diff options
3904 files changed, 501128 insertions, 39 deletions
diff --git a/.envrc b/.envrc new file mode 100644 index 000000000000..162f060fc4ab --- /dev/null +++ b/.envrc @@ -0,0 +1,5 @@ +# Configure the local PATH to contain tools which are fetched ad-hoc +# from Nix. + +export PATH="${PWD}/bin:${PATH}" +export REPO_ROOT="${PWD}" diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000000..572b385fefbf --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,23 @@ +# This file contains commits that should be ignored when calculating +# git blame layers. +# +# You should add it to your local configuration using: +# git config blame.ignoreRevsFile .git-blame-ignore-revs + +# style(3p/nix): Reformat project in Google C++ style +0f2cf531f705d370321843e5ba9135b2ebdb5d19 + +# style(3p/nix): Reformat all includes to match new style +c758de9d22506eb279c5abe61f621e5c8f61af95 + +# style(3p/nix): Reformat all includes to match new style +c758de9d22506eb279c5abe61f621e5c8f61af95 + +# style(3p/nix): Add braces around single-line conditionals +867055133d3f487e52dd44149f76347c2c28bf10 + +# style(3p/nix): Add braces around single-line for-loops +1841d93ccbe5792a17f5b9a22e65ec898c7c2668 + +# style(3p/nix): Final act in the brace-wrapping saga +39087321811e81e26a1a47d6967df1088dcf0e95 diff --git a/.gitignore b/.gitignore index 9e4b0ee5b48e..0b135e70341b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ -*.elc -*-pkg.el -*-autoloads.el +# Ignore the garbage folder, in which I slowly assemble a bunch of +# trash locally that might be valuable in the future. +garbage/ + +# Ignore Nix result symlinks +result +result-* diff --git a/.hgignore b/.hgignore new file mode 100644 index 000000000000..6b8710a711f3 --- /dev/null +++ b/.hgignore @@ -0,0 +1 @@ +.git diff --git a/.mailmap b/.mailmap new file mode 100644 index 000000000000..06e6c9a44997 --- /dev/null +++ b/.mailmap @@ -0,0 +1 @@ +Alyssa Ross <hi@alyssa.is> diff --git a/.nixery/README.md b/.nixery/README.md new file mode 100644 index 000000000000..176209130023 --- /dev/null +++ b/.nixery/README.md @@ -0,0 +1,13 @@ +Nixery set +========== + +This folder exports a special import of the depot Nix structure that is +compatible with Nixery, by extending nixpkgs with a `tvl` attribute containing +the depot. + +This is required because Nixery expects its package set to look like nixpkgs at +the top-level. + +In the future we might want to patch Nixery to not require this (e.g. make it +possible to pass `third_party.nixpkgs` as a key at which to find the nixpkgs +structure). diff --git a/.nixery/default.nix b/.nixery/default.nix new file mode 100644 index 000000000000..19da286ee345 --- /dev/null +++ b/.nixery/default.nix @@ -0,0 +1,6 @@ +# See README.md +{ depot ? import ../. {}, ... }: + +depot.third_party.nixpkgs.extend(_: _: { + tvl = depot; +}) diff --git a/.rgignore b/.rgignore new file mode 100644 index 000000000000..6c19dfbda4bc --- /dev/null +++ b/.rgignore @@ -0,0 +1 @@ +third_party/git diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 000000000000..6ad0d5341a4b --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1,22 @@ +# This configuration file changes some defaults from the ones +# documented on https://rust-lang.github.io/rustfmt/ +# +# All other settings are left at the defaults. + +edition = "2018" +newline_style = "Unix" +use_try_shorthand = true + +# Unstable settings that we want in the future, once they are +# available: +# +# combine_control_expr = false +# comment_width = 100 +# condense_wildcard_suffixes = true +# format_code_in_doc_comments = true +# inline_attribute_width = 100 +# match_block_trailing_comma = true +# merge_imports = true +# normalize_comments = true +# overflow_delimited_expr = true +# wrap_comments = true diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..bdc72a2e03f2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2019 Vincent Ambo +Copyright (c) 2020-2021 The TVL Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/OWNERS b/OWNERS new file mode 100644 index 000000000000..6c47725d4f19 --- /dev/null +++ b/OWNERS @@ -0,0 +1,4 @@ +inherited: false +owners: + - tazjin + - lukegb diff --git a/README.md b/README.md index 6d7e0dd1ff17..4049e2272d26 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,121 @@ -# Emacs X Window Manager - -EXWM (Emacs X Window Manager) is a full-featured tiling X window manager -for Emacs built on top of [XELB](https://github.com/ch11ng/xelb). -It features: -+ Fully keyboard-driven operations -+ Hybrid layout modes (tiling & stacking) -+ Dynamic workspace support -+ ICCCM/EWMH compliance -+ (Optional) RandR (multi-monitor) support -+ (Optional) Builtin system tray -+ (Optional) Builtin input method - -Please check out the -[screenshots](https://github.com/ch11ng/exwm/wiki/Screenshots) -to get an overview of what EXWM is capable of, -and the [user guide](https://github.com/ch11ng/exwm/wiki) -for a detailed explanation of its usage. - -**Note**: If you install EXWM from source, it's recommended to install -XELB also from source (otherwise install both from GNU ELPA). +depot +===== + +[](https://buildkite.com/tvl/depot) + +This repository is the [monorepo][] for the community around [The +Virus Lounge][tvl], containing our personal tools and infrastructure. +Everything in here is built using [Nix][]. + +A large portion of the software here is very self-referential, meaning that it +exists to sustain the operation of the repository. This is the case because we +partially see this as [an experiment][] in tooling for monorepos. + +# Highlights + +## Services + +* Source code is available primarily through Sourcegraph on + [cs.tvl.fyi](https://cs.tvl.fyi), where it is searchable and even semantically + indexed. A lower-tech view of the repository is also available via cgit on + [code.tvl.fyi](https://code.tvl.fyi). + + The repository can be cloned using `git` from `https://cl.tvl.fyi/depot`. + +* All code in the depot, with the exception of code that is checked in to + individual `//users` folders, needs to be reviewed. We use Gerrit on + [cl.tvl.fyi](https://cl.tvl.fyi) for this. + +* Issues are tracked via our own issue tracker on + [b.tvl.fyi](https://b.tvl.fyi). Its source code lives at + [`//web/panettone/`][panettone]. + +* Smaller todo-list entries which do not warrant a separate issue are listed at + [todo.tvl.fyi](https://todo.tvl.fyi). + +* We use Buildkite for CI. Recent builds are listed on + [tvl.fyi/builds](https://tvl.fyi/builds) and pipelines are configured + dynamically via + [`//ops/pipelines`](https://cs.tvl.fyi/depot/-/tree/ops/pipelines). + +* A search service that makes TVL services available via textual + shortcuts is available: [atward](https://at.tvl.fyi) + +All services that we host are deployed on NixOS machines that we manage. Their +configuration is tracked in `//ops/{modules,machines}`. + +## Nix + +* [`//nix/readTree`](https://cs.tvl.fyi/depot/-/blob/nix/readTree/README.md) + contains the Nix code which automatically registers projects in our Nix + attribute hierarchy based on their in-tree location +* `//nix/yants` contains **Y**et **A**nother **N**ix **T**ype **S**ystem, which + we use for a variety of things throughout the repository +* `//nix/buildGo` implements a Nix library that can build Go software in the + style of Bazel's `rules_go`. Go programs in this repository are built using + this library. +* `//nix/buildLisp` implements a Nix library that can build Common Lisp + software. Currently only SBCL is supported. Lisp programs in this repository + are built using this library. +* `//web/bubblegum` contains a CGI-based web framework written in Nix. +* `//nix/nint`: A shebang-compatible interpreter wrapper for Nix. +* `//tvix` contains initial work towards a modular architecture for Nix. +* `//third_party/nix` contains [our fork][tvix] of the Nix package manager. + +We have a variety of other tools and libraries in the `//nix` folder which may +be of interest. + +## Packages / Libraries + +* `//net/alcoholic_jwt` contains an easy-to-use JWT-validation library for Rust +* `//net/crimp` contains a high-level HTTP client using cURL for Rust +* `//tools/emacs-pkgs` contains various useful Emacs libraries, for example: + * `dottime.el` provides [dottime][] in the Emacs modeline + * `nix-util.el` provides editing utilities for Nix files + * `term-switcher.el` is an ivy-function for switching between vterm buffers + * `tvl.el` provides helper functions for interacting with the TVL monorepo +* `//lisp/klatre` provides a grab-bag utility library for Common Lisp + +## User packages + +Contributors to the repository have user directories under +[`//users`](https://cs.tvl.fyi/depot@canon/-/tree/users), which can be used for +personal or experimental code that does not require review. + +Some examples: + +* `//users/tazjin/homepage` && `//users/tazjin/blog`: A Nix-based static site + generator which generates the web page and Atom feed for + [tazj.in](https://tazj.in) +* `//users/grfn/xanthous`: A (WIP) TUI RPG, written in Haskell. +* `//users/tazjin/emacs`: tazjin's Emacs & EXWM configuration +* `//users/tazjin/finito`: A persistent finite-state machine library for Rust. + +# Licensing + +Unless otherwise stated in a subdirectory, all code is licensed under the MIT +license. See [LICENSE](./LICENSE) for details. + +# Contributing + +If you'd like to contribute to any of the tools in here, please check out the +[contribution guidelines](./docs/CONTRIBUTING.md) and our [code of +conduct](./docs/CODE_OF_CONDUCT.md). + +IRC users can find us in [`#tvl`][tvl-irc] on [hackint][], which is also +reachable [via XMPP][hackint-xmpp] at [`#tvl@irc.hackint.org`][tvl-xmpp] (sic!). + +Hackint also provide a [web chat][tvl-webchat]. + +[monorepo]: https://en.wikipedia.org/wiki/Monorepo +[tvl]: https://tvl.fyi +[Nix]: https://nixos.org/nix +[an experiment]: https://tvl.fyi/monorepo-doc +[panettone]: https://cs.tvl.fyi/depot@canon/-/tree/web/panettone +[tvix]: https://cs.tvl.fyi/depot/-/blob/third_party/nix/README.md +[dottime]: https://dotti.me +[tvl-irc]: ircs://irc.hackint.org:6697/#tvl +[hackint]: https://hackint.org/ +[hackint-xmpp]: https://hackint.org/transport/xmpp +[tvl-xmpp]: xmpp:#tvl@irc.hackint.org?join +[tvl-webchat]: https://webirc.hackint.org/#ircs://irc.hackint.org/#tvl diff --git a/RULES b/RULES new file mode 100644 index 000000000000..c6089c4bfd79 --- /dev/null +++ b/RULES @@ -0,0 +1 @@ +1. Only those who pay the Golden Price may bear The Wheel (for the most part). diff --git a/bin/__dispatch.sh b/bin/__dispatch.sh new file mode 100755 index 000000000000..0a049c1ecb20 --- /dev/null +++ b/bin/__dispatch.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# This script dispatches invocations transparently to programs instantiated from +# Nix. +# +# To add a new tool, insert it into the case statement below by setting `attr` +# to the key in nixpkgs which represents the program you want to run. +set -ueo pipefail + +readonly REPO_ROOT=$(dirname "$0")/.. +TARGET_TOOL=$(basename "$0") + +case "${TARGET_TOOL}" in + age) + attr="third_party.nixpkgs-age" + ;; + age-keygen) + attr="third_party.nixpkgs.age" + ;; + depot-build) + attr="tools.depot-build" + ;; + depot-nixpkgs-update) + attr="tools.depot-nixpkgs-update" + ;; + gerrit) + attr="tools.gerrit-cli" + ;; + gerrit-update) + attr="tools.gerrit-update" + ;; + hash-password) + attr="tools.hash-password" + ;; + kontemplate) + attr="ops.kontemplate" + ;; + meson) + attr="third_party.nixpkgs.meson" + ;; + ninja) + attr="third_party.nixpkgs.ninja" + ;; + nint) + attr="nix.nint" + ;; + perf-flamegraph) + attr="tools.perf-flamegraph" + ;; + rebuild-system) + attr="ops.nixos.rebuild-system" + ;; + rink) + attr="third_party.nixpkgs.rink" + ;; + stern) + attr="third_party.nixpkgs.stern" + ;; + *) + echo "The tool '${TARGET_TOOL}' is currently not installed in this repository." + exit 1 + ;; +esac + +result=$(nix-build --no-out-link --attr "${attr}" "${REPO_ROOT}") +PATH="${result}/bin:$PATH" + +exec "${TARGET_TOOL}" "${@}" diff --git a/bin/age b/bin/age new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/age @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/age-keygen b/bin/age-keygen new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/age-keygen @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/depot-build b/bin/depot-build new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/depot-build @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/depot-nixpkgs-update b/bin/depot-nixpkgs-update new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/depot-nixpkgs-update @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/gerrit b/bin/gerrit new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/gerrit @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/gerrit-update b/bin/gerrit-update new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/gerrit-update @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/hash-password b/bin/hash-password new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/hash-password @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/kontemplate b/bin/kontemplate new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/kontemplate @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/meson b/bin/meson new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/meson @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/ninja b/bin/ninja new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/ninja @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/nint b/bin/nint new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/nint @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/perf-flamegraph b/bin/perf-flamegraph new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/perf-flamegraph @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/rebuild-system b/bin/rebuild-system new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/rebuild-system @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/rink b/bin/rink new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/rink @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/bin/stern b/bin/stern new file mode 120000 index 000000000000..8390ec9c9652 --- /dev/null +++ b/bin/stern @@ -0,0 +1 @@ +__dispatch.sh \ No newline at end of file diff --git a/buf.yaml b/buf.yaml new file mode 100644 index 000000000000..42c769f2e4fd --- /dev/null +++ b/buf.yaml @@ -0,0 +1,13 @@ +build: + roots: + #- proto + - third_party +lint: + ignore: + - nix/buildGo + use: + - BASIC + - FILE_LOWER_SNAKE_CASE + except: + - ENUM_VALUE_UPPER_SNAKE_CASE + - PACKAGE_DIRECTORY_MATCH diff --git a/corp/LICENSE b/corp/LICENSE new file mode 100644 index 000000000000..f29fc8403880 --- /dev/null +++ b/corp/LICENSE @@ -0,0 +1,4 @@ +Copyright 2021 ООО ТВЛ + +Code under this folder may be redistributed as part of the TVL depot +repository. All other usage rights for this code are reserved. diff --git a/corp/OWNERS b/corp/OWNERS new file mode 100644 index 000000000000..4bc08d35f67f --- /dev/null +++ b/corp/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - tvl-employees diff --git a/corp/website/content.md b/corp/website/content.md new file mode 100644 index 000000000000..f7ca9786fbea --- /dev/null +++ b/corp/website/content.md @@ -0,0 +1,26 @@ +The Virus Lounge +================ + +---------------- + +<img class="tvl-logo" src="https://static.tvl.su/latest/logo-animated.svg" + alt="Virus with lambda-shaped spike proteins sitting on an armchair"> + +Welcome to the corporate face of [The Virus Lounge][tvl-fyi]. + +We provide technology consulting around a variety of topics, for +example: + +* Advice and setup of organisation-wide monorepos for effective + developer workflows, including associated tooling like CI/CD +* Assistance with anything related to Nix/NixOS +* Software development in various languages (Rust, Common Lisp, + Erlang, Java and more) + +We might be able to help you with other things on request. + +Note: We are still in the process of getting started and have limited +capacity at the moment. If you would like our help, please reach out +at **contact {at} tvl.su** for a discussion. + +[tvl-fyi]: https://tvl.fyi diff --git a/corp/website/default.nix b/corp/website/default.nix new file mode 100644 index 000000000000..8740041d9be7 --- /dev/null +++ b/corp/website/default.nix @@ -0,0 +1,36 @@ +{ depot, pkgs, ... }: + + +let + # https://developers.google.com/search/docs/advanced/structured-data/logo + structuredData = { + "@context" = "https://schema.org"; + "@type" = "Organisation"; + url = "https://tvl.su"; + logo = "https://static.tvl.fyi/${depot.web.static.drvHash}/logo-animated.svg"; + }; + index = depot.web.tvl.template { + title = "TVL (The Virus Lounge) - Software consulting"; + content = builtins.readFile ./content.md; + extraFooter = "\n|\n © ООО ТВЛ"; + + # TODO(tazjin): The `.tvl-logo` thing can probably go in the shared CSS. + extraHead = '' + <meta name="description" content="TVL provides technology consulting for monorepos, Nix, and other SRE/DevOps/Software Engineering topics."> + <script type="application/ld+json"> + ${builtins.toJSON structuredData} + </script> + <style> + .tvl-logo { + width: 60%; + display: block; + margin-left: auto; + margin-right: auto; + } + </style> + ''; + }; +in pkgs.runCommandNoCC "corp-website" {} '' + mkdir $out + cp ${index} $out/index.html +'' diff --git a/default.nix b/default.nix new file mode 100644 index 000000000000..7ccd7413dbd9 --- /dev/null +++ b/default.nix @@ -0,0 +1,111 @@ +# This file sets up the top-level package set by traversing the package tree +# (see //nix/readTree for details) and constructing a matching attribute set +# tree. + +{ nixpkgsBisectPath ? null, ... }@args: + +let + inherit (builtins) + filter + ; + + readTree = import ./nix/readTree {}; + + # Disallow access to //users from other depot parts. + usersFilter = readTree.restrictFolder { + folder = "users"; + reason = '' + Code under //users is not considered stable or dependable in the + wider depot context. If a project under //users is required by + something else, please move it to a different depot path. + ''; + + exceptions = [ + # whitby is allowed to access //users for several reasons: + # + # 1. User SSH keys are set in //users. + # 2. Some personal websites or demo projects are served from it. + [ "ops" "machines" "whitby" ] + + # Due to evaluation order this also affects these targets. + # TODO(tazjin): Can this one be removed somehow? + [ "ops" "nixos" ] + [ "ops" "machines" "all-systems" ] + ]; + }; + + # Disallow access to //corp from other depot parts. + corpFilter = readTree.restrictFolder { + folder = "corp"; + reason = '' + Code under //corp may use incompatible licensing terms with + other depot parts and should not be used anywhere else. + ''; + + exceptions = [ + # For the same reason as above, whitby is exempt to serve the + # corp website. + [ "ops" "machines" "whitby" ] + [ "ops" "nixos" ] + [ "ops" "machines" "all-systems" ] + ]; + }; + + readDepot = depotArgs: readTree { + args = depotArgs; + path = ./.; + filter = parts: args: corpFilter parts (usersFilter parts args); + scopedArgs = { + __findFile = _: _: throw "Do not import from NIX_PATH in the depot!"; + }; + }; + + # To determine build targets, we walk through the depot tree and + # fetch attributes that were imported by readTree and are buildable. + # + # Any build target that contains `meta.ci = false` will be skipped. + + # Is this tree node eligible for build inclusion? + eligible = node: (node ? outPath) && (node.meta.ci or true); + +in readTree.fix(self: (readDepot { + depot = self; + + # Pass third_party as 'pkgs' (for compatibility with external + # imports for certain subdirectories) + pkgs = self.third_party.nixpkgs; + + # Expose lib attribute to packages. + lib = self.third_party.nixpkgs.lib; + + # Pass arguments passed to the entire depot through, for packages + # that would like to add functionality based on this. + # + # Note that it is intended for exceptional circumstance, such as + # debugging by bisecting nixpkgs. + externalArgs = args; +}) // { + # Make the path to the depot available for things that might need it + # (e.g. NixOS module inclusions) + path = self.third_party.nixpkgs.lib.cleanSource ./.; + + # List of all buildable targets, for CI purposes. + # + # Note: To prevent infinite recursion, this *must* be a nested + # attribute set (which does not have a __readTree attribute). + ci.targets = readTree.gather eligible (self // { + # remove the pipelines themselves from the set over which to + # generate pipelines because that also leads to infinite + # recursion. + ops = self.ops // { pipelines = null; }; + + # remove nixpkgs from the set, for obvious reasons. + third_party = self.third_party // { nixpkgs = null; }; + }); + + # Derivation that gcroots all depot targets. + ci.gcroot = self.third_party.nixpkgs.symlinkJoin { + name = "depot-gcroot"; + paths = self.ci.targets; + }; +}) diff --git a/docs/CODE_OF_CONDUCT.md b/docs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..0e46bbedb0cd --- /dev/null +++ b/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,29 @@ +A SERMON ON ETHICS AND LOVE +=========================== + +One day Mal-2 asked the messenger spirit Saint Gulik to approach the +Goddess and request Her presence for some desperate advice. Shortly +afterwards the radio came on by itself, and an ethereal female Voice +said **YES?** + +"O! Eris! Blessed Mother of Man! Queen of Chaos! Daughter of Discord! +Concubine of Confusion! O! Exquisite Lady, I beseech You to lift a +heavy burden from my heart!" + +**WHAT BOTHERS YOU, MAL? YOU DON'T SOUND WELL.** + +"I am filled with fear and tormented with terrible visions of pain. +Everywhere people are hurting one another, the planet is rampant with +injustices, whole societies plunder groups of their own people, +mothers imprison sons, children perish while brothers war. O, woe." + +**WHAT IS THE MATTER WITH THAT, IF IT IS WHAT YOU WANT TO DO?** + +"But nobody Wants it! Everybody hates it." + +**OH. WELL, THEN *STOP*.** + +At which moment She turned herself into an aspirin commercial and left +The Polyfather stranded alone with his species. + +SINISTER DEXTER HAS A BROKEN SPIROMETER. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 000000000000..dd138d3bfcdb --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,120 @@ +Contribution Guidelines +======================= + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Contribution Guidelines](#contribution-guidelines) + - [Before making a change](#before-making-a-change) + - [Commit messages](#commit-messages) + - [Commit content](#commit-content) + - [Code quality](#code-quality) + - [Builds & tests](#builds--tests) + - [Submitting changes](#submitting-changes) + +<!-- markdown-toc end --> + +This is a loose set of "guidelines" for contributing to the depot. Please note +that we will not accept any patches that don't follow these guidelines. + +Also consider the [code of conduct](./CODE_OF_CONDUCT.md). No really, +you should. + +## Before making a change + +Before making a change, consider your motivation for making the change. +Documentation updates, bug fixes and the like are *always* welcome. + +When adding a feature you should consider whether it is only useful for your +particular use-case or whether it is generally applicable for other users of the +project. + +When in doubt - just ask! You can reach out to us at +[depot@tazj.in](mailto:depot@tazj.in) or on Twitter, IRC, etc. + +## Commit messages + +All commit messages should be structured like this: + +``` +type(scope): Subject line with at most a 68 character length + +Body of the commit message with an empty line between subject and +body. This text should explain what the change does and why it has +been made, *especially* if it introduces a new feature. + +Relevant issues should be mentioned if they exist. +``` + +Where `type` can be one of: + +* `feat`: A new feature has been introduced +* `fix`: An issue of some kind has been fixed +* `docs`: Documentation or comments have been updated +* `style`: Formatting changes only +* `refactor`: Hopefully self-explanatory! +* `test`: Added missing tests / fixed tests +* `chore`: Maintenance work +* `subtree`: Operations involving `git subtree` + +And `scope` should refer to some kind of logical grouping inside of the project. + +It does not make sense to include the full path unless it aids in +disambiguating. For example, when changing the configuration of the host +`whitby` at `//ops/machines/whitby` it is enough to write `feat(whitby): ...`. + +Please take a look at the existing commit log for examples. + +## Commit content + +Multiple changes should be divided into multiple git commits whenever possible. +Common sense applies. + +The fix for a single-line whitespace issue is fine to include in a different +commit. Introducing a new feature and refactoring (unrelated) code in the same +commit is not fine. + +`git commit -a` is generally **taboo**. + +In my experience making "sane" commits becomes *significantly* easier as +developer tooling is improved. The interface to `git` that I recommend is +[magit][]. Even if you are not yet an Emacs user, it makes sense to install +Emacs just to be able to use magit - it is really that good. + +For staging sane chunks on the command line with only git, consider `git add +-p`. + +## Code quality + +This one should go without saying - but please ensure that your code quality +does not fall below the rest of the project. This is of course very subjective, +but as an example if you place code that throws away errors into a block in +which errors are handled properly your change will be rejected. + +In my experience there is a strong correlation between the visual appearance of +a code block and its quality. This is a simple way to sanity-check your work +while squinting and keeping some distance from your screen ;-) + +## Builds & tests + +All projects are built using [Nix][] to avoid "build pollution" via the user's +environment. + +If you have Nix installed and are contributing to a project tracked in this +repository, you can usually build the project by calling `nix-build -A +path.to.project`. + +For example, to build a project located at `//tools/foo` you would call +`nix-build -A tools.foo` + +If the project has tests, check that they still work before submitting your +change. + +## Submitting changes + +The code review & change submission process is described in the [code +review][] documentation. + +[magit]: https://magit.vc/ +[Nix]: https://nixos.org/nix/ +[code review]: ./REVIEWS.md diff --git a/docs/REVIEWS.md b/docs/REVIEWS.md new file mode 100644 index 000000000000..e1c657b33344 --- /dev/null +++ b/docs/REVIEWS.md @@ -0,0 +1,154 @@ +TVL Code Reviews +================ + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [TVL Code Reviews](#tvl-code-reviews) + - [Gerrit setup](#gerrit-setup) + - [Gerrit workflows](#gerrit-workflows) + - [Review process & approvals](#review-process--approvals) + - [Registration](#registration) + - [Submitting changes via email](#submitting-changes-via-email) + +<!-- markdown-toc end --> + + +This document describes the TVL code review process & tooling. If you are +looking for general contribution guidelines, please look at the [general +contribution guidelines](./CONTRIBUTING.md). + +All changes are tracked at [cl.tvl.fyi](https://cl.tvl.fyi) using Gerrit. See +[Registration](#registration) for information on how to register an account. + +## Gerrit setup + +Gerrit uses the concept of change IDs to track commits across rebases and other +operations that might change their hashes, and link them to unique changes in +Gerrit. + +First, [tell Gerrit][Gerrit SSH] about your SSH keys. + +Then, to make using Gerrit smooth for users, the repository should be cloned and +a commit hook should be installed as follows: + +``` +git clone "ssh://$USER@code.tvl.fyi:29418/depot" +scp -p -P 29418 $USER@code.tvl.fyi:hooks/commit-msg "depot/.git/hooks/" +``` + +If you have a previous clone of the depot via HTTP you can use `git remote +set-url` to update the origin URL and install the hook in the same way as above. + +## Gerrit workflows + +The developer workflow on Gerrit is quite different from what GitHub-users are +used to. + +The depot does not have branches (other than Gerrit's internal metadata refs) +and all development happens at `HEAD`. + +Every time you create a new commit the change hook will insert a unique +`Change-Id` tag into the commit message. Once you are satisfied with the state +of your commit and want to submit it for review, you push it to a git ref called +`refs/for/canon`. This designates the commits as changelists (CLs) targeted for +the `canon` branch. + +Sending a change for review is done by pushing to a special target. You can set +this to be the default push target through your git configuration: + +``` +git config remote.origin.url "ssh://$USER@code.tvl.fyi:29418/depot" +git config remote.origin.push HEAD:refs/for/canon +``` + +Then, after making your change, push to the default, or to a special target: + +``` +Example: +git commit -m 'docs(REVIEWS): Fixed all the errors in the reviews docs' +git push origin + +# Uploading a work-in-progress CL: +git push origin HEAD:refs/for/canon%wip +``` + +TIP: Every individual commit will become a separate change. We do not merge +related commits, but instead submit them one by one. Be aware that if you are +expecting a different behaviour and attempt something like an unsquashed subtree +merge, you will produce a *lot* of CLs. This is strongly discouraged. + +During your review, the reviewer(s) might ask you to make changes. You can +simply amend your commit(s) and push to the same ref. Gerrit will automatically +update your changes. + +Read more about the Gerrit workflow in the [Gerrit walkthrough][]. + +## Review process & approvals + +Each user has the ability to create their own users directory in +`//users/<username>` in which they can submit code without review from other +contributors (they will still need to +2 their own changes, and the initial +check-in of the `OWNERS` file needs to be reviewed). + +You can set a directory like this up for yourself by proposing a change similar +to [CL/246](https://cl.tvl.fyi/c/depot/+/246). + +For all paths outside of `//users`, code review is required. We have no strict +guidelines about the review process itself, as we're not a megacorp, but we have +formalised checks before submitting: + +* At least one person who is [an owner][OWNERS] of the codepath must have given + a +2 review +* The commit message must conform to our [guidelines][] +* No code review comments must be left unresolved + +If all these conditions are fulfilled, the **change author submits their change +themselves**. + +## Registration + +If you would like to have an account on the Gerrit instance, follow these +instructions: + +1. Be a member of `#tvl` on [hackint][]. +2. Clone the depot locally (via `git clone "https://cl.tvl.fyi/depot"`). +3. Create a user entry in our LDAP server in [ops/users][ops-users]. + + We recommend using ARGON2 password hashes, which can be created + with the `slappasswd` tool if OpenLDAP was compiled with ARGON2 + support. + + For convenience, we provide a wrapper script for this that you can + build with `nix-build -A tools.hash-password` in a depot checkout. + Alternatively, if you have `direnv` installed, you can add the + depot to your allowlist and just run `hash-password` which should + be added to your `$PATH` by `direnv`. + + You can probably create ARGON2 hashes with other tools, but that is + your job to figure out. +4. Create a commit adding yourself (see e.g. + [CL/2671](https://cl.tvl.fyi/c/depot/+/2671)) +5. Submit the commit via email (see below). + +## Submitting changes via email + +You can submit a patch via email to `depot@tazj.in` and it will be added to +Gerrit by a contributor. + +Create an appropriate commit locally and send it us using either of these options: + +* `git format-patch`: This will create a `.patch` file which you should email to + us. +* `git send-email`: If configured on your system, this will take care of the + whole emailing process for you. + +The email address is a [public group][]. + +[Gerrit SSH]: https://cl.tvl.fyi/settings/#SSHKeys +[Gerrit walkthrough]: https://gerrit-review.googlesource.com/Documentation/intro-gerrit-walkthrough.html +[OWNERS]: https://cl.tvl.fyi/plugins/owners/Documentation/config.md +[guidelines]: ./CONTRIBUTING.md#commit-messages +[ops-users]: ../ops/users/default.nix +[public group]: https://groups.google.com/a/tazj.in/forum/?hl=en#!forum/depot +[hackint]: https://hackint.org diff --git a/docs/designs/SPARSE_CHECKOUTS.md b/docs/designs/SPARSE_CHECKOUTS.md new file mode 100644 index 000000000000..7bd4963f61ea --- /dev/null +++ b/docs/designs/SPARSE_CHECKOUTS.md @@ -0,0 +1,37 @@ +Below is a prototype for a script to create Git sparse checkouts of the depot. +The script below works today with relatively recent versions of git. + +Open items: + + - Read-increment-write the checkout ID from a file in .git. + - `NICE_CHECKOUT_ROOT` should be a git configuration value. + - `tvl-get-depends` will be a script that contacts the build farm and asks for + the closure of a given source directory, using [depot-scan]. + +```bash +DEPOT_ROOT="${depot.path}" +XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}" +CLIENT_ROOT="$XDG_DATA_HOME/tvlc/clients" +NICE_CHECKOUT_ROOT="$HOME/tvlc" +CHECKOUT_ID=1 +CHECKOUT_NAME=myclient # command line variables + +assertAbsolutePath "$CLIENT_ROOT" + +mkdir "$CLIENT_ROOT"/"$CHECKOUT_ID" +ln -s "$CLIENT_ROOT"/"$CHECKOUT_ID" "$NICE_CHECKOUT_ROOT"/"$CHECKOUT_NAME" + +cd "$DEPOT_ROOT" +git worktree add --no-checkout -b "tvlc-$CHECKOUT_ID" "$CLIENT_ROOT/$CHECKOUT_ID/" canon +# BUG: git not creating the /info/ subdir +mkdir "$DEPOT_ROOT"/.git/worktrees/"$CHECKOUT_ID"/info + +cd "$CLIENT_ROOT/$CHECKOUT_ID" +git sparse-checkout init --cone +git sparse-checkout set "$TARGET_DIR" nix/readTree overrides +tvl-get-depends "$TARGET_DIR" | xargs git sparse-checkout add + +cd "$NICE_CHECKOUT_ROOT"/"$CHECKOUT_NAME" +``` + +[depot-scan]: ../users/edef/depot-scan.nix diff --git a/fun/clbot/backoffutil/backoffutil.go b/fun/clbot/backoffutil/backoffutil.go new file mode 100644 index 000000000000..1b1ea5f9d012 --- /dev/null +++ b/fun/clbot/backoffutil/backoffutil.go @@ -0,0 +1,43 @@ +// Package backoffutil provides useful utilities for backoff. +package backoffutil + +import ( + "time" + + backoff "github.com/cenkalti/backoff/v4" +) + +// ZeroStartingBackOff is a backoff.BackOff that returns "0" as the first Duration after a reset. +// This is useful for constructing loops and just enforcing a backoff duration on every loop, rather than incorporating this logic into the loop directly. +type ZeroStartingBackOff struct { + bo backoff.BackOff + initial bool +} + +// NewZeroStartingBackOff creates a new ZeroStartingBackOff. +func NewZeroStartingBackOff(bo backoff.BackOff) *ZeroStartingBackOff { + return &ZeroStartingBackOff{bo: bo, initial: true} +} + +// NewDefaultBackOff creates a sensibly configured BackOff that starts at zero. +func NewDefaultBackOff() backoff.BackOff { + ebo := backoff.NewExponentialBackOff() + ebo.MaxElapsedTime = 0 + return NewZeroStartingBackOff(ebo) +} + +// NextBackOff returns the next back off duration to use. +// For the first call after a call to Reset(), this is 0. For each subsequent duration, the underlying BackOff is consulted. +func (bo *ZeroStartingBackOff) NextBackOff() time.Duration { + if bo.initial == true { + bo.initial = false + return 0 + } + return bo.bo.NextBackOff() +} + +// Reset resets to the initial state, and also passes a Reset through to the underlying BackOff. +func (bo *ZeroStartingBackOff) Reset() { + bo.initial = true + bo.bo.Reset() +} diff --git a/fun/clbot/backoffutil/default.nix b/fun/clbot/backoffutil/default.nix new file mode 100644 index 000000000000..78585da236e3 --- /dev/null +++ b/fun/clbot/backoffutil/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +let + inherit (depot.third_party) gopkgs; +in +depot.nix.buildGo.package { + name = "code.tvl.fyi/fun/clbot/backoffutil"; + srcs = [ + ./backoffutil.go + ]; + deps = [ + gopkgs."github.com".cenkalti.backoff.gopkg + ]; +} diff --git a/fun/clbot/clbot.go b/fun/clbot/clbot.go new file mode 100644 index 000000000000..a97d28e0f73d --- /dev/null +++ b/fun/clbot/clbot.go @@ -0,0 +1,274 @@ +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "io/ioutil" + "net" + "os" + "os/signal" + "strings" + "time" + + "code.tvl.fyi/fun/clbot/backoffutil" + "code.tvl.fyi/fun/clbot/gerrit" + "code.tvl.fyi/fun/clbot/gerrit/gerritevents" + log "github.com/golang/glog" + "golang.org/x/crypto/ssh" + "gopkg.in/irc.v3" +) + +var ( + gerritAddr = flag.String("gerrit_host", "cl.tvl.fyi:29418", "Gerrit SSH host:port") + gerritSSHHostKey = flag.String("gerrit_ssh_pubkey", "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIUNYBYPCCBNDFSd0BuCR+8kgeuJ7IA5S2nTNQmkQUYNyXK+ot5os7rHtCk96+grd5+J8jFCuFBWisUe8h8NC0Q=", "Gerrit SSH public key") + gerritSSHTimeout = flag.Duration("gerrit_tcp_timeout", 5*time.Second, "Gerrit SSH TCP connect timeout") + + gerritAuthUsername = flag.String("gerrit_ssh_auth_username", "", "Gerrit SSH username") + gerritAuthKeyPath = flag.String("gerrit_ssh_auth_key", "", "Gerrit SSH private key path") + + ircServer = flag.String("irc_server", "irc.hackint.org:6697", "IRC server to connect to") + ircTls = flag.Bool("irc_tls", false, "Does the server connection need TLS?") + ircNick = flag.String("irc_nick", "clbot", "Nick to use when connecting to IRC") + ircUser = flag.String("irc_user", "clbot", "User string to use for IRC") + ircName = flag.String("irc_name", "clbot", "Name string to use for IRC") + ircChannel = flag.String("irc_channel", "#tvl", "Channel to send messages to") + ircPassword = flag.String("irc_pass", "", "Password to use for IRC") + ircSendLimit = flag.Duration("irc_send_limit", 100*time.Millisecond, "Delay between messages") + ircSendBurst = flag.Int("irc_send_burst", 10, "Number of messages which can be sent in a burst") + + notifyRepo = flag.String("notify_repo", "depot", "Repo name to notify about") + notifyBranches = stringSetFlag{} +) + +func init() { + flag.Var(¬ifyBranches, "notify_branches", "Branch names (comma-separated, or repeated flags, or both) to notify users about") +} + +type stringSetFlag map[string]bool + +func (f stringSetFlag) String() string { + return fmt.Sprintf("%q", map[string]bool(f)) +} +func (f stringSetFlag) Set(s string) error { + if s == "" { + return nil + } + for _, k := range strings.Split(s, ",") { + if k != "" { + f[k] = true + } + } + return nil +} + +func mustFixedHostKey(f string) ssh.HostKeyCallback { + pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(f)) + if err != nil { + log.Exitf("ParseAuthorizedKey(%q): %v", f, err) + } + return ssh.FixedHostKey(pk) +} + +func mustPrivateKey(p string) ssh.AuthMethod { + pkBytes, err := ioutil.ReadFile(p) + if err != nil { + log.Exitf("reading SSH private key from %q: %v", p, err) + } + pk, err := ssh.ParsePrivateKey(pkBytes) + if err != nil { + log.Exitf("parsing private key from %q: %v", p, err) + } + return ssh.PublicKeys(pk) +} + +var shutdownFuncs []func() + +func callOnShutdown(f func()) { + shutdownFuncs = append(shutdownFuncs, f) +} + +// Unicode U+200B zero-width-space, to avoid triggering other bots +// or highlighting people on IRC. +const zeroWidthSpace = "\u200b" + +func runIRC(ctx context.Context, ircCfg irc.ClientConfig, sendMsg <-chan string) { + bo := backoffutil.NewDefaultBackOff() + ircCfg.Handler = irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + if m.Command == "NOTICE" && m.Prefix.Name == "NickServ" && strings.Contains(m.Trailing(), "dentified") { + // We're probably identified now, go join the channel. + c.Writef("JOIN %s", *ircChannel) + } + }) + for { + timer := time.NewTimer(bo.NextBackOff()) + select { + case <-ctx.Done(): + timer.Stop() + return + case <-timer.C: + break + } + + (func() { + connectedStart := time.Now() + + var ircConn net.Conn + var err error + + if *ircTls { + ircConn, err = tls.Dial("tcp", *ircServer, nil) + } else { + ircConn, err = net.Dial("tcp", *ircServer) + } + + if err != nil { + log.Errorf("connecting to IRC at tcp/%s (tls: %v): %v", *ircServer, *ircTls, err) + return + } + + ircClient := irc.NewClient(ircConn, ircCfg) + ircClientCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + for { + select { + case <-ircClientCtx.Done(): + return + case msg := <-sendMsg: + log.Infof("sending message %q to %v", msg, *ircChannel) + ircClient.Writef("PRIVMSG %s :%s%s", *ircChannel, zeroWidthSpace, msg) + } + } + }() + log.Infof("connecting to IRC on tcp/%s", *ircServer) + if err := ircClient.RunContext(ircClientCtx); err != nil { + connectedEnd := time.Now() + connectedFor := connectedEnd.Sub(connectedStart) + if connectedFor > 60*time.Second { + bo.Reset() + } + log.Errorf("IRC RunContext: %v", err) + return + } + })() + } +} + +func username(p gerritevents.PatchSet) string { + options := []string{ + p.Uploader.Username, + p.Uploader.Name, + p.Uploader.Email, + } + for _, opt := range options { + if opt != "" { + return opt + } + } + return "UNKNOWN USER" +} + +// noping inserts a Unicode zero-width space between the first and rest characters of `user` +// in an effort to avoid pinging that user on IRC. +func noping(user string) string { + un := []rune(user) + return string(un[0:1]) + zeroWidthSpace + string(un[1:]) +} + +// Apply noping to each instance of the username in the supplied +// message. With this users will not be pinged for their own CLs, but +// they will be notified if someone else writes a CL that includes +// their username. +func nopingAll(username, message string) string { + return strings.ReplaceAll(message, username, noping(username)) +} + +func patchSetURL(c gerritevents.Change, p gerritevents.PatchSet) string { + return fmt.Sprintf("https://cl.tvl.fyi/%d", c.Number) +} + +func main() { + flag.Parse() + failed := false + if *gerritAuthUsername == "" { + log.Errorf("gerrit_ssh_auth_username must be set") + failed = true + } + if *gerritAuthKeyPath == "" { + log.Errorf("gerrit_ssh_auth_key must be set") + failed = true + } + if failed { + os.Exit(2) + } + + shutdownCh := make(chan os.Signal) + signal.Notify(shutdownCh, os.Interrupt) + go func() { + <-shutdownCh + signal.Reset(os.Interrupt) + for n := len(shutdownFuncs) - 1; n >= 0; n-- { + shutdownFuncs[n]() + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + callOnShutdown(cancel) + cfg := &ssh.ClientConfig{ + User: *gerritAuthUsername, + Auth: []ssh.AuthMethod{mustPrivateKey(*gerritAuthKeyPath)}, + HostKeyCallback: mustFixedHostKey(*gerritSSHHostKey), + Timeout: *gerritSSHTimeout, + } + cfg.SetDefaults() + + gw, err := gerrit.New(ctx, "tcp", *gerritAddr, cfg) + if err != nil { + log.Exitf("gerrit.New(%q): %v", *gerritAddr, err) + } + callOnShutdown(func() { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + gw.Close(ctx) + }) + + sendMsgChan := make(chan string, 5) + go func() { + for e := range gw.Events() { + var parsedMsg string + switch e := e.(type) { + case *gerritevents.PatchSetCreated: + if e.Change.Project != *notifyRepo || !notifyBranches[e.Change.Branch] || e.PatchSet.Number != 1 { + continue + } + user := username(e.PatchSet) + parsedMsg = nopingAll(user, fmt.Sprintf("CL/%d proposed by %s - %s - %s", e.Change.Number, user, e.Change.Subject, patchSetURL(e.Change, e.PatchSet))) + case *gerritevents.ChangeMerged: + if e.Change.Project != *notifyRepo || !notifyBranches[e.Change.Branch] { + continue + } + user := username(e.PatchSet) + parsedMsg = nopingAll(user, fmt.Sprintf("CL/%d applied by %s - %s - %s", e.Change.Number, user, e.Change.Subject, patchSetURL(e.Change, e.PatchSet))) + } + if parsedMsg != "" { + sendMsgChan <- parsedMsg + } + } + }() + + ircCtx, ircCancel := context.WithCancel(ctx) + callOnShutdown(ircCancel) + go runIRC(ircCtx, irc.ClientConfig{ + Nick: *ircNick, + User: *ircUser, + Name: *ircName, + Pass: *ircPassword, + + SendLimit: *ircSendLimit, + SendBurst: *ircSendBurst, + }, sendMsgChan) + + <-ctx.Done() +} diff --git a/fun/clbot/default.nix b/fun/clbot/default.nix new file mode 100644 index 000000000000..e6b9c2fb9bf9 --- /dev/null +++ b/fun/clbot/default.nix @@ -0,0 +1,19 @@ +{ depot, ... }@args: + +let + clbot = depot.fun.clbot; + gopkgs = depot.third_party.gopkgs; +in +depot.nix.buildGo.program { + name = "clbot"; + srcs = [ + ./clbot.go + ]; + deps = [ + clbot.gerrit + gopkgs."github.com".davecgh.go-spew.spew.gopkg + gopkgs."github.com".golang.glog.gopkg + gopkgs."golang.org".x.crypto.ssh.gopkg + gopkgs."gopkg.in"."irc.v3".gopkg + ]; +} diff --git a/fun/clbot/gerrit/default.nix b/fun/clbot/gerrit/default.nix new file mode 100644 index 000000000000..3b6ce0a7392f --- /dev/null +++ b/fun/clbot/gerrit/default.nix @@ -0,0 +1,18 @@ +{ depot, ... }: + +let + inherit (depot.fun) clbot; + inherit (depot.third_party) gopkgs; +in +depot.nix.buildGo.package { + name = "code.tvl.fyi/fun/clbot/gerrit"; + srcs = [ + ./watcher.go + ]; + deps = [ + clbot.gerrit.gerritevents + clbot.backoffutil + gopkgs."github.com".golang.glog.gopkg + gopkgs."golang.org".x.crypto.ssh.gopkg + ]; +} diff --git a/fun/clbot/gerrit/gerritevents/default.nix b/fun/clbot/gerrit/gerritevents/default.nix new file mode 100644 index 000000000000..024451858bbc --- /dev/null +++ b/fun/clbot/gerrit/gerritevents/default.nix @@ -0,0 +1,10 @@ +{ depot, ... }: + +depot.nix.buildGo.package { + name = "code.tvl.fyi/fun/clbot/gerrit/gerritevents"; + srcs = [ + ./time.go + ./types.go + ./events.go + ]; +} diff --git a/fun/clbot/gerrit/gerritevents/events.go b/fun/clbot/gerrit/gerritevents/events.go new file mode 100644 index 000000000000..c02b30f76ea8 --- /dev/null +++ b/fun/clbot/gerrit/gerritevents/events.go @@ -0,0 +1,321 @@ +package gerritevents + +import ( + "encoding/json" + "fmt" +) + +var events = map[string]func() Event{} + +func registerEvent(e func() Event) { + t := e().EventType() + if _, ok := events[t]; ok { + panic(fmt.Sprintf("%s already registered", t)) + } + events[t] = e +} + +// These events are taken from https://cl.tvl.fyi/Documentation/cmd-stream-events.html. + +// Event is implemented by Gerrit event structs. +type Event interface { + EventType() string +} + +type simpleEvent struct { + Type string `json:"type"` +} + +// Parse parses a Gerrit event from JSON. +func Parse(bs []byte) (Event, error) { + var s simpleEvent + if err := json.Unmarshal(bs, &s); err != nil { + return nil, fmt.Errorf("unmarshalling %q as Gerrit Event: %v", string(bs), err) + } + ef, ok := events[s.Type] + if !ok { + return nil, fmt.Errorf("unknown event type %q", s.Type) + } + e := ef() + if err := json.Unmarshal(bs, e); err != nil { + return nil, fmt.Errorf("unmarshalling %q as Gerrit Event %q: %v", string(bs), e.EventType(), err) + } + return e, nil +} + +// AssigneeChanged indicates that a change's assignee has been changed. +type AssigneeChanged struct { + Type string `json:"type"` + Change Change `json:"change"` + Changer Account `json:"changer"` + OldAssignee Account `json:"oldAssignee"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (AssigneeChanged) EventType() string { return "assignee-changed" } + +func init() { + registerEvent(func() Event { return &AssigneeChanged{} }) +} + +// ChangeAbandoned indicates that a change has been abandoned. +type ChangeAbandoned struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Abandoner Account `json:"abandoner"` + Reason string `json:"reason"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (ChangeAbandoned) EventType() string { return "change-abandoned" } + +func init() { + registerEvent(func() Event { return &ChangeAbandoned{} }) +} + +// ChangeDeleted indicates that a change has been deleted. +type ChangeDeleted struct { + Type string `json:"type"` + Change Change `json:"change"` + Deleter Account `json:"deleter"` +} + +// EventType implements Event. +func (ChangeDeleted) EventType() string { return "change-deleted" } + +func init() { + registerEvent(func() Event { return &ChangeDeleted{} }) +} + +// ChangeMerged indicates that a change has been merged into the target branch. +type ChangeMerged struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Submitter Account `json:"submitter"` + NewRev string `json:"newRev"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (ChangeMerged) EventType() string { return "change-merged" } + +func init() { + registerEvent(func() Event { return &ChangeMerged{} }) +} + +// ChangeRestored indicates a change has been restored (i.e. un-abandoned). +type ChangeRestored struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Restorer Account `json:"restorer"` + Reason string `json:"reason"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (ChangeRestored) EventType() string { return "change-restored" } + +func init() { + registerEvent(func() Event { return &ChangeRestored{} }) +} + +// CommentAdded indicates someone has commented on a patchset. +type CommentAdded struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Author Account `json:"author"` + Approvals []Approval `json:"approvals"` + Comment string `json:"comment"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (CommentAdded) EventType() string { return "comment-added" } + +func init() { + registerEvent(func() Event { return &CommentAdded{} }) +} + +// DroppedOutput indicates that some events may be missing from the stream. +type DroppedOutput struct { + Type string `json:"type"` +} + +// EventType implements Event. +func (DroppedOutput) EventType() string { return "dropped-output" } + +func init() { + registerEvent(func() Event { return &DroppedOutput{} }) +} + +// HashtagsChanged indicates that someone has added or removed hashtags from a change. +type HashtagsChanged struct { + Type string `json:"type"` + Change Change `json:"change"` + Editor Account `json:"editor"` + Added []string `json:"added"` + Removed []string `json:"removed"` + Hashtags []string `json:"hashtags"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (HashtagsChanged) EventType() string { return "hashtags-changed" } + +func init() { + registerEvent(func() Event { return &HashtagsChanged{} }) +} + +// ProjectCreated indicates that a new project has been created. +type ProjectCreated struct { + Type string `json:"type"` + ProjectName string `json:"projectName"` + ProjectHead string `json:"projectHead"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (ProjectCreated) EventType() string { return "project-created" } + +func init() { + registerEvent(func() Event { return &ProjectCreated{} }) +} + +// PatchSetCreated indicates that a new patchset has been added to a change. +type PatchSetCreated struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Uploader Account `json:"uploader"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (PatchSetCreated) EventType() string { return "patchset-created" } + +func init() { + registerEvent(func() Event { return &PatchSetCreated{} }) +} + +// RefUpdated indicates that a ref has been updated. +type RefUpdated struct { + Type string `json:"type"` + Submitter Account `json:"submitter"` + RefUpdate RefUpdate `json:"refUpdate"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (RefUpdated) EventType() string { return "ref-updated" } + +func init() { + registerEvent(func() Event { return &RefUpdated{} }) +} + +// ReviewerAdded indicates that a reviewer has been added to a change. +type ReviewerAdded struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Reviewer Account `json:"reviewer"` + Adder Account `json:"adder"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (ReviewerAdded) EventType() string { return "reviewer-added" } + +func init() { + registerEvent(func() Event { return &ReviewerAdded{} }) +} + +// ReviewerDeleted indicates that a reviewer has been removed from a change, possibly removing one or more approvals. +type ReviewerDeleted struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Reviewer Account `json:"reviewer"` + Remover Account `json:"remover"` + Approvals []Approval `json:"approvals"` + Comment string `json:"comment"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (ReviewerDeleted) EventType() string { return "reviewer-deleted" } + +func init() { + registerEvent(func() Event { return &ReviewerDeleted{} }) +} + +// TopicChanged indicates that the topic attached to a change has been changed. +type TopicChanged struct { + Type string `json:"type"` + Change Change `json:"change"` + Changer Account `json:"changer"` + OldTopic string `json:"oldTopic"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (TopicChanged) EventType() string { return "topic-changed" } + +func init() { + registerEvent(func() Event { return &TopicChanged{} }) +} + +// WIPStateChanged indicates that the work-in-progress state of a change has changed. +type WIPStateChanged struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Changer Account `json:"changer"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (WIPStateChanged) EventType() string { return "wip-state-changed" } + +func init() { + registerEvent(func() Event { return &WIPStateChanged{} }) +} + +// PrivateStateChanged indicates that the private state of a change has changed. +type PrivateStateChanged struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Changer Account `json:"changer"` + EventCreatedOn Time `json:"eventCreatedOn"` +} + +// EventType implements Event. +func (PrivateStateChanged) EventType() string { return "private-state-changed" } + +func init() { + registerEvent(func() Event { return &PrivateStateChanged{} }) +} + +// VoteDeleted indicates that an approval vote has been deleted from a change. +type VoteDeleted struct { + Type string `json:"type"` + Change Change `json:"change"` + PatchSet PatchSet `json:"patchSet"` + Reviewer Account `json:"reviewer"` + Remover Account `json:"remover"` + Approvals []Approval `json:"approvals"` + Comment string `json:"comment"` +} + +// EventType implements Event. +func (VoteDeleted) EventType() string { return "vote-deleted" } + +func init() { + registerEvent(func() Event { return &VoteDeleted{} }) +} diff --git a/fun/clbot/gerrit/gerritevents/time.go b/fun/clbot/gerrit/gerritevents/time.go new file mode 100644 index 000000000000..7fbfaa3f5c47 --- /dev/null +++ b/fun/clbot/gerrit/gerritevents/time.go @@ -0,0 +1,38 @@ +package gerritevents + +import ( + "fmt" + "strconv" + "time" +) + +// Time is a time.Time that is formatted as a Unix timestamp in JSON. +type Time struct { + time.Time +} + +// UnmarshalJSON unmarshals a Unix timestamp into a Time. +func (t *Time) UnmarshalJSON(bs []byte) error { + if string(bs) == "null" { + return nil + } + u, err := strconv.ParseInt(string(bs), 10, 64) + if err != nil { + return err + } + t.Time = time.Unix(u, 0) + return nil +} + +// MarshalJSON marshals a Time into a Unix timestamp. +func (t *Time) MarshalJSON() ([]byte, error) { + if t.IsZero() { + return []byte("null"), nil + } + return []byte(fmt.Sprintf("%d", t.Unix())), nil +} + +// IsSet returns true if the time.Time is non-zero. +func (t *Time) IsSet() bool { + return !t.IsZero() +} diff --git a/fun/clbot/gerrit/gerritevents/types.go b/fun/clbot/gerrit/gerritevents/types.go new file mode 100644 index 000000000000..75987a2da601 --- /dev/null +++ b/fun/clbot/gerrit/gerritevents/types.go @@ -0,0 +1,221 @@ +package gerritevents + +// These types are taken from https://cl.tvl.fyi/Documentation/json.html. + +// Account is a Gerrit account (or just a Git name+email pair). +type Account struct { + Name string `json:"name"` + Email string `json:"email"` + Username string `json:"username"` +} + +// ChangeStatus represents the states a change can be in. +type ChangeStatus string + +const ( + // ChangeStatusNew is the state a change is in during review. + ChangeStatusNew ChangeStatus = "NEW" + + // ChangeStatusMerged indicates a change was merged to the target branch. + ChangeStatusMerged ChangeStatus = "MERGED" + + // ChangeStatusAbandoned indicates a change was marked as abandoned. + ChangeStatusAbandoned ChangeStatus = "ABANDONED" +) + +// Message is a message left by a reviewer. +type Message struct { + Timestamp Time `json:"timestamp"` + Reviewer Account `json:"reviewer"` + Message string `json:"message"` +} + +// TrackingID allows storing identifiers from external systems, i.e. bug trackers. +type TrackingID struct { + System string `json:"system"` + ID string `json:"id"` +} + +// ChangeKind indicates the different changes that can be made to a change. +type ChangeKind string + +const ( + // ChangeKindRework indicates a non-trivial content change. + ChangeKindRework ChangeKind = "REWORK" + + // ChangeKindTrivialRebase indicates a conflict-free merge between the new parent and the prior patch set. + ChangeKindTrivialRebase ChangeKind = "TRIVIAL_REBASE" + + // ChangeKindMergeFirstParentUpdate indicates a conflict-free change of the first parent of a merge commit. + ChangeKindMergeFirstParentUpdate ChangeKind = "MERGE_FIRST_PARENT_UPDATE" + + // ChangeKindNoCodeChange indicates no code change (the tree and parent trees are unchanged) - commit message probably changed. + ChangeKindNoCodeChange ChangeKind = "NO_CODE_CHANGE" + + // ChangeKindNoChange indicates nothing changes: the commit message, tree, and parent tree are unchanged. + ChangeKindNoChange ChangeKind = "NO_CHANGE" +) + +// Approval represents the current and past state of an approval label. +type Approval struct { + Type string `json:"type"` + Description string `json:"description"` + Value string `json:"value"` + OldValue *string `json:"oldValue"` + GrantedOn *Time `json:"grantedOn"` + By *Account `json:"by"` +} + +// PatchSetComment is a single comment left on a patchset. +type PatchSetComment struct { + File string `json:"file"` + Line int `json:"line"` + Reviewer Account `json:"reviewer"` + Message string `json:"message"` +} + +// FilePatchType represents the different modifications that can be made to a file by a patchset. +type FilePatchType string + +const ( + // FilePatchTypeAdded indicates the file did not exist, and this patchset adds it to the tree. + FilePatchTypeAdded FilePatchType = "ADDED" + + // FilePatchTypeModified indicates the file exists before and after this patchset. + FilePatchTypeModified FilePatchType = "MODIFIED" + + // FilePatchTypeDeleted indicates the file is removed by this patchset. + FilePatchTypeDeleted FilePatchType = "DELETED" + + // FilePatchTypeRenamed indicates the file has a different name before this patchset than after. + FilePatchTypeRenamed FilePatchType = "RENAMED" + + // FilePatchTypeCopied indicates the file was copied from a different file. + FilePatchTypeCopied FilePatchType = "COPIED" + + // FilePatchTypeRewrite indicates the file had a significant quantity of content changed. + FilePatchTypeRewrite FilePatchType = "REWRITE" +) + +// File represents a file in a patchset as well as how it is being modified. +type File struct { + File string `json:"file"` + FileOld string `json:"fileOld"` + Type FilePatchType `json:"type"` +} + +// PatchSet represents a single patchset within a change. +type PatchSet struct { + Number int `json:"number"` + Revision string `json:"revision"` + Parents []string `json:"parents"` + Ref string `json:"ref"` + Uploader Account `json:"uploader"` + Author Account `json:"author"` + CreatedOn Time `json:"createdOn"` + Kind ChangeKind `json:"kind"` + Approvals []Approval `json:"approvals"` + Comments []PatchSetComment `json:"comments"` + Files []File `json:"file"` + SizeInsertions int `json:"sizeInsertions"` + SizeDeletions int `json:"sizeDeletions"` +} + +// Dependency represents a change on which this change is dependent. +type Dependency struct { + ID string `json:"id"` + Number int `json:"number"` + Revision string `json:"revision"` + Ref string `json:"ref"` + IsCurrentPatchSet bool `json:"isCurrentPatchSet"` +} + +// SubmitStatus indicates whether this change has met the submit conditions and is ready to submit. +type SubmitStatus string + +const ( + // SubmitStatusOK indicates this change is ready to submit - all submit requirements are met. + SubmitStatusOK SubmitStatus = "OK" + + // SubmitStatusNotReady indicates this change cannot yet be submitted. + SubmitStatusNotReady SubmitStatus = "NOT_READY" + + // SubmitStatusRuleError indicates the submit rules could not be evaluted. Administrator intervention is required. + SubmitStatusRuleError SubmitStatus = "RULE_ERROR" +) + +// LabelStatus indicates whether this label permits submission and if the label can be granted by anyone. +type LabelStatus string + +const ( + // LabelStatusOK indicates that this label provides what is necessary for submission (e.g. CR+2). + LabelStatusOK LabelStatus = "OK" + + // LabelStatusReject indicates this label prevents submission (e.g. CR-2). + LabelStatusReject LabelStatus = "REJECT" + + // LabelStatusNeed indicates this label is required for submission, but has not been satisfied (e.g. CR0). + LabelStatusNeed LabelStatus = "NEED" + + // LabelStatusMay indicates this label is not required for submission. It may or may not be set. + LabelStatusMay LabelStatus = "MAY" + + // LabelStatusImpossible indicates this label is required for submission, but cannot be satisfied. The ACLs on this label may be set incorrectly. + LabelStatusImpossible LabelStatus = "IMPOSSIBLE" +) + +// Label represents the status of a particular label. +type Label struct { + Label string `json:"label"` + Status LabelStatus `json:"status"` + By Account `json:"by"` +} + +// Requirement represents a submit requirement. +type Requirement struct { + FallbackText string `json:"fallbackText"` + Type string `json:"type"` + // TODO(lukegb): data +} + +// SubmitRecord represents the current submission state of a change. +type SubmitRecord struct { + Status SubmitStatus `json:"status"` + Labels []Label `json:"labels"` + Requirements []Requirement `json:"requirements"` +} + +// Change represents a Gerrit CL. +type Change struct { + Project string `json:"project"` + Branch string `json:"branch"` + Topic string `json:"topic"` + ID string `json:"id"` + Number int `json:"number"` + Subject string `json:"subject"` + Owner Account `json:"owner"` + URL string `json:"url"` + CommitMessage string `json:"commitMessage"` + CreatedOn Time `json:"createdOn"` + LastUpdated *Time `json:"lastUpdated"` + Open bool `json:"open"` + Status ChangeStatus `json:"status"` + Private bool `json:"private"` + WIP bool `json:"wip"` + Comments []Message `json:"comments"` + TrackingIDs []TrackingID `json:"trackingIds"` + CurrentPatchSet *PatchSet `json:"currentPatchSet"` + PatchSets []PatchSet `json:"patchSets"` + DependsOn []Dependency `json:"dependsOn"` + NeededBy []Dependency `json:"neededBy"` + SubmitRecords []SubmitRecord `json:"submitRecord"` + AllReviewers []Account `json:"allReviewers"` +} + +// RefUpdate represents a change in a ref. +type RefUpdate struct { + OldRev string `json:"oldRev"` + NewRev string `json:"newRev"` + RefName string `json:"refName"` + Project string `json:"project"` +} diff --git a/fun/clbot/gerrit/watcher.go b/fun/clbot/gerrit/watcher.go new file mode 100644 index 000000000000..d45876129b3a --- /dev/null +++ b/fun/clbot/gerrit/watcher.go @@ -0,0 +1,252 @@ +// Package gerrit implements a watcher for Gerrit events. +package gerrit + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + "time" + + "code.tvl.fyi/fun/clbot/backoffutil" + "code.tvl.fyi/fun/clbot/gerrit/gerritevents" + log "github.com/golang/glog" + "golang.org/x/crypto/ssh" +) + +// closer provides an embeddable implementation of Close which awaits a main loop acknowledging it has stopped. +type closer struct { + stop chan struct{} + stopped chan struct{} +} + +// newCloser returns a closer with the channels initialised. +func newCloser() closer { + return closer{ + stop: make(chan struct{}), + stopped: make(chan struct{}), + } +} + +// Close stops the main loop, waiting for the main loop to stop until it stops or the context is cancelled, whichever happens first. +func (c *closer) Close(ctx context.Context) error { + select { + case <-c.stopped: + return nil + case <-c.stop: + return nil + case <-ctx.Done(): + return ctx.Err() + default: + } + close(c.stop) + select { + case <-c.stopped: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// lineWriter is an io.Writer which splits on \n and outputs each line (with no trailing newline) to its output channel. +type lineWriter struct { + buf string + out chan string +} + +// Write accepts a slice of bytes containing zero or more new lines. +// If the contained channel is non-buffering or is full, this will block. +func (w *lineWriter) Write(p []byte) (n int, err error) { + w.buf += string(p) + pieces := strings.Split(w.buf, "\n") + w.buf = pieces[len(pieces)-1] + for n := 0; n < len(pieces)-1; n++ { + w.out <- pieces[n] + } + return len(p), nil +} + +// restartingClient is a simple SSH client that repeatedly connects to an SSH server, runs a command, and outputs the lines output by it on stdout onto a channel. +type restartingClient struct { + closer + + network string + addr string + cfg *ssh.ClientConfig + + exec string + output chan string + shutdown func() +} + +var ( + errStopConnect = errors.New("gerrit: told to stop reconnecting by remote server") +) + +func (c *restartingClient) runOnce() error { + netConn, err := net.Dial(c.network, c.addr) + if err != nil { + return fmt.Errorf("connecting to %v/%v: %w", c.network, c.addr, err) + } + defer netConn.Close() + + sshConn, newCh, newReq, err := ssh.NewClientConn(netConn, c.addr, c.cfg) + if err != nil { + return fmt.Errorf("creating SSH connection to %v/%v: %w", c.network, c.addr, err) + } + defer sshConn.Close() + + goAway := false + passedThroughReqs := make(chan *ssh.Request) + go func() { + defer close(passedThroughReqs) + for req := range newReq { + if req.Type == "goaway" { + goAway = true + log.Warningf("remote end %v/%v told me to go away!", c.network, c.addr) + sshConn.Close() + netConn.Close() + } + passedThroughReqs <- req + } + }() + + cl := ssh.NewClient(sshConn, newCh, passedThroughReqs) + + sess, err := cl.NewSession() + if err != nil { + return fmt.Errorf("NewSession on %v/%v: %w", c.network, c.addr, err) + } + defer sess.Close() + + sess.Stdout = &lineWriter{out: c.output} + + if err := sess.Start(c.exec); err != nil { + return fmt.Errorf("Start(%q) on %v/%v: %w", c.exec, c.network, c.addr, err) + } + + log.Infof("connected to %v/%v", c.network, c.addr) + + done := make(chan struct{}) + go func() { + sess.Wait() + close(done) + }() + go func() { + select { + case <-c.stop: + sess.Close() + case <-done: + } + return + }() + <-done + + if goAway { + return errStopConnect + } + return nil +} + +func (c *restartingClient) run() { + defer close(c.stopped) + bo := backoffutil.NewDefaultBackOff() + for { + timer := time.NewTimer(bo.NextBackOff()) + select { + case <-c.stop: + timer.Stop() + return + case <-timer.C: + break + } + if err := c.runOnce(); err == errStopConnect { + if c.shutdown != nil { + c.shutdown() + return + } + } else if err != nil { + log.Errorf("SSH: %v", err) + } else { + bo.Reset() + } + } +} + +// Output returns the channel on which each newline-delimited string output by the executed command's stdout can be received. +func (c *restartingClient) Output() <-chan string { + return c.output +} + +// dialRestartingClient creates a new restartingClient. +func dialRestartingClient(network, addr string, config *ssh.ClientConfig, exec string, shutdown func()) (*restartingClient, error) { + c := &restartingClient{ + closer: newCloser(), + network: network, + addr: addr, + cfg: config, + exec: exec, + output: make(chan string), + shutdown: shutdown, + } + go c.run() + return c, nil +} + +// Watcher watches +type Watcher struct { + closer + c *restartingClient + + output chan gerritevents.Event +} + +// Close shuts down the SSH client connection, if any, and closes the output channel. +// It blocks until shutdown is complete or until the context is cancelled, whichever comes first. +func (w *Watcher) Close(ctx context.Context) { + w.c.Close(ctx) + w.closer.Close(ctx) +} + +func (w *Watcher) run() { + defer close(w.stopped) + defer close(w.output) + for { + select { + case <-w.stop: + return + case o := <-w.c.Output(): + ev, err := gerritevents.Parse([]byte(o)) + if err != nil { + log.Errorf("failed to parse event %v: %v", o, err) + continue + } + w.output <- ev + } + } +} + +// Events returns the channel upon which parsed Gerrit events can be received. +func (w *Watcher) Events() <-chan gerritevents.Event { + return w.output +} + +// New returns a running Watcher from which events can be read. +// It will begin connecting to the provided address immediately. +func New(ctx context.Context, network, addr string, cfg *ssh.ClientConfig) (*Watcher, error) { + wc := newCloser() + rc, err := dialRestartingClient(network, addr, cfg, "gerrit stream-events", func() { + wc.Close(context.Background()) + }) + if err != nil { + return nil, fmt.Errorf("dialRestartingClient: %w", err) + } + w := &Watcher{ + closer: wc, + c: rc, + output: make(chan gerritevents.Event), + } + go w.run() + return w, nil +} diff --git a/fun/clbot/gerrit/watcher_test.go b/fun/clbot/gerrit/watcher_test.go new file mode 100644 index 000000000000..ae69b2fc4c18 --- /dev/null +++ b/fun/clbot/gerrit/watcher_test.go @@ -0,0 +1,190 @@ +package gerrit + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/subtle" + "fmt" + "net" + "testing" + "time" + + "code.tvl.fyi/fun/clbot/gerrit/gerritevents" + log "github.com/golang/glog" + "github.com/google/go-cmp/cmp" + "golang.org/x/crypto/ssh" +) + +var ( + sshServerSigner, sshServerPublicKey = mustNewKey() + sshClientSigner, sshClientPublicKey = mustNewKey() +) + +func mustNewKey() (ssh.Signer, ssh.PublicKey) { + key, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + if err != nil { + panic(err) + } + signer, err := ssh.NewSignerFromKey(key) + if err != nil { + panic(err) + } + publicKey, err := ssh.NewPublicKey(key.Public()) + if err != nil { + panic(err) + } + return signer, publicKey +} + +func newSSHServer(lines string) (addr string, cleanup func(), err error) { + config := &ssh.ServerConfig{ + PublicKeyCallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) { + pkBytes := pubKey.Marshal() + wantPKBytes := sshClientPublicKey.Marshal() + if subtle.ConstantTimeCompare(pkBytes, wantPKBytes) == 0 { + return nil, fmt.Errorf("unauthorized") + } + return &ssh.Permissions{}, nil + }, + } + config.AddHostKey(sshServerSigner) + + ln, err := net.Listen("tcp", ":0") + if err != nil { + log.Fatalf("Listen on tcp/:0: %v", err) + } + handle := func(conn net.Conn) { + defer conn.Close() + + sc, newchch, newreqch, err := ssh.NewServerConn(conn, config) + if err != nil { + log.Fatalf("NewServerConn: %v", err) + } + go ssh.DiscardRequests(newreqch) + for newCh := range newchch { + if newCh.ChannelType() != "session" { + newCh.Reject(ssh.UnknownChannelType, "unknown channel type") + continue + } + + channel, reqs, err := newCh.Accept() + if err != nil { + log.Fatalf("Could not accept channel: %v", err) + } + go func(in <-chan *ssh.Request) { + for req := range in { + req.Reply(req.Type == "exec", nil) + } + }(reqs) + channel.Write([]byte(lines)) + sc.SendRequest("goaway", false, nil) + } + } + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go handle(conn) + } + }() + + cleanup = func() { + ln.Close() + } + return ln.Addr().String(), cleanup, err +} + +func ts(s string) gerritevents.Time { + t, err := time.Parse("2006-01-02 15:04:05 -0700 MST", s) + if err != nil { + panic(err) + } + return gerritevents.Time{t} +} + +func optStr(s string) *string { return &s } + +func TestWatcher(t *testing.T) { + tcs := []struct { + name string + lines string + want []gerritevents.Event + }{{ + name: "no events", + }, { + name: "single test event", + lines: `{"author":{"name":"tazjin","email":"mail@tazj.in","username":"tazjin"},"approvals":[{"type":"Code-Review","description":"Code-Review","value":"2","oldValue":"0"}],"comment":"Patch Set 3: Code-Review+2","patchSet":{"number":3,"revision":"6fe272d3f82c6efdfe1167fab98bf918efc03fe5","parents":["d984b6018cf68c7e8b7169b475d90134fbcee767"],"ref":"refs/changes/44/244/3","uploader":{"name":"tazjin","email":"mail@tazj.in","username":"tazjin"},"createdOn":1592081910,"author":{"name":"tazjin","email":"mail@tazj.in","username":"tazjin"},"kind":"REWORK","sizeInsertions":83,"sizeDeletions":-156},"change":{"project":"depot","branch":"master","id":"I546c701145fa204b7ba7518a8a56a783588629e0","number":244,"subject":"refactor(ops/nixos): Move my NixOS configurations to //users/tazjin","owner":{"name":"tazjin","email":"mail@tazj.in","username":"tazjin"},"url":"https://cl.tvl.fyi/c/depot/+/244","commitMessage":"refactor(ops/nixos): Move my NixOS configurations to //users/tazjin\n\nNixOS modules move one level up because it\u0027s unlikely that //ops/nixos\nwill contain actual systems at this point (they\u0027re user-specific).\n\nThis is the first users folder, so it is also added to the root\nreadTree invocation for the repository.\n\nChange-Id: I546c701145fa204b7ba7518a8a56a783588629e0\n","createdOn":1592081577,"status":"NEW"},"project":"depot","refName":"refs/heads/master","changeKey":{"id":"I546c701145fa204b7ba7518a8a56a783588629e0"},"type":"comment-added","eventCreatedOn":1592081929} +`, + want: []gerritevents.Event{ + &gerritevents.CommentAdded{ + Type: "comment-added", + Change: gerritevents.Change{ + Project: "depot", + Branch: "master", + ID: "I546c701145fa204b7ba7518a8a56a783588629e0", + Number: 244, + Subject: "refactor(ops/nixos): Move my NixOS configurations to //users/tazjin", + Owner: gerritevents.Account{Name: "tazjin", Email: "mail@tazj.in", Username: "tazjin"}, + URL: "https://cl.tvl.fyi/c/depot/+/244", + CommitMessage: "refactor(ops/nixos): Move my NixOS configurations to //users/tazjin\n\nNixOS modules move one level up because it's unlikely that //ops/nixos\nwill contain actual systems at this point (they're user-specific).\n\nThis is the first users folder, so it is also added to the root\nreadTree invocation for the repository.\n\nChange-Id: I546c701145fa204b7ba7518a8a56a783588629e0\n", + CreatedOn: ts("2020-06-13 21:52:57 +0100 BST"), + Status: "NEW", + }, + PatchSet: gerritevents.PatchSet{ + Number: 3, + Revision: "6fe272d3f82c6efdfe1167fab98bf918efc03fe5", + Parents: []string{"d984b6018cf68c7e8b7169b475d90134fbcee767"}, + Ref: "refs/changes/44/244/3", + Uploader: gerritevents.Account{Name: "tazjin", Email: "mail@tazj.in", Username: "tazjin"}, + Author: gerritevents.Account{Name: "tazjin", Email: "mail@tazj.in", Username: "tazjin"}, + CreatedOn: ts("2020-06-13 21:58:30 +0100 BST"), + Kind: "REWORK", + SizeInsertions: 83, + SizeDeletions: -156, + }, + Author: gerritevents.Account{Name: "tazjin", Email: "mail@tazj.in", Username: "tazjin"}, + Approvals: []gerritevents.Approval{{Type: "Code-Review", Description: "Code-Review", Value: "2", OldValue: optStr("0")}}, + Comment: "Patch Set 3: Code-Review+2", + EventCreatedOn: ts("2020-06-13 21:58:49 +0100 BST"), + }, + }, + }} + for _, tc := range tcs { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + serverAddr, cleanup, err := newSSHServer(tc.lines) + if err != nil { + t.Fatalf("newSSHServer: %v", err) + } + t.Cleanup(cleanup) + + config := &ssh.ClientConfig{ + User: "bert", + Auth: []ssh.AuthMethod{ssh.PublicKeys(sshClientSigner)}, + HostKeyCallback: ssh.FixedHostKey(sshServerPublicKey), + Timeout: 10 * time.Millisecond, + } + w, err := New(ctx, "tcp", serverAddr, config) + if err != nil { + t.Fatalf("New: %v", err) + } + + var gotEvents []gerritevents.Event + for ev := range w.Events() { + gotEvents = append(gotEvents, ev) + } + if diff := cmp.Diff(gotEvents, tc.want); diff != "" { + t.Errorf("got events != want events: diff:\n%v", diff) + } + }) + } +} diff --git a/fun/clbot/go.mod b/fun/clbot/go.mod new file mode 100644 index 000000000000..f1e366ba9d21 --- /dev/null +++ b/fun/clbot/go.mod @@ -0,0 +1,12 @@ +module code.tvl.fyi/fun/clbot + +go 1.14 + +require ( + github.com/cenkalti/backoff/v4 v4.0.2 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b + github.com/google/go-cmp v0.4.1 + golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 + gopkg.in/irc.v3 v3.1.3 +) diff --git a/fun/clbot/go.sum b/fun/clbot/go.sum new file mode 100644 index 000000000000..d5434526e875 --- /dev/null +++ b/fun/clbot/go.sum @@ -0,0 +1,31 @@ +github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs= +github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= +golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/irc.v3 v3.1.3 h1:yeTiJ365882L8h4AnBKYfesD92y5R5ZhGiylu9DfcPY= +gopkg.in/irc.v3 v3.1.3/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/fun/defer_rs/.gitignore b/fun/defer_rs/.gitignore new file mode 100644 index 000000000000..6aa106405a4b --- /dev/null +++ b/fun/defer_rs/.gitignore @@ -0,0 +1,3 @@ +/target/ +**/*.rs.bk +Cargo.lock diff --git a/fun/defer_rs/Cargo.toml b/fun/defer_rs/Cargo.toml new file mode 100644 index 000000000000..0fcd60373ff7 --- /dev/null +++ b/fun/defer_rs/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "defer" +version = "0.1.0" +authors = ["Vincent Ambo <tazjin@gmail.com>"] + +[dependencies] diff --git a/fun/defer_rs/README.md b/fun/defer_rs/README.md new file mode 100644 index 000000000000..160158d177b8 --- /dev/null +++ b/fun/defer_rs/README.md @@ -0,0 +1,53 @@ +defer in Rust +============= + +After a Hacker News discussion about implementing Go's `defer` keyword in C++, +I stumbled upon [this comment](https://news.ycombinator.com/item?id=15523589) +and more specifically this response to it by "Occivink": + +> There's plenty of one-time cases where you don't want to declare an entire +> class but still enjoy scope-based functions. + +Specificall the "don't want to declare an entire class" suggests that languages +like C++ have high friction for explaining your desired invariant (cleanup is +run when `$thing` is destroyed) to the compiler. + +It seems like most languages either hand-wave this away (*cough* Java *cough*) +or use what seems like a workaround (`defer`). + +Rust has the so-called `Drop` trait, which is a typeclass that contains a single +method with no return value that is run when a variable is dropped (i.e. goes out +of scope). + +This works fine for most general cases - i.e. closing file handlers - but can +get complicated if other use-cases of `defer` are considered: + +* returning an error-value by mutating a reference in the enclosing scope (oh boy) +* deferring a decision about when/whether to run cleanup to the caller + +While thinking about how to do this with the `Drop` trait I realised that `defer` +can actually be trivially implemented in Rust, using `Drop`. + +A simple implementation of `defer` can be seen in [defer.rs](examples/defer.rs), +an implementation using shared mutable state for error returns is in the file +[defer-with-error.rs](examples/defer-with-error.rs) and an implementation that +allows cleanup to be *cancelled* (don't _actually_ do this, it leaks a pointer) +is in [undefer.rs](examples/undefer.rs). + +Whether any of this is actually useful is not up to me to decide. I haven't +actually had a real-life need for this. + +You can run the examples with `cargo run --example defer`, etc. + +## Notes + +* `Drop` is not guaranteed to run in case of panics or program aborts, if you + need support for that check out [scopeguard](https://github.com/bluss/scopeguard) +* `undefer` could be implemented safely by, for example, carrying a boolean that + by default causes execution to happen but can be flipped to disable it + +## Further reading: + +* [The Pain Of Real Linear Types in Rust](https://gankro.github.io/blah/linear-rust/) +* [Go's defer](https://tour.golang.org/flowcontrol/12) +* [Rust's Drop](https://doc.rust-lang.org/std/ops/trait.Drop.html) diff --git a/fun/defer_rs/examples/defer-with-error.rs b/fun/defer_rs/examples/defer-with-error.rs new file mode 100644 index 000000000000..26d56d77cf1b --- /dev/null +++ b/fun/defer_rs/examples/defer-with-error.rs @@ -0,0 +1,70 @@ +// Go's defer in Rust, with error value return. + +use std::rc::Rc; +use std::sync::RwLock; + +struct Defer<F: Fn()> { + f: F +} + +impl <F: Fn()> Drop for Defer<F> { + fn drop(&mut self) { + (self.f)() + } +} + +// Only added this for Go-syntax familiarity ;-) +fn defer<F: Fn()>(f: F) -> Defer<F> { + Defer { f } +} + +// Convenience type synonym. This is a reference-counted smart pointer to +// a shareable, mutable variable. +// Rust does not allow willy-nilly mutation of shared variables, so explicit +// write-locking must be performed. +type ErrorHandle<T> = Rc<RwLock<Option<T>>>; + +/////////////////// +// Usage example // +/////////////////// + +#[derive(Debug)] // Debug trait for some default way to print the type. +enum Error { DropError } + +fn main() { + // Create a place to store the error. + let drop_err: ErrorHandle<Error> = Default::default(); // create empty error + + // Introduce an arbitrary scope block (so that we still have control after + // the defer runs): + { + let mut i = 1; + + // Rc types are safe to clone and share for multiple ownership. + let err_handle = drop_err.clone(); + + // Call defer and let the closure own the cloned handle to the error: + let token = defer(move || { + // do something! + println!("Value is: {}", i); + + // ... oh no, it went wrong! + *err_handle.write().unwrap() = Some(Error::DropError); + }); + + i += 1; + println!("Value is: {}", i); + + // token goes out of scope here - drop() is called. + } + + match *drop_err.read().unwrap() { + Some(ref err) => println!("Oh no, an error occured: {:?}!", err), + None => println!("Phew, everything went well.") + }; +} + +// Prints: +// Value is: 2 +// Value is: 1 +// Oh no, an error occured: DropError! diff --git a/fun/defer_rs/examples/defer.rs b/fun/defer_rs/examples/defer.rs new file mode 100644 index 000000000000..eadac795f8bc --- /dev/null +++ b/fun/defer_rs/examples/defer.rs @@ -0,0 +1,31 @@ +// Go's defer in Rust! + +struct Defer<F: Fn()> { + f: F +} + +impl <F: Fn()> Drop for Defer<F> { + fn drop(&mut self) { + (self.f)() + } +} + +// Only added this for Go-syntax familiarity ;-) +fn defer<F: Fn()>(f: F) -> Defer<F> { + Defer { f } +} + +fn main() { + let mut i = 1; + + // Calling it "token" ... could be something else. The lifetime of this + // controls when the action is run. + let _token = defer(move || println!("Value is: {}", i)); + + i += 1; + println!("Value is: {}", i); +} + +// Prints: +// Value is: 2 +// Value is: 1 diff --git a/fun/defer_rs/examples/undefer.rs b/fun/defer_rs/examples/undefer.rs new file mode 100644 index 000000000000..17ad8a6b5485 --- /dev/null +++ b/fun/defer_rs/examples/undefer.rs @@ -0,0 +1,40 @@ +// Go's defer in Rust, with a little twist! + +struct Defer<F: Fn()> { + f: F +} + +impl <F: Fn()> Drop for Defer<F> { + fn drop(&mut self) { + (self.f)() + } +} + +// Only added this for Go-syntax familiarity ;-) +fn defer<F: Fn()>(f: F) -> Defer<F> { + Defer { f } +} + +// Changed your mind about the defer? +// (Note: This leaks the closure! Don't actually do this!) +fn undefer<F: Fn()>(token: Defer<F>) { + use std::mem; + mem::forget(token); +} + +fn main() { + let mut i = 1; + + // Calling it "token" ... could be something else. The lifetime of this + // controls when the action is run. + let token = defer(move || println!("Value is: {}", i)); + + i += 1; + println!("Value is: {}", i); + + // Oh, now I changed my mind about the previous defer: + undefer(token); +} + +// Prints: +// Value is: 2 diff --git a/fun/elblog/.gitignore b/fun/elblog/.gitignore new file mode 100644 index 000000000000..c531d9867f6c --- /dev/null +++ b/fun/elblog/.gitignore @@ -0,0 +1 @@ +*.elc diff --git a/fun/elblog/README.md b/fun/elblog/README.md new file mode 100644 index 000000000000..994b1138ef8f --- /dev/null +++ b/fun/elblog/README.md @@ -0,0 +1,11 @@ +elblog +====== + +This is a simple blogging software written in Emacs Lisp. + +The idea is that it should be able to do most of the things [my actual blog][] +does at the moment. + +No documentation exists for now besides the commit messages, but it works! + +[my actual blog]: https://www.tazj.in/ diff --git a/fun/elblog/blog.css b/fun/elblog/blog.css new file mode 100644 index 000000000000..0d021f78e89b --- /dev/null +++ b/fun/elblog/blog.css @@ -0,0 +1,37 @@ +<style type="text/css"> +body { + margin: 40px auto; + max-width: 800px; + line-height: 1.6; + font-size: 18px; + color: #383838; + padding: 0 10px +} +h1, h2, h3 { + line-height: 1.2 +} +.footer { + text-align: right; +} +.lod { + text-align: center; +} +.unstyled-link { + color: inherit; + text-decoration: none; +} +.uncoloured-link { + color: inherit; +} +.date { + text-align: right; + font-style: italic; + float: right; +} +.inline { + display: inline; +} +.navigation { + text-align: center; +} +</style> diff --git a/fun/elblog/blog.el b/fun/elblog/blog.el new file mode 100644 index 000000000000..102aa3791481 --- /dev/null +++ b/fun/elblog/blog.el @@ -0,0 +1,123 @@ +;;; blog.el --- A simple org-mode & elnode blog software. +;;; -*- lexical-binding: t; -*- + +(require 'dash) +(require 'elnode) +(require 'f) +(require 'ht) + +;; Definition of customization options + +(defgroup elblog nil + "Configuration for the Emacs Lisp blog software" + :link '(url-link "https://github.com/tazjin/elblog")) + +(defcustom elblog-port 8010 + "Port to run elblog's HTTP server on" + :group 'elblog + :type 'integer) + +(defcustom elblog-host "localhost" + "Host for elblog's HTTP server to listen on" + :group 'elblog + :type 'string) + +(defcustom elblog-title "Elblog" + "Title text for this elblog instance" + :group 'elblog + :type 'string) + +(defcustom elblog-article-directory nil + "Directory in which elblog articles are stored" + :group 'elblog + :type 'string) + +(defcustom elblog-additional-routes '() + "Additional Elnode routes to register in the Elblog instance" + :group 'elblog + :type '(alist :key-type regexp :value-type function)) + +;; Declare user-configurable variables needed at runtime. + +(defvar elblog-articles (ht-create) + "A hash-table of blog articles. This is used for looking up articles from + URL fragments as well as for rendering the index.") + +;; HTML templating setup + +(defun template-preamble () + "Templates the preamble snippet with the correct blog title." + (format (f-read-text "preamble.html") elblog-title)) + +(defun configure-org-html-export () + "Configure org-mode settings for elblog's HTML templating to work correctly." + (setq org-html-postamble t) + (setq org-html-doctype "html5") + (setq org-html-head-include-scripts nil) + (setq org-html-style-default (f-read-text "blog.css")) + (setq org-html-preamble-format `(("en" ,(template-preamble)))) + (setq org-html-postamble-format `(("en" ,(f-read-text "postamble.html"))))) + +;; Article fetching & rendering functions + +(defun render-org-buffer (input-buffer &optional force) + "Renders an org-mode buffer as HTML and returns the name of the output buffer." + (letrec ((output-buffer (concat (buffer-name input-buffer) "-rendered")) + ;; Don't re-render articles unless forced. + (must-render (or force + (not (get-buffer output-buffer))))) + (if (and input-buffer must-render) + (with-current-buffer input-buffer + (org-export-to-buffer 'html output-buffer nil nil t))) + (if input-buffer output-buffer nil))) + +(defun get-buffer-string (buffer) + "Returns the contents of the specified buffer as a string." + (with-current-buffer (get-buffer buffer) + (buffer-string))) + +(defvar-local article-not-found + '(404 . "<html><body><p>Oh no, the article was not found.</p></body></html>")) + +(defvar-local text-html '("Content-Type" . "text/html")) + +(defun render-article (article) + "Renders an article, if it exists." + (letrec ((rendered (-some->> + (ht-get elblog-articles article) + (concat elblog-article-directory) + (find-file) + (render-org-buffer)))) + (if rendered `(200 . ,(get-buffer-string rendered)) + article-not-found))) + +(defun blog-post-handler (httpcon) + "This handler servers a blog post from the configured blog post directory." + (let ((response (render-article (elnode-http-mapping httpcon 1)))) + (elnode-http-start httpcon (car response) text-html) + (elnode-http-return httpcon (cdr response)))) + +;; Web server implementation + +(defvar elblog-routes + '(("^.*//\\(.*\\)" . blog-post-handler)) + "The default routes available in elblog. They can be extended by the user +by setting the elblog-additional-routes customize option.") + +(defun elblog-handler (httpcon) + (elnode-hostpath-dispatcher + httpcon + (-concat elblog-additional-routes elblog-routes))) + +(defun start-elblog () + (interactive) + (configure-org-html-export) + (elnode-start 'elblog-handler + :port elblog-port + :host elblog-host)) + +(defun stop-elblog () + (interactive) + (elnode-stop elblog-port)) + +(provide 'elblog) diff --git a/fun/elblog/postamble.html b/fun/elblog/postamble.html new file mode 100644 index 000000000000..16a26218a0c0 --- /dev/null +++ b/fun/elblog/postamble.html @@ -0,0 +1,9 @@ +<hr> +<footer><p class="footer">Served with <a class="uncoloured-link" href="https://github.com/tazjin/elblog">Emacs</a>.</p> + <p class="footer"> + <a class="uncoloured-link" href="https://twitter.com/tazjin">Twitter</a> + | + <a class="uncoloured-link" href="mailto:blog@tazj.in">Mail</a> + </p> + <p class="lod">ಠ_ಠ</p> +</footer> diff --git a/fun/elblog/preamble.html b/fun/elblog/preamble.html new file mode 100644 index 000000000000..be74b9207e72 --- /dev/null +++ b/fun/elblog/preamble.html @@ -0,0 +1,6 @@ +<header> + <h1> + <a class="unstyled-link" href="/">%s</a> + </h1> + <hr> +</header> diff --git a/fun/gemma/CODE_OF_CONDUCT.md b/fun/gemma/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..c4013ac13ebc --- /dev/null +++ b/fun/gemma/CODE_OF_CONDUCT.md @@ -0,0 +1,20 @@ +A SERMON ON ETHICS AND LOVE +=========================== + +One day Mal-2 asked the messenger spirit Saint Gulik to approach the Goddess and request Her presence for some desperate advice. Shortly afterwards the radio came on by itself, and an ethereal female Voice said **YES?** + +"O! Eris! Blessed Mother of Man! Queen of Chaos! Daughter of Discord! Concubine of Confusion! O! Exquisite Lady, I beseech You to lift a heavy burden from my heart!" + +**WHAT BOTHERS YOU, MAL? YOU DON'T SOUND WELL.** + +"I am filled with fear and tormented with terrible visions of pain. Everywhere people are hurting one another, the planet is rampant with injustices, whole societies plunder groups of their own people, mothers imprison sons, children perish while brothers war. O, woe." + +**WHAT IS THE MATTER WITH THAT, IF IT IS WHAT YOU WANT TO DO?** + +"But nobody Wants it! Everybody hates it." + +**OH. WELL, THEN *STOP*.** + +At which moment She turned herself into an aspirin commercial and left The Polyfather stranded alone with his species. + +SINISTER DEXTER HAS A BROKEN SPIROMETER. diff --git a/fun/gemma/LICENSE b/fun/gemma/LICENSE new file mode 100644 index 000000000000..94a9ed024d38 --- /dev/null +++ b/fun/gemma/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/fun/gemma/README.md b/fun/gemma/README.md new file mode 100644 index 000000000000..064742c0091f --- /dev/null +++ b/fun/gemma/README.md @@ -0,0 +1,96 @@ +# Gemma + +Gemma is a simple application to track *recurring* tasks, named after Gemma +Hartley who [wrote an article][] about task distribution issues in households. + +## Background + +(Skip this if you just want the technical bits) + +Gemma's article launched a discussion in my friend circle about what causes an +uneven distribution of household workload. I theorised that this is not so much +a gender issue, but mostly a discoverability issue. + +Usually one person in a household is aware of what needs to be done, but in many +cases the "overhead" of delegating the tasks would actually take more time than +simply doing the task. + +I theorise further that the person (or people) who do a smaller share of the +household work would often do the work if they had a convenient way to become +aware of what needs to be done. Many times the "household manager" has the +function of tracking non-obvious tasks like when bedsheets were last changed - +shouldn't it be possible to actually distribute this information somehow? + +## The Project + +This project is an initial attempt at sketching out a little application that +aids with reminding users of recurring tasks. Some basic ideas: + +* The system should be blame-free. +* There should be as little usage overhead as possible so that people actually + do use it. +* It should work mostly passively without much user interaction. + +I believe that the basic (*very* simple) idea behind Gemma solves these issues. +Unfortunately my living situation changed before I actually got to test this out +in a real-life situation involving multiple people, but feedback from other +potential test subjects would be welcome! :) + +## Overview + +Gemma is a Common Lisp application in which a list of recurring tasks is +declared, together with the *maximum interval* at which they should be completed +(in days). Example: + +```lisp +;; Bathroom tasks +(deftask bathroom/wipe-mirror 7) +(deftask bathroom/wipe-counter 7) + +;; Bedroom tasks +(deftask bedroom/change-sheets 7) +(deftask bedroom/vacuum 10) + +;; Kitchen tasks +(deftask kitchen/trash 3) +(deftask kitchen/wipe-counters 3) +(deftask kitchen/vacuum 5 "Kitchen has more crumbs and such!") + +;; Entire place +(deftask clean-windows 60) +``` + +These tasks are marked with their last completion time and tracked by Gemma. A +simple Elm-based frontend application displays the tasks sorted by their +"urgency" and features a button to mark a task as completed: + + + +Marking a task as completed resets its counter and moves it to the bottom of the +task list. + +In theory this *should be it*, the frontend is made available to household +members in some easily accessible place (e.g. an old phone glued to the fridge!) +and people should attempt to develop a habit of checking what needs to be done +occasionally. + +The "household manager" still exists as a role of the household because someone +is entering the tasks into the application, but if my theory about people not +actually being actively *unwilling* to do tasks is correct this could help a +lot. + +## Usage + +(*Note*: Gemma is alpha software so the below is clearly not the final goal) + +Right now using this is non-trivial, but I'll eventually make a better +distribution. Basically you need to know Common Lisp (in which case you'll know +how to get the backend running) and have `elm-reactor` installed to run the +development version of the frontend application. + +Gemma is configured via a configuration file that should be located either at +`/etc/gemma/config.lisp` or at a custom location specified via the environment +variable `GEMMA_CONFIG`. Have a look at the `config.lisp` file in the repository +root for an example. + +[wrote an article]: http://www.harpersbazaar.com/culture/features/a12063822/emotional-labor-gender-equality/ diff --git a/fun/gemma/config.lisp b/fun/gemma/config.lisp new file mode 100644 index 000000000000..54f8e5f34462 --- /dev/null +++ b/fun/gemma/config.lisp @@ -0,0 +1,21 @@ +;; Example configuration file for Gemma + +(config :port 4242 + :data-dir "/tmp/gemma/") + +(deftask bathroom/wipe-mirror 7) +(deftask bathroom/wipe-counter 7) + +;; Bedroom tasks +(deftask bedroom/change-sheets 7) +(deftask bedroom/vacuum 10) + +;; Kitchen tasks +(deftask kitchen/normal-trash 3) +(deftask kitchen/green-trash 5) +(deftask kitchen/blue-trash 5) +(deftask kitchen/wipe-counters 3) +(deftask kitchen/vacuum 5 "Kitchen has more crumbs and such!") + +;; Entire place +(deftask clean-windows 60) diff --git a/fun/gemma/default.nix b/fun/gemma/default.nix new file mode 100644 index 000000000000..0b5e67b1594b --- /dev/null +++ b/fun/gemma/default.nix @@ -0,0 +1,56 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs) cacert iana-etc libredirect stdenv runCommandNoCC writeText; + elmPackages = depot.third_party.elmPackages_0_18; + + frontend = stdenv.mkDerivation { + name = "gemma-frontend.html"; + src = ./frontend; + buildInputs = [ cacert iana-etc elmPackages.elm ]; + + # The individual Elm packages this requires are not packaged and I + # can't be bothered to do that now, so lets open the escape hatch: + outputHashAlgo = "sha256"; + outputHash = "000xhds5bsig3kbi7dhgbv9h7myacf34bqvw7avvz7m5mwnqlqg7"; + + phases = [ "unpackPhase" "buildPhase" ]; + buildPhase = '' + export NIX_REDIRECTS=/etc/protocols=${iana-etc}/etc/protocols \ + LD_PRELOAD=${libredirect}/lib/libredirect.so + + export SYSTEM_CERTIFICATE_PATH=${cacert}/etc/ssl/certs + + mkdir .home && export HOME="$PWD/.home" + elm-make --yes Main.elm --output $out + ''; + }; + + injectFrontend = writeText "gemma-frontend.lisp" '' + (in-package :gemma) + (setq *static-file-location* "${runCommandNoCC "frontend" {} '' + mkdir -p $out + cp ${frontend} $out/index.html + ''}/") + ''; +in depot.nix.buildLisp.program { + name = "gemma"; + + deps = with depot.third_party.lisp; [ + cl-json + cl-prevalence + hunchentoot + local-time + ]; + + srcs = [ + ./src/gemma.lisp + injectFrontend + ]; + + # depends on SBCL + brokenOn = [ + "ccl" + "ecl" + ]; +} diff --git a/fun/gemma/frontend/Main.elm b/fun/gemma/frontend/Main.elm new file mode 100644 index 000000000000..e449908e499c --- /dev/null +++ b/fun/gemma/frontend/Main.elm @@ -0,0 +1,221 @@ +-- Copyright (C) 2016-2017 Vincent Ambo <mail@tazj.in> +-- +-- This file is part of Gemma. +-- +-- Gemma is free software: you can redistribute it and/or modify it +-- under the terms of the GNU General Public License as published by +-- the Free Software Foundation, either version 3 of the License, or +-- (at your option) any later version. + + +module Main exposing (..) + +import Html exposing (Html, text, div, span) +import Html.Attributes exposing (style) +import Json.Decode exposing (..) +import Http +import Time + + +-- Material design imports + +import Material +import Material.Card as Card +import Material.Color as Color +import Material.Grid exposing (grid, cell, size, Device(..)) +import Material.Layout as Layout +import Material.Scheme as Scheme +import Material.Options as Options +import Material.Elevation as Elevation +import Material.Button as Button + + +-- API interface to Gemma + + +type alias Task = + { name : String + , description : Maybe String + , remaining : Int + } + + +emptyStringFilter s = + if s == "" then + Nothing + else + Just s + + +decodeEmptyString : Decoder (Maybe String) +decodeEmptyString = + map emptyStringFilter string + + +decodeTask : Decoder Task +decodeTask = + map3 Task + (field "name" string) + (field "description" decodeEmptyString) + (field "remaining" int) + + +loadTasks : Cmd Msg +loadTasks = + let + request = + Http.get "/tasks" (list decodeTask) + in + Http.send NewTasks request + + +completeTask : Task -> Cmd Msg +completeTask task = + let + request = + Http.getString + (String.concat + [ "/complete?task=" + , task.name + ] + ) + in + Http.send (\_ -> LoadTasks) request + + + +-- Elm architecture implementation + + +type Msg + = None + | LoadTasks + | NewTasks (Result Http.Error (List Task)) + | Mdl (Material.Msg Msg) + | Complete Task + + +type alias Model = + { tasks : List Task + , error : Maybe String + , mdl : Material.Model + } + + +update : Msg -> Model -> ( Model, Cmd Msg ) +update msg model = + case msg of + LoadTasks -> + ( model, loadTasks ) + + Complete task -> + ( model, completeTask task ) + + NewTasks (Ok tasks) -> + ( { model | tasks = tasks, error = Nothing }, Cmd.none ) + + NewTasks (Err err) -> + ( { model | error = Just (toString err) }, Cmd.none ) + + _ -> + ( model, Cmd.none ) + + + +-- View implementation + + +white = + Color.text Color.white + + +taskColor : Task -> Color.Hue +taskColor task = + if task.remaining > 2 then + Color.Green + else if task.remaining < 0 then + Color.Red + else + Color.Yellow + + +within : Task -> String +within task = + if task.remaining < 0 then + "This task is overdue!" + else if task.remaining > 2 then + String.concat + [ "Relax, this task has " + , toString task.remaining + , " days left before it is due." + ] + else + String.concat + [ "This task should be completed within " + , toString task.remaining + , " days. Consider doing it now!" + ] + + +renderTask : Model -> Task -> Html Msg +renderTask model task = + Card.view + [ Color.background (Color.color (taskColor task) Color.S800) + , Elevation.e3 + ] + [ Card.title [] [ Card.head [ white ] [ text task.name ] ] + , Card.text [ white ] + [ text (Maybe.withDefault "" task.description) + , Html.br [] [] + , text (within task) + ] + , Card.actions + [ Card.border ] + [ Button.render Mdl + [ 0 ] + model.mdl + [ white, Button.ripple, Button.accent, Options.onClick (Complete task) ] + [ text "Completed" ] + ] + ] + + +gemmaView : Model -> Html Msg +gemmaView model = + grid [] + (List.map (\t -> cell [ size All 4 ] [ renderTask model t ]) + model.tasks + ) + + +view : Model -> Html Msg +view model = + gemmaView model |> Scheme.top + + + +-- subscriptions : Model -> Sub Msg + + +subscriptions model = + Sub.batch + [ Material.subscriptions Mdl model + , Time.every (15 * Time.second) (\_ -> LoadTasks) + ] + + +main : Program Never Model Msg +main = + let + model = + { tasks = [] + , error = Nothing + , mdl = Material.model + } + in + Html.program + { init = ( model, Cmd.batch [ loadTasks, Material.init Mdl ] ) + , view = view + , update = update + , subscriptions = subscriptions + } diff --git a/fun/gemma/frontend/elm-package.json b/fun/gemma/frontend/elm-package.json new file mode 100644 index 000000000000..2ae541ae0b95 --- /dev/null +++ b/fun/gemma/frontend/elm-package.json @@ -0,0 +1,17 @@ +{ + "version": "1.0.0", + "summary": "helpful summary of your project, less than 80 characters", + "repository": "https://github.com/user/project.git", + "license": "BSD3", + "source-directories": [ + "." + ], + "exposed-modules": [], + "dependencies": { + "elm-lang/core": "5.1.1 <= v < 6.0.0", + "elm-lang/html": "2.0.0 <= v < 3.0.0", + "elm-lang/http": "1.0.0 <= v < 2.0.0", + "debois/elm-mdl": "8.1.0 <= v < 9.0.0" + }, + "elm-version": "0.18.0 <= v < 0.19.0" +} diff --git a/fun/gemma/src/gemma.lisp b/fun/gemma/src/gemma.lisp new file mode 100644 index 000000000000..20a76caae629 --- /dev/null +++ b/fun/gemma/src/gemma.lisp @@ -0,0 +1,192 @@ +;; Copyright (C) 2016-2020 Vincent Ambo <mail@tazj.in> +;; +;; This file is part of Gemma. +;; +;; Gemma is free software: you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +(defpackage gemma + (:use :cl + :local-time + :cl-json) + (:import-from :sb-posix :getenv) + (:shadowing-import-from :sb-posix :getcwd) + (:export :start-gemma :config :main)) +(in-package :gemma) + +;; TODO: Store an average of how many days it was between task +;; completions. Some of the current numbers are just guesses +;; anyways. + +(defmacro in-case-of (x &body body) + "Evaluate BODY if X is non-nil, binding the value of X to IT." + `(let ((it ,x)) + (when it ,@body))) + +;; Set default configuration parameters +(defvar *gemma-port* 4242 + "Port on which the Gemma web server listens.") + +(defvar *gemma-acceptor* nil + "Hunchentoot acceptor for Gemma's web server.") + +(defvar *static-file-location* "frontend/" + "Folder from which to serve static assets. If built inside of Nix, + the path is injected during the build.") + +(defvar *p-tasks* nil + "All tasks registered in this Gemma instance.") + +(defun initialise-persistence (data-dir) + (setq *p-tasks* (cl-prevalence:make-prevalence-system data-dir)) + + ;; Initialise database ID counter + (or (> (length (cl-prevalence:find-all-objects *p-tasks* 'task)) 0) + (cl-prevalence:tx-create-id-counter *p-tasks*))) + +(defun config (&key port data-dir) + "Configuration function for use in the Gemma configuration file." + + (in-package :gemma) + (in-case-of port (defparameter *gemma-port* it)) + (initialise-persistence (or data-dir "data/"))) + +;; +;; Define task management system +;; + +(defclass task () + ((id :reader id + :initarg :id) + + ;; (Unique) name of the task + (name :type symbol + :initarg :name + :accessor name-of) + + ;; Maximum completion interval + (days :type integer + :initarg :days + :accessor days-of) + + ;; Optional description + (description :type string + :initarg :description + :accessor description-of) + + ;; Last completion time + (done-at :type timestamp + :initarg :done-at + :accessor last-done-at))) + +(defmacro deftask (task-name days &optional description) + (unless (get-task task-name) + `(progn (cl-prevalence:tx-create-object + *p-tasks* + 'task + (quote ((name ,task-name) + (days ,days) + (description ,(or description "")) + (done-at ,(now))))) + (cl-prevalence:snapshot *p-tasks*)))) + +(defun get-task (name) + (cl-prevalence:find-object-with-slot *p-tasks* 'task 'name name)) + +(defun list-tasks () + (cl-prevalence:find-all-objects *p-tasks* 'task)) + +(defun days-remaining (task) + "Returns the number of days remaining before the supplied TASK reaches its +maximum interval." + (let* ((expires-at (timestamp+ (last-done-at task) + (days-of task) :day)) + (secs-until-expiry (timestamp-difference expires-at (now)))) + (round (/ secs-until-expiry 60 60 24)))) + +(defun sort-tasks (tasks) + "Sorts TASKS in descending order by number of days remaining." + (sort (copy-list tasks) + (lambda (t1 t2) (< (days-remaining t1) + (days-remaining t2))))) + +(defun complete-task (name &optional at) + "Mark the task with NAME as completed, either now or AT specified time." + (cl-prevalence:tx-change-object-slots *p-tasks* 'task + (id (get-task name)) + `((done-at ,(or at (now))))) + (cl-prevalence:snapshot *p-tasks*)) + +;; +;; Define web API +;; + +(defun response-for (task) + "Create a response object to be JSON encoded for TASK." + `((:name . ,(name-of task)) + (:description . ,(description-of task)) + (:remaining . ,(days-remaining task)))) + +(defun start-gemma () + (in-package :gemma) + + ;; Load configuration + (load (pathname (or (getenv "GEMMA_CONFIG") + "/etc/gemma/config.lisp"))) + + ;; Set up web server + (setq *gemma-acceptor* (make-instance 'hunchentoot:easy-acceptor + :port *gemma-port* + :document-root *static-file-location*)) + (hunchentoot:start *gemma-acceptor*) + + ;; Task listing handler + (hunchentoot:define-easy-handler + (get-tasks :uri "/tasks") () + + (setf (hunchentoot:content-type*) "application/json") + (setf (hunchentoot:header-out "Access-Control-Allow-Origin") "*") + (encode-json-to-string + ;; Construct a frontend-friendly representation of the tasks. + (mapcar #'response-for (sort-tasks (list-tasks))))) + + ;; Task completion handler + (hunchentoot:define-easy-handler + (complete-task-handler :uri "/complete") (task) + (setf (hunchentoot:content-type*) "application/json") + (let* ((key (find-symbol (camel-case-to-lisp task) "GEMMA"))) + (format t "Marking task ~A as completed" key) + (complete-task key) + (encode-json-to-string (response-for (get-task key)))))) + +(defun main () + "This function serves as the entrypoint for ASDF-built executables. + It joins the Hunchentoot server thread to keep the process running + for as long as the server is alive." + + (start-gemma) + (sb-thread:join-thread + (find-if (lambda (th) + (string= (sb-thread:thread-name th) + (format nil "hunchentoot-listener-*:~A" *gemma-port*))) + (sb-thread:list-all-threads)))) + +;; Experimentation / testing stuff + +(defun randomise-completion-times () + "Set some random completion timestamps for all tasks" + (mapcar + (lambda (task) + (complete-task (name-of task) + (timestamp- (now) + (random 14) + :day))) + (cl-prevalence:find-all-objects *p-tasks* 'task))) + +(defun clear-all-tasks () + (mapcar (lambda (task) (cl-prevalence:tx-delete-object *p-tasks* 'task (id task))) + (cl-prevalence:find-all-objects *p-tasks* 'task))) + +;; (randomise-completion-times) diff --git a/fun/idual/README.md b/fun/idual/README.md new file mode 100644 index 000000000000..922047617f25 --- /dev/null +++ b/fun/idual/README.md @@ -0,0 +1,34 @@ +# iDual light control + +This folder contains some tooling for controlling iDual LED lights +(which use infrared controls) using a "Broadlink RM Pro" infrared +controller. + +The supported colour codes of the iDual remote are stored in +`codes.txt`. + +The point of this is to make it possible for me to automate my lights +in the morning, so that I can actually get out of bed. + +## Capturing codes + +Capturing codes is relatively easy, assuming that the broadlink device +is set up: + +```python +import broadlink +import base64 + +devices = broadlink.discover(timeout=5) +devices[0].auth() +``` + +For each code, the procedure is as follows: + +```python +devices[0].find_rf_packet() +# wait until this returns True + +devices[0].check_data() +# this will return the code +``` diff --git a/fun/idual/default.nix b/fun/idual/default.nix new file mode 100644 index 000000000000..0f87f4ae1a76 --- /dev/null +++ b/fun/idual/default.nix @@ -0,0 +1,24 @@ +{ depot, pkgs, lib, ... }: + +let + inherit (pkgs) python3 python3Packages; + + opts = { + pname = "idualctl"; + version = "0.1"; + src = ./.; + + propagatedBuildInputs = [ + depot.third_party.python.broadlink + ]; + }; + package = python3Packages.buildPythonPackage opts; + script = python3Packages.buildPythonApplication opts; +in depot.nix.readTree.drvTargets { + inherit script; + python = python3.withPackages (_: [ package ]); + setAlarm = pkgs.writeShellScriptBin "set-alarm" '' + echo "setting an alarm for ''${1}" + ${pkgs.systemd}/bin/systemd-run --user --on-calendar="''${1} Europe/London" --unit=light-alarm.service + ''; +} diff --git a/fun/idual/idual/__init__.py b/fun/idual/idual/__init__.py new file mode 100644 index 000000000000..35c3bbae9039 --- /dev/null +++ b/fun/idual/idual/__init__.py @@ -0,0 +1,65 @@ +import base64 +import broadlink +import time +import sys + +commands = { + # system commands + 'on' : 'JgBIAAABK5AVERQ2FBEUERQSFBEUERQSFBEUNhQ2FDUUNhQ2FDYUNRU1FBIUERQRFBIUERQRFBIUERQ2FDYUNRQ2FDYUNhQ1FQANBQ==', + 'off' : 'JgBIAAABLJAUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDUVNRQ2FDYUNhQRFDYUERQSFBEUERQSFBEUNhQRFDYUNhQ2FDUUNhQ2FAANBQ==', + 'darker' : 'JgBIAAABLI8VERQ2FBEUERURFBEUEhQRFBEUNhQ2FDYUNRU1FDYUNhQRFBIUERQRFDYUEhQRFBEVNRQ2FDYUNhQRFDYUNhQ1FQANBQ==', + 'brighter' : 'JgBIAAABLI8VERQ2FBEUERQSFBEUEhQRFBEUNhQ2FDUVNRQ2FDYUNRU1FBIUERQ2FBEUEhQRFBEUEhQ1FTUUEhQ1FTUUNhQ2FAANBQ==', + + # presets + 'candle' : 'JgBQAAABKZISExI4EhMSFBITEhQRFBITEhQROBI4EjgSOBE4EjgSOBI4ERQSExIUEjgRFBITEhQSExI4EjgROBIUEjcSOBI4EgAFJgABKUkSAA0FAAAAAAAAAAA=', + 'bulb' : 'JgBYAAABK5AUERQ2FBEVERQRFBEVERQRFBEVNRQ2FDYUNRU1FDYUNhQRFDYUNRURFBEUEhQRFBEUNhQRFREUNhQ1FDYUNhQ2FAAFIwABKkgVAAxOAAErRxUADQU=', + 'sun' : 'JgBQAAABLI8VERQ2FBEUERURFBEUEhQRFBEUNhQ2FDYUNRU1FTUUNhQSFDUUEhQ2FBEUERURFBEUNhQSFDUUEhQ2FDUVNRQ2FAAFJQABK0cVAA0FAAAAAAAAAAA=', + 'cold' : 'JgBQAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDYUNRQ2FDYUNhQ1FTUUEhQRFBEUEhQRFBIUERQRFDYUNhQ2FDYUNRQ2FAAFJAABK0cVAA0FAAAAAAAAAAA=', + 'eve_dark' : 'JgBQAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDYUNRU1FDYUNhQRFDYUERQSFDUUEhQRFBIUNRQSFDUUNhQSFDUUNhQ2FAAFIwABLEYVAA0FAAAAAAAAAAA=', + 'eve_fade' : 'JgBIAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDYUNRQ2FDYUNhQ1FDYUNhQRFDYUERQSFBEUEhQRFBEUNhQSEzYUNhQ2FAANBQ==', + 'reading' : 'JgBQAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDUVNRQ2FDYUNhQ1FDYUEhQ1FBIUERQRFBIUERQSFDUUEhQ1FTUUNhQ2FAAFJAABK0YVAA0FAAAAAAAAAAA=', + 'yoga' : 'JgBYAAABLI8VERQ1FREUERQSFBEUERURFBEUNhQ1FTUUNhQ2FDYUNRURFBEUNhQRFBIUERMSExMTNxM2ExMTNxM2EzcTNxM3EwAFJQABKkgTAAxRAAErRxUADQU=', + 'morning' : 'JgBQAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDUVNRQ2FDYUNRU1FDYUERURFDUVERQRFBIUERQRFTUUNhQRFDYUNhQ2FAAFIwABK0cVAA0FAAAAAAAAAAA=', + 'colours' : 'JgBQAAABLI8VERQ2FBEUERQSFBEUERURFBEUNhQ1FTUUNhQ2FDYUNRQSFBEUERQ2FDYUERQSFBEUNhQ1FTUUEhQRFDYUNRU1FAAFJAABKkcVAA0FAAAAAAAAAAA=', + 'random' : 'JgBQAAABK5AUEhQ1FREUERQSFBEUERQSExIUNhQ1EzcTNxM3EzYTNxMTEhMTNxM2ExMTEhMSExMTNxM2ExMTEhM3EzcTNhM3EwAFJQABK0cVAA0FAAAAAAAAAAA=', + 'island' : 'JgBQAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDUVNRQ2FDYUNRU1FBIUNRURFBEUEhQRFBEUEhQ1FREUNhQ1FDYUNhQ2FAAFIwABK0cVAA0FAAAAAAAAAAA=', + 'forest' : 'JgBQAAABK5AUEhQ1FREUERQSFBEUERQSFBEUNhQ1FTUUNhQ2FDUVNRQSFBEUNhQRFDYUERQSFBEUNhQ2FBEUNhQRFDYUNhQ1FQAFIwABK0cVAA0FAAAAAAAAAAA=', + 'ocean' : 'JgBQAAABK5AUEhQ1FREUERQSFBEUERQRFREUNhQ1FTUUNhQ2FDUVNRQ2FBEUEhQ1FTUUEhQRFBEUEhQ1FTUUEhQRFDYUNRU1FAAFJAABK0cVAA0FAAAAAAAAAAA=', + 'fire' : 'JgBQAAABK5AUERQ2FBEUEhQRFBEUEhQRFBEUNhQ2FDUVNRQ2FDYUNRU1FBIUNRU1FBIUERQRFBIUERQ2FBEUEhQ1FTUUNhQ2FAAFIwABLEYVAA0FAAAAAAAAAAA=', + 'love' : 'JgBQAAABL5AUEhQ1FREUERQSFBEUERQSFBEUNhQ1FDYUNhQ2FDUVNRQSFDUUNhQSFBEUERQSFBEUNhQRFBIUNRQ2FDYUNhQ1FAAFJAABK0cVAA0FAAAAAAAAAAA=', + + # colour commands + 'red' : 'JgBIAAABK5AUERQ2FBEUEhQRFBEVERQRFBEUNhQ2FDYUNRU1FDYUNhQRFDYUNhQ1FREUERQSFBEUNhQRFBIUERQ2FDUVNRQ2FAANBQ==', + 'yellow' : 'JgBIAAABLI8UEhQ2FBEUERQSFBEUEhMSFBEUNhQ2FDUUNhQ2FDYUNRQ2FDYUNhQRFBIUERQRFBIUERQSExIUNhQ1FDYUNhQ2FAANBQ==', + 'green' : 'JgBIAAABK5AUEhQ1FREUERQSFBEUERQSFBEUNhQ1FDYUNhQ2FDUVNRQSFBEUERQ2FBIUERQRFBIUNRU1FDYUERQ2FDYUNhQ1FQANBQ==', + 'blue' : 'JgBIAAABK5AUERQ2FBEUEhQRFBITEhQRFBEUNhQ2FDYUNRQ2FDYUNhQ2ExIUNhQRFDYUERQSExIUERQ2FBEUNhQSEzYUNhQ2FAANBQ==', + 'saturate' : 'JgBIAAABK5AUERQ2FBEUEhQRFBIUERQRFBEUNhQ2FDYUNRU1FDYUNhQRFDYUERQ2FDYUERQSFBEUNhQRFDYUERQSFDUUNhQ2FAANBQ==', + 'desaturate' : 'JgBIAAABLI8VERQ2FBEUERQSFBEUERURFBEUNhQ1FTUUNhQ2FDYUNRQ2FDYUNhQ1FREUERQSFBEUERQSFBEUERQ2FDYUNhQ1FQANBQ==', +} + +def cmd(name): + return base64.b64decode(commands[name]) + +class LightController: + def __init__(self): + self.devices = broadlink.discover(timeout=10, max_devices=2) + if self.devices == []: + raise Exception('no devices found') + for device in self.devices: + device.auth() + + def send_cmd(self, name, iterations=5): + "Send a command, repeatedly for reliability" + packet = cmd(name) + for i in range(iterations): + for device in self.devices: + device.send_data(packet) + + def wakey(self): + "Turn on the lights in the morning, try repeatedly for reasons." + print('Turning on the lights. Wakey, wakey!') + for i in range(5): + self.send_cmd('random') + time.sleep(0.3) + self.send_cmd('on') + time.sleep(0.3) diff --git a/fun/idual/idualctl b/fun/idual/idualctl new file mode 100644 index 000000000000..10a85eba0af0 --- /dev/null +++ b/fun/idual/idualctl @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +import idual +import sys + +def help(): + print('Available commands:') + for cmd in idual.commands: + print('- ' + cmd) + sys.exit(0) + +def handle(ctrl, cmd): + if cmd == 'help': + help() + elif cmd == 'wakey': + ctrl.wakey() + sys.exit(0) + elif cmd == 'on': + print('Turning on the lights') + ctrl.send_cmd(cmd) + elif cmd == 'off': + print('Turning off the lights') + ctrl.send_cmd(cmd) + elif cmd in idual.commands: + print('Sending ' + cmd + '-command') + ctrl.send_cmd(cmd) + else: + print('unknown command \'' + cmd + '\'') + sys.exit(1) + +if __name__ == "__main__": + if len(sys.argv) == 1: + help() + + print('Initialising light controller') + ctrl = idual.LightController() + + for cmd in sys.argv[1:]: + handle(ctrl, cmd) diff --git a/fun/idual/setup.py b/fun/idual/setup.py new file mode 100644 index 000000000000..f10c3b86f24e --- /dev/null +++ b/fun/idual/setup.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from setuptools import setup + +setup( + name='idualctl', + version='0.1', + author='Vincent Ambo', + author_email='mail@tazj.in', + url='https://git.tazj.in/about/fun/idual', + packages=['idual'], + scripts = ['idualctl'], + install_requires=['broadlink>=0.13.2'], + include_package_data=True, +) diff --git a/fun/owothia/.envrc b/fun/owothia/.envrc new file mode 100644 index 000000000000..051d09d292a8 --- /dev/null +++ b/fun/owothia/.envrc @@ -0,0 +1 @@ +eval "$(lorri direnv)" diff --git a/fun/owothia/.gitignore b/fun/owothia/.gitignore new file mode 100644 index 000000000000..8e850e7a0af2 --- /dev/null +++ b/fun/owothia/.gitignore @@ -0,0 +1,30 @@ +dist +dist-* +build/ +cabal-dev +*.o +*.hi +*.hie +*.chi +*.chs.h +*.dyn_o +*.dyn_hi +.hpc +.hsenv +.cabal-sandbox/ +cabal.sandbox.config +*.prof +*.aux +*.hp +*.eventlog +.stack-work/ +cabal.project.local +cabal.project.local~ +.HTF/ +.ghc.environment.* + +# from nix-build +result + +# grr +*_flymake.hs diff --git a/fun/owothia/chatter.patch b/fun/owothia/chatter.patch new file mode 100644 index 000000000000..c2a6179bfbbf --- /dev/null +++ b/fun/owothia/chatter.patch @@ -0,0 +1,19 @@ +diff --git a/src/NLP/POS/LiteralTagger.hs b/src/NLP/POS/LiteralTagger.hs +index 913bee8..3c2f71d 100644 +--- a/src/NLP/POS/LiteralTagger.hs ++++ b/src/NLP/POS/LiteralTagger.hs +@@ -1,4 +1,4 @@ +-{-# LANGUAGE OverloadedStrings #-} ++{-# LANGUAGE OverloadedStrings, PackageImports #-} + module NLP.POS.LiteralTagger + ( tag + , tagSentence +@@ -27,7 +27,7 @@ import NLP.FullStop (segment) + import NLP.Types ( tagUNK, Sentence, TaggedSentence(..), applyTags + , Tag, POSTagger(..), CaseSensitive(..), tokens, showTok) + import Text.Regex.TDFA +-import Text.Regex.TDFA.Text (compile) ++import "regex-tdfa" Text.Regex.TDFA.Text (compile) + + taggerID :: ByteString + taggerID = pack "NLP.POS.LiteralTagger" diff --git a/fun/owothia/default.nix b/fun/owothia/default.nix new file mode 100644 index 000000000000..b70d0525c152 --- /dev/null +++ b/fun/owothia/default.nix @@ -0,0 +1,12 @@ +{ depot ? (import ../../../. {}) +, pkgs ? depot.third_party.nixpkgs +, ... }: + +let + basePkg = pkgs.haskellPackages.callPackage ./pkg.nix { }; +in + +pkgs.haskell.lib.overrideSrc basePkg { + src = depot.third_party.gitignoreSource ./.; + version = "canon"; +} diff --git a/fun/owothia/hie.yaml b/fun/owothia/hie.yaml new file mode 100644 index 000000000000..16a6c1526259 --- /dev/null +++ b/fun/owothia/hie.yaml @@ -0,0 +1,4 @@ +cradle: + cabal: + - path: './app' + component: 'exe:owothia' diff --git a/fun/owothia/owothia.cabal b/fun/owothia/owothia.cabal new file mode 100644 index 000000000000..ef5477ea1bf2 --- /dev/null +++ b/fun/owothia/owothia.cabal @@ -0,0 +1,53 @@ +cabal-version: 2.2 +name: owothia +version: 0.0.1.0 + +executable owothia + main-is: Main.hs + build-depends: base + , relude + , irc-client + , lens + , chatter + , containers + , text + , bytestring + , random + , envy + + mixins: base hiding (Prelude) + , relude (Relude as Prelude) + + hs-source-dirs: + src + + default-extensions: + BlockArguments + ConstraintKinds + DataKinds + DeriveAnyClass + DeriveGeneric + DerivingStrategies + DerivingVia + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTSyntax + GeneralizedNewtypeDeriving + KindSignatures + LambdaCase + MultiWayIf + NoStarIsType + OverloadedStrings + PolyKinds + RankNTypes + ScopedTypeVariables + TupleSections + TypeApplications + TypeFamilies + TypeOperators + ViewPatterns + + ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -O2 + + default-language: Haskell2010 diff --git a/fun/owothia/pkg.nix b/fun/owothia/pkg.nix new file mode 100644 index 000000000000..d0941a848958 --- /dev/null +++ b/fun/owothia/pkg.nix @@ -0,0 +1,16 @@ +{ mkDerivation, base, bytestring, chatter, containers, envy +, irc-client, lens, lib, random, relude, text +}: +mkDerivation { + pname = "owothia"; + version = "0.0.1.0"; + src = ./.; + isLibrary = false; + isExecutable = true; + executableHaskellDepends = [ + base bytestring chatter containers envy irc-client lens random + relude text + ]; + license = "unknown"; + hydraPlatforms = lib.platforms.none; +} diff --git a/fun/owothia/regex-tdfa-text.patch b/fun/owothia/regex-tdfa-text.patch new file mode 100644 index 000000000000..6b2c34654382 --- /dev/null +++ b/fun/owothia/regex-tdfa-text.patch @@ -0,0 +1,40 @@ +diff --git a/Text/Regex/TDFA/Text.hs b/Text/Regex/TDFA/Text.hs +index c4ef9db..9299272 100644 +--- a/Text/Regex/TDFA/Text.hs ++++ b/Text/Regex/TDFA/Text.hs +@@ -38,13 +38,6 @@ import Text.Regex.TDFA.NewDFA.Uncons(Uncons(uncons)) + import qualified Text.Regex.TDFA.NewDFA.Engine as Engine(execMatch) + import qualified Text.Regex.TDFA.NewDFA.Tester as Tester(matchTest) + +-instance Extract T.Text where +- before = T.take; after = T.drop; empty = T.empty +- +-instance Uncons T.Text where +- {- INLINE uncons #-} +- uncons = T.uncons +- + instance RegexContext Regex T.Text T.Text where + match = polymatch + matchM = polymatchM +diff --git a/Text/Regex/TDFA/Text/Lazy.hs b/Text/Regex/TDFA/Text/Lazy.hs +index 73ca4a0..52958fb 100644 +--- a/Text/Regex/TDFA/Text/Lazy.hs ++++ b/Text/Regex/TDFA/Text/Lazy.hs +@@ -38,17 +38,10 @@ import Text.Regex.TDFA.NewDFA.Uncons(Uncons(uncons)) + import qualified Text.Regex.TDFA.NewDFA.Engine as Engine(execMatch) + import qualified Text.Regex.TDFA.NewDFA.Tester as Tester(matchTest) + +-instance Extract L.Text where +- before = L.take . toEnum; after = L.drop . toEnum; empty = L.empty +- + instance RegexContext Regex L.Text L.Text where + match = polymatch + matchM = polymatchM + +-instance Uncons L.Text where +- {- INLINE uncons #-} +- uncons = L.uncons +- + instance RegexMaker Regex CompOption ExecOption L.Text where + makeRegexOptsM c e source = makeRegexOptsM c e (L.unpack source) + diff --git a/fun/owothia/shell.nix b/fun/owothia/shell.nix new file mode 100644 index 000000000000..1ad70c907baa --- /dev/null +++ b/fun/owothia/shell.nix @@ -0,0 +1,22 @@ +{ pkgs ? (import ../../../. {}).third_party, ... }: + +let + inherit (pkgs) + haskellPackages + haskell + gitignoreSource + ; +in + +(haskellPackages.extend (haskell.lib.packageSourceOverrides { + owothia = gitignoreSource ./.; +})).shellFor { + packages = p: [ p.owothia ]; + withHoogle = true; + doBenchmark = true; + buildInputs = with haskellPackages; [ + cabal-install + hlint + haskell-language-server + ]; +} diff --git a/fun/owothia/src/Main.hs b/fun/owothia/src/Main.hs new file mode 100644 index 000000000000..3bf5e51dba2f --- /dev/null +++ b/fun/owothia/src/Main.hs @@ -0,0 +1,168 @@ +{-# LANGUAGE TemplateHaskell #-} +module Main where + +import Network.IRC.Client +import Control.Lens +import NLP.POS +import NLP.Types (POSTagger) +import qualified NLP.Types.Tags as Tags +import NLP.Types.Tree +import qualified NLP.Corpora.Conll as Conll +import NLP.Corpora.Conll (Tag) +import qualified Data.ByteString as BS +import System.Random +import System.Envy +import System.IO as S +import Data.Maybe +import Data.Foldable (traverse_) +import qualified Data.Text +-------------------------------------------------------------------------------- + +data Config = Config + { _owoChance :: Int + , _ircServer :: ByteString + , _ircPort :: Int + , _ircServerPassword :: Maybe Text + , _nickservPassword :: Maybe Text + , _ircNick :: Maybe Text + , _ircIdent :: Maybe Text + , _ircChannels :: [Text] + } + deriving stock (Show, Eq, Generic) +makeLenses ''Config + +instance Var [Text] where + toVar ts = show ts + fromVar s = readMaybe s >>= (pure . map Data.Text.pack) + +instance FromEnv Config where + fromEnv _ = + Config <$> env "OWO_CHANCE" + <*> env "IRC_SERVER" + <*> env "IRC_PORT" + <*> envMaybe "IRC_SERVER_PASSWORD" + <*> envMaybe "NICKSERV_PASSWORD" + <*> envMaybe "IRC_NICK" + <*> envMaybe "IRC_IDENT" + <*> env "IRC_CHANNELS" + +stopWord :: Text -> Bool +stopWord "'s" = True +stopWord "\"" = True +stopWord "is" = True +stopWord "are" = True +stopWord "am" = True +stopWord "were" = True +stopWord "was" = True +stopWord "be" = True +stopWord _ = False + +pickVerb :: POS Tag -> Maybe Text +pickVerb (POS Conll.VB (Token verb)) = Just verb +pickVerb (POS Conll.VBD (Token verb)) = Just verb +pickVerb (POS Conll.VBG (Token verb)) = Just verb +pickVerb (POS Conll.VBN (Token verb)) = Just verb +pickVerb (POS Conll.VBZ (Token verb)) = Just verb +pickVerb _ = Nothing + +pickNoun :: POS Tag -> Maybe Text +pickNoun (POS Conll.NN (Token noun)) = Just noun +pickNoun _ = Nothing + +randomPOS + :: Tags.Tag tag + => (POS tag -> Maybe Text) + -> POSTagger tag + -> Text + -> IO (Maybe Text) +randomPOS pickPOS tagger s = do + let candidates + = filter (not . stopWord) + . mapMaybe pickPOS + $ tag tagger s >>= \(TaggedSent ps) -> ps + i <- randomRIO (0, length candidates - 1) + pure $ candidates ^? ix i + +doOwo :: MonadIO m => Config -> m Bool +doOwo conf = do + n <- liftIO (randomRIO @Int (0, conf ^. owoChance)) + pure $ n == 0 + +data OwoType = Noun | Verb + deriving stock (Show, Eq) + +instance Random OwoType where + random = over _1 (bool Noun Verb) . random + randomR = const random + +vowels :: [Char] +vowels = "aeiou" + +article :: Text -> Text +article (x :< _) | x `elem` vowels = "an" +article _ = "a" + +owo :: OwoType -> Text -> Text +owo Noun n = mconcat + [ "I'm " + , article n + , " " + , n + , if "o" `Data.Text.isSuffixOf` n + then "wo" + else " owo" + ] +owo Verb v = v <> " me owo" + +pickOwo :: OwoType -> POS Tag -> Maybe Text +pickOwo Verb = pickVerb +pickOwo Noun = pickNoun + +randomOwo :: OwoType -> POSTagger Tag -> Text -> IO (Maybe Text) +randomOwo = randomPOS . pickOwo + +owothiaHandler :: Config -> Text -> IORef Bool -> POSTagger Tag -> EventHandler s +owothiaHandler conf nick state tagger = EventHandler Just $ \src ev -> do + hasIdentified <- readIORef state + when (not hasIdentified) $ do + nickservAuth + traverse_ (send . Join) (conf ^. ircChannels) + writeIORef state True + + when ("You are now identified" `BS.isInfixOf` (ev ^. raw)) $ + traverse_ (send . Join) (conf ^. ircChannels) + + case (src, ev ^. message) of + (Channel chan nick, Privmsg _ (Right m)) -> do + willOwo <- doOwo conf + when willOwo $ owoMessage chan m + _ -> pure() + + pure () + + where + owoMessage chan m = do + owoType <- liftIO randomIO + mWord <- liftIO $ randomOwo owoType tagger m + for_ mWord $ \word -> send $ Privmsg chan $ Right $ owo owoType word + nickservAuthMsg = "IDENTIFY " <> nick <> " " <> fromJust (conf ^. nickservPassword) + nickservAuth = send $ Privmsg "NickServ" $ Right nickservAuthMsg + +main :: IO () +main = do + conf <- either fail pure =<< decodeEnv + tagger <- defaultTagger + state <- newIORef $ not . isJust $ (conf ^. nickservPassword) + S.hSetBuffering stdout LineBuffering + let nick = fromMaybe "owothia" (conf ^. ircNick) + conn = + plainConnection (conf ^. ircServer) (conf ^. ircPort) + & realname .~ "Owothia Revströwö" + & password .~ (conf ^. ircServerPassword) + & username .~ fromMaybe "owothia" (conf ^. ircIdent) + & logfunc .~ stdoutLogger + cfg = + defaultInstanceConfig nick + & channels .~ (conf ^. ircChannels) + & handlers %~ (owothiaHandler conf nick state tagger : ) + runClient conn cfg () diff --git a/fun/paroxysm/.gitignore b/fun/paroxysm/.gitignore new file mode 100644 index 000000000000..8cb01f2e3d35 --- /dev/null +++ b/fun/paroxysm/.gitignore @@ -0,0 +1,5 @@ +/target +irc.toml +paroxysm-irc.toml +paroxysm.toml +**/*.rs.bk diff --git a/fun/paroxysm/Cargo.lock b/fun/paroxysm/Cargo.lock new file mode 100644 index 000000000000..c0962dfff693 --- /dev/null +++ b/fun/paroxysm/Cargo.lock @@ -0,0 +1,1619 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bufstream" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "cc" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits 0.2.14", + "time", + "winapi 0.3.9", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "config" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9107d78ed62b3fa5a86e7d18e647abed48cfd8f8fab6c72f4cdb982d196f7e6" +dependencies = [ + "lazy_static 1.4.0", + "nom", + "rust-ini", + "serde 1.0.130", + "serde-hjson", + "serde_json", + "toml", + "yaml-rust", +] + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + +[[package]] +name = "crimp" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe8f9a320ad9c1a2e3bacedaa281587bd297fb10a10179fd39f777049d04794" +dependencies = [ + "curl", + "serde 1.0.130", + "serde_json", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils", + "lazy_static 1.4.0", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static 1.4.0", +] + +[[package]] +name = "curl" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa3b8db7f3341ddef15786d250106334d4a6c4b0ae4a46cd77082777d9849b9" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "curl-sys" +version = "0.4.49+curl-7.79.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f44960aea24a786a46907b8824ebc0e66ca06bf4e4978408c7499620343483" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi 0.3.9", +] + +[[package]] +name = "diesel" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b28135ecf6b7d446b43e27e225622a038cc4e2930a1022f51cdb97ada19b8e4d" +dependencies = [ + "bitflags", + "byteorder", + "chrono", + "diesel_derives", + "pq-sys", + "r2d2", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + +[[package]] +name = "env_logger" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "instant" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "716d3d89f35ac6a34fd0eed635395f4c3b76fa889338a4632e5231a8684216bd" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "irc" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb7666c9ae95dc77b874467e347bde3789773b6f48887fb3384bfe29552b466" +dependencies = [ + "bufstream", + "bytes", + "chrono", + "encoding", + "failure", + "futures", + "log", + "native-tls", + "serde 1.0.130", + "serde_derive", + "tokio-codec", + "tokio-core", + "tokio-io", + "tokio-mockstream", + "tokio-timer 0.1.2", + "tokio-tls", + "toml", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6" + +[[package]] +name = "libz-sys" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linked-hash-map" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" +dependencies = [ + "serde 0.8.23", + "serde_test", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab 0.4.4", + "winapi 0.2.8", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "native-tls" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" +dependencies = [ + "lazy_static 1.4.0", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" +dependencies = [ + "num-traits 0.2.14", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39f37e50073ccad23b6d09bcb5b263f4e76d3bb6038e4a3c08e52162ffa8abc2" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "openssl" +version = "0.10.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d9facdb76fec0b73c406f125d44d86fdad818d66fef0531eec9233ca425ff4a" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69df2d8dfc6ce3aaf44b40dec6f487d5a886516cf6879c49e98e0710f310a058" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api 0.4.5", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version", + "smallvec 0.6.14", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.10", + "smallvec 1.7.0", + "winapi 0.3.9", +] + +[[package]] +name = "paroxysm" +version = "0.1.0" +dependencies = [ + "chrono", + "config", + "crimp", + "diesel", + "env_logger", + "failure", + "irc", + "lazy_static 1.4.0", + "log", + "rand 0.7.3", + "regex", + "serde 1.0.130", +] + +[[package]] +name = "pkg-config" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "pq-sys" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ac25eee5a0582f45a67e837e350d784e7003bd29a5f460796772061ca49ffda" +dependencies = [ + "vcpkg", +] + +[[package]] +name = "proc-macro2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log", + "parking_lot 0.11.2", + "scheduled-thread-pool", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom 0.2.3", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core 0.6.3", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "rust-ini" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static 1.4.0", + "winapi 0.3.9", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot 0.11.2", +] + +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde-hjson" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b833c5ad67d52ced5f5938b2980f32a9c1c5ef047f0b4fb3127e7a423c76153" +dependencies = [ + "lazy_static 0.2.11", + "linked-hash-map 0.3.0", + "num-traits 0.1.43", + "regex", + "serde 0.8.23", +] + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +dependencies = [ + "itoa", + "ryu", + "serde 1.0.130", +] + +[[package]] +name = "serde_test" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" +dependencies = [ + "serde 0.8.23", +] + +[[package]] +name = "slab" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" + +[[package]] +name = "slab" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" + +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "syn" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand 0.8.4", + "redox_syscall 0.2.10", + "remove_dir_all", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes", + "futures", + "mio", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer 0.2.13", + "tokio-udp", + "tokio-uds", +] + +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes", + "futures", + "tokio-io", +] + +[[package]] +name = "tokio-core" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87b1395334443abca552f63d4f61d0486f12377c2ba8b368e523f89e828cffd4" +dependencies = [ + "bytes", + "futures", + "iovec", + "log", + "mio", + "scoped-tls", + "tokio", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-timer 0.2.13", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils", + "futures", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes", + "futures", + "log", +] + +[[package]] +name = "tokio-mockstream" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41bfc436ef8b7f60c19adf3df086330ae9992385e4d8c53b17a323cad288e155" +dependencies = [ + "futures", + "tokio-io", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils", + "futures", + "lazy_static 1.4.0", + "log", + "mio", + "num_cpus", + "parking_lot 0.9.0", + "slab 0.4.4", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes", + "futures", + "iovec", + "mio", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "futures", + "lazy_static 1.4.0", + "log", + "num_cpus", + "slab 0.4.4", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6131e780037787ff1b3f8aad9da83bca02438b72277850dd6ad0d455e0e20efc" +dependencies = [ + "futures", + "slab 0.3.0", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils", + "futures", + "slab 0.4.4", + "tokio-executor", +] + +[[package]] +name = "tokio-tls" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "354b8cd83825b3c20217a9dc174d6a0c67441a2fae5c41bcb1ea6679f6ae0f7c" +dependencies = [ + "futures", + "native-tls", + "tokio-io", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes", + "futures", + "log", + "mio", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes", + "futures", + "iovec", + "libc", + "log", + "mio", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +dependencies = [ + "serde 1.0.130", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map 0.5.4", +] diff --git a/fun/paroxysm/Cargo.toml b/fun/paroxysm/Cargo.toml new file mode 100644 index 000000000000..4d282285fd97 --- /dev/null +++ b/fun/paroxysm/Cargo.toml @@ -0,0 +1,22 @@ +[package] +authors = ["eeeeeta <eta@theta.eu.org>"] +edition = "2018" +name = "paroxysm" +version = "0.1.0" + +[dependencies] +chrono = "0.4" +config = "0.9" +crimp = "0.2" +env_logger = "0.7" +failure = "0.1" +irc = "0.13" +lazy_static = "1.4" +log = "0.4" +rand = "0.7" +regex = "1.3" +serde = { version = "1.0", features = [ "derive" ] } + +[dependencies.diesel] +features = [ "postgres", "chrono", "r2d2" ] +version = "1.4" diff --git a/fun/paroxysm/OWNERS b/fun/paroxysm/OWNERS new file mode 100644 index 000000000000..7f8beb1aa7d7 --- /dev/null +++ b/fun/paroxysm/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - eta diff --git a/fun/paroxysm/README.md b/fun/paroxysm/README.md new file mode 100644 index 000000000000..a595530982da --- /dev/null +++ b/fun/paroxysm/README.md @@ -0,0 +1,19 @@ +paroxysm +======== + +`paroxysm` is a bot for [internet relay chat +(IRC)](https://en.wikipedia.org/wiki/Internet_Relay_Chat) that lets you store +small pieces of information, called *factoids*, and retrieve them later. It's +useful for organising frequently-used information to avoid repeating oneself in +a busy chatroom, as well as making little todo lists or notes to self in a +private chatroom. + +It was directly inspired by the +[LearnDB](https://github.com/crawl/sequell/blob/master/docs/learndb.md) +functionality offered in `##crawl` on chat.freenode.net, and uses similar +syntax. + +## Usage instructions + +Will come soon; the project is very much still in beta, and is subject to +change. diff --git a/fun/paroxysm/default.nix b/fun/paroxysm/default.nix new file mode 100644 index 000000000000..e4ce4df1ae38 --- /dev/null +++ b/fun/paroxysm/default.nix @@ -0,0 +1,14 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + name = "paroxysm"; + version = "0.0.2"; + src = ./.; + + buildInputs = with pkgs; [ + openssl + pkgconfig + postgresql.lib + curl + ]; +} diff --git a/fun/paroxysm/docker/default.nix b/fun/paroxysm/docker/default.nix new file mode 100644 index 000000000000..cb5b2758ec63 --- /dev/null +++ b/fun/paroxysm/docker/default.nix @@ -0,0 +1,7 @@ +{ depot, pkgs, ... }: + +pkgs.dockerTools.buildLayeredImage { + name = "paroxysm"; + contents = [ depot.fun.paroxysm ]; + config.Entrypoint = [ "${depot.fun.paroxysm}/bin/paroxysm" ]; +} diff --git a/fun/paroxysm/migrations/20181209140247_initial/down.sql b/fun/paroxysm/migrations/20181209140247_initial/down.sql new file mode 100644 index 000000000000..aa02f4f63f92 --- /dev/null +++ b/fun/paroxysm/migrations/20181209140247_initial/down.sql @@ -0,0 +1,2 @@ +DROP TABLE entries; +DROP TABLE keywords; diff --git a/fun/paroxysm/migrations/20181209140247_initial/up.sql b/fun/paroxysm/migrations/20181209140247_initial/up.sql new file mode 100644 index 000000000000..e8b52d5a9b91 --- /dev/null +++ b/fun/paroxysm/migrations/20181209140247_initial/up.sql @@ -0,0 +1,15 @@ +CREATE TABLE keywords ( + id SERIAL PRIMARY KEY, + name VARCHAR UNIQUE NOT NULL, + chan VARCHAR NOT NULL, + UNIQUE(name, chan) +); + +CREATE TABLE entries ( + id SERIAL PRIMARY KEY, + keyword_id INT NOT NULL REFERENCES keywords ON DELETE CASCADE, + idx INT NOT NULL, + text VARCHAR NOT NULL, + creation_ts TIMESTAMP NOT NULL, + created_by VARCHAR NOT NULL +); diff --git a/fun/paroxysm/migrations/20181218142013_fix_unique/down.sql b/fun/paroxysm/migrations/20181218142013_fix_unique/down.sql new file mode 100644 index 000000000000..291a97c5ce1f --- /dev/null +++ b/fun/paroxysm/migrations/20181218142013_fix_unique/down.sql @@ -0,0 +1 @@ +-- This file should undo anything in `up.sql` \ No newline at end of file diff --git a/fun/paroxysm/migrations/20181218142013_fix_unique/up.sql b/fun/paroxysm/migrations/20181218142013_fix_unique/up.sql new file mode 100644 index 000000000000..4885ae5edefe --- /dev/null +++ b/fun/paroxysm/migrations/20181218142013_fix_unique/up.sql @@ -0,0 +1 @@ +ALTER TABLE keywords DROP CONSTRAINT IF EXISTS keywords_name_key; diff --git a/fun/paroxysm/src/cfg.rs b/fun/paroxysm/src/cfg.rs new file mode 100644 index 000000000000..cfb2e2073e27 --- /dev/null +++ b/fun/paroxysm/src/cfg.rs @@ -0,0 +1,12 @@ +use serde::Deserialize; +use std::collections::HashSet; + +#[derive(Deserialize)] +pub struct Config { + pub database_url: String, + pub irc_config_path: String, + #[serde(default)] + pub admins: HashSet<String>, + #[serde(default)] + pub log_filter: Option<String>, +} diff --git a/fun/paroxysm/src/keyword.rs b/fun/paroxysm/src/keyword.rs new file mode 100644 index 000000000000..1b2b6ce592a0 --- /dev/null +++ b/fun/paroxysm/src/keyword.rs @@ -0,0 +1,220 @@ +use crate::models::{Entry, Keyword, NewEntry, NewKeyword}; +use diesel::pg::PgConnection; +use diesel::prelude::*; +use failure::format_err; +use failure::Error; +use std::borrow::Cow; + +/// Maximum number of times we'll follow a `see: ` pointer. +const RECURSION_LIMIT: usize = 5; + +pub struct KeywordDetails { + pub keyword: Keyword, + pub entries: Vec<Entry>, +} + +impl KeywordDetails { + pub fn learn(&mut self, nick: &str, text: &str, dbc: &PgConnection) -> Result<usize, Error> { + let now = ::chrono::Utc::now().naive_utc(); + let ins = NewEntry { + keyword_id: self.keyword.id, + idx: (self.entries.len() + 1) as _, + text, + creation_ts: now, + created_by: nick, + }; + let new = { + use crate::schema::entries; + ::diesel::insert_into(entries::table) + .values(ins) + .get_result(dbc)? + }; + self.entries.push(new); + Ok(self.entries.len()) + } + + pub fn process_moves(&mut self, moves: &[(i32, i32)], dbc: &PgConnection) -> Result<(), Error> { + for (oid, new_idx) in moves { + { + use crate::schema::entries::dsl::*; + ::diesel::update(entries.filter(id.eq(oid))) + .set(idx.eq(new_idx)) + .execute(dbc)?; + } + } + self.entries = Self::get_entries(self.keyword.id, dbc)?; + Ok(()) + } + + pub fn swap(&mut self, idx_a: usize, idx_b: usize, dbc: &PgConnection) -> Result<(), Error> { + let mut moves = vec![]; + for ent in self.entries.iter() { + if ent.idx == idx_a as i32 { + moves.push((ent.id, idx_b as i32)); + } + if ent.idx == idx_b as i32 { + moves.push((ent.id, idx_a as i32)); + } + } + if moves.len() != 2 { + Err(format_err!("Invalid swap operation."))?; + } + self.process_moves(&moves, dbc)?; + Ok(()) + } + + pub fn update(&mut self, idx: usize, val: &str, dbc: &PgConnection) -> Result<(), Error> { + let ent = self + .entries + .get_mut(idx.saturating_sub(1)) + .ok_or(format_err!("No such element to update."))?; + { + use crate::schema::entries::dsl::*; + ::diesel::update(entries.filter(id.eq(ent.id))) + .set(text.eq(val)) + .execute(dbc)?; + } + ent.text = val.to_string(); + Ok(()) + } + + pub fn delete(&mut self, idx: usize, dbc: &PgConnection) -> Result<(), Error> { + // step 1: delete the element + { + let ent = self + .entries + .get(idx.saturating_sub(1)) + .ok_or(format_err!("No such element to delete."))?; + { + use crate::schema::entries::dsl::*; + ::diesel::delete(entries.filter(id.eq(ent.id))).execute(dbc)?; + } + } + // step 2: move all the elements in front of it back one + let mut moves = vec![]; + for ent in self.entries.iter() { + if idx > ent.idx as _ { + moves.push((ent.id, ent.idx.saturating_sub(1))); + } + } + self.process_moves(&moves, dbc)?; + Ok(()) + } + + pub fn add_zwsp_to_name(name: &str) -> Option<String> { + let second_index = name.char_indices().nth(1).map(|(i, _)| i)?; + let (start, end) = name.split_at(second_index); + Some(format!("{}{}", start, end)) + } + + pub fn format_entry(&self, idx: usize) -> Option<String> { + self.format_entry_colours(idx, true) + } + + pub fn format_entry_colours(&self, idx: usize, with_colours: bool) -> Option<String> { + if let Some(ent) = self.entries.get(idx.saturating_sub(1)) { + let gen_clr = if self.keyword.chan == "*" && with_colours { + "\x0307" + } else { + "" + }; + let zwsp_name = Self::add_zwsp_to_name(&self.keyword.name) + .unwrap_or_else(|| self.keyword.name.clone()); + Some(format!( + "{}{}{name}{}[{idx}/{total}]{}: {text} {}[{date}]{}", + if with_colours { "\x02" } else { "" }, + gen_clr, + if with_colours { "\x0f\x0315" } else { "" }, + if with_colours { "\x0f" } else { "" }, + if with_colours { "\x0f\x0314" } else { "" }, + if with_colours { "\x0f" } else { "" }, + name = zwsp_name, + idx = idx, + total = self.entries.len(), + text = ent.text, + date = ent.creation_ts.date() + )) + } else { + None + } + } + + pub fn get_or_create(word: &str, c: &str, dbc: &PgConnection) -> Result<Self, Error> { + if let Some(ret) = Self::get(word, c, dbc)? { + Ok(ret) + } else { + Ok(Self::create(word, c, dbc)?) + } + } + + pub fn create(word: &str, c: &str, dbc: &PgConnection) -> Result<Self, Error> { + let val = NewKeyword { + name: word, + chan: c, + }; + let ret: Keyword = { + use crate::schema::keywords; + ::diesel::insert_into(keywords::table) + .values(val) + .get_result(dbc)? + }; + Ok(KeywordDetails { + keyword: ret, + entries: vec![], + }) + } + + fn get_entries(kid: i32, dbc: &PgConnection) -> Result<Vec<Entry>, Error> { + let entries: Vec<Entry> = { + use crate::schema::entries::dsl::*; + entries + .filter(keyword_id.eq(kid)) + .order_by(idx.asc()) + .load(dbc)? + }; + Ok(entries) + } + + fn get_inner<'a, T: Into<Cow<'a, str>>>( + word: T, + c: &str, + dbc: &PgConnection, + recursion_count: usize, + ) -> Result<Option<Self>, Error> { + let word = word.into(); + let keyword: Option<Keyword> = { + use crate::schema::keywords::dsl::*; + keywords + .filter(name.ilike(word).and(chan.eq(c).or(chan.eq("*")))) + .first(dbc) + .optional()? + }; + if let Some(k) = keyword { + let entries = Self::get_entries(k.id, dbc)?; + if let Some(e0) = entries.get(0) { + if e0.text.starts_with("see: ") { + if recursion_count > RECURSION_LIMIT { + // Oh dear. + Err(format_err!("Halt. You're having a bit too much fun."))? + } + let new_word = e0.text.replace("see: ", ""); + return Self::get_inner(new_word, c, dbc, recursion_count + 1); + } + } + Ok(Some(KeywordDetails { + keyword: k, + entries, + })) + } else { + Ok(None) + } + } + + pub fn get<'a, T: Into<Cow<'a, str>>>( + word: T, + c: &str, + dbc: &PgConnection, + ) -> Result<Option<Self>, Error> { + Self::get_inner(word, c, dbc, 0) + } +} diff --git a/fun/paroxysm/src/main.rs b/fun/paroxysm/src/main.rs new file mode 100644 index 000000000000..11a0e7bf60b5 --- /dev/null +++ b/fun/paroxysm/src/main.rs @@ -0,0 +1,395 @@ +// TODO(tazjin): Upgrade to a Diesel version with public derive +// macros. +#[macro_use] +extern crate diesel; + +use crate::cfg::Config; +use crate::keyword::KeywordDetails; +use diesel::pg::PgConnection; +use diesel::r2d2::{ConnectionManager, Pool}; +use failure::format_err; +use failure::Error; +use irc::client::prelude::*; +use lazy_static::lazy_static; +use log::{debug, info, warn}; +use rand::rngs::ThreadRng; +use rand::{thread_rng, Rng}; +use regex::{Captures, Regex}; +use std::collections::HashMap; +use std::fmt::Display; + +mod cfg; +mod keyword; +mod models; +mod schema; + +lazy_static! { + static ref LEARN_RE: Regex = + Regex::new(r#"^\?\?(?P<gen>!)?\s*(?P<subj>[^\[:]*):\s*(?P<val>.*)"#).unwrap(); + static ref QUERY_RE: Regex = + Regex::new(r#"^\?\?\s*(?P<subj>[^\[:]*)(?P<idx>\[[^\]]+\])?"#).unwrap(); + static ref QLAST_RE: Regex = Regex::new(r#"^\?\?\s*(?P<subj>[^\[:]*)!"#).unwrap(); + static ref INCREMENT_RE: Regex = + Regex::new(r#"^\?\?(?P<gen>!)?\s*(?P<subj>[^\[:]*)(?P<incrdecr>\+\+|\-\-)"#).unwrap(); + static ref MOVE_RE: Regex = + Regex::new(r#"^\?\?(?P<gen>!)?\s*(?P<subj>[^\[:]*)(?P<idx>\[[^\]]+\])->(?P<new_idx>.*)"#) + .unwrap(); +} + +pub struct App { + client: IrcClient, + pg: Pool<ConnectionManager<PgConnection>>, + rng: ThreadRng, + cfg: Config, + last_msgs: HashMap<String, HashMap<String, String>>, +} + +impl App { + pub fn report_error<T: Display>( + &mut self, + nick: &str, + chan: &str, + msg: T, + ) -> Result<(), Error> { + self.client + .send_notice(nick, format!("[{}] \x0304Error:\x0f {}", chan, msg))?; + Ok(()) + } + + pub fn keyword_from_captures( + &mut self, + learn: &::regex::Captures, + nick: &str, + chan: &str, + ) -> Result<KeywordDetails, Error> { + let db = self.pg.get()?; + debug!("Fetching keyword for captures: {:?}", learn); + let subj = &learn["subj"]; + let learn_chan = if learn.name("gen").is_some() { + "*" + } else { + chan + }; + if !chan.starts_with("#") && learn_chan != "*" { + Err(format_err!("Only general entries may be taught via PM."))?; + } + debug!("Fetching keyword '{}' for chan {}", subj, learn_chan); + let kwd = KeywordDetails::get_or_create(subj, learn_chan, &db)?; + if kwd.keyword.chan == "*" && !self.cfg.admins.contains(nick) { + Err(format_err!( + "Only administrators can create or modify general entries." + ))?; + } + Ok(kwd) + } + + pub fn handle_move( + &mut self, + target: &str, + nick: &str, + chan: &str, + mv: Captures, + ) -> Result<(), Error> { + let db = self.pg.get()?; + let idx = &mv["idx"]; + let idx = match idx[1..(idx.len() - 1)].parse::<usize>() { + Ok(i) => i, + Err(e) => Err(format_err!("Could not parse index: {}", e))?, + }; + let new_idx = match mv["new_idx"].parse::<i32>() { + Ok(i) => i, + Err(e) => Err(format_err!("Could not parse target index: {}", e))?, + }; + let mut kwd = self.keyword_from_captures(&mv, nick, chan)?; + if new_idx < 0 { + kwd.delete(idx, &db)?; + self.client.send_notice( + target, + format!("\x02{}\x0f: Deleted entry {}.", kwd.keyword.name, idx), + )?; + } else { + kwd.swap(idx, new_idx as _, &db)?; + self.client.send_notice( + target, + format!( + "\x02{}\x0f: Swapped entries {} and {}.", + kwd.keyword.name, idx, new_idx + ), + )?; + } + Ok(()) + } + + pub fn handle_learn( + &mut self, + target: &str, + nick: &str, + chan: &str, + learn: Captures, + ) -> Result<(), Error> { + let db = self.pg.get()?; + let val = &learn["val"]; + let mut kwd = self.keyword_from_captures(&learn, nick, chan)?; + let idx = kwd.learn(nick, val, &db)?; + self.client + .send_notice(target, kwd.format_entry(idx).unwrap())?; + Ok(()) + } + + pub fn handle_insert_last_quote( + &mut self, + target: &str, + nick: &str, + chan: &str, + qlast: Captures, + ) -> Result<(), Error> { + let db = self.pg.get()?; + let nick_to_grab = &qlast["subj"].to_ascii_lowercase(); + let mut kwd = self.keyword_from_captures(&qlast, nick, chan)?; + let chan_lastmsgs = self + .last_msgs + .entry(chan.to_string()) + .or_insert(HashMap::new()); + // Use `nick` here, so things like "grfn: see glittershark" work. + let val = if let Some(last) = chan_lastmsgs.get(nick_to_grab) { + if last.starts_with("\x01ACTION ") { + // Yes, this is inefficient, but it's better than writing some hacky CTCP parsing code + // I guess (also, characters are hard, so just blindly slicing seems like a bad idea) + format!( + "* {} {}", + nick_to_grab, + last.replace("\x01ACTION ", "").replace("\x01", "") + ) + } else { + format!("<{}> {}", nick_to_grab, last) + } + } else { + Err(format_err!("I dunno what {} said...", kwd.keyword.name))? + }; + let idx = kwd.learn(nick, &val, &db)?; + self.client + .send_notice(target, kwd.format_entry(idx).unwrap())?; + Ok(()) + } + + pub fn handle_increment( + &mut self, + target: &str, + nick: &str, + chan: &str, + icr: Captures, + ) -> Result<(), Error> { + let db = self.pg.get()?; + let mut kwd = self.keyword_from_captures(&icr, nick, chan)?; + let is_incr = &icr["incrdecr"] == "++"; + let now = chrono::Utc::now().naive_utc().date(); + let mut idx = None; + for (i, ent) in kwd.entries.iter().enumerate() { + if ent.creation_ts.date() == now { + if let Ok(val) = ent.text.parse::<i32>() { + let val = if is_incr { val + 1 } else { val - 1 }; + idx = Some((i + 1, val)); + } + } + } + if let Some((i, val)) = idx { + kwd.update(i, &val.to_string(), &db)?; + self.client + .send_notice(target, kwd.format_entry(i).unwrap())?; + } else { + let val = if is_incr { 1 } else { -1 }; + let idx = kwd.learn(nick, &val.to_string(), &db)?; + self.client + .send_notice(target, kwd.format_entry(idx).unwrap())?; + } + Ok(()) + } + + pub fn handle_query( + &mut self, + target: &str, + nick: &str, + chan: &str, + query: Captures, + ) -> Result<(), Error> { + let db = self.pg.get()?; + let subj = &query["subj"]; + let idx = match query.name("idx") { + Some(i) => { + let i = i.as_str(); + match &i[1..(i.len() - 1)] { + "*" => Some(-1), + x => x.parse::<usize>().map(|x| x as i32).ok(), + } + } + None => None, + }; + debug!("Querying {} with idx {:?}", subj, idx); + match KeywordDetails::get(subj, chan, &db)? { + Some(kwd) => { + if let Some(mut idx) = idx { + if idx == -1 { + // 'get all entries' ('*' parses into this) + // step 1: make a blob of all the quotes + let mut data_to_upload = String::new(); + for i in 0..kwd.entries.len() { + data_to_upload + .push_str(&kwd.format_entry_colours(i + 1, false).unwrap()); + data_to_upload.push('\n'); + } + // step 2: attempt to POST it to eta's pastebin + // TODO(eta): make configurable + let response = crimp::Request::put("https://eta.st/lx/upload") + .user_agent("paroxysm/0.0.2 crimp/0.2")? + .header("Linx-Expiry", "7200")? // 2 hours + .body("text/plain", data_to_upload.as_bytes()) + .timeout(std::time::Duration::from_secs(2))? + .send()? + .as_string()?; + // step 3: tell the world about it + if response.status != 200 { + Err(format_err!( + "upload returned {}: {}", + response.status, + response.body + ))? + } + self.client.send_notice( + target, + format!( + "\x02{}\x0f: uploaded {} quotes to \x02\x0311{}\x0f (will expire in \x0224\x0f hours)", + subj, + kwd.entries.len(), + response.body + ) + )?; + } else { + if idx == 0 { + idx = 1; + } + if let Some(ent) = kwd.format_entry(idx as _) { + self.client.send_notice(target, ent)?; + } else { + let pluralised = if kwd.entries.len() == 1 { + "entry" + } else { + "entries" + }; + self.client.send_notice( + target, + format!( + "\x02{}\x0f: only has \x02\x0304{}\x0f {}", + subj, + kwd.entries.len(), + pluralised + ), + )?; + } + } + } else { + let entry = if kwd.entries.len() < 2 { + 1 // because [1, 1) does not a range make + } else { + self.rng.gen_range(1, kwd.entries.len()) + }; + if let Some(ent) = kwd.format_entry(entry) { + self.client.send_notice(target, ent)?; + } else { + self.client + .send_notice(target, format!("\x02{}\x0f: no entries yet", subj))?; + } + } + } + None => { + // If someone just posts "??????????", don't spam the channel with + // an error message (but do allow joke entries to appear if set). + if !subj.chars().all(|c| c == '?' || c == ' ') { + self.client + .send_notice(target, format!("\x02{}\x0f: never heard of it", subj))?; + } + } + } + Ok(()) + } + + pub fn handle_privmsg(&mut self, from: &str, chan: &str, msg: &str) -> Result<(), Error> { + let nick = from.split("!").next().ok_or(format_err!( + "Received PRIVMSG from a source without nickname (failed to split n!u@h)" + ))?; + let target = if chan.starts_with("#") { chan } else { nick }; + debug!("[{}] <{}> {}", chan, nick, msg); + if let Some(learn) = LEARN_RE.captures(msg) { + self.handle_learn(target, nick, chan, learn)?; + } else if let Some(qlast) = QLAST_RE.captures(msg) { + self.handle_insert_last_quote(target, nick, chan, qlast)?; + } else if let Some(mv) = MOVE_RE.captures(msg) { + self.handle_move(target, nick, chan, mv)?; + } else if let Some(icr) = INCREMENT_RE.captures(msg) { + self.handle_increment(target, nick, chan, icr)?; + } else if let Some(query) = QUERY_RE.captures(msg) { + self.handle_query(target, nick, chan, query)?; + } else { + let chan_lastmsgs = self + .last_msgs + .entry(chan.to_string()) + .or_insert(HashMap::new()); + chan_lastmsgs.insert(nick.to_string().to_ascii_lowercase(), msg.to_string()); + } + Ok(()) + } + + pub fn handle_msg(&mut self, m: Message) -> Result<(), Error> { + match m.command { + Command::PRIVMSG(channel, message) => { + if let Some(src) = m.prefix { + if let Err(e) = self.handle_privmsg(&src, &channel, &message) { + warn!("error handling command in {} (src {}): {}", channel, src, e); + if let Some(nick) = src.split("!").next() { + self.report_error(nick, &channel, e)?; + } + } + } + } + Command::INVITE(nick, channel) => { + if self.cfg.admins.contains(&nick) { + info!("Joining {} after admin invite", channel); + self.client.send_join(channel)?; + } + } + _ => {} + } + Ok(()) + } +} + +fn main() -> Result<(), Error> { + println!("[+] loading configuration"); + let default_log_filter = "paroxysm=info".to_string(); + let mut settings = config::Config::default(); + settings.merge(config::Environment::with_prefix("PARX"))?; + let cfg: Config = settings.try_into()?; + let env = env_logger::Env::new() + .default_filter_or(cfg.log_filter.clone().unwrap_or(default_log_filter)); + env_logger::init_from_env(env); + info!("paroxysm starting up"); + info!("connecting to database at {}", cfg.database_url); + let pg = Pool::new(ConnectionManager::new(&cfg.database_url))?; + info!("connecting to IRC using config {}", cfg.irc_config_path); + let client = IrcClient::new(&cfg.irc_config_path)?; + client.identify()?; + let st = client.stream(); + let mut app = App { + client, + pg, + cfg, + rng: thread_rng(), + last_msgs: HashMap::new(), + }; + info!("running!"); + st.for_each_incoming(|m| { + if let Err(e) = app.handle_msg(m) { + warn!("Error processing message: {}", e); + } + })?; + Ok(()) +} diff --git a/fun/paroxysm/src/models.rs b/fun/paroxysm/src/models.rs new file mode 100644 index 000000000000..721efbbb2e61 --- /dev/null +++ b/fun/paroxysm/src/models.rs @@ -0,0 +1,36 @@ +use crate::schema::{entries, keywords}; +use chrono::NaiveDateTime; + +#[derive(Queryable)] +pub struct Keyword { + pub id: i32, + pub name: String, + pub chan: String, +} + +#[derive(Queryable)] +pub struct Entry { + pub id: i32, + pub keyword_id: i32, + pub idx: i32, + pub text: String, + pub creation_ts: NaiveDateTime, + pub created_by: String, +} + +#[derive(Insertable)] +#[table_name = "keywords"] +pub struct NewKeyword<'a> { + pub name: &'a str, + pub chan: &'a str, +} + +#[derive(Insertable)] +#[table_name = "entries"] +pub struct NewEntry<'a> { + pub keyword_id: i32, + pub idx: i32, + pub text: &'a str, + pub creation_ts: NaiveDateTime, + pub created_by: &'a str, +} diff --git a/fun/paroxysm/src/schema.rs b/fun/paroxysm/src/schema.rs new file mode 100644 index 000000000000..ef4044531ee7 --- /dev/null +++ b/fun/paroxysm/src/schema.rs @@ -0,0 +1,18 @@ +table! { + entries (id) { + id -> Int4, + keyword_id -> Int4, + idx -> Int4, + text -> Varchar, + creation_ts -> Timestamp, + created_by -> Varchar, + } +} + +table! { + keywords (id) { + id -> Int4, + name -> Varchar, + chan -> Varchar, + } +} diff --git a/fun/quinistry/.gitignore b/fun/quinistry/.gitignore new file mode 100644 index 000000000000..622119552e47 --- /dev/null +++ b/fun/quinistry/.gitignore @@ -0,0 +1,2 @@ +.idea/ +quinistry \ No newline at end of file diff --git a/fun/quinistry/README.md b/fun/quinistry/README.md new file mode 100644 index 000000000000..de197a219ebf --- /dev/null +++ b/fun/quinistry/README.md @@ -0,0 +1,63 @@ +Quinistry +========= + +*A simple Docker registry quine.* + +## What? + +This is an example project for a from-scratch implementation of an HTTP server compatible with the [Docker Registry V2][] +protocol. + +It serves a single image called `quinistry:latest` which is a Docker image that runs quinistry itself, therefore it is a +sort of Docker registry [quine][]. + +The official documentation does not contain enough information to actually implement this protocol (which I assume is +intentional), but a bit of trial&error lead there anyways. I've added comments to parts of the code to clear up things +that may be helpful to other developers in the future. + +## Example + +``` +# Run quinistry: +vincent@urdhva ~/go/src/github.com/tazjin/quinistry (git)-[master] % ./quinistry +2017/03/16 14:11:56 Starting quinistry + +# Pull the quinistry image from itself: +vincent@urdhva ~ % docker pull localhost:8080/quinistry +Using default tag: latest +latest: Pulling from quinistry +7bf1a8b18466: Already exists +Digest: sha256:d5cd4490901ef04b4e28e4ccc03a1d25fe3461200cf4d7166aab86fcd495e22e +Status: Downloaded newer image for localhost:8080/quinistry:latest + +# Quinistry will log: +2017/03/16 14:14:03 Acknowleding V2 API: GET /v2/ +2017/03/16 14:14:03 Serving manifest: GET /v2/quinistry/manifests/latest +2017/03/16 14:14:03 Serving config: GET /v2/quinistry/blobs/sha256:fbb165c48849de16017aa398aa9bb08fd1c00eaa7c150b6c2af37312913db279 + +# Run the downloaded image: +vincent@urdhva ~ % docker run -p 8090:8080 localhost:8080/quinistry +2017/03/16 13:15:18 Starting quinistry + +# And download it again from itself: +vincent@urdhva ~ % docker pull localhost:8090/quinistry +Using default tag: latest +latest: Pulling from quinistry +7bf1a8b18466: Already exists +Digest: sha256:11141d95ddce0bac9ffa32ab1e8bc94748ed923e87762c68858dc41d11a46d3f +Status: Downloaded newer image for localhost:8090/quinistry:latest +``` + +## Building + +Quinistry creates a Docker image that only contains a statically linked `main` binary. As this package makes use of +`net/http`, Go will (by default) link against `libc` for DNS resolution and create a dynamic binary instead. + +To disable this, `build` the project with `-tags netgo`: + +``` +go build -tags netgo +``` + +[Docker Registry V2]: https://docs.docker.com/registry/spec/api/ +[quine]: https://en.wikipedia.org/wiki/Quine_(computing) \ No newline at end of file diff --git a/fun/quinistry/const.go b/fun/quinistry/const.go new file mode 100644 index 000000000000..173fa9efc390 --- /dev/null +++ b/fun/quinistry/const.go @@ -0,0 +1,12 @@ +package main + +// HTTP content types + +const ImageConfigMediaType string = "application/vnd.docker.container.image.v1+json" +const ManifestMediaType string = "application/vnd.docker.distribution.manifest.v2+json" +const LayerMediaType string = "application/vnd.docker.image.rootfs.diff.tar.gzip" + +// HTTP header names + +const ContentType string = "Content-Type" +const DigestHeader string = "Docker-Content-Digest" diff --git a/fun/quinistry/default.nix b/fun/quinistry/default.nix new file mode 100644 index 000000000000..b12c5e6f0c29 --- /dev/null +++ b/fun/quinistry/default.nix @@ -0,0 +1,11 @@ +{ depot, ... }: + +depot.nix.buildGo.program { + name = "quinistry"; + srcs = [ + ./const.go + ./image.go + ./main.go + ./types.go + ]; +} diff --git a/fun/quinistry/image.go b/fun/quinistry/image.go new file mode 100644 index 000000000000..3daeac34d6bb --- /dev/null +++ b/fun/quinistry/image.go @@ -0,0 +1,150 @@ +// The code in this file creates a Docker image layer containing the binary of the +// application itself. + +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "os" + "time" +) + +// This function creates a Docker-image digest (i.e. SHA256 hash with +// algorithm-specification prefix) +func Digest(b []byte) string { + hash := sha256.New() + hash.Write(b) + + return fmt.Sprintf("sha256:%x", hash.Sum(nil)) +} + +func GetImageOfCurrentExecutable() Image { + binary := getCurrentBinary() + tarArchive := createTarArchive(&map[string][]byte{ + "/main": binary, + }) + + configJson, configElem := createConfig([]string{Digest(tarArchive)}) + compressed := gzipArchive("Quinistry image", tarArchive) + manifest := createManifest(&configElem, &compressed) + manifestJson, _ := json.Marshal(manifest) + + return Image{ + Layer: compressed, + LayerDigest: Digest(compressed), + Manifest: manifestJson, + ManifestDigest: Digest(manifestJson), + Config: configJson, + ConfigDigest: Digest(configJson), + } + +} + +func getCurrentBinary() []byte { + path, _ := os.Executable() + file, _ := ioutil.ReadFile(path) + return file +} + +func createTarArchive(files *map[string][]byte) []byte { + buf := new(bytes.Buffer) + w := tar.NewWriter(buf) + + for name, file := range *files { + hdr := &tar.Header{ + Name: name, + // Everything is executable \o/ + Mode: 0755, + Size: int64(len(file)), + } + w.WriteHeader(hdr) + w.Write(file) + } + + if err := w.Close(); err != nil { + log.Fatalln(err) + os.Exit(1) + } + + return buf.Bytes() +} + +func gzipArchive(name string, archive []byte) []byte { + buf := new(bytes.Buffer) + w := gzip.NewWriter(buf) + w.Name = name + w.Write(archive) + + if err := w.Close(); err != nil { + log.Fatalln(err) + os.Exit(1) + } + + return buf.Bytes() +} + +func createConfig(layerDigests []string) (configJson []byte, elem Element) { + now := time.Now() + + imageConfig := &ImageConfig{ + Cmd: []string{"/main"}, + Env: []string{"PATH=/"}, + } + + rootFs := RootFs{ + DiffIds: layerDigests, + Type: "layers", + } + + history := []History{ + { + Created: now, + CreatedBy: "Quinistry magic", + }, + } + + config := Config{ + Created: now, + Author: "tazjin", + Architecture: "amd64", + Os: "linux", + Config: imageConfig, + RootFs: rootFs, + History: history, + } + + configJson, _ = json.Marshal(config) + + elem = Element{ + MediaType: ImageConfigMediaType, + Size: len(configJson), + Digest: Digest(configJson), + } + + return +} + +func createManifest(config *Element, layer *[]byte) Manifest { + layers := []Element{ + { + MediaType: LayerMediaType, + Size: len(*layer), + // Layers must contain the digest of the *gzipped* layer. + Digest: Digest(*layer), + }, + } + + return Manifest{ + SchemaVersion: 2, + MediaType: ManifestMediaType, + Config: *config, + Layers: layers, + } +} diff --git a/fun/quinistry/k8s/child.yaml b/fun/quinistry/k8s/child.yaml new file mode 100644 index 000000000000..aa2e318262cd --- /dev/null +++ b/fun/quinistry/k8s/child.yaml @@ -0,0 +1,27 @@ +# This is a child quinistry, running via an image served off the parent. +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: quinistry-gen2 + labels: + k8s-app: quinistry + quinistry/role: child + quinistry/generation: '2' +spec: + template: + metadata: + labels: + k8s-app: quinistry + quinistry/role: child + quinistry/generation: '2' + spec: + containers: + - name: quinistry + # Bootstrap via Docker Hub (or any other registry) + image: localhost:5000/quinistry + ports: + - name: registry + containerPort: 8080 + # Incremented hostPort, + hostPort: 5001 diff --git a/fun/quinistry/k8s/parent.yaml b/fun/quinistry/k8s/parent.yaml new file mode 100644 index 000000000000..0db2fe300e1e --- /dev/null +++ b/fun/quinistry/k8s/parent.yaml @@ -0,0 +1,27 @@ +# This is a bootstrapped Quinistry DaemonSet. The initial image +# comes from Docker Hub +--- +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: quinistry + labels: + k8s-app: quinistry + quinistry/role: parent + quinistry/generation: '1' +spec: + template: + metadata: + labels: + k8s-app: quinistry + quinistry/role: parent + quinistry/generation: '1' + spec: + containers: + - name: quinistry + # Bootstrap via Docker Hub (or any other registry) + image: tazjin/quinistry + ports: + - name: registry + containerPort: 8080 + hostPort: 5000 diff --git a/fun/quinistry/main.go b/fun/quinistry/main.go new file mode 100644 index 000000000000..50b47418d1a8 --- /dev/null +++ b/fun/quinistry/main.go @@ -0,0 +1,57 @@ +package main + +import ( + "fmt" + "log" + "net/http" +) + +func main() { + log.Println("Starting quinistry") + + image := GetImageOfCurrentExecutable() + + layerUri := fmt.Sprintf("/v2/quinistry/blobs/%s", image.LayerDigest) + configUri := fmt.Sprintf("/v2/quinistry/blobs/%s", image.ConfigDigest) + + log.Fatal(http.ListenAndServe(":8080", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Acknowledge that we speak V2 + if r.RequestURI == "/v2/" { + logRequest("Acknowleding V2 API", r) + fmt.Fprintln(w) + return + } + + // Serve manifest + if r.RequestURI == "/v2/quinistry/manifests/latest" { + logRequest("Serving manifest", r) + w.Header().Set(ContentType, ManifestMediaType) + w.Header().Add(DigestHeader, image.ManifestDigest) + w.Write(image.Manifest) + return + } + + // Serve actual image layer + if r.RequestURI == layerUri { + logRequest("Serving image layer blob", r) + w.Header().Add(DigestHeader, image.LayerDigest) + w.Write(image.Layer) + return + } + + // Serve image config + if r.RequestURI == configUri { + logRequest("Serving config", r) + w.Header().Set("Content-Type", ImageConfigMediaType) + w.Header().Set(DigestHeader, image.ConfigDigest) + w.Write(image.Config) + return + } + + log.Printf("Unhandled request: %v\n", *r) + }))) +} + +func logRequest(msg string, r *http.Request) { + log.Printf("%s: %s %s\n", msg, r.Method, r.RequestURI) +} diff --git a/fun/quinistry/types.go b/fun/quinistry/types.go new file mode 100644 index 000000000000..498cbac2f2ab --- /dev/null +++ b/fun/quinistry/types.go @@ -0,0 +1,79 @@ +package main + +import "time" + +// This type represents the rootfs-key of the Docker image config. +// It specifies the digest (i.e. usually SHA256 hash) of the tar'ed, but NOT +// compressed image layers. +type RootFs struct { + // The digests of the non-compressed FS layers. + DiffIds []string `json:"diff_ids"` + + // Type should always be set to "layers" + Type string `json:"type"` +} + +// This type represents an entry in the Docker image config's history key. +// Every history element "belongs" to a filesystem layer. +type History struct { + Created time.Time `json:"created"` + CreatedBy string `json:"created_by"` +} + +// This type represents runtime-configuration for the Docker image. +// A lot of possible keys are omitted here, see: +// https://github.com/docker/docker/blob/master/image/spec/v1.2.md#image-json-description +type ImageConfig struct { + Cmd []string + Env []string +} + +// This type represents the Docker image configuration +type Config struct { + Created time.Time `json:"created"` + Author string `json:"author"` + + // Architecture should be "amd64" + Architecture string `json:"architecture"` + + // OS should be "linux" + Os string `json:"os"` + + // Configuration can be set to 'nil', in which case all options have to be + // supplied at container launch time. + Config *ImageConfig `json:"config"` + + // Filesystem layers and history elements have to be in the same order. + RootFs RootFs `json:"rootfs"` + History []History `json:"history"` +} + +// This type represents any manifest +type Element struct { + MediaType string `json:"mediaType"` + Size int `json:"size"` + Digest string `json:"digest"` +} + +// This type represents a Docker image manifest as used by the registry +// protocol V2. +type Manifest struct { + SchemaVersion int `json:"schemaVersion"` // Must be 2 + MediaType string `json:"mediaType"` // Use ManifestMediaType const + Config Element `json:"config"` + Layers []Element `json:"layers"` +} + +// A really "dumb" representation of an image, with its data blob and related +// metadata. +// Note: This is not a registry API type. +type Image struct { + Layer []byte + LayerDigest string + + Manifest []byte + ManifestDigest string + + Config []byte + ConfigDigest string +} diff --git a/fun/streamTVL/default.nix b/fun/streamTVL/default.nix new file mode 100644 index 000000000000..8d8263c9dd3f --- /dev/null +++ b/fun/streamTVL/default.nix @@ -0,0 +1,19 @@ +{ pkgs, ... }: + +pkgs.writeShellScriptBin "start-tvl-stream" '' + env LD_LIBRARY_PATH=/run/opengl-driver/lib/ ${pkgs.ffmpeg-full}/bin/ffmpeg \ + -vsync 0 \ + -hwaccel cuvid \ + -init_hw_device cuda=0 -filter_hw_device 0 \ + -f x11grab \ + -video_size 1920x1080 \ + -framerate 60 \ + -thread_queue_size 256 \ + -i :0.0+0,0 \ + -filter:v "format=nv12,hwupload,scale_npp=w=1280:h=720:interp_algo=lanczos" \ + -c:v h264_nvenc \ + -preset:v llhq \ + -rc:v cbr_ld_hq \ + -an \ + -f flv rtmp://tazj.in:1935/tvl +'' diff --git a/fun/tvl-ebooks/OWNERS b/fun/tvl-ebooks/OWNERS new file mode 100644 index 000000000000..7dd8c27a574c --- /dev/null +++ b/fun/tvl-ebooks/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - ben \ No newline at end of file diff --git a/fun/tvl-ebooks/default.nix b/fun/tvl-ebooks/default.nix new file mode 100644 index 000000000000..fde6e058220e --- /dev/null +++ b/fun/tvl-ebooks/default.nix @@ -0,0 +1,7 @@ +{ pkgs, ... }: + +pkgs.buildGoModule { + name = "tvl-ebooks"; + vendorSha256 = "1p7bazh2vbhvvm559bcvfff9s4yy4q9jmklxr3sfp97inwpv6hzy"; + src = ./.; +} diff --git a/fun/tvl-ebooks/ebook-client/main.go b/fun/tvl-ebooks/ebook-client/main.go new file mode 100644 index 000000000000..7954184dd7be --- /dev/null +++ b/fun/tvl-ebooks/ebook-client/main.go @@ -0,0 +1,170 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "net" + "sync" + "time" + + "github.com/go-redis/redis" + "gopkg.in/irc.v3" +) + +var messageBeat chan bool +var firstMessage chan bool +var client *irc.Client +var safeLock sync.Mutex + +func main() { + nick := flag.String("nick", "NONE", "the ircnick you want") + from := flag.String("ip", "[::1]", "src address") + flag.Parse() + + localAddrDialier := &net.Dialer{ + LocalAddr: &net.TCPAddr{ + IP: net.ParseIP(*from), + Port: 0, + }, + } + + conn, err := tls.DialWithDialer(localAddrDialier, "tcp", "chat.freenode.net:6697", &tls.Config{}) + if err != nil { + log.Fatalln(err) + } + + messageBeat = make(chan bool) + firstMessage = make(chan bool, 10) + go ircKeepalive() + + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + go func() { + for { + time.Sleep(time.Second) + r := redisc.Ping() + if r.Err() != nil { + redisc = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + } + redisc.Set(fmt.Sprintf("alive-%s", *nick), "yes", time.Second*5) + } + }() + + if *nick == "NONE" { + log.Fatalf("You must set a nick") + } + + go func() { + <-firstMessage + for { + psub := redisc.Subscribe(fmt.Sprintf("irc-%s", *nick)) + + for { + msg, err := psub.ReceiveMessage() + if err != nil { + break + } + client.WriteMessage(&irc.Message{ + Command: "PRIVMSG", + Params: []string{ + "##tvl-ebooks", + msg.Payload, + }, + }) + } + time.Sleep(time.Second * 10) + } + + }() + + go func() { + <-firstMessage + for { + psub := redisc.Subscribe(fmt.Sprintf("raw-irc-%s", *nick)) + + for { + msg, err := psub.ReceiveMessage() + if err != nil { + break + } + im := irc.Message{} + err = json.Unmarshal([]byte(msg.Payload), &im) + if err == nil { + client.WriteMessage(&im) + } + } + time.Sleep(time.Second * 10) + } + + }() + + seenMsgBefore := false + config := irc.ClientConfig{ + Nick: *nick, + User: *nick, + Name: fmt.Sprintf("%s Ebooks", *nick), + Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + b, _ := json.Marshal(m) + log.Printf("%#v", string(b)) + + messageBeat <- true + + if !seenMsgBefore { + firstMessage <- true + seenMsgBefore = true + } + res := redisc.Publish("ebook", string(b)) + if res.Err() != nil { + log.Printf("Publish error! %#v", err) + } + if m.Command == "001" { + // 001 is a welcome event, so we join channels there + c.Write("JOIN ##tvl-ebooks") + } + // else if m.Command == "PRIVMSG" && c.FromChannel(m) { + // // // Create a handler on all messages. + // // c.WriteMessage(&irc.Message{ + // // Command: "PRIVMSG", + // // Params: []string{ + // // m.Params[0], + // // m.Trailing(), + // // }, + // // }) + // } + }), + } + + // Create the client + client = irc.NewClient(conn, config) + err = client.Run() + if err != nil { + log.Fatalln(err) + } +} + +func ircKeepalive() { + tt := time.NewTimer(time.Second) + lastPing := time.Now() + for { + select { + case <-tt.C: + if time.Since(lastPing) > time.Minute*5 { + log.Fatalf("It's been too long since the last IRC message, blowing up") + } + break + case <-messageBeat: + lastPing = time.Now() + } + } +} diff --git a/fun/tvl-ebooks/go.mod b/fun/tvl-ebooks/go.mod new file mode 100644 index 000000000000..154fbf8216ef --- /dev/null +++ b/fun/tvl-ebooks/go.mod @@ -0,0 +1,8 @@ +module github.com/benjojo/tvl-ebooks + +go 1.14 + +require ( + github.com/go-redis/redis v6.15.9+incompatible + gopkg.in/irc.v3 v3.1.3 +) diff --git a/fun/tvl-ebooks/go.sum b/fun/tvl-ebooks/go.sum new file mode 100644 index 000000000000..bd3ff8761342 --- /dev/null +++ b/fun/tvl-ebooks/go.sum @@ -0,0 +1,11 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/irc.v3 v3.1.3 h1:yeTiJ365882L8h4AnBKYfesD92y5R5ZhGiylu9DfcPY= +gopkg.in/irc.v3 v3.1.3/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/fun/tvl-ebooks/irc-ingest/main.go b/fun/tvl-ebooks/irc-ingest/main.go new file mode 100644 index 000000000000..bbd607405cdf --- /dev/null +++ b/fun/tvl-ebooks/irc-ingest/main.go @@ -0,0 +1,99 @@ +package main + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "log" + "os" + "time" + + "github.com/go-redis/redis" + "gopkg.in/irc.v3" +) + +var messageBeat chan bool + +func main() { + conn, err := tls.Dial("tcp", "bnc.irccloud.com:6697", nil) + if err != nil { + log.Fatalln(err) + } + + messageBeat = make(chan bool) + go ircKeepalive() + + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + go func() { + for { + time.Sleep(time.Second) + r := redisc.Ping() + if r.Err() != nil { + redisc = redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + } + } + }() + + config := irc.ClientConfig{ + Nick: "Benjojo", + Pass: os.Getenv("IRCCLOUD"), + User: "Benjojo", + Name: "Ben Cox", + Handler: irc.HandlerFunc(func(c *irc.Client, m *irc.Message) { + b, _ := json.Marshal(m) + // log.Printf("%#v", string(b)) + + messageBeat <- true + res := redisc.Publish("irccloud", string(b)) + if res.Err() != nil { + log.Printf("Publish error! %#v", err) + } + // if m.Command == "001" { + // // 001 is a welcome event, so we join channels there + // // c.Write("JOIN #bot-test-chan") + // } else if m.Command == "PRIVMSG" && c.FromChannel(m) { + // // // Create a handler on all messages. + // // c.WriteMessage(&irc.Message{ + // // Command: "PRIVMSG", + // // Params: []string{ + // // m.Params[0], + // // m.Trailing(), + // // }, + // // }) + // } + }), + } + + // Create the client + client := irc.NewClient(conn, config) + err = client.Run() + if err != nil { + log.Fatalln(err) + } +} + +func ircKeepalive() { + tt := time.NewTimer(time.Second) + lastPing := time.Now() + for { + select { + case <-tt.C: + if time.Since(lastPing) > time.Minute*5 { + log.Fatalf("It's been too long since the last IRC message, blowing up") + } + break + case <-messageBeat: + lastPing = time.Now() + } + } + +} diff --git a/fun/tvl-ebooks/make-v6/main.go b/fun/tvl-ebooks/make-v6/main.go new file mode 100644 index 000000000000..6d4d0047bbac --- /dev/null +++ b/fun/tvl-ebooks/make-v6/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "crypto/rand" + "log" + "net" +) + +func main() { + // 2a0c:2f07:29:9999:6564:5298:8413:4652 + ip := net.ParseIP("2a0c:2f07:29::") + + rand.Read(ip[6:]) + + if ip[7] > 0xaa { + ip[4] = 0x03 + ip[5] = 0x84 + if ip[7] > 0xdd { + ip[4] = 0x08 + ip[5] = 0x64 + } + } + + log.Printf("%s\n", ip) + // +} diff --git a/fun/tvl-ebooks/mkov-engine/main.go b/fun/tvl-ebooks/mkov-engine/main.go new file mode 100644 index 000000000000..ba4863414029 --- /dev/null +++ b/fun/tvl-ebooks/mkov-engine/main.go @@ -0,0 +1,246 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "math/rand" + "strconv" + "strings" + "time" + + "github.com/go-redis/redis" +) + +type incomingIRC struct { + Command string `json:"Command"` + Host string `json:"Host"` + Name string `json:"Name"` + Params []string `json:"Params"` + User string `json:"User"` +} + +var suppressionUsernames map[string]bool +var noMkov map[string]bool + +func main() { + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + fireaway := make(chan incomingIRC, 10) + suppressionUsernames = make(map[string]bool) + + suppressionList := redisc.HGetAll("suppressionList") + suppressionListA, _ := suppressionList.Result() + + suppressionListMap, _ := stringMaptoIntMap(suppressionListA) + for v, _ := range suppressionListMap { + suppressionUsernames[v] = true + suppressionUsernames[strings.ToLower(v)] = true + } + + noMkov = make(map[string]bool) + + noMkovRedis := redisc.HGetAll("nomkov") + noMkovRedisA, _ := noMkovRedis.Result() + + noMkovMap, _ := stringMaptoIntMap(noMkovRedisA) + for v, _ := range noMkovMap { + noMkov[v] = true + noMkov[strings.ToLower(v)] = true + } + + go func() { + for { + irccloudFeed := redisc.Subscribe("irccloud") + for { + msg, err := irccloudFeed.ReceiveMessage() + if err != nil { + break + } + imsg := incomingIRC{} + err = json.Unmarshal([]byte(msg.Payload), &imsg) + if err != nil { + log.Printf("Json decoding error from irccloud feed %s", err) + continue + } + + if imsg.Command == "PRIVMSG" { + if len(imsg.Params) == 2 { + if imsg.Params[0] == "##tvl" || imsg.Params[0] == "##tvlbot" { + fireaway <- imsg + } + } + } + } + time.Sleep(time.Second) + } + }() + + for msg := range fireaway { + // Learn + learnFromMessage(msg, redisc) + msg2 := generateMesasge(msg, redisc) + + // Check if we have a active log in for that user + ttl := redisc.TTL("alive-" + msg.Name + "-eb") + ttld, err := ttl.Result() + if err == nil { + redisc.Publish("irc-"+msg.Name+"-eb", msg2) + if ttld == 0 || ttld.Seconds() == -2 { + redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2) + } + } else { + redisc.Publish("irc-tvlebooks-eb", "<"+fmt.Sprintf("%s.%s", string(msg.Name[0]), msg.Name[1:])+"-eb> "+msg2) + } + } +} + +func generateMesasge(msg incomingIRC, redisc *redis.Client) string { + text := msg.Params[1] + username := strings.ToLower(msg.Name) + suppressionUsernames[username] = true + suppressionUsernames[username+":"] = true + suppressionUsernames[msg.Name] = true + suppressionUsernames[msg.Name+":"] = true + redisc.HIncrBy("suppressionList", msg.Name, 1) + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + lastWord := propwords(msg.Name, words[0], redisc) + + if noMkov[username] { + lastWord = blockoutWord(lastWord) + words[0] = blockoutWord(words[0]) + } + + lastWord = filterHighlights(lastWord) + + if lastWord == "_END_" { + if noMkov[username] { + return blockoutWord(words[0]) + } + return words[0] + } + outputMsg := words[0] + " " + lastWord + " " + + for { + lastWord = propwords(username, lastWord, redisc) + if lastWord == "" || lastWord == "_END_" { + return outputMsg + } + + if noMkov[username] { + lastWord = blockoutWord(lastWord) + } + + lastWord = filterHighlights(lastWord) + + outputMsg += lastWord + " " + if len(outputMsg) > 100 { + return outputMsg + } + } +} + +// filterHighlights: tries to prevent highlights by checking against +// a map called suppressionUsernames +func filterHighlights(in string) string { + for username := range suppressionUsernames { + if strings.Contains(in, username) { + if len(in) < 2 { + in = fmt.Sprintf("%s.%s", string(in[0]), in[1:]) + return in + } + } + } + return in +} + +func blockoutWord(in string) string { + block := "" + for i := 0; i < len(in); i++ { + block += "█" + } + return block +} + +func propwords(username string, start string, redisc *redis.Client) string { + userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start)) + userHashMap, err := userHash.Result() + if err != nil { + genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start)) + userHashMap, err = genericHash.Result() + } + + userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap) + if totalVectors == 0 { + return "" + } + targetRand := rand.Intn(totalVectors) + progresRand := 0 + + for k, v := range userIntHashMap { + progresRand += v + if targetRand > progresRand { + return k + } + } + + for k, _ := range userIntHashMap { + return k + } + + return "" +} + +func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) { + outMap = make(map[string]int) + + for k, v := range in { + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + total += int(i) + outMap[k] = int(i) + } + + return outMap, total +} + +func learnFromMessage(msg incomingIRC, redisc *redis.Client) { + text := msg.Params[1] + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + username := msg.Name + + for k, word := range words { + // HINCRBY myhash field 1 + nextWord := "_END_" + if len(words)-1 != k { + nextWord = words[k+1] + } + + if !noMkov[username] { + redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1) + } + redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1) + } +} diff --git a/fun/tvl-ebooks/parse-logs/main.go b/fun/tvl-ebooks/parse-logs/main.go new file mode 100644 index 000000000000..a2d0d20eeb9c --- /dev/null +++ b/fun/tvl-ebooks/parse-logs/main.go @@ -0,0 +1,174 @@ +package main + +import ( + "bufio" + "fmt" + "log" + "math/rand" + "os" + "regexp" + "strconv" + "strings" + + "github.com/go-redis/redis" +) + +type incomingIRC struct { + Command string `json:"Command"` + Host string `json:"Host"` + Name string `json:"Name"` + Params []string `json:"Params"` + User string `json:"User"` +} + +var quicklogMatch = regexp.MustCompile(`<(\w+)> (.+)`) + +func main() { + redisc := redis.NewClient(&redis.Options{ + Addr: fmt.Sprintf("127.0.0.1:%d", 6379), + Password: "", // no password set + DB: 0, // use default DB + }) + + fireaway := make(chan incomingIRC, 10) + go func() { + f, err := os.Open("tvl.txt") + if err != nil { + log.Printf("aaa %v", err) + os.Exit(0) + } + + bio := bufio.NewReader(f) + for { + line, _, err := bio.ReadLine() + if err != nil { + break + } + + sline := string(line) + + offset := strings.Index(sline, "]") + + notime := sline[offset+1:] + + if quicklogMatch.MatchString(notime) { + bits := quicklogMatch.FindAllStringSubmatch(notime, -1) + if len(bits) != 0 { + if len(bits[0]) != 0 { + a := make([]string, 2) + a[1] = bits[0][2] + ic := incomingIRC{ + Name: bits[0][1], + Params: a, + } + log.Printf("aa %#v", ic) + + fireaway <- ic + } + } + } + + } + + }() + + for msg := range fireaway { + // Learn + learnFromMessage(msg, redisc) + // os.Exit(0) + } +} + +func generateMesasge(msg incomingIRC, redisc *redis.Client) string { + text := msg.Params[1] + username := msg.Name + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + lastWord := propwords(username, words[0], redisc) + outputMsg := words[0] + " " + lastWord + " " + + for { + lastWord = propwords(username, words[0], redisc) + if lastWord == "" || lastWord == "_END_" { + return outputMsg + } + + outputMsg += lastWord + " " + if len(outputMsg) > 100 { + return outputMsg + } + } +} + +func propwords(username string, start string, redisc *redis.Client) string { + userHash := redisc.HGetAll(fmt.Sprintf("%s-%s", username, start)) + userHashMap, err := userHash.Result() + if err != nil { + genericHash := redisc.HGetAll(fmt.Sprintf("generic-%s", start)) + userHashMap, err = genericHash.Result() + } + + userIntHashMap, totalVectors := stringMaptoIntMap(userHashMap) + targetRand := rand.Intn(totalVectors) + progresRand := 0 + + for k, v := range userIntHashMap { + progresRand += v + if targetRand > progresRand { + return k + } + } + + for k, _ := range userIntHashMap { + return k + } + + return "" +} + +func stringMaptoIntMap(in map[string]string) (outMap map[string]int, total int) { + outMap = make(map[string]int) + + for k, v := range in { + i, err := strconv.ParseInt(v, 10, 64) + if err != nil { + continue + } + total += int(i) + outMap[k] = int(i) + } + + return outMap, total +} + +func learnFromMessage(msg incomingIRC, redisc *redis.Client) { + text := msg.Params[1] + + text = strings.ToLower(text) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ",", "", -1) + text = strings.Replace(text, ".", "", -1) + text = strings.Replace(text, "!", "", -1) + text = strings.Replace(text, "?", "", -1) + + words := strings.Split(text, " ") + username := msg.Name + + for k, word := range words { + // HINCRBY myhash field 1 + nextWord := "_END_" + if len(words)-1 != k { + nextWord = words[k+1] + } + + redisc.HIncrBy(fmt.Sprintf("%s-%s", username, word), nextWord, 1) + redisc.HIncrBy(fmt.Sprintf("generic-%s", word), nextWord, 1) + } +} diff --git a/fun/uggc/default.nix b/fun/uggc/default.nix new file mode 100644 index 000000000000..ca622666dcd5 --- /dev/null +++ b/fun/uggc/default.nix @@ -0,0 +1,20 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.third_party) gopkgs; + + uggc = depot.nix.buildGo.program { + name = "uggc"; + srcs = [ + ./main.go + ]; + deps = [ + gopkgs."github.com".pkg.browser.gopkg + ]; + }; +in uggc.overrideAttrs(old: { + buildCommand = old.buildCommand + '' + install -D ${./uggc.desktop} $out/share/applications/uggc.desktop + sed "s|@out@|$out|g" -i $out/share/applications/uggc.desktop + ''; +}) diff --git a/fun/uggc/main.go b/fun/uggc/main.go new file mode 100644 index 000000000000..94f8cf092507 --- /dev/null +++ b/fun/uggc/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/pkg/browser" +) + +func rot13(r rune) rune { + if 'a' <= r && r <= 'm' { + return r + ('n' - 'a') + } else if 'n' <= r && r <= 'z' { + return r - ('n' - 'a') + } + if 'A' <= r && r <= 'M' { + return r + ('N' - 'A') + } else if 'N' <= r && r <= 'Z' { + return r - ('N' - 'A') + } + return r +} + +func main() { + if len(os.Args) == 0 { + fmt.Println("usage: uggc [rot13-encoded URL]") + return + } + urlText := strings.Join(os.Args[1:], " ") + corrected := strings.Map(rot13, urlText) + + err := browser.OpenURL(corrected) + + if err != nil { + fmt.Println("could not launch browser:", err) + } +} diff --git a/fun/uggc/uggc.desktop b/fun/uggc/uggc.desktop new file mode 100644 index 000000000000..c4eeff7f5897 --- /dev/null +++ b/fun/uggc/uggc.desktop @@ -0,0 +1,7 @@ +[Desktop Entry] +Type=Application +Name=Rot13 URL Handler +Exec=@out@/bin/uggc +StartupNotify=false +MimeType=x-scheme-handler/uggc; +MimeType=x-scheme-handler/uggcf; diff --git a/fun/uggc/uggc.reg b/fun/uggc/uggc.reg new file mode 100644 index 000000000000..bfb8e1952f7e --- /dev/null +++ b/fun/uggc/uggc.reg @@ -0,0 +1,23 @@ +Windows Registry Editor Version 5.00 + +[HKEY_CLASSES_ROOT\uggc] +"URL Protocol"="" +@="URL:Rot13 HTTP URL Protocol" + +[HKEY_CLASSES_ROOT\uggcf] +"URL Protocol"="" +@="URL:Rot13 HTTPS URL Protocol" + +[HKEY_CLASSES_ROOT\uggc\shell] + +[HKEY_CLASSES_ROOT\uggcf\shell] + +[HKEY_CLASSES_ROOT\uggc\shell\open] + +[HKEY_CLASSES_ROOT\uggcf\shell\open] + +[HKEY_CLASSES_ROOT\uggc\shell\open\command] +@="\"C:\\Program Files\\uggc\\uggc.exe\" \"%1\"" + +[HKEY_CLASSES_ROOT\uggcf\shell\open\command] +@="\"C:\\Program Files\\uggc\\uggc.exe\" \"%1\"" diff --git a/fun/watchblob/README.md b/fun/watchblob/README.md new file mode 100644 index 000000000000..712c96cd95d4 --- /dev/null +++ b/fun/watchblob/README.md @@ -0,0 +1,35 @@ +Watchblob - WatchGuard VPN on Linux +=================================== + +This tiny helper tool makes it possible to use WatchGuard / Firebox / <<whatever +they are actually called>> VPNs that use multi-factor authentication on Linux. + +Rather than using OpenVPN's built-in dynamic challenge/response protocol, WatchGuard +has opted for a separate implementation negotiating credentials outside of the +OpenVPN protocol, which makes it impossible to start those connections solely by +using the `openvpn` CLI and configuration files. + +What this application does has been reverse-engineered from the "WatchGuard Mobile VPN +with SSL" application on OS X. + +I've published a [blog post](https://www.tazj.in/en/1486830338) describing the process +and what is actually going on in this protocol. + +## Installation + +Make sure you have Go installed and `GOPATH` configured, then simply +`go get github.com/tazjin/watchblob/...`. + +## Usage + +Right now the usage is very simple. Make sure you have the correct OpenVPN client +config ready (this is normally supplied by the WatchGuard UI) simply run: + +``` +watchblob vpnserver.somedomain.org username p4ssw0rd +``` + +The server responds with a challenge which is displayed to the user, wait until you +receive the SMS code or whatever and enter it. `watchblob` then completes the +credential negotiation and you may proceed to log in with OpenVPN using your username +and *the OTP token* (**not** your password) as credentials. diff --git a/fun/watchblob/default.nix b/fun/watchblob/default.nix new file mode 100644 index 000000000000..dc32e4d1cefb --- /dev/null +++ b/fun/watchblob/default.nix @@ -0,0 +1,13 @@ +{ depot, ... }: + +depot.nix.buildGo.program { + name = "watchblob"; + srcs = [ + ./main.go + ./urls.go + ]; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.crypto.ssh.terminal.gopkg + ]; +} diff --git a/fun/watchblob/main.go b/fun/watchblob/main.go new file mode 100644 index 000000000000..a7ab65d3d196 --- /dev/null +++ b/fun/watchblob/main.go @@ -0,0 +1,108 @@ +package main + +import ( + "bufio" + "encoding/xml" + "fmt" + "golang.org/x/crypto/ssh/terminal" + "net/http" + "os" + "strings" + "syscall" +) + +// The XML response returned by the WatchGuard server +type Resp struct { + Action string `xml:"action"` + LogonStatus int `xml:"logon_status"` + LogonId int `xml:"logon_id"` + Error string `xml:"errStr"` + Challenge string `xml:"chaStr"` +} + +func main() { + args := os.Args[1:] + + if len(args) != 1 { + fmt.Fprintln(os.Stderr, "Usage: watchblob <vpn-host>") + os.Exit(1) + } + + host := args[0] + + username, password, err := readCredentials() + if err != nil { + fmt.Fprintf(os.Stderr, "Could not read credentials: %v\n", err) + } + + fmt.Printf("Requesting challenge from %s as user %s\n", host, username) + challenge, err := triggerChallengeResponse(&host, &username, &password) + + if err != nil || challenge.LogonStatus != 4 { + fmt.Fprintln(os.Stderr, "Did not receive challenge from server") + fmt.Fprintf(os.Stderr, "Response: %v\nError: %v\n", challenge, err) + os.Exit(1) + } + + token := getToken(&challenge) + err = logon(&host, &challenge, &token) + + if err != nil { + fmt.Fprintf(os.Stderr, "Logon failed: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Login succeeded, you may now (quickly) authenticate OpenVPN with %s as your password\n", token) +} + +func readCredentials() (string, string, error) { + fmt.Printf("Username: ") + reader := bufio.NewReader(os.Stdin) + username, err := reader.ReadString('\n') + + fmt.Printf("Password: ") + password, err := terminal.ReadPassword(syscall.Stdin) + fmt.Println() + + // If an error occured, I don't care about which one it is. + return strings.TrimSpace(username), strings.TrimSpace(string(password)), err +} + +func triggerChallengeResponse(host *string, username *string, password *string) (r Resp, err error) { + return request(templateUrl(host, templateChallengeTriggerUri(username, password))) +} + +func getToken(challenge *Resp) string { + fmt.Println(challenge.Challenge) + + reader := bufio.NewReader(os.Stdin) + token, _ := reader.ReadString('\n') + + return strings.TrimSpace(token) +} + +func logon(host *string, challenge *Resp, token *string) (err error) { + resp, err := request(templateUrl(host, templateResponseUri(challenge.LogonId, token))) + if err != nil { + return + } + + if resp.LogonStatus != 1 { + err = fmt.Errorf("Challenge/response authentication failed: %v", resp) + } + + return +} + +func request(url string) (r Resp, err error) { + resp, err := http.Get(url) + if err != nil { + return + } + + defer resp.Body.Close() + decoder := xml.NewDecoder(resp.Body) + + err = decoder.Decode(&r) + return +} diff --git a/fun/watchblob/main_test.go b/fun/watchblob/main_test.go new file mode 100644 index 000000000000..1af52d0cd4eb --- /dev/null +++ b/fun/watchblob/main_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "encoding/xml" + "reflect" + "testing" +) + +func TestUnmarshalChallengeRespones(t *testing.T) { + var testXml string = ` +<?xml version="1.0" encoding="UTF-8"?> +<resp> + <action>sslvpn_logon</action> + <logon_status>4</logon_status> + <auth-domain-list> + <auth-domain> + <name>RADIUS</name> + </auth-domain> + </auth-domain-list> + <logon_id>441</logon_id> + <chaStr>Enter Your 6 Digit Passcode </chaStr> +</resp>` + + var r Resp + xml.Unmarshal([]byte(testXml), &r) + + expected := Resp{ + Action: "sslvpn_logon", + LogonStatus: 4, + LogonId: 441, + Challenge: "Enter Your 6 Digit Passcode ", + } + + assertEqual(t, expected, r) +} + +func TestUnmarshalLoginError(t *testing.T) { + var testXml string = ` +<?xml version="1.0" encoding="UTF-8"?> +<resp> + <action>sslvpn_logon</action> + <logon_status>2</logon_status> + <auth-domain-list> + <auth-domain> + <name>RADIUS</name> + </auth-domain> + </auth-domain-list> + <errStr>501</errStr> +</resp>` + + var r Resp + xml.Unmarshal([]byte(testXml), &r) + + expected := Resp{ + Action: "sslvpn_logon", + LogonStatus: 2, + Error: "501", + } + + assertEqual(t, expected, r) +} + +func TestUnmarshalLoginSuccess(t *testing.T) { + var testXml string = ` +<?xml version="1.0" encoding="UTF-8"?> +<resp> + <action>sslvpn_logon</action> + <logon_status>1</logon_status> + <auth-domain-list> + <auth-domain> + <name>RADIUS</name> + </auth-domain> + </auth-domain-list> +</resp> +` + var r Resp + xml.Unmarshal([]byte(testXml), &r) + + expected := Resp{ + Action: "sslvpn_logon", + LogonStatus: 1, + } + + assertEqual(t, expected, r) +} + +func assertEqual(t *testing.T, expected interface{}, result interface{}) { + if !reflect.DeepEqual(expected, result) { + t.Errorf( + "Unmarshaled values did not match.\nExpected: %v\nResult: %v\n", + expected, result, + ) + + t.Fail() + } +} diff --git a/fun/watchblob/urls.go b/fun/watchblob/urls.go new file mode 100644 index 000000000000..37f65e0faef4 --- /dev/null +++ b/fun/watchblob/urls.go @@ -0,0 +1,37 @@ +package main + +import ( + "fmt" + "net/url" + "strconv" +) + +const urlFormat string = "https://%s%s" +const uriFormat = "/?%s" + +func templateChallengeTriggerUri(username *string, password *string) string { + v := url.Values{} + v.Set("action", "sslvpn_logon") + v.Set("style", "fw_logon_progress.xsl") + v.Set("fw_logon_type", "logon") + v.Set("fw_domain", "Firebox-DB") + v.Set("fw_username", *username) + v.Set("fw_password", *password) + + return fmt.Sprintf(uriFormat, v.Encode()) +} + +func templateResponseUri(logonId int, token *string) string { + v := url.Values{} + v.Set("action", "sslvpn_logon") + v.Set("style", "fw_logon_progress.xsl") + v.Set("fw_logon_type", "response") + v.Set("response", *token) + v.Set("fw_logon_id", strconv.Itoa(logonId)) + + return fmt.Sprintf(uriFormat, v.Encode()) +} + +func templateUrl(baseUrl *string, uri string) string { + return fmt.Sprintf(urlFormat, *baseUrl, uri) +} diff --git a/fun/wcl/default.nix b/fun/wcl/default.nix new file mode 100644 index 000000000000..e4d9804c807c --- /dev/null +++ b/fun/wcl/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +depot.nix.buildLisp.program { + name = "wc"; + + srcs = [ + ./wc.lisp + ]; + + deps = with depot.third_party.lisp; [ + unix-opts + iterate + ]; +} diff --git a/fun/wcl/wc.lisp b/fun/wcl/wc.lisp new file mode 100644 index 000000000000..83c7e8c4f1cc --- /dev/null +++ b/fun/wcl/wc.lisp @@ -0,0 +1,34 @@ +(defpackage wc + (:use #:cl #:iterate) + (:export :main)) +(in-package :wc) +(declaim (optimize (speed 3) (safety 0))) + +(defun main () + (let ((filename (cadr (opts:argv))) + (space (char-code #\Space)) + (newline (char-code #\Newline))) + (with-open-file (file-stream filename :element-type '(unsigned-byte 8)) + (iter + (for byte in-stream file-stream using #'read-byte) + (for previous-byte previous byte) + (for is-newline = (eql newline byte)) + + ;; Count each byte + (sum 1 into bytes) + + ;; Count every newline + (counting is-newline into newlines) + + ;; Count every "word", unless the preceding character already + ;; was a space or we are at the beginning of the file. + (when (or (eql space previous-byte) + (eql newline previous-byte) + (not previous-byte)) + (next-iteration)) + + (counting (or is-newline (eql space byte)) + into words) + + (declare (fixnum bytes newlines words)) + (finally (format t " ~A ~A ~A ~A~%" newlines words bytes filename)))))) diff --git a/fun/🕰️/OWNERS b/fun/🕰️/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/fun/🕰️/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/fun/🕰️/bin.lisp b/fun/🕰️/bin.lisp new file mode 100644 index 000000000000..1f5a2b2e0518 --- /dev/null +++ b/fun/🕰️/bin.lisp @@ -0,0 +1,91 @@ +(defpackage 🕰️.bin + (:shadow :describe) + (:use :cl :opts) + (:import-from :uiop :quit) + (:import-from :local-time + :now :timestamp-subtimezone :+utc-zone+ + :*default-timezone* :define-timezone) + (:import-from :klatre :format-dottime-offset) + (:import-from :🕰️ :⌚) + (:export :🚂)) + +(in-package :🕰️.bin) +(declaim (optimize (safety 3))) + +(opts:define-opts + (:name :help + :description "Print this help text" + :short #\h + :long "help") + (:name :dot-time + :description "Use pseudo dot-time format (implies -u)" + :short #\d + :long "dot-time") + (:name :utc + :description "Display time in UTC instead of local time" + :short #\u + :long "utc") + (:name :no-newline + :description "Don't print a trailing newline" + :short #\n + :long "no-newline")) + +(defun make-slash-terminated (str) + (if (eq (char str (1- (length str))) #\/) + str + (concatenate 'string str "/"))) + +; TODO(sterni): upstream this into local-time +(defun setup-default-timezone () + (let* ((tz (remove #\: (uiop:getenv "TZ") :count 1)) + (tz-dir (uiop:getenv "TZDIR")) + (tz-file (if (and tz tz-dir) + (merge-pathnames + (pathname tz) + (pathname (make-slash-terminated tz-dir))) + (pathname "/etc/localtime")))) + (handler-case + (define-timezone *default-timezone* tz-file :load t) + (t () (setf *default-timezone* +utc-zone+))))) + + +(defun 🚂 () + (let ((ts (now))) + (multiple-value-bind (options free-args) + (handler-case (opts:get-opts) + ; only handle subset of conditions that can happen here + (opts:unknown-option (c) + (format t "error: unknown option ~s~%" (opts:option c)) + (quit 100))) + + ; check if we have any free args we don't know what to do with + (when (> (length free-args) 0) + (write-string "error: unexpected command line argument(s): ") + (loop for arg in free-args + do (progn (write-string arg) (write-char #\space))) + (write-char #\newline) + (quit 100)) + + ; print help and exit + (when (getf options :help) + (opts:describe :usage-of "🕰️") + (quit 0)) + + ; reinit *default-timezone* as it is cached from compilation + (setup-default-timezone) + ; dot-time implies UTC, naturally + (when (getf options :dot-time) + (setf (getf options :utc) t)) + ; print clock face + (format t "~A" (⌚ ts (if (getf options :utc) + local-time:+utc-zone+ + local-time:*default-timezone*))) + ; render dot-time offset if necessary + (when (getf options :dot-time) + (multiple-value-bind (offset-secs _dst _name) + (timestamp-subtimezone ts local-time:*default-timezone*) + (write-string + (format-dottime-offset (round (/ offset-secs 3600)))))) + ; write newline if necessary + (when (not (getf options :no-newline)) + (write-char #\newline))))) diff --git a/fun/🕰️/default.nix b/fun/🕰️/default.nix new file mode 100644 index 000000000000..230d9f02f137 --- /dev/null +++ b/fun/🕰️/default.nix @@ -0,0 +1,43 @@ +{ depot, ... }: + +let + inherit (depot.nix) + buildLisp + ; + + lib = buildLisp.library { + name = "lib🕰️"; + deps = [ + depot.third_party.lisp.local-time + ]; + + srcs = [ + ./lib.lisp + ]; + }; + + bin = buildLisp.program { + name = "🕰️"; + deps = [ + depot.third_party.lisp.unix-opts + depot.lisp.klatre + { + default = buildLisp.bundled "asdf"; + sbcl = buildLisp.bundled "uiop"; + } + lib + ]; + + srcs = [ + ./bin.lisp + ]; + + main = "🕰️.bin:🚂"; + + brokenOn = [ + "ecl" # refuses to create non-ASCII paths even on POSIX… + ]; + }; +in bin // { + inherit lib; +} diff --git a/fun/🕰️/lib.lisp b/fun/🕰️/lib.lisp new file mode 100644 index 000000000000..d23db033743e --- /dev/null +++ b/fun/🕰️/lib.lisp @@ -0,0 +1,32 @@ +(defpackage 🕰️ + (:use :cl) + (:import-from :local-time + :timestamp-subtimezone :*default-timezone* :sec-of) + (:export :⌚)) + +(in-package :🕰️) +(declaim (optimize (safety 3))) + +(defparameter *clock-emojis* + (vector #\🕛 #\🕧 ; 00:00 - 00:30 + #\🕐 #\🕜 ; 01:00 - 01:30 + #\🕑 #\🕝 ; 00:00 - 00:30 + #\🕒 #\🕞 ; 00:00 - 00:30 + #\🕓 #\🕟 ; 00:00 - 00:30 + #\🕔 #\🕠 ; 00:00 - 00:30 + #\🕕 #\🕡 ; 00:00 - 00:30 + #\🕖 #\🕢 ; 00:00 - 00:30 + #\🕗 #\🕣 ; 00:00 - 00:30 + #\🕘 #\🕤 ; 00:00 - 00:30 + #\🕙 #\🕥 ; 00:00 - 00:30 + #\🕚 #\🕦)) ; 11:00 - 11:30 + +(defun ⌚ (timestamp &optional (tz *default-timezone*)) + "Convert a LOCAL-TIME:TIMESTAMP into the nearest Unicode clock face. + Use TZ (which defaults to LOCAL-TIME:*DEFAULT-TIMEZONE*) to determine + the UTC offset to factor when determining the local clock face." + (let* ((offset (multiple-value-bind (offset-secs _dst _name) + (timestamp-subtimezone timestamp tz) + offset-secs)) + (secs (+ (sec-of timestamp) offset))) + (elt *clock-emojis* (mod (round (/ secs 1800)) 24)))) diff --git a/lisp/dns/README.md b/lisp/dns/README.md new file mode 100644 index 000000000000..c95d2e9a687e --- /dev/null +++ b/lisp/dns/README.md @@ -0,0 +1,75 @@ +dns +=== + +This library is a DNS-over-HTTPS client for Common Lisp. + +The ambition is to transform it into a fully-featured DNS resolver +instead of piggy-backing on the HTTPS implementation, but ... +baby-steps! + +Note that there is no Common Lisp HTTP client that fully supports the +HTTP2 protocol at the moment, so you can not expect this library to +provide equivalent performance to a native DNS resolver (yet). + +## API + +The API is kept as simple as it can be. + +### Types + +The types of this library are implemented as several structs that +support binary (de-)serialisation via [lisp-binary][]. + +The existing structs are as follows and directly implement the +corresponding definitions from [RFC 1035][]: + +* `dns-header` +* `dns-question` +* `dns-rr` +* `dns-message` + +All relevant field accessors for these structs are exported and can be +used to inspect query results. + +### Functions + +All lookup functions are of the type `(function (string &key doh-url) +(dns-message))` and signal a `dns:doh-error` condition for +unsuccessful requests. + +If `:doh-url` is unspecified, Google's public DNS-over-HTTPS servers +at [dns.google][https://dns.google] will be used. + +Currently implemented lookup functions: + +* `lookup-a` +* `lookup-mx` +* `lookup-txt` + +## Example usage + +```lisp +DNS> (dns-message-answer (lookup-a "git.tazj.in.")) +#(#S(DNS-RR + :NAME #S(QNAME :START-AT 29 :NAMES #(12)) + :TYPE A + :CLASS 1 + :TTL 286 + :RDLENGTH 4 + :RDATA #(34 98 120 189))) +``` + +## TODO + +Various things in this library are currently broken because I only +implemented it to work for my blog setup, but these things will be +ironed out. + +Most importantly, the following needs to be fixed: + +* Each qname *fragment* needs to track its offset, not each qname. +* The RDATA for a TXT record can have multiple counted strings. +* qnames should be canonicalised after parsing. + +[lisp-binary]: https://github.com/j3pic/lisp-binary +[RFC 1035]: https://tools.ietf.org/html/rfc1035 diff --git a/lisp/dns/client.lisp b/lisp/dns/client.lisp new file mode 100644 index 000000000000..cee7bceb54a0 --- /dev/null +++ b/lisp/dns/client.lisp @@ -0,0 +1,85 @@ +;; Implementation of a DoH-client, see RFC 8484 (DNS Queries over +;; HTTPS (DoH)) + +(in-package #:dns) + +(defvar *doh-base-url* "https://dns.google/resolve" + "Base URL of the service providing DNS-over-HTTP(S). Defaults to the + Google-hosted API.") + +(define-condition doh-error (error) + ((query-name :initarg :query-name + :reader doh-error-query-name + :type string) + (query-type :initarg :query-type + :reader doh-error-query-type + :type string) + (doh-url :initarg :doh-url + :reader doh-error-doh-url + :type string) + (status-code :initarg :status-code + :reader doh-error-status-code + :type integer) + (response-body :initarg :response-body + :reader doh-error-response-body + :type (or nil (vector (unsigned-byte 8)) string))) + + (:report (lambda (condition stream) + (let ((url (doh-error-doh-url condition)) + (status (doh-error-status-code condition)) + (body (doh-error-response-body condition))) + (format stream "DoH service at '~A' responded with non-success (~A): ~%~%~A" + url status body))))) + +(defun lookup-generic (name type doh-url) + (multiple-value-bind (body status) + (drakma:http-request doh-url + :decode-content t + ;; TODO(tazjin): Figure out why 'want-stream' doesn't work + :parameters `(("type" . ,type) + ("name" . ,name) + ("ct" . "application/dns-message"))) + (if (= 200 status) + (dns-message-answer + (read-binary 'dns-message (flexi-streams:make-in-memory-input-stream body))) + + (restart-case (error 'doh-error + :query-name name + :query-type type + :doh-url doh-url + :status-code status + :response-body body) + (call-with-other-name (new-name) + :interactive (lambda () (list (the string (read)))) + :test (lambda (c) (typep c 'doh-error)) + (lookup-generic new-name type doh-url)) + + (call-with-other-type (new-type) + :interactive (lambda () (list (the string (read)))) + :test (lambda (c) (typep c 'doh-error)) + (lookup-generic name new-type doh-url)) + + (call-with-other-url (new-url) + :interactive (lambda () (list (the string (read)))) + :test (lambda (c) (typep c 'doh-error)) + (lookup-generic name type new-url)))))) + +(defun lookup-a (name &key (doh-url *doh-base-url*)) + "Look up the A records at NAME." + (lookup-generic name "A" doh-url)) + +(defun lookup-txt (name &key (doh-url *doh-base-url*)) + "Look up the TXT records at NAME." + (lookup-generic name "TXT" doh-url)) + +(defun lookup-mx (name &key (doh-url *doh-base-url*)) + "Look up the MX records at NAME." + (lookup-generic name "MX" doh-url)) + +(defun lookup-cname (name &key (doh-url *doh-base-url*)) + "Look up the CNAME records at NAME." + (lookup-generic name "CNAME" doh-url)) + +(defun lookup-ns (name &key (doh-url *doh-base-url*)) + "Look up the NS records at NAME." + (lookup-generic name "NS" doh-url)) diff --git a/lisp/dns/default.nix b/lisp/dns/default.nix new file mode 100644 index 000000000000..cb2445b460c9 --- /dev/null +++ b/lisp/dns/default.nix @@ -0,0 +1,21 @@ +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "dns"; + + deps = with depot.third_party.lisp; [ + drakma + lisp-binary + iterate + ]; + + srcs = [ + ./package.lisp + ./message.lisp + ./client.lisp + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/lisp/dns/message.lisp b/lisp/dns/message.lisp new file mode 100644 index 000000000000..46243ac8d3f0 --- /dev/null +++ b/lisp/dns/message.lisp @@ -0,0 +1,407 @@ +(in-package :dns) + +;; 3.3. Standard RRs + +;; The following RR definitions are expected to occur, at least +;; potentially, in all classes. In particular, NS, SOA, CNAME, and PTR +;; will be used in all classes, and have the same format in all classes. +;; Because their RDATA format is known, all domain names in the RDATA +;; section of these RRs may be compressed. + +;; <domain-name> is a domain name represented as a series of labels, and +;; terminated by a label with zero length. <character-string> is a single +;; length octet followed by that number of characters. <character-string> +;; is treated as binary information, and can be up to 256 characters in +;; length (including the length octet). + + +;; 3.3.11. NS RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / NSDNAME / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; NSDNAME A <domain-name> which specifies a host which should be +;; authoritative for the specified class and domain. + +;; NS records cause both the usual additional section processing to locate +;; a type A record, and, when used in a referral, a special search of the +;; zone in which they reside for glue information. + +;; The NS RR states that the named host should be expected to have a zone +;; starting at owner name of the specified class. Note that the class may +;; not indicate the protocol family which should be used to communicate +;; with the host, although it is typically a strong hint. For example, +;; hosts which are name servers for either Internet (IN) or Hesiod (HS) +;; class information are normally queried using IN class protocols. + +;; 3.3.12. PTR RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / PTRDNAME / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; PTRDNAME A <domain-name> which points to some location in the +;; domain name space. + +;; PTR records cause no additional section processing. These RRs are used +;; in special domains to point to some other location in the domain space. +;; These records are simple data, and don't imply any special processing +;; similar to that performed by CNAME, which identifies aliases. See the +;; description of the IN-ADDR.ARPA domain for an example. + +;; 3.3.13. SOA RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / MNAME / +;; / / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / RNAME / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | SERIAL | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | REFRESH | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | RETRY | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | EXPIRE | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; | MINIMUM | +;; | | +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; MNAME The <domain-name> of the name server that was the +;; original or primary source of data for this zone. + +;; RNAME A <domain-name> which specifies the mailbox of the +;; person responsible for this zone. + +;; SERIAL The unsigned 32 bit version number of the original copy +;; of the zone. Zone transfers preserve this value. This +;; value wraps and should be compared using sequence space +;; arithmetic. + +;; REFRESH A 32 bit time interval before the zone should be +;; refreshed. + +;; RETRY A 32 bit time interval that should elapse before a +;; failed refresh should be retried. + +;; EXPIRE A 32 bit time value that specifies the upper limit on +;; the time interval that can elapse before the zone is no +;; longer authoritative. + +;; MINIMUM The unsigned 32 bit minimum TTL field that should be +;; exported with any RR from this zone. + +;; SOA records cause no additional section processing. + +;; All times are in units of seconds. + +;; Most of these fields are pertinent only for name server maintenance +;; operations. However, MINIMUM is used in all query operations that +;; retrieve RRs from a zone. Whenever a RR is sent in a response to a +;; query, the TTL field is set to the maximum of the TTL field from the RR +;; and the MINIMUM field in the appropriate SOA. Thus MINIMUM is a lower +;; bound on the TTL field for all RRs in a zone. Note that this use of +;; MINIMUM should occur when the RRs are copied into the response and not +;; when the zone is loaded from a master file or via a zone transfer. The +;; reason for this provison is to allow future dynamic update facilities to +;; change the SOA RR with known semantics. + + +;; 3.3.14. TXT RDATA format + +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ +;; / TXT-DATA / +;; +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ + +;; where: + +;; TXT-DATA + +;; TXT RRs are used to hold descriptive text. The semantics of the text +;; depends on the domain where it is found. + +(defbinary dns-header (:byte-order :big-endian) + ;; A 16 bit identifier assigned by the program that + ;; generates any kind of query. This identifier is copied + ;; the corresponding reply and can be used by the requester + ;; to match up replies to outstanding queries. + (id 0 :type 16) + + ;; A one bit field that specifies whether this message is a + ;; query (0), or a response (1). + (qr 0 :type 1) + + ;; A four bit field that specifies kind of query in this + ;; message. This value is set by the originator of a query + ;; and copied into the response. The values are: + ;; + ;; 0 a standard query (QUERY) + ;; 1 an inverse query (IQUERY) + ;; 2 a server status request (STATUS) + ;; 3-15 reserved for future use + (opcode 0 :type 4) + + ;; Authoritative Answer - this bit is valid in responses, + ;; and specifies that the responding name server is an + ;; authority for the domain name in question section. + (aa nil :type 1) + + ;; TrunCation - specifies that this message was truncated + ;; due to length greater than that permitted on the + ;; transmission channel. + (tc nil :type 1) + + ;; Recursion Desired - this bit may be set in a query and + ;; is copied into the response. If RD is set, it directs + ;; the name server to pursue the query recursively. + ;; Recursive query support is optional. + (rd nil :type 1) + + ;; Recursion Available - this be is set or cleared in a + ;; response, and denotes whether recursive query support is + ;; available in the name server. + (ra nil :type 1) + + ;; Reserved for future use. Must be zero in all queries and + ;; responses. + (z 0 :type 3) + + ;; Response code - this 4 bit field is set as part of + ;; responses. The values have the following + ;; interpretation: + ;; 0 No error condition + ;; 1 Format error - The name server was + ;; unable to interpret the query. + ;; 2 Server failure - The name server was + ;; unable to process this query due to a + ;; problem with the name server. + ;; 3 Name Error - Meaningful only for + ;; responses from an authoritative name + ;; server, this code signifies that the + ;; domain name referenced in the query does + ;; not exist. + ;; 4 Not Implemented - The name server does + ;; not support the requested kind of query. + ;; 5 Refused - The name server refuses to + ;; perform the specified operation for + ;; policy reasons. For example, a name + ;; server may not wish to provide the + ;; information to the particular requester, + ;; or a name server may not wish to perform + ;; a particular operation (e.g., zone + ;; transfer) for particular data. + ;; 6-15 Reserved for future use. + (rcode 0 :type 4) + + ;; an unsigned 16 bit integer specifying the number of + ;; entries in the question section. + (qdcount 0 :type 16) + + ;; an unsigned 16 bit integer specifying the number of + ;; resource records in the answer section. + (ancount 0 :type 16) + + ;; an unsigned 16 bit integer specifying the number of name + ;; server resource records in the authority records + ;; section. + (nscount 0 :type 16) + + ;; an unsigned 16 bit integer specifying the number of + ;; resource records in the additional records section. + (arcount 0 :type 16)) + + +;; Representation of DNS QNAMEs. +;; +;; A QNAME can be either made up entirely of labels, which is +;; basically a list of strings, or be terminated with a pointer to an +;; offset within the original message. + +(deftype qname-field () + '(or + ;; pointer + (unsigned-byte 14) + ;; label + string)) + +(defstruct qname + (start-at 0 :type (unsigned-byte 14)) + (names #() :type (vector qname-field))) + +;; Domain names in questions and resource records are represented as a +;; sequence of labels, where each label consists of a length octet +;; followed by that number of octets. +;; +;; The domain name terminates with the zero length octet for the null +;; label of the root. Note that this field may be an odd number of +;; octets; no padding is used. +(declaim (ftype (function (stream) (values qname integer)) read-qname)) +(defun read-qname (stream) + "Reads a DNS QNAME from STREAM." + + (let ((start-at (file-position stream))) + (iter (for byte next (read-byte stream)) + ;; Each fragment is collected into this byte vector pre-allocated + ;; with the correct size. + (for fragment = (make-array byte :element-type '(unsigned-byte 8) + :fill-pointer 0)) + + ;; If the bit sequence (1 1) is encountered at the beginning of + ;; the fragment, a qname pointer is being read. + (let ((byte-copy byte)) + (when (equal #b11 (lisp-binary/integer:pop-bits 2 8 byte-copy)) + (let ((next (read-byte stream))) + (lisp-binary/integer:push-bits byte-copy 8 next) + (collect next into fragments result-type vector) + (sum 2 into size) + (finish)))) + + ;; Total size is needed, count for each iteration byte, plus its + ;; own value. + (sum (+ 1 byte) into size) + (until (equal byte 0)) + + ;; On each iteration, this will interpret the current byte as an + ;; unsigned integer and read from STREAM an equivalent amount of + ;; times to assemble the current fragment. + ;; + ;; Advancing the stream like this also ensures that the next + ;; iteration occurs on a new fragment or the final terminating + ;; byte. + (dotimes (_ byte (collect (babel:octets-to-string fragment) + into fragments result-type vector)) + (vector-push (read-byte stream) fragment)) + + (finally (return (values (make-qname :start-at start-at + :names fragments) + size)))))) + +(declaim (ftype (function (stream qname)) write-qname)) +(defun write-qname (stream qname) + "Write a DNS qname to STREAM." + + ;; Write each fragment starting with its (byte-) length, followed by + ;; the bytes. + (iter (for fragment in-vector (qname-names qname)) + (for bytes = (babel:string-to-octets fragment)) + (write-byte (length bytes) stream) + (iter (for byte in-vector bytes) + (write-byte byte stream))) + + ;; Always finish off the serialisation with a null-byte! + (write-byte 0 stream)) + +(define-enum dns-type 2 + (:byte-order :big-endian) + + ;; http://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml + (A 1) + (NS 2) + (CNAME 5) + (SOA 6) + (PTR 12) + (MX 15) + (TXT 16) + (SRV 33) + (AAAA 28) + + ;; ANY typically wants SOA, MX, NS and MX + (ANY 255)) + +(defbinary dns-question (:byte-order :big-endian :export t) + ;; a domain name represented + (qname "" :type (custom :lisp-type qname + :reader #'read-qname + :writer #'write-qname)) + + ;; a two octet code which specifies the type of the query. + (qtype 0 :type dns-type) + + ;; a two octet code that specifies the class of the query. For + ;; example, the QCLASS field is IN for the Internet. + (qclass 0 :type 16)) + +(defbinary dns-rr (:byte-order :big-endian :export t) + (name nil :type (custom :lisp-type qname + :reader #'read-qname + :writer #'write-qname)) + + ;; two octets containing one of the RR type codes. This field + ;; specifies the meaning of the data in the RDATA field. + (type 0 :type dns-type) + + ;; two octets which specify the class of the data in the RDATA + ;; field. + (class 0 :type 16) + + ;; a 32 bit unsigned integer that specifies the time interval (in + ;; seconds) that the resource record may be cached before it should + ;; be discarded. Zero values are interpreted to mean that the RR + ;; can only be used for the transaction in progress, and should not + ;; be cached. + (ttl 0 :type 32) + + ;; an unsigned 16 bit integer that specifies the length in octets + ;; of the RDATA field. + (rdlength 0 :type 16) + + ;; a variable length string of octets that describes the resource. + ;; The format of this information varies according to the TYPE and + ;; CLASS of the resource record. For example, the if the TYPE is A + ;; and the CLASS is IN, the RDATA field is a 4 octet ARPA Internet + ;; address. + (rdata #() :type (eval (case type + ;; A 32-bit internet address in its + ;; canonical representation of 4 integers. + ((A) '(simple-array (unsigned-byte 8) (4))) + + ;; TODO(tazjin): Deal with multiple strings in single RRDATA + ;; One or more <character-string>s. + ((TXT) '(counted-string 1)) + + ;; A <domain-name> which specifies the + ;; canonical or primary name for the + ;; owner. The owner name is an alias. + ((CNAME) '(custom + :lisp-type qname + :reader #'read-qname + :writer #'write-qname)) + + ;; A <domain-name> which specifies a host + ;; which should be authoritative for the + ;; specified class and domain. + ((NS) '(custom + :lisp-type qname + :reader #'read-qname + :writer #'write-qname)) + (otherwise `(simple-array (unsigned-byte 8) (,rdlength))))))) + +(defbinary dns-message (:byte-order :big-endian :export t) + (header nil :type dns-header) + + ;; the question for the name server + (question #() :type (simple-array dns-question ((dns-header-qdcount header)))) + + ;; ;; RRs answering the question + ;; (answer #() :type (simple-array (unsigned-byte 8) (16))) + (answer #() :type (simple-array dns-rr ((dns-header-ancount header)))) + + ;; ;; ;; RRs pointing toward an authority + (authority #() :type (simple-array dns-rr ((dns-header-nscount header)))) + + ;; ;; RRs holding additional information + (additional #() :type (simple-array dns-rr ((dns-header-arcount header))))) diff --git a/lisp/dns/package.lisp b/lisp/dns/package.lisp new file mode 100644 index 000000000000..2b8bfaa8bcc4 --- /dev/null +++ b/lisp/dns/package.lisp @@ -0,0 +1,11 @@ +(defpackage #:dns + (:documentation "Simple DNS resolver in Common Lisp") + (:use #:cl #:iterate #:lisp-binary) + (:export + ;; Individual lookup functions + #:lookup-txt #:lookup-mx #:lookup-cname #:lookup-a #:lookup-ns + + ;; Useful accessors + #:dns-message-header #:dns-message-answer #:dns-message-question + #:dns-rr-name #:dns-rr-type #:dns-rr-ttl #:dns-rr-rdata + #:dns-question-qname #:dns-question-qtype)) diff --git a/lisp/klatre/OWNERS b/lisp/klatre/OWNERS new file mode 100644 index 000000000000..ce7e0e37ee4f --- /dev/null +++ b/lisp/klatre/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - grfn diff --git a/lisp/klatre/default.nix b/lisp/klatre/default.nix new file mode 100644 index 000000000000..2c7bb6490a9f --- /dev/null +++ b/lisp/klatre/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "klatre"; + + deps = with depot.third_party.lisp; [ + local-time + ]; + + srcs = [ + ./package.lisp + ./klatre.lisp + ]; +} diff --git a/lisp/klatre/klatre.lisp b/lisp/klatre/klatre.lisp new file mode 100644 index 000000000000..79c7259752c6 --- /dev/null +++ b/lisp/klatre/klatre.lisp @@ -0,0 +1,119 @@ +(in-package #:klatre) +(declaim (optimize (safety 3))) + +(defmacro comment (&rest _) + (declare (ignore _))) + +(defun posp (n) (> n 0)) + +;;; Sequence utilities + +(defun slice (vector start end) + (make-array (- end start) + :element-type (array-element-type vector) + :displaced-to vector + :displaced-index-offset start)) + +(defun chunk-vector (size vector &key start end sharedp) + (check-type size (integer 1)) + (loop + with slicer = (if sharedp #'slice #'subseq) + and low = (or start 0) + and high = (or end (length vector)) + for s from low below high by size + for e from (+ low size) by size + collect (funcall slicer vector s (min e high)))) + +(defun chunk-list/unbounded (size list) + (loop + for front = list then next + for next = (nthcdr size front) + collect (ldiff front next) + while next)) + +(defun chunk-list/bounded (size list upper-limit) + (loop + for front = list then next + for next = (nthcdr (min size upper-limit) front) + collect (ldiff front next) + do (decf upper-limit size) + while (and next (plusp upper-limit)))) + +(defun chunk-list (size list &key (start 0) end) + "Returns successive chunks of list of size SIZE, starting at START and ending +at END." + (declare (inline chunk-list/bounded chunk-list/unbounded)) + (check-type size (integer 1)) + (let ((list (nthcdr start list))) + (when list + (if end + (chunk-list/bounded size list (- end start)) + (chunk-list/unbounded size list))))) + +(defun mapconcat (func lst sep) + "Apply FUNC to each element of LST, and concat the results as strings, +separated by SEP." + (check-type lst cons) + (check-type sep (simple-array character (*))) + (let ((vs (make-array 0 + :element-type 'character + :fill-pointer 0 + :adjustable t)) + (lsep (length sep))) + (mapcar #'(lambda (str) + (let ((nstr (the (simple-array character (*)) + (funcall func str)))) + (dotimes (j (length nstr) j) + (vector-push-extend (char nstr (the fixnum j)) vs)) + (dotimes (k lsep k) + (vector-push-extend (char sep (the fixnum k)) vs)))) + lst) + vs)) + +;;; +;;; String handling +;;; + +(defparameter dottime-format + '((:year 4) #\- (:month 2) #\- (:day 2) + #\T + (:hour 2) #\· (:min 2)) + "`:LOCAL-TIME' format specifier for dottime") + +(defun format-dottime (timestamp &optional (offset 0)) + "Return TIMESTAMP formatted as dottime, with a specified offset or +00" + (check-type timestamp local-time:timestamp) + (concatenate 'string + (local-time:format-timestring nil timestamp + :format dottime-format + :timezone local-time:+utc-zone+) + (format-dottime-offset offset))) + +(defun format-dottime-offset (offset) + "Render OFFSET in hours in the format specified by dottime." + (check-type offset integer) + (concatenate 'string + ; render sign manually since format prints it after padding + (if (>= offset 0) "+" "-") + (format nil "~2,'0D" (abs offset)))) + +(comment + (format-dottime (local-time:now)) + (format-dottime (local-time:now) 2)) + +(defun try-parse-integer (str) + "Attempt to parse STR as an integer, returning nil if it is invalid." + (check-type str string) + (handler-case (parse-integer str) + (#+sbcl sb-int:simple-parse-error + #-sbcl parse-error (_) (declare (ignore _)) nil))) + +;;; +;;; Function utilities +;;; + +(defun partial (f &rest args) + "Return a function that calls F with ARGS prepended to any remaining + arguments" + (lambda (&rest more-args) + (apply f (append args more-args)))) diff --git a/lisp/klatre/package.lisp b/lisp/klatre/package.lisp new file mode 100644 index 000000000000..41174bbb3cf7 --- /dev/null +++ b/lisp/klatre/package.lisp @@ -0,0 +1,16 @@ +(defpackage #:klatre + (:documentation "Grab-bag utility library for Common Lisp") + (:use #:cl) + (:export + ;; Miscellanious utilities + #:comment #:posp + + ;; Sequence functions + #:chunk-list #:mapconcat + + ;; String handling + #:+dottime-format+ #:format-dottime + #:try-parse-integer #:format-dottime-offset + + ;; Function utilities + #:partial)) diff --git a/net/alcoholic_jwt/.gitignore b/net/alcoholic_jwt/.gitignore new file mode 100644 index 000000000000..143b1ca01474 --- /dev/null +++ b/net/alcoholic_jwt/.gitignore @@ -0,0 +1,4 @@ + +/target/ +**/*.rs.bk +Cargo.lock diff --git a/net/alcoholic_jwt/Cargo.lock b/net/alcoholic_jwt/Cargo.lock new file mode 100644 index 000000000000..4f541cd8acc7 --- /dev/null +++ b/net/alcoholic_jwt/Cargo.lock @@ -0,0 +1,192 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "alcoholic_jwt" +version = "1.0.1" +dependencies = [ + "base64", + "openssl", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "bitflags" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da1976d75adbe5fbc88130ecd119529cf1cc6a93ae1546d8696ee66f0d21af1" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "libc" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "openssl" +version = "0.10.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-sys" +version = "0.9.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "proc-macro2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" + +[[package]] +name = "serde_derive" +version = "1.0.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" diff --git a/net/alcoholic_jwt/Cargo.toml b/net/alcoholic_jwt/Cargo.toml new file mode 100644 index 000000000000..4430e0687978 --- /dev/null +++ b/net/alcoholic_jwt/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "alcoholic_jwt" +description = "Library for validation of RS256 JWTs" +version = "1.0.1" +authors = ["Vincent Ambo <mail@tazj.in>"] +keywords = ["jwt", "token", "jwks"] +categories = ["authentication"] +license = "GPL-3.0-or-later" +repository = "https://code.tvl.fyi/tree/net/alcoholic_jwt" + +[dependencies] +base64 = "0.10" +openssl = "0.10" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" diff --git a/net/alcoholic_jwt/LICENSE b/net/alcoholic_jwt/LICENSE new file mode 100644 index 000000000000..94a9ed024d38 --- /dev/null +++ b/net/alcoholic_jwt/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/net/alcoholic_jwt/README.md b/net/alcoholic_jwt/README.md new file mode 100644 index 000000000000..b9ff57df4be3 --- /dev/null +++ b/net/alcoholic_jwt/README.md @@ -0,0 +1,62 @@ +alcoholic_jwt +============= + +[](https://travis-ci.org/aprilabank/alcoholic_jwt) + +This is a library for **validation** of **RS256** JWTs using keys from +a JWKS. Nothing more, nothing less. + +RS256 is the most commonly used asymmetric signature mechanism for +JWTs, encountered in for example [Google][]'s or [Aprila][]'s APIs. + +The name of the library stems from the potential side-effects of +trying to use the other Rust libraries that are made for similar +purposes. + +## Usage overview + +You are retrieving JWTs from some authentication provider that uses +`RS256` signatures and provides its public keys in [JWKS][] format. + +Example for a token that provides the key ID used for signing in the +[`kid` claim][]: + +```rust +extern crate alcoholic_jwt; + +use alcoholic_jwt::{JWKS, Validation, validate, token_kid}; + +// The function implied here would usually perform an HTTP-GET +// on the JWKS-URL for an authentication provider and deserialize +// the result into the `alcoholic_jwt::JWKS`-struct. +let jwks: JWKS = jwks_fetching_function(); + +let token: String = some_token_fetching_function(); + +// Several types of built-in validations are provided: +let validations = vec![ + Validation::Issuer("auth.test.aprila.no".into()), + Validation::SubjectPresent, +]; + +// If a JWKS contains multiple keys, the correct KID first +// needs to be fetched from the token headers. +let kid = token_kid(&token) + .expect("Failed to decode token headers") + .expect("No 'kid' claim present in token"); + +let jwk = jwks.find(&kid).expect("Specified key not found in set"); + +validate(token, jwk, validations).expect("Token validation has failed!"); +``` + +## Under the hood + +This library aims to only use trustworthy off-the-shelf components to +do the work. Cryptographic operations are provided by the `openssl` +crate, JSON-serialisation is provided by `serde_json`. + +[Google]: https://www.google.com/ +[Aprila]: https://www.aprila.no/ +[JWKS]: https://tools.ietf.org/html/rfc7517 +[`kid` claim]: https://tools.ietf.org/html/rfc7515#section-4.1.4 diff --git a/net/alcoholic_jwt/default.nix b/net/alcoholic_jwt/default.nix new file mode 100644 index 000000000000..c6b84fb7aba3 --- /dev/null +++ b/net/alcoholic_jwt/default.nix @@ -0,0 +1,9 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; + buildInputs = with pkgs; [ + openssl + pkgconfig + ]; +} diff --git a/net/alcoholic_jwt/src/lib.rs b/net/alcoholic_jwt/src/lib.rs new file mode 100644 index 000000000000..4acd8d1e90db --- /dev/null +++ b/net/alcoholic_jwt/src/lib.rs @@ -0,0 +1,465 @@ +// Copyright (C) 2018 Aprila Bank ASA +// +// alcoholic_jwt is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +//! Implements a library for for **validation** of **RS256** JWTs +//! using keys from a JWKS. Nothing more, nothing less. +//! +//! The name of the library stems from the potential side-effects of +//! trying to use the other Rust libraries that are made for similar +//! purposes. +//! +//! This library is specifically aimed at developers that consume +//! tokens from services which provide their RSA public keys in +//! [JWKS][] format. +//! +//! ## Usage example (token with `kid`-claim) +//! +//! ```rust +//! # extern crate serde_json; +//! extern crate alcoholic_jwt; +//! +//! use alcoholic_jwt::{JWKS, Validation, validate, token_kid}; +//! +//! # fn some_token_fetching_function() -> &'static str { +//! # "eyJraWQiOiI4ckRxOFB3MEZaY2FvWFdURVZRbzcrVGYyWXpTTDFmQnhOS1BDZWJhYWk0PSIsImFsZyI6IlJTMjU2IiwidHlwIjoiSldUIn0.eyJpc3MiOiJhdXRoLnRlc3QuYXByaWxhLm5vIiwiaWF0IjoxNTM2MDUwNjkzLCJleHAiOjE1MzYwNTQyOTMsInN1YiI6IjQyIiwiZXh0Ijoic21va2V0ZXN0IiwicHJ2IjoiYXJpc3RpIiwic2NwIjoicHJvY2VzcyJ9.gOLsv98109qLkmRK6Dn7WWRHLW7o8W78WZcWvFZoxPLzVO0qvRXXRLYc9h5chpfvcWreLZ4f1cOdvxv31_qnCRSQQPOeQ7r7hj_sPEDzhKjk-q2aoNHaGGJg1vabI--9EFkFsGQfoS7UbMMssS44dgR68XEnKtjn0Vys-Vzbvz_CBSCH6yQhRLik2SU2jR2L7BoFvh4LGZ6EKoQWzm8Z-CHXLGLUs4Hp5aPhF46dGzgAzwlPFW4t9G4DciX1uB4vv1XnfTc5wqJch6ltjKMde1GZwLR757a8dJSBcmGWze3UNE2YH_VLD7NCwH2kkqr3gh8rn7lWKG4AUIYPxsw9CB" +//! # } +//! +//! # fn jwks_fetching_function() -> JWKS { +//! # let jwks_json = "{\"keys\":[{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"8rDq8Pw0FZcaoXWTEVQo7+Tf2YzSL1fBxNKPCebaai4=\",\"n\":\"l4UTgk1zr-8C8utt0E57DtBV6qqAPWzVRrIuQS2j0_hp2CviaNl5XzGRDnB8gwk0Hx95YOhJupAe6RNq5ok3fDdxL7DLvppJNRLz3Ag9CsmDLcbXgNEQys33fBJaPw1v3GcaFC4tisU5p-o1f5RfWwvwdBtdBfGiwT1GRvbc5sFx6M4iYjg9uv1lNKW60PqSJW4iDYrfqzZmB0zF1SJ0BL_rnQZ1Wi_UkFmNe9arM8W9tI9T3Ie59HITFuyVSTCt6qQEtSfa1e5PiBaVuV3qoFI2jPBiVZQ6LPGBWEDyz4QtrHLdECPPoTF30NN6TSVwwlRbCuUUrdNdXdjYe2dMFQ\",\"e\":\"DhaD5zC7mzaDvHO192wKT_9sfsVmdy8w8T8C9VG17_b1jG2srd3cmc6Ycw-0blDf53Wrpi9-KGZXKHX6_uIuJK249WhkP7N1SHrTJxO0sUJ8AhK482PLF09Qtu6cUfJqY1X1y1S2vACJZItU4Vjr3YAfiVGQXeA8frAf7Sm4O1CBStCyg6yCcIbGojII0jfh2vSB-GD9ok1F69Nmk-R-bClyqMCV_Oq-5a0gqClVS8pDyGYMgKTww2RHgZaFSUcG13KeLMQsG2UOB2OjSC8FkOXK00NBlAjU3d0Vv-IamaLIszO7FQBY3Oh0uxNOvIE9ofQyCOpB-xIK6V9CTTphxw\"}]}"; +//! # serde_json::from_str(jwks_json).unwrap() +//! # } +//! # +//! // The function implied here would usually perform an HTTP-GET +//! // on the JWKS-URL for an authentication provider and deserialize +//! // the result into the `alcoholic_jwt::JWKS`-struct. +//! let jwks: JWKS = jwks_fetching_function(); +//! +//! let token = some_token_fetching_function(); +//! +//! // Several types of built-in validations are provided: +//! let validations = vec![ +//! Validation::Issuer("auth.test.aprila.no".into()), +//! Validation::SubjectPresent, +//! ]; +//! +//! // If a JWKS contains multiple keys, the correct KID first +//! // needs to be fetched from the token headers. +//! let kid = token_kid(&token) +//! .expect("Failed to decode token headers") +//! .expect("No 'kid' claim present in token"); +//! +//! let jwk = jwks.find(&kid).expect("Specified key not found in set"); +//! +//! validate(token, jwk, validations).expect("Token validation has failed!"); +//! ``` +//! +//! [JWKS]: https://tools.ietf.org/html/rfc7517 + +#[macro_use] extern crate serde_derive; + +extern crate base64; +extern crate openssl; +extern crate serde; +extern crate serde_json; + +use base64::{URL_SAFE_NO_PAD, Config, DecodeError}; +use openssl::bn::BigNum; +use openssl::error::ErrorStack; +use openssl::hash::MessageDigest; +use openssl::pkey::{Public, PKey}; +use openssl::rsa::Rsa; +use openssl::sign::Verifier; +use serde::de::DeserializeOwned; +use serde_json::Value; +use std::time::{UNIX_EPOCH, Duration, SystemTime}; + +#[cfg(test)] +mod tests; + +/// URL-safe character set without padding that allows trailing bits, +/// which appear in some JWT implementations. +/// +/// Note: The functions on `base64::Config` are not marked `const`, +/// and the constructors are not exported, which is why this is +/// implemented as a function. +fn jwt_forgiving() -> Config { + URL_SAFE_NO_PAD.decode_allow_trailing_bits(true) +} + +/// JWT algorithm used. The only supported algorithm is currently +/// RS256. +#[derive(Clone, Deserialize, Debug)] +enum KeyAlgorithm { RS256 } + +/// Type of key contained in a JWT. The only supported key type is +/// currently RSA. +#[derive(Clone, Deserialize, Debug)] +enum KeyType { RSA } + +/// Representation of a single JSON Web Key. See [RFC +/// 7517](https://tools.ietf.org/html/rfc7517#section-4). +#[allow(dead_code)] // kty & alg only constrain deserialisation, but aren't used +#[derive(Clone, Debug, Deserialize)] +pub struct JWK { + kty: KeyType, + alg: Option<KeyAlgorithm>, + kid: Option<String>, + + // Shared modulus + n: String, + + // Public key exponent + e: String, +} + +/// Representation of a set of JSON Web Keys. See [RFC +/// 7517](https://tools.ietf.org/html/rfc7517#section-5). +#[derive(Clone, Debug, Deserialize)] +pub struct JWKS { + // This is a vector instead of some kind of map-like structure + // because key IDs are in fact optional. + // + // Technically having multiple keys with the same KID would not + // violate the JWKS-definition either, but behaviour in that case + // is unspecified. + keys: Vec<JWK>, +} + +impl JWKS { + /// Attempt to find a JWK by its key ID. + pub fn find(&self, kid: &str) -> Option<&JWK> { + self.keys.iter().find(|jwk| jwk.kid == Some(kid.into())) + } +} + +/// Representation of an undecoded JSON Web Token. See [RFC +/// 7519](https://tools.ietf.org/html/rfc7519). +struct JWT<'a> (&'a str); + +/// Representation of a decoded and validated JSON Web Token. +/// +/// Specific claim fields are only decoded internally in the library +/// for validation purposes, while it is generally up to the consumer +/// of the validated JWT what structure they would like to impose. +pub struct ValidJWT { + /// JOSE header of the JSON Web Token. Certain fields are + /// guaranteed to be present in this header, consult section 5 of + /// RFC7519 for more information. + pub headers: Value, + + /// Claims (i.e. primary data) contained in the JSON Web Token. + /// While there are several registered and recommended headers + /// (consult section 4.1 of RFC7519), the presence of no field is + /// guaranteed in these. + pub claims: Value, +} + +/// Possible token claim validations. This enumeration only covers +/// common use-cases, for other types of validations the user is +/// encouraged to inspect the claim set manually. +pub enum Validation { + /// Validate that the issuer ("iss") claim matches a specified + /// value. + Issuer(String), + + /// Validate that the audience ("aud") claim matches a specified + /// value. + Audience(String), + + /// Validate that a subject value is present. + SubjectPresent, + + /// Validate that the expiry time of the token ("exp"-claim) has + /// not yet been reached. + NotExpired, +} + +/// Possible results of a token validation. +#[derive(Debug)] +pub enum ValidationError { + /// Invalid number of token components (not a JWT?) + InvalidComponents, + + /// Token segments had invalid base64-encoding. + InvalidBase64(DecodeError), + + /// Decoding of the provided JWK failed. + InvalidJWK, + + /// Signature validation failed, i.e. because of a non-matching + /// public key. + InvalidSignature, + + /// An OpenSSL operation failed along the way at a point at which + /// a more specific error variant could not be constructed. + OpenSSL(ErrorStack), + + /// JSON decoding into a provided type failed. + JSON(serde_json::Error), + + /// One or more claim validations failed. This variant contains + /// human-readable validation errors. + InvalidClaims(Vec<&'static str>), +} + +type JWTResult<T> = Result<T, ValidationError>; + +impl From<ErrorStack> for ValidationError { + fn from(err: ErrorStack) -> Self { ValidationError::OpenSSL(err) } +} + +impl From<serde_json::Error> for ValidationError { + fn from(err: serde_json::Error) -> Self { ValidationError::JSON(err) } +} + +impl From<DecodeError> for ValidationError { + fn from(err: DecodeError) -> Self { ValidationError::InvalidBase64(err) } +} + +/// Attempt to extract the `kid`-claim out of a JWT's header claims. +/// +/// This function is normally used when a token provider has multiple +/// public keys in rotation at the same time that could all still have +/// valid tokens issued under them. +/// +/// This is only safe if the key set containing the currently allowed +/// key IDs is fetched from a trusted source. +pub fn token_kid(token: &str) -> JWTResult<Option<String>> { + // Fetch the header component of the JWT by splitting it out and + // dismissing the rest. + let parts: Vec<&str> = token.splitn(2, '.').collect(); + if parts.len() != 2 { + return Err(ValidationError::InvalidComponents); + } + + // Decode only the first part of the token into a specialised + // representation: + #[derive(Deserialize)] + struct KidOnly { + kid: Option<String>, + } + + let kid_only: KidOnly = deserialize_part(parts[0])?; + + Ok(kid_only.kid) +} + +/// Validate the signature of a JSON Web Token and optionally apply +/// claim validations. Signatures are always verified before claims, +/// and if a signature verification passes *all* claim validations are +/// run and returned. +/// +/// If validation succeeds a representation of the token is returned +/// that contains the header and claims as simple JSON values. +/// +/// It is the user's task to ensure that the correct JWK is passed in +/// for validation. +pub fn validate(token: &str, + jwk: &JWK, + validations: Vec<Validation>) -> JWTResult<ValidJWT> { + let jwt = JWT(token); + let public_key = public_key_from_jwk(&jwk)?; + validate_jwt_signature(&jwt, public_key)?; + + // Split out all three parts of the JWT this time, deserialising + // the first and second as appropriate. + let parts: Vec<&str> = jwt.0.splitn(3, '.').collect(); + if parts.len() != 3 { + // This is unlikely considering that validation has already + // been performed at this point, but better safe than sorry. + return Err(ValidationError::InvalidComponents) + } + + // Perform claim validations before constructing the valid token: + let partial_claims = deserialize_part(parts[1])?; + validate_claims(partial_claims, validations)?; + + let headers = deserialize_part(parts[0])?; + let claims = deserialize_part(parts[1])?; + let valid_jwt = ValidJWT { headers, claims }; + + Ok(valid_jwt) +} + +// Internal implementation +// +// The functions in the following section are not part of the public +// API of this library. + +/// Decode a single key fragment (base64-url encoded integer) to an +/// OpenSSL BigNum. +fn decode_fragment(fragment: &str) -> JWTResult<BigNum> { + let bytes = base64::decode_config(fragment, jwt_forgiving()) + .map_err(|_| ValidationError::InvalidJWK)?; + + BigNum::from_slice(&bytes).map_err(Into::into) +} + +/// Decode an RSA public key from a JWK by constructing it directly +/// from the public RSA key fragments. +fn public_key_from_jwk(jwk: &JWK) -> JWTResult<Rsa<Public>> { + let jwk_n = decode_fragment(&jwk.n)?; + let jwk_e = decode_fragment(&jwk.e)?; + Rsa::from_public_components(jwk_n, jwk_e).map_err(Into::into) +} + +/// Decode a base64-URL encoded string and deserialise the resulting +/// JSON. +fn deserialize_part<T: DeserializeOwned>(part: &str) -> JWTResult<T> { + let json = base64::decode_config(part, jwt_forgiving())?; + serde_json::from_slice(&json).map_err(Into::into) +} + +/// Validate the signature on a JWT using a provided public key. +/// +/// A JWT is made up of three components (headers, claims, signature) +/// - only the first two are part of the signed data. +fn validate_jwt_signature(jwt: &JWT, key: Rsa<Public>) -> JWTResult<()> { + let key = PKey::from_rsa(key)?; + let mut verifier = Verifier::new(MessageDigest::sha256(), &key)?; + + // Split the token from the back to a maximum of two elements. + // There are technically three components using the same separator + // ('.'), but we are interested in the first two together and + // splitting them is unnecessary. + let token_parts: Vec<&str> = jwt.0.rsplitn(2, '.').collect(); + if token_parts.len() != 2 { + return Err(ValidationError::InvalidComponents); + } + + // Second element of the vector will be the signed payload. + let data = token_parts[1]; + + // First element of the vector will be the (encoded) signature. + let sig_b64 = token_parts[0]; + let sig = base64::decode_config(sig_b64, jwt_forgiving())?; + + // Verify signature by inserting the payload data and checking it + // against the decoded signature. + verifier.update(data.as_bytes())?; + + match verifier.verify(&sig)? { + true => Ok(()), + false => Err(ValidationError::InvalidSignature), + } +} + +/// Internal helper enum for PartialClaims that supports single or +/// multiple audiences +#[derive(Deserialize)] +#[serde(untagged)] +enum Audience { + Single(String), + Multi(Vec<String>) +} + +/// Internal helper struct for claims that are relevant for claim +/// validations. +#[derive(Deserialize)] +struct PartialClaims { + aud: Option<Audience>, + iss: Option<String>, + sub: Option<String>, + exp: Option<u64>, +} + +/// Apply a single validation to the claim set of a token. +fn apply_validation(claims: &PartialClaims, + validation: Validation) -> Result<(), &'static str> { + match validation { + // Validate that an 'iss' claim is present and matches the + // supplied value. + Validation::Issuer(iss) => { + match claims.iss { + None => Err("'iss' claim is missing"), + Some(ref claim) => if *claim == iss { + Ok(()) + } else { + Err("'iss' claim does not match") + } + } + }, + + // Validate that an 'aud' claim is present and matches the + // supplied value. + Validation::Audience(aud) => { + match claims.aud { + None => Err("'aud' claim is missing"), + Some(Audience::Single(ref claim)) => if *claim == aud { + Ok(()) + } else { + Err("'aud' claim does not match") + }, + Some(Audience::Multi(ref claims)) => if claims.contains(&aud) { + Ok(()) + } else { + Err("'aud' claim does not match") + } + } + }, + + Validation::SubjectPresent => match claims.sub { + Some(_) => Ok(()), + None => Err("'sub' claim is missing"), + }, + + Validation::NotExpired => match claims.exp { + None => Err("'exp' claim is missing"), + Some(exp) => { + // Determine the current timestamp in seconds since + // the UNIX epoch. + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + // this is an unrecoverable, critical error. There + // aren't many ways this can occur, other than + // system time being set into the far future or + // this library being used in some sort of future + // museum. + .expect("system time is likely incorrect"); + + // Convert the expiry time (which is also in epoch + // seconds) to a duration. + let exp_duration = Duration::from_secs(exp); + + // The token has not expired if the expiry duration is + // larger than (i.e. in the future from) the current + // time. + if exp_duration > now { + Ok(()) + } else { + Err("token has expired") + } + } + }, + } +} + +/// Apply all requested validations to a partial claim set. +fn validate_claims(claims: PartialClaims, + validations: Vec<Validation>) -> JWTResult<()> { + let validation_errors: Vec<_> = validations.into_iter() + .map(|v| apply_validation(&claims, v)) + .filter_map(|result| match result { + Ok(_) => None, + Err(err) => Some(err), + }) + .collect(); + + if validation_errors.is_empty() { + Ok(()) + } else { + Err(ValidationError::InvalidClaims(validation_errors)) + } +} diff --git a/net/alcoholic_jwt/src/tests.rs b/net/alcoholic_jwt/src/tests.rs new file mode 100644 index 000000000000..81890986f8a2 --- /dev/null +++ b/net/alcoholic_jwt/src/tests.rs @@ -0,0 +1,61 @@ +// Copyright (C) 2018 Aprila Bank ASA +// +// alcoholic_jwt is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +use super::*; + +#[test] +fn test_fragment_decoding() { + let fragment = "ngRRjNbXgPW29oNtF0JgsyyfTwPyEL0u_X16s453X2AOc33XGFxVKLEQ7R_TiMenaKcr-tPifYqgps_deyi0XOr4I3SOdOMtAVKDZJCANe--CANOHZb-meIfjKhCHisvT90fm5Apd6qPRVsXsZ7A8pmClZHKM5fwZUkBv8NsPLm2Xy2sGOZIiwP_7z8m3j0abUzniPQsx2b3xcWimB9vRtshFHN1KgPUf1ALQ5xzLfJnlFkCxC7kmOxKC7_NpQ4kJR_DKzKFV_r3HxTqf-jddHcXIrrMcLQXCSyeLQtLaz7whQ4F-EfL42z4XgwPr4ji3sct2gWL13EqlbE5DDxLKQ"; + let bignum = decode_fragment(fragment).expect("Failed to decode fragment"); + + let expected = "19947781743618558124649689124245117083485690334420160711273532766920651190711502679542723943527557680293732686428091794139998732541701457212387600480039297092835433997837314251024513773285252960725418984381935183495143908023024822433135775773958512751261112853383693442999603704969543668619221464654540065497665889289271044207667765128672709218996183649696030570183970367596949687544839066873508106034650634722970893169823917299050098551447676778961773465887890052852528696684907153295689693676910831376066659456592813140662563597179711588277621736656871685099184755908108451080261403193680966083938080206832839445289"; + assert_eq!(expected, format!("{}", bignum), "Decoded fragment should match "); +} + +#[test] +fn test_decode_find_jwks() { + let json = "{\"keys\":[{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"mUjI\\/rIMLLtung35BKZfdbrqtlEAAYJ4JX\\/SKvnLxJc=\",\"n\":\"ngRRjNbXgPW29oNtF0JgsyyfTwPyEL0u_X16s453X2AOc33XGFxVKLEQ7R_TiMenaKcr-tPifYqgps_deyi0XOr4I3SOdOMtAVKDZJCANe--CANOHZb-meIfjKhCHisvT90fm5Apd6qPRVsXsZ7A8pmClZHKM5fwZUkBv8NsPLm2Xy2sGOZIiwP_7z8m3j0abUzniPQsx2b3xcWimB9vRtshFHN1KgPUf1ALQ5xzLfJnlFkCxC7kmOxKC7_NpQ4kJR_DKzKFV_r3HxTqf-jddHcXIrrMcLQXCSyeLQtLaz7whQ4F-EfL42z4XgwPr4ji3sct2gWL13EqlbE5DDxLKQ\",\"e\":\"GK7oLCDbNPAF59LhvyseqcG04hDnPs58qGYolr_HHmaR4lulWJ90ozx6e4Ut363yKG2p9vwvivR5UIC-aLPtqT2qr-OtjhBFzUFVaMGZ6mPCvMKk0AgMYdOHvWTgBSqQtNJTvl1yYLnhcWyoE2fLQhoEbY9qUyCBCEOScXOZRDpnmBtz5I8q5yYMV6a920J24T_IYbxHgkGcEU2SGg-b1cOMD7Rja7vCfV---CQ2pR4leQ0jufzudDoe7z3mziJm-Ihcdrz2Ujy5kPEMdz6R55prJ-ENKrkD_X4u5aSlSRaetwmHS3oAVkjr1JwUNbqnpM-kOqieqHEp8LUmez-Znw\"}]}"; + let jwks: JWKS = serde_json::from_str(json).expect("Failed to decode JWKS"); + let jwk = jwks.find("mUjI/rIMLLtung35BKZfdbrqtlEAAYJ4JX/SKvnLxJc=") + .expect("Failed to find required JWK"); + + public_key_from_jwk(&jwk).expect("Failed to construct public key from JWK"); +} + +#[test] +fn test_token_kid() { + let jwt = "eyJraWQiOiI4ckRxOFB3MEZaY2FvWFdURVZRbzcrVGYyWXpTTDFmQnhOS1BDZWJhYWk0PSIsImFsZyI6IlJTMjU2IiwidHlwIjoiSldUIn0.eyJpc3MiOiJhdXRoLnRlc3QuYXByaWxhLm5vIiwiaWF0IjoxNTM2MDUwNjkzLCJleHAiOjE1MzYwNTQyOTMsInN1YiI6IjQyIiwiZXh0Ijoic21va2V0ZXN0IiwicHJ2IjoiYXJpc3RpIiwic2NwIjoicHJvY2VzcyJ9.gOLsv98109qLkmRK6Dn7WWRHLW7o8W78WZcWvFZoxPLzVO0qvRXXRLYc9h5chpfvcWreLZ4f1cOdvxv31_qnCRSQQPOeQ7r7hj_sPEDzhKjk-q2aoNHaGGJg1vabI--9EFkFsGQfoS7UbMMssS44dgR68XEnKtjn0Vys-Vzbvz_CBSCH6yQhRLik2SU2jR2L7BoFvh4LGZ6EKoQWzm8Z-CHXLGLUs4Hp5aPhF46dGzgAzwlPFW4t9G4DciX1uB4vv1XnfTc5wqJch6ltjKMde1GZwLR757a8dJSBcmGWze3UNE2YH_VLD7NCwH2kkqr3gh8rn7lWKG4AUIYPxsw9CB"; + + let kid = token_kid(&jwt).expect("Failed to extract token KID"); + assert_eq!(Some("8rDq8Pw0FZcaoXWTEVQo7+Tf2YzSL1fBxNKPCebaai4=".into()), + kid, "Extracted KID did not match expected KID"); +} + +#[test] +fn test_validate_jwt() { + let jwks_json = "{\"keys\":[{\"kty\":\"RSA\",\"alg\":\"RS256\",\"use\":\"sig\",\"kid\":\"8rDq8Pw0FZcaoXWTEVQo7+Tf2YzSL1fBxNKPCebaai4=\",\"n\":\"l4UTgk1zr-8C8utt0E57DtBV6qqAPWzVRrIuQS2j0_hp2CviaNl5XzGRDnB8gwk0Hx95YOhJupAe6RNq5ok3fDdxL7DLvppJNRLz3Ag9CsmDLcbXgNEQys33fBJaPw1v3GcaFC4tisU5p-o1f5RfWwvwdBtdBfGiwT1GRvbc5sFx6M4iYjg9uv1lNKW60PqSJW4iDYrfqzZmB0zF1SJ0BL_rnQZ1Wi_UkFmNe9arM8W9tI9T3Ie59HITFuyVSTCt6qQEtSfa1e5PiBaVuV3qoFI2jPBiVZQ6LPGBWEDyz4QtrHLdECPPoTF30NN6TSVwwlRbCuUUrdNdXdjYe2dMFQ\",\"e\":\"DhaD5zC7mzaDvHO192wKT_9sfsVmdy8w8T8C9VG17_b1jG2srd3cmc6Ycw-0blDf53Wrpi9-KGZXKHX6_uIuJK249WhkP7N1SHrTJxO0sUJ8AhK482PLF09Qtu6cUfJqY1X1y1S2vACJZItU4Vjr3YAfiVGQXeA8frAf7Sm4O1CBStCyg6yCcIbGojII0jfh2vSB-GD9ok1F69Nmk-R-bClyqMCV_Oq-5a0gqClVS8pDyGYMgKTww2RHgZaFSUcG13KeLMQsG2UOB2OjSC8FkOXK00NBlAjU3d0Vv-IamaLIszO7FQBY3Oh0uxNOvIE9ofQyCOpB-xIK6V9CTTphxw\"}]}"; + + let jwks: JWKS = serde_json::from_str(jwks_json) + .expect("Failed to decode JWKS"); + + let jwk = jwks.find("8rDq8Pw0FZcaoXWTEVQo7+Tf2YzSL1fBxNKPCebaai4=") + .expect("Failed to find required JWK"); + + let pkey = public_key_from_jwk(&jwk).expect("Failed to construct public key"); + + let jwt = JWT("eyJraWQiOiI4ckRxOFB3MEZaY2FvWFdURVZRbzcrVGYyWXpTTDFmQnhOS1BDZWJhYWk0PSIsImFsZyI6IlJTMjU2IiwidHlwIjoiSldUIn0.eyJpc3MiOiJhdXRoLnRlc3QuYXByaWxhLm5vIiwiaWF0IjoxNTM2MDUwNjkzLCJleHAiOjE1MzYwNTQyOTMsInN1YiI6IjQyIiwiZXh0Ijoic21va2V0ZXN0IiwicHJ2IjoiYXJpc3RpIiwic2NwIjoicHJvY2VzcyJ9.gOLsv98109qLkmRK6Dn7WWRHLW7o8W78WZcWvFZoxPLzVO0qvRXXRLYc9h5chpfvcWreLZ4f1cOdvxv31_qnCRSQQPOeQ7r7hj_sPEDzhKjk-q2aoNHaGGJg1vabI--9EFkFsGQfoS7UbMMssS44dgR68XEnKtjn0Vys-Vzbvz_CBSCH6yQhRLik2SU2jR2L7BoFvh4LGZ6EKoQWzm8Z-CHXLGLUs4Hp5aPhF46dGzgAzwlPFW4t9G4DciX1uB4vv1XnfTc5wqJch6ltjKMde1GZwLR757a8dJSBcmGWze3UNE2YH_VLD7NCwH2kkqr3gh8rn7lWKG4AUIYPxsw9CB".into()); + + validate_jwt_signature(&jwt, pkey).expect("Validation failed unexpectedly"); +} diff --git a/net/crimp/.gitignore b/net/crimp/.gitignore new file mode 100644 index 000000000000..693699042b1a --- /dev/null +++ b/net/crimp/.gitignore @@ -0,0 +1,3 @@ +/target +**/*.rs.bk +Cargo.lock diff --git a/net/crimp/Cargo.lock b/net/crimp/Cargo.lock new file mode 100644 index 000000000000..9875e6004b38 --- /dev/null +++ b/net/crimp/Cargo.lock @@ -0,0 +1,178 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "cc" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" + +[[package]] +name = "crimp" +version = "0.2.2" +dependencies = [ + "curl", + "serde", + "serde_json", +] + +[[package]] +name = "curl" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "003cb79c1c6d1c93344c7e1201bb51c2148f24ec2bd9c253709d6b2efb796515" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi", +] + +[[package]] +name = "curl-sys" +version = "0.4.45+curl-7.78.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de9e5a72b1c744eb5dd20b2be4d7eb84625070bb5c4ab9b347b70464ab1e62eb" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" + +[[package]] +name = "libz-sys" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "serde" +version = "1.0.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8" + +[[package]] +name = "serde_json" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b10da19a12ad094b59d870ebde26a45402e5b470add4b5fd03c5048a32127" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "socket2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/net/crimp/Cargo.toml b/net/crimp/Cargo.toml new file mode 100644 index 000000000000..13c80f1f6950 --- /dev/null +++ b/net/crimp/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "crimp" +description = "Higher-level Rust API for cURL bindings" +version = "0.2.2" +authors = ["Vincent Ambo <mail@tazj.in>"] +keywords = [ "http", "curl" ] +categories = [ "api-bindings" ] +license = "GPL-3.0-or-later" +repository = "https://github.com/tazjin/crimp" + +[features] +default = [ "json" ] +json = [ "serde", "serde_json"] + +[dependencies] +curl = "0.4" +serde = { version = "1.0", optional = true } +serde_json = { version = "1.0", optional = true } diff --git a/net/crimp/LICENSE b/net/crimp/LICENSE new file mode 100644 index 000000000000..94a9ed024d38 --- /dev/null +++ b/net/crimp/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/net/crimp/README.md b/net/crimp/README.md new file mode 100644 index 000000000000..791c354fc1d9 --- /dev/null +++ b/net/crimp/README.md @@ -0,0 +1,15 @@ +crimp +===== + +[](https://travis-ci.org/tazjin/crimp) +[](https://crates.io/crates/crimp) +[](https://docs.rs/crimp) + +Crimp is an HTTP client interface on top of the [Rust bindings][] to +cURL. + +The documentation for this crate is primarily in the [module +documentation][] + +[Rust bindings]: https://docs.rs/curl +[module documentation]: https://docs.rs/crimp diff --git a/net/crimp/default.nix b/net/crimp/default.nix new file mode 100644 index 000000000000..c6b84fb7aba3 --- /dev/null +++ b/net/crimp/default.nix @@ -0,0 +1,9 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; + buildInputs = with pkgs; [ + openssl + pkgconfig + ]; +} diff --git a/net/crimp/src/lib.rs b/net/crimp/src/lib.rs new file mode 100644 index 000000000000..b52ebc3ef09c --- /dev/null +++ b/net/crimp/src/lib.rs @@ -0,0 +1,502 @@ +// crimp - Higher-level Rust cURL API +// +// Copyright (C) 2019 Vincent Ambo +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +//! # crimp +//! +//! This library provides a simplified API over the [cURL Rust +//! bindings][] that resemble that of higher-level libraries such as +//! [reqwest][]. All calls are synchronous. +//! +//! `crimp` is intended to be used in situations where HTTP client +//! functionality is desired without adding a significant number of +//! dependencies or sacrificing too much usability. +//! +//! Using `crimp` to make HTTP requests is done using a simple +//! builder-pattern style API. For example, to make a `GET`-request +//! and print the result to `stdout`: +//! +//! ```rust +//! use crimp::Request; +//! +//! let response = Request::get("http://httpbin.org/get") +//! .user_agent("crimp test suite").unwrap() +//! .send().unwrap() +//! .as_string().unwrap(); +//! +//! println!("Status: {}\nBody: {}", response.status, response.body); +//! # assert_eq!(response.status, 200); +//! ``` +//! +//! If a feature from the underlying cURL library is missing, the +//! `Request::raw` method can be used as an escape hatch to deal with +//! the handle directly. Should you find yourself doing this, please +//! [file an issue][]. +//! +//! `crimp` does not currently expose functionality for re-using a +//! cURL Easy handle, meaning that keep-alive of HTTP connections and +//! the like is not supported. +//! +//! ## Cargo features +//! +//! All optional features are enabled by default. +//! +//! * `json`: Adds `Request::json` and `Response::as_json` methods +//! which can be used for convenient serialisation of +//! request/response bodies using `serde_json`. This feature adds a +//! dependency on the `serde` and `serde_json` crates. +//! +//! ## Initialisation +//! +//! It is recommended to call the underlying `curl::init` method +//! (re-exported as `crimp::init`) when launching your application to +//! initialise the cURL library. This is not required, but will +//! otherwise occur the first time a request is made. +//! +//! [cURL Rust bindings]: https://docs.rs/curl +//! [reqwest]: https://docs.rs/reqwest +//! [file an issue]: https://github.com/tazjin/crimp/issues + +extern crate curl; + +#[cfg(feature = "json")] extern crate serde; +#[cfg(feature = "json")] extern crate serde_json; + +pub use curl::init; + +use curl::easy::{Auth, Easy, Form, List, Transfer, ReadError, WriteError}; +use std::collections::HashMap; +use std::io::Write; +use std::path::Path; +use std::string::{FromUtf8Error, ToString}; +use std::time::Duration; + +#[cfg(feature = "json")] use serde::Serialize; +#[cfg(feature = "json")] use serde::de::DeserializeOwned; + +#[cfg(test)] +mod tests; + +/// HTTP method to use for the request. +enum Method { + Get, Post, Put, Patch, Delete +} + +/// Certificate types for client-certificate key pairs. +pub enum CertType { + P12, PEM, DER +} + +/// Builder structure for an HTTP request. +/// +/// This is the primary API-type in `crimp`. After creating a new +/// request its parameters are modified using the various builder +/// methods until it is consumed by `send()`. +pub struct Request<'a> { + url: &'a str, + method: Method, + handle: Easy, + headers: List, + body: Body<'a>, +} + +enum Body<'a> { + NoBody, + Form(Form), + + Bytes { + content_type: &'a str, + data: &'a [u8], + }, + + #[cfg(feature = "json")] + Json(Vec<u8>), +} + +/// HTTP responses structure containing response data and headers. +/// +/// By default the `send()` function of the `Request` structure will +/// return a `Response<Vec<u8>>`. Convenience helpers exist for +/// decoding a string via `Response::as_string` or to a +/// `serde`-compatible type with `Response::as_json` (if the +/// `json`-feature is enabled). +#[derive(Debug, PartialEq)] +pub struct Response<T> { + /// HTTP status code of the response. + pub status: u32, + + /// HTTP headers returned from the remote. + pub headers: HashMap<String, String>, + + /// Body data from the HTTP response. + pub body: T, +} + +impl <'a> Request<'a> { + /// Initiate an HTTP request with the given method and URL. + fn new(method: Method, url: &'a str) -> Self { + Request { + url, + method, + handle: Easy::new(), + headers: List::new(), + body: Body::NoBody, + } + } + + /// Initiate a GET request with the given URL. + pub fn get(url: &'a str) -> Self { Request::new(Method::Get, url) } + + /// Initiate a POST request with the given URL. + pub fn post(url: &'a str) -> Self { Request::new(Method::Post, url) } + + /// Initiate a PUT request with the given URL. + pub fn put(url: &'a str) -> Self { Request::new(Method::Put, url) } + + /// Initiate a PATCH request with the given URL. + pub fn patch(url: &'a str) -> Self { Request::new(Method::Patch, url) } + + /// Initiate a DELETE request with the given URL. + pub fn delete(url: &'a str) -> Self { Request::new(Method::Delete, url) } + + /// Add an HTTP header to a request. + pub fn header(mut self, k: &str, v: &str) -> Result<Self, curl::Error> { + self.headers.append(&format!("{}: {}", k, v))?; + Ok(self) + } + + /// Set the `User-Agent` for this request. By default this will be + /// set to cURL's standard user agent. + pub fn user_agent(mut self, agent: &str) -> Result<Self, curl::Error> { + self.handle.useragent(agent)?; + Ok(self) + } + + /// Set the `Authorization` header to a `Bearer` value with the + /// supplied token. + pub fn bearer_auth(mut self, token: &str) -> Result<Self, curl::Error> { + self.headers.append(&format!("Authorization: Bearer {}", token))?; + Ok(self) + } + + /// Set the `Authorization` header to a basic authentication value + /// from the supplied username and password. + pub fn basic_auth(mut self, username: &str, password: &str) -> Result<Self, curl::Error> { + let mut auth = Auth::new(); + auth.basic(true); + self.handle.username(username)?; + self.handle.password(password)?; + self.handle.http_auth(&auth)?; + Ok(self) + } + + /// Configure a TLS client certificate on the request. + /// + /// Depending on whether the certificate file contains the private + /// key or not, calling `tls_client_key` may be required in + /// addition. + /// + /// Consult the documentation for the `ssl_cert` and `ssl_key` + /// functions in `curl::easy::Easy2` for details on supported + /// formats and defaults. + pub fn tls_client_cert<P: AsRef<Path>>(mut self, cert_type: CertType, cert: P) + -> Result<Self, curl::Error> { + self.handle.ssl_cert(cert)?; + self.handle.ssl_cert_type(match cert_type { + CertType::P12 => "P12", + CertType::PEM => "PEM", + CertType::DER => "DER", + })?; + + Ok(self) + } + + /// Configure a TLS client certificate key on the request. + /// + /// Note that this does **not** need to be called again for + /// PKCS12-encoded key pairs which are set via `tls_client_cert`. + /// + /// Currently only PEM-encoded key files are supported. + pub fn tls_client_key<P: AsRef<Path>>(mut self, key: P) -> Result<Self, curl::Error> { + self.handle.ssl_key(key)?; + Ok(self) + } + + /// Configure an encryption password for a TLS client certificate + /// key on the request. + /// + /// This is required in case of an encrypted private key that + /// should be used. + pub fn tls_key_password(mut self, password: &str) -> Result<Self, curl::Error> { + self.handle.key_password(password)?; + Ok(self) + } + + /// Configure a timeout for the request after which the request + /// will be aborted. + pub fn timeout(mut self, timeout: Duration) -> Result<Self, curl::Error> { + self.handle.timeout(timeout)?; + Ok(self) + } + + /// Set custom configuration on the cURL `Easy` handle. + /// + /// This function can be considered an "escape-hatch" from the + /// high-level API which lets users access the internal + /// `curl::easy::Easy` handle and configure options on it + /// directly. + /// + /// ``` + /// # use crimp::Request; + /// let response = Request::get("https://httpbin.org/get") + /// .with_handle(|mut handle| handle.referer("Example-Referer")).unwrap() + /// .send().unwrap(); + /// # + /// # assert!(response.is_success()); + /// ``` + pub fn with_handle<F>(mut self, function: F) -> Result<Self, curl::Error> + where F: FnOnce(&mut Easy) -> Result<(), curl::Error> { + function(&mut self.handle)?; + Ok(self) + } + + /// Add a byte-array body to a request using the specified + /// `Content-Type`. + pub fn body(mut self, content_type: &'a str, data: &'a [u8]) -> Self { + self.body = Body::Bytes { data, content_type }; + self + } + + /// Add a form-encoded body to a request using the `curl::Form` + /// type. + /// + /// ```rust + /// # extern crate curl; + /// # extern crate serde_json; + /// # use crimp::*; + /// # use serde_json::{Value, json}; + /// use curl::easy::Form; + /// + /// let mut form = Form::new(); + /// form.part("some-name") + /// .contents("some-data".as_bytes()) + /// .add().unwrap(); + /// + /// let response = Request::post("https://httpbin.org/post") + /// .user_agent("crimp test suite").unwrap() + /// .form(form) + /// .send().unwrap(); + /// # + /// # assert_eq!(200, response.status, "form POST should succeed"); + /// # assert_eq!( + /// # response.as_json::<Value>().unwrap().body.get("form").unwrap(), + /// # &json!({"some-name": "some-data"}), + /// # "posted form data should match", + /// # ); + /// ``` + /// + /// See the documentation of `curl::easy::Form` for details on how + /// to construct a form body. + pub fn form(mut self, form: Form) -> Self { + self.body = Body::Form(form); + self + } + + /// Add a JSON-encoded body from a serializable type. + #[cfg(feature = "json")] + pub fn json<T: Serialize>(mut self, body: &T) -> Result<Self, serde_json::Error> { + let json = serde_json::to_vec(body)?; + self.body = Body::Json(json); + Ok(self) + } + + /// Send the HTTP request and return a response structure + /// containing the raw body. + pub fn send(mut self) -> Result<Response<Vec<u8>>, curl::Error> { + // Configure request basics: + self.handle.url(self.url)?; + + match self.method { + Method::Get => self.handle.get(true)?, + Method::Post => self.handle.post(true)?, + Method::Put => self.handle.put(true)?, + Method::Patch => self.handle.custom_request("PATCH")?, + Method::Delete => self.handle.custom_request("DELETE")?, + } + + // Create structures in which to store the response data: + let mut headers = HashMap::new(); + let mut body = vec![]; + + // Submit a form value to cURL if it is set and proceed + // pretending that there is no body, as the handling of this + // type of body happens under-the-hood. + if let Body::Form(form) = self.body { + self.handle.httppost(form)?; + self.body = Body::NoBody; + } + + // Optionally set content type if a body payload is configured + // and configure the expected body size (or form payload). + match self.body { + Body::Bytes { content_type, data } => { + self.handle.post_field_size(data.len() as u64)?; + self.headers.append(&format!("Content-Type: {}", content_type))?; + }, + + #[cfg(feature = "json")] + Body::Json(ref data) => { + self.handle.post_field_size(data.len() as u64)?; + self.headers.append("Content-Type: application/json")?; + }, + + // Do not set content-type header at all if there is no + // body, or if the form handler was invoked above. + _ => (), + }; + + // Configure headers on the request: + self.handle.http_headers(self.headers)?; + + { + // Take a scoped transfer from the Easy handle. This makes it + // possible to write data into the above local buffers without + // fighting the borrow-checker: + let mut transfer = self.handle.transfer(); + + // Write the payload if it exists: + match self.body { + Body::Bytes { data, .. } => chunked_read_function(&mut transfer, data)?, + + #[cfg(feature = "json")] + Body::Json(ref json) => chunked_read_function(&mut transfer, json)?, + + // Do nothing if there is no body or if the body is a + // form. + _ => (), + }; + + // Read one header per invocation. Request processing is + // terminated if any header is malformed: + transfer.header_function(|header| { + // Headers are expected to be valid UTF-8 data. If they + // are not, the conversion is lossy. + // + // Technically it is legal for HTTP requests to use + // different encodings, but we don't interface with such + // services for hygienic reasons. + let header = String::from_utf8_lossy(header); + let split = header.splitn(2, ':').collect::<Vec<_>>(); + + // "Malformed" headers are skipped. In most cases this + // will only be the HTTP version statement. + if split.len() != 2 { + return true; + } + + headers.insert( + split[0].trim().to_string(), split[1].trim().to_string() + ); + true + })?; + + // Read the body to the allocated buffer. + transfer.write_function(|data| { + let len = data.len(); + body.write_all(data) + .map(|_| len) + .map_err(|_| WriteError::Pause) + })?; + + transfer.perform()?; + } + + Ok(Response { + status: self.handle.response_code()?, + headers, + body + }) + } +} + +/// Provide a data chunk potentially larger than cURL's initial write +/// buffer to the data reading callback by tracking the offset off +/// already written data. +/// +/// As we manually set the expected upload size, cURL will call the +/// read callback repeatedly until it has all the data it needs. +fn chunked_read_function<'easy, 'data>(transfer: &mut Transfer<'easy, 'data>, + data: &'data [u8]) -> Result<(), curl::Error> { + let mut data = data; + + transfer.read_function(move |mut into| { + let written = into.write(data) + .map_err(|_| ReadError::Abort)?; + + data = &data[written..]; + + Ok(written) + }) +} + +impl <T> Response<T> { + /// Check whether the status code of this HTTP response is a + /// success (i.e. in the 200-299 range). + pub fn is_success(&self) -> bool { + self.status >= 200 && self.status < 300 + } + + /// Check whether a request succeeded using `Request::is_success` + /// and let users provide a closure that creates a custom error + /// from the request if it did not. + /// + /// This function exists for convenience to avoid having to write + /// repetitive `if !response.is_success() { ... }` blocks. + pub fn error_for_status<F, E>(self, closure: F) -> Result<Self, E> + where F: FnOnce(Self) -> E { + if !self.is_success() { + return Err(closure(self)) + } + + Ok(self) + } +} + +impl Response<Vec<u8>> { + /// Attempt to parse the HTTP response body as a UTF-8 encoded + /// string. + pub fn as_string(self) -> Result<Response<String>, FromUtf8Error> { + let body = String::from_utf8(self.body)?; + + Ok(Response { + body, + status: self.status, + headers: self.headers, + }) + } + + /// Attempt to deserialize the HTTP response body from JSON. + #[cfg(feature = "json")] + pub fn as_json<T: DeserializeOwned>(self) -> Result<Response<T>, serde_json::Error> { + let deserialized = serde_json::from_slice(&self.body)?; + + Ok(Response { + body: deserialized, + status: self.status, + headers: self.headers, + }) + } +} diff --git a/net/crimp/src/tests.rs b/net/crimp/src/tests.rs new file mode 100644 index 000000000000..6c2bc4f5b37a --- /dev/null +++ b/net/crimp/src/tests.rs @@ -0,0 +1,152 @@ +// All tests expect an httpbin instance to be available at +// `http://localhost:4662`. +// +// This is easily spun up using Docker by running: +// +// docker run --rm -p 4662:80 kennethreitz/httpbin + +use super::*; +use serde_json::{Value, json}; + +// These tests check whether the correct HTTP method is used in the +// requests. + +#[test] +fn test_http_get() { + let resp = Request::get("http://127.0.0.1:4662/get") + .send().expect("failed to send request"); + + assert!(resp.is_success(), "request should have succeeded"); +} + +#[test] +fn test_http_delete() { + let resp = Request::delete("http://127.0.0.1:4662/delete") + .send().expect("failed to send request"); + + assert_eq!(200, resp.status, "response status should be 200 OK"); +} + +#[test] +fn test_http_put() { + let resp = Request::put("http://127.0.0.1:4662/put") + .send().expect("failed to send request"); + + assert_eq!(200, resp.status, "response status should be 200 OK"); +} + +#[test] +fn test_http_patch() { + let resp = Request::patch("http://127.0.0.1:4662/patch") + .send().expect("failed to send request"); + + assert_eq!(200, resp.status, "response status should be 200 OK"); +} + +// These tests perform various requests with different body payloads +// and verify that those were received correctly by the remote side. + +#[test] +fn test_http_post() { + let body = "test body"; + let response = Request::post("http://127.0.0.1:4662/post") + .user_agent("crimp test suite").expect("failed to set user-agent") + .timeout(Duration::from_secs(5)).expect("failed to set request timeout") + .body("text/plain", &body.as_bytes()) + .send().expect("failed to send request") + .as_json::<Value>().expect("failed to deserialize response"); + + let data = response.body; + + assert_eq!(200, response.status, "response status should be 200 OK"); + + assert_eq!(data.get("data").unwrap(), &json!("test body"), + "test body should have been POSTed"); + + assert_eq!( + data.get("headers").unwrap().get("Content-Type").unwrap(), + &json!("text/plain"), + "Content-Type should be `text/plain`", + ); +} + +#[cfg(feature = "json")] #[test] +fn test_http_post_json() { + let body = json!({ + "purpose": "testing!" + }); + + let response = Request::post("http://127.0.0.1:4662/post") + .user_agent("crimp test suite").expect("failed to set user-agent") + .timeout(Duration::from_secs(5)).expect("failed to set request timeout") + .json(&body).expect("request serialization failed") + .send().expect("failed to send request") + .as_json::<Value>().expect("failed to deserialize response"); + + + let data = response.body; + + assert_eq!(200, response.status, "response status should be 200 OK"); + + assert_eq!(data.get("json").unwrap(), &body, + "test body should have been POSTed"); + + assert_eq!( + data.get("headers").unwrap().get("Content-Type").unwrap(), + &json!("application/json"), + "Content-Type should be `application/json`", + ); +} + +// Tests for different authentication methods that are supported +// out-of-the-box: + +#[test] +fn test_bearer_auth() { + let response = Request::get("http://127.0.0.1:4662/bearer") + .bearer_auth("some-token").expect("failed to set auth header") + .send().expect("failed to send request"); + + assert!(response.is_success(), "authorized request should succeed"); +} + +#[test] +fn test_basic_auth() { + let request = Request::get("http://127.0.0.1:4662/basic-auth/alan_watts/oneness"); + + let response = request + .basic_auth("alan_watts", "oneness").expect("failed to set auth header") + .send().expect("failed to send request"); + + assert!(response.is_success(), "authorized request should succeed"); +} + +#[test] +fn test_large_body() { + // By default cURL buffers seem to be 2^16 bytes in size. The test + // size is therefore 2^16+1. + const BODY_SIZE: usize = 65537; + + let resp = Request::post("http://127.0.0.1:4662/post") + .body("application/octet-stream", &[0; BODY_SIZE]) + .send().expect("sending request") + .as_json::<Value>().expect("JSON deserialisation"); + + // httpbin returns the uploaded data as a string in the `data` + // field. + let data = resp.body.get("data").unwrap().as_str().unwrap(); + + assert_eq!(BODY_SIZE, data.len(), "uploaded data length should be correct"); +} + +// Tests for various other features. + +#[test] +fn test_error_for_status() { + let response = Request::get("http://127.0.0.1:4662/patch") + .send().expect("failed to send request") + .error_for_status(|resp| format!("Response error code: {}", resp.status)); + + assert_eq!(Err("Response error code: 405".into()), response, + "returned error should be converted into Result::Err"); +} diff --git a/net/stomp_erl/.gitignore b/net/stomp_erl/.gitignore new file mode 100644 index 000000000000..8e46d5a07f8f --- /dev/null +++ b/net/stomp_erl/.gitignore @@ -0,0 +1,10 @@ +.eunit +deps +*.o +*.beam +*.plt +erl_crash.dump +ebin +rel/example_project +.concrete/DEV_MODE +.rebar diff --git a/net/stomp_erl/Makefile b/net/stomp_erl/Makefile new file mode 100644 index 000000000000..b3bc54673d0b --- /dev/null +++ b/net/stomp_erl/Makefile @@ -0,0 +1,8 @@ +PROJECT = stomp +PROJECT_DESCRIPTION = STOMP client for Erlang +PROJECT_VERSION = 0.1.0 + +# Whitespace to be used when creating files from templates. +SP = 4 + +include erlang.mk diff --git a/net/stomp_erl/README.md b/net/stomp_erl/README.md new file mode 100644 index 000000000000..21c95a11a2e9 --- /dev/null +++ b/net/stomp_erl/README.md @@ -0,0 +1,78 @@ +STOMP on Erlang +=============== + +`stomp.erl` is a simple Erlang client for the [STOMP protocol][] in version 1.2. + +Currently only subscribing to queues is supported. + +It provides an application called `stomp` which takes configuration of the form: + +```erlang +[{stomp, #{host => "stomp-server.somedomain.sexy", % required + port => 61613, % optional + login => <<"someuser">>, % optional + passcode => <<"hunter2>>, % optional + }}]. +``` + +## Types + +The following types are used in `stomp.erl`, you can include them from +`stomp.hrl`: + +```erlang +%% Client ack modes, refer to the STOMP protocol documentation +-type ack_mode() :: client | client_individual | auto. + +%% Subscriptions are enumerated from 0 +-type sub_id() :: integer(). + +%% Message IDs (for acknowledgements) are simple strings. They are +%% extracted from the 'ack' field of the header in client or client-individual +%% mode, and from the 'message-id' field in auto mode. +-type message_id() :: binary(). + +%% A STOMP message as received from a queue subscription +-record(stomp_msg, { headers :: #{ binary() => binary() }, + body :: binary() }. +-type stomp_msg() :: #stomp_msg{}. +``` + +Once the application starts it will register a process under the name +`stomp_worker` and expose the following API: + +## Subscribing to a queue + +```erlang +%% Subscribe to a destination, receive the subscription ID +-spec subscribe(binary(), % Destination (e.g. <<"/queue/lizards">>) + ack_mode(), % Client-acknowledgement mode + -> {ok, sub_id()}. +``` + +This synchronous call subscribes to a message queue. The `stomp_worker` will +link itself to the caller and forward received messages as +`{msg, sub_id(), stomp_msg()}`. + +Depending on the acknowledgement mode specified on connecting, the subscriber +may have to acknowledge receival of messages. + +## Acknowledging messages + +```erlang +%% Acknowledge a message ID. +%% This is not required in auto mode. In client mode it will acknowledge the +%% received messages up to the ID specified. In client-individual mode every +%% single message has to be acknowledged. +-spec ack(sub_id(), message_id()) -> ok. + +%% Explicitly "unacknowledge" a message +-spec nack(sub_id(), message_id()) -> ok. +``` + +Both of these calls are asynchronous and will return immediately. Note that in +the case of the `stomp_worker` crashing before a message acknowledgement is +handled, the message *may* be delivered again. Your consumer needs to be able to +handle this. + +[STOMP protocol]: https://stomp.github.io/stomp-specification-1.2.html diff --git a/net/stomp_erl/include/stomp.hrl b/net/stomp_erl/include/stomp.hrl new file mode 100644 index 000000000000..30c933b56302 --- /dev/null +++ b/net/stomp_erl/include/stomp.hrl @@ -0,0 +1,22 @@ +%% Client ack modes, refer to the STOMP protocol documentation +-type ack_mode() :: client | client_individual | auto. + +%% Subscriptions are enumerated from 0 +-type sub_id() :: integer(). + +%% Message IDs (for acknowledgements) are simple strings. They are +%% extracted from the 'ack' field of the header in client or client-individual +%% mode, and from the 'message-id' field in auto mode. +-type message_id() :: binary(). + +%% A destination can be a queue, or something else. +%% Example: <<"/queue/lizards">> +-type destination() :: binary(). + +%% A STOMP message as received from a queue subscription +-record(stomp_msg, { headers :: #{ binary() => binary() }, + body :: binary() }). +-type stomp_msg() :: #stomp_msg{}. + +%% STOMP frame components +-type headers() :: #{binary() => binary()}. diff --git a/net/stomp_erl/src/stomp.app.src b/net/stomp_erl/src/stomp.app.src new file mode 100644 index 000000000000..baf0e271d1f0 --- /dev/null +++ b/net/stomp_erl/src/stomp.app.src @@ -0,0 +1,7 @@ +{application, stomp, [{description, "STOMP client for Erlang"}, + {vsn, "0.1.0"}, + {modules, [stomp_app, stomp_sup, stomp_worker]}, + {registered, [stomp_worker]}, + {env, []}, + {applications, [kernel, stdlib]}, + {mod, {stomp_app, []}}]}. diff --git a/net/stomp_erl/src/stomp_app.erl b/net/stomp_erl/src/stomp_app.erl new file mode 100644 index 000000000000..2ba3e69f992a --- /dev/null +++ b/net/stomp_erl/src/stomp_app.erl @@ -0,0 +1,11 @@ +-module(stomp_app). +-behaviour(application). + +-export([start/2]). +-export([stop/1]). + +start(_Type, _Args) -> + stomp_sup:start_link(). + +stop(_State) -> + ok. diff --git a/net/stomp_erl/src/stomp_sup.erl b/net/stomp_erl/src/stomp_sup.erl new file mode 100644 index 000000000000..3a298bc9bf8b --- /dev/null +++ b/net/stomp_erl/src/stomp_sup.erl @@ -0,0 +1,22 @@ +-module(stomp_sup). +-behaviour(supervisor). + +-export([start_link/0]). +-export([init/1]). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + Procs = [stomp_spec()], + {ok, {{one_for_one, 1, 5}, Procs}}. + +%% Private + +stomp_spec() -> + #{id => stomp_proc, + start => {stomp_worker, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + module => [stomp_worker]}. diff --git a/net/stomp_erl/src/stomp_worker.erl b/net/stomp_erl/src/stomp_worker.erl new file mode 100644 index 000000000000..80981d37ab52 --- /dev/null +++ b/net/stomp_erl/src/stomp_worker.erl @@ -0,0 +1,193 @@ +-module(stomp_worker). +-behaviour(gen_server). + +%% API. +-export([start_link/0]). + +%% gen_server. +-export([init/1]). +-export([handle_call/3]). +-export([handle_cast/2]). +-export([handle_info/2]). +-export([terminate/2]). +-export([code_change/3]). + +%% Testing +-compile(export_all). + +-include("stomp.hrl"). + +%% State of a stomp_worker +-record(state, {connection :: port(), + next_sub :: sub_id(), + subscriptions :: #{ destination() => sub_id() }, + subscribers :: #{ destination() => pid() } + }). +-type state() :: #state{}. + +%% API implementation + +-spec start_link() -> {ok, pid()}. +start_link() -> + {ok, Pid} = gen_server:start_link(?MODULE, [], []), + register(?MODULE, Pid), + {ok, Pid}. + +%% gen_server implementation + +-spec init(any()) -> {ok, state()}. +init(_Args) -> + %% Fetch configuration from app config + {ok, Host} = application:get_env(stomp, host), + Port = application:get_env(stomp, port, 61613), + Login = application:get_env(stomp, login), + Pass = application:get_env(stomp, passcode), + + %% Catch exit signals from linked processes (subscribers dying) + process_flag(trap_exit, true), + + %% Establish connection + {ok, Conn} = connect(Host, Port, Login, Pass), + + {ok, #state{connection = Conn, + next_sub = 0, + subscriptions = #{}, + subscribers = #{}}}. + +%% Handle subscription calls +handle_call({subscribe, Dest, Ack}, From, State) -> + %% Subscribe to new destination + SubId = State#state.next_sub, + ok = subscribe(State#state.connection, SubId, Dest, Ack), + + %% Add subscription and subscriber to state + Subscriptions = maps:put(SubId, Dest, State#state.subscriptions), + Subscribers = maps:put(SubId, From, State#state.subscribers), + NextSub = SubId + 1, + NewState = State#state{subscriptions = Subscriptions, + subscribers = Subscribers, + next_sub = NextSub }, + + {reply, {ok, SubId}, NewState}; +handle_call(_Req, _From, State) -> + {reply, ignored, State}. + +handle_info({tcp, Conn, Frame}, State) when Conn =:= State#state.connection -> + handle_frame(Frame, State); +handle_info(_Msg, State) -> + {noreply, State}. + +%% Unused gen_server callbacks + +handle_cast(_Msg, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% Private functions + +-spec connect(list(), integer(), any(), any()) -> {ok, port()}. +connect(Host, Port, Login, Pass) -> + %% STOMP CONNECT frame + Connect = connect_frame(Host, Login, Pass), + + %% TODO: Configurable buffer size + %% Frames larger than the user-level buffer will be truncated, so it should + %% never be smaller than the largest expected messages. + {ok, Socket} = gen_tcp:connect(Host, Port, [binary, + {packet, line}, + {line_delimiter, $\0}, + {buffer, 262144}]), + + ok = gen_tcp:send(Socket, Connect), + {ok, Socket}. + +-spec subscribe(port(), sub_id(), destination(), ack_mode()) -> ok. +subscribe(Socket, Id, Queue, Ack) -> + {ok, SubscribeFrame} = subscribe_frame(Id, Queue, Ack), + gen_tcp:send(Socket, SubscribeFrame). + +%%% Parsing STOMP frames + +handle_frame(<<"MESSAGE", "\n", _Frame/binary>>, State) -> + {noreply, State}; +handle_frame(Frame, State) -> + io:format("Received unknown frame ~p", [Frame]), + {noreply, State}. + + +%% Parse out headers into a map +-spec parse_headers(binary()) -> headers(). +parse_headers(HeadersBin) -> + Headers = binary:split(HeadersBin, <<"\n">>, [global]), + ToPairs = fun(H, M) -> [K,V | []] = binary:split(H, <<":">>), + maps:put(K, V, M) + end, + {ok, lists:mapfoldl(ToPairs, #{}, Headers)}. + +%%% Making STOMP protocol frames + +%% Format a header +-spec format_header({binary(), binary()}) -> binary(). +format_header({Key, Val}) -> + <<Key/binary, ":", Val/binary, "\n">>. + +%% Build a single STOMP frame +-spec make_frame(binary(), + headers(), + binary()) + -> {ok, iolist()}. +make_frame(Command, HeaderMap, Body) -> + Headers = lists:map(fun format_header/1, maps:to_list(HeaderMap)), + Frame = [Command, <<"\n">>, Headers, <<"\n">>, Body, <<0>>], + {ok, Frame}. + +%%% Default frames + +-spec connect_frame(list(), any(), any()) -> iolist(). +connect_frame(Host, {ok, Login}, {ok, Pass}) -> + make_frame(<<"CONNECT">>, + #{<<"accept-version">> => <<"1.2">>, + <<"host">> => Host, + <<"login">> => Login, + <<"passcode">> => Pass, + <<"heart-beat">> => <<"0,5000">>}, + []); +connect_frame(Host, _Login, _Pass) -> + make_frame(<<"CONNECT">>, + #{<<"accept-version">> => <<"1.2">>, + <<"host">> => Host, + %% Expect a server heartbeat every 5 seconds, let the server + %% expect one every 10. We don't actually check this and just + %% echo server heartbeats. + %% TODO: For now the server is told not to expect replies due to + %% a weird behaviour. + <<"heart-beat">> => <<"0,5000">>}, + []). + + +-spec subscribe_frame(sub_id(), destination(), ack_mode()) -> iolist(). +subscribe_frame(Id, Queue, Ack) -> + make_frame(<<"SUBSCRIBE">>, + #{<<"id">> => integer_to_binary(Id), + <<"destination">> => Queue, + <<"ack">> => ack_mode_to_binary(Ack)}, + []). + +-spec ack_mode_to_binary(ack_mode()) -> binary(). +ack_mode_to_binary(AckMode) -> + case AckMode of + auto -> <<"auto">>; + client -> <<"client">>; + client_individual -> <<"client-individual">> + end. + +%% -spec ack_frame(binary()) -> iolist(). +%% ack_frame(MessageID) -> +%% make_frame(<<"ACK">>, +%% [{"id", MessageID}], +%% []). diff --git a/nix/OWNERS b/nix/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/binify/default.nix b/nix/binify/default.nix new file mode 100644 index 000000000000..d40930fd3334 --- /dev/null +++ b/nix/binify/default.nix @@ -0,0 +1,16 @@ +{ pkgs, lib, ... }: + +# Create a store path where the executable `exe` +# is linked to $out/bin/${name}. +# This is useful for e.g. including it as a “package” +# in `buildInputs` of a shell.nix. +# +# For example, if I have the exeutable /nix/store/…-hello, +# I can make it into /nix/store/…-binify-hello/bin/hello +# with `binify { exe = …; name = "hello" }`. +{ exe, name }: + +pkgs.runCommandLocal "${name}-bin" {} '' + mkdir -p $out/bin + ln -sT ${lib.escapeShellArg exe} $out/bin/${lib.escapeShellArg name} +'' diff --git a/nix/bufCheck/default.nix b/nix/bufCheck/default.nix new file mode 100644 index 000000000000..039303ba684b --- /dev/null +++ b/nix/bufCheck/default.nix @@ -0,0 +1,9 @@ +# Check protobuf syntax and breaking. +# +{ depot, pkgs, ... }: + +pkgs.writeShellScriptBin "ci-buf-check" '' + ${depot.third_party.bufbuild}/bin/buf check lint --input . + # Report-only + ${depot.third_party.bufbuild}/bin/buf check breaking --input "." --against-input "./.git#branch=canon" || true +'' diff --git a/nix/buildGo/.skip-subtree b/nix/buildGo/.skip-subtree new file mode 100644 index 000000000000..8db1f814f653 --- /dev/null +++ b/nix/buildGo/.skip-subtree @@ -0,0 +1,2 @@ +Subdirectories of this folder should not be imported since they are +internal to buildGo.nix and incompatible with readTree. diff --git a/nix/buildGo/README.md b/nix/buildGo/README.md new file mode 100644 index 000000000000..37e0c06933f9 --- /dev/null +++ b/nix/buildGo/README.md @@ -0,0 +1,140 @@ +buildGo.nix +=========== + +This is an alternative [Nix][] build system for [Go][]. It supports building Go +libraries and programs, and even automatically generating Protobuf & gRPC +libraries. + +*Note:* This will probably end up being folded into [Nixery][]. + +## Background + +Most language-specific Nix tooling outsources the build to existing +language-specific build tooling, which essentially means that Nix ends up being +a wrapper around all sorts of external build systems. + +However, systems like [Bazel][] take an alternative approach in which the +compiler is invoked directly and the composition of programs and libraries stays +within a single homogeneous build system. + +Users don't need to learn per-language build systems and especially for +companies with large monorepo-setups ([like Google][]) this has huge +productivity impact. + +This project is an attempt to prove that Nix can be used in a similar style to +build software directly, rather than shelling out to other build systems. + +## Example + +Given a program layout like this: + +``` +. +├── lib <-- some library component +│ ├── bar.go +│ └── foo.go +├── api.proto <-- gRPC API definition +├── main.go <-- program implementation +└── default.nix <-- build instructions +``` + +The contents of `default.nix` could look like this: + +```nix +{ buildGo }: + +let + api = buildGo.grpc { + name = "someapi"; + proto = ./api.proto; + }; + + lib = buildGo.package { + name = "somelib"; + srcs = [ + ./lib/bar.go + ./lib/foo.go + ]; + }; +in buildGo.program { + name = "my-program"; + deps = [ api lib ]; + + srcs = [ + ./main.go + ]; +} +``` + +(If you don't know how to read Nix, check out [nix-1p][]) + +## Usage + +`buildGo` exposes five different functions: + +* `buildGo.program`: Build a Go binary out of the specified source files. + + | parameter | type | use | required? | + |-----------|-------------------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the program (and resulting executable) | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no | + | `x_defs` | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no | + +* `buildGo.package`: Build a Go library out of the specified source files. + + | parameter | type | use | required? | + |-----------|--------------|------------------------------------------------|-----------| + | `name` | `string` | Name of the library | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no | + | `path` | `string` | Go import path for the resulting library | no | + +* `buildGo.external`: Build an externally defined Go library or program. + + This function performs analysis on the supplied source code (which + can use the standard Go tooling layout) and creates a tree of all + the packages contained within. + + This exists for compatibility with external libraries that were not + defined using buildGo. + + | parameter | type | use | required? | + |-----------|----------------|-----------------------------------------------|-----------| + | `path` | `string` | Go import path for the resulting package | yes | + | `src` | `path` | Path to the source **directory** | yes | + | `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no | + + For some examples of how `buildGo.external` is used, check out + [`proto.nix`](./proto.nix). + +* `buildGo.proto`: Build a Go library out of the specified Protobuf definition. + + | parameter | type | use | required? | + |-------------|-------------|--------------------------------------------------|-----------| + | `name` | `string` | Name for the resulting library | yes | + | `proto` | `path` | Path to the Protobuf definition file | yes | + | `path` | `string` | Import path for the resulting Go library | no | + | `extraDeps` | `list<drv>` | Additional Go dependencies to add to the library | no | + +* `buildGo.grpc`: Build a Go library out of the specified gRPC definition. + + The parameters are identical to `buildGo.proto`. + +## Current status + +This project is work-in-progress. Crucially it is lacking the following features: + +* feature flag parity with Bazel's Go rules +* documentation building +* test execution + +There are still some open questions around how to structure some of those +features in Nix. + +[Nix]: https://nixos.org/nix/ +[Go]: https://golang.org/ +[Nixery]: https://github.com/google/nixery +[Bazel]: https://bazel.build/ +[like Google]: https://ai.google/research/pubs/pub45424 +[nix-1p]: https://github.com/tazjin/nix-1p diff --git a/nix/buildGo/default.nix b/nix/buildGo/default.nix new file mode 100644 index 000000000000..a2396dc3f770 --- /dev/null +++ b/nix/buildGo/default.nix @@ -0,0 +1,135 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# buildGo provides Nix functions to build Go packages in the style of Bazel's +# rules_go. + +{ pkgs ? import <nixpkgs> {} +, ... }: + +let + inherit (builtins) + attrNames + baseNameOf + dirOf + elemAt + filter + listToAttrs + map + match + readDir + replaceStrings + toString; + + inherit (pkgs) lib go runCommand fetchFromGitHub protobuf symlinkJoin; + + # Helpers for low-level Go compiler invocations + spaceOut = lib.concatStringsSep " "; + + includeDepSrc = dep: "-I ${dep}"; + includeSources = deps: spaceOut (map includeDepSrc deps); + + includeDepLib = dep: "-L ${dep}"; + includeLibs = deps: spaceOut (map includeDepLib deps); + + srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1; + srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}"; + srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs); + + allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps))); + + xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs)); + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + + # Add an `overrideGo` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Go builders. + makeOverridable = f: orig: (f orig) // { + overrideGo = new: makeOverridable f (orig // (new orig)); + }; + + # High-level build functions + + # Build a Go program out of the specified files and dependencies. + program = { name, srcs, deps ? [], x_defs ? {} }: + let uniqueDeps = allDeps (map (d: d.gopkg) deps); + in runCommand name {} '' + ${go}/bin/go tool compile -o ${name}.a -trimpath=$PWD -trimpath=${go} ${includeSources uniqueDeps} ${spaceOut srcs} + mkdir -p $out/bin + export GOROOT_FINAL=go + ${go}/bin/go tool link -o $out/bin/${name} -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a + ''; + + # Build a Go library assembled out of the specified files. + # + # This outputs both the sources and compiled binary, as both are + # needed when downstream packages depend on it. + package = { name, srcs, deps ? [], path ? name, sfiles ? [] }: + let + uniqueDeps = allDeps (map (d: d.gopkg) deps); + + # The build steps below need to be executed conditionally for Go + # assembly if the analyser detected any *.s files. + # + # This is required for several popular packages (e.g. x/sys). + ifAsm = do: lib.optionalString (sfiles != []) do; + asmBuild = ifAsm '' + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles} + ${go}/bin/go tool asm -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles} + ''; + asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h"; + asmPack = ifAsm '' + ${go}/bin/go tool pack r $out/${path}.a ./asm.o + ''; + + gopkg = (runCommand "golib-${name}" {} '' + mkdir -p $out/${path} + ${srcList path (map (s: "${s}") srcs)} + ${asmBuild} + ${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs} + ${asmPack} + '') // { + inherit gopkg; + goDeps = uniqueDeps; + goImportPath = path; + }; + in gopkg; + + # Build a tree of Go libraries out of an external Go source + # directory that follows the standard Go layout and was not built + # with buildGo.nix. + # + # The derivation for each actual package will reside in an attribute + # named "gopkg", and an attribute named "gobin" for binaries. + external = import ./external { inherit pkgs program package; }; + + # Import support libraries needed for protobuf & gRPC support + protoLibs = import ./proto.nix { + inherit external; + }; + + # Build a Go library out of the specified protobuf definition. + proto = { name, proto, path ? name, goPackage ? name, extraDeps ? [] }: (makeOverridable package) { + inherit name path; + deps = [ protoLibs.goProto.proto.gopkg ] ++ extraDeps; + srcs = lib.singleton (runCommand "goproto-${name}.pb.go" {} '' + cp ${proto} ${baseNameOf proto} + ${protobuf}/bin/protoc --plugin=${protoLibs.goProto.protoc-gen-go.gopkg}/bin/protoc-gen-go \ + --go_out=plugins=grpc,import_path=${baseNameOf path}:. ${baseNameOf proto} + mv ./${goPackage}/*.pb.go $out + ''); + }; + + # Build a Go library out of the specified gRPC definition. + grpc = args: proto (args // { extraDeps = [ protoLibs.goGrpc.gopkg ]; }); + +in { + # Only the high-level builder functions are exposed, but made + # overrideable. + program = makeOverridable program; + package = makeOverridable package; + proto = makeOverridable proto; + grpc = makeOverridable grpc; + external = makeOverridable external; +} diff --git a/nix/buildGo/example/default.nix b/nix/buildGo/example/default.nix new file mode 100644 index 000000000000..99c0a7d79bd6 --- /dev/null +++ b/nix/buildGo/example/default.nix @@ -0,0 +1,47 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 + +# This file provides examples for how to use the various builder +# functions provided by `buildGo`. +# +# The features used in the example are not exhaustive, but should give +# users a quick introduction to how to use buildGo. + +let + buildGo = import ../default.nix {}; + + # Example use of buildGo.package, which creates an importable Go + # package from the specified source files. + examplePackage = buildGo.package { + name = "example"; + srcs = [ + ./lib.go + ]; + }; + + # Example use of buildGo.proto, which generates a Go library from a + # Protobuf definition file. + exampleProto = buildGo.proto { + name = "exampleproto"; + proto = ./thing.proto; + }; + + # Example use of buildGo.program, which builds an executable using + # the specified name and dependencies (which in turn must have been + # created via buildGo.package etc.) +in buildGo.program { + name = "example"; + + srcs = [ + ./main.go + ]; + + deps = [ + examplePackage + exampleProto + ]; + + x_defs = { + "main.Flag" = "successfully"; + }; +} diff --git a/nix/buildGo/example/lib.go b/nix/buildGo/example/lib.go new file mode 100644 index 000000000000..8a61370e994c --- /dev/null +++ b/nix/buildGo/example/lib.go @@ -0,0 +1,9 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +package example + +// UUID returns a totally random, carefully chosen UUID +func UUID() string { + return "3640932f-ad40-4bc9-b45d-f504a0f5910a" +} diff --git a/nix/buildGo/example/main.go b/nix/buildGo/example/main.go new file mode 100644 index 000000000000..bbcedbff8726 --- /dev/null +++ b/nix/buildGo/example/main.go @@ -0,0 +1,25 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 +// +// Package main provides a tiny example program for the Bazel-style +// Nix build system for Go. + +package main + +import ( + "example" + "exampleproto" + "fmt" +) + +var Flag string = "unsuccessfully" + +func main() { + thing := exampleproto.Thing{ + Id: example.UUID(), + KindOfThing: "test thing", + } + + fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing) + fmt.Printf("The flag has been %s set\n", Flag) +} diff --git a/nix/buildGo/example/thing.proto b/nix/buildGo/example/thing.proto new file mode 100644 index 000000000000..0f6d6575e008 --- /dev/null +++ b/nix/buildGo/example/thing.proto @@ -0,0 +1,10 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +syntax = "proto3"; +package example; + +message Thing { + string id = 1; + string kind_of_thing = 2; +} diff --git a/nix/buildGo/external/default.nix b/nix/buildGo/external/default.nix new file mode 100644 index 000000000000..6540faf04c36 --- /dev/null +++ b/nix/buildGo/external/default.nix @@ -0,0 +1,95 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +{ pkgs, program, package }: + +let + inherit (builtins) + elemAt + foldl' + fromJSON + head + length + listToAttrs + readFile + replaceStrings + tail + throw; + + inherit (pkgs) lib runCommand go jq ripgrep; + + pathToName = p: replaceStrings ["/"] ["_"] (toString p); + + # Collect all non-vendored dependencies from the Go standard library + # into a file that can be used to filter them out when processing + # dependencies. + stdlibPackages = runCommand "stdlib-pkgs.json" {} '' + export HOME=$PWD + export GOPATH=/dev/null + ${go}/bin/go list std | \ + ${ripgrep}/bin/rg -v 'vendor' | \ + ${jq}/bin/jq -R '.' | \ + ${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \ + > $out + ''; + + analyser = program { + name = "analyser"; + + srcs = [ + ./main.go + ]; + + x_defs = { + "main.stdlibList" = "${stdlibPackages}"; + }; + }; + + mkset = path: value: + if path == [] then { gopkg = value; } + else { "${head path}" = mkset (tail path) value; }; + + last = l: elemAt l ((length l) - 1); + + toPackage = self: src: path: depMap: entry: + let + localDeps = map (d: lib.attrByPath (d ++ [ "gopkg" ]) ( + throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'" + ) self) entry.localDeps; + + foreignDeps = map (d: lib.attrByPath [ d.path ] ( + throw "missing foreign dependency '${d.path}' in '${path}, imported at ${d.position}'" + ) depMap) entry.foreignDeps; + + args = { + srcs = map (f: src + ("/" + f)) entry.files; + deps = localDeps ++ foreignDeps; + }; + + libArgs = args // { + name = pathToName entry.name; + path = lib.concatStringsSep "/" ([ path ] ++ entry.locator); + sfiles = map (f: src + ("/" + f)) entry.sfiles; + }; + + binArgs = args // { + name = (last ((lib.splitString "/" path) ++ entry.locator)); + }; + in if entry.isCommand then (program binArgs) else (package libArgs); + +in { src, path, deps ? [] }: let + # Build a map of dependencies (from their import paths to their + # derivation) so that they can be conditionally imported only in + # sub-packages that require them. + depMap = listToAttrs (map (d: { + name = d.goImportPath; + value = d; + }) (map (d: d.gopkg) deps)); + + name = pathToName path; + analysisOutput = runCommand "${name}-structure.json" {} '' + ${analyser}/bin/analyser -path ${path} -source ${src} > $out + ''; + analysis = fromJSON (readFile analysisOutput); +in lib.fix(self: foldl' lib.recursiveUpdate {} ( + map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis +)) diff --git a/nix/buildGo/external/main.go b/nix/buildGo/external/main.go new file mode 100644 index 000000000000..a77c43b371e9 --- /dev/null +++ b/nix/buildGo/external/main.go @@ -0,0 +1,201 @@ +// Copyright 2019 Google LLC. +// SPDX-License-Identifier: Apache-2.0 + +// This tool analyses external (i.e. not built with `buildGo.nix`) Go +// packages to determine a build plan that Nix can import. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "go/build" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" +) + +// Path to a JSON file describing all standard library import paths. +// This file is generated and set here by Nix during the build +// process. +var stdlibList string + +// pkg describes a single Go package within the specified source +// directory. +// +// Return information includes the local (relative from project root) +// and external (none-stdlib) dependencies of this package. +type pkg struct { + Name string `json:"name"` + Locator []string `json:"locator"` + Files []string `json:"files"` + SFiles []string `json:"sfiles"` + LocalDeps [][]string `json:"localDeps"` + ForeignDeps []foreignDep `json:"foreignDeps"` + IsCommand bool `json:"isCommand"` +} + +type foreignDep struct { + Path string `json:"path"` + // filename, column and line number of the import, if known + Position string `json:"position"` +} + +// findGoDirs returns a filepath.WalkFunc that identifies all +// directories that contain Go source code in a certain tree. +func findGoDirs(at string) ([]string, error) { + dirSet := make(map[string]bool) + + err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + name := info.Name() + // Skip folders that are guaranteed to not be relevant + if info.IsDir() && (name == "testdata" || name == ".git") { + return filepath.SkipDir + } + + // If the current file is a Go file, then the directory is popped + // (i.e. marked as a Go directory). + if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") { + dirSet[filepath.Dir(path)] = true + } + + return nil + }) + + if err != nil { + return nil, err + } + + goDirs := []string{} + for k, _ := range dirSet { + goDirs = append(goDirs, k) + } + + return goDirs, nil +} + +// analysePackage loads and analyses the imports of a single Go +// package, returning the data that is required by the Nix code to +// generate a derivation for this package. +func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) { + ctx := build.Default + ctx.CgoEnabled = false + + p, err := ctx.ImportDir(source, build.IgnoreVendor) + if err != nil { + return pkg{}, err + } + + local := [][]string{} + foreign := []foreignDep{} + + for _, i := range p.Imports { + if stdlib[i] { + continue + } + + if i == importpath { + local = append(local, []string{}) + } else if strings.HasPrefix(i, importpath+"/") { + local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/")) + } else { + // The import positions is a map keyed on the import name. + // The value is a list, presumably because an import can appear + // multiple times in a package. Let’s just take the first one, + // should be enough for a good error message. + firstPos := p.ImportPos[i][0].String() + foreign = append(foreign, foreignDep{Path: i, Position: firstPos}) + } + } + + prefix := strings.TrimPrefix(source, root+"/") + + locator := []string{} + if len(prefix) != len(source) { + locator = strings.Split(prefix, "/") + } else { + // Otherwise, the locator is empty since its the root package and + // no prefix should be added to files. + prefix = "" + } + + files := []string{} + for _, f := range p.GoFiles { + files = append(files, path.Join(prefix, f)) + } + + sfiles := []string{} + for _, f := range p.SFiles { + sfiles = append(sfiles, path.Join(prefix, f)) + } + + return pkg{ + Name: path.Join(importpath, prefix), + Locator: locator, + Files: files, + SFiles: sfiles, + LocalDeps: local, + ForeignDeps: foreign, + IsCommand: p.IsCommand(), + }, nil +} + +func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) { + f, err := ioutil.ReadFile(from) + if err != nil { + return + } + + err = json.Unmarshal(f, &pkgs) + return +} + +func main() { + source := flag.String("source", "", "path to directory with sources to process") + path := flag.String("path", "", "import path for the package") + + flag.Parse() + + if *source == "" { + log.Fatalf("-source flag must be specified") + } + + stdlibPkgs, err := loadStdlibPkgs(stdlibList) + if err != nil { + log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err) + } + + goDirs, err := findGoDirs(*source) + if err != nil { + log.Fatalf("failed to walk source directory '%s': %s", *source, err) + } + + all := []pkg{} + for _, d := range goDirs { + analysed, err := analysePackage(*source, d, *path, stdlibPkgs) + + // If the Go source analysis returned "no buildable Go files", + // that directory should be skipped. + // + // This might be due to `+build` flags on the platform and other + // reasons (such as test files). + if _, ok := err.(*build.NoGoError); ok { + continue + } + + if err != nil { + log.Fatalf("failed to analyse package at %q: %s", d, err) + } + all = append(all, analysed) + } + + j, _ := json.Marshal(all) + fmt.Println(string(j)) +} diff --git a/nix/buildGo/proto.nix b/nix/buildGo/proto.nix new file mode 100644 index 000000000000..57e9a5ec98b0 --- /dev/null +++ b/nix/buildGo/proto.nix @@ -0,0 +1,84 @@ +# Copyright 2019 Google LLC. +# SPDX-License-Identifier: Apache-2.0 +# +# This file provides derivations for the dependencies of a gRPC +# service in Go. + +{ external }: + +let + inherit (builtins) fetchGit map; +in rec { + goProto = external { + path = "github.com/golang/protobuf"; + src = fetchGit { + url = "https://github.com/golang/protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + }; + }; + + xnet = external { + path = "golang.org/x/net"; + + src = fetchGit { + url = "https://go.googlesource.com/net"; + rev = "ffdde105785063a81acd95bdf89ea53f6e0aac2d"; + }; + + deps = [ + xtext.secure.bidirule + xtext.unicode.bidi + xtext.unicode.norm + ]; + }; + + xsys = external { + path = "golang.org/x/sys"; + src = fetchGit { + url = "https://go.googlesource.com/sys"; + rev = "bd437916bb0eb726b873ee8e9b2dcf212d32e2fd"; + }; + }; + + xtext = external { + path = "golang.org/x/text"; + src = fetchGit { + url = "https://go.googlesource.com/text"; + rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; + }; + }; + + genproto = external { + path = "google.golang.org/genproto"; + src = fetchGit { + url = "https://github.com/google/go-genproto"; + rev = "83cc0476cb11ea0da33dacd4c6354ab192de6fe6"; + }; + + deps = with goProto; [ + proto + ptypes.any + ]; + }; + + goGrpc = external { + path = "google.golang.org/grpc"; + deps = ([ + xnet.trace + xnet.http2 + xsys.unix + xnet.http2.hpack + genproto.googleapis.rpc.status + ] ++ (with goProto; [ + proto + ptypes + ptypes.duration + ptypes.timestamp + ])); + + src = fetchGit { + url = "https://github.com/grpc/grpc-go"; + rev = "d8e3da36ac481ef00e510ca119f6b68177713689"; + }; + }; +} diff --git a/nix/buildLisp/README.md b/nix/buildLisp/README.md new file mode 100644 index 000000000000..214700750d7f --- /dev/null +++ b/nix/buildLisp/README.md @@ -0,0 +1,254 @@ +buildLisp.nix +============= + +This is a build system for Common Lisp, written in Nix. + +It aims to offer an alternative to ASDF for users who live in a +Nix-based ecosystem. This offers several advantages over ASDF: + +* Simpler (almost logic-less) package definitions +* Easy linking of native dependencies (from Nix) +* Composability with Nix tooling for other languages +* Effective, per-system caching strategies +* Easy overriding of dependencies and whatnot +* Convenient support for multiple Common Lisp implementations +* ... and more! + +The project is still in its early stages and some important +restrictions should be highlighted: + +* Extending `buildLisp` with support for a custom implementation + currently requires some knowledge of internals and may not be + considered stable yet. +* Parallel compilation is not possible: Since buildLisp doesn't encode + dependencies between components (i. e. source files) like ASDF, + it must compile source files in sequence to avoid errors due to + undefined symbols. + +## Usage + +`buildLisp` exposes four different functions: + +* `buildLisp.library`: Builds a collection of Lisp files into a library. + + | parameter | type | use | required? | + |-----------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the library | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies | no | + | `native` | `list<drv>` | List of native dependencies | no | + | `test` | see "Tests" | Specification for test suite | no | + | `implementation` | see "Implementations" | Common Lisp implementation to use | no | + + The output of invoking this is a directory containing a FASL file + that is the concatenated result of all compiled sources. + +* `buildLisp.program`: Builds an executable program out of Lisp files. + + | parameter | type | use | required? | + |-----------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the program | yes | + | `srcs` | `list<path>` | List of paths to source files | yes | + | `deps` | `list<drv>` | List of dependencies | no | + | `native` | `list<drv>` | List of native dependencies | no | + | `main` | `string` | Entrypoint function | no | + | `test` | see "Tests" | Specification for test suite | no | + | `implementation` | see "Implementations" | Common Lisp implementation to use | no | + + The `main` parameter should be the name of a function and defaults + to `${name}:main` (i.e. the *exported* `main` function of the + package named after the program). + + The output of invoking this is a directory containing a + `bin/${name}`. + +* `buildLisp.bundled`: Creates a virtual dependency on a built-in library. + + Certain libraries ship with Lisp implementations, for example + UIOP/ASDF are commonly included but many implementations also ship + internals (such as SBCLs various `sb-*` libraries). + + This function takes a single string argument that is the name of a + built-in library and returns a "package" that simply requires this + library. + +## Tests + +Both `buildLisp.library` and `buildLisp.program` take an optional argument +`tests`, which has the following supported fields: + + | parameter | type | use | required? | + |--------------|--------------|-------------------------------|-----------| + | `name` | `string` | Name of the test suite | no | + | `expression` | `string` | Lisp expression to run tests | yes | + | `srcs` | `list<path>` | List of paths to source files | no | + | `native` | `list<drv>` | List of native dependencies | no | + +the `expression` parameter should be a Lisp expression and will be evaluated +after loading all sources and dependencies (including library/program +dependencies). It must return a non-`NIL` value if the test suite has passed. + +## Example + +Using buildLisp could look like this: + +```nix +{ buildLisp, lispPkgs }: + +let libExample = buildLisp.library { + name = "lib-example"; + srcs = [ ./lib.lisp ]; + + deps = with lispPkgs; [ + (buildLisp.bundled "sb-posix") + iterate + cl-ppcre + ]; +}; +in buildLisp.program { + name = "example"; + deps = [ libExample ]; + srcs = [ ./main.lisp ]; + tests = { + deps = [ lispPkgs.fiveam ]; + srcs = [ ./tests.lisp ]; + expression = "(fiveam:run!)"; + }; +} +``` + +## Development REPLs + +`buildLisp` builds loadable variants of both `program` and `library` derivations +(usually FASL files). Therefore it can provide a convenient way to obtain an +instance of any implementation preloaded with `buildLisp`-derivations. This +is especially useful to use as a host for Sly or SLIME. + +* `buildLisp.sbcl.lispWith`, `buildLisp.ccl.lispWith`, ...: + Creates a wrapper script preloading a Lisp implementation with various dependencies. + + This function takes a single argument which is a list of Lisp + libraries programs or programs. The desired Lisp implementation + will load all given derivations and all their dependencies on + startup. + + The shortcut `buildLisp.sbclWith` for `buildLisp.sbcl.lispWith` is also provided. + +* `repl` passthru attribute: `derivation.repl` is provided as a shortcut + for `buildLisp.${implementationName}.lispWith [ derivation ]`. + `derivation.ccl.repl`, `derivation.sbcl.repl` etc. work as well, of course + (see also "Implementations" section). + +## Implementations + +Both `buildLisp.library` and `buildLisp.program` allow specifying a different +Common Lisp implementation than the default one (which is SBCL). When an +implementation is passed, `buildLisp` makes sure all dependencies are built +with that implementation as well since build artifacts from different +implementation will be incompatible with each other. + +The argument taken by `implementation` is a special attribute set which +describes how to do certain tasks for a given implementation, like building +or loading a library. In case you want to use a custom implementation +description, the precise structure needed is documented in `buildLisp`'s +source code for now. `buildLisp` also exposes the following already +working implementation sets: + +* `buildLisp.sbcl`: [SBCL][sbcl], our default implementation + +* `buildLisp.ccl`: [CCL][ccl], similar to SBCL, but with very good macOS support + +* `buildLisp.ecl`: [ECL][ecl] setup to produce statically linked binaries and + libraries. Note that its runtime library is LGPL, so [extra conditions][lgpl-static] + must be fulfilled when distributing binaries produced this way. + +* Support for ABCL is planned. + +For every of these “known” implementations, `buildLisp` will create a `passthru` +attribute named like the implementation which points to a variant of the derivation +built with said implementation. Say we have a derivation, `myDrv`, built using SBCL: +While `myDrv` and `myDrv.sbcl` are built using SBCL, `myDrv.ecl`, `myDrv.ccl` etc. +build the derivation and all its dependencies using ECL and CCL respectively. + +This is useful to test portability of your derivation, but is also used internally +to speed up the “normalization” of the dependency graph. Thus it is important to +make sure that your custom implementation's name doesn't clash with one of the +“known” ones. + +## Handling Implementation Specifics + +When targeting multiple Common Lisp implementation, it is often necessary to +handle differing interfaces for OS interaction or to make use of special +implementation features. For this reason, `buildLisp` allows specifying +dependencies and source files for specific implementations only. This can +be utilized by having an attribute set in the list for the `deps` or `srcs` +argument: `buildLisp` will pick the value of the attribute named like the +used implementation or `default` and ignore the set completely if both +are missing. + +```nix +{ buildLisp, lispPkgs }: + +buildLisp.library { + name = "mylib"; + + srcs = [ + # These are included always of course + ./package.lisp + ./portable-lib.lisp + + # Choose right impl-* file + { + sbcl = ./impl-sbcl.lisp; + ccl = ./impl-ccl.lisp; + ecl = ./impl-ecl.lisp; + } + + # We can also use this to inject extra files + { ecl = ./extra-ecl-optimizations.lisp; } + ]; + + deps = [ + # Use SBCL's special bundled package, flexi-streams otherwise + { + sbcl = buildLisp.bundled "sb-rotate-byte"; + default = lispPkgs.flexi-streams; + } + ]; +} +``` + +Additionally a `brokenOn` parameter is accepted which takes a list of +implementation names on which the derivation is not expected to work. +This only influences `meta.targets` which is read by depot's CI to +check which variants (see "Implementations") of the derivation to +build, so it may not be useful outside of depot. + +## Influencing the Lisp Runtime + +Lisp implementations which create an executable by dumping an image +usually parse a few implementation-specific command line options on +executable startup that influence runtime settings related to things +like GC. `buildLisp` generates a wrapper which makes sure that this +never interferes with the argument parsing implemented in the actual +application, but sometimes it is useful to run an executable with +special settings. To allow this, the content of `NIX_BUILDLISP_LISP_ARGS` +is passed to the lisp implementation. + +For example, you can make the underlying SBCL print its version for +any executable built with `buildLisp` (and SBCL) like this: + +```console +$ env NIX_BUILDLISP_LISP_ARGS="--version" ./result/bin/🕰️ +SBCL 2.1.2.nixos +``` + +In practice you'd probably want to specify options like +`--dynamic-space-size` or `--tls-limit` (try passing `--help` for a +full list). Naturally, these options are completely different for +different implementations. + +[sbcl]: http://www.sbcl.org/ +[ccl]: https://ccl.clozure.com/ +[ecl]: https://common-lisp.net/project/ecl/ +[lgpl-static]: https://www.gnu.org/licenses/gpl-faq.en.html#LGPLStaticVsDynamic diff --git a/nix/buildLisp/default.nix b/nix/buildLisp/default.nix new file mode 100644 index 000000000000..1dae0efeb198 --- /dev/null +++ b/nix/buildLisp/default.nix @@ -0,0 +1,736 @@ +# buildLisp provides Nix functions to build Common Lisp packages, +# targeting SBCL. +# +# buildLisp is designed to enforce conventions and do away with the +# free-for-all of existing Lisp build systems. + +{ pkgs ? import <nixpkgs> {}, ... }: + +let + inherit (builtins) map elemAt match filter; + inherit (pkgs) lib runCommandNoCC makeWrapper writeText writeShellScriptBin sbcl ecl-static ccl; + inherit (pkgs.stdenv) targetPlatform; + + # + # Internal helper definitions + # + + defaultImplementation = impls.sbcl; + + # Many Common Lisp implementations (like ECL and CCL) will occasionally drop + # you into an interactive debugger even when executing something as a script. + # In nix builds we don't want such a situation: Any error should make the + # script exit non-zero. Luckily the ANSI standard specifies *debugger-hook* + # which is invoked before the debugger letting us just do that. + disableDebugger = writeText "disable-debugger.lisp" '' + (setf *debugger-hook* + (lambda (error hook) + (declare (ignore hook)) + (format *error-output* "~%Unhandled error: ~a~%" error) + #+ccl (quit 1) + #+ecl (ext:quit 1))) + ''; + + # Process a list of arbitrary values which also contains “implementation + # filter sets” which describe conditonal inclusion of elements depending + # on the CL implementation used. Elements are processed in the following + # manner: + # + # * Paths, strings, derivations are left as is + # * A non-derivation attribute set is processed like this: + # 1. If it has an attribute equal to impl.name, replace with its value. + # 2. Alternatively use the value of the "default" attribute. + # 3. In all other cases delete the element from the list. + # + # This can be used to express dependencies or source files which are specific + # to certain implementations: + # + # srcs = [ + # # mixable with unconditional entries + # ./package.lisp + # + # # implementation specific source files + # { + # ccl = ./impl-ccl.lisp; + # sbcl = ./impl-sbcl.lisp; + # ecl = ./impl-ecl.lisp; + # } + # ]; + # + # deps = [ + # # this dependency is ignored if impl.name != "sbcl" + # { sbcl = buildLisp.bundled "sb-posix"; } + # + # # only special casing for a single implementation + # { + # sbcl = buildLisp.bundled "uiop"; + # default = buildLisp.bundled "asdf"; + # } + # ]; + implFilter = impl: xs: + let + isFilterSet = x: builtins.isAttrs x && !(lib.isDerivation x); + in builtins.map ( + x: if isFilterSet x then x.${impl.name} or x.default else x + ) (builtins.filter ( + x: !(isFilterSet x) || x ? ${impl.name} || x ? default + ) xs); + + # Generates lisp code which instructs the given lisp implementation to load + # all the given dependencies. + genLoadLispGeneric = impl: deps: + lib.concatStringsSep "\n" + (map (lib: "(load \"${lib}/${lib.lispName}.${impl.faslExt}\")") + (allDeps impl deps)); + + # 'genTestLispGeneric' generates a Lisp file that loads all sources and deps + # and executes expression for a given implementation description. + genTestLispGeneric = impl: { name, srcs, deps, expression }: writeText "${name}.lisp" '' + ;; Dependencies + ${impl.genLoadLisp deps} + + ;; Sources + ${lib.concatStringsSep "\n" (map (src: "(load \"${src}\")") srcs)} + + ;; Test expression + (unless ${expression} + (exit :code 1)) + ''; + + # 'dependsOn' determines whether Lisp library 'b' depends on 'a'. + dependsOn = a: b: builtins.elem a b.lispDeps; + + # 'allDeps' flattens the list of dependencies (and their + # dependencies) into one ordered list of unique deps which + # all use the given implementation. + allDeps = impl: deps: let + # The override _should_ propagate itself recursively, as every derivation + # would only expose its actually used dependencies. Use implementation + # attribute created by withExtras if present, override in all other cases + # (mainly bundled). + deps' = builtins.map (dep: dep."${impl.name}" or (dep.overrideLisp (_: { + implementation = impl; + }))) deps; + in (lib.toposort dependsOn (lib.unique ( + lib.flatten (deps' ++ (map (d: d.lispDeps) deps')) + ))).result; + + # 'allNative' extracts all native dependencies of a dependency list + # to ensure that library load paths are set correctly during all + # compilations and program assembly. + allNative = native: deps: lib.unique ( + lib.flatten (native ++ (map (d: d.lispNativeDeps) deps)) + ); + + # Add an `overrideLisp` attribute to a function result that works + # similar to `overrideAttrs`, but is used specifically for the + # arguments passed to Lisp builders. + makeOverridable = f: orig: (f orig) // { + overrideLisp = new: makeOverridable f (orig // (new orig)); + }; + + # This is a wrapper arround 'makeOverridable' which performs its + # function, but also adds a the following additional attributes to the + # resulting derivation, namely a repl attribute which builds a `lispWith` + # derivation for the current implementation and additional attributes for + # every all implementations. So `drv.sbcl` would build the derivation + # with SBCL regardless of what was specified in the initial arguments. + withExtras = f: args: + let + drv = (makeOverridable f) args; + in lib.fix (self: + drv.overrideLisp (old: + let + implementation = old.implementation or defaultImplementation; + brokenOn = old.brokenOn or []; + targets = lib.subtractLists brokenOn + (builtins.attrNames impls); + in { + passthru = (old.passthru or {}) // { + repl = implementation.lispWith [ self ]; + + # meta is done via passthru to minimize rebuilds caused by overriding + meta = (old.passthru.meta or {}) // { + inherit targets; + }; + } // builtins.listToAttrs (builtins.map (impl: { + inherit (impl) name; + value = self.overrideLisp (_: { + implementation = impl; + }); + }) (builtins.attrValues impls)); + }) // { + overrideLisp = new: withExtras f (args // new args); + }); + + # 'testSuite' builds a Common Lisp test suite that loads all of srcs and deps, + # and then executes expression to check its result + testSuite = { name, expression, srcs, deps ? [], native ? [], implementation }: + let + lispNativeDeps = allNative native deps; + lispDeps = allDeps implementation (implFilter implementation deps); + filteredSrcs = implFilter implementation srcs; + in runCommandNoCC name { + LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; + LANG = "C.UTF-8"; + } '' + echo "Running test suite ${name}" + + ${implementation.runScript} ${ + implementation.genTestLisp { + inherit name expression; + srcs = filteredSrcs; + deps = lispDeps; + } + } | tee $out + + echo "Test suite ${name} succeeded" + ''; + + # 'impls' is an attribute set of attribute sets which describe how to do common + # tasks when building for different Common Lisp implementations. Each + # implementation set has the following members: + # + # Required members: + # + # - runScript :: string + # Describes how to invoke the implementation from the shell, so it runs a + # lisp file as a script and exits. + # - faslExt :: string + # File extension of the implementations loadable (FASL) files. + # Implementations are free to generate native object files, but with the way + # buildLisp works it is required that we can also 'load' libraries, so + # (additionally) building a FASL or equivalent is required. + # - genLoadLisp :: [ dependency ] -> string + # Returns lisp code to 'load' the given dependencies. 'genLoadLispGeneric' + # should work for most dependencies. + # - genCompileLisp :: { name, srcs, deps } -> file + # Builds a lisp file which instructs the implementation to build a library + # from the given source files when executed. After running at least + # the file "$out/${name}.${impls.${implementation}.faslExt}" should have + # been created. + # - genDumpLisp :: { name, main, deps } -> file + # Builds a lisp file which instructs the implementation to build an + # executable which runs 'main' (and exits) where 'main' is available from + # 'deps'. The executable should be created as "$out/bin/${name}", usually + # by dumping the lisp image with the replaced toplevel function replaced. + # - wrapProgram :: boolean + # Whether to wrap the resulting binary / image with a wrapper script setting + # `LD_LIBRARY_PATH`. + # - genTestLisp :: { name, srcs, deps, expression } -> file + # Builds a lisp file which loads the given 'deps' and 'srcs' files and + # then evaluates 'expression'. Depending on whether 'expression' returns + # true or false, the script must exit with a zero or non-zero exit code. + # 'genTestLispGeneric' will work for most implementations. + # - lispWith :: [ dependency ] -> drv + # Builds a script (or dumped image) which when executed loads (or has + # loaded) all given dependencies. When built this should create an executable + # at "$out/bin/${implementation}". + # + # Optional members: + # + # - bundled :: string -> library + # Allows giving an implementation specific builder for a bundled library. + # This function is used as a replacement for the internal defaultBundled + # function and only needs to support one implementation. The returned derivation + # must behave like one built by 'library' (in particular have the same files + # available in "$out" and the same 'passthru' attributes), but may be built + # completely differently. + impls = lib.mapAttrs (name: v: { inherit name; } // v) { + sbcl = { + runScript = "${sbcl}/bin/sbcl --script"; + faslExt = "fasl"; + + # 'genLoadLisp' generates Lisp code that instructs SBCL to load all + # the provided Lisp libraries. + genLoadLisp = genLoadLispGeneric impls.sbcl; + + # 'genCompileLisp' generates a Lisp file that instructs SBCL to + # compile the provided list of Lisp source files to "$out/${name}.fasl". + genCompileLisp = { name, srcs, deps }: writeText "sbcl-compile.lisp" '' + ;; This file compiles the specified sources into the Nix build + ;; directory, creating one FASL file for each source. + (require 'sb-posix) + + ${impls.sbcl.genLoadLisp deps} + + (defun nix-compile-lisp (srcfile) + (let ((outfile (make-pathname :type "fasl" + :directory (or (sb-posix:getenv "NIX_BUILD_TOP") + (error "not running in a Nix build")) + :name (substitute #\- #\/ srcfile)))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :output-file outfile) + (if failure-p (sb-posix:exit 1) + (progn + ;; For the case of multiple files belonging to the same + ;; library being compiled, load them in order: + (load out-truename) + + ;; Return pathname as a string for cat-ting it later + (namestring out-truename)))))) + + (let ((*compile-verbose* t) + (catted-fasl (make-pathname :type "fasl" + :directory (or (sb-posix:getenv "out") + (error "not running in a Nix build")) + :name "${name}"))) + + (with-open-file (file catted-fasl + :direction :output + :if-does-not-exist :create) + + ;; SBCL's FASL files can just be bundled together using cat + (sb-ext:run-program "cat" + (mapcar #'nix-compile-lisp + ;; These forms were inserted by the Nix build: + '(${ + lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs + })) + :output file :search t))) + ''; + + # 'genDumpLisp' generates a Lisp file that instructs SBCL to dump + # the currently loaded image as an executable to $out/bin/$name. + # + # TODO(tazjin): Compression is currently unsupported because the + # SBCL in nixpkgs is, by default, not compiled with zlib support. + genDumpLisp = { name, main, deps }: writeText "sbcl-dump.lisp" '' + (require 'sb-posix) + + ${impls.sbcl.genLoadLisp deps} + + (let* ((bindir (concatenate 'string (sb-posix:getenv "out") "/bin")) + (outpath (make-pathname :name "${name}" + :directory bindir))) + (save-lisp-and-die outpath + :executable t + :toplevel + (lambda () + ;; Filter out everything prior to the `--` we + ;; insert in the wrapper to prevent SBCL from + ;; parsing arguments at startup + (setf sb-ext:*posix-argv* + (delete "--" sb-ext:*posix-argv* + :test #'string= :count 1)) + (${main})) + :purify t)) + ''; + + wrapProgram = true; + + genTestLisp = genTestLispGeneric impls.sbcl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.sbcl deps); + in writeShellScriptBin "sbcl" '' + export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}" + export LANG="C.UTF-8" + exec ${sbcl}/bin/sbcl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.sbcl.genLoadLisp lispDeps)}" + } $@ + ''; + }; + ecl = { + runScript = "${ecl-static}/bin/ecl --load ${disableDebugger} --shell"; + faslExt = "fasc"; + genLoadLisp = genLoadLispGeneric impls.ecl; + genCompileLisp = { name, srcs, deps }: writeText "ecl-compile.lisp" '' + ;; This seems to be required to bring make the 'c' package available + ;; early, otherwise ECL tends to fail with a read failure… + (ext:install-c-compiler) + + ;; Load dependencies + ${impls.ecl.genLoadLisp deps} + + (defun getenv-or-fail (var) + (or (ext:getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + (defun nix-compile-file (srcfile &key native) + "Compile the given srcfile into a compilation unit in :out-dir using + a unique name based on srcfile as the filename which is returned after + compilation. If :native is true, create an native object file, + otherwise a byte-compile fasc file is built and immediately loaded." + + (let* ((unique-name (substitute #\_ #\/ srcfile)) + (out-file (make-pathname :type (if native "o" "fasc") + :directory (getenv-or-fail "NIX_BUILD_TOP") + :name unique-name))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :system-p native + :load (not native) + :output-file out-file + :verbose t :print t) + (if failure-p (ext:quit 1) out-truename)))) + + (let* ((out-dir (getenv-or-fail "out")) + (nix-build-dir (getenv-or-fail "NIX_BUILD_TOP")) + (srcs + ;; These forms are inserted by the Nix build + '(${lib.concatMapStringsSep "\n" (src: "\"${src}\"") srcs}))) + + ;; First, we'll byte compile loadable FASL files and load them + ;; immediately. Since we are using a statically linked ECL, there's + ;; no way to load native objects, so we rely on byte compilation + ;; for all our loading — which is crucial in compilation of course. + (ext:install-bytecodes-compiler) + + ;; ECL's bytecode FASLs can just be concatenated to create a bundle + ;; at least since a recent bugfix which we apply as a patch. + ;; See also: https://gitlab.com/embeddable-common-lisp/ecl/-/issues/649 + (let ((bundle-out (make-pathname :type "fasc" :name "${name}" + :directory out-dir))) + + (with-open-file (fasc-stream bundle-out :direction :output) + (ext:run-program "cat" + (mapcar (lambda (f) + (namestring + (nix-compile-file f :native nil))) + srcs) + :output fasc-stream))) + + (ext:install-c-compiler) + + ;; Build a (natively compiled) static archive (.a) file. We want to + ;; use this for (statically) linking an executable later. The bytecode + ;; dance is only required because we can't load such archives. + (c:build-static-library + (make-pathname :type "a" :name "${name}" :directory out-dir) + :lisp-files (mapcar (lambda (x) + (nix-compile-file x :native t)) + srcs))) + ''; + genDumpLisp = { name, main, deps }: writeText "ecl-dump.lisp" '' + (defun getenv-or-fail (var) + (or (ext:getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + ${impls.ecl.genLoadLisp deps} + + ;; makes a 'c' package available that can link executables + (ext:install-c-compiler) + + (c:build-program + (make-pathname :name "${name}" + :directory (concatenate 'string + (getenv-or-fail "out") + "/bin")) + :epilogue-code `(progn + ;; UIOP doesn't understand ECL, so we need to make it + ;; aware that we are a proper executable, causing it + ;; to handle argument parsing and such properly. Since + ;; this needs to work even when we're not using UIOP, + ;; we need to do some compile-time acrobatics. + ,(when (find-package 'uiop) + `(setf ,(find-symbol "*IMAGE-DUMPED-P*" :uiop) :executable)) + ;; Run the actual application… + (${main}) + ;; … and exit. + (ext:quit)) + ;; ECL can't remember these from its own build… + :ld-flags '("-static") + :lisp-files + ;; The following forms are inserted by the Nix build + '(${ + lib.concatMapStrings (dep: '' + "${dep}/${dep.lispName}.a" + '') (allDeps impls.ecl deps) + })) + ''; + + wrapProgram = false; + + genTestLisp = genTestLispGeneric impls.ecl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ecl deps); + in writeShellScriptBin "ecl" '' + exec ${ecl-static}/bin/ecl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.ecl.genLoadLisp lispDeps)}" + } $@ + ''; + + bundled = name: runCommandNoCC "${name}-cllib" { + passthru = { + lispName = name; + lispNativeDeps = []; + lispDeps = []; + lispBinary = false; + repl = impls.ecl.lispWith [ (impls.ecl.bundled name) ]; + }; + } '' + mkdir -p "$out" + ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/${name}.${impls.ecl.faslExt}" -t "$out" + ln -s "${ecl-static}/lib/ecl-${ecl-static.version}/lib${name}.a" "$out/${name}.a" + ''; + }; + ccl = { + # Relatively bespoke wrapper script necessary to make CCL just™ execute + # a lisp file as a script. + runScript = pkgs.writers.writeBash "ccl" '' + # don't print intro message etc. + args=("--quiet") + + # makes CCL crash on error instead of entering the debugger + args+=("--load" "${disableDebugger}") + + # load files from command line in order + for f in "$@"; do + args+=("--load" "$f") + done + + # Exit if everything was processed successfully + args+=("--eval" "(quit)") + + exec ${ccl}/bin/ccl ''${args[@]} + ''; + + # See https://ccl.clozure.com/docs/ccl.html#building-definitions + faslExt = + /**/ if targetPlatform.isPowerPC && targetPlatform.is32bit then "pfsl" + else if targetPlatform.isPowerPC && targetPlatform.is64bit then "p64fsl" + else if targetPlatform.isx86_64 && targetPlatform.isLinux then "lx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isLinux then "lx32fsl" + else if targetPlatform.isAarch32 && targetPlatform.isLinux then "lafsl" + else if targetPlatform.isx86_32 && targetPlatform.isDarwin then "dx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl" + else if targetPlatform.isx86_64 && targetPlatform.isDarwin then "dx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isFreeBSD then "fx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isFreeBSD then "fx64fsl" + else if targetPlatform.isx86_32 && targetPlatform.isWindows then "wx32fsl" + else if targetPlatform.isx86_64 && targetPlatform.isWindows then "wx64fsl" + else builtins.throw "Don't know what FASLs are called for this platform: " + + pkgs.stdenv.targetPlatform.system; + + genLoadLisp = genLoadLispGeneric impls.ccl; + + genCompileLisp = { name, srcs, deps }: writeText "ccl-compile.lisp" '' + ${impls.ccl.genLoadLisp deps} + + (defun getenv-or-fail (var) + (or (getenv var) + (error (format nil "Missing expected environment variable ~A" var)))) + + (defun nix-compile-file (srcfile) + "Trivial wrapper around COMPILE-FILE which causes CCL to exit if + compilation fails and LOADs the compiled file on success." + (let ((output (make-pathname :name (substitute #\_ #\/ srcfile) + :type "${impls.ccl.faslExt}" + :directory (getenv-or-fail "NIX_BUILD_TOP")))) + (multiple-value-bind (out-truename _warnings-p failure-p) + (compile-file srcfile :output-file output :print t :verbose t) + (declare (ignore _warnings-p)) + (if failure-p (quit 1) + (progn (load out-truename) out-truename))))) + + (fasl-concatenate (make-pathname :name "${name}" :type "${impls.ccl.faslExt}" + :directory (getenv-or-fail "out")) + (mapcar #'nix-compile-file + ;; These forms where inserted by the Nix build + '(${ + lib.concatMapStrings (src: '' + "${src}" + '') srcs + }))) + ''; + + genDumpLisp = { name, main, deps }: writeText "ccl-dump.lisp" '' + ${impls.ccl.genLoadLisp deps} + + (let* ((out (or (getenv "out") (error "Not running in a Nix build"))) + (bindir (concatenate 'string out "/bin/")) + (executable (make-pathname :directory bindir :name "${name}"))) + + (save-application executable + :purify t + :error-handler :quit + :toplevel-function + (lambda () + ;; Filter out everything prior to the `--` we + ;; insert in the wrapper to prevent SBCL from + ;; parsing arguments at startup + (setf ccl:*command-line-argument-list* + (delete "--" ccl:*command-line-argument-list* + :test #'string= :count 1)) + (${main})) + :mode #o755 + ;; TODO(sterni): use :native t on macOS + :prepend-kernel t)) + ''; + + wrapProgram = true; + + genTestLisp = genTestLispGeneric impls.ccl; + + lispWith = deps: + let lispDeps = filter (d: !d.lispBinary) (allDeps impls.ccl deps); + in writeShellScriptBin "ccl" '' + export LD_LIBRARY_PATH="${lib.makeLibraryPath (allNative [] lispDeps)}" + exec ${ccl}/bin/ccl ${ + lib.optionalString (deps != []) + "--load ${writeText "load.lisp" (impls.ccl.genLoadLisp lispDeps)}" + } "$@" + ''; + }; + }; + + # + # Public API functions + # + + # 'library' builds a list of Common Lisp files into an implementation + # specific library format, usually a single FASL file, which can then be + # loaded and built into an executable via 'program'. + library = + { name + , implementation ? defaultImplementation + , brokenOn ? [] # TODO(sterni): make this a warning + , srcs + , deps ? [] + , native ? [] + , tests ? null + , passthru ? {} + }: + let + filteredDeps = implFilter implementation deps; + filteredSrcs = implFilter implementation srcs; + lispNativeDeps = (allNative native filteredDeps); + lispDeps = allDeps implementation filteredDeps; + testDrv = if ! isNull tests + then testSuite { + name = tests.name or "${name}-test"; + srcs = filteredSrcs ++ (tests.srcs or []); + deps = filteredDeps ++ (tests.deps or []); + expression = tests.expression; + inherit implementation; + } + else null; + in lib.fix (self: runCommandNoCC "${name}-cllib" { + LD_LIBRARY_PATH = lib.makeLibraryPath lispNativeDeps; + LANG = "C.UTF-8"; + passthru = passthru // { + inherit lispNativeDeps lispDeps; + lispName = name; + lispBinary = false; + tests = testDrv; + }; + } '' + ${if ! isNull testDrv + then "echo 'Test ${testDrv} succeeded'" + else "echo 'No tests run'"} + + mkdir $out + + ${implementation.runScript} ${ + implementation.genCompileLisp { + srcs = filteredSrcs; + inherit name; + deps = lispDeps; + } + } + ''); + + # 'program' creates an executable, usually containing a dumped image of the + # specified sources and dependencies. + program = + { name + , implementation ? defaultImplementation + , brokenOn ? [] # TODO(sterni): make this a warning + , main ? "${name}:main" + , srcs + , deps ? [] + , native ? [] + , tests ? null + , passthru ? {} + }: + let + filteredSrcs = implFilter implementation srcs; + filteredDeps = implFilter implementation deps; + lispDeps = allDeps implementation filteredDeps; + libPath = lib.makeLibraryPath (allNative native lispDeps); + # overriding is used internally to propagate the implementation to use + selfLib = (makeOverridable library) { + inherit name native brokenOn; + deps = lispDeps; + srcs = filteredSrcs; + }; + testDrv = if ! isNull tests + then testSuite { + name = tests.name or "${name}-test"; + srcs = + ( # testSuite does run implFilter as well + filteredSrcs ++ (tests.srcs or [])); + deps = filteredDeps ++ (tests.deps or []); + expression = tests.expression; + inherit implementation; + } + else null; + in lib.fix (self: runCommandNoCC "${name}" { + nativeBuildInputs = [ makeWrapper ]; + LD_LIBRARY_PATH = libPath; + LANG = "C.UTF-8"; + passthru = passthru // { + lispName = name; + lispDeps = [ selfLib ]; + lispNativeDeps = native; + lispBinary = true; + tests = testDrv; + }; + } ('' + ${if ! isNull testDrv + then "echo 'Test ${testDrv} succeeded'" + else ""} + mkdir -p $out/bin + + ${implementation.runScript} ${ + implementation.genDumpLisp { + inherit name main; + deps = ([ selfLib ] ++ lispDeps); + } + } + '' + lib.optionalString implementation.wrapProgram '' + wrapProgram $out/bin/${name} \ + --prefix LD_LIBRARY_PATH : "${libPath}" \ + --add-flags "\$NIX_BUILDLISP_LISP_ARGS --" + '')); + + # 'bundled' creates a "library" which makes a built-in package available, + # such as any of SBCL's sb-* packages or ASDF. By default this is done + # by calling 'require', but implementations are free to provide their + # own specific bundled function. + bundled = name: + let + # TODO(sterni): allow overriding args to underlying 'library' (e. g. srcs) + defaultBundled = implementation: name: library { + inherit name implementation; + srcs = lib.singleton (builtins.toFile "${name}.lisp" "(require '${name})"); + }; + + bundled' = + { implementation ? defaultImplementation + , name + }: + implementation.bundled or (defaultBundled implementation) name; + + in (makeOverridable bundled') { + inherit name; + }; + +in { + library = withExtras library; + program = withExtras program; + inherit bundled; + + # 'sbclWith' creates an image with the specified libraries / + # programs loaded in SBCL. + sbclWith = impls.sbcl.lispWith; + + inherit (impls) + sbcl + ecl + ccl + ; +} diff --git a/nix/buildLisp/example/default.nix b/nix/buildLisp/example/default.nix new file mode 100644 index 000000000000..6a518e4964a1 --- /dev/null +++ b/nix/buildLisp/example/default.nix @@ -0,0 +1,32 @@ +{ depot, ... }: + +let + inherit (depot.nix) buildLisp; + + # Example Lisp library. + # + # Currently the `name` attribute is only used for the derivation + # itself, it has no practical implications. + libExample = buildLisp.library { + name = "lib-example"; + srcs = [ + ./lib.lisp + ]; + }; + +# Example Lisp program. +# +# This builds & writes an executable for a program using the library +# above to disk. +# +# By default, buildLisp.program expects the entry point to be +# `$name:main`. This can be overridden by configuring the `main` +# attribute. +in buildLisp.program { + name = "example"; + deps = [ libExample ]; + + srcs = [ + ./main.lisp + ]; +} diff --git a/nix/buildLisp/example/lib.lisp b/nix/buildLisp/example/lib.lisp new file mode 100644 index 000000000000..e557de4ae5fd --- /dev/null +++ b/nix/buildLisp/example/lib.lisp @@ -0,0 +1,6 @@ +(defpackage lib-example + (:use :cl) + (:export :who)) +(in-package :lib-example) + +(defun who () "edef") diff --git a/nix/buildLisp/example/main.lisp b/nix/buildLisp/example/main.lisp new file mode 100644 index 000000000000..a29390cf4dba --- /dev/null +++ b/nix/buildLisp/example/main.lisp @@ -0,0 +1,7 @@ +(defpackage example + (:use :cl :lib-example) + (:export :main)) +(in-package :example) + +(defun main () + (format t "i <3 ~A~%" (who))) diff --git a/nix/buildManPages/OWNERS b/nix/buildManPages/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/nix/buildManPages/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/nix/buildManPages/default.nix b/nix/buildManPages/default.nix new file mode 100644 index 000000000000..fe6d145f6cee --- /dev/null +++ b/nix/buildManPages/default.nix @@ -0,0 +1,76 @@ +{ depot, pkgs, lib, ... }: + +let + inherit (pkgs) + gzip + mandoc + coreutils + ; + + inherit (depot.nix) + runExecline + getBins + ; + + bins = getBins mandoc [ "mandoc" ] + // getBins gzip [ "gzip" ] + // getBins coreutils [ "mkdir" "ln" "cp" ] + ; + + defaultGzip = true; + + basename = gzip: { name, section, ... }: + "${name}.${toString section}${lib.optionalString gzip ".gz"}"; + + manDir = { section, ... }: + "\${out}/share/man/man${toString section}"; + + target = gzip: args: + "${manDir args}/${basename gzip args}"; + + buildManPage = + { requireLint ? false + , gzip ? defaultGzip + , ... + }: + { content + , ... + }@page: let + source = builtins.toFile (basename false page) content; + in runExecline (basename gzip page) {} ([ + (if requireLint then "if" else "foreground") [ + bins.mandoc "-mdoc" "-T" "lint" source + ] + "importas" "out" "out" + ] ++ (if gzip then [ + "redirfd" "-w" "1" "$out" + bins.gzip "-c" source + ] else [ + bins.cp "--reflink=auto" source "$out" + ])); + + buildManPages = + name: + { derivationArgs ? {} + , gzip ? defaultGzip + , ... + }@args: + pages: + runExecline "${name}-man-pages" { + inherit derivationArgs; + } ([ + "importas" "out" "out" + ] ++ lib.concatMap ({ name, section, content }@page: [ + "if" [ bins.mkdir "-p" (manDir page) ] + "if" [ + bins.ln "-s" + (buildManPage args page) + (target gzip page) + ] + ]) pages); + +in { + __functor = _: buildManPages; + + single = buildManPage; +} diff --git a/nix/drvSeqL/default.nix b/nix/drvSeqL/default.nix new file mode 100644 index 000000000000..3339289b3bd1 --- /dev/null +++ b/nix/drvSeqL/default.nix @@ -0,0 +1,43 @@ +{ depot, lib, pkgs, ... }: + +let + + inherit (depot.nix.yants) + defun + list + drv + ; + + /* Realize drvDeps, then return drvOut if that succeds. + * This can be used to make drvOut depend on the + * build success of all drvDeps without making each drvDep + * a dependency of drvOut. + * => drvOut is not rebuilt if drvDep changes + */ + drvSeqL = defun [ (list drv) drv drv ] + (drvDeps: drvOut: + let + drvOutOutputs = drvOut.outputs or ["out"]; + in + pkgs.runCommandLocal drvOut.name { + # we inherit all attributes in order to replicate + # the original derivation as much as possible + outputs = drvOutOutputs; + passthru = drvOut.drvAttrs; + # depend on drvDeps (by putting it in builder context) + inherit drvDeps; + } + # the outputs of the original derivation are replicated + # by creating a symlink to the old output path + (lib.concatMapStrings (output: '' + target=${lib.escapeShellArg drvOut.${output}} + # if the target is already a symlink, follow it until it’s not; + # this is done to prevent too many dereferences + target=$(readlink -e "$target") + # link to the output + ln -s "$target" "${"$"}${output}" + '') drvOutOutputs)); + +in { + __functor = _: drvSeqL; +} diff --git a/nix/emptyDerivation/OWNERS b/nix/emptyDerivation/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/nix/emptyDerivation/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/nix/emptyDerivation/default.nix b/nix/emptyDerivation/default.nix new file mode 100644 index 000000000000..4165d4fd9ac1 --- /dev/null +++ b/nix/emptyDerivation/default.nix @@ -0,0 +1,20 @@ +{ depot, pkgs, ... }: + +let + emptyDerivation = import ./emptyDerivation.nix { + inherit pkgs; + inherit (pkgs) stdenv; + inherit (depot.nix) getBins; + }; + + tests = import ./tests.nix { + inherit emptyDerivation; + inherit pkgs; + inherit (depot.nix) writeExecline getBins; + inherit (depot.nix.runTestsuite) runTestsuite it assertEq; + }; + +in { + __functor = _: emptyDerivation; + inherit tests; +} diff --git a/nix/emptyDerivation/emptyDerivation.nix b/nix/emptyDerivation/emptyDerivation.nix new file mode 100644 index 000000000000..5e84abe2d505 --- /dev/null +++ b/nix/emptyDerivation/emptyDerivation.nix @@ -0,0 +1,34 @@ +{ stdenv, pkgs, getBins }: + +# The empty derivation. All it does is touch $out. +# Basically the unit value for derivations. +# +# In addition to simple test situations which require +# a derivation, we set __functor, so you can call it +# as a function and pass an attrset. The set you pass +# is `//`-merged with the attrset before calling derivation, +# so you can use this to add more fields. + +let + bins = getBins pkgs.s6-portable-utils [ "s6-touch" ] + // getBins pkgs.execline [ "importas" "exec" ]; + + emptiness = { + name = "empty-derivation"; + + # TODO(Profpatsch): can we get system from tvl? + inherit (stdenv) system; + + builder = bins.exec; + args = [ + bins.importas "out" "out" + bins.s6-touch "$out" + ]; + }; + +in (derivation emptiness) // { + # This allows us to call the empty derivation + # like a function and override fields/add new fields. + __functor = _: overrides: + derivation (emptiness // overrides); +} diff --git a/nix/emptyDerivation/tests.nix b/nix/emptyDerivation/tests.nix new file mode 100644 index 000000000000..053603b02772 --- /dev/null +++ b/nix/emptyDerivation/tests.nix @@ -0,0 +1,32 @@ +{ emptyDerivation, getBins, pkgs, writeExecline, runTestsuite, it, assertEq }: + +let + bins = getBins pkgs.s6-portable-utils [ "s6-echo" ]; + + empty = it "is just an empty path" [ + (assertEq "path empty" + (builtins.readFile emptyDerivation) + "") + ]; + + fooOut = emptyDerivation { + builder = writeExecline "foo-builder" {} [ + "importas" "out" "out" + "redirfd" "-w" "1" "$out" + bins.s6-echo "-n" "foo" + ]; + }; + + overrideBuilder = it "can override the builder" [ + (assertEq "output is foo" + (builtins.readFile fooOut) + "foo") + (assertEq "can add new drv variables" + (emptyDerivation { foo = "bar"; }).foo + "bar") + ]; + +in runTestsuite "emptyDerivation" [ + empty + overrideBuilder +] diff --git a/nix/escapeExecline/default.nix b/nix/escapeExecline/default.nix new file mode 100644 index 000000000000..deef5c2c4ec8 --- /dev/null +++ b/nix/escapeExecline/default.nix @@ -0,0 +1,29 @@ +{ lib, ... }: +let + # replaces " and \ to \" and \\ respectively and quote with " + # e.g. + # a"b\c -> "a\"b\\c" + # a\"bc -> "a\\\"bc" + escapeExeclineArg = arg: + ''"${builtins.replaceStrings [ ''"'' ''\'' ] [ ''\"'' ''\\'' ] (toString arg)}"''; + + # Escapes an execline (list of execline strings) to be passed to execlineb + # Give it a nested list of strings. Nested lists are interpolated as execline + # blocks ({}). + # Everything is quoted correctly. + # + # Example: + # escapeExecline [ "if" [ "somecommand" ] "true" ] + # == ''"if" { "somecommand" } "true"'' + escapeExecline = execlineList: lib.concatStringsSep " " + (let + go = arg: + if builtins.isString arg then [(escapeExeclineArg arg)] + else if builtins.isPath arg then [(escapeExeclineArg "${arg}")] + else if lib.isDerivation arg then [(escapeExeclineArg arg)] + else if builtins.isList arg then [ "{" ] ++ builtins.concatMap go arg ++ [ "}" ] + else abort "escapeExecline can only hande nested lists of strings, was ${lib.generators.toPretty {} arg}"; + in builtins.concatMap go execlineList); + +in +escapeExecline diff --git a/nix/getBins/default.nix b/nix/getBins/default.nix new file mode 100644 index 000000000000..5ba7584ed844 --- /dev/null +++ b/nix/getBins/default.nix @@ -0,0 +1,48 @@ +{ lib, pkgs, depot, ... }: + +# Takes a derivation and a list of binary names +# and returns an attribute set of `name -> path`. +# The list can also contain renames in the form of +# `{ use, as }`, which goes `as -> usePath`. +# +# It is usually used to construct an attrset `bins` +# containing all the binaries required in a file, +# similar to a simple import system. +# +# Example: +# +# bins = getBins pkgs.hello [ "hello" ] +# // getBins pkgs.coreutils [ "printf" "ln" "echo" ] +# // getBins pkgs.execline +# [ { use = "if"; as = "execlineIf" } ] +# // getBins pkgs.s6-portable-utils +# [ { use = "s6-test"; as = "test" } +# { use = "s6-cat"; as = "cat" } +# ]; +# +# provides +# bins.{hello,printf,ln,echo,execlineIf,test,cat} +# + +let + getBins = drv: xs: + let f = x: + # TODO(Profpatsch): typecheck + let x' = if builtins.isString x then { use = x; as = x; } else x; + in { + name = x'.as; + value = "${lib.getBin drv}/bin/${x'.use}"; + }; + in builtins.listToAttrs (builtins.map f xs); + + + tests = import ./tests.nix { + inherit getBins; + inherit (depot.nix) writeScriptBin; + inherit (depot.nix.runTestsuite) assertEq it runTestsuite; + }; + +in { + __functor = _: getBins; + inherit tests; +} diff --git a/nix/getBins/tests.nix b/nix/getBins/tests.nix new file mode 100644 index 000000000000..ff81deb5f1ec --- /dev/null +++ b/nix/getBins/tests.nix @@ -0,0 +1,40 @@ +{ writeScriptBin, assertEq, it, runTestsuite, getBins }: + +let + drv = writeScriptBin "hello" "it’s me"; + drv2 = writeScriptBin "goodbye" "tschau"; + + bins = getBins drv [ + "hello" + { use = "hello"; as = "also-hello"; } + ] + // getBins drv2 [ "goodbye" ] + ; + + simple = it "path is equal to the executable name" [ + (assertEq "path" + bins.hello + "${drv}/bin/hello") + (assertEq "content" + (builtins.readFile bins.hello) + "it’s me") + ]; + + useAs = it "use/as can be used to rename attributes" [ + (assertEq "path" + bins.also-hello + "${drv}/bin/hello") + ]; + + secondDrv = it "by merging attrsets you can build up bins" [ + (assertEq "path" + bins.goodbye + "${drv2}/bin/goodbye") + ]; + +in + runTestsuite "getBins" [ + simple + useAs + secondDrv + ] diff --git a/nix/mergePatch/default.nix b/nix/mergePatch/default.nix new file mode 100644 index 000000000000..0f80b93d4c65 --- /dev/null +++ b/nix/mergePatch/default.nix @@ -0,0 +1,186 @@ +{ lib, depot, ... }: +/* + JSON Merge-Patch for nix + Spec: https://tools.ietf.org/html/rfc7396 + + An algorithm for changing and removing fields in nested objects. + + For example, given the following original document: + + { + a = "b"; + c = { + d = "e"; + f = "g"; + } + } + + Changing the value of `a` and removing `f` can be achieved by merging the patch + + { + a = "z"; + c.f = null; + } + + which results in + + { + a = "z"; + c = { + d = "e"; + }; + } + + Pseudo-code: + define MergePatch(Target, Patch): + if Patch is an Object: + if Target is not an Object: + Target = {} # Ignore the contents and set it to an empty Object + for each Name/Value pair in Patch: + if Value is null: + if Name exists in Target: + remove the Name/Value pair from Target + else: + Target[Name] = MergePatch(Target[Name], Value) + return Target + else: + return Patch +*/ + +let + foldlAttrs = op: init: attrs: + lib.foldl' op init + (lib.mapAttrsToList lib.nameValuePair attrs); + + mergePatch = target: patch: + if lib.isAttrs patch + then + let target' = if lib.isAttrs target then target else {}; + in foldlAttrs + (acc: patchEl: + if patchEl.value == null + then removeAttrs acc [ patchEl.name ] + else acc // { + ${patchEl.name} = + mergePatch + (acc.${patchEl.name} or "unnused") + patchEl.value; + }) + target' + patch + else patch; + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + tests = + let + # example target from the RFC + testTarget = { + a = "b"; + c = { + d = "e"; + f = "g"; + }; + }; + # example patch from the RFC + testPatch = { + a = "z"; + c.f = null; + }; + emptyPatch = it "the empty patch returns the original target" [ + (assertEq "id" + (mergePatch testTarget {}) + testTarget) + ]; + nonAttrs = it "one side is a non-attrset value" [ + (assertEq "target is a value means the value is replaced by the patch" + (mergePatch 42 testPatch) + (mergePatch {} testPatch)) + (assertEq "patch is a value means it replaces target alltogether" + (mergePatch testTarget 42) + 42) + ]; + rfcExamples = it "the examples from the RFC" [ + (assertEq "a subset is deleted and overwritten" + (mergePatch testTarget testPatch) { + a = "z"; + c = { + d = "e"; + }; + }) + (assertEq "a more complicated example from the example section" + (mergePatch { + title = "Goodbye!"; + author = { + givenName = "John"; + familyName = "Doe"; + }; + tags = [ "example" "sample" ]; + content = "This will be unchanged"; + } { + title = "Hello!"; + phoneNumber = "+01-123-456-7890"; + author.familyName = null; + tags = [ "example" ]; + }) + { + title = "Hello!"; + phoneNumber = "+01-123-456-7890"; + author = { + givenName = "John"; + }; + tags = [ "example" ]; + content = "This will be unchanged"; + }) + ]; + + rfcTests = + let + r = index: target: patch: res: + (assertEq "test number ${toString index}" + (mergePatch target patch) + res); + in it "the test suite from the RFC" [ + (r 1 {"a" = "b";} {"a" = "c";} {"a" = "c";}) + (r 2 {"a" = "b";} {"b" = "c";} {"a" = "b"; "b" = "c";}) + (r 3 {"a" = "b";} {"a" = null;} {}) + (r 4 {"a" = "b"; "b" = "c";} + {"a" = null;} + {"b" = "c";}) + (r 5 {"a" = ["b"];} {"a" = "c";} {"a" = "c";}) + (r 6 {"a" = "c";} {"a" = ["b"];} {"a" = ["b"];}) + (r 7 {"a" = {"b" = "c";}; } + {"a" = {"b" = "d"; "c" = null;};} + {"a" = {"b" = "d";};}) + (r 8 {"a" = [{"b" = "c";}];} + {"a" = [1];} + {"a" = [1];}) + (r 9 ["a" "b"] ["c" "d"] ["c" "d"]) + (r 10 {"a" = "b";} ["c"] ["c"]) + (r 11 {"a" = "foo";} null null) + (r 12 {"a" = "foo";} "bar" "bar") + (r 13 {"e" = null;} {"a" = 1;} {"e" = null; "a" = 1;}) + (r 14 [1 2] + {"a" = "b"; "c" = null;} + {"a" = "b";}) + (r 15 {} + {"a" = {"bb" = {"ccc" = null;};};} + {"a" = {"bb" = {};};}) + ]; + + in runTestsuite "mergePatch" [ + emptyPatch + nonAttrs + rfcExamples + rfcTests + ]; + +in { + __functor = _: mergePatch; + + inherit tests; +} diff --git a/nix/netstring/attrsToKeyValList.nix b/nix/netstring/attrsToKeyValList.nix new file mode 100644 index 000000000000..2805d0fbcef6 --- /dev/null +++ b/nix/netstring/attrsToKeyValList.nix @@ -0,0 +1,33 @@ +{ depot, lib, ... }: + +# Convert an attrset of strings to a list of key/value netstring pairs. +# A good minimally viable json replacement if all you need is to iterate. +# You can use e.g. `forstdin -Ed '' item` in execline to split the items +# and then get the key and value via `multidefine -d '' $item { key value }` +# +# Example: +# { foo = "bar"; x = "abc"; } +# => "12:3:foo,3:bar,,10:1:x,3:abc,," +# +# Example with runExecline: +# nix.runExecline "test" { +# stdin = nix.netstring.attrsToKeyValList { +# foo = "bar"; +# x = "abc"; +# }; +# } [ +# "forstdin" "-Ed" "" "item" +# "multidefine" "-d" "" "$item" [ "key" "value" ] +# "${pkgs.coreutils}/bin/echo" "\${key} -> \${value}" +# ] + +# will print: +# foo -> bar +# x -> abc +attrs: +lib.concatStrings + (lib.mapAttrsToList + (k: v: depot.nix.netstring.fromString + ( depot.nix.netstring.fromString k + + depot.nix.netstring.fromString v)) + attrs) diff --git a/nix/netstring/fromString.nix b/nix/netstring/fromString.nix new file mode 100644 index 000000000000..dcd688a8b0b3 --- /dev/null +++ b/nix/netstring/fromString.nix @@ -0,0 +1,10 @@ +{ ... }: +# convert any nix string into a netstring +# (prefixed by its length) according to https://en.wikipedia.org/wiki/Netstring +# +# Examples: +# netstring.fromString "foo" +# => "3:foo," +# netstring.fromString "" +# => "0:," +s: "${toString (builtins.stringLength s)}:${s}," diff --git a/nix/nint/OWNERS b/nix/nint/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/nix/nint/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/nix/nint/README.md b/nix/nint/README.md new file mode 100644 index 000000000000..369a8276199a --- /dev/null +++ b/nix/nint/README.md @@ -0,0 +1,93 @@ +# nint — Nix INTerpreter + +`nint` is a shebang compatible interpreter for nix. It is currently +implemented as a fairly trivial wrapper around `nix-instantiate --eval`. +It allows to run nix expressions as command line tools if they conform +to the following calling convention: + +* Every nix script needs to evaluate to a function which takes an + attribute set as its single argument. Ideally a set pattern with + an ellipsis should be used. By default `nint` passes the following + arguments: + + * `currentDir`: the current working directory as a nix path + * `argv`: a list of arguments to the invokation including the + program name at `builtins.head argv`. + * Extra arguments can be manually passed as described below. + +* The return value must either be + + * A string which is rendered to `stdout`. + + * An attribute set with the following optional attributes: + + * `stdout`: A string that's rendered to `stdout` + * `stderr`: A string that's rendered to `stderr` + * `exit`: A number which is used as an exit code. + If missing, nint always exits with 0 (or equivalent). + +## Usage + +``` +nint [ --arg ARG VALUE … ] script.nix [ ARGS … ] +``` + +Instead of `--arg`, `--argstr` can also be used. They both work +like the flags of the same name for `nix-instantiate` and may +be specified any number of times as long as they are passed +*before* the nix expression to run. + +Below is a shebang which also passes `depot` as an argument +(note the usage of `env -S` to get around the shebang limitation +to two arguments). + +```nix +#!/usr/bin/env -S nint --arg depot /path/to/depot +``` + +## Limitations + +* No side effects except for writing to `stdout`. + +* Output is not streaming, i. e. even if the output is incrementally + calculated, nothing will be printed until the full output is available. + With plain nix strings we can't do better anyways. + +* Limited error handling for the script, no way to set the exit code etc. + +Some of these limitations may be possible to address in the future by using +an alternative nix interpreter and a more elaborate calling convention. + +## Example + +Below is a (very simple) implementation of a `ls(1)`-like program in nix: + +```nix +#!/usr/bin/env nint +{ currentDir, argv, ... }: + +let + lib = import <nixpkgs/lib>; + + dirs = + let + args = builtins.tail argv; + in + if args == [] + then [ currentDir ] + else args; + + makeAbsolute = p: + if builtins.isPath p + then p + else if builtins.match "^/.*" p != null + then p + else "${toString currentDir}/${p}"; +in + + lib.concatStringsSep "\n" + (lib.flatten + (builtins.map + (d: (builtins.attrNames (builtins.readDir (makeAbsolute d)))) + dirs)) + "\n" +``` diff --git a/nix/nint/default.nix b/nix/nint/default.nix new file mode 100644 index 000000000000..5cf83d15d6f3 --- /dev/null +++ b/nix/nint/default.nix @@ -0,0 +1,14 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.nix.writers) + rustSimpleBin + ; +in + + rustSimpleBin { + name = "nint"; + dependencies = [ + depot.third_party.rust-crates.serde_json + ]; + } (builtins.readFile ./nint.rs) diff --git a/nix/nint/nint.rs b/nix/nint/nint.rs new file mode 100644 index 000000000000..1fa4dccb4f86 --- /dev/null +++ b/nix/nint/nint.rs @@ -0,0 +1,156 @@ +extern crate serde_json; + +use serde_json::Value; +use std::ffi::OsString; +use std::os::unix::ffi::{OsStringExt, OsStrExt}; +use std::io::{Error, ErrorKind, Write, stdout, stderr}; +use std::process::Command; +use std::convert::{TryFrom}; + +fn render_nix_string(s: &OsString) -> OsString { + let mut rendered = Vec::new(); + + rendered.extend(b"\""); + + for b in s.as_os_str().as_bytes() { + match char::from(*b) { + '\"' => rendered.extend(b"\\\""), + '\\' => rendered.extend(b"\\\\"), + '$' => rendered.extend(b"\\$"), + _ => rendered.push(*b), + } + } + + rendered.extend(b"\""); + + OsString::from_vec(rendered) +} + +fn render_nix_list(arr: &[OsString]) -> OsString { + let mut rendered = Vec::new(); + + rendered.extend(b"[ "); + + for el in arr { + rendered.extend(render_nix_string(el).as_os_str().as_bytes()); + rendered.extend(b" "); + } + + rendered.extend(b"]"); + + OsString::from_vec(rendered) +} + +/// Slightly overkill helper macro which takes a `Map<String, Value>` obtained +/// from `Value::Object` and an output name (`stderr` or `stdout`) as an +/// identifier. If a value exists for the given output in the object it gets +/// written to the appropriate output. +macro_rules! handle_set_output { + ($map_name:ident, $output_name:ident) => { + match $map_name.get(stringify!($output_name)) { + Some(Value::String(s)) => + $output_name().write_all(s.as_bytes()), + Some(_) => Err( + Error::new( + ErrorKind::Other, + format!("Attribute {} must be a string!", stringify!($output_name)), + ) + ), + None => Ok(()), + } + } +} + +fn main() -> std::io::Result<()> { + let mut nix_args = Vec::new(); + + let mut args = std::env::args_os().into_iter(); + let mut in_args = true; + + let mut argv: Vec<OsString> = Vec::new(); + + // skip argv[0] + args.next(); + + loop { + let arg = match args.next() { + Some(a) => a, + None => break, + }; + + if !arg.to_str().map(|s| s.starts_with("-")).unwrap_or(false) { + in_args = false; + } + + if in_args { + match(arg.to_str()) { + Some("--arg") | Some("--argstr") => { + nix_args.push(arg); + nix_args.push(args.next().unwrap()); + nix_args.push(args.next().unwrap()); + Ok(()) + } + _ => Err(Error::new(ErrorKind::Other, "unknown argument")), + }? + } else { + argv.push(arg); + } + } + + if argv.len() < 1 { + Err(Error::new(ErrorKind::Other, "missing argv")) + } else { + let cd = std::env::current_dir()?.into_os_string(); + + nix_args.push(OsString::from("--arg")); + nix_args.push(OsString::from("currentDir")); + nix_args.push(cd); + + nix_args.push(OsString::from("--arg")); + nix_args.push(OsString::from("argv")); + nix_args.push(render_nix_list(&argv[..])); + + nix_args.push(OsString::from("--eval")); + nix_args.push(OsString::from("--strict")); + nix_args.push(OsString::from("--json")); + + nix_args.push(argv[0].clone()); + + let run = Command::new("nix-instantiate") + .args(nix_args) + .output()?; + + match serde_json::from_slice(&run.stdout[..]) { + Ok(Value::String(s)) => stdout().write_all(s.as_bytes()), + Ok(Value::Object(m)) => { + handle_set_output!(m, stdout)?; + handle_set_output!(m, stderr)?; + + match m.get("exit") { + Some(Value::Number(n)) => { + let code = n.as_i64().and_then(|v| i32::try_from(v).ok()); + + match code { + Some(i) => std::process::exit(i), + None => Err( + Error::new( + ErrorKind::Other, + "Attribute exit is not an i32" + ) + ), + } + }, + Some(_) => Err( + Error::new(ErrorKind::Other, "exit must be a number") + ), + None => Ok(()), + } + }, + Ok(_) => Err(Error::new(ErrorKind::Other, "output must be a string or an object")), + _ => { + stderr().write_all(&run.stderr[..]); + Err(Error::new(ErrorKind::Other, "internal nix error")) + }, + } + } +} diff --git a/nix/readTree/README.md b/nix/readTree/README.md new file mode 100644 index 000000000000..f8bbe2255e5e --- /dev/null +++ b/nix/readTree/README.md @@ -0,0 +1,92 @@ +readTree +======== + +This is a Nix program that builds up an attribute set tree for a large +repository based on the filesystem layout. + +It is in fact the tool that lays out the attribute set of this repository. + +As an example, consider a root (`.`) of a repository and a layout such as: + +``` +. +├── third_party +│ ├── default.nix +│ └── rustpkgs +│ ├── aho-corasick.nix +│ └── serde.nix +└── tools + ├── cheddar + │ └── default.nix + └── roquefort.nix +``` + +When `readTree` is called on that tree, it will construct an attribute set with +this shape: + +```nix +{ + tools = { + cheddar = ...; + roquefort = ...; + }; + + third_party = { + # the `default.nix` of this folder might have had arbitrary other + # attributes here, such as this: + favouriteColour = "orange"; + + rustpkgs = { + aho-corasick = ...; + serde = ...; + }; + }; +} +``` + +Every imported Nix file that yields an attribute set will have a `__readTree = +true;` attribute merged into it. + +## Traversal logic + +`readTree` will follow any subdirectories of a tree and import all Nix files, +with some exceptions: + +* A folder can declare that its children are off-limit by containing a + `.skip-subtree` file. Since the content of the file is not checked, it can be + useful to leave a note for a human in the file. +* If a folder contains a `default.nix` file, no *sibling* Nix files will be + imported - however children are traversed as normal. +* If a folder contains a `default.nix` it is loaded and, if it evaluates to a + set, *merged* with the children. If it evaluates to anything else the children + are *not traversed*. +* The `default.nix` of the top-level folder on which readTree is + called is **not** read to avoid infinite recursion (as, presumably, + this file is where readTree itself is called). + +Traversal is lazy, `readTree` will only build up the tree as requested. This +currently has the downside that directories with no importable files end up in +the tree as empty nodes (`{}`). + +## Import structure + +`readTree` is called with an argument set containing a few parameters: + +* `path`: Initial path at which to start the traversal. +* `args`: Arguments to pass to all imports. +* `filter`: (optional) A function to filter the argument set on each + import based on the location in the tree. This can be used to, for + example, implement a "visibility" system inside of a tree. +* `scopedArgs`: (optional) An argument set that is passed to all + imported files via `builtins.scopedImport`. This will forcefully + override the given values in the import scope, use with care! + +The package headers in this repository follow the form `{ pkgs, ... }:` where +`pkgs` is a fixed-point of the entire package tree (see the `default.nix` at the +root of the depot). + +In theory `readTree` can pass arguments of different shapes, but I have found +this to be a good solution for the most part. + +Note that `readTree` does not currently make functions overridable, though it is +feasible that it could do that in the future. diff --git a/nix/readTree/default.nix b/nix/readTree/default.nix new file mode 100644 index 000000000000..2ad8e40f6c28 --- /dev/null +++ b/nix/readTree/default.nix @@ -0,0 +1,230 @@ +# Copyright (c) 2019 Vincent Ambo +# Copyright (c) 2020-2021 The TVL Authors +# SPDX-License-Identifier: MIT +# +# Provides a function to automatically read a a filesystem structure +# into a Nix attribute set. +# +# Called with an attribute set taking the following arguments: +# +# path: Path to a directory from which to start reading the tree. +# +# args: Argument set to pass to each imported file. +# +# filter: Function to filter `args` based on the tree location. This should +# be a function of the form `args -> location -> args`, where the +# location is a list of strings representing the path components of +# the current readTree target. Optional. +{ ... }: + +let + inherit (builtins) + attrNames + concatMap + concatStringsSep + elem + elemAt + filter + hasAttr + head + isAttrs + listToAttrs + map + match + readDir + substring; + + argsWithPath = args: parts: + let meta.locatedAt = parts; + in meta // (if isAttrs args then args else args meta); + + readDirVisible = path: + let + children = readDir path; + isVisible = f: f == ".skip-subtree" || (substring 0 1 f) != "."; + names = filter isVisible (attrNames children); + in listToAttrs (map (name: { + inherit name; + value = children.${name}; + }) names); + + # Create a mark containing the location of this attribute and + # a list of all child attribute names added by readTree. + marker = parts: children: { + __readTree = parts; + __readTreeChildren = builtins.attrNames children; + }; + + # Import a file and enforce our calling convention + importFile = args: scopedArgs: path: parts: filter: + let + importedFile = if scopedArgs != {} + then builtins.scopedImport scopedArgs path + else import path; + pathType = builtins.typeOf importedFile; + in + if pathType != "lambda" + then builtins.throw "readTree: trying to import ${toString path}, but it’s a ${pathType}, you need to make it a function like { depot, pkgs, ... }" + else importedFile (filter parts (argsWithPath args parts)); + + nixFileName = file: + let res = match "(.*)\\.nix" file; + in if res == null then null else head res; + + readTree = { args, initPath, rootDir, parts, argsFilter, scopedArgs }: + let + dir = readDirVisible initPath; + joinChild = c: initPath + ("/" + c); + + self = if rootDir + then { __readTree = []; } + else importFile args scopedArgs initPath parts argsFilter; + + # Import subdirectories of the current one, unless the special + # `.skip-subtree` file exists which makes readTree ignore the + # children. + # + # This file can optionally contain information on why the tree + # should be ignored, but its content is not inspected by + # readTree + filterDir = f: dir."${f}" == "directory"; + children = if hasAttr ".skip-subtree" dir then [] else map (c: { + name = c; + value = readTree { + inherit argsFilter scopedArgs; + args = args; + initPath = (joinChild c); + rootDir = false; + parts = (parts ++ [ c ]); + }; + }) (filter filterDir (attrNames dir)); + + # Import Nix files + nixFiles = filter (f: f != null) (map nixFileName (attrNames dir)); + nixChildren = map (c: let + p = joinChild (c + ".nix"); + childParts = parts ++ [ c ]; + imported = importFile args scopedArgs p childParts argsFilter; + in { + name = c; + value = + if isAttrs imported + then imported // marker childParts {} + else imported; + }) nixFiles; + + nodeValue = if dir ? "default.nix" then self else {}; + + allChildren = listToAttrs ( + if dir ? "default.nix" + then children + else nixChildren ++ children + ); + + in + if isAttrs nodeValue + then nodeValue // allChildren // (marker parts allChildren) + else nodeValue; + + # Function can be used to find all readTree targets within an + # attribute set. + # + # This function will gather physical targets, that is targets which + # correspond directly to a location in the repository, as well as + # subtargets (specified in the meta.targets attribute of a node). + # + # This can be used to discover targets for inclusion in CI + # pipelines. + # + # Called with the arguments: + # + # eligible: Function to determine whether the given derivation + # should be included in the build. + gather = eligible: node: + if node ? __readTree then + # Include the node itself if it is eligible. + (if eligible node then [ node ] else []) + # Include eligible children of the node + ++ concatMap (gather eligible) (map (attr: node."${attr}") node.__readTreeChildren) + # Include specified sub-targets of the node + ++ filter eligible (map + (k: (node."${k}" or {}) // { + # Keep the same tree location, but explicitly mark this + # node as a subtarget. + __readTree = node.__readTree; + __readTreeChildren = []; + __subtarget = k; + }) + (node.meta.targets or [])) + else []; + + # Determine whether a given value is a derivation. + # Copied from nixpkgs/lib for cases where lib is not available yet. + isDerivation = x: isAttrs x && x ? type && x.type == "derivation"; +in { + inherit gather; + + __functor = _: + { path + , args + , filter ? (_parts: x: x) + , scopedArgs ? {} }: + readTree { + inherit args scopedArgs; + argsFilter = filter; + initPath = path; + rootDir = true; + parts = []; + }; + + # In addition to readTree itself, some functionality is exposed that + # is useful for users of readTree. + + # Create a readTree filter disallowing access to the specified + # top-level folder in the repository, except for specific exceptions + # specified by their (full) paths. + # + # Called with the arguments: + # + # folder: Name of the restricted top-level folder (e.g. 'experimental') + # + # exceptions: List of readTree parts (e.g. [ [ "services" "some-app" ] ]), + # which should be able to access the restricted folder. + # + # reason: Textual explanation for the restriction (included in errors) + restrictFolder = { folder, exceptions ? [], reason }: parts: args: + if (elemAt parts 0) == folder || elem parts exceptions + then args + else args // { + depot = args.depot // { + "${folder}" = throw '' + Access to targets under //${folder} is not permitted from + other repository paths. Specific exceptions are configured + at the top-level. + + ${reason} + At location: ${builtins.concatStringsSep "." parts} + ''; + }; + }; + + # This definition of fix is identical to <nixpkgs>.lib.fix, but is + # provided here for cases where readTree is used before nixpkgs can + # be imported. + # + # It is often required to create the args attribute set. + fix = f: let x = f x; in x; + + # Takes an attribute set and adds a meta.targets attribute to it + # which contains all direct children of the attribute set which are + # derivations. + # + # Type: attrs -> attrs + drvTargets = attrs: attrs // { + meta = { + targets = builtins.filter + (x: isDerivation attrs."${x}") + (builtins.attrNames attrs); + } // (attrs.meta or {}); + }; +} diff --git a/nix/readTree/tests/.skip-subtree b/nix/readTree/tests/.skip-subtree new file mode 100644 index 000000000000..46952a4ca635 --- /dev/null +++ b/nix/readTree/tests/.skip-subtree @@ -0,0 +1 @@ +These tests call their own readTree, so the toplevel one shouldn’t bother diff --git a/nix/readTree/tests/default.nix b/nix/readTree/tests/default.nix new file mode 100644 index 000000000000..3354a4fe5e75 --- /dev/null +++ b/nix/readTree/tests/default.nix @@ -0,0 +1,127 @@ +{ depot, lib, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + ; + + tree-ex = depot.nix.readTree { + path = ./test-example; + args = {}; + }; + + example = it "corresponds to the README example" [ + (assertEq "third_party attrset" + (lib.isAttrs tree-ex.third_party + && (! lib.isDerivation tree-ex.third_party)) + true) + (assertEq "third_party attrset other attribute" + tree-ex.third_party.favouriteColour + "orange") + (assertEq "rustpkgs attrset aho-corasick" + tree-ex.third_party.rustpkgs.aho-corasick + "aho-corasick") + (assertEq "rustpkgs attrset serde" + tree-ex.third_party.rustpkgs.serde + "serde") + (assertEq "tools cheddear" + "cheddar" + tree-ex.tools.cheddar) + (assertEq "tools roquefort" + tree-ex.tools.roquefort + "roquefort") + ]; + + tree-tl = depot.nix.readTree { + path = ./test-tree-traversal; + args = {}; + }; + + traversal-logic = it "corresponds to the traversal logic in the README" [ + (assertEq "skip subtree default.nix is read" + tree-tl.skip-subtree.but + "the default.nix is still read") + (assertEq "skip subtree a/default.nix is skipped" + (tree-tl.skip-subtree ? a) + false) + (assertEq "skip subtree b/c.nix is skipped" + (tree-tl.skip-subtree ? b) + false) + (assertEq "skip subtree a/default.nix would be read without .skip-subtree" + (tree-tl.no-skip-subtree.a) + "am I subtree yet?") + (assertEq "skip subtree b/c.nix would be read without .skip-subtree" + (tree-tl.no-skip-subtree.b.c) + "cool") + + (assertEq "default.nix attrset is merged with siblings" + tree-tl.default-nix.no + "siblings should be read") + (assertEq "default.nix means sibling isn’t read" + (tree-tl.default-nix ? sibling) + false) + (assertEq "default.nix means subdirs are still read and merged into default.nix" + (tree-tl.default-nix.subdir.a) + "but I’m picked up") + + (assertEq "default.nix can be not an attrset" + tree-tl.default-nix.no-merge + "I’m not merged with any children") + (assertEq "default.nix is not an attrset -> children are not merged" + (tree-tl.default-nix.no-merge ? subdir) + false) + + (assertEq "default.nix can contain a derivation" + (lib.isDerivation tree-tl.default-nix.can-be-drv) + true) + (assertEq "Even if default.nix is a derivation, children are traversed and merged" + tree-tl.default-nix.can-be-drv.subdir.a + "Picked up through the drv") + (assertEq "default.nix drv is not changed by readTree" + tree-tl.default-nix.can-be-drv + (import ./test-tree-traversal/default-nix/can-be-drv/default.nix {})) + ]; + + # these each call readTree themselves because the throws have to happen inside assertThrows + wrong = it "cannot read these files and will complain" [ + (assertThrows "this file is not a function" + (depot.nix.readTree { + path = ./test-wrong-not-a-function; + args = {}; + }).not-a-function) + # can’t test for that, assertThrows can’t catch this error + # (assertThrows "this file is a function but doesn’t have dots" + # (depot.nix.readTree {} ./test-wrong-no-dots).no-dots-in-function) + ]; + + read-markers = depot.nix.readTree { + path = ./test-marker; + args = {}; + }; + + assertMarkerByPath = path: + assertEq "${lib.concatStringsSep "." path} is marked correctly" + (lib.getAttrFromPath path read-markers).__readTree path; + + markers = it "marks nodes correctly" [ + (assertMarkerByPath [ "directory-marked" ]) + (assertMarkerByPath [ "directory-marked" "nested" ]) + (assertMarkerByPath [ "file-children" "one" ]) + (assertMarkerByPath [ "file-children" "two" ]) + (assertEq "nix file children are marked correctly" + read-markers.file-children.__readTreeChildren [ "one" "two" ]) + (assertEq "directory children are marked correctly" + read-markers.directory-marked.__readTreeChildren [ "nested" ]) + (assertEq "absence of children is marked" + read-markers.directory-marked.nested.__readTreeChildren [ ]) + ]; + +in runTestsuite "readTree" [ + example + traversal-logic + wrong + markers +] diff --git a/nix/readTree/tests/test-example/third_party/default.nix b/nix/readTree/tests/test-example/third_party/default.nix new file mode 100644 index 000000000000..27d87242189d --- /dev/null +++ b/nix/readTree/tests/test-example/third_party/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + favouriteColour = "orange"; +} diff --git a/nix/readTree/tests/test-example/third_party/rustpkgs/aho-corasick.nix b/nix/readTree/tests/test-example/third_party/rustpkgs/aho-corasick.nix new file mode 100644 index 000000000000..f964fa583c43 --- /dev/null +++ b/nix/readTree/tests/test-example/third_party/rustpkgs/aho-corasick.nix @@ -0,0 +1 @@ +{ ... }: "aho-corasick" diff --git a/nix/readTree/tests/test-example/third_party/rustpkgs/serde.nix b/nix/readTree/tests/test-example/third_party/rustpkgs/serde.nix new file mode 100644 index 000000000000..54b3c0503f12 --- /dev/null +++ b/nix/readTree/tests/test-example/third_party/rustpkgs/serde.nix @@ -0,0 +1 @@ +{ ... }: "serde" diff --git a/nix/readTree/tests/test-example/tools/cheddar/default.nix b/nix/readTree/tests/test-example/tools/cheddar/default.nix new file mode 100644 index 000000000000..a1919442d86d --- /dev/null +++ b/nix/readTree/tests/test-example/tools/cheddar/default.nix @@ -0,0 +1 @@ +{ ... }: "cheddar" diff --git a/nix/readTree/tests/test-example/tools/roquefort.nix b/nix/readTree/tests/test-example/tools/roquefort.nix new file mode 100644 index 000000000000..03c7492bb6b1 --- /dev/null +++ b/nix/readTree/tests/test-example/tools/roquefort.nix @@ -0,0 +1 @@ +{ ... }: "roquefort" diff --git a/nix/readTree/tests/test-marker/directory-marked/default.nix b/nix/readTree/tests/test-marker/directory-marked/default.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/directory-marked/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-marker/directory-marked/nested/default.nix b/nix/readTree/tests/test-marker/directory-marked/nested/default.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/directory-marked/nested/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-marker/file-children/one.nix b/nix/readTree/tests/test-marker/file-children/one.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/file-children/one.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-marker/file-children/two.nix b/nix/readTree/tests/test-marker/file-children/two.nix new file mode 100644 index 000000000000..a3f961128e79 --- /dev/null +++ b/nix/readTree/tests/test-marker/file-children/two.nix @@ -0,0 +1,3 @@ +{ ... }: + +{} diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/default.nix b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/default.nix new file mode 100644 index 000000000000..95d13d3c2750 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/default.nix @@ -0,0 +1,7 @@ +{ ... }: +derivation { + name = "im-a-drv"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" ''echo "" > $out'' ]; +} diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/subdir/a.nix b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/subdir/a.nix new file mode 100644 index 000000000000..2ee2d648f042 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/can-be-drv/subdir/a.nix @@ -0,0 +1,3 @@ +{ ... }: + +"Picked up through the drv" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/default.nix b/nix/readTree/tests/test-tree-traversal/default-nix/default.nix new file mode 100644 index 000000000000..6003b5398305 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + no = "siblings should be read"; +} diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/default.nix b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/default.nix new file mode 100644 index 000000000000..c469533fbe5b --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +"I’m not merged with any children" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/subdir/a.nix b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/subdir/a.nix new file mode 100644 index 000000000000..2008c2d2411f --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/no-merge/subdir/a.nix @@ -0,0 +1 @@ +"not accessible since parent default.nix is not an attrset" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/sibling.nix b/nix/readTree/tests/test-tree-traversal/default-nix/sibling.nix new file mode 100644 index 000000000000..8c57f2c16137 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/sibling.nix @@ -0,0 +1 @@ +"I’m left alone" diff --git a/nix/readTree/tests/test-tree-traversal/default-nix/subdir/a.nix b/nix/readTree/tests/test-tree-traversal/default-nix/subdir/a.nix new file mode 100644 index 000000000000..cf0ac2c8f3cf --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/default-nix/subdir/a.nix @@ -0,0 +1,3 @@ +{ ... }: + +"but I’m picked up" diff --git a/nix/readTree/tests/test-tree-traversal/no-skip-subtree/a/default.nix b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/a/default.nix new file mode 100644 index 000000000000..a586ce534ce2 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/a/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +"am I subtree yet?" diff --git a/nix/readTree/tests/test-tree-traversal/no-skip-subtree/b/c.nix b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/b/c.nix new file mode 100644 index 000000000000..06216c417be9 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/b/c.nix @@ -0,0 +1,3 @@ +{ ... }: + +"cool" diff --git a/nix/readTree/tests/test-tree-traversal/no-skip-subtree/default.nix b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/default.nix new file mode 100644 index 000000000000..3d9f241cddb5 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/no-skip-subtree/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + but = "the default.nix is still read"; +} diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/.skip-subtree b/nix/readTree/tests/test-tree-traversal/skip-subtree/.skip-subtree new file mode 100644 index 000000000000..87271ba5e101 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/.skip-subtree @@ -0,0 +1 @@ +this file makes subdirs be skipped, I hope diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/a/default.nix b/nix/readTree/tests/test-tree-traversal/skip-subtree/a/default.nix new file mode 100644 index 000000000000..a586ce534ce2 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/a/default.nix @@ -0,0 +1,3 @@ +{ ... }: + +"am I subtree yet?" diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/b/c.nix b/nix/readTree/tests/test-tree-traversal/skip-subtree/b/c.nix new file mode 100644 index 000000000000..06216c417be9 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/b/c.nix @@ -0,0 +1,3 @@ +{ ... }: + +"cool" diff --git a/nix/readTree/tests/test-tree-traversal/skip-subtree/default.nix b/nix/readTree/tests/test-tree-traversal/skip-subtree/default.nix new file mode 100644 index 000000000000..3d9f241cddb5 --- /dev/null +++ b/nix/readTree/tests/test-tree-traversal/skip-subtree/default.nix @@ -0,0 +1,5 @@ +{ ... }: + +{ + but = "the default.nix is still read"; +} diff --git a/nix/readTree/tests/test-wrong-no-dots/no-dots-in-function.nix b/nix/readTree/tests/test-wrong-no-dots/no-dots-in-function.nix new file mode 100644 index 000000000000..4681253af831 --- /dev/null +++ b/nix/readTree/tests/test-wrong-no-dots/no-dots-in-function.nix @@ -0,0 +1,3 @@ +{}: + +"This is a function, but readTree wants to pass a bunch of arguments, and not having dots means we depend on exactly which arguments." diff --git a/nix/readTree/tests/test-wrong-not-a-function/not-a-function.nix b/nix/readTree/tests/test-wrong-not-a-function/not-a-function.nix new file mode 100644 index 000000000000..f46ee2a35565 --- /dev/null +++ b/nix/readTree/tests/test-wrong-not-a-function/not-a-function.nix @@ -0,0 +1 @@ +"This file needs to be a function, otherwise readTree doesn’t like it!" diff --git a/nix/renderMarkdown/default.nix b/nix/renderMarkdown/default.nix new file mode 100644 index 000000000000..3e5a59954b20 --- /dev/null +++ b/nix/renderMarkdown/default.nix @@ -0,0 +1,8 @@ +# Render a Markdown file to HTML. +{ depot, pkgs, ... }: + +with depot.nix.yants; + +defun [ path drv ] (file: pkgs.runCommandNoCC "${file}.rendered.html" {} '' + cat ${file} | ${depot.tools.cheddar}/bin/cheddar --about-filter ${file} > $out +'') diff --git a/nix/runExecline/default.nix b/nix/runExecline/default.nix new file mode 100644 index 000000000000..fd92203d0146 --- /dev/null +++ b/nix/runExecline/default.nix @@ -0,0 +1,30 @@ +{ depot, pkgs, lib, ... }: +let + runExecline = import ./runExecline.nix { + inherit (pkgs) stdenv; + inherit (depot.nix) escapeExecline getBins; + inherit pkgs lib; + }; + + runExeclineLocal = name: args: execline: + runExecline name + (args // { + derivationArgs = args.derivationArgs or {} // { + preferLocalBuild = true; + allowSubstitutes = false; + }; + }) + execline; + + tests = import ./tests.nix { + inherit runExecline runExeclineLocal; + inherit (depot.nix) getBins writeScript; + inherit (pkgs) stdenv coreutils; + inherit pkgs; + }; + +in { + __functor = _: runExecline; + local = runExeclineLocal; + inherit tests; +} diff --git a/nix/runExecline/runExecline.nix b/nix/runExecline/runExecline.nix new file mode 100644 index 000000000000..0e45080735bb --- /dev/null +++ b/nix/runExecline/runExecline.nix @@ -0,0 +1,122 @@ +{ pkgs, stdenv, lib, getBins, escapeExecline }: + +# runExecline is a primitive building block +# for writing non-kitchen sink builders. +# +# It’s conceptually similar to `runCommand`, +# but instead of concatenating bash scripts left +# and right, it actually *uses* the features of +# `derivation`, passing things to `args` +# and making it possible to overwrite the `builder` +# in a sensible manner. +# +# Additionally, it provides a way to pass a nix string +# to `stdin` of the build script. +# +# Similar to //nix/writeExecline, the passed script is +# not a string, but a nested list of nix lists +# representing execline blocks. Escaping is +# done by the implementation, the user can just use +# normal nix strings. +# +# Example: +# +# runExecline "my-drv" { stdin = "hi!"; } [ +# "importas" "out" "out" +# # this pipes stdout of s6-cat to $out +# # and s6-cat redirects from stdin to stdout +# "redirfd" "-w" "1" "$out" bins.s6-cat +# ] +# +# which creates a derivation with "hi!" in $out. +# +# See ./tests.nix for more examples. + + +let + bins = getBins pkgs.execline [ + "execlineb" + { use = "if"; as = "execlineIf"; } + "redirfd" + "importas" + "exec" + ] + // getBins pkgs.s6-portable-utils [ + "s6-cat" + "s6-grep" + "s6-touch" + "s6-test" + "s6-chmod" + ]; + +in + +# TODO: move name into the attrset +name: +{ +# a string to pass as stdin to the execline script +stdin ? "" +# a program wrapping the acutal execline invocation; +# should be in Bernstein-chaining style +, builderWrapper ? bins.exec +# additional arguments to pass to the derivation +, derivationArgs ? {} +}: +# the execline script as a nested list of string, +# representing the blocks; +# see docs of `escapeExecline`. +execline: + +# those arguments can’t be overwritten +assert !derivationArgs ? system; +assert !derivationArgs ? name; +assert !derivationArgs ? builder; +assert !derivationArgs ? args; + +derivation (derivationArgs // { + # TODO(Profpatsch): what about cross? + inherit (stdenv) system; + inherit name; + + # okay, `builtins.toFile` does not accept strings + # that reference drv outputs. This means we need + # to pass the script and stdin as envvar; + # this might clash with another passed envar, + # so we give it a long & unique name + _runExeclineScript = + let + in escapeExecline execline; + _runExeclineStdin = stdin; + passAsFile = [ + "_runExeclineScript" + "_runExeclineStdin" + ] ++ derivationArgs.passAsFile or []; + + # the default, exec acts as identity executable + builder = builderWrapper; + + args = [ + bins.importas # import script file as $script + "-ui" # drop the envvar afterwards + "script" # substitution name + "_runExeclineScriptPath" # passed script file + + bins.importas # do the same for $stdin + "-ui" + "stdin" + "_runExeclineStdinPath" + + bins.redirfd # now we + "-r" # read the file + "0" # into the stdin of execlineb + "$stdin" # that was given via stdin + + bins.execlineb # the actual invocation + # TODO(Profpatsch): depending on the use-case, -S0 might not be enough + # in all use-cases, then a wrapper for execlineb arguments + # should be added (-P, -S, -s). + "-S0" # set $@ inside the execline script + "-W" # die on syntax error + "$script" # substituted by importas + ]; +}) diff --git a/nix/runExecline/tests.nix b/nix/runExecline/tests.nix new file mode 100644 index 000000000000..d2f5a1780c16 --- /dev/null +++ b/nix/runExecline/tests.nix @@ -0,0 +1,80 @@ +{ stdenv, pkgs, runExecline, runExeclineLocal, getBins, writeScript +# https://www.mail-archive.com/skaware@list.skarnet.org/msg01256.html +, coreutils }: + +let + + bins = getBins coreutils [ "mv" ] + // getBins pkgs.execline [ + "execlineb" + { use = "if"; as = "execlineIf"; } + "redirfd" + "importas" + ] + // getBins pkgs.s6-portable-utils [ + "s6-chmod" + "s6-grep" + "s6-touch" + "s6-cat" + "s6-test" + ]; + + # execline block of depth 1 + block = args: builtins.map (arg: " ${arg}") args ++ [ "" ]; + + # derivation that tests whether a given line exists + # in the given file. Does not use runExecline, because + # that should be tested after all. + fileHasLine = line: file: derivation { + name = "run-execline-test-file-${file.name}-has-line"; + inherit (stdenv) system; + builder = bins.execlineIf; + args = + (block [ + bins.redirfd "-r" "0" file # read file to stdin + bins.s6-grep "-F" "-q" line # and grep for the line + ]) + ++ [ + # if the block succeeded, touch $out + bins.importas "-ui" "out" "out" + bins.s6-touch "$out" + ]; + preferLocalBuild = true; + allowSubstitutes = false; + }; + + # basic test that touches out + basic = runExeclineLocal "run-execline-test-basic" { + } [ + "importas" "-ui" "out" "out" + "${bins.s6-touch}" "$out" + ]; + + # whether the stdin argument works as intended + stdin = fileHasLine "foo" (runExeclineLocal "run-execline-test-stdin" { + stdin = "foo\nbar\nfoo"; + } [ + "importas" "-ui" "out" "out" + # this pipes stdout of s6-cat to $out + # and s6-cat redirects from stdin to stdout + "redirfd" "-w" "1" "$out" bins.s6-cat + ]); + + + wrapWithVar = runExeclineLocal "run-execline-test-wrap-with-var" { + builderWrapper = writeScript "var-wrapper" '' + #!${bins.execlineb} -S0 + export myvar myvalue $@ + ''; + } [ + "importas" "-ui" "v" "myvar" + "if" [ bins.s6-test "myvalue" "=" "$v" ] + "importas" "out" "out" + bins.s6-touch "$out" + ]; + +in [ + basic + stdin + wrapWithVar +] diff --git a/nix/runTestsuite/default.nix b/nix/runTestsuite/default.nix new file mode 100644 index 000000000000..9eb507099678 --- /dev/null +++ b/nix/runTestsuite/default.nix @@ -0,0 +1,179 @@ +{ lib, pkgs, depot, ... }: + +# Run a nix testsuite. +# +# The tests are simple assertions on the nix level, +# and can use derivation outputs if IfD is enabled. +# +# You build a testsuite by bundling assertions into +# “it”s and then bundling the “it”s into a testsuite. +# +# Running the testsuite will abort evaluation if +# any assertion fails. +# +# Example: +# +# runTestsuite "myFancyTestsuite" [ +# (it "does an assertion" [ +# (assertEq "42 is equal to 42" "42" "42") +# (assertEq "also 23" 23 23) +# ]) +# (it "frmbls the brlbr" [ +# (assertEq true false) +# ]) +# ] +# +# will fail the second it group because true is not false. + +let + inherit (depot.nix.yants) + sum + struct + string + any + defun + list + drv + bool + ; + + bins = depot.nix.getBins pkgs.coreutils [ "printf" ] + // depot.nix.getBins pkgs.s6-portable-utils [ "s6-touch" "s6-false" "s6-cat" ]; + + # Returns true if the given expression throws when `deepSeq`-ed + throws = expr: + !(builtins.tryEval (builtins.deepSeq expr {})).success; + + # rewrite the builtins.partition result + # to use `ok` and `err` instead of `right` and `wrong`. + partitionTests = pred: xs: + let res = builtins.partition pred xs; + in { + ok = res.right; + err = res.wrong; + }; + + AssertErrorContext = + sum "AssertErrorContext" { + not-equal = struct "not-equal" { + left = any; + right = any; + }; + should-throw = struct "should-throw" { + expr = any; + }; + unexpected-throw = struct "unexpected-throw" { }; + }; + + # The result of an assert, + # either it’s true (yep) or false (nope). + # If it's nope we return an additional context + # attribute which gives details on the failure + # depending on the type of assert performed. + AssertResult = + sum "AssertResult" { + yep = struct "yep" { + test = string; + }; + nope = struct "nope" { + test = string; + context = AssertErrorContext; + }; + }; + + # Result of an it. An it is a bunch of asserts + # bundled up with a good description of what is tested. + ItResult = + struct "ItResult" { + it-desc = string; + asserts = list AssertResult; + }; + + # If the given boolean is true return a positive AssertResult. + # If the given boolean is false return a negative AssertResult + # with the provided AssertErrorContext describing the failure. + # + # This function is intended as a generic assert to implement + # more assert types and is not exposed to the user. + assertBoolContext = defun [ AssertErrorContext string bool AssertResult ] + (context: desc: res: + if res + then { yep = { test = desc; }; } + else { nope = { + test = desc; + inherit context; + }; + }); + + # assert that left and right values are equal + assertEq = defun [ string any any AssertResult ] + (desc: left: right: + let + context = { not-equal = { inherit left right; }; }; + in + assertBoolContext context desc (left == right)); + + # assert that the expression throws when `deepSeq`-ed + assertThrows = defun [ string any AssertResult ] + (desc: expr: + let + context = { should-throw = { inherit expr; }; }; + in + assertBoolContext context desc (throws expr)); + + # assert that the expression does not throw when `deepSeq`-ed + assertDoesNotThrow = defun [ string any AssertResult ] + (desc: expr: + assertBoolContext { unexpected-throw = { }; } desc (!(throws expr))); + + # Annotate a bunch of asserts with a descriptive name + it = desc: asserts: { + it-desc = desc; + inherit asserts; + }; + + # Run a bunch of its and check whether all asserts are yep. + # If not, abort evaluation with `throw` + # and print the result of the test suite. + # + # Takes a test suite name as first argument. + runTestsuite = defun [ string (list ItResult) drv ] + (name: itResults: + let + goodAss = ass: AssertResult.match ass { + yep = _: true; + nope = _: false; + }; + res = partitionTests (it: + (partitionTests goodAss it.asserts).err == [] + ) itResults; + prettyRes = lib.generators.toPretty {} res; + in + if res.err == [] + then depot.nix.runExecline.local "testsuite-${name}-successful" {} [ + "importas" "out" "out" + # force derivation to rebuild if test case list changes + "ifelse" [ bins.s6-false ] [ + bins.printf "" (builtins.hashString "sha512" prettyRes) + ] + "if" [ bins.printf "%s\n" "testsuite ${name} successful!" ] + bins.s6-touch "$out" + ] + else depot.nix.runExecline.local "testsuite-${name}-failed" { + stdin = prettyRes + "\n"; + } [ + "importas" "out" "out" + "if" [ bins.printf "%s\n" "testsuite ${name} failed!" ] + "if" [ bins.s6-cat ] + "exit" "1" + ]); + +in { + inherit + assertEq + assertThrows + assertDoesNotThrow + it + runTestsuite + ; +} diff --git a/nix/sparseTree/OWNERS b/nix/sparseTree/OWNERS new file mode 100644 index 000000000000..fdf6d7204051 --- /dev/null +++ b/nix/sparseTree/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni \ No newline at end of file diff --git a/nix/sparseTree/default.nix b/nix/sparseTree/default.nix new file mode 100644 index 000000000000..5184f33d5c46 --- /dev/null +++ b/nix/sparseTree/default.nix @@ -0,0 +1,70 @@ +# Build a “sparse” version of a given directory, only including contained files +# and directories if they are listed in a supplied list: +# +# # A very minimal depot +# sparseTree ./depot [ +# ./default.nix +# ./depot/nix/readTree/default.nix +# ./third_party/nixpkgs +# ./third_party/overlays +# ] +{ pkgs, lib, ... }: + +# root path to use as a reference point +root: +# list of paths below `root` that should be +# included in the resulting directory +# +# If path, need to refer to the actual file / directory to be included. +# If a string, it is treated as a string relative to the root. +paths: + +let + rootLength = builtins.stringLength (toString root); + + # Count slashes in a path. + # + # Type: path -> int + depth = path: lib.pipe path [ + toString + (builtins.split "/") + (builtins.filter builtins.isList) + builtins.length + ]; + + # (Parent) directories will be created from deepest to shallowest + # which should mean no conflicts are caused unless both a child + # and its parent directory are in the list of paths. + # TODO(sterni): improve error messages in such cases + fromDeepest = lib.sort (a: b: depth a < depth b) paths; + + # Create a set which contains the source path to copy / symlink and + # it's destination, so the path below the destination root including + # a leading slash. Additionally some sanity checking is done. + makeSymlink = path: + let + withLeading = p: if builtins.substring 0 1 p == "/" then p else "/" + p; + fullPath = + /**/ if builtins.isPath path then path + else if builtins.isString path then (root + withLeading path) + else builtins.throw "Unsupported path type ${builtins.typeOf path}"; + strPath = toString fullPath; + contextPath = "${fullPath}"; + belowRoot = builtins.substring rootLength (-1) strPath; + prefix = builtins.substring 0 rootLength strPath; + in assert toString root == prefix; { + src = contextPath; + dst = belowRoot; + }; + + symlinks = builtins.map makeSymlink fromDeepest; +in + +# TODO(sterni): teach readTree to also read symlinked directories, +# so we ln -sT instead of cp -aT. +pkgs.runCommandNoCC "sparse-${builtins.baseNameOf root}" {} ( + lib.concatMapStrings ({ src, dst }: '' + mkdir -p "$(dirname "$out${dst}")" + cp -aT --reflink=auto "${src}" "$out${dst}" + '') symlinks +) diff --git a/nix/tag/default.nix b/nix/tag/default.nix new file mode 100644 index 000000000000..9c55e6263b39 --- /dev/null +++ b/nix/tag/default.nix @@ -0,0 +1,162 @@ +{ depot, lib, ... }: +let + # Takes a tag, checks whether it is an attrset with one element, + # if so sets `isTag` to `true` and sets the name and value. + # If not, sets `isTag` to `false` and sets `errmsg`. + verifyTag = tag: + let cases = builtins.attrNames tag; + len = builtins.length cases; + in + if builtins.length cases == 1 + then let name = builtins.head cases; in { + isTag = true; + name = name; + val = tag.${name}; + errmsg = null; + } + else { + isTag = false; + errmsg = + ( "match: an instance of a sum is an attrset " + + "with exactly one element, yours had ${toString len}" + + ", namely: ${lib.generators.toPretty {} cases}" ); + name = null; + val = null; + }; + + # Returns the tag name of a given tag attribute set. + # Throws if the tag is invalid. + # + # Type: tag -> string + tagName = tag: (assertIsTag tag).name; + + # Returns the tagged value of a given tag attribute set. + # Throws if the tag is invalid. + # + # Type: tag -> any + tagValue = tag: (assertIsTag tag).val; + + # like `verifyTag`, but throws the error message if it is not a tag. + assertIsTag = tag: + let res = verifyTag tag; in + assert res.isTag || throw res.errmsg; + { inherit (res) name val; }; + + + # Discriminator for values. + # Goes through a list of tagged predicates `{ <tag> = <pred>; }` + # and returns the value inside the tag + # for which the first predicate applies, `{ <tag> = v; }`. + # They can then later be matched on with `match`. + # + # `defTag` is the tag that is assigned if there is no match. + # + # Examples: + # discrDef "smol" [ + # { biggerFive = i: i > 5; } + # { negative = i: i < 0; } + # ] (-100) + # => { negative = -100; } + # discrDef "smol" [ + # { biggerFive = i: i > 5; } + # { negative = i: i < 0; } + # ] 1 + # => { smol = 1; } + discrDef = defTag: fs: v: + let res = lib.findFirst + (t: t.val v) + null + (map assertIsTag fs); + in + if res == null + then { ${defTag} = v; } + else { ${res.name} = v; }; + + # Like `discrDef`, but fail if there is no match. + discr = fs: v: + let res = discrDef null fs v; in + assert lib.assertMsg (res != null) + "tag.discr: No predicate found that matches ${lib.generators.toPretty {} v}"; + res; + + # The canonical pattern matching primitive. + # A sum value is an attribute set with one element, + # whose key is the name of the variant and + # whose value is the content of the variant. + # `matcher` is an attribute set which enumerates + # all possible variants as keys and provides a function + # which handles each variant’s content. + # You should make an effort to return values of the same + # type in your matcher, or new sums. + # + # Example: + # let + # success = { res = 42; }; + # failure = { err = "no answer"; }; + # matcher = { + # res = i: i + 1; + # err = _: 0; + # }; + # in + # match success matcher == 43 + # && match failure matcher == 0; + # + match = sum: matcher: + let cases = builtins.attrNames sum; + in assert + let len = builtins.length cases; in + lib.assertMsg (len == 1) + ( "match: an instance of a sum is an attrset " + + "with exactly one element, yours had ${toString len}" + + ", namely: ${lib.generators.toPretty {} cases}" ); + let case = builtins.head cases; + in assert + lib.assertMsg (matcher ? ${case}) + ( "match: \"${case}\" is not a valid case of this sum, " + + "the matcher accepts: ${lib.generators.toPretty {} + (builtins.attrNames matcher)}" ); + matcher.${case} sum.${case}; + + # A `match` with the arguments flipped. + # “Lam” stands for “lambda”, because it can be used like the + # `\case` LambdaCase statement in Haskell, to create a curried + # “matcher” function ready to take a value. + # + # Example: + # lib.pipe { foo = 42; } [ + # (matchLam { + # foo = i: if i < 23 then { small = i; } else { big = i; }; + # bar = _: { small = 5; }; + # }) + # (matchLam { + # small = i: "yay it was small"; + # big = i: "whoo it was big!"; + # }) + # ] + # => "whoo it was big!"; + matchLam = matcher: sum: match sum matcher; + + tests = import ./tests.nix { + inherit + depot + lib + verifyTag + discr + discrDef + match + matchLam + ; + }; + +in { + inherit + verifyTag + tagName + tagValue + discr + discrDef + match + matchLam + tests + ; +} diff --git a/nix/tag/tests.nix b/nix/tag/tests.nix new file mode 100644 index 000000000000..8c9c73807472 --- /dev/null +++ b/nix/tag/tests.nix @@ -0,0 +1,88 @@ +{ depot, lib, verifyTag, discr, discrDef, match, matchLam }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + assertEq + it + ; + + isTag-test = it "checks whether something is a tag" [ + (assertEq "is Tag" + (verifyTag { foo = "bar"; }) + { + isTag = true; + name = "foo"; + val = "bar"; + errmsg = null; + }) + (assertEq "is not Tag" + (removeAttrs (verifyTag { foo = "bar"; baz = 42; }) ["errmsg"]) + { + isTag = false; + name = null; + val = null; + }) + ]; + + discr-test = it "can discr things" [ + (assertEq "id" + (discr [ + { a = lib.const true; } + ] "x") + { a = "x"; }) + (assertEq "bools here, ints there" + (discr [ + { bool = lib.isBool; } + { int = lib.isInt; } + ] 25) + { int = 25; }) + (assertEq "bools here, ints there 2" + (discr [ + { bool = lib.isBool; } + { int = lib.isInt; } + ] true) + { bool = true; }) + (assertEq "fallback to default" + (discrDef "def" [ + { bool = lib.isBool; } + { int = lib.isInt; } + ] "foo") + { def = "foo"; }) + ]; + + match-test = it "can match things" [ + (assertEq "match example" + (let + success = { res = 42; }; + failure = { err = "no answer"; }; + matcher = { + res = i: i + 1; + err = _: 0; + }; + in { + one = match success matcher; + two = match failure matcher; + }) + { one = 43; + two = 0; }) + (assertEq "matchLam & pipe" + (lib.pipe { foo = 42; } [ + (matchLam { + foo = i: if i < 23 then { small = i; } else { big = i; }; + bar = _: { small = 5; }; + }) + (matchLam { + small = i: "yay it was small"; + big = i: "whoo it was big!"; + }) + ]) + "whoo it was big!") + ]; + +in + runTestsuite "tag" [ + isTag-test + discr-test + match-test + ] diff --git a/nix/tailscale/default.nix b/nix/tailscale/default.nix new file mode 100644 index 000000000000..8d6a0f661b92 --- /dev/null +++ b/nix/tailscale/default.nix @@ -0,0 +1,30 @@ +# This file defines a Nix helper function to create Tailscale ACL files. +# +# https://tailscale.com/kb/1018/install-acls + +{ depot, pkgs, ... }: + +with depot.nix.yants; + +let + inherit (builtins) toFile toJSON; + + acl = struct "acl" { + Action = enum [ "accept" "reject" ]; + Users = list string; + Ports = list string; + }; + + acls = list entry; + + aclConfig = struct "aclConfig" { + # Static group mappings from group names to lists of users + Groups = option (attrs (list string)); + + # Hostname aliases to use in place of IPs + Hosts = option (attrs string); + + # Actual ACL entries + ACLs = list acl; + }; +in config: pkgs.writeText "tailscale-acl.json" (toJSON (aclConfig config)) diff --git a/nix/utils/OWNERS b/nix/utils/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/nix/utils/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/nix/utils/default.nix b/nix/utils/default.nix new file mode 100644 index 000000000000..f65d4712a03d --- /dev/null +++ b/nix/utils/default.nix @@ -0,0 +1,180 @@ +{ depot, lib, ... }: + +let + /* Get the basename of a store path without + the leading hash. + + Type: (path | drv | string) -> string + + Example: + storePathName ./foo.c + => "foo.c" + + storePathName (writeText "foo.c" "int main() { return 0; }") + => "foo.c" + + storePathName "${hello}/bin/hello" + => "hello" + */ + storePathName = p: + if lib.isDerivation p + then p.name + else if builtins.isPath p + then builtins.baseNameOf p + else if builtins.isString p + then + let + # strip leading storeDir and trailing slashes + noStoreDir = lib.removeSuffix "/" + (lib.removePrefix "${builtins.storeDir}/" p); + # a basename of a child of a store path isn't really + # referring to a store path, so removing the string + # context is safe (e. g. "hello" for "${hello}/bin/hello"). + basename = builtins.unsafeDiscardStringContext + (builtins.baseNameOf p); + in + # If p is a direct child of storeDir, we need to remove + # the leading hash as well to make sure that: + # `storePathName drv == storePathName (toString drv)`. + if noStoreDir == basename + then builtins.substring 33 (-1) basename + else basename + else builtins.throw "Don't know how to get (base)name of " + + lib.generators.toPretty {} p; + + /* Query the type of a path exposing the same information as would be by + `builtins.readDir`, but for a single, specific target path. + + The information is returned as a tagged value, i. e. an attribute set with + exactly one attribute where the type of the path is encoded in the name + of the single attribute. The allowed tags and values are as follows: + + * `regular`: is a regular file, always `true` if returned + * `directory`: is a directory, always `true` if returned + * `missing`: path does not exist, always `true` if returned + * `symlink`: path is a symlink, value is a string describing the type + of its realpath which may be either: + + * `"directory"`: realpath of the symlink is a directory + * `"regular-or-missing`": realpath of the symlink is either a regular + file or does not exist. Due to limitations of the Nix expression + language, we can't tell which. + + Type: path(-like) -> tag + + `tag` refers to the attribute set format of `//nix/tag`. + + Example: + pathType ./foo.c + => { regular = true; } + + pathType /home/lukas + => { directory = true; } + + pathType ./result + => { symlink = "directory"; } + + pathType ./link-to-file + => { symlink = "regular-or-missing"; } + + pathType /does/not/exist + => { missing = true; } + + # Check if a path exists + !(pathType /file ? missing) + + # Check if a path is a directory or a symlink to a directory + # A handy shorthand for this is provided as `realPathIsDirectory`. + pathType /path ? directory || (pathType /path).symlink or null == "directory" + + # Match on the result using //nix/tag + nix.tag.match (nix.utils.pathType ./result) { + symlink = v: "symlink to ${v}"; + directory = _: "directory"; + regular = _: "regular"; + missing = _: "path does not exist"; + } + => "symlink to directory" + + # Query path type + nix.tag.tagName (pathType /path) + */ + pathType = path: + let + # baseNameOf is very annoyed if we proceed with string context. + # We need to call toString to prevent unsafeDiscardStringContext + # from importing a path into store which messes with base- and + # dirname of course. + path'= builtins.unsafeDiscardStringContext (toString path); + # To read the containing directory we absolutely need + # to keep the string context, otherwise a derivation + # would not be realized before our check (at eval time) + containingDir = builtins.readDir (builtins.dirOf path); + # Construct tag to use for the value + thisPathType = containingDir.${builtins.baseNameOf path'} or "missing"; + # Trick to check if the symlink target exists and is a directory: + # if we append a "/." to the string version of the path, Nix won't + # canocalize it (which would strip any "/." in the path), so if + # path' + "/." exists, we know that the symlink points to an existing + # directory. If not, either the target doesn't exist or is a regular file. + # TODO(sterni): is there a way to check reliably if the symlink target exists? + isSymlinkDir = builtins.pathExists (path' + "/."); + in { + ${thisPathType} = + /**/ if thisPathType != "symlink" then true + else if isSymlinkDir then "directory" + else "regular-or-missing"; + }; + + pathType' = path: + let + p = pathType path; + in + if p ? missing + then builtins.throw "${lib.generators.toPretty {} path} does not exist" + else p; + + /* Check whether the given path is a directory. + Throws if the path in question doesn't exist. + + Type: path(-like) -> bool + */ + isDirectory = path: pathType' path ? directory; + + /* Checks whether the given path is a directory or + a symlink to a directory. Throws if the path in + question doesn't exist. + + Warning: Does not throw if the target file or + directory doesn't exist, but the symlink does. + + Type: path(-like) -> bool + */ + realPathIsDirectory = path: let + pt = pathType' path; + in pt ? directory || pt.symlink or null == "directory"; + + /* Check whether the given path is a regular file. + Throws if the path in question doesn't exist. + + Type: path(-like) -> bool + */ + isRegularFile = path: pathType' path ? regular; + + /* Check whether the given path is a symbolic link. + Throws if the path in question doesn't exist. + + Type: path(-like) -> bool + */ + isSymlink = path: pathType' path ? symlink; + +in { + inherit + storePathName + pathType + isDirectory + realPathIsDirectory + isRegularFile + isSymlink + ; +} diff --git a/nix/utils/tests/.skip-subtree b/nix/utils/tests/.skip-subtree new file mode 100644 index 000000000000..1efb1b9476ba --- /dev/null +++ b/nix/utils/tests/.skip-subtree @@ -0,0 +1 @@ +subdirectories are just test cases diff --git a/nix/utils/tests/default.nix b/nix/utils/tests/default.nix new file mode 100644 index 000000000000..1ba68486077e --- /dev/null +++ b/nix/utils/tests/default.nix @@ -0,0 +1,104 @@ +{ depot, lib, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + assertDoesNotThrow + ; + + inherit (depot.nix.utils) + isDirectory + realPathIsDirectory + isRegularFile + isSymlink + pathType + storePathName + ; + + assertUtilsPred = msg: act: exp: [ + (assertDoesNotThrow "${msg} does not throw" act) + (assertEq msg (builtins.tryEval act).value exp) + ]; + + pathPredicates = it "judges paths correctly" (lib.flatten [ + # isDirectory + (assertUtilsPred "directory isDirectory" + (isDirectory ./directory) true) + (assertUtilsPred "symlink not isDirectory" + (isDirectory ./symlink-directory) false) + (assertUtilsPred "file not isDirectory" + (isDirectory ./directory/file) false) + # realPathIsDirectory + (assertUtilsPred "directory realPathIsDirectory" + (realPathIsDirectory ./directory) true) + (assertUtilsPred "symlink to directory realPathIsDirectory" + (realPathIsDirectory ./symlink-directory) true) + (assertUtilsPred "realPathIsDirectory resolves chained symlinks" + (realPathIsDirectory ./symlink-symlink-directory) true) + # isRegularFile + (assertUtilsPred "file isRegularFile" + (isRegularFile ./directory/file) true) + (assertUtilsPred "symlink not isRegularFile" + (isRegularFile ./symlink-file) false) + (assertUtilsPred "directory not isRegularFile" + (isRegularFile ./directory) false) + # isSymlink + (assertUtilsPred "symlink to file isSymlink" + (isSymlink ./symlink-file) true) + (assertUtilsPred "symlink to directory isSymlink" + (isSymlink ./symlink-directory) true) + (assertUtilsPred "symlink to symlink isSymlink" + (isSymlink ./symlink-symlink-file) true) + (assertUtilsPred "symlink to missing file isSymlink" + (isSymlink ./missing) true) + (assertUtilsPred "directory not isSymlink" + (isSymlink ./directory) false) + (assertUtilsPred "file not isSymlink" + (isSymlink ./directory/file) false) + # missing files throw + (assertThrows "isDirectory throws on missing file" + (isDirectory ./does-not-exist)) + (assertThrows "realPathIsDirectory throws on missing file" + (realPathIsDirectory ./does-not-exist)) + (assertThrows "isRegularFile throws on missing file" + (isRegularFile ./does-not-exist)) + (assertThrows "isSymlink throws on missing file" + (isSymlink ./does-not-exist)) + ]); + + symlinkPathTypeTests = it "correctly judges symlinks" [ + (assertEq "symlinks to directories are detected correcty" + ((pathType ./symlink-directory).symlink or null) "directory") + (assertEq "symlinks to symlinks to directories are detected correctly" + ((pathType ./symlink-symlink-directory).symlink or null) "directory") + (assertEq "symlinks to files are detected-ish" + ((pathType ./symlink-file).symlink or null) "regular-or-missing") + (assertEq "symlinks to symlinks to files are detected-ish" + ((pathType ./symlink-symlink-file).symlink or null) "regular-or-missing") + (assertEq "symlinks to nowhere are not distinguished from files" + ((pathType ./missing).symlink or null) "regular-or-missing") + ]; + + cheddarStorePath = + builtins.unsafeDiscardStringContext depot.tools.cheddar.outPath; + + storePathNameTests = it "correctly gets the basename of a store path" [ + (assertEq "base name of a derivation" + (storePathName depot.tools.cheddar) depot.tools.cheddar.name) + (assertEq "base name of a store path string" + (storePathName cheddarStorePath) depot.tools.cheddar.name) + (assertEq "base name of a path within a store path" + (storePathName "${cheddarStorePath}/bin/cheddar") "cheddar") + (assertEq "base name of a path" + (storePathName ../default.nix) "default.nix") + ]; +in + +runTestsuite "nix.utils" [ + pathPredicates + symlinkPathTypeTests + storePathNameTests +] diff --git a/nix/utils/tests/directory/file b/nix/utils/tests/directory/file new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/nix/utils/tests/directory/file diff --git a/nix/utils/tests/missing b/nix/utils/tests/missing new file mode 120000 index 000000000000..cfa0a46515b5 --- /dev/null +++ b/nix/utils/tests/missing @@ -0,0 +1 @@ +does-not-exist \ No newline at end of file diff --git a/nix/utils/tests/symlink-directory b/nix/utils/tests/symlink-directory new file mode 120000 index 000000000000..6d0450cc2421 --- /dev/null +++ b/nix/utils/tests/symlink-directory @@ -0,0 +1 @@ +directory \ No newline at end of file diff --git a/nix/utils/tests/symlink-file b/nix/utils/tests/symlink-file new file mode 120000 index 000000000000..cd5d01792aa4 --- /dev/null +++ b/nix/utils/tests/symlink-file @@ -0,0 +1 @@ +directory/file \ No newline at end of file diff --git a/nix/utils/tests/symlink-symlink-directory b/nix/utils/tests/symlink-symlink-directory new file mode 120000 index 000000000000..5d6ba5247eff --- /dev/null +++ b/nix/utils/tests/symlink-symlink-directory @@ -0,0 +1 @@ +symlink-directory \ No newline at end of file diff --git a/nix/utils/tests/symlink-symlink-file b/nix/utils/tests/symlink-symlink-file new file mode 120000 index 000000000000..f8d9ce3659f4 --- /dev/null +++ b/nix/utils/tests/symlink-symlink-file @@ -0,0 +1 @@ +symlink-file \ No newline at end of file diff --git a/nix/writeElispBin/default.nix b/nix/writeElispBin/default.nix new file mode 100644 index 000000000000..c116607e0aeb --- /dev/null +++ b/nix/writeElispBin/default.nix @@ -0,0 +1,18 @@ +{ depot, pkgs, ... }: + +{ name, src, deps ? (_: []), emacs ? pkgs.emacs27-nox }: + +let + inherit (pkgs) emacsPackages emacsPackagesGen; + inherit (builtins) isString toFile; + + finalEmacs = (emacsPackagesGen emacs).emacsWithPackages deps; + + srcFile = if isString src + then toFile "${name}.el" src + else src; + +in depot.nix.writeScriptBin name '' + #!/bin/sh + ${finalEmacs}/bin/emacs --batch --no-site-file --script ${srcFile} $@ +'' diff --git a/nix/writeExecline/default.nix b/nix/writeExecline/default.nix new file mode 100644 index 000000000000..8626aa46080f --- /dev/null +++ b/nix/writeExecline/default.nix @@ -0,0 +1,38 @@ +{ pkgs, depot, ... }: + +# Write an execline script, represented as nested nix lists. +# Everything is escaped correctly. +# https://skarnet.org/software/execline/ + +# TODO(Profpatsch) upstream into nixpkgs + +name: +{ + # "var": substitute readNArgs variables and start $@ + # from the (readNArgs+1)th argument + # "var-full": substitute readNArgs variables and start $@ from $0 + # "env": don’t substitute, set # and 0…n environment vaariables, where n=$# + # "none": don’t substitute or set any positional arguments + # "env-no-push": like "env", but bypass the push-phase. Not recommended. + argMode ? "var", + # Number of arguments to be substituted as variables (passed to "var"/"-s" or "var-full"/"-S" + readNArgs ? 0, +}: +# Nested list of lists of commands. +# Inner lists are translated to execline blocks. +argList: + +let + env = + if argMode == "var" then "s${toString readNArgs}" + else if argMode == "var-full" then "S${toString readNArgs}" + else if argMode == "env" then "" + else if argMode == "none" then "P" + else if argMode == "env-no-push" then "p" + else abort ''"${toString argMode}" is not a valid argMode, use one of "var", "var-full", "env", "none", "env-no-push".''; + +in + depot.nix.writeScript name '' + #!${pkgs.execline}/bin/execlineb -W${env} + ${depot.nix.escapeExecline argList} + '' diff --git a/nix/writeScript/default.nix b/nix/writeScript/default.nix new file mode 100644 index 000000000000..e8e6e0fa10ac --- /dev/null +++ b/nix/writeScript/default.nix @@ -0,0 +1,29 @@ +{ pkgs, depot, ... }: + +# Write the given string to $out +# and make it executable. + +let + bins = depot.nix.getBins pkgs.s6-portable-utils [ + "s6-cat" + "s6-chmod" + ]; + +in +name: +# string of the executable script that is put in $out +script: + +depot.nix.runExecline name { + stdin = script; + derivationArgs = { + preferLocalBuild = true; + allowSubstitutes = false; + }; +} [ + "importas" "out" "out" + # this pipes stdout of s6-cat to $out + # and s6-cat redirects from stdin to stdout + "if" [ "redirfd" "-w" "1" "$out" bins.s6-cat ] + bins.s6-chmod "0755" "$out" +] diff --git a/nix/writeScriptBin/default.nix b/nix/writeScriptBin/default.nix new file mode 100644 index 000000000000..ed26cf197e1e --- /dev/null +++ b/nix/writeScriptBin/default.nix @@ -0,0 +1,12 @@ +{ depot, ... }: + +# Like writeScript, +# but put the script into `$out/bin/${name}`. + +name: +script: + +depot.nix.binify { + exe = (depot.nix.writeScript name script); + inherit name; +} diff --git a/nix/writers/default.nix b/nix/writers/default.nix new file mode 100644 index 000000000000..b0795ab2c886 --- /dev/null +++ b/nix/writers/default.nix @@ -0,0 +1,93 @@ +{ depot, pkgs, lib, ... }: + +let + bins = depot.nix.getBins pkgs.s6-portable-utils [ "s6-ln" "s6-ls" "s6-touch" ] + ; + + linkTo = name: path: depot.nix.runExecline.local name {} [ + "importas" "out" "out" + bins.s6-ln "-s" path "$out" + ]; + + # Build a rust executable, $out is the executable. + rustSimple = args@{name, ...}: src: + linkTo name "${rustSimpleBin args src}/bin/${name}"; + + # Like `rustSimple`, but put the binary in `$out/bin/`. + rustSimpleBin = { + name, + dependencies ? [], + doCheck ? true, + }: src: + (if doCheck then testRustSimple else pkgs.lib.id) + (pkgs.buildRustCrate ({ + pname = name; + version = "1.0.0"; + crateName = name; + crateBin = [ name ]; + dependencies = dependencies; + src = pkgs.runCommandLocal "write-main.rs" { + src = src; + passAsFile = [ "src" ]; + } '' + mkdir -p $out/src/bin + cp "$srcPath" $out/src/bin/${name}.rs + find $out + ''; + })); + + # Build a rust library, that can be used as dependency to `rustSimple`. + # Wrapper around `pkgs.buildRustCrate`, takes all its arguments. + rustSimpleLib = { + name, + dependencies ? [], + doCheck ? true, + }: src: + (if doCheck then testRustSimple else pkgs.lib.id) + (pkgs.buildRustCrate ({ + pname = name; + version = "1.0.0"; + crateName = name; + dependencies = dependencies; + src = pkgs.runCommandLocal "write-lib.rs" { + src = src; + passAsFile = [ "src" ]; + } '' + mkdir -p $out/src + cp "$srcPath" $out/src/lib.rs + find $out + ''; + })); + + /* Takes a `buildRustCrate` derivation as an input, + * builds it with `{ buildTests = true; }` and runs + * all tests found in its `tests` dir. If they are + * all successful, `$out` will point to the crate + * built with `{ buildTests = false; }`, otherwise + * it will fail to build. + * + * See also `nix.drvSeqL` which is used to implement + * this behavior. + */ + testRustSimple = rustDrv: + let + crate = buildTests: rustDrv.override { inherit buildTests; }; + tests = depot.nix.runExecline.local "${rustDrv.name}-tests-run" {} [ + "importas" "out" "out" + "if" [ + "pipeline" [ bins.s6-ls "${crate true}/tests" ] + "forstdin" "-o0" "test" + "importas" "test" "test" + "${crate true}/tests/$test" + ] + bins.s6-touch "$out" + ]; + in depot.nix.drvSeqL [ tests ] (crate false); + +in { + inherit + rustSimple + rustSimpleBin + rustSimpleLib + ; +} diff --git a/nix/writers/tests/rust.nix b/nix/writers/tests/rust.nix new file mode 100644 index 000000000000..8a12c95ec7da --- /dev/null +++ b/nix/writers/tests/rust.nix @@ -0,0 +1,68 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.nix.writers) + rustSimple + rustSimpleLib + rustSimpleBin + ; + + inherit (pkgs) + coreutils + ; + + run = drv: depot.nix.runExecline.local "run-${drv.name}" {} [ + "if" [ drv ] + "importas" "out" "out" + "${coreutils}/bin/touch" "$out" + ]; + + rustTransitiveLib = rustSimpleLib { + name = "transitive"; + } '' + pub fn transitive(s: &str) -> String { + let mut new = s.to_string(); + new.push_str(" 1 2 3"); + new + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn test_transitive() { + assert_eq!(transitive("foo").as_str(), "foo 1 2 3") + } + } + ''; + + rustTestLib = rustSimpleLib { + name = "test_lib"; + dependencies = [ rustTransitiveLib ]; + } '' + extern crate transitive; + use transitive::{transitive}; + pub fn test() -> String { + transitive("test") + } + ''; + + rustWithLib = run (rustSimple { + name = "rust-with-lib"; + dependencies = [ rustTestLib ]; + } '' + extern crate test_lib; + + fn main() { + assert_eq!(test_lib::test(), String::from("test 1 2 3")); + } + ''); + + +in depot.nix.readTree.drvTargets { + inherit + rustTransitiveLib + rustWithLib + ; +} diff --git a/nix/yants/README.md b/nix/yants/README.md new file mode 100644 index 000000000000..d17ea61b52d1 --- /dev/null +++ b/nix/yants/README.md @@ -0,0 +1,88 @@ +yants +===== + +This is a tiny type-checker for data in Nix, written in Nix. + +# Features + +* Checking of primitive types (`int`, `string` etc.) +* Checking polymorphic types (`option`, `list`, `either`) +* Defining & checking struct/record types +* Defining & matching enum types +* Defining & matching sum types +* Defining function signatures (including curried functions) +* Types are composable! `option string`! `list (either int (option float))`! +* Type errors also compose! + +Currently lacking: + +* Any kind of inference +* Convenient syntax for attribute-set function signatures + +## Primitives & simple polymorphism + + + +## Structs + + + +## Nested structs! + + + +## Enums! + + + +## Functions! + + + +# Usage + +Yants can be imported from its `default.nix`. A single attribute (`lib`) can be +passed, which will otherwise be imported from `<nixpkgs>`. + +TIP: You do not need to clone my whole repository to use Yants! It is split out +into the `nix/yants` branch which you can clone with, for example, `git clone -b +nix/yants https://git.tazj.in yants`. + +Examples for the most common import methods would be: + +1. Import into scope with `with`: + ```nix + with (import ./default.nix {}); + # ... Nix code that uses yants ... + ``` + +2. Import as a named variable: + ```nix + let yants = import ./default.nix {}; + in yants.string "foo" # or other uses ... + ```` + +3. Overlay into `pkgs.lib`: + ```nix + # wherever you import your package set (e.g. from <nixpkgs>): + import <nixpkgs> { + overlays = [ + (self: super: { + lib = super.lib // { yants = import ./default.nix { inherit (super) lib; }; }; + }) + ]; + } + + # yants now lives at lib.yants, besides the other library functions! + ``` + +Please see my [Nix one-pager](https://github.com/tazjin/nix-1p) for more generic +information about the Nix language and what the above constructs mean. + +# Stability + +The current API of Yants is **not yet** considered stable, but it works fine and +should continue to do so even if used at an older version. + +Yants' tests use Nix versions above 2.2 - compatibility with older versions is +not guaranteed. diff --git a/nix/yants/default.nix b/nix/yants/default.nix new file mode 100644 index 000000000000..058444d27be9 --- /dev/null +++ b/nix/yants/default.nix @@ -0,0 +1,326 @@ +# Copyright 2019 Google LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Provides a "type-system" for Nix that provides various primitive & +# polymorphic types as well as the ability to define & check records. +# +# All types (should) compose as expected. + +{ lib ? (import <nixpkgs> {}).lib, ... }: + +with builtins; let + prettyPrint = lib.generators.toPretty {}; + + # typedef' :: struct { + # name = string; + # checkType = function; (a -> result) + # checkToBool = option function; (result -> bool) + # toError = option function; (a -> result -> string) + # def = option any; + # match = option function; + # } -> type + # -> (a -> b) + # -> (b -> bool) + # -> (a -> b -> string) + # -> type + # + # This function creates an attribute set that acts as a type. + # + # It receives a type name, a function that is used to perform a + # check on an arbitrary value, a function that can translate the + # return of that check to a boolean that informs whether the value + # is type-conformant, and a function that can construct error + # messages from the check result. + # + # This function is the low-level primitive used to create types. For + # many cases the higher-level 'typedef' function is more appropriate. + typedef' = { name, checkType + , checkToBool ? (result: result.ok) + , toError ? (_: result: result.err) + , def ? null + , match ? null }: { + inherit name checkToBool toError; + + # check :: a -> bool + # + # This function is used to determine whether a given type is + # conformant. + check = value: checkToBool (checkType value); + + # checkType :: a -> struct { ok = bool; err = option string; } + # + # This function checks whether the passed value is type conformant + # and returns an optional type error string otherwise. + inherit checkType; + + # __functor :: a -> a + # + # This function checks whether the passed value is type conformant + # and throws an error if it is not. + # + # The name of this function is a special attribute in Nix that + # makes it possible to execute a type attribute set like a normal + # function. + __functor = self: value: + let result = self.checkType value; + in if checkToBool result then value + else throw (toError value result); + }; + + typeError = type: val: + "expected type '${type}', but value '${prettyPrint val}' is of type '${typeOf val}'"; + + # typedef :: string -> (a -> bool) -> type + # + # typedef is the simplified version of typedef' which uses a default + # error message constructor. + typedef = name: check: typedef' { + inherit name; + checkType = v: + let res = check v; + in { + ok = res; + } // (lib.optionalAttrs (!res) { + err = typeError name v; + }); + }; + + checkEach = name: t: l: foldl' (acc: e: + let res = t.checkType e; + isT = t.checkToBool res; + in { + ok = acc.ok && isT; + err = if isT + then acc.err + else acc.err + "${prettyPrint e}: ${t.toError e res}\n"; + }) { ok = true; err = "expected type ${name}, but found:\n"; } l; +in lib.fix (self: { + # Primitive types + any = typedef "any" (_: true); + unit = typedef "unit" (v: v == {}); + int = typedef "int" isInt; + bool = typedef "bool" isBool; + float = typedef "float" isFloat; + string = typedef "string" isString; + path = typedef "path" (x: typeOf x == "path"); + drv = typedef "derivation" (x: isAttrs x && x ? "type" && x.type == "derivation"); + function = typedef "function" (x: isFunction x || (isAttrs x && x ? "__functor" + && isFunction x.__functor)); + + # Type for types themselves. Useful when defining polymorphic types. + type = typedef "type" (x: + isAttrs x + && hasAttr "name" x && self.string.check x.name + && hasAttr "checkType" x && self.function.check x.checkType + && hasAttr "checkToBool" x && self.function.check x.checkToBool + && hasAttr "toError" x && self.function.check x.toError + ); + + # Polymorphic types + option = t: typedef' rec { + name = "option<${t.name}>"; + checkType = v: + let res = t.checkType v; + in { + ok = isNull v || (self.type t).checkToBool res; + err = "expected type ${name}, but value does not conform to '${t.name}': " + + t.toError v res; + }; + }; + + eitherN = tn: typedef "either<${concatStringsSep ", " (map (x: x.name) tn)}>" + (x: any (t: (self.type t).check x) tn); + + either = t1: t2: self.eitherN [ t1 t2 ]; + + list = t: typedef' rec { + name = "list<${t.name}>"; + + checkType = v: if isList v + then checkEach name (self.type t) v + else { + ok = false; + err = typeError name v; + }; + }; + + attrs = t: typedef' rec { + name = "attrs<${t.name}>"; + + checkType = v: if isAttrs v + then checkEach name (self.type t) (attrValues v) + else { + ok = false; + err = typeError name v; + }; + }; + + # Structs / record types + # + # Checks that all fields match their declared types, no optional + # fields are missing and no unexpected fields occur in the struct. + # + # Anonymous structs are supported (e.g. for nesting) by omitting the + # name. + # + # TODO: Support open records? + struct = + # Struct checking is more involved than the simpler types above. + # To make the actual type definition more readable, several + # helpers are defined below. + let + # checkField checks an individual field of the struct against + # its definition and creates a typecheck result. These results + # are aggregated during the actual checking. + checkField = def: name: value: let result = def.checkType value; in rec { + ok = def.checkToBool result; + err = if !ok && isNull value + then "missing required ${def.name} field '${name}'\n" + else "field '${name}': ${def.toError value result}\n"; + }; + + # checkExtraneous determines whether a (closed) struct contains + # any fields that are not part of the definition. + checkExtraneous = def: has: acc: + if (length has) == 0 then acc + else if (hasAttr (head has) def) + then checkExtraneous def (tail has) acc + else checkExtraneous def (tail has) { + ok = false; + err = acc.err + "unexpected struct field '${head has}'\n"; + }; + + # checkStruct combines all structure checks and creates one + # typecheck result from them + checkStruct = def: value: + let + init = { ok = true; err = ""; }; + extraneous = checkExtraneous def (attrNames value) init; + + checkedFields = map (n: + let v = if hasAttr n value then value."${n}" else null; + in checkField def."${n}" n v) (attrNames def); + + combined = foldl' (acc: res: { + ok = acc.ok && res.ok; + err = if !res.ok then acc.err + res.err else acc.err; + }) init checkedFields; + in { + ok = combined.ok && extraneous.ok; + err = combined.err + extraneous.err; + }; + + struct' = name: def: typedef' { + inherit name def; + checkType = value: if isAttrs value + then (checkStruct (self.attrs self.type def) value) + else { ok = false; err = typeError name value; }; + + toError = _: result: "expected '${name}'-struct, but found:\n" + result.err; + }; + in arg: if isString arg then (struct' arg) else (struct' "anon" arg); + + # Enums & pattern matching + enum = + let + plain = name: def: typedef' { + inherit name def; + + checkType = (x: isString x && elem x def); + checkToBool = x: x; + toError = value: _: "'${prettyPrint value} is not a member of enum ${name}"; + }; + enum' = name: def: lib.fix (e: (plain name def) // { + match = x: actions: deepSeq (map e (attrNames actions)) ( + let + actionKeys = attrNames actions; + missing = foldl' (m: k: if (elem k actionKeys) then m else m ++ [ k ]) [] def; + in if (length missing) > 0 + then throw "Missing match action for members: ${prettyPrint missing}" + else actions."${e x}"); + }); + in arg: if isString arg then (enum' arg) else (enum' "anon" arg); + + # Sum types + # + # The representation of a sum type is an attribute set with only one + # value, where the key of the value denotes the variant of the type. + sum = + let + plain = name: def: typedef' { + inherit name def; + checkType = (x: + let variant = elemAt (attrNames x) 0; + in if isAttrs x && length (attrNames x) == 1 && hasAttr variant def + then let t = def."${variant}"; + v = x."${variant}"; + res = t.checkType v; + in if t.checkToBool res + then { ok = true; } + else { + ok = false; + err = "while checking '${name}' variant '${variant}': " + + t.toError v res; + } + else { ok = false; err = typeError name x; } + ); + }; + sum' = name: def: lib.fix (s: (plain name def) // { + match = x: actions: + let variant = deepSeq (s x) (elemAt (attrNames x) 0); + actionKeys = attrNames actions; + defKeys = attrNames def; + missing = foldl' (m: k: if (elem k actionKeys) then m else m ++ [ k ]) [] defKeys; + in if (length missing) > 0 + then throw "Missing match action for variants: ${prettyPrint missing}" + else actions."${variant}" x."${variant}"; + }); + in arg: if isString arg then (sum' arg) else (sum' "anon" arg); + + # Typed function definitions + # + # These definitions wrap the supplied function in type-checking + # forms that are evaluated when the function is called. + # + # Note that typed functions themselves are not types and can not be + # used to check values for conformity. + defun = + let + mkFunc = sig: f: { + inherit sig; + __toString = self: foldl' (s: t: "${s} -> ${t.name}") + "λ :: ${(head self.sig).name}" (tail self.sig); + __functor = _: f; + }; + + defun' = sig: func: if length sig > 2 + then mkFunc sig (x: defun' (tail sig) (func ((head sig) x))) + else mkFunc sig (x: ((head (tail sig)) (func ((head sig) x)))); + + in sig: func: if length sig < 2 + then (throw "Signature must at least have two types (a -> b)") + else defun' sig func; + + # Restricting types + # + # `restrict` wraps a type `t`, and uses a predicate `pred` to further + # restrict the values, giving the restriction a descriptive `name`. + # + # First, the wrapped type definition is checked (e.g. int) and then the + # value is checked with the predicate, so the predicate can already + # depend on the value being of the wrapped type. + restrict = name: pred: t: + let restriction = "${t.name}[${name}]"; in typedef' { + name = restriction; + checkType = v: + let res = t.checkType v; + in + if !(t.checkToBool res) + then res + else { + ok = pred v; + err = "${prettyPrint v} does not conform to restriction '${restriction}'"; + }; + }; + +}) diff --git a/nix/yants/screenshots/enums.png b/nix/yants/screenshots/enums.png new file mode 100644 index 000000000000..71673e7ab63c --- /dev/null +++ b/nix/yants/screenshots/enums.png Binary files differdiff --git a/nix/yants/screenshots/functions.png b/nix/yants/screenshots/functions.png new file mode 100644 index 000000000000..30ed50f8327b --- /dev/null +++ b/nix/yants/screenshots/functions.png Binary files differdiff --git a/nix/yants/screenshots/nested-structs.png b/nix/yants/screenshots/nested-structs.png new file mode 100644 index 000000000000..6b03ed65ceb7 --- /dev/null +++ b/nix/yants/screenshots/nested-structs.png Binary files differdiff --git a/nix/yants/screenshots/simple.png b/nix/yants/screenshots/simple.png new file mode 100644 index 000000000000..05a302cc6b9d --- /dev/null +++ b/nix/yants/screenshots/simple.png Binary files differdiff --git a/nix/yants/screenshots/structs.png b/nix/yants/screenshots/structs.png new file mode 100644 index 000000000000..fcbcf6415fad --- /dev/null +++ b/nix/yants/screenshots/structs.png Binary files differdiff --git a/nix/yants/tests/default.nix b/nix/yants/tests/default.nix new file mode 100644 index 000000000000..9a0b2403e124 --- /dev/null +++ b/nix/yants/tests/default.nix @@ -0,0 +1,156 @@ +{ depot, pkgs, ... }: + +with depot.nix.yants; + +# Note: Derivations are not included in the tests below as they cause +# issues with deepSeq. + +let + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + assertDoesNotThrow + ; + + # this derivation won't throw if evaluated with deepSeq + # unlike most things even remotely related with nixpkgs + trivialDerivation = derivation { + name = "trivial-derivation"; + inherit (pkgs.stdenv) system; + builder = "/bin/sh"; + args = [ "-c" "echo hello > $out" ]; + }; + + testPrimitives = it "checks that all primitive types match" [ + (assertDoesNotThrow "unit type" (unit {})) + (assertDoesNotThrow "int type" (int 15)) + (assertDoesNotThrow "bool type" (bool false)) + (assertDoesNotThrow "float type" (float 13.37)) + (assertDoesNotThrow "string type" (string "Hello!")) + (assertDoesNotThrow "function type" (function (x: x * 2))) + (assertDoesNotThrow "path type" (path /nix)) + (assertDoesNotThrow "derivation type" (drv trivialDerivation)) + ]; + + testPoly = it "checks that polymorphic types work as intended" [ + (assertDoesNotThrow "option type" (option int null)) + (assertDoesNotThrow "list type" (list string [ "foo" "bar" ])) + (assertDoesNotThrow "either type" (either int float 42)) + ]; + + # Test that structures work as planned. + person = struct "person" { + name = string; + age = int; + + contact = option (struct { + email = string; + phone = option string; + }); + }; + + testStruct = it "checks that structures work as intended" [ + (assertDoesNotThrow "person struct" (person { + name = "Brynhjulf"; + age = 42; + contact.email = "brynhjulf@yants.nix"; + })) + ]; + + # Test enum definitions & matching + colour = enum "colour" [ "red" "blue" "green" ]; + colourMatcher = { + red = "It is in fact red!"; + blue = "It should not be blue!"; + green = "It should not be green!"; + }; + + testEnum = it "checks enum definitions and matching" [ + (assertEq "enum is matched correctly" + "It is in fact red!" (colour.match "red" colourMatcher)) + (assertThrows "out of bounds enum fails" + (colour.match "alpha" (colourMatcher // { + alpha = "This should never happen"; + })) + ) + ]; + + # Test sum type definitions + creature = sum "creature" { + human = struct { + name = string; + age = option int; + }; + + pet = enum "pet" [ "dog" "lizard" "cat" ]; + }; + some-human = creature { + human = { + name = "Brynhjulf"; + age = 42; + }; + }; + + testSum = it "checks sum types definitions and matching" [ + (assertDoesNotThrow "creature sum type" some-human) + (assertEq "sum type is matched correctly" + "It's a human named Brynhjulf" (creature.match some-human { + human = v: "It's a human named ${v.name}"; + pet = v: "It's not supposed to be a pet!"; + }) + ) + ]; + + # Test curried function definitions + func = defun [ string int string ] + (name: age: "${name} is ${toString age} years old"); + + testFunctions = it "checks function definitions" [ + (assertDoesNotThrow "function application" (func "Brynhjulf" 42)) + ]; + + # Test that all types are types. + assertIsType = name: t: + assertDoesNotThrow "${name} is a type" (type t); + testTypes = it "checks that all types are types" [ + (assertIsType "any" any) + (assertIsType "bool" bool) + (assertIsType "drv" drv) + (assertIsType "float" float) + (assertIsType "int" int) + (assertIsType "string" string) + (assertIsType "path" path) + + (assertIsType "attrs int" (attrs int)) + (assertIsType "eitherN [ ... ]" (eitherN [ int string bool ])) + (assertIsType "either int string" (either int string)) + (assertIsType "enum [ ... ]" (enum [ "foo" "bar" ])) + (assertIsType "list string" (list string)) + (assertIsType "option int" (option int)) + (assertIsType "option (list string)" (option (list string))) + (assertIsType "struct { ... }" (struct { a = int; b = option string; })) + (assertIsType "sum { ... }" (sum { a = int; b = option string; })) + ]; + + testRestrict = it "checks restrict types" [ + (assertDoesNotThrow "< 42" ((restrict "< 42" (i: i < 42) int) 25)) + (assertDoesNotThrow "list length < 3" + ((restrict "not too long" (l: builtins.length l < 3) (list int)) [ 1 2 ])) + (assertDoesNotThrow "list eq 5" + (list (restrict "eq 5" (v: v == 5) any) [ 5 5 5 ])) + ]; + +in + runTestsuite "yants" [ + testPrimitives + testPoly + testStruct + testEnum + testSum + testFunctions + testTypes + testRestrict + ] diff --git a/ops/besadii/default.nix b/ops/besadii/default.nix new file mode 100644 index 000000000000..1199c56cfb94 --- /dev/null +++ b/ops/besadii/default.nix @@ -0,0 +1,8 @@ +# This program is used as a Gerrit hook to trigger builds on +# Buildkite, Sourcegraph reindexing and other maintenance tasks. +{ depot, ... }: + +depot.nix.buildGo.program { + name = "besadii"; + srcs = [ ./main.go ]; +} diff --git a/ops/besadii/main.go b/ops/besadii/main.go new file mode 100644 index 000000000000..ef4f4fe42489 --- /dev/null +++ b/ops/besadii/main.go @@ -0,0 +1,510 @@ +// Copyright 2019-2020 Google LLC. +// SPDX-License-Identifier: Apache-2.0 +// +// besadii is a small CLI tool that is invoked as a hook by various +// programs to cause CI-related actions. +// +// It supports the following modes & operations: +// +// Gerrit (ref-updated) hook: +// - Trigger Buildkite CI builds +// - Trigger SourceGraph repository index updates +// +// Buildkite (post-command) hook: +// - Submit CL verification status back to Gerrit +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "log/syslog" + "net/http" + "net/mail" + "os" + "os/user" + "path" + "regexp" + "strconv" +) + +// Regular expression to extract change ID out of a URL +var changeIdRegexp = regexp.MustCompile(`^.*/(\d+)$`) + +// besadii configuration file structure +type config struct { + // Required configuration for Buildkite<>Gerrit monorepo + // integration. + Repository string `json:"repository"` + Branch string `json:"branch"` + GerritUrl string `json:"gerritUrl"` + GerritUser string `json:"gerritUser"` + GerritPassword string `json:"gerritPassword"` + BuildkiteOrg string `json:"buildkiteOrg"` + BuildkiteProject string `json:"buildkiteProject"` + BuildkiteToken string `json:"buildkiteToken"` + + // Optional configuration for Sourcegraph trigger updates. + SourcegraphUrl string `json:"sourcegraphUrl"` + SourcegraphToken string `json:"sourcegraphToken"` +} + +// buildTrigger represents the information passed to besadii when it +// is invoked as a Gerrit hook. +// +// https://gerrit.googlesource.com/plugins/hooks/+/HEAD/src/main/resources/Documentation/hooks.md +type buildTrigger struct { + project string + ref string + commit string + author string + email string + + changeId string + patchset string +} + +type Author struct { + Name string `json:"name"` + Email string `json:"email"` +} + +// Build is the representation of a Buildkite build as described on +// https://buildkite.com/docs/apis/rest-api/builds#create-a-build +type Build struct { + Commit string `json:"commit"` + Branch string `json:"branch"` + Author Author `json:"author"` + Env map[string]string `json:"env"` +} + +// BuildResponse is the representation of Buildkite's success response +// after triggering a build. This has many fields, but we only need +// one of them. +type buildResponse struct { + WebUrl string `json:"web_url"` +} + +// reviewInput is a struct representing the data submitted to Gerrit +// to post a review on a CL. +// +// https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#review-input +type reviewInput struct { + Message string `json:"message"` + Labels map[string]int `json:"labels,omitempty"` + OmitDuplicateComments bool `json:"omit_duplicate_comments"` + IgnoreDefaultAttentionSetRules bool `json:"ignore_default_attention_set_rules"` + Tag string `json:"tag"` +} + +func defaultConfigLocation() (string, error) { + usr, err := user.Current() + if err != nil { + return "", fmt.Errorf("failed to get current user: %w", err) + } + + return path.Join(usr.HomeDir, "besadii.json"), nil +} + +func loadConfig() (*config, error) { + configPath := os.Getenv("BESADII_CONFIG") + + if configPath == "" { + var err error + configPath, err = defaultConfigLocation() + if err != nil { + return nil, fmt.Errorf("failed to get config location: %w", err) + } + } + + configJson, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to load besadii config: %w", err) + } + + var cfg config + err = json.Unmarshal(configJson, &cfg) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal besadii config: %w", err) + } + + // Rudimentary config validation logic + if cfg.SourcegraphUrl != "" && cfg.SourcegraphToken == "" { + return nil, fmt.Errorf("'SourcegraphToken' must be set if 'SourcegraphUrl' is set") + } + + if cfg.Repository == "" || cfg.Branch == "" { + return nil, fmt.Errorf("missing repository configuration (required: repository, branch)") + } + + if cfg.GerritUrl == "" || cfg.GerritUser == "" || cfg.GerritPassword == "" { + return nil, fmt.Errorf("missing Gerrit configuration (required: gerritUrl, gerritUser, gerritPassword)") + } + + if cfg.BuildkiteOrg == "" || cfg.BuildkiteProject == "" || cfg.BuildkiteToken == "" { + return nil, fmt.Errorf("mising Buildkite configuration (required: buildkiteOrg, buildkiteProject, buildkiteToken)") + } + + return &cfg, nil +} + +// updateGerrit posts a comment on a Gerrit CL to indicate the current build status. +func updateGerrit(cfg *config, review reviewInput, changeId, patchset string) { + body, _ := json.Marshal(review) + reader := ioutil.NopCloser(bytes.NewReader(body)) + + url := fmt.Sprintf("%s/a/changes/%s/revisions/%s/review", cfg.GerritUrl, changeId, patchset) + req, err := http.NewRequest("POST", url, reader) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to create an HTTP request: %w", err) + os.Exit(1) + } + + req.SetBasicAuth(cfg.GerritUser, cfg.GerritPassword) + req.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + fmt.Errorf("failed to update CL on Gerrit: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + respBody, _ := ioutil.ReadAll(resp.Body) + fmt.Fprintf(os.Stderr, "received non-success response from Gerrit: %s (%v)", respBody, resp.Status) + } else { + fmt.Printf("Added CI status comment on %s/c/%s/+/%s/%s", cfg.GerritUrl, cfg.Repository, changeId, patchset) + } +} + +// Trigger a build of a given branch & commit on Buildkite +func triggerBuild(cfg *config, log *syslog.Writer, trigger *buildTrigger) error { + env := make(map[string]string) + + // Pass information about the originating Gerrit change to the + // build, if it is for a patchset. + // + // This information is later used by besadii when invoked by Gerrit + // to communicate the build status back to Gerrit. + headBuild := true + if trigger.changeId != "" && trigger.patchset != "" { + env["GERRIT_CHANGE_ID"] = trigger.changeId + env["GERRIT_PATCHSET"] = trigger.patchset + headBuild = false + } + + build := Build{ + Commit: trigger.commit, + Branch: trigger.ref, + Env: env, + Author: Author{ + Name: trigger.author, + Email: trigger.email, + }, + } + + body, _ := json.Marshal(build) + reader := ioutil.NopCloser(bytes.NewReader(body)) + + bkUrl := fmt.Sprintf("https://api.buildkite.com/v2/organizations/%s/pipelines/%s/builds", cfg.BuildkiteOrg, cfg.BuildkiteProject) + req, err := http.NewRequest("POST", bkUrl, reader) + if err != nil { + return fmt.Errorf("failed to create an HTTP request: %w", err) + } + + req.Header.Add("Authorization", "Bearer "+cfg.BuildkiteToken) + req.Header.Add("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + // This might indicate a temporary error on the Buildkite side. + return fmt.Errorf("failed to send Buildkite request: %w", err) + } + defer resp.Body.Close() + + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read Buildkite response body: %w", err) + } + + if resp.StatusCode != http.StatusCreated { + return fmt.Errorf("received non-success response from Buildkite: %s (%v)", respBody, resp.Status) + } + + var buildResp buildResponse + err = json.Unmarshal(respBody, &buildResp) + if err != nil { + return fmt.Errorf("failed to unmarshal build response: %w", err) + } + + fmt.Fprintf(log, "triggered build for ref %q at commit %q: %s", trigger.ref, trigger.commit, buildResp.WebUrl) + + // For builds of the HEAD branch there is nothing else to do + if headBuild { + return nil + } + + // Report the status back to the Gerrit CL so that users can click + // through to the running build. + msg := fmt.Sprintf("Started build for patchset #%s of cl/%s: %s", trigger.patchset, trigger.changeId, buildResp.WebUrl) + review := reviewInput{ + Message: msg, + OmitDuplicateComments: true, + Tag: "autogenerated:buildkite~trigger", + + // Do not update the attention set for this comment. + IgnoreDefaultAttentionSetRules: true, + } + updateGerrit(cfg, review, trigger.changeId, trigger.patchset) + + return nil +} + +// Trigger a Sourcegraph repository index update. +// +// https://docs.sourcegraph.com/admin/repo/webhooks +func triggerIndexUpdate(cfg *config, log *syslog.Writer) error { + req, err := http.NewRequest("POST", cfg.SourcegraphUrl, nil) + if err != nil { + return err + } + + req.Header.Add("Authorization", "token "+cfg.SourcegraphToken) + + _, err = http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("failed to trigger Sourcegraph index update: %w", err) + } + + log.Info("triggered sourcegraph index update") + return nil +} + +// Gerrit passes more flags than we want, but Rob Pike decided[0] in +// 2013 that the Go art project will not allow users to ignore flags +// because he "doesn't like it". This function allows users to ignore +// flags. +// +// [0]: https://github.com/golang/go/issues/6112#issuecomment-66083768 +func ignoreFlags(ignore []string) { + for _, f := range ignore { + flag.String(f, "", "flag to ignore") + } +} + +// Extract the username & email from Gerrit's uploader flag and set it +// on the trigger struct, for display in Buildkite. +func extractChangeUploader(uploader string, trigger *buildTrigger) error { + // Gerrit passes the uploader in another extra layer of quotes. + uploader, err := strconv.Unquote(uploader) + if err != nil { + return fmt.Errorf("failed to unquote email - forgot quotes on manual invocation?: %w", err) + } + + // Extract the uploader username & email from the input passed by + // Gerrit (in RFC 5322 format). + addr, err := mail.ParseAddress(uploader) + if err != nil { + return fmt.Errorf("invalid change uploader (%s): %w", uploader, err) + } + + trigger.author = addr.Name + trigger.email = addr.Address + + return nil +} + +// Extract the buildtrigger struct out of the flags passed to besadii +// when invoked as Gerrit's 'patchset-created' hook. This hook is used +// for triggering CI on in-progress CLs. +func buildTriggerFromPatchsetCreated(cfg *config) (*buildTrigger, error) { + // Information that needs to be returned + var trigger buildTrigger + + // Information that is only needed for parsing + var targetBranch, changeUrl, uploader string + + flag.StringVar(&trigger.project, "project", "", "Gerrit project") + flag.StringVar(&trigger.commit, "commit", "", "commit hash") + flag.StringVar(&trigger.patchset, "patchset", "", "patchset ID") + + flag.StringVar(&targetBranch, "branch", "", "CL target branch") + flag.StringVar(&changeUrl, "change-url", "", "HTTPS URL of change") + flag.StringVar(&uploader, "uploader", "", "Change uploader name & email") + + // patchset-created also passes various flags which we don't need. + ignoreFlags([]string{"kind", "topic", "change", "uploader-username", "change-owner", "change-owner-username"}) + + flag.Parse() + + // Parse username & email + err := extractChangeUploader(uploader, &trigger) + if err != nil { + return nil, err + } + + // If the patchset is not for the HEAD branch of the monorepo, then + // we can ignore it. It might be some other kind of change + // (refs/meta/config or Gerrit-internal), but it is not an error. + if trigger.project != cfg.Repository || targetBranch != cfg.Branch { + return nil, nil + } + + // Change ID is not directly passed in the numeric format, so we + // need to extract it out of the URL + matches := changeIdRegexp.FindStringSubmatch(changeUrl) + trigger.changeId = matches[1] + + // Construct the CL ref from which the build should happen. + changeId, _ := strconv.Atoi(trigger.changeId) + trigger.ref = fmt.Sprintf( + "refs/changes/%02d/%s/%s", + changeId%100, trigger.changeId, trigger.patchset, + ) + + return &trigger, nil +} + +// Extract the buildtrigger struct out of the flags passed to besadii +// when invoked as Gerrit's 'change-merged' hook. This hook is used +// for triggering HEAD builds after change submission. +func buildTriggerFromChangeMerged(cfg *config) (*buildTrigger, error) { + // Information that needs to be returned + var trigger buildTrigger + + // Information that is only needed for parsing + var targetBranch, submitter string + + flag.StringVar(&trigger.project, "project", "", "Gerrit project") + flag.StringVar(&trigger.commit, "commit", "", "Commit hash") + flag.StringVar(&submitter, "submitter", "", "Submitter email & username") + flag.StringVar(&targetBranch, "branch", "", "CL target branch") + + // Ignore extra flags passed by change-merged + ignoreFlags([]string{"change", "topic", "change-url", "submitter-username", "newrev", "change-owner", "change-owner-username"}) + + flag.Parse() + + // Parse username & email + err := extractChangeUploader(submitter, &trigger) + if err != nil { + return nil, err + } + + // If the patchset is not for the HEAD branch of the monorepo, then + // we can ignore it. + if trigger.project != cfg.Repository || targetBranch != cfg.Branch { + return nil, nil + } + + trigger.ref = "refs/heads/" + targetBranch + + return &trigger, nil +} + +func gerritHookMain(cfg *config, log *syslog.Writer, trigger *buildTrigger) { + if trigger == nil { + // The hook was not for something we care about. + os.Exit(0) + } + + err := triggerBuild(cfg, log, trigger) + + if err != nil { + log.Err(fmt.Sprintf("failed to trigger Buildkite build: %s", err)) + } + + if cfg.SourcegraphUrl != "" && trigger.ref == "refs/heads/canon" { + err = triggerIndexUpdate(cfg, log) + if err != nil { + log.Err(fmt.Sprintf("failed to trigger sourcegraph index update: %s", err)) + } + } +} + +func postCommandMain(cfg *config) { + changeId := os.Getenv("GERRIT_CHANGE_ID") + patchset := os.Getenv("GERRIT_PATCHSET") + + if changeId == "" || patchset == "" { + // If these variables are unset, but the hook was invoked, the + // build was most likely for a branch and not for a CL - no status + // needs to be reported back to Gerrit! + fmt.Println("This isn't a CL build, nothing to do. Have a nice day!") + return + } + + if os.Getenv("BUILDKITE_LABEL") != ":duck:" { + // this is not the build stage, don't do anything. + return + } + + var verified int + var verb string + + if os.Getenv("BUILDKITE_COMMAND_EXIT_STATUS") == "0" { + verified = 1 // Verified: +1 in Gerrit + verb = "passed" + } else { + verified = -1 + verb = "failed" + } + + msg := fmt.Sprintf("Build of patchset %s %s: %s", patchset, verb, os.Getenv("BUILDKITE_BUILD_URL")) + review := reviewInput{ + Message: msg, + OmitDuplicateComments: true, + Labels: map[string]int{ + "Verified": verified, + }, + + // Update the attention set if we are failing this patchset. + IgnoreDefaultAttentionSetRules: verified == 1, + + Tag: "autogenerated:buildkite~result", + } + updateGerrit(cfg, review, changeId, patchset) +} + +func main() { + // Logging happens in syslog because it's almost impossible to get + // output out of Gerrit hooks otherwise. + log, err := syslog.New(syslog.LOG_INFO|syslog.LOG_USER, "besadii") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to open syslog: %s\n", err) + os.Exit(1) + } + + log.Info(fmt.Sprintf("besadii called with arguments: %v", os.Args)) + + bin := path.Base(os.Args[0]) + cfg, err := loadConfig() + + if err != nil { + log.Crit(fmt.Sprintf("besadii configuration error: %v", err)) + os.Exit(4) + } + + if bin == "patchset-created" { + trigger, err := buildTriggerFromPatchsetCreated(cfg) + if err != nil { + log.Crit(fmt.Sprintf("failed to parse 'patchset-created' invocation from args: %v", err)) + os.Exit(1) + } + gerritHookMain(cfg, log, trigger) + } else if bin == "change-merged" { + trigger, err := buildTriggerFromChangeMerged(cfg) + if err != nil { + log.Crit(fmt.Sprintf("failed to parse 'change-merged' invocation from args: %v", err)) + os.Exit(1) + } + gerritHookMain(cfg, log, trigger) + } else if bin == "post-command" { + postCommandMain(cfg) + } else { + fmt.Fprintf(os.Stderr, "besadii does not know how to be invoked as %q, sorry!", bin) + os.Exit(1) + } +} diff --git a/ops/deploy-whitby/default.nix b/ops/deploy-whitby/default.nix new file mode 100644 index 000000000000..f4bd257c4f51 --- /dev/null +++ b/ops/deploy-whitby/default.nix @@ -0,0 +1,31 @@ +{ pkgs, ... }: + +pkgs.stdenv.mkDerivation { + name = "deploy-whitby"; + + phases = [ "installPhase" "installCheckPhase" ]; + + nativeBuildInputs = with pkgs; [ + makeWrapper + ]; + + installPhase = '' + mkdir -p $out/bin + makeWrapper ${./deploy-whitby.sh} $out/bin/deploy-whitby.sh \ + --prefix PATH : ${with pkgs; lib.makeBinPath [ + ansi2html + git + jq + nix-diff + ]} + ''; + + installCheckInputs = with pkgs; [ + shellcheck + ]; + + doInstallCheck = true; + installCheckPhase = '' + shellcheck $out/bin/deploy-whitby.sh + ''; +} diff --git a/ops/deploy-whitby/deploy-whitby.sh b/ops/deploy-whitby/deploy-whitby.sh new file mode 100755 index 000000000000..e41ec681f170 --- /dev/null +++ b/ops/deploy-whitby/deploy-whitby.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +set -Ceuo pipefail + +HTML_ROOT="${HTML_ROOT:-/var/html/deploys.tvl.fyi}" +URL_BASE="${URL_BASE:-https://deploys.tvl.fyi/diff}" +IRCCAT_PORT="${IRCCAT_PORT:-4722}" + +drv_hash() { + basename "$1" | sed 's/-.*//' +} + +new_rev="$1" + +if [ -z "$new_rev" ]; then + >&2 echo "Usage: $0 <new_rev>" + exit 1 +fi + +if [ -d "/tmp/deploy.worktree" ]; then + >&2 echo "/tmp/deploy.worktree exists - exiting in case another deploy is currently running" + exit 1 +fi + +worktree_dir=/tmp/worktree_dir + +cleanup() { + rm -rf "$worktree_dir" +} +trap cleanup EXIT + +git clone https://cl.tvl.fyi/depot "$worktree_dir" --reference /depot +git -C "$worktree_dir" checkout "$new_rev" + +current=$(nix show-derivation /run/current-system | jq -r 'keys | .[0]') +new=$(nix-instantiate -A ops.nixos.whitbySystem "$worktree_dir") + +diff_filename="$(drv_hash "$current")..$(drv_hash "$new").html" +nix-diff "$current" "$new" --color always \ + | ansi2html \ + >| "$HTML_ROOT/diff/$diff_filename" +chmod a+r "$HTML_ROOT/diff/$diff_filename" + +echo "#tvl whitby is being deployed! system diff: $URL_BASE/$diff_filename" \ + | nc -w 5 -N localhost "$IRCCAT_PORT" + +# TODO(grfn): Actually do the deploy diff --git a/ops/dns/README.md b/ops/dns/README.md new file mode 100644 index 000000000000..2290299fe46c --- /dev/null +++ b/ops/dns/README.md @@ -0,0 +1,11 @@ +DNS configuration +================= + +This folder contains configuration for our DNS zones. The zones are hosted with +Google Cloud DNS, which supports zone-file based import/export. + +Currently there is no automation to deploy these zones, but CI will check their +integrity. + +*Note: While each zone file specifies an SOA record, it only exists to satisfy +`named-checkzone`. Cloud DNS manages this record for us.* diff --git a/ops/dns/default.nix b/ops/dns/default.nix new file mode 100644 index 000000000000..136a4c58dca7 --- /dev/null +++ b/ops/dns/default.nix @@ -0,0 +1,13 @@ +# Performs simple (local-only) validity checks on DNS zones. +{ depot, pkgs, ... }: + +let + checkZone = zone: file: pkgs.runCommandNoCC "${zone}-check" {} '' + ${pkgs.bind}/bin/named-checkzone -i local ${zone} ${file} | tee $out + ''; + +in depot.nix.readTree.drvTargets { + nixery-dev = checkZone "nixery.dev" ./nixery.dev.zone; + tvl-fyi = checkZone "tvl.fyi" ./tvl.fyi.zone; + tvl-su = checkZone "tvl.su" ./tvl.su.zone; +} diff --git a/ops/dns/nixery.dev.zone b/ops/dns/nixery.dev.zone new file mode 100644 index 000000000000..44cabab29bd8 --- /dev/null +++ b/ops/dns/nixery.dev.zone @@ -0,0 +1,10 @@ +;; Google Cloud DNS zone for nixery.dev +nixery.dev. 21600 IN SOA ns-cloud-b1.googledomains.com. cloud-dns-hostmaster.google.com. 5 21600 3600 259200 300 +nixery.dev. 21600 IN NS ns-cloud-b1.googledomains.com. +nixery.dev. 21600 IN NS ns-cloud-b2.googledomains.com. +nixery.dev. 21600 IN NS ns-cloud-b3.googledomains.com. +nixery.dev. 21600 IN NS ns-cloud-b4.googledomains.com. + +;; Records for pointing nixery.dev to whitby +nixery.dev. 300 IN A 49.12.129.211 +nixery.dev. 300 IN AAAA 2a01:4f8:242:5b21:0:feed:edef:beef diff --git a/ops/dns/tvl.fyi.zone b/ops/dns/tvl.fyi.zone new file mode 100644 index 000000000000..d1961c6a7a82 --- /dev/null +++ b/ops/dns/tvl.fyi.zone @@ -0,0 +1,39 @@ +;; Google Cloud DNS zone for tvl.fyi. +;; +;; This zone is hosted in the project 'tvl-fyi', and registered via +;; Google Domains. +tvl.fyi. 21600 IN SOA ns-cloud-b1.googledomains.com. cloud-dns-hostmaster.google.com. 20 21600 3600 259200 300 +tvl.fyi. 21600 IN NS ns-cloud-b1.googledomains.com. +tvl.fyi. 21600 IN NS ns-cloud-b2.googledomains.com. +tvl.fyi. 21600 IN NS ns-cloud-b3.googledomains.com. +tvl.fyi. 21600 IN NS ns-cloud-b4.googledomains.com. + +;; Mail forwarding (via domains.google) +tvl.fyi. 3600 IN MX 5 gmr-smtp-in.l.google.com. +tvl.fyi. 3600 IN MX 10 alt1.gmr-smtp-in.l.google.com. +tvl.fyi. 3600 IN MX 20 alt2.gmr-smtp-in.l.google.com. +tvl.fyi. 3600 IN MX 30 alt3.gmr-smtp-in.l.google.com. +tvl.fyi. 3600 IN MX 40 alt4.gmr-smtp-in.l.google.com. + +;; Landing website is hosted on whitby on the apex. +tvl.fyi. 21600 IN A 49.12.129.211 +tvl.fyi. 21600 IN AAAA 2a01:4f8:242:5b21:0:feed:edef:beef + +;; TVL infrastructure +whitby.tvl.fyi. 21600 IN A 49.12.129.211 +whitby.tvl.fyi. 21600 IN AAAA 2a01:4f8:242:5b21:0:feed:edef:beef + +;; TVL services +at.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +atward.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +b.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +cache.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +cl.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +code.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +cs.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +deploys.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +images.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +login.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +static.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +status.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. +todo.tvl.fyi. 21600 IN CNAME whitby.tvl.fyi. diff --git a/ops/dns/tvl.su.zone b/ops/dns/tvl.su.zone new file mode 100644 index 000000000000..da46752f13ba --- /dev/null +++ b/ops/dns/tvl.su.zone @@ -0,0 +1,51 @@ +;; Google Cloud DNS for tvl.su. +;; +;; This zone is hosted in the project 'tvl-fyi', and registered via +;; NIC.RU. +;; +;; This zone is mostly identical to tvl.fyi and will eventually become +;; the primary zone. +tvl.su. 21600 IN SOA ns-cloud-b1.googledomains.com. cloud-dns-hostmaster.google.com. 33 21600 3600 259200 300 +tvl.su. 21600 IN NS ns-cloud-b1.googledomains.com. +tvl.su. 21600 IN NS ns-cloud-b2.googledomains.com. +tvl.su. 21600 IN NS ns-cloud-b3.googledomains.com. +tvl.su. 21600 IN NS ns-cloud-b4.googledomains.com. + +;; Landing website is hosted on whitby on the apex. +tvl.su. 21600 IN A 49.12.129.211 +tvl.su. 21600 IN AAAA 2a01:4f8:242:5b21:0:feed:edef:beef + +;; TVL infrastructure +whitby.tvl.su. 21600 IN A 49.12.129.211 +whitby.tvl.su. 21600 IN AAAA 2a01:4f8:242:5b21:0:feed:edef:beef + +;; TVL services +at.tvl.su. 21600 IN CNAME whitby.tvl.su. +atward.tvl.su. 21600 IN CNAME whitby.tvl.su. +b.tvl.su. 21600 IN CNAME whitby.tvl.su. +cache.tvl.su. 21600 IN CNAME whitby.tvl.su. +cl.tvl.su. 21600 IN CNAME whitby.tvl.su. +code.tvl.su. 21600 IN CNAME whitby.tvl.su. +cs.tvl.su. 21600 IN CNAME whitby.tvl.su. +images.tvl.su. 21600 IN CNAME whitby.tvl.su. +login.tvl.su. 21600 IN CNAME whitby.tvl.su. +static.tvl.su. 21600 IN CNAME whitby.tvl.su. +status.tvl.su. 21600 IN CNAME whitby.tvl.su. +todo.tvl.su. 21600 IN CNAME whitby.tvl.su. + +;; Google Workspaces domain verification +tvl.su. 21600 IN TXT "google-site-verification=3ksTBzFK3lZlzD3ddBfpaHs9qasfAiYBmvbW2T_ejH4" + +;; Google Workspaces email configuration +tvl.su. 21600 IN MX 1 aspmx.l.google.com. +tvl.su. 21600 IN MX 5 alt1.aspmx.l.google.com. +tvl.su. 21600 IN MX 5 alt2.aspmx.l.google.com. +tvl.su. 21600 IN MX 10 alt3.aspmx.l.google.com. +tvl.su. 21600 IN MX 10 alt4.aspmx.l.google.com. +tvl.su. 21600 IN TXT "v=spf1 include:_spf.google.com ~all" +google._domainkey.tvl.su. 21600 IN TXT ("v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAlqCbnGa8oPwrudJK60l6MJj3NBnwj8wAPXNGtYy2SXrOBi7FT+ySwW7ATpfv6Xq9zGDUWJsENPUlFmvDiUs7Qi4scnNvSO1L+sDseB9/q1m3gMFVnTuieDO/" "T+KKkg0+uYgMM7YX5PahsAAJJ+EMb/r4afl3tcBMPR64VveKQ0hiSHA4zIYPsB9FB+b8S5C46uyY0r6WR7IzGjq2Gzb1do0kxvaKItTITWLSImcUu5ZZuXOUKJb441frVBWur5lXaYuedkxb1IRTTK0V/mBODE1D7k73MxGrqlzaMPdCqz+c3hRE18WVUkBTYjANVXDrs3yzBBVxaIAeu++vkO6BvQIDAQAB") + +;; Google Workspaces site aliases +docs.tvl.su. 21600 IN CNAME ghs.googlehosted.com. +groups.tvl.su. 21600 IN CNAME ghs.googlehosted.com. +mail.tvl.su. 21600 IN CNAME ghs.googlehosted.com. diff --git a/ops/gerrit-tvl/HttpModule.java b/ops/gerrit-tvl/HttpModule.java new file mode 100644 index 000000000000..6d785c0817fc --- /dev/null +++ b/ops/gerrit-tvl/HttpModule.java @@ -0,0 +1,14 @@ +package su.tvl.gerrit; + +import com.google.gerrit.extensions.registration.DynamicSet; +import com.google.gerrit.extensions.webui.JavaScriptPlugin; +import com.google.gerrit.extensions.webui.WebUiPlugin; +import com.google.inject.servlet.ServletModule; + +public final class HttpModule extends ServletModule { + + @Override + protected void configureServlets() { + DynamicSet.bind(binder(), WebUiPlugin.class).toInstance(new JavaScriptPlugin("tvl.js")); + } +} diff --git a/ops/gerrit-tvl/MANIFEST.MF b/ops/gerrit-tvl/MANIFEST.MF new file mode 100644 index 000000000000..bfe4eedeb6ac --- /dev/null +++ b/ops/gerrit-tvl/MANIFEST.MF @@ -0,0 +1,2 @@ +Gerrit-HttpModule: su.tvl.gerrit.HttpModule +Gerrit-PluginName: tvl diff --git a/ops/gerrit-tvl/README.md b/ops/gerrit-tvl/README.md new file mode 100644 index 000000000000..1b88600f197f --- /dev/null +++ b/ops/gerrit-tvl/README.md @@ -0,0 +1,6 @@ +# gerrit-tvl + +A Gerrit plugin that does TVL-specific things. + +You probably want to take inspiration from this rather than using it directly, +as it has a variety of TVL-ish assumptions baked into it. diff --git a/ops/gerrit-tvl/default.nix b/ops/gerrit-tvl/default.nix new file mode 100644 index 000000000000..f3bec7a3a242 --- /dev/null +++ b/ops/gerrit-tvl/default.nix @@ -0,0 +1,33 @@ +{ depot, pkgs, lib, ... }: + +let + classPath = lib.concatStringsSep ":" [ + "${depot.third_party.gerrit}/share/api/extension-api_deploy.jar" + ]; +in +pkgs.stdenvNoCC.mkDerivation rec { + name = "${pname}-${version}.jar"; + pname = "gerrit-tvl"; + version = "0.0.1"; + + src = ./.; + + nativeBuildInputs = with pkgs; [ + jdk + ]; + + buildPhase = '' + mkdir $NIX_BUILD_TOP/build + + # Build Java components. + export JAVAC="javac -cp ${classPath} -d $NIX_BUILD_TOP/build --release 11" + $JAVAC ./HttpModule.java + + # Install static files. + cp -R static $NIX_BUILD_TOP/build/static + ''; + + installPhase = '' + jar --create --file $out --manifest $src/MANIFEST.MF -C $NIX_BUILD_TOP/build . + ''; +} diff --git a/ops/gerrit-tvl/static/tvl.js b/ops/gerrit-tvl/static/tvl.js new file mode 100644 index 000000000000..470eb79d6689 --- /dev/null +++ b/ops/gerrit-tvl/static/tvl.js @@ -0,0 +1,134 @@ +// vim: set noai ts=2 sw=2 et: */ + +// This is a read-only Buildkite token: it was generated by lukegb@, and has +// read_builds, read_build_logs, and read_pipelines permissions. +const BUILDKITE_TOKEN = 'a150658fb61062e432f13a032962d70fa9352088'; + +function encodeParams(p) { + const pieces = []; + for (let k of Object.getOwnPropertyNames(p)) { + pieces.push(`${encodeURIComponent(k)}=${encodeURIComponent(p[k])}`); + } + return pieces.join('&'); +} + +function formatDuration(from, to) { + const millisecondsTook = Math.floor(to.valueOf() - from.valueOf()); + if (millisecondsTook < 2000) return `${millisecondsTook} ms`; + const secondsTook = Math.floor(millisecondsTook / 1000); + if (secondsTook < 100) return `${secondsTook} seconds`; + const minutesTook = Math.floor(secondsTook / 60); + if (minutesTook < 60) return `${minutesTook} minutes`; + const hoursTook = Math.floor(minutesTook / 60); + const minutesRemainder = minutesTook - (hoursTook * 60); + return `${hoursTook}hr ${minutesRemainder}min`; +} + +const tvlChecksProvider = { + async fetch(change) { + let {changeNumber, patchsetNumber, repo} = change; + + const experiments = window.ENABLED_EXPERIMENTS || []; + if (experiments.includes("UiFeature__tvl_check_debug")) { + changeNumber = 2872; + patchsetNumber = 4; + repo = 'depot'; + } + + if (repo !== 'depot') { + // We only handle TVL's depot at the moment. + return {responseCode: 'OK'}; + } + + const params = { + // besadii uses the patchset ref as the branch name. + branch: `refs/changes/${changeNumber.toString().slice(-2)}/${changeNumber}/${patchsetNumber}`, + }; + const url = `https://api.buildkite.com/v2/organizations/tvl/pipelines/depot/builds?${encodeParams(params)}`; + const resp = await fetch(url, { + headers: { + Authorization: `Bearer ${BUILDKITE_TOKEN}`, + }, + }); + const respJSON = await resp.json(); + + const runs = []; + for (let i = 0; i < respJSON.length; i++) { + const attempt = respJSON.length - i; + const build = respJSON[i]; + + for (let job of build.jobs) { + // TODO(lukegb): add the ability to retry these (sometimes whitby runs out of disk...) + const checkRun = { + attempt: attempt, + externalId: job.id, + checkName: job.name, + checkDescription: job.command, + checkLink: job.web_url, + status: { + 'running': 'RUNNING', + 'scheduled': 'RUNNABLE', + 'passed': 'COMPLETED', + 'failed': 'COMPLETED', + 'blocked': 'RUNNABLE', + 'canceled': 'COMPLETED', + 'canceling': 'RUNNING', + 'skipped': 'COMPLETED', + 'not_run': 'COMPLETED', + }[job.state], + labelName: 'Verified', + }; + if (job.scheduled_at) { + checkRun.scheduledTimestamp = new Date(job.scheduled_at); + } + if (job.started_at) { + checkRun.startedTimestamp = new Date(job.started_at); + } + if (job.finished_at) { + checkRun.finishedTimestamp = new Date(job.finished_at); + } + + let statusDescription = job.state; + if (checkRun.startedTimestamp && checkRun.finishedTimestamp) { + statusDescription = `${statusDescription} in ${formatDuration(checkRun.startedTimestamp, checkRun.finishedTimestamp)}`; + } else if (checkRun.startedTimestamp) { + statusDescription = `${statusDescription} for ${formatDuration(checkRun.startedTimestamp, new Date())}`; + } else if (checkRun.scheduledTimestamp) { + statusDescription = `${statusDescription} for ${formatDuration(checkRun.scheduledTimestamp, new Date())}`; + } + checkRun.statusDescription = statusDescription; + + if (job.state === 'failed') { + const result = { + // TODO(lukegb): get the log as the message here (the Gerrit + // implementation doesn't yet seem to support newlines in message + // strings...) + links: [{ + url: job.web_url, + tooltip: "Buildkite", + primary: true, + icon: 'EXTERNAL', + }], + category: 'ERROR', + summary: `${job.command} failed`, + }; + checkRun.results = [result]; + } + + console.log(checkRun); + runs.push(checkRun); + } + } + + return { + responseCode: 'OK', + runs: runs, + }; + }, +}; + +Gerrit.install(plugin => { + console.log('TVL plugin initialising'); + + plugin.checks().register(tvlChecksProvider); +}); diff --git a/ops/journaldriver/.gitignore b/ops/journaldriver/.gitignore new file mode 100644 index 000000000000..29e65519ba35 --- /dev/null +++ b/ops/journaldriver/.gitignore @@ -0,0 +1,3 @@ +result +/target +**/*.rs.bk diff --git a/ops/journaldriver/Cargo.lock b/ops/journaldriver/Cargo.lock new file mode 100644 index 000000000000..f7c697068f13 --- /dev/null +++ b/ops/journaldriver/Cargo.lock @@ -0,0 +1,818 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "aho-corasick" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ascii" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "atty" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "backtrace-sys" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "base64" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "chrono" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "chunked_transfer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cookie" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "core-foundation" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "core-foundation-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cstr-argument" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "failure_derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "humantime" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "itoa" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "journaldriver" +version = "1.1.0" +dependencies = [ + "chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "medallion 2.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "systemd 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ureq 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "lazy_static" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.43" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libsystemd-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "medallion" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memchr" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "native-tls" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)", + "schannel 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-integer" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num-traits" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "openssl" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "openssl-sys" +version = "0.9.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pkg-config" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "proc-macro2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "qstring" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "quote" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand_core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_syscall" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "redox_termios" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ryu" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "safemem" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "schannel" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "security-framework" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "security-framework-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde_derive" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serde_json" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.14.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "synstructure" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "systemd" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cstr-argument 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "libsystemd-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-cstr 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tempfile" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termcolor" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "termion" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "thread_local" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ucd-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ureq" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ascii 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)", + "chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cookie 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "qstring 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)", + "url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", + "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-cstr" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "utf8-ranges" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vcpkg" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "wincolor" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum aho-corasick 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "68f56c7353e5a9547cbd76ed90f7bb5ffc3ba09d4ea9bd1d8c06c8b1142eeb5a" +"checksum ascii 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a5fc969a8ce2c9c0c4b0429bb8431544f6658283c8326ba5ff8c762b75369335" +"checksum atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9a7d5b8723950951411ee34d271d99dddcc2035a16ab25310ea2c8cfd4369652" +"checksum backtrace 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "89a47830402e9981c5c41223151efcced65a0510c13097c769cede7efb34782a" +"checksum backtrace-sys 0.1.24 (registry+https://github.com/rust-lang/crates.io-index)" = "c66d56ac8dabd07f6aacdaf633f4b8262f5b3601a810a0dcddffd5c22c69daa0" +"checksum base64 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +"checksum bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "228047a76f468627ca71776ecdebd732a3423081fcf5125585bcd7c49886ce12" +"checksum byteorder 1.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "90492c5858dd7d2e78691cfb89f90d273a2800fc11d98f60786e5d87e2f83781" +"checksum cc 1.0.25 (registry+https://github.com/rust-lang/crates.io-index)" = "f159dfd43363c4d08055a07703eb7a3406b0dac4d0584d96965a3262db3c9d16" +"checksum cfg-if 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0c4e7bb64a8ebb0d856483e1e682ea3422f883c5f5615a90d51a2c82fe87fdd3" +"checksum chrono 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "45912881121cb26fad7c38c17ba7daa18764771836b34fab7d3fbd93ed633878" +"checksum chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87" +"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +"checksum cookie 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1465f8134efa296b4c19db34d909637cb2bf0f7aaf21299e23e18fa29ac557cf" +"checksum core-foundation 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "286e0b41c3a20da26536c6000a280585d519fd07b3956b43aed8a79e9edce980" +"checksum core-foundation-sys 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "716c271e8613ace48344f723b60b900a93150271e5be206212d052bbc0883efa" +"checksum cstr-argument 0.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "514570a4b719329df37f93448a70df2baac553020d0eb43a8dfa9c1f5ba7b658" +"checksum env_logger 0.5.13 (registry+https://github.com/rust-lang/crates.io-index)" = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" +"checksum failure 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7efb22686e4a466b1ec1a15c2898f91fa9cb340452496dca654032de20ff95b9" +"checksum failure_derive 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "946d0e98a50d9831f5d589038d2ca7f8f455b1c21028c0db0e84116a12696426" +"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +"checksum foreign-types-shared 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +"checksum humantime 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0484fda3e7007f2a4a0d9c3a703ca38c71c54c55602ce4660c419fd32e188c9e" +"checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +"checksum itoa 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "1306f3464951f30e30d12373d31c79fbd52d236e5e896fd92f96ec7babbbe60b" +"checksum lazy_static 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca488b89a5657b0a2ecd45b95609b3e848cf1755da332a0da46e2b2b1cb371a7" +"checksum libc 0.2.43 (registry+https://github.com/rust-lang/crates.io-index)" = "76e3a3ef172f1a0b9a9ff0dd1491ae5e6c948b94479a3021819ba7d860c8645d" +"checksum libsystemd-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e751b723417158e0949ba470bee4affd6f1dd6b67622b5240d79186631b6a0d9" +"checksum log 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fcce5fa49cc693c312001daf1d13411c4a5283796bac1084299ea3e567113f" +"checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +"checksum medallion 2.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b2e6f0713b388174fc3de9b63a0a63dfcee191a8abc8e06c0a9c6d80821c1891" +"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +"checksum memchr 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4b3629fe9fdbff6daa6c33b90f7c08355c1aca05a3d01fa8063b822fcf185f3b" +"checksum native-tls 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8b0a7bd714e83db15676d31caf968ad7318e9cc35f93c85a90231c8f22867549" +"checksum num-integer 0.1.39 (registry+https://github.com/rust-lang/crates.io-index)" = "e83d528d2677f0518c570baf2b7abdcf0cd2d248860b68507bdcb3e91d4c0cea" +"checksum num-traits 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0b3a5d7cc97d6d30d8b9bc8fa19bf45349ffe46241e8816f50f62f6d6aaabee1" +"checksum openssl 0.10.12 (registry+https://github.com/rust-lang/crates.io-index)" = "5e2e79eede055813a3ac52fb3915caf8e1c9da2dec1587871aec9f6f7b48508d" +"checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" +"checksum openssl-sys 0.9.36 (registry+https://github.com/rust-lang/crates.io-index)" = "409d77eeb492a1aebd6eb322b2ee72ff7c7496b4434d98b3bf8be038755de65e" +"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" +"checksum pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "676e8eb2b1b4c9043511a9b7bea0915320d7e502b0a079fb03f9635a5252b18c" +"checksum proc-macro2 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "3d7b7eaaa90b4a90a932a9ea6666c95a389e424eff347f0f793979289429feee" +"checksum qstring 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "545ec057a36a93e25fb5883baed912e4984af4e2543bbf0e3463d962e0408469" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" +"checksum quote 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "dd636425967c33af890042c483632d33fa7a18f19ad1d7ea72e8998c6ef8dea5" +"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c" +"checksum rand_core 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1961a422c4d189dfb50ffa9320bf1f2a9bd54ecb92792fb9477f99a1045f3372" +"checksum rand_core 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0905b6b7079ec73b314d4c748701f6931eb79fd97c668caa3f1899b22b32c6db" +"checksum redox_syscall 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "c214e91d3ecf43e9a4e41e578973adeb14b474f2bee858742d127af75a0112b1" +"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" +"checksum regex 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2069749032ea3ec200ca51e4a31df41759190a88edca0d2d86ee8bedf7073341" +"checksum regex-syntax 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "747ba3b235651f6e2f67dfa8bcdcd073ddb7c243cb21c442fc12395dfcac212d" +"checksum remove_dir_all 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3488ba1b9a2084d38645c4c08276a1752dcbf2c7130d74f1569681ad5d2799c5" +"checksum rustc-demangle 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)" = "bcfe5b13211b4d78e5c2cadfebd7769197d95c639c35a50057eb4c05de811395" +"checksum ryu 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "7153dd96dade874ab973e098cb62fcdbb89a03682e46b144fd09550998d4a4a7" +"checksum safemem 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8dca453248a96cb0749e36ccdfe2b0b4e54a61bfef89fb97ec621eb8e0a93dd9" +"checksum schannel 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "0e1a231dc10abf6749cfa5d7767f25888d484201accbd919b66ab5413c502d56" +"checksum security-framework 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "697d3f3c23a618272ead9e1fb259c1411102b31c6af8b93f1d64cca9c3b0e8e0" +"checksum security-framework-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ab01dfbe5756785b5b4d46e0289e5a18071dfa9a7c2b24213ea00b9ef9b665bf" +"checksum serde 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "84257ccd054dc351472528c8587b4de2dbf0dc0fe2e634030c1a90bfdacebaa9" +"checksum serde_derive 1.0.79 (registry+https://github.com/rust-lang/crates.io-index)" = "31569d901045afbff7a9479f793177fe9259819aff10ab4f89ef69bbc5f567fe" +"checksum serde_json 1.0.32 (registry+https://github.com/rust-lang/crates.io-index)" = "43344e7ce05d0d8280c5940cabb4964bea626aa58b1ec0e8c73fa2a8512a38ce" +"checksum syn 0.14.9 (registry+https://github.com/rust-lang/crates.io-index)" = "261ae9ecaa397c42b960649561949d69311f08eeaea86a65696e6e46517cf741" +"checksum syn 0.15.8 (registry+https://github.com/rust-lang/crates.io-index)" = "356d1c5043597c40489e9af2d2498c7fefc33e99b7d75b43be336c8a59b3e45e" +"checksum synstructure 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "85bb9b7550d063ea184027c9b8c20ac167cd36d3e06b3a40bceb9d746dc1a7b7" +"checksum systemd 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1b62a732355787f960c25536210ae0a981aca2e5dae9dab8491bdae39613ce48" +"checksum tempfile 3.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "55c1195ef8513f3273d55ff59fe5da6940287a0d7a98331254397f464833675b" +"checksum termcolor 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4096add70612622289f2fdcdbd5086dc81c1e2675e6ae58d6c4f62a16c6d7f2f" +"checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" +"checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" +"checksum time 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "d825be0eb33fda1a7e68012d51e9c7f451dc1a69391e7fdc197060bb8c56667b" +"checksum ucd-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd2be2d6639d0f8fe6cdda291ad456e23629558d466e2789d2c3e9892bda285d" +"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +"checksum unicode-normalization 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "6a0180bc61fc5a987082bfa111f4cc95c4caff7f9799f3e46df09163a937aa25" +"checksum unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" +"checksum ureq 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5f3f941c0434783c82e46d30508834be5f3c1f2c85dd1b98f0681984c7be8e03" +"checksum url 1.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2a321979c09843d272956e73700d12c4e7d3d92b2ee112b31548aef0d4efc5a6" +"checksum utf8-cstr 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "55bcbb425141152b10d5693095950b51c3745d019363fc2929ffd8f61449b628" +"checksum utf8-ranges 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fd70f467df6810094968e2fce0ee1bd0e87157aceb026a8c083bcf5e25b9efe4" +"checksum vcpkg 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "def296d3eb3b12371b2c7d0e83bfe1403e4db2d7a0bba324a12b21c4ee13143d" +"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" +"checksum winapi 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92c1eb33641e276cfa214a0522acad57be5c56b10cb348b3c5117db75f3ac4b0" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-util 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "afc5508759c5bf4285e61feb862b6083c8480aec864fa17a81fdec6f69b461ab" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" diff --git a/ops/journaldriver/Cargo.toml b/ops/journaldriver/Cargo.toml new file mode 100644 index 000000000000..248b22807fe1 --- /dev/null +++ b/ops/journaldriver/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "journaldriver" +version = "1.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +license = "GPL-3.0-or-later" + +[dependencies] +chrono = { version = "0.4", features = [ "serde" ]} +env_logger = "0.5" +failure = "0.1" +lazy_static = "1.0" +log = "0.4" +medallion = "2.2" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +systemd = "0.3" +ureq = { version = "0.6.2", features = [ "json" ]} + +[build-dependencies] +pkg-config = "0.3" diff --git a/ops/journaldriver/README.md b/ops/journaldriver/README.md new file mode 100644 index 000000000000..4dc9de0f617b --- /dev/null +++ b/ops/journaldriver/README.md @@ -0,0 +1,152 @@ +journaldriver +============= + +This is a small daemon used to forward logs from `journald` (systemd's +logging service) to [Stackdriver Logging][]. + +Many existing log services are written in inefficient dynamic +languages with error-prone "cover every possible use-case" +configuration. `journaldriver` instead aims to fit a specific use-case +very well, instead of covering every possible logging setup. + +`journaldriver` can be run on GCP-instances with no additional +configuration as authentication tokens are retrieved from the +[metadata server][]. + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Features](#features) +- [Usage on Google Cloud Platform](#usage-on-google-cloud-platform) +- [Usage outside of Google Cloud Platform](#usage-outside-of-google-cloud-platform) +- [Log levels / severities / priorities](#log-levels--severities--priorities) +- [NixOS module](#nixos-module) +- [Stackdriver Error Reporting](#stackdriver-error-reporting) + +<!-- markdown-toc end --> + +# Features + +* `journaldriver` persists the last forwarded position in the journal + and will resume forwarding at the same position after a restart +* `journaldriver` will recognise log entries in JSON format and + forward them appropriately to make structured log entries available + in Stackdriver +* `journaldriver` can be used outside of GCP by configuring static + credentials +* `journaldriver` will recognise journald's log priority levels and + convert them into equivalent Stackdriver log severity levels + +# Usage on Google Cloud Platform + +`journaldriver` does not require any configuration when running on GCP +instances. + +1. Install `journaldriver` on the instance from which you wish to + forward logs. + +2. Ensure that the instance has the appropriate permissions to write + to Stackdriver. Google continously changes how IAM is implemented + on GCP, so you will have to refer to [Google's documentation][]. + + By default instances have the required permissions if Stackdriver + Logging support is enabled in the project. + +3. Start `journaldriver`, for example via `systemd`. + +# Usage outside of Google Cloud Platform + +When running outside of GCP, the following extra steps need to be +performed: + +1. Create a Google Cloud Platform service account with the "Log + Writer" role and download its private key in JSON-format. +2. When starting `journaldriver`, configure the following environment + variables: + + * `GOOGLE_CLOUD_PROJECT`: Name of the GCP project to which logs + should be written. + * `GOOGLE_APPLICATION_CREDENTIALS`: Filesystem path to the + JSON-file containing the service account's private key. + * `LOG_STREAM`: Name of the target log stream in Stackdriver Logging. + This will be automatically created if it does not yet exist. + * `LOG_NAME`: Name of the target log to write to. This defaults to + `journaldriver` if unset, but it is recommended to - for + example - set it to the machine hostname. + +# Log levels / severities / priorities + +`journaldriver` recognises [journald's priorities][] and converts them +into [equivalent severities][] in Stackdriver. Both sets of values +correspond to standard `syslog` priorities. + +The easiest way to emit log messages with priorites from an +application is to use [priority prefixes][], which are compatible with +structured log messages. + +For example, to emit a simple warning message (structured and +unstructured): + +``` +$ echo '<4>{"fnord":true, "msg":"structured log (warning)"}' | systemd-cat +$ echo '<4>unstructured log (warning)' | systemd-cat +``` + +# NixOS module + +The NixOS package repository [contains a module][] for setting up +`journaldriver` on NixOS machines. NixOS by default uses `systemd` for +service management and `journald` for logging, which means that log +output from most services will be captured automatically. + +On a GCP instance the only required option is this: + +```nix +services.journaldriver.enable = true; +``` + +When running outside of GCP, the configuration looks as follows: + +```nix +services.journaldriver = { + enable = true; + logStream = "prod-environment"; + logName = "hostname"; + googleCloudProject = "gcp-project-name"; + applicationCredentials = keyFile; +}; +``` + +**Note**: The `journaldriver`-module is included in stable releases of +NixOS since NixOS 18.09. + +# Stackdriver Error Reporting + +The [Stackdriver Error Reporting][] service of Google's monitoring +toolbox supports automatically detecting and correlating errors from +log entries. + +To use this functionality log messages must be logged in the expected +[log format][]. + +*Note*: Reporting errors from non-GCP instances requires that the +`LOG_STREAM` environment variable is set to the special value +`global`. + +This value changes the monitored resource descriptor from a log stream +to the project-global stream. Due to a limitation in Stackdriver Error +Reporting, this is the only way to correctly ingest errors from +non-GCP machines. Please see [issue #4][] for more information about +this. + +[Stackdriver Logging]: https://cloud.google.com/logging/ +[metadata server]: https://cloud.google.com/compute/docs/storing-retrieving-metadata +[Google's documentation]: https://cloud.google.com/logging/docs/access-control +[NixOS]: https://nixos.org/ +[contains a module]: https://github.com/NixOS/nixpkgs/pull/42134 +[journald's priorities]: http://0pointer.de/public/systemd-man/sd-daemon.html +[equivalent severities]: https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#logseverity +[priority prefixes]: http://0pointer.de/public/systemd-man/sd-daemon.html +[Stackdriver Error Reporting]: https://cloud.google.com/error-reporting/ +[log format]: https://cloud.google.com/error-reporting/docs/formatting-error-messages +[issue #4]: https://github.com/tazjin/journaldriver/issues/4 diff --git a/ops/journaldriver/build.rs b/ops/journaldriver/build.rs new file mode 100644 index 000000000000..d64c82a88a71 --- /dev/null +++ b/ops/journaldriver/build.rs @@ -0,0 +1,6 @@ +extern crate pkg_config; + +fn main() { + pkg_config::probe_library("libsystemd") + .expect("Could not probe libsystemd"); +} diff --git a/ops/journaldriver/default.nix b/ops/journaldriver/default.nix new file mode 100644 index 000000000000..d2413e74cc92 --- /dev/null +++ b/ops/journaldriver/default.nix @@ -0,0 +1,9 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; + + buildInputs = with pkgs; [ + pkgconfig openssl systemd.dev + ]; +} diff --git a/ops/journaldriver/src/main.rs b/ops/journaldriver/src/main.rs new file mode 100644 index 000000000000..9886af1c3696 --- /dev/null +++ b/ops/journaldriver/src/main.rs @@ -0,0 +1,665 @@ +// Copyright (C) 2018 Vincent Ambo <mail@tazj.in> +// +// journaldriver is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +//! This file implements journaldriver, a small application that +//! forwards logs from journald (systemd's log facility) to +//! Stackdriver Logging. +//! +//! Log entries are read continously from journald and are forwarded +//! to Stackdriver in batches. +//! +//! Stackdriver Logging has a concept of monitored resources. In the +//! simplest case this monitored resource will be the GCE instance on +//! which journaldriver is running. +//! +//! Information about the instance, the project and required security +//! credentials are retrieved from Google's metadata instance on GCP. +//! +//! To run journaldriver on non-GCP machines, users must specify the +//! `GOOGLE_APPLICATION_CREDENTIALS`, `GOOGLE_CLOUD_PROJECT` and +//! `LOG_NAME` environment variables. + +#[macro_use] extern crate failure; +#[macro_use] extern crate log; +#[macro_use] extern crate serde_derive; +#[macro_use] extern crate serde_json; +#[macro_use] extern crate lazy_static; + +extern crate chrono; +extern crate env_logger; +extern crate medallion; +extern crate serde; +extern crate systemd; +extern crate ureq; + +use chrono::offset::LocalResult; +use chrono::prelude::{DateTime, TimeZone, Utc}; +use failure::ResultExt; +use serde_json::{from_str, Value}; +use std::env; +use std::fs::{self, File, rename}; +use std::io::{self, Read, ErrorKind, Write}; +use std::mem; +use std::path::PathBuf; +use std::process; +use std::time::{Duration, Instant}; +use systemd::journal::{Journal, JournalFiles, JournalRecord, JournalSeek}; + +#[cfg(test)] +mod tests; + +const LOGGING_SERVICE: &str = "https://logging.googleapis.com/google.logging.v2.LoggingServiceV2"; +const ENTRIES_WRITE_URL: &str = "https://logging.googleapis.com/v2/entries:write"; +const METADATA_TOKEN_URL: &str = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; +const METADATA_ID_URL: &str = "http://metadata.google.internal/computeMetadata/v1/instance/id"; +const METADATA_ZONE_URL: &str = "http://metadata.google.internal/computeMetadata/v1/instance/zone"; +const METADATA_PROJECT_URL: &str = "http://metadata.google.internal/computeMetadata/v1/project/project-id"; + +/// Convenience type alias for results using failure's `Error` type. +type Result<T> = std::result::Result<T, failure::Error>; + +/// Representation of static service account credentials for GCP. +#[derive(Debug, Deserialize)] +struct Credentials { + /// PEM encoded private key + private_key: String, + + /// `kid` of this private key + private_key_id: String, + + /// "email" address of the service account + client_email: String, +} + +lazy_static! { + /// ID of the GCP project to which to send logs. + static ref PROJECT_ID: String = get_project_id(); + + /// Name of the log to write to (this should only be manually + /// configured if not running on GCP): + static ref LOG_NAME: String = env::var("LOG_NAME") + .unwrap_or("journaldriver".into()); + + /// Service account credentials (if configured) + static ref SERVICE_ACCOUNT_CREDENTIALS: Option<Credentials> = + env::var("GOOGLE_APPLICATION_CREDENTIALS").ok() + .and_then(|path| File::open(path).ok()) + .and_then(|file| serde_json::from_reader(file).ok()); + + /// Descriptor of the currently monitored instance. Refer to the + /// documentation of `determine_monitored_resource` for more + /// information. + static ref MONITORED_RESOURCE: Value = determine_monitored_resource(); + + /// Path to the directory in which journaldriver should persist + /// its cursor state. + static ref CURSOR_DIR: PathBuf = env::var("CURSOR_POSITION_DIR") + .unwrap_or("/var/lib/journaldriver".into()) + .into(); + + /// Path to the cursor position file itself. + static ref CURSOR_FILE: PathBuf = { + let mut path = CURSOR_DIR.clone(); + path.push("cursor.pos"); + path + }; + + /// Path to the temporary file used for cursor position writes. + static ref CURSOR_TMP_FILE: PathBuf = { + let mut path = CURSOR_DIR.clone(); + path.push("cursor.tmp"); + path + }; +} + +/// Convenience helper for retrieving values from the metadata server. +fn get_metadata(url: &str) -> Result<String> { + let response = ureq::get(url) + .set("Metadata-Flavor", "Google") + .timeout_connect(5000) + .timeout_read(5000) + .call(); + + if response.ok() { + // Whitespace is trimmed to remove newlines from responses. + let body = response.into_string() + .context("Failed to decode metadata response")? + .trim().to_string(); + + Ok(body) + } else { + let status = response.status_line().to_string(); + let body = response.into_string() + .unwrap_or_else(|e| format!("Metadata body error: {}", e)); + bail!("Metadata failure: {} ({})", body, status) + } +} + +/// Convenience helper for determining the project ID. +fn get_project_id() -> String { + env::var("GOOGLE_CLOUD_PROJECT") + .map_err(Into::into) + .or_else(|_: failure::Error| get_metadata(METADATA_PROJECT_URL)) + .expect("Could not determine project ID") +} + +/// Determines the monitored resource descriptor used in Stackdriver +/// logs. On GCP this will be set to the instance ID as returned by +/// the metadata server. +/// +/// On non-GCP machines the value is determined by using the +/// `GOOGLE_CLOUD_PROJECT` and `LOG_STREAM` environment variables. +/// +/// [issue #4]: https://github.com/tazjin/journaldriver/issues/4 +fn determine_monitored_resource() -> Value { + if let Ok(log) = env::var("LOG_STREAM") { + // The special value `global` is recognised as a log stream name that + // results in a `global`-type resource descriptor. This is useful in + // cases where Stackdriver Error Reporting is intended to be used on + // a non-GCE instance. See [issue #4][] for details. + if log == "global" { + return json!({ + "type": "global", + "labels": { + "project_id": PROJECT_ID.as_str(), + } + }); + } + + json!({ + "type": "logging_log", + "labels": { + "project_id": PROJECT_ID.as_str(), + "name": log, + } + }) + } else { + let instance_id = get_metadata(METADATA_ID_URL) + .expect("Could not determine instance ID"); + + let zone = get_metadata(METADATA_ZONE_URL) + .expect("Could not determine instance zone"); + + json!({ + "type": "gce_instance", + "labels": { + "project_id": PROJECT_ID.as_str(), + "instance_id": instance_id, + "zone": zone, + } + }) + } +} + +/// Represents the response returned by the metadata server's token +/// endpoint. The token is normally valid for an hour. +#[derive(Deserialize)] +struct TokenResponse { + expires_in: u64, + access_token: String, +} + +/// Struct used to store a token together with a sensible +/// representation of when it expires. +struct Token { + token: String, + fetched_at: Instant, + expires: Duration, +} + +impl Token { + /// Does this token need to be renewed? + fn is_expired(&self) -> bool { + self.fetched_at.elapsed() > self.expires + } +} + +/// Retrieves a token from the GCP metadata service. Retrieving these +/// tokens requires no additional authentication. +fn get_metadata_token() -> Result<Token> { + let body = get_metadata(METADATA_TOKEN_URL)?; + let token: TokenResponse = from_str(&body)?; + + debug!("Fetched new token from metadata service"); + + Ok(Token { + fetched_at: Instant::now(), + expires: Duration::from_secs(token.expires_in / 2), + token: token.access_token, + }) +} + +/// Signs a token using static client credentials configured for a +/// service account. This service account must have been given the +/// `Log Writer` role in Google Cloud IAM. +/// +/// The process for creating and signing these tokens is described +/// here: +/// +/// https://developers.google.com/identity/protocols/OAuth2ServiceAccount#jwt-auth +fn sign_service_account_token(credentials: &Credentials) -> Result<Token> { + use medallion::{Algorithm, Header, Payload}; + + let iat = Utc::now(); + let exp = iat.checked_add_signed(chrono::Duration::seconds(3600)) + .ok_or_else(|| format_err!("Failed to calculate token expiry"))?; + + let header = Header { + alg: Algorithm::RS256, + headers: Some(json!({ + "kid": credentials.private_key_id, + })), + }; + + let payload: Payload<()> = Payload { + iss: Some(credentials.client_email.clone()), + sub: Some(credentials.client_email.clone()), + aud: Some(LOGGING_SERVICE.to_string()), + iat: Some(iat.timestamp() as u64), + exp: Some(exp.timestamp() as u64), + ..Default::default() + }; + + let token = medallion::Token::new(header, payload) + .sign(credentials.private_key.as_bytes()) + .context("Signing service account token failed")?; + + debug!("Signed new service account token"); + + Ok(Token { + token, + fetched_at: Instant::now(), + expires: Duration::from_secs(3000), + }) +} + +/// Retrieve the authentication token either by using static client +/// credentials, or by talking to the metadata server. +/// +/// Which behaviour is used is controlled by the environment variable +/// `GOOGLE_APPLICATION_CREDENTIALS`, which should be configured to +/// point at a JSON private key file if service account authentication +/// is to be used. +fn get_token() -> Result<Token> { + if let Some(credentials) = SERVICE_ACCOUNT_CREDENTIALS.as_ref() { + sign_service_account_token(credentials) + } else { + get_metadata_token() + } +} + +/// This structure represents the different types of payloads +/// supported by journaldriver. +/// +/// Currently log entries can either contain plain text messages or +/// structured payloads in JSON-format. +#[derive(Debug, Serialize, PartialEq)] +#[serde(untagged)] +enum Payload { + TextPayload { + #[serde(rename = "textPayload")] + text_payload: String, + }, + JsonPayload { + #[serde(rename = "jsonPayload")] + json_payload: Value, + }, +} + +/// Attempt to parse a log message as JSON and return it as a +/// structured payload. If parsing fails, return the entry in plain +/// text format. +fn message_to_payload(message: Option<String>) -> Payload { + match message { + None => Payload::TextPayload { text_payload: "empty log entry".into() }, + Some(text_payload) => { + // Attempt to deserialize the text payload as a generic + // JSON value. + if let Ok(json_payload) = serde_json::from_str::<Value>(&text_payload) { + // If JSON-parsing succeeded on the payload, check + // whether we parsed an object (Stackdriver does not + // expect other types of JSON payload) and return it + // in that case. + if json_payload.is_object() { + return Payload::JsonPayload { json_payload } + } + } + + Payload::TextPayload { text_payload } + } + } +} + +/// Attempt to parse journald's microsecond timestamps into a UTC +/// timestamp. +/// +/// Parse errors are dismissed and returned as empty options: There +/// simply aren't any useful fallback mechanisms other than defaulting +/// to ingestion time for journaldriver's use-case. +fn parse_microseconds(input: String) -> Option<DateTime<Utc>> { + if input.len() != 16 { + return None; + } + + let seconds: i64 = (&input[..10]).parse().ok()?; + let micros: u32 = (&input[10..]).parse().ok()?; + + match Utc.timestamp_opt(seconds, micros * 1000) { + LocalResult::Single(time) => Some(time), + _ => None, + } +} + +/// Converts a journald log message priority to a +/// Stackdriver-compatible severity number. +/// +/// Both Stackdriver and journald specify equivalent +/// severities/priorities. Conveniently, the names are the same. +/// Inconveniently, the numbers are not. +/// +/// For more information on the journald priorities, consult these +/// man-pages: +/// +/// * systemd.journal-fields(7) (section 'PRIORITY') +/// * sd-daemon(3) +/// * systemd.exec(5) (section 'SyslogLevelPrefix') +/// +/// Note that priorities can be logged by applications via the prefix +/// concept described in these man pages, without interfering with +/// structured JSON-payloads. +/// +/// For more information on the Stackdriver severity levels, please +/// consult Google's documentation: +/// +/// https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity +/// +/// Any unknown priority values result in no severity being set. +fn priority_to_severity(priority: String) -> Option<u32> { + match priority.as_ref() { + "0" => Some(800), // emerg + "1" => Some(700), // alert + "2" => Some(600), // crit + "3" => Some(500), // err + "4" => Some(400), // warning + "5" => Some(300), // notice + "6" => Some(200), // info + "7" => Some(100), // debug + _ => None, + } +} + +/// This structure represents a log entry in the format expected by +/// the Stackdriver API. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct LogEntry { + labels: Value, + + #[serde(skip_serializing_if = "Option::is_none")] + timestamp: Option<DateTime<Utc>>, + + #[serde(flatten)] + payload: Payload, + + // https://cloud.google.com/logging/docs/reference/v2/rest/v2/LogEntry#LogSeverity + #[serde(skip_serializing_if = "Option::is_none")] + severity: Option<u32>, +} + +impl From<JournalRecord> for LogEntry { + // Converts from the fields contained in a journald record to the + // representation required by Stackdriver Logging. + // + // The fields are documented in systemd.journal-fields(7). + fn from(mut record: JournalRecord) -> LogEntry { + // The message field is technically just a convention, but + // journald seems to default to it when ingesting unit + // output. + let payload = message_to_payload(record.remove("MESSAGE")); + + // Presumably this is always set, but who can be sure + // about anything in this world. + let hostname = record.remove("_HOSTNAME"); + + // The unit is seemingly missing on kernel entries, but + // present on all others. + let unit = record.remove("_SYSTEMD_UNIT"); + + // The source timestamp (if present) is specified in + // microseconds since epoch. + // + // If it is not present or can not be parsed, journaldriver + // will not send a timestamp for the log entry and it will + // default to the ingestion time. + let timestamp = record + .remove("_SOURCE_REALTIME_TIMESTAMP") + .and_then(parse_microseconds); + + // Journald uses syslogd's concept of priority. No idea if this is + // always present, but it's optional in the Stackdriver API, so we just + // omit it if we can't find or parse it. + let severity = record + .remove("PRIORITY") + .and_then(priority_to_severity); + + LogEntry { + payload, + timestamp, + labels: json!({ + "host": hostname, + "unit": unit.unwrap_or_else(|| "syslog".into()), + }), + severity, + } + } +} + +/// Attempt to read from the journal. If no new entry is present, +/// await the next one up to the specified timeout. +fn receive_next_record(timeout: Duration, journal: &mut Journal) + -> Result<Option<JournalRecord>> { + let next_record = journal.next_record()?; + if next_record.is_some() { + return Ok(next_record); + } + + Ok(journal.await_next_record(Some(timeout))?) +} + +/// This function starts a double-looped, blocking receiver. It will +/// buffer messages for half a second before flushing them to +/// Stackdriver. +fn receiver_loop(mut journal: Journal) -> Result<()> { + let mut token = get_token()?; + + let mut buf: Vec<LogEntry> = Vec::new(); + let iteration = Duration::from_millis(500); + + loop { + trace!("Beginning outer iteration"); + let now = Instant::now(); + + loop { + if now.elapsed() > iteration { + break; + } + + if let Ok(Some(entry)) = receive_next_record(iteration, &mut journal) { + trace!("Received a new entry"); + buf.push(entry.into()); + } + } + + if !buf.is_empty() { + let to_flush = mem::replace(&mut buf, Vec::new()); + flush(&mut token, to_flush, journal.cursor()?)?; + } + + trace!("Done outer iteration"); + } +} + +/// Writes the current cursor into `/var/journaldriver/cursor.pos`. To +/// avoid issues with journaldriver being terminated while the cursor +/// is still being written, this will first write the cursor into a +/// temporary file and then move it. +fn persist_cursor(cursor: String) -> Result<()> { + // This code exists to aid in tracking down if there are other + // causes of issue #2 than what has already been taken care of. + // + // One theory is that journald (or the Rust library to interface + // with it) may occasionally return empty cursor strings. If this + // is ever the case, we would like to know about it. + if cursor.is_empty() { + error!("Received empty journald cursor position, refusing to persist!"); + error!("Please report this message at https://github.com/tazjin/journaldriver/issues/2"); + return Ok(()) + } + + let mut file = File::create(&*CURSOR_TMP_FILE) + .context("Failed to create cursor file")?; + + write!(file, "{}", cursor).context("Failed to write cursor file")?; + + rename(&*CURSOR_TMP_FILE, &*CURSOR_FILE) + .context("Failed to move cursor file") + .map_err(Into::into) +} + +/// Flushes all drained records to Stackdriver. Any Stackdriver +/// message can at most contain 1000 log entries which means they are +/// chunked up here. +/// +/// In some cases large payloads seem to cause errors in Stackdriver - +/// the chunks are therefore made smaller here. +/// +/// If flushing is successful the last cursor position will be +/// persisted to disk. +fn flush(token: &mut Token, + entries: Vec<LogEntry>, + cursor: String) -> Result<()> { + if token.is_expired() { + debug!("Refreshing Google metadata access token"); + let new_token = get_token()?; + *token = new_token; + } + + for chunk in entries.chunks(750) { + let request = prepare_request(chunk); + if let Err(write_error) = write_entries(token, request) { + error!("Failed to write {} entries: {}", chunk.len(), write_error) + } else { + debug!("Wrote {} entries to Stackdriver", chunk.len()) + } + } + + persist_cursor(cursor) +} + +/// Convert a slice of log entries into the format expected by +/// Stackdriver. This format is documented here: +/// +/// https://cloud.google.com/logging/docs/reference/v2/rest/v2/entries/write +fn prepare_request(entries: &[LogEntry]) -> Value { + json!({ + "logName": format!("projects/{}/logs/{}", PROJECT_ID.as_str(), LOG_NAME.as_str()), + "resource": &*MONITORED_RESOURCE, + "entries": entries, + "partialSuccess": true + }) +} + +/// Perform the log entry insertion in Stackdriver Logging. +fn write_entries(token: &Token, request: Value) -> Result<()> { + let response = ureq::post(ENTRIES_WRITE_URL) + .set("Authorization", format!("Bearer {}", token.token).as_str()) + // The timeout values are set relatively high, not because of + // an expectation of Stackdriver being slow but just to + // eventually hit an error case in case of network troubles. + // Presumably no request in a functioning environment will + // ever hit these limits. + .timeout_connect(2000) + .timeout_read(5000) + .send_json(request); + + if response.ok() { + Ok(()) + } else { + let status = response.status_line().to_string(); + let body = response.into_string() + .unwrap_or_else(|_| "no response body".into()); + bail!("Write failure: {} ({})", body, status) + } +} + +/// Attempt to read the initial cursor position from the configured +/// file. If there is no initial cursor position set, read from the +/// tail of the log. +/// +/// The only "acceptable" error when reading the cursor position is +/// the cursor position file not existing, other errors are fatal +/// because they indicate a misconfiguration of journaldriver. +fn initial_cursor() -> Result<JournalSeek> { + let read_result: io::Result<String> = (|| { + let mut contents = String::new(); + let mut file = File::open(&*CURSOR_FILE)?; + file.read_to_string(&mut contents)?; + Ok(contents.trim().into()) + })(); + + match read_result { + Ok(cursor) => Ok(JournalSeek::Cursor { cursor }), + Err(ref err) if err.kind() == ErrorKind::NotFound => { + info!("No previous cursor position, reading from journal tail"); + Ok(JournalSeek::Tail) + }, + Err(err) => { + (Err(err).context("Could not read cursor position"))? + } + } +} + +fn main () { + env_logger::init(); + + // The directory in which cursor positions are persisted should + // have been created: + if !CURSOR_DIR.exists() { + error!("Cursor directory at '{:?}' does not exist", *CURSOR_DIR); + process::exit(1); + } + + let cursor_position_dir = CURSOR_FILE.parent() + .expect("Invalid cursor position file path"); + + fs::create_dir_all(cursor_position_dir) + .expect("Could not create directory to store cursor position in"); + + let mut journal = Journal::open(JournalFiles::All, false, true) + .expect("Failed to open systemd journal"); + + let seek_position = initial_cursor() + .expect("Failed to determine initial cursor position"); + + match journal.seek(seek_position) { + Ok(cursor) => info!("Opened journal at cursor '{}'", cursor), + Err(err) => { + error!("Failed to set initial journal position: {}", err); + process::exit(1) + } + } + + receiver_loop(journal).expect("log receiver encountered an unexpected error"); +} diff --git a/ops/journaldriver/src/tests.rs b/ops/journaldriver/src/tests.rs new file mode 100644 index 000000000000..779add7a7012 --- /dev/null +++ b/ops/journaldriver/src/tests.rs @@ -0,0 +1,95 @@ +use super::*; +use serde_json::to_string; + +#[test] +fn test_text_entry_serialization() { + let entry = LogEntry { + labels: Value::Null, + timestamp: None, + payload: Payload::TextPayload { + text_payload: "test entry".into(), + }, + severity: None, + }; + + let expected = "{\"labels\":null,\"textPayload\":\"test entry\"}"; + let result = to_string(&entry).expect("serialization failed"); + + assert_eq!(expected, result, "Plain text payload should serialize correctly") +} + +#[test] +fn test_json_entry_serialization() { + let entry = LogEntry { + labels: Value::Null, + timestamp: None, + payload: Payload::JsonPayload { + json_payload: json!({ + "message": "JSON test" + }) + }, + severity: None, + }; + + let expected = "{\"labels\":null,\"jsonPayload\":{\"message\":\"JSON test\"}}"; + let result = to_string(&entry).expect("serialization failed"); + + assert_eq!(expected, result, "JSOn payload should serialize correctly") +} + +#[test] +fn test_plain_text_payload() { + let message = "plain text payload".into(); + let payload = message_to_payload(Some(message)); + let expected = Payload::TextPayload { + text_payload: "plain text payload".into(), + }; + + assert_eq!(expected, payload, "Plain text payload should be detected correctly"); +} + +#[test] +fn test_empty_payload() { + let payload = message_to_payload(None); + let expected = Payload::TextPayload { + text_payload: "empty log entry".into(), + }; + + assert_eq!(expected, payload, "Empty payload should be handled correctly"); +} + +#[test] +fn test_json_payload() { + let message = "{\"someKey\":\"someValue\", \"otherKey\": 42}".into(); + let payload = message_to_payload(Some(message)); + let expected = Payload::JsonPayload { + json_payload: json!({ + "someKey": "someValue", + "otherKey": 42 + }) + }; + + assert_eq!(expected, payload, "JSON payload should be detected correctly"); +} + +#[test] +fn test_json_no_object() { + // This message can be parsed as valid JSON, but it is not an + // object - it should be returned as a plain-text payload. + let message = "42".into(); + let payload = message_to_payload(Some(message)); + let expected = Payload::TextPayload { + text_payload: "42".into(), + }; + + assert_eq!(expected, payload, "Non-object JSON payload should be plain text"); +} + +#[test] +fn test_parse_microseconds() { + let input: String = "1529175149291187".into(); + let expected: DateTime<Utc> = "2018-06-16T18:52:29.291187Z" + .to_string().parse().unwrap(); + + assert_eq!(Some(expected), parse_microseconds(input)); +} diff --git a/ops/kontemplate/.gitignore b/ops/kontemplate/.gitignore new file mode 100644 index 000000000000..53a04aab3a89 --- /dev/null +++ b/ops/kontemplate/.gitignore @@ -0,0 +1,2 @@ +.idea/ +release/ diff --git a/ops/kontemplate/LICENSE b/ops/kontemplate/LICENSE new file mode 100644 index 000000000000..94a9ed024d38 --- /dev/null +++ b/ops/kontemplate/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/ops/kontemplate/README.md b/ops/kontemplate/README.md new file mode 100644 index 000000000000..803a1c4f16fb --- /dev/null +++ b/ops/kontemplate/README.md @@ -0,0 +1,185 @@ +Kontemplate - A simple Kubernetes templater +=========================================== + +Kontemplate is a simple CLI tool that can take sets of Kubernetes resource files +with placeholders and insert values per environment. + +This tool was made because in many cases all I want in terms of Kubernetes +configuration is simple value interpolation per environment (i.e. Kubernetes +cluster), but with the same deployment files. + +In my experience this is often enough and more complex solutions such as +[Helm][] are not required. + +Check out a Kontemplate setup example and the feature list below! + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Kontemplate - A simple Kubernetes templater](#kontemplate---a-simple-kubernetes-templater) + - [Features](#features) + - [Example](#example) + - [Installation](#installation) + - [Homebrew](#homebrew) + - [Arch Linux](#arch-linux) + - [Building repeatably from source](#building-repeatably-from-source) + - [Building from source](#building-from-source) + - [Usage](#usage) + - [Contributing](#contributing) + +<!-- markdown-toc end --> + +## Features + +* [Simple, yet powerful templates](docs/templates.md) +* [Clean cluster configuration files](docs/cluster-config.md) +* [Resources organised as simple resource sets](docs/resource-sets.md) +* Integration with pass +* Integration with kubectl + +## Example + +Kontemplate lets you describe resources as you normally would in a simple folder structure: + +``` +. +├── prod-cluster.yaml +└── some-api + ├── deployment.yaml + └── service.yaml +``` + +This example has all resources belonging to `some-api` (no file naming conventions enforced at all!) in the `some-api` +folder and the configuration for the cluster `prod-cluster` in the corresponding file. + +Lets take a short look at `prod-cluster.yaml`: + +```yaml +--- +context: k8s.prod.mydomain.com +global: + globalVar: lizards +include: + - name: some-api + values: + version: 1.0-0e6884d + importantFeature: true + apiPort: 4567 +``` + +Those values are then templated into the resource files of `some-api`. That's it! + +You can also set up more complicated folder structures for organisation, for example: + +``` +. +├── api +│ ├── image-api +│ │ └── deployment.yaml +│ └── music-api +│ └── deployment.yaml +│ │ └── default.json +├── frontend +│ ├── main-app +│ │ ├── deployment.yaml +│ │ └── service.yaml +│ └── user-page +│ ├── deployment.yaml +│ └── service.yaml +├── prod-cluster.yaml +└── test-cluster.yaml +``` + +And selectively template or apply resources with a command such as +`kontemplate apply test-cluster.yaml --include api --include frontend/user-page` +to only update the `api` resource sets and the `frontend/user-page` resource set. + +## Installation + +It is recommended to install Kontemplate from the [Nix](https://nixos.org/) package set, +where it is available since NixOS 17.09 as `kontemplate`. + +If using Nix is not an option for you, several other methods of installation are +available: + +### Binary releases + +Signed binary releases are available on the [releases page][] for Linux, OS X, FreeBSD and +Windows. + +Releases are signed with the GPG key `DCF34CFAC1AC44B87E26333136EE34814F6D294A`. + +### Building from source + +You can clone Kontemplate either by cloning the full TVL +[depot][https://code.tvl.fyi] or by just cloning the kontemplate +subtree like so: + + git clone https://code.tvl.fyi/depot.git:/ops/kontemplate.git + +The `go` tooling can be used as normal with this cloned repository. In +a full clone of the depot, Nix can be used to build Kontemplate: + + nix-build -A ops.kontemplate + +## Usage + +You must have `kubectl` installed to use Kontemplate effectively. + +``` +usage: kontemplate [<flags>] <command> [<args> ...] + +simple Kubernetes resource templating + +Flags: + -h, --help Show context-sensitive help (also try --help-long and --help-man). + -i, --include=INCLUDE ... Resource sets to include explicitly + -e, --exclude=EXCLUDE ... Resource sets to exclude explicitly + +Commands: + help [<command>...] + Show help. + + template <file> + Template resource sets and print them + + apply [<flags>] <file> + Template resources and pass to 'kubectl apply' + + replace <file> + Template resources and pass to 'kubectl replace' + + delete <file> + Template resources and pass to 'kubectl delete' + + create <file> + Template resources and pass to 'kubectl create' + +``` + +Examples: + +``` +# Look at output for a specific resource set and check to see if it's correct ... +kontemplate template example/prod-cluster.yaml -i some-api + +# ... maybe do a dry-run to see what kubectl would do: +kontemplate apply example/prod-cluster.yaml --dry-run + +# And actually apply it if you like what you see: +kontemplate apply example/prod-cluster.yaml +``` + +Check out the feature list and the individual feature documentation above. Then you should be good to go! + +## Contributing + +Feel free to contribute pull requests, file bugs and open issues with feature suggestions! + +Kontemplate is licensed under the GPLv3, a copy of the license and its terms can be found +in the `LICENSE` file. + +Please follow the [code of conduct](CODE_OF_CONDUCT.md). + +[Helm]: https://helm.sh/ +[releases page]: https://github.com/tazjin/kontemplate/releases diff --git a/ops/kontemplate/build-release.sh b/ops/kontemplate/build-release.sh new file mode 100755 index 000000000000..e4258c53dd09 --- /dev/null +++ b/ops/kontemplate/build-release.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -ueo pipefail + +# Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +# +# This file is part of Kontemplate. +# +# Kontemplate is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +readonly GIT_HASH="$(git rev-parse --short HEAD)" +readonly LDFLAGS="-X main.gitHash=${GIT_HASH} -w -s" +readonly VERSION="1.8.0-${GIT_HASH}" + +function binary-name() { + local os="${1}" + local target="${2}" + if [ "${os}" = "windows" ]; then + echo -n "${target}/kontemplate.exe" + else + echo -n "${target}/kontemplate" + fi +} + +function build-for() { + local os="${1}" + local arch="${2}" + local target="release/${os}/${arch}" + local bin=$(binary-name "${os}" "${target}") + + echo "Building kontemplate for ${os}-${arch} in ${target}" + + mkdir -p "${target}" + + env GOOS="${os}" GOARCH="${arch}" go build \ + -ldflags "${LDFLAGS}" \ + -o "${bin}" \ + -tags netgo +} + +function sign-for() { + local os="${1}" + local arch="${2}" + local target="release/${os}/${arch}" + local bin=$(binary-name "${os}" "${target}") + local tar="release/kontemplate-${VERSION}-${os}-${arch}.tar.gz" + + echo "Packing release into ${tar}" + tar czvf "${tar}" -C "${target}" $(basename "${bin}") + + local hash=$(sha256sum "${tar}") + echo "Signing kontemplate release tarball for ${os}-${arch} with SHA256 ${hash}" + gpg --armor --detach-sig --sign "${tar}" +} + +case "${1}" in + "build") + # Build releases for various operating systems: + build-for "linux" "amd64" + build-for "darwin" "amd64" + build-for "windows" "amd64" + build-for "freebsd" "amd64" + exit 0 + ;; + "sign") + # Bundle and sign releases: + sign-for "linux" "amd64" + sign-for "darwin" "amd64" + sign-for "windows" "amd64" + sign-for "freebsd" "amd64" + exit 0 + ;; +esac diff --git a/ops/kontemplate/context/context.go b/ops/kontemplate/context/context.go new file mode 100644 index 000000000000..2d0378a0ec23 --- /dev/null +++ b/ops/kontemplate/context/context.go @@ -0,0 +1,266 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package context + +import ( + "fmt" + "path" + "strings" + + "github.com/tazjin/kontemplate/util" +) + +type ResourceSet struct { + // Name of the resource set. This can be used in include/exclude statements during kontemplate runs. + Name string `json:"name"` + + // Path to the folder containing the files for this resource set. This defaults to the value of the 'name' field + // if unset. + Path string `json:"path"` + + // Values to include when interpolating resources from this resource set. + Values map[string]interface{} `json:"values"` + + // Args to pass on to kubectl for this resource set. + Args []string `json:"args"` + + // Nested resource sets to include + Include []ResourceSet `json:"include"` + + // Parent resource set for flattened resource sets. Should not be manually specified. + Parent string +} + +type Context struct { + // The name of the kubectl context + Name string `json:"context"` + + // Global variables that should be accessible by all resource sets + Global map[string]interface{} `json:"global"` + + // File names of YAML or JSON files including extra variables that should be globally accessible + VariableImportFiles []string `json:"import"` + + // The resource sets to include in this context + ResourceSets []ResourceSet `json:"include"` + + // Variables imported from additional files + ImportedVars map[string]interface{} + + // Explicitly set variables (via `--var`) that should override all others + ExplicitVars map[string]interface{} + + // This field represents the absolute path to the context base directory and should not be manually specified. + BaseDir string +} + +func contextLoadingError(filename string, cause error) error { + return fmt.Errorf("Context loading failed on file %s due to: \n%v", filename, cause) +} + +// Attempt to load and deserialise a Context from the specified file. +func LoadContext(filename string, explicitVars *[]string) (*Context, error) { + var ctx Context + err := util.LoadData(filename, &ctx) + + if err != nil { + return nil, contextLoadingError(filename, err) + } + + ctx.BaseDir = path.Dir(filename) + + // Prepare the resource sets by resolving parents etc. + ctx.ResourceSets = flattenPrepareResourceSetPaths(&ctx.BaseDir, &ctx.ResourceSets) + + // Add variables explicitly specified on the command line + ctx.ExplicitVars, err = loadExplicitVars(explicitVars) + if err != nil { + return nil, fmt.Errorf("Error setting explicit variables: %v\n", err) + } + + // Add variables loaded from import files + ctx.ImportedVars, err = ctx.loadImportedVariables() + if err != nil { + return nil, contextLoadingError(filename, err) + } + + // Merge variables defined at different levels. The + // `mergeContextValues` function is documented with the merge + // hierarchy. + ctx.ResourceSets = ctx.mergeContextValues() + + if err != nil { + return nil, contextLoadingError(filename, err) + } + + return &ctx, nil +} + +// Kontemplate supports specifying additional variable files with the +// `import` keyword. This function loads those variable files and +// merges them together with the context's other global variables. +func (ctx *Context) loadImportedVariables() (map[string]interface{}, error) { + allImportedVars := make(map[string]interface{}) + + for _, file := range ctx.VariableImportFiles { + // Ensure that the filename is not merged with the baseDir if + // it is set to an absolute path. + var filePath string + if path.IsAbs(file) { + filePath = file + } else { + filePath = path.Join(ctx.BaseDir, file) + } + + var importedVars map[string]interface{} + err := util.LoadData(filePath, &importedVars) + + if err != nil { + return nil, err + } + + allImportedVars = *util.Merge(&allImportedVars, &importedVars) + } + + return allImportedVars, nil +} + +// Correctly prepares the file paths for resource sets by inferring implicit paths and flattening resource set +// collections, i.e. resource sets that themselves have an additional 'include' field set. +// Those will be regarded as a short-hand for including multiple resource sets from a subfolder. +// See https://github.com/tazjin/kontemplate/issues/9 for more information. +func flattenPrepareResourceSetPaths(baseDir *string, rs *[]ResourceSet) []ResourceSet { + flattened := make([]ResourceSet, 0) + + for _, r := range *rs { + // If a path is not explicitly specified it should default to the resource set name. + // This is also the classic behaviour prior to kontemplate 1.2 + if r.Path == "" { + r.Path = r.Name + } + + // Paths are made absolute by resolving them relative to the context base, + // unless absolute paths were specified. + if !path.IsAbs(r.Path) { + r.Path = path.Join(*baseDir, r.Path) + } + + if len(r.Include) == 0 { + flattened = append(flattened, r) + } else { + for _, subResourceSet := range r.Include { + if subResourceSet.Path == "" { + subResourceSet.Path = subResourceSet.Name + } + + subResourceSet.Parent = r.Name + subResourceSet.Name = path.Join(r.Name, subResourceSet.Name) + subResourceSet.Path = path.Join(r.Path, subResourceSet.Path) + subResourceSet.Values = *util.Merge(&r.Values, &subResourceSet.Values) + flattened = append(flattened, subResourceSet) + } + } + } + + return flattened +} + +// Merges the context and resource set variables according in the +// desired precedence order. +// +// For now the reasoning behind the merge order is from least specific +// in relation to the cluster configuration, which means that the +// precedence is (in ascending order): +// +// 1. Default values in resource sets. +// 2. Values imported from files (via `import:`) +// 3. Global values in a cluster configuration +// 4. Values set in a resource set's `include`-section +// 5. Explicit values set on the CLI (`--var`) +// +// For a discussion on the reasoning behind this order, please consult +// https://github.com/tazjin/kontemplate/issues/142 +func (ctx *Context) mergeContextValues() []ResourceSet { + updated := make([]ResourceSet, len(ctx.ResourceSets)) + + // Merging has to happen separately for every individual + // resource set to make use of the default values: + for i, rs := range ctx.ResourceSets { + // Begin by loading default values from the resource + // sets configuration. + // + // Resource sets are used across different cluster + // contexts and the default values in them have the + // lowest precedence. + defaultValues := loadDefaultValues(&rs, ctx) + + // Continue by merging default values with values + // imported from external files. Those values are also + // used across cluster contexts, but have higher + // precedence than defaults. + merged := util.Merge(defaultValues, &ctx.ImportedVars) + + // Merge global values defined in the cluster context: + merged = util.Merge(merged, &ctx.Global) + + // Merge values configured in the resource set's + // `include` section: + merged = util.Merge(merged, &rs.Values) + + // Merge values defined explicitly on the CLI: + merged = util.Merge(merged, &ctx.ExplicitVars) + + // Continue with the newly merged resource set: + rs.Values = *merged + updated[i] = rs + } + + return updated +} + +// Loads default values for a resource set collection from +// path/to/set/default.{json|yaml}. +func loadDefaultValues(rs *ResourceSet, c *Context) *map[string]interface{} { + var defaultVars map[string]interface{} + + for _, filename := range util.DefaultFilenames { + err := util.LoadData(path.Join(rs.Path, filename), &defaultVars) + if err == nil { + return &defaultVars + } + } + + // The actual error is not inspected here. The reasoning for + // this is that in case of serious problems (e.g. permission + // issues with the folder / folder not existing) failure will + // occur a bit later anyways. + // + // Otherwise we'd have to differentiate between + // file-not-found-errors (no default values specified) and + // other errors here. + return &rs.Values +} + +// Prepares the variables specified explicitly via `--var` when +// executing kontemplate for adding to the context. +func loadExplicitVars(vars *[]string) (map[string]interface{}, error) { + explicitVars := make(map[string]interface{}, len(*vars)) + + for _, v := range *vars { + varParts := strings.SplitN(v, "=", 2) + if len(varParts) != 2 { + return nil, fmt.Errorf(`invalid explicit variable provided (%s), name and value should be separated with "="`, v) + } + + explicitVars[varParts[0]] = varParts[1] + } + + return explicitVars, nil +} diff --git a/ops/kontemplate/context/context_test.go b/ops/kontemplate/context/context_test.go new file mode 100644 index 000000000000..7ecd9d587d24 --- /dev/null +++ b/ops/kontemplate/context/context_test.go @@ -0,0 +1,353 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package context + +import ( + "reflect" + "testing" +) + +var noExplicitVars []string = make([]string, 0) + +func TestLoadFlatContextFromFile(t *testing.T) { + ctx, err := LoadContext("testdata/flat-test.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + Global: map[string]interface{}{ + "globalVar": "lizards", + }, + ResourceSets: []ResourceSet{ + { + Name: "some-api", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "apiPort": float64(4567), // yep! + "importantFeature": true, + "version": "1.0-0e6884d", + "globalVar": "lizards", + }, + Include: nil, + Parent: "", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestLoadContextWithArgs(t *testing.T) { + ctx, err := LoadContext("testdata/flat-with-args-test.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "some-api", + Path: "testdata/some-api", + Values: make(map[string]interface{}, 0), + Args: []string{ + "--as=some-user", + "--as-group=hello:world", + "--as-banana", + "true", + }, + Include: nil, + Parent: "", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestLoadContextWithResourceSetCollections(t *testing.T) { + ctx, err := LoadContext("testdata/collections-test.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + Global: map[string]interface{}{ + "globalVar": "lizards", + }, + ResourceSets: []ResourceSet{ + { + Name: "some-api", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "apiPort": float64(4567), // yep! + "importantFeature": true, + "version": "1.0-0e6884d", + "globalVar": "lizards", + }, + Include: nil, + Parent: "", + }, + { + Name: "collection/nested", + Path: "testdata/collection/nested", + Values: map[string]interface{}{ + "lizards": "good", + "globalVar": "lizards", + }, + Include: nil, + Parent: "collection", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } + +} + +func TestSubresourceVariableInheritance(t *testing.T) { + ctx, err := LoadContext("testdata/parent-variables.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "parent/child", + Path: "testdata/parent/child", + Values: map[string]interface{}{ + "foo": "bar", + "bar": "baz", + }, + Include: nil, + Parent: "parent", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded and expected context did not match") + t.Fail() + } +} + +func TestSubresourceVariableInheritanceOverride(t *testing.T) { + ctx, err := LoadContext("testdata/parent-variable-override.yaml", &noExplicitVars) + + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "parent/child", + Path: "testdata/parent/child", + Values: map[string]interface{}{ + "foo": "newvalue", + }, + Include: nil, + Parent: "parent", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded and expected context did not match") + t.Fail() + } +} + +func TestDefaultValuesLoading(t *testing.T) { + ctx, err := LoadContext("testdata/default-loading.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + rs := ctx.ResourceSets[0] + if rs.Values["defaultValues"] != "loaded" { + t.Errorf("Default values not loaded from YAML file") + t.Fail() + } + + if rs.Values["override"] != "notAtAll" { + t.Error("Default values should not override other values") + t.Fail() + } +} + +func TestImportValuesLoading(t *testing.T) { + ctx, err := LoadContext("testdata/import-vars-simple.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + expected := map[string]interface{}{ + "override": "true", + "music": map[string]interface{}{ + "artist": "Pallida", + "track": "Tractor Beam", + }, + } + + if !reflect.DeepEqual(ctx.ImportedVars, expected) { + t.Error("Expected imported values after loading imports did not match!") + t.Fail() + } +} + +func TestExplicitPathLoading(t *testing.T) { + ctx, err := LoadContext("testdata/explicit-path.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "some-api-europe", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "location": "europe", + }, + Include: nil, + Parent: "", + }, + { + Name: "some-api-asia", + Path: "testdata/some-api", + Values: map[string]interface{}{ + "location": "asia", + }, + Include: nil, + Parent: "", + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestExplicitSubresourcePathLoading(t *testing.T) { + ctx, err := LoadContext("testdata/explicit-subresource-path.yaml", &noExplicitVars) + if err != nil { + t.Error(err) + t.Fail() + } + + expected := Context{ + Name: "k8s.prod.mydomain.com", + ResourceSets: []ResourceSet{ + { + Name: "parent/child", + Path: "testdata/parent-path/child-path", + Parent: "parent", + Values: make(map[string]interface{}, 0), + }, + }, + BaseDir: "testdata", + ImportedVars: make(map[string]interface{}, 0), + ExplicitVars: make(map[string]interface{}, 0), + } + + if !reflect.DeepEqual(*ctx, expected) { + t.Error("Loaded context and expected context did not match") + t.Fail() + } +} + +func TestSetVariablesFromArguments(t *testing.T) { + vars := []string{"version=some-service-version"} + ctx, _ := LoadContext("testdata/default-loading.yaml", &vars) + + if version := ctx.ExplicitVars["version"]; version != "some-service-version" { + t.Errorf(`Expected variable "version" to have value "some-service-version" but was "%s"`, version) + } +} + +func TestSetInvalidVariablesFromArguments(t *testing.T) { + vars := []string{"version: some-service-version"} + _, err := LoadContext("testdata/default-loading.yaml", &vars) + + if err == nil { + t.Error("Expected invalid variable to return an error") + } +} + +// This test ensures that variables are merged in the correct order. +// Please consult the test data in `testdata/merging`. +func TestValueMergePrecedence(t *testing.T) { + cliVars:= []string{"cliVar=cliVar"} + ctx, _ := LoadContext("testdata/merging/context.yaml", &cliVars) + + expected := map[string]interface{}{ + "defaultVar": "defaultVar", + "importVar": "importVar", + "globalVar": "globalVar", + "includeVar": "includeVar", + "cliVar": "cliVar", + } + + result := ctx.ResourceSets[0].Values + + if !reflect.DeepEqual(expected, result) { + t.Errorf("Merged values did not match expected result: \n%v", result) + t.Fail() + } +} diff --git a/ops/kontemplate/context/testdata/collections-test.yaml b/ops/kontemplate/context/testdata/collections-test.yaml new file mode 100644 index 000000000000..a619c8cfddcc --- /dev/null +++ b/ops/kontemplate/context/testdata/collections-test.yaml @@ -0,0 +1,15 @@ +--- +context: k8s.prod.mydomain.com +global: + globalVar: lizards +include: + - name: some-api + values: + version: 1.0-0e6884d + importantFeature: true + apiPort: 4567 + - name: collection + include: + - name: nested + values: + lizards: good diff --git a/ops/kontemplate/context/testdata/default-loading.yaml b/ops/kontemplate/context/testdata/default-loading.yaml new file mode 100644 index 000000000000..d589c99b4eaf --- /dev/null +++ b/ops/kontemplate/context/testdata/default-loading.yaml @@ -0,0 +1,6 @@ +--- +context: default-loading +include: + - name: default + values: + override: notAtAll \ No newline at end of file diff --git a/ops/kontemplate/context/testdata/default/default.yaml b/ops/kontemplate/context/testdata/default/default.yaml new file mode 100644 index 000000000000..0ffa3cd81f24 --- /dev/null +++ b/ops/kontemplate/context/testdata/default/default.yaml @@ -0,0 +1,2 @@ +defaultValues: loaded +override: noop \ No newline at end of file diff --git a/ops/kontemplate/context/testdata/explicit-path.yaml b/ops/kontemplate/context/testdata/explicit-path.yaml new file mode 100644 index 000000000000..2c81f83c0919 --- /dev/null +++ b/ops/kontemplate/context/testdata/explicit-path.yaml @@ -0,0 +1,11 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: some-api-europe + path: some-api + values: + location: europe + - name: some-api-asia + path: some-api + values: + location: asia diff --git a/ops/kontemplate/context/testdata/explicit-subresource-path.yaml b/ops/kontemplate/context/testdata/explicit-subresource-path.yaml new file mode 100644 index 000000000000..6cf86183229e --- /dev/null +++ b/ops/kontemplate/context/testdata/explicit-subresource-path.yaml @@ -0,0 +1,8 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: parent + path: parent-path + include: + - name: child + path: child-path diff --git a/ops/kontemplate/context/testdata/flat-test.yaml b/ops/kontemplate/context/testdata/flat-test.yaml new file mode 100644 index 000000000000..dd7804f719c3 --- /dev/null +++ b/ops/kontemplate/context/testdata/flat-test.yaml @@ -0,0 +1,10 @@ +--- +context: k8s.prod.mydomain.com +global: + globalVar: lizards +include: + - name: some-api + values: + version: 1.0-0e6884d + importantFeature: true + apiPort: 4567 diff --git a/ops/kontemplate/context/testdata/flat-with-args-test.yaml b/ops/kontemplate/context/testdata/flat-with-args-test.yaml new file mode 100644 index 000000000000..29d3334fb54d --- /dev/null +++ b/ops/kontemplate/context/testdata/flat-with-args-test.yaml @@ -0,0 +1,9 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: some-api + args: + - --as=some-user + - --as-group=hello:world + - --as-banana + - "true" diff --git a/ops/kontemplate/context/testdata/import-vars-simple.yaml b/ops/kontemplate/context/testdata/import-vars-simple.yaml new file mode 100644 index 000000000000..12244e1ab174 --- /dev/null +++ b/ops/kontemplate/context/testdata/import-vars-simple.yaml @@ -0,0 +1,5 @@ +--- +context: k8s.prod.mydomain.com +import: + - test-vars.yaml +include: [] diff --git a/ops/kontemplate/context/testdata/merging/context.yaml b/ops/kontemplate/context/testdata/merging/context.yaml new file mode 100644 index 000000000000..df30d3d8cbe3 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/context.yaml @@ -0,0 +1,15 @@ +# This context file is intended to test the merge hierarchy of +# variables defined at different levels. +--- +context: merging.in.kontemplate.works +global: + globalVar: globalVar + includeVar: should be overridden (global) + cliVar: should be overridden (global) +import: + - import-vars.yaml +include: + - name: resource + values: + includeVar: includeVar + cliVar: should be overridden (include) diff --git a/ops/kontemplate/context/testdata/merging/import-vars.yaml b/ops/kontemplate/context/testdata/merging/import-vars.yaml new file mode 100644 index 000000000000..2a51352571a6 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/import-vars.yaml @@ -0,0 +1,4 @@ +importVar: importVar +globalVar: should be overridden (import) +includeVar: should be overridden (import) +cliVar: should be overridden (import) diff --git a/ops/kontemplate/context/testdata/merging/resource/default.yaml b/ops/kontemplate/context/testdata/merging/resource/default.yaml new file mode 100644 index 000000000000..040a19aaba25 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/resource/default.yaml @@ -0,0 +1,5 @@ +defaultVar: defaultVar +importVar: should be overridden (default) +globalVar: should be overridden (default) +includeVar: should be overridden (default) +cliVar: should be overridden (default) diff --git a/ops/kontemplate/context/testdata/merging/resource/output.yaml b/ops/kontemplate/context/testdata/merging/resource/output.yaml new file mode 100644 index 000000000000..5920b2720780 --- /dev/null +++ b/ops/kontemplate/context/testdata/merging/resource/output.yaml @@ -0,0 +1,5 @@ +defaultVar: {{ .defaultVar }} +importVar: {{ .importVar }} +globalVar: {{ .globalVar }} +includeVar: {{ .includeVar }} +cliVar: {{ .cliVar }} diff --git a/ops/kontemplate/context/testdata/parent-variable-override.yaml b/ops/kontemplate/context/testdata/parent-variable-override.yaml new file mode 100644 index 000000000000..42676c3028fe --- /dev/null +++ b/ops/kontemplate/context/testdata/parent-variable-override.yaml @@ -0,0 +1,10 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: parent + values: + foo: bar + include: + - name: child + values: + foo: newvalue diff --git a/ops/kontemplate/context/testdata/parent-variables.yaml b/ops/kontemplate/context/testdata/parent-variables.yaml new file mode 100644 index 000000000000..8459fd30405b --- /dev/null +++ b/ops/kontemplate/context/testdata/parent-variables.yaml @@ -0,0 +1,10 @@ +--- +context: k8s.prod.mydomain.com +include: + - name: parent + values: + foo: bar + include: + - name: child + values: + bar: baz diff --git a/ops/kontemplate/context/testdata/test-vars-override.yaml b/ops/kontemplate/context/testdata/test-vars-override.yaml new file mode 100644 index 000000000000..5215c559c136 --- /dev/null +++ b/ops/kontemplate/context/testdata/test-vars-override.yaml @@ -0,0 +1,3 @@ +--- +override: 3 +place: Oslo diff --git a/ops/kontemplate/context/testdata/test-vars.yaml b/ops/kontemplate/context/testdata/test-vars.yaml new file mode 100644 index 000000000000..af27bdc455bf --- /dev/null +++ b/ops/kontemplate/context/testdata/test-vars.yaml @@ -0,0 +1,5 @@ +--- +override: 'true' +music: + artist: Pallida + track: Tractor Beam diff --git a/ops/kontemplate/default.nix b/ops/kontemplate/default.nix new file mode 100644 index 000000000000..1190869c3fda --- /dev/null +++ b/ops/kontemplate/default.nix @@ -0,0 +1,36 @@ +# Copyright (C) 2016-2021 Vincent Ambo <mail@tazj.in> +# +# This file is part of Kontemplate. +# +# Kontemplate is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This file is the Nix derivation used to install Kontemplate on +# Nix-based systems. + +{ lib, pkgs, ... }: + +pkgs.buildGoPackage rec { + name = "kontemplate-${version}"; + version = "canon"; + src = ./.; + goPackagePath = "github.com/tazjin/kontemplate"; + goDeps = ./deps.nix; + buildInputs = [ pkgs.parallel ]; + + # Enable checks and configure check-phase to include vet: + doCheck = true; + preCheck = '' + for pkg in $(getGoDirs ""); do + buildGoDir vet "$pkg" + done + ''; + + meta = with lib; { + description = "A resource templating helper for Kubernetes"; + homepage = "http://kontemplate.works/"; + license = licenses.gpl3; + }; +} diff --git a/ops/kontemplate/deps.nix b/ops/kontemplate/deps.nix new file mode 100644 index 000000000000..7693968bd550 --- /dev/null +++ b/ops/kontemplate/deps.nix @@ -0,0 +1,111 @@ +# This file was generated by https://github.com/kamilchm/go2nix v1.3.0 +[ + { + goPackagePath = "github.com/Masterminds/goutils"; + fetch = { + type = "git"; + url = "https://github.com/Masterminds/goutils"; + rev = "41ac8693c5c10a92ea1ff5ac3a7f95646f6123b0"; + sha256 = "180px47gj936qyk5bkv5mbbgiil9abdjq6kwkf7sq70vyi9mcfiq"; + }; + } + { + goPackagePath = "github.com/Masterminds/semver"; + fetch = { + type = "git"; + url = "https://github.com/Masterminds/semver"; + rev = "5bc3b9184d48f1412b300b87a200cf020d9254cf"; + sha256 = "1vdfm653v50jf63cw0kg2hslx50cn4mk6lj3p51bi11jrg48kfng"; + }; + } + { + goPackagePath = "github.com/Masterminds/sprig"; + fetch = { + type = "git"; + url = "https://github.com/Masterminds/sprig"; + rev = "6f509977777c33eae63b2136d97f7b976cb971cc"; + sha256 = "05h9k6fhjxnpwlihj3z02q9kvqvnq53jix0ab84sx0666bci3cdh"; + }; + } + { + goPackagePath = "github.com/alecthomas/template"; + fetch = { + type = "git"; + url = "https://github.com/alecthomas/template"; + rev = "fb15b899a75114aa79cc930e33c46b577cc664b1"; + sha256 = "1vlasv4dgycydh5wx6jdcvz40zdv90zz1h7836z7lhsi2ymvii26"; + }; + } + { + goPackagePath = "github.com/alecthomas/units"; + fetch = { + type = "git"; + url = "https://github.com/alecthomas/units"; + rev = "c3de453c63f4bdb4dadffab9805ec00426c505f7"; + sha256 = "0js37zlgv37y61j4a2d46jh72xm5kxmpaiw0ya9v944bjpc386my"; + }; + } + { + goPackagePath = "github.com/ghodss/yaml"; + fetch = { + type = "git"; + url = "https://github.com/ghodss/yaml"; + rev = "25d852aebe32c875e9c044af3eef9c7dc6bc777f"; + sha256 = "1w9yq0bxzygc4qwkwwiy7k1k1yviaspcqqv18255k2xkjv5ipccz"; + }; + } + { + goPackagePath = "github.com/google/uuid"; + fetch = { + type = "git"; + url = "https://github.com/google/uuid"; + rev = "c2e93f3ae59f2904160ceaab466009f965df46d6"; + sha256 = "0zw8fvl6jqg0fmv6kmvhss0g4gkrbvgyvl2zgy5wdbdlgp4fja0h"; + }; + } + { + goPackagePath = "github.com/huandu/xstrings"; + fetch = { + type = "git"; + url = "https://github.com/huandu/xstrings"; + rev = "8bbcf2f9ccb55755e748b7644164cd4bdce94c1d"; + sha256 = "1ivvc95514z63k7cpz71l0dwlanffmsh1pijhaqmp41kfiby8rsx"; + }; + } + { + goPackagePath = "github.com/imdario/mergo"; + fetch = { + type = "git"; + url = "https://github.com/imdario/mergo"; + rev = "4c317f2286be3bd0c4f1a0e622edc6398ec4656d"; + sha256 = "0bihha1qsgfjk14yv1hwddv3d8dzxpbjlaxwwyys6lhgxz1cr9h9"; + }; + } + { + goPackagePath = "golang.org/x/crypto"; + fetch = { + type = "git"; + url = "https://go.googlesource.com/crypto"; + rev = "9756ffdc24725223350eb3266ffb92590d28f278"; + sha256 = "0q7hxaaq6lp0v8qqzifvysl47z5rfdlrxkh3d29vsl3wyby3dxl8"; + }; + } + { + goPackagePath = "gopkg.in/alecthomas/kingpin.v2"; + fetch = { + type = "git"; + url = "https://gopkg.in/alecthomas/kingpin.v2"; + rev = "947dcec5ba9c011838740e680966fd7087a71d0d"; + sha256 = "0mndnv3hdngr3bxp7yxfd47cas4prv98sqw534mx7vp38gd88n5r"; + }; + } + { + goPackagePath = "gopkg.in/yaml.v2"; + fetch = { + type = "git"; + url = "https://gopkg.in/yaml.v2"; + rev = "51d6538a90f86fe93ac480b35f37b2be17fef232"; + sha256 = "01wj12jzsdqlnidpyjssmj0r4yavlqy7dwrg7adqd8dicjc4ncsa"; + }; + } +] diff --git a/ops/kontemplate/docs/cluster-config.md b/ops/kontemplate/docs/cluster-config.md new file mode 100644 index 000000000000..4e870161796d --- /dev/null +++ b/ops/kontemplate/docs/cluster-config.md @@ -0,0 +1,106 @@ +Cluster configuration +========================== + +Every cluster (or "environment") that requires individual configuration is specified in +a very simple YAML file in Kontemplate. + +An example file for a hypothetical test environment could look like this: + +```yaml +--- +context: k8s.test.mydomain.com +global: + clusterName: test-cluster + defaultReplicas: 2 +import: + - test-secrets.yaml +include: + - name: gateway + path: tools/nginx + values: + tlsDomains: + - test.oslo.pub + - test.tazj.in + - path: backend + values: + env: test + include: + - name: blog + values: + url: test.tazj.in + - name: pub-service +``` + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Cluster configuration](#cluster-configuration) + - [Fields](#fields) + - [`context`](#context) + - [`global`](#global) + - [`import`](#import) + - [`include`](#include) + - [External variables](#external-variables) + +<!-- markdown-toc end --> + +## Fields + +This is documentation for the individual fields in a cluster context file. + +### `context` + +The `context` field contains the name of the kubectl-context. You can list context names with +'kubectl config get-contexts'. + +This must be set here so that Kontemplate can use the correct context when calling kubectl. + +This field is **required** for `kubectl`-wrapping commands. It can be left out if only the `template`-command is used. + +### `global` + +The `global` field contains a key/value map of variables that should be available to all resource +sets in the cluster. + +This field is **optional**. + +### `import` + +The `import` field contains the file names of additional YAML or JSON files from which global +variables should be loaded. Using this field makes it possible to keep certain configuration that +is the same for some, but not all, clusters in a common place. + +This field is **optional**. + +### `include` + +The `include` field contains the actual resource sets to be included in the cluster. + +Information about the structure of resource sets can be found in the [resource set documentation][]. + +This field is **required**. + +## External variables + +As mentioned above, extra variables can be loaded from additional YAML or JSON files. Assuming you +have a file called `test-secrets.yaml` which contains variables that should be shared between a `test` +and `dev` cluster, you could import it in your context as such: + +```yaml +# test-secrets.yaml: +mySecretVar: foo-bar-12345 + +# test-cluster.yaml: +context: k8s.test.mydomain.com +import: + - test-secrets.yaml + +# dev-cluster.yaml: +context: k8s.dev.mydomain.com +import: + - test-secrets.yaml +``` + +The variable `mySecretVar` is then available as a global variable. + +[resource set documentation]: resource-sets.md diff --git a/ops/kontemplate/docs/resource-sets.md b/ops/kontemplate/docs/resource-sets.md new file mode 100644 index 000000000000..1444dd491249 --- /dev/null +++ b/ops/kontemplate/docs/resource-sets.md @@ -0,0 +1,170 @@ +Resource Sets +================ + +Resource sets are collections of Kubernetes resources that should be passed to `kubectl` together. + +Technically a resource set is simply a folder with a few YAML and/or JSON templates in it. + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Resource Sets](#resource-sets) +- [Creating resource sets](#creating-resource-sets) + - [Default variables](#default-variables) +- [Including resource sets](#including-resource-sets) + - [Fields](#fields) + - [`name`](#name) + - [`path`](#path) + - [`values`](#values) + - [`args`](#args) + - [`include`](#include) + - [Multiple includes](#multiple-includes) + - [Nesting resource sets](#nesting-resource-sets) + - [Caveats](#caveats) + +<!-- markdown-toc end --> + +# Creating resource sets + +Simply create a folder in your Kontemplate repository and place a YAML or JSON file in it. These +files get interpreted as [templates][] during Kontemplate runs and variables (as well as template +logic or functions) will be interpolated. + +Refer to the template documentation for information on how to write templates. + +## Default variables + +Sometimes it is useful to specify default values for variables that should be interpolated during +a run if the [cluster configuration][] does not specify a variable explicitly. + +This can be done simply by placing a `default.yaml` or `default.json` file in the resource set +folder and filling it with key/value pairs of the intended default variables. + +Kontemplate will error during interpolation if any variables are left unspecified. + +# Including resource sets + +Under the cluster configuration `include` key resource sets are included and required variables +are specified. For example: + +```yaml +include: + - name: some-api + values: + version: 1.2-SNAPSHOT +``` + +This will include a resource set from a folder called `some-api` and set the specified `version` variable. + +## Fields + +The available fields when including a resource set are these: + +### `name` + +The `name` field contains the name of the resource set. This name can be used to refer to the resource set +when specifying explicit includes or excludes during a run. + +By default it is assumed that the `name` is the path to the resource set folder, but this can be overridden. + +This field is **required**. + +### `path` + +The `path` field specifies an explicit path to a resource set folder in the case that it should differ from +the resource set's `name`. + +This field is **optional**. + +### `values` + +The `values` field specifies key/values pairs of variables that should be available during templating. + +This field is **optional**. + +### `args` + +The `args` field specifies a list of arguments that should be passed to `kubectl`. + +This field is **optional**. + +### `include` + +The `include` field specifies additional resource sets that should be included and that should inherit the +variables of this resource set. + +The fully qualified names of "nested" resource sets are set to `${PARENT_NAME}/${CHILD_NAME}` and paths are +merged in the same way. + +This makes it easy to organise different resource sets as "groups" to include / exclude them collectively +during runs. + +This field is **optional**. + +## Multiple includes + +Resource sets can be included multiple times with different configurations. In this case it is recommended +to set the `path` and `name` fields explicitly. For example: + +```yaml +include: + - name: forwarder-europe + path: tools/forwarder + values: + source: europe + - name: forwarder-asia + path: tools/forwarder + values: + source: asia +``` + +The two different configurations can be referred to by their set names, but will use the same resource +templates with different configurations. + +## Nesting resource sets + +As mentioned above for the `include` field, resource sets can be nested. This lets users group resource +sets in logical ways using simple folder structures. + +Assuming a folder structure like: + +``` +├── backend +│ ├── auth-api +│ ├── message-api +│ └── order-api +└── frontend + ├── app-page + └── login-page +``` + +With each of these folders being a resource set, they could be included in a cluster configuration like so: + +```yaml +include: + - name: backend + include: + - name: auth-api + - name: message-api + - name: order-api + - name: frontend: + include: + - name: app-page + - name: login-page +``` + +Kontemplate could then be run with, for example, `--include backend` to only include the resource sets nested +in the backend group. Specific resource sets can also be targeted, for example as `--include backend/order-api`. + +Variables specified in the parent resource set are inherited by the children. + +### Caveats + +Two caveats apply that users should be aware of: + +1. The parent resource set can not contain any resource templates itself. + +2. Only one level of nesting is supported. Specifying `include` again on a nested resource set will be ignored. + +[templates]: templates.md +[cluster configuration]: cluster-config.md diff --git a/ops/kontemplate/docs/templates.md b/ops/kontemplate/docs/templates.md new file mode 100644 index 000000000000..32da205108de --- /dev/null +++ b/ops/kontemplate/docs/templates.md @@ -0,0 +1,153 @@ +Kontemplate templates +===================== + +The template file format is based on Go's [templating engine][] in combination +with a small extension library called [sprig][] that adds additional template +functions. + +Go templates can either simply display variables or build more complicated +*pipelines* in which variables are passed to functions for further processing, +or in which conditionals are evaluated for more complex template logic. + +It is recommended that you check out the Golang [documentation][] for the templating +engine in addition to the cherry-picked features listed here. + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Kontemplate templates](#kontemplate-templates) + - [Basic variable interpolation](#basic-variable-interpolation) + - [Example:](#example) + - [Template functions](#template-functions) + - [Examples:](#examples) + - [Conditionals & ranges](#conditionals--ranges) + - [Caveats](#caveats) + +<!-- markdown-toc end --> + +## Basic variable interpolation + +The basic template format uses `{{ .variableName }}` as the interpolation format. + +### Example: + +Assuming that you include a resource set as such: + +``` +- name: api-gateway + values: + internalHost: http://my-internal-host/ +``` + +And the api-gateway resource set includes a ConfigMap (some fields left out for +the example): + +``` +# api-gateway/configmap.yaml: +--- +kind: ConfigMap +metadata: + name: api-gateway-config +data: + internalHost: {{ .internalHost }} +``` + +The resulting output will be: + +``` + +--- +kind: ConfigMap +metadata: + name: api-gateway-config +data: + internalHost: http://my-internal-host/ +``` + +## Template functions + +Go templates support template functions which you can think of as a sort of +shell-like pipeline where text flows through transformations from left to +right. + +Some template functions come from Go's standard library and are listed in the +[Go documentation][]. In addition the functions declared by [sprig][] are +available in kontemplate, as well as five custom functions: + +* `json`: Encodes any supplied data structure as JSON. +* `gitHEAD`: Retrieves the commit hash at Git `HEAD`. +* `passLookup`: Looks up the supplied key in [pass][]. +* `insertFile`: Insert the contents of the given file in the resource + set folder as a string. +* `insertTemplate`: Insert the contents of the given template in the resource + set folder as a string. + +## Examples: + +``` +# With the following values: +name: Donald +certKeyPath: my-website/cert-key + +# The following interpolations are possible: + +{{ .name | upper }} +-> DONALD + +{{ .name | upper | repeat 2 }} +-> DONALD DONALD + +{{ .certKeyPath | passLookup }} +-> Returns content of 'my-website/cert-key' from pass + +{{ gitHEAD }} +-> Returns the Git commit hash at HEAD. +``` + +## Conditionals & ranges + +Some logic is supported in Golang templates and can be used in Kontemplate, too. + +With the following values: + +``` +useKube2IAM: true +servicePorts: + - 8080 + - 9090 +``` + +The following interpolations are possible: + +``` +# Conditionally insert something in the template: +metadata: + annotations: + foo: bar + {{ if .useKube2IAM -}} iam.amazonaws.com/role: my-api {{- end }} +``` + +``` +# Iterate over a list of values +ports: + {{ range .servicePorts }} + - port: {{ . }} + {{ end }} +``` + +Check out the Golang documentation (linked above) for more information about template logic. + +## Caveats + +Kontemplate does not by itself parse any of the content of the templates, which +means that it does not validate whether the resources you supply are valid YAML +or JSON. + +You can perform some validation by using `kontemplate apply --dry-run` which +will make use of the Dry-Run functionality in `kubectl`. + +[templating engine]: https://golang.org/pkg/text/template/ +[documentation]: https://golang.org/pkg/text/template/ +[sprig]: http://masterminds.github.io/sprig/ +[Go documentation]: https://golang.org/pkg/text/template/#hdr-Functions +[pass]: https://www.passwordstore.org/ diff --git a/ops/kontemplate/docs/tips-and-tricks.md b/ops/kontemplate/docs/tips-and-tricks.md new file mode 100644 index 000000000000..5401ac91e5a3 --- /dev/null +++ b/ops/kontemplate/docs/tips-and-tricks.md @@ -0,0 +1,77 @@ +Kontemplate tips & tricks +========================= + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +**Table of Contents** + +- [Kontemplate tips & tricks](#kontemplate-tips--tricks) + - [Update Deployments when ConfigMaps change](#update-deployments-when-configmaps-change) + - [direnv & pass](#direnv--pass) + +<!-- markdown-toc end --> + +## Update Deployments when ConfigMaps change + +Kubernetes does [not currently][] have the ability to perform rolling updates +of Deployments and other resource types when `ConfigMap` or `Secret` objects +are updated. + +It is possible to make use of annotations and templating functions in +Kontemplate to force updates to these resources anyways. + +For example: + +```yaml +# A ConfigMap that contains some configuration for your app +--- +kind: ConfigMap +metadata: + name: app-config +data: + app.conf: | + name: {{ .appName }} + foo: bar +``` + +Now whenever the `appName` variable changes or we make an edit to the +`ConfigMap` we would like to update the `Deployment` making use of it, too. We +can do this by adding a hash of the parsed template to the annotations of the +created `Pod` objects: + +```yaml + +--- +kind: Deployment +metadata: + name: app +spec: + template: + metadata: + annotations: + configHash: {{ insertTemplate "app-config.yaml" | sha256sum }} + spec: + containers: + - name: app + # Some details omitted ... + volumeMounts: + - name: config + mountPath: /etc/app/ + volumes: + - name: config + configMap: + name: app-config +``` + +Now any change to the `ConfigMap` - either by directly editing the yaml file or +via a changed template variable - will cause the annotation to change, +triggering a rolling update of all relevant pods. + +## direnv & pass + +Users of `pass` may have multiple different password stores on their machines. +Assuming that `kontemplate` configuration exists somewhere on the filesystem +per project, it is easy to use [direnv][] to switch to the correct +`PASSWORD_STORE_DIR` variable when entering the folder. + +[not currently]: https://github.com/kubernetes/kubernetes/issues/22368 +[direnv]: https://direnv.net/ diff --git a/ops/kontemplate/example/other-config.yaml b/ops/kontemplate/example/other-config.yaml new file mode 100644 index 000000000000..87370569c46e --- /dev/null +++ b/ops/kontemplate/example/other-config.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: extensions/v1beta1 +kind: ConfigMap +metadata: + name: other-config +data: + globalData: {{ .globalVar }} diff --git a/ops/kontemplate/example/prod-cluster.json b/ops/kontemplate/example/prod-cluster.json new file mode 100644 index 000000000000..70e2365f1710 --- /dev/null +++ b/ops/kontemplate/example/prod-cluster.json @@ -0,0 +1,16 @@ +{ + "context": "k8s.prod.mydomain.com", + "global": { + "globalVar": "lizards" + }, + "include": [ + { + "name": "some-api", + "values": { + "version": "1.0-SNAPSHOT-0e6884d", + "importantFeature": true, + "apiPort": 4567 + } + } + ] +} diff --git a/ops/kontemplate/example/prod-cluster.yaml b/ops/kontemplate/example/prod-cluster.yaml new file mode 100644 index 000000000000..9f300a492080 --- /dev/null +++ b/ops/kontemplate/example/prod-cluster.yaml @@ -0,0 +1,17 @@ +--- +context: k8s.prod.mydomain.com +global: + globalVar: lizards +include: + # By default resource sets are included from a folder with the same + # name as the resource set's name + - name: some-api + values: + version: 1.0-0e6884d + importantFeature: true + apiPort: 4567 + + # Paths can also be specified manually (and point at single template + # files!) + - name: other-config + path: other-config.yaml diff --git a/ops/kontemplate/example/some-api/some-api.yaml b/ops/kontemplate/example/some-api/some-api.yaml new file mode 100644 index 000000000000..f0188f9dbde3 --- /dev/null +++ b/ops/kontemplate/example/some-api/some-api.yaml @@ -0,0 +1,52 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: secret-certificate +data: + cert.pem: {{ passLookup "my/secret/certificate" | b64enc }} +--- +apiVersion: extensions/v1beta1 +kind: ConfigMap +metadata: + name: some-config +data: + # The content of the example configuration file is templated in here + # by the 'insertFile' function and indented for YAML-compatibility + # with the 'indent' function: + some.cfg: | +{{ insertFile "some.cfg" | indent 4 }} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: some-api +spec: + replicas: 1 + template: + metadata: + labels: + app: some-api + spec: + containers: + - image: my.container.repo/some-api:{{ .version }} + name: some-api + env: + - name: ENABLE_IMPORTANT_FEATURE + value: {{ .importantFeature }} + - name: SOME_GLOBAL_VAR + value: {{ .globalVar }} +--- +apiVersion: v1 +kind: Service +metadata: + name: some-api + labels: + app: some-api +spec: + selector: + app: some-api + ports: + - port: 80 + targetPort: {{ .apiPort }} + name: http diff --git a/ops/kontemplate/example/some-api/some.cfg b/ops/kontemplate/example/some-api/some.cfg new file mode 100644 index 000000000000..733d5e16789a --- /dev/null +++ b/ops/kontemplate/example/some-api/some.cfg @@ -0,0 +1,4 @@ +{ + "something": 1542, + "other-thing": "да" +} diff --git a/ops/kontemplate/image/Dockerfile b/ops/kontemplate/image/Dockerfile new file mode 100644 index 000000000000..a40fa83b0867 --- /dev/null +++ b/ops/kontemplate/image/Dockerfile @@ -0,0 +1,15 @@ +FROM alpine:3.10 + +ADD hashes /root/hashes +ADD https://storage.googleapis.com/kubernetes-release/release/v1.15.3/bin/linux/amd64/kubectl /usr/bin/kubectl +ADD https://github.com/tazjin/kontemplate/releases/download/v1.8.0/kontemplate-1.8.0-6c3b299-linux-amd64.tar.gz /tmp/kontemplate.tar.gz + +# Pass release version is 1.7.3 +ADD https://raw.githubusercontent.com/zx2c4/password-store/74fdfb5022f317ad48d449e29543710bdad1afda/src/password-store.sh /usr/bin/pass + +RUN sha256sum -c /root/hashes && \ + apk add -U bash tree gnupg git && \ + chmod +x /usr/bin/kubectl /usr/bin/pass && \ + tar xzvf /tmp/kontemplate.tar.gz && \ + mv kontemplate /usr/bin/kontemplate && \ + /usr/bin/kontemplate version diff --git a/ops/kontemplate/image/README.md b/ops/kontemplate/image/README.md new file mode 100644 index 000000000000..fe04765401be --- /dev/null +++ b/ops/kontemplate/image/README.md @@ -0,0 +1,12 @@ +Kontemplate Docker image +======================== + +This builds a simple Docker image available on the Docker Hub as `tazjin/kontemplate`. + +Builds are automated based on the Dockerfile contained here. + +It contains both `kontemplate` and `kubectl` and can be used as part of container-based +CI pipelines. + +`pass` and its dependencies are also installed to enable the use of the `passLookup` +template function if desired. diff --git a/ops/kontemplate/image/hashes b/ops/kontemplate/image/hashes new file mode 100644 index 000000000000..bfd87c0201e7 --- /dev/null +++ b/ops/kontemplate/image/hashes @@ -0,0 +1,2 @@ +a39dfdd77e4655acaabe301285cf389cb5fc8145060f5677dc93db1cc20911a4 /tmp/kontemplate.tar.gz +6e805054a1fb2280abb53f75b57a1b92bf9c66ffe0d2cdcd46e81b079d93c322 /usr/bin/kubectl diff --git a/ops/kontemplate/main.go b/ops/kontemplate/main.go new file mode 100644 index 000000000000..e55d42465c6b --- /dev/null +++ b/ops/kontemplate/main.go @@ -0,0 +1,242 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see <http://www.gnu.org/licenses/>. + +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/tazjin/kontemplate/context" + "github.com/tazjin/kontemplate/templater" + "gopkg.in/alecthomas/kingpin.v2" +) + +const version string = "1.8.0" + +// This variable will be initialised by the Go linker during the builder +var gitHash string + +var ( + app = kingpin.New("kontemplate", "simple Kubernetes resource templating") + + // Global flags + includes = app.Flag("include", "Resource sets to include explicitly").Short('i').Strings() + excludes = app.Flag("exclude", "Resource sets to exclude explicitly").Short('e').Strings() + variables = app.Flag("var", "Provide variables to templates explicitly").Strings() + kubectlBin = app.Flag("kubectl", "Path to the kubectl binary (default 'kubectl')").Default("kubectl").String() + + // Commands + template = app.Command("template", "Template resource sets and print them") + templateFile = template.Arg("file", "Cluster configuration file to use").Required().String() + templateOutputDir = template.Flag("output", "Output directory in which to save templated files instead of printing them").Short('o').String() + + apply = app.Command("apply", "Template resources and pass to 'kubectl apply'") + applyFile = apply.Arg("file", "Cluster configuration file to use").Required().String() + applyDryRun = apply.Flag("dry-run", "Print remote operations without executing them").Default("false").Bool() + + replace = app.Command("replace", "Template resources and pass to 'kubectl replace'") + replaceFile = replace.Arg("file", "Cluster configuration file to use").Required().String() + + delete = app.Command("delete", "Template resources and pass to 'kubectl delete'") + deleteFile = delete.Arg("file", "Cluster configuration file to use").Required().String() + + create = app.Command("create", "Template resources and pass to 'kubectl create'") + createFile = create.Arg("file", "Cluster configuration file to use").Required().String() + + versionCmd = app.Command("version", "Show kontemplate version") +) + +func main() { + app.HelpFlag.Short('h') + + switch kingpin.MustParse(app.Parse(os.Args[1:])) { + case template.FullCommand(): + templateCommand() + + case apply.FullCommand(): + applyCommand() + + case replace.FullCommand(): + replaceCommand() + + case delete.FullCommand(): + deleteCommand() + + case create.FullCommand(): + createCommand() + + case versionCmd.FullCommand(): + versionCommand() + } +} + +func versionCommand() { + if gitHash == "" { + fmt.Printf("Kontemplate version %s (git commit unknown)\n", version) + } else { + fmt.Printf("Kontemplate version %s (git commit: %s)\n", version, gitHash) + } +} + +func templateCommand() { + _, resourceSets := loadContextAndResources(templateFile) + + for _, rs := range *resourceSets { + if len(rs.Resources) == 0 { + fmt.Fprintf(os.Stderr, "Warning: Resource set '%s' does not exist or contains no valid templates\n", rs.Name) + continue + } + + if *templateOutputDir != "" { + templateIntoDirectory(templateOutputDir, rs) + } else { + for _, r := range rs.Resources { + fmt.Fprintf(os.Stderr, "Rendered file %s/%s:\n", rs.Name, r.Filename) + fmt.Println(r.Rendered) + } + } + } +} + +func templateIntoDirectory(outputDir *string, rs templater.RenderedResourceSet) { + // Attempt to create the output directory if it does not + // already exist: + if err := os.MkdirAll(*templateOutputDir, 0775); err != nil { + app.Fatalf("Could not create output directory: %v\n", err) + } + + // Nested resource sets may contain slashes in their names. + // These are replaced with dashes for the purpose of writing a + // flat list of output files: + setName := strings.Replace(rs.Name, "/", "-", -1) + + for _, r := range rs.Resources { + filename := fmt.Sprintf("%s/%s-%s", *templateOutputDir, setName, r.Filename) + fmt.Fprintf(os.Stderr, "Writing file %s\n", filename) + + file, err := os.Create(filename) + if err != nil { + app.Fatalf("Could not create file %s: %v\n", filename, err) + } + + _, err = fmt.Fprintf(file, r.Rendered) + if err != nil { + app.Fatalf("Error writing file %s: %v\n", filename, err) + } + } +} + +func applyCommand() { + ctx, resources := loadContextAndResources(applyFile) + + var kubectlArgs []string + + if *applyDryRun { + kubectlArgs = []string{"apply", "-f", "-", "--dry-run"} + } else { + kubectlArgs = []string{"apply", "-f", "-"} + } + + if err := runKubectlWithResources(ctx, &kubectlArgs, resources); err != nil { + failWithKubectlError(err) + } +} + +func replaceCommand() { + ctx, resources := loadContextAndResources(replaceFile) + args := []string{"replace", "--save-config=true", "-f", "-"} + + if err := runKubectlWithResources(ctx, &args, resources); err != nil { + failWithKubectlError(err) + } +} + +func deleteCommand() { + ctx, resources := loadContextAndResources(deleteFile) + args := []string{"delete", "-f", "-"} + + if err := runKubectlWithResources(ctx, &args, resources); err != nil { + failWithKubectlError(err) + } +} + +func createCommand() { + ctx, resources := loadContextAndResources(createFile) + args := []string{"create", "--save-config=true", "-f", "-"} + + if err := runKubectlWithResources(ctx, &args, resources); err != nil { + failWithKubectlError(err) + } +} + +func loadContextAndResources(file *string) (*context.Context, *[]templater.RenderedResourceSet) { + ctx, err := context.LoadContext(*file, variables) + if err != nil { + app.Fatalf("Error loading context: %v\n", err) + } + + resources, err := templater.LoadAndApplyTemplates(includes, excludes, ctx) + if err != nil { + app.Fatalf("Error templating resource sets: %v\n", err) + } + + return ctx, &resources +} + +func runKubectlWithResources(c *context.Context, kubectlArgs *[]string, resourceSets *[]templater.RenderedResourceSet) error { + argsWithContext := append(*kubectlArgs, fmt.Sprintf("--context=%s", c.Name)) + + for _, rs := range *resourceSets { + if len(rs.Resources) == 0 { + fmt.Fprintf(os.Stderr, "Warning: Resource set '%s' contains no valid templates\n", rs.Name) + continue + } + + argsWithResourceSetArgs := append(argsWithContext, rs.Args...) + + kubectl := exec.Command(*kubectlBin, argsWithResourceSetArgs...) + + stdin, err := kubectl.StdinPipe() + if err != nil { + return fmt.Errorf("kubectl error: %v", err) + } + + kubectl.Stdout = os.Stdout + kubectl.Stderr = os.Stderr + + if err = kubectl.Start(); err != nil { + return fmt.Errorf("kubectl error: %v", err) + } + + for _, r := range rs.Resources { + fmt.Printf("Passing file %s/%s to kubectl\n", rs.Name, r.Filename) + fmt.Fprintln(stdin, r.Rendered) + } + stdin.Close() + + if err = kubectl.Wait(); err != nil { + return err + } + } + + return nil +} + +func failWithKubectlError(err error) { + app.Fatalf("Kubectl error: %v\n", err) +} diff --git a/ops/kontemplate/release.nix b/ops/kontemplate/release.nix new file mode 100644 index 000000000000..8a04109526cf --- /dev/null +++ b/ops/kontemplate/release.nix @@ -0,0 +1,54 @@ +# Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +# +# This file is part of Kontemplate. +# +# Kontemplate is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This file is the Nix derivation used to build release binaries for +# several different architectures and operating systems. + +let pkgs = import ((import <nixpkgs> {}).fetchFromGitHub { + owner = "NixOS"; + repo = "nixpkgs-channels"; + rev = "541d9cce8af7a490fb9085305939569567cb58e6"; + sha256 = "0jgz72hhzkd5vyq5v69vpljjlnf0lqaz7fh327bvb3cvmwbfxrja"; +}) {}; +in with pkgs; buildGoPackage rec { + name = "kontemplate-${version}"; + version = "canon"; + src = ./.; + goPackagePath = "github.com/tazjin/kontemplate"; + goDeps = ./deps.nix; + + # This configuration enables the building of statically linked + # executables. For some reason, those will have multiple references + # to the Go compiler's installation path in them, which is the + # reason for setting the 'allowGoReference' flag. + dontStrip = true; # Linker configuration handles stripping + allowGoReference = true; + CGO_ENABLED="0"; + GOCACHE="off"; + + # Configure release builds via the "build-matrix" script: + buildInputs = [ git ]; + buildPhase = '' + cd go/src/${goPackagePath} + patchShebangs build-release.sh + ./build-release.sh build + ''; + + outputs = [ "out" ]; + installPhase = '' + mkdir $out + cp -r release/ $out + ''; + + meta = with lib; { + description = "A resource templating helper for Kubernetes"; + homepage = "http://kontemplate.works/"; + license = licenses.gpl3; + }; +} diff --git a/ops/kontemplate/templater/dns.go b/ops/kontemplate/templater/dns.go new file mode 100644 index 000000000000..6cd974dd9324 --- /dev/null +++ b/ops/kontemplate/templater/dns.go @@ -0,0 +1,35 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This file contains the implementation of a template function for retrieving +// IP addresses from DNS + +package templater + +import ( + "fmt" + "net" + "os" +) + +func GetIPsFromDNS(host string) ([]interface{}, error) { + fmt.Fprintf(os.Stderr, "Attempting to look up IP for %s in DNS\n", host) + ips, err := net.LookupIP(host) + + if err != nil { + return nil, fmt.Errorf("IP address lookup failed: %v", err) + } + + var result []interface{} = make([]interface{}, len(ips)) + for i, ip := range ips { + result[i] = ip + } + + return result, nil +} diff --git a/ops/kontemplate/templater/pass.go b/ops/kontemplate/templater/pass.go new file mode 100644 index 000000000000..f7fbcb433de5 --- /dev/null +++ b/ops/kontemplate/templater/pass.go @@ -0,0 +1,34 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This file contains the implementation of a template function for retrieving +// variables from 'pass', the standard UNIX password manager. + +package templater + +import ( + "fmt" + "os" + "os/exec" + "strings" +) + +func GetFromPass(key string) (string, error) { + fmt.Fprintf(os.Stderr, "Attempting to look up %s in pass\n", key) + pass := exec.Command("pass", "show", key) + + output, err := pass.CombinedOutput() + if err != nil { + return "", fmt.Errorf("Pass lookup failed: %s (%v)", output, err) + } + + trimmed := strings.TrimSpace(string(output)) + + return trimmed, nil +} diff --git a/ops/kontemplate/templater/templater.go b/ops/kontemplate/templater/templater.go new file mode 100644 index 000000000000..a8f0c670a603 --- /dev/null +++ b/ops/kontemplate/templater/templater.go @@ -0,0 +1,236 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package templater + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "strings" + "text/template" + + "github.com/Masterminds/sprig" + "github.com/tazjin/kontemplate/context" + "github.com/tazjin/kontemplate/util" +) + +const failOnMissingKeys string = "missingkey=error" + +type RenderedResource struct { + Filename string + Rendered string +} + +type RenderedResourceSet struct { + Name string + Resources []RenderedResource + Args []string +} + +func LoadAndApplyTemplates(include *[]string, exclude *[]string, c *context.Context) ([]RenderedResourceSet, error) { + limitedResourceSets := applyLimits(&c.ResourceSets, include, exclude) + renderedResourceSets := make([]RenderedResourceSet, 0) + + if len(*limitedResourceSets) == 0 { + return renderedResourceSets, fmt.Errorf("No valid resource sets included!") + } + + for _, rs := range *limitedResourceSets { + set, err := processResourceSet(c, &rs) + + if err != nil { + return nil, err + } + + renderedResourceSets = append(renderedResourceSets, *set) + } + + return renderedResourceSets, nil +} + +func processResourceSet(ctx *context.Context, rs *context.ResourceSet) (*RenderedResourceSet, error) { + fmt.Fprintf(os.Stderr, "Loading resources for %s\n", rs.Name) + + fileInfo, err := os.Stat(rs.Path) + if err != nil { + return nil, err + } + + var files []os.FileInfo + var resources []RenderedResource + + // Treat single-file resource paths separately from resource + // sets containing multiple templates + if fileInfo.IsDir() { + // Explicitly discard this error, which will give us an empty + // list of files instead. + // This will end up printing a warning to the user, but it + // won't stop the rest of the process. + files, _ = ioutil.ReadDir(rs.Path) + resources, err = processFiles(ctx, rs, files) + if err != nil { + return nil, err + } + } else { + resource, err := templateFile(ctx, rs, rs.Path) + if err != nil { + return nil, err + } + + resources = []RenderedResource{resource} + } + + return &RenderedResourceSet{ + Name: rs.Name, + Resources: resources, + Args: rs.Args, + }, nil +} + +func processFiles(ctx *context.Context, rs *context.ResourceSet, files []os.FileInfo) ([]RenderedResource, error) { + resources := make([]RenderedResource, 0) + + for _, file := range files { + if !file.IsDir() && isResourceFile(file) { + path := path.Join(rs.Path, file.Name()) + res, err := templateFile(ctx, rs, path) + + if err != nil { + return resources, err + } + + resources = append(resources, res) + } + } + + return resources, nil +} + +func templateFile(ctx *context.Context, rs *context.ResourceSet, filepath string) (RenderedResource, error) { + var resource RenderedResource + + tpl, err := template.New(path.Base(filepath)).Funcs(templateFuncs(ctx, rs)).Option(failOnMissingKeys).ParseFiles(filepath) + if err != nil { + return resource, fmt.Errorf("Could not load template %s: %v", filepath, err) + } + + var b bytes.Buffer + err = tpl.Execute(&b, rs.Values) + if err != nil { + return resource, fmt.Errorf("Error while templating %s: %v", filepath, err) + } + + resource = RenderedResource{ + Filename: path.Base(filepath), + Rendered: b.String(), + } + + return resource, nil +} + +// Applies the limits of explicitly included or excluded resources and returns the updated resource set. +// Exclude takes priority over include +func applyLimits(rs *[]context.ResourceSet, include *[]string, exclude *[]string) *[]context.ResourceSet { + if len(*include) == 0 && len(*exclude) == 0 { + return rs + } + + // Exclude excluded resource sets + excluded := make([]context.ResourceSet, 0) + for _, r := range *rs { + if !matchesResourceSet(exclude, &r) { + excluded = append(excluded, r) + } + } + + // Include included resource sets + if len(*include) == 0 { + return &excluded + } + included := make([]context.ResourceSet, 0) + for _, r := range excluded { + if matchesResourceSet(include, &r) { + included = append(included, r) + } + } + + return &included +} + +// Check whether an include/exclude string slice matches a resource set +func matchesResourceSet(s *[]string, rs *context.ResourceSet) bool { + for _, r := range *s { + r = strings.TrimSuffix(r, "/") + if r == rs.Name || r == rs.Parent { + return true + } + } + + return false +} + +func templateFuncs(c *context.Context, rs *context.ResourceSet) template.FuncMap { + m := sprig.TxtFuncMap() + m["json"] = func(data interface{}) string { + b, _ := json.Marshal(data) + return string(b) + } + m["passLookup"] = GetFromPass + m["gitHEAD"] = func() (string, error) { + out, err := exec.Command("git", "-C", c.BaseDir, "rev-parse", "HEAD").Output() + if err != nil { + return "", err + } + output := strings.TrimSpace(string(out)) + return output, nil + } + m["lookupIPAddr"] = GetIPsFromDNS + m["insertFile"] = func(file string) (string, error) { + data, err := ioutil.ReadFile(path.Join(rs.Path, file)) + if err != nil { + return "", err + } + + return string(data), nil + } + m["insertTemplate"] = func(file string) (string, error) { + data, err := templateFile(c, rs, path.Join(rs.Path, file)) + if err != nil { + return "", err + } + + return data.Rendered, nil + } + m["default"] = func(defaultVal interface{}, varName string) interface{} { + if val, ok := rs.Values[varName]; ok { + return val + } + + return defaultVal + } + return m +} + +// Checks whether a file is a resource file (i.e. is YAML or JSON) and not a default values file. +func isResourceFile(f os.FileInfo) bool { + for _, defaultFile := range util.DefaultFilenames { + if f.Name() == defaultFile { + return false + } + } + + return strings.HasSuffix(f.Name(), "yaml") || + strings.HasSuffix(f.Name(), "yml") || + strings.HasSuffix(f.Name(), "json") +} diff --git a/ops/kontemplate/templater/templater_test.go b/ops/kontemplate/templater/templater_test.go new file mode 100644 index 000000000000..c20858c203b4 --- /dev/null +++ b/ops/kontemplate/templater/templater_test.go @@ -0,0 +1,205 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package templater + +import ( + "github.com/tazjin/kontemplate/context" + "reflect" + "strings" + "testing" +) + +func TestApplyNoLimits(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet2", + }, + } + + result := applyLimits(&resources, &[]string{}, &[]string{}) + + if !reflect.DeepEqual(resources, *result) { + t.Error("Resource set slice changed, but shouldn't have.") + t.Errorf("Expected: %v\nResult: %v\n", resources, *result) + t.Fail() + } +} + +func TestApplyIncludeLimits(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet2", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + includes := []string{"testResourceSet1", "included"} + + result := applyLimits(&resources, &includes, &[]string{}) + + expected := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + if !reflect.DeepEqual(expected, *result) { + t.Error("Result does not contain expected resource sets.") + t.Errorf("Expected: %v\nResult: %v\n", expected, *result) + t.Fail() + } +} + +func TestApplyExcludeLimits(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet2", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + exclude := []string{"testResourceSet2"} + + result := applyLimits(&resources, &[]string{}, &exclude) + + expected := []context.ResourceSet{ + { + Name: "testResourceSet1", + }, + { + Name: "testResourceSet3", + Parent: "included", + }, + } + + if !reflect.DeepEqual(expected, *result) { + t.Error("Result does not contain expected resource sets.") + t.Errorf("Expected: %v\nResult: %v\n", expected, *result) + t.Fail() + } +} + +func TestApplyLimitsExcludeIncludePrecedence(t *testing.T) { + resources := []context.ResourceSet{ + { + Name: "collection/nested1", + Parent: "collection", + }, + { + Name: "collection/nested2", + Parent: "collection", + }, + { + Name: "collection/nested3", + Parent: "collection", + }, + { + Name: "something-else", + }, + } + + include := []string{"collection"} + exclude := []string{"collection/nested2"} + + result := applyLimits(&resources, &include, &exclude) + + expected := []context.ResourceSet{ + { + Name: "collection/nested1", + Parent: "collection", + }, + { + Name: "collection/nested3", + Parent: "collection", + }, + } + + if !reflect.DeepEqual(expected, *result) { + t.Error("Result does not contain expected resource sets.") + t.Errorf("Expected: %v\nResult: %v\n", expected, *result) + t.Fail() + } +} + +func TestFailOnMissingKeys(t *testing.T) { + ctx := context.Context{} + resourceSet := context.ResourceSet{} + + _, err := templateFile(&ctx, &resourceSet, "testdata/test-template.txt") + + if err == nil { + t.Errorf("Template with missing keys should have failed.\n") + t.Fail() + } + + if !strings.Contains(err.Error(), "map has no entry for key \"testName\"") { + t.Errorf("Templating failed with unexpected error: %v\n", err) + } +} + +func TestDefaultTemplateFunction(t *testing.T) { + ctx := context.Context{} + resourceSet := context.ResourceSet{} + + res, err := templateFile(&ctx, &resourceSet, "testdata/test-default.txt") + + if err != nil { + t.Errorf("Templating with default values should have succeeded.\n") + t.Fail() + } + + if res.Rendered != "defaultValue\n" { + t.Error("Result does not contain expected rendered default value.") + t.Fail() + } +} + +func TestInsertTemplateFunction(t *testing.T) { + ctx := context.Context{} + resourceSet := context.ResourceSet{ + Path: "testdata", + Values: map[string]interface{}{ + "testName": "TestInsertTemplateFunction", + }, + } + + res, err := templateFile(&ctx, &resourceSet, "testdata/test-insertTemplate.txt") + + if err != nil { + t.Error(err) + t.Errorf("Templating with an insertTemplate call should have succeeded.\n") + t.Fail() + } + + if res.Rendered != "Inserting \"Template for test TestInsertTemplateFunction\".\n" { + t.Error("Result does not contain expected rendered template value.") + t.Error(res.Rendered) + t.Fail() + } +} diff --git a/ops/kontemplate/templater/testdata/test-default.txt b/ops/kontemplate/templater/testdata/test-default.txt new file mode 100644 index 000000000000..4f7997bd69d2 --- /dev/null +++ b/ops/kontemplate/templater/testdata/test-default.txt @@ -0,0 +1 @@ +{{ default "defaultValue" "missingVar" }} diff --git a/ops/kontemplate/templater/testdata/test-insertTemplate.txt b/ops/kontemplate/templater/testdata/test-insertTemplate.txt new file mode 100644 index 000000000000..8155e174fe00 --- /dev/null +++ b/ops/kontemplate/templater/testdata/test-insertTemplate.txt @@ -0,0 +1 @@ +Inserting "{{ insertTemplate "test-template.txt" | trim }}". diff --git a/ops/kontemplate/templater/testdata/test-template.txt b/ops/kontemplate/templater/testdata/test-template.txt new file mode 100644 index 000000000000..06f1cfc630c3 --- /dev/null +++ b/ops/kontemplate/templater/testdata/test-template.txt @@ -0,0 +1 @@ +Template for test {{ .testName }} diff --git a/ops/kontemplate/util/util.go b/ops/kontemplate/util/util.go new file mode 100644 index 000000000000..56fa1e3fc9c5 --- /dev/null +++ b/ops/kontemplate/util/util.go @@ -0,0 +1,58 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package util + +import ( + "io/ioutil" + + "github.com/ghodss/yaml" +) + +// Filenames excluded from templating for the purpose of containing default variable values inside a resource set. +var DefaultFilenames []string = []string{"default.yml", "default.yaml", "default.json"} + +// Merges two maps together. Values from the second map override values in the first map. +// The returned map is new if anything was changed. +func Merge(in1 *map[string]interface{}, in2 *map[string]interface{}) *map[string]interface{} { + if in1 == nil || len(*in1) == 0 { + return in2 + } + + if in2 == nil || len(*in2) == 0 { + return in1 + } + + new := make(map[string]interface{}) + for k, v := range *in1 { + new[k] = v + } + + for k, v := range *in2 { + new[k] = v + } + + return &new +} + +// Loads either a YAML or JSON file from the specified path and +// deserialises it into the provided interface. +func LoadData(filename string, addr interface{}) error { + file, err := ioutil.ReadFile(filename) + if err != nil { + return err + } + + err = yaml.Unmarshal(file, addr) + if err != nil { + return err + } + + return nil +} diff --git a/ops/kontemplate/util/util_test.go b/ops/kontemplate/util/util_test.go new file mode 100644 index 000000000000..53c56081758c --- /dev/null +++ b/ops/kontemplate/util/util_test.go @@ -0,0 +1,83 @@ +// Copyright (C) 2016-2019 Vincent Ambo <mail@tazj.in> +// +// This file is part of Kontemplate. +// +// Kontemplate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package util + +import ( + "reflect" + "testing" +) + +func TestMergeWithEmptyMap(t *testing.T) { + testMap := map[string]interface{}{ + "foo": "bar", + } + + empty := make(map[string]interface{}) + + res1 := Merge(&testMap, &empty) + res2 := Merge(&empty, &testMap) + + if res1 != &testMap || res2 != &testMap { + t.Error("A new map was returned incorrectly.") + t.Fail() + } +} + +func TestMergeWithNilMap(t *testing.T) { + testMap := map[string]interface{}{ + "foo": "bar", + } + + res1 := Merge(&testMap, nil) + res2 := Merge(nil, &testMap) + + if res1 != &testMap || res2 != &testMap { + t.Error("A new map was returned incorrectly.") + t.Fail() + } +} + +func TestMergeMaps(t *testing.T) { + map1 := map[string]interface{}{ + "foo": "bar", + } + + map2 := map[string]interface{}{ + "bar": "baz", + } + + result := Merge(&map1, &map2) + expected := map[string]interface{}{ + "foo": "bar", + "bar": "baz", + } + + if !reflect.DeepEqual(*result, expected) { + t.Error("Maps were merged incorrectly.") + t.Fail() + } +} + +func TestMergeMapsPrecedence(t *testing.T) { + map1 := map[string]interface{}{ + "foo": "incorrect", + } + + map2 := map[string]interface{}{ + "foo": "correct", + } + + result := Merge(&map1, &map2) + + if (*result)["foo"] != "correct" { + t.Error("Map merge precedence test failed.") + t.Fail() + } +} diff --git a/ops/machines/all-systems.nix b/ops/machines/all-systems.nix new file mode 100644 index 000000000000..b80436a06c4a --- /dev/null +++ b/ops/machines/all-systems.nix @@ -0,0 +1,15 @@ +{ depot, ... }: + +(with depot.ops.machines; [ + whitby +]) ++ + +(with depot.users.tazjin.nixos; [ + camden + frog + tverskoy +]) ++ + +(with depot.users.grfn.system.system; [ + yeren +]) diff --git a/ops/machines/whitby/OWNERS b/ops/machines/whitby/OWNERS new file mode 100644 index 000000000000..b1b749e871e1 --- /dev/null +++ b/ops/machines/whitby/OWNERS @@ -0,0 +1,6 @@ +inherited: false + +# Want in on this list? Try paying! +owners: + - lukegb + - tazjin diff --git a/ops/machines/whitby/README.md b/ops/machines/whitby/README.md new file mode 100644 index 000000000000..55287c541256 --- /dev/null +++ b/ops/machines/whitby/README.md @@ -0,0 +1,5 @@ +whitby +====== + +`whitby.tvl.fyi` is our dedicated server providing continuous +integration services and other random nonsense. diff --git a/ops/machines/whitby/default.nix b/ops/machines/whitby/default.nix new file mode 100644 index 000000000000..20abd72be8e2 --- /dev/null +++ b/ops/machines/whitby/default.nix @@ -0,0 +1,600 @@ +{ depot, lib, pkgs, ... }: # readTree options +{ config, ... }: # passed by module system + +let + inherit (builtins) listToAttrs; + inherit (lib) range; +in { + imports = [ + "${depot.path}/ops/modules/atward.nix" + "${depot.path}/ops/modules/automatic-gc.nix" + "${depot.path}/ops/modules/clbot.nix" + "${depot.path}/ops/modules/git-serving.nix" + "${depot.path}/ops/modules/irccat.nix" + "${depot.path}/ops/modules/monorepo-gerrit.nix" + "${depot.path}/ops/modules/nixery.nix" + "${depot.path}/ops/modules/owothia.nix" + "${depot.path}/ops/modules/panettone.nix" + "${depot.path}/ops/modules/paroxysm.nix" + "${depot.path}/ops/modules/restic.nix" + "${depot.path}/ops/modules/smtprelay.nix" + "${depot.path}/ops/modules/sourcegraph.nix" + "${depot.path}/ops/modules/tvl-buildkite.nix" + "${depot.path}/ops/modules/tvl-slapd/default.nix" + "${depot.path}/ops/modules/tvl-sso/default.nix" + "${depot.path}/ops/modules/www/atward.tvl.fyi.nix" + "${depot.path}/ops/modules/www/b.tvl.fyi.nix" + "${depot.path}/ops/modules/www/cache.tvl.su.nix" + "${depot.path}/ops/modules/www/cl.tvl.fyi.nix" + "${depot.path}/ops/modules/www/code.tvl.fyi.nix" + "${depot.path}/ops/modules/www/cs.tvl.fyi.nix" + "${depot.path}/ops/modules/www/deploys.tvl.fyi.nix" + "${depot.path}/ops/modules/www/images.tvl.fyi.nix" + "${depot.path}/ops/modules/www/login.tvl.fyi.nix" + "${depot.path}/ops/modules/www/nixery.dev.nix" + "${depot.path}/ops/modules/www/static.tvl.fyi.nix" + "${depot.path}/ops/modules/www/status.tvl.su.nix" + "${depot.path}/ops/modules/www/tazj.in.nix" + "${depot.path}/ops/modules/www/todo.tvl.fyi.nix" + "${depot.path}/ops/modules/www/tvl.fyi.nix" + "${depot.path}/ops/modules/www/tvl.su.nix" + "${depot.path}/ops/modules/www/wigglydonke.rs.nix" + "${pkgs.path}/nixos/modules/services/web-apps/gerrit.nix" + ]; + + hardware = { + enableRedistributableFirmware = true; + cpu.amd.updateMicrocode = true; + }; + + boot = { + tmpOnTmpfs = true; + kernelModules = [ "kvm-amd" ]; + supportedFilesystems = [ "zfs" ]; + + initrd = { + availableKernelModules = [ + "igb" "xhci_pci" "nvme" "ahci" "usbhid" "usb_storage" "sr_mod" + ]; + + # Enable SSH in the initrd so that we can enter disk encryption + # passwords remotely. + network = { + enable = true; + ssh = { + enable = true; + port = 2222; + authorizedKeys = + depot.users.tazjin.keys.all + ++ depot.users.lukegb.keys.all + ++ [ depot.users.grfn.keys.whitby ]; + + hostKeys = [ + /etc/secrets/initrd_host_ed25519_key + ]; + }; + + # this will launch the zfs password prompt on login and kill the + # other prompt + postCommands = '' + echo "zfs load-key -a && killall zfs" >> /root/.profile + ''; + }; + }; + + kernel.sysctl = { + "net.ipv4.tcp_congestion_control" = "bbr"; + }; + + loader.grub = { + enable = true; + version = 2; + efiSupport = true; + efiInstallAsRemovable = true; + device = "/dev/disk/by-id/nvme-SAMSUNG_MZQLB1T9HAJR-00007_S439NA0N201620"; + }; + + zfs.requestEncryptionCredentials = true; + }; + + fileSystems = { + "/" = { + device = "zroot/root"; + fsType = "zfs"; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/073E-7FBD"; + fsType = "vfat"; + }; + + "/nix" = { + device = "zroot/nix"; + fsType = "zfs"; + }; + + "/home" = { + device = "zroot/home"; + fsType = "zfs"; + }; + }; + + networking = { + # Glass is boring, but Luke doesn't like Wapping - the Prospect of + # Whitby, however, is quite a pleasant establishment. + hostName = "whitby"; + domain = "tvl.fyi"; + hostId = "b38ca543"; + useDHCP = false; + + # Don't use Hetzner's DNS servers. + nameservers = [ + "8.8.8.8" + "8.8.4.4" + ]; + + defaultGateway6 = { + address = "fe80::1"; + interface = "enp196s0"; + }; + + firewall.allowedTCPPorts = [ 22 80 443 4238 8443 29418 ]; + firewall.allowedUDPPorts = [ 8443 ]; + + interfaces.enp196s0.useDHCP = true; + interfaces.enp196s0.ipv6.addresses = [ + { + address = "2a01:04f8:0242:5b21::feed:edef:beef"; + prefixLength = 64; + } + ]; + }; + + # Generate an immutable /etc/resolv.conf from the nameserver settings + # above (otherwise DHCP overwrites it): + environment.etc."resolv.conf" = with lib; { + source = pkgs.writeText "resolv.conf" '' + ${concatStringsSep "\n" (map (ns: "nameserver ${ns}") config.networking.nameservers)} + options edns0 + ''; + }; + + # Disable background git gc system-wide, as it has a tendency to break CI. + environment.etc."gitconfig".source = pkgs.writeText "gitconfig" '' + [gc] + autoDetach = false + ''; + + time.timeZone = "UTC"; + + nix = { + nrBuildUsers = 256; + maxJobs = lib.mkDefault 64; + extraOptions = '' + secret-key-files = /etc/secrets/nix-cache-privkey + ''; + + trustedUsers = [ + "grfn" + "lukegb" + "tazjin" + "sterni" + ]; + + sshServe = { + enable = true; + keys = with depot.users; + tazjin.keys.all + ++ lukegb.keys.all + ++ [ grfn.keys.whitby ] + ++ sterni.keys.all + ; + }; + }; + + programs.mtr.enable = true; + programs.mosh.enable = true; + services.openssh = { + enable = true; + passwordAuthentication = false; + challengeResponseAuthentication = false; + }; + + # Automatically collect garbage from the Nix store. + services.depot.automatic-gc = { + enable = true; + interval = "1 hour"; + diskThreshold = 200; # GiB + maxFreed = 420; # GiB + preserveGenerations = "90d"; + }; + + # Run a handful of Buildkite agents to support parallel builds. + services.depot.buildkite = { + enable = true; + agentCount = 32; + }; + + # Start a local SMTP relay to Gmail (used by gerrit) + services.depot.smtprelay = { + enable = true; + args = { + listen = ":2525"; + remote_host = "smtp.gmail.com:587"; + remote_auth = "plain"; + remote_user = "tvlbot@tazj.in"; + }; + }; + + # Start a ZNC instance which bounces for tvlbot and owothia. + services.znc = { + enable = true; + useLegacyConfig = false; + config = { + LoadModule = [ + "webadmin" + "adminlog" + ]; + + User.admin = { + Admin = true; + Pass.password = { + Method = "sha256"; + Hash = "bb00aa8239de484c2925b1c3f6a196fb7612633f001daa9b674f83abe7e1103f"; + Salt = "TiB0Ochb1CrtpMTl;2;j"; + }; + }; + + Listener.l = { + Host = "localhost"; + Port = 2627; # bncr + SSL = false; + }; + }; + }; + + # Start the Gerrit->IRC bot + services.depot.clbot = { + enable = true; + channels = [ "#tvl" ]; + + # See //fun/clbot for details. + flags = { + gerrit_host = "cl.tvl.fyi:29418"; + gerrit_ssh_auth_username = "clbot"; + gerrit_ssh_auth_key = "/etc/secrets/id_clbot"; + + irc_server = "localhost:${toString config.services.znc.config.Listener.l.Port}"; + irc_user = "tvlbot"; + irc_nick = "tvlbot"; + + notify_branches = "canon,refs/meta/config"; + notify_repo = "depot"; + + # This secret is read from an environment variable, which is + # populated from /etc/secrets/clbot + irc_pass = "$CLBOT_PASS"; + }; + }; + + services.depot = { + # Run a SourceGraph code search instance + sourcegraph.enable = true; + + # Run the Panettone issue tracker + panettone = { + enable = true; + dbUser = "panettone"; + dbName = "panettone"; + secretsFile = "/etc/secrets/panettone"; + irccatChannel = "#tvl"; + }; + + # Run the first cursed bot (quote bot) + paroxysm.enable = true; + + # Run the second cursed bot + owothia = { + enable = true; + ircServer = "localhost"; + ircPort = config.services.znc.config.Listener.l.Port; + }; + + # Run irccat to forward messages to IRC + irccat = { + enable = true; + config = { + tcp.listen = ":4722"; # "ircc" + irc = { + server = "localhost:${toString config.services.znc.config.Listener.l.Port}"; + tls = false; + nick = "tvlbot"; + # Note: irccat means 'ident' where it says 'realname', so + # this is critical for connecting to ZNC. + realname = "tvlbot"; + channels = [ + "#tvl" + ]; + }; + }; + }; + + # Run atward, the search engine redirection thing. + atward.enable = true; + + # Run a Nixery instance + nixery.enable = true; + + # Run cgit & josh to serve git + git-serving.enable = true; + + # Configure backups to GleSYS + restic = { + enable = true; + paths = [ + "/var/backup/postgresql" + "/var/lib/grafana" + "/var/lib/znc" + ]; + }; + }; + + services.postgresql = { + enable = true; + enableTCPIP = true; + + authentication = lib.mkForce '' + local all all trust + host all all 127.0.0.1/32 password + host all all ::1/128 password + hostnossl all all 127.0.0.1/32 password + hostnossl all all ::1/128 password + ''; + + ensureDatabases = [ + "panettone" + ]; + + ensureUsers = [{ + name = "panettone"; + ensurePermissions = { + "DATABASE panettone" = "ALL PRIVILEGES"; + }; + }]; + }; + + services.postgresqlBackup = { + enable = true; + databases = [ + "tvldb" + "panettone" + ]; + }; + + services.shadowsocks = { + enable = true; + port = 8443; + passwordFile = "/etc/secrets/shadowsocks-secret.sec"; + }; + + services.nix-serve = { + enable = true; + port = 6443; + secretKeyFile = "/etc/secrets/nix-cache-key.sec"; + bindAddress = "localhost"; + }; + + services.fail2ban.enable = true; + + environment.systemPackages = with pkgs; [ + bb + curl + emacs-nox + git + htop + nano + rxvt_unicode.terminfo + vim + zfs + zfstools + ]; + + services.journaldriver = { + enable = true; + googleCloudProject = "tvl-fyi"; + logStream = "whitby"; + applicationCredentials = "/var/lib/journaldriver/key.json"; + }; + + # Configure Prometheus & Grafana. Exporter configuration for + # Prometheus is inside the respective service modules. + services.prometheus = { + enable = true; + exporters.node = { + enable = true; + + enabledCollectors = [ + "logind" + "processes" + "systemd" + ]; + }; + + scrapeConfigs = [{ + job_name = "node"; + scrape_interval = "5s"; + static_configs = [{ + targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; + }]; + }]; + }; + + services.grafana = { + enable = true; + port = 4723; # "graf" on phone keyboard + domain = "status.tvl.su"; + rootUrl = "https://status.tvl.su"; + analytics.reporting.enable = false; + extraOptions = let + options = { + auth = { + generic_oauth = { + enabled = true; + client_id = "OAUTH-TVL-grafana-f1A1EmHLDT"; + scopes = "openid profile email"; + name = "TVL"; + email_attribute_path = "mail"; + login_attribute_path = "sub"; + name_attribute_path = "displayName"; + auth_url = "https://login.tvl.fyi/oidc/authorize"; + token_url = "https://login.tvl.fyi/oidc/accessToken"; + api_url = "https://login.tvl.fyi/oidc/profile"; + + # Give lukegb, grfn, tazjin "Admin" rights. + role_attribute_path = "((sub == 'lukegb' || sub == 'grfn' || sub == 'tazjin') && 'Admin') || 'Editor'"; + + # Allow creating new Grafana accounts from OAuth accounts. + allow_sign_up = true; + }; + anonymous = { + enabled = true; + org_name = "The Virus Lounge"; + org_role = "Viewer"; + }; + basic.enabled = false; + oauth_auto_login = true; + disable_login_form = true; + }; + }; + inherit (builtins) typeOf replaceStrings listToAttrs concatLists; + inherit (lib) toUpper mapAttrsToList nameValuePair concatStringsSep; + + # Take ["auth" "generic_oauth" "enabled"] and turn it into OPTIONS_GENERIC_OAUTH_ENABLED. + encodeName = raw: replaceStrings ["."] ["_"] (toUpper (concatStringsSep "_" raw)); + + # Turn an option value into a string, but we want bools to be sensible strings and not "1" or "". + optionToString = value: + if (typeOf value) == "bool" then + if value then "true" else "false" + else builtins.toString value; + + # Turn an nested options attrset into a flat listToAttrs-compatible list. + encodeOptions = prefix: inp: concatLists (mapAttrsToList (name: value: + if (typeOf value) == "set" + then encodeOptions (prefix ++ [name]) value + else [ (nameValuePair (encodeName (prefix ++ [name])) (optionToString value)) ] + ) inp); + in listToAttrs (encodeOptions [] options); + + provision = { + enable = true; + datasources = [{ + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + }]; + }; + }; + # Contains GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET. + systemd.services.grafana.serviceConfig.EnvironmentFile = "/etc/secrets/grafana"; + + security.sudo.extraRules = [ + { + groups = ["wheel"]; + commands = [{ command = "ALL"; options = ["NOPASSWD"]; }]; + } + ]; + + users = { + users.tazjin = { + isNormalUser = true; + extraGroups = [ "git" "wheel" ]; + shell = pkgs.fish; + openssh.authorizedKeys.keys = depot.users.tazjin.keys.all; + }; + + users.lukegb = { + isNormalUser = true; + extraGroups = [ "git" "wheel" ]; + openssh.authorizedKeys.keys = depot.users.lukegb.keys.all; + }; + + users.grfn = { + isNormalUser = true; + extraGroups = [ "git" "wheel" ]; + openssh.authorizedKeys.keys = [ + depot.users.grfn.keys.whitby + ]; + }; + + users.isomer = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.isomer.keys.all; + }; + + users.riking = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.riking.keys.u2f ++ depot.users.riking.keys.passworded; + }; + + users.edef = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.edef.keys.all; + }; + + users.qyliss = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.qyliss.keys.all; + }; + + users.eta = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.eta.keys.whitby; + }; + + users.cynthia = { + isNormalUser = true; # I'm normal OwO :3 + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.cynthia.keys.all; + }; + + users.firefly = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.firefly.keys.whitby; + }; + + users.sterni = { + isNormalUser = true; + extraGroups = [ "git" "wheel" ]; + openssh.authorizedKeys.keys = depot.users.sterni.keys.all; + }; + + users.flokli = { + isNormalUser = true; + extraGroups = [ "git" ]; + openssh.authorizedKeys.keys = depot.users.flokli.keys.all; + }; + + # Set up a user & group for git shenanigans + groups.git = {}; + users.git = { + group = "git"; + isSystemUser = true; + createHome = true; + home = "/var/lib/git"; + }; + }; + + security.acme = { + acceptTerms = true; + email = "certs@tvl.fyi"; + }; + + system.stateVersion = "20.03"; +} diff --git a/ops/modules/.skip-subtree b/ops/modules/.skip-subtree new file mode 100644 index 000000000000..09520f8c831f --- /dev/null +++ b/ops/modules/.skip-subtree @@ -0,0 +1 @@ +NixOS modules are not readTree compatible. diff --git a/ops/modules/README.md b/ops/modules/README.md new file mode 100644 index 000000000000..595b4c3344c6 --- /dev/null +++ b/ops/modules/README.md @@ -0,0 +1,7 @@ +NixOS modules +============= + +This folder contains various NixOS modules shared by our NixOS +configurations. + +It is not read by `readTree`. diff --git a/ops/modules/atward.nix b/ops/modules/atward.nix new file mode 100644 index 000000000000..354f9ebdd3cb --- /dev/null +++ b/ops/modules/atward.nix @@ -0,0 +1,37 @@ +{ depot, config, lib, pkgs, ... }: + +let + cfg = config.services.depot.atward; + description = "atward - (attempt to) cleverly route queries"; +in { + options.services.depot.atward = { + enable = lib.mkEnableOption description; + + host = lib.mkOption { + type = lib.types.str; + default = "[::1]"; + description = "Host on which atward should listen"; + }; + + port = lib.mkOption { + type = lib.types.int; + default = 28973; + description = "Port on which atward should listen"; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.atward = { + inherit description; + script = "${depot.web.atward}/bin/atward"; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + DynamicUser = true; + Restart = "always"; + }; + + environment.ATWARD_LISTEN_ADDRESS = "${cfg.host}:${toString cfg.port}"; + }; + }; +} diff --git a/ops/modules/automatic-gc.nix b/ops/modules/automatic-gc.nix new file mode 100644 index 000000000000..634785721024 --- /dev/null +++ b/ops/modules/automatic-gc.nix @@ -0,0 +1,91 @@ +# Defines a service for automatically collecting Nix garbage +# periodically, without relying on the (ostensibly broken) Nix options +# for min/max space available. +{ config, lib, pkgs, ... }: + +let + cfg = config.services.depot.automatic-gc; + description = "Automatically collect Nix garbage"; + + GiBtoKiB = n: n * 1024 * 1024; + GiBtoBytes = n: n * 1024 * 1024 * 1024; + + gcScript = pkgs.writeShellScript "automatic-nix-gc" '' + set -ueo pipefail + + readonly MIN_THRESHOLD_KIB="${toString (GiBtoKiB cfg.diskThreshold)}" + readonly MAX_FREED_BYTES="${toString (GiBtoBytes cfg.maxFreed)}" + readonly GEN_THRESHOLD="${cfg.preserveGenerations}" + readonly AVAILABLE_KIB=$(df --sync /nix --output=avail | tail -n1) + + if [ "''${AVAILABLE_KIB}" -lt "''${MIN_THRESHOLD_KIB}" ]; then + echo "Have ''${AVAILABLE_KIB} KiB, but want ''${MIN_THRESHOLD_KIB} KiB." + echo "Triggering Nix garbage collection up to ''${MAX_FREED_BYTES} bytes." + set -x + ${config.nix.package}/bin/nix-collect-garbage \ + --delete-older-than "''${GEN_THRESHOLD}" \ + --max-freed "''${MAX_FREED_BYTES}" + else + echo "Skipping GC, enough space available" + fi + ''; +in { + options.services.depot.automatic-gc = { + enable = lib.mkEnableOption description; + + interval = lib.mkOption { + type = lib.types.str; + example = "1h"; + description = '' + Interval between garbage collection runs, specified in + systemd.time(7) format. + ''; + }; + + diskThreshold = lib.mkOption { + type = lib.types.int; + example = "100"; + description = '' + Minimum amount of space that needs to be available (in GiB) on + the partition holding /nix. Garbage collection is triggered if + it falls below this. + ''; + }; + + maxFreed = lib.mkOption { + type = lib.types.int; + example = "420"; + description = '' + Maximum amount of space to free in a single GC run, in GiB. + ''; + }; + + preserveGenerations = lib.mkOption { + type = lib.types.str; + default = "90d"; + description = '' + Preserve NixOS generations younger than the specified value, + in the format expected by nix-collect-garbage(1). + ''; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.automatic-gc = { + inherit description; + script = "${gcScript}"; + serviceConfig.Type = "oneshot"; + }; + + systemd.timers.automatic-gc = { + inherit description; + requisite = [ "nix-daemon.service" ]; + wantedBy = [ "multi-user.target" ]; + + timerConfig = { + OnActiveSec = "1"; + OnUnitActiveSec = cfg.interval; + }; + }; + }; +} diff --git a/ops/modules/clbot.nix b/ops/modules/clbot.nix new file mode 100644 index 000000000000..71ff2fbc3288 --- /dev/null +++ b/ops/modules/clbot.nix @@ -0,0 +1,75 @@ +# Module that configures CLBot, our Gerrit->IRC info bridge. +{ depot, config, lib, pkgs, ... }: + +let + inherit (builtins) attrValues concatStringsSep mapAttrs readFile; + inherit (pkgs) runCommandNoCC; + + inherit (lib) + listToAttrs + mkEnableOption + mkIf + mkOption + removeSuffix + types; + + description = "Bot to forward CL notifications"; + cfg = config.services.depot.clbot; + + mkFlags = flags: + concatStringsSep " " + (attrValues (mapAttrs (key: value: "-${key} \"${toString value}\"") flags)); + + # Escapes a unit name for use in systemd + systemdEscape = name: removeSuffix "\n" (readFile (runCommandNoCC "unit-name" {} '' + ${pkgs.systemd}/bin/systemd-escape '${name}' >> $out + '')); + + mkUnit = flags: channel: { + name = "clbot-${systemdEscape channel}"; + value = { + description = "${description} to ${channel}"; + wantedBy = [ "multi-user.target" ]; + + script = "${depot.fun.clbot}/bin/clbot ${mkFlags (cfg.flags // { + irc_channel = channel; + })} -alsologtostderr"; + + serviceConfig = { + User = "clbot"; + EnvironmentFile = "/etc/secrets/clbot"; + Restart = "always"; + }; + }; + }; +in { + options.services.depot.clbot = { + enable = mkEnableOption description; + + flags = mkOption { + type = types.attrsOf types.str; + description = "Key value pairs for command line flags"; + }; + + channels = mkOption { + type = with types; listOf str; + description = "Channels in which to post (generates one unit per channel)"; + }; + }; + + config = mkIf cfg.enable { + # This does not use DynamicUser because we need to make some files + # (notably the SSH private key) readable by this user outside of + # the module. + users = { + groups.clbot = {}; + + users.clbot = { + group = "clbot"; + isSystemUser = true; + }; + }; + + systemd.services = listToAttrs (map (mkUnit cfg.flags) cfg.channels); + }; +} diff --git a/ops/modules/default.nix b/ops/modules/default.nix new file mode 100644 index 000000000000..8bdfecdf41b0 --- /dev/null +++ b/ops/modules/default.nix @@ -0,0 +1,2 @@ +# Make readTree happy at this level. +_: {} diff --git a/ops/modules/git-serving.nix b/ops/modules/git-serving.nix new file mode 100644 index 000000000000..6b8bef29b15a --- /dev/null +++ b/ops/modules/git-serving.nix @@ -0,0 +1,53 @@ +# Configures public git-serving infrastructure for TVL, this involves: +# +# 1. cgit (running at code.tvl.fyi) for web views of the repository +# 2. josh (for cloning the repository and its distinct subtrees) +# +# We also run Sourcegraph for browsing the repository, but this is +# currently configured in a separate module +# (//ops/modules/sourcegraph.nix) +# +# TODO(tazjin): Move //web/cgit-taz configuration in here instead. +{ config, depot, lib, pkgs, ... }: + +let + cfg = config.services.depot.git-serving; +in { + options.services.depot.git-serving = with lib; { + enable = mkEnableOption "Enable cgit & josh configuration"; + + joshPort = mkOption { + description = "Port on which josh should listen"; + type = types.int; + default = 5674; + }; + }; + + config = lib.mkIf cfg.enable { + # Run cgit for the depot. The onion here is nginx(thttpd(cgit)). + systemd.services.cgit = { + wantedBy = [ "multi-user.target" ]; + script = "${depot.web.cgit-taz}/bin/cgit-launch"; + + serviceConfig = { + Restart = "on-failure"; + User = "git"; + Group = "git"; + }; + }; + + # Run josh for the depot. + systemd.services.josh = { + description = "josh - partial cloning of monorepos"; + wantedBy = [ "multi-user.target" ]; + path = [ pkgs.git pkgs.bash ]; + + serviceConfig = { + DynamicUser = true; + StateDirectory = "josh"; + Restart = "always"; + ExecStart = "${depot.third_party.josh}/bin/josh-proxy --no-background --local /var/lib/josh --port ${toString cfg.joshPort} --remote https://cl.tvl.fyi/"; + }; + }; + }; +} diff --git a/ops/modules/irccat.nix b/ops/modules/irccat.nix new file mode 100644 index 000000000000..e4b30b73553e --- /dev/null +++ b/ops/modules/irccat.nix @@ -0,0 +1,49 @@ +{ depot, config, lib, pkgs, ... }: + +let + cfg = config.services.depot.irccat; + description = "irccat - forward messages to IRC"; + + # irccat expects to read its configuration from the *current + # directory*, and its configuration contains secrets. + # + # To make this work we construct the JSON configuration file and + # then recursively merge it with an on-disk secret using jq on + # service launch. + configJson = pkgs.writeText "irccat.json" (builtins.toJSON cfg.config); + configMerge = pkgs.writeShellScript "merge-irccat-config" '' + if [ ! -f "/etc/secrets/irccat.json" ]; then + echo "irccat secrets file is missing" + exit 1 + fi + + # jq's * is the recursive merge operator + ${pkgs.jq}/bin/jq -s '.[0] * .[1]' ${configJson} /etc/secrets/irccat.json \ + > /var/lib/irccat/irccat.json + ''; +in { + options.services.depot.irccat = { + enable = lib.mkEnableOption description; + + config = lib.mkOption { + type = lib.types.attrs; # varying value types + description = "Configuration structure (unchecked!)"; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.irccat = { + inherit description; + preStart = "${configMerge}"; + script = "${depot.third_party.irccat}/bin/irccat"; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + DynamicUser = true; + StateDirectory = "irccat"; + WorkingDirectory = "/var/lib/irccat"; + Restart = "always"; + }; + }; + }; +} diff --git a/ops/modules/monorepo-gerrit.nix b/ops/modules/monorepo-gerrit.nix new file mode 100644 index 000000000000..57f2edc846bb --- /dev/null +++ b/ops/modules/monorepo-gerrit.nix @@ -0,0 +1,150 @@ +# Gerrit configuration for the TVL monorepo +{ depot, pkgs, config, lib, ... }: + +let + cfg = config.services.gerrit; + + besadiiWithConfig = name: pkgs.writeShellScript "besadii-whitby" '' + export BESADII_CONFIG=/etc/secrets/besadii.json + exec -a ${name} ${depot.ops.besadii}/bin/besadii "$@" + ''; + + gerritHooks = pkgs.runCommandNoCC "gerrit-hooks" {} '' + mkdir -p $out + ln -s ${besadiiWithConfig "change-merged"} $out/change-merged + ln -s ${besadiiWithConfig "patchset-created"} $out/patchset-created + ''; +in { + services.gerrit = { + enable = true; + listenAddress = "[::]:4778"; # 4778 - grrt + serverId = "4fdfa107-4df9-4596-8e0a-1d2bbdd96e36"; + builtinPlugins = [ + "download-commands" + "hooks" + ]; + + plugins = with depot.third_party.gerrit_plugins; [ + owners + oauth + depot.ops.gerrit-tvl + ]; + + package = depot.third_party.gerrit; + + jvmHeapLimit = "4g"; + + # In some NixOS channel bump, the default version of OpenJDK has + # changed to one that is incompatible with our current version of + # Gerrit. + # + # TODO(tazjin): Update Gerrit and remove this when possible. + jvmPackage = pkgs.openjdk11_headless; + + settings = { + core.packedGitLimit = "100m"; + log.jsonLogging = true; + log.textLogging = false; + sshd.advertisedAddress = "code.tvl.fyi:29418"; + hooks.path = "${gerritHooks}"; + cache.web_sessions.maxAge = "3 months"; + plugins.allowRemoteAdmin = false; + change.enableAttentionSet = true; + change.enableAssignee = false; + + # Configures gerrit for being reverse-proxied by nginx as per + # https://gerrit-review.googlesource.com/Documentation/config-reverseproxy.html + gerrit = { + canonicalWebUrl = "https://cl.tvl.fyi"; + docUrl = "/Documentation"; + }; + + httpd.listenUrl = "proxy-https://${cfg.listenAddress}"; + + download.command = [ + "checkout" + "cherry_pick" + "format_patch" + "pull" + ]; + + # Configure for cgit. + gitweb = { + type = "custom"; + url = "https://code.tvl.fyi"; + project = "/"; + revision = "/commit/?id=\${commit}"; + branch = "/log/?h=\${branch}"; + tag = "/tag/?h=\${tag}"; + roottree = "/tree/?h=\${commit}"; + file = "/tree/\${file}?h=\${commit}"; + filehistory = "/log/\${file}?h=\${branch}"; + linkname = "cgit"; + }; + + # Auto-link panettone bug links + commentlink.panettone = { + match = "b/(\\\\d+)"; + html = "<a href=\"https://b.tvl.fyi/issues/$1\">b/$1</a>"; + }; + + # Auto-link other CLs + commentlink.gerrit = { + match = "cl/(\\\\d+)"; + html = "<a href=\"https://cl.tvl.fyi/$1\">cl/$1</a>"; + }; + + # Configures integration with CAS, which then integrates with a variety + # of backends. + auth.type = "OAUTH"; + plugin.gerrit-oauth-provider-cas-oauth = { + root-url = "https://login.tvl.fyi"; + client-id = "OAUTH-TVL-gerrit-Fv0d8Aizz5"; + # client-secret is set in /var/lib/gerrit/etc/secure.config. + }; + + # Allow users to add additional email addresses to their accounts. + oauth.allowRegisterNewEmail = true; + + # Use Gerrit's built-in HTTP passwords, rather than trying to use the + # password against the backing OAuth provider. + auth.gitBasicAuthPolicy = "HTTP"; + + # Email sending (emails are relayed via the tazj.in domain's + # GSuite currently). + # + # Note that sendemail.smtpPass is stored in + # $site_path/etc/secure.config and is *not* controlled by Nix. + # + # Receiving email is not currently supported. + sendemail = { + enable = true; + html = false; + connectTimeout = "10sec"; + from = "TVL Code Review <tvlbot@tazj.in>"; + includeDiff = true; + smtpEncryption = "none"; + smtpServer = "localhost"; + smtpServerPort = 2525; + }; + }; + }; + + systemd.services.gerrit = { + serviceConfig = { + # There seems to be no easy way to get `DynamicUser` to play + # well with other services (e.g. by using SupplementaryGroups, + # which seem to have no effect) so we force the DynamicUser + # setting for the Gerrit service to be disabled and reuse the + # existing 'git' user. + DynamicUser = lib.mkForce false; + User = "git"; + Group = "git"; + }; + }; + + services.depot.restic = { + paths = [ "/var/lib/gerrit" ]; + exclude = [ "/var/lib/gerrit/tmp" ]; + }; +} diff --git a/ops/modules/nixery.nix b/ops/modules/nixery.nix new file mode 100644 index 000000000000..60d151045756 --- /dev/null +++ b/ops/modules/nixery.nix @@ -0,0 +1,42 @@ +# NixOS module to run Nixery, currently with local-storage as the +# backend for storing/serving image layers. +{ depot, config, lib, pkgs, ... }: + +let + cfg = config.services.depot.nixery; + description = "Nixery - container images on-demand"; + storagePath = "/var/lib/nixery/${pkgs.nixpkgsCommits.unstable}"; +in { + options.services.depot.nixery = { + enable = lib.mkEnableOption description; + + port = lib.mkOption { + type = lib.types.int; + default = 45243; # "image" + description = "Port on which Nixery should listen"; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.nixery = { + inherit description; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + DynamicUser = true; + StateDirectory = "nixery"; + Restart = "always"; + ExecStartPre = "${pkgs.coreutils}/bin/mkdir -p ${storagePath}"; + ExecStart = "${depot.third_party.nixery.nixery-bin}/bin/nixery"; + }; + + environment = { + PORT = toString cfg.port; + NIXERY_PKGS_PATH = pkgs.path; + NIXERY_STORAGE_BACKEND = "filesystem"; + NIX_TIMEOUT = "60"; # seconds + STORAGE_PATH = storagePath; + }; + }; + }; +} diff --git a/ops/modules/owothia.nix b/ops/modules/owothia.nix new file mode 100644 index 000000000000..909481873704 --- /dev/null +++ b/ops/modules/owothia.nix @@ -0,0 +1,67 @@ +# Run the owothia IRC bot. +{ depot, config, lib, pkgs, ... }: + +let + cfg = config.services.depot.owothia; + description = "owothia - i'm a service owo"; +in { + options.services.depot.owothia = { + enable = lib.mkEnableOption description; + + secretsFile = lib.mkOption { + type = lib.types.str; + description = "File path from which systemd should read secrets"; + default = "/etc/secrets/owothia"; + }; + + owoChance = lib.mkOption { + type = lib.types.int; + description = "How likely is owo?"; + default = 200; + }; + + ircServer = lib.mkOption { + type = lib.types.str; + description = "IRC server hostname"; + }; + + ircPort = lib.mkOption { + type = lib.types.int; + description = "IRC server port"; + }; + + ircIdent = lib.mkOption { + type = lib.types.str; + description = "IRC username"; + default = "owothia"; + }; + + ircChannels = lib.mkOption { + type = with lib.types; listOf str; + description = "IRC channels to join"; + default = [ "#tvl" ]; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.owothia = { + inherit description; + script = "${depot.fun.owothia}/bin/owothia"; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + DynamicUser = true; + Restart = "always"; + EnvironmentFile = cfg.secretsFile; + }; + + environment = { + OWO_CHANCE = toString cfg.owoChance; + IRC_SERVER = cfg.ircServer; + IRC_PORT = toString cfg.ircPort; + IRC_IDENT = cfg.ircIdent; + IRC_CHANNELS = builtins.toJSON cfg.ircChannels; + }; + }; + }; +} diff --git a/ops/modules/panettone.nix b/ops/modules/panettone.nix new file mode 100644 index 000000000000..51a7468578f4 --- /dev/null +++ b/ops/modules/panettone.nix @@ -0,0 +1,103 @@ +{ depot, config, lib, pkgs, ... }: + +let + cfg = config.services.depot.panettone; +in { + options.services.depot.panettone = with lib; { + enable = mkEnableOption "Panettone issue tracker"; + + port = mkOption { + description = "Port on which Panettone should listen"; + type = types.int; + default = 7268; + }; + + dbHost = mkOption { + description = "Postgresql host to connect to for Panettone"; + type = types.str; + default = "localhost"; + }; + + dbName = mkOption { + description = "Name of the database for Panettone"; + type = types.str; + default = "panettone"; + }; + + dbUser = mkOption { + description = "Name of the database user for Panettone"; + type = types.str; + default = "panettone"; + }; + + secretsFile = mkOption { + description = '' + Path to a file containing secrets, in the format accepted + by systemd's EnvironmentFile + ''; + type = types.str; + }; + + irccatHost = mkOption { + description = "Hostname for the irccat instance"; + type = types.str; + default = "localhost"; + }; + + irccatPort = mkOption { + description = "Port for the irccat instance"; + type = types.int; + default = 4722; + }; + + irccatChannel = mkOption { + description = "IRC channels to post to via irccat"; + type = types.str; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [{ + assertion = + cfg.dbHost != "localhost" || config.services.postgresql.enable; + message = "Panettone requires a postgresql database"; + } { + assertion = + cfg.dbHost != "localhost" || config.services.postgresql.enableTCPIP; + message = "Panettone can only connect to the postgresql database over TCP"; + } { + assertion = + cfg.dbHost != "localhost" || (lib.any + (user: user.name == cfg.dbUser) + config.services.postgresql.ensureUsers); + message = "Panettone requires a database user"; + } { + assertion = + cfg.dbHost != "localhost" || (lib.any + (db: db == cfg.dbName) + config.services.postgresql.ensureDatabases); + message = "Panettone requires a database"; + }]; + + systemd.services.panettone = { + wantedBy = [ "multi-user.target" ]; + script = "${depot.web.panettone}/bin/panettone"; + + serviceConfig = { + DynamicUser = true; + Restart = "always"; + EnvironmentFile = cfg.secretsFile; + }; + + environment = { + PANETTONE_PORT = toString cfg.port; + PGHOST = "localhost"; + PGUSER = cfg.dbUser; + PGDATABASE = cfg.dbName; + IRCCATHOST = cfg.irccatHost; + IRCCATPORT = toString cfg.irccatPort; + ISSUECHANNEL = cfg.irccatChannel; + }; + }; + }; +} diff --git a/ops/modules/paroxysm.nix b/ops/modules/paroxysm.nix new file mode 100644 index 000000000000..cd9cd3866e47 --- /dev/null +++ b/ops/modules/paroxysm.nix @@ -0,0 +1,27 @@ +{ depot, config, lib, pkgs, ... }: + +let + cfg = config.services.depot.paroxysm; + description = "TVL's majestic IRC bot"; +in { + options.services.depot.paroxysm.enable = lib.mkEnableOption description; + + config = lib.mkIf cfg.enable { + systemd.services.paroxysm = { + inherit description; + script = "${depot.fun.paroxysm}/bin/paroxysm"; + wantedBy = [ "multi-user.target" ]; + + environment = { + PARX_DATABASE_URL = "postgresql://tvldb:tvldb@localhost/tvldb"; + PARX_IRC_CONFIG_PATH = "/var/lib/paroxysm/irc.toml"; + }; + + serviceConfig = { + DynamicUser = true; + StateDirectory = "paroxysm"; + Restart = "always"; + }; + }; + }; +} diff --git a/ops/modules/prometheus-fail2ban-exporter.nix b/ops/modules/prometheus-fail2ban-exporter.nix new file mode 100644 index 000000000000..349364f9b7ed --- /dev/null +++ b/ops/modules/prometheus-fail2ban-exporter.nix @@ -0,0 +1,52 @@ +{ config, lib, pkgs, depot, ... }: + +let + cfg = config.services.prometheus-fail2ban-exporter; +in + +{ + options.services.prometheus-fail2ban-exporter = with lib; { + enable = mkEnableOption "Prometheus Fail2ban Exporter"; + + interval = mkOption { + description = "Systemd calendar expression for how often to run the interval"; + type = types.string; + default = "minutely"; + example = "hourly"; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services."prometheus-fail2ban-exporter" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "fail2ban.service" ]; + serviceConfig = { + User = "root"; + Type = "oneshot"; + ExecStart = pkgs.writeShellScript "prometheus-fail2ban-exporter" '' + set -eo pipefail + mkdir -p /var/lib/prometheus/node-exporter + exec prometheus-fail2ban-exporter + ''; + }; + + path = [ + pkgs.fail2ban + depot.third_party.prometheus-fail2ban-exporter + ]; + }; + + systemd.timers."prometheus-fail2ban-exporter" = { + wantedBy = [ "multi-user.target" ]; + timerConfig.OnCalendar = cfg.interval; + }; + + services.prometheus.exporters.node = { + enabledCollectors = [ "textfile" ]; + + extraFlags = [ + "--collector.textfile.directory=/var/lib/prometheus/node-exporter" + ]; + }; + }; +} diff --git a/ops/modules/quassel.nix b/ops/modules/quassel.nix new file mode 100644 index 000000000000..9c8692629a2a --- /dev/null +++ b/ops/modules/quassel.nix @@ -0,0 +1,76 @@ +# A more modern module for running Quassel. +{ config, lib, pkgs, ... }: + +let + cfg = config.services.depot.quassel; + quasselDaemon = pkgs.quassel.override { + monolithic = false; + enableDaemon = true; + withKDE = false; + }; +in { + options.services.depot.quassel = with lib; { + enable = mkEnableOption "Quassel IRC daemon"; + + acmeHost = mkOption { + description = "ACME host to use for the Quassel TLS certificate"; + type = lib.types.str; + }; + + bindAddresses = mkOption { + description = "Addresses Quassel will bind to/listen on"; + default = [ "127.0.0.1" ]; + }; + + logLevel = mkOption { + description = "Log level for Quassel Core"; + default = "Info"; + type = lib.types.enum [ + "Debug" + "Info" + "Warning" + "Error" + ]; + }; + + port = mkOption { + default = 6698; + description = '' + The port number the Quassel daemon will be listening to. + ''; + }; + }; + + config = with lib; mkIf cfg.enable { + systemd.services.quassel = { + description = "Quassel IRC daemon"; + wantedBy = [ "multi-user.target" ]; + + script = concatStringsSep " " [ + "${quasselDaemon}/bin/quasselcore" + "--listen=${concatStringsSep "," cfg.bindAddresses}" + "--port=${toString cfg.port}" + "--configdir=/var/lib/quassel" + "--require-ssl" + "--ssl-cert=/var/lib/acme/${cfg.acmeHost}/full.pem" + "--loglevel=${cfg.logLevel}" + ]; + + serviceConfig = { + Restart = "always"; + User = "quassel"; + Group = "quassel"; + StateDirectory = "quassel"; + }; + }; + + users = { + users.quassel = { + isSystemUser = true; + group = "quassel"; + }; + + groups.quassel = {}; + }; + }; +} diff --git a/ops/modules/restic.nix b/ops/modules/restic.nix new file mode 100644 index 000000000000..1aacf68973e3 --- /dev/null +++ b/ops/modules/restic.nix @@ -0,0 +1,61 @@ +# Configure restic backups to S3-compatible storage, in our case +# GleSYS object storage. +# +# Conventions: +# - restic's cache lives in /var/backup/restic/cache +# - repository password lives in /var/backup/restic/secret +# - object storage credentials in /var/backup/restic/glesys-key +{ config, lib, pkgs, ... }: + +let + cfg = config.services.depot.restic; + description = "Restic backups to GleSYS"; + mkStringOption = default: lib.mkOption { + inherit default; + type = lib.types.str; + }; +in { + options.services.depot.restic = { + enable = lib.mkEnableOption description; + bucketEndpoint = mkStringOption "objects.dc-sto1.glesys.net"; + bucketName = mkStringOption "aged-resonance"; + bucketCredentials = mkStringOption "/var/backup/restic/glesys-key"; + repository = mkStringOption config.networking.hostName; + interval = mkStringOption "hourly"; + + paths = with lib; mkOption { + description = "Directories that should be backed up"; + type = types.listOf types.str; + }; + + exclude = with lib; mkOption { + description = "Files that should be excluded from backups"; + type = types.listOf types.str; + }; + }; + + config = lib.mkIf cfg.enable { + systemd.services.restic = { + description = "Backups to GleSYS"; + + script = "${pkgs.restic}/bin/restic backup ${lib.concatStringsSep " " cfg.paths}"; + + environment = { + RESTIC_REPOSITORY = "s3:${cfg.bucketEndpoint}/${cfg.bucketName}/${cfg.repository}"; + AWS_SHARED_CREDENTIALS_FILE = cfg.bucketCredentials; + RESTIC_PASSWORD_FILE = "/var/backup/restic/secret"; + RESTIC_CACHE_DIR = "/var/backup/restic/cache"; + + RESTIC_EXCLUDE_FILE = + builtins.toFile "exclude-files" (lib.concatStringsSep "\n" cfg.exclude); + }; + }; + + systemd.timers.restic = { + wantedBy = [ "multi-user.target" ]; + timerConfig.OnCalendar = cfg.interval; + }; + + environment.systemPackages = [ pkgs.restic ]; + }; +} diff --git a/ops/modules/smtprelay.nix b/ops/modules/smtprelay.nix new file mode 100644 index 000000000000..d8e03b5794b0 --- /dev/null +++ b/ops/modules/smtprelay.nix @@ -0,0 +1,53 @@ +# NixOS module for configuring the simple SMTP relay. +{ depot, pkgs, config, lib, ... }: + +let + inherit (builtins) attrValues mapAttrs; + inherit (lib) + concatStringsSep + mkEnableOption + mkIf + mkOption + types +; + + cfg = config.services.depot.smtprelay; + description = "Simple SMTP relay"; + + # Configuration values that are always overridden. In particular, + # `config` is specified to always load $StateDirectory/secure.config + # (so that passwords can be loaded from there) and logging is pinned + # to stdout for journald compatibility. + overrideArgs = { + logfile = ""; + config = "/var/lib/smtprelay/secure.config"; + }; + + # Creates the command line argument string for the service. + prepareArgs = args: + concatStringsSep " " + (attrValues (mapAttrs (key: value: "-${key} '${toString value}'") + (args // overrideArgs))); +in { + options.services.depot.smtprelay = { + enable = mkEnableOption description; + args = mkOption { + type = types.attrsOf types.str; + description = "Key value pairs for command line arguments"; + }; + }; + + config = mkIf cfg.enable { + systemd.services.smtprelay = { + inherit description; + script = "${depot.third_party.smtprelay}/bin/smtprelay ${prepareArgs cfg.args}"; + wantedBy = [ "multi-user.target" ]; + + serviceConfig = { + Restart = "always"; + StateDirectory = "smtprelay"; + DynamicUser = true; + }; + }; + }; +} diff --git a/ops/modules/sourcegraph.nix b/ops/modules/sourcegraph.nix new file mode 100644 index 000000000000..a72cd75d477d --- /dev/null +++ b/ops/modules/sourcegraph.nix @@ -0,0 +1,58 @@ +# Run sourcegraph, including its entire machinery, in a container. +# Running it outside of a container is a futile endeavour for now. +{ depot, config, pkgs, lib, ... }: + +let + cfg = config.services.depot.sourcegraph; +in { + options.services.depot.sourcegraph = with lib; { + enable = mkEnableOption "SourceGraph code search engine"; + + port = mkOption { + description = "Port on which SourceGraph should listen"; + type = types.int; + default = 3463; + }; + + cheddarPort = mkOption { + description = "Port on which cheddar should listen"; + type = types.int; + default = 4238; + }; + }; + + config = lib.mkIf cfg.enable { + # Run a cheddar syntax highlighting server + systemd.services.cheddar-server = { + wantedBy = [ "multi-user.target" ]; + script = "${depot.tools.cheddar}/bin/cheddar --listen 0.0.0.0:${toString cfg.cheddarPort} --sourcegraph-server"; + + serviceConfig = { + DynamicUser = true; + Restart = "always"; + }; + }; + + virtualisation.oci-containers.containers.sourcegraph = { + image = "sourcegraph/server:3.31.2"; + + ports = [ + "127.0.0.1:${toString cfg.port}:7080" + ]; + + volumes = [ + "/var/lib/sourcegraph/etc:/etc/sourcegraph" + "/var/lib/sourcegraph/data:/var/opt/sourcegraph" + ]; + + # TODO(tazjin): Figure out what changed in the protocol. + # environment.SRC_SYNTECT_SERVER = "http://172.17.0.1:${toString cfg.cheddarPort}"; + + # Sourcegraph needs a higher nofile limit, it logs warnings + # otherwise (unclear whether it actually affects the service). + extraOptions = [ + "--ulimit" "nofile=10000:10000" + ]; + }; + }; +} diff --git a/ops/modules/tvl-buildkite.nix b/ops/modules/tvl-buildkite.nix new file mode 100644 index 000000000000..a0236a5e7956 --- /dev/null +++ b/ops/modules/tvl-buildkite.nix @@ -0,0 +1,54 @@ +# Configuration for the TVL buildkite agents. +{ config, depot, pkgs, lib, ... }: + +let + cfg = config.services.depot.buildkite; + agents = lib.range 1 cfg.agentCount; + description = "Buildkite agents for TVL"; + + besadiiWithConfig = name: pkgs.writeShellScript "besadii-whitby" '' + export BESADII_CONFIG=/etc/secrets/besadii.json + exec -a ${name} ${depot.ops.besadii}/bin/besadii "$@" + ''; + + # All Buildkite hooks are actually besadii, but it's being invoked + # with different names. + buildkiteHooks = pkgs.runCommandNoCC "buildkite-hooks" {} '' + mkdir -p $out/bin + ln -s ${besadiiWithConfig "post-command"} $out/bin/post-command + ''; +in { + options.services.depot.buildkite = { + enable = lib.mkEnableOption description; + agentCount = lib.mkOption { + type = lib.types.int; + description = "Number of Buildkite agents to launch"; + }; + }; + + config = lib.mkIf cfg.enable { + # Run the Buildkite agents using the default upstream module. + services.buildkite-agents = builtins.listToAttrs (map (n: rec { + name = "whitby-${toString n}"; + value = { + inherit name; + enable = true; + tokenPath = "/etc/secrets/buildkite-agent-token"; + hooks.post-command = "${buildkiteHooks}/bin/post-command"; + }; + }) agents); + + # Set up a group for all Buildkite agent users + users = { + groups.buildkite-agents = {}; + users = builtins.listToAttrs (map (n: rec { + name = "buildkite-agent-whitby-${toString n}"; + value = { + isSystemUser = true; + group = lib.mkForce "buildkite-agents"; + extraGroups = [ name ]; + }; + }) agents); + }; + }; +} diff --git a/ops/modules/tvl-slapd/default.nix b/ops/modules/tvl-slapd/default.nix new file mode 100644 index 000000000000..dbcf139338ea --- /dev/null +++ b/ops/modules/tvl-slapd/default.nix @@ -0,0 +1,80 @@ +# Configures an OpenLDAP instance for TVL +# +# TODO(tazjin): Configure ldaps:// +{ depot, lib, pkgs, ... }: + +with depot.nix.yants; + +let + user = struct { + username = string; + email = string; + password = string; + displayName = option string; + }; + + toLdif = defun [ user string ] (u: '' + dn: cn=${u.username},ou=users,dc=tvl,dc=fyi + objectClass: organizationalPerson + objectClass: inetOrgPerson + sn: ${u.username} + cn: ${u.username} + displayName: ${u.displayName or u.username} + mail: ${u.email} + userPassword: ${u.password} + ''); + + inherit (depot.ops) users; + +in { + services.openldap = { + enable = true; + + settings.children = { + "olcDatabase={1}mdb".attrs = { + objectClass = [ "olcDatabaseConfig" "olcMdbConfig" ]; + olcDatabase = "{1}mdb"; + olcDbDirectory = "/var/lib/openldap"; + olcSuffix = "dc=tvl,dc=fyi"; + olcAccess = "to * by * read"; + olcRootDN = "cn=admin,dc=tvl,dc=fyi"; + olcRootPW = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$OfcgkOQ96VQ3aJj7NfA9vQ$oS6HQOkYl/bUYg4SejpltQYy7kvqx/RUxvoR4zo1vXU"; + }; + + "cn=module{0}".attrs = { + objectClass = "olcModuleList"; + olcModuleLoad = "pw-argon2"; + }; + + "cn=schema".includes = + map (schema: "${pkgs.openldap}/etc/schema/${schema}.ldif") + [ "core" "cosine" "inetorgperson" "nis" ]; + }; + + # Contents are immutable at runtime, and adding user accounts etc. + # is done statically in the LDIF-formatted contents in this folder. + declarativeContents."dc=tvl,dc=fyi" = '' + dn: dc=tvl,dc=fyi + dc: tvl + o: TVL LDAP server + description: Root entry for tvl.fyi + objectClass: top + objectClass: dcObject + objectClass: organization + + dn: ou=users,dc=tvl,dc=fyi + ou: users + description: All users in TVL + objectClass: top + objectClass: organizationalUnit + + dn: ou=groups,dc=tvl,dc=fyi + ou: groups + description: All groups in TVL + objectClass: top + objectClass: organizationalUnit + + ${lib.concatStringsSep "\n" (map toLdif users)} + ''; + }; +} diff --git a/ops/modules/tvl-sso/default.nix b/ops/modules/tvl-sso/default.nix new file mode 100644 index 000000000000..8b413114cc69 --- /dev/null +++ b/ops/modules/tvl-sso/default.nix @@ -0,0 +1,28 @@ +# Configures an Apereo CAS instance for TVL SSO +{ depot, ... }: + +let + inherit (depot.third_party) apereo-cas; +in { + config = { + environment.systemPackages = [ apereo-cas ]; + systemd.services.apereo-cas = { + description = "Apereo CAS Single Sign On server"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment.JDK_JAVA_OPTIONS = "-Xmx512M -Xms512M"; + serviceConfig = { + User = "apereo-cas"; + Group = "apereo-cas"; + ExecStart = "${apereo-cas}/bin/cas"; + EnvironmentFile = "/etc/cas/secrets"; + Restart = "always"; + }; + }; + users.users.apereo-cas = { + isSystemUser = true; + group = "apereo-cas"; + }; + users.groups.apereo-cas = {}; + }; +} diff --git a/ops/modules/v4l2loopback.nix b/ops/modules/v4l2loopback.nix new file mode 100644 index 000000000000..636b2ff6cf27 --- /dev/null +++ b/ops/modules/v4l2loopback.nix @@ -0,0 +1,12 @@ +{ config, lib, pkgs, ... }: + +{ + boot = { + extraModulePackages = [ config.boot.kernelPackages.v4l2loopback ]; + kernelModules = [ "v4l2loopback" ]; + extraModprobeConfig = '' + options v4l2loopback exclusive_caps=1 + ''; + }; +} + diff --git a/ops/modules/www/atward.tvl.fyi.nix b/ops/modules/www/atward.tvl.fyi.nix new file mode 100644 index 000000000000..6b3672dd75cb --- /dev/null +++ b/ops/modules/www/atward.tvl.fyi.nix @@ -0,0 +1,33 @@ +# Serve atward, the query redirection ... thing. +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + # Short link support (i.e. plain http://at) for users with a + # configured tvl.fyi/tvl.su search domain. + services.nginx.virtualHosts."at-shortlink" = { + serverName = "at"; + extraConfig = "return 302 https://atward.tvl.fyi$request_uri;"; + }; + + services.nginx.virtualHosts."atward" = { + serverName = "atward.tvl.fyi"; + enableACME = true; + forceSSL = true; + + serverAliases = [ + "atward.tvl.su" + "at.tvl.fyi" + "at.tvl.su" + ]; + + locations."/" = { + proxyPass = "http://localhost:${toString config.services.depot.atward.port}"; + }; + }; + }; +} diff --git a/ops/modules/www/b.tvl.fyi.nix b/ops/modules/www/b.tvl.fyi.nix new file mode 100644 index 000000000000..45f6c6ed5141 --- /dev/null +++ b/ops/modules/www/b.tvl.fyi.nix @@ -0,0 +1,32 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."b-shortlink" = { + serverName = "b"; + extraConfig = "return 302 https://b.tvl.fyi$request_uri;"; + }; + + services.nginx.virtualHosts."b.tvl.fyi" = { + serverName = "b.tvl.fyi"; + serverAliases = [ "b.tvl.su" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + # Forward short links to issues to the issue itself (b/32) + location ~ ^/(\d+)$ { + return 302 https://b.tvl.fyi/issues$request_uri; + } + + location / { + proxy_pass http://localhost:${toString config.services.depot.panettone.port}; + } + ''; + }; + }; +} diff --git a/ops/modules/www/base.nix b/ops/modules/www/base.nix new file mode 100644 index 000000000000..cfa9bf0bc6a8 --- /dev/null +++ b/ops/modules/www/base.nix @@ -0,0 +1,40 @@ +{ config, pkgs, ... }: + +{ + config = { + services.nginx = { + enable = true; + enableReload = true; + + recommendedTlsSettings = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + + appendHttpConfig = '' + add_header Permissions-Policy "interest-cohort=()"; + ''; + }; + + # NixOS 20.03 broke nginx and I can't be bothered to debug it + # anymore, all solution attempts have failed, so here's a + # brute-force fix. + # + # TODO(tazjin): Find a link to the upstream issue and see if + # they've sorted it after ~20.09 + systemd.services.fix-nginx = { + script = "${pkgs.coreutils}/bin/chown -f -R nginx: /var/spool/nginx /var/cache/nginx"; + + serviceConfig = { + User = "root"; + Type = "oneshot"; + }; + }; + + systemd.timers.fix-nginx = { + wantedBy = [ "multi-user.target" ]; + timerConfig = { + OnCalendar = "minutely"; + }; + }; + }; +} diff --git a/ops/modules/www/cache.tvl.su.nix b/ops/modules/www/cache.tvl.su.nix new file mode 100644 index 000000000000..182306bebff1 --- /dev/null +++ b/ops/modules/www/cache.tvl.su.nix @@ -0,0 +1,26 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."cache.tvl.su" = { + serverName = "cache.tvl.su"; + serverAliases = [ "cache.tvl.fyi" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location = /cache-key.pub { + alias /etc/secrets/nix-cache-key.pub; + } + + location / { + proxy_pass http://localhost:${toString config.services.nix-serve.port}; + } + ''; + }; + }; +} diff --git a/ops/modules/www/cl.tvl.fyi.nix b/ops/modules/www/cl.tvl.fyi.nix new file mode 100644 index 000000000000..470122c395ea --- /dev/null +++ b/ops/modules/www/cl.tvl.fyi.nix @@ -0,0 +1,30 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."cl-shortlink" = { + serverName = "cl"; + extraConfig = "return 302 https://cl.tvl.fyi$request_uri;"; + }; + + services.nginx.virtualHosts.gerrit = { + serverName = "cl.tvl.fyi"; + serverAliases = [ "cl.tvl.su" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location / { + proxy_pass http://localhost:4778; + proxy_set_header X-Forwarded-For $remote_addr; + # The :443 suffix is a workaround for https://b.tvl.fyi/issues/88. + proxy_set_header Host $host:443; + } + ''; + }; + }; +} diff --git a/ops/modules/www/code.tvl.fyi.nix b/ops/modules/www/code.tvl.fyi.nix new file mode 100644 index 000000000000..4c182d34f28d --- /dev/null +++ b/ops/modules/www/code.tvl.fyi.nix @@ -0,0 +1,45 @@ +{ depot, config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts.cgit = { + serverName = "code.tvl.fyi"; + serverAliases = [ "code.tvl.su" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + # Serve the rendered Tvix component SVG. + # + # TODO(tazjin): Implement a way of serving this dynamically + location = /about/tvix/docs/component-flow.svg { + alias ${depot.tvix.docs.svg}/component-flow.svg; + } + + # Git operations on depot.git hit josh + location /depot.git { + proxy_pass http://localhost:${toString config.services.depot.git-serving.joshPort}; + } + + # Git clone operations on '/' should be redirected to josh now. + location = /info/refs { + return 302 https://code.tvl.fyi/depot.git/info/refs$is_args$args; + } + + # Static assets must always hit the root. + location ~ ^/(favicon\.ico|cgit\.(css|png))$ { + proxy_pass http://localhost:2448; + } + + # Everything else is forwarded to cgit for the web view + location / { + proxy_pass http://localhost:2448/cgit.cgi/depot/; + } + ''; + }; + }; +} diff --git a/ops/modules/www/cs.tvl.fyi.nix b/ops/modules/www/cs.tvl.fyi.nix new file mode 100644 index 000000000000..fac814baf064 --- /dev/null +++ b/ops/modules/www/cs.tvl.fyi.nix @@ -0,0 +1,31 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."cs.tvl.fyi" = { + serverName = "cs.tvl.fyi"; + serverAliases = [ "cs.tvl.su" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location = / { + return 301 https://cs.tvl.fyi/depot; + } + + location / { + proxy_set_header X-Sg-Auth "Anonymous"; + proxy_pass http://localhost:${toString config.services.depot.sourcegraph.port}; + } + + location /users/Anonymous/settings { + return 301 https://cs.tvl.fyi; + } + ''; + }; + }; +} diff --git a/ops/modules/www/deploys.tvl.fyi.nix b/ops/modules/www/deploys.tvl.fyi.nix new file mode 100644 index 000000000000..ffbe225b58a3 --- /dev/null +++ b/ops/modules/www/deploys.tvl.fyi.nix @@ -0,0 +1,22 @@ +{ pkgs, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + # Ensure the directory for deployment diffs exists. + systemd.tmpfiles.rules = [ + "d /var/html/deploys.tvl.fyi/diff 0755 nginx nginx -" + ]; + + services.nginx.virtualHosts."deploys.tvl.fyi" = { + enableACME = true; + forceSSL = true; + root = "/var/html/deploys.tvl.fyi"; + }; + + services.depot.restic.paths = [ "/var/html/deploys.tvl.fyi" ]; + }; +} diff --git a/ops/modules/www/images.tvl.fyi.nix b/ops/modules/www/images.tvl.fyi.nix new file mode 100644 index 000000000000..7d027b2991ab --- /dev/null +++ b/ops/modules/www/images.tvl.fyi.nix @@ -0,0 +1,22 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."images.tvl.fyi" = { + serverName = "images.tvl.fyi"; + serverAliases = [ "images.tvl.su" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location / { + proxy_pass http://localhost:${toString config.services.depot.nixery.port}; + } + ''; + }; + }; +} diff --git a/ops/modules/www/login.tvl.fyi.nix b/ops/modules/www/login.tvl.fyi.nix new file mode 100644 index 000000000000..dd2e96324186 --- /dev/null +++ b/ops/modules/www/login.tvl.fyi.nix @@ -0,0 +1,24 @@ +{ ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."login.tvl.fyi" = { + serverName = "login.tvl.fyi"; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location / { + proxy_pass http://localhost:8444; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Host $host; + } + ''; + }; + }; +} diff --git a/ops/modules/www/nixery.dev.nix b/ops/modules/www/nixery.dev.nix new file mode 100644 index 000000000000..05dc88c66a07 --- /dev/null +++ b/ops/modules/www/nixery.dev.nix @@ -0,0 +1,21 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."nixery.dev" = { + serverName = "nixery.dev"; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location / { + proxy_pass http://localhost:${toString config.services.depot.nixery.port}; + } + ''; + }; + }; +} diff --git a/ops/modules/www/static.tvl.fyi.nix b/ops/modules/www/static.tvl.fyi.nix new file mode 100644 index 000000000000..7312f78ecf42 --- /dev/null +++ b/ops/modules/www/static.tvl.fyi.nix @@ -0,0 +1,42 @@ +# Host the static assets at static.tvl.fyi +# +# All assets are served from $base/$drvhash/$file, but can also be +# included with `latest/` which will return a (non-permanent!) +# redirect to the real location. +# +# For all purposes within depot, using the drvhash of web.static is +# recommended. +{ depot, pkgs, ... }: + +let staticHash = depot.web.static.drvHash; +in { + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."static.tvl.fyi" = { + serverAliases = [ "static.tvl.su" ]; + enableACME = true; + forceSSL = true; + + extraConfig = '' + location = / { + add_header Content-Type text/plain; + return 200 "looking for tvl.fyi or tvl.su?"; + } + + location /latest { + rewrite ^/latest/(.*) /${staticHash}/$1 redirect; + } + + location /${staticHash}/ { + alias ${depot.web.static}/; + expires max; + add_header Access-Control-Allow-Origin "*"; + add_header Cache-Control "public"; + } + ''; + }; + }; +} diff --git a/ops/modules/www/status.tvl.su.nix b/ops/modules/www/status.tvl.su.nix new file mode 100644 index 000000000000..2bb6093c1472 --- /dev/null +++ b/ops/modules/www/status.tvl.su.nix @@ -0,0 +1,25 @@ +{ config, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."status-fyi" = { + serverName = "status.tvl.fyi"; + enableACME = true; + extraConfig = "return 302 https://status.tvl.su$request_uri;"; + }; + + services.nginx.virtualHosts.grafana = { + serverName = "status.tvl.su"; + enableACME = true; + forceSSL = true; + + locations."/" = { + proxyPass = "http://localhost:${toString config.services.grafana.port}"; + }; + }; + }; +} diff --git a/ops/modules/www/tazj.in.nix b/ops/modules/www/tazj.in.nix new file mode 100644 index 000000000000..7d658a5ec4c1 --- /dev/null +++ b/ops/modules/www/tazj.in.nix @@ -0,0 +1,40 @@ +# serve tazjin's website & blog +{ depot, config, lib, pkgs, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."tazj.in" = { + enableACME = true; + forceSSL = true; + root = depot.users.tazjin.homepage; + + extraConfig = '' + ${depot.users.tazjin.blog.oldRedirects} + location /blog/ { + alias ${depot.users.tazjin.blog.rendered}/; + + if ($request_uri ~ ^/(.*)\.html$) { + return 302 /$1; + } + + try_files $uri $uri.html $uri/ =404; + } + + # Temporary place for serving static files. + location /blobs/ { + alias /var/lib/tazjins-blobs/; + } + ''; + }; + + services.nginx.virtualHosts."git.tazj.in" = { + enableACME = true; + forceSSL = true; + extraConfig = "return 301 https://code.tvl.fyi$request_uri;"; + }; + }; +} diff --git a/ops/modules/www/todo.tvl.fyi.nix b/ops/modules/www/todo.tvl.fyi.nix new file mode 100644 index 000000000000..b53f5437e7ab --- /dev/null +++ b/ops/modules/www/todo.tvl.fyi.nix @@ -0,0 +1,25 @@ +{ depot, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."todo.tvl.fyi" = { + serverName = "todo.tvl.fyi"; + serverAliases = [ "todo.tvl.su" ]; + root = depot.web.todolist; + enableACME = true; + forceSSL = true; + + extraConfig = '' + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; + + location ~* \.(webp|woff2)$ { + add_header Cache-Control "public, max-age=31536000"; + } + ''; + }; + }; +} diff --git a/ops/modules/www/tvl.fyi.nix b/ops/modules/www/tvl.fyi.nix new file mode 100644 index 000000000000..f422bb84878a --- /dev/null +++ b/ops/modules/www/tvl.fyi.nix @@ -0,0 +1,43 @@ +{ depot, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."tvl.fyi" = { + serverName = "tvl.fyi"; + root = depot.web.tvl; + enableACME = true; + forceSSL = true; + + extraConfig = '' + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; + + rewrite ^/builds/?$ https://buildkite.com/tvl/depot/ last; + + rewrite ^/monorepo-doc/?$ https://docs.google.com/document/d/1nnyByXcH0F6GOmEezNOUa2RFelpeRpDToBLYD_CtjWE/edit?usp=sharing last; + + rewrite ^/irc/?$ ircs://irc.hackint.org:6697/#tvl last; + rewrite ^/webchat/?$ https://webirc.hackint.org/#ircs://irc.hackint.org/#tvl last; + + location ~* \.(webp|woff2)$ { + add_header Cache-Control "public, max-age=31536000"; + } + + location /blog { + if ($request_uri ~ ^/(.*)\.html$) { + return 302 /$1; + } + + try_files $uri $uri.html $uri/ =404; + } + + location = /blog { + return 302 /; + } + ''; + }; + }; +} diff --git a/ops/modules/www/tvl.su.nix b/ops/modules/www/tvl.su.nix new file mode 100644 index 000000000000..a7c4f6a21721 --- /dev/null +++ b/ops/modules/www/tvl.su.nix @@ -0,0 +1,20 @@ +{ depot, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."tvl.su" = { + serverName = "tvl.su"; + root = depot.corp.website; + enableACME = true; + forceSSL = true; + + extraConfig = '' + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; + ''; + }; + }; +} diff --git a/ops/modules/www/wigglydonke.rs.nix b/ops/modules/www/wigglydonke.rs.nix new file mode 100644 index 000000000000..3d85e4eb9843 --- /dev/null +++ b/ops/modules/www/wigglydonke.rs.nix @@ -0,0 +1,15 @@ +{ depot, lib, pkgs, ... }: + +{ + imports = [ + ./base.nix + ]; + + config = { + services.nginx.virtualHosts."wigglydonke.rs" = { + enableACME = true; + forceSSL = true; + root = "${depot.path + "/users/grfn/wigglydonke.rs"}"; + }; + }; +} diff --git a/ops/mq_cli/.gitignore b/ops/mq_cli/.gitignore new file mode 100644 index 000000000000..5bd19a47c3b3 --- /dev/null +++ b/ops/mq_cli/.gitignore @@ -0,0 +1,4 @@ +/target/ +**/*.rs.bk +.idea/ +*.iml diff --git a/ops/mq_cli/CODE_OF_CONDUCT.md b/ops/mq_cli/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..c4013ac13ebc --- /dev/null +++ b/ops/mq_cli/CODE_OF_CONDUCT.md @@ -0,0 +1,20 @@ +A SERMON ON ETHICS AND LOVE +=========================== + +One day Mal-2 asked the messenger spirit Saint Gulik to approach the Goddess and request Her presence for some desperate advice. Shortly afterwards the radio came on by itself, and an ethereal female Voice said **YES?** + +"O! Eris! Blessed Mother of Man! Queen of Chaos! Daughter of Discord! Concubine of Confusion! O! Exquisite Lady, I beseech You to lift a heavy burden from my heart!" + +**WHAT BOTHERS YOU, MAL? YOU DON'T SOUND WELL.** + +"I am filled with fear and tormented with terrible visions of pain. Everywhere people are hurting one another, the planet is rampant with injustices, whole societies plunder groups of their own people, mothers imprison sons, children perish while brothers war. O, woe." + +**WHAT IS THE MATTER WITH THAT, IF IT IS WHAT YOU WANT TO DO?** + +"But nobody Wants it! Everybody hates it." + +**OH. WELL, THEN *STOP*.** + +At which moment She turned herself into an aspirin commercial and left The Polyfather stranded alone with his species. + +SINISTER DEXTER HAS A BROKEN SPIROMETER. diff --git a/ops/mq_cli/Cargo.lock b/ops/mq_cli/Cargo.lock new file mode 100644 index 000000000000..f418d77c34ff --- /dev/null +++ b/ops/mq_cli/Cargo.lock @@ -0,0 +1,159 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", + "vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hermit-abi" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "mq" +version = "1.0.0" +dependencies = [ + "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", + "posix_mq 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "posix_mq" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +"checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +"checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +"checksum posix_mq 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13ae339e13cc96902a4597a5aab6b76473093969c55d36ba33f6a7bf3268573f" +"checksum strsim 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +"checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" +"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +"checksum winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/ops/mq_cli/Cargo.toml b/ops/mq_cli/Cargo.toml new file mode 100644 index 000000000000..b412d88787fa --- /dev/null +++ b/ops/mq_cli/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "mq" +version = "1.0.0" +authors = ["Vincent Ambo <mail@tazj.in>"] + +[dependencies] +clap = "2.33" +libc = "0.2" +nix = "0.16" +posix_mq = "0.9" diff --git a/ops/mq_cli/LICENSE b/ops/mq_cli/LICENSE new file mode 100644 index 000000000000..e289cbab81e0 --- /dev/null +++ b/ops/mq_cli/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2017-2018 Langler AS +Copyright (c) 2019-2020 Vincent Ambo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ops/mq_cli/README.md b/ops/mq_cli/README.md new file mode 100644 index 000000000000..e612553e7447 --- /dev/null +++ b/ops/mq_cli/README.md @@ -0,0 +1,31 @@ +mq-cli +====== + +This project provides a very simple CLI interface to [POSIX message queues][]. + +It can be used to create and inspect queues, as well as send and +receive messages from them. + +``` +1.0.0 +Administrate and inspect POSIX message queues + +USAGE: + mq <SUBCOMMAND> + +FLAGS: + -h, --help Prints help information + -V, --version Prints version information + +SUBCOMMANDS: + create Create a new queue + help Prints this message or the help of the given subcommand(s) + inspect inspect details about a queue + ls list message queues + receive Receive a message from a queue + rlimit Get the message queue rlimit + send Send a message to a queue +``` + + +[POSIX message queues]: https://linux.die.net/man/7/mq_overview diff --git a/ops/mq_cli/default.nix b/ops/mq_cli/default.nix new file mode 100644 index 000000000000..6b0e32009a66 --- /dev/null +++ b/ops/mq_cli/default.nix @@ -0,0 +1,3 @@ +{ depot, ... }: + +depot.third_party.naersk.buildPackage ./. diff --git a/ops/mq_cli/src/main.rs b/ops/mq_cli/src/main.rs new file mode 100644 index 000000000000..55ff0064295d --- /dev/null +++ b/ops/mq_cli/src/main.rs @@ -0,0 +1,223 @@ +extern crate clap; +extern crate posix_mq; +extern crate libc; +extern crate nix; + +use clap::{App, SubCommand, Arg, ArgMatches, AppSettings}; +use posix_mq::{Name, Queue, Message}; +use std::fs::{read_dir, File}; +use std::io::{self, Read, Write}; +use std::process::exit; + +fn run_ls() { + let mqueues = read_dir("/dev/mqueue") + .expect("Could not read message queues"); + + for queue in mqueues { + let path = queue.unwrap().path(); + let status = { + let mut file = File::open(&path) + .expect("Could not open queue file"); + + let mut content = String::new(); + file.read_to_string(&mut content).expect("Could not read queue file"); + + content + }; + + let queue_name = path.components().last().unwrap() + .as_os_str() + .to_string_lossy(); + + println!("/{}: {}", queue_name, status) + }; +} + +fn run_inspect(queue_name: &str) { + let name = Name::new(queue_name).expect("Invalid queue name"); + let queue = Queue::open(name).expect("Could not open queue"); + + println!("Queue {}:\n", queue_name); + println!("Max. message size: {} bytes", queue.max_size()); + println!("Max. # of pending messages: {}", queue.max_pending()); +} + +fn run_create(cmd: &ArgMatches) { + if let Some(rlimit) = cmd.value_of("rlimit") { + set_rlimit(rlimit.parse().expect("Invalid rlimit value")); + } + + let name = Name::new(cmd.value_of("queue").unwrap()) + .expect("Invalid queue name"); + + let max_pending: i64 = cmd.value_of("max-pending").unwrap().parse().unwrap(); + let max_size: i64 = cmd.value_of("max-size").unwrap().parse().unwrap(); + + let queue = Queue::create(name, max_pending, max_size * 1024); + + match queue { + Ok(_) => println!("Queue created successfully"), + Err(e) => { + writeln!(io::stderr(), "Could not create queue: {}", e).ok(); + exit(1); + }, + }; +} + +fn run_receive(queue_name: &str) { + let name = Name::new(queue_name).expect("Invalid queue name"); + let queue = Queue::open(name).expect("Could not open queue"); + + let message = match queue.receive() { + Ok(msg) => msg, + Err(e) => { + writeln!(io::stderr(), "Failed to receive message: {}", e).ok(); + exit(1); + } + }; + + // Attempt to write the message out as a string, but write out raw bytes if it turns out to not + // be UTF-8 encoded data. + match String::from_utf8(message.data.clone()) { + Ok(string) => println!("{}", string), + Err(_) => { + writeln!(io::stderr(), "Message not UTF-8 encoded!").ok(); + io::stdout().write(message.data.as_ref()).ok(); + } + }; +} + +fn run_send(queue_name: &str, content: &str) { + let name = Name::new(queue_name).expect("Invalid queue name"); + let queue = Queue::open(name).expect("Could not open queue"); + + let message = Message { + data: content.as_bytes().to_vec(), + priority: 0, + }; + + match queue.send(&message) { + Ok(_) => (), + Err(e) => { + writeln!(io::stderr(), "Could not send message: {}", e).ok(); + exit(1); + } + } +} + +fn run_rlimit() { + let mut rlimit = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + + let mut errno = 0; + unsafe { + let res = libc::getrlimit(libc::RLIMIT_MSGQUEUE, &mut rlimit); + if res != 0 { + errno = nix::errno::errno(); + } + }; + + if errno != 0 { + writeln!(io::stderr(), "Could not get message queue rlimit: {}", errno).ok(); + } else { + println!("Message queue rlimit:"); + println!("Current limit: {}", rlimit.rlim_cur); + println!("Maximum limit: {}", rlimit.rlim_max); + } +} + +fn set_rlimit(new_limit: u64) { + let rlimit = libc::rlimit { + rlim_cur: new_limit, + rlim_max: new_limit, + }; + + let mut errno: i32 = 0; + unsafe { + let res = libc::setrlimit(libc::RLIMIT_MSGQUEUE, &rlimit); + if res != 0 { + errno = nix::errno::errno(); + } + } + + match errno { + 0 => println!("Set RLIMIT_MSGQUEUE hard limit to {}", new_limit), + _ => { + // Not mapping these error codes to messages for now, the user can + // look up the meaning in setrlimit(2). + panic!("Could not set hard limit: {}", errno); + } + }; +} + +fn main() { + let ls = SubCommand::with_name("ls").about("list message queues"); + + let queue_arg = Arg::with_name("queue").required(true).takes_value(true); + + let rlimit_arg = Arg::with_name("rlimit") + .help("RLIMIT_MSGQUEUE to set for this command") + .long("rlimit") + .takes_value(true); + + let inspect = SubCommand::with_name("inspect") + .about("inspect details about a queue") + .arg(&queue_arg); + + let create = SubCommand::with_name("create") + .about("Create a new queue") + .arg(&queue_arg) + .arg(&rlimit_arg) + .arg(Arg::with_name("max-size") + .help("maximum message size (in kB)") + .long("max-size") + .required(true) + .takes_value(true)) + .arg(Arg::with_name("max-pending") + .help("maximum # of pending messages") + .long("max-pending") + .required(true) + .takes_value(true)); + + let receive = SubCommand::with_name("receive") + .about("Receive a message from a queue") + .arg(&queue_arg); + + let send = SubCommand::with_name("send") + .about("Send a message to a queue") + .arg(&queue_arg) + .arg(Arg::with_name("message") + .help("the message to send") + .required(true)); + + let rlimit = SubCommand::with_name("rlimit") + .about("Get the message queue rlimit") + .setting(AppSettings::SubcommandRequiredElseHelp); + + let matches = App::new("mq") + .setting(AppSettings::SubcommandRequiredElseHelp) + .version("1.0.0") + .about("Administrate and inspect POSIX message queues") + .subcommand(ls) + .subcommand(inspect) + .subcommand(create) + .subcommand(receive) + .subcommand(send) + .subcommand(rlimit) + .get_matches(); + + match matches.subcommand() { + ("ls", _) => run_ls(), + ("inspect", Some(cmd)) => run_inspect(cmd.value_of("queue").unwrap()), + ("create", Some(cmd)) => run_create(cmd), + ("receive", Some(cmd)) => run_receive(cmd.value_of("queue").unwrap()), + ("send", Some(cmd)) => run_send( + cmd.value_of("queue").unwrap(), + cmd.value_of("message").unwrap() + ), + ("rlimit", _) => run_rlimit(), + _ => unimplemented!(), + } +} diff --git a/ops/nixos.nix b/ops/nixos.nix new file mode 100644 index 000000000000..19ba0168c07e --- /dev/null +++ b/ops/nixos.nix @@ -0,0 +1,52 @@ +# Helper functions for instantiating depot-compatible NixOS machines. +{ depot, lib, pkgs, ... }@args: + +let inherit (lib) findFirst isAttrs; +in rec { + # This provides our standard set of arguments to all NixOS modules. + baseModule = { ... }: { + # Ensure that pkgs == third_party.nix + nixpkgs.pkgs = depot.third_party.nixpkgs; + nix.nixPath = [ + "nixos=${pkgs.path}" + "nixpkgs=${pkgs.path}" + ]; + }; + + nixosFor = configuration: (depot.third_party.nixos { + configuration = { ... }: { + imports = [ + baseModule + configuration + ]; + }; + + specialArgs = { + inherit (args) depot; + }; + }); + + findSystem = hostname: + (findFirst + (system: system.config.networking.hostName == hostname) + (throw "${hostname} is not a known NixOS host") + (map nixosFor depot.ops.machines.all-systems)); + + rebuild-system = pkgs.writeShellScriptBin "rebuild-system" '' + set -ue + if [[ $EUID -ne 0 ]]; then + echo "Oh no! Only root is allowed to rebuild the system!" >&2 + exit 1 + fi + + echo "Rebuilding NixOS for $HOSTNAME" + system=$(nix-build -E "((import ${toString depot.path} {}).ops.nixos.findSystem \"$HOSTNAME\").system" --no-out-link --show-trace) + + nix-env -p /nix/var/nix/profiles/system --set $system + $system/bin/switch-to-configuration switch + ''; + + # Systems that should be built in CI + whitbySystem = (nixosFor depot.ops.machines.whitby).system; + meta.targets = [ "whitbySystem" ]; +} diff --git a/ops/pipelines/README.md b/ops/pipelines/README.md new file mode 100644 index 000000000000..a3f94fd23143 --- /dev/null +++ b/ops/pipelines/README.md @@ -0,0 +1,5 @@ +This folder contains the dynamic configuration for our [Buildkite CI +setup](https://tvl.fyi/builds). + +The configuration is built and dynamically loaded by Buildkite at the start of +each CI pipeline. diff --git a/ops/pipelines/depot.nix b/ops/pipelines/depot.nix new file mode 100644 index 000000000000..f2db69a78ff3 --- /dev/null +++ b/ops/pipelines/depot.nix @@ -0,0 +1,134 @@ +# This file configures the primary build pipeline used for the +# top-level list of depot targets. +# +# It outputs a "YAML" (actually JSON) file which is evaluated and +# submitted to Buildkite at the start of each build. This means we can +# dynamically configure the pipeline execution here. +{ depot, lib, pkgs, ... }: + +let + inherit (builtins) concatStringsSep foldl' map toJSON; + inherit (pkgs) symlinkJoin writeText; + + # Create an expression that builds the target at the specified + # location. + mkBuildExpr = target: + let + descend = expr: attr: "builtins.getAttr \"${attr}\" (${expr})"; + targetExpr = foldl' descend "import ./. {}" target.__readTree; + subtargetExpr = descend targetExpr target.__subtarget; + in if target ? __subtarget then subtargetExpr else targetExpr; + + # Create a pipeline label from the targets tree location. + mkLabel = target: + let label = concatStringsSep "/" target.__readTree; + in if target ? __subtarget + then "${label}:${target.__subtarget}" + else label; + + # Create a pipeline step from a single target. + mkStep = target: { + command = let + drvPath = builtins.unsafeDiscardStringContext target.drvPath; + in lib.concatStringsSep " " [ + # First try to realise the drvPath of the target so we don't evaluate twice. + # Nix has no concept of depending on a derivation file without depending on + # at least one of its `outPath`s, so we need to discard the string context + # if we don't want to build everything during pipeline construction. + "nix-store --realise '${drvPath}'" + # However, Nix doesn't track references of store paths to derivations, so + # there's no guarantee that the derivation file is not garbage collected. + # To handle this case we fall back to an ordinary build if the derivation + # file is missing. + "|| (test ! -f '${drvPath}' && nix-build -E '${mkBuildExpr target}' --show-trace)" + ]; + label = ":nix: ${mkLabel target}"; + + # Skip build steps if their out path has already been built. + skip = let + shouldSkip = with builtins; + # Only skip in real Buildkite builds + (getEnv "BUILDKITE_BUILD_ID" != "") && + # Always build everything for the canon branch. + (getEnv "BUILDKITE_BRANCH" != "refs/heads/canon") && + # Discard string context to avoid realising the store path during + # pipeline construction. + (pathExists (unsafeDiscardStringContext target.outPath)); + in if shouldSkip then "Target was already built." else false; + }; + + # Protobuf check step which validates that changes to .proto files + # between revisions don't cause backwards-incompatible or otherwise + # flawed changes. + protoCheck = { + command = "${depot.nix.bufCheck}/bin/ci-buf-check"; + label = ":water_buffalo:"; + }; + + # This defines the build pipeline, using the pipeline format + # documented on https://buildkite.com/docs/pipelines/defining-steps + # + # Pipeline steps need to stay in order. + pipeline.steps = + # Create build steps for each CI target + (map mkStep depot.ci.targets) + + ++ [ + # Simultaneously run protobuf checks + protoCheck + + # Wait for all previous checks to complete + ({ + wait = null; + continue_on_failure = true; + }) + + # Wait for all steps to complete, then exit with success or + # failure depending on whether any other steps failed. + # + # This information is checked by querying the Buildkite GraphQL + # API and fetching the count of failed steps. + # + # This step must be :duck:! (yes, really!) + ({ + command = let duck = pkgs.writeShellScript "duck" '' + set -ueo pipefail + + readonly FAILED_JOBS=$(${pkgs.curl}/bin/curl 'https://graphql.buildkite.com/v1' \ + --silent \ + -H "Authorization: Bearer $(cat /etc/secrets/buildkite-besadii)" \ + -d "{\"query\": \"query BuildStatusQuery { build(uuid: \\\"$BUILDKITE_BUILD_ID\\\") { jobs(passed: false) { count } } }\"}" | \ + ${pkgs.jq}/bin/jq -r '.data.build.jobs.count') + + echo "$FAILED_JOBS build jobs failed." + + if (( $FAILED_JOBS > 0 )); then + exit 1 + fi + ''; in "${duck}"; + + label = ":duck:"; + key = ":duck:"; + }) + + # After duck, on success, create a gcroot if the build branch is + # canon. + # + # We care that this anchors *most* of the depot, in practice + # it's unimportant if there is a build race and we get +-1 of + # the targets. + # + # Unfortunately this requires a third evaluation of the graph, + # but since it happens after :duck: it should not affect the + # timing of status reporting back to Gerrit. + ({ + command = "nix-instantiate -A ci.gcroot --add-root /nix/var/nix/gcroots/depot/canon"; + label = ":anchor:"; + "if" = ''build.branch == "refs/heads/canon"''; + depends_on = [{ + step = ":duck:"; + allow_failure = false; + }]; + }) + ]; +in (writeText "depot.yaml" (toJSON pipeline)) diff --git a/ops/pipelines/fallback.yaml b/ops/pipelines/fallback.yaml new file mode 100644 index 000000000000..73308d937b0c --- /dev/null +++ b/ops/pipelines/fallback.yaml @@ -0,0 +1,8 @@ +# This build configuration provides a fallback which marks a build as +# failed. This is used if evaluating the build configuration fails, +# for example because of a syntax error in Nix code. +--- +steps: + - command: "echo 'Nix evaluation failed!' && exit 1" + # This step *must* be :duck: to trigger the correct hook. + label: ":duck:" diff --git a/ops/pipelines/static-pipeline.yaml b/ops/pipelines/static-pipeline.yaml new file mode 100644 index 000000000000..c864aea65714 --- /dev/null +++ b/ops/pipelines/static-pipeline.yaml @@ -0,0 +1,31 @@ +# This file defines the static Buildkite pipeline which attempts to +# create the dynamic pipeline of all depot targets. +# +# If something fails during the creation of the pipeline, the fallback +# is executed instead which will simply report an error to Gerrit. +--- +steps: + - label: ":llama:" + command: | + function fallback() { + echo 'Using fallback pipeline ...' + buildkite-agent pipeline upload ops/pipelines/fallback.yaml + exit + } + + nix-build -A ops.pipelines.depot -o depot.yaml --show-trace || fallback + buildkite-agent pipeline upload depot.yaml || fallback + + # Create a revision number for the current commit for builds on + # canon. + # + # This writes data back to Gerrit using the Buildkite agent + # credentials injected through a git credentials helper. + # + # Revision numbers are defined as the number of commits in the + # lineage of HEAD, following only the first parent of merges. + - label: ":git:" + if: "build.branch == 'refs/heads/canon'" + command: | + git -c 'credential.helper=/etc/secrets/buildkite-credential-helper' \ + push origin "HEAD:refs/r/$(git rev-list --count --first-parent HEAD)" diff --git a/ops/posix_mq.rs/.gitignore b/ops/posix_mq.rs/.gitignore new file mode 100644 index 000000000000..e5b6fdb28e32 --- /dev/null +++ b/ops/posix_mq.rs/.gitignore @@ -0,0 +1,3 @@ +/target/ +**/*.rs.bk +.idea/ diff --git a/ops/posix_mq.rs/CODE_OF_CONDUCT.md b/ops/posix_mq.rs/CODE_OF_CONDUCT.md new file mode 100644 index 000000000000..c4013ac13ebc --- /dev/null +++ b/ops/posix_mq.rs/CODE_OF_CONDUCT.md @@ -0,0 +1,20 @@ +A SERMON ON ETHICS AND LOVE +=========================== + +One day Mal-2 asked the messenger spirit Saint Gulik to approach the Goddess and request Her presence for some desperate advice. Shortly afterwards the radio came on by itself, and an ethereal female Voice said **YES?** + +"O! Eris! Blessed Mother of Man! Queen of Chaos! Daughter of Discord! Concubine of Confusion! O! Exquisite Lady, I beseech You to lift a heavy burden from my heart!" + +**WHAT BOTHERS YOU, MAL? YOU DON'T SOUND WELL.** + +"I am filled with fear and tormented with terrible visions of pain. Everywhere people are hurting one another, the planet is rampant with injustices, whole societies plunder groups of their own people, mothers imprison sons, children perish while brothers war. O, woe." + +**WHAT IS THE MATTER WITH THAT, IF IT IS WHAT YOU WANT TO DO?** + +"But nobody Wants it! Everybody hates it." + +**OH. WELL, THEN *STOP*.** + +At which moment She turned herself into an aspirin commercial and left The Polyfather stranded alone with his species. + +SINISTER DEXTER HAS A BROKEN SPIROMETER. diff --git a/ops/posix_mq.rs/Cargo.lock b/ops/posix_mq.rs/Cargo.lock new file mode 100644 index 000000000000..fdd0086c4dd3 --- /dev/null +++ b/ops/posix_mq.rs/Cargo.lock @@ -0,0 +1,54 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cc" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.66" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "nix" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "posix_mq" +version = "0.9.0" +dependencies = [ + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" +"checksum libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)" = "d515b1f41455adea1313a4a2ac8a8a477634fbae63cc6100e3aebb207ce61558" +"checksum nix 0.16.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" diff --git a/ops/posix_mq.rs/Cargo.toml b/ops/posix_mq.rs/Cargo.toml new file mode 100644 index 000000000000..d72e87a3dcef --- /dev/null +++ b/ops/posix_mq.rs/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "posix_mq" +version = "0.9.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +description = "(Higher-level) Rust bindings to POSIX message queues" +license = "MIT" +repository = "https://git.tazj.in/tree/ops/posix_mq.rs" + +[dependencies] +nix = "0.16" +libc = "0.2" diff --git a/ops/posix_mq.rs/LICENSE b/ops/posix_mq.rs/LICENSE new file mode 100644 index 000000000000..2389546b1383 --- /dev/null +++ b/ops/posix_mq.rs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-2020 Vincent Ambo + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ops/posix_mq.rs/README.md b/ops/posix_mq.rs/README.md new file mode 100644 index 000000000000..9370c6c08740 --- /dev/null +++ b/ops/posix_mq.rs/README.md @@ -0,0 +1,33 @@ +posix_mq +======== + +[](https://travis-ci.org/aprilabank/posix_mq.rs) +[](https://crates.io/crates/posix_mq) + +This is a simple, relatively high-level library for the POSIX [message queue API][]. It wraps the lower-level API in a +simpler interface with more robust error handling. + +Check out this project's [sister library][] in Kotlin. + +Usage example: + +```rust +// Values that need to undergo validation are wrapped in safe types: +let name = Name::new("/test-queue").unwrap(); + +// Queue creation with system defaults is simple: +let queue = Queue::open_or_create(name).expect("Opening queue failed"); + +// Sending a message: +let message = Message { + data: "test-message".as_bytes().to_vec(), + priority: 0, +}; +queue.send(&message).expect("message sending failed"); + +// ... and receiving it! +let result = queue.receive().expect("message receiving failed"); +``` + +[message queue API]: https://linux.die.net/man/7/mq_overview +[sister library]: https://github.com/aprilabank/posix_mq.kt diff --git a/ops/posix_mq.rs/default.nix b/ops/posix_mq.rs/default.nix new file mode 100644 index 000000000000..6b0e32009a66 --- /dev/null +++ b/ops/posix_mq.rs/default.nix @@ -0,0 +1,3 @@ +{ depot, ... }: + +depot.third_party.naersk.buildPackage ./. diff --git a/ops/posix_mq.rs/src/error.rs b/ops/posix_mq.rs/src/error.rs new file mode 100644 index 000000000000..1ef585c01efb --- /dev/null +++ b/ops/posix_mq.rs/src/error.rs @@ -0,0 +1,130 @@ +use nix; +use std::error; +use std::fmt; +use std::io; +use std::num; + +/// This module implements a simple error type to match the errors that can be thrown from the C +/// functions as well as some extra errors resulting from internal validations. +/// +/// As this crate exposes an opinionated API to the POSIX queues certain errors have been +/// ignored: +/// +/// * ETIMEDOUT: The low-level timed functions are not exported and this error can not occur. +/// * EAGAIN: Non-blocking queue calls are not supported. +/// * EINVAL: Same reason as ETIMEDOUT +/// * EMSGSIZE: The message size is immutable after queue creation and this crate checks it. +/// * ENAMETOOLONG: This crate performs name validation +/// +/// If an unexpected error is encountered it will be wrapped appropriately and should be reported +/// as a bug on https://github.com/aprilabank/posix_mq.rs + +#[derive(Debug)] +pub enum Error { + // These errors are raised inside of the library + InvalidQueueName(&'static str), + ValueReadingError(io::Error), + MessageSizeExceeded(), + MaximumMessageSizeExceeded(), + MaximumMessageCountExceeded(), + + // These errors match what is described in the man pages (from mq_overview(7) onwards). + PermissionDenied(), + InvalidQueueDescriptor(), + QueueCallInterrupted(), + QueueAlreadyExists(), + QueueNotFound(), + InsufficientMemory(), + InsufficientSpace(), + + // These two are (hopefully) unlikely in modern systems + ProcessFileDescriptorLimitReached(), + SystemFileDescriptorLimitReached(), + + // If an unhandled / unknown / unexpected error occurs this error will be used. + // In those cases bug reports would be welcome! + UnknownForeignError(nix::errno::Errno), + + // Some other unexpected / unknown error occured. This is probably an error from + // the nix crate. Bug reports also welcome for this! + UnknownInternalError(Option<nix::Error>), +} + +impl error::Error for Error { + fn description(&self) -> &str { + use Error::*; + match *self { + // This error contains more sensible description strings already + InvalidQueueName(e) => e, + ValueReadingError(_) => "error reading system configuration for message queues", + MessageSizeExceeded() => "message is larger than maximum size for specified queue", + MaximumMessageSizeExceeded() => "specified queue message size exceeds system maximum", + MaximumMessageCountExceeded() => "specified queue message count exceeds system maximum", + PermissionDenied() => "permission to the specified queue was denied", + InvalidQueueDescriptor() => "the internal queue descriptor was invalid", + QueueCallInterrupted() => "queue method interrupted by signal", + QueueAlreadyExists() => "the specified queue already exists", + QueueNotFound() => "the specified queue could not be found", + InsufficientMemory() => "insufficient memory to call queue method", + InsufficientSpace() => "insufficient space to call queue method", + ProcessFileDescriptorLimitReached() => + "maximum number of process file descriptors reached", + SystemFileDescriptorLimitReached() => + "maximum number of system file descriptors reached", + UnknownForeignError(_) => "unknown foreign error occured: please report a bug!", + UnknownInternalError(_) => "unknown internal error occured: please report a bug!", + } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // Explicitly import this to gain access to Error::description() + use std::error::Error; + f.write_str(self.description()) + } +} + +/// This from implementation is used to translate errors from the lower-level +/// C-calls into sensible Rust errors. +impl From<nix::Error> for Error { + fn from(e: nix::Error) -> Self { + match e { + nix::Error::Sys(e) => match_errno(e), + _ => Error::UnknownInternalError(Some(e)), + } + } +} + +// This implementation is used when reading system queue settings. +impl From<io::Error> for Error { + fn from(e: io::Error) -> Self { + Error::ValueReadingError(e) + } +} + +// This implementation is used when parsing system queue settings. The unknown error is returned +// here because the system is probably seriously broken if those files don't contain numbers. +impl From<num::ParseIntError> for Error { + fn from(_: num::ParseIntError) -> Self { + Error::UnknownInternalError(None) + } +} + + +fn match_errno(err: nix::errno::Errno) -> Error { + use nix::errno::Errno::*; + + match err { + EACCES => Error::PermissionDenied(), + EBADF => Error::InvalidQueueDescriptor(), + EINTR => Error::QueueCallInterrupted(), + EEXIST => Error::QueueAlreadyExists(), + EMFILE => Error::ProcessFileDescriptorLimitReached(), + ENFILE => Error::SystemFileDescriptorLimitReached(), + ENOENT => Error::QueueNotFound(), + ENOMEM => Error::InsufficientMemory(), + ENOSPC => Error::InsufficientSpace(), + _ => Error::UnknownForeignError(err), + } +} diff --git a/ops/posix_mq.rs/src/lib.rs b/ops/posix_mq.rs/src/lib.rs new file mode 100644 index 000000000000..057601eccfbd --- /dev/null +++ b/ops/posix_mq.rs/src/lib.rs @@ -0,0 +1,269 @@ +extern crate nix; +extern crate libc; + +use error::Error; +use libc::mqd_t; +use nix::mqueue; +use nix::sys::stat; +use std::ffi::CString; +use std::fs::File; +use std::io::Read; +use std::string::ToString; +use std::ops::Drop; + +pub mod error; + +#[cfg(test)] +mod tests; + +/// Wrapper type for queue names that performs basic validation of queue names before calling +/// out to C code. +#[derive(Debug, Clone, PartialEq)] +pub struct Name(CString); + +impl Name { + pub fn new<S: ToString>(s: S) -> Result<Self, Error> { + let string = s.to_string(); + + if !string.starts_with('/') { + return Err(Error::InvalidQueueName("Queue name must start with '/'")); + } + + // The C library has a special error return for this case, so I assume people must actually + // have tried just using '/' as a queue name. + if string.len() == 1 { + return Err(Error::InvalidQueueName( + "Queue name must be a slash followed by one or more characters" + )); + } + + if string.len() > 255 { + return Err(Error::InvalidQueueName("Queue name must not exceed 255 characters")); + } + + if string.matches('/').count() > 1 { + return Err(Error::InvalidQueueName("Queue name can not contain more than one slash")); + } + + // TODO: What error is being thrown away here? Is it possible? + Ok(Name(CString::new(string).unwrap())) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Message { + pub data: Vec<u8>, + pub priority: u32, +} + +/// Represents an open queue descriptor to a POSIX message queue. This carries information +/// about the queue's limitations (i.e. maximum message size and maximum message count). +#[derive(Debug)] +pub struct Queue { + name: Name, + + /// Internal file/queue descriptor. + queue_descriptor: mqd_t, + + /// Maximum number of pending messages in this queue. + max_pending: i64, + + /// Maximum size of this queue. + max_size: usize, +} + +impl Queue { + /// Creates a new queue and fails if it already exists. + /// By default the queue will be read/writable by the current user with no access for other + /// users. + /// Linux users can change this setting themselves by modifying the queue file in /dev/mqueue. + pub fn create(name: Name, max_pending: i64, max_size: i64) -> Result<Queue, Error> { + if max_pending > read_i64_from_file(MSG_MAX)? { + return Err(Error::MaximumMessageCountExceeded()); + } + + if max_size > read_i64_from_file(MSGSIZE_MAX)? { + return Err(Error::MaximumMessageSizeExceeded()); + } + + let oflags = { + let mut flags = mqueue::MQ_OFlag::empty(); + // Put queue in r/w mode + flags.toggle(mqueue::MQ_OFlag::O_RDWR); + // Enable queue creation + flags.toggle(mqueue::MQ_OFlag::O_CREAT); + // Fail if queue exists already + flags.toggle(mqueue::MQ_OFlag::O_EXCL); + flags + }; + + let attr = mqueue::MqAttr::new( + 0, max_pending, max_size, 0 + ); + + let queue_descriptor = mqueue::mq_open( + &name.0, + oflags, + default_mode(), + Some(&attr), + )?; + + Ok(Queue { + name, + queue_descriptor, + max_pending, + max_size: max_size as usize, + }) + } + + /// Opens an existing queue. + pub fn open(name: Name) -> Result<Queue, Error> { + // No extra flags need to be constructed as the default is to open and fail if the + // queue does not exist yet - which is what we want here. + let oflags = mqueue::MQ_OFlag::O_RDWR; + let queue_descriptor = mqueue::mq_open( + &name.0, + oflags, + default_mode(), + None, + )?; + + let attr = mq_getattr(queue_descriptor)?; + + Ok(Queue { + name, + queue_descriptor, + max_pending: attr.mq_maxmsg, + max_size: attr.mq_msgsize as usize, + }) + } + + /// Opens an existing queue or creates a new queue with the OS default settings. + pub fn open_or_create(name: Name) -> Result<Queue, Error> { + let oflags = { + let mut flags = mqueue::MQ_OFlag::empty(); + // Put queue in r/w mode + flags.toggle(mqueue::MQ_OFlag::O_RDWR); + // Enable queue creation + flags.toggle(mqueue::MQ_OFlag::O_CREAT); + flags + }; + + let default_pending = read_i64_from_file(MSG_DEFAULT)?; + let default_size = read_i64_from_file(MSGSIZE_DEFAULT)?; + let attr = mqueue::MqAttr::new( + 0, default_pending, default_size, 0 + ); + + let queue_descriptor = mqueue::mq_open( + &name.0, + oflags, + default_mode(), + Some(&attr), + )?; + + let actual_attr = mq_getattr(queue_descriptor)?; + + Ok(Queue { + name, + queue_descriptor, + max_pending: actual_attr.mq_maxmsg, + max_size: actual_attr.mq_msgsize as usize, + }) + } + + /// Delete a message queue from the system. This method will make the queue unavailable for + /// other processes after their current queue descriptors have been closed. + pub fn delete(self) -> Result<(), Error> { + mqueue::mq_unlink(&self.name.0)?; + drop(self); + Ok(()) + } + + /// Send a message to the message queue. + /// If the queue is full this call will block until a message has been consumed. + pub fn send(&self, msg: &Message) -> Result<(), Error> { + if msg.data.len() > self.max_size as usize { + return Err(Error::MessageSizeExceeded()); + } + + mqueue::mq_send( + self.queue_descriptor, + msg.data.as_ref(), + msg.priority, + ).map_err(|e| e.into()) + } + + /// Receive a message from the message queue. + /// If the queue is empty this call will block until a message arrives. + pub fn receive(&self) -> Result<Message, Error> { + let mut data: Vec<u8> = vec![0; self.max_size as usize]; + let mut priority: u32 = 0; + + let msg_size = mqueue::mq_receive( + self.queue_descriptor, + data.as_mut(), + &mut priority, + )?; + + data.truncate(msg_size); + Ok(Message { data, priority }) + } + + pub fn max_pending(&self) -> i64 { + self.max_pending + } + + pub fn max_size(&self) -> usize { + self.max_size + } +} + +impl Drop for Queue { + fn drop(&mut self) { + // Attempt to close the queue descriptor and discard any possible errors. + // The only error thrown in the C-code is EINVAL, which would mean that the + // descriptor has already been closed. + mqueue::mq_close(self.queue_descriptor).ok(); + } +} + +// Creates the default queue mode (0600). +fn default_mode() -> stat::Mode { + let mut mode = stat::Mode::empty(); + mode.toggle(stat::Mode::S_IRUSR); + mode.toggle(stat::Mode::S_IWUSR); + mode +} + +/// This file defines the default number of maximum pending messages in a queue. +const MSG_DEFAULT: &'static str = "/proc/sys/fs/mqueue/msg_default"; + +/// This file defines the system maximum number of pending messages in a queue. +const MSG_MAX: &'static str = "/proc/sys/fs/mqueue/msg_max"; + +/// This file defines the default maximum size of messages in a queue. +const MSGSIZE_DEFAULT: &'static str = "/proc/sys/fs/mqueue/msgsize_default"; + +/// This file defines the system maximum size for messages in a queue. +const MSGSIZE_MAX: &'static str = "/proc/sys/fs/mqueue/msgsize_max"; + +/// This method is used in combination with the above constants to find system limits. +fn read_i64_from_file(name: &str) -> Result<i64, Error> { + let mut file = File::open(name.to_string())?; + let mut content = String::new(); + file.read_to_string(&mut content)?; + Ok(content.trim().parse()?) +} + +/// The mq_getattr implementation in the nix crate hides the maximum message size and count, which +/// is very impractical. +/// To work around it, this method calls the C-function directly. +fn mq_getattr(mqd: mqd_t) -> Result<libc::mq_attr, Error> { + use std::mem; + let mut attr = unsafe { mem::uninitialized::<libc::mq_attr>() }; + let res = unsafe { libc::mq_getattr(mqd, &mut attr) }; + nix::errno::Errno::result(res) + .map(|_| attr) + .map_err(|e| e.into()) +} diff --git a/ops/posix_mq.rs/src/tests.rs b/ops/posix_mq.rs/src/tests.rs new file mode 100644 index 000000000000..7a08876aeacd --- /dev/null +++ b/ops/posix_mq.rs/src/tests.rs @@ -0,0 +1,22 @@ +use super::*; + +#[test] +fn test_open_delete() { + // Simple test with default queue settings + let name = Name::new("/test-queue").unwrap(); + let queue = Queue::open_or_create(name) + .expect("Opening queue failed"); + + let message = Message { + data: "test-message".as_bytes().to_vec(), + priority: 0, + }; + + queue.send(&message).expect("message sending failed"); + + let result = queue.receive().expect("message receiving failed"); + + assert_eq!(message, result); + + queue.delete().expect("deleting queue failed"); +} diff --git a/ops/users/default.nix b/ops/users/default.nix new file mode 100644 index 000000000000..f9bf9a132862 --- /dev/null +++ b/ops/users/default.nix @@ -0,0 +1,152 @@ +{ ... }: + +[ + { + username = "adisbladis"; + email = "adisbladis@gmail.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$wdgoLRrUgZuz0Kin9YiNgQ$E40VIgzgpMpylZqkfByTKiWQnerupfuf7LDgOsU8tJA"; + } + { + username = "andi"; + email = "andi@notmuch.email"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$8lefg7+8UPAEh9Ott8zH0A$7YuLRraTC1IgxTNTxFJF03AWmqBS3GX2+vfD4XVTrb0"; + } + { + username = "cschilling"; + email = "christian.schilling.de@gmail.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$9VN3IS6ViW5FFbVKWOZI6Q$gZxuYAYk0Opq4E5i8cbcNjfznCQNc+RiP7Xv1CUnrQU"; + } + { + username = "cynthia"; + email = "me@cynthia.re"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=4,p=1$TxjbMGenhEmkyYLrg5uGhbr60THB86YeRZg5bPdiTJo$k9gbRlAPjmxwdUwzbavvsAVkckgQZ0jS2oTtvZBPysk"; + } + { + username = "edef"; + email = "edef@edef.eu"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$OORx4ERbkgvTmuYCJA8cIw$i5qaBzHkRVw7Tl+wZsTFTDqJwF0vuZqhW3VpknMYMc0"; + } + { + username = "ericvolp12"; + email = "ericvolp12@gmail.com"; + password = "{SSHA}pSepaQ+/5KBLfJtRR5rfxGU8goAsXgvk"; + } + { + username = "eta"; + email = "tvl@eta.st"; + password = "{SSHA}sOR5xzi7Lfv376XGQA8Hf6jyhTvo0XYc"; + } + { + username = "etu"; + email = "etu@failar.nu"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$RUrW8C9mWAkBSlkwSTH5dw$n3FXTeu41nDQfvJPI7TT3tcgwPmPJl8hPtaZ58qLq9A"; + } + { + username = "firefly"; + email = "firefly@firefly.nu"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$RYVVkFoi3A1yYkI8J2zUwg$GUERvgHvU8SGjQmilDJGZu50hYRAHw+ejtuL+Skygs8"; + } + { + username = "flokli"; + email = "flokli@flokli.de"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$TrezbwIY5TKLnJiii0wafQ$K0S2p9I8tiqP907nkgoK6IbG9ia4IuDiylTcIs5pesw"; + } + { + username = "grfn"; + email = "grfn@gws.fyi"; + password = "{SSHA}i7PSAsXwJT3jjmmvU77aar/tU/YPDCEO"; + } + { + username = "htbf"; + email = "h-tvl@htbf.dev"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$2iVXQQfd26icaIguHJg/CQ$hA9ziqn7kQ06AV6uQxJCGXoG8f+LWmH+nVlk00a1n/c"; + } + { + username = "isomer"; + email = "isomer@tvl.fyi"; + password = "{SSHA}OhWQkPJgH1rRJqYIaMUbbKC4iLEzvCev"; + } + { + username = "kn"; + email = "klemens@posteo.de"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$CoRZInysud4sduDoMjVOCw$/bdvAvyPO2DPxOcHlBiG2+rbTGF9XAcHUhPurxiIpZM"; + } + { + username = "lukegb"; + email = "lukegb@tvl.fyi"; + password = "{SSHA}7a85VNhpFElFw+N5xcjgGmt4HnBsaGp4"; + } + { + username = "nyanotech"; + email = "nyanotechnology@gmail.com"; + password = "{SSHA}NIJ2RCRb1+Q4Bs63cyE91VZyiN47DG6y"; + } + { + username = "Profpatsch"; + email = "mail@profpatsch.de"; + password = "{SSHA}jcFXxRplMFxH4gpa0X5VdUzW64T95TwQ"; + } + { + username = "sterni"; + email = "sternenseemann@systemli.org"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$+NbF1izPMGqN5bASCBDV9g$aqBVplHwiyDpflZUmLtjkLWzKhxi7hwjm5fOwfbKohU"; + } + { + username = "qyliss"; + displayName = "Alyssa Ross"; + email = "hi@alyssa.is"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$+uTpAKrN452D8wa7OFqPnw$GYi9/zns5iJCXDp1VuTPPsa35M5vkD6+rC8riT8cEHI"; + } + { + username = "riking"; + displayName = "kanepyork"; + email = "rikingcoding@gmail.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$o2OcfhfKOry+UrcmODyQCw$qloaQgoIRDESwaA3yqPxxy8sgLk3mrjYFBbF41elVrM"; + } + { + username = "tazjin"; + email = "mail@tazj.in"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$wOPEl9D3kSke//oLtbvqrg$j0npwwXgaXQ/emefKUwL59tH8hdmtzbgH2rQzWSmE2Y"; + } + { + username = "implr"; + email = "implr@hackerspace.pl"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$SHRFps5sVgyUXYdmqGPw9g$tEx9DwKK1RjWlw52GLwOZ/iHep+QJboaZE83f1pXSwQ"; + } + { + username = "ben"; + email = "tvl@benjojo.co.uk"; + password = "{SSHA}Zi48mSPsRMEPhff44w4RHi0SjjyhjWk1"; + } + { + username = "jamie"; + email = "jamie@kwiius.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$OkAMHVAfQ3nJhBffYJwk7Q$JV3DrF9eOU+4VL6I+nkaMUUOMqWuNzdp7N7U5Xwa3fg"; + } + { + username = "milan"; + email = "milan@petabyte.dev"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$VQAHgOqYVr7mzjEr8IAMdQ$eAXvy58eRkjg+96RKBCwUoRDpNyGDdes4rVtxoQbaeI"; + } + { + username = "ezemtsov"; + email = "eugene.zemtsov@gmail.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$eAEjTm0+JD0+p0o/GA8JmQ$uRtHCT+B/DBNr1rlOcLlROrkYsMj2T9ns/E9Ep3gJ1A"; + } + { + username = "mdjnsn"; + email = "mdj@mikejohnson.xyz"; + displayName = "Mike Johnson"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$77+f5DFbuzs5myyIUN2nHg$xyXkRhIHFVaPMZUhxPk1uxMpLeEmU3BeyQjDsNPlJVw"; + } + { + username = "smitop"; + email = "me@smitop.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$H78rQtmhlzrPEifbXPoCVw$IBg7ePTm/u+e8r2A8aJ4iaaQBzMUw1isS9YJAZ8aT3o"; + } + { + username = "asmundo"; + email = "asmundo@gmail.com"; + password = "{ARGON2}$argon2id$v=19$m=65536,t=2,p=1$wUpPczumK9/8z18pCp6QAw$KvAHhIc2ugeiyeLvxBlqlM87fZDjdwKEQpkAzq4aaCs"; + } +] diff --git a/third_party/README.md b/third_party/README.md new file mode 100644 index 000000000000..267f23469775 --- /dev/null +++ b/third_party/README.md @@ -0,0 +1,13 @@ +Third-Party Code +================ + +Code under this folder is one of the following: + +1. Externally developed dependencies which have been imported ("vendored") into + this repository. These dependencies come with their own licenses and whatever + else. + +2. Code that is developed inside of this repository, but released to an external + repository via [Copybara][]. + +[Copybara]: https://github.com/google/copybara diff --git a/third_party/abseil_cpp/.clang-format b/third_party/abseil_cpp/.clang-format new file mode 100644 index 000000000000..06ea346a1067 --- /dev/null +++ b/third_party/abseil_cpp/.clang-format @@ -0,0 +1,4 @@ +--- +Language: Cpp +BasedOnStyle: Google +... diff --git a/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/00-bug_report.md b/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/00-bug_report.md new file mode 100644 index 000000000000..1edf3de0ba47 --- /dev/null +++ b/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/00-bug_report.md @@ -0,0 +1,41 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: 'bug' +assignees: '' +--- + +**Describe the bug** + +Include a clear and concise description of what the problem is, including what +you expected to happen, and what actually happened. + +**Steps to reproduce the bug** + +It's important that we are able to reproduce the problem that you are +experiencing. Please provide all code and relevant steps to reproduce the +problem, including your `BUILD`/`CMakeLists.txt` file and build commands. Links +to a GitHub branch or [godbolt.org](https://godbolt.org/) that demonstrate the +problem are also helpful. + +**What version of Abseil are you using?** + +**What operating system and version are you using** + +If you are using a Linux distribution please include the name and version of the +distribution as well. + +**What compiler and version are you using?** + +Please include the output of `gcc -v` or `clang -v`, or the equivalent for your +compiler. + +**What build system are you using?** + +Please include the output of `bazel --version` or `cmake --version`, or the +equivalent for your build system. + +**Additional context** + +Add any other context about the problem here. diff --git a/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/90-question.md b/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/90-question.md new file mode 100644 index 000000000000..84cf349189c3 --- /dev/null +++ b/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/90-question.md @@ -0,0 +1,7 @@ +--- +name: Question +about: Have a question? Ask us anything! :-) +title: '' +labels: 'question' +assignees: '' +--- diff --git a/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/config.yml b/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000000..9794ae1db8bc --- /dev/null +++ b/third_party/abseil_cpp/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enables: true diff --git a/third_party/abseil_cpp/.gitignore b/third_party/abseil_cpp/.gitignore new file mode 100644 index 000000000000..d54fa5a91fe6 --- /dev/null +++ b/third_party/abseil_cpp/.gitignore @@ -0,0 +1,15 @@ +# Ignore all bazel-* symlinks. +/bazel-* +# Ignore Bazel verbose explanations +--verbose_explanations +# Ignore CMake usual build directory +build +# Ignore Vim files +*.swp +# Ignore QtCreator Project file +CMakeLists.txt.user +# Ignore VS Code files +.vscode/* +# Ignore generated python artifacts +*.pyc +copts/__pycache__/ diff --git a/third_party/abseil_cpp/.skip-subtree b/third_party/abseil_cpp/.skip-subtree new file mode 100644 index 000000000000..d49b47f75a16 --- /dev/null +++ b/third_party/abseil_cpp/.skip-subtree @@ -0,0 +1 @@ +Third-party code with non-depot layout. diff --git a/third_party/abseil_cpp/ABSEIL_ISSUE_TEMPLATE.md b/third_party/abseil_cpp/ABSEIL_ISSUE_TEMPLATE.md new file mode 100644 index 000000000000..ed5461f166c3 --- /dev/null +++ b/third_party/abseil_cpp/ABSEIL_ISSUE_TEMPLATE.md @@ -0,0 +1,22 @@ +Please submit a new Abseil Issue using the template below: + +## [Short title of proposed API change(s)] + +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- + +## Background + +[Provide the background information that is required in order to evaluate the +proposed API changes. No controversial claims should be made here. If there are +design constraints that need to be considered, they should be presented here +**along with justification for those constraints**. Linking to other docs is +good, but please keep the **pertinent information as self contained** as +possible in this section.] + +## Proposed API Change (s) + +[Please clearly describe the API change(s) being proposed. If multiple changes, +please keep them clearly distinguished. When possible, **use example code +snippets to illustrate before-after API usages**. List pros-n-cons. Highlight +the main questions that you want to be answered. Given the Abseil project compatibility requirements, describe why the API change is safe.] diff --git a/third_party/abseil_cpp/AUTHORS b/third_party/abseil_cpp/AUTHORS new file mode 100644 index 000000000000..976d31defc26 --- /dev/null +++ b/third_party/abseil_cpp/AUTHORS @@ -0,0 +1,6 @@ +# This is the list of Abseil authors for copyright purposes. +# +# This does not necessarily list everyone who has contributed code, since in +# some cases, their employer may be the copyright holder. To see the full list +# of contributors, see the revision history in source control. +Google Inc. diff --git a/third_party/abseil_cpp/BUILD.bazel b/third_party/abseil_cpp/BUILD.bazel new file mode 100644 index 000000000000..79fb0ecd73ef --- /dev/null +++ b/third_party/abseil_cpp/BUILD.bazel @@ -0,0 +1,25 @@ +# +# Copyright 2020 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +# Expose license for external usage through bazel. +exports_files([ + "AUTHORS", + "LICENSE", +]) diff --git a/third_party/abseil_cpp/CMake/AbseilDll.cmake b/third_party/abseil_cpp/CMake/AbseilDll.cmake new file mode 100644 index 000000000000..e0ff2492e7f2 --- /dev/null +++ b/third_party/abseil_cpp/CMake/AbseilDll.cmake @@ -0,0 +1,514 @@ +include(CMakeParseArguments) + +set(ABSL_INTERNAL_DLL_FILES + "algorithm/algorithm.h" + "algorithm/container.h" + "base/attributes.h" + "base/call_once.h" + "base/casts.h" + "base/config.h" + "base/const_init.h" + "base/dynamic_annotations.h" + "base/internal/atomic_hook.h" + "base/internal/bits.h" + "base/internal/cycleclock.cc" + "base/internal/cycleclock.h" + "base/internal/direct_mmap.h" + "base/internal/dynamic_annotations.h" + "base/internal/endian.h" + "base/internal/errno_saver.h" + "base/internal/exponential_biased.cc" + "base/internal/exponential_biased.h" + "base/internal/fast_type_id.h" + "base/internal/hide_ptr.h" + "base/internal/identity.h" + "base/internal/invoke.h" + "base/internal/inline_variable.h" + "base/internal/low_level_alloc.cc" + "base/internal/low_level_alloc.h" + "base/internal/low_level_scheduling.h" + "base/internal/per_thread_tls.h" + "base/internal/periodic_sampler.cc" + "base/internal/periodic_sampler.h" + "base/internal/pretty_function.h" + "base/internal/raw_logging.cc" + "base/internal/raw_logging.h" + "base/internal/scheduling_mode.h" + "base/internal/scoped_set_env.cc" + "base/internal/scoped_set_env.h" + "base/internal/strerror.h" + "base/internal/strerror.cc" + "base/internal/spinlock.cc" + "base/internal/spinlock.h" + "base/internal/spinlock_wait.cc" + "base/internal/spinlock_wait.h" + "base/internal/sysinfo.cc" + "base/internal/sysinfo.h" + "base/internal/thread_annotations.h" + "base/internal/thread_identity.cc" + "base/internal/thread_identity.h" + "base/internal/throw_delegate.cc" + "base/internal/throw_delegate.h" + "base/internal/tsan_mutex_interface.h" + "base/internal/unaligned_access.h" + "base/internal/unscaledcycleclock.cc" + "base/internal/unscaledcycleclock.h" + "base/log_severity.cc" + "base/log_severity.h" + "base/macros.h" + "base/optimization.h" + "base/options.h" + "base/policy_checks.h" + "base/port.h" + "base/thread_annotations.h" + "container/btree_map.h" + "container/btree_set.h" + "container/fixed_array.h" + "container/flat_hash_map.h" + "container/flat_hash_set.h" + "container/inlined_vector.h" + "container/internal/btree.h" + "container/internal/btree_container.h" + "container/internal/common.h" + "container/internal/compressed_tuple.h" + "container/internal/container_memory.h" + "container/internal/counting_allocator.h" + "container/internal/hash_function_defaults.h" + "container/internal/hash_policy_traits.h" + "container/internal/hashtable_debug.h" + "container/internal/hashtable_debug_hooks.h" + "container/internal/hashtablez_sampler.cc" + "container/internal/hashtablez_sampler.h" + "container/internal/hashtablez_sampler_force_weak_definition.cc" + "container/internal/have_sse.h" + "container/internal/inlined_vector.h" + "container/internal/layout.h" + "container/internal/node_hash_policy.h" + "container/internal/raw_hash_map.h" + "container/internal/raw_hash_set.cc" + "container/internal/raw_hash_set.h" + "container/internal/tracked.h" + "container/node_hash_map.h" + "container/node_hash_set.h" + "debugging/failure_signal_handler.cc" + "debugging/failure_signal_handler.h" + "debugging/leak_check.h" + "debugging/leak_check_disable.cc" + "debugging/stacktrace.cc" + "debugging/stacktrace.h" + "debugging/symbolize.cc" + "debugging/symbolize.h" + "debugging/internal/address_is_readable.cc" + "debugging/internal/address_is_readable.h" + "debugging/internal/demangle.cc" + "debugging/internal/demangle.h" + "debugging/internal/elf_mem_image.cc" + "debugging/internal/elf_mem_image.h" + "debugging/internal/examine_stack.cc" + "debugging/internal/examine_stack.h" + "debugging/internal/stack_consumption.cc" + "debugging/internal/stack_consumption.h" + "debugging/internal/stacktrace_config.h" + "debugging/internal/symbolize.h" + "debugging/internal/vdso_support.cc" + "debugging/internal/vdso_support.h" + "functional/internal/front_binder.h" + "functional/bind_front.h" + "functional/function_ref.h" + "functional/internal/function_ref.h" + "hash/hash.h" + "hash/internal/city.h" + "hash/internal/city.cc" + "hash/internal/hash.h" + "hash/internal/hash.cc" + "hash/internal/spy_hash_state.h" + "memory/memory.h" + "meta/type_traits.h" + "numeric/int128.cc" + "numeric/int128.h" + "random/bernoulli_distribution.h" + "random/beta_distribution.h" + "random/bit_gen_ref.h" + "random/discrete_distribution.cc" + "random/discrete_distribution.h" + "random/distributions.h" + "random/exponential_distribution.h" + "random/gaussian_distribution.cc" + "random/gaussian_distribution.h" + "random/internal/distribution_caller.h" + "random/internal/fastmath.h" + "random/internal/fast_uniform_bits.h" + "random/internal/generate_real.h" + "random/internal/iostream_state_saver.h" + "random/internal/mock_helpers.h" + "random/internal/nonsecure_base.h" + "random/internal/pcg_engine.h" + "random/internal/platform.h" + "random/internal/pool_urbg.cc" + "random/internal/pool_urbg.h" + "random/internal/randen.cc" + "random/internal/randen.h" + "random/internal/randen_detect.cc" + "random/internal/randen_detect.h" + "random/internal/randen_engine.h" + "random/internal/randen_hwaes.cc" + "random/internal/randen_hwaes.h" + "random/internal/randen_round_keys.cc" + "random/internal/randen_slow.cc" + "random/internal/randen_slow.h" + "random/internal/randen_traits.h" + "random/internal/salted_seed_seq.h" + "random/internal/seed_material.cc" + "random/internal/seed_material.h" + "random/internal/sequence_urbg.h" + "random/internal/traits.h" + "random/internal/uniform_helper.h" + "random/internal/wide_multiply.h" + "random/log_uniform_int_distribution.h" + "random/poisson_distribution.h" + "random/random.h" + "random/seed_gen_exception.cc" + "random/seed_gen_exception.h" + "random/seed_sequences.cc" + "random/seed_sequences.h" + "random/uniform_int_distribution.h" + "random/uniform_real_distribution.h" + "random/zipf_distribution.h" + "status/internal/status_internal.h" + "status/internal/statusor_internal.h" + "status/status.h" + "status/status.cc" + "status/statusor.h" + "status/statusor.cc" + "status/status_payload_printer.h" + "status/status_payload_printer.cc" + "strings/ascii.cc" + "strings/ascii.h" + "strings/charconv.cc" + "strings/charconv.h" + "strings/cord.cc" + "strings/cord.h" + "strings/escaping.cc" + "strings/escaping.h" + "strings/internal/cord_internal.h" + "strings/internal/charconv_bigint.cc" + "strings/internal/charconv_bigint.h" + "strings/internal/charconv_parse.cc" + "strings/internal/charconv_parse.h" + "strings/internal/stl_type_traits.h" + "strings/internal/string_constant.h" + "strings/match.cc" + "strings/match.h" + "strings/numbers.cc" + "strings/numbers.h" + "strings/str_format.h" + "strings/str_cat.cc" + "strings/str_cat.h" + "strings/str_join.h" + "strings/str_replace.cc" + "strings/str_replace.h" + "strings/str_split.cc" + "strings/str_split.h" + "strings/string_view.cc" + "strings/string_view.h" + "strings/strip.h" + "strings/substitute.cc" + "strings/substitute.h" + "strings/internal/char_map.h" + "strings/internal/escaping.h" + "strings/internal/escaping.cc" + "strings/internal/memutil.cc" + "strings/internal/memutil.h" + "strings/internal/ostringstream.cc" + "strings/internal/ostringstream.h" + "strings/internal/pow10_helper.cc" + "strings/internal/pow10_helper.h" + "strings/internal/resize_uninitialized.h" + "strings/internal/str_format/arg.cc" + "strings/internal/str_format/arg.h" + "strings/internal/str_format/bind.cc" + "strings/internal/str_format/bind.h" + "strings/internal/str_format/checker.h" + "strings/internal/str_format/extension.cc" + "strings/internal/str_format/extension.h" + "strings/internal/str_format/float_conversion.cc" + "strings/internal/str_format/float_conversion.h" + "strings/internal/str_format/output.cc" + "strings/internal/str_format/output.h" + "strings/internal/str_format/parser.cc" + "strings/internal/str_format/parser.h" + "strings/internal/str_join_internal.h" + "strings/internal/str_split_internal.h" + "strings/internal/utf8.cc" + "strings/internal/utf8.h" + "synchronization/barrier.cc" + "synchronization/barrier.h" + "synchronization/blocking_counter.cc" + "synchronization/blocking_counter.h" + "synchronization/mutex.cc" + "synchronization/mutex.h" + "synchronization/notification.cc" + "synchronization/notification.h" + "synchronization/internal/create_thread_identity.cc" + "synchronization/internal/create_thread_identity.h" + "synchronization/internal/futex.h" + "synchronization/internal/graphcycles.cc" + "synchronization/internal/graphcycles.h" + "synchronization/internal/kernel_timeout.h" + "synchronization/internal/per_thread_sem.cc" + "synchronization/internal/per_thread_sem.h" + "synchronization/internal/thread_pool.h" + "synchronization/internal/waiter.cc" + "synchronization/internal/waiter.h" + "time/civil_time.cc" + "time/civil_time.h" + "time/clock.cc" + "time/clock.h" + "time/duration.cc" + "time/format.cc" + "time/time.cc" + "time/time.h" + "time/internal/cctz/include/cctz/civil_time.h" + "time/internal/cctz/include/cctz/civil_time_detail.h" + "time/internal/cctz/include/cctz/time_zone.h" + "time/internal/cctz/include/cctz/zone_info_source.h" + "time/internal/cctz/src/civil_time_detail.cc" + "time/internal/cctz/src/time_zone_fixed.cc" + "time/internal/cctz/src/time_zone_fixed.h" + "time/internal/cctz/src/time_zone_format.cc" + "time/internal/cctz/src/time_zone_if.cc" + "time/internal/cctz/src/time_zone_if.h" + "time/internal/cctz/src/time_zone_impl.cc" + "time/internal/cctz/src/time_zone_impl.h" + "time/internal/cctz/src/time_zone_info.cc" + "time/internal/cctz/src/time_zone_info.h" + "time/internal/cctz/src/time_zone_libc.cc" + "time/internal/cctz/src/time_zone_libc.h" + "time/internal/cctz/src/time_zone_lookup.cc" + "time/internal/cctz/src/time_zone_posix.cc" + "time/internal/cctz/src/time_zone_posix.h" + "time/internal/cctz/src/tzfile.h" + "time/internal/cctz/src/zone_info_source.cc" + "types/any.h" + "types/bad_any_cast.cc" + "types/bad_any_cast.h" + "types/bad_optional_access.cc" + "types/bad_optional_access.h" + "types/bad_variant_access.cc" + "types/bad_variant_access.h" + "types/compare.h" + "types/internal/conformance_aliases.h" + "types/internal/conformance_archetype.h" + "types/internal/conformance_profile.h" + "types/internal/parentheses.h" + "types/internal/transform_args.h" + "types/internal/variant.h" + "types/optional.h" + "types/internal/optional.h" + "types/span.h" + "types/internal/span.h" + "types/variant.h" + "utility/utility.h" +) + +set(ABSL_INTERNAL_DLL_TARGETS + "stacktrace" + "symbolize" + "examine_stack" + "failure_signal_handler" + "debugging_internal" + "demangle_internal" + "leak_check" + "leak_check_disable" + "stack_consumption" + "debugging" + "hash" + "spy_hash_state" + "city" + "memory" + "strings" + "strings_internal" + "cord" + "str_format" + "str_format_internal" + "pow10_helper" + "int128" + "numeric" + "utility" + "any" + "bad_any_cast" + "bad_any_cast_impl" + "span" + "optional" + "bad_optional_access" + "bad_variant_access" + "variant" + "compare" + "algorithm" + "algorithm_container" + "graphcycles_internal" + "kernel_timeout_internal" + "synchronization" + "thread_pool" + "bind_front" + "function_ref" + "atomic_hook" + "log_severity" + "raw_logging_internal" + "spinlock_wait" + "config" + "dynamic_annotations" + "core_headers" + "malloc_internal" + "base_internal" + "base" + "throw_delegate" + "pretty_function" + "endian" + "bits" + "exponential_biased" + "periodic_sampler" + "scoped_set_env" + "type_traits" + "meta" + "random_random" + "random_bit_gen_ref" + "random_distributions" + "random_seed_gen_exception" + "random_seed_sequences" + "random_internal_traits" + "random_internal_distribution_caller" + "random_internal_distributions" + "random_internal_fast_uniform_bits" + "random_internal_seed_material" + "random_internal_pool_urbg" + "random_internal_explicit_seed_seq" + "random_internal_sequence_urbg" + "random_internal_salted_seed_seq" + "random_internal_iostream_state_saver" + "random_internal_generate_real" + "random_internal_wide_multiply" + "random_internal_fastmath" + "random_internal_nonsecure_base" + "random_internal_pcg_engine" + "random_internal_randen_engine" + "random_internal_platform" + "random_internal_randen" + "random_internal_randen_slow" + "random_internal_randen_hwaes" + "random_internal_randen_hwaes_impl" + "random_internal_uniform_helper" + "status" + "time" + "civil_time" + "time_zone" + "container" + "btree" + "compressed_tuple" + "fixed_array" + "inlined_vector_internal" + "inlined_vector" + "counting_allocator" + "flat_hash_map" + "flat_hash_set" + "node_hash_map" + "node_hash_set" + "container_memory" + "hash_function_defaults" + "hash_policy_traits" + "hashtablez_sampler" + "hashtable_debug" + "hashtable_debug_hooks" + "have_sse" + "node_hash_policy" + "raw_hash_map" + "container_common" + "raw_hash_set" + "layout" + "tracked" +) + +function(absl_internal_dll_contains) + cmake_parse_arguments(ABSL_INTERNAL_DLL + "" + "OUTPUT;TARGET" + "" + ${ARGN} + ) + + STRING(REGEX REPLACE "^absl::" "" _target ${ABSL_INTERNAL_DLL_TARGET}) + + list(FIND + ABSL_INTERNAL_DLL_TARGETS + "${_target}" + _index) + + if (${_index} GREATER -1) + set(${ABSL_INTERNAL_DLL_OUTPUT} 1 PARENT_SCOPE) + else() + set(${ABSL_INTERNAL_DLL_OUTPUT} 0 PARENT_SCOPE) + endif() +endfunction() + +function(absl_internal_dll_targets) + cmake_parse_arguments(ABSL_INTERNAL_DLL + "" + "OUTPUT" + "DEPS" + ${ARGN} + ) + + set(_deps "") + foreach(dep IN LISTS ABSL_INTERNAL_DLL_DEPS) + absl_internal_dll_contains(TARGET ${dep} OUTPUT _contains) + if (_contains) + list(APPEND _deps abseil_dll) + else() + list(APPEND _deps ${dep}) + endif() + endforeach() + + # Because we may have added the DLL multiple times + list(REMOVE_DUPLICATES _deps) + set(${ABSL_INTERNAL_DLL_OUTPUT} "${_deps}" PARENT_SCOPE) +endfunction() + +function(absl_make_dll) + add_library( + abseil_dll + SHARED + "${ABSL_INTERNAL_DLL_FILES}" + ) + target_link_libraries( + abseil_dll + PRIVATE + ${ABSL_DEFAULT_LINKOPTS} + ) + set_property(TARGET abseil_dll PROPERTY LINKER_LANGUAGE "CXX") + target_include_directories( + abseil_dll + PUBLIC + "$<BUILD_INTERFACE:${ABSL_COMMON_INCLUDE_DIRS}>" + $<INSTALL_INTERFACE:${ABSL_INSTALL_INCLUDEDIR}> + ) + + target_compile_options( + abseil_dll + PRIVATE + ${ABSL_DEFAULT_COPTS} + ) + + target_compile_definitions( + abseil_dll + PRIVATE + ABSL_BUILD_DLL + NOMINMAX + INTERFACE + ${ABSL_CC_LIB_DEFINES} + ) + install(TARGETS abseil_dll EXPORT ${PROJECT_NAME}Targets + RUNTIME DESTINATION ${ABSL_INSTALL_BINDIR} + LIBRARY DESTINATION ${ABSL_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${ABSL_INSTALL_LIBDIR} + ) +endfunction() diff --git a/third_party/abseil_cpp/CMake/AbseilHelpers.cmake b/third_party/abseil_cpp/CMake/AbseilHelpers.cmake new file mode 100644 index 000000000000..e88507de562c --- /dev/null +++ b/third_party/abseil_cpp/CMake/AbseilHelpers.cmake @@ -0,0 +1,402 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +include(CMakeParseArguments) +include(AbseilConfigureCopts) +include(AbseilDll) +include(AbseilInstallDirs) + +# The IDE folder for Abseil that will be used if Abseil is included in a CMake +# project that sets +# set_property(GLOBAL PROPERTY USE_FOLDERS ON) +# For example, Visual Studio supports folders. +if(NOT DEFINED ABSL_IDE_FOLDER) + set(ABSL_IDE_FOLDER Abseil) +endif() + +# absl_cc_library() +# +# CMake function to imitate Bazel's cc_library rule. +# +# Parameters: +# NAME: name of target (see Note) +# HDRS: List of public header files for the library +# SRCS: List of source files for the library +# DEPS: List of other libraries to be linked in to the binary targets +# COPTS: List of private compile options +# DEFINES: List of public defines +# LINKOPTS: List of link options +# PUBLIC: Add this so that this library will be exported under absl:: +# Also in IDE, target will appear in Abseil folder while non PUBLIC will be in Abseil/internal. +# TESTONLY: When added, this target will only be built if user passes -DABSL_RUN_TESTS=ON to CMake. +# +# Note: +# By default, absl_cc_library will always create a library named absl_${NAME}, +# and alias target absl::${NAME}. The absl:: form should always be used. +# This is to reduce namespace pollution. +# +# absl_cc_library( +# NAME +# awesome +# HDRS +# "a.h" +# SRCS +# "a.cc" +# ) +# absl_cc_library( +# NAME +# fantastic_lib +# SRCS +# "b.cc" +# DEPS +# absl::awesome # not "awesome" ! +# PUBLIC +# ) +# +# absl_cc_library( +# NAME +# main_lib +# ... +# DEPS +# absl::fantastic_lib +# ) +# +# TODO: Implement "ALWAYSLINK" +function(absl_cc_library) + cmake_parse_arguments(ABSL_CC_LIB + "DISABLE_INSTALL;PUBLIC;TESTONLY" + "NAME" + "HDRS;SRCS;COPTS;DEFINES;LINKOPTS;DEPS" + ${ARGN} + ) + + if(ABSL_CC_LIB_TESTONLY AND NOT ABSL_RUN_TESTS) + return() + endif() + + if(ABSL_ENABLE_INSTALL) + set(_NAME "${ABSL_CC_LIB_NAME}") + else() + set(_NAME "absl_${ABSL_CC_LIB_NAME}") + endif() + + # Check if this is a header-only library + # Note that as of February 2019, many popular OS's (for example, Ubuntu + # 16.04 LTS) only come with cmake 3.5 by default. For this reason, we can't + # use list(FILTER...) + set(ABSL_CC_SRCS "${ABSL_CC_LIB_SRCS}") + foreach(src_file IN LISTS ABSL_CC_SRCS) + if(${src_file} MATCHES ".*\\.(h|inc)") + list(REMOVE_ITEM ABSL_CC_SRCS "${src_file}") + endif() + endforeach() + + if("${ABSL_CC_SRCS}" STREQUAL "") + set(ABSL_CC_LIB_IS_INTERFACE 1) + else() + set(ABSL_CC_LIB_IS_INTERFACE 0) + endif() + + # Determine this build target's relationship to the DLL. It's one of four things: + # 1. "dll" -- This target is part of the DLL + # 2. "dll_dep" -- This target is not part of the DLL, but depends on the DLL. + # Note that we assume any target not in the DLL depends on the + # DLL. This is not a technical necessity but a convenience + # which happens to be true, because nearly every target is + # part of the DLL. + # 3. "shared" -- This is a shared library, perhaps on a non-windows platform + # where DLL doesn't make sense. + # 4. "static" -- This target does not depend on the DLL and should be built + # statically. + if (${ABSL_BUILD_DLL}) + if(ABSL_ENABLE_INSTALL) + absl_internal_dll_contains(TARGET ${_NAME} OUTPUT _in_dll) + else() + absl_internal_dll_contains(TARGET ${ABSL_CC_LIB_NAME} OUTPUT _in_dll) + endif() + if (${_in_dll}) + # This target should be replaced by the DLL + set(_build_type "dll") + set(ABSL_CC_LIB_IS_INTERFACE 1) + else() + # Building a DLL, but this target is not part of the DLL + set(_build_type "dll_dep") + endif() + elseif(BUILD_SHARED_LIBS) + set(_build_type "shared") + else() + set(_build_type "static") + endif() + + # Generate a pkg-config file for every library: + if(${_build_type} STREQUAL "static" OR ${_build_type} STREQUAL "shared") + if(NOT ABSL_CC_LIB_TESTONLY) + if(absl_VERSION) + set(PC_VERSION "${absl_VERSION}") + else() + set(PC_VERSION "head") + endif() + foreach(dep ${ABSL_CC_LIB_DEPS}) + if(${dep} MATCHES "^absl::(.*)") + set(PC_DEPS "${PC_DEPS} absl_${CMAKE_MATCH_1} = ${PC_VERSION}") + endif() + endforeach() + foreach(cflag ${ABSL_CC_LIB_COPTS}) + if(${cflag} MATCHES "^(-Wno|/wd)") + # These flags are needed to suppress warnings that might fire in our headers. + set(PC_CFLAGS "${PC_CFLAGS} ${cflag}") + elseif(${cflag} MATCHES "^(-W|/w[1234eo])") + # Don't impose our warnings on others. + else() + set(PC_CFLAGS "${PC_CFLAGS} ${cflag}") + endif() + endforeach() + FILE(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/lib/pkgconfig/absl_${_NAME}.pc" CONTENT "\ +prefix=${CMAKE_INSTALL_PREFIX}\n\ +exec_prefix=\${prefix}\n\ +libdir=\${prefix}/lib\n\ +includedir=\${prefix}/include\n\ +\n\ +Name: absl_${_NAME}\n\ +Description: Abseil ${_NAME} library\n\ +URL: https://abseil.io/\n\ +Version: ${PC_VERSION}\n\ +Requires.private:${PC_DEPS}\n\ +Libs: -L\${libdir} $<JOIN:${ABSL_CC_LIB_LINKOPTS}, > $<$<NOT:$<BOOL:${ABSL_CC_LIB_IS_INTERFACE}>>:-labsl_${_NAME}>\n\ +Cflags: -I\${includedir}${PC_CFLAGS}\n") + INSTALL(FILES "${CMAKE_BINARY_DIR}/lib/pkgconfig/absl_${_NAME}.pc" + DESTINATION "${CMAKE_INSTALL_PREFIX}/lib/pkgconfig") + endif() + endif() + + if(NOT ABSL_CC_LIB_IS_INTERFACE) + if(${_build_type} STREQUAL "dll_dep") + # This target depends on the DLL. When adding dependencies to this target, + # any depended-on-target which is contained inside the DLL is replaced + # with a dependency on the DLL. + add_library(${_NAME} STATIC "") + target_sources(${_NAME} PRIVATE ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS}) + absl_internal_dll_targets( + DEPS ${ABSL_CC_LIB_DEPS} + OUTPUT _dll_deps + ) + target_link_libraries(${_NAME} + PUBLIC ${_dll_deps} + PRIVATE + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + + if (ABSL_CC_LIB_TESTONLY) + set(_gtest_link_define "GTEST_LINKED_AS_SHARED_LIBRARY=1") + else() + set(_gtest_link_define) + endif() + + target_compile_definitions(${_NAME} + PUBLIC + ABSL_CONSUME_DLL + "${_gtest_link_define}" + ) + + elseif(${_build_type} STREQUAL "static" OR ${_build_type} STREQUAL "shared") + add_library(${_NAME} "") + target_sources(${_NAME} PRIVATE ${ABSL_CC_LIB_SRCS} ${ABSL_CC_LIB_HDRS}) + target_link_libraries(${_NAME} + PUBLIC ${ABSL_CC_LIB_DEPS} + PRIVATE + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + else() + message(FATAL_ERROR "Invalid build type: ${_build_type}") + endif() + + # Linker language can be inferred from sources, but in the case of DLLs we + # don't have any .cc files so it would be ambiguous. We could set it + # explicitly only in the case of DLLs but, because "CXX" is always the + # correct linker language for static or for shared libraries, we set it + # unconditionally. + set_property(TARGET ${_NAME} PROPERTY LINKER_LANGUAGE "CXX") + + target_include_directories(${_NAME} + PUBLIC + "$<BUILD_INTERFACE:${ABSL_COMMON_INCLUDE_DIRS}>" + $<INSTALL_INTERFACE:${ABSL_INSTALL_INCLUDEDIR}> + ) + target_compile_options(${_NAME} + PRIVATE ${ABSL_CC_LIB_COPTS}) + target_compile_definitions(${_NAME} PUBLIC ${ABSL_CC_LIB_DEFINES}) + + # Add all Abseil targets to a a folder in the IDE for organization. + if(ABSL_CC_LIB_PUBLIC) + set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}) + elseif(ABSL_CC_LIB_TESTONLY) + set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}/test) + else() + set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}/internal) + endif() + + # INTERFACE libraries can't have the CXX_STANDARD property set + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${ABSL_CXX_STANDARD}) + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + + # When being installed, we lose the absl_ prefix. We want to put it back + # to have properly named lib files. This is a no-op when we are not being + # installed. + if(ABSL_ENABLE_INSTALL) + set_target_properties(${_NAME} PROPERTIES + OUTPUT_NAME "absl_${_NAME}" + # TODO(b/173696973): Figure out how to set SOVERSION for LTS releases. + SOVERSION 0 + ) + endif() + else() + # Generating header-only library + add_library(${_NAME} INTERFACE) + target_include_directories(${_NAME} + INTERFACE + "$<BUILD_INTERFACE:${ABSL_COMMON_INCLUDE_DIRS}>" + $<INSTALL_INTERFACE:${ABSL_INSTALL_INCLUDEDIR}> + ) + + if (${_build_type} STREQUAL "dll") + set(ABSL_CC_LIB_DEPS abseil_dll) + endif() + + target_link_libraries(${_NAME} + INTERFACE + ${ABSL_CC_LIB_DEPS} + ${ABSL_CC_LIB_LINKOPTS} + ${ABSL_DEFAULT_LINKOPTS} + ) + target_compile_definitions(${_NAME} INTERFACE ${ABSL_CC_LIB_DEFINES}) + endif() + + # TODO currently we don't install googletest alongside abseil sources, so + # installed abseil can't be tested. + if(NOT ABSL_CC_LIB_TESTONLY AND ABSL_ENABLE_INSTALL) + install(TARGETS ${_NAME} EXPORT ${PROJECT_NAME}Targets + RUNTIME DESTINATION ${ABSL_INSTALL_BINDIR} + LIBRARY DESTINATION ${ABSL_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${ABSL_INSTALL_LIBDIR} + ) + endif() + + add_library(absl::${ABSL_CC_LIB_NAME} ALIAS ${_NAME}) +endfunction() + +# absl_cc_test() +# +# CMake function to imitate Bazel's cc_test rule. +# +# Parameters: +# NAME: name of target (see Usage below) +# SRCS: List of source files for the binary +# DEPS: List of other libraries to be linked in to the binary targets +# COPTS: List of private compile options +# DEFINES: List of public defines +# LINKOPTS: List of link options +# +# Note: +# By default, absl_cc_test will always create a binary named absl_${NAME}. +# This will also add it to ctest list as absl_${NAME}. +# +# Usage: +# absl_cc_library( +# NAME +# awesome +# HDRS +# "a.h" +# SRCS +# "a.cc" +# PUBLIC +# ) +# +# absl_cc_test( +# NAME +# awesome_test +# SRCS +# "awesome_test.cc" +# DEPS +# absl::awesome +# gmock +# gtest_main +# ) +function(absl_cc_test) + if(NOT ABSL_RUN_TESTS) + return() + endif() + + cmake_parse_arguments(ABSL_CC_TEST + "" + "NAME" + "SRCS;COPTS;DEFINES;LINKOPTS;DEPS" + ${ARGN} + ) + + set(_NAME "absl_${ABSL_CC_TEST_NAME}") + + add_executable(${_NAME} "") + target_sources(${_NAME} PRIVATE ${ABSL_CC_TEST_SRCS}) + target_include_directories(${_NAME} + PUBLIC ${ABSL_COMMON_INCLUDE_DIRS} + PRIVATE ${GMOCK_INCLUDE_DIRS} ${GTEST_INCLUDE_DIRS} + ) + + if (${ABSL_BUILD_DLL}) + target_compile_definitions(${_NAME} + PUBLIC + ${ABSL_CC_TEST_DEFINES} + ABSL_CONSUME_DLL + GTEST_LINKED_AS_SHARED_LIBRARY=1 + ) + + # Replace dependencies on targets inside the DLL with abseil_dll itself. + absl_internal_dll_targets( + DEPS ${ABSL_CC_TEST_DEPS} + OUTPUT ABSL_CC_TEST_DEPS + ) + else() + target_compile_definitions(${_NAME} + PUBLIC + ${ABSL_CC_TEST_DEFINES} + ) + endif() + target_compile_options(${_NAME} + PRIVATE ${ABSL_CC_TEST_COPTS} + ) + + target_link_libraries(${_NAME} + PUBLIC ${ABSL_CC_TEST_DEPS} + PRIVATE ${ABSL_CC_TEST_LINKOPTS} + ) + # Add all Abseil targets to a folder in the IDE for organization. + set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}/test) + + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${ABSL_CXX_STANDARD}) + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + + add_test(NAME ${_NAME} COMMAND ${_NAME}) +endfunction() + + +function(check_target my_target) + if(NOT TARGET ${my_target}) + message(FATAL_ERROR " ABSL: compiling absl requires a ${my_target} CMake target in your project, + see CMake/README.md for more details") + endif(NOT TARGET ${my_target}) +endfunction() diff --git a/third_party/abseil_cpp/CMake/AbseilInstallDirs.cmake b/third_party/abseil_cpp/CMake/AbseilInstallDirs.cmake new file mode 100644 index 000000000000..6fc914b60f62 --- /dev/null +++ b/third_party/abseil_cpp/CMake/AbseilInstallDirs.cmake @@ -0,0 +1,20 @@ +include(GNUInstallDirs) + +# absl_VERSION is only set if we are an LTS release being installed, in which +# case it may be into a system directory and so we need to make subdirectories +# for each installed version of Abseil. This mechanism is implemented in +# Abseil's internal Copybara (https://github.com/google/copybara) workflows and +# isn't visible in the CMake buildsystem itself. + +if(absl_VERSION) + set(ABSL_SUBDIR "${PROJECT_NAME}_${PROJECT_VERSION}") + set(ABSL_INSTALL_BINDIR "${CMAKE_INSTALL_BINDIR}/${ABSL_SUBDIR}") + set(ABSL_INSTALL_CONFIGDIR "${CMAKE_INSTALL_LIBDIR}/cmake/${ABSL_SUBDIR}") + set(ABSL_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}/${ABSL_SUBDIR}") + set(ABSL_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}/${ABSL_SUBDIR}") +else() + set(ABSL_INSTALL_BINDIR "${CMAKE_INSTALL_BINDIR}") + set(ABSL_INSTALL_CONFIGDIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME}") + set(ABSL_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_INCLUDEDIR}") + set(ABSL_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}") +endif() diff --git a/third_party/abseil_cpp/CMake/Googletest/CMakeLists.txt.in b/third_party/abseil_cpp/CMake/Googletest/CMakeLists.txt.in new file mode 100644 index 000000000000..5769e3a97b62 --- /dev/null +++ b/third_party/abseil_cpp/CMake/Googletest/CMakeLists.txt.in @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 2.8.2) + +project(googletest-external NONE) + +include(ExternalProject) +ExternalProject_Add(googletest + URL "${absl_gtest_download_url}" # May be empty + SOURCE_DIR "${absl_gtest_src_dir}" + BINARY_DIR "${absl_gtest_build_dir}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/third_party/abseil_cpp/CMake/Googletest/DownloadGTest.cmake b/third_party/abseil_cpp/CMake/Googletest/DownloadGTest.cmake new file mode 100644 index 000000000000..9d071c917011 --- /dev/null +++ b/third_party/abseil_cpp/CMake/Googletest/DownloadGTest.cmake @@ -0,0 +1,41 @@ +# Integrates googletest at configure time. Based on the instructions at +# https://github.com/google/googletest/tree/master/googletest#incorporating-into-an-existing-cmake-project + +# Set up the external googletest project, downloading the latest from Github +# master if requested. +configure_file( + ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.in + ${CMAKE_BINARY_DIR}/googletest-external/CMakeLists.txt +) + +set(ABSL_SAVE_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) +set(ABSL_SAVE_CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) +if (BUILD_SHARED_LIBS) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_CREATE_SHARED_LIBRARY=1") +endif() + +# Configure and build the googletest source. +execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-external ) +if(result) + message(FATAL_ERROR "CMake step for googletest failed: ${result}") +endif() + +execute_process(COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/googletest-external) +if(result) + message(FATAL_ERROR "Build step for googletest failed: ${result}") +endif() + +set(CMAKE_CXX_FLAGS ${ABSL_SAVE_CMAKE_CXX_FLAGS}) +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${ABSL_SAVE_CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + +# Prevent overriding the parent project's compiler/linker settings on Windows +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + +# Add googletest directly to our build. This defines the gtest and gtest_main +# targets. +add_subdirectory(${absl_gtest_src_dir} ${absl_gtest_build_dir} EXCLUDE_FROM_ALL) diff --git a/third_party/abseil_cpp/CMake/README.md b/third_party/abseil_cpp/CMake/README.md new file mode 100644 index 000000000000..8f73475a1621 --- /dev/null +++ b/third_party/abseil_cpp/CMake/README.md @@ -0,0 +1,101 @@ +# Abseil CMake Build Instructions + +Abseil comes with a CMake build script ([CMakeLists.txt](../CMakeLists.txt)) +that can be used on a wide range of platforms ("C" stands for cross-platform.). +If you don't have CMake installed already, you can download it for free from +<https://www.cmake.org/>. + +CMake works by generating native makefiles or build projects that can +be used in the compiler environment of your choice. + +For API/ABI compatibility reasons, we strongly recommend building Abseil in a +subdirectory of your project or as an embedded dependency. + +## Incorporating Abseil Into a CMake Project + +The recommendations below are similar to those for using CMake within the +googletest framework +(<https://github.com/google/googletest/blob/master/googletest/README.md#incorporating-into-an-existing-cmake-project>) + +### Step-by-Step Instructions + +1. If you want to build the Abseil tests, integrate the Abseil dependency +[Google Test](https://github.com/google/googletest) into your CMake project. To disable Abseil tests, you have to pass +`-DBUILD_TESTING=OFF` when configuring your project with CMake. + +2. Download Abseil and copy it into a subdirectory in your CMake project or add +Abseil as a [git submodule](https://git-scm.com/docs/git-submodule) in your +CMake project. + +3. You can then use the CMake command +[`add_subdirectory()`](https://cmake.org/cmake/help/latest/command/add_subdirectory.html) +to include Abseil directly in your CMake project. + +4. Add the **absl::** target you wish to use to the +[`target_link_libraries()`](https://cmake.org/cmake/help/latest/command/target_link_libraries.html) +section of your executable or of your library.<br> +Here is a short CMakeLists.txt example of a project file using Abseil. + +```cmake +cmake_minimum_required(VERSION 3.5) +project(my_project) + +# Pick the C++ standard to compile with. +# Abseil currently supports C++11, C++14, and C++17. +set(CMAKE_CXX_STANDARD 11) + +add_subdirectory(abseil-cpp) + +add_executable(my_exe source.cpp) +target_link_libraries(my_exe absl::base absl::synchronization absl::strings) +``` + +### Running Abseil Tests with CMake + +Use the `-DABSL_RUN_TESTS=ON` flag to run Abseil tests. Note that if the `-DBUILD_TESTING=OFF` flag is passed then Abseil tests will not be run. + +You will need to provide Abseil with a Googletest dependency. There are two +options for how to do this: + +* Use `-DABSL_USE_GOOGLETEST_HEAD`. This will automatically download the latest +Googletest source into the build directory at configure time. Googletest will +then be compiled directly alongside Abseil's tests. +* Manually integrate Googletest with your build. See +https://github.com/google/googletest/blob/master/googletest/README.md#using-cmake +for more information on using Googletest in a CMake project. + +For example, to run just the Abseil tests, you could use this script: + +``` +cd path/to/abseil-cpp +mkdir build +cd build +cmake -DABSL_USE_GOOGLETEST_HEAD=ON -DABSL_RUN_TESTS=ON .. +make -j +ctest +``` + +Currently, we only run our tests with CMake in a Linux environment, but we are +working on the rest of our supported platforms. See +https://github.com/abseil/abseil-cpp/projects/1 and +https://github.com/abseil/abseil-cpp/issues/109 for more information. + +### Available Abseil CMake Public Targets + +Here's a non-exhaustive list of Abseil CMake public targets: + +```cmake +absl::algorithm +absl::base +absl::debugging +absl::flat_hash_map +absl::flags +absl::memory +absl::meta +absl::numeric +absl::random_random +absl::strings +absl::synchronization +absl::time +absl::utility +``` diff --git a/third_party/abseil_cpp/CMake/abslConfig.cmake.in b/third_party/abseil_cpp/CMake/abslConfig.cmake.in new file mode 100644 index 000000000000..62d246d01e00 --- /dev/null +++ b/third_party/abseil_cpp/CMake/abslConfig.cmake.in @@ -0,0 +1,8 @@ +# absl CMake configuration file. + +include(CMakeFindDependencyMacro) +find_dependency(Threads) + +@PACKAGE_INIT@ + +include ("${CMAKE_CURRENT_LIST_DIR}/@PROJECT_NAME@Targets.cmake") diff --git a/third_party/abseil_cpp/CMake/install_test_project/CMakeLists.txt b/third_party/abseil_cpp/CMake/install_test_project/CMakeLists.txt new file mode 100644 index 000000000000..06b797e9eddc --- /dev/null +++ b/third_party/abseil_cpp/CMake/install_test_project/CMakeLists.txt @@ -0,0 +1,27 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A simple CMakeLists.txt for testing cmake installation + +cmake_minimum_required(VERSION 3.5) +project(absl_cmake_testing CXX) + +set(CMAKE_CXX_STANDARD 11) + +add_executable(simple simple.cc) + +find_package(absl REQUIRED) + +target_link_libraries(simple absl::strings) diff --git a/third_party/abseil_cpp/CMake/install_test_project/simple.cc b/third_party/abseil_cpp/CMake/install_test_project/simple.cc new file mode 100644 index 000000000000..e9e352912bab --- /dev/null +++ b/third_party/abseil_cpp/CMake/install_test_project/simple.cc @@ -0,0 +1,23 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <iostream> +#include "absl/strings/substitute.h" + +int main(int argc, char** argv) { + for (int i = 0; i < argc; ++i) { + std::cout << absl::Substitute("Arg $0: $1\n", i, argv[i]); + } +} diff --git a/third_party/abseil_cpp/CMake/install_test_project/test.sh b/third_party/abseil_cpp/CMake/install_test_project/test.sh new file mode 100755 index 000000000000..ddc7726b6c2d --- /dev/null +++ b/third_party/abseil_cpp/CMake/install_test_project/test.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# "Unit" and integration tests for Absl CMake installation + +# TODO(absl-team): This script isn't fully hermetic because +# -DABSL_USE_GOOGLETEST_HEAD=ON means that this script isn't pinned to a fixed +# version of GoogleTest. This means that an upstream change to GoogleTest could +# break this test. Fix this by allowing this script to pin to a known-good +# version of GoogleTest. + +# Fail on any error. Treat unset variables an error. Print commands as executed. +set -euox pipefail + +install_absl() { + pushd "${absl_build_dir}" + if [[ "${#}" -eq 1 ]]; then + cmake -DCMAKE_INSTALL_PREFIX="${1}" "${absl_dir}" + else + cmake "${absl_dir}" + fi + cmake --build . --target install -- -j + popd +} + +uninstall_absl() { + xargs rm < "${absl_build_dir}"/install_manifest.txt + rm -rf "${absl_build_dir}" + mkdir -p "${absl_build_dir}" +} + +lts_install="" + +while getopts ":l" lts; do + case "${lts}" in + l ) + lts_install="true" + ;; + esac +done + +absl_dir=/abseil-cpp +absl_build_dir=/buildfs/absl-build +project_dir="${absl_dir}"/CMake/install_test_project +project_build_dir=/buildfs/project-build + +mkdir -p "${absl_build_dir}" +mkdir -p "${project_build_dir}" + +if [[ "${lts_install}" ]]; then + install_dir="/usr/local" +else + install_dir="${project_build_dir}"/install +fi +mkdir -p "${install_dir}" + +# Test build, install, and link against installed abseil +pushd "${project_build_dir}" +if [[ "${lts_install}" ]]; then + install_absl + cmake "${project_dir}" +else + install_absl "${install_dir}" + cmake "${project_dir}" -DCMAKE_PREFIX_PATH="${install_dir}" +fi + +cmake --build . --target simple + +output="$(${project_build_dir}/simple "printme" 2>&1)" +if [[ "${output}" != *"Arg 1: printme"* ]]; then + echo "Faulty output on simple project:" + echo "${output}" + exit 1 +fi + +popd + +# Test that we haven't accidentally made absl::abslblah +pushd "${install_dir}" + +# Starting in CMake 3.12 the default install dir is lib$bit_width +if [[ -d lib64 ]]; then + libdir="lib64" +elif [[ -d lib ]]; then + libdir="lib" +else + echo "ls *, */*, */*/*:" + ls * + ls */* + ls */*/* + echo "unknown lib dir" +fi + +if [[ "${lts_install}" ]]; then + # LTS versions append the date of the release to the subdir. + # 9999/99/99 is the dummy date used in the local_lts workflow. + absl_subdir="absl_99999999" +else + absl_subdir="absl" +fi + +if ! grep absl::strings "${libdir}/cmake/${absl_subdir}/abslTargets.cmake"; then + cat "${libdir}"/cmake/absl/abslTargets.cmake + echo "CMake targets named incorrectly" + exit 1 +fi + +pushd "${HOME}" +cat > hello-abseil.cc << EOF +#include <cstdlib> + +#include "absl/strings/str_format.h" + +int main(int argc, char **argv) { + absl::PrintF("Hello Abseil!\n"); + return EXIT_SUCCESS; +} +EOF +export PKG_CONFIG_PATH="${install_dir}/${libdir}/pkgconfig" +pc_args=($(pkg-config --cflags --libs --static absl_str_format)) +g++ -static -o hello-abseil hello-abseil.cc "${pc_args[@]}" +hello="$(./hello-abseil)" +[[ "${hello}" == "Hello Abseil!" ]] +popd + +uninstall_absl +popd + +if [[ ! "${lts_install}" ]]; then + # Test that we warn if installed without a prefix or a system prefix + output="$(install_absl 2>&1)" + if [[ "${output}" != *"Please set CMAKE_INSTALL_PREFIX"* ]]; then + echo "Install without prefix didn't warn as expected. Output:" + echo "${output}" + exit 1 + fi + uninstall_absl + + output="$(install_absl /usr 2>&1)" + if [[ "${output}" != *"Please set CMAKE_INSTALL_PREFIX"* ]]; then + echo "Install with /usr didn't warn as expected. Output:" + echo "${output}" + exit 1 + fi + uninstall_absl +fi + +echo "Install test complete!" +exit 0 diff --git a/third_party/abseil_cpp/CMakeLists.txt b/third_party/abseil_cpp/CMakeLists.txt new file mode 100644 index 000000000000..2120cb0097a3 --- /dev/null +++ b/third_party/abseil_cpp/CMakeLists.txt @@ -0,0 +1,200 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Most widely used distributions have cmake 3.5 or greater available as of March +# 2019. A notable exception is RHEL-7 (CentOS7). You can install a current +# version of CMake by first installing Extra Packages for Enterprise Linux +# (https://fedoraproject.org/wiki/EPEL#Extra_Packages_for_Enterprise_Linux_.28EPEL.29) +# and then issuing `yum install cmake3` on the command line. +cmake_minimum_required(VERSION 3.5) + +# Compiler id for Apple Clang is now AppleClang. +if (POLICY CMP0025) + cmake_policy(SET CMP0025 NEW) +endif (POLICY CMP0025) + +# if command can use IN_LIST +if (POLICY CMP0057) + cmake_policy(SET CMP0057 NEW) +endif (POLICY CMP0057) + +# Project version variables are the empty string if version is unspecified +if (POLICY CMP0048) + cmake_policy(SET CMP0048 NEW) +endif (POLICY CMP0048) + +# option() honor variables +if (POLICY CMP0077) + cmake_policy(SET CMP0077 NEW) +endif (POLICY CMP0077) + +project(absl CXX) + +# Output directory is correct by default for most build setups. However, when +# building Abseil as a DLL, it is important to have the DLL in the same +# directory as the executable using it. Thus, we put all executables in a single +# /bin directory. +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + +# when absl is included as subproject (i.e. using add_subdirectory(abseil-cpp)) +# in the source tree of a project that uses it, install rules are disabled. +if(NOT "^${CMAKE_SOURCE_DIR}$" STREQUAL "^${PROJECT_SOURCE_DIR}$") + option(ABSL_ENABLE_INSTALL "Enable install rule" OFF) +else() + option(ABSL_ENABLE_INSTALL "Enable install rule" ON) +endif() + +list(APPEND CMAKE_MODULE_PATH + ${CMAKE_CURRENT_LIST_DIR}/CMake + ${CMAKE_CURRENT_LIST_DIR}/absl/copts +) + +# Linking Abseil to shared libraries (such as Nix) requires PIC. +set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) + +include(AbseilInstallDirs) +include(CMakePackageConfigHelpers) +include(AbseilDll) +include(AbseilHelpers) + + +## +## Using absl targets +## +## all public absl targets are +## exported with the absl:: prefix +## +## e.g absl::base absl::synchronization absl::strings .... +## +## DO NOT rely on the internal targets outside of the prefix + + +# include current path +list(APPEND ABSL_COMMON_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}) + +if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + set(ABSL_USING_CLANG ON) +else() + set(ABSL_USING_CLANG OFF) +endif() + +# find dependencies +## pthread +find_package(Threads REQUIRED) + +option(ABSL_USE_EXTERNAL_GOOGLETEST + "If ON, Abseil will assume that the targets for GoogleTest are already provided by the including project. This makes sense when Abseil is used with add_subproject." OFF) + +option(ABSL_USE_GOOGLETEST_HEAD + "If ON, abseil will download HEAD from GoogleTest at config time." OFF) + +set(ABSL_GOOGLETEST_DOWNLOAD_URL "" CACHE STRING "If set, download GoogleTest from this URL") + +set(ABSL_LOCAL_GOOGLETEST_DIR "/usr/src/googletest" CACHE PATH + "If ABSL_USE_GOOGLETEST_HEAD is OFF and ABSL_GOOGLETEST_URL is not set, specifies the directory of a local GoogleTest checkout." + ) + +option(ABSL_RUN_TESTS "If ON, Abseil tests will be run." OFF) + +if(${ABSL_RUN_TESTS}) + # enable CTest. This will set BUILD_TESTING to ON unless otherwise specified + # on the command line + include(CTest) + + ## check targets + if (NOT ABSL_USE_EXTERNAL_GOOGLETEST) + set(absl_gtest_build_dir ${CMAKE_BINARY_DIR}/googletest-build) + if(ABSL_USE_GOOGLETEST_HEAD AND ABSL_GOOGLETEST_DOWNLOAD_URL) + message(FATAL_ERROR "Do not set both ABSL_USE_GOOGLETEST_HEAD and ABSL_GOOGLETEST_DOWNLOAD_URL") + endif() + if(ABSL_USE_GOOGLETEST_HEAD) + set(absl_gtest_download_url "https://github.com/google/googletest/archive/master.zip") + elseif(ABSL_GOOGLETEST_DOWNLOAD_URL) + set(absl_gtest_download_url ${ABSL_GOOGLETEST_DOWNLOAD_URL}) + endif() + if(absl_gtest_download_url) + set(absl_gtest_src_dir ${CMAKE_BINARY_DIR}/googletest-src) + else() + set(absl_gtest_src_dir ${ABSL_LOCAL_GOOGLETEST_DIR}) + endif() + include(CMake/Googletest/DownloadGTest.cmake) + endif() + + check_target(gtest) + check_target(gtest_main) + check_target(gmock) + + list(APPEND ABSL_TEST_COMMON_LIBRARIES + gtest_main + gtest + gmock + ${CMAKE_THREAD_LIBS_INIT} + ) +endif() + +add_subdirectory(absl) + +if(ABSL_ENABLE_INSTALL) + # absl:lts-remove-begin(system installation is supported for LTS releases) + # We don't support system-wide installation + list(APPEND SYSTEM_INSTALL_DIRS "/usr/local" "/usr" "/opt/" "/opt/local" "c:/Program Files/${PROJECT_NAME}") + if(NOT DEFINED CMAKE_INSTALL_PREFIX OR CMAKE_INSTALL_PREFIX IN_LIST SYSTEM_INSTALL_DIRS) + message(WARNING "\ + The default and system-level install directories are unsupported except in LTS \ + releases of Abseil. Please set CMAKE_INSTALL_PREFIX to install Abseil in your \ + source or build tree directly.\ + ") + endif() + # absl:lts-remove-end + + # install as a subdirectory only + install(EXPORT ${PROJECT_NAME}Targets + NAMESPACE absl:: + DESTINATION "${ABSL_INSTALL_CONFIGDIR}" + ) + + configure_package_config_file( + CMake/abslConfig.cmake.in + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + INSTALL_DESTINATION "${ABSL_INSTALL_CONFIGDIR}" + ) + install(FILES "${PROJECT_BINARY_DIR}/${PROJECT_NAME}Config.cmake" + DESTINATION "${ABSL_INSTALL_CONFIGDIR}" + ) + + # Abseil only has a version in LTS releases. This mechanism is accomplished + # Abseil's internal Copybara (https://github.com/google/copybara) workflows and + # isn't visible in the CMake buildsystem itself. + if(absl_VERSION) + write_basic_package_version_file( + "${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + COMPATIBILITY ExactVersion + ) + + install(FILES "${PROJECT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake" + DESTINATION ${ABSL_INSTALL_CONFIGDIR} + ) + endif() # absl_VERSION + + install(DIRECTORY absl + DESTINATION ${ABSL_INSTALL_INCLUDEDIR} + FILES_MATCHING + PATTERN "*.inc" + PATTERN "*.h" + PATTERN "copts" EXCLUDE + PATTERN "testdata" EXCLUDE + ) +endif() # ABSL_ENABLE_INSTALL diff --git a/third_party/abseil_cpp/CONTRIBUTING.md b/third_party/abseil_cpp/CONTRIBUTING.md new file mode 100644 index 000000000000..9dadae937601 --- /dev/null +++ b/third_party/abseil_cpp/CONTRIBUTING.md @@ -0,0 +1,141 @@ +# How to Contribute to Abseil + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +NOTE: If you are new to GitHub, please start by reading [Pull Request +howto](https://help.github.com/articles/about-pull-requests/) + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution, +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to <https://cla.developers.google.com/> to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Contribution Guidelines + +Potential contributors sometimes ask us if the Abseil project is the appropriate +home for their utility library code or for specific functions implementing +missing portions of the standard. Often, the answer to this question is "no". +We’d like to articulate our thinking on this issue so that our choices can be +understood by everyone and so that contributors can have a better intuition +about whether Abseil might be interested in adopting a new library. + +### Priorities + +Although our mission is to augment the C++ standard library, our goal is not to +provide a full forward-compatible implementation of the latest standard. For us +to consider a library for inclusion in Abseil, it is not enough that a library +is useful. We generally choose to release a library when it meets at least one +of the following criteria: + +* **Widespread usage** - Using our internal codebase to help gauge usage, most + of the libraries we've released have tens of thousands of users. +* **Anticipated widespread usage** - Pre-adoption of some standard-compliant + APIs may not have broad adoption initially but can be expected to pick up + usage when it replaces legacy APIs. `absl::from_chars`, for example, + replaces existing code that converts strings to numbers and will therefore + likely see usage growth. +* **High impact** - APIs that provide a key solution to a specific problem, + such as `absl::FixedArray`, have higher impact than usage numbers may signal + and are released because of their importance. +* **Direct support for a library that falls under one of the above** - When we + want access to a smaller library as an implementation detail for a + higher-priority library we plan to release, we may release it, as we did + with portions of `absl/meta/type_traits.h`. One consequence of this is that + the presence of a library in Abseil does not necessarily mean that other + similar libraries would be a high priority. + +### API Freeze Consequences + +Via the +[Abseil Compatibility Guidelines](https://abseil.io/about/compatibility), we +have promised a large degree of API stability. In particular, we will not make +backward-incompatible changes to released APIs without also shipping a tool or +process that can upgrade our users' code. We are not yet at the point of easily +releasing such tools. Therefore, at this time, shipping a library establishes an +API contract which is borderline unchangeable. (We can add new functionality, +but we cannot easily change existing behavior.) This constraint forces us to +very carefully review all APIs that we ship. + + +## Coding Style + +To keep the source consistent, readable, diffable and easy to merge, we use a +fairly rigid coding style, as defined by the +[google-styleguide](https://github.com/google/styleguide) project. All patches +will be expected to conform to the style outlined +[here](https://google.github.io/styleguide/cppguide.html). + +## Guidelines for Pull Requests + +* If you are a Googler, it is preferable to first create an internal CL and + have it reviewed and submitted. The code propagation process will deliver + the change to GitHub. + +* Create **small PRs** that are narrowly focused on **addressing a single + concern**. We often receive PRs that are trying to fix several things at a + time, but if only one fix is considered acceptable, nothing gets merged and + both author's & review's time is wasted. Create more PRs to address + different concerns and everyone will be happy. + +* For speculative changes, consider opening an [Abseil + issue](https://github.com/abseil/abseil-cpp/issues) and discussing it first. + If you are suggesting a behavioral or API change, consider starting with an + [Abseil proposal template](ABSEIL_ISSUE_TEMPLATE.md). + +* Provide a good **PR description** as a record of **what** change is being + made and **why** it was made. Link to a GitHub issue if it exists. + +* Don't fix code style and formatting unless you are already changing that + line to address an issue. Formatting of modified lines may be done using + `git clang-format`. PRs with irrelevant changes won't be merged. If + you do want to fix formatting or style, do that in a separate PR. + +* Unless your PR is trivial, you should expect there will be reviewer comments + that you'll need to address before merging. We expect you to be reasonably + responsive to those comments, otherwise the PR will be closed after 2-3 + weeks of inactivity. + +* Maintain **clean commit history** and use **meaningful commit messages**. + PRs with messy commit history are difficult to review and won't be merged. + Use `rebase -i upstream/master` to curate your commit history and/or to + bring in latest changes from master (but avoid rebasing in the middle of a + code review). + +* Keep your PR up to date with upstream/master (if there are merge conflicts, + we can't really merge your change). + +* **All tests need to be passing** before your change can be merged. We + recommend you **run tests locally** (see below) + +* Exceptions to the rules can be made if there's a compelling reason for doing + so. That is - the rules are here to serve us, not the other way around, and + the rules need to be serving their intended purpose to be valuable. + +* All submissions, including submissions by project members, require review. + +## Running Tests + +If you have [Bazel](https://bazel.build/) installed, use `bazel test +--test_tag_filters="-benchmark" ...` to run the unit tests. + +If you are running the Linux operating system and have +[Docker](https://www.docker.com/) installed, you can also run the `linux_*.sh` +scripts under the `ci/`(https://github.com/abseil/abseil-cpp/tree/master/ci) +directory to test Abseil under a variety of conditions. + +## Abseil Committers + +The current members of the Abseil engineering team are the only committers at +present. + +## Release Process + +Abseil lives at head, where latest-and-greatest code can be found. diff --git a/third_party/abseil_cpp/FAQ.md b/third_party/abseil_cpp/FAQ.md new file mode 100644 index 000000000000..78028fc09f1f --- /dev/null +++ b/third_party/abseil_cpp/FAQ.md @@ -0,0 +1,164 @@ +# Abseil FAQ + +## Is Abseil the right home for my utility library? + +Most often the answer to the question is "no." As both the [About +Abseil](https://abseil.io/about/) page and our [contributing +guidelines](https://github.com/abseil/abseil-cpp/blob/master/CONTRIBUTING.md#contribution-guidelines) +explain, Abseil contains a variety of core C++ library code that is widely used +at [Google](https://www.google.com/). As such, Abseil's primary purpose is to be +used as a dependency by Google's open source C++ projects. While we do hope that +Abseil is also useful to the C++ community at large, this added constraint also +means that we are unlikely to accept a contribution of utility code that isn't +already widely used by Google. + +## How to I set the C++ dialect used to build Abseil? + +The short answer is that whatever mechanism you choose, you need to make sure +that you set this option consistently at the global level for your entire +project. If, for example, you want to set the C++ dialect to C++17, with +[Bazel](https://bazel/build/) as the build system and `gcc` or `clang` as the +compiler, there several ways to do this: +* Pass `--cxxopt=-std=c++17` on the command line (for example, `bazel build + --cxxopt=-std=c++17 ...`) +* Set the environment variable `BAZEL_CXXOPTS` (for example, + `BAZEL_CXXOPTS=-std=c++17`) +* Add `build --cxxopt=-std=c++17` to your [`.bazelrc` + file](https://docs.bazel.build/versions/master/guide.html#bazelrc) + +If you are using CMake as the build system, you'll need to add a line like +`set(CMAKE_CXX_STANDARD 17)` to your top level `CMakeLists.txt` file. See the +[CMake build +instructions](https://github.com/abseil/abseil-cpp/blob/master/CMake/README.md) +for more information. + +For a longer answer to this question and to understand why some other approaches +don't work, see the answer to ["What is ABI and why don't you recommend using a +pre-compiled version of +Abseil?"](#what-is-abi-and-why-dont-you-recommend-using-a-pre-compiled-version-of-abseil) + +## What is ABI and why don't you recommend using a pre-compiled version of Abseil? + +For the purposes of this discussion, you can think of +[ABI](https://en.wikipedia.org/wiki/Application_binary_interface) as the +compiled representation of the interfaces in code. This is in contrast to +[API](https://en.wikipedia.org/wiki/Application_programming_interface), which +you can think of as the interfaces as defined by the code itself. [Abseil has a +strong promise of API compatibility, but does not make any promise of ABI +compatibility](https://abseil.io/about/compatibility). Let's take a look at what +this means in practice. + +You might be tempted to do something like this in a +[Bazel](https://bazel.build/) `BUILD` file: + +``` +# DON'T DO THIS!!! +cc_library( + name = "my_library", + srcs = ["my_library.cc"], + copts = ["-std=c++17"], # May create a mixed-mode compile! + deps = ["@com_google_absl//absl/strings"], +) +``` + +Applying `-std=c++17` to an individual target in your `BUILD` file is going to +compile that specific target in C++17 mode, but it isn't going to ensure the +Abseil library is built in C++17 mode, since the Abseil library itself is a +different build target. If your code includes an Abseil header, then your +program may contain conflicting definitions of the same +class/function/variable/enum, etc. As a rule, all compile options that affect +the ABI of a program need to be applied to the entire build on a global basis. + +C++ has something called the [One Definition +Rule](https://en.wikipedia.org/wiki/One_Definition_Rule) (ODR). C++ doesn't +allow multiple definitions of the same class/function/variable/enum, etc. ODR +violations sometimes result in linker errors, but linkers do not always catch +violations. Uncaught ODR violations can result in strange runtime behaviors or +crashes that can be hard to debug. + +If you build the Abseil library and your code using different compile options +that affect ABI, there is a good chance you will run afoul of the One Definition +Rule. Examples of GCC compile options that affect ABI include (but aren't +limited to) language dialect (e.g. `-std=`), optimization level (e.g. `-O2`), +code generation flags (e.g. `-fexceptions`), and preprocessor defines +(e.g. `-DNDEBUG`). + +If you use a pre-compiled version of Abseil, (for example, from your Linux +distribution package manager or from something like +[vcpkg](https://github.com/microsoft/vcpkg)) you have to be very careful to +ensure ABI compatibility across the components of your program. The only way you +can be sure your program is going to be correct regarding ABI is to ensure +you've used the exact same compile options as were used to build the +pre-compiled library. This does not mean that Abseil cannot work as part of a +Linux distribution since a knowledgeable binary packager will have ensured that +all packages have been built with consistent compile options. This is one of the +reasons we warn against - though do not outright reject - using Abseil as a +pre-compiled library. + +Another possible way that you might afoul of ABI issues is if you accidentally +include two versions of Abseil in your program. Multiple versions of Abseil can +end up within the same binary if your program uses the Abseil library and +another library also transitively depends on Abseil (resulting in what is +sometimes called the diamond dependency problem). In cases such as this you must +structure your build so that all libraries use the same version of Abseil. +[Abseil's strong promise of API compatibility between +releases](https://abseil.io/about/compatibility) means the latest "HEAD" release +of Abseil is almost certainly the right choice if you are doing as we recommend +and building all of your code from source. + +For these reasons we recommend you avoid pre-compiled code and build the Abseil +library yourself in a consistent manner with the rest of your code. + +## What is "live at head" and how do I do it? + +From Abseil's point-of-view, "live at head" means that every Abseil source +release (which happens on an almost daily basis) is either API compatible with +the previous release, or comes with an automated tool that you can run over code +to make it compatible. In practice, the need to use an automated tool is +extremely rare. This means that upgrading from one source release to another +should be a routine practice that can and should be performed often. + +We recommend you update to the [latest commit in the `master` branch of +Abseil](https://github.com/abseil/abseil-cpp/commits/master) as often as +possible. Not only will you pick up bug fixes more quickly, but if you have good +automated testing, you will catch and be able to fix any [Hyrum's +Law](https://www.hyrumslaw.com/) dependency problems on an incremental basis +instead of being overwhelmed by them and having difficulty isolating them if you +wait longer between updates. + +If you are using the [Bazel](https://bazel.build/) build system and its +[external dependencies](https://docs.bazel.build/versions/master/external.html) +feature, updating the +[`http_archive`](https://docs.bazel.build/versions/master/repo/http.html#http_archive) +rule in your +[`WORKSPACE`](https://docs.bazel.build/versions/master/be/workspace.html) for +`com_google_abseil` to point to the [latest commit in the `master` branch of +Abseil](https://github.com/abseil/abseil-cpp/commits/master) is all you need to +do. For example, on February 11, 2020, the latest commit to the master branch +was `98eb410c93ad059f9bba1bf43f5bb916fc92a5ea`. To update to this commit, you +would add the following snippet to your `WORKSPACE` file: + +``` +http_archive( + name = "com_google_absl", + urls = ["https://github.com/abseil/abseil-cpp/archive/98eb410c93ad059f9bba1bf43f5bb916fc92a5ea.zip"], # 2020-02-11T18:50:53Z + strip_prefix = "abseil-cpp-98eb410c93ad059f9bba1bf43f5bb916fc92a5ea", + sha256 = "aabf6c57e3834f8dc3873a927f37eaf69975d4b28117fc7427dfb1c661542a87", +) +``` + +To get the `sha256` of this URL, run `curl -sL --output - +https://github.com/abseil/abseil-cpp/archive/98eb410c93ad059f9bba1bf43f5bb916fc92a5ea.zip +| sha256sum -`. + +You can commit the updated `WORKSPACE` file to your source control every time +you update, and if you have good automated testing, you might even consider +automating this. + +One thing we don't recommend is using GitHub's `master.zip` files (for example +[https://github.com/abseil/abseil-cpp/archive/master.zip](https://github.com/abseil/abseil-cpp/archive/master.zip)), +which are always the latest commit in the `master` branch, to implement live at +head. Since these `master.zip` URLs are not versioned, you will lose build +reproducibility. In addition, some build systems, including Bazel, will simply +cache this file, which means you won't actually be updating to the latest +release until your cache is cleared or invalidated. diff --git a/third_party/abseil_cpp/LICENSE b/third_party/abseil_cpp/LICENSE new file mode 100644 index 000000000000..ccd61dcfe3db --- /dev/null +++ b/third_party/abseil_cpp/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/third_party/abseil_cpp/LTS.md b/third_party/abseil_cpp/LTS.md new file mode 100644 index 000000000000..ade8b17c7367 --- /dev/null +++ b/third_party/abseil_cpp/LTS.md @@ -0,0 +1,16 @@ +# Long Term Support (LTS) Branches + +This repository contains periodic snapshots of the Abseil codebase that are +Long Term Support (LTS) branches. An LTS branch allows you to use a known +version of Abseil without interfering with other projects which may also, in +turn, use Abseil. (For more information about our releases, see the +[Abseil Release Management](https://abseil.io/about/releases) guide.) + +## LTS Branches + +The following lists LTS branches and the dates on which they have been released: + +* [LTS Branch December 18, 2018](https://github.com/abseil/abseil-cpp/tree/lts_2018_12_18/) +* [LTS Branch June 20, 2018](https://github.com/abseil/abseil-cpp/tree/lts_2018_06_20/) +* [LTS Branch August 8, 2019](https://github.com/abseil/abseil-cpp/tree/lts_2019_08_08/) +* [LTS Branch February 25, 2020](https://github.com/abseil/abseil-cpp/tree/lts_2020_02_25/) diff --git a/third_party/abseil_cpp/README.md b/third_party/abseil_cpp/README.md new file mode 100644 index 000000000000..85de569658c6 --- /dev/null +++ b/third_party/abseil_cpp/README.md @@ -0,0 +1,114 @@ +# Abseil - C++ Common Libraries + +The repository contains the Abseil C++ library code. Abseil is an open-source +collection of C++ code (compliant to C++11) designed to augment the C++ +standard library. + +## Table of Contents + +- [About Abseil](#about) +- [Quickstart](#quickstart) +- [Building Abseil](#build) +- [Codemap](#codemap) +- [License](#license) +- [Links](#links) + +<a name="about"></a> +## About Abseil + +Abseil is an open-source collection of C++ library code designed to augment +the C++ standard library. The Abseil library code is collected from Google's +own C++ code base, has been extensively tested and used in production, and +is the same code we depend on in our daily coding lives. + +In some cases, Abseil provides pieces missing from the C++ standard; in +others, Abseil provides alternatives to the standard for special needs +we've found through usage in the Google code base. We denote those cases +clearly within the library code we provide you. + +Abseil is not meant to be a competitor to the standard library; we've +just found that many of these utilities serve a purpose within our code +base, and we now want to provide those resources to the C++ community as +a whole. + +<a name="quickstart"></a> +## Quickstart + +If you want to just get started, make sure you at least run through the +[Abseil Quickstart](https://abseil.io/docs/cpp/quickstart). The Quickstart +contains information about setting up your development environment, downloading +the Abseil code, running tests, and getting a simple binary working. + +<a name="build"></a> +## Building Abseil + +[Bazel](https://bazel.build) is the official build system for Abseil, +which is supported on most major platforms (Linux, Windows, macOS, for example) +and compilers. See the [quickstart](https://abseil.io/docs/cpp/quickstart) for +more information on building Abseil using the Bazel build system. + +<a name="cmake"></a> +If you require CMake support, please check the +[CMake build instructions](CMake/README.md). + +## Codemap + +Abseil contains the following C++ library components: + +* [`base`](absl/base/) Abseil Fundamentals + <br /> The `base` library contains initialization code and other code which + all other Abseil code depends on. Code within `base` may not depend on any + other code (other than the C++ standard library). +* [`algorithm`](absl/algorithm/) + <br /> The `algorithm` library contains additions to the C++ `<algorithm>` + library and container-based versions of such algorithms. +* [`container`](absl/container/) + <br /> The `container` library contains additional STL-style containers, + including Abseil's unordered "Swiss table" containers. +* [`debugging`](absl/debugging/) + <br /> The `debugging` library contains code useful for enabling leak + checks, and stacktrace and symbolization utilities. +* [`hash`](absl/hash/) + <br /> The `hash` library contains the hashing framework and default hash + functor implementations for hashable types in Abseil. +* [`memory`](absl/memory/) + <br /> The `memory` library contains C++11-compatible versions of + `std::make_unique()` and related memory management facilities. +* [`meta`](absl/meta/) + <br /> The `meta` library contains C++11-compatible versions of type checks + available within C++14 and C++17 versions of the C++ `<type_traits>` library. +* [`numeric`](absl/numeric/) + <br /> The `numeric` library contains C++11-compatible 128-bit integers. +* [`strings`](absl/strings/) + <br /> The `strings` library contains a variety of strings routines and + utilities, including a C++11-compatible version of the C++17 + `std::string_view` type. +* [`synchronization`](absl/synchronization/) + <br /> The `synchronization` library contains concurrency primitives (Abseil's + `absl::Mutex` class, an alternative to `std::mutex`) and a variety of + synchronization abstractions. +* [`time`](absl/time/) + <br /> The `time` library contains abstractions for computing with absolute + points in time, durations of time, and formatting and parsing time within + time zones. +* [`types`](absl/types/) + <br /> The `types` library contains non-container utility types, like a + C++11-compatible version of the C++17 `std::optional` type. +* [`utility`](absl/utility/) + <br /> The `utility` library contains utility and helper code. + +## License + +The Abseil C++ library is licensed under the terms of the Apache +license. See [LICENSE](LICENSE) for more information. + +## Links + +For more information about Abseil: + +* Consult our [Abseil Introduction](https://abseil.io/about/intro) +* Read [Why Adopt Abseil](https://abseil.io/about/philosophy) to understand our + design philosophy. +* Peruse our + [Abseil Compatibility Guarantees](https://abseil.io/about/compatibility) to + understand both what we promise to you, and what we expect of you in return. diff --git a/third_party/abseil_cpp/UPGRADES.md b/third_party/abseil_cpp/UPGRADES.md new file mode 100644 index 000000000000..35599d0878d1 --- /dev/null +++ b/third_party/abseil_cpp/UPGRADES.md @@ -0,0 +1,17 @@ +# C++ Upgrade Tools + +Abseil may occassionally release API-breaking changes. As noted in our +[Compatibility Guidelines][compatibility-guide], we will aim to provide a tool +to do the work of effecting such API-breaking changes, when absolutely +necessary. + +These tools will be listed on the [C++ Upgrade Tools][upgrade-tools] guide on +https://abseil.io. + +For more information, the [C++ Automated Upgrade Guide][api-upgrades-guide] +outlines this process. + +[compatibility-guide]: https://abseil.io/about/compatibility +[api-upgrades-guide]: https://abseil.io/docs/cpp/tools/api-upgrades +[upgrade-tools]: https://abseil.io/docs/cpp/tools/upgrades/ + diff --git a/third_party/abseil_cpp/WORKSPACE b/third_party/abseil_cpp/WORKSPACE new file mode 100644 index 000000000000..ed90d2ba9587 --- /dev/null +++ b/third_party/abseil_cpp/WORKSPACE @@ -0,0 +1,45 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +workspace(name = "com_google_absl") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# GoogleTest/GoogleMock framework. Used by most unit-tests. +http_archive( + name = "com_google_googletest", + # Keep this URL in sync with ABSL_GOOGLETEST_COMMIT in ci/cmake_common.sh. + urls = ["https://github.com/google/googletest/archive/8567b09290fe402cf01923e2131c5635b8ed851b.zip"], # 2020-06-12T22:24:28Z + strip_prefix = "googletest-8567b09290fe402cf01923e2131c5635b8ed851b", + sha256 = "9a8a166eb6a56c7b3d7b19dc2c946fe4778fd6f21c7a12368ad3b836d8f1be48", +) + +# Google benchmark. +http_archive( + name = "com_github_google_benchmark", + urls = ["https://github.com/google/benchmark/archive/16703ff83c1ae6d53e5155df3bb3ab0bc96083be.zip"], + strip_prefix = "benchmark-16703ff83c1ae6d53e5155df3bb3ab0bc96083be", + sha256 = "59f918c8ccd4d74b6ac43484467b500f1d64b40cc1010daa055375b322a43ba3", +) + +# C++ rules for Bazel. +http_archive( + name = "rules_cc", + sha256 = "9a446e9dd9c1bb180c86977a8dc1e9e659550ae732ae58bd2e8fd51e15b2c91d", + strip_prefix = "rules_cc-262ebec3c2296296526740db4aefce68c80de7fa", + urls = [ + "https://github.com/bazelbuild/rules_cc/archive/262ebec3c2296296526740db4aefce68c80de7fa.zip", + ], +) diff --git a/third_party/abseil_cpp/absl/BUILD.bazel b/third_party/abseil_cpp/absl/BUILD.bazel new file mode 100644 index 000000000000..6da20c49d28d --- /dev/null +++ b/third_party/abseil_cpp/absl/BUILD.bazel @@ -0,0 +1,65 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +config_setting( + name = "clang_compiler", + flag_values = { + "@bazel_tools//tools/cpp:compiler": "clang", + }, + visibility = [":__subpackages__"], +) + +config_setting( + name = "osx", + constraint_values = [ + "@bazel_tools//platforms:osx", + ], +) + +config_setting( + name = "ios", + constraint_values = [ + "@bazel_tools//platforms:ios", + ], +) + +config_setting( + name = "windows", + constraint_values = [ + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:windows", + ], + visibility = [":__subpackages__"], +) + +config_setting( + name = "ppc", + values = { + "cpu": "ppc", + }, + visibility = [":__subpackages__"], +) + +config_setting( + name = "wasm", + values = { + "cpu": "wasm32", + }, + visibility = [":__subpackages__"], +) diff --git a/third_party/abseil_cpp/absl/CMakeLists.txt b/third_party/abseil_cpp/absl/CMakeLists.txt new file mode 100644 index 000000000000..fbfa7822b594 --- /dev/null +++ b/third_party/abseil_cpp/absl/CMakeLists.txt @@ -0,0 +1,37 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_subdirectory(base) +add_subdirectory(algorithm) +add_subdirectory(container) +add_subdirectory(debugging) +add_subdirectory(flags) +add_subdirectory(functional) +add_subdirectory(hash) +add_subdirectory(memory) +add_subdirectory(meta) +add_subdirectory(numeric) +add_subdirectory(random) +add_subdirectory(status) +add_subdirectory(strings) +add_subdirectory(synchronization) +add_subdirectory(time) +add_subdirectory(types) +add_subdirectory(utility) + +if (${ABSL_BUILD_DLL}) + absl_make_dll() +endif() diff --git a/third_party/abseil_cpp/absl/abseil.podspec.gen.py b/third_party/abseil_cpp/absl/abseil.podspec.gen.py new file mode 100755 index 000000000000..63752980d09b --- /dev/null +++ b/third_party/abseil_cpp/absl/abseil.podspec.gen.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""This script generates abseil.podspec from all BUILD.bazel files. + +This is expected to run on abseil git repository with Bazel 1.0 on Linux. +It recursively analyzes BUILD.bazel files using query command of Bazel to +dump its build rules in XML format. From these rules, it constructs podspec +structure. +""" + +import argparse +import collections +import os +import re +import subprocess +import xml.etree.ElementTree + +# Template of root podspec. +SPEC_TEMPLATE = """ +# This file has been automatically generated from a script. +# Please make modifications to `abseil.podspec.gen.py` instead. +Pod::Spec.new do |s| + s.name = 'abseil' + s.version = '${version}' + s.summary = 'Abseil Common Libraries (C++) from Google' + s.homepage = 'https://abseil.io' + s.license = 'Apache License, Version 2.0' + s.authors = { 'Abseil Team' => 'abseil-io@googlegroups.com' } + s.source = { + :git => 'https://github.com/abseil/abseil-cpp.git', + :tag => '${tag}', + } + s.module_name = 'absl' + s.header_mappings_dir = 'absl' + s.header_dir = 'absl' + s.libraries = 'c++' + s.compiler_flags = '-Wno-everything' + s.pod_target_xcconfig = { + 'USER_HEADER_SEARCH_PATHS' => '$(inherited) "$(PODS_TARGET_SRCROOT)"', + 'USE_HEADERMAP' => 'NO', + 'ALWAYS_SEARCH_USER_PATHS' => 'NO', + } + s.ios.deployment_target = '9.0' + s.osx.deployment_target = '10.10' + s.tvos.deployment_target = '9.0' + s.watchos.deployment_target = '2.0' +""" + +# Rule object representing the rule of Bazel BUILD. +Rule = collections.namedtuple( + "Rule", "type name package srcs hdrs textual_hdrs deps visibility testonly") + + +def get_elem_value(elem, name): + """Returns the value of XML element with the given name.""" + for child in elem: + if child.attrib.get("name") != name: + continue + if child.tag == "string": + return child.attrib.get("value") + if child.tag == "boolean": + return child.attrib.get("value") == "true" + if child.tag == "list": + return [nested_child.attrib.get("value") for nested_child in child] + raise "Cannot recognize tag: " + child.tag + return None + + +def normalize_paths(paths): + """Returns the list of normalized path.""" + # e.g. ["//absl/strings:dir/header.h"] -> ["absl/strings/dir/header.h"] + return [path.lstrip("/").replace(":", "/") for path in paths] + + +def parse_rule(elem, package): + """Returns a rule from bazel XML rule.""" + return Rule( + type=elem.attrib["class"], + name=get_elem_value(elem, "name"), + package=package, + srcs=normalize_paths(get_elem_value(elem, "srcs") or []), + hdrs=normalize_paths(get_elem_value(elem, "hdrs") or []), + textual_hdrs=normalize_paths(get_elem_value(elem, "textual_hdrs") or []), + deps=get_elem_value(elem, "deps") or [], + visibility=get_elem_value(elem, "visibility") or [], + testonly=get_elem_value(elem, "testonly") or False) + + +def read_build(package): + """Runs bazel query on given package file and returns all cc rules.""" + result = subprocess.check_output( + ["bazel", "query", package + ":all", "--output", "xml"]) + root = xml.etree.ElementTree.fromstring(result) + return [ + parse_rule(elem, package) + for elem in root + if elem.tag == "rule" and elem.attrib["class"].startswith("cc_") + ] + + +def collect_rules(root_path): + """Collects and returns all rules from root path recursively.""" + rules = [] + for cur, _, _ in os.walk(root_path): + build_path = os.path.join(cur, "BUILD.bazel") + if os.path.exists(build_path): + rules.extend(read_build("//" + cur)) + return rules + + +def relevant_rule(rule): + """Returns true if a given rule is relevant when generating a podspec.""" + return ( + # cc_library only (ignore cc_test, cc_binary) + rule.type == "cc_library" and + # ignore empty rule + (rule.hdrs + rule.textual_hdrs + rule.srcs) and + # ignore test-only rule + not rule.testonly) + + +def get_spec_var(depth): + """Returns the name of variable for spec with given depth.""" + return "s" if depth == 0 else "s{}".format(depth) + + +def get_spec_name(label): + """Converts the label of bazel rule to the name of podspec.""" + assert label.startswith("//absl/"), "{} doesn't start with //absl/".format( + label) + # e.g. //absl/apple/banana -> abseil/apple/banana + return "abseil/" + label[7:] + + +def write_podspec(f, rules, args): + """Writes a podspec from given rules and args.""" + rule_dir = build_rule_directory(rules)["abseil"] + # Write root part with given arguments + spec = re.sub(r"\$\{(\w+)\}", lambda x: args[x.group(1)], + SPEC_TEMPLATE).lstrip() + f.write(spec) + # Write all target rules + write_podspec_map(f, rule_dir, 0) + f.write("end\n") + + +def build_rule_directory(rules): + """Builds a tree-style rule directory from given rules.""" + rule_dir = {} + for rule in rules: + cur = rule_dir + for frag in get_spec_name(rule.package).split("/"): + cur = cur.setdefault(frag, {}) + cur[rule.name] = rule + return rule_dir + + +def write_podspec_map(f, cur_map, depth): + """Writes podspec from rule map recursively.""" + for key, value in sorted(cur_map.items()): + indent = " " * (depth + 1) + f.write("{indent}{var0}.subspec '{key}' do |{var1}|\n".format( + indent=indent, + key=key, + var0=get_spec_var(depth), + var1=get_spec_var(depth + 1))) + if isinstance(value, dict): + write_podspec_map(f, value, depth + 1) + else: + write_podspec_rule(f, value, depth + 1) + f.write("{indent}end\n".format(indent=indent)) + + +def write_podspec_rule(f, rule, depth): + """Writes podspec from given rule.""" + indent = " " * (depth + 1) + spec_var = get_spec_var(depth) + # Puts all files in hdrs, textual_hdrs, and srcs into source_files. + # Since CocoaPods treats header_files a bit differently from bazel, + # this won't generate a header_files field so that all source_files + # are considered as header files. + srcs = sorted(set(rule.hdrs + rule.textual_hdrs + rule.srcs)) + write_indented_list( + f, "{indent}{var}.source_files = ".format(indent=indent, var=spec_var), + srcs) + # Writes dependencies of this rule. + for dep in sorted(rule.deps): + name = get_spec_name(dep.replace(":", "/")) + f.write("{indent}{var}.dependency '{dep}'\n".format( + indent=indent, var=spec_var, dep=name)) + + +def write_indented_list(f, leading, values): + """Writes leading values in an indented style.""" + f.write(leading) + f.write((",\n" + " " * len(leading)).join("'{}'".format(v) for v in values)) + f.write("\n") + + +def generate(args): + """Generates a podspec file from all BUILD files under absl directory.""" + rules = filter(relevant_rule, collect_rules("absl")) + with open(args.output, "wt") as f: + write_podspec(f, rules, vars(args)) + + +def main(): + parser = argparse.ArgumentParser( + description="Generates abseil.podspec from BUILD.bazel") + parser.add_argument( + "-v", "--version", help="The version of podspec", required=True) + parser.add_argument( + "-t", + "--tag", + default=None, + help="The name of git tag (default: version)") + parser.add_argument( + "-o", + "--output", + default="abseil.podspec", + help="The name of output file (default: abseil.podspec)") + args = parser.parse_args() + if args.tag is None: + args.tag = args.version + generate(args) + + +if __name__ == "__main__": + main() diff --git a/third_party/abseil_cpp/absl/algorithm/BUILD.bazel b/third_party/abseil_cpp/absl/algorithm/BUILD.bazel new file mode 100644 index 000000000000..a3002b7dcd5f --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/BUILD.bazel @@ -0,0 +1,91 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "algorithm", + hdrs = ["algorithm.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + ], +) + +cc_test( + name = "algorithm_test", + size = "small", + srcs = ["algorithm_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":algorithm", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "algorithm_benchmark", + srcs = ["equal_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + deps = [ + ":algorithm", + "//absl/base:core_headers", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "container", + hdrs = [ + "container.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":algorithm", + "//absl/base:core_headers", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "container_test", + srcs = ["container_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":container", + "//absl/base", + "//absl/base:core_headers", + "//absl/memory", + "//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/algorithm/CMakeLists.txt b/third_party/abseil_cpp/absl/algorithm/CMakeLists.txt new file mode 100644 index 000000000000..56cd0fb85b50 --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/CMakeLists.txt @@ -0,0 +1,69 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + algorithm + HDRS + "algorithm.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_test( + NAME + algorithm_test + SRCS + "algorithm_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::algorithm + gmock_main +) + +absl_cc_library( + NAME + algorithm_container + HDRS + "container.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::meta + PUBLIC +) + +absl_cc_test( + NAME + container_test + SRCS + "container_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::algorithm_container + absl::base + absl::core_headers + absl::memory + absl::span + gmock_main +) diff --git a/third_party/abseil_cpp/absl/algorithm/algorithm.h b/third_party/abseil_cpp/absl/algorithm/algorithm.h new file mode 100644 index 000000000000..e9b473387278 --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/algorithm.h @@ -0,0 +1,159 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: algorithm.h +// ----------------------------------------------------------------------------- +// +// This header file contains Google extensions to the standard <algorithm> C++ +// header. + +#ifndef ABSL_ALGORITHM_ALGORITHM_H_ +#define ABSL_ALGORITHM_ALGORITHM_H_ + +#include <algorithm> +#include <iterator> +#include <type_traits> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace algorithm_internal { + +// Performs comparisons with operator==, similar to C++14's `std::equal_to<>`. +struct EqualTo { + template <typename T, typename U> + bool operator()(const T& a, const U& b) const { + return a == b; + } +}; + +template <typename InputIter1, typename InputIter2, typename Pred> +bool EqualImpl(InputIter1 first1, InputIter1 last1, InputIter2 first2, + InputIter2 last2, Pred pred, std::input_iterator_tag, + std::input_iterator_tag) { + while (true) { + if (first1 == last1) return first2 == last2; + if (first2 == last2) return false; + if (!pred(*first1, *first2)) return false; + ++first1; + ++first2; + } +} + +template <typename InputIter1, typename InputIter2, typename Pred> +bool EqualImpl(InputIter1 first1, InputIter1 last1, InputIter2 first2, + InputIter2 last2, Pred&& pred, std::random_access_iterator_tag, + std::random_access_iterator_tag) { + return (last1 - first1 == last2 - first2) && + std::equal(first1, last1, first2, std::forward<Pred>(pred)); +} + +// When we are using our own internal predicate that just applies operator==, we +// forward to the non-predicate form of std::equal. This enables an optimization +// in libstdc++ that can result in std::memcmp being used for integer types. +template <typename InputIter1, typename InputIter2> +bool EqualImpl(InputIter1 first1, InputIter1 last1, InputIter2 first2, + InputIter2 last2, algorithm_internal::EqualTo /* unused */, + std::random_access_iterator_tag, + std::random_access_iterator_tag) { + return (last1 - first1 == last2 - first2) && + std::equal(first1, last1, first2); +} + +template <typename It> +It RotateImpl(It first, It middle, It last, std::true_type) { + return std::rotate(first, middle, last); +} + +template <typename It> +It RotateImpl(It first, It middle, It last, std::false_type) { + std::rotate(first, middle, last); + return std::next(first, std::distance(middle, last)); +} + +} // namespace algorithm_internal + +// equal() +// +// Compares the equality of two ranges specified by pairs of iterators, using +// the given predicate, returning true iff for each corresponding iterator i1 +// and i2 in the first and second range respectively, pred(*i1, *i2) == true +// +// This comparison takes at most min(`last1` - `first1`, `last2` - `first2`) +// invocations of the predicate. Additionally, if InputIter1 and InputIter2 are +// both random-access iterators, and `last1` - `first1` != `last2` - `first2`, +// then the predicate is never invoked and the function returns false. +// +// This is a C++11-compatible implementation of C++14 `std::equal`. See +// https://en.cppreference.com/w/cpp/algorithm/equal for more information. +template <typename InputIter1, typename InputIter2, typename Pred> +bool equal(InputIter1 first1, InputIter1 last1, InputIter2 first2, + InputIter2 last2, Pred&& pred) { + return algorithm_internal::EqualImpl( + first1, last1, first2, last2, std::forward<Pred>(pred), + typename std::iterator_traits<InputIter1>::iterator_category{}, + typename std::iterator_traits<InputIter2>::iterator_category{}); +} + +// Overload of equal() that performs comparison of two ranges specified by pairs +// of iterators using operator==. +template <typename InputIter1, typename InputIter2> +bool equal(InputIter1 first1, InputIter1 last1, InputIter2 first2, + InputIter2 last2) { + return absl::equal(first1, last1, first2, last2, + algorithm_internal::EqualTo{}); +} + +// linear_search() +// +// Performs a linear search for `value` using the iterator `first` up to +// but not including `last`, returning true if [`first`, `last`) contains an +// element equal to `value`. +// +// A linear search is of O(n) complexity which is guaranteed to make at most +// n = (`last` - `first`) comparisons. A linear search over short containers +// may be faster than a binary search, even when the container is sorted. +template <typename InputIterator, typename EqualityComparable> +bool linear_search(InputIterator first, InputIterator last, + const EqualityComparable& value) { + return std::find(first, last, value) != last; +} + +// rotate() +// +// Performs a left rotation on a range of elements (`first`, `last`) such that +// `middle` is now the first element. `rotate()` returns an iterator pointing to +// the first element before rotation. This function is exactly the same as +// `std::rotate`, but fixes a bug in gcc +// <= 4.9 where `std::rotate` returns `void` instead of an iterator. +// +// The complexity of this algorithm is the same as that of `std::rotate`, but if +// `ForwardIterator` is not a random-access iterator, then `absl::rotate` +// performs an additional pass over the range to construct the return value. +template <typename ForwardIterator> +ForwardIterator rotate(ForwardIterator first, ForwardIterator middle, + ForwardIterator last) { + return algorithm_internal::RotateImpl( + first, middle, last, + std::is_same<decltype(std::rotate(first, middle, last)), + ForwardIterator>()); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_ALGORITHM_ALGORITHM_H_ diff --git a/third_party/abseil_cpp/absl/algorithm/algorithm_test.cc b/third_party/abseil_cpp/absl/algorithm/algorithm_test.cc new file mode 100644 index 000000000000..81fccb61353e --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/algorithm_test.cc @@ -0,0 +1,182 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/algorithm/algorithm.h" + +#include <algorithm> +#include <list> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { + +TEST(EqualTest, DefaultComparisonRandomAccess) { + std::vector<int> v1{1, 2, 3}; + std::vector<int> v2 = v1; + std::vector<int> v3 = {1, 2}; + std::vector<int> v4 = {1, 2, 4}; + + EXPECT_TRUE(absl::equal(v1.begin(), v1.end(), v2.begin(), v2.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v3.begin(), v3.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v4.begin(), v4.end())); +} + +TEST(EqualTest, DefaultComparison) { + std::list<int> lst1{1, 2, 3}; + std::list<int> lst2 = lst1; + std::list<int> lst3{1, 2}; + std::list<int> lst4{1, 2, 4}; + + EXPECT_TRUE(absl::equal(lst1.begin(), lst1.end(), lst2.begin(), lst2.end())); + EXPECT_FALSE(absl::equal(lst1.begin(), lst1.end(), lst3.begin(), lst3.end())); + EXPECT_FALSE(absl::equal(lst1.begin(), lst1.end(), lst4.begin(), lst4.end())); +} + +TEST(EqualTest, EmptyRange) { + std::vector<int> v1{1, 2, 3}; + std::vector<int> empty1; + std::vector<int> empty2; + + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), empty1.begin(), empty1.end())); + EXPECT_FALSE(absl::equal(empty1.begin(), empty1.end(), v1.begin(), v1.end())); + EXPECT_TRUE( + absl::equal(empty1.begin(), empty1.end(), empty2.begin(), empty2.end())); +} + +TEST(EqualTest, MixedIterTypes) { + std::vector<int> v1{1, 2, 3}; + std::list<int> lst1{v1.begin(), v1.end()}; + std::list<int> lst2{1, 2, 4}; + std::list<int> lst3{1, 2}; + + EXPECT_TRUE(absl::equal(v1.begin(), v1.end(), lst1.begin(), lst1.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), lst2.begin(), lst2.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), lst3.begin(), lst3.end())); +} + +TEST(EqualTest, MixedValueTypes) { + std::vector<int> v1{1, 2, 3}; + std::vector<char> v2{1, 2, 3}; + std::vector<char> v3{1, 2}; + std::vector<char> v4{1, 2, 4}; + + EXPECT_TRUE(absl::equal(v1.begin(), v1.end(), v2.begin(), v2.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v3.begin(), v3.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v4.begin(), v4.end())); +} + +TEST(EqualTest, WeirdIterators) { + std::vector<bool> v1{true, false}; + std::vector<bool> v2 = v1; + std::vector<bool> v3{true}; + std::vector<bool> v4{true, true, true}; + + EXPECT_TRUE(absl::equal(v1.begin(), v1.end(), v2.begin(), v2.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v3.begin(), v3.end())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v4.begin(), v4.end())); +} + +TEST(EqualTest, CustomComparison) { + int n[] = {1, 2, 3, 4}; + std::vector<int*> v1{&n[0], &n[1], &n[2]}; + std::vector<int*> v2 = v1; + std::vector<int*> v3{&n[0], &n[1], &n[3]}; + std::vector<int*> v4{&n[0], &n[1]}; + + auto eq = [](int* a, int* b) { return *a == *b; }; + + EXPECT_TRUE(absl::equal(v1.begin(), v1.end(), v2.begin(), v2.end(), eq)); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v3.begin(), v3.end(), eq)); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v4.begin(), v4.end(), eq)); +} + +TEST(EqualTest, MoveOnlyPredicate) { + std::vector<int> v1{1, 2, 3}; + std::vector<int> v2{4, 5, 6}; + + // move-only equality predicate + struct Eq { + Eq() = default; + Eq(Eq &&) = default; + Eq(const Eq &) = delete; + Eq &operator=(const Eq &) = delete; + bool operator()(const int a, const int b) const { return a == b; } + }; + + EXPECT_TRUE(absl::equal(v1.begin(), v1.end(), v1.begin(), v1.end(), Eq())); + EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), v2.begin(), v2.end(), Eq())); +} + +struct CountingTrivialPred { + int* count; + bool operator()(int, int) const { + ++*count; + return true; + } +}; + +TEST(EqualTest, RandomAccessComplexity) { + std::vector<int> v1{1, 1, 3}; + std::vector<int> v2 = v1; + std::vector<int> v3{1, 2}; + + do { + int count = 0; + absl::equal(v1.begin(), v1.end(), v2.begin(), v2.end(), + CountingTrivialPred{&count}); + EXPECT_LE(count, 3); + } while (std::next_permutation(v2.begin(), v2.end())); + + int count = 0; + absl::equal(v1.begin(), v1.end(), v3.begin(), v3.end(), + CountingTrivialPred{&count}); + EXPECT_EQ(count, 0); +} + +class LinearSearchTest : public testing::Test { + protected: + LinearSearchTest() : container_{1, 2, 3} {} + + static bool Is3(int n) { return n == 3; } + static bool Is4(int n) { return n == 4; } + + std::vector<int> container_; +}; + +TEST_F(LinearSearchTest, linear_search) { + EXPECT_TRUE(absl::linear_search(container_.begin(), container_.end(), 3)); + EXPECT_FALSE(absl::linear_search(container_.begin(), container_.end(), 4)); +} + +TEST_F(LinearSearchTest, linear_searchConst) { + const std::vector<int> *const const_container = &container_; + EXPECT_TRUE( + absl::linear_search(const_container->begin(), const_container->end(), 3)); + EXPECT_FALSE( + absl::linear_search(const_container->begin(), const_container->end(), 4)); +} + +TEST(RotateTest, Rotate) { + std::vector<int> v{0, 1, 2, 3, 4}; + EXPECT_EQ(*absl::rotate(v.begin(), v.begin() + 2, v.end()), 0); + EXPECT_THAT(v, testing::ElementsAreArray({2, 3, 4, 0, 1})); + + std::list<int> l{0, 1, 2, 3, 4}; + EXPECT_EQ(*absl::rotate(l.begin(), std::next(l.begin(), 3), l.end()), 0); + EXPECT_THAT(l, testing::ElementsAreArray({3, 4, 0, 1, 2})); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/algorithm/container.h b/third_party/abseil_cpp/absl/algorithm/container.h new file mode 100644 index 000000000000..6398438f08ce --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/container.h @@ -0,0 +1,1764 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: container.h +// ----------------------------------------------------------------------------- +// +// This header file provides Container-based versions of algorithmic functions +// within the C++ standard library. The following standard library sets of +// functions are covered within this file: +// +// * Algorithmic <iterator> functions +// * Algorithmic <numeric> functions +// * <algorithm> functions +// +// The standard library functions operate on iterator ranges; the functions +// within this API operate on containers, though many return iterator ranges. +// +// All functions within this API are named with a `c_` prefix. Calls such as +// `absl::c_xx(container, ...) are equivalent to std:: functions such as +// `std::xx(std::begin(cont), std::end(cont), ...)`. Functions that act on +// iterators but not conceptually on iterator ranges (e.g. `std::iter_swap`) +// have no equivalent here. +// +// For template parameter and variable naming, `C` indicates the container type +// to which the function is applied, `Pred` indicates the predicate object type +// to be used by the function and `T` indicates the applicable element type. + +#ifndef ABSL_ALGORITHM_CONTAINER_H_ +#define ABSL_ALGORITHM_CONTAINER_H_ + +#include <algorithm> +#include <cassert> +#include <iterator> +#include <numeric> +#include <type_traits> +#include <unordered_map> +#include <unordered_set> +#include <utility> +#include <vector> + +#include "absl/algorithm/algorithm.h" +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_algorithm_internal { + +// NOTE: it is important to defer to ADL lookup for building with C++ modules, +// especially for headers like <valarray> which are not visible from this file +// but specialize std::begin and std::end. +using std::begin; +using std::end; + +// The type of the iterator given by begin(c) (possibly std::begin(c)). +// ContainerIter<const vector<T>> gives vector<T>::const_iterator, +// while ContainerIter<vector<T>> gives vector<T>::iterator. +template <typename C> +using ContainerIter = decltype(begin(std::declval<C&>())); + +// An MSVC bug involving template parameter substitution requires us to use +// decltype() here instead of just std::pair. +template <typename C1, typename C2> +using ContainerIterPairType = + decltype(std::make_pair(ContainerIter<C1>(), ContainerIter<C2>())); + +template <typename C> +using ContainerDifferenceType = + decltype(std::distance(std::declval<ContainerIter<C>>(), + std::declval<ContainerIter<C>>())); + +template <typename C> +using ContainerPointerType = + typename std::iterator_traits<ContainerIter<C>>::pointer; + +// container_algorithm_internal::c_begin and +// container_algorithm_internal::c_end are abbreviations for proper ADL +// lookup of std::begin and std::end, i.e. +// using std::begin; +// using std::end; +// std::foo(begin(c), end(c)); +// becomes +// std::foo(container_algorithm_internal::begin(c), +// container_algorithm_internal::end(c)); +// These are meant for internal use only. + +template <typename C> +ContainerIter<C> c_begin(C& c) { return begin(c); } + +template <typename C> +ContainerIter<C> c_end(C& c) { return end(c); } + +template <typename T> +struct IsUnorderedContainer : std::false_type {}; + +template <class Key, class T, class Hash, class KeyEqual, class Allocator> +struct IsUnorderedContainer< + std::unordered_map<Key, T, Hash, KeyEqual, Allocator>> : std::true_type {}; + +template <class Key, class Hash, class KeyEqual, class Allocator> +struct IsUnorderedContainer<std::unordered_set<Key, Hash, KeyEqual, Allocator>> + : std::true_type {}; + +// container_algorithm_internal::c_size. It is meant for internal use only. + +template <class C> +auto c_size(C& c) -> decltype(c.size()) { + return c.size(); +} + +template <class T, std::size_t N> +constexpr std::size_t c_size(T (&)[N]) { + return N; +} + +} // namespace container_algorithm_internal + +// PUBLIC API + +//------------------------------------------------------------------------------ +// Abseil algorithm.h functions +//------------------------------------------------------------------------------ + +// c_linear_search() +// +// Container-based version of absl::linear_search() for performing a linear +// search within a container. +template <typename C, typename EqualityComparable> +bool c_linear_search(const C& c, EqualityComparable&& value) { + return linear_search(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<EqualityComparable>(value)); +} + +//------------------------------------------------------------------------------ +// <iterator> algorithms +//------------------------------------------------------------------------------ + +// c_distance() +// +// Container-based version of the <iterator> `std::distance()` function to +// return the number of elements within a container. +template <typename C> +container_algorithm_internal::ContainerDifferenceType<const C> c_distance( + const C& c) { + return std::distance(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Non-modifying sequence operations +//------------------------------------------------------------------------------ + +// c_all_of() +// +// Container-based version of the <algorithm> `std::all_of()` function to +// test a condition on all elements within a container. +template <typename C, typename Pred> +bool c_all_of(const C& c, Pred&& pred) { + return std::all_of(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_any_of() +// +// Container-based version of the <algorithm> `std::any_of()` function to +// test if any element in a container fulfills a condition. +template <typename C, typename Pred> +bool c_any_of(const C& c, Pred&& pred) { + return std::any_of(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_none_of() +// +// Container-based version of the <algorithm> `std::none_of()` function to +// test if no elements in a container fulfill a condition. +template <typename C, typename Pred> +bool c_none_of(const C& c, Pred&& pred) { + return std::none_of(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_for_each() +// +// Container-based version of the <algorithm> `std::for_each()` function to +// apply a function to a container's elements. +template <typename C, typename Function> +decay_t<Function> c_for_each(C&& c, Function&& f) { + return std::for_each(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Function>(f)); +} + +// c_find() +// +// Container-based version of the <algorithm> `std::find()` function to find +// the first element containing the passed value within a container value. +template <typename C, typename T> +container_algorithm_internal::ContainerIter<C> c_find(C& c, T&& value) { + return std::find(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<T>(value)); +} + +// c_find_if() +// +// Container-based version of the <algorithm> `std::find_if()` function to find +// the first element in a container matching the given condition. +template <typename C, typename Pred> +container_algorithm_internal::ContainerIter<C> c_find_if(C& c, Pred&& pred) { + return std::find_if(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_find_if_not() +// +// Container-based version of the <algorithm> `std::find_if_not()` function to +// find the first element in a container not matching the given condition. +template <typename C, typename Pred> +container_algorithm_internal::ContainerIter<C> c_find_if_not(C& c, + Pred&& pred) { + return std::find_if_not(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_find_end() +// +// Container-based version of the <algorithm> `std::find_end()` function to +// find the last subsequence within a container. +template <typename Sequence1, typename Sequence2> +container_algorithm_internal::ContainerIter<Sequence1> c_find_end( + Sequence1& sequence, Sequence2& subsequence) { + return std::find_end(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + container_algorithm_internal::c_begin(subsequence), + container_algorithm_internal::c_end(subsequence)); +} + +// Overload of c_find_end() for using a predicate evaluation other than `==` as +// the function's test condition. +template <typename Sequence1, typename Sequence2, typename BinaryPredicate> +container_algorithm_internal::ContainerIter<Sequence1> c_find_end( + Sequence1& sequence, Sequence2& subsequence, BinaryPredicate&& pred) { + return std::find_end(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + container_algorithm_internal::c_begin(subsequence), + container_algorithm_internal::c_end(subsequence), + std::forward<BinaryPredicate>(pred)); +} + +// c_find_first_of() +// +// Container-based version of the <algorithm> `std::find_first_of()` function to +// find the first element within the container that is also within the options +// container. +template <typename C1, typename C2> +container_algorithm_internal::ContainerIter<C1> c_find_first_of(C1& container, + C2& options) { + return std::find_first_of(container_algorithm_internal::c_begin(container), + container_algorithm_internal::c_end(container), + container_algorithm_internal::c_begin(options), + container_algorithm_internal::c_end(options)); +} + +// Overload of c_find_first_of() for using a predicate evaluation other than +// `==` as the function's test condition. +template <typename C1, typename C2, typename BinaryPredicate> +container_algorithm_internal::ContainerIter<C1> c_find_first_of( + C1& container, C2& options, BinaryPredicate&& pred) { + return std::find_first_of(container_algorithm_internal::c_begin(container), + container_algorithm_internal::c_end(container), + container_algorithm_internal::c_begin(options), + container_algorithm_internal::c_end(options), + std::forward<BinaryPredicate>(pred)); +} + +// c_adjacent_find() +// +// Container-based version of the <algorithm> `std::adjacent_find()` function to +// find equal adjacent elements within a container. +template <typename Sequence> +container_algorithm_internal::ContainerIter<Sequence> c_adjacent_find( + Sequence& sequence) { + return std::adjacent_find(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_adjacent_find() for using a predicate evaluation other than +// `==` as the function's test condition. +template <typename Sequence, typename BinaryPredicate> +container_algorithm_internal::ContainerIter<Sequence> c_adjacent_find( + Sequence& sequence, BinaryPredicate&& pred) { + return std::adjacent_find(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<BinaryPredicate>(pred)); +} + +// c_count() +// +// Container-based version of the <algorithm> `std::count()` function to count +// values that match within a container. +template <typename C, typename T> +container_algorithm_internal::ContainerDifferenceType<const C> c_count( + const C& c, T&& value) { + return std::count(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<T>(value)); +} + +// c_count_if() +// +// Container-based version of the <algorithm> `std::count_if()` function to +// count values matching a condition within a container. +template <typename C, typename Pred> +container_algorithm_internal::ContainerDifferenceType<const C> c_count_if( + const C& c, Pred&& pred) { + return std::count_if(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_mismatch() +// +// Container-based version of the <algorithm> `std::mismatch()` function to +// return the first element where two ordered containers differ. Applies `==` to +// the first N elements of `c1` and `c2`, where N = min(size(c1), size(c2)). +template <typename C1, typename C2> +container_algorithm_internal::ContainerIterPairType<C1, C2> +c_mismatch(C1& c1, C2& c2) { + auto first1 = container_algorithm_internal::c_begin(c1); + auto last1 = container_algorithm_internal::c_end(c1); + auto first2 = container_algorithm_internal::c_begin(c2); + auto last2 = container_algorithm_internal::c_end(c2); + + for (; first1 != last1 && first2 != last2; ++first1, (void)++first2) { + // Negates equality because Cpp17EqualityComparable doesn't require clients + // to overload both `operator==` and `operator!=`. + if (!(*first1 == *first2)) { + break; + } + } + + return std::make_pair(first1, first2); +} + +// Overload of c_mismatch() for using a predicate evaluation other than `==` as +// the function's test condition. Applies `pred`to the first N elements of `c1` +// and `c2`, where N = min(size(c1), size(c2)). +template <typename C1, typename C2, typename BinaryPredicate> +container_algorithm_internal::ContainerIterPairType<C1, C2> +c_mismatch(C1& c1, C2& c2, BinaryPredicate pred) { + auto first1 = container_algorithm_internal::c_begin(c1); + auto last1 = container_algorithm_internal::c_end(c1); + auto first2 = container_algorithm_internal::c_begin(c2); + auto last2 = container_algorithm_internal::c_end(c2); + + for (; first1 != last1 && first2 != last2; ++first1, (void)++first2) { + if (!pred(*first1, *first2)) { + break; + } + } + + return std::make_pair(first1, first2); +} + +// c_equal() +// +// Container-based version of the <algorithm> `std::equal()` function to +// test whether two containers are equal. +// +// NOTE: the semantics of c_equal() are slightly different than those of +// equal(): while the latter iterates over the second container only up to the +// size of the first container, c_equal() also checks whether the container +// sizes are equal. This better matches expectations about c_equal() based on +// its signature. +// +// Example: +// vector v1 = <1, 2, 3>; +// vector v2 = <1, 2, 3, 4>; +// equal(std::begin(v1), std::end(v1), std::begin(v2)) returns true +// c_equal(v1, v2) returns false + +template <typename C1, typename C2> +bool c_equal(const C1& c1, const C2& c2) { + return ((container_algorithm_internal::c_size(c1) == + container_algorithm_internal::c_size(c2)) && + std::equal(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2))); +} + +// Overload of c_equal() for using a predicate evaluation other than `==` as +// the function's test condition. +template <typename C1, typename C2, typename BinaryPredicate> +bool c_equal(const C1& c1, const C2& c2, BinaryPredicate&& pred) { + return ((container_algorithm_internal::c_size(c1) == + container_algorithm_internal::c_size(c2)) && + std::equal(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + std::forward<BinaryPredicate>(pred))); +} + +// c_is_permutation() +// +// Container-based version of the <algorithm> `std::is_permutation()` function +// to test whether a container is a permutation of another. +template <typename C1, typename C2> +bool c_is_permutation(const C1& c1, const C2& c2) { + using std::begin; + using std::end; + return c1.size() == c2.size() && + std::is_permutation(begin(c1), end(c1), begin(c2)); +} + +// Overload of c_is_permutation() for using a predicate evaluation other than +// `==` as the function's test condition. +template <typename C1, typename C2, typename BinaryPredicate> +bool c_is_permutation(const C1& c1, const C2& c2, BinaryPredicate&& pred) { + using std::begin; + using std::end; + return c1.size() == c2.size() && + std::is_permutation(begin(c1), end(c1), begin(c2), + std::forward<BinaryPredicate>(pred)); +} + +// c_search() +// +// Container-based version of the <algorithm> `std::search()` function to search +// a container for a subsequence. +template <typename Sequence1, typename Sequence2> +container_algorithm_internal::ContainerIter<Sequence1> c_search( + Sequence1& sequence, Sequence2& subsequence) { + return std::search(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + container_algorithm_internal::c_begin(subsequence), + container_algorithm_internal::c_end(subsequence)); +} + +// Overload of c_search() for using a predicate evaluation other than +// `==` as the function's test condition. +template <typename Sequence1, typename Sequence2, typename BinaryPredicate> +container_algorithm_internal::ContainerIter<Sequence1> c_search( + Sequence1& sequence, Sequence2& subsequence, BinaryPredicate&& pred) { + return std::search(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + container_algorithm_internal::c_begin(subsequence), + container_algorithm_internal::c_end(subsequence), + std::forward<BinaryPredicate>(pred)); +} + +// c_search_n() +// +// Container-based version of the <algorithm> `std::search_n()` function to +// search a container for the first sequence of N elements. +template <typename Sequence, typename Size, typename T> +container_algorithm_internal::ContainerIter<Sequence> c_search_n( + Sequence& sequence, Size count, T&& value) { + return std::search_n(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), count, + std::forward<T>(value)); +} + +// Overload of c_search_n() for using a predicate evaluation other than +// `==` as the function's test condition. +template <typename Sequence, typename Size, typename T, + typename BinaryPredicate> +container_algorithm_internal::ContainerIter<Sequence> c_search_n( + Sequence& sequence, Size count, T&& value, BinaryPredicate&& pred) { + return std::search_n(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), count, + std::forward<T>(value), + std::forward<BinaryPredicate>(pred)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Modifying sequence operations +//------------------------------------------------------------------------------ + +// c_copy() +// +// Container-based version of the <algorithm> `std::copy()` function to copy a +// container's elements into an iterator. +template <typename InputSequence, typename OutputIterator> +OutputIterator c_copy(const InputSequence& input, OutputIterator output) { + return std::copy(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), output); +} + +// c_copy_n() +// +// Container-based version of the <algorithm> `std::copy_n()` function to copy a +// container's first N elements into an iterator. +template <typename C, typename Size, typename OutputIterator> +OutputIterator c_copy_n(const C& input, Size n, OutputIterator output) { + return std::copy_n(container_algorithm_internal::c_begin(input), n, output); +} + +// c_copy_if() +// +// Container-based version of the <algorithm> `std::copy_if()` function to copy +// a container's elements satisfying some condition into an iterator. +template <typename InputSequence, typename OutputIterator, typename Pred> +OutputIterator c_copy_if(const InputSequence& input, OutputIterator output, + Pred&& pred) { + return std::copy_if(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), output, + std::forward<Pred>(pred)); +} + +// c_copy_backward() +// +// Container-based version of the <algorithm> `std::copy_backward()` function to +// copy a container's elements in reverse order into an iterator. +template <typename C, typename BidirectionalIterator> +BidirectionalIterator c_copy_backward(const C& src, + BidirectionalIterator dest) { + return std::copy_backward(container_algorithm_internal::c_begin(src), + container_algorithm_internal::c_end(src), dest); +} + +// c_move() +// +// Container-based version of the <algorithm> `std::move()` function to move +// a container's elements into an iterator. +template <typename C, typename OutputIterator> +OutputIterator c_move(C&& src, OutputIterator dest) { + return std::move(container_algorithm_internal::c_begin(src), + container_algorithm_internal::c_end(src), dest); +} + +// c_move_backward() +// +// Container-based version of the <algorithm> `std::move_backward()` function to +// move a container's elements into an iterator in reverse order. +template <typename C, typename BidirectionalIterator> +BidirectionalIterator c_move_backward(C&& src, BidirectionalIterator dest) { + return std::move_backward(container_algorithm_internal::c_begin(src), + container_algorithm_internal::c_end(src), dest); +} + +// c_swap_ranges() +// +// Container-based version of the <algorithm> `std::swap_ranges()` function to +// swap a container's elements with another container's elements. Swaps the +// first N elements of `c1` and `c2`, where N = min(size(c1), size(c2)). +template <typename C1, typename C2> +container_algorithm_internal::ContainerIter<C2> c_swap_ranges(C1& c1, C2& c2) { + auto first1 = container_algorithm_internal::c_begin(c1); + auto last1 = container_algorithm_internal::c_end(c1); + auto first2 = container_algorithm_internal::c_begin(c2); + auto last2 = container_algorithm_internal::c_end(c2); + + using std::swap; + for (; first1 != last1 && first2 != last2; ++first1, (void)++first2) { + swap(*first1, *first2); + } + return first2; +} + +// c_transform() +// +// Container-based version of the <algorithm> `std::transform()` function to +// transform a container's elements using the unary operation, storing the +// result in an iterator pointing to the last transformed element in the output +// range. +template <typename InputSequence, typename OutputIterator, typename UnaryOp> +OutputIterator c_transform(const InputSequence& input, OutputIterator output, + UnaryOp&& unary_op) { + return std::transform(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), output, + std::forward<UnaryOp>(unary_op)); +} + +// Overload of c_transform() for performing a transformation using a binary +// predicate. Applies `binary_op` to the first N elements of `c1` and `c2`, +// where N = min(size(c1), size(c2)). +template <typename InputSequence1, typename InputSequence2, + typename OutputIterator, typename BinaryOp> +OutputIterator c_transform(const InputSequence1& input1, + const InputSequence2& input2, OutputIterator output, + BinaryOp&& binary_op) { + auto first1 = container_algorithm_internal::c_begin(input1); + auto last1 = container_algorithm_internal::c_end(input1); + auto first2 = container_algorithm_internal::c_begin(input2); + auto last2 = container_algorithm_internal::c_end(input2); + for (; first1 != last1 && first2 != last2; + ++first1, (void)++first2, ++output) { + *output = binary_op(*first1, *first2); + } + + return output; +} + +// c_replace() +// +// Container-based version of the <algorithm> `std::replace()` function to +// replace a container's elements of some value with a new value. The container +// is modified in place. +template <typename Sequence, typename T> +void c_replace(Sequence& sequence, const T& old_value, const T& new_value) { + std::replace(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), old_value, + new_value); +} + +// c_replace_if() +// +// Container-based version of the <algorithm> `std::replace_if()` function to +// replace a container's elements of some value with a new value based on some +// condition. The container is modified in place. +template <typename C, typename Pred, typename T> +void c_replace_if(C& c, Pred&& pred, T&& new_value) { + std::replace_if(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred), std::forward<T>(new_value)); +} + +// c_replace_copy() +// +// Container-based version of the <algorithm> `std::replace_copy()` function to +// replace a container's elements of some value with a new value and return the +// results within an iterator. +template <typename C, typename OutputIterator, typename T> +OutputIterator c_replace_copy(const C& c, OutputIterator result, T&& old_value, + T&& new_value) { + return std::replace_copy(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result, + std::forward<T>(old_value), + std::forward<T>(new_value)); +} + +// c_replace_copy_if() +// +// Container-based version of the <algorithm> `std::replace_copy_if()` function +// to replace a container's elements of some value with a new value based on +// some condition, and return the results within an iterator. +template <typename C, typename OutputIterator, typename Pred, typename T> +OutputIterator c_replace_copy_if(const C& c, OutputIterator result, Pred&& pred, + T&& new_value) { + return std::replace_copy_if(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result, + std::forward<Pred>(pred), + std::forward<T>(new_value)); +} + +// c_fill() +// +// Container-based version of the <algorithm> `std::fill()` function to fill a +// container with some value. +template <typename C, typename T> +void c_fill(C& c, T&& value) { + std::fill(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), std::forward<T>(value)); +} + +// c_fill_n() +// +// Container-based version of the <algorithm> `std::fill_n()` function to fill +// the first N elements in a container with some value. +template <typename C, typename Size, typename T> +void c_fill_n(C& c, Size n, T&& value) { + std::fill_n(container_algorithm_internal::c_begin(c), n, + std::forward<T>(value)); +} + +// c_generate() +// +// Container-based version of the <algorithm> `std::generate()` function to +// assign a container's elements to the values provided by the given generator. +template <typename C, typename Generator> +void c_generate(C& c, Generator&& gen) { + std::generate(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Generator>(gen)); +} + +// c_generate_n() +// +// Container-based version of the <algorithm> `std::generate_n()` function to +// assign a container's first N elements to the values provided by the given +// generator. +template <typename C, typename Size, typename Generator> +container_algorithm_internal::ContainerIter<C> c_generate_n(C& c, Size n, + Generator&& gen) { + return std::generate_n(container_algorithm_internal::c_begin(c), n, + std::forward<Generator>(gen)); +} + +// Note: `c_xx()` <algorithm> container versions for `remove()`, `remove_if()`, +// and `unique()` are omitted, because it's not clear whether or not such +// functions should call erase on their supplied sequences afterwards. Either +// behavior would be surprising for a different set of users. + +// c_remove_copy() +// +// Container-based version of the <algorithm> `std::remove_copy()` function to +// copy a container's elements while removing any elements matching the given +// `value`. +template <typename C, typename OutputIterator, typename T> +OutputIterator c_remove_copy(const C& c, OutputIterator result, T&& value) { + return std::remove_copy(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result, + std::forward<T>(value)); +} + +// c_remove_copy_if() +// +// Container-based version of the <algorithm> `std::remove_copy_if()` function +// to copy a container's elements while removing any elements matching the given +// condition. +template <typename C, typename OutputIterator, typename Pred> +OutputIterator c_remove_copy_if(const C& c, OutputIterator result, + Pred&& pred) { + return std::remove_copy_if(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result, + std::forward<Pred>(pred)); +} + +// c_unique_copy() +// +// Container-based version of the <algorithm> `std::unique_copy()` function to +// copy a container's elements while removing any elements containing duplicate +// values. +template <typename C, typename OutputIterator> +OutputIterator c_unique_copy(const C& c, OutputIterator result) { + return std::unique_copy(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result); +} + +// Overload of c_unique_copy() for using a predicate evaluation other than +// `==` for comparing uniqueness of the element values. +template <typename C, typename OutputIterator, typename BinaryPredicate> +OutputIterator c_unique_copy(const C& c, OutputIterator result, + BinaryPredicate&& pred) { + return std::unique_copy(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), result, + std::forward<BinaryPredicate>(pred)); +} + +// c_reverse() +// +// Container-based version of the <algorithm> `std::reverse()` function to +// reverse a container's elements. +template <typename Sequence> +void c_reverse(Sequence& sequence) { + std::reverse(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// c_reverse_copy() +// +// Container-based version of the <algorithm> `std::reverse()` function to +// reverse a container's elements and write them to an iterator range. +template <typename C, typename OutputIterator> +OutputIterator c_reverse_copy(const C& sequence, OutputIterator result) { + return std::reverse_copy(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + result); +} + +// c_rotate() +// +// Container-based version of the <algorithm> `std::rotate()` function to +// shift a container's elements leftward such that the `middle` element becomes +// the first element in the container. +template <typename C, + typename Iterator = container_algorithm_internal::ContainerIter<C>> +Iterator c_rotate(C& sequence, Iterator middle) { + return absl::rotate(container_algorithm_internal::c_begin(sequence), middle, + container_algorithm_internal::c_end(sequence)); +} + +// c_rotate_copy() +// +// Container-based version of the <algorithm> `std::rotate_copy()` function to +// shift a container's elements leftward such that the `middle` element becomes +// the first element in a new iterator range. +template <typename C, typename OutputIterator> +OutputIterator c_rotate_copy( + const C& sequence, + container_algorithm_internal::ContainerIter<const C> middle, + OutputIterator result) { + return std::rotate_copy(container_algorithm_internal::c_begin(sequence), + middle, container_algorithm_internal::c_end(sequence), + result); +} + +// c_shuffle() +// +// Container-based version of the <algorithm> `std::shuffle()` function to +// randomly shuffle elements within the container using a `gen()` uniform random +// number generator. +template <typename RandomAccessContainer, typename UniformRandomBitGenerator> +void c_shuffle(RandomAccessContainer& c, UniformRandomBitGenerator&& gen) { + std::shuffle(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<UniformRandomBitGenerator>(gen)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Partition functions +//------------------------------------------------------------------------------ + +// c_is_partitioned() +// +// Container-based version of the <algorithm> `std::is_partitioned()` function +// to test whether all elements in the container for which `pred` returns `true` +// precede those for which `pred` is `false`. +template <typename C, typename Pred> +bool c_is_partitioned(const C& c, Pred&& pred) { + return std::is_partitioned(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_partition() +// +// Container-based version of the <algorithm> `std::partition()` function +// to rearrange all elements in a container in such a way that all elements for +// which `pred` returns `true` precede all those for which it returns `false`, +// returning an iterator to the first element of the second group. +template <typename C, typename Pred> +container_algorithm_internal::ContainerIter<C> c_partition(C& c, Pred&& pred) { + return std::partition(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_stable_partition() +// +// Container-based version of the <algorithm> `std::stable_partition()` function +// to rearrange all elements in a container in such a way that all elements for +// which `pred` returns `true` precede all those for which it returns `false`, +// preserving the relative ordering between the two groups. The function returns +// an iterator to the first element of the second group. +template <typename C, typename Pred> +container_algorithm_internal::ContainerIter<C> c_stable_partition(C& c, + Pred&& pred) { + return std::stable_partition(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +// c_partition_copy() +// +// Container-based version of the <algorithm> `std::partition_copy()` function +// to partition a container's elements and return them into two iterators: one +// for which `pred` returns `true`, and one for which `pred` returns `false.` + +template <typename C, typename OutputIterator1, typename OutputIterator2, + typename Pred> +std::pair<OutputIterator1, OutputIterator2> c_partition_copy( + const C& c, OutputIterator1 out_true, OutputIterator2 out_false, + Pred&& pred) { + return std::partition_copy(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), out_true, + out_false, std::forward<Pred>(pred)); +} + +// c_partition_point() +// +// Container-based version of the <algorithm> `std::partition_point()` function +// to return the first element of an already partitioned container for which +// the given `pred` is not `true`. +template <typename C, typename Pred> +container_algorithm_internal::ContainerIter<C> c_partition_point(C& c, + Pred&& pred) { + return std::partition_point(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Pred>(pred)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Sorting functions +//------------------------------------------------------------------------------ + +// c_sort() +// +// Container-based version of the <algorithm> `std::sort()` function +// to sort elements in ascending order of their values. +template <typename C> +void c_sort(C& c) { + std::sort(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// Overload of c_sort() for performing a `comp` comparison other than the +// default `operator<`. +template <typename C, typename Compare> +void c_sort(C& c, Compare&& comp) { + std::sort(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +// c_stable_sort() +// +// Container-based version of the <algorithm> `std::stable_sort()` function +// to sort elements in ascending order of their values, preserving the order +// of equivalents. +template <typename C> +void c_stable_sort(C& c) { + std::stable_sort(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// Overload of c_stable_sort() for performing a `comp` comparison other than the +// default `operator<`. +template <typename C, typename Compare> +void c_stable_sort(C& c, Compare&& comp) { + std::stable_sort(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +// c_is_sorted() +// +// Container-based version of the <algorithm> `std::is_sorted()` function +// to evaluate whether the given container is sorted in ascending order. +template <typename C> +bool c_is_sorted(const C& c) { + return std::is_sorted(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// c_is_sorted() overload for performing a `comp` comparison other than the +// default `operator<`. +template <typename C, typename Compare> +bool c_is_sorted(const C& c, Compare&& comp) { + return std::is_sorted(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +// c_partial_sort() +// +// Container-based version of the <algorithm> `std::partial_sort()` function +// to rearrange elements within a container such that elements before `middle` +// are sorted in ascending order. +template <typename RandomAccessContainer> +void c_partial_sort( + RandomAccessContainer& sequence, + container_algorithm_internal::ContainerIter<RandomAccessContainer> middle) { + std::partial_sort(container_algorithm_internal::c_begin(sequence), middle, + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_partial_sort() for performing a `comp` comparison other than +// the default `operator<`. +template <typename RandomAccessContainer, typename Compare> +void c_partial_sort( + RandomAccessContainer& sequence, + container_algorithm_internal::ContainerIter<RandomAccessContainer> middle, + Compare&& comp) { + std::partial_sort(container_algorithm_internal::c_begin(sequence), middle, + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_partial_sort_copy() +// +// Container-based version of the <algorithm> `std::partial_sort_copy()` +// function to sort the elements in the given range `result` within the larger +// `sequence` in ascending order (and using `result` as the output parameter). +// At most min(result.last - result.first, sequence.last - sequence.first) +// elements from the sequence will be stored in the result. +template <typename C, typename RandomAccessContainer> +container_algorithm_internal::ContainerIter<RandomAccessContainer> +c_partial_sort_copy(const C& sequence, RandomAccessContainer& result) { + return std::partial_sort_copy(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + container_algorithm_internal::c_begin(result), + container_algorithm_internal::c_end(result)); +} + +// Overload of c_partial_sort_copy() for performing a `comp` comparison other +// than the default `operator<`. +template <typename C, typename RandomAccessContainer, typename Compare> +container_algorithm_internal::ContainerIter<RandomAccessContainer> +c_partial_sort_copy(const C& sequence, RandomAccessContainer& result, + Compare&& comp) { + return std::partial_sort_copy(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + container_algorithm_internal::c_begin(result), + container_algorithm_internal::c_end(result), + std::forward<Compare>(comp)); +} + +// c_is_sorted_until() +// +// Container-based version of the <algorithm> `std::is_sorted_until()` function +// to return the first element within a container that is not sorted in +// ascending order as an iterator. +template <typename C> +container_algorithm_internal::ContainerIter<C> c_is_sorted_until(C& c) { + return std::is_sorted_until(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// Overload of c_is_sorted_until() for performing a `comp` comparison other than +// the default `operator<`. +template <typename C, typename Compare> +container_algorithm_internal::ContainerIter<C> c_is_sorted_until( + C& c, Compare&& comp) { + return std::is_sorted_until(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +// c_nth_element() +// +// Container-based version of the <algorithm> `std::nth_element()` function +// to rearrange the elements within a container such that the `nth` element +// would be in that position in an ordered sequence; other elements may be in +// any order, except that all preceding `nth` will be less than that element, +// and all following `nth` will be greater than that element. +template <typename RandomAccessContainer> +void c_nth_element( + RandomAccessContainer& sequence, + container_algorithm_internal::ContainerIter<RandomAccessContainer> nth) { + std::nth_element(container_algorithm_internal::c_begin(sequence), nth, + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_nth_element() for performing a `comp` comparison other than +// the default `operator<`. +template <typename RandomAccessContainer, typename Compare> +void c_nth_element( + RandomAccessContainer& sequence, + container_algorithm_internal::ContainerIter<RandomAccessContainer> nth, + Compare&& comp) { + std::nth_element(container_algorithm_internal::c_begin(sequence), nth, + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Binary Search +//------------------------------------------------------------------------------ + +// c_lower_bound() +// +// Container-based version of the <algorithm> `std::lower_bound()` function +// to return an iterator pointing to the first element in a sorted container +// which does not compare less than `value`. +template <typename Sequence, typename T> +container_algorithm_internal::ContainerIter<Sequence> c_lower_bound( + Sequence& sequence, T&& value) { + return std::lower_bound(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value)); +} + +// Overload of c_lower_bound() for performing a `comp` comparison other than +// the default `operator<`. +template <typename Sequence, typename T, typename Compare> +container_algorithm_internal::ContainerIter<Sequence> c_lower_bound( + Sequence& sequence, T&& value, Compare&& comp) { + return std::lower_bound(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value), std::forward<Compare>(comp)); +} + +// c_upper_bound() +// +// Container-based version of the <algorithm> `std::upper_bound()` function +// to return an iterator pointing to the first element in a sorted container +// which is greater than `value`. +template <typename Sequence, typename T> +container_algorithm_internal::ContainerIter<Sequence> c_upper_bound( + Sequence& sequence, T&& value) { + return std::upper_bound(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value)); +} + +// Overload of c_upper_bound() for performing a `comp` comparison other than +// the default `operator<`. +template <typename Sequence, typename T, typename Compare> +container_algorithm_internal::ContainerIter<Sequence> c_upper_bound( + Sequence& sequence, T&& value, Compare&& comp) { + return std::upper_bound(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value), std::forward<Compare>(comp)); +} + +// c_equal_range() +// +// Container-based version of the <algorithm> `std::equal_range()` function +// to return an iterator pair pointing to the first and last elements in a +// sorted container which compare equal to `value`. +template <typename Sequence, typename T> +container_algorithm_internal::ContainerIterPairType<Sequence, Sequence> +c_equal_range(Sequence& sequence, T&& value) { + return std::equal_range(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value)); +} + +// Overload of c_equal_range() for performing a `comp` comparison other than +// the default `operator<`. +template <typename Sequence, typename T, typename Compare> +container_algorithm_internal::ContainerIterPairType<Sequence, Sequence> +c_equal_range(Sequence& sequence, T&& value, Compare&& comp) { + return std::equal_range(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value), std::forward<Compare>(comp)); +} + +// c_binary_search() +// +// Container-based version of the <algorithm> `std::binary_search()` function +// to test if any element in the sorted container contains a value equivalent to +// 'value'. +template <typename Sequence, typename T> +bool c_binary_search(Sequence&& sequence, T&& value) { + return std::binary_search(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value)); +} + +// Overload of c_binary_search() for performing a `comp` comparison other than +// the default `operator<`. +template <typename Sequence, typename T, typename Compare> +bool c_binary_search(Sequence&& sequence, T&& value, Compare&& comp) { + return std::binary_search(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value), + std::forward<Compare>(comp)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Merge functions +//------------------------------------------------------------------------------ + +// c_merge() +// +// Container-based version of the <algorithm> `std::merge()` function +// to merge two sorted containers into a single sorted iterator. +template <typename C1, typename C2, typename OutputIterator> +OutputIterator c_merge(const C1& c1, const C2& c2, OutputIterator result) { + return std::merge(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), result); +} + +// Overload of c_merge() for performing a `comp` comparison other than +// the default `operator<`. +template <typename C1, typename C2, typename OutputIterator, typename Compare> +OutputIterator c_merge(const C1& c1, const C2& c2, OutputIterator result, + Compare&& comp) { + return std::merge(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), result, + std::forward<Compare>(comp)); +} + +// c_inplace_merge() +// +// Container-based version of the <algorithm> `std::inplace_merge()` function +// to merge a supplied iterator `middle` into a container. +template <typename C> +void c_inplace_merge(C& c, + container_algorithm_internal::ContainerIter<C> middle) { + std::inplace_merge(container_algorithm_internal::c_begin(c), middle, + container_algorithm_internal::c_end(c)); +} + +// Overload of c_inplace_merge() for performing a merge using a `comp` other +// than `operator<`. +template <typename C, typename Compare> +void c_inplace_merge(C& c, + container_algorithm_internal::ContainerIter<C> middle, + Compare&& comp) { + std::inplace_merge(container_algorithm_internal::c_begin(c), middle, + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +// c_includes() +// +// Container-based version of the <algorithm> `std::includes()` function +// to test whether a sorted container `c1` entirely contains another sorted +// container `c2`. +template <typename C1, typename C2> +bool c_includes(const C1& c1, const C2& c2) { + return std::includes(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2)); +} + +// Overload of c_includes() for performing a merge using a `comp` other than +// `operator<`. +template <typename C1, typename C2, typename Compare> +bool c_includes(const C1& c1, const C2& c2, Compare&& comp) { + return std::includes(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), + std::forward<Compare>(comp)); +} + +// c_set_union() +// +// Container-based version of the <algorithm> `std::set_union()` function +// to return an iterator containing the union of two containers; duplicate +// values are not copied into the output. +template <typename C1, typename C2, typename OutputIterator, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_union(const C1& c1, const C2& c2, OutputIterator output) { + return std::set_union(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output); +} + +// Overload of c_set_union() for performing a merge using a `comp` other than +// `operator<`. +template <typename C1, typename C2, typename OutputIterator, typename Compare, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_union(const C1& c1, const C2& c2, OutputIterator output, + Compare&& comp) { + return std::set_union(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output, + std::forward<Compare>(comp)); +} + +// c_set_intersection() +// +// Container-based version of the <algorithm> `std::set_intersection()` function +// to return an iterator containing the intersection of two containers. +template <typename C1, typename C2, typename OutputIterator, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_intersection(const C1& c1, const C2& c2, + OutputIterator output) { + return std::set_intersection(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output); +} + +// Overload of c_set_intersection() for performing a merge using a `comp` other +// than `operator<`. +template <typename C1, typename C2, typename OutputIterator, typename Compare, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_intersection(const C1& c1, const C2& c2, + OutputIterator output, Compare&& comp) { + return std::set_intersection(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output, + std::forward<Compare>(comp)); +} + +// c_set_difference() +// +// Container-based version of the <algorithm> `std::set_difference()` function +// to return an iterator containing elements present in the first container but +// not in the second. +template <typename C1, typename C2, typename OutputIterator, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_difference(const C1& c1, const C2& c2, + OutputIterator output) { + return std::set_difference(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output); +} + +// Overload of c_set_difference() for performing a merge using a `comp` other +// than `operator<`. +template <typename C1, typename C2, typename OutputIterator, typename Compare, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_difference(const C1& c1, const C2& c2, + OutputIterator output, Compare&& comp) { + return std::set_difference(container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output, + std::forward<Compare>(comp)); +} + +// c_set_symmetric_difference() +// +// Container-based version of the <algorithm> `std::set_symmetric_difference()` +// function to return an iterator containing elements present in either one +// container or the other, but not both. +template <typename C1, typename C2, typename OutputIterator, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_symmetric_difference(const C1& c1, const C2& c2, + OutputIterator output) { + return std::set_symmetric_difference( + container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output); +} + +// Overload of c_set_symmetric_difference() for performing a merge using a +// `comp` other than `operator<`. +template <typename C1, typename C2, typename OutputIterator, typename Compare, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C1>::value, + void>::type, + typename = typename std::enable_if< + !container_algorithm_internal::IsUnorderedContainer<C2>::value, + void>::type> +OutputIterator c_set_symmetric_difference(const C1& c1, const C2& c2, + OutputIterator output, + Compare&& comp) { + return std::set_symmetric_difference( + container_algorithm_internal::c_begin(c1), + container_algorithm_internal::c_end(c1), + container_algorithm_internal::c_begin(c2), + container_algorithm_internal::c_end(c2), output, + std::forward<Compare>(comp)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Heap functions +//------------------------------------------------------------------------------ + +// c_push_heap() +// +// Container-based version of the <algorithm> `std::push_heap()` function +// to push a value onto a container heap. +template <typename RandomAccessContainer> +void c_push_heap(RandomAccessContainer& sequence) { + std::push_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_push_heap() for performing a push operation on a heap using a +// `comp` other than `operator<`. +template <typename RandomAccessContainer, typename Compare> +void c_push_heap(RandomAccessContainer& sequence, Compare&& comp) { + std::push_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_pop_heap() +// +// Container-based version of the <algorithm> `std::pop_heap()` function +// to pop a value from a heap container. +template <typename RandomAccessContainer> +void c_pop_heap(RandomAccessContainer& sequence) { + std::pop_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_pop_heap() for performing a pop operation on a heap using a +// `comp` other than `operator<`. +template <typename RandomAccessContainer, typename Compare> +void c_pop_heap(RandomAccessContainer& sequence, Compare&& comp) { + std::pop_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_make_heap() +// +// Container-based version of the <algorithm> `std::make_heap()` function +// to make a container a heap. +template <typename RandomAccessContainer> +void c_make_heap(RandomAccessContainer& sequence) { + std::make_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_make_heap() for performing heap comparisons using a +// `comp` other than `operator<` +template <typename RandomAccessContainer, typename Compare> +void c_make_heap(RandomAccessContainer& sequence, Compare&& comp) { + std::make_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_sort_heap() +// +// Container-based version of the <algorithm> `std::sort_heap()` function +// to sort a heap into ascending order (after which it is no longer a heap). +template <typename RandomAccessContainer> +void c_sort_heap(RandomAccessContainer& sequence) { + std::sort_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_sort_heap() for performing heap comparisons using a +// `comp` other than `operator<` +template <typename RandomAccessContainer, typename Compare> +void c_sort_heap(RandomAccessContainer& sequence, Compare&& comp) { + std::sort_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_is_heap() +// +// Container-based version of the <algorithm> `std::is_heap()` function +// to check whether the given container is a heap. +template <typename RandomAccessContainer> +bool c_is_heap(const RandomAccessContainer& sequence) { + return std::is_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_is_heap() for performing heap comparisons using a +// `comp` other than `operator<` +template <typename RandomAccessContainer, typename Compare> +bool c_is_heap(const RandomAccessContainer& sequence, Compare&& comp) { + return std::is_heap(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_is_heap_until() +// +// Container-based version of the <algorithm> `std::is_heap_until()` function +// to find the first element in a given container which is not in heap order. +template <typename RandomAccessContainer> +container_algorithm_internal::ContainerIter<RandomAccessContainer> +c_is_heap_until(RandomAccessContainer& sequence) { + return std::is_heap_until(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_is_heap_until() for performing heap comparisons using a +// `comp` other than `operator<` +template <typename RandomAccessContainer, typename Compare> +container_algorithm_internal::ContainerIter<RandomAccessContainer> +c_is_heap_until(RandomAccessContainer& sequence, Compare&& comp) { + return std::is_heap_until(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Min/max +//------------------------------------------------------------------------------ + +// c_min_element() +// +// Container-based version of the <algorithm> `std::min_element()` function +// to return an iterator pointing to the element with the smallest value, using +// `operator<` to make the comparisons. +template <typename Sequence> +container_algorithm_internal::ContainerIter<Sequence> c_min_element( + Sequence& sequence) { + return std::min_element(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_min_element() for performing a `comp` comparison other than +// `operator<`. +template <typename Sequence, typename Compare> +container_algorithm_internal::ContainerIter<Sequence> c_min_element( + Sequence& sequence, Compare&& comp) { + return std::min_element(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_max_element() +// +// Container-based version of the <algorithm> `std::max_element()` function +// to return an iterator pointing to the element with the largest value, using +// `operator<` to make the comparisons. +template <typename Sequence> +container_algorithm_internal::ContainerIter<Sequence> c_max_element( + Sequence& sequence) { + return std::max_element(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence)); +} + +// Overload of c_max_element() for performing a `comp` comparison other than +// `operator<`. +template <typename Sequence, typename Compare> +container_algorithm_internal::ContainerIter<Sequence> c_max_element( + Sequence& sequence, Compare&& comp) { + return std::max_element(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<Compare>(comp)); +} + +// c_minmax_element() +// +// Container-based version of the <algorithm> `std::minmax_element()` function +// to return a pair of iterators pointing to the elements containing the +// smallest and largest values, respectively, using `operator<` to make the +// comparisons. +template <typename C> +container_algorithm_internal::ContainerIterPairType<C, C> +c_minmax_element(C& c) { + return std::minmax_element(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// Overload of c_minmax_element() for performing `comp` comparisons other than +// `operator<`. +template <typename C, typename Compare> +container_algorithm_internal::ContainerIterPairType<C, C> +c_minmax_element(C& c, Compare&& comp) { + return std::minmax_element(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +//------------------------------------------------------------------------------ +// <algorithm> Lexicographical Comparisons +//------------------------------------------------------------------------------ + +// c_lexicographical_compare() +// +// Container-based version of the <algorithm> `std::lexicographical_compare()` +// function to lexicographically compare (e.g. sort words alphabetically) two +// container sequences. The comparison is performed using `operator<`. Note +// that capital letters ("A-Z") have ASCII values less than lowercase letters +// ("a-z"). +template <typename Sequence1, typename Sequence2> +bool c_lexicographical_compare(Sequence1&& sequence1, Sequence2&& sequence2) { + return std::lexicographical_compare( + container_algorithm_internal::c_begin(sequence1), + container_algorithm_internal::c_end(sequence1), + container_algorithm_internal::c_begin(sequence2), + container_algorithm_internal::c_end(sequence2)); +} + +// Overload of c_lexicographical_compare() for performing a lexicographical +// comparison using a `comp` operator instead of `operator<`. +template <typename Sequence1, typename Sequence2, typename Compare> +bool c_lexicographical_compare(Sequence1&& sequence1, Sequence2&& sequence2, + Compare&& comp) { + return std::lexicographical_compare( + container_algorithm_internal::c_begin(sequence1), + container_algorithm_internal::c_end(sequence1), + container_algorithm_internal::c_begin(sequence2), + container_algorithm_internal::c_end(sequence2), + std::forward<Compare>(comp)); +} + +// c_next_permutation() +// +// Container-based version of the <algorithm> `std::next_permutation()` function +// to rearrange a container's elements into the next lexicographically greater +// permutation. +template <typename C> +bool c_next_permutation(C& c) { + return std::next_permutation(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// Overload of c_next_permutation() for performing a lexicographical +// comparison using a `comp` operator instead of `operator<`. +template <typename C, typename Compare> +bool c_next_permutation(C& c, Compare&& comp) { + return std::next_permutation(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +// c_prev_permutation() +// +// Container-based version of the <algorithm> `std::prev_permutation()` function +// to rearrange a container's elements into the next lexicographically lesser +// permutation. +template <typename C> +bool c_prev_permutation(C& c) { + return std::prev_permutation(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c)); +} + +// Overload of c_prev_permutation() for performing a lexicographical +// comparison using a `comp` operator instead of `operator<`. +template <typename C, typename Compare> +bool c_prev_permutation(C& c, Compare&& comp) { + return std::prev_permutation(container_algorithm_internal::c_begin(c), + container_algorithm_internal::c_end(c), + std::forward<Compare>(comp)); +} + +//------------------------------------------------------------------------------ +// <numeric> algorithms +//------------------------------------------------------------------------------ + +// c_iota() +// +// Container-based version of the <algorithm> `std::iota()` function +// to compute successive values of `value`, as if incremented with `++value` +// after each element is written. and write them to the container. +template <typename Sequence, typename T> +void c_iota(Sequence& sequence, T&& value) { + std::iota(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(value)); +} +// c_accumulate() +// +// Container-based version of the <algorithm> `std::accumulate()` function +// to accumulate the element values of a container to `init` and return that +// accumulation by value. +// +// Note: Due to a language technicality this function has return type +// absl::decay_t<T>. As a user of this function you can casually read +// this as "returns T by value" and assume it does the right thing. +template <typename Sequence, typename T> +decay_t<T> c_accumulate(const Sequence& sequence, T&& init) { + return std::accumulate(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(init)); +} + +// Overload of c_accumulate() for using a binary operations other than +// addition for computing the accumulation. +template <typename Sequence, typename T, typename BinaryOp> +decay_t<T> c_accumulate(const Sequence& sequence, T&& init, + BinaryOp&& binary_op) { + return std::accumulate(container_algorithm_internal::c_begin(sequence), + container_algorithm_internal::c_end(sequence), + std::forward<T>(init), + std::forward<BinaryOp>(binary_op)); +} + +// c_inner_product() +// +// Container-based version of the <algorithm> `std::inner_product()` function +// to compute the cumulative inner product of container element pairs. +// +// Note: Due to a language technicality this function has return type +// absl::decay_t<T>. As a user of this function you can casually read +// this as "returns T by value" and assume it does the right thing. +template <typename Sequence1, typename Sequence2, typename T> +decay_t<T> c_inner_product(const Sequence1& factors1, const Sequence2& factors2, + T&& sum) { + return std::inner_product(container_algorithm_internal::c_begin(factors1), + container_algorithm_internal::c_end(factors1), + container_algorithm_internal::c_begin(factors2), + std::forward<T>(sum)); +} + +// Overload of c_inner_product() for using binary operations other than +// `operator+` (for computing the accumulation) and `operator*` (for computing +// the product between the two container's element pair). +template <typename Sequence1, typename Sequence2, typename T, + typename BinaryOp1, typename BinaryOp2> +decay_t<T> c_inner_product(const Sequence1& factors1, const Sequence2& factors2, + T&& sum, BinaryOp1&& op1, BinaryOp2&& op2) { + return std::inner_product(container_algorithm_internal::c_begin(factors1), + container_algorithm_internal::c_end(factors1), + container_algorithm_internal::c_begin(factors2), + std::forward<T>(sum), std::forward<BinaryOp1>(op1), + std::forward<BinaryOp2>(op2)); +} + +// c_adjacent_difference() +// +// Container-based version of the <algorithm> `std::adjacent_difference()` +// function to compute the difference between each element and the one preceding +// it and write it to an iterator. +template <typename InputSequence, typename OutputIt> +OutputIt c_adjacent_difference(const InputSequence& input, + OutputIt output_first) { + return std::adjacent_difference(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), + output_first); +} + +// Overload of c_adjacent_difference() for using a binary operation other than +// subtraction to compute the adjacent difference. +template <typename InputSequence, typename OutputIt, typename BinaryOp> +OutputIt c_adjacent_difference(const InputSequence& input, + OutputIt output_first, BinaryOp&& op) { + return std::adjacent_difference(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), + output_first, std::forward<BinaryOp>(op)); +} + +// c_partial_sum() +// +// Container-based version of the <algorithm> `std::partial_sum()` function +// to compute the partial sum of the elements in a sequence and write them +// to an iterator. The partial sum is the sum of all element values so far in +// the sequence. +template <typename InputSequence, typename OutputIt> +OutputIt c_partial_sum(const InputSequence& input, OutputIt output_first) { + return std::partial_sum(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), + output_first); +} + +// Overload of c_partial_sum() for using a binary operation other than addition +// to compute the "partial sum". +template <typename InputSequence, typename OutputIt, typename BinaryOp> +OutputIt c_partial_sum(const InputSequence& input, OutputIt output_first, + BinaryOp&& op) { + return std::partial_sum(container_algorithm_internal::c_begin(input), + container_algorithm_internal::c_end(input), + output_first, std::forward<BinaryOp>(op)); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_ALGORITHM_CONTAINER_H_ diff --git a/third_party/abseil_cpp/absl/algorithm/container_test.cc b/third_party/abseil_cpp/absl/algorithm/container_test.cc new file mode 100644 index 000000000000..605afc8040d7 --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/container_test.cc @@ -0,0 +1,1124 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/algorithm/container.h" + +#include <functional> +#include <initializer_list> +#include <iterator> +#include <list> +#include <memory> +#include <ostream> +#include <random> +#include <set> +#include <unordered_set> +#include <utility> +#include <valarray> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/casts.h" +#include "absl/base/macros.h" +#include "absl/memory/memory.h" +#include "absl/types/span.h" + +namespace { + +using ::testing::Each; +using ::testing::ElementsAre; +using ::testing::Gt; +using ::testing::IsNull; +using ::testing::Lt; +using ::testing::Pointee; +using ::testing::Truly; +using ::testing::UnorderedElementsAre; + +// Most of these tests just check that the code compiles, not that it +// does the right thing. That's fine since the functions just forward +// to the STL implementation. +class NonMutatingTest : public testing::Test { + protected: + std::unordered_set<int> container_ = {1, 2, 3}; + std::list<int> sequence_ = {1, 2, 3}; + std::vector<int> vector_ = {1, 2, 3}; + int array_[3] = {1, 2, 3}; +}; + +struct AccumulateCalls { + void operator()(int value) { calls.push_back(value); } + std::vector<int> calls; +}; + +bool Predicate(int value) { return value < 3; } +bool BinPredicate(int v1, int v2) { return v1 < v2; } +bool Equals(int v1, int v2) { return v1 == v2; } +bool IsOdd(int x) { return x % 2 != 0; } + +TEST_F(NonMutatingTest, Distance) { + EXPECT_EQ(container_.size(), absl::c_distance(container_)); + EXPECT_EQ(sequence_.size(), absl::c_distance(sequence_)); + EXPECT_EQ(vector_.size(), absl::c_distance(vector_)); + EXPECT_EQ(ABSL_ARRAYSIZE(array_), absl::c_distance(array_)); + + // Works with a temporary argument. + EXPECT_EQ(vector_.size(), absl::c_distance(std::vector<int>(vector_))); +} + +TEST_F(NonMutatingTest, Distance_OverloadedBeginEnd) { + // Works with classes which have custom ADL-selected overloads of std::begin + // and std::end. + std::initializer_list<int> a = {1, 2, 3}; + std::valarray<int> b = {1, 2, 3}; + EXPECT_EQ(3, absl::c_distance(a)); + EXPECT_EQ(3, absl::c_distance(b)); + + // It is assumed that other c_* functions use the same mechanism for + // ADL-selecting begin/end overloads. +} + +TEST_F(NonMutatingTest, ForEach) { + AccumulateCalls c = absl::c_for_each(container_, AccumulateCalls()); + // Don't rely on the unordered_set's order. + std::sort(c.calls.begin(), c.calls.end()); + EXPECT_EQ(vector_, c.calls); + + // Works with temporary container, too. + AccumulateCalls c2 = + absl::c_for_each(std::unordered_set<int>(container_), AccumulateCalls()); + std::sort(c2.calls.begin(), c2.calls.end()); + EXPECT_EQ(vector_, c2.calls); +} + +TEST_F(NonMutatingTest, FindReturnsCorrectType) { + auto it = absl::c_find(container_, 3); + EXPECT_EQ(3, *it); + absl::c_find(absl::implicit_cast<const std::list<int>&>(sequence_), 3); +} + +TEST_F(NonMutatingTest, FindIf) { absl::c_find_if(container_, Predicate); } + +TEST_F(NonMutatingTest, FindIfNot) { + absl::c_find_if_not(container_, Predicate); +} + +TEST_F(NonMutatingTest, FindEnd) { + absl::c_find_end(sequence_, vector_); + absl::c_find_end(vector_, sequence_); +} + +TEST_F(NonMutatingTest, FindEndWithPredicate) { + absl::c_find_end(sequence_, vector_, BinPredicate); + absl::c_find_end(vector_, sequence_, BinPredicate); +} + +TEST_F(NonMutatingTest, FindFirstOf) { + absl::c_find_first_of(container_, sequence_); + absl::c_find_first_of(sequence_, container_); +} + +TEST_F(NonMutatingTest, FindFirstOfWithPredicate) { + absl::c_find_first_of(container_, sequence_, BinPredicate); + absl::c_find_first_of(sequence_, container_, BinPredicate); +} + +TEST_F(NonMutatingTest, AdjacentFind) { absl::c_adjacent_find(sequence_); } + +TEST_F(NonMutatingTest, AdjacentFindWithPredicate) { + absl::c_adjacent_find(sequence_, BinPredicate); +} + +TEST_F(NonMutatingTest, Count) { EXPECT_EQ(1, absl::c_count(container_, 3)); } + +TEST_F(NonMutatingTest, CountIf) { + EXPECT_EQ(2, absl::c_count_if(container_, Predicate)); + const std::unordered_set<int>& const_container = container_; + EXPECT_EQ(2, absl::c_count_if(const_container, Predicate)); +} + +TEST_F(NonMutatingTest, Mismatch) { + // Testing necessary as absl::c_mismatch executes logic. + { + auto result = absl::c_mismatch(vector_, sequence_); + EXPECT_EQ(result.first, vector_.end()); + EXPECT_EQ(result.second, sequence_.end()); + } + { + auto result = absl::c_mismatch(sequence_, vector_); + EXPECT_EQ(result.first, sequence_.end()); + EXPECT_EQ(result.second, vector_.end()); + } + + sequence_.back() = 5; + { + auto result = absl::c_mismatch(vector_, sequence_); + EXPECT_EQ(result.first, std::prev(vector_.end())); + EXPECT_EQ(result.second, std::prev(sequence_.end())); + } + { + auto result = absl::c_mismatch(sequence_, vector_); + EXPECT_EQ(result.first, std::prev(sequence_.end())); + EXPECT_EQ(result.second, std::prev(vector_.end())); + } + + sequence_.pop_back(); + { + auto result = absl::c_mismatch(vector_, sequence_); + EXPECT_EQ(result.first, std::prev(vector_.end())); + EXPECT_EQ(result.second, sequence_.end()); + } + { + auto result = absl::c_mismatch(sequence_, vector_); + EXPECT_EQ(result.first, sequence_.end()); + EXPECT_EQ(result.second, std::prev(vector_.end())); + } + { + struct NoNotEquals { + constexpr bool operator==(NoNotEquals) const { return true; } + constexpr bool operator!=(NoNotEquals) const = delete; + }; + std::vector<NoNotEquals> first; + std::list<NoNotEquals> second; + + // Check this still compiles. + absl::c_mismatch(first, second); + } +} + +TEST_F(NonMutatingTest, MismatchWithPredicate) { + // Testing necessary as absl::c_mismatch executes logic. + { + auto result = absl::c_mismatch(vector_, sequence_, BinPredicate); + EXPECT_EQ(result.first, vector_.begin()); + EXPECT_EQ(result.second, sequence_.begin()); + } + { + auto result = absl::c_mismatch(sequence_, vector_, BinPredicate); + EXPECT_EQ(result.first, sequence_.begin()); + EXPECT_EQ(result.second, vector_.begin()); + } + + sequence_.front() = 0; + { + auto result = absl::c_mismatch(vector_, sequence_, BinPredicate); + EXPECT_EQ(result.first, vector_.begin()); + EXPECT_EQ(result.second, sequence_.begin()); + } + { + auto result = absl::c_mismatch(sequence_, vector_, BinPredicate); + EXPECT_EQ(result.first, std::next(sequence_.begin())); + EXPECT_EQ(result.second, std::next(vector_.begin())); + } + + sequence_.clear(); + { + auto result = absl::c_mismatch(vector_, sequence_, BinPredicate); + EXPECT_EQ(result.first, vector_.begin()); + EXPECT_EQ(result.second, sequence_.end()); + } + { + auto result = absl::c_mismatch(sequence_, vector_, BinPredicate); + EXPECT_EQ(result.first, sequence_.end()); + EXPECT_EQ(result.second, vector_.begin()); + } +} + +TEST_F(NonMutatingTest, Equal) { + EXPECT_TRUE(absl::c_equal(vector_, sequence_)); + EXPECT_TRUE(absl::c_equal(sequence_, vector_)); + EXPECT_TRUE(absl::c_equal(sequence_, array_)); + EXPECT_TRUE(absl::c_equal(array_, vector_)); + + // Test that behavior appropriately differs from that of equal(). + std::vector<int> vector_plus = {1, 2, 3}; + vector_plus.push_back(4); + EXPECT_FALSE(absl::c_equal(vector_plus, sequence_)); + EXPECT_FALSE(absl::c_equal(sequence_, vector_plus)); + EXPECT_FALSE(absl::c_equal(array_, vector_plus)); +} + +TEST_F(NonMutatingTest, EqualWithPredicate) { + EXPECT_TRUE(absl::c_equal(vector_, sequence_, Equals)); + EXPECT_TRUE(absl::c_equal(sequence_, vector_, Equals)); + EXPECT_TRUE(absl::c_equal(array_, sequence_, Equals)); + EXPECT_TRUE(absl::c_equal(vector_, array_, Equals)); + + // Test that behavior appropriately differs from that of equal(). + std::vector<int> vector_plus = {1, 2, 3}; + vector_plus.push_back(4); + EXPECT_FALSE(absl::c_equal(vector_plus, sequence_, Equals)); + EXPECT_FALSE(absl::c_equal(sequence_, vector_plus, Equals)); + EXPECT_FALSE(absl::c_equal(vector_plus, array_, Equals)); +} + +TEST_F(NonMutatingTest, IsPermutation) { + auto vector_permut_ = vector_; + std::next_permutation(vector_permut_.begin(), vector_permut_.end()); + EXPECT_TRUE(absl::c_is_permutation(vector_permut_, sequence_)); + EXPECT_TRUE(absl::c_is_permutation(sequence_, vector_permut_)); + + // Test that behavior appropriately differs from that of is_permutation(). + std::vector<int> vector_plus = {1, 2, 3}; + vector_plus.push_back(4); + EXPECT_FALSE(absl::c_is_permutation(vector_plus, sequence_)); + EXPECT_FALSE(absl::c_is_permutation(sequence_, vector_plus)); +} + +TEST_F(NonMutatingTest, IsPermutationWithPredicate) { + auto vector_permut_ = vector_; + std::next_permutation(vector_permut_.begin(), vector_permut_.end()); + EXPECT_TRUE(absl::c_is_permutation(vector_permut_, sequence_, Equals)); + EXPECT_TRUE(absl::c_is_permutation(sequence_, vector_permut_, Equals)); + + // Test that behavior appropriately differs from that of is_permutation(). + std::vector<int> vector_plus = {1, 2, 3}; + vector_plus.push_back(4); + EXPECT_FALSE(absl::c_is_permutation(vector_plus, sequence_, Equals)); + EXPECT_FALSE(absl::c_is_permutation(sequence_, vector_plus, Equals)); +} + +TEST_F(NonMutatingTest, Search) { + absl::c_search(sequence_, vector_); + absl::c_search(vector_, sequence_); + absl::c_search(array_, sequence_); +} + +TEST_F(NonMutatingTest, SearchWithPredicate) { + absl::c_search(sequence_, vector_, BinPredicate); + absl::c_search(vector_, sequence_, BinPredicate); +} + +TEST_F(NonMutatingTest, SearchN) { absl::c_search_n(sequence_, 3, 1); } + +TEST_F(NonMutatingTest, SearchNWithPredicate) { + absl::c_search_n(sequence_, 3, 1, BinPredicate); +} + +TEST_F(NonMutatingTest, LowerBound) { + std::list<int>::iterator i = absl::c_lower_bound(sequence_, 3); + ASSERT_TRUE(i != sequence_.end()); + EXPECT_EQ(2, std::distance(sequence_.begin(), i)); + EXPECT_EQ(3, *i); +} + +TEST_F(NonMutatingTest, LowerBoundWithPredicate) { + std::vector<int> v(vector_); + std::sort(v.begin(), v.end(), std::greater<int>()); + std::vector<int>::iterator i = absl::c_lower_bound(v, 3, std::greater<int>()); + EXPECT_TRUE(i == v.begin()); + EXPECT_EQ(3, *i); +} + +TEST_F(NonMutatingTest, UpperBound) { + std::list<int>::iterator i = absl::c_upper_bound(sequence_, 1); + ASSERT_TRUE(i != sequence_.end()); + EXPECT_EQ(1, std::distance(sequence_.begin(), i)); + EXPECT_EQ(2, *i); +} + +TEST_F(NonMutatingTest, UpperBoundWithPredicate) { + std::vector<int> v(vector_); + std::sort(v.begin(), v.end(), std::greater<int>()); + std::vector<int>::iterator i = absl::c_upper_bound(v, 1, std::greater<int>()); + EXPECT_EQ(3, i - v.begin()); + EXPECT_TRUE(i == v.end()); +} + +TEST_F(NonMutatingTest, EqualRange) { + std::pair<std::list<int>::iterator, std::list<int>::iterator> p = + absl::c_equal_range(sequence_, 2); + EXPECT_EQ(1, std::distance(sequence_.begin(), p.first)); + EXPECT_EQ(2, std::distance(sequence_.begin(), p.second)); +} + +TEST_F(NonMutatingTest, EqualRangeArray) { + auto p = absl::c_equal_range(array_, 2); + EXPECT_EQ(1, std::distance(std::begin(array_), p.first)); + EXPECT_EQ(2, std::distance(std::begin(array_), p.second)); +} + +TEST_F(NonMutatingTest, EqualRangeWithPredicate) { + std::vector<int> v(vector_); + std::sort(v.begin(), v.end(), std::greater<int>()); + std::pair<std::vector<int>::iterator, std::vector<int>::iterator> p = + absl::c_equal_range(v, 2, std::greater<int>()); + EXPECT_EQ(1, std::distance(v.begin(), p.first)); + EXPECT_EQ(2, std::distance(v.begin(), p.second)); +} + +TEST_F(NonMutatingTest, BinarySearch) { + EXPECT_TRUE(absl::c_binary_search(vector_, 2)); + EXPECT_TRUE(absl::c_binary_search(std::vector<int>(vector_), 2)); +} + +TEST_F(NonMutatingTest, BinarySearchWithPredicate) { + std::vector<int> v(vector_); + std::sort(v.begin(), v.end(), std::greater<int>()); + EXPECT_TRUE(absl::c_binary_search(v, 2, std::greater<int>())); + EXPECT_TRUE( + absl::c_binary_search(std::vector<int>(v), 2, std::greater<int>())); +} + +TEST_F(NonMutatingTest, MinElement) { + std::list<int>::iterator i = absl::c_min_element(sequence_); + ASSERT_TRUE(i != sequence_.end()); + EXPECT_EQ(*i, 1); +} + +TEST_F(NonMutatingTest, MinElementWithPredicate) { + std::list<int>::iterator i = + absl::c_min_element(sequence_, std::greater<int>()); + ASSERT_TRUE(i != sequence_.end()); + EXPECT_EQ(*i, 3); +} + +TEST_F(NonMutatingTest, MaxElement) { + std::list<int>::iterator i = absl::c_max_element(sequence_); + ASSERT_TRUE(i != sequence_.end()); + EXPECT_EQ(*i, 3); +} + +TEST_F(NonMutatingTest, MaxElementWithPredicate) { + std::list<int>::iterator i = + absl::c_max_element(sequence_, std::greater<int>()); + ASSERT_TRUE(i != sequence_.end()); + EXPECT_EQ(*i, 1); +} + +TEST_F(NonMutatingTest, LexicographicalCompare) { + EXPECT_FALSE(absl::c_lexicographical_compare(sequence_, sequence_)); + + std::vector<int> v; + v.push_back(1); + v.push_back(2); + v.push_back(4); + + EXPECT_TRUE(absl::c_lexicographical_compare(sequence_, v)); + EXPECT_TRUE(absl::c_lexicographical_compare(std::list<int>(sequence_), v)); +} + +TEST_F(NonMutatingTest, LexicographicalCopmareWithPredicate) { + EXPECT_FALSE(absl::c_lexicographical_compare(sequence_, sequence_, + std::greater<int>())); + + std::vector<int> v; + v.push_back(1); + v.push_back(2); + v.push_back(4); + + EXPECT_TRUE( + absl::c_lexicographical_compare(v, sequence_, std::greater<int>())); + EXPECT_TRUE(absl::c_lexicographical_compare( + std::vector<int>(v), std::list<int>(sequence_), std::greater<int>())); +} + +TEST_F(NonMutatingTest, Includes) { + std::set<int> s(vector_.begin(), vector_.end()); + s.insert(4); + EXPECT_TRUE(absl::c_includes(s, vector_)); +} + +TEST_F(NonMutatingTest, IncludesWithPredicate) { + std::vector<int> v = {3, 2, 1}; + std::set<int, std::greater<int>> s(v.begin(), v.end()); + s.insert(4); + EXPECT_TRUE(absl::c_includes(s, v, std::greater<int>())); +} + +class NumericMutatingTest : public testing::Test { + protected: + std::list<int> list_ = {1, 2, 3}; + std::vector<int> output_; +}; + +TEST_F(NumericMutatingTest, Iota) { + absl::c_iota(list_, 5); + std::list<int> expected{5, 6, 7}; + EXPECT_EQ(list_, expected); +} + +TEST_F(NonMutatingTest, Accumulate) { + EXPECT_EQ(absl::c_accumulate(sequence_, 4), 1 + 2 + 3 + 4); +} + +TEST_F(NonMutatingTest, AccumulateWithBinaryOp) { + EXPECT_EQ(absl::c_accumulate(sequence_, 4, std::multiplies<int>()), + 1 * 2 * 3 * 4); +} + +TEST_F(NonMutatingTest, AccumulateLvalueInit) { + int lvalue = 4; + EXPECT_EQ(absl::c_accumulate(sequence_, lvalue), 1 + 2 + 3 + 4); +} + +TEST_F(NonMutatingTest, AccumulateWithBinaryOpLvalueInit) { + int lvalue = 4; + EXPECT_EQ(absl::c_accumulate(sequence_, lvalue, std::multiplies<int>()), + 1 * 2 * 3 * 4); +} + +TEST_F(NonMutatingTest, InnerProduct) { + EXPECT_EQ(absl::c_inner_product(sequence_, vector_, 1000), + 1000 + 1 * 1 + 2 * 2 + 3 * 3); +} + +TEST_F(NonMutatingTest, InnerProductWithBinaryOps) { + EXPECT_EQ(absl::c_inner_product(sequence_, vector_, 10, + std::multiplies<int>(), std::plus<int>()), + 10 * (1 + 1) * (2 + 2) * (3 + 3)); +} + +TEST_F(NonMutatingTest, InnerProductLvalueInit) { + int lvalue = 1000; + EXPECT_EQ(absl::c_inner_product(sequence_, vector_, lvalue), + 1000 + 1 * 1 + 2 * 2 + 3 * 3); +} + +TEST_F(NonMutatingTest, InnerProductWithBinaryOpsLvalueInit) { + int lvalue = 10; + EXPECT_EQ(absl::c_inner_product(sequence_, vector_, lvalue, + std::multiplies<int>(), std::plus<int>()), + 10 * (1 + 1) * (2 + 2) * (3 + 3)); +} + +TEST_F(NumericMutatingTest, AdjacentDifference) { + auto last = absl::c_adjacent_difference(list_, std::back_inserter(output_)); + *last = 1000; + std::vector<int> expected{1, 2 - 1, 3 - 2, 1000}; + EXPECT_EQ(output_, expected); +} + +TEST_F(NumericMutatingTest, AdjacentDifferenceWithBinaryOp) { + auto last = absl::c_adjacent_difference(list_, std::back_inserter(output_), + std::multiplies<int>()); + *last = 1000; + std::vector<int> expected{1, 2 * 1, 3 * 2, 1000}; + EXPECT_EQ(output_, expected); +} + +TEST_F(NumericMutatingTest, PartialSum) { + auto last = absl::c_partial_sum(list_, std::back_inserter(output_)); + *last = 1000; + std::vector<int> expected{1, 1 + 2, 1 + 2 + 3, 1000}; + EXPECT_EQ(output_, expected); +} + +TEST_F(NumericMutatingTest, PartialSumWithBinaryOp) { + auto last = absl::c_partial_sum(list_, std::back_inserter(output_), + std::multiplies<int>()); + *last = 1000; + std::vector<int> expected{1, 1 * 2, 1 * 2 * 3, 1000}; + EXPECT_EQ(output_, expected); +} + +TEST_F(NonMutatingTest, LinearSearch) { + EXPECT_TRUE(absl::c_linear_search(container_, 3)); + EXPECT_FALSE(absl::c_linear_search(container_, 4)); +} + +TEST_F(NonMutatingTest, AllOf) { + const std::vector<int>& v = vector_; + EXPECT_FALSE(absl::c_all_of(v, [](int x) { return x > 1; })); + EXPECT_TRUE(absl::c_all_of(v, [](int x) { return x > 0; })); +} + +TEST_F(NonMutatingTest, AnyOf) { + const std::vector<int>& v = vector_; + EXPECT_TRUE(absl::c_any_of(v, [](int x) { return x > 2; })); + EXPECT_FALSE(absl::c_any_of(v, [](int x) { return x > 5; })); +} + +TEST_F(NonMutatingTest, NoneOf) { + const std::vector<int>& v = vector_; + EXPECT_FALSE(absl::c_none_of(v, [](int x) { return x > 2; })); + EXPECT_TRUE(absl::c_none_of(v, [](int x) { return x > 5; })); +} + +TEST_F(NonMutatingTest, MinMaxElementLess) { + std::pair<std::vector<int>::const_iterator, std::vector<int>::const_iterator> + p = absl::c_minmax_element(vector_, std::less<int>()); + EXPECT_TRUE(p.first == vector_.begin()); + EXPECT_TRUE(p.second == vector_.begin() + 2); +} + +TEST_F(NonMutatingTest, MinMaxElementGreater) { + std::pair<std::vector<int>::const_iterator, std::vector<int>::const_iterator> + p = absl::c_minmax_element(vector_, std::greater<int>()); + EXPECT_TRUE(p.first == vector_.begin() + 2); + EXPECT_TRUE(p.second == vector_.begin()); +} + +TEST_F(NonMutatingTest, MinMaxElementNoPredicate) { + std::pair<std::vector<int>::const_iterator, std::vector<int>::const_iterator> + p = absl::c_minmax_element(vector_); + EXPECT_TRUE(p.first == vector_.begin()); + EXPECT_TRUE(p.second == vector_.begin() + 2); +} + +class SortingTest : public testing::Test { + protected: + std::list<int> sorted_ = {1, 2, 3, 4}; + std::list<int> unsorted_ = {2, 4, 1, 3}; + std::list<int> reversed_ = {4, 3, 2, 1}; +}; + +TEST_F(SortingTest, IsSorted) { + EXPECT_TRUE(absl::c_is_sorted(sorted_)); + EXPECT_FALSE(absl::c_is_sorted(unsorted_)); + EXPECT_FALSE(absl::c_is_sorted(reversed_)); +} + +TEST_F(SortingTest, IsSortedWithPredicate) { + EXPECT_FALSE(absl::c_is_sorted(sorted_, std::greater<int>())); + EXPECT_FALSE(absl::c_is_sorted(unsorted_, std::greater<int>())); + EXPECT_TRUE(absl::c_is_sorted(reversed_, std::greater<int>())); +} + +TEST_F(SortingTest, IsSortedUntil) { + EXPECT_EQ(1, *absl::c_is_sorted_until(unsorted_)); + EXPECT_EQ(4, *absl::c_is_sorted_until(unsorted_, std::greater<int>())); +} + +TEST_F(SortingTest, NthElement) { + std::vector<int> unsorted = {2, 4, 1, 3}; + absl::c_nth_element(unsorted, unsorted.begin() + 2); + EXPECT_THAT(unsorted, ElementsAre(Lt(3), Lt(3), 3, Gt(3))); + absl::c_nth_element(unsorted, unsorted.begin() + 2, std::greater<int>()); + EXPECT_THAT(unsorted, ElementsAre(Gt(2), Gt(2), 2, Lt(2))); +} + +TEST(MutatingTest, IsPartitioned) { + EXPECT_TRUE( + absl::c_is_partitioned(std::vector<int>{1, 3, 5, 2, 4, 6}, IsOdd)); + EXPECT_FALSE( + absl::c_is_partitioned(std::vector<int>{1, 2, 3, 4, 5, 6}, IsOdd)); + EXPECT_FALSE( + absl::c_is_partitioned(std::vector<int>{2, 4, 6, 1, 3, 5}, IsOdd)); +} + +TEST(MutatingTest, Partition) { + std::vector<int> actual = {1, 2, 3, 4, 5}; + absl::c_partition(actual, IsOdd); + EXPECT_THAT(actual, Truly([](const std::vector<int>& c) { + return absl::c_is_partitioned(c, IsOdd); + })); +} + +TEST(MutatingTest, StablePartition) { + std::vector<int> actual = {1, 2, 3, 4, 5}; + absl::c_stable_partition(actual, IsOdd); + EXPECT_THAT(actual, ElementsAre(1, 3, 5, 2, 4)); +} + +TEST(MutatingTest, PartitionCopy) { + const std::vector<int> initial = {1, 2, 3, 4, 5}; + std::vector<int> odds, evens; + auto ends = absl::c_partition_copy(initial, back_inserter(odds), + back_inserter(evens), IsOdd); + *ends.first = 7; + *ends.second = 6; + EXPECT_THAT(odds, ElementsAre(1, 3, 5, 7)); + EXPECT_THAT(evens, ElementsAre(2, 4, 6)); +} + +TEST(MutatingTest, PartitionPoint) { + const std::vector<int> initial = {1, 3, 5, 2, 4}; + auto middle = absl::c_partition_point(initial, IsOdd); + EXPECT_EQ(2, *middle); +} + +TEST(MutatingTest, CopyMiddle) { + const std::vector<int> initial = {4, -1, -2, -3, 5}; + const std::list<int> input = {1, 2, 3}; + const std::vector<int> expected = {4, 1, 2, 3, 5}; + + std::list<int> test_list(initial.begin(), initial.end()); + absl::c_copy(input, ++test_list.begin()); + EXPECT_EQ(std::list<int>(expected.begin(), expected.end()), test_list); + + std::vector<int> test_vector = initial; + absl::c_copy(input, test_vector.begin() + 1); + EXPECT_EQ(expected, test_vector); +} + +TEST(MutatingTest, CopyFrontInserter) { + const std::list<int> initial = {4, 5}; + const std::list<int> input = {1, 2, 3}; + const std::list<int> expected = {3, 2, 1, 4, 5}; + + std::list<int> test_list = initial; + absl::c_copy(input, std::front_inserter(test_list)); + EXPECT_EQ(expected, test_list); +} + +TEST(MutatingTest, CopyBackInserter) { + const std::vector<int> initial = {4, 5}; + const std::list<int> input = {1, 2, 3}; + const std::vector<int> expected = {4, 5, 1, 2, 3}; + + std::list<int> test_list(initial.begin(), initial.end()); + absl::c_copy(input, std::back_inserter(test_list)); + EXPECT_EQ(std::list<int>(expected.begin(), expected.end()), test_list); + + std::vector<int> test_vector = initial; + absl::c_copy(input, std::back_inserter(test_vector)); + EXPECT_EQ(expected, test_vector); +} + +TEST(MutatingTest, CopyN) { + const std::vector<int> initial = {1, 2, 3, 4, 5}; + const std::vector<int> expected = {1, 2}; + std::vector<int> actual; + absl::c_copy_n(initial, 2, back_inserter(actual)); + EXPECT_EQ(expected, actual); +} + +TEST(MutatingTest, CopyIf) { + const std::list<int> input = {1, 2, 3}; + std::vector<int> output; + absl::c_copy_if(input, std::back_inserter(output), + [](int i) { return i != 2; }); + EXPECT_THAT(output, ElementsAre(1, 3)); +} + +TEST(MutatingTest, CopyBackward) { + std::vector<int> actual = {1, 2, 3, 4, 5}; + std::vector<int> expected = {1, 2, 1, 2, 3}; + absl::c_copy_backward(absl::MakeSpan(actual.data(), 3), actual.end()); + EXPECT_EQ(expected, actual); +} + +TEST(MutatingTest, Move) { + std::vector<std::unique_ptr<int>> src; + src.emplace_back(absl::make_unique<int>(1)); + src.emplace_back(absl::make_unique<int>(2)); + src.emplace_back(absl::make_unique<int>(3)); + src.emplace_back(absl::make_unique<int>(4)); + src.emplace_back(absl::make_unique<int>(5)); + + std::vector<std::unique_ptr<int>> dest = {}; + absl::c_move(src, std::back_inserter(dest)); + EXPECT_THAT(src, Each(IsNull())); + EXPECT_THAT(dest, ElementsAre(Pointee(1), Pointee(2), Pointee(3), Pointee(4), + Pointee(5))); +} + +TEST(MutatingTest, MoveBackward) { + std::vector<std::unique_ptr<int>> actual; + actual.emplace_back(absl::make_unique<int>(1)); + actual.emplace_back(absl::make_unique<int>(2)); + actual.emplace_back(absl::make_unique<int>(3)); + actual.emplace_back(absl::make_unique<int>(4)); + actual.emplace_back(absl::make_unique<int>(5)); + auto subrange = absl::MakeSpan(actual.data(), 3); + absl::c_move_backward(subrange, actual.end()); + EXPECT_THAT(actual, ElementsAre(IsNull(), IsNull(), Pointee(1), Pointee(2), + Pointee(3))); +} + +TEST(MutatingTest, MoveWithRvalue) { + auto MakeRValueSrc = [] { + std::vector<std::unique_ptr<int>> src; + src.emplace_back(absl::make_unique<int>(1)); + src.emplace_back(absl::make_unique<int>(2)); + src.emplace_back(absl::make_unique<int>(3)); + return src; + }; + + std::vector<std::unique_ptr<int>> dest = MakeRValueSrc(); + absl::c_move(MakeRValueSrc(), std::back_inserter(dest)); + EXPECT_THAT(dest, ElementsAre(Pointee(1), Pointee(2), Pointee(3), Pointee(1), + Pointee(2), Pointee(3))); +} + +TEST(MutatingTest, SwapRanges) { + std::vector<int> odds = {2, 4, 6}; + std::vector<int> evens = {1, 3, 5}; + absl::c_swap_ranges(odds, evens); + EXPECT_THAT(odds, ElementsAre(1, 3, 5)); + EXPECT_THAT(evens, ElementsAre(2, 4, 6)); + + odds.pop_back(); + absl::c_swap_ranges(odds, evens); + EXPECT_THAT(odds, ElementsAre(2, 4)); + EXPECT_THAT(evens, ElementsAre(1, 3, 6)); + + absl::c_swap_ranges(evens, odds); + EXPECT_THAT(odds, ElementsAre(1, 3)); + EXPECT_THAT(evens, ElementsAre(2, 4, 6)); +} + +TEST_F(NonMutatingTest, Transform) { + std::vector<int> x{0, 2, 4}, y, z; + auto end = absl::c_transform(x, back_inserter(y), std::negate<int>()); + EXPECT_EQ(std::vector<int>({0, -2, -4}), y); + *end = 7; + EXPECT_EQ(std::vector<int>({0, -2, -4, 7}), y); + + y = {1, 3, 0}; + end = absl::c_transform(x, y, back_inserter(z), std::plus<int>()); + EXPECT_EQ(std::vector<int>({1, 5, 4}), z); + *end = 7; + EXPECT_EQ(std::vector<int>({1, 5, 4, 7}), z); + + z.clear(); + y.pop_back(); + end = absl::c_transform(x, y, std::back_inserter(z), std::plus<int>()); + EXPECT_EQ(std::vector<int>({1, 5}), z); + *end = 7; + EXPECT_EQ(std::vector<int>({1, 5, 7}), z); + + z.clear(); + std::swap(x, y); + end = absl::c_transform(x, y, std::back_inserter(z), std::plus<int>()); + EXPECT_EQ(std::vector<int>({1, 5}), z); + *end = 7; + EXPECT_EQ(std::vector<int>({1, 5, 7}), z); +} + +TEST(MutatingTest, Replace) { + const std::vector<int> initial = {1, 2, 3, 1, 4, 5}; + const std::vector<int> expected = {4, 2, 3, 4, 4, 5}; + + std::vector<int> test_vector = initial; + absl::c_replace(test_vector, 1, 4); + EXPECT_EQ(expected, test_vector); + + std::list<int> test_list(initial.begin(), initial.end()); + absl::c_replace(test_list, 1, 4); + EXPECT_EQ(std::list<int>(expected.begin(), expected.end()), test_list); +} + +TEST(MutatingTest, ReplaceIf) { + std::vector<int> actual = {1, 2, 3, 4, 5}; + const std::vector<int> expected = {0, 2, 0, 4, 0}; + + absl::c_replace_if(actual, IsOdd, 0); + EXPECT_EQ(expected, actual); +} + +TEST(MutatingTest, ReplaceCopy) { + const std::vector<int> initial = {1, 2, 3, 1, 4, 5}; + const std::vector<int> expected = {4, 2, 3, 4, 4, 5}; + + std::vector<int> actual; + absl::c_replace_copy(initial, back_inserter(actual), 1, 4); + EXPECT_EQ(expected, actual); +} + +TEST(MutatingTest, Sort) { + std::vector<int> test_vector = {2, 3, 1, 4}; + absl::c_sort(test_vector); + EXPECT_THAT(test_vector, ElementsAre(1, 2, 3, 4)); +} + +TEST(MutatingTest, SortWithPredicate) { + std::vector<int> test_vector = {2, 3, 1, 4}; + absl::c_sort(test_vector, std::greater<int>()); + EXPECT_THAT(test_vector, ElementsAre(4, 3, 2, 1)); +} + +// For absl::c_stable_sort tests. Needs an operator< that does not cover all +// fields so that the test can check the sort preserves order of equal elements. +struct Element { + int key; + int value; + friend bool operator<(const Element& e1, const Element& e2) { + return e1.key < e2.key; + } + // Make gmock print useful diagnostics. + friend std::ostream& operator<<(std::ostream& o, const Element& e) { + return o << "{" << e.key << ", " << e.value << "}"; + } +}; + +MATCHER_P2(IsElement, key, value, "") { + return arg.key == key && arg.value == value; +} + +TEST(MutatingTest, StableSort) { + std::vector<Element> test_vector = {{1, 1}, {2, 1}, {2, 0}, {1, 0}, {2, 2}}; + absl::c_stable_sort(test_vector); + EXPECT_THAT(test_vector, + ElementsAre(IsElement(1, 1), IsElement(1, 0), IsElement(2, 1), + IsElement(2, 0), IsElement(2, 2))); +} + +TEST(MutatingTest, StableSortWithPredicate) { + std::vector<Element> test_vector = {{1, 1}, {2, 1}, {2, 0}, {1, 0}, {2, 2}}; + absl::c_stable_sort(test_vector, [](const Element& e1, const Element& e2) { + return e2 < e1; + }); + EXPECT_THAT(test_vector, + ElementsAre(IsElement(2, 1), IsElement(2, 0), IsElement(2, 2), + IsElement(1, 1), IsElement(1, 0))); +} + +TEST(MutatingTest, ReplaceCopyIf) { + const std::vector<int> initial = {1, 2, 3, 4, 5}; + const std::vector<int> expected = {0, 2, 0, 4, 0}; + + std::vector<int> actual; + absl::c_replace_copy_if(initial, back_inserter(actual), IsOdd, 0); + EXPECT_EQ(expected, actual); +} + +TEST(MutatingTest, Fill) { + std::vector<int> actual(5); + absl::c_fill(actual, 1); + EXPECT_THAT(actual, ElementsAre(1, 1, 1, 1, 1)); +} + +TEST(MutatingTest, FillN) { + std::vector<int> actual(5, 0); + absl::c_fill_n(actual, 2, 1); + EXPECT_THAT(actual, ElementsAre(1, 1, 0, 0, 0)); +} + +TEST(MutatingTest, Generate) { + std::vector<int> actual(5); + int x = 0; + absl::c_generate(actual, [&x]() { return ++x; }); + EXPECT_THAT(actual, ElementsAre(1, 2, 3, 4, 5)); +} + +TEST(MutatingTest, GenerateN) { + std::vector<int> actual(5, 0); + int x = 0; + absl::c_generate_n(actual, 3, [&x]() { return ++x; }); + EXPECT_THAT(actual, ElementsAre(1, 2, 3, 0, 0)); +} + +TEST(MutatingTest, RemoveCopy) { + std::vector<int> actual; + absl::c_remove_copy(std::vector<int>{1, 2, 3}, back_inserter(actual), 2); + EXPECT_THAT(actual, ElementsAre(1, 3)); +} + +TEST(MutatingTest, RemoveCopyIf) { + std::vector<int> actual; + absl::c_remove_copy_if(std::vector<int>{1, 2, 3}, back_inserter(actual), + IsOdd); + EXPECT_THAT(actual, ElementsAre(2)); +} + +TEST(MutatingTest, UniqueCopy) { + std::vector<int> actual; + absl::c_unique_copy(std::vector<int>{1, 2, 2, 2, 3, 3, 2}, + back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(1, 2, 3, 2)); +} + +TEST(MutatingTest, UniqueCopyWithPredicate) { + std::vector<int> actual; + absl::c_unique_copy(std::vector<int>{1, 2, 3, -1, -2, -3, 1}, + back_inserter(actual), + [](int x, int y) { return (x < 0) == (y < 0); }); + EXPECT_THAT(actual, ElementsAre(1, -1, 1)); +} + +TEST(MutatingTest, Reverse) { + std::vector<int> test_vector = {1, 2, 3, 4}; + absl::c_reverse(test_vector); + EXPECT_THAT(test_vector, ElementsAre(4, 3, 2, 1)); + + std::list<int> test_list = {1, 2, 3, 4}; + absl::c_reverse(test_list); + EXPECT_THAT(test_list, ElementsAre(4, 3, 2, 1)); +} + +TEST(MutatingTest, ReverseCopy) { + std::vector<int> actual; + absl::c_reverse_copy(std::vector<int>{1, 2, 3, 4}, back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(4, 3, 2, 1)); +} + +TEST(MutatingTest, Rotate) { + std::vector<int> actual = {1, 2, 3, 4}; + auto it = absl::c_rotate(actual, actual.begin() + 2); + EXPECT_THAT(actual, testing::ElementsAreArray({3, 4, 1, 2})); + EXPECT_EQ(*it, 1); +} + +TEST(MutatingTest, RotateCopy) { + std::vector<int> initial = {1, 2, 3, 4}; + std::vector<int> actual; + auto end = + absl::c_rotate_copy(initial, initial.begin() + 2, back_inserter(actual)); + *end = 5; + EXPECT_THAT(actual, ElementsAre(3, 4, 1, 2, 5)); +} + +TEST(MutatingTest, Shuffle) { + std::vector<int> actual = {1, 2, 3, 4, 5}; + absl::c_shuffle(actual, std::random_device()); + EXPECT_THAT(actual, UnorderedElementsAre(1, 2, 3, 4, 5)); +} + +TEST(MutatingTest, PartialSort) { + std::vector<int> sequence{5, 3, 42, 0}; + absl::c_partial_sort(sequence, sequence.begin() + 2); + EXPECT_THAT(absl::MakeSpan(sequence.data(), 2), ElementsAre(0, 3)); + absl::c_partial_sort(sequence, sequence.begin() + 2, std::greater<int>()); + EXPECT_THAT(absl::MakeSpan(sequence.data(), 2), ElementsAre(42, 5)); +} + +TEST(MutatingTest, PartialSortCopy) { + const std::vector<int> initial = {5, 3, 42, 0}; + std::vector<int> actual(2); + absl::c_partial_sort_copy(initial, actual); + EXPECT_THAT(actual, ElementsAre(0, 3)); + absl::c_partial_sort_copy(initial, actual, std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(42, 5)); +} + +TEST(MutatingTest, Merge) { + std::vector<int> actual; + absl::c_merge(std::vector<int>{1, 3, 5}, std::vector<int>{2, 4}, + back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(1, 2, 3, 4, 5)); +} + +TEST(MutatingTest, MergeWithComparator) { + std::vector<int> actual; + absl::c_merge(std::vector<int>{5, 3, 1}, std::vector<int>{4, 2}, + back_inserter(actual), std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(5, 4, 3, 2, 1)); +} + +TEST(MutatingTest, InplaceMerge) { + std::vector<int> actual = {1, 3, 5, 2, 4}; + absl::c_inplace_merge(actual, actual.begin() + 3); + EXPECT_THAT(actual, ElementsAre(1, 2, 3, 4, 5)); +} + +TEST(MutatingTest, InplaceMergeWithComparator) { + std::vector<int> actual = {5, 3, 1, 4, 2}; + absl::c_inplace_merge(actual, actual.begin() + 3, std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(5, 4, 3, 2, 1)); +} + +class SetOperationsTest : public testing::Test { + protected: + std::vector<int> a_ = {1, 2, 3}; + std::vector<int> b_ = {1, 3, 5}; + + std::vector<int> a_reversed_ = {3, 2, 1}; + std::vector<int> b_reversed_ = {5, 3, 1}; +}; + +TEST_F(SetOperationsTest, SetUnion) { + std::vector<int> actual; + absl::c_set_union(a_, b_, back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(1, 2, 3, 5)); +} + +TEST_F(SetOperationsTest, SetUnionWithComparator) { + std::vector<int> actual; + absl::c_set_union(a_reversed_, b_reversed_, back_inserter(actual), + std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(5, 3, 2, 1)); +} + +TEST_F(SetOperationsTest, SetIntersection) { + std::vector<int> actual; + absl::c_set_intersection(a_, b_, back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(1, 3)); +} + +TEST_F(SetOperationsTest, SetIntersectionWithComparator) { + std::vector<int> actual; + absl::c_set_intersection(a_reversed_, b_reversed_, back_inserter(actual), + std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(3, 1)); +} + +TEST_F(SetOperationsTest, SetDifference) { + std::vector<int> actual; + absl::c_set_difference(a_, b_, back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(2)); +} + +TEST_F(SetOperationsTest, SetDifferenceWithComparator) { + std::vector<int> actual; + absl::c_set_difference(a_reversed_, b_reversed_, back_inserter(actual), + std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(2)); +} + +TEST_F(SetOperationsTest, SetSymmetricDifference) { + std::vector<int> actual; + absl::c_set_symmetric_difference(a_, b_, back_inserter(actual)); + EXPECT_THAT(actual, ElementsAre(2, 5)); +} + +TEST_F(SetOperationsTest, SetSymmetricDifferenceWithComparator) { + std::vector<int> actual; + absl::c_set_symmetric_difference(a_reversed_, b_reversed_, + back_inserter(actual), std::greater<int>()); + EXPECT_THAT(actual, ElementsAre(5, 2)); +} + +TEST(HeapOperationsTest, WithoutComparator) { + std::vector<int> heap = {1, 2, 3}; + EXPECT_FALSE(absl::c_is_heap(heap)); + absl::c_make_heap(heap); + EXPECT_TRUE(absl::c_is_heap(heap)); + heap.push_back(4); + EXPECT_EQ(3, absl::c_is_heap_until(heap) - heap.begin()); + absl::c_push_heap(heap); + EXPECT_EQ(4, heap[0]); + absl::c_pop_heap(heap); + EXPECT_EQ(4, heap[3]); + absl::c_make_heap(heap); + absl::c_sort_heap(heap); + EXPECT_THAT(heap, ElementsAre(1, 2, 3, 4)); + EXPECT_FALSE(absl::c_is_heap(heap)); +} + +TEST(HeapOperationsTest, WithComparator) { + using greater = std::greater<int>; + std::vector<int> heap = {3, 2, 1}; + EXPECT_FALSE(absl::c_is_heap(heap, greater())); + absl::c_make_heap(heap, greater()); + EXPECT_TRUE(absl::c_is_heap(heap, greater())); + heap.push_back(0); + EXPECT_EQ(3, absl::c_is_heap_until(heap, greater()) - heap.begin()); + absl::c_push_heap(heap, greater()); + EXPECT_EQ(0, heap[0]); + absl::c_pop_heap(heap, greater()); + EXPECT_EQ(0, heap[3]); + absl::c_make_heap(heap, greater()); + absl::c_sort_heap(heap, greater()); + EXPECT_THAT(heap, ElementsAre(3, 2, 1, 0)); + EXPECT_FALSE(absl::c_is_heap(heap, greater())); +} + +TEST(MutatingTest, PermutationOperations) { + std::vector<int> initial = {1, 2, 3, 4}; + std::vector<int> permuted = initial; + + absl::c_next_permutation(permuted); + EXPECT_TRUE(absl::c_is_permutation(initial, permuted)); + EXPECT_TRUE(absl::c_is_permutation(initial, permuted, std::equal_to<int>())); + + std::vector<int> permuted2 = initial; + absl::c_prev_permutation(permuted2, std::greater<int>()); + EXPECT_EQ(permuted, permuted2); + + absl::c_prev_permutation(permuted); + EXPECT_EQ(initial, permuted); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/algorithm/equal_benchmark.cc b/third_party/abseil_cpp/absl/algorithm/equal_benchmark.cc new file mode 100644 index 000000000000..7bf62c9a7f56 --- /dev/null +++ b/third_party/abseil_cpp/absl/algorithm/equal_benchmark.cc @@ -0,0 +1,126 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <cstring> + +#include "benchmark/benchmark.h" +#include "absl/algorithm/algorithm.h" + +namespace { + +// The range of sequence sizes to benchmark. +constexpr int kMinBenchmarkSize = 1024; +constexpr int kMaxBenchmarkSize = 8 * 1024 * 1024; + +// A user-defined type for use in equality benchmarks. Note that we expect +// std::memcmp to win for this type: libstdc++'s std::equal only defers to +// memcmp for integral types. This is because it is not straightforward to +// guarantee that std::memcmp would produce a result "as-if" compared by +// operator== for other types (example gotchas: NaN floats, structs with +// padding). +struct EightBits { + explicit EightBits(int /* unused */) : data(0) {} + bool operator==(const EightBits& rhs) const { return data == rhs.data; } + uint8_t data; +}; + +template <typename T> +void BM_absl_equal_benchmark(benchmark::State& state) { + std::vector<T> xs(state.range(0), T(0)); + std::vector<T> ys = xs; + while (state.KeepRunning()) { + const bool same = absl::equal(xs.begin(), xs.end(), ys.begin(), ys.end()); + benchmark::DoNotOptimize(same); + } +} + +template <typename T> +void BM_std_equal_benchmark(benchmark::State& state) { + std::vector<T> xs(state.range(0), T(0)); + std::vector<T> ys = xs; + while (state.KeepRunning()) { + const bool same = std::equal(xs.begin(), xs.end(), ys.begin()); + benchmark::DoNotOptimize(same); + } +} + +template <typename T> +void BM_memcmp_benchmark(benchmark::State& state) { + std::vector<T> xs(state.range(0), T(0)); + std::vector<T> ys = xs; + while (state.KeepRunning()) { + const bool same = + std::memcmp(xs.data(), ys.data(), xs.size() * sizeof(T)) == 0; + benchmark::DoNotOptimize(same); + } +} + +// The expectation is that the compiler should be able to elide the equality +// comparison altogether for sufficiently simple types. +template <typename T> +void BM_absl_equal_self_benchmark(benchmark::State& state) { + std::vector<T> xs(state.range(0), T(0)); + while (state.KeepRunning()) { + const bool same = absl::equal(xs.begin(), xs.end(), xs.begin(), xs.end()); + benchmark::DoNotOptimize(same); + } +} + +BENCHMARK_TEMPLATE(BM_absl_equal_benchmark, uint8_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_std_equal_benchmark, uint8_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_memcmp_benchmark, uint8_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_absl_equal_self_benchmark, uint8_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); + +BENCHMARK_TEMPLATE(BM_absl_equal_benchmark, uint16_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_std_equal_benchmark, uint16_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_memcmp_benchmark, uint16_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_absl_equal_self_benchmark, uint16_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); + +BENCHMARK_TEMPLATE(BM_absl_equal_benchmark, uint32_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_std_equal_benchmark, uint32_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_memcmp_benchmark, uint32_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_absl_equal_self_benchmark, uint32_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); + +BENCHMARK_TEMPLATE(BM_absl_equal_benchmark, uint64_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_std_equal_benchmark, uint64_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_memcmp_benchmark, uint64_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_absl_equal_self_benchmark, uint64_t) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); + +BENCHMARK_TEMPLATE(BM_absl_equal_benchmark, EightBits) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_std_equal_benchmark, EightBits) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_memcmp_benchmark, EightBits) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); +BENCHMARK_TEMPLATE(BM_absl_equal_self_benchmark, EightBits) + ->Range(kMinBenchmarkSize, kMaxBenchmarkSize); + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/BUILD.bazel b/third_party/abseil_cpp/absl/base/BUILD.bazel new file mode 100644 index 000000000000..9d96abeb33ae --- /dev/null +++ b/third_party/abseil_cpp/absl/base/BUILD.bazel @@ -0,0 +1,818 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "atomic_hook", + hdrs = ["internal/atomic_hook.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ":core_headers", + ], +) + +cc_library( + name = "errno_saver", + hdrs = ["internal/errno_saver.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [":config"], +) + +cc_library( + name = "log_severity", + srcs = ["log_severity.cc"], + hdrs = ["log_severity.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":core_headers", + ], +) + +cc_library( + name = "raw_logging_internal", + srcs = ["internal/raw_logging.cc"], + hdrs = ["internal/raw_logging.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":atomic_hook", + ":config", + ":core_headers", + ":log_severity", + ], +) + +cc_library( + name = "spinlock_wait", + srcs = [ + "internal/spinlock_akaros.inc", + "internal/spinlock_linux.inc", + "internal/spinlock_posix.inc", + "internal/spinlock_wait.cc", + "internal/spinlock_win32.inc", + ], + hdrs = ["internal/spinlock_wait.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/base:__pkg__", + ], + deps = [ + ":base_internal", + ":core_headers", + ":errno_saver", + ], +) + +cc_library( + name = "config", + hdrs = [ + "config.h", + "options.h", + "policy_checks.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, +) + +cc_library( + name = "dynamic_annotations", + srcs = [ + "internal/dynamic_annotations.h", + ], + hdrs = [ + "dynamic_annotations.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":core_headers", + ], +) + +cc_library( + name = "core_headers", + srcs = [ + "internal/thread_annotations.h", + ], + hdrs = [ + "attributes.h", + "const_init.h", + "macros.h", + "optimization.h", + "port.h", + "thread_annotations.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ], +) + +cc_library( + name = "malloc_internal", + srcs = [ + "internal/low_level_alloc.cc", + ], + hdrs = [ + "internal/direct_mmap.h", + "internal/low_level_alloc.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = select({ + "//absl:windows": [], + "//absl:wasm": [], + "//conditions:default": ["-pthread"], + }) + ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//visibility:public", + ], + deps = [ + ":base", + ":base_internal", + ":config", + ":core_headers", + ":dynamic_annotations", + ":raw_logging_internal", + ], +) + +cc_library( + name = "base_internal", + hdrs = [ + "internal/hide_ptr.h", + "internal/identity.h", + "internal/inline_variable.h", + "internal/invoke.h", + "internal/scheduling_mode.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + "//absl/meta:type_traits", + ], +) + +cc_library( + name = "base", + srcs = [ + "internal/cycleclock.cc", + "internal/spinlock.cc", + "internal/sysinfo.cc", + "internal/thread_identity.cc", + "internal/unscaledcycleclock.cc", + ], + hdrs = [ + "call_once.h", + "casts.h", + "internal/cycleclock.h", + "internal/low_level_scheduling.h", + "internal/per_thread_tls.h", + "internal/spinlock.h", + "internal/sysinfo.h", + "internal/thread_identity.h", + "internal/tsan_mutex_interface.h", + "internal/unscaledcycleclock.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = select({ + "//absl:windows": [ + "-DEFAULTLIB:advapi32.lib", + ], + "//absl:wasm": [], + "//conditions:default": ["-pthread"], + }) + ABSL_DEFAULT_LINKOPTS, + deps = [ + ":atomic_hook", + ":base_internal", + ":config", + ":core_headers", + ":dynamic_annotations", + ":log_severity", + ":raw_logging_internal", + ":spinlock_wait", + "//absl/meta:type_traits", + ], +) + +cc_library( + name = "atomic_hook_test_helper", + testonly = 1, + srcs = ["internal/atomic_hook_test_helper.cc"], + hdrs = ["internal/atomic_hook_test_helper.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":atomic_hook", + ":core_headers", + ], +) + +cc_test( + name = "atomic_hook_test", + size = "small", + srcs = ["internal/atomic_hook_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":atomic_hook", + ":atomic_hook_test_helper", + ":core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "bit_cast_test", + size = "small", + srcs = [ + "bit_cast_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base", + ":core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "throw_delegate", + srcs = ["internal/throw_delegate.cc"], + hdrs = ["internal/throw_delegate.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ":raw_logging_internal", + ], +) + +cc_test( + name = "throw_delegate_test", + srcs = ["throw_delegate_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":throw_delegate", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "errno_saver_test", + size = "small", + srcs = ["internal/errno_saver_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":errno_saver", + ":strerror", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "exception_testing", + testonly = 1, + hdrs = ["internal/exception_testing.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "pretty_function", + hdrs = ["internal/pretty_function.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//absl:__subpackages__"], +) + +cc_library( + name = "exception_safety_testing", + testonly = 1, + srcs = ["internal/exception_safety_testing.cc"], + hdrs = ["internal/exception_safety_testing.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":pretty_function", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/utility", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "exception_safety_testing_test", + srcs = ["exception_safety_testing_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":exception_safety_testing", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "inline_variable_test", + size = "small", + srcs = [ + "inline_variable_test.cc", + "inline_variable_test_a.cc", + "inline_variable_test_b.cc", + "internal/inline_variable_testing.h", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "invoke_test", + size = "small", + srcs = ["invoke_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base_internal", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +# Common test library made available for use in non-absl code that overrides +# AbslInternalSpinLockDelay and AbslInternalSpinLockWake. +cc_library( + name = "spinlock_test_common", + testonly = 1, + srcs = ["spinlock_test_common.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base", + ":base_internal", + ":config", + ":core_headers", + "//absl/synchronization", + "@com_google_googletest//:gtest", + ], + alwayslink = 1, +) + +cc_test( + name = "spinlock_test", + size = "medium", + srcs = ["spinlock_test_common.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base", + ":base_internal", + ":config", + ":core_headers", + "//absl/synchronization", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "spinlock_benchmark_common", + testonly = 1, + srcs = ["internal/spinlock_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/base:__pkg__", + ], + deps = [ + ":base", + ":base_internal", + ":raw_logging_internal", + "//absl/synchronization", + "@com_github_google_benchmark//:benchmark_main", + ], + alwayslink = 1, +) + +cc_binary( + name = "spinlock_benchmark", + testonly = 1, + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":spinlock_benchmark_common", + ], +) + +cc_library( + name = "endian", + hdrs = [ + "internal/endian.h", + "internal/unaligned_access.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":core_headers", + ], +) + +cc_test( + name = "endian_test", + srcs = ["internal/endian_test.cc"], + copts = ABSL_TEST_COPTS, + deps = [ + ":config", + ":endian", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "config_test", + srcs = ["config_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + "//absl/synchronization:thread_pool", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "call_once_test", + srcs = ["call_once_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base", + ":core_headers", + "//absl/synchronization", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "raw_logging_test", + srcs = ["raw_logging_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":raw_logging_internal", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "sysinfo_test", + size = "small", + srcs = ["internal/sysinfo_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base", + "//absl/synchronization", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "low_level_alloc_test", + size = "medium", + srcs = ["internal/low_level_alloc_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["no_test_ios_x86_64"], + deps = [ + ":malloc_internal", + "//absl/container:node_hash_map", + ], +) + +cc_test( + name = "thread_identity_test", + size = "small", + srcs = ["internal/thread_identity_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":base", + ":core_headers", + "//absl/synchronization", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "thread_identity_benchmark", + srcs = ["internal/thread_identity_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":base", + "//absl/synchronization", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "bits", + hdrs = ["internal/bits.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ":core_headers", + ], +) + +cc_test( + name = "bits_test", + size = "small", + srcs = ["internal/bits_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "exponential_biased", + srcs = ["internal/exponential_biased.cc"], + hdrs = ["internal/exponential_biased.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ":core_headers", + ], +) + +cc_test( + name = "exponential_biased_test", + size = "small", + srcs = ["internal/exponential_biased_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":exponential_biased", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "periodic_sampler", + srcs = ["internal/periodic_sampler.cc"], + hdrs = ["internal/periodic_sampler.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":core_headers", + ":exponential_biased", + ], +) + +cc_test( + name = "periodic_sampler_test", + size = "small", + srcs = ["internal/periodic_sampler_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":core_headers", + ":periodic_sampler", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "periodic_sampler_benchmark", + testonly = 1, + srcs = ["internal/periodic_sampler_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":core_headers", + ":periodic_sampler", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "scoped_set_env", + testonly = 1, + srcs = ["internal/scoped_set_env.cc"], + hdrs = ["internal/scoped_set_env.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ":raw_logging_internal", + ], +) + +cc_test( + name = "scoped_set_env_test", + size = "small", + srcs = ["internal/scoped_set_env_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":scoped_set_env", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "log_severity_test", + size = "small", + srcs = ["log_severity_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":log_severity", + "//absl/flags:flag_internal", + "//absl/flags:marshalling", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "strerror", + srcs = ["internal/strerror.cc"], + hdrs = ["internal/strerror.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ":core_headers", + ":errno_saver", + ], +) + +cc_test( + name = "strerror_test", + size = "small", + srcs = ["internal/strerror_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":strerror", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "strerror_benchmark", + testonly = 1, + srcs = ["internal/strerror_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strerror", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "fast_type_id", + hdrs = ["internal/fast_type_id.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ], +) + +cc_test( + name = "fast_type_id_test", + size = "small", + srcs = ["internal/fast_type_id_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":fast_type_id", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "unique_small_name_test", + size = "small", + srcs = ["internal/unique_small_name_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + linkstatic = 1, + deps = [ + ":core_headers", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "optimization_test", + size = "small", + srcs = ["optimization_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":core_headers", + "//absl/types:optional", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/base/CMakeLists.txt b/third_party/abseil_cpp/absl/base/CMakeLists.txt new file mode 100644 index 000000000000..9ff5aa243ca1 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/CMakeLists.txt @@ -0,0 +1,717 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +find_library(LIBRT rt) + +absl_cc_library( + NAME + atomic_hook + HDRS + "internal/atomic_hook.h" + DEPS + absl::config + absl::core_headers + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + errno_saver + HDRS + "internal/errno_saver.h" + DEPS + absl::config + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + log_severity + HDRS + "log_severity.h" + SRCS + "log_severity.cc" + DEPS + absl::core_headers + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + raw_logging_internal + HDRS + "internal/raw_logging.h" + SRCS + "internal/raw_logging.cc" + DEPS + absl::atomic_hook + absl::config + absl::core_headers + absl::log_severity + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + spinlock_wait + HDRS + "internal/spinlock_wait.h" + SRCS + "internal/spinlock_akaros.inc" + "internal/spinlock_linux.inc" + "internal/spinlock_posix.inc" + "internal/spinlock_wait.cc" + "internal/spinlock_win32.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::core_headers + absl::errno_saver +) + +absl_cc_library( + NAME + config + HDRS + "config.h" + "options.h" + "policy_checks.h" + COPTS + ${ABSL_DEFAULT_COPTS} + PUBLIC +) + +absl_cc_library( + NAME + dynamic_annotations + HDRS + "dynamic_annotations.h" + SRCS + "internal/dynamic_annotations.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + core_headers + HDRS + "attributes.h" + "const_init.h" + "macros.h" + "optimization.h" + "port.h" + "thread_annotations.h" + "internal/thread_annotations.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + malloc_internal + HDRS + "internal/direct_mmap.h" + "internal/low_level_alloc.h" + SRCS + "internal/low_level_alloc.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::raw_logging_internal + Threads::Threads +) + +absl_cc_library( + NAME + base_internal + HDRS + "internal/hide_ptr.h" + "internal/identity.h" + "internal/inline_variable.h" + "internal/invoke.h" + "internal/scheduling_mode.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::type_traits +) + +absl_cc_library( + NAME + base + HDRS + "call_once.h" + "casts.h" + "internal/cycleclock.h" + "internal/low_level_scheduling.h" + "internal/per_thread_tls.h" + "internal/spinlock.h" + "internal/sysinfo.h" + "internal/thread_identity.h" + "internal/tsan_mutex_interface.h" + "internal/unscaledcycleclock.h" + SRCS + "internal/cycleclock.cc" + "internal/spinlock.cc" + "internal/sysinfo.cc" + "internal/thread_identity.cc" + "internal/unscaledcycleclock.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + $<$<BOOL:${LIBRT}>:-lrt> + $<$<BOOL:${MINGW}>:"advapi32"> + DEPS + absl::atomic_hook + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::log_severity + absl::raw_logging_internal + absl::spinlock_wait + absl::type_traits + Threads::Threads + PUBLIC +) + +absl_cc_library( + NAME + throw_delegate + HDRS + "internal/throw_delegate.h" + SRCS + "internal/throw_delegate.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +absl_cc_library( + NAME + exception_testing + HDRS + "internal/exception_testing.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + gtest + TESTONLY +) + +absl_cc_library( + NAME + pretty_function + HDRS + "internal/pretty_function.h" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + exception_safety_testing + HDRS + "internal/exception_safety_testing.h" + SRCS + "internal/exception_safety_testing.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::pretty_function + absl::memory + absl::meta + absl::strings + absl::utility + gtest + TESTONLY +) + +absl_cc_test( + NAME + absl_exception_safety_testing_test + SRCS + "exception_safety_testing_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::exception_safety_testing + absl::memory + gtest_main +) + +absl_cc_library( + NAME + atomic_hook_test_helper + SRCS + "internal/atomic_hook_test_helper.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::atomic_hook + absl::core_headers + TESTONLY +) + +absl_cc_test( + NAME + atomic_hook_test + SRCS + "internal/atomic_hook_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::atomic_hook_test_helper + absl::atomic_hook + absl::core_headers + gmock + gtest_main +) + +absl_cc_test( + NAME + bit_cast_test + SRCS + "bit_cast_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::core_headers + gtest_main +) + +absl_cc_test( + NAME + errno_saver_test + SRCS + "internal/errno_saver_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::errno_saver + absl::strerror + gmock + gtest_main +) + +absl_cc_test( + NAME + throw_delegate_test + SRCS + "throw_delegate_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::config + absl::throw_delegate + gtest_main +) + +absl_cc_test( + NAME + inline_variable_test + SRCS + "internal/inline_variable_testing.h" + "inline_variable_test.cc" + "inline_variable_test_a.cc" + "inline_variable_test_b.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base_internal + gtest_main +) + +absl_cc_test( + NAME + invoke_test + SRCS + "invoke_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base_internal + absl::memory + absl::strings + gmock + gtest_main +) + +absl_cc_library( + NAME + spinlock_test_common + SRCS + "spinlock_test_common.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::config + absl::base_internal + absl::core_headers + absl::synchronization + gtest + TESTONLY +) + +# On bazel BUILD this target use "alwayslink = 1" which is not implemented here +absl_cc_test( + NAME + spinlock_test + SRCS + "spinlock_test_common.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::synchronization + gtest_main +) + +absl_cc_library( + NAME + endian + HDRS + "internal/endian.h" + "internal/unaligned_access.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + PUBLIC +) + +absl_cc_test( + NAME + endian_test + SRCS + "internal/endian_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::config + absl::endian + gtest_main +) + +absl_cc_test( + NAME + config_test + SRCS + "config_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::synchronization + gtest_main +) + +absl_cc_test( + NAME + call_once_test + SRCS + "call_once_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::core_headers + absl::synchronization + gtest_main +) + +absl_cc_test( + NAME + raw_logging_test + SRCS + "raw_logging_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::raw_logging_internal + absl::strings + gtest_main +) + +absl_cc_test( + NAME + sysinfo_test + SRCS + "internal/sysinfo_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::synchronization + gtest_main +) + +absl_cc_test( + NAME + low_level_alloc_test + SRCS + "internal/low_level_alloc_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::malloc_internal + absl::node_hash_map + Threads::Threads +) + +absl_cc_test( + NAME + thread_identity_test + SRCS + "internal/thread_identity_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::core_headers + absl::synchronization + Threads::Threads + gtest_main +) + +absl_cc_library( + NAME + bits + HDRS + "internal/bits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_test( + NAME + bits_test + SRCS + "internal/bits_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::bits + gtest_main +) + +absl_cc_library( + NAME + exponential_biased + SRCS + "internal/exponential_biased.cc" + HDRS + "internal/exponential_biased.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_test( + NAME + exponential_biased_test + SRCS + "internal/exponential_biased_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::exponential_biased + absl::strings + gmock_main +) + +absl_cc_library( + NAME + periodic_sampler + SRCS + "internal/periodic_sampler.cc" + HDRS + "internal/periodic_sampler.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::exponential_biased +) + +absl_cc_test( + NAME + periodic_sampler_test + SRCS + "internal/periodic_sampler_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::core_headers + absl::periodic_sampler + gmock_main +) + +absl_cc_library( + NAME + scoped_set_env + SRCS + "internal/scoped_set_env.cc" + HDRS + "internal/scoped_set_env.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +absl_cc_test( + NAME + scoped_set_env_test + SRCS + "internal/scoped_set_env_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::scoped_set_env + gtest_main +) + +absl_cc_test( + NAME + cmake_thread_test + SRCS + "internal/cmake_thread_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base +) + +absl_cc_test( + NAME + log_severity_test + SRCS + "log_severity_test.cc" + DEPS + absl::flags_internal + absl::flags_marshalling + absl::log_severity + absl::strings + gmock + gtest_main +) + +absl_cc_library( + NAME + strerror + SRCS + "internal/strerror.cc" + HDRS + "internal/strerror.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::errno_saver +) + +absl_cc_test( + NAME + strerror_test + SRCS + "internal/strerror_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strerror + absl::strings + gmock + gtest_main +) + +absl_cc_library( + NAME + fast_type_id + HDRS + "internal/fast_type_id.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_test( + NAME + fast_type_id_test + SRCS + "internal/fast_type_id_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::fast_type_id + gtest_main +) + +absl_cc_test( + NAME + optimization_test + SRCS + "optimization_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::core_headers + absl::optional + gtest_main +) diff --git a/third_party/abseil_cpp/absl/base/attributes.h b/third_party/abseil_cpp/absl/base/attributes.h new file mode 100644 index 000000000000..f1d3cfe4d188 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/attributes.h @@ -0,0 +1,683 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This header file defines macros for declaring attributes for functions, +// types, and variables. +// +// These macros are used within Abseil and allow the compiler to optimize, where +// applicable, certain function calls. +// +// This file is used for both C and C++! +// +// Most macros here are exposing GCC or Clang features, and are stubbed out for +// other compilers. +// +// GCC attributes documentation: +// https://gcc.gnu.org/onlinedocs/gcc-4.7.0/gcc/Function-Attributes.html +// https://gcc.gnu.org/onlinedocs/gcc-4.7.0/gcc/Variable-Attributes.html +// https://gcc.gnu.org/onlinedocs/gcc-4.7.0/gcc/Type-Attributes.html +// +// Most attributes in this file are already supported by GCC 4.7. However, some +// of them are not supported in older version of Clang. Thus, we check +// `__has_attribute()` first. If the check fails, we check if we are on GCC and +// assume the attribute exists on GCC (which is verified on GCC 4.7). + +#ifndef ABSL_BASE_ATTRIBUTES_H_ +#define ABSL_BASE_ATTRIBUTES_H_ + +#include "absl/base/config.h" + +// ABSL_HAVE_ATTRIBUTE +// +// A function-like feature checking macro that is a wrapper around +// `__has_attribute`, which is defined by GCC 5+ and Clang and evaluates to a +// nonzero constant integer if the attribute is supported or 0 if not. +// +// It evaluates to zero if `__has_attribute` is not defined by the compiler. +// +// GCC: https://gcc.gnu.org/gcc-5/changes.html +// Clang: https://clang.llvm.org/docs/LanguageExtensions.html +#ifdef __has_attribute +#define ABSL_HAVE_ATTRIBUTE(x) __has_attribute(x) +#else +#define ABSL_HAVE_ATTRIBUTE(x) 0 +#endif + +// ABSL_HAVE_CPP_ATTRIBUTE +// +// A function-like feature checking macro that accepts C++11 style attributes. +// It's a wrapper around `__has_cpp_attribute`, defined by ISO C++ SD-6 +// (https://en.cppreference.com/w/cpp/experimental/feature_test). If we don't +// find `__has_cpp_attribute`, will evaluate to 0. +#if defined(__cplusplus) && defined(__has_cpp_attribute) +// NOTE: requiring __cplusplus above should not be necessary, but +// works around https://bugs.llvm.org/show_bug.cgi?id=23435. +#define ABSL_HAVE_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +#define ABSL_HAVE_CPP_ATTRIBUTE(x) 0 +#endif + +// ----------------------------------------------------------------------------- +// Function Attributes +// ----------------------------------------------------------------------------- +// +// GCC: https://gcc.gnu.org/onlinedocs/gcc/Function-Attributes.html +// Clang: https://clang.llvm.org/docs/AttributeReference.html + +// ABSL_PRINTF_ATTRIBUTE +// ABSL_SCANF_ATTRIBUTE +// +// Tells the compiler to perform `printf` format string checking if the +// compiler supports it; see the 'format' attribute in +// <https://gcc.gnu.org/onlinedocs/gcc-4.7.0/gcc/Function-Attributes.html>. +// +// Note: As the GCC manual states, "[s]ince non-static C++ methods +// have an implicit 'this' argument, the arguments of such methods +// should be counted from two, not one." +#if ABSL_HAVE_ATTRIBUTE(format) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_PRINTF_ATTRIBUTE(string_index, first_to_check) \ + __attribute__((__format__(__printf__, string_index, first_to_check))) +#define ABSL_SCANF_ATTRIBUTE(string_index, first_to_check) \ + __attribute__((__format__(__scanf__, string_index, first_to_check))) +#else +#define ABSL_PRINTF_ATTRIBUTE(string_index, first_to_check) +#define ABSL_SCANF_ATTRIBUTE(string_index, first_to_check) +#endif + +// ABSL_ATTRIBUTE_ALWAYS_INLINE +// ABSL_ATTRIBUTE_NOINLINE +// +// Forces functions to either inline or not inline. Introduced in gcc 3.1. +#if ABSL_HAVE_ATTRIBUTE(always_inline) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_ALWAYS_INLINE __attribute__((always_inline)) +#define ABSL_HAVE_ATTRIBUTE_ALWAYS_INLINE 1 +#else +#define ABSL_ATTRIBUTE_ALWAYS_INLINE +#endif + +#if ABSL_HAVE_ATTRIBUTE(noinline) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_NOINLINE __attribute__((noinline)) +#define ABSL_HAVE_ATTRIBUTE_NOINLINE 1 +#else +#define ABSL_ATTRIBUTE_NOINLINE +#endif + +// ABSL_ATTRIBUTE_NO_TAIL_CALL +// +// Prevents the compiler from optimizing away stack frames for functions which +// end in a call to another function. +#if ABSL_HAVE_ATTRIBUTE(disable_tail_calls) +#define ABSL_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 +#define ABSL_ATTRIBUTE_NO_TAIL_CALL __attribute__((disable_tail_calls)) +#elif defined(__GNUC__) && !defined(__clang__) +#define ABSL_HAVE_ATTRIBUTE_NO_TAIL_CALL 1 +#define ABSL_ATTRIBUTE_NO_TAIL_CALL \ + __attribute__((optimize("no-optimize-sibling-calls"))) +#else +#define ABSL_ATTRIBUTE_NO_TAIL_CALL +#define ABSL_HAVE_ATTRIBUTE_NO_TAIL_CALL 0 +#endif + +// ABSL_ATTRIBUTE_WEAK +// +// Tags a function as weak for the purposes of compilation and linking. +// Weak attributes currently do not work properly in LLVM's Windows backend, +// so disable them there. See https://bugs.llvm.org/show_bug.cgi?id=37598 +// for further information. +// The MinGW compiler doesn't complain about the weak attribute until the link +// step, presumably because Windows doesn't use ELF binaries. +#if (ABSL_HAVE_ATTRIBUTE(weak) || \ + (defined(__GNUC__) && !defined(__clang__))) && \ + !(defined(__llvm__) && defined(_WIN32)) && !defined(__MINGW32__) +#undef ABSL_ATTRIBUTE_WEAK +#define ABSL_ATTRIBUTE_WEAK __attribute__((weak)) +#define ABSL_HAVE_ATTRIBUTE_WEAK 1 +#else +#define ABSL_ATTRIBUTE_WEAK +#define ABSL_HAVE_ATTRIBUTE_WEAK 0 +#endif + +// ABSL_ATTRIBUTE_NONNULL +// +// Tells the compiler either (a) that a particular function parameter +// should be a non-null pointer, or (b) that all pointer arguments should +// be non-null. +// +// Note: As the GCC manual states, "[s]ince non-static C++ methods +// have an implicit 'this' argument, the arguments of such methods +// should be counted from two, not one." +// +// Args are indexed starting at 1. +// +// For non-static class member functions, the implicit `this` argument +// is arg 1, and the first explicit argument is arg 2. For static class member +// functions, there is no implicit `this`, and the first explicit argument is +// arg 1. +// +// Example: +// +// /* arg_a cannot be null, but arg_b can */ +// void Function(void* arg_a, void* arg_b) ABSL_ATTRIBUTE_NONNULL(1); +// +// class C { +// /* arg_a cannot be null, but arg_b can */ +// void Method(void* arg_a, void* arg_b) ABSL_ATTRIBUTE_NONNULL(2); +// +// /* arg_a cannot be null, but arg_b can */ +// static void StaticMethod(void* arg_a, void* arg_b) +// ABSL_ATTRIBUTE_NONNULL(1); +// }; +// +// If no arguments are provided, then all pointer arguments should be non-null. +// +// /* No pointer arguments may be null. */ +// void Function(void* arg_a, void* arg_b, int arg_c) ABSL_ATTRIBUTE_NONNULL(); +// +// NOTE: The GCC nonnull attribute actually accepts a list of arguments, but +// ABSL_ATTRIBUTE_NONNULL does not. +#if ABSL_HAVE_ATTRIBUTE(nonnull) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_NONNULL(arg_index) __attribute__((nonnull(arg_index))) +#else +#define ABSL_ATTRIBUTE_NONNULL(...) +#endif + +// ABSL_ATTRIBUTE_NORETURN +// +// Tells the compiler that a given function never returns. +#if ABSL_HAVE_ATTRIBUTE(noreturn) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_NORETURN __attribute__((noreturn)) +#elif defined(_MSC_VER) +#define ABSL_ATTRIBUTE_NORETURN __declspec(noreturn) +#else +#define ABSL_ATTRIBUTE_NORETURN +#endif + +// ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS +// +// Tells the AddressSanitizer (or other memory testing tools) to ignore a given +// function. Useful for cases when a function reads random locations on stack, +// calls _exit from a cloned subprocess, deliberately accesses buffer +// out of bounds or does other scary things with memory. +// NOTE: GCC supports AddressSanitizer(asan) since 4.8. +// https://gcc.gnu.org/gcc-4.8/changes.html +#if ABSL_HAVE_ATTRIBUTE(no_sanitize_address) +#define ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#else +#define ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS +#endif + +// ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY +// +// Tells the MemorySanitizer to relax the handling of a given function. All "Use +// of uninitialized value" warnings from such functions will be suppressed, and +// all values loaded from memory will be considered fully initialized. This +// attribute is similar to the ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS attribute +// above, but deals with initialized-ness rather than addressability issues. +// NOTE: MemorySanitizer(msan) is supported by Clang but not GCC. +#if ABSL_HAVE_ATTRIBUTE(no_sanitize_memory) +#define ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +#else +#define ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY +#endif + +// ABSL_ATTRIBUTE_NO_SANITIZE_THREAD +// +// Tells the ThreadSanitizer to not instrument a given function. +// NOTE: GCC supports ThreadSanitizer(tsan) since 4.8. +// https://gcc.gnu.org/gcc-4.8/changes.html +#if ABSL_HAVE_ATTRIBUTE(no_sanitize_thread) +#define ABSL_ATTRIBUTE_NO_SANITIZE_THREAD __attribute__((no_sanitize_thread)) +#else +#define ABSL_ATTRIBUTE_NO_SANITIZE_THREAD +#endif + +// ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED +// +// Tells the UndefinedSanitizer to ignore a given function. Useful for cases +// where certain behavior (eg. division by zero) is being used intentionally. +// NOTE: GCC supports UndefinedBehaviorSanitizer(ubsan) since 4.9. +// https://gcc.gnu.org/gcc-4.9/changes.html +#if ABSL_HAVE_ATTRIBUTE(no_sanitize_undefined) +#define ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED \ + __attribute__((no_sanitize_undefined)) +#elif ABSL_HAVE_ATTRIBUTE(no_sanitize) +#define ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED \ + __attribute__((no_sanitize("undefined"))) +#else +#define ABSL_ATTRIBUTE_NO_SANITIZE_UNDEFINED +#endif + +// ABSL_ATTRIBUTE_NO_SANITIZE_CFI +// +// Tells the ControlFlowIntegrity sanitizer to not instrument a given function. +// See https://clang.llvm.org/docs/ControlFlowIntegrity.html for details. +#if ABSL_HAVE_ATTRIBUTE(no_sanitize) +#define ABSL_ATTRIBUTE_NO_SANITIZE_CFI __attribute__((no_sanitize("cfi"))) +#else +#define ABSL_ATTRIBUTE_NO_SANITIZE_CFI +#endif + +// ABSL_ATTRIBUTE_NO_SANITIZE_SAFESTACK +// +// Tells the SafeStack to not instrument a given function. +// See https://clang.llvm.org/docs/SafeStack.html for details. +#if ABSL_HAVE_ATTRIBUTE(no_sanitize) +#define ABSL_ATTRIBUTE_NO_SANITIZE_SAFESTACK \ + __attribute__((no_sanitize("safe-stack"))) +#else +#define ABSL_ATTRIBUTE_NO_SANITIZE_SAFESTACK +#endif + +// ABSL_ATTRIBUTE_RETURNS_NONNULL +// +// Tells the compiler that a particular function never returns a null pointer. +#if ABSL_HAVE_ATTRIBUTE(returns_nonnull) || \ + (defined(__GNUC__) && \ + (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \ + !defined(__clang__)) +#define ABSL_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull)) +#else +#define ABSL_ATTRIBUTE_RETURNS_NONNULL +#endif + +// ABSL_HAVE_ATTRIBUTE_SECTION +// +// Indicates whether labeled sections are supported. Weak symbol support is +// a prerequisite. Labeled sections are not supported on Darwin/iOS. +#ifdef ABSL_HAVE_ATTRIBUTE_SECTION +#error ABSL_HAVE_ATTRIBUTE_SECTION cannot be directly set +#elif (ABSL_HAVE_ATTRIBUTE(section) || \ + (defined(__GNUC__) && !defined(__clang__))) && \ + !defined(__APPLE__) && ABSL_HAVE_ATTRIBUTE_WEAK +#define ABSL_HAVE_ATTRIBUTE_SECTION 1 + +// ABSL_ATTRIBUTE_SECTION +// +// Tells the compiler/linker to put a given function into a section and define +// `__start_ ## name` and `__stop_ ## name` symbols to bracket the section. +// This functionality is supported by GNU linker. Any function annotated with +// `ABSL_ATTRIBUTE_SECTION` must not be inlined, or it will be placed into +// whatever section its caller is placed into. +// +#ifndef ABSL_ATTRIBUTE_SECTION +#define ABSL_ATTRIBUTE_SECTION(name) \ + __attribute__((section(#name))) __attribute__((noinline)) +#endif + + +// ABSL_ATTRIBUTE_SECTION_VARIABLE +// +// Tells the compiler/linker to put a given variable into a section and define +// `__start_ ## name` and `__stop_ ## name` symbols to bracket the section. +// This functionality is supported by GNU linker. +#ifndef ABSL_ATTRIBUTE_SECTION_VARIABLE +#define ABSL_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name))) +#endif + +// ABSL_DECLARE_ATTRIBUTE_SECTION_VARS +// +// A weak section declaration to be used as a global declaration +// for ABSL_ATTRIBUTE_SECTION_START|STOP(name) to compile and link +// even without functions with ABSL_ATTRIBUTE_SECTION(name). +// ABSL_DEFINE_ATTRIBUTE_SECTION should be in the exactly one file; it's +// a no-op on ELF but not on Mach-O. +// +#ifndef ABSL_DECLARE_ATTRIBUTE_SECTION_VARS +#define ABSL_DECLARE_ATTRIBUTE_SECTION_VARS(name) \ + extern char __start_##name[] ABSL_ATTRIBUTE_WEAK; \ + extern char __stop_##name[] ABSL_ATTRIBUTE_WEAK +#endif +#ifndef ABSL_DEFINE_ATTRIBUTE_SECTION_VARS +#define ABSL_INIT_ATTRIBUTE_SECTION_VARS(name) +#define ABSL_DEFINE_ATTRIBUTE_SECTION_VARS(name) +#endif + +// ABSL_ATTRIBUTE_SECTION_START +// +// Returns `void*` pointers to start/end of a section of code with +// functions having ABSL_ATTRIBUTE_SECTION(name). +// Returns 0 if no such functions exist. +// One must ABSL_DECLARE_ATTRIBUTE_SECTION_VARS(name) for this to compile and +// link. +// +#define ABSL_ATTRIBUTE_SECTION_START(name) \ + (reinterpret_cast<void *>(__start_##name)) +#define ABSL_ATTRIBUTE_SECTION_STOP(name) \ + (reinterpret_cast<void *>(__stop_##name)) + +#else // !ABSL_HAVE_ATTRIBUTE_SECTION + +#define ABSL_HAVE_ATTRIBUTE_SECTION 0 + +// provide dummy definitions +#define ABSL_ATTRIBUTE_SECTION(name) +#define ABSL_ATTRIBUTE_SECTION_VARIABLE(name) +#define ABSL_INIT_ATTRIBUTE_SECTION_VARS(name) +#define ABSL_DEFINE_ATTRIBUTE_SECTION_VARS(name) +#define ABSL_DECLARE_ATTRIBUTE_SECTION_VARS(name) +#define ABSL_ATTRIBUTE_SECTION_START(name) (reinterpret_cast<void *>(0)) +#define ABSL_ATTRIBUTE_SECTION_STOP(name) (reinterpret_cast<void *>(0)) + +#endif // ABSL_ATTRIBUTE_SECTION + +// ABSL_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC +// +// Support for aligning the stack on 32-bit x86. +#if ABSL_HAVE_ATTRIBUTE(force_align_arg_pointer) || \ + (defined(__GNUC__) && !defined(__clang__)) +#if defined(__i386__) +#define ABSL_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC \ + __attribute__((force_align_arg_pointer)) +#define ABSL_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) +#elif defined(__x86_64__) +#define ABSL_REQUIRE_STACK_ALIGN_TRAMPOLINE (1) +#define ABSL_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC +#else // !__i386__ && !__x86_64 +#define ABSL_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) +#define ABSL_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC +#endif // __i386__ +#else +#define ABSL_ATTRIBUTE_STACK_ALIGN_FOR_OLD_LIBC +#define ABSL_REQUIRE_STACK_ALIGN_TRAMPOLINE (0) +#endif + +// ABSL_MUST_USE_RESULT +// +// Tells the compiler to warn about unused results. +// +// When annotating a function, it must appear as the first part of the +// declaration or definition. The compiler will warn if the return value from +// such a function is unused: +// +// ABSL_MUST_USE_RESULT Sprocket* AllocateSprocket(); +// AllocateSprocket(); // Triggers a warning. +// +// When annotating a class, it is equivalent to annotating every function which +// returns an instance. +// +// class ABSL_MUST_USE_RESULT Sprocket {}; +// Sprocket(); // Triggers a warning. +// +// Sprocket MakeSprocket(); +// MakeSprocket(); // Triggers a warning. +// +// Note that references and pointers are not instances: +// +// Sprocket* SprocketPointer(); +// SprocketPointer(); // Does *not* trigger a warning. +// +// ABSL_MUST_USE_RESULT allows using cast-to-void to suppress the unused result +// warning. For that, warn_unused_result is used only for clang but not for gcc. +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66425 +// +// Note: past advice was to place the macro after the argument list. +#if ABSL_HAVE_ATTRIBUTE(nodiscard) +#define ABSL_MUST_USE_RESULT [[nodiscard]] +#elif defined(__clang__) && ABSL_HAVE_ATTRIBUTE(warn_unused_result) +#define ABSL_MUST_USE_RESULT __attribute__((warn_unused_result)) +#else +#define ABSL_MUST_USE_RESULT +#endif + +// ABSL_ATTRIBUTE_HOT, ABSL_ATTRIBUTE_COLD +// +// Tells GCC that a function is hot or cold. GCC can use this information to +// improve static analysis, i.e. a conditional branch to a cold function +// is likely to be not-taken. +// This annotation is used for function declarations. +// +// Example: +// +// int foo() ABSL_ATTRIBUTE_HOT; +#if ABSL_HAVE_ATTRIBUTE(hot) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_HOT __attribute__((hot)) +#else +#define ABSL_ATTRIBUTE_HOT +#endif + +#if ABSL_HAVE_ATTRIBUTE(cold) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_COLD __attribute__((cold)) +#else +#define ABSL_ATTRIBUTE_COLD +#endif + +// ABSL_XRAY_ALWAYS_INSTRUMENT, ABSL_XRAY_NEVER_INSTRUMENT, ABSL_XRAY_LOG_ARGS +// +// We define the ABSL_XRAY_ALWAYS_INSTRUMENT and ABSL_XRAY_NEVER_INSTRUMENT +// macro used as an attribute to mark functions that must always or never be +// instrumented by XRay. Currently, this is only supported in Clang/LLVM. +// +// For reference on the LLVM XRay instrumentation, see +// http://llvm.org/docs/XRay.html. +// +// A function with the XRAY_ALWAYS_INSTRUMENT macro attribute in its declaration +// will always get the XRay instrumentation sleds. These sleds may introduce +// some binary size and runtime overhead and must be used sparingly. +// +// These attributes only take effect when the following conditions are met: +// +// * The file/target is built in at least C++11 mode, with a Clang compiler +// that supports XRay attributes. +// * The file/target is built with the -fxray-instrument flag set for the +// Clang/LLVM compiler. +// * The function is defined in the translation unit (the compiler honors the +// attribute in either the definition or the declaration, and must match). +// +// There are cases when, even when building with XRay instrumentation, users +// might want to control specifically which functions are instrumented for a +// particular build using special-case lists provided to the compiler. These +// special case lists are provided to Clang via the +// -fxray-always-instrument=... and -fxray-never-instrument=... flags. The +// attributes in source take precedence over these special-case lists. +// +// To disable the XRay attributes at build-time, users may define +// ABSL_NO_XRAY_ATTRIBUTES. Do NOT define ABSL_NO_XRAY_ATTRIBUTES on specific +// packages/targets, as this may lead to conflicting definitions of functions at +// link-time. +// +// XRay isn't currently supported on Android: +// https://github.com/android/ndk/issues/368 +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::xray_always_instrument) && \ + !defined(ABSL_NO_XRAY_ATTRIBUTES) && !defined(__ANDROID__) +#define ABSL_XRAY_ALWAYS_INSTRUMENT [[clang::xray_always_instrument]] +#define ABSL_XRAY_NEVER_INSTRUMENT [[clang::xray_never_instrument]] +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::xray_log_args) +#define ABSL_XRAY_LOG_ARGS(N) \ + [[clang::xray_always_instrument, clang::xray_log_args(N)]] +#else +#define ABSL_XRAY_LOG_ARGS(N) [[clang::xray_always_instrument]] +#endif +#else +#define ABSL_XRAY_ALWAYS_INSTRUMENT +#define ABSL_XRAY_NEVER_INSTRUMENT +#define ABSL_XRAY_LOG_ARGS(N) +#endif + +// ABSL_ATTRIBUTE_REINITIALIZES +// +// Indicates that a member function reinitializes the entire object to a known +// state, independent of the previous state of the object. +// +// The clang-tidy check bugprone-use-after-move allows member functions marked +// with this attribute to be called on objects that have been moved from; +// without the attribute, this would result in a use-after-move warning. +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::reinitializes) +#define ABSL_ATTRIBUTE_REINITIALIZES [[clang::reinitializes]] +#else +#define ABSL_ATTRIBUTE_REINITIALIZES +#endif + +// ----------------------------------------------------------------------------- +// Variable Attributes +// ----------------------------------------------------------------------------- + +// ABSL_ATTRIBUTE_UNUSED +// +// Prevents the compiler from complaining about variables that appear unused. +#if ABSL_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) +#undef ABSL_ATTRIBUTE_UNUSED +#define ABSL_ATTRIBUTE_UNUSED __attribute__((__unused__)) +#else +#define ABSL_ATTRIBUTE_UNUSED +#endif + +// ABSL_ATTRIBUTE_INITIAL_EXEC +// +// Tells the compiler to use "initial-exec" mode for a thread-local variable. +// See http://people.redhat.com/drepper/tls.pdf for the gory details. +#if ABSL_HAVE_ATTRIBUTE(tls_model) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_INITIAL_EXEC __attribute__((tls_model("initial-exec"))) +#else +#define ABSL_ATTRIBUTE_INITIAL_EXEC +#endif + +// ABSL_ATTRIBUTE_PACKED +// +// Instructs the compiler not to use natural alignment for a tagged data +// structure, but instead to reduce its alignment to 1. This attribute can +// either be applied to members of a structure or to a structure in its +// entirety. Applying this attribute (judiciously) to a structure in its +// entirety to optimize the memory footprint of very commonly-used structs is +// fine. Do not apply this attribute to a structure in its entirety if the +// purpose is to control the offsets of the members in the structure. Instead, +// apply this attribute only to structure members that need it. +// +// When applying ABSL_ATTRIBUTE_PACKED only to specific structure members the +// natural alignment of structure members not annotated is preserved. Aligned +// member accesses are faster than non-aligned member accesses even if the +// targeted microprocessor supports non-aligned accesses. +#if ABSL_HAVE_ATTRIBUTE(packed) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_PACKED __attribute__((__packed__)) +#else +#define ABSL_ATTRIBUTE_PACKED +#endif + +// ABSL_ATTRIBUTE_FUNC_ALIGN +// +// Tells the compiler to align the function start at least to certain +// alignment boundary +#if ABSL_HAVE_ATTRIBUTE(aligned) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_ATTRIBUTE_FUNC_ALIGN(bytes) __attribute__((aligned(bytes))) +#else +#define ABSL_ATTRIBUTE_FUNC_ALIGN(bytes) +#endif + +// ABSL_FALLTHROUGH_INTENDED +// +// Annotates implicit fall-through between switch labels, allowing a case to +// indicate intentional fallthrough and turn off warnings about any lack of a +// `break` statement. The ABSL_FALLTHROUGH_INTENDED macro should be followed by +// a semicolon and can be used in most places where `break` can, provided that +// no statements exist between it and the next switch label. +// +// Example: +// +// switch (x) { +// case 40: +// case 41: +// if (truth_is_out_there) { +// ++x; +// ABSL_FALLTHROUGH_INTENDED; // Use instead of/along with annotations +// // in comments +// } else { +// return x; +// } +// case 42: +// ... +// +// Notes: when compiled with clang in C++11 mode, the ABSL_FALLTHROUGH_INTENDED +// macro is expanded to the [[clang::fallthrough]] attribute, which is analysed +// when performing switch labels fall-through diagnostic +// (`-Wimplicit-fallthrough`). See clang documentation on language extensions +// for details: +// https://clang.llvm.org/docs/AttributeReference.html#fallthrough-clang-fallthrough +// +// When used with unsupported compilers, the ABSL_FALLTHROUGH_INTENDED macro +// has no effect on diagnostics. In any case this macro has no effect on runtime +// behavior and performance of code. + +#ifdef ABSL_FALLTHROUGH_INTENDED +#error "ABSL_FALLTHROUGH_INTENDED should not be defined." +#endif + +// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported. +#if defined(__clang__) && defined(__has_warning) +#if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#define ABSL_FALLTHROUGH_INTENDED [[clang::fallthrough]] +#endif +#elif defined(__GNUC__) && __GNUC__ >= 7 +#define ABSL_FALLTHROUGH_INTENDED [[gnu::fallthrough]] +#endif + +#ifndef ABSL_FALLTHROUGH_INTENDED +#define ABSL_FALLTHROUGH_INTENDED \ + do { \ + } while (0) +#endif + +// ABSL_DEPRECATED() +// +// Marks a deprecated class, struct, enum, function, method and variable +// declarations. The macro argument is used as a custom diagnostic message (e.g. +// suggestion of a better alternative). +// +// Examples: +// +// class ABSL_DEPRECATED("Use Bar instead") Foo {...}; +// +// ABSL_DEPRECATED("Use Baz() instead") void Bar() {...} +// +// template <typename T> +// ABSL_DEPRECATED("Use DoThat() instead") +// void DoThis(); +// +// Every usage of a deprecated entity will trigger a warning when compiled with +// clang's `-Wdeprecated-declarations` option. This option is turned off by +// default, but the warnings will be reported by clang-tidy. +#if defined(__clang__) && __cplusplus >= 201103L +#define ABSL_DEPRECATED(message) __attribute__((deprecated(message))) +#endif + +#ifndef ABSL_DEPRECATED +#define ABSL_DEPRECATED(message) +#endif + +// ABSL_CONST_INIT +// +// A variable declaration annotated with the `ABSL_CONST_INIT` attribute will +// not compile (on supported platforms) unless the variable has a constant +// initializer. This is useful for variables with static and thread storage +// duration, because it guarantees that they will not suffer from the so-called +// "static init order fiasco". Prefer to put this attribute on the most visible +// declaration of the variable, if there's more than one, because code that +// accesses the variable can then use the attribute for optimization. +// +// Example: +// +// class MyClass { +// public: +// ABSL_CONST_INIT static MyType my_var; +// }; +// +// MyType MyClass::my_var = MakeMyType(...); +// +// Note that this attribute is redundant if the variable is declared constexpr. +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) +#define ABSL_CONST_INIT [[clang::require_constant_initialization]] +#else +#define ABSL_CONST_INIT +#endif // ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) + +#endif // ABSL_BASE_ATTRIBUTES_H_ diff --git a/third_party/abseil_cpp/absl/base/bit_cast_test.cc b/third_party/abseil_cpp/absl/base/bit_cast_test.cc new file mode 100644 index 000000000000..8a3a41ea0231 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/bit_cast_test.cc @@ -0,0 +1,109 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit test for bit_cast template. + +#include <cstdint> +#include <cstring> + +#include "gtest/gtest.h" +#include "absl/base/casts.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +template <int N> +struct marshall { char buf[N]; }; + +template <typename T> +void TestMarshall(const T values[], int num_values) { + for (int i = 0; i < num_values; ++i) { + T t0 = values[i]; + marshall<sizeof(T)> m0 = absl::bit_cast<marshall<sizeof(T)> >(t0); + T t1 = absl::bit_cast<T>(m0); + marshall<sizeof(T)> m1 = absl::bit_cast<marshall<sizeof(T)> >(t1); + ASSERT_EQ(0, memcmp(&t0, &t1, sizeof(T))); + ASSERT_EQ(0, memcmp(&m0, &m1, sizeof(T))); + } +} + +// Convert back and forth to an integral type. The C++ standard does +// not guarantee this will work, but we test that this works on all the +// platforms we support. +// +// Likewise, we below make assumptions about sizeof(float) and +// sizeof(double) which the standard does not guarantee, but which hold on the +// platforms we support. + +template <typename T, typename I> +void TestIntegral(const T values[], int num_values) { + for (int i = 0; i < num_values; ++i) { + T t0 = values[i]; + I i0 = absl::bit_cast<I>(t0); + T t1 = absl::bit_cast<T>(i0); + I i1 = absl::bit_cast<I>(t1); + ASSERT_EQ(0, memcmp(&t0, &t1, sizeof(T))); + ASSERT_EQ(i0, i1); + } +} + +TEST(BitCast, Bool) { + static const bool bool_list[] = { false, true }; + TestMarshall<bool>(bool_list, ABSL_ARRAYSIZE(bool_list)); +} + +TEST(BitCast, Int32) { + static const int32_t int_list[] = + { 0, 1, 100, 2147483647, -1, -100, -2147483647, -2147483647-1 }; + TestMarshall<int32_t>(int_list, ABSL_ARRAYSIZE(int_list)); +} + +TEST(BitCast, Int64) { + static const int64_t int64_list[] = + { 0, 1, 1LL << 40, -1, -(1LL<<40) }; + TestMarshall<int64_t>(int64_list, ABSL_ARRAYSIZE(int64_list)); +} + +TEST(BitCast, Uint64) { + static const uint64_t uint64_list[] = + { 0, 1, 1LLU << 40, 1LLU << 63 }; + TestMarshall<uint64_t>(uint64_list, ABSL_ARRAYSIZE(uint64_list)); +} + +TEST(BitCast, Float) { + static const float float_list[] = + { 0.0f, 1.0f, -1.0f, 10.0f, -10.0f, + 1e10f, 1e20f, 1e-10f, 1e-20f, + 2.71828f, 3.14159f }; + TestMarshall<float>(float_list, ABSL_ARRAYSIZE(float_list)); + TestIntegral<float, int>(float_list, ABSL_ARRAYSIZE(float_list)); + TestIntegral<float, unsigned>(float_list, ABSL_ARRAYSIZE(float_list)); +} + +TEST(BitCast, Double) { + static const double double_list[] = + { 0.0, 1.0, -1.0, 10.0, -10.0, + 1e10, 1e100, 1e-10, 1e-100, + 2.718281828459045, + 3.141592653589793238462643383279502884197169399375105820974944 }; + TestMarshall<double>(double_list, ABSL_ARRAYSIZE(double_list)); + TestIntegral<double, int64_t>(double_list, ABSL_ARRAYSIZE(double_list)); + TestIntegral<double, uint64_t>(double_list, ABSL_ARRAYSIZE(double_list)); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/call_once.h b/third_party/abseil_cpp/absl/base/call_once.h new file mode 100644 index 000000000000..5b468af855f1 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/call_once.h @@ -0,0 +1,226 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: call_once.h +// ----------------------------------------------------------------------------- +// +// This header file provides an Abseil version of `std::call_once` for invoking +// a given function at most once, across all threads. This Abseil version is +// faster than the C++11 version and incorporates the C++17 argument-passing +// fix, so that (for example) non-const references may be passed to the invoked +// function. + +#ifndef ABSL_BASE_CALL_ONCE_H_ +#define ABSL_BASE_CALL_ONCE_H_ + +#include <algorithm> +#include <atomic> +#include <cstdint> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/invoke.h" +#include "absl/base/internal/low_level_scheduling.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/spinlock_wait.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class once_flag; + +namespace base_internal { +std::atomic<uint32_t>* ControlWord(absl::once_flag* flag); +} // namespace base_internal + +// call_once() +// +// For all invocations using a given `once_flag`, invokes a given `fn` exactly +// once across all threads. The first call to `call_once()` with a particular +// `once_flag` argument (that does not throw an exception) will run the +// specified function with the provided `args`; other calls with the same +// `once_flag` argument will not run the function, but will wait +// for the provided function to finish running (if it is still running). +// +// This mechanism provides a safe, simple, and fast mechanism for one-time +// initialization in a multi-threaded process. +// +// Example: +// +// class MyInitClass { +// public: +// ... +// mutable absl::once_flag once_; +// +// MyInitClass* init() const { +// absl::call_once(once_, &MyInitClass::Init, this); +// return ptr_; +// } +// +template <typename Callable, typename... Args> +void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args); + +// once_flag +// +// Objects of this type are used to distinguish calls to `call_once()` and +// ensure the provided function is only invoked once across all threads. This +// type is not copyable or movable. However, it has a `constexpr` +// constructor, and is safe to use as a namespace-scoped global variable. +class once_flag { + public: + constexpr once_flag() : control_(0) {} + once_flag(const once_flag&) = delete; + once_flag& operator=(const once_flag&) = delete; + + private: + friend std::atomic<uint32_t>* base_internal::ControlWord(once_flag* flag); + std::atomic<uint32_t> control_; +}; + +//------------------------------------------------------------------------------ +// End of public interfaces. +// Implementation details follow. +//------------------------------------------------------------------------------ + +namespace base_internal { + +// Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to +// initialize entities used by the scheduler implementation. +template <typename Callable, typename... Args> +void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args); + +// Disables scheduling while on stack when scheduling mode is non-cooperative. +// No effect for cooperative scheduling modes. +class SchedulingHelper { + public: + explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) { + if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) { + guard_result_ = base_internal::SchedulingGuard::DisableRescheduling(); + } + } + + ~SchedulingHelper() { + if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) { + base_internal::SchedulingGuard::EnableRescheduling(guard_result_); + } + } + + private: + base_internal::SchedulingMode mode_; + bool guard_result_; +}; + +// Bit patterns for call_once state machine values. Internal implementation +// detail, not for use by clients. +// +// The bit patterns are arbitrarily chosen from unlikely values, to aid in +// debugging. However, kOnceInit must be 0, so that a zero-initialized +// once_flag will be valid for immediate use. +enum { + kOnceInit = 0, + kOnceRunning = 0x65C2937B, + kOnceWaiter = 0x05A308D2, + // A very small constant is chosen for kOnceDone so that it fit in a single + // compare with immediate instruction for most common ISAs. This is verified + // for x86, POWER and ARM. + kOnceDone = 221, // Random Number +}; + +template <typename Callable, typename... Args> +ABSL_ATTRIBUTE_NOINLINE +void CallOnceImpl(std::atomic<uint32_t>* control, + base_internal::SchedulingMode scheduling_mode, Callable&& fn, + Args&&... args) { +#ifndef NDEBUG + { + uint32_t old_control = control->load(std::memory_order_relaxed); + if (old_control != kOnceInit && + old_control != kOnceRunning && + old_control != kOnceWaiter && + old_control != kOnceDone) { + ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx", + static_cast<unsigned long>(old_control)); // NOLINT + } + } +#endif // NDEBUG + static const base_internal::SpinLockWaitTransition trans[] = { + {kOnceInit, kOnceRunning, true}, + {kOnceRunning, kOnceWaiter, false}, + {kOnceDone, kOnceDone, true}}; + + // Must do this before potentially modifying control word's state. + base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode); + // Short circuit the simplest case to avoid procedure call overhead. + // The base_internal::SpinLockWait() call returns either kOnceInit or + // kOnceDone. If it returns kOnceDone, it must have loaded the control word + // with std::memory_order_acquire and seen a value of kOnceDone. + uint32_t old_control = kOnceInit; + if (control->compare_exchange_strong(old_control, kOnceRunning, + std::memory_order_relaxed) || + base_internal::SpinLockWait(control, ABSL_ARRAYSIZE(trans), trans, + scheduling_mode) == kOnceInit) { + base_internal::invoke(std::forward<Callable>(fn), + std::forward<Args>(args)...); + // The call to SpinLockWake below is an optimization, because the waiter + // in SpinLockWait is waiting with a short timeout. The atomic load/store + // sequence is slightly faster than an atomic exchange: + // old_control = control->exchange(base_internal::kOnceDone, + // std::memory_order_release); + // We opt for a slightly faster case when there are no waiters, in spite + // of longer tail latency when there are waiters. + old_control = control->load(std::memory_order_relaxed); + control->store(base_internal::kOnceDone, std::memory_order_release); + if (old_control == base_internal::kOnceWaiter) { + base_internal::SpinLockWake(control, true); + } + } // else *control is already kOnceDone +} + +inline std::atomic<uint32_t>* ControlWord(once_flag* flag) { + return &flag->control_; +} + +template <typename Callable, typename... Args> +void LowLevelCallOnce(absl::once_flag* flag, Callable&& fn, Args&&... args) { + std::atomic<uint32_t>* once = base_internal::ControlWord(flag); + uint32_t s = once->load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { + base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY, + std::forward<Callable>(fn), + std::forward<Args>(args)...); + } +} + +} // namespace base_internal + +template <typename Callable, typename... Args> +void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) { + std::atomic<uint32_t>* once = base_internal::ControlWord(&flag); + uint32_t s = once->load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { + base_internal::CallOnceImpl( + once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL, + std::forward<Callable>(fn), std::forward<Args>(args)...); + } +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_CALL_ONCE_H_ diff --git a/third_party/abseil_cpp/absl/base/call_once_test.cc b/third_party/abseil_cpp/absl/base/call_once_test.cc new file mode 100644 index 000000000000..11d26c44d1a4 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/call_once_test.cc @@ -0,0 +1,107 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/call_once.h" + +#include <thread> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/const_init.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +absl::once_flag once; + +ABSL_CONST_INIT Mutex counters_mu(absl::kConstInit); + +int running_thread_count ABSL_GUARDED_BY(counters_mu) = 0; +int call_once_invoke_count ABSL_GUARDED_BY(counters_mu) = 0; +int call_once_finished_count ABSL_GUARDED_BY(counters_mu) = 0; +int call_once_return_count ABSL_GUARDED_BY(counters_mu) = 0; +bool done_blocking ABSL_GUARDED_BY(counters_mu) = false; + +// Function to be called from absl::call_once. Waits for a notification. +void WaitAndIncrement() { + counters_mu.Lock(); + ++call_once_invoke_count; + counters_mu.Unlock(); + + counters_mu.LockWhen(Condition(&done_blocking)); + ++call_once_finished_count; + counters_mu.Unlock(); +} + +void ThreadBody() { + counters_mu.Lock(); + ++running_thread_count; + counters_mu.Unlock(); + + absl::call_once(once, WaitAndIncrement); + + counters_mu.Lock(); + ++call_once_return_count; + counters_mu.Unlock(); +} + +// Returns true if all threads are set up for the test. +bool ThreadsAreSetup(void*) ABSL_EXCLUSIVE_LOCKS_REQUIRED(counters_mu) { + // All ten threads must be running, and WaitAndIncrement should be blocked. + return running_thread_count == 10 && call_once_invoke_count == 1; +} + +TEST(CallOnceTest, ExecutionCount) { + std::vector<std::thread> threads; + + // Start 10 threads all calling call_once on the same once_flag. + for (int i = 0; i < 10; ++i) { + threads.emplace_back(ThreadBody); + } + + + // Wait until all ten threads have started, and WaitAndIncrement has been + // invoked. + counters_mu.LockWhen(Condition(ThreadsAreSetup, nullptr)); + + // WaitAndIncrement should have been invoked by exactly one call_once() + // instance. That thread should be blocking on a notification, and all other + // call_once instances should be blocking as well. + EXPECT_EQ(call_once_invoke_count, 1); + EXPECT_EQ(call_once_finished_count, 0); + EXPECT_EQ(call_once_return_count, 0); + + // Allow WaitAndIncrement to finish executing. Once it does, the other + // call_once waiters will be unblocked. + done_blocking = true; + counters_mu.Unlock(); + + for (std::thread& thread : threads) { + thread.join(); + } + + counters_mu.Lock(); + EXPECT_EQ(call_once_invoke_count, 1); + EXPECT_EQ(call_once_finished_count, 1); + EXPECT_EQ(call_once_return_count, 10); + counters_mu.Unlock(); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/casts.h b/third_party/abseil_cpp/absl/base/casts.h new file mode 100644 index 000000000000..83c691265ff3 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/casts.h @@ -0,0 +1,187 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: casts.h +// ----------------------------------------------------------------------------- +// +// This header file defines casting templates to fit use cases not covered by +// the standard casts provided in the C++ standard. As with all cast operations, +// use these with caution and only if alternatives do not exist. + +#ifndef ABSL_BASE_CASTS_H_ +#define ABSL_BASE_CASTS_H_ + +#include <cstring> +#include <memory> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/identity.h" +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace internal_casts { + +template <class Dest, class Source> +struct is_bitcastable + : std::integral_constant< + bool, + sizeof(Dest) == sizeof(Source) && + type_traits_internal::is_trivially_copyable<Source>::value && + type_traits_internal::is_trivially_copyable<Dest>::value && + std::is_default_constructible<Dest>::value> {}; + +} // namespace internal_casts + +// implicit_cast() +// +// Performs an implicit conversion between types following the language +// rules for implicit conversion; if an implicit conversion is otherwise +// allowed by the language in the given context, this function performs such an +// implicit conversion. +// +// Example: +// +// // If the context allows implicit conversion: +// From from; +// To to = from; +// +// // Such code can be replaced by: +// implicit_cast<To>(from); +// +// An `implicit_cast()` may also be used to annotate numeric type conversions +// that, although safe, may produce compiler warnings (such as `long` to `int`). +// Additionally, an `implicit_cast()` is also useful within return statements to +// indicate a specific implicit conversion is being undertaken. +// +// Example: +// +// return implicit_cast<double>(size_in_bytes) / capacity_; +// +// Annotating code with `implicit_cast()` allows you to explicitly select +// particular overloads and template instantiations, while providing a safer +// cast than `reinterpret_cast()` or `static_cast()`. +// +// Additionally, an `implicit_cast()` can be used to allow upcasting within a +// type hierarchy where incorrect use of `static_cast()` could accidentally +// allow downcasting. +// +// Finally, an `implicit_cast()` can be used to perform implicit conversions +// from unrelated types that otherwise couldn't be implicitly cast directly; +// C++ will normally only implicitly cast "one step" in such conversions. +// +// That is, if C is a type which can be implicitly converted to B, with B being +// a type that can be implicitly converted to A, an `implicit_cast()` can be +// used to convert C to B (which the compiler can then implicitly convert to A +// using language rules). +// +// Example: +// +// // Assume an object C is convertible to B, which is implicitly convertible +// // to A +// A a = implicit_cast<B>(C); +// +// Such implicit cast chaining may be useful within template logic. +template <typename To> +constexpr To implicit_cast(typename absl::internal::identity_t<To> to) { + return to; +} + +// bit_cast() +// +// Performs a bitwise cast on a type without changing the underlying bit +// representation of that type's value. The two types must be of the same size +// and both types must be trivially copyable. As with most casts, use with +// caution. A `bit_cast()` might be needed when you need to temporarily treat a +// type as some other type, such as in the following cases: +// +// * Serialization (casting temporarily to `char *` for those purposes is +// always allowed by the C++ standard) +// * Managing the individual bits of a type within mathematical operations +// that are not normally accessible through that type +// * Casting non-pointer types to pointer types (casting the other way is +// allowed by `reinterpret_cast()` but round-trips cannot occur the other +// way). +// +// Example: +// +// float f = 3.14159265358979; +// int i = bit_cast<int32_t>(f); +// // i = 0x40490fdb +// +// Casting non-pointer types to pointer types and then dereferencing them +// traditionally produces undefined behavior. +// +// Example: +// +// // WRONG +// float f = 3.14159265358979; // WRONG +// int i = * reinterpret_cast<int*>(&f); // WRONG +// +// The address-casting method produces undefined behavior according to the ISO +// C++ specification section [basic.lval]. Roughly, this section says: if an +// object in memory has one type, and a program accesses it with a different +// type, the result is undefined behavior for most values of "different type". +// +// Such casting results in type punning: holding an object in memory of one type +// and reading its bits back using a different type. A `bit_cast()` avoids this +// issue by implementing its casts using `memcpy()`, which avoids introducing +// this undefined behavior. +// +// NOTE: The requirements here are more strict than the bit_cast of standard +// proposal p0476 due to the need for workarounds and lack of intrinsics. +// Specifically, this implementation also requires `Dest` to be +// default-constructible. +template < + typename Dest, typename Source, + typename std::enable_if<internal_casts::is_bitcastable<Dest, Source>::value, + int>::type = 0> +inline Dest bit_cast(const Source& source) { + Dest dest; + memcpy(static_cast<void*>(std::addressof(dest)), + static_cast<const void*>(std::addressof(source)), sizeof(dest)); + return dest; +} + +// NOTE: This overload is only picked if the requirements of bit_cast are +// not met. It is therefore UB, but is provided temporarily as previous +// versions of this function template were unchecked. Do not use this in +// new code. +template < + typename Dest, typename Source, + typename std::enable_if< + !internal_casts::is_bitcastable<Dest, Source>::value, + int>::type = 0> +ABSL_DEPRECATED( + "absl::bit_cast type requirements were violated. Update the types " + "being used such that they are the same size and are both " + "TriviallyCopyable.") +inline Dest bit_cast(const Source& source) { + static_assert(sizeof(Dest) == sizeof(Source), + "Source and destination types should have equal sizes."); + + Dest dest; + memcpy(&dest, &source, sizeof(dest)); + return dest; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_CASTS_H_ diff --git a/third_party/abseil_cpp/absl/base/config.h b/third_party/abseil_cpp/absl/base/config.h new file mode 100644 index 000000000000..3f7f32b9e6e3 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/config.h @@ -0,0 +1,714 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: config.h +// ----------------------------------------------------------------------------- +// +// This header file defines a set of macros for checking the presence of +// important compiler and platform features. Such macros can be used to +// produce portable code by parameterizing compilation based on the presence or +// lack of a given feature. +// +// We define a "feature" as some interface we wish to program to: for example, +// a library function or system call. A value of `1` indicates support for +// that feature; any other value indicates the feature support is undefined. +// +// Example: +// +// Suppose a programmer wants to write a program that uses the 'mmap()' system +// call. The Abseil macro for that feature (`ABSL_HAVE_MMAP`) allows you to +// selectively include the `mmap.h` header and bracket code using that feature +// in the macro: +// +// #include "absl/base/config.h" +// +// #ifdef ABSL_HAVE_MMAP +// #include "sys/mman.h" +// #endif //ABSL_HAVE_MMAP +// +// ... +// #ifdef ABSL_HAVE_MMAP +// void *ptr = mmap(...); +// ... +// #endif // ABSL_HAVE_MMAP + +#ifndef ABSL_BASE_CONFIG_H_ +#define ABSL_BASE_CONFIG_H_ + +// Included for the __GLIBC__ macro (or similar macros on other systems). +#include <limits.h> + +#ifdef __cplusplus +// Included for __GLIBCXX__, _LIBCPP_VERSION +#include <cstddef> +#endif // __cplusplus + +#if defined(__APPLE__) +// Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, +// __IPHONE_8_0. +#include <Availability.h> +#include <TargetConditionals.h> +#endif + +#include "absl/base/options.h" +#include "absl/base/policy_checks.h" + +// Helper macro to convert a CPP variable to a string literal. +#define ABSL_INTERNAL_DO_TOKEN_STR(x) #x +#define ABSL_INTERNAL_TOKEN_STR(x) ABSL_INTERNAL_DO_TOKEN_STR(x) + +// ----------------------------------------------------------------------------- +// Abseil namespace annotations +// ----------------------------------------------------------------------------- + +// ABSL_NAMESPACE_BEGIN/ABSL_NAMESPACE_END +// +// An annotation placed at the beginning/end of each `namespace absl` scope. +// This is used to inject an inline namespace. +// +// The proper way to write Abseil code in the `absl` namespace is: +// +// namespace absl { +// ABSL_NAMESPACE_BEGIN +// +// void Foo(); // absl::Foo(). +// +// ABSL_NAMESPACE_END +// } // namespace absl +// +// Users of Abseil should not use these macros, because users of Abseil should +// not write `namespace absl {` in their own code for any reason. (Abseil does +// not support forward declarations of its own types, nor does it support +// user-provided specialization of Abseil templates. Code that violates these +// rules may be broken without warning.) +#if !defined(ABSL_OPTION_USE_INLINE_NAMESPACE) || \ + !defined(ABSL_OPTION_INLINE_NAMESPACE_NAME) +#error options.h is misconfigured. +#endif + +// Check that ABSL_OPTION_INLINE_NAMESPACE_NAME is neither "head" nor "" +#if defined(__cplusplus) && ABSL_OPTION_USE_INLINE_NAMESPACE == 1 + +#define ABSL_INTERNAL_INLINE_NAMESPACE_STR \ + ABSL_INTERNAL_TOKEN_STR(ABSL_OPTION_INLINE_NAMESPACE_NAME) + +static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != '\0', + "options.h misconfigured: ABSL_OPTION_INLINE_NAMESPACE_NAME must " + "not be empty."); +static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || + ABSL_INTERNAL_INLINE_NAMESPACE_STR[1] != 'e' || + ABSL_INTERNAL_INLINE_NAMESPACE_STR[2] != 'a' || + ABSL_INTERNAL_INLINE_NAMESPACE_STR[3] != 'd' || + ABSL_INTERNAL_INLINE_NAMESPACE_STR[4] != '\0', + "options.h misconfigured: ABSL_OPTION_INLINE_NAMESPACE_NAME must " + "be changed to a new, unique identifier name."); + +#endif + +#if ABSL_OPTION_USE_INLINE_NAMESPACE == 0 +#define ABSL_NAMESPACE_BEGIN +#define ABSL_NAMESPACE_END +#elif ABSL_OPTION_USE_INLINE_NAMESPACE == 1 +#define ABSL_NAMESPACE_BEGIN \ + inline namespace ABSL_OPTION_INLINE_NAMESPACE_NAME { +#define ABSL_NAMESPACE_END } +#else +#error options.h is misconfigured. +#endif + +// ----------------------------------------------------------------------------- +// Compiler Feature Checks +// ----------------------------------------------------------------------------- + +// ABSL_HAVE_BUILTIN() +// +// Checks whether the compiler supports a Clang Feature Checking Macro, and if +// so, checks whether it supports the provided builtin function "x" where x +// is one of the functions noted in +// https://clang.llvm.org/docs/LanguageExtensions.html +// +// Note: Use this macro to avoid an extra level of #ifdef __has_builtin check. +// http://releases.llvm.org/3.3/tools/clang/docs/LanguageExtensions.html +#ifdef __has_builtin +#define ABSL_HAVE_BUILTIN(x) __has_builtin(x) +#else +#define ABSL_HAVE_BUILTIN(x) 0 +#endif + +#if defined(__is_identifier) +#define ABSL_INTERNAL_HAS_KEYWORD(x) !(__is_identifier(x)) +#else +#define ABSL_INTERNAL_HAS_KEYWORD(x) 0 +#endif + +#ifdef __has_feature +#define ABSL_HAVE_FEATURE(f) __has_feature(f) +#else +#define ABSL_HAVE_FEATURE(f) 0 +#endif + +// ABSL_HAVE_TLS is defined to 1 when __thread should be supported. +// We assume __thread is supported on Linux when compiled with Clang or compiled +// against libstdc++ with _GLIBCXX_HAVE_TLS defined. +#ifdef ABSL_HAVE_TLS +#error ABSL_HAVE_TLS cannot be directly set +#elif defined(__linux__) && (defined(__clang__) || defined(_GLIBCXX_HAVE_TLS)) +#define ABSL_HAVE_TLS 1 +#endif + +// ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE +// +// Checks whether `std::is_trivially_destructible<T>` is supported. +// +// Notes: All supported compilers using libc++ support this feature, as does +// gcc >= 4.8.1 using libstdc++, and Visual Studio. +#ifdef ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE +#error ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set +#elif defined(_LIBCPP_VERSION) || \ + (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \ + (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \ + defined(_MSC_VER) +#define ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1 +#endif + +// ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE +// +// Checks whether `std::is_trivially_default_constructible<T>` and +// `std::is_trivially_copy_constructible<T>` are supported. + +// ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE +// +// Checks whether `std::is_trivially_copy_assignable<T>` is supported. + +// Notes: Clang with libc++ supports these features, as does gcc >= 5.1 with +// either libc++ or libstdc++, and Visual Studio (but not NVCC). +#if defined(ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) +#error ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set +#elif defined(ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE) +#error ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set +#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ + (!defined(__clang__) && defined(__GNUC__) && \ + (__GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ >= 4)) && \ + (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ + (defined(_MSC_VER) && !defined(__NVCC__)) +#define ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1 +#define ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1 +#endif + +// ABSL_HAVE_SOURCE_LOCATION_CURRENT +// +// Indicates whether `absl::SourceLocation::current()` will return useful +// information in some contexts. +#ifndef ABSL_HAVE_SOURCE_LOCATION_CURRENT +#if ABSL_INTERNAL_HAS_KEYWORD(__builtin_LINE) && \ + ABSL_INTERNAL_HAS_KEYWORD(__builtin_FILE) +#define ABSL_HAVE_SOURCE_LOCATION_CURRENT 1 +#endif +#endif + +// ABSL_HAVE_THREAD_LOCAL +// +// Checks whether C++11's `thread_local` storage duration specifier is +// supported. +#ifdef ABSL_HAVE_THREAD_LOCAL +#error ABSL_HAVE_THREAD_LOCAL cannot be directly set +#elif defined(__APPLE__) +// Notes: +// * Xcode's clang did not support `thread_local` until version 8, and +// even then not for all iOS < 9.0. +// * Xcode 9.3 started disallowing `thread_local` for 32-bit iOS simulator +// targeting iOS 9.x. +// * Xcode 10 moves the deployment target check for iOS < 9.0 to link time +// making ABSL_HAVE_FEATURE unreliable there. +// +#if ABSL_HAVE_FEATURE(cxx_thread_local) && \ + !(TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0) +#define ABSL_HAVE_THREAD_LOCAL 1 +#endif +#else // !defined(__APPLE__) +#define ABSL_HAVE_THREAD_LOCAL 1 +#endif + +// There are platforms for which TLS should not be used even though the compiler +// makes it seem like it's supported (Android NDK < r12b for example). +// This is primarily because of linker problems and toolchain misconfiguration: +// Abseil does not intend to support this indefinitely. Currently, the newest +// toolchain that we intend to support that requires this behavior is the +// r11 NDK - allowing for a 5 year support window on that means this option +// is likely to be removed around June of 2021. +// TLS isn't supported until NDK r12b per +// https://developer.android.com/ndk/downloads/revision_history.html +// Since NDK r16, `__NDK_MAJOR__` and `__NDK_MINOR__` are defined in +// <android/ndk-version.h>. For NDK < r16, users should define these macros, +// e.g. `-D__NDK_MAJOR__=11 -D__NKD_MINOR__=0` for NDK r11. +#if defined(__ANDROID__) && defined(__clang__) +#if __has_include(<android/ndk-version.h>) +#include <android/ndk-version.h> +#endif // __has_include(<android/ndk-version.h>) +#if defined(__ANDROID__) && defined(__clang__) && defined(__NDK_MAJOR__) && \ + defined(__NDK_MINOR__) && \ + ((__NDK_MAJOR__ < 12) || ((__NDK_MAJOR__ == 12) && (__NDK_MINOR__ < 1))) +#undef ABSL_HAVE_TLS +#undef ABSL_HAVE_THREAD_LOCAL +#endif +#endif // defined(__ANDROID__) && defined(__clang__) + +// ABSL_HAVE_INTRINSIC_INT128 +// +// Checks whether the __int128 compiler extension for a 128-bit integral type is +// supported. +// +// Note: __SIZEOF_INT128__ is defined by Clang and GCC when __int128 is +// supported, but we avoid using it in certain cases: +// * On Clang: +// * Building using Clang for Windows, where the Clang runtime library has +// 128-bit support only on LP64 architectures, but Windows is LLP64. +// * On Nvidia's nvcc: +// * nvcc also defines __GNUC__ and __SIZEOF_INT128__, but not all versions +// actually support __int128. +#ifdef ABSL_HAVE_INTRINSIC_INT128 +#error ABSL_HAVE_INTRINSIC_INT128 cannot be directly set +#elif defined(__SIZEOF_INT128__) +#if (defined(__clang__) && !defined(_WIN32)) || \ + (defined(__CUDACC__) && __CUDACC_VER_MAJOR__ >= 9) || \ + (defined(__GNUC__) && !defined(__clang__) && !defined(__CUDACC__)) +#define ABSL_HAVE_INTRINSIC_INT128 1 +#elif defined(__CUDACC__) +// __CUDACC_VER__ is a full version number before CUDA 9, and is defined to a +// string explaining that it has been removed starting with CUDA 9. We use +// nested #ifs because there is no short-circuiting in the preprocessor. +// NOTE: `__CUDACC__` could be undefined while `__CUDACC_VER__` is defined. +#if __CUDACC_VER__ >= 70000 +#define ABSL_HAVE_INTRINSIC_INT128 1 +#endif // __CUDACC_VER__ >= 70000 +#endif // defined(__CUDACC__) +#endif // ABSL_HAVE_INTRINSIC_INT128 + +// ABSL_HAVE_EXCEPTIONS +// +// Checks whether the compiler both supports and enables exceptions. Many +// compilers support a "no exceptions" mode that disables exceptions. +// +// Generally, when ABSL_HAVE_EXCEPTIONS is not defined: +// +// * Code using `throw` and `try` may not compile. +// * The `noexcept` specifier will still compile and behave as normal. +// * The `noexcept` operator may still return `false`. +// +// For further details, consult the compiler's documentation. +#ifdef ABSL_HAVE_EXCEPTIONS +#error ABSL_HAVE_EXCEPTIONS cannot be directly set. + +#elif defined(__clang__) + +#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) +// Clang >= 3.6 +#if ABSL_HAVE_FEATURE(cxx_exceptions) +#define ABSL_HAVE_EXCEPTIONS 1 +#endif // ABSL_HAVE_FEATURE(cxx_exceptions) +#else +// Clang < 3.6 +// http://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html#the-exceptions-macro +#if defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) +#define ABSL_HAVE_EXCEPTIONS 1 +#endif // defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) +#endif // __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) + +// Handle remaining special cases and default to exceptions being supported. +#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ + !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \ + !(defined(_MSC_VER) && !defined(_CPPUNWIND)) +#define ABSL_HAVE_EXCEPTIONS 1 +#endif + +// ----------------------------------------------------------------------------- +// Platform Feature Checks +// ----------------------------------------------------------------------------- + +// Currently supported operating systems and associated preprocessor +// symbols: +// +// Linux and Linux-derived __linux__ +// Android __ANDROID__ (implies __linux__) +// Linux (non-Android) __linux__ && !__ANDROID__ +// Darwin (macOS and iOS) __APPLE__ +// Akaros (http://akaros.org) __ros__ +// Windows _WIN32 +// NaCL __native_client__ +// AsmJS __asmjs__ +// WebAssembly __wasm__ +// Fuchsia __Fuchsia__ +// +// Note that since Android defines both __ANDROID__ and __linux__, one +// may probe for either Linux or Android by simply testing for __linux__. + +// ABSL_HAVE_MMAP +// +// Checks whether the platform has an mmap(2) implementation as defined in +// POSIX.1-2001. +#ifdef ABSL_HAVE_MMAP +#error ABSL_HAVE_MMAP cannot be directly set +#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ + defined(__ros__) || defined(__native_client__) || defined(__asmjs__) || \ + defined(__wasm__) || defined(__Fuchsia__) || defined(__sun) || \ + defined(__ASYLO__) || defined(__myriad2__) +#define ABSL_HAVE_MMAP 1 +#endif + +// ABSL_HAVE_PTHREAD_GETSCHEDPARAM +// +// Checks whether the platform implements the pthread_(get|set)schedparam(3) +// functions as defined in POSIX.1-2001. +#ifdef ABSL_HAVE_PTHREAD_GETSCHEDPARAM +#error ABSL_HAVE_PTHREAD_GETSCHEDPARAM cannot be directly set +#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ + defined(__ros__) +#define ABSL_HAVE_PTHREAD_GETSCHEDPARAM 1 +#endif + +// ABSL_HAVE_SCHED_YIELD +// +// Checks whether the platform implements sched_yield(2) as defined in +// POSIX.1-2001. +#ifdef ABSL_HAVE_SCHED_YIELD +#error ABSL_HAVE_SCHED_YIELD cannot be directly set +#elif defined(__linux__) || defined(__ros__) || defined(__native_client__) +#define ABSL_HAVE_SCHED_YIELD 1 +#endif + +// ABSL_HAVE_SEMAPHORE_H +// +// Checks whether the platform supports the <semaphore.h> header and sem_init(3) +// family of functions as standardized in POSIX.1-2001. +// +// Note: While Apple provides <semaphore.h> for both iOS and macOS, it is +// explicitly deprecated and will cause build failures if enabled for those +// platforms. We side-step the issue by not defining it here for Apple +// platforms. +#ifdef ABSL_HAVE_SEMAPHORE_H +#error ABSL_HAVE_SEMAPHORE_H cannot be directly set +#elif defined(__linux__) || defined(__ros__) +#define ABSL_HAVE_SEMAPHORE_H 1 +#endif + +// ABSL_HAVE_ALARM +// +// Checks whether the platform supports the <signal.h> header and alarm(2) +// function as standardized in POSIX.1-2001. +#ifdef ABSL_HAVE_ALARM +#error ABSL_HAVE_ALARM cannot be directly set +#elif defined(__GOOGLE_GRTE_VERSION__) +// feature tests for Google's GRTE +#define ABSL_HAVE_ALARM 1 +#elif defined(__GLIBC__) +// feature test for glibc +#define ABSL_HAVE_ALARM 1 +#elif defined(_MSC_VER) +// feature tests for Microsoft's library +#elif defined(__MINGW32__) +// mingw32 doesn't provide alarm(2): +// https://osdn.net/projects/mingw/scm/git/mingw-org-wsl/blobs/5.2-trunk/mingwrt/include/unistd.h +// mingw-w64 provides a no-op implementation: +// https://sourceforge.net/p/mingw-w64/mingw-w64/ci/master/tree/mingw-w64-crt/misc/alarm.c +#elif defined(__EMSCRIPTEN__) +// emscripten doesn't support signals +#elif defined(__Fuchsia__) +// Signals don't exist on fuchsia. +#elif defined(__native_client__) +#else +// other standard libraries +#define ABSL_HAVE_ALARM 1 +#endif + +// ABSL_IS_LITTLE_ENDIAN +// ABSL_IS_BIG_ENDIAN +// +// Checks the endianness of the platform. +// +// Notes: uses the built in endian macros provided by GCC (since 4.6) and +// Clang (since 3.2); see +// https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html. +// Otherwise, if _WIN32, assume little endian. Otherwise, bail with an error. +#if defined(ABSL_IS_BIG_ENDIAN) +#error "ABSL_IS_BIG_ENDIAN cannot be directly set." +#endif +#if defined(ABSL_IS_LITTLE_ENDIAN) +#error "ABSL_IS_LITTLE_ENDIAN cannot be directly set." +#endif + +#if (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +#define ABSL_IS_LITTLE_ENDIAN 1 +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \ + __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define ABSL_IS_BIG_ENDIAN 1 +#elif defined(_WIN32) +#define ABSL_IS_LITTLE_ENDIAN 1 +#else +#error "absl endian detection needs to be set up for your compiler" +#endif + +// macOS 10.13 and iOS 10.11 don't let you use <any>, <optional>, or <variant> +// even though the headers exist and are publicly noted to work. See +// https://github.com/abseil/abseil-cpp/issues/207 and +// https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes +// libc++ spells out the availability requirements in the file +// llvm-project/libcxx/include/__config via the #define +// _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS. +#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \ + ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400) || \ + (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 120000) || \ + (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000) || \ + (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 120000)) +#define ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE 1 +#else +#define ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE 0 +#endif + +// ABSL_HAVE_STD_ANY +// +// Checks whether C++17 std::any is available by checking whether <any> exists. +#ifdef ABSL_HAVE_STD_ANY +#error "ABSL_HAVE_STD_ANY cannot be directly set." +#endif + +#ifdef __has_include +#if __has_include(<any>) && __cplusplus >= 201703L && \ + !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE +#define ABSL_HAVE_STD_ANY 1 +#endif +#endif + +// ABSL_HAVE_STD_OPTIONAL +// +// Checks whether C++17 std::optional is available. +#ifdef ABSL_HAVE_STD_OPTIONAL +#error "ABSL_HAVE_STD_OPTIONAL cannot be directly set." +#endif + +#ifdef __has_include +#if __has_include(<optional>) && __cplusplus >= 201703L && \ + !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE +#define ABSL_HAVE_STD_OPTIONAL 1 +#endif +#endif + +// ABSL_HAVE_STD_VARIANT +// +// Checks whether C++17 std::variant is available. +#ifdef ABSL_HAVE_STD_VARIANT +#error "ABSL_HAVE_STD_VARIANT cannot be directly set." +#endif + +#ifdef __has_include +#if __has_include(<variant>) && __cplusplus >= 201703L && \ + !ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE +#define ABSL_HAVE_STD_VARIANT 1 +#endif +#endif + +// ABSL_HAVE_STD_STRING_VIEW +// +// Checks whether C++17 std::string_view is available. +#ifdef ABSL_HAVE_STD_STRING_VIEW +#error "ABSL_HAVE_STD_STRING_VIEW cannot be directly set." +#endif + +#ifdef __has_include +#if __has_include(<string_view>) && __cplusplus >= 201703L +#define ABSL_HAVE_STD_STRING_VIEW 1 +#endif +#endif + +// For MSVC, `__has_include` is supported in VS 2017 15.3, which is later than +// the support for <optional>, <any>, <string_view>, <variant>. So we use +// _MSC_VER to check whether we have VS 2017 RTM (when <optional>, <any>, +// <string_view>, <variant> is implemented) or higher. Also, `__cplusplus` is +// not correctly set by MSVC, so we use `_MSVC_LANG` to check the language +// version. +// TODO(zhangxy): fix tests before enabling aliasing for `std::any`. +#if defined(_MSC_VER) && _MSC_VER >= 1910 && \ + ((defined(_MSVC_LANG) && _MSVC_LANG > 201402) || __cplusplus > 201402) +// #define ABSL_HAVE_STD_ANY 1 +#define ABSL_HAVE_STD_OPTIONAL 1 +#define ABSL_HAVE_STD_VARIANT 1 +#define ABSL_HAVE_STD_STRING_VIEW 1 +#endif + +// ABSL_USES_STD_ANY +// +// Indicates whether absl::any is an alias for std::any. +#if !defined(ABSL_OPTION_USE_STD_ANY) +#error options.h is misconfigured. +#elif ABSL_OPTION_USE_STD_ANY == 0 || \ + (ABSL_OPTION_USE_STD_ANY == 2 && !defined(ABSL_HAVE_STD_ANY)) +#undef ABSL_USES_STD_ANY +#elif ABSL_OPTION_USE_STD_ANY == 1 || \ + (ABSL_OPTION_USE_STD_ANY == 2 && defined(ABSL_HAVE_STD_ANY)) +#define ABSL_USES_STD_ANY 1 +#else +#error options.h is misconfigured. +#endif + +// ABSL_USES_STD_OPTIONAL +// +// Indicates whether absl::optional is an alias for std::optional. +#if !defined(ABSL_OPTION_USE_STD_OPTIONAL) +#error options.h is misconfigured. +#elif ABSL_OPTION_USE_STD_OPTIONAL == 0 || \ + (ABSL_OPTION_USE_STD_OPTIONAL == 2 && !defined(ABSL_HAVE_STD_OPTIONAL)) +#undef ABSL_USES_STD_OPTIONAL +#elif ABSL_OPTION_USE_STD_OPTIONAL == 1 || \ + (ABSL_OPTION_USE_STD_OPTIONAL == 2 && defined(ABSL_HAVE_STD_OPTIONAL)) +#define ABSL_USES_STD_OPTIONAL 1 +#else +#error options.h is misconfigured. +#endif + +// ABSL_USES_STD_VARIANT +// +// Indicates whether absl::variant is an alias for std::variant. +#if !defined(ABSL_OPTION_USE_STD_VARIANT) +#error options.h is misconfigured. +#elif ABSL_OPTION_USE_STD_VARIANT == 0 || \ + (ABSL_OPTION_USE_STD_VARIANT == 2 && !defined(ABSL_HAVE_STD_VARIANT)) +#undef ABSL_USES_STD_VARIANT +#elif ABSL_OPTION_USE_STD_VARIANT == 1 || \ + (ABSL_OPTION_USE_STD_VARIANT == 2 && defined(ABSL_HAVE_STD_VARIANT)) +#define ABSL_USES_STD_VARIANT 1 +#else +#error options.h is misconfigured. +#endif + +// ABSL_USES_STD_STRING_VIEW +// +// Indicates whether absl::string_view is an alias for std::string_view. +#if !defined(ABSL_OPTION_USE_STD_STRING_VIEW) +#error options.h is misconfigured. +#elif ABSL_OPTION_USE_STD_STRING_VIEW == 0 || \ + (ABSL_OPTION_USE_STD_STRING_VIEW == 2 && \ + !defined(ABSL_HAVE_STD_STRING_VIEW)) +#undef ABSL_USES_STD_STRING_VIEW +#elif ABSL_OPTION_USE_STD_STRING_VIEW == 1 || \ + (ABSL_OPTION_USE_STD_STRING_VIEW == 2 && \ + defined(ABSL_HAVE_STD_STRING_VIEW)) +#define ABSL_USES_STD_STRING_VIEW 1 +#else +#error options.h is misconfigured. +#endif + +// In debug mode, MSVC 2017's std::variant throws a EXCEPTION_ACCESS_VIOLATION +// SEH exception from emplace for variant<SomeStruct> when constructing the +// struct can throw. This defeats some of variant_test and +// variant_exception_safety_test. +#if defined(_MSC_VER) && _MSC_VER >= 1700 && defined(_DEBUG) +#define ABSL_INTERNAL_MSVC_2017_DBG_MODE +#endif + +// ABSL_INTERNAL_MANGLED_NS +// ABSL_INTERNAL_MANGLED_BACKREFERENCE +// +// Internal macros for building up mangled names in our internal fork of CCTZ. +// This implementation detail is only needed and provided for the MSVC build. +// +// These macros both expand to string literals. ABSL_INTERNAL_MANGLED_NS is +// the mangled spelling of the `absl` namespace, and +// ABSL_INTERNAL_MANGLED_BACKREFERENCE is a back-reference integer representing +// the proper count to skip past the CCTZ fork namespace names. (This number +// is one larger when there is an inline namespace name to skip.) +#if defined(_MSC_VER) +#if ABSL_OPTION_USE_INLINE_NAMESPACE == 0 +#define ABSL_INTERNAL_MANGLED_NS "absl" +#define ABSL_INTERNAL_MANGLED_BACKREFERENCE "5" +#else +#define ABSL_INTERNAL_MANGLED_NS \ + ABSL_INTERNAL_TOKEN_STR(ABSL_OPTION_INLINE_NAMESPACE_NAME) "@absl" +#define ABSL_INTERNAL_MANGLED_BACKREFERENCE "6" +#endif +#endif + +#undef ABSL_INTERNAL_HAS_KEYWORD + +// ABSL_DLL +// +// When building Abseil as a DLL, this macro expands to `__declspec(dllexport)` +// so we can annotate symbols appropriately as being exported. When used in +// headers consuming a DLL, this macro expands to `__declspec(dllimport)` so +// that consumers know the symbol is defined inside the DLL. In all other cases, +// the macro expands to nothing. +#if defined(_MSC_VER) +#if defined(ABSL_BUILD_DLL) +#define ABSL_DLL __declspec(dllexport) +#elif defined(ABSL_CONSUME_DLL) +#define ABSL_DLL __declspec(dllimport) +#else +#define ABSL_DLL +#endif +#else +#define ABSL_DLL +#endif // defined(_MSC_VER) + +// ABSL_HAVE_MEMORY_SANITIZER +// +// MemorySanitizer (MSan) is a detector of uninitialized reads. It consists of +// a compiler instrumentation module and a run-time library. +#ifdef ABSL_HAVE_MEMORY_SANITIZER +#error "ABSL_HAVE_MEMORY_SANITIZER cannot be directly set." +#elif defined(MEMORY_SANITIZER) +// The MEMORY_SANITIZER macro is deprecated but we will continue to honor it +// for now. +#define ABSL_HAVE_MEMORY_SANITIZER 1 +#elif defined(__SANITIZE_MEMORY__) +#define ABSL_HAVE_MEMORY_SANITIZER 1 +#elif !defined(__native_client__) && ABSL_HAVE_FEATURE(memory_sanitizer) +#define ABSL_HAVE_MEMORY_SANITIZER 1 +#endif + +// ABSL_HAVE_THREAD_SANITIZER +// +// ThreadSanitizer (TSan) is a fast data race detector. +#ifdef ABSL_HAVE_THREAD_SANITIZER +#error "ABSL_HAVE_THREAD_SANITIZER cannot be directly set." +#elif defined(THREAD_SANITIZER) +// The THREAD_SANITIZER macro is deprecated but we will continue to honor it +// for now. +#define ABSL_HAVE_THREAD_SANITIZER 1 +#elif defined(__SANITIZE_THREAD__) +#define ABSL_HAVE_THREAD_SANITIZER 1 +#elif ABSL_HAVE_FEATURE(thread_sanitizer) +#define ABSL_HAVE_THREAD_SANITIZER 1 +#endif + +// ABSL_HAVE_ADDRESS_SANITIZER +// +// AddressSanitizer (ASan) is a fast memory error detector. +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +#error "ABSL_HAVE_ADDRESS_SANITIZER cannot be directly set." +#elif defined(ADDRESS_SANITIZER) +// The ADDRESS_SANITIZER macro is deprecated but we will continue to honor it +// for now. +#define ABSL_HAVE_ADDRESS_SANITIZER 1 +#elif defined(__SANITIZE_ADDRESS__) +#define ABSL_HAVE_ADDRESS_SANITIZER 1 +#elif ABSL_HAVE_FEATURE(address_sanitizer) +#define ABSL_HAVE_ADDRESS_SANITIZER 1 +#endif + +#endif // ABSL_BASE_CONFIG_H_ diff --git a/third_party/abseil_cpp/absl/base/config_test.cc b/third_party/abseil_cpp/absl/base/config_test.cc new file mode 100644 index 000000000000..7e0c033de59f --- /dev/null +++ b/third_party/abseil_cpp/absl/base/config_test.cc @@ -0,0 +1,60 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/config.h" + +#include <cstdint> + +#include "gtest/gtest.h" +#include "absl/synchronization/internal/thread_pool.h" + +namespace { + +TEST(ConfigTest, Endianness) { + union { + uint32_t value; + uint8_t data[sizeof(uint32_t)]; + } number; + number.data[0] = 0x00; + number.data[1] = 0x01; + number.data[2] = 0x02; + number.data[3] = 0x03; +#if defined(ABSL_IS_LITTLE_ENDIAN) && defined(ABSL_IS_BIG_ENDIAN) +#error Both ABSL_IS_LITTLE_ENDIAN and ABSL_IS_BIG_ENDIAN are defined +#elif defined(ABSL_IS_LITTLE_ENDIAN) + EXPECT_EQ(UINT32_C(0x03020100), number.value); +#elif defined(ABSL_IS_BIG_ENDIAN) + EXPECT_EQ(UINT32_C(0x00010203), number.value); +#else +#error Unknown endianness +#endif +} + +#if defined(ABSL_HAVE_THREAD_LOCAL) +TEST(ConfigTest, ThreadLocal) { + static thread_local int mine_mine_mine = 16; + EXPECT_EQ(16, mine_mine_mine); + { + absl::synchronization_internal::ThreadPool pool(1); + pool.Schedule([&] { + EXPECT_EQ(16, mine_mine_mine); + mine_mine_mine = 32; + EXPECT_EQ(32, mine_mine_mine); + }); + } + EXPECT_EQ(16, mine_mine_mine); +} +#endif + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/const_init.h b/third_party/abseil_cpp/absl/base/const_init.h new file mode 100644 index 000000000000..16520b61d95b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/const_init.h @@ -0,0 +1,76 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// kConstInit +// ----------------------------------------------------------------------------- +// +// A constructor tag used to mark an object as safe for use as a global +// variable, avoiding the usual lifetime issues that can affect globals. + +#ifndef ABSL_BASE_CONST_INIT_H_ +#define ABSL_BASE_CONST_INIT_H_ + +#include "absl/base/config.h" + +// In general, objects with static storage duration (such as global variables) +// can trigger tricky object lifetime situations. Attempting to access them +// from the constructors or destructors of other global objects can result in +// undefined behavior, unless their constructors and destructors are designed +// with this issue in mind. +// +// The normal way to deal with this issue in C++11 is to use constant +// initialization and trivial destructors. +// +// Constant initialization is guaranteed to occur before any other code +// executes. Constructors that are declared 'constexpr' are eligible for +// constant initialization. You can annotate a variable declaration with the +// ABSL_CONST_INIT macro to express this intent. For compilers that support +// it, this annotation will cause a compilation error for declarations that +// aren't subject to constant initialization (perhaps because a runtime value +// was passed as a constructor argument). +// +// On program shutdown, lifetime issues can be avoided on global objects by +// ensuring that they contain trivial destructors. A class has a trivial +// destructor unless it has a user-defined destructor, a virtual method or base +// class, or a data member or base class with a non-trivial destructor of its +// own. Objects with static storage duration and a trivial destructor are not +// cleaned up on program shutdown, and are thus safe to access from other code +// running during shutdown. +// +// For a few core Abseil classes, we make a best effort to allow for safe global +// instances, even though these classes have non-trivial destructors. These +// objects can be created with the absl::kConstInit tag. For example: +// ABSL_CONST_INIT absl::Mutex global_mutex(absl::kConstInit); +// +// The line above declares a global variable of type absl::Mutex which can be +// accessed at any point during startup or shutdown. global_mutex's destructor +// will still run, but will not invalidate the object. Note that C++ specifies +// that accessing an object after its destructor has run results in undefined +// behavior, but this pattern works on the toolchains we support. +// +// The absl::kConstInit tag should only be used to define objects with static +// or thread_local storage duration. + +namespace absl { +ABSL_NAMESPACE_BEGIN + +enum ConstInitType { + kConstInit, +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_CONST_INIT_H_ diff --git a/third_party/abseil_cpp/absl/base/dynamic_annotations.h b/third_party/abseil_cpp/absl/base/dynamic_annotations.h new file mode 100644 index 000000000000..545f8cbc916b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/dynamic_annotations.h @@ -0,0 +1,482 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines dynamic annotations for use with dynamic analysis tool +// such as valgrind, PIN, etc. +// +// Dynamic annotation is a source code annotation that affects the generated +// code (that is, the annotation is not a comment). Each such annotation is +// attached to a particular instruction and/or to a particular object (address) +// in the program. +// +// The annotations that should be used by users are macros in all upper-case +// (e.g., ABSL_ANNOTATE_THREAD_NAME). +// +// Actual implementation of these macros may differ depending on the dynamic +// analysis tool being used. +// +// This file supports the following configurations: +// - Dynamic Annotations enabled (with static thread-safety warnings disabled). +// In this case, macros expand to functions implemented by Thread Sanitizer, +// when building with TSan. When not provided an external implementation, +// dynamic_annotations.cc provides no-op implementations. +// +// - Static Clang thread-safety warnings enabled. +// When building with a Clang compiler that supports thread-safety warnings, +// a subset of annotations can be statically-checked at compile-time. We +// expand these macros to static-inline functions that can be analyzed for +// thread-safety, but afterwards elided when building the final binary. +// +// - All annotations are disabled. +// If neither Dynamic Annotations nor Clang thread-safety warnings are +// enabled, then all annotation-macros expand to empty. + +#ifndef ABSL_BASE_DYNAMIC_ANNOTATIONS_H_ +#define ABSL_BASE_DYNAMIC_ANNOTATIONS_H_ + +#include <stddef.h> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#ifdef __cplusplus +#include "absl/base/macros.h" +#endif + +// TODO(rogeeff): Remove after the backward compatibility period. +#include "absl/base/internal/dynamic_annotations.h" // IWYU pragma: export + +// ------------------------------------------------------------------------- +// Decide which features are enabled. + +#ifdef ABSL_HAVE_THREAD_SANITIZER + +#define ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED 1 +#define ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED 1 +#define ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED 1 +#define ABSL_INTERNAL_ANNOTALYSIS_ENABLED 0 +#define ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED 1 + +#else + +#define ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED 0 +#define ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED 0 +#define ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED 0 + +// Clang provides limited support for static thread-safety analysis through a +// feature called Annotalysis. We configure macro-definitions according to +// whether Annotalysis support is available. When running in opt-mode, GCC +// will issue a warning, if these attributes are compiled. Only include them +// when compiling using Clang. + +#if defined(__clang__) +#define ABSL_INTERNAL_ANNOTALYSIS_ENABLED 1 +#if !defined(SWIG) +#define ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED 1 +#endif +#else +#define ABSL_INTERNAL_ANNOTALYSIS_ENABLED 0 +#endif + +// Read/write annotations are enabled in Annotalysis mode; disabled otherwise. +#define ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED \ + ABSL_INTERNAL_ANNOTALYSIS_ENABLED + +#endif // ABSL_HAVE_THREAD_SANITIZER + +#ifdef __cplusplus +#define ABSL_INTERNAL_BEGIN_EXTERN_C extern "C" { +#define ABSL_INTERNAL_END_EXTERN_C } // extern "C" +#define ABSL_INTERNAL_GLOBAL_SCOPED(F) ::F +#define ABSL_INTERNAL_STATIC_INLINE inline +#else +#define ABSL_INTERNAL_BEGIN_EXTERN_C // empty +#define ABSL_INTERNAL_END_EXTERN_C // empty +#define ABSL_INTERNAL_GLOBAL_SCOPED(F) F +#define ABSL_INTERNAL_STATIC_INLINE static inline +#endif + +// ------------------------------------------------------------------------- +// Define race annotations. + +#if ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED == 1 + +// ------------------------------------------------------------- +// Annotations that suppress errors. It is usually better to express the +// program's synchronization using the other annotations, but these can be used +// when all else fails. + +// Report that we may have a benign race at `pointer`, with size +// "sizeof(*(pointer))". `pointer` must be a non-void* pointer. Insert at the +// point where `pointer` has been allocated, preferably close to the point +// where the race happens. See also ABSL_ANNOTATE_BENIGN_RACE_STATIC. +#define ABSL_ANNOTATE_BENIGN_RACE(pointer, description) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateBenignRaceSized) \ + (__FILE__, __LINE__, pointer, sizeof(*(pointer)), description) + +// Same as ABSL_ANNOTATE_BENIGN_RACE(`address`, `description`), but applies to +// the memory range [`address`, `address`+`size`). +#define ABSL_ANNOTATE_BENIGN_RACE_SIZED(address, size, description) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateBenignRaceSized) \ + (__FILE__, __LINE__, address, size, description) + +// Enable (`enable`!=0) or disable (`enable`==0) race detection for all threads. +// This annotation could be useful if you want to skip expensive race analysis +// during some period of program execution, e.g. during initialization. +#define ABSL_ANNOTATE_ENABLE_RACE_DETECTION(enable) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateEnableRaceDetection) \ + (__FILE__, __LINE__, enable) + +// ------------------------------------------------------------- +// Annotations useful for debugging. + +// Report the current thread `name` to a race detector. +#define ABSL_ANNOTATE_THREAD_NAME(name) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateThreadName)(__FILE__, __LINE__, name) + +// ------------------------------------------------------------- +// Annotations useful when implementing locks. They are not normally needed by +// modules that merely use locks. The `lock` argument is a pointer to the lock +// object. + +// Report that a lock has been created at address `lock`. +#define ABSL_ANNOTATE_RWLOCK_CREATE(lock) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockCreate)(__FILE__, __LINE__, lock) + +// Report that a linker initialized lock has been created at address `lock`. +#ifdef ABSL_HAVE_THREAD_SANITIZER +#define ABSL_ANNOTATE_RWLOCK_CREATE_STATIC(lock) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockCreateStatic) \ + (__FILE__, __LINE__, lock) +#else +#define ABSL_ANNOTATE_RWLOCK_CREATE_STATIC(lock) \ + ABSL_ANNOTATE_RWLOCK_CREATE(lock) +#endif + +// Report that the lock at address `lock` is about to be destroyed. +#define ABSL_ANNOTATE_RWLOCK_DESTROY(lock) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockDestroy)(__FILE__, __LINE__, lock) + +// Report that the lock at address `lock` has been acquired. +// `is_w`=1 for writer lock, `is_w`=0 for reader lock. +#define ABSL_ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockAcquired) \ + (__FILE__, __LINE__, lock, is_w) + +// Report that the lock at address `lock` is about to be released. +// `is_w`=1 for writer lock, `is_w`=0 for reader lock. +#define ABSL_ANNOTATE_RWLOCK_RELEASED(lock, is_w) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockReleased) \ + (__FILE__, __LINE__, lock, is_w) + +// Apply ABSL_ANNOTATE_BENIGN_RACE_SIZED to a static variable `static_var`. +#define ABSL_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ + namespace { \ + class static_var##_annotator { \ + public: \ + static_var##_annotator() { \ + ABSL_ANNOTATE_BENIGN_RACE_SIZED(&static_var, sizeof(static_var), \ + #static_var ": " description); \ + } \ + }; \ + static static_var##_annotator the##static_var##_annotator; \ + } // namespace + +// Function prototypes of annotations provided by the compiler-based sanitizer +// implementation. +ABSL_INTERNAL_BEGIN_EXTERN_C +void AnnotateRWLockCreate(const char* file, int line, + const volatile void* lock); +void AnnotateRWLockCreateStatic(const char* file, int line, + const volatile void* lock); +void AnnotateRWLockDestroy(const char* file, int line, + const volatile void* lock); +void AnnotateRWLockAcquired(const char* file, int line, + const volatile void* lock, long is_w); // NOLINT +void AnnotateRWLockReleased(const char* file, int line, + const volatile void* lock, long is_w); // NOLINT +void AnnotateBenignRace(const char* file, int line, + const volatile void* address, const char* description); +void AnnotateBenignRaceSized(const char* file, int line, + const volatile void* address, size_t size, + const char* description); +void AnnotateThreadName(const char* file, int line, const char* name); +void AnnotateEnableRaceDetection(const char* file, int line, int enable); +ABSL_INTERNAL_END_EXTERN_C + +#else // ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED == 0 + +#define ABSL_ANNOTATE_RWLOCK_CREATE(lock) // empty +#define ABSL_ANNOTATE_RWLOCK_CREATE_STATIC(lock) // empty +#define ABSL_ANNOTATE_RWLOCK_DESTROY(lock) // empty +#define ABSL_ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) // empty +#define ABSL_ANNOTATE_RWLOCK_RELEASED(lock, is_w) // empty +#define ABSL_ANNOTATE_BENIGN_RACE(address, description) // empty +#define ABSL_ANNOTATE_BENIGN_RACE_SIZED(address, size, description) // empty +#define ABSL_ANNOTATE_THREAD_NAME(name) // empty +#define ABSL_ANNOTATE_ENABLE_RACE_DETECTION(enable) // empty +#define ABSL_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) // empty + +#endif // ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED + +// ------------------------------------------------------------------------- +// Define memory annotations. + +#ifdef ABSL_HAVE_MEMORY_SANITIZER + +#include <sanitizer/msan_interface.h> + +#define ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ + __msan_unpoison(address, size) + +#define ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(address, size) \ + __msan_allocated_memory(address, size) + +#else // !defined(ABSL_HAVE_MEMORY_SANITIZER) + +// TODO(rogeeff): remove this branch +#ifdef ABSL_HAVE_THREAD_SANITIZER +#define ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ + do { \ + (void)(address); \ + (void)(size); \ + } while (0) +#define ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(address, size) \ + do { \ + (void)(address); \ + (void)(size); \ + } while (0) +#else + +#define ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(address, size) // empty +#define ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(address, size) // empty + +#endif + +#endif // ABSL_HAVE_MEMORY_SANITIZER + +// ------------------------------------------------------------------------- +// Define IGNORE_READS_BEGIN/_END attributes. + +#if defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) + +#define ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE \ + __attribute((exclusive_lock_function("*"))) +#define ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE \ + __attribute((unlock_function("*"))) + +#else // !defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) + +#define ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE // empty +#define ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE // empty + +#endif // defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) + +// ------------------------------------------------------------------------- +// Define IGNORE_READS_BEGIN/_END annotations. + +#if ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED == 1 + +// Request the analysis tool to ignore all reads in the current thread until +// ABSL_ANNOTATE_IGNORE_READS_END is called. Useful to ignore intentional racey +// reads, while still checking other reads and all writes. +// See also ABSL_ANNOTATE_UNPROTECTED_READ. +#define ABSL_ANNOTATE_IGNORE_READS_BEGIN() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreReadsBegin)(__FILE__, __LINE__) + +// Stop ignoring reads. +#define ABSL_ANNOTATE_IGNORE_READS_END() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreReadsEnd)(__FILE__, __LINE__) + +// Function prototypes of annotations provided by the compiler-based sanitizer +// implementation. +ABSL_INTERNAL_BEGIN_EXTERN_C +void AnnotateIgnoreReadsBegin(const char* file, int line) + ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE; +void AnnotateIgnoreReadsEnd(const char* file, + int line) ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE; +ABSL_INTERNAL_END_EXTERN_C + +#elif defined(ABSL_INTERNAL_ANNOTALYSIS_ENABLED) + +// When Annotalysis is enabled without Dynamic Annotations, the use of +// static-inline functions allows the annotations to be read at compile-time, +// while still letting the compiler elide the functions from the final build. +// +// TODO(delesley) -- The exclusive lock here ignores writes as well, but +// allows IGNORE_READS_AND_WRITES to work properly. + +#define ABSL_ANNOTATE_IGNORE_READS_BEGIN() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AbslInternalAnnotateIgnoreReadsBegin)() + +#define ABSL_ANNOTATE_IGNORE_READS_END() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AbslInternalAnnotateIgnoreReadsEnd)() + +ABSL_INTERNAL_STATIC_INLINE void AbslInternalAnnotateIgnoreReadsBegin() + ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE {} + +ABSL_INTERNAL_STATIC_INLINE void AbslInternalAnnotateIgnoreReadsEnd() + ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE {} + +#else + +#define ABSL_ANNOTATE_IGNORE_READS_BEGIN() // empty +#define ABSL_ANNOTATE_IGNORE_READS_END() // empty + +#endif + +// ------------------------------------------------------------------------- +// Define IGNORE_WRITES_BEGIN/_END annotations. + +#if ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED == 1 + +// Similar to ABSL_ANNOTATE_IGNORE_READS_BEGIN, but ignore writes instead. +#define ABSL_ANNOTATE_IGNORE_WRITES_BEGIN() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreWritesBegin)(__FILE__, __LINE__) + +// Stop ignoring writes. +#define ABSL_ANNOTATE_IGNORE_WRITES_END() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreWritesEnd)(__FILE__, __LINE__) + +// Function prototypes of annotations provided by the compiler-based sanitizer +// implementation. +ABSL_INTERNAL_BEGIN_EXTERN_C +void AnnotateIgnoreWritesBegin(const char* file, int line); +void AnnotateIgnoreWritesEnd(const char* file, int line); +ABSL_INTERNAL_END_EXTERN_C + +#else + +#define ABSL_ANNOTATE_IGNORE_WRITES_BEGIN() // empty +#define ABSL_ANNOTATE_IGNORE_WRITES_END() // empty + +#endif + +// ------------------------------------------------------------------------- +// Define the ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_* annotations using the more +// primitive annotations defined above. +// +// Instead of doing +// ABSL_ANNOTATE_IGNORE_READS_BEGIN(); +// ... = x; +// ABSL_ANNOTATE_IGNORE_READS_END(); +// one can use +// ... = ABSL_ANNOTATE_UNPROTECTED_READ(x); + +#if defined(ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED) + +// Start ignoring all memory accesses (both reads and writes). +#define ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() \ + do { \ + ABSL_ANNOTATE_IGNORE_READS_BEGIN(); \ + ABSL_ANNOTATE_IGNORE_WRITES_BEGIN(); \ + } while (0) + +// Stop ignoring both reads and writes. +#define ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END() \ + do { \ + ABSL_ANNOTATE_IGNORE_WRITES_END(); \ + ABSL_ANNOTATE_IGNORE_READS_END(); \ + } while (0) + +#ifdef __cplusplus +// ABSL_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. +#define ABSL_ANNOTATE_UNPROTECTED_READ(x) \ + absl::base_internal::AnnotateUnprotectedRead(x) + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +template <typename T> +inline T AnnotateUnprotectedRead(const volatile T& x) { // NOLINT + ABSL_ANNOTATE_IGNORE_READS_BEGIN(); + T res = x; + ABSL_ANNOTATE_IGNORE_READS_END(); + return res; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif + +#else + +#define ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() // empty +#define ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END() // empty +#define ABSL_ANNOTATE_UNPROTECTED_READ(x) (x) + +#endif + +#ifdef __cplusplus +#ifdef ABSL_HAVE_THREAD_SANITIZER +ABSL_INTERNAL_BEGIN_EXTERN_C +int RunningOnValgrind(); +double ValgrindSlowdown(); +ABSL_INTERNAL_END_EXTERN_C +#else +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +ABSL_DEPRECATED( + "Don't use this interface. It is misleading and is being deleted.") +ABSL_ATTRIBUTE_ALWAYS_INLINE inline int RunningOnValgrind() { return 0; } +ABSL_DEPRECATED( + "Don't use this interface. It is misleading and is being deleted.") +ABSL_ATTRIBUTE_ALWAYS_INLINE inline double ValgrindSlowdown() { return 1.0; } +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +using absl::base_internal::RunningOnValgrind; +using absl::base_internal::ValgrindSlowdown; +#endif +#endif + +// ------------------------------------------------------------------------- +// Address sanitizer annotations + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +// Describe the current state of a contiguous container such as e.g. +// std::vector or std::string. For more details see +// sanitizer/common_interface_defs.h, which is provided by the compiler. +#include <sanitizer/common_interface_defs.h> + +#define ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(beg, end, old_mid, new_mid) \ + __sanitizer_annotate_contiguous_container(beg, end, old_mid, new_mid) +#define ABSL_ADDRESS_SANITIZER_REDZONE(name) \ + struct { \ + char x[8] __attribute__((aligned(8))); \ + } name + +#else + +#define ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(beg, end, old_mid, new_mid) // empty +#define ABSL_ADDRESS_SANITIZER_REDZONE(name) static_assert(true, "") + +#endif // ABSL_HAVE_ADDRESS_SANITIZER + +// ------------------------------------------------------------------------- +// Undefine the macros intended only for this file. + +#undef ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_ANNOTALYSIS_ENABLED +#undef ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_BEGIN_EXTERN_C +#undef ABSL_INTERNAL_END_EXTERN_C +#undef ABSL_INTERNAL_STATIC_INLINE + +#endif // ABSL_BASE_DYNAMIC_ANNOTATIONS_H_ diff --git a/third_party/abseil_cpp/absl/base/exception_safety_testing_test.cc b/third_party/abseil_cpp/absl/base/exception_safety_testing_test.cc new file mode 100644 index 000000000000..a59be29e919c --- /dev/null +++ b/third_party/abseil_cpp/absl/base/exception_safety_testing_test.cc @@ -0,0 +1,956 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/exception_safety_testing.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + +#include <cstddef> +#include <exception> +#include <iostream> +#include <list> +#include <type_traits> +#include <vector> + +#include "gtest/gtest-spi.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" + +namespace testing { + +namespace { + +using ::testing::exceptions_internal::SetCountdown; +using ::testing::exceptions_internal::TestException; +using ::testing::exceptions_internal::UnsetCountdown; + +// EXPECT_NO_THROW can't inspect the thrown inspection in general. +template <typename F> +void ExpectNoThrow(const F& f) { + try { + f(); + } catch (const TestException& e) { + ADD_FAILURE() << "Unexpected exception thrown from " << e.what(); + } +} + +TEST(ThrowingValueTest, Throws) { + SetCountdown(); + EXPECT_THROW(ThrowingValue<> bomb, TestException); + + // It's not guaranteed that every operator only throws *once*. The default + // ctor only throws once, though, so use it to make sure we only throw when + // the countdown hits 0 + SetCountdown(2); + ExpectNoThrow([]() { ThrowingValue<> bomb; }); + ExpectNoThrow([]() { ThrowingValue<> bomb; }); + EXPECT_THROW(ThrowingValue<> bomb, TestException); + + UnsetCountdown(); +} + +// Tests that an operation throws when the countdown is at 0, doesn't throw when +// the countdown doesn't hit 0, and doesn't modify the state of the +// ThrowingValue if it throws +template <typename F> +void TestOp(const F& f) { + ExpectNoThrow(f); + + SetCountdown(); + EXPECT_THROW(f(), TestException); + UnsetCountdown(); +} + +TEST(ThrowingValueTest, ThrowingCtors) { + ThrowingValue<> bomb; + + TestOp([]() { ThrowingValue<> bomb(1); }); + TestOp([&]() { ThrowingValue<> bomb1 = bomb; }); + TestOp([&]() { ThrowingValue<> bomb1 = std::move(bomb); }); +} + +TEST(ThrowingValueTest, ThrowingAssignment) { + ThrowingValue<> bomb, bomb1; + + TestOp([&]() { bomb = bomb1; }); + TestOp([&]() { bomb = std::move(bomb1); }); + + // Test that when assignment throws, the assignment should fail (lhs != rhs) + // and strong guarantee fails (lhs != lhs_copy). + { + ThrowingValue<> lhs(39), rhs(42); + ThrowingValue<> lhs_copy(lhs); + SetCountdown(); + EXPECT_THROW(lhs = rhs, TestException); + UnsetCountdown(); + EXPECT_NE(lhs, rhs); + EXPECT_NE(lhs_copy, lhs); + } + { + ThrowingValue<> lhs(39), rhs(42); + ThrowingValue<> lhs_copy(lhs), rhs_copy(rhs); + SetCountdown(); + EXPECT_THROW(lhs = std::move(rhs), TestException); + UnsetCountdown(); + EXPECT_NE(lhs, rhs_copy); + EXPECT_NE(lhs_copy, lhs); + } +} + +TEST(ThrowingValueTest, ThrowingComparisons) { + ThrowingValue<> bomb1, bomb2; + TestOp([&]() { return bomb1 == bomb2; }); + TestOp([&]() { return bomb1 != bomb2; }); + TestOp([&]() { return bomb1 < bomb2; }); + TestOp([&]() { return bomb1 <= bomb2; }); + TestOp([&]() { return bomb1 > bomb2; }); + TestOp([&]() { return bomb1 >= bomb2; }); +} + +TEST(ThrowingValueTest, ThrowingArithmeticOps) { + ThrowingValue<> bomb1(1), bomb2(2); + + TestOp([&bomb1]() { +bomb1; }); + TestOp([&bomb1]() { -bomb1; }); + TestOp([&bomb1]() { ++bomb1; }); + TestOp([&bomb1]() { bomb1++; }); + TestOp([&bomb1]() { --bomb1; }); + TestOp([&bomb1]() { bomb1--; }); + + TestOp([&]() { bomb1 + bomb2; }); + TestOp([&]() { bomb1 - bomb2; }); + TestOp([&]() { bomb1* bomb2; }); + TestOp([&]() { bomb1 / bomb2; }); + TestOp([&]() { bomb1 << 1; }); + TestOp([&]() { bomb1 >> 1; }); +} + +TEST(ThrowingValueTest, ThrowingLogicalOps) { + ThrowingValue<> bomb1, bomb2; + + TestOp([&bomb1]() { !bomb1; }); + TestOp([&]() { bomb1&& bomb2; }); + TestOp([&]() { bomb1 || bomb2; }); +} + +TEST(ThrowingValueTest, ThrowingBitwiseOps) { + ThrowingValue<> bomb1, bomb2; + + TestOp([&bomb1]() { ~bomb1; }); + TestOp([&]() { bomb1& bomb2; }); + TestOp([&]() { bomb1 | bomb2; }); + TestOp([&]() { bomb1 ^ bomb2; }); +} + +TEST(ThrowingValueTest, ThrowingCompoundAssignmentOps) { + ThrowingValue<> bomb1(1), bomb2(2); + + TestOp([&]() { bomb1 += bomb2; }); + TestOp([&]() { bomb1 -= bomb2; }); + TestOp([&]() { bomb1 *= bomb2; }); + TestOp([&]() { bomb1 /= bomb2; }); + TestOp([&]() { bomb1 %= bomb2; }); + TestOp([&]() { bomb1 &= bomb2; }); + TestOp([&]() { bomb1 |= bomb2; }); + TestOp([&]() { bomb1 ^= bomb2; }); + TestOp([&]() { bomb1 *= bomb2; }); +} + +TEST(ThrowingValueTest, ThrowingStreamOps) { + ThrowingValue<> bomb; + + TestOp([&]() { + std::istringstream stream; + stream >> bomb; + }); + TestOp([&]() { + std::stringstream stream; + stream << bomb; + }); +} + +// Tests the operator<< of ThrowingValue by forcing ConstructorTracker to emit +// a nonfatal failure that contains the string representation of the Thrower +TEST(ThrowingValueTest, StreamOpsOutput) { + using ::testing::TypeSpec; + exceptions_internal::ConstructorTracker ct(exceptions_internal::countdown); + + // Test default spec list (kEverythingThrows) + EXPECT_NONFATAL_FAILURE( + { + using Thrower = ThrowingValue<TypeSpec{}>; + auto thrower = Thrower(123); + thrower.~Thrower(); + }, + "ThrowingValue<>(123)"); + + // Test with one item in spec list (kNoThrowCopy) + EXPECT_NONFATAL_FAILURE( + { + using Thrower = ThrowingValue<TypeSpec::kNoThrowCopy>; + auto thrower = Thrower(234); + thrower.~Thrower(); + }, + "ThrowingValue<kNoThrowCopy>(234)"); + + // Test with multiple items in spec list (kNoThrowMove, kNoThrowNew) + EXPECT_NONFATAL_FAILURE( + { + using Thrower = + ThrowingValue<TypeSpec::kNoThrowMove | TypeSpec::kNoThrowNew>; + auto thrower = Thrower(345); + thrower.~Thrower(); + }, + "ThrowingValue<kNoThrowMove | kNoThrowNew>(345)"); + + // Test with all items in spec list (kNoThrowCopy, kNoThrowMove, kNoThrowNew) + EXPECT_NONFATAL_FAILURE( + { + using Thrower = ThrowingValue<static_cast<TypeSpec>(-1)>; + auto thrower = Thrower(456); + thrower.~Thrower(); + }, + "ThrowingValue<kNoThrowCopy | kNoThrowMove | kNoThrowNew>(456)"); +} + +template <typename F> +void TestAllocatingOp(const F& f) { + ExpectNoThrow(f); + + SetCountdown(); + EXPECT_THROW(f(), exceptions_internal::TestBadAllocException); + UnsetCountdown(); +} + +TEST(ThrowingValueTest, ThrowingAllocatingOps) { + // make_unique calls unqualified operator new, so these exercise the + // ThrowingValue overloads. + TestAllocatingOp([]() { return absl::make_unique<ThrowingValue<>>(1); }); + TestAllocatingOp([]() { return absl::make_unique<ThrowingValue<>[]>(2); }); +} + +TEST(ThrowingValueTest, NonThrowingMoveCtor) { + ThrowingValue<TypeSpec::kNoThrowMove> nothrow_ctor; + + SetCountdown(); + ExpectNoThrow([¬hrow_ctor]() { + ThrowingValue<TypeSpec::kNoThrowMove> nothrow1 = std::move(nothrow_ctor); + }); + UnsetCountdown(); +} + +TEST(ThrowingValueTest, NonThrowingMoveAssign) { + ThrowingValue<TypeSpec::kNoThrowMove> nothrow_assign1, nothrow_assign2; + + SetCountdown(); + ExpectNoThrow([¬hrow_assign1, ¬hrow_assign2]() { + nothrow_assign1 = std::move(nothrow_assign2); + }); + UnsetCountdown(); +} + +TEST(ThrowingValueTest, ThrowingCopyCtor) { + ThrowingValue<> tv; + + TestOp([&]() { ThrowingValue<> tv_copy(tv); }); +} + +TEST(ThrowingValueTest, ThrowingCopyAssign) { + ThrowingValue<> tv1, tv2; + + TestOp([&]() { tv1 = tv2; }); +} + +TEST(ThrowingValueTest, NonThrowingCopyCtor) { + ThrowingValue<TypeSpec::kNoThrowCopy> nothrow_ctor; + + SetCountdown(); + ExpectNoThrow([¬hrow_ctor]() { + ThrowingValue<TypeSpec::kNoThrowCopy> nothrow1(nothrow_ctor); + }); + UnsetCountdown(); +} + +TEST(ThrowingValueTest, NonThrowingCopyAssign) { + ThrowingValue<TypeSpec::kNoThrowCopy> nothrow_assign1, nothrow_assign2; + + SetCountdown(); + ExpectNoThrow([¬hrow_assign1, ¬hrow_assign2]() { + nothrow_assign1 = nothrow_assign2; + }); + UnsetCountdown(); +} + +TEST(ThrowingValueTest, ThrowingSwap) { + ThrowingValue<> bomb1, bomb2; + TestOp([&]() { std::swap(bomb1, bomb2); }); +} + +TEST(ThrowingValueTest, NonThrowingSwap) { + ThrowingValue<TypeSpec::kNoThrowMove> bomb1, bomb2; + ExpectNoThrow([&]() { std::swap(bomb1, bomb2); }); +} + +TEST(ThrowingValueTest, NonThrowingAllocation) { + ThrowingValue<TypeSpec::kNoThrowNew>* allocated; + ThrowingValue<TypeSpec::kNoThrowNew>* array; + + ExpectNoThrow([&allocated]() { + allocated = new ThrowingValue<TypeSpec::kNoThrowNew>(1); + delete allocated; + }); + ExpectNoThrow([&array]() { + array = new ThrowingValue<TypeSpec::kNoThrowNew>[2]; + delete[] array; + }); +} + +TEST(ThrowingValueTest, NonThrowingDelete) { + auto* allocated = new ThrowingValue<>(1); + auto* array = new ThrowingValue<>[2]; + + SetCountdown(); + ExpectNoThrow([allocated]() { delete allocated; }); + SetCountdown(); + ExpectNoThrow([array]() { delete[] array; }); + + UnsetCountdown(); +} + +TEST(ThrowingValueTest, NonThrowingPlacementDelete) { + constexpr int kArrayLen = 2; + // We intentionally create extra space to store the tag allocated by placement + // new[]. + constexpr int kStorageLen = 4; + + alignas(ThrowingValue<>) unsigned char buf[sizeof(ThrowingValue<>)]; + alignas(ThrowingValue<>) unsigned char + array_buf[sizeof(ThrowingValue<>[kStorageLen])]; + auto* placed = new (&buf) ThrowingValue<>(1); + auto placed_array = new (&array_buf) ThrowingValue<>[kArrayLen]; + + SetCountdown(); + ExpectNoThrow([placed, &buf]() { + placed->~ThrowingValue<>(); + ThrowingValue<>::operator delete(placed, &buf); + }); + + SetCountdown(); + ExpectNoThrow([&, placed_array]() { + for (int i = 0; i < kArrayLen; ++i) placed_array[i].~ThrowingValue<>(); + ThrowingValue<>::operator delete[](placed_array, &array_buf); + }); + + UnsetCountdown(); +} + +TEST(ThrowingValueTest, NonThrowingDestructor) { + auto* allocated = new ThrowingValue<>(); + + SetCountdown(); + ExpectNoThrow([allocated]() { delete allocated; }); + UnsetCountdown(); +} + +TEST(ThrowingBoolTest, ThrowingBool) { + ThrowingBool t = true; + + // Test that it's contextually convertible to bool + if (t) { // NOLINT(whitespace/empty_if_body) + } + EXPECT_TRUE(t); + + TestOp([&]() { (void)!t; }); +} + +TEST(ThrowingAllocatorTest, MemoryManagement) { + // Just exercise the memory management capabilities under LSan to make sure we + // don't leak. + ThrowingAllocator<int> int_alloc; + int* ip = int_alloc.allocate(1); + int_alloc.deallocate(ip, 1); + int* i_array = int_alloc.allocate(2); + int_alloc.deallocate(i_array, 2); + + ThrowingAllocator<ThrowingValue<>> tv_alloc; + ThrowingValue<>* ptr = tv_alloc.allocate(1); + tv_alloc.deallocate(ptr, 1); + ThrowingValue<>* tv_array = tv_alloc.allocate(2); + tv_alloc.deallocate(tv_array, 2); +} + +TEST(ThrowingAllocatorTest, CallsGlobalNew) { + ThrowingAllocator<ThrowingValue<>, AllocSpec::kNoThrowAllocate> nothrow_alloc; + ThrowingValue<>* ptr; + + SetCountdown(); + // This will only throw if ThrowingValue::new is called. + ExpectNoThrow([&]() { ptr = nothrow_alloc.allocate(1); }); + nothrow_alloc.deallocate(ptr, 1); + + UnsetCountdown(); +} + +TEST(ThrowingAllocatorTest, ThrowingConstructors) { + ThrowingAllocator<int> int_alloc; + int* ip = nullptr; + + SetCountdown(); + EXPECT_THROW(ip = int_alloc.allocate(1), TestException); + ExpectNoThrow([&]() { ip = int_alloc.allocate(1); }); + + *ip = 1; + SetCountdown(); + EXPECT_THROW(int_alloc.construct(ip, 2), TestException); + EXPECT_EQ(*ip, 1); + int_alloc.deallocate(ip, 1); + + UnsetCountdown(); +} + +TEST(ThrowingAllocatorTest, NonThrowingConstruction) { + { + ThrowingAllocator<int, AllocSpec::kNoThrowAllocate> int_alloc; + int* ip = nullptr; + + SetCountdown(); + ExpectNoThrow([&]() { ip = int_alloc.allocate(1); }); + + SetCountdown(); + ExpectNoThrow([&]() { int_alloc.construct(ip, 2); }); + + EXPECT_EQ(*ip, 2); + int_alloc.deallocate(ip, 1); + + UnsetCountdown(); + } + + { + ThrowingAllocator<int> int_alloc; + int* ip = nullptr; + ExpectNoThrow([&]() { ip = int_alloc.allocate(1); }); + ExpectNoThrow([&]() { int_alloc.construct(ip, 2); }); + EXPECT_EQ(*ip, 2); + int_alloc.deallocate(ip, 1); + } + + { + ThrowingAllocator<ThrowingValue<>, AllocSpec::kNoThrowAllocate> + nothrow_alloc; + ThrowingValue<>* ptr; + + SetCountdown(); + ExpectNoThrow([&]() { ptr = nothrow_alloc.allocate(1); }); + + SetCountdown(); + ExpectNoThrow( + [&]() { nothrow_alloc.construct(ptr, 2, testing::nothrow_ctor); }); + + EXPECT_EQ(ptr->Get(), 2); + nothrow_alloc.destroy(ptr); + nothrow_alloc.deallocate(ptr, 1); + + UnsetCountdown(); + } + + { + ThrowingAllocator<int> a; + + SetCountdown(); + ExpectNoThrow([&]() { ThrowingAllocator<double> a1 = a; }); + + SetCountdown(); + ExpectNoThrow([&]() { ThrowingAllocator<double> a1 = std::move(a); }); + + UnsetCountdown(); + } +} + +TEST(ThrowingAllocatorTest, ThrowingAllocatorConstruction) { + ThrowingAllocator<int> a; + TestOp([]() { ThrowingAllocator<int> a; }); + TestOp([&]() { a.select_on_container_copy_construction(); }); +} + +TEST(ThrowingAllocatorTest, State) { + ThrowingAllocator<int> a1, a2; + EXPECT_NE(a1, a2); + + auto a3 = a1; + EXPECT_EQ(a3, a1); + int* ip = a1.allocate(1); + EXPECT_EQ(a3, a1); + a3.deallocate(ip, 1); + EXPECT_EQ(a3, a1); +} + +TEST(ThrowingAllocatorTest, InVector) { + std::vector<ThrowingValue<>, ThrowingAllocator<ThrowingValue<>>> v; + for (int i = 0; i < 20; ++i) v.push_back({}); + for (int i = 0; i < 20; ++i) v.pop_back(); +} + +TEST(ThrowingAllocatorTest, InList) { + std::list<ThrowingValue<>, ThrowingAllocator<ThrowingValue<>>> l; + for (int i = 0; i < 20; ++i) l.push_back({}); + for (int i = 0; i < 20; ++i) l.pop_back(); + for (int i = 0; i < 20; ++i) l.push_front({}); + for (int i = 0; i < 20; ++i) l.pop_front(); +} + +template <typename TesterInstance, typename = void> +struct NullaryTestValidator : public std::false_type {}; + +template <typename TesterInstance> +struct NullaryTestValidator< + TesterInstance, + absl::void_t<decltype(std::declval<TesterInstance>().Test())>> + : public std::true_type {}; + +template <typename TesterInstance> +bool HasNullaryTest(const TesterInstance&) { + return NullaryTestValidator<TesterInstance>::value; +} + +void DummyOp(void*) {} + +template <typename TesterInstance, typename = void> +struct UnaryTestValidator : public std::false_type {}; + +template <typename TesterInstance> +struct UnaryTestValidator< + TesterInstance, + absl::void_t<decltype(std::declval<TesterInstance>().Test(DummyOp))>> + : public std::true_type {}; + +template <typename TesterInstance> +bool HasUnaryTest(const TesterInstance&) { + return UnaryTestValidator<TesterInstance>::value; +} + +TEST(ExceptionSafetyTesterTest, IncompleteTypesAreNotTestable) { + using T = exceptions_internal::UninitializedT; + auto op = [](T* t) {}; + auto inv = [](T*) { return testing::AssertionSuccess(); }; + auto fac = []() { return absl::make_unique<T>(); }; + + // Test that providing operation and inveriants still does not allow for the + // the invocation of .Test() and .Test(op) because it lacks a factory + auto without_fac = + testing::MakeExceptionSafetyTester().WithOperation(op).WithContracts( + inv, testing::strong_guarantee); + EXPECT_FALSE(HasNullaryTest(without_fac)); + EXPECT_FALSE(HasUnaryTest(without_fac)); + + // Test that providing contracts and factory allows the invocation of + // .Test(op) but does not allow for .Test() because it lacks an operation + auto without_op = testing::MakeExceptionSafetyTester() + .WithContracts(inv, testing::strong_guarantee) + .WithFactory(fac); + EXPECT_FALSE(HasNullaryTest(without_op)); + EXPECT_TRUE(HasUnaryTest(without_op)); + + // Test that providing operation and factory still does not allow for the + // the invocation of .Test() and .Test(op) because it lacks contracts + auto without_inv = + testing::MakeExceptionSafetyTester().WithOperation(op).WithFactory(fac); + EXPECT_FALSE(HasNullaryTest(without_inv)); + EXPECT_FALSE(HasUnaryTest(without_inv)); +} + +struct ExampleStruct {}; + +std::unique_ptr<ExampleStruct> ExampleFunctionFactory() { + return absl::make_unique<ExampleStruct>(); +} + +void ExampleFunctionOperation(ExampleStruct*) {} + +testing::AssertionResult ExampleFunctionContract(ExampleStruct*) { + return testing::AssertionSuccess(); +} + +struct { + std::unique_ptr<ExampleStruct> operator()() const { + return ExampleFunctionFactory(); + } +} example_struct_factory; + +struct { + void operator()(ExampleStruct*) const {} +} example_struct_operation; + +struct { + testing::AssertionResult operator()(ExampleStruct* example_struct) const { + return ExampleFunctionContract(example_struct); + } +} example_struct_contract; + +auto example_lambda_factory = []() { return ExampleFunctionFactory(); }; + +auto example_lambda_operation = [](ExampleStruct*) {}; + +auto example_lambda_contract = [](ExampleStruct* example_struct) { + return ExampleFunctionContract(example_struct); +}; + +// Testing that function references, pointers, structs with operator() and +// lambdas can all be used with ExceptionSafetyTester +TEST(ExceptionSafetyTesterTest, MixedFunctionTypes) { + // function reference + EXPECT_TRUE(testing::MakeExceptionSafetyTester() + .WithFactory(ExampleFunctionFactory) + .WithOperation(ExampleFunctionOperation) + .WithContracts(ExampleFunctionContract) + .Test()); + + // function pointer + EXPECT_TRUE(testing::MakeExceptionSafetyTester() + .WithFactory(&ExampleFunctionFactory) + .WithOperation(&ExampleFunctionOperation) + .WithContracts(&ExampleFunctionContract) + .Test()); + + // struct + EXPECT_TRUE(testing::MakeExceptionSafetyTester() + .WithFactory(example_struct_factory) + .WithOperation(example_struct_operation) + .WithContracts(example_struct_contract) + .Test()); + + // lambda + EXPECT_TRUE(testing::MakeExceptionSafetyTester() + .WithFactory(example_lambda_factory) + .WithOperation(example_lambda_operation) + .WithContracts(example_lambda_contract) + .Test()); +} + +struct NonNegative { + bool operator==(const NonNegative& other) const { return i == other.i; } + int i; +}; + +testing::AssertionResult CheckNonNegativeInvariants(NonNegative* g) { + if (g->i >= 0) { + return testing::AssertionSuccess(); + } + return testing::AssertionFailure() + << "i should be non-negative but is " << g->i; +} + +struct { + template <typename T> + void operator()(T* t) const { + (*t)(); + } +} invoker; + +auto tester = + testing::MakeExceptionSafetyTester().WithOperation(invoker).WithContracts( + CheckNonNegativeInvariants); +auto strong_tester = tester.WithContracts(testing::strong_guarantee); + +struct FailsBasicGuarantee : public NonNegative { + void operator()() { + --i; + ThrowingValue<> bomb; + ++i; + } +}; + +TEST(ExceptionCheckTest, BasicGuaranteeFailure) { + EXPECT_FALSE(tester.WithInitialValue(FailsBasicGuarantee{}).Test()); +} + +struct FollowsBasicGuarantee : public NonNegative { + void operator()() { + ++i; + ThrowingValue<> bomb; + } +}; + +TEST(ExceptionCheckTest, BasicGuarantee) { + EXPECT_TRUE(tester.WithInitialValue(FollowsBasicGuarantee{}).Test()); +} + +TEST(ExceptionCheckTest, StrongGuaranteeFailure) { + EXPECT_FALSE(strong_tester.WithInitialValue(FailsBasicGuarantee{}).Test()); + EXPECT_FALSE(strong_tester.WithInitialValue(FollowsBasicGuarantee{}).Test()); +} + +struct BasicGuaranteeWithExtraContracts : public NonNegative { + // After operator(), i is incremented. If operator() throws, i is set to 9999 + void operator()() { + int old_i = i; + i = kExceptionSentinel; + ThrowingValue<> bomb; + i = ++old_i; + } + + static constexpr int kExceptionSentinel = 9999; +}; +constexpr int BasicGuaranteeWithExtraContracts::kExceptionSentinel; + +TEST(ExceptionCheckTest, BasicGuaranteeWithExtraContracts) { + auto tester_with_val = + tester.WithInitialValue(BasicGuaranteeWithExtraContracts{}); + EXPECT_TRUE(tester_with_val.Test()); + EXPECT_TRUE( + tester_with_val + .WithContracts([](BasicGuaranteeWithExtraContracts* o) { + if (o->i == BasicGuaranteeWithExtraContracts::kExceptionSentinel) { + return testing::AssertionSuccess(); + } + return testing::AssertionFailure() + << "i should be " + << BasicGuaranteeWithExtraContracts::kExceptionSentinel + << ", but is " << o->i; + }) + .Test()); +} + +struct FollowsStrongGuarantee : public NonNegative { + void operator()() { ThrowingValue<> bomb; } +}; + +TEST(ExceptionCheckTest, StrongGuarantee) { + EXPECT_TRUE(tester.WithInitialValue(FollowsStrongGuarantee{}).Test()); + EXPECT_TRUE(strong_tester.WithInitialValue(FollowsStrongGuarantee{}).Test()); +} + +struct HasReset : public NonNegative { + void operator()() { + i = -1; + ThrowingValue<> bomb; + i = 1; + } + + void reset() { i = 0; } +}; + +testing::AssertionResult CheckHasResetContracts(HasReset* h) { + h->reset(); + return testing::AssertionResult(h->i == 0); +} + +TEST(ExceptionCheckTest, ModifyingChecker) { + auto set_to_1000 = [](FollowsBasicGuarantee* g) { + g->i = 1000; + return testing::AssertionSuccess(); + }; + auto is_1000 = [](FollowsBasicGuarantee* g) { + return testing::AssertionResult(g->i == 1000); + }; + auto increment = [](FollowsStrongGuarantee* g) { + ++g->i; + return testing::AssertionSuccess(); + }; + + EXPECT_FALSE(tester.WithInitialValue(FollowsBasicGuarantee{}) + .WithContracts(set_to_1000, is_1000) + .Test()); + EXPECT_TRUE(strong_tester.WithInitialValue(FollowsStrongGuarantee{}) + .WithContracts(increment) + .Test()); + EXPECT_TRUE(testing::MakeExceptionSafetyTester() + .WithInitialValue(HasReset{}) + .WithContracts(CheckHasResetContracts) + .Test(invoker)); +} + +TEST(ExceptionSafetyTesterTest, ResetsCountdown) { + auto test = + testing::MakeExceptionSafetyTester() + .WithInitialValue(ThrowingValue<>()) + .WithContracts([](ThrowingValue<>*) { return AssertionSuccess(); }) + .WithOperation([](ThrowingValue<>*) {}); + ASSERT_TRUE(test.Test()); + // If the countdown isn't reset because there were no exceptions thrown, then + // this will fail with a termination from an unhandled exception + EXPECT_TRUE(test.Test()); +} + +struct NonCopyable : public NonNegative { + NonCopyable(const NonCopyable&) = delete; + NonCopyable() : NonNegative{0} {} + + void operator()() { ThrowingValue<> bomb; } +}; + +TEST(ExceptionCheckTest, NonCopyable) { + auto factory = []() { return absl::make_unique<NonCopyable>(); }; + EXPECT_TRUE(tester.WithFactory(factory).Test()); + EXPECT_TRUE(strong_tester.WithFactory(factory).Test()); +} + +struct NonEqualityComparable : public NonNegative { + void operator()() { ThrowingValue<> bomb; } + + void ModifyOnThrow() { + ++i; + ThrowingValue<> bomb; + static_cast<void>(bomb); + --i; + } +}; + +TEST(ExceptionCheckTest, NonEqualityComparable) { + auto nec_is_strong = [](NonEqualityComparable* nec) { + return testing::AssertionResult(nec->i == NonEqualityComparable().i); + }; + auto strong_nec_tester = tester.WithInitialValue(NonEqualityComparable{}) + .WithContracts(nec_is_strong); + + EXPECT_TRUE(strong_nec_tester.Test()); + EXPECT_FALSE(strong_nec_tester.Test( + [](NonEqualityComparable* n) { n->ModifyOnThrow(); })); +} + +template <typename T> +struct ExhaustivenessTester { + void operator()() { + successes |= 1; + T b1; + static_cast<void>(b1); + successes |= (1 << 1); + T b2; + static_cast<void>(b2); + successes |= (1 << 2); + T b3; + static_cast<void>(b3); + successes |= (1 << 3); + } + + bool operator==(const ExhaustivenessTester<ThrowingValue<>>&) const { + return true; + } + + static unsigned char successes; +}; + +struct { + template <typename T> + testing::AssertionResult operator()(ExhaustivenessTester<T>*) const { + return testing::AssertionSuccess(); + } +} CheckExhaustivenessTesterContracts; + +template <typename T> +unsigned char ExhaustivenessTester<T>::successes = 0; + +TEST(ExceptionCheckTest, Exhaustiveness) { + auto exhaust_tester = testing::MakeExceptionSafetyTester() + .WithContracts(CheckExhaustivenessTesterContracts) + .WithOperation(invoker); + + EXPECT_TRUE( + exhaust_tester.WithInitialValue(ExhaustivenessTester<int>{}).Test()); + EXPECT_EQ(ExhaustivenessTester<int>::successes, 0xF); + + EXPECT_TRUE( + exhaust_tester.WithInitialValue(ExhaustivenessTester<ThrowingValue<>>{}) + .WithContracts(testing::strong_guarantee) + .Test()); + EXPECT_EQ(ExhaustivenessTester<ThrowingValue<>>::successes, 0xF); +} + +struct LeaksIfCtorThrows : private exceptions_internal::TrackedObject { + LeaksIfCtorThrows() : TrackedObject(ABSL_PRETTY_FUNCTION) { + ++counter; + ThrowingValue<> v; + static_cast<void>(v); + --counter; + } + LeaksIfCtorThrows(const LeaksIfCtorThrows&) noexcept + : TrackedObject(ABSL_PRETTY_FUNCTION) {} + static int counter; +}; +int LeaksIfCtorThrows::counter = 0; + +TEST(ExceptionCheckTest, TestLeakyCtor) { + testing::TestThrowingCtor<LeaksIfCtorThrows>(); + EXPECT_EQ(LeaksIfCtorThrows::counter, 1); + LeaksIfCtorThrows::counter = 0; +} + +struct Tracked : private exceptions_internal::TrackedObject { + Tracked() : TrackedObject(ABSL_PRETTY_FUNCTION) {} +}; + +TEST(ConstructorTrackerTest, CreatedBefore) { + Tracked a, b, c; + exceptions_internal::ConstructorTracker ct(exceptions_internal::countdown); +} + +TEST(ConstructorTrackerTest, CreatedAfter) { + exceptions_internal::ConstructorTracker ct(exceptions_internal::countdown); + Tracked a, b, c; +} + +TEST(ConstructorTrackerTest, NotDestroyedAfter) { + alignas(Tracked) unsigned char storage[sizeof(Tracked)]; + EXPECT_NONFATAL_FAILURE( + { + exceptions_internal::ConstructorTracker ct( + exceptions_internal::countdown); + new (&storage) Tracked(); + }, + "not destroyed"); +} + +TEST(ConstructorTrackerTest, DestroyedTwice) { + exceptions_internal::ConstructorTracker ct(exceptions_internal::countdown); + EXPECT_NONFATAL_FAILURE( + { + Tracked t; + t.~Tracked(); + }, + "re-destroyed"); +} + +TEST(ConstructorTrackerTest, ConstructedTwice) { + exceptions_internal::ConstructorTracker ct(exceptions_internal::countdown); + alignas(Tracked) unsigned char storage[sizeof(Tracked)]; + EXPECT_NONFATAL_FAILURE( + { + new (&storage) Tracked(); + new (&storage) Tracked(); + reinterpret_cast<Tracked*>(&storage)->~Tracked(); + }, + "re-constructed"); +} + +TEST(ThrowingValueTraitsTest, RelationalOperators) { + ThrowingValue<> a, b; + EXPECT_TRUE((std::is_convertible<decltype(a == b), bool>::value)); + EXPECT_TRUE((std::is_convertible<decltype(a != b), bool>::value)); + EXPECT_TRUE((std::is_convertible<decltype(a < b), bool>::value)); + EXPECT_TRUE((std::is_convertible<decltype(a <= b), bool>::value)); + EXPECT_TRUE((std::is_convertible<decltype(a > b), bool>::value)); + EXPECT_TRUE((std::is_convertible<decltype(a >= b), bool>::value)); +} + +TEST(ThrowingAllocatorTraitsTest, Assignablility) { + EXPECT_TRUE(absl::is_move_assignable<ThrowingAllocator<int>>::value); + EXPECT_TRUE(absl::is_copy_assignable<ThrowingAllocator<int>>::value); + EXPECT_TRUE(std::is_nothrow_move_assignable<ThrowingAllocator<int>>::value); + EXPECT_TRUE(std::is_nothrow_copy_assignable<ThrowingAllocator<int>>::value); +} + +} // namespace + +} // namespace testing + +#endif // ABSL_HAVE_EXCEPTIONS diff --git a/third_party/abseil_cpp/absl/base/inline_variable_test.cc b/third_party/abseil_cpp/absl/base/inline_variable_test.cc new file mode 100644 index 000000000000..37a40e1e40a6 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/inline_variable_test.cc @@ -0,0 +1,64 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <type_traits> + +#include "absl/base/internal/inline_variable.h" +#include "absl/base/internal/inline_variable_testing.h" + +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace inline_variable_testing_internal { +namespace { + +TEST(InlineVariableTest, Constexpr) { + static_assert(inline_variable_foo.value == 5, ""); + static_assert(other_inline_variable_foo.value == 5, ""); + static_assert(inline_variable_int == 5, ""); + static_assert(other_inline_variable_int == 5, ""); +} + +TEST(InlineVariableTest, DefaultConstructedIdentityEquality) { + EXPECT_EQ(get_foo_a().value, 5); + EXPECT_EQ(get_foo_b().value, 5); + EXPECT_EQ(&get_foo_a(), &get_foo_b()); +} + +TEST(InlineVariableTest, DefaultConstructedIdentityInequality) { + EXPECT_NE(&inline_variable_foo, &other_inline_variable_foo); +} + +TEST(InlineVariableTest, InitializedIdentityEquality) { + EXPECT_EQ(get_int_a(), 5); + EXPECT_EQ(get_int_b(), 5); + EXPECT_EQ(&get_int_a(), &get_int_b()); +} + +TEST(InlineVariableTest, InitializedIdentityInequality) { + EXPECT_NE(&inline_variable_int, &other_inline_variable_int); +} + +TEST(InlineVariableTest, FunPtrType) { + static_assert( + std::is_same<void(*)(), + std::decay<decltype(inline_variable_fun_ptr)>::type>::value, + ""); +} + +} // namespace +} // namespace inline_variable_testing_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/inline_variable_test_a.cc b/third_party/abseil_cpp/absl/base/inline_variable_test_a.cc new file mode 100644 index 000000000000..f96a58d9b422 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/inline_variable_test_a.cc @@ -0,0 +1,27 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/inline_variable_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace inline_variable_testing_internal { + +const Foo& get_foo_a() { return inline_variable_foo; } + +const int& get_int_a() { return inline_variable_int; } + +} // namespace inline_variable_testing_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/inline_variable_test_b.cc b/third_party/abseil_cpp/absl/base/inline_variable_test_b.cc new file mode 100644 index 000000000000..038adc30a9fb --- /dev/null +++ b/third_party/abseil_cpp/absl/base/inline_variable_test_b.cc @@ -0,0 +1,27 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/inline_variable_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace inline_variable_testing_internal { + +const Foo& get_foo_b() { return inline_variable_foo; } + +const int& get_int_b() { return inline_variable_int; } + +} // namespace inline_variable_testing_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/atomic_hook.h b/third_party/abseil_cpp/absl/base/internal/atomic_hook.h new file mode 100644 index 000000000000..ae21cd7fe50a --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/atomic_hook.h @@ -0,0 +1,200 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_ATOMIC_HOOK_H_ +#define ABSL_BASE_INTERNAL_ATOMIC_HOOK_H_ + +#include <atomic> +#include <cassert> +#include <cstdint> +#include <utility> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" + +#if defined(_MSC_VER) && !defined(__clang__) +#define ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT 0 +#else +#define ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT 1 +#endif + +#if defined(_MSC_VER) +#define ABSL_HAVE_WORKING_ATOMIC_POINTER 0 +#else +#define ABSL_HAVE_WORKING_ATOMIC_POINTER 1 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +template <typename T> +class AtomicHook; + +// To workaround AtomicHook not being constant-initializable on some platforms, +// prefer to annotate instances with `ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES` +// instead of `ABSL_CONST_INIT`. +#if ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT +#define ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_CONST_INIT +#else +#define ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +#endif + +// `AtomicHook` is a helper class, templatized on a raw function pointer type, +// for implementing Abseil customization hooks. It is a callable object that +// dispatches to the registered hook. Objects of type `AtomicHook` must have +// static or thread storage duration. +// +// A default constructed object performs a no-op (and returns a default +// constructed object) if no hook has been registered. +// +// Hooks can be pre-registered via constant initialization, for example: +// +// ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static AtomicHook<void(*)()> +// my_hook(DefaultAction); +// +// and then changed at runtime via a call to `Store()`. +// +// Reads and writes guarantee memory_order_acquire/memory_order_release +// semantics. +template <typename ReturnType, typename... Args> +class AtomicHook<ReturnType (*)(Args...)> { + public: + using FnPtr = ReturnType (*)(Args...); + + // Constructs an object that by default performs a no-op (and + // returns a default constructed object) when no hook as been registered. + constexpr AtomicHook() : AtomicHook(DummyFunction) {} + + // Constructs an object that by default dispatches to/returns the + // pre-registered default_fn when no hook has been registered at runtime. +#if ABSL_HAVE_WORKING_ATOMIC_POINTER && ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT + explicit constexpr AtomicHook(FnPtr default_fn) + : hook_(default_fn), default_fn_(default_fn) {} +#elif ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT + explicit constexpr AtomicHook(FnPtr default_fn) + : hook_(kUninitialized), default_fn_(default_fn) {} +#else + // As of January 2020, on all known versions of MSVC this constructor runs in + // the global constructor sequence. If `Store()` is called by a dynamic + // initializer, we want to preserve the value, even if this constructor runs + // after the call to `Store()`. If not, `hook_` will be + // zero-initialized by the linker and we have no need to set it. + // https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html + explicit constexpr AtomicHook(FnPtr default_fn) + : /* hook_(deliberately omitted), */ default_fn_(default_fn) { + static_assert(kUninitialized == 0, "here we rely on zero-initialization"); + } +#endif + + // Stores the provided function pointer as the value for this hook. + // + // This is intended to be called once. Multiple calls are legal only if the + // same function pointer is provided for each call. The store is implemented + // as a memory_order_release operation, and read accesses are implemented as + // memory_order_acquire. + void Store(FnPtr fn) { + bool success = DoStore(fn); + static_cast<void>(success); + assert(success); + } + + // Invokes the registered callback. If no callback has yet been registered, a + // default-constructed object of the appropriate type is returned instead. + template <typename... CallArgs> + ReturnType operator()(CallArgs&&... args) const { + return DoLoad()(std::forward<CallArgs>(args)...); + } + + // Returns the registered callback, or nullptr if none has been registered. + // Useful if client code needs to conditionalize behavior based on whether a + // callback was registered. + // + // Note that atomic_hook.Load()() and atomic_hook() have different semantics: + // operator()() will perform a no-op if no callback was registered, while + // Load()() will dereference a null function pointer. Prefer operator()() to + // Load()() unless you must conditionalize behavior on whether a hook was + // registered. + FnPtr Load() const { + FnPtr ptr = DoLoad(); + return (ptr == DummyFunction) ? nullptr : ptr; + } + + private: + static ReturnType DummyFunction(Args...) { + return ReturnType(); + } + + // Current versions of MSVC (as of September 2017) have a broken + // implementation of std::atomic<T*>: Its constructor attempts to do the + // equivalent of a reinterpret_cast in a constexpr context, which is not + // allowed. + // + // This causes an issue when building with LLVM under Windows. To avoid this, + // we use a less-efficient, intptr_t-based implementation on Windows. +#if ABSL_HAVE_WORKING_ATOMIC_POINTER + // Return the stored value, or DummyFunction if no value has been stored. + FnPtr DoLoad() const { return hook_.load(std::memory_order_acquire); } + + // Store the given value. Returns false if a different value was already + // stored to this object. + bool DoStore(FnPtr fn) { + assert(fn); + FnPtr expected = default_fn_; + const bool store_succeeded = hook_.compare_exchange_strong( + expected, fn, std::memory_order_acq_rel, std::memory_order_acquire); + const bool same_value_already_stored = (expected == fn); + return store_succeeded || same_value_already_stored; + } + + std::atomic<FnPtr> hook_; +#else // !ABSL_HAVE_WORKING_ATOMIC_POINTER + // Use a sentinel value unlikely to be the address of an actual function. + static constexpr intptr_t kUninitialized = 0; + + static_assert(sizeof(intptr_t) >= sizeof(FnPtr), + "intptr_t can't contain a function pointer"); + + FnPtr DoLoad() const { + const intptr_t value = hook_.load(std::memory_order_acquire); + if (value == kUninitialized) { + return default_fn_; + } + return reinterpret_cast<FnPtr>(value); + } + + bool DoStore(FnPtr fn) { + assert(fn); + const auto value = reinterpret_cast<intptr_t>(fn); + intptr_t expected = kUninitialized; + const bool store_succeeded = hook_.compare_exchange_strong( + expected, value, std::memory_order_acq_rel, std::memory_order_acquire); + const bool same_value_already_stored = (expected == value); + return store_succeeded || same_value_already_stored; + } + + std::atomic<intptr_t> hook_; +#endif + + const FnPtr default_fn_; +}; + +#undef ABSL_HAVE_WORKING_ATOMIC_POINTER +#undef ABSL_HAVE_WORKING_CONSTEXPR_STATIC_INIT + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_ATOMIC_HOOK_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/atomic_hook_test.cc b/third_party/abseil_cpp/absl/base/internal/atomic_hook_test.cc new file mode 100644 index 000000000000..e577a8fd930b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/atomic_hook_test.cc @@ -0,0 +1,97 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/atomic_hook.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/internal/atomic_hook_test_helper.h" + +namespace { + +using ::testing::Eq; + +int value = 0; +void TestHook(int x) { value = x; } + +TEST(AtomicHookTest, NoDefaultFunction) { + ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + void (*)(int)> + hook; + value = 0; + + // Test the default DummyFunction. + EXPECT_TRUE(hook.Load() == nullptr); + EXPECT_EQ(value, 0); + hook(1); + EXPECT_EQ(value, 0); + + // Test a stored hook. + hook.Store(TestHook); + EXPECT_TRUE(hook.Load() == TestHook); + EXPECT_EQ(value, 0); + hook(1); + EXPECT_EQ(value, 1); + + // Calling Store() with the same hook should not crash. + hook.Store(TestHook); + EXPECT_TRUE(hook.Load() == TestHook); + EXPECT_EQ(value, 1); + hook(2); + EXPECT_EQ(value, 2); +} + +TEST(AtomicHookTest, WithDefaultFunction) { + // Set the default value to TestHook at compile-time. + ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + void (*)(int)> + hook(TestHook); + value = 0; + + // Test the default value is TestHook. + EXPECT_TRUE(hook.Load() == TestHook); + EXPECT_EQ(value, 0); + hook(1); + EXPECT_EQ(value, 1); + + // Calling Store() with the same hook should not crash. + hook.Store(TestHook); + EXPECT_TRUE(hook.Load() == TestHook); + EXPECT_EQ(value, 1); + hook(2); + EXPECT_EQ(value, 2); +} + +ABSL_CONST_INIT int override_func_calls = 0; +void OverrideFunc() { override_func_calls++; } +static struct OverrideInstaller { + OverrideInstaller() { absl::atomic_hook_internal::func.Store(OverrideFunc); } +} override_installer; + +TEST(AtomicHookTest, DynamicInitFromAnotherTU) { + // MSVC 14.2 doesn't do constexpr static init correctly; in particular it + // tends to sequence static init (i.e. defaults) of `AtomicHook` objects + // after their dynamic init (i.e. overrides), overwriting whatever value was + // written during dynamic init. This regression test validates the fix. + // https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html + EXPECT_THAT(absl::atomic_hook_internal::default_func_calls, Eq(0)); + EXPECT_THAT(override_func_calls, Eq(0)); + absl::atomic_hook_internal::func(); + EXPECT_THAT(absl::atomic_hook_internal::default_func_calls, Eq(0)); + EXPECT_THAT(override_func_calls, Eq(1)); + EXPECT_THAT(absl::atomic_hook_internal::func.Load(), Eq(OverrideFunc)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/atomic_hook_test_helper.cc b/third_party/abseil_cpp/absl/base/internal/atomic_hook_test_helper.cc new file mode 100644 index 000000000000..537d47cd2d2f --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/atomic_hook_test_helper.cc @@ -0,0 +1,32 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/atomic_hook_test_helper.h" + +#include "absl/base/attributes.h" +#include "absl/base/internal/atomic_hook.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace atomic_hook_internal { + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<VoidF> + func(DefaultFunc); +ABSL_CONST_INIT int default_func_calls = 0; +void DefaultFunc() { default_func_calls++; } +void RegisterFunc(VoidF f) { func.Store(f); } + +} // namespace atomic_hook_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/atomic_hook_test_helper.h b/third_party/abseil_cpp/absl/base/internal/atomic_hook_test_helper.h new file mode 100644 index 000000000000..3e72b4977d2f --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/atomic_hook_test_helper.h @@ -0,0 +1,34 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_ATOMIC_HOOK_TEST_HELPER_H_ +#define ABSL_BASE_ATOMIC_HOOK_TEST_HELPER_H_ + +#include "absl/base/internal/atomic_hook.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace atomic_hook_internal { + +using VoidF = void (*)(); +extern absl::base_internal::AtomicHook<VoidF> func; +extern int default_func_calls; +void DefaultFunc(); +void RegisterFunc(VoidF func); + +} // namespace atomic_hook_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_ATOMIC_HOOK_TEST_HELPER_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/bits.h b/third_party/abseil_cpp/absl/base/internal/bits.h new file mode 100644 index 000000000000..81648e2c3957 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/bits.h @@ -0,0 +1,219 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_BITS_H_ +#define ABSL_BASE_INTERNAL_BITS_H_ + +// This file contains bitwise ops which are implementation details of various +// absl libraries. + +#include <cstdint> + +#include "absl/base/config.h" + +// Clang on Windows has __builtin_clzll; otherwise we need to use the +// windows intrinsic functions. +#if defined(_MSC_VER) && !defined(__clang__) +#include <intrin.h> +#if defined(_M_X64) +#pragma intrinsic(_BitScanReverse64) +#pragma intrinsic(_BitScanForward64) +#endif +#pragma intrinsic(_BitScanReverse) +#pragma intrinsic(_BitScanForward) +#endif + +#include "absl/base/attributes.h" + +#if defined(_MSC_VER) && !defined(__clang__) +// We can achieve something similar to attribute((always_inline)) with MSVC by +// using the __forceinline keyword, however this is not perfect. MSVC is +// much less aggressive about inlining, and even with the __forceinline keyword. +#define ABSL_BASE_INTERNAL_FORCEINLINE __forceinline +#else +// Use default attribute inline. +#define ABSL_BASE_INTERNAL_FORCEINLINE inline ABSL_ATTRIBUTE_ALWAYS_INLINE +#endif + + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64Slow(uint64_t n) { + int zeroes = 60; + if (n >> 32) { + zeroes -= 32; + n >>= 32; + } + if (n >> 16) { + zeroes -= 16; + n >>= 16; + } + if (n >> 8) { + zeroes -= 8; + n >>= 8; + } + if (n >> 4) { + zeroes -= 4; + n >>= 4; + } + return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros64(uint64_t n) { +#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) + // MSVC does not have __buitin_clzll. Use _BitScanReverse64. + unsigned long result = 0; // NOLINT(runtime/int) + if (_BitScanReverse64(&result, n)) { + return 63 - result; + } + return 64; +#elif defined(_MSC_VER) && !defined(__clang__) + // MSVC does not have __buitin_clzll. Compose two calls to _BitScanReverse + unsigned long result = 0; // NOLINT(runtime/int) + if ((n >> 32) && + _BitScanReverse(&result, static_cast<unsigned long>(n >> 32))) { + return 31 - result; + } + if (_BitScanReverse(&result, static_cast<unsigned long>(n))) { + return 63 - result; + } + return 64; +#elif defined(__GNUC__) || defined(__clang__) + // Use __builtin_clzll, which uses the following instructions: + // x86: bsr + // ARM64: clz + // PPC: cntlzd + static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) + "__builtin_clzll does not take 64-bit arg"); + + // Handle 0 as a special case because __builtin_clzll(0) is undefined. + if (n == 0) { + return 64; + } + return __builtin_clzll(n); +#else + return CountLeadingZeros64Slow(n); +#endif +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32Slow(uint64_t n) { + int zeroes = 28; + if (n >> 16) { + zeroes -= 16; + n >>= 16; + } + if (n >> 8) { + zeroes -= 8; + n >>= 8; + } + if (n >> 4) { + zeroes -= 4; + n >>= 4; + } + return "\4\3\2\2\1\1\1\1\0\0\0\0\0\0\0"[n] + zeroes; +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountLeadingZeros32(uint32_t n) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long result = 0; // NOLINT(runtime/int) + if (_BitScanReverse(&result, n)) { + return 31 - result; + } + return 32; +#elif defined(__GNUC__) || defined(__clang__) + // Use __builtin_clz, which uses the following instructions: + // x86: bsr + // ARM64: clz + // PPC: cntlzd + static_assert(sizeof(int) == sizeof(n), + "__builtin_clz does not take 32-bit arg"); + + // Handle 0 as a special case because __builtin_clz(0) is undefined. + if (n == 0) { + return 32; + } + return __builtin_clz(n); +#else + return CountLeadingZeros32Slow(n); +#endif +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64Slow(uint64_t n) { + int c = 63; + n &= ~n + 1; + if (n & 0x00000000FFFFFFFF) c -= 32; + if (n & 0x0000FFFF0000FFFF) c -= 16; + if (n & 0x00FF00FF00FF00FF) c -= 8; + if (n & 0x0F0F0F0F0F0F0F0F) c -= 4; + if (n & 0x3333333333333333) c -= 2; + if (n & 0x5555555555555555) c -= 1; + return c; +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero64(uint64_t n) { +#if defined(_MSC_VER) && !defined(__clang__) && defined(_M_X64) + unsigned long result = 0; // NOLINT(runtime/int) + _BitScanForward64(&result, n); + return result; +#elif defined(_MSC_VER) && !defined(__clang__) + unsigned long result = 0; // NOLINT(runtime/int) + if (static_cast<uint32_t>(n) == 0) { + _BitScanForward(&result, static_cast<unsigned long>(n >> 32)); + return result + 32; + } + _BitScanForward(&result, static_cast<unsigned long>(n)); + return result; +#elif defined(__GNUC__) || defined(__clang__) + static_assert(sizeof(unsigned long long) == sizeof(n), // NOLINT(runtime/int) + "__builtin_ctzll does not take 64-bit arg"); + return __builtin_ctzll(n); +#else + return CountTrailingZerosNonZero64Slow(n); +#endif +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32Slow(uint32_t n) { + int c = 31; + n &= ~n + 1; + if (n & 0x0000FFFF) c -= 16; + if (n & 0x00FF00FF) c -= 8; + if (n & 0x0F0F0F0F) c -= 4; + if (n & 0x33333333) c -= 2; + if (n & 0x55555555) c -= 1; + return c; +} + +ABSL_BASE_INTERNAL_FORCEINLINE int CountTrailingZerosNonZero32(uint32_t n) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long result = 0; // NOLINT(runtime/int) + _BitScanForward(&result, n); + return result; +#elif defined(__GNUC__) || defined(__clang__) + static_assert(sizeof(int) == sizeof(n), + "__builtin_ctz does not take 32-bit arg"); + return __builtin_ctz(n); +#else + return CountTrailingZerosNonZero32Slow(n); +#endif +} + +#undef ABSL_BASE_INTERNAL_FORCEINLINE + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_BITS_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/bits_test.cc b/third_party/abseil_cpp/absl/base/internal/bits_test.cc new file mode 100644 index 000000000000..7855fa629724 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/bits_test.cc @@ -0,0 +1,97 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/bits.h" + +#include "gtest/gtest.h" + +namespace { + +int CLZ64(uint64_t n) { + int fast = absl::base_internal::CountLeadingZeros64(n); + int slow = absl::base_internal::CountLeadingZeros64Slow(n); + EXPECT_EQ(fast, slow) << n; + return fast; +} + +TEST(BitsTest, CountLeadingZeros64) { + EXPECT_EQ(64, CLZ64(uint64_t{})); + EXPECT_EQ(0, CLZ64(~uint64_t{})); + + for (int index = 0; index < 64; index++) { + uint64_t x = static_cast<uint64_t>(1) << index; + const auto cnt = 63 - index; + ASSERT_EQ(cnt, CLZ64(x)) << index; + ASSERT_EQ(cnt, CLZ64(x + x - 1)) << index; + } +} + +int CLZ32(uint32_t n) { + int fast = absl::base_internal::CountLeadingZeros32(n); + int slow = absl::base_internal::CountLeadingZeros32Slow(n); + EXPECT_EQ(fast, slow) << n; + return fast; +} + +TEST(BitsTest, CountLeadingZeros32) { + EXPECT_EQ(32, CLZ32(uint32_t{})); + EXPECT_EQ(0, CLZ32(~uint32_t{})); + + for (int index = 0; index < 32; index++) { + uint32_t x = static_cast<uint32_t>(1) << index; + const auto cnt = 31 - index; + ASSERT_EQ(cnt, CLZ32(x)) << index; + ASSERT_EQ(cnt, CLZ32(x + x - 1)) << index; + ASSERT_EQ(CLZ64(x), CLZ32(x) + 32); + } +} + +int CTZ64(uint64_t n) { + int fast = absl::base_internal::CountTrailingZerosNonZero64(n); + int slow = absl::base_internal::CountTrailingZerosNonZero64Slow(n); + EXPECT_EQ(fast, slow) << n; + return fast; +} + +TEST(BitsTest, CountTrailingZerosNonZero64) { + EXPECT_EQ(0, CTZ64(~uint64_t{})); + + for (int index = 0; index < 64; index++) { + uint64_t x = static_cast<uint64_t>(1) << index; + const auto cnt = index; + ASSERT_EQ(cnt, CTZ64(x)) << index; + ASSERT_EQ(cnt, CTZ64(~(x - 1))) << index; + } +} + +int CTZ32(uint32_t n) { + int fast = absl::base_internal::CountTrailingZerosNonZero32(n); + int slow = absl::base_internal::CountTrailingZerosNonZero32Slow(n); + EXPECT_EQ(fast, slow) << n; + return fast; +} + +TEST(BitsTest, CountTrailingZerosNonZero32) { + EXPECT_EQ(0, CTZ32(~uint32_t{})); + + for (int index = 0; index < 32; index++) { + uint32_t x = static_cast<uint32_t>(1) << index; + const auto cnt = index; + ASSERT_EQ(cnt, CTZ32(x)) << index; + ASSERT_EQ(cnt, CTZ32(~(x - 1))) << index; + } +} + + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/cmake_thread_test.cc b/third_party/abseil_cpp/absl/base/internal/cmake_thread_test.cc new file mode 100644 index 000000000000..f70bb24eb7b4 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/cmake_thread_test.cc @@ -0,0 +1,22 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <iostream> +#include "absl/base/internal/thread_identity.h" + +int main() { + auto* tid = absl::base_internal::CurrentThreadIdentityIfPresent(); + // Make sure the above call can't be optimized out + std::cout << (void*)tid << std::endl; +} diff --git a/third_party/abseil_cpp/absl/base/internal/cycleclock.cc b/third_party/abseil_cpp/absl/base/internal/cycleclock.cc new file mode 100644 index 000000000000..0e65005b8914 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/cycleclock.cc @@ -0,0 +1,107 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The implementation of CycleClock::Frequency. +// +// NOTE: only i386 and x86_64 have been well tested. +// PPC, sparc, alpha, and ia64 are based on +// http://peter.kuscsik.com/wordpress/?p=14 +// with modifications by m3b. See also +// https://setisvn.ssl.berkeley.edu/svn/lib/fftw-3.0.1/kernel/cycle.h + +#include "absl/base/internal/cycleclock.h" + +#include <atomic> +#include <chrono> // NOLINT(build/c++11) + +#include "absl/base/internal/unscaledcycleclock.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +#if ABSL_USE_UNSCALED_CYCLECLOCK + +namespace { + +#ifdef NDEBUG +#ifdef ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY +// Not debug mode and the UnscaledCycleClock frequency is the CPU +// frequency. Scale the CycleClock to prevent overflow if someone +// tries to represent the time as cycles since the Unix epoch. +static constexpr int32_t kShift = 1; +#else +// Not debug mode and the UnscaledCycleClock isn't operating at the +// raw CPU frequency. There is no need to do any scaling, so don't +// needlessly sacrifice precision. +static constexpr int32_t kShift = 0; +#endif +#else +// In debug mode use a different shift to discourage depending on a +// particular shift value. +static constexpr int32_t kShift = 2; +#endif + +static constexpr double kFrequencyScale = 1.0 / (1 << kShift); +static std::atomic<CycleClockSourceFunc> cycle_clock_source; + +CycleClockSourceFunc LoadCycleClockSource() { + // Optimize for the common case (no callback) by first doing a relaxed load; + // this is significantly faster on non-x86 platforms. + if (cycle_clock_source.load(std::memory_order_relaxed) == nullptr) { + return nullptr; + } + // This corresponds to the store(std::memory_order_release) in + // CycleClockSource::Register, and makes sure that any updates made prior to + // registering the callback are visible to this thread before the callback is + // invoked. + return cycle_clock_source.load(std::memory_order_acquire); +} + +} // namespace + +int64_t CycleClock::Now() { + auto fn = LoadCycleClockSource(); + if (fn == nullptr) { + return base_internal::UnscaledCycleClock::Now() >> kShift; + } + return fn() >> kShift; +} + +double CycleClock::Frequency() { + return kFrequencyScale * base_internal::UnscaledCycleClock::Frequency(); +} + +void CycleClockSource::Register(CycleClockSourceFunc source) { + // Corresponds to the load(std::memory_order_acquire) in LoadCycleClockSource. + cycle_clock_source.store(source, std::memory_order_release); +} + +#else + +int64_t CycleClock::Now() { + return std::chrono::duration_cast<std::chrono::nanoseconds>( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} + +double CycleClock::Frequency() { + return 1e9; +} + +#endif + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/cycleclock.h b/third_party/abseil_cpp/absl/base/internal/cycleclock.h new file mode 100644 index 000000000000..a18b58444560 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/cycleclock.h @@ -0,0 +1,94 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// ----------------------------------------------------------------------------- +// File: cycleclock.h +// ----------------------------------------------------------------------------- +// +// This header file defines a `CycleClock`, which yields the value and frequency +// of a cycle counter that increments at a rate that is approximately constant. +// +// NOTE: +// +// The cycle counter frequency is not necessarily related to the core clock +// frequency and should not be treated as such. That is, `CycleClock` cycles are +// not necessarily "CPU cycles" and code should not rely on that behavior, even +// if experimentally observed. +// +// An arbitrary offset may have been added to the counter at power on. +// +// On some platforms, the rate and offset of the counter may differ +// slightly when read from different CPUs of a multiprocessor. Usually, +// we try to ensure that the operating system adjusts values periodically +// so that values agree approximately. If you need stronger guarantees, +// consider using alternate interfaces. +// +// The CPU is not required to maintain the ordering of a cycle counter read +// with respect to surrounding instructions. + +#ifndef ABSL_BASE_INTERNAL_CYCLECLOCK_H_ +#define ABSL_BASE_INTERNAL_CYCLECLOCK_H_ + +#include <cstdint> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// ----------------------------------------------------------------------------- +// CycleClock +// ----------------------------------------------------------------------------- +class CycleClock { + public: + // CycleClock::Now() + // + // Returns the value of a cycle counter that counts at a rate that is + // approximately constant. + static int64_t Now(); + + // CycleClock::Frequency() + // + // Returns the amount by which `CycleClock::Now()` increases per second. Note + // that this value may not necessarily match the core CPU clock frequency. + static double Frequency(); + + private: + CycleClock() = delete; // no instances + CycleClock(const CycleClock&) = delete; + CycleClock& operator=(const CycleClock&) = delete; +}; + +using CycleClockSourceFunc = int64_t (*)(); + +class CycleClockSource { + private: + // CycleClockSource::Register() + // + // Register a function that provides an alternate source for the unscaled CPU + // cycle count value. The source function must be async signal safe, must not + // call CycleClock::Now(), and must have a frequency that matches that of the + // unscaled clock used by CycleClock. A nullptr value resets CycleClock to use + // the default source. + static void Register(CycleClockSourceFunc source); +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_CYCLECLOCK_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/direct_mmap.h b/third_party/abseil_cpp/absl/base/internal/direct_mmap.h new file mode 100644 index 000000000000..16accf096604 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/direct_mmap.h @@ -0,0 +1,166 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Functions for directly invoking mmap() via syscall, avoiding the case where +// mmap() has been locally overridden. + +#ifndef ABSL_BASE_INTERNAL_DIRECT_MMAP_H_ +#define ABSL_BASE_INTERNAL_DIRECT_MMAP_H_ + +#include "absl/base/config.h" + +#if ABSL_HAVE_MMAP + +#include <sys/mman.h> + +#ifdef __linux__ + +#include <sys/types.h> +#ifdef __BIONIC__ +#include <sys/syscall.h> +#else +#include <syscall.h> +#endif + +#include <linux/unistd.h> +#include <unistd.h> +#include <cerrno> +#include <cstdarg> +#include <cstdint> + +#ifdef __mips__ +// Include definitions of the ABI currently in use. +#ifdef __BIONIC__ +// Android doesn't have sgidefs.h, but does have asm/sgidefs.h, which has the +// definitions we need. +#include <asm/sgidefs.h> +#else +#include <sgidefs.h> +#endif // __BIONIC__ +#endif // __mips__ + +// SYS_mmap and SYS_munmap are not defined in Android. +#ifdef __BIONIC__ +extern "C" void* __mmap2(void*, size_t, int, int, int, size_t); +#if defined(__NR_mmap) && !defined(SYS_mmap) +#define SYS_mmap __NR_mmap +#endif +#ifndef SYS_munmap +#define SYS_munmap __NR_munmap +#endif +#endif // __BIONIC__ + +#if defined(__NR_mmap2) && !defined(SYS_mmap2) +#define SYS_mmap2 __NR_mmap2 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// Platform specific logic extracted from +// https://chromium.googlesource.com/linux-syscall-support/+/master/linux_syscall_support.h +inline void* DirectMmap(void* start, size_t length, int prot, int flags, int fd, + off64_t offset) noexcept { +#if defined(__i386__) || defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) || \ + (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || \ + (defined(__PPC__) && !defined(__PPC64__)) || \ + (defined(__riscv) && __riscv_xlen == 32) || \ + (defined(__s390__) && !defined(__s390x__)) + // On these architectures, implement mmap with mmap2. + static int pagesize = 0; + if (pagesize == 0) { +#if defined(__wasm__) || defined(__asmjs__) + pagesize = getpagesize(); +#else + pagesize = sysconf(_SC_PAGESIZE); +#endif + } + if (offset < 0 || offset % pagesize != 0) { + errno = EINVAL; + return MAP_FAILED; + } +#ifdef __BIONIC__ + // SYS_mmap2 has problems on Android API level <= 16. + // Workaround by invoking __mmap2() instead. + return __mmap2(start, length, prot, flags, fd, offset / pagesize); +#else + return reinterpret_cast<void*>( + syscall(SYS_mmap2, start, length, prot, flags, fd, + static_cast<off_t>(offset / pagesize))); +#endif +#elif defined(__s390x__) + // On s390x, mmap() arguments are passed in memory. + unsigned long buf[6] = {reinterpret_cast<unsigned long>(start), // NOLINT + static_cast<unsigned long>(length), // NOLINT + static_cast<unsigned long>(prot), // NOLINT + static_cast<unsigned long>(flags), // NOLINT + static_cast<unsigned long>(fd), // NOLINT + static_cast<unsigned long>(offset)}; // NOLINT + return reinterpret_cast<void*>(syscall(SYS_mmap, buf)); +#elif defined(__x86_64__) +// The x32 ABI has 32 bit longs, but the syscall interface is 64 bit. +// We need to explicitly cast to an unsigned 64 bit type to avoid implicit +// sign extension. We can't cast pointers directly because those are +// 32 bits, and gcc will dump ugly warnings about casting from a pointer +// to an integer of a different size. We also need to make sure __off64_t +// isn't truncated to 32-bits under x32. +#define MMAP_SYSCALL_ARG(x) ((uint64_t)(uintptr_t)(x)) + return reinterpret_cast<void*>( + syscall(SYS_mmap, MMAP_SYSCALL_ARG(start), MMAP_SYSCALL_ARG(length), + MMAP_SYSCALL_ARG(prot), MMAP_SYSCALL_ARG(flags), + MMAP_SYSCALL_ARG(fd), static_cast<uint64_t>(offset))); +#undef MMAP_SYSCALL_ARG +#else // Remaining 64-bit aritectures. + static_assert(sizeof(unsigned long) == 8, "Platform is not 64-bit"); + return reinterpret_cast<void*>( + syscall(SYS_mmap, start, length, prot, flags, fd, offset)); +#endif +} + +inline int DirectMunmap(void* start, size_t length) { + return static_cast<int>(syscall(SYS_munmap, start, length)); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#else // !__linux__ + +// For non-linux platforms where we have mmap, just dispatch directly to the +// actual mmap()/munmap() methods. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +inline void* DirectMmap(void* start, size_t length, int prot, int flags, int fd, + off_t offset) { + return mmap(start, length, prot, flags, fd, offset); +} + +inline int DirectMunmap(void* start, size_t length) { + return munmap(start, length); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // __linux__ + +#endif // ABSL_HAVE_MMAP + +#endif // ABSL_BASE_INTERNAL_DIRECT_MMAP_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/dynamic_annotations.h b/third_party/abseil_cpp/absl/base/internal/dynamic_annotations.h new file mode 100644 index 000000000000..b23c5ec1c41e --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/dynamic_annotations.h @@ -0,0 +1,398 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file defines dynamic annotations for use with dynamic analysis tool +// such as valgrind, PIN, etc. +// +// Dynamic annotation is a source code annotation that affects the generated +// code (that is, the annotation is not a comment). Each such annotation is +// attached to a particular instruction and/or to a particular object (address) +// in the program. +// +// The annotations that should be used by users are macros in all upper-case +// (e.g., ANNOTATE_THREAD_NAME). +// +// Actual implementation of these macros may differ depending on the dynamic +// analysis tool being used. +// +// This file supports the following configurations: +// - Dynamic Annotations enabled (with static thread-safety warnings disabled). +// In this case, macros expand to functions implemented by Thread Sanitizer, +// when building with TSan. When not provided an external implementation, +// dynamic_annotations.cc provides no-op implementations. +// +// - Static Clang thread-safety warnings enabled. +// When building with a Clang compiler that supports thread-safety warnings, +// a subset of annotations can be statically-checked at compile-time. We +// expand these macros to static-inline functions that can be analyzed for +// thread-safety, but afterwards elided when building the final binary. +// +// - All annotations are disabled. +// If neither Dynamic Annotations nor Clang thread-safety warnings are +// enabled, then all annotation-macros expand to empty. + +#ifndef ABSL_BASE_INTERNAL_DYNAMIC_ANNOTATIONS_H_ +#define ABSL_BASE_INTERNAL_DYNAMIC_ANNOTATIONS_H_ + +#include <stddef.h> + +#include "absl/base/config.h" + +// ------------------------------------------------------------------------- +// Decide which features are enabled + +#ifndef DYNAMIC_ANNOTATIONS_ENABLED +#define DYNAMIC_ANNOTATIONS_ENABLED 0 +#endif + +#if defined(__clang__) && !defined(SWIG) +#define ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED 1 +#endif + +#if DYNAMIC_ANNOTATIONS_ENABLED != 0 + +#define ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED 1 +#define ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED 1 +#define ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED 1 +#define ABSL_INTERNAL_ANNOTALYSIS_ENABLED 0 +#define ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED 1 + +#else + +#define ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED 0 +#define ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED 0 +#define ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED 0 + +// Clang provides limited support for static thread-safety analysis through a +// feature called Annotalysis. We configure macro-definitions according to +// whether Annotalysis support is available. When running in opt-mode, GCC +// will issue a warning, if these attributes are compiled. Only include them +// when compiling using Clang. + +// ANNOTALYSIS_ENABLED == 1 when IGNORE_READ_ATTRIBUTE_ENABLED == 1 +#define ABSL_INTERNAL_ANNOTALYSIS_ENABLED \ + defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) +// Read/write annotations are enabled in Annotalysis mode; disabled otherwise. +#define ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED \ + ABSL_INTERNAL_ANNOTALYSIS_ENABLED +#endif + +// Memory annotations are also made available to LLVM's Memory Sanitizer +#if defined(ABSL_HAVE_MEMORY_SANITIZER) && !defined(__native_client__) +#define ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED 1 +#endif + +#ifndef ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED +#define ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED 0 +#endif + +#ifdef __cplusplus +#define ABSL_INTERNAL_BEGIN_EXTERN_C extern "C" { +#define ABSL_INTERNAL_END_EXTERN_C } // extern "C" +#define ABSL_INTERNAL_GLOBAL_SCOPED(F) ::F +#define ABSL_INTERNAL_STATIC_INLINE inline +#else +#define ABSL_INTERNAL_BEGIN_EXTERN_C // empty +#define ABSL_INTERNAL_END_EXTERN_C // empty +#define ABSL_INTERNAL_GLOBAL_SCOPED(F) F +#define ABSL_INTERNAL_STATIC_INLINE static inline +#endif + +// ------------------------------------------------------------------------- +// Define race annotations. + +#if ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED == 1 + +// ------------------------------------------------------------- +// Annotations that suppress errors. It is usually better to express the +// program's synchronization using the other annotations, but these can be used +// when all else fails. + +// Report that we may have a benign race at `pointer`, with size +// "sizeof(*(pointer))". `pointer` must be a non-void* pointer. Insert at the +// point where `pointer` has been allocated, preferably close to the point +// where the race happens. See also ANNOTATE_BENIGN_RACE_STATIC. +#define ANNOTATE_BENIGN_RACE(pointer, description) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateBenignRaceSized) \ + (__FILE__, __LINE__, pointer, sizeof(*(pointer)), description) + +// Same as ANNOTATE_BENIGN_RACE(`address`, `description`), but applies to +// the memory range [`address`, `address`+`size`). +#define ANNOTATE_BENIGN_RACE_SIZED(address, size, description) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateBenignRaceSized) \ + (__FILE__, __LINE__, address, size, description) + +// Enable (`enable`!=0) or disable (`enable`==0) race detection for all threads. +// This annotation could be useful if you want to skip expensive race analysis +// during some period of program execution, e.g. during initialization. +#define ANNOTATE_ENABLE_RACE_DETECTION(enable) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateEnableRaceDetection) \ + (__FILE__, __LINE__, enable) + +// ------------------------------------------------------------- +// Annotations useful for debugging. + +// Report the current thread `name` to a race detector. +#define ANNOTATE_THREAD_NAME(name) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateThreadName)(__FILE__, __LINE__, name) + +// ------------------------------------------------------------- +// Annotations useful when implementing locks. They are not normally needed by +// modules that merely use locks. The `lock` argument is a pointer to the lock +// object. + +// Report that a lock has been created at address `lock`. +#define ANNOTATE_RWLOCK_CREATE(lock) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockCreate)(__FILE__, __LINE__, lock) + +// Report that a linker initialized lock has been created at address `lock`. +#ifdef ABSL_HAVE_THREAD_SANITIZER +#define ANNOTATE_RWLOCK_CREATE_STATIC(lock) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockCreateStatic) \ + (__FILE__, __LINE__, lock) +#else +#define ANNOTATE_RWLOCK_CREATE_STATIC(lock) ANNOTATE_RWLOCK_CREATE(lock) +#endif + +// Report that the lock at address `lock` is about to be destroyed. +#define ANNOTATE_RWLOCK_DESTROY(lock) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockDestroy)(__FILE__, __LINE__, lock) + +// Report that the lock at address `lock` has been acquired. +// `is_w`=1 for writer lock, `is_w`=0 for reader lock. +#define ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockAcquired) \ + (__FILE__, __LINE__, lock, is_w) + +// Report that the lock at address `lock` is about to be released. +// `is_w`=1 for writer lock, `is_w`=0 for reader lock. +#define ANNOTATE_RWLOCK_RELEASED(lock, is_w) \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateRWLockReleased) \ + (__FILE__, __LINE__, lock, is_w) + +// Apply ANNOTATE_BENIGN_RACE_SIZED to a static variable `static_var`. +#define ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ + namespace { \ + class static_var##_annotator { \ + public: \ + static_var##_annotator() { \ + ANNOTATE_BENIGN_RACE_SIZED(&static_var, sizeof(static_var), \ + #static_var ": " description); \ + } \ + }; \ + static static_var##_annotator the##static_var##_annotator; \ + } // namespace + +#else // ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED == 0 + +#define ANNOTATE_RWLOCK_CREATE(lock) // empty +#define ANNOTATE_RWLOCK_CREATE_STATIC(lock) // empty +#define ANNOTATE_RWLOCK_DESTROY(lock) // empty +#define ANNOTATE_RWLOCK_ACQUIRED(lock, is_w) // empty +#define ANNOTATE_RWLOCK_RELEASED(lock, is_w) // empty +#define ANNOTATE_BENIGN_RACE(address, description) // empty +#define ANNOTATE_BENIGN_RACE_SIZED(address, size, description) // empty +#define ANNOTATE_THREAD_NAME(name) // empty +#define ANNOTATE_ENABLE_RACE_DETECTION(enable) // empty +#define ANNOTATE_BENIGN_RACE_STATIC(static_var, description) // empty + +#endif // ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED + +// ------------------------------------------------------------------------- +// Define memory annotations. + +#if ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED == 1 + +#include <sanitizer/msan_interface.h> + +#define ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ + __msan_unpoison(address, size) + +#define ANNOTATE_MEMORY_IS_UNINITIALIZED(address, size) \ + __msan_allocated_memory(address, size) + +#else // ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED == 0 + +#if DYNAMIC_ANNOTATIONS_ENABLED == 1 +#define ANNOTATE_MEMORY_IS_INITIALIZED(address, size) \ + do { \ + (void)(address); \ + (void)(size); \ + } while (0) +#define ANNOTATE_MEMORY_IS_UNINITIALIZED(address, size) \ + do { \ + (void)(address); \ + (void)(size); \ + } while (0) +#else +#define ANNOTATE_MEMORY_IS_INITIALIZED(address, size) // empty +#define ANNOTATE_MEMORY_IS_UNINITIALIZED(address, size) // empty +#endif + +#endif // ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED + +// ------------------------------------------------------------------------- +// Define IGNORE_READS_BEGIN/_END attributes. + +#if defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) + +#define ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE \ + __attribute((exclusive_lock_function("*"))) +#define ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE \ + __attribute((unlock_function("*"))) + +#else // !defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) + +#define ABSL_INTERNAL_IGNORE_READS_BEGIN_ATTRIBUTE // empty +#define ABSL_INTERNAL_IGNORE_READS_END_ATTRIBUTE // empty + +#endif // defined(ABSL_INTERNAL_IGNORE_READS_ATTRIBUTE_ENABLED) + +// ------------------------------------------------------------------------- +// Define IGNORE_READS_BEGIN/_END annotations. + +#if ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED == 1 + +// Request the analysis tool to ignore all reads in the current thread until +// ANNOTATE_IGNORE_READS_END is called. Useful to ignore intentional racey +// reads, while still checking other reads and all writes. +// See also ANNOTATE_UNPROTECTED_READ. +#define ANNOTATE_IGNORE_READS_BEGIN() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreReadsBegin)(__FILE__, __LINE__) + +// Stop ignoring reads. +#define ANNOTATE_IGNORE_READS_END() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreReadsEnd)(__FILE__, __LINE__) + +#elif defined(ABSL_INTERNAL_ANNOTALYSIS_ENABLED) + +// When Annotalysis is enabled without Dynamic Annotations, the use of +// static-inline functions allows the annotations to be read at compile-time, +// while still letting the compiler elide the functions from the final build. +// +// TODO(delesley) -- The exclusive lock here ignores writes as well, but +// allows IGNORE_READS_AND_WRITES to work properly. + +#define ANNOTATE_IGNORE_READS_BEGIN() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AbslInternalAnnotateIgnoreReadsBegin)() + +#define ANNOTATE_IGNORE_READS_END() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AbslInternalAnnotateIgnoreReadsEnd)() + +#else + +#define ANNOTATE_IGNORE_READS_BEGIN() // empty +#define ANNOTATE_IGNORE_READS_END() // empty + +#endif + +// ------------------------------------------------------------------------- +// Define IGNORE_WRITES_BEGIN/_END annotations. + +#if ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED == 1 + +// Similar to ANNOTATE_IGNORE_READS_BEGIN, but ignore writes instead. +#define ANNOTATE_IGNORE_WRITES_BEGIN() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreWritesBegin)(__FILE__, __LINE__) + +// Stop ignoring writes. +#define ANNOTATE_IGNORE_WRITES_END() \ + ABSL_INTERNAL_GLOBAL_SCOPED(AnnotateIgnoreWritesEnd)(__FILE__, __LINE__) + +#else + +#define ANNOTATE_IGNORE_WRITES_BEGIN() // empty +#define ANNOTATE_IGNORE_WRITES_END() // empty + +#endif + +// ------------------------------------------------------------------------- +// Define the ANNOTATE_IGNORE_READS_AND_WRITES_* annotations using the more +// primitive annotations defined above. +// +// Instead of doing +// ANNOTATE_IGNORE_READS_BEGIN(); +// ... = x; +// ANNOTATE_IGNORE_READS_END(); +// one can use +// ... = ANNOTATE_UNPROTECTED_READ(x); + +#if defined(ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED) + +// Start ignoring all memory accesses (both reads and writes). +#define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() \ + do { \ + ANNOTATE_IGNORE_READS_BEGIN(); \ + ANNOTATE_IGNORE_WRITES_BEGIN(); \ + } while (0) + +// Stop ignoring both reads and writes. +#define ANNOTATE_IGNORE_READS_AND_WRITES_END() \ + do { \ + ANNOTATE_IGNORE_WRITES_END(); \ + ANNOTATE_IGNORE_READS_END(); \ + } while (0) + +#ifdef __cplusplus +// ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. +#define ANNOTATE_UNPROTECTED_READ(x) \ + absl::base_internal::AnnotateUnprotectedRead(x) + +#endif + +#else + +#define ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN() // empty +#define ANNOTATE_IGNORE_READS_AND_WRITES_END() // empty +#define ANNOTATE_UNPROTECTED_READ(x) (x) + +#endif + +// ------------------------------------------------------------------------- +// Address sanitizer annotations + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +// Describe the current state of a contiguous container such as e.g. +// std::vector or std::string. For more details see +// sanitizer/common_interface_defs.h, which is provided by the compiler. +#include <sanitizer/common_interface_defs.h> + +#define ANNOTATE_CONTIGUOUS_CONTAINER(beg, end, old_mid, new_mid) \ + __sanitizer_annotate_contiguous_container(beg, end, old_mid, new_mid) +#define ADDRESS_SANITIZER_REDZONE(name) \ + struct { \ + char x[8] __attribute__((aligned(8))); \ + } name + +#else + +#define ANNOTATE_CONTIGUOUS_CONTAINER(beg, end, old_mid, new_mid) +#define ADDRESS_SANITIZER_REDZONE(name) static_assert(true, "") + +#endif // ABSL_HAVE_ADDRESS_SANITIZER + +// ------------------------------------------------------------------------- +// Undefine the macros intended only for this file. + +#undef ABSL_INTERNAL_RACE_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_MEMORY_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_READS_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_WRITES_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_ANNOTALYSIS_ENABLED +#undef ABSL_INTERNAL_READS_WRITES_ANNOTATIONS_ENABLED +#undef ABSL_INTERNAL_BEGIN_EXTERN_C +#undef ABSL_INTERNAL_END_EXTERN_C +#undef ABSL_INTERNAL_STATIC_INLINE + +#endif // ABSL_BASE_INTERNAL_DYNAMIC_ANNOTATIONS_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/endian.h b/third_party/abseil_cpp/absl/base/internal/endian.h new file mode 100644 index 000000000000..9677530e8de3 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/endian.h @@ -0,0 +1,266 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_ENDIAN_H_ +#define ABSL_BASE_INTERNAL_ENDIAN_H_ + +// The following guarantees declaration of the byte swap functions +#ifdef _MSC_VER +#include <stdlib.h> // NOLINT(build/include) +#elif defined(__FreeBSD__) +#include <sys/endian.h> +#elif defined(__GLIBC__) +#include <byteswap.h> // IWYU pragma: export +#endif + +#include <cstdint> +#include "absl/base/config.h" +#include "absl/base/internal/unaligned_access.h" +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Use compiler byte-swapping intrinsics if they are available. 32-bit +// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. +// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0. +// For simplicity, we enable them all only for GCC 4.8.0 or later. +#if defined(__clang__) || \ + (defined(__GNUC__) && \ + ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5)) +inline uint64_t gbswap_64(uint64_t host_int) { + return __builtin_bswap64(host_int); +} +inline uint32_t gbswap_32(uint32_t host_int) { + return __builtin_bswap32(host_int); +} +inline uint16_t gbswap_16(uint16_t host_int) { + return __builtin_bswap16(host_int); +} + +#elif defined(_MSC_VER) +inline uint64_t gbswap_64(uint64_t host_int) { + return _byteswap_uint64(host_int); +} +inline uint32_t gbswap_32(uint32_t host_int) { + return _byteswap_ulong(host_int); +} +inline uint16_t gbswap_16(uint16_t host_int) { + return _byteswap_ushort(host_int); +} + +#else +inline uint64_t gbswap_64(uint64_t host_int) { +#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) + // Adapted from /usr/include/byteswap.h. Not available on Mac. + if (__builtin_constant_p(host_int)) { + return __bswap_constant_64(host_int); + } else { + uint64_t result; + __asm__("bswap %0" : "=r"(result) : "0"(host_int)); + return result; + } +#elif defined(__GLIBC__) + return bswap_64(host_int); +#else + return (((host_int & uint64_t{0xFF}) << 56) | + ((host_int & uint64_t{0xFF00}) << 40) | + ((host_int & uint64_t{0xFF0000}) << 24) | + ((host_int & uint64_t{0xFF000000}) << 8) | + ((host_int & uint64_t{0xFF00000000}) >> 8) | + ((host_int & uint64_t{0xFF0000000000}) >> 24) | + ((host_int & uint64_t{0xFF000000000000}) >> 40) | + ((host_int & uint64_t{0xFF00000000000000}) >> 56)); +#endif // bswap_64 +} + +inline uint32_t gbswap_32(uint32_t host_int) { +#if defined(__GLIBC__) + return bswap_32(host_int); +#else + return (((host_int & uint32_t{0xFF}) << 24) | + ((host_int & uint32_t{0xFF00}) << 8) | + ((host_int & uint32_t{0xFF0000}) >> 8) | + ((host_int & uint32_t{0xFF000000}) >> 24)); +#endif +} + +inline uint16_t gbswap_16(uint16_t host_int) { +#if defined(__GLIBC__) + return bswap_16(host_int); +#else + return (((host_int & uint16_t{0xFF}) << 8) | + ((host_int & uint16_t{0xFF00}) >> 8)); +#endif +} + +#endif // intrinsics available + +#ifdef ABSL_IS_LITTLE_ENDIAN + +// Definitions for ntohl etc. that don't require us to include +// netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather +// than just #defining them because in debug mode, gcc doesn't +// correctly handle the (rather involved) definitions of bswap_32. +// gcc guarantees that inline functions are as fast as macros, so +// this isn't a performance hit. +inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); } +inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); } +inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); } + +#elif defined ABSL_IS_BIG_ENDIAN + +// These definitions are simpler on big-endian machines +// These are functions instead of macros to avoid self-assignment warnings +// on calls such as "i = ghtnol(i);". This also provides type checking. +inline uint16_t ghtons(uint16_t x) { return x; } +inline uint32_t ghtonl(uint32_t x) { return x; } +inline uint64_t ghtonll(uint64_t x) { return x; } + +#else +#error \ + "Unsupported byte order: Either ABSL_IS_BIG_ENDIAN or " \ + "ABSL_IS_LITTLE_ENDIAN must be defined" +#endif // byte order + +inline uint16_t gntohs(uint16_t x) { return ghtons(x); } +inline uint32_t gntohl(uint32_t x) { return ghtonl(x); } +inline uint64_t gntohll(uint64_t x) { return ghtonll(x); } + +// Utilities to convert numbers between the current hosts's native byte +// order and little-endian byte order +// +// Load/Store methods are alignment safe +namespace little_endian { +// Conversion functions. +#ifdef ABSL_IS_LITTLE_ENDIAN + +inline uint16_t FromHost16(uint16_t x) { return x; } +inline uint16_t ToHost16(uint16_t x) { return x; } + +inline uint32_t FromHost32(uint32_t x) { return x; } +inline uint32_t ToHost32(uint32_t x) { return x; } + +inline uint64_t FromHost64(uint64_t x) { return x; } +inline uint64_t ToHost64(uint64_t x) { return x; } + +inline constexpr bool IsLittleEndian() { return true; } + +#elif defined ABSL_IS_BIG_ENDIAN + +inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } +inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } + +inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } +inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } + +inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } +inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } + +inline constexpr bool IsLittleEndian() { return false; } + +#endif /* ENDIAN */ + +// Functions to do unaligned loads and stores in little-endian order. +inline uint16_t Load16(const void *p) { + return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); +} + +inline void Store16(void *p, uint16_t v) { + ABSL_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); +} + +inline uint32_t Load32(const void *p) { + return ToHost32(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); +} + +inline void Store32(void *p, uint32_t v) { + ABSL_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); +} + +inline uint64_t Load64(const void *p) { + return ToHost64(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); +} + +inline void Store64(void *p, uint64_t v) { + ABSL_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); +} + +} // namespace little_endian + +// Utilities to convert numbers between the current hosts's native byte +// order and big-endian byte order (same as network byte order) +// +// Load/Store methods are alignment safe +namespace big_endian { +#ifdef ABSL_IS_LITTLE_ENDIAN + +inline uint16_t FromHost16(uint16_t x) { return gbswap_16(x); } +inline uint16_t ToHost16(uint16_t x) { return gbswap_16(x); } + +inline uint32_t FromHost32(uint32_t x) { return gbswap_32(x); } +inline uint32_t ToHost32(uint32_t x) { return gbswap_32(x); } + +inline uint64_t FromHost64(uint64_t x) { return gbswap_64(x); } +inline uint64_t ToHost64(uint64_t x) { return gbswap_64(x); } + +inline constexpr bool IsLittleEndian() { return true; } + +#elif defined ABSL_IS_BIG_ENDIAN + +inline uint16_t FromHost16(uint16_t x) { return x; } +inline uint16_t ToHost16(uint16_t x) { return x; } + +inline uint32_t FromHost32(uint32_t x) { return x; } +inline uint32_t ToHost32(uint32_t x) { return x; } + +inline uint64_t FromHost64(uint64_t x) { return x; } +inline uint64_t ToHost64(uint64_t x) { return x; } + +inline constexpr bool IsLittleEndian() { return false; } + +#endif /* ENDIAN */ + +// Functions to do unaligned loads and stores in big-endian order. +inline uint16_t Load16(const void *p) { + return ToHost16(ABSL_INTERNAL_UNALIGNED_LOAD16(p)); +} + +inline void Store16(void *p, uint16_t v) { + ABSL_INTERNAL_UNALIGNED_STORE16(p, FromHost16(v)); +} + +inline uint32_t Load32(const void *p) { + return ToHost32(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); +} + +inline void Store32(void *p, uint32_t v) { + ABSL_INTERNAL_UNALIGNED_STORE32(p, FromHost32(v)); +} + +inline uint64_t Load64(const void *p) { + return ToHost64(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); +} + +inline void Store64(void *p, uint64_t v) { + ABSL_INTERNAL_UNALIGNED_STORE64(p, FromHost64(v)); +} + +} // namespace big_endian + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_ENDIAN_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/endian_test.cc b/third_party/abseil_cpp/absl/base/internal/endian_test.cc new file mode 100644 index 000000000000..a1691b1f82c0 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/endian_test.cc @@ -0,0 +1,263 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/endian.h" + +#include <algorithm> +#include <cstdint> +#include <limits> +#include <random> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +const uint64_t kInitialNumber{0x0123456789abcdef}; +const uint64_t k64Value{kInitialNumber}; +const uint32_t k32Value{0x01234567}; +const uint16_t k16Value{0x0123}; +const int kNumValuesToTest = 1000000; +const int kRandomSeed = 12345; + +#if defined(ABSL_IS_BIG_ENDIAN) +const uint64_t kInitialInNetworkOrder{kInitialNumber}; +const uint64_t k64ValueLE{0xefcdab8967452301}; +const uint32_t k32ValueLE{0x67452301}; +const uint16_t k16ValueLE{0x2301}; + +const uint64_t k64ValueBE{kInitialNumber}; +const uint32_t k32ValueBE{k32Value}; +const uint16_t k16ValueBE{k16Value}; +#elif defined(ABSL_IS_LITTLE_ENDIAN) +const uint64_t kInitialInNetworkOrder{0xefcdab8967452301}; +const uint64_t k64ValueLE{kInitialNumber}; +const uint32_t k32ValueLE{k32Value}; +const uint16_t k16ValueLE{k16Value}; + +const uint64_t k64ValueBE{0xefcdab8967452301}; +const uint32_t k32ValueBE{0x67452301}; +const uint16_t k16ValueBE{0x2301}; +#endif + +std::vector<uint16_t> GenerateAllUint16Values() { + std::vector<uint16_t> result; + result.reserve(size_t{1} << (sizeof(uint16_t) * 8)); + for (uint32_t i = std::numeric_limits<uint16_t>::min(); + i <= std::numeric_limits<uint16_t>::max(); ++i) { + result.push_back(static_cast<uint16_t>(i)); + } + return result; +} + +template<typename T> +std::vector<T> GenerateRandomIntegers(size_t num_values_to_test) { + std::vector<T> result; + result.reserve(num_values_to_test); + std::mt19937_64 rng(kRandomSeed); + for (size_t i = 0; i < num_values_to_test; ++i) { + result.push_back(rng()); + } + return result; +} + +void ManualByteSwap(char* bytes, int length) { + if (length == 1) + return; + + EXPECT_EQ(0, length % 2); + for (int i = 0; i < length / 2; ++i) { + int j = (length - 1) - i; + using std::swap; + swap(bytes[i], bytes[j]); + } +} + +template<typename T> +inline T UnalignedLoad(const char* p) { + static_assert( + sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8, + "Unexpected type size"); + + switch (sizeof(T)) { + case 1: return *reinterpret_cast<const T*>(p); + case 2: + return ABSL_INTERNAL_UNALIGNED_LOAD16(p); + case 4: + return ABSL_INTERNAL_UNALIGNED_LOAD32(p); + case 8: + return ABSL_INTERNAL_UNALIGNED_LOAD64(p); + default: + // Suppresses invalid "not all control paths return a value" on MSVC + return {}; + } +} + +template <typename T, typename ByteSwapper> +static void GBSwapHelper(const std::vector<T>& host_values_to_test, + const ByteSwapper& byte_swapper) { + // Test byte_swapper against a manual byte swap. + for (typename std::vector<T>::const_iterator it = host_values_to_test.begin(); + it != host_values_to_test.end(); ++it) { + T host_value = *it; + + char actual_value[sizeof(host_value)]; + memcpy(actual_value, &host_value, sizeof(host_value)); + byte_swapper(actual_value); + + char expected_value[sizeof(host_value)]; + memcpy(expected_value, &host_value, sizeof(host_value)); + ManualByteSwap(expected_value, sizeof(host_value)); + + ASSERT_EQ(0, memcmp(actual_value, expected_value, sizeof(host_value))) + << "Swap output for 0x" << std::hex << host_value << " does not match. " + << "Expected: 0x" << UnalignedLoad<T>(expected_value) << "; " + << "actual: 0x" << UnalignedLoad<T>(actual_value); + } +} + +void Swap16(char* bytes) { + ABSL_INTERNAL_UNALIGNED_STORE16( + bytes, gbswap_16(ABSL_INTERNAL_UNALIGNED_LOAD16(bytes))); +} + +void Swap32(char* bytes) { + ABSL_INTERNAL_UNALIGNED_STORE32( + bytes, gbswap_32(ABSL_INTERNAL_UNALIGNED_LOAD32(bytes))); +} + +void Swap64(char* bytes) { + ABSL_INTERNAL_UNALIGNED_STORE64( + bytes, gbswap_64(ABSL_INTERNAL_UNALIGNED_LOAD64(bytes))); +} + +TEST(EndianessTest, Uint16) { + GBSwapHelper(GenerateAllUint16Values(), &Swap16); +} + +TEST(EndianessTest, Uint32) { + GBSwapHelper(GenerateRandomIntegers<uint32_t>(kNumValuesToTest), &Swap32); +} + +TEST(EndianessTest, Uint64) { + GBSwapHelper(GenerateRandomIntegers<uint64_t>(kNumValuesToTest), &Swap64); +} + +TEST(EndianessTest, ghtonll_gntohll) { + // Test that absl::ghtonl compiles correctly + uint32_t test = 0x01234567; + EXPECT_EQ(absl::gntohl(absl::ghtonl(test)), test); + + uint64_t comp = absl::ghtonll(kInitialNumber); + EXPECT_EQ(comp, kInitialInNetworkOrder); + comp = absl::gntohll(kInitialInNetworkOrder); + EXPECT_EQ(comp, kInitialNumber); + + // Test that htonll and ntohll are each others' inverse functions on a + // somewhat assorted batch of numbers. 37 is chosen to not be anything + // particularly nice base 2. + uint64_t value = 1; + for (int i = 0; i < 100; ++i) { + comp = absl::ghtonll(absl::gntohll(value)); + EXPECT_EQ(value, comp); + comp = absl::gntohll(absl::ghtonll(value)); + EXPECT_EQ(value, comp); + value *= 37; + } +} + +TEST(EndianessTest, little_endian) { + // Check little_endian uint16_t. + uint64_t comp = little_endian::FromHost16(k16Value); + EXPECT_EQ(comp, k16ValueLE); + comp = little_endian::ToHost16(k16ValueLE); + EXPECT_EQ(comp, k16Value); + + // Check little_endian uint32_t. + comp = little_endian::FromHost32(k32Value); + EXPECT_EQ(comp, k32ValueLE); + comp = little_endian::ToHost32(k32ValueLE); + EXPECT_EQ(comp, k32Value); + + // Check little_endian uint64_t. + comp = little_endian::FromHost64(k64Value); + EXPECT_EQ(comp, k64ValueLE); + comp = little_endian::ToHost64(k64ValueLE); + EXPECT_EQ(comp, k64Value); + + // Check little-endian Load and store functions. + uint16_t u16Buf; + uint32_t u32Buf; + uint64_t u64Buf; + + little_endian::Store16(&u16Buf, k16Value); + EXPECT_EQ(u16Buf, k16ValueLE); + comp = little_endian::Load16(&u16Buf); + EXPECT_EQ(comp, k16Value); + + little_endian::Store32(&u32Buf, k32Value); + EXPECT_EQ(u32Buf, k32ValueLE); + comp = little_endian::Load32(&u32Buf); + EXPECT_EQ(comp, k32Value); + + little_endian::Store64(&u64Buf, k64Value); + EXPECT_EQ(u64Buf, k64ValueLE); + comp = little_endian::Load64(&u64Buf); + EXPECT_EQ(comp, k64Value); +} + +TEST(EndianessTest, big_endian) { + // Check big-endian Load and store functions. + uint16_t u16Buf; + uint32_t u32Buf; + uint64_t u64Buf; + + unsigned char buffer[10]; + big_endian::Store16(&u16Buf, k16Value); + EXPECT_EQ(u16Buf, k16ValueBE); + uint64_t comp = big_endian::Load16(&u16Buf); + EXPECT_EQ(comp, k16Value); + + big_endian::Store32(&u32Buf, k32Value); + EXPECT_EQ(u32Buf, k32ValueBE); + comp = big_endian::Load32(&u32Buf); + EXPECT_EQ(comp, k32Value); + + big_endian::Store64(&u64Buf, k64Value); + EXPECT_EQ(u64Buf, k64ValueBE); + comp = big_endian::Load64(&u64Buf); + EXPECT_EQ(comp, k64Value); + + big_endian::Store16(buffer + 1, k16Value); + EXPECT_EQ(u16Buf, k16ValueBE); + comp = big_endian::Load16(buffer + 1); + EXPECT_EQ(comp, k16Value); + + big_endian::Store32(buffer + 1, k32Value); + EXPECT_EQ(u32Buf, k32ValueBE); + comp = big_endian::Load32(buffer + 1); + EXPECT_EQ(comp, k32Value); + + big_endian::Store64(buffer + 1, k64Value); + EXPECT_EQ(u64Buf, k64ValueBE); + comp = big_endian::Load64(buffer + 1); + EXPECT_EQ(comp, k64Value); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/errno_saver.h b/third_party/abseil_cpp/absl/base/internal/errno_saver.h new file mode 100644 index 000000000000..251de510fc93 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/errno_saver.h @@ -0,0 +1,43 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_ERRNO_SAVER_H_ +#define ABSL_BASE_INTERNAL_ERRNO_SAVER_H_ + +#include <cerrno> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// `ErrnoSaver` captures the value of `errno` upon construction and restores it +// upon deletion. It is used in low-level code and must be super fast. Do not +// add instrumentation, even in debug modes. +class ErrnoSaver { + public: + ErrnoSaver() : saved_errno_(errno) {} + ~ErrnoSaver() { errno = saved_errno_; } + int operator()() const { return saved_errno_; } + + private: + const int saved_errno_; +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_ERRNO_SAVER_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/errno_saver_test.cc b/third_party/abseil_cpp/absl/base/internal/errno_saver_test.cc new file mode 100644 index 000000000000..e9b742c588b0 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/errno_saver_test.cc @@ -0,0 +1,45 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/errno_saver.h" + +#include <cerrno> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/strerror.h" + +namespace { +using ::testing::Eq; + +struct ErrnoPrinter { + int no; +}; +std::ostream &operator<<(std::ostream &os, ErrnoPrinter ep) { + return os << absl::base_internal::StrError(ep.no) << " [" << ep.no << "]"; +} +bool operator==(ErrnoPrinter one, ErrnoPrinter two) { return one.no == two.no; } + +TEST(ErrnoSaverTest, Works) { + errno = EDOM; + { + absl::base_internal::ErrnoSaver errno_saver; + EXPECT_THAT(ErrnoPrinter{errno}, Eq(ErrnoPrinter{EDOM})); + errno = ERANGE; + EXPECT_THAT(ErrnoPrinter{errno}, Eq(ErrnoPrinter{ERANGE})); + EXPECT_THAT(ErrnoPrinter{errno_saver()}, Eq(ErrnoPrinter{EDOM})); + } + EXPECT_THAT(ErrnoPrinter{errno}, Eq(ErrnoPrinter{EDOM})); +} +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/exception_safety_testing.cc b/third_party/abseil_cpp/absl/base/internal/exception_safety_testing.cc new file mode 100644 index 000000000000..6ccac41864b1 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/exception_safety_testing.cc @@ -0,0 +1,79 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/exception_safety_testing.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" + +namespace testing { + +exceptions_internal::NoThrowTag nothrow_ctor; + +exceptions_internal::StrongGuaranteeTagType strong_guarantee; + +exceptions_internal::ExceptionSafetyTestBuilder<> MakeExceptionSafetyTester() { + return {}; +} + +namespace exceptions_internal { + +int countdown = -1; + +ConstructorTracker* ConstructorTracker::current_tracker_instance_ = nullptr; + +void MaybeThrow(absl::string_view msg, bool throw_bad_alloc) { + if (countdown-- == 0) { + if (throw_bad_alloc) throw TestBadAllocException(msg); + throw TestException(msg); + } +} + +testing::AssertionResult FailureMessage(const TestException& e, + int countdown) noexcept { + return testing::AssertionFailure() << "Exception thrown from " << e.what(); +} + +std::string GetSpecString(TypeSpec spec) { + std::string out; + absl::string_view sep; + const auto append = [&](absl::string_view s) { + absl::StrAppend(&out, sep, s); + sep = " | "; + }; + if (static_cast<bool>(TypeSpec::kNoThrowCopy & spec)) { + append("kNoThrowCopy"); + } + if (static_cast<bool>(TypeSpec::kNoThrowMove & spec)) { + append("kNoThrowMove"); + } + if (static_cast<bool>(TypeSpec::kNoThrowNew & spec)) { + append("kNoThrowNew"); + } + return out; +} + +std::string GetSpecString(AllocSpec spec) { + return static_cast<bool>(AllocSpec::kNoThrowAllocate & spec) + ? "kNoThrowAllocate" + : ""; +} + +} // namespace exceptions_internal + +} // namespace testing + +#endif // ABSL_HAVE_EXCEPTIONS diff --git a/third_party/abseil_cpp/absl/base/internal/exception_safety_testing.h b/third_party/abseil_cpp/absl/base/internal/exception_safety_testing.h new file mode 100644 index 000000000000..6ba89d05dfca --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/exception_safety_testing.h @@ -0,0 +1,1101 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Utilities for testing exception-safety + +#ifndef ABSL_BASE_INTERNAL_EXCEPTION_SAFETY_TESTING_H_ +#define ABSL_BASE_INTERNAL_EXCEPTION_SAFETY_TESTING_H_ + +#include "absl/base/config.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + +#include <cstddef> +#include <cstdint> +#include <functional> +#include <initializer_list> +#include <iosfwd> +#include <string> +#include <tuple> +#include <unordered_map> + +#include "gtest/gtest.h" +#include "absl/base/internal/pretty_function.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/utility/utility.h" + +namespace testing { + +enum class TypeSpec; +enum class AllocSpec; + +constexpr TypeSpec operator|(TypeSpec a, TypeSpec b) { + using T = absl::underlying_type_t<TypeSpec>; + return static_cast<TypeSpec>(static_cast<T>(a) | static_cast<T>(b)); +} + +constexpr TypeSpec operator&(TypeSpec a, TypeSpec b) { + using T = absl::underlying_type_t<TypeSpec>; + return static_cast<TypeSpec>(static_cast<T>(a) & static_cast<T>(b)); +} + +constexpr AllocSpec operator|(AllocSpec a, AllocSpec b) { + using T = absl::underlying_type_t<AllocSpec>; + return static_cast<AllocSpec>(static_cast<T>(a) | static_cast<T>(b)); +} + +constexpr AllocSpec operator&(AllocSpec a, AllocSpec b) { + using T = absl::underlying_type_t<AllocSpec>; + return static_cast<AllocSpec>(static_cast<T>(a) & static_cast<T>(b)); +} + +namespace exceptions_internal { + +std::string GetSpecString(TypeSpec); +std::string GetSpecString(AllocSpec); + +struct NoThrowTag {}; +struct StrongGuaranteeTagType {}; + +// A simple exception class. We throw this so that test code can catch +// exceptions specifically thrown by ThrowingValue. +class TestException { + public: + explicit TestException(absl::string_view msg) : msg_(msg) {} + virtual ~TestException() {} + virtual const char* what() const noexcept { return msg_.c_str(); } + + private: + std::string msg_; +}; + +// TestBadAllocException exists because allocation functions must throw an +// exception which can be caught by a handler of std::bad_alloc. We use a child +// class of std::bad_alloc so we can customise the error message, and also +// derive from TestException so we don't accidentally end up catching an actual +// bad_alloc exception in TestExceptionSafety. +class TestBadAllocException : public std::bad_alloc, public TestException { + public: + explicit TestBadAllocException(absl::string_view msg) : TestException(msg) {} + using TestException::what; +}; + +extern int countdown; + +// Allows the countdown variable to be set manually (defaulting to the initial +// value of 0) +inline void SetCountdown(int i = 0) { countdown = i; } +// Sets the countdown to the terminal value -1 +inline void UnsetCountdown() { SetCountdown(-1); } + +void MaybeThrow(absl::string_view msg, bool throw_bad_alloc = false); + +testing::AssertionResult FailureMessage(const TestException& e, + int countdown) noexcept; + +struct TrackedAddress { + bool is_alive; + std::string description; +}; + +// Inspects the constructions and destructions of anything inheriting from +// TrackedObject. This allows us to safely "leak" TrackedObjects, as +// ConstructorTracker will destroy everything left over in its destructor. +class ConstructorTracker { + public: + explicit ConstructorTracker(int count) : countdown_(count) { + assert(current_tracker_instance_ == nullptr); + current_tracker_instance_ = this; + } + + ~ConstructorTracker() { + assert(current_tracker_instance_ == this); + current_tracker_instance_ = nullptr; + + for (auto& it : address_map_) { + void* address = it.first; + TrackedAddress& tracked_address = it.second; + if (tracked_address.is_alive) { + ADD_FAILURE() << ErrorMessage(address, tracked_address.description, + countdown_, "Object was not destroyed."); + } + } + } + + static void ObjectConstructed(void* address, std::string description) { + if (!CurrentlyTracking()) return; + + TrackedAddress& tracked_address = + current_tracker_instance_->address_map_[address]; + if (tracked_address.is_alive) { + ADD_FAILURE() << ErrorMessage( + address, tracked_address.description, + current_tracker_instance_->countdown_, + "Object was re-constructed. Current object was constructed by " + + description); + } + tracked_address = {true, std::move(description)}; + } + + static void ObjectDestructed(void* address) { + if (!CurrentlyTracking()) return; + + auto it = current_tracker_instance_->address_map_.find(address); + // Not tracked. Ignore. + if (it == current_tracker_instance_->address_map_.end()) return; + + TrackedAddress& tracked_address = it->second; + if (!tracked_address.is_alive) { + ADD_FAILURE() << ErrorMessage(address, tracked_address.description, + current_tracker_instance_->countdown_, + "Object was re-destroyed."); + } + tracked_address.is_alive = false; + } + + private: + static bool CurrentlyTracking() { + return current_tracker_instance_ != nullptr; + } + + static std::string ErrorMessage(void* address, + const std::string& address_description, + int countdown, + const std::string& error_description) { + return absl::Substitute( + "With coundtown at $0:\n" + " $1\n" + " Object originally constructed by $2\n" + " Object address: $3\n", + countdown, error_description, address_description, address); + } + + std::unordered_map<void*, TrackedAddress> address_map_; + int countdown_; + + static ConstructorTracker* current_tracker_instance_; +}; + +class TrackedObject { + public: + TrackedObject(const TrackedObject&) = delete; + TrackedObject(TrackedObject&&) = delete; + + protected: + explicit TrackedObject(std::string description) { + ConstructorTracker::ObjectConstructed(this, std::move(description)); + } + + ~TrackedObject() noexcept { ConstructorTracker::ObjectDestructed(this); } +}; +} // namespace exceptions_internal + +extern exceptions_internal::NoThrowTag nothrow_ctor; + +extern exceptions_internal::StrongGuaranteeTagType strong_guarantee; + +// A test class which is convertible to bool. The conversion can be +// instrumented to throw at a controlled time. +class ThrowingBool { + public: + ThrowingBool(bool b) noexcept : b_(b) {} // NOLINT(runtime/explicit) + operator bool() const { // NOLINT + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return b_; + } + + private: + bool b_; +}; + +/* + * Configuration enum for the ThrowingValue type that defines behavior for the + * lifetime of the instance. Use testing::nothrow_ctor to prevent the integer + * constructor from throwing. + * + * kEverythingThrows: Every operation can throw an exception + * kNoThrowCopy: Copy construction and copy assignment will not throw + * kNoThrowMove: Move construction and move assignment will not throw + * kNoThrowNew: Overloaded operators new and new[] will not throw + */ +enum class TypeSpec { + kEverythingThrows = 0, + kNoThrowCopy = 1, + kNoThrowMove = 1 << 1, + kNoThrowNew = 1 << 2, +}; + +/* + * A testing class instrumented to throw an exception at a controlled time. + * + * ThrowingValue implements a slightly relaxed version of the Regular concept -- + * that is it's a value type with the expected semantics. It also implements + * arithmetic operations. It doesn't implement member and pointer operators + * like operator-> or operator[]. + * + * ThrowingValue can be instrumented to have certain operations be noexcept by + * using compile-time bitfield template arguments. That is, to make an + * ThrowingValue which has noexcept move construction/assignment and noexcept + * copy construction/assignment, use the following: + * ThrowingValue<testing::kNoThrowMove | testing::kNoThrowCopy> my_thrwr{val}; + */ +template <TypeSpec Spec = TypeSpec::kEverythingThrows> +class ThrowingValue : private exceptions_internal::TrackedObject { + static constexpr bool IsSpecified(TypeSpec spec) { + return static_cast<bool>(Spec & spec); + } + + static constexpr int kDefaultValue = 0; + static constexpr int kBadValue = 938550620; + + public: + ThrowingValue() : TrackedObject(GetInstanceString(kDefaultValue)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ = kDefaultValue; + } + + ThrowingValue(const ThrowingValue& other) noexcept( + IsSpecified(TypeSpec::kNoThrowCopy)) + : TrackedObject(GetInstanceString(other.dummy_)) { + if (!IsSpecified(TypeSpec::kNoThrowCopy)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + } + dummy_ = other.dummy_; + } + + ThrowingValue(ThrowingValue&& other) noexcept( + IsSpecified(TypeSpec::kNoThrowMove)) + : TrackedObject(GetInstanceString(other.dummy_)) { + if (!IsSpecified(TypeSpec::kNoThrowMove)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + } + dummy_ = other.dummy_; + } + + explicit ThrowingValue(int i) : TrackedObject(GetInstanceString(i)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ = i; + } + + ThrowingValue(int i, exceptions_internal::NoThrowTag) noexcept + : TrackedObject(GetInstanceString(i)), dummy_(i) {} + + // absl expects nothrow destructors + ~ThrowingValue() noexcept = default; + + ThrowingValue& operator=(const ThrowingValue& other) noexcept( + IsSpecified(TypeSpec::kNoThrowCopy)) { + dummy_ = kBadValue; + if (!IsSpecified(TypeSpec::kNoThrowCopy)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + } + dummy_ = other.dummy_; + return *this; + } + + ThrowingValue& operator=(ThrowingValue&& other) noexcept( + IsSpecified(TypeSpec::kNoThrowMove)) { + dummy_ = kBadValue; + if (!IsSpecified(TypeSpec::kNoThrowMove)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + } + dummy_ = other.dummy_; + return *this; + } + + // Arithmetic Operators + ThrowingValue operator+(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ + other.dummy_, nothrow_ctor); + } + + ThrowingValue operator+() const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_, nothrow_ctor); + } + + ThrowingValue operator-(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ - other.dummy_, nothrow_ctor); + } + + ThrowingValue operator-() const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(-dummy_, nothrow_ctor); + } + + ThrowingValue& operator++() { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + ++dummy_; + return *this; + } + + ThrowingValue operator++(int) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + auto out = ThrowingValue(dummy_, nothrow_ctor); + ++dummy_; + return out; + } + + ThrowingValue& operator--() { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + --dummy_; + return *this; + } + + ThrowingValue operator--(int) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + auto out = ThrowingValue(dummy_, nothrow_ctor); + --dummy_; + return out; + } + + ThrowingValue operator*(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ * other.dummy_, nothrow_ctor); + } + + ThrowingValue operator/(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ / other.dummy_, nothrow_ctor); + } + + ThrowingValue operator%(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ % other.dummy_, nothrow_ctor); + } + + ThrowingValue operator<<(int shift) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ << shift, nothrow_ctor); + } + + ThrowingValue operator>>(int shift) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ >> shift, nothrow_ctor); + } + + // Comparison Operators + // NOTE: We use `ThrowingBool` instead of `bool` because most STL + // types/containers requires T to be convertible to bool. + friend ThrowingBool operator==(const ThrowingValue& a, + const ThrowingValue& b) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return a.dummy_ == b.dummy_; + } + friend ThrowingBool operator!=(const ThrowingValue& a, + const ThrowingValue& b) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return a.dummy_ != b.dummy_; + } + friend ThrowingBool operator<(const ThrowingValue& a, + const ThrowingValue& b) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return a.dummy_ < b.dummy_; + } + friend ThrowingBool operator<=(const ThrowingValue& a, + const ThrowingValue& b) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return a.dummy_ <= b.dummy_; + } + friend ThrowingBool operator>(const ThrowingValue& a, + const ThrowingValue& b) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return a.dummy_ > b.dummy_; + } + friend ThrowingBool operator>=(const ThrowingValue& a, + const ThrowingValue& b) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return a.dummy_ >= b.dummy_; + } + + // Logical Operators + ThrowingBool operator!() const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return !dummy_; + } + + ThrowingBool operator&&(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return dummy_ && other.dummy_; + } + + ThrowingBool operator||(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return dummy_ || other.dummy_; + } + + // Bitwise Logical Operators + ThrowingValue operator~() const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(~dummy_, nothrow_ctor); + } + + ThrowingValue operator&(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ & other.dummy_, nothrow_ctor); + } + + ThrowingValue operator|(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ | other.dummy_, nothrow_ctor); + } + + ThrowingValue operator^(const ThrowingValue& other) const { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return ThrowingValue(dummy_ ^ other.dummy_, nothrow_ctor); + } + + // Compound Assignment operators + ThrowingValue& operator+=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ += other.dummy_; + return *this; + } + + ThrowingValue& operator-=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ -= other.dummy_; + return *this; + } + + ThrowingValue& operator*=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ *= other.dummy_; + return *this; + } + + ThrowingValue& operator/=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ /= other.dummy_; + return *this; + } + + ThrowingValue& operator%=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ %= other.dummy_; + return *this; + } + + ThrowingValue& operator&=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ &= other.dummy_; + return *this; + } + + ThrowingValue& operator|=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ |= other.dummy_; + return *this; + } + + ThrowingValue& operator^=(const ThrowingValue& other) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ ^= other.dummy_; + return *this; + } + + ThrowingValue& operator<<=(int shift) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ <<= shift; + return *this; + } + + ThrowingValue& operator>>=(int shift) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ >>= shift; + return *this; + } + + // Pointer operators + void operator&() const = delete; // NOLINT(runtime/operator) + + // Stream operators + friend std::ostream& operator<<(std::ostream& os, const ThrowingValue& tv) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return os << GetInstanceString(tv.dummy_); + } + + friend std::istream& operator>>(std::istream& is, const ThrowingValue&) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + return is; + } + + // Memory management operators + // Args.. allows us to overload regular and placement new in one shot + template <typename... Args> + static void* operator new(size_t s, Args&&... args) noexcept( + IsSpecified(TypeSpec::kNoThrowNew)) { + if (!IsSpecified(TypeSpec::kNoThrowNew)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + } + return ::operator new(s, std::forward<Args>(args)...); + } + + template <typename... Args> + static void* operator new[](size_t s, Args&&... args) noexcept( + IsSpecified(TypeSpec::kNoThrowNew)) { + if (!IsSpecified(TypeSpec::kNoThrowNew)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + } + return ::operator new[](s, std::forward<Args>(args)...); + } + + // Abseil doesn't support throwing overloaded operator delete. These are + // provided so a throwing operator-new can clean up after itself. + // + // We provide both regular and templated operator delete because if only the + // templated version is provided as we did with operator new, the compiler has + // no way of knowing which overload of operator delete to call. See + // https://en.cppreference.com/w/cpp/memory/new/operator_delete and + // https://en.cppreference.com/w/cpp/language/delete for the gory details. + void operator delete(void* p) noexcept { ::operator delete(p); } + + template <typename... Args> + void operator delete(void* p, Args&&... args) noexcept { + ::operator delete(p, std::forward<Args>(args)...); + } + + void operator delete[](void* p) noexcept { return ::operator delete[](p); } + + template <typename... Args> + void operator delete[](void* p, Args&&... args) noexcept { + return ::operator delete[](p, std::forward<Args>(args)...); + } + + // Non-standard access to the actual contained value. No need for this to + // throw. + int& Get() noexcept { return dummy_; } + const int& Get() const noexcept { return dummy_; } + + private: + static std::string GetInstanceString(int dummy) { + return absl::StrCat("ThrowingValue<", + exceptions_internal::GetSpecString(Spec), ">(", dummy, + ")"); + } + + int dummy_; +}; +// While not having to do with exceptions, explicitly delete comma operator, to +// make sure we don't use it on user-supplied types. +template <TypeSpec Spec, typename T> +void operator,(const ThrowingValue<Spec>&, T&&) = delete; +template <TypeSpec Spec, typename T> +void operator,(T&&, const ThrowingValue<Spec>&) = delete; + +/* + * Configuration enum for the ThrowingAllocator type that defines behavior for + * the lifetime of the instance. + * + * kEverythingThrows: Calls to the member functions may throw + * kNoThrowAllocate: Calls to the member functions will not throw + */ +enum class AllocSpec { + kEverythingThrows = 0, + kNoThrowAllocate = 1, +}; + +/* + * An allocator type which is instrumented to throw at a controlled time, or not + * to throw, using AllocSpec. The supported settings are the default of every + * function which is allowed to throw in a conforming allocator possibly + * throwing, or nothing throws, in line with the ABSL_ALLOCATOR_THROWS + * configuration macro. + */ +template <typename T, AllocSpec Spec = AllocSpec::kEverythingThrows> +class ThrowingAllocator : private exceptions_internal::TrackedObject { + static constexpr bool IsSpecified(AllocSpec spec) { + return static_cast<bool>(Spec & spec); + } + + public: + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using void_pointer = void*; + using const_void_pointer = const void*; + using value_type = T; + using size_type = size_t; + using difference_type = ptrdiff_t; + + using is_nothrow = + std::integral_constant<bool, Spec == AllocSpec::kNoThrowAllocate>; + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + using is_always_equal = std::false_type; + + ThrowingAllocator() : TrackedObject(GetInstanceString(next_id_)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION); + dummy_ = std::make_shared<const int>(next_id_++); + } + + template <typename U> + ThrowingAllocator(const ThrowingAllocator<U, Spec>& other) noexcept // NOLINT + : TrackedObject(GetInstanceString(*other.State())), + dummy_(other.State()) {} + + // According to C++11 standard [17.6.3.5], Table 28, the move/copy ctors of + // allocator shall not exit via an exception, thus they are marked noexcept. + ThrowingAllocator(const ThrowingAllocator& other) noexcept + : TrackedObject(GetInstanceString(*other.State())), + dummy_(other.State()) {} + + template <typename U> + ThrowingAllocator(ThrowingAllocator<U, Spec>&& other) noexcept // NOLINT + : TrackedObject(GetInstanceString(*other.State())), + dummy_(std::move(other.State())) {} + + ThrowingAllocator(ThrowingAllocator&& other) noexcept + : TrackedObject(GetInstanceString(*other.State())), + dummy_(std::move(other.State())) {} + + ~ThrowingAllocator() noexcept = default; + + ThrowingAllocator& operator=(const ThrowingAllocator& other) noexcept { + dummy_ = other.State(); + return *this; + } + + template <typename U> + ThrowingAllocator& operator=( + const ThrowingAllocator<U, Spec>& other) noexcept { + dummy_ = other.State(); + return *this; + } + + template <typename U> + ThrowingAllocator& operator=(ThrowingAllocator<U, Spec>&& other) noexcept { + dummy_ = std::move(other.State()); + return *this; + } + + template <typename U> + struct rebind { + using other = ThrowingAllocator<U, Spec>; + }; + + pointer allocate(size_type n) noexcept( + IsSpecified(AllocSpec::kNoThrowAllocate)) { + ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); + return static_cast<pointer>(::operator new(n * sizeof(T))); + } + + pointer allocate(size_type n, const_void_pointer) noexcept( + IsSpecified(AllocSpec::kNoThrowAllocate)) { + return allocate(n); + } + + void deallocate(pointer ptr, size_type) noexcept { + ReadState(); + ::operator delete(static_cast<void*>(ptr)); + } + + template <typename U, typename... Args> + void construct(U* ptr, Args&&... args) noexcept( + IsSpecified(AllocSpec::kNoThrowAllocate)) { + ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); + ::new (static_cast<void*>(ptr)) U(std::forward<Args>(args)...); + } + + template <typename U> + void destroy(U* p) noexcept { + ReadState(); + p->~U(); + } + + size_type max_size() const noexcept { + return (std::numeric_limits<difference_type>::max)() / sizeof(value_type); + } + + ThrowingAllocator select_on_container_copy_construction() noexcept( + IsSpecified(AllocSpec::kNoThrowAllocate)) { + auto& out = *this; + ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); + return out; + } + + template <typename U> + bool operator==(const ThrowingAllocator<U, Spec>& other) const noexcept { + return dummy_ == other.dummy_; + } + + template <typename U> + bool operator!=(const ThrowingAllocator<U, Spec>& other) const noexcept { + return dummy_ != other.dummy_; + } + + template <typename, AllocSpec> + friend class ThrowingAllocator; + + private: + static std::string GetInstanceString(int dummy) { + return absl::StrCat("ThrowingAllocator<", + exceptions_internal::GetSpecString(Spec), ">(", dummy, + ")"); + } + + const std::shared_ptr<const int>& State() const { return dummy_; } + std::shared_ptr<const int>& State() { return dummy_; } + + void ReadState() { + // we know that this will never be true, but the compiler doesn't, so this + // should safely force a read of the value. + if (*dummy_ < 0) std::abort(); + } + + void ReadStateAndMaybeThrow(absl::string_view msg) const { + if (!IsSpecified(AllocSpec::kNoThrowAllocate)) { + exceptions_internal::MaybeThrow( + absl::Substitute("Allocator id $0 threw from $1", *dummy_, msg)); + } + } + + static int next_id_; + std::shared_ptr<const int> dummy_; +}; + +template <typename T, AllocSpec Spec> +int ThrowingAllocator<T, Spec>::next_id_ = 0; + +// Tests for resource leaks by attempting to construct a T using args repeatedly +// until successful, using the countdown method. Side effects can then be +// tested for resource leaks. +template <typename T, typename... Args> +void TestThrowingCtor(Args&&... args) { + struct Cleanup { + ~Cleanup() { exceptions_internal::UnsetCountdown(); } + } c; + for (int count = 0;; ++count) { + exceptions_internal::ConstructorTracker ct(count); + exceptions_internal::SetCountdown(count); + try { + T temp(std::forward<Args>(args)...); + static_cast<void>(temp); + break; + } catch (const exceptions_internal::TestException&) { + } + } +} + +// Tests the nothrow guarantee of the provided nullary operation. If the an +// exception is thrown, the result will be AssertionFailure(). Otherwise, it +// will be AssertionSuccess(). +template <typename Operation> +testing::AssertionResult TestNothrowOp(const Operation& operation) { + struct Cleanup { + Cleanup() { exceptions_internal::SetCountdown(); } + ~Cleanup() { exceptions_internal::UnsetCountdown(); } + } c; + try { + operation(); + return testing::AssertionSuccess(); + } catch (const exceptions_internal::TestException&) { + return testing::AssertionFailure() + << "TestException thrown during call to operation() when nothrow " + "guarantee was expected."; + } catch (...) { + return testing::AssertionFailure() + << "Unknown exception thrown during call to operation() when " + "nothrow guarantee was expected."; + } +} + +namespace exceptions_internal { + +// Dummy struct for ExceptionSafetyTestBuilder<> partial state. +struct UninitializedT {}; + +template <typename T> +class DefaultFactory { + public: + explicit DefaultFactory(const T& t) : t_(t) {} + std::unique_ptr<T> operator()() const { return absl::make_unique<T>(t_); } + + private: + T t_; +}; + +template <size_t LazyContractsCount, typename LazyFactory, + typename LazyOperation> +using EnableIfTestable = typename absl::enable_if_t< + LazyContractsCount != 0 && + !std::is_same<LazyFactory, UninitializedT>::value && + !std::is_same<LazyOperation, UninitializedT>::value>; + +template <typename Factory = UninitializedT, + typename Operation = UninitializedT, typename... Contracts> +class ExceptionSafetyTestBuilder; + +} // namespace exceptions_internal + +/* + * Constructs an empty ExceptionSafetyTestBuilder. All + * ExceptionSafetyTestBuilder objects are immutable and all With[thing] mutation + * methods return new instances of ExceptionSafetyTestBuilder. + * + * In order to test a T for exception safety, a factory for that T, a testable + * operation, and at least one contract callback returning an assertion + * result must be applied using the respective methods. + */ +exceptions_internal::ExceptionSafetyTestBuilder<> MakeExceptionSafetyTester(); + +namespace exceptions_internal { +template <typename T> +struct IsUniquePtr : std::false_type {}; + +template <typename T, typename D> +struct IsUniquePtr<std::unique_ptr<T, D>> : std::true_type {}; + +template <typename Factory> +struct FactoryPtrTypeHelper { + using type = decltype(std::declval<const Factory&>()()); + + static_assert(IsUniquePtr<type>::value, "Factories must return a unique_ptr"); +}; + +template <typename Factory> +using FactoryPtrType = typename FactoryPtrTypeHelper<Factory>::type; + +template <typename Factory> +using FactoryElementType = typename FactoryPtrType<Factory>::element_type; + +template <typename T> +class ExceptionSafetyTest { + using Factory = std::function<std::unique_ptr<T>()>; + using Operation = std::function<void(T*)>; + using Contract = std::function<AssertionResult(T*)>; + + public: + template <typename... Contracts> + explicit ExceptionSafetyTest(const Factory& f, const Operation& op, + const Contracts&... contracts) + : factory_(f), operation_(op), contracts_{WrapContract(contracts)...} {} + + AssertionResult Test() const { + for (int count = 0;; ++count) { + exceptions_internal::ConstructorTracker ct(count); + + for (const auto& contract : contracts_) { + auto t_ptr = factory_(); + try { + SetCountdown(count); + operation_(t_ptr.get()); + // Unset for the case that the operation throws no exceptions, which + // would leave the countdown set and break the *next* exception safety + // test after this one. + UnsetCountdown(); + return AssertionSuccess(); + } catch (const exceptions_internal::TestException& e) { + if (!contract(t_ptr.get())) { + return AssertionFailure() << e.what() << " failed contract check"; + } + } + } + } + } + + private: + template <typename ContractFn> + Contract WrapContract(const ContractFn& contract) { + return [contract](T* t_ptr) { return AssertionResult(contract(t_ptr)); }; + } + + Contract WrapContract(StrongGuaranteeTagType) { + return [this](T* t_ptr) { return AssertionResult(*factory_() == *t_ptr); }; + } + + Factory factory_; + Operation operation_; + std::vector<Contract> contracts_; +}; + +/* + * Builds a tester object that tests if performing a operation on a T follows + * exception safety guarantees. Verification is done via contract assertion + * callbacks applied to T instances post-throw. + * + * Template parameters for ExceptionSafetyTestBuilder: + * + * - Factory: The factory object (passed in via tester.WithFactory(...) or + * tester.WithInitialValue(...)) must be invocable with the signature + * `std::unique_ptr<T> operator()() const` where T is the type being tested. + * It is used for reliably creating identical T instances to test on. + * + * - Operation: The operation object (passsed in via tester.WithOperation(...) + * or tester.Test(...)) must be invocable with the signature + * `void operator()(T*) const` where T is the type being tested. It is used + * for performing steps on a T instance that may throw and that need to be + * checked for exception safety. Each call to the operation will receive a + * fresh T instance so it's free to modify and destroy the T instances as it + * pleases. + * + * - Contracts...: The contract assertion callback objects (passed in via + * tester.WithContracts(...)) must be invocable with the signature + * `testing::AssertionResult operator()(T*) const` where T is the type being + * tested. Contract assertion callbacks are provided T instances post-throw. + * They must return testing::AssertionSuccess when the type contracts of the + * provided T instance hold. If the type contracts of the T instance do not + * hold, they must return testing::AssertionFailure. Execution order of + * Contracts... is unspecified. They will each individually get a fresh T + * instance so they are free to modify and destroy the T instances as they + * please. + */ +template <typename Factory, typename Operation, typename... Contracts> +class ExceptionSafetyTestBuilder { + public: + /* + * Returns a new ExceptionSafetyTestBuilder with an included T factory based + * on the provided T instance. The existing factory will not be included in + * the newly created tester instance. The created factory returns a new T + * instance by copy-constructing the provided const T& t. + * + * Preconditions for tester.WithInitialValue(const T& t): + * + * - The const T& t object must be copy-constructible where T is the type + * being tested. For non-copy-constructible objects, use the method + * tester.WithFactory(...). + */ + template <typename T> + ExceptionSafetyTestBuilder<DefaultFactory<T>, Operation, Contracts...> + WithInitialValue(const T& t) const { + return WithFactory(DefaultFactory<T>(t)); + } + + /* + * Returns a new ExceptionSafetyTestBuilder with the provided T factory + * included. The existing factory will not be included in the newly-created + * tester instance. This method is intended for use with types lacking a copy + * constructor. Types that can be copy-constructed should instead use the + * method tester.WithInitialValue(...). + */ + template <typename NewFactory> + ExceptionSafetyTestBuilder<absl::decay_t<NewFactory>, Operation, Contracts...> + WithFactory(const NewFactory& new_factory) const { + return {new_factory, operation_, contracts_}; + } + + /* + * Returns a new ExceptionSafetyTestBuilder with the provided testable + * operation included. The existing operation will not be included in the + * newly created tester. + */ + template <typename NewOperation> + ExceptionSafetyTestBuilder<Factory, absl::decay_t<NewOperation>, Contracts...> + WithOperation(const NewOperation& new_operation) const { + return {factory_, new_operation, contracts_}; + } + + /* + * Returns a new ExceptionSafetyTestBuilder with the provided MoreContracts... + * combined with the Contracts... that were already included in the instance + * on which the method was called. Contracts... cannot be removed or replaced + * once added to an ExceptionSafetyTestBuilder instance. A fresh object must + * be created in order to get an empty Contracts... list. + * + * In addition to passing in custom contract assertion callbacks, this method + * accepts `testing::strong_guarantee` as an argument which checks T instances + * post-throw against freshly created T instances via operator== to verify + * that any state changes made during the execution of the operation were + * properly rolled back. + */ + template <typename... MoreContracts> + ExceptionSafetyTestBuilder<Factory, Operation, Contracts..., + absl::decay_t<MoreContracts>...> + WithContracts(const MoreContracts&... more_contracts) const { + return { + factory_, operation_, + std::tuple_cat(contracts_, std::tuple<absl::decay_t<MoreContracts>...>( + more_contracts...))}; + } + + /* + * Returns a testing::AssertionResult that is the reduced result of the + * exception safety algorithm. The algorithm short circuits and returns + * AssertionFailure after the first contract callback returns an + * AssertionFailure. Otherwise, if all contract callbacks return an + * AssertionSuccess, the reduced result is AssertionSuccess. + * + * The passed-in testable operation will not be saved in a new tester instance + * nor will it modify/replace the existing tester instance. This is useful + * when each operation being tested is unique and does not need to be reused. + * + * Preconditions for tester.Test(const NewOperation& new_operation): + * + * - May only be called after at least one contract assertion callback and a + * factory or initial value have been provided. + */ + template < + typename NewOperation, + typename = EnableIfTestable<sizeof...(Contracts), Factory, NewOperation>> + testing::AssertionResult Test(const NewOperation& new_operation) const { + return TestImpl(new_operation, absl::index_sequence_for<Contracts...>()); + } + + /* + * Returns a testing::AssertionResult that is the reduced result of the + * exception safety algorithm. The algorithm short circuits and returns + * AssertionFailure after the first contract callback returns an + * AssertionFailure. Otherwise, if all contract callbacks return an + * AssertionSuccess, the reduced result is AssertionSuccess. + * + * Preconditions for tester.Test(): + * + * - May only be called after at least one contract assertion callback, a + * factory or initial value and a testable operation have been provided. + */ + template < + typename LazyOperation = Operation, + typename = EnableIfTestable<sizeof...(Contracts), Factory, LazyOperation>> + testing::AssertionResult Test() const { + return Test(operation_); + } + + private: + template <typename, typename, typename...> + friend class ExceptionSafetyTestBuilder; + + friend ExceptionSafetyTestBuilder<> testing::MakeExceptionSafetyTester(); + + ExceptionSafetyTestBuilder() {} + + ExceptionSafetyTestBuilder(const Factory& f, const Operation& o, + const std::tuple<Contracts...>& i) + : factory_(f), operation_(o), contracts_(i) {} + + template <typename SelectedOperation, size_t... Indices> + testing::AssertionResult TestImpl(SelectedOperation selected_operation, + absl::index_sequence<Indices...>) const { + return ExceptionSafetyTest<FactoryElementType<Factory>>( + factory_, selected_operation, std::get<Indices>(contracts_)...) + .Test(); + } + + Factory factory_; + Operation operation_; + std::tuple<Contracts...> contracts_; +}; + +} // namespace exceptions_internal + +} // namespace testing + +#endif // ABSL_HAVE_EXCEPTIONS + +#endif // ABSL_BASE_INTERNAL_EXCEPTION_SAFETY_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/exception_testing.h b/third_party/abseil_cpp/absl/base/internal/exception_testing.h new file mode 100644 index 000000000000..01b546557117 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/exception_testing.h @@ -0,0 +1,42 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Testing utilities for ABSL types which throw exceptions. + +#ifndef ABSL_BASE_INTERNAL_EXCEPTION_TESTING_H_ +#define ABSL_BASE_INTERNAL_EXCEPTION_TESTING_H_ + +#include "gtest/gtest.h" +#include "absl/base/config.h" + +// ABSL_BASE_INTERNAL_EXPECT_FAIL tests either for a specified thrown exception +// if exceptions are enabled, or for death with a specified text in the error +// message +#ifdef ABSL_HAVE_EXCEPTIONS + +#define ABSL_BASE_INTERNAL_EXPECT_FAIL(expr, exception_t, text) \ + EXPECT_THROW(expr, exception_t) + +#elif defined(__ANDROID__) +// Android asserts do not log anywhere that gtest can currently inspect. +// So we expect exit, but cannot match the message. +#define ABSL_BASE_INTERNAL_EXPECT_FAIL(expr, exception_t, text) \ + EXPECT_DEATH(expr, ".*") +#else +#define ABSL_BASE_INTERNAL_EXPECT_FAIL(expr, exception_t, text) \ + EXPECT_DEATH_IF_SUPPORTED(expr, text) + +#endif + +#endif // ABSL_BASE_INTERNAL_EXCEPTION_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/exponential_biased.cc b/third_party/abseil_cpp/absl/base/internal/exponential_biased.cc new file mode 100644 index 000000000000..1b30c061e3bd --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/exponential_biased.cc @@ -0,0 +1,93 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/exponential_biased.h" + +#include <stdint.h> + +#include <algorithm> +#include <atomic> +#include <cmath> +#include <limits> + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// The algorithm generates a random number between 0 and 1 and applies the +// inverse cumulative distribution function for an exponential. Specifically: +// Let m be the inverse of the sample period, then the probability +// distribution function is m*exp(-mx) so the CDF is +// p = 1 - exp(-mx), so +// q = 1 - p = exp(-mx) +// log_e(q) = -mx +// -log_e(q)/m = x +// log_2(q) * (-log_e(2) * 1/m) = x +// In the code, q is actually in the range 1 to 2**26, hence the -26 below +int64_t ExponentialBiased::GetSkipCount(int64_t mean) { + if (ABSL_PREDICT_FALSE(!initialized_)) { + Initialize(); + } + + uint64_t rng = NextRandom(rng_); + rng_ = rng; + + // Take the top 26 bits as the random number + // (This plus the 1<<58 sampling bound give a max possible step of + // 5194297183973780480 bytes.) + // The uint32_t cast is to prevent a (hard-to-reproduce) NAN + // under piii debug for some binaries. + double q = static_cast<uint32_t>(rng >> (kPrngNumBits - 26)) + 1.0; + // Put the computed p-value through the CDF of a geometric. + double interval = bias_ + (std::log2(q) - 26) * (-std::log(2.0) * mean); + // Very large values of interval overflow int64_t. To avoid that, we will + // cheat and clamp any huge values to (int64_t max)/2. This is a potential + // source of bias, but the mean would need to be such a large value that it's + // not likely to come up. For example, with a mean of 1e18, the probability of + // hitting this condition is about 1/1000. For a mean of 1e17, standard + // calculators claim that this event won't happen. + if (interval > static_cast<double>(std::numeric_limits<int64_t>::max() / 2)) { + // Assume huge values are bias neutral, retain bias for next call. + return std::numeric_limits<int64_t>::max() / 2; + } + double value = std::round(interval); + bias_ = interval - value; + return value; +} + +int64_t ExponentialBiased::GetStride(int64_t mean) { + return GetSkipCount(mean - 1) + 1; +} + +void ExponentialBiased::Initialize() { + // We don't get well distributed numbers from `this` so we call NextRandom() a + // bunch to mush the bits around. We use a global_rand to handle the case + // where the same thread (by memory address) gets created and destroyed + // repeatedly. + ABSL_CONST_INIT static std::atomic<uint32_t> global_rand(0); + uint64_t r = reinterpret_cast<uint64_t>(this) + + global_rand.fetch_add(1, std::memory_order_relaxed); + for (int i = 0; i < 20; ++i) { + r = NextRandom(r); + } + rng_ = r; + initialized_ = true; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/exponential_biased.h b/third_party/abseil_cpp/absl/base/internal/exponential_biased.h new file mode 100644 index 000000000000..94f79a3378db --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/exponential_biased.h @@ -0,0 +1,130 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ +#define ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ + +#include <stdint.h> + +#include "absl/base/config.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// ExponentialBiased provides a small and fast random number generator for a +// rounded exponential distribution. This generator manages very little state, +// and imposes no synchronization overhead. This makes it useful in specialized +// scenarios requiring minimum overhead, such as stride based periodic sampling. +// +// ExponentialBiased provides two closely related functions, GetSkipCount() and +// GetStride(), both returning a rounded integer defining a number of events +// required before some event with a given mean probability occurs. +// +// The distribution is useful to generate a random wait time or some periodic +// event with a given mean probability. For example, if an action is supposed to +// happen on average once every 'N' events, then we can get a random 'stride' +// counting down how long before the event to happen. For example, if we'd want +// to sample one in every 1000 'Frobber' calls, our code could look like this: +// +// Frobber::Frobber() { +// stride_ = exponential_biased_.GetStride(1000); +// } +// +// void Frobber::Frob(int arg) { +// if (--stride == 0) { +// SampleFrob(arg); +// stride_ = exponential_biased_.GetStride(1000); +// } +// ... +// } +// +// The rounding of the return value creates a bias, especially for smaller means +// where the distribution of the fraction is not evenly distributed. We correct +// this bias by tracking the fraction we rounded up or down on each iteration, +// effectively tracking the distance between the cumulative value, and the +// rounded cumulative value. For example, given a mean of 2: +// +// raw = 1.63076, cumulative = 1.63076, rounded = 2, bias = -0.36923 +// raw = 0.14624, cumulative = 1.77701, rounded = 2, bias = 0.14624 +// raw = 4.93194, cumulative = 6.70895, rounded = 7, bias = -0.06805 +// raw = 0.24206, cumulative = 6.95101, rounded = 7, bias = 0.24206 +// etc... +// +// Adjusting with rounding bias is relatively trivial: +// +// double value = bias_ + exponential_distribution(mean)(); +// double rounded_value = std::round(value); +// bias_ = value - rounded_value; +// return rounded_value; +// +// This class is thread-compatible. +class ExponentialBiased { + public: + // The number of bits set by NextRandom. + static constexpr int kPrngNumBits = 48; + + // `GetSkipCount()` returns the number of events to skip before some chosen + // event happens. For example, randomly tossing a coin, we will on average + // throw heads once before we get tails. We can simulate random coin tosses + // using GetSkipCount() as: + // + // ExponentialBiased eb; + // for (...) { + // int number_of_heads_before_tail = eb.GetSkipCount(1); + // for (int flips = 0; flips < number_of_heads_before_tail; ++flips) { + // printf("head..."); + // } + // printf("tail\n"); + // } + // + int64_t GetSkipCount(int64_t mean); + + // GetStride() returns the number of events required for a specific event to + // happen. See the class comments for a usage example. `GetStride()` is + // equivalent to `GetSkipCount(mean - 1) + 1`. When to use `GetStride()` or + // `GetSkipCount()` depends mostly on what best fits the use case. + int64_t GetStride(int64_t mean); + + // Computes a random number in the range [0, 1<<(kPrngNumBits+1) - 1] + // + // This is public to enable testing. + static uint64_t NextRandom(uint64_t rnd); + + private: + void Initialize(); + + uint64_t rng_{0}; + double bias_{0}; + bool initialized_{false}; +}; + +// Returns the next prng value. +// pRNG is: aX+b mod c with a = 0x5DEECE66D, b = 0xB, c = 1<<48 +// This is the lrand64 generator. +inline uint64_t ExponentialBiased::NextRandom(uint64_t rnd) { + const uint64_t prng_mult = uint64_t{0x5DEECE66D}; + const uint64_t prng_add = 0xB; + const uint64_t prng_mod_power = 48; + const uint64_t prng_mod_mask = + ~((~static_cast<uint64_t>(0)) << prng_mod_power); + return (prng_mult * rnd + prng_add) & prng_mod_mask; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/exponential_biased_test.cc b/third_party/abseil_cpp/absl/base/internal/exponential_biased_test.cc new file mode 100644 index 000000000000..075583ca6fd4 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/exponential_biased_test.cc @@ -0,0 +1,199 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/exponential_biased.h" + +#include <stddef.h> + +#include <cmath> +#include <cstdint> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" + +using ::testing::Ge; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +MATCHER_P2(IsBetween, a, b, + absl::StrCat(std::string(negation ? "isn't" : "is"), " between ", a, + " and ", b)) { + return a <= arg && arg <= b; +} + +// Tests of the quality of the random numbers generated +// This uses the Anderson Darling test for uniformity. +// See "Evaluating the Anderson-Darling Distribution" by Marsaglia +// for details. + +// Short cut version of ADinf(z), z>0 (from Marsaglia) +// This returns the p-value for Anderson Darling statistic in +// the limit as n-> infinity. For finite n, apply the error fix below. +double AndersonDarlingInf(double z) { + if (z < 2) { + return exp(-1.2337141 / z) / sqrt(z) * + (2.00012 + + (0.247105 - + (0.0649821 - (0.0347962 - (0.011672 - 0.00168691 * z) * z) * z) * + z) * + z); + } + return exp( + -exp(1.0776 - + (2.30695 - + (0.43424 - (0.082433 - (0.008056 - 0.0003146 * z) * z) * z) * z) * + z)); +} + +// Corrects the approximation error in AndersonDarlingInf for small values of n +// Add this to AndersonDarlingInf to get a better approximation +// (from Marsaglia) +double AndersonDarlingErrFix(int n, double x) { + if (x > 0.8) { + return (-130.2137 + + (745.2337 - + (1705.091 - (1950.646 - (1116.360 - 255.7844 * x) * x) * x) * x) * + x) / + n; + } + double cutoff = 0.01265 + 0.1757 / n; + if (x < cutoff) { + double t = x / cutoff; + t = sqrt(t) * (1 - t) * (49 * t - 102); + return t * (0.0037 / (n * n) + 0.00078 / n + 0.00006) / n; + } else { + double t = (x - cutoff) / (0.8 - cutoff); + t = -0.00022633 + + (6.54034 - (14.6538 - (14.458 - (8.259 - 1.91864 * t) * t) * t) * t) * + t; + return t * (0.04213 + 0.01365 / n) / n; + } +} + +// Returns the AndersonDarling p-value given n and the value of the statistic +double AndersonDarlingPValue(int n, double z) { + double ad = AndersonDarlingInf(z); + double errfix = AndersonDarlingErrFix(n, ad); + return ad + errfix; +} + +double AndersonDarlingStatistic(const std::vector<double>& random_sample) { + int n = random_sample.size(); + double ad_sum = 0; + for (int i = 0; i < n; i++) { + ad_sum += (2 * i + 1) * + std::log(random_sample[i] * (1 - random_sample[n - 1 - i])); + } + double ad_statistic = -n - 1 / static_cast<double>(n) * ad_sum; + return ad_statistic; +} + +// Tests if the array of doubles is uniformly distributed. +// Returns the p-value of the Anderson Darling Statistic +// for the given set of sorted random doubles +// See "Evaluating the Anderson-Darling Distribution" by +// Marsaglia and Marsaglia for details. +double AndersonDarlingTest(const std::vector<double>& random_sample) { + double ad_statistic = AndersonDarlingStatistic(random_sample); + double p = AndersonDarlingPValue(random_sample.size(), ad_statistic); + return p; +} + +TEST(ExponentialBiasedTest, CoinTossDemoWithGetSkipCount) { + ExponentialBiased eb; + for (int runs = 0; runs < 10; ++runs) { + for (int flips = eb.GetSkipCount(1); flips > 0; --flips) { + printf("head..."); + } + printf("tail\n"); + } + int heads = 0; + for (int i = 0; i < 10000000; i += 1 + eb.GetSkipCount(1)) { + ++heads; + } + printf("Heads = %d (%f%%)\n", heads, 100.0 * heads / 10000000); +} + +TEST(ExponentialBiasedTest, SampleDemoWithStride) { + ExponentialBiased eb; + int stride = eb.GetStride(10); + int samples = 0; + for (int i = 0; i < 10000000; ++i) { + if (--stride == 0) { + ++samples; + stride = eb.GetStride(10); + } + } + printf("Samples = %d (%f%%)\n", samples, 100.0 * samples / 10000000); +} + + +// Testing that NextRandom generates uniform random numbers. Applies the +// Anderson-Darling test for uniformity +TEST(ExponentialBiasedTest, TestNextRandom) { + for (auto n : std::vector<int>({ + 10, // Check short-range correlation + 100, 1000, + 10000 // Make sure there's no systemic error + })) { + uint64_t x = 1; + // This assumes that the prng returns 48 bit numbers + uint64_t max_prng_value = static_cast<uint64_t>(1) << 48; + // Initialize. + for (int i = 1; i <= 20; i++) { + x = ExponentialBiased::NextRandom(x); + } + std::vector<uint64_t> int_random_sample(n); + // Collect samples + for (int i = 0; i < n; i++) { + int_random_sample[i] = x; + x = ExponentialBiased::NextRandom(x); + } + // First sort them... + std::sort(int_random_sample.begin(), int_random_sample.end()); + std::vector<double> random_sample(n); + // Convert them to uniform randoms (in the range [0,1]) + for (int i = 0; i < n; i++) { + random_sample[i] = + static_cast<double>(int_random_sample[i]) / max_prng_value; + } + // Now compute the Anderson-Darling statistic + double ad_pvalue = AndersonDarlingTest(random_sample); + EXPECT_GT(std::min(ad_pvalue, 1 - ad_pvalue), 0.0001) + << "prng is not uniform: n = " << n << " p = " << ad_pvalue; + } +} + +// The generator needs to be available as a thread_local and as a static +// variable. +TEST(ExponentialBiasedTest, InitializationModes) { + ABSL_CONST_INIT static ExponentialBiased eb_static; + EXPECT_THAT(eb_static.GetSkipCount(2), Ge(0)); + +#ifdef ABSL_HAVE_THREAD_LOCAL + thread_local ExponentialBiased eb_thread; + EXPECT_THAT(eb_thread.GetSkipCount(2), Ge(0)); +#endif + + ExponentialBiased eb_stack; + EXPECT_THAT(eb_stack.GetSkipCount(2), Ge(0)); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/fast_type_id.h b/third_party/abseil_cpp/absl/base/internal/fast_type_id.h new file mode 100644 index 000000000000..3db59e83745b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/fast_type_id.h @@ -0,0 +1,48 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_FAST_TYPE_ID_H_ +#define ABSL_BASE_INTERNAL_FAST_TYPE_ID_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +template <typename Type> +struct FastTypeTag { + constexpr static char dummy_var = 0; +}; + +template <typename Type> +constexpr char FastTypeTag<Type>::dummy_var; + +// FastTypeId<Type>() evaluates at compile/link-time to a unique pointer for the +// passed-in type. These are meant to be good match for keys into maps or +// straight up comparisons. +using FastTypeIdType = const void*; + +template <typename Type> +constexpr inline FastTypeIdType FastTypeId() { + return &FastTypeTag<Type>::dummy_var; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_FAST_TYPE_ID_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/fast_type_id_test.cc b/third_party/abseil_cpp/absl/base/internal/fast_type_id_test.cc new file mode 100644 index 000000000000..16f3c1458bdf --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/fast_type_id_test.cc @@ -0,0 +1,123 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/fast_type_id.h" + +#include <cstdint> +#include <map> +#include <vector> + +#include "gtest/gtest.h" + +namespace { +namespace bi = absl::base_internal; + +// NOLINTNEXTLINE +#define PRIM_TYPES(A) \ + A(bool) \ + A(short) \ + A(unsigned short) \ + A(int) \ + A(unsigned int) \ + A(long) \ + A(unsigned long) \ + A(long long) \ + A(unsigned long long) \ + A(float) \ + A(double) \ + A(long double) + +TEST(FastTypeIdTest, PrimitiveTypes) { + bi::FastTypeIdType type_ids[] = { +#define A(T) bi::FastTypeId<T>(), + PRIM_TYPES(A) +#undef A +#define A(T) bi::FastTypeId<const T>(), + PRIM_TYPES(A) +#undef A +#define A(T) bi::FastTypeId<volatile T>(), + PRIM_TYPES(A) +#undef A +#define A(T) bi::FastTypeId<const volatile T>(), + PRIM_TYPES(A) +#undef A + }; + size_t total_type_ids = sizeof(type_ids) / sizeof(bi::FastTypeIdType); + + for (int i = 0; i < total_type_ids; ++i) { + EXPECT_EQ(type_ids[i], type_ids[i]); + for (int j = 0; j < i; ++j) { + EXPECT_NE(type_ids[i], type_ids[j]); + } + } +} + +#define FIXED_WIDTH_TYPES(A) \ + A(int8_t) \ + A(uint8_t) \ + A(int16_t) \ + A(uint16_t) \ + A(int32_t) \ + A(uint32_t) \ + A(int64_t) \ + A(uint64_t) + +TEST(FastTypeIdTest, FixedWidthTypes) { + bi::FastTypeIdType type_ids[] = { +#define A(T) bi::FastTypeId<T>(), + FIXED_WIDTH_TYPES(A) +#undef A +#define A(T) bi::FastTypeId<const T>(), + FIXED_WIDTH_TYPES(A) +#undef A +#define A(T) bi::FastTypeId<volatile T>(), + FIXED_WIDTH_TYPES(A) +#undef A +#define A(T) bi::FastTypeId<const volatile T>(), + FIXED_WIDTH_TYPES(A) +#undef A + }; + size_t total_type_ids = sizeof(type_ids) / sizeof(bi::FastTypeIdType); + + for (int i = 0; i < total_type_ids; ++i) { + EXPECT_EQ(type_ids[i], type_ids[i]); + for (int j = 0; j < i; ++j) { + EXPECT_NE(type_ids[i], type_ids[j]); + } + } +} + +TEST(FastTypeIdTest, AliasTypes) { + using int_alias = int; + EXPECT_EQ(bi::FastTypeId<int_alias>(), bi::FastTypeId<int>()); +} + +TEST(FastTypeIdTest, TemplateSpecializations) { + EXPECT_NE(bi::FastTypeId<std::vector<int>>(), + bi::FastTypeId<std::vector<long>>()); + + EXPECT_NE((bi::FastTypeId<std::map<int, float>>()), + (bi::FastTypeId<std::map<int, double>>())); +} + +struct Base {}; +struct Derived : Base {}; +struct PDerived : private Base {}; + +TEST(FastTypeIdTest, Inheritance) { + EXPECT_NE(bi::FastTypeId<Base>(), bi::FastTypeId<Derived>()); + EXPECT_NE(bi::FastTypeId<Base>(), bi::FastTypeId<PDerived>()); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/hide_ptr.h b/third_party/abseil_cpp/absl/base/internal/hide_ptr.h new file mode 100644 index 000000000000..1dba80909a9b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/hide_ptr.h @@ -0,0 +1,51 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_HIDE_PTR_H_ +#define ABSL_BASE_INTERNAL_HIDE_PTR_H_ + +#include <cstdint> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// Arbitrary value with high bits set. Xor'ing with it is unlikely +// to map one valid pointer to another valid pointer. +constexpr uintptr_t HideMask() { + return (uintptr_t{0xF03A5F7BU} << (sizeof(uintptr_t) - 4) * 8) | 0xF03A5F7BU; +} + +// Hide a pointer from the leak checker. For internal use only. +// Differs from absl::IgnoreLeak(ptr) in that absl::IgnoreLeak(ptr) causes ptr +// and all objects reachable from ptr to be ignored by the leak checker. +template <class T> +inline uintptr_t HidePtr(T* ptr) { + return reinterpret_cast<uintptr_t>(ptr) ^ HideMask(); +} + +// Return a pointer that has been hidden from the leak checker. +// For internal use only. +template <class T> +inline T* UnhidePtr(uintptr_t hidden) { + return reinterpret_cast<T*>(hidden ^ HideMask()); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_HIDE_PTR_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/identity.h b/third_party/abseil_cpp/absl/base/internal/identity.h new file mode 100644 index 000000000000..a3154ed7bc59 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/identity.h @@ -0,0 +1,37 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_IDENTITY_H_ +#define ABSL_BASE_INTERNAL_IDENTITY_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace internal { + +template <typename T> +struct identity { + typedef T type; +}; + +template <typename T> +using identity_t = typename identity<T>::type; + +} // namespace internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_IDENTITY_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/inline_variable.h b/third_party/abseil_cpp/absl/base/internal/inline_variable.h new file mode 100644 index 000000000000..130d8c2476dd --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/inline_variable.h @@ -0,0 +1,107 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_INLINE_VARIABLE_EMULATION_H_ +#define ABSL_BASE_INTERNAL_INLINE_VARIABLE_EMULATION_H_ + +#include <type_traits> + +#include "absl/base/internal/identity.h" + +// File: +// This file define a macro that allows the creation of or emulation of C++17 +// inline variables based on whether or not the feature is supported. + +//////////////////////////////////////////////////////////////////////////////// +// Macro: ABSL_INTERNAL_INLINE_CONSTEXPR(type, name, init) +// +// Description: +// Expands to the equivalent of an inline constexpr instance of the specified +// `type` and `name`, initialized to the value `init`. If the compiler being +// used is detected as supporting actual inline variables as a language +// feature, then the macro expands to an actual inline variable definition. +// +// Requires: +// `type` is a type that is usable in an extern variable declaration. +// +// Requires: `name` is a valid identifier +// +// Requires: +// `init` is an expression that can be used in the following definition: +// constexpr type name = init; +// +// Usage: +// +// // Equivalent to: `inline constexpr size_t variant_npos = -1;` +// ABSL_INTERNAL_INLINE_CONSTEXPR(size_t, variant_npos, -1); +// +// Differences in implementation: +// For a direct, language-level inline variable, decltype(name) will be the +// type that was specified along with const qualification, whereas for +// emulated inline variables, decltype(name) may be different (in practice +// it will likely be a reference type). +//////////////////////////////////////////////////////////////////////////////// + +#ifdef __cpp_inline_variables + +// Clang's -Wmissing-variable-declarations option erroneously warned that +// inline constexpr objects need to be pre-declared. This has now been fixed, +// but we will need to support this workaround for people building with older +// versions of clang. +// +// Bug: https://bugs.llvm.org/show_bug.cgi?id=35862 +// +// Note: +// identity_t is used here so that the const and name are in the +// appropriate place for pointer types, reference types, function pointer +// types, etc.. +#if defined(__clang__) +#define ABSL_INTERNAL_EXTERN_DECL(type, name) \ + extern const ::absl::internal::identity_t<type> name; +#else // Otherwise, just define the macro to do nothing. +#define ABSL_INTERNAL_EXTERN_DECL(type, name) +#endif // defined(__clang__) + +// See above comment at top of file for details. +#define ABSL_INTERNAL_INLINE_CONSTEXPR(type, name, init) \ + ABSL_INTERNAL_EXTERN_DECL(type, name) \ + inline constexpr ::absl::internal::identity_t<type> name = init + +#else + +// See above comment at top of file for details. +// +// Note: +// identity_t is used here so that the const and name are in the +// appropriate place for pointer types, reference types, function pointer +// types, etc.. +#define ABSL_INTERNAL_INLINE_CONSTEXPR(var_type, name, init) \ + template <class /*AbslInternalDummy*/ = void> \ + struct AbslInternalInlineVariableHolder##name { \ + static constexpr ::absl::internal::identity_t<var_type> kInstance = init; \ + }; \ + \ + template <class AbslInternalDummy> \ + constexpr ::absl::internal::identity_t<var_type> \ + AbslInternalInlineVariableHolder##name<AbslInternalDummy>::kInstance; \ + \ + static constexpr const ::absl::internal::identity_t<var_type>& \ + name = /* NOLINT */ \ + AbslInternalInlineVariableHolder##name<>::kInstance; \ + static_assert(sizeof(void (*)(decltype(name))) != 0, \ + "Silence unused variable warnings.") + +#endif // __cpp_inline_variables + +#endif // ABSL_BASE_INTERNAL_INLINE_VARIABLE_EMULATION_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/inline_variable_testing.h b/third_party/abseil_cpp/absl/base/internal/inline_variable_testing.h new file mode 100644 index 000000000000..3856b9f80f20 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/inline_variable_testing.h @@ -0,0 +1,46 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INLINE_VARIABLE_TESTING_H_ +#define ABSL_BASE_INLINE_VARIABLE_TESTING_H_ + +#include "absl/base/internal/inline_variable.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace inline_variable_testing_internal { + +struct Foo { + int value = 5; +}; + +ABSL_INTERNAL_INLINE_CONSTEXPR(Foo, inline_variable_foo, {}); +ABSL_INTERNAL_INLINE_CONSTEXPR(Foo, other_inline_variable_foo, {}); + +ABSL_INTERNAL_INLINE_CONSTEXPR(int, inline_variable_int, 5); +ABSL_INTERNAL_INLINE_CONSTEXPR(int, other_inline_variable_int, 5); + +ABSL_INTERNAL_INLINE_CONSTEXPR(void(*)(), inline_variable_fun_ptr, nullptr); + +const Foo& get_foo_a(); +const Foo& get_foo_b(); + +const int& get_int_a(); +const int& get_int_b(); + +} // namespace inline_variable_testing_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INLINE_VARIABLE_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/invoke.h b/third_party/abseil_cpp/absl/base/internal/invoke.h new file mode 100644 index 000000000000..5c71f328234f --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/invoke.h @@ -0,0 +1,187 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// absl::base_internal::invoke(f, args...) is an implementation of +// INVOKE(f, args...) from section [func.require] of the C++ standard. +// +// [func.require] +// Define INVOKE (f, t1, t2, ..., tN) as follows: +// 1. (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T +// and t1 is an object of type T or a reference to an object of type T or a +// reference to an object of a type derived from T; +// 2. ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a +// class T and t1 is not one of the types described in the previous item; +// 3. t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is +// an object of type T or a reference to an object of type T or a reference +// to an object of a type derived from T; +// 4. (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1 +// is not one of the types described in the previous item; +// 5. f(t1, t2, ..., tN) in all other cases. +// +// The implementation is SFINAE-friendly: substitution failure within invoke() +// isn't an error. + +#ifndef ABSL_BASE_INTERNAL_INVOKE_H_ +#define ABSL_BASE_INTERNAL_INVOKE_H_ + +#include <algorithm> +#include <type_traits> +#include <utility> + +#include "absl/meta/type_traits.h" + +// The following code is internal implementation detail. See the comment at the +// top of this file for the API documentation. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// The five classes below each implement one of the clauses from the definition +// of INVOKE. The inner class template Accept<F, Args...> checks whether the +// clause is applicable; static function template Invoke(f, args...) does the +// invocation. +// +// By separating the clause selection logic from invocation we make sure that +// Invoke() does exactly what the standard says. + +template <typename Derived> +struct StrippedAccept { + template <typename... Args> + struct Accept : Derived::template AcceptImpl<typename std::remove_cv< + typename std::remove_reference<Args>::type>::type...> {}; +}; + +// (t1.*f)(t2, ..., tN) when f is a pointer to a member function of a class T +// and t1 is an object of type T or a reference to an object of type T or a +// reference to an object of a type derived from T. +struct MemFunAndRef : StrippedAccept<MemFunAndRef> { + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename MemFunType, typename C, typename Obj, typename... Args> + struct AcceptImpl<MemFunType C::*, Obj, Args...> + : std::integral_constant<bool, std::is_base_of<C, Obj>::value && + absl::is_function<MemFunType>::value> { + }; + + template <typename MemFun, typename Obj, typename... Args> + static decltype((std::declval<Obj>().* + std::declval<MemFun>())(std::declval<Args>()...)) + Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { + return (std::forward<Obj>(obj).* + std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); + } +}; + +// ((*t1).*f)(t2, ..., tN) when f is a pointer to a member function of a +// class T and t1 is not one of the types described in the previous item. +struct MemFunAndPtr : StrippedAccept<MemFunAndPtr> { + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename MemFunType, typename C, typename Ptr, typename... Args> + struct AcceptImpl<MemFunType C::*, Ptr, Args...> + : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value && + absl::is_function<MemFunType>::value> { + }; + + template <typename MemFun, typename Ptr, typename... Args> + static decltype(((*std::declval<Ptr>()).* + std::declval<MemFun>())(std::declval<Args>()...)) + Invoke(MemFun&& mem_fun, Ptr&& ptr, Args&&... args) { + return ((*std::forward<Ptr>(ptr)).* + std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); + } +}; + +// t1.*f when N == 1 and f is a pointer to member data of a class T and t1 is +// an object of type T or a reference to an object of type T or a reference +// to an object of a type derived from T. +struct DataMemAndRef : StrippedAccept<DataMemAndRef> { + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename R, typename C, typename Obj> + struct AcceptImpl<R C::*, Obj> + : std::integral_constant<bool, std::is_base_of<C, Obj>::value && + !absl::is_function<R>::value> {}; + + template <typename DataMem, typename Ref> + static decltype(std::declval<Ref>().*std::declval<DataMem>()) Invoke( + DataMem&& data_mem, Ref&& ref) { + return std::forward<Ref>(ref).*std::forward<DataMem>(data_mem); + } +}; + +// (*t1).*f when N == 1 and f is a pointer to member data of a class T and t1 +// is not one of the types described in the previous item. +struct DataMemAndPtr : StrippedAccept<DataMemAndPtr> { + template <typename... Args> + struct AcceptImpl : std::false_type {}; + + template <typename R, typename C, typename Ptr> + struct AcceptImpl<R C::*, Ptr> + : std::integral_constant<bool, !std::is_base_of<C, Ptr>::value && + !absl::is_function<R>::value> {}; + + template <typename DataMem, typename Ptr> + static decltype((*std::declval<Ptr>()).*std::declval<DataMem>()) Invoke( + DataMem&& data_mem, Ptr&& ptr) { + return (*std::forward<Ptr>(ptr)).*std::forward<DataMem>(data_mem); + } +}; + +// f(t1, t2, ..., tN) in all other cases. +struct Callable { + // Callable doesn't have Accept because it's the last clause that gets picked + // when none of the previous clauses are applicable. + template <typename F, typename... Args> + static decltype(std::declval<F>()(std::declval<Args>()...)) Invoke( + F&& f, Args&&... args) { + return std::forward<F>(f)(std::forward<Args>(args)...); + } +}; + +// Resolves to the first matching clause. +template <typename... Args> +struct Invoker { + typedef typename std::conditional< + MemFunAndRef::Accept<Args...>::value, MemFunAndRef, + typename std::conditional< + MemFunAndPtr::Accept<Args...>::value, MemFunAndPtr, + typename std::conditional< + DataMemAndRef::Accept<Args...>::value, DataMemAndRef, + typename std::conditional<DataMemAndPtr::Accept<Args...>::value, + DataMemAndPtr, Callable>::type>::type>:: + type>::type type; +}; + +// The result type of Invoke<F, Args...>. +template <typename F, typename... Args> +using invoke_result_t = decltype(Invoker<F, Args...>::type::Invoke( + std::declval<F>(), std::declval<Args>()...)); + +// Invoke(f, args...) is an implementation of INVOKE(f, args...) from section +// [func.require] of the C++ standard. +template <typename F, typename... Args> +invoke_result_t<F, Args...> invoke(F&& f, Args&&... args) { + return Invoker<F, Args...>::type::Invoke(std::forward<F>(f), + std::forward<Args>(args)...); +} +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_INVOKE_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/low_level_alloc.cc b/third_party/abseil_cpp/absl/base/internal/low_level_alloc.cc new file mode 100644 index 000000000000..229ab9162d12 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/low_level_alloc.cc @@ -0,0 +1,620 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A low-level allocator that can be used by other low-level +// modules without introducing dependency cycles. +// This allocator is slow and wasteful of memory; +// it should not be used when performance is key. + +#include "absl/base/internal/low_level_alloc.h" + +#include <type_traits> + +#include "absl/base/call_once.h" +#include "absl/base/config.h" +#include "absl/base/internal/direct_mmap.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/macros.h" +#include "absl/base/thread_annotations.h" + +// LowLevelAlloc requires that the platform support low-level +// allocation of virtual memory. Platforms lacking this cannot use +// LowLevelAlloc. +#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING + +#ifndef _WIN32 +#include <pthread.h> +#include <signal.h> +#include <sys/mman.h> +#include <unistd.h> +#else +#include <windows.h> +#endif + +#include <string.h> +#include <algorithm> +#include <atomic> +#include <cerrno> +#include <cstddef> +#include <new> // for placement-new + +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" + +// MAP_ANONYMOUS +#if defined(__APPLE__) +// For mmap, Linux defines both MAP_ANONYMOUS and MAP_ANON and says MAP_ANON is +// deprecated. In Darwin, MAP_ANON is all there is. +#if !defined MAP_ANONYMOUS +#define MAP_ANONYMOUS MAP_ANON +#endif // !MAP_ANONYMOUS +#endif // __APPLE__ + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// A first-fit allocator with amortized logarithmic free() time. + +// --------------------------------------------------------------------------- +static const int kMaxLevel = 30; + +namespace { +// This struct describes one allocated block, or one free block. +struct AllocList { + struct Header { + // Size of entire region, including this field. Must be + // first. Valid in both allocated and unallocated blocks. + uintptr_t size; + + // kMagicAllocated or kMagicUnallocated xor this. + uintptr_t magic; + + // Pointer to parent arena. + LowLevelAlloc::Arena *arena; + + // Aligns regions to 0 mod 2*sizeof(void*). + void *dummy_for_alignment; + } header; + + // Next two fields: in unallocated blocks: freelist skiplist data + // in allocated blocks: overlaps with client data + + // Levels in skiplist used. + int levels; + + // Actually has levels elements. The AllocList node may not have room + // for all kMaxLevel entries. See max_fit in LLA_SkiplistLevels(). + AllocList *next[kMaxLevel]; +}; +} // namespace + +// --------------------------------------------------------------------------- +// A trivial skiplist implementation. This is used to keep the freelist +// in address order while taking only logarithmic time per insert and delete. + +// An integer approximation of log2(size/base) +// Requires size >= base. +static int IntLog2(size_t size, size_t base) { + int result = 0; + for (size_t i = size; i > base; i >>= 1) { // i == floor(size/2**result) + result++; + } + // floor(size / 2**result) <= base < floor(size / 2**(result-1)) + // => log2(size/(base+1)) <= result < 1+log2(size/base) + // => result ~= log2(size/base) + return result; +} + +// Return a random integer n: p(n)=1/(2**n) if 1 <= n; p(n)=0 if n < 1. +static int Random(uint32_t *state) { + uint32_t r = *state; + int result = 1; + while ((((r = r*1103515245 + 12345) >> 30) & 1) == 0) { + result++; + } + *state = r; + return result; +} + +// Return a number of skiplist levels for a node of size bytes, where +// base is the minimum node size. Compute level=log2(size / base)+n +// where n is 1 if random is false and otherwise a random number generated with +// the standard distribution for a skiplist: See Random() above. +// Bigger nodes tend to have more skiplist levels due to the log2(size / base) +// term, so first-fit searches touch fewer nodes. "level" is clipped so +// level<kMaxLevel and next[level-1] will fit in the node. +// 0 < LLA_SkiplistLevels(x,y,false) <= LLA_SkiplistLevels(x,y,true) < kMaxLevel +static int LLA_SkiplistLevels(size_t size, size_t base, uint32_t *random) { + // max_fit is the maximum number of levels that will fit in a node for the + // given size. We can't return more than max_fit, no matter what the + // random number generator says. + size_t max_fit = (size - offsetof(AllocList, next)) / sizeof(AllocList *); + int level = IntLog2(size, base) + (random != nullptr ? Random(random) : 1); + if (static_cast<size_t>(level) > max_fit) level = static_cast<int>(max_fit); + if (level > kMaxLevel-1) level = kMaxLevel - 1; + ABSL_RAW_CHECK(level >= 1, "block not big enough for even one level"); + return level; +} + +// Return "atleast", the first element of AllocList *head s.t. *atleast >= *e. +// For 0 <= i < head->levels, set prev[i] to "no_greater", where no_greater +// points to the last element at level i in the AllocList less than *e, or is +// head if no such element exists. +static AllocList *LLA_SkiplistSearch(AllocList *head, + AllocList *e, AllocList **prev) { + AllocList *p = head; + for (int level = head->levels - 1; level >= 0; level--) { + for (AllocList *n; (n = p->next[level]) != nullptr && n < e; p = n) { + } + prev[level] = p; + } + return (head->levels == 0) ? nullptr : prev[0]->next[0]; +} + +// Insert element *e into AllocList *head. Set prev[] as LLA_SkiplistSearch. +// Requires that e->levels be previously set by the caller (using +// LLA_SkiplistLevels()) +static void LLA_SkiplistInsert(AllocList *head, AllocList *e, + AllocList **prev) { + LLA_SkiplistSearch(head, e, prev); + for (; head->levels < e->levels; head->levels++) { // extend prev pointers + prev[head->levels] = head; // to all *e's levels + } + for (int i = 0; i != e->levels; i++) { // add element to list + e->next[i] = prev[i]->next[i]; + prev[i]->next[i] = e; + } +} + +// Remove element *e from AllocList *head. Set prev[] as LLA_SkiplistSearch(). +// Requires that e->levels be previous set by the caller (using +// LLA_SkiplistLevels()) +static void LLA_SkiplistDelete(AllocList *head, AllocList *e, + AllocList **prev) { + AllocList *found = LLA_SkiplistSearch(head, e, prev); + ABSL_RAW_CHECK(e == found, "element not in freelist"); + for (int i = 0; i != e->levels && prev[i]->next[i] == e; i++) { + prev[i]->next[i] = e->next[i]; + } + while (head->levels > 0 && head->next[head->levels - 1] == nullptr) { + head->levels--; // reduce head->levels if level unused + } +} + +// --------------------------------------------------------------------------- +// Arena implementation + +// Metadata for an LowLevelAlloc arena instance. +struct LowLevelAlloc::Arena { + // Constructs an arena with the given LowLevelAlloc flags. + explicit Arena(uint32_t flags_value); + + base_internal::SpinLock mu; + // Head of free list, sorted by address + AllocList freelist ABSL_GUARDED_BY(mu); + // Count of allocated blocks + int32_t allocation_count ABSL_GUARDED_BY(mu); + // flags passed to NewArena + const uint32_t flags; + // Result of sysconf(_SC_PAGESIZE) + const size_t pagesize; + // Lowest power of two >= max(16, sizeof(AllocList)) + const size_t round_up; + // Smallest allocation block size + const size_t min_size; + // PRNG state + uint32_t random ABSL_GUARDED_BY(mu); +}; + +namespace { +// Static storage space for the lazily-constructed, default global arena +// instances. We require this space because the whole point of LowLevelAlloc +// is to avoid relying on malloc/new. +alignas(LowLevelAlloc::Arena) unsigned char default_arena_storage[sizeof( + LowLevelAlloc::Arena)]; +alignas(LowLevelAlloc::Arena) unsigned char unhooked_arena_storage[sizeof( + LowLevelAlloc::Arena)]; +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING +alignas( + LowLevelAlloc::Arena) unsigned char unhooked_async_sig_safe_arena_storage + [sizeof(LowLevelAlloc::Arena)]; +#endif + +// We must use LowLevelCallOnce here to construct the global arenas, rather than +// using function-level statics, to avoid recursively invoking the scheduler. +absl::once_flag create_globals_once; + +void CreateGlobalArenas() { + new (&default_arena_storage) + LowLevelAlloc::Arena(LowLevelAlloc::kCallMallocHook); + new (&unhooked_arena_storage) LowLevelAlloc::Arena(0); +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + new (&unhooked_async_sig_safe_arena_storage) + LowLevelAlloc::Arena(LowLevelAlloc::kAsyncSignalSafe); +#endif +} + +// Returns a global arena that does not call into hooks. Used by NewArena() +// when kCallMallocHook is not set. +LowLevelAlloc::Arena* UnhookedArena() { + base_internal::LowLevelCallOnce(&create_globals_once, CreateGlobalArenas); + return reinterpret_cast<LowLevelAlloc::Arena*>(&unhooked_arena_storage); +} + +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING +// Returns a global arena that is async-signal safe. Used by NewArena() when +// kAsyncSignalSafe is set. +LowLevelAlloc::Arena *UnhookedAsyncSigSafeArena() { + base_internal::LowLevelCallOnce(&create_globals_once, CreateGlobalArenas); + return reinterpret_cast<LowLevelAlloc::Arena *>( + &unhooked_async_sig_safe_arena_storage); +} +#endif + +} // namespace + +// Returns the default arena, as used by LowLevelAlloc::Alloc() and friends. +LowLevelAlloc::Arena *LowLevelAlloc::DefaultArena() { + base_internal::LowLevelCallOnce(&create_globals_once, CreateGlobalArenas); + return reinterpret_cast<LowLevelAlloc::Arena*>(&default_arena_storage); +} + +// magic numbers to identify allocated and unallocated blocks +static const uintptr_t kMagicAllocated = 0x4c833e95U; +static const uintptr_t kMagicUnallocated = ~kMagicAllocated; + +namespace { +class ABSL_SCOPED_LOCKABLE ArenaLock { + public: + explicit ArenaLock(LowLevelAlloc::Arena *arena) + ABSL_EXCLUSIVE_LOCK_FUNCTION(arena->mu) + : arena_(arena) { +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { + sigset_t all; + sigfillset(&all); + mask_valid_ = pthread_sigmask(SIG_BLOCK, &all, &mask_) == 0; + } +#endif + arena_->mu.Lock(); + } + ~ArenaLock() { ABSL_RAW_CHECK(left_, "haven't left Arena region"); } + void Leave() ABSL_UNLOCK_FUNCTION() { + arena_->mu.Unlock(); +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if (mask_valid_) { + const int err = pthread_sigmask(SIG_SETMASK, &mask_, nullptr); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_sigmask failed: %d", err); + } + } +#endif + left_ = true; + } + + private: + bool left_ = false; // whether left region +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + bool mask_valid_ = false; + sigset_t mask_; // old mask of blocked signals +#endif + LowLevelAlloc::Arena *arena_; + ArenaLock(const ArenaLock &) = delete; + ArenaLock &operator=(const ArenaLock &) = delete; +}; +} // namespace + +// create an appropriate magic number for an object at "ptr" +// "magic" should be kMagicAllocated or kMagicUnallocated +inline static uintptr_t Magic(uintptr_t magic, AllocList::Header *ptr) { + return magic ^ reinterpret_cast<uintptr_t>(ptr); +} + +namespace { +size_t GetPageSize() { +#ifdef _WIN32 + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + return std::max(system_info.dwPageSize, system_info.dwAllocationGranularity); +#elif defined(__wasm__) || defined(__asmjs__) + return getpagesize(); +#else + return sysconf(_SC_PAGESIZE); +#endif +} + +size_t RoundedUpBlockSize() { + // Round up block sizes to a power of two close to the header size. + size_t round_up = 16; + while (round_up < sizeof(AllocList::Header)) { + round_up += round_up; + } + return round_up; +} + +} // namespace + +LowLevelAlloc::Arena::Arena(uint32_t flags_value) + : mu(base_internal::SCHEDULE_KERNEL_ONLY), + allocation_count(0), + flags(flags_value), + pagesize(GetPageSize()), + round_up(RoundedUpBlockSize()), + min_size(2 * round_up), + random(0) { + freelist.header.size = 0; + freelist.header.magic = + Magic(kMagicUnallocated, &freelist.header); + freelist.header.arena = this; + freelist.levels = 0; + memset(freelist.next, 0, sizeof(freelist.next)); +} + +// L < meta_data_arena->mu +LowLevelAlloc::Arena *LowLevelAlloc::NewArena(int32_t flags) { + Arena *meta_data_arena = DefaultArena(); +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if ((flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { + meta_data_arena = UnhookedAsyncSigSafeArena(); + } else // NOLINT(readability/braces) +#endif + if ((flags & LowLevelAlloc::kCallMallocHook) == 0) { + meta_data_arena = UnhookedArena(); + } + Arena *result = + new (AllocWithArena(sizeof (*result), meta_data_arena)) Arena(flags); + return result; +} + +// L < arena->mu, L < arena->arena->mu +bool LowLevelAlloc::DeleteArena(Arena *arena) { + ABSL_RAW_CHECK( + arena != nullptr && arena != DefaultArena() && arena != UnhookedArena(), + "may not delete default arena"); + ArenaLock section(arena); + if (arena->allocation_count != 0) { + section.Leave(); + return false; + } + while (arena->freelist.next[0] != nullptr) { + AllocList *region = arena->freelist.next[0]; + size_t size = region->header.size; + arena->freelist.next[0] = region->next[0]; + ABSL_RAW_CHECK( + region->header.magic == Magic(kMagicUnallocated, ®ion->header), + "bad magic number in DeleteArena()"); + ABSL_RAW_CHECK(region->header.arena == arena, + "bad arena pointer in DeleteArena()"); + ABSL_RAW_CHECK(size % arena->pagesize == 0, + "empty arena has non-page-aligned block size"); + ABSL_RAW_CHECK(reinterpret_cast<uintptr_t>(region) % arena->pagesize == 0, + "empty arena has non-page-aligned block"); + int munmap_result; +#ifdef _WIN32 + munmap_result = VirtualFree(region, 0, MEM_RELEASE); + ABSL_RAW_CHECK(munmap_result != 0, + "LowLevelAlloc::DeleteArena: VitualFree failed"); +#else +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) == 0) { + munmap_result = munmap(region, size); + } else { + munmap_result = base_internal::DirectMunmap(region, size); + } +#else + munmap_result = munmap(region, size); +#endif // ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if (munmap_result != 0) { + ABSL_RAW_LOG(FATAL, "LowLevelAlloc::DeleteArena: munmap failed: %d", + errno); + } +#endif // _WIN32 + } + section.Leave(); + arena->~Arena(); + Free(arena); + return true; +} + +// --------------------------------------------------------------------------- + +// Addition, checking for overflow. The intent is to die if an external client +// manages to push through a request that would cause arithmetic to fail. +static inline uintptr_t CheckedAdd(uintptr_t a, uintptr_t b) { + uintptr_t sum = a + b; + ABSL_RAW_CHECK(sum >= a, "LowLevelAlloc arithmetic overflow"); + return sum; +} + +// Return value rounded up to next multiple of align. +// align must be a power of two. +static inline uintptr_t RoundUp(uintptr_t addr, uintptr_t align) { + return CheckedAdd(addr, align - 1) & ~(align - 1); +} + +// Equivalent to "return prev->next[i]" but with sanity checking +// that the freelist is in the correct order, that it +// consists of regions marked "unallocated", and that no two regions +// are adjacent in memory (they should have been coalesced). +// L >= arena->mu +static AllocList *Next(int i, AllocList *prev, LowLevelAlloc::Arena *arena) { + ABSL_RAW_CHECK(i < prev->levels, "too few levels in Next()"); + AllocList *next = prev->next[i]; + if (next != nullptr) { + ABSL_RAW_CHECK( + next->header.magic == Magic(kMagicUnallocated, &next->header), + "bad magic number in Next()"); + ABSL_RAW_CHECK(next->header.arena == arena, "bad arena pointer in Next()"); + if (prev != &arena->freelist) { + ABSL_RAW_CHECK(prev < next, "unordered freelist"); + ABSL_RAW_CHECK(reinterpret_cast<char *>(prev) + prev->header.size < + reinterpret_cast<char *>(next), + "malformed freelist"); + } + } + return next; +} + +// Coalesce list item "a" with its successor if they are adjacent. +static void Coalesce(AllocList *a) { + AllocList *n = a->next[0]; + if (n != nullptr && reinterpret_cast<char *>(a) + a->header.size == + reinterpret_cast<char *>(n)) { + LowLevelAlloc::Arena *arena = a->header.arena; + a->header.size += n->header.size; + n->header.magic = 0; + n->header.arena = nullptr; + AllocList *prev[kMaxLevel]; + LLA_SkiplistDelete(&arena->freelist, n, prev); + LLA_SkiplistDelete(&arena->freelist, a, prev); + a->levels = LLA_SkiplistLevels(a->header.size, arena->min_size, + &arena->random); + LLA_SkiplistInsert(&arena->freelist, a, prev); + } +} + +// Adds block at location "v" to the free list +// L >= arena->mu +static void AddToFreelist(void *v, LowLevelAlloc::Arena *arena) { + AllocList *f = reinterpret_cast<AllocList *>( + reinterpret_cast<char *>(v) - sizeof (f->header)); + ABSL_RAW_CHECK(f->header.magic == Magic(kMagicAllocated, &f->header), + "bad magic number in AddToFreelist()"); + ABSL_RAW_CHECK(f->header.arena == arena, + "bad arena pointer in AddToFreelist()"); + f->levels = LLA_SkiplistLevels(f->header.size, arena->min_size, + &arena->random); + AllocList *prev[kMaxLevel]; + LLA_SkiplistInsert(&arena->freelist, f, prev); + f->header.magic = Magic(kMagicUnallocated, &f->header); + Coalesce(f); // maybe coalesce with successor + Coalesce(prev[0]); // maybe coalesce with predecessor +} + +// Frees storage allocated by LowLevelAlloc::Alloc(). +// L < arena->mu +void LowLevelAlloc::Free(void *v) { + if (v != nullptr) { + AllocList *f = reinterpret_cast<AllocList *>( + reinterpret_cast<char *>(v) - sizeof (f->header)); + LowLevelAlloc::Arena *arena = f->header.arena; + ArenaLock section(arena); + AddToFreelist(v, arena); + ABSL_RAW_CHECK(arena->allocation_count > 0, "nothing in arena to free"); + arena->allocation_count--; + section.Leave(); + } +} + +// allocates and returns a block of size bytes, to be freed with Free() +// L < arena->mu +static void *DoAllocWithArena(size_t request, LowLevelAlloc::Arena *arena) { + void *result = nullptr; + if (request != 0) { + AllocList *s; // will point to region that satisfies request + ArenaLock section(arena); + // round up with header + size_t req_rnd = RoundUp(CheckedAdd(request, sizeof (s->header)), + arena->round_up); + for (;;) { // loop until we find a suitable region + // find the minimum levels that a block of this size must have + int i = LLA_SkiplistLevels(req_rnd, arena->min_size, nullptr) - 1; + if (i < arena->freelist.levels) { // potential blocks exist + AllocList *before = &arena->freelist; // predecessor of s + while ((s = Next(i, before, arena)) != nullptr && + s->header.size < req_rnd) { + before = s; + } + if (s != nullptr) { // we found a region + break; + } + } + // we unlock before mmap() both because mmap() may call a callback hook, + // and because it may be slow. + arena->mu.Unlock(); + // mmap generous 64K chunks to decrease + // the chances/impact of fragmentation: + size_t new_pages_size = RoundUp(req_rnd, arena->pagesize * 16); + void *new_pages; +#ifdef _WIN32 + new_pages = VirtualAlloc(0, new_pages_size, + MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); + ABSL_RAW_CHECK(new_pages != nullptr, "VirtualAlloc failed"); +#else +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if ((arena->flags & LowLevelAlloc::kAsyncSignalSafe) != 0) { + new_pages = base_internal::DirectMmap(nullptr, new_pages_size, + PROT_WRITE|PROT_READ, MAP_ANONYMOUS|MAP_PRIVATE, -1, 0); + } else { + new_pages = mmap(nullptr, new_pages_size, PROT_WRITE | PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); + } +#else + new_pages = mmap(nullptr, new_pages_size, PROT_WRITE | PROT_READ, + MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); +#endif // ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + if (new_pages == MAP_FAILED) { + ABSL_RAW_LOG(FATAL, "mmap error: %d", errno); + } + +#endif // _WIN32 + arena->mu.Lock(); + s = reinterpret_cast<AllocList *>(new_pages); + s->header.size = new_pages_size; + // Pretend the block is allocated; call AddToFreelist() to free it. + s->header.magic = Magic(kMagicAllocated, &s->header); + s->header.arena = arena; + AddToFreelist(&s->levels, arena); // insert new region into free list + } + AllocList *prev[kMaxLevel]; + LLA_SkiplistDelete(&arena->freelist, s, prev); // remove from free list + // s points to the first free region that's big enough + if (CheckedAdd(req_rnd, arena->min_size) <= s->header.size) { + // big enough to split + AllocList *n = reinterpret_cast<AllocList *> + (req_rnd + reinterpret_cast<char *>(s)); + n->header.size = s->header.size - req_rnd; + n->header.magic = Magic(kMagicAllocated, &n->header); + n->header.arena = arena; + s->header.size = req_rnd; + AddToFreelist(&n->levels, arena); + } + s->header.magic = Magic(kMagicAllocated, &s->header); + ABSL_RAW_CHECK(s->header.arena == arena, ""); + arena->allocation_count++; + section.Leave(); + result = &s->levels; + } + ABSL_ANNOTATE_MEMORY_IS_UNINITIALIZED(result, request); + return result; +} + +void *LowLevelAlloc::Alloc(size_t request) { + void *result = DoAllocWithArena(request, DefaultArena()); + return result; +} + +void *LowLevelAlloc::AllocWithArena(size_t request, Arena *arena) { + ABSL_RAW_CHECK(arena != nullptr, "must pass a valid arena"); + void *result = DoAllocWithArena(request, arena); + return result; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/third_party/abseil_cpp/absl/base/internal/low_level_alloc.h b/third_party/abseil_cpp/absl/base/internal/low_level_alloc.h new file mode 100644 index 000000000000..db91951c825f --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/low_level_alloc.h @@ -0,0 +1,126 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_ +#define ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_ + +// A simple thread-safe memory allocator that does not depend on +// mutexes or thread-specific data. It is intended to be used +// sparingly, and only when malloc() would introduce an unwanted +// dependency, such as inside the heap-checker, or the Mutex +// implementation. + +// IWYU pragma: private, include "base/low_level_alloc.h" + +#include <sys/types.h> + +#include <cstdint> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" + +// LowLevelAlloc requires that the platform support low-level +// allocation of virtual memory. Platforms lacking this cannot use +// LowLevelAlloc. +#ifdef ABSL_LOW_LEVEL_ALLOC_MISSING +#error ABSL_LOW_LEVEL_ALLOC_MISSING cannot be directly set +#elif !defined(ABSL_HAVE_MMAP) && !defined(_WIN32) +#define ABSL_LOW_LEVEL_ALLOC_MISSING 1 +#endif + +// Using LowLevelAlloc with kAsyncSignalSafe isn't supported on Windows or +// asm.js / WebAssembly. +// See https://kripken.github.io/emscripten-site/docs/porting/pthreads.html +// for more information. +#ifdef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING +#error ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING cannot be directly set +#elif defined(_WIN32) || defined(__asmjs__) || defined(__wasm__) +#define ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING 1 +#endif + +#include <cstddef> + +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +class LowLevelAlloc { + public: + struct Arena; // an arena from which memory may be allocated + + // Returns a pointer to a block of at least "request" bytes + // that have been newly allocated from the specific arena. + // for Alloc() call the DefaultArena() is used. + // Returns 0 if passed request==0. + // Does not return 0 under other circumstances; it crashes if memory + // is not available. + static void *Alloc(size_t request) ABSL_ATTRIBUTE_SECTION(malloc_hook); + static void *AllocWithArena(size_t request, Arena *arena) + ABSL_ATTRIBUTE_SECTION(malloc_hook); + + // Deallocates a region of memory that was previously allocated with + // Alloc(). Does nothing if passed 0. "s" must be either 0, + // or must have been returned from a call to Alloc() and not yet passed to + // Free() since that call to Alloc(). The space is returned to the arena + // from which it was allocated. + static void Free(void *s) ABSL_ATTRIBUTE_SECTION(malloc_hook); + + // ABSL_ATTRIBUTE_SECTION(malloc_hook) for Alloc* and Free + // are to put all callers of MallocHook::Invoke* in this module + // into special section, + // so that MallocHook::GetCallerStackTrace can function accurately. + + // Create a new arena. + // The root metadata for the new arena is allocated in the + // meta_data_arena; the DefaultArena() can be passed for meta_data_arena. + // These values may be ored into flags: + enum { + // Report calls to Alloc() and Free() via the MallocHook interface. + // Set in the DefaultArena. + kCallMallocHook = 0x0001, + +#ifndef ABSL_LOW_LEVEL_ALLOC_ASYNC_SIGNAL_SAFE_MISSING + // Make calls to Alloc(), Free() be async-signal-safe. Not set in + // DefaultArena(). Not supported on all platforms. + kAsyncSignalSafe = 0x0002, +#endif + }; + // Construct a new arena. The allocation of the underlying metadata honors + // the provided flags. For example, the call NewArena(kAsyncSignalSafe) + // is itself async-signal-safe, as well as generatating an arena that provides + // async-signal-safe Alloc/Free. + static Arena *NewArena(int32_t flags); + + // Destroys an arena allocated by NewArena and returns true, + // provided no allocated blocks remain in the arena. + // If allocated blocks remain in the arena, does nothing and + // returns false. + // It is illegal to attempt to destroy the DefaultArena(). + static bool DeleteArena(Arena *arena); + + // The default arena that always exists. + static Arena *DefaultArena(); + + private: + LowLevelAlloc(); // no instances +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_LOW_LEVEL_ALLOC_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/low_level_alloc_test.cc b/third_party/abseil_cpp/absl/base/internal/low_level_alloc_test.cc new file mode 100644 index 000000000000..2f2eaffa032e --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/low_level_alloc_test.cc @@ -0,0 +1,162 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/low_level_alloc.h" + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <thread> // NOLINT(build/c++11) +#include <unordered_map> +#include <utility> + +#include "absl/container/node_hash_map.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +// This test doesn't use gtest since it needs to test that everything +// works before main(). +#define TEST_ASSERT(x) \ + if (!(x)) { \ + printf("TEST_ASSERT(%s) FAILED ON LINE %d\n", #x, __LINE__); \ + abort(); \ + } + +// a block of memory obtained from the allocator +struct BlockDesc { + char *ptr; // pointer to memory + int len; // number of bytes + int fill; // filled with data starting with this +}; + +// Check that the pattern placed in the block d +// by RandomizeBlockDesc is still there. +static void CheckBlockDesc(const BlockDesc &d) { + for (int i = 0; i != d.len; i++) { + TEST_ASSERT((d.ptr[i] & 0xff) == ((d.fill + i) & 0xff)); + } +} + +// Fill the block "*d" with a pattern +// starting with a random byte. +static void RandomizeBlockDesc(BlockDesc *d) { + d->fill = rand() & 0xff; + for (int i = 0; i != d->len; i++) { + d->ptr[i] = (d->fill + i) & 0xff; + } +} + +// Use to indicate to the malloc hooks that +// this calls is from LowLevelAlloc. +static bool using_low_level_alloc = false; + +// n times, toss a coin, and based on the outcome +// either allocate a new block or deallocate an old block. +// New blocks are placed in a std::unordered_map with a random key +// and initialized with RandomizeBlockDesc(). +// If keys conflict, the older block is freed. +// Old blocks are always checked with CheckBlockDesc() +// before being freed. At the end of the run, +// all remaining allocated blocks are freed. +// If use_new_arena is true, use a fresh arena, and then delete it. +// If call_malloc_hook is true and user_arena is true, +// allocations and deallocations are reported via the MallocHook +// interface. +static void Test(bool use_new_arena, bool call_malloc_hook, int n) { + typedef absl::node_hash_map<int, BlockDesc> AllocMap; + AllocMap allocated; + AllocMap::iterator it; + BlockDesc block_desc; + int rnd; + LowLevelAlloc::Arena *arena = 0; + if (use_new_arena) { + int32_t flags = call_malloc_hook ? LowLevelAlloc::kCallMallocHook : 0; + arena = LowLevelAlloc::NewArena(flags); + } + for (int i = 0; i != n; i++) { + if (i != 0 && i % 10000 == 0) { + printf("."); + fflush(stdout); + } + + switch (rand() & 1) { // toss a coin + case 0: // coin came up heads: add a block + using_low_level_alloc = true; + block_desc.len = rand() & 0x3fff; + block_desc.ptr = + reinterpret_cast<char *>( + arena == 0 + ? LowLevelAlloc::Alloc(block_desc.len) + : LowLevelAlloc::AllocWithArena(block_desc.len, arena)); + using_low_level_alloc = false; + RandomizeBlockDesc(&block_desc); + rnd = rand(); + it = allocated.find(rnd); + if (it != allocated.end()) { + CheckBlockDesc(it->second); + using_low_level_alloc = true; + LowLevelAlloc::Free(it->second.ptr); + using_low_level_alloc = false; + it->second = block_desc; + } else { + allocated[rnd] = block_desc; + } + break; + case 1: // coin came up tails: remove a block + it = allocated.begin(); + if (it != allocated.end()) { + CheckBlockDesc(it->second); + using_low_level_alloc = true; + LowLevelAlloc::Free(it->second.ptr); + using_low_level_alloc = false; + allocated.erase(it); + } + break; + } + } + // remove all remaining blocks + while ((it = allocated.begin()) != allocated.end()) { + CheckBlockDesc(it->second); + using_low_level_alloc = true; + LowLevelAlloc::Free(it->second.ptr); + using_low_level_alloc = false; + allocated.erase(it); + } + if (use_new_arena) { + TEST_ASSERT(LowLevelAlloc::DeleteArena(arena)); + } +} + +// LowLevelAlloc is designed to be safe to call before main(). +static struct BeforeMain { + BeforeMain() { + Test(false, false, 50000); + Test(true, false, 50000); + Test(true, true, 50000); + } +} before_main; + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +int main(int argc, char *argv[]) { + // The actual test runs in the global constructor of `before_main`. + printf("PASS\n"); + return 0; +} diff --git a/third_party/abseil_cpp/absl/base/internal/low_level_scheduling.h b/third_party/abseil_cpp/absl/base/internal/low_level_scheduling.h new file mode 100644 index 000000000000..9baccc0659be --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/low_level_scheduling.h @@ -0,0 +1,134 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Core interfaces and definitions used by by low-level interfaces such as +// SpinLock. + +#ifndef ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_ +#define ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_ + +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/macros.h" + +// The following two declarations exist so SchedulingGuard may friend them with +// the appropriate language linkage. These callbacks allow libc internals, such +// as function level statics, to schedule cooperatively when locking. +extern "C" bool __google_disable_rescheduling(void); +extern "C" void __google_enable_rescheduling(bool disable_result); + +namespace absl { +ABSL_NAMESPACE_BEGIN +class CondVar; +class Mutex; + +namespace synchronization_internal { +int MutexDelay(int32_t c, int mode); +} // namespace synchronization_internal + +namespace base_internal { + +class SchedulingHelper; // To allow use of SchedulingGuard. +class SpinLock; // To allow use of SchedulingGuard. + +// SchedulingGuard +// Provides guard semantics that may be used to disable cooperative rescheduling +// of the calling thread within specific program blocks. This is used to +// protect resources (e.g. low-level SpinLocks or Domain code) that cooperative +// scheduling depends on. +// +// Domain implementations capable of rescheduling in reaction to involuntary +// kernel thread actions (e.g blocking due to a pagefault or syscall) must +// guarantee that an annotated thread is not allowed to (cooperatively) +// reschedule until the annotated region is complete. +// +// It is an error to attempt to use a cooperatively scheduled resource (e.g. +// Mutex) within a rescheduling-disabled region. +// +// All methods are async-signal safe. +class SchedulingGuard { + public: + // Returns true iff the calling thread may be cooperatively rescheduled. + static bool ReschedulingIsAllowed(); + SchedulingGuard(const SchedulingGuard&) = delete; + SchedulingGuard& operator=(const SchedulingGuard&) = delete; + + private: + // Disable cooperative rescheduling of the calling thread. It may still + // initiate scheduling operations (e.g. wake-ups), however, it may not itself + // reschedule. Nestable. The returned result is opaque, clients should not + // attempt to interpret it. + // REQUIRES: Result must be passed to a pairing EnableScheduling(). + static bool DisableRescheduling(); + + // Marks the end of a rescheduling disabled region, previously started by + // DisableRescheduling(). + // REQUIRES: Pairs with innermost call (and result) of DisableRescheduling(). + static void EnableRescheduling(bool disable_result); + + // A scoped helper for {Disable, Enable}Rescheduling(). + // REQUIRES: destructor must run in same thread as constructor. + struct ScopedDisable { + ScopedDisable() { disabled = SchedulingGuard::DisableRescheduling(); } + ~ScopedDisable() { SchedulingGuard::EnableRescheduling(disabled); } + + bool disabled; + }; + + // A scoped helper to enable rescheduling temporarily. + // REQUIRES: destructor must run in same thread as constructor. + class ScopedEnable { + public: + ScopedEnable(); + ~ScopedEnable(); + + private: + int scheduling_disabled_depth_; + }; + + // Access to SchedulingGuard is explicitly permitted. + friend class absl::CondVar; + friend class absl::Mutex; + friend class SchedulingHelper; + friend class SpinLock; + friend int absl::synchronization_internal::MutexDelay(int32_t c, int mode); +}; + +//------------------------------------------------------------------------------ +// End of public interfaces. +//------------------------------------------------------------------------------ + +inline bool SchedulingGuard::ReschedulingIsAllowed() { + return false; +} + +inline bool SchedulingGuard::DisableRescheduling() { + return false; +} + +inline void SchedulingGuard::EnableRescheduling(bool /* disable_result */) { + return; +} + +inline SchedulingGuard::ScopedEnable::ScopedEnable() + : scheduling_disabled_depth_(0) {} +inline SchedulingGuard::ScopedEnable::~ScopedEnable() { + ABSL_RAW_CHECK(scheduling_disabled_depth_ == 0, "disable unused warning"); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_LOW_LEVEL_SCHEDULING_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/per_thread_tls.h b/third_party/abseil_cpp/absl/base/internal/per_thread_tls.h new file mode 100644 index 000000000000..cf5e97a0470e --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/per_thread_tls.h @@ -0,0 +1,52 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_PER_THREAD_TLS_H_ +#define ABSL_BASE_INTERNAL_PER_THREAD_TLS_H_ + +// This header defines two macros: +// +// If the platform supports thread-local storage: +// +// * ABSL_PER_THREAD_TLS_KEYWORD is the C keyword needed to declare a +// thread-local variable +// * ABSL_PER_THREAD_TLS is 1 +// +// Otherwise: +// +// * ABSL_PER_THREAD_TLS_KEYWORD is empty +// * ABSL_PER_THREAD_TLS is 0 +// +// Microsoft C supports thread-local storage. +// GCC supports it if the appropriate version of glibc is available, +// which the programmer can indicate by defining ABSL_HAVE_TLS + +#include "absl/base/port.h" // For ABSL_HAVE_TLS + +#if defined(ABSL_PER_THREAD_TLS) +#error ABSL_PER_THREAD_TLS cannot be directly set +#elif defined(ABSL_PER_THREAD_TLS_KEYWORD) +#error ABSL_PER_THREAD_TLS_KEYWORD cannot be directly set +#elif defined(ABSL_HAVE_TLS) +#define ABSL_PER_THREAD_TLS_KEYWORD __thread +#define ABSL_PER_THREAD_TLS 1 +#elif defined(_MSC_VER) +#define ABSL_PER_THREAD_TLS_KEYWORD __declspec(thread) +#define ABSL_PER_THREAD_TLS 1 +#else +#define ABSL_PER_THREAD_TLS_KEYWORD +#define ABSL_PER_THREAD_TLS 0 +#endif + +#endif // ABSL_BASE_INTERNAL_PER_THREAD_TLS_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/periodic_sampler.cc b/third_party/abseil_cpp/absl/base/internal/periodic_sampler.cc new file mode 100644 index 000000000000..520dabbaa006 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/periodic_sampler.cc @@ -0,0 +1,53 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/periodic_sampler.h" + +#include <atomic> + +#include "absl/base/internal/exponential_biased.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +int64_t PeriodicSamplerBase::GetExponentialBiased(int period) noexcept { + return rng_.GetStride(period); +} + +bool PeriodicSamplerBase::SubtleConfirmSample() noexcept { + int current_period = period(); + + // Deal with period case 0 (always off) and 1 (always on) + if (ABSL_PREDICT_FALSE(current_period < 2)) { + stride_ = 0; + return current_period == 1; + } + + // Check if this is the first call to Sample() + if (ABSL_PREDICT_FALSE(stride_ == 1)) { + stride_ = static_cast<uint64_t>(-GetExponentialBiased(current_period)); + if (static_cast<int64_t>(stride_) < -1) { + ++stride_; + return false; + } + } + + stride_ = static_cast<uint64_t>(-GetExponentialBiased(current_period)); + return true; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/periodic_sampler.h b/third_party/abseil_cpp/absl/base/internal/periodic_sampler.h new file mode 100644 index 000000000000..f8a86796b117 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/periodic_sampler.h @@ -0,0 +1,211 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ +#define ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ + +#include <stdint.h> + +#include <atomic> + +#include "absl/base/internal/exponential_biased.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// PeriodicSamplerBase provides the basic period sampler implementation. +// +// This is the base class for the templated PeriodicSampler class, which holds +// a global std::atomic value identified by a user defined tag, such that +// each specific PeriodSampler implementation holds its own global period. +// +// PeriodicSamplerBase is thread-compatible except where stated otherwise. +class PeriodicSamplerBase { + public: + // PeriodicSamplerBase is trivial / copyable / movable / destructible. + PeriodicSamplerBase() = default; + PeriodicSamplerBase(PeriodicSamplerBase&&) = default; + PeriodicSamplerBase(const PeriodicSamplerBase&) = default; + + // Returns true roughly once every `period` calls. This is established by a + // randomly picked `stride` that is counted down on each call to `Sample`. + // This stride is picked such that the probability of `Sample()` returning + // true is 1 in `period`. + inline bool Sample() noexcept; + + // The below methods are intended for optimized use cases where the + // size of the inlined fast path code is highly important. Applications + // should use the `Sample()` method unless they have proof that their + // specific use case requires the optimizations offered by these methods. + // + // An example of such a use case is SwissTable sampling. All sampling checks + // are in inlined SwissTable methods, and the number of call sites is huge. + // In this case, the inlined code size added to each translation unit calling + // SwissTable methods is non-trivial. + // + // The `SubtleMaybeSample()` function spuriously returns true even if the + // function should not be sampled, applications MUST match each call to + // 'SubtleMaybeSample()' returning true with a `SubtleConfirmSample()` call, + // and use the result of the latter as the sampling decision. + // In other words: the code should logically be equivalent to: + // + // if (SubtleMaybeSample() && SubtleConfirmSample()) { + // // Sample this call + // } + // + // In the 'inline-size' optimized case, the `SubtleConfirmSample()` call can + // be placed out of line, for example, the typical use case looks as follows: + // + // // --- frobber.h ----------- + // void FrobberSampled(); + // + // inline void FrobberImpl() { + // // ... + // } + // + // inline void Frobber() { + // if (ABSL_PREDICT_FALSE(sampler.SubtleMaybeSample())) { + // FrobberSampled(); + // } else { + // FrobberImpl(); + // } + // } + // + // // --- frobber.cc ----------- + // void FrobberSampled() { + // if (!sampler.SubtleConfirmSample())) { + // // Spurious false positive + // FrobberImpl(); + // return; + // } + // + // // Sampled execution + // // ... + // } + inline bool SubtleMaybeSample() noexcept; + bool SubtleConfirmSample() noexcept; + + protected: + // We explicitly don't use a virtual destructor as this class is never + // virtually destroyed, and it keeps the class trivial, which avoids TLS + // prologue and epilogue code for our TLS instances. + ~PeriodicSamplerBase() = default; + + // Returns the next stride for our sampler. + // This function is virtual for testing purposes only. + virtual int64_t GetExponentialBiased(int period) noexcept; + + private: + // Returns the current period of this sampler. Thread-safe. + virtual int period() const noexcept = 0; + + // Keep and decrement stride_ as an unsigned integer, but compare the value + // to zero casted as a signed int. clang and msvc do not create optimum code + // if we use signed for the combined decrement and sign comparison. + // + // Below 3 alternative options, all compiles generate the best code + // using the unsigned increment <---> signed int comparison option. + // + // Option 1: + // int64_t stride_; + // if (ABSL_PREDICT_TRUE(++stride_ < 0)) { ... } + // + // GCC x64 (OK) : https://gcc.godbolt.org/z/R5MzzA + // GCC ppc (OK) : https://gcc.godbolt.org/z/z7NZAt + // Clang x64 (BAD): https://gcc.godbolt.org/z/t4gPsd + // ICC x64 (OK) : https://gcc.godbolt.org/z/rE6s8W + // MSVC x64 (OK) : https://gcc.godbolt.org/z/ARMXqS + // + // Option 2: + // int64_t stride_ = 0; + // if (ABSL_PREDICT_TRUE(--stride_ >= 0)) { ... } + // + // GCC x64 (OK) : https://gcc.godbolt.org/z/jSQxYK + // GCC ppc (OK) : https://gcc.godbolt.org/z/VJdYaA + // Clang x64 (BAD): https://gcc.godbolt.org/z/Xm4NjX + // ICC x64 (OK) : https://gcc.godbolt.org/z/4snaFd + // MSVC x64 (BAD): https://gcc.godbolt.org/z/BgnEKE + // + // Option 3: + // uint64_t stride_; + // if (ABSL_PREDICT_TRUE(static_cast<int64_t>(++stride_) < 0)) { ... } + // + // GCC x64 (OK) : https://gcc.godbolt.org/z/bFbfPy + // GCC ppc (OK) : https://gcc.godbolt.org/z/S9KkUE + // Clang x64 (OK) : https://gcc.godbolt.org/z/UYzRb4 + // ICC x64 (OK) : https://gcc.godbolt.org/z/ptTNfD + // MSVC x64 (OK) : https://gcc.godbolt.org/z/76j4-5 + uint64_t stride_ = 0; + ExponentialBiased rng_; +}; + +inline bool PeriodicSamplerBase::SubtleMaybeSample() noexcept { + // See comments on `stride_` for the unsigned increment / signed compare. + if (ABSL_PREDICT_TRUE(static_cast<int64_t>(++stride_) < 0)) { + return false; + } + return true; +} + +inline bool PeriodicSamplerBase::Sample() noexcept { + return ABSL_PREDICT_FALSE(SubtleMaybeSample()) ? SubtleConfirmSample() + : false; +} + +// PeriodicSampler is a concreted periodic sampler implementation. +// The user provided Tag identifies the implementation, and is required to +// isolate the global state of this instance from other instances. +// +// Typical use case: +// +// struct HashTablezTag {}; +// thread_local PeriodicSampler sampler; +// +// void HashTableSamplingLogic(...) { +// if (sampler.Sample()) { +// HashTableSlowSamplePath(...); +// } +// } +// +template <typename Tag, int default_period = 0> +class PeriodicSampler final : public PeriodicSamplerBase { + public: + ~PeriodicSampler() = default; + + int period() const noexcept final { + return period_.load(std::memory_order_relaxed); + } + + // Sets the global period for this sampler. Thread-safe. + // Setting a period of 0 disables the sampler, i.e., every call to Sample() + // will return false. Setting a period of 1 puts the sampler in 'always on' + // mode, i.e., every call to Sample() returns true. + static void SetGlobalPeriod(int period) { + period_.store(period, std::memory_order_relaxed); + } + + private: + static std::atomic<int> period_; +}; + +template <typename Tag, int default_period> +std::atomic<int> PeriodicSampler<Tag, default_period>::period_(default_period); + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/periodic_sampler_benchmark.cc b/third_party/abseil_cpp/absl/base/internal/periodic_sampler_benchmark.cc new file mode 100644 index 000000000000..5ad469ce79c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/periodic_sampler_benchmark.cc @@ -0,0 +1,79 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "absl/base/internal/periodic_sampler.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +template <typename Sampler> +void BM_Sample(Sampler* sampler, benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(sampler); + benchmark::DoNotOptimize(sampler->Sample()); + } +} + +template <typename Sampler> +void BM_SampleMinunumInlined(Sampler* sampler, benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(sampler); + if (ABSL_PREDICT_FALSE(sampler->SubtleMaybeSample())) { + benchmark::DoNotOptimize(sampler->SubtleConfirmSample()); + } + } +} + +void BM_PeriodicSampler_TinySample(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 10> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_TinySample); + +void BM_PeriodicSampler_ShortSample(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 1024> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_ShortSample); + +void BM_PeriodicSampler_LongSample(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 1024 * 1024> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_LongSample); + +void BM_PeriodicSampler_LongSampleMinunumInlined(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 1024 * 1024> sampler; + BM_SampleMinunumInlined(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_LongSampleMinunumInlined); + +void BM_PeriodicSampler_Disabled(benchmark::State& state) { + struct Tag {}; + PeriodicSampler<Tag, 0> sampler; + BM_Sample(&sampler, state); +} +BENCHMARK(BM_PeriodicSampler_Disabled); + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/periodic_sampler_test.cc b/third_party/abseil_cpp/absl/base/internal/periodic_sampler_test.cc new file mode 100644 index 000000000000..3b301e37ab98 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/periodic_sampler_test.cc @@ -0,0 +1,177 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/periodic_sampler.h" + +#include <thread> // NOLINT(build/c++11) + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +using testing::Eq; +using testing::Return; +using testing::StrictMock; + +class MockPeriodicSampler : public PeriodicSamplerBase { + public: + virtual ~MockPeriodicSampler() = default; + + MOCK_METHOD(int, period, (), (const, noexcept)); + MOCK_METHOD(int64_t, GetExponentialBiased, (int), (noexcept)); +}; + +TEST(PeriodicSamplerBaseTest, Sample) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)) + .WillOnce(Return(2)) + .WillOnce(Return(3)) + .WillOnce(Return(4)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, ImmediatelySample) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)) + .WillOnce(Return(1)) + .WillOnce(Return(2)) + .WillOnce(Return(3)); + + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, Disabled) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(0)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, AlwaysOn) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).Times(3).WillRepeatedly(Return(1)); + + EXPECT_TRUE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, Disable) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).WillOnce(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)).WillOnce(Return(3)); + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + + EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(0)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerBaseTest, Enable) { + StrictMock<MockPeriodicSampler> sampler; + + EXPECT_CALL(sampler, period()).WillOnce(Return(0)); + EXPECT_FALSE(sampler.Sample()); + + EXPECT_CALL(sampler, period()).Times(2).WillRepeatedly(Return(16)); + EXPECT_CALL(sampler, GetExponentialBiased(16)) + .Times(2) + .WillRepeatedly(Return(3)); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); + EXPECT_TRUE(sampler.Sample()); + + EXPECT_FALSE(sampler.Sample()); + EXPECT_FALSE(sampler.Sample()); +} + +TEST(PeriodicSamplerTest, ConstructConstInit) { + struct Tag {}; + ABSL_CONST_INIT static PeriodicSampler<Tag> sampler; + (void)sampler; +} + +TEST(PeriodicSamplerTest, DefaultPeriod0) { + struct Tag {}; + PeriodicSampler<Tag> sampler; + EXPECT_THAT(sampler.period(), Eq(0)); +} + +TEST(PeriodicSamplerTest, DefaultPeriod) { + struct Tag {}; + PeriodicSampler<Tag, 100> sampler; + EXPECT_THAT(sampler.period(), Eq(100)); +} + +TEST(PeriodicSamplerTest, SetGlobalPeriod) { + struct Tag1 {}; + struct Tag2 {}; + PeriodicSampler<Tag1, 25> sampler1; + PeriodicSampler<Tag2, 50> sampler2; + + EXPECT_THAT(sampler1.period(), Eq(25)); + EXPECT_THAT(sampler2.period(), Eq(50)); + + std::thread thread([] { + PeriodicSampler<Tag1, 25> sampler1; + PeriodicSampler<Tag2, 50> sampler2; + EXPECT_THAT(sampler1.period(), Eq(25)); + EXPECT_THAT(sampler2.period(), Eq(50)); + sampler1.SetGlobalPeriod(10); + sampler2.SetGlobalPeriod(20); + }); + thread.join(); + + EXPECT_THAT(sampler1.period(), Eq(10)); + EXPECT_THAT(sampler2.period(), Eq(20)); +} + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/pretty_function.h b/third_party/abseil_cpp/absl/base/internal/pretty_function.h new file mode 100644 index 000000000000..35d51676dc2e --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/pretty_function.h @@ -0,0 +1,33 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_PRETTY_FUNCTION_H_ +#define ABSL_BASE_INTERNAL_PRETTY_FUNCTION_H_ + +// ABSL_PRETTY_FUNCTION +// +// In C++11, __func__ gives the undecorated name of the current function. That +// is, "main", not "int main()". Various compilers give extra macros to get the +// decorated function name, including return type and arguments, to +// differentiate between overload sets. ABSL_PRETTY_FUNCTION is a portable +// version of these macros which forwards to the correct macro on each compiler. +#if defined(_MSC_VER) +#define ABSL_PRETTY_FUNCTION __FUNCSIG__ +#elif defined(__GNUC__) +#define ABSL_PRETTY_FUNCTION __PRETTY_FUNCTION__ +#else +#error "Unsupported compiler" +#endif + +#endif // ABSL_BASE_INTERNAL_PRETTY_FUNCTION_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/raw_logging.cc b/third_party/abseil_cpp/absl/base/internal/raw_logging.cc new file mode 100644 index 000000000000..ae8754c6a132 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/raw_logging.cc @@ -0,0 +1,240 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/raw_logging.h" + +#include <stddef.h> +#include <cstdarg> +#include <cstdio> +#include <cstdlib> +#include <cstring> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/atomic_hook.h" +#include "absl/base/log_severity.h" + +// We know how to perform low-level writes to stderr in POSIX and Windows. For +// these platforms, we define the token ABSL_LOW_LEVEL_WRITE_SUPPORTED. +// Much of raw_logging.cc becomes a no-op when we can't output messages, +// although a FATAL ABSL_RAW_LOG message will still abort the process. + +// ABSL_HAVE_POSIX_WRITE is defined when the platform provides posix write() +// (as from unistd.h) +// +// This preprocessor token is also defined in raw_io.cc. If you need to copy +// this, consider moving both to config.h instead. +#if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ + defined(__Fuchsia__) || defined(__native_client__) || \ + defined(__EMSCRIPTEN__) || defined(__ASYLO__) + +#include <unistd.h> + +#define ABSL_HAVE_POSIX_WRITE 1 +#define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 +#else +#undef ABSL_HAVE_POSIX_WRITE +#endif + +// ABSL_HAVE_SYSCALL_WRITE is defined when the platform provides the syscall +// syscall(SYS_write, /*int*/ fd, /*char* */ buf, /*size_t*/ len); +// for low level operations that want to avoid libc. +#if (defined(__linux__) || defined(__FreeBSD__)) && !defined(__ANDROID__) +#include <sys/syscall.h> +#define ABSL_HAVE_SYSCALL_WRITE 1 +#define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 +#else +#undef ABSL_HAVE_SYSCALL_WRITE +#endif + +#ifdef _WIN32 +#include <io.h> + +#define ABSL_HAVE_RAW_IO 1 +#define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 +#else +#undef ABSL_HAVE_RAW_IO +#endif + +// TODO(gfalcon): We want raw-logging to work on as many platforms as possible. +// Explicitly #error out when not ABSL_LOW_LEVEL_WRITE_SUPPORTED, except for a +// selected set of platforms for which we expect not to be able to raw log. + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + absl::raw_logging_internal::LogPrefixHook> + log_prefix_hook; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static absl::base_internal::AtomicHook< + absl::raw_logging_internal::AbortHook> + abort_hook; + +#ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED +static const char kTruncated[] = " ... (message truncated)\n"; + +// sprintf the format to the buffer, adjusting *buf and *size to reflect the +// consumed bytes, and return whether the message fit without truncation. If +// truncation occurred, if possible leave room in the buffer for the message +// kTruncated[]. +inline static bool VADoRawLog(char** buf, int* size, const char* format, + va_list ap) ABSL_PRINTF_ATTRIBUTE(3, 0); +inline static bool VADoRawLog(char** buf, int* size, + const char* format, va_list ap) { + int n = vsnprintf(*buf, *size, format, ap); + bool result = true; + if (n < 0 || n > *size) { + result = false; + if (static_cast<size_t>(*size) > sizeof(kTruncated)) { + n = *size - sizeof(kTruncated); // room for truncation message + } else { + n = 0; // no room for truncation message + } + } + *size -= n; + *buf += n; + return result; +} +#endif // ABSL_LOW_LEVEL_WRITE_SUPPORTED + +static constexpr int kLogBufSize = 3000; + +namespace { + +// CAVEAT: vsnprintf called from *DoRawLog below has some (exotic) code paths +// that invoke malloc() and getenv() that might acquire some locks. + +// Helper for RawLog below. +// *DoRawLog writes to *buf of *size and move them past the written portion. +// It returns true iff there was no overflow or error. +bool DoRawLog(char** buf, int* size, const char* format, ...) + ABSL_PRINTF_ATTRIBUTE(3, 4); +bool DoRawLog(char** buf, int* size, const char* format, ...) { + va_list ap; + va_start(ap, format); + int n = vsnprintf(*buf, *size, format, ap); + va_end(ap); + if (n < 0 || n > *size) return false; + *size -= n; + *buf += n; + return true; +} + +void RawLogVA(absl::LogSeverity severity, const char* file, int line, + const char* format, va_list ap) ABSL_PRINTF_ATTRIBUTE(4, 0); +void RawLogVA(absl::LogSeverity severity, const char* file, int line, + const char* format, va_list ap) { + char buffer[kLogBufSize]; + char* buf = buffer; + int size = sizeof(buffer); +#ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED + bool enabled = true; +#else + bool enabled = false; +#endif + +#ifdef ABSL_MIN_LOG_LEVEL + if (severity < static_cast<absl::LogSeverity>(ABSL_MIN_LOG_LEVEL) && + severity < absl::LogSeverity::kFatal) { + enabled = false; + } +#endif + + auto log_prefix_hook_ptr = log_prefix_hook.Load(); + if (log_prefix_hook_ptr) { + enabled = log_prefix_hook_ptr(severity, file, line, &buf, &size); + } else { + if (enabled) { + DoRawLog(&buf, &size, "[%s : %d] RAW: ", file, line); + } + } + const char* const prefix_end = buf; + +#ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED + if (enabled) { + bool no_chop = VADoRawLog(&buf, &size, format, ap); + if (no_chop) { + DoRawLog(&buf, &size, "\n"); + } else { + DoRawLog(&buf, &size, "%s", kTruncated); + } + absl::raw_logging_internal::SafeWriteToStderr(buffer, strlen(buffer)); + } +#else + static_cast<void>(format); + static_cast<void>(ap); +#endif + + // Abort the process after logging a FATAL message, even if the output itself + // was suppressed. + if (severity == absl::LogSeverity::kFatal) { + abort_hook(file, line, buffer, prefix_end, buffer + kLogBufSize); + abort(); + } +} + +} // namespace + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace raw_logging_internal { +void SafeWriteToStderr(const char *s, size_t len) { +#if defined(ABSL_HAVE_SYSCALL_WRITE) + syscall(SYS_write, STDERR_FILENO, s, len); +#elif defined(ABSL_HAVE_POSIX_WRITE) + write(STDERR_FILENO, s, len); +#elif defined(ABSL_HAVE_RAW_IO) + _write(/* stderr */ 2, s, len); +#else + // stderr logging unsupported on this platform + (void) s; + (void) len; +#endif +} + +void RawLog(absl::LogSeverity severity, const char* file, int line, + const char* format, ...) ABSL_PRINTF_ATTRIBUTE(4, 5); +void RawLog(absl::LogSeverity severity, const char* file, int line, + const char* format, ...) { + va_list ap; + va_start(ap, format); + RawLogVA(severity, file, line, format, ap); + va_end(ap); +} + +// Non-formatting version of RawLog(). +// +// TODO(gfalcon): When string_view no longer depends on base, change this +// interface to take its message as a string_view instead. +static void DefaultInternalLog(absl::LogSeverity severity, const char* file, + int line, const std::string& message) { + RawLog(severity, file, line, "%s", message.c_str()); +} + +bool RawLoggingFullySupported() { +#ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED + return true; +#else // !ABSL_LOW_LEVEL_WRITE_SUPPORTED + return false; +#endif // !ABSL_LOW_LEVEL_WRITE_SUPPORTED +} + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL + absl::base_internal::AtomicHook<InternalLogFunction> + internal_log_function(DefaultInternalLog); + +void RegisterInternalLogFunction(InternalLogFunction func) { + internal_log_function.Store(func); +} + +} // namespace raw_logging_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/raw_logging.h b/third_party/abseil_cpp/absl/base/internal/raw_logging.h new file mode 100644 index 000000000000..20f4291b162b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/raw_logging.h @@ -0,0 +1,187 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Thread-safe logging routines that do not allocate any memory or +// acquire any locks, and can therefore be used by low-level memory +// allocation, synchronization, and signal-handling code. + +#ifndef ABSL_BASE_INTERNAL_RAW_LOGGING_H_ +#define ABSL_BASE_INTERNAL_RAW_LOGGING_H_ + +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/atomic_hook.h" +#include "absl/base/log_severity.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" + +// This is similar to LOG(severity) << format..., but +// * it is to be used ONLY by low-level modules that can't use normal LOG() +// * it is designed to be a low-level logger that does not allocate any +// memory and does not need any locks, hence: +// * it logs straight and ONLY to STDERR w/o buffering +// * it uses an explicit printf-format and arguments list +// * it will silently chop off really long message strings +// Usage example: +// ABSL_RAW_LOG(ERROR, "Failed foo with %i: %s", status, error); +// This will print an almost standard log line like this to stderr only: +// E0821 211317 file.cc:123] RAW: Failed foo with 22: bad_file + +#define ABSL_RAW_LOG(severity, ...) \ + do { \ + constexpr const char* absl_raw_logging_internal_basename = \ + ::absl::raw_logging_internal::Basename(__FILE__, \ + sizeof(__FILE__) - 1); \ + ::absl::raw_logging_internal::RawLog(ABSL_RAW_LOGGING_INTERNAL_##severity, \ + absl_raw_logging_internal_basename, \ + __LINE__, __VA_ARGS__); \ + } while (0) + +// Similar to CHECK(condition) << message, but for low-level modules: +// we use only ABSL_RAW_LOG that does not allocate memory. +// We do not want to provide args list here to encourage this usage: +// if (!cond) ABSL_RAW_LOG(FATAL, "foo ...", hard_to_compute_args); +// so that the args are not computed when not needed. +#define ABSL_RAW_CHECK(condition, message) \ + do { \ + if (ABSL_PREDICT_FALSE(!(condition))) { \ + ABSL_RAW_LOG(FATAL, "Check %s failed: %s", #condition, message); \ + } \ + } while (0) + +// ABSL_INTERNAL_LOG and ABSL_INTERNAL_CHECK work like the RAW variants above, +// except that if the richer log library is linked into the binary, we dispatch +// to that instead. This is potentially useful for internal logging and +// assertions, where we are using RAW_LOG neither for its async-signal-safety +// nor for its non-allocating nature, but rather because raw logging has very +// few other dependencies. +// +// The API is a subset of the above: each macro only takes two arguments. Use +// StrCat if you need to build a richer message. +#define ABSL_INTERNAL_LOG(severity, message) \ + do { \ + constexpr const char* absl_raw_logging_internal_filename = __FILE__; \ + ::absl::raw_logging_internal::internal_log_function( \ + ABSL_RAW_LOGGING_INTERNAL_##severity, \ + absl_raw_logging_internal_filename, __LINE__, message); \ + if (ABSL_RAW_LOGGING_INTERNAL_##severity == ::absl::LogSeverity::kFatal) \ + ABSL_INTERNAL_UNREACHABLE; \ + } while (0) + +#define ABSL_INTERNAL_CHECK(condition, message) \ + do { \ + if (ABSL_PREDICT_FALSE(!(condition))) { \ + std::string death_message = "Check " #condition " failed: "; \ + death_message += std::string(message); \ + ABSL_INTERNAL_LOG(FATAL, death_message); \ + } \ + } while (0) + +#define ABSL_RAW_LOGGING_INTERNAL_INFO ::absl::LogSeverity::kInfo +#define ABSL_RAW_LOGGING_INTERNAL_WARNING ::absl::LogSeverity::kWarning +#define ABSL_RAW_LOGGING_INTERNAL_ERROR ::absl::LogSeverity::kError +#define ABSL_RAW_LOGGING_INTERNAL_FATAL ::absl::LogSeverity::kFatal +#define ABSL_RAW_LOGGING_INTERNAL_LEVEL(severity) \ + ::absl::NormalizeLogSeverity(severity) + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace raw_logging_internal { + +// Helper function to implement ABSL_RAW_LOG +// Logs format... at "severity" level, reporting it +// as called from file:line. +// This does not allocate memory or acquire locks. +void RawLog(absl::LogSeverity severity, const char* file, int line, + const char* format, ...) ABSL_PRINTF_ATTRIBUTE(4, 5); + +// Writes the provided buffer directly to stderr, in a safe, low-level manner. +// +// In POSIX this means calling write(), which is async-signal safe and does +// not malloc. If the platform supports the SYS_write syscall, we invoke that +// directly to side-step any libc interception. +void SafeWriteToStderr(const char *s, size_t len); + +// compile-time function to get the "base" filename, that is, the part of +// a filename after the last "/" or "\" path separator. The search starts at +// the end of the string; the second parameter is the length of the string. +constexpr const char* Basename(const char* fname, int offset) { + return offset == 0 || fname[offset - 1] == '/' || fname[offset - 1] == '\\' + ? fname + offset + : Basename(fname, offset - 1); +} + +// For testing only. +// Returns true if raw logging is fully supported. When it is not +// fully supported, no messages will be emitted, but a log at FATAL +// severity will cause an abort. +// +// TODO(gfalcon): Come up with a better name for this method. +bool RawLoggingFullySupported(); + +// Function type for a raw_logging customization hook for suppressing messages +// by severity, and for writing custom prefixes on non-suppressed messages. +// +// The installed hook is called for every raw log invocation. The message will +// be logged to stderr only if the hook returns true. FATAL errors will cause +// the process to abort, even if writing to stderr is suppressed. The hook is +// also provided with an output buffer, where it can write a custom log message +// prefix. +// +// The raw_logging system does not allocate memory or grab locks. User-provided +// hooks must avoid these operations, and must not throw exceptions. +// +// 'severity' is the severity level of the message being written. +// 'file' and 'line' are the file and line number where the ABSL_RAW_LOG macro +// was located. +// 'buffer' and 'buf_size' are pointers to the buffer and buffer size. If the +// hook writes a prefix, it must increment *buffer and decrement *buf_size +// accordingly. +using LogPrefixHook = bool (*)(absl::LogSeverity severity, const char* file, + int line, char** buffer, int* buf_size); + +// Function type for a raw_logging customization hook called to abort a process +// when a FATAL message is logged. If the provided AbortHook() returns, the +// logging system will call abort(). +// +// 'file' and 'line' are the file and line number where the ABSL_RAW_LOG macro +// was located. +// The NUL-terminated logged message lives in the buffer between 'buf_start' +// and 'buf_end'. 'prefix_end' points to the first non-prefix character of the +// buffer (as written by the LogPrefixHook.) +using AbortHook = void (*)(const char* file, int line, const char* buf_start, + const char* prefix_end, const char* buf_end); + +// Internal logging function for ABSL_INTERNAL_LOG to dispatch to. +// +// TODO(gfalcon): When string_view no longer depends on base, change this +// interface to take its message as a string_view instead. +using InternalLogFunction = void (*)(absl::LogSeverity severity, + const char* file, int line, + const std::string& message); + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL extern base_internal::AtomicHook< + InternalLogFunction> + internal_log_function; + +void RegisterInternalLogFunction(InternalLogFunction func); + +} // namespace raw_logging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_RAW_LOGGING_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/scheduling_mode.h b/third_party/abseil_cpp/absl/base/internal/scheduling_mode.h new file mode 100644 index 000000000000..8be5ab6dd3c0 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/scheduling_mode.h @@ -0,0 +1,58 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Core interfaces and definitions used by by low-level interfaces such as +// SpinLock. + +#ifndef ABSL_BASE_INTERNAL_SCHEDULING_MODE_H_ +#define ABSL_BASE_INTERNAL_SCHEDULING_MODE_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// Used to describe how a thread may be scheduled. Typically associated with +// the declaration of a resource supporting synchronized access. +// +// SCHEDULE_COOPERATIVE_AND_KERNEL: +// Specifies that when waiting, a cooperative thread (e.g. a Fiber) may +// reschedule (using base::scheduling semantics); allowing other cooperative +// threads to proceed. +// +// SCHEDULE_KERNEL_ONLY: (Also described as "non-cooperative") +// Specifies that no cooperative scheduling semantics may be used, even if the +// current thread is itself cooperatively scheduled. This means that +// cooperative threads will NOT allow other cooperative threads to execute in +// their place while waiting for a resource of this type. Host operating system +// semantics (e.g. a futex) may still be used. +// +// When optional, clients should strongly prefer SCHEDULE_COOPERATIVE_AND_KERNEL +// by default. SCHEDULE_KERNEL_ONLY should only be used for resources on which +// base::scheduling (e.g. the implementation of a Scheduler) may depend. +// +// NOTE: Cooperative resources may not be nested below non-cooperative ones. +// This means that it is invalid to to acquire a SCHEDULE_COOPERATIVE_AND_KERNEL +// resource if a SCHEDULE_KERNEL_ONLY resource is already held. +enum SchedulingMode { + SCHEDULE_KERNEL_ONLY = 0, // Allow scheduling only the host OS. + SCHEDULE_COOPERATIVE_AND_KERNEL, // Also allow cooperative scheduling. +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_SCHEDULING_MODE_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/scoped_set_env.cc b/third_party/abseil_cpp/absl/base/internal/scoped_set_env.cc new file mode 100644 index 000000000000..8a934cb51191 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/scoped_set_env.cc @@ -0,0 +1,81 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/scoped_set_env.h" + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include <cstdlib> + +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +namespace { + +#ifdef _WIN32 +const int kMaxEnvVarValueSize = 1024; +#endif + +void SetEnvVar(const char* name, const char* value) { +#ifdef _WIN32 + SetEnvironmentVariableA(name, value); +#else + if (value == nullptr) { + ::unsetenv(name); + } else { + ::setenv(name, value, 1); + } +#endif +} + +} // namespace + +ScopedSetEnv::ScopedSetEnv(const char* var_name, const char* new_value) + : var_name_(var_name), was_unset_(false) { +#ifdef _WIN32 + char buf[kMaxEnvVarValueSize]; + auto get_res = GetEnvironmentVariableA(var_name_.c_str(), buf, sizeof(buf)); + ABSL_INTERNAL_CHECK(get_res < sizeof(buf), "value exceeds buffer size"); + + if (get_res == 0) { + was_unset_ = (GetLastError() == ERROR_ENVVAR_NOT_FOUND); + } else { + old_value_.assign(buf, get_res); + } + + SetEnvironmentVariableA(var_name_.c_str(), new_value); +#else + const char* val = ::getenv(var_name_.c_str()); + if (val == nullptr) { + was_unset_ = true; + } else { + old_value_ = val; + } +#endif + + SetEnvVar(var_name_.c_str(), new_value); +} + +ScopedSetEnv::~ScopedSetEnv() { + SetEnvVar(var_name_.c_str(), was_unset_ ? nullptr : old_value_.c_str()); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/scoped_set_env.h b/third_party/abseil_cpp/absl/base/internal/scoped_set_env.h new file mode 100644 index 000000000000..19ec7b5d8a04 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/scoped_set_env.h @@ -0,0 +1,45 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_SCOPED_SET_ENV_H_ +#define ABSL_BASE_INTERNAL_SCOPED_SET_ENV_H_ + +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +class ScopedSetEnv { + public: + ScopedSetEnv(const char* var_name, const char* new_value); + ~ScopedSetEnv(); + + private: + std::string var_name_; + std::string old_value_; + + // True if the environment variable was initially not set. + bool was_unset_; +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_SCOPED_SET_ENV_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/scoped_set_env_test.cc b/third_party/abseil_cpp/absl/base/internal/scoped_set_env_test.cc new file mode 100644 index 000000000000..5cbad246c647 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/scoped_set_env_test.cc @@ -0,0 +1,99 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include "gtest/gtest.h" +#include "absl/base/internal/scoped_set_env.h" + +namespace { + +using absl::base_internal::ScopedSetEnv; + +std::string GetEnvVar(const char* name) { +#ifdef _WIN32 + char buf[1024]; + auto get_res = GetEnvironmentVariableA(name, buf, sizeof(buf)); + if (get_res >= sizeof(buf)) { + return "TOO_BIG"; + } + + if (get_res == 0) { + return "UNSET"; + } + + return std::string(buf, get_res); +#else + const char* val = ::getenv(name); + if (val == nullptr) { + return "UNSET"; + } + + return val; +#endif +} + +TEST(ScopedSetEnvTest, SetNonExistingVarToString) { + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "UNSET"); + + { + ScopedSetEnv scoped_set("SCOPED_SET_ENV_TEST_VAR", "value"); + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "value"); + } + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "UNSET"); +} + +TEST(ScopedSetEnvTest, SetNonExistingVarToNull) { + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "UNSET"); + + { + ScopedSetEnv scoped_set("SCOPED_SET_ENV_TEST_VAR", nullptr); + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "UNSET"); + } + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "UNSET"); +} + +TEST(ScopedSetEnvTest, SetExistingVarToString) { + ScopedSetEnv scoped_set("SCOPED_SET_ENV_TEST_VAR", "value"); + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "value"); + + { + ScopedSetEnv scoped_set("SCOPED_SET_ENV_TEST_VAR", "new_value"); + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "new_value"); + } + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "value"); +} + +TEST(ScopedSetEnvTest, SetExistingVarToNull) { + ScopedSetEnv scoped_set("SCOPED_SET_ENV_TEST_VAR", "value"); + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "value"); + + { + ScopedSetEnv scoped_set("SCOPED_SET_ENV_TEST_VAR", nullptr); + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "UNSET"); + } + + EXPECT_EQ(GetEnvVar("SCOPED_SET_ENV_TEST_VAR"), "value"); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock.cc b/third_party/abseil_cpp/absl/base/internal/spinlock.cc new file mode 100644 index 000000000000..a7d44f3eb091 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock.cc @@ -0,0 +1,220 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/spinlock.h" + +#include <algorithm> +#include <atomic> +#include <limits> + +#include "absl/base/attributes.h" +#include "absl/base/internal/atomic_hook.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/spinlock_wait.h" +#include "absl/base/internal/sysinfo.h" /* For NumCPUs() */ +#include "absl/base/call_once.h" + +// Description of lock-word: +// 31..00: [............................3][2][1][0] +// +// [0]: kSpinLockHeld +// [1]: kSpinLockCooperative +// [2]: kSpinLockDisabledScheduling +// [31..3]: ONLY kSpinLockSleeper OR +// Wait time in cycles >> PROFILE_TIMESTAMP_SHIFT +// +// Detailed descriptions: +// +// Bit [0]: The lock is considered held iff kSpinLockHeld is set. +// +// Bit [1]: Eligible waiters (e.g. Fibers) may co-operatively reschedule when +// contended iff kSpinLockCooperative is set. +// +// Bit [2]: This bit is exclusive from bit [1]. It is used only by a +// non-cooperative lock. When set, indicates that scheduling was +// successfully disabled when the lock was acquired. May be unset, +// even if non-cooperative, if a ThreadIdentity did not yet exist at +// time of acquisition. +// +// Bit [3]: If this is the only upper bit ([31..3]) set then this lock was +// acquired without contention, however, at least one waiter exists. +// +// Otherwise, bits [31..3] represent the time spent by the current lock +// holder to acquire the lock. There may be outstanding waiter(s). + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES static base_internal::AtomicHook<void (*)( + const void *lock, int64_t wait_cycles)> + submit_profile_data; + +void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock, + int64_t wait_cycles)) { + submit_profile_data.Store(fn); +} + +// Static member variable definitions. +constexpr uint32_t SpinLock::kSpinLockHeld; +constexpr uint32_t SpinLock::kSpinLockCooperative; +constexpr uint32_t SpinLock::kSpinLockDisabledScheduling; +constexpr uint32_t SpinLock::kSpinLockSleeper; +constexpr uint32_t SpinLock::kWaitTimeMask; + +// Uncommon constructors. +SpinLock::SpinLock(base_internal::SchedulingMode mode) + : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) { + ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static); +} + +// Monitor the lock to see if its value changes within some time period +// (adaptive_spin_count loop iterations). The last value read from the lock +// is returned from the method. +uint32_t SpinLock::SpinLoop() { + // We are already in the slow path of SpinLock, initialize the + // adaptive_spin_count here. + ABSL_CONST_INIT static absl::once_flag init_adaptive_spin_count; + ABSL_CONST_INIT static int adaptive_spin_count = 0; + base_internal::LowLevelCallOnce(&init_adaptive_spin_count, []() { + adaptive_spin_count = base_internal::NumCPUs() > 1 ? 1000 : 1; + }); + + int c = adaptive_spin_count; + uint32_t lock_value; + do { + lock_value = lockword_.load(std::memory_order_relaxed); + } while ((lock_value & kSpinLockHeld) != 0 && --c > 0); + return lock_value; +} + +void SpinLock::SlowLock() { + uint32_t lock_value = SpinLoop(); + lock_value = TryLockInternal(lock_value, 0); + if ((lock_value & kSpinLockHeld) == 0) { + return; + } + + base_internal::SchedulingMode scheduling_mode; + if ((lock_value & kSpinLockCooperative) != 0) { + scheduling_mode = base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL; + } else { + scheduling_mode = base_internal::SCHEDULE_KERNEL_ONLY; + } + + // The lock was not obtained initially, so this thread needs to wait for + // it. Record the current timestamp in the local variable wait_start_time + // so the total wait time can be stored in the lockword once this thread + // obtains the lock. + int64_t wait_start_time = CycleClock::Now(); + uint32_t wait_cycles = 0; + int lock_wait_call_count = 0; + while ((lock_value & kSpinLockHeld) != 0) { + // If the lock is currently held, but not marked as having a sleeper, mark + // it as having a sleeper. + if ((lock_value & kWaitTimeMask) == 0) { + // Here, just "mark" that the thread is going to sleep. Don't store the + // lock wait time in the lock as that will cause the current lock + // owner to think it experienced contention. + if (lockword_.compare_exchange_strong( + lock_value, lock_value | kSpinLockSleeper, + std::memory_order_relaxed, std::memory_order_relaxed)) { + // Successfully transitioned to kSpinLockSleeper. Pass + // kSpinLockSleeper to the SpinLockWait routine to properly indicate + // the last lock_value observed. + lock_value |= kSpinLockSleeper; + } else if ((lock_value & kSpinLockHeld) == 0) { + // Lock is free again, so try and acquire it before sleeping. The + // new lock state will be the number of cycles this thread waited if + // this thread obtains the lock. + lock_value = TryLockInternal(lock_value, wait_cycles); + continue; // Skip the delay at the end of the loop. + } + } + + // SpinLockDelay() calls into fiber scheduler, we need to see + // synchronization there to avoid false positives. + ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0); + // Wait for an OS specific delay. + base_internal::SpinLockDelay(&lockword_, lock_value, ++lock_wait_call_count, + scheduling_mode); + ABSL_TSAN_MUTEX_POST_DIVERT(this, 0); + // Spin again after returning from the wait routine to give this thread + // some chance of obtaining the lock. + lock_value = SpinLoop(); + wait_cycles = EncodeWaitCycles(wait_start_time, CycleClock::Now()); + lock_value = TryLockInternal(lock_value, wait_cycles); + } +} + +void SpinLock::SlowUnlock(uint32_t lock_value) { + base_internal::SpinLockWake(&lockword_, + false); // wake waiter if necessary + + // If our acquisition was contended, collect contentionz profile info. We + // reserve a unitary wait time to represent that a waiter exists without our + // own acquisition having been contended. + if ((lock_value & kWaitTimeMask) != kSpinLockSleeper) { + const uint64_t wait_cycles = DecodeWaitCycles(lock_value); + ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0); + submit_profile_data(this, wait_cycles); + ABSL_TSAN_MUTEX_POST_DIVERT(this, 0); + } +} + +// We use the upper 29 bits of the lock word to store the time spent waiting to +// acquire this lock. This is reported by contentionz profiling. Since the +// lower bits of the cycle counter wrap very quickly on high-frequency +// processors we divide to reduce the granularity to 2^kProfileTimestampShift +// sized units. On a 4Ghz machine this will lose track of wait times greater +// than (2^29/4 Ghz)*128 =~ 17.2 seconds. Such waits should be extremely rare. +static constexpr int kProfileTimestampShift = 7; + +// We currently reserve the lower 3 bits. +static constexpr int kLockwordReservedShift = 3; + +uint32_t SpinLock::EncodeWaitCycles(int64_t wait_start_time, + int64_t wait_end_time) { + static const int64_t kMaxWaitTime = + std::numeric_limits<uint32_t>::max() >> kLockwordReservedShift; + int64_t scaled_wait_time = + (wait_end_time - wait_start_time) >> kProfileTimestampShift; + + // Return a representation of the time spent waiting that can be stored in + // the lock word's upper bits. + uint32_t clamped = static_cast<uint32_t>( + std::min(scaled_wait_time, kMaxWaitTime) << kLockwordReservedShift); + + if (clamped == 0) { + return kSpinLockSleeper; // Just wake waiters, but don't record contention. + } + // Bump up value if necessary to avoid returning kSpinLockSleeper. + const uint32_t kMinWaitTime = + kSpinLockSleeper + (1 << kLockwordReservedShift); + if (clamped == kSpinLockSleeper) { + return kMinWaitTime; + } + return clamped; +} + +uint64_t SpinLock::DecodeWaitCycles(uint32_t lock_value) { + // Cast to uint32_t first to ensure bits [63:32] are cleared. + const uint64_t scaled_wait_time = + static_cast<uint32_t>(lock_value & kWaitTimeMask); + return scaled_wait_time << (kProfileTimestampShift - kLockwordReservedShift); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock.h b/third_party/abseil_cpp/absl/base/internal/spinlock.h new file mode 100644 index 000000000000..e6ac9e6400b2 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock.h @@ -0,0 +1,237 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Most users requiring mutual exclusion should use Mutex. +// SpinLock is provided for use in three situations: +// - for use in code that Mutex itself depends on +// - to get a faster fast-path release under low contention (without an +// atomic read-modify-write) In return, SpinLock has worse behaviour under +// contention, which is why Mutex is preferred in most situations. +// - for async signal safety (see below) + +// SpinLock is async signal safe. If a spinlock is used within a signal +// handler, all code that acquires the lock must ensure that the signal cannot +// arrive while they are holding the lock. Typically, this is done by blocking +// the signal. + +#ifndef ABSL_BASE_INTERNAL_SPINLOCK_H_ +#define ABSL_BASE_INTERNAL_SPINLOCK_H_ + +#include <stdint.h> +#include <sys/types.h> + +#include <atomic> + +#include "absl/base/attributes.h" +#include "absl/base/const_init.h" +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/low_level_scheduling.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/tsan_mutex_interface.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/base/thread_annotations.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +class ABSL_LOCKABLE SpinLock { + public: + SpinLock() : lockword_(kSpinLockCooperative) { + ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static); + } + + // Constructors that allow non-cooperative spinlocks to be created for use + // inside thread schedulers. Normal clients should not use these. + explicit SpinLock(base_internal::SchedulingMode mode); + + // Constructor for global SpinLock instances. See absl/base/const_init.h. + constexpr SpinLock(absl::ConstInitType, base_internal::SchedulingMode mode) + : lockword_(IsCooperative(mode) ? kSpinLockCooperative : 0) {} + + // For global SpinLock instances prefer trivial destructor when possible. + // Default but non-trivial destructor in some build configurations causes an + // extra static initializer. +#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE + ~SpinLock() { ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); } +#else + ~SpinLock() = default; +#endif + + // Acquire this SpinLock. + inline void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { + ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); + if (!TryLockImpl()) { + SlowLock(); + } + ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); + } + + // Try to acquire this SpinLock without blocking and return true if the + // acquisition was successful. If the lock was not acquired, false is + // returned. If this SpinLock is free at the time of the call, TryLock + // will return true with high probability. + inline bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); + bool res = TryLockImpl(); + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_try_lock | (res ? 0 : __tsan_mutex_try_lock_failed), + 0); + return res; + } + + // Release this SpinLock, which must be held by the calling thread. + inline void Unlock() ABSL_UNLOCK_FUNCTION() { + ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0); + uint32_t lock_value = lockword_.load(std::memory_order_relaxed); + lock_value = lockword_.exchange(lock_value & kSpinLockCooperative, + std::memory_order_release); + + if ((lock_value & kSpinLockDisabledScheduling) != 0) { + base_internal::SchedulingGuard::EnableRescheduling(true); + } + if ((lock_value & kWaitTimeMask) != 0) { + // Collect contentionz profile info, and speed the wakeup of any waiter. + // The wait_cycles value indicates how long this thread spent waiting + // for the lock. + SlowUnlock(lock_value); + } + ABSL_TSAN_MUTEX_POST_UNLOCK(this, 0); + } + + // Determine if the lock is held. When the lock is held by the invoking + // thread, true will always be returned. Intended to be used as + // CHECK(lock.IsHeld()). + inline bool IsHeld() const { + return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0; + } + + protected: + // These should not be exported except for testing. + + // Store number of cycles between wait_start_time and wait_end_time in a + // lock value. + static uint32_t EncodeWaitCycles(int64_t wait_start_time, + int64_t wait_end_time); + + // Extract number of wait cycles in a lock value. + static uint64_t DecodeWaitCycles(uint32_t lock_value); + + // Provide access to protected method above. Use for testing only. + friend struct SpinLockTest; + + private: + // lockword_ is used to store the following: + // + // bit[0] encodes whether a lock is being held. + // bit[1] encodes whether a lock uses cooperative scheduling. + // bit[2] encodes whether a lock disables scheduling. + // bit[3:31] encodes time a lock spent on waiting as a 29-bit unsigned int. + static constexpr uint32_t kSpinLockHeld = 1; + static constexpr uint32_t kSpinLockCooperative = 2; + static constexpr uint32_t kSpinLockDisabledScheduling = 4; + static constexpr uint32_t kSpinLockSleeper = 8; + // Includes kSpinLockSleeper. + static constexpr uint32_t kWaitTimeMask = + ~(kSpinLockHeld | kSpinLockCooperative | kSpinLockDisabledScheduling); + + // Returns true if the provided scheduling mode is cooperative. + static constexpr bool IsCooperative( + base_internal::SchedulingMode scheduling_mode) { + return scheduling_mode == base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL; + } + + uint32_t TryLockInternal(uint32_t lock_value, uint32_t wait_cycles); + void SlowLock() ABSL_ATTRIBUTE_COLD; + void SlowUnlock(uint32_t lock_value) ABSL_ATTRIBUTE_COLD; + uint32_t SpinLoop(); + + inline bool TryLockImpl() { + uint32_t lock_value = lockword_.load(std::memory_order_relaxed); + return (TryLockInternal(lock_value, 0) & kSpinLockHeld) == 0; + } + + std::atomic<uint32_t> lockword_; + + SpinLock(const SpinLock&) = delete; + SpinLock& operator=(const SpinLock&) = delete; +}; + +// Corresponding locker object that arranges to acquire a spinlock for +// the duration of a C++ scope. +class ABSL_SCOPED_LOCKABLE SpinLockHolder { + public: + inline explicit SpinLockHolder(SpinLock* l) ABSL_EXCLUSIVE_LOCK_FUNCTION(l) + : lock_(l) { + l->Lock(); + } + inline ~SpinLockHolder() ABSL_UNLOCK_FUNCTION() { lock_->Unlock(); } + + SpinLockHolder(const SpinLockHolder&) = delete; + SpinLockHolder& operator=(const SpinLockHolder&) = delete; + + private: + SpinLock* lock_; +}; + +// Register a hook for profiling support. +// +// The function pointer registered here will be called whenever a spinlock is +// contended. The callback is given an opaque handle to the contended spinlock +// and the number of wait cycles. This is thread-safe, but only a single +// profiler can be registered. It is an error to call this function multiple +// times with different arguments. +void RegisterSpinLockProfiler(void (*fn)(const void* lock, + int64_t wait_cycles)); + +//------------------------------------------------------------------------------ +// Public interface ends here. +//------------------------------------------------------------------------------ + +// If (result & kSpinLockHeld) == 0, then *this was successfully locked. +// Otherwise, returns last observed value for lockword_. +inline uint32_t SpinLock::TryLockInternal(uint32_t lock_value, + uint32_t wait_cycles) { + if ((lock_value & kSpinLockHeld) != 0) { + return lock_value; + } + + uint32_t sched_disabled_bit = 0; + if ((lock_value & kSpinLockCooperative) == 0) { + // For non-cooperative locks we must make sure we mark ourselves as + // non-reschedulable before we attempt to CompareAndSwap. + if (base_internal::SchedulingGuard::DisableRescheduling()) { + sched_disabled_bit = kSpinLockDisabledScheduling; + } + } + + if (!lockword_.compare_exchange_strong( + lock_value, + kSpinLockHeld | lock_value | wait_cycles | sched_disabled_bit, + std::memory_order_acquire, std::memory_order_relaxed)) { + base_internal::SchedulingGuard::EnableRescheduling(sched_disabled_bit != 0); + } + + return lock_value; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_SPINLOCK_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_akaros.inc b/third_party/abseil_cpp/absl/base/internal/spinlock_akaros.inc new file mode 100644 index 000000000000..bc468940fc5d --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_akaros.inc @@ -0,0 +1,35 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file is an Akaros-specific part of spinlock_wait.cc + +#include <atomic> + +#include "absl/base/internal/scheduling_mode.h" + +extern "C" { + +ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( + std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, + int /* loop */, absl::base_internal::SchedulingMode /* mode */) { + // In Akaros, one must take care not to call anything that could cause a + // malloc(), a blocking system call, or a uthread_yield() while holding a + // spinlock. Our callers assume will not call into libraries or other + // arbitrary code. +} + +ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake( + std::atomic<uint32_t>* /* lock_word */, bool /* all */) {} + +} // extern "C" diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_benchmark.cc b/third_party/abseil_cpp/absl/base/internal/spinlock_benchmark.cc new file mode 100644 index 000000000000..0451c65f9559 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_benchmark.cc @@ -0,0 +1,52 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// See also //absl/synchronization:mutex_benchmark for a comparison of SpinLock +// and Mutex performance under varying levels of contention. + +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/spinlock.h" +#include "absl/synchronization/internal/create_thread_identity.h" +#include "benchmark/benchmark.h" + +namespace { + +template <absl::base_internal::SchedulingMode scheduling_mode> +static void BM_SpinLock(benchmark::State& state) { + // Ensure a ThreadIdentity is installed. + ABSL_INTERNAL_CHECK( + absl::synchronization_internal::GetOrCreateCurrentThreadIdentity() != + nullptr, + "GetOrCreateCurrentThreadIdentity() failed"); + + static auto* spinlock = new absl::base_internal::SpinLock(scheduling_mode); + for (auto _ : state) { + absl::base_internal::SpinLockHolder holder(spinlock); + } +} + +BENCHMARK_TEMPLATE(BM_SpinLock, + absl::base_internal::SCHEDULE_KERNEL_ONLY) + ->UseRealTime() + ->Threads(1) + ->ThreadPerCpu(); + +BENCHMARK_TEMPLATE(BM_SpinLock, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) + ->UseRealTime() + ->Threads(1) + ->ThreadPerCpu(); + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_linux.inc b/third_party/abseil_cpp/absl/base/internal/spinlock_linux.inc new file mode 100644 index 000000000000..e31c6ed477c0 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_linux.inc @@ -0,0 +1,74 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file is a Linux-specific part of spinlock_wait.cc + +#include <linux/futex.h> +#include <sys/syscall.h> +#include <unistd.h> + +#include <atomic> +#include <climits> +#include <cstdint> +#include <ctime> + +#include "absl/base/attributes.h" +#include "absl/base/internal/errno_saver.h" + +// The SpinLock lockword is `std::atomic<uint32_t>`. Here we assert that +// `std::atomic<uint32_t>` is bitwise equivalent of the `int` expected +// by SYS_futex. We also assume that reads/writes done to the lockword +// by SYS_futex have rational semantics with regard to the +// std::atomic<> API. C++ provides no guarantees of these assumptions, +// but they are believed to hold in practice. +static_assert(sizeof(std::atomic<uint32_t>) == sizeof(int), + "SpinLock lockword has the wrong size for a futex"); + +// Some Android headers are missing these definitions even though they +// support these futex operations. +#ifdef __BIONIC__ +#ifndef SYS_futex +#define SYS_futex __NR_futex +#endif +#ifndef FUTEX_PRIVATE_FLAG +#define FUTEX_PRIVATE_FLAG 128 +#endif +#endif + +#if defined(__NR_futex_time64) && !defined(SYS_futex_time64) +#define SYS_futex_time64 __NR_futex_time64 +#endif + +#if defined(SYS_futex_time64) && !defined(SYS_futex) +#define SYS_futex SYS_futex_time64 +#endif + +extern "C" { + +ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( + std::atomic<uint32_t> *w, uint32_t value, int loop, + absl::base_internal::SchedulingMode) { + absl::base_internal::ErrnoSaver errno_saver; + struct timespec tm; + tm.tv_sec = 0; + tm.tv_nsec = absl::base_internal::SpinLockSuggestedDelayNS(loop); + syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, &tm); +} + +ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake(std::atomic<uint32_t> *w, + bool all) { + syscall(SYS_futex, w, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, all ? INT_MAX : 1, 0); +} + +} // extern "C" diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_posix.inc b/third_party/abseil_cpp/absl/base/internal/spinlock_posix.inc new file mode 100644 index 000000000000..fcd21b151b6e --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_posix.inc @@ -0,0 +1,46 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file is a Posix-specific part of spinlock_wait.cc + +#include <sched.h> + +#include <atomic> +#include <ctime> + +#include "absl/base/internal/errno_saver.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/port.h" + +extern "C" { + +ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockDelay( + std::atomic<uint32_t>* /* lock_word */, uint32_t /* value */, int loop, + absl::base_internal::SchedulingMode /* mode */) { + absl::base_internal::ErrnoSaver errno_saver; + if (loop == 0) { + } else if (loop == 1) { + sched_yield(); + } else { + struct timespec tm; + tm.tv_sec = 0; + tm.tv_nsec = absl::base_internal::SpinLockSuggestedDelayNS(loop); + nanosleep(&tm, nullptr); + } +} + +ABSL_ATTRIBUTE_WEAK void AbslInternalSpinLockWake( + std::atomic<uint32_t>* /* lock_word */, bool /* all */) {} + +} // extern "C" diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_wait.cc b/third_party/abseil_cpp/absl/base/internal/spinlock_wait.cc new file mode 100644 index 000000000000..fa824be1c008 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_wait.cc @@ -0,0 +1,81 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The OS-specific header included below must provide two calls: +// AbslInternalSpinLockDelay() and AbslInternalSpinLockWake(). +// See spinlock_wait.h for the specs. + +#include <atomic> +#include <cstdint> + +#include "absl/base/internal/spinlock_wait.h" + +#if defined(_WIN32) +#include "absl/base/internal/spinlock_win32.inc" +#elif defined(__linux__) +#include "absl/base/internal/spinlock_linux.inc" +#elif defined(__akaros__) +#include "absl/base/internal/spinlock_akaros.inc" +#else +#include "absl/base/internal/spinlock_posix.inc" +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// See spinlock_wait.h for spec. +uint32_t SpinLockWait(std::atomic<uint32_t> *w, int n, + const SpinLockWaitTransition trans[], + base_internal::SchedulingMode scheduling_mode) { + int loop = 0; + for (;;) { + uint32_t v = w->load(std::memory_order_acquire); + int i; + for (i = 0; i != n && v != trans[i].from; i++) { + } + if (i == n) { + SpinLockDelay(w, v, ++loop, scheduling_mode); // no matching transition + } else if (trans[i].to == v || // null transition + w->compare_exchange_strong(v, trans[i].to, + std::memory_order_acquire, + std::memory_order_relaxed)) { + if (trans[i].done) return v; + } + } +} + +static std::atomic<uint64_t> delay_rand; + +// Return a suggested delay in nanoseconds for iteration number "loop" +int SpinLockSuggestedDelayNS(int loop) { + // Weak pseudo-random number generator to get some spread between threads + // when many are spinning. + uint64_t r = delay_rand.load(std::memory_order_relaxed); + r = 0x5deece66dLL * r + 0xb; // numbers from nrand48() + delay_rand.store(r, std::memory_order_relaxed); + + if (loop < 0 || loop > 32) { // limit loop to 0..32 + loop = 32; + } + const int kMinDelay = 128 << 10; // 128us + // Double delay every 8 iterations, up to 16x (2ms). + int delay = kMinDelay << (loop / 8); + // Randomize in delay..2*delay range, for resulting 128us..4ms range. + return delay | ((delay - 1) & static_cast<int>(r)); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_wait.h b/third_party/abseil_cpp/absl/base/internal/spinlock_wait.h new file mode 100644 index 000000000000..169bc749fbc5 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_wait.h @@ -0,0 +1,93 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_SPINLOCK_WAIT_H_ +#define ABSL_BASE_INTERNAL_SPINLOCK_WAIT_H_ + +// Operations to make atomic transitions on a word, and to allow +// waiting for those transitions to become possible. + +#include <stdint.h> +#include <atomic> + +#include "absl/base/internal/scheduling_mode.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// SpinLockWait() waits until it can perform one of several transitions from +// "from" to "to". It returns when it performs a transition where done==true. +struct SpinLockWaitTransition { + uint32_t from; + uint32_t to; + bool done; +}; + +// Wait until *w can transition from trans[i].from to trans[i].to for some i +// satisfying 0<=i<n && trans[i].done, atomically make the transition, +// then return the old value of *w. Make any other atomic transitions +// where !trans[i].done, but continue waiting. +uint32_t SpinLockWait(std::atomic<uint32_t> *w, int n, + const SpinLockWaitTransition trans[], + SchedulingMode scheduling_mode); + +// If possible, wake some thread that has called SpinLockDelay(w, ...). If +// "all" is true, wake all such threads. This call is a hint, and on some +// systems it may be a no-op; threads calling SpinLockDelay() will always wake +// eventually even if SpinLockWake() is never called. +void SpinLockWake(std::atomic<uint32_t> *w, bool all); + +// Wait for an appropriate spin delay on iteration "loop" of a +// spin loop on location *w, whose previously observed value was "value". +// SpinLockDelay() may do nothing, may yield the CPU, may sleep a clock tick, +// or may wait for a delay that can be truncated by a call to SpinLockWake(w). +// In all cases, it must return in bounded time even if SpinLockWake() is not +// called. +void SpinLockDelay(std::atomic<uint32_t> *w, uint32_t value, int loop, + base_internal::SchedulingMode scheduling_mode); + +// Helper used by AbslInternalSpinLockDelay. +// Returns a suggested delay in nanoseconds for iteration number "loop". +int SpinLockSuggestedDelayNS(int loop); + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +// In some build configurations we pass --detect-odr-violations to the +// gold linker. This causes it to flag weak symbol overrides as ODR +// violations. Because ODR only applies to C++ and not C, +// --detect-odr-violations ignores symbols not mangled with C++ names. +// By changing our extension points to be extern "C", we dodge this +// check. +extern "C" { +void AbslInternalSpinLockWake(std::atomic<uint32_t> *w, bool all); +void AbslInternalSpinLockDelay( + std::atomic<uint32_t> *w, uint32_t value, int loop, + absl::base_internal::SchedulingMode scheduling_mode); +} + +inline void absl::base_internal::SpinLockWake(std::atomic<uint32_t> *w, + bool all) { + AbslInternalSpinLockWake(w, all); +} + +inline void absl::base_internal::SpinLockDelay( + std::atomic<uint32_t> *w, uint32_t value, int loop, + absl::base_internal::SchedulingMode scheduling_mode) { + AbslInternalSpinLockDelay(w, value, loop, scheduling_mode); +} + +#endif // ABSL_BASE_INTERNAL_SPINLOCK_WAIT_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/spinlock_win32.inc b/third_party/abseil_cpp/absl/base/internal/spinlock_win32.inc new file mode 100644 index 000000000000..78654b5b5966 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/spinlock_win32.inc @@ -0,0 +1,37 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file is a Win32-specific part of spinlock_wait.cc + +#include <windows.h> +#include <atomic> +#include "absl/base/internal/scheduling_mode.h" + +extern "C" { + +void AbslInternalSpinLockDelay(std::atomic<uint32_t>* /* lock_word */, + uint32_t /* value */, int loop, + absl::base_internal::SchedulingMode /* mode */) { + if (loop == 0) { + } else if (loop == 1) { + Sleep(0); + } else { + Sleep(absl::base_internal::SpinLockSuggestedDelayNS(loop) / 1000000); + } +} + +void AbslInternalSpinLockWake(std::atomic<uint32_t>* /* lock_word */, + bool /* all */) {} + +} // extern "C" diff --git a/third_party/abseil_cpp/absl/base/internal/strerror.cc b/third_party/abseil_cpp/absl/base/internal/strerror.cc new file mode 100644 index 000000000000..d66ba1201135 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/strerror.cc @@ -0,0 +1,88 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/strerror.h" + +#include <array> +#include <cerrno> +#include <cstddef> +#include <cstdio> +#include <cstring> +#include <string> +#include <type_traits> + +#include "absl/base/internal/errno_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +const char* StrErrorAdaptor(int errnum, char* buf, size_t buflen) { +#if defined(_WIN32) + int rc = strerror_s(buf, buflen, errnum); + buf[buflen - 1] = '\0'; // guarantee NUL termination + if (rc == 0 && strncmp(buf, "Unknown error", buflen) == 0) *buf = '\0'; + return buf; +#else + // The type of `ret` is platform-specific; both of these branches must compile + // either way but only one will execute on any given platform: + auto ret = strerror_r(errnum, buf, buflen); + if (std::is_same<decltype(ret), int>::value) { + // XSI `strerror_r`; `ret` is `int`: + if (ret) *buf = '\0'; + return buf; + } else { + // GNU `strerror_r`; `ret` is `char *`: + return reinterpret_cast<const char*>(ret); + } +#endif +} + +std::string StrErrorInternal(int errnum) { + absl::base_internal::ErrnoSaver errno_saver; + char buf[100]; + const char* str = StrErrorAdaptor(errnum, buf, sizeof buf); + if (*str == '\0') { + snprintf(buf, sizeof buf, "Unknown error %d", errnum); + str = buf; + } + return str; +} + +// kSysNerr is the number of errors from a recent glibc. `StrError()` falls back +// to `StrErrorAdaptor()` if the value is larger than this. +constexpr int kSysNerr = 135; + +std::array<std::string, kSysNerr>* NewStrErrorTable() { + auto* table = new std::array<std::string, kSysNerr>; + for (int i = 0; i < static_cast<int>(table->size()); ++i) { + (*table)[i] = StrErrorInternal(i); + } + return table; +} + +} // namespace + +std::string StrError(int errnum) { + static const auto* table = NewStrErrorTable(); + if (errnum >= 0 && errnum < static_cast<int>(table->size())) { + return (*table)[errnum]; + } + return StrErrorInternal(errnum); +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/strerror.h b/third_party/abseil_cpp/absl/base/internal/strerror.h new file mode 100644 index 000000000000..350097366eed --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/strerror.h @@ -0,0 +1,39 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_STRERROR_H_ +#define ABSL_BASE_INTERNAL_STRERROR_H_ + +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// A portable and thread-safe alternative to C89's `strerror`. +// +// The C89 specification of `strerror` is not suitable for use in a +// multi-threaded application as the returned string may be changed by calls to +// `strerror` from another thread. The many non-stdlib alternatives differ +// enough in their names, availability, and semantics to justify this wrapper +// around them. `errno` will not be modified by a call to `absl::StrError`. +std::string StrError(int errnum); + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_STRERROR_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/strerror_benchmark.cc b/third_party/abseil_cpp/absl/base/internal/strerror_benchmark.cc new file mode 100644 index 000000000000..c9ab14a89dbb --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/strerror_benchmark.cc @@ -0,0 +1,29 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cerrno> +#include <cstdio> +#include <string> + +#include "absl/base/internal/strerror.h" +#include "benchmark/benchmark.h" + +namespace { +void BM_AbslStrError(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(absl::base_internal::StrError(ERANGE)); + } +} +BENCHMARK(BM_AbslStrError); +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/strerror_test.cc b/third_party/abseil_cpp/absl/base/internal/strerror_test.cc new file mode 100644 index 000000000000..a53da97f92c9 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/strerror_test.cc @@ -0,0 +1,86 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/strerror.h" + +#include <atomic> +#include <cerrno> +#include <cstdio> +#include <cstring> +#include <string> +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/match.h" + +namespace { +using ::testing::AnyOf; +using ::testing::Eq; + +TEST(StrErrorTest, ValidErrorCode) { + errno = ERANGE; + EXPECT_THAT(absl::base_internal::StrError(EDOM), Eq(strerror(EDOM))); + EXPECT_THAT(errno, Eq(ERANGE)); +} + +TEST(StrErrorTest, InvalidErrorCode) { + errno = ERANGE; + EXPECT_THAT(absl::base_internal::StrError(-1), + AnyOf(Eq("No error information"), Eq("Unknown error -1"))); + EXPECT_THAT(errno, Eq(ERANGE)); +} + +TEST(StrErrorTest, MultipleThreads) { + // In this test, we will start up 2 threads and have each one call + // StrError 1000 times, each time with a different errnum. We + // expect that StrError(errnum) will return a string equal to the + // one returned by strerror(errnum), if the code is known. Since + // strerror is known to be thread-hostile, collect all the expected + // strings up front. + const int kNumCodes = 1000; + std::vector<std::string> expected_strings(kNumCodes); + for (int i = 0; i < kNumCodes; ++i) { + expected_strings[i] = strerror(i); + } + + std::atomic_int counter(0); + auto thread_fun = [&]() { + for (int i = 0; i < kNumCodes; ++i) { + ++counter; + errno = ERANGE; + const std::string value = absl::base_internal::StrError(i); + // Only the GNU implementation is guaranteed to provide the + // string "Unknown error nnn". POSIX doesn't say anything. + if (!absl::StartsWith(value, "Unknown error ")) { + EXPECT_THAT(absl::base_internal::StrError(i), Eq(expected_strings[i])); + } + EXPECT_THAT(errno, Eq(ERANGE)); + } + }; + + const int kNumThreads = 100; + std::vector<std::thread> threads; + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread(thread_fun)); + } + for (auto& thread : threads) { + thread.join(); + } + + EXPECT_THAT(counter, Eq(kNumThreads * kNumCodes)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/sysinfo.cc b/third_party/abseil_cpp/absl/base/internal/sysinfo.cc new file mode 100644 index 000000000000..4a3b205034a7 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/sysinfo.cc @@ -0,0 +1,439 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/sysinfo.h" + +#include "absl/base/attributes.h" + +#ifdef _WIN32 +#include <windows.h> +#else +#include <fcntl.h> +#include <pthread.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> +#endif + +#ifdef __linux__ +#include <sys/syscall.h> +#endif + +#if defined(__APPLE__) || defined(__FreeBSD__) +#include <sys/sysctl.h> +#endif + +#if defined(__myriad2__) +#include <rtems.h> +#endif + +#include <string.h> + +#include <cassert> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <ctime> +#include <limits> +#include <thread> // NOLINT(build/c++11) +#include <utility> +#include <vector> + +#include "absl/base/call_once.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/internal/unscaledcycleclock.h" +#include "absl/base/thread_annotations.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +static int GetNumCPUs() { +#if defined(__myriad2__) + return 1; +#else + // Other possibilities: + // - Read /sys/devices/system/cpu/online and use cpumask_parse() + // - sysconf(_SC_NPROCESSORS_ONLN) + return std::thread::hardware_concurrency(); +#endif +} + +#if defined(_WIN32) + +static double GetNominalCPUFrequency() { +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) && \ + !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + // UWP apps don't have access to the registry and currently don't provide an + // API informing about CPU nominal frequency. + return 1.0; +#else +#pragma comment(lib, "advapi32.lib") // For Reg* functions. + HKEY key; + // Use the Reg* functions rather than the SH functions because shlwapi.dll + // pulls in gdi32.dll which makes process destruction much more costly. + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, + "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", 0, + KEY_READ, &key) == ERROR_SUCCESS) { + DWORD type = 0; + DWORD data = 0; + DWORD data_size = sizeof(data); + auto result = RegQueryValueExA(key, "~MHz", 0, &type, + reinterpret_cast<LPBYTE>(&data), &data_size); + RegCloseKey(key); + if (result == ERROR_SUCCESS && type == REG_DWORD && + data_size == sizeof(data)) { + return data * 1e6; // Value is MHz. + } + } + return 1.0; +#endif // WINAPI_PARTITION_APP && !WINAPI_PARTITION_DESKTOP +} + +#elif defined(CTL_HW) && defined(HW_CPU_FREQ) + +static double GetNominalCPUFrequency() { + unsigned freq; + size_t size = sizeof(freq); + int mib[2] = {CTL_HW, HW_CPU_FREQ}; + if (sysctl(mib, 2, &freq, &size, nullptr, 0) == 0) { + return static_cast<double>(freq); + } + return 1.0; +} + +#else + +// Helper function for reading a long from a file. Returns true if successful +// and the memory location pointed to by value is set to the value read. +static bool ReadLongFromFile(const char *file, long *value) { + bool ret = false; + int fd = open(file, O_RDONLY); + if (fd != -1) { + char line[1024]; + char *err; + memset(line, '\0', sizeof(line)); + int len = read(fd, line, sizeof(line) - 1); + if (len <= 0) { + ret = false; + } else { + const long temp_value = strtol(line, &err, 10); + if (line[0] != '\0' && (*err == '\n' || *err == '\0')) { + *value = temp_value; + ret = true; + } + } + close(fd); + } + return ret; +} + +#if defined(ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY) + +// Reads a monotonic time source and returns a value in +// nanoseconds. The returned value uses an arbitrary epoch, not the +// Unix epoch. +static int64_t ReadMonotonicClockNanos() { + struct timespec t; +#ifdef CLOCK_MONOTONIC_RAW + int rc = clock_gettime(CLOCK_MONOTONIC_RAW, &t); +#else + int rc = clock_gettime(CLOCK_MONOTONIC, &t); +#endif + if (rc != 0) { + perror("clock_gettime() failed"); + abort(); + } + return int64_t{t.tv_sec} * 1000000000 + t.tv_nsec; +} + +class UnscaledCycleClockWrapperForInitializeFrequency { + public: + static int64_t Now() { return base_internal::UnscaledCycleClock::Now(); } +}; + +struct TimeTscPair { + int64_t time; // From ReadMonotonicClockNanos(). + int64_t tsc; // From UnscaledCycleClock::Now(). +}; + +// Returns a pair of values (monotonic kernel time, TSC ticks) that +// approximately correspond to each other. This is accomplished by +// doing several reads and picking the reading with the lowest +// latency. This approach is used to minimize the probability that +// our thread was preempted between clock reads. +static TimeTscPair GetTimeTscPair() { + int64_t best_latency = std::numeric_limits<int64_t>::max(); + TimeTscPair best; + for (int i = 0; i < 10; ++i) { + int64_t t0 = ReadMonotonicClockNanos(); + int64_t tsc = UnscaledCycleClockWrapperForInitializeFrequency::Now(); + int64_t t1 = ReadMonotonicClockNanos(); + int64_t latency = t1 - t0; + if (latency < best_latency) { + best_latency = latency; + best.time = t0; + best.tsc = tsc; + } + } + return best; +} + +// Measures and returns the TSC frequency by taking a pair of +// measurements approximately `sleep_nanoseconds` apart. +static double MeasureTscFrequencyWithSleep(int sleep_nanoseconds) { + auto t0 = GetTimeTscPair(); + struct timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = sleep_nanoseconds; + while (nanosleep(&ts, &ts) != 0 && errno == EINTR) {} + auto t1 = GetTimeTscPair(); + double elapsed_ticks = t1.tsc - t0.tsc; + double elapsed_time = (t1.time - t0.time) * 1e-9; + return elapsed_ticks / elapsed_time; +} + +// Measures and returns the TSC frequency by calling +// MeasureTscFrequencyWithSleep(), doubling the sleep interval until the +// frequency measurement stabilizes. +static double MeasureTscFrequency() { + double last_measurement = -1.0; + int sleep_nanoseconds = 1000000; // 1 millisecond. + for (int i = 0; i < 8; ++i) { + double measurement = MeasureTscFrequencyWithSleep(sleep_nanoseconds); + if (measurement * 0.99 < last_measurement && + last_measurement < measurement * 1.01) { + // Use the current measurement if it is within 1% of the + // previous measurement. + return measurement; + } + last_measurement = measurement; + sleep_nanoseconds *= 2; + } + return last_measurement; +} + +#endif // ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY + +static double GetNominalCPUFrequency() { + long freq = 0; + + // Google's production kernel has a patch to export the TSC + // frequency through sysfs. If the kernel is exporting the TSC + // frequency use that. There are issues where cpuinfo_max_freq + // cannot be relied on because the BIOS may be exporting an invalid + // p-state (on x86) or p-states may be used to put the processor in + // a new mode (turbo mode). Essentially, those frequencies cannot + // always be relied upon. The same reasons apply to /proc/cpuinfo as + // well. + if (ReadLongFromFile("/sys/devices/system/cpu/cpu0/tsc_freq_khz", &freq)) { + return freq * 1e3; // Value is kHz. + } + +#if defined(ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY) + // On these platforms, the TSC frequency is the nominal CPU + // frequency. But without having the kernel export it directly + // though /sys/devices/system/cpu/cpu0/tsc_freq_khz, there is no + // other way to reliably get the TSC frequency, so we have to + // measure it ourselves. Some CPUs abuse cpuinfo_max_freq by + // exporting "fake" frequencies for implementing new features. For + // example, Intel's turbo mode is enabled by exposing a p-state + // value with a higher frequency than that of the real TSC + // rate. Because of this, we prefer to measure the TSC rate + // ourselves on i386 and x86-64. + return MeasureTscFrequency(); +#else + + // If CPU scaling is in effect, we want to use the *maximum* + // frequency, not whatever CPU speed some random processor happens + // to be using now. + if (ReadLongFromFile("/sys/devices/system/cpu/cpu0/cpufreq/cpuinfo_max_freq", + &freq)) { + return freq * 1e3; // Value is kHz. + } + + return 1.0; +#endif // !ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY +} + +#endif + +ABSL_CONST_INIT static once_flag init_num_cpus_once; +ABSL_CONST_INIT static int num_cpus = 0; + +// NumCPUs() may be called before main() and before malloc is properly +// initialized, therefore this must not allocate memory. +int NumCPUs() { + base_internal::LowLevelCallOnce( + &init_num_cpus_once, []() { num_cpus = GetNumCPUs(); }); + return num_cpus; +} + +// A default frequency of 0.0 might be dangerous if it is used in division. +ABSL_CONST_INIT static once_flag init_nominal_cpu_frequency_once; +ABSL_CONST_INIT static double nominal_cpu_frequency = 1.0; + +// NominalCPUFrequency() may be called before main() and before malloc is +// properly initialized, therefore this must not allocate memory. +double NominalCPUFrequency() { + base_internal::LowLevelCallOnce( + &init_nominal_cpu_frequency_once, + []() { nominal_cpu_frequency = GetNominalCPUFrequency(); }); + return nominal_cpu_frequency; +} + +#if defined(_WIN32) + +pid_t GetTID() { + return pid_t{GetCurrentThreadId()}; +} + +#elif defined(__linux__) + +#ifndef SYS_gettid +#define SYS_gettid __NR_gettid +#endif + +pid_t GetTID() { + return syscall(SYS_gettid); +} + +#elif defined(__akaros__) + +pid_t GetTID() { + // Akaros has a concept of "vcore context", which is the state the program + // is forced into when we need to make a user-level scheduling decision, or + // run a signal handler. This is analogous to the interrupt context that a + // CPU might enter if it encounters some kind of exception. + // + // There is no current thread context in vcore context, but we need to give + // a reasonable answer if asked for a thread ID (e.g., in a signal handler). + // Thread 0 always exists, so if we are in vcore context, we return that. + // + // Otherwise, we know (since we are using pthreads) that the uthread struct + // current_uthread is pointing to is the first element of a + // struct pthread_tcb, so we extract and return the thread ID from that. + // + // TODO(dcross): Akaros anticipates moving the thread ID to the uthread + // structure at some point. We should modify this code to remove the cast + // when that happens. + if (in_vcore_context()) + return 0; + return reinterpret_cast<struct pthread_tcb *>(current_uthread)->id; +} + +#elif defined(__myriad2__) + +pid_t GetTID() { + uint32_t tid; + rtems_task_ident(RTEMS_SELF, 0, &tid); + return tid; +} + +#else + +// Fallback implementation of GetTID using pthread_getspecific. +ABSL_CONST_INIT static once_flag tid_once; +ABSL_CONST_INIT static pthread_key_t tid_key; +ABSL_CONST_INIT static absl::base_internal::SpinLock tid_lock( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); + +// We set a bit per thread in this array to indicate that an ID is in +// use. ID 0 is unused because it is the default value returned by +// pthread_getspecific(). +ABSL_CONST_INIT static std::vector<uint32_t> *tid_array + ABSL_GUARDED_BY(tid_lock) = nullptr; +static constexpr int kBitsPerWord = 32; // tid_array is uint32_t. + +// Returns the TID to tid_array. +static void FreeTID(void *v) { + intptr_t tid = reinterpret_cast<intptr_t>(v); + int word = tid / kBitsPerWord; + uint32_t mask = ~(1u << (tid % kBitsPerWord)); + absl::base_internal::SpinLockHolder lock(&tid_lock); + assert(0 <= word && static_cast<size_t>(word) < tid_array->size()); + (*tid_array)[word] &= mask; +} + +static void InitGetTID() { + if (pthread_key_create(&tid_key, FreeTID) != 0) { + // The logging system calls GetTID() so it can't be used here. + perror("pthread_key_create failed"); + abort(); + } + + // Initialize tid_array. + absl::base_internal::SpinLockHolder lock(&tid_lock); + tid_array = new std::vector<uint32_t>(1); + (*tid_array)[0] = 1; // ID 0 is never-allocated. +} + +// Return a per-thread small integer ID from pthread's thread-specific data. +pid_t GetTID() { + absl::call_once(tid_once, InitGetTID); + + intptr_t tid = reinterpret_cast<intptr_t>(pthread_getspecific(tid_key)); + if (tid != 0) { + return tid; + } + + int bit; // tid_array[word] = 1u << bit; + size_t word; + { + // Search for the first unused ID. + absl::base_internal::SpinLockHolder lock(&tid_lock); + // First search for a word in the array that is not all ones. + word = 0; + while (word < tid_array->size() && ~(*tid_array)[word] == 0) { + ++word; + } + if (word == tid_array->size()) { + tid_array->push_back(0); // No space left, add kBitsPerWord more IDs. + } + // Search for a zero bit in the word. + bit = 0; + while (bit < kBitsPerWord && (((*tid_array)[word] >> bit) & 1) != 0) { + ++bit; + } + tid = (word * kBitsPerWord) + bit; + (*tid_array)[word] |= 1u << bit; // Mark the TID as allocated. + } + + if (pthread_setspecific(tid_key, reinterpret_cast<void *>(tid)) != 0) { + perror("pthread_setspecific failed"); + abort(); + } + + return static_cast<pid_t>(tid); +} + +#endif + +// GetCachedTID() caches the thread ID in thread-local storage (which is a +// userspace construct) to avoid unnecessary system calls. Without this caching, +// it can take roughly 98ns, while it takes roughly 1ns with this caching. +pid_t GetCachedTID() { +#ifdef ABSL_HAVE_THREAD_LOCAL + static thread_local pid_t thread_id = GetTID(); + return thread_id; +#else + return GetTID(); +#endif // ABSL_HAVE_THREAD_LOCAL +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/sysinfo.h b/third_party/abseil_cpp/absl/base/internal/sysinfo.h new file mode 100644 index 000000000000..119cf1f0e85b --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/sysinfo.h @@ -0,0 +1,74 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file includes routines to find out characteristics +// of the machine a program is running on. It is undoubtedly +// system-dependent. + +// Functions listed here that accept a pid_t as an argument act on the +// current process if the pid_t argument is 0 +// All functions here are thread-hostile due to file caching unless +// commented otherwise. + +#ifndef ABSL_BASE_INTERNAL_SYSINFO_H_ +#define ABSL_BASE_INTERNAL_SYSINFO_H_ + +#ifndef _WIN32 +#include <sys/types.h> +#endif + +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// Nominal core processor cycles per second of each processor. This is _not_ +// necessarily the frequency of the CycleClock counter (see cycleclock.h) +// Thread-safe. +double NominalCPUFrequency(); + +// Number of logical processors (hyperthreads) in system. Thread-safe. +int NumCPUs(); + +// Return the thread id of the current thread, as told by the system. +// No two currently-live threads implemented by the OS shall have the same ID. +// Thread ids of exited threads may be reused. Multiple user-level threads +// may have the same thread ID if multiplexed on the same OS thread. +// +// On Linux, you may send a signal to the resulting ID with kill(). However, +// it is recommended for portability that you use pthread_kill() instead. +#ifdef _WIN32 +// On Windows, process id and thread id are of the same type according to the +// return types of GetProcessId() and GetThreadId() are both DWORD, an unsigned +// 32-bit type. +using pid_t = uint32_t; +#endif +pid_t GetTID(); + +// Like GetTID(), but caches the result in thread-local storage in order +// to avoid unnecessary system calls. Note that there are some cases where +// one must call through to GetTID directly, which is why this exists as a +// separate function. For example, GetCachedTID() is not safe to call in +// an asynchronous signal-handling context nor right after a call to fork(). +pid_t GetCachedTID(); + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_SYSINFO_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/sysinfo_test.cc b/third_party/abseil_cpp/absl/base/internal/sysinfo_test.cc new file mode 100644 index 000000000000..fa8b88b1dc07 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/sysinfo_test.cc @@ -0,0 +1,100 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/sysinfo.h" + +#ifndef _WIN32 +#include <sys/types.h> +#include <unistd.h> +#endif + +#include <thread> // NOLINT(build/c++11) +#include <unordered_set> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/synchronization/barrier.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +TEST(SysinfoTest, NumCPUs) { + EXPECT_NE(NumCPUs(), 0) + << "NumCPUs() should not have the default value of 0"; +} + +TEST(SysinfoTest, NominalCPUFrequency) { +#if !(defined(__aarch64__) && defined(__linux__)) && !defined(__EMSCRIPTEN__) + EXPECT_GE(NominalCPUFrequency(), 1000.0) + << "NominalCPUFrequency() did not return a reasonable value"; +#else + // Aarch64 cannot read the CPU frequency from sysfs, so we get back 1.0. + // Emscripten does not have a sysfs to read from at all. + EXPECT_EQ(NominalCPUFrequency(), 1.0) + << "CPU frequency detection was fixed! Please update unittest."; +#endif +} + +TEST(SysinfoTest, GetTID) { + EXPECT_EQ(GetTID(), GetTID()); // Basic compile and equality test. +#ifdef __native_client__ + // Native Client has a race condition bug that leads to memory + // exaustion when repeatedly creating and joining threads. + // https://bugs.chromium.org/p/nativeclient/issues/detail?id=1027 + return; +#endif + // Test that TIDs are unique to each thread. + // Uses a few loops to exercise implementations that reallocate IDs. + for (int i = 0; i < 10; ++i) { + constexpr int kNumThreads = 10; + Barrier all_threads_done(kNumThreads); + std::vector<std::thread> threads; + + Mutex mutex; + std::unordered_set<pid_t> tids; + + for (int j = 0; j < kNumThreads; ++j) { + threads.push_back(std::thread([&]() { + pid_t id = GetTID(); + { + MutexLock lock(&mutex); + ASSERT_TRUE(tids.find(id) == tids.end()); + tids.insert(id); + } + // We can't simply join the threads here. The threads need to + // be alive otherwise the TID might have been reallocated to + // another live thread. + all_threads_done.Block(); + })); + } + for (auto& thread : threads) { + thread.join(); + } + } +} + +#ifdef __linux__ +TEST(SysinfoTest, LinuxGetTID) { + // On Linux, for the main thread, GetTID()==getpid() is guaranteed by the API. + EXPECT_EQ(GetTID(), getpid()); +} +#endif + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/thread_annotations.h b/third_party/abseil_cpp/absl/base/internal/thread_annotations.h new file mode 100644 index 000000000000..4dab6a9c150a --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/thread_annotations.h @@ -0,0 +1,271 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: thread_annotations.h +// ----------------------------------------------------------------------------- +// +// WARNING: This is a backwards compatible header and it will be removed after +// the migration to prefixed thread annotations is finished; please include +// "absl/base/thread_annotations.h". +// +// This header file contains macro definitions for thread safety annotations +// that allow developers to document the locking policies of multi-threaded +// code. The annotations can also help program analysis tools to identify +// potential thread safety issues. +// +// These annotations are implemented using compiler attributes. Using the macros +// defined here instead of raw attributes allow for portability and future +// compatibility. +// +// When referring to mutexes in the arguments of the attributes, you should +// use variable names or more complex expressions (e.g. my_object->mutex_) +// that evaluate to a concrete mutex object whenever possible. If the mutex +// you want to refer to is not in scope, you may use a member pointer +// (e.g. &MyClass::mutex_) to refer to a mutex in some (unknown) object. + +#ifndef ABSL_BASE_INTERNAL_THREAD_ANNOTATIONS_H_ +#define ABSL_BASE_INTERNAL_THREAD_ANNOTATIONS_H_ + +#if defined(__clang__) +#define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) +#else +#define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op +#endif + +// GUARDED_BY() +// +// Documents if a shared field or global variable needs to be protected by a +// mutex. GUARDED_BY() allows the user to specify a particular mutex that +// should be held when accessing the annotated variable. +// +// Although this annotation (and PT_GUARDED_BY, below) cannot be applied to +// local variables, a local variable and its associated mutex can often be +// combined into a small class or struct, thereby allowing the annotation. +// +// Example: +// +// class Foo { +// Mutex mu_; +// int p1_ GUARDED_BY(mu_); +// ... +// }; +#define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) + +// PT_GUARDED_BY() +// +// Documents if the memory location pointed to by a pointer should be guarded +// by a mutex when dereferencing the pointer. +// +// Example: +// class Foo { +// Mutex mu_; +// int *p1_ PT_GUARDED_BY(mu_); +// ... +// }; +// +// Note that a pointer variable to a shared memory location could itself be a +// shared variable. +// +// Example: +// +// // `q_`, guarded by `mu1_`, points to a shared memory location that is +// // guarded by `mu2_`: +// int *q_ GUARDED_BY(mu1_) PT_GUARDED_BY(mu2_); +#define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) + +// ACQUIRED_AFTER() / ACQUIRED_BEFORE() +// +// Documents the acquisition order between locks that can be held +// simultaneously by a thread. For any two locks that need to be annotated +// to establish an acquisition order, only one of them needs the annotation. +// (i.e. You don't have to annotate both locks with both ACQUIRED_AFTER +// and ACQUIRED_BEFORE.) +// +// As with GUARDED_BY, this is only applicable to mutexes that are shared +// fields or global variables. +// +// Example: +// +// Mutex m1_; +// Mutex m2_ ACQUIRED_AFTER(m1_); +#define ACQUIRED_AFTER(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) + +#define ACQUIRED_BEFORE(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) + +// EXCLUSIVE_LOCKS_REQUIRED() / SHARED_LOCKS_REQUIRED() +// +// Documents a function that expects a mutex to be held prior to entry. +// The mutex is expected to be held both on entry to, and exit from, the +// function. +// +// An exclusive lock allows read-write access to the guarded data member(s), and +// only one thread can acquire a lock exclusively at any one time. A shared lock +// allows read-only access, and any number of threads can acquire a shared lock +// concurrently. +// +// Generally, non-const methods should be annotated with +// EXCLUSIVE_LOCKS_REQUIRED, while const methods should be annotated with +// SHARED_LOCKS_REQUIRED. +// +// Example: +// +// Mutex mu1, mu2; +// int a GUARDED_BY(mu1); +// int b GUARDED_BY(mu2); +// +// void foo() EXCLUSIVE_LOCKS_REQUIRED(mu1, mu2) { ... } +// void bar() const SHARED_LOCKS_REQUIRED(mu1, mu2) { ... } +#define EXCLUSIVE_LOCKS_REQUIRED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) + +#define SHARED_LOCKS_REQUIRED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) + +// LOCKS_EXCLUDED() +// +// Documents the locks acquired in the body of the function. These locks +// cannot be held when calling this function (as Abseil's `Mutex` locks are +// non-reentrant). +#define LOCKS_EXCLUDED(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) + +// LOCK_RETURNED() +// +// Documents a function that returns a mutex without acquiring it. For example, +// a public getter method that returns a pointer to a private mutex should +// be annotated with LOCK_RETURNED. +#define LOCK_RETURNED(x) \ + THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) + +// LOCKABLE +// +// Documents if a class/type is a lockable type (such as the `Mutex` class). +#define LOCKABLE \ + THREAD_ANNOTATION_ATTRIBUTE__(lockable) + +// SCOPED_LOCKABLE +// +// Documents if a class does RAII locking (such as the `MutexLock` class). +// The constructor should use `LOCK_FUNCTION()` to specify the mutex that is +// acquired, and the destructor should use `UNLOCK_FUNCTION()` with no +// arguments; the analysis will assume that the destructor unlocks whatever the +// constructor locked. +#define SCOPED_LOCKABLE \ + THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) + +// EXCLUSIVE_LOCK_FUNCTION() +// +// Documents functions that acquire a lock in the body of a function, and do +// not release it. +#define EXCLUSIVE_LOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) + +// SHARED_LOCK_FUNCTION() +// +// Documents functions that acquire a shared (reader) lock in the body of a +// function, and do not release it. +#define SHARED_LOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) + +// UNLOCK_FUNCTION() +// +// Documents functions that expect a lock to be held on entry to the function, +// and release it in the body of the function. +#define UNLOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) + +// EXCLUSIVE_TRYLOCK_FUNCTION() / SHARED_TRYLOCK_FUNCTION() +// +// Documents functions that try to acquire a lock, and return success or failure +// (or a non-boolean value that can be interpreted as a boolean). +// The first argument should be `true` for functions that return `true` on +// success, or `false` for functions that return `false` on success. The second +// argument specifies the mutex that is locked on success. If unspecified, this +// mutex is assumed to be `this`. +#define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) + +#define SHARED_TRYLOCK_FUNCTION(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) + +// ASSERT_EXCLUSIVE_LOCK() / ASSERT_SHARED_LOCK() +// +// Documents functions that dynamically check to see if a lock is held, and fail +// if it is not held. +#define ASSERT_EXCLUSIVE_LOCK(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) + +#define ASSERT_SHARED_LOCK(...) \ + THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) + +// NO_THREAD_SAFETY_ANALYSIS +// +// Turns off thread safety checking within the body of a particular function. +// This annotation is used to mark functions that are known to be correct, but +// the locking behavior is more complicated than the analyzer can handle. +#define NO_THREAD_SAFETY_ANALYSIS \ + THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) + +//------------------------------------------------------------------------------ +// Tool-Supplied Annotations +//------------------------------------------------------------------------------ + +// TS_UNCHECKED should be placed around lock expressions that are not valid +// C++ syntax, but which are present for documentation purposes. These +// annotations will be ignored by the analysis. +#define TS_UNCHECKED(x) "" + +// TS_FIXME is used to mark lock expressions that are not valid C++ syntax. +// It is used by automated tools to mark and disable invalid expressions. +// The annotation should either be fixed, or changed to TS_UNCHECKED. +#define TS_FIXME(x) "" + +// Like NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body of +// a particular function. However, this attribute is used to mark functions +// that are incorrect and need to be fixed. It is used by automated tools to +// avoid breaking the build when the analysis is updated. +// Code owners are expected to eventually fix the routine. +#define NO_THREAD_SAFETY_ANALYSIS_FIXME NO_THREAD_SAFETY_ANALYSIS + +// Similar to NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a GUARDED_BY +// annotation that needs to be fixed, because it is producing thread safety +// warning. It disables the GUARDED_BY. +#define GUARDED_BY_FIXME(x) + +// Disables warnings for a single read operation. This can be used to avoid +// warnings when it is known that the read is not actually involved in a race, +// but the compiler cannot confirm that. +#define TS_UNCHECKED_READ(x) thread_safety_analysis::ts_unchecked_read(x) + + +namespace thread_safety_analysis { + +// Takes a reference to a guarded data member, and returns an unguarded +// reference. +template <typename T> +inline const T& ts_unchecked_read(const T& v) NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +template <typename T> +inline T& ts_unchecked_read(T& v) NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +} // namespace thread_safety_analysis + +#endif // ABSL_BASE_INTERNAL_THREAD_ANNOTATIONS_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/thread_identity.cc b/third_party/abseil_cpp/absl/base/internal/thread_identity.cc new file mode 100644 index 000000000000..6ea010ed0d31 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/thread_identity.cc @@ -0,0 +1,155 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/thread_identity.h" + +#ifndef _WIN32 +#include <pthread.h> +#include <signal.h> +#endif + +#include <atomic> +#include <cassert> +#include <memory> + +#include "absl/base/attributes.h" +#include "absl/base/call_once.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +#if ABSL_THREAD_IDENTITY_MODE != ABSL_THREAD_IDENTITY_MODE_USE_CPP11 +namespace { +// Used to co-ordinate one-time creation of our pthread_key +absl::once_flag init_thread_identity_key_once; +pthread_key_t thread_identity_pthread_key; +std::atomic<bool> pthread_key_initialized(false); + +void AllocateThreadIdentityKey(ThreadIdentityReclaimerFunction reclaimer) { + pthread_key_create(&thread_identity_pthread_key, reclaimer); + pthread_key_initialized.store(true, std::memory_order_release); +} +} // namespace +#endif + +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ + ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 +// The actual TLS storage for a thread's currently associated ThreadIdentity. +// This is referenced by inline accessors in the header. +// "protected" visibility ensures that if multiple instances of Abseil code +// exist within a process (via dlopen() or similar), references to +// thread_identity_ptr from each instance of the code will refer to +// *different* instances of this ptr. +// Apple platforms have the visibility attribute, but issue a compile warning +// that protected visibility is unsupported. +#if ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__) +__attribute__((visibility("protected"))) +#endif // ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__) +#if ABSL_PER_THREAD_TLS +// Prefer __thread to thread_local as benchmarks indicate it is a bit faster. +ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* thread_identity_ptr = nullptr; +#elif defined(ABSL_HAVE_THREAD_LOCAL) +thread_local ThreadIdentity* thread_identity_ptr = nullptr; +#endif // ABSL_PER_THREAD_TLS +#endif // TLS or CPP11 + +void SetCurrentThreadIdentity( + ThreadIdentity* identity, ThreadIdentityReclaimerFunction reclaimer) { + assert(CurrentThreadIdentityIfPresent() == nullptr); + // Associate our destructor. + // NOTE: This call to pthread_setspecific is currently the only immovable + // barrier to CurrentThreadIdentity() always being async signal safe. +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC + // NOTE: Not async-safe. But can be open-coded. + absl::call_once(init_thread_identity_key_once, AllocateThreadIdentityKey, + reclaimer); + +#if defined(__EMSCRIPTEN__) || defined(__MINGW32__) + // Emscripten and MinGW pthread implementations does not support signals. + // See https://kripken.github.io/emscripten-site/docs/porting/pthreads.html + // for more information. + pthread_setspecific(thread_identity_pthread_key, + reinterpret_cast<void*>(identity)); +#else + // We must mask signals around the call to setspecific as with current glibc, + // a concurrent getspecific (needed for GetCurrentThreadIdentityIfPresent()) + // may zero our value. + // + // While not officially async-signal safe, getspecific within a signal handler + // is otherwise OK. + sigset_t all_signals; + sigset_t curr_signals; + sigfillset(&all_signals); + pthread_sigmask(SIG_SETMASK, &all_signals, &curr_signals); + pthread_setspecific(thread_identity_pthread_key, + reinterpret_cast<void*>(identity)); + pthread_sigmask(SIG_SETMASK, &curr_signals, nullptr); +#endif // !__EMSCRIPTEN__ && !__MINGW32__ + +#elif ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS + // NOTE: Not async-safe. But can be open-coded. + absl::call_once(init_thread_identity_key_once, AllocateThreadIdentityKey, + reclaimer); + pthread_setspecific(thread_identity_pthread_key, + reinterpret_cast<void*>(identity)); + thread_identity_ptr = identity; +#elif ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + thread_local std::unique_ptr<ThreadIdentity, ThreadIdentityReclaimerFunction> + holder(identity, reclaimer); + thread_identity_ptr = identity; +#else +#error Unimplemented ABSL_THREAD_IDENTITY_MODE +#endif +} + +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ + ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + +// Please see the comment on `CurrentThreadIdentityIfPresent` in +// thread_identity.h. Because DLLs cannot expose thread_local variables in +// headers, we opt for the correct-but-slower option of placing the definition +// of this function only in a translation unit inside DLL. +#if defined(ABSL_BUILD_DLL) || defined(ABSL_CONSUME_DLL) +ThreadIdentity* CurrentThreadIdentityIfPresent() { return thread_identity_ptr; } +#endif +#endif + +void ClearCurrentThreadIdentity() { +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ + ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + thread_identity_ptr = nullptr; +#elif ABSL_THREAD_IDENTITY_MODE == \ + ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC + // pthread_setspecific expected to clear value on destruction + assert(CurrentThreadIdentityIfPresent() == nullptr); +#endif +} + +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC +ThreadIdentity* CurrentThreadIdentityIfPresent() { + bool initialized = pthread_key_initialized.load(std::memory_order_acquire); + if (!initialized) { + return nullptr; + } + return reinterpret_cast<ThreadIdentity*>( + pthread_getspecific(thread_identity_pthread_key)); +} +#endif + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/thread_identity.h b/third_party/abseil_cpp/absl/base/internal/thread_identity.h new file mode 100644 index 000000000000..d2a65fd81126 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/thread_identity.h @@ -0,0 +1,260 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Each active thread has an ThreadIdentity that may represent the thread in +// various level interfaces. ThreadIdentity objects are never deallocated. +// When a thread terminates, its ThreadIdentity object may be reused for a +// thread created later. + +#ifndef ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_ +#define ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_ + +#ifndef _WIN32 +#include <pthread.h> +// Defines __GOOGLE_GRTE_VERSION__ (via glibc-specific features.h) when +// supported. +#include <unistd.h> +#endif + +#include <atomic> +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/base/internal/per_thread_tls.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +struct SynchLocksHeld; +struct SynchWaitParams; + +namespace base_internal { + +class SpinLock; +struct ThreadIdentity; + +// Used by the implementation of absl::Mutex and absl::CondVar. +struct PerThreadSynch { + // The internal representation of absl::Mutex and absl::CondVar rely + // on the alignment of PerThreadSynch. Both store the address of the + // PerThreadSynch in the high-order bits of their internal state, + // which means the low kLowZeroBits of the address of PerThreadSynch + // must be zero. + static constexpr int kLowZeroBits = 8; + static constexpr int kAlignment = 1 << kLowZeroBits; + + // Returns the associated ThreadIdentity. + // This can be implemented as a cast because we guarantee + // PerThreadSynch is the first element of ThreadIdentity. + ThreadIdentity* thread_identity() { + return reinterpret_cast<ThreadIdentity*>(this); + } + + PerThreadSynch *next; // Circular waiter queue; initialized to 0. + PerThreadSynch *skip; // If non-zero, all entries in Mutex queue + // up to and including "skip" have same + // condition as this, and will be woken later + bool may_skip; // if false while on mutex queue, a mutex unlocker + // is using this PerThreadSynch as a terminator. Its + // skip field must not be filled in because the loop + // might then skip over the terminator. + bool wake; // This thread is to be woken from a Mutex. + // If "x" is on a waiter list for a mutex, "x->cond_waiter" is true iff the + // waiter is waiting on the mutex as part of a CV Wait or Mutex Await. + // + // The value of "x->cond_waiter" is meaningless if "x" is not on a + // Mutex waiter list. + bool cond_waiter; + bool maybe_unlocking; // Valid at head of Mutex waiter queue; + // true if UnlockSlow could be searching + // for a waiter to wake. Used for an optimization + // in Enqueue(). true is always a valid value. + // Can be reset to false when the unlocker or any + // writer releases the lock, or a reader fully + // releases the lock. It may not be set to false + // by a reader that decrements the count to + // non-zero. protected by mutex spinlock + bool suppress_fatal_errors; // If true, try to proceed even in the face + // of broken invariants. This is used within + // fatal signal handlers to improve the + // chances of debug logging information being + // output successfully. + int priority; // Priority of thread (updated every so often). + + // State values: + // kAvailable: This PerThreadSynch is available. + // kQueued: This PerThreadSynch is unavailable, it's currently queued on a + // Mutex or CondVar waistlist. + // + // Transitions from kQueued to kAvailable require a release + // barrier. This is needed as a waiter may use "state" to + // independently observe that it's no longer queued. + // + // Transitions from kAvailable to kQueued require no barrier, they + // are externally ordered by the Mutex. + enum State { + kAvailable, + kQueued + }; + std::atomic<State> state; + + // The wait parameters of the current wait. waitp is null if the + // thread is not waiting. Transitions from null to non-null must + // occur before the enqueue commit point (state = kQueued in + // Enqueue() and CondVarEnqueue()). Transitions from non-null to + // null must occur after the wait is finished (state = kAvailable in + // Mutex::Block() and CondVar::WaitCommon()). This field may be + // changed only by the thread that describes this PerThreadSynch. A + // special case is Fer(), which calls Enqueue() on another thread, + // but with an identical SynchWaitParams pointer, thus leaving the + // pointer unchanged. + SynchWaitParams* waitp; + + intptr_t readers; // Number of readers in mutex. + + // When priority will next be read (cycles). + int64_t next_priority_read_cycles; + + // Locks held; used during deadlock detection. + // Allocated in Synch_GetAllLocks() and freed in ReclaimThreadIdentity(). + SynchLocksHeld *all_locks; +}; + +// The instances of this class are allocated in NewThreadIdentity() with an +// alignment of PerThreadSynch::kAlignment. +struct ThreadIdentity { + // Must be the first member. The Mutex implementation requires that + // the PerThreadSynch object associated with each thread is + // PerThreadSynch::kAlignment aligned. We provide this alignment on + // ThreadIdentity itself. + PerThreadSynch per_thread_synch; + + // Private: Reserved for absl::synchronization_internal::Waiter. + struct WaiterState { + char data[128]; + } waiter_state; + + // Used by PerThreadSem::{Get,Set}ThreadBlockedCounter(). + std::atomic<int>* blocked_count_ptr; + + // The following variables are mostly read/written just by the + // thread itself. The only exception is that these are read by + // a ticker thread as a hint. + std::atomic<int> ticker; // Tick counter, incremented once per second. + std::atomic<int> wait_start; // Ticker value when thread started waiting. + std::atomic<bool> is_idle; // Has thread become idle yet? + + ThreadIdentity* next; +}; + +// Returns the ThreadIdentity object representing the calling thread; guaranteed +// to be unique for its lifetime. The returned object will remain valid for the +// program's lifetime; although it may be re-assigned to a subsequent thread. +// If one does not exist, return nullptr instead. +// +// Does not malloc(*), and is async-signal safe. +// [*] Technically pthread_setspecific() does malloc on first use; however this +// is handled internally within tcmalloc's initialization already. +// +// New ThreadIdentity objects can be constructed and associated with a thread +// by calling GetOrCreateCurrentThreadIdentity() in per-thread-sem.h. +ThreadIdentity* CurrentThreadIdentityIfPresent(); + +using ThreadIdentityReclaimerFunction = void (*)(void*); + +// Sets the current thread identity to the given value. 'reclaimer' is a +// pointer to the global function for cleaning up instances on thread +// destruction. +void SetCurrentThreadIdentity(ThreadIdentity* identity, + ThreadIdentityReclaimerFunction reclaimer); + +// Removes the currently associated ThreadIdentity from the running thread. +// This must be called from inside the ThreadIdentityReclaimerFunction, and only +// from that function. +void ClearCurrentThreadIdentity(); + +// May be chosen at compile time via: -DABSL_FORCE_THREAD_IDENTITY_MODE=<mode +// index> +#ifdef ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC +#error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be direcly set +#else +#define ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC 0 +#endif + +#ifdef ABSL_THREAD_IDENTITY_MODE_USE_TLS +#error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be direcly set +#else +#define ABSL_THREAD_IDENTITY_MODE_USE_TLS 1 +#endif + +#ifdef ABSL_THREAD_IDENTITY_MODE_USE_CPP11 +#error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be direcly set +#else +#define ABSL_THREAD_IDENTITY_MODE_USE_CPP11 2 +#endif + +#ifdef ABSL_THREAD_IDENTITY_MODE +#error ABSL_THREAD_IDENTITY_MODE cannot be direcly set +#elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE) +#define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE +#elif defined(_WIN32) && !defined(__MINGW32__) +#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 +#elif defined(__APPLE__) && defined(ABSL_HAVE_THREAD_LOCAL) +#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_CPP11 +#elif ABSL_PER_THREAD_TLS && defined(__GOOGLE_GRTE_VERSION__) && \ + (__GOOGLE_GRTE_VERSION__ >= 20140228L) +// Support for async-safe TLS was specifically added in GRTEv4. It's not +// present in the upstream eglibc. +// Note: Current default for production systems. +#define ABSL_THREAD_IDENTITY_MODE ABSL_THREAD_IDENTITY_MODE_USE_TLS +#else +#define ABSL_THREAD_IDENTITY_MODE \ + ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC +#endif + +#if ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_TLS || \ + ABSL_THREAD_IDENTITY_MODE == ABSL_THREAD_IDENTITY_MODE_USE_CPP11 + +#if ABSL_PER_THREAD_TLS +ABSL_CONST_INIT extern ABSL_PER_THREAD_TLS_KEYWORD ThreadIdentity* + thread_identity_ptr; +#elif defined(ABSL_HAVE_THREAD_LOCAL) +ABSL_CONST_INIT extern thread_local ThreadIdentity* thread_identity_ptr; +#else +#error Thread-local storage not detected on this platform +#endif + +// thread_local variables cannot be in headers exposed by DLLs. However, it is +// important for performance reasons in general that +// `CurrentThreadIdentityIfPresent` be inlined. This is not possible across a +// DLL boundary so, with DLLs, we opt to have the function not be inlined. Note +// that `CurrentThreadIdentityIfPresent` is declared above so we can exclude +// this entire inline definition when compiling as a DLL. +#if !defined(ABSL_BUILD_DLL) && !defined(ABSL_CONSUME_DLL) +inline ThreadIdentity* CurrentThreadIdentityIfPresent() { + return thread_identity_ptr; +} +#endif + +#elif ABSL_THREAD_IDENTITY_MODE != \ + ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC +#error Unknown ABSL_THREAD_IDENTITY_MODE +#endif + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_THREAD_IDENTITY_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/thread_identity_benchmark.cc b/third_party/abseil_cpp/absl/base/internal/thread_identity_benchmark.cc new file mode 100644 index 000000000000..0ae10f2b1e8a --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/thread_identity_benchmark.cc @@ -0,0 +1,38 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/synchronization/internal/create_thread_identity.h" +#include "absl/synchronization/internal/per_thread_sem.h" + +namespace { + +void BM_SafeCurrentThreadIdentity(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::synchronization_internal::GetOrCreateCurrentThreadIdentity()); + } +} +BENCHMARK(BM_SafeCurrentThreadIdentity); + +void BM_UnsafeCurrentThreadIdentity(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::base_internal::CurrentThreadIdentityIfPresent()); + } +} +BENCHMARK(BM_UnsafeCurrentThreadIdentity); + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/thread_identity_test.cc b/third_party/abseil_cpp/absl/base/internal/thread_identity_test.cc new file mode 100644 index 000000000000..46a6f7434629 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/thread_identity_test.cc @@ -0,0 +1,129 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/thread_identity.h" + +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/macros.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/internal/per_thread_sem.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +ABSL_CONST_INIT static absl::base_internal::SpinLock map_lock( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); +ABSL_CONST_INIT static int num_identities_reused ABSL_GUARDED_BY(map_lock); + +static const void* const kCheckNoIdentity = reinterpret_cast<void*>(1); + +static void TestThreadIdentityCurrent(const void* assert_no_identity) { + ThreadIdentity* identity; + + // We have to test this conditionally, because if the test framework relies + // on Abseil, then some previous action may have already allocated an + // identity. + if (assert_no_identity == kCheckNoIdentity) { + identity = CurrentThreadIdentityIfPresent(); + EXPECT_TRUE(identity == nullptr); + } + + identity = synchronization_internal::GetOrCreateCurrentThreadIdentity(); + EXPECT_TRUE(identity != nullptr); + ThreadIdentity* identity_no_init; + identity_no_init = CurrentThreadIdentityIfPresent(); + EXPECT_TRUE(identity == identity_no_init); + + // Check that per_thread_synch is correctly aligned. + EXPECT_EQ(0, reinterpret_cast<intptr_t>(&identity->per_thread_synch) % + PerThreadSynch::kAlignment); + EXPECT_EQ(identity, identity->per_thread_synch.thread_identity()); + + absl::base_internal::SpinLockHolder l(&map_lock); + num_identities_reused++; +} + +TEST(ThreadIdentityTest, BasicIdentityWorks) { + // This tests for the main() thread. + TestThreadIdentityCurrent(nullptr); +} + +TEST(ThreadIdentityTest, BasicIdentityWorksThreaded) { + // Now try the same basic test with multiple threads being created and + // destroyed. This makes sure that: + // - New threads are created without a ThreadIdentity. + // - We re-allocate ThreadIdentity objects from the free-list. + // - If a thread implementation chooses to recycle threads, that + // correct re-initialization occurs. + static const int kNumLoops = 3; + static const int kNumThreads = 32; + for (int iter = 0; iter < kNumLoops; iter++) { + std::vector<std::thread> threads; + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back( + std::thread(TestThreadIdentityCurrent, kCheckNoIdentity)); + } + for (auto& thread : threads) { + thread.join(); + } + } + + // We should have recycled ThreadIdentity objects above; while (external) + // library threads allocating their own identities may preclude some + // reuse, we should have sufficient repetitions to exclude this. + absl::base_internal::SpinLockHolder l(&map_lock); + EXPECT_LT(kNumThreads, num_identities_reused); +} + +TEST(ThreadIdentityTest, ReusedThreadIdentityMutexTest) { + // This test repeatly creates and joins a series of threads, each of + // which acquires and releases shared Mutex locks. This verifies + // Mutex operations work correctly under a reused + // ThreadIdentity. Note that the most likely failure mode of this + // test is a crash or deadlock. + static const int kNumLoops = 10; + static const int kNumThreads = 12; + static const int kNumMutexes = 3; + static const int kNumLockLoops = 5; + + Mutex mutexes[kNumMutexes]; + for (int iter = 0; iter < kNumLoops; ++iter) { + std::vector<std::thread> threads; + for (int thread = 0; thread < kNumThreads; ++thread) { + threads.push_back(std::thread([&]() { + for (int l = 0; l < kNumLockLoops; ++l) { + for (int m = 0; m < kNumMutexes; ++m) { + MutexLock lock(&mutexes[m]); + } + } + })); + } + for (auto& thread : threads) { + thread.join(); + } + } +} + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/throw_delegate.cc b/third_party/abseil_cpp/absl/base/internal/throw_delegate.cc new file mode 100644 index 000000000000..c260ff1eed63 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/throw_delegate.cc @@ -0,0 +1,212 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/throw_delegate.h" + +#include <cstdlib> +#include <functional> +#include <new> +#include <stdexcept> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// NOTE: The various STL exception throwing functions are placed within the +// #ifdef blocks so the symbols aren't exposed on platforms that don't support +// them, such as the Android NDK. For example, ANGLE fails to link when building +// within AOSP without them, since the STL functions don't exist. +namespace { +#ifdef ABSL_HAVE_EXCEPTIONS +template <typename T> +[[noreturn]] void Throw(const T& error) { + throw error; +} +#endif +} // namespace + +void ThrowStdLogicError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::logic_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdLogicError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::logic_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} +void ThrowStdInvalidArgument(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::invalid_argument(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdInvalidArgument(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::invalid_argument(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdDomainError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::domain_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdDomainError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::domain_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdLengthError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::length_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdLengthError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::length_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdOutOfRange(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::out_of_range(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdOutOfRange(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::out_of_range(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdRuntimeError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::runtime_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdRuntimeError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::runtime_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdRangeError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::range_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdRangeError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::range_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdOverflowError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::overflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdOverflowError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::overflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdUnderflowError(const std::string& what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::underflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg.c_str()); + std::abort(); +#endif +} +void ThrowStdUnderflowError(const char* what_arg) { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::underflow_error(what_arg)); +#else + ABSL_RAW_LOG(FATAL, "%s", what_arg); + std::abort(); +#endif +} + +void ThrowStdBadFunctionCall() { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::bad_function_call()); +#else + std::abort(); +#endif +} + +void ThrowStdBadAlloc() { +#ifdef ABSL_HAVE_EXCEPTIONS + Throw(std::bad_alloc()); +#else + std::abort(); +#endif +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/internal/throw_delegate.h b/third_party/abseil_cpp/absl/base/internal/throw_delegate.h new file mode 100644 index 000000000000..075f5272543a --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/throw_delegate.h @@ -0,0 +1,75 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_THROW_DELEGATE_H_ +#define ABSL_BASE_INTERNAL_THROW_DELEGATE_H_ + +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// Helper functions that allow throwing exceptions consistently from anywhere. +// The main use case is for header-based libraries (eg templates), as they will +// be built by many different targets with their own compiler options. +// In particular, this will allow a safe way to throw exceptions even if the +// caller is compiled with -fno-exceptions. This is intended for implementing +// things like map<>::at(), which the standard documents as throwing an +// exception on error. +// +// Using other techniques like #if tricks could lead to ODR violations. +// +// You shouldn't use it unless you're writing code that you know will be built +// both with and without exceptions and you need to conform to an interface +// that uses exceptions. + +[[noreturn]] void ThrowStdLogicError(const std::string& what_arg); +[[noreturn]] void ThrowStdLogicError(const char* what_arg); +[[noreturn]] void ThrowStdInvalidArgument(const std::string& what_arg); +[[noreturn]] void ThrowStdInvalidArgument(const char* what_arg); +[[noreturn]] void ThrowStdDomainError(const std::string& what_arg); +[[noreturn]] void ThrowStdDomainError(const char* what_arg); +[[noreturn]] void ThrowStdLengthError(const std::string& what_arg); +[[noreturn]] void ThrowStdLengthError(const char* what_arg); +[[noreturn]] void ThrowStdOutOfRange(const std::string& what_arg); +[[noreturn]] void ThrowStdOutOfRange(const char* what_arg); +[[noreturn]] void ThrowStdRuntimeError(const std::string& what_arg); +[[noreturn]] void ThrowStdRuntimeError(const char* what_arg); +[[noreturn]] void ThrowStdRangeError(const std::string& what_arg); +[[noreturn]] void ThrowStdRangeError(const char* what_arg); +[[noreturn]] void ThrowStdOverflowError(const std::string& what_arg); +[[noreturn]] void ThrowStdOverflowError(const char* what_arg); +[[noreturn]] void ThrowStdUnderflowError(const std::string& what_arg); +[[noreturn]] void ThrowStdUnderflowError(const char* what_arg); + +[[noreturn]] void ThrowStdBadFunctionCall(); +[[noreturn]] void ThrowStdBadAlloc(); + +// ThrowStdBadArrayNewLength() cannot be consistently supported because +// std::bad_array_new_length is missing in libstdc++ until 4.9.0. +// https://gcc.gnu.org/onlinedocs/gcc-4.8.3/libstdc++/api/a01379_source.html +// https://gcc.gnu.org/onlinedocs/gcc-4.9.0/libstdc++/api/a01327_source.html +// libcxx (as of 3.2) and msvc (as of 2015) both have it. +// [[noreturn]] void ThrowStdBadArrayNewLength(); + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_THROW_DELEGATE_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/tsan_mutex_interface.h b/third_party/abseil_cpp/absl/base/internal/tsan_mutex_interface.h new file mode 100644 index 000000000000..39207d8a5c42 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/tsan_mutex_interface.h @@ -0,0 +1,68 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file is intended solely for spinlock.h. +// It provides ThreadSanitizer annotations for custom mutexes. +// See <sanitizer/tsan_interface.h> for meaning of these annotations. + +#ifndef ABSL_BASE_INTERNAL_TSAN_MUTEX_INTERFACE_H_ +#define ABSL_BASE_INTERNAL_TSAN_MUTEX_INTERFACE_H_ + +#include "absl/base/config.h" + +// ABSL_INTERNAL_HAVE_TSAN_INTERFACE +// Macro intended only for internal use. +// +// Checks whether LLVM Thread Sanitizer interfaces are available. +// First made available in LLVM 5.0 (Sep 2017). +#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE +#error "ABSL_INTERNAL_HAVE_TSAN_INTERFACE cannot be directly set." +#endif + +#if defined(ABSL_HAVE_THREAD_SANITIZER) && defined(__has_include) +#if __has_include(<sanitizer/tsan_interface.h>) +#define ABSL_INTERNAL_HAVE_TSAN_INTERFACE 1 +#endif +#endif + +#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE +#include <sanitizer/tsan_interface.h> + +#define ABSL_TSAN_MUTEX_CREATE __tsan_mutex_create +#define ABSL_TSAN_MUTEX_DESTROY __tsan_mutex_destroy +#define ABSL_TSAN_MUTEX_PRE_LOCK __tsan_mutex_pre_lock +#define ABSL_TSAN_MUTEX_POST_LOCK __tsan_mutex_post_lock +#define ABSL_TSAN_MUTEX_PRE_UNLOCK __tsan_mutex_pre_unlock +#define ABSL_TSAN_MUTEX_POST_UNLOCK __tsan_mutex_post_unlock +#define ABSL_TSAN_MUTEX_PRE_SIGNAL __tsan_mutex_pre_signal +#define ABSL_TSAN_MUTEX_POST_SIGNAL __tsan_mutex_post_signal +#define ABSL_TSAN_MUTEX_PRE_DIVERT __tsan_mutex_pre_divert +#define ABSL_TSAN_MUTEX_POST_DIVERT __tsan_mutex_post_divert + +#else + +#define ABSL_TSAN_MUTEX_CREATE(...) +#define ABSL_TSAN_MUTEX_DESTROY(...) +#define ABSL_TSAN_MUTEX_PRE_LOCK(...) +#define ABSL_TSAN_MUTEX_POST_LOCK(...) +#define ABSL_TSAN_MUTEX_PRE_UNLOCK(...) +#define ABSL_TSAN_MUTEX_POST_UNLOCK(...) +#define ABSL_TSAN_MUTEX_PRE_SIGNAL(...) +#define ABSL_TSAN_MUTEX_POST_SIGNAL(...) +#define ABSL_TSAN_MUTEX_PRE_DIVERT(...) +#define ABSL_TSAN_MUTEX_POST_DIVERT(...) + +#endif + +#endif // ABSL_BASE_INTERNAL_TSAN_MUTEX_INTERFACE_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/unaligned_access.h b/third_party/abseil_cpp/absl/base/internal/unaligned_access.h new file mode 100644 index 000000000000..093dd9b499f0 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/unaligned_access.h @@ -0,0 +1,82 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_BASE_INTERNAL_UNALIGNED_ACCESS_H_ +#define ABSL_BASE_INTERNAL_UNALIGNED_ACCESS_H_ + +#include <string.h> + +#include <cstdint> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" + +// unaligned APIs + +// Portable handling of unaligned loads, stores, and copies. + +// The unaligned API is C++ only. The declarations use C++ features +// (namespaces, inline) which are absent or incompatible in C. +#if defined(__cplusplus) +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +inline uint16_t UnalignedLoad16(const void *p) { + uint16_t t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint32_t UnalignedLoad32(const void *p) { + uint32_t t; + memcpy(&t, p, sizeof t); + return t; +} + +inline uint64_t UnalignedLoad64(const void *p) { + uint64_t t; + memcpy(&t, p, sizeof t); + return t; +} + +inline void UnalignedStore16(void *p, uint16_t v) { memcpy(p, &v, sizeof v); } + +inline void UnalignedStore32(void *p, uint32_t v) { memcpy(p, &v, sizeof v); } + +inline void UnalignedStore64(void *p, uint64_t v) { memcpy(p, &v, sizeof v); } + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#define ABSL_INTERNAL_UNALIGNED_LOAD16(_p) \ + (absl::base_internal::UnalignedLoad16(_p)) +#define ABSL_INTERNAL_UNALIGNED_LOAD32(_p) \ + (absl::base_internal::UnalignedLoad32(_p)) +#define ABSL_INTERNAL_UNALIGNED_LOAD64(_p) \ + (absl::base_internal::UnalignedLoad64(_p)) + +#define ABSL_INTERNAL_UNALIGNED_STORE16(_p, _val) \ + (absl::base_internal::UnalignedStore16(_p, _val)) +#define ABSL_INTERNAL_UNALIGNED_STORE32(_p, _val) \ + (absl::base_internal::UnalignedStore32(_p, _val)) +#define ABSL_INTERNAL_UNALIGNED_STORE64(_p, _val) \ + (absl::base_internal::UnalignedStore64(_p, _val)) + +#endif // defined(__cplusplus), end of unaligned API + +#endif // ABSL_BASE_INTERNAL_UNALIGNED_ACCESS_H_ diff --git a/third_party/abseil_cpp/absl/base/internal/unique_small_name_test.cc b/third_party/abseil_cpp/absl/base/internal/unique_small_name_test.cc new file mode 100644 index 000000000000..ff8c2b3fb427 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/unique_small_name_test.cc @@ -0,0 +1,77 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gtest/gtest.h" +#include "absl/base/optimization.h" +#include "absl/strings/string_view.h" + +// This test by itself does not do anything fancy, but it serves as binary I can +// query in shell test. + +namespace { + +template <class T> +void DoNotOptimize(const T& var) { +#ifdef __GNUC__ + asm volatile("" : "+m"(const_cast<T&>(var))); +#else + std::cout << (void*)&var; +#endif +} + +int very_long_int_variable_name ABSL_INTERNAL_UNIQUE_SMALL_NAME() = 0; +char very_long_str_variable_name[] ABSL_INTERNAL_UNIQUE_SMALL_NAME() = "abc"; + +TEST(UniqueSmallName, NonAutomaticVar) { + EXPECT_EQ(very_long_int_variable_name, 0); + EXPECT_EQ(absl::string_view(very_long_str_variable_name), "abc"); +} + +int VeryLongFreeFunctionName() ABSL_INTERNAL_UNIQUE_SMALL_NAME(); + +TEST(UniqueSmallName, FreeFunction) { + DoNotOptimize(&VeryLongFreeFunctionName); + + EXPECT_EQ(VeryLongFreeFunctionName(), 456); +} + +int VeryLongFreeFunctionName() { return 456; } + +struct VeryLongStructName { + explicit VeryLongStructName(int i); + + int VeryLongMethodName() ABSL_INTERNAL_UNIQUE_SMALL_NAME(); + + static int VeryLongStaticMethodName() ABSL_INTERNAL_UNIQUE_SMALL_NAME(); + + private: + int fld; +}; + +TEST(UniqueSmallName, Struct) { + VeryLongStructName var(10); + + DoNotOptimize(var); + DoNotOptimize(&VeryLongStructName::VeryLongMethodName); + DoNotOptimize(&VeryLongStructName::VeryLongStaticMethodName); + + EXPECT_EQ(var.VeryLongMethodName(), 10); + EXPECT_EQ(VeryLongStructName::VeryLongStaticMethodName(), 123); +} + +VeryLongStructName::VeryLongStructName(int i) : fld(i) {} +int VeryLongStructName::VeryLongMethodName() { return fld; } +int VeryLongStructName::VeryLongStaticMethodName() { return 123; } + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/internal/unscaledcycleclock.cc b/third_party/abseil_cpp/absl/base/internal/unscaledcycleclock.cc new file mode 100644 index 000000000000..1545288cf2de --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/unscaledcycleclock.cc @@ -0,0 +1,138 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/unscaledcycleclock.h" + +#if ABSL_USE_UNSCALED_CYCLECLOCK + +#if defined(_WIN32) +#include <intrin.h> +#endif + +#if defined(__powerpc__) || defined(__ppc__) +#ifdef __GLIBC__ +#include <sys/platform/ppc.h> +#elif defined(__FreeBSD__) +#include <sys/sysctl.h> +#include <sys/types.h> +#endif +#endif + +#include "absl/base/internal/sysinfo.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +#if defined(__i386__) + +int64_t UnscaledCycleClock::Now() { + int64_t ret; + __asm__ volatile("rdtsc" : "=A"(ret)); + return ret; +} + +double UnscaledCycleClock::Frequency() { + return base_internal::NominalCPUFrequency(); +} + +#elif defined(__x86_64__) + +int64_t UnscaledCycleClock::Now() { + uint64_t low, high; + __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); + return (high << 32) | low; +} + +double UnscaledCycleClock::Frequency() { + return base_internal::NominalCPUFrequency(); +} + +#elif defined(__powerpc__) || defined(__ppc__) + +int64_t UnscaledCycleClock::Now() { +#ifdef __GLIBC__ + return __ppc_get_timebase(); +#else +#ifdef __powerpc64__ + int64_t tbr; + asm volatile("mfspr %0, 268" : "=r"(tbr)); + return tbr; +#else + int32_t tbu, tbl, tmp; + asm volatile( + "0:\n" + "mftbu %[hi32]\n" + "mftb %[lo32]\n" + "mftbu %[tmp]\n" + "cmpw %[tmp],%[hi32]\n" + "bne 0b\n" + : [ hi32 ] "=r"(tbu), [ lo32 ] "=r"(tbl), [ tmp ] "=r"(tmp)); + return (static_cast<int64_t>(tbu) << 32) | tbl; +#endif +#endif +} + +double UnscaledCycleClock::Frequency() { +#ifdef __GLIBC__ + return __ppc_get_timebase_freq(); +#elif defined(__FreeBSD__) + static once_flag init_timebase_frequency_once; + static double timebase_frequency = 0.0; + base_internal::LowLevelCallOnce(&init_timebase_frequency_once, [&]() { + size_t length = sizeof(timebase_frequency); + sysctlbyname("kern.timecounter.tc.timebase.frequency", &timebase_frequency, + &length, nullptr, 0); + }); + return timebase_frequency; +#else +#error Must implement UnscaledCycleClock::Frequency() +#endif +} + +#elif defined(__aarch64__) + +// System timer of ARMv8 runs at a different frequency than the CPU's. +// The frequency is fixed, typically in the range 1-50MHz. It can be +// read at CNTFRQ special register. We assume the OS has set up +// the virtual timer properly. +int64_t UnscaledCycleClock::Now() { + int64_t virtual_timer_value; + asm volatile("mrs %0, cntvct_el0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +double UnscaledCycleClock::Frequency() { + uint64_t aarch64_timer_frequency; + asm volatile("mrs %0, cntfrq_el0" : "=r"(aarch64_timer_frequency)); + return aarch64_timer_frequency; +} + +#elif defined(_M_IX86) || defined(_M_X64) + +#pragma intrinsic(__rdtsc) + +int64_t UnscaledCycleClock::Now() { return __rdtsc(); } + +double UnscaledCycleClock::Frequency() { + return base_internal::NominalCPUFrequency(); +} + +#endif + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USE_UNSCALED_CYCLECLOCK diff --git a/third_party/abseil_cpp/absl/base/internal/unscaledcycleclock.h b/third_party/abseil_cpp/absl/base/internal/unscaledcycleclock.h new file mode 100644 index 000000000000..82f2c87a95a9 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/internal/unscaledcycleclock.h @@ -0,0 +1,124 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// UnscaledCycleClock +// An UnscaledCycleClock yields the value and frequency of a cycle counter +// that increments at a rate that is approximately constant. +// This class is for internal use only, you should consider using CycleClock +// instead. +// +// Notes: +// The cycle counter frequency is not necessarily the core clock frequency. +// That is, CycleCounter cycles are not necessarily "CPU cycles". +// +// An arbitrary offset may have been added to the counter at power on. +// +// On some platforms, the rate and offset of the counter may differ +// slightly when read from different CPUs of a multiprocessor. Usually, +// we try to ensure that the operating system adjusts values periodically +// so that values agree approximately. If you need stronger guarantees, +// consider using alternate interfaces. +// +// The CPU is not required to maintain the ordering of a cycle counter read +// with respect to surrounding instructions. + +#ifndef ABSL_BASE_INTERNAL_UNSCALEDCYCLECLOCK_H_ +#define ABSL_BASE_INTERNAL_UNSCALEDCYCLECLOCK_H_ + +#include <cstdint> + +#if defined(__APPLE__) +#include <TargetConditionals.h> +#endif + +#include "absl/base/port.h" + +// The following platforms have an implementation of a hardware counter. +#if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || \ + defined(__powerpc__) || defined(__ppc__) || \ + defined(_M_IX86) || defined(_M_X64) +#define ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION 1 +#else +#define ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION 0 +#endif + +// The following platforms often disable access to the hardware +// counter (through a sandbox) even if the underlying hardware has a +// usable counter. The CycleTimer interface also requires a *scaled* +// CycleClock that runs at atleast 1 MHz. We've found some Android +// ARM64 devices where this is not the case, so we disable it by +// default on Android ARM64. +#if defined(__native_client__) || \ + (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) || \ + (defined(__ANDROID__) && defined(__aarch64__)) +#define ABSL_USE_UNSCALED_CYCLECLOCK_DEFAULT 0 +#else +#define ABSL_USE_UNSCALED_CYCLECLOCK_DEFAULT 1 +#endif + +// UnscaledCycleClock is an optional internal feature. +// Use "#if ABSL_USE_UNSCALED_CYCLECLOCK" to test for its presence. +// Can be overridden at compile-time via -DABSL_USE_UNSCALED_CYCLECLOCK=0|1 +#if !defined(ABSL_USE_UNSCALED_CYCLECLOCK) +#define ABSL_USE_UNSCALED_CYCLECLOCK \ + (ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION && \ + ABSL_USE_UNSCALED_CYCLECLOCK_DEFAULT) +#endif + +#if ABSL_USE_UNSCALED_CYCLECLOCK + +// This macro can be used to test if UnscaledCycleClock::Frequency() +// is NominalCPUFrequency() on a particular platform. +#if (defined(__i386__) || defined(__x86_64__) || \ + defined(_M_IX86) || defined(_M_X64)) +#define ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +class UnscaledCycleClockWrapperForGetCurrentTime; +} // namespace time_internal + +namespace base_internal { +class CycleClock; +class UnscaledCycleClockWrapperForInitializeFrequency; + +class UnscaledCycleClock { + private: + UnscaledCycleClock() = delete; + + // Return the value of a cycle counter that counts at a rate that is + // approximately constant. + static int64_t Now(); + + // Return the how much UnscaledCycleClock::Now() increases per second. + // This is not necessarily the core CPU clock frequency. + // It may be the nominal value report by the kernel, rather than a measured + // value. + static double Frequency(); + + // Allowed users + friend class base_internal::CycleClock; + friend class time_internal::UnscaledCycleClockWrapperForGetCurrentTime; + friend class base_internal::UnscaledCycleClockWrapperForInitializeFrequency; +}; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USE_UNSCALED_CYCLECLOCK + +#endif // ABSL_BASE_INTERNAL_UNSCALEDCYCLECLOCK_H_ diff --git a/third_party/abseil_cpp/absl/base/invoke_test.cc b/third_party/abseil_cpp/absl/base/invoke_test.cc new file mode 100644 index 000000000000..bcdef36c3bfa --- /dev/null +++ b/third_party/abseil_cpp/absl/base/invoke_test.cc @@ -0,0 +1,229 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/invoke.h" + +#include <functional> +#include <memory> +#include <string> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { +namespace { + +int Function(int a, int b) { return a - b; } + +int Sink(std::unique_ptr<int> p) { + return *p; +} + +std::unique_ptr<int> Factory(int n) { + return make_unique<int>(n); +} + +void NoOp() {} + +struct ConstFunctor { + int operator()(int a, int b) const { return a - b; } +}; + +struct MutableFunctor { + int operator()(int a, int b) { return a - b; } +}; + +struct EphemeralFunctor { + int operator()(int a, int b) && { return a - b; } +}; + +struct OverloadedFunctor { + template <typename... Args> + std::string operator()(const Args&... args) & { + return StrCat("&", args...); + } + template <typename... Args> + std::string operator()(const Args&... args) const& { + return StrCat("const&", args...); + } + template <typename... Args> + std::string operator()(const Args&... args) && { + return StrCat("&&", args...); + } +}; + +struct Class { + int Method(int a, int b) { return a - b; } + int ConstMethod(int a, int b) const { return a - b; } + int RefMethod(int a, int b) & { return a - b; } + int RefRefMethod(int a, int b) && { return a - b; } + int NoExceptMethod(int a, int b) noexcept { return a - b; } + int VolatileMethod(int a, int b) volatile { return a - b; } + + int member; +}; + +struct FlipFlop { + int ConstMethod() const { return member; } + FlipFlop operator*() const { return {-member}; } + + int member; +}; + +// CallMaybeWithArg(f) resolves either to invoke(f) or invoke(f, 42), depending +// on which one is valid. +template <typename F> +decltype(base_internal::invoke(std::declval<const F&>())) CallMaybeWithArg( + const F& f) { + return base_internal::invoke(f); +} + +template <typename F> +decltype(base_internal::invoke(std::declval<const F&>(), 42)) CallMaybeWithArg( + const F& f) { + return base_internal::invoke(f, 42); +} + +TEST(InvokeTest, Function) { + EXPECT_EQ(1, base_internal::invoke(Function, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Function, 3, 2)); +} + +TEST(InvokeTest, NonCopyableArgument) { + EXPECT_EQ(42, base_internal::invoke(Sink, make_unique<int>(42))); +} + +TEST(InvokeTest, NonCopyableResult) { + EXPECT_THAT(base_internal::invoke(Factory, 42), ::testing::Pointee(42)); +} + +TEST(InvokeTest, VoidResult) { base_internal::invoke(NoOp); } + +TEST(InvokeTest, ConstFunctor) { + EXPECT_EQ(1, base_internal::invoke(ConstFunctor(), 3, 2)); +} + +TEST(InvokeTest, MutableFunctor) { + MutableFunctor f; + EXPECT_EQ(1, base_internal::invoke(f, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(MutableFunctor(), 3, 2)); +} + +TEST(InvokeTest, EphemeralFunctor) { + EphemeralFunctor f; + EXPECT_EQ(1, base_internal::invoke(std::move(f), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(EphemeralFunctor(), 3, 2)); +} + +TEST(InvokeTest, OverloadedFunctor) { + OverloadedFunctor f; + const OverloadedFunctor& cf = f; + + EXPECT_EQ("&", base_internal::invoke(f)); + EXPECT_EQ("& 42", base_internal::invoke(f, " 42")); + + EXPECT_EQ("const&", base_internal::invoke(cf)); + EXPECT_EQ("const& 42", base_internal::invoke(cf, " 42")); + + EXPECT_EQ("&&", base_internal::invoke(std::move(f))); + + OverloadedFunctor f2; + EXPECT_EQ("&& 42", base_internal::invoke(std::move(f2), " 42")); +} + +TEST(InvokeTest, ReferenceWrapper) { + ConstFunctor cf; + MutableFunctor mf; + EXPECT_EQ(1, base_internal::invoke(std::cref(cf), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(std::ref(cf), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(std::ref(mf), 3, 2)); +} + +TEST(InvokeTest, MemberFunction) { + std::unique_ptr<Class> p(new Class); + std::unique_ptr<const Class> cp(new Class); + std::unique_ptr<volatile Class> vp(new Class); + + EXPECT_EQ(1, base_internal::invoke(&Class::Method, p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::Method, p.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::Method, *p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::RefMethod, p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::RefMethod, p.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::RefMethod, *p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::RefRefMethod, std::move(*p), 3, + 2)); // NOLINT + EXPECT_EQ(1, base_internal::invoke(&Class::NoExceptMethod, p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::NoExceptMethod, p.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::NoExceptMethod, *p, 3, 2)); + + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, p.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, *p, 3, 2)); + + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, cp, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, cp.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, *cp, 3, 2)); + + EXPECT_EQ(1, base_internal::invoke(&Class::VolatileMethod, p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::VolatileMethod, p.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::VolatileMethod, *p, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::VolatileMethod, vp, 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::VolatileMethod, vp.get(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::VolatileMethod, *vp, 3, 2)); + + EXPECT_EQ(1, + base_internal::invoke(&Class::Method, make_unique<Class>(), 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, make_unique<Class>(), + 3, 2)); + EXPECT_EQ(1, base_internal::invoke(&Class::ConstMethod, + make_unique<const Class>(), 3, 2)); +} + +TEST(InvokeTest, DataMember) { + std::unique_ptr<Class> p(new Class{42}); + std::unique_ptr<const Class> cp(new Class{42}); + EXPECT_EQ(42, base_internal::invoke(&Class::member, p)); + EXPECT_EQ(42, base_internal::invoke(&Class::member, *p)); + EXPECT_EQ(42, base_internal::invoke(&Class::member, p.get())); + + base_internal::invoke(&Class::member, p) = 42; + base_internal::invoke(&Class::member, p.get()) = 42; + + EXPECT_EQ(42, base_internal::invoke(&Class::member, cp)); + EXPECT_EQ(42, base_internal::invoke(&Class::member, *cp)); + EXPECT_EQ(42, base_internal::invoke(&Class::member, cp.get())); +} + +TEST(InvokeTest, FlipFlop) { + FlipFlop obj = {42}; + // This call could resolve to (obj.*&FlipFlop::ConstMethod)() or + // ((*obj).*&FlipFlop::ConstMethod)(). We verify that it's the former. + EXPECT_EQ(42, base_internal::invoke(&FlipFlop::ConstMethod, obj)); + EXPECT_EQ(42, base_internal::invoke(&FlipFlop::member, obj)); +} + +TEST(InvokeTest, SfinaeFriendly) { + CallMaybeWithArg(NoOp); + EXPECT_THAT(CallMaybeWithArg(Factory), ::testing::Pointee(42)); +} + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/log_severity.cc b/third_party/abseil_cpp/absl/base/log_severity.cc new file mode 100644 index 000000000000..72312afd367e --- /dev/null +++ b/third_party/abseil_cpp/absl/base/log_severity.cc @@ -0,0 +1,27 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/log_severity.h" + +#include <ostream> + +namespace absl { +ABSL_NAMESPACE_BEGIN + +std::ostream& operator<<(std::ostream& os, absl::LogSeverity s) { + if (s == absl::NormalizeLogSeverity(s)) return os << absl::LogSeverityName(s); + return os << "absl::LogSeverity(" << static_cast<int>(s) << ")"; +} +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/log_severity.h b/third_party/abseil_cpp/absl/base/log_severity.h new file mode 100644 index 000000000000..045f17f81db9 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/log_severity.h @@ -0,0 +1,121 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_LOG_SEVERITY_H_ +#define ABSL_BASE_LOG_SEVERITY_H_ + +#include <array> +#include <ostream> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::LogSeverity +// +// Four severity levels are defined. Logging APIs should terminate the program +// when a message is logged at severity `kFatal`; the other levels have no +// special semantics. +// +// Values other than the four defined levels (e.g. produced by `static_cast`) +// are valid, but their semantics when passed to a function, macro, or flag +// depend on the function, macro, or flag. The usual behavior is to normalize +// such values to a defined severity level, however in some cases values other +// than the defined levels are useful for comparison. +// +// Exmaple: +// +// // Effectively disables all logging: +// SetMinLogLevel(static_cast<absl::LogSeverity>(100)); +// +// Abseil flags may be defined with type `LogSeverity`. Dependency layering +// constraints require that the `AbslParseFlag()` overload be declared and +// defined in the flags library itself rather than here. The `AbslUnparseFlag()` +// overload is defined there as well for consistency. +// +// absl::LogSeverity Flag String Representation +// +// An `absl::LogSeverity` has a string representation used for parsing +// command-line flags based on the enumerator name (e.g. `kFatal`) or +// its unprefixed name (without the `k`) in any case-insensitive form. (E.g. +// "FATAL", "fatal" or "Fatal" are all valid.) Unparsing such flags produces an +// unprefixed string representation in all caps (e.g. "FATAL") or an integer. +// +// Additionally, the parser accepts arbitrary integers (as if the type were +// `int`). +// +// Examples: +// +// --my_log_level=kInfo +// --my_log_level=INFO +// --my_log_level=info +// --my_log_level=0 +// +// Unparsing a flag produces the same result as `absl::LogSeverityName()` for +// the standard levels and a base-ten integer otherwise. +enum class LogSeverity : int { + kInfo = 0, + kWarning = 1, + kError = 2, + kFatal = 3, +}; + +// LogSeverities() +// +// Returns an iterable of all standard `absl::LogSeverity` values, ordered from +// least to most severe. +constexpr std::array<absl::LogSeverity, 4> LogSeverities() { + return {{absl::LogSeverity::kInfo, absl::LogSeverity::kWarning, + absl::LogSeverity::kError, absl::LogSeverity::kFatal}}; +} + +// LogSeverityName() +// +// Returns the all-caps string representation (e.g. "INFO") of the specified +// severity level if it is one of the standard levels and "UNKNOWN" otherwise. +constexpr const char* LogSeverityName(absl::LogSeverity s) { + return s == absl::LogSeverity::kInfo + ? "INFO" + : s == absl::LogSeverity::kWarning + ? "WARNING" + : s == absl::LogSeverity::kError + ? "ERROR" + : s == absl::LogSeverity::kFatal ? "FATAL" : "UNKNOWN"; +} + +// NormalizeLogSeverity() +// +// Values less than `kInfo` normalize to `kInfo`; values greater than `kFatal` +// normalize to `kError` (**NOT** `kFatal`). +constexpr absl::LogSeverity NormalizeLogSeverity(absl::LogSeverity s) { + return s < absl::LogSeverity::kInfo + ? absl::LogSeverity::kInfo + : s > absl::LogSeverity::kFatal ? absl::LogSeverity::kError : s; +} +constexpr absl::LogSeverity NormalizeLogSeverity(int s) { + return absl::NormalizeLogSeverity(static_cast<absl::LogSeverity>(s)); +} + +// operator<< +// +// The exact representation of a streamed `absl::LogSeverity` is deliberately +// unspecified; do not rely on it. +std::ostream& operator<<(std::ostream& os, absl::LogSeverity s); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_LOG_SEVERITY_H_ diff --git a/third_party/abseil_cpp/absl/base/log_severity_test.cc b/third_party/abseil_cpp/absl/base/log_severity_test.cc new file mode 100644 index 000000000000..2c6872b00a64 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/log_severity_test.cc @@ -0,0 +1,204 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/log_severity.h" + +#include <cstdint> +#include <ios> +#include <limits> +#include <ostream> +#include <sstream> +#include <string> +#include <tuple> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/flags/internal/flag.h" +#include "absl/flags/marshalling.h" +#include "absl/strings/str_cat.h" + +namespace { +using ::testing::Eq; +using ::testing::IsFalse; +using ::testing::IsTrue; +using ::testing::TestWithParam; +using ::testing::Values; + +std::string StreamHelper(absl::LogSeverity value) { + std::ostringstream stream; + stream << value; + return stream.str(); +} + +TEST(StreamTest, Works) { + EXPECT_THAT(StreamHelper(static_cast<absl::LogSeverity>(-100)), + Eq("absl::LogSeverity(-100)")); + EXPECT_THAT(StreamHelper(absl::LogSeverity::kInfo), Eq("INFO")); + EXPECT_THAT(StreamHelper(absl::LogSeverity::kWarning), Eq("WARNING")); + EXPECT_THAT(StreamHelper(absl::LogSeverity::kError), Eq("ERROR")); + EXPECT_THAT(StreamHelper(absl::LogSeverity::kFatal), Eq("FATAL")); + EXPECT_THAT(StreamHelper(static_cast<absl::LogSeverity>(4)), + Eq("absl::LogSeverity(4)")); +} + +static_assert( + absl::flags_internal::FlagUseOneWordStorage<absl::LogSeverity>::value, + "Flags of type absl::LogSeverity ought to be lock-free."); + +using ParseFlagFromOutOfRangeIntegerTest = TestWithParam<int64_t>; +INSTANTIATE_TEST_SUITE_P( + Instantiation, ParseFlagFromOutOfRangeIntegerTest, + Values(static_cast<int64_t>(std::numeric_limits<int>::min()) - 1, + static_cast<int64_t>(std::numeric_limits<int>::max()) + 1)); +TEST_P(ParseFlagFromOutOfRangeIntegerTest, ReturnsError) { + const std::string to_parse = absl::StrCat(GetParam()); + absl::LogSeverity value; + std::string error; + EXPECT_THAT(absl::ParseFlag(to_parse, &value, &error), IsFalse()) << value; +} + +using ParseFlagFromAlmostOutOfRangeIntegerTest = TestWithParam<int>; +INSTANTIATE_TEST_SUITE_P(Instantiation, + ParseFlagFromAlmostOutOfRangeIntegerTest, + Values(std::numeric_limits<int>::min(), + std::numeric_limits<int>::max())); +TEST_P(ParseFlagFromAlmostOutOfRangeIntegerTest, YieldsExpectedValue) { + const auto expected = static_cast<absl::LogSeverity>(GetParam()); + const std::string to_parse = absl::StrCat(GetParam()); + absl::LogSeverity value; + std::string error; + ASSERT_THAT(absl::ParseFlag(to_parse, &value, &error), IsTrue()) << error; + EXPECT_THAT(value, Eq(expected)); +} + +using ParseFlagFromIntegerMatchingEnumeratorTest = + TestWithParam<std::tuple<absl::string_view, absl::LogSeverity>>; +INSTANTIATE_TEST_SUITE_P( + Instantiation, ParseFlagFromIntegerMatchingEnumeratorTest, + Values(std::make_tuple("0", absl::LogSeverity::kInfo), + std::make_tuple(" 0", absl::LogSeverity::kInfo), + std::make_tuple("-0", absl::LogSeverity::kInfo), + std::make_tuple("+0", absl::LogSeverity::kInfo), + std::make_tuple("00", absl::LogSeverity::kInfo), + std::make_tuple("0 ", absl::LogSeverity::kInfo), + std::make_tuple("0x0", absl::LogSeverity::kInfo), + std::make_tuple("1", absl::LogSeverity::kWarning), + std::make_tuple("+1", absl::LogSeverity::kWarning), + std::make_tuple("2", absl::LogSeverity::kError), + std::make_tuple("3", absl::LogSeverity::kFatal))); +TEST_P(ParseFlagFromIntegerMatchingEnumeratorTest, YieldsExpectedValue) { + const absl::string_view to_parse = std::get<0>(GetParam()); + const absl::LogSeverity expected = std::get<1>(GetParam()); + absl::LogSeverity value; + std::string error; + ASSERT_THAT(absl::ParseFlag(to_parse, &value, &error), IsTrue()) << error; + EXPECT_THAT(value, Eq(expected)); +} + +using ParseFlagFromOtherIntegerTest = + TestWithParam<std::tuple<absl::string_view, int>>; +INSTANTIATE_TEST_SUITE_P(Instantiation, ParseFlagFromOtherIntegerTest, + Values(std::make_tuple("-1", -1), + std::make_tuple("4", 4), + std::make_tuple("010", 10), + std::make_tuple("0x10", 16))); +TEST_P(ParseFlagFromOtherIntegerTest, YieldsExpectedValue) { + const absl::string_view to_parse = std::get<0>(GetParam()); + const auto expected = static_cast<absl::LogSeverity>(std::get<1>(GetParam())); + absl::LogSeverity value; + std::string error; + ASSERT_THAT(absl::ParseFlag(to_parse, &value, &error), IsTrue()) << error; + EXPECT_THAT(value, Eq(expected)); +} + +using ParseFlagFromEnumeratorTest = + TestWithParam<std::tuple<absl::string_view, absl::LogSeverity>>; +INSTANTIATE_TEST_SUITE_P( + Instantiation, ParseFlagFromEnumeratorTest, + Values(std::make_tuple("INFO", absl::LogSeverity::kInfo), + std::make_tuple("info", absl::LogSeverity::kInfo), + std::make_tuple("kInfo", absl::LogSeverity::kInfo), + std::make_tuple("iNfO", absl::LogSeverity::kInfo), + std::make_tuple("kInFo", absl::LogSeverity::kInfo), + std::make_tuple("WARNING", absl::LogSeverity::kWarning), + std::make_tuple("warning", absl::LogSeverity::kWarning), + std::make_tuple("kWarning", absl::LogSeverity::kWarning), + std::make_tuple("WaRnInG", absl::LogSeverity::kWarning), + std::make_tuple("KwArNiNg", absl::LogSeverity::kWarning), + std::make_tuple("ERROR", absl::LogSeverity::kError), + std::make_tuple("error", absl::LogSeverity::kError), + std::make_tuple("kError", absl::LogSeverity::kError), + std::make_tuple("eRrOr", absl::LogSeverity::kError), + std::make_tuple("kErRoR", absl::LogSeverity::kError), + std::make_tuple("FATAL", absl::LogSeverity::kFatal), + std::make_tuple("fatal", absl::LogSeverity::kFatal), + std::make_tuple("kFatal", absl::LogSeverity::kFatal), + std::make_tuple("FaTaL", absl::LogSeverity::kFatal), + std::make_tuple("KfAtAl", absl::LogSeverity::kFatal))); +TEST_P(ParseFlagFromEnumeratorTest, YieldsExpectedValue) { + const absl::string_view to_parse = std::get<0>(GetParam()); + const absl::LogSeverity expected = std::get<1>(GetParam()); + absl::LogSeverity value; + std::string error; + ASSERT_THAT(absl::ParseFlag(to_parse, &value, &error), IsTrue()) << error; + EXPECT_THAT(value, Eq(expected)); +} + +using ParseFlagFromGarbageTest = TestWithParam<absl::string_view>; +INSTANTIATE_TEST_SUITE_P(Instantiation, ParseFlagFromGarbageTest, + Values("", "\0", " ", "garbage", "kkinfo", "I")); +TEST_P(ParseFlagFromGarbageTest, ReturnsError) { + const absl::string_view to_parse = GetParam(); + absl::LogSeverity value; + std::string error; + EXPECT_THAT(absl::ParseFlag(to_parse, &value, &error), IsFalse()) << value; +} + +using UnparseFlagToEnumeratorTest = + TestWithParam<std::tuple<absl::LogSeverity, absl::string_view>>; +INSTANTIATE_TEST_SUITE_P( + Instantiation, UnparseFlagToEnumeratorTest, + Values(std::make_tuple(absl::LogSeverity::kInfo, "INFO"), + std::make_tuple(absl::LogSeverity::kWarning, "WARNING"), + std::make_tuple(absl::LogSeverity::kError, "ERROR"), + std::make_tuple(absl::LogSeverity::kFatal, "FATAL"))); +TEST_P(UnparseFlagToEnumeratorTest, ReturnsExpectedValueAndRoundTrips) { + const absl::LogSeverity to_unparse = std::get<0>(GetParam()); + const absl::string_view expected = std::get<1>(GetParam()); + const std::string stringified_value = absl::UnparseFlag(to_unparse); + EXPECT_THAT(stringified_value, Eq(expected)); + absl::LogSeverity reparsed_value; + std::string error; + EXPECT_THAT(absl::ParseFlag(stringified_value, &reparsed_value, &error), + IsTrue()); + EXPECT_THAT(reparsed_value, Eq(to_unparse)); +} + +using UnparseFlagToOtherIntegerTest = TestWithParam<int>; +INSTANTIATE_TEST_SUITE_P(Instantiation, UnparseFlagToOtherIntegerTest, + Values(std::numeric_limits<int>::min(), -1, 4, + std::numeric_limits<int>::max())); +TEST_P(UnparseFlagToOtherIntegerTest, ReturnsExpectedValueAndRoundTrips) { + const absl::LogSeverity to_unparse = + static_cast<absl::LogSeverity>(GetParam()); + const std::string expected = absl::StrCat(GetParam()); + const std::string stringified_value = absl::UnparseFlag(to_unparse); + EXPECT_THAT(stringified_value, Eq(expected)); + absl::LogSeverity reparsed_value; + std::string error; + EXPECT_THAT(absl::ParseFlag(stringified_value, &reparsed_value, &error), + IsTrue()); + EXPECT_THAT(reparsed_value, Eq(to_unparse)); +} +} // namespace diff --git a/third_party/abseil_cpp/absl/base/macros.h b/third_party/abseil_cpp/absl/base/macros.h new file mode 100644 index 000000000000..3e085a916bbe --- /dev/null +++ b/third_party/abseil_cpp/absl/base/macros.h @@ -0,0 +1,158 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: macros.h +// ----------------------------------------------------------------------------- +// +// This header file defines the set of language macros used within Abseil code. +// For the set of macros used to determine supported compilers and platforms, +// see absl/base/config.h instead. +// +// This code is compiled directly on many platforms, including client +// platforms like Windows, Mac, and embedded systems. Before making +// any changes here, make sure that you're not breaking any platforms. + +#ifndef ABSL_BASE_MACROS_H_ +#define ABSL_BASE_MACROS_H_ + +#include <cassert> +#include <cstddef> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" + +// ABSL_ARRAYSIZE() +// +// Returns the number of elements in an array as a compile-time constant, which +// can be used in defining new arrays. If you use this macro on a pointer by +// mistake, you will get a compile-time error. +#define ABSL_ARRAYSIZE(array) \ + (sizeof(::absl::macros_internal::ArraySizeHelper(array))) + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace macros_internal { +// Note: this internal template function declaration is used by ABSL_ARRAYSIZE. +// The function doesn't need a definition, as we only use its type. +template <typename T, size_t N> +auto ArraySizeHelper(const T (&array)[N]) -> char (&)[N]; +} // namespace macros_internal +ABSL_NAMESPACE_END +} // namespace absl + +// ABSL_BAD_CALL_IF() +// +// Used on a function overload to trap bad calls: any call that matches the +// overload will cause a compile-time error. This macro uses a clang-specific +// "enable_if" attribute, as described at +// https://clang.llvm.org/docs/AttributeReference.html#enable-if +// +// Overloads which use this macro should be bracketed by +// `#ifdef ABSL_BAD_CALL_IF`. +// +// Example: +// +// int isdigit(int c); +// #ifdef ABSL_BAD_CALL_IF +// int isdigit(int c) +// ABSL_BAD_CALL_IF(c <= -1 || c > 255, +// "'c' must have the value of an unsigned char or EOF"); +// #endif // ABSL_BAD_CALL_IF +#if ABSL_HAVE_ATTRIBUTE(enable_if) +#define ABSL_BAD_CALL_IF(expr, msg) \ + __attribute__((enable_if(expr, "Bad call trap"), unavailable(msg))) +#endif + +// ABSL_ASSERT() +// +// In C++11, `assert` can't be used portably within constexpr functions. +// ABSL_ASSERT functions as a runtime assert but works in C++11 constexpr +// functions. Example: +// +// constexpr double Divide(double a, double b) { +// return ABSL_ASSERT(b != 0), a / b; +// } +// +// This macro is inspired by +// https://akrzemi1.wordpress.com/2017/05/18/asserts-in-constexpr-functions/ +#if defined(NDEBUG) +#define ABSL_ASSERT(expr) \ + (false ? static_cast<void>(expr) : static_cast<void>(0)) +#else +#define ABSL_ASSERT(expr) \ + (ABSL_PREDICT_TRUE((expr)) ? static_cast<void>(0) \ + : [] { assert(false && #expr); }()) // NOLINT +#endif + +// `ABSL_INTERNAL_HARDENING_ABORT()` controls how `ABSL_HARDENING_ASSERT()` +// aborts the program in release mode (when NDEBUG is defined). The +// implementation should abort the program as quickly as possible and ideally it +// should not be possible to ignore the abort request. +#if (ABSL_HAVE_BUILTIN(__builtin_trap) && \ + ABSL_HAVE_BUILTIN(__builtin_unreachable)) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_INTERNAL_HARDENING_ABORT() \ + do { \ + __builtin_trap(); \ + __builtin_unreachable(); \ + } while (false) +#else +#define ABSL_INTERNAL_HARDENING_ABORT() abort() +#endif + +// ABSL_HARDENING_ASSERT() +// +// `ABSL_HARDENING_ASSERT()` is like `ABSL_ASSERT()`, but used to implement +// runtime assertions that should be enabled in hardened builds even when +// `NDEBUG` is defined. +// +// When `NDEBUG` is not defined, `ABSL_HARDENING_ASSERT()` is identical to +// `ABSL_ASSERT()`. +// +// See `ABSL_OPTION_HARDENED` in `absl/base/options.h` for more information on +// hardened mode. +#if ABSL_OPTION_HARDENED == 1 && defined(NDEBUG) +#define ABSL_HARDENING_ASSERT(expr) \ + (ABSL_PREDICT_TRUE((expr)) ? static_cast<void>(0) \ + : [] { ABSL_INTERNAL_HARDENING_ABORT(); }()) +#else +#define ABSL_HARDENING_ASSERT(expr) ABSL_ASSERT(expr) +#endif + +#ifdef ABSL_HAVE_EXCEPTIONS +#define ABSL_INTERNAL_TRY try +#define ABSL_INTERNAL_CATCH_ANY catch (...) +#define ABSL_INTERNAL_RETHROW do { throw; } while (false) +#else // ABSL_HAVE_EXCEPTIONS +#define ABSL_INTERNAL_TRY if (true) +#define ABSL_INTERNAL_CATCH_ANY else if (false) +#define ABSL_INTERNAL_RETHROW do {} while (false) +#endif // ABSL_HAVE_EXCEPTIONS + +// `ABSL_INTERNAL_UNREACHABLE` is an unreachable statement. A program which +// reaches one has undefined behavior, and the compiler may optimize +// accordingly. +#if defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable) +#define ABSL_INTERNAL_UNREACHABLE __builtin_unreachable() +#elif defined(_MSC_VER) +#define ABSL_INTERNAL_UNREACHABLE __assume(0) +#else +#define ABSL_INTERNAL_UNREACHABLE +#endif + +#endif // ABSL_BASE_MACROS_H_ diff --git a/third_party/abseil_cpp/absl/base/optimization.h b/third_party/abseil_cpp/absl/base/optimization.h new file mode 100644 index 000000000000..393fc3a423ec --- /dev/null +++ b/third_party/abseil_cpp/absl/base/optimization.h @@ -0,0 +1,241 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: optimization.h +// ----------------------------------------------------------------------------- +// +// This header file defines portable macros for performance optimization. + +#ifndef ABSL_BASE_OPTIMIZATION_H_ +#define ABSL_BASE_OPTIMIZATION_H_ + +#include "absl/base/config.h" + +// ABSL_BLOCK_TAIL_CALL_OPTIMIZATION +// +// Instructs the compiler to avoid optimizing tail-call recursion. Use of this +// macro is useful when you wish to preserve the existing function order within +// a stack trace for logging, debugging, or profiling purposes. +// +// Example: +// +// int f() { +// int result = g(); +// ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); +// return result; +// } +#if defined(__pnacl__) +#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } +#elif defined(__clang__) +// Clang will not tail call given inline volatile assembly. +#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") +#elif defined(__GNUC__) +// GCC will not tail call given inline volatile assembly. +#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("") +#elif defined(_MSC_VER) +#include <intrin.h> +// The __nop() intrinsic blocks the optimisation. +#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __nop() +#else +#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; } +#endif + +// ABSL_CACHELINE_SIZE +// +// Explicitly defines the size of the L1 cache for purposes of alignment. +// Setting the cacheline size allows you to specify that certain objects be +// aligned on a cacheline boundary with `ABSL_CACHELINE_ALIGNED` declarations. +// (See below.) +// +// NOTE: this macro should be replaced with the following C++17 features, when +// those are generally available: +// +// * `std::hardware_constructive_interference_size` +// * `std::hardware_destructive_interference_size` +// +// See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html +// for more information. +#if defined(__GNUC__) +// Cache line alignment +#if defined(__i386__) || defined(__x86_64__) +#define ABSL_CACHELINE_SIZE 64 +#elif defined(__powerpc64__) +#define ABSL_CACHELINE_SIZE 128 +#elif defined(__aarch64__) +// We would need to read special register ctr_el0 to find out L1 dcache size. +// This value is a good estimate based on a real aarch64 machine. +#define ABSL_CACHELINE_SIZE 64 +#elif defined(__arm__) +// Cache line sizes for ARM: These values are not strictly correct since +// cache line sizes depend on implementations, not architectures. There +// are even implementations with cache line sizes configurable at boot +// time. +#if defined(__ARM_ARCH_5T__) +#define ABSL_CACHELINE_SIZE 32 +#elif defined(__ARM_ARCH_7A__) +#define ABSL_CACHELINE_SIZE 64 +#endif +#endif + +#ifndef ABSL_CACHELINE_SIZE +// A reasonable default guess. Note that overestimates tend to waste more +// space, while underestimates tend to waste more time. +#define ABSL_CACHELINE_SIZE 64 +#endif + +// ABSL_CACHELINE_ALIGNED +// +// Indicates that the declared object be cache aligned using +// `ABSL_CACHELINE_SIZE` (see above). Cacheline aligning objects allows you to +// load a set of related objects in the L1 cache for performance improvements. +// Cacheline aligning objects properly allows constructive memory sharing and +// prevents destructive (or "false") memory sharing. +// +// NOTE: this macro should be replaced with usage of `alignas()` using +// `std::hardware_constructive_interference_size` and/or +// `std::hardware_destructive_interference_size` when available within C++17. +// +// See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html +// for more information. +// +// On some compilers, `ABSL_CACHELINE_ALIGNED` expands to an `__attribute__` +// or `__declspec` attribute. For compilers where this is not known to work, +// the macro expands to nothing. +// +// No further guarantees are made here. The result of applying the macro +// to variables and types is always implementation-defined. +// +// WARNING: It is easy to use this attribute incorrectly, even to the point +// of causing bugs that are difficult to diagnose, crash, etc. It does not +// of itself guarantee that objects are aligned to a cache line. +// +// NOTE: Some compilers are picky about the locations of annotations such as +// this attribute, so prefer to put it at the beginning of your declaration. +// For example, +// +// ABSL_CACHELINE_ALIGNED static Foo* foo = ... +// +// class ABSL_CACHELINE_ALIGNED Bar { ... +// +// Recommendations: +// +// 1) Consult compiler documentation; this comment is not kept in sync as +// toolchains evolve. +// 2) Verify your use has the intended effect. This often requires inspecting +// the generated machine code. +// 3) Prefer applying this attribute to individual variables. Avoid +// applying it to types. This tends to localize the effect. +#define ABSL_CACHELINE_ALIGNED __attribute__((aligned(ABSL_CACHELINE_SIZE))) +#elif defined(_MSC_VER) +#define ABSL_CACHELINE_SIZE 64 +#define ABSL_CACHELINE_ALIGNED __declspec(align(ABSL_CACHELINE_SIZE)) +#else +#define ABSL_CACHELINE_SIZE 64 +#define ABSL_CACHELINE_ALIGNED +#endif + +// ABSL_PREDICT_TRUE, ABSL_PREDICT_FALSE +// +// Enables the compiler to prioritize compilation using static analysis for +// likely paths within a boolean branch. +// +// Example: +// +// if (ABSL_PREDICT_TRUE(expression)) { +// return result; // Faster if more likely +// } else { +// return 0; +// } +// +// Compilers can use the information that a certain branch is not likely to be +// taken (for instance, a CHECK failure) to optimize for the common case in +// the absence of better information (ie. compiling gcc with `-fprofile-arcs`). +// +// Recommendation: Modern CPUs dynamically predict branch execution paths, +// typically with accuracy greater than 97%. As a result, annotating every +// branch in a codebase is likely counterproductive; however, annotating +// specific branches that are both hot and consistently mispredicted is likely +// to yield performance improvements. +#if ABSL_HAVE_BUILTIN(__builtin_expect) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_PREDICT_FALSE(x) (__builtin_expect(false || (x), false)) +#define ABSL_PREDICT_TRUE(x) (__builtin_expect(false || (x), true)) +#else +#define ABSL_PREDICT_FALSE(x) (x) +#define ABSL_PREDICT_TRUE(x) (x) +#endif + +// ABSL_INTERNAL_ASSUME(cond) +// Informs the compiler that a condition is always true and that it can assume +// it to be true for optimization purposes. The call has undefined behavior if +// the condition is false. +// In !NDEBUG mode, the condition is checked with an assert(). +// NOTE: The expression must not have side effects, as it will only be evaluated +// in some compilation modes and not others. +// +// Example: +// +// int x = ...; +// ABSL_INTERNAL_ASSUME(x >= 0); +// // The compiler can optimize the division to a simple right shift using the +// // assumption specified above. +// int y = x / 16; +// +#if !defined(NDEBUG) +#define ABSL_INTERNAL_ASSUME(cond) assert(cond) +#elif ABSL_HAVE_BUILTIN(__builtin_assume) +#define ABSL_INTERNAL_ASSUME(cond) __builtin_assume(cond) +#elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable) +#define ABSL_INTERNAL_ASSUME(cond) \ + do { \ + if (!(cond)) __builtin_unreachable(); \ + } while (0) +#elif defined(_MSC_VER) +#define ABSL_INTERNAL_ASSUME(cond) __assume(cond) +#else +#define ABSL_INTERNAL_ASSUME(cond) \ + do { \ + static_cast<void>(false && (cond)); \ + } while (0) +#endif + +// ABSL_INTERNAL_UNIQUE_SMALL_NAME(cond) +// This macro forces small unique name on a static file level symbols like +// static local variables or static functions. This is intended to be used in +// macro definitions to optimize the cost of generated code. Do NOT use it on +// symbols exported from translation unit since it may casue a link time +// conflict. +// +// Example: +// +// #define MY_MACRO(txt) +// namespace { +// char VeryVeryLongVarName[] ABSL_INTERNAL_UNIQUE_SMALL_NAME() = txt; +// const char* VeryVeryLongFuncName() ABSL_INTERNAL_UNIQUE_SMALL_NAME(); +// const char* VeryVeryLongFuncName() { return txt; } +// } +// + +#if defined(__GNUC__) +#define ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x) #x +#define ABSL_INTERNAL_UNIQUE_SMALL_NAME1(x) ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x) +#define ABSL_INTERNAL_UNIQUE_SMALL_NAME() \ + asm(ABSL_INTERNAL_UNIQUE_SMALL_NAME1(.absl.__COUNTER__)) +#else +#define ABSL_INTERNAL_UNIQUE_SMALL_NAME() +#endif + +#endif // ABSL_BASE_OPTIMIZATION_H_ diff --git a/third_party/abseil_cpp/absl/base/optimization_test.cc b/third_party/abseil_cpp/absl/base/optimization_test.cc new file mode 100644 index 000000000000..e83369f32261 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/optimization_test.cc @@ -0,0 +1,129 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/optimization.h" + +#include "gtest/gtest.h" +#include "absl/types/optional.h" + +namespace { + +// Tests for the ABSL_PREDICT_TRUE and ABSL_PREDICT_FALSE macros. +// The tests only verify that the macros are functionally correct - i.e. code +// behaves as if they weren't used. They don't try to check their impact on +// optimization. + +TEST(PredictTest, PredictTrue) { + EXPECT_TRUE(ABSL_PREDICT_TRUE(true)); + EXPECT_FALSE(ABSL_PREDICT_TRUE(false)); + EXPECT_TRUE(ABSL_PREDICT_TRUE(1 == 1)); + EXPECT_FALSE(ABSL_PREDICT_TRUE(1 == 2)); + + if (ABSL_PREDICT_TRUE(false)) ADD_FAILURE(); + if (!ABSL_PREDICT_TRUE(true)) ADD_FAILURE(); + + EXPECT_TRUE(ABSL_PREDICT_TRUE(true) && true); + EXPECT_TRUE(ABSL_PREDICT_TRUE(true) || false); +} + +TEST(PredictTest, PredictFalse) { + EXPECT_TRUE(ABSL_PREDICT_FALSE(true)); + EXPECT_FALSE(ABSL_PREDICT_FALSE(false)); + EXPECT_TRUE(ABSL_PREDICT_FALSE(1 == 1)); + EXPECT_FALSE(ABSL_PREDICT_FALSE(1 == 2)); + + if (ABSL_PREDICT_FALSE(false)) ADD_FAILURE(); + if (!ABSL_PREDICT_FALSE(true)) ADD_FAILURE(); + + EXPECT_TRUE(ABSL_PREDICT_FALSE(true) && true); + EXPECT_TRUE(ABSL_PREDICT_FALSE(true) || false); +} + +TEST(PredictTest, OneEvaluation) { + // Verify that the expression is only evaluated once. + int x = 0; + if (ABSL_PREDICT_TRUE((++x) == 0)) ADD_FAILURE(); + EXPECT_EQ(x, 1); + if (ABSL_PREDICT_FALSE((++x) == 0)) ADD_FAILURE(); + EXPECT_EQ(x, 2); +} + +TEST(PredictTest, OperatorOrder) { + // Verify that operator order inside and outside the macro behaves well. + // These would fail for a naive '#define ABSL_PREDICT_TRUE(x) x' + EXPECT_TRUE(ABSL_PREDICT_TRUE(1 && 2) == true); + EXPECT_TRUE(ABSL_PREDICT_FALSE(1 && 2) == true); + EXPECT_TRUE(!ABSL_PREDICT_TRUE(1 == 2)); + EXPECT_TRUE(!ABSL_PREDICT_FALSE(1 == 2)); +} + +TEST(PredictTest, Pointer) { + const int x = 3; + const int *good_intptr = &x; + const int *null_intptr = nullptr; + EXPECT_TRUE(ABSL_PREDICT_TRUE(good_intptr)); + EXPECT_FALSE(ABSL_PREDICT_TRUE(null_intptr)); + EXPECT_TRUE(ABSL_PREDICT_FALSE(good_intptr)); + EXPECT_FALSE(ABSL_PREDICT_FALSE(null_intptr)); +} + +TEST(PredictTest, Optional) { + // Note: An optional's truth value is the value's existence, not its truth. + absl::optional<bool> has_value(false); + absl::optional<bool> no_value; + EXPECT_TRUE(ABSL_PREDICT_TRUE(has_value)); + EXPECT_FALSE(ABSL_PREDICT_TRUE(no_value)); + EXPECT_TRUE(ABSL_PREDICT_FALSE(has_value)); + EXPECT_FALSE(ABSL_PREDICT_FALSE(no_value)); +} + +class ImplictlyConvertibleToBool { + public: + explicit ImplictlyConvertibleToBool(bool value) : value_(value) {} + operator bool() const { // NOLINT(google-explicit-constructor) + return value_; + } + + private: + bool value_; +}; + +TEST(PredictTest, ImplicitBoolConversion) { + const ImplictlyConvertibleToBool is_true(true); + const ImplictlyConvertibleToBool is_false(false); + if (!ABSL_PREDICT_TRUE(is_true)) ADD_FAILURE(); + if (ABSL_PREDICT_TRUE(is_false)) ADD_FAILURE(); + if (!ABSL_PREDICT_FALSE(is_true)) ADD_FAILURE(); + if (ABSL_PREDICT_FALSE(is_false)) ADD_FAILURE(); +} + +class ExplictlyConvertibleToBool { + public: + explicit ExplictlyConvertibleToBool(bool value) : value_(value) {} + explicit operator bool() const { return value_; } + + private: + bool value_; +}; + +TEST(PredictTest, ExplicitBoolConversion) { + const ExplictlyConvertibleToBool is_true(true); + const ExplictlyConvertibleToBool is_false(false); + if (!ABSL_PREDICT_TRUE(is_true)) ADD_FAILURE(); + if (ABSL_PREDICT_TRUE(is_false)) ADD_FAILURE(); + if (!ABSL_PREDICT_FALSE(is_true)) ADD_FAILURE(); + if (ABSL_PREDICT_FALSE(is_false)) ADD_FAILURE(); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/options.h b/third_party/abseil_cpp/absl/base/options.h new file mode 100644 index 000000000000..230bf1eecc4a --- /dev/null +++ b/third_party/abseil_cpp/absl/base/options.h @@ -0,0 +1,238 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: options.h +// ----------------------------------------------------------------------------- +// +// This file contains Abseil configuration options for setting specific +// implementations instead of letting Abseil determine which implementation to +// use at compile-time. Setting these options may be useful for package or build +// managers who wish to guarantee ABI stability within binary builds (which are +// otherwise difficult to enforce). +// +// *** IMPORTANT NOTICE FOR PACKAGE MANAGERS: It is important that +// maintainers of package managers who wish to package Abseil read and +// understand this file! *** +// +// Abseil contains a number of possible configuration endpoints, based on +// parameters such as the detected platform, language version, or command-line +// flags used to invoke the underlying binary. As is the case with all +// libraries, binaries which contain Abseil code must ensure that separate +// packages use the same compiled copy of Abseil to avoid a diamond dependency +// problem, which can occur if two packages built with different Abseil +// configuration settings are linked together. Diamond dependency problems in +// C++ may manifest as violations to the One Definition Rule (ODR) (resulting in +// linker errors), or undefined behavior (resulting in crashes). +// +// Diamond dependency problems can be avoided if all packages utilize the same +// exact version of Abseil. Building from source code with the same compilation +// parameters is the easiest way to avoid such dependency problems. However, for +// package managers who cannot control such compilation parameters, we are +// providing the file to allow you to inject ABI (Application Binary Interface) +// stability across builds. Settings options in this file will neither change +// API nor ABI, providing a stable copy of Abseil between packages. +// +// Care must be taken to keep options within these configurations isolated +// from any other dynamic settings, such as command-line flags which could alter +// these options. This file is provided specifically to help build and package +// managers provide a stable copy of Abseil within their libraries and binaries; +// other developers should not have need to alter the contents of this file. +// +// ----------------------------------------------------------------------------- +// Usage +// ----------------------------------------------------------------------------- +// +// For any particular package release, set the appropriate definitions within +// this file to whatever value makes the most sense for your package(s). Note +// that, by default, most of these options, at the moment, affect the +// implementation of types; future options may affect other implementation +// details. +// +// NOTE: the defaults within this file all assume that Abseil can select the +// proper Abseil implementation at compile-time, which will not be sufficient +// to guarantee ABI stability to package managers. + +#ifndef ABSL_BASE_OPTIONS_H_ +#define ABSL_BASE_OPTIONS_H_ + +// Include a standard library header to allow configuration based on the +// standard library in use. +#ifdef __cplusplus +#include <ciso646> +#endif + +// ----------------------------------------------------------------------------- +// Type Compatibility Options +// ----------------------------------------------------------------------------- +// +// ABSL_OPTION_USE_STD_ANY +// +// This option controls whether absl::any is implemented as an alias to +// std::any, or as an independent implementation. +// +// A value of 0 means to use Abseil's implementation. This requires only C++11 +// support, and is expected to work on every toolchain we support. +// +// A value of 1 means to use an alias to std::any. This requires that all code +// using Abseil is built in C++17 mode or later. +// +// A value of 2 means to detect the C++ version being used to compile Abseil, +// and use an alias only if a working std::any is available. This option is +// useful when you are building your entire program, including all of its +// dependencies, from source. It should not be used otherwise -- for example, +// if you are distributing Abseil in a binary package manager -- since in +// mode 2, absl::any will name a different type, with a different mangled name +// and binary layout, depending on the compiler flags passed by the end user. +// For more info, see https://abseil.io/about/design/dropin-types. +// +// User code should not inspect this macro. To check in the preprocessor if +// absl::any is a typedef of std::any, use the feature macro ABSL_USES_STD_ANY. + +#define ABSL_OPTION_USE_STD_ANY 2 + + +// ABSL_OPTION_USE_STD_OPTIONAL +// +// This option controls whether absl::optional is implemented as an alias to +// std::optional, or as an independent implementation. +// +// A value of 0 means to use Abseil's implementation. This requires only C++11 +// support, and is expected to work on every toolchain we support. +// +// A value of 1 means to use an alias to std::optional. This requires that all +// code using Abseil is built in C++17 mode or later. +// +// A value of 2 means to detect the C++ version being used to compile Abseil, +// and use an alias only if a working std::optional is available. This option +// is useful when you are building your program from source. It should not be +// used otherwise -- for example, if you are distributing Abseil in a binary +// package manager -- since in mode 2, absl::optional will name a different +// type, with a different mangled name and binary layout, depending on the +// compiler flags passed by the end user. For more info, see +// https://abseil.io/about/design/dropin-types. + +// User code should not inspect this macro. To check in the preprocessor if +// absl::optional is a typedef of std::optional, use the feature macro +// ABSL_USES_STD_OPTIONAL. + +#define ABSL_OPTION_USE_STD_OPTIONAL 2 + + +// ABSL_OPTION_USE_STD_STRING_VIEW +// +// This option controls whether absl::string_view is implemented as an alias to +// std::string_view, or as an independent implementation. +// +// A value of 0 means to use Abseil's implementation. This requires only C++11 +// support, and is expected to work on every toolchain we support. +// +// A value of 1 means to use an alias to std::string_view. This requires that +// all code using Abseil is built in C++17 mode or later. +// +// A value of 2 means to detect the C++ version being used to compile Abseil, +// and use an alias only if a working std::string_view is available. This +// option is useful when you are building your program from source. It should +// not be used otherwise -- for example, if you are distributing Abseil in a +// binary package manager -- since in mode 2, absl::string_view will name a +// different type, with a different mangled name and binary layout, depending on +// the compiler flags passed by the end user. For more info, see +// https://abseil.io/about/design/dropin-types. +// +// User code should not inspect this macro. To check in the preprocessor if +// absl::string_view is a typedef of std::string_view, use the feature macro +// ABSL_USES_STD_STRING_VIEW. + +#define ABSL_OPTION_USE_STD_STRING_VIEW 2 + +// ABSL_OPTION_USE_STD_VARIANT +// +// This option controls whether absl::variant is implemented as an alias to +// std::variant, or as an independent implementation. +// +// A value of 0 means to use Abseil's implementation. This requires only C++11 +// support, and is expected to work on every toolchain we support. +// +// A value of 1 means to use an alias to std::variant. This requires that all +// code using Abseil is built in C++17 mode or later. +// +// A value of 2 means to detect the C++ version being used to compile Abseil, +// and use an alias only if a working std::variant is available. This option +// is useful when you are building your program from source. It should not be +// used otherwise -- for example, if you are distributing Abseil in a binary +// package manager -- since in mode 2, absl::variant will name a different +// type, with a different mangled name and binary layout, depending on the +// compiler flags passed by the end user. For more info, see +// https://abseil.io/about/design/dropin-types. +// +// User code should not inspect this macro. To check in the preprocessor if +// absl::variant is a typedef of std::variant, use the feature macro +// ABSL_USES_STD_VARIANT. + +#define ABSL_OPTION_USE_STD_VARIANT 2 + + +// ABSL_OPTION_USE_INLINE_NAMESPACE +// ABSL_OPTION_INLINE_NAMESPACE_NAME +// +// These options controls whether all entities in the absl namespace are +// contained within an inner inline namespace. This does not affect the +// user-visible API of Abseil, but it changes the mangled names of all symbols. +// +// This can be useful as a version tag if you are distributing Abseil in +// precompiled form. This will prevent a binary library build of Abseil with +// one inline namespace being used with headers configured with a different +// inline namespace name. Binary packagers are reminded that Abseil does not +// guarantee any ABI stability in Abseil, so any update of Abseil or +// configuration change in such a binary package should be combined with a +// new, unique value for the inline namespace name. +// +// A value of 0 means not to use inline namespaces. +// +// A value of 1 means to use an inline namespace with the given name inside +// namespace absl. If this is set, ABSL_OPTION_INLINE_NAMESPACE_NAME must also +// be changed to a new, unique identifier name. In particular "head" is not +// allowed. + +#define ABSL_OPTION_USE_INLINE_NAMESPACE 0 +#define ABSL_OPTION_INLINE_NAMESPACE_NAME head + +// ABSL_OPTION_HARDENED +// +// This option enables a "hardened" build in release mode (in this context, +// release mode is defined as a build where the `NDEBUG` macro is defined). +// +// A value of 0 means that "hardened" mode is not enabled. +// +// A value of 1 means that "hardened" mode is enabled. +// +// Hardened builds have additional security checks enabled when `NDEBUG` is +// defined. Defining `NDEBUG` is normally used to turn `assert()` macro into a +// no-op, as well as disabling other bespoke program consistency checks. By +// defining ABSL_OPTION_HARDENED to 1, a select set of checks remain enabled in +// release mode. These checks guard against programming errors that may lead to +// security vulnerabilities. In release mode, when one of these programming +// errors is encountered, the program will immediately abort, possibly without +// any attempt at logging. +// +// The checks enabled by this option are not free; they do incur runtime cost. +// +// The checks enabled by this option are always active when `NDEBUG` is not +// defined, even in the case when ABSL_OPTION_HARDENED is defined to 0. The +// checks enabled by this option may abort the program in a different way and +// log additional information when `NDEBUG` is not defined. + +#define ABSL_OPTION_HARDENED 0 + +#endif // ABSL_BASE_OPTIONS_H_ diff --git a/third_party/abseil_cpp/absl/base/policy_checks.h b/third_party/abseil_cpp/absl/base/policy_checks.h new file mode 100644 index 000000000000..06b324391685 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/policy_checks.h @@ -0,0 +1,111 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: policy_checks.h +// ----------------------------------------------------------------------------- +// +// This header enforces a minimum set of policies at build time, such as the +// supported compiler and library versions. Unsupported configurations are +// reported with `#error`. This enforcement is best effort, so successfully +// compiling this header does not guarantee a supported configuration. + +#ifndef ABSL_BASE_POLICY_CHECKS_H_ +#define ABSL_BASE_POLICY_CHECKS_H_ + +// Included for the __GLIBC_PREREQ macro used below. +#include <limits.h> + +// Included for the _STLPORT_VERSION macro used below. +#if defined(__cplusplus) +#include <cstddef> +#endif + +// ----------------------------------------------------------------------------- +// Operating System Check +// ----------------------------------------------------------------------------- + +#if defined(__CYGWIN__) +#error "Cygwin is not supported." +#endif + +// ----------------------------------------------------------------------------- +// Toolchain Check +// ----------------------------------------------------------------------------- + +// We support MSVC++ 14.0 update 2 and later. +// This minimum will go up. +#if defined(_MSC_FULL_VER) && _MSC_FULL_VER < 190023918 && !defined(__clang__) +#error "This package requires Visual Studio 2015 Update 2 or higher." +#endif + +// We support gcc 4.7 and later. +// This minimum will go up. +#if defined(__GNUC__) && !defined(__clang__) +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#error "This package requires gcc 4.7 or higher." +#endif +#endif + +// We support Apple Xcode clang 4.2.1 (version 421.11.65) and later. +// This corresponds to Apple Xcode version 4.5. +// This minimum will go up. +#if defined(__apple_build_version__) && __apple_build_version__ < 4211165 +#error "This package requires __apple_build_version__ of 4211165 or higher." +#endif + +// ----------------------------------------------------------------------------- +// C++ Version Check +// ----------------------------------------------------------------------------- + +// Enforce C++11 as the minimum. Note that Visual Studio has not +// advanced __cplusplus despite being good enough for our purposes, so +// so we exempt it from the check. +#if defined(__cplusplus) && !defined(_MSC_VER) +#if __cplusplus < 201103L +#error "C++ versions less than C++11 are not supported." +#endif +#endif + +// ----------------------------------------------------------------------------- +// Standard Library Check +// ----------------------------------------------------------------------------- + +#if defined(_STLPORT_VERSION) +#error "STLPort is not supported." +#endif + +// ----------------------------------------------------------------------------- +// `char` Size Check +// ----------------------------------------------------------------------------- + +// Abseil currently assumes CHAR_BIT == 8. If you would like to use Abseil on a +// platform where this is not the case, please provide us with the details about +// your platform so we can consider relaxing this requirement. +#if CHAR_BIT != 8 +#error "Abseil assumes CHAR_BIT == 8." +#endif + +// ----------------------------------------------------------------------------- +// `int` Size Check +// ----------------------------------------------------------------------------- + +// Abseil currently assumes that an int is 4 bytes. If you would like to use +// Abseil on a platform where this is not the case, please provide us with the +// details about your platform so we can consider relaxing this requirement. +#if INT_MAX < 2147483647 +#error "Abseil assumes that int is at least 4 bytes. " +#endif + +#endif // ABSL_BASE_POLICY_CHECKS_H_ diff --git a/third_party/abseil_cpp/absl/base/port.h b/third_party/abseil_cpp/absl/base/port.h new file mode 100644 index 000000000000..6c28068d4f40 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/port.h @@ -0,0 +1,26 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This files is a forwarding header for other headers containing various +// portability macros and functions. +// This file is used for both C and C++! + +#ifndef ABSL_BASE_PORT_H_ +#define ABSL_BASE_PORT_H_ + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" + +#endif // ABSL_BASE_PORT_H_ diff --git a/third_party/abseil_cpp/absl/base/raw_logging_test.cc b/third_party/abseil_cpp/absl/base/raw_logging_test.cc new file mode 100644 index 000000000000..3d30bd386115 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/raw_logging_test.cc @@ -0,0 +1,79 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This test serves primarily as a compilation test for base/raw_logging.h. +// Raw logging testing is covered by logging_unittest.cc, which is not as +// portable as this test. + +#include "absl/base/internal/raw_logging.h" + +#include <tuple> + +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" + +namespace { + +TEST(RawLoggingCompilationTest, Log) { + ABSL_RAW_LOG(INFO, "RAW INFO: %d", 1); + ABSL_RAW_LOG(INFO, "RAW INFO: %d %d", 1, 2); + ABSL_RAW_LOG(INFO, "RAW INFO: %d %d %d", 1, 2, 3); + ABSL_RAW_LOG(INFO, "RAW INFO: %d %d %d %d", 1, 2, 3, 4); + ABSL_RAW_LOG(INFO, "RAW INFO: %d %d %d %d %d", 1, 2, 3, 4, 5); + ABSL_RAW_LOG(WARNING, "RAW WARNING: %d", 1); + ABSL_RAW_LOG(ERROR, "RAW ERROR: %d", 1); +} + +TEST(RawLoggingCompilationTest, PassingCheck) { + ABSL_RAW_CHECK(true, "RAW CHECK"); +} + +// Not all platforms support output from raw log, so we don't verify any +// particular output for RAW check failures (expecting the empty string +// accomplishes this). This test is primarily a compilation test, but we +// are verifying process death when EXPECT_DEATH works for a platform. +const char kExpectedDeathOutput[] = ""; + +TEST(RawLoggingDeathTest, FailingCheck) { + EXPECT_DEATH_IF_SUPPORTED(ABSL_RAW_CHECK(1 == 0, "explanation"), + kExpectedDeathOutput); +} + +TEST(RawLoggingDeathTest, LogFatal) { + EXPECT_DEATH_IF_SUPPORTED(ABSL_RAW_LOG(FATAL, "my dog has fleas"), + kExpectedDeathOutput); +} + +TEST(InternalLog, CompilationTest) { + ABSL_INTERNAL_LOG(INFO, "Internal Log"); + std::string log_msg = "Internal Log"; + ABSL_INTERNAL_LOG(INFO, log_msg); + + ABSL_INTERNAL_LOG(INFO, log_msg + " 2"); + + float d = 1.1f; + ABSL_INTERNAL_LOG(INFO, absl::StrCat("Internal log ", 3, " + ", d)); +} + +TEST(InternalLogDeathTest, FailingCheck) { + EXPECT_DEATH_IF_SUPPORTED(ABSL_INTERNAL_CHECK(1 == 0, "explanation"), + kExpectedDeathOutput); +} + +TEST(InternalLogDeathTest, LogFatal) { + EXPECT_DEATH_IF_SUPPORTED(ABSL_INTERNAL_LOG(FATAL, "my dog has fleas"), + kExpectedDeathOutput); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/base/spinlock_test_common.cc b/third_party/abseil_cpp/absl/base/spinlock_test_common.cc new file mode 100644 index 000000000000..dee266e4f4c0 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/spinlock_test_common.cc @@ -0,0 +1,271 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A bunch of threads repeatedly hash an array of ints protected by a +// spinlock. If the spinlock is working properly, all elements of the +// array should be equal at the end of the test. + +#include <cstdint> +#include <limits> +#include <random> +#include <thread> // NOLINT(build/c++11) +#include <type_traits> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/low_level_scheduling.h" +#include "absl/base/internal/scheduling_mode.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/internal/sysinfo.h" +#include "absl/base/macros.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/notification.h" + +constexpr int32_t kNumThreads = 10; +constexpr int32_t kIters = 1000; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// This is defined outside of anonymous namespace so that it can be +// a friend of SpinLock to access protected methods for testing. +struct SpinLockTest { + static uint32_t EncodeWaitCycles(int64_t wait_start_time, + int64_t wait_end_time) { + return SpinLock::EncodeWaitCycles(wait_start_time, wait_end_time); + } + static uint64_t DecodeWaitCycles(uint32_t lock_value) { + return SpinLock::DecodeWaitCycles(lock_value); + } +}; + +namespace { + +static constexpr int kArrayLength = 10; +static uint32_t values[kArrayLength]; + +ABSL_CONST_INIT static SpinLock static_cooperative_spinlock( + absl::kConstInit, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL); +ABSL_CONST_INIT static SpinLock static_noncooperative_spinlock( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); + +// Simple integer hash function based on the public domain lookup2 hash. +// http://burtleburtle.net/bob/c/lookup2.c +static uint32_t Hash32(uint32_t a, uint32_t c) { + uint32_t b = 0x9e3779b9UL; // The golden ratio; an arbitrary value. + a -= b; a -= c; a ^= (c >> 13); + b -= c; b -= a; b ^= (a << 8); + c -= a; c -= b; c ^= (b >> 13); + a -= b; a -= c; a ^= (c >> 12); + b -= c; b -= a; b ^= (a << 16); + c -= a; c -= b; c ^= (b >> 5); + a -= b; a -= c; a ^= (c >> 3); + b -= c; b -= a; b ^= (a << 10); + c -= a; c -= b; c ^= (b >> 15); + return c; +} + +static void TestFunction(int thread_salt, SpinLock* spinlock) { + for (int i = 0; i < kIters; i++) { + SpinLockHolder h(spinlock); + for (int j = 0; j < kArrayLength; j++) { + const int index = (j + thread_salt) % kArrayLength; + values[index] = Hash32(values[index], thread_salt); + std::this_thread::yield(); + } + } +} + +static void ThreadedTest(SpinLock* spinlock) { + std::vector<std::thread> threads; + for (int i = 0; i < kNumThreads; ++i) { + threads.push_back(std::thread(TestFunction, i, spinlock)); + } + for (auto& thread : threads) { + thread.join(); + } + + SpinLockHolder h(spinlock); + for (int i = 1; i < kArrayLength; i++) { + EXPECT_EQ(values[0], values[i]); + } +} + +#ifndef ABSL_HAVE_THREAD_SANITIZER +static_assert(std::is_trivially_destructible<SpinLock>(), ""); +#endif + +TEST(SpinLock, StackNonCooperativeDisablesScheduling) { + SpinLock spinlock(base_internal::SCHEDULE_KERNEL_ONLY); + spinlock.Lock(); + EXPECT_FALSE(base_internal::SchedulingGuard::ReschedulingIsAllowed()); + spinlock.Unlock(); +} + +TEST(SpinLock, StaticNonCooperativeDisablesScheduling) { + static_noncooperative_spinlock.Lock(); + EXPECT_FALSE(base_internal::SchedulingGuard::ReschedulingIsAllowed()); + static_noncooperative_spinlock.Unlock(); +} + +TEST(SpinLock, WaitCyclesEncoding) { + // These are implementation details not exported by SpinLock. + const int kProfileTimestampShift = 7; + const int kLockwordReservedShift = 3; + const uint32_t kSpinLockSleeper = 8; + + // We should be able to encode up to (1^kMaxCycleBits - 1) without clamping + // but the lower kProfileTimestampShift will be dropped. + const int kMaxCyclesShift = + 32 - kLockwordReservedShift + kProfileTimestampShift; + const uint64_t kMaxCycles = (int64_t{1} << kMaxCyclesShift) - 1; + + // These bits should be zero after encoding. + const uint32_t kLockwordReservedMask = (1 << kLockwordReservedShift) - 1; + + // These bits are dropped when wait cycles are encoded. + const uint64_t kProfileTimestampMask = (1 << kProfileTimestampShift) - 1; + + // Test a bunch of random values + std::default_random_engine generator; + // Shift to avoid overflow below. + std::uniform_int_distribution<uint64_t> time_distribution( + 0, std::numeric_limits<uint64_t>::max() >> 4); + std::uniform_int_distribution<uint64_t> cycle_distribution(0, kMaxCycles); + + for (int i = 0; i < 100; i++) { + int64_t start_time = time_distribution(generator); + int64_t cycles = cycle_distribution(generator); + int64_t end_time = start_time + cycles; + uint32_t lock_value = SpinLockTest::EncodeWaitCycles(start_time, end_time); + EXPECT_EQ(0, lock_value & kLockwordReservedMask); + uint64_t decoded = SpinLockTest::DecodeWaitCycles(lock_value); + EXPECT_EQ(0, decoded & kProfileTimestampMask); + EXPECT_EQ(cycles & ~kProfileTimestampMask, decoded); + } + + // Test corner cases + int64_t start_time = time_distribution(generator); + EXPECT_EQ(kSpinLockSleeper, + SpinLockTest::EncodeWaitCycles(start_time, start_time)); + EXPECT_EQ(0, SpinLockTest::DecodeWaitCycles(0)); + EXPECT_EQ(0, SpinLockTest::DecodeWaitCycles(kLockwordReservedMask)); + EXPECT_EQ(kMaxCycles & ~kProfileTimestampMask, + SpinLockTest::DecodeWaitCycles(~kLockwordReservedMask)); + + // Check that we cannot produce kSpinLockSleeper during encoding. + int64_t sleeper_cycles = + kSpinLockSleeper << (kProfileTimestampShift - kLockwordReservedShift); + uint32_t sleeper_value = + SpinLockTest::EncodeWaitCycles(start_time, start_time + sleeper_cycles); + EXPECT_NE(sleeper_value, kSpinLockSleeper); + + // Test clamping + uint32_t max_value = + SpinLockTest::EncodeWaitCycles(start_time, start_time + kMaxCycles); + uint64_t max_value_decoded = SpinLockTest::DecodeWaitCycles(max_value); + uint64_t expected_max_value_decoded = kMaxCycles & ~kProfileTimestampMask; + EXPECT_EQ(expected_max_value_decoded, max_value_decoded); + + const int64_t step = (1 << kProfileTimestampShift); + uint32_t after_max_value = + SpinLockTest::EncodeWaitCycles(start_time, start_time + kMaxCycles + step); + uint64_t after_max_value_decoded = + SpinLockTest::DecodeWaitCycles(after_max_value); + EXPECT_EQ(expected_max_value_decoded, after_max_value_decoded); + + uint32_t before_max_value = SpinLockTest::EncodeWaitCycles( + start_time, start_time + kMaxCycles - step); + uint64_t before_max_value_decoded = + SpinLockTest::DecodeWaitCycles(before_max_value); + EXPECT_GT(expected_max_value_decoded, before_max_value_decoded); +} + +TEST(SpinLockWithThreads, StackSpinLock) { + SpinLock spinlock; + ThreadedTest(&spinlock); +} + +TEST(SpinLockWithThreads, StackCooperativeSpinLock) { + SpinLock spinlock(base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL); + ThreadedTest(&spinlock); +} + +TEST(SpinLockWithThreads, StackNonCooperativeSpinLock) { + SpinLock spinlock(base_internal::SCHEDULE_KERNEL_ONLY); + ThreadedTest(&spinlock); +} + +TEST(SpinLockWithThreads, StaticCooperativeSpinLock) { + ThreadedTest(&static_cooperative_spinlock); +} + +TEST(SpinLockWithThreads, StaticNonCooperativeSpinLock) { + ThreadedTest(&static_noncooperative_spinlock); +} + +TEST(SpinLockWithThreads, DoesNotDeadlock) { + struct Helper { + static void NotifyThenLock(Notification* locked, SpinLock* spinlock, + BlockingCounter* b) { + locked->WaitForNotification(); // Wait for LockThenWait() to hold "s". + b->DecrementCount(); + SpinLockHolder l(spinlock); + } + + static void LockThenWait(Notification* locked, SpinLock* spinlock, + BlockingCounter* b) { + SpinLockHolder l(spinlock); + locked->Notify(); + b->Wait(); + } + + static void DeadlockTest(SpinLock* spinlock, int num_spinners) { + Notification locked; + BlockingCounter counter(num_spinners); + std::vector<std::thread> threads; + + threads.push_back( + std::thread(Helper::LockThenWait, &locked, spinlock, &counter)); + for (int i = 0; i < num_spinners; ++i) { + threads.push_back( + std::thread(Helper::NotifyThenLock, &locked, spinlock, &counter)); + } + + for (auto& thread : threads) { + thread.join(); + } + } + }; + + SpinLock stack_cooperative_spinlock( + base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL); + SpinLock stack_noncooperative_spinlock(base_internal::SCHEDULE_KERNEL_ONLY); + Helper::DeadlockTest(&stack_cooperative_spinlock, + base_internal::NumCPUs() * 2); + Helper::DeadlockTest(&stack_noncooperative_spinlock, + base_internal::NumCPUs() * 2); + Helper::DeadlockTest(&static_cooperative_spinlock, + base_internal::NumCPUs() * 2); + Helper::DeadlockTest(&static_noncooperative_spinlock, + base_internal::NumCPUs() * 2); +} + +} // namespace +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/base/thread_annotations.h b/third_party/abseil_cpp/absl/base/thread_annotations.h new file mode 100644 index 000000000000..e23fff1d2013 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/thread_annotations.h @@ -0,0 +1,335 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: thread_annotations.h +// ----------------------------------------------------------------------------- +// +// This header file contains macro definitions for thread safety annotations +// that allow developers to document the locking policies of multi-threaded +// code. The annotations can also help program analysis tools to identify +// potential thread safety issues. +// +// These annotations are implemented using compiler attributes. Using the macros +// defined here instead of raw attributes allow for portability and future +// compatibility. +// +// When referring to mutexes in the arguments of the attributes, you should +// use variable names or more complex expressions (e.g. my_object->mutex_) +// that evaluate to a concrete mutex object whenever possible. If the mutex +// you want to refer to is not in scope, you may use a member pointer +// (e.g. &MyClass::mutex_) to refer to a mutex in some (unknown) object. + +#ifndef ABSL_BASE_THREAD_ANNOTATIONS_H_ +#define ABSL_BASE_THREAD_ANNOTATIONS_H_ + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +// TODO(mbonadei): Remove after the backward compatibility period. +#include "absl/base/internal/thread_annotations.h" // IWYU pragma: export + +// ABSL_GUARDED_BY() +// +// Documents if a shared field or global variable needs to be protected by a +// mutex. ABSL_GUARDED_BY() allows the user to specify a particular mutex that +// should be held when accessing the annotated variable. +// +// Although this annotation (and ABSL_PT_GUARDED_BY, below) cannot be applied to +// local variables, a local variable and its associated mutex can often be +// combined into a small class or struct, thereby allowing the annotation. +// +// Example: +// +// class Foo { +// Mutex mu_; +// int p1_ ABSL_GUARDED_BY(mu_); +// ... +// }; +#if ABSL_HAVE_ATTRIBUTE(guarded_by) +#define ABSL_GUARDED_BY(x) __attribute__((guarded_by(x))) +#else +#define ABSL_GUARDED_BY(x) +#endif + +// ABSL_PT_GUARDED_BY() +// +// Documents if the memory location pointed to by a pointer should be guarded +// by a mutex when dereferencing the pointer. +// +// Example: +// class Foo { +// Mutex mu_; +// int *p1_ ABSL_PT_GUARDED_BY(mu_); +// ... +// }; +// +// Note that a pointer variable to a shared memory location could itself be a +// shared variable. +// +// Example: +// +// // `q_`, guarded by `mu1_`, points to a shared memory location that is +// // guarded by `mu2_`: +// int *q_ ABSL_GUARDED_BY(mu1_) ABSL_PT_GUARDED_BY(mu2_); +#if ABSL_HAVE_ATTRIBUTE(pt_guarded_by) +#define ABSL_PT_GUARDED_BY(x) __attribute__((pt_guarded_by(x))) +#else +#define ABSL_PT_GUARDED_BY(x) +#endif + +// ABSL_ACQUIRED_AFTER() / ABSL_ACQUIRED_BEFORE() +// +// Documents the acquisition order between locks that can be held +// simultaneously by a thread. For any two locks that need to be annotated +// to establish an acquisition order, only one of them needs the annotation. +// (i.e. You don't have to annotate both locks with both ABSL_ACQUIRED_AFTER +// and ABSL_ACQUIRED_BEFORE.) +// +// As with ABSL_GUARDED_BY, this is only applicable to mutexes that are shared +// fields or global variables. +// +// Example: +// +// Mutex m1_; +// Mutex m2_ ABSL_ACQUIRED_AFTER(m1_); +#if ABSL_HAVE_ATTRIBUTE(acquired_after) +#define ABSL_ACQUIRED_AFTER(...) __attribute__((acquired_after(__VA_ARGS__))) +#else +#define ABSL_ACQUIRED_AFTER(...) +#endif + +#if ABSL_HAVE_ATTRIBUTE(acquired_before) +#define ABSL_ACQUIRED_BEFORE(...) __attribute__((acquired_before(__VA_ARGS__))) +#else +#define ABSL_ACQUIRED_BEFORE(...) +#endif + +// ABSL_EXCLUSIVE_LOCKS_REQUIRED() / ABSL_SHARED_LOCKS_REQUIRED() +// +// Documents a function that expects a mutex to be held prior to entry. +// The mutex is expected to be held both on entry to, and exit from, the +// function. +// +// An exclusive lock allows read-write access to the guarded data member(s), and +// only one thread can acquire a lock exclusively at any one time. A shared lock +// allows read-only access, and any number of threads can acquire a shared lock +// concurrently. +// +// Generally, non-const methods should be annotated with +// ABSL_EXCLUSIVE_LOCKS_REQUIRED, while const methods should be annotated with +// ABSL_SHARED_LOCKS_REQUIRED. +// +// Example: +// +// Mutex mu1, mu2; +// int a ABSL_GUARDED_BY(mu1); +// int b ABSL_GUARDED_BY(mu2); +// +// void foo() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu1, mu2) { ... } +// void bar() const ABSL_SHARED_LOCKS_REQUIRED(mu1, mu2) { ... } +#if ABSL_HAVE_ATTRIBUTE(exclusive_locks_required) +#define ABSL_EXCLUSIVE_LOCKS_REQUIRED(...) \ + __attribute__((exclusive_locks_required(__VA_ARGS__))) +#else +#define ABSL_EXCLUSIVE_LOCKS_REQUIRED(...) +#endif + +#if ABSL_HAVE_ATTRIBUTE(shared_locks_required) +#define ABSL_SHARED_LOCKS_REQUIRED(...) \ + __attribute__((shared_locks_required(__VA_ARGS__))) +#else +#define ABSL_SHARED_LOCKS_REQUIRED(...) +#endif + +// ABSL_LOCKS_EXCLUDED() +// +// Documents the locks acquired in the body of the function. These locks +// cannot be held when calling this function (as Abseil's `Mutex` locks are +// non-reentrant). +#if ABSL_HAVE_ATTRIBUTE(locks_excluded) +#define ABSL_LOCKS_EXCLUDED(...) __attribute__((locks_excluded(__VA_ARGS__))) +#else +#define ABSL_LOCKS_EXCLUDED(...) +#endif + +// ABSL_LOCK_RETURNED() +// +// Documents a function that returns a mutex without acquiring it. For example, +// a public getter method that returns a pointer to a private mutex should +// be annotated with ABSL_LOCK_RETURNED. +#if ABSL_HAVE_ATTRIBUTE(lock_returned) +#define ABSL_LOCK_RETURNED(x) __attribute__((lock_returned(x))) +#else +#define ABSL_LOCK_RETURNED(x) +#endif + +// ABSL_LOCKABLE +// +// Documents if a class/type is a lockable type (such as the `Mutex` class). +#if ABSL_HAVE_ATTRIBUTE(lockable) +#define ABSL_LOCKABLE __attribute__((lockable)) +#else +#define ABSL_LOCKABLE +#endif + +// ABSL_SCOPED_LOCKABLE +// +// Documents if a class does RAII locking (such as the `MutexLock` class). +// The constructor should use `LOCK_FUNCTION()` to specify the mutex that is +// acquired, and the destructor should use `UNLOCK_FUNCTION()` with no +// arguments; the analysis will assume that the destructor unlocks whatever the +// constructor locked. +#if ABSL_HAVE_ATTRIBUTE(scoped_lockable) +#define ABSL_SCOPED_LOCKABLE __attribute__((scoped_lockable)) +#else +#define ABSL_SCOPED_LOCKABLE +#endif + +// ABSL_EXCLUSIVE_LOCK_FUNCTION() +// +// Documents functions that acquire a lock in the body of a function, and do +// not release it. +#if ABSL_HAVE_ATTRIBUTE(exclusive_lock_function) +#define ABSL_EXCLUSIVE_LOCK_FUNCTION(...) \ + __attribute__((exclusive_lock_function(__VA_ARGS__))) +#else +#define ABSL_EXCLUSIVE_LOCK_FUNCTION(...) +#endif + +// ABSL_SHARED_LOCK_FUNCTION() +// +// Documents functions that acquire a shared (reader) lock in the body of a +// function, and do not release it. +#if ABSL_HAVE_ATTRIBUTE(shared_lock_function) +#define ABSL_SHARED_LOCK_FUNCTION(...) \ + __attribute__((shared_lock_function(__VA_ARGS__))) +#else +#define ABSL_SHARED_LOCK_FUNCTION(...) +#endif + +// ABSL_UNLOCK_FUNCTION() +// +// Documents functions that expect a lock to be held on entry to the function, +// and release it in the body of the function. +#if ABSL_HAVE_ATTRIBUTE(unlock_function) +#define ABSL_UNLOCK_FUNCTION(...) __attribute__((unlock_function(__VA_ARGS__))) +#else +#define ABSL_UNLOCK_FUNCTION(...) +#endif + +// ABSL_EXCLUSIVE_TRYLOCK_FUNCTION() / ABSL_SHARED_TRYLOCK_FUNCTION() +// +// Documents functions that try to acquire a lock, and return success or failure +// (or a non-boolean value that can be interpreted as a boolean). +// The first argument should be `true` for functions that return `true` on +// success, or `false` for functions that return `false` on success. The second +// argument specifies the mutex that is locked on success. If unspecified, this +// mutex is assumed to be `this`. +#if ABSL_HAVE_ATTRIBUTE(exclusive_trylock_function) +#define ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(...) \ + __attribute__((exclusive_trylock_function(__VA_ARGS__))) +#else +#define ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(...) +#endif + +#if ABSL_HAVE_ATTRIBUTE(shared_trylock_function) +#define ABSL_SHARED_TRYLOCK_FUNCTION(...) \ + __attribute__((shared_trylock_function(__VA_ARGS__))) +#else +#define ABSL_SHARED_TRYLOCK_FUNCTION(...) +#endif + +// ABSL_ASSERT_EXCLUSIVE_LOCK() / ABSL_ASSERT_SHARED_LOCK() +// +// Documents functions that dynamically check to see if a lock is held, and fail +// if it is not held. +#if ABSL_HAVE_ATTRIBUTE(assert_exclusive_lock) +#define ABSL_ASSERT_EXCLUSIVE_LOCK(...) \ + __attribute__((assert_exclusive_lock(__VA_ARGS__))) +#else +#define ABSL_ASSERT_EXCLUSIVE_LOCK(...) +#endif + +#if ABSL_HAVE_ATTRIBUTE(assert_shared_lock) +#define ABSL_ASSERT_SHARED_LOCK(...) \ + __attribute__((assert_shared_lock(__VA_ARGS__))) +#else +#define ABSL_ASSERT_SHARED_LOCK(...) +#endif + +// ABSL_NO_THREAD_SAFETY_ANALYSIS +// +// Turns off thread safety checking within the body of a particular function. +// This annotation is used to mark functions that are known to be correct, but +// the locking behavior is more complicated than the analyzer can handle. +#if ABSL_HAVE_ATTRIBUTE(no_thread_safety_analysis) +#define ABSL_NO_THREAD_SAFETY_ANALYSIS \ + __attribute__((no_thread_safety_analysis)) +#else +#define ABSL_NO_THREAD_SAFETY_ANALYSIS +#endif + +//------------------------------------------------------------------------------ +// Tool-Supplied Annotations +//------------------------------------------------------------------------------ + +// ABSL_TS_UNCHECKED should be placed around lock expressions that are not valid +// C++ syntax, but which are present for documentation purposes. These +// annotations will be ignored by the analysis. +#define ABSL_TS_UNCHECKED(x) "" + +// ABSL_TS_FIXME is used to mark lock expressions that are not valid C++ syntax. +// It is used by automated tools to mark and disable invalid expressions. +// The annotation should either be fixed, or changed to ABSL_TS_UNCHECKED. +#define ABSL_TS_FIXME(x) "" + +// Like ABSL_NO_THREAD_SAFETY_ANALYSIS, this turns off checking within the body +// of a particular function. However, this attribute is used to mark functions +// that are incorrect and need to be fixed. It is used by automated tools to +// avoid breaking the build when the analysis is updated. +// Code owners are expected to eventually fix the routine. +#define ABSL_NO_THREAD_SAFETY_ANALYSIS_FIXME ABSL_NO_THREAD_SAFETY_ANALYSIS + +// Similar to ABSL_NO_THREAD_SAFETY_ANALYSIS_FIXME, this macro marks a +// ABSL_GUARDED_BY annotation that needs to be fixed, because it is producing +// thread safety warning. It disables the ABSL_GUARDED_BY. +#define ABSL_GUARDED_BY_FIXME(x) + +// Disables warnings for a single read operation. This can be used to avoid +// warnings when it is known that the read is not actually involved in a race, +// but the compiler cannot confirm that. +#define ABSL_TS_UNCHECKED_READ(x) absl::base_internal::ts_unchecked_read(x) + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +// Takes a reference to a guarded data member, and returns an unguarded +// reference. +// Do not used this function directly, use ABSL_TS_UNCHECKED_READ instead. +template <typename T> +inline const T& ts_unchecked_read(const T& v) ABSL_NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +template <typename T> +inline T& ts_unchecked_read(T& v) ABSL_NO_THREAD_SAFETY_ANALYSIS { + return v; +} + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_THREAD_ANNOTATIONS_H_ diff --git a/third_party/abseil_cpp/absl/base/throw_delegate_test.cc b/third_party/abseil_cpp/absl/base/throw_delegate_test.cc new file mode 100644 index 000000000000..5ba4ce55e6d5 --- /dev/null +++ b/third_party/abseil_cpp/absl/base/throw_delegate_test.cc @@ -0,0 +1,107 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/throw_delegate.h" + +#include <functional> +#include <new> +#include <stdexcept> + +#include "absl/base/config.h" +#include "gtest/gtest.h" + +namespace { + +using absl::base_internal::ThrowStdLogicError; +using absl::base_internal::ThrowStdInvalidArgument; +using absl::base_internal::ThrowStdDomainError; +using absl::base_internal::ThrowStdLengthError; +using absl::base_internal::ThrowStdOutOfRange; +using absl::base_internal::ThrowStdRuntimeError; +using absl::base_internal::ThrowStdRangeError; +using absl::base_internal::ThrowStdOverflowError; +using absl::base_internal::ThrowStdUnderflowError; +using absl::base_internal::ThrowStdBadFunctionCall; +using absl::base_internal::ThrowStdBadAlloc; + +constexpr const char* what_arg = "The quick brown fox jumps over the lazy dog"; + +template <typename E> +void ExpectThrowChar(void (*f)(const char*)) { +#ifdef ABSL_HAVE_EXCEPTIONS + try { + f(what_arg); + FAIL() << "Didn't throw"; + } catch (const E& e) { + EXPECT_STREQ(e.what(), what_arg); + } +#else + EXPECT_DEATH_IF_SUPPORTED(f(what_arg), what_arg); +#endif +} + +template <typename E> +void ExpectThrowString(void (*f)(const std::string&)) { +#ifdef ABSL_HAVE_EXCEPTIONS + try { + f(what_arg); + FAIL() << "Didn't throw"; + } catch (const E& e) { + EXPECT_STREQ(e.what(), what_arg); + } +#else + EXPECT_DEATH_IF_SUPPORTED(f(what_arg), what_arg); +#endif +} + +template <typename E> +void ExpectThrowNoWhat(void (*f)()) { +#ifdef ABSL_HAVE_EXCEPTIONS + try { + f(); + FAIL() << "Didn't throw"; + } catch (const E& e) { + } +#else + EXPECT_DEATH_IF_SUPPORTED(f(), ""); +#endif +} + +TEST(ThrowHelper, Test) { + // Not using EXPECT_THROW because we want to check the .what() message too. + ExpectThrowChar<std::logic_error>(ThrowStdLogicError); + ExpectThrowChar<std::invalid_argument>(ThrowStdInvalidArgument); + ExpectThrowChar<std::domain_error>(ThrowStdDomainError); + ExpectThrowChar<std::length_error>(ThrowStdLengthError); + ExpectThrowChar<std::out_of_range>(ThrowStdOutOfRange); + ExpectThrowChar<std::runtime_error>(ThrowStdRuntimeError); + ExpectThrowChar<std::range_error>(ThrowStdRangeError); + ExpectThrowChar<std::overflow_error>(ThrowStdOverflowError); + ExpectThrowChar<std::underflow_error>(ThrowStdUnderflowError); + + ExpectThrowString<std::logic_error>(ThrowStdLogicError); + ExpectThrowString<std::invalid_argument>(ThrowStdInvalidArgument); + ExpectThrowString<std::domain_error>(ThrowStdDomainError); + ExpectThrowString<std::length_error>(ThrowStdLengthError); + ExpectThrowString<std::out_of_range>(ThrowStdOutOfRange); + ExpectThrowString<std::runtime_error>(ThrowStdRuntimeError); + ExpectThrowString<std::range_error>(ThrowStdRangeError); + ExpectThrowString<std::overflow_error>(ThrowStdOverflowError); + ExpectThrowString<std::underflow_error>(ThrowStdUnderflowError); + + ExpectThrowNoWhat<std::bad_function_call>(ThrowStdBadFunctionCall); + ExpectThrowNoWhat<std::bad_alloc>(ThrowStdBadAlloc); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/container/BUILD.bazel b/third_party/abseil_cpp/absl/container/BUILD.bazel new file mode 100644 index 000000000000..8e72ad03547c --- /dev/null +++ b/third_party/abseil_cpp/absl/container/BUILD.bazel @@ -0,0 +1,921 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "compressed_tuple", + hdrs = ["internal/compressed_tuple.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/utility", + ], +) + +cc_test( + name = "compressed_tuple_test", + srcs = ["internal/compressed_tuple_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":compressed_tuple", + ":test_instance_tracker", + "//absl/memory", + "//absl/types:any", + "//absl/types:optional", + "//absl/utility", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "fixed_array", + hdrs = ["fixed_array.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":compressed_tuple", + "//absl/algorithm", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:dynamic_annotations", + "//absl/base:throw_delegate", + "//absl/memory", + ], +) + +cc_test( + name = "fixed_array_test", + srcs = ["fixed_array_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":counting_allocator", + ":fixed_array", + "//absl/base:config", + "//absl/base:exception_testing", + "//absl/hash:hash_testing", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "fixed_array_exception_safety_test", + srcs = ["fixed_array_exception_safety_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":fixed_array", + "//absl/base:config", + "//absl/base:exception_safety_testing", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "fixed_array_benchmark", + srcs = ["fixed_array_benchmark.cc"], + copts = ABSL_TEST_COPTS + ["$(STACK_FRAME_UNLIMITED)"], + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + deps = [ + ":fixed_array", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "inlined_vector_internal", + hdrs = ["internal/inlined_vector.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":compressed_tuple", + "//absl/base:core_headers", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/types:span", + ], +) + +cc_library( + name = "inlined_vector", + hdrs = ["inlined_vector.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":inlined_vector_internal", + "//absl/algorithm", + "//absl/base:core_headers", + "//absl/base:throw_delegate", + "//absl/memory", + ], +) + +cc_library( + name = "counting_allocator", + testonly = 1, + hdrs = ["internal/counting_allocator.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = ["//absl/base:config"], +) + +cc_test( + name = "inlined_vector_test", + srcs = ["inlined_vector_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":counting_allocator", + ":inlined_vector", + ":test_instance_tracker", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:exception_testing", + "//absl/base:raw_logging_internal", + "//absl/hash:hash_testing", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "inlined_vector_benchmark", + srcs = ["inlined_vector_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + deps = [ + ":inlined_vector", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/strings", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "inlined_vector_exception_safety_test", + srcs = ["inlined_vector_exception_safety_test.cc"], + copts = ABSL_TEST_COPTS, + deps = [ + ":inlined_vector", + "//absl/base:config", + "//absl/base:exception_safety_testing", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "test_instance_tracker", + testonly = 1, + srcs = ["internal/test_instance_tracker.cc"], + hdrs = ["internal/test_instance_tracker.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = ["//absl/types:compare"], +) + +cc_test( + name = "test_instance_tracker_test", + srcs = ["internal/test_instance_tracker_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":test_instance_tracker", + "@com_google_googletest//:gtest_main", + ], +) + +NOTEST_TAGS_NONMOBILE = [ + "no_test_darwin_x86_64", + "no_test_loonix", +] + +NOTEST_TAGS_MOBILE = [ + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + "no_test_ios_x86_64", +] + +NOTEST_TAGS = NOTEST_TAGS_MOBILE + NOTEST_TAGS_NONMOBILE + +cc_library( + name = "flat_hash_map", + hdrs = ["flat_hash_map.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":container_memory", + ":hash_function_defaults", + ":raw_hash_map", + "//absl/algorithm:container", + "//absl/memory", + ], +) + +cc_test( + name = "flat_hash_map_test", + srcs = ["flat_hash_map_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":flat_hash_map", + ":hash_generator_testing", + ":unordered_map_constructor_test", + ":unordered_map_lookup_test", + ":unordered_map_members_test", + ":unordered_map_modifiers_test", + "//absl/base:raw_logging_internal", + "//absl/types:any", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "flat_hash_set", + hdrs = ["flat_hash_set.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":container_memory", + ":hash_function_defaults", + ":raw_hash_set", + "//absl/algorithm:container", + "//absl/base:core_headers", + "//absl/memory", + ], +) + +cc_test( + name = "flat_hash_set_test", + srcs = ["flat_hash_set_test.cc"], + copts = ABSL_TEST_COPTS + ["-DUNORDERED_SET_CXX17"], + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":flat_hash_set", + ":hash_generator_testing", + ":unordered_set_constructor_test", + ":unordered_set_lookup_test", + ":unordered_set_members_test", + ":unordered_set_modifiers_test", + "//absl/base:raw_logging_internal", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "node_hash_map", + hdrs = ["node_hash_map.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":container_memory", + ":hash_function_defaults", + ":node_hash_policy", + ":raw_hash_map", + "//absl/algorithm:container", + "//absl/memory", + ], +) + +cc_test( + name = "node_hash_map_test", + srcs = ["node_hash_map_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":hash_generator_testing", + ":node_hash_map", + ":tracked", + ":unordered_map_constructor_test", + ":unordered_map_lookup_test", + ":unordered_map_members_test", + ":unordered_map_modifiers_test", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "node_hash_set", + hdrs = ["node_hash_set.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_function_defaults", + ":node_hash_policy", + ":raw_hash_set", + "//absl/algorithm:container", + "//absl/memory", + ], +) + +cc_test( + name = "node_hash_set_test", + srcs = ["node_hash_set_test.cc"], + copts = ABSL_TEST_COPTS + ["-DUNORDERED_SET_CXX17"], + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":node_hash_set", + ":unordered_set_constructor_test", + ":unordered_set_lookup_test", + ":unordered_set_members_test", + ":unordered_set_modifiers_test", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "container_memory", + hdrs = ["internal/container_memory.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_test( + name = "container_memory_test", + srcs = ["internal/container_memory_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":container_memory", + ":test_instance_tracker", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "hash_function_defaults", + hdrs = ["internal/hash_function_defaults.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/hash", + "//absl/strings", + "//absl/strings:cord", + ], +) + +cc_test( + name = "hash_function_defaults_test", + srcs = ["internal/hash_function_defaults_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS, + deps = [ + ":hash_function_defaults", + "//absl/hash", + "//absl/random", + "//absl/strings", + "//absl/strings:cord", + "//absl/strings:cord_test_helpers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "hash_generator_testing", + testonly = 1, + srcs = ["internal/hash_generator_testing.cc"], + hdrs = ["internal/hash_generator_testing.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_policy_testing", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/strings", + ], +) + +cc_library( + name = "hash_policy_testing", + testonly = 1, + hdrs = ["internal/hash_policy_testing.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/hash", + "//absl/strings", + ], +) + +cc_test( + name = "hash_policy_testing_test", + srcs = ["internal/hash_policy_testing_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_policy_testing", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "hash_policy_traits", + hdrs = ["internal/hash_policy_traits.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/meta:type_traits"], +) + +cc_test( + name = "hash_policy_traits_test", + srcs = ["internal/hash_policy_traits_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_policy_traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "hashtable_debug", + hdrs = ["internal/hashtable_debug.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hashtable_debug_hooks", + ], +) + +cc_library( + name = "hashtable_debug_hooks", + hdrs = ["internal/hashtable_debug_hooks.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + ], +) + +cc_library( + name = "hashtablez_sampler", + srcs = [ + "internal/hashtablez_sampler.cc", + "internal/hashtablez_sampler_force_weak_definition.cc", + ], + hdrs = ["internal/hashtablez_sampler.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":have_sse", + "//absl/base", + "//absl/base:core_headers", + "//absl/base:exponential_biased", + "//absl/debugging:stacktrace", + "//absl/memory", + "//absl/synchronization", + "//absl/utility", + ], +) + +cc_test( + name = "hashtablez_sampler_test", + srcs = ["internal/hashtablez_sampler_test.cc"], + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hashtablez_sampler", + ":have_sse", + "//absl/base:core_headers", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "node_hash_policy", + hdrs = ["internal/node_hash_policy.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/base:config"], +) + +cc_test( + name = "node_hash_policy_test", + srcs = ["internal/node_hash_policy_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_policy_traits", + ":node_hash_policy", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "raw_hash_map", + hdrs = ["internal/raw_hash_map.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":container_memory", + ":raw_hash_set", + "//absl/base:throw_delegate", + ], +) + +cc_library( + name = "have_sse", + hdrs = ["internal/have_sse.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], +) + +cc_library( + name = "common", + hdrs = ["internal/common.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/meta:type_traits", + "//absl/types:optional", + ], +) + +cc_library( + name = "raw_hash_set", + srcs = ["internal/raw_hash_set.cc"], + hdrs = ["internal/raw_hash_set.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":common", + ":compressed_tuple", + ":container_memory", + ":hash_policy_traits", + ":hashtable_debug_hooks", + ":hashtablez_sampler", + ":have_sse", + ":layout", + "//absl/base:bits", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_test( + name = "raw_hash_set_test", + srcs = ["internal/raw_hash_set_test.cc"], + copts = ABSL_TEST_COPTS, + linkstatic = 1, + tags = NOTEST_TAGS, + deps = [ + ":container_memory", + ":hash_function_defaults", + ":hash_policy_testing", + ":hashtable_debug", + ":raw_hash_set", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "raw_hash_set_allocator_test", + size = "small", + srcs = ["internal/raw_hash_set_allocator_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":raw_hash_set", + ":tracked", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "layout", + hdrs = ["internal/layout.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/types:span", + "//absl/utility", + ], +) + +cc_test( + name = "layout_test", + size = "small", + srcs = ["internal/layout_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS, + visibility = ["//visibility:private"], + deps = [ + ":layout", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "tracked", + testonly = 1, + hdrs = ["internal/tracked.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + ], +) + +cc_library( + name = "unordered_map_constructor_test", + testonly = 1, + hdrs = ["internal/unordered_map_constructor_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_generator_testing", + ":hash_policy_testing", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_map_lookup_test", + testonly = 1, + hdrs = ["internal/unordered_map_lookup_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_generator_testing", + ":hash_policy_testing", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_map_modifiers_test", + testonly = 1, + hdrs = ["internal/unordered_map_modifiers_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_generator_testing", + ":hash_policy_testing", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_set_constructor_test", + testonly = 1, + hdrs = ["internal/unordered_set_constructor_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_generator_testing", + ":hash_policy_testing", + "//absl/meta:type_traits", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_set_members_test", + testonly = 1, + hdrs = ["internal/unordered_set_members_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/meta:type_traits", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_map_members_test", + testonly = 1, + hdrs = ["internal/unordered_map_members_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/meta:type_traits", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_set_lookup_test", + testonly = 1, + hdrs = ["internal/unordered_set_lookup_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_generator_testing", + ":hash_policy_testing", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "unordered_set_modifiers_test", + testonly = 1, + hdrs = ["internal/unordered_set_modifiers_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash_generator_testing", + ":hash_policy_testing", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "unordered_set_test", + srcs = ["internal/unordered_set_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":unordered_set_constructor_test", + ":unordered_set_lookup_test", + ":unordered_set_members_test", + ":unordered_set_modifiers_test", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "unordered_map_test", + srcs = ["internal/unordered_map_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = NOTEST_TAGS_NONMOBILE, + deps = [ + ":unordered_map_constructor_test", + ":unordered_map_lookup_test", + ":unordered_map_members_test", + ":unordered_map_modifiers_test", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "btree", + srcs = [ + "internal/btree.h", + "internal/btree_container.h", + ], + hdrs = [ + "btree_map.h", + "btree_set.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:public"], + deps = [ + ":common", + ":compressed_tuple", + ":container_memory", + ":layout", + "//absl/base:core_headers", + "//absl/base:throw_delegate", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/strings:cord", + "//absl/types:compare", + "//absl/utility", + ], +) + +cc_library( + name = "btree_test_common", + testonly = 1, + hdrs = ["btree_test.h"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":btree", + ":flat_hash_set", + "//absl/strings", + "//absl/strings:cord", + "//absl/time", + ], +) + +cc_test( + name = "btree_test", + size = "large", + srcs = [ + "btree_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 10, + visibility = ["//visibility:private"], + deps = [ + ":btree", + ":btree_test_common", + ":counting_allocator", + ":test_instance_tracker", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/flags:flag", + "//absl/hash:hash_testing", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/types:compare", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "btree_benchmark", + testonly = 1, + srcs = [ + "btree_benchmark.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":btree", + ":btree_test_common", + ":flat_hash_map", + ":flat_hash_set", + ":hashtable_debug", + "//absl/base:raw_logging_internal", + "//absl/flags:flag", + "//absl/hash", + "//absl/memory", + "//absl/strings:cord", + "//absl/strings:str_format", + "//absl/time", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/third_party/abseil_cpp/absl/container/CMakeLists.txt b/third_party/abseil_cpp/absl/container/CMakeLists.txt new file mode 100644 index 000000000000..eb202c459b27 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/CMakeLists.txt @@ -0,0 +1,905 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# This is deprecated and will be removed in the future. It also doesn't do +# anything anyways. Prefer to use the library associated with the API you are +# using. +absl_cc_library( + NAME + container + PUBLIC +) + +absl_cc_library( + NAME + btree + HDRS + "btree_map.h" + "btree_set.h" + "internal/btree.h" + "internal/btree_container.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::container_common + absl::compare + absl::compressed_tuple + absl::container_memory + absl::cord + absl::core_headers + absl::layout + absl::memory + absl::strings + absl::throw_delegate + absl::type_traits + absl::utility +) + +absl_cc_library( + NAME + btree_test_common + hdrs + "btree_test.h" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::btree + absl::cord + absl::flat_hash_set + absl::strings + absl::time + TESTONLY +) + +absl_cc_test( + NAME + btree_test + SRCS + "btree_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::btree + absl::btree_test_common + absl::compare + absl::core_headers + absl::counting_allocator + absl::flags + absl::hash_testing + absl::raw_logging_internal + absl::strings + absl::test_instance_tracker + absl::type_traits + gmock_main +) + +absl_cc_library( + NAME + compressed_tuple + HDRS + "internal/compressed_tuple.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::utility + PUBLIC +) + +absl_cc_test( + NAME + compressed_tuple_test + SRCS + "internal/compressed_tuple_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::any + absl::compressed_tuple + absl::memory + absl::optional + absl::test_instance_tracker + absl::utility + gmock_main +) + +absl_cc_library( + NAME + fixed_array + HDRS + "fixed_array.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compressed_tuple + absl::algorithm + absl::config + absl::core_headers + absl::dynamic_annotations + absl::throw_delegate + absl::memory + PUBLIC +) + +absl_cc_test( + NAME + fixed_array_test + SRCS + "fixed_array_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::fixed_array + absl::counting_allocator + absl::config + absl::exception_testing + absl::hash_testing + absl::memory + gmock_main +) + +absl_cc_test( + NAME + fixed_array_exception_safety_test + SRCS + "fixed_array_exception_safety_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::fixed_array + absl::config + absl::exception_safety_testing + gmock_main +) + +absl_cc_library( + NAME + inlined_vector_internal + HDRS + "internal/inlined_vector.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compressed_tuple + absl::core_headers + absl::memory + absl::span + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + inlined_vector + HDRS + "inlined_vector.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::inlined_vector_internal + absl::throw_delegate + absl::memory + PUBLIC +) + +absl_cc_library( + NAME + counting_allocator + HDRS + "internal/counting_allocator.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +absl_cc_test( + NAME + inlined_vector_test + SRCS + "inlined_vector_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::counting_allocator + absl::inlined_vector + absl::test_instance_tracker + absl::config + absl::core_headers + absl::exception_testing + absl::hash_testing + absl::memory + absl::raw_logging_internal + absl::strings + gmock_main +) + +absl_cc_test( + NAME + inlined_vector_exception_safety_test + SRCS + "inlined_vector_exception_safety_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::inlined_vector + absl::config + absl::exception_safety_testing + gmock_main +) + +absl_cc_library( + NAME + test_instance_tracker + HDRS + "internal/test_instance_tracker.h" + SRCS + "internal/test_instance_tracker.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::compare + TESTONLY +) + +absl_cc_test( + NAME + test_instance_tracker_test + SRCS + "internal/test_instance_tracker_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::test_instance_tracker + gmock_main +) + +absl_cc_library( + NAME + flat_hash_map + HDRS + "flat_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::hash_function_defaults + absl::raw_hash_map + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_test( + NAME + flat_hash_map_test + SRCS + "flat_hash_map_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flat_hash_map + absl::hash_generator_testing + absl::unordered_map_constructor_test + absl::unordered_map_lookup_test + absl::unordered_map_members_test + absl::unordered_map_modifiers_test + absl::any + absl::raw_logging_internal + gmock_main +) + +absl_cc_library( + NAME + flat_hash_set + HDRS + "flat_hash_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::hash_function_defaults + absl::raw_hash_set + absl::algorithm_container + absl::core_headers + absl::memory + PUBLIC +) + +absl_cc_test( + NAME + flat_hash_set_test + SRCS + "flat_hash_set_test.cc" + COPTS + ${ABSL_TEST_COPTS} + "-DUNORDERED_SET_CXX17" + DEPS + absl::flat_hash_set + absl::hash_generator_testing + absl::unordered_set_constructor_test + absl::unordered_set_lookup_test + absl::unordered_set_members_test + absl::unordered_set_modifiers_test + absl::memory + absl::raw_logging_internal + absl::strings + gmock_main +) + +absl_cc_library( + NAME + node_hash_map + HDRS + "node_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::hash_function_defaults + absl::node_hash_policy + absl::raw_hash_map + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_test( + NAME + node_hash_map_test + SRCS + "node_hash_map_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::node_hash_map + absl::tracked + absl::unordered_map_constructor_test + absl::unordered_map_lookup_test + absl::unordered_map_members_test + absl::unordered_map_modifiers_test + gmock_main +) + +absl_cc_library( + NAME + node_hash_set + HDRS + "node_hash_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::hash_function_defaults + absl::node_hash_policy + absl::raw_hash_set + absl::algorithm_container + absl::memory + PUBLIC +) + +absl_cc_test( + NAME + node_hash_set_test + SRCS + "node_hash_set_test.cc" + COPTS + ${ABSL_TEST_COPTS} + "-DUNORDERED_SET_CXX17" + DEPS + absl::hash_generator_testing + absl::node_hash_set + absl::unordered_set_constructor_test + absl::unordered_set_lookup_test + absl::unordered_set_members_test + absl::unordered_set_modifiers_test + gmock_main +) + +absl_cc_library( + NAME + container_memory + HDRS + "internal/container_memory.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::memory + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_test( + NAME + container_memory_test + SRCS + "internal/container_memory_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::container_memory + absl::strings + absl::test_instance_tracker + gmock_main +) + +absl_cc_library( + NAME + hash_function_defaults + HDRS + "internal/hash_function_defaults.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord + absl::hash + absl::strings + PUBLIC +) + +absl_cc_test( + NAME + hash_function_defaults_test + SRCS + "internal/hash_function_defaults_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::cord + absl::cord_test_helpers + absl::hash_function_defaults + absl::hash + absl::random_random + absl::strings + gmock_main +) + +absl_cc_library( + NAME + hash_generator_testing + HDRS + "internal/hash_generator_testing.h" + SRCS + "internal/hash_generator_testing.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_policy_testing + absl::memory + absl::meta + absl::strings + TESTONLY +) + +absl_cc_library( + NAME + hash_policy_testing + HDRS + "internal/hash_policy_testing.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash + absl::strings + TESTONLY +) + +absl_cc_test( + NAME + hash_policy_testing_test + SRCS + "internal/hash_policy_testing_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_policy_testing + gmock_main +) + +absl_cc_library( + NAME + hash_policy_traits + HDRS + "internal/hash_policy_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::meta + PUBLIC +) + +absl_cc_test( + NAME + hash_policy_traits_test + SRCS + "internal/hash_policy_traits_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_policy_traits + gmock_main +) + +absl_cc_library( + NAME + hashtablez_sampler + HDRS + "internal/hashtablez_sampler.h" + SRCS + "internal/hashtablez_sampler.cc" + "internal/hashtablez_sampler_force_weak_definition.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::exponential_biased + absl::have_sse + absl::synchronization +) + +absl_cc_test( + NAME + hashtablez_sampler_test + SRCS + "internal/hashtablez_sampler_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hashtablez_sampler + absl::have_sse + gmock_main +) + +absl_cc_library( + NAME + hashtable_debug + HDRS + "internal/hashtable_debug.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::hashtable_debug_hooks +) + +absl_cc_library( + NAME + hashtable_debug_hooks + HDRS + "internal/hashtable_debug_hooks.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_library( + NAME + have_sse + HDRS + "internal/have_sse.h" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +absl_cc_library( + NAME + node_hash_policy + HDRS + "internal/node_hash_policy.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_test( + NAME + node_hash_policy_test + SRCS + "internal/node_hash_policy_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_policy_traits + absl::node_hash_policy + gmock_main +) + +absl_cc_library( + NAME + raw_hash_map + HDRS + "internal/raw_hash_map.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::container_memory + absl::raw_hash_set + absl::throw_delegate + PUBLIC +) + +absl_cc_library( + NAME + container_common + HDRS + "internal/common.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::type_traits +) + +absl_cc_library( + NAME + raw_hash_set + HDRS + "internal/raw_hash_set.h" + SRCS + "internal/raw_hash_set.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::compressed_tuple + absl::config + absl::container_common + absl::container_memory + absl::core_headers + absl::endian + absl::hash_policy_traits + absl::hashtable_debug_hooks + absl::have_sse + absl::layout + absl::memory + absl::meta + absl::optional + absl::utility + absl::hashtablez_sampler + PUBLIC +) + +absl_cc_test( + NAME + raw_hash_set_test + SRCS + "internal/raw_hash_set_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::container_memory + absl::hash_function_defaults + absl::hash_policy_testing + absl::hashtable_debug + absl::raw_hash_set + absl::base + absl::config + absl::core_headers + absl::raw_logging_internal + absl::strings + gmock_main +) + +absl_cc_test( + NAME + raw_hash_set_allocator_test + SRCS + "internal/raw_hash_set_allocator_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::raw_hash_set + absl::tracked + absl::core_headers + gmock_main +) + +absl_cc_library( + NAME + layout + HDRS + "internal/layout.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::meta + absl::strings + absl::span + absl::utility + PUBLIC +) + +absl_cc_test( + NAME + layout_test + SRCS + "internal/layout_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::layout + absl::config + absl::core_headers + absl::raw_logging_internal + absl::span + gmock_main +) + +absl_cc_library( + NAME + tracked + HDRS + "internal/tracked.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + TESTONLY +) + +absl_cc_library( + NAME + unordered_map_constructor_test + HDRS + "internal/unordered_map_constructor_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::hash_policy_testing + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_map_lookup_test + HDRS + "internal/unordered_map_lookup_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::hash_policy_testing + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_map_members_test + HDRS + "internal/unordered_map_members_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::type_traits + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_map_modifiers_test + HDRS + "internal/unordered_map_modifiers_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::hash_policy_testing + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_set_constructor_test + HDRS + "internal/unordered_set_constructor_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::hash_policy_testing + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_set_lookup_test + HDRS + "internal/unordered_set_lookup_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::hash_policy_testing + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_set_members_test + HDRS + "internal/unordered_set_members_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::type_traits + gmock + TESTONLY +) + +absl_cc_library( + NAME + unordered_set_modifiers_test + HDRS + "internal/unordered_set_modifiers_test.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::hash_generator_testing + absl::hash_policy_testing + gmock + TESTONLY +) + +absl_cc_test( + NAME + unordered_set_test + SRCS + "internal/unordered_set_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::unordered_set_constructor_test + absl::unordered_set_lookup_test + absl::unordered_set_members_test + absl::unordered_set_modifiers_test + gmock_main +) + +absl_cc_test( + NAME + unordered_map_test + SRCS + "internal/unordered_map_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::unordered_map_constructor_test + absl::unordered_map_lookup_test + absl::unordered_map_members_test + absl::unordered_map_modifiers_test + gmock_main +) diff --git a/third_party/abseil_cpp/absl/container/btree_benchmark.cc b/third_party/abseil_cpp/absl/container/btree_benchmark.cc new file mode 100644 index 000000000000..467986768aa1 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/btree_benchmark.cc @@ -0,0 +1,735 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdint.h> + +#include <algorithm> +#include <functional> +#include <map> +#include <numeric> +#include <random> +#include <set> +#include <string> +#include <type_traits> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/btree_map.h" +#include "absl/container/btree_set.h" +#include "absl/container/btree_test.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/container/internal/hashtable_debug.h" +#include "absl/flags/flag.h" +#include "absl/hash/hash.h" +#include "absl/memory/memory.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_format.h" +#include "absl/time/time.h" +#include "benchmark/benchmark.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +constexpr size_t kBenchmarkValues = 1 << 20; + +// How many times we add and remove sub-batches in one batch of *AddRem +// benchmarks. +constexpr size_t kAddRemBatchSize = 1 << 2; + +// Generates n values in the range [0, 4 * n]. +template <typename V> +std::vector<V> GenerateValues(int n) { + constexpr int kSeed = 23; + return GenerateValuesWithSeed<V>(n, 4 * n, kSeed); +} + +// Benchmark insertion of values into a container. +template <typename T> +void BM_InsertImpl(benchmark::State& state, bool sorted) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + if (sorted) { + std::sort(values.begin(), values.end()); + } + T container(values.begin(), values.end()); + + // Remove and re-insert 10% of the keys per batch. + const int batch_size = (kBenchmarkValues + 9) / 10; + while (state.KeepRunningBatch(batch_size)) { + state.PauseTiming(); + const auto i = static_cast<int>(state.iterations()); + + for (int j = i; j < i + batch_size; j++) { + int x = j % kBenchmarkValues; + container.erase(key_of_value(values[x])); + } + + state.ResumeTiming(); + + for (int j = i; j < i + batch_size; j++) { + int x = j % kBenchmarkValues; + container.insert(values[x]); + } + } +} + +template <typename T> +void BM_Insert(benchmark::State& state) { + BM_InsertImpl<T>(state, false); +} + +template <typename T> +void BM_InsertSorted(benchmark::State& state) { + BM_InsertImpl<T>(state, true); +} + +// container::insert sometimes returns a pair<iterator, bool> and sometimes +// returns an iterator (for multi- containers). +template <typename Iter> +Iter GetIterFromInsert(const std::pair<Iter, bool>& pair) { + return pair.first; +} +template <typename Iter> +Iter GetIterFromInsert(const Iter iter) { + return iter; +} + +// Benchmark insertion of values into a container at the end. +template <typename T> +void BM_InsertEnd(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + + T container; + const int kSize = 10000; + for (int i = 0; i < kSize; ++i) { + container.insert(Generator<V>(kSize)(i)); + } + V v = Generator<V>(kSize)(kSize - 1); + typename T::key_type k = key_of_value(v); + + auto it = container.find(k); + while (state.KeepRunning()) { + // Repeatedly removing then adding v. + container.erase(it); + it = GetIterFromInsert(container.insert(v)); + } +} + +// Benchmark inserting the first few elements in a container. In b-tree, this is +// when the root node grows. +template <typename T> +void BM_InsertSmall(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + + const int kSize = 8; + std::vector<V> values = GenerateValues<V>(kSize); + T container; + + while (state.KeepRunningBatch(kSize)) { + for (int i = 0; i < kSize; ++i) { + benchmark::DoNotOptimize(container.insert(values[i])); + } + state.PauseTiming(); + // Do not measure the time it takes to clear the container. + container.clear(); + state.ResumeTiming(); + } +} + +template <typename T> +void BM_LookupImpl(benchmark::State& state, bool sorted) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + if (sorted) { + std::sort(values.begin(), values.end()); + } + T container(values.begin(), values.end()); + + while (state.KeepRunning()) { + int idx = state.iterations() % kBenchmarkValues; + benchmark::DoNotOptimize(container.find(key_of_value(values[idx]))); + } +} + +// Benchmark lookup of values in a container. +template <typename T> +void BM_Lookup(benchmark::State& state) { + BM_LookupImpl<T>(state, false); +} + +// Benchmark lookup of values in a full container, meaning that values +// are inserted in-order to take advantage of biased insertion, which +// yields a full tree. +template <typename T> +void BM_FullLookup(benchmark::State& state) { + BM_LookupImpl<T>(state, true); +} + +// Benchmark deletion of values from a container. +template <typename T> +void BM_Delete(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + T container(values.begin(), values.end()); + + // Remove and re-insert 10% of the keys per batch. + const int batch_size = (kBenchmarkValues + 9) / 10; + while (state.KeepRunningBatch(batch_size)) { + const int i = state.iterations(); + + for (int j = i; j < i + batch_size; j++) { + int x = j % kBenchmarkValues; + container.erase(key_of_value(values[x])); + } + + state.PauseTiming(); + for (int j = i; j < i + batch_size; j++) { + int x = j % kBenchmarkValues; + container.insert(values[x]); + } + state.ResumeTiming(); + } +} + +// Benchmark deletion of multiple values from a container. +template <typename T> +void BM_DeleteRange(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + T container(values.begin(), values.end()); + + // Remove and re-insert 10% of the keys per batch. + const int batch_size = (kBenchmarkValues + 9) / 10; + while (state.KeepRunningBatch(batch_size)) { + const int i = state.iterations(); + + const int start_index = i % kBenchmarkValues; + + state.PauseTiming(); + { + std::vector<V> removed; + removed.reserve(batch_size); + auto itr = container.find(key_of_value(values[start_index])); + auto start = itr; + for (int j = 0; j < batch_size; j++) { + if (itr == container.end()) { + state.ResumeTiming(); + container.erase(start, itr); + state.PauseTiming(); + itr = container.begin(); + start = itr; + } + removed.push_back(*itr++); + } + + state.ResumeTiming(); + container.erase(start, itr); + state.PauseTiming(); + + container.insert(removed.begin(), removed.end()); + } + state.ResumeTiming(); + } +} + +// Benchmark steady-state insert (into first half of range) and remove (from +// second half of range), treating the container approximately like a queue with +// log-time access for all elements. This benchmark does not test the case where +// insertion and removal happen in the same region of the tree. This benchmark +// counts two value constructors. +template <typename T> +void BM_QueueAddRem(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + + ABSL_RAW_CHECK(kBenchmarkValues % 2 == 0, "for performance"); + + T container; + + const size_t half = kBenchmarkValues / 2; + std::vector<int> remove_keys(half); + std::vector<int> add_keys(half); + + // We want to do the exact same work repeatedly, and the benchmark can end + // after a different number of iterations depending on the speed of the + // individual run so we use a large batch size here and ensure that we do + // deterministic work every batch. + while (state.KeepRunningBatch(half * kAddRemBatchSize)) { + state.PauseTiming(); + + container.clear(); + + for (size_t i = 0; i < half; ++i) { + remove_keys[i] = i; + add_keys[i] = i; + } + constexpr int kSeed = 5; + std::mt19937_64 rand(kSeed); + std::shuffle(remove_keys.begin(), remove_keys.end(), rand); + std::shuffle(add_keys.begin(), add_keys.end(), rand); + + // Note needs lazy generation of values. + Generator<V> g(kBenchmarkValues * kAddRemBatchSize); + + for (size_t i = 0; i < half; ++i) { + container.insert(g(add_keys[i])); + container.insert(g(half + remove_keys[i])); + } + + // There are three parts each of size "half": + // 1 is being deleted from [offset - half, offset) + // 2 is standing [offset, offset + half) + // 3 is being inserted into [offset + half, offset + 2 * half) + size_t offset = 0; + + for (size_t i = 0; i < kAddRemBatchSize; ++i) { + std::shuffle(remove_keys.begin(), remove_keys.end(), rand); + std::shuffle(add_keys.begin(), add_keys.end(), rand); + offset += half; + + state.ResumeTiming(); + for (size_t idx = 0; idx < half; ++idx) { + container.erase(key_of_value(g(offset - half + remove_keys[idx]))); + container.insert(g(offset + half + add_keys[idx])); + } + state.PauseTiming(); + } + state.ResumeTiming(); + } +} + +// Mixed insertion and deletion in the same range using pre-constructed values. +template <typename T> +void BM_MixedAddRem(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + + ABSL_RAW_CHECK(kBenchmarkValues % 2 == 0, "for performance"); + + T container; + + // Create two random shuffles + std::vector<int> remove_keys(kBenchmarkValues); + std::vector<int> add_keys(kBenchmarkValues); + + // We want to do the exact same work repeatedly, and the benchmark can end + // after a different number of iterations depending on the speed of the + // individual run so we use a large batch size here and ensure that we do + // deterministic work every batch. + while (state.KeepRunningBatch(kBenchmarkValues * kAddRemBatchSize)) { + state.PauseTiming(); + + container.clear(); + + constexpr int kSeed = 7; + std::mt19937_64 rand(kSeed); + + std::vector<V> values = GenerateValues<V>(kBenchmarkValues * 2); + + // Insert the first half of the values (already in random order) + container.insert(values.begin(), values.begin() + kBenchmarkValues); + + // Insert the first half of the values (already in random order) + for (size_t i = 0; i < kBenchmarkValues; ++i) { + // remove_keys and add_keys will be swapped before each round, + // therefore fill add_keys here w/ the keys being inserted, so + // they'll be the first to be removed. + remove_keys[i] = i + kBenchmarkValues; + add_keys[i] = i; + } + + for (size_t i = 0; i < kAddRemBatchSize; ++i) { + remove_keys.swap(add_keys); + std::shuffle(remove_keys.begin(), remove_keys.end(), rand); + std::shuffle(add_keys.begin(), add_keys.end(), rand); + + state.ResumeTiming(); + for (size_t idx = 0; idx < kBenchmarkValues; ++idx) { + container.erase(key_of_value(values[remove_keys[idx]])); + container.insert(values[add_keys[idx]]); + } + state.PauseTiming(); + } + state.ResumeTiming(); + } +} + +// Insertion at end, removal from the beginning. This benchmark +// counts two value constructors. +// TODO(ezb): we could add a GenerateNext version of generator that could reduce +// noise for string-like types. +template <typename T> +void BM_Fifo(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + + T container; + // Need lazy generation of values as state.max_iterations is large. + Generator<V> g(kBenchmarkValues + state.max_iterations); + + for (int i = 0; i < kBenchmarkValues; i++) { + container.insert(g(i)); + } + + while (state.KeepRunning()) { + container.erase(container.begin()); + container.insert(container.end(), g(state.iterations() + kBenchmarkValues)); + } +} + +// Iteration (forward) through the tree +template <typename T> +void BM_FwdIter(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + using R = typename T::value_type const*; + + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + T container(values.begin(), values.end()); + + auto iter = container.end(); + + R r = nullptr; + + while (state.KeepRunning()) { + if (iter == container.end()) iter = container.begin(); + r = &(*iter); + ++iter; + } + + benchmark::DoNotOptimize(r); +} + +// Benchmark random range-construction of a container. +template <typename T> +void BM_RangeConstructionImpl(benchmark::State& state, bool sorted) { + using V = typename remove_pair_const<typename T::value_type>::type; + + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + if (sorted) { + std::sort(values.begin(), values.end()); + } + { + T container(values.begin(), values.end()); + } + + while (state.KeepRunning()) { + T container(values.begin(), values.end()); + benchmark::DoNotOptimize(container); + } +} + +template <typename T> +void BM_InsertRangeRandom(benchmark::State& state) { + BM_RangeConstructionImpl<T>(state, false); +} + +template <typename T> +void BM_InsertRangeSorted(benchmark::State& state) { + BM_RangeConstructionImpl<T>(state, true); +} + +#define STL_ORDERED_TYPES(value) \ + using stl_set_##value = std::set<value>; \ + using stl_map_##value = std::map<value, intptr_t>; \ + using stl_multiset_##value = std::multiset<value>; \ + using stl_multimap_##value = std::multimap<value, intptr_t> + +using StdString = std::string; +STL_ORDERED_TYPES(int32_t); +STL_ORDERED_TYPES(int64_t); +STL_ORDERED_TYPES(StdString); +STL_ORDERED_TYPES(Cord); +STL_ORDERED_TYPES(Time); + +#define STL_UNORDERED_TYPES(value) \ + using stl_unordered_set_##value = std::unordered_set<value>; \ + using stl_unordered_map_##value = std::unordered_map<value, intptr_t>; \ + using flat_hash_set_##value = flat_hash_set<value>; \ + using flat_hash_map_##value = flat_hash_map<value, intptr_t>; \ + using stl_unordered_multiset_##value = std::unordered_multiset<value>; \ + using stl_unordered_multimap_##value = \ + std::unordered_multimap<value, intptr_t> + +#define STL_UNORDERED_TYPES_CUSTOM_HASH(value, hash) \ + using stl_unordered_set_##value = std::unordered_set<value, hash>; \ + using stl_unordered_map_##value = std::unordered_map<value, intptr_t, hash>; \ + using flat_hash_set_##value = flat_hash_set<value, hash>; \ + using flat_hash_map_##value = flat_hash_map<value, intptr_t, hash>; \ + using stl_unordered_multiset_##value = std::unordered_multiset<value, hash>; \ + using stl_unordered_multimap_##value = \ + std::unordered_multimap<value, intptr_t, hash> + +STL_UNORDERED_TYPES_CUSTOM_HASH(Cord, absl::Hash<absl::Cord>); + +STL_UNORDERED_TYPES(int32_t); +STL_UNORDERED_TYPES(int64_t); +STL_UNORDERED_TYPES(StdString); +STL_UNORDERED_TYPES_CUSTOM_HASH(Time, absl::Hash<absl::Time>); + +#define BTREE_TYPES(value) \ + using btree_256_set_##value = \ + btree_set<value, std::less<value>, std::allocator<value>>; \ + using btree_256_map_##value = \ + btree_map<value, intptr_t, std::less<value>, \ + std::allocator<std::pair<const value, intptr_t>>>; \ + using btree_256_multiset_##value = \ + btree_multiset<value, std::less<value>, std::allocator<value>>; \ + using btree_256_multimap_##value = \ + btree_multimap<value, intptr_t, std::less<value>, \ + std::allocator<std::pair<const value, intptr_t>>> + +BTREE_TYPES(int32_t); +BTREE_TYPES(int64_t); +BTREE_TYPES(StdString); +BTREE_TYPES(Cord); +BTREE_TYPES(Time); + +#define MY_BENCHMARK4(type, func) \ + void BM_##type##_##func(benchmark::State& state) { BM_##func<type>(state); } \ + BENCHMARK(BM_##type##_##func) + +#define MY_BENCHMARK3(type) \ + MY_BENCHMARK4(type, Insert); \ + MY_BENCHMARK4(type, InsertSorted); \ + MY_BENCHMARK4(type, InsertEnd); \ + MY_BENCHMARK4(type, InsertSmall); \ + MY_BENCHMARK4(type, Lookup); \ + MY_BENCHMARK4(type, FullLookup); \ + MY_BENCHMARK4(type, Delete); \ + MY_BENCHMARK4(type, DeleteRange); \ + MY_BENCHMARK4(type, QueueAddRem); \ + MY_BENCHMARK4(type, MixedAddRem); \ + MY_BENCHMARK4(type, Fifo); \ + MY_BENCHMARK4(type, FwdIter); \ + MY_BENCHMARK4(type, InsertRangeRandom); \ + MY_BENCHMARK4(type, InsertRangeSorted) + +#define MY_BENCHMARK2_SUPPORTS_MULTI_ONLY(type) \ + MY_BENCHMARK3(stl_##type); \ + MY_BENCHMARK3(stl_unordered_##type); \ + MY_BENCHMARK3(btree_256_##type) + +#define MY_BENCHMARK2(type) \ + MY_BENCHMARK2_SUPPORTS_MULTI_ONLY(type); \ + MY_BENCHMARK3(flat_hash_##type) + +// Define MULTI_TESTING to see benchmarks for multi-containers also. +// +// You can use --copt=-DMULTI_TESTING. +#ifdef MULTI_TESTING +#define MY_BENCHMARK(type) \ + MY_BENCHMARK2(set_##type); \ + MY_BENCHMARK2(map_##type); \ + MY_BENCHMARK2_SUPPORTS_MULTI_ONLY(multiset_##type); \ + MY_BENCHMARK2_SUPPORTS_MULTI_ONLY(multimap_##type) +#else +#define MY_BENCHMARK(type) \ + MY_BENCHMARK2(set_##type); \ + MY_BENCHMARK2(map_##type) +#endif + +MY_BENCHMARK(int32_t); +MY_BENCHMARK(int64_t); +MY_BENCHMARK(StdString); +MY_BENCHMARK(Cord); +MY_BENCHMARK(Time); + +// Define a type whose size and cost of moving are independently customizable. +// When sizeof(value_type) increases, we expect btree to no longer have as much +// cache-locality advantage over STL. When cost of moving increases, we expect +// btree to actually do more work than STL because it has to move values around +// and STL doesn't have to. +template <int Size, int Copies> +struct BigType { + BigType() : BigType(0) {} + explicit BigType(int x) { std::iota(values.begin(), values.end(), x); } + + void Copy(const BigType& other) { + for (int i = 0; i < Size && i < Copies; ++i) values[i] = other.values[i]; + // If Copies > Size, do extra copies. + for (int i = Size, idx = 0; i < Copies; ++i) { + int64_t tmp = other.values[idx]; + benchmark::DoNotOptimize(tmp); + idx = idx + 1 == Size ? 0 : idx + 1; + } + } + + BigType(const BigType& other) { Copy(other); } + BigType& operator=(const BigType& other) { + Copy(other); + return *this; + } + + // Compare only the first Copies elements if Copies is less than Size. + bool operator<(const BigType& other) const { + return std::lexicographical_compare( + values.begin(), values.begin() + std::min(Size, Copies), + other.values.begin(), other.values.begin() + std::min(Size, Copies)); + } + bool operator==(const BigType& other) const { + return std::equal(values.begin(), values.begin() + std::min(Size, Copies), + other.values.begin()); + } + + // Support absl::Hash. + template <typename State> + friend State AbslHashValue(State h, const BigType& b) { + for (int i = 0; i < Size && i < Copies; ++i) + h = State::combine(std::move(h), b.values[i]); + return h; + } + + std::array<int64_t, Size> values; +}; + +#define BIG_TYPE_BENCHMARKS(SIZE, COPIES) \ + using stl_set_size##SIZE##copies##COPIES = std::set<BigType<SIZE, COPIES>>; \ + using stl_map_size##SIZE##copies##COPIES = \ + std::map<BigType<SIZE, COPIES>, intptr_t>; \ + using stl_multiset_size##SIZE##copies##COPIES = \ + std::multiset<BigType<SIZE, COPIES>>; \ + using stl_multimap_size##SIZE##copies##COPIES = \ + std::multimap<BigType<SIZE, COPIES>, intptr_t>; \ + using stl_unordered_set_size##SIZE##copies##COPIES = \ + std::unordered_set<BigType<SIZE, COPIES>, \ + absl::Hash<BigType<SIZE, COPIES>>>; \ + using stl_unordered_map_size##SIZE##copies##COPIES = \ + std::unordered_map<BigType<SIZE, COPIES>, intptr_t, \ + absl::Hash<BigType<SIZE, COPIES>>>; \ + using flat_hash_set_size##SIZE##copies##COPIES = \ + flat_hash_set<BigType<SIZE, COPIES>>; \ + using flat_hash_map_size##SIZE##copies##COPIES = \ + flat_hash_map<BigType<SIZE, COPIES>, intptr_t>; \ + using stl_unordered_multiset_size##SIZE##copies##COPIES = \ + std::unordered_multiset<BigType<SIZE, COPIES>, \ + absl::Hash<BigType<SIZE, COPIES>>>; \ + using stl_unordered_multimap_size##SIZE##copies##COPIES = \ + std::unordered_multimap<BigType<SIZE, COPIES>, intptr_t, \ + absl::Hash<BigType<SIZE, COPIES>>>; \ + using btree_256_set_size##SIZE##copies##COPIES = \ + btree_set<BigType<SIZE, COPIES>>; \ + using btree_256_map_size##SIZE##copies##COPIES = \ + btree_map<BigType<SIZE, COPIES>, intptr_t>; \ + using btree_256_multiset_size##SIZE##copies##COPIES = \ + btree_multiset<BigType<SIZE, COPIES>>; \ + using btree_256_multimap_size##SIZE##copies##COPIES = \ + btree_multimap<BigType<SIZE, COPIES>, intptr_t>; \ + MY_BENCHMARK(size##SIZE##copies##COPIES) + +// Define BIG_TYPE_TESTING to see benchmarks for more big types. +// +// You can use --copt=-DBIG_TYPE_TESTING. +#ifndef NODESIZE_TESTING +#ifdef BIG_TYPE_TESTING +BIG_TYPE_BENCHMARKS(1, 4); +BIG_TYPE_BENCHMARKS(4, 1); +BIG_TYPE_BENCHMARKS(4, 4); +BIG_TYPE_BENCHMARKS(1, 8); +BIG_TYPE_BENCHMARKS(8, 1); +BIG_TYPE_BENCHMARKS(8, 8); +BIG_TYPE_BENCHMARKS(1, 16); +BIG_TYPE_BENCHMARKS(16, 1); +BIG_TYPE_BENCHMARKS(16, 16); +BIG_TYPE_BENCHMARKS(1, 32); +BIG_TYPE_BENCHMARKS(32, 1); +BIG_TYPE_BENCHMARKS(32, 32); +#else +BIG_TYPE_BENCHMARKS(32, 32); +#endif +#endif + +// Benchmark using unique_ptrs to large value types. In order to be able to use +// the same benchmark code as the other types, use a type that holds a +// unique_ptr and has a copy constructor. +template <int Size> +struct BigTypePtr { + BigTypePtr() : BigTypePtr(0) {} + explicit BigTypePtr(int x) { + ptr = absl::make_unique<BigType<Size, Size>>(x); + } + BigTypePtr(const BigTypePtr& other) { + ptr = absl::make_unique<BigType<Size, Size>>(*other.ptr); + } + BigTypePtr(BigTypePtr&& other) noexcept = default; + BigTypePtr& operator=(const BigTypePtr& other) { + ptr = absl::make_unique<BigType<Size, Size>>(*other.ptr); + } + BigTypePtr& operator=(BigTypePtr&& other) noexcept = default; + + bool operator<(const BigTypePtr& other) const { return *ptr < *other.ptr; } + bool operator==(const BigTypePtr& other) const { return *ptr == *other.ptr; } + + std::unique_ptr<BigType<Size, Size>> ptr; +}; + +template <int Size> +double ContainerInfo(const btree_set<BigTypePtr<Size>>& b) { + const double bytes_used = + b.bytes_used() + b.size() * sizeof(BigType<Size, Size>); + const double bytes_per_value = bytes_used / b.size(); + BtreeContainerInfoLog(b, bytes_used, bytes_per_value); + return bytes_per_value; +} +template <int Size> +double ContainerInfo(const btree_map<int, BigTypePtr<Size>>& b) { + const double bytes_used = + b.bytes_used() + b.size() * sizeof(BigType<Size, Size>); + const double bytes_per_value = bytes_used / b.size(); + BtreeContainerInfoLog(b, bytes_used, bytes_per_value); + return bytes_per_value; +} + +#define BIG_TYPE_PTR_BENCHMARKS(SIZE) \ + using stl_set_size##SIZE##copies##SIZE##ptr = std::set<BigType<SIZE, SIZE>>; \ + using stl_map_size##SIZE##copies##SIZE##ptr = \ + std::map<int, BigType<SIZE, SIZE>>; \ + using stl_unordered_set_size##SIZE##copies##SIZE##ptr = \ + std::unordered_set<BigType<SIZE, SIZE>, \ + absl::Hash<BigType<SIZE, SIZE>>>; \ + using stl_unordered_map_size##SIZE##copies##SIZE##ptr = \ + std::unordered_map<int, BigType<SIZE, SIZE>>; \ + using flat_hash_set_size##SIZE##copies##SIZE##ptr = \ + flat_hash_set<BigType<SIZE, SIZE>>; \ + using flat_hash_map_size##SIZE##copies##SIZE##ptr = \ + flat_hash_map<int, BigTypePtr<SIZE>>; \ + using btree_256_set_size##SIZE##copies##SIZE##ptr = \ + btree_set<BigTypePtr<SIZE>>; \ + using btree_256_map_size##SIZE##copies##SIZE##ptr = \ + btree_map<int, BigTypePtr<SIZE>>; \ + MY_BENCHMARK3(stl_set_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(stl_unordered_set_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(flat_hash_set_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(btree_256_set_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(stl_map_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(stl_unordered_map_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(flat_hash_map_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3(btree_256_map_size##SIZE##copies##SIZE##ptr) + +BIG_TYPE_PTR_BENCHMARKS(32); + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/btree_map.h b/third_party/abseil_cpp/absl/container/btree_map.h new file mode 100644 index 000000000000..abc09b0ac068 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/btree_map.h @@ -0,0 +1,769 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: btree_map.h +// ----------------------------------------------------------------------------- +// +// This header file defines B-tree maps: sorted associative containers mapping +// keys to values. +// +// * `absl::btree_map<>` +// * `absl::btree_multimap<>` +// +// These B-tree types are similar to the corresponding types in the STL +// (`std::map` and `std::multimap`) and generally conform to the STL interfaces +// of those types. However, because they are implemented using B-trees, they +// are more efficient in most situations. +// +// Unlike `std::map` and `std::multimap`, which are commonly implemented using +// red-black tree nodes, B-tree maps use more generic B-tree nodes able to hold +// multiple values per node. Holding multiple values per node often makes +// B-tree maps perform better than their `std::map` counterparts, because +// multiple entries can be checked within the same cache hit. +// +// However, these types should not be considered drop-in replacements for +// `std::map` and `std::multimap` as there are some API differences, which are +// noted in this header file. +// +// Importantly, insertions and deletions may invalidate outstanding iterators, +// pointers, and references to elements. Such invalidations are typically only +// an issue if insertion and deletion operations are interleaved with the use of +// more than one iterator, pointer, or reference simultaneously. For this +// reason, `insert()` and `erase()` return a valid iterator at the current +// position. + +#ifndef ABSL_CONTAINER_BTREE_MAP_H_ +#define ABSL_CONTAINER_BTREE_MAP_H_ + +#include "absl/container/internal/btree.h" // IWYU pragma: export +#include "absl/container/internal/btree_container.h" // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::btree_map<> +// +// An `absl::btree_map<K, V>` is an ordered associative container of +// unique keys and associated values designed to be a more efficient replacement +// for `std::map` (in most cases). +// +// Keys are sorted using an (optional) comparison function, which defaults to +// `std::less<K>`. +// +// An `absl::btree_map<K, V>` uses a default allocator of +// `std::allocator<std::pair<const K, V>>` to allocate (and deallocate) +// nodes, and construct and destruct values within those nodes. You may +// instead specify a custom allocator `A` (which in turn requires specifying a +// custom comparator `C`) as in `absl::btree_map<K, V, C, A>`. +// +template <typename Key, typename Value, typename Compare = std::less<Key>, + typename Alloc = std::allocator<std::pair<const Key, Value>>> +class btree_map + : public container_internal::btree_map_container< + container_internal::btree<container_internal::map_params< + Key, Value, Compare, Alloc, /*TargetNodeSize=*/256, + /*Multi=*/false>>> { + using Base = typename btree_map::btree_map_container; + + public: + // Constructors and Assignment Operators + // + // A `btree_map` supports the same overload set as `std::map` + // for construction and assignment: + // + // * Default constructor + // + // absl::btree_map<int, std::string> map1; + // + // * Initializer List constructor + // + // absl::btree_map<int, std::string> map2 = + // {{1, "huey"}, {2, "dewey"}, {3, "louie"},}; + // + // * Copy constructor + // + // absl::btree_map<int, std::string> map3(map2); + // + // * Copy assignment operator + // + // absl::btree_map<int, std::string> map4; + // map4 = map3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::btree_map<int, std::string> map5(std::move(map4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::btree_map<int, std::string> map6; + // map6 = std::move(map5); + // + // * Range constructor + // + // std::vector<std::pair<int, std::string>> v = {{1, "a"}, {2, "b"}}; + // absl::btree_map<int, std::string> map7(v.begin(), v.end()); + btree_map() {} + using Base::Base; + + // btree_map::begin() + // + // Returns an iterator to the beginning of the `btree_map`. + using Base::begin; + + // btree_map::cbegin() + // + // Returns a const iterator to the beginning of the `btree_map`. + using Base::cbegin; + + // btree_map::end() + // + // Returns an iterator to the end of the `btree_map`. + using Base::end; + + // btree_map::cend() + // + // Returns a const iterator to the end of the `btree_map`. + using Base::cend; + + // btree_map::empty() + // + // Returns whether or not the `btree_map` is empty. + using Base::empty; + + // btree_map::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `btree_map` under current memory constraints. This value can be thought + // of as the largest value of `std::distance(begin(), end())` for a + // `btree_map<Key, T>`. + using Base::max_size; + + // btree_map::size() + // + // Returns the number of elements currently within the `btree_map`. + using Base::size; + + // btree_map::clear() + // + // Removes all elements from the `btree_map`. Invalidates any references, + // pointers, or iterators referring to contained elements. + using Base::clear; + + // btree_map::erase() + // + // Erases elements within the `btree_map`. If an erase occurs, any references, + // pointers, or iterators are invalidated. + // Overloads are listed below. + // + // iterator erase(iterator position): + // iterator erase(const_iterator position): + // + // Erases the element at `position` of the `btree_map`, returning + // the iterator pointing to the element after the one that was erased + // (or end() if none exists). + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning + // the iterator pointing to the element after the interval that was erased + // (or end() if none exists). + // + // template <typename K> size_type erase(const K& key): + // + // Erases the element with the matching key, if it exists, returning the + // number of elements erased (0 or 1). + using Base::erase; + + // btree_map::insert() + // + // Inserts an element of the specified value into the `btree_map`, + // returning an iterator pointing to the newly inserted element, provided that + // an element with the given key does not already exist. If an insertion + // occurs, any references, pointers, or iterators are invalidated. + // Overloads are listed below. + // + // std::pair<iterator,bool> insert(const value_type& value): + // + // Inserts a value into the `btree_map`. Returns a pair consisting of an + // iterator to the inserted element (or to the element that prevented the + // insertion) and a bool denoting whether the insertion took place. + // + // std::pair<iterator,bool> insert(value_type&& value): + // + // Inserts a moveable value into the `btree_map`. Returns a pair + // consisting of an iterator to the inserted element (or to the element that + // prevented the insertion) and a bool denoting whether the insertion took + // place. + // + // iterator insert(const_iterator hint, const value_type& value): + // iterator insert(const_iterator hint, value_type&& value): + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element, or to the existing element that prevented the + // insertion. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // void insert(std::initializer_list<init_type> ilist): + // + // Inserts the elements within the initializer list `ilist`. + using Base::insert; + + // btree_map::insert_or_assign() + // + // Inserts an element of the specified value into the `btree_map` provided + // that a value with the given key does not already exist, or replaces the + // corresponding mapped type with the forwarded `obj` argument if a key for + // that value already exists, returning an iterator pointing to the newly + // inserted element. Overloads are listed below. + // + // pair<iterator, bool> insert_or_assign(const key_type& k, M&& obj): + // pair<iterator, bool> insert_or_assign(key_type&& k, M&& obj): + // + // Inserts/Assigns (or moves) the element of the specified key into the + // `btree_map`. If the returned bool is true, insertion took place, and if + // it's false, assignment took place. + // + // iterator insert_or_assign(const_iterator hint, + // const key_type& k, M&& obj): + // iterator insert_or_assign(const_iterator hint, key_type&& k, M&& obj): + // + // Inserts/Assigns (or moves) the element of the specified key into the + // `btree_map` using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. + using Base::insert_or_assign; + + // btree_map::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_map`, provided that no element with the given key + // already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. Prefer `try_emplace()` unless your key is not + // copyable or moveable. + // + // If an insertion occurs, any references, pointers, or iterators are + // invalidated. + using Base::emplace; + + // btree_map::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_map`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search, and only inserts + // provided that no element with the given key already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. Prefer `try_emplace()` unless your key is not + // copyable or moveable. + // + // If an insertion occurs, any references, pointers, or iterators are + // invalidated. + using Base::emplace_hint; + + // btree_map::try_emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_map`, provided that no element with the given key + // already exists. Unlike `emplace()`, if an element with the given key + // already exists, we guarantee that no element is constructed. + // + // If an insertion occurs, any references, pointers, or iterators are + // invalidated. + // + // Overloads are listed below. + // + // std::pair<iterator, bool> try_emplace(const key_type& k, Args&&... args): + // std::pair<iterator, bool> try_emplace(key_type&& k, Args&&... args): + // + // Inserts (via copy or move) the element of the specified key into the + // `btree_map`. + // + // iterator try_emplace(const_iterator hint, + // const key_type& k, Args&&... args): + // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args): + // + // Inserts (via copy or move) the element of the specified key into the + // `btree_map` using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. + using Base::try_emplace; + + // btree_map::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the element at the indicated position and returns a node handle + // owning that extracted data. + // + // template <typename K> node_type extract(const K& k): + // + // Extracts the element with the key matching the passed key value and + // returns a node handle owning that extracted data. If the `btree_map` + // does not contain an element with a matching key, this function returns an + // empty node handle. + // + // NOTE: when compiled in an earlier version of C++ than C++17, + // `node_type::key()` returns a const reference to the key instead of a + // mutable reference. We cannot safely return a mutable reference without + // std::launder (which is not available before C++17). + // + // NOTE: In this context, `node_type` refers to the C++17 concept of a + // move-only type that owns and provides access to the elements in associative + // containers (https://en.cppreference.com/w/cpp/container/node_handle). + // It does NOT refer to the data layout of the underlying btree. + using Base::extract; + + // btree_map::merge() + // + // Extracts elements from a given `source` btree_map into this + // `btree_map`. If the destination `btree_map` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // btree_map::swap(btree_map& other) + // + // Exchanges the contents of this `btree_map` with those of the `other` + // btree_map, avoiding invocation of any move, copy, or swap operations on + // individual elements. + // + // All iterators and references on the `btree_map` remain valid, excepting + // for the past-the-end iterator, which is invalidated. + using Base::swap; + + // btree_map::at() + // + // Returns a reference to the mapped value of the element with key equivalent + // to the passed key. + using Base::at; + + // btree_map::contains() + // + // template <typename K> bool contains(const K& key) const: + // + // Determines whether an element comparing equal to the given `key` exists + // within the `btree_map`, returning `true` if so or `false` otherwise. + // + // Supports heterogeneous lookup, provided that the map is provided a + // compatible heterogeneous comparator. + using Base::contains; + + // btree_map::count() + // + // template <typename K> size_type count(const K& key) const: + // + // Returns the number of elements comparing equal to the given `key` within + // the `btree_map`. Note that this function will return either `1` or `0` + // since duplicate elements are not allowed within a `btree_map`. + // + // Supports heterogeneous lookup, provided that the map is provided a + // compatible heterogeneous comparator. + using Base::count; + + // btree_map::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `btree_map`. + using Base::equal_range; + + // btree_map::find() + // + // template <typename K> iterator find(const K& key): + // template <typename K> const_iterator find(const K& key) const: + // + // Finds an element with the passed `key` within the `btree_map`. + // + // Supports heterogeneous lookup, provided that the map is provided a + // compatible heterogeneous comparator. + using Base::find; + + // btree_map::operator[]() + // + // Returns a reference to the value mapped to the passed key within the + // `btree_map`, performing an `insert()` if the key does not already + // exist. + // + // If an insertion occurs, any references, pointers, or iterators are + // invalidated. Otherwise iterators are not affected and references are not + // invalidated. Overloads are listed below. + // + // T& operator[](key_type&& key): + // T& operator[](const key_type& key): + // + // Inserts a value_type object constructed in-place if the element with the + // given key does not exist. + using Base::operator[]; + + // btree_map::get_allocator() + // + // Returns the allocator function associated with this `btree_map`. + using Base::get_allocator; + + // btree_map::key_comp(); + // + // Returns the key comparator associated with this `btree_map`. + using Base::key_comp; + + // btree_map::value_comp(); + // + // Returns the value comparator associated with this `btree_map`. + using Base::value_comp; +}; + +// absl::swap(absl::btree_map<>, absl::btree_map<>) +// +// Swaps the contents of two `absl::btree_map` containers. +template <typename K, typename V, typename C, typename A> +void swap(btree_map<K, V, C, A> &x, btree_map<K, V, C, A> &y) { + return x.swap(y); +} + +// absl::erase_if(absl::btree_map<>, Pred) +// +// Erases all elements that satisfy the predicate pred from the container. +template <typename K, typename V, typename C, typename A, typename Pred> +void erase_if(btree_map<K, V, C, A> &map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } +} + +// absl::btree_multimap +// +// An `absl::btree_multimap<K, V>` is an ordered associative container of +// keys and associated values designed to be a more efficient replacement for +// `std::multimap` (in most cases). Unlike `absl::btree_map`, a B-tree multimap +// allows multiple elements with equivalent keys. +// +// Keys are sorted using an (optional) comparison function, which defaults to +// `std::less<K>`. +// +// An `absl::btree_multimap<K, V>` uses a default allocator of +// `std::allocator<std::pair<const K, V>>` to allocate (and deallocate) +// nodes, and construct and destruct values within those nodes. You may +// instead specify a custom allocator `A` (which in turn requires specifying a +// custom comparator `C`) as in `absl::btree_multimap<K, V, C, A>`. +// +template <typename Key, typename Value, typename Compare = std::less<Key>, + typename Alloc = std::allocator<std::pair<const Key, Value>>> +class btree_multimap + : public container_internal::btree_multimap_container< + container_internal::btree<container_internal::map_params< + Key, Value, Compare, Alloc, /*TargetNodeSize=*/256, + /*Multi=*/true>>> { + using Base = typename btree_multimap::btree_multimap_container; + + public: + // Constructors and Assignment Operators + // + // A `btree_multimap` supports the same overload set as `std::multimap` + // for construction and assignment: + // + // * Default constructor + // + // absl::btree_multimap<int, std::string> map1; + // + // * Initializer List constructor + // + // absl::btree_multimap<int, std::string> map2 = + // {{1, "huey"}, {2, "dewey"}, {3, "louie"},}; + // + // * Copy constructor + // + // absl::btree_multimap<int, std::string> map3(map2); + // + // * Copy assignment operator + // + // absl::btree_multimap<int, std::string> map4; + // map4 = map3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::btree_multimap<int, std::string> map5(std::move(map4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::btree_multimap<int, std::string> map6; + // map6 = std::move(map5); + // + // * Range constructor + // + // std::vector<std::pair<int, std::string>> v = {{1, "a"}, {2, "b"}}; + // absl::btree_multimap<int, std::string> map7(v.begin(), v.end()); + btree_multimap() {} + using Base::Base; + + // btree_multimap::begin() + // + // Returns an iterator to the beginning of the `btree_multimap`. + using Base::begin; + + // btree_multimap::cbegin() + // + // Returns a const iterator to the beginning of the `btree_multimap`. + using Base::cbegin; + + // btree_multimap::end() + // + // Returns an iterator to the end of the `btree_multimap`. + using Base::end; + + // btree_multimap::cend() + // + // Returns a const iterator to the end of the `btree_multimap`. + using Base::cend; + + // btree_multimap::empty() + // + // Returns whether or not the `btree_multimap` is empty. + using Base::empty; + + // btree_multimap::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `btree_multimap` under current memory constraints. This value can be + // thought of as the largest value of `std::distance(begin(), end())` for a + // `btree_multimap<Key, T>`. + using Base::max_size; + + // btree_multimap::size() + // + // Returns the number of elements currently within the `btree_multimap`. + using Base::size; + + // btree_multimap::clear() + // + // Removes all elements from the `btree_multimap`. Invalidates any references, + // pointers, or iterators referring to contained elements. + using Base::clear; + + // btree_multimap::erase() + // + // Erases elements within the `btree_multimap`. If an erase occurs, any + // references, pointers, or iterators are invalidated. + // Overloads are listed below. + // + // iterator erase(iterator position): + // iterator erase(const_iterator position): + // + // Erases the element at `position` of the `btree_multimap`, returning + // the iterator pointing to the element after the one that was erased + // (or end() if none exists). + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning + // the iterator pointing to the element after the interval that was erased + // (or end() if none exists). + // + // template <typename K> size_type erase(const K& key): + // + // Erases the elements matching the key, if any exist, returning the + // number of elements erased. + using Base::erase; + + // btree_multimap::insert() + // + // Inserts an element of the specified value into the `btree_multimap`, + // returning an iterator pointing to the newly inserted element. + // Any references, pointers, or iterators are invalidated. Overloads are + // listed below. + // + // iterator insert(const value_type& value): + // + // Inserts a value into the `btree_multimap`, returning an iterator to the + // inserted element. + // + // iterator insert(value_type&& value): + // + // Inserts a moveable value into the `btree_multimap`, returning an iterator + // to the inserted element. + // + // iterator insert(const_iterator hint, const value_type& value): + // iterator insert(const_iterator hint, value_type&& value): + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // void insert(std::initializer_list<init_type> ilist): + // + // Inserts the elements within the initializer list `ilist`. + using Base::insert; + + // btree_multimap::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_multimap`. Any references, pointers, or iterators are + // invalidated. + using Base::emplace; + + // btree_multimap::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_multimap`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search. + // + // Any references, pointers, or iterators are invalidated. + using Base::emplace_hint; + + // btree_multimap::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the element at the indicated position and returns a node handle + // owning that extracted data. + // + // template <typename K> node_type extract(const K& k): + // + // Extracts the element with the key matching the passed key value and + // returns a node handle owning that extracted data. If the `btree_multimap` + // does not contain an element with a matching key, this function returns an + // empty node handle. + // + // NOTE: when compiled in an earlier version of C++ than C++17, + // `node_type::key()` returns a const reference to the key instead of a + // mutable reference. We cannot safely return a mutable reference without + // std::launder (which is not available before C++17). + // + // NOTE: In this context, `node_type` refers to the C++17 concept of a + // move-only type that owns and provides access to the elements in associative + // containers (https://en.cppreference.com/w/cpp/container/node_handle). + // It does NOT refer to the data layout of the underlying btree. + using Base::extract; + + // btree_multimap::merge() + // + // Extracts elements from a given `source` btree_multimap into this + // `btree_multimap`. If the destination `btree_multimap` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // btree_multimap::swap(btree_multimap& other) + // + // Exchanges the contents of this `btree_multimap` with those of the `other` + // btree_multimap, avoiding invocation of any move, copy, or swap operations + // on individual elements. + // + // All iterators and references on the `btree_multimap` remain valid, + // excepting for the past-the-end iterator, which is invalidated. + using Base::swap; + + // btree_multimap::contains() + // + // template <typename K> bool contains(const K& key) const: + // + // Determines whether an element comparing equal to the given `key` exists + // within the `btree_multimap`, returning `true` if so or `false` otherwise. + // + // Supports heterogeneous lookup, provided that the map is provided a + // compatible heterogeneous comparator. + using Base::contains; + + // btree_multimap::count() + // + // template <typename K> size_type count(const K& key) const: + // + // Returns the number of elements comparing equal to the given `key` within + // the `btree_multimap`. + // + // Supports heterogeneous lookup, provided that the map is provided a + // compatible heterogeneous comparator. + using Base::count; + + // btree_multimap::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `btree_multimap`. + using Base::equal_range; + + // btree_multimap::find() + // + // template <typename K> iterator find(const K& key): + // template <typename K> const_iterator find(const K& key) const: + // + // Finds an element with the passed `key` within the `btree_multimap`. + // + // Supports heterogeneous lookup, provided that the map is provided a + // compatible heterogeneous comparator. + using Base::find; + + // btree_multimap::get_allocator() + // + // Returns the allocator function associated with this `btree_multimap`. + using Base::get_allocator; + + // btree_multimap::key_comp(); + // + // Returns the key comparator associated with this `btree_multimap`. + using Base::key_comp; + + // btree_multimap::value_comp(); + // + // Returns the value comparator associated with this `btree_multimap`. + using Base::value_comp; +}; + +// absl::swap(absl::btree_multimap<>, absl::btree_multimap<>) +// +// Swaps the contents of two `absl::btree_multimap` containers. +template <typename K, typename V, typename C, typename A> +void swap(btree_multimap<K, V, C, A> &x, btree_multimap<K, V, C, A> &y) { + return x.swap(y); +} + +// absl::erase_if(absl::btree_multimap<>, Pred) +// +// Erases all elements that satisfy the predicate pred from the container. +template <typename K, typename V, typename C, typename A, typename Pred> +void erase_if(btree_multimap<K, V, C, A> &map, Pred pred) { + for (auto it = map.begin(); it != map.end();) { + if (pred(*it)) { + it = map.erase(it); + } else { + ++it; + } + } +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_BTREE_MAP_H_ diff --git a/third_party/abseil_cpp/absl/container/btree_set.h b/third_party/abseil_cpp/absl/container/btree_set.h new file mode 100644 index 000000000000..21ef0a032a85 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/btree_set.h @@ -0,0 +1,683 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: btree_set.h +// ----------------------------------------------------------------------------- +// +// This header file defines B-tree sets: sorted associative containers of +// values. +// +// * `absl::btree_set<>` +// * `absl::btree_multiset<>` +// +// These B-tree types are similar to the corresponding types in the STL +// (`std::set` and `std::multiset`) and generally conform to the STL interfaces +// of those types. However, because they are implemented using B-trees, they +// are more efficient in most situations. +// +// Unlike `std::set` and `std::multiset`, which are commonly implemented using +// red-black tree nodes, B-tree sets use more generic B-tree nodes able to hold +// multiple values per node. Holding multiple values per node often makes +// B-tree sets perform better than their `std::set` counterparts, because +// multiple entries can be checked within the same cache hit. +// +// However, these types should not be considered drop-in replacements for +// `std::set` and `std::multiset` as there are some API differences, which are +// noted in this header file. +// +// Importantly, insertions and deletions may invalidate outstanding iterators, +// pointers, and references to elements. Such invalidations are typically only +// an issue if insertion and deletion operations are interleaved with the use of +// more than one iterator, pointer, or reference simultaneously. For this +// reason, `insert()` and `erase()` return a valid iterator at the current +// position. + +#ifndef ABSL_CONTAINER_BTREE_SET_H_ +#define ABSL_CONTAINER_BTREE_SET_H_ + +#include "absl/container/internal/btree.h" // IWYU pragma: export +#include "absl/container/internal/btree_container.h" // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::btree_set<> +// +// An `absl::btree_set<K>` is an ordered associative container of unique key +// values designed to be a more efficient replacement for `std::set` (in most +// cases). +// +// Keys are sorted using an (optional) comparison function, which defaults to +// `std::less<K>`. +// +// An `absl::btree_set<K>` uses a default allocator of `std::allocator<K>` to +// allocate (and deallocate) nodes, and construct and destruct values within +// those nodes. You may instead specify a custom allocator `A` (which in turn +// requires specifying a custom comparator `C`) as in +// `absl::btree_set<K, C, A>`. +// +template <typename Key, typename Compare = std::less<Key>, + typename Alloc = std::allocator<Key>> +class btree_set + : public container_internal::btree_set_container< + container_internal::btree<container_internal::set_params< + Key, Compare, Alloc, /*TargetNodeSize=*/256, + /*Multi=*/false>>> { + using Base = typename btree_set::btree_set_container; + + public: + // Constructors and Assignment Operators + // + // A `btree_set` supports the same overload set as `std::set` + // for construction and assignment: + // + // * Default constructor + // + // absl::btree_set<std::string> set1; + // + // * Initializer List constructor + // + // absl::btree_set<std::string> set2 = + // {{"huey"}, {"dewey"}, {"louie"},}; + // + // * Copy constructor + // + // absl::btree_set<std::string> set3(set2); + // + // * Copy assignment operator + // + // absl::btree_set<std::string> set4; + // set4 = set3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::btree_set<std::string> set5(std::move(set4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::btree_set<std::string> set6; + // set6 = std::move(set5); + // + // * Range constructor + // + // std::vector<std::string> v = {"a", "b"}; + // absl::btree_set<std::string> set7(v.begin(), v.end()); + btree_set() {} + using Base::Base; + + // btree_set::begin() + // + // Returns an iterator to the beginning of the `btree_set`. + using Base::begin; + + // btree_set::cbegin() + // + // Returns a const iterator to the beginning of the `btree_set`. + using Base::cbegin; + + // btree_set::end() + // + // Returns an iterator to the end of the `btree_set`. + using Base::end; + + // btree_set::cend() + // + // Returns a const iterator to the end of the `btree_set`. + using Base::cend; + + // btree_set::empty() + // + // Returns whether or not the `btree_set` is empty. + using Base::empty; + + // btree_set::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `btree_set` under current memory constraints. This value can be thought + // of as the largest value of `std::distance(begin(), end())` for a + // `btree_set<Key>`. + using Base::max_size; + + // btree_set::size() + // + // Returns the number of elements currently within the `btree_set`. + using Base::size; + + // btree_set::clear() + // + // Removes all elements from the `btree_set`. Invalidates any references, + // pointers, or iterators referring to contained elements. + using Base::clear; + + // btree_set::erase() + // + // Erases elements within the `btree_set`. Overloads are listed below. + // + // iterator erase(iterator position): + // iterator erase(const_iterator position): + // + // Erases the element at `position` of the `btree_set`, returning + // the iterator pointing to the element after the one that was erased + // (or end() if none exists). + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning + // the iterator pointing to the element after the interval that was erased + // (or end() if none exists). + // + // template <typename K> size_type erase(const K& key): + // + // Erases the element with the matching key, if it exists, returning the + // number of elements erased (0 or 1). + using Base::erase; + + // btree_set::insert() + // + // Inserts an element of the specified value into the `btree_set`, + // returning an iterator pointing to the newly inserted element, provided that + // an element with the given key does not already exist. If an insertion + // occurs, any references, pointers, or iterators are invalidated. + // Overloads are listed below. + // + // std::pair<iterator,bool> insert(const value_type& value): + // + // Inserts a value into the `btree_set`. Returns a pair consisting of an + // iterator to the inserted element (or to the element that prevented the + // insertion) and a bool denoting whether the insertion took place. + // + // std::pair<iterator,bool> insert(value_type&& value): + // + // Inserts a moveable value into the `btree_set`. Returns a pair + // consisting of an iterator to the inserted element (or to the element that + // prevented the insertion) and a bool denoting whether the insertion took + // place. + // + // iterator insert(const_iterator hint, const value_type& value): + // iterator insert(const_iterator hint, value_type&& value): + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element, or to the existing element that prevented the + // insertion. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // void insert(std::initializer_list<init_type> ilist): + // + // Inserts the elements within the initializer list `ilist`. + using Base::insert; + + // btree_set::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_set`, provided that no element with the given key + // already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. + // + // If an insertion occurs, any references, pointers, or iterators are + // invalidated. + using Base::emplace; + + // btree_set::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_set`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search, and only inserts + // provided that no element with the given key already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. + // + // If an insertion occurs, any references, pointers, or iterators are + // invalidated. + using Base::emplace_hint; + + // btree_set::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the element at the indicated position and returns a node handle + // owning that extracted data. + // + // template <typename K> node_type extract(const K& k): + // + // Extracts the element with the key matching the passed key value and + // returns a node handle owning that extracted data. If the `btree_set` + // does not contain an element with a matching key, this function returns an + // empty node handle. + // + // NOTE: In this context, `node_type` refers to the C++17 concept of a + // move-only type that owns and provides access to the elements in associative + // containers (https://en.cppreference.com/w/cpp/container/node_handle). + // It does NOT refer to the data layout of the underlying btree. + using Base::extract; + + // btree_set::merge() + // + // Extracts elements from a given `source` btree_set into this + // `btree_set`. If the destination `btree_set` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // btree_set::swap(btree_set& other) + // + // Exchanges the contents of this `btree_set` with those of the `other` + // btree_set, avoiding invocation of any move, copy, or swap operations on + // individual elements. + // + // All iterators and references on the `btree_set` remain valid, excepting + // for the past-the-end iterator, which is invalidated. + using Base::swap; + + // btree_set::contains() + // + // template <typename K> bool contains(const K& key) const: + // + // Determines whether an element comparing equal to the given `key` exists + // within the `btree_set`, returning `true` if so or `false` otherwise. + // + // Supports heterogeneous lookup, provided that the set is provided a + // compatible heterogeneous comparator. + using Base::contains; + + // btree_set::count() + // + // template <typename K> size_type count(const K& key) const: + // + // Returns the number of elements comparing equal to the given `key` within + // the `btree_set`. Note that this function will return either `1` or `0` + // since duplicate elements are not allowed within a `btree_set`. + // + // Supports heterogeneous lookup, provided that the set is provided a + // compatible heterogeneous comparator. + using Base::count; + + // btree_set::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `btree_set`. + using Base::equal_range; + + // btree_set::find() + // + // template <typename K> iterator find(const K& key): + // template <typename K> const_iterator find(const K& key) const: + // + // Finds an element with the passed `key` within the `btree_set`. + // + // Supports heterogeneous lookup, provided that the set is provided a + // compatible heterogeneous comparator. + using Base::find; + + // btree_set::get_allocator() + // + // Returns the allocator function associated with this `btree_set`. + using Base::get_allocator; + + // btree_set::key_comp(); + // + // Returns the key comparator associated with this `btree_set`. + using Base::key_comp; + + // btree_set::value_comp(); + // + // Returns the value comparator associated with this `btree_set`. The keys to + // sort the elements are the values themselves, therefore `value_comp` and its + // sibling member function `key_comp` are equivalent. + using Base::value_comp; +}; + +// absl::swap(absl::btree_set<>, absl::btree_set<>) +// +// Swaps the contents of two `absl::btree_set` containers. +template <typename K, typename C, typename A> +void swap(btree_set<K, C, A> &x, btree_set<K, C, A> &y) { + return x.swap(y); +} + +// absl::erase_if(absl::btree_set<>, Pred) +// +// Erases all elements that satisfy the predicate pred from the container. +template <typename K, typename C, typename A, typename Pred> +void erase_if(btree_set<K, C, A> &set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; + } + } +} + +// absl::btree_multiset<> +// +// An `absl::btree_multiset<K>` is an ordered associative container of +// keys and associated values designed to be a more efficient replacement +// for `std::multiset` (in most cases). Unlike `absl::btree_set`, a B-tree +// multiset allows equivalent elements. +// +// Keys are sorted using an (optional) comparison function, which defaults to +// `std::less<K>`. +// +// An `absl::btree_multiset<K>` uses a default allocator of `std::allocator<K>` +// to allocate (and deallocate) nodes, and construct and destruct values within +// those nodes. You may instead specify a custom allocator `A` (which in turn +// requires specifying a custom comparator `C`) as in +// `absl::btree_multiset<K, C, A>`. +// +template <typename Key, typename Compare = std::less<Key>, + typename Alloc = std::allocator<Key>> +class btree_multiset + : public container_internal::btree_multiset_container< + container_internal::btree<container_internal::set_params< + Key, Compare, Alloc, /*TargetNodeSize=*/256, + /*Multi=*/true>>> { + using Base = typename btree_multiset::btree_multiset_container; + + public: + // Constructors and Assignment Operators + // + // A `btree_multiset` supports the same overload set as `std::set` + // for construction and assignment: + // + // * Default constructor + // + // absl::btree_multiset<std::string> set1; + // + // * Initializer List constructor + // + // absl::btree_multiset<std::string> set2 = + // {{"huey"}, {"dewey"}, {"louie"},}; + // + // * Copy constructor + // + // absl::btree_multiset<std::string> set3(set2); + // + // * Copy assignment operator + // + // absl::btree_multiset<std::string> set4; + // set4 = set3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::btree_multiset<std::string> set5(std::move(set4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::btree_multiset<std::string> set6; + // set6 = std::move(set5); + // + // * Range constructor + // + // std::vector<std::string> v = {"a", "b"}; + // absl::btree_multiset<std::string> set7(v.begin(), v.end()); + btree_multiset() {} + using Base::Base; + + // btree_multiset::begin() + // + // Returns an iterator to the beginning of the `btree_multiset`. + using Base::begin; + + // btree_multiset::cbegin() + // + // Returns a const iterator to the beginning of the `btree_multiset`. + using Base::cbegin; + + // btree_multiset::end() + // + // Returns an iterator to the end of the `btree_multiset`. + using Base::end; + + // btree_multiset::cend() + // + // Returns a const iterator to the end of the `btree_multiset`. + using Base::cend; + + // btree_multiset::empty() + // + // Returns whether or not the `btree_multiset` is empty. + using Base::empty; + + // btree_multiset::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `btree_multiset` under current memory constraints. This value can be + // thought of as the largest value of `std::distance(begin(), end())` for a + // `btree_multiset<Key>`. + using Base::max_size; + + // btree_multiset::size() + // + // Returns the number of elements currently within the `btree_multiset`. + using Base::size; + + // btree_multiset::clear() + // + // Removes all elements from the `btree_multiset`. Invalidates any references, + // pointers, or iterators referring to contained elements. + using Base::clear; + + // btree_multiset::erase() + // + // Erases elements within the `btree_multiset`. Overloads are listed below. + // + // iterator erase(iterator position): + // iterator erase(const_iterator position): + // + // Erases the element at `position` of the `btree_multiset`, returning + // the iterator pointing to the element after the one that was erased + // (or end() if none exists). + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning + // the iterator pointing to the element after the interval that was erased + // (or end() if none exists). + // + // template <typename K> size_type erase(const K& key): + // + // Erases the elements matching the key, if any exist, returning the + // number of elements erased. + using Base::erase; + + // btree_multiset::insert() + // + // Inserts an element of the specified value into the `btree_multiset`, + // returning an iterator pointing to the newly inserted element. + // Any references, pointers, or iterators are invalidated. Overloads are + // listed below. + // + // iterator insert(const value_type& value): + // + // Inserts a value into the `btree_multiset`, returning an iterator to the + // inserted element. + // + // iterator insert(value_type&& value): + // + // Inserts a moveable value into the `btree_multiset`, returning an iterator + // to the inserted element. + // + // iterator insert(const_iterator hint, const value_type& value): + // iterator insert(const_iterator hint, value_type&& value): + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // void insert(std::initializer_list<init_type> ilist): + // + // Inserts the elements within the initializer list `ilist`. + using Base::insert; + + // btree_multiset::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_multiset`. Any references, pointers, or iterators are + // invalidated. + using Base::emplace; + + // btree_multiset::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `btree_multiset`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search. + // + // Any references, pointers, or iterators are invalidated. + using Base::emplace_hint; + + // btree_multiset::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the element at the indicated position and returns a node handle + // owning that extracted data. + // + // template <typename K> node_type extract(const K& k): + // + // Extracts the element with the key matching the passed key value and + // returns a node handle owning that extracted data. If the `btree_multiset` + // does not contain an element with a matching key, this function returns an + // empty node handle. + // + // NOTE: In this context, `node_type` refers to the C++17 concept of a + // move-only type that owns and provides access to the elements in associative + // containers (https://en.cppreference.com/w/cpp/container/node_handle). + // It does NOT refer to the data layout of the underlying btree. + using Base::extract; + + // btree_multiset::merge() + // + // Extracts elements from a given `source` btree_multiset into this + // `btree_multiset`. If the destination `btree_multiset` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // btree_multiset::swap(btree_multiset& other) + // + // Exchanges the contents of this `btree_multiset` with those of the `other` + // btree_multiset, avoiding invocation of any move, copy, or swap operations + // on individual elements. + // + // All iterators and references on the `btree_multiset` remain valid, + // excepting for the past-the-end iterator, which is invalidated. + using Base::swap; + + // btree_multiset::contains() + // + // template <typename K> bool contains(const K& key) const: + // + // Determines whether an element comparing equal to the given `key` exists + // within the `btree_multiset`, returning `true` if so or `false` otherwise. + // + // Supports heterogeneous lookup, provided that the set is provided a + // compatible heterogeneous comparator. + using Base::contains; + + // btree_multiset::count() + // + // template <typename K> size_type count(const K& key) const: + // + // Returns the number of elements comparing equal to the given `key` within + // the `btree_multiset`. + // + // Supports heterogeneous lookup, provided that the set is provided a + // compatible heterogeneous comparator. + using Base::count; + + // btree_multiset::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `btree_multiset`. + using Base::equal_range; + + // btree_multiset::find() + // + // template <typename K> iterator find(const K& key): + // template <typename K> const_iterator find(const K& key) const: + // + // Finds an element with the passed `key` within the `btree_multiset`. + // + // Supports heterogeneous lookup, provided that the set is provided a + // compatible heterogeneous comparator. + using Base::find; + + // btree_multiset::get_allocator() + // + // Returns the allocator function associated with this `btree_multiset`. + using Base::get_allocator; + + // btree_multiset::key_comp(); + // + // Returns the key comparator associated with this `btree_multiset`. + using Base::key_comp; + + // btree_multiset::value_comp(); + // + // Returns the value comparator associated with this `btree_multiset`. The + // keys to sort the elements are the values themselves, therefore `value_comp` + // and its sibling member function `key_comp` are equivalent. + using Base::value_comp; +}; + +// absl::swap(absl::btree_multiset<>, absl::btree_multiset<>) +// +// Swaps the contents of two `absl::btree_multiset` containers. +template <typename K, typename C, typename A> +void swap(btree_multiset<K, C, A> &x, btree_multiset<K, C, A> &y) { + return x.swap(y); +} + +// absl::erase_if(absl::btree_multiset<>, Pred) +// +// Erases all elements that satisfy the predicate pred from the container. +template <typename K, typename C, typename A, typename Pred> +void erase_if(btree_multiset<K, C, A> &set, Pred pred) { + for (auto it = set.begin(); it != set.end();) { + if (pred(*it)) { + it = set.erase(it); + } else { + ++it; + } + } +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_BTREE_SET_H_ diff --git a/third_party/abseil_cpp/absl/container/btree_test.cc b/third_party/abseil_cpp/absl/container/btree_test.cc new file mode 100644 index 000000000000..9b1b6436c7ca --- /dev/null +++ b/third_party/abseil_cpp/absl/container/btree_test.cc @@ -0,0 +1,2827 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/btree_test.h" + +#include <cstdint> +#include <limits> +#include <map> +#include <memory> +#include <stdexcept> +#include <string> +#include <type_traits> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/container/btree_map.h" +#include "absl/container/btree_set.h" +#include "absl/container/internal/counting_allocator.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/flags/flag.h" +#include "absl/hash/hash_testing.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/types/compare.h" + +ABSL_FLAG(int, test_values, 10000, "The number of values to use for tests"); + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::absl::test_internal::CopyableMovableInstance; +using ::absl::test_internal::InstanceTracker; +using ::absl::test_internal::MovableOnlyInstance; +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::IsEmpty; +using ::testing::IsNull; +using ::testing::Pair; +using ::testing::SizeIs; + +template <typename T, typename U> +void CheckPairEquals(const T &x, const U &y) { + ABSL_INTERNAL_CHECK(x == y, "Values are unequal."); +} + +template <typename T, typename U, typename V, typename W> +void CheckPairEquals(const std::pair<T, U> &x, const std::pair<V, W> &y) { + CheckPairEquals(x.first, y.first); + CheckPairEquals(x.second, y.second); +} +} // namespace + +// The base class for a sorted associative container checker. TreeType is the +// container type to check and CheckerType is the container type to check +// against. TreeType is expected to be btree_{set,map,multiset,multimap} and +// CheckerType is expected to be {set,map,multiset,multimap}. +template <typename TreeType, typename CheckerType> +class base_checker { + public: + using key_type = typename TreeType::key_type; + using value_type = typename TreeType::value_type; + using key_compare = typename TreeType::key_compare; + using pointer = typename TreeType::pointer; + using const_pointer = typename TreeType::const_pointer; + using reference = typename TreeType::reference; + using const_reference = typename TreeType::const_reference; + using size_type = typename TreeType::size_type; + using difference_type = typename TreeType::difference_type; + using iterator = typename TreeType::iterator; + using const_iterator = typename TreeType::const_iterator; + using reverse_iterator = typename TreeType::reverse_iterator; + using const_reverse_iterator = typename TreeType::const_reverse_iterator; + + public: + base_checker() : const_tree_(tree_) {} + base_checker(const base_checker &other) + : tree_(other.tree_), const_tree_(tree_), checker_(other.checker_) {} + template <typename InputIterator> + base_checker(InputIterator b, InputIterator e) + : tree_(b, e), const_tree_(tree_), checker_(b, e) {} + + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + + template <typename IterType, typename CheckerIterType> + IterType iter_check(IterType tree_iter, CheckerIterType checker_iter) const { + if (tree_iter == tree_.end()) { + ABSL_INTERNAL_CHECK(checker_iter == checker_.end(), + "Checker iterator not at end."); + } else { + CheckPairEquals(*tree_iter, *checker_iter); + } + return tree_iter; + } + template <typename IterType, typename CheckerIterType> + IterType riter_check(IterType tree_iter, CheckerIterType checker_iter) const { + if (tree_iter == tree_.rend()) { + ABSL_INTERNAL_CHECK(checker_iter == checker_.rend(), + "Checker iterator not at rend."); + } else { + CheckPairEquals(*tree_iter, *checker_iter); + } + return tree_iter; + } + void value_check(const value_type &v) { + typename KeyOfValue<typename TreeType::key_type, + typename TreeType::value_type>::type key_of_value; + const key_type &key = key_of_value(v); + CheckPairEquals(*find(key), v); + lower_bound(key); + upper_bound(key); + equal_range(key); + contains(key); + count(key); + } + void erase_check(const key_type &key) { + EXPECT_FALSE(tree_.contains(key)); + EXPECT_EQ(tree_.find(key), const_tree_.end()); + EXPECT_FALSE(const_tree_.contains(key)); + EXPECT_EQ(const_tree_.find(key), tree_.end()); + EXPECT_EQ(tree_.equal_range(key).first, + const_tree_.equal_range(key).second); + } + + iterator lower_bound(const key_type &key) { + return iter_check(tree_.lower_bound(key), checker_.lower_bound(key)); + } + const_iterator lower_bound(const key_type &key) const { + return iter_check(tree_.lower_bound(key), checker_.lower_bound(key)); + } + iterator upper_bound(const key_type &key) { + return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); + } + const_iterator upper_bound(const key_type &key) const { + return iter_check(tree_.upper_bound(key), checker_.upper_bound(key)); + } + std::pair<iterator, iterator> equal_range(const key_type &key) { + std::pair<typename CheckerType::iterator, typename CheckerType::iterator> + checker_res = checker_.equal_range(key); + std::pair<iterator, iterator> tree_res = tree_.equal_range(key); + iter_check(tree_res.first, checker_res.first); + iter_check(tree_res.second, checker_res.second); + return tree_res; + } + std::pair<const_iterator, const_iterator> equal_range( + const key_type &key) const { + std::pair<typename CheckerType::const_iterator, + typename CheckerType::const_iterator> + checker_res = checker_.equal_range(key); + std::pair<const_iterator, const_iterator> tree_res = tree_.equal_range(key); + iter_check(tree_res.first, checker_res.first); + iter_check(tree_res.second, checker_res.second); + return tree_res; + } + iterator find(const key_type &key) { + return iter_check(tree_.find(key), checker_.find(key)); + } + const_iterator find(const key_type &key) const { + return iter_check(tree_.find(key), checker_.find(key)); + } + bool contains(const key_type &key) const { return find(key) != end(); } + size_type count(const key_type &key) const { + size_type res = checker_.count(key); + EXPECT_EQ(res, tree_.count(key)); + return res; + } + + base_checker &operator=(const base_checker &other) { + tree_ = other.tree_; + checker_ = other.checker_; + return *this; + } + + int erase(const key_type &key) { + int size = tree_.size(); + int res = checker_.erase(key); + EXPECT_EQ(res, tree_.count(key)); + EXPECT_EQ(res, tree_.erase(key)); + EXPECT_EQ(tree_.count(key), 0); + EXPECT_EQ(tree_.size(), size - res); + erase_check(key); + return res; + } + iterator erase(iterator iter) { + key_type key = iter.key(); + int size = tree_.size(); + int count = tree_.count(key); + auto checker_iter = checker_.lower_bound(key); + for (iterator tmp(tree_.lower_bound(key)); tmp != iter; ++tmp) { + ++checker_iter; + } + auto checker_next = checker_iter; + ++checker_next; + checker_.erase(checker_iter); + iter = tree_.erase(iter); + EXPECT_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), size - 1); + EXPECT_EQ(tree_.count(key), count - 1); + if (count == 1) { + erase_check(key); + } + return iter_check(iter, checker_next); + } + + void erase(iterator begin, iterator end) { + int size = tree_.size(); + int count = std::distance(begin, end); + auto checker_begin = checker_.lower_bound(begin.key()); + for (iterator tmp(tree_.lower_bound(begin.key())); tmp != begin; ++tmp) { + ++checker_begin; + } + auto checker_end = + end == tree_.end() ? checker_.end() : checker_.lower_bound(end.key()); + if (end != tree_.end()) { + for (iterator tmp(tree_.lower_bound(end.key())); tmp != end; ++tmp) { + ++checker_end; + } + } + const auto checker_ret = checker_.erase(checker_begin, checker_end); + const auto tree_ret = tree_.erase(begin, end); + EXPECT_EQ(std::distance(checker_.begin(), checker_ret), + std::distance(tree_.begin(), tree_ret)); + EXPECT_EQ(tree_.size(), checker_.size()); + EXPECT_EQ(tree_.size(), size - count); + } + + void clear() { + tree_.clear(); + checker_.clear(); + } + void swap(base_checker &other) { + tree_.swap(other.tree_); + checker_.swap(other.checker_); + } + + void verify() const { + tree_.verify(); + EXPECT_EQ(tree_.size(), checker_.size()); + + // Move through the forward iterators using increment. + auto checker_iter = checker_.begin(); + const_iterator tree_iter(tree_.begin()); + for (; tree_iter != tree_.end(); ++tree_iter, ++checker_iter) { + CheckPairEquals(*tree_iter, *checker_iter); + } + + // Move through the forward iterators using decrement. + for (int n = tree_.size() - 1; n >= 0; --n) { + iter_check(tree_iter, checker_iter); + --tree_iter; + --checker_iter; + } + EXPECT_EQ(tree_iter, tree_.begin()); + EXPECT_EQ(checker_iter, checker_.begin()); + + // Move through the reverse iterators using increment. + auto checker_riter = checker_.rbegin(); + const_reverse_iterator tree_riter(tree_.rbegin()); + for (; tree_riter != tree_.rend(); ++tree_riter, ++checker_riter) { + CheckPairEquals(*tree_riter, *checker_riter); + } + + // Move through the reverse iterators using decrement. + for (int n = tree_.size() - 1; n >= 0; --n) { + riter_check(tree_riter, checker_riter); + --tree_riter; + --checker_riter; + } + EXPECT_EQ(tree_riter, tree_.rbegin()); + EXPECT_EQ(checker_riter, checker_.rbegin()); + } + + const TreeType &tree() const { return tree_; } + + size_type size() const { + EXPECT_EQ(tree_.size(), checker_.size()); + return tree_.size(); + } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { + EXPECT_EQ(tree_.empty(), checker_.empty()); + return tree_.empty(); + } + + protected: + TreeType tree_; + const TreeType &const_tree_; + CheckerType checker_; +}; + +namespace { +// A checker for unique sorted associative containers. TreeType is expected to +// be btree_{set,map} and CheckerType is expected to be {set,map}. +template <typename TreeType, typename CheckerType> +class unique_checker : public base_checker<TreeType, CheckerType> { + using super_type = base_checker<TreeType, CheckerType>; + + public: + using iterator = typename super_type::iterator; + using value_type = typename super_type::value_type; + + public: + unique_checker() : super_type() {} + unique_checker(const unique_checker &other) : super_type(other) {} + template <class InputIterator> + unique_checker(InputIterator b, InputIterator e) : super_type(b, e) {} + unique_checker &operator=(const unique_checker &) = default; + + // Insertion routines. + std::pair<iterator, bool> insert(const value_type &v) { + int size = this->tree_.size(); + std::pair<typename CheckerType::iterator, bool> checker_res = + this->checker_.insert(v); + std::pair<iterator, bool> tree_res = this->tree_.insert(v); + CheckPairEquals(*tree_res.first, *checker_res.first); + EXPECT_EQ(tree_res.second, checker_res.second); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + tree_res.second); + return tree_res; + } + iterator insert(iterator position, const value_type &v) { + int size = this->tree_.size(); + std::pair<typename CheckerType::iterator, bool> checker_res = + this->checker_.insert(v); + iterator tree_res = this->tree_.insert(position, v); + CheckPairEquals(*tree_res, *checker_res.first); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + checker_res.second); + return tree_res; + } + template <typename InputIterator> + void insert(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert(*b); + } + } +}; + +// A checker for multiple sorted associative containers. TreeType is expected +// to be btree_{multiset,multimap} and CheckerType is expected to be +// {multiset,multimap}. +template <typename TreeType, typename CheckerType> +class multi_checker : public base_checker<TreeType, CheckerType> { + using super_type = base_checker<TreeType, CheckerType>; + + public: + using iterator = typename super_type::iterator; + using value_type = typename super_type::value_type; + + public: + multi_checker() : super_type() {} + multi_checker(const multi_checker &other) : super_type(other) {} + template <class InputIterator> + multi_checker(InputIterator b, InputIterator e) : super_type(b, e) {} + multi_checker &operator=(const multi_checker &) = default; + + // Insertion routines. + iterator insert(const value_type &v) { + int size = this->tree_.size(); + auto checker_res = this->checker_.insert(v); + iterator tree_res = this->tree_.insert(v); + CheckPairEquals(*tree_res, *checker_res); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + 1); + return tree_res; + } + iterator insert(iterator position, const value_type &v) { + int size = this->tree_.size(); + auto checker_res = this->checker_.insert(v); + iterator tree_res = this->tree_.insert(position, v); + CheckPairEquals(*tree_res, *checker_res); + EXPECT_EQ(this->tree_.size(), this->checker_.size()); + EXPECT_EQ(this->tree_.size(), size + 1); + return tree_res; + } + template <typename InputIterator> + void insert(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert(*b); + } + } +}; + +template <typename T, typename V> +void DoTest(const char *name, T *b, const std::vector<V> &values) { + typename KeyOfValue<typename T::key_type, V>::type key_of_value; + + T &mutable_b = *b; + const T &const_b = *b; + + // Test insert. + for (int i = 0; i < values.size(); ++i) { + mutable_b.insert(values[i]); + mutable_b.value_check(values[i]); + } + ASSERT_EQ(mutable_b.size(), values.size()); + + const_b.verify(); + + // Test copy constructor. + T b_copy(const_b); + EXPECT_EQ(b_copy.size(), const_b.size()); + for (int i = 0; i < values.size(); ++i) { + CheckPairEquals(*b_copy.find(key_of_value(values[i])), values[i]); + } + + // Test range constructor. + T b_range(const_b.begin(), const_b.end()); + EXPECT_EQ(b_range.size(), const_b.size()); + for (int i = 0; i < values.size(); ++i) { + CheckPairEquals(*b_range.find(key_of_value(values[i])), values[i]); + } + + // Test range insertion for values that already exist. + b_range.insert(b_copy.begin(), b_copy.end()); + b_range.verify(); + + // Test range insertion for new values. + b_range.clear(); + b_range.insert(b_copy.begin(), b_copy.end()); + EXPECT_EQ(b_range.size(), b_copy.size()); + for (int i = 0; i < values.size(); ++i) { + CheckPairEquals(*b_range.find(key_of_value(values[i])), values[i]); + } + + // Test assignment to self. Nothing should change. + b_range.operator=(b_range); + EXPECT_EQ(b_range.size(), b_copy.size()); + + // Test assignment of new values. + b_range.clear(); + b_range = b_copy; + EXPECT_EQ(b_range.size(), b_copy.size()); + + // Test swap. + b_range.clear(); + b_range.swap(b_copy); + EXPECT_EQ(b_copy.size(), 0); + EXPECT_EQ(b_range.size(), const_b.size()); + for (int i = 0; i < values.size(); ++i) { + CheckPairEquals(*b_range.find(key_of_value(values[i])), values[i]); + } + b_range.swap(b_copy); + + // Test non-member function swap. + swap(b_range, b_copy); + EXPECT_EQ(b_copy.size(), 0); + EXPECT_EQ(b_range.size(), const_b.size()); + for (int i = 0; i < values.size(); ++i) { + CheckPairEquals(*b_range.find(key_of_value(values[i])), values[i]); + } + swap(b_range, b_copy); + + // Test erase via values. + for (int i = 0; i < values.size(); ++i) { + mutable_b.erase(key_of_value(values[i])); + // Erasing a non-existent key should have no effect. + ASSERT_EQ(mutable_b.erase(key_of_value(values[i])), 0); + } + + const_b.verify(); + EXPECT_EQ(const_b.size(), 0); + + // Test erase via iterators. + mutable_b = b_copy; + for (int i = 0; i < values.size(); ++i) { + mutable_b.erase(mutable_b.find(key_of_value(values[i]))); + } + + const_b.verify(); + EXPECT_EQ(const_b.size(), 0); + + // Test insert with hint. + for (int i = 0; i < values.size(); i++) { + mutable_b.insert(mutable_b.upper_bound(key_of_value(values[i])), values[i]); + } + + const_b.verify(); + + // Test range erase. + mutable_b.erase(mutable_b.begin(), mutable_b.end()); + EXPECT_EQ(mutable_b.size(), 0); + const_b.verify(); + + // First half. + mutable_b = b_copy; + typename T::iterator mutable_iter_end = mutable_b.begin(); + for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_end; + mutable_b.erase(mutable_b.begin(), mutable_iter_end); + EXPECT_EQ(mutable_b.size(), values.size() - values.size() / 2); + const_b.verify(); + + // Second half. + mutable_b = b_copy; + typename T::iterator mutable_iter_begin = mutable_b.begin(); + for (int i = 0; i < values.size() / 2; ++i) ++mutable_iter_begin; + mutable_b.erase(mutable_iter_begin, mutable_b.end()); + EXPECT_EQ(mutable_b.size(), values.size() / 2); + const_b.verify(); + + // Second quarter. + mutable_b = b_copy; + mutable_iter_begin = mutable_b.begin(); + for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_begin; + mutable_iter_end = mutable_iter_begin; + for (int i = 0; i < values.size() / 4; ++i) ++mutable_iter_end; + mutable_b.erase(mutable_iter_begin, mutable_iter_end); + EXPECT_EQ(mutable_b.size(), values.size() - values.size() / 4); + const_b.verify(); + + mutable_b.clear(); +} + +template <typename T> +void ConstTest() { + using value_type = typename T::value_type; + typename KeyOfValue<typename T::key_type, value_type>::type key_of_value; + + T mutable_b; + const T &const_b = mutable_b; + + // Insert a single value into the container and test looking it up. + value_type value = Generator<value_type>(2)(2); + mutable_b.insert(value); + EXPECT_TRUE(mutable_b.contains(key_of_value(value))); + EXPECT_NE(mutable_b.find(key_of_value(value)), const_b.end()); + EXPECT_TRUE(const_b.contains(key_of_value(value))); + EXPECT_NE(const_b.find(key_of_value(value)), mutable_b.end()); + EXPECT_EQ(*const_b.lower_bound(key_of_value(value)), value); + EXPECT_EQ(const_b.upper_bound(key_of_value(value)), const_b.end()); + EXPECT_EQ(*const_b.equal_range(key_of_value(value)).first, value); + + // We can only create a non-const iterator from a non-const container. + typename T::iterator mutable_iter(mutable_b.begin()); + EXPECT_EQ(mutable_iter, const_b.begin()); + EXPECT_NE(mutable_iter, const_b.end()); + EXPECT_EQ(const_b.begin(), mutable_iter); + EXPECT_NE(const_b.end(), mutable_iter); + typename T::reverse_iterator mutable_riter(mutable_b.rbegin()); + EXPECT_EQ(mutable_riter, const_b.rbegin()); + EXPECT_NE(mutable_riter, const_b.rend()); + EXPECT_EQ(const_b.rbegin(), mutable_riter); + EXPECT_NE(const_b.rend(), mutable_riter); + + // We can create a const iterator from a non-const iterator. + typename T::const_iterator const_iter(mutable_iter); + EXPECT_EQ(const_iter, mutable_b.begin()); + EXPECT_NE(const_iter, mutable_b.end()); + EXPECT_EQ(mutable_b.begin(), const_iter); + EXPECT_NE(mutable_b.end(), const_iter); + typename T::const_reverse_iterator const_riter(mutable_riter); + EXPECT_EQ(const_riter, mutable_b.rbegin()); + EXPECT_NE(const_riter, mutable_b.rend()); + EXPECT_EQ(mutable_b.rbegin(), const_riter); + EXPECT_NE(mutable_b.rend(), const_riter); + + // Make sure various methods can be invoked on a const container. + const_b.verify(); + ASSERT_TRUE(!const_b.empty()); + EXPECT_EQ(const_b.size(), 1); + EXPECT_GT(const_b.max_size(), 0); + EXPECT_TRUE(const_b.contains(key_of_value(value))); + EXPECT_EQ(const_b.count(key_of_value(value)), 1); +} + +template <typename T, typename C> +void BtreeTest() { + ConstTest<T>(); + + using V = typename remove_pair_const<typename T::value_type>::type; + const std::vector<V> random_values = GenerateValuesWithSeed<V>( + absl::GetFlag(FLAGS_test_values), 4 * absl::GetFlag(FLAGS_test_values), + testing::GTEST_FLAG(random_seed)); + + unique_checker<T, C> container; + + // Test key insertion/deletion in sorted order. + std::vector<V> sorted_values(random_values); + std::sort(sorted_values.begin(), sorted_values.end()); + DoTest("sorted: ", &container, sorted_values); + + // Test key insertion/deletion in reverse sorted order. + std::reverse(sorted_values.begin(), sorted_values.end()); + DoTest("rsorted: ", &container, sorted_values); + + // Test key insertion/deletion in random order. + DoTest("random: ", &container, random_values); +} + +template <typename T, typename C> +void BtreeMultiTest() { + ConstTest<T>(); + + using V = typename remove_pair_const<typename T::value_type>::type; + const std::vector<V> random_values = GenerateValuesWithSeed<V>( + absl::GetFlag(FLAGS_test_values), 4 * absl::GetFlag(FLAGS_test_values), + testing::GTEST_FLAG(random_seed)); + + multi_checker<T, C> container; + + // Test keys in sorted order. + std::vector<V> sorted_values(random_values); + std::sort(sorted_values.begin(), sorted_values.end()); + DoTest("sorted: ", &container, sorted_values); + + // Test keys in reverse sorted order. + std::reverse(sorted_values.begin(), sorted_values.end()); + DoTest("rsorted: ", &container, sorted_values); + + // Test keys in random order. + DoTest("random: ", &container, random_values); + + // Test keys in random order w/ duplicates. + std::vector<V> duplicate_values(random_values); + duplicate_values.insert(duplicate_values.end(), random_values.begin(), + random_values.end()); + DoTest("duplicates:", &container, duplicate_values); + + // Test all identical keys. + std::vector<V> identical_values(100); + std::fill(identical_values.begin(), identical_values.end(), + Generator<V>(2)(2)); + DoTest("identical: ", &container, identical_values); +} + +template <typename T> +struct PropagatingCountingAlloc : public CountingAllocator<T> { + using propagate_on_container_copy_assignment = std::true_type; + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; + + using Base = CountingAllocator<T>; + using Base::Base; + + template <typename U> + explicit PropagatingCountingAlloc(const PropagatingCountingAlloc<U> &other) + : Base(other.bytes_used_) {} + + template <typename U> + struct rebind { + using other = PropagatingCountingAlloc<U>; + }; +}; + +template <typename T> +void BtreeAllocatorTest() { + using value_type = typename T::value_type; + + int64_t bytes1 = 0, bytes2 = 0; + PropagatingCountingAlloc<T> allocator1(&bytes1); + PropagatingCountingAlloc<T> allocator2(&bytes2); + Generator<value_type> generator(1000); + + // Test that we allocate properly aligned memory. If we don't, then Layout + // will assert fail. + auto unused1 = allocator1.allocate(1); + auto unused2 = allocator2.allocate(1); + + // Test copy assignment + { + T b1(typename T::key_compare(), allocator1); + T b2(typename T::key_compare(), allocator2); + + int64_t original_bytes1 = bytes1; + b1.insert(generator(0)); + EXPECT_GT(bytes1, original_bytes1); + + // This should propagate the allocator. + b1 = b2; + EXPECT_EQ(b1.size(), 0); + EXPECT_EQ(b2.size(), 0); + EXPECT_EQ(bytes1, original_bytes1); + + for (int i = 1; i < 1000; i++) { + b1.insert(generator(i)); + } + + // We should have allocated out of allocator2. + EXPECT_GT(bytes2, bytes1); + } + + // Test move assignment + { + T b1(typename T::key_compare(), allocator1); + T b2(typename T::key_compare(), allocator2); + + int64_t original_bytes1 = bytes1; + b1.insert(generator(0)); + EXPECT_GT(bytes1, original_bytes1); + + // This should propagate the allocator. + b1 = std::move(b2); + EXPECT_EQ(b1.size(), 0); + EXPECT_EQ(bytes1, original_bytes1); + + for (int i = 1; i < 1000; i++) { + b1.insert(generator(i)); + } + + // We should have allocated out of allocator2. + EXPECT_GT(bytes2, bytes1); + } + + // Test swap + { + T b1(typename T::key_compare(), allocator1); + T b2(typename T::key_compare(), allocator2); + + int64_t original_bytes1 = bytes1; + b1.insert(generator(0)); + EXPECT_GT(bytes1, original_bytes1); + + // This should swap the allocators. + swap(b1, b2); + EXPECT_EQ(b1.size(), 0); + EXPECT_EQ(b2.size(), 1); + EXPECT_GT(bytes1, original_bytes1); + + for (int i = 1; i < 1000; i++) { + b1.insert(generator(i)); + } + + // We should have allocated out of allocator2. + EXPECT_GT(bytes2, bytes1); + } + + allocator1.deallocate(unused1, 1); + allocator2.deallocate(unused2, 1); +} + +template <typename T> +void BtreeMapTest() { + using value_type = typename T::value_type; + using mapped_type = typename T::mapped_type; + + mapped_type m = Generator<mapped_type>(0)(0); + (void)m; + + T b; + + // Verify we can insert using operator[]. + for (int i = 0; i < 1000; i++) { + value_type v = Generator<value_type>(1000)(i); + b[v.first] = v.second; + } + EXPECT_EQ(b.size(), 1000); + + // Test whether we can use the "->" operator on iterators and + // reverse_iterators. This stresses the btree_map_params::pair_pointer + // mechanism. + EXPECT_EQ(b.begin()->first, Generator<value_type>(1000)(0).first); + EXPECT_EQ(b.begin()->second, Generator<value_type>(1000)(0).second); + EXPECT_EQ(b.rbegin()->first, Generator<value_type>(1000)(999).first); + EXPECT_EQ(b.rbegin()->second, Generator<value_type>(1000)(999).second); +} + +template <typename T> +void BtreeMultiMapTest() { + using mapped_type = typename T::mapped_type; + mapped_type m = Generator<mapped_type>(0)(0); + (void)m; +} + +template <typename K, int N = 256> +void SetTest() { + EXPECT_EQ( + sizeof(absl::btree_set<K>), + 2 * sizeof(void *) + sizeof(typename absl::btree_set<K>::size_type)); + using BtreeSet = absl::btree_set<K>; + using CountingBtreeSet = + absl::btree_set<K, std::less<K>, PropagatingCountingAlloc<K>>; + BtreeTest<BtreeSet, std::set<K>>(); + BtreeAllocatorTest<CountingBtreeSet>(); +} + +template <typename K, int N = 256> +void MapTest() { + EXPECT_EQ( + sizeof(absl::btree_map<K, K>), + 2 * sizeof(void *) + sizeof(typename absl::btree_map<K, K>::size_type)); + using BtreeMap = absl::btree_map<K, K>; + using CountingBtreeMap = + absl::btree_map<K, K, std::less<K>, + PropagatingCountingAlloc<std::pair<const K, K>>>; + BtreeTest<BtreeMap, std::map<K, K>>(); + BtreeAllocatorTest<CountingBtreeMap>(); + BtreeMapTest<BtreeMap>(); +} + +TEST(Btree, set_int32) { SetTest<int32_t>(); } +TEST(Btree, set_int64) { SetTest<int64_t>(); } +TEST(Btree, set_string) { SetTest<std::string>(); } +TEST(Btree, set_cord) { SetTest<absl::Cord>(); } +TEST(Btree, set_pair) { SetTest<std::pair<int, int>>(); } +TEST(Btree, map_int32) { MapTest<int32_t>(); } +TEST(Btree, map_int64) { MapTest<int64_t>(); } +TEST(Btree, map_string) { MapTest<std::string>(); } +TEST(Btree, map_cord) { MapTest<absl::Cord>(); } +TEST(Btree, map_pair) { MapTest<std::pair<int, int>>(); } + +template <typename K, int N = 256> +void MultiSetTest() { + EXPECT_EQ( + sizeof(absl::btree_multiset<K>), + 2 * sizeof(void *) + sizeof(typename absl::btree_multiset<K>::size_type)); + using BtreeMSet = absl::btree_multiset<K>; + using CountingBtreeMSet = + absl::btree_multiset<K, std::less<K>, PropagatingCountingAlloc<K>>; + BtreeMultiTest<BtreeMSet, std::multiset<K>>(); + BtreeAllocatorTest<CountingBtreeMSet>(); +} + +template <typename K, int N = 256> +void MultiMapTest() { + EXPECT_EQ(sizeof(absl::btree_multimap<K, K>), + 2 * sizeof(void *) + + sizeof(typename absl::btree_multimap<K, K>::size_type)); + using BtreeMMap = absl::btree_multimap<K, K>; + using CountingBtreeMMap = + absl::btree_multimap<K, K, std::less<K>, + PropagatingCountingAlloc<std::pair<const K, K>>>; + BtreeMultiTest<BtreeMMap, std::multimap<K, K>>(); + BtreeMultiMapTest<BtreeMMap>(); + BtreeAllocatorTest<CountingBtreeMMap>(); +} + +TEST(Btree, multiset_int32) { MultiSetTest<int32_t>(); } +TEST(Btree, multiset_int64) { MultiSetTest<int64_t>(); } +TEST(Btree, multiset_string) { MultiSetTest<std::string>(); } +TEST(Btree, multiset_cord) { MultiSetTest<absl::Cord>(); } +TEST(Btree, multiset_pair) { MultiSetTest<std::pair<int, int>>(); } +TEST(Btree, multimap_int32) { MultiMapTest<int32_t>(); } +TEST(Btree, multimap_int64) { MultiMapTest<int64_t>(); } +TEST(Btree, multimap_string) { MultiMapTest<std::string>(); } +TEST(Btree, multimap_cord) { MultiMapTest<absl::Cord>(); } +TEST(Btree, multimap_pair) { MultiMapTest<std::pair<int, int>>(); } + +struct CompareIntToString { + bool operator()(const std::string &a, const std::string &b) const { + return a < b; + } + bool operator()(const std::string &a, int b) const { + return a < absl::StrCat(b); + } + bool operator()(int a, const std::string &b) const { + return absl::StrCat(a) < b; + } + using is_transparent = void; +}; + +struct NonTransparentCompare { + template <typename T, typename U> + bool operator()(const T &t, const U &u) const { + // Treating all comparators as transparent can cause inefficiencies (see + // N3657 C++ proposal). Test that for comparators without 'is_transparent' + // alias (like this one), we do not attempt heterogeneous lookup. + EXPECT_TRUE((std::is_same<T, U>())); + return t < u; + } +}; + +template <typename T> +bool CanEraseWithEmptyBrace(T t, decltype(t.erase({})) *) { + return true; +} + +template <typename T> +bool CanEraseWithEmptyBrace(T, ...) { + return false; +} + +template <typename T> +void TestHeterogeneous(T table) { + auto lb = table.lower_bound("3"); + EXPECT_EQ(lb, table.lower_bound(3)); + EXPECT_NE(lb, table.lower_bound(4)); + EXPECT_EQ(lb, table.lower_bound({"3"})); + EXPECT_NE(lb, table.lower_bound({})); + + auto ub = table.upper_bound("3"); + EXPECT_EQ(ub, table.upper_bound(3)); + EXPECT_NE(ub, table.upper_bound(5)); + EXPECT_EQ(ub, table.upper_bound({"3"})); + EXPECT_NE(ub, table.upper_bound({})); + + auto er = table.equal_range("3"); + EXPECT_EQ(er, table.equal_range(3)); + EXPECT_NE(er, table.equal_range(4)); + EXPECT_EQ(er, table.equal_range({"3"})); + EXPECT_NE(er, table.equal_range({})); + + auto it = table.find("3"); + EXPECT_EQ(it, table.find(3)); + EXPECT_NE(it, table.find(4)); + EXPECT_EQ(it, table.find({"3"})); + EXPECT_NE(it, table.find({})); + + EXPECT_TRUE(table.contains(3)); + EXPECT_FALSE(table.contains(4)); + EXPECT_TRUE(table.count({"3"})); + EXPECT_FALSE(table.contains({})); + + EXPECT_EQ(1, table.count(3)); + EXPECT_EQ(0, table.count(4)); + EXPECT_EQ(1, table.count({"3"})); + EXPECT_EQ(0, table.count({})); + + auto copy = table; + copy.erase(3); + EXPECT_EQ(table.size() - 1, copy.size()); + copy.erase(4); + EXPECT_EQ(table.size() - 1, copy.size()); + copy.erase({"5"}); + EXPECT_EQ(table.size() - 2, copy.size()); + EXPECT_FALSE(CanEraseWithEmptyBrace(table, nullptr)); + + // Also run it with const T&. + if (std::is_class<T>()) TestHeterogeneous<const T &>(table); +} + +TEST(Btree, HeterogeneousLookup) { + TestHeterogeneous(btree_set<std::string, CompareIntToString>{"1", "3", "5"}); + TestHeterogeneous(btree_map<std::string, int, CompareIntToString>{ + {"1", 1}, {"3", 3}, {"5", 5}}); + TestHeterogeneous( + btree_multiset<std::string, CompareIntToString>{"1", "3", "5"}); + TestHeterogeneous(btree_multimap<std::string, int, CompareIntToString>{ + {"1", 1}, {"3", 3}, {"5", 5}}); + + // Only maps have .at() + btree_map<std::string, int, CompareIntToString> map{ + {"", -1}, {"1", 1}, {"3", 3}, {"5", 5}}; + EXPECT_EQ(1, map.at(1)); + EXPECT_EQ(3, map.at({"3"})); + EXPECT_EQ(-1, map.at({})); + const auto &cmap = map; + EXPECT_EQ(1, cmap.at(1)); + EXPECT_EQ(3, cmap.at({"3"})); + EXPECT_EQ(-1, cmap.at({})); +} + +TEST(Btree, NoHeterogeneousLookupWithoutAlias) { + using StringSet = absl::btree_set<std::string, NonTransparentCompare>; + StringSet s; + ASSERT_TRUE(s.insert("hello").second); + ASSERT_TRUE(s.insert("world").second); + EXPECT_TRUE(s.end() == s.find("blah")); + EXPECT_TRUE(s.begin() == s.lower_bound("hello")); + EXPECT_EQ(1, s.count("world")); + EXPECT_TRUE(s.contains("hello")); + EXPECT_TRUE(s.contains("world")); + EXPECT_FALSE(s.contains("blah")); + + using StringMultiSet = + absl::btree_multiset<std::string, NonTransparentCompare>; + StringMultiSet ms; + ms.insert("hello"); + ms.insert("world"); + ms.insert("world"); + EXPECT_TRUE(ms.end() == ms.find("blah")); + EXPECT_TRUE(ms.begin() == ms.lower_bound("hello")); + EXPECT_EQ(2, ms.count("world")); + EXPECT_TRUE(ms.contains("hello")); + EXPECT_TRUE(ms.contains("world")); + EXPECT_FALSE(ms.contains("blah")); +} + +TEST(Btree, DefaultTransparent) { + { + // `int` does not have a default transparent comparator. + // The input value is converted to key_type. + btree_set<int> s = {1}; + double d = 1.1; + EXPECT_EQ(s.begin(), s.find(d)); + EXPECT_TRUE(s.contains(d)); + } + + { + // `std::string` has heterogeneous support. + btree_set<std::string> s = {"A"}; + EXPECT_EQ(s.begin(), s.find(absl::string_view("A"))); + EXPECT_TRUE(s.contains(absl::string_view("A"))); + } +} + +class StringLike { + public: + StringLike() = default; + + StringLike(const char *s) : s_(s) { // NOLINT + ++constructor_calls_; + } + + bool operator<(const StringLike &a) const { return s_ < a.s_; } + + static void clear_constructor_call_count() { constructor_calls_ = 0; } + + static int constructor_calls() { return constructor_calls_; } + + private: + static int constructor_calls_; + std::string s_; +}; + +int StringLike::constructor_calls_ = 0; + +TEST(Btree, HeterogeneousLookupDoesntDegradePerformance) { + using StringSet = absl::btree_set<StringLike>; + StringSet s; + for (int i = 0; i < 100; ++i) { + ASSERT_TRUE(s.insert(absl::StrCat(i).c_str()).second); + } + StringLike::clear_constructor_call_count(); + s.find("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); + + StringLike::clear_constructor_call_count(); + s.contains("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); + + StringLike::clear_constructor_call_count(); + s.count("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); + + StringLike::clear_constructor_call_count(); + s.lower_bound("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); + + StringLike::clear_constructor_call_count(); + s.upper_bound("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); + + StringLike::clear_constructor_call_count(); + s.equal_range("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); + + StringLike::clear_constructor_call_count(); + s.erase("50"); + ASSERT_EQ(1, StringLike::constructor_calls()); +} + +// Verify that swapping btrees swaps the key comparison functors and that we can +// use non-default constructible comparators. +struct SubstringLess { + SubstringLess() = delete; + explicit SubstringLess(int length) : n(length) {} + bool operator()(const std::string &a, const std::string &b) const { + return absl::string_view(a).substr(0, n) < + absl::string_view(b).substr(0, n); + } + int n; +}; + +TEST(Btree, SwapKeyCompare) { + using SubstringSet = absl::btree_set<std::string, SubstringLess>; + SubstringSet s1(SubstringLess(1), SubstringSet::allocator_type()); + SubstringSet s2(SubstringLess(2), SubstringSet::allocator_type()); + + ASSERT_TRUE(s1.insert("a").second); + ASSERT_FALSE(s1.insert("aa").second); + + ASSERT_TRUE(s2.insert("a").second); + ASSERT_TRUE(s2.insert("aa").second); + ASSERT_FALSE(s2.insert("aaa").second); + + swap(s1, s2); + + ASSERT_TRUE(s1.insert("b").second); + ASSERT_TRUE(s1.insert("bb").second); + ASSERT_FALSE(s1.insert("bbb").second); + + ASSERT_TRUE(s2.insert("b").second); + ASSERT_FALSE(s2.insert("bb").second); +} + +TEST(Btree, UpperBoundRegression) { + // Regress a bug where upper_bound would default-construct a new key_compare + // instead of copying the existing one. + using SubstringSet = absl::btree_set<std::string, SubstringLess>; + SubstringSet my_set(SubstringLess(3)); + my_set.insert("aab"); + my_set.insert("abb"); + // We call upper_bound("aaa"). If this correctly uses the length 3 + // comparator, aaa < aab < abb, so we should get aab as the result. + // If it instead uses the default-constructed length 2 comparator, + // aa == aa < ab, so we'll get abb as our result. + SubstringSet::iterator it = my_set.upper_bound("aaa"); + ASSERT_TRUE(it != my_set.end()); + EXPECT_EQ("aab", *it); +} + +TEST(Btree, Comparison) { + const int kSetSize = 1201; + absl::btree_set<int64_t> my_set; + for (int i = 0; i < kSetSize; ++i) { + my_set.insert(i); + } + absl::btree_set<int64_t> my_set_copy(my_set); + EXPECT_TRUE(my_set_copy == my_set); + EXPECT_TRUE(my_set == my_set_copy); + EXPECT_FALSE(my_set_copy != my_set); + EXPECT_FALSE(my_set != my_set_copy); + + my_set.insert(kSetSize); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + my_set.erase(kSetSize - 1); + EXPECT_FALSE(my_set_copy == my_set); + EXPECT_FALSE(my_set == my_set_copy); + EXPECT_TRUE(my_set_copy != my_set); + EXPECT_TRUE(my_set != my_set_copy); + + absl::btree_map<std::string, int64_t> my_map; + for (int i = 0; i < kSetSize; ++i) { + my_map[std::string(i, 'a')] = i; + } + absl::btree_map<std::string, int64_t> my_map_copy(my_map); + EXPECT_TRUE(my_map_copy == my_map); + EXPECT_TRUE(my_map == my_map_copy); + EXPECT_FALSE(my_map_copy != my_map); + EXPECT_FALSE(my_map != my_map_copy); + + ++my_map_copy[std::string(7, 'a')]; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map_copy = my_map; + my_map["hello"] = kSetSize; + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); + + my_map.erase(std::string(kSetSize - 1, 'a')); + EXPECT_FALSE(my_map_copy == my_map); + EXPECT_FALSE(my_map == my_map_copy); + EXPECT_TRUE(my_map_copy != my_map); + EXPECT_TRUE(my_map != my_map_copy); +} + +TEST(Btree, RangeCtorSanity) { + std::vector<int> ivec; + ivec.push_back(1); + std::map<int, int> imap; + imap.insert(std::make_pair(1, 2)); + absl::btree_multiset<int> tmset(ivec.begin(), ivec.end()); + absl::btree_multimap<int, int> tmmap(imap.begin(), imap.end()); + absl::btree_set<int> tset(ivec.begin(), ivec.end()); + absl::btree_map<int, int> tmap(imap.begin(), imap.end()); + EXPECT_EQ(1, tmset.size()); + EXPECT_EQ(1, tmmap.size()); + EXPECT_EQ(1, tset.size()); + EXPECT_EQ(1, tmap.size()); +} + +} // namespace + +class BtreeNodePeer { + public: + // Yields the size of a leaf node with a specific number of values. + template <typename ValueType> + constexpr static size_t GetTargetNodeSize(size_t target_values_per_node) { + return btree_node< + set_params<ValueType, std::less<ValueType>, std::allocator<ValueType>, + /*TargetNodeSize=*/256, // This parameter isn't used here. + /*Multi=*/false>>::SizeWithNValues(target_values_per_node); + } + + // Yields the number of values in a (non-root) leaf node for this btree. + template <typename Btree> + constexpr static size_t GetNumValuesPerNode() { + return btree_node<typename Btree::params_type>::kNodeValues; + } + + template <typename Btree> + constexpr static size_t GetMaxFieldType() { + return std::numeric_limits< + typename btree_node<typename Btree::params_type>::field_type>::max(); + } + + template <typename Btree> + constexpr static bool UsesLinearNodeSearch() { + return btree_node<typename Btree::params_type>::use_linear_search::value; + } +}; + +namespace { + +class BtreeMapTest : public ::testing::Test { + public: + struct Key {}; + struct Cmp { + template <typename T> + bool operator()(T, T) const { + return false; + } + }; + + struct KeyLin { + using absl_btree_prefer_linear_node_search = std::true_type; + }; + struct CmpLin : Cmp { + using absl_btree_prefer_linear_node_search = std::true_type; + }; + + struct KeyBin { + using absl_btree_prefer_linear_node_search = std::false_type; + }; + struct CmpBin : Cmp { + using absl_btree_prefer_linear_node_search = std::false_type; + }; + + template <typename K, typename C> + static bool IsLinear() { + return BtreeNodePeer::UsesLinearNodeSearch<absl::btree_map<K, int, C>>(); + } +}; + +TEST_F(BtreeMapTest, TestLinearSearchPreferredForKeyLinearViaAlias) { + // Test requesting linear search by directly exporting an alias. + EXPECT_FALSE((IsLinear<Key, Cmp>())); + EXPECT_TRUE((IsLinear<KeyLin, Cmp>())); + EXPECT_TRUE((IsLinear<Key, CmpLin>())); + EXPECT_TRUE((IsLinear<KeyLin, CmpLin>())); +} + +TEST_F(BtreeMapTest, LinearChoiceTree) { + // Cmp has precedence, and is forcing binary + EXPECT_FALSE((IsLinear<Key, CmpBin>())); + EXPECT_FALSE((IsLinear<KeyLin, CmpBin>())); + EXPECT_FALSE((IsLinear<KeyBin, CmpBin>())); + EXPECT_FALSE((IsLinear<int, CmpBin>())); + EXPECT_FALSE((IsLinear<std::string, CmpBin>())); + // Cmp has precedence, and is forcing linear + EXPECT_TRUE((IsLinear<Key, CmpLin>())); + EXPECT_TRUE((IsLinear<KeyLin, CmpLin>())); + EXPECT_TRUE((IsLinear<KeyBin, CmpLin>())); + EXPECT_TRUE((IsLinear<int, CmpLin>())); + EXPECT_TRUE((IsLinear<std::string, CmpLin>())); + // Cmp has no preference, Key determines linear vs binary. + EXPECT_FALSE((IsLinear<Key, Cmp>())); + EXPECT_TRUE((IsLinear<KeyLin, Cmp>())); + EXPECT_FALSE((IsLinear<KeyBin, Cmp>())); + // arithmetic key w/ std::less or std::greater: linear + EXPECT_TRUE((IsLinear<int, std::less<int>>())); + EXPECT_TRUE((IsLinear<double, std::greater<double>>())); + // arithmetic key w/ custom compare: binary + EXPECT_FALSE((IsLinear<int, Cmp>())); + // non-arithmetic key: binary + EXPECT_FALSE((IsLinear<std::string, std::less<std::string>>())); +} + +TEST(Btree, BtreeMapCanHoldMoveOnlyTypes) { + absl::btree_map<std::string, std::unique_ptr<std::string>> m; + + std::unique_ptr<std::string> &v = m["A"]; + EXPECT_TRUE(v == nullptr); + v.reset(new std::string("X")); + + auto iter = m.find("A"); + EXPECT_EQ("X", *iter->second); +} + +TEST(Btree, InitializerListConstructor) { + absl::btree_set<std::string> set({"a", "b"}); + EXPECT_EQ(set.count("a"), 1); + EXPECT_EQ(set.count("b"), 1); + + absl::btree_multiset<int> mset({1, 1, 4}); + EXPECT_EQ(mset.count(1), 2); + EXPECT_EQ(mset.count(4), 1); + + absl::btree_map<int, int> map({{1, 5}, {2, 10}}); + EXPECT_EQ(map[1], 5); + EXPECT_EQ(map[2], 10); + + absl::btree_multimap<int, int> mmap({{1, 5}, {1, 10}}); + auto range = mmap.equal_range(1); + auto it = range.first; + ASSERT_NE(it, range.second); + EXPECT_EQ(it->second, 5); + ASSERT_NE(++it, range.second); + EXPECT_EQ(it->second, 10); + EXPECT_EQ(++it, range.second); +} + +TEST(Btree, InitializerListInsert) { + absl::btree_set<std::string> set; + set.insert({"a", "b"}); + EXPECT_EQ(set.count("a"), 1); + EXPECT_EQ(set.count("b"), 1); + + absl::btree_multiset<int> mset; + mset.insert({1, 1, 4}); + EXPECT_EQ(mset.count(1), 2); + EXPECT_EQ(mset.count(4), 1); + + absl::btree_map<int, int> map; + map.insert({{1, 5}, {2, 10}}); + // Test that inserting one element using an initializer list also works. + map.insert({3, 15}); + EXPECT_EQ(map[1], 5); + EXPECT_EQ(map[2], 10); + EXPECT_EQ(map[3], 15); + + absl::btree_multimap<int, int> mmap; + mmap.insert({{1, 5}, {1, 10}}); + auto range = mmap.equal_range(1); + auto it = range.first; + ASSERT_NE(it, range.second); + EXPECT_EQ(it->second, 5); + ASSERT_NE(++it, range.second); + EXPECT_EQ(it->second, 10); + EXPECT_EQ(++it, range.second); +} + +template <typename Compare, typename K> +void AssertKeyCompareToAdapted() { + using Adapted = typename key_compare_to_adapter<Compare>::type; + static_assert(!std::is_same<Adapted, Compare>::value, + "key_compare_to_adapter should have adapted this comparator."); + static_assert( + std::is_same<absl::weak_ordering, + absl::result_of_t<Adapted(const K &, const K &)>>::value, + "Adapted comparator should be a key-compare-to comparator."); +} +template <typename Compare, typename K> +void AssertKeyCompareToNotAdapted() { + using Unadapted = typename key_compare_to_adapter<Compare>::type; + static_assert( + std::is_same<Unadapted, Compare>::value, + "key_compare_to_adapter shouldn't have adapted this comparator."); + static_assert( + std::is_same<bool, + absl::result_of_t<Unadapted(const K &, const K &)>>::value, + "Un-adapted comparator should return bool."); +} + +TEST(Btree, KeyCompareToAdapter) { + AssertKeyCompareToAdapted<std::less<std::string>, std::string>(); + AssertKeyCompareToAdapted<std::greater<std::string>, std::string>(); + AssertKeyCompareToAdapted<std::less<absl::string_view>, absl::string_view>(); + AssertKeyCompareToAdapted<std::greater<absl::string_view>, + absl::string_view>(); + AssertKeyCompareToAdapted<std::less<absl::Cord>, absl::Cord>(); + AssertKeyCompareToAdapted<std::greater<absl::Cord>, absl::Cord>(); + AssertKeyCompareToNotAdapted<std::less<int>, int>(); + AssertKeyCompareToNotAdapted<std::greater<int>, int>(); +} + +TEST(Btree, RValueInsert) { + InstanceTracker tracker; + + absl::btree_set<MovableOnlyInstance> set; + set.insert(MovableOnlyInstance(1)); + set.insert(MovableOnlyInstance(3)); + MovableOnlyInstance two(2); + set.insert(set.find(MovableOnlyInstance(3)), std::move(two)); + auto it = set.find(MovableOnlyInstance(2)); + ASSERT_NE(it, set.end()); + ASSERT_NE(++it, set.end()); + EXPECT_EQ(it->value(), 3); + + absl::btree_multiset<MovableOnlyInstance> mset; + MovableOnlyInstance zero(0); + MovableOnlyInstance zero2(0); + mset.insert(std::move(zero)); + mset.insert(mset.find(MovableOnlyInstance(0)), std::move(zero2)); + EXPECT_EQ(mset.count(MovableOnlyInstance(0)), 2); + + absl::btree_map<int, MovableOnlyInstance> map; + std::pair<const int, MovableOnlyInstance> p1 = {1, MovableOnlyInstance(5)}; + std::pair<const int, MovableOnlyInstance> p2 = {2, MovableOnlyInstance(10)}; + std::pair<const int, MovableOnlyInstance> p3 = {3, MovableOnlyInstance(15)}; + map.insert(std::move(p1)); + map.insert(std::move(p3)); + map.insert(map.find(3), std::move(p2)); + ASSERT_NE(map.find(2), map.end()); + EXPECT_EQ(map.find(2)->second.value(), 10); + + absl::btree_multimap<int, MovableOnlyInstance> mmap; + std::pair<const int, MovableOnlyInstance> p4 = {1, MovableOnlyInstance(5)}; + std::pair<const int, MovableOnlyInstance> p5 = {1, MovableOnlyInstance(10)}; + mmap.insert(std::move(p4)); + mmap.insert(mmap.find(1), std::move(p5)); + auto range = mmap.equal_range(1); + auto it1 = range.first; + ASSERT_NE(it1, range.second); + EXPECT_EQ(it1->second.value(), 10); + ASSERT_NE(++it1, range.second); + EXPECT_EQ(it1->second.value(), 5); + EXPECT_EQ(++it1, range.second); + + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.swaps(), 0); +} + +// A btree set with a specific number of values per node. +template <typename Key, int TargetValuesPerNode, typename Cmp = std::less<Key>> +class SizedBtreeSet + : public btree_set_container<btree< + set_params<Key, Cmp, std::allocator<Key>, + BtreeNodePeer::GetTargetNodeSize<Key>(TargetValuesPerNode), + /*Multi=*/false>>> { + using Base = typename SizedBtreeSet::btree_set_container; + + public: + SizedBtreeSet() {} + using Base::Base; +}; + +template <typename Set> +void ExpectOperationCounts(const int expected_moves, + const int expected_comparisons, + const std::vector<int> &values, + InstanceTracker *tracker, Set *set) { + for (const int v : values) set->insert(MovableOnlyInstance(v)); + set->clear(); + EXPECT_EQ(tracker->moves(), expected_moves); + EXPECT_EQ(tracker->comparisons(), expected_comparisons); + EXPECT_EQ(tracker->copies(), 0); + EXPECT_EQ(tracker->swaps(), 0); + tracker->ResetCopiesMovesSwaps(); +} + +// Note: when the values in this test change, it is expected to have an impact +// on performance. +TEST(Btree, MovesComparisonsCopiesSwapsTracking) { + InstanceTracker tracker; + // Note: this is minimum number of values per node. + SizedBtreeSet<MovableOnlyInstance, /*TargetValuesPerNode=*/3> set3; + // Note: this is the default number of values per node for a set of int32s + // (with 64-bit pointers). + SizedBtreeSet<MovableOnlyInstance, /*TargetValuesPerNode=*/61> set61; + SizedBtreeSet<MovableOnlyInstance, /*TargetValuesPerNode=*/100> set100; + + // Don't depend on flags for random values because then the expectations will + // fail if the flags change. + std::vector<int> values = + GenerateValuesWithSeed<int>(10000, 1 << 22, /*seed=*/23); + + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<decltype(set3)>(), 3); + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<decltype(set61)>(), 61); + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<decltype(set100)>(), 100); + if (sizeof(void *) == 8) { + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<absl::btree_set<int32_t>>(), + BtreeNodePeer::GetNumValuesPerNode<decltype(set61)>()); + } + + // Test key insertion/deletion in random order. + ExpectOperationCounts(45281, 132551, values, &tracker, &set3); + ExpectOperationCounts(386718, 129807, values, &tracker, &set61); + ExpectOperationCounts(586761, 130310, values, &tracker, &set100); + + // Test key insertion/deletion in sorted order. + std::sort(values.begin(), values.end()); + ExpectOperationCounts(26638, 92134, values, &tracker, &set3); + ExpectOperationCounts(20208, 87757, values, &tracker, &set61); + ExpectOperationCounts(20124, 96583, values, &tracker, &set100); + + // Test key insertion/deletion in reverse sorted order. + std::reverse(values.begin(), values.end()); + ExpectOperationCounts(49951, 119325, values, &tracker, &set3); + ExpectOperationCounts(338813, 118266, values, &tracker, &set61); + ExpectOperationCounts(534529, 125279, values, &tracker, &set100); +} + +struct MovableOnlyInstanceThreeWayCompare { + absl::weak_ordering operator()(const MovableOnlyInstance &a, + const MovableOnlyInstance &b) const { + return a.compare(b); + } +}; + +// Note: when the values in this test change, it is expected to have an impact +// on performance. +TEST(Btree, MovesComparisonsCopiesSwapsTrackingThreeWayCompare) { + InstanceTracker tracker; + // Note: this is minimum number of values per node. + SizedBtreeSet<MovableOnlyInstance, /*TargetValuesPerNode=*/3, + MovableOnlyInstanceThreeWayCompare> + set3; + // Note: this is the default number of values per node for a set of int32s + // (with 64-bit pointers). + SizedBtreeSet<MovableOnlyInstance, /*TargetValuesPerNode=*/61, + MovableOnlyInstanceThreeWayCompare> + set61; + SizedBtreeSet<MovableOnlyInstance, /*TargetValuesPerNode=*/100, + MovableOnlyInstanceThreeWayCompare> + set100; + + // Don't depend on flags for random values because then the expectations will + // fail if the flags change. + std::vector<int> values = + GenerateValuesWithSeed<int>(10000, 1 << 22, /*seed=*/23); + + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<decltype(set3)>(), 3); + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<decltype(set61)>(), 61); + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<decltype(set100)>(), 100); + if (sizeof(void *) == 8) { + EXPECT_EQ(BtreeNodePeer::GetNumValuesPerNode<absl::btree_set<int32_t>>(), + BtreeNodePeer::GetNumValuesPerNode<decltype(set61)>()); + } + + // Test key insertion/deletion in random order. + ExpectOperationCounts(45281, 122560, values, &tracker, &set3); + ExpectOperationCounts(386718, 119816, values, &tracker, &set61); + ExpectOperationCounts(586761, 120319, values, &tracker, &set100); + + // Test key insertion/deletion in sorted order. + std::sort(values.begin(), values.end()); + ExpectOperationCounts(26638, 92134, values, &tracker, &set3); + ExpectOperationCounts(20208, 87757, values, &tracker, &set61); + ExpectOperationCounts(20124, 96583, values, &tracker, &set100); + + // Test key insertion/deletion in reverse sorted order. + std::reverse(values.begin(), values.end()); + ExpectOperationCounts(49951, 109326, values, &tracker, &set3); + ExpectOperationCounts(338813, 108267, values, &tracker, &set61); + ExpectOperationCounts(534529, 115280, values, &tracker, &set100); +} + +struct NoDefaultCtor { + int num; + explicit NoDefaultCtor(int i) : num(i) {} + + friend bool operator<(const NoDefaultCtor &a, const NoDefaultCtor &b) { + return a.num < b.num; + } +}; + +TEST(Btree, BtreeMapCanHoldNoDefaultCtorTypes) { + absl::btree_map<NoDefaultCtor, NoDefaultCtor> m; + + for (int i = 1; i <= 99; ++i) { + SCOPED_TRACE(i); + EXPECT_TRUE(m.emplace(NoDefaultCtor(i), NoDefaultCtor(100 - i)).second); + } + EXPECT_FALSE(m.emplace(NoDefaultCtor(78), NoDefaultCtor(0)).second); + + auto iter99 = m.find(NoDefaultCtor(99)); + ASSERT_NE(iter99, m.end()); + EXPECT_EQ(iter99->second.num, 1); + + auto iter1 = m.find(NoDefaultCtor(1)); + ASSERT_NE(iter1, m.end()); + EXPECT_EQ(iter1->second.num, 99); + + auto iter50 = m.find(NoDefaultCtor(50)); + ASSERT_NE(iter50, m.end()); + EXPECT_EQ(iter50->second.num, 50); + + auto iter25 = m.find(NoDefaultCtor(25)); + ASSERT_NE(iter25, m.end()); + EXPECT_EQ(iter25->second.num, 75); +} + +TEST(Btree, BtreeMultimapCanHoldNoDefaultCtorTypes) { + absl::btree_multimap<NoDefaultCtor, NoDefaultCtor> m; + + for (int i = 1; i <= 99; ++i) { + SCOPED_TRACE(i); + m.emplace(NoDefaultCtor(i), NoDefaultCtor(100 - i)); + } + + auto iter99 = m.find(NoDefaultCtor(99)); + ASSERT_NE(iter99, m.end()); + EXPECT_EQ(iter99->second.num, 1); + + auto iter1 = m.find(NoDefaultCtor(1)); + ASSERT_NE(iter1, m.end()); + EXPECT_EQ(iter1->second.num, 99); + + auto iter50 = m.find(NoDefaultCtor(50)); + ASSERT_NE(iter50, m.end()); + EXPECT_EQ(iter50->second.num, 50); + + auto iter25 = m.find(NoDefaultCtor(25)); + ASSERT_NE(iter25, m.end()); + EXPECT_EQ(iter25->second.num, 75); +} + +TEST(Btree, MapAt) { + absl::btree_map<int, int> map = {{1, 2}, {2, 4}}; + EXPECT_EQ(map.at(1), 2); + EXPECT_EQ(map.at(2), 4); + map.at(2) = 8; + const absl::btree_map<int, int> &const_map = map; + EXPECT_EQ(const_map.at(1), 2); + EXPECT_EQ(const_map.at(2), 8); +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(map.at(3), std::out_of_range); +#else + EXPECT_DEATH_IF_SUPPORTED(map.at(3), "absl::btree_map::at"); +#endif +} + +TEST(Btree, BtreeMultisetEmplace) { + const int value_to_insert = 123456; + absl::btree_multiset<int> s; + auto iter = s.emplace(value_to_insert); + ASSERT_NE(iter, s.end()); + EXPECT_EQ(*iter, value_to_insert); + auto iter2 = s.emplace(value_to_insert); + EXPECT_NE(iter2, iter); + ASSERT_NE(iter2, s.end()); + EXPECT_EQ(*iter2, value_to_insert); + auto result = s.equal_range(value_to_insert); + EXPECT_EQ(std::distance(result.first, result.second), 2); +} + +TEST(Btree, BtreeMultisetEmplaceHint) { + const int value_to_insert = 123456; + absl::btree_multiset<int> s; + auto iter = s.emplace(value_to_insert); + ASSERT_NE(iter, s.end()); + EXPECT_EQ(*iter, value_to_insert); + auto emplace_iter = s.emplace_hint(iter, value_to_insert); + EXPECT_NE(emplace_iter, iter); + ASSERT_NE(emplace_iter, s.end()); + EXPECT_EQ(*emplace_iter, value_to_insert); +} + +TEST(Btree, BtreeMultimapEmplace) { + const int key_to_insert = 123456; + const char value0[] = "a"; + absl::btree_multimap<int, std::string> s; + auto iter = s.emplace(key_to_insert, value0); + ASSERT_NE(iter, s.end()); + EXPECT_EQ(iter->first, key_to_insert); + EXPECT_EQ(iter->second, value0); + const char value1[] = "b"; + auto iter2 = s.emplace(key_to_insert, value1); + EXPECT_NE(iter2, iter); + ASSERT_NE(iter2, s.end()); + EXPECT_EQ(iter2->first, key_to_insert); + EXPECT_EQ(iter2->second, value1); + auto result = s.equal_range(key_to_insert); + EXPECT_EQ(std::distance(result.first, result.second), 2); +} + +TEST(Btree, BtreeMultimapEmplaceHint) { + const int key_to_insert = 123456; + const char value0[] = "a"; + absl::btree_multimap<int, std::string> s; + auto iter = s.emplace(key_to_insert, value0); + ASSERT_NE(iter, s.end()); + EXPECT_EQ(iter->first, key_to_insert); + EXPECT_EQ(iter->second, value0); + const char value1[] = "b"; + auto emplace_iter = s.emplace_hint(iter, key_to_insert, value1); + EXPECT_NE(emplace_iter, iter); + ASSERT_NE(emplace_iter, s.end()); + EXPECT_EQ(emplace_iter->first, key_to_insert); + EXPECT_EQ(emplace_iter->second, value1); +} + +TEST(Btree, ConstIteratorAccessors) { + absl::btree_set<int> set; + for (int i = 0; i < 100; ++i) { + set.insert(i); + } + + auto it = set.cbegin(); + auto r_it = set.crbegin(); + for (int i = 0; i < 100; ++i, ++it, ++r_it) { + ASSERT_EQ(*it, i); + ASSERT_EQ(*r_it, 99 - i); + } + EXPECT_EQ(it, set.cend()); + EXPECT_EQ(r_it, set.crend()); +} + +TEST(Btree, StrSplitCompatible) { + const absl::btree_set<std::string> split_set = absl::StrSplit("a,b,c", ','); + const absl::btree_set<std::string> expected_set = {"a", "b", "c"}; + + EXPECT_EQ(split_set, expected_set); +} + +// We can't use EXPECT_EQ/etc. to compare absl::weak_ordering because they +// convert literal 0 to int and absl::weak_ordering can only be compared with +// literal 0. Defining this function allows for avoiding ClangTidy warnings. +bool Identity(const bool b) { return b; } + +TEST(Btree, ValueComp) { + absl::btree_set<int> s; + EXPECT_TRUE(s.value_comp()(1, 2)); + EXPECT_FALSE(s.value_comp()(2, 2)); + EXPECT_FALSE(s.value_comp()(2, 1)); + + absl::btree_map<int, int> m1; + EXPECT_TRUE(m1.value_comp()(std::make_pair(1, 0), std::make_pair(2, 0))); + EXPECT_FALSE(m1.value_comp()(std::make_pair(2, 0), std::make_pair(2, 0))); + EXPECT_FALSE(m1.value_comp()(std::make_pair(2, 0), std::make_pair(1, 0))); + + absl::btree_map<std::string, int> m2; + EXPECT_TRUE(Identity( + m2.value_comp()(std::make_pair("a", 0), std::make_pair("b", 0)) < 0)); + EXPECT_TRUE(Identity( + m2.value_comp()(std::make_pair("b", 0), std::make_pair("b", 0)) == 0)); + EXPECT_TRUE(Identity( + m2.value_comp()(std::make_pair("b", 0), std::make_pair("a", 0)) > 0)); +} + +TEST(Btree, DefaultConstruction) { + absl::btree_set<int> s; + absl::btree_map<int, int> m; + absl::btree_multiset<int> ms; + absl::btree_multimap<int, int> mm; + + EXPECT_TRUE(s.empty()); + EXPECT_TRUE(m.empty()); + EXPECT_TRUE(ms.empty()); + EXPECT_TRUE(mm.empty()); +} + +TEST(Btree, SwissTableHashable) { + static constexpr int kValues = 10000; + std::vector<int> values(kValues); + std::iota(values.begin(), values.end(), 0); + std::vector<std::pair<int, int>> map_values; + for (int v : values) map_values.emplace_back(v, -v); + + using set = absl::btree_set<int>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + set{}, + set{1}, + set{2}, + set{1, 2}, + set{2, 1}, + set(values.begin(), values.end()), + set(values.rbegin(), values.rend()), + })); + + using mset = absl::btree_multiset<int>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + mset{}, + mset{1}, + mset{1, 1}, + mset{2}, + mset{2, 2}, + mset{1, 2}, + mset{1, 1, 2}, + mset{1, 2, 2}, + mset{1, 1, 2, 2}, + mset(values.begin(), values.end()), + mset(values.rbegin(), values.rend()), + })); + + using map = absl::btree_map<int, int>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + map{}, + map{{1, 0}}, + map{{1, 1}}, + map{{2, 0}}, + map{{2, 2}}, + map{{1, 0}, {2, 1}}, + map(map_values.begin(), map_values.end()), + map(map_values.rbegin(), map_values.rend()), + })); + + using mmap = absl::btree_multimap<int, int>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + mmap{}, + mmap{{1, 0}}, + mmap{{1, 1}}, + mmap{{1, 0}, {1, 1}}, + mmap{{1, 1}, {1, 0}}, + mmap{{2, 0}}, + mmap{{2, 2}}, + mmap{{1, 0}, {2, 1}}, + mmap(map_values.begin(), map_values.end()), + mmap(map_values.rbegin(), map_values.rend()), + })); +} + +TEST(Btree, ComparableSet) { + absl::btree_set<int> s1 = {1, 2}; + absl::btree_set<int> s2 = {2, 3}; + EXPECT_LT(s1, s2); + EXPECT_LE(s1, s2); + EXPECT_LE(s1, s1); + EXPECT_GT(s2, s1); + EXPECT_GE(s2, s1); + EXPECT_GE(s1, s1); +} + +TEST(Btree, ComparableSetsDifferentLength) { + absl::btree_set<int> s1 = {1, 2}; + absl::btree_set<int> s2 = {1, 2, 3}; + EXPECT_LT(s1, s2); + EXPECT_LE(s1, s2); + EXPECT_GT(s2, s1); + EXPECT_GE(s2, s1); +} + +TEST(Btree, ComparableMultiset) { + absl::btree_multiset<int> s1 = {1, 2}; + absl::btree_multiset<int> s2 = {2, 3}; + EXPECT_LT(s1, s2); + EXPECT_LE(s1, s2); + EXPECT_LE(s1, s1); + EXPECT_GT(s2, s1); + EXPECT_GE(s2, s1); + EXPECT_GE(s1, s1); +} + +TEST(Btree, ComparableMap) { + absl::btree_map<int, int> s1 = {{1, 2}}; + absl::btree_map<int, int> s2 = {{2, 3}}; + EXPECT_LT(s1, s2); + EXPECT_LE(s1, s2); + EXPECT_LE(s1, s1); + EXPECT_GT(s2, s1); + EXPECT_GE(s2, s1); + EXPECT_GE(s1, s1); +} + +TEST(Btree, ComparableMultimap) { + absl::btree_multimap<int, int> s1 = {{1, 2}}; + absl::btree_multimap<int, int> s2 = {{2, 3}}; + EXPECT_LT(s1, s2); + EXPECT_LE(s1, s2); + EXPECT_LE(s1, s1); + EXPECT_GT(s2, s1); + EXPECT_GE(s2, s1); + EXPECT_GE(s1, s1); +} + +TEST(Btree, ComparableSetWithCustomComparator) { + // As specified by + // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2012/n3337.pdf section + // [container.requirements.general].12, ordering associative containers always + // uses default '<' operator + // - even if otherwise the container uses custom functor. + absl::btree_set<int, std::greater<int>> s1 = {1, 2}; + absl::btree_set<int, std::greater<int>> s2 = {2, 3}; + EXPECT_LT(s1, s2); + EXPECT_LE(s1, s2); + EXPECT_LE(s1, s1); + EXPECT_GT(s2, s1); + EXPECT_GE(s2, s1); + EXPECT_GE(s1, s1); +} + +TEST(Btree, EraseReturnsIterator) { + absl::btree_set<int> set = {1, 2, 3, 4, 5}; + auto result_it = set.erase(set.begin(), set.find(3)); + EXPECT_EQ(result_it, set.find(3)); + result_it = set.erase(set.find(5)); + EXPECT_EQ(result_it, set.end()); +} + +TEST(Btree, ExtractAndInsertNodeHandleSet) { + absl::btree_set<int> src1 = {1, 2, 3, 4, 5}; + auto nh = src1.extract(src1.find(3)); + EXPECT_THAT(src1, ElementsAre(1, 2, 4, 5)); + absl::btree_set<int> other; + absl::btree_set<int>::insert_return_type res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(3)); + EXPECT_EQ(res.position, other.find(3)); + EXPECT_TRUE(res.inserted); + EXPECT_TRUE(res.node.empty()); + + absl::btree_set<int> src2 = {3, 4}; + nh = src2.extract(src2.find(3)); + EXPECT_THAT(src2, ElementsAre(4)); + res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(3)); + EXPECT_EQ(res.position, other.find(3)); + EXPECT_FALSE(res.inserted); + ASSERT_FALSE(res.node.empty()); + EXPECT_EQ(res.node.value(), 3); +} + +template <typename Set> +void TestExtractWithTrackingForSet() { + InstanceTracker tracker; + { + Set s; + // Add enough elements to make sure we test internal nodes too. + const size_t kSize = 1000; + while (s.size() < kSize) { + s.insert(MovableOnlyInstance(s.size())); + } + for (int i = 0; i < kSize; ++i) { + // Extract with key + auto nh = s.extract(MovableOnlyInstance(i)); + EXPECT_EQ(s.size(), kSize - 1); + EXPECT_EQ(nh.value().value(), i); + // Insert with node + s.insert(std::move(nh)); + EXPECT_EQ(s.size(), kSize); + + // Extract with iterator + auto it = s.find(MovableOnlyInstance(i)); + nh = s.extract(it); + EXPECT_EQ(s.size(), kSize - 1); + EXPECT_EQ(nh.value().value(), i); + // Insert with node and hint + s.insert(s.begin(), std::move(nh)); + EXPECT_EQ(s.size(), kSize); + } + } + EXPECT_EQ(0, tracker.instances()); +} + +template <typename Map> +void TestExtractWithTrackingForMap() { + InstanceTracker tracker; + { + Map m; + // Add enough elements to make sure we test internal nodes too. + const size_t kSize = 1000; + while (m.size() < kSize) { + m.insert( + {CopyableMovableInstance(m.size()), MovableOnlyInstance(m.size())}); + } + for (int i = 0; i < kSize; ++i) { + // Extract with key + auto nh = m.extract(CopyableMovableInstance(i)); + EXPECT_EQ(m.size(), kSize - 1); + EXPECT_EQ(nh.key().value(), i); + EXPECT_EQ(nh.mapped().value(), i); + // Insert with node + m.insert(std::move(nh)); + EXPECT_EQ(m.size(), kSize); + + // Extract with iterator + auto it = m.find(CopyableMovableInstance(i)); + nh = m.extract(it); + EXPECT_EQ(m.size(), kSize - 1); + EXPECT_EQ(nh.key().value(), i); + EXPECT_EQ(nh.mapped().value(), i); + // Insert with node and hint + m.insert(m.begin(), std::move(nh)); + EXPECT_EQ(m.size(), kSize); + } + } + EXPECT_EQ(0, tracker.instances()); +} + +TEST(Btree, ExtractTracking) { + TestExtractWithTrackingForSet<absl::btree_set<MovableOnlyInstance>>(); + TestExtractWithTrackingForSet<absl::btree_multiset<MovableOnlyInstance>>(); + TestExtractWithTrackingForMap< + absl::btree_map<CopyableMovableInstance, MovableOnlyInstance>>(); + TestExtractWithTrackingForMap< + absl::btree_multimap<CopyableMovableInstance, MovableOnlyInstance>>(); +} + +TEST(Btree, ExtractAndInsertNodeHandleMultiSet) { + absl::btree_multiset<int> src1 = {1, 2, 3, 3, 4, 5}; + auto nh = src1.extract(src1.find(3)); + EXPECT_THAT(src1, ElementsAre(1, 2, 3, 4, 5)); + absl::btree_multiset<int> other; + auto res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(3)); + EXPECT_EQ(res, other.find(3)); + + absl::btree_multiset<int> src2 = {3, 4}; + nh = src2.extract(src2.find(3)); + EXPECT_THAT(src2, ElementsAre(4)); + res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(3, 3)); + EXPECT_EQ(res, ++other.find(3)); +} + +TEST(Btree, ExtractAndInsertNodeHandleMap) { + absl::btree_map<int, int> src1 = {{1, 2}, {3, 4}, {5, 6}}; + auto nh = src1.extract(src1.find(3)); + EXPECT_THAT(src1, ElementsAre(Pair(1, 2), Pair(5, 6))); + absl::btree_map<int, int> other; + absl::btree_map<int, int>::insert_return_type res = + other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(Pair(3, 4))); + EXPECT_EQ(res.position, other.find(3)); + EXPECT_TRUE(res.inserted); + EXPECT_TRUE(res.node.empty()); + + absl::btree_map<int, int> src2 = {{3, 6}}; + nh = src2.extract(src2.find(3)); + EXPECT_TRUE(src2.empty()); + res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(Pair(3, 4))); + EXPECT_EQ(res.position, other.find(3)); + EXPECT_FALSE(res.inserted); + ASSERT_FALSE(res.node.empty()); + EXPECT_EQ(res.node.key(), 3); + EXPECT_EQ(res.node.mapped(), 6); +} + +TEST(Btree, ExtractAndInsertNodeHandleMultiMap) { + absl::btree_multimap<int, int> src1 = {{1, 2}, {3, 4}, {5, 6}}; + auto nh = src1.extract(src1.find(3)); + EXPECT_THAT(src1, ElementsAre(Pair(1, 2), Pair(5, 6))); + absl::btree_multimap<int, int> other; + auto res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(Pair(3, 4))); + EXPECT_EQ(res, other.find(3)); + + absl::btree_multimap<int, int> src2 = {{3, 6}}; + nh = src2.extract(src2.find(3)); + EXPECT_TRUE(src2.empty()); + res = other.insert(std::move(nh)); + EXPECT_THAT(other, ElementsAre(Pair(3, 4), Pair(3, 6))); + EXPECT_EQ(res, ++other.begin()); +} + +// For multisets, insert with hint also affects correctness because we need to +// insert immediately before the hint if possible. +struct InsertMultiHintData { + int key; + int not_key; + bool operator==(const InsertMultiHintData other) const { + return key == other.key && not_key == other.not_key; + } +}; + +struct InsertMultiHintDataKeyCompare { + using is_transparent = void; + bool operator()(const InsertMultiHintData a, + const InsertMultiHintData b) const { + return a.key < b.key; + } + bool operator()(const int a, const InsertMultiHintData b) const { + return a < b.key; + } + bool operator()(const InsertMultiHintData a, const int b) const { + return a.key < b; + } +}; + +TEST(Btree, InsertHintNodeHandle) { + // For unique sets, insert with hint is just a performance optimization. + // Test that insert works correctly when the hint is right or wrong. + { + absl::btree_set<int> src = {1, 2, 3, 4, 5}; + auto nh = src.extract(src.find(3)); + EXPECT_THAT(src, ElementsAre(1, 2, 4, 5)); + absl::btree_set<int> other = {0, 100}; + // Test a correct hint. + auto it = other.insert(other.lower_bound(3), std::move(nh)); + EXPECT_THAT(other, ElementsAre(0, 3, 100)); + EXPECT_EQ(it, other.find(3)); + + nh = src.extract(src.find(5)); + // Test an incorrect hint. + it = other.insert(other.end(), std::move(nh)); + EXPECT_THAT(other, ElementsAre(0, 3, 5, 100)); + EXPECT_EQ(it, other.find(5)); + } + + absl::btree_multiset<InsertMultiHintData, InsertMultiHintDataKeyCompare> src = + {{1, 2}, {3, 4}, {3, 5}}; + auto nh = src.extract(src.lower_bound(3)); + EXPECT_EQ(nh.value(), (InsertMultiHintData{3, 4})); + absl::btree_multiset<InsertMultiHintData, InsertMultiHintDataKeyCompare> + other = {{3, 1}, {3, 2}, {3, 3}}; + auto it = other.insert(--other.end(), std::move(nh)); + EXPECT_THAT( + other, ElementsAre(InsertMultiHintData{3, 1}, InsertMultiHintData{3, 2}, + InsertMultiHintData{3, 4}, InsertMultiHintData{3, 3})); + EXPECT_EQ(it, --(--other.end())); + + nh = src.extract(src.find(3)); + EXPECT_EQ(nh.value(), (InsertMultiHintData{3, 5})); + it = other.insert(other.begin(), std::move(nh)); + EXPECT_THAT(other, + ElementsAre(InsertMultiHintData{3, 5}, InsertMultiHintData{3, 1}, + InsertMultiHintData{3, 2}, InsertMultiHintData{3, 4}, + InsertMultiHintData{3, 3})); + EXPECT_EQ(it, other.begin()); +} + +struct IntCompareToCmp { + absl::weak_ordering operator()(int a, int b) const { + if (a < b) return absl::weak_ordering::less; + if (a > b) return absl::weak_ordering::greater; + return absl::weak_ordering::equivalent; + } +}; + +TEST(Btree, MergeIntoUniqueContainers) { + absl::btree_set<int, IntCompareToCmp> src1 = {1, 2, 3}; + absl::btree_multiset<int> src2 = {3, 4, 4, 5}; + absl::btree_set<int> dst; + + dst.merge(src1); + EXPECT_TRUE(src1.empty()); + EXPECT_THAT(dst, ElementsAre(1, 2, 3)); + dst.merge(src2); + EXPECT_THAT(src2, ElementsAre(3, 4)); + EXPECT_THAT(dst, ElementsAre(1, 2, 3, 4, 5)); +} + +TEST(Btree, MergeIntoUniqueContainersWithCompareTo) { + absl::btree_set<int, IntCompareToCmp> src1 = {1, 2, 3}; + absl::btree_multiset<int> src2 = {3, 4, 4, 5}; + absl::btree_set<int, IntCompareToCmp> dst; + + dst.merge(src1); + EXPECT_TRUE(src1.empty()); + EXPECT_THAT(dst, ElementsAre(1, 2, 3)); + dst.merge(src2); + EXPECT_THAT(src2, ElementsAre(3, 4)); + EXPECT_THAT(dst, ElementsAre(1, 2, 3, 4, 5)); +} + +TEST(Btree, MergeIntoMultiContainers) { + absl::btree_set<int, IntCompareToCmp> src1 = {1, 2, 3}; + absl::btree_multiset<int> src2 = {3, 4, 4, 5}; + absl::btree_multiset<int> dst; + + dst.merge(src1); + EXPECT_TRUE(src1.empty()); + EXPECT_THAT(dst, ElementsAre(1, 2, 3)); + dst.merge(src2); + EXPECT_TRUE(src2.empty()); + EXPECT_THAT(dst, ElementsAre(1, 2, 3, 3, 4, 4, 5)); +} + +TEST(Btree, MergeIntoMultiContainersWithCompareTo) { + absl::btree_set<int, IntCompareToCmp> src1 = {1, 2, 3}; + absl::btree_multiset<int> src2 = {3, 4, 4, 5}; + absl::btree_multiset<int, IntCompareToCmp> dst; + + dst.merge(src1); + EXPECT_TRUE(src1.empty()); + EXPECT_THAT(dst, ElementsAre(1, 2, 3)); + dst.merge(src2); + EXPECT_TRUE(src2.empty()); + EXPECT_THAT(dst, ElementsAre(1, 2, 3, 3, 4, 4, 5)); +} + +TEST(Btree, MergeIntoMultiMapsWithDifferentComparators) { + absl::btree_map<int, int, IntCompareToCmp> src1 = {{1, 1}, {2, 2}, {3, 3}}; + absl::btree_multimap<int, int, std::greater<int>> src2 = { + {5, 5}, {4, 1}, {4, 4}, {3, 2}}; + absl::btree_multimap<int, int> dst; + + dst.merge(src1); + EXPECT_TRUE(src1.empty()); + EXPECT_THAT(dst, ElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3))); + dst.merge(src2); + EXPECT_TRUE(src2.empty()); + EXPECT_THAT(dst, ElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), Pair(3, 2), + Pair(4, 1), Pair(4, 4), Pair(5, 5))); +} + +TEST(Btree, MergeIntoSetMovableOnly) { + absl::btree_set<MovableOnlyInstance> src; + src.insert(MovableOnlyInstance(1)); + absl::btree_multiset<MovableOnlyInstance> dst1; + dst1.insert(MovableOnlyInstance(2)); + absl::btree_set<MovableOnlyInstance> dst2; + + // Test merge into multiset. + dst1.merge(src); + + EXPECT_TRUE(src.empty()); + // ElementsAre/ElementsAreArray don't work with move-only types. + ASSERT_THAT(dst1, SizeIs(2)); + EXPECT_EQ(*dst1.begin(), MovableOnlyInstance(1)); + EXPECT_EQ(*std::next(dst1.begin()), MovableOnlyInstance(2)); + + // Test merge into set. + dst2.merge(dst1); + + EXPECT_TRUE(dst1.empty()); + ASSERT_THAT(dst2, SizeIs(2)); + EXPECT_EQ(*dst2.begin(), MovableOnlyInstance(1)); + EXPECT_EQ(*std::next(dst2.begin()), MovableOnlyInstance(2)); +} + +struct KeyCompareToWeakOrdering { + template <typename T> + absl::weak_ordering operator()(const T &a, const T &b) const { + return a < b ? absl::weak_ordering::less + : a == b ? absl::weak_ordering::equivalent + : absl::weak_ordering::greater; + } +}; + +struct KeyCompareToStrongOrdering { + template <typename T> + absl::strong_ordering operator()(const T &a, const T &b) const { + return a < b ? absl::strong_ordering::less + : a == b ? absl::strong_ordering::equal + : absl::strong_ordering::greater; + } +}; + +TEST(Btree, UserProvidedKeyCompareToComparators) { + absl::btree_set<int, KeyCompareToWeakOrdering> weak_set = {1, 2, 3}; + EXPECT_TRUE(weak_set.contains(2)); + EXPECT_FALSE(weak_set.contains(4)); + + absl::btree_set<int, KeyCompareToStrongOrdering> strong_set = {1, 2, 3}; + EXPECT_TRUE(strong_set.contains(2)); + EXPECT_FALSE(strong_set.contains(4)); +} + +TEST(Btree, TryEmplaceBasicTest) { + absl::btree_map<int, std::string> m; + + // Should construct a string from the literal. + m.try_emplace(1, "one"); + EXPECT_EQ(1, m.size()); + + // Try other string constructors and const lvalue key. + const int key(42); + m.try_emplace(key, 3, 'a'); + m.try_emplace(2, std::string("two")); + + EXPECT_TRUE(std::is_sorted(m.begin(), m.end())); + EXPECT_THAT(m, ElementsAreArray(std::vector<std::pair<int, std::string>>{ + {1, "one"}, {2, "two"}, {42, "aaa"}})); +} + +TEST(Btree, TryEmplaceWithHintWorks) { + // Use a counting comparator here to verify that hint is used. + int calls = 0; + auto cmp = [&calls](int x, int y) { + ++calls; + return x < y; + }; + using Cmp = decltype(cmp); + + absl::btree_map<int, int, Cmp> m(cmp); + for (int i = 0; i < 128; ++i) { + m.emplace(i, i); + } + + // Sanity check for the comparator + calls = 0; + m.emplace(127, 127); + EXPECT_GE(calls, 4); + + // Try with begin hint: + calls = 0; + auto it = m.try_emplace(m.begin(), -1, -1); + EXPECT_EQ(129, m.size()); + EXPECT_EQ(it, m.begin()); + EXPECT_LE(calls, 2); + + // Try with end hint: + calls = 0; + std::pair<int, int> pair1024 = {1024, 1024}; + it = m.try_emplace(m.end(), pair1024.first, pair1024.second); + EXPECT_EQ(130, m.size()); + EXPECT_EQ(it, --m.end()); + EXPECT_LE(calls, 2); + + // Try value already present, bad hint; ensure no duplicate added: + calls = 0; + it = m.try_emplace(m.end(), 16, 17); + EXPECT_EQ(130, m.size()); + EXPECT_GE(calls, 4); + EXPECT_EQ(it, m.find(16)); + + // Try value already present, hint points directly to it: + calls = 0; + it = m.try_emplace(it, 16, 17); + EXPECT_EQ(130, m.size()); + EXPECT_LE(calls, 2); + EXPECT_EQ(it, m.find(16)); + + m.erase(2); + EXPECT_EQ(129, m.size()); + auto hint = m.find(3); + // Try emplace in the middle of two other elements. + calls = 0; + m.try_emplace(hint, 2, 2); + EXPECT_EQ(130, m.size()); + EXPECT_LE(calls, 2); + + EXPECT_TRUE(std::is_sorted(m.begin(), m.end())); +} + +TEST(Btree, TryEmplaceWithBadHint) { + absl::btree_map<int, int> m = {{1, 1}, {9, 9}}; + + // Bad hint (too small), should still emplace: + auto it = m.try_emplace(m.begin(), 2, 2); + EXPECT_EQ(it, ++m.begin()); + EXPECT_THAT(m, ElementsAreArray( + std::vector<std::pair<int, int>>{{1, 1}, {2, 2}, {9, 9}})); + + // Bad hint, too large this time: + it = m.try_emplace(++(++m.begin()), 0, 0); + EXPECT_EQ(it, m.begin()); + EXPECT_THAT(m, ElementsAreArray(std::vector<std::pair<int, int>>{ + {0, 0}, {1, 1}, {2, 2}, {9, 9}})); +} + +TEST(Btree, TryEmplaceMaintainsSortedOrder) { + absl::btree_map<int, std::string> m; + std::pair<int, std::string> pair5 = {5, "five"}; + + // Test both lvalue & rvalue emplace. + m.try_emplace(10, "ten"); + m.try_emplace(pair5.first, pair5.second); + EXPECT_EQ(2, m.size()); + EXPECT_TRUE(std::is_sorted(m.begin(), m.end())); + + int int100{100}; + m.try_emplace(int100, "hundred"); + m.try_emplace(1, "one"); + EXPECT_EQ(4, m.size()); + EXPECT_TRUE(std::is_sorted(m.begin(), m.end())); +} + +TEST(Btree, TryEmplaceWithHintAndNoValueArgsWorks) { + absl::btree_map<int, int> m; + m.try_emplace(m.end(), 1); + EXPECT_EQ(0, m[1]); +} + +TEST(Btree, TryEmplaceWithHintAndMultipleValueArgsWorks) { + absl::btree_map<int, std::string> m; + m.try_emplace(m.end(), 1, 10, 'a'); + EXPECT_EQ(std::string(10, 'a'), m[1]); +} + +TEST(Btree, MoveAssignmentAllocatorPropagation) { + InstanceTracker tracker; + + int64_t bytes1 = 0, bytes2 = 0; + PropagatingCountingAlloc<MovableOnlyInstance> allocator1(&bytes1); + PropagatingCountingAlloc<MovableOnlyInstance> allocator2(&bytes2); + std::less<MovableOnlyInstance> cmp; + + // Test propagating allocator_type. + { + absl::btree_set<MovableOnlyInstance, std::less<MovableOnlyInstance>, + PropagatingCountingAlloc<MovableOnlyInstance>> + set1(cmp, allocator1), set2(cmp, allocator2); + + for (int i = 0; i < 100; ++i) set1.insert(MovableOnlyInstance(i)); + + tracker.ResetCopiesMovesSwaps(); + set2 = std::move(set1); + EXPECT_EQ(tracker.moves(), 0); + } + // Test non-propagating allocator_type with equal allocators. + { + absl::btree_set<MovableOnlyInstance, std::less<MovableOnlyInstance>, + CountingAllocator<MovableOnlyInstance>> + set1(cmp, allocator1), set2(cmp, allocator1); + + for (int i = 0; i < 100; ++i) set1.insert(MovableOnlyInstance(i)); + + tracker.ResetCopiesMovesSwaps(); + set2 = std::move(set1); + EXPECT_EQ(tracker.moves(), 0); + } + // Test non-propagating allocator_type with different allocators. + { + absl::btree_set<MovableOnlyInstance, std::less<MovableOnlyInstance>, + CountingAllocator<MovableOnlyInstance>> + set1(cmp, allocator1), set2(cmp, allocator2); + + for (int i = 0; i < 100; ++i) set1.insert(MovableOnlyInstance(i)); + + tracker.ResetCopiesMovesSwaps(); + set2 = std::move(set1); + EXPECT_GE(tracker.moves(), 100); + } +} + +TEST(Btree, EmptyTree) { + absl::btree_set<int> s; + EXPECT_TRUE(s.empty()); + EXPECT_EQ(s.size(), 0); + EXPECT_GT(s.max_size(), 0); +} + +bool IsEven(int k) { return k % 2 == 0; } + +TEST(Btree, EraseIf) { + // Test that erase_if works with all the container types and supports lambdas. + { + absl::btree_set<int> s = {1, 3, 5, 6, 100}; + erase_if(s, [](int k) { return k > 3; }); + EXPECT_THAT(s, ElementsAre(1, 3)); + } + { + absl::btree_multiset<int> s = {1, 3, 3, 5, 6, 6, 100}; + erase_if(s, [](int k) { return k <= 3; }); + EXPECT_THAT(s, ElementsAre(5, 6, 6, 100)); + } + { + absl::btree_map<int, int> m = {{1, 1}, {3, 3}, {6, 6}, {100, 100}}; + erase_if(m, [](std::pair<const int, int> kv) { return kv.first > 3; }); + EXPECT_THAT(m, ElementsAre(Pair(1, 1), Pair(3, 3))); + } + { + absl::btree_multimap<int, int> m = {{1, 1}, {3, 3}, {3, 6}, + {6, 6}, {6, 7}, {100, 6}}; + erase_if(m, [](std::pair<const int, int> kv) { return kv.second == 6; }); + EXPECT_THAT(m, ElementsAre(Pair(1, 1), Pair(3, 3), Pair(6, 7))); + } + // Test that erasing all elements from a large set works and test support for + // function pointers. + { + absl::btree_set<int> s; + for (int i = 0; i < 1000; ++i) s.insert(2 * i); + erase_if(s, IsEven); + EXPECT_THAT(s, IsEmpty()); + } + // Test that erase_if supports other format of function pointers. + { + absl::btree_set<int> s = {1, 3, 5, 6, 100}; + erase_if(s, &IsEven); + EXPECT_THAT(s, ElementsAre(1, 3, 5)); + } +} + +TEST(Btree, InsertOrAssign) { + absl::btree_map<int, int> m = {{1, 1}, {3, 3}}; + using value_type = typename decltype(m)::value_type; + + auto ret = m.insert_or_assign(4, 4); + EXPECT_EQ(*ret.first, value_type(4, 4)); + EXPECT_TRUE(ret.second); + ret = m.insert_or_assign(3, 100); + EXPECT_EQ(*ret.first, value_type(3, 100)); + EXPECT_FALSE(ret.second); + + auto hint_ret = m.insert_or_assign(ret.first, 3, 200); + EXPECT_EQ(*hint_ret, value_type(3, 200)); + hint_ret = m.insert_or_assign(m.find(1), 0, 1); + EXPECT_EQ(*hint_ret, value_type(0, 1)); + // Test with bad hint. + hint_ret = m.insert_or_assign(m.end(), -1, 1); + EXPECT_EQ(*hint_ret, value_type(-1, 1)); + + EXPECT_THAT(m, ElementsAre(Pair(-1, 1), Pair(0, 1), Pair(1, 1), Pair(3, 200), + Pair(4, 4))); +} + +TEST(Btree, InsertOrAssignMovableOnly) { + absl::btree_map<int, MovableOnlyInstance> m; + using value_type = typename decltype(m)::value_type; + + auto ret = m.insert_or_assign(4, MovableOnlyInstance(4)); + EXPECT_EQ(*ret.first, value_type(4, MovableOnlyInstance(4))); + EXPECT_TRUE(ret.second); + ret = m.insert_or_assign(4, MovableOnlyInstance(100)); + EXPECT_EQ(*ret.first, value_type(4, MovableOnlyInstance(100))); + EXPECT_FALSE(ret.second); + + auto hint_ret = m.insert_or_assign(ret.first, 3, MovableOnlyInstance(200)); + EXPECT_EQ(*hint_ret, value_type(3, MovableOnlyInstance(200))); + + EXPECT_EQ(m.size(), 2); +} + +TEST(Btree, BitfieldArgument) { + union { + int n : 1; + }; + n = 0; + absl::btree_map<int, int> m; + m.erase(n); + m.count(n); + m.find(n); + m.contains(n); + m.equal_range(n); + m.insert_or_assign(n, n); + m.insert_or_assign(m.end(), n, n); + m.try_emplace(n); + m.try_emplace(m.end(), n); + m.at(n); + m[n]; +} + +TEST(Btree, SetRangeConstructorAndInsertSupportExplicitConversionComparable) { + const absl::string_view names[] = {"n1", "n2"}; + + absl::btree_set<std::string> name_set1{std::begin(names), std::end(names)}; + EXPECT_THAT(name_set1, ElementsAreArray(names)); + + absl::btree_set<std::string> name_set2; + name_set2.insert(std::begin(names), std::end(names)); + EXPECT_THAT(name_set2, ElementsAreArray(names)); +} + +// A type that is explicitly convertible from int and counts constructor calls. +struct ConstructorCounted { + explicit ConstructorCounted(int i) : i(i) { ++constructor_calls; } + bool operator==(int other) const { return i == other; } + + int i; + static int constructor_calls; +}; +int ConstructorCounted::constructor_calls = 0; + +struct ConstructorCountedCompare { + bool operator()(int a, const ConstructorCounted &b) const { return a < b.i; } + bool operator()(const ConstructorCounted &a, int b) const { return a.i < b; } + bool operator()(const ConstructorCounted &a, + const ConstructorCounted &b) const { + return a.i < b.i; + } + using is_transparent = void; +}; + +TEST(Btree, + SetRangeConstructorAndInsertExplicitConvComparableLimitConstruction) { + const int i[] = {0, 1, 1}; + ConstructorCounted::constructor_calls = 0; + + absl::btree_set<ConstructorCounted, ConstructorCountedCompare> set{ + std::begin(i), std::end(i)}; + EXPECT_THAT(set, ElementsAre(0, 1)); + EXPECT_EQ(ConstructorCounted::constructor_calls, 2); + + set.insert(std::begin(i), std::end(i)); + EXPECT_THAT(set, ElementsAre(0, 1)); + EXPECT_EQ(ConstructorCounted::constructor_calls, 2); +} + +TEST(Btree, + SetRangeConstructorAndInsertSupportExplicitConversionNonComparable) { + const int i[] = {0, 1}; + + absl::btree_set<std::vector<void *>> s1{std::begin(i), std::end(i)}; + EXPECT_THAT(s1, ElementsAre(IsEmpty(), ElementsAre(IsNull()))); + + absl::btree_set<std::vector<void *>> s2; + s2.insert(std::begin(i), std::end(i)); + EXPECT_THAT(s2, ElementsAre(IsEmpty(), ElementsAre(IsNull()))); +} + +// libstdc++ included with GCC 4.9 has a bug in the std::pair constructors that +// prevents explicit conversions between pair types. +// We only run this test for the libstdc++ from GCC 7 or newer because we can't +// reliably check the libstdc++ version prior to that release. +#if !defined(__GLIBCXX__) || \ + (defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE >= 7) +TEST(Btree, MapRangeConstructorAndInsertSupportExplicitConversionComparable) { + const std::pair<absl::string_view, int> names[] = {{"n1", 1}, {"n2", 2}}; + + absl::btree_map<std::string, int> name_map1{std::begin(names), + std::end(names)}; + EXPECT_THAT(name_map1, ElementsAre(Pair("n1", 1), Pair("n2", 2))); + + absl::btree_map<std::string, int> name_map2; + name_map2.insert(std::begin(names), std::end(names)); + EXPECT_THAT(name_map2, ElementsAre(Pair("n1", 1), Pair("n2", 2))); +} + +TEST(Btree, + MapRangeConstructorAndInsertExplicitConvComparableLimitConstruction) { + const std::pair<int, int> i[] = {{0, 1}, {1, 2}, {1, 3}}; + ConstructorCounted::constructor_calls = 0; + + absl::btree_map<ConstructorCounted, int, ConstructorCountedCompare> map{ + std::begin(i), std::end(i)}; + EXPECT_THAT(map, ElementsAre(Pair(0, 1), Pair(1, 2))); + EXPECT_EQ(ConstructorCounted::constructor_calls, 2); + + map.insert(std::begin(i), std::end(i)); + EXPECT_THAT(map, ElementsAre(Pair(0, 1), Pair(1, 2))); + EXPECT_EQ(ConstructorCounted::constructor_calls, 2); +} + +TEST(Btree, + MapRangeConstructorAndInsertSupportExplicitConversionNonComparable) { + const std::pair<int, int> i[] = {{0, 1}, {1, 2}}; + + absl::btree_map<std::vector<void *>, int> m1{std::begin(i), std::end(i)}; + EXPECT_THAT(m1, + ElementsAre(Pair(IsEmpty(), 1), Pair(ElementsAre(IsNull()), 2))); + + absl::btree_map<std::vector<void *>, int> m2; + m2.insert(std::begin(i), std::end(i)); + EXPECT_THAT(m2, + ElementsAre(Pair(IsEmpty(), 1), Pair(ElementsAre(IsNull()), 2))); +} + +TEST(Btree, HeterogeneousTryEmplace) { + absl::btree_map<std::string, int> m; + std::string s = "key"; + absl::string_view sv = s; + m.try_emplace(sv, 1); + EXPECT_EQ(m[s], 1); + + m.try_emplace(m.end(), sv, 2); + EXPECT_EQ(m[s], 1); +} + +TEST(Btree, HeterogeneousOperatorMapped) { + absl::btree_map<std::string, int> m; + std::string s = "key"; + absl::string_view sv = s; + m[sv] = 1; + EXPECT_EQ(m[s], 1); + + m[sv] = 2; + EXPECT_EQ(m[s], 2); +} + +TEST(Btree, HeterogeneousInsertOrAssign) { + absl::btree_map<std::string, int> m; + std::string s = "key"; + absl::string_view sv = s; + m.insert_or_assign(sv, 1); + EXPECT_EQ(m[s], 1); + + m.insert_or_assign(m.end(), sv, 2); + EXPECT_EQ(m[s], 2); +} +#endif + +// This test requires std::launder for mutable key access in node handles. +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 +TEST(Btree, NodeHandleMutableKeyAccess) { + { + absl::btree_map<std::string, std::string> map; + + map["key1"] = "mapped"; + + auto nh = map.extract(map.begin()); + nh.key().resize(3); + map.insert(std::move(nh)); + + EXPECT_THAT(map, ElementsAre(Pair("key", "mapped"))); + } + // Also for multimap. + { + absl::btree_multimap<std::string, std::string> map; + + map.emplace("key1", "mapped"); + + auto nh = map.extract(map.begin()); + nh.key().resize(3); + map.insert(std::move(nh)); + + EXPECT_THAT(map, ElementsAre(Pair("key", "mapped"))); + } +} +#endif + +struct MultiKey { + int i1; + int i2; +}; + +bool operator==(const MultiKey a, const MultiKey b) { + return a.i1 == b.i1 && a.i2 == b.i2; +} + +// A heterogeneous comparator that has different equivalence classes for +// different lookup types. +struct MultiKeyComp { + using is_transparent = void; + bool operator()(const MultiKey a, const MultiKey b) const { + if (a.i1 != b.i1) return a.i1 < b.i1; + return a.i2 < b.i2; + } + bool operator()(const int a, const MultiKey b) const { return a < b.i1; } + bool operator()(const MultiKey a, const int b) const { return a.i1 < b; } +}; + +TEST(Btree, MultiKeyEqualRange) { + absl::btree_set<MultiKey, MultiKeyComp> set; + + for (int i = 0; i < 100; ++i) { + for (int j = 0; j < 100; ++j) { + set.insert({i, j}); + } + } + + for (int i = 0; i < 100; ++i) { + auto equal_range = set.equal_range(i); + EXPECT_EQ(equal_range.first->i1, i); + EXPECT_EQ(equal_range.first->i2, 0); + EXPECT_EQ(std::distance(equal_range.first, equal_range.second), 100) << i; + } +} + +TEST(Btree, MultiKeyErase) { + absl::btree_set<MultiKey, MultiKeyComp> set = { + {1, 1}, {2, 1}, {2, 2}, {3, 1}}; + EXPECT_EQ(set.erase(2), 2); + EXPECT_THAT(set, ElementsAre(MultiKey{1, 1}, MultiKey{3, 1})); +} + +TEST(Btree, MultiKeyCount) { + const absl::btree_set<MultiKey, MultiKeyComp> set = { + {1, 1}, {2, 1}, {2, 2}, {3, 1}}; + EXPECT_EQ(set.count(2), 2); +} + +TEST(Btree, AllocConstructor) { + using Alloc = CountingAllocator<int>; + using Set = absl::btree_set<int, std::less<int>, Alloc>; + int64_t bytes_used = 0; + Alloc alloc(&bytes_used); + Set set(alloc); + + set.insert({1, 2, 3}); + + EXPECT_THAT(set, ElementsAre(1, 2, 3)); + EXPECT_GT(bytes_used, set.size() * sizeof(int)); +} + +TEST(Btree, AllocInitializerListConstructor) { + using Alloc = CountingAllocator<int>; + using Set = absl::btree_set<int, std::less<int>, Alloc>; + int64_t bytes_used = 0; + Alloc alloc(&bytes_used); + Set set({1, 2, 3}, alloc); + + EXPECT_THAT(set, ElementsAre(1, 2, 3)); + EXPECT_GT(bytes_used, set.size() * sizeof(int)); +} + +TEST(Btree, AllocRangeConstructor) { + using Alloc = CountingAllocator<int>; + using Set = absl::btree_set<int, std::less<int>, Alloc>; + int64_t bytes_used = 0; + Alloc alloc(&bytes_used); + std::vector<int> v = {1, 2, 3}; + Set set(v.begin(), v.end(), alloc); + + EXPECT_THAT(set, ElementsAre(1, 2, 3)); + EXPECT_GT(bytes_used, set.size() * sizeof(int)); +} + +TEST(Btree, AllocCopyConstructor) { + using Alloc = CountingAllocator<int>; + using Set = absl::btree_set<int, std::less<int>, Alloc>; + int64_t bytes_used1 = 0; + Alloc alloc1(&bytes_used1); + Set set1(alloc1); + + set1.insert({1, 2, 3}); + + int64_t bytes_used2 = 0; + Alloc alloc2(&bytes_used2); + Set set2(set1, alloc2); + + EXPECT_THAT(set1, ElementsAre(1, 2, 3)); + EXPECT_THAT(set2, ElementsAre(1, 2, 3)); + EXPECT_GT(bytes_used1, set1.size() * sizeof(int)); + EXPECT_EQ(bytes_used1, bytes_used2); +} + +TEST(Btree, AllocMoveConstructor_SameAlloc) { + using Alloc = CountingAllocator<int>; + using Set = absl::btree_set<int, std::less<int>, Alloc>; + int64_t bytes_used = 0; + Alloc alloc(&bytes_used); + Set set1(alloc); + + set1.insert({1, 2, 3}); + + const int64_t original_bytes_used = bytes_used; + EXPECT_GT(original_bytes_used, set1.size() * sizeof(int)); + + Set set2(std::move(set1), alloc); + + EXPECT_THAT(set2, ElementsAre(1, 2, 3)); + EXPECT_EQ(bytes_used, original_bytes_used); +} + +TEST(Btree, AllocMoveConstructor_DifferentAlloc) { + using Alloc = CountingAllocator<int>; + using Set = absl::btree_set<int, std::less<int>, Alloc>; + int64_t bytes_used1 = 0; + Alloc alloc1(&bytes_used1); + Set set1(alloc1); + + set1.insert({1, 2, 3}); + + const int64_t original_bytes_used = bytes_used1; + EXPECT_GT(original_bytes_used, set1.size() * sizeof(int)); + + int64_t bytes_used2 = 0; + Alloc alloc2(&bytes_used2); + Set set2(std::move(set1), alloc2); + + EXPECT_THAT(set2, ElementsAre(1, 2, 3)); + // We didn't free these bytes allocated by `set1` yet. + EXPECT_EQ(bytes_used1, original_bytes_used); + EXPECT_EQ(bytes_used2, original_bytes_used); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/btree_test.h b/third_party/abseil_cpp/absl/container/btree_test.h new file mode 100644 index 000000000000..624908072d59 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/btree_test.h @@ -0,0 +1,166 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_BTREE_TEST_H_ +#define ABSL_CONTAINER_BTREE_TEST_H_ + +#include <algorithm> +#include <cassert> +#include <random> +#include <string> +#include <utility> +#include <vector> + +#include "absl/container/btree_map.h" +#include "absl/container/btree_set.h" +#include "absl/container/flat_hash_set.h" +#include "absl/strings/cord.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// Like remove_const but propagates the removal through std::pair. +template <typename T> +struct remove_pair_const { + using type = typename std::remove_const<T>::type; +}; +template <typename T, typename U> +struct remove_pair_const<std::pair<T, U> > { + using type = std::pair<typename remove_pair_const<T>::type, + typename remove_pair_const<U>::type>; +}; + +// Utility class to provide an accessor for a key given a value. The default +// behavior is to treat the value as a pair and return the first element. +template <typename K, typename V> +struct KeyOfValue { + struct type { + const K& operator()(const V& p) const { return p.first; } + }; +}; + +// Partial specialization of KeyOfValue class for when the key and value are +// the same type such as in set<> and btree_set<>. +template <typename K> +struct KeyOfValue<K, K> { + struct type { + const K& operator()(const K& k) const { return k; } + }; +}; + +inline char* GenerateDigits(char buf[16], unsigned val, unsigned maxval) { + assert(val <= maxval); + constexpr unsigned kBase = 64; // avoid integer division. + unsigned p = 15; + buf[p--] = 0; + while (maxval > 0) { + buf[p--] = ' ' + (val % kBase); + val /= kBase; + maxval /= kBase; + } + return buf + p + 1; +} + +template <typename K> +struct Generator { + int maxval; + explicit Generator(int m) : maxval(m) {} + K operator()(int i) const { + assert(i <= maxval); + return K(i); + } +}; + +template <> +struct Generator<absl::Time> { + int maxval; + explicit Generator(int m) : maxval(m) {} + absl::Time operator()(int i) const { return absl::FromUnixMillis(i); } +}; + +template <> +struct Generator<std::string> { + int maxval; + explicit Generator(int m) : maxval(m) {} + std::string operator()(int i) const { + char buf[16]; + return GenerateDigits(buf, i, maxval); + } +}; + +template <> +struct Generator<Cord> { + int maxval; + explicit Generator(int m) : maxval(m) {} + Cord operator()(int i) const { + char buf[16]; + return Cord(GenerateDigits(buf, i, maxval)); + } +}; + +template <typename T, typename U> +struct Generator<std::pair<T, U> > { + Generator<typename remove_pair_const<T>::type> tgen; + Generator<typename remove_pair_const<U>::type> ugen; + + explicit Generator(int m) : tgen(m), ugen(m) {} + std::pair<T, U> operator()(int i) const { + return std::make_pair(tgen(i), ugen(i)); + } +}; + +// Generate n values for our tests and benchmarks. Value range is [0, maxval]. +inline std::vector<int> GenerateNumbersWithSeed(int n, int maxval, int seed) { + // NOTE: Some tests rely on generated numbers not changing between test runs. + // We use std::minstd_rand0 because it is well-defined, but don't use + // std::uniform_int_distribution because platforms use different algorithms. + std::minstd_rand0 rng(seed); + + std::vector<int> values; + absl::flat_hash_set<int> unique_values; + if (values.size() < n) { + for (int i = values.size(); i < n; i++) { + int value; + do { + value = static_cast<int>(rng()) % (maxval + 1); + } while (!unique_values.insert(value).second); + + values.push_back(value); + } + } + return values; +} + +// Generates n values in the range [0, maxval]. +template <typename V> +std::vector<V> GenerateValuesWithSeed(int n, int maxval, int seed) { + const std::vector<int> nums = GenerateNumbersWithSeed(n, maxval, seed); + Generator<V> gen(maxval); + std::vector<V> vec; + + vec.reserve(n); + for (int i = 0; i < n; i++) { + vec.push_back(gen(nums[i])); + } + + return vec; +} + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_BTREE_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/fixed_array.h b/third_party/abseil_cpp/absl/container/fixed_array.h new file mode 100644 index 000000000000..fcb3e545b243 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/fixed_array.h @@ -0,0 +1,532 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: fixed_array.h +// ----------------------------------------------------------------------------- +// +// A `FixedArray<T>` represents a non-resizable array of `T` where the length of +// the array can be determined at run-time. It is a good replacement for +// non-standard and deprecated uses of `alloca()` and variable length arrays +// within the GCC extension. (See +// https://gcc.gnu.org/onlinedocs/gcc/Variable-Length.html). +// +// `FixedArray` allocates small arrays inline, keeping performance fast by +// avoiding heap operations. It also helps reduce the chances of +// accidentally overflowing your stack if large input is passed to +// your function. + +#ifndef ABSL_CONTAINER_FIXED_ARRAY_H_ +#define ABSL_CONTAINER_FIXED_ARRAY_H_ + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <initializer_list> +#include <iterator> +#include <limits> +#include <memory> +#include <new> +#include <type_traits> + +#include "absl/algorithm/algorithm.h" +#include "absl/base/config.h" +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/throw_delegate.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +constexpr static auto kFixedArrayUseDefault = static_cast<size_t>(-1); + +// ----------------------------------------------------------------------------- +// FixedArray +// ----------------------------------------------------------------------------- +// +// A `FixedArray` provides a run-time fixed-size array, allocating a small array +// inline for efficiency. +// +// Most users should not specify an `inline_elements` argument and let +// `FixedArray` automatically determine the number of elements +// to store inline based on `sizeof(T)`. If `inline_elements` is specified, the +// `FixedArray` implementation will use inline storage for arrays with a +// length <= `inline_elements`. +// +// Note that a `FixedArray` constructed with a `size_type` argument will +// default-initialize its values by leaving trivially constructible types +// uninitialized (e.g. int, int[4], double), and others default-constructed. +// This matches the behavior of c-style arrays and `std::array`, but not +// `std::vector`. +// +// Note that `FixedArray` does not provide a public allocator; if it requires a +// heap allocation, it will do so with global `::operator new[]()` and +// `::operator delete[]()`, even if T provides class-scope overrides for these +// operators. +template <typename T, size_t N = kFixedArrayUseDefault, + typename A = std::allocator<T>> +class FixedArray { + static_assert(!std::is_array<T>::value || std::extent<T>::value > 0, + "Arrays with unknown bounds cannot be used with FixedArray."); + + static constexpr size_t kInlineBytesDefault = 256; + + using AllocatorTraits = std::allocator_traits<A>; + // std::iterator_traits isn't guaranteed to be SFINAE-friendly until C++17, + // but this seems to be mostly pedantic. + template <typename Iterator> + using EnableIfForwardIterator = absl::enable_if_t<std::is_convertible< + typename std::iterator_traits<Iterator>::iterator_category, + std::forward_iterator_tag>::value>; + static constexpr bool NoexceptCopyable() { + return std::is_nothrow_copy_constructible<StorageElement>::value && + absl::allocator_is_nothrow<allocator_type>::value; + } + static constexpr bool NoexceptMovable() { + return std::is_nothrow_move_constructible<StorageElement>::value && + absl::allocator_is_nothrow<allocator_type>::value; + } + static constexpr bool DefaultConstructorIsNonTrivial() { + return !absl::is_trivially_default_constructible<StorageElement>::value; + } + + public: + using allocator_type = typename AllocatorTraits::allocator_type; + using value_type = typename AllocatorTraits::value_type; + using pointer = typename AllocatorTraits::pointer; + using const_pointer = typename AllocatorTraits::const_pointer; + using reference = value_type&; + using const_reference = const value_type&; + using size_type = typename AllocatorTraits::size_type; + using difference_type = typename AllocatorTraits::difference_type; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + + static constexpr size_type inline_elements = + (N == kFixedArrayUseDefault ? kInlineBytesDefault / sizeof(value_type) + : static_cast<size_type>(N)); + + FixedArray( + const FixedArray& other, + const allocator_type& a = allocator_type()) noexcept(NoexceptCopyable()) + : FixedArray(other.begin(), other.end(), a) {} + + FixedArray( + FixedArray&& other, + const allocator_type& a = allocator_type()) noexcept(NoexceptMovable()) + : FixedArray(std::make_move_iterator(other.begin()), + std::make_move_iterator(other.end()), a) {} + + // Creates an array object that can store `n` elements. + // Note that trivially constructible elements will be uninitialized. + explicit FixedArray(size_type n, const allocator_type& a = allocator_type()) + : storage_(n, a) { + if (DefaultConstructorIsNonTrivial()) { + memory_internal::ConstructRange(storage_.alloc(), storage_.begin(), + storage_.end()); + } + } + + // Creates an array initialized with `n` copies of `val`. + FixedArray(size_type n, const value_type& val, + const allocator_type& a = allocator_type()) + : storage_(n, a) { + memory_internal::ConstructRange(storage_.alloc(), storage_.begin(), + storage_.end(), val); + } + + // Creates an array initialized with the size and contents of `init_list`. + FixedArray(std::initializer_list<value_type> init_list, + const allocator_type& a = allocator_type()) + : FixedArray(init_list.begin(), init_list.end(), a) {} + + // Creates an array initialized with the elements from the input + // range. The array's size will always be `std::distance(first, last)`. + // REQUIRES: Iterator must be a forward_iterator or better. + template <typename Iterator, EnableIfForwardIterator<Iterator>* = nullptr> + FixedArray(Iterator first, Iterator last, + const allocator_type& a = allocator_type()) + : storage_(std::distance(first, last), a) { + memory_internal::CopyRange(storage_.alloc(), storage_.begin(), first, last); + } + + ~FixedArray() noexcept { + for (auto* cur = storage_.begin(); cur != storage_.end(); ++cur) { + AllocatorTraits::destroy(storage_.alloc(), cur); + } + } + + // Assignments are deleted because they break the invariant that the size of a + // `FixedArray` never changes. + void operator=(FixedArray&&) = delete; + void operator=(const FixedArray&) = delete; + + // FixedArray::size() + // + // Returns the length of the fixed array. + size_type size() const { return storage_.size(); } + + // FixedArray::max_size() + // + // Returns the largest possible value of `std::distance(begin(), end())` for a + // `FixedArray<T>`. This is equivalent to the most possible addressable bytes + // over the number of bytes taken by T. + constexpr size_type max_size() const { + return (std::numeric_limits<difference_type>::max)() / sizeof(value_type); + } + + // FixedArray::empty() + // + // Returns whether or not the fixed array is empty. + bool empty() const { return size() == 0; } + + // FixedArray::memsize() + // + // Returns the memory size of the fixed array in bytes. + size_t memsize() const { return size() * sizeof(value_type); } + + // FixedArray::data() + // + // Returns a const T* pointer to elements of the `FixedArray`. This pointer + // can be used to access (but not modify) the contained elements. + const_pointer data() const { return AsValueType(storage_.begin()); } + + // Overload of FixedArray::data() to return a T* pointer to elements of the + // fixed array. This pointer can be used to access and modify the contained + // elements. + pointer data() { return AsValueType(storage_.begin()); } + + // FixedArray::operator[] + // + // Returns a reference the ith element of the fixed array. + // REQUIRES: 0 <= i < size() + reference operator[](size_type i) { + ABSL_HARDENING_ASSERT(i < size()); + return data()[i]; + } + + // Overload of FixedArray::operator()[] to return a const reference to the + // ith element of the fixed array. + // REQUIRES: 0 <= i < size() + const_reference operator[](size_type i) const { + ABSL_HARDENING_ASSERT(i < size()); + return data()[i]; + } + + // FixedArray::at + // + // Bounds-checked access. Returns a reference to the ith element of the fixed + // array, or throws std::out_of_range + reference at(size_type i) { + if (ABSL_PREDICT_FALSE(i >= size())) { + base_internal::ThrowStdOutOfRange("FixedArray::at failed bounds check"); + } + return data()[i]; + } + + // Overload of FixedArray::at() to return a const reference to the ith element + // of the fixed array. + const_reference at(size_type i) const { + if (ABSL_PREDICT_FALSE(i >= size())) { + base_internal::ThrowStdOutOfRange("FixedArray::at failed bounds check"); + } + return data()[i]; + } + + // FixedArray::front() + // + // Returns a reference to the first element of the fixed array. + reference front() { + ABSL_HARDENING_ASSERT(!empty()); + return data()[0]; + } + + // Overload of FixedArray::front() to return a reference to the first element + // of a fixed array of const values. + const_reference front() const { + ABSL_HARDENING_ASSERT(!empty()); + return data()[0]; + } + + // FixedArray::back() + // + // Returns a reference to the last element of the fixed array. + reference back() { + ABSL_HARDENING_ASSERT(!empty()); + return data()[size() - 1]; + } + + // Overload of FixedArray::back() to return a reference to the last element + // of a fixed array of const values. + const_reference back() const { + ABSL_HARDENING_ASSERT(!empty()); + return data()[size() - 1]; + } + + // FixedArray::begin() + // + // Returns an iterator to the beginning of the fixed array. + iterator begin() { return data(); } + + // Overload of FixedArray::begin() to return a const iterator to the + // beginning of the fixed array. + const_iterator begin() const { return data(); } + + // FixedArray::cbegin() + // + // Returns a const iterator to the beginning of the fixed array. + const_iterator cbegin() const { return begin(); } + + // FixedArray::end() + // + // Returns an iterator to the end of the fixed array. + iterator end() { return data() + size(); } + + // Overload of FixedArray::end() to return a const iterator to the end of the + // fixed array. + const_iterator end() const { return data() + size(); } + + // FixedArray::cend() + // + // Returns a const iterator to the end of the fixed array. + const_iterator cend() const { return end(); } + + // FixedArray::rbegin() + // + // Returns a reverse iterator from the end of the fixed array. + reverse_iterator rbegin() { return reverse_iterator(end()); } + + // Overload of FixedArray::rbegin() to return a const reverse iterator from + // the end of the fixed array. + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + + // FixedArray::crbegin() + // + // Returns a const reverse iterator from the end of the fixed array. + const_reverse_iterator crbegin() const { return rbegin(); } + + // FixedArray::rend() + // + // Returns a reverse iterator from the beginning of the fixed array. + reverse_iterator rend() { return reverse_iterator(begin()); } + + // Overload of FixedArray::rend() for returning a const reverse iterator + // from the beginning of the fixed array. + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // FixedArray::crend() + // + // Returns a reverse iterator from the beginning of the fixed array. + const_reverse_iterator crend() const { return rend(); } + + // FixedArray::fill() + // + // Assigns the given `value` to all elements in the fixed array. + void fill(const value_type& val) { std::fill(begin(), end(), val); } + + // Relational operators. Equality operators are elementwise using + // `operator==`, while order operators order FixedArrays lexicographically. + friend bool operator==(const FixedArray& lhs, const FixedArray& rhs) { + return absl::equal(lhs.begin(), lhs.end(), rhs.begin(), rhs.end()); + } + + friend bool operator!=(const FixedArray& lhs, const FixedArray& rhs) { + return !(lhs == rhs); + } + + friend bool operator<(const FixedArray& lhs, const FixedArray& rhs) { + return std::lexicographical_compare(lhs.begin(), lhs.end(), rhs.begin(), + rhs.end()); + } + + friend bool operator>(const FixedArray& lhs, const FixedArray& rhs) { + return rhs < lhs; + } + + friend bool operator<=(const FixedArray& lhs, const FixedArray& rhs) { + return !(rhs < lhs); + } + + friend bool operator>=(const FixedArray& lhs, const FixedArray& rhs) { + return !(lhs < rhs); + } + + template <typename H> + friend H AbslHashValue(H h, const FixedArray& v) { + return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()), + v.size()); + } + + private: + // StorageElement + // + // For FixedArrays with a C-style-array value_type, StorageElement is a POD + // wrapper struct called StorageElementWrapper that holds the value_type + // instance inside. This is needed for construction and destruction of the + // entire array regardless of how many dimensions it has. For all other cases, + // StorageElement is just an alias of value_type. + // + // Maintainer's Note: The simpler solution would be to simply wrap value_type + // in a struct whether it's an array or not. That causes some paranoid + // diagnostics to misfire, believing that 'data()' returns a pointer to a + // single element, rather than the packed array that it really is. + // e.g.: + // + // FixedArray<char> buf(1); + // sprintf(buf.data(), "foo"); + // + // error: call to int __builtin___sprintf_chk(etc...) + // will always overflow destination buffer [-Werror] + // + template <typename OuterT, typename InnerT = absl::remove_extent_t<OuterT>, + size_t InnerN = std::extent<OuterT>::value> + struct StorageElementWrapper { + InnerT array[InnerN]; + }; + + using StorageElement = + absl::conditional_t<std::is_array<value_type>::value, + StorageElementWrapper<value_type>, value_type>; + + static pointer AsValueType(pointer ptr) { return ptr; } + static pointer AsValueType(StorageElementWrapper<value_type>* ptr) { + return std::addressof(ptr->array); + } + + static_assert(sizeof(StorageElement) == sizeof(value_type), ""); + static_assert(alignof(StorageElement) == alignof(value_type), ""); + + class NonEmptyInlinedStorage { + public: + StorageElement* data() { return reinterpret_cast<StorageElement*>(buff_); } + void AnnotateConstruct(size_type n); + void AnnotateDestruct(size_type n); + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + void* RedzoneBegin() { return &redzone_begin_; } + void* RedzoneEnd() { return &redzone_end_ + 1; } +#endif // ABSL_HAVE_ADDRESS_SANITIZER + + private: + ABSL_ADDRESS_SANITIZER_REDZONE(redzone_begin_); + alignas(StorageElement) char buff_[sizeof(StorageElement[inline_elements])]; + ABSL_ADDRESS_SANITIZER_REDZONE(redzone_end_); + }; + + class EmptyInlinedStorage { + public: + StorageElement* data() { return nullptr; } + void AnnotateConstruct(size_type) {} + void AnnotateDestruct(size_type) {} + }; + + using InlinedStorage = + absl::conditional_t<inline_elements == 0, EmptyInlinedStorage, + NonEmptyInlinedStorage>; + + // Storage + // + // An instance of Storage manages the inline and out-of-line memory for + // instances of FixedArray. This guarantees that even when construction of + // individual elements fails in the FixedArray constructor body, the + // destructor for Storage will still be called and out-of-line memory will be + // properly deallocated. + // + class Storage : public InlinedStorage { + public: + Storage(size_type n, const allocator_type& a) + : size_alloc_(n, a), data_(InitializeData()) {} + + ~Storage() noexcept { + if (UsingInlinedStorage(size())) { + InlinedStorage::AnnotateDestruct(size()); + } else { + AllocatorTraits::deallocate(alloc(), AsValueType(begin()), size()); + } + } + + size_type size() const { return size_alloc_.template get<0>(); } + StorageElement* begin() const { return data_; } + StorageElement* end() const { return begin() + size(); } + allocator_type& alloc() { return size_alloc_.template get<1>(); } + + private: + static bool UsingInlinedStorage(size_type n) { + return n <= inline_elements; + } + + StorageElement* InitializeData() { + if (UsingInlinedStorage(size())) { + InlinedStorage::AnnotateConstruct(size()); + return InlinedStorage::data(); + } else { + return reinterpret_cast<StorageElement*>( + AllocatorTraits::allocate(alloc(), size())); + } + } + + // `CompressedTuple` takes advantage of EBCO for stateless `allocator_type`s + container_internal::CompressedTuple<size_type, allocator_type> size_alloc_; + StorageElement* data_; + }; + + Storage storage_; +}; + +template <typename T, size_t N, typename A> +constexpr size_t FixedArray<T, N, A>::kInlineBytesDefault; + +template <typename T, size_t N, typename A> +constexpr typename FixedArray<T, N, A>::size_type + FixedArray<T, N, A>::inline_elements; + +template <typename T, size_t N, typename A> +void FixedArray<T, N, A>::NonEmptyInlinedStorage::AnnotateConstruct( + typename FixedArray<T, N, A>::size_type n) { +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + if (!n) return; + ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(data(), RedzoneEnd(), RedzoneEnd(), + data() + n); + ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(RedzoneBegin(), data(), data(), + RedzoneBegin()); +#endif // ABSL_HAVE_ADDRESS_SANITIZER + static_cast<void>(n); // Mark used when not in asan mode +} + +template <typename T, size_t N, typename A> +void FixedArray<T, N, A>::NonEmptyInlinedStorage::AnnotateDestruct( + typename FixedArray<T, N, A>::size_type n) { +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + if (!n) return; + ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(data(), RedzoneEnd(), data() + n, + RedzoneEnd()); + ABSL_ANNOTATE_CONTIGUOUS_CONTAINER(RedzoneBegin(), data(), RedzoneBegin(), + data()); +#endif // ABSL_HAVE_ADDRESS_SANITIZER + static_cast<void>(n); // Mark used when not in asan mode +} +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_FIXED_ARRAY_H_ diff --git a/third_party/abseil_cpp/absl/container/fixed_array_benchmark.cc b/third_party/abseil_cpp/absl/container/fixed_array_benchmark.cc new file mode 100644 index 000000000000..3c7a5a723450 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/fixed_array_benchmark.cc @@ -0,0 +1,67 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> + +#include <string> + +#include "benchmark/benchmark.h" +#include "absl/container/fixed_array.h" + +namespace { + +// For benchmarking -- simple class with constructor and destructor that +// set an int to a constant.. +class SimpleClass { + public: + SimpleClass() : i(3) {} + ~SimpleClass() { i = 0; } + + private: + int i; +}; + +template <typename C, size_t stack_size> +void BM_FixedArray(benchmark::State& state) { + const int size = state.range(0); + for (auto _ : state) { + absl::FixedArray<C, stack_size> fa(size); + benchmark::DoNotOptimize(fa.data()); + } +} +BENCHMARK_TEMPLATE(BM_FixedArray, char, absl::kFixedArrayUseDefault) + ->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, char, 0)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, char, 1)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, char, 16)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, char, 256)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, char, 65536)->Range(0, 1 << 16); + +BENCHMARK_TEMPLATE(BM_FixedArray, SimpleClass, absl::kFixedArrayUseDefault) + ->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, SimpleClass, 0)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, SimpleClass, 1)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, SimpleClass, 16)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, SimpleClass, 256)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, SimpleClass, 65536)->Range(0, 1 << 16); + +BENCHMARK_TEMPLATE(BM_FixedArray, std::string, absl::kFixedArrayUseDefault) + ->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, std::string, 0)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, std::string, 1)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, std::string, 16)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, std::string, 256)->Range(0, 1 << 16); +BENCHMARK_TEMPLATE(BM_FixedArray, std::string, 65536)->Range(0, 1 << 16); + +} // namespace diff --git a/third_party/abseil_cpp/absl/container/fixed_array_exception_safety_test.cc b/third_party/abseil_cpp/absl/container/fixed_array_exception_safety_test.cc new file mode 100644 index 000000000000..e5f59299b358 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/fixed_array_exception_safety_test.cc @@ -0,0 +1,201 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/config.h" +#include "absl/container/fixed_array.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + +#include <initializer_list> + +#include "gtest/gtest.h" +#include "absl/base/internal/exception_safety_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +constexpr size_t kInlined = 25; +constexpr size_t kSmallSize = kInlined / 2; +constexpr size_t kLargeSize = kInlined * 2; + +constexpr int kInitialValue = 5; +constexpr int kUpdatedValue = 10; + +using ::testing::TestThrowingCtor; + +using Thrower = testing::ThrowingValue<testing::TypeSpec::kEverythingThrows>; +using ThrowAlloc = + testing::ThrowingAllocator<Thrower, testing::AllocSpec::kEverythingThrows>; +using MoveThrower = testing::ThrowingValue<testing::TypeSpec::kNoThrowMove>; +using MoveThrowAlloc = + testing::ThrowingAllocator<MoveThrower, + testing::AllocSpec::kEverythingThrows>; + +using FixedArr = absl::FixedArray<Thrower, kInlined>; +using FixedArrWithAlloc = absl::FixedArray<Thrower, kInlined, ThrowAlloc>; + +using MoveFixedArr = absl::FixedArray<MoveThrower, kInlined>; +using MoveFixedArrWithAlloc = + absl::FixedArray<MoveThrower, kInlined, MoveThrowAlloc>; + +TEST(FixedArrayExceptionSafety, CopyConstructor) { + auto small = FixedArr(kSmallSize); + TestThrowingCtor<FixedArr>(small); + + auto large = FixedArr(kLargeSize); + TestThrowingCtor<FixedArr>(large); +} + +TEST(FixedArrayExceptionSafety, CopyConstructorWithAlloc) { + auto small = FixedArrWithAlloc(kSmallSize); + TestThrowingCtor<FixedArrWithAlloc>(small); + + auto large = FixedArrWithAlloc(kLargeSize); + TestThrowingCtor<FixedArrWithAlloc>(large); +} + +TEST(FixedArrayExceptionSafety, MoveConstructor) { + TestThrowingCtor<FixedArr>(FixedArr(kSmallSize)); + TestThrowingCtor<FixedArr>(FixedArr(kLargeSize)); + + // TypeSpec::kNoThrowMove + TestThrowingCtor<MoveFixedArr>(MoveFixedArr(kSmallSize)); + TestThrowingCtor<MoveFixedArr>(MoveFixedArr(kLargeSize)); +} + +TEST(FixedArrayExceptionSafety, MoveConstructorWithAlloc) { + TestThrowingCtor<FixedArrWithAlloc>(FixedArrWithAlloc(kSmallSize)); + TestThrowingCtor<FixedArrWithAlloc>(FixedArrWithAlloc(kLargeSize)); + + // TypeSpec::kNoThrowMove + TestThrowingCtor<MoveFixedArrWithAlloc>(MoveFixedArrWithAlloc(kSmallSize)); + TestThrowingCtor<MoveFixedArrWithAlloc>(MoveFixedArrWithAlloc(kLargeSize)); +} + +TEST(FixedArrayExceptionSafety, SizeConstructor) { + TestThrowingCtor<FixedArr>(kSmallSize); + TestThrowingCtor<FixedArr>(kLargeSize); +} + +TEST(FixedArrayExceptionSafety, SizeConstructorWithAlloc) { + TestThrowingCtor<FixedArrWithAlloc>(kSmallSize); + TestThrowingCtor<FixedArrWithAlloc>(kLargeSize); +} + +TEST(FixedArrayExceptionSafety, SizeValueConstructor) { + TestThrowingCtor<FixedArr>(kSmallSize, Thrower()); + TestThrowingCtor<FixedArr>(kLargeSize, Thrower()); +} + +TEST(FixedArrayExceptionSafety, SizeValueConstructorWithAlloc) { + TestThrowingCtor<FixedArrWithAlloc>(kSmallSize, Thrower()); + TestThrowingCtor<FixedArrWithAlloc>(kLargeSize, Thrower()); +} + +TEST(FixedArrayExceptionSafety, IteratorConstructor) { + auto small = FixedArr(kSmallSize); + TestThrowingCtor<FixedArr>(small.begin(), small.end()); + + auto large = FixedArr(kLargeSize); + TestThrowingCtor<FixedArr>(large.begin(), large.end()); +} + +TEST(FixedArrayExceptionSafety, IteratorConstructorWithAlloc) { + auto small = FixedArrWithAlloc(kSmallSize); + TestThrowingCtor<FixedArrWithAlloc>(small.begin(), small.end()); + + auto large = FixedArrWithAlloc(kLargeSize); + TestThrowingCtor<FixedArrWithAlloc>(large.begin(), large.end()); +} + +TEST(FixedArrayExceptionSafety, InitListConstructor) { + constexpr int small_inlined = 3; + using SmallFixedArr = absl::FixedArray<Thrower, small_inlined>; + + TestThrowingCtor<SmallFixedArr>(std::initializer_list<Thrower>{}); + // Test inlined allocation + TestThrowingCtor<SmallFixedArr>( + std::initializer_list<Thrower>{Thrower{}, Thrower{}}); + // Test out of line allocation + TestThrowingCtor<SmallFixedArr>(std::initializer_list<Thrower>{ + Thrower{}, Thrower{}, Thrower{}, Thrower{}, Thrower{}}); +} + +TEST(FixedArrayExceptionSafety, InitListConstructorWithAlloc) { + constexpr int small_inlined = 3; + using SmallFixedArrWithAlloc = + absl::FixedArray<Thrower, small_inlined, ThrowAlloc>; + + TestThrowingCtor<SmallFixedArrWithAlloc>(std::initializer_list<Thrower>{}); + // Test inlined allocation + TestThrowingCtor<SmallFixedArrWithAlloc>( + std::initializer_list<Thrower>{Thrower{}, Thrower{}}); + // Test out of line allocation + TestThrowingCtor<SmallFixedArrWithAlloc>(std::initializer_list<Thrower>{ + Thrower{}, Thrower{}, Thrower{}, Thrower{}, Thrower{}}); +} + +template <typename FixedArrT> +testing::AssertionResult ReadMemory(FixedArrT* fixed_arr) { + int sum = 0; + for (const auto& thrower : *fixed_arr) { + sum += thrower.Get(); + } + return testing::AssertionSuccess() << "Values sum to [" << sum << "]"; +} + +TEST(FixedArrayExceptionSafety, Fill) { + auto test_fill = testing::MakeExceptionSafetyTester() + .WithContracts(ReadMemory<FixedArr>) + .WithOperation([&](FixedArr* fixed_arr_ptr) { + auto thrower = + Thrower(kUpdatedValue, testing::nothrow_ctor); + fixed_arr_ptr->fill(thrower); + }); + + EXPECT_TRUE( + test_fill.WithInitialValue(FixedArr(kSmallSize, Thrower(kInitialValue))) + .Test()); + EXPECT_TRUE( + test_fill.WithInitialValue(FixedArr(kLargeSize, Thrower(kInitialValue))) + .Test()); +} + +TEST(FixedArrayExceptionSafety, FillWithAlloc) { + auto test_fill = testing::MakeExceptionSafetyTester() + .WithContracts(ReadMemory<FixedArrWithAlloc>) + .WithOperation([&](FixedArrWithAlloc* fixed_arr_ptr) { + auto thrower = + Thrower(kUpdatedValue, testing::nothrow_ctor); + fixed_arr_ptr->fill(thrower); + }); + + EXPECT_TRUE(test_fill + .WithInitialValue( + FixedArrWithAlloc(kSmallSize, Thrower(kInitialValue))) + .Test()); + EXPECT_TRUE(test_fill + .WithInitialValue( + FixedArrWithAlloc(kLargeSize, Thrower(kInitialValue))) + .Test()); +} + +} // namespace + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HAVE_EXCEPTIONS diff --git a/third_party/abseil_cpp/absl/container/fixed_array_test.cc b/third_party/abseil_cpp/absl/container/fixed_array_test.cc new file mode 100644 index 000000000000..49598e7a053c --- /dev/null +++ b/third_party/abseil_cpp/absl/container/fixed_array_test.cc @@ -0,0 +1,837 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/fixed_array.h" + +#include <stdio.h> + +#include <cstring> +#include <list> +#include <memory> +#include <numeric> +#include <scoped_allocator> +#include <stdexcept> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/exception_testing.h" +#include "absl/base/options.h" +#include "absl/container/internal/counting_allocator.h" +#include "absl/hash/hash_testing.h" +#include "absl/memory/memory.h" + +using ::testing::ElementsAreArray; + +namespace { + +// Helper routine to determine if a absl::FixedArray used stack allocation. +template <typename ArrayType> +static bool IsOnStack(const ArrayType& a) { + return a.size() <= ArrayType::inline_elements; +} + +class ConstructionTester { + public: + ConstructionTester() : self_ptr_(this), value_(0) { constructions++; } + ~ConstructionTester() { + assert(self_ptr_ == this); + self_ptr_ = nullptr; + destructions++; + } + + // These are incremented as elements are constructed and destructed so we can + // be sure all elements are properly cleaned up. + static int constructions; + static int destructions; + + void CheckConstructed() { assert(self_ptr_ == this); } + + void set(int value) { value_ = value; } + int get() { return value_; } + + private: + // self_ptr_ should always point to 'this' -- that's how we can be sure the + // constructor has been called. + ConstructionTester* self_ptr_; + int value_; +}; + +int ConstructionTester::constructions = 0; +int ConstructionTester::destructions = 0; + +// ThreeInts will initialize its three ints to the value stored in +// ThreeInts::counter. The constructor increments counter so that each object +// in an array of ThreeInts will have different values. +class ThreeInts { + public: + ThreeInts() { + x_ = counter; + y_ = counter; + z_ = counter; + ++counter; + } + + static int counter; + + int x_, y_, z_; +}; + +int ThreeInts::counter = 0; + +TEST(FixedArrayTest, CopyCtor) { + absl::FixedArray<int, 10> on_stack(5); + std::iota(on_stack.begin(), on_stack.end(), 0); + absl::FixedArray<int, 10> stack_copy = on_stack; + EXPECT_THAT(stack_copy, ElementsAreArray(on_stack)); + EXPECT_TRUE(IsOnStack(stack_copy)); + + absl::FixedArray<int, 10> allocated(15); + std::iota(allocated.begin(), allocated.end(), 0); + absl::FixedArray<int, 10> alloced_copy = allocated; + EXPECT_THAT(alloced_copy, ElementsAreArray(allocated)); + EXPECT_FALSE(IsOnStack(alloced_copy)); +} + +TEST(FixedArrayTest, MoveCtor) { + absl::FixedArray<std::unique_ptr<int>, 10> on_stack(5); + for (int i = 0; i < 5; ++i) { + on_stack[i] = absl::make_unique<int>(i); + } + + absl::FixedArray<std::unique_ptr<int>, 10> stack_copy = std::move(on_stack); + for (int i = 0; i < 5; ++i) EXPECT_EQ(*(stack_copy[i]), i); + EXPECT_EQ(stack_copy.size(), on_stack.size()); + + absl::FixedArray<std::unique_ptr<int>, 10> allocated(15); + for (int i = 0; i < 15; ++i) { + allocated[i] = absl::make_unique<int>(i); + } + + absl::FixedArray<std::unique_ptr<int>, 10> alloced_copy = + std::move(allocated); + for (int i = 0; i < 15; ++i) EXPECT_EQ(*(alloced_copy[i]), i); + EXPECT_EQ(allocated.size(), alloced_copy.size()); +} + +TEST(FixedArrayTest, SmallObjects) { + // Small object arrays + { + // Short arrays should be on the stack + absl::FixedArray<int> array(4); + EXPECT_TRUE(IsOnStack(array)); + } + + { + // Large arrays should be on the heap + absl::FixedArray<int> array(1048576); + EXPECT_FALSE(IsOnStack(array)); + } + + { + // Arrays of <= default size should be on the stack + absl::FixedArray<int, 100> array(100); + EXPECT_TRUE(IsOnStack(array)); + } + + { + // Arrays of > default size should be on the heap + absl::FixedArray<int, 100> array(101); + EXPECT_FALSE(IsOnStack(array)); + } + + { + // Arrays with different size elements should use approximately + // same amount of stack space + absl::FixedArray<int> array1(0); + absl::FixedArray<char> array2(0); + EXPECT_LE(sizeof(array1), sizeof(array2) + 100); + EXPECT_LE(sizeof(array2), sizeof(array1) + 100); + } + + { + // Ensure that vectors are properly constructed inside a fixed array. + absl::FixedArray<std::vector<int>> array(2); + EXPECT_EQ(0, array[0].size()); + EXPECT_EQ(0, array[1].size()); + } + + { + // Regardless of absl::FixedArray implementation, check that a type with a + // low alignment requirement and a non power-of-two size is initialized + // correctly. + ThreeInts::counter = 1; + absl::FixedArray<ThreeInts> array(2); + EXPECT_EQ(1, array[0].x_); + EXPECT_EQ(1, array[0].y_); + EXPECT_EQ(1, array[0].z_); + EXPECT_EQ(2, array[1].x_); + EXPECT_EQ(2, array[1].y_); + EXPECT_EQ(2, array[1].z_); + } +} + +TEST(FixedArrayTest, AtThrows) { + absl::FixedArray<int> a = {1, 2, 3}; + EXPECT_EQ(a.at(2), 3); + ABSL_BASE_INTERNAL_EXPECT_FAIL(a.at(3), std::out_of_range, + "failed bounds check"); +} + +TEST(FixedArrayTest, Hardened) { +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + absl::FixedArray<int> a = {1, 2, 3}; + EXPECT_EQ(a[2], 3); + EXPECT_DEATH_IF_SUPPORTED(a[3], ""); + EXPECT_DEATH_IF_SUPPORTED(a[-1], ""); + + absl::FixedArray<int> empty(0); + EXPECT_DEATH_IF_SUPPORTED(empty[0], ""); + EXPECT_DEATH_IF_SUPPORTED(empty[-1], ""); + EXPECT_DEATH_IF_SUPPORTED(empty.front(), ""); + EXPECT_DEATH_IF_SUPPORTED(empty.back(), ""); +#endif +} + +TEST(FixedArrayRelationalsTest, EqualArrays) { + for (int i = 0; i < 10; ++i) { + absl::FixedArray<int, 5> a1(i); + std::iota(a1.begin(), a1.end(), 0); + absl::FixedArray<int, 5> a2(a1.begin(), a1.end()); + + EXPECT_TRUE(a1 == a2); + EXPECT_FALSE(a1 != a2); + EXPECT_TRUE(a2 == a1); + EXPECT_FALSE(a2 != a1); + EXPECT_FALSE(a1 < a2); + EXPECT_FALSE(a1 > a2); + EXPECT_FALSE(a2 < a1); + EXPECT_FALSE(a2 > a1); + EXPECT_TRUE(a1 <= a2); + EXPECT_TRUE(a1 >= a2); + EXPECT_TRUE(a2 <= a1); + EXPECT_TRUE(a2 >= a1); + } +} + +TEST(FixedArrayRelationalsTest, UnequalArrays) { + for (int i = 1; i < 10; ++i) { + absl::FixedArray<int, 5> a1(i); + std::iota(a1.begin(), a1.end(), 0); + absl::FixedArray<int, 5> a2(a1.begin(), a1.end()); + --a2[i / 2]; + + EXPECT_FALSE(a1 == a2); + EXPECT_TRUE(a1 != a2); + EXPECT_FALSE(a2 == a1); + EXPECT_TRUE(a2 != a1); + EXPECT_FALSE(a1 < a2); + EXPECT_TRUE(a1 > a2); + EXPECT_TRUE(a2 < a1); + EXPECT_FALSE(a2 > a1); + EXPECT_FALSE(a1 <= a2); + EXPECT_TRUE(a1 >= a2); + EXPECT_TRUE(a2 <= a1); + EXPECT_FALSE(a2 >= a1); + } +} + +template <int stack_elements> +static void TestArray(int n) { + SCOPED_TRACE(n); + SCOPED_TRACE(stack_elements); + ConstructionTester::constructions = 0; + ConstructionTester::destructions = 0; + { + absl::FixedArray<ConstructionTester, stack_elements> array(n); + + EXPECT_THAT(array.size(), n); + EXPECT_THAT(array.memsize(), sizeof(ConstructionTester) * n); + EXPECT_THAT(array.begin() + n, array.end()); + + // Check that all elements were constructed + for (int i = 0; i < n; i++) { + array[i].CheckConstructed(); + } + // Check that no other elements were constructed + EXPECT_THAT(ConstructionTester::constructions, n); + + // Test operator[] + for (int i = 0; i < n; i++) { + array[i].set(i); + } + for (int i = 0; i < n; i++) { + EXPECT_THAT(array[i].get(), i); + EXPECT_THAT(array.data()[i].get(), i); + } + + // Test data() + for (int i = 0; i < n; i++) { + array.data()[i].set(i + 1); + } + for (int i = 0; i < n; i++) { + EXPECT_THAT(array[i].get(), i + 1); + EXPECT_THAT(array.data()[i].get(), i + 1); + } + } // Close scope containing 'array'. + + // Check that all constructed elements were destructed. + EXPECT_EQ(ConstructionTester::constructions, + ConstructionTester::destructions); +} + +template <int elements_per_inner_array, int inline_elements> +static void TestArrayOfArrays(int n) { + SCOPED_TRACE(n); + SCOPED_TRACE(inline_elements); + SCOPED_TRACE(elements_per_inner_array); + ConstructionTester::constructions = 0; + ConstructionTester::destructions = 0; + { + using InnerArray = ConstructionTester[elements_per_inner_array]; + // Heap-allocate the FixedArray to avoid blowing the stack frame. + auto array_ptr = + absl::make_unique<absl::FixedArray<InnerArray, inline_elements>>(n); + auto& array = *array_ptr; + + ASSERT_EQ(array.size(), n); + ASSERT_EQ(array.memsize(), + sizeof(ConstructionTester) * elements_per_inner_array * n); + ASSERT_EQ(array.begin() + n, array.end()); + + // Check that all elements were constructed + for (int i = 0; i < n; i++) { + for (int j = 0; j < elements_per_inner_array; j++) { + (array[i])[j].CheckConstructed(); + } + } + // Check that no other elements were constructed + ASSERT_EQ(ConstructionTester::constructions, n * elements_per_inner_array); + + // Test operator[] + for (int i = 0; i < n; i++) { + for (int j = 0; j < elements_per_inner_array; j++) { + (array[i])[j].set(i * elements_per_inner_array + j); + } + } + for (int i = 0; i < n; i++) { + for (int j = 0; j < elements_per_inner_array; j++) { + ASSERT_EQ((array[i])[j].get(), i * elements_per_inner_array + j); + ASSERT_EQ((array.data()[i])[j].get(), i * elements_per_inner_array + j); + } + } + + // Test data() + for (int i = 0; i < n; i++) { + for (int j = 0; j < elements_per_inner_array; j++) { + (array.data()[i])[j].set((i + 1) * elements_per_inner_array + j); + } + } + for (int i = 0; i < n; i++) { + for (int j = 0; j < elements_per_inner_array; j++) { + ASSERT_EQ((array[i])[j].get(), (i + 1) * elements_per_inner_array + j); + ASSERT_EQ((array.data()[i])[j].get(), + (i + 1) * elements_per_inner_array + j); + } + } + } // Close scope containing 'array'. + + // Check that all constructed elements were destructed. + EXPECT_EQ(ConstructionTester::constructions, + ConstructionTester::destructions); +} + +TEST(IteratorConstructorTest, NonInline) { + int const kInput[] = {2, 3, 5, 7, 11, 13, 17}; + absl::FixedArray<int, ABSL_ARRAYSIZE(kInput) - 1> const fixed( + kInput, kInput + ABSL_ARRAYSIZE(kInput)); + ASSERT_EQ(ABSL_ARRAYSIZE(kInput), fixed.size()); + for (size_t i = 0; i < ABSL_ARRAYSIZE(kInput); ++i) { + ASSERT_EQ(kInput[i], fixed[i]); + } +} + +TEST(IteratorConstructorTest, Inline) { + int const kInput[] = {2, 3, 5, 7, 11, 13, 17}; + absl::FixedArray<int, ABSL_ARRAYSIZE(kInput)> const fixed( + kInput, kInput + ABSL_ARRAYSIZE(kInput)); + ASSERT_EQ(ABSL_ARRAYSIZE(kInput), fixed.size()); + for (size_t i = 0; i < ABSL_ARRAYSIZE(kInput); ++i) { + ASSERT_EQ(kInput[i], fixed[i]); + } +} + +TEST(IteratorConstructorTest, NonPod) { + char const* kInput[] = {"red", "orange", "yellow", "green", + "blue", "indigo", "violet"}; + absl::FixedArray<std::string> const fixed(kInput, + kInput + ABSL_ARRAYSIZE(kInput)); + ASSERT_EQ(ABSL_ARRAYSIZE(kInput), fixed.size()); + for (size_t i = 0; i < ABSL_ARRAYSIZE(kInput); ++i) { + ASSERT_EQ(kInput[i], fixed[i]); + } +} + +TEST(IteratorConstructorTest, FromEmptyVector) { + std::vector<int> const empty; + absl::FixedArray<int> const fixed(empty.begin(), empty.end()); + EXPECT_EQ(0, fixed.size()); + EXPECT_EQ(empty.size(), fixed.size()); +} + +TEST(IteratorConstructorTest, FromNonEmptyVector) { + int const kInput[] = {2, 3, 5, 7, 11, 13, 17}; + std::vector<int> const items(kInput, kInput + ABSL_ARRAYSIZE(kInput)); + absl::FixedArray<int> const fixed(items.begin(), items.end()); + ASSERT_EQ(items.size(), fixed.size()); + for (size_t i = 0; i < items.size(); ++i) { + ASSERT_EQ(items[i], fixed[i]); + } +} + +TEST(IteratorConstructorTest, FromBidirectionalIteratorRange) { + int const kInput[] = {2, 3, 5, 7, 11, 13, 17}; + std::list<int> const items(kInput, kInput + ABSL_ARRAYSIZE(kInput)); + absl::FixedArray<int> const fixed(items.begin(), items.end()); + EXPECT_THAT(fixed, testing::ElementsAreArray(kInput)); +} + +TEST(InitListConstructorTest, InitListConstruction) { + absl::FixedArray<int> fixed = {1, 2, 3}; + EXPECT_THAT(fixed, testing::ElementsAreArray({1, 2, 3})); +} + +TEST(FillConstructorTest, NonEmptyArrays) { + absl::FixedArray<int> stack_array(4, 1); + EXPECT_THAT(stack_array, testing::ElementsAreArray({1, 1, 1, 1})); + + absl::FixedArray<int, 0> heap_array(4, 1); + EXPECT_THAT(stack_array, testing::ElementsAreArray({1, 1, 1, 1})); +} + +TEST(FillConstructorTest, EmptyArray) { + absl::FixedArray<int> empty_fill(0, 1); + absl::FixedArray<int> empty_size(0); + EXPECT_EQ(empty_fill, empty_size); +} + +TEST(FillConstructorTest, NotTriviallyCopyable) { + std::string str = "abcd"; + absl::FixedArray<std::string> strings = {str, str, str, str}; + + absl::FixedArray<std::string> array(4, str); + EXPECT_EQ(array, strings); +} + +TEST(FillConstructorTest, Disambiguation) { + absl::FixedArray<size_t> a(1, 2); + EXPECT_THAT(a, testing::ElementsAre(2)); +} + +TEST(FixedArrayTest, ManySizedArrays) { + std::vector<int> sizes; + for (int i = 1; i < 100; i++) sizes.push_back(i); + for (int i = 100; i <= 1000; i += 100) sizes.push_back(i); + for (int n : sizes) { + TestArray<0>(n); + TestArray<1>(n); + TestArray<64>(n); + TestArray<1000>(n); + } +} + +TEST(FixedArrayTest, ManySizedArraysOfArraysOf1) { + for (int n = 1; n < 1000; n++) { + ASSERT_NO_FATAL_FAILURE((TestArrayOfArrays<1, 0>(n))); + ASSERT_NO_FATAL_FAILURE((TestArrayOfArrays<1, 1>(n))); + ASSERT_NO_FATAL_FAILURE((TestArrayOfArrays<1, 64>(n))); + ASSERT_NO_FATAL_FAILURE((TestArrayOfArrays<1, 1000>(n))); + } +} + +TEST(FixedArrayTest, ManySizedArraysOfArraysOf2) { + for (int n = 1; n < 1000; n++) { + TestArrayOfArrays<2, 0>(n); + TestArrayOfArrays<2, 1>(n); + TestArrayOfArrays<2, 64>(n); + TestArrayOfArrays<2, 1000>(n); + } +} + +// If value_type is put inside of a struct container, +// we might evoke this error in a hardened build unless data() is carefully +// written, so check on that. +// error: call to int __builtin___sprintf_chk(etc...) +// will always overflow destination buffer [-Werror] +TEST(FixedArrayTest, AvoidParanoidDiagnostics) { + absl::FixedArray<char, 32> buf(32); + sprintf(buf.data(), "foo"); // NOLINT(runtime/printf) +} + +TEST(FixedArrayTest, TooBigInlinedSpace) { + struct TooBig { + char c[1 << 20]; + }; // too big for even one on the stack + + // Simulate the data members of absl::FixedArray, a pointer and a size_t. + struct Data { + TooBig* p; + size_t size; + }; + + // Make sure TooBig objects are not inlined for 0 or default size. + static_assert(sizeof(absl::FixedArray<TooBig, 0>) == sizeof(Data), + "0-sized absl::FixedArray should have same size as Data."); + static_assert(alignof(absl::FixedArray<TooBig, 0>) == alignof(Data), + "0-sized absl::FixedArray should have same alignment as Data."); + static_assert(sizeof(absl::FixedArray<TooBig>) == sizeof(Data), + "default-sized absl::FixedArray should have same size as Data"); + static_assert( + alignof(absl::FixedArray<TooBig>) == alignof(Data), + "default-sized absl::FixedArray should have same alignment as Data."); +} + +// PickyDelete EXPECTs its class-scope deallocation funcs are unused. +struct PickyDelete { + PickyDelete() {} + ~PickyDelete() {} + void operator delete(void* p) { + EXPECT_TRUE(false) << __FUNCTION__; + ::operator delete(p); + } + void operator delete[](void* p) { + EXPECT_TRUE(false) << __FUNCTION__; + ::operator delete[](p); + } +}; + +TEST(FixedArrayTest, UsesGlobalAlloc) { absl::FixedArray<PickyDelete, 0> a(5); } + +TEST(FixedArrayTest, Data) { + static const int kInput[] = {2, 3, 5, 7, 11, 13, 17}; + absl::FixedArray<int> fa(std::begin(kInput), std::end(kInput)); + EXPECT_EQ(fa.data(), &*fa.begin()); + EXPECT_EQ(fa.data(), &fa[0]); + + const absl::FixedArray<int>& cfa = fa; + EXPECT_EQ(cfa.data(), &*cfa.begin()); + EXPECT_EQ(cfa.data(), &cfa[0]); +} + +TEST(FixedArrayTest, Empty) { + absl::FixedArray<int> empty(0); + absl::FixedArray<int> inline_filled(1); + absl::FixedArray<int, 0> heap_filled(1); + EXPECT_TRUE(empty.empty()); + EXPECT_FALSE(inline_filled.empty()); + EXPECT_FALSE(heap_filled.empty()); +} + +TEST(FixedArrayTest, FrontAndBack) { + absl::FixedArray<int, 3 * sizeof(int)> inlined = {1, 2, 3}; + EXPECT_EQ(inlined.front(), 1); + EXPECT_EQ(inlined.back(), 3); + + absl::FixedArray<int, 0> allocated = {1, 2, 3}; + EXPECT_EQ(allocated.front(), 1); + EXPECT_EQ(allocated.back(), 3); + + absl::FixedArray<int> one_element = {1}; + EXPECT_EQ(one_element.front(), one_element.back()); +} + +TEST(FixedArrayTest, ReverseIteratorInlined) { + absl::FixedArray<int, 5 * sizeof(int)> a = {0, 1, 2, 3, 4}; + + int counter = 5; + for (absl::FixedArray<int>::reverse_iterator iter = a.rbegin(); + iter != a.rend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + + counter = 5; + for (absl::FixedArray<int>::const_reverse_iterator iter = a.rbegin(); + iter != a.rend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + + counter = 5; + for (auto iter = a.crbegin(); iter != a.crend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); +} + +TEST(FixedArrayTest, ReverseIteratorAllocated) { + absl::FixedArray<int, 0> a = {0, 1, 2, 3, 4}; + + int counter = 5; + for (absl::FixedArray<int>::reverse_iterator iter = a.rbegin(); + iter != a.rend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + + counter = 5; + for (absl::FixedArray<int>::const_reverse_iterator iter = a.rbegin(); + iter != a.rend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + + counter = 5; + for (auto iter = a.crbegin(); iter != a.crend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); +} + +TEST(FixedArrayTest, Fill) { + absl::FixedArray<int, 5 * sizeof(int)> inlined(5); + int fill_val = 42; + inlined.fill(fill_val); + for (int i : inlined) EXPECT_EQ(i, fill_val); + + absl::FixedArray<int, 0> allocated(5); + allocated.fill(fill_val); + for (int i : allocated) EXPECT_EQ(i, fill_val); + + // It doesn't do anything, just make sure this compiles. + absl::FixedArray<int> empty(0); + empty.fill(fill_val); +} + +#ifndef __GNUC__ +TEST(FixedArrayTest, DefaultCtorDoesNotValueInit) { + using T = char; + constexpr auto capacity = 10; + using FixedArrType = absl::FixedArray<T, capacity>; + constexpr auto scrubbed_bits = 0x95; + constexpr auto length = capacity / 2; + + alignas(FixedArrType) unsigned char buff[sizeof(FixedArrType)]; + std::memset(std::addressof(buff), scrubbed_bits, sizeof(FixedArrType)); + + FixedArrType* arr = + ::new (static_cast<void*>(std::addressof(buff))) FixedArrType(length); + EXPECT_THAT(*arr, testing::Each(scrubbed_bits)); + arr->~FixedArrType(); +} +#endif // __GNUC__ + +TEST(AllocatorSupportTest, CountInlineAllocations) { + constexpr size_t inlined_size = 4; + using Alloc = absl::container_internal::CountingAllocator<int>; + using AllocFxdArr = absl::FixedArray<int, inlined_size, Alloc>; + + int64_t allocated = 0; + int64_t active_instances = 0; + + { + const int ia[] = {0, 1, 2, 3, 4, 5, 6, 7}; + + Alloc alloc(&allocated, &active_instances); + + AllocFxdArr arr(ia, ia + inlined_size, alloc); + static_cast<void>(arr); + } + + EXPECT_EQ(allocated, 0); + EXPECT_EQ(active_instances, 0); +} + +TEST(AllocatorSupportTest, CountOutoflineAllocations) { + constexpr size_t inlined_size = 4; + using Alloc = absl::container_internal::CountingAllocator<int>; + using AllocFxdArr = absl::FixedArray<int, inlined_size, Alloc>; + + int64_t allocated = 0; + int64_t active_instances = 0; + + { + const int ia[] = {0, 1, 2, 3, 4, 5, 6, 7}; + Alloc alloc(&allocated, &active_instances); + + AllocFxdArr arr(ia, ia + ABSL_ARRAYSIZE(ia), alloc); + + EXPECT_EQ(allocated, arr.size() * sizeof(int)); + static_cast<void>(arr); + } + + EXPECT_EQ(active_instances, 0); +} + +TEST(AllocatorSupportTest, CountCopyInlineAllocations) { + constexpr size_t inlined_size = 4; + using Alloc = absl::container_internal::CountingAllocator<int>; + using AllocFxdArr = absl::FixedArray<int, inlined_size, Alloc>; + + int64_t allocated1 = 0; + int64_t allocated2 = 0; + int64_t active_instances = 0; + Alloc alloc(&allocated1, &active_instances); + Alloc alloc2(&allocated2, &active_instances); + + { + int initial_value = 1; + + AllocFxdArr arr1(inlined_size / 2, initial_value, alloc); + + EXPECT_EQ(allocated1, 0); + + AllocFxdArr arr2(arr1, alloc2); + + EXPECT_EQ(allocated2, 0); + static_cast<void>(arr1); + static_cast<void>(arr2); + } + + EXPECT_EQ(active_instances, 0); +} + +TEST(AllocatorSupportTest, CountCopyOutoflineAllocations) { + constexpr size_t inlined_size = 4; + using Alloc = absl::container_internal::CountingAllocator<int>; + using AllocFxdArr = absl::FixedArray<int, inlined_size, Alloc>; + + int64_t allocated1 = 0; + int64_t allocated2 = 0; + int64_t active_instances = 0; + Alloc alloc(&allocated1, &active_instances); + Alloc alloc2(&allocated2, &active_instances); + + { + int initial_value = 1; + + AllocFxdArr arr1(inlined_size * 2, initial_value, alloc); + + EXPECT_EQ(allocated1, arr1.size() * sizeof(int)); + + AllocFxdArr arr2(arr1, alloc2); + + EXPECT_EQ(allocated2, inlined_size * 2 * sizeof(int)); + static_cast<void>(arr1); + static_cast<void>(arr2); + } + + EXPECT_EQ(active_instances, 0); +} + +TEST(AllocatorSupportTest, SizeValAllocConstructor) { + using testing::AllOf; + using testing::Each; + using testing::SizeIs; + + constexpr size_t inlined_size = 4; + using Alloc = absl::container_internal::CountingAllocator<int>; + using AllocFxdArr = absl::FixedArray<int, inlined_size, Alloc>; + + { + auto len = inlined_size / 2; + auto val = 0; + int64_t allocated = 0; + AllocFxdArr arr(len, val, Alloc(&allocated)); + + EXPECT_EQ(allocated, 0); + EXPECT_THAT(arr, AllOf(SizeIs(len), Each(0))); + } + + { + auto len = inlined_size * 2; + auto val = 0; + int64_t allocated = 0; + AllocFxdArr arr(len, val, Alloc(&allocated)); + + EXPECT_EQ(allocated, len * sizeof(int)); + EXPECT_THAT(arr, AllOf(SizeIs(len), Each(0))); + } +} + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +TEST(FixedArrayTest, AddressSanitizerAnnotations1) { + absl::FixedArray<int, 32> a(10); + int* raw = a.data(); + raw[0] = 0; + raw[9] = 0; + EXPECT_DEATH_IF_SUPPORTED(raw[-2] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[-1] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[10] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[31] = 0, "container-overflow"); +} + +TEST(FixedArrayTest, AddressSanitizerAnnotations2) { + absl::FixedArray<char, 17> a(12); + char* raw = a.data(); + raw[0] = 0; + raw[11] = 0; + EXPECT_DEATH_IF_SUPPORTED(raw[-7] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[-1] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[12] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[17] = 0, "container-overflow"); +} + +TEST(FixedArrayTest, AddressSanitizerAnnotations3) { + absl::FixedArray<uint64_t, 20> a(20); + uint64_t* raw = a.data(); + raw[0] = 0; + raw[19] = 0; + EXPECT_DEATH_IF_SUPPORTED(raw[-1] = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[20] = 0, "container-overflow"); +} + +TEST(FixedArrayTest, AddressSanitizerAnnotations4) { + absl::FixedArray<ThreeInts> a(10); + ThreeInts* raw = a.data(); + raw[0] = ThreeInts(); + raw[9] = ThreeInts(); + // Note: raw[-1] is pointing to 12 bytes before the container range. However, + // there is only a 8-byte red zone before the container range, so we only + // access the last 4 bytes of the struct to make sure it stays within the red + // zone. + EXPECT_DEATH_IF_SUPPORTED(raw[-1].z_ = 0, "container-overflow"); + EXPECT_DEATH_IF_SUPPORTED(raw[10] = ThreeInts(), "container-overflow"); + // The actual size of storage is kDefaultBytes=256, 21*12 = 252, + // so reading raw[21] should still trigger the correct warning. + EXPECT_DEATH_IF_SUPPORTED(raw[21] = ThreeInts(), "container-overflow"); +} +#endif // ABSL_HAVE_ADDRESS_SANITIZER + +TEST(FixedArrayTest, AbslHashValueWorks) { + using V = absl::FixedArray<int>; + std::vector<V> cases; + + // Generate a variety of vectors some of these are small enough for the inline + // space but are stored out of line. + for (int i = 0; i < 10; ++i) { + V v(i); + for (int j = 0; j < i; ++j) { + v[j] = j; + } + cases.push_back(v); + } + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(cases)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/container/flat_hash_map.h b/third_party/abseil_cpp/absl/container/flat_hash_map.h new file mode 100644 index 000000000000..74def0df0e33 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/flat_hash_map.h @@ -0,0 +1,606 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: flat_hash_map.h +// ----------------------------------------------------------------------------- +// +// An `absl::flat_hash_map<K, V>` is an unordered associative container of +// unique keys and associated values designed to be a more efficient replacement +// for `std::unordered_map`. Like `unordered_map`, search, insertion, and +// deletion of map elements can be done as an `O(1)` operation. However, +// `flat_hash_map` (and other unordered associative containers known as the +// collection of Abseil "Swiss tables") contain other optimizations that result +// in both memory and computation advantages. +// +// In most cases, your default choice for a hash map should be a map of type +// `flat_hash_map`. + +#ifndef ABSL_CONTAINER_FLAT_HASH_MAP_H_ +#define ABSL_CONTAINER_FLAT_HASH_MAP_H_ + +#include <cstddef> +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export +#include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +template <class K, class V> +struct FlatHashMapPolicy; +} // namespace container_internal + +// ----------------------------------------------------------------------------- +// absl::flat_hash_map +// ----------------------------------------------------------------------------- +// +// An `absl::flat_hash_map<K, V>` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_map<K, V>` with +// the following notable differences: +// +// * Requires keys that are CopyConstructible +// * Requires values that are MoveConstructible +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Invalidates any references and pointers to elements within the table after +// `rehash()`. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash map. +// * Returns `void` from the `erase(iterator)` overload. +// +// By default, `flat_hash_map` uses the `absl::Hash` hashing framework. +// All fundamental and Abseil types that support the `absl::Hash` framework have +// a compatible equality operator for comparing insertions into `flat_hash_map`. +// If your type is not yet supported by the `absl::Hash` framework, see +// absl/hash/hash.h for information on extending Abseil hashing to user-defined +// types. +// +// NOTE: A `flat_hash_map` stores its value types directly inside its +// implementation array to avoid memory indirection. Because a `flat_hash_map` +// is designed to move data when rehashed, map values will not retain pointer +// stability. If you require pointer stability, or if your values are large, +// consider using `absl::flat_hash_map<Key, std::unique_ptr<Value>>` instead. +// If your types are not moveable or you require pointer stability for keys, +// consider `absl::node_hash_map`. +// +// Example: +// +// // Create a flat hash map of three strings (that map to strings) +// absl::flat_hash_map<std::string, std::string> ducks = +// {{"a", "huey"}, {"b", "dewey"}, {"c", "louie"}}; +// +// // Insert a new element into the flat hash map +// ducks.insert({"d", "donald"}); +// +// // Force a rehash of the flat hash map +// ducks.rehash(0); +// +// // Find the element with the key "b" +// std::string search_key = "b"; +// auto result = ducks.find(search_key); +// if (result != ducks.end()) { +// std::cout << "Result: " << result->second << std::endl; +// } +template <class K, class V, + class Hash = absl::container_internal::hash_default_hash<K>, + class Eq = absl::container_internal::hash_default_eq<K>, + class Allocator = std::allocator<std::pair<const K, V>>> +class flat_hash_map : public absl::container_internal::raw_hash_map< + absl::container_internal::FlatHashMapPolicy<K, V>, + Hash, Eq, Allocator> { + using Base = typename flat_hash_map::raw_hash_map; + + public: + // Constructors and Assignment Operators + // + // A flat_hash_map supports the same overload set as `std::unordered_map` + // for construction and assignment: + // + // * Default constructor + // + // // No allocation for the table's elements is made. + // absl::flat_hash_map<int, std::string> map1; + // + // * Initializer List constructor + // + // absl::flat_hash_map<int, std::string> map2 = + // {{1, "huey"}, {2, "dewey"}, {3, "louie"},}; + // + // * Copy constructor + // + // absl::flat_hash_map<int, std::string> map3(map2); + // + // * Copy assignment operator + // + // // Hash functor and Comparator are copied as well + // absl::flat_hash_map<int, std::string> map4; + // map4 = map3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::flat_hash_map<int, std::string> map5(std::move(map4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::flat_hash_map<int, std::string> map6; + // map6 = std::move(map5); + // + // * Range constructor + // + // std::vector<std::pair<int, std::string>> v = {{1, "a"}, {2, "b"}}; + // absl::flat_hash_map<int, std::string> map7(v.begin(), v.end()); + flat_hash_map() {} + using Base::Base; + + // flat_hash_map::begin() + // + // Returns an iterator to the beginning of the `flat_hash_map`. + using Base::begin; + + // flat_hash_map::cbegin() + // + // Returns a const iterator to the beginning of the `flat_hash_map`. + using Base::cbegin; + + // flat_hash_map::cend() + // + // Returns a const iterator to the end of the `flat_hash_map`. + using Base::cend; + + // flat_hash_map::end() + // + // Returns an iterator to the end of the `flat_hash_map`. + using Base::end; + + // flat_hash_map::capacity() + // + // Returns the number of element slots (assigned, deleted, and empty) + // available within the `flat_hash_map`. + // + // NOTE: this member function is particular to `absl::flat_hash_map` and is + // not provided in the `std::unordered_map` API. + using Base::capacity; + + // flat_hash_map::empty() + // + // Returns whether or not the `flat_hash_map` is empty. + using Base::empty; + + // flat_hash_map::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `flat_hash_map` under current memory constraints. This value can be thought + // of the largest value of `std::distance(begin(), end())` for a + // `flat_hash_map<K, V>`. + using Base::max_size; + + // flat_hash_map::size() + // + // Returns the number of elements currently within the `flat_hash_map`. + using Base::size; + + // flat_hash_map::clear() + // + // Removes all elements from the `flat_hash_map`. Invalidates any references, + // pointers, or iterators referring to contained elements. + // + // NOTE: this operation may shrink the underlying buffer. To avoid shrinking + // the underlying buffer call `erase(begin(), end())`. + using Base::clear; + + // flat_hash_map::erase() + // + // Erases elements within the `flat_hash_map`. Erasing does not trigger a + // rehash. Overloads are listed below. + // + // void erase(const_iterator pos): + // + // Erases the element at `position` of the `flat_hash_map`, returning + // `void`. + // + // NOTE: returning `void` in this case is different than that of STL + // containers in general and `std::unordered_map` in particular (which + // return an iterator to the element following the erased element). If that + // iterator is needed, simply post increment the iterator: + // + // map.erase(it++); + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning an + // iterator pointing to `last`. + // + // size_type erase(const key_type& key): + // + // Erases the element with the matching key, if it exists, returning the + // number of elements erased (0 or 1). + using Base::erase; + + // flat_hash_map::insert() + // + // Inserts an element of the specified value into the `flat_hash_map`, + // returning an iterator pointing to the newly inserted element, provided that + // an element with the given key does not already exist. If rehashing occurs + // due to the insertion, all iterators are invalidated. Overloads are listed + // below. + // + // std::pair<iterator,bool> insert(const init_type& value): + // + // Inserts a value into the `flat_hash_map`. Returns a pair consisting of an + // iterator to the inserted element (or to the element that prevented the + // insertion) and a bool denoting whether the insertion took place. + // + // std::pair<iterator,bool> insert(T&& value): + // std::pair<iterator,bool> insert(init_type&& value): + // + // Inserts a moveable value into the `flat_hash_map`. Returns a pair + // consisting of an iterator to the inserted element (or to the element that + // prevented the insertion) and a bool denoting whether the insertion took + // place. + // + // iterator insert(const_iterator hint, const init_type& value): + // iterator insert(const_iterator hint, T&& value): + // iterator insert(const_iterator hint, init_type&& value); + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element, or to the existing element that prevented the + // insertion. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently, for `flat_hash_map` we guarantee the + // first match is inserted. + // + // void insert(std::initializer_list<init_type> ilist): + // + // Inserts the elements within the initializer list `ilist`. + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently within the initializer list, for + // `flat_hash_map` we guarantee the first match is inserted. + using Base::insert; + + // flat_hash_map::insert_or_assign() + // + // Inserts an element of the specified value into the `flat_hash_map` provided + // that a value with the given key does not already exist, or replaces it with + // the element value if a key for that value already exists, returning an + // iterator pointing to the newly inserted element. If rehashing occurs due + // to the insertion, all existing iterators are invalidated. Overloads are + // listed below. + // + // pair<iterator, bool> insert_or_assign(const init_type& k, T&& obj): + // pair<iterator, bool> insert_or_assign(init_type&& k, T&& obj): + // + // Inserts/Assigns (or moves) the element of the specified key into the + // `flat_hash_map`. + // + // iterator insert_or_assign(const_iterator hint, + // const init_type& k, T&& obj): + // iterator insert_or_assign(const_iterator hint, init_type&& k, T&& obj): + // + // Inserts/Assigns (or moves) the element of the specified key into the + // `flat_hash_map` using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. + using Base::insert_or_assign; + + // flat_hash_map::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `flat_hash_map`, provided that no element with the given key + // already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. Prefer `try_emplace()` unless your key is not + // copyable or moveable. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace; + + // flat_hash_map::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `flat_hash_map`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search, and only inserts + // provided that no element with the given key already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. Prefer `try_emplace()` unless your key is not + // copyable or moveable. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace_hint; + + // flat_hash_map::try_emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `flat_hash_map`, provided that no element with the given key + // already exists. Unlike `emplace()`, if an element with the given key + // already exists, we guarantee that no element is constructed. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + // Overloads are listed below. + // + // pair<iterator, bool> try_emplace(const key_type& k, Args&&... args): + // pair<iterator, bool> try_emplace(key_type&& k, Args&&... args): + // + // Inserts (via copy or move) the element of the specified key into the + // `flat_hash_map`. + // + // iterator try_emplace(const_iterator hint, + // const init_type& k, Args&&... args): + // iterator try_emplace(const_iterator hint, init_type&& k, Args&&... args): + // + // Inserts (via copy or move) the element of the specified key into the + // `flat_hash_map` using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. + // + // All `try_emplace()` overloads make the same guarantees regarding rvalue + // arguments as `std::unordered_map::try_emplace()`, namely that these + // functions will not move from rvalue arguments if insertions do not happen. + using Base::try_emplace; + + // flat_hash_map::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the key,value pair of the element at the indicated position and + // returns a node handle owning that extracted data. + // + // node_type extract(const key_type& x): + // + // Extracts the key,value pair of the element with a key matching the passed + // key value and returns a node handle owning that extracted data. If the + // `flat_hash_map` does not contain an element with a matching key, this + // function returns an empty node handle. + // + // NOTE: when compiled in an earlier version of C++ than C++17, + // `node_type::key()` returns a const reference to the key instead of a + // mutable reference. We cannot safely return a mutable reference without + // std::launder (which is not available before C++17). + using Base::extract; + + // flat_hash_map::merge() + // + // Extracts elements from a given `source` flat hash map into this + // `flat_hash_map`. If the destination `flat_hash_map` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // flat_hash_map::swap(flat_hash_map& other) + // + // Exchanges the contents of this `flat_hash_map` with those of the `other` + // flat hash map, avoiding invocation of any move, copy, or swap operations on + // individual elements. + // + // All iterators and references on the `flat_hash_map` remain valid, excepting + // for the past-the-end iterator, which is invalidated. + // + // `swap()` requires that the flat hash map's hashing and key equivalence + // functions be Swappable, and are exchanged using unqualified calls to + // non-member `swap()`. If the map's allocator has + // `std::allocator_traits<allocator_type>::propagate_on_container_swap::value` + // set to `true`, the allocators are also exchanged using an unqualified call + // to non-member `swap()`; otherwise, the allocators are not swapped. + using Base::swap; + + // flat_hash_map::rehash(count) + // + // Rehashes the `flat_hash_map`, setting the number of slots to be at least + // the passed value. If the new number of slots increases the load factor more + // than the current maximum load factor + // (`count` < `size()` / `max_load_factor()`), then the new number of slots + // will be at least `size()` / `max_load_factor()`. + // + // To force a rehash, pass rehash(0). + // + // NOTE: unlike behavior in `std::unordered_map`, references are also + // invalidated upon a `rehash()`. + using Base::rehash; + + // flat_hash_map::reserve(count) + // + // Sets the number of slots in the `flat_hash_map` to the number needed to + // accommodate at least `count` total elements without exceeding the current + // maximum load factor, and may rehash the container if needed. + using Base::reserve; + + // flat_hash_map::at() + // + // Returns a reference to the mapped value of the element with key equivalent + // to the passed key. + using Base::at; + + // flat_hash_map::contains() + // + // Determines whether an element with a key comparing equal to the given `key` + // exists within the `flat_hash_map`, returning `true` if so or `false` + // otherwise. + using Base::contains; + + // flat_hash_map::count(const Key& key) const + // + // Returns the number of elements with a key comparing equal to the given + // `key` within the `flat_hash_map`. note that this function will return + // either `1` or `0` since duplicate keys are not allowed within a + // `flat_hash_map`. + using Base::count; + + // flat_hash_map::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `flat_hash_map`. + using Base::equal_range; + + // flat_hash_map::find() + // + // Finds an element with the passed `key` within the `flat_hash_map`. + using Base::find; + + // flat_hash_map::operator[]() + // + // Returns a reference to the value mapped to the passed key within the + // `flat_hash_map`, performing an `insert()` if the key does not already + // exist. + // + // If an insertion occurs and results in a rehashing of the container, all + // iterators are invalidated. Otherwise iterators are not affected and + // references are not invalidated. Overloads are listed below. + // + // T& operator[](const Key& key): + // + // Inserts an init_type object constructed in-place if the element with the + // given key does not exist. + // + // T& operator[](Key&& key): + // + // Inserts an init_type object constructed in-place provided that an element + // with the given key does not exist. + using Base::operator[]; + + // flat_hash_map::bucket_count() + // + // Returns the number of "buckets" within the `flat_hash_map`. Note that + // because a flat hash map contains all elements within its internal storage, + // this value simply equals the current capacity of the `flat_hash_map`. + using Base::bucket_count; + + // flat_hash_map::load_factor() + // + // Returns the current load factor of the `flat_hash_map` (the average number + // of slots occupied with a value within the hash map). + using Base::load_factor; + + // flat_hash_map::max_load_factor() + // + // Manages the maximum load factor of the `flat_hash_map`. Overloads are + // listed below. + // + // float flat_hash_map::max_load_factor() + // + // Returns the current maximum load factor of the `flat_hash_map`. + // + // void flat_hash_map::max_load_factor(float ml) + // + // Sets the maximum load factor of the `flat_hash_map` to the passed value. + // + // NOTE: This overload is provided only for API compatibility with the STL; + // `flat_hash_map` will ignore any set load factor and manage its rehashing + // internally as an implementation detail. + using Base::max_load_factor; + + // flat_hash_map::get_allocator() + // + // Returns the allocator function associated with this `flat_hash_map`. + using Base::get_allocator; + + // flat_hash_map::hash_function() + // + // Returns the hashing function used to hash the keys within this + // `flat_hash_map`. + using Base::hash_function; + + // flat_hash_map::key_eq() + // + // Returns the function used for comparing keys equality. + using Base::key_eq; +}; + +// erase_if(flat_hash_map<>, Pred) +// +// Erases all elements that satisfy the predicate `pred` from the container `c`. +template <typename K, typename V, typename H, typename E, typename A, + typename Predicate> +void erase_if(flat_hash_map<K, V, H, E, A>& c, Predicate pred) { + container_internal::EraseIf(pred, &c); +} + +namespace container_internal { + +template <class K, class V> +struct FlatHashMapPolicy { + using slot_policy = container_internal::map_slot_policy<K, V>; + using slot_type = typename slot_policy::slot_type; + using key_type = K; + using mapped_type = V; + using init_type = std::pair</*non const*/ key_type, mapped_type>; + + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + slot_policy::construct(alloc, slot, std::forward<Args>(args)...); + } + + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + slot_policy::destroy(alloc, slot); + } + + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + slot_policy::transfer(alloc, new_slot, old_slot); + } + + template <class F, class... Args> + static decltype(absl::container_internal::DecomposePair( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return absl::container_internal::DecomposePair(std::forward<F>(f), + std::forward<Args>(args)...); + } + + static size_t space_used(const slot_type*) { return 0; } + + static std::pair<const K, V>& element(slot_type* slot) { return slot->value; } + + static V& value(std::pair<const K, V>* kv) { return kv->second; } + static const V& value(const std::pair<const K, V>* kv) { return kv->second; } +}; + +} // namespace container_internal + +namespace container_algorithm_internal { + +// Specialization of trait in absl/algorithm/container.h +template <class Key, class T, class Hash, class KeyEqual, class Allocator> +struct IsUnorderedContainer< + absl::flat_hash_map<Key, T, Hash, KeyEqual, Allocator>> : std::true_type {}; + +} // namespace container_algorithm_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_FLAT_HASH_MAP_H_ diff --git a/third_party/abseil_cpp/absl/container/flat_hash_map_test.cc b/third_party/abseil_cpp/absl/container/flat_hash_map_test.cc new file mode 100644 index 000000000000..89ec60c916ed --- /dev/null +++ b/third_party/abseil_cpp/absl/container/flat_hash_map_test.cc @@ -0,0 +1,288 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/flat_hash_map.h" + +#include <memory> + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/unordered_map_constructor_test.h" +#include "absl/container/internal/unordered_map_lookup_test.h" +#include "absl/container/internal/unordered_map_members_test.h" +#include "absl/container/internal/unordered_map_modifiers_test.h" +#include "absl/types/any.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { +using ::absl::container_internal::hash_internal::Enum; +using ::absl::container_internal::hash_internal::EnumClass; +using ::testing::_; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +// Check that absl::flat_hash_map works in a global constructor. +struct BeforeMain { + BeforeMain() { + absl::flat_hash_map<int, int> x; + x.insert({1, 1}); + ABSL_RAW_CHECK(x.find(0) == x.end(), "x should not contain 0"); + auto it = x.find(1); + ABSL_RAW_CHECK(it != x.end(), "x should contain 1"); + ABSL_RAW_CHECK(it->second, "1 should map to 1"); + } +}; +const BeforeMain before_main; + +template <class K, class V> +using Map = flat_hash_map<K, V, StatefulTestingHash, StatefulTestingEqual, + Alloc<std::pair<const K, V>>>; + +static_assert(!std::is_standard_layout<NonStandardLayout>(), ""); + +using MapTypes = + ::testing::Types<Map<int, int>, Map<std::string, int>, + Map<Enum, std::string>, Map<EnumClass, int>, + Map<int, NonStandardLayout>, Map<NonStandardLayout, int>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashMap, ConstructorTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashMap, LookupTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashMap, MembersTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashMap, ModifiersTest, MapTypes); + +using UniquePtrMapTypes = ::testing::Types<Map<int, std::unique_ptr<int>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashMap, UniquePtrModifiersTest, + UniquePtrMapTypes); + +TEST(FlatHashMap, StandardLayout) { + struct Int { + explicit Int(size_t value) : value(value) {} + Int() : value(0) { ADD_FAILURE(); } + Int(const Int& other) : value(other.value) { ADD_FAILURE(); } + Int(Int&&) = default; + bool operator==(const Int& other) const { return value == other.value; } + size_t value; + }; + static_assert(std::is_standard_layout<Int>(), ""); + + struct Hash { + size_t operator()(const Int& obj) const { return obj.value; } + }; + + // Verify that neither the key nor the value get default-constructed or + // copy-constructed. + { + flat_hash_map<Int, Int, Hash> m; + m.try_emplace(Int(1), Int(2)); + m.try_emplace(Int(3), Int(4)); + m.erase(Int(1)); + m.rehash(2 * m.bucket_count()); + } + { + flat_hash_map<Int, Int, Hash> m; + m.try_emplace(Int(1), Int(2)); + m.try_emplace(Int(3), Int(4)); + m.erase(Int(1)); + m.clear(); + } +} + +// gcc becomes unhappy if this is inside the method, so pull it out here. +struct balast {}; + +TEST(FlatHashMap, IteratesMsan) { + // Because SwissTable randomizes on pointer addresses, we keep old tables + // around to ensure we don't reuse old memory. + std::vector<absl::flat_hash_map<int, balast>> garbage; + for (int i = 0; i < 100; ++i) { + absl::flat_hash_map<int, balast> t; + for (int j = 0; j < 100; ++j) { + t[j]; + for (const auto& p : t) EXPECT_THAT(p, Pair(_, _)); + } + garbage.push_back(std::move(t)); + } +} + +// Demonstration of the "Lazy Key" pattern. This uses heterogeneous insert to +// avoid creating expensive key elements when the item is already present in the +// map. +struct LazyInt { + explicit LazyInt(size_t value, int* tracker) + : value(value), tracker(tracker) {} + + explicit operator size_t() const { + ++*tracker; + return value; + } + + size_t value; + int* tracker; +}; + +struct Hash { + using is_transparent = void; + int* tracker; + size_t operator()(size_t obj) const { + ++*tracker; + return obj; + } + size_t operator()(const LazyInt& obj) const { + ++*tracker; + return obj.value; + } +}; + +struct Eq { + using is_transparent = void; + bool operator()(size_t lhs, size_t rhs) const { + return lhs == rhs; + } + bool operator()(size_t lhs, const LazyInt& rhs) const { + return lhs == rhs.value; + } +}; + +TEST(FlatHashMap, LazyKeyPattern) { + // hashes are only guaranteed in opt mode, we use assertions to track internal + // state that can cause extra calls to hash. + int conversions = 0; + int hashes = 0; + flat_hash_map<size_t, size_t, Hash, Eq> m(0, Hash{&hashes}); + m.reserve(3); + + m[LazyInt(1, &conversions)] = 1; + EXPECT_THAT(m, UnorderedElementsAre(Pair(1, 1))); + EXPECT_EQ(conversions, 1); +#ifdef NDEBUG + EXPECT_EQ(hashes, 1); +#endif + + m[LazyInt(1, &conversions)] = 2; + EXPECT_THAT(m, UnorderedElementsAre(Pair(1, 2))); + EXPECT_EQ(conversions, 1); +#ifdef NDEBUG + EXPECT_EQ(hashes, 2); +#endif + + m.try_emplace(LazyInt(2, &conversions), 3); + EXPECT_THAT(m, UnorderedElementsAre(Pair(1, 2), Pair(2, 3))); + EXPECT_EQ(conversions, 2); +#ifdef NDEBUG + EXPECT_EQ(hashes, 3); +#endif + + m.try_emplace(LazyInt(2, &conversions), 4); + EXPECT_THAT(m, UnorderedElementsAre(Pair(1, 2), Pair(2, 3))); + EXPECT_EQ(conversions, 2); +#ifdef NDEBUG + EXPECT_EQ(hashes, 4); +#endif +} + +TEST(FlatHashMap, BitfieldArgument) { + union { + int n : 1; + }; + n = 0; + flat_hash_map<int, int> m; + m.erase(n); + m.count(n); + m.prefetch(n); + m.find(n); + m.contains(n); + m.equal_range(n); + m.insert_or_assign(n, n); + m.insert_or_assign(m.end(), n, n); + m.try_emplace(n); + m.try_emplace(m.end(), n); + m.at(n); + m[n]; +} + +TEST(FlatHashMap, MergeExtractInsert) { + // We can't test mutable keys, or non-copyable keys with flat_hash_map. + // Test that the nodes have the proper API. + absl::flat_hash_map<int, int> m = {{1, 7}, {2, 9}}; + auto node = m.extract(1); + EXPECT_TRUE(node); + EXPECT_EQ(node.key(), 1); + EXPECT_EQ(node.mapped(), 7); + EXPECT_THAT(m, UnorderedElementsAre(Pair(2, 9))); + + node.mapped() = 17; + m.insert(std::move(node)); + EXPECT_THAT(m, UnorderedElementsAre(Pair(1, 17), Pair(2, 9))); +} + +bool FirstIsEven(std::pair<const int, int> p) { return p.first % 2 == 0; } + +TEST(FlatHashMap, EraseIf) { + // Erase all elements. + { + flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, [](std::pair<const int, int>) { return true; }); + EXPECT_THAT(s, IsEmpty()); + } + // Erase no elements. + { + flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, [](std::pair<const int, int>) { return false; }); + EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), + Pair(4, 4), Pair(5, 5))); + } + // Erase specific elements. + { + flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, + [](std::pair<const int, int> kvp) { return kvp.first % 2 == 1; }); + EXPECT_THAT(s, UnorderedElementsAre(Pair(2, 2), Pair(4, 4))); + } + // Predicate is function reference. + { + flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, FirstIsEven); + EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); + } + // Predicate is function pointer. + { + flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, &FirstIsEven); + EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); + } +} + +// This test requires std::launder for mutable key access in node handles. +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 +TEST(FlatHashMap, NodeHandleMutableKeyAccess) { + flat_hash_map<std::string, std::string> map; + + map["key1"] = "mapped"; + + auto nh = map.extract(map.begin()); + nh.key().resize(3); + map.insert(std::move(nh)); + + EXPECT_THAT(map, testing::ElementsAre(Pair("key", "mapped"))); +} +#endif + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/flat_hash_set.h b/third_party/abseil_cpp/absl/container/flat_hash_set.h new file mode 100644 index 000000000000..6b89da65714c --- /dev/null +++ b/third_party/abseil_cpp/absl/container/flat_hash_set.h @@ -0,0 +1,504 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: flat_hash_set.h +// ----------------------------------------------------------------------------- +// +// An `absl::flat_hash_set<T>` is an unordered associative container designed to +// be a more efficient replacement for `std::unordered_set`. Like +// `unordered_set`, search, insertion, and deletion of set elements can be done +// as an `O(1)` operation. However, `flat_hash_set` (and other unordered +// associative containers known as the collection of Abseil "Swiss tables") +// contain other optimizations that result in both memory and computation +// advantages. +// +// In most cases, your default choice for a hash set should be a set of type +// `flat_hash_set`. +#ifndef ABSL_CONTAINER_FLAT_HASH_SET_H_ +#define ABSL_CONTAINER_FLAT_HASH_SET_H_ + +#include <type_traits> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/base/macros.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export +#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +template <typename T> +struct FlatHashSetPolicy; +} // namespace container_internal + +// ----------------------------------------------------------------------------- +// absl::flat_hash_set +// ----------------------------------------------------------------------------- +// +// An `absl::flat_hash_set<T>` is an unordered associative container which has +// been optimized for both speed and memory footprint in most common use cases. +// Its interface is similar to that of `std::unordered_set<T>` with the +// following notable differences: +// +// * Requires keys that are CopyConstructible +// * Supports heterogeneous lookup, through `find()` and `insert()`, provided +// that the set is provided a compatible heterogeneous hashing function and +// equality operator. +// * Invalidates any references and pointers to elements within the table after +// `rehash()`. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash set. +// * Returns `void` from the `erase(iterator)` overload. +// +// By default, `flat_hash_set` uses the `absl::Hash` hashing framework. All +// fundamental and Abseil types that support the `absl::Hash` framework have a +// compatible equality operator for comparing insertions into `flat_hash_map`. +// If your type is not yet supported by the `absl::Hash` framework, see +// absl/hash/hash.h for information on extending Abseil hashing to user-defined +// types. +// +// NOTE: A `flat_hash_set` stores its keys directly inside its implementation +// array to avoid memory indirection. Because a `flat_hash_set` is designed to +// move data when rehashed, set keys will not retain pointer stability. If you +// require pointer stability, consider using +// `absl::flat_hash_set<std::unique_ptr<T>>`. If your type is not moveable and +// you require pointer stability, consider `absl::node_hash_set` instead. +// +// Example: +// +// // Create a flat hash set of three strings +// absl::flat_hash_set<std::string> ducks = +// {"huey", "dewey", "louie"}; +// +// // Insert a new element into the flat hash set +// ducks.insert("donald"); +// +// // Force a rehash of the flat hash set +// ducks.rehash(0); +// +// // See if "dewey" is present +// if (ducks.contains("dewey")) { +// std::cout << "We found dewey!" << std::endl; +// } +template <class T, class Hash = absl::container_internal::hash_default_hash<T>, + class Eq = absl::container_internal::hash_default_eq<T>, + class Allocator = std::allocator<T>> +class flat_hash_set + : public absl::container_internal::raw_hash_set< + absl::container_internal::FlatHashSetPolicy<T>, Hash, Eq, Allocator> { + using Base = typename flat_hash_set::raw_hash_set; + + public: + // Constructors and Assignment Operators + // + // A flat_hash_set supports the same overload set as `std::unordered_map` + // for construction and assignment: + // + // * Default constructor + // + // // No allocation for the table's elements is made. + // absl::flat_hash_set<std::string> set1; + // + // * Initializer List constructor + // + // absl::flat_hash_set<std::string> set2 = + // {{"huey"}, {"dewey"}, {"louie"},}; + // + // * Copy constructor + // + // absl::flat_hash_set<std::string> set3(set2); + // + // * Copy assignment operator + // + // // Hash functor and Comparator are copied as well + // absl::flat_hash_set<std::string> set4; + // set4 = set3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::flat_hash_set<std::string> set5(std::move(set4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::flat_hash_set<std::string> set6; + // set6 = std::move(set5); + // + // * Range constructor + // + // std::vector<std::string> v = {"a", "b"}; + // absl::flat_hash_set<std::string> set7(v.begin(), v.end()); + flat_hash_set() {} + using Base::Base; + + // flat_hash_set::begin() + // + // Returns an iterator to the beginning of the `flat_hash_set`. + using Base::begin; + + // flat_hash_set::cbegin() + // + // Returns a const iterator to the beginning of the `flat_hash_set`. + using Base::cbegin; + + // flat_hash_set::cend() + // + // Returns a const iterator to the end of the `flat_hash_set`. + using Base::cend; + + // flat_hash_set::end() + // + // Returns an iterator to the end of the `flat_hash_set`. + using Base::end; + + // flat_hash_set::capacity() + // + // Returns the number of element slots (assigned, deleted, and empty) + // available within the `flat_hash_set`. + // + // NOTE: this member function is particular to `absl::flat_hash_set` and is + // not provided in the `std::unordered_map` API. + using Base::capacity; + + // flat_hash_set::empty() + // + // Returns whether or not the `flat_hash_set` is empty. + using Base::empty; + + // flat_hash_set::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `flat_hash_set` under current memory constraints. This value can be thought + // of the largest value of `std::distance(begin(), end())` for a + // `flat_hash_set<T>`. + using Base::max_size; + + // flat_hash_set::size() + // + // Returns the number of elements currently within the `flat_hash_set`. + using Base::size; + + // flat_hash_set::clear() + // + // Removes all elements from the `flat_hash_set`. Invalidates any references, + // pointers, or iterators referring to contained elements. + // + // NOTE: this operation may shrink the underlying buffer. To avoid shrinking + // the underlying buffer call `erase(begin(), end())`. + using Base::clear; + + // flat_hash_set::erase() + // + // Erases elements within the `flat_hash_set`. Erasing does not trigger a + // rehash. Overloads are listed below. + // + // void erase(const_iterator pos): + // + // Erases the element at `position` of the `flat_hash_set`, returning + // `void`. + // + // NOTE: returning `void` in this case is different than that of STL + // containers in general and `std::unordered_set` in particular (which + // return an iterator to the element following the erased element). If that + // iterator is needed, simply post increment the iterator: + // + // set.erase(it++); + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning an + // iterator pointing to `last`. + // + // size_type erase(const key_type& key): + // + // Erases the element with the matching key, if it exists, returning the + // number of elements erased (0 or 1). + using Base::erase; + + // flat_hash_set::insert() + // + // Inserts an element of the specified value into the `flat_hash_set`, + // returning an iterator pointing to the newly inserted element, provided that + // an element with the given key does not already exist. If rehashing occurs + // due to the insertion, all iterators are invalidated. Overloads are listed + // below. + // + // std::pair<iterator,bool> insert(const T& value): + // + // Inserts a value into the `flat_hash_set`. Returns a pair consisting of an + // iterator to the inserted element (or to the element that prevented the + // insertion) and a bool denoting whether the insertion took place. + // + // std::pair<iterator,bool> insert(T&& value): + // + // Inserts a moveable value into the `flat_hash_set`. Returns a pair + // consisting of an iterator to the inserted element (or to the element that + // prevented the insertion) and a bool denoting whether the insertion took + // place. + // + // iterator insert(const_iterator hint, const T& value): + // iterator insert(const_iterator hint, T&& value): + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element, or to the existing element that prevented the + // insertion. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently, for `flat_hash_set` we guarantee the + // first match is inserted. + // + // void insert(std::initializer_list<T> ilist): + // + // Inserts the elements within the initializer list `ilist`. + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently within the initializer list, for + // `flat_hash_set` we guarantee the first match is inserted. + using Base::insert; + + // flat_hash_set::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `flat_hash_set`, provided that no element with the given key + // already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace; + + // flat_hash_set::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `flat_hash_set`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search, and only inserts + // provided that no element with the given key already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace_hint; + + // flat_hash_set::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the element at the indicated position and returns a node handle + // owning that extracted data. + // + // node_type extract(const key_type& x): + // + // Extracts the element with the key matching the passed key value and + // returns a node handle owning that extracted data. If the `flat_hash_set` + // does not contain an element with a matching key, this function returns an + // empty node handle. + using Base::extract; + + // flat_hash_set::merge() + // + // Extracts elements from a given `source` flat hash set into this + // `flat_hash_set`. If the destination `flat_hash_set` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // flat_hash_set::swap(flat_hash_set& other) + // + // Exchanges the contents of this `flat_hash_set` with those of the `other` + // flat hash map, avoiding invocation of any move, copy, or swap operations on + // individual elements. + // + // All iterators and references on the `flat_hash_set` remain valid, excepting + // for the past-the-end iterator, which is invalidated. + // + // `swap()` requires that the flat hash set's hashing and key equivalence + // functions be Swappable, and are exchaged using unqualified calls to + // non-member `swap()`. If the map's allocator has + // `std::allocator_traits<allocator_type>::propagate_on_container_swap::value` + // set to `true`, the allocators are also exchanged using an unqualified call + // to non-member `swap()`; otherwise, the allocators are not swapped. + using Base::swap; + + // flat_hash_set::rehash(count) + // + // Rehashes the `flat_hash_set`, setting the number of slots to be at least + // the passed value. If the new number of slots increases the load factor more + // than the current maximum load factor + // (`count` < `size()` / `max_load_factor()`), then the new number of slots + // will be at least `size()` / `max_load_factor()`. + // + // To force a rehash, pass rehash(0). + // + // NOTE: unlike behavior in `std::unordered_set`, references are also + // invalidated upon a `rehash()`. + using Base::rehash; + + // flat_hash_set::reserve(count) + // + // Sets the number of slots in the `flat_hash_set` to the number needed to + // accommodate at least `count` total elements without exceeding the current + // maximum load factor, and may rehash the container if needed. + using Base::reserve; + + // flat_hash_set::contains() + // + // Determines whether an element comparing equal to the given `key` exists + // within the `flat_hash_set`, returning `true` if so or `false` otherwise. + using Base::contains; + + // flat_hash_set::count(const Key& key) const + // + // Returns the number of elements comparing equal to the given `key` within + // the `flat_hash_set`. note that this function will return either `1` or `0` + // since duplicate elements are not allowed within a `flat_hash_set`. + using Base::count; + + // flat_hash_set::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `flat_hash_set`. + using Base::equal_range; + + // flat_hash_set::find() + // + // Finds an element with the passed `key` within the `flat_hash_set`. + using Base::find; + + // flat_hash_set::bucket_count() + // + // Returns the number of "buckets" within the `flat_hash_set`. Note that + // because a flat hash map contains all elements within its internal storage, + // this value simply equals the current capacity of the `flat_hash_set`. + using Base::bucket_count; + + // flat_hash_set::load_factor() + // + // Returns the current load factor of the `flat_hash_set` (the average number + // of slots occupied with a value within the hash map). + using Base::load_factor; + + // flat_hash_set::max_load_factor() + // + // Manages the maximum load factor of the `flat_hash_set`. Overloads are + // listed below. + // + // float flat_hash_set::max_load_factor() + // + // Returns the current maximum load factor of the `flat_hash_set`. + // + // void flat_hash_set::max_load_factor(float ml) + // + // Sets the maximum load factor of the `flat_hash_set` to the passed value. + // + // NOTE: This overload is provided only for API compatibility with the STL; + // `flat_hash_set` will ignore any set load factor and manage its rehashing + // internally as an implementation detail. + using Base::max_load_factor; + + // flat_hash_set::get_allocator() + // + // Returns the allocator function associated with this `flat_hash_set`. + using Base::get_allocator; + + // flat_hash_set::hash_function() + // + // Returns the hashing function used to hash the keys within this + // `flat_hash_set`. + using Base::hash_function; + + // flat_hash_set::key_eq() + // + // Returns the function used for comparing keys equality. + using Base::key_eq; +}; + +// erase_if(flat_hash_set<>, Pred) +// +// Erases all elements that satisfy the predicate `pred` from the container `c`. +template <typename T, typename H, typename E, typename A, typename Predicate> +void erase_if(flat_hash_set<T, H, E, A>& c, Predicate pred) { + container_internal::EraseIf(pred, &c); +} + +namespace container_internal { + +template <class T> +struct FlatHashSetPolicy { + using slot_type = T; + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + absl::allocator_traits<Allocator>::construct(*alloc, slot, + std::forward<Args>(args)...); + } + + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + absl::allocator_traits<Allocator>::destroy(*alloc, slot); + } + + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(*old_slot)); + destroy(alloc, old_slot); + } + + static T& element(slot_type* slot) { return *slot; } + + template <class F, class... Args> + static decltype(absl::container_internal::DecomposeValue( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return absl::container_internal::DecomposeValue( + std::forward<F>(f), std::forward<Args>(args)...); + } + + static size_t space_used(const T*) { return 0; } +}; +} // namespace container_internal + +namespace container_algorithm_internal { + +// Specialization of trait in absl/algorithm/container.h +template <class Key, class Hash, class KeyEqual, class Allocator> +struct IsUnorderedContainer<absl::flat_hash_set<Key, Hash, KeyEqual, Allocator>> + : std::true_type {}; + +} // namespace container_algorithm_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_FLAT_HASH_SET_H_ diff --git a/third_party/abseil_cpp/absl/container/flat_hash_set_test.cc b/third_party/abseil_cpp/absl/container/flat_hash_set_test.cc new file mode 100644 index 000000000000..8f6f9944ca4e --- /dev/null +++ b/third_party/abseil_cpp/absl/container/flat_hash_set_test.cc @@ -0,0 +1,178 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/flat_hash_set.h" + +#include <vector> + +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/unordered_set_constructor_test.h" +#include "absl/container/internal/unordered_set_lookup_test.h" +#include "absl/container/internal/unordered_set_members_test.h" +#include "absl/container/internal/unordered_set_modifiers_test.h" +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::absl::container_internal::hash_internal::Enum; +using ::absl::container_internal::hash_internal::EnumClass; +using ::testing::IsEmpty; +using ::testing::Pointee; +using ::testing::UnorderedElementsAre; +using ::testing::UnorderedElementsAreArray; + +// Check that absl::flat_hash_set works in a global constructor. +struct BeforeMain { + BeforeMain() { + absl::flat_hash_set<int> x; + x.insert(1); + ABSL_RAW_CHECK(!x.contains(0), "x should not contain 0"); + ABSL_RAW_CHECK(x.contains(1), "x should contain 1"); + } +}; +const BeforeMain before_main; + +template <class T> +using Set = + absl::flat_hash_set<T, StatefulTestingHash, StatefulTestingEqual, Alloc<T>>; + +using SetTypes = + ::testing::Types<Set<int>, Set<std::string>, Set<Enum>, Set<EnumClass>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashSet, ConstructorTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashSet, LookupTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashSet, MembersTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(FlatHashSet, ModifiersTest, SetTypes); + +TEST(FlatHashSet, EmplaceString) { + std::vector<std::string> v = {"a", "b"}; + absl::flat_hash_set<absl::string_view> hs(v.begin(), v.end()); + EXPECT_THAT(hs, UnorderedElementsAreArray(v)); +} + +TEST(FlatHashSet, BitfieldArgument) { + union { + int n : 1; + }; + n = 0; + absl::flat_hash_set<int> s = {n}; + s.insert(n); + s.insert(s.end(), n); + s.insert({n}); + s.erase(n); + s.count(n); + s.prefetch(n); + s.find(n); + s.contains(n); + s.equal_range(n); +} + +TEST(FlatHashSet, MergeExtractInsert) { + struct Hash { + size_t operator()(const std::unique_ptr<int>& p) const { return *p; } + }; + struct Eq { + bool operator()(const std::unique_ptr<int>& a, + const std::unique_ptr<int>& b) const { + return *a == *b; + } + }; + absl::flat_hash_set<std::unique_ptr<int>, Hash, Eq> set1, set2; + set1.insert(absl::make_unique<int>(7)); + set1.insert(absl::make_unique<int>(17)); + + set2.insert(absl::make_unique<int>(7)); + set2.insert(absl::make_unique<int>(19)); + + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17))); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(19))); + + set1.merge(set2); + + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17), Pointee(19))); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); + + auto node = set1.extract(absl::make_unique<int>(7)); + EXPECT_TRUE(node); + EXPECT_THAT(node.value(), Pointee(7)); + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(17), Pointee(19))); + + auto insert_result = set2.insert(std::move(node)); + EXPECT_FALSE(node); + EXPECT_FALSE(insert_result.inserted); + EXPECT_TRUE(insert_result.node); + EXPECT_THAT(insert_result.node.value(), Pointee(7)); + EXPECT_EQ(**insert_result.position, 7); + EXPECT_NE(insert_result.position->get(), insert_result.node.value().get()); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); + + node = set1.extract(absl::make_unique<int>(17)); + EXPECT_TRUE(node); + EXPECT_THAT(node.value(), Pointee(17)); + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(19))); + + node.value() = absl::make_unique<int>(23); + + insert_result = set2.insert(std::move(node)); + EXPECT_FALSE(node); + EXPECT_TRUE(insert_result.inserted); + EXPECT_FALSE(insert_result.node); + EXPECT_EQ(**insert_result.position, 23); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(23))); +} + +bool IsEven(int k) { return k % 2 == 0; } + +TEST(FlatHashSet, EraseIf) { + // Erase all elements. + { + flat_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, [](int) { return true; }); + EXPECT_THAT(s, IsEmpty()); + } + // Erase no elements. + { + flat_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, [](int) { return false; }); + EXPECT_THAT(s, UnorderedElementsAre(1, 2, 3, 4, 5)); + } + // Erase specific elements. + { + flat_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, [](int k) { return k % 2 == 1; }); + EXPECT_THAT(s, UnorderedElementsAre(2, 4)); + } + // Predicate is function reference. + { + flat_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, IsEven); + EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); + } + // Predicate is function pointer. + { + flat_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, &IsEven); + EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); + } +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/inlined_vector.h b/third_party/abseil_cpp/absl/container/inlined_vector.h new file mode 100644 index 000000000000..90bb96e8325c --- /dev/null +++ b/third_party/abseil_cpp/absl/container/inlined_vector.h @@ -0,0 +1,845 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: inlined_vector.h +// ----------------------------------------------------------------------------- +// +// This header file contains the declaration and definition of an "inlined +// vector" which behaves in an equivalent fashion to a `std::vector`, except +// that storage for small sequences of the vector are provided inline without +// requiring any heap allocation. +// +// An `absl::InlinedVector<T, N>` specifies the default capacity `N` as one of +// its template parameters. Instances where `size() <= N` hold contained +// elements in inline space. Typically `N` is very small so that sequences that +// are expected to be short do not require allocations. +// +// An `absl::InlinedVector` does not usually require a specific allocator. If +// the inlined vector grows beyond its initial constraints, it will need to +// allocate (as any normal `std::vector` would). This is usually performed with +// the default allocator (defined as `std::allocator<T>`). Optionally, a custom +// allocator type may be specified as `A` in `absl::InlinedVector<T, N, A>`. + +#ifndef ABSL_CONTAINER_INLINED_VECTOR_H_ +#define ABSL_CONTAINER_INLINED_VECTOR_H_ + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdlib> +#include <cstring> +#include <initializer_list> +#include <iterator> +#include <memory> +#include <type_traits> +#include <utility> + +#include "absl/algorithm/algorithm.h" +#include "absl/base/internal/throw_delegate.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" +#include "absl/container/internal/inlined_vector.h" +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +// ----------------------------------------------------------------------------- +// InlinedVector +// ----------------------------------------------------------------------------- +// +// An `absl::InlinedVector` is designed to be a drop-in replacement for +// `std::vector` for use cases where the vector's size is sufficiently small +// that it can be inlined. If the inlined vector does grow beyond its estimated +// capacity, it will trigger an initial allocation on the heap, and will behave +// as a `std::vector`. The API of the `absl::InlinedVector` within this file is +// designed to cover the same API footprint as covered by `std::vector`. +template <typename T, size_t N, typename A = std::allocator<T>> +class InlinedVector { + static_assert(N > 0, "`absl::InlinedVector` requires an inlined capacity."); + + using Storage = inlined_vector_internal::Storage<T, N, A>; + + using AllocatorTraits = typename Storage::AllocatorTraits; + using RValueReference = typename Storage::RValueReference; + using MoveIterator = typename Storage::MoveIterator; + using IsMemcpyOk = typename Storage::IsMemcpyOk; + + template <typename Iterator> + using IteratorValueAdapter = + typename Storage::template IteratorValueAdapter<Iterator>; + using CopyValueAdapter = typename Storage::CopyValueAdapter; + using DefaultValueAdapter = typename Storage::DefaultValueAdapter; + + template <typename Iterator> + using EnableIfAtLeastForwardIterator = absl::enable_if_t< + inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value>; + template <typename Iterator> + using DisableIfAtLeastForwardIterator = absl::enable_if_t< + !inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value>; + + public: + using allocator_type = typename Storage::allocator_type; + using value_type = typename Storage::value_type; + using pointer = typename Storage::pointer; + using const_pointer = typename Storage::const_pointer; + using size_type = typename Storage::size_type; + using difference_type = typename Storage::difference_type; + using reference = typename Storage::reference; + using const_reference = typename Storage::const_reference; + using iterator = typename Storage::iterator; + using const_iterator = typename Storage::const_iterator; + using reverse_iterator = typename Storage::reverse_iterator; + using const_reverse_iterator = typename Storage::const_reverse_iterator; + + // --------------------------------------------------------------------------- + // InlinedVector Constructors and Destructor + // --------------------------------------------------------------------------- + + // Creates an empty inlined vector with a value-initialized allocator. + InlinedVector() noexcept(noexcept(allocator_type())) : storage_() {} + + // Creates an empty inlined vector with a copy of `alloc`. + explicit InlinedVector(const allocator_type& alloc) noexcept + : storage_(alloc) {} + + // Creates an inlined vector with `n` copies of `value_type()`. + explicit InlinedVector(size_type n, + const allocator_type& alloc = allocator_type()) + : storage_(alloc) { + storage_.Initialize(DefaultValueAdapter(), n); + } + + // Creates an inlined vector with `n` copies of `v`. + InlinedVector(size_type n, const_reference v, + const allocator_type& alloc = allocator_type()) + : storage_(alloc) { + storage_.Initialize(CopyValueAdapter(v), n); + } + + // Creates an inlined vector with copies of the elements of `list`. + InlinedVector(std::initializer_list<value_type> list, + const allocator_type& alloc = allocator_type()) + : InlinedVector(list.begin(), list.end(), alloc) {} + + // Creates an inlined vector with elements constructed from the provided + // forward iterator range [`first`, `last`). + // + // NOTE: the `enable_if` prevents ambiguous interpretation between a call to + // this constructor with two integral arguments and a call to the above + // `InlinedVector(size_type, const_reference)` constructor. + template <typename ForwardIterator, + EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr> + InlinedVector(ForwardIterator first, ForwardIterator last, + const allocator_type& alloc = allocator_type()) + : storage_(alloc) { + storage_.Initialize(IteratorValueAdapter<ForwardIterator>(first), + std::distance(first, last)); + } + + // Creates an inlined vector with elements constructed from the provided input + // iterator range [`first`, `last`). + template <typename InputIterator, + DisableIfAtLeastForwardIterator<InputIterator>* = nullptr> + InlinedVector(InputIterator first, InputIterator last, + const allocator_type& alloc = allocator_type()) + : storage_(alloc) { + std::copy(first, last, std::back_inserter(*this)); + } + + // Creates an inlined vector by copying the contents of `other` using + // `other`'s allocator. + InlinedVector(const InlinedVector& other) + : InlinedVector(other, *other.storage_.GetAllocPtr()) {} + + // Creates an inlined vector by copying the contents of `other` using `alloc`. + InlinedVector(const InlinedVector& other, const allocator_type& alloc) + : storage_(alloc) { + if (IsMemcpyOk::value && !other.storage_.GetIsAllocated()) { + storage_.MemcpyFrom(other.storage_); + } else { + storage_.Initialize(IteratorValueAdapter<const_pointer>(other.data()), + other.size()); + } + } + + // Creates an inlined vector by moving in the contents of `other` without + // allocating. If `other` contains allocated memory, the newly-created inlined + // vector will take ownership of that memory. However, if `other` does not + // contain allocated memory, the newly-created inlined vector will perform + // element-wise move construction of the contents of `other`. + // + // NOTE: since no allocation is performed for the inlined vector in either + // case, the `noexcept(...)` specification depends on whether moving the + // underlying objects can throw. It is assumed assumed that... + // a) move constructors should only throw due to allocation failure. + // b) if `value_type`'s move constructor allocates, it uses the same + // allocation function as the inlined vector's allocator. + // Thus, the move constructor is non-throwing if the allocator is non-throwing + // or `value_type`'s move constructor is specified as `noexcept`. + InlinedVector(InlinedVector&& other) noexcept( + absl::allocator_is_nothrow<allocator_type>::value || + std::is_nothrow_move_constructible<value_type>::value) + : storage_(*other.storage_.GetAllocPtr()) { + if (IsMemcpyOk::value) { + storage_.MemcpyFrom(other.storage_); + + other.storage_.SetInlinedSize(0); + } else if (other.storage_.GetIsAllocated()) { + storage_.SetAllocatedData(other.storage_.GetAllocatedData(), + other.storage_.GetAllocatedCapacity()); + storage_.SetAllocatedSize(other.storage_.GetSize()); + + other.storage_.SetInlinedSize(0); + } else { + IteratorValueAdapter<MoveIterator> other_values( + MoveIterator(other.storage_.GetInlinedData())); + + inlined_vector_internal::ConstructElements( + storage_.GetAllocPtr(), storage_.GetInlinedData(), &other_values, + other.storage_.GetSize()); + + storage_.SetInlinedSize(other.storage_.GetSize()); + } + } + + // Creates an inlined vector by moving in the contents of `other` with a copy + // of `alloc`. + // + // NOTE: if `other`'s allocator is not equal to `alloc`, even if `other` + // contains allocated memory, this move constructor will still allocate. Since + // allocation is performed, this constructor can only be `noexcept` if the + // specified allocator is also `noexcept`. + InlinedVector(InlinedVector&& other, const allocator_type& alloc) noexcept( + absl::allocator_is_nothrow<allocator_type>::value) + : storage_(alloc) { + if (IsMemcpyOk::value) { + storage_.MemcpyFrom(other.storage_); + + other.storage_.SetInlinedSize(0); + } else if ((*storage_.GetAllocPtr() == *other.storage_.GetAllocPtr()) && + other.storage_.GetIsAllocated()) { + storage_.SetAllocatedData(other.storage_.GetAllocatedData(), + other.storage_.GetAllocatedCapacity()); + storage_.SetAllocatedSize(other.storage_.GetSize()); + + other.storage_.SetInlinedSize(0); + } else { + storage_.Initialize( + IteratorValueAdapter<MoveIterator>(MoveIterator(other.data())), + other.size()); + } + } + + ~InlinedVector() {} + + // --------------------------------------------------------------------------- + // InlinedVector Member Accessors + // --------------------------------------------------------------------------- + + // `InlinedVector::empty()` + // + // Returns whether the inlined vector contains no elements. + bool empty() const noexcept { return !size(); } + + // `InlinedVector::size()` + // + // Returns the number of elements in the inlined vector. + size_type size() const noexcept { return storage_.GetSize(); } + + // `InlinedVector::max_size()` + // + // Returns the maximum number of elements the inlined vector can hold. + size_type max_size() const noexcept { + // One bit of the size storage is used to indicate whether the inlined + // vector contains allocated memory. As a result, the maximum size that the + // inlined vector can express is half of the max for `size_type`. + return (std::numeric_limits<size_type>::max)() / 2; + } + + // `InlinedVector::capacity()` + // + // Returns the number of elements that could be stored in the inlined vector + // without requiring a reallocation. + // + // NOTE: for most inlined vectors, `capacity()` should be equal to the + // template parameter `N`. For inlined vectors which exceed this capacity, + // they will no longer be inlined and `capacity()` will equal the capactity of + // the allocated memory. + size_type capacity() const noexcept { + return storage_.GetIsAllocated() ? storage_.GetAllocatedCapacity() + : storage_.GetInlinedCapacity(); + } + + // `InlinedVector::data()` + // + // Returns a `pointer` to the elements of the inlined vector. This pointer + // can be used to access and modify the contained elements. + // + // NOTE: only elements within [`data()`, `data() + size()`) are valid. + pointer data() noexcept { + return storage_.GetIsAllocated() ? storage_.GetAllocatedData() + : storage_.GetInlinedData(); + } + + // Overload of `InlinedVector::data()` that returns a `const_pointer` to the + // elements of the inlined vector. This pointer can be used to access but not + // modify the contained elements. + // + // NOTE: only elements within [`data()`, `data() + size()`) are valid. + const_pointer data() const noexcept { + return storage_.GetIsAllocated() ? storage_.GetAllocatedData() + : storage_.GetInlinedData(); + } + + // `InlinedVector::operator[](...)` + // + // Returns a `reference` to the `i`th element of the inlined vector. + reference operator[](size_type i) { + ABSL_HARDENING_ASSERT(i < size()); + return data()[i]; + } + + // Overload of `InlinedVector::operator[](...)` that returns a + // `const_reference` to the `i`th element of the inlined vector. + const_reference operator[](size_type i) const { + ABSL_HARDENING_ASSERT(i < size()); + return data()[i]; + } + + // `InlinedVector::at(...)` + // + // Returns a `reference` to the `i`th element of the inlined vector. + // + // NOTE: if `i` is not within the required range of `InlinedVector::at(...)`, + // in both debug and non-debug builds, `std::out_of_range` will be thrown. + reference at(size_type i) { + if (ABSL_PREDICT_FALSE(i >= size())) { + base_internal::ThrowStdOutOfRange( + "`InlinedVector::at(size_type)` failed bounds check"); + } + return data()[i]; + } + + // Overload of `InlinedVector::at(...)` that returns a `const_reference` to + // the `i`th element of the inlined vector. + // + // NOTE: if `i` is not within the required range of `InlinedVector::at(...)`, + // in both debug and non-debug builds, `std::out_of_range` will be thrown. + const_reference at(size_type i) const { + if (ABSL_PREDICT_FALSE(i >= size())) { + base_internal::ThrowStdOutOfRange( + "`InlinedVector::at(size_type) const` failed bounds check"); + } + return data()[i]; + } + + // `InlinedVector::front()` + // + // Returns a `reference` to the first element of the inlined vector. + reference front() { + ABSL_HARDENING_ASSERT(!empty()); + return data()[0]; + } + + // Overload of `InlinedVector::front()` that returns a `const_reference` to + // the first element of the inlined vector. + const_reference front() const { + ABSL_HARDENING_ASSERT(!empty()); + return data()[0]; + } + + // `InlinedVector::back()` + // + // Returns a `reference` to the last element of the inlined vector. + reference back() { + ABSL_HARDENING_ASSERT(!empty()); + return data()[size() - 1]; + } + + // Overload of `InlinedVector::back()` that returns a `const_reference` to the + // last element of the inlined vector. + const_reference back() const { + ABSL_HARDENING_ASSERT(!empty()); + return data()[size() - 1]; + } + + // `InlinedVector::begin()` + // + // Returns an `iterator` to the beginning of the inlined vector. + iterator begin() noexcept { return data(); } + + // Overload of `InlinedVector::begin()` that returns a `const_iterator` to + // the beginning of the inlined vector. + const_iterator begin() const noexcept { return data(); } + + // `InlinedVector::end()` + // + // Returns an `iterator` to the end of the inlined vector. + iterator end() noexcept { return data() + size(); } + + // Overload of `InlinedVector::end()` that returns a `const_iterator` to the + // end of the inlined vector. + const_iterator end() const noexcept { return data() + size(); } + + // `InlinedVector::cbegin()` + // + // Returns a `const_iterator` to the beginning of the inlined vector. + const_iterator cbegin() const noexcept { return begin(); } + + // `InlinedVector::cend()` + // + // Returns a `const_iterator` to the end of the inlined vector. + const_iterator cend() const noexcept { return end(); } + + // `InlinedVector::rbegin()` + // + // Returns a `reverse_iterator` from the end of the inlined vector. + reverse_iterator rbegin() noexcept { return reverse_iterator(end()); } + + // Overload of `InlinedVector::rbegin()` that returns a + // `const_reverse_iterator` from the end of the inlined vector. + const_reverse_iterator rbegin() const noexcept { + return const_reverse_iterator(end()); + } + + // `InlinedVector::rend()` + // + // Returns a `reverse_iterator` from the beginning of the inlined vector. + reverse_iterator rend() noexcept { return reverse_iterator(begin()); } + + // Overload of `InlinedVector::rend()` that returns a `const_reverse_iterator` + // from the beginning of the inlined vector. + const_reverse_iterator rend() const noexcept { + return const_reverse_iterator(begin()); + } + + // `InlinedVector::crbegin()` + // + // Returns a `const_reverse_iterator` from the end of the inlined vector. + const_reverse_iterator crbegin() const noexcept { return rbegin(); } + + // `InlinedVector::crend()` + // + // Returns a `const_reverse_iterator` from the beginning of the inlined + // vector. + const_reverse_iterator crend() const noexcept { return rend(); } + + // `InlinedVector::get_allocator()` + // + // Returns a copy of the inlined vector's allocator. + allocator_type get_allocator() const { return *storage_.GetAllocPtr(); } + + // --------------------------------------------------------------------------- + // InlinedVector Member Mutators + // --------------------------------------------------------------------------- + + // `InlinedVector::operator=(...)` + // + // Replaces the elements of the inlined vector with copies of the elements of + // `list`. + InlinedVector& operator=(std::initializer_list<value_type> list) { + assign(list.begin(), list.end()); + + return *this; + } + + // Overload of `InlinedVector::operator=(...)` that replaces the elements of + // the inlined vector with copies of the elements of `other`. + InlinedVector& operator=(const InlinedVector& other) { + if (ABSL_PREDICT_TRUE(this != std::addressof(other))) { + const_pointer other_data = other.data(); + assign(other_data, other_data + other.size()); + } + + return *this; + } + + // Overload of `InlinedVector::operator=(...)` that moves the elements of + // `other` into the inlined vector. + // + // NOTE: as a result of calling this overload, `other` is left in a valid but + // unspecified state. + InlinedVector& operator=(InlinedVector&& other) { + if (ABSL_PREDICT_TRUE(this != std::addressof(other))) { + if (IsMemcpyOk::value || other.storage_.GetIsAllocated()) { + inlined_vector_internal::DestroyElements(storage_.GetAllocPtr(), data(), + size()); + storage_.DeallocateIfAllocated(); + storage_.MemcpyFrom(other.storage_); + + other.storage_.SetInlinedSize(0); + } else { + storage_.Assign(IteratorValueAdapter<MoveIterator>( + MoveIterator(other.storage_.GetInlinedData())), + other.size()); + } + } + + return *this; + } + + // `InlinedVector::assign(...)` + // + // Replaces the contents of the inlined vector with `n` copies of `v`. + void assign(size_type n, const_reference v) { + storage_.Assign(CopyValueAdapter(v), n); + } + + // Overload of `InlinedVector::assign(...)` that replaces the contents of the + // inlined vector with copies of the elements of `list`. + void assign(std::initializer_list<value_type> list) { + assign(list.begin(), list.end()); + } + + // Overload of `InlinedVector::assign(...)` to replace the contents of the + // inlined vector with the range [`first`, `last`). + // + // NOTE: this overload is for iterators that are "forward" category or better. + template <typename ForwardIterator, + EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr> + void assign(ForwardIterator first, ForwardIterator last) { + storage_.Assign(IteratorValueAdapter<ForwardIterator>(first), + std::distance(first, last)); + } + + // Overload of `InlinedVector::assign(...)` to replace the contents of the + // inlined vector with the range [`first`, `last`). + // + // NOTE: this overload is for iterators that are "input" category. + template <typename InputIterator, + DisableIfAtLeastForwardIterator<InputIterator>* = nullptr> + void assign(InputIterator first, InputIterator last) { + size_type i = 0; + for (; i < size() && first != last; ++i, static_cast<void>(++first)) { + data()[i] = *first; + } + + erase(data() + i, data() + size()); + std::copy(first, last, std::back_inserter(*this)); + } + + // `InlinedVector::resize(...)` + // + // Resizes the inlined vector to contain `n` elements. + // + // NOTE: If `n` is smaller than `size()`, extra elements are destroyed. If `n` + // is larger than `size()`, new elements are value-initialized. + void resize(size_type n) { + ABSL_HARDENING_ASSERT(n <= max_size()); + storage_.Resize(DefaultValueAdapter(), n); + } + + // Overload of `InlinedVector::resize(...)` that resizes the inlined vector to + // contain `n` elements. + // + // NOTE: if `n` is smaller than `size()`, extra elements are destroyed. If `n` + // is larger than `size()`, new elements are copied-constructed from `v`. + void resize(size_type n, const_reference v) { + ABSL_HARDENING_ASSERT(n <= max_size()); + storage_.Resize(CopyValueAdapter(v), n); + } + + // `InlinedVector::insert(...)` + // + // Inserts a copy of `v` at `pos`, returning an `iterator` to the newly + // inserted element. + iterator insert(const_iterator pos, const_reference v) { + return emplace(pos, v); + } + + // Overload of `InlinedVector::insert(...)` that inserts `v` at `pos` using + // move semantics, returning an `iterator` to the newly inserted element. + iterator insert(const_iterator pos, RValueReference v) { + return emplace(pos, std::move(v)); + } + + // Overload of `InlinedVector::insert(...)` that inserts `n` contiguous copies + // of `v` starting at `pos`, returning an `iterator` pointing to the first of + // the newly inserted elements. + iterator insert(const_iterator pos, size_type n, const_reference v) { + ABSL_HARDENING_ASSERT(pos >= begin()); + ABSL_HARDENING_ASSERT(pos <= end()); + + if (ABSL_PREDICT_TRUE(n != 0)) { + value_type dealias = v; + return storage_.Insert(pos, CopyValueAdapter(dealias), n); + } else { + return const_cast<iterator>(pos); + } + } + + // Overload of `InlinedVector::insert(...)` that inserts copies of the + // elements of `list` starting at `pos`, returning an `iterator` pointing to + // the first of the newly inserted elements. + iterator insert(const_iterator pos, std::initializer_list<value_type> list) { + return insert(pos, list.begin(), list.end()); + } + + // Overload of `InlinedVector::insert(...)` that inserts the range [`first`, + // `last`) starting at `pos`, returning an `iterator` pointing to the first + // of the newly inserted elements. + // + // NOTE: this overload is for iterators that are "forward" category or better. + template <typename ForwardIterator, + EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr> + iterator insert(const_iterator pos, ForwardIterator first, + ForwardIterator last) { + ABSL_HARDENING_ASSERT(pos >= begin()); + ABSL_HARDENING_ASSERT(pos <= end()); + + if (ABSL_PREDICT_TRUE(first != last)) { + return storage_.Insert(pos, IteratorValueAdapter<ForwardIterator>(first), + std::distance(first, last)); + } else { + return const_cast<iterator>(pos); + } + } + + // Overload of `InlinedVector::insert(...)` that inserts the range [`first`, + // `last`) starting at `pos`, returning an `iterator` pointing to the first + // of the newly inserted elements. + // + // NOTE: this overload is for iterators that are "input" category. + template <typename InputIterator, + DisableIfAtLeastForwardIterator<InputIterator>* = nullptr> + iterator insert(const_iterator pos, InputIterator first, InputIterator last) { + ABSL_HARDENING_ASSERT(pos >= begin()); + ABSL_HARDENING_ASSERT(pos <= end()); + + size_type index = std::distance(cbegin(), pos); + for (size_type i = index; first != last; ++i, static_cast<void>(++first)) { + insert(data() + i, *first); + } + + return iterator(data() + index); + } + + // `InlinedVector::emplace(...)` + // + // Constructs and inserts an element using `args...` in the inlined vector at + // `pos`, returning an `iterator` pointing to the newly emplaced element. + template <typename... Args> + iterator emplace(const_iterator pos, Args&&... args) { + ABSL_HARDENING_ASSERT(pos >= begin()); + ABSL_HARDENING_ASSERT(pos <= end()); + + value_type dealias(std::forward<Args>(args)...); + return storage_.Insert(pos, + IteratorValueAdapter<MoveIterator>( + MoveIterator(std::addressof(dealias))), + 1); + } + + // `InlinedVector::emplace_back(...)` + // + // Constructs and inserts an element using `args...` in the inlined vector at + // `end()`, returning a `reference` to the newly emplaced element. + template <typename... Args> + reference emplace_back(Args&&... args) { + return storage_.EmplaceBack(std::forward<Args>(args)...); + } + + // `InlinedVector::push_back(...)` + // + // Inserts a copy of `v` in the inlined vector at `end()`. + void push_back(const_reference v) { static_cast<void>(emplace_back(v)); } + + // Overload of `InlinedVector::push_back(...)` for inserting `v` at `end()` + // using move semantics. + void push_back(RValueReference v) { + static_cast<void>(emplace_back(std::move(v))); + } + + // `InlinedVector::pop_back()` + // + // Destroys the element at `back()`, reducing the size by `1`. + void pop_back() noexcept { + ABSL_HARDENING_ASSERT(!empty()); + + AllocatorTraits::destroy(*storage_.GetAllocPtr(), data() + (size() - 1)); + storage_.SubtractSize(1); + } + + // `InlinedVector::erase(...)` + // + // Erases the element at `pos`, returning an `iterator` pointing to where the + // erased element was located. + // + // NOTE: may return `end()`, which is not dereferencable. + iterator erase(const_iterator pos) { + ABSL_HARDENING_ASSERT(pos >= begin()); + ABSL_HARDENING_ASSERT(pos < end()); + + return storage_.Erase(pos, pos + 1); + } + + // Overload of `InlinedVector::erase(...)` that erases every element in the + // range [`from`, `to`), returning an `iterator` pointing to where the first + // erased element was located. + // + // NOTE: may return `end()`, which is not dereferencable. + iterator erase(const_iterator from, const_iterator to) { + ABSL_HARDENING_ASSERT(from >= begin()); + ABSL_HARDENING_ASSERT(from <= to); + ABSL_HARDENING_ASSERT(to <= end()); + + if (ABSL_PREDICT_TRUE(from != to)) { + return storage_.Erase(from, to); + } else { + return const_cast<iterator>(from); + } + } + + // `InlinedVector::clear()` + // + // Destroys all elements in the inlined vector, setting the size to `0` and + // deallocating any held memory. + void clear() noexcept { + inlined_vector_internal::DestroyElements(storage_.GetAllocPtr(), data(), + size()); + storage_.DeallocateIfAllocated(); + + storage_.SetInlinedSize(0); + } + + // `InlinedVector::reserve(...)` + // + // Ensures that there is enough room for at least `n` elements. + void reserve(size_type n) { storage_.Reserve(n); } + + // `InlinedVector::shrink_to_fit()` + // + // Reduces memory usage by freeing unused memory. After being called, calls to + // `capacity()` will be equal to `max(N, size())`. + // + // If `size() <= N` and the inlined vector contains allocated memory, the + // elements will all be moved to the inlined space and the allocated memory + // will be deallocated. + // + // If `size() > N` and `size() < capacity()`, the elements will be moved to a + // smaller allocation. + void shrink_to_fit() { + if (storage_.GetIsAllocated()) { + storage_.ShrinkToFit(); + } + } + + // `InlinedVector::swap(...)` + // + // Swaps the contents of the inlined vector with `other`. + void swap(InlinedVector& other) { + if (ABSL_PREDICT_TRUE(this != std::addressof(other))) { + storage_.Swap(std::addressof(other.storage_)); + } + } + + private: + template <typename H, typename TheT, size_t TheN, typename TheA> + friend H AbslHashValue(H h, const absl::InlinedVector<TheT, TheN, TheA>& a); + + Storage storage_; +}; + +// ----------------------------------------------------------------------------- +// InlinedVector Non-Member Functions +// ----------------------------------------------------------------------------- + +// `swap(...)` +// +// Swaps the contents of two inlined vectors. +template <typename T, size_t N, typename A> +void swap(absl::InlinedVector<T, N, A>& a, + absl::InlinedVector<T, N, A>& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); +} + +// `operator==(...)` +// +// Tests for value-equality of two inlined vectors. +template <typename T, size_t N, typename A> +bool operator==(const absl::InlinedVector<T, N, A>& a, + const absl::InlinedVector<T, N, A>& b) { + auto a_data = a.data(); + auto b_data = b.data(); + return absl::equal(a_data, a_data + a.size(), b_data, b_data + b.size()); +} + +// `operator!=(...)` +// +// Tests for value-inequality of two inlined vectors. +template <typename T, size_t N, typename A> +bool operator!=(const absl::InlinedVector<T, N, A>& a, + const absl::InlinedVector<T, N, A>& b) { + return !(a == b); +} + +// `operator<(...)` +// +// Tests whether the value of an inlined vector is less than the value of +// another inlined vector using a lexicographical comparison algorithm. +template <typename T, size_t N, typename A> +bool operator<(const absl::InlinedVector<T, N, A>& a, + const absl::InlinedVector<T, N, A>& b) { + auto a_data = a.data(); + auto b_data = b.data(); + return std::lexicographical_compare(a_data, a_data + a.size(), b_data, + b_data + b.size()); +} + +// `operator>(...)` +// +// Tests whether the value of an inlined vector is greater than the value of +// another inlined vector using a lexicographical comparison algorithm. +template <typename T, size_t N, typename A> +bool operator>(const absl::InlinedVector<T, N, A>& a, + const absl::InlinedVector<T, N, A>& b) { + return b < a; +} + +// `operator<=(...)` +// +// Tests whether the value of an inlined vector is less than or equal to the +// value of another inlined vector using a lexicographical comparison algorithm. +template <typename T, size_t N, typename A> +bool operator<=(const absl::InlinedVector<T, N, A>& a, + const absl::InlinedVector<T, N, A>& b) { + return !(b < a); +} + +// `operator>=(...)` +// +// Tests whether the value of an inlined vector is greater than or equal to the +// value of another inlined vector using a lexicographical comparison algorithm. +template <typename T, size_t N, typename A> +bool operator>=(const absl::InlinedVector<T, N, A>& a, + const absl::InlinedVector<T, N, A>& b) { + return !(a < b); +} + +// `AbslHashValue(...)` +// +// Provides `absl::Hash` support for `absl::InlinedVector`. It is uncommon to +// call this directly. +template <typename H, typename T, size_t N, typename A> +H AbslHashValue(H h, const absl::InlinedVector<T, N, A>& a) { + auto size = a.size(); + return H::combine(H::combine_contiguous(std::move(h), a.data(), size), size); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INLINED_VECTOR_H_ diff --git a/third_party/abseil_cpp/absl/container/inlined_vector_benchmark.cc b/third_party/abseil_cpp/absl/container/inlined_vector_benchmark.cc new file mode 100644 index 000000000000..b8dafe932320 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/inlined_vector_benchmark.cc @@ -0,0 +1,807 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <array> +#include <string> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/container/inlined_vector.h" +#include "absl/strings/str_cat.h" + +namespace { + +void BM_InlinedVectorFill(benchmark::State& state) { + const int len = state.range(0); + absl::InlinedVector<int, 8> v; + v.reserve(len); + for (auto _ : state) { + v.resize(0); // Use resize(0) as InlinedVector releases storage on clear(). + for (int i = 0; i < len; ++i) { + v.push_back(i); + } + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_InlinedVectorFill)->Range(1, 256); + +void BM_InlinedVectorFillRange(benchmark::State& state) { + const int len = state.range(0); + const std::vector<int> src(len, len); + absl::InlinedVector<int, 8> v; + v.reserve(len); + for (auto _ : state) { + benchmark::DoNotOptimize(src); + v.assign(src.begin(), src.end()); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_InlinedVectorFillRange)->Range(1, 256); + +void BM_StdVectorFill(benchmark::State& state) { + const int len = state.range(0); + std::vector<int> v; + v.reserve(len); + for (auto _ : state) { + v.clear(); + for (int i = 0; i < len; ++i) { + v.push_back(i); + } + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_StdVectorFill)->Range(1, 256); + +// The purpose of the next two benchmarks is to verify that +// absl::InlinedVector is efficient when moving is more efficent than +// copying. To do so, we use strings that are larger than the short +// string optimization. +bool StringRepresentedInline(std::string s) { + const char* chars = s.data(); + std::string s1 = std::move(s); + return s1.data() != chars; +} + +int GetNonShortStringOptimizationSize() { + for (int i = 24; i <= 192; i *= 2) { + if (!StringRepresentedInline(std::string(i, 'A'))) { + return i; + } + } + ABSL_RAW_LOG( + FATAL, + "Failed to find a string larger than the short string optimization"); + return -1; +} + +void BM_InlinedVectorFillString(benchmark::State& state) { + const int len = state.range(0); + const int no_sso = GetNonShortStringOptimizationSize(); + std::string strings[4] = {std::string(no_sso, 'A'), std::string(no_sso, 'B'), + std::string(no_sso, 'C'), std::string(no_sso, 'D')}; + + for (auto _ : state) { + absl::InlinedVector<std::string, 8> v; + for (int i = 0; i < len; i++) { + v.push_back(strings[i & 3]); + } + } + state.SetItemsProcessed(static_cast<int64_t>(state.iterations()) * len); +} +BENCHMARK(BM_InlinedVectorFillString)->Range(0, 1024); + +void BM_StdVectorFillString(benchmark::State& state) { + const int len = state.range(0); + const int no_sso = GetNonShortStringOptimizationSize(); + std::string strings[4] = {std::string(no_sso, 'A'), std::string(no_sso, 'B'), + std::string(no_sso, 'C'), std::string(no_sso, 'D')}; + + for (auto _ : state) { + std::vector<std::string> v; + for (int i = 0; i < len; i++) { + v.push_back(strings[i & 3]); + } + } + state.SetItemsProcessed(static_cast<int64_t>(state.iterations()) * len); +} +BENCHMARK(BM_StdVectorFillString)->Range(0, 1024); + +struct Buffer { // some arbitrary structure for benchmarking. + char* base; + int length; + int capacity; + void* user_data; +}; + +void BM_InlinedVectorAssignments(benchmark::State& state) { + const int len = state.range(0); + using BufferVec = absl::InlinedVector<Buffer, 2>; + + BufferVec src; + src.resize(len); + + BufferVec dst; + for (auto _ : state) { + benchmark::DoNotOptimize(dst); + benchmark::DoNotOptimize(src); + dst = src; + } +} +BENCHMARK(BM_InlinedVectorAssignments) + ->Arg(0) + ->Arg(1) + ->Arg(2) + ->Arg(3) + ->Arg(4) + ->Arg(20); + +void BM_CreateFromContainer(benchmark::State& state) { + for (auto _ : state) { + absl::InlinedVector<int, 4> src{1, 2, 3}; + benchmark::DoNotOptimize(src); + absl::InlinedVector<int, 4> dst(std::move(src)); + benchmark::DoNotOptimize(dst); + } +} +BENCHMARK(BM_CreateFromContainer); + +struct LargeCopyableOnly { + LargeCopyableOnly() : d(1024, 17) {} + LargeCopyableOnly(const LargeCopyableOnly& o) = default; + LargeCopyableOnly& operator=(const LargeCopyableOnly& o) = default; + + std::vector<int> d; +}; + +struct LargeCopyableSwappable { + LargeCopyableSwappable() : d(1024, 17) {} + + LargeCopyableSwappable(const LargeCopyableSwappable& o) = default; + + LargeCopyableSwappable& operator=(LargeCopyableSwappable o) { + using std::swap; + swap(*this, o); + return *this; + } + + friend void swap(LargeCopyableSwappable& a, LargeCopyableSwappable& b) { + using std::swap; + swap(a.d, b.d); + } + + std::vector<int> d; +}; + +struct LargeCopyableMovable { + LargeCopyableMovable() : d(1024, 17) {} + // Use implicitly defined copy and move. + + std::vector<int> d; +}; + +struct LargeCopyableMovableSwappable { + LargeCopyableMovableSwappable() : d(1024, 17) {} + LargeCopyableMovableSwappable(const LargeCopyableMovableSwappable& o) = + default; + LargeCopyableMovableSwappable(LargeCopyableMovableSwappable&& o) = default; + + LargeCopyableMovableSwappable& operator=(LargeCopyableMovableSwappable o) { + using std::swap; + swap(*this, o); + return *this; + } + LargeCopyableMovableSwappable& operator=(LargeCopyableMovableSwappable&& o) = + default; + + friend void swap(LargeCopyableMovableSwappable& a, + LargeCopyableMovableSwappable& b) { + using std::swap; + swap(a.d, b.d); + } + + std::vector<int> d; +}; + +template <typename ElementType> +void BM_SwapElements(benchmark::State& state) { + const int len = state.range(0); + using Vec = absl::InlinedVector<ElementType, 32>; + Vec a(len); + Vec b; + for (auto _ : state) { + using std::swap; + benchmark::DoNotOptimize(a); + benchmark::DoNotOptimize(b); + swap(a, b); + } +} +BENCHMARK_TEMPLATE(BM_SwapElements, LargeCopyableOnly)->Range(0, 1024); +BENCHMARK_TEMPLATE(BM_SwapElements, LargeCopyableSwappable)->Range(0, 1024); +BENCHMARK_TEMPLATE(BM_SwapElements, LargeCopyableMovable)->Range(0, 1024); +BENCHMARK_TEMPLATE(BM_SwapElements, LargeCopyableMovableSwappable) + ->Range(0, 1024); + +// The following benchmark is meant to track the efficiency of the vector size +// as a function of stored type via the benchmark label. It is not meant to +// output useful sizeof operator performance. The loop is a dummy operation +// to fulfill the requirement of running the benchmark. +template <typename VecType> +void BM_Sizeof(benchmark::State& state) { + int size = 0; + for (auto _ : state) { + VecType vec; + size = sizeof(vec); + } + state.SetLabel(absl::StrCat("sz=", size)); +} +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<char, 1>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<char, 4>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<char, 7>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<char, 8>); + +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<int, 1>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<int, 4>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<int, 7>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<int, 8>); + +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<void*, 1>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<void*, 4>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<void*, 7>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<void*, 8>); + +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<std::string, 1>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<std::string, 4>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<std::string, 7>); +BENCHMARK_TEMPLATE(BM_Sizeof, absl::InlinedVector<std::string, 8>); + +void BM_InlinedVectorIndexInlined(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v[4]); + } +} +BENCHMARK(BM_InlinedVectorIndexInlined); + +void BM_InlinedVectorIndexExternal(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v[4]); + } +} +BENCHMARK(BM_InlinedVectorIndexExternal); + +void BM_StdVectorIndex(benchmark::State& state) { + std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v[4]); + } +} +BENCHMARK(BM_StdVectorIndex); + +void BM_InlinedVectorDataInlined(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.data()); + } +} +BENCHMARK(BM_InlinedVectorDataInlined); + +void BM_InlinedVectorDataExternal(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.data()); + } + state.SetItemsProcessed(16 * static_cast<int64_t>(state.iterations())); +} +BENCHMARK(BM_InlinedVectorDataExternal); + +void BM_StdVectorData(benchmark::State& state) { + std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.data()); + } + state.SetItemsProcessed(16 * static_cast<int64_t>(state.iterations())); +} +BENCHMARK(BM_StdVectorData); + +void BM_InlinedVectorSizeInlined(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.size()); + } +} +BENCHMARK(BM_InlinedVectorSizeInlined); + +void BM_InlinedVectorSizeExternal(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.size()); + } +} +BENCHMARK(BM_InlinedVectorSizeExternal); + +void BM_StdVectorSize(benchmark::State& state) { + std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.size()); + } +} +BENCHMARK(BM_StdVectorSize); + +void BM_InlinedVectorEmptyInlined(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.empty()); + } +} +BENCHMARK(BM_InlinedVectorEmptyInlined); + +void BM_InlinedVectorEmptyExternal(benchmark::State& state) { + absl::InlinedVector<int, 8> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.empty()); + } +} +BENCHMARK(BM_InlinedVectorEmptyExternal); + +void BM_StdVectorEmpty(benchmark::State& state) { + std::vector<int> v = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + for (auto _ : state) { + benchmark::DoNotOptimize(v); + benchmark::DoNotOptimize(v.empty()); + } +} +BENCHMARK(BM_StdVectorEmpty); + +constexpr size_t kInlinedCapacity = 4; +constexpr size_t kLargeSize = kInlinedCapacity * 2; +constexpr size_t kSmallSize = kInlinedCapacity / 2; +constexpr size_t kBatchSize = 100; + +#define ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_FunctionTemplate, T) \ + BENCHMARK_TEMPLATE(BM_FunctionTemplate, T, kLargeSize); \ + BENCHMARK_TEMPLATE(BM_FunctionTemplate, T, kSmallSize) + +#define ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_FunctionTemplate, T) \ + BENCHMARK_TEMPLATE(BM_FunctionTemplate, T, kLargeSize, kLargeSize); \ + BENCHMARK_TEMPLATE(BM_FunctionTemplate, T, kLargeSize, kSmallSize); \ + BENCHMARK_TEMPLATE(BM_FunctionTemplate, T, kSmallSize, kLargeSize); \ + BENCHMARK_TEMPLATE(BM_FunctionTemplate, T, kSmallSize, kSmallSize) + +template <typename T> +using InlVec = absl::InlinedVector<T, kInlinedCapacity>; + +struct TrivialType { + size_t val; +}; + +class NontrivialType { + public: + ABSL_ATTRIBUTE_NOINLINE NontrivialType() : val_() { + benchmark::DoNotOptimize(*this); + } + + ABSL_ATTRIBUTE_NOINLINE NontrivialType(const NontrivialType& other) + : val_(other.val_) { + benchmark::DoNotOptimize(*this); + } + + ABSL_ATTRIBUTE_NOINLINE NontrivialType& operator=( + const NontrivialType& other) { + val_ = other.val_; + benchmark::DoNotOptimize(*this); + return *this; + } + + ABSL_ATTRIBUTE_NOINLINE ~NontrivialType() noexcept { + benchmark::DoNotOptimize(*this); + } + + private: + size_t val_; +}; + +template <typename T, typename PrepareVecFn, typename TestVecFn> +void BatchedBenchmark(benchmark::State& state, PrepareVecFn prepare_vec, + TestVecFn test_vec) { + std::array<InlVec<T>, kBatchSize> vector_batch{}; + + while (state.KeepRunningBatch(kBatchSize)) { + // Prepare batch + state.PauseTiming(); + for (size_t i = 0; i < kBatchSize; ++i) { + prepare_vec(vector_batch.data() + i, i); + } + benchmark::DoNotOptimize(vector_batch); + state.ResumeTiming(); + + // Test batch + for (size_t i = 0; i < kBatchSize; ++i) { + test_vec(vector_batch.data() + i, i); + } + } +} + +template <typename T, size_t ToSize> +void BM_ConstructFromSize(benchmark::State& state) { + using VecT = InlVec<T>; + auto size = ToSize; + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->~VecT(); }, + /* test_vec = */ + [&](void* ptr, size_t) { + benchmark::DoNotOptimize(size); + ::new (ptr) VecT(size); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromSize, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromSize, NontrivialType); + +template <typename T, size_t ToSize> +void BM_ConstructFromSizeRef(benchmark::State& state) { + using VecT = InlVec<T>; + auto size = ToSize; + auto ref = T(); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->~VecT(); }, + /* test_vec = */ + [&](void* ptr, size_t) { + benchmark::DoNotOptimize(size); + benchmark::DoNotOptimize(ref); + ::new (ptr) VecT(size, ref); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromSizeRef, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromSizeRef, NontrivialType); + +template <typename T, size_t ToSize> +void BM_ConstructFromRange(benchmark::State& state) { + using VecT = InlVec<T>; + std::array<T, ToSize> arr{}; + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->~VecT(); }, + /* test_vec = */ + [&](void* ptr, size_t) { + benchmark::DoNotOptimize(arr); + ::new (ptr) VecT(arr.begin(), arr.end()); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromRange, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromRange, NontrivialType); + +template <typename T, size_t ToSize> +void BM_ConstructFromCopy(benchmark::State& state) { + using VecT = InlVec<T>; + VecT other_vec(ToSize); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { vec->~VecT(); }, + /* test_vec = */ + [&](void* ptr, size_t) { + benchmark::DoNotOptimize(other_vec); + ::new (ptr) VecT(other_vec); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromCopy, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromCopy, NontrivialType); + +template <typename T, size_t ToSize> +void BM_ConstructFromMove(benchmark::State& state) { + using VecT = InlVec<T>; + std::array<VecT, kBatchSize> vector_batch{}; + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [&](InlVec<T>* vec, size_t i) { + vector_batch[i].clear(); + vector_batch[i].resize(ToSize); + vec->~VecT(); + }, + /* test_vec = */ + [&](void* ptr, size_t i) { + benchmark::DoNotOptimize(vector_batch[i]); + ::new (ptr) VecT(std::move(vector_batch[i])); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromMove, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_ConstructFromMove, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_AssignSizeRef(benchmark::State& state) { + auto size = ToSize; + auto ref = T(); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->resize(FromSize); }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t) { + benchmark::DoNotOptimize(size); + benchmark::DoNotOptimize(ref); + vec->assign(size, ref); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignSizeRef, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignSizeRef, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_AssignRange(benchmark::State& state) { + std::array<T, ToSize> arr{}; + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->resize(FromSize); }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t) { + benchmark::DoNotOptimize(arr); + vec->assign(arr.begin(), arr.end()); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignRange, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignRange, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_AssignFromCopy(benchmark::State& state) { + InlVec<T> other_vec(ToSize); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->resize(FromSize); }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t) { + benchmark::DoNotOptimize(other_vec); + *vec = other_vec; + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignFromCopy, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignFromCopy, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_AssignFromMove(benchmark::State& state) { + using VecT = InlVec<T>; + std::array<VecT, kBatchSize> vector_batch{}; + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [&](InlVec<T>* vec, size_t i) { + vector_batch[i].clear(); + vector_batch[i].resize(ToSize); + vec->resize(FromSize); + }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t i) { + benchmark::DoNotOptimize(vector_batch[i]); + *vec = std::move(vector_batch[i]); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignFromMove, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_AssignFromMove, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_ResizeSize(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [](InlVec<T>* vec, size_t) { vec->resize(ToSize); }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_ResizeSize, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_ResizeSize, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_ResizeSizeRef(benchmark::State& state) { + auto t = T(); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t) { + benchmark::DoNotOptimize(t); + vec->resize(ToSize, t); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_ResizeSizeRef, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_ResizeSizeRef, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_InsertSizeRef(benchmark::State& state) { + auto t = T(); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t) { + benchmark::DoNotOptimize(t); + auto* pos = vec->data() + (vec->size() / 2); + vec->insert(pos, t); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_InsertSizeRef, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_InsertSizeRef, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_InsertRange(benchmark::State& state) { + InlVec<T> other_vec(ToSize); + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t) { + benchmark::DoNotOptimize(other_vec); + auto* pos = vec->data() + (vec->size() / 2); + vec->insert(pos, other_vec.begin(), other_vec.end()); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_InsertRange, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_InsertRange, NontrivialType); + +template <typename T, size_t FromSize> +void BM_EmplaceBack(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [](InlVec<T>* vec, size_t) { vec->emplace_back(); }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_EmplaceBack, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_EmplaceBack, NontrivialType); + +template <typename T, size_t FromSize> +void BM_PopBack(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [](InlVec<T>* vec, size_t) { vec->pop_back(); }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_PopBack, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_PopBack, NontrivialType); + +template <typename T, size_t FromSize> +void BM_EraseOne(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [](InlVec<T>* vec, size_t) { + auto* pos = vec->data() + (vec->size() / 2); + vec->erase(pos); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_EraseOne, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_EraseOne, NontrivialType); + +template <typename T, size_t FromSize> +void BM_EraseRange(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [](InlVec<T>* vec, size_t) { + auto* pos = vec->data() + (vec->size() / 2); + vec->erase(pos, pos + 1); + }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_EraseRange, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_EraseRange, NontrivialType); + +template <typename T, size_t FromSize> +void BM_Clear(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ [](InlVec<T>* vec, size_t) { vec->resize(FromSize); }, + /* test_vec = */ [](InlVec<T>* vec, size_t) { vec->clear(); }); +} +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_Clear, TrivialType); +ABSL_INTERNAL_BENCHMARK_ONE_SIZE(BM_Clear, NontrivialType); + +template <typename T, size_t FromSize, size_t ToCapacity> +void BM_Reserve(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(FromSize); + }, + /* test_vec = */ + [](InlVec<T>* vec, size_t) { vec->reserve(ToCapacity); }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_Reserve, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_Reserve, NontrivialType); + +template <typename T, size_t FromCapacity, size_t ToCapacity> +void BM_ShrinkToFit(benchmark::State& state) { + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [](InlVec<T>* vec, size_t) { + vec->clear(); + vec->resize(ToCapacity); + vec->reserve(FromCapacity); + }, + /* test_vec = */ [](InlVec<T>* vec, size_t) { vec->shrink_to_fit(); }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_ShrinkToFit, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_ShrinkToFit, NontrivialType); + +template <typename T, size_t FromSize, size_t ToSize> +void BM_Swap(benchmark::State& state) { + using VecT = InlVec<T>; + std::array<VecT, kBatchSize> vector_batch{}; + BatchedBenchmark<T>( + state, + /* prepare_vec = */ + [&](InlVec<T>* vec, size_t i) { + vector_batch[i].clear(); + vector_batch[i].resize(ToSize); + vec->resize(FromSize); + }, + /* test_vec = */ + [&](InlVec<T>* vec, size_t i) { + using std::swap; + benchmark::DoNotOptimize(vector_batch[i]); + swap(*vec, vector_batch[i]); + }); +} +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_Swap, TrivialType); +ABSL_INTERNAL_BENCHMARK_TWO_SIZE(BM_Swap, NontrivialType); + +} // namespace diff --git a/third_party/abseil_cpp/absl/container/inlined_vector_exception_safety_test.cc b/third_party/abseil_cpp/absl/container/inlined_vector_exception_safety_test.cc new file mode 100644 index 000000000000..0e6a05b5f6a7 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/inlined_vector_exception_safety_test.cc @@ -0,0 +1,508 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/inlined_vector.h" + +#include "absl/base/config.h" + +#if defined(ABSL_HAVE_EXCEPTIONS) + +#include <array> +#include <initializer_list> +#include <iterator> +#include <memory> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/base/internal/exception_safety_testing.h" + +namespace { + +constexpr size_t kInlinedCapacity = 4; +constexpr size_t kLargeSize = kInlinedCapacity * 2; +constexpr size_t kSmallSize = kInlinedCapacity / 2; + +using Thrower = testing::ThrowingValue<>; +using MovableThrower = testing::ThrowingValue<testing::TypeSpec::kNoThrowMove>; +using ThrowAlloc = testing::ThrowingAllocator<Thrower>; + +using ThrowerVec = absl::InlinedVector<Thrower, kInlinedCapacity>; +using MovableThrowerVec = absl::InlinedVector<MovableThrower, kInlinedCapacity>; + +using ThrowAllocThrowerVec = + absl::InlinedVector<Thrower, kInlinedCapacity, ThrowAlloc>; +using ThrowAllocMovableThrowerVec = + absl::InlinedVector<MovableThrower, kInlinedCapacity, ThrowAlloc>; + +// In GCC, if an element of a `std::initializer_list` throws during construction +// the elements that were constructed before it are not destroyed. This causes +// incorrect exception safety test failures. Thus, `testing::nothrow_ctor` is +// required. See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66139 +#define ABSL_INTERNAL_MAKE_INIT_LIST(T, N) \ + (N > kInlinedCapacity \ + ? std::initializer_list<T>{T(0, testing::nothrow_ctor), \ + T(1, testing::nothrow_ctor), \ + T(2, testing::nothrow_ctor), \ + T(3, testing::nothrow_ctor), \ + T(4, testing::nothrow_ctor), \ + T(5, testing::nothrow_ctor), \ + T(6, testing::nothrow_ctor), \ + T(7, testing::nothrow_ctor)} \ + \ + : std::initializer_list<T>{T(0, testing::nothrow_ctor), \ + T(1, testing::nothrow_ctor)}) +static_assert(kLargeSize == 8, "Must update ABSL_INTERNAL_MAKE_INIT_LIST(...)"); +static_assert(kSmallSize == 2, "Must update ABSL_INTERNAL_MAKE_INIT_LIST(...)"); + +template <typename TheVecT, size_t... TheSizes> +class TestParams { + public: + using VecT = TheVecT; + constexpr static size_t GetSizeAt(size_t i) { return kSizes[1 + i]; } + + private: + constexpr static size_t kSizes[1 + sizeof...(TheSizes)] = {1, TheSizes...}; +}; + +using NoSizeTestParams = + ::testing::Types<TestParams<ThrowerVec>, TestParams<MovableThrowerVec>, + TestParams<ThrowAllocThrowerVec>, + TestParams<ThrowAllocMovableThrowerVec>>; + +using OneSizeTestParams = + ::testing::Types<TestParams<ThrowerVec, kLargeSize>, + TestParams<ThrowerVec, kSmallSize>, + TestParams<MovableThrowerVec, kLargeSize>, + TestParams<MovableThrowerVec, kSmallSize>, + TestParams<ThrowAllocThrowerVec, kLargeSize>, + TestParams<ThrowAllocThrowerVec, kSmallSize>, + TestParams<ThrowAllocMovableThrowerVec, kLargeSize>, + TestParams<ThrowAllocMovableThrowerVec, kSmallSize>>; + +using TwoSizeTestParams = ::testing::Types< + TestParams<ThrowerVec, kLargeSize, kLargeSize>, + TestParams<ThrowerVec, kLargeSize, kSmallSize>, + TestParams<ThrowerVec, kSmallSize, kLargeSize>, + TestParams<ThrowerVec, kSmallSize, kSmallSize>, + TestParams<MovableThrowerVec, kLargeSize, kLargeSize>, + TestParams<MovableThrowerVec, kLargeSize, kSmallSize>, + TestParams<MovableThrowerVec, kSmallSize, kLargeSize>, + TestParams<MovableThrowerVec, kSmallSize, kSmallSize>, + TestParams<ThrowAllocThrowerVec, kLargeSize, kLargeSize>, + TestParams<ThrowAllocThrowerVec, kLargeSize, kSmallSize>, + TestParams<ThrowAllocThrowerVec, kSmallSize, kLargeSize>, + TestParams<ThrowAllocThrowerVec, kSmallSize, kSmallSize>, + TestParams<ThrowAllocMovableThrowerVec, kLargeSize, kLargeSize>, + TestParams<ThrowAllocMovableThrowerVec, kLargeSize, kSmallSize>, + TestParams<ThrowAllocMovableThrowerVec, kSmallSize, kLargeSize>, + TestParams<ThrowAllocMovableThrowerVec, kSmallSize, kSmallSize>>; + +template <typename> +struct NoSizeTest : ::testing::Test {}; +TYPED_TEST_SUITE(NoSizeTest, NoSizeTestParams); + +template <typename> +struct OneSizeTest : ::testing::Test {}; +TYPED_TEST_SUITE(OneSizeTest, OneSizeTestParams); + +template <typename> +struct TwoSizeTest : ::testing::Test {}; +TYPED_TEST_SUITE(TwoSizeTest, TwoSizeTestParams); + +template <typename VecT> +bool InlinedVectorInvariants(VecT* vec) { + if (*vec != *vec) return false; + if (vec->size() > vec->capacity()) return false; + if (vec->size() > vec->max_size()) return false; + if (vec->capacity() > vec->max_size()) return false; + if (vec->data() != std::addressof(vec->at(0))) return false; + if (vec->data() != vec->begin()) return false; + if (*vec->data() != *vec->begin()) return false; + if (vec->begin() > vec->end()) return false; + if ((vec->end() - vec->begin()) != vec->size()) return false; + if (std::distance(vec->begin(), vec->end()) != vec->size()) return false; + return true; +} + +// Function that always returns false is correct, but refactoring is required +// for clarity. It's needed to express that, as a contract, certain operations +// should not throw at all. Execution of this function means an exception was +// thrown and thus the test should fail. +// TODO(johnsoncj): Add `testing::NoThrowGuarantee` to the framework +template <typename VecT> +bool NoThrowGuarantee(VecT* /* vec */) { + return false; +} + +TYPED_TEST(NoSizeTest, DefaultConstructor) { + using VecT = typename TypeParam::VecT; + using allocator_type = typename VecT::allocator_type; + + testing::TestThrowingCtor<VecT>(); + + testing::TestThrowingCtor<VecT>(allocator_type{}); +} + +TYPED_TEST(OneSizeTest, SizeConstructor) { + using VecT = typename TypeParam::VecT; + using allocator_type = typename VecT::allocator_type; + constexpr static auto size = TypeParam::GetSizeAt(0); + + testing::TestThrowingCtor<VecT>(size); + + testing::TestThrowingCtor<VecT>(size, allocator_type{}); +} + +TYPED_TEST(OneSizeTest, SizeRefConstructor) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + using allocator_type = typename VecT::allocator_type; + constexpr static auto size = TypeParam::GetSizeAt(0); + + testing::TestThrowingCtor<VecT>(size, value_type{}); + + testing::TestThrowingCtor<VecT>(size, value_type{}, allocator_type{}); +} + +TYPED_TEST(OneSizeTest, InitializerListConstructor) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + using allocator_type = typename VecT::allocator_type; + constexpr static auto size = TypeParam::GetSizeAt(0); + + testing::TestThrowingCtor<VecT>( + ABSL_INTERNAL_MAKE_INIT_LIST(value_type, size)); + + testing::TestThrowingCtor<VecT>( + ABSL_INTERNAL_MAKE_INIT_LIST(value_type, size), allocator_type{}); +} + +TYPED_TEST(OneSizeTest, RangeConstructor) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + using allocator_type = typename VecT::allocator_type; + constexpr static auto size = TypeParam::GetSizeAt(0); + + std::array<value_type, size> arr{}; + + testing::TestThrowingCtor<VecT>(arr.begin(), arr.end()); + + testing::TestThrowingCtor<VecT>(arr.begin(), arr.end(), allocator_type{}); +} + +TYPED_TEST(OneSizeTest, CopyConstructor) { + using VecT = typename TypeParam::VecT; + using allocator_type = typename VecT::allocator_type; + constexpr static auto size = TypeParam::GetSizeAt(0); + + VecT other_vec{size}; + + testing::TestThrowingCtor<VecT>(other_vec); + + testing::TestThrowingCtor<VecT>(other_vec, allocator_type{}); +} + +TYPED_TEST(OneSizeTest, MoveConstructor) { + using VecT = typename TypeParam::VecT; + using allocator_type = typename VecT::allocator_type; + constexpr static auto size = TypeParam::GetSizeAt(0); + + if (!absl::allocator_is_nothrow<allocator_type>::value) { + testing::TestThrowingCtor<VecT>(VecT{size}); + + testing::TestThrowingCtor<VecT>(VecT{size}, allocator_type{}); + } +} + +TYPED_TEST(TwoSizeTest, Assign) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + constexpr static auto from_size = TypeParam::GetSizeAt(0); + constexpr static auto to_size = TypeParam::GetSizeAt(1); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{from_size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + *vec = ABSL_INTERNAL_MAKE_INIT_LIST(value_type, to_size); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + VecT other_vec{to_size}; + *vec = other_vec; + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + VecT other_vec{to_size}; + *vec = std::move(other_vec); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + value_type val{}; + vec->assign(to_size, val); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + vec->assign(ABSL_INTERNAL_MAKE_INIT_LIST(value_type, to_size)); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + std::array<value_type, to_size> arr{}; + vec->assign(arr.begin(), arr.end()); + })); +} + +TYPED_TEST(TwoSizeTest, Resize) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + constexpr static auto from_size = TypeParam::GetSizeAt(0); + constexpr static auto to_size = TypeParam::GetSizeAt(1); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{from_size}) + .WithContracts(InlinedVectorInvariants<VecT>, + testing::strong_guarantee); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + vec->resize(to_size); // + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + vec->resize(to_size, value_type{}); // + })); +} + +TYPED_TEST(OneSizeTest, Insert) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + constexpr static auto from_size = TypeParam::GetSizeAt(0); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{from_size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + vec->insert(it, value_type{}); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + vec->insert(it, value_type{}); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->end(); + vec->insert(it, value_type{}); + })); +} + +TYPED_TEST(TwoSizeTest, Insert) { + using VecT = typename TypeParam::VecT; + using value_type = typename VecT::value_type; + constexpr static auto from_size = TypeParam::GetSizeAt(0); + constexpr static auto count = TypeParam::GetSizeAt(1); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{from_size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + vec->insert(it, count, value_type{}); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + vec->insert(it, count, value_type{}); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->end(); + vec->insert(it, count, value_type{}); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + vec->insert(it, ABSL_INTERNAL_MAKE_INIT_LIST(value_type, count)); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + vec->insert(it, ABSL_INTERNAL_MAKE_INIT_LIST(value_type, count)); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->end(); + vec->insert(it, ABSL_INTERNAL_MAKE_INIT_LIST(value_type, count)); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + std::array<value_type, count> arr{}; + vec->insert(it, arr.begin(), arr.end()); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + std::array<value_type, count> arr{}; + vec->insert(it, arr.begin(), arr.end()); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->end(); + std::array<value_type, count> arr{}; + vec->insert(it, arr.begin(), arr.end()); + })); +} + +TYPED_TEST(OneSizeTest, EmplaceBack) { + using VecT = typename TypeParam::VecT; + constexpr static auto size = TypeParam::GetSizeAt(0); + + // For testing calls to `emplace_back(...)` that reallocate. + VecT full_vec{size}; + full_vec.resize(full_vec.capacity()); + + // For testing calls to `emplace_back(...)` that don't reallocate. + VecT nonfull_vec{size}; + nonfull_vec.reserve(size + 1); + + auto tester = testing::MakeExceptionSafetyTester().WithContracts( + InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.WithInitialValue(nonfull_vec).Test([](VecT* vec) { + vec->emplace_back(); + })); + + EXPECT_TRUE(tester.WithInitialValue(full_vec).Test( + [](VecT* vec) { vec->emplace_back(); })); +} + +TYPED_TEST(OneSizeTest, PopBack) { + using VecT = typename TypeParam::VecT; + constexpr static auto size = TypeParam::GetSizeAt(0); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{size}) + .WithContracts(NoThrowGuarantee<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + vec->pop_back(); // + })); +} + +TYPED_TEST(OneSizeTest, Erase) { + using VecT = typename TypeParam::VecT; + constexpr static auto size = TypeParam::GetSizeAt(0); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + vec->erase(it); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + vec->erase(it); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() - 1); + vec->erase(it); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + vec->erase(it, it); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + vec->erase(it, it); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() - 1); + vec->erase(it, it); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin(); + vec->erase(it, it + 1); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() / 2); + vec->erase(it, it + 1); + })); + EXPECT_TRUE(tester.Test([](VecT* vec) { + auto it = vec->begin() + (vec->size() - 1); + vec->erase(it, it + 1); + })); +} + +TYPED_TEST(OneSizeTest, Clear) { + using VecT = typename TypeParam::VecT; + constexpr static auto size = TypeParam::GetSizeAt(0); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{size}) + .WithContracts(NoThrowGuarantee<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + vec->clear(); // + })); +} + +TYPED_TEST(TwoSizeTest, Reserve) { + using VecT = typename TypeParam::VecT; + constexpr static auto from_size = TypeParam::GetSizeAt(0); + constexpr static auto to_capacity = TypeParam::GetSizeAt(1); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{from_size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { vec->reserve(to_capacity); })); +} + +TYPED_TEST(OneSizeTest, ShrinkToFit) { + using VecT = typename TypeParam::VecT; + constexpr static auto size = TypeParam::GetSizeAt(0); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + vec->shrink_to_fit(); // + })); +} + +TYPED_TEST(TwoSizeTest, Swap) { + using VecT = typename TypeParam::VecT; + constexpr static auto from_size = TypeParam::GetSizeAt(0); + constexpr static auto to_size = TypeParam::GetSizeAt(1); + + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(VecT{from_size}) + .WithContracts(InlinedVectorInvariants<VecT>); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + VecT other_vec{to_size}; + vec->swap(other_vec); + })); + + EXPECT_TRUE(tester.Test([](VecT* vec) { + using std::swap; + VecT other_vec{to_size}; + swap(*vec, other_vec); + })); +} + +} // namespace + +#endif // defined(ABSL_HAVE_EXCEPTIONS) diff --git a/third_party/abseil_cpp/absl/container/inlined_vector_test.cc b/third_party/abseil_cpp/absl/container/inlined_vector_test.cc new file mode 100644 index 000000000000..98aff33498b2 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/inlined_vector_test.cc @@ -0,0 +1,1815 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/inlined_vector.h" + +#include <algorithm> +#include <forward_list> +#include <list> +#include <memory> +#include <scoped_allocator> +#include <sstream> +#include <stdexcept> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/internal/exception_testing.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/base/options.h" +#include "absl/container/internal/counting_allocator.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/hash/hash_testing.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" + +namespace { + +using absl::container_internal::CountingAllocator; +using absl::test_internal::CopyableMovableInstance; +using absl::test_internal::CopyableOnlyInstance; +using absl::test_internal::InstanceTracker; +using testing::AllOf; +using testing::Each; +using testing::ElementsAre; +using testing::ElementsAreArray; +using testing::Eq; +using testing::Gt; +using testing::PrintToString; + +using IntVec = absl::InlinedVector<int, 8>; + +MATCHER_P(SizeIs, n, "") { + return testing::ExplainMatchResult(n, arg.size(), result_listener); +} + +MATCHER_P(CapacityIs, n, "") { + return testing::ExplainMatchResult(n, arg.capacity(), result_listener); +} + +MATCHER_P(ValueIs, e, "") { + return testing::ExplainMatchResult(e, arg.value(), result_listener); +} + +// TODO(bsamwel): Add support for movable-only types. + +// Test fixture for typed tests on BaseCountedInstance derived classes, see +// test_instance_tracker.h. +template <typename T> +class InstanceTest : public ::testing::Test {}; +TYPED_TEST_SUITE_P(InstanceTest); + +// A simple reference counted class to make sure that the proper elements are +// destroyed in the erase(begin, end) test. +class RefCounted { + public: + RefCounted(int value, int* count) : value_(value), count_(count) { Ref(); } + + RefCounted(const RefCounted& v) : value_(v.value_), count_(v.count_) { + Ref(); + } + + ~RefCounted() { + Unref(); + count_ = nullptr; + } + + friend void swap(RefCounted& a, RefCounted& b) { + using std::swap; + swap(a.value_, b.value_); + swap(a.count_, b.count_); + } + + RefCounted& operator=(RefCounted v) { + using std::swap; + swap(*this, v); + return *this; + } + + void Ref() const { + ABSL_RAW_CHECK(count_ != nullptr, ""); + ++(*count_); + } + + void Unref() const { + --(*count_); + ABSL_RAW_CHECK(*count_ >= 0, ""); + } + + int value_; + int* count_; +}; + +using RefCountedVec = absl::InlinedVector<RefCounted, 8>; + +// A class with a vtable pointer +class Dynamic { + public: + virtual ~Dynamic() {} +}; + +using DynamicVec = absl::InlinedVector<Dynamic, 8>; + +// Append 0..len-1 to *v +template <typename Container> +static void Fill(Container* v, int len, int offset = 0) { + for (int i = 0; i < len; i++) { + v->push_back(i + offset); + } +} + +static IntVec Fill(int len, int offset = 0) { + IntVec v; + Fill(&v, len, offset); + return v; +} + +TEST(IntVec, SimpleOps) { + for (int len = 0; len < 20; len++) { + IntVec v; + const IntVec& cv = v; // const alias + + Fill(&v, len); + EXPECT_EQ(len, v.size()); + EXPECT_LE(len, v.capacity()); + + for (int i = 0; i < len; i++) { + EXPECT_EQ(i, v[i]); + EXPECT_EQ(i, v.at(i)); + } + EXPECT_EQ(v.begin(), v.data()); + EXPECT_EQ(cv.begin(), cv.data()); + + int counter = 0; + for (IntVec::iterator iter = v.begin(); iter != v.end(); ++iter) { + EXPECT_EQ(counter, *iter); + counter++; + } + EXPECT_EQ(counter, len); + + counter = 0; + for (IntVec::const_iterator iter = v.begin(); iter != v.end(); ++iter) { + EXPECT_EQ(counter, *iter); + counter++; + } + EXPECT_EQ(counter, len); + + counter = 0; + for (IntVec::const_iterator iter = v.cbegin(); iter != v.cend(); ++iter) { + EXPECT_EQ(counter, *iter); + counter++; + } + EXPECT_EQ(counter, len); + + if (len > 0) { + EXPECT_EQ(0, v.front()); + EXPECT_EQ(len - 1, v.back()); + v.pop_back(); + EXPECT_EQ(len - 1, v.size()); + for (int i = 0; i < v.size(); ++i) { + EXPECT_EQ(i, v[i]); + EXPECT_EQ(i, v.at(i)); + } + } + } +} + +TEST(IntVec, PopBackNoOverflow) { + IntVec v = {1}; + v.pop_back(); + EXPECT_EQ(v.size(), 0); +} + +TEST(IntVec, AtThrows) { + IntVec v = {1, 2, 3}; + EXPECT_EQ(v.at(2), 3); + ABSL_BASE_INTERNAL_EXPECT_FAIL(v.at(3), std::out_of_range, + "failed bounds check"); +} + +TEST(IntVec, ReverseIterator) { + for (int len = 0; len < 20; len++) { + IntVec v; + Fill(&v, len); + + int counter = len; + for (IntVec::reverse_iterator iter = v.rbegin(); iter != v.rend(); ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + + counter = len; + for (IntVec::const_reverse_iterator iter = v.rbegin(); iter != v.rend(); + ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + + counter = len; + for (IntVec::const_reverse_iterator iter = v.crbegin(); iter != v.crend(); + ++iter) { + counter--; + EXPECT_EQ(counter, *iter); + } + EXPECT_EQ(counter, 0); + } +} + +TEST(IntVec, Erase) { + for (int len = 1; len < 20; len++) { + for (int i = 0; i < len; ++i) { + IntVec v; + Fill(&v, len); + v.erase(v.begin() + i); + EXPECT_EQ(len - 1, v.size()); + for (int j = 0; j < i; ++j) { + EXPECT_EQ(j, v[j]); + } + for (int j = i; j < len - 1; ++j) { + EXPECT_EQ(j + 1, v[j]); + } + } + } +} + +TEST(IntVec, Hardened) { + IntVec v; + Fill(&v, 10); + EXPECT_EQ(v[9], 9); +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + EXPECT_DEATH_IF_SUPPORTED(v[10], ""); + EXPECT_DEATH_IF_SUPPORTED(v[-1], ""); +#endif +} + +// At the end of this test loop, the elements between [erase_begin, erase_end) +// should have reference counts == 0, and all others elements should have +// reference counts == 1. +TEST(RefCountedVec, EraseBeginEnd) { + for (int len = 1; len < 20; ++len) { + for (int erase_begin = 0; erase_begin < len; ++erase_begin) { + for (int erase_end = erase_begin; erase_end <= len; ++erase_end) { + std::vector<int> counts(len, 0); + RefCountedVec v; + for (int i = 0; i < len; ++i) { + v.push_back(RefCounted(i, &counts[i])); + } + + int erase_len = erase_end - erase_begin; + + v.erase(v.begin() + erase_begin, v.begin() + erase_end); + + EXPECT_EQ(len - erase_len, v.size()); + + // Check the elements before the first element erased. + for (int i = 0; i < erase_begin; ++i) { + EXPECT_EQ(i, v[i].value_); + } + + // Check the elements after the first element erased. + for (int i = erase_begin; i < v.size(); ++i) { + EXPECT_EQ(i + erase_len, v[i].value_); + } + + // Check that the elements at the beginning are preserved. + for (int i = 0; i < erase_begin; ++i) { + EXPECT_EQ(1, counts[i]); + } + + // Check that the erased elements are destroyed + for (int i = erase_begin; i < erase_end; ++i) { + EXPECT_EQ(0, counts[i]); + } + + // Check that the elements at the end are preserved. + for (int i = erase_end; i < len; ++i) { + EXPECT_EQ(1, counts[i]); + } + } + } + } +} + +struct NoDefaultCtor { + explicit NoDefaultCtor(int) {} +}; +struct NoCopy { + NoCopy() {} + NoCopy(const NoCopy&) = delete; +}; +struct NoAssign { + NoAssign() {} + NoAssign& operator=(const NoAssign&) = delete; +}; +struct MoveOnly { + MoveOnly() {} + MoveOnly(MoveOnly&&) = default; + MoveOnly& operator=(MoveOnly&&) = default; +}; +TEST(InlinedVectorTest, NoDefaultCtor) { + absl::InlinedVector<NoDefaultCtor, 1> v(10, NoDefaultCtor(2)); + (void)v; +} +TEST(InlinedVectorTest, NoCopy) { + absl::InlinedVector<NoCopy, 1> v(10); + (void)v; +} +TEST(InlinedVectorTest, NoAssign) { + absl::InlinedVector<NoAssign, 1> v(10); + (void)v; +} +TEST(InlinedVectorTest, MoveOnly) { + absl::InlinedVector<MoveOnly, 2> v; + v.push_back(MoveOnly{}); + v.push_back(MoveOnly{}); + v.push_back(MoveOnly{}); + v.erase(v.begin()); + v.push_back(MoveOnly{}); + v.erase(v.begin(), v.begin() + 1); + v.insert(v.begin(), MoveOnly{}); + v.emplace(v.begin()); + v.emplace(v.begin(), MoveOnly{}); +} +TEST(InlinedVectorTest, Noexcept) { + EXPECT_TRUE(std::is_nothrow_move_constructible<IntVec>::value); + EXPECT_TRUE((std::is_nothrow_move_constructible< + absl::InlinedVector<MoveOnly, 2>>::value)); + + struct MoveCanThrow { + MoveCanThrow(MoveCanThrow&&) {} + }; + EXPECT_EQ(absl::default_allocator_is_nothrow::value, + (std::is_nothrow_move_constructible< + absl::InlinedVector<MoveCanThrow, 2>>::value)); +} + +TEST(InlinedVectorTest, EmplaceBack) { + absl::InlinedVector<std::pair<std::string, int>, 1> v; + + auto& inlined_element = v.emplace_back("answer", 42); + EXPECT_EQ(&inlined_element, &v[0]); + EXPECT_EQ(inlined_element.first, "answer"); + EXPECT_EQ(inlined_element.second, 42); + + auto& allocated_element = v.emplace_back("taxicab", 1729); + EXPECT_EQ(&allocated_element, &v[1]); + EXPECT_EQ(allocated_element.first, "taxicab"); + EXPECT_EQ(allocated_element.second, 1729); +} + +TEST(InlinedVectorTest, ShrinkToFitGrowingVector) { + absl::InlinedVector<std::pair<std::string, int>, 1> v; + + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 1); + + v.emplace_back("answer", 42); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 1); + + v.emplace_back("taxicab", 1729); + EXPECT_GE(v.capacity(), 2); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 2); + + v.reserve(100); + EXPECT_GE(v.capacity(), 100); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 2); +} + +TEST(InlinedVectorTest, ShrinkToFitEdgeCases) { + { + absl::InlinedVector<std::pair<std::string, int>, 1> v; + v.emplace_back("answer", 42); + v.emplace_back("taxicab", 1729); + EXPECT_GE(v.capacity(), 2); + v.pop_back(); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 1); + EXPECT_EQ(v[0].first, "answer"); + EXPECT_EQ(v[0].second, 42); + } + + { + absl::InlinedVector<std::string, 2> v(100); + v.resize(0); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 2); // inlined capacity + } + + { + absl::InlinedVector<std::string, 2> v(100); + v.resize(1); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 2); // inlined capacity + } + + { + absl::InlinedVector<std::string, 2> v(100); + v.resize(2); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 2); + } + + { + absl::InlinedVector<std::string, 2> v(100); + v.resize(3); + v.shrink_to_fit(); + EXPECT_EQ(v.capacity(), 3); + } +} + +TEST(IntVec, Insert) { + for (int len = 0; len < 20; len++) { + for (int pos = 0; pos <= len; pos++) { + { + // Single element + std::vector<int> std_v; + Fill(&std_v, len); + IntVec v; + Fill(&v, len); + + std_v.insert(std_v.begin() + pos, 9999); + IntVec::iterator it = v.insert(v.cbegin() + pos, 9999); + EXPECT_THAT(v, ElementsAreArray(std_v)); + EXPECT_EQ(it, v.cbegin() + pos); + } + { + // n elements + std::vector<int> std_v; + Fill(&std_v, len); + IntVec v; + Fill(&v, len); + + IntVec::size_type n = 5; + std_v.insert(std_v.begin() + pos, n, 9999); + IntVec::iterator it = v.insert(v.cbegin() + pos, n, 9999); + EXPECT_THAT(v, ElementsAreArray(std_v)); + EXPECT_EQ(it, v.cbegin() + pos); + } + { + // Iterator range (random access iterator) + std::vector<int> std_v; + Fill(&std_v, len); + IntVec v; + Fill(&v, len); + + const std::vector<int> input = {9999, 8888, 7777}; + std_v.insert(std_v.begin() + pos, input.cbegin(), input.cend()); + IntVec::iterator it = + v.insert(v.cbegin() + pos, input.cbegin(), input.cend()); + EXPECT_THAT(v, ElementsAreArray(std_v)); + EXPECT_EQ(it, v.cbegin() + pos); + } + { + // Iterator range (forward iterator) + std::vector<int> std_v; + Fill(&std_v, len); + IntVec v; + Fill(&v, len); + + const std::forward_list<int> input = {9999, 8888, 7777}; + std_v.insert(std_v.begin() + pos, input.cbegin(), input.cend()); + IntVec::iterator it = + v.insert(v.cbegin() + pos, input.cbegin(), input.cend()); + EXPECT_THAT(v, ElementsAreArray(std_v)); + EXPECT_EQ(it, v.cbegin() + pos); + } + { + // Iterator range (input iterator) + std::vector<int> std_v; + Fill(&std_v, len); + IntVec v; + Fill(&v, len); + + std_v.insert(std_v.begin() + pos, {9999, 8888, 7777}); + std::istringstream input("9999 8888 7777"); + IntVec::iterator it = + v.insert(v.cbegin() + pos, std::istream_iterator<int>(input), + std::istream_iterator<int>()); + EXPECT_THAT(v, ElementsAreArray(std_v)); + EXPECT_EQ(it, v.cbegin() + pos); + } + { + // Initializer list + std::vector<int> std_v; + Fill(&std_v, len); + IntVec v; + Fill(&v, len); + + std_v.insert(std_v.begin() + pos, {9999, 8888}); + IntVec::iterator it = v.insert(v.cbegin() + pos, {9999, 8888}); + EXPECT_THAT(v, ElementsAreArray(std_v)); + EXPECT_EQ(it, v.cbegin() + pos); + } + } + } +} + +TEST(RefCountedVec, InsertConstructorDestructor) { + // Make sure the proper construction/destruction happen during insert + // operations. + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + for (int pos = 0; pos <= len; pos++) { + SCOPED_TRACE(pos); + std::vector<int> counts(len, 0); + int inserted_count = 0; + RefCountedVec v; + for (int i = 0; i < len; ++i) { + SCOPED_TRACE(i); + v.push_back(RefCounted(i, &counts[i])); + } + + EXPECT_THAT(counts, Each(Eq(1))); + + RefCounted insert_element(9999, &inserted_count); + EXPECT_EQ(1, inserted_count); + v.insert(v.begin() + pos, insert_element); + EXPECT_EQ(2, inserted_count); + // Check that the elements at the end are preserved. + EXPECT_THAT(counts, Each(Eq(1))); + EXPECT_EQ(2, inserted_count); + } + } +} + +TEST(IntVec, Resize) { + for (int len = 0; len < 20; len++) { + IntVec v; + Fill(&v, len); + + // Try resizing up and down by k elements + static const int kResizeElem = 1000000; + for (int k = 0; k < 10; k++) { + // Enlarging resize + v.resize(len + k, kResizeElem); + EXPECT_EQ(len + k, v.size()); + EXPECT_LE(len + k, v.capacity()); + for (int i = 0; i < len + k; i++) { + if (i < len) { + EXPECT_EQ(i, v[i]); + } else { + EXPECT_EQ(kResizeElem, v[i]); + } + } + + // Shrinking resize + v.resize(len, kResizeElem); + EXPECT_EQ(len, v.size()); + EXPECT_LE(len, v.capacity()); + for (int i = 0; i < len; i++) { + EXPECT_EQ(i, v[i]); + } + } + } +} + +TEST(IntVec, InitWithLength) { + for (int len = 0; len < 20; len++) { + IntVec v(len, 7); + EXPECT_EQ(len, v.size()); + EXPECT_LE(len, v.capacity()); + for (int i = 0; i < len; i++) { + EXPECT_EQ(7, v[i]); + } + } +} + +TEST(IntVec, CopyConstructorAndAssignment) { + for (int len = 0; len < 20; len++) { + IntVec v; + Fill(&v, len); + EXPECT_EQ(len, v.size()); + EXPECT_LE(len, v.capacity()); + + IntVec v2(v); + EXPECT_TRUE(v == v2) << PrintToString(v) << PrintToString(v2); + + for (int start_len = 0; start_len < 20; start_len++) { + IntVec v3; + Fill(&v3, start_len, 99); // Add dummy elements that should go away + v3 = v; + EXPECT_TRUE(v == v3) << PrintToString(v) << PrintToString(v3); + } + } +} + +TEST(IntVec, AliasingCopyAssignment) { + for (int len = 0; len < 20; ++len) { + IntVec original; + Fill(&original, len); + IntVec dup = original; + dup = *&dup; + EXPECT_EQ(dup, original); + } +} + +TEST(IntVec, MoveConstructorAndAssignment) { + for (int len = 0; len < 20; len++) { + IntVec v_in; + const int inlined_capacity = v_in.capacity(); + Fill(&v_in, len); + EXPECT_EQ(len, v_in.size()); + EXPECT_LE(len, v_in.capacity()); + + { + IntVec v_temp(v_in); + auto* old_data = v_temp.data(); + IntVec v_out(std::move(v_temp)); + EXPECT_TRUE(v_in == v_out) << PrintToString(v_in) << PrintToString(v_out); + if (v_in.size() > inlined_capacity) { + // Allocation is moved as a whole, data stays in place. + EXPECT_TRUE(v_out.data() == old_data); + } else { + EXPECT_FALSE(v_out.data() == old_data); + } + } + for (int start_len = 0; start_len < 20; start_len++) { + IntVec v_out; + Fill(&v_out, start_len, 99); // Add dummy elements that should go away + IntVec v_temp(v_in); + auto* old_data = v_temp.data(); + v_out = std::move(v_temp); + EXPECT_TRUE(v_in == v_out) << PrintToString(v_in) << PrintToString(v_out); + if (v_in.size() > inlined_capacity) { + // Allocation is moved as a whole, data stays in place. + EXPECT_TRUE(v_out.data() == old_data); + } else { + EXPECT_FALSE(v_out.data() == old_data); + } + } + } +} + +class NotTriviallyDestructible { + public: + NotTriviallyDestructible() : p_(new int(1)) {} + explicit NotTriviallyDestructible(int i) : p_(new int(i)) {} + + NotTriviallyDestructible(const NotTriviallyDestructible& other) + : p_(new int(*other.p_)) {} + + NotTriviallyDestructible& operator=(const NotTriviallyDestructible& other) { + p_ = absl::make_unique<int>(*other.p_); + return *this; + } + + bool operator==(const NotTriviallyDestructible& other) const { + return *p_ == *other.p_; + } + + private: + std::unique_ptr<int> p_; +}; + +TEST(AliasingTest, Emplace) { + for (int i = 2; i < 20; ++i) { + absl::InlinedVector<NotTriviallyDestructible, 10> vec; + for (int j = 0; j < i; ++j) { + vec.push_back(NotTriviallyDestructible(j)); + } + vec.emplace(vec.begin(), vec[0]); + EXPECT_EQ(vec[0], vec[1]); + vec.emplace(vec.begin() + i / 2, vec[i / 2]); + EXPECT_EQ(vec[i / 2], vec[i / 2 + 1]); + vec.emplace(vec.end() - 1, vec.back()); + EXPECT_EQ(vec[vec.size() - 2], vec.back()); + } +} + +TEST(AliasingTest, InsertWithCount) { + for (int i = 1; i < 20; ++i) { + absl::InlinedVector<NotTriviallyDestructible, 10> vec; + for (int j = 0; j < i; ++j) { + vec.push_back(NotTriviallyDestructible(j)); + } + for (int n = 0; n < 5; ++n) { + // We use back where we can because it's guaranteed to become invalidated + vec.insert(vec.begin(), n, vec.back()); + auto b = vec.begin(); + EXPECT_TRUE( + std::all_of(b, b + n, [&vec](const NotTriviallyDestructible& x) { + return x == vec.back(); + })); + + auto m_idx = vec.size() / 2; + vec.insert(vec.begin() + m_idx, n, vec.back()); + auto m = vec.begin() + m_idx; + EXPECT_TRUE( + std::all_of(m, m + n, [&vec](const NotTriviallyDestructible& x) { + return x == vec.back(); + })); + + // We want distinct values so the equality test is meaningful, + // vec[vec.size() - 1] is also almost always invalidated. + auto old_e = vec.size() - 1; + auto val = vec[old_e]; + vec.insert(vec.end(), n, vec[old_e]); + auto e = vec.begin() + old_e; + EXPECT_TRUE(std::all_of( + e, e + n, + [&val](const NotTriviallyDestructible& x) { return x == val; })); + } + } +} + +TEST(OverheadTest, Storage) { + // Check for size overhead. + // In particular, ensure that std::allocator doesn't cost anything to store. + // The union should be absorbing some of the allocation bookkeeping overhead + // in the larger vectors, leaving only the size_ field as overhead. + + struct T { void* val; }; + size_t expected_overhead = sizeof(T); + + EXPECT_EQ((2 * expected_overhead), + sizeof(absl::InlinedVector<T, 1>) - sizeof(T[1])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 2>) - sizeof(T[2])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 3>) - sizeof(T[3])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 4>) - sizeof(T[4])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 5>) - sizeof(T[5])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 6>) - sizeof(T[6])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 7>) - sizeof(T[7])); + EXPECT_EQ(expected_overhead, + sizeof(absl::InlinedVector<T, 8>) - sizeof(T[8])); +} + +TEST(IntVec, Clear) { + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + IntVec v; + Fill(&v, len); + v.clear(); + EXPECT_EQ(0, v.size()); + EXPECT_EQ(v.begin(), v.end()); + } +} + +TEST(IntVec, Reserve) { + for (int len = 0; len < 20; len++) { + IntVec v; + Fill(&v, len); + + for (int newlen = 0; newlen < 100; newlen++) { + const int* start_rep = v.data(); + v.reserve(newlen); + const int* final_rep = v.data(); + if (newlen <= len) { + EXPECT_EQ(start_rep, final_rep); + } + EXPECT_LE(newlen, v.capacity()); + + // Filling up to newlen should not change rep + while (v.size() < newlen) { + v.push_back(0); + } + EXPECT_EQ(final_rep, v.data()); + } + } +} + +TEST(StringVec, SelfRefPushBack) { + std::vector<std::string> std_v; + absl::InlinedVector<std::string, 4> v; + const std::string s = "A quite long string to ensure heap."; + std_v.push_back(s); + v.push_back(s); + for (int i = 0; i < 20; ++i) { + EXPECT_THAT(v, ElementsAreArray(std_v)); + + v.push_back(v.back()); + std_v.push_back(std_v.back()); + } + EXPECT_THAT(v, ElementsAreArray(std_v)); +} + +TEST(StringVec, SelfRefPushBackWithMove) { + std::vector<std::string> std_v; + absl::InlinedVector<std::string, 4> v; + const std::string s = "A quite long string to ensure heap."; + std_v.push_back(s); + v.push_back(s); + for (int i = 0; i < 20; ++i) { + EXPECT_EQ(v.back(), std_v.back()); + + v.push_back(std::move(v.back())); + std_v.push_back(std::move(std_v.back())); + } + EXPECT_EQ(v.back(), std_v.back()); +} + +TEST(StringVec, SelfMove) { + const std::string s = "A quite long string to ensure heap."; + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + absl::InlinedVector<std::string, 8> v; + for (int i = 0; i < len; ++i) { + SCOPED_TRACE(i); + v.push_back(s); + } + // Indirection necessary to avoid compiler warning. + v = std::move(*(&v)); + // Ensure that the inlined vector is still in a valid state by copying it. + // We don't expect specific contents since a self-move results in an + // unspecified valid state. + std::vector<std::string> copy(v.begin(), v.end()); + } +} + +TEST(IntVec, Swap) { + for (int l1 = 0; l1 < 20; l1++) { + SCOPED_TRACE(l1); + for (int l2 = 0; l2 < 20; l2++) { + SCOPED_TRACE(l2); + IntVec a = Fill(l1, 0); + IntVec b = Fill(l2, 100); + { + using std::swap; + swap(a, b); + } + EXPECT_EQ(l1, b.size()); + EXPECT_EQ(l2, a.size()); + for (int i = 0; i < l1; i++) { + SCOPED_TRACE(i); + EXPECT_EQ(i, b[i]); + } + for (int i = 0; i < l2; i++) { + SCOPED_TRACE(i); + EXPECT_EQ(100 + i, a[i]); + } + } + } +} + +TYPED_TEST_P(InstanceTest, Swap) { + using Instance = TypeParam; + using InstanceVec = absl::InlinedVector<Instance, 8>; + for (int l1 = 0; l1 < 20; l1++) { + SCOPED_TRACE(l1); + for (int l2 = 0; l2 < 20; l2++) { + SCOPED_TRACE(l2); + InstanceTracker tracker; + InstanceVec a, b; + const size_t inlined_capacity = a.capacity(); + auto min_len = std::min(l1, l2); + auto max_len = std::max(l1, l2); + for (int i = 0; i < l1; i++) a.push_back(Instance(i)); + for (int i = 0; i < l2; i++) b.push_back(Instance(100 + i)); + EXPECT_EQ(tracker.instances(), l1 + l2); + tracker.ResetCopiesMovesSwaps(); + { + using std::swap; + swap(a, b); + } + EXPECT_EQ(tracker.instances(), l1 + l2); + if (a.size() > inlined_capacity && b.size() > inlined_capacity) { + EXPECT_EQ(tracker.swaps(), 0); // Allocations are swapped. + EXPECT_EQ(tracker.moves(), 0); + } else if (a.size() <= inlined_capacity && b.size() <= inlined_capacity) { + EXPECT_EQ(tracker.swaps(), min_len); + EXPECT_EQ((tracker.moves() ? tracker.moves() : tracker.copies()), + max_len - min_len); + } else { + // One is allocated and the other isn't. The allocation is transferred + // without copying elements, and the inlined instances are copied/moved. + EXPECT_EQ(tracker.swaps(), 0); + EXPECT_EQ((tracker.moves() ? tracker.moves() : tracker.copies()), + min_len); + } + + EXPECT_EQ(l1, b.size()); + EXPECT_EQ(l2, a.size()); + for (int i = 0; i < l1; i++) { + EXPECT_EQ(i, b[i].value()); + } + for (int i = 0; i < l2; i++) { + EXPECT_EQ(100 + i, a[i].value()); + } + } + } +} + +TEST(IntVec, EqualAndNotEqual) { + IntVec a, b; + EXPECT_TRUE(a == b); + EXPECT_FALSE(a != b); + + a.push_back(3); + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + + b.push_back(3); + EXPECT_TRUE(a == b); + EXPECT_FALSE(a != b); + + b.push_back(7); + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + + a.push_back(6); + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + + a.clear(); + b.clear(); + for (int i = 0; i < 100; i++) { + a.push_back(i); + b.push_back(i); + EXPECT_TRUE(a == b); + EXPECT_FALSE(a != b); + + b[i] = b[i] + 1; + EXPECT_FALSE(a == b); + EXPECT_TRUE(a != b); + + b[i] = b[i] - 1; // Back to before + EXPECT_TRUE(a == b); + EXPECT_FALSE(a != b); + } +} + +TEST(IntVec, RelationalOps) { + IntVec a, b; + EXPECT_FALSE(a < b); + EXPECT_FALSE(b < a); + EXPECT_FALSE(a > b); + EXPECT_FALSE(b > a); + EXPECT_TRUE(a <= b); + EXPECT_TRUE(b <= a); + EXPECT_TRUE(a >= b); + EXPECT_TRUE(b >= a); + b.push_back(3); + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); + EXPECT_FALSE(a > b); + EXPECT_TRUE(b > a); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(b <= a); + EXPECT_FALSE(a >= b); + EXPECT_TRUE(b >= a); +} + +TYPED_TEST_P(InstanceTest, CountConstructorsDestructors) { + using Instance = TypeParam; + using InstanceVec = absl::InlinedVector<Instance, 8>; + InstanceTracker tracker; + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + tracker.ResetCopiesMovesSwaps(); + + InstanceVec v; + const size_t inlined_capacity = v.capacity(); + for (int i = 0; i < len; i++) { + v.push_back(Instance(i)); + } + EXPECT_EQ(tracker.instances(), len); + EXPECT_GE(tracker.copies() + tracker.moves(), + len); // More due to reallocation. + tracker.ResetCopiesMovesSwaps(); + + // Enlarging resize() must construct some objects + tracker.ResetCopiesMovesSwaps(); + v.resize(len + 10, Instance(100)); + EXPECT_EQ(tracker.instances(), len + 10); + if (len <= inlined_capacity && len + 10 > inlined_capacity) { + EXPECT_EQ(tracker.copies() + tracker.moves(), 10 + len); + } else { + // Only specify a minimum number of copies + moves. We don't want to + // depend on the reallocation policy here. + EXPECT_GE(tracker.copies() + tracker.moves(), + 10); // More due to reallocation. + } + + // Shrinking resize() must destroy some objects + tracker.ResetCopiesMovesSwaps(); + v.resize(len, Instance(100)); + EXPECT_EQ(tracker.instances(), len); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 0); + + // reserve() must not increase the number of initialized objects + SCOPED_TRACE("reserve"); + v.reserve(len + 1000); + EXPECT_EQ(tracker.instances(), len); + EXPECT_EQ(tracker.copies() + tracker.moves(), len); + + // pop_back() and erase() must destroy one object + if (len > 0) { + tracker.ResetCopiesMovesSwaps(); + v.pop_back(); + EXPECT_EQ(tracker.instances(), len - 1); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 0); + + if (!v.empty()) { + tracker.ResetCopiesMovesSwaps(); + v.erase(v.begin()); + EXPECT_EQ(tracker.instances(), len - 2); + EXPECT_EQ(tracker.copies() + tracker.moves(), len - 2); + } + } + + tracker.ResetCopiesMovesSwaps(); + int instances_before_empty_erase = tracker.instances(); + v.erase(v.begin(), v.begin()); + EXPECT_EQ(tracker.instances(), instances_before_empty_erase); + EXPECT_EQ(tracker.copies() + tracker.moves(), 0); + } +} + +TYPED_TEST_P(InstanceTest, CountConstructorsDestructorsOnCopyConstruction) { + using Instance = TypeParam; + using InstanceVec = absl::InlinedVector<Instance, 8>; + InstanceTracker tracker; + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + tracker.ResetCopiesMovesSwaps(); + + InstanceVec v; + for (int i = 0; i < len; i++) { + v.push_back(Instance(i)); + } + EXPECT_EQ(tracker.instances(), len); + EXPECT_GE(tracker.copies() + tracker.moves(), + len); // More due to reallocation. + tracker.ResetCopiesMovesSwaps(); + { // Copy constructor should create 'len' more instances. + InstanceVec v_copy(v); + EXPECT_EQ(tracker.instances(), len + len); + EXPECT_EQ(tracker.copies(), len); + EXPECT_EQ(tracker.moves(), 0); + } + EXPECT_EQ(tracker.instances(), len); + } +} + +TYPED_TEST_P(InstanceTest, CountConstructorsDestructorsOnMoveConstruction) { + using Instance = TypeParam; + using InstanceVec = absl::InlinedVector<Instance, 8>; + InstanceTracker tracker; + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + tracker.ResetCopiesMovesSwaps(); + + InstanceVec v; + const size_t inlined_capacity = v.capacity(); + for (int i = 0; i < len; i++) { + v.push_back(Instance(i)); + } + EXPECT_EQ(tracker.instances(), len); + EXPECT_GE(tracker.copies() + tracker.moves(), + len); // More due to reallocation. + tracker.ResetCopiesMovesSwaps(); + { + InstanceVec v_copy(std::move(v)); + if (len > inlined_capacity) { + // Allocation is moved as a whole. + EXPECT_EQ(tracker.instances(), len); + EXPECT_EQ(tracker.live_instances(), len); + // Tests an implementation detail, don't rely on this in your code. + EXPECT_EQ(v.size(), 0); // NOLINT misc-use-after-move + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 0); + } else { + EXPECT_EQ(tracker.instances(), len + len); + if (Instance::supports_move()) { + EXPECT_EQ(tracker.live_instances(), len); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), len); + } else { + EXPECT_EQ(tracker.live_instances(), len + len); + EXPECT_EQ(tracker.copies(), len); + EXPECT_EQ(tracker.moves(), 0); + } + } + EXPECT_EQ(tracker.swaps(), 0); + } + } +} + +TYPED_TEST_P(InstanceTest, CountConstructorsDestructorsOnAssignment) { + using Instance = TypeParam; + using InstanceVec = absl::InlinedVector<Instance, 8>; + InstanceTracker tracker; + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + for (int longorshort = 0; longorshort <= 1; ++longorshort) { + SCOPED_TRACE(longorshort); + tracker.ResetCopiesMovesSwaps(); + + InstanceVec longer, shorter; + for (int i = 0; i < len; i++) { + longer.push_back(Instance(i)); + shorter.push_back(Instance(i)); + } + longer.push_back(Instance(len)); + EXPECT_EQ(tracker.instances(), len + len + 1); + EXPECT_GE(tracker.copies() + tracker.moves(), + len + len + 1); // More due to reallocation. + + tracker.ResetCopiesMovesSwaps(); + if (longorshort) { + shorter = longer; + EXPECT_EQ(tracker.instances(), (len + 1) + (len + 1)); + EXPECT_GE(tracker.copies() + tracker.moves(), + len + 1); // More due to reallocation. + } else { + longer = shorter; + EXPECT_EQ(tracker.instances(), len + len); + EXPECT_EQ(tracker.copies() + tracker.moves(), len); + } + } + } +} + +TYPED_TEST_P(InstanceTest, CountConstructorsDestructorsOnMoveAssignment) { + using Instance = TypeParam; + using InstanceVec = absl::InlinedVector<Instance, 8>; + InstanceTracker tracker; + for (int len = 0; len < 20; len++) { + SCOPED_TRACE(len); + for (int longorshort = 0; longorshort <= 1; ++longorshort) { + SCOPED_TRACE(longorshort); + tracker.ResetCopiesMovesSwaps(); + + InstanceVec longer, shorter; + const int inlined_capacity = longer.capacity(); + for (int i = 0; i < len; i++) { + longer.push_back(Instance(i)); + shorter.push_back(Instance(i)); + } + longer.push_back(Instance(len)); + EXPECT_EQ(tracker.instances(), len + len + 1); + EXPECT_GE(tracker.copies() + tracker.moves(), + len + len + 1); // More due to reallocation. + + tracker.ResetCopiesMovesSwaps(); + int src_len; + if (longorshort) { + src_len = len + 1; + shorter = std::move(longer); + } else { + src_len = len; + longer = std::move(shorter); + } + if (src_len > inlined_capacity) { + // Allocation moved as a whole. + EXPECT_EQ(tracker.instances(), src_len); + EXPECT_EQ(tracker.live_instances(), src_len); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 0); + } else { + // Elements are all copied. + EXPECT_EQ(tracker.instances(), src_len + src_len); + if (Instance::supports_move()) { + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), src_len); + EXPECT_EQ(tracker.live_instances(), src_len); + } else { + EXPECT_EQ(tracker.copies(), src_len); + EXPECT_EQ(tracker.moves(), 0); + EXPECT_EQ(tracker.live_instances(), src_len + src_len); + } + } + EXPECT_EQ(tracker.swaps(), 0); + } + } +} + +TEST(CountElemAssign, SimpleTypeWithInlineBacking) { + for (size_t original_size = 0; original_size <= 5; ++original_size) { + SCOPED_TRACE(original_size); + // Original contents are [12345, 12345, ...] + std::vector<int> original_contents(original_size, 12345); + + absl::InlinedVector<int, 2> v(original_contents.begin(), + original_contents.end()); + v.assign(2, 123); + EXPECT_THAT(v, AllOf(SizeIs(2), ElementsAre(123, 123))); + if (original_size <= 2) { + // If the original had inline backing, it should stay inline. + EXPECT_EQ(2, v.capacity()); + } + } +} + +TEST(CountElemAssign, SimpleTypeWithAllocation) { + for (size_t original_size = 0; original_size <= 5; ++original_size) { + SCOPED_TRACE(original_size); + // Original contents are [12345, 12345, ...] + std::vector<int> original_contents(original_size, 12345); + + absl::InlinedVector<int, 2> v(original_contents.begin(), + original_contents.end()); + v.assign(3, 123); + EXPECT_THAT(v, AllOf(SizeIs(3), ElementsAre(123, 123, 123))); + EXPECT_LE(v.size(), v.capacity()); + } +} + +TYPED_TEST_P(InstanceTest, CountElemAssignInlineBacking) { + using Instance = TypeParam; + for (size_t original_size = 0; original_size <= 5; ++original_size) { + SCOPED_TRACE(original_size); + // Original contents are [12345, 12345, ...] + std::vector<Instance> original_contents(original_size, Instance(12345)); + + absl::InlinedVector<Instance, 2> v(original_contents.begin(), + original_contents.end()); + v.assign(2, Instance(123)); + EXPECT_THAT(v, AllOf(SizeIs(2), ElementsAre(ValueIs(123), ValueIs(123)))); + if (original_size <= 2) { + // If the original had inline backing, it should stay inline. + EXPECT_EQ(2, v.capacity()); + } + } +} + +template <typename Instance> +void InstanceCountElemAssignWithAllocationTest() { + for (size_t original_size = 0; original_size <= 5; ++original_size) { + SCOPED_TRACE(original_size); + // Original contents are [12345, 12345, ...] + std::vector<Instance> original_contents(original_size, Instance(12345)); + + absl::InlinedVector<Instance, 2> v(original_contents.begin(), + original_contents.end()); + v.assign(3, Instance(123)); + EXPECT_THAT(v, AllOf(SizeIs(3), ElementsAre(ValueIs(123), ValueIs(123), + ValueIs(123)))); + EXPECT_LE(v.size(), v.capacity()); + } +} +TEST(CountElemAssign, WithAllocationCopyableInstance) { + InstanceCountElemAssignWithAllocationTest<CopyableOnlyInstance>(); +} +TEST(CountElemAssign, WithAllocationCopyableMovableInstance) { + InstanceCountElemAssignWithAllocationTest<CopyableMovableInstance>(); +} + +TEST(RangedConstructor, SimpleType) { + std::vector<int> source_v = {4, 5, 6}; + // First try to fit in inline backing + absl::InlinedVector<int, 4> v(source_v.begin(), source_v.end()); + EXPECT_EQ(3, v.size()); + EXPECT_EQ(4, v.capacity()); // Indication that we're still on inlined storage + EXPECT_EQ(4, v[0]); + EXPECT_EQ(5, v[1]); + EXPECT_EQ(6, v[2]); + + // Now, force a re-allocate + absl::InlinedVector<int, 2> realloc_v(source_v.begin(), source_v.end()); + EXPECT_EQ(3, realloc_v.size()); + EXPECT_LT(2, realloc_v.capacity()); + EXPECT_EQ(4, realloc_v[0]); + EXPECT_EQ(5, realloc_v[1]); + EXPECT_EQ(6, realloc_v[2]); +} + +// Test for ranged constructors using Instance as the element type and +// SourceContainer as the source container type. +template <typename Instance, typename SourceContainer, int inlined_capacity> +void InstanceRangedConstructorTestForContainer() { + InstanceTracker tracker; + SourceContainer source_v = {Instance(0), Instance(1)}; + tracker.ResetCopiesMovesSwaps(); + absl::InlinedVector<Instance, inlined_capacity> v(source_v.begin(), + source_v.end()); + EXPECT_EQ(2, v.size()); + EXPECT_LT(1, v.capacity()); + EXPECT_EQ(0, v[0].value()); + EXPECT_EQ(1, v[1].value()); + EXPECT_EQ(tracker.copies(), 2); + EXPECT_EQ(tracker.moves(), 0); +} + +template <typename Instance, int inlined_capacity> +void InstanceRangedConstructorTestWithCapacity() { + // Test with const and non-const, random access and non-random-access sources. + // TODO(bsamwel): Test with an input iterator source. + { + SCOPED_TRACE("std::list"); + InstanceRangedConstructorTestForContainer<Instance, std::list<Instance>, + inlined_capacity>(); + { + SCOPED_TRACE("const std::list"); + InstanceRangedConstructorTestForContainer< + Instance, const std::list<Instance>, inlined_capacity>(); + } + { + SCOPED_TRACE("std::vector"); + InstanceRangedConstructorTestForContainer<Instance, std::vector<Instance>, + inlined_capacity>(); + } + { + SCOPED_TRACE("const std::vector"); + InstanceRangedConstructorTestForContainer< + Instance, const std::vector<Instance>, inlined_capacity>(); + } + } +} + +TYPED_TEST_P(InstanceTest, RangedConstructor) { + using Instance = TypeParam; + SCOPED_TRACE("capacity=1"); + InstanceRangedConstructorTestWithCapacity<Instance, 1>(); + SCOPED_TRACE("capacity=2"); + InstanceRangedConstructorTestWithCapacity<Instance, 2>(); +} + +TEST(RangedConstructor, ElementsAreConstructed) { + std::vector<std::string> source_v = {"cat", "dog"}; + + // Force expansion and re-allocation of v. Ensures that when the vector is + // expanded that new elements are constructed. + absl::InlinedVector<std::string, 1> v(source_v.begin(), source_v.end()); + EXPECT_EQ("cat", v[0]); + EXPECT_EQ("dog", v[1]); +} + +TEST(RangedAssign, SimpleType) { + // Test for all combinations of original sizes (empty and non-empty inline, + // and out of line) and target sizes. + for (size_t original_size = 0; original_size <= 5; ++original_size) { + SCOPED_TRACE(original_size); + // Original contents are [12345, 12345, ...] + std::vector<int> original_contents(original_size, 12345); + + for (size_t target_size = 0; target_size <= 5; ++target_size) { + SCOPED_TRACE(target_size); + + // New contents are [3, 4, ...] + std::vector<int> new_contents; + for (size_t i = 0; i < target_size; ++i) { + new_contents.push_back(i + 3); + } + + absl::InlinedVector<int, 3> v(original_contents.begin(), + original_contents.end()); + v.assign(new_contents.begin(), new_contents.end()); + + EXPECT_EQ(new_contents.size(), v.size()); + EXPECT_LE(new_contents.size(), v.capacity()); + if (target_size <= 3 && original_size <= 3) { + // Storage should stay inline when target size is small. + EXPECT_EQ(3, v.capacity()); + } + EXPECT_THAT(v, ElementsAreArray(new_contents)); + } + } +} + +// Returns true if lhs and rhs have the same value. +template <typename Instance> +static bool InstanceValuesEqual(const Instance& lhs, const Instance& rhs) { + return lhs.value() == rhs.value(); +} + +// Test for ranged assign() using Instance as the element type and +// SourceContainer as the source container type. +template <typename Instance, typename SourceContainer> +void InstanceRangedAssignTestForContainer() { + // Test for all combinations of original sizes (empty and non-empty inline, + // and out of line) and target sizes. + for (size_t original_size = 0; original_size <= 5; ++original_size) { + SCOPED_TRACE(original_size); + // Original contents are [12345, 12345, ...] + std::vector<Instance> original_contents(original_size, Instance(12345)); + + for (size_t target_size = 0; target_size <= 5; ++target_size) { + SCOPED_TRACE(target_size); + + // New contents are [3, 4, ...] + // Generate data using a non-const container, because SourceContainer + // itself may be const. + // TODO(bsamwel): Test with an input iterator. + std::vector<Instance> new_contents_in; + for (size_t i = 0; i < target_size; ++i) { + new_contents_in.push_back(Instance(i + 3)); + } + SourceContainer new_contents(new_contents_in.begin(), + new_contents_in.end()); + + absl::InlinedVector<Instance, 3> v(original_contents.begin(), + original_contents.end()); + v.assign(new_contents.begin(), new_contents.end()); + + EXPECT_EQ(new_contents.size(), v.size()); + EXPECT_LE(new_contents.size(), v.capacity()); + if (target_size <= 3 && original_size <= 3) { + // Storage should stay inline when target size is small. + EXPECT_EQ(3, v.capacity()); + } + EXPECT_TRUE(std::equal(v.begin(), v.end(), new_contents.begin(), + InstanceValuesEqual<Instance>)); + } + } +} + +TYPED_TEST_P(InstanceTest, RangedAssign) { + using Instance = TypeParam; + // Test with const and non-const, random access and non-random-access sources. + // TODO(bsamwel): Test with an input iterator source. + SCOPED_TRACE("std::list"); + InstanceRangedAssignTestForContainer<Instance, std::list<Instance>>(); + SCOPED_TRACE("const std::list"); + InstanceRangedAssignTestForContainer<Instance, const std::list<Instance>>(); + SCOPED_TRACE("std::vector"); + InstanceRangedAssignTestForContainer<Instance, std::vector<Instance>>(); + SCOPED_TRACE("const std::vector"); + InstanceRangedAssignTestForContainer<Instance, const std::vector<Instance>>(); +} + +TEST(InitializerListConstructor, SimpleTypeWithInlineBacking) { + EXPECT_THAT((absl::InlinedVector<int, 4>{4, 5, 6}), + AllOf(SizeIs(3), CapacityIs(4), ElementsAre(4, 5, 6))); +} + +TEST(InitializerListConstructor, SimpleTypeWithReallocationRequired) { + EXPECT_THAT((absl::InlinedVector<int, 2>{4, 5, 6}), + AllOf(SizeIs(3), CapacityIs(Gt(2)), ElementsAre(4, 5, 6))); +} + +TEST(InitializerListConstructor, DisparateTypesInList) { + EXPECT_THAT((absl::InlinedVector<int, 2>{-7, 8ULL}), ElementsAre(-7, 8)); + + EXPECT_THAT((absl::InlinedVector<std::string, 2>{"foo", std::string("bar")}), + ElementsAre("foo", "bar")); +} + +TEST(InitializerListConstructor, ComplexTypeWithInlineBacking) { + EXPECT_THAT((absl::InlinedVector<CopyableMovableInstance, 1>{ + CopyableMovableInstance(0)}), + AllOf(SizeIs(1), CapacityIs(1), ElementsAre(ValueIs(0)))); +} + +TEST(InitializerListConstructor, ComplexTypeWithReallocationRequired) { + EXPECT_THAT( + (absl::InlinedVector<CopyableMovableInstance, 1>{ + CopyableMovableInstance(0), CopyableMovableInstance(1)}), + AllOf(SizeIs(2), CapacityIs(Gt(1)), ElementsAre(ValueIs(0), ValueIs(1)))); +} + +TEST(InitializerListAssign, SimpleTypeFitsInlineBacking) { + for (size_t original_size = 0; original_size <= 4; ++original_size) { + SCOPED_TRACE(original_size); + + absl::InlinedVector<int, 2> v1(original_size, 12345); + const size_t original_capacity_v1 = v1.capacity(); + v1.assign({3}); + EXPECT_THAT( + v1, AllOf(SizeIs(1), CapacityIs(original_capacity_v1), ElementsAre(3))); + + absl::InlinedVector<int, 2> v2(original_size, 12345); + const size_t original_capacity_v2 = v2.capacity(); + v2 = {3}; + EXPECT_THAT( + v2, AllOf(SizeIs(1), CapacityIs(original_capacity_v2), ElementsAre(3))); + } +} + +TEST(InitializerListAssign, SimpleTypeDoesNotFitInlineBacking) { + for (size_t original_size = 0; original_size <= 4; ++original_size) { + SCOPED_TRACE(original_size); + absl::InlinedVector<int, 2> v1(original_size, 12345); + v1.assign({3, 4, 5}); + EXPECT_THAT(v1, AllOf(SizeIs(3), ElementsAre(3, 4, 5))); + EXPECT_LE(3, v1.capacity()); + + absl::InlinedVector<int, 2> v2(original_size, 12345); + v2 = {3, 4, 5}; + EXPECT_THAT(v2, AllOf(SizeIs(3), ElementsAre(3, 4, 5))); + EXPECT_LE(3, v2.capacity()); + } +} + +TEST(InitializerListAssign, DisparateTypesInList) { + absl::InlinedVector<int, 2> v_int1; + v_int1.assign({-7, 8ULL}); + EXPECT_THAT(v_int1, ElementsAre(-7, 8)); + + absl::InlinedVector<int, 2> v_int2; + v_int2 = {-7, 8ULL}; + EXPECT_THAT(v_int2, ElementsAre(-7, 8)); + + absl::InlinedVector<std::string, 2> v_string1; + v_string1.assign({"foo", std::string("bar")}); + EXPECT_THAT(v_string1, ElementsAre("foo", "bar")); + + absl::InlinedVector<std::string, 2> v_string2; + v_string2 = {"foo", std::string("bar")}; + EXPECT_THAT(v_string2, ElementsAre("foo", "bar")); +} + +TYPED_TEST_P(InstanceTest, InitializerListAssign) { + using Instance = TypeParam; + for (size_t original_size = 0; original_size <= 4; ++original_size) { + SCOPED_TRACE(original_size); + absl::InlinedVector<Instance, 2> v(original_size, Instance(12345)); + const size_t original_capacity = v.capacity(); + v.assign({Instance(3)}); + EXPECT_THAT(v, AllOf(SizeIs(1), CapacityIs(original_capacity), + ElementsAre(ValueIs(3)))); + } + for (size_t original_size = 0; original_size <= 4; ++original_size) { + SCOPED_TRACE(original_size); + absl::InlinedVector<Instance, 2> v(original_size, Instance(12345)); + v.assign({Instance(3), Instance(4), Instance(5)}); + EXPECT_THAT( + v, AllOf(SizeIs(3), ElementsAre(ValueIs(3), ValueIs(4), ValueIs(5)))); + EXPECT_LE(3, v.capacity()); + } +} + +REGISTER_TYPED_TEST_CASE_P(InstanceTest, Swap, CountConstructorsDestructors, + CountConstructorsDestructorsOnCopyConstruction, + CountConstructorsDestructorsOnMoveConstruction, + CountConstructorsDestructorsOnAssignment, + CountConstructorsDestructorsOnMoveAssignment, + CountElemAssignInlineBacking, RangedConstructor, + RangedAssign, InitializerListAssign); + +using InstanceTypes = + ::testing::Types<CopyableOnlyInstance, CopyableMovableInstance>; +INSTANTIATE_TYPED_TEST_CASE_P(InstanceTestOnTypes, InstanceTest, InstanceTypes); + +TEST(DynamicVec, DynamicVecCompiles) { + DynamicVec v; + (void)v; +} + +TEST(AllocatorSupportTest, Constructors) { + using MyAlloc = CountingAllocator<int>; + using AllocVec = absl::InlinedVector<int, 4, MyAlloc>; + const int ia[] = {0, 1, 2, 3, 4, 5, 6, 7}; + int64_t allocated = 0; + MyAlloc alloc(&allocated); + { AllocVec ABSL_ATTRIBUTE_UNUSED v; } + { AllocVec ABSL_ATTRIBUTE_UNUSED v(alloc); } + { AllocVec ABSL_ATTRIBUTE_UNUSED v(ia, ia + ABSL_ARRAYSIZE(ia), alloc); } + { AllocVec ABSL_ATTRIBUTE_UNUSED v({1, 2, 3}, alloc); } + + AllocVec v2; + { AllocVec ABSL_ATTRIBUTE_UNUSED v(v2, alloc); } + { AllocVec ABSL_ATTRIBUTE_UNUSED v(std::move(v2), alloc); } +} + +TEST(AllocatorSupportTest, CountAllocations) { + using MyAlloc = CountingAllocator<int>; + using AllocVec = absl::InlinedVector<int, 4, MyAlloc>; + const int ia[] = {0, 1, 2, 3, 4, 5, 6, 7}; + int64_t allocated = 0; + MyAlloc alloc(&allocated); + { + AllocVec ABSL_ATTRIBUTE_UNUSED v(ia, ia + 4, alloc); + EXPECT_THAT(allocated, 0); + } + EXPECT_THAT(allocated, 0); + { + AllocVec ABSL_ATTRIBUTE_UNUSED v(ia, ia + ABSL_ARRAYSIZE(ia), alloc); + EXPECT_THAT(allocated, v.size() * sizeof(int)); + } + EXPECT_THAT(allocated, 0); + { + AllocVec v(4, 1, alloc); + EXPECT_THAT(allocated, 0); + + int64_t allocated2 = 0; + MyAlloc alloc2(&allocated2); + AllocVec v2(v, alloc2); + EXPECT_THAT(allocated2, 0); + + int64_t allocated3 = 0; + MyAlloc alloc3(&allocated3); + AllocVec v3(std::move(v), alloc3); + EXPECT_THAT(allocated3, 0); + } + EXPECT_THAT(allocated, 0); + { + AllocVec v(8, 2, alloc); + EXPECT_THAT(allocated, v.size() * sizeof(int)); + + int64_t allocated2 = 0; + MyAlloc alloc2(&allocated2); + AllocVec v2(v, alloc2); + EXPECT_THAT(allocated2, v2.size() * sizeof(int)); + + int64_t allocated3 = 0; + MyAlloc alloc3(&allocated3); + AllocVec v3(std::move(v), alloc3); + EXPECT_THAT(allocated3, v3.size() * sizeof(int)); + } + EXPECT_EQ(allocated, 0); + { + // Test shrink_to_fit deallocations. + AllocVec v(8, 2, alloc); + EXPECT_EQ(allocated, 8 * sizeof(int)); + v.resize(5); + EXPECT_EQ(allocated, 8 * sizeof(int)); + v.shrink_to_fit(); + EXPECT_EQ(allocated, 5 * sizeof(int)); + v.resize(4); + EXPECT_EQ(allocated, 5 * sizeof(int)); + v.shrink_to_fit(); + EXPECT_EQ(allocated, 0); + } +} + +TEST(AllocatorSupportTest, SwapBothAllocated) { + using MyAlloc = CountingAllocator<int>; + using AllocVec = absl::InlinedVector<int, 4, MyAlloc>; + int64_t allocated1 = 0; + int64_t allocated2 = 0; + { + const int ia1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + const int ia2[] = {0, 1, 2, 3, 4, 5, 6, 7, 8}; + MyAlloc a1(&allocated1); + MyAlloc a2(&allocated2); + AllocVec v1(ia1, ia1 + ABSL_ARRAYSIZE(ia1), a1); + AllocVec v2(ia2, ia2 + ABSL_ARRAYSIZE(ia2), a2); + EXPECT_LT(v1.capacity(), v2.capacity()); + EXPECT_THAT(allocated1, v1.capacity() * sizeof(int)); + EXPECT_THAT(allocated2, v2.capacity() * sizeof(int)); + v1.swap(v2); + EXPECT_THAT(v1, ElementsAreArray(ia2)); + EXPECT_THAT(v2, ElementsAreArray(ia1)); + EXPECT_THAT(allocated1, v2.capacity() * sizeof(int)); + EXPECT_THAT(allocated2, v1.capacity() * sizeof(int)); + } + EXPECT_THAT(allocated1, 0); + EXPECT_THAT(allocated2, 0); +} + +TEST(AllocatorSupportTest, SwapOneAllocated) { + using MyAlloc = CountingAllocator<int>; + using AllocVec = absl::InlinedVector<int, 4, MyAlloc>; + int64_t allocated1 = 0; + int64_t allocated2 = 0; + { + const int ia1[] = {0, 1, 2, 3, 4, 5, 6, 7}; + const int ia2[] = {0, 1, 2, 3}; + MyAlloc a1(&allocated1); + MyAlloc a2(&allocated2); + AllocVec v1(ia1, ia1 + ABSL_ARRAYSIZE(ia1), a1); + AllocVec v2(ia2, ia2 + ABSL_ARRAYSIZE(ia2), a2); + EXPECT_THAT(allocated1, v1.capacity() * sizeof(int)); + EXPECT_THAT(allocated2, 0); + v1.swap(v2); + EXPECT_THAT(v1, ElementsAreArray(ia2)); + EXPECT_THAT(v2, ElementsAreArray(ia1)); + EXPECT_THAT(allocated1, v2.capacity() * sizeof(int)); + EXPECT_THAT(allocated2, 0); + EXPECT_TRUE(v2.get_allocator() == a1); + EXPECT_TRUE(v1.get_allocator() == a2); + } + EXPECT_THAT(allocated1, 0); + EXPECT_THAT(allocated2, 0); +} + +TEST(AllocatorSupportTest, ScopedAllocatorWorksInlined) { + using StdVector = std::vector<int, CountingAllocator<int>>; + using Alloc = CountingAllocator<StdVector>; + using ScopedAlloc = std::scoped_allocator_adaptor<Alloc>; + using AllocVec = absl::InlinedVector<StdVector, 1, ScopedAlloc>; + + int64_t total_allocated_byte_count = 0; + + AllocVec inlined_case(ScopedAlloc(Alloc(+&total_allocated_byte_count))); + + // Called only once to remain inlined + inlined_case.emplace_back(); + + int64_t absl_responsible_for_count = total_allocated_byte_count; + + // MSVC's allocator preemptively allocates in debug mode +#if !defined(_MSC_VER) + EXPECT_EQ(absl_responsible_for_count, 0); +#endif // !defined(_MSC_VER) + + inlined_case[0].emplace_back(); + EXPECT_GT(total_allocated_byte_count, absl_responsible_for_count); + + inlined_case.clear(); + inlined_case.shrink_to_fit(); + EXPECT_EQ(total_allocated_byte_count, 0); +} + +TEST(AllocatorSupportTest, ScopedAllocatorWorksAllocated) { + using StdVector = std::vector<int, CountingAllocator<int>>; + using Alloc = CountingAllocator<StdVector>; + using ScopedAlloc = std::scoped_allocator_adaptor<Alloc>; + using AllocVec = absl::InlinedVector<StdVector, 1, ScopedAlloc>; + + int64_t total_allocated_byte_count = 0; + + AllocVec allocated_case(ScopedAlloc(Alloc(+&total_allocated_byte_count))); + + // Called twice to force into being allocated + allocated_case.emplace_back(); + allocated_case.emplace_back(); + + int64_t absl_responsible_for_count = total_allocated_byte_count; + EXPECT_GT(absl_responsible_for_count, 0); + + allocated_case[1].emplace_back(); + EXPECT_GT(total_allocated_byte_count, absl_responsible_for_count); + + allocated_case.clear(); + allocated_case.shrink_to_fit(); + EXPECT_EQ(total_allocated_byte_count, 0); +} + +TEST(AllocatorSupportTest, SizeAllocConstructor) { + constexpr int inlined_size = 4; + using Alloc = CountingAllocator<int>; + using AllocVec = absl::InlinedVector<int, inlined_size, Alloc>; + + { + auto len = inlined_size / 2; + int64_t allocated = 0; + auto v = AllocVec(len, Alloc(&allocated)); + + // Inline storage used; allocator should not be invoked + EXPECT_THAT(allocated, 0); + EXPECT_THAT(v, AllOf(SizeIs(len), Each(0))); + } + + { + auto len = inlined_size * 2; + int64_t allocated = 0; + auto v = AllocVec(len, Alloc(&allocated)); + + // Out of line storage used; allocation of 8 elements expected + EXPECT_THAT(allocated, len * sizeof(int)); + EXPECT_THAT(v, AllOf(SizeIs(len), Each(0))); + } +} + +TEST(InlinedVectorTest, MinimumAllocatorCompilesUsingTraits) { + using T = int; + using A = std::allocator<T>; + using ATraits = absl::allocator_traits<A>; + + struct MinimumAllocator { + using value_type = T; + + value_type* allocate(size_t n) { + A a; + return ATraits::allocate(a, n); + } + + void deallocate(value_type* p, size_t n) { + A a; + ATraits::deallocate(a, p, n); + } + }; + + absl::InlinedVector<T, 1, MinimumAllocator> vec; + vec.emplace_back(); + vec.resize(0); +} + +TEST(InlinedVectorTest, AbslHashValueWorks) { + using V = absl::InlinedVector<int, 4>; + std::vector<V> cases; + + // Generate a variety of vectors some of these are small enough for the inline + // space but are stored out of line. + for (int i = 0; i < 10; ++i) { + V v; + for (int j = 0; j < i; ++j) { + v.push_back(j); + } + cases.push_back(v); + v.resize(i % 4); + cases.push_back(v); + } + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(cases)); +} + +} // anonymous namespace diff --git a/third_party/abseil_cpp/absl/container/internal/btree.h b/third_party/abseil_cpp/absl/container/internal/btree.h new file mode 100644 index 000000000000..f2fc31df8d41 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/btree.h @@ -0,0 +1,2587 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A btree implementation of the STL set and map interfaces. A btree is smaller +// and generally also faster than STL set/map (refer to the benchmarks below). +// The red-black tree implementation of STL set/map has an overhead of 3 +// pointers (left, right and parent) plus the node color information for each +// stored value. So a set<int32_t> consumes 40 bytes for each value stored in +// 64-bit mode. This btree implementation stores multiple values on fixed +// size nodes (usually 256 bytes) and doesn't store child pointers for leaf +// nodes. The result is that a btree_set<int32_t> may use much less memory per +// stored value. For the random insertion benchmark in btree_bench.cc, a +// btree_set<int32_t> with node-size of 256 uses 5.1 bytes per stored value. +// +// The packing of multiple values on to each node of a btree has another effect +// besides better space utilization: better cache locality due to fewer cache +// lines being accessed. Better cache locality translates into faster +// operations. +// +// CAVEATS +// +// Insertions and deletions on a btree can cause splitting, merging or +// rebalancing of btree nodes. And even without these operations, insertions +// and deletions on a btree will move values around within a node. In both +// cases, the result is that insertions and deletions can invalidate iterators +// pointing to values other than the one being inserted/deleted. Therefore, this +// container does not provide pointer stability. This is notably different from +// STL set/map which takes care to not invalidate iterators on insert/erase +// except, of course, for iterators pointing to the value being erased. A +// partial workaround when erasing is available: erase() returns an iterator +// pointing to the item just after the one that was erased (or end() if none +// exists). + +#ifndef ABSL_CONTAINER_INTERNAL_BTREE_H_ +#define ABSL_CONTAINER_INTERNAL_BTREE_H_ + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <functional> +#include <iterator> +#include <limits> +#include <new> +#include <string> +#include <type_traits> +#include <utility> + +#include "absl/base/macros.h" +#include "absl/container/internal/common.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/layout.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/cord.h" +#include "absl/strings/string_view.h" +#include "absl/types/compare.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// A helper class that indicates if the Compare parameter is a key-compare-to +// comparator. +template <typename Compare, typename T> +using btree_is_key_compare_to = + std::is_convertible<absl::result_of_t<Compare(const T &, const T &)>, + absl::weak_ordering>; + +struct StringBtreeDefaultLess { + using is_transparent = void; + + StringBtreeDefaultLess() = default; + + // Compatibility constructor. + StringBtreeDefaultLess(std::less<std::string>) {} // NOLINT + StringBtreeDefaultLess(std::less<string_view>) {} // NOLINT + + absl::weak_ordering operator()(absl::string_view lhs, + absl::string_view rhs) const { + return compare_internal::compare_result_as_ordering(lhs.compare(rhs)); + } + StringBtreeDefaultLess(std::less<absl::Cord>) {} // NOLINT + absl::weak_ordering operator()(const absl::Cord &lhs, + const absl::Cord &rhs) const { + return compare_internal::compare_result_as_ordering(lhs.Compare(rhs)); + } + absl::weak_ordering operator()(const absl::Cord &lhs, + absl::string_view rhs) const { + return compare_internal::compare_result_as_ordering(lhs.Compare(rhs)); + } + absl::weak_ordering operator()(absl::string_view lhs, + const absl::Cord &rhs) const { + return compare_internal::compare_result_as_ordering(-rhs.Compare(lhs)); + } +}; + +struct StringBtreeDefaultGreater { + using is_transparent = void; + + StringBtreeDefaultGreater() = default; + + StringBtreeDefaultGreater(std::greater<std::string>) {} // NOLINT + StringBtreeDefaultGreater(std::greater<string_view>) {} // NOLINT + + absl::weak_ordering operator()(absl::string_view lhs, + absl::string_view rhs) const { + return compare_internal::compare_result_as_ordering(rhs.compare(lhs)); + } + StringBtreeDefaultGreater(std::greater<absl::Cord>) {} // NOLINT + absl::weak_ordering operator()(const absl::Cord &lhs, + const absl::Cord &rhs) const { + return compare_internal::compare_result_as_ordering(rhs.Compare(lhs)); + } + absl::weak_ordering operator()(const absl::Cord &lhs, + absl::string_view rhs) const { + return compare_internal::compare_result_as_ordering(-lhs.Compare(rhs)); + } + absl::weak_ordering operator()(absl::string_view lhs, + const absl::Cord &rhs) const { + return compare_internal::compare_result_as_ordering(rhs.Compare(lhs)); + } +}; + +// A helper class to convert a boolean comparison into a three-way "compare-to" +// comparison that returns an `absl::weak_ordering`. This helper +// class is specialized for less<std::string>, greater<std::string>, +// less<string_view>, greater<string_view>, less<absl::Cord>, and +// greater<absl::Cord>. +// +// key_compare_to_adapter is provided so that btree users +// automatically get the more efficient compare-to code when using common +// Abseil string types with common comparison functors. +// These string-like specializations also turn on heterogeneous lookup by +// default. +template <typename Compare> +struct key_compare_to_adapter { + using type = Compare; +}; + +template <> +struct key_compare_to_adapter<std::less<std::string>> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter<std::greater<std::string>> { + using type = StringBtreeDefaultGreater; +}; + +template <> +struct key_compare_to_adapter<std::less<absl::string_view>> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter<std::greater<absl::string_view>> { + using type = StringBtreeDefaultGreater; +}; + +template <> +struct key_compare_to_adapter<std::less<absl::Cord>> { + using type = StringBtreeDefaultLess; +}; + +template <> +struct key_compare_to_adapter<std::greater<absl::Cord>> { + using type = StringBtreeDefaultGreater; +}; + +// Detects an 'absl_btree_prefer_linear_node_search' member. This is +// a protocol used as an opt-in or opt-out of linear search. +// +// For example, this would be useful for key types that wrap an integer +// and define their own cheap operator<(). For example: +// +// class K { +// public: +// using absl_btree_prefer_linear_node_search = std::true_type; +// ... +// private: +// friend bool operator<(K a, K b) { return a.k_ < b.k_; } +// int k_; +// }; +// +// btree_map<K, V> m; // Uses linear search +// +// If T has the preference tag, then it has a preference. +// Btree will use the tag's truth value. +template <typename T, typename = void> +struct has_linear_node_search_preference : std::false_type {}; +template <typename T, typename = void> +struct prefers_linear_node_search : std::false_type {}; +template <typename T> +struct has_linear_node_search_preference< + T, absl::void_t<typename T::absl_btree_prefer_linear_node_search>> + : std::true_type {}; +template <typename T> +struct prefers_linear_node_search< + T, absl::void_t<typename T::absl_btree_prefer_linear_node_search>> + : T::absl_btree_prefer_linear_node_search {}; + +template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, + bool Multi, typename SlotPolicy> +struct common_params { + // If Compare is a common comparator for a string-like type, then we adapt it + // to use heterogeneous lookup and to be a key-compare-to comparator. + using key_compare = typename key_compare_to_adapter<Compare>::type; + // True when key_compare has been adapted to StringBtreeDefault{Less,Greater}. + using is_key_compare_adapted = + absl::negation<std::is_same<key_compare, Compare>>; + // A type which indicates if we have a key-compare-to functor or a plain old + // key-compare functor. + using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>; + + using allocator_type = Alloc; + using key_type = Key; + using size_type = std::make_signed<size_t>::type; + using difference_type = ptrdiff_t; + + // True if this is a multiset or multimap. + using is_multi_container = std::integral_constant<bool, Multi>; + + using slot_policy = SlotPolicy; + using slot_type = typename slot_policy::slot_type; + using value_type = typename slot_policy::value_type; + using init_type = typename slot_policy::mutable_value_type; + using pointer = value_type *; + using const_pointer = const value_type *; + using reference = value_type &; + using const_reference = const value_type &; + + enum { + kTargetNodeSize = TargetNodeSize, + + // Upper bound for the available space for values. This is largest for leaf + // nodes, which have overhead of at least a pointer + 4 bytes (for storing + // 3 field_types and an enum). + kNodeValueSpace = + TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), + }; + + // This is an integral type large enough to hold as many + // ValueSize-values as will fit a node of TargetNodeSize bytes. + using node_count_type = + absl::conditional_t<(kNodeValueSpace / sizeof(value_type) > + (std::numeric_limits<uint8_t>::max)()), + uint16_t, uint8_t>; // NOLINT + + // The following methods are necessary for passing this struct as PolicyTraits + // for node_handle and/or are used within btree. + static value_type &element(slot_type *slot) { + return slot_policy::element(slot); + } + static const value_type &element(const slot_type *slot) { + return slot_policy::element(slot); + } + template <class... Args> + static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { + slot_policy::construct(alloc, slot, std::forward<Args>(args)...); + } + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + slot_policy::construct(alloc, slot, other); + } + static void destroy(Alloc *alloc, slot_type *slot) { + slot_policy::destroy(alloc, slot); + } + static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { + construct(alloc, new_slot, old_slot); + destroy(alloc, old_slot); + } + static void swap(Alloc *alloc, slot_type *a, slot_type *b) { + slot_policy::swap(alloc, a, b); + } + static void move(Alloc *alloc, slot_type *src, slot_type *dest) { + slot_policy::move(alloc, src, dest); + } +}; + +// A parameters structure for holding the type parameters for a btree_map. +// Compare and Alloc should be nothrow copy-constructible. +template <typename Key, typename Data, typename Compare, typename Alloc, + int TargetNodeSize, bool Multi> +struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, + map_slot_policy<Key, Data>> { + using super_type = typename map_params::common_params; + using mapped_type = Data; + // This type allows us to move keys when it is safe to do so. It is safe + // for maps in which value_type and mutable_value_type are layout compatible. + using slot_policy = typename super_type::slot_policy; + using slot_type = typename super_type::slot_type; + using value_type = typename super_type::value_type; + using init_type = typename super_type::init_type; + + using key_compare = typename super_type::key_compare; + // Inherit from key_compare for empty base class optimization. + struct value_compare : private key_compare { + value_compare() = default; + explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} + + template <typename T, typename U> + auto operator()(const T &left, const U &right) const + -> decltype(std::declval<key_compare>()(left.first, right.first)) { + return key_compare::operator()(left.first, right.first); + } + }; + using is_map_container = std::true_type; + + template <typename V> + static auto key(const V &value) -> decltype(value.first) { + return value.first; + } + static const Key &key(const slot_type *s) { return slot_policy::key(s); } + static const Key &key(slot_type *s) { return slot_policy::key(s); } + // For use in node handle. + static auto mutable_key(slot_type *s) + -> decltype(slot_policy::mutable_key(s)) { + return slot_policy::mutable_key(s); + } + static mapped_type &value(value_type *value) { return value->second; } +}; + +// This type implements the necessary functions from the +// absl::container_internal::slot_type interface. +template <typename Key> +struct set_slot_policy { + using slot_type = Key; + using value_type = Key; + using mutable_value_type = Key; + + static value_type &element(slot_type *slot) { return *slot; } + static const value_type &element(const slot_type *slot) { return *slot; } + + template <typename Alloc, class... Args> + static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { + absl::allocator_traits<Alloc>::construct(*alloc, slot, + std::forward<Args>(args)...); + } + + template <typename Alloc> + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + absl::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other)); + } + + template <typename Alloc> + static void destroy(Alloc *alloc, slot_type *slot) { + absl::allocator_traits<Alloc>::destroy(*alloc, slot); + } + + template <typename Alloc> + static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { + using std::swap; + swap(*a, *b); + } + + template <typename Alloc> + static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { + *dest = std::move(*src); + } +}; + +// A parameters structure for holding the type parameters for a btree_set. +// Compare and Alloc should be nothrow copy-constructible. +template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, + bool Multi> +struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, + set_slot_policy<Key>> { + using value_type = Key; + using slot_type = typename set_params::common_params::slot_type; + using value_compare = typename set_params::common_params::key_compare; + using is_map_container = std::false_type; + + template <typename V> + static const V &key(const V &value) { return value; } + static const Key &key(const slot_type *slot) { return *slot; } + static const Key &key(slot_type *slot) { return *slot; } +}; + +// An adapter class that converts a lower-bound compare into an upper-bound +// compare. Note: there is no need to make a version of this adapter specialized +// for key-compare-to functors because the upper-bound (the first value greater +// than the input) is never an exact match. +template <typename Compare> +struct upper_bound_adapter { + explicit upper_bound_adapter(const Compare &c) : comp(c) {} + template <typename K1, typename K2> + bool operator()(const K1 &a, const K2 &b) const { + // Returns true when a is not greater than b. + return !compare_internal::compare_result_as_less_than(comp(b, a)); + } + + private: + Compare comp; +}; + +enum class MatchKind : uint8_t { kEq, kNe }; + +template <typename V, bool IsCompareTo> +struct SearchResult { + V value; + MatchKind match; + + static constexpr bool HasMatch() { return true; } + bool IsEq() const { return match == MatchKind::kEq; } +}; + +// When we don't use CompareTo, `match` is not present. +// This ensures that callers can't use it accidentally when it provides no +// useful information. +template <typename V> +struct SearchResult<V, false> { + SearchResult() {} + explicit SearchResult(V value) : value(value) {} + SearchResult(V value, MatchKind /*match*/) : value(value) {} + + V value; + + static constexpr bool HasMatch() { return false; } + static constexpr bool IsEq() { return false; } +}; + +// A node in the btree holding. The same node type is used for both internal +// and leaf nodes in the btree, though the nodes are allocated in such a way +// that the children array is only valid in internal nodes. +template <typename Params> +class btree_node { + using is_key_compare_to = typename Params::is_key_compare_to; + using is_multi_container = typename Params::is_multi_container; + using field_type = typename Params::node_count_type; + using allocator_type = typename Params::allocator_type; + using slot_type = typename Params::slot_type; + + public: + using params_type = Params; + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using key_compare = typename Params::key_compare; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + + // Btree decides whether to use linear node search as follows: + // - If the comparator expresses a preference, use that. + // - If the key expresses a preference, use that. + // - If the key is arithmetic and the comparator is std::less or + // std::greater, choose linear. + // - Otherwise, choose binary. + // TODO(ezb): Might make sense to add condition(s) based on node-size. + using use_linear_search = std::integral_constant< + bool, + has_linear_node_search_preference<key_compare>::value + ? prefers_linear_node_search<key_compare>::value + : has_linear_node_search_preference<key_type>::value + ? prefers_linear_node_search<key_type>::value + : std::is_arithmetic<key_type>::value && + (std::is_same<std::less<key_type>, key_compare>::value || + std::is_same<std::greater<key_type>, + key_compare>::value)>; + + // This class is organized by gtl::Layout as if it had the following + // structure: + // // A pointer to the node's parent. + // btree_node *parent; + // + // // The position of the node in the node's parent. + // field_type position; + // // The index of the first populated value in `values`. + // // TODO(ezb): right now, `start` is always 0. Update insertion/merge + // // logic to allow for floating storage within nodes. + // field_type start; + // // The index after the last populated value in `values`. Currently, this + // // is the same as the count of values. + // field_type finish; + // // The maximum number of values the node can hold. This is an integer in + // // [1, kNodeValues] for root leaf nodes, kNodeValues for non-root leaf + // // nodes, and kInternalNodeMaxCount (as a sentinel value) for internal + // // nodes (even though there are still kNodeValues values in the node). + // // TODO(ezb): make max_count use only 4 bits and record log2(capacity) + // // to free extra bits for is_root, etc. + // field_type max_count; + // + // // The array of values. The capacity is `max_count` for leaf nodes and + // // kNodeValues for internal nodes. Only the values in + // // [start, finish) have been initialized and are valid. + // slot_type values[max_count]; + // + // // The array of child pointers. The keys in children[i] are all less + // // than key(i). The keys in children[i + 1] are all greater than key(i). + // // There are 0 children for leaf nodes and kNodeValues + 1 children for + // // internal nodes. + // btree_node *children[kNodeValues + 1]; + // + // This class is only constructed by EmptyNodeType. Normally, pointers to the + // layout above are allocated, cast to btree_node*, and de-allocated within + // the btree implementation. + ~btree_node() = default; + btree_node(btree_node const &) = delete; + btree_node &operator=(btree_node const &) = delete; + + // Public for EmptyNodeType. + constexpr static size_type Alignment() { + static_assert(LeafLayout(1).Alignment() == InternalLayout().Alignment(), + "Alignment of all nodes must be equal."); + return InternalLayout().Alignment(); + } + + protected: + btree_node() = default; + + private: + using layout_type = absl::container_internal::Layout<btree_node *, field_type, + slot_type, btree_node *>; + constexpr static size_type SizeWithNValues(size_type n) { + return layout_type(/*parent*/ 1, + /*position, start, finish, max_count*/ 4, + /*values*/ n, + /*children*/ 0) + .AllocSize(); + } + // A lower bound for the overhead of fields other than values in a leaf node. + constexpr static size_type MinimumOverhead() { + return SizeWithNValues(1) - sizeof(value_type); + } + + // Compute how many values we can fit onto a leaf node taking into account + // padding. + constexpr static size_type NodeTargetValues(const int begin, const int end) { + return begin == end ? begin + : SizeWithNValues((begin + end) / 2 + 1) > + params_type::kTargetNodeSize + ? NodeTargetValues(begin, (begin + end) / 2) + : NodeTargetValues((begin + end) / 2 + 1, end); + } + + enum { + kTargetNodeSize = params_type::kTargetNodeSize, + kNodeTargetValues = NodeTargetValues(0, params_type::kTargetNodeSize), + + // We need a minimum of 3 values per internal node in order to perform + // splitting (1 value for the two nodes involved in the split and 1 value + // propagated to the parent as the delimiter for the split). + kNodeValues = kNodeTargetValues >= 3 ? kNodeTargetValues : 3, + + // The node is internal (i.e. is not a leaf node) if and only if `max_count` + // has this value. + kInternalNodeMaxCount = 0, + }; + + // Leaves can have less than kNodeValues values. + constexpr static layout_type LeafLayout(const int max_values = kNodeValues) { + return layout_type(/*parent*/ 1, + /*position, start, finish, max_count*/ 4, + /*values*/ max_values, + /*children*/ 0); + } + constexpr static layout_type InternalLayout() { + return layout_type(/*parent*/ 1, + /*position, start, finish, max_count*/ 4, + /*values*/ kNodeValues, + /*children*/ kNodeValues + 1); + } + constexpr static size_type LeafSize(const int max_values = kNodeValues) { + return LeafLayout(max_values).AllocSize(); + } + constexpr static size_type InternalSize() { + return InternalLayout().AllocSize(); + } + + // N is the index of the type in the Layout definition. + // ElementType<N> is the Nth type in the Layout definition. + template <size_type N> + inline typename layout_type::template ElementType<N> *GetField() { + // We assert that we don't read from values that aren't there. + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer<N>(reinterpret_cast<char *>(this)); + } + template <size_type N> + inline const typename layout_type::template ElementType<N> *GetField() const { + assert(N < 3 || !leaf()); + return InternalLayout().template Pointer<N>( + reinterpret_cast<const char *>(this)); + } + void set_parent(btree_node *p) { *GetField<0>() = p; } + field_type &mutable_finish() { return GetField<1>()[2]; } + slot_type *slot(int i) { return &GetField<2>()[i]; } + slot_type *start_slot() { return slot(start()); } + slot_type *finish_slot() { return slot(finish()); } + const slot_type *slot(int i) const { return &GetField<2>()[i]; } + void set_position(field_type v) { GetField<1>()[0] = v; } + void set_start(field_type v) { GetField<1>()[1] = v; } + void set_finish(field_type v) { GetField<1>()[2] = v; } + // This method is only called by the node init methods. + void set_max_count(field_type v) { GetField<1>()[3] = v; } + + public: + // Whether this is a leaf node or not. This value doesn't change after the + // node is created. + bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } + + // Getter for the position of this node in its parent. + field_type position() const { return GetField<1>()[0]; } + + // Getter for the offset of the first value in the `values` array. + field_type start() const { + // TODO(ezb): when floating storage is implemented, return GetField<1>()[1]; + assert(GetField<1>()[1] == 0); + return 0; + } + + // Getter for the offset after the last value in the `values` array. + field_type finish() const { return GetField<1>()[2]; } + + // Getters for the number of values stored in this node. + field_type count() const { + assert(finish() >= start()); + return finish() - start(); + } + field_type max_count() const { + // Internal nodes have max_count==kInternalNodeMaxCount. + // Leaf nodes have max_count in [1, kNodeValues]. + const field_type max_count = GetField<1>()[3]; + return max_count == field_type{kInternalNodeMaxCount} + ? field_type{kNodeValues} + : max_count; + } + + // Getter for the parent of this node. + btree_node *parent() const { return *GetField<0>(); } + // Getter for whether the node is the root of the tree. The parent of the + // root of the tree is the leftmost node in the tree which is guaranteed to + // be a leaf. + bool is_root() const { return parent()->leaf(); } + void make_root() { + assert(parent()->is_root()); + set_parent(parent()->parent()); + } + + // Getters for the key/value at position i in the node. + const key_type &key(int i) const { return params_type::key(slot(i)); } + reference value(int i) { return params_type::element(slot(i)); } + const_reference value(int i) const { return params_type::element(slot(i)); } + + // Getters/setter for the child at position i in the node. + btree_node *child(int i) const { return GetField<3>()[i]; } + btree_node *start_child() const { return child(start()); } + btree_node *&mutable_child(int i) { return GetField<3>()[i]; } + void clear_child(int i) { + absl::container_internal::SanitizerPoisonObject(&mutable_child(i)); + } + void set_child(int i, btree_node *c) { + absl::container_internal::SanitizerUnpoisonObject(&mutable_child(i)); + mutable_child(i) = c; + c->set_position(i); + } + void init_child(int i, btree_node *c) { + set_child(i, c); + c->set_parent(this); + } + + // Returns the position of the first value whose key is not less than k. + template <typename K> + SearchResult<int, is_key_compare_to::value> lower_bound( + const K &k, const key_compare &comp) const { + return use_linear_search::value ? linear_search(k, comp) + : binary_search(k, comp); + } + // Returns the position of the first value whose key is greater than k. + template <typename K> + int upper_bound(const K &k, const key_compare &comp) const { + auto upper_compare = upper_bound_adapter<key_compare>(comp); + return use_linear_search::value ? linear_search(k, upper_compare).value + : binary_search(k, upper_compare).value; + } + + template <typename K, typename Compare> + SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> + linear_search(const K &k, const Compare &comp) const { + return linear_search_impl(k, start(), finish(), comp, + btree_is_key_compare_to<Compare, key_type>()); + } + + template <typename K, typename Compare> + SearchResult<int, btree_is_key_compare_to<Compare, key_type>::value> + binary_search(const K &k, const Compare &comp) const { + return binary_search_impl(k, start(), finish(), comp, + btree_is_key_compare_to<Compare, key_type>()); + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using plain compare. + template <typename K, typename Compare> + SearchResult<int, false> linear_search_impl( + const K &k, int s, const int e, const Compare &comp, + std::false_type /* IsCompareTo */) const { + while (s < e) { + if (!comp(key(s), k)) { + break; + } + ++s; + } + return SearchResult<int, false>{s}; + } + + // Returns the position of the first value whose key is not less than k using + // linear search performed using compare-to. + template <typename K, typename Compare> + SearchResult<int, true> linear_search_impl( + const K &k, int s, const int e, const Compare &comp, + std::true_type /* IsCompareTo */) const { + while (s < e) { + const absl::weak_ordering c = comp(key(s), k); + if (c == 0) { + return {s, MatchKind::kEq}; + } else if (c > 0) { + break; + } + ++s; + } + return {s, MatchKind::kNe}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using plain compare. + template <typename K, typename Compare> + SearchResult<int, false> binary_search_impl( + const K &k, int s, int e, const Compare &comp, + std::false_type /* IsCompareTo */) const { + while (s != e) { + const int mid = (s + e) >> 1; + if (comp(key(mid), k)) { + s = mid + 1; + } else { + e = mid; + } + } + return SearchResult<int, false>{s}; + } + + // Returns the position of the first value whose key is not less than k using + // binary search performed using compare-to. + template <typename K, typename CompareTo> + SearchResult<int, true> binary_search_impl( + const K &k, int s, int e, const CompareTo &comp, + std::true_type /* IsCompareTo */) const { + if (is_multi_container::value) { + MatchKind exact_match = MatchKind::kNe; + while (s != e) { + const int mid = (s + e) >> 1; + const absl::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else { + e = mid; + if (c == 0) { + // Need to return the first value whose key is not less than k, + // which requires continuing the binary search if this is a + // multi-container. + exact_match = MatchKind::kEq; + } + } + } + return {s, exact_match}; + } else { // Not a multi-container. + while (s != e) { + const int mid = (s + e) >> 1; + const absl::weak_ordering c = comp(key(mid), k); + if (c < 0) { + s = mid + 1; + } else if (c > 0) { + e = mid; + } else { + return {mid, MatchKind::kEq}; + } + } + return {s, MatchKind::kNe}; + } + } + + // Emplaces a value at position i, shifting all existing values and + // children at positions >= i to the right by 1. + template <typename... Args> + void emplace_value(size_type i, allocator_type *alloc, Args &&... args); + + // Removes the values at positions [i, i + to_erase), shifting all existing + // values and children after that range to the left by to_erase. Clears all + // children between [i, i + to_erase). + void remove_values(field_type i, field_type to_erase, allocator_type *alloc); + + // Rebalances a node with its right sibling. + void rebalance_right_to_left(int to_move, btree_node *right, + allocator_type *alloc); + void rebalance_left_to_right(int to_move, btree_node *right, + allocator_type *alloc); + + // Splits a node, moving a portion of the node's values to its right sibling. + void split(int insert_position, btree_node *dest, allocator_type *alloc); + + // Merges a node with its right sibling, moving all of the values and the + // delimiting key in the parent node onto itself, and deleting the src node. + void merge(btree_node *src, allocator_type *alloc); + + // Node allocation/deletion routines. + void init_leaf(btree_node *parent, int max_count) { + set_parent(parent); + set_position(0); + set_start(0); + set_finish(0); + set_max_count(max_count); + absl::container_internal::SanitizerPoisonMemoryRegion( + start_slot(), max_count * sizeof(slot_type)); + } + void init_internal(btree_node *parent) { + init_leaf(parent, kNodeValues); + // Set `max_count` to a sentinel value to indicate that this node is + // internal. + set_max_count(kInternalNodeMaxCount); + absl::container_internal::SanitizerPoisonMemoryRegion( + &mutable_child(start()), (kNodeValues + 1) * sizeof(btree_node *)); + } + + static void deallocate(const size_type size, btree_node *node, + allocator_type *alloc) { + absl::container_internal::Deallocate<Alignment()>(alloc, node, size); + } + + // Deletes a node and all of its children. + static void clear_and_delete(btree_node *node, allocator_type *alloc); + + private: + template <typename... Args> + void value_init(const field_type i, allocator_type *alloc, Args &&... args) { + absl::container_internal::SanitizerUnpoisonObject(slot(i)); + params_type::construct(alloc, slot(i), std::forward<Args>(args)...); + } + void value_destroy(const field_type i, allocator_type *alloc) { + params_type::destroy(alloc, slot(i)); + absl::container_internal::SanitizerPoisonObject(slot(i)); + } + void value_destroy_n(const field_type i, const field_type n, + allocator_type *alloc) { + for (slot_type *s = slot(i), *end = slot(i + n); s != end; ++s) { + params_type::destroy(alloc, s); + absl::container_internal::SanitizerPoisonObject(s); + } + } + + static void transfer(slot_type *dest, slot_type *src, allocator_type *alloc) { + absl::container_internal::SanitizerUnpoisonObject(dest); + params_type::transfer(alloc, dest, src); + absl::container_internal::SanitizerPoisonObject(src); + } + + // Transfers value from slot `src_i` in `src_node` to slot `dest_i` in `this`. + void transfer(const size_type dest_i, const size_type src_i, + btree_node *src_node, allocator_type *alloc) { + transfer(slot(dest_i), src_node->slot(src_i), alloc); + } + + // Transfers `n` values starting at value `src_i` in `src_node` into the + // values starting at value `dest_i` in `this`. + void transfer_n(const size_type n, const size_type dest_i, + const size_type src_i, btree_node *src_node, + allocator_type *alloc) { + for (slot_type *src = src_node->slot(src_i), *end = src + n, + *dest = slot(dest_i); + src != end; ++src, ++dest) { + transfer(dest, src, alloc); + } + } + + // Same as above, except that we start at the end and work our way to the + // beginning. + void transfer_n_backward(const size_type n, const size_type dest_i, + const size_type src_i, btree_node *src_node, + allocator_type *alloc) { + for (slot_type *src = src_node->slot(src_i + n - 1), *end = src - n, + *dest = slot(dest_i + n - 1); + src != end; --src, --dest) { + transfer(dest, src, alloc); + } + } + + template <typename P> + friend class btree; + template <typename N, typename R, typename P> + friend struct btree_iterator; + friend class BtreeNodePeer; +}; + +template <typename Node, typename Reference, typename Pointer> +struct btree_iterator { + private: + using key_type = typename Node::key_type; + using size_type = typename Node::size_type; + using params_type = typename Node::params_type; + + using node_type = Node; + using normal_node = typename std::remove_const<Node>::type; + using const_node = const Node; + using normal_pointer = typename params_type::pointer; + using normal_reference = typename params_type::reference; + using const_pointer = typename params_type::const_pointer; + using const_reference = typename params_type::const_reference; + using slot_type = typename params_type::slot_type; + + using iterator = + btree_iterator<normal_node, normal_reference, normal_pointer>; + using const_iterator = + btree_iterator<const_node, const_reference, const_pointer>; + + public: + // These aliases are public for std::iterator_traits. + using difference_type = typename Node::difference_type; + using value_type = typename params_type::value_type; + using pointer = Pointer; + using reference = Reference; + using iterator_category = std::bidirectional_iterator_tag; + + btree_iterator() : node(nullptr), position(-1) {} + explicit btree_iterator(Node *n) : node(n), position(n->start()) {} + btree_iterator(Node *n, int p) : node(n), position(p) {} + + // NOTE: this SFINAE allows for implicit conversions from iterator to + // const_iterator, but it specifically avoids defining copy constructors so + // that btree_iterator can be trivially copyable. This is for performance and + // binary size reasons. + template <typename N, typename R, typename P, + absl::enable_if_t< + std::is_same<btree_iterator<N, R, P>, iterator>::value && + std::is_same<btree_iterator, const_iterator>::value, + int> = 0> + btree_iterator(const btree_iterator<N, R, P> &other) // NOLINT + : node(other.node), position(other.position) {} + + private: + // This SFINAE allows explicit conversions from const_iterator to + // iterator, but also avoids defining a copy constructor. + // NOTE: the const_cast is safe because this constructor is only called by + // non-const methods and the container owns the nodes. + template <typename N, typename R, typename P, + absl::enable_if_t< + std::is_same<btree_iterator<N, R, P>, const_iterator>::value && + std::is_same<btree_iterator, iterator>::value, + int> = 0> + explicit btree_iterator(const btree_iterator<N, R, P> &other) + : node(const_cast<node_type *>(other.node)), position(other.position) {} + + // Increment/decrement the iterator. + void increment() { + if (node->leaf() && ++position < node->finish()) { + return; + } + increment_slow(); + } + void increment_slow(); + + void decrement() { + if (node->leaf() && --position >= node->start()) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + public: + bool operator==(const iterator &other) const { + return node == other.node && position == other.position; + } + bool operator==(const const_iterator &other) const { + return node == other.node && position == other.position; + } + bool operator!=(const iterator &other) const { + return node != other.node || position != other.position; + } + bool operator!=(const const_iterator &other) const { + return node != other.node || position != other.position; + } + + // Accessors for the key/value the iterator is pointing at. + reference operator*() const { + ABSL_HARDENING_ASSERT(node != nullptr); + ABSL_HARDENING_ASSERT(node->start() <= position); + ABSL_HARDENING_ASSERT(node->finish() > position); + return node->value(position); + } + pointer operator->() const { return &operator*(); } + + btree_iterator &operator++() { + increment(); + return *this; + } + btree_iterator &operator--() { + decrement(); + return *this; + } + btree_iterator operator++(int) { + btree_iterator tmp = *this; + ++*this; + return tmp; + } + btree_iterator operator--(int) { + btree_iterator tmp = *this; + --*this; + return tmp; + } + + private: + template <typename Params> + friend class btree; + template <typename Tree> + friend class btree_container; + template <typename Tree> + friend class btree_set_container; + template <typename Tree> + friend class btree_map_container; + template <typename Tree> + friend class btree_multiset_container; + template <typename N, typename R, typename P> + friend struct btree_iterator; + template <typename TreeType, typename CheckerType> + friend class base_checker; + + const key_type &key() const { return node->key(position); } + slot_type *slot() { return node->slot(position); } + + // The node in the tree the iterator is pointing at. + Node *node; + // The position within the node of the tree the iterator is pointing at. + // NOTE: this is an int rather than a field_type because iterators can point + // to invalid positions (such as -1) in certain circumstances. + int position; +}; + +template <typename Params> +class btree { + using node_type = btree_node<Params>; + using is_key_compare_to = typename Params::is_key_compare_to; + using init_type = typename Params::init_type; + using field_type = typename node_type::field_type; + using is_multi_container = typename Params::is_multi_container; + using is_key_compare_adapted = typename Params::is_key_compare_adapted; + + // We use a static empty node for the root/leftmost/rightmost of empty btrees + // in order to avoid branching in begin()/end(). + struct alignas(node_type::Alignment()) EmptyNodeType : node_type { + using field_type = typename node_type::field_type; + node_type *parent; + field_type position = 0; + field_type start = 0; + field_type finish = 0; + // max_count must be != kInternalNodeMaxCount (so that this node is regarded + // as a leaf node). max_count() is never called when the tree is empty. + field_type max_count = node_type::kInternalNodeMaxCount + 1; + +#ifdef _MSC_VER + // MSVC has constexpr code generations bugs here. + EmptyNodeType() : parent(this) {} +#else + constexpr EmptyNodeType(node_type *p) : parent(p) {} +#endif + }; + + static node_type *EmptyNode() { +#ifdef _MSC_VER + static EmptyNodeType *empty_node = new EmptyNodeType; + // This assert fails on some other construction methods. + assert(empty_node->parent == empty_node); + return empty_node; +#else + static constexpr EmptyNodeType empty_node( + const_cast<EmptyNodeType *>(&empty_node)); + return const_cast<EmptyNodeType *>(&empty_node); +#endif + } + + enum : uint32_t { + kNodeValues = node_type::kNodeValues, + kMinNodeValues = kNodeValues / 2, + }; + + struct node_stats { + using size_type = typename Params::size_type; + + node_stats(size_type l, size_type i) : leaf_nodes(l), internal_nodes(i) {} + + node_stats &operator+=(const node_stats &other) { + leaf_nodes += other.leaf_nodes; + internal_nodes += other.internal_nodes; + return *this; + } + + size_type leaf_nodes; + size_type internal_nodes; + }; + + public: + using key_type = typename Params::key_type; + using value_type = typename Params::value_type; + using size_type = typename Params::size_type; + using difference_type = typename Params::difference_type; + using key_compare = typename Params::key_compare; + using value_compare = typename Params::value_compare; + using allocator_type = typename Params::allocator_type; + using reference = typename Params::reference; + using const_reference = typename Params::const_reference; + using pointer = typename Params::pointer; + using const_pointer = typename Params::const_pointer; + using iterator = btree_iterator<node_type, reference, pointer>; + using const_iterator = typename iterator::const_iterator; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using node_handle_type = node_handle<Params, Params, allocator_type>; + + // Internal types made public for use by btree_container types. + using params_type = Params; + using slot_type = typename Params::slot_type; + + private: + // For use in copy_or_move_values_in_order. + const value_type &maybe_move_from_iterator(const_iterator it) { return *it; } + value_type &&maybe_move_from_iterator(iterator it) { return std::move(*it); } + + // Copies or moves (depending on the template parameter) the values in + // other into this btree in their order in other. This btree must be empty + // before this method is called. This method is used in copy construction, + // copy assignment, and move assignment. + template <typename Btree> + void copy_or_move_values_in_order(Btree &other); + + // Validates that various assumptions/requirements are true at compile time. + constexpr static bool static_assert_validation(); + + public: + btree(const key_compare &comp, const allocator_type &alloc) + : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} + + btree(const btree &other) : btree(other, other.allocator()) {} + btree(const btree &other, const allocator_type &alloc) + : btree(other.key_comp(), alloc) { + copy_or_move_values_in_order(other); + } + btree(btree &&other) noexcept + : root_(std::move(other.root_)), + rightmost_(absl::exchange(other.rightmost_, EmptyNode())), + size_(absl::exchange(other.size_, 0)) { + other.mutable_root() = EmptyNode(); + } + btree(btree &&other, const allocator_type &alloc) + : btree(other.key_comp(), alloc) { + if (alloc == other.allocator()) { + swap(other); + } else { + // Move values from `other` one at a time when allocators are different. + copy_or_move_values_in_order(other); + } + } + + ~btree() { + // Put static_asserts in destructor to avoid triggering them before the type + // is complete. + static_assert(static_assert_validation(), "This call must be elided."); + clear(); + } + + // Assign the contents of other to *this. + btree &operator=(const btree &other); + btree &operator=(btree &&other) noexcept; + + iterator begin() { return iterator(leftmost()); } + const_iterator begin() const { return const_iterator(leftmost()); } + iterator end() { return iterator(rightmost_, rightmost_->finish()); } + const_iterator end() const { + return const_iterator(rightmost_, rightmost_->finish()); + } + reverse_iterator rbegin() { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const { + return const_reverse_iterator(end()); + } + reverse_iterator rend() { return reverse_iterator(begin()); } + const_reverse_iterator rend() const { + return const_reverse_iterator(begin()); + } + + // Finds the first element whose key is not less than key. + template <typename K> + iterator lower_bound(const K &key) { + return internal_end(internal_lower_bound(key).value); + } + template <typename K> + const_iterator lower_bound(const K &key) const { + return internal_end(internal_lower_bound(key).value); + } + + // Finds the first element whose key is greater than key. + template <typename K> + iterator upper_bound(const K &key) { + return internal_end(internal_upper_bound(key)); + } + template <typename K> + const_iterator upper_bound(const K &key) const { + return internal_end(internal_upper_bound(key)); + } + + // Finds the range of values which compare equal to key. The first member of + // the returned pair is equal to lower_bound(key). The second member of the + // pair is equal to upper_bound(key). + template <typename K> + std::pair<iterator, iterator> equal_range(const K &key); + template <typename K> + std::pair<const_iterator, const_iterator> equal_range(const K &key) const { + return const_cast<btree *>(this)->equal_range(key); + } + + // Inserts a value into the btree only if it does not already exist. The + // boolean return value indicates whether insertion succeeded or failed. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template <typename K, typename... Args> + std::pair<iterator, bool> insert_unique(const K &key, Args &&... args); + + // Inserts with hint. Checks to see if the value should be placed immediately + // before `position` in the tree. If so, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_unique() were made. + // Requirement: if `key` already exists in the btree, does not consume `args`. + // Requirement: `key` is never referenced after consuming `args`. + template <typename K, typename... Args> + std::pair<iterator, bool> insert_hint_unique(iterator position, + const K &key, + Args &&... args); + + // Insert a range of values into the btree. + // Note: the first overload avoids constructing a value_type if the key + // already exists in the btree. + template <typename InputIterator, + typename = decltype(std::declval<const key_compare &>()( + params_type::key(*std::declval<InputIterator>()), + std::declval<const key_type &>()))> + void insert_iterator_unique(InputIterator b, InputIterator e, int); + // We need the second overload for cases in which we need to construct a + // value_type in order to compare it with the keys already in the btree. + template <typename InputIterator> + void insert_iterator_unique(InputIterator b, InputIterator e, char); + + // Inserts a value into the btree. + template <typename ValueType> + iterator insert_multi(const key_type &key, ValueType &&v); + + // Inserts a value into the btree. + template <typename ValueType> + iterator insert_multi(ValueType &&v) { + return insert_multi(params_type::key(v), std::forward<ValueType>(v)); + } + + // Insert with hint. Check to see if the value should be placed immediately + // before position in the tree. If it does, then the insertion will take + // amortized constant time. If not, the insertion will take amortized + // logarithmic time as if a call to insert_multi(v) were made. + template <typename ValueType> + iterator insert_hint_multi(iterator position, ValueType &&v); + + // Insert a range of values into the btree. + template <typename InputIterator> + void insert_iterator_multi(InputIterator b, InputIterator e); + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + // Requirement: does not read the value at `*iter`. + iterator erase(iterator iter); + + // Erases range. Returns the number of keys erased and an iterator pointing + // to the element after the last erased element. + std::pair<size_type, iterator> erase_range(iterator begin, iterator end); + + // Finds the iterator corresponding to a key or returns end() if the key is + // not present. + template <typename K> + iterator find(const K &key) { + return internal_end(internal_find(key)); + } + template <typename K> + const_iterator find(const K &key) const { + return internal_end(internal_find(key)); + } + + // Clear the btree, deleting all of the values it contains. + void clear(); + + // Swaps the contents of `this` and `other`. + void swap(btree &other); + + const key_compare &key_comp() const noexcept { + return root_.template get<0>(); + } + template <typename K1, typename K2> + bool compare_keys(const K1 &a, const K2 &b) const { + return compare_internal::compare_result_as_less_than(key_comp()(a, b)); + } + + value_compare value_comp() const { return value_compare(key_comp()); } + + // Verifies the structure of the btree. + void verify() const; + + // Size routines. + size_type size() const { return size_; } + size_type max_size() const { return (std::numeric_limits<size_type>::max)(); } + bool empty() const { return size_ == 0; } + + // The height of the btree. An empty tree will have height 0. + size_type height() const { + size_type h = 0; + if (!empty()) { + // Count the length of the chain from the leftmost node up to the + // root. We actually count from the root back around to the level below + // the root, but the calculation is the same because of the circularity + // of that traversal. + const node_type *n = root(); + do { + ++h; + n = n->parent(); + } while (n != root()); + } + return h; + } + + // The number of internal, leaf and total nodes used by the btree. + size_type leaf_nodes() const { return internal_stats(root()).leaf_nodes; } + size_type internal_nodes() const { + return internal_stats(root()).internal_nodes; + } + size_type nodes() const { + node_stats stats = internal_stats(root()); + return stats.leaf_nodes + stats.internal_nodes; + } + + // The total number of bytes used by the btree. + size_type bytes_used() const { + node_stats stats = internal_stats(root()); + if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { + return sizeof(*this) + node_type::LeafSize(root()->max_count()); + } else { + return sizeof(*this) + stats.leaf_nodes * node_type::LeafSize() + + stats.internal_nodes * node_type::InternalSize(); + } + } + + // The average number of bytes used per value stored in the btree. + static double average_bytes_per_value() { + // Returns the number of bytes per value on a leaf node that is 75% + // full. Experimentally, this matches up nicely with the computed number of + // bytes per value in trees that had their values inserted in random order. + return node_type::LeafSize() / (kNodeValues * 0.75); + } + + // The fullness of the btree. Computed as the number of elements in the btree + // divided by the maximum number of elements a tree with the current number + // of nodes could hold. A value of 1 indicates perfect space + // utilization. Smaller values indicate space wastage. + // Returns 0 for empty trees. + double fullness() const { + if (empty()) return 0.0; + return static_cast<double>(size()) / (nodes() * kNodeValues); + } + // The overhead of the btree structure in bytes per node. Computed as the + // total number of bytes used by the btree minus the number of bytes used for + // storing elements divided by the number of elements. + // Returns 0 for empty trees. + double overhead() const { + if (empty()) return 0.0; + return (bytes_used() - size() * sizeof(value_type)) / + static_cast<double>(size()); + } + + // The allocator used by the btree. + allocator_type get_allocator() const { return allocator(); } + + private: + // Internal accessor routines. + node_type *root() { return root_.template get<2>(); } + const node_type *root() const { return root_.template get<2>(); } + node_type *&mutable_root() noexcept { return root_.template get<2>(); } + key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } + + // The leftmost node is stored as the parent of the root node. + node_type *leftmost() { return root()->parent(); } + const node_type *leftmost() const { return root()->parent(); } + + // Allocator routines. + allocator_type *mutable_allocator() noexcept { + return &root_.template get<1>(); + } + const allocator_type &allocator() const noexcept { + return root_.template get<1>(); + } + + // Allocates a correctly aligned node of at least size bytes using the + // allocator. + node_type *allocate(const size_type size) { + return reinterpret_cast<node_type *>( + absl::container_internal::Allocate<node_type::Alignment()>( + mutable_allocator(), size)); + } + + // Node creation/deletion routines. + node_type *new_internal_node(node_type *parent) { + node_type *n = allocate(node_type::InternalSize()); + n->init_internal(parent); + return n; + } + node_type *new_leaf_node(node_type *parent) { + node_type *n = allocate(node_type::LeafSize()); + n->init_leaf(parent, kNodeValues); + return n; + } + node_type *new_leaf_root_node(const int max_count) { + node_type *n = allocate(node_type::LeafSize(max_count)); + n->init_leaf(/*parent=*/n, max_count); + return n; + } + + // Deletion helper routines. + iterator rebalance_after_delete(iterator iter); + + // Rebalances or splits the node iter points to. + void rebalance_or_split(iterator *iter); + + // Merges the values of left, right and the delimiting key on their parent + // onto left, removing the delimiting key and deleting right. + void merge_nodes(node_type *left, node_type *right); + + // Tries to merge node with its left or right sibling, and failing that, + // rebalance with its left or right sibling. Returns true if a merge + // occurred, at which point it is no longer valid to access node. Returns + // false if no merging took place. + bool try_merge_or_rebalance(iterator *iter); + + // Tries to shrink the height of the tree by 1. + void try_shrink(); + + iterator internal_end(iterator iter) { + return iter.node != nullptr ? iter : end(); + } + const_iterator internal_end(const_iterator iter) const { + return iter.node != nullptr ? iter : end(); + } + + // Emplaces a value into the btree immediately before iter. Requires that + // key(v) <= iter.key() and (--iter).key() <= key(v). + template <typename... Args> + iterator internal_emplace(iterator iter, Args &&... args); + + // Returns an iterator pointing to the first value >= the value "iter" is + // pointing at. Note that "iter" might be pointing to an invalid location such + // as iter.position == iter.node->finish(). This routine simply moves iter up + // in the tree to a valid location. + // Requires: iter.node is non-null. + template <typename IterType> + static IterType internal_last(IterType iter); + + // Returns an iterator pointing to the leaf position at which key would + // reside in the tree, unless there is an exact match - in which case, the + // result may not be on a leaf. When there's a three-way comparator, we can + // return whether there was an exact match. This allows the caller to avoid a + // subsequent comparison to determine if an exact match was made, which is + // important for keys with expensive comparison, such as strings. + template <typename K> + SearchResult<iterator, is_key_compare_to::value> internal_locate( + const K &key) const; + + // Internal routine which implements lower_bound(). + template <typename K> + SearchResult<iterator, is_key_compare_to::value> internal_lower_bound( + const K &key) const; + + // Internal routine which implements upper_bound(). + template <typename K> + iterator internal_upper_bound(const K &key) const; + + // Internal routine which implements find(). + template <typename K> + iterator internal_find(const K &key) const; + + // Verifies the tree structure of node. + int internal_verify(const node_type *node, const key_type *lo, + const key_type *hi) const; + + node_stats internal_stats(const node_type *node) const { + // The root can be a static empty node. + if (node == nullptr || (node == root() && empty())) { + return node_stats(0, 0); + } + if (node->leaf()) { + return node_stats(1, 0); + } + node_stats res(0, 1); + for (int i = node->start(); i <= node->finish(); ++i) { + res += internal_stats(node->child(i)); + } + return res; + } + + // We use compressed tuple in order to save space because key_compare and + // allocator_type are usually empty. + absl::container_internal::CompressedTuple<key_compare, allocator_type, + node_type *> + root_; + + // A pointer to the rightmost node. Note that the leftmost node is stored as + // the root's parent. + node_type *rightmost_; + + // Number of values. + size_type size_; +}; + +//// +// btree_node methods +template <typename P> +template <typename... Args> +inline void btree_node<P>::emplace_value(const size_type i, + allocator_type *alloc, + Args &&... args) { + assert(i >= start()); + assert(i <= finish()); + // Shift old values to create space for new value and then construct it in + // place. + if (i < finish()) { + transfer_n_backward(finish() - i, /*dest_i=*/i + 1, /*src_i=*/i, this, + alloc); + } + value_init(i, alloc, std::forward<Args>(args)...); + set_finish(finish() + 1); + + if (!leaf() && finish() > i + 1) { + for (int j = finish(); j > i + 1; --j) { + set_child(j, child(j - 1)); + } + clear_child(i + 1); + } +} + +template <typename P> +inline void btree_node<P>::remove_values(const field_type i, + const field_type to_erase, + allocator_type *alloc) { + // Transfer values after the removed range into their new places. + value_destroy_n(i, to_erase, alloc); + const field_type orig_finish = finish(); + const field_type src_i = i + to_erase; + transfer_n(orig_finish - src_i, i, src_i, this, alloc); + + if (!leaf()) { + // Delete all children between begin and end. + for (int j = 0; j < to_erase; ++j) { + clear_and_delete(child(i + j + 1), alloc); + } + // Rotate children after end into new positions. + for (int j = i + to_erase + 1; j <= orig_finish; ++j) { + set_child(j - to_erase, child(j)); + clear_child(j); + } + } + set_finish(orig_finish - to_erase); +} + +template <typename P> +void btree_node<P>::rebalance_right_to_left(const int to_move, + btree_node *right, + allocator_type *alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(right->count() >= count()); + assert(to_move >= 1); + assert(to_move <= right->count()); + + // 1) Move the delimiting value in the parent to the left node. + transfer(finish(), position(), parent(), alloc); + + // 2) Move the (to_move - 1) values from the right node to the left node. + transfer_n(to_move - 1, finish() + 1, right->start(), right, alloc); + + // 3) Move the new delimiting value to the parent from the right node. + parent()->transfer(position(), right->start() + to_move - 1, right, alloc); + + // 4) Shift the values in the right node to their correct positions. + right->transfer_n(right->count() - to_move, right->start(), + right->start() + to_move, right, alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = 0; i < to_move; ++i) { + init_child(finish() + i + 1, right->child(i)); + } + for (int i = right->start(); i <= right->finish() - to_move; ++i) { + assert(i + to_move <= right->max_count()); + right->init_child(i, right->child(i + to_move)); + right->clear_child(i + to_move); + } + } + + // Fixup `finish` on the left and right nodes. + set_finish(finish() + to_move); + right->set_finish(right->finish() - to_move); +} + +template <typename P> +void btree_node<P>::rebalance_left_to_right(const int to_move, + btree_node *right, + allocator_type *alloc) { + assert(parent() == right->parent()); + assert(position() + 1 == right->position()); + assert(count() >= right->count()); + assert(to_move >= 1); + assert(to_move <= count()); + + // Values in the right node are shifted to the right to make room for the + // new to_move values. Then, the delimiting value in the parent and the + // other (to_move - 1) values in the left node are moved into the right node. + // Lastly, a new delimiting value is moved from the left node into the + // parent, and the remaining empty left node entries are destroyed. + + // 1) Shift existing values in the right node to their correct positions. + right->transfer_n_backward(right->count(), right->start() + to_move, + right->start(), right, alloc); + + // 2) Move the delimiting value in the parent to the right node. + right->transfer(right->start() + to_move - 1, position(), parent(), alloc); + + // 3) Move the (to_move - 1) values from the left node to the right node. + right->transfer_n(to_move - 1, right->start(), finish() - (to_move - 1), this, + alloc); + + // 4) Move the new delimiting value to the parent from the left node. + parent()->transfer(position(), finish() - to_move, this, alloc); + + if (!leaf()) { + // Move the child pointers from the left to the right node. + for (int i = right->finish(); i >= right->start(); --i) { + right->init_child(i + to_move, right->child(i)); + right->clear_child(i); + } + for (int i = 1; i <= to_move; ++i) { + right->init_child(i - 1, child(finish() - to_move + i)); + clear_child(finish() - to_move + i); + } + } + + // Fixup the counts on the left and right nodes. + set_finish(finish() - to_move); + right->set_finish(right->finish() + to_move); +} + +template <typename P> +void btree_node<P>::split(const int insert_position, btree_node *dest, + allocator_type *alloc) { + assert(dest->count() == 0); + assert(max_count() == kNodeValues); + + // We bias the split based on the position being inserted. If we're + // inserting at the beginning of the left node then bias the split to put + // more values on the right node. If we're inserting at the end of the + // right node then bias the split to put more values on the left node. + if (insert_position == start()) { + dest->set_finish(dest->start() + finish() - 1); + } else if (insert_position == kNodeValues) { + dest->set_finish(dest->start()); + } else { + dest->set_finish(dest->start() + count() / 2); + } + set_finish(finish() - dest->count()); + assert(count() >= 1); + + // Move values from the left sibling to the right sibling. + dest->transfer_n(dest->count(), dest->start(), finish(), this, alloc); + + // The split key is the largest value in the left sibling. + --mutable_finish(); + parent()->emplace_value(position(), alloc, finish_slot()); + value_destroy(finish(), alloc); + parent()->init_child(position() + 1, dest); + + if (!leaf()) { + for (int i = dest->start(), j = finish() + 1; i <= dest->finish(); + ++i, ++j) { + assert(child(j) != nullptr); + dest->init_child(i, child(j)); + clear_child(j); + } + } +} + +template <typename P> +void btree_node<P>::merge(btree_node *src, allocator_type *alloc) { + assert(parent() == src->parent()); + assert(position() + 1 == src->position()); + + // Move the delimiting value to the left node. + value_init(finish(), alloc, parent()->slot(position())); + + // Move the values from the right to the left node. + transfer_n(src->count(), finish() + 1, src->start(), src, alloc); + + if (!leaf()) { + // Move the child pointers from the right to the left node. + for (int i = src->start(), j = finish() + 1; i <= src->finish(); ++i, ++j) { + init_child(j, src->child(i)); + src->clear_child(i); + } + } + + // Fixup `finish` on the src and dest nodes. + set_finish(start() + 1 + count() + src->count()); + src->set_finish(src->start()); + + // Remove the value on the parent node and delete the src node. + parent()->remove_values(position(), /*to_erase=*/1, alloc); +} + +template <typename P> +void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { + if (node->leaf()) { + node->value_destroy_n(node->start(), node->count(), alloc); + deallocate(LeafSize(node->max_count()), node, alloc); + return; + } + if (node->count() == 0) { + deallocate(InternalSize(), node, alloc); + return; + } + + // The parent of the root of the subtree we are deleting. + btree_node *delete_root_parent = node->parent(); + + // Navigate to the leftmost leaf under node, and then delete upwards. + while (!node->leaf()) node = node->start_child(); + // Use `int` because `pos` needs to be able to hold `kNodeValues+1`, which + // isn't guaranteed to be a valid `field_type`. + int pos = node->position(); + btree_node *parent = node->parent(); + for (;;) { + // In each iteration of the next loop, we delete one leaf node and go right. + assert(pos <= parent->finish()); + do { + node = parent->child(pos); + if (!node->leaf()) { + // Navigate to the leftmost leaf under node. + while (!node->leaf()) node = node->start_child(); + pos = node->position(); + parent = node->parent(); + } + node->value_destroy_n(node->start(), node->count(), alloc); + deallocate(LeafSize(node->max_count()), node, alloc); + ++pos; + } while (pos <= parent->finish()); + + // Once we've deleted all children of parent, delete parent and go up/right. + assert(pos > parent->finish()); + do { + node = parent; + pos = node->position(); + parent = node->parent(); + node->value_destroy_n(node->start(), node->count(), alloc); + deallocate(InternalSize(), node, alloc); + if (parent == delete_root_parent) return; + ++pos; + } while (pos > parent->finish()); + } +} + +//// +// btree_iterator methods +template <typename N, typename R, typename P> +void btree_iterator<N, R, P>::increment_slow() { + if (node->leaf()) { + assert(position >= node->finish()); + btree_iterator save(*this); + while (position == node->finish() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position(); + node = node->parent(); + } + // TODO(ezb): assert we aren't incrementing end() instead of handling. + if (position == node->finish()) { + *this = save; + } + } else { + assert(position < node->finish()); + node = node->child(position + 1); + while (!node->leaf()) { + node = node->start_child(); + } + position = node->start(); + } +} + +template <typename N, typename R, typename P> +void btree_iterator<N, R, P>::decrement_slow() { + if (node->leaf()) { + assert(position <= -1); + btree_iterator save(*this); + while (position < node->start() && !node->is_root()) { + assert(node->parent()->child(node->position()) == node); + position = node->position() - 1; + node = node->parent(); + } + // TODO(ezb): assert we aren't decrementing begin() instead of handling. + if (position < node->start()) { + *this = save; + } + } else { + assert(position >= node->start()); + node = node->child(position); + while (!node->leaf()) { + node = node->child(node->finish()); + } + position = node->finish() - 1; + } +} + +//// +// btree methods +template <typename P> +template <typename Btree> +void btree<P>::copy_or_move_values_in_order(Btree &other) { + static_assert(std::is_same<btree, Btree>::value || + std::is_same<const btree, Btree>::value, + "Btree type must be same or const."); + assert(empty()); + + // We can avoid key comparisons because we know the order of the + // values is the same order we'll store them in. + auto iter = other.begin(); + if (iter == other.end()) return; + insert_multi(maybe_move_from_iterator(iter)); + ++iter; + for (; iter != other.end(); ++iter) { + // If the btree is not empty, we can just insert the new value at the end + // of the tree. + internal_emplace(end(), maybe_move_from_iterator(iter)); + } +} + +template <typename P> +constexpr bool btree<P>::static_assert_validation() { + static_assert(std::is_nothrow_copy_constructible<key_compare>::value, + "Key comparison must be nothrow copy constructible"); + static_assert(std::is_nothrow_copy_constructible<allocator_type>::value, + "Allocator must be nothrow copy constructible"); + static_assert(type_traits_internal::is_trivially_copyable<iterator>::value, + "iterator not trivially copyable."); + + // Note: We assert that kTargetValues, which is computed from + // Params::kTargetNodeSize, must fit the node_type::field_type. + static_assert( + kNodeValues < (1 << (8 * sizeof(typename node_type::field_type))), + "target node size too large"); + + // Verify that key_compare returns an absl::{weak,strong}_ordering or bool. + using compare_result_type = + absl::result_of_t<key_compare(key_type, key_type)>; + static_assert( + std::is_same<compare_result_type, bool>::value || + std::is_convertible<compare_result_type, absl::weak_ordering>::value, + "key comparison function must return absl::{weak,strong}_ordering or " + "bool."); + + // Test the assumption made in setting kNodeValueSpace. + static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, + "node space assumption incorrect"); + + return true; +} + +template <typename P> +template <typename K> +auto btree<P>::equal_range(const K &key) -> std::pair<iterator, iterator> { + const SearchResult<iterator, is_key_compare_to::value> res = + internal_lower_bound(key); + const iterator lower = internal_end(res.value); + if (res.HasMatch() ? !res.IsEq() + : lower == end() || compare_keys(key, lower.key())) { + return {lower, lower}; + } + + const iterator next = std::next(lower); + // When the comparator is heterogeneous, we can't assume that comparison with + // non-`key_type` will be equivalent to `key_type` comparisons so there + // could be multiple equivalent keys even in a unique-container. But for + // heterogeneous comparisons from the default string adapted comparators, we + // don't need to worry about this. + if (!is_multi_container::value && + (std::is_same<K, key_type>::value || is_key_compare_adapted::value)) { + // The next iterator after lower must point to a key greater than `key`. + // Note: if this assert fails, then it may indicate that the comparator does + // not meet the equivalence requirements for Compare + // (see https://en.cppreference.com/w/cpp/named_req/Compare). + assert(next == end() || compare_keys(key, next.key())); + return {lower, next}; + } + // Try once more to avoid the call to upper_bound() if there's only one + // equivalent key. This should prevent all calls to upper_bound() in cases of + // unique-containers with heterogeneous comparators in which all comparison + // operators have the same equivalence classes. + if (next == end() || compare_keys(key, next.key())) return {lower, next}; + + // In this case, we need to call upper_bound() to avoid worst case O(N) + // behavior if we were to iterate over equal keys. + return {lower, upper_bound(key)}; +} + +template <typename P> +template <typename K, typename... Args> +auto btree<P>::insert_unique(const K &key, Args &&... args) + -> std::pair<iterator, bool> { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + SearchResult<iterator, is_key_compare_to::value> res = internal_locate(key); + iterator iter = res.value; + + if (res.HasMatch()) { + if (res.IsEq()) { + // The key already exists in the tree, do nothing. + return {iter, false}; + } + } else { + iterator last = internal_last(iter); + if (last.node && !compare_keys(key, last.key())) { + // The key already exists in the tree, do nothing. + return {last, false}; + } + } + return {internal_emplace(iter, std::forward<Args>(args)...), true}; +} + +template <typename P> +template <typename K, typename... Args> +inline auto btree<P>::insert_hint_unique(iterator position, const K &key, + Args &&... args) + -> std::pair<iterator, bool> { + if (!empty()) { + if (position == end() || compare_keys(key, position.key())) { + if (position == begin() || compare_keys(std::prev(position).key(), key)) { + // prev.key() < key < position.key() + return {internal_emplace(position, std::forward<Args>(args)...), true}; + } + } else if (compare_keys(position.key(), key)) { + ++position; + if (position == end() || compare_keys(key, position.key())) { + // {original `position`}.key() < key < {current `position`}.key() + return {internal_emplace(position, std::forward<Args>(args)...), true}; + } + } else { + // position.key() == key + return {position, false}; + } + } + return insert_unique(key, std::forward<Args>(args)...); +} + +template <typename P> +template <typename InputIterator, typename> +void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e, int) { + for (; b != e; ++b) { + insert_hint_unique(end(), params_type::key(*b), *b); + } +} + +template <typename P> +template <typename InputIterator> +void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e, char) { + for (; b != e; ++b) { + init_type value(*b); + insert_hint_unique(end(), params_type::key(value), std::move(value)); + } +} + +template <typename P> +template <typename ValueType> +auto btree<P>::insert_multi(const key_type &key, ValueType &&v) -> iterator { + if (empty()) { + mutable_root() = rightmost_ = new_leaf_root_node(1); + } + + iterator iter = internal_upper_bound(key); + if (iter.node == nullptr) { + iter = end(); + } + return internal_emplace(iter, std::forward<ValueType>(v)); +} + +template <typename P> +template <typename ValueType> +auto btree<P>::insert_hint_multi(iterator position, ValueType &&v) -> iterator { + if (!empty()) { + const key_type &key = params_type::key(v); + if (position == end() || !compare_keys(position.key(), key)) { + if (position == begin() || + !compare_keys(key, std::prev(position).key())) { + // prev.key() <= key <= position.key() + return internal_emplace(position, std::forward<ValueType>(v)); + } + } else { + ++position; + if (position == end() || !compare_keys(position.key(), key)) { + // {original `position`}.key() < key < {current `position`}.key() + return internal_emplace(position, std::forward<ValueType>(v)); + } + } + } + return insert_multi(std::forward<ValueType>(v)); +} + +template <typename P> +template <typename InputIterator> +void btree<P>::insert_iterator_multi(InputIterator b, InputIterator e) { + for (; b != e; ++b) { + insert_hint_multi(end(), *b); + } +} + +template <typename P> +auto btree<P>::operator=(const btree &other) -> btree & { + if (this != &other) { + clear(); + + *mutable_key_comp() = other.key_comp(); + if (absl::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + *mutable_allocator() = other.allocator(); + } + + copy_or_move_values_in_order(other); + } + return *this; +} + +template <typename P> +auto btree<P>::operator=(btree &&other) noexcept -> btree & { + if (this != &other) { + clear(); + + using std::swap; + if (absl::allocator_traits< + allocator_type>::propagate_on_container_copy_assignment::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, other.root_); + swap(rightmost_, other.rightmost_); + swap(size_, other.size_); + } else { + if (allocator() == other.allocator()) { + swap(mutable_root(), other.mutable_root()); + swap(*mutable_key_comp(), *other.mutable_key_comp()); + swap(rightmost_, other.rightmost_); + swap(size_, other.size_); + } else { + // We aren't allowed to propagate the allocator and the allocator is + // different so we can't take over its memory. We must move each element + // individually. We need both `other` and `this` to have `other`s key + // comparator while moving the values so we can't swap the key + // comparators. + *mutable_key_comp() = other.key_comp(); + copy_or_move_values_in_order(other); + } + } + } + return *this; +} + +template <typename P> +auto btree<P>::erase(iterator iter) -> iterator { + bool internal_delete = false; + if (!iter.node->leaf()) { + // Deletion of a value on an internal node. First, move the largest value + // from our left child here, then delete that position (in remove_values() + // below). We can get to the largest value from our left child by + // decrementing iter. + iterator internal_iter(iter); + --iter; + assert(iter.node->leaf()); + params_type::move(mutable_allocator(), iter.node->slot(iter.position), + internal_iter.node->slot(internal_iter.position)); + internal_delete = true; + } + + // Delete the key from the leaf. + iter.node->remove_values(iter.position, /*to_erase=*/1, mutable_allocator()); + --size_; + + // We want to return the next value after the one we just erased. If we + // erased from an internal node (internal_delete == true), then the next + // value is ++(++iter). If we erased from a leaf node (internal_delete == + // false) then the next value is ++iter. Note that ++iter may point to an + // internal node and the value in the internal node may move to a leaf node + // (iter.node) when rebalancing is performed at the leaf level. + + iterator res = rebalance_after_delete(iter); + + // If we erased from an internal node, advance the iterator. + if (internal_delete) { + ++res; + } + return res; +} + +template <typename P> +auto btree<P>::rebalance_after_delete(iterator iter) -> iterator { + // Merge/rebalance as we walk back up the tree. + iterator res(iter); + bool first_iteration = true; + for (;;) { + if (iter.node == root()) { + try_shrink(); + if (empty()) { + return end(); + } + break; + } + if (iter.node->count() >= kMinNodeValues) { + break; + } + bool merged = try_merge_or_rebalance(&iter); + // On the first iteration, we should update `res` with `iter` because `res` + // may have been invalidated. + if (first_iteration) { + res = iter; + first_iteration = false; + } + if (!merged) { + break; + } + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + } + + // Adjust our return value. If we're pointing at the end of a node, advance + // the iterator. + if (res.position == res.node->finish()) { + res.position = res.node->finish() - 1; + ++res; + } + + return res; +} + +template <typename P> +auto btree<P>::erase_range(iterator begin, iterator end) + -> std::pair<size_type, iterator> { + difference_type count = std::distance(begin, end); + assert(count >= 0); + + if (count == 0) { + return {0, begin}; + } + + if (count == size_) { + clear(); + return {count, this->end()}; + } + + if (begin.node == end.node) { + assert(end.position > begin.position); + begin.node->remove_values(begin.position, end.position - begin.position, + mutable_allocator()); + size_ -= count; + return {count, rebalance_after_delete(begin)}; + } + + const size_type target_size = size_ - count; + while (size_ > target_size) { + if (begin.node->leaf()) { + const size_type remaining_to_erase = size_ - target_size; + const size_type remaining_in_node = begin.node->finish() - begin.position; + const size_type to_erase = + (std::min)(remaining_to_erase, remaining_in_node); + begin.node->remove_values(begin.position, to_erase, mutable_allocator()); + size_ -= to_erase; + begin = rebalance_after_delete(begin); + } else { + begin = erase(begin); + } + } + return {count, begin}; +} + +template <typename P> +void btree<P>::clear() { + if (!empty()) { + node_type::clear_and_delete(root(), mutable_allocator()); + } + mutable_root() = EmptyNode(); + rightmost_ = EmptyNode(); + size_ = 0; +} + +template <typename P> +void btree<P>::swap(btree &other) { + using std::swap; + if (absl::allocator_traits< + allocator_type>::propagate_on_container_swap::value) { + // Note: `root_` also contains the allocator and the key comparator. + swap(root_, other.root_); + } else { + // It's undefined behavior if the allocators are unequal here. + assert(allocator() == other.allocator()); + swap(mutable_root(), other.mutable_root()); + swap(*mutable_key_comp(), *other.mutable_key_comp()); + } + swap(rightmost_, other.rightmost_); + swap(size_, other.size_); +} + +template <typename P> +void btree<P>::verify() const { + assert(root() != nullptr); + assert(leftmost() != nullptr); + assert(rightmost_ != nullptr); + assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); + assert(leftmost() == (++const_iterator(root(), -1)).node); + assert(rightmost_ == (--const_iterator(root(), root()->finish())).node); + assert(leftmost()->leaf()); + assert(rightmost_->leaf()); +} + +template <typename P> +void btree<P>::rebalance_or_split(iterator *iter) { + node_type *&node = iter->node; + int &insert_position = iter->position; + assert(node->count() == node->max_count()); + assert(kNodeValues == node->max_count()); + + // First try to make room on the node by rebalancing. + node_type *parent = node->parent(); + if (node != root()) { + if (node->position() > parent->start()) { + // Try rebalancing with our left sibling. + node_type *left = parent->child(node->position() - 1); + assert(left->max_count() == kNodeValues); + if (left->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the end of the right node then we bias rebalancing to + // fill up the left node. + int to_move = (kNodeValues - left->count()) / + (1 + (insert_position < static_cast<int>(kNodeValues))); + to_move = (std::max)(1, to_move); + + if (insert_position - to_move >= node->start() || + left->count() + to_move < static_cast<int>(kNodeValues)) { + left->rebalance_right_to_left(to_move, node, mutable_allocator()); + + assert(node->max_count() - node->count() == to_move); + insert_position = insert_position - to_move; + if (insert_position < node->start()) { + insert_position = insert_position + left->count() + 1; + node = left; + } + + assert(node->count() < node->max_count()); + return; + } + } + } + + if (node->position() < parent->finish()) { + // Try rebalancing with our right sibling. + node_type *right = parent->child(node->position() + 1); + assert(right->max_count() == kNodeValues); + if (right->count() < kNodeValues) { + // We bias rebalancing based on the position being inserted. If we're + // inserting at the beginning of the left node then we bias rebalancing + // to fill up the right node. + int to_move = (static_cast<int>(kNodeValues) - right->count()) / + (1 + (insert_position > node->start())); + to_move = (std::max)(1, to_move); + + if (insert_position <= node->finish() - to_move || + right->count() + to_move < static_cast<int>(kNodeValues)) { + node->rebalance_left_to_right(to_move, right, mutable_allocator()); + + if (insert_position > node->finish()) { + insert_position = insert_position - node->count() - 1; + node = right; + } + + assert(node->count() < node->max_count()); + return; + } + } + } + + // Rebalancing failed, make sure there is room on the parent node for a new + // value. + assert(parent->max_count() == kNodeValues); + if (parent->count() == kNodeValues) { + iterator parent_iter(node->parent(), node->position()); + rebalance_or_split(&parent_iter); + } + } else { + // Rebalancing not possible because this is the root node. + // Create a new root node and set the current root node as the child of the + // new root. + parent = new_internal_node(parent); + parent->init_child(parent->start(), root()); + mutable_root() = parent; + // If the former root was a leaf node, then it's now the rightmost node. + assert(!parent->start_child()->leaf() || + parent->start_child() == rightmost_); + } + + // Split the node. + node_type *split_node; + if (node->leaf()) { + split_node = new_leaf_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + if (rightmost_ == node) rightmost_ = split_node; + } else { + split_node = new_internal_node(parent); + node->split(insert_position, split_node, mutable_allocator()); + } + + if (insert_position > node->finish()) { + insert_position = insert_position - node->count() - 1; + node = split_node; + } +} + +template <typename P> +void btree<P>::merge_nodes(node_type *left, node_type *right) { + left->merge(right, mutable_allocator()); + if (rightmost_ == right) rightmost_ = left; +} + +template <typename P> +bool btree<P>::try_merge_or_rebalance(iterator *iter) { + node_type *parent = iter->node->parent(); + if (iter->node->position() > parent->start()) { + // Try merging with our left sibling. + node_type *left = parent->child(iter->node->position() - 1); + assert(left->max_count() == kNodeValues); + if (1U + left->count() + iter->node->count() <= kNodeValues) { + iter->position += 1 + left->count(); + merge_nodes(left, iter->node); + iter->node = left; + return true; + } + } + if (iter->node->position() < parent->finish()) { + // Try merging with our right sibling. + node_type *right = parent->child(iter->node->position() + 1); + assert(right->max_count() == kNodeValues); + if (1U + iter->node->count() + right->count() <= kNodeValues) { + merge_nodes(iter->node, right); + return true; + } + // Try rebalancing with our right sibling. We don't perform rebalancing if + // we deleted the first element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the front of the tree. + if (right->count() > kMinNodeValues && + (iter->node->count() == 0 || iter->position > iter->node->start())) { + int to_move = (right->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, right->count() - 1); + iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); + return false; + } + } + if (iter->node->position() > parent->start()) { + // Try rebalancing with our left sibling. We don't perform rebalancing if + // we deleted the last element from iter->node and the node is not + // empty. This is a small optimization for the common pattern of deleting + // from the back of the tree. + node_type *left = parent->child(iter->node->position() - 1); + if (left->count() > kMinNodeValues && + (iter->node->count() == 0 || iter->position < iter->node->finish())) { + int to_move = (left->count() - iter->node->count()) / 2; + to_move = (std::min)(to_move, left->count() - 1); + left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); + iter->position += to_move; + return false; + } + } + return false; +} + +template <typename P> +void btree<P>::try_shrink() { + node_type *orig_root = root(); + if (orig_root->count() > 0) { + return; + } + // Deleted the last item on the root node, shrink the height of the tree. + if (orig_root->leaf()) { + assert(size() == 0); + mutable_root() = rightmost_ = EmptyNode(); + } else { + node_type *child = orig_root->start_child(); + child->make_root(); + mutable_root() = child; + } + node_type::clear_and_delete(orig_root, mutable_allocator()); +} + +template <typename P> +template <typename IterType> +inline IterType btree<P>::internal_last(IterType iter) { + assert(iter.node != nullptr); + while (iter.position == iter.node->finish()) { + iter.position = iter.node->position(); + iter.node = iter.node->parent(); + if (iter.node->leaf()) { + iter.node = nullptr; + break; + } + } + return iter; +} + +template <typename P> +template <typename... Args> +inline auto btree<P>::internal_emplace(iterator iter, Args &&... args) + -> iterator { + if (!iter.node->leaf()) { + // We can't insert on an internal node. Instead, we'll insert after the + // previous value which is guaranteed to be on a leaf node. + --iter; + ++iter.position; + } + const field_type max_count = iter.node->max_count(); + allocator_type *alloc = mutable_allocator(); + if (iter.node->count() == max_count) { + // Make room in the leaf for the new item. + if (max_count < kNodeValues) { + // Insertion into the root where the root is smaller than the full node + // size. Simply grow the size of the root node. + assert(iter.node == root()); + iter.node = + new_leaf_root_node((std::min<int>)(kNodeValues, 2 * max_count)); + // Transfer the values from the old root to the new root. + node_type *old_root = root(); + node_type *new_root = iter.node; + new_root->transfer_n(old_root->count(), new_root->start(), + old_root->start(), old_root, alloc); + new_root->set_finish(old_root->finish()); + old_root->set_finish(old_root->start()); + node_type::clear_and_delete(old_root, alloc); + mutable_root() = rightmost_ = new_root; + } else { + rebalance_or_split(&iter); + } + } + iter.node->emplace_value(iter.position, alloc, std::forward<Args>(args)...); + ++size_; + return iter; +} + +template <typename P> +template <typename K> +inline auto btree<P>::internal_locate(const K &key) const + -> SearchResult<iterator, is_key_compare_to::value> { + iterator iter(const_cast<node_type *>(root())); + for (;;) { + SearchResult<int, is_key_compare_to::value> res = + iter.node->lower_bound(key, key_comp()); + iter.position = res.value; + if (res.IsEq()) { + return {iter, MatchKind::kEq}; + } + // Note: in the non-key-compare-to case, we don't need to walk all the way + // down the tree if the keys are equal, but determining equality would + // require doing an extra comparison on each node on the way down, and we + // will need to go all the way to the leaf node in the expected case. + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + // Note: in the non-key-compare-to case, the key may actually be equivalent + // here (and the MatchKind::kNe is ignored). + return {iter, MatchKind::kNe}; +} + +template <typename P> +template <typename K> +auto btree<P>::internal_lower_bound(const K &key) const + -> SearchResult<iterator, is_key_compare_to::value> { + iterator iter(const_cast<node_type *>(root())); + SearchResult<int, is_key_compare_to::value> res; + bool seen_eq = false; + for (;;) { + res = iter.node->lower_bound(key, key_comp()); + iter.position = res.value; + // TODO(ezb): we should be able to terminate early on IsEq() if there can't + // be multiple equivalent keys in container for this lookup type. + if (iter.node->leaf()) { + break; + } + seen_eq = seen_eq || res.IsEq(); + iter.node = iter.node->child(iter.position); + } + if (res.IsEq()) return {iter, MatchKind::kEq}; + return {internal_last(iter), seen_eq ? MatchKind::kEq : MatchKind::kNe}; +} + +template <typename P> +template <typename K> +auto btree<P>::internal_upper_bound(const K &key) const -> iterator { + iterator iter(const_cast<node_type *>(root())); + for (;;) { + iter.position = iter.node->upper_bound(key, key_comp()); + if (iter.node->leaf()) { + break; + } + iter.node = iter.node->child(iter.position); + } + return internal_last(iter); +} + +template <typename P> +template <typename K> +auto btree<P>::internal_find(const K &key) const -> iterator { + SearchResult<iterator, is_key_compare_to::value> res = internal_locate(key); + if (res.HasMatch()) { + if (res.IsEq()) { + return res.value; + } + } else { + const iterator iter = internal_last(res.value); + if (iter.node != nullptr && !compare_keys(key, iter.key())) { + return iter; + } + } + return {nullptr, 0}; +} + +template <typename P> +int btree<P>::internal_verify(const node_type *node, const key_type *lo, + const key_type *hi) const { + assert(node->count() > 0); + assert(node->count() <= node->max_count()); + if (lo) { + assert(!compare_keys(node->key(node->start()), *lo)); + } + if (hi) { + assert(!compare_keys(*hi, node->key(node->finish() - 1))); + } + for (int i = node->start() + 1; i < node->finish(); ++i) { + assert(!compare_keys(node->key(i), node->key(i - 1))); + } + int count = node->count(); + if (!node->leaf()) { + for (int i = node->start(); i <= node->finish(); ++i) { + assert(node->child(i) != nullptr); + assert(node->child(i)->parent() == node); + assert(node->child(i)->position() == i); + count += internal_verify(node->child(i), + i == node->start() ? lo : &node->key(i - 1), + i == node->finish() ? hi : &node->key(i)); + } + } + return count; +} + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_BTREE_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/btree_container.h b/third_party/abseil_cpp/absl/container/internal/btree_container.h new file mode 100644 index 000000000000..887eda4122f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/btree_container.h @@ -0,0 +1,683 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_BTREE_CONTAINER_H_ +#define ABSL_CONTAINER_INTERNAL_BTREE_CONTAINER_H_ + +#include <algorithm> +#include <initializer_list> +#include <iterator> +#include <utility> + +#include "absl/base/internal/throw_delegate.h" +#include "absl/container/internal/btree.h" // IWYU pragma: export +#include "absl/container/internal/common.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// A common base class for btree_set, btree_map, btree_multiset, and +// btree_multimap. +template <typename Tree> +class btree_container { + using params_type = typename Tree::params_type; + + protected: + // Alias used for heterogeneous lookup functions. + // `key_arg<K>` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template <class K> + using key_arg = + typename KeyArg<IsTransparent<typename Tree::key_compare>::value>:: + template type<K, typename Tree::key_type>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using difference_type = typename Tree::difference_type; + using key_compare = typename Tree::key_compare; + using value_compare = typename Tree::value_compare; + using allocator_type = typename Tree::allocator_type; + using reference = typename Tree::reference; + using const_reference = typename Tree::const_reference; + using pointer = typename Tree::pointer; + using const_pointer = typename Tree::const_pointer; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using reverse_iterator = typename Tree::reverse_iterator; + using const_reverse_iterator = typename Tree::const_reverse_iterator; + using node_type = typename Tree::node_handle_type; + + // Constructors/assignments. + btree_container() : tree_(key_compare(), allocator_type()) {} + explicit btree_container(const key_compare &comp, + const allocator_type &alloc = allocator_type()) + : tree_(comp, alloc) {} + explicit btree_container(const allocator_type &alloc) + : tree_(key_compare(), alloc) {} + + btree_container(const btree_container &other) + : btree_container(other, absl::allocator_traits<allocator_type>:: + select_on_container_copy_construction( + other.get_allocator())) {} + btree_container(const btree_container &other, const allocator_type &alloc) + : tree_(other.tree_, alloc) {} + + btree_container(btree_container &&other) noexcept( + std::is_nothrow_move_constructible<Tree>::value) = default; + btree_container(btree_container &&other, const allocator_type &alloc) + : tree_(std::move(other.tree_), alloc) {} + + btree_container &operator=(const btree_container &other) = default; + btree_container &operator=(btree_container &&other) noexcept( + std::is_nothrow_move_assignable<Tree>::value) = default; + + // Iterator routines. + iterator begin() { return tree_.begin(); } + const_iterator begin() const { return tree_.begin(); } + const_iterator cbegin() const { return tree_.begin(); } + iterator end() { return tree_.end(); } + const_iterator end() const { return tree_.end(); } + const_iterator cend() const { return tree_.end(); } + reverse_iterator rbegin() { return tree_.rbegin(); } + const_reverse_iterator rbegin() const { return tree_.rbegin(); } + const_reverse_iterator crbegin() const { return tree_.rbegin(); } + reverse_iterator rend() { return tree_.rend(); } + const_reverse_iterator rend() const { return tree_.rend(); } + const_reverse_iterator crend() const { return tree_.rend(); } + + // Lookup routines. + template <typename K = key_type> + size_type count(const key_arg<K> &key) const { + auto equal_range = this->equal_range(key); + return std::distance(equal_range.first, equal_range.second); + } + template <typename K = key_type> + iterator find(const key_arg<K> &key) { + return tree_.find(key); + } + template <typename K = key_type> + const_iterator find(const key_arg<K> &key) const { + return tree_.find(key); + } + template <typename K = key_type> + bool contains(const key_arg<K> &key) const { + return find(key) != end(); + } + template <typename K = key_type> + iterator lower_bound(const key_arg<K> &key) { + return tree_.lower_bound(key); + } + template <typename K = key_type> + const_iterator lower_bound(const key_arg<K> &key) const { + return tree_.lower_bound(key); + } + template <typename K = key_type> + iterator upper_bound(const key_arg<K> &key) { + return tree_.upper_bound(key); + } + template <typename K = key_type> + const_iterator upper_bound(const key_arg<K> &key) const { + return tree_.upper_bound(key); + } + template <typename K = key_type> + std::pair<iterator, iterator> equal_range(const key_arg<K> &key) { + return tree_.equal_range(key); + } + template <typename K = key_type> + std::pair<const_iterator, const_iterator> equal_range( + const key_arg<K> &key) const { + return tree_.equal_range(key); + } + + // Deletion routines. Note that there is also a deletion routine that is + // specific to btree_set_container/btree_multiset_container. + + // Erase the specified iterator from the btree. The iterator must be valid + // (i.e. not equal to end()). Return an iterator pointing to the node after + // the one that was erased (or end() if none exists). + iterator erase(const_iterator iter) { return tree_.erase(iterator(iter)); } + iterator erase(iterator iter) { return tree_.erase(iter); } + iterator erase(const_iterator first, const_iterator last) { + return tree_.erase_range(iterator(first), iterator(last)).second; + } + template <typename K = key_type> + size_type erase(const key_arg<K> &key) { + auto equal_range = this->equal_range(key); + return tree_.erase_range(equal_range.first, equal_range.second).first; + } + + // Extract routines. + node_type extract(iterator position) { + // Use Move instead of Transfer, because the rebalancing code expects to + // have a valid object to scribble metadata bits on top of. + auto node = CommonAccess::Move<node_type>(get_allocator(), position.slot()); + erase(position); + return node; + } + node_type extract(const_iterator position) { + return extract(iterator(position)); + } + + // Utility routines. + void clear() { tree_.clear(); } + void swap(btree_container &other) { tree_.swap(other.tree_); } + void verify() const { tree_.verify(); } + + // Size routines. + size_type size() const { return tree_.size(); } + size_type max_size() const { return tree_.max_size(); } + bool empty() const { return tree_.empty(); } + + friend bool operator==(const btree_container &x, const btree_container &y) { + if (x.size() != y.size()) return false; + return std::equal(x.begin(), x.end(), y.begin()); + } + + friend bool operator!=(const btree_container &x, const btree_container &y) { + return !(x == y); + } + + friend bool operator<(const btree_container &x, const btree_container &y) { + return std::lexicographical_compare(x.begin(), x.end(), y.begin(), y.end()); + } + + friend bool operator>(const btree_container &x, const btree_container &y) { + return y < x; + } + + friend bool operator<=(const btree_container &x, const btree_container &y) { + return !(y < x); + } + + friend bool operator>=(const btree_container &x, const btree_container &y) { + return !(x < y); + } + + // The allocator used by the btree. + allocator_type get_allocator() const { return tree_.get_allocator(); } + + // The key comparator used by the btree. + key_compare key_comp() const { return tree_.key_comp(); } + value_compare value_comp() const { return tree_.value_comp(); } + + // Support absl::Hash. + template <typename State> + friend State AbslHashValue(State h, const btree_container &b) { + for (const auto &v : b) { + h = State::combine(std::move(h), v); + } + return State::combine(std::move(h), b.size()); + } + + protected: + Tree tree_; +}; + +// A common base class for btree_set and btree_map. +template <typename Tree> +class btree_set_container : public btree_container<Tree> { + using super_type = btree_container<Tree>; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + friend class BtreeNodePeer; + + protected: + template <class K> + using key_arg = typename super_type::template key_arg<K>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + using insert_return_type = InsertReturnType<iterator, node_type>; + + // Inherit constructors. + using super_type::super_type; + btree_set_container() {} + + // Range constructors. + template <class InputIterator> + btree_set_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + template <class InputIterator> + btree_set_container(InputIterator b, InputIterator e, + const allocator_type &alloc) + : btree_set_container(b, e, key_compare(), alloc) {} + + // Initializer list constructors. + btree_set_container(std::initializer_list<init_type> init, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : btree_set_container(init.begin(), init.end(), comp, alloc) {} + btree_set_container(std::initializer_list<init_type> init, + const allocator_type &alloc) + : btree_set_container(init.begin(), init.end(), alloc) {} + + // Insertion routines. + std::pair<iterator, bool> insert(const value_type &v) { + return this->tree_.insert_unique(params_type::key(v), v); + } + std::pair<iterator, bool> insert(value_type &&v) { + return this->tree_.insert_unique(params_type::key(v), std::move(v)); + } + template <typename... Args> + std::pair<iterator, bool> emplace(Args &&... args) { + init_type v(std::forward<Args>(args)...); + return this->tree_.insert_unique(params_type::key(v), std::move(v)); + } + iterator insert(const_iterator hint, const value_type &v) { + return this->tree_ + .insert_hint_unique(iterator(hint), params_type::key(v), v) + .first; + } + iterator insert(const_iterator hint, value_type &&v) { + return this->tree_ + .insert_hint_unique(iterator(hint), params_type::key(v), std::move(v)) + .first; + } + template <typename... Args> + iterator emplace_hint(const_iterator hint, Args &&... args) { + init_type v(std::forward<Args>(args)...); + return this->tree_ + .insert_hint_unique(iterator(hint), params_type::key(v), std::move(v)) + .first; + } + template <typename InputIterator> + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_unique(b, e, 0); + } + void insert(std::initializer_list<init_type> init) { + this->tree_.insert_iterator_unique(init.begin(), init.end(), 0); + } + insert_return_type insert(node_type &&node) { + if (!node) return {this->end(), false, node_type()}; + std::pair<iterator, bool> res = + this->tree_.insert_unique(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) { + CommonAccess::Destroy(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + iterator insert(const_iterator hint, node_type &&node) { + if (!node) return this->end(); + std::pair<iterator, bool> res = this->tree_.insert_hint_unique( + iterator(hint), params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + if (res.second) CommonAccess::Destroy(&node); + return res.first; + } + + // Node extraction routines. + // TODO(ezb): when the comparator is heterogeneous and has different + // equivalence classes for different lookup types, we should extract the first + // equivalent value if there are multiple. + template <typename K = key_type> + node_type extract(const key_arg<K> &key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + using super_type::extract; + + // Merge routines. + // Moves elements from `src` into `this`. If the element already exists in + // `this`, it is left unmodified in `src`. + template < + typename T, + typename absl::enable_if_t< + absl::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &src) { // NOLINT + for (auto src_it = src.begin(); src_it != src.end();) { + if (insert(std::move(params_type::element(src_it.slot()))).second) { + src_it = src.erase(src_it); + } else { + ++src_it; + } + } + } + + template < + typename T, + typename absl::enable_if_t< + absl::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &&src) { + merge(src); + } +}; + +// Base class for btree_map. +template <typename Tree> +class btree_map_container : public btree_set_container<Tree> { + using super_type = btree_set_container<Tree>; + using params_type = typename Tree::params_type; + friend class BtreeNodePeer; + + private: + template <class K> + using key_arg = typename super_type::template key_arg<K>; + + public: + using key_type = typename Tree::key_type; + using mapped_type = typename params_type::mapped_type; + using value_type = typename Tree::value_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + + // Inherit constructors. + using super_type::super_type; + btree_map_container() {} + + // Insertion routines. + // Note: the nullptr template arguments and extra `const M&` overloads allow + // for supporting bitfield arguments. + template <typename K = key_type, class M> + std::pair<iterator, bool> insert_or_assign(const key_arg<K> &k, + const M &obj) { + return insert_or_assign_impl(k, obj); + } + template <typename K = key_type, class M, K * = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K> &&k, const M &obj) { + return insert_or_assign_impl(std::forward<K>(k), obj); + } + template <typename K = key_type, class M, M * = nullptr> + std::pair<iterator, bool> insert_or_assign(const key_arg<K> &k, M &&obj) { + return insert_or_assign_impl(k, std::forward<M>(obj)); + } + template <typename K = key_type, class M, K * = nullptr, M * = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K> &&k, M &&obj) { + return insert_or_assign_impl(std::forward<K>(k), std::forward<M>(obj)); + } + template <typename K = key_type, class M> + iterator insert_or_assign(const_iterator hint, const key_arg<K> &k, + const M &obj) { + return insert_or_assign_hint_impl(hint, k, obj); + } + template <typename K = key_type, class M, K * = nullptr> + iterator insert_or_assign(const_iterator hint, key_arg<K> &&k, const M &obj) { + return insert_or_assign_hint_impl(hint, std::forward<K>(k), obj); + } + template <typename K = key_type, class M, M * = nullptr> + iterator insert_or_assign(const_iterator hint, const key_arg<K> &k, M &&obj) { + return insert_or_assign_hint_impl(hint, k, std::forward<M>(obj)); + } + template <typename K = key_type, class M, K * = nullptr, M * = nullptr> + iterator insert_or_assign(const_iterator hint, key_arg<K> &&k, M &&obj) { + return insert_or_assign_hint_impl(hint, std::forward<K>(k), + std::forward<M>(obj)); + } + + template <typename K = key_type, typename... Args, + typename absl::enable_if_t< + !std::is_convertible<K, const_iterator>::value, int> = 0> + std::pair<iterator, bool> try_emplace(const key_arg<K> &k, Args &&... args) { + return try_emplace_impl(k, std::forward<Args>(args)...); + } + template <typename K = key_type, typename... Args, + typename absl::enable_if_t< + !std::is_convertible<K, const_iterator>::value, int> = 0> + std::pair<iterator, bool> try_emplace(key_arg<K> &&k, Args &&... args) { + return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); + } + template <typename K = key_type, typename... Args> + iterator try_emplace(const_iterator hint, const key_arg<K> &k, + Args &&... args) { + return try_emplace_hint_impl(hint, k, std::forward<Args>(args)...); + } + template <typename K = key_type, typename... Args> + iterator try_emplace(const_iterator hint, key_arg<K> &&k, Args &&... args) { + return try_emplace_hint_impl(hint, std::forward<K>(k), + std::forward<Args>(args)...); + } + + template <typename K = key_type> + mapped_type &operator[](const key_arg<K> &k) { + return try_emplace(k).first->second; + } + template <typename K = key_type> + mapped_type &operator[](key_arg<K> &&k) { + return try_emplace(std::forward<K>(k)).first->second; + } + + template <typename K = key_type> + mapped_type &at(const key_arg<K> &key) { + auto it = this->find(key); + if (it == this->end()) + base_internal::ThrowStdOutOfRange("absl::btree_map::at"); + return it->second; + } + template <typename K = key_type> + const mapped_type &at(const key_arg<K> &key) const { + auto it = this->find(key); + if (it == this->end()) + base_internal::ThrowStdOutOfRange("absl::btree_map::at"); + return it->second; + } + + private: + // Note: when we call `std::forward<M>(obj)` twice, it's safe because + // insert_unique/insert_hint_unique are guaranteed to not consume `obj` when + // `ret.second` is false. + template <class K, class M> + std::pair<iterator, bool> insert_or_assign_impl(K &&k, M &&obj) { + const std::pair<iterator, bool> ret = + this->tree_.insert_unique(k, std::forward<K>(k), std::forward<M>(obj)); + if (!ret.second) ret.first->second = std::forward<M>(obj); + return ret; + } + template <class K, class M> + iterator insert_or_assign_hint_impl(const_iterator hint, K &&k, M &&obj) { + const std::pair<iterator, bool> ret = this->tree_.insert_hint_unique( + iterator(hint), k, std::forward<K>(k), std::forward<M>(obj)); + if (!ret.second) ret.first->second = std::forward<M>(obj); + return ret.first; + } + + template <class K, class... Args> + std::pair<iterator, bool> try_emplace_impl(K &&k, Args &&... args) { + return this->tree_.insert_unique( + k, std::piecewise_construct, std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + } + template <class K, class... Args> + iterator try_emplace_hint_impl(const_iterator hint, K &&k, Args &&... args) { + return this->tree_ + .insert_hint_unique(iterator(hint), k, std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)) + .first; + } +}; + +// A common base class for btree_multiset and btree_multimap. +template <typename Tree> +class btree_multiset_container : public btree_container<Tree> { + using super_type = btree_container<Tree>; + using params_type = typename Tree::params_type; + using init_type = typename params_type::init_type; + using is_key_compare_to = typename params_type::is_key_compare_to; + + template <class K> + using key_arg = typename super_type::template key_arg<K>; + + public: + using key_type = typename Tree::key_type; + using value_type = typename Tree::value_type; + using size_type = typename Tree::size_type; + using key_compare = typename Tree::key_compare; + using allocator_type = typename Tree::allocator_type; + using iterator = typename Tree::iterator; + using const_iterator = typename Tree::const_iterator; + using node_type = typename super_type::node_type; + + // Inherit constructors. + using super_type::super_type; + btree_multiset_container() {} + + // Range constructors. + template <class InputIterator> + btree_multiset_container(InputIterator b, InputIterator e, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : super_type(comp, alloc) { + insert(b, e); + } + template <class InputIterator> + btree_multiset_container(InputIterator b, InputIterator e, + const allocator_type &alloc) + : btree_multiset_container(b, e, key_compare(), alloc) {} + + // Initializer list constructors. + btree_multiset_container(std::initializer_list<init_type> init, + const key_compare &comp = key_compare(), + const allocator_type &alloc = allocator_type()) + : btree_multiset_container(init.begin(), init.end(), comp, alloc) {} + btree_multiset_container(std::initializer_list<init_type> init, + const allocator_type &alloc) + : btree_multiset_container(init.begin(), init.end(), alloc) {} + + // Insertion routines. + iterator insert(const value_type &v) { return this->tree_.insert_multi(v); } + iterator insert(value_type &&v) { + return this->tree_.insert_multi(std::move(v)); + } + iterator insert(const_iterator hint, const value_type &v) { + return this->tree_.insert_hint_multi(iterator(hint), v); + } + iterator insert(const_iterator hint, value_type &&v) { + return this->tree_.insert_hint_multi(iterator(hint), std::move(v)); + } + template <typename InputIterator> + void insert(InputIterator b, InputIterator e) { + this->tree_.insert_iterator_multi(b, e); + } + void insert(std::initializer_list<init_type> init) { + this->tree_.insert_iterator_multi(init.begin(), init.end()); + } + template <typename... Args> + iterator emplace(Args &&... args) { + return this->tree_.insert_multi(init_type(std::forward<Args>(args)...)); + } + template <typename... Args> + iterator emplace_hint(const_iterator hint, Args &&... args) { + return this->tree_.insert_hint_multi( + iterator(hint), init_type(std::forward<Args>(args)...)); + } + iterator insert(node_type &&node) { + if (!node) return this->end(); + iterator res = + this->tree_.insert_multi(params_type::key(CommonAccess::GetSlot(node)), + CommonAccess::GetSlot(node)); + CommonAccess::Destroy(&node); + return res; + } + iterator insert(const_iterator hint, node_type &&node) { + if (!node) return this->end(); + iterator res = this->tree_.insert_hint_multi( + iterator(hint), + std::move(params_type::element(CommonAccess::GetSlot(node)))); + CommonAccess::Destroy(&node); + return res; + } + + // Node extraction routines. + // TODO(ezb): we are supposed to extract the first equivalent key if there are + // multiple, but this isn't guaranteed to extract the first one. + template <typename K = key_type> + node_type extract(const key_arg<K> &key) { + auto it = this->find(key); + return it == this->end() ? node_type() : extract(it); + } + using super_type::extract; + + // Merge routines. + // Moves all elements from `src` into `this`. + template < + typename T, + typename absl::enable_if_t< + absl::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &src) { // NOLINT + for (auto src_it = src.begin(), end = src.end(); src_it != end; ++src_it) { + insert(std::move(params_type::element(src_it.slot()))); + } + src.clear(); + } + + template < + typename T, + typename absl::enable_if_t< + absl::conjunction< + std::is_same<value_type, typename T::value_type>, + std::is_same<allocator_type, typename T::allocator_type>, + std::is_same<typename params_type::is_map_container, + typename T::params_type::is_map_container>>::value, + int> = 0> + void merge(btree_container<T> &&src) { + merge(src); + } +}; + +// A base class for btree_multimap. +template <typename Tree> +class btree_multimap_container : public btree_multiset_container<Tree> { + using super_type = btree_multiset_container<Tree>; + using params_type = typename Tree::params_type; + + public: + using mapped_type = typename params_type::mapped_type; + + // Inherit constructors. + using super_type::super_type; + btree_multimap_container() {} +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_BTREE_CONTAINER_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/common.h b/third_party/abseil_cpp/absl/container/internal/common.h new file mode 100644 index 000000000000..030e9d4ab07d --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/common.h @@ -0,0 +1,206 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_CONTAINER_H_ +#define ABSL_CONTAINER_INTERNAL_CONTAINER_H_ + +#include <cassert> +#include <type_traits> + +#include "absl/meta/type_traits.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class, class = void> +struct IsTransparent : std::false_type {}; +template <class T> +struct IsTransparent<T, absl::void_t<typename T::is_transparent>> + : std::true_type {}; + +template <bool is_transparent> +struct KeyArg { + // Transparent. Forward `K`. + template <typename K, typename key_type> + using type = K; +}; + +template <> +struct KeyArg<false> { + // Not transparent. Always use `key_type`. + template <typename K, typename key_type> + using type = key_type; +}; + +// The node_handle concept from C++17. +// We specialize node_handle for sets and maps. node_handle_base holds the +// common API of both. +template <typename PolicyTraits, typename Alloc> +class node_handle_base { + protected: + using slot_type = typename PolicyTraits::slot_type; + + public: + using allocator_type = Alloc; + + constexpr node_handle_base() = default; + node_handle_base(node_handle_base&& other) noexcept { + *this = std::move(other); + } + ~node_handle_base() { destroy(); } + node_handle_base& operator=(node_handle_base&& other) noexcept { + destroy(); + if (!other.empty()) { + alloc_ = other.alloc_; + PolicyTraits::transfer(alloc(), slot(), other.slot()); + other.reset(); + } + return *this; + } + + bool empty() const noexcept { return !alloc_; } + explicit operator bool() const noexcept { return !empty(); } + allocator_type get_allocator() const { return *alloc_; } + + protected: + friend struct CommonAccess; + + struct transfer_tag_t {}; + node_handle_base(transfer_tag_t, const allocator_type& a, slot_type* s) + : alloc_(a) { + PolicyTraits::transfer(alloc(), slot(), s); + } + + struct move_tag_t {}; + node_handle_base(move_tag_t, const allocator_type& a, slot_type* s) + : alloc_(a) { + PolicyTraits::construct(alloc(), slot(), s); + } + + void destroy() { + if (!empty()) { + PolicyTraits::destroy(alloc(), slot()); + reset(); + } + } + + void reset() { + assert(alloc_.has_value()); + alloc_ = absl::nullopt; + } + + slot_type* slot() const { + assert(!empty()); + return reinterpret_cast<slot_type*>(std::addressof(slot_space_)); + } + allocator_type* alloc() { return std::addressof(*alloc_); } + + private: + absl::optional<allocator_type> alloc_ = {}; + alignas(slot_type) mutable unsigned char slot_space_[sizeof(slot_type)] = {}; +}; + +// For sets. +template <typename Policy, typename PolicyTraits, typename Alloc, + typename = void> +class node_handle : public node_handle_base<PolicyTraits, Alloc> { + using Base = node_handle_base<PolicyTraits, Alloc>; + + public: + using value_type = typename PolicyTraits::value_type; + + constexpr node_handle() {} + + value_type& value() const { return PolicyTraits::element(this->slot()); } + + private: + friend struct CommonAccess; + + using Base::Base; +}; + +// For maps. +template <typename Policy, typename PolicyTraits, typename Alloc> +class node_handle<Policy, PolicyTraits, Alloc, + absl::void_t<typename Policy::mapped_type>> + : public node_handle_base<PolicyTraits, Alloc> { + using Base = node_handle_base<PolicyTraits, Alloc>; + using slot_type = typename PolicyTraits::slot_type; + + public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + + constexpr node_handle() {} + + // When C++17 is available, we can use std::launder to provide mutable + // access to the key. Otherwise, we provide const access. + auto key() const + -> decltype(PolicyTraits::mutable_key(std::declval<slot_type*>())) { + return PolicyTraits::mutable_key(this->slot()); + } + + mapped_type& mapped() const { + return PolicyTraits::value(&PolicyTraits::element(this->slot())); + } + + private: + friend struct CommonAccess; + + using Base::Base; +}; + +// Provide access to non-public node-handle functions. +struct CommonAccess { + template <typename Node> + static auto GetSlot(const Node& node) -> decltype(node.slot()) { + return node.slot(); + } + + template <typename Node> + static void Destroy(Node* node) { + node->destroy(); + } + + template <typename Node> + static void Reset(Node* node) { + node->reset(); + } + + template <typename T, typename... Args> + static T Transfer(Args&&... args) { + return T(typename T::transfer_tag_t{}, std::forward<Args>(args)...); + } + + template <typename T, typename... Args> + static T Move(Args&&... args) { + return T(typename T::move_tag_t{}, std::forward<Args>(args)...); + } +}; + +// Implement the insert_return_type<> concept of C++17. +template <class Iterator, class NodeType> +struct InsertReturnType { + Iterator position; + bool inserted; + NodeType node; +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_CONTAINER_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/compressed_tuple.h b/third_party/abseil_cpp/absl/container/internal/compressed_tuple.h new file mode 100644 index 000000000000..5ebe16494249 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/compressed_tuple.h @@ -0,0 +1,290 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Helper class to perform the Empty Base Optimization. +// Ts can contain classes and non-classes, empty or not. For the ones that +// are empty classes, we perform the optimization. If all types in Ts are empty +// classes, then CompressedTuple<Ts...> is itself an empty class. +// +// To access the members, use member get<N>() function. +// +// Eg: +// absl::container_internal::CompressedTuple<int, T1, T2, T3> value(7, t1, t2, +// t3); +// assert(value.get<0>() == 7); +// T1& t1 = value.get<1>(); +// const T2& t2 = value.get<2>(); +// ... +// +// https://en.cppreference.com/w/cpp/language/ebo + +#ifndef ABSL_CONTAINER_INTERNAL_COMPRESSED_TUPLE_H_ +#define ABSL_CONTAINER_INTERNAL_COMPRESSED_TUPLE_H_ + +#include <initializer_list> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/utility/utility.h" + +#if defined(_MSC_VER) && !defined(__NVCC__) +// We need to mark these classes with this declspec to ensure that +// CompressedTuple happens. +#define ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC __declspec(empty_bases) +#else +#define ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <typename... Ts> +class CompressedTuple; + +namespace internal_compressed_tuple { + +template <typename D, size_t I> +struct Elem; +template <typename... B, size_t I> +struct Elem<CompressedTuple<B...>, I> + : std::tuple_element<I, std::tuple<B...>> {}; +template <typename D, size_t I> +using ElemT = typename Elem<D, I>::type; + +// Use the __is_final intrinsic if available. Where it's not available, classes +// declared with the 'final' specifier cannot be used as CompressedTuple +// elements. +// TODO(sbenza): Replace this with std::is_final in C++14. +template <typename T> +constexpr bool IsFinal() { +#if defined(__clang__) || defined(__GNUC__) + return __is_final(T); +#else + return false; +#endif +} + +// We can't use EBCO on other CompressedTuples because that would mean that we +// derive from multiple Storage<> instantiations with the same I parameter, +// and potentially from multiple identical Storage<> instantiations. So anytime +// we use type inheritance rather than encapsulation, we mark +// CompressedTupleImpl, to make this easy to detect. +struct uses_inheritance {}; + +template <typename T> +constexpr bool ShouldUseBase() { + return std::is_class<T>::value && std::is_empty<T>::value && !IsFinal<T>() && + !std::is_base_of<uses_inheritance, T>::value; +} + +// The storage class provides two specializations: +// - For empty classes, it stores T as a base class. +// - For everything else, it stores T as a member. +template <typename T, size_t I, +#if defined(_MSC_VER) + bool UseBase = + ShouldUseBase<typename std::enable_if<true, T>::type>()> +#else + bool UseBase = ShouldUseBase<T>()> +#endif +struct Storage { + T value; + constexpr Storage() = default; + template <typename V> + explicit constexpr Storage(absl::in_place_t, V&& v) + : value(absl::forward<V>(v)) {} + constexpr const T& get() const& { return value; } + T& get() & { return value; } + constexpr const T&& get() const&& { return absl::move(*this).value; } + T&& get() && { return std::move(*this).value; } +}; + +template <typename T, size_t I> +struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC Storage<T, I, true> : T { + constexpr Storage() = default; + + template <typename V> + explicit constexpr Storage(absl::in_place_t, V&& v) + : T(absl::forward<V>(v)) {} + + constexpr const T& get() const& { return *this; } + T& get() & { return *this; } + constexpr const T&& get() const&& { return absl::move(*this); } + T&& get() && { return std::move(*this); } +}; + +template <typename D, typename I, bool ShouldAnyUseBase> +struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl; + +template <typename... Ts, size_t... I, bool ShouldAnyUseBase> +struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl< + CompressedTuple<Ts...>, absl::index_sequence<I...>, ShouldAnyUseBase> + // We use the dummy identity function through std::integral_constant to + // convince MSVC of accepting and expanding I in that context. Without it + // you would get: + // error C3548: 'I': parameter pack cannot be used in this context + : uses_inheritance, + Storage<Ts, std::integral_constant<size_t, I>::value>... { + constexpr CompressedTupleImpl() = default; + template <typename... Vs> + explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args) + : Storage<Ts, I>(absl::in_place, absl::forward<Vs>(args))... {} + friend CompressedTuple<Ts...>; +}; + +template <typename... Ts, size_t... I> +struct ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTupleImpl< + CompressedTuple<Ts...>, absl::index_sequence<I...>, false> + // We use the dummy identity function as above... + : Storage<Ts, std::integral_constant<size_t, I>::value, false>... { + constexpr CompressedTupleImpl() = default; + template <typename... Vs> + explicit constexpr CompressedTupleImpl(absl::in_place_t, Vs&&... args) + : Storage<Ts, I, false>(absl::in_place, absl::forward<Vs>(args))... {} + friend CompressedTuple<Ts...>; +}; + +std::false_type Or(std::initializer_list<std::false_type>); +std::true_type Or(std::initializer_list<bool>); + +// MSVC requires this to be done separately rather than within the declaration +// of CompressedTuple below. +template <typename... Ts> +constexpr bool ShouldAnyUseBase() { + return decltype( + Or({std::integral_constant<bool, ShouldUseBase<Ts>()>()...})){}; +} + +template <typename T, typename V> +using TupleElementMoveConstructible = + typename std::conditional<std::is_reference<T>::value, + std::is_convertible<V, T>, + std::is_constructible<T, V&&>>::type; + +template <bool SizeMatches, class T, class... Vs> +struct TupleMoveConstructible : std::false_type {}; + +template <class... Ts, class... Vs> +struct TupleMoveConstructible<true, CompressedTuple<Ts...>, Vs...> + : std::integral_constant< + bool, absl::conjunction< + TupleElementMoveConstructible<Ts, Vs&&>...>::value> {}; + +template <typename T> +struct compressed_tuple_size; + +template <typename... Es> +struct compressed_tuple_size<CompressedTuple<Es...>> + : public std::integral_constant<std::size_t, sizeof...(Es)> {}; + +template <class T, class... Vs> +struct TupleItemsMoveConstructible + : std::integral_constant< + bool, TupleMoveConstructible<compressed_tuple_size<T>::value == + sizeof...(Vs), + T, Vs...>::value> {}; + +} // namespace internal_compressed_tuple + +// Helper class to perform the Empty Base Class Optimization. +// Ts can contain classes and non-classes, empty or not. For the ones that +// are empty classes, we perform the CompressedTuple. If all types in Ts are +// empty classes, then CompressedTuple<Ts...> is itself an empty class. (This +// does not apply when one or more of those empty classes is itself an empty +// CompressedTuple.) +// +// To access the members, use member .get<N>() function. +// +// Eg: +// absl::container_internal::CompressedTuple<int, T1, T2, T3> value(7, t1, t2, +// t3); +// assert(value.get<0>() == 7); +// T1& t1 = value.get<1>(); +// const T2& t2 = value.get<2>(); +// ... +// +// https://en.cppreference.com/w/cpp/language/ebo +template <typename... Ts> +class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple + : private internal_compressed_tuple::CompressedTupleImpl< + CompressedTuple<Ts...>, absl::index_sequence_for<Ts...>, + internal_compressed_tuple::ShouldAnyUseBase<Ts...>()> { + private: + template <int I> + using ElemT = internal_compressed_tuple::ElemT<CompressedTuple, I>; + + template <int I> + using StorageT = internal_compressed_tuple::Storage<ElemT<I>, I>; + + public: + // There seems to be a bug in MSVC dealing in which using '=default' here will + // cause the compiler to ignore the body of other constructors. The work- + // around is to explicitly implement the default constructor. +#if defined(_MSC_VER) + constexpr CompressedTuple() : CompressedTuple::CompressedTupleImpl() {} +#else + constexpr CompressedTuple() = default; +#endif + explicit constexpr CompressedTuple(const Ts&... base) + : CompressedTuple::CompressedTupleImpl(absl::in_place, base...) {} + + template <typename First, typename... Vs, + absl::enable_if_t< + absl::conjunction< + // Ensure we are not hiding default copy/move constructors. + absl::negation<std::is_same<void(CompressedTuple), + void(absl::decay_t<First>)>>, + internal_compressed_tuple::TupleItemsMoveConstructible< + CompressedTuple<Ts...>, First, Vs...>>::value, + bool> = true> + explicit constexpr CompressedTuple(First&& first, Vs&&... base) + : CompressedTuple::CompressedTupleImpl(absl::in_place, + absl::forward<First>(first), + absl::forward<Vs>(base)...) {} + + template <int I> + ElemT<I>& get() & { + return StorageT<I>::get(); + } + + template <int I> + constexpr const ElemT<I>& get() const& { + return StorageT<I>::get(); + } + + template <int I> + ElemT<I>&& get() && { + return std::move(*this).StorageT<I>::get(); + } + + template <int I> + constexpr const ElemT<I>&& get() const&& { + return absl::move(*this).StorageT<I>::get(); + } +}; + +// Explicit specialization for a zero-element tuple +// (needed to avoid ambiguous overloads for the default constructor). +template <> +class ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC CompressedTuple<> {}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_INTERNAL_COMPRESSED_TUPLE_DECLSPEC + +#endif // ABSL_CONTAINER_INTERNAL_COMPRESSED_TUPLE_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/compressed_tuple_test.cc b/third_party/abseil_cpp/absl/container/internal/compressed_tuple_test.cc new file mode 100644 index 000000000000..62a7483ee311 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/compressed_tuple_test.cc @@ -0,0 +1,409 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/compressed_tuple.h" + +#include <memory> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/memory/memory.h" +#include "absl/types/any.h" +#include "absl/types/optional.h" +#include "absl/utility/utility.h" + +// These are declared at global scope purely so that error messages +// are smaller and easier to understand. +enum class CallType { kConstRef, kConstMove }; + +template <int> +struct Empty { + constexpr CallType value() const& { return CallType::kConstRef; } + constexpr CallType value() const&& { return CallType::kConstMove; } +}; + +template <typename T> +struct NotEmpty { + T value; +}; + +template <typename T, typename U> +struct TwoValues { + T value1; + U value2; +}; + + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using absl::test_internal::CopyableMovableInstance; +using absl::test_internal::InstanceTracker; + +TEST(CompressedTupleTest, Sizeof) { + EXPECT_EQ(sizeof(int), sizeof(CompressedTuple<int>)); + EXPECT_EQ(sizeof(int), sizeof(CompressedTuple<int, Empty<0>>)); + EXPECT_EQ(sizeof(int), sizeof(CompressedTuple<int, Empty<0>, Empty<1>>)); + EXPECT_EQ(sizeof(int), + sizeof(CompressedTuple<int, Empty<0>, Empty<1>, Empty<2>>)); + + EXPECT_EQ(sizeof(TwoValues<int, double>), + sizeof(CompressedTuple<int, NotEmpty<double>>)); + EXPECT_EQ(sizeof(TwoValues<int, double>), + sizeof(CompressedTuple<int, Empty<0>, NotEmpty<double>>)); + EXPECT_EQ(sizeof(TwoValues<int, double>), + sizeof(CompressedTuple<int, Empty<0>, NotEmpty<double>, Empty<1>>)); +} + +TEST(CompressedTupleTest, OneMoveOnRValueConstructionTemp) { + InstanceTracker tracker; + CompressedTuple<CopyableMovableInstance> x1(CopyableMovableInstance(1)); + EXPECT_EQ(tracker.instances(), 1); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_LE(tracker.moves(), 1); + EXPECT_EQ(x1.get<0>().value(), 1); +} + +TEST(CompressedTupleTest, OneMoveOnRValueConstructionMove) { + InstanceTracker tracker; + + CopyableMovableInstance i1(1); + CompressedTuple<CopyableMovableInstance> x1(std::move(i1)); + EXPECT_EQ(tracker.instances(), 2); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_LE(tracker.moves(), 1); + EXPECT_EQ(x1.get<0>().value(), 1); +} + +TEST(CompressedTupleTest, OneMoveOnRValueConstructionMixedTypes) { + InstanceTracker tracker; + CopyableMovableInstance i1(1); + CopyableMovableInstance i2(2); + Empty<0> empty; + CompressedTuple<CopyableMovableInstance, CopyableMovableInstance&, Empty<0>> + x1(std::move(i1), i2, empty); + EXPECT_EQ(x1.get<0>().value(), 1); + EXPECT_EQ(x1.get<1>().value(), 2); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 1); +} + +struct IncompleteType; +CompressedTuple<CopyableMovableInstance, IncompleteType&, Empty<0>> +MakeWithIncomplete(CopyableMovableInstance i1, + IncompleteType& t, // NOLINT + Empty<0> empty) { + return CompressedTuple<CopyableMovableInstance, IncompleteType&, Empty<0>>{ + std::move(i1), t, empty}; +} + +struct IncompleteType {}; +TEST(CompressedTupleTest, OneMoveOnRValueConstructionWithIncompleteType) { + InstanceTracker tracker; + CopyableMovableInstance i1(1); + Empty<0> empty; + struct DerivedType : IncompleteType {int value = 0;}; + DerivedType fd; + fd.value = 7; + + CompressedTuple<CopyableMovableInstance, IncompleteType&, Empty<0>> x1 = + MakeWithIncomplete(std::move(i1), fd, empty); + + EXPECT_EQ(x1.get<0>().value(), 1); + EXPECT_EQ(static_cast<DerivedType&>(x1.get<1>()).value, 7); + + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 2); +} + +TEST(CompressedTupleTest, + OneMoveOnRValueConstructionMixedTypes_BraceInitPoisonPillExpected) { + InstanceTracker tracker; + CopyableMovableInstance i1(1); + CopyableMovableInstance i2(2); + CompressedTuple<CopyableMovableInstance, CopyableMovableInstance&, Empty<0>> + x1(std::move(i1), i2, {}); // NOLINT + EXPECT_EQ(x1.get<0>().value(), 1); + EXPECT_EQ(x1.get<1>().value(), 2); + EXPECT_EQ(tracker.instances(), 3); + // We are forced into the `const Ts&...` constructor (invoking copies) + // because we need it to deduce the type of `{}`. + // std::tuple also has this behavior. + // Note, this test is proof that this is expected behavior, but it is not + // _desired_ behavior. + EXPECT_EQ(tracker.copies(), 1); + EXPECT_EQ(tracker.moves(), 0); +} + +TEST(CompressedTupleTest, OneCopyOnLValueConstruction) { + InstanceTracker tracker; + CopyableMovableInstance i1(1); + + CompressedTuple<CopyableMovableInstance> x1(i1); + EXPECT_EQ(tracker.copies(), 1); + EXPECT_EQ(tracker.moves(), 0); + + tracker.ResetCopiesMovesSwaps(); + + CopyableMovableInstance i2(2); + const CopyableMovableInstance& i2_ref = i2; + CompressedTuple<CopyableMovableInstance> x2(i2_ref); + EXPECT_EQ(tracker.copies(), 1); + EXPECT_EQ(tracker.moves(), 0); +} + +TEST(CompressedTupleTest, OneMoveOnRValueAccess) { + InstanceTracker tracker; + CopyableMovableInstance i1(1); + CompressedTuple<CopyableMovableInstance> x(std::move(i1)); + tracker.ResetCopiesMovesSwaps(); + + CopyableMovableInstance i2 = std::move(x).get<0>(); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 1); +} + +TEST(CompressedTupleTest, OneCopyOnLValueAccess) { + InstanceTracker tracker; + + CompressedTuple<CopyableMovableInstance> x(CopyableMovableInstance(0)); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 1); + + CopyableMovableInstance t = x.get<0>(); + EXPECT_EQ(tracker.copies(), 1); + EXPECT_EQ(tracker.moves(), 1); +} + +TEST(CompressedTupleTest, ZeroCopyOnRefAccess) { + InstanceTracker tracker; + + CompressedTuple<CopyableMovableInstance> x(CopyableMovableInstance(0)); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 1); + + CopyableMovableInstance& t1 = x.get<0>(); + const CopyableMovableInstance& t2 = x.get<0>(); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 1); + EXPECT_EQ(t1.value(), 0); + EXPECT_EQ(t2.value(), 0); +} + +TEST(CompressedTupleTest, Access) { + struct S { + std::string x; + }; + CompressedTuple<int, Empty<0>, S> x(7, {}, S{"ABC"}); + EXPECT_EQ(sizeof(x), sizeof(TwoValues<int, S>)); + EXPECT_EQ(7, x.get<0>()); + EXPECT_EQ("ABC", x.get<2>().x); +} + +TEST(CompressedTupleTest, NonClasses) { + CompressedTuple<int, const char*> x(7, "ABC"); + EXPECT_EQ(7, x.get<0>()); + EXPECT_STREQ("ABC", x.get<1>()); +} + +TEST(CompressedTupleTest, MixClassAndNonClass) { + CompressedTuple<int, const char*, Empty<0>, NotEmpty<double>> x(7, "ABC", {}, + {1.25}); + struct Mock { + int v; + const char* p; + double d; + }; + EXPECT_EQ(sizeof(x), sizeof(Mock)); + EXPECT_EQ(7, x.get<0>()); + EXPECT_STREQ("ABC", x.get<1>()); + EXPECT_EQ(1.25, x.get<3>().value); +} + +TEST(CompressedTupleTest, Nested) { + CompressedTuple<int, CompressedTuple<int>, + CompressedTuple<int, CompressedTuple<int>>> + x(1, CompressedTuple<int>(2), + CompressedTuple<int, CompressedTuple<int>>(3, CompressedTuple<int>(4))); + EXPECT_EQ(1, x.get<0>()); + EXPECT_EQ(2, x.get<1>().get<0>()); + EXPECT_EQ(3, x.get<2>().get<0>()); + EXPECT_EQ(4, x.get<2>().get<1>().get<0>()); + + CompressedTuple<Empty<0>, Empty<0>, + CompressedTuple<Empty<0>, CompressedTuple<Empty<0>>>> + y; + std::set<Empty<0>*> empties{&y.get<0>(), &y.get<1>(), &y.get<2>().get<0>(), + &y.get<2>().get<1>().get<0>()}; +#ifdef _MSC_VER + // MSVC has a bug where many instances of the same base class are layed out in + // the same address when using __declspec(empty_bases). + // This will be fixed in a future version of MSVC. + int expected = 1; +#else + int expected = 4; +#endif + EXPECT_EQ(expected, sizeof(y)); + EXPECT_EQ(expected, empties.size()); + EXPECT_EQ(sizeof(y), sizeof(Empty<0>) * empties.size()); + + EXPECT_EQ(4 * sizeof(char), + sizeof(CompressedTuple<CompressedTuple<char, char>, + CompressedTuple<char, char>>)); + EXPECT_TRUE((std::is_empty<CompressedTuple<Empty<0>, Empty<1>>>::value)); + + // Make sure everything still works when things are nested. + struct CT_Empty : CompressedTuple<Empty<0>> {}; + CompressedTuple<Empty<0>, CT_Empty> nested_empty; + auto contained = nested_empty.get<0>(); + auto nested = nested_empty.get<1>().get<0>(); + EXPECT_TRUE((std::is_same<decltype(contained), decltype(nested)>::value)); +} + +TEST(CompressedTupleTest, Reference) { + int i = 7; + std::string s = "Very long string that goes in the heap"; + CompressedTuple<int, int&, std::string, std::string&> x(i, i, s, s); + + // Sanity check. We should have not moved from `s` + EXPECT_EQ(s, "Very long string that goes in the heap"); + + EXPECT_EQ(x.get<0>(), x.get<1>()); + EXPECT_NE(&x.get<0>(), &x.get<1>()); + EXPECT_EQ(&x.get<1>(), &i); + + EXPECT_EQ(x.get<2>(), x.get<3>()); + EXPECT_NE(&x.get<2>(), &x.get<3>()); + EXPECT_EQ(&x.get<3>(), &s); +} + +TEST(CompressedTupleTest, NoElements) { + CompressedTuple<> x; + static_cast<void>(x); // Silence -Wunused-variable. + EXPECT_TRUE(std::is_empty<CompressedTuple<>>::value); +} + +TEST(CompressedTupleTest, MoveOnlyElements) { + CompressedTuple<std::unique_ptr<std::string>> str_tup( + absl::make_unique<std::string>("str")); + + CompressedTuple<CompressedTuple<std::unique_ptr<std::string>>, + std::unique_ptr<int>> + x(std::move(str_tup), absl::make_unique<int>(5)); + + EXPECT_EQ(*x.get<0>().get<0>(), "str"); + EXPECT_EQ(*x.get<1>(), 5); + + std::unique_ptr<std::string> x0 = std::move(x.get<0>()).get<0>(); + std::unique_ptr<int> x1 = std::move(x).get<1>(); + + EXPECT_EQ(*x0, "str"); + EXPECT_EQ(*x1, 5); +} + +TEST(CompressedTupleTest, MoveConstructionMoveOnlyElements) { + CompressedTuple<std::unique_ptr<std::string>> base( + absl::make_unique<std::string>("str")); + EXPECT_EQ(*base.get<0>(), "str"); + + CompressedTuple<std::unique_ptr<std::string>> copy(std::move(base)); + EXPECT_EQ(*copy.get<0>(), "str"); +} + +TEST(CompressedTupleTest, AnyElements) { + any a(std::string("str")); + CompressedTuple<any, any&> x(any(5), a); + EXPECT_EQ(absl::any_cast<int>(x.get<0>()), 5); + EXPECT_EQ(absl::any_cast<std::string>(x.get<1>()), "str"); + + a = 0.5f; + EXPECT_EQ(absl::any_cast<float>(x.get<1>()), 0.5); +} + +TEST(CompressedTupleTest, Constexpr) { + struct NonTrivialStruct { + constexpr NonTrivialStruct() = default; + constexpr int value() const { return v; } + int v = 5; + }; + struct TrivialStruct { + TrivialStruct() = default; + constexpr int value() const { return v; } + int v; + }; + constexpr CompressedTuple<int, double, CompressedTuple<int>, Empty<0>> x( + 7, 1.25, CompressedTuple<int>(5), {}); + constexpr int x0 = x.get<0>(); + constexpr double x1 = x.get<1>(); + constexpr int x2 = x.get<2>().get<0>(); + constexpr CallType x3 = x.get<3>().value(); + + EXPECT_EQ(x0, 7); + EXPECT_EQ(x1, 1.25); + EXPECT_EQ(x2, 5); + EXPECT_EQ(x3, CallType::kConstRef); + +#if !defined(__GNUC__) || defined(__clang__) || __GNUC__ > 4 + constexpr CompressedTuple<Empty<0>, TrivialStruct, int> trivial = {}; + constexpr CallType trivial0 = trivial.get<0>().value(); + constexpr int trivial1 = trivial.get<1>().value(); + constexpr int trivial2 = trivial.get<2>(); + + EXPECT_EQ(trivial0, CallType::kConstRef); + EXPECT_EQ(trivial1, 0); + EXPECT_EQ(trivial2, 0); +#endif + + constexpr CompressedTuple<Empty<0>, NonTrivialStruct, absl::optional<int>> + non_trivial = {}; + constexpr CallType non_trivial0 = non_trivial.get<0>().value(); + constexpr int non_trivial1 = non_trivial.get<1>().value(); + constexpr absl::optional<int> non_trivial2 = non_trivial.get<2>(); + + EXPECT_EQ(non_trivial0, CallType::kConstRef); + EXPECT_EQ(non_trivial1, 5); + EXPECT_EQ(non_trivial2, absl::nullopt); + + static constexpr char data[] = "DEF"; + constexpr CompressedTuple<const char*> z(data); + constexpr const char* z1 = z.get<0>(); + EXPECT_EQ(std::string(z1), std::string(data)); + +#if defined(__clang__) + // An apparent bug in earlier versions of gcc claims these are ambiguous. + constexpr int x2m = absl::move(x.get<2>()).get<0>(); + constexpr CallType x3m = absl::move(x).get<3>().value(); + EXPECT_EQ(x2m, 5); + EXPECT_EQ(x3m, CallType::kConstMove); +#endif +} + +#if defined(__clang__) || defined(__GNUC__) +TEST(CompressedTupleTest, EmptyFinalClass) { + struct S final { + int f() const { return 5; } + }; + CompressedTuple<S> x; + EXPECT_EQ(x.get<0>().f(), 5); +} +#endif + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/container_memory.h b/third_party/abseil_cpp/absl/container/internal/container_memory.h new file mode 100644 index 000000000000..e67529ecb6e6 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/container_memory.h @@ -0,0 +1,460 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_CONTAINER_MEMORY_H_ +#define ABSL_CONTAINER_INTERNAL_CONTAINER_MEMORY_H_ + +#include <cassert> +#include <cstddef> +#include <memory> +#include <new> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/base/config.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +#include <sanitizer/asan_interface.h> +#endif + +#ifdef ABSL_HAVE_MEMORY_SANITIZER +#include <sanitizer/msan_interface.h> +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <size_t Alignment> +struct alignas(Alignment) AlignedType {}; + +// Allocates at least n bytes aligned to the specified alignment. +// Alignment must be a power of 2. It must be positive. +// +// Note that many allocators don't honor alignment requirements above certain +// threshold (usually either alignof(std::max_align_t) or alignof(void*)). +// Allocate() doesn't apply alignment corrections. If the underlying allocator +// returns insufficiently alignment pointer, that's what you are going to get. +template <size_t Alignment, class Alloc> +void* Allocate(Alloc* alloc, size_t n) { + static_assert(Alignment > 0, ""); + assert(n && "n must be positive"); + using M = AlignedType<Alignment>; + using A = typename absl::allocator_traits<Alloc>::template rebind_alloc<M>; + using AT = typename absl::allocator_traits<Alloc>::template rebind_traits<M>; + // On macOS, "mem_alloc" is a #define with one argument defined in + // rpc/types.h, so we can't name the variable "mem_alloc" and initialize it + // with the "foo(bar)" syntax. + A my_mem_alloc(*alloc); + void* p = AT::allocate(my_mem_alloc, (n + sizeof(M) - 1) / sizeof(M)); + assert(reinterpret_cast<uintptr_t>(p) % Alignment == 0 && + "allocator does not respect alignment"); + return p; +} + +// The pointer must have been previously obtained by calling +// Allocate<Alignment>(alloc, n). +template <size_t Alignment, class Alloc> +void Deallocate(Alloc* alloc, void* p, size_t n) { + static_assert(Alignment > 0, ""); + assert(n && "n must be positive"); + using M = AlignedType<Alignment>; + using A = typename absl::allocator_traits<Alloc>::template rebind_alloc<M>; + using AT = typename absl::allocator_traits<Alloc>::template rebind_traits<M>; + // On macOS, "mem_alloc" is a #define with one argument defined in + // rpc/types.h, so we can't name the variable "mem_alloc" and initialize it + // with the "foo(bar)" syntax. + A my_mem_alloc(*alloc); + AT::deallocate(my_mem_alloc, static_cast<M*>(p), + (n + sizeof(M) - 1) / sizeof(M)); +} + +namespace memory_internal { + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +template <class Alloc, class T, class Tuple, size_t... I> +void ConstructFromTupleImpl(Alloc* alloc, T* ptr, Tuple&& t, + absl::index_sequence<I...>) { + absl::allocator_traits<Alloc>::construct( + *alloc, ptr, std::get<I>(std::forward<Tuple>(t))...); +} + +template <class T, class F> +struct WithConstructedImplF { + template <class... Args> + decltype(std::declval<F>()(std::declval<T>())) operator()( + Args&&... args) const { + return std::forward<F>(f)(T(std::forward<Args>(args)...)); + } + F&& f; +}; + +template <class T, class Tuple, size_t... Is, class F> +decltype(std::declval<F>()(std::declval<T>())) WithConstructedImpl( + Tuple&& t, absl::index_sequence<Is...>, F&& f) { + return WithConstructedImplF<T, F>{std::forward<F>(f)}( + std::get<Is>(std::forward<Tuple>(t))...); +} + +template <class T, size_t... Is> +auto TupleRefImpl(T&& t, absl::index_sequence<Is...>) + -> decltype(std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...)) { + return std::forward_as_tuple(std::get<Is>(std::forward<T>(t))...); +} + +// Returns a tuple of references to the elements of the input tuple. T must be a +// tuple. +template <class T> +auto TupleRef(T&& t) -> decltype( + TupleRefImpl(std::forward<T>(t), + absl::make_index_sequence< + std::tuple_size<typename std::decay<T>::type>::value>())) { + return TupleRefImpl( + std::forward<T>(t), + absl::make_index_sequence< + std::tuple_size<typename std::decay<T>::type>::value>()); +} + +template <class F, class K, class V> +decltype(std::declval<F>()(std::declval<const K&>(), std::piecewise_construct, + std::declval<std::tuple<K>>(), std::declval<V>())) +DecomposePairImpl(F&& f, std::pair<std::tuple<K>, V> p) { + const auto& key = std::get<0>(p.first); + return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); +} + +} // namespace memory_internal + +// Constructs T into uninitialized storage pointed by `ptr` using the args +// specified in the tuple. +template <class Alloc, class T, class Tuple> +void ConstructFromTuple(Alloc* alloc, T* ptr, Tuple&& t) { + memory_internal::ConstructFromTupleImpl( + alloc, ptr, std::forward<Tuple>(t), + absl::make_index_sequence< + std::tuple_size<typename std::decay<Tuple>::type>::value>()); +} + +// Constructs T using the args specified in the tuple and calls F with the +// constructed value. +template <class T, class Tuple, class F> +decltype(std::declval<F>()(std::declval<T>())) WithConstructed( + Tuple&& t, F&& f) { + return memory_internal::WithConstructedImpl<T>( + std::forward<Tuple>(t), + absl::make_index_sequence< + std::tuple_size<typename std::decay<Tuple>::type>::value>(), + std::forward<F>(f)); +} + +// Given arguments of an std::pair's consructor, PairArgs() returns a pair of +// tuples with references to the passed arguments. The tuples contain +// constructor arguments for the first and the second elements of the pair. +// +// The following two snippets are equivalent. +// +// 1. std::pair<F, S> p(args...); +// +// 2. auto a = PairArgs(args...); +// std::pair<F, S> p(std::piecewise_construct, +// std::move(p.first), std::move(p.second)); +inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; } +template <class F, class S> +std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) { + return {std::piecewise_construct, std::forward_as_tuple(std::forward<F>(f)), + std::forward_as_tuple(std::forward<S>(s))}; +} +template <class F, class S> +std::pair<std::tuple<const F&>, std::tuple<const S&>> PairArgs( + const std::pair<F, S>& p) { + return PairArgs(p.first, p.second); +} +template <class F, class S> +std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(std::pair<F, S>&& p) { + return PairArgs(std::forward<F>(p.first), std::forward<S>(p.second)); +} +template <class F, class S> +auto PairArgs(std::piecewise_construct_t, F&& f, S&& s) + -> decltype(std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), + memory_internal::TupleRef(std::forward<S>(s)))) { + return std::make_pair(memory_internal::TupleRef(std::forward<F>(f)), + memory_internal::TupleRef(std::forward<S>(s))); +} + +// A helper function for implementing apply() in map policies. +template <class F, class... Args> +auto DecomposePair(F&& f, Args&&... args) + -> decltype(memory_internal::DecomposePairImpl( + std::forward<F>(f), PairArgs(std::forward<Args>(args)...))) { + return memory_internal::DecomposePairImpl( + std::forward<F>(f), PairArgs(std::forward<Args>(args)...)); +} + +// A helper function for implementing apply() in set policies. +template <class F, class Arg> +decltype(std::declval<F>()(std::declval<const Arg&>(), std::declval<Arg>())) +DecomposeValue(F&& f, Arg&& arg) { + const auto& key = arg; + return std::forward<F>(f)(key, std::forward<Arg>(arg)); +} + +// Helper functions for asan and msan. +inline void SanitizerPoisonMemoryRegion(const void* m, size_t s) { +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + ASAN_POISON_MEMORY_REGION(m, s); +#endif +#ifdef ABSL_HAVE_MEMORY_SANITIZER + __msan_poison(m, s); +#endif + (void)m; + (void)s; +} + +inline void SanitizerUnpoisonMemoryRegion(const void* m, size_t s) { +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + ASAN_UNPOISON_MEMORY_REGION(m, s); +#endif +#ifdef ABSL_HAVE_MEMORY_SANITIZER + __msan_unpoison(m, s); +#endif + (void)m; + (void)s; +} + +template <typename T> +inline void SanitizerPoisonObject(const T* object) { + SanitizerPoisonMemoryRegion(object, sizeof(T)); +} + +template <typename T> +inline void SanitizerUnpoisonObject(const T* object) { + SanitizerUnpoisonMemoryRegion(object, sizeof(T)); +} + +namespace memory_internal { + +// If Pair is a standard-layout type, OffsetOf<Pair>::kFirst and +// OffsetOf<Pair>::kSecond are equivalent to offsetof(Pair, first) and +// offsetof(Pair, second) respectively. Otherwise they are -1. +// +// The purpose of OffsetOf is to avoid calling offsetof() on non-standard-layout +// type, which is non-portable. +template <class Pair, class = std::true_type> +struct OffsetOf { + static constexpr size_t kFirst = static_cast<size_t>(-1); + static constexpr size_t kSecond = static_cast<size_t>(-1); +}; + +template <class Pair> +struct OffsetOf<Pair, typename std::is_standard_layout<Pair>::type> { + static constexpr size_t kFirst = offsetof(Pair, first); + static constexpr size_t kSecond = offsetof(Pair, second); +}; + +template <class K, class V> +struct IsLayoutCompatible { + private: + struct Pair { + K first; + V second; + }; + + // Is P layout-compatible with Pair? + template <class P> + static constexpr bool LayoutCompatible() { + return std::is_standard_layout<P>() && sizeof(P) == sizeof(Pair) && + alignof(P) == alignof(Pair) && + memory_internal::OffsetOf<P>::kFirst == + memory_internal::OffsetOf<Pair>::kFirst && + memory_internal::OffsetOf<P>::kSecond == + memory_internal::OffsetOf<Pair>::kSecond; + } + + public: + // Whether pair<const K, V> and pair<K, V> are layout-compatible. If they are, + // then it is safe to store them in a union and read from either. + static constexpr bool value = std::is_standard_layout<K>() && + std::is_standard_layout<Pair>() && + memory_internal::OffsetOf<Pair>::kFirst == 0 && + LayoutCompatible<std::pair<K, V>>() && + LayoutCompatible<std::pair<const K, V>>(); +}; + +} // namespace memory_internal + +// The internal storage type for key-value containers like flat_hash_map. +// +// It is convenient for the value_type of a flat_hash_map<K, V> to be +// pair<const K, V>; the "const K" prevents accidental modification of the key +// when dealing with the reference returned from find() and similar methods. +// However, this creates other problems; we want to be able to emplace(K, V) +// efficiently with move operations, and similarly be able to move a +// pair<K, V> in insert(). +// +// The solution is this union, which aliases the const and non-const versions +// of the pair. This also allows flat_hash_map<const K, V> to work, even though +// that has the same efficiency issues with move in emplace() and insert() - +// but people do it anyway. +// +// If kMutableKeys is false, only the value member can be accessed. +// +// If kMutableKeys is true, key can be accessed through all slots while value +// and mutable_value must be accessed only via INITIALIZED slots. Slots are +// created and destroyed via mutable_value so that the key can be moved later. +// +// Accessing one of the union fields while the other is active is safe as +// long as they are layout-compatible, which is guaranteed by the definition of +// kMutableKeys. For C++11, the relevant section of the standard is +// https://timsong-cpp.github.io/cppwp/n3337/class.mem#19 (9.2.19) +template <class K, class V> +union map_slot_type { + map_slot_type() {} + ~map_slot_type() = delete; + using value_type = std::pair<const K, V>; + using mutable_value_type = + std::pair<absl::remove_const_t<K>, absl::remove_const_t<V>>; + + value_type value; + mutable_value_type mutable_value; + absl::remove_const_t<K> key; +}; + +template <class K, class V> +struct map_slot_policy { + using slot_type = map_slot_type<K, V>; + using value_type = std::pair<const K, V>; + using mutable_value_type = std::pair<K, V>; + + private: + static void emplace(slot_type* slot) { + // The construction of union doesn't do anything at runtime but it allows us + // to access its members without violating aliasing rules. + new (slot) slot_type; + } + // If pair<const K, V> and pair<K, V> are layout-compatible, we can accept one + // or the other via slot_type. We are also free to access the key via + // slot_type::key in this case. + using kMutableKeys = memory_internal::IsLayoutCompatible<K, V>; + + public: + static value_type& element(slot_type* slot) { return slot->value; } + static const value_type& element(const slot_type* slot) { + return slot->value; + } + + // When C++17 is available, we can use std::launder to provide mutable + // access to the key for use in node handle. +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 + static K& mutable_key(slot_type* slot) { + // Still check for kMutableKeys so that we can avoid calling std::launder + // unless necessary because it can interfere with optimizations. + return kMutableKeys::value ? slot->key + : *std::launder(const_cast<K*>( + std::addressof(slot->value.first))); + } +#else // !(defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606) + static const K& mutable_key(slot_type* slot) { return key(slot); } +#endif + + static const K& key(const slot_type* slot) { + return kMutableKeys::value ? slot->key : slot->value.first; + } + + template <class Allocator, class... Args> + static void construct(Allocator* alloc, slot_type* slot, Args&&... args) { + emplace(slot); + if (kMutableKeys::value) { + absl::allocator_traits<Allocator>::construct(*alloc, &slot->mutable_value, + std::forward<Args>(args)...); + } else { + absl::allocator_traits<Allocator>::construct(*alloc, &slot->value, + std::forward<Args>(args)...); + } + } + + // Construct this slot by moving from another slot. + template <class Allocator> + static void construct(Allocator* alloc, slot_type* slot, slot_type* other) { + emplace(slot); + if (kMutableKeys::value) { + absl::allocator_traits<Allocator>::construct( + *alloc, &slot->mutable_value, std::move(other->mutable_value)); + } else { + absl::allocator_traits<Allocator>::construct(*alloc, &slot->value, + std::move(other->value)); + } + } + + template <class Allocator> + static void destroy(Allocator* alloc, slot_type* slot) { + if (kMutableKeys::value) { + absl::allocator_traits<Allocator>::destroy(*alloc, &slot->mutable_value); + } else { + absl::allocator_traits<Allocator>::destroy(*alloc, &slot->value); + } + } + + template <class Allocator> + static void transfer(Allocator* alloc, slot_type* new_slot, + slot_type* old_slot) { + emplace(new_slot); + if (kMutableKeys::value) { + absl::allocator_traits<Allocator>::construct( + *alloc, &new_slot->mutable_value, std::move(old_slot->mutable_value)); + } else { + absl::allocator_traits<Allocator>::construct(*alloc, &new_slot->value, + std::move(old_slot->value)); + } + destroy(alloc, old_slot); + } + + template <class Allocator> + static void swap(Allocator* alloc, slot_type* a, slot_type* b) { + if (kMutableKeys::value) { + using std::swap; + swap(a->mutable_value, b->mutable_value); + } else { + value_type tmp = std::move(a->value); + absl::allocator_traits<Allocator>::destroy(*alloc, &a->value); + absl::allocator_traits<Allocator>::construct(*alloc, &a->value, + std::move(b->value)); + absl::allocator_traits<Allocator>::destroy(*alloc, &b->value); + absl::allocator_traits<Allocator>::construct(*alloc, &b->value, + std::move(tmp)); + } + } + + template <class Allocator> + static void move(Allocator* alloc, slot_type* src, slot_type* dest) { + if (kMutableKeys::value) { + dest->mutable_value = std::move(src->mutable_value); + } else { + absl::allocator_traits<Allocator>::destroy(*alloc, &dest->value); + absl::allocator_traits<Allocator>::construct(*alloc, &dest->value, + std::move(src->value)); + } + } +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_CONTAINER_MEMORY_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/container_memory_test.cc b/third_party/abseil_cpp/absl/container/internal/container_memory_test.cc new file mode 100644 index 000000000000..6a7fcd29ba90 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/container_memory_test.cc @@ -0,0 +1,256 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/container_memory.h" + +#include <cstdint> +#include <tuple> +#include <typeindex> +#include <typeinfo> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::absl::test_internal::CopyableMovableInstance; +using ::absl::test_internal::InstanceTracker; +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Gt; +using ::testing::Pair; + +TEST(Memory, AlignmentLargerThanBase) { + std::allocator<int8_t> alloc; + void* mem = Allocate<2>(&alloc, 3); + EXPECT_EQ(0, reinterpret_cast<uintptr_t>(mem) % 2); + memcpy(mem, "abc", 3); + Deallocate<2>(&alloc, mem, 3); +} + +TEST(Memory, AlignmentSmallerThanBase) { + std::allocator<int64_t> alloc; + void* mem = Allocate<2>(&alloc, 3); + EXPECT_EQ(0, reinterpret_cast<uintptr_t>(mem) % 2); + memcpy(mem, "abc", 3); + Deallocate<2>(&alloc, mem, 3); +} + +std::map<std::type_index, int>& AllocationMap() { + static auto* map = new std::map<std::type_index, int>; + return *map; +} + +template <typename T> +struct TypeCountingAllocator { + TypeCountingAllocator() = default; + template <typename U> + TypeCountingAllocator(const TypeCountingAllocator<U>&) {} // NOLINT + + using value_type = T; + + T* allocate(size_t n, const void* = nullptr) { + AllocationMap()[typeid(T)] += n; + return std::allocator<T>().allocate(n); + } + void deallocate(T* p, std::size_t n) { + AllocationMap()[typeid(T)] -= n; + return std::allocator<T>().deallocate(p, n); + } +}; + +TEST(Memory, AllocateDeallocateMatchType) { + TypeCountingAllocator<int> alloc; + void* mem = Allocate<1>(&alloc, 1); + // Verify that it was allocated + EXPECT_THAT(AllocationMap(), ElementsAre(Pair(_, Gt(0)))); + Deallocate<1>(&alloc, mem, 1); + // Verify that the deallocation matched. + EXPECT_THAT(AllocationMap(), ElementsAre(Pair(_, 0))); +} + +class Fixture : public ::testing::Test { + using Alloc = std::allocator<std::string>; + + public: + Fixture() { ptr_ = std::allocator_traits<Alloc>::allocate(*alloc(), 1); } + ~Fixture() override { + std::allocator_traits<Alloc>::destroy(*alloc(), ptr_); + std::allocator_traits<Alloc>::deallocate(*alloc(), ptr_, 1); + } + std::string* ptr() { return ptr_; } + Alloc* alloc() { return &alloc_; } + + private: + Alloc alloc_; + std::string* ptr_; +}; + +TEST_F(Fixture, ConstructNoArgs) { + ConstructFromTuple(alloc(), ptr(), std::forward_as_tuple()); + EXPECT_EQ(*ptr(), ""); +} + +TEST_F(Fixture, ConstructOneArg) { + ConstructFromTuple(alloc(), ptr(), std::forward_as_tuple("abcde")); + EXPECT_EQ(*ptr(), "abcde"); +} + +TEST_F(Fixture, ConstructTwoArg) { + ConstructFromTuple(alloc(), ptr(), std::forward_as_tuple(5, 'a')); + EXPECT_EQ(*ptr(), "aaaaa"); +} + +TEST(PairArgs, NoArgs) { + EXPECT_THAT(PairArgs(), + Pair(std::forward_as_tuple(), std::forward_as_tuple())); +} + +TEST(PairArgs, TwoArgs) { + EXPECT_EQ( + std::make_pair(std::forward_as_tuple(1), std::forward_as_tuple('A')), + PairArgs(1, 'A')); +} + +TEST(PairArgs, Pair) { + EXPECT_EQ( + std::make_pair(std::forward_as_tuple(1), std::forward_as_tuple('A')), + PairArgs(std::make_pair(1, 'A'))); +} + +TEST(PairArgs, Piecewise) { + EXPECT_EQ( + std::make_pair(std::forward_as_tuple(1), std::forward_as_tuple('A')), + PairArgs(std::piecewise_construct, std::forward_as_tuple(1), + std::forward_as_tuple('A'))); +} + +TEST(WithConstructed, Simple) { + EXPECT_EQ(1, WithConstructed<absl::string_view>( + std::make_tuple(std::string("a")), + [](absl::string_view str) { return str.size(); })); +} + +template <class F, class Arg> +decltype(DecomposeValue(std::declval<F>(), std::declval<Arg>())) +DecomposeValueImpl(int, F&& f, Arg&& arg) { + return DecomposeValue(std::forward<F>(f), std::forward<Arg>(arg)); +} + +template <class F, class Arg> +const char* DecomposeValueImpl(char, F&& f, Arg&& arg) { + return "not decomposable"; +} + +template <class F, class Arg> +decltype(DecomposeValueImpl(0, std::declval<F>(), std::declval<Arg>())) +TryDecomposeValue(F&& f, Arg&& arg) { + return DecomposeValueImpl(0, std::forward<F>(f), std::forward<Arg>(arg)); +} + +TEST(DecomposeValue, Decomposable) { + auto f = [](const int& x, int&& y) { + EXPECT_EQ(&x, &y); + EXPECT_EQ(42, x); + return 'A'; + }; + EXPECT_EQ('A', TryDecomposeValue(f, 42)); +} + +TEST(DecomposeValue, NotDecomposable) { + auto f = [](void*) { + ADD_FAILURE() << "Must not be called"; + return 'A'; + }; + EXPECT_STREQ("not decomposable", TryDecomposeValue(f, 42)); +} + +template <class F, class... Args> +decltype(DecomposePair(std::declval<F>(), std::declval<Args>()...)) +DecomposePairImpl(int, F&& f, Args&&... args) { + return DecomposePair(std::forward<F>(f), std::forward<Args>(args)...); +} + +template <class F, class... Args> +const char* DecomposePairImpl(char, F&& f, Args&&... args) { + return "not decomposable"; +} + +template <class F, class... Args> +decltype(DecomposePairImpl(0, std::declval<F>(), std::declval<Args>()...)) +TryDecomposePair(F&& f, Args&&... args) { + return DecomposePairImpl(0, std::forward<F>(f), std::forward<Args>(args)...); +} + +TEST(DecomposePair, Decomposable) { + auto f = [](const int& x, std::piecewise_construct_t, std::tuple<int&&> k, + std::tuple<double>&& v) { + EXPECT_EQ(&x, &std::get<0>(k)); + EXPECT_EQ(42, x); + EXPECT_EQ(0.5, std::get<0>(v)); + return 'A'; + }; + EXPECT_EQ('A', TryDecomposePair(f, 42, 0.5)); + EXPECT_EQ('A', TryDecomposePair(f, std::make_pair(42, 0.5))); + EXPECT_EQ('A', TryDecomposePair(f, std::piecewise_construct, + std::make_tuple(42), std::make_tuple(0.5))); +} + +TEST(DecomposePair, NotDecomposable) { + auto f = [](...) { + ADD_FAILURE() << "Must not be called"; + return 'A'; + }; + EXPECT_STREQ("not decomposable", + TryDecomposePair(f)); + EXPECT_STREQ("not decomposable", + TryDecomposePair(f, std::piecewise_construct, std::make_tuple(), + std::make_tuple(0.5))); +} + +TEST(MapSlotPolicy, ConstKeyAndValue) { + using slot_policy = map_slot_policy<const CopyableMovableInstance, + const CopyableMovableInstance>; + using slot_type = typename slot_policy::slot_type; + + union Slots { + Slots() {} + ~Slots() {} + slot_type slots[100]; + } slots; + + std::allocator< + std::pair<const CopyableMovableInstance, const CopyableMovableInstance>> + alloc; + InstanceTracker tracker; + slot_policy::construct(&alloc, &slots.slots[0], CopyableMovableInstance(1), + CopyableMovableInstance(1)); + for (int i = 0; i < 99; ++i) { + slot_policy::transfer(&alloc, &slots.slots[i + 1], &slots.slots[i]); + } + slot_policy::destroy(&alloc, &slots.slots[99]); + + EXPECT_EQ(tracker.copies(), 0); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/counting_allocator.h b/third_party/abseil_cpp/absl/container/internal/counting_allocator.h new file mode 100644 index 000000000000..927cf0825579 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/counting_allocator.h @@ -0,0 +1,114 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_COUNTING_ALLOCATOR_H_ +#define ABSL_CONTAINER_INTERNAL_COUNTING_ALLOCATOR_H_ + +#include <cstdint> +#include <memory> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// This is a stateful allocator, but the state lives outside of the +// allocator (in whatever test is using the allocator). This is odd +// but helps in tests where the allocator is propagated into nested +// containers - that chain of allocators uses the same state and is +// thus easier to query for aggregate allocation information. +template <typename T> +class CountingAllocator { + public: + using Allocator = std::allocator<T>; + using AllocatorTraits = std::allocator_traits<Allocator>; + using value_type = typename AllocatorTraits::value_type; + using pointer = typename AllocatorTraits::pointer; + using const_pointer = typename AllocatorTraits::const_pointer; + using size_type = typename AllocatorTraits::size_type; + using difference_type = typename AllocatorTraits::difference_type; + + CountingAllocator() = default; + explicit CountingAllocator(int64_t* bytes_used) : bytes_used_(bytes_used) {} + CountingAllocator(int64_t* bytes_used, int64_t* instance_count) + : bytes_used_(bytes_used), instance_count_(instance_count) {} + + template <typename U> + CountingAllocator(const CountingAllocator<U>& x) + : bytes_used_(x.bytes_used_), instance_count_(x.instance_count_) {} + + pointer allocate( + size_type n, + typename AllocatorTraits::const_void_pointer hint = nullptr) { + Allocator allocator; + pointer ptr = AllocatorTraits::allocate(allocator, n, hint); + if (bytes_used_ != nullptr) { + *bytes_used_ += n * sizeof(T); + } + return ptr; + } + + void deallocate(pointer p, size_type n) { + Allocator allocator; + AllocatorTraits::deallocate(allocator, p, n); + if (bytes_used_ != nullptr) { + *bytes_used_ -= n * sizeof(T); + } + } + + template <typename U, typename... Args> + void construct(U* p, Args&&... args) { + Allocator allocator; + AllocatorTraits::construct(allocator, p, std::forward<Args>(args)...); + if (instance_count_ != nullptr) { + *instance_count_ += 1; + } + } + + template <typename U> + void destroy(U* p) { + Allocator allocator; + AllocatorTraits::destroy(allocator, p); + if (instance_count_ != nullptr) { + *instance_count_ -= 1; + } + } + + template <typename U> + class rebind { + public: + using other = CountingAllocator<U>; + }; + + friend bool operator==(const CountingAllocator& a, + const CountingAllocator& b) { + return a.bytes_used_ == b.bytes_used_ && + a.instance_count_ == b.instance_count_; + } + + friend bool operator!=(const CountingAllocator& a, + const CountingAllocator& b) { + return !(a == b); + } + + int64_t* bytes_used_ = nullptr; + int64_t* instance_count_ = nullptr; +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_COUNTING_ALLOCATOR_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hash_function_defaults.h b/third_party/abseil_cpp/absl/container/internal/hash_function_defaults.h new file mode 100644 index 000000000000..0683422ad89b --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_function_defaults.h @@ -0,0 +1,161 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Define the default Hash and Eq functions for SwissTable containers. +// +// std::hash<T> and std::equal_to<T> are not appropriate hash and equal +// functions for SwissTable containers. There are two reasons for this. +// +// SwissTable containers are power of 2 sized containers: +// +// This means they use the lower bits of the hash value to find the slot for +// each entry. The typical hash function for integral types is the identity. +// This is a very weak hash function for SwissTable and any power of 2 sized +// hashtable implementation which will lead to excessive collisions. For +// SwissTable we use murmur3 style mixing to reduce collisions to a minimum. +// +// SwissTable containers support heterogeneous lookup: +// +// In order to make heterogeneous lookup work, hash and equal functions must be +// polymorphic. At the same time they have to satisfy the same requirements the +// C++ standard imposes on hash functions and equality operators. That is: +// +// if hash_default_eq<T>(a, b) returns true for any a and b of type T, then +// hash_default_hash<T>(a) must equal hash_default_hash<T>(b) +// +// For SwissTable containers this requirement is relaxed to allow a and b of +// any and possibly different types. Note that like the standard the hash and +// equal functions are still bound to T. This is important because some type U +// can be hashed by/tested for equality differently depending on T. A notable +// example is `const char*`. `const char*` is treated as a c-style string when +// the hash function is hash<std::string> but as a pointer when the hash +// function is hash<void*>. +// +#ifndef ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ +#define ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ + +#include <stdint.h> +#include <cstddef> +#include <memory> +#include <string> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/hash/hash.h" +#include "absl/strings/cord.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// The hash of an object of type T is computed by using absl::Hash. +template <class T, class E = void> +struct HashEq { + using Hash = absl::Hash<T>; + using Eq = std::equal_to<T>; +}; + +struct StringHash { + using is_transparent = void; + + size_t operator()(absl::string_view v) const { + return absl::Hash<absl::string_view>{}(v); + } + size_t operator()(const absl::Cord& v) const { + return absl::Hash<absl::Cord>{}(v); + } +}; + +// Supports heterogeneous lookup for string-like elements. +struct StringHashEq { + using Hash = StringHash; + struct Eq { + using is_transparent = void; + bool operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs == rhs; + } + bool operator()(const absl::Cord& lhs, const absl::Cord& rhs) const { + return lhs == rhs; + } + bool operator()(const absl::Cord& lhs, absl::string_view rhs) const { + return lhs == rhs; + } + bool operator()(absl::string_view lhs, const absl::Cord& rhs) const { + return lhs == rhs; + } + }; +}; + +template <> +struct HashEq<std::string> : StringHashEq {}; +template <> +struct HashEq<absl::string_view> : StringHashEq {}; +template <> +struct HashEq<absl::Cord> : StringHashEq {}; + +// Supports heterogeneous lookup for pointers and smart pointers. +template <class T> +struct HashEq<T*> { + struct Hash { + using is_transparent = void; + template <class U> + size_t operator()(const U& ptr) const { + return absl::Hash<const T*>{}(HashEq::ToPtr(ptr)); + } + }; + struct Eq { + using is_transparent = void; + template <class A, class B> + bool operator()(const A& a, const B& b) const { + return HashEq::ToPtr(a) == HashEq::ToPtr(b); + } + }; + + private: + static const T* ToPtr(const T* ptr) { return ptr; } + template <class U, class D> + static const T* ToPtr(const std::unique_ptr<U, D>& ptr) { + return ptr.get(); + } + template <class U> + static const T* ToPtr(const std::shared_ptr<U>& ptr) { + return ptr.get(); + } +}; + +template <class T, class D> +struct HashEq<std::unique_ptr<T, D>> : HashEq<T*> {}; +template <class T> +struct HashEq<std::shared_ptr<T>> : HashEq<T*> {}; + +// This header's visibility is restricted. If you need to access the default +// hasher please use the container's ::hasher alias instead. +// +// Example: typename Hash = typename absl::flat_hash_map<K, V>::hasher +template <class T> +using hash_default_hash = typename container_internal::HashEq<T>::Hash; + +// This header's visibility is restricted. If you need to access the default +// key equal please use the container's ::key_equal alias instead. +// +// Example: typename Eq = typename absl::flat_hash_map<K, V, Hash>::key_equal +template <class T> +using hash_default_eq = typename container_internal::HashEq<T>::Eq; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_HASH_FUNCTION_DEFAULTS_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hash_function_defaults_test.cc b/third_party/abseil_cpp/absl/container/internal/hash_function_defaults_test.cc new file mode 100644 index 000000000000..59576b8edebc --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_function_defaults_test.cc @@ -0,0 +1,383 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hash_function_defaults.h" + +#include <functional> +#include <type_traits> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/random/random.h" +#include "absl/strings/cord.h" +#include "absl/strings/cord_test_helpers.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::testing::Types; + +TEST(Eq, Int32) { + hash_default_eq<int32_t> eq; + EXPECT_TRUE(eq(1, 1u)); + EXPECT_TRUE(eq(1, char{1})); + EXPECT_TRUE(eq(1, true)); + EXPECT_TRUE(eq(1, double{1.1})); + EXPECT_FALSE(eq(1, char{2})); + EXPECT_FALSE(eq(1, 2u)); + EXPECT_FALSE(eq(1, false)); + EXPECT_FALSE(eq(1, 2.)); +} + +TEST(Hash, Int32) { + hash_default_hash<int32_t> hash; + auto h = hash(1); + EXPECT_EQ(h, hash(1u)); + EXPECT_EQ(h, hash(char{1})); + EXPECT_EQ(h, hash(true)); + EXPECT_EQ(h, hash(double{1.1})); + EXPECT_NE(h, hash(2u)); + EXPECT_NE(h, hash(char{2})); + EXPECT_NE(h, hash(false)); + EXPECT_NE(h, hash(2.)); +} + +enum class MyEnum { A, B, C, D }; + +TEST(Eq, Enum) { + hash_default_eq<MyEnum> eq; + EXPECT_TRUE(eq(MyEnum::A, MyEnum::A)); + EXPECT_FALSE(eq(MyEnum::A, MyEnum::B)); +} + +TEST(Hash, Enum) { + hash_default_hash<MyEnum> hash; + + for (MyEnum e : {MyEnum::A, MyEnum::B, MyEnum::C}) { + auto h = hash(e); + EXPECT_EQ(h, hash_default_hash<int>{}(static_cast<int>(e))); + EXPECT_NE(h, hash(MyEnum::D)); + } +} + +using StringTypes = ::testing::Types<std::string, absl::string_view>; + +template <class T> +struct EqString : ::testing::Test { + hash_default_eq<T> key_eq; +}; + +TYPED_TEST_SUITE(EqString, StringTypes); + +template <class T> +struct HashString : ::testing::Test { + hash_default_hash<T> hasher; +}; + +TYPED_TEST_SUITE(HashString, StringTypes); + +TYPED_TEST(EqString, Works) { + auto eq = this->key_eq; + EXPECT_TRUE(eq("a", "a")); + EXPECT_TRUE(eq("a", absl::string_view("a"))); + EXPECT_TRUE(eq("a", std::string("a"))); + EXPECT_FALSE(eq("a", "b")); + EXPECT_FALSE(eq("a", absl::string_view("b"))); + EXPECT_FALSE(eq("a", std::string("b"))); +} + +TYPED_TEST(HashString, Works) { + auto hash = this->hasher; + auto h = hash("a"); + EXPECT_EQ(h, hash(absl::string_view("a"))); + EXPECT_EQ(h, hash(std::string("a"))); + EXPECT_NE(h, hash(absl::string_view("b"))); + EXPECT_NE(h, hash(std::string("b"))); +} + +struct NoDeleter { + template <class T> + void operator()(const T* ptr) const {} +}; + +using PointerTypes = + ::testing::Types<const int*, int*, std::unique_ptr<const int>, + std::unique_ptr<const int, NoDeleter>, + std::unique_ptr<int>, std::unique_ptr<int, NoDeleter>, + std::shared_ptr<const int>, std::shared_ptr<int>>; + +template <class T> +struct EqPointer : ::testing::Test { + hash_default_eq<T> key_eq; +}; + +TYPED_TEST_SUITE(EqPointer, PointerTypes); + +template <class T> +struct HashPointer : ::testing::Test { + hash_default_hash<T> hasher; +}; + +TYPED_TEST_SUITE(HashPointer, PointerTypes); + +TYPED_TEST(EqPointer, Works) { + int dummy; + auto eq = this->key_eq; + auto sptr = std::make_shared<int>(); + std::shared_ptr<const int> csptr = sptr; + int* ptr = sptr.get(); + const int* cptr = ptr; + std::unique_ptr<int, NoDeleter> uptr(ptr); + std::unique_ptr<const int, NoDeleter> cuptr(ptr); + + EXPECT_TRUE(eq(ptr, cptr)); + EXPECT_TRUE(eq(ptr, sptr)); + EXPECT_TRUE(eq(ptr, uptr)); + EXPECT_TRUE(eq(ptr, csptr)); + EXPECT_TRUE(eq(ptr, cuptr)); + EXPECT_FALSE(eq(&dummy, cptr)); + EXPECT_FALSE(eq(&dummy, sptr)); + EXPECT_FALSE(eq(&dummy, uptr)); + EXPECT_FALSE(eq(&dummy, csptr)); + EXPECT_FALSE(eq(&dummy, cuptr)); +} + +TEST(Hash, DerivedAndBase) { + struct Base {}; + struct Derived : Base {}; + + hash_default_hash<Base*> hasher; + + Base base; + Derived derived; + EXPECT_NE(hasher(&base), hasher(&derived)); + EXPECT_EQ(hasher(static_cast<Base*>(&derived)), hasher(&derived)); + + auto dp = std::make_shared<Derived>(); + EXPECT_EQ(hasher(static_cast<Base*>(dp.get())), hasher(dp)); +} + +TEST(Hash, FunctionPointer) { + using Func = int (*)(); + hash_default_hash<Func> hasher; + hash_default_eq<Func> eq; + + Func p1 = [] { return 1; }, p2 = [] { return 2; }; + EXPECT_EQ(hasher(p1), hasher(p1)); + EXPECT_TRUE(eq(p1, p1)); + + EXPECT_NE(hasher(p1), hasher(p2)); + EXPECT_FALSE(eq(p1, p2)); +} + +TYPED_TEST(HashPointer, Works) { + int dummy; + auto hash = this->hasher; + auto sptr = std::make_shared<int>(); + std::shared_ptr<const int> csptr = sptr; + int* ptr = sptr.get(); + const int* cptr = ptr; + std::unique_ptr<int, NoDeleter> uptr(ptr); + std::unique_ptr<const int, NoDeleter> cuptr(ptr); + + EXPECT_EQ(hash(ptr), hash(cptr)); + EXPECT_EQ(hash(ptr), hash(sptr)); + EXPECT_EQ(hash(ptr), hash(uptr)); + EXPECT_EQ(hash(ptr), hash(csptr)); + EXPECT_EQ(hash(ptr), hash(cuptr)); + EXPECT_NE(hash(&dummy), hash(cptr)); + EXPECT_NE(hash(&dummy), hash(sptr)); + EXPECT_NE(hash(&dummy), hash(uptr)); + EXPECT_NE(hash(&dummy), hash(csptr)); + EXPECT_NE(hash(&dummy), hash(cuptr)); +} + +TEST(EqCord, Works) { + hash_default_eq<absl::Cord> eq; + const absl::string_view a_string_view = "a"; + const absl::Cord a_cord(a_string_view); + const absl::string_view b_string_view = "b"; + const absl::Cord b_cord(b_string_view); + + EXPECT_TRUE(eq(a_cord, a_cord)); + EXPECT_TRUE(eq(a_cord, a_string_view)); + EXPECT_TRUE(eq(a_string_view, a_cord)); + EXPECT_FALSE(eq(a_cord, b_cord)); + EXPECT_FALSE(eq(a_cord, b_string_view)); + EXPECT_FALSE(eq(b_string_view, a_cord)); +} + +TEST(HashCord, Works) { + hash_default_hash<absl::Cord> hash; + const absl::string_view a_string_view = "a"; + const absl::Cord a_cord(a_string_view); + const absl::string_view b_string_view = "b"; + const absl::Cord b_cord(b_string_view); + + EXPECT_EQ(hash(a_cord), hash(a_cord)); + EXPECT_EQ(hash(b_cord), hash(b_cord)); + EXPECT_EQ(hash(a_string_view), hash(a_cord)); + EXPECT_EQ(hash(b_string_view), hash(b_cord)); + EXPECT_EQ(hash(absl::Cord("")), hash("")); + EXPECT_EQ(hash(absl::Cord()), hash(absl::string_view())); + + EXPECT_NE(hash(a_cord), hash(b_cord)); + EXPECT_NE(hash(a_cord), hash(b_string_view)); + EXPECT_NE(hash(a_string_view), hash(b_cord)); + EXPECT_NE(hash(a_string_view), hash(b_string_view)); +} + +void NoOpReleaser(absl::string_view data, void* arg) {} + +TEST(HashCord, FragmentedCordWorks) { + hash_default_hash<absl::Cord> hash; + absl::Cord c = absl::MakeFragmentedCord({"a", "b", "c"}); + EXPECT_FALSE(c.TryFlat().has_value()); + EXPECT_EQ(hash(c), hash("abc")); +} + +TEST(HashCord, FragmentedLongCordWorks) { + hash_default_hash<absl::Cord> hash; + // Crete some large strings which do not fit on the stack. + std::string a(65536, 'a'); + std::string b(65536, 'b'); + absl::Cord c = absl::MakeFragmentedCord({a, b}); + EXPECT_FALSE(c.TryFlat().has_value()); + EXPECT_EQ(hash(c), hash(a + b)); +} + +TEST(HashCord, RandomCord) { + hash_default_hash<absl::Cord> hash; + auto bitgen = absl::BitGen(); + for (int i = 0; i < 1000; ++i) { + const int number_of_segments = absl::Uniform(bitgen, 0, 10); + std::vector<std::string> pieces; + for (size_t s = 0; s < number_of_segments; ++s) { + std::string str; + str.resize(absl::Uniform(bitgen, 0, 4096)); + // MSVC needed the explicit return type in the lambda. + std::generate(str.begin(), str.end(), [&]() -> char { + return static_cast<char>(absl::Uniform<unsigned char>(bitgen)); + }); + pieces.push_back(str); + } + absl::Cord c = absl::MakeFragmentedCord(pieces); + EXPECT_EQ(hash(c), hash(std::string(c))); + } +} + +// Cartesian product of (std::string, absl::string_view) +// with (std::string, absl::string_view, const char*, absl::Cord). +using StringTypesCartesianProduct = Types< + // clang-format off + std::pair<absl::Cord, std::string>, + std::pair<absl::Cord, absl::string_view>, + std::pair<absl::Cord, absl::Cord>, + std::pair<absl::Cord, const char*>, + + std::pair<std::string, absl::Cord>, + std::pair<absl::string_view, absl::Cord>, + + std::pair<absl::string_view, std::string>, + std::pair<absl::string_view, absl::string_view>, + std::pair<absl::string_view, const char*>>; +// clang-format on + +constexpr char kFirstString[] = "abc123"; +constexpr char kSecondString[] = "ijk456"; + +template <typename T> +struct StringLikeTest : public ::testing::Test { + typename T::first_type a1{kFirstString}; + typename T::second_type b1{kFirstString}; + typename T::first_type a2{kSecondString}; + typename T::second_type b2{kSecondString}; + hash_default_eq<typename T::first_type> eq; + hash_default_hash<typename T::first_type> hash; +}; + +TYPED_TEST_CASE_P(StringLikeTest); + +TYPED_TEST_P(StringLikeTest, Eq) { + EXPECT_TRUE(this->eq(this->a1, this->b1)); + EXPECT_TRUE(this->eq(this->b1, this->a1)); +} + +TYPED_TEST_P(StringLikeTest, NotEq) { + EXPECT_FALSE(this->eq(this->a1, this->b2)); + EXPECT_FALSE(this->eq(this->b2, this->a1)); +} + +TYPED_TEST_P(StringLikeTest, HashEq) { + EXPECT_EQ(this->hash(this->a1), this->hash(this->b1)); + EXPECT_EQ(this->hash(this->a2), this->hash(this->b2)); + // It would be a poor hash function which collides on these strings. + EXPECT_NE(this->hash(this->a1), this->hash(this->b2)); +} + +TYPED_TEST_SUITE(StringLikeTest, StringTypesCartesianProduct); + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +enum Hash : size_t { + kStd = 0x1, // std::hash +#ifdef _MSC_VER + kExtension = kStd, // In MSVC, std::hash == ::hash +#else // _MSC_VER + kExtension = 0x2, // ::hash (GCC extension) +#endif // _MSC_VER +}; + +// H is a bitmask of Hash enumerations. +// Hashable<H> is hashable via all means specified in H. +template <int H> +struct Hashable { + static constexpr bool HashableBy(Hash h) { return h & H; } +}; + +namespace std { +template <int H> +struct hash<Hashable<H>> { + template <class E = Hashable<H>, + class = typename std::enable_if<E::HashableBy(kStd)>::type> + size_t operator()(E) const { + return kStd; + } +}; +} // namespace std + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +template <class T> +size_t Hash(const T& v) { + return hash_default_hash<T>()(v); +} + +TEST(Delegate, HashDispatch) { + EXPECT_EQ(Hash(kStd), Hash(Hashable<kStd>())); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/hash_generator_testing.cc b/third_party/abseil_cpp/absl/container/internal/hash_generator_testing.cc new file mode 100644 index 000000000000..59cc5aac7ab8 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_generator_testing.cc @@ -0,0 +1,76 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hash_generator_testing.h" + +#include <deque> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace hash_internal { +namespace { + +class RandomDeviceSeedSeq { + public: + using result_type = typename std::random_device::result_type; + + template <class Iterator> + void generate(Iterator start, Iterator end) { + while (start != end) { + *start = gen_(); + ++start; + } + } + + private: + std::random_device gen_; +}; + +} // namespace + +std::mt19937_64* GetSharedRng() { + static auto* rng = [] { + RandomDeviceSeedSeq seed_seq; + return new std::mt19937_64(seed_seq); + }(); + return rng; +} + +std::string Generator<std::string>::operator()() const { + // NOLINTNEXTLINE(runtime/int) + std::uniform_int_distribution<short> chars(0x20, 0x7E); + std::string res; + res.resize(32); + std::generate(res.begin(), res.end(), + [&]() { return chars(*GetSharedRng()); }); + return res; +} + +absl::string_view Generator<absl::string_view>::operator()() const { + static auto* arena = new std::deque<std::string>(); + // NOLINTNEXTLINE(runtime/int) + std::uniform_int_distribution<short> chars(0x20, 0x7E); + arena->emplace_back(); + auto& res = arena->back(); + res.resize(32); + std::generate(res.begin(), res.end(), + [&]() { return chars(*GetSharedRng()); }); + return res; +} + +} // namespace hash_internal +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/hash_generator_testing.h b/third_party/abseil_cpp/absl/container/internal/hash_generator_testing.h new file mode 100644 index 000000000000..6869fe45e8c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_generator_testing.h @@ -0,0 +1,161 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Generates random values for testing. Specialized only for the few types we +// care about. + +#ifndef ABSL_CONTAINER_INTERNAL_HASH_GENERATOR_TESTING_H_ +#define ABSL_CONTAINER_INTERNAL_HASH_GENERATOR_TESTING_H_ + +#include <stdint.h> + +#include <algorithm> +#include <iosfwd> +#include <random> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/container/internal/hash_policy_testing.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace hash_internal { +namespace generator_internal { + +template <class Container, class = void> +struct IsMap : std::false_type {}; + +template <class Map> +struct IsMap<Map, absl::void_t<typename Map::mapped_type>> : std::true_type {}; + +} // namespace generator_internal + +std::mt19937_64* GetSharedRng(); + +enum Enum { + kEnumEmpty, + kEnumDeleted, +}; + +enum class EnumClass : uint64_t { + kEmpty, + kDeleted, +}; + +inline std::ostream& operator<<(std::ostream& o, const EnumClass& ec) { + return o << static_cast<uint64_t>(ec); +} + +template <class T, class E = void> +struct Generator; + +template <class T> +struct Generator<T, typename std::enable_if<std::is_integral<T>::value>::type> { + T operator()() const { + std::uniform_int_distribution<T> dist; + return dist(*GetSharedRng()); + } +}; + +template <> +struct Generator<Enum> { + Enum operator()() const { + std::uniform_int_distribution<typename std::underlying_type<Enum>::type> + dist; + while (true) { + auto variate = dist(*GetSharedRng()); + if (variate != kEnumEmpty && variate != kEnumDeleted) + return static_cast<Enum>(variate); + } + } +}; + +template <> +struct Generator<EnumClass> { + EnumClass operator()() const { + std::uniform_int_distribution< + typename std::underlying_type<EnumClass>::type> + dist; + while (true) { + EnumClass variate = static_cast<EnumClass>(dist(*GetSharedRng())); + if (variate != EnumClass::kEmpty && variate != EnumClass::kDeleted) + return static_cast<EnumClass>(variate); + } + } +}; + +template <> +struct Generator<std::string> { + std::string operator()() const; +}; + +template <> +struct Generator<absl::string_view> { + absl::string_view operator()() const; +}; + +template <> +struct Generator<NonStandardLayout> { + NonStandardLayout operator()() const { + return NonStandardLayout(Generator<std::string>()()); + } +}; + +template <class K, class V> +struct Generator<std::pair<K, V>> { + std::pair<K, V> operator()() const { + return std::pair<K, V>(Generator<typename std::decay<K>::type>()(), + Generator<typename std::decay<V>::type>()()); + } +}; + +template <class... Ts> +struct Generator<std::tuple<Ts...>> { + std::tuple<Ts...> operator()() const { + return std::tuple<Ts...>(Generator<typename std::decay<Ts>::type>()()...); + } +}; + +template <class T> +struct Generator<std::unique_ptr<T>> { + std::unique_ptr<T> operator()() const { + return absl::make_unique<T>(Generator<T>()()); + } +}; + +template <class U> +struct Generator<U, absl::void_t<decltype(std::declval<U&>().key()), + decltype(std::declval<U&>().value())>> + : Generator<std::pair< + typename std::decay<decltype(std::declval<U&>().key())>::type, + typename std::decay<decltype(std::declval<U&>().value())>::type>> {}; + +template <class Container> +using GeneratedType = decltype( + std::declval<const Generator< + typename std::conditional<generator_internal::IsMap<Container>::value, + typename Container::value_type, + typename Container::key_type>::type>&>()()); + +} // namespace hash_internal +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_HASH_GENERATOR_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hash_policy_testing.h b/third_party/abseil_cpp/absl/container/internal/hash_policy_testing.h new file mode 100644 index 000000000000..01c40d2e5cff --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_policy_testing.h @@ -0,0 +1,184 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Utilities to help tests verify that hash tables properly handle stateful +// allocators and hash functions. + +#ifndef ABSL_CONTAINER_INTERNAL_HASH_POLICY_TESTING_H_ +#define ABSL_CONTAINER_INTERNAL_HASH_POLICY_TESTING_H_ + +#include <cstdlib> +#include <limits> +#include <memory> +#include <ostream> +#include <type_traits> +#include <utility> +#include <vector> + +#include "absl/hash/hash.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace hash_testing_internal { + +template <class Derived> +struct WithId { + WithId() : id_(next_id<Derived>()) {} + WithId(const WithId& that) : id_(that.id_) {} + WithId(WithId&& that) : id_(that.id_) { that.id_ = 0; } + WithId& operator=(const WithId& that) { + id_ = that.id_; + return *this; + } + WithId& operator=(WithId&& that) { + id_ = that.id_; + that.id_ = 0; + return *this; + } + + size_t id() const { return id_; } + + friend bool operator==(const WithId& a, const WithId& b) { + return a.id_ == b.id_; + } + friend bool operator!=(const WithId& a, const WithId& b) { return !(a == b); } + + protected: + explicit WithId(size_t id) : id_(id) {} + + private: + size_t id_; + + template <class T> + static size_t next_id() { + // 0 is reserved for moved from state. + static size_t gId = 1; + return gId++; + } +}; + +} // namespace hash_testing_internal + +struct NonStandardLayout { + NonStandardLayout() {} + explicit NonStandardLayout(std::string s) : value(std::move(s)) {} + virtual ~NonStandardLayout() {} + + friend bool operator==(const NonStandardLayout& a, + const NonStandardLayout& b) { + return a.value == b.value; + } + friend bool operator!=(const NonStandardLayout& a, + const NonStandardLayout& b) { + return a.value != b.value; + } + + template <typename H> + friend H AbslHashValue(H h, const NonStandardLayout& v) { + return H::combine(std::move(h), v.value); + } + + std::string value; +}; + +struct StatefulTestingHash + : absl::container_internal::hash_testing_internal::WithId< + StatefulTestingHash> { + template <class T> + size_t operator()(const T& t) const { + return absl::Hash<T>{}(t); + } +}; + +struct StatefulTestingEqual + : absl::container_internal::hash_testing_internal::WithId< + StatefulTestingEqual> { + template <class T, class U> + bool operator()(const T& t, const U& u) const { + return t == u; + } +}; + +// It is expected that Alloc() == Alloc() for all allocators so we cannot use +// WithId base. We need to explicitly assign ids. +template <class T = int> +struct Alloc : std::allocator<T> { + using propagate_on_container_swap = std::true_type; + + // Using old paradigm for this to ensure compatibility. + explicit Alloc(size_t id = 0) : id_(id) {} + + Alloc(const Alloc&) = default; + Alloc& operator=(const Alloc&) = default; + + template <class U> + Alloc(const Alloc<U>& that) : std::allocator<T>(that), id_(that.id()) {} + + template <class U> + struct rebind { + using other = Alloc<U>; + }; + + size_t id() const { return id_; } + + friend bool operator==(const Alloc& a, const Alloc& b) { + return a.id_ == b.id_; + } + friend bool operator!=(const Alloc& a, const Alloc& b) { return !(a == b); } + + private: + size_t id_ = (std::numeric_limits<size_t>::max)(); +}; + +template <class Map> +auto items(const Map& m) -> std::vector< + std::pair<typename Map::key_type, typename Map::mapped_type>> { + using std::get; + std::vector<std::pair<typename Map::key_type, typename Map::mapped_type>> res; + res.reserve(m.size()); + for (const auto& v : m) res.emplace_back(get<0>(v), get<1>(v)); + return res; +} + +template <class Set> +auto keys(const Set& s) + -> std::vector<typename std::decay<typename Set::key_type>::type> { + std::vector<typename std::decay<typename Set::key_type>::type> res; + res.reserve(s.size()); + for (const auto& v : s) res.emplace_back(v); + return res; +} + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +// ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS is false for glibcxx versions +// where the unordered containers are missing certain constructors that +// take allocator arguments. This test is defined ad-hoc for the platforms +// we care about (notably Crosstool 17) because libstdcxx's useless +// versioning scheme precludes a more principled solution. +// From GCC-4.9 Changelog: (src: https://gcc.gnu.org/gcc-4.9/changes.html) +// "the unordered associative containers in <unordered_map> and <unordered_set> +// meet the allocator-aware container requirements;" +#if (defined(__GLIBCXX__) && __GLIBCXX__ <= 20140425 ) || \ +( __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 9 )) +#define ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS 0 +#else +#define ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS 1 +#endif + +#endif // ABSL_CONTAINER_INTERNAL_HASH_POLICY_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hash_policy_testing_test.cc b/third_party/abseil_cpp/absl/container/internal/hash_policy_testing_test.cc new file mode 100644 index 000000000000..f0b20fe345e2 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_policy_testing_test.cc @@ -0,0 +1,45 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hash_policy_testing.h" + +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +TEST(_, Hash) { + StatefulTestingHash h1; + EXPECT_EQ(1, h1.id()); + StatefulTestingHash h2; + EXPECT_EQ(2, h2.id()); + StatefulTestingHash h1c(h1); + EXPECT_EQ(1, h1c.id()); + StatefulTestingHash h2m(std::move(h2)); + EXPECT_EQ(2, h2m.id()); + EXPECT_EQ(0, h2.id()); + StatefulTestingHash h3; + EXPECT_EQ(3, h3.id()); + h3 = StatefulTestingHash(); + EXPECT_EQ(4, h3.id()); + h3 = std::move(h1); + EXPECT_EQ(1, h3.id()); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/hash_policy_traits.h b/third_party/abseil_cpp/absl/container/internal/hash_policy_traits.h new file mode 100644 index 000000000000..46c97b18a227 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_policy_traits.h @@ -0,0 +1,208 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_HASH_POLICY_TRAITS_H_ +#define ABSL_CONTAINER_INTERNAL_HASH_POLICY_TRAITS_H_ + +#include <cstddef> +#include <memory> +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// Defines how slots are initialized/destroyed/moved. +template <class Policy, class = void> +struct hash_policy_traits { + // The type of the keys stored in the hashtable. + using key_type = typename Policy::key_type; + + private: + struct ReturnKey { + // When C++17 is available, we can use std::launder to provide mutable + // access to the key for use in node handle. +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 + template <class Key, + absl::enable_if_t<std::is_lvalue_reference<Key>::value, int> = 0> + static key_type& Impl(Key&& k, int) { + return *std::launder( + const_cast<key_type*>(std::addressof(std::forward<Key>(k)))); + } +#endif + + template <class Key> + static Key Impl(Key&& k, char) { + return std::forward<Key>(k); + } + + // When Key=T&, we forward the lvalue reference. + // When Key=T, we return by value to avoid a dangling reference. + // eg, for string_hash_map. + template <class Key, class... Args> + auto operator()(Key&& k, const Args&...) const + -> decltype(Impl(std::forward<Key>(k), 0)) { + return Impl(std::forward<Key>(k), 0); + } + }; + + template <class P = Policy, class = void> + struct ConstantIteratorsImpl : std::false_type {}; + + template <class P> + struct ConstantIteratorsImpl<P, absl::void_t<typename P::constant_iterators>> + : P::constant_iterators {}; + + public: + // The actual object stored in the hash table. + using slot_type = typename Policy::slot_type; + + // The argument type for insertions into the hashtable. This is different + // from value_type for increased performance. See initializer_list constructor + // and insert() member functions for more details. + using init_type = typename Policy::init_type; + + using reference = decltype(Policy::element(std::declval<slot_type*>())); + using pointer = typename std::remove_reference<reference>::type*; + using value_type = typename std::remove_reference<reference>::type; + + // Policies can set this variable to tell raw_hash_set that all iterators + // should be constant, even `iterator`. This is useful for set-like + // containers. + // Defaults to false if not provided by the policy. + using constant_iterators = ConstantIteratorsImpl<>; + + // PRECONDITION: `slot` is UNINITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template <class Alloc, class... Args> + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + Policy::construct(alloc, slot, std::forward<Args>(args)...); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is UNINITIALIZED + template <class Alloc> + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::destroy(alloc, slot); + } + + // Transfers the `old_slot` to `new_slot`. Any memory allocated by the + // allocator inside `old_slot` to `new_slot` can be transferred. + // + // OPTIONAL: defaults to: + // + // clone(new_slot, std::move(*old_slot)); + // destroy(old_slot); + // + // PRECONDITION: `new_slot` is UNINITIALIZED and `old_slot` is INITIALIZED + // POSTCONDITION: `new_slot` is INITIALIZED and `old_slot` is + // UNINITIALIZED + template <class Alloc> + static void transfer(Alloc* alloc, slot_type* new_slot, slot_type* old_slot) { + transfer_impl(alloc, new_slot, old_slot, 0); + } + + // PRECONDITION: `slot` is INITIALIZED + // POSTCONDITION: `slot` is INITIALIZED + template <class P = Policy> + static auto element(slot_type* slot) -> decltype(P::element(slot)) { + return P::element(slot); + } + + // Returns the amount of memory owned by `slot`, exclusive of `sizeof(*slot)`. + // + // If `slot` is nullptr, returns the constant amount of memory owned by any + // full slot or -1 if slots own variable amounts of memory. + // + // PRECONDITION: `slot` is INITIALIZED or nullptr + template <class P = Policy> + static size_t space_used(const slot_type* slot) { + return P::space_used(slot); + } + + // Provides generalized access to the key for elements, both for elements in + // the table and for elements that have not yet been inserted (or even + // constructed). We would like an API that allows us to say: `key(args...)` + // but we cannot do that for all cases, so we use this more general API that + // can be used for many things, including the following: + // + // - Given an element in a table, get its key. + // - Given an element initializer, get its key. + // - Given `emplace()` arguments, get the element key. + // + // Implementations of this must adhere to a very strict technical + // specification around aliasing and consuming arguments: + // + // Let `value_type` be the result type of `element()` without ref- and + // cv-qualifiers. The first argument is a functor, the rest are constructor + // arguments for `value_type`. Returns `std::forward<F>(f)(k, xs...)`, where + // `k` is the element key, and `xs...` are the new constructor arguments for + // `value_type`. It's allowed for `k` to alias `xs...`, and for both to alias + // `ts...`. The key won't be touched once `xs...` are used to construct an + // element; `ts...` won't be touched at all, which allows `apply()` to consume + // any rvalues among them. + // + // If `value_type` is constructible from `Ts&&...`, `Policy::apply()` must not + // trigger a hard compile error unless it originates from `f`. In other words, + // `Policy::apply()` must be SFINAE-friendly. If `value_type` is not + // constructible from `Ts&&...`, either SFINAE or a hard compile error is OK. + // + // If `Ts...` is `[cv] value_type[&]` or `[cv] init_type[&]`, + // `Policy::apply()` must work. A compile error is not allowed, SFINAE or not. + template <class F, class... Ts, class P = Policy> + static auto apply(F&& f, Ts&&... ts) + -> decltype(P::apply(std::forward<F>(f), std::forward<Ts>(ts)...)) { + return P::apply(std::forward<F>(f), std::forward<Ts>(ts)...); + } + + // Returns the "key" portion of the slot. + // Used for node handle manipulation. + template <class P = Policy> + static auto mutable_key(slot_type* slot) + -> decltype(P::apply(ReturnKey(), element(slot))) { + return P::apply(ReturnKey(), element(slot)); + } + + // Returns the "value" (as opposed to the "key") portion of the element. Used + // by maps to implement `operator[]`, `at()` and `insert_or_assign()`. + template <class T, class P = Policy> + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + + private: + // Use auto -> decltype as an enabler. + template <class Alloc, class P = Policy> + static auto transfer_impl(Alloc* alloc, slot_type* new_slot, + slot_type* old_slot, int) + -> decltype((void)P::transfer(alloc, new_slot, old_slot)) { + P::transfer(alloc, new_slot, old_slot); + } + template <class Alloc> + static void transfer_impl(Alloc* alloc, slot_type* new_slot, + slot_type* old_slot, char) { + construct(alloc, new_slot, std::move(element(old_slot))); + destroy(alloc, old_slot); + } +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_HASH_POLICY_TRAITS_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hash_policy_traits_test.cc b/third_party/abseil_cpp/absl/container/internal/hash_policy_traits_test.cc new file mode 100644 index 000000000000..6ef8b9e05fb1 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hash_policy_traits_test.cc @@ -0,0 +1,144 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hash_policy_traits.h" + +#include <functional> +#include <memory> +#include <new> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::testing::MockFunction; +using ::testing::Return; +using ::testing::ReturnRef; + +using Alloc = std::allocator<int>; +using Slot = int; + +struct PolicyWithoutOptionalOps { + using slot_type = Slot; + using key_type = Slot; + using init_type = Slot; + + static std::function<void(void*, Slot*, Slot)> construct; + static std::function<void(void*, Slot*)> destroy; + + static std::function<Slot&(Slot*)> element; + static int apply(int v) { return apply_impl(v); } + static std::function<int(int)> apply_impl; + static std::function<Slot&(Slot*)> value; +}; + +std::function<void(void*, Slot*, Slot)> PolicyWithoutOptionalOps::construct; +std::function<void(void*, Slot*)> PolicyWithoutOptionalOps::destroy; + +std::function<Slot&(Slot*)> PolicyWithoutOptionalOps::element; +std::function<int(int)> PolicyWithoutOptionalOps::apply_impl; +std::function<Slot&(Slot*)> PolicyWithoutOptionalOps::value; + +struct PolicyWithOptionalOps : PolicyWithoutOptionalOps { + static std::function<void(void*, Slot*, Slot*)> transfer; +}; + +std::function<void(void*, Slot*, Slot*)> PolicyWithOptionalOps::transfer; + +struct Test : ::testing::Test { + Test() { + PolicyWithoutOptionalOps::construct = [&](void* a1, Slot* a2, Slot a3) { + construct.Call(a1, a2, std::move(a3)); + }; + PolicyWithoutOptionalOps::destroy = [&](void* a1, Slot* a2) { + destroy.Call(a1, a2); + }; + + PolicyWithoutOptionalOps::element = [&](Slot* a1) -> Slot& { + return element.Call(a1); + }; + PolicyWithoutOptionalOps::apply_impl = [&](int a1) -> int { + return apply.Call(a1); + }; + PolicyWithoutOptionalOps::value = [&](Slot* a1) -> Slot& { + return value.Call(a1); + }; + + PolicyWithOptionalOps::transfer = [&](void* a1, Slot* a2, Slot* a3) { + return transfer.Call(a1, a2, a3); + }; + } + + std::allocator<int> alloc; + int a = 53; + + MockFunction<void(void*, Slot*, Slot)> construct; + MockFunction<void(void*, Slot*)> destroy; + + MockFunction<Slot&(Slot*)> element; + MockFunction<int(int)> apply; + MockFunction<Slot&(Slot*)> value; + + MockFunction<void(void*, Slot*, Slot*)> transfer; +}; + +TEST_F(Test, construct) { + EXPECT_CALL(construct, Call(&alloc, &a, 53)); + hash_policy_traits<PolicyWithoutOptionalOps>::construct(&alloc, &a, 53); +} + +TEST_F(Test, destroy) { + EXPECT_CALL(destroy, Call(&alloc, &a)); + hash_policy_traits<PolicyWithoutOptionalOps>::destroy(&alloc, &a); +} + +TEST_F(Test, element) { + int b = 0; + EXPECT_CALL(element, Call(&a)).WillOnce(ReturnRef(b)); + EXPECT_EQ(&b, &hash_policy_traits<PolicyWithoutOptionalOps>::element(&a)); +} + +TEST_F(Test, apply) { + EXPECT_CALL(apply, Call(42)).WillOnce(Return(1337)); + EXPECT_EQ(1337, (hash_policy_traits<PolicyWithoutOptionalOps>::apply(42))); +} + +TEST_F(Test, value) { + int b = 0; + EXPECT_CALL(value, Call(&a)).WillOnce(ReturnRef(b)); + EXPECT_EQ(&b, &hash_policy_traits<PolicyWithoutOptionalOps>::value(&a)); +} + +TEST_F(Test, without_transfer) { + int b = 42; + EXPECT_CALL(element, Call(&b)).WillOnce(::testing::ReturnRef(b)); + EXPECT_CALL(construct, Call(&alloc, &a, b)); + EXPECT_CALL(destroy, Call(&alloc, &b)); + hash_policy_traits<PolicyWithoutOptionalOps>::transfer(&alloc, &a, &b); +} + +TEST_F(Test, with_transfer) { + int b = 42; + EXPECT_CALL(transfer, Call(&alloc, &a, &b)); + hash_policy_traits<PolicyWithOptionalOps>::transfer(&alloc, &a, &b); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/hashtable_debug.h b/third_party/abseil_cpp/absl/container/internal/hashtable_debug.h new file mode 100644 index 000000000000..19d52121d688 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hashtable_debug.h @@ -0,0 +1,110 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This library provides APIs to debug the probing behavior of hash tables. +// +// In general, the probing behavior is a black box for users and only the +// side effects can be measured in the form of performance differences. +// These APIs give a glimpse on the actual behavior of the probing algorithms in +// these hashtables given a specified hash function and a set of elements. +// +// The probe count distribution can be used to assess the quality of the hash +// function for that particular hash table. Note that a hash function that +// performs well in one hash table implementation does not necessarily performs +// well in a different one. +// +// This library supports std::unordered_{set,map}, dense_hash_{set,map} and +// absl::{flat,node,string}_hash_{set,map}. + +#ifndef ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_H_ +#define ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_H_ + +#include <cstddef> +#include <algorithm> +#include <type_traits> +#include <vector> + +#include "absl/container/internal/hashtable_debug_hooks.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// Returns the number of probes required to lookup `key`. Returns 0 for a +// search with no collisions. Higher values mean more hash collisions occurred; +// however, the exact meaning of this number varies according to the container +// type. +template <typename C> +size_t GetHashtableDebugNumProbes( + const C& c, const typename C::key_type& key) { + return absl::container_internal::hashtable_debug_internal:: + HashtableDebugAccess<C>::GetNumProbes(c, key); +} + +// Gets a histogram of the number of probes for each elements in the container. +// The sum of all the values in the vector is equal to container.size(). +template <typename C> +std::vector<size_t> GetHashtableDebugNumProbesHistogram(const C& container) { + std::vector<size_t> v; + for (auto it = container.begin(); it != container.end(); ++it) { + size_t num_probes = GetHashtableDebugNumProbes( + container, + absl::container_internal::hashtable_debug_internal::GetKey<C>(*it, 0)); + v.resize((std::max)(v.size(), num_probes + 1)); + v[num_probes]++; + } + return v; +} + +struct HashtableDebugProbeSummary { + size_t total_elements; + size_t total_num_probes; + double mean; +}; + +// Gets a summary of the probe count distribution for the elements in the +// container. +template <typename C> +HashtableDebugProbeSummary GetHashtableDebugProbeSummary(const C& container) { + auto probes = GetHashtableDebugNumProbesHistogram(container); + HashtableDebugProbeSummary summary = {}; + for (size_t i = 0; i < probes.size(); ++i) { + summary.total_elements += probes[i]; + summary.total_num_probes += probes[i] * i; + } + summary.mean = 1.0 * summary.total_num_probes / summary.total_elements; + return summary; +} + +// Returns the number of bytes requested from the allocator by the container +// and not freed. +template <typename C> +size_t AllocatedByteSize(const C& c) { + return absl::container_internal::hashtable_debug_internal:: + HashtableDebugAccess<C>::AllocatedByteSize(c); +} + +// Returns a tight lower bound for AllocatedByteSize(c) where `c` is of type `C` +// and `c.size()` is equal to `num_elements`. +template <typename C> +size_t LowerBoundAllocatedByteSize(size_t num_elements) { + return absl::container_internal::hashtable_debug_internal:: + HashtableDebugAccess<C>::LowerBoundAllocatedByteSize(num_elements); +} + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hashtable_debug_hooks.h b/third_party/abseil_cpp/absl/container/internal/hashtable_debug_hooks.h new file mode 100644 index 000000000000..3e9ea5954e0a --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hashtable_debug_hooks.h @@ -0,0 +1,85 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Provides the internal API for hashtable_debug.h. + +#ifndef ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_HOOKS_H_ +#define ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_HOOKS_H_ + +#include <cstddef> + +#include <algorithm> +#include <type_traits> +#include <vector> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace hashtable_debug_internal { + +// If it is a map, call get<0>(). +using std::get; +template <typename T, typename = typename T::mapped_type> +auto GetKey(const typename T::value_type& pair, int) -> decltype(get<0>(pair)) { + return get<0>(pair); +} + +// If it is not a map, return the value directly. +template <typename T> +const typename T::key_type& GetKey(const typename T::key_type& key, char) { + return key; +} + +// Containers should specialize this to provide debug information for that +// container. +template <class Container, typename Enabler = void> +struct HashtableDebugAccess { + // Returns the number of probes required to find `key` in `c`. The "number of + // probes" is a concept that can vary by container. Implementations should + // return 0 when `key` was found in the minimum number of operations and + // should increment the result for each non-trivial operation required to find + // `key`. + // + // The default implementation uses the bucket api from the standard and thus + // works for `std::unordered_*` containers. + static size_t GetNumProbes(const Container& c, + const typename Container::key_type& key) { + if (!c.bucket_count()) return {}; + size_t num_probes = 0; + size_t bucket = c.bucket(key); + for (auto it = c.begin(bucket), e = c.end(bucket);; ++it, ++num_probes) { + if (it == e) return num_probes; + if (c.key_eq()(key, GetKey<Container>(*it, 0))) return num_probes; + } + } + + // Returns the number of bytes requested from the allocator by the container + // and not freed. + // + // static size_t AllocatedByteSize(const Container& c); + + // Returns a tight lower bound for AllocatedByteSize(c) where `c` is of type + // `Container` and `c.size()` is equal to `num_elements`. + // + // static size_t LowerBoundAllocatedByteSize(size_t num_elements); +}; + +} // namespace hashtable_debug_internal +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_HASHTABLE_DEBUG_HOOKS_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler.cc b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler.cc new file mode 100644 index 000000000000..e4484fbb1b43 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler.cc @@ -0,0 +1,270 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hashtablez_sampler.h" + +#include <atomic> +#include <cassert> +#include <cmath> +#include <functional> +#include <limits> + +#include "absl/base/attributes.h" +#include "absl/base/internal/exponential_biased.h" +#include "absl/container/internal/have_sse.h" +#include "absl/debugging/stacktrace.h" +#include "absl/memory/memory.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +constexpr int HashtablezInfo::kMaxStackDepth; + +namespace { +ABSL_CONST_INIT std::atomic<bool> g_hashtablez_enabled{ + false +}; +ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_sample_parameter{1 << 10}; +ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_max_samples{1 << 20}; + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +ABSL_PER_THREAD_TLS_KEYWORD absl::base_internal::ExponentialBiased + g_exponential_biased_generator; +#endif + +} // namespace + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +ABSL_PER_THREAD_TLS_KEYWORD int64_t global_next_sample = 0; +#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + +HashtablezSampler& HashtablezSampler::Global() { + static auto* sampler = new HashtablezSampler(); + return *sampler; +} + +HashtablezSampler::DisposeCallback HashtablezSampler::SetDisposeCallback( + DisposeCallback f) { + return dispose_.exchange(f, std::memory_order_relaxed); +} + +HashtablezInfo::HashtablezInfo() { PrepareForSampling(); } +HashtablezInfo::~HashtablezInfo() = default; + +void HashtablezInfo::PrepareForSampling() { + capacity.store(0, std::memory_order_relaxed); + size.store(0, std::memory_order_relaxed); + num_erases.store(0, std::memory_order_relaxed); + num_rehashes.store(0, std::memory_order_relaxed); + max_probe_length.store(0, std::memory_order_relaxed); + total_probe_length.store(0, std::memory_order_relaxed); + hashes_bitwise_or.store(0, std::memory_order_relaxed); + hashes_bitwise_and.store(~size_t{}, std::memory_order_relaxed); + + create_time = absl::Now(); + // The inliner makes hardcoded skip_count difficult (especially when combined + // with LTO). We use the ability to exclude stacks by regex when encoding + // instead. + depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, + /* skip_count= */ 0); + dead = nullptr; +} + +HashtablezSampler::HashtablezSampler() + : dropped_samples_(0), size_estimate_(0), all_(nullptr), dispose_(nullptr) { + absl::MutexLock l(&graveyard_.init_mu); + graveyard_.dead = &graveyard_; +} + +HashtablezSampler::~HashtablezSampler() { + HashtablezInfo* s = all_.load(std::memory_order_acquire); + while (s != nullptr) { + HashtablezInfo* next = s->next; + delete s; + s = next; + } +} + +void HashtablezSampler::PushNew(HashtablezInfo* sample) { + sample->next = all_.load(std::memory_order_relaxed); + while (!all_.compare_exchange_weak(sample->next, sample, + std::memory_order_release, + std::memory_order_relaxed)) { + } +} + +void HashtablezSampler::PushDead(HashtablezInfo* sample) { + if (auto* dispose = dispose_.load(std::memory_order_relaxed)) { + dispose(*sample); + } + + absl::MutexLock graveyard_lock(&graveyard_.init_mu); + absl::MutexLock sample_lock(&sample->init_mu); + sample->dead = graveyard_.dead; + graveyard_.dead = sample; +} + +HashtablezInfo* HashtablezSampler::PopDead() { + absl::MutexLock graveyard_lock(&graveyard_.init_mu); + + // The list is circular, so eventually it collapses down to + // graveyard_.dead == &graveyard_ + // when it is empty. + HashtablezInfo* sample = graveyard_.dead; + if (sample == &graveyard_) return nullptr; + + absl::MutexLock sample_lock(&sample->init_mu); + graveyard_.dead = sample->dead; + sample->PrepareForSampling(); + return sample; +} + +HashtablezInfo* HashtablezSampler::Register() { + int64_t size = size_estimate_.fetch_add(1, std::memory_order_relaxed); + if (size > g_hashtablez_max_samples.load(std::memory_order_relaxed)) { + size_estimate_.fetch_sub(1, std::memory_order_relaxed); + dropped_samples_.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + + HashtablezInfo* sample = PopDead(); + if (sample == nullptr) { + // Resurrection failed. Hire a new warlock. + sample = new HashtablezInfo(); + PushNew(sample); + } + + return sample; +} + +void HashtablezSampler::Unregister(HashtablezInfo* sample) { + PushDead(sample); + size_estimate_.fetch_sub(1, std::memory_order_relaxed); +} + +int64_t HashtablezSampler::Iterate( + const std::function<void(const HashtablezInfo& stack)>& f) { + HashtablezInfo* s = all_.load(std::memory_order_acquire); + while (s != nullptr) { + absl::MutexLock l(&s->init_mu); + if (s->dead == nullptr) { + f(*s); + } + s = s->next; + } + + return dropped_samples_.load(std::memory_order_relaxed); +} + +static bool ShouldForceSampling() { + enum ForceState { + kDontForce, + kForce, + kUninitialized + }; + ABSL_CONST_INIT static std::atomic<ForceState> global_state{ + kUninitialized}; + ForceState state = global_state.load(std::memory_order_relaxed); + if (ABSL_PREDICT_TRUE(state == kDontForce)) return false; + + if (state == kUninitialized) { + state = AbslContainerInternalSampleEverything() ? kForce : kDontForce; + global_state.store(state, std::memory_order_relaxed); + } + return state == kForce; +} + +HashtablezInfo* SampleSlow(int64_t* next_sample) { + if (ABSL_PREDICT_FALSE(ShouldForceSampling())) { + *next_sample = 1; + return HashtablezSampler::Global().Register(); + } + +#if !defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + *next_sample = std::numeric_limits<int64_t>::max(); + return nullptr; +#else + bool first = *next_sample < 0; + *next_sample = g_exponential_biased_generator.GetStride( + g_hashtablez_sample_parameter.load(std::memory_order_relaxed)); + // Small values of interval are equivalent to just sampling next time. + ABSL_ASSERT(*next_sample >= 1); + + // g_hashtablez_enabled can be dynamically flipped, we need to set a threshold + // low enough that we will start sampling in a reasonable time, so we just use + // the default sampling rate. + if (!g_hashtablez_enabled.load(std::memory_order_relaxed)) return nullptr; + + // We will only be negative on our first count, so we should just retry in + // that case. + if (first) { + if (ABSL_PREDICT_TRUE(--*next_sample > 0)) return nullptr; + return SampleSlow(next_sample); + } + + return HashtablezSampler::Global().Register(); +#endif +} + +void UnsampleSlow(HashtablezInfo* info) { + HashtablezSampler::Global().Unregister(info); +} + +void RecordInsertSlow(HashtablezInfo* info, size_t hash, + size_t distance_from_desired) { + // SwissTables probe in groups of 16, so scale this to count items probes and + // not offset from desired. + size_t probe_length = distance_from_desired; +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 + probe_length /= 16; +#else + probe_length /= 8; +#endif + + info->hashes_bitwise_and.fetch_and(hash, std::memory_order_relaxed); + info->hashes_bitwise_or.fetch_or(hash, std::memory_order_relaxed); + info->max_probe_length.store( + std::max(info->max_probe_length.load(std::memory_order_relaxed), + probe_length), + std::memory_order_relaxed); + info->total_probe_length.fetch_add(probe_length, std::memory_order_relaxed); + info->size.fetch_add(1, std::memory_order_relaxed); +} + +void SetHashtablezEnabled(bool enabled) { + g_hashtablez_enabled.store(enabled, std::memory_order_release); +} + +void SetHashtablezSampleParameter(int32_t rate) { + if (rate > 0) { + g_hashtablez_sample_parameter.store(rate, std::memory_order_release); + } else { + ABSL_RAW_LOG(ERROR, "Invalid hashtablez sample rate: %lld", + static_cast<long long>(rate)); // NOLINT(runtime/int) + } +} + +void SetHashtablezMaxSamples(int32_t max) { + if (max > 0) { + g_hashtablez_max_samples.store(max, std::memory_order_release); + } else { + ABSL_RAW_LOG(ERROR, "Invalid hashtablez max samples: %lld", + static_cast<long long>(max)); // NOLINT(runtime/int) + } +} + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler.h b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler.h new file mode 100644 index 000000000000..394348da58f5 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler.h @@ -0,0 +1,321 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: hashtablez_sampler.h +// ----------------------------------------------------------------------------- +// +// This header file defines the API for a low level library to sample hashtables +// and collect runtime statistics about them. +// +// `HashtablezSampler` controls the lifecycle of `HashtablezInfo` objects which +// store information about a single sample. +// +// `Record*` methods store information into samples. +// `Sample()` and `Unsample()` make use of a single global sampler with +// properties controlled by the flags hashtablez_enabled, +// hashtablez_sample_rate, and hashtablez_max_samples. +// +// WARNING +// +// Using this sampling API may cause sampled Swiss tables to use the global +// allocator (operator `new`) in addition to any custom allocator. If you +// are using a table in an unusual circumstance where allocation or calling a +// linux syscall is unacceptable, this could interfere. +// +// This utility is internal-only. Use at your own risk. + +#ifndef ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_ +#define ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_ + +#include <atomic> +#include <functional> +#include <memory> +#include <vector> + +#include "absl/base/internal/per_thread_tls.h" +#include "absl/base/optimization.h" +#include "absl/container/internal/have_sse.h" +#include "absl/synchronization/mutex.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// Stores information about a sampled hashtable. All mutations to this *must* +// be made through `Record*` functions below. All reads from this *must* only +// occur in the callback to `HashtablezSampler::Iterate`. +struct HashtablezInfo { + // Constructs the object but does not fill in any fields. + HashtablezInfo(); + ~HashtablezInfo(); + HashtablezInfo(const HashtablezInfo&) = delete; + HashtablezInfo& operator=(const HashtablezInfo&) = delete; + + // Puts the object into a clean state, fills in the logically `const` members, + // blocking for any readers that are currently sampling the object. + void PrepareForSampling() ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); + + // These fields are mutated by the various Record* APIs and need to be + // thread-safe. + std::atomic<size_t> capacity; + std::atomic<size_t> size; + std::atomic<size_t> num_erases; + std::atomic<size_t> num_rehashes; + std::atomic<size_t> max_probe_length; + std::atomic<size_t> total_probe_length; + std::atomic<size_t> hashes_bitwise_or; + std::atomic<size_t> hashes_bitwise_and; + + // `HashtablezSampler` maintains intrusive linked lists for all samples. See + // comments on `HashtablezSampler::all_` for details on these. `init_mu` + // guards the ability to restore the sample to a pristine state. This + // prevents races with sampling and resurrecting an object. + absl::Mutex init_mu; + HashtablezInfo* next; + HashtablezInfo* dead ABSL_GUARDED_BY(init_mu); + + // All of the fields below are set by `PrepareForSampling`, they must not be + // mutated in `Record*` functions. They are logically `const` in that sense. + // These are guarded by init_mu, but that is not externalized to clients, who + // can only read them during `HashtablezSampler::Iterate` which will hold the + // lock. + static constexpr int kMaxStackDepth = 64; + absl::Time create_time; + int32_t depth; + void* stack[kMaxStackDepth]; +}; + +inline void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length) { +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 + total_probe_length /= 16; +#else + total_probe_length /= 8; +#endif + info->total_probe_length.store(total_probe_length, std::memory_order_relaxed); + info->num_erases.store(0, std::memory_order_relaxed); + // There is only one concurrent writer, so `load` then `store` is sufficient + // instead of using `fetch_add`. + info->num_rehashes.store( + 1 + info->num_rehashes.load(std::memory_order_relaxed), + std::memory_order_relaxed); +} + +inline void RecordStorageChangedSlow(HashtablezInfo* info, size_t size, + size_t capacity) { + info->size.store(size, std::memory_order_relaxed); + info->capacity.store(capacity, std::memory_order_relaxed); + if (size == 0) { + // This is a clear, reset the total/num_erases too. + info->total_probe_length.store(0, std::memory_order_relaxed); + info->num_erases.store(0, std::memory_order_relaxed); + } +} + +void RecordInsertSlow(HashtablezInfo* info, size_t hash, + size_t distance_from_desired); + +inline void RecordEraseSlow(HashtablezInfo* info) { + info->size.fetch_sub(1, std::memory_order_relaxed); + // There is only one concurrent writer, so `load` then `store` is sufficient + // instead of using `fetch_add`. + info->num_erases.store( + 1 + info->num_erases.load(std::memory_order_relaxed), + std::memory_order_relaxed); +} + +HashtablezInfo* SampleSlow(int64_t* next_sample); +void UnsampleSlow(HashtablezInfo* info); + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +#error ABSL_INTERNAL_HASHTABLEZ_SAMPLE cannot be directly set +#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +class HashtablezInfoHandle { + public: + explicit HashtablezInfoHandle() : info_(nullptr) {} + explicit HashtablezInfoHandle(HashtablezInfo* info) : info_(info) {} + ~HashtablezInfoHandle() { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + UnsampleSlow(info_); + } + + HashtablezInfoHandle(const HashtablezInfoHandle&) = delete; + HashtablezInfoHandle& operator=(const HashtablezInfoHandle&) = delete; + + HashtablezInfoHandle(HashtablezInfoHandle&& o) noexcept + : info_(absl::exchange(o.info_, nullptr)) {} + HashtablezInfoHandle& operator=(HashtablezInfoHandle&& o) noexcept { + if (ABSL_PREDICT_FALSE(info_ != nullptr)) { + UnsampleSlow(info_); + } + info_ = absl::exchange(o.info_, nullptr); + return *this; + } + + inline void RecordStorageChanged(size_t size, size_t capacity) { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + RecordStorageChangedSlow(info_, size, capacity); + } + + inline void RecordRehash(size_t total_probe_length) { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + RecordRehashSlow(info_, total_probe_length); + } + + inline void RecordInsert(size_t hash, size_t distance_from_desired) { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + RecordInsertSlow(info_, hash, distance_from_desired); + } + + inline void RecordErase() { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + RecordEraseSlow(info_); + } + + friend inline void swap(HashtablezInfoHandle& lhs, + HashtablezInfoHandle& rhs) { + std::swap(lhs.info_, rhs.info_); + } + + private: + friend class HashtablezInfoHandlePeer; + HashtablezInfo* info_; +}; +#else +// Ensure that when Hashtablez is turned off at compile time, HashtablezInfo can +// be removed by the linker, in order to reduce the binary size. +class HashtablezInfoHandle { + public: + explicit HashtablezInfoHandle() = default; + explicit HashtablezInfoHandle(std::nullptr_t) {} + + inline void RecordStorageChanged(size_t /*size*/, size_t /*capacity*/) {} + inline void RecordRehash(size_t /*total_probe_length*/) {} + inline void RecordInsert(size_t /*hash*/, size_t /*distance_from_desired*/) {} + inline void RecordErase() {} + + friend inline void swap(HashtablezInfoHandle& /*lhs*/, + HashtablezInfoHandle& /*rhs*/) {} +}; +#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +extern ABSL_PER_THREAD_TLS_KEYWORD int64_t global_next_sample; +#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + +// Returns an RAII sampling handle that manages registration and unregistation +// with the global sampler. +inline HashtablezInfoHandle Sample() { +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + if (ABSL_PREDICT_TRUE(--global_next_sample > 0)) { + return HashtablezInfoHandle(nullptr); + } + return HashtablezInfoHandle(SampleSlow(&global_next_sample)); +#else + return HashtablezInfoHandle(nullptr); +#endif // !ABSL_PER_THREAD_TLS +} + +// Holds samples and their associated stack traces with a soft limit of +// `SetHashtablezMaxSamples()`. +// +// Thread safe. +class HashtablezSampler { + public: + // Returns a global Sampler. + static HashtablezSampler& Global(); + + HashtablezSampler(); + ~HashtablezSampler(); + + // Registers for sampling. Returns an opaque registration info. + HashtablezInfo* Register(); + + // Unregisters the sample. + void Unregister(HashtablezInfo* sample); + + // The dispose callback will be called on all samples the moment they are + // being unregistered. Only affects samples that are unregistered after the + // callback has been set. + // Returns the previous callback. + using DisposeCallback = void (*)(const HashtablezInfo&); + DisposeCallback SetDisposeCallback(DisposeCallback f); + + // Iterates over all the registered `StackInfo`s. Returning the number of + // samples that have been dropped. + int64_t Iterate(const std::function<void(const HashtablezInfo& stack)>& f); + + private: + void PushNew(HashtablezInfo* sample); + void PushDead(HashtablezInfo* sample); + HashtablezInfo* PopDead(); + + std::atomic<size_t> dropped_samples_; + std::atomic<size_t> size_estimate_; + + // Intrusive lock free linked lists for tracking samples. + // + // `all_` records all samples (they are never removed from this list) and is + // terminated with a `nullptr`. + // + // `graveyard_.dead` is a circular linked list. When it is empty, + // `graveyard_.dead == &graveyard`. The list is circular so that + // every item on it (even the last) has a non-null dead pointer. This allows + // `Iterate` to determine if a given sample is live or dead using only + // information on the sample itself. + // + // For example, nodes [A, B, C, D, E] with [A, C, E] alive and [B, D] dead + // looks like this (G is the Graveyard): + // + // +---+ +---+ +---+ +---+ +---+ + // all -->| A |--->| B |--->| C |--->| D |--->| E | + // | | | | | | | | | | + // +---+ | | +->| |-+ | | +->| |-+ | | + // | G | +---+ | +---+ | +---+ | +---+ | +---+ + // | | | | | | + // | | --------+ +--------+ | + // +---+ | + // ^ | + // +--------------------------------------+ + // + std::atomic<HashtablezInfo*> all_; + HashtablezInfo graveyard_; + + std::atomic<DisposeCallback> dispose_; +}; + +// Enables or disables sampling for Swiss tables. +void SetHashtablezEnabled(bool enabled); + +// Sets the rate at which Swiss tables will be sampled. +void SetHashtablezSampleParameter(int32_t rate); + +// Sets a soft max for the number of samples that will be kept. +void SetHashtablezMaxSamples(int32_t max); + +// Configuration override. +// This allows process-wide sampling without depending on order of +// initialization of static storage duration objects. +// The definition of this constant is weak, which allows us to inject a +// different value for it at link time. +extern "C" bool AbslContainerInternalSampleEverything(); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_HASHTABLEZ_SAMPLER_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc new file mode 100644 index 000000000000..78b9d362acf3 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler_force_weak_definition.cc @@ -0,0 +1,30 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hashtablez_sampler.h" + +#include "absl/base/attributes.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// See hashtablez_sampler.h for details. +extern "C" ABSL_ATTRIBUTE_WEAK bool AbslContainerInternalSampleEverything() { + return false; +} + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler_test.cc b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler_test.cc new file mode 100644 index 000000000000..8d10a1e94030 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/hashtablez_sampler_test.cc @@ -0,0 +1,371 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/hashtablez_sampler.h" + +#include <atomic> +#include <limits> +#include <random> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/container/internal/have_sse.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +constexpr int kProbeLength = 16; +#else +constexpr int kProbeLength = 8; +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +class HashtablezInfoHandlePeer { + public: + static bool IsSampled(const HashtablezInfoHandle& h) { + return h.info_ != nullptr; + } + + static HashtablezInfo* GetInfo(HashtablezInfoHandle* h) { return h->info_; } +}; +#else +class HashtablezInfoHandlePeer { + public: + static bool IsSampled(const HashtablezInfoHandle&) { return false; } + static HashtablezInfo* GetInfo(HashtablezInfoHandle*) { return nullptr; } +}; +#endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + +namespace { +using ::absl::synchronization_internal::ThreadPool; +using ::testing::IsEmpty; +using ::testing::UnorderedElementsAre; + +std::vector<size_t> GetSizes(HashtablezSampler* s) { + std::vector<size_t> res; + s->Iterate([&](const HashtablezInfo& info) { + res.push_back(info.size.load(std::memory_order_acquire)); + }); + return res; +} + +HashtablezInfo* Register(HashtablezSampler* s, size_t size) { + auto* info = s->Register(); + assert(info != nullptr); + info->size.store(size); + return info; +} + +TEST(HashtablezInfoTest, PrepareForSampling) { + absl::Time test_start = absl::Now(); + HashtablezInfo info; + absl::MutexLock l(&info.init_mu); + info.PrepareForSampling(); + + EXPECT_EQ(info.capacity.load(), 0); + EXPECT_EQ(info.size.load(), 0); + EXPECT_EQ(info.num_erases.load(), 0); + EXPECT_EQ(info.num_rehashes.load(), 0); + EXPECT_EQ(info.max_probe_length.load(), 0); + EXPECT_EQ(info.total_probe_length.load(), 0); + EXPECT_EQ(info.hashes_bitwise_or.load(), 0); + EXPECT_EQ(info.hashes_bitwise_and.load(), ~size_t{}); + EXPECT_GE(info.create_time, test_start); + + info.capacity.store(1, std::memory_order_relaxed); + info.size.store(1, std::memory_order_relaxed); + info.num_erases.store(1, std::memory_order_relaxed); + info.max_probe_length.store(1, std::memory_order_relaxed); + info.total_probe_length.store(1, std::memory_order_relaxed); + info.hashes_bitwise_or.store(1, std::memory_order_relaxed); + info.hashes_bitwise_and.store(1, std::memory_order_relaxed); + info.create_time = test_start - absl::Hours(20); + + info.PrepareForSampling(); + EXPECT_EQ(info.capacity.load(), 0); + EXPECT_EQ(info.size.load(), 0); + EXPECT_EQ(info.num_erases.load(), 0); + EXPECT_EQ(info.num_rehashes.load(), 0); + EXPECT_EQ(info.max_probe_length.load(), 0); + EXPECT_EQ(info.total_probe_length.load(), 0); + EXPECT_EQ(info.hashes_bitwise_or.load(), 0); + EXPECT_EQ(info.hashes_bitwise_and.load(), ~size_t{}); + EXPECT_GE(info.create_time, test_start); +} + +TEST(HashtablezInfoTest, RecordStorageChanged) { + HashtablezInfo info; + absl::MutexLock l(&info.init_mu); + info.PrepareForSampling(); + RecordStorageChangedSlow(&info, 17, 47); + EXPECT_EQ(info.size.load(), 17); + EXPECT_EQ(info.capacity.load(), 47); + RecordStorageChangedSlow(&info, 20, 20); + EXPECT_EQ(info.size.load(), 20); + EXPECT_EQ(info.capacity.load(), 20); +} + +TEST(HashtablezInfoTest, RecordInsert) { + HashtablezInfo info; + absl::MutexLock l(&info.init_mu); + info.PrepareForSampling(); + EXPECT_EQ(info.max_probe_length.load(), 0); + RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); + EXPECT_EQ(info.max_probe_length.load(), 6); + EXPECT_EQ(info.hashes_bitwise_and.load(), 0x0000FF00); + EXPECT_EQ(info.hashes_bitwise_or.load(), 0x0000FF00); + RecordInsertSlow(&info, 0x000FF000, 4 * kProbeLength); + EXPECT_EQ(info.max_probe_length.load(), 6); + EXPECT_EQ(info.hashes_bitwise_and.load(), 0x0000F000); + EXPECT_EQ(info.hashes_bitwise_or.load(), 0x000FFF00); + RecordInsertSlow(&info, 0x00FF0000, 12 * kProbeLength); + EXPECT_EQ(info.max_probe_length.load(), 12); + EXPECT_EQ(info.hashes_bitwise_and.load(), 0x00000000); + EXPECT_EQ(info.hashes_bitwise_or.load(), 0x00FFFF00); +} + +TEST(HashtablezInfoTest, RecordErase) { + HashtablezInfo info; + absl::MutexLock l(&info.init_mu); + info.PrepareForSampling(); + EXPECT_EQ(info.num_erases.load(), 0); + EXPECT_EQ(info.size.load(), 0); + RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); + EXPECT_EQ(info.size.load(), 1); + RecordEraseSlow(&info); + EXPECT_EQ(info.size.load(), 0); + EXPECT_EQ(info.num_erases.load(), 1); +} + +TEST(HashtablezInfoTest, RecordRehash) { + HashtablezInfo info; + absl::MutexLock l(&info.init_mu); + info.PrepareForSampling(); + RecordInsertSlow(&info, 0x1, 0); + RecordInsertSlow(&info, 0x2, kProbeLength); + RecordInsertSlow(&info, 0x4, kProbeLength); + RecordInsertSlow(&info, 0x8, 2 * kProbeLength); + EXPECT_EQ(info.size.load(), 4); + EXPECT_EQ(info.total_probe_length.load(), 4); + + RecordEraseSlow(&info); + RecordEraseSlow(&info); + EXPECT_EQ(info.size.load(), 2); + EXPECT_EQ(info.total_probe_length.load(), 4); + EXPECT_EQ(info.num_erases.load(), 2); + + RecordRehashSlow(&info, 3 * kProbeLength); + EXPECT_EQ(info.size.load(), 2); + EXPECT_EQ(info.total_probe_length.load(), 3); + EXPECT_EQ(info.num_erases.load(), 0); + EXPECT_EQ(info.num_rehashes.load(), 1); +} + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +TEST(HashtablezSamplerTest, SmallSampleParameter) { + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); + + for (int i = 0; i < 1000; ++i) { + int64_t next_sample = 0; + HashtablezInfo* sample = SampleSlow(&next_sample); + EXPECT_GT(next_sample, 0); + EXPECT_NE(sample, nullptr); + UnsampleSlow(sample); + } +} + +TEST(HashtablezSamplerTest, LargeSampleParameter) { + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(std::numeric_limits<int32_t>::max()); + + for (int i = 0; i < 1000; ++i) { + int64_t next_sample = 0; + HashtablezInfo* sample = SampleSlow(&next_sample); + EXPECT_GT(next_sample, 0); + EXPECT_NE(sample, nullptr); + UnsampleSlow(sample); + } +} + +TEST(HashtablezSamplerTest, Sample) { + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); + int64_t num_sampled = 0; + int64_t total = 0; + double sample_rate = 0.0; + for (int i = 0; i < 1000000; ++i) { + HashtablezInfoHandle h = Sample(); + ++total; + if (HashtablezInfoHandlePeer::IsSampled(h)) { + ++num_sampled; + } + sample_rate = static_cast<double>(num_sampled) / total; + if (0.005 < sample_rate && sample_rate < 0.015) break; + } + EXPECT_NEAR(sample_rate, 0.01, 0.005); +} + +TEST(HashtablezSamplerTest, Handle) { + auto& sampler = HashtablezSampler::Global(); + HashtablezInfoHandle h(sampler.Register()); + auto* info = HashtablezInfoHandlePeer::GetInfo(&h); + info->hashes_bitwise_and.store(0x12345678, std::memory_order_relaxed); + + bool found = false; + sampler.Iterate([&](const HashtablezInfo& h) { + if (&h == info) { + EXPECT_EQ(h.hashes_bitwise_and.load(), 0x12345678); + found = true; + } + }); + EXPECT_TRUE(found); + + h = HashtablezInfoHandle(); + found = false; + sampler.Iterate([&](const HashtablezInfo& h) { + if (&h == info) { + // this will only happen if some other thread has resurrected the info + // the old handle was using. + if (h.hashes_bitwise_and.load() == 0x12345678) { + found = true; + } + } + }); + EXPECT_FALSE(found); +} +#endif + + +TEST(HashtablezSamplerTest, Registration) { + HashtablezSampler sampler; + auto* info1 = Register(&sampler, 1); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(1)); + + auto* info2 = Register(&sampler, 2); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(1, 2)); + info1->size.store(3); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(3, 2)); + + sampler.Unregister(info1); + sampler.Unregister(info2); +} + +TEST(HashtablezSamplerTest, Unregistration) { + HashtablezSampler sampler; + std::vector<HashtablezInfo*> infos; + for (size_t i = 0; i < 3; ++i) { + infos.push_back(Register(&sampler, i)); + } + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 1, 2)); + + sampler.Unregister(infos[1]); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 2)); + + infos.push_back(Register(&sampler, 3)); + infos.push_back(Register(&sampler, 4)); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 2, 3, 4)); + sampler.Unregister(infos[3]); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 2, 4)); + + sampler.Unregister(infos[0]); + sampler.Unregister(infos[2]); + sampler.Unregister(infos[4]); + EXPECT_THAT(GetSizes(&sampler), IsEmpty()); +} + +TEST(HashtablezSamplerTest, MultiThreaded) { + HashtablezSampler sampler; + Notification stop; + ThreadPool pool(10); + + for (int i = 0; i < 10; ++i) { + pool.Schedule([&sampler, &stop]() { + std::random_device rd; + std::mt19937 gen(rd()); + + std::vector<HashtablezInfo*> infoz; + while (!stop.HasBeenNotified()) { + if (infoz.empty()) { + infoz.push_back(sampler.Register()); + } + switch (std::uniform_int_distribution<>(0, 2)(gen)) { + case 0: { + infoz.push_back(sampler.Register()); + break; + } + case 1: { + size_t p = + std::uniform_int_distribution<>(0, infoz.size() - 1)(gen); + HashtablezInfo* info = infoz[p]; + infoz[p] = infoz.back(); + infoz.pop_back(); + sampler.Unregister(info); + break; + } + case 2: { + absl::Duration oldest = absl::ZeroDuration(); + sampler.Iterate([&](const HashtablezInfo& info) { + oldest = std::max(oldest, absl::Now() - info.create_time); + }); + ASSERT_GE(oldest, absl::ZeroDuration()); + break; + } + } + } + }); + } + // The threads will hammer away. Give it a little bit of time for tsan to + // spot errors. + absl::SleepFor(absl::Seconds(3)); + stop.Notify(); +} + +TEST(HashtablezSamplerTest, Callback) { + HashtablezSampler sampler; + + auto* info1 = Register(&sampler, 1); + auto* info2 = Register(&sampler, 2); + + static const HashtablezInfo* expected; + + auto callback = [](const HashtablezInfo& info) { + // We can't use `info` outside of this callback because the object will be + // disposed as soon as we return from here. + EXPECT_EQ(&info, expected); + }; + + // Set the callback. + EXPECT_EQ(sampler.SetDisposeCallback(callback), nullptr); + expected = info1; + sampler.Unregister(info1); + + // Unset the callback. + EXPECT_EQ(callback, sampler.SetDisposeCallback(nullptr)); + expected = nullptr; // no more calls. + sampler.Unregister(info2); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/have_sse.h b/third_party/abseil_cpp/absl/container/internal/have_sse.h new file mode 100644 index 000000000000..e75e1a16d327 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/have_sse.h @@ -0,0 +1,50 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Shared config probing for SSE instructions used in Swiss tables. +#ifndef ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ +#define ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ + +#ifndef ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#if defined(__SSE2__) || \ + (defined(_MSC_VER) && \ + (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) +#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 1 +#else +#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 0 +#endif +#endif + +#ifndef ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 +#ifdef __SSSE3__ +#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 1 +#else +#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 0 +#endif +#endif + +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 && \ + !ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#error "Bad configuration!" +#endif + +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#include <emmintrin.h> +#endif + +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 +#include <tmmintrin.h> +#endif + +#endif // ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/inlined_vector.h b/third_party/abseil_cpp/absl/container/internal/inlined_vector.h new file mode 100644 index 000000000000..c98c25c44221 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/inlined_vector.h @@ -0,0 +1,895 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_INTERNAL_H_ +#define ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_INTERNAL_H_ + +#include <algorithm> +#include <cstddef> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> +#include <utility> + +#include "absl/base/macros.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace inlined_vector_internal { + +template <typename Iterator> +using IsAtLeastForwardIterator = std::is_convertible< + typename std::iterator_traits<Iterator>::iterator_category, + std::forward_iterator_tag>; + +template <typename AllocatorType, + typename ValueType = + typename absl::allocator_traits<AllocatorType>::value_type> +using IsMemcpyOk = + absl::conjunction<std::is_same<AllocatorType, std::allocator<ValueType>>, + absl::is_trivially_copy_constructible<ValueType>, + absl::is_trivially_copy_assignable<ValueType>, + absl::is_trivially_destructible<ValueType>>; + +template <typename AllocatorType, typename Pointer, typename SizeType> +void DestroyElements(AllocatorType* alloc_ptr, Pointer destroy_first, + SizeType destroy_size) { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + + if (destroy_first != nullptr) { + for (auto i = destroy_size; i != 0;) { + --i; + AllocatorTraits::destroy(*alloc_ptr, destroy_first + i); + } + +#if !defined(NDEBUG) + { + using ValueType = typename AllocatorTraits::value_type; + + // Overwrite unused memory with `0xab` so we can catch uninitialized + // usage. + // + // Cast to `void*` to tell the compiler that we don't care that we might + // be scribbling on a vtable pointer. + void* memory_ptr = destroy_first; + auto memory_size = destroy_size * sizeof(ValueType); + std::memset(memory_ptr, 0xab, memory_size); + } +#endif // !defined(NDEBUG) + } +} + +template <typename AllocatorType, typename Pointer, typename ValueAdapter, + typename SizeType> +void ConstructElements(AllocatorType* alloc_ptr, Pointer construct_first, + ValueAdapter* values_ptr, SizeType construct_size) { + for (SizeType i = 0; i < construct_size; ++i) { + ABSL_INTERNAL_TRY { + values_ptr->ConstructNext(alloc_ptr, construct_first + i); + } + ABSL_INTERNAL_CATCH_ANY { + inlined_vector_internal::DestroyElements(alloc_ptr, construct_first, i); + ABSL_INTERNAL_RETHROW; + } + } +} + +template <typename Pointer, typename ValueAdapter, typename SizeType> +void AssignElements(Pointer assign_first, ValueAdapter* values_ptr, + SizeType assign_size) { + for (SizeType i = 0; i < assign_size; ++i) { + values_ptr->AssignNext(assign_first + i); + } +} + +template <typename AllocatorType> +struct StorageView { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + using Pointer = typename AllocatorTraits::pointer; + using SizeType = typename AllocatorTraits::size_type; + + Pointer data; + SizeType size; + SizeType capacity; +}; + +template <typename AllocatorType, typename Iterator> +class IteratorValueAdapter { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + using Pointer = typename AllocatorTraits::pointer; + + public: + explicit IteratorValueAdapter(const Iterator& it) : it_(it) {} + + void ConstructNext(AllocatorType* alloc_ptr, Pointer construct_at) { + AllocatorTraits::construct(*alloc_ptr, construct_at, *it_); + ++it_; + } + + void AssignNext(Pointer assign_at) { + *assign_at = *it_; + ++it_; + } + + private: + Iterator it_; +}; + +template <typename AllocatorType> +class CopyValueAdapter { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + using ValueType = typename AllocatorTraits::value_type; + using Pointer = typename AllocatorTraits::pointer; + using ConstPointer = typename AllocatorTraits::const_pointer; + + public: + explicit CopyValueAdapter(const ValueType& v) : ptr_(std::addressof(v)) {} + + void ConstructNext(AllocatorType* alloc_ptr, Pointer construct_at) { + AllocatorTraits::construct(*alloc_ptr, construct_at, *ptr_); + } + + void AssignNext(Pointer assign_at) { *assign_at = *ptr_; } + + private: + ConstPointer ptr_; +}; + +template <typename AllocatorType> +class DefaultValueAdapter { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + using ValueType = typename AllocatorTraits::value_type; + using Pointer = typename AllocatorTraits::pointer; + + public: + explicit DefaultValueAdapter() {} + + void ConstructNext(AllocatorType* alloc_ptr, Pointer construct_at) { + AllocatorTraits::construct(*alloc_ptr, construct_at); + } + + void AssignNext(Pointer assign_at) { *assign_at = ValueType(); } +}; + +template <typename AllocatorType> +class AllocationTransaction { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + using Pointer = typename AllocatorTraits::pointer; + using SizeType = typename AllocatorTraits::size_type; + + public: + explicit AllocationTransaction(AllocatorType* alloc_ptr) + : alloc_data_(*alloc_ptr, nullptr) {} + + ~AllocationTransaction() { + if (DidAllocate()) { + AllocatorTraits::deallocate(GetAllocator(), GetData(), GetCapacity()); + } + } + + AllocationTransaction(const AllocationTransaction&) = delete; + void operator=(const AllocationTransaction&) = delete; + + AllocatorType& GetAllocator() { return alloc_data_.template get<0>(); } + Pointer& GetData() { return alloc_data_.template get<1>(); } + SizeType& GetCapacity() { return capacity_; } + + bool DidAllocate() { return GetData() != nullptr; } + Pointer Allocate(SizeType capacity) { + GetData() = AllocatorTraits::allocate(GetAllocator(), capacity); + GetCapacity() = capacity; + return GetData(); + } + + void Reset() { + GetData() = nullptr; + GetCapacity() = 0; + } + + private: + container_internal::CompressedTuple<AllocatorType, Pointer> alloc_data_; + SizeType capacity_ = 0; +}; + +template <typename AllocatorType> +class ConstructionTransaction { + using AllocatorTraits = absl::allocator_traits<AllocatorType>; + using Pointer = typename AllocatorTraits::pointer; + using SizeType = typename AllocatorTraits::size_type; + + public: + explicit ConstructionTransaction(AllocatorType* alloc_ptr) + : alloc_data_(*alloc_ptr, nullptr) {} + + ~ConstructionTransaction() { + if (DidConstruct()) { + inlined_vector_internal::DestroyElements(std::addressof(GetAllocator()), + GetData(), GetSize()); + } + } + + ConstructionTransaction(const ConstructionTransaction&) = delete; + void operator=(const ConstructionTransaction&) = delete; + + AllocatorType& GetAllocator() { return alloc_data_.template get<0>(); } + Pointer& GetData() { return alloc_data_.template get<1>(); } + SizeType& GetSize() { return size_; } + + bool DidConstruct() { return GetData() != nullptr; } + template <typename ValueAdapter> + void Construct(Pointer data, ValueAdapter* values_ptr, SizeType size) { + inlined_vector_internal::ConstructElements(std::addressof(GetAllocator()), + data, values_ptr, size); + GetData() = data; + GetSize() = size; + } + void Commit() { + GetData() = nullptr; + GetSize() = 0; + } + + private: + container_internal::CompressedTuple<AllocatorType, Pointer> alloc_data_; + SizeType size_ = 0; +}; + +template <typename T, size_t N, typename A> +class Storage { + public: + using AllocatorTraits = absl::allocator_traits<A>; + using allocator_type = typename AllocatorTraits::allocator_type; + using value_type = typename AllocatorTraits::value_type; + using pointer = typename AllocatorTraits::pointer; + using const_pointer = typename AllocatorTraits::const_pointer; + using size_type = typename AllocatorTraits::size_type; + using difference_type = typename AllocatorTraits::difference_type; + + using reference = value_type&; + using const_reference = const value_type&; + using RValueReference = value_type&&; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using MoveIterator = std::move_iterator<iterator>; + using IsMemcpyOk = inlined_vector_internal::IsMemcpyOk<allocator_type>; + + using StorageView = inlined_vector_internal::StorageView<allocator_type>; + + template <typename Iterator> + using IteratorValueAdapter = + inlined_vector_internal::IteratorValueAdapter<allocator_type, Iterator>; + using CopyValueAdapter = + inlined_vector_internal::CopyValueAdapter<allocator_type>; + using DefaultValueAdapter = + inlined_vector_internal::DefaultValueAdapter<allocator_type>; + + using AllocationTransaction = + inlined_vector_internal::AllocationTransaction<allocator_type>; + using ConstructionTransaction = + inlined_vector_internal::ConstructionTransaction<allocator_type>; + + static size_type NextCapacity(size_type current_capacity) { + return current_capacity * 2; + } + + static size_type ComputeCapacity(size_type current_capacity, + size_type requested_capacity) { + return (std::max)(NextCapacity(current_capacity), requested_capacity); + } + + // --------------------------------------------------------------------------- + // Storage Constructors and Destructor + // --------------------------------------------------------------------------- + + Storage() : metadata_() {} + + explicit Storage(const allocator_type& alloc) : metadata_(alloc, {}) {} + + ~Storage() { + pointer data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData(); + inlined_vector_internal::DestroyElements(GetAllocPtr(), data, GetSize()); + DeallocateIfAllocated(); + } + + // --------------------------------------------------------------------------- + // Storage Member Accessors + // --------------------------------------------------------------------------- + + size_type& GetSizeAndIsAllocated() { return metadata_.template get<1>(); } + + const size_type& GetSizeAndIsAllocated() const { + return metadata_.template get<1>(); + } + + size_type GetSize() const { return GetSizeAndIsAllocated() >> 1; } + + bool GetIsAllocated() const { return GetSizeAndIsAllocated() & 1; } + + pointer GetAllocatedData() { return data_.allocated.allocated_data; } + + const_pointer GetAllocatedData() const { + return data_.allocated.allocated_data; + } + + pointer GetInlinedData() { + return reinterpret_cast<pointer>( + std::addressof(data_.inlined.inlined_data[0])); + } + + const_pointer GetInlinedData() const { + return reinterpret_cast<const_pointer>( + std::addressof(data_.inlined.inlined_data[0])); + } + + size_type GetAllocatedCapacity() const { + return data_.allocated.allocated_capacity; + } + + size_type GetInlinedCapacity() const { return static_cast<size_type>(N); } + + StorageView MakeStorageView() { + return GetIsAllocated() + ? StorageView{GetAllocatedData(), GetSize(), + GetAllocatedCapacity()} + : StorageView{GetInlinedData(), GetSize(), GetInlinedCapacity()}; + } + + allocator_type* GetAllocPtr() { + return std::addressof(metadata_.template get<0>()); + } + + const allocator_type* GetAllocPtr() const { + return std::addressof(metadata_.template get<0>()); + } + + // --------------------------------------------------------------------------- + // Storage Member Mutators + // --------------------------------------------------------------------------- + + template <typename ValueAdapter> + void Initialize(ValueAdapter values, size_type new_size); + + template <typename ValueAdapter> + void Assign(ValueAdapter values, size_type new_size); + + template <typename ValueAdapter> + void Resize(ValueAdapter values, size_type new_size); + + template <typename ValueAdapter> + iterator Insert(const_iterator pos, ValueAdapter values, + size_type insert_count); + + template <typename... Args> + reference EmplaceBack(Args&&... args); + + iterator Erase(const_iterator from, const_iterator to); + + void Reserve(size_type requested_capacity); + + void ShrinkToFit(); + + void Swap(Storage* other_storage_ptr); + + void SetIsAllocated() { + GetSizeAndIsAllocated() |= static_cast<size_type>(1); + } + + void UnsetIsAllocated() { + GetSizeAndIsAllocated() &= ((std::numeric_limits<size_type>::max)() - 1); + } + + void SetSize(size_type size) { + GetSizeAndIsAllocated() = + (size << 1) | static_cast<size_type>(GetIsAllocated()); + } + + void SetAllocatedSize(size_type size) { + GetSizeAndIsAllocated() = (size << 1) | static_cast<size_type>(1); + } + + void SetInlinedSize(size_type size) { + GetSizeAndIsAllocated() = size << static_cast<size_type>(1); + } + + void AddSize(size_type count) { + GetSizeAndIsAllocated() += count << static_cast<size_type>(1); + } + + void SubtractSize(size_type count) { + assert(count <= GetSize()); + + GetSizeAndIsAllocated() -= count << static_cast<size_type>(1); + } + + void SetAllocatedData(pointer data, size_type capacity) { + data_.allocated.allocated_data = data; + data_.allocated.allocated_capacity = capacity; + } + + void AcquireAllocatedData(AllocationTransaction* allocation_tx_ptr) { + SetAllocatedData(allocation_tx_ptr->GetData(), + allocation_tx_ptr->GetCapacity()); + + allocation_tx_ptr->Reset(); + } + + void MemcpyFrom(const Storage& other_storage) { + assert(IsMemcpyOk::value || other_storage.GetIsAllocated()); + + GetSizeAndIsAllocated() = other_storage.GetSizeAndIsAllocated(); + data_ = other_storage.data_; + } + + void DeallocateIfAllocated() { + if (GetIsAllocated()) { + AllocatorTraits::deallocate(*GetAllocPtr(), GetAllocatedData(), + GetAllocatedCapacity()); + } + } + + private: + using Metadata = + container_internal::CompressedTuple<allocator_type, size_type>; + + struct Allocated { + pointer allocated_data; + size_type allocated_capacity; + }; + + struct Inlined { + alignas(value_type) char inlined_data[sizeof(value_type[N])]; + }; + + union Data { + Allocated allocated; + Inlined inlined; + }; + + template <typename... Args> + ABSL_ATTRIBUTE_NOINLINE reference EmplaceBackSlow(Args&&... args); + + Metadata metadata_; + Data data_; +}; + +template <typename T, size_t N, typename A> +template <typename ValueAdapter> +auto Storage<T, N, A>::Initialize(ValueAdapter values, size_type new_size) + -> void { + // Only callable from constructors! + assert(!GetIsAllocated()); + assert(GetSize() == 0); + + pointer construct_data; + if (new_size > GetInlinedCapacity()) { + // Because this is only called from the `InlinedVector` constructors, it's + // safe to take on the allocation with size `0`. If `ConstructElements(...)` + // throws, deallocation will be automatically handled by `~Storage()`. + size_type new_capacity = ComputeCapacity(GetInlinedCapacity(), new_size); + construct_data = AllocatorTraits::allocate(*GetAllocPtr(), new_capacity); + SetAllocatedData(construct_data, new_capacity); + SetIsAllocated(); + } else { + construct_data = GetInlinedData(); + } + + inlined_vector_internal::ConstructElements(GetAllocPtr(), construct_data, + &values, new_size); + + // Since the initial size was guaranteed to be `0` and the allocated bit is + // already correct for either case, *adding* `new_size` gives us the correct + // result faster than setting it directly. + AddSize(new_size); +} + +template <typename T, size_t N, typename A> +template <typename ValueAdapter> +auto Storage<T, N, A>::Assign(ValueAdapter values, size_type new_size) -> void { + StorageView storage_view = MakeStorageView(); + + AllocationTransaction allocation_tx(GetAllocPtr()); + + absl::Span<value_type> assign_loop; + absl::Span<value_type> construct_loop; + absl::Span<value_type> destroy_loop; + + if (new_size > storage_view.capacity) { + size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); + construct_loop = {allocation_tx.Allocate(new_capacity), new_size}; + destroy_loop = {storage_view.data, storage_view.size}; + } else if (new_size > storage_view.size) { + assign_loop = {storage_view.data, storage_view.size}; + construct_loop = {storage_view.data + storage_view.size, + new_size - storage_view.size}; + } else { + assign_loop = {storage_view.data, new_size}; + destroy_loop = {storage_view.data + new_size, storage_view.size - new_size}; + } + + inlined_vector_internal::AssignElements(assign_loop.data(), &values, + assign_loop.size()); + + inlined_vector_internal::ConstructElements( + GetAllocPtr(), construct_loop.data(), &values, construct_loop.size()); + + inlined_vector_internal::DestroyElements(GetAllocPtr(), destroy_loop.data(), + destroy_loop.size()); + + if (allocation_tx.DidAllocate()) { + DeallocateIfAllocated(); + AcquireAllocatedData(&allocation_tx); + SetIsAllocated(); + } + + SetSize(new_size); +} + +template <typename T, size_t N, typename A> +template <typename ValueAdapter> +auto Storage<T, N, A>::Resize(ValueAdapter values, size_type new_size) -> void { + StorageView storage_view = MakeStorageView(); + auto* const base = storage_view.data; + const size_type size = storage_view.size; + auto* alloc = GetAllocPtr(); + if (new_size <= size) { + // Destroy extra old elements. + inlined_vector_internal::DestroyElements(alloc, base + new_size, + size - new_size); + } else if (new_size <= storage_view.capacity) { + // Construct new elements in place. + inlined_vector_internal::ConstructElements(alloc, base + size, &values, + new_size - size); + } else { + // Steps: + // a. Allocate new backing store. + // b. Construct new elements in new backing store. + // c. Move existing elements from old backing store to now. + // d. Destroy all elements in old backing store. + // Use transactional wrappers for the first two steps so we can roll + // back if necessary due to exceptions. + AllocationTransaction allocation_tx(alloc); + size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); + pointer new_data = allocation_tx.Allocate(new_capacity); + + ConstructionTransaction construction_tx(alloc); + construction_tx.Construct(new_data + size, &values, new_size - size); + + IteratorValueAdapter<MoveIterator> move_values((MoveIterator(base))); + inlined_vector_internal::ConstructElements(alloc, new_data, &move_values, + size); + + inlined_vector_internal::DestroyElements(alloc, base, size); + construction_tx.Commit(); + DeallocateIfAllocated(); + AcquireAllocatedData(&allocation_tx); + SetIsAllocated(); + } + SetSize(new_size); +} + +template <typename T, size_t N, typename A> +template <typename ValueAdapter> +auto Storage<T, N, A>::Insert(const_iterator pos, ValueAdapter values, + size_type insert_count) -> iterator { + StorageView storage_view = MakeStorageView(); + + size_type insert_index = + std::distance(const_iterator(storage_view.data), pos); + size_type insert_end_index = insert_index + insert_count; + size_type new_size = storage_view.size + insert_count; + + if (new_size > storage_view.capacity) { + AllocationTransaction allocation_tx(GetAllocPtr()); + ConstructionTransaction construction_tx(GetAllocPtr()); + ConstructionTransaction move_construciton_tx(GetAllocPtr()); + + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(storage_view.data)); + + size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); + pointer new_data = allocation_tx.Allocate(new_capacity); + + construction_tx.Construct(new_data + insert_index, &values, insert_count); + + move_construciton_tx.Construct(new_data, &move_values, insert_index); + + inlined_vector_internal::ConstructElements( + GetAllocPtr(), new_data + insert_end_index, &move_values, + storage_view.size - insert_index); + + inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, + storage_view.size); + + construction_tx.Commit(); + move_construciton_tx.Commit(); + DeallocateIfAllocated(); + AcquireAllocatedData(&allocation_tx); + + SetAllocatedSize(new_size); + return iterator(new_data + insert_index); + } else { + size_type move_construction_destination_index = + (std::max)(insert_end_index, storage_view.size); + + ConstructionTransaction move_construction_tx(GetAllocPtr()); + + IteratorValueAdapter<MoveIterator> move_construction_values( + MoveIterator(storage_view.data + + (move_construction_destination_index - insert_count))); + absl::Span<value_type> move_construction = { + storage_view.data + move_construction_destination_index, + new_size - move_construction_destination_index}; + + pointer move_assignment_values = storage_view.data + insert_index; + absl::Span<value_type> move_assignment = { + storage_view.data + insert_end_index, + move_construction_destination_index - insert_end_index}; + + absl::Span<value_type> insert_assignment = {move_assignment_values, + move_construction.size()}; + + absl::Span<value_type> insert_construction = { + insert_assignment.data() + insert_assignment.size(), + insert_count - insert_assignment.size()}; + + move_construction_tx.Construct(move_construction.data(), + &move_construction_values, + move_construction.size()); + + for (pointer destination = move_assignment.data() + move_assignment.size(), + last_destination = move_assignment.data(), + source = move_assignment_values + move_assignment.size(); + ;) { + --destination; + --source; + if (destination < last_destination) break; + *destination = std::move(*source); + } + + inlined_vector_internal::AssignElements(insert_assignment.data(), &values, + insert_assignment.size()); + + inlined_vector_internal::ConstructElements( + GetAllocPtr(), insert_construction.data(), &values, + insert_construction.size()); + + move_construction_tx.Commit(); + + AddSize(insert_count); + return iterator(storage_view.data + insert_index); + } +} + +template <typename T, size_t N, typename A> +template <typename... Args> +auto Storage<T, N, A>::EmplaceBack(Args&&... args) -> reference { + StorageView storage_view = MakeStorageView(); + const auto n = storage_view.size; + if (ABSL_PREDICT_TRUE(n != storage_view.capacity)) { + // Fast path; new element fits. + pointer last_ptr = storage_view.data + n; + AllocatorTraits::construct(*GetAllocPtr(), last_ptr, + std::forward<Args>(args)...); + AddSize(1); + return *last_ptr; + } + // TODO(b/173712035): Annotate with musttail attribute to prevent regression. + return EmplaceBackSlow(std::forward<Args>(args)...); +} + +template <typename T, size_t N, typename A> +template <typename... Args> +auto Storage<T, N, A>::EmplaceBackSlow(Args&&... args) -> reference { + StorageView storage_view = MakeStorageView(); + AllocationTransaction allocation_tx(GetAllocPtr()); + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(storage_view.data)); + size_type new_capacity = NextCapacity(storage_view.capacity); + pointer construct_data = allocation_tx.Allocate(new_capacity); + pointer last_ptr = construct_data + storage_view.size; + + // Construct new element. + AllocatorTraits::construct(*GetAllocPtr(), last_ptr, + std::forward<Args>(args)...); + // Move elements from old backing store to new backing store. + ABSL_INTERNAL_TRY { + inlined_vector_internal::ConstructElements( + GetAllocPtr(), allocation_tx.GetData(), &move_values, + storage_view.size); + } + ABSL_INTERNAL_CATCH_ANY { + AllocatorTraits::destroy(*GetAllocPtr(), last_ptr); + ABSL_INTERNAL_RETHROW; + } + // Destroy elements in old backing store. + inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, + storage_view.size); + + DeallocateIfAllocated(); + AcquireAllocatedData(&allocation_tx); + SetIsAllocated(); + AddSize(1); + return *last_ptr; +} + +template <typename T, size_t N, typename A> +auto Storage<T, N, A>::Erase(const_iterator from, const_iterator to) + -> iterator { + StorageView storage_view = MakeStorageView(); + + size_type erase_size = std::distance(from, to); + size_type erase_index = + std::distance(const_iterator(storage_view.data), from); + size_type erase_end_index = erase_index + erase_size; + + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(storage_view.data + erase_end_index)); + + inlined_vector_internal::AssignElements(storage_view.data + erase_index, + &move_values, + storage_view.size - erase_end_index); + + inlined_vector_internal::DestroyElements( + GetAllocPtr(), storage_view.data + (storage_view.size - erase_size), + erase_size); + + SubtractSize(erase_size); + return iterator(storage_view.data + erase_index); +} + +template <typename T, size_t N, typename A> +auto Storage<T, N, A>::Reserve(size_type requested_capacity) -> void { + StorageView storage_view = MakeStorageView(); + + if (ABSL_PREDICT_FALSE(requested_capacity <= storage_view.capacity)) return; + + AllocationTransaction allocation_tx(GetAllocPtr()); + + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(storage_view.data)); + + size_type new_capacity = + ComputeCapacity(storage_view.capacity, requested_capacity); + pointer new_data = allocation_tx.Allocate(new_capacity); + + inlined_vector_internal::ConstructElements(GetAllocPtr(), new_data, + &move_values, storage_view.size); + + inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, + storage_view.size); + + DeallocateIfAllocated(); + AcquireAllocatedData(&allocation_tx); + SetIsAllocated(); +} + +template <typename T, size_t N, typename A> +auto Storage<T, N, A>::ShrinkToFit() -> void { + // May only be called on allocated instances! + assert(GetIsAllocated()); + + StorageView storage_view{GetAllocatedData(), GetSize(), + GetAllocatedCapacity()}; + + if (ABSL_PREDICT_FALSE(storage_view.size == storage_view.capacity)) return; + + AllocationTransaction allocation_tx(GetAllocPtr()); + + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(storage_view.data)); + + pointer construct_data; + if (storage_view.size > GetInlinedCapacity()) { + size_type new_capacity = storage_view.size; + construct_data = allocation_tx.Allocate(new_capacity); + } else { + construct_data = GetInlinedData(); + } + + ABSL_INTERNAL_TRY { + inlined_vector_internal::ConstructElements(GetAllocPtr(), construct_data, + &move_values, storage_view.size); + } + ABSL_INTERNAL_CATCH_ANY { + SetAllocatedData(storage_view.data, storage_view.capacity); + ABSL_INTERNAL_RETHROW; + } + + inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, + storage_view.size); + + AllocatorTraits::deallocate(*GetAllocPtr(), storage_view.data, + storage_view.capacity); + + if (allocation_tx.DidAllocate()) { + AcquireAllocatedData(&allocation_tx); + } else { + UnsetIsAllocated(); + } +} + +template <typename T, size_t N, typename A> +auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { + using std::swap; + assert(this != other_storage_ptr); + + if (GetIsAllocated() && other_storage_ptr->GetIsAllocated()) { + swap(data_.allocated, other_storage_ptr->data_.allocated); + } else if (!GetIsAllocated() && !other_storage_ptr->GetIsAllocated()) { + Storage* small_ptr = this; + Storage* large_ptr = other_storage_ptr; + if (small_ptr->GetSize() > large_ptr->GetSize()) swap(small_ptr, large_ptr); + + for (size_type i = 0; i < small_ptr->GetSize(); ++i) { + swap(small_ptr->GetInlinedData()[i], large_ptr->GetInlinedData()[i]); + } + + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(large_ptr->GetInlinedData() + small_ptr->GetSize())); + + inlined_vector_internal::ConstructElements( + large_ptr->GetAllocPtr(), + small_ptr->GetInlinedData() + small_ptr->GetSize(), &move_values, + large_ptr->GetSize() - small_ptr->GetSize()); + + inlined_vector_internal::DestroyElements( + large_ptr->GetAllocPtr(), + large_ptr->GetInlinedData() + small_ptr->GetSize(), + large_ptr->GetSize() - small_ptr->GetSize()); + } else { + Storage* allocated_ptr = this; + Storage* inlined_ptr = other_storage_ptr; + if (!allocated_ptr->GetIsAllocated()) swap(allocated_ptr, inlined_ptr); + + StorageView allocated_storage_view{allocated_ptr->GetAllocatedData(), + allocated_ptr->GetSize(), + allocated_ptr->GetAllocatedCapacity()}; + + IteratorValueAdapter<MoveIterator> move_values( + MoveIterator(inlined_ptr->GetInlinedData())); + + ABSL_INTERNAL_TRY { + inlined_vector_internal::ConstructElements( + inlined_ptr->GetAllocPtr(), allocated_ptr->GetInlinedData(), + &move_values, inlined_ptr->GetSize()); + } + ABSL_INTERNAL_CATCH_ANY { + allocated_ptr->SetAllocatedData(allocated_storage_view.data, + allocated_storage_view.capacity); + ABSL_INTERNAL_RETHROW; + } + + inlined_vector_internal::DestroyElements(inlined_ptr->GetAllocPtr(), + inlined_ptr->GetInlinedData(), + inlined_ptr->GetSize()); + + inlined_ptr->SetAllocatedData(allocated_storage_view.data, + allocated_storage_view.capacity); + } + + swap(GetSizeAndIsAllocated(), other_storage_ptr->GetSizeAndIsAllocated()); + swap(*GetAllocPtr(), *other_storage_ptr->GetAllocPtr()); +} + +} // namespace inlined_vector_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_INLINED_VECTOR_INTERNAL_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/layout.h b/third_party/abseil_cpp/absl/container/internal/layout.h new file mode 100644 index 000000000000..233678331543 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/layout.h @@ -0,0 +1,743 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// MOTIVATION AND TUTORIAL +// +// If you want to put in a single heap allocation N doubles followed by M ints, +// it's easy if N and M are known at compile time. +// +// struct S { +// double a[N]; +// int b[M]; +// }; +// +// S* p = new S; +// +// But what if N and M are known only in run time? Class template Layout to the +// rescue! It's a portable generalization of the technique known as struct hack. +// +// // This object will tell us everything we need to know about the memory +// // layout of double[N] followed by int[M]. It's structurally identical to +// // size_t[2] that stores N and M. It's very cheap to create. +// const Layout<double, int> layout(N, M); +// +// // Allocate enough memory for both arrays. `AllocSize()` tells us how much +// // memory is needed. We are free to use any allocation function we want as +// // long as it returns aligned memory. +// std::unique_ptr<unsigned char[]> p(new unsigned char[layout.AllocSize()]); +// +// // Obtain the pointer to the array of doubles. +// // Equivalent to `reinterpret_cast<double*>(p.get())`. +// // +// // We could have written layout.Pointer<0>(p) instead. If all the types are +// // unique you can use either form, but if some types are repeated you must +// // use the index form. +// double* a = layout.Pointer<double>(p.get()); +// +// // Obtain the pointer to the array of ints. +// // Equivalent to `reinterpret_cast<int*>(p.get() + N * 8)`. +// int* b = layout.Pointer<int>(p); +// +// If we are unable to specify sizes of all fields, we can pass as many sizes as +// we can to `Partial()`. In return, it'll allow us to access the fields whose +// locations and sizes can be computed from the provided information. +// `Partial()` comes in handy when the array sizes are embedded into the +// allocation. +// +// // size_t[1] containing N, size_t[1] containing M, double[N], int[M]. +// using L = Layout<size_t, size_t, double, int>; +// +// unsigned char* Allocate(size_t n, size_t m) { +// const L layout(1, 1, n, m); +// unsigned char* p = new unsigned char[layout.AllocSize()]; +// *layout.Pointer<0>(p) = n; +// *layout.Pointer<1>(p) = m; +// return p; +// } +// +// void Use(unsigned char* p) { +// // First, extract N and M. +// // Specify that the first array has only one element. Using `prefix` we +// // can access the first two arrays but not more. +// constexpr auto prefix = L::Partial(1); +// size_t n = *prefix.Pointer<0>(p); +// size_t m = *prefix.Pointer<1>(p); +// +// // Now we can get pointers to the payload. +// const L layout(1, 1, n, m); +// double* a = layout.Pointer<double>(p); +// int* b = layout.Pointer<int>(p); +// } +// +// The layout we used above combines fixed-size with dynamically-sized fields. +// This is quite common. Layout is optimized for this use case and generates +// optimal code. All computations that can be performed at compile time are +// indeed performed at compile time. +// +// Efficiency tip: The order of fields matters. In `Layout<T1, ..., TN>` try to +// ensure that `alignof(T1) >= ... >= alignof(TN)`. This way you'll have no +// padding in between arrays. +// +// You can manually override the alignment of an array by wrapping the type in +// `Aligned<T, N>`. `Layout<..., Aligned<T, N>, ...>` has exactly the same API +// and behavior as `Layout<..., T, ...>` except that the first element of the +// array of `T` is aligned to `N` (the rest of the elements follow without +// padding). `N` cannot be less than `alignof(T)`. +// +// `AllocSize()` and `Pointer()` are the most basic methods for dealing with +// memory layouts. Check out the reference or code below to discover more. +// +// EXAMPLE +// +// // Immutable move-only string with sizeof equal to sizeof(void*). The +// // string size and the characters are kept in the same heap allocation. +// class CompactString { +// public: +// CompactString(const char* s = "") { +// const size_t size = strlen(s); +// // size_t[1] followed by char[size + 1]. +// const L layout(1, size + 1); +// p_.reset(new unsigned char[layout.AllocSize()]); +// // If running under ASAN, mark the padding bytes, if any, to catch +// // memory errors. +// layout.PoisonPadding(p_.get()); +// // Store the size in the allocation. +// *layout.Pointer<size_t>(p_.get()) = size; +// // Store the characters in the allocation. +// memcpy(layout.Pointer<char>(p_.get()), s, size + 1); +// } +// +// size_t size() const { +// // Equivalent to reinterpret_cast<size_t&>(*p). +// return *L::Partial().Pointer<size_t>(p_.get()); +// } +// +// const char* c_str() const { +// // Equivalent to reinterpret_cast<char*>(p.get() + sizeof(size_t)). +// // The argument in Partial(1) specifies that we have size_t[1] in front +// // of the characters. +// return L::Partial(1).Pointer<char>(p_.get()); +// } +// +// private: +// // Our heap allocation contains a size_t followed by an array of chars. +// using L = Layout<size_t, char>; +// std::unique_ptr<unsigned char[]> p_; +// }; +// +// int main() { +// CompactString s = "hello"; +// assert(s.size() == 5); +// assert(strcmp(s.c_str(), "hello") == 0); +// } +// +// DOCUMENTATION +// +// The interface exported by this file consists of: +// - class `Layout<>` and its public members. +// - The public members of class `internal_layout::LayoutImpl<>`. That class +// isn't intended to be used directly, and its name and template parameter +// list are internal implementation details, but the class itself provides +// most of the functionality in this file. See comments on its members for +// detailed documentation. +// +// `Layout<T1,... Tn>::Partial(count1,..., countm)` (where `m` <= `n`) returns a +// `LayoutImpl<>` object. `Layout<T1,..., Tn> layout(count1,..., countn)` +// creates a `Layout` object, which exposes the same functionality by inheriting +// from `LayoutImpl<>`. + +#ifndef ABSL_CONTAINER_INTERNAL_LAYOUT_H_ +#define ABSL_CONTAINER_INTERNAL_LAYOUT_H_ + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> + +#include <ostream> +#include <string> +#include <tuple> +#include <type_traits> +#include <typeinfo> +#include <utility> + +#include "absl/base/config.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" +#include "absl/utility/utility.h" + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +#include <sanitizer/asan_interface.h> +#endif + +#if defined(__GXX_RTTI) +#define ABSL_INTERNAL_HAS_CXA_DEMANGLE +#endif + +#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE +#include <cxxabi.h> +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// A type wrapper that instructs `Layout` to use the specific alignment for the +// array. `Layout<..., Aligned<T, N>, ...>` has exactly the same API +// and behavior as `Layout<..., T, ...>` except that the first element of the +// array of `T` is aligned to `N` (the rest of the elements follow without +// padding). +// +// Requires: `N >= alignof(T)` and `N` is a power of 2. +template <class T, size_t N> +struct Aligned; + +namespace internal_layout { + +template <class T> +struct NotAligned {}; + +template <class T, size_t N> +struct NotAligned<const Aligned<T, N>> { + static_assert(sizeof(T) == 0, "Aligned<T, N> cannot be const-qualified"); +}; + +template <size_t> +using IntToSize = size_t; + +template <class> +using TypeToSize = size_t; + +template <class T> +struct Type : NotAligned<T> { + using type = T; +}; + +template <class T, size_t N> +struct Type<Aligned<T, N>> { + using type = T; +}; + +template <class T> +struct SizeOf : NotAligned<T>, std::integral_constant<size_t, sizeof(T)> {}; + +template <class T, size_t N> +struct SizeOf<Aligned<T, N>> : std::integral_constant<size_t, sizeof(T)> {}; + +// Note: workaround for https://gcc.gnu.org/PR88115 +template <class T> +struct AlignOf : NotAligned<T> { + static constexpr size_t value = alignof(T); +}; + +template <class T, size_t N> +struct AlignOf<Aligned<T, N>> { + static_assert(N % alignof(T) == 0, + "Custom alignment can't be lower than the type's alignment"); + static constexpr size_t value = N; +}; + +// Does `Ts...` contain `T`? +template <class T, class... Ts> +using Contains = absl::disjunction<std::is_same<T, Ts>...>; + +template <class From, class To> +using CopyConst = + typename std::conditional<std::is_const<From>::value, const To, To>::type; + +// Note: We're not qualifying this with absl:: because it doesn't compile under +// MSVC. +template <class T> +using SliceType = Span<T>; + +// This namespace contains no types. It prevents functions defined in it from +// being found by ADL. +namespace adl_barrier { + +template <class Needle, class... Ts> +constexpr size_t Find(Needle, Needle, Ts...) { + static_assert(!Contains<Needle, Ts...>(), "Duplicate element type"); + return 0; +} + +template <class Needle, class T, class... Ts> +constexpr size_t Find(Needle, T, Ts...) { + return adl_barrier::Find(Needle(), Ts()...) + 1; +} + +constexpr bool IsPow2(size_t n) { return !(n & (n - 1)); } + +// Returns `q * m` for the smallest `q` such that `q * m >= n`. +// Requires: `m` is a power of two. It's enforced by IsLegalElementType below. +constexpr size_t Align(size_t n, size_t m) { return (n + m - 1) & ~(m - 1); } + +constexpr size_t Min(size_t a, size_t b) { return b < a ? b : a; } + +constexpr size_t Max(size_t a) { return a; } + +template <class... Ts> +constexpr size_t Max(size_t a, size_t b, Ts... rest) { + return adl_barrier::Max(b < a ? a : b, rest...); +} + +template <class T> +std::string TypeName() { + std::string out; + int status = 0; + char* demangled = nullptr; +#ifdef ABSL_INTERNAL_HAS_CXA_DEMANGLE + demangled = abi::__cxa_demangle(typeid(T).name(), nullptr, nullptr, &status); +#endif + if (status == 0 && demangled != nullptr) { // Demangling succeeded. + absl::StrAppend(&out, "<", demangled, ">"); + free(demangled); + } else { +#if defined(__GXX_RTTI) || defined(_CPPRTTI) + absl::StrAppend(&out, "<", typeid(T).name(), ">"); +#endif + } + return out; +} + +} // namespace adl_barrier + +template <bool C> +using EnableIf = typename std::enable_if<C, int>::type; + +// Can `T` be a template argument of `Layout`? +template <class T> +using IsLegalElementType = std::integral_constant< + bool, !std::is_reference<T>::value && !std::is_volatile<T>::value && + !std::is_reference<typename Type<T>::type>::value && + !std::is_volatile<typename Type<T>::type>::value && + adl_barrier::IsPow2(AlignOf<T>::value)>; + +template <class Elements, class SizeSeq, class OffsetSeq> +class LayoutImpl; + +// Public base class of `Layout` and the result type of `Layout::Partial()`. +// +// `Elements...` contains all template arguments of `Layout` that created this +// instance. +// +// `SizeSeq...` is `[0, NumSizes)` where `NumSizes` is the number of arguments +// passed to `Layout::Partial()` or `Layout::Layout()`. +// +// `OffsetSeq...` is `[0, NumOffsets)` where `NumOffsets` is +// `Min(sizeof...(Elements), NumSizes + 1)` (the number of arrays for which we +// can compute offsets). +template <class... Elements, size_t... SizeSeq, size_t... OffsetSeq> +class LayoutImpl<std::tuple<Elements...>, absl::index_sequence<SizeSeq...>, + absl::index_sequence<OffsetSeq...>> { + private: + static_assert(sizeof...(Elements) > 0, "At least one field is required"); + static_assert(absl::conjunction<IsLegalElementType<Elements>...>::value, + "Invalid element type (see IsLegalElementType)"); + + enum { + NumTypes = sizeof...(Elements), + NumSizes = sizeof...(SizeSeq), + NumOffsets = sizeof...(OffsetSeq), + }; + + // These are guaranteed by `Layout`. + static_assert(NumOffsets == adl_barrier::Min(NumTypes, NumSizes + 1), + "Internal error"); + static_assert(NumTypes > 0, "Internal error"); + + // Returns the index of `T` in `Elements...`. Results in a compilation error + // if `Elements...` doesn't contain exactly one instance of `T`. + template <class T> + static constexpr size_t ElementIndex() { + static_assert(Contains<Type<T>, Type<typename Type<Elements>::type>...>(), + "Type not found"); + return adl_barrier::Find(Type<T>(), + Type<typename Type<Elements>::type>()...); + } + + template <size_t N> + using ElementAlignment = + AlignOf<typename std::tuple_element<N, std::tuple<Elements...>>::type>; + + public: + // Element types of all arrays packed in a tuple. + using ElementTypes = std::tuple<typename Type<Elements>::type...>; + + // Element type of the Nth array. + template <size_t N> + using ElementType = typename std::tuple_element<N, ElementTypes>::type; + + constexpr explicit LayoutImpl(IntToSize<SizeSeq>... sizes) + : size_{sizes...} {} + + // Alignment of the layout, equal to the strictest alignment of all elements. + // All pointers passed to the methods of layout must be aligned to this value. + static constexpr size_t Alignment() { + return adl_barrier::Max(AlignOf<Elements>::value...); + } + + // Offset in bytes of the Nth array. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Offset<0>() == 0); // The ints starts from 0. + // assert(x.Offset<1>() == 16); // The doubles starts from 16. + // + // Requires: `N <= NumSizes && N < sizeof...(Ts)`. + template <size_t N, EnableIf<N == 0> = 0> + constexpr size_t Offset() const { + return 0; + } + + template <size_t N, EnableIf<N != 0> = 0> + constexpr size_t Offset() const { + static_assert(N < NumOffsets, "Index out of bounds"); + return adl_barrier::Align( + Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1], + ElementAlignment<N>::value); + } + + // Offset in bytes of the array with the specified element type. There must + // be exactly one such array and its zero-based index must be at most + // `NumSizes`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Offset<int>() == 0); // The ints starts from 0. + // assert(x.Offset<double>() == 16); // The doubles starts from 16. + template <class T> + constexpr size_t Offset() const { + return Offset<ElementIndex<T>()>(); + } + + // Offsets in bytes of all arrays for which the offsets are known. + constexpr std::array<size_t, NumOffsets> Offsets() const { + return {{Offset<OffsetSeq>()...}}; + } + + // The number of elements in the Nth array. This is the Nth argument of + // `Layout::Partial()` or `Layout::Layout()` (zero-based). + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Size<0>() == 3); + // assert(x.Size<1>() == 4); + // + // Requires: `N < NumSizes`. + template <size_t N> + constexpr size_t Size() const { + static_assert(N < NumSizes, "Index out of bounds"); + return size_[N]; + } + + // The number of elements in the array with the specified element type. + // There must be exactly one such array and its zero-based index must be + // at most `NumSizes`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // assert(x.Size<int>() == 3); + // assert(x.Size<double>() == 4); + template <class T> + constexpr size_t Size() const { + return Size<ElementIndex<T>()>(); + } + + // The number of elements of all arrays for which they are known. + constexpr std::array<size_t, NumSizes> Sizes() const { + return {{Size<SizeSeq>()...}}; + } + + // Pointer to the beginning of the Nth array. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // int* ints = x.Pointer<0>(p); + // double* doubles = x.Pointer<1>(p); + // + // Requires: `N <= NumSizes && N < sizeof...(Ts)`. + // Requires: `p` is aligned to `Alignment()`. + template <size_t N, class Char> + CopyConst<Char, ElementType<N>>* Pointer(Char* p) const { + using C = typename std::remove_const<Char>::type; + static_assert( + std::is_same<C, char>() || std::is_same<C, unsigned char>() || + std::is_same<C, signed char>(), + "The argument must be a pointer to [const] [signed|unsigned] char"); + constexpr size_t alignment = Alignment(); + (void)alignment; + assert(reinterpret_cast<uintptr_t>(p) % alignment == 0); + return reinterpret_cast<CopyConst<Char, ElementType<N>>*>(p + Offset<N>()); + } + + // Pointer to the beginning of the array with the specified element type. + // There must be exactly one such array and its zero-based index must be at + // most `NumSizes`. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // int* ints = x.Pointer<int>(p); + // double* doubles = x.Pointer<double>(p); + // + // Requires: `p` is aligned to `Alignment()`. + template <class T, class Char> + CopyConst<Char, T>* Pointer(Char* p) const { + return Pointer<ElementIndex<T>()>(p); + } + + // Pointers to all arrays for which pointers are known. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // + // int* ints; + // double* doubles; + // std::tie(ints, doubles) = x.Pointers(p); + // + // Requires: `p` is aligned to `Alignment()`. + // + // Note: We're not using ElementType alias here because it does not compile + // under MSVC. + template <class Char> + std::tuple<CopyConst< + Char, typename std::tuple_element<OffsetSeq, ElementTypes>::type>*...> + Pointers(Char* p) const { + return std::tuple<CopyConst<Char, ElementType<OffsetSeq>>*...>( + Pointer<OffsetSeq>(p)...); + } + + // The Nth array. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // Span<int> ints = x.Slice<0>(p); + // Span<double> doubles = x.Slice<1>(p); + // + // Requires: `N < NumSizes`. + // Requires: `p` is aligned to `Alignment()`. + template <size_t N, class Char> + SliceType<CopyConst<Char, ElementType<N>>> Slice(Char* p) const { + return SliceType<CopyConst<Char, ElementType<N>>>(Pointer<N>(p), Size<N>()); + } + + // The array with the specified element type. There must be exactly one + // such array and its zero-based index must be less than `NumSizes`. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // Span<int> ints = x.Slice<int>(p); + // Span<double> doubles = x.Slice<double>(p); + // + // Requires: `p` is aligned to `Alignment()`. + template <class T, class Char> + SliceType<CopyConst<Char, T>> Slice(Char* p) const { + return Slice<ElementIndex<T>()>(p); + } + + // All arrays with known sizes. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; + // + // Span<int> ints; + // Span<double> doubles; + // std::tie(ints, doubles) = x.Slices(p); + // + // Requires: `p` is aligned to `Alignment()`. + // + // Note: We're not using ElementType alias here because it does not compile + // under MSVC. + template <class Char> + std::tuple<SliceType<CopyConst< + Char, typename std::tuple_element<SizeSeq, ElementTypes>::type>>...> + Slices(Char* p) const { + // Workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63875 (fixed + // in 6.1). + (void)p; + return std::tuple<SliceType<CopyConst<Char, ElementType<SizeSeq>>>...>( + Slice<SizeSeq>(p)...); + } + + // The size of the allocation that fits all arrays. + // + // // int[3], 4 bytes of padding, double[4]. + // Layout<int, double> x(3, 4); + // unsigned char* p = new unsigned char[x.AllocSize()]; // 48 bytes + // + // Requires: `NumSizes == sizeof...(Ts)`. + constexpr size_t AllocSize() const { + static_assert(NumTypes == NumSizes, "You must specify sizes of all fields"); + return Offset<NumTypes - 1>() + + SizeOf<ElementType<NumTypes - 1>>() * size_[NumTypes - 1]; + } + + // If built with --config=asan, poisons padding bytes (if any) in the + // allocation. The pointer must point to a memory block at least + // `AllocSize()` bytes in length. + // + // `Char` must be `[const] [signed|unsigned] char`. + // + // Requires: `p` is aligned to `Alignment()`. + template <class Char, size_t N = NumOffsets - 1, EnableIf<N == 0> = 0> + void PoisonPadding(const Char* p) const { + Pointer<0>(p); // verify the requirements on `Char` and `p` + } + + template <class Char, size_t N = NumOffsets - 1, EnableIf<N != 0> = 0> + void PoisonPadding(const Char* p) const { + static_assert(N < NumOffsets, "Index out of bounds"); + (void)p; +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + PoisonPadding<Char, N - 1>(p); + // The `if` is an optimization. It doesn't affect the observable behaviour. + if (ElementAlignment<N - 1>::value % ElementAlignment<N>::value) { + size_t start = + Offset<N - 1>() + SizeOf<ElementType<N - 1>>() * size_[N - 1]; + ASAN_POISON_MEMORY_REGION(p + start, Offset<N>() - start); + } +#endif + } + + // Human-readable description of the memory layout. Useful for debugging. + // Slow. + // + // // char[5], 3 bytes of padding, int[3], 4 bytes of padding, followed + // // by an unknown number of doubles. + // auto x = Layout<char, int, double>::Partial(5, 3); + // assert(x.DebugString() == + // "@0<char>(1)[5]; @8<int>(4)[3]; @24<double>(8)"); + // + // Each field is in the following format: @offset<type>(sizeof)[size] (<type> + // may be missing depending on the target platform). For example, + // @8<int>(4)[3] means that at offset 8 we have an array of ints, where each + // int is 4 bytes, and we have 3 of those ints. The size of the last field may + // be missing (as in the example above). Only fields with known offsets are + // described. Type names may differ across platforms: one compiler might + // produce "unsigned*" where another produces "unsigned int *". + std::string DebugString() const { + const auto offsets = Offsets(); + const size_t sizes[] = {SizeOf<ElementType<OffsetSeq>>()...}; + const std::string types[] = { + adl_barrier::TypeName<ElementType<OffsetSeq>>()...}; + std::string res = absl::StrCat("@0", types[0], "(", sizes[0], ")"); + for (size_t i = 0; i != NumOffsets - 1; ++i) { + absl::StrAppend(&res, "[", size_[i], "]; @", offsets[i + 1], types[i + 1], + "(", sizes[i + 1], ")"); + } + // NumSizes is a constant that may be zero. Some compilers cannot see that + // inside the if statement "size_[NumSizes - 1]" must be valid. + int last = static_cast<int>(NumSizes) - 1; + if (NumTypes == NumSizes && last >= 0) { + absl::StrAppend(&res, "[", size_[last], "]"); + } + return res; + } + + private: + // Arguments of `Layout::Partial()` or `Layout::Layout()`. + size_t size_[NumSizes > 0 ? NumSizes : 1]; +}; + +template <size_t NumSizes, class... Ts> +using LayoutType = LayoutImpl< + std::tuple<Ts...>, absl::make_index_sequence<NumSizes>, + absl::make_index_sequence<adl_barrier::Min(sizeof...(Ts), NumSizes + 1)>>; + +} // namespace internal_layout + +// Descriptor of arrays of various types and sizes laid out in memory one after +// another. See the top of the file for documentation. +// +// Check out the public API of internal_layout::LayoutImpl above. The type is +// internal to the library but its methods are public, and they are inherited +// by `Layout`. +template <class... Ts> +class Layout : public internal_layout::LayoutType<sizeof...(Ts), Ts...> { + public: + static_assert(sizeof...(Ts) > 0, "At least one field is required"); + static_assert( + absl::conjunction<internal_layout::IsLegalElementType<Ts>...>::value, + "Invalid element type (see IsLegalElementType)"); + + // The result type of `Partial()` with `NumSizes` arguments. + template <size_t NumSizes> + using PartialType = internal_layout::LayoutType<NumSizes, Ts...>; + + // `Layout` knows the element types of the arrays we want to lay out in + // memory but not the number of elements in each array. + // `Partial(size1, ..., sizeN)` allows us to specify the latter. The + // resulting immutable object can be used to obtain pointers to the + // individual arrays. + // + // It's allowed to pass fewer array sizes than the number of arrays. E.g., + // if all you need is to the offset of the second array, you only need to + // pass one argument -- the number of elements in the first array. + // + // // int[3] followed by 4 bytes of padding and an unknown number of + // // doubles. + // auto x = Layout<int, double>::Partial(3); + // // doubles start at byte 16. + // assert(x.Offset<1>() == 16); + // + // If you know the number of elements in all arrays, you can still call + // `Partial()` but it's more convenient to use the constructor of `Layout`. + // + // Layout<int, double> x(3, 5); + // + // Note: The sizes of the arrays must be specified in number of elements, + // not in bytes. + // + // Requires: `sizeof...(Sizes) <= sizeof...(Ts)`. + // Requires: all arguments are convertible to `size_t`. + template <class... Sizes> + static constexpr PartialType<sizeof...(Sizes)> Partial(Sizes&&... sizes) { + static_assert(sizeof...(Sizes) <= sizeof...(Ts), ""); + return PartialType<sizeof...(Sizes)>(absl::forward<Sizes>(sizes)...); + } + + // Creates a layout with the sizes of all arrays specified. If you know + // only the sizes of the first N arrays (where N can be zero), you can use + // `Partial()` defined above. The constructor is essentially equivalent to + // calling `Partial()` and passing in all array sizes; the constructor is + // provided as a convenient abbreviation. + // + // Note: The sizes of the arrays must be specified in number of elements, + // not in bytes. + constexpr explicit Layout(internal_layout::TypeToSize<Ts>... sizes) + : internal_layout::LayoutType<sizeof...(Ts), Ts...>(sizes...) {} +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_LAYOUT_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/layout_test.cc b/third_party/abseil_cpp/absl/container/internal/layout_test.cc new file mode 100644 index 000000000000..1d7158ffc0b9 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/layout_test.cc @@ -0,0 +1,1635 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/layout.h" + +// We need ::max_align_t because some libstdc++ versions don't provide +// std::max_align_t +#include <stddef.h> + +#include <cstdint> +#include <memory> +#include <sstream> +#include <type_traits> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::absl::Span; +using ::testing::ElementsAre; + +size_t Distance(const void* from, const void* to) { + ABSL_RAW_CHECK(from <= to, "Distance must be non-negative"); + return static_cast<const char*>(to) - static_cast<const char*>(from); +} + +template <class Expected, class Actual> +Expected Type(Actual val) { + static_assert(std::is_same<Expected, Actual>(), ""); + return val; +} + +// Helper classes to test different size and alignments. +struct alignas(8) Int128 { + uint64_t a, b; + friend bool operator==(Int128 lhs, Int128 rhs) { + return std::tie(lhs.a, lhs.b) == std::tie(rhs.a, rhs.b); + } + + static std::string Name() { + return internal_layout::adl_barrier::TypeName<Int128>(); + } +}; + +// int64_t is *not* 8-byte aligned on all platforms! +struct alignas(8) Int64 { + int64_t a; + friend bool operator==(Int64 lhs, Int64 rhs) { + return lhs.a == rhs.a; + } +}; + +// Properties of types that this test relies on. +static_assert(sizeof(int8_t) == 1, ""); +static_assert(alignof(int8_t) == 1, ""); +static_assert(sizeof(int16_t) == 2, ""); +static_assert(alignof(int16_t) == 2, ""); +static_assert(sizeof(int32_t) == 4, ""); +static_assert(alignof(int32_t) == 4, ""); +static_assert(sizeof(Int64) == 8, ""); +static_assert(alignof(Int64) == 8, ""); +static_assert(sizeof(Int128) == 16, ""); +static_assert(alignof(Int128) == 8, ""); + +template <class Expected, class Actual> +void SameType() { + static_assert(std::is_same<Expected, Actual>(), ""); +} + +TEST(Layout, ElementType) { + { + using L = Layout<int32_t>; + SameType<int32_t, L::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial())::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial(0))::ElementType<0>>(); + } + { + using L = Layout<int32_t, int32_t>; + SameType<int32_t, L::ElementType<0>>(); + SameType<int32_t, L::ElementType<1>>(); + SameType<int32_t, decltype(L::Partial())::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial())::ElementType<1>>(); + SameType<int32_t, decltype(L::Partial(0))::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial(0))::ElementType<1>>(); + } + { + using L = Layout<int8_t, int32_t, Int128>; + SameType<int8_t, L::ElementType<0>>(); + SameType<int32_t, L::ElementType<1>>(); + SameType<Int128, L::ElementType<2>>(); + SameType<int8_t, decltype(L::Partial())::ElementType<0>>(); + SameType<int8_t, decltype(L::Partial(0))::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial(0))::ElementType<1>>(); + SameType<int8_t, decltype(L::Partial(0, 0))::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial(0, 0))::ElementType<1>>(); + SameType<Int128, decltype(L::Partial(0, 0))::ElementType<2>>(); + SameType<int8_t, decltype(L::Partial(0, 0, 0))::ElementType<0>>(); + SameType<int32_t, decltype(L::Partial(0, 0, 0))::ElementType<1>>(); + SameType<Int128, decltype(L::Partial(0, 0, 0))::ElementType<2>>(); + } +} + +TEST(Layout, ElementTypes) { + { + using L = Layout<int32_t>; + SameType<std::tuple<int32_t>, L::ElementTypes>(); + SameType<std::tuple<int32_t>, decltype(L::Partial())::ElementTypes>(); + SameType<std::tuple<int32_t>, decltype(L::Partial(0))::ElementTypes>(); + } + { + using L = Layout<int32_t, int32_t>; + SameType<std::tuple<int32_t, int32_t>, L::ElementTypes>(); + SameType<std::tuple<int32_t, int32_t>, + decltype(L::Partial())::ElementTypes>(); + SameType<std::tuple<int32_t, int32_t>, + decltype(L::Partial(0))::ElementTypes>(); + } + { + using L = Layout<int8_t, int32_t, Int128>; + SameType<std::tuple<int8_t, int32_t, Int128>, L::ElementTypes>(); + SameType<std::tuple<int8_t, int32_t, Int128>, + decltype(L::Partial())::ElementTypes>(); + SameType<std::tuple<int8_t, int32_t, Int128>, + decltype(L::Partial(0))::ElementTypes>(); + SameType<std::tuple<int8_t, int32_t, Int128>, + decltype(L::Partial(0, 0))::ElementTypes>(); + SameType<std::tuple<int8_t, int32_t, Int128>, + decltype(L::Partial(0, 0, 0))::ElementTypes>(); + } +} + +TEST(Layout, OffsetByIndex) { + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial().Offset<0>()); + EXPECT_EQ(0, L::Partial(3).Offset<0>()); + EXPECT_EQ(0, L(3).Offset<0>()); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(0, L::Partial().Offset<0>()); + EXPECT_EQ(0, L::Partial(3).Offset<0>()); + EXPECT_EQ(12, L::Partial(3).Offset<1>()); + EXPECT_EQ(0, L::Partial(3, 5).Offset<0>()); + EXPECT_EQ(12, L::Partial(3, 5).Offset<1>()); + EXPECT_EQ(0, L(3, 5).Offset<0>()); + EXPECT_EQ(12, L(3, 5).Offset<1>()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(0, L::Partial().Offset<0>()); + EXPECT_EQ(0, L::Partial(0).Offset<0>()); + EXPECT_EQ(0, L::Partial(0).Offset<1>()); + EXPECT_EQ(0, L::Partial(1).Offset<0>()); + EXPECT_EQ(4, L::Partial(1).Offset<1>()); + EXPECT_EQ(0, L::Partial(5).Offset<0>()); + EXPECT_EQ(8, L::Partial(5).Offset<1>()); + EXPECT_EQ(0, L::Partial(0, 0).Offset<0>()); + EXPECT_EQ(0, L::Partial(0, 0).Offset<1>()); + EXPECT_EQ(0, L::Partial(0, 0).Offset<2>()); + EXPECT_EQ(0, L::Partial(1, 0).Offset<0>()); + EXPECT_EQ(4, L::Partial(1, 0).Offset<1>()); + EXPECT_EQ(8, L::Partial(1, 0).Offset<2>()); + EXPECT_EQ(0, L::Partial(5, 3).Offset<0>()); + EXPECT_EQ(8, L::Partial(5, 3).Offset<1>()); + EXPECT_EQ(24, L::Partial(5, 3).Offset<2>()); + EXPECT_EQ(0, L::Partial(0, 0, 0).Offset<0>()); + EXPECT_EQ(0, L::Partial(0, 0, 0).Offset<1>()); + EXPECT_EQ(0, L::Partial(0, 0, 0).Offset<2>()); + EXPECT_EQ(0, L::Partial(1, 0, 0).Offset<0>()); + EXPECT_EQ(4, L::Partial(1, 0, 0).Offset<1>()); + EXPECT_EQ(8, L::Partial(1, 0, 0).Offset<2>()); + EXPECT_EQ(0, L::Partial(5, 3, 1).Offset<0>()); + EXPECT_EQ(24, L::Partial(5, 3, 1).Offset<2>()); + EXPECT_EQ(8, L::Partial(5, 3, 1).Offset<1>()); + EXPECT_EQ(0, L(5, 3, 1).Offset<0>()); + EXPECT_EQ(24, L(5, 3, 1).Offset<2>()); + EXPECT_EQ(8, L(5, 3, 1).Offset<1>()); + } +} + +TEST(Layout, OffsetByType) { + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial().Offset<int32_t>()); + EXPECT_EQ(0, L::Partial(3).Offset<int32_t>()); + EXPECT_EQ(0, L(3).Offset<int32_t>()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(0, L::Partial().Offset<int8_t>()); + EXPECT_EQ(0, L::Partial(0).Offset<int8_t>()); + EXPECT_EQ(0, L::Partial(0).Offset<int32_t>()); + EXPECT_EQ(0, L::Partial(1).Offset<int8_t>()); + EXPECT_EQ(4, L::Partial(1).Offset<int32_t>()); + EXPECT_EQ(0, L::Partial(5).Offset<int8_t>()); + EXPECT_EQ(8, L::Partial(5).Offset<int32_t>()); + EXPECT_EQ(0, L::Partial(0, 0).Offset<int8_t>()); + EXPECT_EQ(0, L::Partial(0, 0).Offset<int32_t>()); + EXPECT_EQ(0, L::Partial(0, 0).Offset<Int128>()); + EXPECT_EQ(0, L::Partial(1, 0).Offset<int8_t>()); + EXPECT_EQ(4, L::Partial(1, 0).Offset<int32_t>()); + EXPECT_EQ(8, L::Partial(1, 0).Offset<Int128>()); + EXPECT_EQ(0, L::Partial(5, 3).Offset<int8_t>()); + EXPECT_EQ(8, L::Partial(5, 3).Offset<int32_t>()); + EXPECT_EQ(24, L::Partial(5, 3).Offset<Int128>()); + EXPECT_EQ(0, L::Partial(0, 0, 0).Offset<int8_t>()); + EXPECT_EQ(0, L::Partial(0, 0, 0).Offset<int32_t>()); + EXPECT_EQ(0, L::Partial(0, 0, 0).Offset<Int128>()); + EXPECT_EQ(0, L::Partial(1, 0, 0).Offset<int8_t>()); + EXPECT_EQ(4, L::Partial(1, 0, 0).Offset<int32_t>()); + EXPECT_EQ(8, L::Partial(1, 0, 0).Offset<Int128>()); + EXPECT_EQ(0, L::Partial(5, 3, 1).Offset<int8_t>()); + EXPECT_EQ(24, L::Partial(5, 3, 1).Offset<Int128>()); + EXPECT_EQ(8, L::Partial(5, 3, 1).Offset<int32_t>()); + EXPECT_EQ(0, L(5, 3, 1).Offset<int8_t>()); + EXPECT_EQ(24, L(5, 3, 1).Offset<Int128>()); + EXPECT_EQ(8, L(5, 3, 1).Offset<int32_t>()); + } +} + +TEST(Layout, Offsets) { + { + using L = Layout<int32_t>; + EXPECT_THAT(L::Partial().Offsets(), ElementsAre(0)); + EXPECT_THAT(L::Partial(3).Offsets(), ElementsAre(0)); + EXPECT_THAT(L(3).Offsets(), ElementsAre(0)); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_THAT(L::Partial().Offsets(), ElementsAre(0)); + EXPECT_THAT(L::Partial(3).Offsets(), ElementsAre(0, 12)); + EXPECT_THAT(L::Partial(3, 5).Offsets(), ElementsAre(0, 12)); + EXPECT_THAT(L(3, 5).Offsets(), ElementsAre(0, 12)); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_THAT(L::Partial().Offsets(), ElementsAre(0)); + EXPECT_THAT(L::Partial(1).Offsets(), ElementsAre(0, 4)); + EXPECT_THAT(L::Partial(5).Offsets(), ElementsAre(0, 8)); + EXPECT_THAT(L::Partial(0, 0).Offsets(), ElementsAre(0, 0, 0)); + EXPECT_THAT(L::Partial(1, 0).Offsets(), ElementsAre(0, 4, 8)); + EXPECT_THAT(L::Partial(5, 3).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(L::Partial(0, 0, 0).Offsets(), ElementsAre(0, 0, 0)); + EXPECT_THAT(L::Partial(1, 0, 0).Offsets(), ElementsAre(0, 4, 8)); + EXPECT_THAT(L::Partial(5, 3, 1).Offsets(), ElementsAre(0, 8, 24)); + EXPECT_THAT(L(5, 3, 1).Offsets(), ElementsAre(0, 8, 24)); + } +} + +TEST(Layout, AllocSize) { + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).AllocSize()); + EXPECT_EQ(12, L::Partial(3).AllocSize()); + EXPECT_EQ(12, L(3).AllocSize()); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(32, L::Partial(3, 5).AllocSize()); + EXPECT_EQ(32, L(3, 5).AllocSize()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(0, L::Partial(0, 0, 0).AllocSize()); + EXPECT_EQ(8, L::Partial(1, 0, 0).AllocSize()); + EXPECT_EQ(8, L::Partial(0, 1, 0).AllocSize()); + EXPECT_EQ(16, L::Partial(0, 0, 1).AllocSize()); + EXPECT_EQ(24, L::Partial(1, 1, 1).AllocSize()); + EXPECT_EQ(136, L::Partial(3, 5, 7).AllocSize()); + EXPECT_EQ(136, L(3, 5, 7).AllocSize()); + } +} + +TEST(Layout, SizeByIndex) { + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).Size<0>()); + EXPECT_EQ(3, L::Partial(3).Size<0>()); + EXPECT_EQ(3, L(3).Size<0>()); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(0, L::Partial(0).Size<0>()); + EXPECT_EQ(3, L::Partial(3).Size<0>()); + EXPECT_EQ(3, L::Partial(3, 5).Size<0>()); + EXPECT_EQ(5, L::Partial(3, 5).Size<1>()); + EXPECT_EQ(3, L(3, 5).Size<0>()); + EXPECT_EQ(5, L(3, 5).Size<1>()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(3, L::Partial(3).Size<0>()); + EXPECT_EQ(3, L::Partial(3, 5).Size<0>()); + EXPECT_EQ(5, L::Partial(3, 5).Size<1>()); + EXPECT_EQ(3, L::Partial(3, 5, 7).Size<0>()); + EXPECT_EQ(5, L::Partial(3, 5, 7).Size<1>()); + EXPECT_EQ(7, L::Partial(3, 5, 7).Size<2>()); + EXPECT_EQ(3, L(3, 5, 7).Size<0>()); + EXPECT_EQ(5, L(3, 5, 7).Size<1>()); + EXPECT_EQ(7, L(3, 5, 7).Size<2>()); + } +} + +TEST(Layout, SizeByType) { + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).Size<int32_t>()); + EXPECT_EQ(3, L::Partial(3).Size<int32_t>()); + EXPECT_EQ(3, L(3).Size<int32_t>()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(3, L::Partial(3).Size<int8_t>()); + EXPECT_EQ(3, L::Partial(3, 5).Size<int8_t>()); + EXPECT_EQ(5, L::Partial(3, 5).Size<int32_t>()); + EXPECT_EQ(3, L::Partial(3, 5, 7).Size<int8_t>()); + EXPECT_EQ(5, L::Partial(3, 5, 7).Size<int32_t>()); + EXPECT_EQ(7, L::Partial(3, 5, 7).Size<Int128>()); + EXPECT_EQ(3, L(3, 5, 7).Size<int8_t>()); + EXPECT_EQ(5, L(3, 5, 7).Size<int32_t>()); + EXPECT_EQ(7, L(3, 5, 7).Size<Int128>()); + } +} + +TEST(Layout, Sizes) { + { + using L = Layout<int32_t>; + EXPECT_THAT(L::Partial().Sizes(), ElementsAre()); + EXPECT_THAT(L::Partial(3).Sizes(), ElementsAre(3)); + EXPECT_THAT(L(3).Sizes(), ElementsAre(3)); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_THAT(L::Partial().Sizes(), ElementsAre()); + EXPECT_THAT(L::Partial(3).Sizes(), ElementsAre(3)); + EXPECT_THAT(L::Partial(3, 5).Sizes(), ElementsAre(3, 5)); + EXPECT_THAT(L(3, 5).Sizes(), ElementsAre(3, 5)); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_THAT(L::Partial().Sizes(), ElementsAre()); + EXPECT_THAT(L::Partial(3).Sizes(), ElementsAre(3)); + EXPECT_THAT(L::Partial(3, 5).Sizes(), ElementsAre(3, 5)); + EXPECT_THAT(L::Partial(3, 5, 7).Sizes(), ElementsAre(3, 5, 7)); + EXPECT_THAT(L(3, 5, 7).Sizes(), ElementsAre(3, 5, 7)); + } +} + +TEST(Layout, PointerByIndex) { + alignas(max_align_t) const unsigned char p[100] = {}; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial().Pointer<0>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L(3).Pointer<0>(p)))); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L::Partial().Pointer<0>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<0>(p)))); + EXPECT_EQ(12, + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<1>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int32_t*>(L::Partial(3, 5).Pointer<0>(p)))); + EXPECT_EQ( + 12, Distance(p, Type<const int32_t*>(L::Partial(3, 5).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L(3, 5).Pointer<0>(p)))); + EXPECT_EQ(12, Distance(p, Type<const int32_t*>(L(3, 5).Pointer<1>(p)))); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial().Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(0).Pointer<0>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int32_t*>(L::Partial(0).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(1).Pointer<0>(p)))); + EXPECT_EQ(4, + Distance(p, Type<const int32_t*>(L::Partial(1).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L::Partial(5).Pointer<0>(p)))); + EXPECT_EQ(8, + Distance(p, Type<const int32_t*>(L::Partial(5).Pointer<1>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int8_t*>(L::Partial(0, 0).Pointer<0>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int32_t*>(L::Partial(0, 0).Pointer<1>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const Int128*>(L::Partial(0, 0).Pointer<2>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int8_t*>(L::Partial(1, 0).Pointer<0>(p)))); + EXPECT_EQ( + 4, Distance(p, Type<const int32_t*>(L::Partial(1, 0).Pointer<1>(p)))); + EXPECT_EQ(8, + Distance(p, Type<const Int128*>(L::Partial(1, 0).Pointer<2>(p)))); + EXPECT_EQ(0, + Distance(p, Type<const int8_t*>(L::Partial(5, 3).Pointer<0>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<const int32_t*>(L::Partial(5, 3).Pointer<1>(p)))); + EXPECT_EQ(24, + Distance(p, Type<const Int128*>(L::Partial(5, 3).Pointer<2>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial(0, 0, 0).Pointer<0>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int32_t*>(L::Partial(0, 0, 0).Pointer<1>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const Int128*>(L::Partial(0, 0, 0).Pointer<2>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial(1, 0, 0).Pointer<0>(p)))); + EXPECT_EQ( + 4, + Distance(p, Type<const int32_t*>(L::Partial(1, 0, 0).Pointer<1>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<const Int128*>(L::Partial(1, 0, 0).Pointer<2>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial(5, 3, 1).Pointer<0>(p)))); + EXPECT_EQ( + 24, + Distance(p, Type<const Int128*>(L::Partial(5, 3, 1).Pointer<2>(p)))); + EXPECT_EQ( + 8, + Distance(p, Type<const int32_t*>(L::Partial(5, 3, 1).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>(L(5, 3, 1).Pointer<0>(p)))); + EXPECT_EQ(24, Distance(p, Type<const Int128*>(L(5, 3, 1).Pointer<2>(p)))); + EXPECT_EQ(8, Distance(p, Type<const int32_t*>(L(5, 3, 1).Pointer<1>(p)))); + } +} + +TEST(Layout, PointerByType) { + alignas(max_align_t) const unsigned char p[100] = {}; + { + using L = Layout<int32_t>; + EXPECT_EQ( + 0, Distance(p, Type<const int32_t*>(L::Partial().Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int32_t*>(L::Partial(3).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>(L(3).Pointer<int32_t>(p)))); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial().Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial(0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int32_t*>(L::Partial(0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial(1).Pointer<int8_t>(p)))); + EXPECT_EQ( + 4, + Distance(p, Type<const int32_t*>(L::Partial(1).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<const int8_t*>(L::Partial(5).Pointer<int8_t>(p)))); + EXPECT_EQ( + 8, + Distance(p, Type<const int32_t*>(L::Partial(5).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int8_t*>(L::Partial(0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>( + L::Partial(0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const Int128*>(L::Partial(0, 0).Pointer<Int128>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int8_t*>(L::Partial(1, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(4, Distance(p, Type<const int32_t*>( + L::Partial(1, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 8, + Distance(p, Type<const Int128*>(L::Partial(1, 0).Pointer<Int128>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<const int8_t*>(L::Partial(5, 3).Pointer<int8_t>(p)))); + EXPECT_EQ(8, Distance(p, Type<const int32_t*>( + L::Partial(5, 3).Pointer<int32_t>(p)))); + EXPECT_EQ( + 24, + Distance(p, Type<const Int128*>(L::Partial(5, 3).Pointer<Int128>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>( + L::Partial(0, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int32_t*>( + L::Partial(0, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<const Int128*>( + L::Partial(0, 0, 0).Pointer<Int128>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>( + L::Partial(1, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ(4, Distance(p, Type<const int32_t*>( + L::Partial(1, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(8, Distance(p, Type<const Int128*>( + L::Partial(1, 0, 0).Pointer<Int128>(p)))); + EXPECT_EQ(0, Distance(p, Type<const int8_t*>( + L::Partial(5, 3, 1).Pointer<int8_t>(p)))); + EXPECT_EQ(24, Distance(p, Type<const Int128*>( + L::Partial(5, 3, 1).Pointer<Int128>(p)))); + EXPECT_EQ(8, Distance(p, Type<const int32_t*>( + L::Partial(5, 3, 1).Pointer<int32_t>(p)))); + EXPECT_EQ(24, + Distance(p, Type<const Int128*>(L(5, 3, 1).Pointer<Int128>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<const int32_t*>(L(5, 3, 1).Pointer<int32_t>(p)))); + } +} + +TEST(Layout, MutablePointerByIndex) { + alignas(max_align_t) unsigned char p[100]; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial().Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(3).Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L(3).Pointer<0>(p)))); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial().Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(3).Pointer<0>(p)))); + EXPECT_EQ(12, Distance(p, Type<int32_t*>(L::Partial(3).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(3, 5).Pointer<0>(p)))); + EXPECT_EQ(12, Distance(p, Type<int32_t*>(L::Partial(3, 5).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L(3, 5).Pointer<0>(p)))); + EXPECT_EQ(12, Distance(p, Type<int32_t*>(L(3, 5).Pointer<1>(p)))); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial().Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0).Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(0).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1).Pointer<0>(p)))); + EXPECT_EQ(4, Distance(p, Type<int32_t*>(L::Partial(1).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5).Pointer<0>(p)))); + EXPECT_EQ(8, Distance(p, Type<int32_t*>(L::Partial(5).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0, 0).Pointer<0>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial(0, 0).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<Int128*>(L::Partial(0, 0).Pointer<2>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1, 0).Pointer<0>(p)))); + EXPECT_EQ(4, Distance(p, Type<int32_t*>(L::Partial(1, 0).Pointer<1>(p)))); + EXPECT_EQ(8, Distance(p, Type<Int128*>(L::Partial(1, 0).Pointer<2>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5, 3).Pointer<0>(p)))); + EXPECT_EQ(8, Distance(p, Type<int32_t*>(L::Partial(5, 3).Pointer<1>(p)))); + EXPECT_EQ(24, Distance(p, Type<Int128*>(L::Partial(5, 3).Pointer<2>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0, 0, 0).Pointer<0>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int32_t*>(L::Partial(0, 0, 0).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<Int128*>(L::Partial(0, 0, 0).Pointer<2>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1, 0, 0).Pointer<0>(p)))); + EXPECT_EQ(4, + Distance(p, Type<int32_t*>(L::Partial(1, 0, 0).Pointer<1>(p)))); + EXPECT_EQ(8, Distance(p, Type<Int128*>(L::Partial(1, 0, 0).Pointer<2>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5, 3, 1).Pointer<0>(p)))); + EXPECT_EQ(24, + Distance(p, Type<Int128*>(L::Partial(5, 3, 1).Pointer<2>(p)))); + EXPECT_EQ(8, + Distance(p, Type<int32_t*>(L::Partial(5, 3, 1).Pointer<1>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L(5, 3, 1).Pointer<0>(p)))); + EXPECT_EQ(24, Distance(p, Type<Int128*>(L(5, 3, 1).Pointer<2>(p)))); + EXPECT_EQ(8, Distance(p, Type<int32_t*>(L(5, 3, 1).Pointer<1>(p)))); + } +} + +TEST(Layout, MutablePointerByType) { + alignas(max_align_t) unsigned char p[100]; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L::Partial().Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int32_t*>(L::Partial(3).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<int32_t*>(L(3).Pointer<int32_t>(p)))); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial().Pointer<int8_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(0).Pointer<int8_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int32_t*>(L::Partial(0).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(1).Pointer<int8_t>(p)))); + EXPECT_EQ(4, + Distance(p, Type<int32_t*>(L::Partial(1).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L::Partial(5).Pointer<int8_t>(p)))); + EXPECT_EQ(8, + Distance(p, Type<int32_t*>(L::Partial(5).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int8_t*>(L::Partial(0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int32_t*>(L::Partial(0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(0, + Distance(p, Type<Int128*>(L::Partial(0, 0).Pointer<Int128>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int8_t*>(L::Partial(1, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 4, Distance(p, Type<int32_t*>(L::Partial(1, 0).Pointer<int32_t>(p)))); + EXPECT_EQ(8, + Distance(p, Type<Int128*>(L::Partial(1, 0).Pointer<Int128>(p)))); + EXPECT_EQ(0, + Distance(p, Type<int8_t*>(L::Partial(5, 3).Pointer<int8_t>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<int32_t*>(L::Partial(5, 3).Pointer<int32_t>(p)))); + EXPECT_EQ(24, + Distance(p, Type<Int128*>(L::Partial(5, 3).Pointer<Int128>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int8_t*>(L::Partial(0, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 0, + Distance(p, Type<int32_t*>(L::Partial(0, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<Int128*>(L::Partial(0, 0, 0).Pointer<Int128>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int8_t*>(L::Partial(1, 0, 0).Pointer<int8_t>(p)))); + EXPECT_EQ( + 4, + Distance(p, Type<int32_t*>(L::Partial(1, 0, 0).Pointer<int32_t>(p)))); + EXPECT_EQ( + 8, Distance(p, Type<Int128*>(L::Partial(1, 0, 0).Pointer<Int128>(p)))); + EXPECT_EQ( + 0, Distance(p, Type<int8_t*>(L::Partial(5, 3, 1).Pointer<int8_t>(p)))); + EXPECT_EQ( + 24, Distance(p, Type<Int128*>(L::Partial(5, 3, 1).Pointer<Int128>(p)))); + EXPECT_EQ( + 8, + Distance(p, Type<int32_t*>(L::Partial(5, 3, 1).Pointer<int32_t>(p)))); + EXPECT_EQ(0, Distance(p, Type<int8_t*>(L(5, 3, 1).Pointer<int8_t>(p)))); + EXPECT_EQ(24, Distance(p, Type<Int128*>(L(5, 3, 1).Pointer<Int128>(p)))); + EXPECT_EQ(8, Distance(p, Type<int32_t*>(L(5, 3, 1).Pointer<int32_t>(p)))); + } +} + +TEST(Layout, Pointers) { + alignas(max_align_t) const unsigned char p[100] = {}; + using L = Layout<int8_t, int8_t, Int128>; + { + const auto x = L::Partial(); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p)), + Type<std::tuple<const int8_t*>>(x.Pointers(p))); + } + { + const auto x = L::Partial(1); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p)), + (Type<std::tuple<const int8_t*, const int8_t*>>(x.Pointers(p)))); + } + { + const auto x = L::Partial(1, 2); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } + { + const auto x = L::Partial(1, 2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } + { + const L x(1, 2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<const int8_t*, const int8_t*, const Int128*>>( + x.Pointers(p)))); + } +} + +TEST(Layout, MutablePointers) { + alignas(max_align_t) unsigned char p[100]; + using L = Layout<int8_t, int8_t, Int128>; + { + const auto x = L::Partial(); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p)), + Type<std::tuple<int8_t*>>(x.Pointers(p))); + } + { + const auto x = L::Partial(1); + EXPECT_EQ(std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p)), + (Type<std::tuple<int8_t*, int8_t*>>(x.Pointers(p)))); + } + { + const auto x = L::Partial(1, 2); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<int8_t*, int8_t*, Int128*>>(x.Pointers(p)))); + } + { + const auto x = L::Partial(1, 2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<int8_t*, int8_t*, Int128*>>(x.Pointers(p)))); + } + { + const L x(1, 2, 3); + EXPECT_EQ( + std::make_tuple(x.Pointer<0>(p), x.Pointer<1>(p), x.Pointer<2>(p)), + (Type<std::tuple<int8_t*, int8_t*, Int128*>>(x.Pointers(p)))); + } +} + +TEST(Layout, SliceByIndexSize) { + alignas(max_align_t) const unsigned char p[100] = {}; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).Slice<0>(p).size()); + EXPECT_EQ(3, L::Partial(3).Slice<0>(p).size()); + EXPECT_EQ(3, L(3).Slice<0>(p).size()); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(3, L::Partial(3).Slice<0>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5).Slice<1>(p).size()); + EXPECT_EQ(5, L(3, 5).Slice<1>(p).size()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(3, L::Partial(3).Slice<0>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5).Slice<0>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5).Slice<1>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5, 7).Slice<0>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5, 7).Slice<1>(p).size()); + EXPECT_EQ(7, L::Partial(3, 5, 7).Slice<2>(p).size()); + EXPECT_EQ(3, L(3, 5, 7).Slice<0>(p).size()); + EXPECT_EQ(5, L(3, 5, 7).Slice<1>(p).size()); + EXPECT_EQ(7, L(3, 5, 7).Slice<2>(p).size()); + } +} + +TEST(Layout, SliceByTypeSize) { + alignas(max_align_t) const unsigned char p[100] = {}; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).Slice<int32_t>(p).size()); + EXPECT_EQ(3, L::Partial(3).Slice<int32_t>(p).size()); + EXPECT_EQ(3, L(3).Slice<int32_t>(p).size()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(3, L::Partial(3).Slice<int8_t>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5).Slice<int8_t>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5).Slice<int32_t>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5, 7).Slice<int8_t>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5, 7).Slice<int32_t>(p).size()); + EXPECT_EQ(7, L::Partial(3, 5, 7).Slice<Int128>(p).size()); + EXPECT_EQ(3, L(3, 5, 7).Slice<int8_t>(p).size()); + EXPECT_EQ(5, L(3, 5, 7).Slice<int32_t>(p).size()); + EXPECT_EQ(7, L(3, 5, 7).Slice<Int128>(p).size()); + } +} + +TEST(Layout, MutableSliceByIndexSize) { + alignas(max_align_t) unsigned char p[100]; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).Slice<0>(p).size()); + EXPECT_EQ(3, L::Partial(3).Slice<0>(p).size()); + EXPECT_EQ(3, L(3).Slice<0>(p).size()); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ(3, L::Partial(3).Slice<0>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5).Slice<1>(p).size()); + EXPECT_EQ(5, L(3, 5).Slice<1>(p).size()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(3, L::Partial(3).Slice<0>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5).Slice<0>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5).Slice<1>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5, 7).Slice<0>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5, 7).Slice<1>(p).size()); + EXPECT_EQ(7, L::Partial(3, 5, 7).Slice<2>(p).size()); + EXPECT_EQ(3, L(3, 5, 7).Slice<0>(p).size()); + EXPECT_EQ(5, L(3, 5, 7).Slice<1>(p).size()); + EXPECT_EQ(7, L(3, 5, 7).Slice<2>(p).size()); + } +} + +TEST(Layout, MutableSliceByTypeSize) { + alignas(max_align_t) unsigned char p[100]; + { + using L = Layout<int32_t>; + EXPECT_EQ(0, L::Partial(0).Slice<int32_t>(p).size()); + EXPECT_EQ(3, L::Partial(3).Slice<int32_t>(p).size()); + EXPECT_EQ(3, L(3).Slice<int32_t>(p).size()); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ(3, L::Partial(3).Slice<int8_t>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5).Slice<int8_t>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5).Slice<int32_t>(p).size()); + EXPECT_EQ(3, L::Partial(3, 5, 7).Slice<int8_t>(p).size()); + EXPECT_EQ(5, L::Partial(3, 5, 7).Slice<int32_t>(p).size()); + EXPECT_EQ(7, L::Partial(3, 5, 7).Slice<Int128>(p).size()); + EXPECT_EQ(3, L(3, 5, 7).Slice<int8_t>(p).size()); + EXPECT_EQ(5, L(3, 5, 7).Slice<int32_t>(p).size()); + EXPECT_EQ(7, L(3, 5, 7).Slice<Int128>(p).size()); + } +} + +TEST(Layout, SliceByIndexData) { + alignas(max_align_t) const unsigned char p[100] = {}; + { + using L = Layout<int32_t>; + EXPECT_EQ( + 0, Distance( + p, Type<Span<const int32_t>>(L::Partial(0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<const int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ(0, + Distance(p, Type<Span<const int32_t>>(L(3).Slice<0>(p)).data())); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ( + 0, Distance( + p, Type<Span<const int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<const int32_t>>(L::Partial(3, 5).Slice<0>(p)).data())); + EXPECT_EQ( + 12, + Distance( + p, Type<Span<const int32_t>>(L::Partial(3, 5).Slice<1>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<const int32_t>>(L(3, 5).Slice<0>(p)).data())); + EXPECT_EQ( + 12, Distance(p, Type<Span<const int32_t>>(L(3, 5).Slice<1>(p)).data())); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ( + 0, Distance( + p, Type<Span<const int8_t>>(L::Partial(0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<const int8_t>>(L::Partial(1).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<const int8_t>>(L::Partial(5).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<const int8_t>>(L::Partial(0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<const int32_t>>(L::Partial(0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<const int8_t>>(L::Partial(1, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 4, + Distance( + p, Type<Span<const int32_t>>(L::Partial(1, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<const int8_t>>(L::Partial(5, 3).Slice<0>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, Type<Span<const int32_t>>(L::Partial(5, 3).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int8_t>>(L::Partial(0, 0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int32_t>>(L::Partial(0, 0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const Int128>>(L::Partial(0, 0, 0).Slice<2>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int8_t>>(L::Partial(1, 0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 4, + Distance( + p, + Type<Span<const int32_t>>(L::Partial(1, 0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, + Type<Span<const Int128>>(L::Partial(1, 0, 0).Slice<2>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int8_t>>(L::Partial(5, 3, 1).Slice<0>(p)).data())); + EXPECT_EQ( + 24, + Distance( + p, + Type<Span<const Int128>>(L::Partial(5, 3, 1).Slice<2>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, + Type<Span<const int32_t>>(L::Partial(5, 3, 1).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<const int8_t>>(L(5, 3, 1).Slice<0>(p)).data())); + EXPECT_EQ( + 24, + Distance(p, Type<Span<const Int128>>(L(5, 3, 1).Slice<2>(p)).data())); + EXPECT_EQ( + 8, + Distance(p, Type<Span<const int32_t>>(L(5, 3, 1).Slice<1>(p)).data())); + } +} + +TEST(Layout, SliceByTypeData) { + alignas(max_align_t) const unsigned char p[100] = {}; + { + using L = Layout<int32_t>; + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int32_t>>(L::Partial(0).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int32_t>>(L::Partial(3).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<const int32_t>>(L(3).Slice<int32_t>(p)).data())); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int8_t>>(L::Partial(0).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int8_t>>(L::Partial(1).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<const int8_t>>(L::Partial(5).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<const int8_t>>(L::Partial(0, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int32_t>>( + L::Partial(0, 0).Slice<int32_t>(p)) + .data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<const int8_t>>(L::Partial(1, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(4, Distance(p, Type<Span<const int32_t>>( + L::Partial(1, 0).Slice<int32_t>(p)) + .data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<const int8_t>>(L::Partial(5, 3).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(8, Distance(p, Type<Span<const int32_t>>( + L::Partial(5, 3).Slice<int32_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int8_t>>( + L::Partial(0, 0, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int32_t>>( + L::Partial(0, 0, 0).Slice<int32_t>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const Int128>>( + L::Partial(0, 0, 0).Slice<Int128>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int8_t>>( + L::Partial(1, 0, 0).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(4, Distance(p, Type<Span<const int32_t>>( + L::Partial(1, 0, 0).Slice<int32_t>(p)) + .data())); + EXPECT_EQ(8, Distance(p, Type<Span<const Int128>>( + L::Partial(1, 0, 0).Slice<Int128>(p)) + .data())); + EXPECT_EQ(0, Distance(p, Type<Span<const int8_t>>( + L::Partial(5, 3, 1).Slice<int8_t>(p)) + .data())); + EXPECT_EQ(24, Distance(p, Type<Span<const Int128>>( + L::Partial(5, 3, 1).Slice<Int128>(p)) + .data())); + EXPECT_EQ(8, Distance(p, Type<Span<const int32_t>>( + L::Partial(5, 3, 1).Slice<int32_t>(p)) + .data())); + EXPECT_EQ( + 0, + Distance(p, + Type<Span<const int8_t>>(L(5, 3, 1).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 24, + Distance(p, + Type<Span<const Int128>>(L(5, 3, 1).Slice<Int128>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, Type<Span<const int32_t>>(L(5, 3, 1).Slice<int32_t>(p)).data())); + } +} + +TEST(Layout, MutableSliceByIndexData) { + alignas(max_align_t) unsigned char p[100]; + { + using L = Layout<int32_t>; + EXPECT_EQ( + 0, Distance(p, Type<Span<int32_t>>(L::Partial(0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ(0, Distance(p, Type<Span<int32_t>>(L(3).Slice<0>(p)).data())); + } + { + using L = Layout<int32_t, int32_t>; + EXPECT_EQ( + 0, Distance(p, Type<Span<int32_t>>(L::Partial(3).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int32_t>>(L::Partial(3, 5).Slice<0>(p)).data())); + EXPECT_EQ( + 12, + Distance(p, Type<Span<int32_t>>(L::Partial(3, 5).Slice<1>(p)).data())); + EXPECT_EQ(0, Distance(p, Type<Span<int32_t>>(L(3, 5).Slice<0>(p)).data())); + EXPECT_EQ(12, Distance(p, Type<Span<int32_t>>(L(3, 5).Slice<1>(p)).data())); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ( + 0, Distance(p, Type<Span<int8_t>>(L::Partial(0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int8_t>>(L::Partial(1).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int8_t>>(L::Partial(5).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int32_t>>(L::Partial(0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(1, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 4, + Distance(p, Type<Span<int32_t>>(L::Partial(1, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(5, 3).Slice<0>(p)).data())); + EXPECT_EQ( + 8, + Distance(p, Type<Span<int32_t>>(L::Partial(5, 3).Slice<1>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int8_t>>(L::Partial(0, 0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int32_t>>(L::Partial(0, 0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<Int128>>(L::Partial(0, 0, 0).Slice<2>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int8_t>>(L::Partial(1, 0, 0).Slice<0>(p)).data())); + EXPECT_EQ( + 4, Distance( + p, Type<Span<int32_t>>(L::Partial(1, 0, 0).Slice<1>(p)).data())); + EXPECT_EQ( + 8, Distance( + p, Type<Span<Int128>>(L::Partial(1, 0, 0).Slice<2>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int8_t>>(L::Partial(5, 3, 1).Slice<0>(p)).data())); + EXPECT_EQ( + 24, Distance( + p, Type<Span<Int128>>(L::Partial(5, 3, 1).Slice<2>(p)).data())); + EXPECT_EQ( + 8, Distance( + p, Type<Span<int32_t>>(L::Partial(5, 3, 1).Slice<1>(p)).data())); + EXPECT_EQ(0, + Distance(p, Type<Span<int8_t>>(L(5, 3, 1).Slice<0>(p)).data())); + EXPECT_EQ(24, + Distance(p, Type<Span<Int128>>(L(5, 3, 1).Slice<2>(p)).data())); + EXPECT_EQ(8, + Distance(p, Type<Span<int32_t>>(L(5, 3, 1).Slice<1>(p)).data())); + } +} + +TEST(Layout, MutableSliceByTypeData) { + alignas(max_align_t) unsigned char p[100]; + { + using L = Layout<int32_t>; + EXPECT_EQ( + 0, Distance( + p, Type<Span<int32_t>>(L::Partial(0).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, Distance( + p, Type<Span<int32_t>>(L::Partial(3).Slice<int32_t>(p)).data())); + EXPECT_EQ(0, + Distance(p, Type<Span<int32_t>>(L(3).Slice<int32_t>(p)).data())); + } + { + using L = Layout<int8_t, int32_t, Int128>; + EXPECT_EQ( + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(0).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(1).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, Type<Span<int8_t>>(L::Partial(5).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, + Type<Span<int8_t>>(L::Partial(0, 0).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, Type<Span<int32_t>>(L::Partial(0, 0).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, + Type<Span<int8_t>>(L::Partial(1, 0).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 4, + Distance( + p, Type<Span<int32_t>>(L::Partial(1, 0).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, + Distance(p, + Type<Span<int8_t>>(L::Partial(5, 3).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, Type<Span<int32_t>>(L::Partial(5, 3).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<int8_t>>(L::Partial(0, 0, 0).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<int32_t>>(L::Partial(0, 0, 0).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<Int128>>(L::Partial(0, 0, 0).Slice<Int128>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<int8_t>>(L::Partial(1, 0, 0).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 4, + Distance( + p, + Type<Span<int32_t>>(L::Partial(1, 0, 0).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, + Type<Span<Int128>>(L::Partial(1, 0, 0).Slice<Int128>(p)).data())); + EXPECT_EQ( + 0, + Distance( + p, + Type<Span<int8_t>>(L::Partial(5, 3, 1).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 24, + Distance( + p, + Type<Span<Int128>>(L::Partial(5, 3, 1).Slice<Int128>(p)).data())); + EXPECT_EQ( + 8, + Distance( + p, + Type<Span<int32_t>>(L::Partial(5, 3, 1).Slice<int32_t>(p)).data())); + EXPECT_EQ( + 0, Distance(p, Type<Span<int8_t>>(L(5, 3, 1).Slice<int8_t>(p)).data())); + EXPECT_EQ( + 24, + Distance(p, Type<Span<Int128>>(L(5, 3, 1).Slice<Int128>(p)).data())); + EXPECT_EQ( + 8, + Distance(p, Type<Span<int32_t>>(L(5, 3, 1).Slice<int32_t>(p)).data())); + } +} + +MATCHER_P(IsSameSlice, slice, "") { + return arg.size() == slice.size() && arg.data() == slice.data(); +} + +template <typename... M> +class TupleMatcher { + public: + explicit TupleMatcher(M... matchers) : matchers_(std::move(matchers)...) {} + + template <typename Tuple> + bool MatchAndExplain(const Tuple& p, + testing::MatchResultListener* /* listener */) const { + static_assert(std::tuple_size<Tuple>::value == sizeof...(M), ""); + return MatchAndExplainImpl( + p, absl::make_index_sequence<std::tuple_size<Tuple>::value>{}); + } + + // For the matcher concept. Left empty as we don't really need the diagnostics + // right now. + void DescribeTo(::std::ostream* os) const {} + void DescribeNegationTo(::std::ostream* os) const {} + + private: + template <typename Tuple, size_t... Is> + bool MatchAndExplainImpl(const Tuple& p, absl::index_sequence<Is...>) const { + // Using std::min as a simple variadic "and". + return std::min( + {true, testing::SafeMatcherCast< + const typename std::tuple_element<Is, Tuple>::type&>( + std::get<Is>(matchers_)) + .Matches(std::get<Is>(p))...}); + } + + std::tuple<M...> matchers_; +}; + +template <typename... M> +testing::PolymorphicMatcher<TupleMatcher<M...>> Tuple(M... matchers) { + return testing::MakePolymorphicMatcher( + TupleMatcher<M...>(std::move(matchers)...)); +} + +TEST(Layout, Slices) { + alignas(max_align_t) const unsigned char p[100] = {}; + using L = Layout<int8_t, int8_t, Int128>; + { + const auto x = L::Partial(); + EXPECT_THAT(Type<std::tuple<>>(x.Slices(p)), Tuple()); + } + { + const auto x = L::Partial(1); + EXPECT_THAT(Type<std::tuple<Span<const int8_t>>>(x.Slices(p)), + Tuple(IsSameSlice(x.Slice<0>(p)))); + } + { + const auto x = L::Partial(1, 2); + EXPECT_THAT( + (Type<std::tuple<Span<const int8_t>, Span<const int8_t>>>(x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)))); + } + { + const auto x = L::Partial(1, 2, 3); + EXPECT_THAT((Type<std::tuple<Span<const int8_t>, Span<const int8_t>, + Span<const Int128>>>(x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } + { + const L x(1, 2, 3); + EXPECT_THAT((Type<std::tuple<Span<const int8_t>, Span<const int8_t>, + Span<const Int128>>>(x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } +} + +TEST(Layout, MutableSlices) { + alignas(max_align_t) unsigned char p[100] = {}; + using L = Layout<int8_t, int8_t, Int128>; + { + const auto x = L::Partial(); + EXPECT_THAT(Type<std::tuple<>>(x.Slices(p)), Tuple()); + } + { + const auto x = L::Partial(1); + EXPECT_THAT(Type<std::tuple<Span<int8_t>>>(x.Slices(p)), + Tuple(IsSameSlice(x.Slice<0>(p)))); + } + { + const auto x = L::Partial(1, 2); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>>>(x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)))); + } + { + const auto x = L::Partial(1, 2, 3); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } + { + const L x(1, 2, 3); + EXPECT_THAT((Type<std::tuple<Span<int8_t>, Span<int8_t>, Span<Int128>>>( + x.Slices(p))), + Tuple(IsSameSlice(x.Slice<0>(p)), IsSameSlice(x.Slice<1>(p)), + IsSameSlice(x.Slice<2>(p)))); + } +} + +TEST(Layout, UnalignedTypes) { + constexpr Layout<unsigned char, unsigned char, unsigned char> x(1, 2, 3); + alignas(max_align_t) unsigned char p[x.AllocSize() + 1]; + EXPECT_THAT(x.Pointers(p + 1), Tuple(p + 1, p + 2, p + 4)); +} + +TEST(Layout, CustomAlignment) { + constexpr Layout<unsigned char, Aligned<unsigned char, 8>> x(1, 2); + alignas(max_align_t) unsigned char p[x.AllocSize()]; + EXPECT_EQ(10, x.AllocSize()); + EXPECT_THAT(x.Pointers(p), Tuple(p + 0, p + 8)); +} + +TEST(Layout, OverAligned) { + constexpr size_t M = alignof(max_align_t); + constexpr Layout<unsigned char, Aligned<unsigned char, 2 * M>> x(1, 3); + alignas(2 * M) unsigned char p[x.AllocSize()]; + EXPECT_EQ(2 * M + 3, x.AllocSize()); + EXPECT_THAT(x.Pointers(p), Tuple(p + 0, p + 2 * M)); +} + +TEST(Layout, Alignment) { + static_assert(Layout<int8_t>::Alignment() == 1, ""); + static_assert(Layout<int32_t>::Alignment() == 4, ""); + static_assert(Layout<Int64>::Alignment() == 8, ""); + static_assert(Layout<Aligned<int8_t, 64>>::Alignment() == 64, ""); + static_assert(Layout<int8_t, int32_t, Int64>::Alignment() == 8, ""); + static_assert(Layout<int8_t, Int64, int32_t>::Alignment() == 8, ""); + static_assert(Layout<int32_t, int8_t, Int64>::Alignment() == 8, ""); + static_assert(Layout<int32_t, Int64, int8_t>::Alignment() == 8, ""); + static_assert(Layout<Int64, int8_t, int32_t>::Alignment() == 8, ""); + static_assert(Layout<Int64, int32_t, int8_t>::Alignment() == 8, ""); +} + +TEST(Layout, ConstexprPartial) { + constexpr size_t M = alignof(max_align_t); + constexpr Layout<unsigned char, Aligned<unsigned char, 2 * M>> x(1, 3); + static_assert(x.Partial(1).template Offset<1>() == 2 * M, ""); +} +// [from, to) +struct Region { + size_t from; + size_t to; +}; + +void ExpectRegionPoisoned(const unsigned char* p, size_t n, bool poisoned) { +#ifdef ABSL_HAVE_ADDRESS_SANITIZER + for (size_t i = 0; i != n; ++i) { + EXPECT_EQ(poisoned, __asan_address_is_poisoned(p + i)); + } +#endif +} + +template <size_t N> +void ExpectPoisoned(const unsigned char (&buf)[N], + std::initializer_list<Region> reg) { + size_t prev = 0; + for (const Region& r : reg) { + ExpectRegionPoisoned(buf + prev, r.from - prev, false); + ExpectRegionPoisoned(buf + r.from, r.to - r.from, true); + prev = r.to; + } + ExpectRegionPoisoned(buf + prev, N - prev, false); +} + +TEST(Layout, PoisonPadding) { + using L = Layout<int8_t, Int64, int32_t, Int128>; + + constexpr size_t n = L::Partial(1, 2, 3, 4).AllocSize(); + { + constexpr auto x = L::Partial(); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {}); + } + { + constexpr auto x = L::Partial(1); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}}); + } + { + constexpr auto x = L::Partial(1, 2); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}}); + } + { + constexpr auto x = L::Partial(1, 2, 3); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } + { + constexpr auto x = L::Partial(1, 2, 3, 4); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } + { + constexpr L x(1, 2, 3, 4); + alignas(max_align_t) const unsigned char c[n] = {}; + x.PoisonPadding(c); + EXPECT_EQ(x.Slices(c), x.Slices(c)); + ExpectPoisoned(c, {{1, 8}, {36, 40}}); + } +} + +TEST(Layout, DebugString) { + { + constexpr auto x = Layout<int8_t, int32_t, int8_t, Int128>::Partial(); + EXPECT_EQ("@0<signed char>(1)", x.DebugString()); + } + { + constexpr auto x = Layout<int8_t, int32_t, int8_t, Int128>::Partial(1); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)", x.DebugString()); + } + { + constexpr auto x = Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2); + EXPECT_EQ("@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)", + x.DebugString()); + } + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2, 3); + EXPECT_EQ( + "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " + "@16" + + Int128::Name() + "(16)", + x.DebugString()); + } + { + constexpr auto x = + Layout<int8_t, int32_t, int8_t, Int128>::Partial(1, 2, 3, 4); + EXPECT_EQ( + "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " + "@16" + + Int128::Name() + "(16)[4]", + x.DebugString()); + } + { + constexpr Layout<int8_t, int32_t, int8_t, Int128> x(1, 2, 3, 4); + EXPECT_EQ( + "@0<signed char>(1)[1]; @4<int>(4)[2]; @12<signed char>(1)[3]; " + "@16" + + Int128::Name() + "(16)[4]", + x.DebugString()); + } +} + +TEST(Layout, CharTypes) { + constexpr Layout<int32_t> x(1); + alignas(max_align_t) char c[x.AllocSize()] = {}; + alignas(max_align_t) unsigned char uc[x.AllocSize()] = {}; + alignas(max_align_t) signed char sc[x.AllocSize()] = {}; + alignas(max_align_t) const char cc[x.AllocSize()] = {}; + alignas(max_align_t) const unsigned char cuc[x.AllocSize()] = {}; + alignas(max_align_t) const signed char csc[x.AllocSize()] = {}; + + Type<int32_t*>(x.Pointer<0>(c)); + Type<int32_t*>(x.Pointer<0>(uc)); + Type<int32_t*>(x.Pointer<0>(sc)); + Type<const int32_t*>(x.Pointer<0>(cc)); + Type<const int32_t*>(x.Pointer<0>(cuc)); + Type<const int32_t*>(x.Pointer<0>(csc)); + + Type<int32_t*>(x.Pointer<int32_t>(c)); + Type<int32_t*>(x.Pointer<int32_t>(uc)); + Type<int32_t*>(x.Pointer<int32_t>(sc)); + Type<const int32_t*>(x.Pointer<int32_t>(cc)); + Type<const int32_t*>(x.Pointer<int32_t>(cuc)); + Type<const int32_t*>(x.Pointer<int32_t>(csc)); + + Type<std::tuple<int32_t*>>(x.Pointers(c)); + Type<std::tuple<int32_t*>>(x.Pointers(uc)); + Type<std::tuple<int32_t*>>(x.Pointers(sc)); + Type<std::tuple<const int32_t*>>(x.Pointers(cc)); + Type<std::tuple<const int32_t*>>(x.Pointers(cuc)); + Type<std::tuple<const int32_t*>>(x.Pointers(csc)); + + Type<Span<int32_t>>(x.Slice<0>(c)); + Type<Span<int32_t>>(x.Slice<0>(uc)); + Type<Span<int32_t>>(x.Slice<0>(sc)); + Type<Span<const int32_t>>(x.Slice<0>(cc)); + Type<Span<const int32_t>>(x.Slice<0>(cuc)); + Type<Span<const int32_t>>(x.Slice<0>(csc)); + + Type<std::tuple<Span<int32_t>>>(x.Slices(c)); + Type<std::tuple<Span<int32_t>>>(x.Slices(uc)); + Type<std::tuple<Span<int32_t>>>(x.Slices(sc)); + Type<std::tuple<Span<const int32_t>>>(x.Slices(cc)); + Type<std::tuple<Span<const int32_t>>>(x.Slices(cuc)); + Type<std::tuple<Span<const int32_t>>>(x.Slices(csc)); +} + +TEST(Layout, ConstElementType) { + constexpr Layout<const int32_t> x(1); + alignas(int32_t) char c[x.AllocSize()] = {}; + const char* cc = c; + const int32_t* p = reinterpret_cast<const int32_t*>(cc); + + EXPECT_EQ(alignof(int32_t), x.Alignment()); + + EXPECT_EQ(0, x.Offset<0>()); + EXPECT_EQ(0, x.Offset<const int32_t>()); + + EXPECT_THAT(x.Offsets(), ElementsAre(0)); + + EXPECT_EQ(1, x.Size<0>()); + EXPECT_EQ(1, x.Size<const int32_t>()); + + EXPECT_THAT(x.Sizes(), ElementsAre(1)); + + EXPECT_EQ(sizeof(int32_t), x.AllocSize()); + + EXPECT_EQ(p, Type<const int32_t*>(x.Pointer<0>(c))); + EXPECT_EQ(p, Type<const int32_t*>(x.Pointer<0>(cc))); + + EXPECT_EQ(p, Type<const int32_t*>(x.Pointer<const int32_t>(c))); + EXPECT_EQ(p, Type<const int32_t*>(x.Pointer<const int32_t>(cc))); + + EXPECT_THAT(Type<std::tuple<const int32_t*>>(x.Pointers(c)), Tuple(p)); + EXPECT_THAT(Type<std::tuple<const int32_t*>>(x.Pointers(cc)), Tuple(p)); + + EXPECT_THAT(Type<Span<const int32_t>>(x.Slice<0>(c)), + IsSameSlice(Span<const int32_t>(p, 1))); + EXPECT_THAT(Type<Span<const int32_t>>(x.Slice<0>(cc)), + IsSameSlice(Span<const int32_t>(p, 1))); + + EXPECT_THAT(Type<Span<const int32_t>>(x.Slice<const int32_t>(c)), + IsSameSlice(Span<const int32_t>(p, 1))); + EXPECT_THAT(Type<Span<const int32_t>>(x.Slice<const int32_t>(cc)), + IsSameSlice(Span<const int32_t>(p, 1))); + + EXPECT_THAT(Type<std::tuple<Span<const int32_t>>>(x.Slices(c)), + Tuple(IsSameSlice(Span<const int32_t>(p, 1)))); + EXPECT_THAT(Type<std::tuple<Span<const int32_t>>>(x.Slices(cc)), + Tuple(IsSameSlice(Span<const int32_t>(p, 1)))); +} + +namespace example { + +// Immutable move-only string with sizeof equal to sizeof(void*). The string +// size and the characters are kept in the same heap allocation. +class CompactString { + public: + CompactString(const char* s = "") { // NOLINT + const size_t size = strlen(s); + // size_t[1], followed by char[size + 1]. + // This statement doesn't allocate memory. + const L layout(1, size + 1); + // AllocSize() tells us how much memory we need to allocate for all our + // data. + p_.reset(new unsigned char[layout.AllocSize()]); + // If running under ASAN, mark the padding bytes, if any, to catch memory + // errors. + layout.PoisonPadding(p_.get()); + // Store the size in the allocation. + // Pointer<size_t>() is a synonym for Pointer<0>(). + *layout.Pointer<size_t>(p_.get()) = size; + // Store the characters in the allocation. + memcpy(layout.Pointer<char>(p_.get()), s, size + 1); + } + + size_t size() const { + // Equivalent to reinterpret_cast<size_t&>(*p). + return *L::Partial().Pointer<size_t>(p_.get()); + } + + const char* c_str() const { + // Equivalent to reinterpret_cast<char*>(p.get() + sizeof(size_t)). + // The argument in Partial(1) specifies that we have size_t[1] in front of + // the characters. + return L::Partial(1).Pointer<char>(p_.get()); + } + + private: + // Our heap allocation contains a size_t followed by an array of chars. + using L = Layout<size_t, char>; + std::unique_ptr<unsigned char[]> p_; +}; + +TEST(CompactString, Works) { + CompactString s = "hello"; + EXPECT_EQ(5, s.size()); + EXPECT_STREQ("hello", s.c_str()); +} + +} // namespace example + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/node_hash_policy.h b/third_party/abseil_cpp/absl/container/internal/node_hash_policy.h new file mode 100644 index 000000000000..4617162f0b32 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/node_hash_policy.h @@ -0,0 +1,92 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Adapts a policy for nodes. +// +// The node policy should model: +// +// struct Policy { +// // Returns a new node allocated and constructed using the allocator, using +// // the specified arguments. +// template <class Alloc, class... Args> +// value_type* new_element(Alloc* alloc, Args&&... args) const; +// +// // Destroys and deallocates node using the allocator. +// template <class Alloc> +// void delete_element(Alloc* alloc, value_type* node) const; +// }; +// +// It may also optionally define `value()` and `apply()`. For documentation on +// these, see hash_policy_traits.h. + +#ifndef ABSL_CONTAINER_INTERNAL_NODE_HASH_POLICY_H_ +#define ABSL_CONTAINER_INTERNAL_NODE_HASH_POLICY_H_ + +#include <cassert> +#include <cstddef> +#include <memory> +#include <type_traits> +#include <utility> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class Reference, class Policy> +struct node_hash_policy { + static_assert(std::is_lvalue_reference<Reference>::value, ""); + + using slot_type = typename std::remove_cv< + typename std::remove_reference<Reference>::type>::type*; + + template <class Alloc, class... Args> + static void construct(Alloc* alloc, slot_type* slot, Args&&... args) { + *slot = Policy::new_element(alloc, std::forward<Args>(args)...); + } + + template <class Alloc> + static void destroy(Alloc* alloc, slot_type* slot) { + Policy::delete_element(alloc, *slot); + } + + template <class Alloc> + static void transfer(Alloc*, slot_type* new_slot, slot_type* old_slot) { + *new_slot = *old_slot; + } + + static size_t space_used(const slot_type* slot) { + if (slot == nullptr) return Policy::element_space_used(nullptr); + return Policy::element_space_used(*slot); + } + + static Reference element(slot_type* slot) { return **slot; } + + template <class T, class P = Policy> + static auto value(T* elem) -> decltype(P::value(elem)) { + return P::value(elem); + } + + template <class... Ts, class P = Policy> + static auto apply(Ts&&... ts) -> decltype(P::apply(std::forward<Ts>(ts)...)) { + return P::apply(std::forward<Ts>(ts)...); + } +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_NODE_HASH_POLICY_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/node_hash_policy_test.cc b/third_party/abseil_cpp/absl/container/internal/node_hash_policy_test.cc new file mode 100644 index 000000000000..84aabba96830 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/node_hash_policy_test.cc @@ -0,0 +1,69 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/node_hash_policy.h" + +#include <memory> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_policy_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::testing::Pointee; + +struct Policy : node_hash_policy<int&, Policy> { + using key_type = int; + using init_type = int; + + template <class Alloc> + static int* new_element(Alloc* alloc, int value) { + return new int(value); + } + + template <class Alloc> + static void delete_element(Alloc* alloc, int* elem) { + delete elem; + } +}; + +using NodePolicy = hash_policy_traits<Policy>; + +struct NodeTest : ::testing::Test { + std::allocator<int> alloc; + int n = 53; + int* a = &n; +}; + +TEST_F(NodeTest, ConstructDestroy) { + NodePolicy::construct(&alloc, &a, 42); + EXPECT_THAT(a, Pointee(42)); + NodePolicy::destroy(&alloc, &a); +} + +TEST_F(NodeTest, transfer) { + int s = 42; + int* b = &s; + NodePolicy::transfer(&alloc, &a, &b); + EXPECT_EQ(&s, a); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/raw_hash_map.h b/third_party/abseil_cpp/absl/container/internal/raw_hash_map.h new file mode 100644 index 000000000000..0a02757ddfb4 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/raw_hash_map.h @@ -0,0 +1,197 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_MAP_H_ +#define ABSL_CONTAINER_INTERNAL_RAW_HASH_MAP_H_ + +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/throw_delegate.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> { + // P is Policy. It's passed as a template argument to support maps that have + // incomplete types as values, as in unordered_map<K, IncompleteType>. + // MappedReference<> may be a non-reference type. + template <class P> + using MappedReference = decltype(P::value( + std::addressof(std::declval<typename raw_hash_map::reference>()))); + + // MappedConstReference<> may be a non-reference type. + template <class P> + using MappedConstReference = decltype(P::value( + std::addressof(std::declval<typename raw_hash_map::const_reference>()))); + + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + + public: + using key_type = typename Policy::key_type; + using mapped_type = typename Policy::mapped_type; + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + + static_assert(!std::is_reference<key_type>::value, ""); + // TODO(alkis): remove this assertion and verify that reference mapped_type is + // supported. + static_assert(!std::is_reference<mapped_type>::value, ""); + + using iterator = typename raw_hash_map::raw_hash_set::iterator; + using const_iterator = typename raw_hash_map::raw_hash_set::const_iterator; + + raw_hash_map() {} + using raw_hash_map::raw_hash_set::raw_hash_set; + + // The last two template parameters ensure that both arguments are rvalues + // (lvalue arguments are handled by the overloads below). This is necessary + // for supporting bitfield arguments. + // + // union { int n : 1; }; + // flat_hash_map<int, int> m; + // m.insert_or_assign(n, n); + template <class K = key_type, class V = mapped_type, K* = nullptr, + V* = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, V&& v) { + return insert_or_assign_impl(std::forward<K>(k), std::forward<V>(v)); + } + + template <class K = key_type, class V = mapped_type, K* = nullptr> + std::pair<iterator, bool> insert_or_assign(key_arg<K>&& k, const V& v) { + return insert_or_assign_impl(std::forward<K>(k), v); + } + + template <class K = key_type, class V = mapped_type, V* = nullptr> + std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, V&& v) { + return insert_or_assign_impl(k, std::forward<V>(v)); + } + + template <class K = key_type, class V = mapped_type> + std::pair<iterator, bool> insert_or_assign(const key_arg<K>& k, const V& v) { + return insert_or_assign_impl(k, v); + } + + template <class K = key_type, class V = mapped_type, K* = nullptr, + V* = nullptr> + iterator insert_or_assign(const_iterator, key_arg<K>&& k, V&& v) { + return insert_or_assign(std::forward<K>(k), std::forward<V>(v)).first; + } + + template <class K = key_type, class V = mapped_type, K* = nullptr> + iterator insert_or_assign(const_iterator, key_arg<K>&& k, const V& v) { + return insert_or_assign(std::forward<K>(k), v).first; + } + + template <class K = key_type, class V = mapped_type, V* = nullptr> + iterator insert_or_assign(const_iterator, const key_arg<K>& k, V&& v) { + return insert_or_assign(k, std::forward<V>(v)).first; + } + + template <class K = key_type, class V = mapped_type> + iterator insert_or_assign(const_iterator, const key_arg<K>& k, const V& v) { + return insert_or_assign(k, v).first; + } + + // All `try_emplace()` overloads make the same guarantees regarding rvalue + // arguments as `std::unordered_map::try_emplace()`, namely that these + // functions will not move from rvalue arguments if insertions do not happen. + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0, + K* = nullptr> + std::pair<iterator, bool> try_emplace(key_arg<K>&& k, Args&&... args) { + return try_emplace_impl(std::forward<K>(k), std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, + typename std::enable_if< + !std::is_convertible<K, const_iterator>::value, int>::type = 0> + std::pair<iterator, bool> try_emplace(const key_arg<K>& k, Args&&... args) { + return try_emplace_impl(k, std::forward<Args>(args)...); + } + + template <class K = key_type, class... Args, K* = nullptr> + iterator try_emplace(const_iterator, key_arg<K>&& k, Args&&... args) { + return try_emplace(std::forward<K>(k), std::forward<Args>(args)...).first; + } + + template <class K = key_type, class... Args> + iterator try_emplace(const_iterator, const key_arg<K>& k, Args&&... args) { + return try_emplace(k, std::forward<Args>(args)...).first; + } + + template <class K = key_type, class P = Policy> + MappedReference<P> at(const key_arg<K>& key) { + auto it = this->find(key); + if (it == this->end()) { + base_internal::ThrowStdOutOfRange( + "absl::container_internal::raw_hash_map<>::at"); + } + return Policy::value(&*it); + } + + template <class K = key_type, class P = Policy> + MappedConstReference<P> at(const key_arg<K>& key) const { + auto it = this->find(key); + if (it == this->end()) { + base_internal::ThrowStdOutOfRange( + "absl::container_internal::raw_hash_map<>::at"); + } + return Policy::value(&*it); + } + + template <class K = key_type, class P = Policy, K* = nullptr> + MappedReference<P> operator[](key_arg<K>&& key) { + return Policy::value(&*try_emplace(std::forward<K>(key)).first); + } + + template <class K = key_type, class P = Policy> + MappedReference<P> operator[](const key_arg<K>& key) { + return Policy::value(&*try_emplace(key).first); + } + + private: + template <class K, class V> + std::pair<iterator, bool> insert_or_assign_impl(K&& k, V&& v) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::forward<K>(k), std::forward<V>(v)); + else + Policy::value(&*this->iterator_at(res.first)) = std::forward<V>(v); + return {this->iterator_at(res.first), res.second}; + } + + template <class K = key_type, class... Args> + std::pair<iterator, bool> try_emplace_impl(K&& k, Args&&... args) { + auto res = this->find_or_prepare_insert(k); + if (res.second) + this->emplace_at(res.first, std::piecewise_construct, + std::forward_as_tuple(std::forward<K>(k)), + std::forward_as_tuple(std::forward<Args>(args)...)); + return {this->iterator_at(res.first), res.second}; + } +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_MAP_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/raw_hash_set.cc b/third_party/abseil_cpp/absl/container/internal/raw_hash_set.cc new file mode 100644 index 000000000000..bfef071f29e6 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/raw_hash_set.cc @@ -0,0 +1,61 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/raw_hash_set.h" + +#include <atomic> +#include <cstddef> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +constexpr size_t Group::kWidth; + +// Returns "random" seed. +inline size_t RandomSeed() { +#ifdef ABSL_HAVE_THREAD_LOCAL + static thread_local size_t counter = 0; + size_t value = ++counter; +#else // ABSL_HAVE_THREAD_LOCAL + static std::atomic<size_t> counter(0); + size_t value = counter.fetch_add(1, std::memory_order_relaxed); +#endif // ABSL_HAVE_THREAD_LOCAL + return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); +} + +bool ShouldInsertBackwards(size_t hash, ctrl_t* ctrl) { + // To avoid problems with weak hashes and single bit tests, we use % 13. + // TODO(kfm,sbenza): revisit after we do unconditional mixing + return (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; +} + +void ConvertDeletedToEmptyAndFullToDeleted( + ctrl_t* ctrl, size_t capacity) { + assert(ctrl[capacity] == kSentinel); + assert(IsValidCapacity(capacity)); + for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { + Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); + } + // Copy the cloned ctrl bytes. + std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); + ctrl[capacity] = kSentinel; +} + + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/raw_hash_set.h b/third_party/abseil_cpp/absl/container/internal/raw_hash_set.h new file mode 100644 index 000000000000..02158c4e0886 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/raw_hash_set.h @@ -0,0 +1,1903 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// An open-addressing +// hashtable with quadratic probing. +// +// This is a low level hashtable on top of which different interfaces can be +// implemented, like flat_hash_set, node_hash_set, string_hash_set, etc. +// +// The table interface is similar to that of std::unordered_set. Notable +// differences are that most member functions support heterogeneous keys when +// BOTH the hash and eq functions are marked as transparent. They do so by +// providing a typedef called `is_transparent`. +// +// When heterogeneous lookup is enabled, functions that take key_type act as if +// they have an overload set like: +// +// iterator find(const key_type& key); +// template <class K> +// iterator find(const K& key); +// +// size_type erase(const key_type& key); +// template <class K> +// size_type erase(const K& key); +// +// std::pair<iterator, iterator> equal_range(const key_type& key); +// template <class K> +// std::pair<iterator, iterator> equal_range(const K& key); +// +// When heterogeneous lookup is disabled, only the explicit `key_type` overloads +// exist. +// +// find() also supports passing the hash explicitly: +// +// iterator find(const key_type& key, size_t hash); +// template <class U> +// iterator find(const U& key, size_t hash); +// +// In addition the pointer to element and iterator stability guarantees are +// weaker: all iterators and pointers are invalidated after a new element is +// inserted. +// +// IMPLEMENTATION DETAILS +// +// The table stores elements inline in a slot array. In addition to the slot +// array the table maintains some control state per slot. The extra state is one +// byte per slot and stores empty or deleted marks, or alternatively 7 bits from +// the hash of an occupied slot. The table is split into logical groups of +// slots, like so: +// +// Group 1 Group 2 Group 3 +// +---------------+---------------+---------------+ +// | | | | | | | | | | | | | | | | | | | | | | | | | +// +---------------+---------------+---------------+ +// +// On lookup the hash is split into two parts: +// - H2: 7 bits (those stored in the control bytes) +// - H1: the rest of the bits +// The groups are probed using H1. For each group the slots are matched to H2 in +// parallel. Because H2 is 7 bits (128 states) and the number of slots per group +// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. +// +// On insert, once the right group is found (as in lookup), its slots are +// filled in order. +// +// On erase a slot is cleared. In case the group did not have any empty slots +// before the erase, the erased slot is marked as deleted. +// +// Groups without empty slots (but maybe with deleted slots) extend the probe +// sequence. The probing algorithm is quadratic. Given N the number of groups, +// the probing function for the i'th probe is: +// +// P(0) = H1 % N +// +// P(i) = (P(i - 1) + i) % N +// +// This probing function guarantees that after N probes, all the groups of the +// table will be probed exactly once. + +#ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ +#define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ + +#include <algorithm> +#include <cmath> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/bits.h" +#include "absl/base/internal/endian.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" +#include "absl/container/internal/common.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hash_policy_traits.h" +#include "absl/container/internal/hashtable_debug_hooks.h" +#include "absl/container/internal/hashtablez_sampler.h" +#include "absl/container/internal/have_sse.h" +#include "absl/container/internal/layout.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <typename AllocType> +void SwapAlloc(AllocType& lhs, AllocType& rhs, + std::true_type /* propagate_on_container_swap */) { + using std::swap; + swap(lhs, rhs); +} +template <typename AllocType> +void SwapAlloc(AllocType& /*lhs*/, AllocType& /*rhs*/, + std::false_type /* propagate_on_container_swap */) {} + +template <size_t Width> +class probe_seq { + public: + probe_seq(size_t hash, size_t mask) { + assert(((mask + 1) & mask) == 0 && "not a mask"); + mask_ = mask; + offset_ = hash & mask_; + } + size_t offset() const { return offset_; } + size_t offset(size_t i) const { return (offset_ + i) & mask_; } + + void next() { + index_ += Width; + offset_ += index_; + offset_ &= mask_; + } + // 0-based probe index. The i-th probe in the probe sequence. + size_t index() const { return index_; } + + private: + size_t mask_; + size_t offset_; + size_t index_ = 0; +}; + +template <class ContainerKey, class Hash, class Eq> +struct RequireUsableKey { + template <class PassedKey, class... Args> + std::pair< + decltype(std::declval<const Hash&>()(std::declval<const PassedKey&>())), + decltype(std::declval<const Eq&>()(std::declval<const ContainerKey&>(), + std::declval<const PassedKey&>()))>* + operator()(const PassedKey&, const Args&...) const; +}; + +template <class E, class Policy, class Hash, class Eq, class... Ts> +struct IsDecomposable : std::false_type {}; + +template <class Policy, class Hash, class Eq, class... Ts> +struct IsDecomposable< + absl::void_t<decltype( + Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), + std::declval<Ts>()...))>, + Policy, Hash, Eq, Ts...> : std::true_type {}; + +// TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. +template <class T> +constexpr bool IsNoThrowSwappable(std::true_type = {} /* is_swappable */) { + using std::swap; + return noexcept(swap(std::declval<T&>(), std::declval<T&>())); +} +template <class T> +constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) { + return false; +} + +template <typename T> +int TrailingZeros(T x) { + return sizeof(T) == 8 ? base_internal::CountTrailingZerosNonZero64( + static_cast<uint64_t>(x)) + : base_internal::CountTrailingZerosNonZero32( + static_cast<uint32_t>(x)); +} + +template <typename T> +int LeadingZeros(T x) { + return sizeof(T) == 8 + ? base_internal::CountLeadingZeros64(static_cast<uint64_t>(x)) + : base_internal::CountLeadingZeros32(static_cast<uint32_t>(x)); +} + +// An abstraction over a bitmask. It provides an easy way to iterate through the +// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), +// this is a true bitmask. On non-SSE, platforms the arithematic used to +// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as +// either 0x00 or 0x80. +// +// For example: +// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 +// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 +template <class T, int SignificantBits, int Shift = 0> +class BitMask { + static_assert(std::is_unsigned<T>::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + + public: + // These are useful for unit tests (gunit). + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + explicit BitMask(T mask) : mask_(mask) {} + BitMask& operator++() { + mask_ &= (mask_ - 1); + return *this; + } + explicit operator bool() const { return mask_ != 0; } + int operator*() const { return LowestBitSet(); } + int LowestBitSet() const { + return container_internal::TrailingZeros(mask_) >> Shift; + } + int HighestBitSet() const { + return (sizeof(T) * CHAR_BIT - container_internal::LeadingZeros(mask_) - + 1) >> + Shift; + } + + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } + + int TrailingZeros() const { + return container_internal::TrailingZeros(mask_) >> Shift; + } + + int LeadingZeros() const { + constexpr int total_significant_bits = SignificantBits << Shift; + constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; + return container_internal::LeadingZeros(mask_ << extra_bits) >> Shift; + } + + private: + friend bool operator==(const BitMask& a, const BitMask& b) { + return a.mask_ == b.mask_; + } + friend bool operator!=(const BitMask& a, const BitMask& b) { + return a.mask_ != b.mask_; + } + + T mask_; +}; + +using ctrl_t = signed char; +using h2_t = uint8_t; + +// The values here are selected for maximum performance. See the static asserts +// below for details. +enum Ctrl : ctrl_t { + kEmpty = -128, // 0b10000000 + kDeleted = -2, // 0b11111110 + kSentinel = -1, // 0b11111111 +}; +static_assert( + kEmpty & kDeleted & kSentinel & 0x80, + "Special markers need to have the MSB to make checking for them efficient"); +static_assert(kEmpty < kSentinel && kDeleted < kSentinel, + "kEmpty and kDeleted must be smaller than kSentinel to make the " + "SIMD test of IsEmptyOrDeleted() efficient"); +static_assert(kSentinel == -1, + "kSentinel must be -1 to elide loading it from memory into SIMD " + "registers (pcmpeqd xmm, xmm)"); +static_assert(kEmpty == -128, + "kEmpty must be -128 to make the SIMD check for its " + "existence efficient (psignb xmm, xmm)"); +static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, + "kEmpty and kDeleted must share an unset bit that is not shared " + "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " + "efficient"); +static_assert(kDeleted == -2, + "kDeleted must be -2 to make the implementation of " + "ConvertSpecialToEmptyAndFullToDeleted efficient"); + +// A single block of empty control bytes for tables without any slots allocated. +// This enables removing a branch in the hot path of find(). +inline ctrl_t* EmptyGroup() { + alignas(16) static constexpr ctrl_t empty_group[] = { + kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, + kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; + return const_cast<ctrl_t*>(empty_group); +} + +// Mixes a randomly generated per-process seed with `hash` and `ctrl` to +// randomize insertion order within groups. +bool ShouldInsertBackwards(size_t hash, ctrl_t* ctrl); + +// Returns a hash seed. +// +// The seed consists of the ctrl_ pointer, which adds enough entropy to ensure +// non-determinism of iteration order in most cases. +inline size_t HashSeed(const ctrl_t* ctrl) { + // The low bits of the pointer have little or no entropy because of + // alignment. We shift the pointer to try to use higher entropy bits. A + // good number seems to be 12 bits, because that aligns with page size. + return reinterpret_cast<uintptr_t>(ctrl) >> 12; +} + +inline size_t H1(size_t hash, const ctrl_t* ctrl) { + return (hash >> 7) ^ HashSeed(ctrl); +} +inline ctrl_t H2(size_t hash) { return hash & 0x7F; } + +inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } +inline bool IsFull(ctrl_t c) { return c >= 0; } +inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } +inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } + +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 + +// https://github.com/abseil/abseil-cpp/issues/209 +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 +// _mm_cmpgt_epi8 is broken under GCC with -funsigned-char +// Work around this by using the portable implementation of Group +// when using -funsigned-char under GCC. +inline __m128i _mm_cmpgt_epi8_fixed(__m128i a, __m128i b) { +#if defined(__GNUC__) && !defined(__clang__) + if (std::is_unsigned<char>::value) { + const __m128i mask = _mm_set1_epi8(0x80); + const __m128i diff = _mm_subs_epi8(b, a); + return _mm_cmpeq_epi8(_mm_and_si128(diff, mask), mask); + } +#endif + return _mm_cmpgt_epi8(a, b); +} + +struct GroupSse2Impl { + static constexpr size_t kWidth = 16; // the number of slots per group + + explicit GroupSse2Impl(const ctrl_t* pos) { + ctrl = _mm_loadu_si128(reinterpret_cast<const __m128i*>(pos)); + } + + // Returns a bitmask representing the positions of slots that match hash. + BitMask<uint32_t, kWidth> Match(h2_t hash) const { + auto match = _mm_set1_epi8(hash); + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); + } + + // Returns a bitmask representing the positions of empty slots. + BitMask<uint32_t, kWidth> MatchEmpty() const { +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 + // This only works because kEmpty is -128. + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); +#else + return Match(static_cast<h2_t>(kEmpty)); +#endif + } + + // Returns a bitmask representing the positions of empty or deleted slots. + BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { + auto special = _mm_set1_epi8(kSentinel); + return BitMask<uint32_t, kWidth>( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); + } + + // Returns the number of trailing empty or deleted elements in the group. + uint32_t CountLeadingEmptyOrDeleted() const { + auto special = _mm_set1_epi8(kSentinel); + return TrailingZeros( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1); + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + auto msbs = _mm_set1_epi8(static_cast<char>(-128)); + auto x126 = _mm_set1_epi8(126); +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 + auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); +#else + auto zero = _mm_setzero_si128(); + auto special_mask = _mm_cmpgt_epi8_fixed(zero, ctrl); + auto res = _mm_or_si128(msbs, _mm_andnot_si128(special_mask, x126)); +#endif + _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), res); + } + + __m128i ctrl; +}; +#endif // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 + +struct GroupPortableImpl { + static constexpr size_t kWidth = 8; + + explicit GroupPortableImpl(const ctrl_t* pos) + : ctrl(little_endian::Load64(pos)) {} + + BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { + // For the technique, see: + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + // (Determine if a word has a byte equal to n). + // + // Caveat: there are false positives but: + // - they only occur if there is a real match + // - they never occur on kEmpty, kDeleted, kSentinel + // - they will be handled gracefully by subsequent checks in code + // + // Example: + // v = 0x1716151413121110 + // hash = 0x12 + // retval = (v - lsbs) & ~v & msbs = 0x0000000080800000 + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); + } + + BitMask<uint64_t, kWidth, 3> MatchEmpty() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); + } + + BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; + return (TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3; + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl & msbs; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } + + uint64_t ctrl; +}; + +#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +using Group = GroupSse2Impl; +#else +using Group = GroupPortableImpl; +#endif + +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_set; + +inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } + +// PRECONDITION: +// IsValidCapacity(capacity) +// ctrl[capacity] == kSentinel +// ctrl[i] != kSentinel for all i < capacity +// Applies mapping for every byte in ctrl: +// DELETED -> EMPTY +// EMPTY -> EMPTY +// FULL -> DELETED +void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity); + +// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. +inline size_t NormalizeCapacity(size_t n) { + return n ? ~size_t{} >> LeadingZeros(n) : 1; +} + +// We use 7/8th as maximum load factor. +// For 16-wide groups, that gives an average of two empty slots per group. +inline size_t CapacityToGrowth(size_t capacity) { + assert(IsValidCapacity(capacity)); + // `capacity*7/8` + if (Group::kWidth == 8 && capacity == 7) { + // x-x/8 does not work when x==7. + return 6; + } + return capacity - capacity / 8; +} +// From desired "growth" to a lowerbound of the necessary capacity. +// Might not be a valid one and required NormalizeCapacity(). +inline size_t GrowthToLowerboundCapacity(size_t growth) { + // `growth*8/7` + if (Group::kWidth == 8 && growth == 7) { + // x+(x-1)/7 does not work when x==7. + return 8; + } + return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); +} + +inline void AssertIsFull(ctrl_t* ctrl) { + ABSL_HARDENING_ASSERT((ctrl != nullptr && IsFull(*ctrl)) && + "Invalid operation on iterator. The element might have " + "been erased, or the table might have rehashed."); +} + +inline void AssertIsValid(ctrl_t* ctrl) { + ABSL_HARDENING_ASSERT((ctrl == nullptr || IsFull(*ctrl)) && + "Invalid operation on iterator. The element might have " + "been erased, or the table might have rehashed."); +} + +struct FindInfo { + size_t offset; + size_t probe_length; +}; + +// The representation of the object has two modes: +// - small: For capacities < kWidth-1 +// - large: For the rest. +// +// Differences: +// - In small mode we are able to use the whole capacity. The extra control +// bytes give us at least one "empty" control byte to stop the iteration. +// This is important to make 1 a valid capacity. +// +// - In small mode only the first `capacity()` control bytes after the +// sentinel are valid. The rest contain dummy kEmpty values that do not +// represent a real slot. This is important to take into account on +// find_first_non_full(), where we never try ShouldInsertBackwards() for +// small tables. +inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; } + +inline probe_seq<Group::kWidth> probe(ctrl_t* ctrl, size_t hash, + size_t capacity) { + return probe_seq<Group::kWidth>(H1(hash, ctrl), capacity); +} + +// Probes the raw_hash_set with the probe sequence for hash and returns the +// pointer to the first empty or deleted slot. +// NOTE: this function must work with tables having both kEmpty and kDelete +// in one group. Such tables appears during drop_deletes_without_resize. +// +// This function is very useful when insertions happen and: +// - the input is already a set +// - there are enough slots +// - the element with the hash is not in the table +inline FindInfo find_first_non_full(ctrl_t* ctrl, size_t hash, + size_t capacity) { + auto seq = probe(ctrl, hash, capacity); + while (true) { + Group g{ctrl + seq.offset()}; + auto mask = g.MatchEmptyOrDeleted(); + if (mask) { +#if !defined(NDEBUG) + // We want to add entropy even when ASLR is not enabled. + // In debug build we will randomly insert in either the front or back of + // the group. + // TODO(kfm,sbenza): revisit after we do unconditional mixing + if (!is_small(capacity) && ShouldInsertBackwards(hash, ctrl)) { + return {seq.offset(mask.HighestBitSet()), seq.index()}; + } +#endif + return {seq.offset(mask.LowestBitSet()), seq.index()}; + } + seq.next(); + assert(seq.index() < capacity && "full table!"); + } +} + +// Policy: a policy defines how to perform different operations on +// the slots of the hashtable (see hash_policy_traits.h for the full interface +// of policy). +// +// Hash: a (possibly polymorphic) functor that hashes keys of the hashtable. The +// functor should accept a key and return size_t as hash. For best performance +// it is important that the hash function provides high entropy across all bits +// of the hash. +// +// Eq: a (possibly polymorphic) functor that compares two keys for equality. It +// should accept two (of possibly different type) keys and return a bool: true +// if they are equal, false if they are not. If two keys compare equal, then +// their hash values as defined by Hash MUST be equal. +// +// Allocator: an Allocator +// [https://en.cppreference.com/w/cpp/named_req/Allocator] with which +// the storage of the hashtable will be allocated and the elements will be +// constructed and destroyed. +template <class Policy, class Hash, class Eq, class Alloc> +class raw_hash_set { + using PolicyTraits = hash_policy_traits<Policy>; + using KeyArgImpl = + KeyArg<IsTransparent<Eq>::value && IsTransparent<Hash>::value>; + + public: + using init_type = typename PolicyTraits::init_type; + using key_type = typename PolicyTraits::key_type; + // TODO(sbenza): Hide slot_type as it is an implementation detail. Needs user + // code fixes! + using slot_type = typename PolicyTraits::slot_type; + using allocator_type = Alloc; + using size_type = size_t; + using difference_type = ptrdiff_t; + using hasher = Hash; + using key_equal = Eq; + using policy_type = Policy; + using value_type = typename PolicyTraits::value_type; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename absl::allocator_traits< + allocator_type>::template rebind_traits<value_type>::pointer; + using const_pointer = typename absl::allocator_traits< + allocator_type>::template rebind_traits<value_type>::const_pointer; + + // Alias used for heterogeneous lookup functions. + // `key_arg<K>` evaluates to `K` when the functors are transparent and to + // `key_type` otherwise. It permits template argument deduction on `K` for the + // transparent case. + template <class K> + using key_arg = typename KeyArgImpl::template type<K, key_type>; + + private: + // Give an early error when key_type is not hashable/eq. + auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); + auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); + + using Layout = absl::container_internal::Layout<ctrl_t, slot_type>; + + static Layout MakeLayout(size_t capacity) { + assert(IsValidCapacity(capacity)); + return Layout(capacity + Group::kWidth + 1, capacity); + } + + using AllocTraits = absl::allocator_traits<allocator_type>; + using SlotAlloc = typename absl::allocator_traits< + allocator_type>::template rebind_alloc<slot_type>; + using SlotAllocTraits = typename absl::allocator_traits< + allocator_type>::template rebind_traits<slot_type>; + + static_assert(std::is_lvalue_reference<reference>::value, + "Policy::element() must return a reference"); + + template <typename T> + struct SameAsElementReference + : std::is_same<typename std::remove_cv< + typename std::remove_reference<reference>::type>::type, + typename std::remove_cv< + typename std::remove_reference<T>::type>::type> {}; + + // An enabler for insert(T&&): T must be convertible to init_type or be the + // same as [cv] value_type [ref]. + // Note: we separate SameAsElementReference into its own type to avoid using + // reference unless we need to. MSVC doesn't seem to like it in some + // cases. + template <class T> + using RequiresInsertable = typename std::enable_if< + absl::disjunction<std::is_convertible<T, init_type>, + SameAsElementReference<T>>::value, + int>::type; + + // RequiresNotInit is a workaround for gcc prior to 7.1. + // See https://godbolt.org/g/Y4xsUh. + template <class T> + using RequiresNotInit = + typename std::enable_if<!std::is_same<T, init_type>::value, int>::type; + + template <class... Ts> + using IsDecomposable = IsDecomposable<void, PolicyTraits, Hash, Eq, Ts...>; + + public: + static_assert(std::is_same<pointer, value_type*>::value, + "Allocators with custom pointer types are not supported"); + static_assert(std::is_same<const_pointer, const value_type*>::value, + "Allocators with custom pointer types are not supported"); + + class iterator { + friend class raw_hash_set; + + public: + using iterator_category = std::forward_iterator_tag; + using value_type = typename raw_hash_set::value_type; + using reference = + absl::conditional_t<PolicyTraits::constant_iterators::value, + const value_type&, value_type&>; + using pointer = absl::remove_reference_t<reference>*; + using difference_type = typename raw_hash_set::difference_type; + + iterator() {} + + // PRECONDITION: not an end() iterator. + reference operator*() const { + AssertIsFull(ctrl_); + return PolicyTraits::element(slot_); + } + + // PRECONDITION: not an end() iterator. + pointer operator->() const { return &operator*(); } + + // PRECONDITION: not an end() iterator. + iterator& operator++() { + AssertIsFull(ctrl_); + ++ctrl_; + ++slot_; + skip_empty_or_deleted(); + return *this; + } + // PRECONDITION: not an end() iterator. + iterator operator++(int) { + auto tmp = *this; + ++*this; + return tmp; + } + + friend bool operator==(const iterator& a, const iterator& b) { + AssertIsValid(a.ctrl_); + AssertIsValid(b.ctrl_); + return a.ctrl_ == b.ctrl_; + } + friend bool operator!=(const iterator& a, const iterator& b) { + return !(a == b); + } + + private: + iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) { + // This assumption helps the compiler know that any non-end iterator is + // not equal to any end iterator. + ABSL_INTERNAL_ASSUME(ctrl != nullptr); + } + + void skip_empty_or_deleted() { + while (IsEmptyOrDeleted(*ctrl_)) { + uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); + ctrl_ += shift; + slot_ += shift; + } + if (ABSL_PREDICT_FALSE(*ctrl_ == kSentinel)) ctrl_ = nullptr; + } + + ctrl_t* ctrl_ = nullptr; + // To avoid uninitialized member warnings, put slot_ in an anonymous union. + // The member is not initialized on singleton and end iterators. + union { + slot_type* slot_; + }; + }; + + class const_iterator { + friend class raw_hash_set; + + public: + using iterator_category = typename iterator::iterator_category; + using value_type = typename raw_hash_set::value_type; + using reference = typename raw_hash_set::const_reference; + using pointer = typename raw_hash_set::const_pointer; + using difference_type = typename raw_hash_set::difference_type; + + const_iterator() {} + // Implicit construction from iterator. + const_iterator(iterator i) : inner_(std::move(i)) {} + + reference operator*() const { return *inner_; } + pointer operator->() const { return inner_.operator->(); } + + const_iterator& operator++() { + ++inner_; + return *this; + } + const_iterator operator++(int) { return inner_++; } + + friend bool operator==(const const_iterator& a, const const_iterator& b) { + return a.inner_ == b.inner_; + } + friend bool operator!=(const const_iterator& a, const const_iterator& b) { + return !(a == b); + } + + private: + const_iterator(const ctrl_t* ctrl, const slot_type* slot) + : inner_(const_cast<ctrl_t*>(ctrl), const_cast<slot_type*>(slot)) {} + + iterator inner_; + }; + + using node_type = node_handle<Policy, hash_policy_traits<Policy>, Alloc>; + using insert_return_type = InsertReturnType<iterator, node_type>; + + raw_hash_set() noexcept( + std::is_nothrow_default_constructible<hasher>::value&& + std::is_nothrow_default_constructible<key_equal>::value&& + std::is_nothrow_default_constructible<allocator_type>::value) {} + + explicit raw_hash_set(size_t bucket_count, const hasher& hash = hasher(), + const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : ctrl_(EmptyGroup()), settings_(0, hash, eq, alloc) { + if (bucket_count) { + capacity_ = NormalizeCapacity(bucket_count); + initialize_slots(); + } + } + + raw_hash_set(size_t bucket_count, const hasher& hash, + const allocator_type& alloc) + : raw_hash_set(bucket_count, hash, key_equal(), alloc) {} + + raw_hash_set(size_t bucket_count, const allocator_type& alloc) + : raw_hash_set(bucket_count, hasher(), key_equal(), alloc) {} + + explicit raw_hash_set(const allocator_type& alloc) + : raw_hash_set(0, hasher(), key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_count = 0, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(bucket_count, hash, eq, alloc) { + insert(first, last); + } + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_count, + const hasher& hash, const allocator_type& alloc) + : raw_hash_set(first, last, bucket_count, hash, key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, size_t bucket_count, + const allocator_type& alloc) + : raw_hash_set(first, last, bucket_count, hasher(), key_equal(), alloc) {} + + template <class InputIter> + raw_hash_set(InputIter first, InputIter last, const allocator_type& alloc) + : raw_hash_set(first, last, 0, hasher(), key_equal(), alloc) {} + + // Instead of accepting std::initializer_list<value_type> as the first + // argument like std::unordered_set<value_type> does, we have two overloads + // that accept std::initializer_list<T> and std::initializer_list<init_type>. + // This is advantageous for performance. + // + // // Turns {"abc", "def"} into std::initializer_list<std::string>, then + // // copies the strings into the set. + // std::unordered_set<std::string> s = {"abc", "def"}; + // + // // Turns {"abc", "def"} into std::initializer_list<const char*>, then + // // copies the strings into the set. + // absl::flat_hash_set<std::string> s = {"abc", "def"}; + // + // The same trick is used in insert(). + // + // The enabler is necessary to prevent this constructor from triggering where + // the copy constructor is meant to be called. + // + // absl::flat_hash_set<int> a, b{a}; + // + // RequiresNotInit<T> is a workaround for gcc prior to 7.1. + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_count = 0, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count = 0, + const hasher& hash = hasher(), const key_equal& eq = key_equal(), + const allocator_type& alloc = allocator_type()) + : raw_hash_set(init.begin(), init.end(), bucket_count, hash, eq, alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_count, + const hasher& hash, const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, + const hasher& hash, const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hash, key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, size_t bucket_count, + const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, size_t bucket_count, + const allocator_type& alloc) + : raw_hash_set(init, bucket_count, hasher(), key_equal(), alloc) {} + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<T> = 0> + raw_hash_set(std::initializer_list<T> init, const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(std::initializer_list<init_type> init, + const allocator_type& alloc) + : raw_hash_set(init, 0, hasher(), key_equal(), alloc) {} + + raw_hash_set(const raw_hash_set& that) + : raw_hash_set(that, AllocTraits::select_on_container_copy_construction( + that.alloc_ref())) {} + + raw_hash_set(const raw_hash_set& that, const allocator_type& a) + : raw_hash_set(0, that.hash_ref(), that.eq_ref(), a) { + reserve(that.size()); + // Because the table is guaranteed to be empty, we can do something faster + // than a full `insert`. + for (const auto& v : that) { + const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); + auto target = find_first_non_full(ctrl_, hash, capacity_); + set_ctrl(target.offset, H2(hash)); + emplace_at(target.offset, v); + infoz_.RecordInsert(hash, target.probe_length); + } + size_ = that.size(); + growth_left() -= that.size(); + } + + raw_hash_set(raw_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible<hasher>::value&& + std::is_nothrow_copy_constructible<key_equal>::value&& + std::is_nothrow_copy_constructible<allocator_type>::value) + : ctrl_(absl::exchange(that.ctrl_, EmptyGroup())), + slots_(absl::exchange(that.slots_, nullptr)), + size_(absl::exchange(that.size_, 0)), + capacity_(absl::exchange(that.capacity_, 0)), + infoz_(absl::exchange(that.infoz_, HashtablezInfoHandle())), + // Hash, equality and allocator are copied instead of moved because + // `that` must be left valid. If Hash is std::function<Key>, moving it + // would create a nullptr functor that cannot be called. + settings_(that.settings_) { + // growth_left was copied above, reset the one from `that`. + that.growth_left() = 0; + } + + raw_hash_set(raw_hash_set&& that, const allocator_type& a) + : ctrl_(EmptyGroup()), + slots_(nullptr), + size_(0), + capacity_(0), + settings_(0, that.hash_ref(), that.eq_ref(), a) { + if (a == that.alloc_ref()) { + std::swap(ctrl_, that.ctrl_); + std::swap(slots_, that.slots_); + std::swap(size_, that.size_); + std::swap(capacity_, that.capacity_); + std::swap(growth_left(), that.growth_left()); + std::swap(infoz_, that.infoz_); + } else { + reserve(that.size()); + // Note: this will copy elements of dense_set and unordered_set instead of + // moving them. This can be fixed if it ever becomes an issue. + for (auto& elem : that) insert(std::move(elem)); + } + } + + raw_hash_set& operator=(const raw_hash_set& that) { + raw_hash_set tmp(that, + AllocTraits::propagate_on_container_copy_assignment::value + ? that.alloc_ref() + : alloc_ref()); + swap(tmp); + return *this; + } + + raw_hash_set& operator=(raw_hash_set&& that) noexcept( + absl::allocator_traits<allocator_type>::is_always_equal::value&& + std::is_nothrow_move_assignable<hasher>::value&& + std::is_nothrow_move_assignable<key_equal>::value) { + // TODO(sbenza): We should only use the operations from the noexcept clause + // to make sure we actually adhere to that contract. + return move_assign( + std::move(that), + typename AllocTraits::propagate_on_container_move_assignment()); + } + + ~raw_hash_set() { destroy_slots(); } + + iterator begin() { + auto it = iterator_at(0); + it.skip_empty_or_deleted(); + return it; + } + iterator end() { return {}; } + + const_iterator begin() const { + return const_cast<raw_hash_set*>(this)->begin(); + } + const_iterator end() const { return {}; } + const_iterator cbegin() const { return begin(); } + const_iterator cend() const { return end(); } + + bool empty() const { return !size(); } + size_t size() const { return size_; } + size_t capacity() const { return capacity_; } + size_t max_size() const { return (std::numeric_limits<size_t>::max)(); } + + ABSL_ATTRIBUTE_REINITIALIZES void clear() { + // Iterating over this container is O(bucket_count()). When bucket_count() + // is much greater than size(), iteration becomes prohibitively expensive. + // For clear() it is more important to reuse the allocated array when the + // container is small because allocation takes comparatively long time + // compared to destruction of the elements of the container. So we pick the + // largest bucket_count() threshold for which iteration is still fast and + // past that we simply deallocate the array. + if (capacity_ > 127) { + destroy_slots(); + } else if (capacity_) { + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + size_ = 0; + reset_ctrl(); + reset_growth_left(); + } + assert(empty()); + infoz_.RecordStorageChanged(0, capacity_); + } + + // This overload kicks in when the argument is an rvalue of insertable and + // decomposable type other than init_type. + // + // flat_hash_map<std::string, int> m; + // m.insert(std::make_pair("abc", 42)); + // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc + // bug. + template <class T, RequiresInsertable<T> = 0, + class T2 = T, + typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, + T* = nullptr> + std::pair<iterator, bool> insert(T&& value) { + return emplace(std::forward<T>(value)); + } + + // This overload kicks in when the argument is a bitfield or an lvalue of + // insertable and decomposable type. + // + // union { int n : 1; }; + // flat_hash_set<int> s; + // s.insert(n); + // + // flat_hash_set<std::string> s; + // const char* p = "hello"; + // s.insert(p); + // + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template < + class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + std::pair<iterator, bool> insert(const T& value) { + return emplace(value); + } + + // This overload kicks in when the argument is an rvalue of init_type. Its + // purpose is to handle brace-init-list arguments. + // + // flat_hash_map<std::string, int> s; + // s.insert({"abc", 42}); + std::pair<iterator, bool> insert(init_type&& value) { + return emplace(std::move(value)); + } + + // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc + // bug. + template <class T, RequiresInsertable<T> = 0, class T2 = T, + typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, + T* = nullptr> + iterator insert(const_iterator, T&& value) { + return insert(std::forward<T>(value)).first; + } + + // TODO(romanp): Once we stop supporting gcc 5.1 and below, replace + // RequiresInsertable<T> with RequiresInsertable<const T&>. + // We are hitting this bug: https://godbolt.org/g/1Vht4f. + template < + class T, RequiresInsertable<T> = 0, + typename std::enable_if<IsDecomposable<const T&>::value, int>::type = 0> + iterator insert(const_iterator, const T& value) { + return insert(value).first; + } + + iterator insert(const_iterator, init_type&& value) { + return insert(std::move(value)).first; + } + + template <class InputIt> + void insert(InputIt first, InputIt last) { + for (; first != last; ++first) insert(*first); + } + + template <class T, RequiresNotInit<T> = 0, RequiresInsertable<const T&> = 0> + void insert(std::initializer_list<T> ilist) { + insert(ilist.begin(), ilist.end()); + } + + void insert(std::initializer_list<init_type> ilist) { + insert(ilist.begin(), ilist.end()); + } + + insert_return_type insert(node_type&& node) { + if (!node) return {end(), false, node_type()}; + const auto& elem = PolicyTraits::element(CommonAccess::GetSlot(node)); + auto res = PolicyTraits::apply( + InsertSlot<false>{*this, std::move(*CommonAccess::GetSlot(node))}, + elem); + if (res.second) { + CommonAccess::Reset(&node); + return {res.first, true, node_type()}; + } else { + return {res.first, false, std::move(node)}; + } + } + + iterator insert(const_iterator, node_type&& node) { + auto res = insert(std::move(node)); + node = std::move(res.node); + return res.position; + } + + // This overload kicks in if we can deduce the key from args. This enables us + // to avoid constructing value_type if an entry with the same key already + // exists. + // + // For example: + // + // flat_hash_map<std::string, std::string> m = {{"abc", "def"}}; + // // Creates no std::string copies and makes no heap allocations. + // m.emplace("abc", "xyz"); + template <class... Args, typename std::enable_if< + IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + return PolicyTraits::apply(EmplaceDecomposable{*this}, + std::forward<Args>(args)...); + } + + // This overload kicks in if we cannot deduce the key from args. It constructs + // value_type unconditionally and then either moves it into the table or + // destroys. + template <class... Args, typename std::enable_if< + !IsDecomposable<Args...>::value, int>::type = 0> + std::pair<iterator, bool> emplace(Args&&... args) { + alignas(slot_type) unsigned char raw[sizeof(slot_type)]; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + + PolicyTraits::construct(&alloc_ref(), slot, std::forward<Args>(args)...); + const auto& elem = PolicyTraits::element(slot); + return PolicyTraits::apply(InsertSlot<true>{*this, std::move(*slot)}, elem); + } + + template <class... Args> + iterator emplace_hint(const_iterator, Args&&... args) { + return emplace(std::forward<Args>(args)...).first; + } + + // Extension API: support for lazy emplace. + // + // Looks up key in the table. If found, returns the iterator to the element. + // Otherwise calls `f` with one argument of type `raw_hash_set::constructor`. + // + // `f` must abide by several restrictions: + // - it MUST call `raw_hash_set::constructor` with arguments as if a + // `raw_hash_set::value_type` is constructed, + // - it MUST NOT access the container before the call to + // `raw_hash_set::constructor`, and + // - it MUST NOT erase the lazily emplaced element. + // Doing any of these is undefined behavior. + // + // For example: + // + // std::unordered_set<ArenaString> s; + // // Makes ArenaStr even if "abc" is in the map. + // s.insert(ArenaString(&arena, "abc")); + // + // flat_hash_set<ArenaStr> s; + // // Makes ArenaStr only if "abc" is not in the map. + // s.lazy_emplace("abc", [&](const constructor& ctor) { + // ctor(&arena, "abc"); + // }); + // + // WARNING: This API is currently experimental. If there is a way to implement + // the same thing with the rest of the API, prefer that. + class constructor { + friend class raw_hash_set; + + public: + template <class... Args> + void operator()(Args&&... args) const { + assert(*slot_); + PolicyTraits::construct(alloc_, *slot_, std::forward<Args>(args)...); + *slot_ = nullptr; + } + + private: + constructor(allocator_type* a, slot_type** slot) : alloc_(a), slot_(slot) {} + + allocator_type* alloc_; + slot_type** slot_; + }; + + template <class K = key_type, class F> + iterator lazy_emplace(const key_arg<K>& key, F&& f) { + auto res = find_or_prepare_insert(key); + if (res.second) { + slot_type* slot = slots_ + res.first; + std::forward<F>(f)(constructor(&alloc_ref(), &slot)); + assert(!slot); + } + return iterator_at(res.first); + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.erase("abc"); + // + // flat_hash_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.erase("abc"); + template <class K = key_type> + size_type erase(const key_arg<K>& key) { + auto it = find(key); + if (it == end()) return 0; + erase(it); + return 1; + } + + // Erases the element pointed to by `it`. Unlike `std::unordered_set::erase`, + // this method returns void to reduce algorithmic complexity to O(1). The + // iterator is invalidated, so any increment should be done before calling + // erase. In order to erase while iterating across a map, use the following + // idiom (which also works for standard containers): + // + // for (auto it = m.begin(), end = m.end(); it != end;) { + // // `erase()` will invalidate `it`, so advance `it` first. + // auto copy_it = it++; + // if (<pred>) { + // m.erase(copy_it); + // } + // } + void erase(const_iterator cit) { erase(cit.inner_); } + + // This overload is necessary because otherwise erase<K>(const K&) would be + // a better match if non-const iterator is passed as an argument. + void erase(iterator it) { + AssertIsFull(it.ctrl_); + PolicyTraits::destroy(&alloc_ref(), it.slot_); + erase_meta_only(it); + } + + iterator erase(const_iterator first, const_iterator last) { + while (first != last) { + erase(first++); + } + return last.inner_; + } + + // Moves elements from `src` into `this`. + // If the element already exists in `this`, it is left unmodified in `src`. + template <typename H, typename E> + void merge(raw_hash_set<Policy, H, E, Alloc>& src) { // NOLINT + assert(this != &src); + for (auto it = src.begin(), e = src.end(); it != e;) { + auto next = std::next(it); + if (PolicyTraits::apply(InsertSlot<false>{*this, std::move(*it.slot_)}, + PolicyTraits::element(it.slot_)) + .second) { + src.erase_meta_only(it); + } + it = next; + } + } + + template <typename H, typename E> + void merge(raw_hash_set<Policy, H, E, Alloc>&& src) { + merge(src); + } + + node_type extract(const_iterator position) { + AssertIsFull(position.inner_.ctrl_); + auto node = + CommonAccess::Transfer<node_type>(alloc_ref(), position.inner_.slot_); + erase_meta_only(position); + return node; + } + + template < + class K = key_type, + typename std::enable_if<!std::is_same<K, iterator>::value, int>::type = 0> + node_type extract(const key_arg<K>& key) { + auto it = find(key); + return it == end() ? node_type() : extract(const_iterator{it}); + } + + void swap(raw_hash_set& that) noexcept( + IsNoThrowSwappable<hasher>() && IsNoThrowSwappable<key_equal>() && + IsNoThrowSwappable<allocator_type>( + typename AllocTraits::propagate_on_container_swap{})) { + using std::swap; + swap(ctrl_, that.ctrl_); + swap(slots_, that.slots_); + swap(size_, that.size_); + swap(capacity_, that.capacity_); + swap(growth_left(), that.growth_left()); + swap(hash_ref(), that.hash_ref()); + swap(eq_ref(), that.eq_ref()); + swap(infoz_, that.infoz_); + SwapAlloc(alloc_ref(), that.alloc_ref(), + typename AllocTraits::propagate_on_container_swap{}); + } + + void rehash(size_t n) { + if (n == 0 && capacity_ == 0) return; + if (n == 0 && size_ == 0) { + destroy_slots(); + infoz_.RecordStorageChanged(0, 0); + return; + } + // bitor is a faster way of doing `max` here. We will round up to the next + // power-of-2-minus-1, so bitor is good enough. + auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); + // n == 0 unconditionally rehashes as per the standard. + if (n == 0 || m > capacity_) { + resize(m); + } + } + + void reserve(size_t n) { + size_t m = GrowthToLowerboundCapacity(n); + if (m > capacity_) { + resize(NormalizeCapacity(m)); + } + } + + // Extension API: support for heterogeneous keys. + // + // std::unordered_set<std::string> s; + // // Turns "abc" into std::string. + // s.count("abc"); + // + // ch_set<std::string> s; + // // Uses "abc" directly without copying it into std::string. + // s.count("abc"); + template <class K = key_type> + size_t count(const key_arg<K>& key) const { + return find(key) == end() ? 0 : 1; + } + + // Issues CPU prefetch instructions for the memory needed to find or insert + // a key. Like all lookup functions, this support heterogeneous keys. + // + // NOTE: This is a very low level operation and should not be used without + // specific benchmarks indicating its importance. + template <class K = key_type> + void prefetch(const key_arg<K>& key) const { + (void)key; +#if defined(__GNUC__) + auto seq = probe(ctrl_, hash_ref()(key), capacity_); + __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); + __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); +#endif // __GNUC__ + } + + // The API of find() has two extensions. + // + // 1. The hash can be passed by the user. It must be equal to the hash of the + // key. + // + // 2. The type of the key argument doesn't have to be key_type. This is so + // called heterogeneous key support. + template <class K = key_type> + iterator find(const key_arg<K>& key, size_t hash) { + auto seq = probe(ctrl_, hash, capacity_); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset(i))))) + return iterator_at(seq.offset(i)); + } + if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return end(); + seq.next(); + assert(seq.index() < capacity_ && "full table!"); + } + } + template <class K = key_type> + iterator find(const key_arg<K>& key) { + return find(key, hash_ref()(key)); + } + + template <class K = key_type> + const_iterator find(const key_arg<K>& key, size_t hash) const { + return const_cast<raw_hash_set*>(this)->find(key, hash); + } + template <class K = key_type> + const_iterator find(const key_arg<K>& key) const { + return find(key, hash_ref()(key)); + } + + template <class K = key_type> + bool contains(const key_arg<K>& key) const { + return find(key) != end(); + } + + template <class K = key_type> + std::pair<iterator, iterator> equal_range(const key_arg<K>& key) { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + template <class K = key_type> + std::pair<const_iterator, const_iterator> equal_range( + const key_arg<K>& key) const { + auto it = find(key); + if (it != end()) return {it, std::next(it)}; + return {it, it}; + } + + size_t bucket_count() const { return capacity_; } + float load_factor() const { + return capacity_ ? static_cast<double>(size()) / capacity_ : 0.0; + } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { + // Does nothing. + } + + hasher hash_function() const { return hash_ref(); } + key_equal key_eq() const { return eq_ref(); } + allocator_type get_allocator() const { return alloc_ref(); } + + friend bool operator==(const raw_hash_set& a, const raw_hash_set& b) { + if (a.size() != b.size()) return false; + const raw_hash_set* outer = &a; + const raw_hash_set* inner = &b; + if (outer->capacity() > inner->capacity()) std::swap(outer, inner); + for (const value_type& elem : *outer) + if (!inner->has_element(elem)) return false; + return true; + } + + friend bool operator!=(const raw_hash_set& a, const raw_hash_set& b) { + return !(a == b); + } + + friend void swap(raw_hash_set& a, + raw_hash_set& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); + } + + private: + template <class Container, typename Enabler> + friend struct absl::container_internal::hashtable_debug_internal:: + HashtableDebugAccess; + + struct FindElement { + template <class K, class... Args> + const_iterator operator()(const K& key, Args&&...) const { + return s.find(key); + } + const raw_hash_set& s; + }; + + struct HashElement { + template <class K, class... Args> + size_t operator()(const K& key, Args&&...) const { + return h(key); + } + const hasher& h; + }; + + template <class K1> + struct EqualElement { + template <class K2, class... Args> + bool operator()(const K2& lhs, Args&&...) const { + return eq(lhs, rhs); + } + const K1& rhs; + const key_equal& eq; + }; + + struct EmplaceDecomposable { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&... args) const { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + s.emplace_at(res.first, std::forward<Args>(args)...); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + }; + + template <bool do_destroy> + struct InsertSlot { + template <class K, class... Args> + std::pair<iterator, bool> operator()(const K& key, Args&&...) && { + auto res = s.find_or_prepare_insert(key); + if (res.second) { + PolicyTraits::transfer(&s.alloc_ref(), s.slots_ + res.first, &slot); + } else if (do_destroy) { + PolicyTraits::destroy(&s.alloc_ref(), &slot); + } + return {s.iterator_at(res.first), res.second}; + } + raw_hash_set& s; + // Constructed slot. Either moved into place or destroyed. + slot_type&& slot; + }; + + // "erases" the object from the container, except that it doesn't actually + // destroy the object. It only updates all the metadata of the class. + // This can be used in conjunction with Policy::transfer to move the object to + // another place. + void erase_meta_only(const_iterator it) { + assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); + --size_; + const size_t index = it.inner_.ctrl_ - ctrl_; + const size_t index_before = (index - Group::kWidth) & capacity_; + const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); + const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); + + // We count how many consecutive non empties we have to the right and to the + // left of `it`. If the sum is >= kWidth then there is at least one probe + // window that might have seen a full group. + bool was_never_full = + empty_before && empty_after && + static_cast<size_t>(empty_after.TrailingZeros() + + empty_before.LeadingZeros()) < Group::kWidth; + + set_ctrl(index, was_never_full ? kEmpty : kDeleted); + growth_left() += was_never_full; + infoz_.RecordErase(); + } + + void initialize_slots() { + assert(capacity_); + // Folks with custom allocators often make unwarranted assumptions about the + // behavior of their classes vis-a-vis trivial destructability and what + // calls they will or wont make. Avoid sampling for people with custom + // allocators to get us out of this mess. This is not a hard guarantee but + // a workaround while we plan the exact guarantee we want to provide. + // + // People are often sloppy with the exact type of their allocator (sometimes + // it has an extra const or is missing the pair, but rebinds made it work + // anyway). To avoid the ambiguity, we work off SlotAlloc which we have + // bound more carefully. + if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && + slots_ == nullptr) { + infoz_ = Sample(); + } + + auto layout = MakeLayout(capacity_); + char* mem = static_cast<char*>( + Allocate<Layout::Alignment()>(&alloc_ref(), layout.AllocSize())); + ctrl_ = reinterpret_cast<ctrl_t*>(layout.template Pointer<0>(mem)); + slots_ = layout.template Pointer<1>(mem); + reset_ctrl(); + reset_growth_left(); + infoz_.RecordStorageChanged(size_, capacity_); + } + + void destroy_slots() { + if (!capacity_) return; + for (size_t i = 0; i != capacity_; ++i) { + if (IsFull(ctrl_[i])) { + PolicyTraits::destroy(&alloc_ref(), slots_ + i); + } + } + auto layout = MakeLayout(capacity_); + // Unpoison before returning the memory to the allocator. + SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize()); + ctrl_ = EmptyGroup(); + slots_ = nullptr; + size_ = 0; + capacity_ = 0; + growth_left() = 0; + } + + void resize(size_t new_capacity) { + assert(IsValidCapacity(new_capacity)); + auto* old_ctrl = ctrl_; + auto* old_slots = slots_; + const size_t old_capacity = capacity_; + capacity_ = new_capacity; + initialize_slots(); + + size_t total_probe_length = 0; + for (size_t i = 0; i != old_capacity; ++i) { + if (IsFull(old_ctrl[i])) { + size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(old_slots + i)); + auto target = find_first_non_full(ctrl_, hash, capacity_); + size_t new_i = target.offset; + total_probe_length += target.probe_length; + set_ctrl(new_i, H2(hash)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); + } + } + if (old_capacity) { + SanitizerUnpoisonMemoryRegion(old_slots, + sizeof(slot_type) * old_capacity); + auto layout = MakeLayout(old_capacity); + Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl, + layout.AllocSize()); + } + infoz_.RecordRehash(total_probe_length); + } + + void drop_deletes_without_resize() ABSL_ATTRIBUTE_NOINLINE { + assert(IsValidCapacity(capacity_)); + assert(!is_small(capacity_)); + // Algorithm: + // - mark all DELETED slots as EMPTY + // - mark all FULL slots as DELETED + // - for each slot marked as DELETED + // hash = Hash(element) + // target = find_first_non_full(hash) + // if target is in the same group + // mark slot as FULL + // else if target is EMPTY + // transfer element to target + // mark slot as EMPTY + // mark target as FULL + // else if target is DELETED + // swap current element with target element + // mark target as FULL + // repeat procedure for current slot with moved from element (target) + ConvertDeletedToEmptyAndFullToDeleted(ctrl_, capacity_); + alignas(slot_type) unsigned char raw[sizeof(slot_type)]; + size_t total_probe_length = 0; + slot_type* slot = reinterpret_cast<slot_type*>(&raw); + for (size_t i = 0; i != capacity_; ++i) { + if (!IsDeleted(ctrl_[i])) continue; + size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, + PolicyTraits::element(slots_ + i)); + auto target = find_first_non_full(ctrl_, hash, capacity_); + size_t new_i = target.offset; + total_probe_length += target.probe_length; + + // Verify if the old and new i fall within the same group wrt the hash. + // If they do, we don't need to move the object as it falls already in the + // best probe we can. + const auto probe_index = [&](size_t pos) { + return ((pos - probe(ctrl_, hash, capacity_).offset()) & capacity_) / + Group::kWidth; + }; + + // Element doesn't move. + if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { + set_ctrl(i, H2(hash)); + continue; + } + if (IsEmpty(ctrl_[new_i])) { + // Transfer element to the empty spot. + // set_ctrl poisons/unpoisons the slots so we have to call it at the + // right time. + set_ctrl(new_i, H2(hash)); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); + set_ctrl(i, kEmpty); + } else { + assert(IsDeleted(ctrl_[new_i])); + set_ctrl(new_i, H2(hash)); + // Until we are done rehashing, DELETED marks previously FULL slots. + // Swap i and new_i elements. + PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); + PolicyTraits::transfer(&alloc_ref(), slots_ + i, slots_ + new_i); + PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slot); + --i; // repeat + } + } + reset_growth_left(); + infoz_.RecordRehash(total_probe_length); + } + + void rehash_and_grow_if_necessary() { + if (capacity_ == 0) { + resize(1); + } else if (size() <= CapacityToGrowth(capacity()) / 2) { + // Squash DELETED without growing if there is enough capacity. + drop_deletes_without_resize(); + } else { + // Otherwise grow the container. + resize(capacity_ * 2 + 1); + } + } + + bool has_element(const value_type& elem) const { + size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, elem); + auto seq = probe(ctrl_, hash, capacity_); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset(i)) == + elem)) + return true; + } + if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return false; + seq.next(); + assert(seq.index() < capacity_ && "full table!"); + } + return false; + } + + // TODO(alkis): Optimize this assuming *this and that don't overlap. + raw_hash_set& move_assign(raw_hash_set&& that, std::true_type) { + raw_hash_set tmp(std::move(that)); + swap(tmp); + return *this; + } + raw_hash_set& move_assign(raw_hash_set&& that, std::false_type) { + raw_hash_set tmp(std::move(that), alloc_ref()); + swap(tmp); + return *this; + } + + protected: + template <class K> + std::pair<size_t, bool> find_or_prepare_insert(const K& key) { + auto hash = hash_ref()(key); + auto seq = probe(ctrl_, hash, capacity_); + while (true) { + Group g{ctrl_ + seq.offset()}; + for (int i : g.Match(H2(hash))) { + if (ABSL_PREDICT_TRUE(PolicyTraits::apply( + EqualElement<K>{key, eq_ref()}, + PolicyTraits::element(slots_ + seq.offset(i))))) + return {seq.offset(i), false}; + } + if (ABSL_PREDICT_TRUE(g.MatchEmpty())) break; + seq.next(); + assert(seq.index() < capacity_ && "full table!"); + } + return {prepare_insert(hash), true}; + } + + size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { + auto target = find_first_non_full(ctrl_, hash, capacity_); + if (ABSL_PREDICT_FALSE(growth_left() == 0 && + !IsDeleted(ctrl_[target.offset]))) { + rehash_and_grow_if_necessary(); + target = find_first_non_full(ctrl_, hash, capacity_); + } + ++size_; + growth_left() -= IsEmpty(ctrl_[target.offset]); + set_ctrl(target.offset, H2(hash)); + infoz_.RecordInsert(hash, target.probe_length); + return target.offset; + } + + // Constructs the value in the space pointed by the iterator. This only works + // after an unsuccessful find_or_prepare_insert() and before any other + // modifications happen in the raw_hash_set. + // + // PRECONDITION: i is an index returned from find_or_prepare_insert(k), where + // k is the key decomposed from `forward<Args>(args)...`, and the bool + // returned by find_or_prepare_insert(k) was true. + // POSTCONDITION: *m.iterator_at(i) == value_type(forward<Args>(args)...). + template <class... Args> + void emplace_at(size_t i, Args&&... args) { + PolicyTraits::construct(&alloc_ref(), slots_ + i, + std::forward<Args>(args)...); + + assert(PolicyTraits::apply(FindElement{*this}, *iterator_at(i)) == + iterator_at(i) && + "constructed value does not match the lookup key"); + } + + iterator iterator_at(size_t i) { return {ctrl_ + i, slots_ + i}; } + const_iterator iterator_at(size_t i) const { return {ctrl_ + i, slots_ + i}; } + + private: + friend struct RawHashSetTestOnlyAccess; + + // Reset all ctrl bytes back to kEmpty, except the sentinel. + void reset_ctrl() { + std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); + ctrl_[capacity_] = kSentinel; + SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); + } + + void reset_growth_left() { + growth_left() = CapacityToGrowth(capacity()) - size_; + } + + // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at + // the end too. + void set_ctrl(size_t i, ctrl_t h) { + assert(i < capacity_); + + if (IsFull(h)) { + SanitizerUnpoisonObject(slots_ + i); + } else { + SanitizerPoisonObject(slots_ + i); + } + + ctrl_[i] = h; + ctrl_[((i - Group::kWidth) & capacity_) + 1 + + ((Group::kWidth - 1) & capacity_)] = h; + } + + size_t& growth_left() { return settings_.template get<0>(); } + + hasher& hash_ref() { return settings_.template get<1>(); } + const hasher& hash_ref() const { return settings_.template get<1>(); } + key_equal& eq_ref() { return settings_.template get<2>(); } + const key_equal& eq_ref() const { return settings_.template get<2>(); } + allocator_type& alloc_ref() { return settings_.template get<3>(); } + const allocator_type& alloc_ref() const { + return settings_.template get<3>(); + } + + // TODO(alkis): Investigate removing some of these fields: + // - ctrl/slots can be derived from each other + // - size can be moved into the slot array + ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] + slot_type* slots_ = nullptr; // [capacity * slot_type] + size_t size_ = 0; // number of full slots + size_t capacity_ = 0; // total number of slots + HashtablezInfoHandle infoz_; + absl::container_internal::CompressedTuple<size_t /* growth_left */, hasher, + key_equal, allocator_type> + settings_{0, hasher{}, key_equal{}, allocator_type{}}; +}; + +// Erases all elements that satisfy the predicate `pred` from the container `c`. +template <typename P, typename H, typename E, typename A, typename Predicate> +void EraseIf(Predicate pred, raw_hash_set<P, H, E, A>* c) { + for (auto it = c->begin(), last = c->end(); it != last;) { + auto copy_it = it++; + if (pred(*copy_it)) { + c->erase(copy_it); + } + } +} + +namespace hashtable_debug_internal { +template <typename Set> +struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { + using Traits = typename Set::PolicyTraits; + using Slot = typename Traits::slot_type; + + static size_t GetNumProbes(const Set& set, + const typename Set::key_type& key) { + size_t num_probes = 0; + size_t hash = set.hash_ref()(key); + auto seq = probe(set.ctrl_, hash, set.capacity_); + while (true) { + container_internal::Group g{set.ctrl_ + seq.offset()}; + for (int i : g.Match(container_internal::H2(hash))) { + if (Traits::apply( + typename Set::template EqualElement<typename Set::key_type>{ + key, set.eq_ref()}, + Traits::element(set.slots_ + seq.offset(i)))) + return num_probes; + ++num_probes; + } + if (g.MatchEmpty()) return num_probes; + seq.next(); + ++num_probes; + } + } + + static size_t AllocatedByteSize(const Set& c) { + size_t capacity = c.capacity_; + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(capacity); + size_t m = layout.AllocSize(); + + size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * c.size(); + } else { + for (size_t i = 0; i != capacity; ++i) { + if (container_internal::IsFull(c.ctrl_[i])) { + m += Traits::space_used(c.slots_ + i); + } + } + } + return m; + } + + static size_t LowerBoundAllocatedByteSize(size_t size) { + size_t capacity = GrowthToLowerboundCapacity(size); + if (capacity == 0) return 0; + auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); + size_t m = layout.AllocSize(); + size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); + if (per_slot != ~size_t{}) { + m += per_slot * size; + } + return m; + } +}; + +} // namespace hashtable_debug_internal +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/raw_hash_set_allocator_test.cc b/third_party/abseil_cpp/absl/container/internal/raw_hash_set_allocator_test.cc new file mode 100644 index 000000000000..e73f53fd637b --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/raw_hash_set_allocator_test.cc @@ -0,0 +1,505 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <limits> +#include <scoped_allocator> + +#include "gtest/gtest.h" +#include "absl/container/internal/raw_hash_set.h" +#include "absl/container/internal/tracked.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +enum AllocSpec { + kPropagateOnCopy = 1, + kPropagateOnMove = 2, + kPropagateOnSwap = 4, +}; + +struct AllocState { + size_t num_allocs = 0; + std::set<void*> owned; +}; + +template <class T, + int Spec = kPropagateOnCopy | kPropagateOnMove | kPropagateOnSwap> +class CheckedAlloc { + public: + template <class, int> + friend class CheckedAlloc; + + using value_type = T; + + CheckedAlloc() {} + explicit CheckedAlloc(size_t id) : id_(id) {} + CheckedAlloc(const CheckedAlloc&) = default; + CheckedAlloc& operator=(const CheckedAlloc&) = default; + + template <class U> + CheckedAlloc(const CheckedAlloc<U, Spec>& that) + : id_(that.id_), state_(that.state_) {} + + template <class U> + struct rebind { + using other = CheckedAlloc<U, Spec>; + }; + + using propagate_on_container_copy_assignment = + std::integral_constant<bool, (Spec & kPropagateOnCopy) != 0>; + + using propagate_on_container_move_assignment = + std::integral_constant<bool, (Spec & kPropagateOnMove) != 0>; + + using propagate_on_container_swap = + std::integral_constant<bool, (Spec & kPropagateOnSwap) != 0>; + + CheckedAlloc select_on_container_copy_construction() const { + if (Spec & kPropagateOnCopy) return *this; + return {}; + } + + T* allocate(size_t n) { + T* ptr = std::allocator<T>().allocate(n); + track_alloc(ptr); + return ptr; + } + void deallocate(T* ptr, size_t n) { + memset(ptr, 0, n * sizeof(T)); // The freed memory must be unpoisoned. + track_dealloc(ptr); + return std::allocator<T>().deallocate(ptr, n); + } + + friend bool operator==(const CheckedAlloc& a, const CheckedAlloc& b) { + return a.id_ == b.id_; + } + friend bool operator!=(const CheckedAlloc& a, const CheckedAlloc& b) { + return !(a == b); + } + + size_t num_allocs() const { return state_->num_allocs; } + + void swap(CheckedAlloc& that) { + using std::swap; + swap(id_, that.id_); + swap(state_, that.state_); + } + + friend void swap(CheckedAlloc& a, CheckedAlloc& b) { a.swap(b); } + + friend std::ostream& operator<<(std::ostream& o, const CheckedAlloc& a) { + return o << "alloc(" << a.id_ << ")"; + } + + private: + void track_alloc(void* ptr) { + AllocState* state = state_.get(); + ++state->num_allocs; + if (!state->owned.insert(ptr).second) + ADD_FAILURE() << *this << " got previously allocated memory: " << ptr; + } + void track_dealloc(void* ptr) { + if (state_->owned.erase(ptr) != 1) + ADD_FAILURE() << *this + << " deleting memory owned by another allocator: " << ptr; + } + + size_t id_ = std::numeric_limits<size_t>::max(); + + std::shared_ptr<AllocState> state_ = std::make_shared<AllocState>(); +}; + +struct Identity { + int32_t operator()(int32_t v) const { return v; } +}; + +struct Policy { + using slot_type = Tracked<int32_t>; + using init_type = Tracked<int32_t>; + using key_type = int32_t; + + template <class allocator_type, class... Args> + static void construct(allocator_type* alloc, slot_type* slot, + Args&&... args) { + std::allocator_traits<allocator_type>::construct( + *alloc, slot, std::forward<Args>(args)...); + } + + template <class allocator_type> + static void destroy(allocator_type* alloc, slot_type* slot) { + std::allocator_traits<allocator_type>::destroy(*alloc, slot); + } + + template <class allocator_type> + static void transfer(allocator_type* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(*old_slot)); + destroy(alloc, old_slot); + } + + template <class F> + static auto apply(F&& f, int32_t v) -> decltype(std::forward<F>(f)(v, v)) { + return std::forward<F>(f)(v, v); + } + + template <class F> + static auto apply(F&& f, const slot_type& v) + -> decltype(std::forward<F>(f)(v.val(), v)) { + return std::forward<F>(f)(v.val(), v); + } + + template <class F> + static auto apply(F&& f, slot_type&& v) + -> decltype(std::forward<F>(f)(v.val(), std::move(v))) { + return std::forward<F>(f)(v.val(), std::move(v)); + } + + static slot_type& element(slot_type* slot) { return *slot; } +}; + +template <int Spec> +struct PropagateTest : public ::testing::Test { + using Alloc = CheckedAlloc<Tracked<int32_t>, Spec>; + + using Table = raw_hash_set<Policy, Identity, std::equal_to<int32_t>, Alloc>; + + PropagateTest() { + EXPECT_EQ(a1, t1.get_allocator()); + EXPECT_NE(a2, t1.get_allocator()); + } + + Alloc a1 = Alloc(1); + Table t1 = Table(0, a1); + Alloc a2 = Alloc(2); +}; + +using PropagateOnAll = + PropagateTest<kPropagateOnCopy | kPropagateOnMove | kPropagateOnSwap>; +using NoPropagateOnCopy = PropagateTest<kPropagateOnMove | kPropagateOnSwap>; +using NoPropagateOnMove = PropagateTest<kPropagateOnCopy | kPropagateOnSwap>; + +TEST_F(PropagateOnAll, Empty) { EXPECT_EQ(0, a1.num_allocs()); } + +TEST_F(PropagateOnAll, InsertAllocates) { + auto it = t1.insert(0).first; + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, InsertDecomposes) { + auto it = t1.insert(0).first; + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); + + EXPECT_FALSE(t1.insert(0).second); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, RehashMoves) { + auto it = t1.insert(0).first; + EXPECT_EQ(0, it->num_moves()); + t1.rehash(2 * t1.capacity()); + EXPECT_EQ(2, a1.num_allocs()); + it = t1.find(0); + EXPECT_EQ(1, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, CopyConstructor) { + auto it = t1.insert(0).first; + Table u(t1); + EXPECT_EQ(2, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(NoPropagateOnCopy, CopyConstructor) { + auto it = t1.insert(0).first; + Table u(t1); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, u.get_allocator().num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(PropagateOnAll, CopyConstructorWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(t1, a1); + EXPECT_EQ(2, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(NoPropagateOnCopy, CopyConstructorWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(t1, a1); + EXPECT_EQ(2, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(PropagateOnAll, CopyConstructorWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(t1, a2); + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, a2.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(NoPropagateOnCopy, CopyConstructorWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(t1, a2); + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, a2.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(PropagateOnAll, MoveConstructor) { + auto it = t1.insert(0).first; + Table u(std::move(t1)); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(NoPropagateOnMove, MoveConstructor) { + auto it = t1.insert(0).first; + Table u(std::move(t1)); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, MoveConstructorWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(std::move(t1), a1); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(NoPropagateOnMove, MoveConstructorWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(std::move(t1), a1); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, MoveConstructorWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(std::move(t1), a2); + it = u.find(0); + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, a2.num_allocs()); + EXPECT_EQ(1, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(NoPropagateOnMove, MoveConstructorWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(std::move(t1), a2); + it = u.find(0); + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, a2.num_allocs()); + EXPECT_EQ(1, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, CopyAssignmentWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(0, a1); + u = t1; + EXPECT_EQ(2, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(NoPropagateOnCopy, CopyAssignmentWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(0, a1); + u = t1; + EXPECT_EQ(2, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(PropagateOnAll, CopyAssignmentWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(0, a2); + u = t1; + EXPECT_EQ(a1, u.get_allocator()); + EXPECT_EQ(2, a1.num_allocs()); + EXPECT_EQ(0, a2.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(NoPropagateOnCopy, CopyAssignmentWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(0, a2); + u = t1; + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, a2.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(1, it->num_copies()); +} + +TEST_F(PropagateOnAll, MoveAssignmentWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(0, a1); + u = std::move(t1); + EXPECT_EQ(a1, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(NoPropagateOnMove, MoveAssignmentWithSameAlloc) { + auto it = t1.insert(0).first; + Table u(0, a1); + u = std::move(t1); + EXPECT_EQ(a1, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, MoveAssignmentWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(0, a2); + u = std::move(t1); + EXPECT_EQ(a1, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, a2.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(NoPropagateOnMove, MoveAssignmentWithDifferentAlloc) { + auto it = t1.insert(0).first; + Table u(0, a2); + u = std::move(t1); + it = u.find(0); + EXPECT_EQ(a2, u.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(1, a2.num_allocs()); + EXPECT_EQ(1, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +TEST_F(PropagateOnAll, Swap) { + auto it = t1.insert(0).first; + Table u(0, a2); + u.swap(t1); + EXPECT_EQ(a1, u.get_allocator()); + EXPECT_EQ(a2, t1.get_allocator()); + EXPECT_EQ(1, a1.num_allocs()); + EXPECT_EQ(0, a2.num_allocs()); + EXPECT_EQ(0, it->num_moves()); + EXPECT_EQ(0, it->num_copies()); +} + +// This allocator is similar to std::pmr::polymorphic_allocator. +// Note the disabled assignment. +template <class T> +class PAlloc { + template <class> + friend class PAlloc; + + public: + // types + using value_type = T; + + // traits + using propagate_on_container_swap = std::false_type; + + PAlloc() noexcept = default; + explicit PAlloc(size_t id) noexcept : id_(id) {} + PAlloc(const PAlloc&) noexcept = default; + PAlloc& operator=(const PAlloc&) noexcept = delete; + + template <class U> + PAlloc(const PAlloc<U>& that) noexcept : id_(that.id_) {} // NOLINT + + template <class U> + struct rebind { + using other = PAlloc<U>; + }; + + constexpr PAlloc select_on_container_copy_construction() const { return {}; } + + // public member functions + T* allocate(size_t) { return new T; } + void deallocate(T* p, size_t) { delete p; } + + friend bool operator==(const PAlloc& a, const PAlloc& b) { + return a.id_ == b.id_; + } + friend bool operator!=(const PAlloc& a, const PAlloc& b) { return !(a == b); } + + private: + size_t id_ = std::numeric_limits<size_t>::max(); +}; + +// This doesn't compile with GCC 5.4 and 5.5 due to a bug in noexcept handing. +#if !defined(__GNUC__) || __GNUC__ != 5 || (__GNUC_MINOR__ != 4 && \ + __GNUC_MINOR__ != 5) +TEST(NoPropagateOn, Swap) { + using PA = PAlloc<char>; + using Table = raw_hash_set<Policy, Identity, std::equal_to<int32_t>, PA>; + + Table t1(PA{1}), t2(PA{2}); + swap(t1, t2); + EXPECT_EQ(t1.get_allocator(), PA(1)); + EXPECT_EQ(t2.get_allocator(), PA(2)); +} +#endif + +TEST(NoPropagateOn, CopyConstruct) { + using PA = PAlloc<char>; + using Table = raw_hash_set<Policy, Identity, std::equal_to<int32_t>, PA>; + + Table t1(PA{1}), t2(t1); + EXPECT_EQ(t1.get_allocator(), PA(1)); + EXPECT_EQ(t2.get_allocator(), PA()); +} + +TEST(NoPropagateOn, Assignment) { + using PA = PAlloc<char>; + using Table = raw_hash_set<Policy, Identity, std::equal_to<int32_t>, PA>; + + Table t1(PA{1}), t2(PA{2}); + t1 = t2; + EXPECT_EQ(t1.get_allocator(), PA(1)); + EXPECT_EQ(t2.get_allocator(), PA(2)); +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/raw_hash_set_test.cc b/third_party/abseil_cpp/absl/container/internal/raw_hash_set_test.cc new file mode 100644 index 000000000000..33d2773de302 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/raw_hash_set_test.cc @@ -0,0 +1,1893 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/raw_hash_set.h" + +#include <cmath> +#include <cstdint> +#include <deque> +#include <functional> +#include <memory> +#include <numeric> +#include <random> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hash_function_defaults.h" +#include "absl/container/internal/hash_policy_testing.h" +#include "absl/container/internal/hashtable_debug.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +struct RawHashSetTestOnlyAccess { + template <typename C> + static auto GetSlots(const C& c) -> decltype(c.slots_) { + return c.slots_; + } +}; + +namespace { + +using ::testing::DoubleNear; +using ::testing::ElementsAre; +using ::testing::Ge; +using ::testing::Lt; +using ::testing::Optional; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +TEST(Util, NormalizeCapacity) { + EXPECT_EQ(1, NormalizeCapacity(0)); + EXPECT_EQ(1, NormalizeCapacity(1)); + EXPECT_EQ(3, NormalizeCapacity(2)); + EXPECT_EQ(3, NormalizeCapacity(3)); + EXPECT_EQ(7, NormalizeCapacity(4)); + EXPECT_EQ(7, NormalizeCapacity(7)); + EXPECT_EQ(15, NormalizeCapacity(8)); + EXPECT_EQ(15, NormalizeCapacity(15)); + EXPECT_EQ(15 * 2 + 1, NormalizeCapacity(15 + 1)); + EXPECT_EQ(15 * 2 + 1, NormalizeCapacity(15 + 2)); +} + +TEST(Util, GrowthAndCapacity) { + // Verify that GrowthToCapacity gives the minimum capacity that has enough + // growth. + for (size_t growth = 0; growth < 10000; ++growth) { + SCOPED_TRACE(growth); + size_t capacity = NormalizeCapacity(GrowthToLowerboundCapacity(growth)); + // The capacity is large enough for `growth` + EXPECT_THAT(CapacityToGrowth(capacity), Ge(growth)); + if (growth != 0 && capacity > 1) { + // There is no smaller capacity that works. + EXPECT_THAT(CapacityToGrowth(capacity / 2), Lt(growth)); + } + } + + for (size_t capacity = Group::kWidth - 1; capacity < 10000; + capacity = 2 * capacity + 1) { + SCOPED_TRACE(capacity); + size_t growth = CapacityToGrowth(capacity); + EXPECT_THAT(growth, Lt(capacity)); + EXPECT_LE(GrowthToLowerboundCapacity(growth), capacity); + EXPECT_EQ(NormalizeCapacity(GrowthToLowerboundCapacity(growth)), capacity); + } +} + +TEST(Util, probe_seq) { + probe_seq<16> seq(0, 127); + auto gen = [&]() { + size_t res = seq.offset(); + seq.next(); + return res; + }; + std::vector<size_t> offsets(8); + std::generate_n(offsets.begin(), 8, gen); + EXPECT_THAT(offsets, ElementsAre(0, 16, 48, 96, 32, 112, 80, 64)); + seq = probe_seq<16>(128, 127); + std::generate_n(offsets.begin(), 8, gen); + EXPECT_THAT(offsets, ElementsAre(0, 16, 48, 96, 32, 112, 80, 64)); +} + +TEST(BitMask, Smoke) { + EXPECT_FALSE((BitMask<uint8_t, 8>(0))); + EXPECT_TRUE((BitMask<uint8_t, 8>(5))); + + EXPECT_THAT((BitMask<uint8_t, 8>(0)), ElementsAre()); + EXPECT_THAT((BitMask<uint8_t, 8>(0x1)), ElementsAre(0)); + EXPECT_THAT((BitMask<uint8_t, 8>(0x2)), ElementsAre(1)); + EXPECT_THAT((BitMask<uint8_t, 8>(0x3)), ElementsAre(0, 1)); + EXPECT_THAT((BitMask<uint8_t, 8>(0x4)), ElementsAre(2)); + EXPECT_THAT((BitMask<uint8_t, 8>(0x5)), ElementsAre(0, 2)); + EXPECT_THAT((BitMask<uint8_t, 8>(0x55)), ElementsAre(0, 2, 4, 6)); + EXPECT_THAT((BitMask<uint8_t, 8>(0xAA)), ElementsAre(1, 3, 5, 7)); +} + +TEST(BitMask, WithShift) { + // See the non-SSE version of Group for details on what this math is for. + uint64_t ctrl = 0x1716151413121110; + uint64_t hash = 0x12; + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = ctrl ^ (lsbs * hash); + uint64_t mask = (x - lsbs) & ~x & msbs; + EXPECT_EQ(0x0000000080800000, mask); + + BitMask<uint64_t, 8, 3> b(mask); + EXPECT_EQ(*b, 2); +} + +TEST(BitMask, LeadingTrailing) { + EXPECT_EQ((BitMask<uint32_t, 16>(0x00001a40).LeadingZeros()), 3); + EXPECT_EQ((BitMask<uint32_t, 16>(0x00001a40).TrailingZeros()), 6); + + EXPECT_EQ((BitMask<uint32_t, 16>(0x00000001).LeadingZeros()), 15); + EXPECT_EQ((BitMask<uint32_t, 16>(0x00000001).TrailingZeros()), 0); + + EXPECT_EQ((BitMask<uint32_t, 16>(0x00008000).LeadingZeros()), 0); + EXPECT_EQ((BitMask<uint32_t, 16>(0x00008000).TrailingZeros()), 15); + + EXPECT_EQ((BitMask<uint64_t, 8, 3>(0x0000008080808000).LeadingZeros()), 3); + EXPECT_EQ((BitMask<uint64_t, 8, 3>(0x0000008080808000).TrailingZeros()), 1); + + EXPECT_EQ((BitMask<uint64_t, 8, 3>(0x0000000000000080).LeadingZeros()), 7); + EXPECT_EQ((BitMask<uint64_t, 8, 3>(0x0000000000000080).TrailingZeros()), 0); + + EXPECT_EQ((BitMask<uint64_t, 8, 3>(0x8000000000000000).LeadingZeros()), 0); + EXPECT_EQ((BitMask<uint64_t, 8, 3>(0x8000000000000000).TrailingZeros()), 7); +} + +TEST(Group, EmptyGroup) { + for (h2_t h = 0; h != 128; ++h) EXPECT_FALSE(Group{EmptyGroup()}.Match(h)); +} + +TEST(Group, Match) { + if (Group::kWidth == 16) { + ctrl_t group[] = {kEmpty, 1, kDeleted, 3, kEmpty, 5, kSentinel, 7, + 7, 5, 3, 1, 1, 1, 1, 1}; + EXPECT_THAT(Group{group}.Match(0), ElementsAre()); + EXPECT_THAT(Group{group}.Match(1), ElementsAre(1, 11, 12, 13, 14, 15)); + EXPECT_THAT(Group{group}.Match(3), ElementsAre(3, 10)); + EXPECT_THAT(Group{group}.Match(5), ElementsAre(5, 9)); + EXPECT_THAT(Group{group}.Match(7), ElementsAre(7, 8)); + } else if (Group::kWidth == 8) { + ctrl_t group[] = {kEmpty, 1, 2, kDeleted, 2, 1, kSentinel, 1}; + EXPECT_THAT(Group{group}.Match(0), ElementsAre()); + EXPECT_THAT(Group{group}.Match(1), ElementsAre(1, 5, 7)); + EXPECT_THAT(Group{group}.Match(2), ElementsAre(2, 4)); + } else { + FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; + } +} + +TEST(Group, MatchEmpty) { + if (Group::kWidth == 16) { + ctrl_t group[] = {kEmpty, 1, kDeleted, 3, kEmpty, 5, kSentinel, 7, + 7, 5, 3, 1, 1, 1, 1, 1}; + EXPECT_THAT(Group{group}.MatchEmpty(), ElementsAre(0, 4)); + } else if (Group::kWidth == 8) { + ctrl_t group[] = {kEmpty, 1, 2, kDeleted, 2, 1, kSentinel, 1}; + EXPECT_THAT(Group{group}.MatchEmpty(), ElementsAre(0)); + } else { + FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; + } +} + +TEST(Group, MatchEmptyOrDeleted) { + if (Group::kWidth == 16) { + ctrl_t group[] = {kEmpty, 1, kDeleted, 3, kEmpty, 5, kSentinel, 7, + 7, 5, 3, 1, 1, 1, 1, 1}; + EXPECT_THAT(Group{group}.MatchEmptyOrDeleted(), ElementsAre(0, 2, 4)); + } else if (Group::kWidth == 8) { + ctrl_t group[] = {kEmpty, 1, 2, kDeleted, 2, 1, kSentinel, 1}; + EXPECT_THAT(Group{group}.MatchEmptyOrDeleted(), ElementsAre(0, 3)); + } else { + FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; + } +} + +TEST(Batch, DropDeletes) { + constexpr size_t kCapacity = 63; + constexpr size_t kGroupWidth = container_internal::Group::kWidth; + std::vector<ctrl_t> ctrl(kCapacity + 1 + kGroupWidth); + ctrl[kCapacity] = kSentinel; + std::vector<ctrl_t> pattern = {kEmpty, 2, kDeleted, 2, kEmpty, 1, kDeleted}; + for (size_t i = 0; i != kCapacity; ++i) { + ctrl[i] = pattern[i % pattern.size()]; + if (i < kGroupWidth - 1) + ctrl[i + kCapacity + 1] = pattern[i % pattern.size()]; + } + ConvertDeletedToEmptyAndFullToDeleted(ctrl.data(), kCapacity); + ASSERT_EQ(ctrl[kCapacity], kSentinel); + for (size_t i = 0; i < kCapacity + 1 + kGroupWidth; ++i) { + ctrl_t expected = pattern[i % (kCapacity + 1) % pattern.size()]; + if (i == kCapacity) expected = kSentinel; + if (expected == kDeleted) expected = kEmpty; + if (IsFull(expected)) expected = kDeleted; + EXPECT_EQ(ctrl[i], expected) + << i << " " << int{pattern[i % pattern.size()]}; + } +} + +TEST(Group, CountLeadingEmptyOrDeleted) { + const std::vector<ctrl_t> empty_examples = {kEmpty, kDeleted}; + const std::vector<ctrl_t> full_examples = {0, 1, 2, 3, 5, 9, 127, kSentinel}; + + for (ctrl_t empty : empty_examples) { + std::vector<ctrl_t> e(Group::kWidth, empty); + EXPECT_EQ(Group::kWidth, Group{e.data()}.CountLeadingEmptyOrDeleted()); + for (ctrl_t full : full_examples) { + for (size_t i = 0; i != Group::kWidth; ++i) { + std::vector<ctrl_t> f(Group::kWidth, empty); + f[i] = full; + EXPECT_EQ(i, Group{f.data()}.CountLeadingEmptyOrDeleted()); + } + std::vector<ctrl_t> f(Group::kWidth, empty); + f[Group::kWidth * 2 / 3] = full; + f[Group::kWidth / 2] = full; + EXPECT_EQ( + Group::kWidth / 2, Group{f.data()}.CountLeadingEmptyOrDeleted()); + } + } +} + +struct IntPolicy { + using slot_type = int64_t; + using key_type = int64_t; + using init_type = int64_t; + + static void construct(void*, int64_t* slot, int64_t v) { *slot = v; } + static void destroy(void*, int64_t*) {} + static void transfer(void*, int64_t* new_slot, int64_t* old_slot) { + *new_slot = *old_slot; + } + + static int64_t& element(slot_type* slot) { return *slot; } + + template <class F> + static auto apply(F&& f, int64_t x) -> decltype(std::forward<F>(f)(x, x)) { + return std::forward<F>(f)(x, x); + } +}; + +class StringPolicy { + template <class F, class K, class V, + class = typename std::enable_if< + std::is_convertible<const K&, absl::string_view>::value>::type> + decltype(std::declval<F>()( + std::declval<const absl::string_view&>(), std::piecewise_construct, + std::declval<std::tuple<K>>(), + std::declval<V>())) static apply_impl(F&& f, + std::pair<std::tuple<K>, V> p) { + const absl::string_view& key = std::get<0>(p.first); + return std::forward<F>(f)(key, std::piecewise_construct, std::move(p.first), + std::move(p.second)); + } + + public: + struct slot_type { + struct ctor {}; + + template <class... Ts> + slot_type(ctor, Ts&&... ts) : pair(std::forward<Ts>(ts)...) {} + + std::pair<std::string, std::string> pair; + }; + + using key_type = std::string; + using init_type = std::pair<std::string, std::string>; + + template <class allocator_type, class... Args> + static void construct(allocator_type* alloc, slot_type* slot, Args... args) { + std::allocator_traits<allocator_type>::construct( + *alloc, slot, typename slot_type::ctor(), std::forward<Args>(args)...); + } + + template <class allocator_type> + static void destroy(allocator_type* alloc, slot_type* slot) { + std::allocator_traits<allocator_type>::destroy(*alloc, slot); + } + + template <class allocator_type> + static void transfer(allocator_type* alloc, slot_type* new_slot, + slot_type* old_slot) { + construct(alloc, new_slot, std::move(old_slot->pair)); + destroy(alloc, old_slot); + } + + static std::pair<std::string, std::string>& element(slot_type* slot) { + return slot->pair; + } + + template <class F, class... Args> + static auto apply(F&& f, Args&&... args) + -> decltype(apply_impl(std::forward<F>(f), + PairArgs(std::forward<Args>(args)...))) { + return apply_impl(std::forward<F>(f), + PairArgs(std::forward<Args>(args)...)); + } +}; + +struct StringHash : absl::Hash<absl::string_view> { + using is_transparent = void; +}; +struct StringEq : std::equal_to<absl::string_view> { + using is_transparent = void; +}; + +struct StringTable + : raw_hash_set<StringPolicy, StringHash, StringEq, std::allocator<int>> { + using Base = typename StringTable::raw_hash_set; + StringTable() {} + using Base::Base; +}; + +struct IntTable + : raw_hash_set<IntPolicy, container_internal::hash_default_hash<int64_t>, + std::equal_to<int64_t>, std::allocator<int64_t>> { + using Base = typename IntTable::raw_hash_set; + using Base::Base; +}; + +template <typename T> +struct CustomAlloc : std::allocator<T> { + CustomAlloc() {} + + template <typename U> + CustomAlloc(const CustomAlloc<U>& other) {} + + template<class U> struct rebind { + using other = CustomAlloc<U>; + }; +}; + +struct CustomAllocIntTable + : raw_hash_set<IntPolicy, container_internal::hash_default_hash<int64_t>, + std::equal_to<int64_t>, CustomAlloc<int64_t>> { + using Base = typename CustomAllocIntTable::raw_hash_set; + using Base::Base; +}; + +struct BadFastHash { + template <class T> + size_t operator()(const T&) const { + return 0; + } +}; + +struct BadTable : raw_hash_set<IntPolicy, BadFastHash, std::equal_to<int>, + std::allocator<int>> { + using Base = typename BadTable::raw_hash_set; + BadTable() {} + using Base::Base; +}; + +TEST(Table, EmptyFunctorOptimization) { + static_assert(std::is_empty<std::equal_to<absl::string_view>>::value, ""); + static_assert(std::is_empty<std::allocator<int>>::value, ""); + + struct MockTable { + void* ctrl; + void* slots; + size_t size; + size_t capacity; + size_t growth_left; + void* infoz; + }; + struct StatelessHash { + size_t operator()(absl::string_view) const { return 0; } + }; + struct StatefulHash : StatelessHash { + size_t dummy; + }; + + EXPECT_EQ( + sizeof(MockTable), + sizeof( + raw_hash_set<StringPolicy, StatelessHash, + std::equal_to<absl::string_view>, std::allocator<int>>)); + + EXPECT_EQ( + sizeof(MockTable) + sizeof(StatefulHash), + sizeof( + raw_hash_set<StringPolicy, StatefulHash, + std::equal_to<absl::string_view>, std::allocator<int>>)); +} + +TEST(Table, Empty) { + IntTable t; + EXPECT_EQ(0, t.size()); + EXPECT_TRUE(t.empty()); +} + +TEST(Table, LookupEmpty) { + IntTable t; + auto it = t.find(0); + EXPECT_TRUE(it == t.end()); +} + +TEST(Table, Insert1) { + IntTable t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 0); + EXPECT_EQ(1, t.size()); + EXPECT_THAT(*t.find(0), 0); +} + +TEST(Table, Insert2) { + IntTable t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 0); + EXPECT_EQ(1, t.size()); + EXPECT_TRUE(t.find(1) == t.end()); + res = t.emplace(1); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 1); + EXPECT_EQ(2, t.size()); + EXPECT_THAT(*t.find(0), 0); + EXPECT_THAT(*t.find(1), 1); +} + +TEST(Table, InsertCollision) { + BadTable t; + EXPECT_TRUE(t.find(1) == t.end()); + auto res = t.emplace(1); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, 1); + EXPECT_EQ(1, t.size()); + + EXPECT_TRUE(t.find(2) == t.end()); + res = t.emplace(2); + EXPECT_THAT(*res.first, 2); + EXPECT_TRUE(res.second); + EXPECT_EQ(2, t.size()); + + EXPECT_THAT(*t.find(1), 1); + EXPECT_THAT(*t.find(2), 2); +} + +// Test that we do not add existent element in case we need to search through +// many groups with deleted elements +TEST(Table, InsertCollisionAndFindAfterDelete) { + BadTable t; // all elements go to the same group. + // Have at least 2 groups with Group::kWidth collisions + // plus some extra collisions in the last group. + constexpr size_t kNumInserts = Group::kWidth * 2 + 5; + for (size_t i = 0; i < kNumInserts; ++i) { + auto res = t.emplace(i); + EXPECT_TRUE(res.second); + EXPECT_THAT(*res.first, i); + EXPECT_EQ(i + 1, t.size()); + } + + // Remove elements one by one and check + // that we still can find all other elements. + for (size_t i = 0; i < kNumInserts; ++i) { + EXPECT_EQ(1, t.erase(i)) << i; + for (size_t j = i + 1; j < kNumInserts; ++j) { + EXPECT_THAT(*t.find(j), j); + auto res = t.emplace(j); + EXPECT_FALSE(res.second) << i << " " << j; + EXPECT_THAT(*res.first, j); + EXPECT_EQ(kNumInserts - i - 1, t.size()); + } + } + EXPECT_TRUE(t.empty()); +} + +TEST(Table, LazyEmplace) { + StringTable t; + bool called = false; + auto it = t.lazy_emplace("abc", [&](const StringTable::constructor& f) { + called = true; + f("abc", "ABC"); + }); + EXPECT_TRUE(called); + EXPECT_THAT(*it, Pair("abc", "ABC")); + called = false; + it = t.lazy_emplace("abc", [&](const StringTable::constructor& f) { + called = true; + f("abc", "DEF"); + }); + EXPECT_FALSE(called); + EXPECT_THAT(*it, Pair("abc", "ABC")); +} + +TEST(Table, ContainsEmpty) { + IntTable t; + + EXPECT_FALSE(t.contains(0)); +} + +TEST(Table, Contains1) { + IntTable t; + + EXPECT_TRUE(t.insert(0).second); + EXPECT_TRUE(t.contains(0)); + EXPECT_FALSE(t.contains(1)); + + EXPECT_EQ(1, t.erase(0)); + EXPECT_FALSE(t.contains(0)); +} + +TEST(Table, Contains2) { + IntTable t; + + EXPECT_TRUE(t.insert(0).second); + EXPECT_TRUE(t.contains(0)); + EXPECT_FALSE(t.contains(1)); + + t.clear(); + EXPECT_FALSE(t.contains(0)); +} + +int decompose_constructed; +struct DecomposeType { + DecomposeType(int i) : i(i) { // NOLINT + ++decompose_constructed; + } + + explicit DecomposeType(const char* d) : DecomposeType(*d) {} + + int i; +}; + +struct DecomposeHash { + using is_transparent = void; + size_t operator()(DecomposeType a) const { return a.i; } + size_t operator()(int a) const { return a; } + size_t operator()(const char* a) const { return *a; } +}; + +struct DecomposeEq { + using is_transparent = void; + bool operator()(DecomposeType a, DecomposeType b) const { return a.i == b.i; } + bool operator()(DecomposeType a, int b) const { return a.i == b; } + bool operator()(DecomposeType a, const char* b) const { return a.i == *b; } +}; + +struct DecomposePolicy { + using slot_type = DecomposeType; + using key_type = DecomposeType; + using init_type = DecomposeType; + + template <typename T> + static void construct(void*, DecomposeType* slot, T&& v) { + *slot = DecomposeType(std::forward<T>(v)); + } + static void destroy(void*, DecomposeType*) {} + static DecomposeType& element(slot_type* slot) { return *slot; } + + template <class F, class T> + static auto apply(F&& f, const T& x) -> decltype(std::forward<F>(f)(x, x)) { + return std::forward<F>(f)(x, x); + } +}; + +template <typename Hash, typename Eq> +void TestDecompose(bool construct_three) { + DecomposeType elem{0}; + const int one = 1; + const char* three_p = "3"; + const auto& three = three_p; + + raw_hash_set<DecomposePolicy, Hash, Eq, std::allocator<int>> set1; + + decompose_constructed = 0; + int expected_constructed = 0; + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.insert(elem); + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.insert(1); + EXPECT_EQ(++expected_constructed, decompose_constructed); + set1.emplace("3"); + EXPECT_EQ(++expected_constructed, decompose_constructed); + EXPECT_EQ(expected_constructed, decompose_constructed); + + { // insert(T&&) + set1.insert(1); + EXPECT_EQ(expected_constructed, decompose_constructed); + } + + { // insert(const T&) + set1.insert(one); + EXPECT_EQ(expected_constructed, decompose_constructed); + } + + { // insert(hint, T&&) + set1.insert(set1.begin(), 1); + EXPECT_EQ(expected_constructed, decompose_constructed); + } + + { // insert(hint, const T&) + set1.insert(set1.begin(), one); + EXPECT_EQ(expected_constructed, decompose_constructed); + } + + { // emplace(...) + set1.emplace(1); + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.emplace("3"); + expected_constructed += construct_three; + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.emplace(one); + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.emplace(three); + expected_constructed += construct_three; + EXPECT_EQ(expected_constructed, decompose_constructed); + } + + { // emplace_hint(...) + set1.emplace_hint(set1.begin(), 1); + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.emplace_hint(set1.begin(), "3"); + expected_constructed += construct_three; + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.emplace_hint(set1.begin(), one); + EXPECT_EQ(expected_constructed, decompose_constructed); + set1.emplace_hint(set1.begin(), three); + expected_constructed += construct_three; + EXPECT_EQ(expected_constructed, decompose_constructed); + } +} + +TEST(Table, Decompose) { + TestDecompose<DecomposeHash, DecomposeEq>(false); + + struct TransparentHashIntOverload { + size_t operator()(DecomposeType a) const { return a.i; } + size_t operator()(int a) const { return a; } + }; + struct TransparentEqIntOverload { + bool operator()(DecomposeType a, DecomposeType b) const { + return a.i == b.i; + } + bool operator()(DecomposeType a, int b) const { return a.i == b; } + }; + TestDecompose<TransparentHashIntOverload, DecomposeEq>(true); + TestDecompose<TransparentHashIntOverload, TransparentEqIntOverload>(true); + TestDecompose<DecomposeHash, TransparentEqIntOverload>(true); +} + +// Returns the largest m such that a table with m elements has the same number +// of buckets as a table with n elements. +size_t MaxDensitySize(size_t n) { + IntTable t; + t.reserve(n); + for (size_t i = 0; i != n; ++i) t.emplace(i); + const size_t c = t.bucket_count(); + while (c == t.bucket_count()) t.emplace(n++); + return t.size() - 1; +} + +struct Modulo1000Hash { + size_t operator()(int x) const { return x % 1000; } +}; + +struct Modulo1000HashTable + : public raw_hash_set<IntPolicy, Modulo1000Hash, std::equal_to<int>, + std::allocator<int>> {}; + +// Test that rehash with no resize happen in case of many deleted slots. +TEST(Table, RehashWithNoResize) { + Modulo1000HashTable t; + // Adding the same length (and the same hash) strings + // to have at least kMinFullGroups groups + // with Group::kWidth collisions. Then fill up to MaxDensitySize; + const size_t kMinFullGroups = 7; + std::vector<int> keys; + for (size_t i = 0; i < MaxDensitySize(Group::kWidth * kMinFullGroups); ++i) { + int k = i * 1000; + t.emplace(k); + keys.push_back(k); + } + const size_t capacity = t.capacity(); + + // Remove elements from all groups except the first and the last one. + // All elements removed from full groups will be marked as kDeleted. + const size_t erase_begin = Group::kWidth / 2; + const size_t erase_end = (t.size() / Group::kWidth - 1) * Group::kWidth; + for (size_t i = erase_begin; i < erase_end; ++i) { + EXPECT_EQ(1, t.erase(keys[i])) << i; + } + keys.erase(keys.begin() + erase_begin, keys.begin() + erase_end); + + auto last_key = keys.back(); + size_t last_key_num_probes = GetHashtableDebugNumProbes(t, last_key); + + // Make sure that we have to make a lot of probes for last key. + ASSERT_GT(last_key_num_probes, kMinFullGroups); + + int x = 1; + // Insert and erase one element, before inplace rehash happen. + while (last_key_num_probes == GetHashtableDebugNumProbes(t, last_key)) { + t.emplace(x); + ASSERT_EQ(capacity, t.capacity()); + // All elements should be there. + ASSERT_TRUE(t.find(x) != t.end()) << x; + for (const auto& k : keys) { + ASSERT_TRUE(t.find(k) != t.end()) << k; + } + t.erase(x); + ++x; + } +} + +TEST(Table, InsertEraseStressTest) { + IntTable t; + const size_t kMinElementCount = 250; + std::deque<int> keys; + size_t i = 0; + for (; i < MaxDensitySize(kMinElementCount); ++i) { + t.emplace(i); + keys.push_back(i); + } + const size_t kNumIterations = 1000000; + for (; i < kNumIterations; ++i) { + ASSERT_EQ(1, t.erase(keys.front())); + keys.pop_front(); + t.emplace(i); + keys.push_back(i); + } +} + +TEST(Table, InsertOverloads) { + StringTable t; + // These should all trigger the insert(init_type) overload. + t.insert({{}, {}}); + t.insert({"ABC", {}}); + t.insert({"DEF", "!!!"}); + + EXPECT_THAT(t, UnorderedElementsAre(Pair("", ""), Pair("ABC", ""), + Pair("DEF", "!!!"))); +} + +TEST(Table, LargeTable) { + IntTable t; + for (int64_t i = 0; i != 100000; ++i) t.emplace(i << 40); + for (int64_t i = 0; i != 100000; ++i) ASSERT_EQ(i << 40, *t.find(i << 40)); +} + +// Timeout if copy is quadratic as it was in Rust. +TEST(Table, EnsureNonQuadraticAsInRust) { + static const size_t kLargeSize = 1 << 15; + + IntTable t; + for (size_t i = 0; i != kLargeSize; ++i) { + t.insert(i); + } + + // If this is quadratic, the test will timeout. + IntTable t2; + for (const auto& entry : t) t2.insert(entry); +} + +TEST(Table, ClearBug) { + IntTable t; + constexpr size_t capacity = container_internal::Group::kWidth - 1; + constexpr size_t max_size = capacity / 2 + 1; + for (size_t i = 0; i < max_size; ++i) { + t.insert(i); + } + ASSERT_EQ(capacity, t.capacity()); + intptr_t original = reinterpret_cast<intptr_t>(&*t.find(2)); + t.clear(); + ASSERT_EQ(capacity, t.capacity()); + for (size_t i = 0; i < max_size; ++i) { + t.insert(i); + } + ASSERT_EQ(capacity, t.capacity()); + intptr_t second = reinterpret_cast<intptr_t>(&*t.find(2)); + // We are checking that original and second are close enough to each other + // that they are probably still in the same group. This is not strictly + // guaranteed. + EXPECT_LT(std::abs(original - second), + capacity * sizeof(IntTable::value_type)); +} + +TEST(Table, Erase) { + IntTable t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_EQ(1, t.size()); + t.erase(res.first); + EXPECT_EQ(0, t.size()); + EXPECT_TRUE(t.find(0) == t.end()); +} + +TEST(Table, EraseMaintainsValidIterator) { + IntTable t; + const int kNumElements = 100; + for (int i = 0; i < kNumElements; i ++) { + EXPECT_TRUE(t.emplace(i).second); + } + EXPECT_EQ(t.size(), kNumElements); + + int num_erase_calls = 0; + auto it = t.begin(); + while (it != t.end()) { + t.erase(it++); + num_erase_calls++; + } + + EXPECT_TRUE(t.empty()); + EXPECT_EQ(num_erase_calls, kNumElements); +} + +// Collect N bad keys by following algorithm: +// 1. Create an empty table and reserve it to 2 * N. +// 2. Insert N random elements. +// 3. Take first Group::kWidth - 1 to bad_keys array. +// 4. Clear the table without resize. +// 5. Go to point 2 while N keys not collected +std::vector<int64_t> CollectBadMergeKeys(size_t N) { + static constexpr int kGroupSize = Group::kWidth - 1; + + auto topk_range = [](size_t b, size_t e, + IntTable* t) -> std::vector<int64_t> { + for (size_t i = b; i != e; ++i) { + t->emplace(i); + } + std::vector<int64_t> res; + res.reserve(kGroupSize); + auto it = t->begin(); + for (size_t i = b; i != e && i != b + kGroupSize; ++i, ++it) { + res.push_back(*it); + } + return res; + }; + + std::vector<int64_t> bad_keys; + bad_keys.reserve(N); + IntTable t; + t.reserve(N * 2); + + for (size_t b = 0; bad_keys.size() < N; b += N) { + auto keys = topk_range(b, b + N, &t); + bad_keys.insert(bad_keys.end(), keys.begin(), keys.end()); + t.erase(t.begin(), t.end()); + EXPECT_TRUE(t.empty()); + } + return bad_keys; +} + +struct ProbeStats { + // Number of elements with specific probe length over all tested tables. + std::vector<size_t> all_probes_histogram; + // Ratios total_probe_length/size for every tested table. + std::vector<double> single_table_ratios; + + friend ProbeStats operator+(const ProbeStats& a, const ProbeStats& b) { + ProbeStats res = a; + res.all_probes_histogram.resize(std::max(res.all_probes_histogram.size(), + b.all_probes_histogram.size())); + std::transform(b.all_probes_histogram.begin(), b.all_probes_histogram.end(), + res.all_probes_histogram.begin(), + res.all_probes_histogram.begin(), std::plus<size_t>()); + res.single_table_ratios.insert(res.single_table_ratios.end(), + b.single_table_ratios.begin(), + b.single_table_ratios.end()); + return res; + } + + // Average ratio total_probe_length/size over tables. + double AvgRatio() const { + return std::accumulate(single_table_ratios.begin(), + single_table_ratios.end(), 0.0) / + single_table_ratios.size(); + } + + // Maximum ratio total_probe_length/size over tables. + double MaxRatio() const { + return *std::max_element(single_table_ratios.begin(), + single_table_ratios.end()); + } + + // Percentile ratio total_probe_length/size over tables. + double PercentileRatio(double Percentile = 0.95) const { + auto r = single_table_ratios; + auto mid = r.begin() + static_cast<size_t>(r.size() * Percentile); + if (mid != r.end()) { + std::nth_element(r.begin(), mid, r.end()); + return *mid; + } else { + return MaxRatio(); + } + } + + // Maximum probe length over all elements and all tables. + size_t MaxProbe() const { return all_probes_histogram.size(); } + + // Fraction of elements with specified probe length. + std::vector<double> ProbeNormalizedHistogram() const { + double total_elements = std::accumulate(all_probes_histogram.begin(), + all_probes_histogram.end(), 0ull); + std::vector<double> res; + for (size_t p : all_probes_histogram) { + res.push_back(p / total_elements); + } + return res; + } + + size_t PercentileProbe(double Percentile = 0.99) const { + size_t idx = 0; + for (double p : ProbeNormalizedHistogram()) { + if (Percentile > p) { + Percentile -= p; + ++idx; + } else { + return idx; + } + } + return idx; + } + + friend std::ostream& operator<<(std::ostream& out, const ProbeStats& s) { + out << "{AvgRatio:" << s.AvgRatio() << ", MaxRatio:" << s.MaxRatio() + << ", PercentileRatio:" << s.PercentileRatio() + << ", MaxProbe:" << s.MaxProbe() << ", Probes=["; + for (double p : s.ProbeNormalizedHistogram()) { + out << p << ","; + } + out << "]}"; + + return out; + } +}; + +struct ExpectedStats { + double avg_ratio; + double max_ratio; + std::vector<std::pair<double, double>> pecentile_ratios; + std::vector<std::pair<double, double>> pecentile_probes; + + friend std::ostream& operator<<(std::ostream& out, const ExpectedStats& s) { + out << "{AvgRatio:" << s.avg_ratio << ", MaxRatio:" << s.max_ratio + << ", PercentileRatios: ["; + for (auto el : s.pecentile_ratios) { + out << el.first << ":" << el.second << ", "; + } + out << "], PercentileProbes: ["; + for (auto el : s.pecentile_probes) { + out << el.first << ":" << el.second << ", "; + } + out << "]}"; + + return out; + } +}; + +void VerifyStats(size_t size, const ExpectedStats& exp, + const ProbeStats& stats) { + EXPECT_LT(stats.AvgRatio(), exp.avg_ratio) << size << " " << stats; + EXPECT_LT(stats.MaxRatio(), exp.max_ratio) << size << " " << stats; + for (auto pr : exp.pecentile_ratios) { + EXPECT_LE(stats.PercentileRatio(pr.first), pr.second) + << size << " " << pr.first << " " << stats; + } + + for (auto pr : exp.pecentile_probes) { + EXPECT_LE(stats.PercentileProbe(pr.first), pr.second) + << size << " " << pr.first << " " << stats; + } +} + +using ProbeStatsPerSize = std::map<size_t, ProbeStats>; + +// Collect total ProbeStats on num_iters iterations of the following algorithm: +// 1. Create new table and reserve it to keys.size() * 2 +// 2. Insert all keys xored with seed +// 3. Collect ProbeStats from final table. +ProbeStats CollectProbeStatsOnKeysXoredWithSeed( + const std::vector<int64_t>& keys, size_t num_iters) { + const size_t reserve_size = keys.size() * 2; + + ProbeStats stats; + + int64_t seed = 0x71b1a19b907d6e33; + while (num_iters--) { + seed = static_cast<int64_t>(static_cast<uint64_t>(seed) * 17 + 13); + IntTable t1; + t1.reserve(reserve_size); + for (const auto& key : keys) { + t1.emplace(key ^ seed); + } + + auto probe_histogram = GetHashtableDebugNumProbesHistogram(t1); + stats.all_probes_histogram.resize( + std::max(stats.all_probes_histogram.size(), probe_histogram.size())); + std::transform(probe_histogram.begin(), probe_histogram.end(), + stats.all_probes_histogram.begin(), + stats.all_probes_histogram.begin(), std::plus<size_t>()); + + size_t total_probe_seq_length = 0; + for (size_t i = 0; i < probe_histogram.size(); ++i) { + total_probe_seq_length += i * probe_histogram[i]; + } + stats.single_table_ratios.push_back(total_probe_seq_length * 1.0 / + keys.size()); + t1.erase(t1.begin(), t1.end()); + } + return stats; +} + +ExpectedStats XorSeedExpectedStats() { + constexpr bool kRandomizesInserts = +#ifdef NDEBUG + false; +#else // NDEBUG + true; +#endif // NDEBUG + + // The effective load factor is larger in non-opt mode because we insert + // elements out of order. + switch (container_internal::Group::kWidth) { + case 8: + if (kRandomizesInserts) { + return {0.05, + 1.0, + {{0.95, 0.5}}, + {{0.95, 0}, {0.99, 2}, {0.999, 4}, {0.9999, 10}}}; + } else { + return {0.05, + 2.0, + {{0.95, 0.1}}, + {{0.95, 0}, {0.99, 2}, {0.999, 4}, {0.9999, 10}}}; + } + case 16: + if (kRandomizesInserts) { + return {0.1, + 1.0, + {{0.95, 0.1}}, + {{0.95, 0}, {0.99, 1}, {0.999, 8}, {0.9999, 15}}}; + } else { + return {0.05, + 1.0, + {{0.95, 0.05}}, + {{0.95, 0}, {0.99, 1}, {0.999, 4}, {0.9999, 10}}}; + } + } + ABSL_RAW_LOG(FATAL, "%s", "Unknown Group width"); + return {}; +} + +TEST(Table, DISABLED_EnsureNonQuadraticTopNXorSeedByProbeSeqLength) { + ProbeStatsPerSize stats; + std::vector<size_t> sizes = {Group::kWidth << 5, Group::kWidth << 10}; + for (size_t size : sizes) { + stats[size] = + CollectProbeStatsOnKeysXoredWithSeed(CollectBadMergeKeys(size), 200); + } + auto expected = XorSeedExpectedStats(); + for (size_t size : sizes) { + auto& stat = stats[size]; + VerifyStats(size, expected, stat); + } +} + +// Collect total ProbeStats on num_iters iterations of the following algorithm: +// 1. Create new table +// 2. Select 10% of keys and insert 10 elements key * 17 + j * 13 +// 3. Collect ProbeStats from final table +ProbeStats CollectProbeStatsOnLinearlyTransformedKeys( + const std::vector<int64_t>& keys, size_t num_iters) { + ProbeStats stats; + + std::random_device rd; + std::mt19937 rng(rd()); + auto linear_transform = [](size_t x, size_t y) { return x * 17 + y * 13; }; + std::uniform_int_distribution<size_t> dist(0, keys.size()-1); + while (num_iters--) { + IntTable t1; + size_t num_keys = keys.size() / 10; + size_t start = dist(rng); + for (size_t i = 0; i != num_keys; ++i) { + for (size_t j = 0; j != 10; ++j) { + t1.emplace(linear_transform(keys[(i + start) % keys.size()], j)); + } + } + + auto probe_histogram = GetHashtableDebugNumProbesHistogram(t1); + stats.all_probes_histogram.resize( + std::max(stats.all_probes_histogram.size(), probe_histogram.size())); + std::transform(probe_histogram.begin(), probe_histogram.end(), + stats.all_probes_histogram.begin(), + stats.all_probes_histogram.begin(), std::plus<size_t>()); + + size_t total_probe_seq_length = 0; + for (size_t i = 0; i < probe_histogram.size(); ++i) { + total_probe_seq_length += i * probe_histogram[i]; + } + stats.single_table_ratios.push_back(total_probe_seq_length * 1.0 / + t1.size()); + t1.erase(t1.begin(), t1.end()); + } + return stats; +} + +ExpectedStats LinearTransformExpectedStats() { + constexpr bool kRandomizesInserts = +#ifdef NDEBUG + false; +#else // NDEBUG + true; +#endif // NDEBUG + + // The effective load factor is larger in non-opt mode because we insert + // elements out of order. + switch (container_internal::Group::kWidth) { + case 8: + if (kRandomizesInserts) { + return {0.1, + 0.5, + {{0.95, 0.3}}, + {{0.95, 0}, {0.99, 1}, {0.999, 8}, {0.9999, 15}}}; + } else { + return {0.15, + 0.5, + {{0.95, 0.3}}, + {{0.95, 0}, {0.99, 3}, {0.999, 15}, {0.9999, 25}}}; + } + case 16: + if (kRandomizesInserts) { + return {0.1, + 0.4, + {{0.95, 0.3}}, + {{0.95, 0}, {0.99, 1}, {0.999, 8}, {0.9999, 15}}}; + } else { + return {0.05, + 0.2, + {{0.95, 0.1}}, + {{0.95, 0}, {0.99, 1}, {0.999, 6}, {0.9999, 10}}}; + } + } + ABSL_RAW_LOG(FATAL, "%s", "Unknown Group width"); + return {}; +} + +TEST(Table, DISABLED_EnsureNonQuadraticTopNLinearTransformByProbeSeqLength) { + ProbeStatsPerSize stats; + std::vector<size_t> sizes = {Group::kWidth << 5, Group::kWidth << 10}; + for (size_t size : sizes) { + stats[size] = CollectProbeStatsOnLinearlyTransformedKeys( + CollectBadMergeKeys(size), 300); + } + auto expected = LinearTransformExpectedStats(); + for (size_t size : sizes) { + auto& stat = stats[size]; + VerifyStats(size, expected, stat); + } +} + +TEST(Table, EraseCollision) { + BadTable t; + + // 1 2 3 + t.emplace(1); + t.emplace(2); + t.emplace(3); + EXPECT_THAT(*t.find(1), 1); + EXPECT_THAT(*t.find(2), 2); + EXPECT_THAT(*t.find(3), 3); + EXPECT_EQ(3, t.size()); + + // 1 DELETED 3 + t.erase(t.find(2)); + EXPECT_THAT(*t.find(1), 1); + EXPECT_TRUE(t.find(2) == t.end()); + EXPECT_THAT(*t.find(3), 3); + EXPECT_EQ(2, t.size()); + + // DELETED DELETED 3 + t.erase(t.find(1)); + EXPECT_TRUE(t.find(1) == t.end()); + EXPECT_TRUE(t.find(2) == t.end()); + EXPECT_THAT(*t.find(3), 3); + EXPECT_EQ(1, t.size()); + + // DELETED DELETED DELETED + t.erase(t.find(3)); + EXPECT_TRUE(t.find(1) == t.end()); + EXPECT_TRUE(t.find(2) == t.end()); + EXPECT_TRUE(t.find(3) == t.end()); + EXPECT_EQ(0, t.size()); +} + +TEST(Table, EraseInsertProbing) { + BadTable t(100); + + // 1 2 3 4 + t.emplace(1); + t.emplace(2); + t.emplace(3); + t.emplace(4); + + // 1 DELETED 3 DELETED + t.erase(t.find(2)); + t.erase(t.find(4)); + + // 1 10 3 11 12 + t.emplace(10); + t.emplace(11); + t.emplace(12); + + EXPECT_EQ(5, t.size()); + EXPECT_THAT(t, UnorderedElementsAre(1, 10, 3, 11, 12)); +} + +TEST(Table, Clear) { + IntTable t; + EXPECT_TRUE(t.find(0) == t.end()); + t.clear(); + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_EQ(1, t.size()); + t.clear(); + EXPECT_EQ(0, t.size()); + EXPECT_TRUE(t.find(0) == t.end()); +} + +TEST(Table, Swap) { + IntTable t; + EXPECT_TRUE(t.find(0) == t.end()); + auto res = t.emplace(0); + EXPECT_TRUE(res.second); + EXPECT_EQ(1, t.size()); + IntTable u; + t.swap(u); + EXPECT_EQ(0, t.size()); + EXPECT_EQ(1, u.size()); + EXPECT_TRUE(t.find(0) == t.end()); + EXPECT_THAT(*u.find(0), 0); +} + +TEST(Table, Rehash) { + IntTable t; + EXPECT_TRUE(t.find(0) == t.end()); + t.emplace(0); + t.emplace(1); + EXPECT_EQ(2, t.size()); + t.rehash(128); + EXPECT_EQ(2, t.size()); + EXPECT_THAT(*t.find(0), 0); + EXPECT_THAT(*t.find(1), 1); +} + +TEST(Table, RehashDoesNotRehashWhenNotNecessary) { + IntTable t; + t.emplace(0); + t.emplace(1); + auto* p = &*t.find(0); + t.rehash(1); + EXPECT_EQ(p, &*t.find(0)); +} + +TEST(Table, RehashZeroDoesNotAllocateOnEmptyTable) { + IntTable t; + t.rehash(0); + EXPECT_EQ(0, t.bucket_count()); +} + +TEST(Table, RehashZeroDeallocatesEmptyTable) { + IntTable t; + t.emplace(0); + t.clear(); + EXPECT_NE(0, t.bucket_count()); + t.rehash(0); + EXPECT_EQ(0, t.bucket_count()); +} + +TEST(Table, RehashZeroForcesRehash) { + IntTable t; + t.emplace(0); + t.emplace(1); + auto* p = &*t.find(0); + t.rehash(0); + EXPECT_NE(p, &*t.find(0)); +} + +TEST(Table, ConstructFromInitList) { + using P = std::pair<std::string, std::string>; + struct Q { + operator P() const { return {}; } + }; + StringTable t = {P(), Q(), {}, {{}, {}}}; +} + +TEST(Table, CopyConstruct) { + IntTable t; + t.emplace(0); + EXPECT_EQ(1, t.size()); + { + IntTable u(t); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } + { + IntTable u{t}; + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } + { + IntTable u = t; + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find(0), 0); + } +} + +TEST(Table, CopyConstructWithAlloc) { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + StringTable u(t, Alloc<std::pair<std::string, std::string>>()); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); +} + +struct ExplicitAllocIntTable + : raw_hash_set<IntPolicy, container_internal::hash_default_hash<int64_t>, + std::equal_to<int64_t>, Alloc<int64_t>> { + ExplicitAllocIntTable() {} +}; + +TEST(Table, AllocWithExplicitCtor) { + ExplicitAllocIntTable t; + EXPECT_EQ(0, t.size()); +} + +TEST(Table, MoveConstruct) { + { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + + StringTable u(std::move(t)); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); + } + { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + + StringTable u{std::move(t)}; + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); + } + { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + + StringTable u = std::move(t); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); + } +} + +TEST(Table, MoveConstructWithAlloc) { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + StringTable u(std::move(t), Alloc<std::pair<std::string, std::string>>()); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); +} + +TEST(Table, CopyAssign) { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + StringTable u; + u = t; + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); +} + +TEST(Table, CopySelfAssign) { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + t = *&t; + EXPECT_EQ(1, t.size()); + EXPECT_THAT(*t.find("a"), Pair("a", "b")); +} + +TEST(Table, MoveAssign) { + StringTable t; + t.emplace("a", "b"); + EXPECT_EQ(1, t.size()); + StringTable u; + u = std::move(t); + EXPECT_EQ(1, u.size()); + EXPECT_THAT(*u.find("a"), Pair("a", "b")); +} + +TEST(Table, Equality) { + StringTable t; + std::vector<std::pair<std::string, std::string>> v = {{"a", "b"}, + {"aa", "bb"}}; + t.insert(std::begin(v), std::end(v)); + StringTable u = t; + EXPECT_EQ(u, t); +} + +TEST(Table, Equality2) { + StringTable t; + std::vector<std::pair<std::string, std::string>> v1 = {{"a", "b"}, + {"aa", "bb"}}; + t.insert(std::begin(v1), std::end(v1)); + StringTable u; + std::vector<std::pair<std::string, std::string>> v2 = {{"a", "a"}, + {"aa", "aa"}}; + u.insert(std::begin(v2), std::end(v2)); + EXPECT_NE(u, t); +} + +TEST(Table, Equality3) { + StringTable t; + std::vector<std::pair<std::string, std::string>> v1 = {{"b", "b"}, + {"bb", "bb"}}; + t.insert(std::begin(v1), std::end(v1)); + StringTable u; + std::vector<std::pair<std::string, std::string>> v2 = {{"a", "a"}, + {"aa", "aa"}}; + u.insert(std::begin(v2), std::end(v2)); + EXPECT_NE(u, t); +} + +TEST(Table, NumDeletedRegression) { + IntTable t; + t.emplace(0); + t.erase(t.find(0)); + // construct over a deleted slot. + t.emplace(0); + t.clear(); +} + +TEST(Table, FindFullDeletedRegression) { + IntTable t; + for (int i = 0; i < 1000; ++i) { + t.emplace(i); + t.erase(t.find(i)); + } + EXPECT_EQ(0, t.size()); +} + +TEST(Table, ReplacingDeletedSlotDoesNotRehash) { + size_t n; + { + // Compute n such that n is the maximum number of elements before rehash. + IntTable t; + t.emplace(0); + size_t c = t.bucket_count(); + for (n = 1; c == t.bucket_count(); ++n) t.emplace(n); + --n; + } + IntTable t; + t.rehash(n); + const size_t c = t.bucket_count(); + for (size_t i = 0; i != n; ++i) t.emplace(i); + EXPECT_EQ(c, t.bucket_count()) << "rehashing threshold = " << n; + t.erase(0); + t.emplace(0); + EXPECT_EQ(c, t.bucket_count()) << "rehashing threshold = " << n; +} + +TEST(Table, NoThrowMoveConstruct) { + ASSERT_TRUE( + std::is_nothrow_copy_constructible<absl::Hash<absl::string_view>>::value); + ASSERT_TRUE(std::is_nothrow_copy_constructible< + std::equal_to<absl::string_view>>::value); + ASSERT_TRUE(std::is_nothrow_copy_constructible<std::allocator<int>>::value); + EXPECT_TRUE(std::is_nothrow_move_constructible<StringTable>::value); +} + +TEST(Table, NoThrowMoveAssign) { + ASSERT_TRUE( + std::is_nothrow_move_assignable<absl::Hash<absl::string_view>>::value); + ASSERT_TRUE( + std::is_nothrow_move_assignable<std::equal_to<absl::string_view>>::value); + ASSERT_TRUE(std::is_nothrow_move_assignable<std::allocator<int>>::value); + ASSERT_TRUE( + absl::allocator_traits<std::allocator<int>>::is_always_equal::value); + EXPECT_TRUE(std::is_nothrow_move_assignable<StringTable>::value); +} + +TEST(Table, NoThrowSwappable) { + ASSERT_TRUE( + container_internal::IsNoThrowSwappable<absl::Hash<absl::string_view>>()); + ASSERT_TRUE(container_internal::IsNoThrowSwappable< + std::equal_to<absl::string_view>>()); + ASSERT_TRUE(container_internal::IsNoThrowSwappable<std::allocator<int>>()); + EXPECT_TRUE(container_internal::IsNoThrowSwappable<StringTable>()); +} + +TEST(Table, HeterogeneousLookup) { + struct Hash { + size_t operator()(int64_t i) const { return i; } + size_t operator()(double i) const { + ADD_FAILURE(); + return i; + } + }; + struct Eq { + bool operator()(int64_t a, int64_t b) const { return a == b; } + bool operator()(double a, int64_t b) const { + ADD_FAILURE(); + return a == b; + } + bool operator()(int64_t a, double b) const { + ADD_FAILURE(); + return a == b; + } + bool operator()(double a, double b) const { + ADD_FAILURE(); + return a == b; + } + }; + + struct THash { + using is_transparent = void; + size_t operator()(int64_t i) const { return i; } + size_t operator()(double i) const { return i; } + }; + struct TEq { + using is_transparent = void; + bool operator()(int64_t a, int64_t b) const { return a == b; } + bool operator()(double a, int64_t b) const { return a == b; } + bool operator()(int64_t a, double b) const { return a == b; } + bool operator()(double a, double b) const { return a == b; } + }; + + raw_hash_set<IntPolicy, Hash, Eq, Alloc<int64_t>> s{0, 1, 2}; + // It will convert to int64_t before the query. + EXPECT_EQ(1, *s.find(double{1.1})); + + raw_hash_set<IntPolicy, THash, TEq, Alloc<int64_t>> ts{0, 1, 2}; + // It will try to use the double, and fail to find the object. + EXPECT_TRUE(ts.find(1.1) == ts.end()); +} + +template <class Table> +using CallFind = decltype(std::declval<Table&>().find(17)); + +template <class Table> +using CallErase = decltype(std::declval<Table&>().erase(17)); + +template <class Table> +using CallExtract = decltype(std::declval<Table&>().extract(17)); + +template <class Table> +using CallPrefetch = decltype(std::declval<Table&>().prefetch(17)); + +template <class Table> +using CallCount = decltype(std::declval<Table&>().count(17)); + +template <template <typename> class C, class Table, class = void> +struct VerifyResultOf : std::false_type {}; + +template <template <typename> class C, class Table> +struct VerifyResultOf<C, Table, absl::void_t<C<Table>>> : std::true_type {}; + +TEST(Table, HeterogeneousLookupOverloads) { + using NonTransparentTable = + raw_hash_set<StringPolicy, absl::Hash<absl::string_view>, + std::equal_to<absl::string_view>, std::allocator<int>>; + + EXPECT_FALSE((VerifyResultOf<CallFind, NonTransparentTable>())); + EXPECT_FALSE((VerifyResultOf<CallErase, NonTransparentTable>())); + EXPECT_FALSE((VerifyResultOf<CallExtract, NonTransparentTable>())); + EXPECT_FALSE((VerifyResultOf<CallPrefetch, NonTransparentTable>())); + EXPECT_FALSE((VerifyResultOf<CallCount, NonTransparentTable>())); + + using TransparentTable = raw_hash_set< + StringPolicy, + absl::container_internal::hash_default_hash<absl::string_view>, + absl::container_internal::hash_default_eq<absl::string_view>, + std::allocator<int>>; + + EXPECT_TRUE((VerifyResultOf<CallFind, TransparentTable>())); + EXPECT_TRUE((VerifyResultOf<CallErase, TransparentTable>())); + EXPECT_TRUE((VerifyResultOf<CallExtract, TransparentTable>())); + EXPECT_TRUE((VerifyResultOf<CallPrefetch, TransparentTable>())); + EXPECT_TRUE((VerifyResultOf<CallCount, TransparentTable>())); +} + +// TODO(alkis): Expand iterator tests. +TEST(Iterator, IsDefaultConstructible) { + StringTable::iterator i; + EXPECT_TRUE(i == StringTable::iterator()); +} + +TEST(ConstIterator, IsDefaultConstructible) { + StringTable::const_iterator i; + EXPECT_TRUE(i == StringTable::const_iterator()); +} + +TEST(Iterator, ConvertsToConstIterator) { + StringTable::iterator i; + EXPECT_TRUE(i == StringTable::const_iterator()); +} + +TEST(Iterator, Iterates) { + IntTable t; + for (size_t i = 3; i != 6; ++i) EXPECT_TRUE(t.emplace(i).second); + EXPECT_THAT(t, UnorderedElementsAre(3, 4, 5)); +} + +TEST(Table, Merge) { + StringTable t1, t2; + t1.emplace("0", "-0"); + t1.emplace("1", "-1"); + t2.emplace("0", "~0"); + t2.emplace("2", "~2"); + + EXPECT_THAT(t1, UnorderedElementsAre(Pair("0", "-0"), Pair("1", "-1"))); + EXPECT_THAT(t2, UnorderedElementsAre(Pair("0", "~0"), Pair("2", "~2"))); + + t1.merge(t2); + EXPECT_THAT(t1, UnorderedElementsAre(Pair("0", "-0"), Pair("1", "-1"), + Pair("2", "~2"))); + EXPECT_THAT(t2, UnorderedElementsAre(Pair("0", "~0"))); +} + +TEST(Nodes, EmptyNodeType) { + using node_type = StringTable::node_type; + node_type n; + EXPECT_FALSE(n); + EXPECT_TRUE(n.empty()); + + EXPECT_TRUE((std::is_same<node_type::allocator_type, + StringTable::allocator_type>::value)); +} + +TEST(Nodes, ExtractInsert) { + constexpr char k0[] = "Very long string zero."; + constexpr char k1[] = "Very long string one."; + constexpr char k2[] = "Very long string two."; + StringTable t = {{k0, ""}, {k1, ""}, {k2, ""}}; + EXPECT_THAT(t, + UnorderedElementsAre(Pair(k0, ""), Pair(k1, ""), Pair(k2, ""))); + + auto node = t.extract(k0); + EXPECT_THAT(t, UnorderedElementsAre(Pair(k1, ""), Pair(k2, ""))); + EXPECT_TRUE(node); + EXPECT_FALSE(node.empty()); + + StringTable t2; + StringTable::insert_return_type res = t2.insert(std::move(node)); + EXPECT_TRUE(res.inserted); + EXPECT_THAT(*res.position, Pair(k0, "")); + EXPECT_FALSE(res.node); + EXPECT_THAT(t2, UnorderedElementsAre(Pair(k0, ""))); + + // Not there. + EXPECT_THAT(t, UnorderedElementsAre(Pair(k1, ""), Pair(k2, ""))); + node = t.extract("Not there!"); + EXPECT_THAT(t, UnorderedElementsAre(Pair(k1, ""), Pair(k2, ""))); + EXPECT_FALSE(node); + + // Inserting nothing. + res = t2.insert(std::move(node)); + EXPECT_FALSE(res.inserted); + EXPECT_EQ(res.position, t2.end()); + EXPECT_FALSE(res.node); + EXPECT_THAT(t2, UnorderedElementsAre(Pair(k0, ""))); + + t.emplace(k0, "1"); + node = t.extract(k0); + + // Insert duplicate. + res = t2.insert(std::move(node)); + EXPECT_FALSE(res.inserted); + EXPECT_THAT(*res.position, Pair(k0, "")); + EXPECT_TRUE(res.node); + EXPECT_FALSE(node); +} + +TEST(Nodes, HintInsert) { + IntTable t = {1, 2, 3}; + auto node = t.extract(1); + EXPECT_THAT(t, UnorderedElementsAre(2, 3)); + auto it = t.insert(t.begin(), std::move(node)); + EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); + EXPECT_EQ(*it, 1); + EXPECT_FALSE(node); + + node = t.extract(2); + EXPECT_THAT(t, UnorderedElementsAre(1, 3)); + // reinsert 2 to make the next insert fail. + t.insert(2); + EXPECT_THAT(t, UnorderedElementsAre(1, 2, 3)); + it = t.insert(t.begin(), std::move(node)); + EXPECT_EQ(*it, 2); + // The node was not emptied by the insert call. + EXPECT_TRUE(node); +} + +IntTable MakeSimpleTable(size_t size) { + IntTable t; + while (t.size() < size) t.insert(t.size()); + return t; +} + +std::vector<int> OrderOfIteration(const IntTable& t) { + return {t.begin(), t.end()}; +} + +// These IterationOrderChanges tests depend on non-deterministic behavior. +// We are injecting non-determinism from the pointer of the table, but do so in +// a way that only the page matters. We have to retry enough times to make sure +// we are touching different memory pages to cause the ordering to change. +// We also need to keep the old tables around to avoid getting the same memory +// blocks over and over. +TEST(Table, IterationOrderChangesByInstance) { + for (size_t size : {2, 6, 12, 20}) { + const auto reference_table = MakeSimpleTable(size); + const auto reference = OrderOfIteration(reference_table); + + std::vector<IntTable> tables; + bool found_difference = false; + for (int i = 0; !found_difference && i < 5000; ++i) { + tables.push_back(MakeSimpleTable(size)); + found_difference = OrderOfIteration(tables.back()) != reference; + } + if (!found_difference) { + FAIL() + << "Iteration order remained the same across many attempts with size " + << size; + } + } +} + +TEST(Table, IterationOrderChangesOnRehash) { + std::vector<IntTable> garbage; + for (int i = 0; i < 5000; ++i) { + auto t = MakeSimpleTable(20); + const auto reference = OrderOfIteration(t); + // Force rehash to the same size. + t.rehash(0); + auto trial = OrderOfIteration(t); + if (trial != reference) { + // We are done. + return; + } + garbage.push_back(std::move(t)); + } + FAIL() << "Iteration order remained the same across many attempts."; +} + +// Verify that pointers are invalidated as soon as a second element is inserted. +// This prevents dependency on pointer stability on small tables. +TEST(Table, UnstablePointers) { + IntTable table; + + const auto addr = [&](int i) { + return reinterpret_cast<uintptr_t>(&*table.find(i)); + }; + + table.insert(0); + const uintptr_t old_ptr = addr(0); + + // This causes a rehash. + table.insert(1); + + EXPECT_NE(old_ptr, addr(0)); +} + +// Confirm that we assert if we try to erase() end(). +TEST(TableDeathTest, EraseOfEndAsserts) { + // Use an assert with side-effects to figure out if they are actually enabled. + bool assert_enabled = false; + assert([&]() { + assert_enabled = true; + return true; + }()); + if (!assert_enabled) return; + + IntTable t; + // Extra simple "regexp" as regexp support is highly varied across platforms. + constexpr char kDeathMsg[] = "Invalid operation on iterator"; + EXPECT_DEATH_IF_SUPPORTED(t.erase(t.end()), kDeathMsg); +} + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +TEST(RawHashSamplerTest, Sample) { + // Enable the feature even if the prod default is off. + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); + + auto& sampler = HashtablezSampler::Global(); + size_t start_size = 0; + start_size += sampler.Iterate([&](const HashtablezInfo&) { ++start_size; }); + + std::vector<IntTable> tables; + for (int i = 0; i < 1000000; ++i) { + tables.emplace_back(); + tables.back().insert(1); + } + size_t end_size = 0; + end_size += sampler.Iterate([&](const HashtablezInfo&) { ++end_size; }); + + EXPECT_NEAR((end_size - start_size) / static_cast<double>(tables.size()), + 0.01, 0.005); +} +#endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE + +TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { + // Enable the feature even if the prod default is off. + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(100); + + auto& sampler = HashtablezSampler::Global(); + size_t start_size = 0; + start_size += sampler.Iterate([&](const HashtablezInfo&) { ++start_size; }); + + std::vector<CustomAllocIntTable> tables; + for (int i = 0; i < 1000000; ++i) { + tables.emplace_back(); + tables.back().insert(1); + } + size_t end_size = 0; + end_size += sampler.Iterate([&](const HashtablezInfo&) { ++end_size; }); + + EXPECT_NEAR((end_size - start_size) / static_cast<double>(tables.size()), + 0.00, 0.001); +} + +#ifdef ABSL_HAVE_ADDRESS_SANITIZER +TEST(Sanitizer, PoisoningUnused) { + IntTable t; + t.reserve(5); + // Insert something to force an allocation. + int64_t& v1 = *t.insert(0).first; + + // Make sure there is something to test. + ASSERT_GT(t.capacity(), 1); + + int64_t* slots = RawHashSetTestOnlyAccess::GetSlots(t); + for (size_t i = 0; i < t.capacity(); ++i) { + EXPECT_EQ(slots + i != &v1, __asan_address_is_poisoned(slots + i)); + } +} + +TEST(Sanitizer, PoisoningOnErase) { + IntTable t; + int64_t& v = *t.insert(0).first; + + EXPECT_FALSE(__asan_address_is_poisoned(&v)); + t.erase(0); + EXPECT_TRUE(__asan_address_is_poisoned(&v)); +} +#endif // ABSL_HAVE_ADDRESS_SANITIZER + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/test_instance_tracker.cc b/third_party/abseil_cpp/absl/container/internal/test_instance_tracker.cc new file mode 100644 index 000000000000..f9947f0475d2 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/test_instance_tracker.cc @@ -0,0 +1,29 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/test_instance_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace test_internal { +int BaseCountedInstance::num_instances_ = 0; +int BaseCountedInstance::num_live_instances_ = 0; +int BaseCountedInstance::num_moves_ = 0; +int BaseCountedInstance::num_copies_ = 0; +int BaseCountedInstance::num_swaps_ = 0; +int BaseCountedInstance::num_comparisons_ = 0; + +} // namespace test_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/test_instance_tracker.h b/third_party/abseil_cpp/absl/container/internal/test_instance_tracker.h new file mode 100644 index 000000000000..5ff6fd714e2b --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/test_instance_tracker.h @@ -0,0 +1,274 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_TEST_INSTANCE_TRACKER_H_ +#define ABSL_CONTAINER_INTERNAL_TEST_INSTANCE_TRACKER_H_ + +#include <cstdlib> +#include <ostream> + +#include "absl/types/compare.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace test_internal { + +// A type that counts number of occurrences of the type, the live occurrences of +// the type, as well as the number of copies, moves, swaps, and comparisons that +// have occurred on the type. This is used as a base class for the copyable, +// copyable+movable, and movable types below that are used in actual tests. Use +// InstanceTracker in tests to track the number of instances. +class BaseCountedInstance { + public: + explicit BaseCountedInstance(int x) : value_(x) { + ++num_instances_; + ++num_live_instances_; + } + BaseCountedInstance(const BaseCountedInstance& x) + : value_(x.value_), is_live_(x.is_live_) { + ++num_instances_; + if (is_live_) ++num_live_instances_; + ++num_copies_; + } + BaseCountedInstance(BaseCountedInstance&& x) + : value_(x.value_), is_live_(x.is_live_) { + x.is_live_ = false; + ++num_instances_; + ++num_moves_; + } + ~BaseCountedInstance() { + --num_instances_; + if (is_live_) --num_live_instances_; + } + + BaseCountedInstance& operator=(const BaseCountedInstance& x) { + value_ = x.value_; + if (is_live_) --num_live_instances_; + is_live_ = x.is_live_; + if (is_live_) ++num_live_instances_; + ++num_copies_; + return *this; + } + BaseCountedInstance& operator=(BaseCountedInstance&& x) { + value_ = x.value_; + if (is_live_) --num_live_instances_; + is_live_ = x.is_live_; + x.is_live_ = false; + ++num_moves_; + return *this; + } + + bool operator==(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ == x.value_; + } + + bool operator!=(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ != x.value_; + } + + bool operator<(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ < x.value_; + } + + bool operator>(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ > x.value_; + } + + bool operator<=(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ <= x.value_; + } + + bool operator>=(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ >= x.value_; + } + + absl::weak_ordering compare(const BaseCountedInstance& x) const { + ++num_comparisons_; + return value_ < x.value_ + ? absl::weak_ordering::less + : value_ == x.value_ ? absl::weak_ordering::equivalent + : absl::weak_ordering::greater; + } + + int value() const { + if (!is_live_) std::abort(); + return value_; + } + + friend std::ostream& operator<<(std::ostream& o, + const BaseCountedInstance& v) { + return o << "[value:" << v.value() << "]"; + } + + // Implementation of efficient swap() that counts swaps. + static void SwapImpl( + BaseCountedInstance& lhs, // NOLINT(runtime/references) + BaseCountedInstance& rhs) { // NOLINT(runtime/references) + using std::swap; + swap(lhs.value_, rhs.value_); + swap(lhs.is_live_, rhs.is_live_); + ++BaseCountedInstance::num_swaps_; + } + + private: + friend class InstanceTracker; + + int value_; + + // Indicates if the value is live, ie it hasn't been moved away from. + bool is_live_ = true; + + // Number of instances. + static int num_instances_; + + // Number of live instances (those that have not been moved away from.) + static int num_live_instances_; + + // Number of times that BaseCountedInstance objects were moved. + static int num_moves_; + + // Number of times that BaseCountedInstance objects were copied. + static int num_copies_; + + // Number of times that BaseCountedInstance objects were swapped. + static int num_swaps_; + + // Number of times that BaseCountedInstance objects were compared. + static int num_comparisons_; +}; + +// Helper to track the BaseCountedInstance instance counters. Expects that the +// number of instances and live_instances are the same when it is constructed +// and when it is destructed. +class InstanceTracker { + public: + InstanceTracker() + : start_instances_(BaseCountedInstance::num_instances_), + start_live_instances_(BaseCountedInstance::num_live_instances_) { + ResetCopiesMovesSwaps(); + } + ~InstanceTracker() { + if (instances() != 0) std::abort(); + if (live_instances() != 0) std::abort(); + } + + // Returns the number of BaseCountedInstance instances both containing valid + // values and those moved away from compared to when the InstanceTracker was + // constructed + int instances() const { + return BaseCountedInstance::num_instances_ - start_instances_; + } + + // Returns the number of live BaseCountedInstance instances compared to when + // the InstanceTracker was constructed + int live_instances() const { + return BaseCountedInstance::num_live_instances_ - start_live_instances_; + } + + // Returns the number of moves on BaseCountedInstance objects since + // construction or since the last call to ResetCopiesMovesSwaps(). + int moves() const { return BaseCountedInstance::num_moves_ - start_moves_; } + + // Returns the number of copies on BaseCountedInstance objects since + // construction or the last call to ResetCopiesMovesSwaps(). + int copies() const { + return BaseCountedInstance::num_copies_ - start_copies_; + } + + // Returns the number of swaps on BaseCountedInstance objects since + // construction or the last call to ResetCopiesMovesSwaps(). + int swaps() const { return BaseCountedInstance::num_swaps_ - start_swaps_; } + + // Returns the number of comparisons on BaseCountedInstance objects since + // construction or the last call to ResetCopiesMovesSwaps(). + int comparisons() const { + return BaseCountedInstance::num_comparisons_ - start_comparisons_; + } + + // Resets the base values for moves, copies, comparisons, and swaps to the + // current values, so that subsequent Get*() calls for moves, copies, + // comparisons, and swaps will compare to the situation at the point of this + // call. + void ResetCopiesMovesSwaps() { + start_moves_ = BaseCountedInstance::num_moves_; + start_copies_ = BaseCountedInstance::num_copies_; + start_swaps_ = BaseCountedInstance::num_swaps_; + start_comparisons_ = BaseCountedInstance::num_comparisons_; + } + + private: + int start_instances_; + int start_live_instances_; + int start_moves_; + int start_copies_; + int start_swaps_; + int start_comparisons_; +}; + +// Copyable, not movable. +class CopyableOnlyInstance : public BaseCountedInstance { + public: + explicit CopyableOnlyInstance(int x) : BaseCountedInstance(x) {} + CopyableOnlyInstance(const CopyableOnlyInstance& rhs) = default; + CopyableOnlyInstance& operator=(const CopyableOnlyInstance& rhs) = default; + + friend void swap(CopyableOnlyInstance& lhs, CopyableOnlyInstance& rhs) { + BaseCountedInstance::SwapImpl(lhs, rhs); + } + + static bool supports_move() { return false; } +}; + +// Copyable and movable. +class CopyableMovableInstance : public BaseCountedInstance { + public: + explicit CopyableMovableInstance(int x) : BaseCountedInstance(x) {} + CopyableMovableInstance(const CopyableMovableInstance& rhs) = default; + CopyableMovableInstance(CopyableMovableInstance&& rhs) = default; + CopyableMovableInstance& operator=(const CopyableMovableInstance& rhs) = + default; + CopyableMovableInstance& operator=(CopyableMovableInstance&& rhs) = default; + + friend void swap(CopyableMovableInstance& lhs, CopyableMovableInstance& rhs) { + BaseCountedInstance::SwapImpl(lhs, rhs); + } + + static bool supports_move() { return true; } +}; + +// Only movable, not default-constructible. +class MovableOnlyInstance : public BaseCountedInstance { + public: + explicit MovableOnlyInstance(int x) : BaseCountedInstance(x) {} + MovableOnlyInstance(MovableOnlyInstance&& other) = default; + MovableOnlyInstance& operator=(MovableOnlyInstance&& other) = default; + + friend void swap(MovableOnlyInstance& lhs, MovableOnlyInstance& rhs) { + BaseCountedInstance::SwapImpl(lhs, rhs); + } + + static bool supports_move() { return true; } +}; + +} // namespace test_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_TEST_INSTANCE_TRACKER_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/test_instance_tracker_test.cc b/third_party/abseil_cpp/absl/container/internal/test_instance_tracker_test.cc new file mode 100644 index 000000000000..1c6a4fa7150d --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/test_instance_tracker_test.cc @@ -0,0 +1,184 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/internal/test_instance_tracker.h" + +#include "gtest/gtest.h" + +namespace { + +using absl::test_internal::CopyableMovableInstance; +using absl::test_internal::CopyableOnlyInstance; +using absl::test_internal::InstanceTracker; +using absl::test_internal::MovableOnlyInstance; + +TEST(TestInstanceTracker, CopyableMovable) { + InstanceTracker tracker; + CopyableMovableInstance src(1); + EXPECT_EQ(1, src.value()) << src; + CopyableMovableInstance copy(src); + CopyableMovableInstance move(std::move(src)); + EXPECT_EQ(1, tracker.copies()); + EXPECT_EQ(1, tracker.moves()); + EXPECT_EQ(0, tracker.swaps()); + EXPECT_EQ(3, tracker.instances()); + EXPECT_EQ(2, tracker.live_instances()); + tracker.ResetCopiesMovesSwaps(); + + CopyableMovableInstance copy_assign(1); + copy_assign = copy; + CopyableMovableInstance move_assign(1); + move_assign = std::move(move); + EXPECT_EQ(1, tracker.copies()); + EXPECT_EQ(1, tracker.moves()); + EXPECT_EQ(0, tracker.swaps()); + EXPECT_EQ(5, tracker.instances()); + EXPECT_EQ(3, tracker.live_instances()); + tracker.ResetCopiesMovesSwaps(); + + { + using std::swap; + swap(move_assign, copy); + swap(copy, move_assign); + EXPECT_EQ(2, tracker.swaps()); + EXPECT_EQ(0, tracker.copies()); + EXPECT_EQ(0, tracker.moves()); + EXPECT_EQ(5, tracker.instances()); + EXPECT_EQ(3, tracker.live_instances()); + } +} + +TEST(TestInstanceTracker, CopyableOnly) { + InstanceTracker tracker; + CopyableOnlyInstance src(1); + EXPECT_EQ(1, src.value()) << src; + CopyableOnlyInstance copy(src); + CopyableOnlyInstance copy2(std::move(src)); // NOLINT + EXPECT_EQ(2, tracker.copies()); + EXPECT_EQ(0, tracker.moves()); + EXPECT_EQ(3, tracker.instances()); + EXPECT_EQ(3, tracker.live_instances()); + tracker.ResetCopiesMovesSwaps(); + + CopyableOnlyInstance copy_assign(1); + copy_assign = copy; + CopyableOnlyInstance copy_assign2(1); + copy_assign2 = std::move(copy2); // NOLINT + EXPECT_EQ(2, tracker.copies()); + EXPECT_EQ(0, tracker.moves()); + EXPECT_EQ(5, tracker.instances()); + EXPECT_EQ(5, tracker.live_instances()); + tracker.ResetCopiesMovesSwaps(); + + { + using std::swap; + swap(src, copy); + swap(copy, src); + EXPECT_EQ(2, tracker.swaps()); + EXPECT_EQ(0, tracker.copies()); + EXPECT_EQ(0, tracker.moves()); + EXPECT_EQ(5, tracker.instances()); + EXPECT_EQ(5, tracker.live_instances()); + } +} + +TEST(TestInstanceTracker, MovableOnly) { + InstanceTracker tracker; + MovableOnlyInstance src(1); + EXPECT_EQ(1, src.value()) << src; + MovableOnlyInstance move(std::move(src)); + MovableOnlyInstance move_assign(2); + move_assign = std::move(move); + EXPECT_EQ(3, tracker.instances()); + EXPECT_EQ(1, tracker.live_instances()); + EXPECT_EQ(2, tracker.moves()); + EXPECT_EQ(0, tracker.copies()); + tracker.ResetCopiesMovesSwaps(); + + { + using std::swap; + MovableOnlyInstance other(2); + swap(move_assign, other); + swap(other, move_assign); + EXPECT_EQ(2, tracker.swaps()); + EXPECT_EQ(0, tracker.copies()); + EXPECT_EQ(0, tracker.moves()); + EXPECT_EQ(4, tracker.instances()); + EXPECT_EQ(2, tracker.live_instances()); + } +} + +TEST(TestInstanceTracker, ExistingInstances) { + CopyableMovableInstance uncounted_instance(1); + CopyableMovableInstance uncounted_live_instance( + std::move(uncounted_instance)); + InstanceTracker tracker; + EXPECT_EQ(0, tracker.instances()); + EXPECT_EQ(0, tracker.live_instances()); + EXPECT_EQ(0, tracker.copies()); + { + CopyableMovableInstance instance1(1); + EXPECT_EQ(1, tracker.instances()); + EXPECT_EQ(1, tracker.live_instances()); + EXPECT_EQ(0, tracker.copies()); + EXPECT_EQ(0, tracker.moves()); + { + InstanceTracker tracker2; + CopyableMovableInstance instance2(instance1); + CopyableMovableInstance instance3(std::move(instance2)); + EXPECT_EQ(3, tracker.instances()); + EXPECT_EQ(2, tracker.live_instances()); + EXPECT_EQ(1, tracker.copies()); + EXPECT_EQ(1, tracker.moves()); + EXPECT_EQ(2, tracker2.instances()); + EXPECT_EQ(1, tracker2.live_instances()); + EXPECT_EQ(1, tracker2.copies()); + EXPECT_EQ(1, tracker2.moves()); + } + EXPECT_EQ(1, tracker.instances()); + EXPECT_EQ(1, tracker.live_instances()); + EXPECT_EQ(1, tracker.copies()); + EXPECT_EQ(1, tracker.moves()); + } + EXPECT_EQ(0, tracker.instances()); + EXPECT_EQ(0, tracker.live_instances()); + EXPECT_EQ(1, tracker.copies()); + EXPECT_EQ(1, tracker.moves()); +} + +TEST(TestInstanceTracker, Comparisons) { + InstanceTracker tracker; + MovableOnlyInstance one(1), two(2); + + EXPECT_EQ(0, tracker.comparisons()); + EXPECT_FALSE(one == two); + EXPECT_EQ(1, tracker.comparisons()); + EXPECT_TRUE(one != two); + EXPECT_EQ(2, tracker.comparisons()); + EXPECT_TRUE(one < two); + EXPECT_EQ(3, tracker.comparisons()); + EXPECT_FALSE(one > two); + EXPECT_EQ(4, tracker.comparisons()); + EXPECT_TRUE(one <= two); + EXPECT_EQ(5, tracker.comparisons()); + EXPECT_FALSE(one >= two); + EXPECT_EQ(6, tracker.comparisons()); + EXPECT_TRUE(one.compare(two) < 0); // NOLINT + EXPECT_EQ(7, tracker.comparisons()); + + tracker.ResetCopiesMovesSwaps(); + EXPECT_EQ(0, tracker.comparisons()); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/container/internal/tracked.h b/third_party/abseil_cpp/absl/container/internal/tracked.h new file mode 100644 index 000000000000..29f5829f7199 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/tracked.h @@ -0,0 +1,83 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_TRACKED_H_ +#define ABSL_CONTAINER_INTERNAL_TRACKED_H_ + +#include <stddef.h> + +#include <memory> +#include <utility> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +// A class that tracks its copies and moves so that it can be queried in tests. +template <class T> +class Tracked { + public: + Tracked() {} + // NOLINTNEXTLINE(runtime/explicit) + Tracked(const T& val) : val_(val) {} + Tracked(const Tracked& that) + : val_(that.val_), + num_moves_(that.num_moves_), + num_copies_(that.num_copies_) { + ++(*num_copies_); + } + Tracked(Tracked&& that) + : val_(std::move(that.val_)), + num_moves_(std::move(that.num_moves_)), + num_copies_(std::move(that.num_copies_)) { + ++(*num_moves_); + } + Tracked& operator=(const Tracked& that) { + val_ = that.val_; + num_moves_ = that.num_moves_; + num_copies_ = that.num_copies_; + ++(*num_copies_); + } + Tracked& operator=(Tracked&& that) { + val_ = std::move(that.val_); + num_moves_ = std::move(that.num_moves_); + num_copies_ = std::move(that.num_copies_); + ++(*num_moves_); + } + + const T& val() const { return val_; } + + friend bool operator==(const Tracked& a, const Tracked& b) { + return a.val_ == b.val_; + } + friend bool operator!=(const Tracked& a, const Tracked& b) { + return !(a == b); + } + + size_t num_copies() { return *num_copies_; } + size_t num_moves() { return *num_moves_; } + + private: + T val_; + std::shared_ptr<size_t> num_moves_ = std::make_shared<size_t>(0); + std::shared_ptr<size_t> num_copies_ = std::make_shared<size_t>(0); +}; + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_TRACKED_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_map_constructor_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_map_constructor_test.h new file mode 100644 index 000000000000..76ee95e6abc5 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_map_constructor_test.h @@ -0,0 +1,489 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_CONSTRUCTOR_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_CONSTRUCTOR_TEST_H_ + +#include <algorithm> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordMap> +class ConstructorTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(ConstructorTest); + +TYPED_TEST_P(ConstructorTest, NoArgs) { + TypeParam m; + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); +} + +TYPED_TEST_P(ConstructorTest, BucketCount) { + TypeParam m(123); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHash) { + using H = typename TypeParam::hasher; + H hasher; + TypeParam m(123, hasher); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHashEqual) { + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + H hasher; + E equal; + TypeParam m(123, hasher, equal); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHashEqualAlloc) { + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +template <typename T> +struct is_std_unordered_map : std::false_type {}; + +template <typename... T> +struct is_std_unordered_map<std::unordered_map<T...>> : std::true_type {}; + +#if defined(UNORDERED_MAP_CXX14) || defined(UNORDERED_MAP_CXX17) +using has_cxx14_std_apis = std::true_type; +#else +using has_cxx14_std_apis = std::false_type; +#endif + +template <typename T> +using expect_cxx14_apis = + absl::disjunction<absl::negation<is_std_unordered_map<T>>, + has_cxx14_std_apis>; + +template <typename TypeParam> +void BucketCountAllocTest(std::false_type) {} + +template <typename TypeParam> +void BucketCountAllocTest(std::true_type) { + using A = typename TypeParam::allocator_type; + A alloc(0); + TypeParam m(123, alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountAlloc) { + BucketCountAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +template <typename TypeParam> +void BucketCountHashAllocTest(std::false_type) {} + +template <typename TypeParam> +void BucketCountHashAllocTest(std::true_type) { + using H = typename TypeParam::hasher; + using A = typename TypeParam::allocator_type; + H hasher; + A alloc(0); + TypeParam m(123, hasher, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHashAlloc) { + BucketCountHashAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +#if ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS +using has_alloc_std_constructors = std::true_type; +#else +using has_alloc_std_constructors = std::false_type; +#endif + +template <typename T> +using expect_alloc_constructors = + absl::disjunction<absl::negation<is_std_unordered_map<T>>, + has_alloc_std_constructors>; + +template <typename TypeParam> +void AllocTest(std::false_type) {} + +template <typename TypeParam> +void AllocTest(std::true_type) { + using A = typename TypeParam::allocator_type; + A alloc(0); + TypeParam m(alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(m, ::testing::UnorderedElementsAre()); +} + +TYPED_TEST_P(ConstructorTest, Alloc) { + AllocTest<TypeParam>(expect_alloc_constructors<TypeParam>()); +} + +TYPED_TEST_P(ConstructorTest, InputIteratorBucketHashEqualAlloc) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end(), 123, hasher, equal, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +template <typename TypeParam> +void InputIteratorBucketAllocTest(std::false_type) {} + +template <typename TypeParam> +void InputIteratorBucketAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using A = typename TypeParam::allocator_type; + A alloc(0); + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end(), 123, alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InputIteratorBucketAlloc) { + InputIteratorBucketAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +template <typename TypeParam> +void InputIteratorBucketHashAllocTest(std::false_type) {} + +template <typename TypeParam> +void InputIteratorBucketHashAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using A = typename TypeParam::allocator_type; + H hasher; + A alloc(0); + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end(), 123, hasher, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InputIteratorBucketHashAlloc) { + InputIteratorBucketHashAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +TYPED_TEST_P(ConstructorTest, CopyConstructor) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam n(m); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +template <typename TypeParam> +void CopyConstructorAllocTest(std::false_type) {} + +template <typename TypeParam> +void CopyConstructorAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam n(m, A(11)); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_NE(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, CopyConstructorAlloc) { + CopyConstructorAllocTest<TypeParam>(expect_alloc_constructors<TypeParam>()); +} + +// TODO(alkis): Test non-propagating allocators on copy constructors. + +TYPED_TEST_P(ConstructorTest, MoveConstructor) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam t(m); + TypeParam n(std::move(t)); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +template <typename TypeParam> +void MoveConstructorAllocTest(std::false_type) {} + +template <typename TypeParam> +void MoveConstructorAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam t(m); + TypeParam n(std::move(t), A(1)); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_NE(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, MoveConstructorAlloc) { + MoveConstructorAllocTest<TypeParam>(expect_alloc_constructors<TypeParam>()); +} + +// TODO(alkis): Test non-propagating allocators on move constructors. + +TYPED_TEST_P(ConstructorTest, InitializerListBucketHashEqualAlloc) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(values, 123, hasher, equal, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +template <typename TypeParam> +void InitializerListBucketAllocTest(std::false_type) {} + +template <typename TypeParam> +void InitializerListBucketAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using A = typename TypeParam::allocator_type; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + A alloc(0); + TypeParam m(values, 123, alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InitializerListBucketAlloc) { + InitializerListBucketAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +template <typename TypeParam> +void InitializerListBucketHashAllocTest(std::false_type) {} + +template <typename TypeParam> +void InitializerListBucketHashAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using A = typename TypeParam::allocator_type; + H hasher; + A alloc(0); + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m(values, 123, hasher, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InitializerListBucketHashAlloc) { + InitializerListBucketHashAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +TYPED_TEST_P(ConstructorTest, Assignment) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); + TypeParam n; + n = m; + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m, n); +} + +// TODO(alkis): Test [non-]propagating allocators on move/copy assignments +// (it depends on traits). + +TYPED_TEST_P(ConstructorTest, MoveAssignment) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); + TypeParam t(m); + TypeParam n; + n = std::move(t); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerList) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m; + m = values; + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); +} + +TYPED_TEST_P(ConstructorTest, AssignmentOverwritesExisting) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}); + TypeParam n({gen()}); + n = m; + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, MoveAssignmentOverwritesExisting) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}); + TypeParam t(m); + TypeParam n({gen()}); + n = std::move(t); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerListOverwritesExisting) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m; + m = values; + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); +} + +TYPED_TEST_P(ConstructorTest, AssignmentOnSelf) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m(values); + m = *&m; // Avoid -Wself-assign + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); +} + +// We cannot test self move as standard states that it leaves standard +// containers in unspecified state (and in practice in causes memory-leak +// according to heap-checker!). + +REGISTER_TYPED_TEST_CASE_P( + ConstructorTest, NoArgs, BucketCount, BucketCountHash, BucketCountHashEqual, + BucketCountHashEqualAlloc, BucketCountAlloc, BucketCountHashAlloc, Alloc, + InputIteratorBucketHashEqualAlloc, InputIteratorBucketAlloc, + InputIteratorBucketHashAlloc, CopyConstructor, CopyConstructorAlloc, + MoveConstructor, MoveConstructorAlloc, InitializerListBucketHashEqualAlloc, + InitializerListBucketAlloc, InitializerListBucketHashAlloc, Assignment, + MoveAssignment, AssignmentFromInitializerList, AssignmentOverwritesExisting, + MoveAssignmentOverwritesExisting, + AssignmentFromInitializerListOverwritesExisting, AssignmentOnSelf); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_CONSTRUCTOR_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_map_lookup_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_map_lookup_test.h new file mode 100644 index 000000000000..e76421e508fe --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_map_lookup_test.h @@ -0,0 +1,117 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_LOOKUP_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_LOOKUP_TEST_H_ + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordMap> +class LookupTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(LookupTest); + +TYPED_TEST_P(LookupTest, At) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + for (const auto& p : values) { + const auto& val = m.at(p.first); + EXPECT_EQ(p.second, val) << ::testing::PrintToString(p.first); + } +} + +TYPED_TEST_P(LookupTest, OperatorBracket) { + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& p : values) { + auto& val = m[p.first]; + EXPECT_EQ(V(), val) << ::testing::PrintToString(p.first); + val = p.second; + } + for (const auto& p : values) + EXPECT_EQ(p.second, m[p.first]) << ::testing::PrintToString(p.first); +} + +TYPED_TEST_P(LookupTest, Count) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& p : values) + EXPECT_EQ(0, m.count(p.first)) << ::testing::PrintToString(p.first); + m.insert(values.begin(), values.end()); + for (const auto& p : values) + EXPECT_EQ(1, m.count(p.first)) << ::testing::PrintToString(p.first); +} + +TYPED_TEST_P(LookupTest, Find) { + using std::get; + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& p : values) + EXPECT_TRUE(m.end() == m.find(p.first)) + << ::testing::PrintToString(p.first); + m.insert(values.begin(), values.end()); + for (const auto& p : values) { + auto it = m.find(p.first); + EXPECT_TRUE(m.end() != it) << ::testing::PrintToString(p.first); + EXPECT_EQ(p.second, get<1>(*it)) << ::testing::PrintToString(p.first); + } +} + +TYPED_TEST_P(LookupTest, EqualRange) { + using std::get; + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& p : values) { + auto r = m.equal_range(p.first); + ASSERT_EQ(0, std::distance(r.first, r.second)); + } + m.insert(values.begin(), values.end()); + for (const auto& p : values) { + auto r = m.equal_range(p.first); + ASSERT_EQ(1, std::distance(r.first, r.second)); + EXPECT_EQ(p.second, get<1>(*r.first)) << ::testing::PrintToString(p.first); + } +} + +REGISTER_TYPED_TEST_CASE_P(LookupTest, At, OperatorBracket, Count, Find, + EqualRange); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_LOOKUP_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_map_members_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_map_members_test.h new file mode 100644 index 000000000000..7d48cdb890bb --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_map_members_test.h @@ -0,0 +1,87 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_MEMBERS_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_MEMBERS_TEST_H_ + +#include <type_traits> +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordMap> +class MembersTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(MembersTest); + +template <typename T> +void UseType() {} + +TYPED_TEST_P(MembersTest, Typedefs) { + EXPECT_TRUE((std::is_same<std::pair<const typename TypeParam::key_type, + typename TypeParam::mapped_type>, + typename TypeParam::value_type>())); + EXPECT_TRUE((absl::conjunction< + absl::negation<std::is_signed<typename TypeParam::size_type>>, + std::is_integral<typename TypeParam::size_type>>())); + EXPECT_TRUE((absl::conjunction< + std::is_signed<typename TypeParam::difference_type>, + std::is_integral<typename TypeParam::difference_type>>())); + EXPECT_TRUE((std::is_convertible< + decltype(std::declval<const typename TypeParam::hasher&>()( + std::declval<const typename TypeParam::key_type&>())), + size_t>())); + EXPECT_TRUE((std::is_convertible< + decltype(std::declval<const typename TypeParam::key_equal&>()( + std::declval<const typename TypeParam::key_type&>(), + std::declval<const typename TypeParam::key_type&>())), + bool>())); + EXPECT_TRUE((std::is_same<typename TypeParam::allocator_type::value_type, + typename TypeParam::value_type>())); + EXPECT_TRUE((std::is_same<typename TypeParam::value_type&, + typename TypeParam::reference>())); + EXPECT_TRUE((std::is_same<const typename TypeParam::value_type&, + typename TypeParam::const_reference>())); + EXPECT_TRUE((std::is_same<typename std::allocator_traits< + typename TypeParam::allocator_type>::pointer, + typename TypeParam::pointer>())); + EXPECT_TRUE( + (std::is_same<typename std::allocator_traits< + typename TypeParam::allocator_type>::const_pointer, + typename TypeParam::const_pointer>())); +} + +TYPED_TEST_P(MembersTest, SimpleFunctions) { + EXPECT_GT(TypeParam().max_size(), 0); +} + +TYPED_TEST_P(MembersTest, BeginEnd) { + TypeParam t = {typename TypeParam::value_type{}}; + EXPECT_EQ(t.begin(), t.cbegin()); + EXPECT_EQ(t.end(), t.cend()); + EXPECT_NE(t.begin(), t.end()); + EXPECT_NE(t.cbegin(), t.cend()); +} + +REGISTER_TYPED_TEST_SUITE_P(MembersTest, Typedefs, SimpleFunctions, BeginEnd); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_MEMBERS_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_map_modifiers_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_map_modifiers_test.h new file mode 100644 index 000000000000..8c9ca779a424 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_map_modifiers_test.h @@ -0,0 +1,318 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_MODIFIERS_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_MODIFIERS_TEST_H_ + +#include <memory> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordMap> +class ModifiersTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(ModifiersTest); + +TYPED_TEST_P(ModifiersTest, Clear) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + m.clear(); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAre()); + EXPECT_TRUE(m.empty()); +} + +TYPED_TEST_P(ModifiersTest, Insert) { + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + auto p = m.insert(val); + EXPECT_TRUE(p.second); + EXPECT_EQ(val, *p.first); + T val2 = {val.first, hash_internal::Generator<V>()()}; + p = m.insert(val2); + EXPECT_FALSE(p.second); + EXPECT_EQ(val, *p.first); +} + +TYPED_TEST_P(ModifiersTest, InsertHint) { + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + auto it = m.insert(m.end(), val); + EXPECT_TRUE(it != m.end()); + EXPECT_EQ(val, *it); + T val2 = {val.first, hash_internal::Generator<V>()()}; + it = m.insert(it, val2); + EXPECT_TRUE(it != m.end()); + EXPECT_EQ(val, *it); +} + +TYPED_TEST_P(ModifiersTest, InsertRange) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + m.insert(values.begin(), values.end()); + ASSERT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); +} + +TYPED_TEST_P(ModifiersTest, InsertOrAssign) { +#ifdef UNORDERED_MAP_CXX17 + using std::get; + using K = typename TypeParam::key_type; + using V = typename TypeParam::mapped_type; + K k = hash_internal::Generator<K>()(); + V val = hash_internal::Generator<V>()(); + TypeParam m; + auto p = m.insert_or_assign(k, val); + EXPECT_TRUE(p.second); + EXPECT_EQ(k, get<0>(*p.first)); + EXPECT_EQ(val, get<1>(*p.first)); + V val2 = hash_internal::Generator<V>()(); + p = m.insert_or_assign(k, val2); + EXPECT_FALSE(p.second); + EXPECT_EQ(k, get<0>(*p.first)); + EXPECT_EQ(val2, get<1>(*p.first)); +#endif +} + +TYPED_TEST_P(ModifiersTest, InsertOrAssignHint) { +#ifdef UNORDERED_MAP_CXX17 + using std::get; + using K = typename TypeParam::key_type; + using V = typename TypeParam::mapped_type; + K k = hash_internal::Generator<K>()(); + V val = hash_internal::Generator<V>()(); + TypeParam m; + auto it = m.insert_or_assign(m.end(), k, val); + EXPECT_TRUE(it != m.end()); + EXPECT_EQ(k, get<0>(*it)); + EXPECT_EQ(val, get<1>(*it)); + V val2 = hash_internal::Generator<V>()(); + it = m.insert_or_assign(it, k, val2); + EXPECT_EQ(k, get<0>(*it)); + EXPECT_EQ(val2, get<1>(*it)); +#endif +} + +TYPED_TEST_P(ModifiersTest, Emplace) { + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + // TODO(alkis): We need a way to run emplace in a more meaningful way. Perhaps + // with test traits/policy. + auto p = m.emplace(val); + EXPECT_TRUE(p.second); + EXPECT_EQ(val, *p.first); + T val2 = {val.first, hash_internal::Generator<V>()()}; + p = m.emplace(val2); + EXPECT_FALSE(p.second); + EXPECT_EQ(val, *p.first); +} + +TYPED_TEST_P(ModifiersTest, EmplaceHint) { + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + // TODO(alkis): We need a way to run emplace in a more meaningful way. Perhaps + // with test traits/policy. + auto it = m.emplace_hint(m.end(), val); + EXPECT_EQ(val, *it); + T val2 = {val.first, hash_internal::Generator<V>()()}; + it = m.emplace_hint(it, val2); + EXPECT_EQ(val, *it); +} + +TYPED_TEST_P(ModifiersTest, TryEmplace) { +#ifdef UNORDERED_MAP_CXX17 + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + // TODO(alkis): We need a way to run emplace in a more meaningful way. Perhaps + // with test traits/policy. + auto p = m.try_emplace(val.first, val.second); + EXPECT_TRUE(p.second); + EXPECT_EQ(val, *p.first); + T val2 = {val.first, hash_internal::Generator<V>()()}; + p = m.try_emplace(val2.first, val2.second); + EXPECT_FALSE(p.second); + EXPECT_EQ(val, *p.first); +#endif +} + +TYPED_TEST_P(ModifiersTest, TryEmplaceHint) { +#ifdef UNORDERED_MAP_CXX17 + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + // TODO(alkis): We need a way to run emplace in a more meaningful way. Perhaps + // with test traits/policy. + auto it = m.try_emplace(m.end(), val.first, val.second); + EXPECT_EQ(val, *it); + T val2 = {val.first, hash_internal::Generator<V>()()}; + it = m.try_emplace(it, val2.first, val2.second); + EXPECT_EQ(val, *it); +#endif +} + +template <class V> +using IfNotVoid = typename std::enable_if<!std::is_void<V>::value, V>::type; + +// In openmap we chose not to return the iterator from erase because that's +// more expensive. As such we adapt erase to return an iterator here. +struct EraseFirst { + template <class Map> + auto operator()(Map* m, int) const + -> IfNotVoid<decltype(m->erase(m->begin()))> { + return m->erase(m->begin()); + } + template <class Map> + typename Map::iterator operator()(Map* m, ...) const { + auto it = m->begin(); + m->erase(it++); + return it; + } +}; + +TYPED_TEST_P(ModifiersTest, Erase) { + using T = hash_internal::GeneratedType<TypeParam>; + using std::get; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + auto& first = *m.begin(); + std::vector<T> values2; + for (const auto& val : values) + if (get<0>(val) != get<0>(first)) values2.push_back(val); + auto it = EraseFirst()(&m, 0); + ASSERT_TRUE(it != m.end()); + EXPECT_EQ(1, std::count(values2.begin(), values2.end(), *it)); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values2.begin(), + values2.end())); +} + +TYPED_TEST_P(ModifiersTest, EraseRange) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + auto it = m.erase(m.begin(), m.end()); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAre()); + EXPECT_TRUE(it == m.end()); +} + +TYPED_TEST_P(ModifiersTest, EraseKey) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_EQ(1, m.erase(values[0].first)); + EXPECT_EQ(0, std::count(m.begin(), m.end(), values[0])); + EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values.begin() + 1, + values.end())); +} + +TYPED_TEST_P(ModifiersTest, Swap) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> v1; + std::vector<T> v2; + std::generate_n(std::back_inserter(v1), 5, hash_internal::Generator<T>()); + std::generate_n(std::back_inserter(v2), 5, hash_internal::Generator<T>()); + TypeParam m1(v1.begin(), v1.end()); + TypeParam m2(v2.begin(), v2.end()); + EXPECT_THAT(items(m1), ::testing::UnorderedElementsAreArray(v1)); + EXPECT_THAT(items(m2), ::testing::UnorderedElementsAreArray(v2)); + m1.swap(m2); + EXPECT_THAT(items(m1), ::testing::UnorderedElementsAreArray(v2)); + EXPECT_THAT(items(m2), ::testing::UnorderedElementsAreArray(v1)); +} + +// TODO(alkis): Write tests for extract. +// TODO(alkis): Write tests for merge. + +REGISTER_TYPED_TEST_CASE_P(ModifiersTest, Clear, Insert, InsertHint, + InsertRange, InsertOrAssign, InsertOrAssignHint, + Emplace, EmplaceHint, TryEmplace, TryEmplaceHint, + Erase, EraseRange, EraseKey, Swap); + +template <typename Type> +struct is_unique_ptr : std::false_type {}; + +template <typename Type> +struct is_unique_ptr<std::unique_ptr<Type>> : std::true_type {}; + +template <class UnordMap> +class UniquePtrModifiersTest : public ::testing::Test { + protected: + UniquePtrModifiersTest() { + static_assert(is_unique_ptr<typename UnordMap::mapped_type>::value, + "UniquePtrModifiersTyest may only be called with a " + "std::unique_ptr value type."); + } +}; + +GTEST_ALLOW_UNINSTANTIATED_PARAMETERIZED_TEST(UniquePtrModifiersTest); + +TYPED_TEST_SUITE_P(UniquePtrModifiersTest); + +// Test that we do not move from rvalue arguments if an insertion does not +// happen. +TYPED_TEST_P(UniquePtrModifiersTest, TryEmplace) { +#ifdef UNORDERED_MAP_CXX17 + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + auto p = m.try_emplace(val.first, std::move(val.second)); + EXPECT_TRUE(p.second); + // A moved from std::unique_ptr is guaranteed to be nullptr. + EXPECT_EQ(val.second, nullptr); + T val2 = {val.first, hash_internal::Generator<V>()()}; + p = m.try_emplace(val2.first, std::move(val2.second)); + EXPECT_FALSE(p.second); + EXPECT_NE(val2.second, nullptr); +#endif +} + +REGISTER_TYPED_TEST_SUITE_P(UniquePtrModifiersTest, TryEmplace); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_MAP_MODIFIERS_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_map_test.cc b/third_party/abseil_cpp/absl/container/internal/unordered_map_test.cc new file mode 100644 index 000000000000..9cbf512f32b2 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_map_test.cc @@ -0,0 +1,50 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <memory> +#include <unordered_map> + +#include "absl/container/internal/unordered_map_constructor_test.h" +#include "absl/container/internal/unordered_map_lookup_test.h" +#include "absl/container/internal/unordered_map_members_test.h" +#include "absl/container/internal/unordered_map_modifiers_test.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using MapTypes = ::testing::Types< + std::unordered_map<int, int, StatefulTestingHash, StatefulTestingEqual, + Alloc<std::pair<const int, int>>>, + std::unordered_map<std::string, std::string, StatefulTestingHash, + StatefulTestingEqual, + Alloc<std::pair<const std::string, std::string>>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedMap, ConstructorTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedMap, LookupTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedMap, MembersTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedMap, ModifiersTest, MapTypes); + +using UniquePtrMapTypes = ::testing::Types<std::unordered_map< + int, std::unique_ptr<int>, StatefulTestingHash, StatefulTestingEqual, + Alloc<std::pair<const int, std::unique_ptr<int>>>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedMap, UniquePtrModifiersTest, + UniquePtrMapTypes); + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_set_constructor_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_set_constructor_test.h new file mode 100644 index 000000000000..41165b05e97b --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_set_constructor_test.h @@ -0,0 +1,496 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_SET_CONSTRUCTOR_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_SET_CONSTRUCTOR_TEST_H_ + +#include <algorithm> +#include <unordered_set> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordMap> +class ConstructorTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(ConstructorTest); + +TYPED_TEST_P(ConstructorTest, NoArgs) { + TypeParam m; + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); +} + +TYPED_TEST_P(ConstructorTest, BucketCount) { + TypeParam m(123); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHash) { + using H = typename TypeParam::hasher; + H hasher; + TypeParam m(123, hasher); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHashEqual) { + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + H hasher; + E equal; + TypeParam m(123, hasher, equal); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHashEqualAlloc) { + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); + + const auto& cm = m; + EXPECT_EQ(cm.hash_function(), hasher); + EXPECT_EQ(cm.key_eq(), equal); + EXPECT_EQ(cm.get_allocator(), alloc); + EXPECT_TRUE(cm.empty()); + EXPECT_THAT(keys(cm), ::testing::UnorderedElementsAre()); + EXPECT_GE(cm.bucket_count(), 123); +} + +template <typename T> +struct is_std_unordered_set : std::false_type {}; + +template <typename... T> +struct is_std_unordered_set<std::unordered_set<T...>> : std::true_type {}; + +#if defined(UNORDERED_SET_CXX14) || defined(UNORDERED_SET_CXX17) +using has_cxx14_std_apis = std::true_type; +#else +using has_cxx14_std_apis = std::false_type; +#endif + +template <typename T> +using expect_cxx14_apis = + absl::disjunction<absl::negation<is_std_unordered_set<T>>, + has_cxx14_std_apis>; + +template <typename TypeParam> +void BucketCountAllocTest(std::false_type) {} + +template <typename TypeParam> +void BucketCountAllocTest(std::true_type) { + using A = typename TypeParam::allocator_type; + A alloc(0); + TypeParam m(123, alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountAlloc) { + BucketCountAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +template <typename TypeParam> +void BucketCountHashAllocTest(std::false_type) {} + +template <typename TypeParam> +void BucketCountHashAllocTest(std::true_type) { + using H = typename TypeParam::hasher; + using A = typename TypeParam::allocator_type; + H hasher; + A alloc(0); + TypeParam m(123, hasher, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, BucketCountHashAlloc) { + BucketCountHashAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +#if ABSL_UNORDERED_SUPPORTS_ALLOC_CTORS +using has_alloc_std_constructors = std::true_type; +#else +using has_alloc_std_constructors = std::false_type; +#endif + +template <typename T> +using expect_alloc_constructors = + absl::disjunction<absl::negation<is_std_unordered_set<T>>, + has_alloc_std_constructors>; + +template <typename TypeParam> +void AllocTest(std::false_type) {} + +template <typename TypeParam> +void AllocTest(std::true_type) { + using A = typename TypeParam::allocator_type; + A alloc(0); + TypeParam m(alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_TRUE(m.empty()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); +} + +TYPED_TEST_P(ConstructorTest, Alloc) { + AllocTest<TypeParam>(expect_alloc_constructors<TypeParam>()); +} + +TYPED_TEST_P(ConstructorTest, InputIteratorBucketHashEqualAlloc) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + std::vector<T> values; + for (size_t i = 0; i != 10; ++i) + values.push_back(hash_internal::Generator<T>()()); + TypeParam m(values.begin(), values.end(), 123, hasher, equal, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +template <typename TypeParam> +void InputIteratorBucketAllocTest(std::false_type) {} + +template <typename TypeParam> +void InputIteratorBucketAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using A = typename TypeParam::allocator_type; + A alloc(0); + std::vector<T> values; + for (size_t i = 0; i != 10; ++i) + values.push_back(hash_internal::Generator<T>()()); + TypeParam m(values.begin(), values.end(), 123, alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InputIteratorBucketAlloc) { + InputIteratorBucketAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +template <typename TypeParam> +void InputIteratorBucketHashAllocTest(std::false_type) {} + +template <typename TypeParam> +void InputIteratorBucketHashAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using A = typename TypeParam::allocator_type; + H hasher; + A alloc(0); + std::vector<T> values; + for (size_t i = 0; i != 10; ++i) + values.push_back(hash_internal::Generator<T>()()); + TypeParam m(values.begin(), values.end(), 123, hasher, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InputIteratorBucketHashAlloc) { + InputIteratorBucketHashAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +TYPED_TEST_P(ConstructorTest, CopyConstructor) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam n(m); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); + EXPECT_NE(TypeParam(0, hasher, equal, alloc), n); +} + +template <typename TypeParam> +void CopyConstructorAllocTest(std::false_type) {} + +template <typename TypeParam> +void CopyConstructorAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam n(m, A(11)); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_NE(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, CopyConstructorAlloc) { + CopyConstructorAllocTest<TypeParam>(expect_alloc_constructors<TypeParam>()); +} + +// TODO(alkis): Test non-propagating allocators on copy constructors. + +TYPED_TEST_P(ConstructorTest, MoveConstructor) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam t(m); + TypeParam n(std::move(t)); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +template <typename TypeParam> +void MoveConstructorAllocTest(std::false_type) {} + +template <typename TypeParam> +void MoveConstructorAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(123, hasher, equal, alloc); + for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + TypeParam t(m); + TypeParam n(std::move(t), A(1)); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_NE(m.get_allocator(), n.get_allocator()); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, MoveConstructorAlloc) { + MoveConstructorAllocTest<TypeParam>(expect_alloc_constructors<TypeParam>()); +} + +// TODO(alkis): Test non-propagating allocators on move constructors. + +TYPED_TEST_P(ConstructorTest, InitializerListBucketHashEqualAlloc) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + TypeParam m(values, 123, hasher, equal, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.key_eq(), equal); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +template <typename TypeParam> +void InitializerListBucketAllocTest(std::false_type) {} + +template <typename TypeParam> +void InitializerListBucketAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using A = typename TypeParam::allocator_type; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + A alloc(0); + TypeParam m(values, 123, alloc); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InitializerListBucketAlloc) { + InitializerListBucketAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +template <typename TypeParam> +void InitializerListBucketHashAllocTest(std::false_type) {} + +template <typename TypeParam> +void InitializerListBucketHashAllocTest(std::true_type) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using A = typename TypeParam::allocator_type; + H hasher; + A alloc(0); + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m(values, 123, hasher, alloc); + EXPECT_EQ(m.hash_function(), hasher); + EXPECT_EQ(m.get_allocator(), alloc); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_GE(m.bucket_count(), 123); +} + +TYPED_TEST_P(ConstructorTest, InitializerListBucketHashAlloc) { + InitializerListBucketHashAllocTest<TypeParam>(expect_cxx14_apis<TypeParam>()); +} + +TYPED_TEST_P(ConstructorTest, CopyAssignment) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); + TypeParam n; + n = m; + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m, n); +} + +// TODO(alkis): Test [non-]propagating allocators on move/copy assignments +// (it depends on traits). + +TYPED_TEST_P(ConstructorTest, MoveAssignment) { + using T = hash_internal::GeneratedType<TypeParam>; + using H = typename TypeParam::hasher; + using E = typename TypeParam::key_equal; + using A = typename TypeParam::allocator_type; + H hasher; + E equal; + A alloc(0); + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); + TypeParam t(m); + TypeParam n; + n = std::move(t); + EXPECT_EQ(m.hash_function(), n.hash_function()); + EXPECT_EQ(m.key_eq(), n.key_eq()); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerList) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m; + m = values; + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); +} + +TYPED_TEST_P(ConstructorTest, AssignmentOverwritesExisting) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}); + TypeParam n({gen()}); + n = m; + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, MoveAssignmentOverwritesExisting) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + TypeParam m({gen(), gen(), gen()}); + TypeParam t(m); + TypeParam n({gen()}); + n = std::move(t); + EXPECT_EQ(m, n); +} + +TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerListOverwritesExisting) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m; + m = values; + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); +} + +TYPED_TEST_P(ConstructorTest, AssignmentOnSelf) { + using T = hash_internal::GeneratedType<TypeParam>; + hash_internal::Generator<T> gen; + std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; + TypeParam m(values); + m = *&m; // Avoid -Wself-assign. + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); +} + +REGISTER_TYPED_TEST_CASE_P( + ConstructorTest, NoArgs, BucketCount, BucketCountHash, BucketCountHashEqual, + BucketCountHashEqualAlloc, BucketCountAlloc, BucketCountHashAlloc, Alloc, + InputIteratorBucketHashEqualAlloc, InputIteratorBucketAlloc, + InputIteratorBucketHashAlloc, CopyConstructor, CopyConstructorAlloc, + MoveConstructor, MoveConstructorAlloc, InitializerListBucketHashEqualAlloc, + InitializerListBucketAlloc, InitializerListBucketHashAlloc, CopyAssignment, + MoveAssignment, AssignmentFromInitializerList, AssignmentOverwritesExisting, + MoveAssignmentOverwritesExisting, + AssignmentFromInitializerListOverwritesExisting, AssignmentOnSelf); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_SET_CONSTRUCTOR_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_set_lookup_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_set_lookup_test.h new file mode 100644 index 000000000000..8f2f4b207ef0 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_set_lookup_test.h @@ -0,0 +1,91 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_SET_LOOKUP_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_SET_LOOKUP_TEST_H_ + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordSet> +class LookupTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(LookupTest); + +TYPED_TEST_P(LookupTest, Count) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& v : values) + EXPECT_EQ(0, m.count(v)) << ::testing::PrintToString(v); + m.insert(values.begin(), values.end()); + for (const auto& v : values) + EXPECT_EQ(1, m.count(v)) << ::testing::PrintToString(v); +} + +TYPED_TEST_P(LookupTest, Find) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& v : values) + EXPECT_TRUE(m.end() == m.find(v)) << ::testing::PrintToString(v); + m.insert(values.begin(), values.end()); + for (const auto& v : values) { + typename TypeParam::iterator it = m.find(v); + static_assert(std::is_same<const typename TypeParam::value_type&, + decltype(*it)>::value, + ""); + static_assert(std::is_same<const typename TypeParam::value_type*, + decltype(it.operator->())>::value, + ""); + EXPECT_TRUE(m.end() != it) << ::testing::PrintToString(v); + EXPECT_EQ(v, *it) << ::testing::PrintToString(v); + } +} + +TYPED_TEST_P(LookupTest, EqualRange) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + for (const auto& v : values) { + auto r = m.equal_range(v); + ASSERT_EQ(0, std::distance(r.first, r.second)); + } + m.insert(values.begin(), values.end()); + for (const auto& v : values) { + auto r = m.equal_range(v); + ASSERT_EQ(1, std::distance(r.first, r.second)); + EXPECT_EQ(v, *r.first); + } +} + +REGISTER_TYPED_TEST_CASE_P(LookupTest, Count, Find, EqualRange); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_SET_LOOKUP_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_set_members_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_set_members_test.h new file mode 100644 index 000000000000..4c5e104af292 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_set_members_test.h @@ -0,0 +1,86 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_SET_MEMBERS_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_SET_MEMBERS_TEST_H_ + +#include <type_traits> +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordSet> +class MembersTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(MembersTest); + +template <typename T> +void UseType() {} + +TYPED_TEST_P(MembersTest, Typedefs) { + EXPECT_TRUE((std::is_same<typename TypeParam::key_type, + typename TypeParam::value_type>())); + EXPECT_TRUE((absl::conjunction< + absl::negation<std::is_signed<typename TypeParam::size_type>>, + std::is_integral<typename TypeParam::size_type>>())); + EXPECT_TRUE((absl::conjunction< + std::is_signed<typename TypeParam::difference_type>, + std::is_integral<typename TypeParam::difference_type>>())); + EXPECT_TRUE((std::is_convertible< + decltype(std::declval<const typename TypeParam::hasher&>()( + std::declval<const typename TypeParam::key_type&>())), + size_t>())); + EXPECT_TRUE((std::is_convertible< + decltype(std::declval<const typename TypeParam::key_equal&>()( + std::declval<const typename TypeParam::key_type&>(), + std::declval<const typename TypeParam::key_type&>())), + bool>())); + EXPECT_TRUE((std::is_same<typename TypeParam::allocator_type::value_type, + typename TypeParam::value_type>())); + EXPECT_TRUE((std::is_same<typename TypeParam::value_type&, + typename TypeParam::reference>())); + EXPECT_TRUE((std::is_same<const typename TypeParam::value_type&, + typename TypeParam::const_reference>())); + EXPECT_TRUE((std::is_same<typename std::allocator_traits< + typename TypeParam::allocator_type>::pointer, + typename TypeParam::pointer>())); + EXPECT_TRUE( + (std::is_same<typename std::allocator_traits< + typename TypeParam::allocator_type>::const_pointer, + typename TypeParam::const_pointer>())); +} + +TYPED_TEST_P(MembersTest, SimpleFunctions) { + EXPECT_GT(TypeParam().max_size(), 0); +} + +TYPED_TEST_P(MembersTest, BeginEnd) { + TypeParam t = {typename TypeParam::value_type{}}; + EXPECT_EQ(t.begin(), t.cbegin()); + EXPECT_EQ(t.end(), t.cend()); + EXPECT_NE(t.begin(), t.end()); + EXPECT_NE(t.cbegin(), t.cend()); +} + +REGISTER_TYPED_TEST_SUITE_P(MembersTest, Typedefs, SimpleFunctions, BeginEnd); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_SET_MEMBERS_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_set_modifiers_test.h b/third_party/abseil_cpp/absl/container/internal/unordered_set_modifiers_test.h new file mode 100644 index 000000000000..26be58d99f21 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_set_modifiers_test.h @@ -0,0 +1,190 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_CONTAINER_INTERNAL_UNORDERED_SET_MODIFIERS_TEST_H_ +#define ABSL_CONTAINER_INTERNAL_UNORDERED_SET_MODIFIERS_TEST_H_ + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/hash_generator_testing.h" +#include "absl/container/internal/hash_policy_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <class UnordSet> +class ModifiersTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(ModifiersTest); + +TYPED_TEST_P(ModifiersTest, Clear) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + m.clear(); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_TRUE(m.empty()); +} + +TYPED_TEST_P(ModifiersTest, Insert) { + using T = hash_internal::GeneratedType<TypeParam>; + T val = hash_internal::Generator<T>()(); + TypeParam m; + auto p = m.insert(val); + EXPECT_TRUE(p.second); + EXPECT_EQ(val, *p.first); + p = m.insert(val); + EXPECT_FALSE(p.second); +} + +TYPED_TEST_P(ModifiersTest, InsertHint) { + using T = hash_internal::GeneratedType<TypeParam>; + T val = hash_internal::Generator<T>()(); + TypeParam m; + auto it = m.insert(m.end(), val); + EXPECT_TRUE(it != m.end()); + EXPECT_EQ(val, *it); + it = m.insert(it, val); + EXPECT_TRUE(it != m.end()); + EXPECT_EQ(val, *it); +} + +TYPED_TEST_P(ModifiersTest, InsertRange) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m; + m.insert(values.begin(), values.end()); + ASSERT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); +} + +TYPED_TEST_P(ModifiersTest, Emplace) { + using T = hash_internal::GeneratedType<TypeParam>; + T val = hash_internal::Generator<T>()(); + TypeParam m; + // TODO(alkis): We need a way to run emplace in a more meaningful way. Perhaps + // with test traits/policy. + auto p = m.emplace(val); + EXPECT_TRUE(p.second); + EXPECT_EQ(val, *p.first); + p = m.emplace(val); + EXPECT_FALSE(p.second); + EXPECT_EQ(val, *p.first); +} + +TYPED_TEST_P(ModifiersTest, EmplaceHint) { + using T = hash_internal::GeneratedType<TypeParam>; + T val = hash_internal::Generator<T>()(); + TypeParam m; + // TODO(alkis): We need a way to run emplace in a more meaningful way. Perhaps + // with test traits/policy. + auto it = m.emplace_hint(m.end(), val); + EXPECT_EQ(val, *it); + it = m.emplace_hint(it, val); + EXPECT_EQ(val, *it); +} + +template <class V> +using IfNotVoid = typename std::enable_if<!std::is_void<V>::value, V>::type; + +// In openmap we chose not to return the iterator from erase because that's +// more expensive. As such we adapt erase to return an iterator here. +struct EraseFirst { + template <class Map> + auto operator()(Map* m, int) const + -> IfNotVoid<decltype(m->erase(m->begin()))> { + return m->erase(m->begin()); + } + template <class Map> + typename Map::iterator operator()(Map* m, ...) const { + auto it = m->begin(); + m->erase(it++); + return it; + } +}; + +TYPED_TEST_P(ModifiersTest, Erase) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + std::vector<T> values2; + for (const auto& val : values) + if (val != *m.begin()) values2.push_back(val); + auto it = EraseFirst()(&m, 0); + ASSERT_TRUE(it != m.end()); + EXPECT_EQ(1, std::count(values2.begin(), values2.end(), *it)); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values2.begin(), + values2.end())); +} + +TYPED_TEST_P(ModifiersTest, EraseRange) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + auto it = m.erase(m.begin(), m.end()); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAre()); + EXPECT_TRUE(it == m.end()); +} + +TYPED_TEST_P(ModifiersTest, EraseKey) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> values; + std::generate_n(std::back_inserter(values), 10, + hash_internal::Generator<T>()); + TypeParam m(values.begin(), values.end()); + ASSERT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); + EXPECT_EQ(1, m.erase(values[0])); + EXPECT_EQ(0, std::count(m.begin(), m.end(), values[0])); + EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values.begin() + 1, + values.end())); +} + +TYPED_TEST_P(ModifiersTest, Swap) { + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> v1; + std::vector<T> v2; + std::generate_n(std::back_inserter(v1), 5, hash_internal::Generator<T>()); + std::generate_n(std::back_inserter(v2), 5, hash_internal::Generator<T>()); + TypeParam m1(v1.begin(), v1.end()); + TypeParam m2(v2.begin(), v2.end()); + EXPECT_THAT(keys(m1), ::testing::UnorderedElementsAreArray(v1)); + EXPECT_THAT(keys(m2), ::testing::UnorderedElementsAreArray(v2)); + m1.swap(m2); + EXPECT_THAT(keys(m1), ::testing::UnorderedElementsAreArray(v2)); + EXPECT_THAT(keys(m2), ::testing::UnorderedElementsAreArray(v1)); +} + +// TODO(alkis): Write tests for extract. +// TODO(alkis): Write tests for merge. + +REGISTER_TYPED_TEST_CASE_P(ModifiersTest, Clear, Insert, InsertHint, + InsertRange, Emplace, EmplaceHint, Erase, EraseRange, + EraseKey, Swap); + +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_INTERNAL_UNORDERED_SET_MODIFIERS_TEST_H_ diff --git a/third_party/abseil_cpp/absl/container/internal/unordered_set_test.cc b/third_party/abseil_cpp/absl/container/internal/unordered_set_test.cc new file mode 100644 index 000000000000..a134b53984bc --- /dev/null +++ b/third_party/abseil_cpp/absl/container/internal/unordered_set_test.cc @@ -0,0 +1,41 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <unordered_set> + +#include "absl/container/internal/unordered_set_constructor_test.h" +#include "absl/container/internal/unordered_set_lookup_test.h" +#include "absl/container/internal/unordered_set_members_test.h" +#include "absl/container/internal/unordered_set_modifiers_test.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using SetTypes = ::testing::Types< + std::unordered_set<int, StatefulTestingHash, StatefulTestingEqual, + Alloc<int>>, + std::unordered_set<std::string, StatefulTestingHash, StatefulTestingEqual, + Alloc<std::string>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedSet, ConstructorTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedSet, LookupTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedSet, MembersTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(UnorderedSet, ModifiersTest, SetTypes); + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/node_hash_map.h b/third_party/abseil_cpp/absl/container/node_hash_map.h new file mode 100644 index 000000000000..7a39f6284cf5 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/node_hash_map.h @@ -0,0 +1,597 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: node_hash_map.h +// ----------------------------------------------------------------------------- +// +// An `absl::node_hash_map<K, V>` is an unordered associative container of +// unique keys and associated values designed to be a more efficient replacement +// for `std::unordered_map`. Like `unordered_map`, search, insertion, and +// deletion of map elements can be done as an `O(1)` operation. However, +// `node_hash_map` (and other unordered associative containers known as the +// collection of Abseil "Swiss tables") contain other optimizations that result +// in both memory and computation advantages. +// +// In most cases, your default choice for a hash map should be a map of type +// `flat_hash_map`. However, if you need pointer stability and cannot store +// a `flat_hash_map` with `unique_ptr` elements, a `node_hash_map` may be a +// valid alternative. As well, if you are migrating your code from using +// `std::unordered_map`, a `node_hash_map` provides a more straightforward +// migration, because it guarantees pointer stability. Consider migrating to +// `node_hash_map` and perhaps converting to a more efficient `flat_hash_map` +// upon further review. + +#ifndef ABSL_CONTAINER_NODE_HASH_MAP_H_ +#define ABSL_CONTAINER_NODE_HASH_MAP_H_ + +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/algorithm/container.h" +#include "absl/container/internal/container_memory.h" +#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export +#include "absl/container/internal/node_hash_policy.h" +#include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +template <class Key, class Value> +class NodeHashMapPolicy; +} // namespace container_internal + +// ----------------------------------------------------------------------------- +// absl::node_hash_map +// ----------------------------------------------------------------------------- +// +// An `absl::node_hash_map<K, V>` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_map<K, V>` with +// the following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash map. +// * Returns `void` from the `erase(iterator)` overload. +// +// By default, `node_hash_map` uses the `absl::Hash` hashing framework. +// All fundamental and Abseil types that support the `absl::Hash` framework have +// a compatible equality operator for comparing insertions into `node_hash_map`. +// If your type is not yet supported by the `absl::Hash` framework, see +// absl/hash/hash.h for information on extending Abseil hashing to user-defined +// types. +// +// Example: +// +// // Create a node hash map of three strings (that map to strings) +// absl::node_hash_map<std::string, std::string> ducks = +// {{"a", "huey"}, {"b", "dewey"}, {"c", "louie"}}; +// +// // Insert a new element into the node hash map +// ducks.insert({"d", "donald"}}; +// +// // Force a rehash of the node hash map +// ducks.rehash(0); +// +// // Find the element with the key "b" +// std::string search_key = "b"; +// auto result = ducks.find(search_key); +// if (result != ducks.end()) { +// std::cout << "Result: " << result->second << std::endl; +// } +template <class Key, class Value, + class Hash = absl::container_internal::hash_default_hash<Key>, + class Eq = absl::container_internal::hash_default_eq<Key>, + class Alloc = std::allocator<std::pair<const Key, Value>>> +class node_hash_map + : public absl::container_internal::raw_hash_map< + absl::container_internal::NodeHashMapPolicy<Key, Value>, Hash, Eq, + Alloc> { + using Base = typename node_hash_map::raw_hash_map; + + public: + // Constructors and Assignment Operators + // + // A node_hash_map supports the same overload set as `std::unordered_map` + // for construction and assignment: + // + // * Default constructor + // + // // No allocation for the table's elements is made. + // absl::node_hash_map<int, std::string> map1; + // + // * Initializer List constructor + // + // absl::node_hash_map<int, std::string> map2 = + // {{1, "huey"}, {2, "dewey"}, {3, "louie"},}; + // + // * Copy constructor + // + // absl::node_hash_map<int, std::string> map3(map2); + // + // * Copy assignment operator + // + // // Hash functor and Comparator are copied as well + // absl::node_hash_map<int, std::string> map4; + // map4 = map3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::node_hash_map<int, std::string> map5(std::move(map4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::node_hash_map<int, std::string> map6; + // map6 = std::move(map5); + // + // * Range constructor + // + // std::vector<std::pair<int, std::string>> v = {{1, "a"}, {2, "b"}}; + // absl::node_hash_map<int, std::string> map7(v.begin(), v.end()); + node_hash_map() {} + using Base::Base; + + // node_hash_map::begin() + // + // Returns an iterator to the beginning of the `node_hash_map`. + using Base::begin; + + // node_hash_map::cbegin() + // + // Returns a const iterator to the beginning of the `node_hash_map`. + using Base::cbegin; + + // node_hash_map::cend() + // + // Returns a const iterator to the end of the `node_hash_map`. + using Base::cend; + + // node_hash_map::end() + // + // Returns an iterator to the end of the `node_hash_map`. + using Base::end; + + // node_hash_map::capacity() + // + // Returns the number of element slots (assigned, deleted, and empty) + // available within the `node_hash_map`. + // + // NOTE: this member function is particular to `absl::node_hash_map` and is + // not provided in the `std::unordered_map` API. + using Base::capacity; + + // node_hash_map::empty() + // + // Returns whether or not the `node_hash_map` is empty. + using Base::empty; + + // node_hash_map::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `node_hash_map` under current memory constraints. This value can be thought + // of as the largest value of `std::distance(begin(), end())` for a + // `node_hash_map<K, V>`. + using Base::max_size; + + // node_hash_map::size() + // + // Returns the number of elements currently within the `node_hash_map`. + using Base::size; + + // node_hash_map::clear() + // + // Removes all elements from the `node_hash_map`. Invalidates any references, + // pointers, or iterators referring to contained elements. + // + // NOTE: this operation may shrink the underlying buffer. To avoid shrinking + // the underlying buffer call `erase(begin(), end())`. + using Base::clear; + + // node_hash_map::erase() + // + // Erases elements within the `node_hash_map`. Erasing does not trigger a + // rehash. Overloads are listed below. + // + // void erase(const_iterator pos): + // + // Erases the element at `position` of the `node_hash_map`, returning + // `void`. + // + // NOTE: this return behavior is different than that of STL containers in + // general and `std::unordered_map` in particular. + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning an + // iterator pointing to `last`. + // + // size_type erase(const key_type& key): + // + // Erases the element with the matching key, if it exists, returning the + // number of elements erased (0 or 1). + using Base::erase; + + // node_hash_map::insert() + // + // Inserts an element of the specified value into the `node_hash_map`, + // returning an iterator pointing to the newly inserted element, provided that + // an element with the given key does not already exist. If rehashing occurs + // due to the insertion, all iterators are invalidated. Overloads are listed + // below. + // + // std::pair<iterator,bool> insert(const init_type& value): + // + // Inserts a value into the `node_hash_map`. Returns a pair consisting of an + // iterator to the inserted element (or to the element that prevented the + // insertion) and a `bool` denoting whether the insertion took place. + // + // std::pair<iterator,bool> insert(T&& value): + // std::pair<iterator,bool> insert(init_type&& value): + // + // Inserts a moveable value into the `node_hash_map`. Returns a `std::pair` + // consisting of an iterator to the inserted element (or to the element that + // prevented the insertion) and a `bool` denoting whether the insertion took + // place. + // + // iterator insert(const_iterator hint, const init_type& value): + // iterator insert(const_iterator hint, T&& value): + // iterator insert(const_iterator hint, init_type&& value); + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element, or to the existing element that prevented the + // insertion. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently, for `node_hash_map` we guarantee the + // first match is inserted. + // + // void insert(std::initializer_list<init_type> ilist): + // + // Inserts the elements within the initializer list `ilist`. + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently within the initializer list, for + // `node_hash_map` we guarantee the first match is inserted. + using Base::insert; + + // node_hash_map::insert_or_assign() + // + // Inserts an element of the specified value into the `node_hash_map` provided + // that a value with the given key does not already exist, or replaces it with + // the element value if a key for that value already exists, returning an + // iterator pointing to the newly inserted element. If rehashing occurs due to + // the insertion, all iterators are invalidated. Overloads are listed + // below. + // + // std::pair<iterator, bool> insert_or_assign(const init_type& k, T&& obj): + // std::pair<iterator, bool> insert_or_assign(init_type&& k, T&& obj): + // + // Inserts/Assigns (or moves) the element of the specified key into the + // `node_hash_map`. + // + // iterator insert_or_assign(const_iterator hint, + // const init_type& k, T&& obj): + // iterator insert_or_assign(const_iterator hint, init_type&& k, T&& obj): + // + // Inserts/Assigns (or moves) the element of the specified key into the + // `node_hash_map` using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. + using Base::insert_or_assign; + + // node_hash_map::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `node_hash_map`, provided that no element with the given key + // already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. Prefer `try_emplace()` unless your key is not + // copyable or moveable. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace; + + // node_hash_map::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `node_hash_map`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search, and only inserts + // provided that no element with the given key already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. Prefer `try_emplace()` unless your key is not + // copyable or moveable. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace_hint; + + // node_hash_map::try_emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `node_hash_map`, provided that no element with the given key + // already exists. Unlike `emplace()`, if an element with the given key + // already exists, we guarantee that no element is constructed. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + // Overloads are listed below. + // + // std::pair<iterator, bool> try_emplace(const key_type& k, Args&&... args): + // std::pair<iterator, bool> try_emplace(key_type&& k, Args&&... args): + // + // Inserts (via copy or move) the element of the specified key into the + // `node_hash_map`. + // + // iterator try_emplace(const_iterator hint, + // const init_type& k, Args&&... args): + // iterator try_emplace(const_iterator hint, init_type&& k, Args&&... args): + // + // Inserts (via copy or move) the element of the specified key into the + // `node_hash_map` using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. + // + // All `try_emplace()` overloads make the same guarantees regarding rvalue + // arguments as `std::unordered_map::try_emplace()`, namely that these + // functions will not move from rvalue arguments if insertions do not happen. + using Base::try_emplace; + + // node_hash_map::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the key,value pair of the element at the indicated position and + // returns a node handle owning that extracted data. + // + // node_type extract(const key_type& x): + // + // Extracts the key,value pair of the element with a key matching the passed + // key value and returns a node handle owning that extracted data. If the + // `node_hash_map` does not contain an element with a matching key, this + // function returns an empty node handle. + // + // NOTE: when compiled in an earlier version of C++ than C++17, + // `node_type::key()` returns a const reference to the key instead of a + // mutable reference. We cannot safely return a mutable reference without + // std::launder (which is not available before C++17). + using Base::extract; + + // node_hash_map::merge() + // + // Extracts elements from a given `source` node hash map into this + // `node_hash_map`. If the destination `node_hash_map` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // node_hash_map::swap(node_hash_map& other) + // + // Exchanges the contents of this `node_hash_map` with those of the `other` + // node hash map, avoiding invocation of any move, copy, or swap operations on + // individual elements. + // + // All iterators and references on the `node_hash_map` remain valid, excepting + // for the past-the-end iterator, which is invalidated. + // + // `swap()` requires that the node hash map's hashing and key equivalence + // functions be Swappable, and are exchaged using unqualified calls to + // non-member `swap()`. If the map's allocator has + // `std::allocator_traits<allocator_type>::propagate_on_container_swap::value` + // set to `true`, the allocators are also exchanged using an unqualified call + // to non-member `swap()`; otherwise, the allocators are not swapped. + using Base::swap; + + // node_hash_map::rehash(count) + // + // Rehashes the `node_hash_map`, setting the number of slots to be at least + // the passed value. If the new number of slots increases the load factor more + // than the current maximum load factor + // (`count` < `size()` / `max_load_factor()`), then the new number of slots + // will be at least `size()` / `max_load_factor()`. + // + // To force a rehash, pass rehash(0). + using Base::rehash; + + // node_hash_map::reserve(count) + // + // Sets the number of slots in the `node_hash_map` to the number needed to + // accommodate at least `count` total elements without exceeding the current + // maximum load factor, and may rehash the container if needed. + using Base::reserve; + + // node_hash_map::at() + // + // Returns a reference to the mapped value of the element with key equivalent + // to the passed key. + using Base::at; + + // node_hash_map::contains() + // + // Determines whether an element with a key comparing equal to the given `key` + // exists within the `node_hash_map`, returning `true` if so or `false` + // otherwise. + using Base::contains; + + // node_hash_map::count(const Key& key) const + // + // Returns the number of elements with a key comparing equal to the given + // `key` within the `node_hash_map`. note that this function will return + // either `1` or `0` since duplicate keys are not allowed within a + // `node_hash_map`. + using Base::count; + + // node_hash_map::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `node_hash_map`. + using Base::equal_range; + + // node_hash_map::find() + // + // Finds an element with the passed `key` within the `node_hash_map`. + using Base::find; + + // node_hash_map::operator[]() + // + // Returns a reference to the value mapped to the passed key within the + // `node_hash_map`, performing an `insert()` if the key does not already + // exist. If an insertion occurs and results in a rehashing of the container, + // all iterators are invalidated. Otherwise iterators are not affected and + // references are not invalidated. Overloads are listed below. + // + // T& operator[](const Key& key): + // + // Inserts an init_type object constructed in-place if the element with the + // given key does not exist. + // + // T& operator[](Key&& key): + // + // Inserts an init_type object constructed in-place provided that an element + // with the given key does not exist. + using Base::operator[]; + + // node_hash_map::bucket_count() + // + // Returns the number of "buckets" within the `node_hash_map`. + using Base::bucket_count; + + // node_hash_map::load_factor() + // + // Returns the current load factor of the `node_hash_map` (the average number + // of slots occupied with a value within the hash map). + using Base::load_factor; + + // node_hash_map::max_load_factor() + // + // Manages the maximum load factor of the `node_hash_map`. Overloads are + // listed below. + // + // float node_hash_map::max_load_factor() + // + // Returns the current maximum load factor of the `node_hash_map`. + // + // void node_hash_map::max_load_factor(float ml) + // + // Sets the maximum load factor of the `node_hash_map` to the passed value. + // + // NOTE: This overload is provided only for API compatibility with the STL; + // `node_hash_map` will ignore any set load factor and manage its rehashing + // internally as an implementation detail. + using Base::max_load_factor; + + // node_hash_map::get_allocator() + // + // Returns the allocator function associated with this `node_hash_map`. + using Base::get_allocator; + + // node_hash_map::hash_function() + // + // Returns the hashing function used to hash the keys within this + // `node_hash_map`. + using Base::hash_function; + + // node_hash_map::key_eq() + // + // Returns the function used for comparing keys equality. + using Base::key_eq; +}; + +// erase_if(node_hash_map<>, Pred) +// +// Erases all elements that satisfy the predicate `pred` from the container `c`. +template <typename K, typename V, typename H, typename E, typename A, + typename Predicate> +void erase_if(node_hash_map<K, V, H, E, A>& c, Predicate pred) { + container_internal::EraseIf(pred, &c); +} + +namespace container_internal { + +template <class Key, class Value> +class NodeHashMapPolicy + : public absl::container_internal::node_hash_policy< + std::pair<const Key, Value>&, NodeHashMapPolicy<Key, Value>> { + using value_type = std::pair<const Key, Value>; + + public: + using key_type = Key; + using mapped_type = Value; + using init_type = std::pair</*non const*/ key_type, mapped_type>; + + template <class Allocator, class... Args> + static value_type* new_element(Allocator* alloc, Args&&... args) { + using PairAlloc = typename absl::allocator_traits< + Allocator>::template rebind_alloc<value_type>; + PairAlloc pair_alloc(*alloc); + value_type* res = + absl::allocator_traits<PairAlloc>::allocate(pair_alloc, 1); + absl::allocator_traits<PairAlloc>::construct(pair_alloc, res, + std::forward<Args>(args)...); + return res; + } + + template <class Allocator> + static void delete_element(Allocator* alloc, value_type* pair) { + using PairAlloc = typename absl::allocator_traits< + Allocator>::template rebind_alloc<value_type>; + PairAlloc pair_alloc(*alloc); + absl::allocator_traits<PairAlloc>::destroy(pair_alloc, pair); + absl::allocator_traits<PairAlloc>::deallocate(pair_alloc, pair, 1); + } + + template <class F, class... Args> + static decltype(absl::container_internal::DecomposePair( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return absl::container_internal::DecomposePair(std::forward<F>(f), + std::forward<Args>(args)...); + } + + static size_t element_space_used(const value_type*) { + return sizeof(value_type); + } + + static Value& value(value_type* elem) { return elem->second; } + static const Value& value(const value_type* elem) { return elem->second; } +}; +} // namespace container_internal + +namespace container_algorithm_internal { + +// Specialization of trait in absl/algorithm/container.h +template <class Key, class T, class Hash, class KeyEqual, class Allocator> +struct IsUnorderedContainer< + absl::node_hash_map<Key, T, Hash, KeyEqual, Allocator>> : std::true_type {}; + +} // namespace container_algorithm_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_NODE_HASH_MAP_H_ diff --git a/third_party/abseil_cpp/absl/container/node_hash_map_test.cc b/third_party/abseil_cpp/absl/container/node_hash_map_test.cc new file mode 100644 index 000000000000..8f59a1e4a210 --- /dev/null +++ b/third_party/abseil_cpp/absl/container/node_hash_map_test.cc @@ -0,0 +1,275 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/node_hash_map.h" + +#include "absl/container/internal/tracked.h" +#include "absl/container/internal/unordered_map_constructor_test.h" +#include "absl/container/internal/unordered_map_lookup_test.h" +#include "absl/container/internal/unordered_map_members_test.h" +#include "absl/container/internal/unordered_map_modifiers_test.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +using ::testing::Field; +using ::testing::IsEmpty; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +using MapTypes = ::testing::Types< + absl::node_hash_map<int, int, StatefulTestingHash, StatefulTestingEqual, + Alloc<std::pair<const int, int>>>, + absl::node_hash_map<std::string, std::string, StatefulTestingHash, + StatefulTestingEqual, + Alloc<std::pair<const std::string, std::string>>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashMap, ConstructorTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashMap, LookupTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashMap, MembersTest, MapTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashMap, ModifiersTest, MapTypes); + +using M = absl::node_hash_map<std::string, Tracked<int>>; + +TEST(NodeHashMap, Emplace) { + M m; + Tracked<int> t(53); + m.emplace("a", t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(1, t.num_copies()); + + m.emplace(std::string("a"), t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(1, t.num_copies()); + + std::string a("a"); + m.emplace(a, t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(1, t.num_copies()); + + const std::string ca("a"); + m.emplace(a, t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(1, t.num_copies()); + + m.emplace(std::make_pair("a", t)); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(2, t.num_copies()); + + m.emplace(std::make_pair(std::string("a"), t)); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(3, t.num_copies()); + + std::pair<std::string, Tracked<int>> p("a", t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(4, t.num_copies()); + m.emplace(p); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(4, t.num_copies()); + + const std::pair<std::string, Tracked<int>> cp("a", t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(5, t.num_copies()); + m.emplace(cp); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(5, t.num_copies()); + + std::pair<const std::string, Tracked<int>> pc("a", t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(6, t.num_copies()); + m.emplace(pc); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(6, t.num_copies()); + + const std::pair<const std::string, Tracked<int>> cpc("a", t); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(7, t.num_copies()); + m.emplace(cpc); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(7, t.num_copies()); + + m.emplace(std::piecewise_construct, std::forward_as_tuple("a"), + std::forward_as_tuple(t)); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(7, t.num_copies()); + + m.emplace(std::piecewise_construct, std::forward_as_tuple(std::string("a")), + std::forward_as_tuple(t)); + ASSERT_EQ(0, t.num_moves()); + ASSERT_EQ(7, t.num_copies()); +} + +TEST(NodeHashMap, AssignRecursive) { + struct Tree { + // Verify that unordered_map<K, IncompleteType> can be instantiated. + absl::node_hash_map<int, Tree> children; + }; + Tree root; + const Tree& child = root.children.emplace().first->second; + // Verify that `lhs = rhs` doesn't read rhs after clearing lhs. + root = child; +} + +TEST(FlatHashMap, MoveOnlyKey) { + struct Key { + Key() = default; + Key(Key&&) = default; + Key& operator=(Key&&) = default; + }; + struct Eq { + bool operator()(const Key&, const Key&) const { return true; } + }; + struct Hash { + size_t operator()(const Key&) const { return 0; } + }; + absl::node_hash_map<Key, int, Hash, Eq> m; + m[Key()]; +} + +struct NonMovableKey { + explicit NonMovableKey(int i) : i(i) {} + NonMovableKey(NonMovableKey&&) = delete; + int i; +}; +struct NonMovableKeyHash { + using is_transparent = void; + size_t operator()(const NonMovableKey& k) const { return k.i; } + size_t operator()(int k) const { return k; } +}; +struct NonMovableKeyEq { + using is_transparent = void; + bool operator()(const NonMovableKey& a, const NonMovableKey& b) const { + return a.i == b.i; + } + bool operator()(const NonMovableKey& a, int b) const { return a.i == b; } +}; + +TEST(NodeHashMap, MergeExtractInsert) { + absl::node_hash_map<NonMovableKey, int, NonMovableKeyHash, NonMovableKeyEq> + set1, set2; + set1.emplace(std::piecewise_construct, std::make_tuple(7), + std::make_tuple(-7)); + set1.emplace(std::piecewise_construct, std::make_tuple(17), + std::make_tuple(-17)); + + set2.emplace(std::piecewise_construct, std::make_tuple(7), + std::make_tuple(-70)); + set2.emplace(std::piecewise_construct, std::make_tuple(19), + std::make_tuple(-190)); + + auto Elem = [](int key, int value) { + return Pair(Field(&NonMovableKey::i, key), value); + }; + + EXPECT_THAT(set1, UnorderedElementsAre(Elem(7, -7), Elem(17, -17))); + EXPECT_THAT(set2, UnorderedElementsAre(Elem(7, -70), Elem(19, -190))); + + // NonMovableKey is neither copyable nor movable. We should still be able to + // move nodes around. + static_assert(!std::is_move_constructible<NonMovableKey>::value, ""); + set1.merge(set2); + + EXPECT_THAT(set1, + UnorderedElementsAre(Elem(7, -7), Elem(17, -17), Elem(19, -190))); + EXPECT_THAT(set2, UnorderedElementsAre(Elem(7, -70))); + + auto node = set1.extract(7); + EXPECT_TRUE(node); + EXPECT_EQ(node.key().i, 7); + EXPECT_EQ(node.mapped(), -7); + EXPECT_THAT(set1, UnorderedElementsAre(Elem(17, -17), Elem(19, -190))); + + auto insert_result = set2.insert(std::move(node)); + EXPECT_FALSE(node); + EXPECT_FALSE(insert_result.inserted); + EXPECT_TRUE(insert_result.node); + EXPECT_EQ(insert_result.node.key().i, 7); + EXPECT_EQ(insert_result.node.mapped(), -7); + EXPECT_THAT(*insert_result.position, Elem(7, -70)); + EXPECT_THAT(set2, UnorderedElementsAre(Elem(7, -70))); + + node = set1.extract(17); + EXPECT_TRUE(node); + EXPECT_EQ(node.key().i, 17); + EXPECT_EQ(node.mapped(), -17); + EXPECT_THAT(set1, UnorderedElementsAre(Elem(19, -190))); + + node.mapped() = 23; + + insert_result = set2.insert(std::move(node)); + EXPECT_FALSE(node); + EXPECT_TRUE(insert_result.inserted); + EXPECT_FALSE(insert_result.node); + EXPECT_THAT(*insert_result.position, Elem(17, 23)); + EXPECT_THAT(set2, UnorderedElementsAre(Elem(7, -70), Elem(17, 23))); +} + +bool FirstIsEven(std::pair<const int, int> p) { return p.first % 2 == 0; } + +TEST(NodeHashMap, EraseIf) { + // Erase all elements. + { + node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, [](std::pair<const int, int>) { return true; }); + EXPECT_THAT(s, IsEmpty()); + } + // Erase no elements. + { + node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, [](std::pair<const int, int>) { return false; }); + EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), + Pair(4, 4), Pair(5, 5))); + } + // Erase specific elements. + { + node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, + [](std::pair<const int, int> kvp) { return kvp.first % 2 == 1; }); + EXPECT_THAT(s, UnorderedElementsAre(Pair(2, 2), Pair(4, 4))); + } + // Predicate is function reference. + { + node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, FirstIsEven); + EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); + } + // Predicate is function pointer. + { + node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; + erase_if(s, &FirstIsEven); + EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); + } +} + +// This test requires std::launder for mutable key access in node handles. +#if defined(__cpp_lib_launder) && __cpp_lib_launder >= 201606 +TEST(NodeHashMap, NodeHandleMutableKeyAccess) { + node_hash_map<std::string, std::string> map; + + map["key1"] = "mapped"; + + auto nh = map.extract(map.begin()); + nh.key().resize(3); + map.insert(std::move(nh)); + + EXPECT_THAT(map, testing::ElementsAre(Pair("key", "mapped"))); +} +#endif + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/container/node_hash_set.h b/third_party/abseil_cpp/absl/container/node_hash_set.h new file mode 100644 index 000000000000..56ce3b66c0be --- /dev/null +++ b/third_party/abseil_cpp/absl/container/node_hash_set.h @@ -0,0 +1,493 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: node_hash_set.h +// ----------------------------------------------------------------------------- +// +// An `absl::node_hash_set<T>` is an unordered associative container designed to +// be a more efficient replacement for `std::unordered_set`. Like +// `unordered_set`, search, insertion, and deletion of map elements can be done +// as an `O(1)` operation. However, `node_hash_set` (and other unordered +// associative containers known as the collection of Abseil "Swiss tables") +// contain other optimizations that result in both memory and computation +// advantages. +// +// In most cases, your default choice for a hash table should be a map of type +// `flat_hash_map` or a set of type `flat_hash_set`. However, if you need +// pointer stability, a `node_hash_set` should be your preferred choice. As +// well, if you are migrating your code from using `std::unordered_set`, a +// `node_hash_set` should be an easy migration. Consider migrating to +// `node_hash_set` and perhaps converting to a more efficient `flat_hash_set` +// upon further review. + +#ifndef ABSL_CONTAINER_NODE_HASH_SET_H_ +#define ABSL_CONTAINER_NODE_HASH_SET_H_ + +#include <type_traits> + +#include "absl/algorithm/container.h" +#include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export +#include "absl/container/internal/node_hash_policy.h" +#include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +template <typename T> +struct NodeHashSetPolicy; +} // namespace container_internal + +// ----------------------------------------------------------------------------- +// absl::node_hash_set +// ----------------------------------------------------------------------------- +// +// An `absl::node_hash_set<T>` is an unordered associative container which +// has been optimized for both speed and memory footprint in most common use +// cases. Its interface is similar to that of `std::unordered_set<T>` with the +// following notable differences: +// +// * Supports heterogeneous lookup, through `find()`, `operator[]()` and +// `insert()`, provided that the map is provided a compatible heterogeneous +// hashing function and equality operator. +// * Contains a `capacity()` member function indicating the number of element +// slots (open, deleted, and empty) within the hash set. +// * Returns `void` from the `erase(iterator)` overload. +// +// By default, `node_hash_set` uses the `absl::Hash` hashing framework. +// All fundamental and Abseil types that support the `absl::Hash` framework have +// a compatible equality operator for comparing insertions into `node_hash_set`. +// If your type is not yet supported by the `absl::Hash` framework, see +// absl/hash/hash.h for information on extending Abseil hashing to user-defined +// types. +// +// Example: +// +// // Create a node hash set of three strings +// absl::node_hash_map<std::string, std::string> ducks = +// {"huey", "dewey", "louie"}; +// +// // Insert a new element into the node hash map +// ducks.insert("donald"}; +// +// // Force a rehash of the node hash map +// ducks.rehash(0); +// +// // See if "dewey" is present +// if (ducks.contains("dewey")) { +// std::cout << "We found dewey!" << std::endl; +// } +template <class T, class Hash = absl::container_internal::hash_default_hash<T>, + class Eq = absl::container_internal::hash_default_eq<T>, + class Alloc = std::allocator<T>> +class node_hash_set + : public absl::container_internal::raw_hash_set< + absl::container_internal::NodeHashSetPolicy<T>, Hash, Eq, Alloc> { + using Base = typename node_hash_set::raw_hash_set; + + public: + // Constructors and Assignment Operators + // + // A node_hash_set supports the same overload set as `std::unordered_map` + // for construction and assignment: + // + // * Default constructor + // + // // No allocation for the table's elements is made. + // absl::node_hash_set<std::string> set1; + // + // * Initializer List constructor + // + // absl::node_hash_set<std::string> set2 = + // {{"huey"}, {"dewey"}, {"louie"}}; + // + // * Copy constructor + // + // absl::node_hash_set<std::string> set3(set2); + // + // * Copy assignment operator + // + // // Hash functor and Comparator are copied as well + // absl::node_hash_set<std::string> set4; + // set4 = set3; + // + // * Move constructor + // + // // Move is guaranteed efficient + // absl::node_hash_set<std::string> set5(std::move(set4)); + // + // * Move assignment operator + // + // // May be efficient if allocators are compatible + // absl::node_hash_set<std::string> set6; + // set6 = std::move(set5); + // + // * Range constructor + // + // std::vector<std::string> v = {"a", "b"}; + // absl::node_hash_set<std::string> set7(v.begin(), v.end()); + node_hash_set() {} + using Base::Base; + + // node_hash_set::begin() + // + // Returns an iterator to the beginning of the `node_hash_set`. + using Base::begin; + + // node_hash_set::cbegin() + // + // Returns a const iterator to the beginning of the `node_hash_set`. + using Base::cbegin; + + // node_hash_set::cend() + // + // Returns a const iterator to the end of the `node_hash_set`. + using Base::cend; + + // node_hash_set::end() + // + // Returns an iterator to the end of the `node_hash_set`. + using Base::end; + + // node_hash_set::capacity() + // + // Returns the number of element slots (assigned, deleted, and empty) + // available within the `node_hash_set`. + // + // NOTE: this member function is particular to `absl::node_hash_set` and is + // not provided in the `std::unordered_map` API. + using Base::capacity; + + // node_hash_set::empty() + // + // Returns whether or not the `node_hash_set` is empty. + using Base::empty; + + // node_hash_set::max_size() + // + // Returns the largest theoretical possible number of elements within a + // `node_hash_set` under current memory constraints. This value can be thought + // of the largest value of `std::distance(begin(), end())` for a + // `node_hash_set<T>`. + using Base::max_size; + + // node_hash_set::size() + // + // Returns the number of elements currently within the `node_hash_set`. + using Base::size; + + // node_hash_set::clear() + // + // Removes all elements from the `node_hash_set`. Invalidates any references, + // pointers, or iterators referring to contained elements. + // + // NOTE: this operation may shrink the underlying buffer. To avoid shrinking + // the underlying buffer call `erase(begin(), end())`. + using Base::clear; + + // node_hash_set::erase() + // + // Erases elements within the `node_hash_set`. Erasing does not trigger a + // rehash. Overloads are listed below. + // + // void erase(const_iterator pos): + // + // Erases the element at `position` of the `node_hash_set`, returning + // `void`. + // + // NOTE: this return behavior is different than that of STL containers in + // general and `std::unordered_map` in particular. + // + // iterator erase(const_iterator first, const_iterator last): + // + // Erases the elements in the open interval [`first`, `last`), returning an + // iterator pointing to `last`. + // + // size_type erase(const key_type& key): + // + // Erases the element with the matching key, if it exists, returning the + // number of elements erased (0 or 1). + using Base::erase; + + // node_hash_set::insert() + // + // Inserts an element of the specified value into the `node_hash_set`, + // returning an iterator pointing to the newly inserted element, provided that + // an element with the given key does not already exist. If rehashing occurs + // due to the insertion, all iterators are invalidated. Overloads are listed + // below. + // + // std::pair<iterator,bool> insert(const T& value): + // + // Inserts a value into the `node_hash_set`. Returns a pair consisting of an + // iterator to the inserted element (or to the element that prevented the + // insertion) and a bool denoting whether the insertion took place. + // + // std::pair<iterator,bool> insert(T&& value): + // + // Inserts a moveable value into the `node_hash_set`. Returns a pair + // consisting of an iterator to the inserted element (or to the element that + // prevented the insertion) and a bool denoting whether the insertion took + // place. + // + // iterator insert(const_iterator hint, const T& value): + // iterator insert(const_iterator hint, T&& value): + // + // Inserts a value, using the position of `hint` as a non-binding suggestion + // for where to begin the insertion search. Returns an iterator to the + // inserted element, or to the existing element that prevented the + // insertion. + // + // void insert(InputIterator first, InputIterator last): + // + // Inserts a range of values [`first`, `last`). + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently, for `node_hash_set` we guarantee the + // first match is inserted. + // + // void insert(std::initializer_list<T> ilist): + // + // Inserts the elements within the initializer list `ilist`. + // + // NOTE: Although the STL does not specify which element may be inserted if + // multiple keys compare equivalently within the initializer list, for + // `node_hash_set` we guarantee the first match is inserted. + using Base::insert; + + // node_hash_set::emplace() + // + // Inserts an element of the specified value by constructing it in-place + // within the `node_hash_set`, provided that no element with the given key + // already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace; + + // node_hash_set::emplace_hint() + // + // Inserts an element of the specified value by constructing it in-place + // within the `node_hash_set`, using the position of `hint` as a non-binding + // suggestion for where to begin the insertion search, and only inserts + // provided that no element with the given key already exists. + // + // The element may be constructed even if there already is an element with the + // key in the container, in which case the newly constructed element will be + // destroyed immediately. + // + // If rehashing occurs due to the insertion, all iterators are invalidated. + using Base::emplace_hint; + + // node_hash_set::extract() + // + // Extracts the indicated element, erasing it in the process, and returns it + // as a C++17-compatible node handle. Overloads are listed below. + // + // node_type extract(const_iterator position): + // + // Extracts the element at the indicated position and returns a node handle + // owning that extracted data. + // + // node_type extract(const key_type& x): + // + // Extracts the element with the key matching the passed key value and + // returns a node handle owning that extracted data. If the `node_hash_set` + // does not contain an element with a matching key, this function returns an + // empty node handle. + using Base::extract; + + // node_hash_set::merge() + // + // Extracts elements from a given `source` flat hash map into this + // `node_hash_set`. If the destination `node_hash_set` already contains an + // element with an equivalent key, that element is not extracted. + using Base::merge; + + // node_hash_set::swap(node_hash_set& other) + // + // Exchanges the contents of this `node_hash_set` with those of the `other` + // flat hash map, avoiding invocation of any move, copy, or swap operations on + // individual elements. + // + // All iterators and references on the `node_hash_set` remain valid, excepting + // for the past-the-end iterator, which is invalidated. + // + // `swap()` requires that the flat hash set's hashing and key equivalence + // functions be Swappable, and are exchaged using unqualified calls to + // non-member `swap()`. If the map's allocator has + // `std::allocator_traits<allocator_type>::propagate_on_container_swap::value` + // set to `true`, the allocators are also exchanged using an unqualified call + // to non-member `swap()`; otherwise, the allocators are not swapped. + using Base::swap; + + // node_hash_set::rehash(count) + // + // Rehashes the `node_hash_set`, setting the number of slots to be at least + // the passed value. If the new number of slots increases the load factor more + // than the current maximum load factor + // (`count` < `size()` / `max_load_factor()`), then the new number of slots + // will be at least `size()` / `max_load_factor()`. + // + // To force a rehash, pass rehash(0). + // + // NOTE: unlike behavior in `std::unordered_set`, references are also + // invalidated upon a `rehash()`. + using Base::rehash; + + // node_hash_set::reserve(count) + // + // Sets the number of slots in the `node_hash_set` to the number needed to + // accommodate at least `count` total elements without exceeding the current + // maximum load factor, and may rehash the container if needed. + using Base::reserve; + + // node_hash_set::contains() + // + // Determines whether an element comparing equal to the given `key` exists + // within the `node_hash_set`, returning `true` if so or `false` otherwise. + using Base::contains; + + // node_hash_set::count(const Key& key) const + // + // Returns the number of elements comparing equal to the given `key` within + // the `node_hash_set`. note that this function will return either `1` or `0` + // since duplicate elements are not allowed within a `node_hash_set`. + using Base::count; + + // node_hash_set::equal_range() + // + // Returns a closed range [first, last], defined by a `std::pair` of two + // iterators, containing all elements with the passed key in the + // `node_hash_set`. + using Base::equal_range; + + // node_hash_set::find() + // + // Finds an element with the passed `key` within the `node_hash_set`. + using Base::find; + + // node_hash_set::bucket_count() + // + // Returns the number of "buckets" within the `node_hash_set`. Note that + // because a flat hash map contains all elements within its internal storage, + // this value simply equals the current capacity of the `node_hash_set`. + using Base::bucket_count; + + // node_hash_set::load_factor() + // + // Returns the current load factor of the `node_hash_set` (the average number + // of slots occupied with a value within the hash map). + using Base::load_factor; + + // node_hash_set::max_load_factor() + // + // Manages the maximum load factor of the `node_hash_set`. Overloads are + // listed below. + // + // float node_hash_set::max_load_factor() + // + // Returns the current maximum load factor of the `node_hash_set`. + // + // void node_hash_set::max_load_factor(float ml) + // + // Sets the maximum load factor of the `node_hash_set` to the passed value. + // + // NOTE: This overload is provided only for API compatibility with the STL; + // `node_hash_set` will ignore any set load factor and manage its rehashing + // internally as an implementation detail. + using Base::max_load_factor; + + // node_hash_set::get_allocator() + // + // Returns the allocator function associated with this `node_hash_set`. + using Base::get_allocator; + + // node_hash_set::hash_function() + // + // Returns the hashing function used to hash the keys within this + // `node_hash_set`. + using Base::hash_function; + + // node_hash_set::key_eq() + // + // Returns the function used for comparing keys equality. + using Base::key_eq; +}; + +// erase_if(node_hash_set<>, Pred) +// +// Erases all elements that satisfy the predicate `pred` from the container `c`. +template <typename T, typename H, typename E, typename A, typename Predicate> +void erase_if(node_hash_set<T, H, E, A>& c, Predicate pred) { + container_internal::EraseIf(pred, &c); +} + +namespace container_internal { + +template <class T> +struct NodeHashSetPolicy + : absl::container_internal::node_hash_policy<T&, NodeHashSetPolicy<T>> { + using key_type = T; + using init_type = T; + using constant_iterators = std::true_type; + + template <class Allocator, class... Args> + static T* new_element(Allocator* alloc, Args&&... args) { + using ValueAlloc = + typename absl::allocator_traits<Allocator>::template rebind_alloc<T>; + ValueAlloc value_alloc(*alloc); + T* res = absl::allocator_traits<ValueAlloc>::allocate(value_alloc, 1); + absl::allocator_traits<ValueAlloc>::construct(value_alloc, res, + std::forward<Args>(args)...); + return res; + } + + template <class Allocator> + static void delete_element(Allocator* alloc, T* elem) { + using ValueAlloc = + typename absl::allocator_traits<Allocator>::template rebind_alloc<T>; + ValueAlloc value_alloc(*alloc); + absl::allocator_traits<ValueAlloc>::destroy(value_alloc, elem); + absl::allocator_traits<ValueAlloc>::deallocate(value_alloc, elem, 1); + } + + template <class F, class... Args> + static decltype(absl::container_internal::DecomposeValue( + std::declval<F>(), std::declval<Args>()...)) + apply(F&& f, Args&&... args) { + return absl::container_internal::DecomposeValue( + std::forward<F>(f), std::forward<Args>(args)...); + } + + static size_t element_space_used(const T*) { return sizeof(T); } +}; +} // namespace container_internal + +namespace container_algorithm_internal { + +// Specialization of trait in absl/algorithm/container.h +template <class Key, class Hash, class KeyEqual, class Allocator> +struct IsUnorderedContainer<absl::node_hash_set<Key, Hash, KeyEqual, Allocator>> + : std::true_type {}; + +} // namespace container_algorithm_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_CONTAINER_NODE_HASH_SET_H_ diff --git a/third_party/abseil_cpp/absl/container/node_hash_set_test.cc b/third_party/abseil_cpp/absl/container/node_hash_set_test.cc new file mode 100644 index 000000000000..7ddad2021d2f --- /dev/null +++ b/third_party/abseil_cpp/absl/container/node_hash_set_test.cc @@ -0,0 +1,143 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/container/node_hash_set.h" + +#include "absl/container/internal/unordered_set_constructor_test.h" +#include "absl/container/internal/unordered_set_lookup_test.h" +#include "absl/container/internal/unordered_set_members_test.h" +#include "absl/container/internal/unordered_set_modifiers_test.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { +using ::absl::container_internal::hash_internal::Enum; +using ::absl::container_internal::hash_internal::EnumClass; +using ::testing::IsEmpty; +using ::testing::Pointee; +using ::testing::UnorderedElementsAre; + +using SetTypes = ::testing::Types< + node_hash_set<int, StatefulTestingHash, StatefulTestingEqual, Alloc<int>>, + node_hash_set<std::string, StatefulTestingHash, StatefulTestingEqual, + Alloc<std::string>>, + node_hash_set<Enum, StatefulTestingHash, StatefulTestingEqual, Alloc<Enum>>, + node_hash_set<EnumClass, StatefulTestingHash, StatefulTestingEqual, + Alloc<EnumClass>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashSet, ConstructorTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashSet, LookupTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashSet, MembersTest, SetTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(NodeHashSet, ModifiersTest, SetTypes); + +TEST(NodeHashSet, MoveableNotCopyableCompiles) { + node_hash_set<std::unique_ptr<void*>> t; + node_hash_set<std::unique_ptr<void*>> u; + u = std::move(t); +} + +TEST(NodeHashSet, MergeExtractInsert) { + struct Hash { + size_t operator()(const std::unique_ptr<int>& p) const { return *p; } + }; + struct Eq { + bool operator()(const std::unique_ptr<int>& a, + const std::unique_ptr<int>& b) const { + return *a == *b; + } + }; + absl::node_hash_set<std::unique_ptr<int>, Hash, Eq> set1, set2; + set1.insert(absl::make_unique<int>(7)); + set1.insert(absl::make_unique<int>(17)); + + set2.insert(absl::make_unique<int>(7)); + set2.insert(absl::make_unique<int>(19)); + + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17))); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(19))); + + set1.merge(set2); + + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(7), Pointee(17), Pointee(19))); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); + + auto node = set1.extract(absl::make_unique<int>(7)); + EXPECT_TRUE(node); + EXPECT_THAT(node.value(), Pointee(7)); + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(17), Pointee(19))); + + auto insert_result = set2.insert(std::move(node)); + EXPECT_FALSE(node); + EXPECT_FALSE(insert_result.inserted); + EXPECT_TRUE(insert_result.node); + EXPECT_THAT(insert_result.node.value(), Pointee(7)); + EXPECT_EQ(**insert_result.position, 7); + EXPECT_NE(insert_result.position->get(), insert_result.node.value().get()); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7))); + + node = set1.extract(absl::make_unique<int>(17)); + EXPECT_TRUE(node); + EXPECT_THAT(node.value(), Pointee(17)); + EXPECT_THAT(set1, UnorderedElementsAre(Pointee(19))); + + node.value() = absl::make_unique<int>(23); + + insert_result = set2.insert(std::move(node)); + EXPECT_FALSE(node); + EXPECT_TRUE(insert_result.inserted); + EXPECT_FALSE(insert_result.node); + EXPECT_EQ(**insert_result.position, 23); + EXPECT_THAT(set2, UnorderedElementsAre(Pointee(7), Pointee(23))); +} + +bool IsEven(int k) { return k % 2 == 0; } + +TEST(NodeHashSet, EraseIf) { + // Erase all elements. + { + node_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, [](int) { return true; }); + EXPECT_THAT(s, IsEmpty()); + } + // Erase no elements. + { + node_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, [](int) { return false; }); + EXPECT_THAT(s, UnorderedElementsAre(1, 2, 3, 4, 5)); + } + // Erase specific elements. + { + node_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, [](int k) { return k % 2 == 1; }); + EXPECT_THAT(s, UnorderedElementsAre(2, 4)); + } + // Predicate is function reference. + { + node_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, IsEven); + EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); + } + // Predicate is function pointer. + { + node_hash_set<int> s = {1, 2, 3, 4, 5}; + erase_if(s, &IsEven); + EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); + } +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/copts/AbseilConfigureCopts.cmake b/third_party/abseil_cpp/absl/copts/AbseilConfigureCopts.cmake new file mode 100644 index 000000000000..acd46d04aa5a --- /dev/null +++ b/third_party/abseil_cpp/absl/copts/AbseilConfigureCopts.cmake @@ -0,0 +1,67 @@ +# See absl/copts/copts.py and absl/copts/generate_copts.py +include(GENERATED_AbseilCopts) + +set(ABSL_LSAN_LINKOPTS "") +set(ABSL_HAVE_LSAN OFF) +set(ABSL_DEFAULT_LINKOPTS "") + +if (BUILD_SHARED_LIBS AND MSVC) + set(ABSL_BUILD_DLL TRUE) + set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON) +else() + set(ABSL_BUILD_DLL FALSE) +endif() + +if("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "x86_64|amd64|AMD64") + if (MSVC) + set(ABSL_RANDOM_RANDEN_COPTS "${ABSL_RANDOM_HWAES_MSVC_X64_FLAGS}") + else() + set(ABSL_RANDOM_RANDEN_COPTS "${ABSL_RANDOM_HWAES_X64_FLAGS}") + endif() +elseif("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "arm.*|aarch64") + if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8") + set(ABSL_RANDOM_RANDEN_COPTS "${ABSL_RANDOM_HWAES_ARM64_FLAGS}") + elseif("${CMAKE_SIZEOF_VOID_P}" STREQUAL "4") + set(ABSL_RANDOM_RANDEN_COPTS "${ABSL_RANDOM_HWAES_ARM32_FLAGS}") + else() + message(WARNING "Value of CMAKE_SIZEOF_VOID_P (${CMAKE_SIZEOF_VOID_P}) is not supported.") + endif() +else() + message(WARNING "Value of CMAKE_SYSTEM_PROCESSOR (${CMAKE_SYSTEM_PROCESSOR}) is unknown and cannot be used to set ABSL_RANDOM_RANDEN_COPTS") + set(ABSL_RANDOM_RANDEN_COPTS "") +endif() + + +if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(ABSL_DEFAULT_COPTS "${ABSL_GCC_FLAGS}") + set(ABSL_TEST_COPTS "${ABSL_GCC_FLAGS};${ABSL_GCC_TEST_FLAGS}") +elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang") + # MATCHES so we get both Clang and AppleClang + if(MSVC) + # clang-cl is half MSVC, half LLVM + set(ABSL_DEFAULT_COPTS "${ABSL_CLANG_CL_FLAGS}") + set(ABSL_TEST_COPTS "${ABSL_CLANG_CL_FLAGS};${ABSL_CLANG_CL_TEST_FLAGS}") + set(ABSL_DEFAULT_LINKOPTS "${ABSL_MSVC_LINKOPTS}") + else() + set(ABSL_DEFAULT_COPTS "${ABSL_LLVM_FLAGS}") + set(ABSL_TEST_COPTS "${ABSL_LLVM_FLAGS};${ABSL_LLVM_TEST_FLAGS}") + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + # AppleClang doesn't have lsan + # https://developer.apple.com/documentation/code_diagnostics + if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.5) + set(ABSL_LSAN_LINKOPTS "-fsanitize=leak") + set(ABSL_HAVE_LSAN ON) + endif() + endif() + endif() +elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") + set(ABSL_DEFAULT_COPTS "${ABSL_MSVC_FLAGS}") + set(ABSL_TEST_COPTS "${ABSL_MSVC_FLAGS};${ABSL_MSVC_TEST_FLAGS}") + set(ABSL_DEFAULT_LINKOPTS "${ABSL_MSVC_LINKOPTS}") +else() + message(WARNING "Unknown compiler: ${CMAKE_CXX_COMPILER}. Building with no default flags") + set(ABSL_DEFAULT_COPTS "") + set(ABSL_TEST_COPTS "") +endif() + +set(ABSL_CXX_STANDARD "${CMAKE_CXX_STANDARD}") diff --git a/third_party/abseil_cpp/absl/copts/GENERATED_AbseilCopts.cmake b/third_party/abseil_cpp/absl/copts/GENERATED_AbseilCopts.cmake new file mode 100644 index 000000000000..51742c9b6b3a --- /dev/null +++ b/third_party/abseil_cpp/absl/copts/GENERATED_AbseilCopts.cmake @@ -0,0 +1,162 @@ +# GENERATED! DO NOT MANUALLY EDIT THIS FILE. +# +# (1) Edit absl/copts/copts.py. +# (2) Run `python <path_to_absl>/copts/generate_copts.py`. + +list(APPEND ABSL_CLANG_CL_FLAGS + "/W3" + "/DNOMINMAX" + "/DWIN32_LEAN_AND_MEAN" + "/D_CRT_SECURE_NO_WARNINGS" + "/D_SCL_SECURE_NO_WARNINGS" + "/D_ENABLE_EXTENDED_ALIGNED_STORAGE" +) + +list(APPEND ABSL_CLANG_CL_TEST_FLAGS + "-Wno-c99-extensions" + "-Wno-deprecated-declarations" + "-Wno-missing-noreturn" + "-Wno-missing-prototypes" + "-Wno-missing-variable-declarations" + "-Wno-null-conversion" + "-Wno-shadow" + "-Wno-shift-sign-overflow" + "-Wno-sign-compare" + "-Wno-unused-function" + "-Wno-unused-member-function" + "-Wno-unused-parameter" + "-Wno-unused-private-field" + "-Wno-unused-template" + "-Wno-used-but-marked-unused" + "-Wno-zero-as-null-pointer-constant" + "-Wno-gnu-zero-variadic-macro-arguments" +) + +list(APPEND ABSL_GCC_FLAGS + "-Wall" + "-Wextra" + "-Wcast-qual" + "-Wconversion-null" + "-Wformat-security" + "-Wmissing-declarations" + "-Woverlength-strings" + "-Wpointer-arith" + "-Wundef" + "-Wunused-local-typedefs" + "-Wunused-result" + "-Wvarargs" + "-Wvla" + "-Wwrite-strings" + "-DNOMINMAX" +) + +list(APPEND ABSL_GCC_TEST_FLAGS + "-Wno-conversion-null" + "-Wno-deprecated-declarations" + "-Wno-missing-declarations" + "-Wno-sign-compare" + "-Wno-unused-function" + "-Wno-unused-parameter" + "-Wno-unused-private-field" +) + +list(APPEND ABSL_LLVM_FLAGS + "-Wall" + "-Wextra" + "-Wcast-qual" + "-Wconversion" + "-Wfloat-overflow-conversion" + "-Wfloat-zero-conversion" + "-Wfor-loop-analysis" + "-Wformat-security" + "-Wgnu-redeclared-enum" + "-Winfinite-recursion" + "-Wliteral-conversion" + "-Wmissing-declarations" + "-Woverlength-strings" + "-Wpointer-arith" + "-Wself-assign" + "-Wshadow" + "-Wstring-conversion" + "-Wtautological-overlap-compare" + "-Wundef" + "-Wuninitialized" + "-Wunreachable-code" + "-Wunused-comparison" + "-Wunused-local-typedefs" + "-Wunused-result" + "-Wvla" + "-Wwrite-strings" + "-Wno-float-conversion" + "-Wno-implicit-float-conversion" + "-Wno-implicit-int-float-conversion" + "-Wno-implicit-int-conversion" + "-Wno-shorten-64-to-32" + "-Wno-sign-conversion" + "-DNOMINMAX" +) + +list(APPEND ABSL_LLVM_TEST_FLAGS + "-Wno-c99-extensions" + "-Wno-deprecated-declarations" + "-Wno-missing-noreturn" + "-Wno-missing-prototypes" + "-Wno-missing-variable-declarations" + "-Wno-null-conversion" + "-Wno-shadow" + "-Wno-shift-sign-overflow" + "-Wno-sign-compare" + "-Wno-unused-function" + "-Wno-unused-member-function" + "-Wno-unused-parameter" + "-Wno-unused-private-field" + "-Wno-unused-template" + "-Wno-used-but-marked-unused" + "-Wno-zero-as-null-pointer-constant" + "-Wno-gnu-zero-variadic-macro-arguments" +) + +list(APPEND ABSL_MSVC_FLAGS + "/W3" + "/DNOMINMAX" + "/DWIN32_LEAN_AND_MEAN" + "/D_CRT_SECURE_NO_WARNINGS" + "/D_SCL_SECURE_NO_WARNINGS" + "/D_ENABLE_EXTENDED_ALIGNED_STORAGE" + "/bigobj" + "/wd4005" + "/wd4068" + "/wd4180" + "/wd4244" + "/wd4267" + "/wd4503" + "/wd4800" +) + +list(APPEND ABSL_MSVC_LINKOPTS + "-ignore:4221" +) + +list(APPEND ABSL_MSVC_TEST_FLAGS + "/wd4018" + "/wd4101" + "/wd4503" + "/wd4996" + "/DNOMINMAX" +) + +list(APPEND ABSL_RANDOM_HWAES_ARM32_FLAGS + "-mfpu=neon" +) + +list(APPEND ABSL_RANDOM_HWAES_ARM64_FLAGS + "-march=armv8-a+crypto" +) + +list(APPEND ABSL_RANDOM_HWAES_MSVC_X64_FLAGS +) + +list(APPEND ABSL_RANDOM_HWAES_X64_FLAGS + "-maes" + "-msse4.1" +) diff --git a/third_party/abseil_cpp/absl/copts/GENERATED_copts.bzl b/third_party/abseil_cpp/absl/copts/GENERATED_copts.bzl new file mode 100644 index 000000000000..6707488f23db --- /dev/null +++ b/third_party/abseil_cpp/absl/copts/GENERATED_copts.bzl @@ -0,0 +1,163 @@ +"""GENERATED! DO NOT MANUALLY EDIT THIS FILE. + +(1) Edit absl/copts/copts.py. +(2) Run `python <path_to_absl>/copts/generate_copts.py`. +""" + +ABSL_CLANG_CL_FLAGS = [ + "/W3", + "/DNOMINMAX", + "/DWIN32_LEAN_AND_MEAN", + "/D_CRT_SECURE_NO_WARNINGS", + "/D_SCL_SECURE_NO_WARNINGS", + "/D_ENABLE_EXTENDED_ALIGNED_STORAGE", +] + +ABSL_CLANG_CL_TEST_FLAGS = [ + "-Wno-c99-extensions", + "-Wno-deprecated-declarations", + "-Wno-missing-noreturn", + "-Wno-missing-prototypes", + "-Wno-missing-variable-declarations", + "-Wno-null-conversion", + "-Wno-shadow", + "-Wno-shift-sign-overflow", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-member-function", + "-Wno-unused-parameter", + "-Wno-unused-private-field", + "-Wno-unused-template", + "-Wno-used-but-marked-unused", + "-Wno-zero-as-null-pointer-constant", + "-Wno-gnu-zero-variadic-macro-arguments", +] + +ABSL_GCC_FLAGS = [ + "-Wall", + "-Wextra", + "-Wcast-qual", + "-Wconversion-null", + "-Wformat-security", + "-Wmissing-declarations", + "-Woverlength-strings", + "-Wpointer-arith", + "-Wundef", + "-Wunused-local-typedefs", + "-Wunused-result", + "-Wvarargs", + "-Wvla", + "-Wwrite-strings", + "-DNOMINMAX", +] + +ABSL_GCC_TEST_FLAGS = [ + "-Wno-conversion-null", + "-Wno-deprecated-declarations", + "-Wno-missing-declarations", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-parameter", + "-Wno-unused-private-field", +] + +ABSL_LLVM_FLAGS = [ + "-Wall", + "-Wextra", + "-Wcast-qual", + "-Wconversion", + "-Wfloat-overflow-conversion", + "-Wfloat-zero-conversion", + "-Wfor-loop-analysis", + "-Wformat-security", + "-Wgnu-redeclared-enum", + "-Winfinite-recursion", + "-Wliteral-conversion", + "-Wmissing-declarations", + "-Woverlength-strings", + "-Wpointer-arith", + "-Wself-assign", + "-Wshadow", + "-Wstring-conversion", + "-Wtautological-overlap-compare", + "-Wundef", + "-Wuninitialized", + "-Wunreachable-code", + "-Wunused-comparison", + "-Wunused-local-typedefs", + "-Wunused-result", + "-Wvla", + "-Wwrite-strings", + "-Wno-float-conversion", + "-Wno-implicit-float-conversion", + "-Wno-implicit-int-float-conversion", + "-Wno-implicit-int-conversion", + "-Wno-shorten-64-to-32", + "-Wno-sign-conversion", + "-DNOMINMAX", +] + +ABSL_LLVM_TEST_FLAGS = [ + "-Wno-c99-extensions", + "-Wno-deprecated-declarations", + "-Wno-missing-noreturn", + "-Wno-missing-prototypes", + "-Wno-missing-variable-declarations", + "-Wno-null-conversion", + "-Wno-shadow", + "-Wno-shift-sign-overflow", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-member-function", + "-Wno-unused-parameter", + "-Wno-unused-private-field", + "-Wno-unused-template", + "-Wno-used-but-marked-unused", + "-Wno-zero-as-null-pointer-constant", + "-Wno-gnu-zero-variadic-macro-arguments", +] + +ABSL_MSVC_FLAGS = [ + "/W3", + "/DNOMINMAX", + "/DWIN32_LEAN_AND_MEAN", + "/D_CRT_SECURE_NO_WARNINGS", + "/D_SCL_SECURE_NO_WARNINGS", + "/D_ENABLE_EXTENDED_ALIGNED_STORAGE", + "/bigobj", + "/wd4005", + "/wd4068", + "/wd4180", + "/wd4244", + "/wd4267", + "/wd4503", + "/wd4800", +] + +ABSL_MSVC_LINKOPTS = [ + "-ignore:4221", +] + +ABSL_MSVC_TEST_FLAGS = [ + "/wd4018", + "/wd4101", + "/wd4503", + "/wd4996", + "/DNOMINMAX", +] + +ABSL_RANDOM_HWAES_ARM32_FLAGS = [ + "-mfpu=neon", +] + +ABSL_RANDOM_HWAES_ARM64_FLAGS = [ + "-march=armv8-a+crypto", +] + +ABSL_RANDOM_HWAES_MSVC_X64_FLAGS = [ +] + +ABSL_RANDOM_HWAES_X64_FLAGS = [ + "-maes", + "-msse4.1", +] diff --git a/third_party/abseil_cpp/absl/copts/configure_copts.bzl b/third_party/abseil_cpp/absl/copts/configure_copts.bzl new file mode 100644 index 000000000000..4d3425444349 --- /dev/null +++ b/third_party/abseil_cpp/absl/copts/configure_copts.bzl @@ -0,0 +1,76 @@ +"""absl specific copts. + +This file simply selects the correct options from the generated files. To +change Abseil copts, edit absl/copts/copts.py +""" + +load( + "//absl:copts/GENERATED_copts.bzl", + "ABSL_CLANG_CL_FLAGS", + "ABSL_CLANG_CL_TEST_FLAGS", + "ABSL_GCC_FLAGS", + "ABSL_GCC_TEST_FLAGS", + "ABSL_LLVM_FLAGS", + "ABSL_LLVM_TEST_FLAGS", + "ABSL_MSVC_FLAGS", + "ABSL_MSVC_LINKOPTS", + "ABSL_MSVC_TEST_FLAGS", + "ABSL_RANDOM_HWAES_ARM32_FLAGS", + "ABSL_RANDOM_HWAES_ARM64_FLAGS", + "ABSL_RANDOM_HWAES_MSVC_X64_FLAGS", + "ABSL_RANDOM_HWAES_X64_FLAGS", +) + +ABSL_DEFAULT_COPTS = select({ + "//absl:windows": ABSL_MSVC_FLAGS, + "//absl:clang_compiler": ABSL_LLVM_FLAGS, + "//conditions:default": ABSL_GCC_FLAGS, +}) + +ABSL_TEST_COPTS = ABSL_DEFAULT_COPTS + select({ + "//absl:windows": ABSL_MSVC_TEST_FLAGS, + "//absl:clang_compiler": ABSL_LLVM_TEST_FLAGS, + "//conditions:default": ABSL_GCC_TEST_FLAGS, +}) + +ABSL_DEFAULT_LINKOPTS = select({ + "//absl:windows": ABSL_MSVC_LINKOPTS, + "//conditions:default": [], +}) + +# ABSL_RANDOM_RANDEN_COPTS blaze copts flags which are required by each +# environment to build an accelerated RandenHwAes library. +ABSL_RANDOM_RANDEN_COPTS = select({ + # APPLE + ":cpu_darwin_x86_64": ABSL_RANDOM_HWAES_X64_FLAGS, + ":cpu_darwin": ABSL_RANDOM_HWAES_X64_FLAGS, + ":cpu_x64_windows_msvc": ABSL_RANDOM_HWAES_MSVC_X64_FLAGS, + ":cpu_x64_windows": ABSL_RANDOM_HWAES_MSVC_X64_FLAGS, + ":cpu_k8": ABSL_RANDOM_HWAES_X64_FLAGS, + ":cpu_ppc": ["-mcrypto"], + + # Supported by default or unsupported. + "//conditions:default": [], +}) + +# absl_random_randen_copts_init: +# Initialize the config targets based on cpu, os, etc. used to select +# the required values for ABSL_RANDOM_RANDEN_COPTS +def absl_random_randen_copts_init(): + """Initialize the config_settings used by ABSL_RANDOM_RANDEN_COPTS.""" + + # CPU configs. + # These configs have consistent flags to enable HWAES intsructions. + cpu_configs = [ + "ppc", + "k8", + "darwin_x86_64", + "darwin", + "x64_windows_msvc", + "x64_windows", + ] + for cpu in cpu_configs: + native.config_setting( + name = "cpu_%s" % cpu, + values = {"cpu": cpu}, + ) diff --git a/third_party/abseil_cpp/absl/copts/copts.py b/third_party/abseil_cpp/absl/copts/copts.py new file mode 100644 index 000000000000..cf52981c58cc --- /dev/null +++ b/third_party/abseil_cpp/absl/copts/copts.py @@ -0,0 +1,162 @@ +"""Abseil compiler options. + +This is the source of truth for Abseil compiler options. To modify Abseil +compilation options: + + (1) Edit the appropriate list in this file based on the platform the flag is + needed on. + (2) Run `<path_to_absl>/copts/generate_copts.py`. + +The generated copts are consumed by configure_copts.bzl and +AbseilConfigureCopts.cmake. +""" + +# /Wall with msvc includes unhelpful warnings such as C4711, C4710, ... +MSVC_BIG_WARNING_FLAGS = [ + "/W3", +] + +LLVM_TEST_DISABLE_WARNINGS_FLAGS = [ + "-Wno-c99-extensions", + "-Wno-deprecated-declarations", + "-Wno-missing-noreturn", + "-Wno-missing-prototypes", + "-Wno-missing-variable-declarations", + "-Wno-null-conversion", + "-Wno-shadow", + "-Wno-shift-sign-overflow", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-member-function", + "-Wno-unused-parameter", + "-Wno-unused-private-field", + "-Wno-unused-template", + "-Wno-used-but-marked-unused", + "-Wno-zero-as-null-pointer-constant", + # gtest depends on this GNU extension being offered. + "-Wno-gnu-zero-variadic-macro-arguments", +] + +MSVC_DEFINES = [ + "/DNOMINMAX", # Don't define min and max macros (windows.h) + # Don't bloat namespace with incompatible winsock versions. + "/DWIN32_LEAN_AND_MEAN", + # Don't warn about usage of insecure C functions. + "/D_CRT_SECURE_NO_WARNINGS", + "/D_SCL_SECURE_NO_WARNINGS", + # Introduced in VS 2017 15.8, allow overaligned types in aligned_storage + "/D_ENABLE_EXTENDED_ALIGNED_STORAGE", +] + +COPT_VARS = { + "ABSL_GCC_FLAGS": [ + "-Wall", + "-Wextra", + "-Wcast-qual", + "-Wconversion-null", + "-Wformat-security", + "-Wmissing-declarations", + "-Woverlength-strings", + "-Wpointer-arith", + "-Wundef", + "-Wunused-local-typedefs", + "-Wunused-result", + "-Wvarargs", + "-Wvla", # variable-length array + "-Wwrite-strings", + # Don't define min and max macros (Build on Windows using gcc) + "-DNOMINMAX", + ], + "ABSL_GCC_TEST_FLAGS": [ + "-Wno-conversion-null", + "-Wno-deprecated-declarations", + "-Wno-missing-declarations", + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-parameter", + "-Wno-unused-private-field", + ], + "ABSL_LLVM_FLAGS": [ + "-Wall", + "-Wextra", + "-Wcast-qual", + "-Wconversion", + "-Wfloat-overflow-conversion", + "-Wfloat-zero-conversion", + "-Wfor-loop-analysis", + "-Wformat-security", + "-Wgnu-redeclared-enum", + "-Winfinite-recursion", + "-Wliteral-conversion", + "-Wmissing-declarations", + "-Woverlength-strings", + "-Wpointer-arith", + "-Wself-assign", + "-Wshadow", + "-Wstring-conversion", + "-Wtautological-overlap-compare", + "-Wundef", + "-Wuninitialized", + "-Wunreachable-code", + "-Wunused-comparison", + "-Wunused-local-typedefs", + "-Wunused-result", + "-Wvla", + "-Wwrite-strings", + # Warnings that are enabled by group warning flags like -Wall that we + # explicitly disable. + "-Wno-float-conversion", + "-Wno-implicit-float-conversion", + "-Wno-implicit-int-float-conversion", + "-Wno-implicit-int-conversion", + "-Wno-shorten-64-to-32", + "-Wno-sign-conversion", + # Don't define min and max macros (Build on Windows using clang) + "-DNOMINMAX", + ], + "ABSL_LLVM_TEST_FLAGS": + LLVM_TEST_DISABLE_WARNINGS_FLAGS, + "ABSL_CLANG_CL_FLAGS": + (MSVC_BIG_WARNING_FLAGS + MSVC_DEFINES), + "ABSL_CLANG_CL_TEST_FLAGS": + LLVM_TEST_DISABLE_WARNINGS_FLAGS, + "ABSL_MSVC_FLAGS": + MSVC_BIG_WARNING_FLAGS + MSVC_DEFINES + [ + # Increase the number of sections available in object files + "/bigobj", + "/wd4005", # macro-redefinition + "/wd4068", # unknown pragma + # qualifier applied to function type has no meaning; ignored + "/wd4180", + # conversion from 'type1' to 'type2', possible loss of data + "/wd4244", + # conversion from 'size_t' to 'type', possible loss of data + "/wd4267", + # The decorated name was longer than the compiler limit + "/wd4503", + # forcing value to bool 'true' or 'false' (performance warning) + "/wd4800", + ], + "ABSL_MSVC_TEST_FLAGS": [ + "/wd4018", # signed/unsigned mismatch + "/wd4101", # unreferenced local variable + "/wd4503", # decorated name length exceeded, name was truncated + "/wd4996", # use of deprecated symbol + "/DNOMINMAX", # disable the min() and max() macros from <windows.h> + ], + "ABSL_MSVC_LINKOPTS": [ + # Object file doesn't export any previously undefined symbols + "-ignore:4221", + ], + # "HWAES" is an abbreviation for "hardware AES" (AES - Advanced Encryption + # Standard). These flags are used for detecting whether or not the target + # architecture has hardware support for AES instructions which can be used + # to improve performance of some random bit generators. + "ABSL_RANDOM_HWAES_ARM64_FLAGS": ["-march=armv8-a+crypto"], + "ABSL_RANDOM_HWAES_ARM32_FLAGS": ["-mfpu=neon"], + "ABSL_RANDOM_HWAES_X64_FLAGS": [ + "-maes", + "-msse4.1", + ], + "ABSL_RANDOM_HWAES_MSVC_X64_FLAGS": [], +} diff --git a/third_party/abseil_cpp/absl/copts/generate_copts.py b/third_party/abseil_cpp/absl/copts/generate_copts.py new file mode 100755 index 000000000000..0e5dc9fad295 --- /dev/null +++ b/third_party/abseil_cpp/absl/copts/generate_copts.py @@ -0,0 +1,109 @@ +#!/usr/bin/python +"""Generate Abseil compile compile option configs. + +Usage: <path_to_absl>/copts/generate_copts.py + +The configs are generated from copts.py. +""" + +from os import path +import sys +from copts import COPT_VARS + + +# Helper functions +def file_header_lines(): + return [ + "GENERATED! DO NOT MANUALLY EDIT THIS FILE.", "", + "(1) Edit absl/copts/copts.py.", + "(2) Run `python <path_to_absl>/copts/generate_copts.py`." + ] + + +def flatten(*lists): + return [item for sublist in lists for item in sublist] + + +def relative_filename(filename): + return path.join(path.dirname(__file__), filename) + + +# Style classes. These contain all the syntactic styling needed to generate a +# copt file for different build tools. +class CMakeStyle(object): + """Style object for CMake copts file.""" + + def separator(self): + return "" + + def list_introducer(self, name): + return "list(APPEND " + name + + def list_closer(self): + return ")\n" + + def docstring(self): + return "\n".join((("# " + line).strip() for line in file_header_lines())) + + def filename(self): + return "GENERATED_AbseilCopts.cmake" + + +class StarlarkStyle(object): + """Style object for Starlark copts file.""" + + def separator(self): + return "," + + def list_introducer(self, name): + return name + " = [" + + def list_closer(self): + return "]\n" + + def docstring(self): + docstring_quotes = "\"\"\"" + return docstring_quotes + "\n".join( + flatten(file_header_lines(), [docstring_quotes])) + + def filename(self): + return "GENERATED_copts.bzl" + + +def copt_list(name, arg_list, style): + """Copt file generation.""" + + make_line = lambda s: " \"" + s + "\"" + style.separator() + external_str_list = [make_line(s) for s in arg_list] + + return "\n".join( + flatten( + [style.list_introducer(name)], + external_str_list, + [style.list_closer()])) + + +def generate_copt_file(style): + """Creates a generated copt file using the given style object. + + Args: + style: either StarlarkStyle() or CMakeStyle() + """ + with open(relative_filename(style.filename()), "w") as f: + f.write(style.docstring()) + f.write("\n") + for var_name, arg_list in sorted(COPT_VARS.items()): + f.write("\n") + f.write(copt_list(var_name, arg_list, style)) + + +def main(argv): + if len(argv) > 1: + raise RuntimeError("generate_copts needs no command line args") + + generate_copt_file(StarlarkStyle()) + generate_copt_file(CMakeStyle()) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/third_party/abseil_cpp/absl/debugging/BUILD.bazel b/third_party/abseil_cpp/absl/debugging/BUILD.bazel new file mode 100644 index 000000000000..cd6e45439657 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/BUILD.bazel @@ -0,0 +1,347 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package( + default_visibility = ["//visibility:public"], +) + +licenses(["notice"]) + +cc_library( + name = "stacktrace", + srcs = [ + "internal/stacktrace_aarch64-inl.inc", + "internal/stacktrace_arm-inl.inc", + "internal/stacktrace_config.h", + "internal/stacktrace_generic-inl.inc", + "internal/stacktrace_powerpc-inl.inc", + "internal/stacktrace_unimplemented-inl.inc", + "internal/stacktrace_win32-inl.inc", + "internal/stacktrace_x86-inl.inc", + "stacktrace.cc", + ], + hdrs = ["stacktrace.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":debugging_internal", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_library( + name = "symbolize", + srcs = [ + "symbolize.cc", + "symbolize_darwin.inc", + "symbolize_elf.inc", + "symbolize_unimplemented.inc", + "symbolize_win32.inc", + ], + hdrs = [ + "internal/symbolize.h", + "symbolize.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS + select({ + "//absl:windows": ["-DEFAULTLIB:dbghelp.lib"], + "//conditions:default": [], + }), + deps = [ + ":debugging_internal", + ":demangle_internal", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:dynamic_annotations", + "//absl/base:malloc_internal", + "//absl/base:raw_logging_internal", + "//absl/strings", + ], +) + +cc_test( + name = "symbolize_test", + srcs = ["symbolize_test.cc"], + copts = ABSL_TEST_COPTS + select({ + "//absl:windows": ["/Z7"], + "//conditions:default": [], + }), + linkopts = ABSL_DEFAULT_LINKOPTS + select({ + "//absl:windows": ["/DEBUG"], + "//conditions:default": [], + }), + deps = [ + ":stack_consumption", + ":symbolize", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "examine_stack", + srcs = [ + "internal/examine_stack.cc", + ], + hdrs = [ + "internal/examine_stack.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":stacktrace", + ":symbolize", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "failure_signal_handler", + srcs = ["failure_signal_handler.cc"], + hdrs = ["failure_signal_handler.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":examine_stack", + ":stacktrace", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:errno_saver", + "//absl/base:raw_logging_internal", + ], +) + +cc_test( + name = "failure_signal_handler_test", + srcs = ["failure_signal_handler_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = select({ + "//absl:windows": [], + "//absl:wasm": [], + "//conditions:default": ["-pthread"], + }) + ABSL_DEFAULT_LINKOPTS, + deps = [ + ":failure_signal_handler", + ":stacktrace", + ":symbolize", + "//absl/base:raw_logging_internal", + "//absl/strings", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "debugging_internal", + srcs = [ + "internal/address_is_readable.cc", + "internal/elf_mem_image.cc", + "internal/vdso_support.cc", + ], + hdrs = [ + "internal/address_is_readable.h", + "internal/elf_mem_image.h", + "internal/vdso_support.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:dynamic_annotations", + "//absl/base:errno_saver", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "demangle_internal", + srcs = ["internal/demangle.cc"], + hdrs = ["internal/demangle.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "demangle_test", + srcs = ["internal/demangle_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":demangle_internal", + ":stack_consumption", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "leak_check", + srcs = ["leak_check.cc"], + hdrs = ["leak_check.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +# Adding a dependency to leak_check_disable will disable +# sanitizer leak checking (asan/lsan) in a test without +# the need to mess around with build features. +cc_library( + name = "leak_check_disable", + srcs = ["leak_check_disable.cc"], + linkopts = ABSL_DEFAULT_LINKOPTS, + linkstatic = 1, + deps = ["//absl/base:config"], + alwayslink = 1, +) + +# These targets exists for use in tests only, explicitly configuring the +# LEAK_SANITIZER macro. It must be linked with -fsanitize=leak for lsan. +ABSL_LSAN_LINKOPTS = select({ + "//absl:clang_compiler": ["-fsanitize=leak"], + "//conditions:default": [], +}) + +cc_library( + name = "leak_check_api_enabled_for_testing", + testonly = 1, + srcs = ["leak_check.cc"], + hdrs = ["leak_check.h"], + copts = select({ + "//absl:clang_compiler": ["-DLEAK_SANITIZER"], + "//conditions:default": [], + }), + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + "//absl/base:config", + ], +) + +cc_library( + name = "leak_check_api_disabled_for_testing", + testonly = 1, + srcs = ["leak_check.cc"], + hdrs = ["leak_check.h"], + copts = ["-ULEAK_SANITIZER"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + "//absl/base:config", + ], +) + +cc_test( + name = "leak_check_test", + srcs = ["leak_check_test.cc"], + copts = select({ + "//absl:clang_compiler": ["-DABSL_EXPECT_LEAK_SANITIZER"], + "//conditions:default": [], + }), + linkopts = ABSL_LSAN_LINKOPTS + ABSL_DEFAULT_LINKOPTS, + tags = ["notsan"], + deps = [ + ":leak_check_api_enabled_for_testing", + "//absl/base", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "leak_check_no_lsan_test", + srcs = ["leak_check_test.cc"], + copts = ["-UABSL_EXPECT_LEAK_SANITIZER"], + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["noasan"], + deps = [ + ":leak_check_api_disabled_for_testing", + "//absl/base", # for raw_logging + "@com_google_googletest//:gtest_main", + ], +) + +# Test that leak checking is skipped when lsan is enabled but +# ":leak_check_disable" is linked in. +# +# This test should fail in the absence of a dependency on ":leak_check_disable" +cc_test( + name = "disabled_leak_check_test", + srcs = ["leak_check_fail_test.cc"], + linkopts = ABSL_LSAN_LINKOPTS + ABSL_DEFAULT_LINKOPTS, + tags = ["notsan"], + deps = [ + ":leak_check_api_enabled_for_testing", + ":leak_check_disable", + "//absl/base", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "stack_consumption", + testonly = 1, + srcs = ["internal/stack_consumption.cc"], + hdrs = ["internal/stack_consumption.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + ], +) + +cc_test( + name = "stack_consumption_test", + srcs = ["internal/stack_consumption_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":stack_consumption", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/debugging/CMakeLists.txt b/third_party/abseil_cpp/absl/debugging/CMakeLists.txt new file mode 100644 index 000000000000..074b44cf1781 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/CMakeLists.txt @@ -0,0 +1,338 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + stacktrace + HDRS + "stacktrace.h" + "internal/stacktrace_aarch64-inl.inc" + "internal/stacktrace_arm-inl.inc" + "internal/stacktrace_config.h" + "internal/stacktrace_generic-inl.inc" + "internal/stacktrace_powerpc-inl.inc" + "internal/stacktrace_unimplemented-inl.inc" + "internal/stacktrace_win32-inl.inc" + "internal/stacktrace_x86-inl.inc" + SRCS + "stacktrace.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::debugging_internal + absl::config + absl::core_headers + PUBLIC +) + +absl_cc_library( + NAME + symbolize + HDRS + "symbolize.h" + "internal/symbolize.h" + SRCS + "symbolize.cc" + "symbolize_darwin.inc" + "symbolize_elf.inc" + "symbolize_unimplemented.inc" + "symbolize_win32.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + $<$<BOOL:${MINGW}>:"dbghelp"> + DEPS + absl::debugging_internal + absl::demangle_internal + absl::base + absl::config + absl::core_headers + absl::dynamic_annotations + absl::malloc_internal + absl::raw_logging_internal + absl::strings + PUBLIC +) + +absl_cc_test( + NAME + symbolize_test + SRCS + "symbolize_test.cc" + COPTS + ${ABSL_TEST_COPTS} + $<$<BOOL:${MSVC}>:-Z7> + LINKOPTS + $<$<BOOL:${MSVC}>:-DEBUG> + DEPS + absl::stack_consumption + absl::symbolize + absl::base + absl::config + absl::core_headers + absl::memory + absl::raw_logging_internal + absl::strings + gmock +) + +absl_cc_library( + NAME + examine_stack + HDRS + "internal/examine_stack.h" + SRCS + "internal/examine_stack.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::stacktrace + absl::symbolize + absl::config + absl::core_headers + absl::raw_logging_internal +) + +absl_cc_library( + NAME + failure_signal_handler + HDRS + "failure_signal_handler.h" + SRCS + "failure_signal_handler.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::examine_stack + absl::stacktrace + absl::base + absl::config + absl::core_headers + absl::errno_saver + absl::raw_logging_internal + PUBLIC +) + +absl_cc_test( + NAME + failure_signal_handler_test + SRCS + "failure_signal_handler_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::failure_signal_handler + absl::stacktrace + absl::symbolize + absl::strings + absl::raw_logging_internal + Threads::Threads + gmock +) + +absl_cc_library( + NAME + debugging_internal + HDRS + "internal/address_is_readable.h" + "internal/elf_mem_image.h" + "internal/vdso_support.h" + SRCS + "internal/address_is_readable.cc" + "internal/elf_mem_image.cc" + "internal/vdso_support.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::config + absl::dynamic_annotations + absl::errno_saver + absl::raw_logging_internal +) + +absl_cc_library( + NAME + demangle_internal + HDRS + "internal/demangle.h" + SRCS + "internal/demangle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::core_headers + PUBLIC +) + +absl_cc_test( + NAME + demangle_test + SRCS + "internal/demangle_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::demangle_internal + absl::stack_consumption + absl::config + absl::core_headers + absl::memory + absl::raw_logging_internal + gmock_main +) + +absl_cc_library( + NAME + leak_check + HDRS + "leak_check.h" + SRCS + "leak_check.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + PUBLIC +) + +absl_cc_library( + NAME + leak_check_disable + SRCS + "leak_check_disable.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + PUBLIC +) + +absl_cc_library( + NAME + leak_check_api_enabled_for_testing + HDRS + "leak_check.h" + SRCS + "leak_check.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + $<$<BOOL:${ABSL_HAVE_LSAN}>:-DLEAK_SANITIZER> + TESTONLY +) + +absl_cc_library( + NAME + leak_check_api_disabled_for_testing + HDRS + "leak_check.h" + SRCS + "leak_check.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + "-ULEAK_SANITIZER" + TESTONLY +) + +absl_cc_test( + NAME + leak_check_test + SRCS + "leak_check_test.cc" + COPTS + ${ABSL_TEST_COPTS} + "$<$<BOOL:${ABSL_HAVE_LSAN}>:-DABSL_EXPECT_LEAK_SANITIZER>" + LINKOPTS + "${ABSL_LSAN_LINKOPTS}" + DEPS + absl::leak_check_api_enabled_for_testing + absl::base + gmock_main +) + +absl_cc_test( + NAME + leak_check_no_lsan_test + SRCS + "leak_check_test.cc" + COPTS + ${ABSL_TEST_COPTS} + "-UABSL_EXPECT_LEAK_SANITIZER" + DEPS + absl::leak_check_api_disabled_for_testing + absl::base + gmock_main +) + +absl_cc_test( + NAME + disabled_leak_check_test + SRCS + "leak_check_fail_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + "${ABSL_LSAN_LINKOPTS}" + DEPS + absl::leak_check_api_enabled_for_testing + absl::leak_check_disable + absl::base + absl::raw_logging_internal + gmock_main +) + +absl_cc_library( + NAME + stack_consumption + HDRS + "internal/stack_consumption.h" + SRCS + "internal/stack_consumption.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::raw_logging_internal + TESTONLY +) + +absl_cc_test( + NAME + stack_consumption_test + SRCS + "internal/stack_consumption_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::stack_consumption + absl::core_headers + absl::raw_logging_internal + gmock_main +) + +# component target +absl_cc_library( + NAME + debugging + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::stacktrace + absl::leak_check + PUBLIC +) diff --git a/third_party/abseil_cpp/absl/debugging/failure_signal_handler.cc b/third_party/abseil_cpp/absl/debugging/failure_signal_handler.cc new file mode 100644 index 000000000000..5d13bdbbbd13 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/failure_signal_handler.cc @@ -0,0 +1,370 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "absl/debugging/failure_signal_handler.h" + +#include "absl/base/config.h" + +#ifdef _WIN32 +#include <windows.h> +#else +#include <unistd.h> +#endif + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif + +#ifdef ABSL_HAVE_MMAP +#include <sys/mman.h> +#endif + +#include <algorithm> +#include <atomic> +#include <cerrno> +#include <csignal> +#include <cstdio> +#include <cstring> +#include <ctime> + +#include "absl/base/attributes.h" +#include "absl/base/internal/errno_saver.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/sysinfo.h" +#include "absl/debugging/internal/examine_stack.h" +#include "absl/debugging/stacktrace.h" + +#ifndef _WIN32 +#define ABSL_HAVE_SIGACTION +// Apple WatchOS and TVOS don't allow sigaltstack +#if !(defined(TARGET_OS_WATCH) && TARGET_OS_WATCH) && \ + !(defined(TARGET_OS_TV) && TARGET_OS_TV) +#define ABSL_HAVE_SIGALTSTACK +#endif +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_CONST_INIT static FailureSignalHandlerOptions fsh_options; + +// Resets the signal handler for signo to the default action for that +// signal, then raises the signal. +static void RaiseToDefaultHandler(int signo) { + signal(signo, SIG_DFL); + raise(signo); +} + +struct FailureSignalData { + const int signo; + const char* const as_string; +#ifdef ABSL_HAVE_SIGACTION + struct sigaction previous_action; + // StructSigaction is used to silence -Wmissing-field-initializers. + using StructSigaction = struct sigaction; + #define FSD_PREVIOUS_INIT FailureSignalData::StructSigaction() +#else + void (*previous_handler)(int); + #define FSD_PREVIOUS_INIT SIG_DFL +#endif +}; + +ABSL_CONST_INIT static FailureSignalData failure_signal_data[] = { + {SIGSEGV, "SIGSEGV", FSD_PREVIOUS_INIT}, + {SIGILL, "SIGILL", FSD_PREVIOUS_INIT}, + {SIGFPE, "SIGFPE", FSD_PREVIOUS_INIT}, + {SIGABRT, "SIGABRT", FSD_PREVIOUS_INIT}, + {SIGTERM, "SIGTERM", FSD_PREVIOUS_INIT}, +#ifndef _WIN32 + {SIGBUS, "SIGBUS", FSD_PREVIOUS_INIT}, + {SIGTRAP, "SIGTRAP", FSD_PREVIOUS_INIT}, +#endif +}; + +#undef FSD_PREVIOUS_INIT + +static void RaiseToPreviousHandler(int signo) { + // Search for the previous handler. + for (const auto& it : failure_signal_data) { + if (it.signo == signo) { +#ifdef ABSL_HAVE_SIGACTION + sigaction(signo, &it.previous_action, nullptr); +#else + signal(signo, it.previous_handler); +#endif + raise(signo); + return; + } + } + + // Not found, use the default handler. + RaiseToDefaultHandler(signo); +} + +namespace debugging_internal { + +const char* FailureSignalToString(int signo) { + for (const auto& it : failure_signal_data) { + if (it.signo == signo) { + return it.as_string; + } + } + return ""; +} + +} // namespace debugging_internal + +#ifdef ABSL_HAVE_SIGALTSTACK + +static bool SetupAlternateStackOnce() { +#if defined(__wasm__) || defined (__asjms__) + const size_t page_mask = getpagesize() - 1; +#else + const size_t page_mask = sysconf(_SC_PAGESIZE) - 1; +#endif + size_t stack_size = (std::max(SIGSTKSZ, 65536) + page_mask) & ~page_mask; +#if defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) + // Account for sanitizer instrumentation requiring additional stack space. + stack_size *= 5; +#endif + + stack_t sigstk; + memset(&sigstk, 0, sizeof(sigstk)); + sigstk.ss_size = stack_size; + +#ifdef ABSL_HAVE_MMAP +#ifndef MAP_STACK +#define MAP_STACK 0 +#endif +#if defined(MAP_ANON) && !defined(MAP_ANONYMOUS) +#define MAP_ANONYMOUS MAP_ANON +#endif + sigstk.ss_sp = mmap(nullptr, sigstk.ss_size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (sigstk.ss_sp == MAP_FAILED) { + ABSL_RAW_LOG(FATAL, "mmap() for alternate signal stack failed"); + } +#else + sigstk.ss_sp = malloc(sigstk.ss_size); + if (sigstk.ss_sp == nullptr) { + ABSL_RAW_LOG(FATAL, "malloc() for alternate signal stack failed"); + } +#endif + + if (sigaltstack(&sigstk, nullptr) != 0) { + ABSL_RAW_LOG(FATAL, "sigaltstack() failed with errno=%d", errno); + } + return true; +} + +#endif + +#ifdef ABSL_HAVE_SIGACTION + +// Sets up an alternate stack for signal handlers once. +// Returns the appropriate flag for sig_action.sa_flags +// if the system supports using an alternate stack. +static int MaybeSetupAlternateStack() { +#ifdef ABSL_HAVE_SIGALTSTACK + ABSL_ATTRIBUTE_UNUSED static const bool kOnce = SetupAlternateStackOnce(); + return SA_ONSTACK; +#else + return 0; +#endif +} + +static void InstallOneFailureHandler(FailureSignalData* data, + void (*handler)(int, siginfo_t*, void*)) { + struct sigaction act; + memset(&act, 0, sizeof(act)); + sigemptyset(&act.sa_mask); + act.sa_flags |= SA_SIGINFO; + // SA_NODEFER is required to handle SIGABRT from + // ImmediateAbortSignalHandler(). + act.sa_flags |= SA_NODEFER; + if (fsh_options.use_alternate_stack) { + act.sa_flags |= MaybeSetupAlternateStack(); + } + act.sa_sigaction = handler; + ABSL_RAW_CHECK(sigaction(data->signo, &act, &data->previous_action) == 0, + "sigaction() failed"); +} + +#else + +static void InstallOneFailureHandler(FailureSignalData* data, + void (*handler)(int)) { + data->previous_handler = signal(data->signo, handler); + ABSL_RAW_CHECK(data->previous_handler != SIG_ERR, "signal() failed"); +} + +#endif + +static void WriteToStderr(const char* data) { + absl::base_internal::ErrnoSaver errno_saver; + absl::raw_logging_internal::SafeWriteToStderr(data, strlen(data)); +} + +static void WriteSignalMessage(int signo, void (*writerfn)(const char*)) { + char buf[64]; + const char* const signal_string = + debugging_internal::FailureSignalToString(signo); + if (signal_string != nullptr && signal_string[0] != '\0') { + snprintf(buf, sizeof(buf), "*** %s received at time=%ld ***\n", + signal_string, + static_cast<long>(time(nullptr))); // NOLINT(runtime/int) + } else { + snprintf(buf, sizeof(buf), "*** Signal %d received at time=%ld ***\n", + signo, static_cast<long>(time(nullptr))); // NOLINT(runtime/int) + } + writerfn(buf); +} + +// `void*` might not be big enough to store `void(*)(const char*)`. +struct WriterFnStruct { + void (*writerfn)(const char*); +}; + +// Many of the absl::debugging_internal::Dump* functions in +// examine_stack.h take a writer function pointer that has a void* arg +// for historical reasons. failure_signal_handler_writer only takes a +// data pointer. This function converts between these types. +static void WriterFnWrapper(const char* data, void* arg) { + static_cast<WriterFnStruct*>(arg)->writerfn(data); +} + +// Convenient wrapper around DumpPCAndFrameSizesAndStackTrace() for signal +// handlers. "noinline" so that GetStackFrames() skips the top-most stack +// frame for this function. +ABSL_ATTRIBUTE_NOINLINE static void WriteStackTrace( + void* ucontext, bool symbolize_stacktrace, + void (*writerfn)(const char*, void*), void* writerfn_arg) { + constexpr int kNumStackFrames = 32; + void* stack[kNumStackFrames]; + int frame_sizes[kNumStackFrames]; + int min_dropped_frames; + int depth = absl::GetStackFramesWithContext( + stack, frame_sizes, kNumStackFrames, + 1, // Do not include this function in stack trace. + ucontext, &min_dropped_frames); + absl::debugging_internal::DumpPCAndFrameSizesAndStackTrace( + absl::debugging_internal::GetProgramCounter(ucontext), stack, frame_sizes, + depth, min_dropped_frames, symbolize_stacktrace, writerfn, writerfn_arg); +} + +// Called by AbslFailureSignalHandler() to write the failure info. It is +// called once with writerfn set to WriteToStderr() and then possibly +// with writerfn set to the user provided function. +static void WriteFailureInfo(int signo, void* ucontext, + void (*writerfn)(const char*)) { + WriterFnStruct writerfn_struct{writerfn}; + WriteSignalMessage(signo, writerfn); + WriteStackTrace(ucontext, fsh_options.symbolize_stacktrace, WriterFnWrapper, + &writerfn_struct); +} + +// absl::SleepFor() can't be used here since AbslInternalSleepFor() +// may be overridden to do something that isn't async-signal-safe on +// some platforms. +static void PortableSleepForSeconds(int seconds) { +#ifdef _WIN32 + Sleep(seconds * 1000); +#else + struct timespec sleep_time; + sleep_time.tv_sec = seconds; + sleep_time.tv_nsec = 0; + while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) {} +#endif +} + +#ifdef ABSL_HAVE_ALARM +// AbslFailureSignalHandler() installs this as a signal handler for +// SIGALRM, then sets an alarm to be delivered to the program after a +// set amount of time. If AbslFailureSignalHandler() hangs for more than +// the alarm timeout, ImmediateAbortSignalHandler() will abort the +// program. +static void ImmediateAbortSignalHandler(int) { + RaiseToDefaultHandler(SIGABRT); +} +#endif + +// absl::base_internal::GetTID() returns pid_t on most platforms, but +// returns absl::base_internal::pid_t on Windows. +using GetTidType = decltype(absl::base_internal::GetTID()); +ABSL_CONST_INIT static std::atomic<GetTidType> failed_tid(0); + +#ifndef ABSL_HAVE_SIGACTION +static void AbslFailureSignalHandler(int signo) { + void* ucontext = nullptr; +#else +static void AbslFailureSignalHandler(int signo, siginfo_t*, void* ucontext) { +#endif + + const GetTidType this_tid = absl::base_internal::GetTID(); + GetTidType previous_failed_tid = 0; + if (!failed_tid.compare_exchange_strong( + previous_failed_tid, static_cast<intptr_t>(this_tid), + std::memory_order_acq_rel, std::memory_order_relaxed)) { + ABSL_RAW_LOG( + ERROR, + "Signal %d raised at PC=%p while already in AbslFailureSignalHandler()", + signo, absl::debugging_internal::GetProgramCounter(ucontext)); + if (this_tid != previous_failed_tid) { + // Another thread is already in AbslFailureSignalHandler(), so wait + // a bit for it to finish. If the other thread doesn't kill us, + // we do so after sleeping. + PortableSleepForSeconds(3); + RaiseToDefaultHandler(signo); + // The recursively raised signal may be blocked until we return. + return; + } + } + +#ifdef ABSL_HAVE_ALARM + // Set an alarm to abort the program in case this code hangs or deadlocks. + if (fsh_options.alarm_on_failure_secs > 0) { + alarm(0); // Cancel any existing alarms. + signal(SIGALRM, ImmediateAbortSignalHandler); + alarm(fsh_options.alarm_on_failure_secs); + } +#endif + + // First write to stderr. + WriteFailureInfo(signo, ucontext, WriteToStderr); + + // Riskier code (because it is less likely to be async-signal-safe) + // goes after this point. + if (fsh_options.writerfn != nullptr) { + WriteFailureInfo(signo, ucontext, fsh_options.writerfn); + } + + if (fsh_options.call_previous_handler) { + RaiseToPreviousHandler(signo); + } else { + RaiseToDefaultHandler(signo); + } +} + +void InstallFailureSignalHandler(const FailureSignalHandlerOptions& options) { + fsh_options = options; + for (auto& it : failure_signal_data) { + InstallOneFailureHandler(&it, AbslFailureSignalHandler); + } +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/failure_signal_handler.h b/third_party/abseil_cpp/absl/debugging/failure_signal_handler.h new file mode 100644 index 000000000000..0c0f585d0fb4 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/failure_signal_handler.h @@ -0,0 +1,121 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: failure_signal_handler.h +// ----------------------------------------------------------------------------- +// +// This file configures the Abseil *failure signal handler* to capture and dump +// useful debugging information (such as a stacktrace) upon program failure. +// +// To use the failure signal handler, call `absl::InstallFailureSignalHandler()` +// very early in your program, usually in the first few lines of main(): +// +// int main(int argc, char** argv) { +// // Initialize the symbolizer to get a human-readable stack trace +// absl::InitializeSymbolizer(argv[0]); +// +// absl::FailureSignalHandlerOptions options; +// absl::InstallFailureSignalHandler(options); +// DoSomethingInteresting(); +// return 0; +// } +// +// Any program that raises a fatal signal (such as `SIGSEGV`, `SIGILL`, +// `SIGFPE`, `SIGABRT`, `SIGTERM`, `SIGBUG`, and `SIGTRAP`) will call the +// installed failure signal handler and provide debugging information to stderr. +// +// Note that you should *not* install the Abseil failure signal handler more +// than once. You may, of course, have another (non-Abseil) failure signal +// handler installed (which would be triggered if Abseil's failure signal +// handler sets `call_previous_handler` to `true`). + +#ifndef ABSL_DEBUGGING_FAILURE_SIGNAL_HANDLER_H_ +#define ABSL_DEBUGGING_FAILURE_SIGNAL_HANDLER_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// FailureSignalHandlerOptions +// +// Struct for holding `absl::InstallFailureSignalHandler()` configuration +// options. +struct FailureSignalHandlerOptions { + // If true, try to symbolize the stacktrace emitted on failure, provided that + // you have initialized a symbolizer for that purpose. (See symbolize.h for + // more information.) + bool symbolize_stacktrace = true; + + // If true, try to run signal handlers on an alternate stack (if supported on + // the given platform). An alternate stack is useful for program crashes due + // to a stack overflow; by running on a alternate stack, the signal handler + // may run even when normal stack space has been exausted. The downside of + // using an alternate stack is that extra memory for the alternate stack needs + // to be pre-allocated. + bool use_alternate_stack = true; + + // If positive, indicates the number of seconds after which the failure signal + // handler is invoked to abort the program. Setting such an alarm is useful in + // cases where the failure signal handler itself may become hung or + // deadlocked. + int alarm_on_failure_secs = 3; + + // If true, call the previously registered signal handler for the signal that + // was received (if one was registered) after the existing signal handler + // runs. This mechanism can be used to chain signal handlers together. + // + // If false, the signal is raised to the default handler for that signal + // (which normally terminates the program). + // + // IMPORTANT: If true, the chained fatal signal handlers must not try to + // recover from the fatal signal. Instead, they should terminate the program + // via some mechanism, like raising the default handler for the signal, or by + // calling `_exit()`. Note that the failure signal handler may put parts of + // the Abseil library into a state from which they cannot recover. + bool call_previous_handler = false; + + // If non-null, indicates a pointer to a callback function that will be called + // upon failure, with a string argument containing failure data. This function + // may be used as a hook to write failure data to a secondary location, such + // as a log file. This function may also be called with null data, as a hint + // to flush any buffered data before the program may be terminated. Consider + // flushing any buffered data in all calls to this function. + // + // Since this function runs within a signal handler, it should be + // async-signal-safe if possible. + // See http://man7.org/linux/man-pages/man7/signal-safety.7.html + void (*writerfn)(const char*) = nullptr; +}; + +// InstallFailureSignalHandler() +// +// Installs a signal handler for the common failure signals `SIGSEGV`, `SIGILL`, +// `SIGFPE`, `SIGABRT`, `SIGTERM`, `SIGBUG`, and `SIGTRAP` (provided they exist +// on the given platform). The failure signal handler dumps program failure data +// useful for debugging in an unspecified format to stderr. This data may +// include the program counter, a stacktrace, and register information on some +// systems; do not rely on an exact format for the output, as it is subject to +// change. +void InstallFailureSignalHandler(const FailureSignalHandlerOptions& options); + +namespace debugging_internal { +const char* FailureSignalToString(int signo); +} // namespace debugging_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_FAILURE_SIGNAL_HANDLER_H_ diff --git a/third_party/abseil_cpp/absl/debugging/failure_signal_handler_test.cc b/third_party/abseil_cpp/absl/debugging/failure_signal_handler_test.cc new file mode 100644 index 000000000000..d8283b2f47cd --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/failure_signal_handler_test.cc @@ -0,0 +1,159 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "absl/debugging/failure_signal_handler.h" + +#include <csignal> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <fstream> + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" + +namespace { + +using testing::StartsWith; + +#if GTEST_HAS_DEATH_TEST + +// For the parameterized death tests. GetParam() returns the signal number. +using FailureSignalHandlerDeathTest = ::testing::TestWithParam<int>; + +// This function runs in a fork()ed process on most systems. +void InstallHandlerAndRaise(int signo) { + absl::InstallFailureSignalHandler(absl::FailureSignalHandlerOptions()); + raise(signo); +} + +TEST_P(FailureSignalHandlerDeathTest, AbslFailureSignal) { + const int signo = GetParam(); + std::string exit_regex = absl::StrCat( + "\\*\\*\\* ", absl::debugging_internal::FailureSignalToString(signo), + " received at time="); +#ifndef _WIN32 + EXPECT_EXIT(InstallHandlerAndRaise(signo), testing::KilledBySignal(signo), + exit_regex); +#else + // Windows doesn't have testing::KilledBySignal(). + EXPECT_DEATH_IF_SUPPORTED(InstallHandlerAndRaise(signo), exit_regex); +#endif +} + +ABSL_CONST_INIT FILE* error_file = nullptr; + +void WriteToErrorFile(const char* msg) { + if (msg != nullptr) { + ABSL_RAW_CHECK(fwrite(msg, strlen(msg), 1, error_file) == 1, + "fwrite() failed"); + } + ABSL_RAW_CHECK(fflush(error_file) == 0, "fflush() failed"); +} + +std::string GetTmpDir() { + // TEST_TMPDIR is set by Bazel. Try the others when not running under Bazel. + static const char* const kTmpEnvVars[] = {"TEST_TMPDIR", "TMPDIR", "TEMP", + "TEMPDIR", "TMP"}; + for (const char* const var : kTmpEnvVars) { + const char* tmp_dir = std::getenv(var); + if (tmp_dir != nullptr) { + return tmp_dir; + } + } + + // Try something reasonable. + return "/tmp"; +} + +// This function runs in a fork()ed process on most systems. +void InstallHandlerWithWriteToFileAndRaise(const char* file, int signo) { + error_file = fopen(file, "w"); + ABSL_RAW_CHECK(error_file != nullptr, "Failed create error_file"); + absl::FailureSignalHandlerOptions options; + options.writerfn = WriteToErrorFile; + absl::InstallFailureSignalHandler(options); + raise(signo); +} + +TEST_P(FailureSignalHandlerDeathTest, AbslFatalSignalsWithWriterFn) { + const int signo = GetParam(); + std::string tmp_dir = GetTmpDir(); + std::string file = absl::StrCat(tmp_dir, "/signo_", signo); + + std::string exit_regex = absl::StrCat( + "\\*\\*\\* ", absl::debugging_internal::FailureSignalToString(signo), + " received at time="); +#ifndef _WIN32 + EXPECT_EXIT(InstallHandlerWithWriteToFileAndRaise(file.c_str(), signo), + testing::KilledBySignal(signo), exit_regex); +#else + // Windows doesn't have testing::KilledBySignal(). + EXPECT_DEATH_IF_SUPPORTED( + InstallHandlerWithWriteToFileAndRaise(file.c_str(), signo), exit_regex); +#endif + + // Open the file in this process and check its contents. + std::fstream error_output(file); + ASSERT_TRUE(error_output.is_open()) << file; + std::string error_line; + std::getline(error_output, error_line); + EXPECT_THAT( + error_line, + StartsWith(absl::StrCat( + "*** ", absl::debugging_internal::FailureSignalToString(signo), + " received at "))); + + if (absl::debugging_internal::StackTraceWorksForTest()) { + std::getline(error_output, error_line); + EXPECT_THAT(error_line, StartsWith("PC: ")); + } +} + +constexpr int kFailureSignals[] = { + SIGSEGV, SIGILL, SIGFPE, SIGABRT, SIGTERM, +#ifndef _WIN32 + SIGBUS, SIGTRAP, +#endif +}; + +std::string SignalParamToString(const ::testing::TestParamInfo<int>& info) { + std::string result = + absl::debugging_internal::FailureSignalToString(info.param); + if (result.empty()) { + result = absl::StrCat(info.param); + } + return result; +} + +INSTANTIATE_TEST_SUITE_P(AbslDeathTest, FailureSignalHandlerDeathTest, + ::testing::ValuesIn(kFailureSignals), + SignalParamToString); + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace + +int main(int argc, char** argv) { + absl::InitializeSymbolizer(argv[0]); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/third_party/abseil_cpp/absl/debugging/internal/address_is_readable.cc b/third_party/abseil_cpp/absl/debugging/internal/address_is_readable.cc new file mode 100644 index 000000000000..329c285f3b58 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/address_is_readable.cc @@ -0,0 +1,139 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// base::AddressIsReadable() probes an address to see whether it is readable, +// without faulting. + +#include "absl/debugging/internal/address_is_readable.h" + +#if !defined(__linux__) || defined(__ANDROID__) + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// On platforms other than Linux, just return true. +bool AddressIsReadable(const void* /* addr */) { return true; } + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#else + +#include <fcntl.h> +#include <sys/syscall.h> +#include <unistd.h> + +#include <atomic> +#include <cerrno> +#include <cstdint> + +#include "absl/base/internal/errno_saver.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Pack a pid and two file descriptors into a 64-bit word, +// using 16, 24, and 24 bits for each respectively. +static uint64_t Pack(uint64_t pid, uint64_t read_fd, uint64_t write_fd) { + ABSL_RAW_CHECK((read_fd >> 24) == 0 && (write_fd >> 24) == 0, + "fd out of range"); + return (pid << 48) | ((read_fd & 0xffffff) << 24) | (write_fd & 0xffffff); +} + +// Unpack x into a pid and two file descriptors, where x was created with +// Pack(). +static void Unpack(uint64_t x, int *pid, int *read_fd, int *write_fd) { + *pid = x >> 48; + *read_fd = (x >> 24) & 0xffffff; + *write_fd = x & 0xffffff; +} + +// Return whether the byte at *addr is readable, without faulting. +// Save and restores errno. Returns true on systems where +// unimplemented. +// This is a namespace-scoped variable for correct zero-initialization. +static std::atomic<uint64_t> pid_and_fds; // initially 0, an invalid pid. + +bool AddressIsReadable(const void *addr) { + absl::base_internal::ErrnoSaver errno_saver; + // We test whether a byte is readable by using write(). Normally, this would + // be done via a cached file descriptor to /dev/null, but linux fails to + // check whether the byte is readable when the destination is /dev/null, so + // we use a cached pipe. We store the pid of the process that created the + // pipe to handle the case where a process forks, and the child closes all + // the file descriptors and then calls this routine. This is not perfect: + // the child could use the routine, then close all file descriptors and then + // use this routine again. But the likely use of this routine is when + // crashing, to test the validity of pages when dumping the stack. Beware + // that we may leak file descriptors, but we're unlikely to leak many. + int bytes_written; + int current_pid = getpid() & 0xffff; // we use only the low order 16 bits + do { // until we do not get EBADF trying to use file descriptors + int pid; + int read_fd; + int write_fd; + uint64_t local_pid_and_fds = pid_and_fds.load(std::memory_order_acquire); + Unpack(local_pid_and_fds, &pid, &read_fd, &write_fd); + while (current_pid != pid) { + int p[2]; + // new pipe + if (pipe(p) != 0) { + ABSL_RAW_LOG(FATAL, "Failed to create pipe, errno=%d", errno); + } + fcntl(p[0], F_SETFD, FD_CLOEXEC); + fcntl(p[1], F_SETFD, FD_CLOEXEC); + uint64_t new_pid_and_fds = Pack(current_pid, p[0], p[1]); + if (pid_and_fds.compare_exchange_strong( + local_pid_and_fds, new_pid_and_fds, std::memory_order_release, + std::memory_order_relaxed)) { + local_pid_and_fds = new_pid_and_fds; // fds exposed to other threads + } else { // fds not exposed to other threads; we can close them. + close(p[0]); + close(p[1]); + local_pid_and_fds = pid_and_fds.load(std::memory_order_acquire); + } + Unpack(local_pid_and_fds, &pid, &read_fd, &write_fd); + } + errno = 0; + // Use syscall(SYS_write, ...) instead of write() to prevent ASAN + // and other checkers from complaining about accesses to arbitrary + // memory. + do { + bytes_written = syscall(SYS_write, write_fd, addr, 1); + } while (bytes_written == -1 && errno == EINTR); + if (bytes_written == 1) { // remove the byte from the pipe + char c; + while (read(read_fd, &c, 1) == -1 && errno == EINTR) { + } + } + if (errno == EBADF) { // Descriptors invalid. + // If pid_and_fds contains the problematic file descriptors we just used, + // this call will forget them, and the loop will try again. + pid_and_fds.compare_exchange_strong(local_pid_and_fds, 0, + std::memory_order_release, + std::memory_order_relaxed); + } + } while (errno == EBADF); + return bytes_written == 1; +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif diff --git a/third_party/abseil_cpp/absl/debugging/internal/address_is_readable.h b/third_party/abseil_cpp/absl/debugging/internal/address_is_readable.h new file mode 100644 index 000000000000..4bbaf4d69bd8 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/address_is_readable.h @@ -0,0 +1,32 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_DEBUGGING_INTERNAL_ADDRESS_IS_READABLE_H_ +#define ABSL_DEBUGGING_INTERNAL_ADDRESS_IS_READABLE_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Return whether the byte at *addr is readable, without faulting. +// Save and restores errno. +bool AddressIsReadable(const void *addr); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_ADDRESS_IS_READABLE_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/demangle.cc b/third_party/abseil_cpp/absl/debugging/internal/demangle.cc new file mode 100644 index 000000000000..46cdb67b1fbf --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/demangle.cc @@ -0,0 +1,1945 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// For reference check out: +// https://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling +// +// Note that we only have partial C++11 support yet. + +#include "absl/debugging/internal/demangle.h" + +#include <cstdint> +#include <cstdio> +#include <limits> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +typedef struct { + const char *abbrev; + const char *real_name; + // Number of arguments in <expression> context, or 0 if disallowed. + int arity; +} AbbrevPair; + +// List of operators from Itanium C++ ABI. +static const AbbrevPair kOperatorList[] = { + // New has special syntax (not currently supported). + {"nw", "new", 0}, + {"na", "new[]", 0}, + + // Works except that the 'gs' prefix is not supported. + {"dl", "delete", 1}, + {"da", "delete[]", 1}, + + {"ps", "+", 1}, // "positive" + {"ng", "-", 1}, // "negative" + {"ad", "&", 1}, // "address-of" + {"de", "*", 1}, // "dereference" + {"co", "~", 1}, + + {"pl", "+", 2}, + {"mi", "-", 2}, + {"ml", "*", 2}, + {"dv", "/", 2}, + {"rm", "%", 2}, + {"an", "&", 2}, + {"or", "|", 2}, + {"eo", "^", 2}, + {"aS", "=", 2}, + {"pL", "+=", 2}, + {"mI", "-=", 2}, + {"mL", "*=", 2}, + {"dV", "/=", 2}, + {"rM", "%=", 2}, + {"aN", "&=", 2}, + {"oR", "|=", 2}, + {"eO", "^=", 2}, + {"ls", "<<", 2}, + {"rs", ">>", 2}, + {"lS", "<<=", 2}, + {"rS", ">>=", 2}, + {"eq", "==", 2}, + {"ne", "!=", 2}, + {"lt", "<", 2}, + {"gt", ">", 2}, + {"le", "<=", 2}, + {"ge", ">=", 2}, + {"nt", "!", 1}, + {"aa", "&&", 2}, + {"oo", "||", 2}, + {"pp", "++", 1}, + {"mm", "--", 1}, + {"cm", ",", 2}, + {"pm", "->*", 2}, + {"pt", "->", 0}, // Special syntax + {"cl", "()", 0}, // Special syntax + {"ix", "[]", 2}, + {"qu", "?", 3}, + {"st", "sizeof", 0}, // Special syntax + {"sz", "sizeof", 1}, // Not a real operator name, but used in expressions. + {nullptr, nullptr, 0}, +}; + +// List of builtin types from Itanium C++ ABI. +// +// Invariant: only one- or two-character type abbreviations here. +static const AbbrevPair kBuiltinTypeList[] = { + {"v", "void", 0}, + {"w", "wchar_t", 0}, + {"b", "bool", 0}, + {"c", "char", 0}, + {"a", "signed char", 0}, + {"h", "unsigned char", 0}, + {"s", "short", 0}, + {"t", "unsigned short", 0}, + {"i", "int", 0}, + {"j", "unsigned int", 0}, + {"l", "long", 0}, + {"m", "unsigned long", 0}, + {"x", "long long", 0}, + {"y", "unsigned long long", 0}, + {"n", "__int128", 0}, + {"o", "unsigned __int128", 0}, + {"f", "float", 0}, + {"d", "double", 0}, + {"e", "long double", 0}, + {"g", "__float128", 0}, + {"z", "ellipsis", 0}, + + {"De", "decimal128", 0}, // IEEE 754r decimal floating point (128 bits) + {"Dd", "decimal64", 0}, // IEEE 754r decimal floating point (64 bits) + {"Dc", "decltype(auto)", 0}, + {"Da", "auto", 0}, + {"Dn", "std::nullptr_t", 0}, // i.e., decltype(nullptr) + {"Df", "decimal32", 0}, // IEEE 754r decimal floating point (32 bits) + {"Di", "char32_t", 0}, + {"Du", "char8_t", 0}, + {"Ds", "char16_t", 0}, + {"Dh", "float16", 0}, // IEEE 754r half-precision float (16 bits) + {nullptr, nullptr, 0}, +}; + +// List of substitutions Itanium C++ ABI. +static const AbbrevPair kSubstitutionList[] = { + {"St", "", 0}, + {"Sa", "allocator", 0}, + {"Sb", "basic_string", 0}, + // std::basic_string<char, std::char_traits<char>,std::allocator<char> > + {"Ss", "string", 0}, + // std::basic_istream<char, std::char_traits<char> > + {"Si", "istream", 0}, + // std::basic_ostream<char, std::char_traits<char> > + {"So", "ostream", 0}, + // std::basic_iostream<char, std::char_traits<char> > + {"Sd", "iostream", 0}, + {nullptr, nullptr, 0}, +}; + +// State needed for demangling. This struct is copied in almost every stack +// frame, so every byte counts. +typedef struct { + int mangled_idx; // Cursor of mangled name. + int out_cur_idx; // Cursor of output string. + int prev_name_idx; // For constructors/destructors. + signed int prev_name_length : 16; // For constructors/destructors. + signed int nest_level : 15; // For nested names. + unsigned int append : 1; // Append flag. + // Note: for some reason MSVC can't pack "bool append : 1" into the same int + // with the above two fields, so we use an int instead. Amusingly it can pack + // "signed bool" as expected, but relying on that to continue to be a legal + // type seems ill-advised (as it's illegal in at least clang). +} ParseState; + +static_assert(sizeof(ParseState) == 4 * sizeof(int), + "unexpected size of ParseState"); + +// One-off state for demangling that's not subject to backtracking -- either +// constant data, data that's intentionally immune to backtracking (steps), or +// data that would never be changed by backtracking anyway (recursion_depth). +// +// Only one copy of this exists for each call to Demangle, so the size of this +// struct is nearly inconsequential. +typedef struct { + const char *mangled_begin; // Beginning of input string. + char *out; // Beginning of output string. + int out_end_idx; // One past last allowed output character. + int recursion_depth; // For stack exhaustion prevention. + int steps; // Cap how much work we'll do, regardless of depth. + ParseState parse_state; // Backtrackable state copied for most frames. +} State; + +namespace { +// Prevent deep recursion / stack exhaustion. +// Also prevent unbounded handling of complex inputs. +class ComplexityGuard { + public: + explicit ComplexityGuard(State *state) : state_(state) { + ++state->recursion_depth; + ++state->steps; + } + ~ComplexityGuard() { --state_->recursion_depth; } + + // 256 levels of recursion seems like a reasonable upper limit on depth. + // 128 is not enough to demagle synthetic tests from demangle_unittest.txt: + // "_ZaaZZZZ..." and "_ZaaZcvZcvZ..." + static constexpr int kRecursionDepthLimit = 256; + + // We're trying to pick a charitable upper-limit on how many parse steps are + // necessary to handle something that a human could actually make use of. + // This is mostly in place as a bound on how much work we'll do if we are + // asked to demangle an mangled name from an untrusted source, so it should be + // much larger than the largest expected symbol, but much smaller than the + // amount of work we can do in, e.g., a second. + // + // Some real-world symbols from an arbitrary binary started failing between + // 2^12 and 2^13, so we multiply the latter by an extra factor of 16 to set + // the limit. + // + // Spending one second on 2^17 parse steps would require each step to take + // 7.6us, or ~30000 clock cycles, so it's safe to say this can be done in + // under a second. + static constexpr int kParseStepsLimit = 1 << 17; + + bool IsTooComplex() const { + return state_->recursion_depth > kRecursionDepthLimit || + state_->steps > kParseStepsLimit; + } + + private: + State *state_; +}; +} // namespace + +// We don't use strlen() in libc since it's not guaranteed to be async +// signal safe. +static size_t StrLen(const char *str) { + size_t len = 0; + while (*str != '\0') { + ++str; + ++len; + } + return len; +} + +// Returns true if "str" has at least "n" characters remaining. +static bool AtLeastNumCharsRemaining(const char *str, int n) { + for (int i = 0; i < n; ++i) { + if (str[i] == '\0') { + return false; + } + } + return true; +} + +// Returns true if "str" has "prefix" as a prefix. +static bool StrPrefix(const char *str, const char *prefix) { + size_t i = 0; + while (str[i] != '\0' && prefix[i] != '\0' && str[i] == prefix[i]) { + ++i; + } + return prefix[i] == '\0'; // Consumed everything in "prefix". +} + +static void InitState(State *state, const char *mangled, char *out, + int out_size) { + state->mangled_begin = mangled; + state->out = out; + state->out_end_idx = out_size; + state->recursion_depth = 0; + state->steps = 0; + + state->parse_state.mangled_idx = 0; + state->parse_state.out_cur_idx = 0; + state->parse_state.prev_name_idx = 0; + state->parse_state.prev_name_length = -1; + state->parse_state.nest_level = -1; + state->parse_state.append = true; +} + +static inline const char *RemainingInput(State *state) { + return &state->mangled_begin[state->parse_state.mangled_idx]; +} + +// Returns true and advances "mangled_idx" if we find "one_char_token" +// at "mangled_idx" position. It is assumed that "one_char_token" does +// not contain '\0'. +static bool ParseOneCharToken(State *state, const char one_char_token) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (RemainingInput(state)[0] == one_char_token) { + ++state->parse_state.mangled_idx; + return true; + } + return false; +} + +// Returns true and advances "mangled_cur" if we find "two_char_token" +// at "mangled_cur" position. It is assumed that "two_char_token" does +// not contain '\0'. +static bool ParseTwoCharToken(State *state, const char *two_char_token) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (RemainingInput(state)[0] == two_char_token[0] && + RemainingInput(state)[1] == two_char_token[1]) { + state->parse_state.mangled_idx += 2; + return true; + } + return false; +} + +// Returns true and advances "mangled_cur" if we find any character in +// "char_class" at "mangled_cur" position. +static bool ParseCharClass(State *state, const char *char_class) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (RemainingInput(state)[0] == '\0') { + return false; + } + const char *p = char_class; + for (; *p != '\0'; ++p) { + if (RemainingInput(state)[0] == *p) { + ++state->parse_state.mangled_idx; + return true; + } + } + return false; +} + +static bool ParseDigit(State *state, int *digit) { + char c = RemainingInput(state)[0]; + if (ParseCharClass(state, "0123456789")) { + if (digit != nullptr) { + *digit = c - '0'; + } + return true; + } + return false; +} + +// This function is used for handling an optional non-terminal. +static bool Optional(bool /*status*/) { return true; } + +// This function is used for handling <non-terminal>+ syntax. +typedef bool (*ParseFunc)(State *); +static bool OneOrMore(ParseFunc parse_func, State *state) { + if (parse_func(state)) { + while (parse_func(state)) { + } + return true; + } + return false; +} + +// This function is used for handling <non-terminal>* syntax. The function +// always returns true and must be followed by a termination token or a +// terminating sequence not handled by parse_func (e.g. +// ParseOneCharToken(state, 'E')). +static bool ZeroOrMore(ParseFunc parse_func, State *state) { + while (parse_func(state)) { + } + return true; +} + +// Append "str" at "out_cur_idx". If there is an overflow, out_cur_idx is +// set to out_end_idx+1. The output string is ensured to +// always terminate with '\0' as long as there is no overflow. +static void Append(State *state, const char *const str, const int length) { + for (int i = 0; i < length; ++i) { + if (state->parse_state.out_cur_idx + 1 < + state->out_end_idx) { // +1 for '\0' + state->out[state->parse_state.out_cur_idx++] = str[i]; + } else { + // signal overflow + state->parse_state.out_cur_idx = state->out_end_idx + 1; + break; + } + } + if (state->parse_state.out_cur_idx < state->out_end_idx) { + state->out[state->parse_state.out_cur_idx] = + '\0'; // Terminate it with '\0' + } +} + +// We don't use equivalents in libc to avoid locale issues. +static bool IsLower(char c) { return c >= 'a' && c <= 'z'; } + +static bool IsAlpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool IsDigit(char c) { return c >= '0' && c <= '9'; } + +// Returns true if "str" is a function clone suffix. These suffixes are used +// by GCC 4.5.x and later versions (and our locally-modified version of GCC +// 4.4.x) to indicate functions which have been cloned during optimization. +// We treat any sequence (.<alpha>+.<digit>+)+ as a function clone suffix. +static bool IsFunctionCloneSuffix(const char *str) { + size_t i = 0; + while (str[i] != '\0') { + // Consume a single .<alpha>+.<digit>+ sequence. + if (str[i] != '.' || !IsAlpha(str[i + 1])) { + return false; + } + i += 2; + while (IsAlpha(str[i])) { + ++i; + } + if (str[i] != '.' || !IsDigit(str[i + 1])) { + return false; + } + i += 2; + while (IsDigit(str[i])) { + ++i; + } + } + return true; // Consumed everything in "str". +} + +static bool EndsWith(State *state, const char chr) { + return state->parse_state.out_cur_idx > 0 && + state->parse_state.out_cur_idx < state->out_end_idx && + chr == state->out[state->parse_state.out_cur_idx - 1]; +} + +// Append "str" with some tweaks, iff "append" state is true. +static void MaybeAppendWithLength(State *state, const char *const str, + const int length) { + if (state->parse_state.append && length > 0) { + // Append a space if the output buffer ends with '<' and "str" + // starts with '<' to avoid <<<. + if (str[0] == '<' && EndsWith(state, '<')) { + Append(state, " ", 1); + } + // Remember the last identifier name for ctors/dtors, + // but only if we haven't yet overflown the buffer. + if (state->parse_state.out_cur_idx < state->out_end_idx && + (IsAlpha(str[0]) || str[0] == '_')) { + state->parse_state.prev_name_idx = state->parse_state.out_cur_idx; + state->parse_state.prev_name_length = length; + } + Append(state, str, length); + } +} + +// Appends a positive decimal number to the output if appending is enabled. +static bool MaybeAppendDecimal(State *state, unsigned int val) { + // Max {32-64}-bit unsigned int is 20 digits. + constexpr size_t kMaxLength = 20; + char buf[kMaxLength]; + + // We can't use itoa or sprintf as neither is specified to be + // async-signal-safe. + if (state->parse_state.append) { + // We can't have a one-before-the-beginning pointer, so instead start with + // one-past-the-end and manipulate one character before the pointer. + char *p = &buf[kMaxLength]; + do { // val=0 is the only input that should write a leading zero digit. + *--p = (val % 10) + '0'; + val /= 10; + } while (p > buf && val != 0); + + // 'p' landed on the last character we set. How convenient. + Append(state, p, kMaxLength - (p - buf)); + } + + return true; +} + +// A convenient wrapper around MaybeAppendWithLength(). +// Returns true so that it can be placed in "if" conditions. +static bool MaybeAppend(State *state, const char *const str) { + if (state->parse_state.append) { + int length = StrLen(str); + MaybeAppendWithLength(state, str, length); + } + return true; +} + +// This function is used for handling nested names. +static bool EnterNestedName(State *state) { + state->parse_state.nest_level = 0; + return true; +} + +// This function is used for handling nested names. +static bool LeaveNestedName(State *state, int16_t prev_value) { + state->parse_state.nest_level = prev_value; + return true; +} + +// Disable the append mode not to print function parameters, etc. +static bool DisableAppend(State *state) { + state->parse_state.append = false; + return true; +} + +// Restore the append mode to the previous state. +static bool RestoreAppend(State *state, bool prev_value) { + state->parse_state.append = prev_value; + return true; +} + +// Increase the nest level for nested names. +static void MaybeIncreaseNestLevel(State *state) { + if (state->parse_state.nest_level > -1) { + ++state->parse_state.nest_level; + } +} + +// Appends :: for nested names if necessary. +static void MaybeAppendSeparator(State *state) { + if (state->parse_state.nest_level >= 1) { + MaybeAppend(state, "::"); + } +} + +// Cancel the last separator if necessary. +static void MaybeCancelLastSeparator(State *state) { + if (state->parse_state.nest_level >= 1 && state->parse_state.append && + state->parse_state.out_cur_idx >= 2) { + state->parse_state.out_cur_idx -= 2; + state->out[state->parse_state.out_cur_idx] = '\0'; + } +} + +// Returns true if the identifier of the given length pointed to by +// "mangled_cur" is anonymous namespace. +static bool IdentifierIsAnonymousNamespace(State *state, int length) { + // Returns true if "anon_prefix" is a proper prefix of "mangled_cur". + static const char anon_prefix[] = "_GLOBAL__N_"; + return (length > static_cast<int>(sizeof(anon_prefix) - 1) && + StrPrefix(RemainingInput(state), anon_prefix)); +} + +// Forward declarations of our parsing functions. +static bool ParseMangledName(State *state); +static bool ParseEncoding(State *state); +static bool ParseName(State *state); +static bool ParseUnscopedName(State *state); +static bool ParseNestedName(State *state); +static bool ParsePrefix(State *state); +static bool ParseUnqualifiedName(State *state); +static bool ParseSourceName(State *state); +static bool ParseLocalSourceName(State *state); +static bool ParseUnnamedTypeName(State *state); +static bool ParseNumber(State *state, int *number_out); +static bool ParseFloatNumber(State *state); +static bool ParseSeqId(State *state); +static bool ParseIdentifier(State *state, int length); +static bool ParseOperatorName(State *state, int *arity); +static bool ParseSpecialName(State *state); +static bool ParseCallOffset(State *state); +static bool ParseNVOffset(State *state); +static bool ParseVOffset(State *state); +static bool ParseCtorDtorName(State *state); +static bool ParseDecltype(State *state); +static bool ParseType(State *state); +static bool ParseCVQualifiers(State *state); +static bool ParseBuiltinType(State *state); +static bool ParseFunctionType(State *state); +static bool ParseBareFunctionType(State *state); +static bool ParseClassEnumType(State *state); +static bool ParseArrayType(State *state); +static bool ParsePointerToMemberType(State *state); +static bool ParseTemplateParam(State *state); +static bool ParseTemplateTemplateParam(State *state); +static bool ParseTemplateArgs(State *state); +static bool ParseTemplateArg(State *state); +static bool ParseBaseUnresolvedName(State *state); +static bool ParseUnresolvedName(State *state); +static bool ParseExpression(State *state); +static bool ParseExprPrimary(State *state); +static bool ParseExprCastValue(State *state); +static bool ParseLocalName(State *state); +static bool ParseLocalNameSuffix(State *state); +static bool ParseDiscriminator(State *state); +static bool ParseSubstitution(State *state, bool accept_std); + +// Implementation note: the following code is a straightforward +// translation of the Itanium C++ ABI defined in BNF with a couple of +// exceptions. +// +// - Support GNU extensions not defined in the Itanium C++ ABI +// - <prefix> and <template-prefix> are combined to avoid infinite loop +// - Reorder patterns to shorten the code +// - Reorder patterns to give greedier functions precedence +// We'll mark "Less greedy than" for these cases in the code +// +// Each parsing function changes the parse state and returns true on +// success, or returns false and doesn't change the parse state (note: +// the parse-steps counter increases regardless of success or failure). +// To ensure that the parse state isn't changed in the latter case, we +// save the original state before we call multiple parsing functions +// consecutively with &&, and restore it if unsuccessful. See +// ParseEncoding() as an example of this convention. We follow the +// convention throughout the code. +// +// Originally we tried to do demangling without following the full ABI +// syntax but it turned out we needed to follow the full syntax to +// parse complicated cases like nested template arguments. Note that +// implementing a full-fledged demangler isn't trivial (libiberty's +// cp-demangle.c has +4300 lines). +// +// Note that (foo) in <(foo) ...> is a modifier to be ignored. +// +// Reference: +// - Itanium C++ ABI +// <https://mentorembedded.github.io/cxx-abi/abi.html#mangling> + +// <mangled-name> ::= _Z <encoding> +static bool ParseMangledName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + return ParseTwoCharToken(state, "_Z") && ParseEncoding(state); +} + +// <encoding> ::= <(function) name> <bare-function-type> +// ::= <(data) name> +// ::= <special-name> +static bool ParseEncoding(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + // Implementing the first two productions together as <name> + // [<bare-function-type>] avoids exponential blowup of backtracking. + // + // Since Optional(...) can't fail, there's no need to copy the state for + // backtracking. + if (ParseName(state) && Optional(ParseBareFunctionType(state))) { + return true; + } + + if (ParseSpecialName(state)) { + return true; + } + return false; +} + +// <name> ::= <nested-name> +// ::= <unscoped-template-name> <template-args> +// ::= <unscoped-name> +// ::= <local-name> +static bool ParseName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (ParseNestedName(state) || ParseLocalName(state)) { + return true; + } + + // We reorganize the productions to avoid re-parsing unscoped names. + // - Inline <unscoped-template-name> productions: + // <name> ::= <substitution> <template-args> + // ::= <unscoped-name> <template-args> + // ::= <unscoped-name> + // - Merge the two productions that start with unscoped-name: + // <name> ::= <unscoped-name> [<template-args>] + + ParseState copy = state->parse_state; + // "std<...>" isn't a valid name. + if (ParseSubstitution(state, /*accept_std=*/false) && + ParseTemplateArgs(state)) { + return true; + } + state->parse_state = copy; + + // Note there's no need to restore state after this since only the first + // subparser can fail. + return ParseUnscopedName(state) && Optional(ParseTemplateArgs(state)); +} + +// <unscoped-name> ::= <unqualified-name> +// ::= St <unqualified-name> +static bool ParseUnscopedName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (ParseUnqualifiedName(state)) { + return true; + } + + ParseState copy = state->parse_state; + if (ParseTwoCharToken(state, "St") && MaybeAppend(state, "std::") && + ParseUnqualifiedName(state)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <ref-qualifer> ::= R // lvalue method reference qualifier +// ::= O // rvalue method reference qualifier +static inline bool ParseRefQualifier(State *state) { + return ParseCharClass(state, "OR"); +} + +// <nested-name> ::= N [<CV-qualifiers>] [<ref-qualifier>] <prefix> +// <unqualified-name> E +// ::= N [<CV-qualifiers>] [<ref-qualifier>] <template-prefix> +// <template-args> E +static bool ParseNestedName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'N') && EnterNestedName(state) && + Optional(ParseCVQualifiers(state)) && + Optional(ParseRefQualifier(state)) && ParsePrefix(state) && + LeaveNestedName(state, copy.nest_level) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + return false; +} + +// This part is tricky. If we literally translate them to code, we'll +// end up infinite loop. Hence we merge them to avoid the case. +// +// <prefix> ::= <prefix> <unqualified-name> +// ::= <template-prefix> <template-args> +// ::= <template-param> +// ::= <substitution> +// ::= # empty +// <template-prefix> ::= <prefix> <(template) unqualified-name> +// ::= <template-param> +// ::= <substitution> +static bool ParsePrefix(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + bool has_something = false; + while (true) { + MaybeAppendSeparator(state); + if (ParseTemplateParam(state) || + ParseSubstitution(state, /*accept_std=*/true) || + ParseUnscopedName(state) || + (ParseOneCharToken(state, 'M') && ParseUnnamedTypeName(state))) { + has_something = true; + MaybeIncreaseNestLevel(state); + continue; + } + MaybeCancelLastSeparator(state); + if (has_something && ParseTemplateArgs(state)) { + return ParsePrefix(state); + } else { + break; + } + } + return true; +} + +// <unqualified-name> ::= <operator-name> +// ::= <ctor-dtor-name> +// ::= <source-name> +// ::= <local-source-name> // GCC extension; see below. +// ::= <unnamed-type-name> +static bool ParseUnqualifiedName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + return (ParseOperatorName(state, nullptr) || ParseCtorDtorName(state) || + ParseSourceName(state) || ParseLocalSourceName(state) || + ParseUnnamedTypeName(state)); +} + +// <source-name> ::= <positive length number> <identifier> +static bool ParseSourceName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + int length = -1; + if (ParseNumber(state, &length) && ParseIdentifier(state, length)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <local-source-name> ::= L <source-name> [<discriminator>] +// +// References: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=31775 +// https://gcc.gnu.org/viewcvs?view=rev&revision=124467 +static bool ParseLocalSourceName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'L') && ParseSourceName(state) && + Optional(ParseDiscriminator(state))) { + return true; + } + state->parse_state = copy; + return false; +} + +// <unnamed-type-name> ::= Ut [<(nonnegative) number>] _ +// ::= <closure-type-name> +// <closure-type-name> ::= Ul <lambda-sig> E [<(nonnegative) number>] _ +// <lambda-sig> ::= <(parameter) type>+ +static bool ParseUnnamedTypeName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + // Type's 1-based index n is encoded as { "", n == 1; itoa(n-2), otherwise }. + // Optionally parse the encoded value into 'which' and add 2 to get the index. + int which = -1; + + // Unnamed type local to function or class. + if (ParseTwoCharToken(state, "Ut") && Optional(ParseNumber(state, &which)) && + which <= std::numeric_limits<int>::max() - 2 && // Don't overflow. + ParseOneCharToken(state, '_')) { + MaybeAppend(state, "{unnamed type#"); + MaybeAppendDecimal(state, 2 + which); + MaybeAppend(state, "}"); + return true; + } + state->parse_state = copy; + + // Closure type. + which = -1; + if (ParseTwoCharToken(state, "Ul") && DisableAppend(state) && + OneOrMore(ParseType, state) && RestoreAppend(state, copy.append) && + ParseOneCharToken(state, 'E') && Optional(ParseNumber(state, &which)) && + which <= std::numeric_limits<int>::max() - 2 && // Don't overflow. + ParseOneCharToken(state, '_')) { + MaybeAppend(state, "{lambda()#"); + MaybeAppendDecimal(state, 2 + which); + MaybeAppend(state, "}"); + return true; + } + state->parse_state = copy; + + return false; +} + +// <number> ::= [n] <non-negative decimal integer> +// If "number_out" is non-null, then *number_out is set to the value of the +// parsed number on success. +static bool ParseNumber(State *state, int *number_out) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + bool negative = false; + if (ParseOneCharToken(state, 'n')) { + negative = true; + } + const char *p = RemainingInput(state); + uint64_t number = 0; + for (; *p != '\0'; ++p) { + if (IsDigit(*p)) { + number = number * 10 + (*p - '0'); + } else { + break; + } + } + // Apply the sign with uint64_t arithmetic so overflows aren't UB. Gives + // "incorrect" results for out-of-range inputs, but negative values only + // appear for literals, which aren't printed. + if (negative) { + number = ~number + 1; + } + if (p != RemainingInput(state)) { // Conversion succeeded. + state->parse_state.mangled_idx += p - RemainingInput(state); + if (number_out != nullptr) { + // Note: possibly truncate "number". + *number_out = number; + } + return true; + } + return false; +} + +// Floating-point literals are encoded using a fixed-length lowercase +// hexadecimal string. +static bool ParseFloatNumber(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + const char *p = RemainingInput(state); + for (; *p != '\0'; ++p) { + if (!IsDigit(*p) && !(*p >= 'a' && *p <= 'f')) { + break; + } + } + if (p != RemainingInput(state)) { // Conversion succeeded. + state->parse_state.mangled_idx += p - RemainingInput(state); + return true; + } + return false; +} + +// The <seq-id> is a sequence number in base 36, +// using digits and upper case letters +static bool ParseSeqId(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + const char *p = RemainingInput(state); + for (; *p != '\0'; ++p) { + if (!IsDigit(*p) && !(*p >= 'A' && *p <= 'Z')) { + break; + } + } + if (p != RemainingInput(state)) { // Conversion succeeded. + state->parse_state.mangled_idx += p - RemainingInput(state); + return true; + } + return false; +} + +// <identifier> ::= <unqualified source code identifier> (of given length) +static bool ParseIdentifier(State *state, int length) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (length < 0 || !AtLeastNumCharsRemaining(RemainingInput(state), length)) { + return false; + } + if (IdentifierIsAnonymousNamespace(state, length)) { + MaybeAppend(state, "(anonymous namespace)"); + } else { + MaybeAppendWithLength(state, RemainingInput(state), length); + } + state->parse_state.mangled_idx += length; + return true; +} + +// <operator-name> ::= nw, and other two letters cases +// ::= cv <type> # (cast) +// ::= v <digit> <source-name> # vendor extended operator +static bool ParseOperatorName(State *state, int *arity) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (!AtLeastNumCharsRemaining(RemainingInput(state), 2)) { + return false; + } + // First check with "cv" (cast) case. + ParseState copy = state->parse_state; + if (ParseTwoCharToken(state, "cv") && MaybeAppend(state, "operator ") && + EnterNestedName(state) && ParseType(state) && + LeaveNestedName(state, copy.nest_level)) { + if (arity != nullptr) { + *arity = 1; + } + return true; + } + state->parse_state = copy; + + // Then vendor extended operators. + if (ParseOneCharToken(state, 'v') && ParseDigit(state, arity) && + ParseSourceName(state)) { + return true; + } + state->parse_state = copy; + + // Other operator names should start with a lower alphabet followed + // by a lower/upper alphabet. + if (!(IsLower(RemainingInput(state)[0]) && + IsAlpha(RemainingInput(state)[1]))) { + return false; + } + // We may want to perform a binary search if we really need speed. + const AbbrevPair *p; + for (p = kOperatorList; p->abbrev != nullptr; ++p) { + if (RemainingInput(state)[0] == p->abbrev[0] && + RemainingInput(state)[1] == p->abbrev[1]) { + if (arity != nullptr) { + *arity = p->arity; + } + MaybeAppend(state, "operator"); + if (IsLower(*p->real_name)) { // new, delete, etc. + MaybeAppend(state, " "); + } + MaybeAppend(state, p->real_name); + state->parse_state.mangled_idx += 2; + return true; + } + } + return false; +} + +// <special-name> ::= TV <type> +// ::= TT <type> +// ::= TI <type> +// ::= TS <type> +// ::= TH <type> # thread-local +// ::= Tc <call-offset> <call-offset> <(base) encoding> +// ::= GV <(object) name> +// ::= T <call-offset> <(base) encoding> +// G++ extensions: +// ::= TC <type> <(offset) number> _ <(base) type> +// ::= TF <type> +// ::= TJ <type> +// ::= GR <name> +// ::= GA <encoding> +// ::= Th <call-offset> <(base) encoding> +// ::= Tv <call-offset> <(base) encoding> +// +// Note: we don't care much about them since they don't appear in +// stack traces. The are special data. +static bool ParseSpecialName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'T') && ParseCharClass(state, "VTISH") && + ParseType(state)) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "Tc") && ParseCallOffset(state) && + ParseCallOffset(state) && ParseEncoding(state)) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "GV") && ParseName(state)) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'T') && ParseCallOffset(state) && + ParseEncoding(state)) { + return true; + } + state->parse_state = copy; + + // G++ extensions + if (ParseTwoCharToken(state, "TC") && ParseType(state) && + ParseNumber(state, nullptr) && ParseOneCharToken(state, '_') && + DisableAppend(state) && ParseType(state)) { + RestoreAppend(state, copy.append); + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'T') && ParseCharClass(state, "FJ") && + ParseType(state)) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "GR") && ParseName(state)) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "GA") && ParseEncoding(state)) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'T') && ParseCharClass(state, "hv") && + ParseCallOffset(state) && ParseEncoding(state)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <call-offset> ::= h <nv-offset> _ +// ::= v <v-offset> _ +static bool ParseCallOffset(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'h') && ParseNVOffset(state) && + ParseOneCharToken(state, '_')) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'v') && ParseVOffset(state) && + ParseOneCharToken(state, '_')) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <nv-offset> ::= <(offset) number> +static bool ParseNVOffset(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + return ParseNumber(state, nullptr); +} + +// <v-offset> ::= <(offset) number> _ <(virtual offset) number> +static bool ParseVOffset(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseNumber(state, nullptr) && ParseOneCharToken(state, '_') && + ParseNumber(state, nullptr)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <ctor-dtor-name> ::= C1 | C2 | C3 | CI1 <base-class-type> | CI2 +// <base-class-type> +// ::= D0 | D1 | D2 +// # GCC extensions: "unified" constructor/destructor. See +// # +// https://github.com/gcc-mirror/gcc/blob/7ad17b583c3643bd4557f29b8391ca7ef08391f5/gcc/cp/mangle.c#L1847 +// ::= C4 | D4 +static bool ParseCtorDtorName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'C')) { + if (ParseCharClass(state, "1234")) { + const char *const prev_name = + state->out + state->parse_state.prev_name_idx; + MaybeAppendWithLength(state, prev_name, + state->parse_state.prev_name_length); + return true; + } else if (ParseOneCharToken(state, 'I') && ParseCharClass(state, "12") && + ParseClassEnumType(state)) { + return true; + } + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'D') && ParseCharClass(state, "0124")) { + const char *const prev_name = state->out + state->parse_state.prev_name_idx; + MaybeAppend(state, "~"); + MaybeAppendWithLength(state, prev_name, + state->parse_state.prev_name_length); + return true; + } + state->parse_state = copy; + return false; +} + +// <decltype> ::= Dt <expression> E # decltype of an id-expression or class +// # member access (C++0x) +// ::= DT <expression> E # decltype of an expression (C++0x) +static bool ParseDecltype(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'D') && ParseCharClass(state, "tT") && + ParseExpression(state) && ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <type> ::= <CV-qualifiers> <type> +// ::= P <type> # pointer-to +// ::= R <type> # reference-to +// ::= O <type> # rvalue reference-to (C++0x) +// ::= C <type> # complex pair (C 2000) +// ::= G <type> # imaginary (C 2000) +// ::= U <source-name> <type> # vendor extended type qualifier +// ::= <builtin-type> +// ::= <function-type> +// ::= <class-enum-type> # note: just an alias for <name> +// ::= <array-type> +// ::= <pointer-to-member-type> +// ::= <template-template-param> <template-args> +// ::= <template-param> +// ::= <decltype> +// ::= <substitution> +// ::= Dp <type> # pack expansion of (C++0x) +// ::= Dv <num-elems> _ # GNU vector extension +// +static bool ParseType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + + // We should check CV-qualifers, and PRGC things first. + // + // CV-qualifiers overlap with some operator names, but an operator name is not + // valid as a type. To avoid an ambiguity that can lead to exponential time + // complexity, refuse to backtrack the CV-qualifiers. + // + // _Z4aoeuIrMvvE + // => _Z 4aoeuI rM v v E + // aoeu<operator%=, void, void> + // => _Z 4aoeuI r Mv v E + // aoeu<void void::* restrict> + // + // By consuming the CV-qualifiers first, the former parse is disabled. + if (ParseCVQualifiers(state)) { + const bool result = ParseType(state); + if (!result) state->parse_state = copy; + return result; + } + state->parse_state = copy; + + // Similarly, these tag characters can overlap with other <name>s resulting in + // two different parse prefixes that land on <template-args> in the same + // place, such as "C3r1xI...". So, disable the "ctor-name = C3" parse by + // refusing to backtrack the tag characters. + if (ParseCharClass(state, "OPRCG")) { + const bool result = ParseType(state); + if (!result) state->parse_state = copy; + return result; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "Dp") && ParseType(state)) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'U') && ParseSourceName(state) && + ParseType(state)) { + return true; + } + state->parse_state = copy; + + if (ParseBuiltinType(state) || ParseFunctionType(state) || + ParseClassEnumType(state) || ParseArrayType(state) || + ParsePointerToMemberType(state) || ParseDecltype(state) || + // "std" on its own isn't a type. + ParseSubstitution(state, /*accept_std=*/false)) { + return true; + } + + if (ParseTemplateTemplateParam(state) && ParseTemplateArgs(state)) { + return true; + } + state->parse_state = copy; + + // Less greedy than <template-template-param> <template-args>. + if (ParseTemplateParam(state)) { + return true; + } + + if (ParseTwoCharToken(state, "Dv") && ParseNumber(state, nullptr) && + ParseOneCharToken(state, '_')) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <CV-qualifiers> ::= [r] [V] [K] +// We don't allow empty <CV-qualifiers> to avoid infinite loop in +// ParseType(). +static bool ParseCVQualifiers(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + int num_cv_qualifiers = 0; + num_cv_qualifiers += ParseOneCharToken(state, 'r'); + num_cv_qualifiers += ParseOneCharToken(state, 'V'); + num_cv_qualifiers += ParseOneCharToken(state, 'K'); + return num_cv_qualifiers > 0; +} + +// <builtin-type> ::= v, etc. # single-character builtin types +// ::= u <source-name> +// ::= Dd, etc. # two-character builtin types +// +// Not supported: +// ::= DF <number> _ # _FloatN (N bits) +// +static bool ParseBuiltinType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + const AbbrevPair *p; + for (p = kBuiltinTypeList; p->abbrev != nullptr; ++p) { + // Guaranteed only 1- or 2-character strings in kBuiltinTypeList. + if (p->abbrev[1] == '\0') { + if (ParseOneCharToken(state, p->abbrev[0])) { + MaybeAppend(state, p->real_name); + return true; + } + } else if (p->abbrev[2] == '\0' && ParseTwoCharToken(state, p->abbrev)) { + MaybeAppend(state, p->real_name); + return true; + } + } + + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'u') && ParseSourceName(state)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <exception-spec> ::= Do # non-throwing +// exception-specification (e.g., +// noexcept, throw()) +// ::= DO <expression> E # computed (instantiation-dependent) +// noexcept +// ::= Dw <type>+ E # dynamic exception specification +// with instantiation-dependent types +static bool ParseExceptionSpec(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + + if (ParseTwoCharToken(state, "Do")) return true; + + ParseState copy = state->parse_state; + if (ParseTwoCharToken(state, "DO") && ParseExpression(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + if (ParseTwoCharToken(state, "Dw") && OneOrMore(ParseType, state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <function-type> ::= [exception-spec] F [Y] <bare-function-type> [O] E +static bool ParseFunctionType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (Optional(ParseExceptionSpec(state)) && ParseOneCharToken(state, 'F') && + Optional(ParseOneCharToken(state, 'Y')) && ParseBareFunctionType(state) && + Optional(ParseOneCharToken(state, 'O')) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + return false; +} + +// <bare-function-type> ::= <(signature) type>+ +static bool ParseBareFunctionType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + DisableAppend(state); + if (OneOrMore(ParseType, state)) { + RestoreAppend(state, copy.append); + MaybeAppend(state, "()"); + return true; + } + state->parse_state = copy; + return false; +} + +// <class-enum-type> ::= <name> +static bool ParseClassEnumType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + return ParseName(state); +} + +// <array-type> ::= A <(positive dimension) number> _ <(element) type> +// ::= A [<(dimension) expression>] _ <(element) type> +static bool ParseArrayType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'A') && ParseNumber(state, nullptr) && + ParseOneCharToken(state, '_') && ParseType(state)) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'A') && Optional(ParseExpression(state)) && + ParseOneCharToken(state, '_') && ParseType(state)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <pointer-to-member-type> ::= M <(class) type> <(member) type> +static bool ParsePointerToMemberType(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'M') && ParseType(state) && ParseType(state)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <template-param> ::= T_ +// ::= T <parameter-2 non-negative number> _ +static bool ParseTemplateParam(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (ParseTwoCharToken(state, "T_")) { + MaybeAppend(state, "?"); // We don't support template substitutions. + return true; + } + + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'T') && ParseNumber(state, nullptr) && + ParseOneCharToken(state, '_')) { + MaybeAppend(state, "?"); // We don't support template substitutions. + return true; + } + state->parse_state = copy; + return false; +} + +// <template-template-param> ::= <template-param> +// ::= <substitution> +static bool ParseTemplateTemplateParam(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + return (ParseTemplateParam(state) || + // "std" on its own isn't a template. + ParseSubstitution(state, /*accept_std=*/false)); +} + +// <template-args> ::= I <template-arg>+ E +static bool ParseTemplateArgs(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + DisableAppend(state); + if (ParseOneCharToken(state, 'I') && OneOrMore(ParseTemplateArg, state) && + ParseOneCharToken(state, 'E')) { + RestoreAppend(state, copy.append); + MaybeAppend(state, "<>"); + return true; + } + state->parse_state = copy; + return false; +} + +// <template-arg> ::= <type> +// ::= <expr-primary> +// ::= J <template-arg>* E # argument pack +// ::= X <expression> E +static bool ParseTemplateArg(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'J') && ZeroOrMore(ParseTemplateArg, state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + // There can be significant overlap between the following leading to + // exponential backtracking: + // + // <expr-primary> ::= L <type> <expr-cast-value> E + // e.g. L 2xxIvE 1 E + // <type> ==> <local-source-name> <template-args> + // e.g. L 2xx IvE + // + // This means parsing an entire <type> twice, and <type> can contain + // <template-arg>, so this can generate exponential backtracking. There is + // only overlap when the remaining input starts with "L <source-name>", so + // parse all cases that can start this way jointly to share the common prefix. + // + // We have: + // + // <template-arg> ::= <type> + // ::= <expr-primary> + // + // First, drop all the productions of <type> that must start with something + // other than 'L'. All that's left is <class-enum-type>; inline it. + // + // <type> ::= <nested-name> # starts with 'N' + // ::= <unscoped-name> + // ::= <unscoped-template-name> <template-args> + // ::= <local-name> # starts with 'Z' + // + // Drop and inline again: + // + // <type> ::= <unscoped-name> + // ::= <unscoped-name> <template-args> + // ::= <substitution> <template-args> # starts with 'S' + // + // Merge the first two, inline <unscoped-name>, drop last: + // + // <type> ::= <unqualified-name> [<template-args>] + // ::= St <unqualified-name> [<template-args>] # starts with 'S' + // + // Drop and inline: + // + // <type> ::= <operator-name> [<template-args>] # starts with lowercase + // ::= <ctor-dtor-name> [<template-args>] # starts with 'C' or 'D' + // ::= <source-name> [<template-args>] # starts with digit + // ::= <local-source-name> [<template-args>] + // ::= <unnamed-type-name> [<template-args>] # starts with 'U' + // + // One more time: + // + // <type> ::= L <source-name> [<template-args>] + // + // Likewise with <expr-primary>: + // + // <expr-primary> ::= L <type> <expr-cast-value> E + // ::= LZ <encoding> E # cannot overlap; drop + // ::= L <mangled_name> E # cannot overlap; drop + // + // By similar reasoning as shown above, the only <type>s starting with + // <source-name> are "<source-name> [<template-args>]". Inline this. + // + // <expr-primary> ::= L <source-name> [<template-args>] <expr-cast-value> E + // + // Now inline both of these into <template-arg>: + // + // <template-arg> ::= L <source-name> [<template-args>] + // ::= L <source-name> [<template-args>] <expr-cast-value> E + // + // Merge them and we're done: + // <template-arg> + // ::= L <source-name> [<template-args>] [<expr-cast-value> E] + if (ParseLocalSourceName(state) && Optional(ParseTemplateArgs(state))) { + copy = state->parse_state; + if (ParseExprCastValue(state) && ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + return true; + } + + // Now that the overlapping cases can't reach this code, we can safely call + // both of these. + if (ParseType(state) || ParseExprPrimary(state)) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'X') && ParseExpression(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + return false; +} + +// <unresolved-type> ::= <template-param> [<template-args>] +// ::= <decltype> +// ::= <substitution> +static inline bool ParseUnresolvedType(State *state) { + // No ComplexityGuard because we don't copy the state in this stack frame. + return (ParseTemplateParam(state) && Optional(ParseTemplateArgs(state))) || + ParseDecltype(state) || ParseSubstitution(state, /*accept_std=*/false); +} + +// <simple-id> ::= <source-name> [<template-args>] +static inline bool ParseSimpleId(State *state) { + // No ComplexityGuard because we don't copy the state in this stack frame. + + // Note: <simple-id> cannot be followed by a parameter pack; see comment in + // ParseUnresolvedType. + return ParseSourceName(state) && Optional(ParseTemplateArgs(state)); +} + +// <base-unresolved-name> ::= <source-name> [<template-args>] +// ::= on <operator-name> [<template-args>] +// ::= dn <destructor-name> +static bool ParseBaseUnresolvedName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + + if (ParseSimpleId(state)) { + return true; + } + + ParseState copy = state->parse_state; + if (ParseTwoCharToken(state, "on") && ParseOperatorName(state, nullptr) && + Optional(ParseTemplateArgs(state))) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "dn") && + (ParseUnresolvedType(state) || ParseSimpleId(state))) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <unresolved-name> ::= [gs] <base-unresolved-name> +// ::= sr <unresolved-type> <base-unresolved-name> +// ::= srN <unresolved-type> <unresolved-qualifier-level>+ E +// <base-unresolved-name> +// ::= [gs] sr <unresolved-qualifier-level>+ E +// <base-unresolved-name> +static bool ParseUnresolvedName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + + ParseState copy = state->parse_state; + if (Optional(ParseTwoCharToken(state, "gs")) && + ParseBaseUnresolvedName(state)) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "sr") && ParseUnresolvedType(state) && + ParseBaseUnresolvedName(state)) { + return true; + } + state->parse_state = copy; + + if (ParseTwoCharToken(state, "sr") && ParseOneCharToken(state, 'N') && + ParseUnresolvedType(state) && + OneOrMore(/* <unresolved-qualifier-level> ::= */ ParseSimpleId, state) && + ParseOneCharToken(state, 'E') && ParseBaseUnresolvedName(state)) { + return true; + } + state->parse_state = copy; + + if (Optional(ParseTwoCharToken(state, "gs")) && + ParseTwoCharToken(state, "sr") && + OneOrMore(/* <unresolved-qualifier-level> ::= */ ParseSimpleId, state) && + ParseOneCharToken(state, 'E') && ParseBaseUnresolvedName(state)) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <expression> ::= <1-ary operator-name> <expression> +// ::= <2-ary operator-name> <expression> <expression> +// ::= <3-ary operator-name> <expression> <expression> <expression> +// ::= cl <expression>+ E +// ::= cv <type> <expression> # type (expression) +// ::= cv <type> _ <expression>* E # type (expr-list) +// ::= st <type> +// ::= <template-param> +// ::= <function-param> +// ::= <expr-primary> +// ::= dt <expression> <unresolved-name> # expr.name +// ::= pt <expression> <unresolved-name> # expr->name +// ::= sp <expression> # argument pack expansion +// ::= sr <type> <unqualified-name> <template-args> +// ::= sr <type> <unqualified-name> +// <function-param> ::= fp <(top-level) CV-qualifiers> _ +// ::= fp <(top-level) CV-qualifiers> <number> _ +// ::= fL <number> p <(top-level) CV-qualifiers> _ +// ::= fL <number> p <(top-level) CV-qualifiers> <number> _ +static bool ParseExpression(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (ParseTemplateParam(state) || ParseExprPrimary(state)) { + return true; + } + + // Object/function call expression. + ParseState copy = state->parse_state; + if (ParseTwoCharToken(state, "cl") && OneOrMore(ParseExpression, state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + // Function-param expression (level 0). + if (ParseTwoCharToken(state, "fp") && Optional(ParseCVQualifiers(state)) && + Optional(ParseNumber(state, nullptr)) && ParseOneCharToken(state, '_')) { + return true; + } + state->parse_state = copy; + + // Function-param expression (level 1+). + if (ParseTwoCharToken(state, "fL") && Optional(ParseNumber(state, nullptr)) && + ParseOneCharToken(state, 'p') && Optional(ParseCVQualifiers(state)) && + Optional(ParseNumber(state, nullptr)) && ParseOneCharToken(state, '_')) { + return true; + } + state->parse_state = copy; + + // Parse the conversion expressions jointly to avoid re-parsing the <type> in + // their common prefix. Parsed as: + // <expression> ::= cv <type> <conversion-args> + // <conversion-args> ::= _ <expression>* E + // ::= <expression> + // + // Also don't try ParseOperatorName after seeing "cv", since ParseOperatorName + // also needs to accept "cv <type>" in other contexts. + if (ParseTwoCharToken(state, "cv")) { + if (ParseType(state)) { + ParseState copy2 = state->parse_state; + if (ParseOneCharToken(state, '_') && ZeroOrMore(ParseExpression, state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy2; + if (ParseExpression(state)) { + return true; + } + } + } else { + // Parse unary, binary, and ternary operator expressions jointly, taking + // care not to re-parse subexpressions repeatedly. Parse like: + // <expression> ::= <operator-name> <expression> + // [<one-to-two-expressions>] + // <one-to-two-expressions> ::= <expression> [<expression>] + int arity = -1; + if (ParseOperatorName(state, &arity) && + arity > 0 && // 0 arity => disabled. + (arity < 3 || ParseExpression(state)) && + (arity < 2 || ParseExpression(state)) && + (arity < 1 || ParseExpression(state))) { + return true; + } + } + state->parse_state = copy; + + // sizeof type + if (ParseTwoCharToken(state, "st") && ParseType(state)) { + return true; + } + state->parse_state = copy; + + // Object and pointer member access expressions. + if ((ParseTwoCharToken(state, "dt") || ParseTwoCharToken(state, "pt")) && + ParseExpression(state) && ParseType(state)) { + return true; + } + state->parse_state = copy; + + // Pointer-to-member access expressions. This parses the same as a binary + // operator, but it's implemented separately because "ds" shouldn't be + // accepted in other contexts that parse an operator name. + if (ParseTwoCharToken(state, "ds") && ParseExpression(state) && + ParseExpression(state)) { + return true; + } + state->parse_state = copy; + + // Parameter pack expansion + if (ParseTwoCharToken(state, "sp") && ParseExpression(state)) { + return true; + } + state->parse_state = copy; + + return ParseUnresolvedName(state); +} + +// <expr-primary> ::= L <type> <(value) number> E +// ::= L <type> <(value) float> E +// ::= L <mangled-name> E +// // A bug in g++'s C++ ABI version 2 (-fabi-version=2). +// ::= LZ <encoding> E +// +// Warning, subtle: the "bug" LZ production above is ambiguous with the first +// production where <type> starts with <local-name>, which can lead to +// exponential backtracking in two scenarios: +// +// - When whatever follows the E in the <local-name> in the first production is +// not a name, we backtrack the whole <encoding> and re-parse the whole thing. +// +// - When whatever follows the <local-name> in the first production is not a +// number and this <expr-primary> may be followed by a name, we backtrack the +// <name> and re-parse it. +// +// Moreover this ambiguity isn't always resolved -- for example, the following +// has two different parses: +// +// _ZaaILZ4aoeuE1x1EvE +// => operator&&<aoeu, x, E, void> +// => operator&&<(aoeu::x)(1), void> +// +// To resolve this, we just do what GCC's demangler does, and refuse to parse +// casts to <local-name> types. +static bool ParseExprPrimary(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + + // The "LZ" special case: if we see LZ, we commit to accept "LZ <encoding> E" + // or fail, no backtracking. + if (ParseTwoCharToken(state, "LZ")) { + if (ParseEncoding(state) && ParseOneCharToken(state, 'E')) { + return true; + } + + state->parse_state = copy; + return false; + } + + // The merged cast production. + if (ParseOneCharToken(state, 'L') && ParseType(state) && + ParseExprCastValue(state)) { + return true; + } + state->parse_state = copy; + + if (ParseOneCharToken(state, 'L') && ParseMangledName(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <number> or <float>, followed by 'E', as described above ParseExprPrimary. +static bool ParseExprCastValue(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + // We have to be able to backtrack after accepting a number because we could + // have e.g. "7fffE", which will accept "7" as a number but then fail to find + // the 'E'. + ParseState copy = state->parse_state; + if (ParseNumber(state, nullptr) && ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + if (ParseFloatNumber(state) && ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + + return false; +} + +// <local-name> ::= Z <(function) encoding> E <(entity) name> [<discriminator>] +// ::= Z <(function) encoding> E s [<discriminator>] +// +// Parsing a common prefix of these two productions together avoids an +// exponential blowup of backtracking. Parse like: +// <local-name> := Z <encoding> E <local-name-suffix> +// <local-name-suffix> ::= s [<discriminator>] +// ::= <name> [<discriminator>] + +static bool ParseLocalNameSuffix(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + + if (MaybeAppend(state, "::") && ParseName(state) && + Optional(ParseDiscriminator(state))) { + return true; + } + + // Since we're not going to overwrite the above "::" by re-parsing the + // <encoding> (whose trailing '\0' byte was in the byte now holding the + // first ':'), we have to rollback the "::" if the <name> parse failed. + if (state->parse_state.append) { + state->out[state->parse_state.out_cur_idx - 2] = '\0'; + } + + return ParseOneCharToken(state, 's') && Optional(ParseDiscriminator(state)); +} + +static bool ParseLocalName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'Z') && ParseEncoding(state) && + ParseOneCharToken(state, 'E') && ParseLocalNameSuffix(state)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <discriminator> := _ <(non-negative) number> +static bool ParseDiscriminator(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, '_') && ParseNumber(state, nullptr)) { + return true; + } + state->parse_state = copy; + return false; +} + +// <substitution> ::= S_ +// ::= S <seq-id> _ +// ::= St, etc. +// +// "St" is special in that it's not valid as a standalone name, and it *is* +// allowed to precede a name without being wrapped in "N...E". This means that +// if we accept it on its own, we can accept "St1a" and try to parse +// template-args, then fail and backtrack, accept "St" on its own, then "1a" as +// an unqualified name and re-parse the same template-args. To block this +// exponential backtracking, we disable it with 'accept_std=false' in +// problematic contexts. +static bool ParseSubstitution(State *state, bool accept_std) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (ParseTwoCharToken(state, "S_")) { + MaybeAppend(state, "?"); // We don't support substitutions. + return true; + } + + ParseState copy = state->parse_state; + if (ParseOneCharToken(state, 'S') && ParseSeqId(state) && + ParseOneCharToken(state, '_')) { + MaybeAppend(state, "?"); // We don't support substitutions. + return true; + } + state->parse_state = copy; + + // Expand abbreviations like "St" => "std". + if (ParseOneCharToken(state, 'S')) { + const AbbrevPair *p; + for (p = kSubstitutionList; p->abbrev != nullptr; ++p) { + if (RemainingInput(state)[0] == p->abbrev[1] && + (accept_std || p->abbrev[1] != 't')) { + MaybeAppend(state, "std"); + if (p->real_name[0] != '\0') { + MaybeAppend(state, "::"); + MaybeAppend(state, p->real_name); + } + ++state->parse_state.mangled_idx; + return true; + } + } + } + state->parse_state = copy; + return false; +} + +// Parse <mangled-name>, optionally followed by either a function-clone suffix +// or version suffix. Returns true only if all of "mangled_cur" was consumed. +static bool ParseTopLevelMangledName(State *state) { + ComplexityGuard guard(state); + if (guard.IsTooComplex()) return false; + if (ParseMangledName(state)) { + if (RemainingInput(state)[0] != '\0') { + // Drop trailing function clone suffix, if any. + if (IsFunctionCloneSuffix(RemainingInput(state))) { + return true; + } + // Append trailing version suffix if any. + // ex. _Z3foo@@GLIBCXX_3.4 + if (RemainingInput(state)[0] == '@') { + MaybeAppend(state, RemainingInput(state)); + return true; + } + return false; // Unconsumed suffix. + } + return true; + } + return false; +} + +static bool Overflowed(const State *state) { + return state->parse_state.out_cur_idx >= state->out_end_idx; +} + +// The demangler entry point. +bool Demangle(const char *mangled, char *out, int out_size) { + State state; + InitState(&state, mangled, out, out_size); + return ParseTopLevelMangledName(&state) && !Overflowed(&state) && + state.parse_state.out_cur_idx > 0; +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/internal/demangle.h b/third_party/abseil_cpp/absl/debugging/internal/demangle.h new file mode 100644 index 000000000000..c314d9bc237c --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/demangle.h @@ -0,0 +1,71 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// An async-signal-safe and thread-safe demangler for Itanium C++ ABI +// (aka G++ V3 ABI). +// +// The demangler is implemented to be used in async signal handlers to +// symbolize stack traces. We cannot use libstdc++'s +// abi::__cxa_demangle() in such signal handlers since it's not async +// signal safe (it uses malloc() internally). +// +// Note that this demangler doesn't support full demangling. More +// specifically, it doesn't print types of function parameters and +// types of template arguments. It just skips them. However, it's +// still very useful to extract basic information such as class, +// function, constructor, destructor, and operator names. +// +// See the implementation note in demangle.cc if you are interested. +// +// Example: +// +// | Mangled Name | The Demangler | abi::__cxa_demangle() +// |---------------|---------------|----------------------- +// | _Z1fv | f() | f() +// | _Z1fi | f() | f(int) +// | _Z3foo3bar | foo() | foo(bar) +// | _Z1fIiEvi | f<>() | void f<int>(int) +// | _ZN1N1fE | N::f | N::f +// | _ZN3Foo3BarEv | Foo::Bar() | Foo::Bar() +// | _Zrm1XS_" | operator%() | operator%(X, X) +// | _ZN3FooC1Ev | Foo::Foo() | Foo::Foo() +// | _Z1fSs | f() | f(std::basic_string<char, +// | | | std::char_traits<char>, +// | | | std::allocator<char> >) +// +// See the unit test for more examples. +// +// Note: we might want to write demanglers for ABIs other than Itanium +// C++ ABI in the future. +// + +#ifndef ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ +#define ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Demangle `mangled`. On success, return true and write the +// demangled symbol name to `out`. Otherwise, return false. +// `out` is modified even if demangling is unsuccessful. +bool Demangle(const char *mangled, char *out, int out_size); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_DEMANGLE_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/demangle_test.cc b/third_party/abseil_cpp/absl/debugging/internal/demangle_test.cc new file mode 100644 index 000000000000..0bed7359d8b4 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/demangle_test.cc @@ -0,0 +1,197 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/internal/demangle.h" + +#include <cstdlib> +#include <string> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/internal/stack_consumption.h" +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +namespace { + +// A wrapper function for Demangle() to make the unit test simple. +static const char *DemangleIt(const char * const mangled) { + static char demangled[4096]; + if (Demangle(mangled, demangled, sizeof(demangled))) { + return demangled; + } else { + return mangled; + } +} + +// Test corner cases of bounary conditions. +TEST(Demangle, CornerCases) { + char tmp[10]; + EXPECT_TRUE(Demangle("_Z6foobarv", tmp, sizeof(tmp))); + // sizeof("foobar()") == 9 + EXPECT_STREQ("foobar()", tmp); + EXPECT_TRUE(Demangle("_Z6foobarv", tmp, 9)); + EXPECT_STREQ("foobar()", tmp); + EXPECT_FALSE(Demangle("_Z6foobarv", tmp, 8)); // Not enough. + EXPECT_FALSE(Demangle("_Z6foobarv", tmp, 1)); + EXPECT_FALSE(Demangle("_Z6foobarv", tmp, 0)); + EXPECT_FALSE(Demangle("_Z6foobarv", nullptr, 0)); // Should not cause SEGV. + EXPECT_FALSE(Demangle("_Z1000000", tmp, 9)); +} + +// Test handling of functions suffixed with .clone.N, which is used +// by GCC 4.5.x (and our locally-modified version of GCC 4.4.x), and +// .constprop.N and .isra.N, which are used by GCC 4.6.x. These +// suffixes are used to indicate functions which have been cloned +// during optimization. We ignore these suffixes. +TEST(Demangle, Clones) { + char tmp[20]; + EXPECT_TRUE(Demangle("_ZL3Foov", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.clone.3", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.constprop.80", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.isra.18", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.isra.2.constprop.18", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + // Invalid (truncated), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.clo", tmp, sizeof(tmp))); + // Invalid (.clone. not followed by number), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.clone.", tmp, sizeof(tmp))); + // Invalid (.clone. followed by non-number), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.clone.foo", tmp, sizeof(tmp))); + // Invalid (.constprop. not followed by number), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.isra.2.constprop.", tmp, sizeof(tmp))); +} + +// Tests that verify that Demangle footprint is within some limit. +// They are not to be run under sanitizers as the sanitizers increase +// stack consumption by about 4x. +#if defined(ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION) && \ + !defined(ABSL_HAVE_ADDRESS_SANITIZER) && \ + !defined(ABSL_HAVE_MEMORY_SANITIZER) && \ + !defined(ABSL_HAVE_THREAD_SANITIZER) + +static const char *g_mangled; +static char g_demangle_buffer[4096]; +static char *g_demangle_result; + +static void DemangleSignalHandler(int signo) { + if (Demangle(g_mangled, g_demangle_buffer, sizeof(g_demangle_buffer))) { + g_demangle_result = g_demangle_buffer; + } else { + g_demangle_result = nullptr; + } +} + +// Call Demangle and figure out the stack footprint of this call. +static const char *DemangleStackConsumption(const char *mangled, + int *stack_consumed) { + g_mangled = mangled; + *stack_consumed = GetSignalHandlerStackConsumption(DemangleSignalHandler); + ABSL_RAW_LOG(INFO, "Stack consumption of Demangle: %d", *stack_consumed); + return g_demangle_result; +} + +// Demangle stack consumption should be within 8kB for simple mangled names +// with some level of nesting. With alternate signal stack we have 64K, +// but some signal handlers run on thread stack, and could have arbitrarily +// little space left (so we don't want to make this number too large). +const int kStackConsumptionUpperLimit = 8192; + +// Returns a mangled name nested to the given depth. +static std::string NestedMangledName(int depth) { + std::string mangled_name = "_Z1a"; + if (depth > 0) { + mangled_name += "IXL"; + mangled_name += NestedMangledName(depth - 1); + mangled_name += "EEE"; + } + return mangled_name; +} + +TEST(Demangle, DemangleStackConsumption) { + // Measure stack consumption of Demangle for nested mangled names of varying + // depth. Since Demangle is implemented as a recursive descent parser, + // stack consumption will grow as the nesting depth increases. By measuring + // the stack consumption for increasing depths, we can see the growing + // impact of any stack-saving changes made to the code for Demangle. + int stack_consumed = 0; + + const char *demangled = + DemangleStackConsumption("_Z6foobarv", &stack_consumed); + EXPECT_STREQ("foobar()", demangled); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); + + const std::string nested_mangled_name0 = NestedMangledName(0); + demangled = DemangleStackConsumption(nested_mangled_name0.c_str(), + &stack_consumed); + EXPECT_STREQ("a", demangled); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); + + const std::string nested_mangled_name1 = NestedMangledName(1); + demangled = DemangleStackConsumption(nested_mangled_name1.c_str(), + &stack_consumed); + EXPECT_STREQ("a<>", demangled); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); + + const std::string nested_mangled_name2 = NestedMangledName(2); + demangled = DemangleStackConsumption(nested_mangled_name2.c_str(), + &stack_consumed); + EXPECT_STREQ("a<>", demangled); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); + + const std::string nested_mangled_name3 = NestedMangledName(3); + demangled = DemangleStackConsumption(nested_mangled_name3.c_str(), + &stack_consumed); + EXPECT_STREQ("a<>", demangled); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); +} + +#endif // Stack consumption tests + +static void TestOnInput(const char* input) { + static const int kOutSize = 1048576; + auto out = absl::make_unique<char[]>(kOutSize); + Demangle(input, out.get(), kOutSize); +} + +TEST(DemangleRegression, NegativeLength) { + TestOnInput("_ZZn4"); +} + +TEST(DemangleRegression, DeeplyNestedArrayType) { + const int depth = 100000; + std::string data = "_ZStI"; + data.reserve(data.size() + 3 * depth + 1); + for (int i = 0; i < depth; i++) { + data += "A1_"; + } + TestOnInput(data.c_str()); +} + +} // namespace +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/internal/elf_mem_image.cc b/third_party/abseil_cpp/absl/debugging/internal/elf_mem_image.cc new file mode 100644 index 000000000000..24cc01302dc5 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/elf_mem_image.cc @@ -0,0 +1,382 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Allow dynamic symbol lookup in an in-memory Elf image. +// + +#include "absl/debugging/internal/elf_mem_image.h" + +#ifdef ABSL_HAVE_ELF_MEM_IMAGE // defined in elf_mem_image.h + +#include <string.h> +#include <cassert> +#include <cstddef> +#include "absl/base/internal/raw_logging.h" + +// From binutils/include/elf/common.h (this doesn't appear to be documented +// anywhere else). +// +// /* This flag appears in a Versym structure. It means that the symbol +// is hidden, and is only visible with an explicit version number. +// This is a GNU extension. */ +// #define VERSYM_HIDDEN 0x8000 +// +// /* This is the mask for the rest of the Versym information. */ +// #define VERSYM_VERSION 0x7fff + +#define VERSYM_VERSION 0x7fff + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +namespace { + +#if __WORDSIZE == 32 +const int kElfClass = ELFCLASS32; +int ElfBind(const ElfW(Sym) *symbol) { return ELF32_ST_BIND(symbol->st_info); } +int ElfType(const ElfW(Sym) *symbol) { return ELF32_ST_TYPE(symbol->st_info); } +#elif __WORDSIZE == 64 +const int kElfClass = ELFCLASS64; +int ElfBind(const ElfW(Sym) *symbol) { return ELF64_ST_BIND(symbol->st_info); } +int ElfType(const ElfW(Sym) *symbol) { return ELF64_ST_TYPE(symbol->st_info); } +#else +const int kElfClass = -1; +int ElfBind(const ElfW(Sym) *) { + ABSL_RAW_LOG(FATAL, "Unexpected word size"); + return 0; +} +int ElfType(const ElfW(Sym) *) { + ABSL_RAW_LOG(FATAL, "Unexpected word size"); + return 0; +} +#endif + +// Extract an element from one of the ELF tables, cast it to desired type. +// This is just a simple arithmetic and a glorified cast. +// Callers are responsible for bounds checking. +template <typename T> +const T *GetTableElement(const ElfW(Ehdr) * ehdr, ElfW(Off) table_offset, + ElfW(Word) element_size, size_t index) { + return reinterpret_cast<const T*>(reinterpret_cast<const char *>(ehdr) + + table_offset + + index * element_size); +} + +} // namespace + +// The value of this variable doesn't matter; it's used only for its +// unique address. +const int ElfMemImage::kInvalidBaseSentinel = 0; + +ElfMemImage::ElfMemImage(const void *base) { + ABSL_RAW_CHECK(base != kInvalidBase, "bad pointer"); + Init(base); +} + +int ElfMemImage::GetNumSymbols() const { + if (!hash_) { + return 0; + } + // See http://www.caldera.com/developers/gabi/latest/ch5.dynamic.html#hash + return hash_[1]; +} + +const ElfW(Sym) *ElfMemImage::GetDynsym(int index) const { + ABSL_RAW_CHECK(index < GetNumSymbols(), "index out of range"); + return dynsym_ + index; +} + +const ElfW(Versym) *ElfMemImage::GetVersym(int index) const { + ABSL_RAW_CHECK(index < GetNumSymbols(), "index out of range"); + return versym_ + index; +} + +const ElfW(Phdr) *ElfMemImage::GetPhdr(int index) const { + ABSL_RAW_CHECK(index < ehdr_->e_phnum, "index out of range"); + return GetTableElement<ElfW(Phdr)>(ehdr_, + ehdr_->e_phoff, + ehdr_->e_phentsize, + index); +} + +const char *ElfMemImage::GetDynstr(ElfW(Word) offset) const { + ABSL_RAW_CHECK(offset < strsize_, "offset out of range"); + return dynstr_ + offset; +} + +const void *ElfMemImage::GetSymAddr(const ElfW(Sym) *sym) const { + if (sym->st_shndx == SHN_UNDEF || sym->st_shndx >= SHN_LORESERVE) { + // Symbol corresponds to "special" (e.g. SHN_ABS) section. + return reinterpret_cast<const void *>(sym->st_value); + } + ABSL_RAW_CHECK(link_base_ < sym->st_value, "symbol out of range"); + return GetTableElement<char>(ehdr_, 0, 1, sym->st_value - link_base_); +} + +const ElfW(Verdef) *ElfMemImage::GetVerdef(int index) const { + ABSL_RAW_CHECK(0 <= index && static_cast<size_t>(index) <= verdefnum_, + "index out of range"); + const ElfW(Verdef) *version_definition = verdef_; + while (version_definition->vd_ndx < index && version_definition->vd_next) { + const char *const version_definition_as_char = + reinterpret_cast<const char *>(version_definition); + version_definition = + reinterpret_cast<const ElfW(Verdef) *>(version_definition_as_char + + version_definition->vd_next); + } + return version_definition->vd_ndx == index ? version_definition : nullptr; +} + +const ElfW(Verdaux) *ElfMemImage::GetVerdefAux( + const ElfW(Verdef) *verdef) const { + return reinterpret_cast<const ElfW(Verdaux) *>(verdef+1); +} + +const char *ElfMemImage::GetVerstr(ElfW(Word) offset) const { + ABSL_RAW_CHECK(offset < strsize_, "offset out of range"); + return dynstr_ + offset; +} + +void ElfMemImage::Init(const void *base) { + ehdr_ = nullptr; + dynsym_ = nullptr; + dynstr_ = nullptr; + versym_ = nullptr; + verdef_ = nullptr; + hash_ = nullptr; + strsize_ = 0; + verdefnum_ = 0; + link_base_ = ~0L; // Sentinel: PT_LOAD .p_vaddr can't possibly be this. + if (!base) { + return; + } + const char *const base_as_char = reinterpret_cast<const char *>(base); + if (base_as_char[EI_MAG0] != ELFMAG0 || base_as_char[EI_MAG1] != ELFMAG1 || + base_as_char[EI_MAG2] != ELFMAG2 || base_as_char[EI_MAG3] != ELFMAG3) { + assert(false); + return; + } + int elf_class = base_as_char[EI_CLASS]; + if (elf_class != kElfClass) { + assert(false); + return; + } + switch (base_as_char[EI_DATA]) { + case ELFDATA2LSB: { + if (__LITTLE_ENDIAN != __BYTE_ORDER) { + assert(false); + return; + } + break; + } + case ELFDATA2MSB: { + if (__BIG_ENDIAN != __BYTE_ORDER) { + assert(false); + return; + } + break; + } + default: { + assert(false); + return; + } + } + + ehdr_ = reinterpret_cast<const ElfW(Ehdr) *>(base); + const ElfW(Phdr) *dynamic_program_header = nullptr; + for (int i = 0; i < ehdr_->e_phnum; ++i) { + const ElfW(Phdr) *const program_header = GetPhdr(i); + switch (program_header->p_type) { + case PT_LOAD: + if (!~link_base_) { + link_base_ = program_header->p_vaddr; + } + break; + case PT_DYNAMIC: + dynamic_program_header = program_header; + break; + } + } + if (!~link_base_ || !dynamic_program_header) { + assert(false); + // Mark this image as not present. Can not recur infinitely. + Init(nullptr); + return; + } + ptrdiff_t relocation = + base_as_char - reinterpret_cast<const char *>(link_base_); + ElfW(Dyn) *dynamic_entry = + reinterpret_cast<ElfW(Dyn) *>(dynamic_program_header->p_vaddr + + relocation); + for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) { + const ElfW(Xword) value = dynamic_entry->d_un.d_val + relocation; + switch (dynamic_entry->d_tag) { + case DT_HASH: + hash_ = reinterpret_cast<ElfW(Word) *>(value); + break; + case DT_SYMTAB: + dynsym_ = reinterpret_cast<ElfW(Sym) *>(value); + break; + case DT_STRTAB: + dynstr_ = reinterpret_cast<const char *>(value); + break; + case DT_VERSYM: + versym_ = reinterpret_cast<ElfW(Versym) *>(value); + break; + case DT_VERDEF: + verdef_ = reinterpret_cast<ElfW(Verdef) *>(value); + break; + case DT_VERDEFNUM: + verdefnum_ = dynamic_entry->d_un.d_val; + break; + case DT_STRSZ: + strsize_ = dynamic_entry->d_un.d_val; + break; + default: + // Unrecognized entries explicitly ignored. + break; + } + } + if (!hash_ || !dynsym_ || !dynstr_ || !versym_ || + !verdef_ || !verdefnum_ || !strsize_) { + assert(false); // invalid VDSO + // Mark this image as not present. Can not recur infinitely. + Init(nullptr); + return; + } +} + +bool ElfMemImage::LookupSymbol(const char *name, + const char *version, + int type, + SymbolInfo *info_out) const { + for (const SymbolInfo& info : *this) { + if (strcmp(info.name, name) == 0 && strcmp(info.version, version) == 0 && + ElfType(info.symbol) == type) { + if (info_out) { + *info_out = info; + } + return true; + } + } + return false; +} + +bool ElfMemImage::LookupSymbolByAddress(const void *address, + SymbolInfo *info_out) const { + for (const SymbolInfo& info : *this) { + const char *const symbol_start = + reinterpret_cast<const char *>(info.address); + const char *const symbol_end = symbol_start + info.symbol->st_size; + if (symbol_start <= address && address < symbol_end) { + if (info_out) { + // Client wants to know details for that symbol (the usual case). + if (ElfBind(info.symbol) == STB_GLOBAL) { + // Strong symbol; just return it. + *info_out = info; + return true; + } else { + // Weak or local. Record it, but keep looking for a strong one. + *info_out = info; + } + } else { + // Client only cares if there is an overlapping symbol. + return true; + } + } + } + return false; +} + +ElfMemImage::SymbolIterator::SymbolIterator(const void *const image, int index) + : index_(index), image_(image) { +} + +const ElfMemImage::SymbolInfo *ElfMemImage::SymbolIterator::operator->() const { + return &info_; +} + +const ElfMemImage::SymbolInfo& ElfMemImage::SymbolIterator::operator*() const { + return info_; +} + +bool ElfMemImage::SymbolIterator::operator==(const SymbolIterator &rhs) const { + return this->image_ == rhs.image_ && this->index_ == rhs.index_; +} + +bool ElfMemImage::SymbolIterator::operator!=(const SymbolIterator &rhs) const { + return !(*this == rhs); +} + +ElfMemImage::SymbolIterator &ElfMemImage::SymbolIterator::operator++() { + this->Update(1); + return *this; +} + +ElfMemImage::SymbolIterator ElfMemImage::begin() const { + SymbolIterator it(this, 0); + it.Update(0); + return it; +} + +ElfMemImage::SymbolIterator ElfMemImage::end() const { + return SymbolIterator(this, GetNumSymbols()); +} + +void ElfMemImage::SymbolIterator::Update(int increment) { + const ElfMemImage *image = reinterpret_cast<const ElfMemImage *>(image_); + ABSL_RAW_CHECK(image->IsPresent() || increment == 0, ""); + if (!image->IsPresent()) { + return; + } + index_ += increment; + if (index_ >= image->GetNumSymbols()) { + index_ = image->GetNumSymbols(); + return; + } + const ElfW(Sym) *symbol = image->GetDynsym(index_); + const ElfW(Versym) *version_symbol = image->GetVersym(index_); + ABSL_RAW_CHECK(symbol && version_symbol, ""); + const char *const symbol_name = image->GetDynstr(symbol->st_name); + const ElfW(Versym) version_index = version_symbol[0] & VERSYM_VERSION; + const ElfW(Verdef) *version_definition = nullptr; + const char *version_name = ""; + if (symbol->st_shndx == SHN_UNDEF) { + // Undefined symbols reference DT_VERNEED, not DT_VERDEF, and + // version_index could well be greater than verdefnum_, so calling + // GetVerdef(version_index) may trigger assertion. + } else { + version_definition = image->GetVerdef(version_index); + } + if (version_definition) { + // I am expecting 1 or 2 auxiliary entries: 1 for the version itself, + // optional 2nd if the version has a parent. + ABSL_RAW_CHECK( + version_definition->vd_cnt == 1 || version_definition->vd_cnt == 2, + "wrong number of entries"); + const ElfW(Verdaux) *version_aux = image->GetVerdefAux(version_definition); + version_name = image->GetVerstr(version_aux->vda_name); + } + info_.name = symbol_name; + info_.version = version_name; + info_.address = image->GetSymAddr(symbol); + info_.symbol = symbol; +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HAVE_ELF_MEM_IMAGE diff --git a/third_party/abseil_cpp/absl/debugging/internal/elf_mem_image.h b/third_party/abseil_cpp/absl/debugging/internal/elf_mem_image.h new file mode 100644 index 000000000000..46bfade3503e --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/elf_mem_image.h @@ -0,0 +1,134 @@ +/* + * Copyright 2017 The Abseil Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Allow dynamic symbol lookup for in-memory Elf images. + +#ifndef ABSL_DEBUGGING_INTERNAL_ELF_MEM_IMAGE_H_ +#define ABSL_DEBUGGING_INTERNAL_ELF_MEM_IMAGE_H_ + +// Including this will define the __GLIBC__ macro if glibc is being +// used. +#include <climits> + +#include "absl/base/config.h" + +// Maybe one day we can rewrite this file not to require the elf +// symbol extensions in glibc, but for right now we need them. +#ifdef ABSL_HAVE_ELF_MEM_IMAGE +#error ABSL_HAVE_ELF_MEM_IMAGE cannot be directly set +#endif + +#if defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) && \ + !defined(__asmjs__) && !defined(__wasm__) +#define ABSL_HAVE_ELF_MEM_IMAGE 1 +#endif + +#ifdef ABSL_HAVE_ELF_MEM_IMAGE + +#include <link.h> // for ElfW + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// An in-memory ELF image (may not exist on disk). +class ElfMemImage { + private: + // Sentinel: there could never be an elf image at &kInvalidBaseSentinel. + static const int kInvalidBaseSentinel; + + public: + // Sentinel: there could never be an elf image at this address. + static constexpr const void *const kInvalidBase = + static_cast<const void*>(&kInvalidBaseSentinel); + + // Information about a single vdso symbol. + // All pointers are into .dynsym, .dynstr, or .text of the VDSO. + // Do not free() them or modify through them. + struct SymbolInfo { + const char *name; // E.g. "__vdso_getcpu" + const char *version; // E.g. "LINUX_2.6", could be "" + // for unversioned symbol. + const void *address; // Relocated symbol address. + const ElfW(Sym) *symbol; // Symbol in the dynamic symbol table. + }; + + // Supports iteration over all dynamic symbols. + class SymbolIterator { + public: + friend class ElfMemImage; + const SymbolInfo *operator->() const; + const SymbolInfo &operator*() const; + SymbolIterator& operator++(); + bool operator!=(const SymbolIterator &rhs) const; + bool operator==(const SymbolIterator &rhs) const; + private: + SymbolIterator(const void *const image, int index); + void Update(int incr); + SymbolInfo info_; + int index_; + const void *const image_; + }; + + + explicit ElfMemImage(const void *base); + void Init(const void *base); + bool IsPresent() const { return ehdr_ != nullptr; } + const ElfW(Phdr)* GetPhdr(int index) const; + const ElfW(Sym)* GetDynsym(int index) const; + const ElfW(Versym)* GetVersym(int index) const; + const ElfW(Verdef)* GetVerdef(int index) const; + const ElfW(Verdaux)* GetVerdefAux(const ElfW(Verdef) *verdef) const; + const char* GetDynstr(ElfW(Word) offset) const; + const void* GetSymAddr(const ElfW(Sym) *sym) const; + const char* GetVerstr(ElfW(Word) offset) const; + int GetNumSymbols() const; + + SymbolIterator begin() const; + SymbolIterator end() const; + + // Look up versioned dynamic symbol in the image. + // Returns false if image is not present, or doesn't contain given + // symbol/version/type combination. + // If info_out is non-null, additional details are filled in. + bool LookupSymbol(const char *name, const char *version, + int symbol_type, SymbolInfo *info_out) const; + + // Find info about symbol (if any) which overlaps given address. + // Returns true if symbol was found; false if image isn't present + // or doesn't have a symbol overlapping given address. + // If info_out is non-null, additional details are filled in. + bool LookupSymbolByAddress(const void *address, SymbolInfo *info_out) const; + + private: + const ElfW(Ehdr) *ehdr_; + const ElfW(Sym) *dynsym_; + const ElfW(Versym) *versym_; + const ElfW(Verdef) *verdef_; + const ElfW(Word) *hash_; + const char *dynstr_; + size_t strsize_; + size_t verdefnum_; + ElfW(Addr) link_base_; // Link-time base (p_vaddr of first PT_LOAD). +}; + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HAVE_ELF_MEM_IMAGE + +#endif // ABSL_DEBUGGING_INTERNAL_ELF_MEM_IMAGE_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/examine_stack.cc b/third_party/abseil_cpp/absl/debugging/internal/examine_stack.cc new file mode 100644 index 000000000000..6e5ff1fbd836 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/examine_stack.cc @@ -0,0 +1,187 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "absl/debugging/internal/examine_stack.h" + +#ifndef _WIN32 +#include <unistd.h> +#endif + +#ifdef __APPLE__ +#include <sys/ucontext.h> +#endif + +#include <csignal> +#include <cstdio> + +#include "absl/base/attributes.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Returns the program counter from signal context, nullptr if +// unknown. vuc is a ucontext_t*. We use void* to avoid the use of +// ucontext_t on non-POSIX systems. +void* GetProgramCounter(void* vuc) { +#ifdef __linux__ + if (vuc != nullptr) { + ucontext_t* context = reinterpret_cast<ucontext_t*>(vuc); +#if defined(__aarch64__) + return reinterpret_cast<void*>(context->uc_mcontext.pc); +#elif defined(__arm__) + return reinterpret_cast<void*>(context->uc_mcontext.arm_pc); +#elif defined(__i386__) + if (14 < ABSL_ARRAYSIZE(context->uc_mcontext.gregs)) + return reinterpret_cast<void*>(context->uc_mcontext.gregs[14]); +#elif defined(__mips__) + return reinterpret_cast<void*>(context->uc_mcontext.pc); +#elif defined(__powerpc64__) + return reinterpret_cast<void*>(context->uc_mcontext.gp_regs[32]); +#elif defined(__powerpc__) + return reinterpret_cast<void*>(context->uc_mcontext.regs->nip); +#elif defined(__riscv) + return reinterpret_cast<void*>(context->uc_mcontext.__gregs[REG_PC]); +#elif defined(__s390__) && !defined(__s390x__) + return reinterpret_cast<void*>(context->uc_mcontext.psw.addr & 0x7fffffff); +#elif defined(__s390__) && defined(__s390x__) + return reinterpret_cast<void*>(context->uc_mcontext.psw.addr); +#elif defined(__x86_64__) + if (16 < ABSL_ARRAYSIZE(context->uc_mcontext.gregs)) + return reinterpret_cast<void*>(context->uc_mcontext.gregs[16]); +#else +#error "Undefined Architecture." +#endif + } +#elif defined(__APPLE__) + if (vuc != nullptr) { + ucontext_t* signal_ucontext = reinterpret_cast<ucontext_t*>(vuc); +#if defined(__aarch64__) + return reinterpret_cast<void*>( + __darwin_arm_thread_state64_get_pc(signal_ucontext->uc_mcontext->__ss)); +#elif defined(__arm__) +#if __DARWIN_UNIX03 + return reinterpret_cast<void*>(signal_ucontext->uc_mcontext->__ss.__pc); +#else + return reinterpret_cast<void*>(signal_ucontext->uc_mcontext->ss.pc); +#endif +#elif defined(__i386__) +#if __DARWIN_UNIX03 + return reinterpret_cast<void*>(signal_ucontext->uc_mcontext->__ss.__eip); +#else + return reinterpret_cast<void*>(signal_ucontext->uc_mcontext->ss.eip); +#endif +#elif defined(__x86_64__) +#if __DARWIN_UNIX03 + return reinterpret_cast<void*>(signal_ucontext->uc_mcontext->__ss.__rip); +#else + return reinterpret_cast<void*>(signal_ucontext->uc_mcontext->ss.rip); +#endif +#endif + } +#elif defined(__akaros__) + auto* ctx = reinterpret_cast<struct user_context*>(vuc); + return reinterpret_cast<void*>(get_user_ctx_pc(ctx)); +#endif + static_cast<void>(vuc); + return nullptr; +} + +// The %p field width for printf() functions is two characters per byte, +// and two extra for the leading "0x". +static constexpr int kPrintfPointerFieldWidth = 2 + 2 * sizeof(void*); + +// Print a program counter, its stack frame size, and its symbol name. +// Note that there is a separate symbolize_pc argument. Return addresses may be +// at the end of the function, and this allows the caller to back up from pc if +// appropriate. +static void DumpPCAndFrameSizeAndSymbol(void (*writerfn)(const char*, void*), + void* writerfn_arg, void* pc, + void* symbolize_pc, int framesize, + const char* const prefix) { + char tmp[1024]; + const char* symbol = "(unknown)"; + if (absl::Symbolize(symbolize_pc, tmp, sizeof(tmp))) { + symbol = tmp; + } + char buf[1024]; + if (framesize <= 0) { + snprintf(buf, sizeof(buf), "%s@ %*p (unknown) %s\n", prefix, + kPrintfPointerFieldWidth, pc, symbol); + } else { + snprintf(buf, sizeof(buf), "%s@ %*p %9d %s\n", prefix, + kPrintfPointerFieldWidth, pc, framesize, symbol); + } + writerfn(buf, writerfn_arg); +} + +// Print a program counter and the corresponding stack frame size. +static void DumpPCAndFrameSize(void (*writerfn)(const char*, void*), + void* writerfn_arg, void* pc, int framesize, + const char* const prefix) { + char buf[100]; + if (framesize <= 0) { + snprintf(buf, sizeof(buf), "%s@ %*p (unknown)\n", prefix, + kPrintfPointerFieldWidth, pc); + } else { + snprintf(buf, sizeof(buf), "%s@ %*p %9d\n", prefix, + kPrintfPointerFieldWidth, pc, framesize); + } + writerfn(buf, writerfn_arg); +} + +void DumpPCAndFrameSizesAndStackTrace( + void* pc, void* const stack[], int frame_sizes[], int depth, + int min_dropped_frames, bool symbolize_stacktrace, + void (*writerfn)(const char*, void*), void* writerfn_arg) { + if (pc != nullptr) { + // We don't know the stack frame size for PC, use 0. + if (symbolize_stacktrace) { + DumpPCAndFrameSizeAndSymbol(writerfn, writerfn_arg, pc, pc, 0, "PC: "); + } else { + DumpPCAndFrameSize(writerfn, writerfn_arg, pc, 0, "PC: "); + } + } + for (int i = 0; i < depth; i++) { + if (symbolize_stacktrace) { + // Pass the previous address of pc as the symbol address because pc is a + // return address, and an overrun may occur when the function ends with a + // call to a function annotated noreturn (e.g. CHECK). Note that we don't + // do this for pc above, as the adjustment is only correct for return + // addresses. + DumpPCAndFrameSizeAndSymbol(writerfn, writerfn_arg, stack[i], + reinterpret_cast<char*>(stack[i]) - 1, + frame_sizes[i], " "); + } else { + DumpPCAndFrameSize(writerfn, writerfn_arg, stack[i], frame_sizes[i], + " "); + } + } + if (min_dropped_frames > 0) { + char buf[100]; + snprintf(buf, sizeof(buf), " @ ... and at least %d more frames\n", + min_dropped_frames); + writerfn(buf, writerfn_arg); + } +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/internal/examine_stack.h b/third_party/abseil_cpp/absl/debugging/internal/examine_stack.h new file mode 100644 index 000000000000..393369131f1e --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/examine_stack.h @@ -0,0 +1,42 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_DEBUGGING_INTERNAL_EXAMINE_STACK_H_ +#define ABSL_DEBUGGING_INTERNAL_EXAMINE_STACK_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Returns the program counter from signal context, or nullptr if +// unknown. `vuc` is a ucontext_t*. We use void* to avoid the use of +// ucontext_t on non-POSIX systems. +void* GetProgramCounter(void* vuc); + +// Uses `writerfn` to dump the program counter, stack trace, and stack +// frame sizes. +void DumpPCAndFrameSizesAndStackTrace( + void* pc, void* const stack[], int frame_sizes[], int depth, + int min_dropped_frames, bool symbolize_stacktrace, + void (*writerfn)(const char*, void*), void* writerfn_arg); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_EXAMINE_STACK_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stack_consumption.cc b/third_party/abseil_cpp/absl/debugging/internal/stack_consumption.cc new file mode 100644 index 000000000000..e3dd51c355ad --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stack_consumption.cc @@ -0,0 +1,185 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/internal/stack_consumption.h" + +#ifdef ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION + +#include <signal.h> +#include <sys/mman.h> +#include <unistd.h> + +#include <string.h> + +#include "absl/base/attributes.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +namespace { + +// This code requires that we know the direction in which the stack +// grows. It is commonly believed that this can be detected by putting +// a variable on the stack and then passing its address to a function +// that compares the address of this variable to the address of a +// variable on the function's own stack. However, this is unspecified +// behavior in C++: If two pointers p and q of the same type point to +// different objects that are not members of the same object or +// elements of the same array or to different functions, or if only +// one of them is null, the results of p<q, p>q, p<=q, and p>=q are +// unspecified. Therefore, instead we hardcode the direction of the +// stack on platforms we know about. +#if defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || \ + defined(__aarch64__) +constexpr bool kStackGrowsDown = true; +#else +#error Need to define kStackGrowsDown +#endif + +// To measure the stack footprint of some code, we create a signal handler +// (for SIGUSR2 say) that exercises this code on an alternate stack. This +// alternate stack is initialized to some known pattern (0x55, 0x55, 0x55, +// ...). We then self-send this signal, and after the signal handler returns, +// look at the alternate stack buffer to see what portion has been touched. +// +// This trick gives us the the stack footprint of the signal handler. But the +// signal handler, even before the code for it is exercised, consumes some +// stack already. We however only want the stack usage of the code inside the +// signal handler. To measure this accurately, we install two signal handlers: +// one that does nothing and just returns, and the user-provided signal +// handler. The difference between the stack consumption of these two signals +// handlers should give us the stack foorprint of interest. + +void EmptySignalHandler(int) {} + +// This is arbitrary value, and could be increase further, at the cost of +// memset()ting it all to known sentinel value. +constexpr int kAlternateStackSize = 64 << 10; // 64KiB + +constexpr int kSafetyMargin = 32; +constexpr char kAlternateStackFillValue = 0x55; + +// These helper functions look at the alternate stack buffer, and figure +// out what portion of this buffer has been touched - this is the stack +// consumption of the signal handler running on this alternate stack. +// This function will return -1 if the alternate stack buffer has not been +// touched. It will abort the program if the buffer has overflowed or is about +// to overflow. +int GetStackConsumption(const void* const altstack) { + const char* begin; + int increment; + if (kStackGrowsDown) { + begin = reinterpret_cast<const char*>(altstack); + increment = 1; + } else { + begin = reinterpret_cast<const char*>(altstack) + kAlternateStackSize - 1; + increment = -1; + } + + for (int usage_count = kAlternateStackSize; usage_count > 0; --usage_count) { + if (*begin != kAlternateStackFillValue) { + ABSL_RAW_CHECK(usage_count <= kAlternateStackSize - kSafetyMargin, + "Buffer has overflowed or is about to overflow"); + return usage_count; + } + begin += increment; + } + + ABSL_RAW_LOG(FATAL, "Unreachable code"); + return -1; +} + +} // namespace + +int GetSignalHandlerStackConsumption(void (*signal_handler)(int)) { + // The alt-signal-stack cannot be heap allocated because there is a + // bug in glibc-2.2 where some signal handler setup code looks at the + // current stack pointer to figure out what thread is currently running. + // Therefore, the alternate stack must be allocated from the main stack + // itself. + void* altstack = mmap(nullptr, kAlternateStackSize, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + ABSL_RAW_CHECK(altstack != MAP_FAILED, "mmap() failed"); + + // Set up the alt-signal-stack (and save the older one). + stack_t sigstk; + memset(&sigstk, 0, sizeof(sigstk)); + sigstk.ss_sp = altstack; + sigstk.ss_size = kAlternateStackSize; + sigstk.ss_flags = 0; + stack_t old_sigstk; + memset(&old_sigstk, 0, sizeof(old_sigstk)); + ABSL_RAW_CHECK(sigaltstack(&sigstk, &old_sigstk) == 0, + "sigaltstack() failed"); + + // Set up SIGUSR1 and SIGUSR2 signal handlers (and save the older ones). + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + struct sigaction old_sa1, old_sa2; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_ONSTACK; + + // SIGUSR1 maps to EmptySignalHandler. + sa.sa_handler = EmptySignalHandler; + ABSL_RAW_CHECK(sigaction(SIGUSR1, &sa, &old_sa1) == 0, "sigaction() failed"); + + // SIGUSR2 maps to signal_handler. + sa.sa_handler = signal_handler; + ABSL_RAW_CHECK(sigaction(SIGUSR2, &sa, &old_sa2) == 0, "sigaction() failed"); + + // Send SIGUSR1 signal and measure the stack consumption of the empty + // signal handler. + // The first signal might use more stack space. Run once and ignore the + // results to get that out of the way. + ABSL_RAW_CHECK(kill(getpid(), SIGUSR1) == 0, "kill() failed"); + + memset(altstack, kAlternateStackFillValue, kAlternateStackSize); + ABSL_RAW_CHECK(kill(getpid(), SIGUSR1) == 0, "kill() failed"); + int base_stack_consumption = GetStackConsumption(altstack); + + // Send SIGUSR2 signal and measure the stack consumption of signal_handler. + ABSL_RAW_CHECK(kill(getpid(), SIGUSR2) == 0, "kill() failed"); + int signal_handler_stack_consumption = GetStackConsumption(altstack); + + // Now restore the old alt-signal-stack and signal handlers. + if (old_sigstk.ss_sp == nullptr && old_sigstk.ss_size == 0 && + (old_sigstk.ss_flags & SS_DISABLE)) { + // https://git.musl-libc.org/cgit/musl/commit/src/signal/sigaltstack.c?id=7829f42a2c8944555439380498ab8b924d0f2070 + // The original stack has ss_size==0 and ss_flags==SS_DISABLE, but some + // versions of musl have a bug that rejects ss_size==0. Work around this by + // setting ss_size to MINSIGSTKSZ, which should be ignored by the kernel + // when SS_DISABLE is set. + old_sigstk.ss_size = MINSIGSTKSZ; + } + ABSL_RAW_CHECK(sigaltstack(&old_sigstk, nullptr) == 0, + "sigaltstack() failed"); + ABSL_RAW_CHECK(sigaction(SIGUSR1, &old_sa1, nullptr) == 0, + "sigaction() failed"); + ABSL_RAW_CHECK(sigaction(SIGUSR2, &old_sa2, nullptr) == 0, + "sigaction() failed"); + + ABSL_RAW_CHECK(munmap(altstack, kAlternateStackSize) == 0, "munmap() failed"); + if (signal_handler_stack_consumption != -1 && base_stack_consumption != -1) { + return signal_handler_stack_consumption - base_stack_consumption; + } + return -1; +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION diff --git a/third_party/abseil_cpp/absl/debugging/internal/stack_consumption.h b/third_party/abseil_cpp/absl/debugging/internal/stack_consumption.h new file mode 100644 index 000000000000..2b5e7151667c --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stack_consumption.h @@ -0,0 +1,50 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Helper function for measuring stack consumption of signal handlers. + +#ifndef ABSL_DEBUGGING_INTERNAL_STACK_CONSUMPTION_H_ +#define ABSL_DEBUGGING_INTERNAL_STACK_CONSUMPTION_H_ + +#include "absl/base/config.h" + +// The code in this module is not portable. +// Use this feature test macro to detect its availability. +#ifdef ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION +#error ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION cannot be set directly +#elif !defined(__APPLE__) && !defined(_WIN32) && \ + (defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || \ + defined(__aarch64__)) +#define ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION 1 + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Returns the stack consumption in bytes for the code exercised by +// signal_handler. To measure stack consumption, signal_handler is registered +// as a signal handler, so the code that it exercises must be async-signal +// safe. The argument of signal_handler is an implementation detail of signal +// handlers and should ignored by the code for signal_handler. Use global +// variables to pass information between your test code and signal_handler. +int GetSignalHandlerStackConsumption(void (*signal_handler)(int)); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION + +#endif // ABSL_DEBUGGING_INTERNAL_STACK_CONSUMPTION_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stack_consumption_test.cc b/third_party/abseil_cpp/absl/debugging/internal/stack_consumption_test.cc new file mode 100644 index 000000000000..80445bf43aab --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stack_consumption_test.cc @@ -0,0 +1,50 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/internal/stack_consumption.h" + +#ifdef ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION + +#include <string.h> + +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +namespace { + +static void SimpleSignalHandler(int signo) { + char buf[100]; + memset(buf, 'a', sizeof(buf)); + + // Never true, but prevents compiler from optimizing buf out. + if (signo == 0) { + ABSL_RAW_LOG(INFO, "%p", static_cast<void*>(buf)); + } +} + +TEST(SignalHandlerStackConsumptionTest, MeasuresStackConsumption) { + // Our handler should consume reasonable number of bytes. + EXPECT_GE(GetSignalHandlerStackConsumption(SimpleSignalHandler), 100); +} + +} // namespace +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc new file mode 100644 index 000000000000..f4859d7c210a --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_aarch64-inl.inc @@ -0,0 +1,199 @@ +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_AARCH64_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_AARCH64_INL_H_ + +// Generate stack tracer for aarch64 + +#if defined(__linux__) +#include <sys/mman.h> +#include <ucontext.h> +#include <unistd.h> +#endif + +#include <atomic> +#include <cassert> +#include <cstdint> +#include <iostream> + +#include "absl/base/attributes.h" +#include "absl/debugging/internal/address_is_readable.h" +#include "absl/debugging/internal/vdso_support.h" // a no-op on non-elf or non-glibc systems +#include "absl/debugging/stacktrace.h" + +static const uintptr_t kUnknownFrameSize = 0; + +#if defined(__linux__) +// Returns the address of the VDSO __kernel_rt_sigreturn function, if present. +static const unsigned char* GetKernelRtSigreturnAddress() { + constexpr uintptr_t kImpossibleAddress = 1; + ABSL_CONST_INIT static std::atomic<uintptr_t> memoized{kImpossibleAddress}; + uintptr_t address = memoized.load(std::memory_order_relaxed); + if (address != kImpossibleAddress) { + return reinterpret_cast<const unsigned char*>(address); + } + + address = reinterpret_cast<uintptr_t>(nullptr); + +#ifdef ABSL_HAVE_VDSO_SUPPORT + absl::debugging_internal::VDSOSupport vdso; + if (vdso.IsPresent()) { + absl::debugging_internal::VDSOSupport::SymbolInfo symbol_info; + auto lookup = [&](int type) { + return vdso.LookupSymbol("__kernel_rt_sigreturn", "LINUX_2.6.39", type, + &symbol_info); + }; + if ((!lookup(STT_FUNC) && !lookup(STT_NOTYPE)) || + symbol_info.address == nullptr) { + // Unexpected: VDSO is present, yet the expected symbol is missing + // or null. + assert(false && "VDSO is present, but doesn't have expected symbol"); + } else { + if (reinterpret_cast<uintptr_t>(symbol_info.address) != + kImpossibleAddress) { + address = reinterpret_cast<uintptr_t>(symbol_info.address); + } else { + assert(false && "VDSO returned invalid address"); + } + } + } +#endif + + memoized.store(address, std::memory_order_relaxed); + return reinterpret_cast<const unsigned char*>(address); +} +#endif // __linux__ + +// Compute the size of a stack frame in [low..high). We assume that +// low < high. Return size of kUnknownFrameSize. +template<typename T> +static inline uintptr_t ComputeStackFrameSize(const T* low, + const T* high) { + const char* low_char_ptr = reinterpret_cast<const char *>(low); + const char* high_char_ptr = reinterpret_cast<const char *>(high); + return low < high ? high_char_ptr - low_char_ptr : kUnknownFrameSize; +} + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return null if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template<bool STRICT_UNWINDING, bool WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static void **NextStackFrame(void **old_frame_pointer, const void *uc) { + void **new_frame_pointer = reinterpret_cast<void**>(*old_frame_pointer); + bool check_frame_size = true; + +#if defined(__linux__) + if (WITH_CONTEXT && uc != nullptr) { + // Check to see if next frame's return address is __kernel_rt_sigreturn. + if (old_frame_pointer[1] == GetKernelRtSigreturnAddress()) { + const ucontext_t *ucv = static_cast<const ucontext_t *>(uc); + // old_frame_pointer[0] is not suitable for unwinding, look at + // ucontext to discover frame pointer before signal. + void **const pre_signal_frame_pointer = + reinterpret_cast<void **>(ucv->uc_mcontext.regs[29]); + + // Check that alleged frame pointer is actually readable. This is to + // prevent "double fault" in case we hit the first fault due to e.g. + // stack corruption. + if (!absl::debugging_internal::AddressIsReadable( + pre_signal_frame_pointer)) + return nullptr; + + // Alleged frame pointer is readable, use it for further unwinding. + new_frame_pointer = pre_signal_frame_pointer; + + // Skip frame size check if we return from a signal. We may be using a + // an alternate stack for signals. + check_frame_size = false; + } + } +#endif + + // aarch64 ABI requires stack pointer to be 16-byte-aligned. + if ((reinterpret_cast<uintptr_t>(new_frame_pointer) & 15) != 0) + return nullptr; + + // Check frame size. In strict mode, we assume frames to be under + // 100,000 bytes. In non-strict mode, we relax the limit to 1MB. + if (check_frame_size) { + const uintptr_t max_size = STRICT_UNWINDING ? 100000 : 1000000; + const uintptr_t frame_size = + ComputeStackFrameSize(old_frame_pointer, new_frame_pointer); + if (frame_size == kUnknownFrameSize || frame_size > max_size) + return nullptr; + } + + return new_frame_pointer; +} + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, + const void *ucp, int *min_dropped_frames) { +#ifdef __GNUC__ + void **frame_pointer = reinterpret_cast<void**>(__builtin_frame_address(0)); +#else +# error reading stack point not yet supported on this platform. +#endif + + skip_count++; // Skip the frame for this function. + int n = 0; + + // The frame pointer points to low address of a frame. The first 64-bit + // word of a frame points to the next frame up the call chain, which normally + // is just after the high address of the current frame. The second word of + // a frame contains return adress of to the caller. To find a pc value + // associated with the current frame, we need to go down a level in the call + // chain. So we remember return the address of the last frame seen. This + // does not work for the first stack frame, which belongs to UnwindImp() but + // we skip the frame for UnwindImp() anyway. + void* prev_return_address = nullptr; + + while (frame_pointer && n < max_depth) { + // The absl::GetStackFrames routine is called when we are in some + // informational context (the failure signal handler for example). + // Use the non-strict unwinding rules to produce a stack trace + // that is as complete as possible (even if it contains a few bogus + // entries in some rare cases). + void **next_frame_pointer = + NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(frame_pointer, ucp); + + if (skip_count > 0) { + skip_count--; + } else { + result[n] = prev_return_address; + if (IS_STACK_FRAMES) { + sizes[n] = ComputeStackFrameSize(frame_pointer, next_frame_pointer); + } + n++; + } + prev_return_address = frame_pointer[1]; + frame_pointer = next_frame_pointer; + } + if (min_dropped_frames != nullptr) { + // Implementation detail: we clamp the max of frames we are willing to + // count, so as not to spend too much time in the loop below. + const int kMaxUnwind = 200; + int j = 0; + for (; frame_pointer != nullptr && j < kMaxUnwind; j++) { + frame_pointer = + NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(frame_pointer, ucp); + } + *min_dropped_frames = j; + } + return n; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return true; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_AARCH64_INL_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_arm-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_arm-inl.inc new file mode 100644 index 000000000000..2a1bf2e88694 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_arm-inl.inc @@ -0,0 +1,134 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This is inspired by Craig Silverstein's PowerPC stacktrace code. + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_ARM_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_ARM_INL_H_ + +#include <cstdint> + +#include "absl/debugging/stacktrace.h" + +// WARNING: +// This only works if all your code is in either ARM or THUMB mode. With +// interworking, the frame pointer of the caller can either be in r11 (ARM +// mode) or r7 (THUMB mode). A callee only saves the frame pointer of its +// mode in a fixed location on its stack frame. If the caller is a different +// mode, there is no easy way to find the frame pointer. It can either be +// still in the designated register or saved on stack along with other callee +// saved registers. + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return nullptr if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template<bool STRICT_UNWINDING> +static void **NextStackFrame(void **old_sp) { + void **new_sp = (void**) old_sp[-1]; + + // Check that the transition from frame pointer old_sp to frame + // pointer new_sp isn't clearly bogus + if (STRICT_UNWINDING) { + // With the stack growing downwards, older stack frame must be + // at a greater address that the current one. + if (new_sp <= old_sp) return nullptr; + // Assume stack frames larger than 100,000 bytes are bogus. + if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return nullptr; + } else { + // In the non-strict mode, allow discontiguous stack frames. + // (alternate-signal-stacks for example). + if (new_sp == old_sp) return nullptr; + // And allow frames upto about 1MB. + if ((new_sp > old_sp) + && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return nullptr; + } + if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return nullptr; + return new_sp; +} + +// This ensures that absl::GetStackTrace sets up the Link Register properly. +#ifdef __GNUC__ +void StacktraceArmDummyFunction() __attribute__((noinline)); +void StacktraceArmDummyFunction() { __asm__ volatile(""); } +#else +# error StacktraceArmDummyFunction() needs to be ported to this platform. +#endif + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, + const void * /* ucp */, int *min_dropped_frames) { +#ifdef __GNUC__ + void **sp = reinterpret_cast<void**>(__builtin_frame_address(0)); +#else +# error reading stack point not yet supported on this platform. +#endif + + // On ARM, the return address is stored in the link register (r14). + // This is not saved on the stack frame of a leaf function. To + // simplify code that reads return addresses, we call a dummy + // function so that the return address of this function is also + // stored in the stack frame. This works at least for gcc. + StacktraceArmDummyFunction(); + + int n = 0; + while (sp && n < max_depth) { + // The absl::GetStackFrames routine is called when we are in some + // informational context (the failure signal handler for example). + // Use the non-strict unwinding rules to produce a stack trace + // that is as complete as possible (even if it contains a few bogus + // entries in some rare cases). + void **next_sp = NextStackFrame<!IS_STACK_FRAMES>(sp); + + if (skip_count > 0) { + skip_count--; + } else { + result[n] = *sp; + + if (IS_STACK_FRAMES) { + if (next_sp > sp) { + sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } + } + n++; + } + sp = next_sp; + } + if (min_dropped_frames != nullptr) { + // Implementation detail: we clamp the max of frames we are willing to + // count, so as not to spend too much time in the loop below. + const int kMaxUnwind = 200; + int j = 0; + for (; sp != nullptr && j < kMaxUnwind; j++) { + sp = NextStackFrame<!IS_STACK_FRAMES>(sp); + } + *min_dropped_frames = j; + } + return n; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return false; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_ARM_INL_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_config.h b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_config.h new file mode 100644 index 000000000000..90af852818e7 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_config.h @@ -0,0 +1,89 @@ +/* + * Copyright 2017 The Abseil Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + + * Defines ABSL_STACKTRACE_INL_HEADER to the *-inl.h containing + * actual unwinder implementation. + * This header is "private" to stacktrace.cc. + * DO NOT include it into any other files. +*/ +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_CONFIG_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_CONFIG_H_ + +#if defined(ABSL_STACKTRACE_INL_HEADER) +#error ABSL_STACKTRACE_INL_HEADER cannot be directly set + +#elif defined(_WIN32) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_win32-inl.inc" + +#elif defined(__APPLE__) +// Thread local support required for UnwindImpl. +// Notes: +// * Xcode's clang did not support `thread_local` until version 8, and +// even then not for all iOS < 9.0. +// * Xcode 9.3 started disallowing `thread_local` for 32-bit iOS simulator +// targeting iOS 9.x. +// * Xcode 10 moves the deployment target check for iOS < 9.0 to link time +// making __has_feature unreliable there. +// +// Otherwise, `__has_feature` is only supported by Clang so it has be inside +// `defined(__APPLE__)` check. +#if __has_feature(cxx_thread_local) && \ + !(TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MIN_REQUIRED < __IPHONE_9_0) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_generic-inl.inc" +#endif + +#elif defined(__linux__) && !defined(__ANDROID__) + +#if defined(NO_FRAME_POINTER) && \ + (defined(__i386__) || defined(__x86_64__) || defined(__aarch64__)) +// Note: The libunwind-based implementation is not available to open-source +// users. +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_libunwind-inl.inc" +#define STACKTRACE_USES_LIBUNWIND 1 +#elif defined(NO_FRAME_POINTER) && defined(__has_include) +#if __has_include(<execinfo.h>) +// Note: When using glibc this may require -funwind-tables to function properly. +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_generic-inl.inc" +#endif +#elif defined(__i386__) || defined(__x86_64__) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_x86-inl.inc" +#elif defined(__ppc__) || defined(__PPC__) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_powerpc-inl.inc" +#elif defined(__aarch64__) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_aarch64-inl.inc" +#elif defined(__has_include) +#if __has_include(<execinfo.h>) +// Note: When using glibc this may require -funwind-tables to function properly. +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_generic-inl.inc" +#endif +#endif + +#endif + +// Fallback to the empty implementation. +#if !defined(ABSL_STACKTRACE_INL_HEADER) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_unimplemented-inl.inc" +#endif + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_CONFIG_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_generic-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_generic-inl.inc new file mode 100644 index 000000000000..b2792a1f3ada --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_generic-inl.inc @@ -0,0 +1,108 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Portable implementation - just use glibc +// +// Note: The glibc implementation may cause a call to malloc. +// This can cause a deadlock in HeapProfiler. + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_GENERIC_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_GENERIC_INL_H_ + +#include <execinfo.h> +#include <atomic> +#include <cstring> + +#include "absl/debugging/stacktrace.h" +#include "absl/base/attributes.h" + +// Sometimes, we can try to get a stack trace from within a stack +// trace, because we don't block signals inside this code (which would be too +// expensive: the two extra system calls per stack trace do matter here). +// That can cause a self-deadlock. +// Protect against such reentrant call by failing to get a stack trace. +// +// We use __thread here because the code here is extremely low level -- it is +// called while collecting stack traces from within malloc and mmap, and thus +// can not call anything which might call malloc or mmap itself. +static __thread int recursive = 0; + +// The stack trace function might be invoked very early in the program's +// execution (e.g. from the very first malloc if using tcmalloc). Also, the +// glibc implementation itself will trigger malloc the first time it is called. +// As such, we suppress usage of backtrace during this early stage of execution. +static std::atomic<bool> disable_stacktraces(true); // Disabled until healthy. +// Waiting until static initializers run seems to be late enough. +// This file is included into stacktrace.cc so this will only run once. +ABSL_ATTRIBUTE_UNUSED static int stacktraces_enabler = []() { + void* unused_stack[1]; + // Force the first backtrace to happen early to get the one-time shared lib + // loading (allocation) out of the way. After the first call it is much safer + // to use backtrace from a signal handler if we crash somewhere later. + backtrace(unused_stack, 1); + disable_stacktraces.store(false, std::memory_order_relaxed); + return 0; +}(); + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, + const void *ucp, int *min_dropped_frames) { + if (recursive || disable_stacktraces.load(std::memory_order_relaxed)) { + return 0; + } + ++recursive; + + static_cast<void>(ucp); // Unused. + static const int kStackLength = 64; + void * stack[kStackLength]; + int size; + + size = backtrace(stack, kStackLength); + skip_count++; // we want to skip the current frame as well + int result_count = size - skip_count; + if (result_count < 0) + result_count = 0; + if (result_count > max_depth) + result_count = max_depth; + for (int i = 0; i < result_count; i++) + result[i] = stack[i + skip_count]; + + if (IS_STACK_FRAMES) { + // No implementation for finding out the stack frame sizes yet. + memset(sizes, 0, sizeof(*sizes) * result_count); + } + if (min_dropped_frames != nullptr) { + if (size - skip_count - max_depth > 0) { + *min_dropped_frames = size - skip_count - max_depth; + } else { + *min_dropped_frames = 0; + } + } + + --recursive; + + return result_count; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return true; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_GENERIC_INL_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_powerpc-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_powerpc-inl.inc new file mode 100644 index 000000000000..2e7c2f404f20 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_powerpc-inl.inc @@ -0,0 +1,248 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Produce stack trace. I'm guessing (hoping!) the code is much like +// for x86. For apple machines, at least, it seems to be; see +// https://developer.apple.com/documentation/mac/runtimehtml/RTArch-59.html +// https://www.linux-foundation.org/spec/ELF/ppc64/PPC-elf64abi-1.9.html#STACK +// Linux has similar code: http://patchwork.ozlabs.org/linuxppc/patch?id=8882 + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_POWERPC_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_POWERPC_INL_H_ + +#if defined(__linux__) +#include <asm/ptrace.h> // for PT_NIP. +#include <ucontext.h> // for ucontext_t +#endif + +#include <unistd.h> +#include <cassert> +#include <cstdint> +#include <cstdio> + +#include "absl/base/attributes.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/internal/address_is_readable.h" +#include "absl/debugging/internal/vdso_support.h" // a no-op on non-elf or non-glibc systems + +// Given a stack pointer, return the saved link register value. +// Note that this is the link register for a callee. +static inline void *StacktracePowerPCGetLR(void **sp) { + // PowerPC has 3 main ABIs, which say where in the stack the + // Link Register is. For DARWIN and AIX (used by apple and + // linux ppc64), it's in sp[2]. For SYSV (used by linux ppc), + // it's in sp[1]. +#if defined(_CALL_AIX) || defined(_CALL_DARWIN) + return *(sp+2); +#elif defined(_CALL_SYSV) + return *(sp+1); +#elif defined(__APPLE__) || defined(__FreeBSD__) || \ + (defined(__linux__) && defined(__PPC64__)) + // This check is in case the compiler doesn't define _CALL_AIX/etc. + return *(sp+2); +#elif defined(__linux) + // This check is in case the compiler doesn't define _CALL_SYSV. + return *(sp+1); +#else +#error Need to specify the PPC ABI for your archiecture. +#endif +} + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return null if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template<bool STRICT_UNWINDING, bool IS_WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static void **NextStackFrame(void **old_sp, const void *uc) { + void **new_sp = (void **) *old_sp; + enum { kStackAlignment = 16 }; + + // Check that the transition from frame pointer old_sp to frame + // pointer new_sp isn't clearly bogus + if (STRICT_UNWINDING) { + // With the stack growing downwards, older stack frame must be + // at a greater address that the current one. + if (new_sp <= old_sp) return nullptr; + // Assume stack frames larger than 100,000 bytes are bogus. + if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return nullptr; + } else { + // In the non-strict mode, allow discontiguous stack frames. + // (alternate-signal-stacks for example). + if (new_sp == old_sp) return nullptr; + // And allow frames upto about 1MB. + if ((new_sp > old_sp) + && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return nullptr; + } + if ((uintptr_t)new_sp % kStackAlignment != 0) return nullptr; + +#if defined(__linux__) + enum StackTraceKernelSymbolStatus { + kNotInitialized = 0, kAddressValid, kAddressInvalid }; + + if (IS_WITH_CONTEXT && uc != nullptr) { + static StackTraceKernelSymbolStatus kernel_symbol_status = + kNotInitialized; // Sentinel: not computed yet. + // Initialize with sentinel value: __kernel_rt_sigtramp_rt64 can not + // possibly be there. + static const unsigned char *kernel_sigtramp_rt64_address = nullptr; + if (kernel_symbol_status == kNotInitialized) { + absl::debugging_internal::VDSOSupport vdso; + if (vdso.IsPresent()) { + absl::debugging_internal::VDSOSupport::SymbolInfo + sigtramp_rt64_symbol_info; + if (!vdso.LookupSymbol( + "__kernel_sigtramp_rt64", "LINUX_2.6.15", + absl::debugging_internal::VDSOSupport::kVDSOSymbolType, + &sigtramp_rt64_symbol_info) || + sigtramp_rt64_symbol_info.address == nullptr) { + // Unexpected: VDSO is present, yet the expected symbol is missing + // or null. + assert(false && "VDSO is present, but doesn't have expected symbol"); + kernel_symbol_status = kAddressInvalid; + } else { + kernel_sigtramp_rt64_address = + reinterpret_cast<const unsigned char *>( + sigtramp_rt64_symbol_info.address); + kernel_symbol_status = kAddressValid; + } + } else { + kernel_symbol_status = kAddressInvalid; + } + } + + if (new_sp != nullptr && + kernel_symbol_status == kAddressValid && + StacktracePowerPCGetLR(new_sp) == kernel_sigtramp_rt64_address) { + const ucontext_t* signal_context = + reinterpret_cast<const ucontext_t*>(uc); + void **const sp_before_signal = + reinterpret_cast<void**>(signal_context->uc_mcontext.gp_regs[PT_R1]); + // Check that alleged sp before signal is nonnull and is reasonably + // aligned. + if (sp_before_signal != nullptr && + ((uintptr_t)sp_before_signal % kStackAlignment) == 0) { + // Check that alleged stack pointer is actually readable. This is to + // prevent a "double fault" in case we hit the first fault due to e.g. + // a stack corruption. + if (absl::debugging_internal::AddressIsReadable(sp_before_signal)) { + // Alleged stack pointer is readable, use it for further unwinding. + new_sp = sp_before_signal; + } + } + } + } +#endif + + return new_sp; +} + +// This ensures that absl::GetStackTrace sets up the Link Register properly. +ABSL_ATTRIBUTE_NOINLINE static void AbslStacktracePowerPCDummyFunction() { + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); +} + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, + const void *ucp, int *min_dropped_frames) { + void **sp; + // Apple macOS uses an old version of gnu as -- both Darwin 7.9.0 (Panther) + // and Darwin 8.8.1 (Tiger) use as 1.38. This means we have to use a + // different asm syntax. I don't know quite the best way to discriminate + // systems using the old as from the new one; I've gone with __APPLE__. +#ifdef __APPLE__ + __asm__ volatile ("mr %0,r1" : "=r" (sp)); +#else + __asm__ volatile ("mr %0,1" : "=r" (sp)); +#endif + + // On PowerPC, the "Link Register" or "Link Record" (LR), is a stack + // entry that holds the return address of the subroutine call (what + // instruction we run after our function finishes). This is the + // same as the stack-pointer of our parent routine, which is what we + // want here. While the compiler will always(?) set up LR for + // subroutine calls, it may not for leaf functions (such as this one). + // This routine forces the compiler (at least gcc) to push it anyway. + AbslStacktracePowerPCDummyFunction(); + + // The LR save area is used by the callee, so the top entry is bogus. + skip_count++; + + int n = 0; + + // Unlike ABIs of X86 and ARM, PowerPC ABIs say that return address (in + // the link register) of a function call is stored in the caller's stack + // frame instead of the callee's. When we look for the return address + // associated with a stack frame, we need to make sure that there is a + // caller frame before it. So we call NextStackFrame before entering the + // loop below and check next_sp instead of sp for loop termination. + // The outermost frame is set up by runtimes and it does not have a + // caller frame, so it is skipped. + + // The absl::GetStackFrames routine is called when we are in some + // informational context (the failure signal handler for example). + // Use the non-strict unwinding rules to produce a stack trace + // that is as complete as possible (even if it contains a few + // bogus entries in some rare cases). + void **next_sp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(sp, ucp); + + while (next_sp && n < max_depth) { + if (skip_count > 0) { + skip_count--; + } else { + result[n] = StacktracePowerPCGetLR(sp); + if (IS_STACK_FRAMES) { + if (next_sp > sp) { + sizes[n] = (uintptr_t)next_sp - (uintptr_t)sp; + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } + } + n++; + } + + sp = next_sp; + next_sp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(sp, ucp); + } + + if (min_dropped_frames != nullptr) { + // Implementation detail: we clamp the max of frames we are willing to + // count, so as not to spend too much time in the loop below. + const int kMaxUnwind = 1000; + int j = 0; + for (; next_sp != nullptr && j < kMaxUnwind; j++) { + next_sp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(next_sp, ucp); + } + *min_dropped_frames = j; + } + return n; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return true; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_POWERPC_INL_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_unimplemented-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_unimplemented-inl.inc new file mode 100644 index 000000000000..5b8fb191b65a --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_unimplemented-inl.inc @@ -0,0 +1,24 @@ +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_UNIMPLEMENTED_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_UNIMPLEMENTED_INL_H_ + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +static int UnwindImpl(void** /* result */, int* /* sizes */, + int /* max_depth */, int /* skip_count */, + const void* /* ucp */, int *min_dropped_frames) { + if (min_dropped_frames != nullptr) { + *min_dropped_frames = 0; + } + return 0; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return false; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_UNIMPLEMENTED_INL_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_win32-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_win32-inl.inc new file mode 100644 index 000000000000..1c666c8b561f --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_win32-inl.inc @@ -0,0 +1,93 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Produces a stack trace for Windows. Normally, one could use +// stacktrace_x86-inl.h or stacktrace_x86_64-inl.h -- and indeed, that +// should work for binaries compiled using MSVC in "debug" mode. +// However, in "release" mode, Windows uses frame-pointer +// optimization, which makes getting a stack trace very difficult. +// +// There are several approaches one can take. One is to use Windows +// intrinsics like StackWalk64. These can work, but have restrictions +// on how successful they can be. Another attempt is to write a +// version of stacktrace_x86-inl.h that has heuristic support for +// dealing with FPO, similar to what WinDbg does (see +// http://www.nynaeve.net/?p=97). There are (non-working) examples of +// these approaches, complete with TODOs, in stacktrace_win32-inl.h#1 +// +// The solution we've ended up doing is to call the undocumented +// windows function RtlCaptureStackBackTrace, which probably doesn't +// work with FPO but at least is fast, and doesn't require a symbol +// server. +// +// This code is inspired by a patch from David Vitek: +// https://code.google.com/p/google-perftools/issues/detail?id=83 + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_WIN32_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_WIN32_INL_H_ + +#include <windows.h> // for GetProcAddress and GetModuleHandle +#include <cassert> + +typedef USHORT NTAPI RtlCaptureStackBackTrace_Function( + IN ULONG frames_to_skip, + IN ULONG frames_to_capture, + OUT PVOID *backtrace, + OUT PULONG backtrace_hash); + +// It is not possible to load RtlCaptureStackBackTrace at static init time in +// UWP. CaptureStackBackTrace is the public version of RtlCaptureStackBackTrace +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP) && \ + !WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +static RtlCaptureStackBackTrace_Function* const RtlCaptureStackBackTrace_fn = + &::CaptureStackBackTrace; +#else +// Load the function we need at static init time, where we don't have +// to worry about someone else holding the loader's lock. +static RtlCaptureStackBackTrace_Function* const RtlCaptureStackBackTrace_fn = + (RtlCaptureStackBackTrace_Function*)GetProcAddress( + GetModuleHandleA("ntdll.dll"), "RtlCaptureStackBackTrace"); +#endif // WINAPI_PARTITION_APP && !WINAPI_PARTITION_DESKTOP + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, + const void*, int* min_dropped_frames) { + int n = 0; + if (!RtlCaptureStackBackTrace_fn) { + // can't find a stacktrace with no function to call + } else { + n = (int)RtlCaptureStackBackTrace_fn(skip_count + 2, max_depth, result, 0); + } + if (IS_STACK_FRAMES) { + // No implementation for finding out the stack frame sizes yet. + memset(sizes, 0, sizeof(*sizes) * n); + } + if (min_dropped_frames != nullptr) { + // Not implemented. + *min_dropped_frames = 0; + } + return n; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return false; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_WIN32_INL_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/stacktrace_x86-inl.inc b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_x86-inl.inc new file mode 100644 index 000000000000..bc320ff75bc5 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/stacktrace_x86-inl.inc @@ -0,0 +1,346 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Produce stack trace + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_X86_INL_INC_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_X86_INL_INC_ + +#if defined(__linux__) && (defined(__i386__) || defined(__x86_64__)) +#include <ucontext.h> // for ucontext_t +#endif + +#if !defined(_WIN32) +#include <unistd.h> +#endif + +#include <cassert> +#include <cstdint> + +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/debugging/internal/address_is_readable.h" +#include "absl/debugging/internal/vdso_support.h" // a no-op on non-elf or non-glibc systems +#include "absl/debugging/stacktrace.h" + +#include "absl/base/internal/raw_logging.h" + +using absl::debugging_internal::AddressIsReadable; + +#if defined(__linux__) && defined(__i386__) +// Count "push %reg" instructions in VDSO __kernel_vsyscall(), +// preceeding "syscall" or "sysenter". +// If __kernel_vsyscall uses frame pointer, answer 0. +// +// kMaxBytes tells how many instruction bytes of __kernel_vsyscall +// to analyze before giving up. Up to kMaxBytes+1 bytes of +// instructions could be accessed. +// +// Here are known __kernel_vsyscall instruction sequences: +// +// SYSENTER (linux-2.6.26/arch/x86/vdso/vdso32/sysenter.S). +// Used on Intel. +// 0xffffe400 <__kernel_vsyscall+0>: push %ecx +// 0xffffe401 <__kernel_vsyscall+1>: push %edx +// 0xffffe402 <__kernel_vsyscall+2>: push %ebp +// 0xffffe403 <__kernel_vsyscall+3>: mov %esp,%ebp +// 0xffffe405 <__kernel_vsyscall+5>: sysenter +// +// SYSCALL (see linux-2.6.26/arch/x86/vdso/vdso32/syscall.S). +// Used on AMD. +// 0xffffe400 <__kernel_vsyscall+0>: push %ebp +// 0xffffe401 <__kernel_vsyscall+1>: mov %ecx,%ebp +// 0xffffe403 <__kernel_vsyscall+3>: syscall +// + +// The sequence below isn't actually expected in Google fleet, +// here only for completeness. Remove this comment from OSS release. + +// i386 (see linux-2.6.26/arch/x86/vdso/vdso32/int80.S) +// 0xffffe400 <__kernel_vsyscall+0>: int $0x80 +// 0xffffe401 <__kernel_vsyscall+1>: ret +// +static const int kMaxBytes = 10; + +// We use assert()s instead of DCHECK()s -- this is too low level +// for DCHECK(). + +static int CountPushInstructions(const unsigned char *const addr) { + int result = 0; + for (int i = 0; i < kMaxBytes; ++i) { + if (addr[i] == 0x89) { + // "mov reg,reg" + if (addr[i + 1] == 0xE5) { + // Found "mov %esp,%ebp". + return 0; + } + ++i; // Skip register encoding byte. + } else if (addr[i] == 0x0F && + (addr[i + 1] == 0x34 || addr[i + 1] == 0x05)) { + // Found "sysenter" or "syscall". + return result; + } else if ((addr[i] & 0xF0) == 0x50) { + // Found "push %reg". + ++result; + } else if (addr[i] == 0xCD && addr[i + 1] == 0x80) { + // Found "int $0x80" + assert(result == 0); + return 0; + } else { + // Unexpected instruction. + assert(false && "unexpected instruction in __kernel_vsyscall"); + return 0; + } + } + // Unexpected: didn't find SYSENTER or SYSCALL in + // [__kernel_vsyscall, __kernel_vsyscall + kMaxBytes) interval. + assert(false && "did not find SYSENTER or SYSCALL in __kernel_vsyscall"); + return 0; +} +#endif + +// Assume stack frames larger than 100,000 bytes are bogus. +static const int kMaxFrameBytes = 100000; + +// Returns the stack frame pointer from signal context, 0 if unknown. +// vuc is a ucontext_t *. We use void* to avoid the use +// of ucontext_t on non-POSIX systems. +static uintptr_t GetFP(const void *vuc) { +#if !defined(__linux__) + static_cast<void>(vuc); // Avoid an unused argument compiler warning. +#else + if (vuc != nullptr) { + auto *uc = reinterpret_cast<const ucontext_t *>(vuc); +#if defined(__i386__) + const auto bp = uc->uc_mcontext.gregs[REG_EBP]; + const auto sp = uc->uc_mcontext.gregs[REG_ESP]; +#elif defined(__x86_64__) + const auto bp = uc->uc_mcontext.gregs[REG_RBP]; + const auto sp = uc->uc_mcontext.gregs[REG_RSP]; +#else + const uintptr_t bp = 0; + const uintptr_t sp = 0; +#endif + // Sanity-check that the base pointer is valid. It should be as long as + // SHRINK_WRAP_FRAME_POINTER is not set, but it's possible that some code in + // the process is compiled with --copt=-fomit-frame-pointer or + // --copt=-momit-leaf-frame-pointer. + // + // TODO(bcmills): -momit-leaf-frame-pointer is currently the default + // behavior when building with clang. Talk to the C++ toolchain team about + // fixing that. + if (bp >= sp && bp - sp <= kMaxFrameBytes) return bp; + + // If bp isn't a plausible frame pointer, return the stack pointer instead. + // If we're lucky, it points to the start of a stack frame; otherwise, we'll + // get one frame of garbage in the stack trace and fail the sanity check on + // the next iteration. + return sp; + } +#endif + return 0; +} + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return null if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template <bool STRICT_UNWINDING, bool WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static void **NextStackFrame(void **old_fp, const void *uc) { + void **new_fp = (void **)*old_fp; + +#if defined(__linux__) && defined(__i386__) + if (WITH_CONTEXT && uc != nullptr) { + // How many "push %reg" instructions are there at __kernel_vsyscall? + // This is constant for a given kernel and processor, so compute + // it only once. + static int num_push_instructions = -1; // Sentinel: not computed yet. + // Initialize with sentinel value: __kernel_rt_sigreturn can not possibly + // be there. + static const unsigned char *kernel_rt_sigreturn_address = nullptr; + static const unsigned char *kernel_vsyscall_address = nullptr; + if (num_push_instructions == -1) { +#ifdef ABSL_HAVE_VDSO_SUPPORT + absl::debugging_internal::VDSOSupport vdso; + if (vdso.IsPresent()) { + absl::debugging_internal::VDSOSupport::SymbolInfo + rt_sigreturn_symbol_info; + absl::debugging_internal::VDSOSupport::SymbolInfo vsyscall_symbol_info; + if (!vdso.LookupSymbol("__kernel_rt_sigreturn", "LINUX_2.5", STT_FUNC, + &rt_sigreturn_symbol_info) || + !vdso.LookupSymbol("__kernel_vsyscall", "LINUX_2.5", STT_FUNC, + &vsyscall_symbol_info) || + rt_sigreturn_symbol_info.address == nullptr || + vsyscall_symbol_info.address == nullptr) { + // Unexpected: 32-bit VDSO is present, yet one of the expected + // symbols is missing or null. + assert(false && "VDSO is present, but doesn't have expected symbols"); + num_push_instructions = 0; + } else { + kernel_rt_sigreturn_address = + reinterpret_cast<const unsigned char *>( + rt_sigreturn_symbol_info.address); + kernel_vsyscall_address = + reinterpret_cast<const unsigned char *>( + vsyscall_symbol_info.address); + num_push_instructions = + CountPushInstructions(kernel_vsyscall_address); + } + } else { + num_push_instructions = 0; + } +#else // ABSL_HAVE_VDSO_SUPPORT + num_push_instructions = 0; +#endif // ABSL_HAVE_VDSO_SUPPORT + } + if (num_push_instructions != 0 && kernel_rt_sigreturn_address != nullptr && + old_fp[1] == kernel_rt_sigreturn_address) { + const ucontext_t *ucv = static_cast<const ucontext_t *>(uc); + // This kernel does not use frame pointer in its VDSO code, + // and so %ebp is not suitable for unwinding. + void **const reg_ebp = + reinterpret_cast<void **>(ucv->uc_mcontext.gregs[REG_EBP]); + const unsigned char *const reg_eip = + reinterpret_cast<unsigned char *>(ucv->uc_mcontext.gregs[REG_EIP]); + if (new_fp == reg_ebp && kernel_vsyscall_address <= reg_eip && + reg_eip - kernel_vsyscall_address < kMaxBytes) { + // We "stepped up" to __kernel_vsyscall, but %ebp is not usable. + // Restore from 'ucv' instead. + void **const reg_esp = + reinterpret_cast<void **>(ucv->uc_mcontext.gregs[REG_ESP]); + // Check that alleged %esp is not null and is reasonably aligned. + if (reg_esp && + ((uintptr_t)reg_esp & (sizeof(reg_esp) - 1)) == 0) { + // Check that alleged %esp is actually readable. This is to prevent + // "double fault" in case we hit the first fault due to e.g. stack + // corruption. + void *const reg_esp2 = reg_esp[num_push_instructions - 1]; + if (AddressIsReadable(reg_esp2)) { + // Alleged %esp is readable, use it for further unwinding. + new_fp = reinterpret_cast<void **>(reg_esp2); + } + } + } + } + } +#endif + + const uintptr_t old_fp_u = reinterpret_cast<uintptr_t>(old_fp); + const uintptr_t new_fp_u = reinterpret_cast<uintptr_t>(new_fp); + + // Check that the transition from frame pointer old_fp to frame + // pointer new_fp isn't clearly bogus. Skip the checks if new_fp + // matches the signal context, so that we don't skip out early when + // using an alternate signal stack. + // + // TODO(bcmills): The GetFP call should be completely unnecessary when + // SHRINK_WRAP_FRAME_POINTER is set (because we should be back in the thread's + // stack by this point), but it is empirically still needed (e.g. when the + // stack includes a call to abort). unw_get_reg returns UNW_EBADREG for some + // frames. Figure out why GetValidFrameAddr and/or libunwind isn't doing what + // it's supposed to. + if (STRICT_UNWINDING && + (!WITH_CONTEXT || uc == nullptr || new_fp_u != GetFP(uc))) { + // With the stack growing downwards, older stack frame must be + // at a greater address that the current one. + if (new_fp_u <= old_fp_u) return nullptr; + if (new_fp_u - old_fp_u > kMaxFrameBytes) return nullptr; + } else { + if (new_fp == nullptr) return nullptr; // skip AddressIsReadable() below + // In the non-strict mode, allow discontiguous stack frames. + // (alternate-signal-stacks for example). + if (new_fp == old_fp) return nullptr; + } + + if (new_fp_u & (sizeof(void *) - 1)) return nullptr; +#ifdef __i386__ + // On 32-bit machines, the stack pointer can be very close to + // 0xffffffff, so we explicitly check for a pointer into the + // last two pages in the address space + if (new_fp_u >= 0xffffe000) return nullptr; +#endif +#if !defined(_WIN32) + if (!STRICT_UNWINDING) { + // Lax sanity checks cause a crash in 32-bit tcmalloc/crash_reason_test + // on AMD-based machines with VDSO-enabled kernels. + // Make an extra sanity check to insure new_fp is readable. + // Note: NextStackFrame<false>() is only called while the program + // is already on its last leg, so it's ok to be slow here. + + if (!AddressIsReadable(new_fp)) { + return nullptr; + } + } +#endif + return new_fp; +} + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +ABSL_ATTRIBUTE_NOINLINE +static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, + const void *ucp, int *min_dropped_frames) { + int n = 0; + void **fp = reinterpret_cast<void **>(__builtin_frame_address(0)); + + while (fp && n < max_depth) { + if (*(fp + 1) == reinterpret_cast<void *>(0)) { + // In 64-bit code, we often see a frame that + // points to itself and has a return address of 0. + break; + } + void **next_fp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(fp, ucp); + if (skip_count > 0) { + skip_count--; + } else { + result[n] = *(fp + 1); + if (IS_STACK_FRAMES) { + if (next_fp > fp) { + sizes[n] = (uintptr_t)next_fp - (uintptr_t)fp; + } else { + // A frame-size of 0 is used to indicate unknown frame size. + sizes[n] = 0; + } + } + n++; + } + fp = next_fp; + } + if (min_dropped_frames != nullptr) { + // Implementation detail: we clamp the max of frames we are willing to + // count, so as not to spend too much time in the loop below. + const int kMaxUnwind = 1000; + int j = 0; + for (; fp != nullptr && j < kMaxUnwind; j++) { + fp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(fp, ucp); + } + *min_dropped_frames = j; + } + return n; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { + return true; +} +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_X86_INL_INC_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/symbolize.h b/third_party/abseil_cpp/absl/debugging/internal/symbolize.h new file mode 100644 index 000000000000..4f26130fbb71 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/symbolize.h @@ -0,0 +1,147 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains internal parts of the Abseil symbolizer. +// Do not depend on the anything in this file, it may change at anytime. + +#ifndef ABSL_DEBUGGING_INTERNAL_SYMBOLIZE_H_ +#define ABSL_DEBUGGING_INTERNAL_SYMBOLIZE_H_ + +#ifdef __cplusplus + +#include <cstddef> +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +#ifdef ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE +#error ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE cannot be directly set +#elif defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) && \ + !defined(__asmjs__) && !defined(__wasm__) +#define ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE 1 + +#include <elf.h> +#include <link.h> // For ElfW() macro. +#include <functional> +#include <string> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// Iterates over all sections, invoking callback on each with the section name +// and the section header. +// +// Returns true on success; otherwise returns false in case of errors. +// +// This is not async-signal-safe. +bool ForEachSection(int fd, + const std::function<bool(absl::string_view name, + const ElfW(Shdr) &)>& callback); + +// Gets the section header for the given name, if it exists. Returns true on +// success. Otherwise, returns false. +bool GetSectionHeaderByName(int fd, const char *name, size_t name_len, + ElfW(Shdr) *out); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE + +#ifdef ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE +#error ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE cannot be directly set +#elif defined(__APPLE__) +#define ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE 1 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +struct SymbolDecoratorArgs { + // The program counter we are getting symbolic name for. + const void *pc; + // 0 for main executable, load address for shared libraries. + ptrdiff_t relocation; + // Read-only file descriptor for ELF image covering "pc", + // or -1 if no such ELF image exists in /proc/self/maps. + int fd; + // Output buffer, size. + // Note: the buffer may not be empty -- default symbolizer may have already + // produced some output, and earlier decorators may have adorned it in + // some way. You are free to replace or augment the contents (within the + // symbol_buf_size limit). + char *const symbol_buf; + size_t symbol_buf_size; + // Temporary scratch space, size. + // Use that space in preference to allocating your own stack buffer to + // conserve stack. + char *const tmp_buf; + size_t tmp_buf_size; + // User-provided argument + void* arg; +}; +using SymbolDecorator = void (*)(const SymbolDecoratorArgs *); + +// Installs a function-pointer as a decorator. Returns a value less than zero +// if the system cannot install the decorator. Otherwise, returns a unique +// identifier corresponding to the decorator. This identifier can be used to +// uninstall the decorator - See RemoveSymbolDecorator() below. +int InstallSymbolDecorator(SymbolDecorator decorator, void* arg); + +// Removes a previously installed function-pointer decorator. Parameter "ticket" +// is the return-value from calling InstallSymbolDecorator(). +bool RemoveSymbolDecorator(int ticket); + +// Remove all installed decorators. Returns true if successful, false if +// symbolization is currently in progress. +bool RemoveAllSymbolDecorators(void); + +// Registers an address range to a file mapping. +// +// Preconditions: +// start <= end +// filename != nullptr +// +// Returns true if the file was successfully registered. +bool RegisterFileMappingHint(const void* start, const void* end, + uint64_t offset, const char* filename); + +// Looks up the file mapping registered by RegisterFileMappingHint for an +// address range. If there is one, the file name is stored in *filename and +// *start and *end are modified to reflect the registered mapping. Returns +// whether any hint was found. +bool GetFileMappingHint(const void** start, const void** end, uint64_t* offset, + const char** filename); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // __cplusplus + +#include <stdbool.h> + +#ifdef __cplusplus +extern "C" +#endif // __cplusplus + + bool + AbslInternalGetFileMappingHint(const void** start, const void** end, + uint64_t* offset, const char** filename); + +#endif // ABSL_DEBUGGING_INTERNAL_SYMBOLIZE_H_ diff --git a/third_party/abseil_cpp/absl/debugging/internal/vdso_support.cc b/third_party/abseil_cpp/absl/debugging/internal/vdso_support.cc new file mode 100644 index 000000000000..6be16d907275 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/vdso_support.cc @@ -0,0 +1,173 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Allow dynamic symbol lookup in the kernel VDSO page. +// +// VDSOSupport -- a class representing kernel VDSO (if present). + +#include "absl/debugging/internal/vdso_support.h" + +#ifdef ABSL_HAVE_VDSO_SUPPORT // defined in vdso_support.h + +#include <errno.h> +#include <fcntl.h> +#include <sys/syscall.h> +#include <unistd.h> + +#if __GLIBC_PREREQ(2, 16) // GLIBC-2.16 implements getauxval. +#include <sys/auxv.h> +#endif + +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/port.h" + +#ifndef AT_SYSINFO_EHDR +#define AT_SYSINFO_EHDR 33 // for crosstoolv10 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +ABSL_CONST_INIT +std::atomic<const void *> VDSOSupport::vdso_base_( + debugging_internal::ElfMemImage::kInvalidBase); + +std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_(&InitAndGetCPU); +VDSOSupport::VDSOSupport() + // If vdso_base_ is still set to kInvalidBase, we got here + // before VDSOSupport::Init has been called. Call it now. + : image_(vdso_base_.load(std::memory_order_relaxed) == + debugging_internal::ElfMemImage::kInvalidBase + ? Init() + : vdso_base_.load(std::memory_order_relaxed)) {} + +// NOTE: we can't use GoogleOnceInit() below, because we can be +// called by tcmalloc, and none of the *once* stuff may be functional yet. +// +// In addition, we hope that the VDSOSupportHelper constructor +// causes this code to run before there are any threads, and before +// InitGoogle() has executed any chroot or setuid calls. +// +// Finally, even if there is a race here, it is harmless, because +// the operation should be idempotent. +const void *VDSOSupport::Init() { + const auto kInvalidBase = debugging_internal::ElfMemImage::kInvalidBase; +#if __GLIBC_PREREQ(2, 16) + if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { + errno = 0; + const void *const sysinfo_ehdr = + reinterpret_cast<const void *>(getauxval(AT_SYSINFO_EHDR)); + if (errno == 0) { + vdso_base_.store(sysinfo_ehdr, std::memory_order_relaxed); + } + } +#endif // __GLIBC_PREREQ(2, 16) + if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { + int fd = open("/proc/self/auxv", O_RDONLY); + if (fd == -1) { + // Kernel too old to have a VDSO. + vdso_base_.store(nullptr, std::memory_order_relaxed); + getcpu_fn_.store(&GetCPUViaSyscall, std::memory_order_relaxed); + return nullptr; + } + ElfW(auxv_t) aux; + while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) { + if (aux.a_type == AT_SYSINFO_EHDR) { + vdso_base_.store(reinterpret_cast<void *>(aux.a_un.a_val), + std::memory_order_relaxed); + break; + } + } + close(fd); + if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { + // Didn't find AT_SYSINFO_EHDR in auxv[]. + vdso_base_.store(nullptr, std::memory_order_relaxed); + } + } + GetCpuFn fn = &GetCPUViaSyscall; // default if VDSO not present. + if (vdso_base_.load(std::memory_order_relaxed)) { + VDSOSupport vdso; + SymbolInfo info; + if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) { + fn = reinterpret_cast<GetCpuFn>(const_cast<void *>(info.address)); + } + } + // Subtle: this code runs outside of any locks; prevent compiler + // from assigning to getcpu_fn_ more than once. + getcpu_fn_.store(fn, std::memory_order_relaxed); + return vdso_base_.load(std::memory_order_relaxed); +} + +const void *VDSOSupport::SetBase(const void *base) { + ABSL_RAW_CHECK(base != debugging_internal::ElfMemImage::kInvalidBase, + "internal error"); + const void *old_base = vdso_base_.load(std::memory_order_relaxed); + vdso_base_.store(base, std::memory_order_relaxed); + image_.Init(base); + // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO. + getcpu_fn_.store(&InitAndGetCPU, std::memory_order_relaxed); + return old_base; +} + +bool VDSOSupport::LookupSymbol(const char *name, + const char *version, + int type, + SymbolInfo *info) const { + return image_.LookupSymbol(name, version, type, info); +} + +bool VDSOSupport::LookupSymbolByAddress(const void *address, + SymbolInfo *info_out) const { + return image_.LookupSymbolByAddress(address, info_out); +} + +// NOLINT on 'long' because this routine mimics kernel api. +long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, // NOLINT(runtime/int) + void *, void *) { +#ifdef SYS_getcpu + return syscall(SYS_getcpu, cpu, nullptr, nullptr); +#else + // x86_64 never implemented sys_getcpu(), except as a VDSO call. + static_cast<void>(cpu); // Avoid an unused argument compiler warning. + errno = ENOSYS; + return -1; +#endif +} + +// Use fast __vdso_getcpu if available. +long VDSOSupport::InitAndGetCPU(unsigned *cpu, // NOLINT(runtime/int) + void *x, void *y) { + Init(); + GetCpuFn fn = getcpu_fn_.load(std::memory_order_relaxed); + ABSL_RAW_CHECK(fn != &InitAndGetCPU, "Init() did not set getcpu_fn_"); + return (*fn)(cpu, x, y); +} + +// This function must be very fast, and may be called from very +// low level (e.g. tcmalloc). Hence I avoid things like +// GoogleOnceInit() and ::operator new. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY +int GetCPU() { + unsigned cpu; + int ret_code = (*VDSOSupport::getcpu_fn_)(&cpu, nullptr, nullptr); + return ret_code == 0 ? cpu : ret_code; +} + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HAVE_VDSO_SUPPORT diff --git a/third_party/abseil_cpp/absl/debugging/internal/vdso_support.h b/third_party/abseil_cpp/absl/debugging/internal/vdso_support.h new file mode 100644 index 000000000000..6562c6c2350a --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/internal/vdso_support.h @@ -0,0 +1,158 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Allow dynamic symbol lookup in the kernel VDSO page. +// +// VDSO stands for "Virtual Dynamic Shared Object" -- a page of +// executable code, which looks like a shared library, but doesn't +// necessarily exist anywhere on disk, and which gets mmap()ed into +// every process by kernels which support VDSO, such as 2.6.x for 32-bit +// executables, and 2.6.24 and above for 64-bit executables. +// +// More details could be found here: +// http://www.trilithium.com/johan/2005/08/linux-gate/ +// +// VDSOSupport -- a class representing kernel VDSO (if present). +// +// Example usage: +// VDSOSupport vdso; +// VDSOSupport::SymbolInfo info; +// typedef (*FN)(unsigned *, void *, void *); +// FN fn = nullptr; +// if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) { +// fn = reinterpret_cast<FN>(info.address); +// } + +#ifndef ABSL_DEBUGGING_INTERNAL_VDSO_SUPPORT_H_ +#define ABSL_DEBUGGING_INTERNAL_VDSO_SUPPORT_H_ + +#include <atomic> + +#include "absl/base/attributes.h" +#include "absl/debugging/internal/elf_mem_image.h" + +#ifdef ABSL_HAVE_ELF_MEM_IMAGE + +#ifdef ABSL_HAVE_VDSO_SUPPORT +#error ABSL_HAVE_VDSO_SUPPORT cannot be directly set +#else +#define ABSL_HAVE_VDSO_SUPPORT 1 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { + +// NOTE: this class may be used from within tcmalloc, and can not +// use any memory allocation routines. +class VDSOSupport { + public: + VDSOSupport(); + + typedef ElfMemImage::SymbolInfo SymbolInfo; + typedef ElfMemImage::SymbolIterator SymbolIterator; + + // On PowerPC64 VDSO symbols can either be of type STT_FUNC or STT_NOTYPE + // depending on how the kernel is built. The kernel is normally built with + // STT_NOTYPE type VDSO symbols. Let's make things simpler first by using a + // compile-time constant. +#ifdef __powerpc64__ + enum { kVDSOSymbolType = STT_NOTYPE }; +#else + enum { kVDSOSymbolType = STT_FUNC }; +#endif + + // Answers whether we have a vdso at all. + bool IsPresent() const { return image_.IsPresent(); } + + // Allow to iterate over all VDSO symbols. + SymbolIterator begin() const { return image_.begin(); } + SymbolIterator end() const { return image_.end(); } + + // Look up versioned dynamic symbol in the kernel VDSO. + // Returns false if VDSO is not present, or doesn't contain given + // symbol/version/type combination. + // If info_out != nullptr, additional details are filled in. + bool LookupSymbol(const char *name, const char *version, + int symbol_type, SymbolInfo *info_out) const; + + // Find info about symbol (if any) which overlaps given address. + // Returns true if symbol was found; false if VDSO isn't present + // or doesn't have a symbol overlapping given address. + // If info_out != nullptr, additional details are filled in. + bool LookupSymbolByAddress(const void *address, SymbolInfo *info_out) const; + + // Used only for testing. Replace real VDSO base with a mock. + // Returns previous value of vdso_base_. After you are done testing, + // you are expected to call SetBase() with previous value, in order to + // reset state to the way it was. + const void *SetBase(const void *s); + + // Computes vdso_base_ and returns it. Should be called as early as + // possible; before any thread creation, chroot or setuid. + static const void *Init(); + + private: + // image_ represents VDSO ELF image in memory. + // image_.ehdr_ == nullptr implies there is no VDSO. + ElfMemImage image_; + + // Cached value of auxv AT_SYSINFO_EHDR, computed once. + // This is a tri-state: + // kInvalidBase => value hasn't been determined yet. + // 0 => there is no VDSO. + // else => vma of VDSO Elf{32,64}_Ehdr. + // + // When testing with mock VDSO, low bit is set. + // The low bit is always available because vdso_base_ is + // page-aligned. + static std::atomic<const void *> vdso_base_; + + // NOLINT on 'long' because these routines mimic kernel api. + // The 'cache' parameter may be used by some versions of the kernel, + // and should be nullptr or point to a static buffer containing at + // least two 'long's. + static long InitAndGetCPU(unsigned *cpu, void *cache, // NOLINT 'long'. + void *unused); + static long GetCPUViaSyscall(unsigned *cpu, void *cache, // NOLINT 'long'. + void *unused); + typedef long (*GetCpuFn)(unsigned *cpu, void *cache, // NOLINT 'long'. + void *unused); + + // This function pointer may point to InitAndGetCPU, + // GetCPUViaSyscall, or __vdso_getcpu at different stages of initialization. + ABSL_CONST_INIT static std::atomic<GetCpuFn> getcpu_fn_; + + friend int GetCPU(void); // Needs access to getcpu_fn_. + + VDSOSupport(const VDSOSupport&) = delete; + VDSOSupport& operator=(const VDSOSupport&) = delete; +}; + +// Same as sched_getcpu() on later glibc versions. +// Return current CPU, using (fast) __vdso_getcpu@LINUX_2.6 if present, +// otherwise use syscall(SYS_getcpu,...). +// May return -1 with errno == ENOSYS if the kernel doesn't +// support SYS_getcpu. +int GetCPU(); + +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HAVE_ELF_MEM_IMAGE + +#endif // ABSL_DEBUGGING_INTERNAL_VDSO_SUPPORT_H_ diff --git a/third_party/abseil_cpp/absl/debugging/leak_check.cc b/third_party/abseil_cpp/absl/debugging/leak_check.cc new file mode 100644 index 000000000000..ff9049559d4d --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/leak_check.cc @@ -0,0 +1,53 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Wrappers around lsan_interface functions. +// When lsan is not linked in, these functions are not available, +// therefore Abseil code which depends on these functions is conditioned on the +// definition of LEAK_SANITIZER. +#include "absl/debugging/leak_check.h" + +#ifndef LEAK_SANITIZER + +namespace absl { +ABSL_NAMESPACE_BEGIN +bool HaveLeakSanitizer() { return false; } +void DoIgnoreLeak(const void*) { } +void RegisterLivePointers(const void*, size_t) { } +void UnRegisterLivePointers(const void*, size_t) { } +LeakCheckDisabler::LeakCheckDisabler() { } +LeakCheckDisabler::~LeakCheckDisabler() { } +ABSL_NAMESPACE_END +} // namespace absl + +#else + +#include <sanitizer/lsan_interface.h> + +namespace absl { +ABSL_NAMESPACE_BEGIN +bool HaveLeakSanitizer() { return true; } +void DoIgnoreLeak(const void* ptr) { __lsan_ignore_object(ptr); } +void RegisterLivePointers(const void* ptr, size_t size) { + __lsan_register_root_region(ptr, size); +} +void UnRegisterLivePointers(const void* ptr, size_t size) { + __lsan_unregister_root_region(ptr, size); +} +LeakCheckDisabler::LeakCheckDisabler() { __lsan_disable(); } +LeakCheckDisabler::~LeakCheckDisabler() { __lsan_enable(); } +ABSL_NAMESPACE_END +} // namespace absl + +#endif // LEAK_SANITIZER diff --git a/third_party/abseil_cpp/absl/debugging/leak_check.h b/third_party/abseil_cpp/absl/debugging/leak_check.h new file mode 100644 index 000000000000..7a5a22dd1caf --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/leak_check.h @@ -0,0 +1,113 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: leak_check.h +// ----------------------------------------------------------------------------- +// +// This file contains functions that affect leak checking behavior within +// targets built with the LeakSanitizer (LSan), a memory leak detector that is +// integrated within the AddressSanitizer (ASan) as an additional component, or +// which can be used standalone. LSan and ASan are included (or can be provided) +// as additional components for most compilers such as Clang, gcc and MSVC. +// Note: this leak checking API is not yet supported in MSVC. +// Leak checking is enabled by default in all ASan builds. +// +// See https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer +// +// ----------------------------------------------------------------------------- +#ifndef ABSL_DEBUGGING_LEAK_CHECK_H_ +#define ABSL_DEBUGGING_LEAK_CHECK_H_ + +#include <cstddef> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// HaveLeakSanitizer() +// +// Returns true if a leak-checking sanitizer (either ASan or standalone LSan) is +// currently built into this target. +bool HaveLeakSanitizer(); + +// DoIgnoreLeak() +// +// Implements `IgnoreLeak()` below. This function should usually +// not be called directly; calling `IgnoreLeak()` is preferred. +void DoIgnoreLeak(const void* ptr); + +// IgnoreLeak() +// +// Instruct the leak sanitizer to ignore leak warnings on the object referenced +// by the passed pointer, as well as all heap objects transitively referenced +// by it. The passed object pointer can point to either the beginning of the +// object or anywhere within it. +// +// Example: +// +// static T* obj = IgnoreLeak(new T(...)); +// +// If the passed `ptr` does not point to an actively allocated object at the +// time `IgnoreLeak()` is called, the call is a no-op; if it is actively +// allocated, the object must not get deallocated later. +// +template <typename T> +T* IgnoreLeak(T* ptr) { + DoIgnoreLeak(ptr); + return ptr; +} + +// LeakCheckDisabler +// +// This helper class indicates that any heap allocations done in the code block +// covered by the scoped object, which should be allocated on the stack, will +// not be reported as leaks. Leak check disabling will occur within the code +// block and any nested function calls within the code block. +// +// Example: +// +// void Foo() { +// LeakCheckDisabler disabler; +// ... code that allocates objects whose leaks should be ignored ... +// } +// +// REQUIRES: Destructor runs in same thread as constructor +class LeakCheckDisabler { + public: + LeakCheckDisabler(); + LeakCheckDisabler(const LeakCheckDisabler&) = delete; + LeakCheckDisabler& operator=(const LeakCheckDisabler&) = delete; + ~LeakCheckDisabler(); +}; + +// RegisterLivePointers() +// +// Registers `ptr[0,size-1]` as pointers to memory that is still actively being +// referenced and for which leak checking should be ignored. This function is +// useful if you store pointers in mapped memory, for memory ranges that we know +// are correct but for which normal analysis would flag as leaked code. +void RegisterLivePointers(const void* ptr, size_t size); + +// UnRegisterLivePointers() +// +// Deregisters the pointers previously marked as active in +// `RegisterLivePointers()`, enabling leak checking of those pointers. +void UnRegisterLivePointers(const void* ptr, size_t size); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_LEAK_CHECK_H_ diff --git a/third_party/abseil_cpp/absl/debugging/leak_check_disable.cc b/third_party/abseil_cpp/absl/debugging/leak_check_disable.cc new file mode 100644 index 000000000000..924d6e3d543b --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/leak_check_disable.cc @@ -0,0 +1,20 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Disable LeakSanitizer when this file is linked in. +// This function overrides __lsan_is_turned_off from sanitizer/lsan_interface.h +extern "C" int __lsan_is_turned_off(); +extern "C" int __lsan_is_turned_off() { + return 1; +} diff --git a/third_party/abseil_cpp/absl/debugging/leak_check_fail_test.cc b/third_party/abseil_cpp/absl/debugging/leak_check_fail_test.cc new file mode 100644 index 000000000000..c49b81a9d9e2 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/leak_check_fail_test.cc @@ -0,0 +1,41 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <memory> +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/leak_check.h" + +namespace { + +TEST(LeakCheckTest, LeakMemory) { + // This test is expected to cause lsan failures on program exit. Therefore the + // test will be run only by leak_check_test.sh, which will verify a + // failed exit code. + + char* foo = strdup("lsan should complain about this leaked string"); + ABSL_RAW_LOG(INFO, "Should detect leaked string %s", foo); +} + +TEST(LeakCheckTest, LeakMemoryAfterDisablerScope) { + // This test is expected to cause lsan failures on program exit. Therefore the + // test will be run only by external_leak_check_test.sh, which will verify a + // failed exit code. + { absl::LeakCheckDisabler disabler; } + char* foo = strdup("lsan should also complain about this leaked string"); + ABSL_RAW_LOG(INFO, "Re-enabled leak detection.Should detect leaked string %s", + foo); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/debugging/leak_check_test.cc b/third_party/abseil_cpp/absl/debugging/leak_check_test.cc new file mode 100644 index 000000000000..b5cc4874880f --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/leak_check_test.cc @@ -0,0 +1,42 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> + +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/leak_check.h" + +namespace { + +TEST(LeakCheckTest, DetectLeakSanitizer) { +#ifdef ABSL_EXPECT_LEAK_SANITIZER + EXPECT_TRUE(absl::HaveLeakSanitizer()); +#else + EXPECT_FALSE(absl::HaveLeakSanitizer()); +#endif +} + +TEST(LeakCheckTest, IgnoreLeakSuppressesLeakedMemoryErrors) { + auto foo = absl::IgnoreLeak(new std::string("some ignored leaked string")); + ABSL_RAW_LOG(INFO, "Ignoring leaked string %s", foo->c_str()); +} + +TEST(LeakCheckTest, LeakCheckDisablerIgnoresLeak) { + absl::LeakCheckDisabler disabler; + auto foo = new std::string("some string leaked while checks are disabled"); + ABSL_RAW_LOG(INFO, "Ignoring leaked string %s", foo->c_str()); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/debugging/stacktrace.cc b/third_party/abseil_cpp/absl/debugging/stacktrace.cc new file mode 100644 index 000000000000..1f7c7d82b2d2 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/stacktrace.cc @@ -0,0 +1,140 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Produce stack trace. +// +// There are three different ways we can try to get the stack trace: +// +// 1) Our hand-coded stack-unwinder. This depends on a certain stack +// layout, which is used by gcc (and those systems using a +// gcc-compatible ABI) on x86 systems, at least since gcc 2.95. +// It uses the frame pointer to do its work. +// +// 2) The libunwind library. This is still in development, and as a +// separate library adds a new dependency, but doesn't need a frame +// pointer. It also doesn't call malloc. +// +// 3) The gdb unwinder -- also the one used by the c++ exception code. +// It's obviously well-tested, but has a fatal flaw: it can call +// malloc() from the unwinder. This is a problem because we're +// trying to use the unwinder to instrument malloc(). +// +// Note: if you add a new implementation here, make sure it works +// correctly when absl::GetStackTrace() is called with max_depth == 0. +// Some code may do that. + +#include "absl/debugging/stacktrace.h" + +#include <atomic> + +#include "absl/base/attributes.h" +#include "absl/base/port.h" +#include "absl/debugging/internal/stacktrace_config.h" + +#if defined(ABSL_STACKTRACE_INL_HEADER) +#include ABSL_STACKTRACE_INL_HEADER +#else +# error Cannot calculate stack trace: will need to write for your environment + +# include "absl/debugging/internal/stacktrace_aarch64-inl.inc" +# include "absl/debugging/internal/stacktrace_arm-inl.inc" +# include "absl/debugging/internal/stacktrace_generic-inl.inc" +# include "absl/debugging/internal/stacktrace_powerpc-inl.inc" +# include "absl/debugging/internal/stacktrace_unimplemented-inl.inc" +# include "absl/debugging/internal/stacktrace_win32-inl.inc" +# include "absl/debugging/internal/stacktrace_x86-inl.inc" +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +typedef int (*Unwinder)(void**, int*, int, int, const void*, int*); +std::atomic<Unwinder> custom; + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +ABSL_ATTRIBUTE_ALWAYS_INLINE inline int Unwind(void** result, int* sizes, + int max_depth, int skip_count, + const void* uc, + int* min_dropped_frames) { + Unwinder f = &UnwindImpl<IS_STACK_FRAMES, IS_WITH_CONTEXT>; + Unwinder g = custom.load(std::memory_order_acquire); + if (g != nullptr) f = g; + + // Add 1 to skip count for the unwinder function itself + int size = (*f)(result, sizes, max_depth, skip_count + 1, uc, + min_dropped_frames); + // To disable tail call to (*f)(...) + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); + return size; +} + +} // anonymous namespace + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackFrames( + void** result, int* sizes, int max_depth, int skip_count) { + return Unwind<true, false>(result, sizes, max_depth, skip_count, nullptr, + nullptr); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int +GetStackFramesWithContext(void** result, int* sizes, int max_depth, + int skip_count, const void* uc, + int* min_dropped_frames) { + return Unwind<true, true>(result, sizes, max_depth, skip_count, uc, + min_dropped_frames); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int GetStackTrace( + void** result, int max_depth, int skip_count) { + return Unwind<false, false>(result, nullptr, max_depth, skip_count, nullptr, + nullptr); +} + +ABSL_ATTRIBUTE_NOINLINE ABSL_ATTRIBUTE_NO_TAIL_CALL int +GetStackTraceWithContext(void** result, int max_depth, int skip_count, + const void* uc, int* min_dropped_frames) { + return Unwind<false, true>(result, nullptr, max_depth, skip_count, uc, + min_dropped_frames); +} + +void SetStackUnwinder(Unwinder w) { + custom.store(w, std::memory_order_release); +} + +int DefaultStackUnwinder(void** pcs, int* sizes, int depth, int skip, + const void* uc, int* min_dropped_frames) { + skip++; // For this function + Unwinder f = nullptr; + if (sizes == nullptr) { + if (uc == nullptr) { + f = &UnwindImpl<false, false>; + } else { + f = &UnwindImpl<false, true>; + } + } else { + if (uc == nullptr) { + f = &UnwindImpl<true, false>; + } else { + f = &UnwindImpl<true, true>; + } + } + volatile int x = 0; + int n = (*f)(pcs, sizes, depth, skip, uc, min_dropped_frames); + x = 1; (void) x; // To disable tail call to (*f)(...) + return n; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/stacktrace.h b/third_party/abseil_cpp/absl/debugging/stacktrace.h new file mode 100644 index 000000000000..0ec0ffdabd42 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/stacktrace.h @@ -0,0 +1,231 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: stacktrace.h +// ----------------------------------------------------------------------------- +// +// This file contains routines to extract the current stack trace and associated +// stack frames. These functions are thread-safe and async-signal-safe. +// +// Note that stack trace functionality is platform dependent and requires +// additional support from the compiler/build system in most cases. (That is, +// this functionality generally only works on platforms/builds that have been +// specifically configured to support it.) +// +// Note: stack traces in Abseil that do not utilize a symbolizer will result in +// frames consisting of function addresses rather than human-readable function +// names. (See symbolize.h for information on symbolizing these values.) + +#ifndef ABSL_DEBUGGING_STACKTRACE_H_ +#define ABSL_DEBUGGING_STACKTRACE_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// GetStackFrames() +// +// Records program counter values for up to `max_depth` frames, skipping the +// most recent `skip_count` stack frames, stores their corresponding values +// and sizes in `results` and `sizes` buffers, and returns the number of frames +// stored. (Note that the frame generated for the `absl::GetStackFrames()` +// routine itself is also skipped.) +// +// Example: +// +// main() { foo(); } +// foo() { bar(); } +// bar() { +// void* result[10]; +// int sizes[10]; +// int depth = absl::GetStackFrames(result, sizes, 10, 1); +// } +// +// The current stack frame would consist of three function calls: `bar()`, +// `foo()`, and then `main()`; however, since the `GetStackFrames()` call sets +// `skip_count` to `1`, it will skip the frame for `bar()`, the most recently +// invoked function call. It will therefore return 2 and fill `result` with +// program counters within the following functions: +// +// result[0] foo() +// result[1] main() +// +// (Note: in practice, a few more entries after `main()` may be added to account +// for startup processes.) +// +// Corresponding stack frame sizes will also be recorded: +// +// sizes[0] 16 +// sizes[1] 16 +// +// (Stack frame sizes of `16` above are just for illustration purposes.) +// +// Stack frame sizes of 0 or less indicate that those frame sizes couldn't +// be identified. +// +// This routine may return fewer stack frame entries than are +// available. Also note that `result` and `sizes` must both be non-null. +extern int GetStackFrames(void** result, int* sizes, int max_depth, + int skip_count); + +// GetStackFramesWithContext() +// +// Records program counter values obtained from a signal handler. Records +// program counter values for up to `max_depth` frames, skipping the most recent +// `skip_count` stack frames, stores their corresponding values and sizes in +// `results` and `sizes` buffers, and returns the number of frames stored. (Note +// that the frame generated for the `absl::GetStackFramesWithContext()` routine +// itself is also skipped.) +// +// The `uc` parameter, if non-null, should be a pointer to a `ucontext_t` value +// passed to a signal handler registered via the `sa_sigaction` field of a +// `sigaction` struct. (See +// http://man7.org/linux/man-pages/man2/sigaction.2.html.) The `uc` value may +// help a stack unwinder to provide a better stack trace under certain +// conditions. `uc` may safely be null. +// +// The `min_dropped_frames` output parameter, if non-null, points to the +// location to note any dropped stack frames, if any, due to buffer limitations +// or other reasons. (This value will be set to `0` if no frames were dropped.) +// The number of total stack frames is guaranteed to be >= skip_count + +// max_depth + *min_dropped_frames. +extern int GetStackFramesWithContext(void** result, int* sizes, int max_depth, + int skip_count, const void* uc, + int* min_dropped_frames); + +// GetStackTrace() +// +// Records program counter values for up to `max_depth` frames, skipping the +// most recent `skip_count` stack frames, stores their corresponding values +// in `results`, and returns the number of frames +// stored. Note that this function is similar to `absl::GetStackFrames()` +// except that it returns the stack trace only, and not stack frame sizes. +// +// Example: +// +// main() { foo(); } +// foo() { bar(); } +// bar() { +// void* result[10]; +// int depth = absl::GetStackTrace(result, 10, 1); +// } +// +// This produces: +// +// result[0] foo +// result[1] main +// .... ... +// +// `result` must not be null. +extern int GetStackTrace(void** result, int max_depth, int skip_count); + +// GetStackTraceWithContext() +// +// Records program counter values obtained from a signal handler. Records +// program counter values for up to `max_depth` frames, skipping the most recent +// `skip_count` stack frames, stores their corresponding values in `results`, +// and returns the number of frames stored. (Note that the frame generated for +// the `absl::GetStackFramesWithContext()` routine itself is also skipped.) +// +// The `uc` parameter, if non-null, should be a pointer to a `ucontext_t` value +// passed to a signal handler registered via the `sa_sigaction` field of a +// `sigaction` struct. (See +// http://man7.org/linux/man-pages/man2/sigaction.2.html.) The `uc` value may +// help a stack unwinder to provide a better stack trace under certain +// conditions. `uc` may safely be null. +// +// The `min_dropped_frames` output parameter, if non-null, points to the +// location to note any dropped stack frames, if any, due to buffer limitations +// or other reasons. (This value will be set to `0` if no frames were dropped.) +// The number of total stack frames is guaranteed to be >= skip_count + +// max_depth + *min_dropped_frames. +extern int GetStackTraceWithContext(void** result, int max_depth, + int skip_count, const void* uc, + int* min_dropped_frames); + +// SetStackUnwinder() +// +// Provides a custom function for unwinding stack frames that will be used in +// place of the default stack unwinder when invoking the static +// GetStack{Frames,Trace}{,WithContext}() functions above. +// +// The arguments passed to the unwinder function will match the +// arguments passed to `absl::GetStackFramesWithContext()` except that sizes +// will be non-null iff the caller is interested in frame sizes. +// +// If unwinder is set to null, we revert to the default stack-tracing behavior. +// +// ***************************************************************************** +// WARNING +// ***************************************************************************** +// +// absl::SetStackUnwinder is not suitable for general purpose use. It is +// provided for custom runtimes. +// Some things to watch out for when calling `absl::SetStackUnwinder()`: +// +// (a) The unwinder may be called from within signal handlers and +// therefore must be async-signal-safe. +// +// (b) Even after a custom stack unwinder has been unregistered, other +// threads may still be in the process of using that unwinder. +// Therefore do not clean up any state that may be needed by an old +// unwinder. +// ***************************************************************************** +extern void SetStackUnwinder(int (*unwinder)(void** pcs, int* sizes, + int max_depth, int skip_count, + const void* uc, + int* min_dropped_frames)); + +// DefaultStackUnwinder() +// +// Records program counter values of up to `max_depth` frames, skipping the most +// recent `skip_count` stack frames, and stores their corresponding values in +// `pcs`. (Note that the frame generated for this call itself is also skipped.) +// This function acts as a generic stack-unwinder; prefer usage of the more +// specific `GetStack{Trace,Frames}{,WithContext}()` functions above. +// +// If you have set your own stack unwinder (with the `SetStackUnwinder()` +// function above, you can still get the default stack unwinder by calling +// `DefaultStackUnwinder()`, which will ignore any previously set stack unwinder +// and use the default one instead. +// +// Because this function is generic, only `pcs` is guaranteed to be non-null +// upon return. It is legal for `sizes`, `uc`, and `min_dropped_frames` to all +// be null when called. +// +// The semantics are the same as the corresponding `GetStack*()` function in the +// case where `absl::SetStackUnwinder()` was never called. Equivalents are: +// +// null sizes | non-nullptr sizes +// |==========================================================| +// null uc | GetStackTrace() | GetStackFrames() | +// non-null uc | GetStackTraceWithContext() | GetStackFramesWithContext() | +// |==========================================================| +extern int DefaultStackUnwinder(void** pcs, int* sizes, int max_depth, + int skip_count, const void* uc, + int* min_dropped_frames); + +namespace debugging_internal { +// Returns true for platforms which are expected to have functioning stack trace +// implementations. Intended to be used for tests which want to exclude +// verification of logic known to be broken because stack traces are not +// working. +extern bool StackTraceWorksForTest(); +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_STACKTRACE_H_ diff --git a/third_party/abseil_cpp/absl/debugging/symbolize.cc b/third_party/abseil_cpp/absl/debugging/symbolize.cc new file mode 100644 index 000000000000..5e4a25d69dce --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize.cc @@ -0,0 +1,36 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/symbolize.h" + +#ifdef _WIN32 +#include <winapifamily.h> +#if !(WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_APP)) || \ + WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) +// UWP doesn't have access to win32 APIs. +#define ABSL_INTERNAL_HAVE_SYMBOLIZE_WIN32 +#endif +#endif + +#if defined(ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE) +#include "absl/debugging/symbolize_elf.inc" +#elif defined(ABSL_INTERNAL_HAVE_SYMBOLIZE_WIN32) +// The Windows Symbolizer only works if PDB files containing the debug info +// are available to the program at runtime. +#include "absl/debugging/symbolize_win32.inc" +#elif defined(__APPLE__) +#include "absl/debugging/symbolize_darwin.inc" +#else +#include "absl/debugging/symbolize_unimplemented.inc" +#endif diff --git a/third_party/abseil_cpp/absl/debugging/symbolize.h b/third_party/abseil_cpp/absl/debugging/symbolize.h new file mode 100644 index 000000000000..43d93a86822e --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize.h @@ -0,0 +1,99 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: symbolize.h +// ----------------------------------------------------------------------------- +// +// This file configures the Abseil symbolizer for use in converting instruction +// pointer addresses (program counters) into human-readable names (function +// calls, etc.) within Abseil code. +// +// The symbolizer may be invoked from several sources: +// +// * Implicitly, through the installation of an Abseil failure signal handler. +// (See failure_signal_handler.h for more information.) +// * By calling `Symbolize()` directly on a program counter you obtain through +// `absl::GetStackTrace()` or `absl::GetStackFrames()`. (See stacktrace.h +// for more information. +// * By calling `Symbolize()` directly on a program counter you obtain through +// other means (which would be platform-dependent). +// +// In all of the above cases, the symbolizer must first be initialized before +// any program counter values can be symbolized. If you are installing a failure +// signal handler, initialize the symbolizer before you do so. +// +// Example: +// +// int main(int argc, char** argv) { +// // Initialize the Symbolizer before installing the failure signal handler +// absl::InitializeSymbolizer(argv[0]); +// +// // Now you may install the failure signal handler +// absl::FailureSignalHandlerOptions options; +// absl::InstallFailureSignalHandler(options); +// +// // Start running your main program +// ... +// return 0; +// } +// +#ifndef ABSL_DEBUGGING_SYMBOLIZE_H_ +#define ABSL_DEBUGGING_SYMBOLIZE_H_ + +#include "absl/debugging/internal/symbolize.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// InitializeSymbolizer() +// +// Initializes the program counter symbolizer, given the path of the program +// (typically obtained through `main()`s `argv[0]`). The Abseil symbolizer +// allows you to read program counters (instruction pointer values) using their +// human-readable names within output such as stack traces. +// +// Example: +// +// int main(int argc, char *argv[]) { +// absl::InitializeSymbolizer(argv[0]); +// // Now you can use the symbolizer +// } +void InitializeSymbolizer(const char* argv0); +// +// Symbolize() +// +// Symbolizes a program counter (instruction pointer value) `pc` and, on +// success, writes the name to `out`. The symbol name is demangled, if possible. +// Note that the symbolized name may be truncated and will be NUL-terminated. +// Demangling is supported for symbols generated by GCC 3.x or newer). Returns +// `false` on failure. +// +// Example: +// +// // Print a program counter and its symbol name. +// static void DumpPCAndSymbol(void *pc) { +// char tmp[1024]; +// const char *symbol = "(unknown)"; +// if (absl::Symbolize(pc, tmp, sizeof(tmp))) { +// symbol = tmp; +// } +// absl::PrintF("%p %s\n", pc, symbol); +// } +bool Symbolize(const void *pc, char *out, int out_size); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_SYMBOLIZE_H_ diff --git a/third_party/abseil_cpp/absl/debugging/symbolize_darwin.inc b/third_party/abseil_cpp/absl/debugging/symbolize_darwin.inc new file mode 100644 index 000000000000..443ce9efc4ce --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize_darwin.inc @@ -0,0 +1,101 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cxxabi.h> +#include <execinfo.h> + +#include <algorithm> +#include <cstring> + +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/internal/demangle.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +void InitializeSymbolizer(const char*) {} + +namespace debugging_internal { +namespace { + +static std::string GetSymbolString(absl::string_view backtrace_line) { + // Example Backtrace lines: + // 0 libimaging_shared.dylib 0x018c152a + // _ZNSt11_Deque_baseIN3nik7mediadb4PageESaIS2_EE17_M_initialize_mapEm + 3478 + // + // or + // 0 libimaging_shared.dylib 0x0000000001895c39 + // _ZN3nik4util19register_shared_ptrINS_3gpu7TextureEEEvPKvS5_ + 39 + // + // or + // 0 mysterious_app 0x0124000120120009 main + 17 + auto address_pos = backtrace_line.find(" 0x"); + if (address_pos == absl::string_view::npos) return std::string(); + absl::string_view symbol_view = backtrace_line.substr(address_pos + 1); + + auto space_pos = symbol_view.find(" "); + if (space_pos == absl::string_view::npos) return std::string(); + symbol_view = symbol_view.substr(space_pos + 1); // to mangled symbol + + auto plus_pos = symbol_view.find(" + "); + if (plus_pos == absl::string_view::npos) return std::string(); + symbol_view = symbol_view.substr(0, plus_pos); // strip remainng + + return std::string(symbol_view); +} + +} // namespace +} // namespace debugging_internal + +bool Symbolize(const void* pc, char* out, int out_size) { + if (out_size <= 0 || pc == nullptr) { + out = nullptr; + return false; + } + + // This allocates a char* array. + char** frame_strings = backtrace_symbols(const_cast<void**>(&pc), 1); + + if (frame_strings == nullptr) return false; + + std::string symbol = debugging_internal::GetSymbolString(frame_strings[0]); + free(frame_strings); + + char tmp_buf[1024]; + if (debugging_internal::Demangle(symbol.c_str(), tmp_buf, sizeof(tmp_buf))) { + size_t len = strlen(tmp_buf); + if (len + 1 <= static_cast<size_t>(out_size)) { // +1 for '\0' + assert(len < sizeof(tmp_buf)); + memmove(out, tmp_buf, len + 1); + } + } else { + strncpy(out, symbol.c_str(), out_size); + } + + if (out[out_size - 1] != '\0') { + // strncpy() does not '\0' terminate when it truncates. + static constexpr char kEllipsis[] = "..."; + int ellipsis_size = std::min<int>(sizeof(kEllipsis) - 1, out_size - 1); + memcpy(out + out_size - ellipsis_size - 1, kEllipsis, ellipsis_size); + out[out_size - 1] = '\0'; + } + + return true; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/symbolize_elf.inc b/third_party/abseil_cpp/absl/debugging/symbolize_elf.inc new file mode 100644 index 000000000000..f4d5727bdeb5 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize_elf.inc @@ -0,0 +1,1560 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This library provides Symbolize() function that symbolizes program +// counters to their corresponding symbol names on linux platforms. +// This library has a minimal implementation of an ELF symbol table +// reader (i.e. it doesn't depend on libelf, etc.). +// +// The algorithm used in Symbolize() is as follows. +// +// 1. Go through a list of maps in /proc/self/maps and find the map +// containing the program counter. +// +// 2. Open the mapped file and find a regular symbol table inside. +// Iterate over symbols in the symbol table and look for the symbol +// containing the program counter. If such a symbol is found, +// obtain the symbol name, and demangle the symbol if possible. +// If the symbol isn't found in the regular symbol table (binary is +// stripped), try the same thing with a dynamic symbol table. +// +// Note that Symbolize() is originally implemented to be used in +// signal handlers, hence it doesn't use malloc() and other unsafe +// operations. It should be both thread-safe and async-signal-safe. +// +// Implementation note: +// +// We don't use heaps but only use stacks. We want to reduce the +// stack consumption so that the symbolizer can run on small stacks. +// +// Here are some numbers collected with GCC 4.1.0 on x86: +// - sizeof(Elf32_Sym) = 16 +// - sizeof(Elf32_Shdr) = 40 +// - sizeof(Elf64_Sym) = 24 +// - sizeof(Elf64_Shdr) = 64 +// +// This implementation is intended to be async-signal-safe but uses some +// functions which are not guaranteed to be so, such as memchr() and +// memmove(). We assume they are async-signal-safe. + +#include <dlfcn.h> +#include <elf.h> +#include <fcntl.h> +#include <link.h> // For ElfW() macro. +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <algorithm> +#include <array> +#include <atomic> +#include <cerrno> +#include <cinttypes> +#include <climits> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> + +#include "absl/base/casts.h" +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/low_level_alloc.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/port.h" +#include "absl/debugging/internal/demangle.h" +#include "absl/debugging/internal/vdso_support.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Value of argv[0]. Used by MaybeInitializeObjFile(). +static char *argv0_value = nullptr; + +void InitializeSymbolizer(const char *argv0) { +#ifdef ABSL_HAVE_VDSO_SUPPORT + // We need to make sure VDSOSupport::Init() is called before any setuid or + // chroot calls, so InitializeSymbolizer() should be called very early in the + // life of a program. + absl::debugging_internal::VDSOSupport::Init(); +#endif + if (argv0_value != nullptr) { + free(argv0_value); + argv0_value = nullptr; + } + if (argv0 != nullptr && argv0[0] != '\0') { + argv0_value = strdup(argv0); + } +} + +namespace debugging_internal { +namespace { + +// Re-runs fn until it doesn't cause EINTR. +#define NO_INTR(fn) \ + do { \ + } while ((fn) < 0 && errno == EINTR) + +// On Linux, ELF_ST_* are defined in <linux/elf.h>. To make this portable +// we define our own ELF_ST_BIND and ELF_ST_TYPE if not available. +#ifndef ELF_ST_BIND +#define ELF_ST_BIND(info) (((unsigned char)(info)) >> 4) +#endif + +#ifndef ELF_ST_TYPE +#define ELF_ST_TYPE(info) (((unsigned char)(info)) & 0xF) +#endif + +// Some platforms use a special .opd section to store function pointers. +const char kOpdSectionName[] = ".opd"; + +#if (defined(__powerpc__) && !(_CALL_ELF > 1)) || defined(__ia64) +// Use opd section for function descriptors on these platforms, the function +// address is the first word of the descriptor. +enum { kPlatformUsesOPDSections = 1 }; +#else // not PPC or IA64 +enum { kPlatformUsesOPDSections = 0 }; +#endif + +// This works for PowerPC & IA64 only. A function descriptor consist of two +// pointers and the first one is the function's entry. +const size_t kFunctionDescriptorSize = sizeof(void *) * 2; + +const int kMaxDecorators = 10; // Seems like a reasonable upper limit. + +struct InstalledSymbolDecorator { + SymbolDecorator fn; + void *arg; + int ticket; +}; + +int g_num_decorators; +InstalledSymbolDecorator g_decorators[kMaxDecorators]; + +struct FileMappingHint { + const void *start; + const void *end; + uint64_t offset; + const char *filename; +}; + +// Protects g_decorators. +// We are using SpinLock and not a Mutex here, because we may be called +// from inside Mutex::Lock itself, and it prohibits recursive calls. +// This happens in e.g. base/stacktrace_syscall_unittest. +// Moreover, we are using only TryLock(), if the decorator list +// is being modified (is busy), we skip all decorators, and possibly +// loose some info. Sorry, that's the best we could do. +ABSL_CONST_INIT absl::base_internal::SpinLock g_decorators_mu( + absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY); + +const int kMaxFileMappingHints = 8; +int g_num_file_mapping_hints; +FileMappingHint g_file_mapping_hints[kMaxFileMappingHints]; +// Protects g_file_mapping_hints. +ABSL_CONST_INIT absl::base_internal::SpinLock g_file_mapping_mu( + absl::kConstInit, absl::base_internal::SCHEDULE_KERNEL_ONLY); + +// Async-signal-safe function to zero a buffer. +// memset() is not guaranteed to be async-signal-safe. +static void SafeMemZero(void* p, size_t size) { + unsigned char *c = static_cast<unsigned char *>(p); + while (size--) { + *c++ = 0; + } +} + +struct ObjFile { + ObjFile() + : filename(nullptr), + start_addr(nullptr), + end_addr(nullptr), + offset(0), + fd(-1), + elf_type(-1) { + SafeMemZero(&elf_header, sizeof(elf_header)); + SafeMemZero(&phdr[0], sizeof(phdr)); + } + + char *filename; + const void *start_addr; + const void *end_addr; + uint64_t offset; + + // The following fields are initialized on the first access to the + // object file. + int fd; + int elf_type; + ElfW(Ehdr) elf_header; + + // PT_LOAD program header describing executable code. + // Normally we expect just one, but SWIFT binaries have two. + std::array<ElfW(Phdr), 2> phdr; +}; + +// Build 4-way associative cache for symbols. Within each cache line, symbols +// are replaced in LRU order. +enum { + ASSOCIATIVITY = 4, +}; +struct SymbolCacheLine { + const void *pc[ASSOCIATIVITY]; + char *name[ASSOCIATIVITY]; + + // age[i] is incremented when a line is accessed. it's reset to zero if the + // i'th entry is read. + uint32_t age[ASSOCIATIVITY]; +}; + +// --------------------------------------------------------------- +// An async-signal-safe arena for LowLevelAlloc +static std::atomic<base_internal::LowLevelAlloc::Arena *> g_sig_safe_arena; + +static base_internal::LowLevelAlloc::Arena *SigSafeArena() { + return g_sig_safe_arena.load(std::memory_order_acquire); +} + +static void InitSigSafeArena() { + if (SigSafeArena() == nullptr) { + base_internal::LowLevelAlloc::Arena *new_arena = + base_internal::LowLevelAlloc::NewArena( + base_internal::LowLevelAlloc::kAsyncSignalSafe); + base_internal::LowLevelAlloc::Arena *old_value = nullptr; + if (!g_sig_safe_arena.compare_exchange_strong(old_value, new_arena, + std::memory_order_release, + std::memory_order_relaxed)) { + // We lost a race to allocate an arena; deallocate. + base_internal::LowLevelAlloc::DeleteArena(new_arena); + } + } +} + +// --------------------------------------------------------------- +// An AddrMap is a vector of ObjFile, using SigSafeArena() for allocation. + +class AddrMap { + public: + AddrMap() : size_(0), allocated_(0), obj_(nullptr) {} + ~AddrMap() { base_internal::LowLevelAlloc::Free(obj_); } + int Size() const { return size_; } + ObjFile *At(int i) { return &obj_[i]; } + ObjFile *Add(); + void Clear(); + + private: + int size_; // count of valid elements (<= allocated_) + int allocated_; // count of allocated elements + ObjFile *obj_; // array of allocated_ elements + AddrMap(const AddrMap &) = delete; + AddrMap &operator=(const AddrMap &) = delete; +}; + +void AddrMap::Clear() { + for (int i = 0; i != size_; i++) { + At(i)->~ObjFile(); + } + size_ = 0; +} + +ObjFile *AddrMap::Add() { + if (size_ == allocated_) { + int new_allocated = allocated_ * 2 + 50; + ObjFile *new_obj_ = + static_cast<ObjFile *>(base_internal::LowLevelAlloc::AllocWithArena( + new_allocated * sizeof(*new_obj_), SigSafeArena())); + if (obj_) { + memcpy(new_obj_, obj_, allocated_ * sizeof(*new_obj_)); + base_internal::LowLevelAlloc::Free(obj_); + } + obj_ = new_obj_; + allocated_ = new_allocated; + } + return new (&obj_[size_++]) ObjFile; +} + +// --------------------------------------------------------------- + +enum FindSymbolResult { SYMBOL_NOT_FOUND = 1, SYMBOL_TRUNCATED, SYMBOL_FOUND }; + +class Symbolizer { + public: + Symbolizer(); + ~Symbolizer(); + const char *GetSymbol(const void *const pc); + + private: + char *CopyString(const char *s) { + int len = strlen(s); + char *dst = static_cast<char *>( + base_internal::LowLevelAlloc::AllocWithArena(len + 1, SigSafeArena())); + ABSL_RAW_CHECK(dst != nullptr, "out of memory"); + memcpy(dst, s, len + 1); + return dst; + } + ObjFile *FindObjFile(const void *const start, + size_t size) ABSL_ATTRIBUTE_NOINLINE; + static bool RegisterObjFile(const char *filename, + const void *const start_addr, + const void *const end_addr, uint64_t offset, + void *arg); + SymbolCacheLine *GetCacheLine(const void *const pc); + const char *FindSymbolInCache(const void *const pc); + const char *InsertSymbolInCache(const void *const pc, const char *name); + void AgeSymbols(SymbolCacheLine *line); + void ClearAddrMap(); + FindSymbolResult GetSymbolFromObjectFile(const ObjFile &obj, + const void *const pc, + const ptrdiff_t relocation, + char *out, int out_size, + char *tmp_buf, int tmp_buf_size); + + enum { + SYMBOL_BUF_SIZE = 3072, + TMP_BUF_SIZE = 1024, + SYMBOL_CACHE_LINES = 128, + }; + + AddrMap addr_map_; + + bool ok_; + bool addr_map_read_; + + char symbol_buf_[SYMBOL_BUF_SIZE]; + + // tmp_buf_ will be used to store arrays of ElfW(Shdr) and ElfW(Sym) + // so we ensure that tmp_buf_ is properly aligned to store either. + alignas(16) char tmp_buf_[TMP_BUF_SIZE]; + static_assert(alignof(ElfW(Shdr)) <= 16, + "alignment of tmp buf too small for Shdr"); + static_assert(alignof(ElfW(Sym)) <= 16, + "alignment of tmp buf too small for Sym"); + + SymbolCacheLine symbol_cache_[SYMBOL_CACHE_LINES]; +}; + +static std::atomic<Symbolizer *> g_cached_symbolizer; + +} // namespace + +static int SymbolizerSize() { +#if defined(__wasm__) || defined(__asmjs__) + int pagesize = getpagesize(); +#else + int pagesize = sysconf(_SC_PAGESIZE); +#endif + return ((sizeof(Symbolizer) - 1) / pagesize + 1) * pagesize; +} + +// Return (and set null) g_cached_symbolized_state if it is not null. +// Otherwise return a new symbolizer. +static Symbolizer *AllocateSymbolizer() { + InitSigSafeArena(); + Symbolizer *symbolizer = + g_cached_symbolizer.exchange(nullptr, std::memory_order_acquire); + if (symbolizer != nullptr) { + return symbolizer; + } + return new (base_internal::LowLevelAlloc::AllocWithArena( + SymbolizerSize(), SigSafeArena())) Symbolizer(); +} + +// Set g_cached_symbolize_state to s if it is null, otherwise +// delete s. +static void FreeSymbolizer(Symbolizer *s) { + Symbolizer *old_cached_symbolizer = nullptr; + if (!g_cached_symbolizer.compare_exchange_strong(old_cached_symbolizer, s, + std::memory_order_release, + std::memory_order_relaxed)) { + s->~Symbolizer(); + base_internal::LowLevelAlloc::Free(s); + } +} + +Symbolizer::Symbolizer() : ok_(true), addr_map_read_(false) { + for (SymbolCacheLine &symbol_cache_line : symbol_cache_) { + for (size_t j = 0; j < ABSL_ARRAYSIZE(symbol_cache_line.name); ++j) { + symbol_cache_line.pc[j] = nullptr; + symbol_cache_line.name[j] = nullptr; + symbol_cache_line.age[j] = 0; + } + } +} + +Symbolizer::~Symbolizer() { + for (SymbolCacheLine &symbol_cache_line : symbol_cache_) { + for (char *s : symbol_cache_line.name) { + base_internal::LowLevelAlloc::Free(s); + } + } + ClearAddrMap(); +} + +// We don't use assert() since it's not guaranteed to be +// async-signal-safe. Instead we define a minimal assertion +// macro. So far, we don't need pretty printing for __FILE__, etc. +#define SAFE_ASSERT(expr) ((expr) ? static_cast<void>(0) : abort()) + +// Read up to "count" bytes from file descriptor "fd" into the buffer +// starting at "buf" while handling short reads and EINTR. On +// success, return the number of bytes read. Otherwise, return -1. +static ssize_t ReadPersistent(int fd, void *buf, size_t count) { + SAFE_ASSERT(fd >= 0); + SAFE_ASSERT(count <= SSIZE_MAX); + char *buf0 = reinterpret_cast<char *>(buf); + size_t num_bytes = 0; + while (num_bytes < count) { + ssize_t len; + NO_INTR(len = read(fd, buf0 + num_bytes, count - num_bytes)); + if (len < 0) { // There was an error other than EINTR. + ABSL_RAW_LOG(WARNING, "read failed: errno=%d", errno); + return -1; + } + if (len == 0) { // Reached EOF. + break; + } + num_bytes += len; + } + SAFE_ASSERT(num_bytes <= count); + return static_cast<ssize_t>(num_bytes); +} + +// Read up to "count" bytes from "offset" in the file pointed by file +// descriptor "fd" into the buffer starting at "buf". On success, +// return the number of bytes read. Otherwise, return -1. +static ssize_t ReadFromOffset(const int fd, void *buf, const size_t count, + const off_t offset) { + off_t off = lseek(fd, offset, SEEK_SET); + if (off == (off_t)-1) { + ABSL_RAW_LOG(WARNING, "lseek(%d, %ju, SEEK_SET) failed: errno=%d", fd, + static_cast<uintmax_t>(offset), errno); + return -1; + } + return ReadPersistent(fd, buf, count); +} + +// Try reading exactly "count" bytes from "offset" bytes in a file +// pointed by "fd" into the buffer starting at "buf" while handling +// short reads and EINTR. On success, return true. Otherwise, return +// false. +static bool ReadFromOffsetExact(const int fd, void *buf, const size_t count, + const off_t offset) { + ssize_t len = ReadFromOffset(fd, buf, count, offset); + return len >= 0 && static_cast<size_t>(len) == count; +} + +// Returns elf_header.e_type if the file pointed by fd is an ELF binary. +static int FileGetElfType(const int fd) { + ElfW(Ehdr) elf_header; + if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + return -1; + } + if (memcmp(elf_header.e_ident, ELFMAG, SELFMAG) != 0) { + return -1; + } + return elf_header.e_type; +} + +// Read the section headers in the given ELF binary, and if a section +// of the specified type is found, set the output to this section header +// and return true. Otherwise, return false. +// To keep stack consumption low, we would like this function to not get +// inlined. +static ABSL_ATTRIBUTE_NOINLINE bool GetSectionHeaderByType( + const int fd, ElfW(Half) sh_num, const off_t sh_offset, ElfW(Word) type, + ElfW(Shdr) * out, char *tmp_buf, int tmp_buf_size) { + ElfW(Shdr) *buf = reinterpret_cast<ElfW(Shdr) *>(tmp_buf); + const int buf_entries = tmp_buf_size / sizeof(buf[0]); + const int buf_bytes = buf_entries * sizeof(buf[0]); + + for (int i = 0; i < sh_num;) { + const ssize_t num_bytes_left = (sh_num - i) * sizeof(buf[0]); + const ssize_t num_bytes_to_read = + (buf_bytes > num_bytes_left) ? num_bytes_left : buf_bytes; + const off_t offset = sh_offset + i * sizeof(buf[0]); + const ssize_t len = ReadFromOffset(fd, buf, num_bytes_to_read, offset); + if (len % sizeof(buf[0]) != 0) { + ABSL_RAW_LOG( + WARNING, + "Reading %zd bytes from offset %ju returned %zd which is not a " + "multiple of %zu.", + num_bytes_to_read, static_cast<uintmax_t>(offset), len, + sizeof(buf[0])); + return false; + } + const ssize_t num_headers_in_buf = len / sizeof(buf[0]); + SAFE_ASSERT(num_headers_in_buf <= buf_entries); + for (int j = 0; j < num_headers_in_buf; ++j) { + if (buf[j].sh_type == type) { + *out = buf[j]; + return true; + } + } + i += num_headers_in_buf; + } + return false; +} + +// There is no particular reason to limit section name to 63 characters, +// but there has (as yet) been no need for anything longer either. +const int kMaxSectionNameLen = 64; + +bool ForEachSection(int fd, + const std::function<bool(absl::string_view name, + const ElfW(Shdr) &)> &callback) { + ElfW(Ehdr) elf_header; + if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + return false; + } + + ElfW(Shdr) shstrtab; + off_t shstrtab_offset = + (elf_header.e_shoff + elf_header.e_shentsize * elf_header.e_shstrndx); + if (!ReadFromOffsetExact(fd, &shstrtab, sizeof(shstrtab), shstrtab_offset)) { + return false; + } + + for (int i = 0; i < elf_header.e_shnum; ++i) { + ElfW(Shdr) out; + off_t section_header_offset = + (elf_header.e_shoff + elf_header.e_shentsize * i); + if (!ReadFromOffsetExact(fd, &out, sizeof(out), section_header_offset)) { + return false; + } + off_t name_offset = shstrtab.sh_offset + out.sh_name; + char header_name[kMaxSectionNameLen]; + ssize_t n_read = + ReadFromOffset(fd, &header_name, kMaxSectionNameLen, name_offset); + if (n_read == -1) { + return false; + } else if (n_read > kMaxSectionNameLen) { + // Long read? + return false; + } + + absl::string_view name(header_name, strnlen(header_name, n_read)); + if (!callback(name, out)) { + break; + } + } + return true; +} + +// name_len should include terminating '\0'. +bool GetSectionHeaderByName(int fd, const char *name, size_t name_len, + ElfW(Shdr) * out) { + char header_name[kMaxSectionNameLen]; + if (sizeof(header_name) < name_len) { + ABSL_RAW_LOG(WARNING, + "Section name '%s' is too long (%zu); " + "section will not be found (even if present).", + name, name_len); + // No point in even trying. + return false; + } + + ElfW(Ehdr) elf_header; + if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + return false; + } + + ElfW(Shdr) shstrtab; + off_t shstrtab_offset = + (elf_header.e_shoff + elf_header.e_shentsize * elf_header.e_shstrndx); + if (!ReadFromOffsetExact(fd, &shstrtab, sizeof(shstrtab), shstrtab_offset)) { + return false; + } + + for (int i = 0; i < elf_header.e_shnum; ++i) { + off_t section_header_offset = + (elf_header.e_shoff + elf_header.e_shentsize * i); + if (!ReadFromOffsetExact(fd, out, sizeof(*out), section_header_offset)) { + return false; + } + off_t name_offset = shstrtab.sh_offset + out->sh_name; + ssize_t n_read = ReadFromOffset(fd, &header_name, name_len, name_offset); + if (n_read < 0) { + return false; + } else if (static_cast<size_t>(n_read) != name_len) { + // Short read -- name could be at end of file. + continue; + } + if (memcmp(header_name, name, name_len) == 0) { + return true; + } + } + return false; +} + +// Compare symbols at in the same address. +// Return true if we should pick symbol1. +static bool ShouldPickFirstSymbol(const ElfW(Sym) & symbol1, + const ElfW(Sym) & symbol2) { + // If one of the symbols is weak and the other is not, pick the one + // this is not a weak symbol. + char bind1 = ELF_ST_BIND(symbol1.st_info); + char bind2 = ELF_ST_BIND(symbol1.st_info); + if (bind1 == STB_WEAK && bind2 != STB_WEAK) return false; + if (bind2 == STB_WEAK && bind1 != STB_WEAK) return true; + + // If one of the symbols has zero size and the other is not, pick the + // one that has non-zero size. + if (symbol1.st_size != 0 && symbol2.st_size == 0) { + return true; + } + if (symbol1.st_size == 0 && symbol2.st_size != 0) { + return false; + } + + // If one of the symbols has no type and the other is not, pick the + // one that has a type. + char type1 = ELF_ST_TYPE(symbol1.st_info); + char type2 = ELF_ST_TYPE(symbol1.st_info); + if (type1 != STT_NOTYPE && type2 == STT_NOTYPE) { + return true; + } + if (type1 == STT_NOTYPE && type2 != STT_NOTYPE) { + return false; + } + + // Pick the first one, if we still cannot decide. + return true; +} + +// Return true if an address is inside a section. +static bool InSection(const void *address, const ElfW(Shdr) * section) { + const char *start = reinterpret_cast<const char *>(section->sh_addr); + size_t size = static_cast<size_t>(section->sh_size); + return start <= address && address < (start + size); +} + +static const char *ComputeOffset(const char *base, ptrdiff_t offset) { + // Note: cast to uintptr_t to avoid undefined behavior when base evaluates to + // zero and offset is non-zero. + return reinterpret_cast<const char *>( + reinterpret_cast<uintptr_t>(base) + offset); +} + +// Read a symbol table and look for the symbol containing the +// pc. Iterate over symbols in a symbol table and look for the symbol +// containing "pc". If the symbol is found, and its name fits in +// out_size, the name is written into out and SYMBOL_FOUND is returned. +// If the name does not fit, truncated name is written into out, +// and SYMBOL_TRUNCATED is returned. Out is NUL-terminated. +// If the symbol is not found, SYMBOL_NOT_FOUND is returned; +// To keep stack consumption low, we would like this function to not get +// inlined. +static ABSL_ATTRIBUTE_NOINLINE FindSymbolResult FindSymbol( + const void *const pc, const int fd, char *out, int out_size, + ptrdiff_t relocation, const ElfW(Shdr) * strtab, const ElfW(Shdr) * symtab, + const ElfW(Shdr) * opd, char *tmp_buf, int tmp_buf_size) { + if (symtab == nullptr) { + return SYMBOL_NOT_FOUND; + } + + // Read multiple symbols at once to save read() calls. + ElfW(Sym) *buf = reinterpret_cast<ElfW(Sym) *>(tmp_buf); + const int buf_entries = tmp_buf_size / sizeof(buf[0]); + + const int num_symbols = symtab->sh_size / symtab->sh_entsize; + + // On platforms using an .opd section (PowerPC & IA64), a function symbol + // has the address of a function descriptor, which contains the real + // starting address. However, we do not always want to use the real + // starting address because we sometimes want to symbolize a function + // pointer into the .opd section, e.g. FindSymbol(&foo,...). + const bool pc_in_opd = + kPlatformUsesOPDSections && opd != nullptr && InSection(pc, opd); + const bool deref_function_descriptor_pointer = + kPlatformUsesOPDSections && opd != nullptr && !pc_in_opd; + + ElfW(Sym) best_match; + SafeMemZero(&best_match, sizeof(best_match)); + bool found_match = false; + for (int i = 0; i < num_symbols;) { + off_t offset = symtab->sh_offset + i * symtab->sh_entsize; + const int num_remaining_symbols = num_symbols - i; + const int entries_in_chunk = std::min(num_remaining_symbols, buf_entries); + const int bytes_in_chunk = entries_in_chunk * sizeof(buf[0]); + const ssize_t len = ReadFromOffset(fd, buf, bytes_in_chunk, offset); + SAFE_ASSERT(len % sizeof(buf[0]) == 0); + const ssize_t num_symbols_in_buf = len / sizeof(buf[0]); + SAFE_ASSERT(num_symbols_in_buf <= entries_in_chunk); + for (int j = 0; j < num_symbols_in_buf; ++j) { + const ElfW(Sym) &symbol = buf[j]; + + // For a DSO, a symbol address is relocated by the loading address. + // We keep the original address for opd redirection below. + const char *const original_start_address = + reinterpret_cast<const char *>(symbol.st_value); + const char *start_address = + ComputeOffset(original_start_address, relocation); + + if (deref_function_descriptor_pointer && + InSection(original_start_address, opd)) { + // The opd section is mapped into memory. Just dereference + // start_address to get the first double word, which points to the + // function entry. + start_address = *reinterpret_cast<const char *const *>(start_address); + } + + // If pc is inside the .opd section, it points to a function descriptor. + const size_t size = pc_in_opd ? kFunctionDescriptorSize : symbol.st_size; + const void *const end_address = ComputeOffset(start_address, size); + if (symbol.st_value != 0 && // Skip null value symbols. + symbol.st_shndx != 0 && // Skip undefined symbols. +#ifdef STT_TLS + ELF_ST_TYPE(symbol.st_info) != STT_TLS && // Skip thread-local data. +#endif // STT_TLS + ((start_address <= pc && pc < end_address) || + (start_address == pc && pc == end_address))) { + if (!found_match || ShouldPickFirstSymbol(symbol, best_match)) { + found_match = true; + best_match = symbol; + } + } + } + i += num_symbols_in_buf; + } + + if (found_match) { + const size_t off = strtab->sh_offset + best_match.st_name; + const ssize_t n_read = ReadFromOffset(fd, out, out_size, off); + if (n_read <= 0) { + // This should never happen. + ABSL_RAW_LOG(WARNING, + "Unable to read from fd %d at offset %zu: n_read = %zd", fd, + off, n_read); + return SYMBOL_NOT_FOUND; + } + ABSL_RAW_CHECK(n_read <= out_size, "ReadFromOffset read too much data."); + + // strtab->sh_offset points into .strtab-like section that contains + // NUL-terminated strings: '\0foo\0barbaz\0...". + // + // sh_offset+st_name points to the start of symbol name, but we don't know + // how long the symbol is, so we try to read as much as we have space for, + // and usually over-read (i.e. there is a NUL somewhere before n_read). + if (memchr(out, '\0', n_read) == nullptr) { + // Either out_size was too small (n_read == out_size and no NUL), or + // we tried to read past the EOF (n_read < out_size) and .strtab is + // corrupt (missing terminating NUL; should never happen for valid ELF). + out[n_read - 1] = '\0'; + return SYMBOL_TRUNCATED; + } + return SYMBOL_FOUND; + } + + return SYMBOL_NOT_FOUND; +} + +// Get the symbol name of "pc" from the file pointed by "fd". Process +// both regular and dynamic symbol tables if necessary. +// See FindSymbol() comment for description of return value. +FindSymbolResult Symbolizer::GetSymbolFromObjectFile( + const ObjFile &obj, const void *const pc, const ptrdiff_t relocation, + char *out, int out_size, char *tmp_buf, int tmp_buf_size) { + ElfW(Shdr) symtab; + ElfW(Shdr) strtab; + ElfW(Shdr) opd; + ElfW(Shdr) *opd_ptr = nullptr; + + // On platforms using an .opd sections for function descriptor, read + // the section header. The .opd section is in data segment and should be + // loaded but we check that it is mapped just to be extra careful. + if (kPlatformUsesOPDSections) { + if (GetSectionHeaderByName(obj.fd, kOpdSectionName, + sizeof(kOpdSectionName) - 1, &opd) && + FindObjFile(reinterpret_cast<const char *>(opd.sh_addr) + relocation, + opd.sh_size) != nullptr) { + opd_ptr = &opd; + } else { + return SYMBOL_NOT_FOUND; + } + } + + // Consult a regular symbol table, then fall back to the dynamic symbol table. + for (const auto symbol_table_type : {SHT_SYMTAB, SHT_DYNSYM}) { + if (!GetSectionHeaderByType(obj.fd, obj.elf_header.e_shnum, + obj.elf_header.e_shoff, symbol_table_type, + &symtab, tmp_buf, tmp_buf_size)) { + continue; + } + if (!ReadFromOffsetExact( + obj.fd, &strtab, sizeof(strtab), + obj.elf_header.e_shoff + symtab.sh_link * sizeof(symtab))) { + continue; + } + const FindSymbolResult rc = + FindSymbol(pc, obj.fd, out, out_size, relocation, &strtab, &symtab, + opd_ptr, tmp_buf, tmp_buf_size); + if (rc != SYMBOL_NOT_FOUND) { + return rc; + } + } + + return SYMBOL_NOT_FOUND; +} + +namespace { +// Thin wrapper around a file descriptor so that the file descriptor +// gets closed for sure. +class FileDescriptor { + public: + explicit FileDescriptor(int fd) : fd_(fd) {} + FileDescriptor(const FileDescriptor &) = delete; + FileDescriptor &operator=(const FileDescriptor &) = delete; + + ~FileDescriptor() { + if (fd_ >= 0) { + NO_INTR(close(fd_)); + } + } + + int get() const { return fd_; } + + private: + const int fd_; +}; + +// Helper class for reading lines from file. +// +// Note: we don't use ProcMapsIterator since the object is big (it has +// a 5k array member) and uses async-unsafe functions such as sscanf() +// and snprintf(). +class LineReader { + public: + explicit LineReader(int fd, char *buf, int buf_len) + : fd_(fd), + buf_len_(buf_len), + buf_(buf), + bol_(buf), + eol_(buf), + eod_(buf) {} + + LineReader(const LineReader &) = delete; + LineReader &operator=(const LineReader &) = delete; + + // Read '\n'-terminated line from file. On success, modify "bol" + // and "eol", then return true. Otherwise, return false. + // + // Note: if the last line doesn't end with '\n', the line will be + // dropped. It's an intentional behavior to make the code simple. + bool ReadLine(const char **bol, const char **eol) { + if (BufferIsEmpty()) { // First time. + const ssize_t num_bytes = ReadPersistent(fd_, buf_, buf_len_); + if (num_bytes <= 0) { // EOF or error. + return false; + } + eod_ = buf_ + num_bytes; + bol_ = buf_; + } else { + bol_ = eol_ + 1; // Advance to the next line in the buffer. + SAFE_ASSERT(bol_ <= eod_); // "bol_" can point to "eod_". + if (!HasCompleteLine()) { + const int incomplete_line_length = eod_ - bol_; + // Move the trailing incomplete line to the beginning. + memmove(buf_, bol_, incomplete_line_length); + // Read text from file and append it. + char *const append_pos = buf_ + incomplete_line_length; + const int capacity_left = buf_len_ - incomplete_line_length; + const ssize_t num_bytes = + ReadPersistent(fd_, append_pos, capacity_left); + if (num_bytes <= 0) { // EOF or error. + return false; + } + eod_ = append_pos + num_bytes; + bol_ = buf_; + } + } + eol_ = FindLineFeed(); + if (eol_ == nullptr) { // '\n' not found. Malformed line. + return false; + } + *eol_ = '\0'; // Replace '\n' with '\0'. + + *bol = bol_; + *eol = eol_; + return true; + } + + private: + char *FindLineFeed() const { + return reinterpret_cast<char *>(memchr(bol_, '\n', eod_ - bol_)); + } + + bool BufferIsEmpty() const { return buf_ == eod_; } + + bool HasCompleteLine() const { + return !BufferIsEmpty() && FindLineFeed() != nullptr; + } + + const int fd_; + const int buf_len_; + char *const buf_; + char *bol_; + char *eol_; + const char *eod_; // End of data in "buf_". +}; +} // namespace + +// Place the hex number read from "start" into "*hex". The pointer to +// the first non-hex character or "end" is returned. +static const char *GetHex(const char *start, const char *end, + uint64_t *const value) { + uint64_t hex = 0; + const char *p; + for (p = start; p < end; ++p) { + int ch = *p; + if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'F') || + (ch >= 'a' && ch <= 'f')) { + hex = (hex << 4) | (ch < 'A' ? ch - '0' : (ch & 0xF) + 9); + } else { // Encountered the first non-hex character. + break; + } + } + SAFE_ASSERT(p <= end); + *value = hex; + return p; +} + +static const char *GetHex(const char *start, const char *end, + const void **const addr) { + uint64_t hex = 0; + const char *p = GetHex(start, end, &hex); + *addr = reinterpret_cast<void *>(hex); + return p; +} + +// Normally we are only interested in "r?x" maps. +// On the PowerPC, function pointers point to descriptors in the .opd +// section. The descriptors themselves are not executable code, so +// we need to relax the check below to "r??". +static bool ShouldUseMapping(const char *const flags) { + return flags[0] == 'r' && (kPlatformUsesOPDSections || flags[2] == 'x'); +} + +// Read /proc/self/maps and run "callback" for each mmapped file found. If +// "callback" returns false, stop scanning and return true. Else continue +// scanning /proc/self/maps. Return true if no parse error is found. +static ABSL_ATTRIBUTE_NOINLINE bool ReadAddrMap( + bool (*callback)(const char *filename, const void *const start_addr, + const void *const end_addr, uint64_t offset, void *arg), + void *arg, void *tmp_buf, int tmp_buf_size) { + // Use /proc/self/task/<pid>/maps instead of /proc/self/maps. The latter + // requires kernel to stop all threads, and is significantly slower when there + // are 1000s of threads. + char maps_path[80]; + snprintf(maps_path, sizeof(maps_path), "/proc/self/task/%d/maps", getpid()); + + int maps_fd; + NO_INTR(maps_fd = open(maps_path, O_RDONLY)); + FileDescriptor wrapped_maps_fd(maps_fd); + if (wrapped_maps_fd.get() < 0) { + ABSL_RAW_LOG(WARNING, "%s: errno=%d", maps_path, errno); + return false; + } + + // Iterate over maps and look for the map containing the pc. Then + // look into the symbol tables inside. + LineReader reader(wrapped_maps_fd.get(), static_cast<char *>(tmp_buf), + tmp_buf_size); + while (true) { + const char *cursor; + const char *eol; + if (!reader.ReadLine(&cursor, &eol)) { // EOF or malformed line. + break; + } + + const char *line = cursor; + const void *start_address; + // Start parsing line in /proc/self/maps. Here is an example: + // + // 08048000-0804c000 r-xp 00000000 08:01 2142121 /bin/cat + // + // We want start address (08048000), end address (0804c000), flags + // (r-xp) and file name (/bin/cat). + + // Read start address. + cursor = GetHex(cursor, eol, &start_address); + if (cursor == eol || *cursor != '-') { + ABSL_RAW_LOG(WARNING, "Corrupt /proc/self/maps line: %s", line); + return false; + } + ++cursor; // Skip '-'. + + // Read end address. + const void *end_address; + cursor = GetHex(cursor, eol, &end_address); + if (cursor == eol || *cursor != ' ') { + ABSL_RAW_LOG(WARNING, "Corrupt /proc/self/maps line: %s", line); + return false; + } + ++cursor; // Skip ' '. + + // Read flags. Skip flags until we encounter a space or eol. + const char *const flags_start = cursor; + while (cursor < eol && *cursor != ' ') { + ++cursor; + } + // We expect at least four letters for flags (ex. "r-xp"). + if (cursor == eol || cursor < flags_start + 4) { + ABSL_RAW_LOG(WARNING, "Corrupt /proc/self/maps: %s", line); + return false; + } + + // Check flags. + if (!ShouldUseMapping(flags_start)) { + continue; // We skip this map. + } + ++cursor; // Skip ' '. + + // Read file offset. + uint64_t offset; + cursor = GetHex(cursor, eol, &offset); + ++cursor; // Skip ' '. + + // Skip to file name. "cursor" now points to dev. We need to skip at least + // two spaces for dev and inode. + int num_spaces = 0; + while (cursor < eol) { + if (*cursor == ' ') { + ++num_spaces; + } else if (num_spaces >= 2) { + // The first non-space character after skipping two spaces + // is the beginning of the file name. + break; + } + ++cursor; + } + + // Check whether this entry corresponds to our hint table for the true + // filename. + bool hinted = + GetFileMappingHint(&start_address, &end_address, &offset, &cursor); + if (!hinted && (cursor == eol || cursor[0] == '[')) { + // not an object file, typically [vdso] or [vsyscall] + continue; + } + if (!callback(cursor, start_address, end_address, offset, arg)) break; + } + return true; +} + +// Find the objfile mapped in address region containing [addr, addr + len). +ObjFile *Symbolizer::FindObjFile(const void *const addr, size_t len) { + for (int i = 0; i < 2; ++i) { + if (!ok_) return nullptr; + + // Read /proc/self/maps if necessary + if (!addr_map_read_) { + addr_map_read_ = true; + if (!ReadAddrMap(RegisterObjFile, this, tmp_buf_, TMP_BUF_SIZE)) { + ok_ = false; + return nullptr; + } + } + + int lo = 0; + int hi = addr_map_.Size(); + while (lo < hi) { + int mid = (lo + hi) / 2; + if (addr < addr_map_.At(mid)->end_addr) { + hi = mid; + } else { + lo = mid + 1; + } + } + if (lo != addr_map_.Size()) { + ObjFile *obj = addr_map_.At(lo); + SAFE_ASSERT(obj->end_addr > addr); + if (addr >= obj->start_addr && + reinterpret_cast<const char *>(addr) + len <= obj->end_addr) + return obj; + } + + // The address mapping may have changed since it was last read. Retry. + ClearAddrMap(); + } + return nullptr; +} + +void Symbolizer::ClearAddrMap() { + for (int i = 0; i != addr_map_.Size(); i++) { + ObjFile *o = addr_map_.At(i); + base_internal::LowLevelAlloc::Free(o->filename); + if (o->fd >= 0) { + NO_INTR(close(o->fd)); + } + } + addr_map_.Clear(); + addr_map_read_ = false; +} + +// Callback for ReadAddrMap to register objfiles in an in-memory table. +bool Symbolizer::RegisterObjFile(const char *filename, + const void *const start_addr, + const void *const end_addr, uint64_t offset, + void *arg) { + Symbolizer *impl = static_cast<Symbolizer *>(arg); + + // Files are supposed to be added in the increasing address order. Make + // sure that's the case. + int addr_map_size = impl->addr_map_.Size(); + if (addr_map_size != 0) { + ObjFile *old = impl->addr_map_.At(addr_map_size - 1); + if (old->end_addr > end_addr) { + ABSL_RAW_LOG(ERROR, + "Unsorted addr map entry: 0x%" PRIxPTR ": %s <-> 0x%" PRIxPTR + ": %s", + reinterpret_cast<uintptr_t>(end_addr), filename, + reinterpret_cast<uintptr_t>(old->end_addr), old->filename); + return true; + } else if (old->end_addr == end_addr) { + // The same entry appears twice. This sometimes happens for [vdso]. + if (old->start_addr != start_addr || + strcmp(old->filename, filename) != 0) { + ABSL_RAW_LOG(ERROR, + "Duplicate addr 0x%" PRIxPTR ": %s <-> 0x%" PRIxPTR ": %s", + reinterpret_cast<uintptr_t>(end_addr), filename, + reinterpret_cast<uintptr_t>(old->end_addr), old->filename); + } + return true; + } + } + ObjFile *obj = impl->addr_map_.Add(); + obj->filename = impl->CopyString(filename); + obj->start_addr = start_addr; + obj->end_addr = end_addr; + obj->offset = offset; + obj->elf_type = -1; // filled on demand + obj->fd = -1; // opened on demand + return true; +} + +// This function wraps the Demangle function to provide an interface +// where the input symbol is demangled in-place. +// To keep stack consumption low, we would like this function to not +// get inlined. +static ABSL_ATTRIBUTE_NOINLINE void DemangleInplace(char *out, int out_size, + char *tmp_buf, + int tmp_buf_size) { + if (Demangle(out, tmp_buf, tmp_buf_size)) { + // Demangling succeeded. Copy to out if the space allows. + int len = strlen(tmp_buf); + if (len + 1 <= out_size) { // +1 for '\0'. + SAFE_ASSERT(len < tmp_buf_size); + memmove(out, tmp_buf, len + 1); + } + } +} + +SymbolCacheLine *Symbolizer::GetCacheLine(const void *const pc) { + uintptr_t pc0 = reinterpret_cast<uintptr_t>(pc); + pc0 >>= 3; // drop the low 3 bits + + // Shuffle bits. + pc0 ^= (pc0 >> 6) ^ (pc0 >> 12) ^ (pc0 >> 18); + return &symbol_cache_[pc0 % SYMBOL_CACHE_LINES]; +} + +void Symbolizer::AgeSymbols(SymbolCacheLine *line) { + for (uint32_t &age : line->age) { + ++age; + } +} + +const char *Symbolizer::FindSymbolInCache(const void *const pc) { + if (pc == nullptr) return nullptr; + + SymbolCacheLine *line = GetCacheLine(pc); + for (size_t i = 0; i < ABSL_ARRAYSIZE(line->pc); ++i) { + if (line->pc[i] == pc) { + AgeSymbols(line); + line->age[i] = 0; + return line->name[i]; + } + } + return nullptr; +} + +const char *Symbolizer::InsertSymbolInCache(const void *const pc, + const char *name) { + SAFE_ASSERT(pc != nullptr); + + SymbolCacheLine *line = GetCacheLine(pc); + uint32_t max_age = 0; + int oldest_index = -1; + for (size_t i = 0; i < ABSL_ARRAYSIZE(line->pc); ++i) { + if (line->pc[i] == nullptr) { + AgeSymbols(line); + line->pc[i] = pc; + line->name[i] = CopyString(name); + line->age[i] = 0; + return line->name[i]; + } + if (line->age[i] >= max_age) { + max_age = line->age[i]; + oldest_index = i; + } + } + + AgeSymbols(line); + ABSL_RAW_CHECK(oldest_index >= 0, "Corrupt cache"); + base_internal::LowLevelAlloc::Free(line->name[oldest_index]); + line->pc[oldest_index] = pc; + line->name[oldest_index] = CopyString(name); + line->age[oldest_index] = 0; + return line->name[oldest_index]; +} + +static void MaybeOpenFdFromSelfExe(ObjFile *obj) { + if (memcmp(obj->start_addr, ELFMAG, SELFMAG) != 0) { + return; + } + int fd = open("/proc/self/exe", O_RDONLY); + if (fd == -1) { + return; + } + // Verify that contents of /proc/self/exe matches in-memory image of + // the binary. This can fail if the "deleted" binary is in fact not + // the main executable, or for binaries that have the first PT_LOAD + // segment smaller than 4K. We do it in four steps so that the + // buffer is smaller and we don't consume too much stack space. + const char *mem = reinterpret_cast<const char *>(obj->start_addr); + for (int i = 0; i < 4; ++i) { + char buf[1024]; + ssize_t n = read(fd, buf, sizeof(buf)); + if (n != sizeof(buf) || memcmp(buf, mem, sizeof(buf)) != 0) { + close(fd); + return; + } + mem += sizeof(buf); + } + obj->fd = fd; +} + +static bool MaybeInitializeObjFile(ObjFile *obj) { + if (obj->fd < 0) { + obj->fd = open(obj->filename, O_RDONLY); + + if (obj->fd < 0) { + // Getting /proc/self/exe here means that we were hinted. + if (strcmp(obj->filename, "/proc/self/exe") == 0) { + // /proc/self/exe may be inaccessible (due to setuid, etc.), so try + // accessing the binary via argv0. + if (argv0_value != nullptr) { + obj->fd = open(argv0_value, O_RDONLY); + } + } else { + MaybeOpenFdFromSelfExe(obj); + } + } + + if (obj->fd < 0) { + ABSL_RAW_LOG(WARNING, "%s: open failed: errno=%d", obj->filename, errno); + return false; + } + obj->elf_type = FileGetElfType(obj->fd); + if (obj->elf_type < 0) { + ABSL_RAW_LOG(WARNING, "%s: wrong elf type: %d", obj->filename, + obj->elf_type); + return false; + } + + if (!ReadFromOffsetExact(obj->fd, &obj->elf_header, sizeof(obj->elf_header), + 0)) { + ABSL_RAW_LOG(WARNING, "%s: failed to read elf header", obj->filename); + return false; + } + const int phnum = obj->elf_header.e_phnum; + const int phentsize = obj->elf_header.e_phentsize; + size_t phoff = obj->elf_header.e_phoff; + size_t num_executable_load_segments = 0; + for (int j = 0; j < phnum; j++) { + ElfW(Phdr) phdr; + if (!ReadFromOffsetExact(obj->fd, &phdr, sizeof(phdr), phoff)) { + ABSL_RAW_LOG(WARNING, "%s: failed to read program header %d", + obj->filename, j); + return false; + } + phoff += phentsize; + constexpr int rx = PF_X | PF_R; + if (phdr.p_type != PT_LOAD || (phdr.p_flags & rx) != rx) { + // Not a LOAD segment, or not executable code. + continue; + } + if (num_executable_load_segments < obj->phdr.size()) { + memcpy(&obj->phdr[num_executable_load_segments++], &phdr, sizeof(phdr)); + } else { + ABSL_RAW_LOG(WARNING, "%s: too many executable LOAD segments", + obj->filename); + break; + } + } + if (num_executable_load_segments == 0) { + // This object has no "r-x" LOAD segments. That's unexpected. + ABSL_RAW_LOG(WARNING, "%s: no executable LOAD segments", obj->filename); + return false; + } + } + return true; +} + +// The implementation of our symbolization routine. If it +// successfully finds the symbol containing "pc" and obtains the +// symbol name, returns pointer to that symbol. Otherwise, returns nullptr. +// If any symbol decorators have been installed via InstallSymbolDecorator(), +// they are called here as well. +// To keep stack consumption low, we would like this function to not +// get inlined. +const char *Symbolizer::GetSymbol(const void *const pc) { + const char *entry = FindSymbolInCache(pc); + if (entry != nullptr) { + return entry; + } + symbol_buf_[0] = '\0'; + + ObjFile *const obj = FindObjFile(pc, 1); + ptrdiff_t relocation = 0; + int fd = -1; + if (obj != nullptr) { + if (MaybeInitializeObjFile(obj)) { + const size_t start_addr = reinterpret_cast<size_t>(obj->start_addr); + if (obj->elf_type == ET_DYN && start_addr >= obj->offset) { + // This object was relocated. + // + // For obj->offset > 0, adjust the relocation since a mapping at offset + // X in the file will have a start address of [true relocation]+X. + relocation = start_addr - obj->offset; + + // Note: some binaries have multiple "rx" LOAD segments. We must + // find the right one. + ElfW(Phdr) *phdr = nullptr; + for (size_t j = 0; j < obj->phdr.size(); j++) { + ElfW(Phdr) &p = obj->phdr[j]; + if (p.p_type != PT_LOAD) { + // We only expect PT_LOADs. This must be PT_NULL that we didn't + // write over (i.e. we exhausted all interesting PT_LOADs). + ABSL_RAW_CHECK(p.p_type == PT_NULL, "unexpected p_type"); + break; + } + if (pc < reinterpret_cast<void *>(start_addr + p.p_memsz)) { + phdr = &p; + break; + } + } + if (phdr == nullptr) { + // That's unexpected. Hope for the best. + ABSL_RAW_LOG( + WARNING, + "%s: unable to find LOAD segment for pc: %p, start_addr: %zx", + obj->filename, pc, start_addr); + } else { + // Adjust relocation in case phdr.p_vaddr != 0. + // This happens for binaries linked with `lld --rosegment`, and for + // binaries linked with BFD `ld -z separate-code`. + relocation -= phdr->p_vaddr - phdr->p_offset; + } + } + + fd = obj->fd; + if (GetSymbolFromObjectFile(*obj, pc, relocation, symbol_buf_, + sizeof(symbol_buf_), tmp_buf_, + sizeof(tmp_buf_)) == SYMBOL_FOUND) { + // Only try to demangle the symbol name if it fit into symbol_buf_. + DemangleInplace(symbol_buf_, sizeof(symbol_buf_), tmp_buf_, + sizeof(tmp_buf_)); + } + } + } else { +#if ABSL_HAVE_VDSO_SUPPORT + VDSOSupport vdso; + if (vdso.IsPresent()) { + VDSOSupport::SymbolInfo symbol_info; + if (vdso.LookupSymbolByAddress(pc, &symbol_info)) { + // All VDSO symbols are known to be short. + size_t len = strlen(symbol_info.name); + ABSL_RAW_CHECK(len + 1 < sizeof(symbol_buf_), + "VDSO symbol unexpectedly long"); + memcpy(symbol_buf_, symbol_info.name, len + 1); + } + } +#endif + } + + if (g_decorators_mu.TryLock()) { + if (g_num_decorators > 0) { + SymbolDecoratorArgs decorator_args = { + pc, relocation, fd, symbol_buf_, sizeof(symbol_buf_), + tmp_buf_, sizeof(tmp_buf_), nullptr}; + for (int i = 0; i < g_num_decorators; ++i) { + decorator_args.arg = g_decorators[i].arg; + g_decorators[i].fn(&decorator_args); + } + } + g_decorators_mu.Unlock(); + } + if (symbol_buf_[0] == '\0') { + return nullptr; + } + symbol_buf_[sizeof(symbol_buf_) - 1] = '\0'; // Paranoia. + return InsertSymbolInCache(pc, symbol_buf_); +} + +bool RemoveAllSymbolDecorators(void) { + if (!g_decorators_mu.TryLock()) { + // Someone else is using decorators. Get out. + return false; + } + g_num_decorators = 0; + g_decorators_mu.Unlock(); + return true; +} + +bool RemoveSymbolDecorator(int ticket) { + if (!g_decorators_mu.TryLock()) { + // Someone else is using decorators. Get out. + return false; + } + for (int i = 0; i < g_num_decorators; ++i) { + if (g_decorators[i].ticket == ticket) { + while (i < g_num_decorators - 1) { + g_decorators[i] = g_decorators[i + 1]; + ++i; + } + g_num_decorators = i; + break; + } + } + g_decorators_mu.Unlock(); + return true; // Decorator is known to be removed. +} + +int InstallSymbolDecorator(SymbolDecorator decorator, void *arg) { + static int ticket = 0; + + if (!g_decorators_mu.TryLock()) { + // Someone else is using decorators. Get out. + return -2; + } + int ret = ticket; + if (g_num_decorators >= kMaxDecorators) { + ret = -1; + } else { + g_decorators[g_num_decorators] = {decorator, arg, ticket++}; + ++g_num_decorators; + } + g_decorators_mu.Unlock(); + return ret; +} + +bool RegisterFileMappingHint(const void *start, const void *end, uint64_t offset, + const char *filename) { + SAFE_ASSERT(start <= end); + SAFE_ASSERT(filename != nullptr); + + InitSigSafeArena(); + + if (!g_file_mapping_mu.TryLock()) { + return false; + } + + bool ret = true; + if (g_num_file_mapping_hints >= kMaxFileMappingHints) { + ret = false; + } else { + // TODO(ckennelly): Move this into a string copy routine. + int len = strlen(filename); + char *dst = static_cast<char *>( + base_internal::LowLevelAlloc::AllocWithArena(len + 1, SigSafeArena())); + ABSL_RAW_CHECK(dst != nullptr, "out of memory"); + memcpy(dst, filename, len + 1); + + auto &hint = g_file_mapping_hints[g_num_file_mapping_hints++]; + hint.start = start; + hint.end = end; + hint.offset = offset; + hint.filename = dst; + } + + g_file_mapping_mu.Unlock(); + return ret; +} + +bool GetFileMappingHint(const void **start, const void **end, uint64_t *offset, + const char **filename) { + if (!g_file_mapping_mu.TryLock()) { + return false; + } + bool found = false; + for (int i = 0; i < g_num_file_mapping_hints; i++) { + if (g_file_mapping_hints[i].start <= *start && + *end <= g_file_mapping_hints[i].end) { + // We assume that the start_address for the mapping is the base + // address of the ELF section, but when [start_address,end_address) is + // not strictly equal to [hint.start, hint.end), that assumption is + // invalid. + // + // This uses the hint's start address (even though hint.start is not + // necessarily equal to start_address) to ensure the correct + // relocation is computed later. + *start = g_file_mapping_hints[i].start; + *end = g_file_mapping_hints[i].end; + *offset = g_file_mapping_hints[i].offset; + *filename = g_file_mapping_hints[i].filename; + found = true; + break; + } + } + g_file_mapping_mu.Unlock(); + return found; +} + +} // namespace debugging_internal + +bool Symbolize(const void *pc, char *out, int out_size) { + // Symbolization is very slow under tsan. + ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN(); + SAFE_ASSERT(out_size >= 0); + debugging_internal::Symbolizer *s = debugging_internal::AllocateSymbolizer(); + const char *name = s->GetSymbol(pc); + bool ok = false; + if (name != nullptr && out_size > 0) { + strncpy(out, name, out_size); + ok = true; + if (out[out_size - 1] != '\0') { + // strncpy() does not '\0' terminate when it truncates. Do so, with + // trailing ellipsis. + static constexpr char kEllipsis[] = "..."; + int ellipsis_size = + std::min(implicit_cast<int>(strlen(kEllipsis)), out_size - 1); + memcpy(out + out_size - ellipsis_size - 1, kEllipsis, ellipsis_size); + out[out_size - 1] = '\0'; + } + } + debugging_internal::FreeSymbolizer(s); + ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END(); + return ok; +} + +ABSL_NAMESPACE_END +} // namespace absl + +extern "C" bool AbslInternalGetFileMappingHint(const void **start, + const void **end, uint64_t *offset, + const char **filename) { + return absl::debugging_internal::GetFileMappingHint(start, end, offset, + filename); +} diff --git a/third_party/abseil_cpp/absl/debugging/symbolize_test.cc b/third_party/abseil_cpp/absl/debugging/symbolize_test.cc new file mode 100644 index 000000000000..a2dd4956c497 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize_test.cc @@ -0,0 +1,557 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/debugging/symbolize.h" + +#ifndef _WIN32 +#include <fcntl.h> +#include <sys/mman.h> +#endif + +#include <cstring> +#include <iostream> +#include <memory> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/casts.h" +#include "absl/base/config.h" +#include "absl/base/internal/per_thread_tls.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/optimization.h" +#include "absl/debugging/internal/stack_consumption.h" +#include "absl/memory/memory.h" +#include "absl/strings/string_view.h" + +using testing::Contains; + +#ifdef _WIN32 +#define ABSL_SYMBOLIZE_TEST_NOINLINE __declspec(noinline) +#else +#define ABSL_SYMBOLIZE_TEST_NOINLINE ABSL_ATTRIBUTE_NOINLINE +#endif + +// Functions to symbolize. Use C linkage to avoid mangled names. +extern "C" { +ABSL_SYMBOLIZE_TEST_NOINLINE void nonstatic_func() { + // The next line makes this a unique function to prevent the compiler from + // folding identical functions together. + volatile int x = __LINE__; + static_cast<void>(x); + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); +} + +ABSL_SYMBOLIZE_TEST_NOINLINE static void static_func() { + // The next line makes this a unique function to prevent the compiler from + // folding identical functions together. + volatile int x = __LINE__; + static_cast<void>(x); + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); +} +} // extern "C" + +struct Foo { + static void func(int x); +}; + +// A C++ method that should have a mangled name. +ABSL_SYMBOLIZE_TEST_NOINLINE void Foo::func(int) { + // The next line makes this a unique function to prevent the compiler from + // folding identical functions together. + volatile int x = __LINE__; + static_cast<void>(x); + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); +} + +// Create functions that will remain in different text sections in the +// final binary when linker option "-z,keep-text-section-prefix" is used. +int ABSL_ATTRIBUTE_SECTION_VARIABLE(.text.unlikely) unlikely_func() { + return 0; +} + +int ABSL_ATTRIBUTE_SECTION_VARIABLE(.text.hot) hot_func() { + return 0; +} + +int ABSL_ATTRIBUTE_SECTION_VARIABLE(.text.startup) startup_func() { + return 0; +} + +int ABSL_ATTRIBUTE_SECTION_VARIABLE(.text.exit) exit_func() { + return 0; +} + +int /*ABSL_ATTRIBUTE_SECTION_VARIABLE(.text)*/ regular_func() { + return 0; +} + +// Thread-local data may confuse the symbolizer, ensure that it does not. +// Variable sizes and order are important. +#if ABSL_PER_THREAD_TLS +static ABSL_PER_THREAD_TLS_KEYWORD char symbolize_test_thread_small[1]; +static ABSL_PER_THREAD_TLS_KEYWORD char + symbolize_test_thread_big[2 * 1024 * 1024]; +#endif + +#if !defined(__EMSCRIPTEN__) +// Used below to hopefully inhibit some compiler/linker optimizations +// that may remove kHpageTextPadding, kPadding0, and kPadding1 from +// the binary. +static volatile bool volatile_bool = false; + +// Force the binary to be large enough that a THP .text remap will succeed. +static constexpr size_t kHpageSize = 1 << 21; +const char kHpageTextPadding[kHpageSize * 4] ABSL_ATTRIBUTE_SECTION_VARIABLE( + .text) = ""; +#endif // !defined(__EMSCRIPTEN__) + +static char try_symbolize_buffer[4096]; + +// A wrapper function for absl::Symbolize() to make the unit test simple. The +// limit must be < sizeof(try_symbolize_buffer). Returns null if +// absl::Symbolize() returns false, otherwise returns try_symbolize_buffer with +// the result of absl::Symbolize(). +static const char *TrySymbolizeWithLimit(void *pc, int limit) { + ABSL_RAW_CHECK(limit <= sizeof(try_symbolize_buffer), + "try_symbolize_buffer is too small"); + + // Use the heap to facilitate heap and buffer sanitizer tools. + auto heap_buffer = absl::make_unique<char[]>(sizeof(try_symbolize_buffer)); + bool found = absl::Symbolize(pc, heap_buffer.get(), limit); + if (found) { + ABSL_RAW_CHECK(strnlen(heap_buffer.get(), limit) < limit, + "absl::Symbolize() did not properly terminate the string"); + strncpy(try_symbolize_buffer, heap_buffer.get(), + sizeof(try_symbolize_buffer) - 1); + try_symbolize_buffer[sizeof(try_symbolize_buffer) - 1] = '\0'; + } + + return found ? try_symbolize_buffer : nullptr; +} + +// A wrapper for TrySymbolizeWithLimit(), with a large limit. +static const char *TrySymbolize(void *pc) { + return TrySymbolizeWithLimit(pc, sizeof(try_symbolize_buffer)); +} + +#if defined(ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE) || \ + defined(ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE) + +TEST(Symbolize, Cached) { + // Compilers should give us pointers to them. + EXPECT_STREQ("nonstatic_func", TrySymbolize((void *)(&nonstatic_func))); + + // The name of an internal linkage symbol is not specified; allow either a + // mangled or an unmangled name here. + const char *static_func_symbol = TrySymbolize((void *)(&static_func)); + EXPECT_TRUE(strcmp("static_func", static_func_symbol) == 0 || + strcmp("static_func()", static_func_symbol) == 0); + + EXPECT_TRUE(nullptr == TrySymbolize(nullptr)); +} + +TEST(Symbolize, Truncation) { + constexpr char kNonStaticFunc[] = "nonstatic_func"; + EXPECT_STREQ("nonstatic_func", + TrySymbolizeWithLimit((void *)(&nonstatic_func), + strlen(kNonStaticFunc) + 1)); + EXPECT_STREQ("nonstatic_...", + TrySymbolizeWithLimit((void *)(&nonstatic_func), + strlen(kNonStaticFunc) + 0)); + EXPECT_STREQ("nonstatic...", + TrySymbolizeWithLimit((void *)(&nonstatic_func), + strlen(kNonStaticFunc) - 1)); + EXPECT_STREQ("n...", TrySymbolizeWithLimit((void *)(&nonstatic_func), 5)); + EXPECT_STREQ("...", TrySymbolizeWithLimit((void *)(&nonstatic_func), 4)); + EXPECT_STREQ("..", TrySymbolizeWithLimit((void *)(&nonstatic_func), 3)); + EXPECT_STREQ(".", TrySymbolizeWithLimit((void *)(&nonstatic_func), 2)); + EXPECT_STREQ("", TrySymbolizeWithLimit((void *)(&nonstatic_func), 1)); + EXPECT_EQ(nullptr, TrySymbolizeWithLimit((void *)(&nonstatic_func), 0)); +} + +TEST(Symbolize, SymbolizeWithDemangling) { + Foo::func(100); + EXPECT_STREQ("Foo::func()", TrySymbolize((void *)(&Foo::func))); +} + +TEST(Symbolize, SymbolizeSplitTextSections) { + EXPECT_STREQ("unlikely_func()", TrySymbolize((void *)(&unlikely_func))); + EXPECT_STREQ("hot_func()", TrySymbolize((void *)(&hot_func))); + EXPECT_STREQ("startup_func()", TrySymbolize((void *)(&startup_func))); + EXPECT_STREQ("exit_func()", TrySymbolize((void *)(&exit_func))); + EXPECT_STREQ("regular_func()", TrySymbolize((void *)(®ular_func))); +} + +// Tests that verify that Symbolize stack footprint is within some limit. +#ifdef ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION + +static void *g_pc_to_symbolize; +static char g_symbolize_buffer[4096]; +static char *g_symbolize_result; + +static void SymbolizeSignalHandler(int signo) { + if (absl::Symbolize(g_pc_to_symbolize, g_symbolize_buffer, + sizeof(g_symbolize_buffer))) { + g_symbolize_result = g_symbolize_buffer; + } else { + g_symbolize_result = nullptr; + } +} + +// Call Symbolize and figure out the stack footprint of this call. +static const char *SymbolizeStackConsumption(void *pc, int *stack_consumed) { + g_pc_to_symbolize = pc; + *stack_consumed = absl::debugging_internal::GetSignalHandlerStackConsumption( + SymbolizeSignalHandler); + return g_symbolize_result; +} + +static int GetStackConsumptionUpperLimit() { + // Symbolize stack consumption should be within 2kB. + int stack_consumption_upper_limit = 2048; +#if defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) + // Account for sanitizer instrumentation requiring additional stack space. + stack_consumption_upper_limit *= 5; +#endif + return stack_consumption_upper_limit; +} + +TEST(Symbolize, SymbolizeStackConsumption) { + int stack_consumed = 0; + + const char *symbol = + SymbolizeStackConsumption((void *)(&nonstatic_func), &stack_consumed); + EXPECT_STREQ("nonstatic_func", symbol); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, GetStackConsumptionUpperLimit()); + + // The name of an internal linkage symbol is not specified; allow either a + // mangled or an unmangled name here. + symbol = SymbolizeStackConsumption((void *)(&static_func), &stack_consumed); + EXPECT_TRUE(strcmp("static_func", symbol) == 0 || + strcmp("static_func()", symbol) == 0); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, GetStackConsumptionUpperLimit()); +} + +TEST(Symbolize, SymbolizeWithDemanglingStackConsumption) { + Foo::func(100); + int stack_consumed = 0; + + const char *symbol = + SymbolizeStackConsumption((void *)(&Foo::func), &stack_consumed); + + EXPECT_STREQ("Foo::func()", symbol); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, GetStackConsumptionUpperLimit()); +} + +#endif // ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION + +#ifndef ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE +// Use a 64K page size for PPC. +const size_t kPageSize = 64 << 10; +// We place a read-only symbols into the .text section and verify that we can +// symbolize them and other symbols after remapping them. +const char kPadding0[kPageSize * 4] ABSL_ATTRIBUTE_SECTION_VARIABLE(.text) = + ""; +const char kPadding1[kPageSize * 4] ABSL_ATTRIBUTE_SECTION_VARIABLE(.text) = + ""; + +static int FilterElfHeader(struct dl_phdr_info *info, size_t size, void *data) { + for (int i = 0; i < info->dlpi_phnum; i++) { + if (info->dlpi_phdr[i].p_type == PT_LOAD && + info->dlpi_phdr[i].p_flags == (PF_R | PF_X)) { + const void *const vaddr = + absl::bit_cast<void *>(info->dlpi_addr + info->dlpi_phdr[i].p_vaddr); + const auto segsize = info->dlpi_phdr[i].p_memsz; + + const char *self_exe; + if (info->dlpi_name != nullptr && info->dlpi_name[0] != '\0') { + self_exe = info->dlpi_name; + } else { + self_exe = "/proc/self/exe"; + } + + absl::debugging_internal::RegisterFileMappingHint( + vaddr, reinterpret_cast<const char *>(vaddr) + segsize, + info->dlpi_phdr[i].p_offset, self_exe); + + return 1; + } + } + + return 1; +} + +TEST(Symbolize, SymbolizeWithMultipleMaps) { + // Force kPadding0 and kPadding1 to be linked in. + if (volatile_bool) { + ABSL_RAW_LOG(INFO, "%s", kPadding0); + ABSL_RAW_LOG(INFO, "%s", kPadding1); + } + + // Verify we can symbolize everything. + char buf[512]; + memset(buf, 0, sizeof(buf)); + absl::Symbolize(kPadding0, buf, sizeof(buf)); + EXPECT_STREQ("kPadding0", buf); + + memset(buf, 0, sizeof(buf)); + absl::Symbolize(kPadding1, buf, sizeof(buf)); + EXPECT_STREQ("kPadding1", buf); + + // Specify a hint for the executable segment. + dl_iterate_phdr(FilterElfHeader, nullptr); + + // Reload at least one page out of kPadding0, kPadding1 + const char *ptrs[] = {kPadding0, kPadding1}; + + for (const char *ptr : ptrs) { + const int kMapFlags = MAP_ANONYMOUS | MAP_PRIVATE; + void *addr = mmap(nullptr, kPageSize, PROT_READ, kMapFlags, 0, 0); + ASSERT_NE(addr, MAP_FAILED); + + // kPadding[0-1] is full of zeroes, so we can remap anywhere within it, but + // we ensure there is at least a full page of padding. + void *remapped = reinterpret_cast<void *>( + reinterpret_cast<uintptr_t>(ptr + kPageSize) & ~(kPageSize - 1ULL)); + + const int kMremapFlags = (MREMAP_MAYMOVE | MREMAP_FIXED); + void *ret = mremap(addr, kPageSize, kPageSize, kMremapFlags, remapped); + ASSERT_NE(ret, MAP_FAILED); + } + + // Invalidate the symbolization cache so we are forced to rely on the hint. + absl::Symbolize(nullptr, buf, sizeof(buf)); + + // Verify we can still symbolize. + const char *expected[] = {"kPadding0", "kPadding1"}; + const size_t offsets[] = {0, kPageSize, 2 * kPageSize, 3 * kPageSize}; + + for (int i = 0; i < 2; i++) { + for (size_t offset : offsets) { + memset(buf, 0, sizeof(buf)); + absl::Symbolize(ptrs[i] + offset, buf, sizeof(buf)); + EXPECT_STREQ(expected[i], buf); + } + } +} + +// Appends string(*args->arg) to args->symbol_buf. +static void DummySymbolDecorator( + const absl::debugging_internal::SymbolDecoratorArgs *args) { + std::string *message = static_cast<std::string *>(args->arg); + strncat(args->symbol_buf, message->c_str(), + args->symbol_buf_size - strlen(args->symbol_buf) - 1); +} + +TEST(Symbolize, InstallAndRemoveSymbolDecorators) { + int ticket_a; + std::string a_message("a"); + EXPECT_GE(ticket_a = absl::debugging_internal::InstallSymbolDecorator( + DummySymbolDecorator, &a_message), + 0); + + int ticket_b; + std::string b_message("b"); + EXPECT_GE(ticket_b = absl::debugging_internal::InstallSymbolDecorator( + DummySymbolDecorator, &b_message), + 0); + + int ticket_c; + std::string c_message("c"); + EXPECT_GE(ticket_c = absl::debugging_internal::InstallSymbolDecorator( + DummySymbolDecorator, &c_message), + 0); + + char *address = reinterpret_cast<char *>(1); + EXPECT_STREQ("abc", TrySymbolize(address++)); + + EXPECT_TRUE(absl::debugging_internal::RemoveSymbolDecorator(ticket_b)); + + EXPECT_STREQ("ac", TrySymbolize(address++)); + + // Cleanup: remove all remaining decorators so other stack traces don't + // get mystery "ac" decoration. + EXPECT_TRUE(absl::debugging_internal::RemoveSymbolDecorator(ticket_a)); + EXPECT_TRUE(absl::debugging_internal::RemoveSymbolDecorator(ticket_c)); +} + +// Some versions of Clang with optimizations enabled seem to be able +// to optimize away the .data section if no variables live in the +// section. This variable should get placed in the .data section, and +// the test below checks for the existence of a .data section. +static int in_data_section = 1; + +TEST(Symbolize, ForEachSection) { + int fd = TEMP_FAILURE_RETRY(open("/proc/self/exe", O_RDONLY)); + ASSERT_NE(fd, -1); + + std::vector<std::string> sections; + ASSERT_TRUE(absl::debugging_internal::ForEachSection( + fd, [§ions](const absl::string_view name, const ElfW(Shdr) &) { + sections.emplace_back(name); + return true; + })); + + // Check for the presence of common section names. + EXPECT_THAT(sections, Contains(".text")); + EXPECT_THAT(sections, Contains(".rodata")); + EXPECT_THAT(sections, Contains(".bss")); + ++in_data_section; + EXPECT_THAT(sections, Contains(".data")); + + close(fd); +} +#endif // !ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE + +// x86 specific tests. Uses some inline assembler. +extern "C" { +inline void *ABSL_ATTRIBUTE_ALWAYS_INLINE inline_func() { + void *pc = nullptr; +#if defined(__i386__) + __asm__ __volatile__("call 1f;\n 1: pop %[PC]" : [ PC ] "=r"(pc)); +#elif defined(__x86_64__) + __asm__ __volatile__("leaq 0(%%rip),%[PC];\n" : [ PC ] "=r"(pc)); +#endif + return pc; +} + +void *ABSL_ATTRIBUTE_NOINLINE non_inline_func() { + void *pc = nullptr; +#if defined(__i386__) + __asm__ __volatile__("call 1f;\n 1: pop %[PC]" : [ PC ] "=r"(pc)); +#elif defined(__x86_64__) + __asm__ __volatile__("leaq 0(%%rip),%[PC];\n" : [ PC ] "=r"(pc)); +#endif + return pc; +} + +void ABSL_ATTRIBUTE_NOINLINE TestWithPCInsideNonInlineFunction() { +#if defined(ABSL_HAVE_ATTRIBUTE_NOINLINE) && \ + (defined(__i386__) || defined(__x86_64__)) + void *pc = non_inline_func(); + const char *symbol = TrySymbolize(pc); + ABSL_RAW_CHECK(symbol != nullptr, "TestWithPCInsideNonInlineFunction failed"); + ABSL_RAW_CHECK(strcmp(symbol, "non_inline_func") == 0, + "TestWithPCInsideNonInlineFunction failed"); + std::cout << "TestWithPCInsideNonInlineFunction passed" << std::endl; +#endif +} + +void ABSL_ATTRIBUTE_NOINLINE TestWithPCInsideInlineFunction() { +#if defined(ABSL_HAVE_ATTRIBUTE_ALWAYS_INLINE) && \ + (defined(__i386__) || defined(__x86_64__)) + void *pc = inline_func(); // Must be inlined. + const char *symbol = TrySymbolize(pc); + ABSL_RAW_CHECK(symbol != nullptr, "TestWithPCInsideInlineFunction failed"); + ABSL_RAW_CHECK(strcmp(symbol, __FUNCTION__) == 0, + "TestWithPCInsideInlineFunction failed"); + std::cout << "TestWithPCInsideInlineFunction passed" << std::endl; +#endif +} +} + +// Test with a return address. +void ABSL_ATTRIBUTE_NOINLINE TestWithReturnAddress() { +#if defined(ABSL_HAVE_ATTRIBUTE_NOINLINE) + void *return_address = __builtin_return_address(0); + const char *symbol = TrySymbolize(return_address); + ABSL_RAW_CHECK(symbol != nullptr, "TestWithReturnAddress failed"); + ABSL_RAW_CHECK(strcmp(symbol, "main") == 0, "TestWithReturnAddress failed"); + std::cout << "TestWithReturnAddress passed" << std::endl; +#endif +} + +#elif defined(_WIN32) +#if !defined(ABSL_CONSUME_DLL) + +TEST(Symbolize, Basics) { + EXPECT_STREQ("nonstatic_func", TrySymbolize((void *)(&nonstatic_func))); + + // The name of an internal linkage symbol is not specified; allow either a + // mangled or an unmangled name here. + const char *static_func_symbol = TrySymbolize((void *)(&static_func)); + ASSERT_TRUE(static_func_symbol != nullptr); + EXPECT_TRUE(strstr(static_func_symbol, "static_func") != nullptr); + + EXPECT_TRUE(nullptr == TrySymbolize(nullptr)); +} + +TEST(Symbolize, Truncation) { + constexpr char kNonStaticFunc[] = "nonstatic_func"; + EXPECT_STREQ("nonstatic_func", + TrySymbolizeWithLimit((void *)(&nonstatic_func), + strlen(kNonStaticFunc) + 1)); + EXPECT_STREQ("nonstatic_...", + TrySymbolizeWithLimit((void *)(&nonstatic_func), + strlen(kNonStaticFunc) + 0)); + EXPECT_STREQ("nonstatic...", + TrySymbolizeWithLimit((void *)(&nonstatic_func), + strlen(kNonStaticFunc) - 1)); + EXPECT_STREQ("n...", TrySymbolizeWithLimit((void *)(&nonstatic_func), 5)); + EXPECT_STREQ("...", TrySymbolizeWithLimit((void *)(&nonstatic_func), 4)); + EXPECT_STREQ("..", TrySymbolizeWithLimit((void *)(&nonstatic_func), 3)); + EXPECT_STREQ(".", TrySymbolizeWithLimit((void *)(&nonstatic_func), 2)); + EXPECT_STREQ("", TrySymbolizeWithLimit((void *)(&nonstatic_func), 1)); + EXPECT_EQ(nullptr, TrySymbolizeWithLimit((void *)(&nonstatic_func), 0)); +} + +TEST(Symbolize, SymbolizeWithDemangling) { + const char *result = TrySymbolize((void *)(&Foo::func)); + ASSERT_TRUE(result != nullptr); + EXPECT_TRUE(strstr(result, "Foo::func") != nullptr) << result; +} + +#endif // !defined(ABSL_CONSUME_DLL) +#else // Symbolizer unimplemented + +TEST(Symbolize, Unimplemented) { + char buf[64]; + EXPECT_FALSE(absl::Symbolize((void *)(&nonstatic_func), buf, sizeof(buf))); + EXPECT_FALSE(absl::Symbolize((void *)(&static_func), buf, sizeof(buf))); + EXPECT_FALSE(absl::Symbolize((void *)(&Foo::func), buf, sizeof(buf))); +} + +#endif + +int main(int argc, char **argv) { +#if !defined(__EMSCRIPTEN__) + // Make sure kHpageTextPadding is linked into the binary. + if (volatile_bool) { + ABSL_RAW_LOG(INFO, "%s", kHpageTextPadding); + } +#endif // !defined(__EMSCRIPTEN__) + +#if ABSL_PER_THREAD_TLS + // Touch the per-thread variables. + symbolize_test_thread_small[0] = 0; + symbolize_test_thread_big[0] = 0; +#endif + + absl::InitializeSymbolizer(argv[0]); + testing::InitGoogleTest(&argc, argv); + +#if defined(ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE) || \ + defined(ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE) + TestWithPCInsideInlineFunction(); + TestWithPCInsideNonInlineFunction(); + TestWithReturnAddress(); +#endif + + return RUN_ALL_TESTS(); +} diff --git a/third_party/abseil_cpp/absl/debugging/symbolize_unimplemented.inc b/third_party/abseil_cpp/absl/debugging/symbolize_unimplemented.inc new file mode 100644 index 000000000000..db24456b0afa --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize_unimplemented.inc @@ -0,0 +1,40 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> + +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace debugging_internal { + +int InstallSymbolDecorator(SymbolDecorator, void*) { return -1; } +bool RemoveSymbolDecorator(int) { return false; } +bool RemoveAllSymbolDecorators(void) { return false; } +bool RegisterFileMappingHint(const void *, const void *, uint64_t, const char *) { + return false; +} +bool GetFileMappingHint(const void **, const void **, uint64_t *, const char **) { + return false; +} + +} // namespace debugging_internal + +void InitializeSymbolizer(const char*) {} +bool Symbolize(const void *, char *, int) { return false; } + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/debugging/symbolize_win32.inc b/third_party/abseil_cpp/absl/debugging/symbolize_win32.inc new file mode 100644 index 000000000000..c3df46f606c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/debugging/symbolize_win32.inc @@ -0,0 +1,81 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// See "Retrieving Symbol Information by Address": +// https://msdn.microsoft.com/en-us/library/windows/desktop/ms680578(v=vs.85).aspx + +#include <windows.h> + +// MSVC header dbghelp.h has a warning for an ignored typedef. +#pragma warning(push) +#pragma warning(disable:4091) +#include <dbghelp.h> +#pragma warning(pop) + +#pragma comment(lib, "dbghelp.lib") + +#include <algorithm> +#include <cstring> + +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +static HANDLE process = NULL; + +void InitializeSymbolizer(const char*) { + if (process != nullptr) { + return; + } + process = GetCurrentProcess(); + + // Symbols are not loaded until a reference is made requiring the + // symbols be loaded. This is the fastest, most efficient way to use + // the symbol handler. + SymSetOptions(SYMOPT_DEFERRED_LOADS | SYMOPT_UNDNAME); + if (!SymInitialize(process, nullptr, true)) { + // GetLastError() returns a Win32 DWORD, but we assign to + // unsigned long long to simplify the ABSL_RAW_LOG case below. The uniform + // initialization guarantees this is not a narrowing conversion. + const unsigned long long error{GetLastError()}; // NOLINT(runtime/int) + ABSL_RAW_LOG(FATAL, "SymInitialize() failed: %llu", error); + } +} + +bool Symbolize(const void* pc, char* out, int out_size) { + if (out_size <= 0) { + return false; + } + alignas(SYMBOL_INFO) char buf[sizeof(SYMBOL_INFO) + MAX_SYM_NAME]; + SYMBOL_INFO* symbol = reinterpret_cast<SYMBOL_INFO*>(buf); + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + symbol->MaxNameLen = MAX_SYM_NAME; + if (!SymFromAddr(process, reinterpret_cast<DWORD64>(pc), nullptr, symbol)) { + return false; + } + strncpy(out, symbol->Name, out_size); + if (out[out_size - 1] != '\0') { + // strncpy() does not '\0' terminate when it truncates. + static constexpr char kEllipsis[] = "..."; + int ellipsis_size = + std::min<int>(sizeof(kEllipsis) - 1, out_size - 1); + memcpy(out + out_size - ellipsis_size - 1, kEllipsis, ellipsis_size); + out[out_size - 1] = '\0'; + } + return true; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/BUILD.bazel b/third_party/abseil_cpp/absl/flags/BUILD.bazel new file mode 100644 index 000000000000..78d6da74d819 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/BUILD.bazel @@ -0,0 +1,514 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "path_util", + hdrs = [ + "internal/path_util.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/flags:__pkg__", + ], + deps = [ + "//absl/base:config", + "//absl/strings", + ], +) + +cc_library( + name = "program_name", + srcs = [ + "internal/program_name.cc", + ], + hdrs = [ + "internal/program_name.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/flags:__pkg__", + ], + deps = [ + ":path_util", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/strings", + "//absl/synchronization", + ], +) + +cc_library( + name = "config", + srcs = [ + "usage_config.cc", + ], + hdrs = [ + "config.h", + "usage_config.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":path_util", + ":program_name", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/strings", + "//absl/synchronization", + ], +) + +cc_library( + name = "marshalling", + srcs = [ + "marshalling.cc", + ], + hdrs = [ + "marshalling.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:log_severity", + "//absl/strings", + "//absl/strings:str_format", + ], +) + +cc_library( + name = "commandlineflag_internal", + srcs = [ + "internal/commandlineflag.cc", + ], + hdrs = [ + "internal/commandlineflag.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:fast_type_id", + ], +) + +cc_library( + name = "commandlineflag", + srcs = [ + "commandlineflag.cc", + ], + hdrs = [ + "commandlineflag.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":commandlineflag_internal", + "//absl/base:config", + "//absl/base:fast_type_id", + "//absl/strings", + "//absl/types:optional", + ], +) + +cc_library( + name = "private_handle_accessor", + srcs = [ + "internal/private_handle_accessor.cc", + ], + hdrs = [ + "internal/private_handle_accessor.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/flags:__pkg__", + ], + deps = [ + ":commandlineflag", + ":commandlineflag_internal", + "//absl/base:config", + "//absl/strings", + ], +) + +cc_library( + name = "reflection", + srcs = [ + "reflection.cc", + ], + hdrs = [ + "internal/registry.h", + "reflection.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":commandlineflag", + ":commandlineflag_internal", + ":config", + ":private_handle_accessor", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/container:flat_hash_map", + "//absl/strings", + "//absl/synchronization", + ], +) + +cc_library( + name = "flag_internal", + srcs = [ + "internal/flag.cc", + ], + hdrs = [ + "internal/flag.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//absl/base:__subpackages__"], + deps = [ + ":commandlineflag", + ":commandlineflag_internal", + ":config", + ":marshalling", + ":reflection", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/synchronization", + "//absl/utility", + ], +) + +cc_library( + name = "flag", + srcs = [ + "flag.cc", + ], + hdrs = [ + "declare.h", + "flag.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":flag_internal", + ":reflection", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/strings", + ], +) + +cc_library( + name = "usage_internal", + srcs = [ + "internal/usage.cc", + ], + hdrs = [ + "internal/usage.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/flags:__pkg__", + ], + deps = [ + ":commandlineflag", + ":config", + ":flag", + ":flag_internal", + ":path_util", + ":private_handle_accessor", + ":program_name", + ":reflection", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/strings", + ], +) + +cc_library( + name = "usage", + srcs = [ + "usage.cc", + ], + hdrs = [ + "usage.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":usage_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/strings", + "//absl/synchronization", + ], +) + +cc_library( + name = "parse", + srcs = ["parse.cc"], + hdrs = [ + "internal/parse.h", + "parse.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":commandlineflag", + ":commandlineflag_internal", + ":config", + ":flag", + ":flag_internal", + ":private_handle_accessor", + ":program_name", + ":reflection", + ":usage", + ":usage_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/strings", + "//absl/synchronization", + ], +) + +############################################################################ +# Unit tests in alphabetical order. + +cc_test( + name = "commandlineflag_test", + size = "small", + srcs = [ + "commandlineflag_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":commandlineflag", + ":commandlineflag_internal", + ":config", + ":flag", + ":private_handle_accessor", + ":reflection", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "config_test", + size = "small", + srcs = [ + "config_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "flag_test", + size = "small", + srcs = [ + "flag_test.cc", + "flag_test_defs.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":flag", + ":flag_internal", + ":marshalling", + ":reflection", + "//absl/base:core_headers", + "//absl/base:malloc_internal", + "//absl/strings", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "flag_benchmark", + testonly = 1, + srcs = [ + "flag_benchmark.cc", + ], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":flag", + ":marshalling", + ":parse", + ":reflection", + "//absl/strings", + "//absl/time", + "//absl/types:optional", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "marshalling_test", + size = "small", + srcs = [ + "marshalling_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":marshalling", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "parse_test", + size = "small", + srcs = [ + "parse_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":flag", + ":parse", + ":reflection", + ":usage_internal", + "//absl/base:raw_logging_internal", + "//absl/base:scoped_set_env", + "//absl/strings", + "//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "path_util_test", + size = "small", + srcs = [ + "internal/path_util_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":path_util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "program_name_test", + size = "small", + srcs = [ + "internal/program_name_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":program_name", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "reflection_test", + size = "small", + srcs = [ + "reflection_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":commandlineflag_internal", + ":flag", + ":marshalling", + ":reflection", + ":usage_internal", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "usage_config_test", + size = "small", + srcs = [ + "usage_config_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":path_util", + ":program_name", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "usage_test", + size = "small", + srcs = [ + "internal/usage_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":config", + ":flag", + ":parse", + ":path_util", + ":program_name", + ":reflection", + ":usage", + ":usage_internal", + "//absl/strings", + "@com_google_googletest//:gtest", + ], +) diff --git a/third_party/abseil_cpp/absl/flags/CMakeLists.txt b/third_party/abseil_cpp/absl/flags/CMakeLists.txt new file mode 100644 index 000000000000..e5083d7b9a87 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/CMakeLists.txt @@ -0,0 +1,451 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_path_util + HDRS + "internal/path_util.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::strings + PUBLIC +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_program_name + SRCS + "internal/program_name.cc" + HDRS + "internal/program_name.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_path_util + absl::strings + absl::synchronization + PUBLIC +) + +absl_cc_library( + NAME + flags_config + SRCS + "usage_config.cc" + HDRS + "config.h" + "usage_config.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_path_util + absl::flags_program_name + absl::core_headers + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_marshalling + SRCS + "marshalling.cc" + HDRS + "marshalling.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::log_severity + absl::strings + absl::str_format +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_commandlineflag_internal + SRCS + "internal/commandlineflag.cc" + HDRS + "internal/commandlineflag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::fast_type_id +) + +absl_cc_library( + NAME + flags_commandlineflag + SRCS + "commandlineflag.cc" + HDRS + "commandlineflag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::fast_type_id + absl::flags_commandlineflag_internal + absl::optional + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_private_handle_accessor + SRCS + "internal/private_handle_accessor.cc" + HDRS + "internal/private_handle_accessor.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::strings +) + +absl_cc_library( + NAME + flags_reflection + SRCS + "reflection.cc" + HDRS + "reflection.h" + "internal/registry.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_private_handle_accessor + absl::flags_config + absl::strings + absl::synchronization + absl::flat_hash_map +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_internal + SRCS + "internal/flag.cc" + HDRS + "internal/flag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_marshalling + absl::synchronization + absl::meta + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + flags + SRCS + "flag.cc" + HDRS + "declare.h" + "flag.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_commandlineflag + absl::flags_config + absl::flags_internal + absl::flags_reflection + absl::base + absl::core_headers + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + flags_usage_internal + SRCS + "internal/usage.cc" + HDRS + "internal/usage.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::flags_config + absl::flags + absl::flags_commandlineflag + absl::flags_internal + absl::flags_path_util + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_usage + SRCS + "usage.cc" + HDRS + "usage.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_usage_internal + absl::strings + absl::synchronization +) + +absl_cc_library( + NAME + flags_parse + SRCS + "parse.cc" + HDRS + "internal/parse.h" + "parse.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::flags_config + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_internal + absl::flags_private_handle_accessor + absl::flags_program_name + absl::flags_reflection + absl::flags_usage + absl::strings + absl::synchronization +) + +############################################################################ +# Unit tests in alpahabetical order. + +absl_cc_test( + NAME + flags_commandlineflag_test + SRCS + "commandlineflag_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags + absl::flags_commandlineflag + absl::flags_commandlineflag_internal + absl::flags_config + absl::flags_private_handle_accessor + absl::flags_reflection + absl::memory + absl::strings + gtest_main +) + +absl_cc_test( + NAME + flags_config_test + SRCS + "config_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_config + gtest_main +) + +absl_cc_test( + NAME + flags_flag_test + SRCS + "flag_test.cc" + "flag_test_defs.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::core_headers + absl::flags + absl::flags_config + absl::flags_internal + absl::flags_marshalling + absl::flags_reflection + absl::strings + absl::time + gtest_main +) + +absl_cc_test( + NAME + flags_marshalling_test + SRCS + "marshalling_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_marshalling + gtest_main +) + +absl_cc_test( + NAME + flags_parse_test + SRCS + "parse_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags + absl::flags_parse + absl::flags_reflection + absl::flags_usage_internal + absl::raw_logging_internal + absl::scoped_set_env + absl::span + absl::strings + gmock_main +) + +absl_cc_test( + NAME + flags_path_util_test + SRCS + "internal/path_util_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_path_util + gtest_main +) + +absl_cc_test( + NAME + flags_program_name_test + SRCS + "internal/program_name_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_program_name + absl::strings + gtest_main +) + +absl_cc_test( + NAME + flags_reflection_test + SRCS + "reflection_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_commandlineflag_internal + absl::flags + absl::flags_reflection + absl::flags_usage + absl::memory + absl::strings + gmock_main +) + +absl_cc_test( + NAME + flags_usage_config_test + SRCS + "usage_config_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_config + absl::flags_path_util + absl::flags_program_name + absl::strings + gtest_main +) + +absl_cc_test( + NAME + flags_usage_test + SRCS + "internal/usage_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flags_config + absl::flags + absl::flags_path_util + absl::flags_program_name + absl::flags_parse + absl::flags_reflection + absl::flags_usage + absl::strings + gtest +) diff --git a/third_party/abseil_cpp/absl/flags/commandlineflag.cc b/third_party/abseil_cpp/absl/flags/commandlineflag.cc new file mode 100644 index 000000000000..9f3b4a5a28ef --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/commandlineflag.cc @@ -0,0 +1,34 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/commandlineflag.h" + +#include <string> + +#include "absl/base/config.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +bool CommandLineFlag::IsRetired() const { return false; } +bool CommandLineFlag::ParseFrom(absl::string_view value, std::string* error) { + return ParseFrom(value, flags_internal::SET_FLAGS_VALUE, + flags_internal::kProgrammaticChange, *error); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/commandlineflag.h b/third_party/abseil_cpp/absl/flags/commandlineflag.h new file mode 100644 index 000000000000..f2fa08977fd4 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/commandlineflag.h @@ -0,0 +1,200 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: commandlineflag.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `CommandLineFlag`, which acts as a type-erased +// handle for accessing metadata about the Abseil Flag in question. +// +// Because an actual Abseil flag is of an unspecified type, you should not +// manipulate or interact directly with objects of that type. Instead, use the +// CommandLineFlag type as an intermediary. +#ifndef ABSL_FLAGS_COMMANDLINEFLAG_H_ +#define ABSL_FLAGS_COMMANDLINEFLAG_H_ + +#include <memory> +#include <string> + +#include "absl/base/config.h" +#include "absl/base/internal/fast_type_id.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { +class PrivateHandleAccessor; +} // namespace flags_internal + +// CommandLineFlag +// +// This type acts as a type-erased handle for an instance of an Abseil Flag and +// holds reflection information pertaining to that flag. Use CommandLineFlag to +// access a flag's name, location, help string etc. +// +// To obtain an absl::CommandLineFlag, invoke `absl::FindCommandLineFlag()` +// passing it the flag name string. +// +// Example: +// +// // Obtain reflection handle for a flag named "flagname". +// const absl::CommandLineFlag* my_flag_data = +// absl::FindCommandLineFlag("flagname"); +// +// // Now you can get flag info from that reflection handle. +// std::string flag_location = my_flag_data->Filename(); +// ... +class CommandLineFlag { + public: + constexpr CommandLineFlag() = default; + + // Not copyable/assignable. + CommandLineFlag(const CommandLineFlag&) = delete; + CommandLineFlag& operator=(const CommandLineFlag&) = delete; + + // absl::CommandLineFlag::IsOfType() + // + // Return true iff flag has type T. + template <typename T> + inline bool IsOfType() const { + return TypeId() == base_internal::FastTypeId<T>(); + } + + // absl::CommandLineFlag::TryGet() + // + // Attempts to retrieve the flag value. Returns value on success, + // absl::nullopt otherwise. + template <typename T> + absl::optional<T> TryGet() const { + if (IsRetired() || !IsOfType<T>()) { + return absl::nullopt; + } + + // Implementation notes: + // + // We are wrapping a union around the value of `T` to serve three purposes: + // + // 1. `U.value` has correct size and alignment for a value of type `T` + // 2. The `U.value` constructor is not invoked since U's constructor does + // not do it explicitly. + // 3. The `U.value` destructor is invoked since U's destructor does it + // explicitly. This makes `U` a kind of RAII wrapper around non default + // constructible value of T, which is destructed when we leave the + // scope. We do need to destroy U.value, which is constructed by + // CommandLineFlag::Read even though we left it in a moved-from state + // after std::move. + // + // All of this serves to avoid requiring `T` being default constructible. + union U { + T value; + U() {} + ~U() { value.~T(); } + }; + U u; + + Read(&u.value); + // allow retired flags to be "read", so we can report invalid access. + if (IsRetired()) { + return absl::nullopt; + } + return std::move(u.value); + } + + // absl::CommandLineFlag::Name() + // + // Returns name of this flag. + virtual absl::string_view Name() const = 0; + + // absl::CommandLineFlag::Filename() + // + // Returns name of the file where this flag is defined. + virtual std::string Filename() const = 0; + + // absl::CommandLineFlag::Help() + // + // Returns help message associated with this flag. + virtual std::string Help() const = 0; + + // absl::CommandLineFlag::IsRetired() + // + // Returns true iff this object corresponds to retired flag. + virtual bool IsRetired() const; + + // absl::CommandLineFlag::DefaultValue() + // + // Returns the default value for this flag. + virtual std::string DefaultValue() const = 0; + + // absl::CommandLineFlag::CurrentValue() + // + // Returns the current value for this flag. + virtual std::string CurrentValue() const = 0; + + // absl::CommandLineFlag::ParseFrom() + // + // Sets the value of the flag based on specified string `value`. If the flag + // was successfully set to new value, it returns true. Otherwise, sets `error` + // to indicate the error, leaves the flag unchanged, and returns false. + bool ParseFrom(absl::string_view value, std::string* error); + + protected: + ~CommandLineFlag() = default; + + private: + friend class flags_internal::PrivateHandleAccessor; + + // Sets the value of the flag based on specified string `value`. If the flag + // was successfully set to new value, it returns true. Otherwise, sets `error` + // to indicate the error, leaves the flag unchanged, and returns false. There + // are three ways to set the flag's value: + // * Update the current flag value + // * Update the flag's default value + // * Update the current flag value if it was never set before + // The mode is selected based on `set_mode` parameter. + virtual bool ParseFrom(absl::string_view value, + flags_internal::FlagSettingMode set_mode, + flags_internal::ValueSource source, + std::string& error) = 0; + + // Returns id of the flag's value type. + virtual flags_internal::FlagFastTypeId TypeId() const = 0; + + // Interface to save flag to some persistent state. Returns current flag state + // or nullptr if flag does not support saving and restoring a state. + virtual std::unique_ptr<flags_internal::FlagStateInterface> SaveState() = 0; + + // Copy-construct a new value of the flag's type in a memory referenced by + // the dst based on the current flag's value. + virtual void Read(void* dst) const = 0; + + // To be deleted. Used to return true if flag's current value originated from + // command line. + virtual bool IsSpecifiedOnCommandLine() const = 0; + + // Validates supplied value usign validator or parseflag routine + virtual bool ValidateInputValue(absl::string_view value) const = 0; + + // Checks that flags default value can be converted to string and back to the + // flag's value type. + virtual void CheckDefaultValueParsingRoundtrip() const = 0; +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_COMMANDLINEFLAG_H_ diff --git a/third_party/abseil_cpp/absl/flags/commandlineflag_test.cc b/third_party/abseil_cpp/absl/flags/commandlineflag_test.cc new file mode 100644 index 000000000000..585db4ba78a0 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/commandlineflag_test.cc @@ -0,0 +1,231 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/commandlineflag.h" + +#include <memory> +#include <string> + +#include "gtest/gtest.h" +#include "absl/flags/flag.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/flags/internal/private_handle_accessor.h" +#include "absl/flags/reflection.h" +#include "absl/flags/usage_config.h" +#include "absl/memory/memory.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +ABSL_FLAG(int, int_flag, 201, "int_flag help"); +ABSL_FLAG(std::string, string_flag, "dflt", + absl::StrCat("string_flag", " help")); +ABSL_RETIRED_FLAG(bool, bool_retired_flag, false, "bool_retired_flag help"); + +// These are only used to test default values. +ABSL_FLAG(int, int_flag2, 201, ""); +ABSL_FLAG(std::string, string_flag2, "dflt", ""); + +namespace { + +namespace flags = absl::flags_internal; + +class CommandLineFlagTest : public testing::Test { + protected: + static void SetUpTestSuite() { + // Install a function to normalize filenames before this test is run. + absl::FlagsUsageConfig default_config; + default_config.normalize_filename = &CommandLineFlagTest::NormalizeFileName; + absl::SetFlagsUsageConfig(default_config); + } + + void SetUp() override { flag_saver_ = absl::make_unique<absl::FlagSaver>(); } + void TearDown() override { flag_saver_.reset(); } + + private: + static std::string NormalizeFileName(absl::string_view fname) { +#ifdef _WIN32 + std::string normalized(fname); + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + fname = normalized; +#endif + return std::string(fname); + } + + std::unique_ptr<absl::FlagSaver> flag_saver_; +}; + +TEST_F(CommandLineFlagTest, TestAttributesAccessMethods) { + auto* flag_01 = absl::FindCommandLineFlag("int_flag"); + + ASSERT_TRUE(flag_01); + EXPECT_EQ(flag_01->Name(), "int_flag"); + EXPECT_EQ(flag_01->Help(), "int_flag help"); + EXPECT_TRUE(!flag_01->IsRetired()); + EXPECT_TRUE(flag_01->IsOfType<int>()); + EXPECT_TRUE(!flag_01->IsOfType<bool>()); + EXPECT_TRUE(!flag_01->IsOfType<std::string>()); + EXPECT_TRUE(absl::EndsWith(flag_01->Filename(), + "absl/flags/commandlineflag_test.cc")) + << flag_01->Filename(); + + auto* flag_02 = absl::FindCommandLineFlag("string_flag"); + + ASSERT_TRUE(flag_02); + EXPECT_EQ(flag_02->Name(), "string_flag"); + EXPECT_EQ(flag_02->Help(), "string_flag help"); + EXPECT_TRUE(!flag_02->IsRetired()); + EXPECT_TRUE(flag_02->IsOfType<std::string>()); + EXPECT_TRUE(!flag_02->IsOfType<bool>()); + EXPECT_TRUE(!flag_02->IsOfType<int>()); + EXPECT_TRUE(absl::EndsWith(flag_02->Filename(), + "absl/flags/commandlineflag_test.cc")) + << flag_02->Filename(); +} + +// -------------------------------------------------------------------- + +TEST_F(CommandLineFlagTest, TestValueAccessMethods) { + absl::SetFlag(&FLAGS_int_flag2, 301); + auto* flag_01 = absl::FindCommandLineFlag("int_flag2"); + + ASSERT_TRUE(flag_01); + EXPECT_EQ(flag_01->CurrentValue(), "301"); + EXPECT_EQ(flag_01->DefaultValue(), "201"); + + absl::SetFlag(&FLAGS_string_flag2, "new_str_value"); + auto* flag_02 = absl::FindCommandLineFlag("string_flag2"); + + ASSERT_TRUE(flag_02); + EXPECT_EQ(flag_02->CurrentValue(), "new_str_value"); + EXPECT_EQ(flag_02->DefaultValue(), "dflt"); +} + +// -------------------------------------------------------------------- + +TEST_F(CommandLineFlagTest, TestParseFromCurrentValue) { + std::string err; + + auto* flag_01 = absl::FindCommandLineFlag("int_flag"); + EXPECT_FALSE( + flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "11", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 11); + EXPECT_FALSE( + flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "-123", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, + err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), -123); + EXPECT_FALSE( + flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(!flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "xyz", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, + err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), -123); + EXPECT_EQ(err, "Illegal value 'xyz' specified for flag 'int_flag'"); + EXPECT_FALSE( + flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(!flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "A1", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), -123); + EXPECT_EQ(err, "Illegal value 'A1' specified for flag 'int_flag'"); + EXPECT_FALSE( + flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "0x10", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, + err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 16); + EXPECT_FALSE( + flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "011", flags::SET_FLAGS_VALUE, flags::kCommandLine, err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 11); + EXPECT_TRUE(flags::PrivateHandleAccessor::IsSpecifiedOnCommandLine(*flag_01)); + + EXPECT_TRUE(!flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, err)); + EXPECT_EQ(err, "Illegal value '' specified for flag 'int_flag'"); + + auto* flag_02 = absl::FindCommandLineFlag("string_flag"); + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_02, "xyz", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, + err)); + EXPECT_EQ(absl::GetFlag(FLAGS_string_flag), "xyz"); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_02, "", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, err)); + EXPECT_EQ(absl::GetFlag(FLAGS_string_flag), ""); +} + +// -------------------------------------------------------------------- + +TEST_F(CommandLineFlagTest, TestParseFromDefaultValue) { + std::string err; + + auto* flag_01 = absl::FindCommandLineFlag("int_flag"); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "111", flags::SET_FLAGS_DEFAULT, flags::kProgrammaticChange, + err)); + EXPECT_EQ(flag_01->DefaultValue(), "111"); + + auto* flag_02 = absl::FindCommandLineFlag("string_flag"); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_02, "abc", flags::SET_FLAGS_DEFAULT, flags::kProgrammaticChange, + err)); + EXPECT_EQ(flag_02->DefaultValue(), "abc"); +} + +// -------------------------------------------------------------------- + +TEST_F(CommandLineFlagTest, TestParseFromIfDefault) { + std::string err; + + auto* flag_01 = absl::FindCommandLineFlag("int_flag"); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "22", flags::SET_FLAG_IF_DEFAULT, flags::kProgrammaticChange, + err)) + << err; + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 22); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "33", flags::SET_FLAG_IF_DEFAULT, flags::kProgrammaticChange, + err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 22); + // EXPECT_EQ(err, "ERROR: int_flag is already set to 22"); + + // Reset back to default value + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "201", flags::SET_FLAGS_VALUE, flags::kProgrammaticChange, + err)); + + EXPECT_TRUE(flags::PrivateHandleAccessor::ParseFrom( + *flag_01, "33", flags::SET_FLAG_IF_DEFAULT, flags::kProgrammaticChange, + err)); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 201); + // EXPECT_EQ(err, "ERROR: int_flag is already set to 201"); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/config.h b/third_party/abseil_cpp/absl/flags/config.h new file mode 100644 index 000000000000..813a9257000f --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/config.h @@ -0,0 +1,87 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_CONFIG_H_ +#define ABSL_FLAGS_CONFIG_H_ + +// Determine if we should strip string literals from the Flag objects. +// By default we strip string literals on mobile platforms. +#if !defined(ABSL_FLAGS_STRIP_NAMES) + +#if defined(__ANDROID__) +#define ABSL_FLAGS_STRIP_NAMES 1 + +#elif defined(__APPLE__) +#include <TargetConditionals.h> +#if defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE +#define ABSL_FLAGS_STRIP_NAMES 1 +#elif defined(TARGET_OS_EMBEDDED) && TARGET_OS_EMBEDDED +#define ABSL_FLAGS_STRIP_NAMES 1 +#endif // TARGET_OS_* +#endif + +#endif // !defined(ABSL_FLAGS_STRIP_NAMES) + +#if !defined(ABSL_FLAGS_STRIP_NAMES) +// If ABSL_FLAGS_STRIP_NAMES wasn't set on the command line or above, +// the default is not to strip. +#define ABSL_FLAGS_STRIP_NAMES 0 +#endif + +#if !defined(ABSL_FLAGS_STRIP_HELP) +// By default, if we strip names, we also strip help. +#define ABSL_FLAGS_STRIP_HELP ABSL_FLAGS_STRIP_NAMES +#endif + +// ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD macro is used for using atomics with +// double words, e.g. absl::Duration. +// For reasons in bug https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80878, modern +// versions of GCC do not support cmpxchg16b instruction in standard atomics. +#ifdef ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD +#error "ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD should not be defined." +#elif defined(__clang__) && defined(__x86_64__) && \ + defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) +#define ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD 1 +#endif + +// ABSL_FLAGS_INTERNAL_HAS_RTTI macro is used for selecting if we can use RTTI +// for flag type identification. +#ifdef ABSL_FLAGS_INTERNAL_HAS_RTTI +#error ABSL_FLAGS_INTERNAL_HAS_RTTI cannot be directly set +#elif !defined(__GNUC__) || defined(__GXX_RTTI) +#define ABSL_FLAGS_INTERNAL_HAS_RTTI 1 +#endif // !defined(__GNUC__) || defined(__GXX_RTTI) + +// These macros represent the "source of truth" for the list of supported +// built-in types. +#define ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(A) \ + A(bool, bool) \ + A(short, short) \ + A(unsigned short, unsigned_short) \ + A(int, int) \ + A(unsigned int, unsigned_int) \ + A(long, long) \ + A(unsigned long, unsigned_long) \ + A(long long, long_long) \ + A(unsigned long long, unsigned_long_long) \ + A(double, double) \ + A(float, float) + +#define ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(A) \ + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(A) \ + A(std::string, std_string) \ + A(std::vector<std::string>, std_vector_of_string) + +#endif // ABSL_FLAGS_CONFIG_H_ diff --git a/third_party/abseil_cpp/absl/flags/config_test.cc b/third_party/abseil_cpp/absl/flags/config_test.cc new file mode 100644 index 000000000000..638998667e70 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/config_test.cc @@ -0,0 +1,61 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/config.h" + +#ifdef __APPLE__ +#include <TargetConditionals.h> +#endif + +#include "gtest/gtest.h" + +#ifndef ABSL_FLAGS_STRIP_NAMES +#error ABSL_FLAGS_STRIP_NAMES is not defined +#endif + +#ifndef ABSL_FLAGS_STRIP_HELP +#error ABSL_FLAGS_STRIP_HELP is not defined +#endif + +namespace { + +// Test that ABSL_FLAGS_STRIP_NAMES and ABSL_FLAGS_STRIP_HELP are configured how +// we expect them to be configured by default. If you override this +// configuration, this test will fail, but the code should still be safe to use. +TEST(FlagsConfigTest, Test) { +#if defined(__ANDROID__) + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 1); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 1); +#elif defined(__myriad2__) + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 0); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 0); +#elif defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 1); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 1); +#elif defined(TARGET_OS_EMBEDDED) && TARGET_OS_EMBEDDED + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 1); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 1); +#elif defined(__APPLE__) + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 0); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 0); +#elif defined(_WIN32) + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 0); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 0); +#elif defined(__linux__) + EXPECT_EQ(ABSL_FLAGS_STRIP_NAMES, 0); + EXPECT_EQ(ABSL_FLAGS_STRIP_HELP, 0); +#endif +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/declare.h b/third_party/abseil_cpp/absl/flags/declare.h new file mode 100644 index 000000000000..b9794d8b85ac --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/declare.h @@ -0,0 +1,65 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: declare.h +// ----------------------------------------------------------------------------- +// +// This file defines the ABSL_DECLARE_FLAG macro, allowing you to declare an +// `absl::Flag` for use within a translation unit. You should place this +// declaration within the header file associated with the .cc file that defines +// and owns the `Flag`. + +#ifndef ABSL_FLAGS_DECLARE_H_ +#define ABSL_FLAGS_DECLARE_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// absl::Flag<T> represents a flag of type 'T' created by ABSL_FLAG. +template <typename T> +class Flag; + +} // namespace flags_internal + +// Flag +// +// Forward declaration of the `absl::Flag` type for use in defining the macro. +#if defined(_MSC_VER) && !defined(__clang__) +template <typename T> +class Flag; +#else +template <typename T> +using Flag = flags_internal::Flag<T>; +#endif + +ABSL_NAMESPACE_END +} // namespace absl + +// ABSL_DECLARE_FLAG() +// +// This macro is a convenience for declaring use of an `absl::Flag` within a +// translation unit. This macro should be used within a header file to +// declare usage of the flag within any .cc file including that header file. +// +// The ABSL_DECLARE_FLAG(type, name) macro expands to: +// +// extern absl::Flag<type> FLAGS_name; +#define ABSL_DECLARE_FLAG(type, name) extern ::absl::Flag<type> FLAGS_##name + +#endif // ABSL_FLAGS_DECLARE_H_ diff --git a/third_party/abseil_cpp/absl/flags/flag.cc b/third_party/abseil_cpp/absl/flags/flag.cc new file mode 100644 index 000000000000..531df1287a17 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/flag.cc @@ -0,0 +1,38 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/flag.h" + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// This global mutex protects on-demand construction of flag objects in MSVC +// builds. +#if defined(_MSC_VER) && !defined(__clang__) + +namespace flags_internal { + +ABSL_CONST_INIT static absl::Mutex construction_guard(absl::kConstInit); + +absl::Mutex* GetGlobalConstructionGuard() { return &construction_guard; } + +} // namespace flags_internal + +#endif + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/flag.h b/third_party/abseil_cpp/absl/flags/flag.h new file mode 100644 index 000000000000..a9cb2b7994d1 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/flag.h @@ -0,0 +1,396 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: flag.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::Flag<T>` type for holding command-line +// flag data, and abstractions to create, get and set such flag data. +// +// It is important to note that this type is **unspecified** (an implementation +// detail) and you do not construct or manipulate actual `absl::Flag<T>` +// instances. Instead, you define and declare flags using the +// `ABSL_FLAG()` and `ABSL_DECLARE_FLAG()` macros, and get and set flag values +// using the `absl::GetFlag()` and `absl::SetFlag()` functions. + +#ifndef ABSL_FLAGS_FLAG_H_ +#define ABSL_FLAGS_FLAG_H_ + +#include <string> +#include <type_traits> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/flags/config.h" +#include "absl/flags/internal/flag.h" +#include "absl/flags/internal/registry.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Flag +// +// An `absl::Flag` holds a command-line flag value, providing a runtime +// parameter to a binary. Such flags should be defined in the global namespace +// and (preferably) in the module containing the binary's `main()` function. +// +// You should not construct and cannot use the `absl::Flag` type directly; +// instead, you should declare flags using the `ABSL_DECLARE_FLAG()` macro +// within a header file, and define your flag using `ABSL_FLAG()` within your +// header's associated `.cc` file. Such flags will be named `FLAGS_name`. +// +// Example: +// +// .h file +// +// // Declares usage of a flag named "FLAGS_count" +// ABSL_DECLARE_FLAG(int, count); +// +// .cc file +// +// // Defines a flag named "FLAGS_count" with a default `int` value of 0. +// ABSL_FLAG(int, count, 0, "Count of items to process"); +// +// No public methods of `absl::Flag<T>` are part of the Abseil Flags API. +#if !defined(_MSC_VER) || defined(__clang__) +template <typename T> +using Flag = flags_internal::Flag<T>; +#else +// MSVC debug builds do not implement initialization with constexpr constructors +// correctly. To work around this we add a level of indirection, so that the +// class `absl::Flag` contains an `internal::Flag*` (instead of being an alias +// to that class) and dynamically allocates an instance when necessary. We also +// forward all calls to internal::Flag methods via trampoline methods. In this +// setup the `absl::Flag` class does not have constructor and virtual methods, +// all the data members are public and thus MSVC is able to initialize it at +// link time. To deal with multiple threads accessing the flag for the first +// time concurrently we use an atomic boolean indicating if flag object is +// initialized. We also employ the double-checked locking pattern where the +// second level of protection is a global Mutex, so if two threads attempt to +// construct the flag concurrently only one wins. +// This solution is based on a recomendation here: +// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html?childToView=648454#comment-648454 + +namespace flags_internal { +absl::Mutex* GetGlobalConstructionGuard(); +} // namespace flags_internal + +template <typename T> +class Flag { + public: + // No constructor and destructor to ensure this is an aggregate type. + // Visual Studio 2015 still requires the constructor for class to be + // constexpr initializable. +#if _MSC_VER <= 1900 + constexpr Flag(const char* name, const char* filename, + const flags_internal::HelpGenFunc help_gen, + const flags_internal::FlagDfltGenFunc default_value_gen) + : name_(name), + filename_(filename), + help_gen_(help_gen), + default_value_gen_(default_value_gen), + inited_(false), + impl_(nullptr) {} +#endif + + flags_internal::Flag<T>& GetImpl() const { + if (!inited_.load(std::memory_order_acquire)) { + absl::MutexLock l(flags_internal::GetGlobalConstructionGuard()); + + if (inited_.load(std::memory_order_acquire)) { + return *impl_; + } + + impl_ = new flags_internal::Flag<T>( + name_, filename_, + {flags_internal::FlagHelpMsg(help_gen_), + flags_internal::FlagHelpKind::kGenFunc}, + {flags_internal::FlagDefaultSrc(default_value_gen_), + flags_internal::FlagDefaultKind::kGenFunc}); + inited_.store(true, std::memory_order_release); + } + + return *impl_; + } + + // Public methods of `absl::Flag<T>` are NOT part of the Abseil Flags API. + // See https://abseil.io/docs/cpp/guides/flags + bool IsRetired() const { return GetImpl().IsRetired(); } + absl::string_view Name() const { return GetImpl().Name(); } + std::string Help() const { return GetImpl().Help(); } + bool IsModified() const { return GetImpl().IsModified(); } + bool IsSpecifiedOnCommandLine() const { + return GetImpl().IsSpecifiedOnCommandLine(); + } + std::string Filename() const { return GetImpl().Filename(); } + std::string DefaultValue() const { return GetImpl().DefaultValue(); } + std::string CurrentValue() const { return GetImpl().CurrentValue(); } + template <typename U> + inline bool IsOfType() const { + return GetImpl().template IsOfType<U>(); + } + T Get() const { + return flags_internal::FlagImplPeer::InvokeGet<T>(GetImpl()); + } + void Set(const T& v) { + flags_internal::FlagImplPeer::InvokeSet(GetImpl(), v); + } + void InvokeCallback() { GetImpl().InvokeCallback(); } + + const CommandLineFlag& Reflect() const { + return flags_internal::FlagImplPeer::InvokeReflect(GetImpl()); + } + + // The data members are logically private, but they need to be public for + // this to be an aggregate type. + const char* name_; + const char* filename_; + const flags_internal::HelpGenFunc help_gen_; + const flags_internal::FlagDfltGenFunc default_value_gen_; + + mutable std::atomic<bool> inited_; + mutable flags_internal::Flag<T>* impl_; +}; +#endif + +// GetFlag() +// +// Returns the value (of type `T`) of an `absl::Flag<T>` instance, by value. Do +// not construct an `absl::Flag<T>` directly and call `absl::GetFlag()`; +// instead, refer to flag's constructed variable name (e.g. `FLAGS_name`). +// Because this function returns by value and not by reference, it is +// thread-safe, but note that the operation may be expensive; as a result, avoid +// `absl::GetFlag()` within any tight loops. +// +// Example: +// +// // FLAGS_count is a Flag of type `int` +// int my_count = absl::GetFlag(FLAGS_count); +// +// // FLAGS_firstname is a Flag of type `std::string` +// std::string first_name = absl::GetFlag(FLAGS_firstname); +template <typename T> +ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag) { + return flags_internal::FlagImplPeer::InvokeGet<T>(flag); +} + +// SetFlag() +// +// Sets the value of an `absl::Flag` to the value `v`. Do not construct an +// `absl::Flag<T>` directly and call `absl::SetFlag()`; instead, use the +// flag's variable name (e.g. `FLAGS_name`). This function is +// thread-safe, but is potentially expensive. Avoid setting flags in general, +// but especially within performance-critical code. +template <typename T> +void SetFlag(absl::Flag<T>* flag, const T& v) { + flags_internal::FlagImplPeer::InvokeSet(*flag, v); +} + +// Overload of `SetFlag()` to allow callers to pass in a value that is +// convertible to `T`. E.g., use this overload to pass a "const char*" when `T` +// is `std::string`. +template <typename T, typename V> +void SetFlag(absl::Flag<T>* flag, const V& v) { + T value(v); + flags_internal::FlagImplPeer::InvokeSet(*flag, value); +} + +// GetFlagReflectionHandle() +// +// Returns the reflection handle corresponding to specified Abseil Flag +// instance. Use this handle to access flag's reflection information, like name, +// location, default value etc. +// +// Example: +// +// std::string = absl::GetFlagReflectionHandle(FLAGS_count).DefaultValue(); + +template <typename T> +const CommandLineFlag& GetFlagReflectionHandle(const absl::Flag<T>& f) { + return flags_internal::FlagImplPeer::InvokeReflect(f); +} + +ABSL_NAMESPACE_END +} // namespace absl + + +// ABSL_FLAG() +// +// This macro defines an `absl::Flag<T>` instance of a specified type `T`: +// +// ABSL_FLAG(T, name, default_value, help); +// +// where: +// +// * `T` is a supported flag type (see the list of types in `marshalling.h`), +// * `name` designates the name of the flag (as a global variable +// `FLAGS_name`), +// * `default_value` is an expression holding the default value for this flag +// (which must be implicitly convertible to `T`), +// * `help` is the help text, which can also be an expression. +// +// This macro expands to a flag named 'FLAGS_name' of type 'T': +// +// absl::Flag<T> FLAGS_name = ...; +// +// Note that all such instances are created as global variables. +// +// For `ABSL_FLAG()` values that you wish to expose to other translation units, +// it is recommended to define those flags within the `.cc` file associated with +// the header where the flag is declared. +// +// Note: do not construct objects of type `absl::Flag<T>` directly. Only use the +// `ABSL_FLAG()` macro for such construction. +#define ABSL_FLAG(Type, name, default_value, help) \ + ABSL_FLAG_IMPL(Type, name, default_value, help) + +// ABSL_FLAG().OnUpdate() +// +// Defines a flag of type `T` with a callback attached: +// +// ABSL_FLAG(T, name, default_value, help).OnUpdate(callback); +// +// After any setting of the flag value, the callback will be called at least +// once. A rapid sequence of changes may be merged together into the same +// callback. No concurrent calls to the callback will be made for the same +// flag. Callbacks are allowed to read the current value of the flag but must +// not mutate that flag. +// +// The update mechanism guarantees "eventual consistency"; if the callback +// derives an auxiliary data structure from the flag value, it is guaranteed +// that eventually the flag value and the derived data structure will be +// consistent. +// +// Note: ABSL_FLAG.OnUpdate() does not have a public definition. Hence, this +// comment serves as its API documentation. + + +// ----------------------------------------------------------------------------- +// Implementation details below this section +// ----------------------------------------------------------------------------- + +// ABSL_FLAG_IMPL macro definition conditional on ABSL_FLAGS_STRIP_NAMES +#if !defined(_MSC_VER) || defined(__clang__) +#define ABSL_FLAG_IMPL_FLAG_PTR(flag) flag +#define ABSL_FLAG_IMPL_HELP_ARG(name) \ + absl::flags_internal::HelpArg<AbslFlagHelpGenFor##name>( \ + FLAGS_help_storage_##name) +#define ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name) \ + absl::flags_internal::DefaultArg<Type, AbslFlagDefaultGenFor##name>(0) +#else +#define ABSL_FLAG_IMPL_FLAG_PTR(flag) flag.GetImpl() +#define ABSL_FLAG_IMPL_HELP_ARG(name) &AbslFlagHelpGenFor##name::NonConst +#define ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name) &AbslFlagDefaultGenFor##name::Gen +#endif + +#if ABSL_FLAGS_STRIP_NAMES +#define ABSL_FLAG_IMPL_FLAGNAME(txt) "" +#define ABSL_FLAG_IMPL_FILENAME() "" +#define ABSL_FLAG_IMPL_REGISTRAR(T, flag) \ + absl::flags_internal::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(flag)) +#else +#define ABSL_FLAG_IMPL_FLAGNAME(txt) txt +#define ABSL_FLAG_IMPL_FILENAME() __FILE__ +#define ABSL_FLAG_IMPL_REGISTRAR(T, flag) \ + absl::flags_internal::FlagRegistrar<T, true>(ABSL_FLAG_IMPL_FLAG_PTR(flag)) +#endif + +// ABSL_FLAG_IMPL macro definition conditional on ABSL_FLAGS_STRIP_HELP + +#if ABSL_FLAGS_STRIP_HELP +#define ABSL_FLAG_IMPL_FLAGHELP(txt) absl::flags_internal::kStrippedFlagHelp +#else +#define ABSL_FLAG_IMPL_FLAGHELP(txt) txt +#endif + +// AbslFlagHelpGenFor##name is used to encapsulate both immediate (method Const) +// and lazy (method NonConst) evaluation of help message expression. We choose +// between the two via the call to HelpArg in absl::Flag instantiation below. +// If help message expression is constexpr evaluable compiler will optimize +// away this whole struct. +// TODO(rogeeff): place these generated structs into local namespace and apply +// ABSL_INTERNAL_UNIQUE_SHORT_NAME. +// TODO(rogeeff): Apply __attribute__((nodebug)) to FLAGS_help_storage_##name +#define ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, txt) \ + struct AbslFlagHelpGenFor##name { \ + /* The expression is run in the caller as part of the */ \ + /* default value argument. That keeps temporaries alive */ \ + /* long enough for NonConst to work correctly. */ \ + static constexpr absl::string_view Value( \ + absl::string_view v = ABSL_FLAG_IMPL_FLAGHELP(txt)) { \ + return v; \ + } \ + static std::string NonConst() { return std::string(Value()); } \ + }; \ + constexpr auto FLAGS_help_storage_##name ABSL_INTERNAL_UNIQUE_SMALL_NAME() \ + ABSL_ATTRIBUTE_SECTION_VARIABLE(flags_help_cold) = \ + absl::flags_internal::HelpStringAsArray<AbslFlagHelpGenFor##name>( \ + 0); + +#define ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ + struct AbslFlagDefaultGenFor##name { \ + Type value = absl::flags_internal::InitDefaultValue<Type>(default_value); \ + static void Gen(void* p) { \ + new (p) Type(AbslFlagDefaultGenFor##name{}.value); \ + } \ + }; + +// ABSL_FLAG_IMPL +// +// Note: Name of registrar object is not arbitrary. It is used to "grab" +// global name for FLAGS_no<flag_name> symbol, thus preventing the possibility +// of defining two flags with names foo and nofoo. +#define ABSL_FLAG_IMPL(Type, name, default_value, help) \ + namespace absl /* block flags in namespaces */ {} \ + ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ + ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, help) \ + ABSL_CONST_INIT absl::Flag<Type> FLAGS_##name{ \ + ABSL_FLAG_IMPL_FLAGNAME(#name), ABSL_FLAG_IMPL_FILENAME(), \ + ABSL_FLAG_IMPL_HELP_ARG(name), ABSL_FLAG_IMPL_DEFAULT_ARG(Type, name)}; \ + extern absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name; \ + absl::flags_internal::FlagRegistrarEmpty FLAGS_no##name = \ + ABSL_FLAG_IMPL_REGISTRAR(Type, FLAGS_##name) + +// ABSL_RETIRED_FLAG +// +// Designates the flag (which is usually pre-existing) as "retired." A retired +// flag is a flag that is now unused by the program, but may still be passed on +// the command line, usually by production scripts. A retired flag is ignored +// and code can't access it at runtime. +// +// This macro registers a retired flag with given name and type, with a name +// identical to the name of the original flag you are retiring. The retired +// flag's type can change over time, so that you can retire code to support a +// custom flag type. +// +// This macro has the same signature as `ABSL_FLAG`. To retire a flag, simply +// replace an `ABSL_FLAG` definition with `ABSL_RETIRED_FLAG`, leaving the +// arguments unchanged (unless of course you actually want to retire the flag +// type at this time as well). +// +// `default_value` is only used as a double check on the type. `explanation` is +// unused. +// TODO(rogeeff): replace RETIRED_FLAGS with FLAGS once forward declarations of +// retired flags are cleaned up. +#define ABSL_RETIRED_FLAG(type, name, default_value, explanation) \ + static absl::flags_internal::RetiredFlag<type> RETIRED_FLAGS_##name; \ + ABSL_ATTRIBUTE_UNUSED static const auto RETIRED_FLAGS_REG_##name = \ + (RETIRED_FLAGS_##name.Retire(#name), \ + ::absl::flags_internal::FlagRegistrarEmpty{}) + +#endif // ABSL_FLAGS_FLAG_H_ diff --git a/third_party/abseil_cpp/absl/flags/flag_benchmark.cc b/third_party/abseil_cpp/absl/flags/flag_benchmark.cc new file mode 100644 index 000000000000..9982b604b3b5 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/flag_benchmark.cc @@ -0,0 +1,159 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdint.h> + +#include <string> +#include <vector> + +#include "absl/flags/flag.h" +#include "absl/flags/marshalling.h" +#include "absl/flags/parse.h" +#include "absl/flags/reflection.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" +#include "absl/types/optional.h" +#include "benchmark/benchmark.h" + +namespace { +using String = std::string; +using VectorOfStrings = std::vector<std::string>; +using AbslDuration = absl::Duration; + +// We do not want to take over marshalling for the types absl::optional<int>, +// absl::optional<std::string> which we do not own. Instead we introduce unique +// "aliases" to these types, which we do. +using AbslOptionalInt = absl::optional<int>; +struct OptionalInt : AbslOptionalInt { + using AbslOptionalInt::AbslOptionalInt; +}; +// Next two functions represent Abseil Flags marshalling for OptionalInt. +bool AbslParseFlag(absl::string_view src, OptionalInt* flag, + std::string* error) { + int val; + if (src.empty()) + flag->reset(); + else if (!absl::ParseFlag(src, &val, error)) + return false; + *flag = val; + return true; +} +std::string AbslUnparseFlag(const OptionalInt& flag) { + return !flag ? "" : absl::UnparseFlag(*flag); +} + +using AbslOptionalString = absl::optional<std::string>; +struct OptionalString : AbslOptionalString { + using AbslOptionalString::AbslOptionalString; +}; +// Next two functions represent Abseil Flags marshalling for OptionalString. +bool AbslParseFlag(absl::string_view src, OptionalString* flag, + std::string* error) { + std::string val; + if (src.empty()) + flag->reset(); + else if (!absl::ParseFlag(src, &val, error)) + return false; + *flag = val; + return true; +} +std::string AbslUnparseFlag(const OptionalString& flag) { + return !flag ? "" : absl::UnparseFlag(*flag); +} + +struct UDT { + UDT() = default; + UDT(const UDT&) {} + UDT& operator=(const UDT&) { return *this; } +}; +// Next two functions represent Abseil Flags marshalling for UDT. +bool AbslParseFlag(absl::string_view, UDT*, std::string*) { return true; } +std::string AbslUnparseFlag(const UDT&) { return ""; } + +} // namespace + +#define BENCHMARKED_TYPES(A) \ + A(bool) \ + A(int16_t) \ + A(uint16_t) \ + A(int32_t) \ + A(uint32_t) \ + A(int64_t) \ + A(uint64_t) \ + A(double) \ + A(float) \ + A(String) \ + A(VectorOfStrings) \ + A(OptionalInt) \ + A(OptionalString) \ + A(AbslDuration) \ + A(UDT) + +#define FLAG_DEF(T) ABSL_FLAG(T, T##_flag, {}, ""); + +BENCHMARKED_TYPES(FLAG_DEF) + +// Register thousands of flags to bloat up the size of the registry. +// This mimics real life production binaries. +#define DEFINE_FLAG_0(name) ABSL_FLAG(int, name, 0, ""); +#define DEFINE_FLAG_1(name) DEFINE_FLAG_0(name##0) DEFINE_FLAG_0(name##1) +#define DEFINE_FLAG_2(name) DEFINE_FLAG_1(name##0) DEFINE_FLAG_1(name##1) +#define DEFINE_FLAG_3(name) DEFINE_FLAG_2(name##0) DEFINE_FLAG_2(name##1) +#define DEFINE_FLAG_4(name) DEFINE_FLAG_3(name##0) DEFINE_FLAG_3(name##1) +#define DEFINE_FLAG_5(name) DEFINE_FLAG_4(name##0) DEFINE_FLAG_4(name##1) +#define DEFINE_FLAG_6(name) DEFINE_FLAG_5(name##0) DEFINE_FLAG_5(name##1) +#define DEFINE_FLAG_7(name) DEFINE_FLAG_6(name##0) DEFINE_FLAG_6(name##1) +#define DEFINE_FLAG_8(name) DEFINE_FLAG_7(name##0) DEFINE_FLAG_7(name##1) +#define DEFINE_FLAG_9(name) DEFINE_FLAG_8(name##0) DEFINE_FLAG_8(name##1) +#define DEFINE_FLAG_10(name) DEFINE_FLAG_9(name##0) DEFINE_FLAG_9(name##1) +#define DEFINE_FLAG_11(name) DEFINE_FLAG_10(name##0) DEFINE_FLAG_10(name##1) +#define DEFINE_FLAG_12(name) DEFINE_FLAG_11(name##0) DEFINE_FLAG_11(name##1) +DEFINE_FLAG_12(bloat_flag_); + +namespace { + +#define BM_GetFlag(T) \ + void BM_GetFlag_##T(benchmark::State& state) { \ + for (auto _ : state) { \ + benchmark::DoNotOptimize(absl::GetFlag(FLAGS_##T##_flag)); \ + } \ + } \ + BENCHMARK(BM_GetFlag_##T); + +BENCHMARKED_TYPES(BM_GetFlag) + +void BM_ThreadedFindCommandLineFlag(benchmark::State& state) { + char dummy[] = "dummy"; + char* argv[] = {dummy}; + // We need to ensure that flags have been parsed. That is where the registry + // is finalized. + absl::ParseCommandLine(1, argv); + + for (auto s : state) { + benchmark::DoNotOptimize( + absl::FindCommandLineFlag("bloat_flag_010101010101")); + } +} +BENCHMARK(BM_ThreadedFindCommandLineFlag)->ThreadRange(1, 16); + +} // namespace + +#define InvokeGetFlag(T) \ + T AbslInvokeGetFlag##T() { return absl::GetFlag(FLAGS_##T##_flag); } \ + int odr##T = (benchmark::DoNotOptimize(AbslInvokeGetFlag##T), 1); + +BENCHMARKED_TYPES(InvokeGetFlag) + +// To veiw disassembly use: gdb ${BINARY} -batch -ex "disassemble /s $FUNC" diff --git a/third_party/abseil_cpp/absl/flags/flag_test.cc b/third_party/abseil_cpp/absl/flags/flag_test.cc new file mode 100644 index 000000000000..654c8122215c --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/flag_test.cc @@ -0,0 +1,906 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/flag.h" + +#include <stddef.h> +#include <stdint.h> + +#include <cmath> +#include <new> +#include <string> +#include <thread> // NOLINT +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/flags/config.h" +#include "absl/flags/declare.h" +#include "absl/flags/internal/flag.h" +#include "absl/flags/marshalling.h" +#include "absl/flags/reflection.h" +#include "absl/flags/usage_config.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/time/time.h" + +ABSL_DECLARE_FLAG(int64_t, mistyped_int_flag); +ABSL_DECLARE_FLAG(std::vector<std::string>, mistyped_string_flag); + +namespace { + +namespace flags = absl::flags_internal; + +std::string TestHelpMsg() { return "dynamic help"; } +#if defined(_MSC_VER) && !defined(__clang__) +std::string TestLiteralHelpMsg() { return "literal help"; } +#endif +template <typename T> +void TestMakeDflt(void* dst) { + new (dst) T{}; +} +void TestCallback() {} + +struct UDT { + UDT() = default; + UDT(const UDT&) = default; +}; +bool AbslParseFlag(absl::string_view, UDT*, std::string*) { return true; } +std::string AbslUnparseFlag(const UDT&) { return ""; } + +class FlagTest : public testing::Test { + protected: + static void SetUpTestSuite() { + // Install a function to normalize filenames before this test is run. + absl::FlagsUsageConfig default_config; + default_config.normalize_filename = &FlagTest::NormalizeFileName; + absl::SetFlagsUsageConfig(default_config); + } + + private: + static std::string NormalizeFileName(absl::string_view fname) { +#ifdef _WIN32 + std::string normalized(fname); + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + fname = normalized; +#endif + return std::string(fname); + } + absl::FlagSaver flag_saver_; +}; + +struct S1 { + S1() = default; + S1(const S1&) = default; + int32_t f1; + int64_t f2; +}; + +struct S2 { + S2() = default; + S2(const S2&) = default; + int64_t f1; + double f2; +}; + +TEST_F(FlagTest, Traits) { + EXPECT_EQ(flags::StorageKind<int>(), + flags::FlagValueStorageKind::kOneWordAtomic); + EXPECT_EQ(flags::StorageKind<bool>(), + flags::FlagValueStorageKind::kOneWordAtomic); + EXPECT_EQ(flags::StorageKind<double>(), + flags::FlagValueStorageKind::kOneWordAtomic); + EXPECT_EQ(flags::StorageKind<int64_t>(), + flags::FlagValueStorageKind::kOneWordAtomic); + +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) + EXPECT_EQ(flags::StorageKind<S1>(), + flags::FlagValueStorageKind::kTwoWordsAtomic); + EXPECT_EQ(flags::StorageKind<S2>(), + flags::FlagValueStorageKind::kTwoWordsAtomic); +#else + EXPECT_EQ(flags::StorageKind<S1>(), + flags::FlagValueStorageKind::kAlignedBuffer); + EXPECT_EQ(flags::StorageKind<S2>(), + flags::FlagValueStorageKind::kAlignedBuffer); +#endif + + EXPECT_EQ(flags::StorageKind<std::string>(), + flags::FlagValueStorageKind::kAlignedBuffer); + EXPECT_EQ(flags::StorageKind<std::vector<std::string>>(), + flags::FlagValueStorageKind::kAlignedBuffer); +} + +// -------------------------------------------------------------------- + +constexpr flags::FlagHelpArg help_arg{flags::FlagHelpMsg("literal help"), + flags::FlagHelpKind::kLiteral}; + +using String = std::string; + +#if !defined(_MSC_VER) || defined(__clang__) +#define DEFINE_CONSTRUCTED_FLAG(T, dflt, dflt_kind) \ + constexpr flags::FlagDefaultArg f1default##T{ \ + flags::FlagDefaultSrc{dflt}, flags::FlagDefaultKind::dflt_kind}; \ + constexpr absl::Flag<T> f1##T{"f1", "file", help_arg, f1default##T}; \ + ABSL_CONST_INIT absl::Flag<T> f2##T { \ + "f2", "file", \ + {flags::FlagHelpMsg(&TestHelpMsg), flags::FlagHelpKind::kGenFunc}, \ + flags::FlagDefaultArg { \ + flags::FlagDefaultSrc(&TestMakeDflt<T>), \ + flags::FlagDefaultKind::kGenFunc \ + } \ + } +#else +#define DEFINE_CONSTRUCTED_FLAG(T, dflt, dflt_kind) \ + constexpr flags::FlagDefaultArg f1default##T{ \ + flags::FlagDefaultSrc{dflt}, flags::FlagDefaultKind::dflt_kind}; \ + constexpr absl::Flag<T> f1##T{"f1", "file", &TestLiteralHelpMsg, \ + &TestMakeDflt<T>}; \ + ABSL_CONST_INIT absl::Flag<T> f2##T { \ + "f2", "file", &TestHelpMsg, &TestMakeDflt<T> \ + } +#endif + +DEFINE_CONSTRUCTED_FLAG(bool, true, kOneWord); +DEFINE_CONSTRUCTED_FLAG(int16_t, 1, kOneWord); +DEFINE_CONSTRUCTED_FLAG(uint16_t, 2, kOneWord); +DEFINE_CONSTRUCTED_FLAG(int32_t, 3, kOneWord); +DEFINE_CONSTRUCTED_FLAG(uint32_t, 4, kOneWord); +DEFINE_CONSTRUCTED_FLAG(int64_t, 5, kOneWord); +DEFINE_CONSTRUCTED_FLAG(uint64_t, 6, kOneWord); +DEFINE_CONSTRUCTED_FLAG(float, 7.8, kOneWord); +DEFINE_CONSTRUCTED_FLAG(double, 9.10, kOneWord); +DEFINE_CONSTRUCTED_FLAG(String, &TestMakeDflt<String>, kGenFunc); +DEFINE_CONSTRUCTED_FLAG(UDT, &TestMakeDflt<UDT>, kGenFunc); + +template <typename T> +bool TestConstructionFor(const absl::Flag<T>& f1, absl::Flag<T>& f2) { + EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Name(), "f1"); + EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Help(), "literal help"); + EXPECT_EQ(absl::GetFlagReflectionHandle(f1).Filename(), "file"); + + flags::FlagRegistrar<T, false>(ABSL_FLAG_IMPL_FLAG_PTR(f2)) + .OnUpdate(TestCallback); + + EXPECT_EQ(absl::GetFlagReflectionHandle(f2).Name(), "f2"); + EXPECT_EQ(absl::GetFlagReflectionHandle(f2).Help(), "dynamic help"); + EXPECT_EQ(absl::GetFlagReflectionHandle(f2).Filename(), "file"); + + return true; +} + +#define TEST_CONSTRUCTED_FLAG(T) TestConstructionFor(f1##T, f2##T); + +TEST_F(FlagTest, TestConstruction) { + TEST_CONSTRUCTED_FLAG(bool); + TEST_CONSTRUCTED_FLAG(int16_t); + TEST_CONSTRUCTED_FLAG(uint16_t); + TEST_CONSTRUCTED_FLAG(int32_t); + TEST_CONSTRUCTED_FLAG(uint32_t); + TEST_CONSTRUCTED_FLAG(int64_t); + TEST_CONSTRUCTED_FLAG(uint64_t); + TEST_CONSTRUCTED_FLAG(float); + TEST_CONSTRUCTED_FLAG(double); + TEST_CONSTRUCTED_FLAG(String); + TEST_CONSTRUCTED_FLAG(UDT); +} + +// -------------------------------------------------------------------- + +} // namespace + +ABSL_DECLARE_FLAG(bool, test_flag_01); +ABSL_DECLARE_FLAG(int, test_flag_02); +ABSL_DECLARE_FLAG(int16_t, test_flag_03); +ABSL_DECLARE_FLAG(uint16_t, test_flag_04); +ABSL_DECLARE_FLAG(int32_t, test_flag_05); +ABSL_DECLARE_FLAG(uint32_t, test_flag_06); +ABSL_DECLARE_FLAG(int64_t, test_flag_07); +ABSL_DECLARE_FLAG(uint64_t, test_flag_08); +ABSL_DECLARE_FLAG(double, test_flag_09); +ABSL_DECLARE_FLAG(float, test_flag_10); +ABSL_DECLARE_FLAG(std::string, test_flag_11); +ABSL_DECLARE_FLAG(absl::Duration, test_flag_12); + +namespace { + +#if !ABSL_FLAGS_STRIP_NAMES + +TEST_F(FlagTest, TestFlagDeclaration) { + // test that we can access flag objects. + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Name(), + "test_flag_01"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_02).Name(), + "test_flag_02"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_03).Name(), + "test_flag_03"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_04).Name(), + "test_flag_04"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_05).Name(), + "test_flag_05"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_06).Name(), + "test_flag_06"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_07).Name(), + "test_flag_07"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_08).Name(), + "test_flag_08"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_09).Name(), + "test_flag_09"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_10).Name(), + "test_flag_10"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_11).Name(), + "test_flag_11"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_12).Name(), + "test_flag_12"); +} +#endif // !ABSL_FLAGS_STRIP_NAMES + +// -------------------------------------------------------------------- + +} // namespace + +ABSL_FLAG(bool, test_flag_01, true, "test flag 01"); +ABSL_FLAG(int, test_flag_02, 1234, "test flag 02"); +ABSL_FLAG(int16_t, test_flag_03, -34, "test flag 03"); +ABSL_FLAG(uint16_t, test_flag_04, 189, "test flag 04"); +ABSL_FLAG(int32_t, test_flag_05, 10765, "test flag 05"); +ABSL_FLAG(uint32_t, test_flag_06, 40000, "test flag 06"); +ABSL_FLAG(int64_t, test_flag_07, -1234567, "test flag 07"); +ABSL_FLAG(uint64_t, test_flag_08, 9876543, "test flag 08"); +ABSL_FLAG(double, test_flag_09, -9.876e-50, "test flag 09"); +ABSL_FLAG(float, test_flag_10, 1.234e12f, "test flag 10"); +ABSL_FLAG(std::string, test_flag_11, "", "test flag 11"); +ABSL_FLAG(absl::Duration, test_flag_12, absl::Minutes(10), "test flag 12"); + +namespace { + +#if !ABSL_FLAGS_STRIP_NAMES +TEST_F(FlagTest, TestFlagDefinition) { + absl::string_view expected_file_name = "absl/flags/flag_test.cc"; + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Name(), + "test_flag_01"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Help(), + "test flag 01"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_01).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_02).Name(), + "test_flag_02"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_02).Help(), + "test flag 02"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_02).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_02).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_03).Name(), + "test_flag_03"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_03).Help(), + "test flag 03"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_03).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_03).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_04).Name(), + "test_flag_04"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_04).Help(), + "test flag 04"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_04).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_04).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_05).Name(), + "test_flag_05"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_05).Help(), + "test flag 05"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_05).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_05).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_06).Name(), + "test_flag_06"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_06).Help(), + "test flag 06"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_06).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_06).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_07).Name(), + "test_flag_07"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_07).Help(), + "test flag 07"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_07).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_07).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_08).Name(), + "test_flag_08"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_08).Help(), + "test flag 08"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_08).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_08).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_09).Name(), + "test_flag_09"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_09).Help(), + "test flag 09"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_09).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_09).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_10).Name(), + "test_flag_10"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_10).Help(), + "test flag 10"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_10).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_10).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_11).Name(), + "test_flag_11"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_11).Help(), + "test flag 11"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_11).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_11).Filename(); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_12).Name(), + "test_flag_12"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_12).Help(), + "test flag 12"); + EXPECT_TRUE(absl::EndsWith( + absl::GetFlagReflectionHandle(FLAGS_test_flag_12).Filename(), + expected_file_name)) + << absl::GetFlagReflectionHandle(FLAGS_test_flag_12).Filename(); +} +#endif // !ABSL_FLAGS_STRIP_NAMES + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestDefault) { + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).DefaultValue(), + "true"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_02).DefaultValue(), + "1234"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_03).DefaultValue(), + "-34"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_04).DefaultValue(), + "189"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_05).DefaultValue(), + "10765"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_06).DefaultValue(), + "40000"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_07).DefaultValue(), + "-1234567"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_08).DefaultValue(), + "9876543"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_09).DefaultValue(), + "-9.876e-50"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_10).DefaultValue(), + "1.234e+12"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_11).DefaultValue(), + ""); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_12).DefaultValue(), + "10m"); + + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_01).CurrentValue(), + "true"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_02).CurrentValue(), + "1234"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_03).CurrentValue(), + "-34"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_04).CurrentValue(), + "189"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_05).CurrentValue(), + "10765"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_06).CurrentValue(), + "40000"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_07).CurrentValue(), + "-1234567"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_08).CurrentValue(), + "9876543"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_09).CurrentValue(), + "-9.876e-50"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_10).CurrentValue(), + "1.234e+12"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_11).CurrentValue(), + ""); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_12).CurrentValue(), + "10m"); + + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_01), true); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_02), 1234); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_03), -34); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_04), 189); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_05), 10765); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_06), 40000); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_07), -1234567); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 9876543); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_09), -9.876e-50, 1e-55); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_10), 1.234e12f, 1e5f); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_11), ""); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_12), absl::Minutes(10)); +} + +// -------------------------------------------------------------------- + +struct NonTriviallyCopyableAggregate { + NonTriviallyCopyableAggregate() = default; + NonTriviallyCopyableAggregate(const NonTriviallyCopyableAggregate& rhs) + : value(rhs.value) {} + NonTriviallyCopyableAggregate& operator=( + const NonTriviallyCopyableAggregate& rhs) { + value = rhs.value; + return *this; + } + + int value; +}; +bool AbslParseFlag(absl::string_view src, NonTriviallyCopyableAggregate* f, + std::string* e) { + return absl::ParseFlag(src, &f->value, e); +} +std::string AbslUnparseFlag(const NonTriviallyCopyableAggregate& ntc) { + return absl::StrCat(ntc.value); +} + +bool operator==(const NonTriviallyCopyableAggregate& ntc1, + const NonTriviallyCopyableAggregate& ntc2) { + return ntc1.value == ntc2.value; +} + +} // namespace + +ABSL_FLAG(bool, test_flag_eb_01, {}, ""); +ABSL_FLAG(int32_t, test_flag_eb_02, {}, ""); +ABSL_FLAG(int64_t, test_flag_eb_03, {}, ""); +ABSL_FLAG(double, test_flag_eb_04, {}, ""); +ABSL_FLAG(std::string, test_flag_eb_05, {}, ""); +ABSL_FLAG(NonTriviallyCopyableAggregate, test_flag_eb_06, {}, ""); + +namespace { + +TEST_F(FlagTest, TestEmptyBracesDefault) { + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_eb_01).DefaultValue(), + "false"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_eb_02).DefaultValue(), + "0"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_eb_03).DefaultValue(), + "0"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_eb_04).DefaultValue(), + "0"); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_eb_05).DefaultValue(), + ""); + EXPECT_EQ(absl::GetFlagReflectionHandle(FLAGS_test_flag_eb_06).DefaultValue(), + "0"); + + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_eb_01), false); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_eb_02), 0); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_eb_03), 0); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_eb_04), 0.0); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_eb_05), ""); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_eb_06), + NonTriviallyCopyableAggregate{}); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestGetSet) { + absl::SetFlag(&FLAGS_test_flag_01, false); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_01), false); + + absl::SetFlag(&FLAGS_test_flag_02, 321); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_02), 321); + + absl::SetFlag(&FLAGS_test_flag_03, 67); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_03), 67); + + absl::SetFlag(&FLAGS_test_flag_04, 1); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_04), 1); + + absl::SetFlag(&FLAGS_test_flag_05, -908); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_05), -908); + + absl::SetFlag(&FLAGS_test_flag_06, 4001); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_06), 4001); + + absl::SetFlag(&FLAGS_test_flag_07, -23456); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_07), -23456); + + absl::SetFlag(&FLAGS_test_flag_08, 975310); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 975310); + + absl::SetFlag(&FLAGS_test_flag_09, 1.00001); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_09), 1.00001, 1e-10); + + absl::SetFlag(&FLAGS_test_flag_10, -3.54f); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_10), -3.54f, 1e-6f); + + absl::SetFlag(&FLAGS_test_flag_11, "asdf"); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_11), "asdf"); + + absl::SetFlag(&FLAGS_test_flag_12, absl::Seconds(110)); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_12), absl::Seconds(110)); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestGetViaReflection) { + auto* handle = absl::FindCommandLineFlag("test_flag_01"); + EXPECT_EQ(*handle->TryGet<bool>(), true); + handle = absl::FindCommandLineFlag("test_flag_02"); + EXPECT_EQ(*handle->TryGet<int>(), 1234); + handle = absl::FindCommandLineFlag("test_flag_03"); + EXPECT_EQ(*handle->TryGet<int16_t>(), -34); + handle = absl::FindCommandLineFlag("test_flag_04"); + EXPECT_EQ(*handle->TryGet<uint16_t>(), 189); + handle = absl::FindCommandLineFlag("test_flag_05"); + EXPECT_EQ(*handle->TryGet<int32_t>(), 10765); + handle = absl::FindCommandLineFlag("test_flag_06"); + EXPECT_EQ(*handle->TryGet<uint32_t>(), 40000); + handle = absl::FindCommandLineFlag("test_flag_07"); + EXPECT_EQ(*handle->TryGet<int64_t>(), -1234567); + handle = absl::FindCommandLineFlag("test_flag_08"); + EXPECT_EQ(*handle->TryGet<uint64_t>(), 9876543); + handle = absl::FindCommandLineFlag("test_flag_09"); + EXPECT_NEAR(*handle->TryGet<double>(), -9.876e-50, 1e-55); + handle = absl::FindCommandLineFlag("test_flag_10"); + EXPECT_NEAR(*handle->TryGet<float>(), 1.234e12f, 1e5f); + handle = absl::FindCommandLineFlag("test_flag_11"); + EXPECT_EQ(*handle->TryGet<std::string>(), ""); + handle = absl::FindCommandLineFlag("test_flag_12"); + EXPECT_EQ(*handle->TryGet<absl::Duration>(), absl::Minutes(10)); +} + +// -------------------------------------------------------------------- + +int GetDflt1() { return 1; } + +} // namespace + +ABSL_FLAG(int, test_int_flag_with_non_const_default, GetDflt1(), + "test int flag non const default"); +ABSL_FLAG(std::string, test_string_flag_with_non_const_default, + absl::StrCat("AAA", "BBB"), "test string flag non const default"); + +namespace { + +TEST_F(FlagTest, TestNonConstexprDefault) { + EXPECT_EQ(absl::GetFlag(FLAGS_test_int_flag_with_non_const_default), 1); + EXPECT_EQ(absl::GetFlag(FLAGS_test_string_flag_with_non_const_default), + "AAABBB"); +} + +// -------------------------------------------------------------------- + +} // namespace + +ABSL_FLAG(bool, test_flag_with_non_const_help, true, + absl::StrCat("test ", "flag ", "non const help")); + +namespace { + +#if !ABSL_FLAGS_STRIP_HELP +TEST_F(FlagTest, TestNonConstexprHelp) { + EXPECT_EQ( + absl::GetFlagReflectionHandle(FLAGS_test_flag_with_non_const_help).Help(), + "test flag non const help"); +} +#endif //! ABSL_FLAGS_STRIP_HELP + +// -------------------------------------------------------------------- + +int cb_test_value = -1; +void TestFlagCB(); + +} // namespace + +ABSL_FLAG(int, test_flag_with_cb, 100, "").OnUpdate(TestFlagCB); + +ABSL_FLAG(int, test_flag_with_lambda_cb, 200, "").OnUpdate([]() { + cb_test_value = absl::GetFlag(FLAGS_test_flag_with_lambda_cb) + + absl::GetFlag(FLAGS_test_flag_with_cb); +}); + +namespace { + +void TestFlagCB() { cb_test_value = absl::GetFlag(FLAGS_test_flag_with_cb); } + +// Tests side-effects of callback invocation. +TEST_F(FlagTest, CallbackInvocation) { + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_with_cb), 100); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_with_lambda_cb), 200); + EXPECT_EQ(cb_test_value, 300); + + absl::SetFlag(&FLAGS_test_flag_with_cb, 1); + EXPECT_EQ(cb_test_value, 1); + + absl::SetFlag(&FLAGS_test_flag_with_lambda_cb, 3); + EXPECT_EQ(cb_test_value, 4); +} + +// -------------------------------------------------------------------- + +struct CustomUDT { + CustomUDT() : a(1), b(1) {} + CustomUDT(int a_, int b_) : a(a_), b(b_) {} + + friend bool operator==(const CustomUDT& f1, const CustomUDT& f2) { + return f1.a == f2.a && f1.b == f2.b; + } + + int a; + int b; +}; +bool AbslParseFlag(absl::string_view in, CustomUDT* f, std::string*) { + std::vector<absl::string_view> parts = + absl::StrSplit(in, ':', absl::SkipWhitespace()); + + if (parts.size() != 2) return false; + + if (!absl::SimpleAtoi(parts[0], &f->a)) return false; + + if (!absl::SimpleAtoi(parts[1], &f->b)) return false; + + return true; +} +std::string AbslUnparseFlag(const CustomUDT& f) { + return absl::StrCat(f.a, ":", f.b); +} + +} // namespace + +ABSL_FLAG(CustomUDT, test_flag_custom_udt, CustomUDT(), "test flag custom UDT"); + +namespace { + +TEST_F(FlagTest, TestCustomUDT) { + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_custom_udt), CustomUDT(1, 1)); + absl::SetFlag(&FLAGS_test_flag_custom_udt, CustomUDT(2, 3)); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_custom_udt), CustomUDT(2, 3)); +} + +// MSVC produces link error on the type mismatch. +// Linux does not have build errors and validations work as expected. +#if !defined(_WIN32) && GTEST_HAS_DEATH_TEST + +using FlagDeathTest = FlagTest; + +TEST_F(FlagDeathTest, TestTypeMismatchValidations) { +#if !defined(NDEBUG) + EXPECT_DEATH_IF_SUPPORTED( + static_cast<void>(absl::GetFlag(FLAGS_mistyped_int_flag)), + "Flag 'mistyped_int_flag' is defined as one type and declared " + "as another"); + EXPECT_DEATH_IF_SUPPORTED( + static_cast<void>(absl::GetFlag(FLAGS_mistyped_string_flag)), + "Flag 'mistyped_string_flag' is defined as one type and " + "declared as another"); +#endif + + EXPECT_DEATH_IF_SUPPORTED( + absl::SetFlag(&FLAGS_mistyped_int_flag, 1), + "Flag 'mistyped_int_flag' is defined as one type and declared " + "as another"); + EXPECT_DEATH_IF_SUPPORTED( + absl::SetFlag(&FLAGS_mistyped_string_flag, std::vector<std::string>{}), + "Flag 'mistyped_string_flag' is defined as one type and declared as " + "another"); +} + +#endif + +// -------------------------------------------------------------------- + +// A contrived type that offers implicit and explicit conversion from specific +// source types. +struct ConversionTestVal { + ConversionTestVal() = default; + explicit ConversionTestVal(int a_in) : a(a_in) {} + + enum class ViaImplicitConv { kTen = 10, kEleven }; + // NOLINTNEXTLINE + ConversionTestVal(ViaImplicitConv from) : a(static_cast<int>(from)) {} + + int a; +}; + +bool AbslParseFlag(absl::string_view in, ConversionTestVal* val_out, + std::string*) { + if (!absl::SimpleAtoi(in, &val_out->a)) { + return false; + } + return true; +} +std::string AbslUnparseFlag(const ConversionTestVal& val) { + return absl::StrCat(val.a); +} + +} // namespace + +// Flag default values can be specified with a value that converts to the flag +// value type implicitly. +ABSL_FLAG(ConversionTestVal, test_flag_implicit_conv, + ConversionTestVal::ViaImplicitConv::kTen, + "test flag init via implicit conversion"); + +namespace { + +TEST_F(FlagTest, CanSetViaImplicitConversion) { + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_implicit_conv).a, 10); + absl::SetFlag(&FLAGS_test_flag_implicit_conv, + ConversionTestVal::ViaImplicitConv::kEleven); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_implicit_conv).a, 11); +} + +// -------------------------------------------------------------------- + +struct NonDfltConstructible { + public: + // This constructor tests that we can initialize the flag with int value + NonDfltConstructible(int i) : value(i) {} // NOLINT + + // This constructor tests that we can't initialize the flag with char value + // but can with explicitly constructed NonDfltConstructible. + explicit NonDfltConstructible(char c) : value(100 + static_cast<int>(c)) {} + + int value; +}; + +bool AbslParseFlag(absl::string_view in, NonDfltConstructible* ndc_out, + std::string*) { + return absl::SimpleAtoi(in, &ndc_out->value); +} +std::string AbslUnparseFlag(const NonDfltConstructible& ndc) { + return absl::StrCat(ndc.value); +} + +} // namespace + +ABSL_FLAG(NonDfltConstructible, ndc_flag1, NonDfltConstructible('1'), + "Flag with non default constructible type"); +ABSL_FLAG(NonDfltConstructible, ndc_flag2, 0, + "Flag with non default constructible type"); + +namespace { + +TEST_F(FlagTest, TestNonDefaultConstructibleType) { + EXPECT_EQ(absl::GetFlag(FLAGS_ndc_flag1).value, '1' + 100); + EXPECT_EQ(absl::GetFlag(FLAGS_ndc_flag2).value, 0); + + absl::SetFlag(&FLAGS_ndc_flag1, NonDfltConstructible('A')); + absl::SetFlag(&FLAGS_ndc_flag2, 25); + + EXPECT_EQ(absl::GetFlag(FLAGS_ndc_flag1).value, 'A' + 100); + EXPECT_EQ(absl::GetFlag(FLAGS_ndc_flag2).value, 25); +} + +} // namespace + +// -------------------------------------------------------------------- + +ABSL_RETIRED_FLAG(bool, old_bool_flag, true, "old descr"); +ABSL_RETIRED_FLAG(int, old_int_flag, (int)std::sqrt(10), "old descr"); +ABSL_RETIRED_FLAG(std::string, old_str_flag, "", absl::StrCat("old ", "descr")); + +bool initializaion_order_fiasco_test = [] { + // Iterate over all the flags during static initialization. + // This should not trigger ASan's initialization-order-fiasco. + auto* handle1 = absl::FindCommandLineFlag("flag_on_separate_file"); + auto* handle2 = absl::FindCommandLineFlag("retired_flag_on_separate_file"); + if (handle1 != nullptr && handle2 != nullptr) { + return handle1->Name() == handle2->Name(); + } + return true; +}(); + +namespace { + +TEST_F(FlagTest, TestRetiredFlagRegistration) { + auto* handle = absl::FindCommandLineFlag("old_bool_flag"); + EXPECT_TRUE(handle->IsOfType<bool>()); + EXPECT_TRUE(handle->IsRetired()); + handle = absl::FindCommandLineFlag("old_int_flag"); + EXPECT_TRUE(handle->IsOfType<int>()); + EXPECT_TRUE(handle->IsRetired()); + handle = absl::FindCommandLineFlag("old_str_flag"); + EXPECT_TRUE(handle->IsOfType<std::string>()); + EXPECT_TRUE(handle->IsRetired()); +} + +} // namespace + +// -------------------------------------------------------------------- + +namespace { + +// User-defined type with small alignment, but size exceeding 16. +struct SmallAlignUDT { + SmallAlignUDT() : c('A'), s(12) {} + char c; + int16_t s; + char bytes[14]; +}; + +bool AbslParseFlag(absl::string_view, SmallAlignUDT*, std::string*) { + return true; +} +std::string AbslUnparseFlag(const SmallAlignUDT&) { return ""; } + +// User-defined type with small size, but not trivially copyable. +struct NonTriviallyCopyableUDT { + NonTriviallyCopyableUDT() : c('A') {} + NonTriviallyCopyableUDT(const NonTriviallyCopyableUDT& rhs) : c(rhs.c) {} + NonTriviallyCopyableUDT& operator=(const NonTriviallyCopyableUDT& rhs) { + c = rhs.c; + return *this; + } + + char c; +}; + +bool AbslParseFlag(absl::string_view, NonTriviallyCopyableUDT*, std::string*) { + return true; +} +std::string AbslUnparseFlag(const NonTriviallyCopyableUDT&) { return ""; } + +} // namespace + +ABSL_FLAG(SmallAlignUDT, test_flag_sa_udt, {}, "help"); +ABSL_FLAG(NonTriviallyCopyableUDT, test_flag_ntc_udt, {}, "help"); + +namespace { + +TEST_F(FlagTest, TestSmallAlignUDT) { + SmallAlignUDT value = absl::GetFlag(FLAGS_test_flag_sa_udt); + EXPECT_EQ(value.c, 'A'); + EXPECT_EQ(value.s, 12); + + value.c = 'B'; + value.s = 45; + absl::SetFlag(&FLAGS_test_flag_sa_udt, value); + value = absl::GetFlag(FLAGS_test_flag_sa_udt); + EXPECT_EQ(value.c, 'B'); + EXPECT_EQ(value.s, 45); +} + +TEST_F(FlagTest, TestNonTriviallyCopyableUDT) { + NonTriviallyCopyableUDT value = absl::GetFlag(FLAGS_test_flag_ntc_udt); + EXPECT_EQ(value.c, 'A'); + + value.c = 'B'; + absl::SetFlag(&FLAGS_test_flag_ntc_udt, value); + value = absl::GetFlag(FLAGS_test_flag_ntc_udt); + EXPECT_EQ(value.c, 'B'); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/flag_test_defs.cc b/third_party/abseil_cpp/absl/flags/flag_test_defs.cc new file mode 100644 index 000000000000..4e1693cdb9c4 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/flag_test_defs.cc @@ -0,0 +1,24 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is used to test the mismatch of the flag type between definition +// and declaration. These are definitions. flag_test.cc contains declarations. +#include <string> +#include "absl/flags/flag.h" + +ABSL_FLAG(int, mistyped_int_flag, 0, ""); +ABSL_FLAG(std::string, mistyped_string_flag, "", ""); +ABSL_FLAG(bool, flag_on_separate_file, false, ""); +ABSL_RETIRED_FLAG(bool, retired_flag_on_separate_file, false, ""); diff --git a/third_party/abseil_cpp/absl/flags/internal/commandlineflag.cc b/third_party/abseil_cpp/absl/flags/internal/commandlineflag.cc new file mode 100644 index 000000000000..4482955c4cb6 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/commandlineflag.cc @@ -0,0 +1,26 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/commandlineflag.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +FlagStateInterface::~FlagStateInterface() {} + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/internal/commandlineflag.h b/third_party/abseil_cpp/absl/flags/internal/commandlineflag.h new file mode 100644 index 000000000000..cb46fe2e979d --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/commandlineflag.h @@ -0,0 +1,68 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_COMMANDLINEFLAG_H_ +#define ABSL_FLAGS_INTERNAL_COMMANDLINEFLAG_H_ + +#include "absl/base/config.h" +#include "absl/base/internal/fast_type_id.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// An alias for flag fast type id. This value identifies the flag value type +// simialarly to typeid(T), without relying on RTTI being available. In most +// cases this id is enough to uniquely identify the flag's value type. In a few +// cases we'll have to resort to using actual RTTI implementation if it is +// available. +using FlagFastTypeId = absl::base_internal::FastTypeIdType; + +// Options that control SetCommandLineOptionWithMode. +enum FlagSettingMode { + // update the flag's value unconditionally (can call this multiple times). + SET_FLAGS_VALUE, + // update the flag's value, but *only if* it has not yet been updated + // with SET_FLAGS_VALUE, SET_FLAG_IF_DEFAULT, or "FLAGS_xxx = nondef". + SET_FLAG_IF_DEFAULT, + // set the flag's default value to this. If the flag has not been updated + // yet (via SET_FLAGS_VALUE, SET_FLAG_IF_DEFAULT, or "FLAGS_xxx = nondef") + // change the flag's current value to the new default value as well. + SET_FLAGS_DEFAULT +}; + +// Options that control ParseFrom: Source of a value. +enum ValueSource { + // Flag is being set by value specified on a command line. + kCommandLine, + // Flag is being set by value specified in the code. + kProgrammaticChange, +}; + +// Handle to FlagState objects. Specific flag state objects will restore state +// of a flag produced this flag state from method CommandLineFlag::SaveState(). +class FlagStateInterface { + public: + virtual ~FlagStateInterface(); + + // Restores the flag originated this object to the saved state. + virtual void Restore() const = 0; +}; + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_COMMANDLINEFLAG_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/flag.cc b/third_party/abseil_cpp/absl/flags/internal/flag.cc new file mode 100644 index 000000000000..1502e7f11d36 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/flag.cc @@ -0,0 +1,568 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/flag.h" + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <array> +#include <atomic> +#include <memory> +#include <new> +#include <string> +#include <typeinfo> + +#include "absl/base/call_once.h" +#include "absl/base/casts.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/flags/config.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/flags/usage_config.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// The help message indicating that the commandline flag has been +// 'stripped'. It will not show up when doing "-help" and its +// variants. The flag is stripped if ABSL_FLAGS_STRIP_HELP is set to 1 +// before including absl/flags/flag.h +const char kStrippedFlagHelp[] = "\001\002\003\004 (unknown) \004\003\002\001"; + +namespace { + +// Currently we only validate flag values for user-defined flag types. +bool ShouldValidateFlagValue(FlagFastTypeId flag_type_id) { +#define DONT_VALIDATE(T, _) \ + if (flag_type_id == base_internal::FastTypeId<T>()) return false; + ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(DONT_VALIDATE) +#undef DONT_VALIDATE + + return true; +} + +// RAII helper used to temporarily unlock and relock `absl::Mutex`. +// This is used when we need to ensure that locks are released while +// invoking user supplied callbacks and then reacquired, since callbacks may +// need to acquire these locks themselves. +class MutexRelock { + public: + explicit MutexRelock(absl::Mutex& mu) : mu_(mu) { mu_.Unlock(); } + ~MutexRelock() { mu_.Lock(); } + + MutexRelock(const MutexRelock&) = delete; + MutexRelock& operator=(const MutexRelock&) = delete; + + private: + absl::Mutex& mu_; +}; + +} // namespace + +/////////////////////////////////////////////////////////////////////////////// +// Persistent state of the flag data. + +class FlagImpl; + +class FlagState : public flags_internal::FlagStateInterface { + public: + template <typename V> + FlagState(FlagImpl& flag_impl, const V& v, bool modified, + bool on_command_line, int64_t counter) + : flag_impl_(flag_impl), + value_(v), + modified_(modified), + on_command_line_(on_command_line), + counter_(counter) {} + + ~FlagState() override { + if (flag_impl_.ValueStorageKind() != FlagValueStorageKind::kAlignedBuffer) + return; + flags_internal::Delete(flag_impl_.op_, value_.heap_allocated); + } + + private: + friend class FlagImpl; + + // Restores the flag to the saved state. + void Restore() const override { + if (!flag_impl_.RestoreState(*this)) return; + + ABSL_INTERNAL_LOG(INFO, + absl::StrCat("Restore saved value of ", flag_impl_.Name(), + " to: ", flag_impl_.CurrentValue())); + } + + // Flag and saved flag data. + FlagImpl& flag_impl_; + union SavedValue { + explicit SavedValue(void* v) : heap_allocated(v) {} + explicit SavedValue(int64_t v) : one_word(v) {} + explicit SavedValue(flags_internal::AlignedTwoWords v) : two_words(v) {} + + void* heap_allocated; + int64_t one_word; + flags_internal::AlignedTwoWords two_words; + } value_; + bool modified_; + bool on_command_line_; + int64_t counter_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Flag implementation, which does not depend on flag value type. + +DynValueDeleter::DynValueDeleter(FlagOpFn op_arg) : op(op_arg) {} + +void DynValueDeleter::operator()(void* ptr) const { + if (op == nullptr) return; + + Delete(op, ptr); +} + +void FlagImpl::Init() { + new (&data_guard_) absl::Mutex; + + auto def_kind = static_cast<FlagDefaultKind>(def_kind_); + + switch (ValueStorageKind()) { + case FlagValueStorageKind::kAlignedBuffer: + // For this storage kind the default_value_ always points to gen_func + // during initialization. + assert(def_kind == FlagDefaultKind::kGenFunc); + (*default_value_.gen_func)(AlignedBufferValue()); + break; + case FlagValueStorageKind::kOneWordAtomic: { + alignas(int64_t) std::array<char, sizeof(int64_t)> buf{}; + if (def_kind == FlagDefaultKind::kGenFunc) { + (*default_value_.gen_func)(buf.data()); + } else { + assert(def_kind != FlagDefaultKind::kDynamicValue); + std::memcpy(buf.data(), &default_value_, Sizeof(op_)); + } + OneWordValue().store(absl::bit_cast<int64_t>(buf), + std::memory_order_release); + break; + } + case FlagValueStorageKind::kTwoWordsAtomic: { + // For this storage kind the default_value_ always points to gen_func + // during initialization. + assert(def_kind == FlagDefaultKind::kGenFunc); + alignas(AlignedTwoWords) std::array<char, sizeof(AlignedTwoWords)> buf{}; + (*default_value_.gen_func)(buf.data()); + auto atomic_value = absl::bit_cast<AlignedTwoWords>(buf); + TwoWordsValue().store(atomic_value, std::memory_order_release); + break; + } + } +} + +absl::Mutex* FlagImpl::DataGuard() const { + absl::call_once(const_cast<FlagImpl*>(this)->init_control_, &FlagImpl::Init, + const_cast<FlagImpl*>(this)); + + // data_guard_ is initialized inside Init. + return reinterpret_cast<absl::Mutex*>(&data_guard_); +} + +void FlagImpl::AssertValidType(FlagFastTypeId rhs_type_id, + const std::type_info* (*gen_rtti)()) const { + FlagFastTypeId lhs_type_id = flags_internal::FastTypeId(op_); + + // `rhs_type_id` is the fast type id corresponding to the declaration + // visibile at the call site. `lhs_type_id` is the fast type id + // corresponding to the type specified in flag definition. They must match + // for this operation to be well-defined. + if (ABSL_PREDICT_TRUE(lhs_type_id == rhs_type_id)) return; + + const std::type_info* lhs_runtime_type_id = + flags_internal::RuntimeTypeId(op_); + const std::type_info* rhs_runtime_type_id = (*gen_rtti)(); + + if (lhs_runtime_type_id == rhs_runtime_type_id) return; + +#if defined(ABSL_FLAGS_INTERNAL_HAS_RTTI) + if (*lhs_runtime_type_id == *rhs_runtime_type_id) return; +#endif + + ABSL_INTERNAL_LOG( + FATAL, absl::StrCat("Flag '", Name(), + "' is defined as one type and declared as another")); +} + +std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const { + void* res = nullptr; + switch (DefaultKind()) { + case FlagDefaultKind::kDynamicValue: + res = flags_internal::Clone(op_, default_value_.dynamic_value); + break; + case FlagDefaultKind::kGenFunc: + res = flags_internal::Alloc(op_); + (*default_value_.gen_func)(res); + break; + default: + res = flags_internal::Clone(op_, &default_value_); + break; + } + return {res, DynValueDeleter{op_}}; +} + +void FlagImpl::StoreValue(const void* src) { + switch (ValueStorageKind()) { + case FlagValueStorageKind::kAlignedBuffer: + Copy(op_, src, AlignedBufferValue()); + break; + case FlagValueStorageKind::kOneWordAtomic: { + int64_t one_word_val = 0; + std::memcpy(&one_word_val, src, Sizeof(op_)); + OneWordValue().store(one_word_val, std::memory_order_release); + break; + } + case FlagValueStorageKind::kTwoWordsAtomic: { + AlignedTwoWords two_words_val{0, 0}; + std::memcpy(&two_words_val, src, Sizeof(op_)); + TwoWordsValue().store(two_words_val, std::memory_order_release); + break; + } + } + + modified_ = true; + ++counter_; + InvokeCallback(); +} + +absl::string_view FlagImpl::Name() const { return name_; } + +std::string FlagImpl::Filename() const { + return flags_internal::GetUsageConfig().normalize_filename(filename_); +} + +std::string FlagImpl::Help() const { + return HelpSourceKind() == FlagHelpKind::kLiteral ? help_.literal + : help_.gen_func(); +} + +FlagFastTypeId FlagImpl::TypeId() const { + return flags_internal::FastTypeId(op_); +} + +bool FlagImpl::IsSpecifiedOnCommandLine() const { + absl::MutexLock l(DataGuard()); + return on_command_line_; +} + +std::string FlagImpl::DefaultValue() const { + absl::MutexLock l(DataGuard()); + + auto obj = MakeInitValue(); + return flags_internal::Unparse(op_, obj.get()); +} + +std::string FlagImpl::CurrentValue() const { + auto* guard = DataGuard(); // Make sure flag initialized + switch (ValueStorageKind()) { + case FlagValueStorageKind::kAlignedBuffer: { + absl::MutexLock l(guard); + return flags_internal::Unparse(op_, AlignedBufferValue()); + } + case FlagValueStorageKind::kOneWordAtomic: { + const auto one_word_val = + absl::bit_cast<std::array<char, sizeof(int64_t)>>( + OneWordValue().load(std::memory_order_acquire)); + return flags_internal::Unparse(op_, one_word_val.data()); + } + case FlagValueStorageKind::kTwoWordsAtomic: { + const auto two_words_val = + absl::bit_cast<std::array<char, sizeof(AlignedTwoWords)>>( + TwoWordsValue().load(std::memory_order_acquire)); + return flags_internal::Unparse(op_, two_words_val.data()); + } + } + + return ""; +} + +void FlagImpl::SetCallback(const FlagCallbackFunc mutation_callback) { + absl::MutexLock l(DataGuard()); + + if (callback_ == nullptr) { + callback_ = new FlagCallback; + } + callback_->func = mutation_callback; + + InvokeCallback(); +} + +void FlagImpl::InvokeCallback() const { + if (!callback_) return; + + // Make a copy of the C-style function pointer that we are about to invoke + // before we release the lock guarding it. + FlagCallbackFunc cb = callback_->func; + + // If the flag has a mutation callback this function invokes it. While the + // callback is being invoked the primary flag's mutex is unlocked and it is + // re-locked back after call to callback is completed. Callback invocation is + // guarded by flag's secondary mutex instead which prevents concurrent + // callback invocation. Note that it is possible for other thread to grab the + // primary lock and update flag's value at any time during the callback + // invocation. This is by design. Callback can get a value of the flag if + // necessary, but it might be different from the value initiated the callback + // and it also can be different by the time the callback invocation is + // completed. Requires that *primary_lock be held in exclusive mode; it may be + // released and reacquired by the implementation. + MutexRelock relock(*DataGuard()); + absl::MutexLock lock(&callback_->guard); + cb(); +} + +std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { + absl::MutexLock l(DataGuard()); + + bool modified = modified_; + bool on_command_line = on_command_line_; + switch (ValueStorageKind()) { + case FlagValueStorageKind::kAlignedBuffer: { + return absl::make_unique<FlagState>( + *this, flags_internal::Clone(op_, AlignedBufferValue()), modified, + on_command_line, counter_); + } + case FlagValueStorageKind::kOneWordAtomic: { + return absl::make_unique<FlagState>( + *this, OneWordValue().load(std::memory_order_acquire), modified, + on_command_line, counter_); + } + case FlagValueStorageKind::kTwoWordsAtomic: { + return absl::make_unique<FlagState>( + *this, TwoWordsValue().load(std::memory_order_acquire), modified, + on_command_line, counter_); + } + } + return nullptr; +} + +bool FlagImpl::RestoreState(const FlagState& flag_state) { + absl::MutexLock l(DataGuard()); + + if (flag_state.counter_ == counter_) { + return false; + } + + switch (ValueStorageKind()) { + case FlagValueStorageKind::kAlignedBuffer: + StoreValue(flag_state.value_.heap_allocated); + break; + case FlagValueStorageKind::kOneWordAtomic: + StoreValue(&flag_state.value_.one_word); + break; + case FlagValueStorageKind::kTwoWordsAtomic: + StoreValue(&flag_state.value_.two_words); + break; + } + + modified_ = flag_state.modified_; + on_command_line_ = flag_state.on_command_line_; + + return true; +} + +template <typename StorageT> +StorageT* FlagImpl::OffsetValue() const { + char* p = reinterpret_cast<char*>(const_cast<FlagImpl*>(this)); + // The offset is deduced via Flag value type specific op_. + size_t offset = flags_internal::ValueOffset(op_); + + return reinterpret_cast<StorageT*>(p + offset); +} + +void* FlagImpl::AlignedBufferValue() const { + assert(ValueStorageKind() == FlagValueStorageKind::kAlignedBuffer); + return OffsetValue<void>(); +} + +std::atomic<int64_t>& FlagImpl::OneWordValue() const { + assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic); + return OffsetValue<FlagOneWordValue>()->value; +} + +std::atomic<AlignedTwoWords>& FlagImpl::TwoWordsValue() const { + assert(ValueStorageKind() == FlagValueStorageKind::kTwoWordsAtomic); + return OffsetValue<FlagTwoWordsValue>()->value; +} + +// Attempts to parse supplied `value` string using parsing routine in the `flag` +// argument. If parsing successful, this function replaces the dst with newly +// parsed value. In case if any error is encountered in either step, the error +// message is stored in 'err' +std::unique_ptr<void, DynValueDeleter> FlagImpl::TryParse( + absl::string_view value, std::string& err) const { + std::unique_ptr<void, DynValueDeleter> tentative_value = MakeInitValue(); + + std::string parse_err; + if (!flags_internal::Parse(op_, value, tentative_value.get(), &parse_err)) { + absl::string_view err_sep = parse_err.empty() ? "" : "; "; + err = absl::StrCat("Illegal value '", value, "' specified for flag '", + Name(), "'", err_sep, parse_err); + return nullptr; + } + + return tentative_value; +} + +void FlagImpl::Read(void* dst) const { + auto* guard = DataGuard(); // Make sure flag initialized + switch (ValueStorageKind()) { + case FlagValueStorageKind::kAlignedBuffer: { + absl::MutexLock l(guard); + flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst); + break; + } + case FlagValueStorageKind::kOneWordAtomic: { + const int64_t one_word_val = + OneWordValue().load(std::memory_order_acquire); + std::memcpy(dst, &one_word_val, Sizeof(op_)); + break; + } + case FlagValueStorageKind::kTwoWordsAtomic: { + const AlignedTwoWords two_words_val = + TwoWordsValue().load(std::memory_order_acquire); + std::memcpy(dst, &two_words_val, Sizeof(op_)); + break; + } + } +} + +void FlagImpl::Write(const void* src) { + absl::MutexLock l(DataGuard()); + + if (ShouldValidateFlagValue(flags_internal::FastTypeId(op_))) { + std::unique_ptr<void, DynValueDeleter> obj{flags_internal::Clone(op_, src), + DynValueDeleter{op_}}; + std::string ignored_error; + std::string src_as_str = flags_internal::Unparse(op_, src); + if (!flags_internal::Parse(op_, src_as_str, obj.get(), &ignored_error)) { + ABSL_INTERNAL_LOG(ERROR, absl::StrCat("Attempt to set flag '", Name(), + "' to invalid value ", src_as_str)); + } + } + + StoreValue(src); +} + +// Sets the value of the flag based on specified string `value`. If the flag +// was successfully set to new value, it returns true. Otherwise, sets `err` +// to indicate the error, leaves the flag unchanged, and returns false. There +// are three ways to set the flag's value: +// * Update the current flag value +// * Update the flag's default value +// * Update the current flag value if it was never set before +// The mode is selected based on 'set_mode' parameter. +bool FlagImpl::ParseFrom(absl::string_view value, FlagSettingMode set_mode, + ValueSource source, std::string& err) { + absl::MutexLock l(DataGuard()); + + switch (set_mode) { + case SET_FLAGS_VALUE: { + // set or modify the flag's value + auto tentative_value = TryParse(value, err); + if (!tentative_value) return false; + + StoreValue(tentative_value.get()); + + if (source == kCommandLine) { + on_command_line_ = true; + } + break; + } + case SET_FLAG_IF_DEFAULT: { + // set the flag's value, but only if it hasn't been set by someone else + if (modified_) { + // TODO(rogeeff): review and fix this semantic. Currently we do not fail + // in this case if flag is modified. This is misleading since the flag's + // value is not updated even though we return true. + // *err = absl::StrCat(Name(), " is already set to ", + // CurrentValue(), "\n"); + // return false; + return true; + } + auto tentative_value = TryParse(value, err); + if (!tentative_value) return false; + + StoreValue(tentative_value.get()); + break; + } + case SET_FLAGS_DEFAULT: { + auto tentative_value = TryParse(value, err); + if (!tentative_value) return false; + + if (DefaultKind() == FlagDefaultKind::kDynamicValue) { + void* old_value = default_value_.dynamic_value; + default_value_.dynamic_value = tentative_value.release(); + tentative_value.reset(old_value); + } else { + default_value_.dynamic_value = tentative_value.release(); + def_kind_ = static_cast<uint8_t>(FlagDefaultKind::kDynamicValue); + } + + if (!modified_) { + // Need to set both default value *and* current, in this case. + StoreValue(default_value_.dynamic_value); + modified_ = false; + } + break; + } + } + + return true; +} + +void FlagImpl::CheckDefaultValueParsingRoundtrip() const { + std::string v = DefaultValue(); + + absl::MutexLock lock(DataGuard()); + + auto dst = MakeInitValue(); + std::string error; + if (!flags_internal::Parse(op_, v, dst.get(), &error)) { + ABSL_INTERNAL_LOG( + FATAL, + absl::StrCat("Flag ", Name(), " (from ", Filename(), + "): string form of default value '", v, + "' could not be parsed; error=", error)); + } + + // We do not compare dst to def since parsing/unparsing may make + // small changes, e.g., precision loss for floating point types. +} + +bool FlagImpl::ValidateInputValue(absl::string_view value) const { + absl::MutexLock l(DataGuard()); + + auto obj = MakeInitValue(); + std::string ignored_error; + return flags_internal::Parse(op_, value, obj.get(), &ignored_error); +} + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/internal/flag.h b/third_party/abseil_cpp/absl/flags/internal/flag.h new file mode 100644 index 000000000000..370d8a02ef04 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/flag.h @@ -0,0 +1,775 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_FLAG_H_ +#define ABSL_FLAGS_INTERNAL_FLAG_H_ + +#include <stddef.h> +#include <stdint.h> + +#include <atomic> +#include <cstring> +#include <memory> +#include <new> +#include <string> +#include <type_traits> +#include <typeinfo> + +#include "absl/base/attributes.h" +#include "absl/base/call_once.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/config.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/flags/internal/registry.h" +#include "absl/flags/marshalling.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Forward declaration of absl::Flag<T> public API. +namespace flags_internal { +template <typename T> +class Flag; +} // namespace flags_internal + +#if defined(_MSC_VER) && !defined(__clang__) +template <typename T> +class Flag; +#else +template <typename T> +using Flag = flags_internal::Flag<T>; +#endif + +template <typename T> +ABSL_MUST_USE_RESULT T GetFlag(const absl::Flag<T>& flag); + +template <typename T> +void SetFlag(absl::Flag<T>* flag, const T& v); + +template <typename T, typename V> +void SetFlag(absl::Flag<T>* flag, const V& v); + +template <typename U> +const CommandLineFlag& GetFlagReflectionHandle(const absl::Flag<U>& f); + +/////////////////////////////////////////////////////////////////////////////// +// Flag value type operations, eg., parsing, copying, etc. are provided +// by function specific to that type with a signature matching FlagOpFn. + +namespace flags_internal { + +enum class FlagOp { + kAlloc, + kDelete, + kCopy, + kCopyConstruct, + kSizeof, + kFastTypeId, + kRuntimeTypeId, + kParse, + kUnparse, + kValueOffset, +}; +using FlagOpFn = void* (*)(FlagOp, const void*, void*, void*); + +// Forward declaration for Flag value specific operations. +template <typename T> +void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3); + +// Allocate aligned memory for a flag value. +inline void* Alloc(FlagOpFn op) { + return op(FlagOp::kAlloc, nullptr, nullptr, nullptr); +} +// Deletes memory interpreting obj as flag value type pointer. +inline void Delete(FlagOpFn op, void* obj) { + op(FlagOp::kDelete, nullptr, obj, nullptr); +} +// Copies src to dst interpreting as flag value type pointers. +inline void Copy(FlagOpFn op, const void* src, void* dst) { + op(FlagOp::kCopy, src, dst, nullptr); +} +// Construct a copy of flag value in a location pointed by dst +// based on src - pointer to the flag's value. +inline void CopyConstruct(FlagOpFn op, const void* src, void* dst) { + op(FlagOp::kCopyConstruct, src, dst, nullptr); +} +// Makes a copy of flag value pointed by obj. +inline void* Clone(FlagOpFn op, const void* obj) { + void* res = flags_internal::Alloc(op); + flags_internal::CopyConstruct(op, obj, res); + return res; +} +// Returns true if parsing of input text is successfull. +inline bool Parse(FlagOpFn op, absl::string_view text, void* dst, + std::string* error) { + return op(FlagOp::kParse, &text, dst, error) != nullptr; +} +// Returns string representing supplied value. +inline std::string Unparse(FlagOpFn op, const void* val) { + std::string result; + op(FlagOp::kUnparse, val, &result, nullptr); + return result; +} +// Returns size of flag value type. +inline size_t Sizeof(FlagOpFn op) { + // This sequence of casts reverses the sequence from + // `flags_internal::FlagOps()` + return static_cast<size_t>(reinterpret_cast<intptr_t>( + op(FlagOp::kSizeof, nullptr, nullptr, nullptr))); +} +// Returns fast type id coresponding to the value type. +inline FlagFastTypeId FastTypeId(FlagOpFn op) { + return reinterpret_cast<FlagFastTypeId>( + op(FlagOp::kFastTypeId, nullptr, nullptr, nullptr)); +} +// Returns fast type id coresponding to the value type. +inline const std::type_info* RuntimeTypeId(FlagOpFn op) { + return reinterpret_cast<const std::type_info*>( + op(FlagOp::kRuntimeTypeId, nullptr, nullptr, nullptr)); +} +// Returns offset of the field value_ from the field impl_ inside of +// absl::Flag<T> data. Given FlagImpl pointer p you can get the +// location of the corresponding value as: +// reinterpret_cast<char*>(p) + ValueOffset(). +inline ptrdiff_t ValueOffset(FlagOpFn op) { + // This sequence of casts reverses the sequence from + // `flags_internal::FlagOps()` + return static_cast<ptrdiff_t>(reinterpret_cast<intptr_t>( + op(FlagOp::kValueOffset, nullptr, nullptr, nullptr))); +} + +// Returns an address of RTTI's typeid(T). +template <typename T> +inline const std::type_info* GenRuntimeTypeId() { +#if defined(ABSL_FLAGS_INTERNAL_HAS_RTTI) + return &typeid(T); +#else + return nullptr; +#endif +} + +/////////////////////////////////////////////////////////////////////////////// +// Flag help auxiliary structs. + +// This is help argument for absl::Flag encapsulating the string literal pointer +// or pointer to function generating it as well as enum descriminating two +// cases. +using HelpGenFunc = std::string (*)(); + +template <size_t N> +struct FixedCharArray { + char value[N]; + + template <size_t... I> + static constexpr FixedCharArray<N> FromLiteralString( + absl::string_view str, absl::index_sequence<I...>) { + return (void)str, FixedCharArray<N>({{str[I]..., '\0'}}); + } +}; + +template <typename Gen, size_t N = Gen::Value().size()> +constexpr FixedCharArray<N + 1> HelpStringAsArray(int) { + return FixedCharArray<N + 1>::FromLiteralString( + Gen::Value(), absl::make_index_sequence<N>{}); +} + +template <typename Gen> +constexpr std::false_type HelpStringAsArray(char) { + return std::false_type{}; +} + +union FlagHelpMsg { + constexpr explicit FlagHelpMsg(const char* help_msg) : literal(help_msg) {} + constexpr explicit FlagHelpMsg(HelpGenFunc help_gen) : gen_func(help_gen) {} + + const char* literal; + HelpGenFunc gen_func; +}; + +enum class FlagHelpKind : uint8_t { kLiteral = 0, kGenFunc = 1 }; + +struct FlagHelpArg { + FlagHelpMsg source; + FlagHelpKind kind; +}; + +extern const char kStrippedFlagHelp[]; + +// These two HelpArg overloads allows us to select at compile time one of two +// way to pass Help argument to absl::Flag. We'll be passing +// AbslFlagHelpGenFor##name as Gen and integer 0 as a single argument to prefer +// first overload if possible. If help message is evaluatable on constexpr +// context We'll be able to make FixedCharArray out of it and we'll choose first +// overload. In this case the help message expression is immediately evaluated +// and is used to construct the absl::Flag. No additionl code is generated by +// ABSL_FLAG Otherwise SFINAE kicks in and first overload is dropped from the +// consideration, in which case the second overload will be used. The second +// overload does not attempt to evaluate the help message expression +// immediately and instead delays the evaluation by returing the function +// pointer (&T::NonConst) genering the help message when necessary. This is +// evaluatable in constexpr context, but the cost is an extra function being +// generated in the ABSL_FLAG code. +template <typename Gen, size_t N> +constexpr FlagHelpArg HelpArg(const FixedCharArray<N>& value) { + return {FlagHelpMsg(value.value), FlagHelpKind::kLiteral}; +} + +template <typename Gen> +constexpr FlagHelpArg HelpArg(std::false_type) { + return {FlagHelpMsg(&Gen::NonConst), FlagHelpKind::kGenFunc}; +} + +/////////////////////////////////////////////////////////////////////////////// +// Flag default value auxiliary structs. + +// Signature for the function generating the initial flag value (usually +// based on default value supplied in flag's definition) +using FlagDfltGenFunc = void (*)(void*); + +union FlagDefaultSrc { + constexpr explicit FlagDefaultSrc(FlagDfltGenFunc gen_func_arg) + : gen_func(gen_func_arg) {} + +#define ABSL_FLAGS_INTERNAL_DFLT_FOR_TYPE(T, name) \ + T name##_value; \ + constexpr explicit FlagDefaultSrc(T value) : name##_value(value) {} // NOLINT + ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(ABSL_FLAGS_INTERNAL_DFLT_FOR_TYPE) +#undef ABSL_FLAGS_INTERNAL_DFLT_FOR_TYPE + + void* dynamic_value; + FlagDfltGenFunc gen_func; +}; + +enum class FlagDefaultKind : uint8_t { + kDynamicValue = 0, + kGenFunc = 1, + kOneWord = 2 // for default values UP to one word in size +}; + +struct FlagDefaultArg { + FlagDefaultSrc source; + FlagDefaultKind kind; +}; + +// This struct and corresponding overload to InitDefaultValue are used to +// facilitate usage of {} as default value in ABSL_FLAG macro. +// TODO(rogeeff): Fix handling types with explicit constructors. +struct EmptyBraces {}; + +template <typename T> +constexpr T InitDefaultValue(T t) { + return t; +} + +template <typename T> +constexpr T InitDefaultValue(EmptyBraces) { + return T{}; +} + +template <typename ValueT, typename GenT, + typename std::enable_if<std::is_integral<ValueT>::value, int>::type = + (GenT{}, 0)> +constexpr FlagDefaultArg DefaultArg(int) { + return {FlagDefaultSrc(GenT{}.value), FlagDefaultKind::kOneWord}; +} + +template <typename ValueT, typename GenT> +constexpr FlagDefaultArg DefaultArg(char) { + return {FlagDefaultSrc(&GenT::Gen), FlagDefaultKind::kGenFunc}; +} + +/////////////////////////////////////////////////////////////////////////////// +// Flag current value auxiliary structs. + +constexpr int64_t UninitializedFlagValue() { return 0xababababababababll; } + +template <typename T> +using FlagUseOneWordStorage = std::integral_constant< + bool, absl::type_traits_internal::is_trivially_copyable<T>::value && + (sizeof(T) <= 8)>; + +#if defined(ABSL_FLAGS_INTERNAL_ATOMIC_DOUBLE_WORD) +// Clang does not always produce cmpxchg16b instruction when alignment of a 16 +// bytes type is not 16. +struct alignas(16) AlignedTwoWords { + int64_t first; + int64_t second; + + bool IsInitialized() const { + return first != flags_internal::UninitializedFlagValue(); + } +}; + +template <typename T> +using FlagUseTwoWordsStorage = std::integral_constant< + bool, absl::type_traits_internal::is_trivially_copyable<T>::value && + (sizeof(T) > 8) && (sizeof(T) <= 16)>; +#else +// This is actually unused and only here to avoid ifdefs in other palces. +struct AlignedTwoWords { + constexpr AlignedTwoWords() noexcept : dummy() {} + constexpr AlignedTwoWords(int64_t, int64_t) noexcept : dummy() {} + char dummy; + + bool IsInitialized() const { + std::abort(); + return true; + } +}; + +// This trait should be type dependent, otherwise SFINAE below will fail +template <typename T> +using FlagUseTwoWordsStorage = + std::integral_constant<bool, sizeof(T) != sizeof(T)>; +#endif + +template <typename T> +using FlagUseBufferStorage = + std::integral_constant<bool, !FlagUseOneWordStorage<T>::value && + !FlagUseTwoWordsStorage<T>::value>; + +enum class FlagValueStorageKind : uint8_t { + kAlignedBuffer = 0, + kOneWordAtomic = 1, + kTwoWordsAtomic = 2 +}; + +template <typename T> +static constexpr FlagValueStorageKind StorageKind() { + return FlagUseBufferStorage<T>::value + ? FlagValueStorageKind::kAlignedBuffer + : FlagUseOneWordStorage<T>::value + ? FlagValueStorageKind::kOneWordAtomic + : FlagValueStorageKind::kTwoWordsAtomic; +} + +struct FlagOneWordValue { + constexpr FlagOneWordValue() : value(UninitializedFlagValue()) {} + + std::atomic<int64_t> value; +}; + +struct FlagTwoWordsValue { + constexpr FlagTwoWordsValue() + : value(AlignedTwoWords{UninitializedFlagValue(), 0}) {} + + std::atomic<AlignedTwoWords> value; +}; + +template <typename T, + FlagValueStorageKind Kind = flags_internal::StorageKind<T>()> +struct FlagValue; + +template <typename T> +struct FlagValue<T, FlagValueStorageKind::kAlignedBuffer> { + bool Get(T&) const { return false; } + + alignas(T) char value[sizeof(T)]; +}; + +template <typename T> +struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { + bool Get(T& dst) const { + int64_t one_word_val = value.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) { + return false; + } + std::memcpy(&dst, static_cast<const void*>(&one_word_val), sizeof(T)); + return true; + } +}; + +template <typename T> +struct FlagValue<T, FlagValueStorageKind::kTwoWordsAtomic> : FlagTwoWordsValue { + bool Get(T& dst) const { + AlignedTwoWords two_words_val = value.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(!two_words_val.IsInitialized())) { + return false; + } + std::memcpy(&dst, static_cast<const void*>(&two_words_val), sizeof(T)); + return true; + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// Flag callback auxiliary structs. + +// Signature for the mutation callback used by watched Flags +// The callback is noexcept. +// TODO(rogeeff): add noexcept after C++17 support is added. +using FlagCallbackFunc = void (*)(); + +struct FlagCallback { + FlagCallbackFunc func; + absl::Mutex guard; // Guard for concurrent callback invocations. +}; + +/////////////////////////////////////////////////////////////////////////////// +// Flag implementation, which does not depend on flag value type. +// The class encapsulates the Flag's data and access to it. + +struct DynValueDeleter { + explicit DynValueDeleter(FlagOpFn op_arg = nullptr); + void operator()(void* ptr) const; + + FlagOpFn op; +}; + +class FlagState; + +class FlagImpl final : public CommandLineFlag { + public: + constexpr FlagImpl(const char* name, const char* filename, FlagOpFn op, + FlagHelpArg help, FlagValueStorageKind value_kind, + FlagDefaultArg default_arg) + : name_(name), + filename_(filename), + op_(op), + help_(help.source), + help_source_kind_(static_cast<uint8_t>(help.kind)), + value_storage_kind_(static_cast<uint8_t>(value_kind)), + def_kind_(static_cast<uint8_t>(default_arg.kind)), + modified_(false), + on_command_line_(false), + counter_(0), + callback_(nullptr), + default_value_(default_arg.source), + data_guard_{} {} + + // Constant access methods + void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + + // Mutating access methods + void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard()); + + // Interfaces to operate on callbacks. + void SetCallback(const FlagCallbackFunc mutation_callback) + ABSL_LOCKS_EXCLUDED(*DataGuard()); + void InvokeCallback() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + + // Used in read/write operations to validate source/target has correct type. + // For example if flag is declared as absl::Flag<int> FLAGS_foo, a call to + // absl::GetFlag(FLAGS_foo) validates that the type of FLAGS_foo is indeed + // int. To do that we pass the "assumed" type id (which is deduced from type + // int) as an argument `type_id`, which is in turn is validated against the + // type id stored in flag object by flag definition statement. + void AssertValidType(FlagFastTypeId type_id, + const std::type_info* (*gen_rtti)()) const; + + private: + template <typename T> + friend class Flag; + friend class FlagState; + + // Ensures that `data_guard_` is initialized and returns it. + absl::Mutex* DataGuard() const + ABSL_LOCK_RETURNED(reinterpret_cast<absl::Mutex*>(data_guard_)); + // Returns heap allocated value of type T initialized with default value. + std::unique_ptr<void, DynValueDeleter> MakeInitValue() const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + // Flag initialization called via absl::call_once. + void Init(); + + // Offset value access methods. One per storage kind. These methods to not + // respect const correctness, so be very carefull using them. + + // This is a shared helper routine which encapsulates most of the magic. Since + // it is only used inside the three routines below, which are defined in + // flag.cc, we can define it in that file as well. + template <typename StorageT> + StorageT* OffsetValue() const; + // This is an accessor for a value stored in an aligned buffer storage. + // Returns a mutable pointer to the start of a buffer. + void* AlignedBufferValue() const; + // This is an accessor for a value stored as one word atomic. Returns a + // mutable reference to an atomic value. + std::atomic<int64_t>& OneWordValue() const; + // This is an accessor for a value stored as two words atomic. Returns a + // mutable reference to an atomic value. + std::atomic<AlignedTwoWords>& TwoWordsValue() const; + + // Attempts to parse supplied `value` string. If parsing is successful, + // returns new value. Otherwise returns nullptr. + std::unique_ptr<void, DynValueDeleter> TryParse(absl::string_view value, + std::string& err) const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + // Stores the flag value based on the pointer to the source. + void StoreValue(const void* src) ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()); + + FlagHelpKind HelpSourceKind() const { + return static_cast<FlagHelpKind>(help_source_kind_); + } + FlagValueStorageKind ValueStorageKind() const { + return static_cast<FlagValueStorageKind>(value_storage_kind_); + } + FlagDefaultKind DefaultKind() const + ABSL_EXCLUSIVE_LOCKS_REQUIRED(*DataGuard()) { + return static_cast<FlagDefaultKind>(def_kind_); + } + + // CommandLineFlag interface implementation + absl::string_view Name() const override; + std::string Filename() const override; + std::string Help() const override; + FlagFastTypeId TypeId() const override; + bool IsSpecifiedOnCommandLine() const override + ABSL_LOCKS_EXCLUDED(*DataGuard()); + std::string DefaultValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + std::string CurrentValue() const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + bool ValidateInputValue(absl::string_view value) const override + ABSL_LOCKS_EXCLUDED(*DataGuard()); + void CheckDefaultValueParsingRoundtrip() const override + ABSL_LOCKS_EXCLUDED(*DataGuard()); + + // Interfaces to save and restore flags to/from persistent state. + // Returns current flag state or nullptr if flag does not support + // saving and restoring a state. + std::unique_ptr<FlagStateInterface> SaveState() override + ABSL_LOCKS_EXCLUDED(*DataGuard()); + + // Restores the flag state to the supplied state object. If there is + // nothing to restore returns false. Otherwise returns true. + bool RestoreState(const FlagState& flag_state) + ABSL_LOCKS_EXCLUDED(*DataGuard()); + + bool ParseFrom(absl::string_view value, FlagSettingMode set_mode, + ValueSource source, std::string& error) override + ABSL_LOCKS_EXCLUDED(*DataGuard()); + + // Immutable flag's state. + + // Flags name passed to ABSL_FLAG as second arg. + const char* const name_; + // The file name where ABSL_FLAG resides. + const char* const filename_; + // Type-specific operations "vtable". + const FlagOpFn op_; + // Help message literal or function to generate it. + const FlagHelpMsg help_; + // Indicates if help message was supplied as literal or generator func. + const uint8_t help_source_kind_ : 1; + // Kind of storage this flag is using for the flag's value. + const uint8_t value_storage_kind_ : 2; + + uint8_t : 0; // The bytes containing the const bitfields must not be + // shared with bytes containing the mutable bitfields. + + // Mutable flag's state (guarded by `data_guard_`). + + // def_kind_ is not guard by DataGuard() since it is accessed in Init without + // locks. + uint8_t def_kind_ : 2; + // Has this flag's value been modified? + bool modified_ : 1 ABSL_GUARDED_BY(*DataGuard()); + // Has this flag been specified on command line. + bool on_command_line_ : 1 ABSL_GUARDED_BY(*DataGuard()); + + // Unique tag for absl::call_once call to initialize this flag. + absl::once_flag init_control_; + + // Mutation counter + int64_t counter_ ABSL_GUARDED_BY(*DataGuard()); + // Optional flag's callback and absl::Mutex to guard the invocations. + FlagCallback* callback_ ABSL_GUARDED_BY(*DataGuard()); + // Either a pointer to the function generating the default value based on the + // value specified in ABSL_FLAG or pointer to the dynamically set default + // value via SetCommandLineOptionWithMode. def_kind_ is used to distinguish + // these two cases. + FlagDefaultSrc default_value_; + + // This is reserved space for an absl::Mutex to guard flag data. It will be + // initialized in FlagImpl::Init via placement new. + // We can't use "absl::Mutex data_guard_", since this class is not literal. + // We do not want to use "absl::Mutex* data_guard_", since this would require + // heap allocation during initialization, which is both slows program startup + // and can fail. Using reserved space + placement new allows us to avoid both + // problems. + alignas(absl::Mutex) mutable char data_guard_[sizeof(absl::Mutex)]; +}; + +/////////////////////////////////////////////////////////////////////////////// +// The Flag object parameterized by the flag's value type. This class implements +// flag reflection handle interface. + +template <typename T> +class Flag { + public: + constexpr Flag(const char* name, const char* filename, FlagHelpArg help, + const FlagDefaultArg default_arg) + : impl_(name, filename, &FlagOps<T>, help, + flags_internal::StorageKind<T>(), default_arg), + value_() {} + + // CommandLineFlag interface + absl::string_view Name() const { return impl_.Name(); } + std::string Filename() const { return impl_.Filename(); } + std::string Help() const { return impl_.Help(); } + // Do not use. To be removed. + bool IsSpecifiedOnCommandLine() const { + return impl_.IsSpecifiedOnCommandLine(); + } + std::string DefaultValue() const { return impl_.DefaultValue(); } + std::string CurrentValue() const { return impl_.CurrentValue(); } + + private: + template <typename, bool> + friend class FlagRegistrar; + friend class FlagImplPeer; + + T Get() const { + // See implementation notes in CommandLineFlag::Get(). + union U { + T value; + U() {} + ~U() { value.~T(); } + }; + U u; + +#if !defined(NDEBUG) + impl_.AssertValidType(base_internal::FastTypeId<T>(), &GenRuntimeTypeId<T>); +#endif + + if (!value_.Get(u.value)) impl_.Read(&u.value); + return std::move(u.value); + } + void Set(const T& v) { + impl_.AssertValidType(base_internal::FastTypeId<T>(), &GenRuntimeTypeId<T>); + impl_.Write(&v); + } + + // Access to the reflection. + const CommandLineFlag& Reflect() const { return impl_; } + + // Flag's data + // The implementation depends on value_ field to be placed exactly after the + // impl_ field, so that impl_ can figure out the offset to the value and + // access it. + FlagImpl impl_; + FlagValue<T> value_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Trampoline for friend access + +class FlagImplPeer { + public: + template <typename T, typename FlagType> + static T InvokeGet(const FlagType& flag) { + return flag.Get(); + } + template <typename FlagType, typename T> + static void InvokeSet(FlagType& flag, const T& v) { + flag.Set(v); + } + template <typename FlagType> + static const CommandLineFlag& InvokeReflect(const FlagType& f) { + return f.Reflect(); + } +}; + +/////////////////////////////////////////////////////////////////////////////// +// Implementation of Flag value specific operations routine. +template <typename T> +void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { + switch (op) { + case FlagOp::kAlloc: { + std::allocator<T> alloc; + return std::allocator_traits<std::allocator<T>>::allocate(alloc, 1); + } + case FlagOp::kDelete: { + T* p = static_cast<T*>(v2); + p->~T(); + std::allocator<T> alloc; + std::allocator_traits<std::allocator<T>>::deallocate(alloc, p, 1); + return nullptr; + } + case FlagOp::kCopy: + *static_cast<T*>(v2) = *static_cast<const T*>(v1); + return nullptr; + case FlagOp::kCopyConstruct: + new (v2) T(*static_cast<const T*>(v1)); + return nullptr; + case FlagOp::kSizeof: + return reinterpret_cast<void*>(static_cast<uintptr_t>(sizeof(T))); + case FlagOp::kFastTypeId: + return const_cast<void*>(base_internal::FastTypeId<T>()); + case FlagOp::kRuntimeTypeId: + return const_cast<std::type_info*>(GenRuntimeTypeId<T>()); + case FlagOp::kParse: { + // Initialize the temporary instance of type T based on current value in + // destination (which is going to be flag's default value). + T temp(*static_cast<T*>(v2)); + if (!absl::ParseFlag<T>(*static_cast<const absl::string_view*>(v1), &temp, + static_cast<std::string*>(v3))) { + return nullptr; + } + *static_cast<T*>(v2) = std::move(temp); + return v2; + } + case FlagOp::kUnparse: + *static_cast<std::string*>(v2) = + absl::UnparseFlag<T>(*static_cast<const T*>(v1)); + return nullptr; + case FlagOp::kValueOffset: { + // Round sizeof(FlagImp) to a multiple of alignof(FlagValue<T>) to get the + // offset of the data. + ptrdiff_t round_to = alignof(FlagValue<T>); + ptrdiff_t offset = + (sizeof(FlagImpl) + round_to - 1) / round_to * round_to; + return reinterpret_cast<void*>(offset); + } + } + return nullptr; +} + +/////////////////////////////////////////////////////////////////////////////// +// This class facilitates Flag object registration and tail expression-based +// flag definition, for example: +// ABSL_FLAG(int, foo, 42, "Foo help").OnUpdate(NotifyFooWatcher); +struct FlagRegistrarEmpty {}; +template <typename T, bool do_register> +class FlagRegistrar { + public: + explicit FlagRegistrar(Flag<T>& flag) : flag_(flag) { + if (do_register) flags_internal::RegisterCommandLineFlag(flag_.impl_); + } + + FlagRegistrar OnUpdate(FlagCallbackFunc cb) && { + flag_.impl_.SetCallback(cb); + return *this; + } + + // Make the registrar "die" gracefully as an empty struct on a line where + // registration happens. Registrar objects are intended to live only as + // temporary. + operator FlagRegistrarEmpty() const { return {}; } // NOLINT + + private: + Flag<T>& flag_; // Flag being registered (not owned). +}; + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_FLAG_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/parse.h b/third_party/abseil_cpp/absl/flags/internal/parse.h new file mode 100644 index 000000000000..de706c898470 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/parse.h @@ -0,0 +1,59 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_PARSE_H_ +#define ABSL_FLAGS_INTERNAL_PARSE_H_ + +#include <string> +#include <vector> + +#include "absl/base/config.h" +#include "absl/flags/declare.h" +#include "absl/strings/string_view.h" + +ABSL_DECLARE_FLAG(std::vector<std::string>, flagfile); +ABSL_DECLARE_FLAG(std::vector<std::string>, fromenv); +ABSL_DECLARE_FLAG(std::vector<std::string>, tryfromenv); +ABSL_DECLARE_FLAG(std::vector<std::string>, undefok); + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +enum class ArgvListAction { kRemoveParsedArgs, kKeepParsedArgs }; +enum class UsageFlagsAction { kHandleUsage, kIgnoreUsage }; +enum class OnUndefinedFlag { + kIgnoreUndefined, + kReportUndefined, + kAbortIfUndefined +}; + +std::vector<char*> ParseCommandLineImpl(int argc, char* argv[], + ArgvListAction arg_list_act, + UsageFlagsAction usage_flag_act, + OnUndefinedFlag on_undef_flag); + +// -------------------------------------------------------------------- +// Inspect original command line + +// Returns true if flag with specified name was either present on the original +// command line or specified in flag file present on the original command line. +bool WasPresentOnCommandLine(absl::string_view flag_name); + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_PARSE_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/path_util.h b/third_party/abseil_cpp/absl/flags/internal/path_util.h new file mode 100644 index 000000000000..a6594d3347e1 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/path_util.h @@ -0,0 +1,62 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_PATH_UTIL_H_ +#define ABSL_FLAGS_INTERNAL_PATH_UTIL_H_ + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// A portable interface that returns the basename of the filename passed as an +// argument. It is similar to basename(3) +// <https://linux.die.net/man/3/basename>. +// For example: +// flags_internal::Basename("a/b/prog/file.cc") +// returns "file.cc" +// flags_internal::Basename("file.cc") +// returns "file.cc" +inline absl::string_view Basename(absl::string_view filename) { + auto last_slash_pos = filename.find_last_of("/\\"); + + return last_slash_pos == absl::string_view::npos + ? filename + : filename.substr(last_slash_pos + 1); +} + +// A portable interface that returns the directory name of the filename +// passed as an argument, including the trailing slash. +// Returns the empty string if a slash is not found in the input file name. +// For example: +// flags_internal::Package("a/b/prog/file.cc") +// returns "a/b/prog/" +// flags_internal::Package("file.cc") +// returns "" +inline absl::string_view Package(absl::string_view filename) { + auto last_slash_pos = filename.find_last_of("/\\"); + + return last_slash_pos == absl::string_view::npos + ? absl::string_view() + : filename.substr(0, last_slash_pos + 1); +} + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_PATH_UTIL_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/path_util_test.cc b/third_party/abseil_cpp/absl/flags/internal/path_util_test.cc new file mode 100644 index 000000000000..2091373c88ea --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/path_util_test.cc @@ -0,0 +1,46 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/path_util.h" + +#include "gtest/gtest.h" + +namespace { + +namespace flags = absl::flags_internal; + +TEST(FlagsPathUtilTest, TestBasename) { + EXPECT_EQ(flags::Basename(""), ""); + EXPECT_EQ(flags::Basename("a.cc"), "a.cc"); + EXPECT_EQ(flags::Basename("dir/a.cc"), "a.cc"); + EXPECT_EQ(flags::Basename("dir1/dir2/a.cc"), "a.cc"); + EXPECT_EQ(flags::Basename("../dir1/dir2/a.cc"), "a.cc"); + EXPECT_EQ(flags::Basename("/dir1/dir2/a.cc"), "a.cc"); + EXPECT_EQ(flags::Basename("/dir1/dir2/../dir3/a.cc"), "a.cc"); +} + +// -------------------------------------------------------------------- + +TEST(FlagsPathUtilTest, TestPackage) { + EXPECT_EQ(flags::Package(""), ""); + EXPECT_EQ(flags::Package("a.cc"), ""); + EXPECT_EQ(flags::Package("dir/a.cc"), "dir/"); + EXPECT_EQ(flags::Package("dir1/dir2/a.cc"), "dir1/dir2/"); + EXPECT_EQ(flags::Package("../dir1/dir2/a.cc"), "../dir1/dir2/"); + EXPECT_EQ(flags::Package("/dir1/dir2/a.cc"), "/dir1/dir2/"); + EXPECT_EQ(flags::Package("/dir1/dir2/../dir3/a.cc"), "/dir1/dir2/../dir3/"); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/internal/private_handle_accessor.cc b/third_party/abseil_cpp/absl/flags/internal/private_handle_accessor.cc new file mode 100644 index 000000000000..a7eb58b6d45f --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/private_handle_accessor.cc @@ -0,0 +1,65 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/private_handle_accessor.h" + +#include <memory> +#include <string> + +#include "absl/base/config.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +FlagFastTypeId PrivateHandleAccessor::TypeId(const CommandLineFlag& flag) { + return flag.TypeId(); +} + +std::unique_ptr<FlagStateInterface> PrivateHandleAccessor::SaveState( + CommandLineFlag& flag) { + return flag.SaveState(); +} + +bool PrivateHandleAccessor::IsSpecifiedOnCommandLine( + const CommandLineFlag& flag) { + return flag.IsSpecifiedOnCommandLine(); +} + +bool PrivateHandleAccessor::ValidateInputValue(const CommandLineFlag& flag, + absl::string_view value) { + return flag.ValidateInputValue(value); +} + +void PrivateHandleAccessor::CheckDefaultValueParsingRoundtrip( + const CommandLineFlag& flag) { + flag.CheckDefaultValueParsingRoundtrip(); +} + +bool PrivateHandleAccessor::ParseFrom(CommandLineFlag& flag, + absl::string_view value, + flags_internal::FlagSettingMode set_mode, + flags_internal::ValueSource source, + std::string& error) { + return flag.ParseFrom(value, set_mode, source, error); +} + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + diff --git a/third_party/abseil_cpp/absl/flags/internal/private_handle_accessor.h b/third_party/abseil_cpp/absl/flags/internal/private_handle_accessor.h new file mode 100644 index 000000000000..c64435cd6199 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/private_handle_accessor.h @@ -0,0 +1,61 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_PRIVATE_HANDLE_ACCESSOR_H_ +#define ABSL_FLAGS_INTERNAL_PRIVATE_HANDLE_ACCESSOR_H_ + +#include <memory> +#include <string> + +#include "absl/base/config.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// This class serves as a trampoline to access private methods of +// CommandLineFlag. This class is intended for use exclusively internally inside +// of the Abseil Flags implementation. +class PrivateHandleAccessor { + public: + // Access to CommandLineFlag::TypeId. + static FlagFastTypeId TypeId(const CommandLineFlag& flag); + + // Access to CommandLineFlag::SaveState. + static std::unique_ptr<FlagStateInterface> SaveState(CommandLineFlag& flag); + + // Access to CommandLineFlag::IsSpecifiedOnCommandLine. + static bool IsSpecifiedOnCommandLine(const CommandLineFlag& flag); + + // Access to CommandLineFlag::ValidateInputValue. + static bool ValidateInputValue(const CommandLineFlag& flag, + absl::string_view value); + + // Access to CommandLineFlag::CheckDefaultValueParsingRoundtrip. + static void CheckDefaultValueParsingRoundtrip(const CommandLineFlag& flag); + + static bool ParseFrom(CommandLineFlag& flag, absl::string_view value, + flags_internal::FlagSettingMode set_mode, + flags_internal::ValueSource source, std::string& error); +}; + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_PRIVATE_HANDLE_ACCESSOR_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/program_name.cc b/third_party/abseil_cpp/absl/flags/internal/program_name.cc new file mode 100644 index 000000000000..51d698da8b31 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/program_name.cc @@ -0,0 +1,60 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/program_name.h" + +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/const_init.h" +#include "absl/base/thread_annotations.h" +#include "absl/flags/internal/path_util.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +ABSL_CONST_INIT static absl::Mutex program_name_guard(absl::kConstInit); +ABSL_CONST_INIT static std::string* program_name + ABSL_GUARDED_BY(program_name_guard) = nullptr; + +std::string ProgramInvocationName() { + absl::MutexLock l(&program_name_guard); + + return program_name ? *program_name : "UNKNOWN"; +} + +std::string ShortProgramInvocationName() { + absl::MutexLock l(&program_name_guard); + + return program_name ? std::string(flags_internal::Basename(*program_name)) + : "UNKNOWN"; +} + +void SetProgramInvocationName(absl::string_view prog_name_str) { + absl::MutexLock l(&program_name_guard); + + if (!program_name) + program_name = new std::string(prog_name_str); + else + program_name->assign(prog_name_str.data(), prog_name_str.size()); +} + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/internal/program_name.h b/third_party/abseil_cpp/absl/flags/internal/program_name.h new file mode 100644 index 000000000000..b99b94fe18ab --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/program_name.h @@ -0,0 +1,50 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_PROGRAM_NAME_H_ +#define ABSL_FLAGS_INTERNAL_PROGRAM_NAME_H_ + +#include <string> + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +// -------------------------------------------------------------------- +// Program name + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// Returns program invocation name or "UNKNOWN" if `SetProgramInvocationName()` +// is never called. At the moment this is always set to argv[0] as part of +// library initialization. +std::string ProgramInvocationName(); + +// Returns base name for program invocation name. For example, if +// ProgramInvocationName() == "a/b/mybinary" +// then +// ShortProgramInvocationName() == "mybinary" +std::string ShortProgramInvocationName(); + +// Sets program invocation name to a new value. Should only be called once +// during program initialization, before any threads are spawned. +void SetProgramInvocationName(absl::string_view prog_name_str); + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_PROGRAM_NAME_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/program_name_test.cc b/third_party/abseil_cpp/absl/flags/internal/program_name_test.cc new file mode 100644 index 000000000000..aff9f6315e4e --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/program_name_test.cc @@ -0,0 +1,61 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/program_name.h" + +#include <string> + +#include "gtest/gtest.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" + +namespace { + +namespace flags = absl::flags_internal; + +TEST(FlagsPathUtilTest, TestProgamNameInterfaces) { + flags::SetProgramInvocationName("absl/flags/program_name_test"); + std::string program_name = flags::ProgramInvocationName(); + for (char& c : program_name) + if (c == '\\') c = '/'; + +#if !defined(__wasm__) && !defined(__asmjs__) + const std::string expect_name = "absl/flags/program_name_test"; + const std::string expect_basename = "program_name_test"; +#else + // For targets that generate javascript or webassembly the invocation name + // has the special value below. + const std::string expect_name = "this.program"; + const std::string expect_basename = "this.program"; +#endif + + EXPECT_TRUE(absl::EndsWith(program_name, expect_name)) << program_name; + EXPECT_EQ(flags::ShortProgramInvocationName(), expect_basename); + + flags::SetProgramInvocationName("a/my_test"); + + EXPECT_EQ(flags::ProgramInvocationName(), "a/my_test"); + EXPECT_EQ(flags::ShortProgramInvocationName(), "my_test"); + + absl::string_view not_null_terminated("absl/aaa/bbb"); + not_null_terminated = not_null_terminated.substr(1, 10); + + flags::SetProgramInvocationName(not_null_terminated); + + EXPECT_EQ(flags::ProgramInvocationName(), "bsl/aaa/bb"); + EXPECT_EQ(flags::ShortProgramInvocationName(), "bb"); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/internal/registry.h b/third_party/abseil_cpp/absl/flags/internal/registry.h new file mode 100644 index 000000000000..a8d9eb9cb0bb --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/registry.h @@ -0,0 +1,97 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_REGISTRY_H_ +#define ABSL_FLAGS_INTERNAL_REGISTRY_H_ + +#include <functional> + +#include "absl/base/config.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/strings/string_view.h" + +// -------------------------------------------------------------------- +// Global flags registry API. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// Executes specified visitor for each non-retired flag in the registry. While +// callback are executed, the registry is locked and can't be changed. +void ForEachFlag(std::function<void(CommandLineFlag&)> visitor); + +//----------------------------------------------------------------------------- + +bool RegisterCommandLineFlag(CommandLineFlag&); + +void FinalizeRegistry(); + +//----------------------------------------------------------------------------- +// Retired registrations: +// +// Retired flag registrations are treated specially. A 'retired' flag is +// provided only for compatibility with automated invocations that still +// name it. A 'retired' flag: +// - is not bound to a C++ FLAGS_ reference. +// - has a type and a value, but that value is intentionally inaccessible. +// - does not appear in --help messages. +// - is fully supported by _all_ flag parsing routines. +// - consumes args normally, and complains about type mismatches in its +// argument. +// - emits a complaint but does not die (e.g. LOG(ERROR)) if it is +// accessed by name through the flags API for parsing or otherwise. +// +// The registrations for a flag happen in an unspecified order as the +// initializers for the namespace-scope objects of a program are run. +// Any number of weak registrations for a flag can weakly define the flag. +// One non-weak registration will upgrade the flag from weak to non-weak. +// Further weak registrations of a non-weak flag are ignored. +// +// This mechanism is designed to support moving dead flags into a +// 'graveyard' library. An example migration: +// +// 0: Remove references to this FLAGS_flagname in the C++ codebase. +// 1: Register as 'retired' in old_lib. +// 2: Make old_lib depend on graveyard. +// 3: Add a redundant 'retired' registration to graveyard. +// 4: Remove the old_lib 'retired' registration. +// 5: Eventually delete the graveyard registration entirely. +// + +// Retire flag with name "name" and type indicated by ops. +void Retire(const char* name, FlagFastTypeId type_id, char* buf); + +constexpr size_t kRetiredFlagObjSize = 3 * sizeof(void*); +constexpr size_t kRetiredFlagObjAlignment = alignof(void*); + +// Registered a retired flag with name 'flag_name' and type 'T'. +template <typename T> +class RetiredFlag { + public: + void Retire(const char* flag_name) { + flags_internal::Retire(flag_name, base_internal::FastTypeId<T>(), buf_); + } + + private: + alignas(kRetiredFlagObjAlignment) char buf_[kRetiredFlagObjSize]; +}; + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_REGISTRY_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/usage.cc b/third_party/abseil_cpp/absl/flags/internal/usage.cc new file mode 100644 index 000000000000..f29d7c9b48f0 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/usage.cc @@ -0,0 +1,524 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/usage.h" + +#include <stdint.h> + +#include <functional> +#include <map> +#include <ostream> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/config.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/flag.h" +#include "absl/flags/internal/flag.h" +#include "absl/flags/internal/path_util.h" +#include "absl/flags/internal/private_handle_accessor.h" +#include "absl/flags/internal/program_name.h" +#include "absl/flags/internal/registry.h" +#include "absl/flags/usage_config.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" + +// Dummy global variables to prevent anyone else defining these. +bool FLAGS_help = false; +bool FLAGS_helpfull = false; +bool FLAGS_helpshort = false; +bool FLAGS_helppackage = false; +bool FLAGS_version = false; +bool FLAGS_only_check_args = false; +bool FLAGS_helpon = false; +bool FLAGS_helpmatch = false; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { +namespace { + +using PerFlagFilter = std::function<bool(const absl::CommandLineFlag&)>; + +// Maximum length size in a human readable format. +constexpr size_t kHrfMaxLineLength = 80; + +// This class is used to emit an XML element with `tag` and `text`. +// It adds opening and closing tags and escapes special characters in the text. +// For example: +// std::cout << XMLElement("title", "Milk & Cookies"); +// prints "<title>Milk & Cookies</title>" +class XMLElement { + public: + XMLElement(absl::string_view tag, absl::string_view txt) + : tag_(tag), txt_(txt) {} + + friend std::ostream& operator<<(std::ostream& out, + const XMLElement& xml_elem) { + out << "<" << xml_elem.tag_ << ">"; + + for (auto c : xml_elem.txt_) { + switch (c) { + case '"': + out << """; + break; + case '\'': + out << "'"; + break; + case '&': + out << "&"; + break; + case '<': + out << "<"; + break; + case '>': + out << ">"; + break; + default: + out << c; + break; + } + } + + return out << "</" << xml_elem.tag_ << ">"; + } + + private: + absl::string_view tag_; + absl::string_view txt_; +}; + +// -------------------------------------------------------------------- +// Helper class to pretty-print info about a flag. + +class FlagHelpPrettyPrinter { + public: + // Pretty printer holds on to the std::ostream& reference to direct an output + // to that stream. + FlagHelpPrettyPrinter(size_t max_line_len, size_t min_line_len, + size_t wrapped_line_indent, std::ostream& out) + : out_(out), + max_line_len_(max_line_len), + min_line_len_(min_line_len), + wrapped_line_indent_(wrapped_line_indent), + line_len_(0), + first_line_(true) {} + + void Write(absl::string_view str, bool wrap_line = false) { + // Empty string - do nothing. + if (str.empty()) return; + + std::vector<absl::string_view> tokens; + if (wrap_line) { + for (auto line : absl::StrSplit(str, absl::ByAnyChar("\n\r"))) { + if (!tokens.empty()) { + // Keep line separators in the input string. + tokens.push_back("\n"); + } + for (auto token : + absl::StrSplit(line, absl::ByAnyChar(" \t"), absl::SkipEmpty())) { + tokens.push_back(token); + } + } + } else { + tokens.push_back(str); + } + + for (auto token : tokens) { + bool new_line = (line_len_ == 0); + + // Respect line separators in the input string. + if (token == "\n") { + EndLine(); + continue; + } + + // Write the token, ending the string first if necessary/possible. + if (!new_line && + (line_len_ + static_cast<int>(token.size()) >= max_line_len_)) { + EndLine(); + new_line = true; + } + + if (new_line) { + StartLine(); + } else { + out_ << ' '; + ++line_len_; + } + + out_ << token; + line_len_ += token.size(); + } + } + + void StartLine() { + if (first_line_) { + line_len_ = min_line_len_; + first_line_ = false; + } else { + line_len_ = min_line_len_ + wrapped_line_indent_; + } + out_ << std::string(line_len_, ' '); + } + void EndLine() { + out_ << '\n'; + line_len_ = 0; + } + + private: + std::ostream& out_; + const size_t max_line_len_; + const size_t min_line_len_; + const size_t wrapped_line_indent_; + size_t line_len_; + bool first_line_; +}; + +void FlagHelpHumanReadable(const CommandLineFlag& flag, std::ostream& out) { + FlagHelpPrettyPrinter printer(kHrfMaxLineLength, 4, 2, out); + + // Flag name. + printer.Write(absl::StrCat("--", flag.Name())); + + // Flag help. + printer.Write(absl::StrCat("(", flag.Help(), ");"), /*wrap_line=*/true); + + // The listed default value will be the actual default from the flag + // definition in the originating source file, unless the value has + // subsequently been modified using SetCommandLineOption() with mode + // SET_FLAGS_DEFAULT. + std::string dflt_val = flag.DefaultValue(); + std::string curr_val = flag.CurrentValue(); + bool is_modified = curr_val != dflt_val; + + if (flag.IsOfType<std::string>()) { + dflt_val = absl::StrCat("\"", dflt_val, "\""); + } + printer.Write(absl::StrCat("default: ", dflt_val, ";")); + + if (is_modified) { + if (flag.IsOfType<std::string>()) { + curr_val = absl::StrCat("\"", curr_val, "\""); + } + printer.Write(absl::StrCat("currently: ", curr_val, ";")); + } + + printer.EndLine(); +} + +// Shows help for every filename which matches any of the filters +// If filters are empty, shows help for every file. +// If a flag's help message has been stripped (e.g. by adding '#define +// STRIP_FLAG_HELP 1' then this flag will not be displayed by '--help' +// and its variants. +void FlagsHelpImpl(std::ostream& out, PerFlagFilter filter_cb, + HelpFormat format, absl::string_view program_usage_message) { + if (format == HelpFormat::kHumanReadable) { + out << flags_internal::ShortProgramInvocationName() << ": " + << program_usage_message << "\n\n"; + } else { + // XML schema is not a part of our public API for now. + out << "<?xml version=\"1.0\"?>\n" + << "<!-- This output should be used with care. We do not report type " + "names for flags with user defined types -->\n" + << "<!-- Prefer flag only_check_args for validating flag inputs -->\n" + // The document. + << "<AllFlags>\n" + // The program name and usage. + << XMLElement("program", flags_internal::ShortProgramInvocationName()) + << '\n' + << XMLElement("usage", program_usage_message) << '\n'; + } + + // Map of package name to + // map of file name to + // vector of flags in the file. + // This map is used to output matching flags grouped by package and file + // name. + std::map<std::string, + std::map<std::string, std::vector<const absl::CommandLineFlag*>>> + matching_flags; + + flags_internal::ForEachFlag([&](absl::CommandLineFlag& flag) { + // Ignore retired flags. + if (flag.IsRetired()) return; + + // If the flag has been stripped, pretend that it doesn't exist. + if (flag.Help() == flags_internal::kStrippedFlagHelp) return; + + // Make sure flag satisfies the filter + if (!filter_cb(flag)) return; + + std::string flag_filename = flag.Filename(); + + matching_flags[std::string(flags_internal::Package(flag_filename))] + [flag_filename] + .push_back(&flag); + }); + + absl::string_view package_separator; // controls blank lines between packages + absl::string_view file_separator; // controls blank lines between files + for (const auto& package : matching_flags) { + if (format == HelpFormat::kHumanReadable) { + out << package_separator; + package_separator = "\n\n"; + } + + file_separator = ""; + for (const auto& flags_in_file : package.second) { + if (format == HelpFormat::kHumanReadable) { + out << file_separator << " Flags from " << flags_in_file.first + << ":\n"; + file_separator = "\n"; + } + + for (const auto* flag : flags_in_file.second) { + flags_internal::FlagHelp(out, *flag, format); + } + } + } + + if (format == HelpFormat::kHumanReadable) { + FlagHelpPrettyPrinter printer(kHrfMaxLineLength, 0, 0, out); + + if (filter_cb && matching_flags.empty()) { + printer.Write("No flags matched.\n", true); + } + printer.EndLine(); + printer.Write( + "Try --helpfull to get a list of all flags or --help=substring " + "shows help for flags which include specified substring in either " + "in the name, or description or path.\n", + true); + } else { + // The end of the document. + out << "</AllFlags>\n"; + } +} + +void FlagsHelpImpl(std::ostream& out, + flags_internal::FlagKindFilter filename_filter_cb, + HelpFormat format, absl::string_view program_usage_message) { + FlagsHelpImpl( + out, + [&](const absl::CommandLineFlag& flag) { + return filename_filter_cb && filename_filter_cb(flag.Filename()); + }, + format, program_usage_message); +} + +} // namespace + +// -------------------------------------------------------------------- +// Produces the help message describing specific flag. +void FlagHelp(std::ostream& out, const CommandLineFlag& flag, + HelpFormat format) { + if (format == HelpFormat::kHumanReadable) + flags_internal::FlagHelpHumanReadable(flag, out); +} + +// -------------------------------------------------------------------- +// Produces the help messages for all flags matching the filename filter. +// If filter is empty produces help messages for all flags. +void FlagsHelp(std::ostream& out, absl::string_view filter, HelpFormat format, + absl::string_view program_usage_message) { + flags_internal::FlagKindFilter filter_cb = [&](absl::string_view filename) { + return filter.empty() || filename.find(filter) != absl::string_view::npos; + }; + flags_internal::FlagsHelpImpl(out, filter_cb, format, program_usage_message); +} + +// -------------------------------------------------------------------- +// Checks all the 'usage' command line flags to see if any have been set. +// If so, handles them appropriately. +int HandleUsageFlags(std::ostream& out, + absl::string_view program_usage_message) { + switch (GetFlagsHelpMode()) { + case HelpMode::kNone: + break; + case HelpMode::kImportant: + flags_internal::FlagsHelpImpl( + out, flags_internal::GetUsageConfig().contains_help_flags, + GetFlagsHelpFormat(), program_usage_message); + return 1; + + case HelpMode::kShort: + flags_internal::FlagsHelpImpl( + out, flags_internal::GetUsageConfig().contains_helpshort_flags, + GetFlagsHelpFormat(), program_usage_message); + return 1; + + case HelpMode::kFull: + flags_internal::FlagsHelp(out, "", GetFlagsHelpFormat(), + program_usage_message); + return 1; + + case HelpMode::kPackage: + flags_internal::FlagsHelpImpl( + out, flags_internal::GetUsageConfig().contains_helppackage_flags, + GetFlagsHelpFormat(), program_usage_message); + + return 1; + + case HelpMode::kMatch: { + std::string substr = GetFlagsHelpMatchSubstr(); + if (substr.empty()) { + // show all options + flags_internal::FlagsHelp(out, substr, GetFlagsHelpFormat(), + program_usage_message); + } else { + auto filter_cb = [&substr](const absl::CommandLineFlag& flag) { + if (absl::StrContains(flag.Name(), substr)) return true; + if (absl::StrContains(flag.Filename(), substr)) return true; + if (absl::StrContains(flag.Help(), substr)) return true; + + return false; + }; + flags_internal::FlagsHelpImpl( + out, filter_cb, HelpFormat::kHumanReadable, program_usage_message); + } + + return 1; + } + case HelpMode::kVersion: + if (flags_internal::GetUsageConfig().version_string) + out << flags_internal::GetUsageConfig().version_string(); + // Unlike help, we may be asking for version in a script, so return 0 + return 0; + + case HelpMode::kOnlyCheckArgs: + return 0; + } + + return -1; +} + +// -------------------------------------------------------------------- +// Globals representing usage reporting flags + +namespace { + +ABSL_CONST_INIT absl::Mutex help_attributes_guard(absl::kConstInit); +ABSL_CONST_INIT std::string* match_substr + ABSL_GUARDED_BY(help_attributes_guard) = nullptr; +ABSL_CONST_INIT HelpMode help_mode ABSL_GUARDED_BY(help_attributes_guard) = + HelpMode::kNone; +ABSL_CONST_INIT HelpFormat help_format ABSL_GUARDED_BY(help_attributes_guard) = + HelpFormat::kHumanReadable; + +} // namespace + +std::string GetFlagsHelpMatchSubstr() { + absl::MutexLock l(&help_attributes_guard); + if (match_substr == nullptr) return ""; + return *match_substr; +} + +void SetFlagsHelpMatchSubstr(absl::string_view substr) { + absl::MutexLock l(&help_attributes_guard); + if (match_substr == nullptr) match_substr = new std::string; + match_substr->assign(substr.data(), substr.size()); +} + +HelpMode GetFlagsHelpMode() { + absl::MutexLock l(&help_attributes_guard); + // Refer to dummy variales to prevent linker dropping them + if (FLAGS_help || FLAGS_helpfull || FLAGS_helpshort || FLAGS_helppackage || + FLAGS_version || FLAGS_only_check_args || FLAGS_helpon || + FLAGS_helpmatch) { + help_mode = HelpMode::kNone; + } + return help_mode; +} + +void SetFlagsHelpMode(HelpMode mode) { + absl::MutexLock l(&help_attributes_guard); + help_mode = mode; +} + +HelpFormat GetFlagsHelpFormat() { + absl::MutexLock l(&help_attributes_guard); + return help_format; +} + +void SetFlagsHelpFormat(HelpFormat format) { + absl::MutexLock l(&help_attributes_guard); + help_format = format; +} + +// Deduces usage flags from the input argument in a form --name=value or +// --name. argument is already split into name and value before we call this +// function. +bool DeduceUsageFlags(absl::string_view name, absl::string_view value) { + if (absl::ConsumePrefix(&name, "help")) { + if (name == "") { + if (value.empty()) { + SetFlagsHelpMode(HelpMode::kImportant); + } else { + SetFlagsHelpMode(HelpMode::kMatch); + SetFlagsHelpMatchSubstr(value); + } + return true; + } + + if (name == "match") { + SetFlagsHelpMode(HelpMode::kMatch); + SetFlagsHelpMatchSubstr(value); + return true; + } + + if (name == "on") { + SetFlagsHelpMode(HelpMode::kMatch); + SetFlagsHelpMatchSubstr(absl::StrCat("/", value, ".")); + return true; + } + + if (name == "full") { + SetFlagsHelpMode(HelpMode::kFull); + return true; + } + + if (name == "short") { + SetFlagsHelpMode(HelpMode::kShort); + return true; + } + + if (name == "package") { + SetFlagsHelpMode(HelpMode::kPackage); + return true; + } + + return false; + } + + if (name == "version") { + SetFlagsHelpMode(HelpMode::kVersion); + return true; + } + + if (name == "only_check_args") { + SetFlagsHelpMode(HelpMode::kOnlyCheckArgs); + return true; + } + + return false; +} + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/internal/usage.h b/third_party/abseil_cpp/absl/flags/internal/usage.h new file mode 100644 index 000000000000..c0bcac57628a --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/usage.h @@ -0,0 +1,104 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_INTERNAL_USAGE_H_ +#define ABSL_FLAGS_INTERNAL_USAGE_H_ + +#include <iosfwd> +#include <string> + +#include "absl/base/config.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/declare.h" +#include "absl/strings/string_view.h" + +// -------------------------------------------------------------------- +// Usage reporting interfaces + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// The format to report the help messages in. +enum class HelpFormat { + kHumanReadable, +}; + +// Streams the help message describing `flag` to `out`. +// The default value for `flag` is included in the output. +void FlagHelp(std::ostream& out, const CommandLineFlag& flag, + HelpFormat format = HelpFormat::kHumanReadable); + +// Produces the help messages for all flags matching the filter. A flag matches +// the filter if it is defined in a file with a filename which includes +// filter string as a substring. You can use '/' and '.' to restrict the +// matching to a specific file names. For example: +// FlagsHelp(out, "/path/to/file."); +// restricts help to only flags which resides in files named like: +// .../path/to/file.<ext> +// for any extension 'ext'. If the filter is empty this function produces help +// messages for all flags. +void FlagsHelp(std::ostream& out, absl::string_view filter, + HelpFormat format, absl::string_view program_usage_message); + +// -------------------------------------------------------------------- + +// If any of the 'usage' related command line flags (listed on the bottom of +// this file) has been set this routine produces corresponding help message in +// the specified output stream and returns: +// 0 - if "version" or "only_check_flags" flags were set and handled. +// 1 - if some other 'usage' related flag was set and handled. +// -1 - if no usage flags were set on a commmand line. +// Non negative return values are expected to be used as an exit code for a +// binary. +int HandleUsageFlags(std::ostream& out, + absl::string_view program_usage_message); + +// -------------------------------------------------------------------- +// Globals representing usage reporting flags + +enum class HelpMode { + kNone, + kImportant, + kShort, + kFull, + kPackage, + kMatch, + kVersion, + kOnlyCheckArgs +}; + +// Returns substring to filter help output (--help=substr argument) +std::string GetFlagsHelpMatchSubstr(); +// Returns the requested help mode. +HelpMode GetFlagsHelpMode(); +// Returns the requested help format. +HelpFormat GetFlagsHelpFormat(); + +// These are corresponding setters to the attributes above. +void SetFlagsHelpMatchSubstr(absl::string_view); +void SetFlagsHelpMode(HelpMode); +void SetFlagsHelpFormat(HelpFormat); + +// Deduces usage flags from the input argument in a form --name=value or +// --name. argument is already split into name and value before we call this +// function. +bool DeduceUsageFlags(absl::string_view name, absl::string_view value); + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_INTERNAL_USAGE_H_ diff --git a/third_party/abseil_cpp/absl/flags/internal/usage_test.cc b/third_party/abseil_cpp/absl/flags/internal/usage_test.cc new file mode 100644 index 000000000000..b5c2487da577 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/internal/usage_test.cc @@ -0,0 +1,493 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/internal/usage.h" + +#include <stdint.h> + +#include <sstream> +#include <string> + +#include "gtest/gtest.h" +#include "absl/flags/flag.h" +#include "absl/flags/internal/parse.h" +#include "absl/flags/internal/path_util.h" +#include "absl/flags/internal/program_name.h" +#include "absl/flags/reflection.h" +#include "absl/flags/usage.h" +#include "absl/flags/usage_config.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" + +ABSL_FLAG(int, usage_reporting_test_flag_01, 101, + "usage_reporting_test_flag_01 help message"); +ABSL_FLAG(bool, usage_reporting_test_flag_02, false, + "usage_reporting_test_flag_02 help message"); +ABSL_FLAG(double, usage_reporting_test_flag_03, 1.03, + "usage_reporting_test_flag_03 help message"); +ABSL_FLAG(int64_t, usage_reporting_test_flag_04, 1000000000000004L, + "usage_reporting_test_flag_04 help message"); + +static const char kTestUsageMessage[] = "Custom usage message"; + +struct UDT { + UDT() = default; + UDT(const UDT&) = default; +}; +bool AbslParseFlag(absl::string_view, UDT*, std::string*) { return true; } +std::string AbslUnparseFlag(const UDT&) { return "UDT{}"; } + +ABSL_FLAG(UDT, usage_reporting_test_flag_05, {}, + "usage_reporting_test_flag_05 help message"); + +ABSL_FLAG( + std::string, usage_reporting_test_flag_06, {}, + "usage_reporting_test_flag_06 help message.\n" + "\n" + "Some more help.\n" + "Even more long long long long long long long long long long long long " + "help message."); + +namespace { + +namespace flags = absl::flags_internal; + +static std::string NormalizeFileName(absl::string_view fname) { +#ifdef _WIN32 + std::string normalized(fname); + std::replace(normalized.begin(), normalized.end(), '\\', '/'); + fname = normalized; +#endif + + auto absl_pos = fname.rfind("absl/"); + if (absl_pos != absl::string_view::npos) { + fname = fname.substr(absl_pos); + } + return std::string(fname); +} + +class UsageReportingTest : public testing::Test { + protected: + UsageReportingTest() { + // Install default config for the use on this unit test. + // Binary may install a custom config before tests are run. + absl::FlagsUsageConfig default_config; + default_config.normalize_filename = &NormalizeFileName; + absl::SetFlagsUsageConfig(default_config); + } + ~UsageReportingTest() override { + flags::SetFlagsHelpMode(flags::HelpMode::kNone); + flags::SetFlagsHelpMatchSubstr(""); + flags::SetFlagsHelpFormat(flags::HelpFormat::kHumanReadable); + } + + private: + absl::FlagSaver flag_saver_; +}; + +// -------------------------------------------------------------------- + +using UsageReportingDeathTest = UsageReportingTest; + +TEST_F(UsageReportingDeathTest, TestSetProgramUsageMessage) { + EXPECT_EQ(absl::ProgramUsageMessage(), kTestUsageMessage); + +#ifndef _WIN32 + // TODO(rogeeff): figure out why this does not work on Windows. + EXPECT_DEATH_IF_SUPPORTED( + absl::SetProgramUsageMessage("custom usage message"), + ".*SetProgramUsageMessage\\(\\) called twice.*"); +#endif +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestFlagHelpHRF_on_flag_01) { + const auto* flag = absl::FindCommandLineFlag("usage_reporting_test_flag_01"); + std::stringstream test_buf; + + flags::FlagHelp(test_buf, *flag, flags::HelpFormat::kHumanReadable); + EXPECT_EQ( + test_buf.str(), + R"( --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; +)"); +} + +TEST_F(UsageReportingTest, TestFlagHelpHRF_on_flag_02) { + const auto* flag = absl::FindCommandLineFlag("usage_reporting_test_flag_02"); + std::stringstream test_buf; + + flags::FlagHelp(test_buf, *flag, flags::HelpFormat::kHumanReadable); + EXPECT_EQ( + test_buf.str(), + R"( --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; +)"); +} + +TEST_F(UsageReportingTest, TestFlagHelpHRF_on_flag_03) { + const auto* flag = absl::FindCommandLineFlag("usage_reporting_test_flag_03"); + std::stringstream test_buf; + + flags::FlagHelp(test_buf, *flag, flags::HelpFormat::kHumanReadable); + EXPECT_EQ( + test_buf.str(), + R"( --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; +)"); +} + +TEST_F(UsageReportingTest, TestFlagHelpHRF_on_flag_04) { + const auto* flag = absl::FindCommandLineFlag("usage_reporting_test_flag_04"); + std::stringstream test_buf; + + flags::FlagHelp(test_buf, *flag, flags::HelpFormat::kHumanReadable); + EXPECT_EQ( + test_buf.str(), + R"( --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; +)"); +} + +TEST_F(UsageReportingTest, TestFlagHelpHRF_on_flag_05) { + const auto* flag = absl::FindCommandLineFlag("usage_reporting_test_flag_05"); + std::stringstream test_buf; + + flags::FlagHelp(test_buf, *flag, flags::HelpFormat::kHumanReadable); + EXPECT_EQ( + test_buf.str(), + R"( --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; +)"); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestFlagsHelpHRF) { + std::string usage_test_flags_out = + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; + --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; + --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; + --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; + --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"; + + std::stringstream test_buf_01; + flags::FlagsHelp(test_buf_01, "usage_test.cc", + flags::HelpFormat::kHumanReadable, kTestUsageMessage); + EXPECT_EQ(test_buf_01.str(), usage_test_flags_out); + + std::stringstream test_buf_02; + flags::FlagsHelp(test_buf_02, "flags/internal/usage_test.cc", + flags::HelpFormat::kHumanReadable, kTestUsageMessage); + EXPECT_EQ(test_buf_02.str(), usage_test_flags_out); + + std::stringstream test_buf_03; + flags::FlagsHelp(test_buf_03, "usage_test", flags::HelpFormat::kHumanReadable, + kTestUsageMessage); + EXPECT_EQ(test_buf_03.str(), usage_test_flags_out); + + std::stringstream test_buf_04; + flags::FlagsHelp(test_buf_04, "flags/invalid_file_name.cc", + flags::HelpFormat::kHumanReadable, kTestUsageMessage); + EXPECT_EQ(test_buf_04.str(), + R"(usage_test: Custom usage message + +No flags matched. + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); + + std::stringstream test_buf_05; + flags::FlagsHelp(test_buf_05, "", flags::HelpFormat::kHumanReadable, + kTestUsageMessage); + std::string test_out = test_buf_05.str(); + absl::string_view test_out_str(test_out); + EXPECT_TRUE( + absl::StartsWith(test_out_str, "usage_test: Custom usage message")); + EXPECT_TRUE(absl::StrContains( + test_out_str, "Flags from absl/flags/internal/usage_test.cc:")); + EXPECT_TRUE( + absl::StrContains(test_out_str, "-usage_reporting_test_flag_01 ")); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestNoUsageFlags) { + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), -1); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_helpshort) { + flags::SetFlagsHelpMode(flags::HelpMode::kShort); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 1); + EXPECT_EQ(test_buf.str(), + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; + --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; + --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; + --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; + --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_help_simple) { + flags::SetFlagsHelpMode(flags::HelpMode::kImportant); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 1); + EXPECT_EQ(test_buf.str(), + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; + --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; + --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; + --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; + --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_help_one_flag) { + flags::SetFlagsHelpMode(flags::HelpMode::kMatch); + flags::SetFlagsHelpMatchSubstr("usage_reporting_test_flag_06"); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 1); + EXPECT_EQ(test_buf.str(), + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_help_multiple_flag) { + flags::SetFlagsHelpMode(flags::HelpMode::kMatch); + flags::SetFlagsHelpMatchSubstr("test_flag"); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 1); + EXPECT_EQ(test_buf.str(), + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; + --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; + --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; + --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; + --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_helppackage) { + flags::SetFlagsHelpMode(flags::HelpMode::kPackage); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 1); + EXPECT_EQ(test_buf.str(), + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; + --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; + --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; + --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; + --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_version) { + flags::SetFlagsHelpMode(flags::HelpMode::kVersion); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 0); +#ifndef NDEBUG + EXPECT_EQ(test_buf.str(), "usage_test\nDebug build (NDEBUG not #defined)\n"); +#else + EXPECT_EQ(test_buf.str(), "usage_test\n"); +#endif +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_only_check_args) { + flags::SetFlagsHelpMode(flags::HelpMode::kOnlyCheckArgs); + + std::stringstream test_buf; + EXPECT_EQ(flags::HandleUsageFlags(test_buf, kTestUsageMessage), 0); + EXPECT_EQ(test_buf.str(), ""); +} + +// -------------------------------------------------------------------- + +TEST_F(UsageReportingTest, TestUsageFlag_helpon) { + flags::SetFlagsHelpMode(flags::HelpMode::kMatch); + flags::SetFlagsHelpMatchSubstr("/bla-bla."); + + std::stringstream test_buf_01; + EXPECT_EQ(flags::HandleUsageFlags(test_buf_01, kTestUsageMessage), 1); + EXPECT_EQ(test_buf_01.str(), + R"(usage_test: Custom usage message + +No flags matched. + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); + + flags::SetFlagsHelpMatchSubstr("/usage_test."); + + std::stringstream test_buf_02; + EXPECT_EQ(flags::HandleUsageFlags(test_buf_02, kTestUsageMessage), 1); + EXPECT_EQ(test_buf_02.str(), + R"(usage_test: Custom usage message + + Flags from absl/flags/internal/usage_test.cc: + --usage_reporting_test_flag_01 (usage_reporting_test_flag_01 help message); + default: 101; + --usage_reporting_test_flag_02 (usage_reporting_test_flag_02 help message); + default: false; + --usage_reporting_test_flag_03 (usage_reporting_test_flag_03 help message); + default: 1.03; + --usage_reporting_test_flag_04 (usage_reporting_test_flag_04 help message); + default: 1000000000000004; + --usage_reporting_test_flag_05 (usage_reporting_test_flag_05 help message); + default: UDT{}; + --usage_reporting_test_flag_06 (usage_reporting_test_flag_06 help message. + + Some more help. + Even more long long long long long long long long long long long long help + message.); default: ""; + +Try --helpfull to get a list of all flags or --help=substring shows help for +flags which include specified substring in either in the name, or description or +path. +)"); +} + +// -------------------------------------------------------------------- + +} // namespace + +int main(int argc, char* argv[]) { + (void)absl::GetFlag(FLAGS_undefok); // Force linking of parse.cc + flags::SetProgramInvocationName("usage_test"); + absl::SetProgramUsageMessage(kTestUsageMessage); + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/third_party/abseil_cpp/absl/flags/marshalling.cc b/third_party/abseil_cpp/absl/flags/marshalling.cc new file mode 100644 index 000000000000..81f9cebd6f41 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/marshalling.cc @@ -0,0 +1,241 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/marshalling.h" + +#include <stddef.h> + +#include <cmath> +#include <limits> +#include <string> +#include <type_traits> +#include <vector> + +#include "absl/base/config.h" +#include "absl/base/log_severity.h" +#include "absl/base/macros.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// -------------------------------------------------------------------- +// AbslParseFlag specializations for boolean type. + +bool AbslParseFlag(absl::string_view text, bool* dst, std::string*) { + const char* kTrue[] = {"1", "t", "true", "y", "yes"}; + const char* kFalse[] = {"0", "f", "false", "n", "no"}; + static_assert(sizeof(kTrue) == sizeof(kFalse), "true_false_equal"); + + text = absl::StripAsciiWhitespace(text); + + for (size_t i = 0; i < ABSL_ARRAYSIZE(kTrue); ++i) { + if (absl::EqualsIgnoreCase(text, kTrue[i])) { + *dst = true; + return true; + } else if (absl::EqualsIgnoreCase(text, kFalse[i])) { + *dst = false; + return true; + } + } + return false; // didn't match a legal input +} + +// -------------------------------------------------------------------- +// AbslParseFlag for integral types. + +// Return the base to use for parsing text as an integer. Leading 0x +// puts us in base 16. But leading 0 does not put us in base 8. It +// caused too many bugs when we had that behavior. +static int NumericBase(absl::string_view text) { + const bool hex = (text.size() >= 2 && text[0] == '0' && + (text[1] == 'x' || text[1] == 'X')); + return hex ? 16 : 10; +} + +template <typename IntType> +inline bool ParseFlagImpl(absl::string_view text, IntType& dst) { + text = absl::StripAsciiWhitespace(text); + + return absl::numbers_internal::safe_strtoi_base(text, &dst, + NumericBase(text)); +} + +bool AbslParseFlag(absl::string_view text, short* dst, std::string*) { + int val; + if (!ParseFlagImpl(text, val)) return false; + if (static_cast<short>(val) != val) // worked, but number out of range + return false; + *dst = static_cast<short>(val); + return true; +} + +bool AbslParseFlag(absl::string_view text, unsigned short* dst, std::string*) { + unsigned int val; + if (!ParseFlagImpl(text, val)) return false; + if (static_cast<unsigned short>(val) != + val) // worked, but number out of range + return false; + *dst = static_cast<unsigned short>(val); + return true; +} + +bool AbslParseFlag(absl::string_view text, int* dst, std::string*) { + return ParseFlagImpl(text, *dst); +} + +bool AbslParseFlag(absl::string_view text, unsigned int* dst, std::string*) { + return ParseFlagImpl(text, *dst); +} + +bool AbslParseFlag(absl::string_view text, long* dst, std::string*) { + return ParseFlagImpl(text, *dst); +} + +bool AbslParseFlag(absl::string_view text, unsigned long* dst, std::string*) { + return ParseFlagImpl(text, *dst); +} + +bool AbslParseFlag(absl::string_view text, long long* dst, std::string*) { + return ParseFlagImpl(text, *dst); +} + +bool AbslParseFlag(absl::string_view text, unsigned long long* dst, + std::string*) { + return ParseFlagImpl(text, *dst); +} + +// -------------------------------------------------------------------- +// AbslParseFlag for floating point types. + +bool AbslParseFlag(absl::string_view text, float* dst, std::string*) { + return absl::SimpleAtof(text, dst); +} + +bool AbslParseFlag(absl::string_view text, double* dst, std::string*) { + return absl::SimpleAtod(text, dst); +} + +// -------------------------------------------------------------------- +// AbslParseFlag for strings. + +bool AbslParseFlag(absl::string_view text, std::string* dst, std::string*) { + dst->assign(text.data(), text.size()); + return true; +} + +// -------------------------------------------------------------------- +// AbslParseFlag for vector of strings. + +bool AbslParseFlag(absl::string_view text, std::vector<std::string>* dst, + std::string*) { + // An empty flag value corresponds to an empty vector, not a vector + // with a single, empty std::string. + if (text.empty()) { + dst->clear(); + return true; + } + *dst = absl::StrSplit(text, ',', absl::AllowEmpty()); + return true; +} + +// -------------------------------------------------------------------- +// AbslUnparseFlag specializations for various builtin flag types. + +std::string Unparse(bool v) { return v ? "true" : "false"; } +std::string Unparse(short v) { return absl::StrCat(v); } +std::string Unparse(unsigned short v) { return absl::StrCat(v); } +std::string Unparse(int v) { return absl::StrCat(v); } +std::string Unparse(unsigned int v) { return absl::StrCat(v); } +std::string Unparse(long v) { return absl::StrCat(v); } +std::string Unparse(unsigned long v) { return absl::StrCat(v); } +std::string Unparse(long long v) { return absl::StrCat(v); } +std::string Unparse(unsigned long long v) { return absl::StrCat(v); } +template <typename T> +std::string UnparseFloatingPointVal(T v) { + // digits10 is guaranteed to roundtrip correctly in string -> value -> string + // conversions, but may not be enough to represent all the values correctly. + std::string digit10_str = + absl::StrFormat("%.*g", std::numeric_limits<T>::digits10, v); + if (std::isnan(v) || std::isinf(v)) return digit10_str; + + T roundtrip_val = 0; + std::string err; + if (absl::ParseFlag(digit10_str, &roundtrip_val, &err) && + roundtrip_val == v) { + return digit10_str; + } + + // max_digits10 is the number of base-10 digits that are necessary to uniquely + // represent all distinct values. + return absl::StrFormat("%.*g", std::numeric_limits<T>::max_digits10, v); +} +std::string Unparse(float v) { return UnparseFloatingPointVal(v); } +std::string Unparse(double v) { return UnparseFloatingPointVal(v); } +std::string AbslUnparseFlag(absl::string_view v) { return std::string(v); } +std::string AbslUnparseFlag(const std::vector<std::string>& v) { + return absl::StrJoin(v, ","); +} + +} // namespace flags_internal + +bool AbslParseFlag(absl::string_view text, absl::LogSeverity* dst, + std::string* err) { + text = absl::StripAsciiWhitespace(text); + if (text.empty()) { + *err = "no value provided"; + return false; + } + if (text.front() == 'k' || text.front() == 'K') text.remove_prefix(1); + if (absl::EqualsIgnoreCase(text, "info")) { + *dst = absl::LogSeverity::kInfo; + return true; + } + if (absl::EqualsIgnoreCase(text, "warning")) { + *dst = absl::LogSeverity::kWarning; + return true; + } + if (absl::EqualsIgnoreCase(text, "error")) { + *dst = absl::LogSeverity::kError; + return true; + } + if (absl::EqualsIgnoreCase(text, "fatal")) { + *dst = absl::LogSeverity::kFatal; + return true; + } + std::underlying_type<absl::LogSeverity>::type numeric_value; + if (absl::ParseFlag(text, &numeric_value, err)) { + *dst = static_cast<absl::LogSeverity>(numeric_value); + return true; + } + *err = "only integers and absl::LogSeverity enumerators are accepted"; + return false; +} + +std::string AbslUnparseFlag(absl::LogSeverity v) { + if (v == absl::NormalizeLogSeverity(v)) return absl::LogSeverityName(v); + return absl::UnparseFlag(static_cast<int>(v)); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/marshalling.h b/third_party/abseil_cpp/absl/flags/marshalling.h new file mode 100644 index 000000000000..0b5033547e97 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/marshalling.h @@ -0,0 +1,264 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: marshalling.h +// ----------------------------------------------------------------------------- +// +// This header file defines the API for extending Abseil flag support to +// custom types, and defines the set of overloads for fundamental types. +// +// Out of the box, the Abseil flags library supports the following types: +// +// * `bool` +// * `int16_t` +// * `uint16_t` +// * `int32_t` +// * `uint32_t` +// * `int64_t` +// * `uint64_t` +// * `float` +// * `double` +// * `std::string` +// * `std::vector<std::string>` +// * `absl::LogSeverity` (provided natively for layering reasons) +// +// Note that support for integral types is implemented using overloads for +// variable-width fundamental types (`short`, `int`, `long`, etc.). However, +// you should prefer the fixed-width integral types (`int32_t`, `uint64_t`, +// etc.) we've noted above within flag definitions. +// +// In addition, several Abseil libraries provide their own custom support for +// Abseil flags. Documentation for these formats is provided in the type's +// `AbslParseFlag()` definition. +// +// The Abseil time library provides the following support for civil time values: +// +// * `absl::CivilSecond` +// * `absl::CivilMinute` +// * `absl::CivilHour` +// * `absl::CivilDay` +// * `absl::CivilMonth` +// * `absl::CivilYear` +// +// and also provides support for the following absolute time values: +// +// * `absl::Duration` +// * `absl::Time` +// +// Additional support for Abseil types will be noted here as it is added. +// +// You can also provide your own custom flags by adding overloads for +// `AbslParseFlag()` and `AbslUnparseFlag()` to your type definitions. (See +// below.) +// +// ----------------------------------------------------------------------------- +// Adding Type Support for Abseil Flags +// ----------------------------------------------------------------------------- +// +// To add support for your user-defined type, add overloads of `AbslParseFlag()` +// and `AbslUnparseFlag()` as free (non-member) functions to your type. If `T` +// is a class type, these functions can be friend function definitions. These +// overloads must be added to the same namespace where the type is defined, so +// that they can be discovered by Argument-Dependent Lookup (ADL). +// +// Example: +// +// namespace foo { +// +// enum OutputMode { kPlainText, kHtml }; +// +// // AbslParseFlag converts from a string to OutputMode. +// // Must be in same namespace as OutputMode. +// +// // Parses an OutputMode from the command line flag value `text. Returns +// // `true` and sets `*mode` on success; returns `false` and sets `*error` +// // on failure. +// bool AbslParseFlag(absl::string_view text, +// OutputMode* mode, +// std::string* error) { +// if (text == "plaintext") { +// *mode = kPlainText; +// return true; +// } +// if (text == "html") { +// *mode = kHtml; +// return true; +// } +// *error = "unknown value for enumeration"; +// return false; +// } +// +// // AbslUnparseFlag converts from an OutputMode to a string. +// // Must be in same namespace as OutputMode. +// +// // Returns a textual flag value corresponding to the OutputMode `mode`. +// std::string AbslUnparseFlag(OutputMode mode) { +// switch (mode) { +// case kPlainText: return "plaintext"; +// case kHtml: return "html"; +// } +// return absl::StrCat(mode); +// } +// +// Notice that neither `AbslParseFlag()` nor `AbslUnparseFlag()` are class +// members, but free functions. `AbslParseFlag/AbslUnparseFlag()` overloads +// for a type should only be declared in the same file and namespace as said +// type. The proper `AbslParseFlag/AbslUnparseFlag()` implementations for a +// given type will be discovered via Argument-Dependent Lookup (ADL). +// +// `AbslParseFlag()` may need, in turn, to parse simpler constituent types +// using `absl::ParseFlag()`. For example, a custom struct `MyFlagType` +// consisting of a `std::pair<int, std::string>` would add an `AbslParseFlag()` +// overload for its `MyFlagType` like so: +// +// Example: +// +// namespace my_flag_type { +// +// struct MyFlagType { +// std::pair<int, std::string> my_flag_data; +// }; +// +// bool AbslParseFlag(absl::string_view text, MyFlagType* flag, +// std::string* err); +// +// std::string AbslUnparseFlag(const MyFlagType&); +// +// // Within the implementation, `AbslParseFlag()` will, in turn invoke +// // `absl::ParseFlag()` on its constituent `int` and `std::string` types +// // (which have built-in Abseil flag support. +// +// bool AbslParseFlag(absl::string_view text, MyFlagType* flag, +// std::string* err) { +// std::pair<absl::string_view, absl::string_view> tokens = +// absl::StrSplit(text, ','); +// if (!absl::ParseFlag(tokens.first, &flag->my_flag_data.first, err)) +// return false; +// if (!absl::ParseFlag(tokens.second, &flag->my_flag_data.second, err)) +// return false; +// return true; +// } +// +// // Similarly, for unparsing, we can simply invoke `absl::UnparseFlag()` on +// // the constituent types. +// std::string AbslUnparseFlag(const MyFlagType& flag) { +// return absl::StrCat(absl::UnparseFlag(flag.my_flag_data.first), +// ",", +// absl::UnparseFlag(flag.my_flag_data.second)); +// } +#ifndef ABSL_FLAGS_MARSHALLING_H_ +#define ABSL_FLAGS_MARSHALLING_H_ + +#include <string> +#include <vector> + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// Overloads of `AbslParseFlag()` and `AbslUnparseFlag()` for fundamental types. +bool AbslParseFlag(absl::string_view, bool*, std::string*); +bool AbslParseFlag(absl::string_view, short*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, unsigned short*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, int*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, unsigned int*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, long*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, unsigned long*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, long long*, std::string*); // NOLINT +bool AbslParseFlag(absl::string_view, unsigned long long*, // NOLINT + std::string*); +bool AbslParseFlag(absl::string_view, float*, std::string*); +bool AbslParseFlag(absl::string_view, double*, std::string*); +bool AbslParseFlag(absl::string_view, std::string*, std::string*); +bool AbslParseFlag(absl::string_view, std::vector<std::string>*, std::string*); + +template <typename T> +bool InvokeParseFlag(absl::string_view input, T* dst, std::string* err) { + // Comment on next line provides a good compiler error message if T + // does not have AbslParseFlag(absl::string_view, T*, std::string*). + return AbslParseFlag(input, dst, err); // Is T missing AbslParseFlag? +} + +// Strings and std:: containers do not have the same overload resolution +// considerations as fundamental types. Naming these 'AbslUnparseFlag' means we +// can avoid the need for additional specializations of Unparse (below). +std::string AbslUnparseFlag(absl::string_view v); +std::string AbslUnparseFlag(const std::vector<std::string>&); + +template <typename T> +std::string Unparse(const T& v) { + // Comment on next line provides a good compiler error message if T does not + // have UnparseFlag. + return AbslUnparseFlag(v); // Is T missing AbslUnparseFlag? +} + +// Overloads for builtin types. +std::string Unparse(bool v); +std::string Unparse(short v); // NOLINT +std::string Unparse(unsigned short v); // NOLINT +std::string Unparse(int v); // NOLINT +std::string Unparse(unsigned int v); // NOLINT +std::string Unparse(long v); // NOLINT +std::string Unparse(unsigned long v); // NOLINT +std::string Unparse(long long v); // NOLINT +std::string Unparse(unsigned long long v); // NOLINT +std::string Unparse(float v); +std::string Unparse(double v); + +} // namespace flags_internal + +// ParseFlag() +// +// Parses a string value into a flag value of type `T`. Do not add overloads of +// this function for your type directly; instead, add an `AbslParseFlag()` +// free function as documented above. +// +// Some implementations of `AbslParseFlag()` for types which consist of other, +// constituent types which already have Abseil flag support, may need to call +// `absl::ParseFlag()` on those consituent string values. (See above.) +template <typename T> +inline bool ParseFlag(absl::string_view input, T* dst, std::string* error) { + return flags_internal::InvokeParseFlag(input, dst, error); +} + +// UnparseFlag() +// +// Unparses a flag value of type `T` into a string value. Do not add overloads +// of this function for your type directly; instead, add an `AbslUnparseFlag()` +// free function as documented above. +// +// Some implementations of `AbslUnparseFlag()` for types which consist of other, +// constituent types which already have Abseil flag support, may want to call +// `absl::UnparseFlag()` on those constituent types. (See above.) +template <typename T> +inline std::string UnparseFlag(const T& v) { + return flags_internal::Unparse(v); +} + +// Overloads for `absl::LogSeverity` can't (easily) appear alongside that type's +// definition because it is layered below flags. See proper documentation in +// base/log_severity.h. +enum class LogSeverity : int; +bool AbslParseFlag(absl::string_view, absl::LogSeverity*, std::string*); +std::string AbslUnparseFlag(absl::LogSeverity); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_MARSHALLING_H_ diff --git a/third_party/abseil_cpp/absl/flags/marshalling_test.cc b/third_party/abseil_cpp/absl/flags/marshalling_test.cc new file mode 100644 index 000000000000..4a64ce11a165 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/marshalling_test.cc @@ -0,0 +1,904 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/marshalling.h" + +#include <stdint.h> + +#include <cmath> +#include <limits> +#include <string> +#include <vector> + +#include "gtest/gtest.h" + +namespace { + +TEST(MarshallingTest, TestBoolParsing) { + std::string err; + bool value; + + // True values. + EXPECT_TRUE(absl::ParseFlag("True", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("true", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("TRUE", &value, &err)); + EXPECT_TRUE(value); + + EXPECT_TRUE(absl::ParseFlag("Yes", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("yes", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("YES", &value, &err)); + EXPECT_TRUE(value); + + EXPECT_TRUE(absl::ParseFlag("t", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("T", &value, &err)); + EXPECT_TRUE(value); + + EXPECT_TRUE(absl::ParseFlag("y", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("Y", &value, &err)); + EXPECT_TRUE(value); + + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_TRUE(value); + + // False values. + EXPECT_TRUE(absl::ParseFlag("False", &value, &err)); + EXPECT_FALSE(value); + EXPECT_TRUE(absl::ParseFlag("false", &value, &err)); + EXPECT_FALSE(value); + EXPECT_TRUE(absl::ParseFlag("FALSE", &value, &err)); + EXPECT_FALSE(value); + + EXPECT_TRUE(absl::ParseFlag("No", &value, &err)); + EXPECT_FALSE(value); + EXPECT_TRUE(absl::ParseFlag("no", &value, &err)); + EXPECT_FALSE(value); + EXPECT_TRUE(absl::ParseFlag("NO", &value, &err)); + EXPECT_FALSE(value); + + EXPECT_TRUE(absl::ParseFlag("f", &value, &err)); + EXPECT_FALSE(value); + EXPECT_TRUE(absl::ParseFlag("F", &value, &err)); + EXPECT_FALSE(value); + + EXPECT_TRUE(absl::ParseFlag("n", &value, &err)); + EXPECT_FALSE(value); + EXPECT_TRUE(absl::ParseFlag("N", &value, &err)); + EXPECT_FALSE(value); + + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_FALSE(value); + + // Whitespace handling. + EXPECT_TRUE(absl::ParseFlag(" true", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag("true ", &value, &err)); + EXPECT_TRUE(value); + EXPECT_TRUE(absl::ParseFlag(" true ", &value, &err)); + EXPECT_TRUE(value); + + // Invalid input. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("11", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("tt", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestInt16Parsing) { + std::string err; + int16_t value; + + // Decimal values. + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("-1", &value, &err)); + EXPECT_EQ(value, -1); + EXPECT_TRUE(absl::ParseFlag("123", &value, &err)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(absl::ParseFlag("-18765", &value, &err)); + EXPECT_EQ(value, -18765); + EXPECT_TRUE(absl::ParseFlag("+3", &value, &err)); + EXPECT_EQ(value, 3); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("-001", &value, &err)); + EXPECT_EQ(value, -1); + EXPECT_TRUE(absl::ParseFlag("0000100", &value, &err)); + EXPECT_EQ(value, 100); + + // Hex values. + EXPECT_TRUE(absl::ParseFlag("0x10", &value, &err)); + EXPECT_EQ(value, 16); + EXPECT_TRUE(absl::ParseFlag("0X234", &value, &err)); + EXPECT_EQ(value, 564); + // TODO(rogeeff): fix below validations + EXPECT_FALSE(absl::ParseFlag("-0x7FFD", &value, &err)); + EXPECT_NE(value, -3); + EXPECT_FALSE(absl::ParseFlag("+0x31", &value, &err)); + EXPECT_NE(value, 49); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10 ", &value, &err)); + EXPECT_EQ(value, 10); + EXPECT_TRUE(absl::ParseFlag(" 11", &value, &err)); + EXPECT_EQ(value, 11); + EXPECT_TRUE(absl::ParseFlag(" 012 ", &value, &err)); + EXPECT_EQ(value, 12); + EXPECT_TRUE(absl::ParseFlag(" 0x22 ", &value, &err)); + EXPECT_EQ(value, 34); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("40000", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2U", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("FFF", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestUint16Parsing) { + std::string err; + uint16_t value; + + // Decimal values. + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("123", &value, &err)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(absl::ParseFlag("+3", &value, &err)); + EXPECT_EQ(value, 3); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("001", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0000100", &value, &err)); + EXPECT_EQ(value, 100); + + // Hex values. + EXPECT_TRUE(absl::ParseFlag("0x10", &value, &err)); + EXPECT_EQ(value, 16); + EXPECT_TRUE(absl::ParseFlag("0X234", &value, &err)); + EXPECT_EQ(value, 564); + // TODO(rogeeff): fix below validations + EXPECT_FALSE(absl::ParseFlag("+0x31", &value, &err)); + EXPECT_NE(value, 49); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10 ", &value, &err)); + EXPECT_EQ(value, 10); + EXPECT_TRUE(absl::ParseFlag(" 11", &value, &err)); + EXPECT_EQ(value, 11); + EXPECT_TRUE(absl::ParseFlag(" 012 ", &value, &err)); + EXPECT_EQ(value, 12); + EXPECT_TRUE(absl::ParseFlag(" 0x22 ", &value, &err)); + EXPECT_EQ(value, 34); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("70000", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("-1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2U", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("FFF", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestInt32Parsing) { + std::string err; + int32_t value; + + // Decimal values. + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("-1", &value, &err)); + EXPECT_EQ(value, -1); + EXPECT_TRUE(absl::ParseFlag("123", &value, &err)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(absl::ParseFlag("-98765", &value, &err)); + EXPECT_EQ(value, -98765); + EXPECT_TRUE(absl::ParseFlag("+3", &value, &err)); + EXPECT_EQ(value, 3); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("-001", &value, &err)); + EXPECT_EQ(value, -1); + EXPECT_TRUE(absl::ParseFlag("0000100", &value, &err)); + EXPECT_EQ(value, 100); + + // Hex values. + EXPECT_TRUE(absl::ParseFlag("0x10", &value, &err)); + EXPECT_EQ(value, 16); + EXPECT_TRUE(absl::ParseFlag("0X234", &value, &err)); + EXPECT_EQ(value, 564); + // TODO(rogeeff): fix below validations + EXPECT_FALSE(absl::ParseFlag("-0x7FFFFFFD", &value, &err)); + EXPECT_NE(value, -3); + EXPECT_FALSE(absl::ParseFlag("+0x31", &value, &err)); + EXPECT_NE(value, 49); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10 ", &value, &err)); + EXPECT_EQ(value, 10); + EXPECT_TRUE(absl::ParseFlag(" 11", &value, &err)); + EXPECT_EQ(value, 11); + EXPECT_TRUE(absl::ParseFlag(" 012 ", &value, &err)); + EXPECT_EQ(value, 12); + EXPECT_TRUE(absl::ParseFlag(" 0x22 ", &value, &err)); + EXPECT_EQ(value, 34); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("70000000000", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2U", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("FFF", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestUint32Parsing) { + std::string err; + uint32_t value; + + // Decimal values. + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("123", &value, &err)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(absl::ParseFlag("+3", &value, &err)); + EXPECT_EQ(value, 3); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0000100", &value, &err)); + EXPECT_EQ(value, 100); + + // Hex values. + EXPECT_TRUE(absl::ParseFlag("0x10", &value, &err)); + EXPECT_EQ(value, 16); + EXPECT_TRUE(absl::ParseFlag("0X234", &value, &err)); + EXPECT_EQ(value, 564); + EXPECT_TRUE(absl::ParseFlag("0xFFFFFFFD", &value, &err)); + EXPECT_EQ(value, 4294967293); + // TODO(rogeeff): fix below validations + EXPECT_FALSE(absl::ParseFlag("+0x31", &value, &err)); + EXPECT_NE(value, 49); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10 ", &value, &err)); + EXPECT_EQ(value, 10); + EXPECT_TRUE(absl::ParseFlag(" 11", &value, &err)); + EXPECT_EQ(value, 11); + EXPECT_TRUE(absl::ParseFlag(" 012 ", &value, &err)); + EXPECT_EQ(value, 12); + EXPECT_TRUE(absl::ParseFlag(" 0x22 ", &value, &err)); + EXPECT_EQ(value, 34); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("140000000000", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("-1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2U", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("FFF", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestInt64Parsing) { + std::string err; + int64_t value; + + // Decimal values. + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("-1", &value, &err)); + EXPECT_EQ(value, -1); + EXPECT_TRUE(absl::ParseFlag("123", &value, &err)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(absl::ParseFlag("-98765", &value, &err)); + EXPECT_EQ(value, -98765); + EXPECT_TRUE(absl::ParseFlag("+3", &value, &err)); + EXPECT_EQ(value, 3); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("001", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0000100", &value, &err)); + EXPECT_EQ(value, 100); + + // Hex values. + EXPECT_TRUE(absl::ParseFlag("0x10", &value, &err)); + EXPECT_EQ(value, 16); + EXPECT_TRUE(absl::ParseFlag("0XFFFAAABBBCCCDDD", &value, &err)); + EXPECT_EQ(value, 1152827684197027293); + // TODO(rogeeff): fix below validation + EXPECT_FALSE(absl::ParseFlag("-0x7FFFFFFFFFFFFFFE", &value, &err)); + EXPECT_NE(value, -2); + EXPECT_FALSE(absl::ParseFlag("+0x31", &value, &err)); + EXPECT_NE(value, 49); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10 ", &value, &err)); + EXPECT_EQ(value, 10); + EXPECT_TRUE(absl::ParseFlag(" 11", &value, &err)); + EXPECT_EQ(value, 11); + EXPECT_TRUE(absl::ParseFlag(" 012 ", &value, &err)); + EXPECT_EQ(value, 12); + EXPECT_TRUE(absl::ParseFlag(" 0x7F ", &value, &err)); + EXPECT_EQ(value, 127); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("0xFFFFFFFFFFFFFFFFFF", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2U", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("FFF", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestUInt64Parsing) { + std::string err; + uint64_t value; + + // Decimal values. + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0", &value, &err)); + EXPECT_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("123", &value, &err)); + EXPECT_EQ(value, 123); + EXPECT_TRUE(absl::ParseFlag("+13", &value, &err)); + EXPECT_EQ(value, 13); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("001", &value, &err)); + EXPECT_EQ(value, 1); + EXPECT_TRUE(absl::ParseFlag("0000300", &value, &err)); + EXPECT_EQ(value, 300); + + // Hex values. + EXPECT_TRUE(absl::ParseFlag("0x10", &value, &err)); + EXPECT_EQ(value, 16); + EXPECT_TRUE(absl::ParseFlag("0XFFFF", &value, &err)); + EXPECT_EQ(value, 65535); + // TODO(rogeeff): fix below validation + EXPECT_FALSE(absl::ParseFlag("+0x31", &value, &err)); + EXPECT_NE(value, 49); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10 ", &value, &err)); + EXPECT_EQ(value, 10); + EXPECT_TRUE(absl::ParseFlag(" 11", &value, &err)); + EXPECT_EQ(value, 11); + EXPECT_TRUE(absl::ParseFlag(" 012 ", &value, &err)); + EXPECT_EQ(value, 12); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("0xFFFFFFFFFFFFFFFFFF", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("-1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2U", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("FFF", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestFloatParsing) { + std::string err; + float value; + + // Ordinary values. + EXPECT_TRUE(absl::ParseFlag("1.3", &value, &err)); + EXPECT_FLOAT_EQ(value, 1.3f); + EXPECT_TRUE(absl::ParseFlag("-0.1", &value, &err)); + EXPECT_DOUBLE_EQ(value, -0.1f); + EXPECT_TRUE(absl::ParseFlag("+0.01", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.01f); + + // Scientific values. + EXPECT_TRUE(absl::ParseFlag("1.2e3", &value, &err)); + EXPECT_DOUBLE_EQ(value, 1.2e3f); + EXPECT_TRUE(absl::ParseFlag("9.8765402e-37", &value, &err)); + EXPECT_DOUBLE_EQ(value, 9.8765402e-37f); + EXPECT_TRUE(absl::ParseFlag("0.11e+3", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.11e+3f); + EXPECT_TRUE(absl::ParseFlag("1.e-2300", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.f); + EXPECT_TRUE(absl::ParseFlag("1.e+2300", &value, &err)); + EXPECT_TRUE(std::isinf(value)); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01.6", &value, &err)); + EXPECT_DOUBLE_EQ(value, 1.6f); + EXPECT_TRUE(absl::ParseFlag("000.0001", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.0001f); + + // Trailing zero values. + EXPECT_TRUE(absl::ParseFlag("-5.1000", &value, &err)); + EXPECT_DOUBLE_EQ(value, -5.1f); + + // Exceptional values. + EXPECT_TRUE(absl::ParseFlag("NaN", &value, &err)); + EXPECT_TRUE(std::isnan(value)); + EXPECT_TRUE(absl::ParseFlag("Inf", &value, &err)); + EXPECT_TRUE(std::isinf(value)); + + // Hex values + EXPECT_TRUE(absl::ParseFlag("0x10.23p12", &value, &err)); + EXPECT_DOUBLE_EQ(value, 66096.f); + EXPECT_TRUE(absl::ParseFlag("-0xF1.A3p-2", &value, &err)); + EXPECT_NEAR(value, -60.4092f, 5e-5f); + EXPECT_TRUE(absl::ParseFlag("+0x0.0AAp-12", &value, &err)); + EXPECT_NEAR(value, 1.01328e-05f, 5e-11f); + EXPECT_TRUE(absl::ParseFlag("0x.01p1", &value, &err)); + EXPECT_NEAR(value, 0.0078125f, 5e-8f); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10.1 ", &value, &err)); + EXPECT_DOUBLE_EQ(value, 10.1f); + EXPECT_TRUE(absl::ParseFlag(" 2.34", &value, &err)); + EXPECT_DOUBLE_EQ(value, 2.34f); + EXPECT_TRUE(absl::ParseFlag(" 5.7 ", &value, &err)); + EXPECT_DOUBLE_EQ(value, 5.7f); + EXPECT_TRUE(absl::ParseFlag(" -0xE0.F3p01 ", &value, &err)); + EXPECT_NEAR(value, -449.8984375f, 5e-8f); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2.3xxx", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("0x0.1pAA", &value, &err)); + // TODO(rogeeff): below assertion should fail + EXPECT_TRUE(absl::ParseFlag("0x0.1", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestDoubleParsing) { + std::string err; + double value; + + // Ordinary values. + EXPECT_TRUE(absl::ParseFlag("1.3", &value, &err)); + EXPECT_DOUBLE_EQ(value, 1.3); + EXPECT_TRUE(absl::ParseFlag("-0.1", &value, &err)); + EXPECT_DOUBLE_EQ(value, -0.1); + EXPECT_TRUE(absl::ParseFlag("+0.01", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.01); + + // Scientific values. + EXPECT_TRUE(absl::ParseFlag("1.2e3", &value, &err)); + EXPECT_DOUBLE_EQ(value, 1.2e3); + EXPECT_TRUE(absl::ParseFlag("9.00000002e-123", &value, &err)); + EXPECT_DOUBLE_EQ(value, 9.00000002e-123); + EXPECT_TRUE(absl::ParseFlag("0.11e+3", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.11e+3); + EXPECT_TRUE(absl::ParseFlag("1.e-2300", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0); + EXPECT_TRUE(absl::ParseFlag("1.e+2300", &value, &err)); + EXPECT_TRUE(std::isinf(value)); + + // Leading zero values. + EXPECT_TRUE(absl::ParseFlag("01.6", &value, &err)); + EXPECT_DOUBLE_EQ(value, 1.6); + EXPECT_TRUE(absl::ParseFlag("000.0001", &value, &err)); + EXPECT_DOUBLE_EQ(value, 0.0001); + + // Trailing zero values. + EXPECT_TRUE(absl::ParseFlag("-5.1000", &value, &err)); + EXPECT_DOUBLE_EQ(value, -5.1); + + // Exceptional values. + EXPECT_TRUE(absl::ParseFlag("NaN", &value, &err)); + EXPECT_TRUE(std::isnan(value)); + EXPECT_TRUE(absl::ParseFlag("nan", &value, &err)); + EXPECT_TRUE(std::isnan(value)); + EXPECT_TRUE(absl::ParseFlag("Inf", &value, &err)); + EXPECT_TRUE(std::isinf(value)); + EXPECT_TRUE(absl::ParseFlag("inf", &value, &err)); + EXPECT_TRUE(std::isinf(value)); + + // Hex values + EXPECT_TRUE(absl::ParseFlag("0x10.23p12", &value, &err)); + EXPECT_DOUBLE_EQ(value, 66096); + EXPECT_TRUE(absl::ParseFlag("-0xF1.A3p-2", &value, &err)); + EXPECT_NEAR(value, -60.4092, 5e-5); + EXPECT_TRUE(absl::ParseFlag("+0x0.0AAp-12", &value, &err)); + EXPECT_NEAR(value, 1.01328e-05, 5e-11); + EXPECT_TRUE(absl::ParseFlag("0x.01p1", &value, &err)); + EXPECT_NEAR(value, 0.0078125, 5e-8); + + // Whitespace handling + EXPECT_TRUE(absl::ParseFlag("10.1 ", &value, &err)); + EXPECT_DOUBLE_EQ(value, 10.1); + EXPECT_TRUE(absl::ParseFlag(" 2.34", &value, &err)); + EXPECT_DOUBLE_EQ(value, 2.34); + EXPECT_TRUE(absl::ParseFlag(" 5.7 ", &value, &err)); + EXPECT_DOUBLE_EQ(value, 5.7); + EXPECT_TRUE(absl::ParseFlag(" -0xE0.F3p01 ", &value, &err)); + EXPECT_NEAR(value, -449.8984375, 5e-8); + + // Invalid values. + EXPECT_FALSE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag(" ", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("--1", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\n", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("\t", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("2.3xxx", &value, &err)); + EXPECT_FALSE(absl::ParseFlag("0x0.1pAA", &value, &err)); + // TODO(rogeeff): below assertion should fail + EXPECT_TRUE(absl::ParseFlag("0x0.1", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestStringParsing) { + std::string err; + std::string value; + + EXPECT_TRUE(absl::ParseFlag("", &value, &err)); + EXPECT_EQ(value, ""); + EXPECT_TRUE(absl::ParseFlag(" ", &value, &err)); + EXPECT_EQ(value, " "); + EXPECT_TRUE(absl::ParseFlag(" ", &value, &err)); + EXPECT_EQ(value, " "); + EXPECT_TRUE(absl::ParseFlag("\n", &value, &err)); + EXPECT_EQ(value, "\n"); + EXPECT_TRUE(absl::ParseFlag("\t", &value, &err)); + EXPECT_EQ(value, "\t"); + EXPECT_TRUE(absl::ParseFlag("asdfg", &value, &err)); + EXPECT_EQ(value, "asdfg"); + EXPECT_TRUE(absl::ParseFlag("asdf ghjk", &value, &err)); + EXPECT_EQ(value, "asdf ghjk"); + EXPECT_TRUE(absl::ParseFlag("a\nb\nc", &value, &err)); + EXPECT_EQ(value, "a\nb\nc"); + EXPECT_TRUE(absl::ParseFlag("asd\0fgh", &value, &err)); + EXPECT_EQ(value, "asd"); + EXPECT_TRUE(absl::ParseFlag("\\\\", &value, &err)); + EXPECT_EQ(value, "\\\\"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestVectorOfStringParsing) { + std::string err; + std::vector<std::string> value; + + EXPECT_TRUE(absl::ParseFlag("", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>{}); + EXPECT_TRUE(absl::ParseFlag("1", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>({"1"})); + EXPECT_TRUE(absl::ParseFlag("a,b", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>({"a", "b"})); + EXPECT_TRUE(absl::ParseFlag("a,b,c,", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>({"a", "b", "c", ""})); + EXPECT_TRUE(absl::ParseFlag("a,,", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>({"a", "", ""})); + EXPECT_TRUE(absl::ParseFlag(",", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>({"", ""})); + EXPECT_TRUE(absl::ParseFlag("a, b,c ", &value, &err)); + EXPECT_EQ(value, std::vector<std::string>({"a", " b", "c "})); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestBoolUnparsing) { + EXPECT_EQ(absl::UnparseFlag(true), "true"); + EXPECT_EQ(absl::UnparseFlag(false), "false"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestInt16Unparsing) { + int16_t value; + + value = 1; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = -1; + EXPECT_EQ(absl::UnparseFlag(value), "-1"); + value = 9876; + EXPECT_EQ(absl::UnparseFlag(value), "9876"); + value = -987; + EXPECT_EQ(absl::UnparseFlag(value), "-987"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestUint16Unparsing) { + uint16_t value; + + value = 1; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = 19876; + EXPECT_EQ(absl::UnparseFlag(value), "19876"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestInt32Unparsing) { + int32_t value; + + value = 1; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = -1; + EXPECT_EQ(absl::UnparseFlag(value), "-1"); + value = 12345; + EXPECT_EQ(absl::UnparseFlag(value), "12345"); + value = -987; + EXPECT_EQ(absl::UnparseFlag(value), "-987"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestUint32Unparsing) { + uint32_t value; + + value = 1; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = 1234500; + EXPECT_EQ(absl::UnparseFlag(value), "1234500"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestInt64Unparsing) { + int64_t value; + + value = 1; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = -1; + EXPECT_EQ(absl::UnparseFlag(value), "-1"); + value = 123456789L; + EXPECT_EQ(absl::UnparseFlag(value), "123456789"); + value = -987654321L; + EXPECT_EQ(absl::UnparseFlag(value), "-987654321"); + value = 0x7FFFFFFFFFFFFFFF; + EXPECT_EQ(absl::UnparseFlag(value), "9223372036854775807"); + value = 0xFFFFFFFFFFFFFFFF; + EXPECT_EQ(absl::UnparseFlag(value), "-1"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestUint64Unparsing) { + uint64_t value; + + value = 1; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = 123456789L; + EXPECT_EQ(absl::UnparseFlag(value), "123456789"); + value = 0xFFFFFFFFFFFFFFFF; + EXPECT_EQ(absl::UnparseFlag(value), "18446744073709551615"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestFloatUnparsing) { + float value; + + value = 1.1f; + EXPECT_EQ(absl::UnparseFlag(value), "1.1"); + value = 0.01f; + EXPECT_EQ(absl::UnparseFlag(value), "0.01"); + value = 1.23e-2f; + EXPECT_EQ(absl::UnparseFlag(value), "0.0123"); + value = -0.71f; + EXPECT_EQ(absl::UnparseFlag(value), "-0.71"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestDoubleUnparsing) { + double value; + + value = 1.1; + EXPECT_EQ(absl::UnparseFlag(value), "1.1"); + value = 0.01; + EXPECT_EQ(absl::UnparseFlag(value), "0.01"); + value = 1.23e-2; + EXPECT_EQ(absl::UnparseFlag(value), "0.0123"); + value = -0.71; + EXPECT_EQ(absl::UnparseFlag(value), "-0.71"); + value = -0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = std::nan(""); + EXPECT_EQ(absl::UnparseFlag(value), "nan"); + value = std::numeric_limits<double>::infinity(); + EXPECT_EQ(absl::UnparseFlag(value), "inf"); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestStringUnparsing) { + EXPECT_EQ(absl::UnparseFlag(""), ""); + EXPECT_EQ(absl::UnparseFlag(" "), " "); + EXPECT_EQ(absl::UnparseFlag("qwerty"), "qwerty"); + EXPECT_EQ(absl::UnparseFlag("ASDFGH"), "ASDFGH"); + EXPECT_EQ(absl::UnparseFlag("\n\t "), "\n\t "); +} + +// -------------------------------------------------------------------- + +template <typename T> +void TestRoundtrip(T v) { + T new_v; + std::string err; + EXPECT_TRUE(absl::ParseFlag(absl::UnparseFlag(v), &new_v, &err)); + EXPECT_EQ(new_v, v); +} + +TEST(MarshallingTest, TestFloatRoundTrip) { + TestRoundtrip(0.1f); + TestRoundtrip(0.12f); + TestRoundtrip(0.123f); + TestRoundtrip(0.1234f); + TestRoundtrip(0.12345f); + TestRoundtrip(0.123456f); + TestRoundtrip(0.1234567f); + TestRoundtrip(0.12345678f); + + TestRoundtrip(0.1e20f); + TestRoundtrip(0.12e20f); + TestRoundtrip(0.123e20f); + TestRoundtrip(0.1234e20f); + TestRoundtrip(0.12345e20f); + TestRoundtrip(0.123456e20f); + TestRoundtrip(0.1234567e20f); + TestRoundtrip(0.12345678e20f); + + TestRoundtrip(0.1e-20f); + TestRoundtrip(0.12e-20f); + TestRoundtrip(0.123e-20f); + TestRoundtrip(0.1234e-20f); + TestRoundtrip(0.12345e-20f); + TestRoundtrip(0.123456e-20f); + TestRoundtrip(0.1234567e-20f); + TestRoundtrip(0.12345678e-20f); +} + +TEST(MarshallingTest, TestDoubleRoundTrip) { + TestRoundtrip(0.1); + TestRoundtrip(0.12); + TestRoundtrip(0.123); + TestRoundtrip(0.1234); + TestRoundtrip(0.12345); + TestRoundtrip(0.123456); + TestRoundtrip(0.1234567); + TestRoundtrip(0.12345678); + TestRoundtrip(0.123456789); + TestRoundtrip(0.1234567891); + TestRoundtrip(0.12345678912); + TestRoundtrip(0.123456789123); + TestRoundtrip(0.1234567891234); + TestRoundtrip(0.12345678912345); + TestRoundtrip(0.123456789123456); + TestRoundtrip(0.1234567891234567); + TestRoundtrip(0.12345678912345678); + + TestRoundtrip(0.1e50); + TestRoundtrip(0.12e50); + TestRoundtrip(0.123e50); + TestRoundtrip(0.1234e50); + TestRoundtrip(0.12345e50); + TestRoundtrip(0.123456e50); + TestRoundtrip(0.1234567e50); + TestRoundtrip(0.12345678e50); + TestRoundtrip(0.123456789e50); + TestRoundtrip(0.1234567891e50); + TestRoundtrip(0.12345678912e50); + TestRoundtrip(0.123456789123e50); + TestRoundtrip(0.1234567891234e50); + TestRoundtrip(0.12345678912345e50); + TestRoundtrip(0.123456789123456e50); + TestRoundtrip(0.1234567891234567e50); + TestRoundtrip(0.12345678912345678e50); + + TestRoundtrip(0.1e-50); + TestRoundtrip(0.12e-50); + TestRoundtrip(0.123e-50); + TestRoundtrip(0.1234e-50); + TestRoundtrip(0.12345e-50); + TestRoundtrip(0.123456e-50); + TestRoundtrip(0.1234567e-50); + TestRoundtrip(0.12345678e-50); + TestRoundtrip(0.123456789e-50); + TestRoundtrip(0.1234567891e-50); + TestRoundtrip(0.12345678912e-50); + TestRoundtrip(0.123456789123e-50); + TestRoundtrip(0.1234567891234e-50); + TestRoundtrip(0.12345678912345e-50); + TestRoundtrip(0.123456789123456e-50); + TestRoundtrip(0.1234567891234567e-50); + TestRoundtrip(0.12345678912345678e-50); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/parse.cc b/third_party/abseil_cpp/absl/flags/parse.cc new file mode 100644 index 000000000000..dd1a6796cabd --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/parse.cc @@ -0,0 +1,823 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/parse.h" + +#include <stdlib.h> + +#include <algorithm> +#include <fstream> +#include <iostream> +#include <iterator> +#include <string> +#include <tuple> +#include <utility> +#include <vector> + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/const_init.h" +#include "absl/base/thread_annotations.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/config.h" +#include "absl/flags/flag.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/flags/internal/flag.h" +#include "absl/flags/internal/parse.h" +#include "absl/flags/internal/private_handle_accessor.h" +#include "absl/flags/internal/program_name.h" +#include "absl/flags/internal/usage.h" +#include "absl/flags/reflection.h" +#include "absl/flags/usage.h" +#include "absl/flags/usage_config.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/synchronization/mutex.h" + +// -------------------------------------------------------------------- + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { +namespace { + +ABSL_CONST_INIT absl::Mutex processing_checks_guard(absl::kConstInit); + +ABSL_CONST_INIT bool flagfile_needs_processing + ABSL_GUARDED_BY(processing_checks_guard) = false; +ABSL_CONST_INIT bool fromenv_needs_processing + ABSL_GUARDED_BY(processing_checks_guard) = false; +ABSL_CONST_INIT bool tryfromenv_needs_processing + ABSL_GUARDED_BY(processing_checks_guard) = false; + +ABSL_CONST_INIT absl::Mutex specified_flags_guard(absl::kConstInit); +ABSL_CONST_INIT std::vector<const CommandLineFlag*>* specified_flags + ABSL_GUARDED_BY(specified_flags_guard) = nullptr; + +struct SpecifiedFlagsCompare { + bool operator()(const CommandLineFlag* a, const CommandLineFlag* b) const { + return a->Name() < b->Name(); + } + bool operator()(const CommandLineFlag* a, absl::string_view b) const { + return a->Name() < b; + } + bool operator()(absl::string_view a, const CommandLineFlag* b) const { + return a < b->Name(); + } +}; + +} // namespace +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +ABSL_FLAG(std::vector<std::string>, flagfile, {}, + "comma-separated list of files to load flags from") + .OnUpdate([]() { + if (absl::GetFlag(FLAGS_flagfile).empty()) return; + + absl::MutexLock l(&absl::flags_internal::processing_checks_guard); + + // Setting this flag twice before it is handled most likely an internal + // error and should be reviewed by developers. + if (absl::flags_internal::flagfile_needs_processing) { + ABSL_INTERNAL_LOG(WARNING, "flagfile set twice before it is handled"); + } + + absl::flags_internal::flagfile_needs_processing = true; + }); +ABSL_FLAG(std::vector<std::string>, fromenv, {}, + "comma-separated list of flags to set from the environment" + " [use 'export FLAGS_flag1=value']") + .OnUpdate([]() { + if (absl::GetFlag(FLAGS_fromenv).empty()) return; + + absl::MutexLock l(&absl::flags_internal::processing_checks_guard); + + // Setting this flag twice before it is handled most likely an internal + // error and should be reviewed by developers. + if (absl::flags_internal::fromenv_needs_processing) { + ABSL_INTERNAL_LOG(WARNING, "fromenv set twice before it is handled."); + } + + absl::flags_internal::fromenv_needs_processing = true; + }); +ABSL_FLAG(std::vector<std::string>, tryfromenv, {}, + "comma-separated list of flags to try to set from the environment if " + "present") + .OnUpdate([]() { + if (absl::GetFlag(FLAGS_tryfromenv).empty()) return; + + absl::MutexLock l(&absl::flags_internal::processing_checks_guard); + + // Setting this flag twice before it is handled most likely an internal + // error and should be reviewed by developers. + if (absl::flags_internal::tryfromenv_needs_processing) { + ABSL_INTERNAL_LOG(WARNING, + "tryfromenv set twice before it is handled."); + } + + absl::flags_internal::tryfromenv_needs_processing = true; + }); + +ABSL_FLAG(std::vector<std::string>, undefok, {}, + "comma-separated list of flag names that it is okay to specify " + "on the command line even if the program does not define a flag " + "with that name"); + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +namespace { + +class ArgsList { + public: + ArgsList() : next_arg_(0) {} + ArgsList(int argc, char* argv[]) : args_(argv, argv + argc), next_arg_(0) {} + explicit ArgsList(const std::vector<std::string>& args) + : args_(args), next_arg_(0) {} + + // Returns success status: true if parsing successful, false otherwise. + bool ReadFromFlagfile(const std::string& flag_file_name); + + int Size() const { return args_.size() - next_arg_; } + int FrontIndex() const { return next_arg_; } + absl::string_view Front() const { return args_[next_arg_]; } + void PopFront() { next_arg_++; } + + private: + std::vector<std::string> args_; + int next_arg_; +}; + +bool ArgsList::ReadFromFlagfile(const std::string& flag_file_name) { + std::ifstream flag_file(flag_file_name); + + if (!flag_file) { + flags_internal::ReportUsageError( + absl::StrCat("Can't open flagfile ", flag_file_name), true); + + return false; + } + + // This argument represents fake argv[0], which should be present in all arg + // lists. + args_.push_back(""); + + std::string line; + bool success = true; + + while (std::getline(flag_file, line)) { + absl::string_view stripped = absl::StripLeadingAsciiWhitespace(line); + + if (stripped.empty() || stripped[0] == '#') { + // Comment or empty line; just ignore. + continue; + } + + if (stripped[0] == '-') { + if (stripped == "--") { + flags_internal::ReportUsageError( + "Flagfile can't contain position arguments or --", true); + + success = false; + break; + } + + args_.push_back(std::string(stripped)); + continue; + } + + flags_internal::ReportUsageError( + absl::StrCat("Unexpected line in the flagfile ", flag_file_name, ": ", + line), + true); + + success = false; + } + + return success; +} + +// -------------------------------------------------------------------- + +// Reads the environment variable with name `name` and stores results in +// `value`. If variable is not present in environment returns false, otherwise +// returns true. +bool GetEnvVar(const char* var_name, std::string& var_value) { +#ifdef _WIN32 + char buf[1024]; + auto get_res = GetEnvironmentVariableA(var_name, buf, sizeof(buf)); + if (get_res >= sizeof(buf)) { + return false; + } + + if (get_res == 0) { + return false; + } + + var_value = std::string(buf, get_res); +#else + const char* val = ::getenv(var_name); + if (val == nullptr) { + return false; + } + + var_value = val; +#endif + + return true; +} + +// -------------------------------------------------------------------- + +// Returns: +// Flag name or empty if arg= -- +// Flag value after = in --flag=value (empty if --foo) +// "Is empty value" status. True if arg= --foo=, false otherwise. This is +// required to separate --foo from --foo=. +// For example: +// arg return values +// "--foo=bar" -> {"foo", "bar", false}. +// "--foo" -> {"foo", "", false}. +// "--foo=" -> {"foo", "", true}. +std::tuple<absl::string_view, absl::string_view, bool> SplitNameAndValue( + absl::string_view arg) { + // Allow -foo and --foo + absl::ConsumePrefix(&arg, "-"); + + if (arg.empty()) { + return std::make_tuple("", "", false); + } + + auto equal_sign_pos = arg.find("="); + + absl::string_view flag_name = arg.substr(0, equal_sign_pos); + + absl::string_view value; + bool is_empty_value = false; + + if (equal_sign_pos != absl::string_view::npos) { + value = arg.substr(equal_sign_pos + 1); + is_empty_value = value.empty(); + } + + return std::make_tuple(flag_name, value, is_empty_value); +} + +// -------------------------------------------------------------------- + +// Returns: +// found flag or nullptr +// is negative in case of --nofoo +std::tuple<CommandLineFlag*, bool> LocateFlag(absl::string_view flag_name) { + CommandLineFlag* flag = absl::FindCommandLineFlag(flag_name); + bool is_negative = false; + + if (!flag && absl::ConsumePrefix(&flag_name, "no")) { + flag = absl::FindCommandLineFlag(flag_name); + is_negative = true; + } + + return std::make_tuple(flag, is_negative); +} + +// -------------------------------------------------------------------- + +// Verify that default values of typed flags must be convertible to string and +// back. +void CheckDefaultValuesParsingRoundtrip() { +#ifndef NDEBUG + flags_internal::ForEachFlag([&](CommandLineFlag& flag) { + if (flag.IsRetired()) return; + +#define ABSL_FLAGS_INTERNAL_IGNORE_TYPE(T, _) \ + if (flag.IsOfType<T>()) return; + + ABSL_FLAGS_INTERNAL_SUPPORTED_TYPES(ABSL_FLAGS_INTERNAL_IGNORE_TYPE) +#undef ABSL_FLAGS_INTERNAL_IGNORE_TYPE + + flags_internal::PrivateHandleAccessor::CheckDefaultValueParsingRoundtrip( + flag); + }); +#endif +} + +// -------------------------------------------------------------------- + +// Returns success status, which is true if we successfully read all flag files, +// in which case new ArgLists are appended to the input_args in a reverse order +// of file names in the input flagfiles list. This order ensures that flags from +// the first flagfile in the input list are processed before the second flagfile +// etc. +bool ReadFlagfiles(const std::vector<std::string>& flagfiles, + std::vector<ArgsList>& input_args) { + bool success = true; + for (auto it = flagfiles.rbegin(); it != flagfiles.rend(); ++it) { + ArgsList al; + + if (al.ReadFromFlagfile(*it)) { + input_args.push_back(al); + } else { + success = false; + } + } + + return success; +} + +// Returns success status, which is true if were able to locate all environment +// variables correctly or if fail_on_absent_in_env is false. The environment +// variable names are expected to be of the form `FLAGS_<flag_name>`, where +// `flag_name` is a string from the input flag_names list. If successful we +// append a single ArgList at the end of the input_args. +bool ReadFlagsFromEnv(const std::vector<std::string>& flag_names, + std::vector<ArgsList>& input_args, + bool fail_on_absent_in_env) { + bool success = true; + std::vector<std::string> args; + + // This argument represents fake argv[0], which should be present in all arg + // lists. + args.push_back(""); + + for (const auto& flag_name : flag_names) { + // Avoid infinite recursion. + if (flag_name == "fromenv" || flag_name == "tryfromenv") { + flags_internal::ReportUsageError( + absl::StrCat("Infinite recursion on flag ", flag_name), true); + + success = false; + continue; + } + + const std::string envname = absl::StrCat("FLAGS_", flag_name); + std::string envval; + if (!GetEnvVar(envname.c_str(), envval)) { + if (fail_on_absent_in_env) { + flags_internal::ReportUsageError( + absl::StrCat(envname, " not found in environment"), true); + + success = false; + } + + continue; + } + + args.push_back(absl::StrCat("--", flag_name, "=", envval)); + } + + if (success) { + input_args.emplace_back(args); + } + + return success; +} + +// -------------------------------------------------------------------- + +// Returns success status, which is true if were able to handle all generator +// flags (flagfile, fromenv, tryfromemv) successfully. +bool HandleGeneratorFlags(std::vector<ArgsList>& input_args, + std::vector<std::string>& flagfile_value) { + bool success = true; + + absl::MutexLock l(&flags_internal::processing_checks_guard); + + // flagfile could have been set either on a command line or + // programmatically before invoking ParseCommandLine. Note that we do not + // actually process arguments specified in the flagfile, but instead + // create a secondary arguments list to be processed along with the rest + // of the comamnd line arguments. Since we always the process most recently + // created list of arguments first, this will result in flagfile argument + // being processed before any other argument in the command line. If + // FLAGS_flagfile contains more than one file name we create multiple new + // levels of arguments in a reverse order of file names. Thus we always + // process arguments from first file before arguments containing in a + // second file, etc. If flagfile contains another + // --flagfile inside of it, it will produce new level of arguments and + // processed before the rest of the flagfile. We are also collecting all + // flagfiles set on original command line. Unlike the rest of the flags, + // this flag can be set multiple times and is expected to be handled + // multiple times. We are collecting them all into a single list and set + // the value of FLAGS_flagfile to that value at the end of the parsing. + if (flags_internal::flagfile_needs_processing) { + auto flagfiles = absl::GetFlag(FLAGS_flagfile); + + if (input_args.size() == 1) { + flagfile_value.insert(flagfile_value.end(), flagfiles.begin(), + flagfiles.end()); + } + + success &= ReadFlagfiles(flagfiles, input_args); + + flags_internal::flagfile_needs_processing = false; + } + + // Similar to flagfile fromenv/tryfromemv can be set both + // programmatically and at runtime on a command line. Unlike flagfile these + // can't be recursive. + if (flags_internal::fromenv_needs_processing) { + auto flags_list = absl::GetFlag(FLAGS_fromenv); + + success &= ReadFlagsFromEnv(flags_list, input_args, true); + + flags_internal::fromenv_needs_processing = false; + } + + if (flags_internal::tryfromenv_needs_processing) { + auto flags_list = absl::GetFlag(FLAGS_tryfromenv); + + success &= ReadFlagsFromEnv(flags_list, input_args, false); + + flags_internal::tryfromenv_needs_processing = false; + } + + return success; +} + +// -------------------------------------------------------------------- + +void ResetGeneratorFlags(const std::vector<std::string>& flagfile_value) { + // Setting flagfile to the value which collates all the values set on a + // command line and programmatically. So if command line looked like + // --flagfile=f1 --flagfile=f2 the final value of the FLAGS_flagfile flag is + // going to be {"f1", "f2"} + if (!flagfile_value.empty()) { + absl::SetFlag(&FLAGS_flagfile, flagfile_value); + absl::MutexLock l(&flags_internal::processing_checks_guard); + flags_internal::flagfile_needs_processing = false; + } + + // fromenv/tryfromenv are set to <undefined> value. + if (!absl::GetFlag(FLAGS_fromenv).empty()) { + absl::SetFlag(&FLAGS_fromenv, {}); + } + if (!absl::GetFlag(FLAGS_tryfromenv).empty()) { + absl::SetFlag(&FLAGS_tryfromenv, {}); + } + + absl::MutexLock l(&flags_internal::processing_checks_guard); + flags_internal::fromenv_needs_processing = false; + flags_internal::tryfromenv_needs_processing = false; +} + +// -------------------------------------------------------------------- + +// Returns: +// success status +// deduced value +// We are also mutating curr_list in case if we need to get a hold of next +// argument in the input. +std::tuple<bool, absl::string_view> DeduceFlagValue(const CommandLineFlag& flag, + absl::string_view value, + bool is_negative, + bool is_empty_value, + ArgsList* curr_list) { + // Value is either an argument suffix after `=` in "--foo=<value>" + // or separate argument in case of "--foo" "<value>". + + // boolean flags have these forms: + // --foo + // --nofoo + // --foo=true + // --foo=false + // --nofoo=<value> is not supported + // --foo <value> is not supported + + // non boolean flags have these forms: + // --foo=<value> + // --foo <value> + // --nofoo is not supported + + if (flag.IsOfType<bool>()) { + if (value.empty()) { + if (is_empty_value) { + // "--bool_flag=" case + flags_internal::ReportUsageError( + absl::StrCat( + "Missing the value after assignment for the boolean flag '", + flag.Name(), "'"), + true); + return std::make_tuple(false, ""); + } + + // "--bool_flag" case + value = is_negative ? "0" : "1"; + } else if (is_negative) { + // "--nobool_flag=Y" case + flags_internal::ReportUsageError( + absl::StrCat("Negative form with assignment is not valid for the " + "boolean flag '", + flag.Name(), "'"), + true); + return std::make_tuple(false, ""); + } + } else if (is_negative) { + // "--noint_flag=1" case + flags_internal::ReportUsageError( + absl::StrCat("Negative form is not valid for the flag '", flag.Name(), + "'"), + true); + return std::make_tuple(false, ""); + } else if (value.empty() && (!is_empty_value)) { + if (curr_list->Size() == 1) { + // "--int_flag" case + flags_internal::ReportUsageError( + absl::StrCat("Missing the value for the flag '", flag.Name(), "'"), + true); + return std::make_tuple(false, ""); + } + + // "--int_flag" "10" case + curr_list->PopFront(); + value = curr_list->Front(); + + // Heuristic to detect the case where someone treats a string arg + // like a bool or just forgets to pass a value: + // --my_string_var --foo=bar + // We look for a flag of string type, whose value begins with a + // dash and corresponds to known flag or standalone --. + if (!value.empty() && value[0] == '-' && flag.IsOfType<std::string>()) { + auto maybe_flag_name = std::get<0>(SplitNameAndValue(value.substr(1))); + + if (maybe_flag_name.empty() || + std::get<0>(LocateFlag(maybe_flag_name)) != nullptr) { + // "--string_flag" "--known_flag" case + ABSL_INTERNAL_LOG( + WARNING, + absl::StrCat("Did you really mean to set flag '", flag.Name(), + "' to the value '", value, "'?")); + } + } + } + + return std::make_tuple(true, value); +} + +// -------------------------------------------------------------------- + +bool CanIgnoreUndefinedFlag(absl::string_view flag_name) { + auto undefok = absl::GetFlag(FLAGS_undefok); + if (std::find(undefok.begin(), undefok.end(), flag_name) != undefok.end()) { + return true; + } + + if (absl::ConsumePrefix(&flag_name, "no") && + std::find(undefok.begin(), undefok.end(), flag_name) != undefok.end()) { + return true; + } + + return false; +} + +} // namespace + +// -------------------------------------------------------------------- + +bool WasPresentOnCommandLine(absl::string_view flag_name) { + absl::MutexLock l(&specified_flags_guard); + ABSL_INTERNAL_CHECK(specified_flags != nullptr, + "ParseCommandLine is not invoked yet"); + + return std::binary_search(specified_flags->begin(), specified_flags->end(), + flag_name, SpecifiedFlagsCompare{}); +} + +// -------------------------------------------------------------------- + +std::vector<char*> ParseCommandLineImpl(int argc, char* argv[], + ArgvListAction arg_list_act, + UsageFlagsAction usage_flag_act, + OnUndefinedFlag on_undef_flag) { + ABSL_INTERNAL_CHECK(argc > 0, "Missing argv[0]"); + + // Once parsing has started we will not have more flag registrations. + // If we did, they would be missing during parsing, which is a problem on + // itself. + flags_internal::FinalizeRegistry(); + + // This routine does not return anything since we abort on failure. + CheckDefaultValuesParsingRoundtrip(); + + std::vector<std::string> flagfile_value; + + std::vector<ArgsList> input_args; + input_args.push_back(ArgsList(argc, argv)); + + std::vector<char*> output_args; + std::vector<char*> positional_args; + output_args.reserve(argc); + + // This is the list of undefined flags. The element of the list is the pair + // consisting of boolean indicating if flag came from command line (vs from + // some flag file we've read) and flag name. + // TODO(rogeeff): Eliminate the first element in the pair after cleanup. + std::vector<std::pair<bool, std::string>> undefined_flag_names; + + // Set program invocation name if it is not set before. + if (ProgramInvocationName() == "UNKNOWN") { + flags_internal::SetProgramInvocationName(argv[0]); + } + output_args.push_back(argv[0]); + + absl::MutexLock l(&specified_flags_guard); + if (specified_flags == nullptr) { + specified_flags = new std::vector<const CommandLineFlag*>; + } else { + specified_flags->clear(); + } + + // Iterate through the list of the input arguments. First level are arguments + // originated from argc/argv. Following levels are arguments originated from + // recursive parsing of flagfile(s). + bool success = true; + while (!input_args.empty()) { + // 10. First we process the built-in generator flags. + success &= HandleGeneratorFlags(input_args, flagfile_value); + + // 30. Select top-most (most recent) arguments list. If it is empty drop it + // and re-try. + ArgsList& curr_list = input_args.back(); + + curr_list.PopFront(); + + if (curr_list.Size() == 0) { + input_args.pop_back(); + continue; + } + + // 40. Pick up the front remaining argument in the current list. If current + // stack of argument lists contains only one element - we are processing an + // argument from the original argv. + absl::string_view arg(curr_list.Front()); + bool arg_from_argv = input_args.size() == 1; + + // 50. If argument does not start with - or is just "-" - this is + // positional argument. + if (!absl::ConsumePrefix(&arg, "-") || arg.empty()) { + ABSL_INTERNAL_CHECK(arg_from_argv, + "Flagfile cannot contain positional argument"); + + positional_args.push_back(argv[curr_list.FrontIndex()]); + continue; + } + + if (arg_from_argv && (arg_list_act == ArgvListAction::kKeepParsedArgs)) { + output_args.push_back(argv[curr_list.FrontIndex()]); + } + + // 60. Split the current argument on '=' to figure out the argument + // name and value. If flag name is empty it means we've got "--". value + // can be empty either if there were no '=' in argument string at all or + // an argument looked like "--foo=". In a latter case is_empty_value is + // true. + absl::string_view flag_name; + absl::string_view value; + bool is_empty_value = false; + + std::tie(flag_name, value, is_empty_value) = SplitNameAndValue(arg); + + // 70. "--" alone means what it does for GNU: stop flags parsing. We do + // not support positional arguments in flagfiles, so we just drop them. + if (flag_name.empty()) { + ABSL_INTERNAL_CHECK(arg_from_argv, + "Flagfile cannot contain positional argument"); + + curr_list.PopFront(); + break; + } + + // 80. Locate the flag based on flag name. Handle both --foo and --nofoo + CommandLineFlag* flag = nullptr; + bool is_negative = false; + std::tie(flag, is_negative) = LocateFlag(flag_name); + + if (flag == nullptr) { + // Usage flags are not modeled as Abseil flags. Locate them separately. + if (flags_internal::DeduceUsageFlags(flag_name, value)) { + continue; + } + + if (on_undef_flag != OnUndefinedFlag::kIgnoreUndefined) { + undefined_flag_names.emplace_back(arg_from_argv, + std::string(flag_name)); + } + continue; + } + + // 90. Deduce flag's value (from this or next argument) + auto curr_index = curr_list.FrontIndex(); + bool value_success = true; + std::tie(value_success, value) = + DeduceFlagValue(*flag, value, is_negative, is_empty_value, &curr_list); + success &= value_success; + + // If above call consumed an argument, it was a standalone value + if (arg_from_argv && (arg_list_act == ArgvListAction::kKeepParsedArgs) && + (curr_index != curr_list.FrontIndex())) { + output_args.push_back(argv[curr_list.FrontIndex()]); + } + + // 100. Set the located flag to a new new value, unless it is retired. + // Setting retired flag fails, but we ignoring it here while also reporting + // access to retired flag. + std::string error; + if (!flags_internal::PrivateHandleAccessor::ParseFrom( + *flag, value, SET_FLAGS_VALUE, kCommandLine, error)) { + if (flag->IsRetired()) continue; + + flags_internal::ReportUsageError(error, true); + success = false; + } else { + specified_flags->push_back(flag); + } + } + + for (const auto& flag_name : undefined_flag_names) { + if (CanIgnoreUndefinedFlag(flag_name.second)) continue; + + flags_internal::ReportUsageError( + absl::StrCat("Unknown command line flag '", flag_name.second, "'"), + true); + + success = false; + } + +#if ABSL_FLAGS_STRIP_NAMES + if (!success) { + flags_internal::ReportUsageError( + "NOTE: command line flags are disabled in this build", true); + } +#endif + + if (!success) { + flags_internal::HandleUsageFlags(std::cout, + ProgramUsageMessage()); + std::exit(1); + } + + if (usage_flag_act == UsageFlagsAction::kHandleUsage) { + int exit_code = flags_internal::HandleUsageFlags( + std::cout, ProgramUsageMessage()); + + if (exit_code != -1) { + std::exit(exit_code); + } + } + + ResetGeneratorFlags(flagfile_value); + + // Reinstate positional args which were intermixed with flags in the arguments + // list. + for (auto arg : positional_args) { + output_args.push_back(arg); + } + + // All the remaining arguments are positional. + if (!input_args.empty()) { + for (int arg_index = input_args.back().FrontIndex(); arg_index < argc; + ++arg_index) { + output_args.push_back(argv[arg_index]); + } + } + + // Trim and sort the vector. + specified_flags->shrink_to_fit(); + std::sort(specified_flags->begin(), specified_flags->end(), + SpecifiedFlagsCompare{}); + return output_args; +} + +} // namespace flags_internal + +// -------------------------------------------------------------------- + +std::vector<char*> ParseCommandLine(int argc, char* argv[]) { + return flags_internal::ParseCommandLineImpl( + argc, argv, flags_internal::ArgvListAction::kRemoveParsedArgs, + flags_internal::UsageFlagsAction::kHandleUsage, + flags_internal::OnUndefinedFlag::kAbortIfUndefined); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/parse.h b/third_party/abseil_cpp/absl/flags/parse.h new file mode 100644 index 000000000000..929de2cb40de --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/parse.h @@ -0,0 +1,60 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: parse.h +// ----------------------------------------------------------------------------- +// +// This file defines the main parsing function for Abseil flags: +// `absl::ParseCommandLine()`. + +#ifndef ABSL_FLAGS_PARSE_H_ +#define ABSL_FLAGS_PARSE_H_ + +#include <vector> + +#include "absl/base/config.h" +#include "absl/flags/internal/parse.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ParseCommandLine() +// +// Parses the set of command-line arguments passed in the `argc` (argument +// count) and `argv[]` (argument vector) parameters from `main()`, assigning +// values to any defined Abseil flags. (Any arguments passed after the +// flag-terminating delimiter (`--`) are treated as positional arguments and +// ignored.) +// +// Any command-line flags (and arguments to those flags) are parsed into Abseil +// Flag values, if those flags are defined. Any undefined flags will either +// return an error, or be ignored if that flag is designated using `undefok` to +// indicate "undefined is OK." +// +// Any command-line positional arguments not part of any command-line flag (or +// arguments to a flag) are returned in a vector, with the program invocation +// name at position 0 of that vector. (Note that this includes positional +// arguments after the flag-terminating delimiter `--`.) +// +// After all flags and flag arguments are parsed, this function looks for any +// built-in usage flags (e.g. `--help`), and if any were specified, it reports +// help messages and then exits the program. +std::vector<char*> ParseCommandLine(int argc, char* argv[]); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_PARSE_H_ diff --git a/third_party/abseil_cpp/absl/flags/parse_test.cc b/third_party/abseil_cpp/absl/flags/parse_test.cc new file mode 100644 index 000000000000..41bc0bc6b3a4 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/parse_test.cc @@ -0,0 +1,929 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/parse.h" + +#include <stdlib.h> + +#include <fstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/scoped_set_env.h" +#include "absl/flags/declare.h" +#include "absl/flags/flag.h" +#include "absl/flags/internal/parse.h" +#include "absl/flags/internal/usage.h" +#include "absl/flags/reflection.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/strings/substitute.h" +#include "absl/types/span.h" + +#ifdef _WIN32 +#include <windows.h> +#endif + +namespace { + +using absl::base_internal::ScopedSetEnv; + +struct UDT { + UDT() = default; + UDT(const UDT&) = default; + UDT(int v) : value(v) {} // NOLINT + + int value; +}; + +bool AbslParseFlag(absl::string_view in, UDT* udt, std::string* err) { + if (in == "A") { + udt->value = 1; + return true; + } + if (in == "AAA") { + udt->value = 10; + return true; + } + + *err = "Use values A, AAA instead"; + return false; +} +std::string AbslUnparseFlag(const UDT& udt) { + return udt.value == 1 ? "A" : "AAA"; +} + +std::string GetTestTmpDirEnvVar(const char* const env_var_name) { +#ifdef _WIN32 + char buf[MAX_PATH]; + auto get_res = GetEnvironmentVariableA(env_var_name, buf, sizeof(buf)); + if (get_res >= sizeof(buf) || get_res == 0) { + return ""; + } + + return std::string(buf, get_res); +#else + const char* val = ::getenv(env_var_name); + if (val == nullptr) { + return ""; + } + + return val; +#endif +} + +const std::string& GetTestTempDir() { + static std::string* temp_dir_name = []() -> std::string* { + std::string* res = new std::string(GetTestTmpDirEnvVar("TEST_TMPDIR")); + + if (res->empty()) { + *res = GetTestTmpDirEnvVar("TMPDIR"); + } + + if (res->empty()) { +#ifdef _WIN32 + char temp_path_buffer[MAX_PATH]; + + auto len = GetTempPathA(MAX_PATH, temp_path_buffer); + if (len < MAX_PATH && len != 0) { + std::string temp_dir_name = temp_path_buffer; + if (!absl::EndsWith(temp_dir_name, "\\")) { + temp_dir_name.push_back('\\'); + } + absl::StrAppend(&temp_dir_name, "parse_test.", GetCurrentProcessId()); + if (CreateDirectoryA(temp_dir_name.c_str(), nullptr)) { + *res = temp_dir_name; + } + } +#else + char temp_dir_template[] = "/tmp/parse_test.XXXXXX"; + if (auto* unique_name = ::mkdtemp(temp_dir_template)) { + *res = unique_name; + } +#endif + } + + if (res->empty()) { + ABSL_INTERNAL_LOG(FATAL, + "Failed to make temporary directory for data files"); + } + +#ifdef _WIN32 + *res += "\\"; +#else + *res += "/"; +#endif + + return res; + }(); + + return *temp_dir_name; +} + +struct FlagfileData { + const absl::string_view file_name; + const absl::Span<const char* const> file_lines; +}; + +// clang-format off +constexpr const char* const ff1_data[] = { + "# comment ", + " # comment ", + "", + " ", + "--int_flag=-1", + " --string_flag=q2w2 ", + " ## ", + " --double_flag=0.1", + "--bool_flag=Y " +}; + +constexpr const char* const ff2_data[] = { + "# Setting legacy flag", + "--legacy_int=1111", + "--legacy_bool", + "--nobool_flag", + "--legacy_str=aqsw", + "--int_flag=100", + " ## =============" +}; +// clang-format on + +// Builds flagfile flag in the flagfile_flag buffer and returns it. This +// function also creates a temporary flagfile based on FlagfileData input. +// We create a flagfile in a temporary directory with the name specified in +// FlagfileData and populate it with lines specifed in FlagfileData. If $0 is +// referenced in any of the lines in FlagfileData they are replaced with +// temporary directory location. This way we can test inclusion of one flagfile +// from another flagfile. +const char* GetFlagfileFlag(const std::vector<FlagfileData>& ffd, + std::string& flagfile_flag) { + flagfile_flag = "--flagfile="; + absl::string_view separator; + for (const auto& flagfile_data : ffd) { + std::string flagfile_name = + absl::StrCat(GetTestTempDir(), flagfile_data.file_name); + + std::ofstream flagfile_out(flagfile_name); + for (auto line : flagfile_data.file_lines) { + flagfile_out << absl::Substitute(line, GetTestTempDir()) << "\n"; + } + + absl::StrAppend(&flagfile_flag, separator, flagfile_name); + separator = ","; + } + + return flagfile_flag.c_str(); +} + +} // namespace + +ABSL_FLAG(int, int_flag, 1, ""); +ABSL_FLAG(double, double_flag, 1.1, ""); +ABSL_FLAG(std::string, string_flag, "a", ""); +ABSL_FLAG(bool, bool_flag, false, ""); +ABSL_FLAG(UDT, udt_flag, -1, ""); +ABSL_RETIRED_FLAG(int, legacy_int, 1, ""); +ABSL_RETIRED_FLAG(bool, legacy_bool, false, ""); +ABSL_RETIRED_FLAG(std::string, legacy_str, "l", ""); + +namespace { + +namespace flags = absl::flags_internal; +using testing::ElementsAreArray; + +class ParseTest : public testing::Test { + public: + ~ParseTest() override { flags::SetFlagsHelpMode(flags::HelpMode::kNone); } + + private: + absl::FlagSaver flag_saver_; +}; + +// -------------------------------------------------------------------- + +template <int N> +std::vector<char*> InvokeParse(const char* (&in_argv)[N]) { + return absl::ParseCommandLine(N, const_cast<char**>(in_argv)); +} + +// -------------------------------------------------------------------- + +template <int N> +void TestParse(const char* (&in_argv)[N], int int_flag_value, + double double_flag_val, absl::string_view string_flag_val, + bool bool_flag_val, int exp_position_args = 0) { + auto out_args = InvokeParse(in_argv); + + EXPECT_EQ(out_args.size(), 1 + exp_position_args); + EXPECT_STREQ(out_args[0], "testbin"); + + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), int_flag_value); + EXPECT_NEAR(absl::GetFlag(FLAGS_double_flag), double_flag_val, 0.0001); + EXPECT_EQ(absl::GetFlag(FLAGS_string_flag), string_flag_val); + EXPECT_EQ(absl::GetFlag(FLAGS_bool_flag), bool_flag_val); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestEmptyArgv) { + const char* in_argv[] = {"testbin"}; + + auto out_args = InvokeParse(in_argv); + + EXPECT_EQ(out_args.size(), 1); + EXPECT_STREQ(out_args[0], "testbin"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidIntArg) { + const char* in_args1[] = { + "testbin", + "--int_flag=10", + }; + TestParse(in_args1, 10, 1.1, "a", false); + + const char* in_args2[] = { + "testbin", + "-int_flag=020", + }; + TestParse(in_args2, 20, 1.1, "a", false); + + const char* in_args3[] = { + "testbin", + "--int_flag", + "-30", + }; + TestParse(in_args3, -30, 1.1, "a", false); + + const char* in_args4[] = { + "testbin", + "-int_flag", + "0x21", + }; + TestParse(in_args4, 33, 1.1, "a", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidDoubleArg) { + const char* in_args1[] = { + "testbin", + "--double_flag=2.3", + }; + TestParse(in_args1, 1, 2.3, "a", false); + + const char* in_args2[] = { + "testbin", + "--double_flag=0x1.2", + }; + TestParse(in_args2, 1, 1.125, "a", false); + + const char* in_args3[] = { + "testbin", + "--double_flag", + "99.7", + }; + TestParse(in_args3, 1, 99.7, "a", false); + + const char* in_args4[] = { + "testbin", + "--double_flag", + "0x20.1", + }; + TestParse(in_args4, 1, 32.0625, "a", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidStringArg) { + const char* in_args1[] = { + "testbin", + "--string_flag=aqswde", + }; + TestParse(in_args1, 1, 1.1, "aqswde", false); + + const char* in_args2[] = { + "testbin", + "-string_flag=a=b=c", + }; + TestParse(in_args2, 1, 1.1, "a=b=c", false); + + const char* in_args3[] = { + "testbin", + "--string_flag", + "zaxscd", + }; + TestParse(in_args3, 1, 1.1, "zaxscd", false); + + const char* in_args4[] = { + "testbin", + "-string_flag", + "--int_flag", + }; + TestParse(in_args4, 1, 1.1, "--int_flag", false); + + const char* in_args5[] = { + "testbin", + "--string_flag", + "--no_a_flag=11", + }; + TestParse(in_args5, 1, 1.1, "--no_a_flag=11", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidBoolArg) { + const char* in_args1[] = { + "testbin", + "--bool_flag", + }; + TestParse(in_args1, 1, 1.1, "a", true); + + const char* in_args2[] = { + "testbin", + "--nobool_flag", + }; + TestParse(in_args2, 1, 1.1, "a", false); + + const char* in_args3[] = { + "testbin", + "--bool_flag=true", + }; + TestParse(in_args3, 1, 1.1, "a", true); + + const char* in_args4[] = { + "testbin", + "-bool_flag=false", + }; + TestParse(in_args4, 1, 1.1, "a", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidUDTArg) { + const char* in_args1[] = { + "testbin", + "--udt_flag=A", + }; + InvokeParse(in_args1); + + EXPECT_EQ(absl::GetFlag(FLAGS_udt_flag).value, 1); + + const char* in_args2[] = {"testbin", "--udt_flag", "AAA"}; + InvokeParse(in_args2); + + EXPECT_EQ(absl::GetFlag(FLAGS_udt_flag).value, 10); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidMultipleArg) { + const char* in_args1[] = { + "testbin", "--bool_flag", "--int_flag=2", + "--double_flag=0.1", "--string_flag=asd", + }; + TestParse(in_args1, 2, 0.1, "asd", true); + + const char* in_args2[] = { + "testbin", "--string_flag=", "--nobool_flag", "--int_flag", + "-011", "--double_flag", "-1e-2", + }; + TestParse(in_args2, -11, -0.01, "", false); + + const char* in_args3[] = { + "testbin", "--int_flag", "-0", "--string_flag", "\"\"", + "--bool_flag=true", "--double_flag=1e18", + }; + TestParse(in_args3, 0, 1e18, "\"\"", true); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestPositionalArgs) { + const char* in_args1[] = { + "testbin", + "p1", + "p2", + }; + TestParse(in_args1, 1, 1.1, "a", false, 2); + + auto out_args1 = InvokeParse(in_args1); + + EXPECT_STREQ(out_args1[1], "p1"); + EXPECT_STREQ(out_args1[2], "p2"); + + const char* in_args2[] = { + "testbin", + "--int_flag=2", + "p1", + }; + TestParse(in_args2, 2, 1.1, "a", false, 1); + + auto out_args2 = InvokeParse(in_args2); + + EXPECT_STREQ(out_args2[1], "p1"); + + const char* in_args3[] = {"testbin", "p1", "--int_flag=3", + "p2", "--bool_flag", "true"}; + TestParse(in_args3, 3, 1.1, "a", true, 3); + + auto out_args3 = InvokeParse(in_args3); + + EXPECT_STREQ(out_args3[1], "p1"); + EXPECT_STREQ(out_args3[2], "p2"); + EXPECT_STREQ(out_args3[3], "true"); + + const char* in_args4[] = { + "testbin", + "--", + "p1", + "p2", + }; + TestParse(in_args4, 3, 1.1, "a", true, 2); + + auto out_args4 = InvokeParse(in_args4); + + EXPECT_STREQ(out_args4[1], "p1"); + EXPECT_STREQ(out_args4[2], "p2"); + + const char* in_args5[] = { + "testbin", "p1", "--int_flag=4", "--", "--bool_flag", "false", "p2", + }; + TestParse(in_args5, 4, 1.1, "a", true, 4); + + auto out_args5 = InvokeParse(in_args5); + + EXPECT_STREQ(out_args5[1], "p1"); + EXPECT_STREQ(out_args5[2], "--bool_flag"); + EXPECT_STREQ(out_args5[3], "false"); + EXPECT_STREQ(out_args5[4], "p2"); +} + +// -------------------------------------------------------------------- + +using ParseDeathTest = ParseTest; + +TEST_F(ParseDeathTest, TestUndefinedArg) { + const char* in_args1[] = { + "testbin", + "--undefined_flag", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args1), + "Unknown command line flag 'undefined_flag'"); + + const char* in_args2[] = { + "testbin", + "--noprefixed_flag", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args2), + "Unknown command line flag 'noprefixed_flag'"); + + const char* in_args3[] = { + "testbin", + "--Int_flag=1", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args3), + "Unknown command line flag 'Int_flag'"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestInvalidBoolFlagFormat) { + const char* in_args1[] = { + "testbin", + "--bool_flag=", + }; + EXPECT_DEATH_IF_SUPPORTED( + InvokeParse(in_args1), + "Missing the value after assignment for the boolean flag 'bool_flag'"); + + const char* in_args2[] = { + "testbin", + "--nobool_flag=true", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args2), + "Negative form with assignment is not valid for the boolean " + "flag 'bool_flag'"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestInvalidNonBoolFlagFormat) { + const char* in_args1[] = { + "testbin", + "--nostring_flag", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args1), + "Negative form is not valid for the flag 'string_flag'"); + + const char* in_args2[] = { + "testbin", + "--int_flag", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args2), + "Missing the value for the flag 'int_flag'"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestInvalidUDTFlagFormat) { + const char* in_args1[] = { + "testbin", + "--udt_flag=1", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args1), + "Illegal value '1' specified for flag 'udt_flag'; Use values A, " + "AAA instead"); + + const char* in_args2[] = { + "testbin", + "--udt_flag", + "AA", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args2), + "Illegal value 'AA' specified for flag 'udt_flag'; Use values " + "A, AAA instead"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestLegacyFlags) { + const char* in_args1[] = { + "testbin", + "--legacy_int=11", + }; + TestParse(in_args1, 1, 1.1, "a", false); + + const char* in_args2[] = { + "testbin", + "--legacy_bool", + }; + TestParse(in_args2, 1, 1.1, "a", false); + + const char* in_args3[] = { + "testbin", "--legacy_int", "22", "--int_flag=2", + "--legacy_bool", "true", "--legacy_str", "--string_flag=qwe", + }; + TestParse(in_args3, 2, 1.1, "a", false, 1); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestSimpleValidFlagfile) { + std::string flagfile_flag; + + const char* in_args1[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff1", absl::MakeConstSpan(ff1_data)}}, + flagfile_flag), + }; + TestParse(in_args1, -1, 0.1, "q2w2 ", true); + + const char* in_args2[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff2", absl::MakeConstSpan(ff2_data)}}, + flagfile_flag), + }; + TestParse(in_args2, 100, 0.1, "q2w2 ", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestValidMultiFlagfile) { + std::string flagfile_flag; + + const char* in_args1[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff2", absl::MakeConstSpan(ff2_data)}, + {"parse_test.ff1", absl::MakeConstSpan(ff1_data)}}, + flagfile_flag), + }; + TestParse(in_args1, -1, 0.1, "q2w2 ", true); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestFlagfileMixedWithRegularFlags) { + std::string flagfile_flag; + + const char* in_args1[] = { + "testbin", "--int_flag=3", + GetFlagfileFlag({{"parse_test.ff1", absl::MakeConstSpan(ff1_data)}}, + flagfile_flag), + "-double_flag=0.2"}; + TestParse(in_args1, -1, 0.2, "q2w2 ", true); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestFlagfileInFlagfile) { + std::string flagfile_flag; + + constexpr const char* const ff3_data[] = { + "--flagfile=$0/parse_test.ff1", + "--flagfile=$0/parse_test.ff2", + }; + + GetFlagfileFlag({{"parse_test.ff2", absl::MakeConstSpan(ff2_data)}, + {"parse_test.ff1", absl::MakeConstSpan(ff1_data)}}, + flagfile_flag); + + const char* in_args1[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff3", absl::MakeConstSpan(ff3_data)}}, + flagfile_flag), + }; + TestParse(in_args1, 100, 0.1, "q2w2 ", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestInvalidFlagfiles) { + std::string flagfile_flag; + + constexpr const char* const ff4_data[] = { + "--unknown_flag=10" + }; + + const char* in_args1[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff4", + absl::MakeConstSpan(ff4_data)}}, flagfile_flag), + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args1), + "Unknown command line flag 'unknown_flag'"); + + constexpr const char* const ff5_data[] = { + "--int_flag 10", + }; + + const char* in_args2[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff5", + absl::MakeConstSpan(ff5_data)}}, flagfile_flag), + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args2), + "Unknown command line flag 'int_flag 10'"); + + constexpr const char* const ff6_data[] = { + "--int_flag=10", "--", "arg1", "arg2", "arg3", + }; + + const char* in_args3[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff6", absl::MakeConstSpan(ff6_data)}}, + flagfile_flag), + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args3), + "Flagfile can't contain position arguments or --"); + + const char* in_args4[] = { + "testbin", + "--flagfile=invalid_flag_file", + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args4), + "Can't open flagfile invalid_flag_file"); + + constexpr const char* const ff7_data[] = { + "--int_flag=10", + "*bin*", + "--str_flag=aqsw", + }; + + const char* in_args5[] = { + "testbin", + GetFlagfileFlag({{"parse_test.ff7", absl::MakeConstSpan(ff7_data)}}, + flagfile_flag), + }; + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args5), + "Unexpected line in the flagfile .*: \\*bin\\*"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestReadingRequiredFlagsFromEnv) { + const char* in_args1[] = {"testbin", + "--fromenv=int_flag,bool_flag,string_flag"}; + + ScopedSetEnv set_int_flag("FLAGS_int_flag", "33"); + ScopedSetEnv set_bool_flag("FLAGS_bool_flag", "True"); + ScopedSetEnv set_string_flag("FLAGS_string_flag", "AQ12"); + + TestParse(in_args1, 33, 1.1, "AQ12", true); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestReadingUnsetRequiredFlagsFromEnv) { + const char* in_args1[] = {"testbin", "--fromenv=int_flag"}; + + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args1), + "FLAGS_int_flag not found in environment"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestRecursiveFlagsFromEnv) { + const char* in_args1[] = {"testbin", "--fromenv=tryfromenv"}; + + ScopedSetEnv set_tryfromenv("FLAGS_tryfromenv", "int_flag"); + + EXPECT_DEATH_IF_SUPPORTED(InvokeParse(in_args1), + "Infinite recursion on flag tryfromenv"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestReadingOptionalFlagsFromEnv) { + const char* in_args1[] = { + "testbin", "--tryfromenv=int_flag,bool_flag,string_flag,other_flag"}; + + ScopedSetEnv set_int_flag("FLAGS_int_flag", "17"); + ScopedSetEnv set_bool_flag("FLAGS_bool_flag", "Y"); + + TestParse(in_args1, 17, 1.1, "a", true); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestReadingFlagsFromEnvMoxedWithRegularFlags) { + const char* in_args1[] = { + "testbin", + "--bool_flag=T", + "--tryfromenv=int_flag,bool_flag", + "--int_flag=-21", + }; + + ScopedSetEnv set_int_flag("FLAGS_int_flag", "-15"); + ScopedSetEnv set_bool_flag("FLAGS_bool_flag", "F"); + + TestParse(in_args1, -21, 1.1, "a", false); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestKeepParsedArgs) { + const char* in_args1[] = { + "testbin", "arg1", "--bool_flag", + "--int_flag=211", "arg2", "--double_flag=1.1", + "--string_flag", "asd", "--", + "arg3", "arg4", + }; + + auto out_args1 = InvokeParse(in_args1); + + EXPECT_THAT( + out_args1, + ElementsAreArray({absl::string_view("testbin"), absl::string_view("arg1"), + absl::string_view("arg2"), absl::string_view("arg3"), + absl::string_view("arg4")})); + + auto out_args2 = flags::ParseCommandLineImpl( + 11, const_cast<char**>(in_args1), flags::ArgvListAction::kKeepParsedArgs, + flags::UsageFlagsAction::kHandleUsage, + flags::OnUndefinedFlag::kAbortIfUndefined); + + EXPECT_THAT( + out_args2, + ElementsAreArray({absl::string_view("testbin"), + absl::string_view("--bool_flag"), + absl::string_view("--int_flag=211"), + absl::string_view("--double_flag=1.1"), + absl::string_view("--string_flag"), + absl::string_view("asd"), absl::string_view("--"), + absl::string_view("arg1"), absl::string_view("arg2"), + absl::string_view("arg3"), absl::string_view("arg4")})); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, TestIgnoreUndefinedFlags) { + const char* in_args1[] = { + "testbin", + "arg1", + "--undef_flag=aa", + "--int_flag=21", + }; + + auto out_args1 = flags::ParseCommandLineImpl( + 4, const_cast<char**>(in_args1), flags::ArgvListAction::kRemoveParsedArgs, + flags::UsageFlagsAction::kHandleUsage, + flags::OnUndefinedFlag::kIgnoreUndefined); + + EXPECT_THAT(out_args1, ElementsAreArray({absl::string_view("testbin"), + absl::string_view("arg1")})); + + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 21); + + const char* in_args2[] = { + "testbin", + "arg1", + "--undef_flag=aa", + "--string_flag=AA", + }; + + auto out_args2 = flags::ParseCommandLineImpl( + 4, const_cast<char**>(in_args2), flags::ArgvListAction::kKeepParsedArgs, + flags::UsageFlagsAction::kHandleUsage, + flags::OnUndefinedFlag::kIgnoreUndefined); + + EXPECT_THAT( + out_args2, + ElementsAreArray( + {absl::string_view("testbin"), absl::string_view("--undef_flag=aa"), + absl::string_view("--string_flag=AA"), absl::string_view("arg1")})); + + EXPECT_EQ(absl::GetFlag(FLAGS_string_flag), "AA"); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestSimpleHelpFlagHandling) { + const char* in_args1[] = { + "testbin", + "--help", + }; + + EXPECT_EXIT(InvokeParse(in_args1), testing::ExitedWithCode(1), ""); + + const char* in_args2[] = { + "testbin", + "--help", + "--int_flag=3", + }; + + auto out_args2 = flags::ParseCommandLineImpl( + 3, const_cast<char**>(in_args2), flags::ArgvListAction::kRemoveParsedArgs, + flags::UsageFlagsAction::kIgnoreUsage, + flags::OnUndefinedFlag::kAbortIfUndefined); + + EXPECT_EQ(flags::GetFlagsHelpMode(), flags::HelpMode::kImportant); + EXPECT_EQ(absl::GetFlag(FLAGS_int_flag), 3); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseDeathTest, TestSubstringHelpFlagHandling) { + const char* in_args1[] = { + "testbin", + "--help=abcd", + }; + + auto out_args1 = flags::ParseCommandLineImpl( + 2, const_cast<char**>(in_args1), flags::ArgvListAction::kRemoveParsedArgs, + flags::UsageFlagsAction::kIgnoreUsage, + flags::OnUndefinedFlag::kAbortIfUndefined); + + EXPECT_EQ(flags::GetFlagsHelpMode(), flags::HelpMode::kMatch); + EXPECT_EQ(flags::GetFlagsHelpMatchSubstr(), "abcd"); + + const char* in_args2[] = {"testbin", "--help", "some_positional_arg"}; + + auto out_args2 = flags::ParseCommandLineImpl( + 3, const_cast<char**>(in_args2), flags::ArgvListAction::kRemoveParsedArgs, + flags::UsageFlagsAction::kIgnoreUsage, + flags::OnUndefinedFlag::kAbortIfUndefined); + + EXPECT_EQ(flags::GetFlagsHelpMode(), flags::HelpMode::kImportant); +} + +// -------------------------------------------------------------------- + +TEST_F(ParseTest, WasPresentOnCommandLine) { + const char* in_args1[] = { + "testbin", "arg1", "--bool_flag", + "--int_flag=211", "arg2", "--double_flag=1.1", + "--string_flag", "asd", "--", + "--some_flag", "arg4", + }; + + InvokeParse(in_args1); + + EXPECT_TRUE(flags::WasPresentOnCommandLine("bool_flag")); + EXPECT_TRUE(flags::WasPresentOnCommandLine("int_flag")); + EXPECT_TRUE(flags::WasPresentOnCommandLine("double_flag")); + EXPECT_TRUE(flags::WasPresentOnCommandLine("string_flag")); + EXPECT_FALSE(flags::WasPresentOnCommandLine("some_flag")); + EXPECT_FALSE(flags::WasPresentOnCommandLine("another_flag")); +} + +// -------------------------------------------------------------------- + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/reflection.cc b/third_party/abseil_cpp/absl/flags/reflection.cc new file mode 100644 index 000000000000..c976d4679601 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/reflection.cc @@ -0,0 +1,337 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/reflection.h" + +#include <assert.h> + +#include <atomic> +#include <map> +#include <string> + +#include "absl/base/config.h" +#include "absl/base/thread_annotations.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/internal/private_handle_accessor.h" +#include "absl/flags/internal/registry.h" +#include "absl/flags/usage_config.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +// -------------------------------------------------------------------- +// FlagRegistry +// A FlagRegistry singleton object holds all flag objects indexed by their +// names so that if you know a flag's name, you can access or set it. If the +// function is named FooLocked(), you must own the registry lock before +// calling the function; otherwise, you should *not* hold the lock, and the +// function will acquire it itself if needed. +// -------------------------------------------------------------------- + +class FlagRegistry { + public: + FlagRegistry() = default; + ~FlagRegistry() = default; + + // Store a flag in this registry. Takes ownership of *flag. + void RegisterFlag(CommandLineFlag& flag); + + void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION(lock_) { lock_.Lock(); } + void Unlock() ABSL_UNLOCK_FUNCTION(lock_) { lock_.Unlock(); } + + // Returns the flag object for the specified name, or nullptr if not found. + // Will emit a warning if a 'retired' flag is specified. + CommandLineFlag* FindFlag(absl::string_view name); + + static FlagRegistry& GlobalRegistry(); // returns a singleton registry + + private: + friend class flags_internal::FlagSaverImpl; // reads all the flags in order + // to copy them + friend void ForEachFlag(std::function<void(CommandLineFlag&)> visitor); + friend void FinalizeRegistry(); + + // The map from name to flag, for FindFlag(). + using FlagMap = std::map<absl::string_view, CommandLineFlag*>; + using FlagIterator = FlagMap::iterator; + using FlagConstIterator = FlagMap::const_iterator; + FlagMap flags_; + std::vector<CommandLineFlag*> flat_flags_; + std::atomic<bool> finalized_flags_{false}; + + absl::Mutex lock_; + + // Disallow + FlagRegistry(const FlagRegistry&); + FlagRegistry& operator=(const FlagRegistry&); +}; + +namespace { + +class FlagRegistryLock { + public: + explicit FlagRegistryLock(FlagRegistry& fr) : fr_(fr) { fr_.Lock(); } + ~FlagRegistryLock() { fr_.Unlock(); } + + private: + FlagRegistry& fr_; +}; + +} // namespace + +CommandLineFlag* FlagRegistry::FindFlag(absl::string_view name) { + if (finalized_flags_.load(std::memory_order_acquire)) { + // We could save some gcus here if we make `Name()` be non-virtual. + // We could move the `const char*` name to the base class. + auto it = std::partition_point( + flat_flags_.begin(), flat_flags_.end(), + [=](CommandLineFlag* f) { return f->Name() < name; }); + if (it != flat_flags_.end() && (*it)->Name() == name) return *it; + } + + FlagRegistryLock frl(*this); + auto it = flags_.find(name); + return it != flags_.end() ? it->second : nullptr; +} + +void FlagRegistry::RegisterFlag(CommandLineFlag& flag) { + FlagRegistryLock registry_lock(*this); + + std::pair<FlagIterator, bool> ins = + flags_.insert(FlagMap::value_type(flag.Name(), &flag)); + if (ins.second == false) { // means the name was already in the map + CommandLineFlag& old_flag = *ins.first->second; + if (flag.IsRetired() != old_flag.IsRetired()) { + // All registrations must agree on the 'retired' flag. + flags_internal::ReportUsageError( + absl::StrCat( + "Retired flag '", flag.Name(), "' was defined normally in file '", + (flag.IsRetired() ? old_flag.Filename() : flag.Filename()), "'."), + true); + } else if (flags_internal::PrivateHandleAccessor::TypeId(flag) != + flags_internal::PrivateHandleAccessor::TypeId(old_flag)) { + flags_internal::ReportUsageError( + absl::StrCat("Flag '", flag.Name(), + "' was defined more than once but with " + "differing types. Defined in files '", + old_flag.Filename(), "' and '", flag.Filename(), "'."), + true); + } else if (old_flag.IsRetired()) { + return; + } else if (old_flag.Filename() != flag.Filename()) { + flags_internal::ReportUsageError( + absl::StrCat("Flag '", flag.Name(), + "' was defined more than once (in files '", + old_flag.Filename(), "' and '", flag.Filename(), "')."), + true); + } else { + flags_internal::ReportUsageError( + absl::StrCat( + "Something is wrong with flag '", flag.Name(), "' in file '", + flag.Filename(), "'. One possibility: file '", flag.Filename(), + "' is being linked both statically and dynamically into this " + "executable. e.g. some files listed as srcs to a test and also " + "listed as srcs of some shared lib deps of the same test."), + true); + } + // All cases above are fatal, except for the retired flags. + std::exit(1); + } +} + +FlagRegistry& FlagRegistry::GlobalRegistry() { + static FlagRegistry* global_registry = new FlagRegistry; + return *global_registry; +} + +// -------------------------------------------------------------------- + +void ForEachFlag(std::function<void(CommandLineFlag&)> visitor) { + FlagRegistry& registry = FlagRegistry::GlobalRegistry(); + + if (registry.finalized_flags_.load(std::memory_order_acquire)) { + for (const auto& i : registry.flat_flags_) visitor(*i); + } + + FlagRegistryLock frl(registry); + for (const auto& i : registry.flags_) visitor(*i.second); +} + +// -------------------------------------------------------------------- + +bool RegisterCommandLineFlag(CommandLineFlag& flag) { + FlagRegistry::GlobalRegistry().RegisterFlag(flag); + return true; +} + +void FinalizeRegistry() { + auto& registry = FlagRegistry::GlobalRegistry(); + FlagRegistryLock frl(registry); + if (registry.finalized_flags_.load(std::memory_order_relaxed)) { + // Was already finalized. Ignore the second time. + return; + } + registry.flat_flags_.reserve(registry.flags_.size()); + for (const auto& f : registry.flags_) { + registry.flat_flags_.push_back(f.second); + } + registry.flags_.clear(); + registry.finalized_flags_.store(true, std::memory_order_release); +} + +// -------------------------------------------------------------------- + +namespace { + +class RetiredFlagObj final : public CommandLineFlag { + public: + constexpr RetiredFlagObj(const char* name, FlagFastTypeId type_id) + : name_(name), type_id_(type_id) {} + + private: + absl::string_view Name() const override { return name_; } + std::string Filename() const override { + OnAccess(); + return "RETIRED"; + } + FlagFastTypeId TypeId() const override { return type_id_; } + std::string Help() const override { + OnAccess(); + return ""; + } + bool IsRetired() const override { return true; } + bool IsSpecifiedOnCommandLine() const override { + OnAccess(); + return false; + } + std::string DefaultValue() const override { + OnAccess(); + return ""; + } + std::string CurrentValue() const override { + OnAccess(); + return ""; + } + + // Any input is valid + bool ValidateInputValue(absl::string_view) const override { + OnAccess(); + return true; + } + + std::unique_ptr<flags_internal::FlagStateInterface> SaveState() override { + return nullptr; + } + + bool ParseFrom(absl::string_view, flags_internal::FlagSettingMode, + flags_internal::ValueSource, std::string&) override { + OnAccess(); + return false; + } + + void CheckDefaultValueParsingRoundtrip() const override { OnAccess(); } + + void Read(void*) const override { OnAccess(); } + + void OnAccess() const { + flags_internal::ReportUsageError( + absl::StrCat("Accessing retired flag '", name_, "'"), false); + } + + // Data members + const char* const name_; + const FlagFastTypeId type_id_; +}; + +} // namespace + +void Retire(const char* name, FlagFastTypeId type_id, char* buf) { + static_assert(sizeof(RetiredFlagObj) == kRetiredFlagObjSize, ""); + static_assert(alignof(RetiredFlagObj) == kRetiredFlagObjAlignment, ""); + auto* flag = ::new (static_cast<void*>(buf)) + flags_internal::RetiredFlagObj(name, type_id); + FlagRegistry::GlobalRegistry().RegisterFlag(*flag); +} + +// -------------------------------------------------------------------- + +class FlagSaverImpl { + public: + FlagSaverImpl() = default; + FlagSaverImpl(const FlagSaverImpl&) = delete; + void operator=(const FlagSaverImpl&) = delete; + + // Saves the flag states from the flag registry into this object. + // It's an error to call this more than once. + void SaveFromRegistry() { + assert(backup_registry_.empty()); // call only once! + flags_internal::ForEachFlag([&](CommandLineFlag& flag) { + if (auto flag_state = + flags_internal::PrivateHandleAccessor::SaveState(flag)) { + backup_registry_.emplace_back(std::move(flag_state)); + } + }); + } + + // Restores the saved flag states into the flag registry. + void RestoreToRegistry() { + for (const auto& flag_state : backup_registry_) { + flag_state->Restore(); + } + } + + private: + std::vector<std::unique_ptr<flags_internal::FlagStateInterface>> + backup_registry_; +}; + +} // namespace flags_internal + +FlagSaver::FlagSaver() : impl_(new flags_internal::FlagSaverImpl) { + impl_->SaveFromRegistry(); +} + +FlagSaver::~FlagSaver() { + if (!impl_) return; + + impl_->RestoreToRegistry(); + delete impl_; +} + +// -------------------------------------------------------------------- + +CommandLineFlag* FindCommandLineFlag(absl::string_view name) { + if (name.empty()) return nullptr; + flags_internal::FlagRegistry& registry = + flags_internal::FlagRegistry::GlobalRegistry(); + return registry.FindFlag(name); +} + +// -------------------------------------------------------------------- + +absl::flat_hash_map<absl::string_view, absl::CommandLineFlag*> GetAllFlags() { + absl::flat_hash_map<absl::string_view, absl::CommandLineFlag*> res; + flags_internal::ForEachFlag([&](CommandLineFlag& flag) { + if (!flag.IsRetired()) res.insert({flag.Name(), &flag}); + }); + return res; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/reflection.h b/third_party/abseil_cpp/absl/flags/reflection.h new file mode 100644 index 000000000000..e6baf5de4b0c --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/reflection.h @@ -0,0 +1,90 @@ +// +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: reflection.h +// ----------------------------------------------------------------------------- +// +// This file defines the routines to access and operate on an Abseil Flag's +// reflection handle. + +#ifndef ABSL_FLAGS_REFLECTION_H_ +#define ABSL_FLAGS_REFLECTION_H_ + +#include <string> + +#include "absl/base/config.h" +#include "absl/container/flat_hash_map.h" +#include "absl/flags/commandlineflag.h" +#include "absl/flags/internal/commandlineflag.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { +class FlagSaverImpl; +} // namespace flags_internal + +// FindCommandLineFlag() +// +// Returns the reflection handle of an Abseil flag of the specified name, or +// `nullptr` if not found. This function will emit a warning if the name of a +// 'retired' flag is specified. +absl::CommandLineFlag* FindCommandLineFlag(absl::string_view name); + +// Returns current state of the Flags registry in a form of mapping from flag +// name to a flag reflection handle. +absl::flat_hash_map<absl::string_view, absl::CommandLineFlag*> GetAllFlags(); + +//------------------------------------------------------------------------------ +// FlagSaver +//------------------------------------------------------------------------------ +// +// A FlagSaver object stores the state of flags in the scope where the FlagSaver +// is defined, allowing modification of those flags within that scope and +// automatic restoration of the flags to their previous state upon leaving the +// scope. +// +// A FlagSaver can be used within tests to temporarily change the test +// environment and restore the test case to its previous state. +// +// Example: +// +// void MyFunc() { +// absl::FlagSaver fs; +// ... +// absl::SetFlag(&FLAGS_myFlag, otherValue); +// ... +// } // scope of FlagSaver left, flags return to previous state +// +// This class is thread-safe. + +class FlagSaver { + public: + FlagSaver(); + ~FlagSaver(); + + FlagSaver(const FlagSaver&) = delete; + void operator=(const FlagSaver&) = delete; + + private: + flags_internal::FlagSaverImpl* impl_; +}; + +//----------------------------------------------------------------------------- + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_REFLECTION_H_ diff --git a/third_party/abseil_cpp/absl/flags/reflection_test.cc b/third_party/abseil_cpp/absl/flags/reflection_test.cc new file mode 100644 index 000000000000..4c80900956a9 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/reflection_test.cc @@ -0,0 +1,267 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/reflection.h" + +#include <memory> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/flags/declare.h" +#include "absl/flags/flag.h" +#include "absl/flags/internal/commandlineflag.h" +#include "absl/flags/marshalling.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" + +ABSL_FLAG(int, int_flag, 1, "int_flag help"); +ABSL_FLAG(std::string, string_flag, "dflt", "string_flag help"); +ABSL_RETIRED_FLAG(bool, bool_retired_flag, false, "bool_retired_flag help"); + +namespace { + +namespace flags = absl::flags_internal; + +class ReflectionTest : public testing::Test { + protected: + void SetUp() override { flag_saver_ = absl::make_unique<absl::FlagSaver>(); } + void TearDown() override { flag_saver_.reset(); } + + private: + std::unique_ptr<absl::FlagSaver> flag_saver_; +}; + +// -------------------------------------------------------------------- + +TEST_F(ReflectionTest, TestFindCommandLineFlag) { + auto* handle = absl::FindCommandLineFlag("some_flag"); + EXPECT_EQ(handle, nullptr); + + handle = absl::FindCommandLineFlag("int_flag"); + EXPECT_NE(handle, nullptr); + + handle = absl::FindCommandLineFlag("string_flag"); + EXPECT_NE(handle, nullptr); + + handle = absl::FindCommandLineFlag("bool_retired_flag"); + EXPECT_NE(handle, nullptr); +} + +// -------------------------------------------------------------------- + +TEST_F(ReflectionTest, TestGetAllFlags) { + auto all_flags = absl::GetAllFlags(); + EXPECT_NE(all_flags.find("int_flag"), all_flags.end()); + EXPECT_EQ(all_flags.find("bool_retired_flag"), all_flags.end()); + EXPECT_EQ(all_flags.find("some_undefined_flag"), all_flags.end()); + + std::vector<absl::string_view> flag_names_first_attempt; + auto all_flags_1 = absl::GetAllFlags(); + for (auto f : all_flags_1) { + flag_names_first_attempt.push_back(f.first); + } + + std::vector<absl::string_view> flag_names_second_attempt; + auto all_flags_2 = absl::GetAllFlags(); + for (auto f : all_flags_2) { + flag_names_second_attempt.push_back(f.first); + } + + EXPECT_THAT(flag_names_first_attempt, + ::testing::UnorderedElementsAreArray(flag_names_second_attempt)); +} + +// -------------------------------------------------------------------- + +struct CustomUDT { + CustomUDT() : a(1), b(1) {} + CustomUDT(int a_, int b_) : a(a_), b(b_) {} + + friend bool operator==(const CustomUDT& f1, const CustomUDT& f2) { + return f1.a == f2.a && f1.b == f2.b; + } + + int a; + int b; +}; +bool AbslParseFlag(absl::string_view in, CustomUDT* f, std::string*) { + std::vector<absl::string_view> parts = + absl::StrSplit(in, ':', absl::SkipWhitespace()); + + if (parts.size() != 2) return false; + + if (!absl::SimpleAtoi(parts[0], &f->a)) return false; + + if (!absl::SimpleAtoi(parts[1], &f->b)) return false; + + return true; +} +std::string AbslUnparseFlag(const CustomUDT& f) { + return absl::StrCat(f.a, ":", f.b); +} + +} // namespace + +// -------------------------------------------------------------------- + +ABSL_FLAG(bool, test_flag_01, true, ""); +ABSL_FLAG(int, test_flag_02, 1234, ""); +ABSL_FLAG(int16_t, test_flag_03, -34, ""); +ABSL_FLAG(uint16_t, test_flag_04, 189, ""); +ABSL_FLAG(int32_t, test_flag_05, 10765, ""); +ABSL_FLAG(uint32_t, test_flag_06, 40000, ""); +ABSL_FLAG(int64_t, test_flag_07, -1234567, ""); +ABSL_FLAG(uint64_t, test_flag_08, 9876543, ""); +ABSL_FLAG(double, test_flag_09, -9.876e-50, ""); +ABSL_FLAG(float, test_flag_10, 1.234e12f, ""); +ABSL_FLAG(std::string, test_flag_11, "", ""); +ABSL_FLAG(absl::Duration, test_flag_12, absl::Minutes(10), ""); +static int counter = 0; +ABSL_FLAG(int, test_flag_13, 200, "").OnUpdate([]() { counter++; }); +ABSL_FLAG(CustomUDT, test_flag_14, {}, ""); + +namespace { + +TEST_F(ReflectionTest, TestFlagSaverInScope) { + { + absl::FlagSaver s; + counter = 0; + absl::SetFlag(&FLAGS_test_flag_01, false); + absl::SetFlag(&FLAGS_test_flag_02, -1021); + absl::SetFlag(&FLAGS_test_flag_03, 6009); + absl::SetFlag(&FLAGS_test_flag_04, 44); + absl::SetFlag(&FLAGS_test_flag_05, +800); + absl::SetFlag(&FLAGS_test_flag_06, -40978756); + absl::SetFlag(&FLAGS_test_flag_07, 23405); + absl::SetFlag(&FLAGS_test_flag_08, 975310); + absl::SetFlag(&FLAGS_test_flag_09, 1.00001); + absl::SetFlag(&FLAGS_test_flag_10, -3.54f); + absl::SetFlag(&FLAGS_test_flag_11, "asdf"); + absl::SetFlag(&FLAGS_test_flag_12, absl::Hours(20)); + absl::SetFlag(&FLAGS_test_flag_13, 4); + absl::SetFlag(&FLAGS_test_flag_14, CustomUDT{-1, -2}); + } + + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_01), true); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_02), 1234); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_03), -34); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_04), 189); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_05), 10765); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_06), 40000); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_07), -1234567); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 9876543); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_09), -9.876e-50, 1e-55); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_10), 1.234e12f, 1e5f); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_11), ""); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_12), absl::Minutes(10)); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_13), 200); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_14), CustomUDT{}); + EXPECT_EQ(counter, 2); +} + +// -------------------------------------------------------------------- + +TEST_F(ReflectionTest, TestFlagSaverVsUpdateViaReflection) { + { + absl::FlagSaver s; + counter = 0; + std::string error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_01")->ParseFrom("false", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_02")->ParseFrom("-4536", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_03")->ParseFrom("111", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_04")->ParseFrom("909", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_05")->ParseFrom("-2004", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_06")->ParseFrom("1000023", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_07")->ParseFrom("69305", &error)) + << error; + EXPECT_TRUE(absl::FindCommandLineFlag("test_flag_08") + ->ParseFrom("1000000001", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_09")->ParseFrom("2.09021", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_10")->ParseFrom("-33.1", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_11")->ParseFrom("ADD_FOO", &error)) + << error; + EXPECT_TRUE(absl::FindCommandLineFlag("test_flag_12") + ->ParseFrom("3h11m16s", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_13")->ParseFrom("0", &error)) + << error; + EXPECT_TRUE( + absl::FindCommandLineFlag("test_flag_14")->ParseFrom("10:1", &error)) + << error; + } + + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_01), true); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_02), 1234); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_03), -34); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_04), 189); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_05), 10765); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_06), 40000); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_07), -1234567); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 9876543); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_09), -9.876e-50, 1e-55); + EXPECT_NEAR(absl::GetFlag(FLAGS_test_flag_10), 1.234e12f, 1e5f); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_11), ""); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_12), absl::Minutes(10)); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_13), 200); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_14), CustomUDT{}); + EXPECT_EQ(counter, 2); +} + +// -------------------------------------------------------------------- + +TEST_F(ReflectionTest, TestMultipleFlagSaversInEnclosedScopes) { + { + absl::FlagSaver s; + absl::SetFlag(&FLAGS_test_flag_08, 10); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 10); + { + absl::FlagSaver s; + absl::SetFlag(&FLAGS_test_flag_08, 20); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 20); + { + absl::FlagSaver s; + absl::SetFlag(&FLAGS_test_flag_08, -200); + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), -200); + } + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 20); + } + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 10); + } + EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_08), 9876543); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/flags/usage.cc b/third_party/abseil_cpp/absl/flags/usage.cc new file mode 100644 index 000000000000..452f667512e8 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/usage.cc @@ -0,0 +1,65 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/flags/usage.h" + +#include <stdlib.h> + +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/const_init.h" +#include "absl/base/thread_annotations.h" +#include "absl/flags/internal/usage.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { +namespace { +ABSL_CONST_INIT absl::Mutex usage_message_guard(absl::kConstInit); +ABSL_CONST_INIT std::string* program_usage_message + ABSL_GUARDED_BY(usage_message_guard) = nullptr; +} // namespace +} // namespace flags_internal + +// -------------------------------------------------------------------- +// Sets the "usage" message to be used by help reporting routines. +void SetProgramUsageMessage(absl::string_view new_usage_message) { + absl::MutexLock l(&flags_internal::usage_message_guard); + + if (flags_internal::program_usage_message != nullptr) { + ABSL_INTERNAL_LOG(FATAL, "SetProgramUsageMessage() called twice."); + std::exit(1); + } + + flags_internal::program_usage_message = new std::string(new_usage_message); +} + +// -------------------------------------------------------------------- +// Returns the usage message set by SetProgramUsageMessage(). +// Note: We able to return string_view here only because calling +// SetProgramUsageMessage twice is prohibited. +absl::string_view ProgramUsageMessage() { + absl::MutexLock l(&flags_internal::usage_message_guard); + + return flags_internal::program_usage_message != nullptr + ? absl::string_view(*flags_internal::program_usage_message) + : "Warning: SetProgramUsageMessage() never called"; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/usage.h b/third_party/abseil_cpp/absl/flags/usage.h new file mode 100644 index 000000000000..ad12ab7ad902 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/usage.h @@ -0,0 +1,43 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FLAGS_USAGE_H_ +#define ABSL_FLAGS_USAGE_H_ + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +// -------------------------------------------------------------------- +// Usage reporting interfaces + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Sets the "usage" message to be used by help reporting routines. +// For example: +// absl::SetProgramUsageMessage( +// absl::StrCat("This program does nothing. Sample usage:\n", argv[0], +// " <uselessarg1> <uselessarg2>")); +// Do not include commandline flags in the usage: we do that for you! +// Note: Calling SetProgramUsageMessage twice will trigger a call to std::exit. +void SetProgramUsageMessage(absl::string_view new_usage_message); + +// Returns the usage message set by SetProgramUsageMessage(). +absl::string_view ProgramUsageMessage(); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FLAGS_USAGE_H_ diff --git a/third_party/abseil_cpp/absl/flags/usage_config.cc b/third_party/abseil_cpp/absl/flags/usage_config.cc new file mode 100644 index 000000000000..ae2f548a570b --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/usage_config.cc @@ -0,0 +1,164 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/usage_config.h" + +#include <functional> +#include <iostream> +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/const_init.h" +#include "absl/base/thread_annotations.h" +#include "absl/flags/internal/path_util.h" +#include "absl/flags/internal/program_name.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/synchronization/mutex.h" + +extern "C" { + +// Additional report of fatal usage error message before we std::exit. Error is +// fatal if is_fatal argument to ReportUsageError is true. +ABSL_ATTRIBUTE_WEAK void AbslInternalReportFatalUsageError(absl::string_view) {} + +} // extern "C" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace flags_internal { + +namespace { + +// -------------------------------------------------------------------- +// Returns true if flags defined in the filename should be reported with +// -helpshort flag. + +bool ContainsHelpshortFlags(absl::string_view filename) { + // By default we only want flags in binary's main. We expect the main + // routine to reside in <program>.cc or <program>-main.cc or + // <program>_main.cc, where the <program> is the name of the binary + // (without .exe on Windows). + auto suffix = flags_internal::Basename(filename); + auto program_name = flags_internal::ShortProgramInvocationName(); + absl::string_view program_name_ref = program_name; +#if defined(_WIN32) + absl::ConsumeSuffix(&program_name_ref, ".exe"); +#endif + if (!absl::ConsumePrefix(&suffix, program_name_ref)) + return false; + return absl::StartsWith(suffix, ".") || absl::StartsWith(suffix, "-main.") || + absl::StartsWith(suffix, "_main."); +} + +// -------------------------------------------------------------------- +// Returns true if flags defined in the filename should be reported with +// -helppackage flag. + +bool ContainsHelppackageFlags(absl::string_view filename) { + // TODO(rogeeff): implement properly when registry is available. + return ContainsHelpshortFlags(filename); +} + +// -------------------------------------------------------------------- +// Generates program version information into supplied output. + +std::string VersionString() { + std::string version_str(flags_internal::ShortProgramInvocationName()); + + version_str += "\n"; + +#if !defined(NDEBUG) + version_str += "Debug build (NDEBUG not #defined)\n"; +#endif + + return version_str; +} + +// -------------------------------------------------------------------- +// Normalizes the filename specific to the build system/filesystem used. + +std::string NormalizeFilename(absl::string_view filename) { + // Skip any leading slashes + auto pos = filename.find_first_not_of("\\/"); + if (pos == absl::string_view::npos) return ""; + + filename.remove_prefix(pos); + return std::string(filename); +} + +// -------------------------------------------------------------------- + +ABSL_CONST_INIT absl::Mutex custom_usage_config_guard(absl::kConstInit); +ABSL_CONST_INIT FlagsUsageConfig* custom_usage_config + ABSL_GUARDED_BY(custom_usage_config_guard) = nullptr; + +} // namespace + +FlagsUsageConfig GetUsageConfig() { + absl::MutexLock l(&custom_usage_config_guard); + + if (custom_usage_config) return *custom_usage_config; + + FlagsUsageConfig default_config; + default_config.contains_helpshort_flags = &ContainsHelpshortFlags; + default_config.contains_help_flags = &ContainsHelppackageFlags; + default_config.contains_helppackage_flags = &ContainsHelppackageFlags; + default_config.version_string = &VersionString; + default_config.normalize_filename = &NormalizeFilename; + + return default_config; +} + +void ReportUsageError(absl::string_view msg, bool is_fatal) { + std::cerr << "ERROR: " << msg << std::endl; + + if (is_fatal) { + AbslInternalReportFatalUsageError(msg); + } +} + +} // namespace flags_internal + +void SetFlagsUsageConfig(FlagsUsageConfig usage_config) { + absl::MutexLock l(&flags_internal::custom_usage_config_guard); + + if (!usage_config.contains_helpshort_flags) + usage_config.contains_helpshort_flags = + flags_internal::ContainsHelpshortFlags; + + if (!usage_config.contains_help_flags) + usage_config.contains_help_flags = flags_internal::ContainsHelppackageFlags; + + if (!usage_config.contains_helppackage_flags) + usage_config.contains_helppackage_flags = + flags_internal::ContainsHelppackageFlags; + + if (!usage_config.version_string) + usage_config.version_string = flags_internal::VersionString; + + if (!usage_config.normalize_filename) + usage_config.normalize_filename = flags_internal::NormalizeFilename; + + if (flags_internal::custom_usage_config) + *flags_internal::custom_usage_config = usage_config; + else + flags_internal::custom_usage_config = new FlagsUsageConfig(usage_config); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/flags/usage_config.h b/third_party/abseil_cpp/absl/flags/usage_config.h new file mode 100644 index 000000000000..96eecea23159 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/usage_config.h @@ -0,0 +1,134 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: usage_config.h +// ----------------------------------------------------------------------------- +// +// This file defines the main usage reporting configuration interfaces and +// documents Abseil's supported built-in usage flags. If these flags are found +// when parsing a command-line, Abseil will exit the program and display +// appropriate help messages. +#ifndef ABSL_FLAGS_USAGE_CONFIG_H_ +#define ABSL_FLAGS_USAGE_CONFIG_H_ + +#include <functional> +#include <string> + +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +// ----------------------------------------------------------------------------- +// Built-in Usage Flags +// ----------------------------------------------------------------------------- +// +// Abseil supports the following built-in usage flags. When passed, these flags +// exit the program and : +// +// * --help +// Shows help on important flags for this binary +// * --helpfull +// Shows help on all flags +// * --helpshort +// Shows help on only the main module for this program +// * --helppackage +// Shows help on all modules in the main package +// * --version +// Shows the version and build info for this binary and exits +// * --only_check_args +// Exits after checking all flags +// * --helpon +// Shows help on the modules named by this flag value +// * --helpmatch +// Shows help on modules whose name contains the specified substring + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace flags_internal { +using FlagKindFilter = std::function<bool (absl::string_view)>; +} // namespace flags_internal + +// FlagsUsageConfig +// +// This structure contains the collection of callbacks for changing the behavior +// of the usage reporting routines in Abseil Flags. +struct FlagsUsageConfig { + // Returns true if flags defined in the given source code file should be + // reported with --helpshort flag. For example, if the file + // "path/to/my/code.cc" defines the flag "--my_flag", and + // contains_helpshort_flags("path/to/my/code.cc") returns true, invoking the + // program with --helpshort will include information about --my_flag in the + // program output. + flags_internal::FlagKindFilter contains_helpshort_flags; + + // Returns true if flags defined in the filename should be reported with + // --help flag. For example, if the file + // "path/to/my/code.cc" defines the flag "--my_flag", and + // contains_help_flags("path/to/my/code.cc") returns true, invoking the + // program with --help will include information about --my_flag in the + // program output. + flags_internal::FlagKindFilter contains_help_flags; + + // Returns true if flags defined in the filename should be reported with + // --helppackage flag. For example, if the file + // "path/to/my/code.cc" defines the flag "--my_flag", and + // contains_helppackage_flags("path/to/my/code.cc") returns true, invoking the + // program with --helppackage will include information about --my_flag in the + // program output. + flags_internal::FlagKindFilter contains_helppackage_flags; + + // Generates string containing program version. This is the string reported + // when user specifies --version in a command line. + std::function<std::string()> version_string; + + // Normalizes the filename specific to the build system/filesystem used. This + // routine is used when we report the information about the flag definition + // location. For instance, if your build resides at some location you do not + // want to expose in the usage output, you can trim it to show only relevant + // part. + // For example: + // normalize_filename("/my_company/some_long_path/src/project/file.cc") + // might produce + // "project/file.cc". + std::function<std::string(absl::string_view)> normalize_filename; +}; + +// SetFlagsUsageConfig() +// +// Sets the usage reporting configuration callbacks. If any of the callbacks are +// not set in usage_config instance, then the default value of the callback is +// used. +void SetFlagsUsageConfig(FlagsUsageConfig usage_config); + +namespace flags_internal { + +FlagsUsageConfig GetUsageConfig(); + +void ReportUsageError(absl::string_view msg, bool is_fatal); + +} // namespace flags_internal +ABSL_NAMESPACE_END +} // namespace absl + +extern "C" { + +// Additional report of fatal usage error message before we std::exit. Error is +// fatal if is_fatal argument to ReportUsageError is true. +void AbslInternalReportFatalUsageError(absl::string_view); + +} // extern "C" + +#endif // ABSL_FLAGS_USAGE_CONFIG_H_ diff --git a/third_party/abseil_cpp/absl/flags/usage_config_test.cc b/third_party/abseil_cpp/absl/flags/usage_config_test.cc new file mode 100644 index 000000000000..e57a8832f645 --- /dev/null +++ b/third_party/abseil_cpp/absl/flags/usage_config_test.cc @@ -0,0 +1,205 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/flags/usage_config.h" + +#include <string> + +#include "gtest/gtest.h" +#include "absl/flags/internal/path_util.h" +#include "absl/flags/internal/program_name.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" + +namespace { + +class FlagsUsageConfigTest : public testing::Test { + protected: + void SetUp() override { + // Install Default config for the use on this unit test. + // Binary may install a custom config before tests are run. + absl::FlagsUsageConfig default_config; + absl::SetFlagsUsageConfig(default_config); + } +}; + +namespace flags = absl::flags_internal; + +bool TstContainsHelpshortFlags(absl::string_view f) { + return absl::StartsWith(flags::Basename(f), "progname."); +} + +bool TstContainsHelppackageFlags(absl::string_view f) { + return absl::EndsWith(flags::Package(f), "aaa/"); +} + +bool TstContainsHelpFlags(absl::string_view f) { + return absl::EndsWith(flags::Package(f), "zzz/"); +} + +std::string TstVersionString() { return "program 1.0.0"; } + +std::string TstNormalizeFilename(absl::string_view filename) { + return std::string(filename.substr(2)); +} + +void TstReportUsageMessage(absl::string_view msg) {} + +// -------------------------------------------------------------------- + +TEST_F(FlagsUsageConfigTest, TestGetSetFlagsUsageConfig) { + EXPECT_TRUE(flags::GetUsageConfig().contains_helpshort_flags); + EXPECT_TRUE(flags::GetUsageConfig().contains_help_flags); + EXPECT_TRUE(flags::GetUsageConfig().contains_helppackage_flags); + EXPECT_TRUE(flags::GetUsageConfig().version_string); + EXPECT_TRUE(flags::GetUsageConfig().normalize_filename); + + absl::FlagsUsageConfig empty_config; + empty_config.contains_helpshort_flags = &TstContainsHelpshortFlags; + empty_config.contains_help_flags = &TstContainsHelpFlags; + empty_config.contains_helppackage_flags = &TstContainsHelppackageFlags; + empty_config.version_string = &TstVersionString; + empty_config.normalize_filename = &TstNormalizeFilename; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_TRUE(flags::GetUsageConfig().contains_helpshort_flags); + EXPECT_TRUE(flags::GetUsageConfig().contains_help_flags); + EXPECT_TRUE(flags::GetUsageConfig().contains_helppackage_flags); + EXPECT_TRUE(flags::GetUsageConfig().version_string); + EXPECT_TRUE(flags::GetUsageConfig().normalize_filename); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagsUsageConfigTest, TestContainsHelpshortFlags) { +#if defined(_WIN32) + flags::SetProgramInvocationName("usage_config_test.exe"); +#else + flags::SetProgramInvocationName("usage_config_test"); +#endif + + auto config = flags::GetUsageConfig(); + EXPECT_TRUE(config.contains_helpshort_flags("adir/cd/usage_config_test.cc")); + EXPECT_TRUE( + config.contains_helpshort_flags("aaaa/usage_config_test-main.cc")); + EXPECT_TRUE(config.contains_helpshort_flags("abc/usage_config_test_main.cc")); + EXPECT_FALSE(config.contains_helpshort_flags("usage_config_main.cc")); + + absl::FlagsUsageConfig empty_config; + empty_config.contains_helpshort_flags = &TstContainsHelpshortFlags; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_TRUE( + flags::GetUsageConfig().contains_helpshort_flags("aaa/progname.cpp")); + EXPECT_FALSE( + flags::GetUsageConfig().contains_helpshort_flags("aaa/progmane.cpp")); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagsUsageConfigTest, TestContainsHelpFlags) { + flags::SetProgramInvocationName("usage_config_test"); + + auto config = flags::GetUsageConfig(); + EXPECT_TRUE(config.contains_help_flags("zzz/usage_config_test.cc")); + EXPECT_TRUE( + config.contains_help_flags("bdir/a/zzz/usage_config_test-main.cc")); + EXPECT_TRUE( + config.contains_help_flags("//aqse/zzz/usage_config_test_main.cc")); + EXPECT_FALSE(config.contains_help_flags("zzz/aa/usage_config_main.cc")); + + absl::FlagsUsageConfig empty_config; + empty_config.contains_help_flags = &TstContainsHelpFlags; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_TRUE(flags::GetUsageConfig().contains_help_flags("zzz/main-body.c")); + EXPECT_FALSE( + flags::GetUsageConfig().contains_help_flags("zzz/dir/main-body.c")); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagsUsageConfigTest, TestContainsHelppackageFlags) { + flags::SetProgramInvocationName("usage_config_test"); + + auto config = flags::GetUsageConfig(); + EXPECT_TRUE(config.contains_helppackage_flags("aaa/usage_config_test.cc")); + EXPECT_TRUE( + config.contains_helppackage_flags("bbdir/aaa/usage_config_test-main.cc")); + EXPECT_TRUE(config.contains_helppackage_flags( + "//aqswde/aaa/usage_config_test_main.cc")); + EXPECT_FALSE(config.contains_helppackage_flags("aadir/usage_config_main.cc")); + + absl::FlagsUsageConfig empty_config; + empty_config.contains_helppackage_flags = &TstContainsHelppackageFlags; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_TRUE( + flags::GetUsageConfig().contains_helppackage_flags("aaa/main-body.c")); + EXPECT_FALSE( + flags::GetUsageConfig().contains_helppackage_flags("aadir/main-body.c")); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagsUsageConfigTest, TestVersionString) { + flags::SetProgramInvocationName("usage_config_test"); + +#ifdef NDEBUG + std::string expected_output = "usage_config_test\n"; +#else + std::string expected_output = + "usage_config_test\nDebug build (NDEBUG not #defined)\n"; +#endif + + EXPECT_EQ(flags::GetUsageConfig().version_string(), expected_output); + + absl::FlagsUsageConfig empty_config; + empty_config.version_string = &TstVersionString; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_EQ(flags::GetUsageConfig().version_string(), "program 1.0.0"); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagsUsageConfigTest, TestNormalizeFilename) { + // This tests the default implementation. + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("a/a.cc"), "a/a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("/a/a.cc"), "a/a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("///a/a.cc"), "a/a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("/"), ""); + + // This tests that the custom implementation is called. + absl::FlagsUsageConfig empty_config; + empty_config.normalize_filename = &TstNormalizeFilename; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("a/a.cc"), "a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("aaa/a.cc"), "a/a.cc"); + + // This tests that the default implementation is called. + empty_config.normalize_filename = nullptr; + absl::SetFlagsUsageConfig(empty_config); + + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("a/a.cc"), "a/a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("/a/a.cc"), "a/a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("///a/a.cc"), "a/a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("\\a\\a.cc"), "a\\a.cc"); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("//"), ""); + EXPECT_EQ(flags::GetUsageConfig().normalize_filename("\\\\"), ""); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/functional/BUILD.bazel b/third_party/abseil_cpp/absl/functional/BUILD.bazel new file mode 100644 index 000000000000..ebd9b99b782f --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/BUILD.bazel @@ -0,0 +1,93 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "bind_front", + srcs = ["internal/front_binder.h"], + hdrs = ["bind_front.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:base_internal", + "//absl/container:compressed_tuple", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_test( + name = "bind_front_test", + srcs = ["bind_front_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bind_front", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "function_ref", + srcs = ["internal/function_ref.h"], + hdrs = ["function_ref.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:base_internal", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "function_ref_test", + size = "small", + srcs = ["function_ref_test.cc"], + copts = ABSL_TEST_COPTS, + deps = [ + ":function_ref", + "//absl/container:test_instance_tracker", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "function_ref_benchmark", + srcs = [ + "function_ref_benchmark.cc", + ], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":function_ref", + "//absl/base:core_headers", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/third_party/abseil_cpp/absl/functional/CMakeLists.txt b/third_party/abseil_cpp/absl/functional/CMakeLists.txt new file mode 100644 index 000000000000..cda914f2cd53 --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/CMakeLists.txt @@ -0,0 +1,72 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + bind_front + SRCS + "internal/front_binder.h" + HDRS + "bind_front.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::compressed_tuple + PUBLIC +) + +absl_cc_test( + NAME + bind_front_test + SRCS + "bind_front_test.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bind_front + absl::memory + gmock_main +) + +absl_cc_library( + NAME + function_ref + SRCS + "internal/function_ref.h" + HDRS + "function_ref.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::meta + PUBLIC +) + +absl_cc_test( + NAME + function_ref_test + SRCS + "function_ref_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::function_ref + absl::memory + absl::test_instance_tracker + gmock_main +) diff --git a/third_party/abseil_cpp/absl/functional/bind_front.h b/third_party/abseil_cpp/absl/functional/bind_front.h new file mode 100644 index 000000000000..5b47970e3577 --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/bind_front.h @@ -0,0 +1,184 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: bind_front.h +// ----------------------------------------------------------------------------- +// +// `absl::bind_front()` returns a functor by binding a number of arguments to +// the front of a provided (usually more generic) functor. Unlike `std::bind`, +// it does not require the use of argument placeholders. The simpler syntax of +// `absl::bind_front()` allows you to avoid known misuses with `std::bind()`. +// +// `absl::bind_front()` is meant as a drop-in replacement for C++20's upcoming +// `std::bind_front()`, which similarly resolves these issues with +// `std::bind()`. Both `bind_front()` alternatives, unlike `std::bind()`, allow +// partial function application. (See +// https://en.wikipedia.org/wiki/Partial_application). + +#ifndef ABSL_FUNCTIONAL_BIND_FRONT_H_ +#define ABSL_FUNCTIONAL_BIND_FRONT_H_ + +#include "absl/functional/internal/front_binder.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// bind_front() +// +// Binds the first N arguments of an invocable object and stores them by value. +// +// Like `std::bind()`, `absl::bind_front()` is implicitly convertible to +// `std::function`. In particular, it may be used as a simpler replacement for +// `std::bind()` in most cases, as it does not require placeholders to be +// specified. More importantly, it provides more reliable correctness guarantees +// than `std::bind()`; while `std::bind()` will silently ignore passing more +// parameters than expected, for example, `absl::bind_front()` will report such +// mis-uses as errors. +// +// absl::bind_front(a...) can be seen as storing the results of +// std::make_tuple(a...). +// +// Example: Binding a free function. +// +// int Minus(int a, int b) { return a - b; } +// +// assert(absl::bind_front(Minus)(3, 2) == 3 - 2); +// assert(absl::bind_front(Minus, 3)(2) == 3 - 2); +// assert(absl::bind_front(Minus, 3, 2)() == 3 - 2); +// +// Example: Binding a member function. +// +// struct Math { +// int Double(int a) const { return 2 * a; } +// }; +// +// Math math; +// +// assert(absl::bind_front(&Math::Double)(&math, 3) == 2 * 3); +// // Stores a pointer to math inside the functor. +// assert(absl::bind_front(&Math::Double, &math)(3) == 2 * 3); +// // Stores a copy of math inside the functor. +// assert(absl::bind_front(&Math::Double, math)(3) == 2 * 3); +// // Stores std::unique_ptr<Math> inside the functor. +// assert(absl::bind_front(&Math::Double, +// std::unique_ptr<Math>(new Math))(3) == 2 * 3); +// +// Example: Using `absl::bind_front()`, instead of `std::bind()`, with +// `std::function`. +// +// class FileReader { +// public: +// void ReadFileAsync(const std::string& filename, std::string* content, +// const std::function<void()>& done) { +// // Calls Executor::Schedule(std::function<void()>). +// Executor::DefaultExecutor()->Schedule( +// absl::bind_front(&FileReader::BlockingRead, this, +// filename, content, done)); +// } +// +// private: +// void BlockingRead(const std::string& filename, std::string* content, +// const std::function<void()>& done) { +// CHECK_OK(file::GetContents(filename, content, {})); +// done(); +// } +// }; +// +// `absl::bind_front()` stores bound arguments explicitly using the type passed +// rather than implicitly based on the type accepted by its functor. +// +// Example: Binding arguments explicitly. +// +// void LogStringView(absl::string_view sv) { +// LOG(INFO) << sv; +// } +// +// Executor* e = Executor::DefaultExecutor(); +// std::string s = "hello"; +// absl::string_view sv = s; +// +// // absl::bind_front(LogStringView, arg) makes a copy of arg and stores it. +// e->Schedule(absl::bind_front(LogStringView, sv)); // ERROR: dangling +// // string_view. +// +// e->Schedule(absl::bind_front(LogStringView, s)); // OK: stores a copy of +// // s. +// +// To store some of the arguments passed to `absl::bind_front()` by reference, +// use std::ref()` and `std::cref()`. +// +// Example: Storing some of the bound arguments by reference. +// +// class Service { +// public: +// void Serve(const Request& req, std::function<void()>* done) { +// // The request protocol buffer won't be deleted until done is called. +// // It's safe to store a reference to it inside the functor. +// Executor::DefaultExecutor()->Schedule( +// absl::bind_front(&Service::BlockingServe, this, std::cref(req), +// done)); +// } +// +// private: +// void BlockingServe(const Request& req, std::function<void()>* done); +// }; +// +// Example: Storing bound arguments by reference. +// +// void Print(const std::string& a, const std::string& b) { +// std::cerr << a << b; +// } +// +// std::string hi = "Hello, "; +// std::vector<std::string> names = {"Chuk", "Gek"}; +// // Doesn't copy hi. +// for_each(names.begin(), names.end(), +// absl::bind_front(Print, std::ref(hi))); +// +// // DO NOT DO THIS: the functor may outlive "hi", resulting in +// // dangling references. +// foo->DoInFuture(absl::bind_front(Print, std::ref(hi), "Guest")); // BAD! +// auto f = absl::bind_front(Print, std::ref(hi), "Guest"); // BAD! +// +// Example: Storing reference-like types. +// +// void Print(absl::string_view a, const std::string& b) { +// std::cerr << a << b; +// } +// +// std::string hi = "Hello, "; +// // Copies "hi". +// absl::bind_front(Print, hi)("Chuk"); +// +// // Compile error: std::reference_wrapper<const string> is not implicitly +// // convertible to string_view. +// // absl::bind_front(Print, std::cref(hi))("Chuk"); +// +// // Doesn't copy "hi". +// absl::bind_front(Print, absl::string_view(hi))("Chuk"); +// +template <class F, class... BoundArgs> +constexpr functional_internal::bind_front_t<F, BoundArgs...> bind_front( + F&& func, BoundArgs&&... args) { + return functional_internal::bind_front_t<F, BoundArgs...>( + absl::in_place, absl::forward<F>(func), + absl::forward<BoundArgs>(args)...); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FUNCTIONAL_BIND_FRONT_H_ diff --git a/third_party/abseil_cpp/absl/functional/bind_front_test.cc b/third_party/abseil_cpp/absl/functional/bind_front_test.cc new file mode 100644 index 000000000000..4801a81caf0d --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/bind_front_test.cc @@ -0,0 +1,231 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/functional/bind_front.h" + +#include <stddef.h> + +#include <functional> +#include <memory> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" + +namespace { + +char CharAt(const char* s, size_t index) { return s[index]; } + +TEST(BindTest, Basics) { + EXPECT_EQ('C', absl::bind_front(CharAt)("ABC", 2)); + EXPECT_EQ('C', absl::bind_front(CharAt, "ABC")(2)); + EXPECT_EQ('C', absl::bind_front(CharAt, "ABC", 2)()); +} + +TEST(BindTest, Lambda) { + auto lambda = [](int x, int y, int z) { return x + y + z; }; + EXPECT_EQ(6, absl::bind_front(lambda)(1, 2, 3)); + EXPECT_EQ(6, absl::bind_front(lambda, 1)(2, 3)); + EXPECT_EQ(6, absl::bind_front(lambda, 1, 2)(3)); + EXPECT_EQ(6, absl::bind_front(lambda, 1, 2, 3)()); +} + +struct Functor { + std::string operator()() & { return "&"; } + std::string operator()() const& { return "const&"; } + std::string operator()() && { return "&&"; } + std::string operator()() const&& { return "const&&"; } +}; + +TEST(BindTest, PerfectForwardingOfBoundArgs) { + auto f = absl::bind_front(Functor()); + const auto& cf = f; + EXPECT_EQ("&", f()); + EXPECT_EQ("const&", cf()); + EXPECT_EQ("&&", std::move(f)()); + EXPECT_EQ("const&&", std::move(cf)()); +} + +struct ArgDescribe { + std::string operator()(int&) const { return "&"; } // NOLINT + std::string operator()(const int&) const { return "const&"; } // NOLINT + std::string operator()(int&&) const { return "&&"; } + std::string operator()(const int&&) const { return "const&&"; } +}; + +TEST(BindTest, PerfectForwardingOfFreeArgs) { + ArgDescribe f; + int i; + EXPECT_EQ("&", absl::bind_front(f)(static_cast<int&>(i))); + EXPECT_EQ("const&", absl::bind_front(f)(static_cast<const int&>(i))); + EXPECT_EQ("&&", absl::bind_front(f)(static_cast<int&&>(i))); + EXPECT_EQ("const&&", absl::bind_front(f)(static_cast<const int&&>(i))); +} + +struct NonCopyableFunctor { + NonCopyableFunctor() = default; + NonCopyableFunctor(const NonCopyableFunctor&) = delete; + NonCopyableFunctor& operator=(const NonCopyableFunctor&) = delete; + const NonCopyableFunctor* operator()() const { return this; } +}; + +TEST(BindTest, RefToFunctor) { + // It won't copy/move the functor and use the original object. + NonCopyableFunctor ncf; + auto bound_ncf = absl::bind_front(std::ref(ncf)); + auto bound_ncf_copy = bound_ncf; + EXPECT_EQ(&ncf, bound_ncf_copy()); +} + +struct Struct { + std::string value; +}; + +TEST(BindTest, StoreByCopy) { + Struct s = {"hello"}; + auto f = absl::bind_front(&Struct::value, s); + auto g = f; + EXPECT_EQ("hello", f()); + EXPECT_EQ("hello", g()); + EXPECT_NE(&s.value, &f()); + EXPECT_NE(&s.value, &g()); + EXPECT_NE(&g(), &f()); +} + +struct NonCopyable { + explicit NonCopyable(const std::string& s) : value(s) {} + NonCopyable(const NonCopyable&) = delete; + NonCopyable& operator=(const NonCopyable&) = delete; + + std::string value; +}; + +const std::string& GetNonCopyableValue(const NonCopyable& n) { return n.value; } + +TEST(BindTest, StoreByRef) { + NonCopyable s("hello"); + auto f = absl::bind_front(&GetNonCopyableValue, std::ref(s)); + EXPECT_EQ("hello", f()); + EXPECT_EQ(&s.value, &f()); + auto g = std::move(f); // NOLINT + EXPECT_EQ("hello", g()); + EXPECT_EQ(&s.value, &g()); + s.value = "goodbye"; + EXPECT_EQ("goodbye", g()); +} + +TEST(BindTest, StoreByCRef) { + NonCopyable s("hello"); + auto f = absl::bind_front(&GetNonCopyableValue, std::cref(s)); + EXPECT_EQ("hello", f()); + EXPECT_EQ(&s.value, &f()); + auto g = std::move(f); // NOLINT + EXPECT_EQ("hello", g()); + EXPECT_EQ(&s.value, &g()); + s.value = "goodbye"; + EXPECT_EQ("goodbye", g()); +} + +const std::string& GetNonCopyableValueByWrapper( + std::reference_wrapper<NonCopyable> n) { + return n.get().value; +} + +TEST(BindTest, StoreByRefInvokeByWrapper) { + NonCopyable s("hello"); + auto f = absl::bind_front(GetNonCopyableValueByWrapper, std::ref(s)); + EXPECT_EQ("hello", f()); + EXPECT_EQ(&s.value, &f()); + auto g = std::move(f); + EXPECT_EQ("hello", g()); + EXPECT_EQ(&s.value, &g()); + s.value = "goodbye"; + EXPECT_EQ("goodbye", g()); +} + +TEST(BindTest, StoreByPointer) { + NonCopyable s("hello"); + auto f = absl::bind_front(&NonCopyable::value, &s); + EXPECT_EQ("hello", f()); + EXPECT_EQ(&s.value, &f()); + auto g = std::move(f); + EXPECT_EQ("hello", g()); + EXPECT_EQ(&s.value, &g()); +} + +int Sink(std::unique_ptr<int> p) { + return *p; +} + +std::unique_ptr<int> Factory(int n) { return absl::make_unique<int>(n); } + +TEST(BindTest, NonCopyableArg) { + EXPECT_EQ(42, absl::bind_front(Sink)(absl::make_unique<int>(42))); + EXPECT_EQ(42, absl::bind_front(Sink, absl::make_unique<int>(42))()); +} + +TEST(BindTest, NonCopyableResult) { + EXPECT_THAT(absl::bind_front(Factory)(42), ::testing::Pointee(42)); + EXPECT_THAT(absl::bind_front(Factory, 42)(), ::testing::Pointee(42)); +} + +// is_copy_constructible<FalseCopyable<unique_ptr<T>> is true but an attempt to +// instantiate the copy constructor leads to a compile error. This is similar +// to how standard containers behave. +template <class T> +struct FalseCopyable { + FalseCopyable() {} + FalseCopyable(const FalseCopyable& other) : m(other.m) {} + FalseCopyable(FalseCopyable&& other) : m(std::move(other.m)) {} + T m; +}; + +int GetMember(FalseCopyable<std::unique_ptr<int>> x) { return *x.m; } + +TEST(BindTest, WrappedMoveOnly) { + FalseCopyable<std::unique_ptr<int>> x; + x.m = absl::make_unique<int>(42); + auto f = absl::bind_front(&GetMember, std::move(x)); + EXPECT_EQ(42, std::move(f)()); +} + +int Plus(int a, int b) { return a + b; } + +TEST(BindTest, ConstExpr) { + constexpr auto f = absl::bind_front(CharAt); + EXPECT_EQ(f("ABC", 1), 'B'); + static constexpr int five = 5; + constexpr auto plus5 = absl::bind_front(Plus, five); + EXPECT_EQ(plus5(1), 6); + + // There seems to be a bug in MSVC dealing constexpr construction of + // char[]. Notice 'plus5' above; 'int' works just fine. +#if !(defined(_MSC_VER) && _MSC_VER < 1910) + static constexpr char data[] = "DEF"; + constexpr auto g = absl::bind_front(CharAt, data); + EXPECT_EQ(g(1), 'E'); +#endif +} + +struct ManglingCall { + int operator()(int, double, std::string) const { return 0; } +}; + +TEST(BindTest, Mangling) { + // We just want to generate a particular instantiation to see its mangling. + absl::bind_front(ManglingCall{}, 1, 3.3)("A"); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/functional/function_ref.h b/third_party/abseil_cpp/absl/functional/function_ref.h new file mode 100644 index 000000000000..6e03ac2e04e9 --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/function_ref.h @@ -0,0 +1,139 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: function_ref.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::FunctionRef` type for holding a +// non-owning reference to an object of any invocable type. This function +// reference is typically most useful as a type-erased argument type for +// accepting function types that neither take ownership nor copy the type; using +// the reference type in this case avoids a copy and an allocation. Best +// practices of other non-owning reference-like objects (such as +// `absl::string_view`) apply here. +// +// An `absl::FunctionRef` is similar in usage to a `std::function` but has the +// following differences: +// +// * It doesn't own the underlying object. +// * It doesn't have a null or empty state. +// * It never performs deep copies or allocations. +// * It's much faster and cheaper to construct. +// * It's trivially copyable and destructable. +// +// Generally, `absl::FunctionRef` should not be used as a return value, data +// member, or to initialize a `std::function`. Such usages will often lead to +// problematic lifetime issues. Once you convert something to an +// `absl::FunctionRef` you cannot make a deep copy later. +// +// This class is suitable for use wherever a "const std::function<>&" +// would be used without making a copy. ForEach functions and other versions of +// the visitor pattern are a good example of when this class should be used. +// +// This class is trivial to copy and should be passed by value. +#ifndef ABSL_FUNCTIONAL_FUNCTION_REF_H_ +#define ABSL_FUNCTIONAL_FUNCTION_REF_H_ + +#include <cassert> +#include <functional> +#include <type_traits> + +#include "absl/functional/internal/function_ref.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// FunctionRef +// +// Dummy class declaration to allow the partial specialization based on function +// types below. +template <typename T> +class FunctionRef; + +// FunctionRef +// +// An `absl::FunctionRef` is a lightweight wrapper to any invokable object with +// a compatible signature. Generally, an `absl::FunctionRef` should only be used +// as an argument type and should be preferred as an argument over a const +// reference to a `std::function`. +// +// Example: +// +// // The following function takes a function callback by const reference +// bool Visitor(const std::function<void(my_proto&, +// absl::string_view)>& callback); +// +// // Assuming that the function is not stored or otherwise copied, it can be +// // replaced by an `absl::FunctionRef`: +// bool Visitor(absl::FunctionRef<void(my_proto&, absl::string_view)> +// callback); +// +// Note: the assignment operator within an `absl::FunctionRef` is intentionally +// deleted to prevent misuse; because the `absl::FunctionRef` does not own the +// underlying type, assignment likely indicates misuse. +template <typename R, typename... Args> +class FunctionRef<R(Args...)> { + private: + // Used to disable constructors for objects that are not compatible with the + // signature of this FunctionRef. + template <typename F, + typename FR = absl::base_internal::invoke_result_t<F, Args&&...>> + using EnableIfCompatible = + typename std::enable_if<std::is_void<R>::value || + std::is_convertible<FR, R>::value>::type; + + public: + // Constructs a FunctionRef from any invokable type. + template <typename F, typename = EnableIfCompatible<const F&>> + FunctionRef(const F& f) // NOLINT(runtime/explicit) + : invoker_(&absl::functional_internal::InvokeObject<F, R, Args...>) { + absl::functional_internal::AssertNonNull(f); + ptr_.obj = &f; + } + + // Overload for function pointers. This eliminates a level of indirection that + // would happen if the above overload was used (it lets us store the pointer + // instead of a pointer to a pointer). + // + // This overload is also used for references to functions, since references to + // functions can decay to function pointers implicitly. + template < + typename F, typename = EnableIfCompatible<F*>, + absl::functional_internal::EnableIf<absl::is_function<F>::value> = 0> + FunctionRef(F* f) // NOLINT(runtime/explicit) + : invoker_(&absl::functional_internal::InvokeFunction<F*, R, Args...>) { + assert(f != nullptr); + ptr_.fun = reinterpret_cast<decltype(ptr_.fun)>(f); + } + + // To help prevent subtle lifetime bugs, FunctionRef is not assignable. + // Typically, it should only be used as an argument type. + FunctionRef& operator=(const FunctionRef& rhs) = delete; + + // Call the underlying object. + R operator()(Args... args) const { + return invoker_(ptr_, std::forward<Args>(args)...); + } + + private: + absl::functional_internal::VoidPtr ptr_; + absl::functional_internal::Invoker<R, Args...> invoker_; +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FUNCTIONAL_FUNCTION_REF_H_ diff --git a/third_party/abseil_cpp/absl/functional/function_ref_benchmark.cc b/third_party/abseil_cpp/absl/functional/function_ref_benchmark.cc new file mode 100644 index 000000000000..045305bfef30 --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/function_ref_benchmark.cc @@ -0,0 +1,142 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/functional/function_ref.h" + +#include <memory> + +#include "benchmark/benchmark.h" +#include "absl/base/attributes.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +int dummy = 0; + +void FreeFunction() { benchmark::DoNotOptimize(dummy); } + +struct TrivialFunctor { + void operator()() const { benchmark::DoNotOptimize(dummy); } +}; + +struct LargeFunctor { + void operator()() const { benchmark::DoNotOptimize(this); } + std::string a, b, c; +}; + +template <typename Function, typename... Args> +void ABSL_ATTRIBUTE_NOINLINE CallFunction(Function f, Args&&... args) { + f(std::forward<Args>(args)...); +} + +template <typename Function, typename Callable, typename... Args> +void ConstructAndCallFunctionBenchmark(benchmark::State& state, + const Callable& c, Args&&... args) { + for (auto _ : state) { + CallFunction<Function>(c, std::forward<Args>(args)...); + } +} + +void BM_TrivialStdFunction(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<std::function<void()>>(state, + TrivialFunctor{}); +} +BENCHMARK(BM_TrivialStdFunction); + +void BM_TrivialFunctionRef(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<FunctionRef<void()>>(state, + TrivialFunctor{}); +} +BENCHMARK(BM_TrivialFunctionRef); + +void BM_LargeStdFunction(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<std::function<void()>>(state, + LargeFunctor{}); +} +BENCHMARK(BM_LargeStdFunction); + +void BM_LargeFunctionRef(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<FunctionRef<void()>>(state, LargeFunctor{}); +} +BENCHMARK(BM_LargeFunctionRef); + +void BM_FunPtrStdFunction(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<std::function<void()>>(state, FreeFunction); +} +BENCHMARK(BM_FunPtrStdFunction); + +void BM_FunPtrFunctionRef(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<FunctionRef<void()>>(state, FreeFunction); +} +BENCHMARK(BM_FunPtrFunctionRef); + +// Doesn't include construction or copy overhead in the loop. +template <typename Function, typename Callable, typename... Args> +void CallFunctionBenchmark(benchmark::State& state, const Callable& c, + Args... args) { + Function f = c; + for (auto _ : state) { + benchmark::DoNotOptimize(&f); + f(args...); + } +} + +struct FunctorWithTrivialArgs { + void operator()(int a, int b, int c) const { + benchmark::DoNotOptimize(a); + benchmark::DoNotOptimize(b); + benchmark::DoNotOptimize(c); + } +}; + +void BM_TrivialArgsStdFunction(benchmark::State& state) { + CallFunctionBenchmark<std::function<void(int, int, int)>>( + state, FunctorWithTrivialArgs{}, 1, 2, 3); +} +BENCHMARK(BM_TrivialArgsStdFunction); + +void BM_TrivialArgsFunctionRef(benchmark::State& state) { + CallFunctionBenchmark<FunctionRef<void(int, int, int)>>( + state, FunctorWithTrivialArgs{}, 1, 2, 3); +} +BENCHMARK(BM_TrivialArgsFunctionRef); + +struct FunctorWithNonTrivialArgs { + void operator()(std::string a, std::string b, std::string c) const { + benchmark::DoNotOptimize(&a); + benchmark::DoNotOptimize(&b); + benchmark::DoNotOptimize(&c); + } +}; + +void BM_NonTrivialArgsStdFunction(benchmark::State& state) { + std::string a, b, c; + CallFunctionBenchmark< + std::function<void(std::string, std::string, std::string)>>( + state, FunctorWithNonTrivialArgs{}, a, b, c); +} +BENCHMARK(BM_NonTrivialArgsStdFunction); + +void BM_NonTrivialArgsFunctionRef(benchmark::State& state) { + std::string a, b, c; + CallFunctionBenchmark< + FunctionRef<void(std::string, std::string, std::string)>>( + state, FunctorWithNonTrivialArgs{}, a, b, c); +} +BENCHMARK(BM_NonTrivialArgsFunctionRef); + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/functional/function_ref_test.cc b/third_party/abseil_cpp/absl/functional/function_ref_test.cc new file mode 100644 index 000000000000..3aa59745873d --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/function_ref_test.cc @@ -0,0 +1,257 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/functional/function_ref.h" + +#include <memory> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/internal/test_instance_tracker.h" +#include "absl/memory/memory.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +void RunFun(FunctionRef<void()> f) { f(); } + +TEST(FunctionRefTest, Lambda) { + bool ran = false; + RunFun([&] { ran = true; }); + EXPECT_TRUE(ran); +} + +int Function() { return 1337; } + +TEST(FunctionRefTest, Function1) { + FunctionRef<int()> ref(&Function); + EXPECT_EQ(1337, ref()); +} + +TEST(FunctionRefTest, Function2) { + FunctionRef<int()> ref(Function); + EXPECT_EQ(1337, ref()); +} + +int NoExceptFunction() noexcept { return 1337; } + +// TODO(jdennett): Add a test for noexcept member functions. +TEST(FunctionRefTest, NoExceptFunction) { + FunctionRef<int()> ref(NoExceptFunction); + EXPECT_EQ(1337, ref()); +} + +TEST(FunctionRefTest, ForwardsArgs) { + auto l = [](std::unique_ptr<int> i) { return *i; }; + FunctionRef<int(std::unique_ptr<int>)> ref(l); + EXPECT_EQ(42, ref(absl::make_unique<int>(42))); +} + +TEST(FunctionRef, ReturnMoveOnly) { + auto l = [] { return absl::make_unique<int>(29); }; + FunctionRef<std::unique_ptr<int>()> ref(l); + EXPECT_EQ(29, *ref()); +} + +TEST(FunctionRef, ManyArgs) { + auto l = [](int a, int b, int c) { return a + b + c; }; + FunctionRef<int(int, int, int)> ref(l); + EXPECT_EQ(6, ref(1, 2, 3)); +} + +TEST(FunctionRef, VoidResultFromNonVoidFunctor) { + bool ran = false; + auto l = [&]() -> int { + ran = true; + return 2; + }; + FunctionRef<void()> ref(l); + ref(); + EXPECT_TRUE(ran); +} + +TEST(FunctionRef, CastFromDerived) { + struct Base {}; + struct Derived : public Base {}; + + Derived d; + auto l1 = [&](Base* b) { EXPECT_EQ(&d, b); }; + FunctionRef<void(Derived*)> ref1(l1); + ref1(&d); + + auto l2 = [&]() -> Derived* { return &d; }; + FunctionRef<Base*()> ref2(l2); + EXPECT_EQ(&d, ref2()); +} + +TEST(FunctionRef, VoidResultFromNonVoidFuncton) { + FunctionRef<void()> ref(Function); + ref(); +} + +TEST(FunctionRef, MemberPtr) { + struct S { + int i; + }; + + S s{1100111}; + auto mem_ptr = &S::i; + FunctionRef<int(const S& s)> ref(mem_ptr); + EXPECT_EQ(1100111, ref(s)); +} + +TEST(FunctionRef, MemberFun) { + struct S { + int i; + int get_i() const { return i; } + }; + + S s{22}; + auto mem_fun_ptr = &S::get_i; + FunctionRef<int(const S& s)> ref(mem_fun_ptr); + EXPECT_EQ(22, ref(s)); +} + +TEST(FunctionRef, MemberFunRefqualified) { + struct S { + int i; + int get_i() && { return i; } + }; + auto mem_fun_ptr = &S::get_i; + S s{22}; + FunctionRef<int(S && s)> ref(mem_fun_ptr); + EXPECT_EQ(22, ref(std::move(s))); +} + +#if !defined(_WIN32) && defined(GTEST_HAS_DEATH_TEST) + +TEST(FunctionRef, MemberFunRefqualifiedNull) { + struct S { + int i; + int get_i() && { return i; } + }; + auto mem_fun_ptr = &S::get_i; + mem_fun_ptr = nullptr; + EXPECT_DEBUG_DEATH({ FunctionRef<int(S && s)> ref(mem_fun_ptr); }, ""); +} + +TEST(FunctionRef, NullMemberPtrAssertFails) { + struct S { + int i; + }; + using MemberPtr = int S::*; + MemberPtr mem_ptr = nullptr; + EXPECT_DEBUG_DEATH({ FunctionRef<int(const S& s)> ref(mem_ptr); }, ""); +} + +#endif // GTEST_HAS_DEATH_TEST + +TEST(FunctionRef, CopiesAndMovesPerPassByValue) { + absl::test_internal::InstanceTracker tracker; + absl::test_internal::CopyableMovableInstance instance(0); + auto l = [](absl::test_internal::CopyableMovableInstance) {}; + FunctionRef<void(absl::test_internal::CopyableMovableInstance)> ref(l); + ref(instance); + EXPECT_EQ(tracker.copies(), 1); + EXPECT_EQ(tracker.moves(), 1); +} + +TEST(FunctionRef, CopiesAndMovesPerPassByRef) { + absl::test_internal::InstanceTracker tracker; + absl::test_internal::CopyableMovableInstance instance(0); + auto l = [](const absl::test_internal::CopyableMovableInstance&) {}; + FunctionRef<void(const absl::test_internal::CopyableMovableInstance&)> ref(l); + ref(instance); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 0); +} + +TEST(FunctionRef, CopiesAndMovesPerPassByValueCallByMove) { + absl::test_internal::InstanceTracker tracker; + absl::test_internal::CopyableMovableInstance instance(0); + auto l = [](absl::test_internal::CopyableMovableInstance) {}; + FunctionRef<void(absl::test_internal::CopyableMovableInstance)> ref(l); + ref(std::move(instance)); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 2); +} + +TEST(FunctionRef, CopiesAndMovesPerPassByValueToRef) { + absl::test_internal::InstanceTracker tracker; + absl::test_internal::CopyableMovableInstance instance(0); + auto l = [](const absl::test_internal::CopyableMovableInstance&) {}; + FunctionRef<void(absl::test_internal::CopyableMovableInstance)> ref(l); + ref(std::move(instance)); + EXPECT_EQ(tracker.copies(), 0); + EXPECT_EQ(tracker.moves(), 1); +} + +TEST(FunctionRef, PassByValueTypes) { + using absl::functional_internal::Invoker; + using absl::functional_internal::VoidPtr; + using absl::test_internal::CopyableMovableInstance; + struct Trivial { + void* p[2]; + }; + struct LargeTrivial { + void* p[3]; + }; + + static_assert(std::is_same<Invoker<void, int>, void (*)(VoidPtr, int)>::value, + "Scalar types should be passed by value"); + static_assert( + std::is_same<Invoker<void, Trivial>, void (*)(VoidPtr, Trivial)>::value, + "Small trivial types should be passed by value"); + static_assert(std::is_same<Invoker<void, LargeTrivial>, + void (*)(VoidPtr, LargeTrivial &&)>::value, + "Large trivial types should be passed by rvalue reference"); + static_assert( + std::is_same<Invoker<void, CopyableMovableInstance>, + void (*)(VoidPtr, CopyableMovableInstance &&)>::value, + "Types with copy/move ctor should be passed by rvalue reference"); + + // References are passed as references. + static_assert( + std::is_same<Invoker<void, int&>, void (*)(VoidPtr, int&)>::value, + "Reference types should be preserved"); + static_assert( + std::is_same<Invoker<void, CopyableMovableInstance&>, + void (*)(VoidPtr, CopyableMovableInstance&)>::value, + "Reference types should be preserved"); + static_assert( + std::is_same<Invoker<void, CopyableMovableInstance&&>, + void (*)(VoidPtr, CopyableMovableInstance &&)>::value, + "Reference types should be preserved"); + + // Make sure the address of an object received by reference is the same as the + // addess of the object passed by the caller. + { + LargeTrivial obj; + auto test = [&obj](LargeTrivial& input) { ASSERT_EQ(&input, &obj); }; + absl::FunctionRef<void(LargeTrivial&)> ref(test); + ref(obj); + } + + { + Trivial obj; + auto test = [&obj](Trivial& input) { ASSERT_EQ(&input, &obj); }; + absl::FunctionRef<void(Trivial&)> ref(test); + ref(obj); + } +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/functional/internal/front_binder.h b/third_party/abseil_cpp/absl/functional/internal/front_binder.h new file mode 100644 index 000000000000..45f52de73d25 --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/internal/front_binder.h @@ -0,0 +1,95 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Implementation details for `absl::bind_front()`. + +#ifndef ABSL_FUNCTIONAL_INTERNAL_FRONT_BINDER_H_ +#define ABSL_FUNCTIONAL_INTERNAL_FRONT_BINDER_H_ + +#include <cstddef> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/invoke.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace functional_internal { + +// Invoke the method, expanding the tuple of bound arguments. +template <class R, class Tuple, size_t... Idx, class... Args> +R Apply(Tuple&& bound, absl::index_sequence<Idx...>, Args&&... free) { + return base_internal::invoke( + absl::forward<Tuple>(bound).template get<Idx>()..., + absl::forward<Args>(free)...); +} + +template <class F, class... BoundArgs> +class FrontBinder { + using BoundArgsT = absl::container_internal::CompressedTuple<F, BoundArgs...>; + using Idx = absl::make_index_sequence<sizeof...(BoundArgs) + 1>; + + BoundArgsT bound_args_; + + public: + template <class... Ts> + constexpr explicit FrontBinder(absl::in_place_t, Ts&&... ts) + : bound_args_(absl::forward<Ts>(ts)...) {} + + template <class... FreeArgs, class R = base_internal::invoke_result_t< + F&, BoundArgs&..., FreeArgs&&...>> + R operator()(FreeArgs&&... free_args) & { + return functional_internal::Apply<R>(bound_args_, Idx(), + absl::forward<FreeArgs>(free_args)...); + } + + template <class... FreeArgs, + class R = base_internal::invoke_result_t< + const F&, const BoundArgs&..., FreeArgs&&...>> + R operator()(FreeArgs&&... free_args) const& { + return functional_internal::Apply<R>(bound_args_, Idx(), + absl::forward<FreeArgs>(free_args)...); + } + + template <class... FreeArgs, class R = base_internal::invoke_result_t< + F&&, BoundArgs&&..., FreeArgs&&...>> + R operator()(FreeArgs&&... free_args) && { + // This overload is called when *this is an rvalue. If some of the bound + // arguments are stored by value or rvalue reference, we move them. + return functional_internal::Apply<R>(absl::move(bound_args_), Idx(), + absl::forward<FreeArgs>(free_args)...); + } + + template <class... FreeArgs, + class R = base_internal::invoke_result_t< + const F&&, const BoundArgs&&..., FreeArgs&&...>> + R operator()(FreeArgs&&... free_args) const&& { + // This overload is called when *this is an rvalue. If some of the bound + // arguments are stored by value or rvalue reference, we move them. + return functional_internal::Apply<R>(absl::move(bound_args_), Idx(), + absl::forward<FreeArgs>(free_args)...); + } +}; + +template <class F, class... BoundArgs> +using bind_front_t = FrontBinder<decay_t<F>, absl::decay_t<BoundArgs>...>; + +} // namespace functional_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FUNCTIONAL_INTERNAL_FRONT_BINDER_H_ diff --git a/third_party/abseil_cpp/absl/functional/internal/function_ref.h b/third_party/abseil_cpp/absl/functional/internal/function_ref.h new file mode 100644 index 000000000000..b5bb8b430a81 --- /dev/null +++ b/third_party/abseil_cpp/absl/functional/internal/function_ref.h @@ -0,0 +1,106 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_FUNCTIONAL_INTERNAL_FUNCTION_REF_H_ +#define ABSL_FUNCTIONAL_INTERNAL_FUNCTION_REF_H_ + +#include <cassert> +#include <functional> +#include <type_traits> + +#include "absl/base/internal/invoke.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace functional_internal { + +// Like a void* that can handle function pointers as well. The standard does not +// allow function pointers to round-trip through void*, but void(*)() is fine. +// +// Note: It's important that this class remains trivial and is the same size as +// a pointer, since this allows the compiler to perform tail-call optimizations +// when the underlying function is a callable object with a matching signature. +union VoidPtr { + const void* obj; + void (*fun)(); +}; + +// Chooses the best type for passing T as an argument. +// Attempt to be close to SystemV AMD64 ABI. Objects with trivial copy ctor are +// passed by value. +template <typename T> +constexpr bool PassByValue() { + return !std::is_lvalue_reference<T>::value && + absl::is_trivially_copy_constructible<T>::value && + absl::is_trivially_copy_assignable< + typename std::remove_cv<T>::type>::value && + std::is_trivially_destructible<T>::value && + sizeof(T) <= 2 * sizeof(void*); +} + +template <typename T> +struct ForwardT : std::conditional<PassByValue<T>(), T, T&&> {}; + +// An Invoker takes a pointer to the type-erased invokable object, followed by +// the arguments that the invokable object expects. +// +// Note: The order of arguments here is an optimization, since member functions +// have an implicit "this" pointer as their first argument, putting VoidPtr +// first allows the compiler to perform tail-call optimization in many cases. +template <typename R, typename... Args> +using Invoker = R (*)(VoidPtr, typename ForwardT<Args>::type...); + +// +// InvokeObject and InvokeFunction provide static "Invoke" functions that can be +// used as Invokers for objects or functions respectively. +// +// static_cast<R> handles the case the return type is void. +template <typename Obj, typename R, typename... Args> +R InvokeObject(VoidPtr ptr, typename ForwardT<Args>::type... args) { + auto o = static_cast<const Obj*>(ptr.obj); + return static_cast<R>( + absl::base_internal::invoke(*o, std::forward<Args>(args)...)); +} + +template <typename Fun, typename R, typename... Args> +R InvokeFunction(VoidPtr ptr, typename ForwardT<Args>::type... args) { + auto f = reinterpret_cast<Fun>(ptr.fun); + return static_cast<R>( + absl::base_internal::invoke(f, std::forward<Args>(args)...)); +} + +template <typename Sig> +void AssertNonNull(const std::function<Sig>& f) { + assert(f != nullptr); + (void)f; +} + +template <typename F> +void AssertNonNull(const F&) {} + +template <typename F, typename C> +void AssertNonNull(F C::*f) { + assert(f != nullptr); + (void)f; +} + +template <bool C> +using EnableIf = typename ::std::enable_if<C, int>::type; + +} // namespace functional_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FUNCTIONAL_INTERNAL_FUNCTION_REF_H_ diff --git a/third_party/abseil_cpp/absl/hash/BUILD.bazel b/third_party/abseil_cpp/absl/hash/BUILD.bazel new file mode 100644 index 000000000000..5b1e2d01fd3c --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/BUILD.bazel @@ -0,0 +1,122 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "hash", + srcs = [ + "internal/hash.cc", + "internal/hash.h", + ], + hdrs = ["hash.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":city", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/container:fixed_array", + "//absl/meta:type_traits", + "//absl/numeric:int128", + "//absl/strings", + "//absl/types:optional", + "//absl/types:variant", + "//absl/utility", + ], +) + +cc_library( + name = "hash_testing", + testonly = 1, + hdrs = ["hash_testing.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":spy_hash_state", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/types:variant", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "hash_test", + srcs = ["hash_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":hash", + ":hash_testing", + ":spy_hash_state", + "//absl/base:core_headers", + "//absl/container:flat_hash_set", + "//absl/meta:type_traits", + "//absl/numeric:int128", + "//absl/strings:cord_test_helpers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "spy_hash_state", + testonly = 1, + hdrs = ["internal/spy_hash_state.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":hash", + "//absl/strings", + "//absl/strings:str_format", + ], +) + +cc_library( + name = "city", + srcs = ["internal/city.cc"], + hdrs = [ + "internal/city.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:endian", + ], +) + +cc_test( + name = "city_test", + srcs = ["internal/city_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":city", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/hash/CMakeLists.txt b/third_party/abseil_cpp/absl/hash/CMakeLists.txt new file mode 100644 index 000000000000..61365e9bb5a1 --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/CMakeLists.txt @@ -0,0 +1,116 @@ +# +# Copyright 2018 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + hash + HDRS + "hash.h" + SRCS + "internal/hash.cc" + "internal/hash.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::endian + absl::fixed_array + absl::meta + absl::int128 + absl::strings + absl::optional + absl::variant + absl::utility + absl::city + PUBLIC +) + +absl_cc_library( + NAME + hash_testing + HDRS + "hash_testing.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::spy_hash_state + absl::meta + absl::strings + absl::variant + gmock + TESTONLY +) + +absl_cc_test( + NAME + hash_test + SRCS + "hash_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::cord_test_helpers + absl::hash + absl::hash_testing + absl::core_headers + absl::flat_hash_set + absl::spy_hash_state + absl::meta + absl::int128 + gmock_main +) + +absl_cc_library( + NAME + spy_hash_state + HDRS + "internal/spy_hash_state.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::hash + absl::strings + absl::str_format + TESTONLY +) + +absl_cc_library( + NAME + city + HDRS + "internal/city.h" + SRCS + "internal/city.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::endian +) + +absl_cc_test( + NAME + city_test + SRCS + "internal/city_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::city + gmock_main +) + diff --git a/third_party/abseil_cpp/absl/hash/hash.h b/third_party/abseil_cpp/absl/hash/hash.h new file mode 100644 index 000000000000..5de132cac8ac --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/hash.h @@ -0,0 +1,325 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: hash.h +// ----------------------------------------------------------------------------- +// +// This header file defines the Abseil `hash` library and the Abseil hashing +// framework. This framework consists of the following: +// +// * The `absl::Hash` functor, which is used to invoke the hasher within the +// Abseil hashing framework. `absl::Hash<T>` supports most basic types and +// a number of Abseil types out of the box. +// * `AbslHashValue`, an extension point that allows you to extend types to +// support Abseil hashing without requiring you to define a hashing +// algorithm. +// * `HashState`, a type-erased class which implements the manipulation of the +// hash state (H) itself, contains member functions `combine()` and +// `combine_contiguous()`, which you can use to contribute to an existing +// hash state when hashing your types. +// +// Unlike `std::hash` or other hashing frameworks, the Abseil hashing framework +// provides most of its utility by abstracting away the hash algorithm (and its +// implementation) entirely. Instead, a type invokes the Abseil hashing +// framework by simply combining its state with the state of known, hashable +// types. Hashing of that combined state is separately done by `absl::Hash`. +// +// One should assume that a hash algorithm is chosen randomly at the start of +// each process. E.g., `absl::Hash<int>{}(9)` in one process and +// `absl::Hash<int>{}(9)` in another process are likely to differ. +// +// `absl::Hash` is intended to strongly mix input bits with a target of passing +// an [Avalanche Test](https://en.wikipedia.org/wiki/Avalanche_effect). +// +// Example: +// +// // Suppose we have a class `Circle` for which we want to add hashing: +// class Circle { +// public: +// ... +// private: +// std::pair<int, int> center_; +// int radius_; +// }; +// +// // To add hashing support to `Circle`, we simply need to add a free +// // (non-member) function `AbslHashValue()`, and return the combined hash +// // state of the existing hash state and the class state. You can add such a +// // free function using a friend declaration within the body of the class: +// class Circle { +// public: +// ... +// template <typename H> +// friend H AbslHashValue(H h, const Circle& c) { +// return H::combine(std::move(h), c.center_, c.radius_); +// } +// ... +// }; +// +// For more information, see Adding Type Support to `absl::Hash` below. +// +#ifndef ABSL_HASH_HASH_H_ +#define ABSL_HASH_HASH_H_ + +#include "absl/hash/internal/hash.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// `absl::Hash` +// ----------------------------------------------------------------------------- +// +// `absl::Hash<T>` is a convenient general-purpose hash functor for any type `T` +// satisfying any of the following conditions (in order): +// +// * T is an arithmetic or pointer type +// * T defines an overload for `AbslHashValue(H, const T&)` for an arbitrary +// hash state `H`. +// - T defines a specialization of `std::hash<T>` +// +// `absl::Hash` intrinsically supports the following types: +// +// * All integral types (including bool) +// * All enum types +// * All floating-point types (although hashing them is discouraged) +// * All pointer types, including nullptr_t +// * std::pair<T1, T2>, if T1 and T2 are hashable +// * std::tuple<Ts...>, if all the Ts... are hashable +// * std::unique_ptr and std::shared_ptr +// * All string-like types including: +// * absl::Cord +// * std::string +// * std::string_view (as well as any instance of std::basic_string that +// uses char and std::char_traits) +// * All the standard sequence containers (provided the elements are hashable) +// * All the standard ordered associative containers (provided the elements are +// hashable) +// * absl types such as the following: +// * absl::string_view +// * absl::InlinedVector +// * absl::FixedArray +// * absl::uint128 +// * absl::Time, absl::Duration, and absl::TimeZone +// +// Note: the list above is not meant to be exhaustive. Additional type support +// may be added, in which case the above list will be updated. +// +// ----------------------------------------------------------------------------- +// absl::Hash Invocation Evaluation +// ----------------------------------------------------------------------------- +// +// When invoked, `absl::Hash<T>` searches for supplied hash functions in the +// following order: +// +// * Natively supported types out of the box (see above) +// * Types for which an `AbslHashValue()` overload is provided (such as +// user-defined types). See "Adding Type Support to `absl::Hash`" below. +// * Types which define a `std::hash<T>` specialization +// +// The fallback to legacy hash functions exists mainly for backwards +// compatibility. If you have a choice, prefer defining an `AbslHashValue` +// overload instead of specializing any legacy hash functors. +// +// ----------------------------------------------------------------------------- +// The Hash State Concept, and using `HashState` for Type Erasure +// ----------------------------------------------------------------------------- +// +// The `absl::Hash` framework relies on the Concept of a "hash state." Such a +// hash state is used in several places: +// +// * Within existing implementations of `absl::Hash<T>` to store the hashed +// state of an object. Note that it is up to the implementation how it stores +// such state. A hash table, for example, may mix the state to produce an +// integer value; a testing framework may simply hold a vector of that state. +// * Within implementations of `AbslHashValue()` used to extend user-defined +// types. (See "Adding Type Support to absl::Hash" below.) +// * Inside a `HashState`, providing type erasure for the concept of a hash +// state, which you can use to extend the `absl::Hash` framework for types +// that are otherwise difficult to extend using `AbslHashValue()`. (See the +// `HashState` class below.) +// +// The "hash state" concept contains two member functions for mixing hash state: +// +// * `H::combine(state, values...)` +// +// Combines an arbitrary number of values into a hash state, returning the +// updated state. Note that the existing hash state is move-only and must be +// passed by value. +// +// Each of the value types T must be hashable by H. +// +// NOTE: +// +// state = H::combine(std::move(state), value1, value2, value3); +// +// must be guaranteed to produce the same hash expansion as +// +// state = H::combine(std::move(state), value1); +// state = H::combine(std::move(state), value2); +// state = H::combine(std::move(state), value3); +// +// * `H::combine_contiguous(state, data, size)` +// +// Combines a contiguous array of `size` elements into a hash state, +// returning the updated state. Note that the existing hash state is +// move-only and must be passed by value. +// +// NOTE: +// +// state = H::combine_contiguous(std::move(state), data, size); +// +// need NOT be guaranteed to produce the same hash expansion as a loop +// (it may perform internal optimizations). If you need this guarantee, use a +// loop instead. +// +// ----------------------------------------------------------------------------- +// Adding Type Support to `absl::Hash` +// ----------------------------------------------------------------------------- +// +// To add support for your user-defined type, add a proper `AbslHashValue()` +// overload as a free (non-member) function. The overload will take an +// existing hash state and should combine that state with state from the type. +// +// Example: +// +// template <typename H> +// H AbslHashValue(H state, const MyType& v) { +// return H::combine(std::move(state), v.field1, ..., v.fieldN); +// } +// +// where `(field1, ..., fieldN)` are the members you would use on your +// `operator==` to define equality. +// +// Notice that `AbslHashValue` is not a class member, but an ordinary function. +// An `AbslHashValue` overload for a type should only be declared in the same +// file and namespace as said type. The proper `AbslHashValue` implementation +// for a given type will be discovered via ADL. +// +// Note: unlike `std::hash', `absl::Hash` should never be specialized. It must +// only be extended by adding `AbslHashValue()` overloads. +// +template <typename T> +using Hash = absl::hash_internal::Hash<T>; + +// HashState +// +// A type erased version of the hash state concept, for use in user-defined +// `AbslHashValue` implementations that can't use templates (such as PImpl +// classes, virtual functions, etc.). The type erasure adds overhead so it +// should be avoided unless necessary. +// +// Note: This wrapper will only erase calls to: +// combine_contiguous(H, const unsigned char*, size_t) +// +// All other calls will be handled internally and will not invoke overloads +// provided by the wrapped class. +// +// Users of this class should still define a template `AbslHashValue` function, +// but can use `absl::HashState::Create(&state)` to erase the type of the hash +// state and dispatch to their private hashing logic. +// +// This state can be used like any other hash state. In particular, you can call +// `HashState::combine()` and `HashState::combine_contiguous()` on it. +// +// Example: +// +// class Interface { +// public: +// template <typename H> +// friend H AbslHashValue(H state, const Interface& value) { +// state = H::combine(std::move(state), std::type_index(typeid(*this))); +// value.HashValue(absl::HashState::Create(&state)); +// return state; +// } +// private: +// virtual void HashValue(absl::HashState state) const = 0; +// }; +// +// class Impl : Interface { +// private: +// void HashValue(absl::HashState state) const override { +// absl::HashState::combine(std::move(state), v1_, v2_); +// } +// int v1_; +// std::string v2_; +// }; +class HashState : public hash_internal::HashStateBase<HashState> { + public: + // HashState::Create() + // + // Create a new `HashState` instance that wraps `state`. All calls to + // `combine()` and `combine_contiguous()` on the new instance will be + // redirected to the original `state` object. The `state` object must outlive + // the `HashState` instance. + template <typename T> + static HashState Create(T* state) { + HashState s; + s.Init(state); + return s; + } + + HashState(const HashState&) = delete; + HashState& operator=(const HashState&) = delete; + HashState(HashState&&) = default; + HashState& operator=(HashState&&) = default; + + // HashState::combine() + // + // Combines an arbitrary number of values into a hash state, returning the + // updated state. + using HashState::HashStateBase::combine; + + // HashState::combine_contiguous() + // + // Combines a contiguous array of `size` elements into a hash state, returning + // the updated state. + static HashState combine_contiguous(HashState hash_state, + const unsigned char* first, size_t size) { + hash_state.combine_contiguous_(hash_state.state_, first, size); + return hash_state; + } + using HashState::HashStateBase::combine_contiguous; + + private: + HashState() = default; + + template <typename T> + static void CombineContiguousImpl(void* p, const unsigned char* first, + size_t size) { + T& state = *static_cast<T*>(p); + state = T::combine_contiguous(std::move(state), first, size); + } + + template <typename T> + void Init(T* state) { + state_ = state; + combine_contiguous_ = &CombineContiguousImpl<T>; + } + + // Do not erase an already erased state. + void Init(HashState* state) { + state_ = state->state_; + combine_contiguous_ = state->combine_contiguous_; + } + + void* state_; + void (*combine_contiguous_)(void*, const unsigned char*, size_t); +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HASH_HASH_H_ diff --git a/third_party/abseil_cpp/absl/hash/hash_test.cc b/third_party/abseil_cpp/absl/hash/hash_test.cc new file mode 100644 index 000000000000..1d2e6cf0df44 --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/hash_test.cc @@ -0,0 +1,976 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/hash/hash.h" + +#include <array> +#include <bitset> +#include <cstring> +#include <deque> +#include <forward_list> +#include <functional> +#include <iterator> +#include <limits> +#include <list> +#include <map> +#include <memory> +#include <numeric> +#include <random> +#include <set> +#include <string> +#include <tuple> +#include <type_traits> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/flat_hash_set.h" +#include "absl/hash/hash_testing.h" +#include "absl/hash/internal/spy_hash_state.h" +#include "absl/meta/type_traits.h" +#include "absl/numeric/int128.h" +#include "absl/strings/cord_test_helpers.h" + +namespace { + +using absl::Hash; +using absl::hash_internal::SpyHashState; + +template <typename T> +class HashValueIntTest : public testing::Test { +}; +TYPED_TEST_SUITE_P(HashValueIntTest); + +template <typename T> +SpyHashState SpyHash(const T& value) { + return SpyHashState::combine(SpyHashState(), value); +} + +// Helper trait to verify if T is hashable. We use absl::Hash's poison status to +// detect it. +template <typename T> +using is_hashable = std::is_default_constructible<absl::Hash<T>>; + +TYPED_TEST_P(HashValueIntTest, BasicUsage) { + EXPECT_TRUE((is_hashable<TypeParam>::value)); + + TypeParam n = 42; + EXPECT_EQ(SpyHash(n), SpyHash(TypeParam{42})); + EXPECT_NE(SpyHash(n), SpyHash(TypeParam{0})); + EXPECT_NE(SpyHash(std::numeric_limits<TypeParam>::max()), + SpyHash(std::numeric_limits<TypeParam>::min())); +} + +TYPED_TEST_P(HashValueIntTest, FastPath) { + // Test the fast-path to make sure the values are the same. + TypeParam n = 42; + EXPECT_EQ(absl::Hash<TypeParam>{}(n), + absl::Hash<std::tuple<TypeParam>>{}(std::tuple<TypeParam>(n))); +} + +REGISTER_TYPED_TEST_CASE_P(HashValueIntTest, BasicUsage, FastPath); +using IntTypes = testing::Types<unsigned char, char, int, int32_t, int64_t, + uint32_t, uint64_t, size_t>; +INSTANTIATE_TYPED_TEST_CASE_P(My, HashValueIntTest, IntTypes); + +enum LegacyEnum { kValue1, kValue2, kValue3 }; + +enum class EnumClass { kValue4, kValue5, kValue6 }; + +TEST(HashValueTest, EnumAndBool) { + EXPECT_TRUE((is_hashable<LegacyEnum>::value)); + EXPECT_TRUE((is_hashable<EnumClass>::value)); + EXPECT_TRUE((is_hashable<bool>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + LegacyEnum::kValue1, LegacyEnum::kValue2, LegacyEnum::kValue3))); + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + EnumClass::kValue4, EnumClass::kValue5, EnumClass::kValue6))); + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(true, false))); +} + +TEST(HashValueTest, FloatingPoint) { + EXPECT_TRUE((is_hashable<float>::value)); + EXPECT_TRUE((is_hashable<double>::value)); + EXPECT_TRUE((is_hashable<long double>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(42.f, 0.f, -0.f, std::numeric_limits<float>::infinity(), + -std::numeric_limits<float>::infinity()))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(42., 0., -0., std::numeric_limits<double>::infinity(), + -std::numeric_limits<double>::infinity()))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + // Add some values with small exponent to test that NORMAL values also + // append their category. + .5L, 1.L, 2.L, 4.L, 42.L, 0.L, -0.L, + 17 * static_cast<long double>(std::numeric_limits<double>::max()), + std::numeric_limits<long double>::infinity(), + -std::numeric_limits<long double>::infinity()))); +} + +TEST(HashValueTest, Pointer) { + EXPECT_TRUE((is_hashable<int*>::value)); + + int i; + int* ptr = &i; + int* n = nullptr; + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(&i, ptr, nullptr, ptr + 1, n))); +} + +TEST(HashValueTest, PointerAlignment) { + // We want to make sure that pointer alignment will not cause bits to be + // stuck. + + constexpr size_t kTotalSize = 1 << 20; + std::unique_ptr<char[]> data(new char[kTotalSize]); + constexpr size_t kLog2NumValues = 5; + constexpr size_t kNumValues = 1 << kLog2NumValues; + + for (size_t align = 1; align < kTotalSize / kNumValues; + align < 8 ? align += 1 : align < 1024 ? align += 8 : align += 32) { + SCOPED_TRACE(align); + ASSERT_LE(align * kNumValues, kTotalSize); + + size_t bits_or = 0; + size_t bits_and = ~size_t{}; + + for (size_t i = 0; i < kNumValues; ++i) { + size_t hash = absl::Hash<void*>()(data.get() + i * align); + bits_or |= hash; + bits_and &= hash; + } + + // Limit the scope to the bits we would be using for Swisstable. + constexpr size_t kMask = (1 << (kLog2NumValues + 7)) - 1; + size_t stuck_bits = (~bits_or | bits_and) & kMask; + EXPECT_EQ(stuck_bits, 0) << "0x" << std::hex << stuck_bits; + } +} + +TEST(HashValueTest, PairAndTuple) { + EXPECT_TRUE((is_hashable<std::pair<int, int>>::value)); + EXPECT_TRUE((is_hashable<std::pair<const int&, const int&>>::value)); + EXPECT_TRUE((is_hashable<std::tuple<int&, int&>>::value)); + EXPECT_TRUE((is_hashable<std::tuple<int&&, int&&>>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::make_pair(0, 42), std::make_pair(0, 42), std::make_pair(42, 0), + std::make_pair(0, 0), std::make_pair(42, 42), std::make_pair(1, 42)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(std::make_tuple(0, 0, 0), std::make_tuple(0, 0, 42), + std::make_tuple(0, 23, 0), std::make_tuple(17, 0, 0), + std::make_tuple(42, 0, 0), std::make_tuple(3, 9, 9), + std::make_tuple(0, 0, -42)))); + + // Test that tuples of lvalue references work (so we need a few lvalues): + int a = 0, b = 1, c = 17, d = 23; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::tie(a, a), std::tie(a, b), std::tie(b, c), std::tie(c, d)))); + + // Test that tuples of rvalue references work: + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::forward_as_tuple(0, 0, 0), std::forward_as_tuple(0, 0, 42), + std::forward_as_tuple(0, 23, 0), std::forward_as_tuple(17, 0, 0), + std::forward_as_tuple(42, 0, 0), std::forward_as_tuple(3, 9, 9), + std::forward_as_tuple(0, 0, -42)))); +} + +TEST(HashValueTest, CombineContiguousWorks) { + std::vector<std::tuple<int>> v1 = {std::make_tuple(1), std::make_tuple(3)}; + std::vector<std::tuple<int>> v2 = {std::make_tuple(1), std::make_tuple(2)}; + + auto vh1 = SpyHash(v1); + auto vh2 = SpyHash(v2); + EXPECT_NE(vh1, vh2); +} + +struct DummyDeleter { + template <typename T> + void operator() (T* ptr) {} +}; + +struct SmartPointerEq { + template <typename T, typename U> + bool operator()(const T& t, const U& u) const { + return GetPtr(t) == GetPtr(u); + } + + template <typename T> + static auto GetPtr(const T& t) -> decltype(&*t) { + return t ? &*t : nullptr; + } + + static std::nullptr_t GetPtr(std::nullptr_t) { return nullptr; } +}; + +TEST(HashValueTest, SmartPointers) { + EXPECT_TRUE((is_hashable<std::unique_ptr<int>>::value)); + EXPECT_TRUE((is_hashable<std::unique_ptr<int, DummyDeleter>>::value)); + EXPECT_TRUE((is_hashable<std::shared_ptr<int>>::value)); + + int i, j; + std::unique_ptr<int, DummyDeleter> unique1(&i); + std::unique_ptr<int, DummyDeleter> unique2(&i); + std::unique_ptr<int, DummyDeleter> unique_other(&j); + std::unique_ptr<int, DummyDeleter> unique_null; + + std::shared_ptr<int> shared1(&i, DummyDeleter()); + std::shared_ptr<int> shared2(&i, DummyDeleter()); + std::shared_ptr<int> shared_other(&j, DummyDeleter()); + std::shared_ptr<int> shared_null; + + // Sanity check of the Eq function. + ASSERT_TRUE(SmartPointerEq{}(unique1, shared1)); + ASSERT_FALSE(SmartPointerEq{}(unique1, shared_other)); + ASSERT_TRUE(SmartPointerEq{}(unique_null, nullptr)); + ASSERT_FALSE(SmartPointerEq{}(shared2, nullptr)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::forward_as_tuple(&i, nullptr, // + unique1, unique2, unique_null, // + absl::make_unique<int>(), // + shared1, shared2, shared_null, // + std::make_shared<int>()), + SmartPointerEq{})); +} + +TEST(HashValueTest, FunctionPointer) { + using Func = int (*)(); + EXPECT_TRUE(is_hashable<Func>::value); + + Func p1 = [] { return 2; }, p2 = [] { return 1; }; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(p1, p2, nullptr))); +} + +struct WrapInTuple { + template <typename T> + std::tuple<int, T, size_t> operator()(const T& t) const { + return std::make_tuple(7, t, 0xdeadbeef); + } +}; + +absl::Cord FlatCord(absl::string_view sv) { + absl::Cord c(sv); + c.Flatten(); + return c; +} + +absl::Cord FragmentedCord(absl::string_view sv) { + if (sv.size() < 2) { + return absl::Cord(sv); + } + size_t halfway = sv.size() / 2; + std::vector<absl::string_view> parts = {sv.substr(0, halfway), + sv.substr(halfway)}; + return absl::MakeFragmentedCord(parts); +} + +TEST(HashValueTest, Strings) { + EXPECT_TRUE((is_hashable<std::string>::value)); + + const std::string small = "foo"; + const std::string dup = "foofoo"; + const std::string large = std::string(2048, 'x'); // multiple of chunk size + const std::string huge = std::string(5000, 'a'); // not a multiple + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( // + std::string(), absl::string_view(), absl::Cord(), // + std::string(""), absl::string_view(""), absl::Cord(""), // + std::string(small), absl::string_view(small), absl::Cord(small), // + std::string(dup), absl::string_view(dup), absl::Cord(dup), // + std::string(large), absl::string_view(large), absl::Cord(large), // + std::string(huge), absl::string_view(huge), FlatCord(huge), // + FragmentedCord(huge)))); + + // Also check that nested types maintain the same hash. + const WrapInTuple t{}; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( // + t(std::string()), t(absl::string_view()), t(absl::Cord()), // + t(std::string("")), t(absl::string_view("")), t(absl::Cord("")), // + t(std::string(small)), t(absl::string_view(small)), // + t(absl::Cord(small)), // + t(std::string(dup)), t(absl::string_view(dup)), t(absl::Cord(dup)), // + t(std::string(large)), t(absl::string_view(large)), // + t(absl::Cord(large)), // + t(std::string(huge)), t(absl::string_view(huge)), // + t(FlatCord(huge)), t(FragmentedCord(huge))))); + + // Make sure that hashing a `const char*` does not use its string-value. + EXPECT_NE(SpyHash(static_cast<const char*>("ABC")), + SpyHash(absl::string_view("ABC"))); +} + +TEST(HashValueTest, WString) { + EXPECT_TRUE((is_hashable<std::wstring>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::wstring(), std::wstring(L"ABC"), std::wstring(L"ABC"), + std::wstring(L"Some other different string"), + std::wstring(L"Iñtërnâtiônàlizætiøn")))); +} + +TEST(HashValueTest, U16String) { + EXPECT_TRUE((is_hashable<std::u16string>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::u16string(), std::u16string(u"ABC"), std::u16string(u"ABC"), + std::u16string(u"Some other different string"), + std::u16string(u"Iñtërnâtiônàlizætiøn")))); +} + +TEST(HashValueTest, U32String) { + EXPECT_TRUE((is_hashable<std::u32string>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + std::u32string(), std::u32string(U"ABC"), std::u32string(U"ABC"), + std::u32string(U"Some other different string"), + std::u32string(U"Iñtërnâtiônàlizætiøn")))); +} + +TEST(HashValueTest, StdArray) { + EXPECT_TRUE((is_hashable<std::array<int, 3>>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(std::array<int, 3>{}, std::array<int, 3>{{0, 23, 42}}))); +} + +TEST(HashValueTest, StdBitset) { + EXPECT_TRUE((is_hashable<std::bitset<257>>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + {std::bitset<2>("00"), std::bitset<2>("01"), std::bitset<2>("10"), + std::bitset<2>("11")})); + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + {std::bitset<5>("10101"), std::bitset<5>("10001"), std::bitset<5>()})); + + constexpr int kNumBits = 256; + std::array<std::string, 6> bit_strings; + bit_strings.fill(std::string(kNumBits, '1')); + bit_strings[1][0] = '0'; + bit_strings[2][1] = '0'; + bit_strings[3][kNumBits / 3] = '0'; + bit_strings[4][kNumBits - 2] = '0'; + bit_strings[5][kNumBits - 1] = '0'; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + {std::bitset<kNumBits>(bit_strings[0].c_str()), + std::bitset<kNumBits>(bit_strings[1].c_str()), + std::bitset<kNumBits>(bit_strings[2].c_str()), + std::bitset<kNumBits>(bit_strings[3].c_str()), + std::bitset<kNumBits>(bit_strings[4].c_str()), + std::bitset<kNumBits>(bit_strings[5].c_str())})); +} // namespace + +template <typename T> +class HashValueSequenceTest : public testing::Test { +}; +TYPED_TEST_SUITE_P(HashValueSequenceTest); + +TYPED_TEST_P(HashValueSequenceTest, BasicUsage) { + EXPECT_TRUE((is_hashable<TypeParam>::value)); + + using ValueType = typename TypeParam::value_type; + auto a = static_cast<ValueType>(0); + auto b = static_cast<ValueType>(23); + auto c = static_cast<ValueType>(42); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(TypeParam(), TypeParam{}, TypeParam{a, b, c}, + TypeParam{a, b}, TypeParam{b, c}))); +} + +REGISTER_TYPED_TEST_CASE_P(HashValueSequenceTest, BasicUsage); +using IntSequenceTypes = + testing::Types<std::deque<int>, std::forward_list<int>, std::list<int>, + std::vector<int>, std::vector<bool>, std::set<int>, + std::multiset<int>>; +INSTANTIATE_TYPED_TEST_CASE_P(My, HashValueSequenceTest, IntSequenceTypes); + +// Private type that only supports AbslHashValue to make sure our chosen hash +// implementation is recursive within absl::Hash. +// It uses std::abs() on the value to provide different bitwise representations +// of the same logical value. +struct Private { + int i; + template <typename H> + friend H AbslHashValue(H h, Private p) { + return H::combine(std::move(h), std::abs(p.i)); + } + + friend bool operator==(Private a, Private b) { + return std::abs(a.i) == std::abs(b.i); + } + + friend std::ostream& operator<<(std::ostream& o, Private p) { + return o << p.i; + } +}; + +// Test helper for combine_piecewise_buffer. It holds a string_view to the +// buffer-to-be-hashed. Its AbslHashValue specialization will split up its +// contents at the character offsets requested. +class PiecewiseHashTester { + public: + // Create a hash view of a buffer to be hashed contiguously. + explicit PiecewiseHashTester(absl::string_view buf) + : buf_(buf), piecewise_(false), split_locations_() {} + + // Create a hash view of a buffer to be hashed piecewise, with breaks at the + // given locations. + PiecewiseHashTester(absl::string_view buf, std::set<size_t> split_locations) + : buf_(buf), + piecewise_(true), + split_locations_(std::move(split_locations)) {} + + template <typename H> + friend H AbslHashValue(H h, const PiecewiseHashTester& p) { + if (!p.piecewise_) { + return H::combine_contiguous(std::move(h), p.buf_.data(), p.buf_.size()); + } + absl::hash_internal::PiecewiseCombiner combiner; + if (p.split_locations_.empty()) { + h = combiner.add_buffer(std::move(h), p.buf_.data(), p.buf_.size()); + return combiner.finalize(std::move(h)); + } + size_t begin = 0; + for (size_t next : p.split_locations_) { + absl::string_view chunk = p.buf_.substr(begin, next - begin); + h = combiner.add_buffer(std::move(h), chunk.data(), chunk.size()); + begin = next; + } + absl::string_view last_chunk = p.buf_.substr(begin); + if (!last_chunk.empty()) { + h = combiner.add_buffer(std::move(h), last_chunk.data(), + last_chunk.size()); + } + return combiner.finalize(std::move(h)); + } + + private: + absl::string_view buf_; + bool piecewise_; + std::set<size_t> split_locations_; +}; + +// Dummy object that hashes as two distinct contiguous buffers, "foo" followed +// by "bar" +struct DummyFooBar { + template <typename H> + friend H AbslHashValue(H h, const DummyFooBar&) { + const char* foo = "foo"; + const char* bar = "bar"; + h = H::combine_contiguous(std::move(h), foo, 3); + h = H::combine_contiguous(std::move(h), bar, 3); + return h; + } +}; + +TEST(HashValueTest, CombinePiecewiseBuffer) { + absl::Hash<PiecewiseHashTester> hash; + + // Check that hashing an empty buffer through the piecewise API works. + EXPECT_EQ(hash(PiecewiseHashTester("")), hash(PiecewiseHashTester("", {}))); + + // Similarly, small buffers should give consistent results + EXPECT_EQ(hash(PiecewiseHashTester("foobar")), + hash(PiecewiseHashTester("foobar", {}))); + EXPECT_EQ(hash(PiecewiseHashTester("foobar")), + hash(PiecewiseHashTester("foobar", {3}))); + + // But hashing "foobar" in pieces gives a different answer than hashing "foo" + // contiguously, then "bar" contiguously. + EXPECT_NE(hash(PiecewiseHashTester("foobar", {3})), + absl::Hash<DummyFooBar>()(DummyFooBar{})); + + // Test hashing a large buffer incrementally, broken up in several different + // ways. Arrange for breaks on and near the stride boundaries to look for + // off-by-one errors in the implementation. + // + // This test is run on a buffer that is a multiple of the stride size, and one + // that isn't. + for (size_t big_buffer_size : {1024 * 2 + 512, 1024 * 3}) { + SCOPED_TRACE(big_buffer_size); + std::string big_buffer; + for (int i = 0; i < big_buffer_size; ++i) { + // Arbitrary string + big_buffer.push_back(32 + (i * (i / 3)) % 64); + } + auto big_buffer_hash = hash(PiecewiseHashTester(big_buffer)); + + const int possible_breaks = 9; + size_t breaks[possible_breaks] = {1, 512, 1023, 1024, 1025, + 1536, 2047, 2048, 2049}; + for (unsigned test_mask = 0; test_mask < (1u << possible_breaks); + ++test_mask) { + SCOPED_TRACE(test_mask); + std::set<size_t> break_locations; + for (int j = 0; j < possible_breaks; ++j) { + if (test_mask & (1u << j)) { + break_locations.insert(breaks[j]); + } + } + EXPECT_EQ( + hash(PiecewiseHashTester(big_buffer, std::move(break_locations))), + big_buffer_hash); + } + } +} + +TEST(HashValueTest, PrivateSanity) { + // Sanity check that Private is working as the tests below expect it to work. + EXPECT_TRUE(is_hashable<Private>::value); + EXPECT_NE(SpyHash(Private{0}), SpyHash(Private{1})); + EXPECT_EQ(SpyHash(Private{1}), SpyHash(Private{1})); +} + +TEST(HashValueTest, Optional) { + EXPECT_TRUE(is_hashable<absl::optional<Private>>::value); + + using O = absl::optional<Private>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(O{}, O{{1}}, O{{-1}}, O{{10}}))); +} + +TEST(HashValueTest, Variant) { + using V = absl::variant<Private, std::string>; + EXPECT_TRUE(is_hashable<V>::value); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + V(Private{1}), V(Private{-1}), V(Private{2}), V("ABC"), V("BCD")))); + +#if ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + struct S {}; + EXPECT_FALSE(is_hashable<absl::variant<S>>::value); +#endif +} + +TEST(HashValueTest, Maps) { + EXPECT_TRUE((is_hashable<std::map<int, std::string>>::value)); + + using M = std::map<int, std::string>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + M{}, M{{0, "foo"}}, M{{1, "foo"}}, M{{0, "bar"}}, M{{1, "bar"}}, + M{{0, "foo"}, {42, "bar"}}, M{{1, "foo"}, {42, "bar"}}, + M{{1, "foo"}, {43, "bar"}}, M{{1, "foo"}, {43, "baz"}}))); + + using MM = std::multimap<int, std::string>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + MM{}, MM{{0, "foo"}}, MM{{1, "foo"}}, MM{{0, "bar"}}, MM{{1, "bar"}}, + MM{{0, "foo"}, {0, "bar"}}, MM{{0, "bar"}, {0, "foo"}}, + MM{{0, "foo"}, {42, "bar"}}, MM{{1, "foo"}, {42, "bar"}}, + MM{{1, "foo"}, {1, "foo"}, {43, "bar"}}, MM{{1, "foo"}, {43, "baz"}}))); +} + +TEST(HashValueTest, ReferenceWrapper) { + EXPECT_TRUE(is_hashable<std::reference_wrapper<Private>>::value); + + Private p1{1}, p10{10}; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + p1, p10, std::ref(p1), std::ref(p10), std::cref(p1), std::cref(p10)))); + + EXPECT_TRUE(is_hashable<std::reference_wrapper<int>>::value); + int one = 1, ten = 10; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + one, ten, std::ref(one), std::ref(ten), std::cref(one), std::cref(ten)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(std::tuple<std::reference_wrapper<int>>(std::ref(one)), + std::tuple<std::reference_wrapper<int>>(std::ref(ten)), + std::tuple<int>(one), std::tuple<int>(ten)))); +} + +template <typename T, typename = void> +struct IsHashCallable : std::false_type {}; + +template <typename T> +struct IsHashCallable<T, absl::void_t<decltype(std::declval<absl::Hash<T>>()( + std::declval<const T&>()))>> : std::true_type {}; + +template <typename T, typename = void> +struct IsAggregateInitializable : std::false_type {}; + +template <typename T> +struct IsAggregateInitializable<T, absl::void_t<decltype(T{})>> + : std::true_type {}; + +TEST(IsHashableTest, ValidHash) { + EXPECT_TRUE((is_hashable<int>::value)); + EXPECT_TRUE(std::is_default_constructible<absl::Hash<int>>::value); + EXPECT_TRUE(std::is_copy_constructible<absl::Hash<int>>::value); + EXPECT_TRUE(std::is_move_constructible<absl::Hash<int>>::value); + EXPECT_TRUE(absl::is_copy_assignable<absl::Hash<int>>::value); + EXPECT_TRUE(absl::is_move_assignable<absl::Hash<int>>::value); + EXPECT_TRUE(IsHashCallable<int>::value); + EXPECT_TRUE(IsAggregateInitializable<absl::Hash<int>>::value); +} + +#if ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ +TEST(IsHashableTest, PoisonHash) { + struct X {}; + EXPECT_FALSE((is_hashable<X>::value)); + EXPECT_FALSE(std::is_default_constructible<absl::Hash<X>>::value); + EXPECT_FALSE(std::is_copy_constructible<absl::Hash<X>>::value); + EXPECT_FALSE(std::is_move_constructible<absl::Hash<X>>::value); + EXPECT_FALSE(absl::is_copy_assignable<absl::Hash<X>>::value); + EXPECT_FALSE(absl::is_move_assignable<absl::Hash<X>>::value); + EXPECT_FALSE(IsHashCallable<X>::value); +#if !defined(__GNUC__) || __GNUC__ < 9 + // This doesn't compile on GCC 9. + EXPECT_FALSE(IsAggregateInitializable<absl::Hash<X>>::value); +#endif +} +#endif // ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + +// Hashable types +// +// These types exist simply to exercise various AbslHashValue behaviors, so +// they are named by what their AbslHashValue overload does. +struct NoOp { + template <typename HashCode> + friend HashCode AbslHashValue(HashCode h, NoOp n) { + return h; + } +}; + +struct EmptyCombine { + template <typename HashCode> + friend HashCode AbslHashValue(HashCode h, EmptyCombine e) { + return HashCode::combine(std::move(h)); + } +}; + +template <typename Int> +struct CombineIterative { + template <typename HashCode> + friend HashCode AbslHashValue(HashCode h, CombineIterative c) { + for (int i = 0; i < 5; ++i) { + h = HashCode::combine(std::move(h), Int(i)); + } + return h; + } +}; + +template <typename Int> +struct CombineVariadic { + template <typename HashCode> + friend HashCode AbslHashValue(HashCode h, CombineVariadic c) { + return HashCode::combine(std::move(h), Int(0), Int(1), Int(2), Int(3), + Int(4)); + } +}; +enum class InvokeTag { + kUniquelyRepresented, + kHashValue, +#if ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ + kLegacyHash, +#endif // ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ + kStdHash, + kNone +}; + +template <InvokeTag T> +using InvokeTagConstant = std::integral_constant<InvokeTag, T>; + +template <InvokeTag... Tags> +struct MinTag; + +template <InvokeTag a, InvokeTag b, InvokeTag... Tags> +struct MinTag<a, b, Tags...> : MinTag<(a < b ? a : b), Tags...> {}; + +template <InvokeTag a> +struct MinTag<a> : InvokeTagConstant<a> {}; + +template <InvokeTag... Tags> +struct CustomHashType { + explicit CustomHashType(size_t val) : value(val) {} + size_t value; +}; + +template <InvokeTag allowed, InvokeTag... tags> +struct EnableIfContained + : std::enable_if<absl::disjunction< + std::integral_constant<bool, allowed == tags>...>::value> {}; + +template < + typename H, InvokeTag... Tags, + typename = typename EnableIfContained<InvokeTag::kHashValue, Tags...>::type> +H AbslHashValue(H state, CustomHashType<Tags...> t) { + static_assert(MinTag<Tags...>::value == InvokeTag::kHashValue, ""); + return H::combine(std::move(state), + t.value + static_cast<int>(InvokeTag::kHashValue)); +} + +} // namespace + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { +template <InvokeTag... Tags> +struct is_uniquely_represented< + CustomHashType<Tags...>, + typename EnableIfContained<InvokeTag::kUniquelyRepresented, Tags...>::type> + : std::true_type {}; +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl + +#if ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ +namespace ABSL_INTERNAL_LEGACY_HASH_NAMESPACE { +template <InvokeTag... Tags> +struct hash<CustomHashType<Tags...>> { + template <InvokeTag... TagsIn, typename = typename EnableIfContained< + InvokeTag::kLegacyHash, TagsIn...>::type> + size_t operator()(CustomHashType<TagsIn...> t) const { + static_assert(MinTag<Tags...>::value == InvokeTag::kLegacyHash, ""); + return t.value + static_cast<int>(InvokeTag::kLegacyHash); + } +}; +} // namespace ABSL_INTERNAL_LEGACY_HASH_NAMESPACE +#endif // ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ + +namespace std { +template <InvokeTag... Tags> // NOLINT +struct hash<CustomHashType<Tags...>> { + template <InvokeTag... TagsIn, typename = typename EnableIfContained< + InvokeTag::kStdHash, TagsIn...>::type> + size_t operator()(CustomHashType<TagsIn...> t) const { + static_assert(MinTag<Tags...>::value == InvokeTag::kStdHash, ""); + return t.value + static_cast<int>(InvokeTag::kStdHash); + } +}; +} // namespace std + +namespace { + +template <typename... T> +void TestCustomHashType(InvokeTagConstant<InvokeTag::kNone>, T...) { + using type = CustomHashType<T::value...>; + SCOPED_TRACE(testing::PrintToString(std::vector<InvokeTag>{T::value...})); + EXPECT_TRUE(is_hashable<type>()); + EXPECT_TRUE(is_hashable<const type>()); + EXPECT_TRUE(is_hashable<const type&>()); + + const size_t offset = static_cast<int>(std::min({T::value...})); + EXPECT_EQ(SpyHash(type(7)), SpyHash(size_t{7 + offset})); +} + +void TestCustomHashType(InvokeTagConstant<InvokeTag::kNone>) { +#if ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + // is_hashable is false if we don't support any of the hooks. + using type = CustomHashType<>; + EXPECT_FALSE(is_hashable<type>()); + EXPECT_FALSE(is_hashable<const type>()); + EXPECT_FALSE(is_hashable<const type&>()); +#endif // ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ +} + +template <InvokeTag Tag, typename... T> +void TestCustomHashType(InvokeTagConstant<Tag> tag, T... t) { + constexpr auto next = static_cast<InvokeTag>(static_cast<int>(Tag) + 1); + TestCustomHashType(InvokeTagConstant<next>(), tag, t...); + TestCustomHashType(InvokeTagConstant<next>(), t...); +} + +TEST(HashTest, CustomHashType) { + TestCustomHashType(InvokeTagConstant<InvokeTag{}>()); +} + +TEST(HashTest, NoOpsAreEquivalent) { + EXPECT_EQ(Hash<NoOp>()({}), Hash<NoOp>()({})); + EXPECT_EQ(Hash<NoOp>()({}), Hash<EmptyCombine>()({})); +} + +template <typename T> +class HashIntTest : public testing::Test { +}; +TYPED_TEST_SUITE_P(HashIntTest); + +TYPED_TEST_P(HashIntTest, BasicUsage) { + EXPECT_NE(Hash<NoOp>()({}), Hash<TypeParam>()(0)); + EXPECT_NE(Hash<NoOp>()({}), + Hash<TypeParam>()(std::numeric_limits<TypeParam>::max())); + if (std::numeric_limits<TypeParam>::min() != 0) { + EXPECT_NE(Hash<NoOp>()({}), + Hash<TypeParam>()(std::numeric_limits<TypeParam>::min())); + } + + EXPECT_EQ(Hash<CombineIterative<TypeParam>>()({}), + Hash<CombineVariadic<TypeParam>>()({})); +} + +REGISTER_TYPED_TEST_CASE_P(HashIntTest, BasicUsage); +using IntTypes = testing::Types<unsigned char, char, int, int32_t, int64_t, + uint32_t, uint64_t, size_t>; +INSTANTIATE_TYPED_TEST_CASE_P(My, HashIntTest, IntTypes); + +struct StructWithPadding { + char c; + int i; + + template <typename H> + friend H AbslHashValue(H hash_state, const StructWithPadding& s) { + return H::combine(std::move(hash_state), s.c, s.i); + } +}; + +static_assert(sizeof(StructWithPadding) > sizeof(char) + sizeof(int), + "StructWithPadding doesn't have padding"); +static_assert(std::is_standard_layout<StructWithPadding>::value, ""); + +// This check has to be disabled because libstdc++ doesn't support it. +// static_assert(std::is_trivially_constructible<StructWithPadding>::value, ""); + +template <typename T> +struct ArraySlice { + T* begin; + T* end; + + template <typename H> + friend H AbslHashValue(H hash_state, const ArraySlice& slice) { + for (auto t = slice.begin; t != slice.end; ++t) { + hash_state = H::combine(std::move(hash_state), *t); + } + return hash_state; + } +}; + +TEST(HashTest, HashNonUniquelyRepresentedType) { + // Create equal StructWithPadding objects that are known to have non-equal + // padding bytes. + static const size_t kNumStructs = 10; + unsigned char buffer1[kNumStructs * sizeof(StructWithPadding)]; + std::memset(buffer1, 0, sizeof(buffer1)); + auto* s1 = reinterpret_cast<StructWithPadding*>(buffer1); + + unsigned char buffer2[kNumStructs * sizeof(StructWithPadding)]; + std::memset(buffer2, 255, sizeof(buffer2)); + auto* s2 = reinterpret_cast<StructWithPadding*>(buffer2); + for (int i = 0; i < kNumStructs; ++i) { + SCOPED_TRACE(i); + s1[i].c = s2[i].c = '0' + i; + s1[i].i = s2[i].i = i; + ASSERT_FALSE(memcmp(buffer1 + i * sizeof(StructWithPadding), + buffer2 + i * sizeof(StructWithPadding), + sizeof(StructWithPadding)) == 0) + << "Bug in test code: objects do not have unequal" + << " object representations"; + } + + EXPECT_EQ(Hash<StructWithPadding>()(s1[0]), Hash<StructWithPadding>()(s2[0])); + EXPECT_EQ(Hash<ArraySlice<StructWithPadding>>()({s1, s1 + kNumStructs}), + Hash<ArraySlice<StructWithPadding>>()({s2, s2 + kNumStructs})); +} + +TEST(HashTest, StandardHashContainerUsage) { + std::unordered_map<int, std::string, Hash<int>> map = {{0, "foo"}, + {42, "bar"}}; + + EXPECT_NE(map.find(0), map.end()); + EXPECT_EQ(map.find(1), map.end()); + EXPECT_NE(map.find(0u), map.end()); +} + +struct ConvertibleFromNoOp { + ConvertibleFromNoOp(NoOp) {} // NOLINT(runtime/explicit) + + template <typename H> + friend H AbslHashValue(H hash_state, ConvertibleFromNoOp) { + return H::combine(std::move(hash_state), 1); + } +}; + +TEST(HashTest, HeterogeneousCall) { + EXPECT_NE(Hash<ConvertibleFromNoOp>()(NoOp()), + Hash<NoOp>()(NoOp())); +} + +TEST(IsUniquelyRepresentedTest, SanityTest) { + using absl::hash_internal::is_uniquely_represented; + + EXPECT_TRUE(is_uniquely_represented<unsigned char>::value); + EXPECT_TRUE(is_uniquely_represented<int>::value); + EXPECT_FALSE(is_uniquely_represented<bool>::value); + EXPECT_FALSE(is_uniquely_represented<int*>::value); +} + +struct IntAndString { + int i; + std::string s; + + template <typename H> + friend H AbslHashValue(H hash_state, IntAndString int_and_string) { + return H::combine(std::move(hash_state), int_and_string.s, + int_and_string.i); + } +}; + +TEST(HashTest, SmallValueOn64ByteBoundary) { + Hash<IntAndString>()(IntAndString{0, std::string(63, '0')}); +} + +struct TypeErased { + size_t n; + + template <typename H> + friend H AbslHashValue(H hash_state, const TypeErased& v) { + v.HashValue(absl::HashState::Create(&hash_state)); + return hash_state; + } + + void HashValue(absl::HashState state) const { + absl::HashState::combine(std::move(state), n); + } +}; + +TEST(HashTest, TypeErased) { + EXPECT_TRUE((is_hashable<TypeErased>::value)); + EXPECT_TRUE((is_hashable<std::pair<TypeErased, int>>::value)); + + EXPECT_EQ(SpyHash(TypeErased{7}), SpyHash(size_t{7})); + EXPECT_NE(SpyHash(TypeErased{7}), SpyHash(size_t{13})); + + EXPECT_EQ(SpyHash(std::make_pair(TypeErased{7}, 17)), + SpyHash(std::make_pair(size_t{7}, 17))); +} + +struct ValueWithBoolConversion { + operator bool() const { return false; } + int i; +}; + +} // namespace +namespace std { +template <> +struct hash<ValueWithBoolConversion> { + size_t operator()(ValueWithBoolConversion v) { return v.i; } +}; +} // namespace std + +namespace { + +TEST(HashTest, DoesNotUseImplicitConversionsToBool) { + EXPECT_NE(absl::Hash<ValueWithBoolConversion>()(ValueWithBoolConversion{0}), + absl::Hash<ValueWithBoolConversion>()(ValueWithBoolConversion{1})); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/hash/hash_testing.h b/third_party/abseil_cpp/absl/hash/hash_testing.h new file mode 100644 index 000000000000..1e1c5741491e --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/hash_testing.h @@ -0,0 +1,378 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_HASH_HASH_TESTING_H_ +#define ABSL_HASH_HASH_TESTING_H_ + +#include <initializer_list> +#include <tuple> +#include <type_traits> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/hash/internal/spy_hash_state.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/str_cat.h" +#include "absl/types/variant.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Run the absl::Hash algorithm over all the elements passed in and verify that +// their hash expansion is congruent with their `==` operator. +// +// It is used in conjunction with EXPECT_TRUE. Failures will output information +// on what requirement failed and on which objects. +// +// Users should pass a collection of types as either an initializer list or a +// container of cases. +// +// EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( +// {v1, v2, ..., vN})); +// +// std::vector<MyType> cases; +// // Fill cases... +// EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(cases)); +// +// Users can pass a variety of types for testing heterogeneous lookup with +// `std::make_tuple`: +// +// EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( +// std::make_tuple(v1, v2, ..., vN))); +// +// +// Ideally, the values passed should provide enough coverage of the `==` +// operator and the AbslHashValue implementations. +// For dynamically sized types, the empty state should usually be included in +// the values. +// +// The function accepts an optional comparator function, in case that `==` is +// not enough for the values provided. +// +// Usage: +// +// EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( +// std::make_tuple(v1, v2, ..., vN), MyCustomEq{})); +// +// It checks the following requirements: +// 1. The expansion for a value is deterministic. +// 2. For any two objects `a` and `b` in the sequence, if `a == b` evaluates +// to true, then their hash expansion must be equal. +// 3. If `a == b` evaluates to false their hash expansion must be unequal. +// 4. If `a == b` evaluates to false neither hash expansion can be a +// suffix of the other. +// 5. AbslHashValue overloads should not be called by the user. They are only +// meant to be called by the framework. Users should call H::combine() and +// H::combine_contiguous(). +// 6. No moved-from instance of the hash state is used in the implementation +// of AbslHashValue. +// +// The values do not have to have the same type. This can be useful for +// equivalent types that support heterogeneous lookup. +// +// A possible reason for breaking (2) is combining state in the hash expansion +// that was not used in `==`. +// For example: +// +// struct Bad2 { +// int a, b; +// template <typename H> +// friend H AbslHashValue(H state, Bad2 x) { +// // Uses a and b. +// return H::combine(std::move(state), x.a, x.b); +// } +// friend bool operator==(Bad2 x, Bad2 y) { +// // Only uses a. +// return x.a == y.a; +// } +// }; +// +// As for (3), breaking this usually means that there is state being passed to +// the `==` operator that is not used in the hash expansion. +// For example: +// +// struct Bad3 { +// int a, b; +// template <typename H> +// friend H AbslHashValue(H state, Bad3 x) { +// // Only uses a. +// return H::combine(std::move(state), x.a); +// } +// friend bool operator==(Bad3 x, Bad3 y) { +// // Uses a and b. +// return x.a == y.a && x.b == y.b; +// } +// }; +// +// Finally, a common way to break 4 is by combining dynamic ranges without +// combining the size of the range. +// For example: +// +// struct Bad4 { +// int *p, size; +// template <typename H> +// friend H AbslHashValue(H state, Bad4 x) { +// return H::combine_contiguous(std::move(state), x.p, x.p + x.size); +// } +// friend bool operator==(Bad4 x, Bad4 y) { +// // Compare two ranges for equality. C++14 code can instead use std::equal. +// return absl::equal(x.p, x.p + x.size, y.p, y.p + y.size); +// } +// }; +// +// An easy solution to this is to combine the size after combining the range, +// like so: +// template <typename H> +// friend H AbslHashValue(H state, Bad4 x) { +// return H::combine( +// H::combine_contiguous(std::move(state), x.p, x.p + x.size), x.size); +// } +// +template <int&... ExplicitBarrier, typename Container> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(const Container& values); + +template <int&... ExplicitBarrier, typename Container, typename Eq> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(const Container& values, Eq equals); + +template <int&..., typename T> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(std::initializer_list<T> values); + +template <int&..., typename T, typename Eq> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(std::initializer_list<T> values, + Eq equals); + +namespace hash_internal { + +struct PrintVisitor { + size_t index; + template <typename T> + std::string operator()(const T* value) const { + return absl::StrCat("#", index, "(", testing::PrintToString(*value), ")"); + } +}; + +template <typename Eq> +struct EqVisitor { + Eq eq; + template <typename T, typename U> + bool operator()(const T* t, const U* u) const { + return eq(*t, *u); + } +}; + +struct ExpandVisitor { + template <typename T> + SpyHashState operator()(const T* value) const { + return SpyHashState::combine(SpyHashState(), *value); + } +}; + +template <typename Container, typename Eq> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(const Container& values, Eq equals) { + using V = typename Container::value_type; + + struct Info { + const V& value; + size_t index; + std::string ToString() const { + return absl::visit(PrintVisitor{index}, value); + } + SpyHashState expand() const { return absl::visit(ExpandVisitor{}, value); } + }; + + using EqClass = std::vector<Info>; + std::vector<EqClass> classes; + + // Gather the values in equivalence classes. + size_t i = 0; + for (const auto& value : values) { + EqClass* c = nullptr; + for (auto& eqclass : classes) { + if (absl::visit(EqVisitor<Eq>{equals}, value, eqclass[0].value)) { + c = &eqclass; + break; + } + } + if (c == nullptr) { + classes.emplace_back(); + c = &classes.back(); + } + c->push_back({value, i}); + ++i; + + // Verify potential errors captured by SpyHashState. + if (auto error = c->back().expand().error()) { + return testing::AssertionFailure() << *error; + } + } + + if (classes.size() < 2) { + return testing::AssertionFailure() + << "At least two equivalence classes are expected."; + } + + // We assume that equality is correctly implemented. + // Now we verify that AbslHashValue is also correctly implemented. + + for (const auto& c : classes) { + // All elements of the equivalence class must have the same hash + // expansion. + const SpyHashState expected = c[0].expand(); + for (const Info& v : c) { + if (v.expand() != v.expand()) { + return testing::AssertionFailure() + << "Hash expansion for " << v.ToString() + << " is non-deterministic."; + } + if (v.expand() != expected) { + return testing::AssertionFailure() + << "Values " << c[0].ToString() << " and " << v.ToString() + << " evaluate as equal but have an unequal hash expansion."; + } + } + + // Elements from other classes must have different hash expansion. + for (const auto& c2 : classes) { + if (&c == &c2) continue; + const SpyHashState c2_hash = c2[0].expand(); + switch (SpyHashState::Compare(expected, c2_hash)) { + case SpyHashState::CompareResult::kEqual: + return testing::AssertionFailure() + << "Values " << c[0].ToString() << " and " << c2[0].ToString() + << " evaluate as unequal but have an equal hash expansion."; + case SpyHashState::CompareResult::kBSuffixA: + return testing::AssertionFailure() + << "Hash expansion of " << c2[0].ToString() + << " is a suffix of the hash expansion of " << c[0].ToString() + << "."; + case SpyHashState::CompareResult::kASuffixB: + return testing::AssertionFailure() + << "Hash expansion of " << c[0].ToString() + << " is a suffix of the hash expansion of " << c2[0].ToString() + << "."; + case SpyHashState::CompareResult::kUnequal: + break; + } + } + } + return testing::AssertionSuccess(); +} + +template <typename... T> +struct TypeSet { + template <typename U, bool = disjunction<std::is_same<T, U>...>::value> + struct Insert { + using type = TypeSet<U, T...>; + }; + template <typename U> + struct Insert<U, true> { + using type = TypeSet; + }; + + template <template <typename...> class C> + using apply = C<T...>; +}; + +template <typename... T> +struct MakeTypeSet : TypeSet<> {}; +template <typename T, typename... Ts> +struct MakeTypeSet<T, Ts...> : MakeTypeSet<Ts...>::template Insert<T>::type {}; + +template <typename... T> +using VariantForTypes = typename MakeTypeSet< + const typename std::decay<T>::type*...>::template apply<absl::variant>; + +template <typename Container> +struct ContainerAsVector { + using V = absl::variant<const typename Container::value_type*>; + using Out = std::vector<V>; + + static Out Do(const Container& values) { + Out out; + for (const auto& v : values) out.push_back(&v); + return out; + } +}; + +template <typename... T> +struct ContainerAsVector<std::tuple<T...>> { + using V = VariantForTypes<T...>; + using Out = std::vector<V>; + + template <size_t... I> + static Out DoImpl(const std::tuple<T...>& tuple, absl::index_sequence<I...>) { + return Out{&std::get<I>(tuple)...}; + } + + static Out Do(const std::tuple<T...>& values) { + return DoImpl(values, absl::index_sequence_for<T...>()); + } +}; + +template <> +struct ContainerAsVector<std::tuple<>> { + static std::vector<VariantForTypes<int>> Do(std::tuple<>) { return {}; } +}; + +struct DefaultEquals { + template <typename T, typename U> + bool operator()(const T& t, const U& u) const { + return t == u; + } +}; + +} // namespace hash_internal + +template <int&..., typename Container> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(const Container& values) { + return hash_internal::VerifyTypeImplementsAbslHashCorrectly( + hash_internal::ContainerAsVector<Container>::Do(values), + hash_internal::DefaultEquals{}); +} + +template <int&..., typename Container, typename Eq> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(const Container& values, Eq equals) { + return hash_internal::VerifyTypeImplementsAbslHashCorrectly( + hash_internal::ContainerAsVector<Container>::Do(values), equals); +} + +template <int&..., typename T> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(std::initializer_list<T> values) { + return hash_internal::VerifyTypeImplementsAbslHashCorrectly( + hash_internal::ContainerAsVector<std::initializer_list<T>>::Do(values), + hash_internal::DefaultEquals{}); +} + +template <int&..., typename T, typename Eq> +ABSL_MUST_USE_RESULT testing::AssertionResult +VerifyTypeImplementsAbslHashCorrectly(std::initializer_list<T> values, + Eq equals) { + return hash_internal::VerifyTypeImplementsAbslHashCorrectly( + hash_internal::ContainerAsVector<std::initializer_list<T>>::Do(values), + equals); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HASH_HASH_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/hash/internal/city.cc b/third_party/abseil_cpp/absl/hash/internal/city.cc new file mode 100644 index 000000000000..5460134e5762 --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/city.cc @@ -0,0 +1,349 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file provides CityHash64() and related functions. +// +// It's probably possible to create even faster hash functions by +// writing a program that systematically explores some of the space of +// possible hash functions, by using SIMD instructions, or by +// compromising on hash quality. + +#include "absl/hash/internal/city.h" + +#include <string.h> // for memcpy and memset +#include <algorithm> + +#include "absl/base/config.h" +#include "absl/base/internal/endian.h" +#include "absl/base/internal/unaligned_access.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +#ifdef ABSL_IS_BIG_ENDIAN +#define uint32_in_expected_order(x) (absl::gbswap_32(x)) +#define uint64_in_expected_order(x) (absl::gbswap_64(x)) +#else +#define uint32_in_expected_order(x) (x) +#define uint64_in_expected_order(x) (x) +#endif + +static uint64_t Fetch64(const char *p) { + return uint64_in_expected_order(ABSL_INTERNAL_UNALIGNED_LOAD64(p)); +} + +static uint32_t Fetch32(const char *p) { + return uint32_in_expected_order(ABSL_INTERNAL_UNALIGNED_LOAD32(p)); +} + +// Some primes between 2^63 and 2^64 for various uses. +static const uint64_t k0 = 0xc3a5c85c97cb3127ULL; +static const uint64_t k1 = 0xb492b66fbe98f273ULL; +static const uint64_t k2 = 0x9ae16a3b2f90404fULL; + +// Magic numbers for 32-bit hashing. Copied from Murmur3. +static const uint32_t c1 = 0xcc9e2d51; +static const uint32_t c2 = 0x1b873593; + +// A 32-bit to 32-bit integer hash copied from Murmur3. +static uint32_t fmix(uint32_t h) { + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +static uint32_t Rotate32(uint32_t val, int shift) { + // Avoid shifting by 32: doing so yields an undefined result. + return shift == 0 ? val : ((val >> shift) | (val << (32 - shift))); +} + +#undef PERMUTE3 +#define PERMUTE3(a, b, c) \ + do { \ + std::swap(a, b); \ + std::swap(a, c); \ + } while (0) + +static uint32_t Mur(uint32_t a, uint32_t h) { + // Helper from Murmur3 for combining two 32-bit values. + a *= c1; + a = Rotate32(a, 17); + a *= c2; + h ^= a; + h = Rotate32(h, 19); + return h * 5 + 0xe6546b64; +} + +static uint32_t Hash32Len13to24(const char *s, size_t len) { + uint32_t a = Fetch32(s - 4 + (len >> 1)); + uint32_t b = Fetch32(s + 4); + uint32_t c = Fetch32(s + len - 8); + uint32_t d = Fetch32(s + (len >> 1)); + uint32_t e = Fetch32(s); + uint32_t f = Fetch32(s + len - 4); + uint32_t h = len; + + return fmix(Mur(f, Mur(e, Mur(d, Mur(c, Mur(b, Mur(a, h))))))); +} + +static uint32_t Hash32Len0to4(const char *s, size_t len) { + uint32_t b = 0; + uint32_t c = 9; + for (size_t i = 0; i < len; i++) { + signed char v = s[i]; + b = b * c1 + v; + c ^= b; + } + return fmix(Mur(b, Mur(len, c))); +} + +static uint32_t Hash32Len5to12(const char *s, size_t len) { + uint32_t a = len, b = len * 5, c = 9, d = b; + a += Fetch32(s); + b += Fetch32(s + len - 4); + c += Fetch32(s + ((len >> 1) & 4)); + return fmix(Mur(c, Mur(b, Mur(a, d)))); +} + +uint32_t CityHash32(const char *s, size_t len) { + if (len <= 24) { + return len <= 12 + ? (len <= 4 ? Hash32Len0to4(s, len) : Hash32Len5to12(s, len)) + : Hash32Len13to24(s, len); + } + + // len > 24 + uint32_t h = len, g = c1 * len, f = g; + + uint32_t a0 = Rotate32(Fetch32(s + len - 4) * c1, 17) * c2; + uint32_t a1 = Rotate32(Fetch32(s + len - 8) * c1, 17) * c2; + uint32_t a2 = Rotate32(Fetch32(s + len - 16) * c1, 17) * c2; + uint32_t a3 = Rotate32(Fetch32(s + len - 12) * c1, 17) * c2; + uint32_t a4 = Rotate32(Fetch32(s + len - 20) * c1, 17) * c2; + h ^= a0; + h = Rotate32(h, 19); + h = h * 5 + 0xe6546b64; + h ^= a2; + h = Rotate32(h, 19); + h = h * 5 + 0xe6546b64; + g ^= a1; + g = Rotate32(g, 19); + g = g * 5 + 0xe6546b64; + g ^= a3; + g = Rotate32(g, 19); + g = g * 5 + 0xe6546b64; + f += a4; + f = Rotate32(f, 19); + f = f * 5 + 0xe6546b64; + size_t iters = (len - 1) / 20; + do { + uint32_t b0 = Rotate32(Fetch32(s) * c1, 17) * c2; + uint32_t b1 = Fetch32(s + 4); + uint32_t b2 = Rotate32(Fetch32(s + 8) * c1, 17) * c2; + uint32_t b3 = Rotate32(Fetch32(s + 12) * c1, 17) * c2; + uint32_t b4 = Fetch32(s + 16); + h ^= b0; + h = Rotate32(h, 18); + h = h * 5 + 0xe6546b64; + f += b1; + f = Rotate32(f, 19); + f = f * c1; + g += b2; + g = Rotate32(g, 18); + g = g * 5 + 0xe6546b64; + h ^= b3 + b1; + h = Rotate32(h, 19); + h = h * 5 + 0xe6546b64; + g ^= b4; + g = absl::gbswap_32(g) * 5; + h += b4 * 5; + h = absl::gbswap_32(h); + f += b0; + PERMUTE3(f, h, g); + s += 20; + } while (--iters != 0); + g = Rotate32(g, 11) * c1; + g = Rotate32(g, 17) * c1; + f = Rotate32(f, 11) * c1; + f = Rotate32(f, 17) * c1; + h = Rotate32(h + g, 19); + h = h * 5 + 0xe6546b64; + h = Rotate32(h, 17) * c1; + h = Rotate32(h + f, 19); + h = h * 5 + 0xe6546b64; + h = Rotate32(h, 17) * c1; + return h; +} + +// Bitwise right rotate. Normally this will compile to a single +// instruction, especially if the shift is a manifest constant. +static uint64_t Rotate(uint64_t val, int shift) { + // Avoid shifting by 64: doing so yields an undefined result. + return shift == 0 ? val : ((val >> shift) | (val << (64 - shift))); +} + +static uint64_t ShiftMix(uint64_t val) { return val ^ (val >> 47); } + +static uint64_t HashLen16(uint64_t u, uint64_t v, uint64_t mul) { + // Murmur-inspired hashing. + uint64_t a = (u ^ v) * mul; + a ^= (a >> 47); + uint64_t b = (v ^ a) * mul; + b ^= (b >> 47); + b *= mul; + return b; +} + +static uint64_t HashLen16(uint64_t u, uint64_t v) { + const uint64_t kMul = 0x9ddfea08eb382d69ULL; + return HashLen16(u, v, kMul); +} + +static uint64_t HashLen0to16(const char *s, size_t len) { + if (len >= 8) { + uint64_t mul = k2 + len * 2; + uint64_t a = Fetch64(s) + k2; + uint64_t b = Fetch64(s + len - 8); + uint64_t c = Rotate(b, 37) * mul + a; + uint64_t d = (Rotate(a, 25) + b) * mul; + return HashLen16(c, d, mul); + } + if (len >= 4) { + uint64_t mul = k2 + len * 2; + uint64_t a = Fetch32(s); + return HashLen16(len + (a << 3), Fetch32(s + len - 4), mul); + } + if (len > 0) { + uint8_t a = s[0]; + uint8_t b = s[len >> 1]; + uint8_t c = s[len - 1]; + uint32_t y = static_cast<uint32_t>(a) + (static_cast<uint32_t>(b) << 8); + uint32_t z = len + (static_cast<uint32_t>(c) << 2); + return ShiftMix(y * k2 ^ z * k0) * k2; + } + return k2; +} + +// This probably works well for 16-byte strings as well, but it may be overkill +// in that case. +static uint64_t HashLen17to32(const char *s, size_t len) { + uint64_t mul = k2 + len * 2; + uint64_t a = Fetch64(s) * k1; + uint64_t b = Fetch64(s + 8); + uint64_t c = Fetch64(s + len - 8) * mul; + uint64_t d = Fetch64(s + len - 16) * k2; + return HashLen16(Rotate(a + b, 43) + Rotate(c, 30) + d, + a + Rotate(b + k2, 18) + c, mul); +} + +// Return a 16-byte hash for 48 bytes. Quick and dirty. +// Callers do best to use "random-looking" values for a and b. +static std::pair<uint64_t, uint64_t> WeakHashLen32WithSeeds( + uint64_t w, uint64_t x, uint64_t y, uint64_t z, uint64_t a, uint64_t b) { + a += w; + b = Rotate(b + a + z, 21); + uint64_t c = a; + a += x; + a += y; + b += Rotate(a, 44); + return std::make_pair(a + z, b + c); +} + +// Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty. +static std::pair<uint64_t, uint64_t> WeakHashLen32WithSeeds(const char *s, + uint64_t a, + uint64_t b) { + return WeakHashLen32WithSeeds(Fetch64(s), Fetch64(s + 8), Fetch64(s + 16), + Fetch64(s + 24), a, b); +} + +// Return an 8-byte hash for 33 to 64 bytes. +static uint64_t HashLen33to64(const char *s, size_t len) { + uint64_t mul = k2 + len * 2; + uint64_t a = Fetch64(s) * k2; + uint64_t b = Fetch64(s + 8); + uint64_t c = Fetch64(s + len - 24); + uint64_t d = Fetch64(s + len - 32); + uint64_t e = Fetch64(s + 16) * k2; + uint64_t f = Fetch64(s + 24) * 9; + uint64_t g = Fetch64(s + len - 8); + uint64_t h = Fetch64(s + len - 16) * mul; + uint64_t u = Rotate(a + g, 43) + (Rotate(b, 30) + c) * 9; + uint64_t v = ((a + g) ^ d) + f + 1; + uint64_t w = absl::gbswap_64((u + v) * mul) + h; + uint64_t x = Rotate(e + f, 42) + c; + uint64_t y = (absl::gbswap_64((v + w) * mul) + g) * mul; + uint64_t z = e + f + c; + a = absl::gbswap_64((x + z) * mul + y) + b; + b = ShiftMix((z + a) * mul + d + h) * mul; + return b + x; +} + +uint64_t CityHash64(const char *s, size_t len) { + if (len <= 32) { + if (len <= 16) { + return HashLen0to16(s, len); + } else { + return HashLen17to32(s, len); + } + } else if (len <= 64) { + return HashLen33to64(s, len); + } + + // For strings over 64 bytes we hash the end first, and then as we + // loop we keep 56 bytes of state: v, w, x, y, and z. + uint64_t x = Fetch64(s + len - 40); + uint64_t y = Fetch64(s + len - 16) + Fetch64(s + len - 56); + uint64_t z = HashLen16(Fetch64(s + len - 48) + len, Fetch64(s + len - 24)); + std::pair<uint64_t, uint64_t> v = + WeakHashLen32WithSeeds(s + len - 64, len, z); + std::pair<uint64_t, uint64_t> w = + WeakHashLen32WithSeeds(s + len - 32, y + k1, x); + x = x * k1 + Fetch64(s); + + // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks. + len = (len - 1) & ~static_cast<size_t>(63); + do { + x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1; + y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1; + x ^= w.second; + y += v.first + Fetch64(s + 40); + z = Rotate(z + w.first, 33) * k1; + v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first); + w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16)); + std::swap(z, x); + s += 64; + len -= 64; + } while (len != 0); + return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z, + HashLen16(v.second, w.second) + x); +} + +uint64_t CityHash64WithSeed(const char *s, size_t len, uint64_t seed) { + return CityHash64WithSeeds(s, len, k2, seed); +} + +uint64_t CityHash64WithSeeds(const char *s, size_t len, uint64_t seed0, + uint64_t seed1) { + return HashLen16(CityHash64(s, len) - seed0, seed1); +} + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/hash/internal/city.h b/third_party/abseil_cpp/absl/hash/internal/city.h new file mode 100644 index 000000000000..393da0b95d9f --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/city.h @@ -0,0 +1,78 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// https://code.google.com/p/cityhash/ +// +// This file provides a few functions for hashing strings. All of them are +// high-quality functions in the sense that they pass standard tests such +// as Austin Appleby's SMHasher. They are also fast. +// +// For 64-bit x86 code, on short strings, we don't know of anything faster than +// CityHash64 that is of comparable quality. We believe our nearest competitor +// is Murmur3. For 64-bit x86 code, CityHash64 is an excellent choice for hash +// tables and most other hashing (excluding cryptography). +// +// For 32-bit x86 code, we don't know of anything faster than CityHash32 that +// is of comparable quality. We believe our nearest competitor is Murmur3A. +// (On 64-bit CPUs, it is typically faster to use the other CityHash variants.) +// +// Functions in the CityHash family are not suitable for cryptography. +// +// Please see CityHash's README file for more details on our performance +// measurements and so on. +// +// WARNING: This code has been only lightly tested on big-endian platforms! +// It is known to work well on little-endian platforms that have a small penalty +// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs. +// It should work on all 32-bit and 64-bit platforms that allow unaligned reads; +// bug reports are welcome. +// +// By the way, for some hash functions, given strings a and b, the hash +// of a+b is easily derived from the hashes of a and b. This property +// doesn't hold for any hash functions in this file. + +#ifndef ABSL_HASH_INTERNAL_CITY_H_ +#define ABSL_HASH_INTERNAL_CITY_H_ + +#include <stdint.h> +#include <stdlib.h> // for size_t. + +#include <utility> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +// Hash function for a byte array. +uint64_t CityHash64(const char *s, size_t len); + +// Hash function for a byte array. For convenience, a 64-bit seed is also +// hashed into the result. +uint64_t CityHash64WithSeed(const char *s, size_t len, uint64_t seed); + +// Hash function for a byte array. For convenience, two seeds are also +// hashed into the result. +uint64_t CityHash64WithSeeds(const char *s, size_t len, uint64_t seed0, + uint64_t seed1); + +// Hash function for a byte array. Most useful in 32-bit binaries. +uint32_t CityHash32(const char *s, size_t len); + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HASH_INTERNAL_CITY_H_ diff --git a/third_party/abseil_cpp/absl/hash/internal/city_test.cc b/third_party/abseil_cpp/absl/hash/internal/city_test.cc new file mode 100644 index 000000000000..251d381d73e1 --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/city_test.cc @@ -0,0 +1,595 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/hash/internal/city.h" + +#include <string.h> +#include <cstdio> +#include <iostream> +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +static const uint64_t k0 = 0xc3a5c85c97cb3127ULL; +static const uint64_t kSeed0 = 1234567; +static const uint64_t kSeed1 = k0; +static const int kDataSize = 1 << 20; +static const int kTestSize = 300; + +static char data[kDataSize]; + +// Initialize data to pseudorandom values. +void setup() { + uint64_t a = 9; + uint64_t b = 777; + for (int i = 0; i < kDataSize; i++) { + a += b; + b += a; + a = (a ^ (a >> 41)) * k0; + b = (b ^ (b >> 41)) * k0 + i; + uint8_t u = b >> 37; + memcpy(data + i, &u, 1); // uint8_t -> char + } +} + +#define C(x) 0x##x##ULL +static const uint64_t testdata[kTestSize][4] = { + {C(9ae16a3b2f90404f), C(75106db890237a4a), C(3feac5f636039766), + C(dc56d17a)}, + {C(541150e87f415e96), C(1aef0d24b3148a1a), C(bacc300e1e82345a), + C(99929334)}, + {C(f3786a4b25827c1), C(34ee1a2bf767bd1c), C(2f15ca2ebfb631f2), C(4252edb7)}, + {C(ef923a7a1af78eab), C(79163b1e1e9a9b18), C(df3b2aca6e1e4a30), + C(ebc34f3c)}, + {C(11df592596f41d88), C(843ec0bce9042f9c), C(cce2ea1e08b1eb30), + C(26f2b463)}, + {C(831f448bdc5600b3), C(62a24be3120a6919), C(1b44098a41e010da), + C(b042c047)}, + {C(3eca803e70304894), C(d80de767e4a920a), C(a51cfbb292efd53d), C(e73bb0a8)}, + {C(1b5a063fb4c7f9f1), C(318dbc24af66dee9), C(10ef7b32d5c719af), + C(91dfdd75)}, + {C(a0f10149a0e538d6), C(69d008c20f87419f), C(41b36376185b3e9e), + C(c87f95de)}, + {C(fb8d9c70660b910b), C(a45b0cc3476bff1b), C(b28d1996144f0207), + C(3f5538ef)}, + {C(236827beae282a46), C(e43970221139c946), C(4f3ac6faa837a3aa), + C(70eb1a1f)}, + {C(c385e435136ecf7c), C(d9d17368ff6c4a08), C(1b31eed4e5251a67), + C(cfd63b83)}, + {C(e3f6828b6017086d), C(21b4d1900554b3b0), C(bef38be1809e24f1), + C(894a52ef)}, + {C(851fff285561dca0), C(4d1277d73cdf416f), C(28ccffa61010ebe2), + C(9cde6a54)}, + {C(61152a63595a96d9), C(d1a3a91ef3a7ba45), C(443b6bb4a493ad0c), + C(6c4898d5)}, + {C(44473e03be306c88), C(30097761f872472a), C(9fd1b669bfad82d7), + C(13e1978e)}, + {C(3ead5f21d344056), C(fb6420393cfb05c3), C(407932394cbbd303), C(51b4ba8)}, + {C(6abbfde37ee03b5b), C(83febf188d2cc113), C(cda7b62d94d5b8ee), + C(b6b06e40)}, + {C(943e7ed63b3c080), C(1ef207e9444ef7f8), C(ef4a9f9f8c6f9b4a), C(240a2f2)}, + {C(d72ce05171ef8a1a), C(c6bd6bd869203894), C(c760e6396455d23a), + C(5dcefc30)}, + {C(4182832b52d63735), C(337097e123eea414), C(b5a72ca0456df910), + C(7a48b105)}, + {C(d6cdae892584a2cb), C(58de0fa4eca17dcd), C(43df30b8f5f1cb00), + C(fd55007b)}, + {C(5c8e90bc267c5ee4), C(e9ae044075d992d9), C(f234cbfd1f0a1e59), + C(6b95894c)}, + {C(bbd7f30ac310a6f3), C(b23b570d2666685f), C(fb13fb08c9814fe7), + C(3360e827)}, + {C(36a097aa49519d97), C(8204380a73c4065), C(77c2004bdd9e276a), C(45177e0b)}, + {C(dc78cb032c49217), C(112464083f83e03a), C(96ae53e28170c0f5), C(7c6fffe4)}, + {C(441593e0da922dfe), C(936ef46061469b32), C(204a1921197ddd87), + C(bbc78da4)}, + {C(2ba3883d71cc2133), C(72f2bbb32bed1a3c), C(27e1bd96d4843251), + C(c5c25d39)}, + {C(f2b6d2adf8423600), C(7514e2f016a48722), C(43045743a50396ba), + C(b6e5d06e)}, + {C(38fffe7f3680d63c), C(d513325255a7a6d1), C(31ed47790f6ca62f), + C(6178504e)}, + {C(b7477bf0b9ce37c6), C(63b1c580a7fd02a4), C(f6433b9f10a5dac), C(bd4c3637)}, + {C(55bdb0e71e3edebd), C(c7ab562bcf0568bc), C(43166332f9ee684f), + C(6e7ac474)}, + {C(782fa1b08b475e7), C(fb7138951c61b23b), C(9829105e234fb11e), C(1fb4b518)}, + {C(c5dc19b876d37a80), C(15ffcff666cfd710), C(e8c30c72003103e2), + C(31d13d6d)}, + {C(5e1141711d2d6706), C(b537f6dee8de6933), C(3af0a1fbbe027c54), + C(26fa72e3)}, + {C(782edf6da001234f), C(f48cbd5c66c48f3), C(808754d1e64e2a32), C(6a7433bf)}, + {C(d26285842ff04d44), C(8f38d71341eacca9), C(5ca436f4db7a883c), + C(4e6df758)}, + {C(c6ab830865a6bae6), C(6aa8e8dd4b98815c), C(efe3846713c371e5), + C(d57f63ea)}, + {C(44b3a1929232892), C(61dca0e914fc217), C(a607cc142096b964), C(52ef73b3)}, + {C(4b603d7932a8de4f), C(fae64c464b8a8f45), C(8fafab75661d602a), C(3cb36c3)}, + {C(4ec0b54cf1566aff), C(30d2c7269b206bf4), C(77c22e82295e1061), + C(72c39bea)}, + {C(ed8b7a4b34954ff7), C(56432de31f4ee757), C(85bd3abaa572b155), + C(a65aa25c)}, + {C(5d28b43694176c26), C(714cc8bc12d060ae), C(3437726273a83fe6), + C(74740539)}, + {C(6a1ef3639e1d202e), C(919bc1bd145ad928), C(30f3f7e48c28a773), + C(c3ae3c26)}, + {C(159f4d9e0307b111), C(3e17914a5675a0c), C(af849bd425047b51), C(f29db8a2)}, + {C(cc0a840725a7e25b), C(57c69454396e193a), C(976eaf7eee0b4540), + C(1ef4cbf4)}, + {C(a2b27ee22f63c3f1), C(9ebde0ce1b3976b2), C(2fe6a92a257af308), + C(a9be6c41)}, + {C(d8f2f234899bcab3), C(b10b037297c3a168), C(debea2c510ceda7f), C(fa31801)}, + {C(584f28543864844f), C(d7cee9fc2d46f20d), C(a38dca5657387205), + C(8331c5d8)}, + {C(a94be46dd9aa41af), C(a57e5b7723d3f9bd), C(34bf845a52fd2f), C(e9876db8)}, + {C(9a87bea227491d20), C(a468657e2b9c43e7), C(af9ba60db8d89ef7), + C(27b0604e)}, + {C(27688c24958d1a5c), C(e3b4a1c9429cf253), C(48a95811f70d64bc), + C(dcec07f2)}, + {C(5d1d37790a1873ad), C(ed9cd4bcc5fa1090), C(ce51cde05d8cd96a), + C(cff0a82a)}, + {C(1f03fd18b711eea9), C(566d89b1946d381a), C(6e96e83fc92563ab), + C(fec83621)}, + {C(f0316f286cf527b6), C(f84c29538de1aa5a), C(7612ed3c923d4a71), C(743d8dc)}, + {C(297008bcb3e3401d), C(61a8e407f82b0c69), C(a4a35bff0524fa0e), + C(64d41d26)}, + {C(43c6252411ee3be), C(b4ca1b8077777168), C(2746dc3f7da1737f), C(acd90c81)}, + {C(ce38a9a54fad6599), C(6d6f4a90b9e8755e), C(c3ecc79ff105de3f), + C(7c746a4b)}, + {C(270a9305fef70cf), C(600193999d884f3a), C(f4d49eae09ed8a1), C(b1047e99)}, + {C(e71be7c28e84d119), C(eb6ace59932736e6), C(70c4397807ba12c5), + C(d1fd1068)}, + {C(b5b58c24b53aaa19), C(d2a6ab0773dd897f), C(ef762fe01ecb5b97), + C(56486077)}, + {C(44dd59bd301995cf), C(3ccabd76493ada1a), C(540db4c87d55ef23), + C(6069be80)}, + {C(b4d4789eb6f2630b), C(bf6973263ce8ef0e), C(d1c75c50844b9d3), C(2078359b)}, + {C(12807833c463737c), C(58e927ea3b3776b4), C(72dd20ef1c2f8ad0), + C(9ea21004)}, + {C(e88419922b87176f), C(bcf32f41a7ddbf6f), C(d6ebefd8085c1a0f), + C(9c9cfe88)}, + {C(105191e0ec8f7f60), C(5918dbfcca971e79), C(6b285c8a944767b9), + C(b70a6ddd)}, + {C(a5b88bf7399a9f07), C(fca3ddfd96461cc4), C(ebe738fdc0282fc6), + C(dea37298)}, + {C(d08c3f5747d84f50), C(4e708b27d1b6f8ac), C(70f70fd734888606), + C(8f480819)}, + {C(2f72d12a40044b4b), C(889689352fec53de), C(f03e6ad87eb2f36), C(30b3b16)}, + {C(aa1f61fdc5c2e11e), C(c2c56cd11277ab27), C(a1e73069fdf1f94f), + C(f31bc4e8)}, + {C(9489b36fe2246244), C(3355367033be74b8), C(5f57c2277cbce516), + C(419f953b)}, + {C(358d7c0476a044cd), C(e0b7b47bcbd8854f), C(ffb42ec696705519), + C(20e9e76d)}, + {C(b0c48df14275265a), C(9da4448975905efa), C(d716618e414ceb6d), + C(646f0ff8)}, + {C(daa70bb300956588), C(410ea6883a240c6d), C(f5c8239fb5673eb3), + C(eeb7eca8)}, + {C(4ec97a20b6c4c7c2), C(5913b1cd454f29fd), C(a9629f9daf06d685), C(8112bb9)}, + {C(5c3323628435a2e8), C(1bea45ce9e72a6e3), C(904f0a7027ddb52e), + C(85a6d477)}, + {C(c1ef26bea260abdb), C(6ee423f2137f9280), C(df2118b946ed0b43), + C(56f76c84)}, + {C(6be7381b115d653a), C(ed046190758ea511), C(de6a45ffc3ed1159), + C(9af45d55)}, + {C(ae3eece1711b2105), C(14fd3f4027f81a4a), C(abb7e45177d151db), + C(d1c33760)}, + {C(376c28588b8fb389), C(6b045e84d8491ed2), C(4e857effb7d4e7dc), + C(c56bbf69)}, + {C(58d943503bb6748f), C(419c6c8e88ac70f6), C(586760cbf3d3d368), + C(abecfb9b)}, + {C(dfff5989f5cfd9a1), C(bcee2e7ea3a96f83), C(681c7874adb29017), + C(8de13255)}, + {C(7fb19eb1a496e8f5), C(d49e5dfdb5c0833f), C(c0d5d7b2f7c48dc7), + C(a98ee299)}, + {C(5dba5b0dadccdbaa), C(4ba8da8ded87fcdc), C(f693fdd25badf2f0), + C(3015f556)}, + {C(688bef4b135a6829), C(8d31d82abcd54e8e), C(f95f8a30d55036d7), + C(5a430e29)}, + {C(d8323be05433a412), C(8d48fa2b2b76141d), C(3d346f23978336a5), + C(2797add0)}, + {C(3b5404278a55a7fc), C(23ca0b327c2d0a81), C(a6d65329571c892c), + C(27d55016)}, + {C(2a96a3f96c5e9bbc), C(8caf8566e212dda8), C(904de559ca16e45e), + C(84945a82)}, + {C(22bebfdcc26d18ff), C(4b4d8dcb10807ba1), C(40265eee30c6b896), + C(3ef7e224)}, + {C(627a2249ec6bbcc2), C(c0578b462a46735a), C(4974b8ee1c2d4f1f), + C(35ed8dc8)}, + {C(3abaf1667ba2f3e0), C(ee78476b5eeadc1), C(7e56ac0a6ca4f3f4), C(6a75e43d)}, + {C(3931ac68c5f1b2c9), C(efe3892363ab0fb0), C(40b707268337cd36), + C(235d9805)}, + {C(b98fb0606f416754), C(46a6e5547ba99c1e), C(c909d82112a8ed2), C(f7d69572)}, + {C(7f7729a33e58fcc4), C(2e4bc1e7a023ead4), C(e707008ea7ca6222), + C(bacd0199)}, + {C(42a0aa9ce82848b3), C(57232730e6bee175), C(f89bb3f370782031), + C(e428f50e)}, + {C(6b2c6d38408a4889), C(de3ef6f68fb25885), C(20754f456c203361), + C(81eaaad3)}, + {C(930380a3741e862a), C(348d28638dc71658), C(89dedcfd1654ea0d), + C(addbd3e3)}, + {C(94808b5d2aa25f9a), C(cec72968128195e0), C(d9f4da2bdc1e130f), + C(e66dbca0)}, + {C(b31abb08ae6e3d38), C(9eb9a95cbd9e8223), C(8019e79b7ee94ea9), + C(afe11fd5)}, + {C(dccb5534a893ea1a), C(ce71c398708c6131), C(fe2396315457c164), + C(a71a406f)}, + {C(6369163565814de6), C(8feb86fb38d08c2f), C(4976933485cc9a20), + C(9d90eaf5)}, + {C(edee4ff253d9f9b3), C(96ef76fb279ef0ad), C(a4d204d179db2460), + C(6665db10)}, + {C(941993df6e633214), C(929bc1beca5b72c6), C(141fc52b8d55572d), + C(9c977cbf)}, + {C(859838293f64cd4c), C(484403b39d44ad79), C(bf674e64d64b9339), + C(ee83ddd4)}, + {C(c19b5648e0d9f555), C(328e47b2b7562993), C(e756b92ba4bd6a51), C(26519cc)}, + {C(f963b63b9006c248), C(9e9bf727ffaa00bc), C(c73bacc75b917e3a), + C(a485a53f)}, + {C(6a8aa0852a8c1f3b), C(c8f1e5e206a21016), C(2aa554aed1ebb524), + C(f62bc412)}, + {C(740428b4d45e5fb8), C(4c95a4ce922cb0a5), C(e99c3ba78feae796), + C(8975a436)}, + {C(658b883b3a872b86), C(2f0e303f0f64827a), C(975337e23dc45e1), C(94ff7f41)}, + {C(6df0a977da5d27d4), C(891dd0e7cb19508), C(fd65434a0b71e680), C(760aa031)}, + {C(a900275464ae07ef), C(11f2cfda34beb4a3), C(9abf91e5a1c38e4), C(3bda76df)}, + {C(810bc8aa0c40bcb0), C(448a019568d01441), C(f60ec52f60d3aeae), + C(498e2e65)}, + {C(22036327deb59ed7), C(adc05ceb97026a02), C(48bff0654262672b), + C(d38deb48)}, + {C(7d14dfa9772b00c8), C(595735efc7eeaed7), C(29872854f94c3507), + C(82b3fb6b)}, + {C(2d777cddb912675d), C(278d7b10722a13f9), C(f5c02bfb7cc078af), + C(e500e25f)}, + {C(f2ec98824e8aa613), C(5eb7e3fb53fe3bed), C(12c22860466e1dd4), + C(bd2bb07c)}, + {C(5e763988e21f487f), C(24189de8065d8dc5), C(d1519d2403b62aa0), + C(3a2b431d)}, + {C(48949dc327bb96ad), C(e1fd21636c5c50b4), C(3f6eb7f13a8712b4), + C(7322a83d)}, + {C(b7c4209fb24a85c5), C(b35feb319c79ce10), C(f0d3de191833b922), + C(a645ca1c)}, + {C(9c9e5be0943d4b05), C(b73dc69e45201cbb), C(aab17180bfe5083d), + C(8909a45a)}, + {C(3898bca4dfd6638d), C(f911ff35efef0167), C(24bdf69e5091fc88), + C(bd30074c)}, + {C(5b5d2557400e68e7), C(98d610033574cee), C(dfd08772ce385deb), C(c17cf001)}, + {C(a927ed8b2bf09bb6), C(606e52f10ae94eca), C(71c2203feb35a9ee), + C(26ffd25a)}, + {C(8d25746414aedf28), C(34b1629d28b33d3a), C(4d5394aea5f82d7b), + C(f1d8ce3c)}, + {C(b5bbdb73458712f2), C(1ff887b3c2a35137), C(7f7231f702d0ace9), + C(3ee8fb17)}, + {C(3d32a26e3ab9d254), C(fc4070574dc30d3a), C(f02629579c2b27c9), + C(a77acc2a)}, + {C(9371d3c35fa5e9a5), C(42967cf4d01f30), C(652d1eeae704145c), C(f4556dee)}, + {C(cbaa3cb8f64f54e0), C(76c3b48ee5c08417), C(9f7d24e87e61ce9), C(de287a64)}, + {C(b2e23e8116c2ba9f), C(7e4d9c0060101151), C(3310da5e5028f367), + C(878e55b9)}, + {C(8aa77f52d7868eb9), C(4d55bd587584e6e2), C(d2db37041f495f5), C(7648486)}, + {C(858fea922c7fe0c3), C(cfe8326bf733bc6f), C(4e5e2018cf8f7dfc), + C(57ac0fb1)}, + {C(46ef25fdec8392b1), C(e48d7b6d42a5cd35), C(56a6fe1c175299ca), + C(d01967ca)}, + {C(8d078f726b2df464), C(b50ee71cdcabb299), C(f4af300106f9c7ba), + C(96ecdf74)}, + {C(35ea86e6960ca950), C(34fe1fe234fc5c76), C(a00207a3dc2a72b7), + C(779f5506)}, + {C(8aee9edbc15dd011), C(51f5839dc8462695), C(b2213e17c37dca2d), + C(3c94c2de)}, + {C(c3e142ba98432dda), C(911d060cab126188), C(b753fbfa8365b844), + C(39f98faf)}, + {C(123ba6b99c8cd8db), C(448e582672ee07c4), C(cebe379292db9e65), + C(7af31199)}, + {C(ba87acef79d14f53), C(b3e0fcae63a11558), C(d5ac313a593a9f45), + C(e341a9d6)}, + {C(bcd3957d5717dc3), C(2da746741b03a007), C(873816f4b1ece472), C(ca24aeeb)}, + {C(61442ff55609168e), C(6447c5fc76e8c9cf), C(6a846de83ae15728), + C(b2252b57)}, + {C(dbe4b1b2d174757f), C(506512da18712656), C(6857f3e0b8dd95f), C(72c81da1)}, + {C(531e8e77b363161c), C(eece0b43e2dae030), C(8294b82c78f34ed1), + C(6b9fce95)}, + {C(f71e9c926d711e2b), C(d77af2853a4ceaa1), C(9aa0d6d76a36fae7), + C(19399857)}, + {C(cb20ac28f52df368), C(e6705ee7880996de), C(9b665cc3ec6972f2), + C(3c57a994)}, + {C(e4a794b4acb94b55), C(89795358057b661b), C(9c4cdcec176d7a70), + C(c053e729)}, + {C(cb942e91443e7208), C(e335de8125567c2a), C(d4d74d268b86df1f), + C(51cbbba7)}, + {C(ecca7563c203f7ba), C(177ae2423ef34bb2), C(f60b7243400c5731), + C(1acde79a)}, + {C(1652cb940177c8b5), C(8c4fe7d85d2a6d6d), C(f6216ad097e54e72), + C(2d160d13)}, + {C(31fed0fc04c13ce8), C(3d5d03dbf7ff240a), C(727c5c9b51581203), + C(787f5801)}, + {C(e7b668947590b9b3), C(baa41ad32938d3fa), C(abcbc8d4ca4b39e4), + C(c9629828)}, + {C(1de2119923e8ef3c), C(6ab27c096cf2fe14), C(8c3658edca958891), + C(be139231)}, + {C(1269df1e69e14fa7), C(992f9d58ac5041b7), C(e97fcf695a7cbbb4), + C(7df699ef)}, + {C(820826d7aba567ff), C(1f73d28e036a52f3), C(41c4c5a73f3b0893), + C(8ce6b96d)}, + {C(ffe0547e4923cef9), C(3534ed49b9da5b02), C(548a273700fba03d), + C(6f9ed99c)}, + {C(72da8d1b11d8bc8b), C(ba94b56b91b681c6), C(4e8cc51bd9b0fc8c), + C(e0244796)}, + {C(d62ab4e3f88fc797), C(ea86c7aeb6283ae4), C(b5b93e09a7fe465), C(4ccf7e75)}, + {C(d0f06c28c7b36823), C(1008cb0874de4bb8), C(d6c7ff816c7a737b), + C(915cef86)}, + {C(99b7042460d72ec6), C(2a53e5e2b8e795c2), C(53a78132d9e1b3e3), + C(5cb59482)}, + {C(4f4dfcfc0ec2bae5), C(841233148268a1b8), C(9248a76ab8be0d3), C(6ca3f532)}, + {C(fe86bf9d4422b9ae), C(ebce89c90641ef9c), C(1c84e2292c0b5659), + C(e24f3859)}, + {C(a90d81060932dbb0), C(8acfaa88c5fbe92b), C(7c6f3447e90f7f3f), + C(adf5a9c7)}, + {C(17938a1b0e7f5952), C(22cadd2f56f8a4be), C(84b0d1183d5ed7c1), + C(32264b75)}, + {C(de9e0cb0e16f6e6d), C(238e6283aa4f6594), C(4fb9c914c2f0a13b), + C(a64b3376)}, + {C(6d4b876d9b146d1a), C(aab2d64ce8f26739), C(d315f93600e83fe5), C(d33890e)}, + {C(e698fa3f54e6ea22), C(bd28e20e7455358c), C(9ace161f6ea76e66), + C(926d4b63)}, + {C(7bc0deed4fb349f7), C(1771aff25dc722fa), C(19ff0644d9681917), + C(d51ba539)}, + {C(db4b15e88533f622), C(256d6d2419b41ce9), C(9d7c5378396765d5), + C(7f37636d)}, + {C(922834735e86ecb2), C(363382685b88328e), C(e9c92960d7144630), + C(b98026c0)}, + {C(30f1d72c812f1eb8), C(b567cd4a69cd8989), C(820b6c992a51f0bc), + C(b877767e)}, + {C(168884267f3817e9), C(5b376e050f637645), C(1c18314abd34497a), C(aefae77)}, + {C(82e78596ee3e56a7), C(25697d9c87f30d98), C(7600a8342834924d), C(f686911)}, + {C(aa2d6cf22e3cc252), C(9b4dec4f5e179f16), C(76fb0fba1d99a99a), + C(3deadf12)}, + {C(7bf5ffd7f69385c7), C(fc077b1d8bc82879), C(9c04e36f9ed83a24), + C(ccf02a4e)}, + {C(e89c8ff9f9c6e34b), C(f54c0f669a49f6c4), C(fc3e46f5d846adef), + C(176c1722)}, + {C(a18fbcdccd11e1f4), C(8248216751dfd65e), C(40c089f208d89d7c), C(26f82ad)}, + {C(2d54f40cc4088b17), C(59d15633b0cd1399), C(a8cc04bb1bffd15b), + C(b5244f42)}, + {C(69276946cb4e87c7), C(62bdbe6183be6fa9), C(3ba9773dac442a1a), + C(49a689e5)}, + {C(668174a3f443df1d), C(407299392da1ce86), C(c2a3f7d7f2c5be28), C(59fcdd3)}, + {C(5e29be847bd5046), C(b561c7f19c8f80c3), C(5e5abd5021ccaeaf), C(4f4b04e9)}, + {C(cd0d79f2164da014), C(4c386bb5c5d6ca0c), C(8e771b03647c3b63), + C(8b00f891)}, + {C(e0e6fc0b1628af1d), C(29be5fb4c27a2949), C(1c3f781a604d3630), + C(16e114f3)}, + {C(2058927664adfd93), C(6e8f968c7963baa5), C(af3dced6fff7c394), + C(d6b6dadc)}, + {C(dc107285fd8e1af7), C(a8641a0609321f3f), C(db06e89ffdc54466), + C(897e20ac)}, + {C(fbba1afe2e3280f1), C(755a5f392f07fce), C(9e44a9a15402809a), C(f996e05d)}, + {C(bfa10785ddc1011b), C(b6e1c4d2f670f7de), C(517d95604e4fcc1f), + C(c4306af6)}, + {C(534cc35f0ee1eb4e), C(b703820f1f3b3dce), C(884aa164cf22363), C(6dcad433)}, + {C(7ca6e3933995dac), C(fd118c77daa8188), C(3aceb7b5e7da6545), C(3c07374d)}, + {C(f0d6044f6efd7598), C(e044d6ba4369856e), C(91968e4f8c8a1a4c), + C(f0f4602c)}, + {C(3d69e52049879d61), C(76610636ea9f74fe), C(e9bf5602f89310c0), + C(3e1ea071)}, + {C(79da242a16acae31), C(183c5f438e29d40), C(6d351710ae92f3de), C(67580f0c)}, + {C(461c82656a74fb57), C(d84b491b275aa0f7), C(8f262cb29a6eb8b2), + C(4e109454)}, + {C(53c1a66d0b13003), C(731f060e6fe797fc), C(daa56811791371e3), C(88a474a7)}, + {C(d3a2efec0f047e9), C(1cabce58853e58ea), C(7a17b2eae3256be4), C(5b5bedd)}, + {C(43c64d7484f7f9b2), C(5da002b64aafaeb7), C(b576c1e45800a716), + C(1aaddfa7)}, + {C(a7dec6ad81cf7fa1), C(180c1ab708683063), C(95e0fd7008d67cff), + C(5be07fd8)}, + {C(5408a1df99d4aff), C(b9565e588740f6bd), C(abf241813b08006e), C(cbca8606)}, + {C(a8b27a6bcaeeed4b), C(aec1eeded6a87e39), C(9daf246d6fed8326), + C(bde64d01)}, + {C(9a952a8246fdc269), C(d0dcfcac74ef278c), C(250f7139836f0f1f), + C(ee90cf33)}, + {C(c930841d1d88684f), C(5eb66eb18b7f9672), C(e455d413008a2546), + C(4305c3ce)}, + {C(94dc6971e3cf071a), C(994c7003b73b2b34), C(ea16e85978694e5), C(4b3a1d76)}, + {C(7fc98006e25cac9), C(77fee0484cda86a7), C(376ec3d447060456), C(a8bb6d80)}, + {C(bd781c4454103f6), C(612197322f49c931), C(b9cf17fd7e5462d5), C(1f9fa607)}, + {C(da60e6b14479f9df), C(3bdccf69ece16792), C(18ebf45c4fecfdc9), + C(8d0e4ed2)}, + {C(4ca56a348b6c4d3), C(60618537c3872514), C(2fbb9f0e65871b09), C(1bf31347)}, + {C(ebd22d4b70946401), C(6863602bf7139017), C(c0b1ac4e11b00666), + C(1ae3fc5b)}, + {C(3cc4693d6cbcb0c), C(501689ea1c70ffa), C(10a4353e9c89e364), C(459c3930)}, + {C(38908e43f7ba5ef0), C(1ab035d4e7781e76), C(41d133e8c0a68ff7), + C(e00c4184)}, + {C(34983ccc6aa40205), C(21802cad34e72bc4), C(1943e8fb3c17bb8), C(ffc7a781)}, + {C(86215c45dcac9905), C(ea546afe851cae4b), C(d85b6457e489e374), + C(6a125480)}, + {C(420fc255c38db175), C(d503cd0f3c1208d1), C(d4684e74c825a0bc), + C(88a1512b)}, + {C(1d7a31f5bc8fe2f9), C(4763991092dcf836), C(ed695f55b97416f4), + C(549bbbe5)}, + {C(94129a84c376a26e), C(c245e859dc231933), C(1b8f74fecf917453), + C(c133d38c)}, + {C(1d3a9809dab05c8d), C(adddeb4f71c93e8), C(ef342eb36631edb), C(fcace348)}, + {C(90fa3ccbd60848da), C(dfa6e0595b569e11), C(e585d067a1f5135d), + C(ed7b6f9a)}, + {C(2dbb4fc71b554514), C(9650e04b86be0f82), C(60f2304fba9274d3), + C(6d907dda)}, + {C(b98bf4274d18374a), C(1b669fd4c7f9a19a), C(b1f5972b88ba2b7a), + C(7a4d48d5)}, + {C(d6781d0b5e18eb68), C(b992913cae09b533), C(58f6021caaee3a40), + C(e686f3db)}, + {C(226651cf18f4884c), C(595052a874f0f51c), C(c9b75162b23bab42), C(cce7c55)}, + {C(a734fb047d3162d6), C(e523170d240ba3a5), C(125a6972809730e8), C(f58b96b)}, + {C(c6df6364a24f75a3), C(c294e2c84c4f5df8), C(a88df65c6a89313b), + C(1bbf6f60)}, + {C(d8d1364c1fbcd10), C(2d7cc7f54832deaa), C(4e22c876a7c57625), C(ce5e0cc2)}, + {C(aae06f9146db885f), C(3598736441e280d9), C(fba339b117083e55), + C(584cfd6f)}, + {C(8955ef07631e3bcc), C(7d70965ea3926f83), C(39aed4134f8b2db6), + C(8f9bbc33)}, + {C(ad611c609cfbe412), C(d3c00b18bf253877), C(90b2172e1f3d0bfd), + C(d7640d95)}, + {C(d5339adc295d5d69), C(b633cc1dcb8b586a), C(ee84184cf5b1aeaf), C(3d12a2b)}, + {C(40d0aeff521375a8), C(77ba1ad7ecebd506), C(547c6f1a7d9df427), + C(aaeafed0)}, + {C(8b2d54ae1a3df769), C(11e7adaee3216679), C(3483781efc563e03), + C(95b9b814)}, + {C(99c175819b4eae28), C(932e8ff9f7a40043), C(ec78dcab07ca9f7c), + C(45fbe66e)}, + {C(2a418335779b82fc), C(af0295987849a76b), C(c12bc5ff0213f46e), + C(b4baa7a8)}, + {C(3b1fc6a3d279e67d), C(70ea1e49c226396), C(25505adcf104697c), C(83e962fe)}, + {C(d97eacdf10f1c3c9), C(b54f4654043a36e0), C(b128f6eb09d1234), C(aac3531c)}, + {C(293a5c1c4e203cd4), C(6b3329f1c130cefe), C(f2e32f8ec76aac91), + C(2b1db7cc)}, + {C(4290e018ffaedde7), C(a14948545418eb5e), C(72d851b202284636), + C(cf00cd31)}, + {C(f919a59cbde8bf2f), C(a56d04203b2dc5a5), C(38b06753ac871e48), + C(7d3c43b8)}, + {C(1d70a3f5521d7fa4), C(fb97b3fdc5891965), C(299d49bbbe3535af), + C(cbd5fac6)}, + {C(6af98d7b656d0d7c), C(d2e99ae96d6b5c0c), C(f63bd1603ef80627), + C(76d0fec4)}, + {C(395b7a8adb96ab75), C(582df7165b20f4a), C(e52bd30e9ff657f9), C(405e3402)}, + {C(3822dd82c7df012f), C(b9029b40bd9f122b), C(fd25b988468266c4), + C(c732c481)}, + {C(79f7efe4a80b951a), C(dd3a3fddfc6c9c41), C(ab4c812f9e27aa40), + C(a8d123c9)}, + {C(ae6e59f5f055921a), C(e9d9b7bf68e82), C(5ce4e4a5b269cc59), C(1e80ad7d)}, + {C(8959dbbf07387d36), C(b4658afce48ea35d), C(8f3f82437d8cb8d6), + C(52aeb863)}, + {C(4739613234278a49), C(99ea5bcd340bf663), C(258640912e712b12), + C(ef7c0c18)}, + {C(420e6c926bc54841), C(96dbbf6f4e7c75cd), C(d8d40fa70c3c67bb), + C(b6ad4b68)}, + {C(c8601bab561bc1b7), C(72b26272a0ff869a), C(56fdfc986d6bc3c4), + C(c1e46b17)}, + {C(b2d294931a0e20eb), C(284ffd9a0815bc38), C(1f8a103aac9bbe6), C(57b8df25)}, + {C(7966f53c37b6c6d7), C(8e6abcfb3aa2b88f), C(7f2e5e0724e5f345), + C(e9fa36d6)}, + {C(be9bb0abd03b7368), C(13bca93a3031be55), C(e864f4f52b55b472), + C(8f8daefc)}, + {C(a08d128c5f1649be), C(a8166c3dbbe19aad), C(cb9f914f829ec62c), C(6e1bb7e)}, + {C(7c386f0ffe0465ac), C(530419c9d843dbf3), C(7450e3a4f72b8d8c), + C(fd0076f0)}, + {C(bb362094e7ef4f8), C(ff3c2a48966f9725), C(55152803acd4a7fe), C(899b17b6)}, + {C(cd80dea24321eea4), C(52b4fdc8130c2b15), C(f3ea100b154bfb82), + C(e3e84e31)}, + {C(d599a04125372c3a), C(313136c56a56f363), C(1e993c3677625832), + C(eef79b6b)}, + {C(dbbf541e9dfda0a), C(1479fceb6db4f844), C(31ab576b59062534), C(868e3315)}, + {C(c2ee3288be4fe2bf), C(c65d2f5ddf32b92), C(af6ecdf121ba5485), C(4639a426)}, + {C(d86603ced1ed4730), C(f9de718aaada7709), C(db8b9755194c6535), + C(f3213646)}, + {C(915263c671b28809), C(a815378e7ad762fd), C(abec6dc9b669f559), + C(17f148e9)}, + {C(2b67cdd38c307a5e), C(cb1d45bb5c9fe1c), C(800baf2a02ec18ad), C(bfd94880)}, + {C(2d107419073b9cd0), C(a96db0740cef8f54), C(ec41ee91b3ecdc1b), + C(bb1fa7f3)}, + {C(f3e9487ec0e26dfc), C(1ab1f63224e837fa), C(119983bb5a8125d8), C(88816b1)}, + {C(1160987c8fe86f7d), C(879e6db1481eb91b), C(d7dcb802bfe6885d), + C(5c2faeb3)}, + {C(eab8112c560b967b), C(97f550b58e89dbae), C(846ed506d304051f), + C(51b5fc6f)}, + {C(1addcf0386d35351), C(b5f436561f8f1484), C(85d38e22181c9bb1), + C(33d94752)}, + {C(d445ba84bf803e09), C(1216c2497038f804), C(2293216ea2237207), + C(b0c92948)}, + {C(37235a096a8be435), C(d9b73130493589c2), C(3b1024f59378d3be), + C(c7171590)}, + {C(763ad6ea2fe1c99d), C(cf7af5368ac1e26b), C(4d5e451b3bb8d3d4), + C(240a67fb)}, + {C(ea627fc84cd1b857), C(85e372494520071f), C(69ec61800845780b), + C(e1843cd5)}, + {C(1f2ffd79f2cdc0c8), C(726a1bc31b337aaa), C(678b7f275ef96434), + C(fda1452b)}, + {C(39a9e146ec4b3210), C(f63f75802a78b1ac), C(e2e22539c94741c3), + C(a2cad330)}, + {C(74cba303e2dd9d6d), C(692699b83289fad1), C(dfb9aa7874678480), + C(53467e16)}, + {C(4cbc2b73a43071e0), C(56c5db4c4ca4e0b7), C(1b275a162f46bd3d), + C(da14a8d0)}, + {C(875638b9715d2221), C(d9ba0615c0c58740), C(616d4be2dfe825aa), + C(67333551)}, + {C(fb686b2782994a8d), C(edee60693756bb48), C(e6bc3cae0ded2ef5), + C(a0ebd66e)}, + {C(ab21d81a911e6723), C(4c31b07354852f59), C(835da384c9384744), + C(4b769593)}, + {C(33d013cc0cd46ecf), C(3de726423aea122c), C(116af51117fe21a9), + C(6aa75624)}, + {C(8ca92c7cd39fae5d), C(317e620e1bf20f1), C(4f0b33bf2194b97f), C(602a3f96)}, + {C(fdde3b03f018f43e), C(38f932946c78660), C(c84084ce946851ee), C(cd183c4d)}, + {C(9c8502050e9c9458), C(d6d2a1a69964beb9), C(1675766f480229b5), + C(960a4d07)}, + {C(348176ca2fa2fdd2), C(3a89c514cc360c2d), C(9f90b8afb318d6d0), + C(9ae998c4)}, + {C(4a3d3dfbbaea130b), C(4e221c920f61ed01), C(553fd6cd1304531f), + C(74e2179d)}, + {C(b371f768cdf4edb9), C(bdef2ace6d2de0f0), C(e05b4100f7f1baec), + C(ee9bae25)}, + {C(7a1d2e96934f61f), C(eb1760ae6af7d961), C(887eb0da063005df), C(b66edf10)}, + {C(8be53d466d4728f2), C(86a5ac8e0d416640), C(984aa464cdb5c8bb), + C(d6209737)}, + {C(829677eb03abf042), C(43cad004b6bc2c0), C(f2f224756803971a), C(b994a88)}, + {C(754435bae3496fc), C(5707fc006f094dcf), C(8951c86ab19d8e40), C(a05d43c0)}, + {C(fda9877ea8e3805f), C(31e868b6ffd521b7), C(b08c90681fb6a0fd), + C(c79f73a8)}, + {C(2e36f523ca8f5eb5), C(8b22932f89b27513), C(331cd6ecbfadc1bb), + C(a490aff5)}, + {C(21a378ef76828208), C(a5c13037fa841da2), C(506d22a53fbe9812), + C(dfad65b4)}, + {C(ccdd5600054b16ca), C(f78846e84204cb7b), C(1f9faec82c24eac9), C(1d07dfb)}, + {C(7854468f4e0cabd0), C(3a3f6b4f098d0692), C(ae2423ec7799d30d), + C(416df9a0)}, + {C(7f88db5346d8f997), C(88eac9aacc653798), C(68a4d0295f8eefa1), + C(1f8fb9cc)}, + {C(bb3fb5fb01d60fcf), C(1b7cc0847a215eb6), C(1246c994437990a1), + C(7abf48e3)}, + {C(2e783e1761acd84d), C(39158042bac975a0), C(1cd21c5a8071188d), + C(dea4e3dd)}, + {C(392058251cf22acc), C(944ec4475ead4620), C(b330a10b5cb94166), + C(c6064f22)}, + {C(adf5c1e5d6419947), C(2a9747bc659d28aa), C(95c5b8cb1f5d62c), C(743bed9c)}, + {C(6bc1db2c2bee5aba), C(e63b0ed635307398), C(7b2eca111f30dbbc), + C(fce254d5)}, + {C(b00f898229efa508), C(83b7590ad7f6985c), C(2780e70a0592e41d), + C(e47ec9d1)}, + {C(b56eb769ce0d9a8c), C(ce196117bfbcaf04), C(b26c3c3797d66165), + C(334a145c)}, + {C(70c0637675b94150), C(259e1669305b0a15), C(46e1dd9fd387a58d), + C(adec1e3c)}, + {C(74c0b8a6821faafe), C(abac39d7491370e7), C(faf0b2a48a4e6aed), + C(f6a9fbf8)}, + {C(5fb5e48ac7b7fa4f), C(a96170f08f5acbc7), C(bbf5c63d4f52a1e5), + C(5398210c)}, +}; + +void TestUnchanging(const uint64_t* expected, int offset, int len) { + EXPECT_EQ(expected[0], CityHash64(data + offset, len)); + EXPECT_EQ(expected[3], CityHash32(data + offset, len)); + EXPECT_EQ(expected[1], CityHash64WithSeed(data + offset, len, kSeed0)); + EXPECT_EQ(expected[2], + CityHash64WithSeeds(data + offset, len, kSeed0, kSeed1)); +} + +TEST(CityHashTest, Unchanging) { + setup(); + int i = 0; + for (; i < kTestSize - 1; i++) { + TestUnchanging(testdata[i], i * i, i); + } + TestUnchanging(testdata[i], 0, kDataSize); +} + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/hash/internal/hash.cc b/third_party/abseil_cpp/absl/hash/internal/hash.cc new file mode 100644 index 000000000000..b44ecb3a6b66 --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/hash.cc @@ -0,0 +1,55 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/hash/internal/hash.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +uint64_t CityHashState::CombineLargeContiguousImpl32(uint64_t state, + const unsigned char* first, + size_t len) { + while (len >= PiecewiseChunkSize()) { + state = + Mix(state, absl::hash_internal::CityHash32(reinterpret_cast<const char*>(first), + PiecewiseChunkSize())); + len -= PiecewiseChunkSize(); + first += PiecewiseChunkSize(); + } + // Handle the remainder. + return CombineContiguousImpl(state, first, len, + std::integral_constant<int, 4>{}); +} + +uint64_t CityHashState::CombineLargeContiguousImpl64(uint64_t state, + const unsigned char* first, + size_t len) { + while (len >= PiecewiseChunkSize()) { + state = + Mix(state, absl::hash_internal::CityHash64(reinterpret_cast<const char*>(first), + PiecewiseChunkSize())); + len -= PiecewiseChunkSize(); + first += PiecewiseChunkSize(); + } + // Handle the remainder. + return CombineContiguousImpl(state, first, len, + std::integral_constant<int, 8>{}); +} + +ABSL_CONST_INIT const void* const CityHashState::kSeed = &kSeed; + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/hash/internal/hash.h b/third_party/abseil_cpp/absl/hash/internal/hash.h new file mode 100644 index 000000000000..b0132da2069d --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/hash.h @@ -0,0 +1,1003 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: hash.h +// ----------------------------------------------------------------------------- +// +#ifndef ABSL_HASH_INTERNAL_HASH_H_ +#define ABSL_HASH_INTERNAL_HASH_H_ + +#include <algorithm> +#include <array> +#include <cmath> +#include <cstring> +#include <deque> +#include <forward_list> +#include <functional> +#include <iterator> +#include <limits> +#include <list> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <tuple> +#include <type_traits> +#include <utility> +#include <vector> + +#include "absl/base/internal/endian.h" +#include "absl/base/port.h" +#include "absl/container/fixed_array.h" +#include "absl/meta/type_traits.h" +#include "absl/numeric/int128.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "absl/utility/utility.h" +#include "absl/hash/internal/city.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +// Internal detail: Large buffers are hashed in smaller chunks. This function +// returns the size of these chunks. +constexpr size_t PiecewiseChunkSize() { return 1024; } + +// PiecewiseCombiner +// +// PiecewiseCombiner is an internal-only helper class for hashing a piecewise +// buffer of `char` or `unsigned char` as though it were contiguous. This class +// provides two methods: +// +// H add_buffer(state, data, size) +// H finalize(state) +// +// `add_buffer` can be called zero or more times, followed by a single call to +// `finalize`. This will produce the same hash expansion as concatenating each +// buffer piece into a single contiguous buffer, and passing this to +// `H::combine_contiguous`. +// +// Example usage: +// PiecewiseCombiner combiner; +// for (const auto& piece : pieces) { +// state = combiner.add_buffer(std::move(state), piece.data, piece.size); +// } +// return combiner.finalize(std::move(state)); +class PiecewiseCombiner { + public: + PiecewiseCombiner() : position_(0) {} + PiecewiseCombiner(const PiecewiseCombiner&) = delete; + PiecewiseCombiner& operator=(const PiecewiseCombiner&) = delete; + + // PiecewiseCombiner::add_buffer() + // + // Appends the given range of bytes to the sequence to be hashed, which may + // modify the provided hash state. + template <typename H> + H add_buffer(H state, const unsigned char* data, size_t size); + template <typename H> + H add_buffer(H state, const char* data, size_t size) { + return add_buffer(std::move(state), + reinterpret_cast<const unsigned char*>(data), size); + } + + // PiecewiseCombiner::finalize() + // + // Finishes combining the hash sequence, which may may modify the provided + // hash state. + // + // Once finalize() is called, add_buffer() may no longer be called. The + // resulting hash state will be the same as if the pieces passed to + // add_buffer() were concatenated into a single flat buffer, and then provided + // to H::combine_contiguous(). + template <typename H> + H finalize(H state); + + private: + unsigned char buf_[PiecewiseChunkSize()]; + size_t position_; +}; + +// HashStateBase +// +// A hash state object represents an intermediate state in the computation +// of an unspecified hash algorithm. `HashStateBase` provides a CRTP style +// base class for hash state implementations. Developers adding type support +// for `absl::Hash` should not rely on any parts of the state object other than +// the following member functions: +// +// * HashStateBase::combine() +// * HashStateBase::combine_contiguous() +// +// A derived hash state class of type `H` must provide a static member function +// with a signature similar to the following: +// +// `static H combine_contiguous(H state, const unsigned char*, size_t)`. +// +// `HashStateBase` will provide a complete implementation for a hash state +// object in terms of this method. +// +// Example: +// +// // Use CRTP to define your derived class. +// struct MyHashState : HashStateBase<MyHashState> { +// static H combine_contiguous(H state, const unsigned char*, size_t); +// using MyHashState::HashStateBase::combine; +// using MyHashState::HashStateBase::combine_contiguous; +// }; +template <typename H> +class HashStateBase { + public: + // HashStateBase::combine() + // + // Combines an arbitrary number of values into a hash state, returning the + // updated state. + // + // Each of the value types `T` must be separately hashable by the Abseil + // hashing framework. + // + // NOTE: + // + // state = H::combine(std::move(state), value1, value2, value3); + // + // is guaranteed to produce the same hash expansion as: + // + // state = H::combine(std::move(state), value1); + // state = H::combine(std::move(state), value2); + // state = H::combine(std::move(state), value3); + template <typename T, typename... Ts> + static H combine(H state, const T& value, const Ts&... values); + static H combine(H state) { return state; } + + // HashStateBase::combine_contiguous() + // + // Combines a contiguous array of `size` elements into a hash state, returning + // the updated state. + // + // NOTE: + // + // state = H::combine_contiguous(std::move(state), data, size); + // + // is NOT guaranteed to produce the same hash expansion as a for-loop (it may + // perform internal optimizations). If you need this guarantee, use the + // for-loop instead. + template <typename T> + static H combine_contiguous(H state, const T* data, size_t size); + + using AbslInternalPiecewiseCombiner = PiecewiseCombiner; +}; + +// is_uniquely_represented +// +// `is_uniquely_represented<T>` is a trait class that indicates whether `T` +// is uniquely represented. +// +// A type is "uniquely represented" if two equal values of that type are +// guaranteed to have the same bytes in their underlying storage. In other +// words, if `a == b`, then `memcmp(&a, &b, sizeof(T))` is guaranteed to be +// zero. This property cannot be detected automatically, so this trait is false +// by default, but can be specialized by types that wish to assert that they are +// uniquely represented. This makes them eligible for certain optimizations. +// +// If you have any doubt whatsoever, do not specialize this template. +// The default is completely safe, and merely disables some optimizations +// that will not matter for most types. Specializing this template, +// on the other hand, can be very hazardous. +// +// To be uniquely represented, a type must not have multiple ways of +// representing the same value; for example, float and double are not +// uniquely represented, because they have distinct representations for +// +0 and -0. Furthermore, the type's byte representation must consist +// solely of user-controlled data, with no padding bits and no compiler- +// controlled data such as vptrs or sanitizer metadata. This is usually +// very difficult to guarantee, because in most cases the compiler can +// insert data and padding bits at its own discretion. +// +// If you specialize this template for a type `T`, you must do so in the file +// that defines that type (or in this file). If you define that specialization +// anywhere else, `is_uniquely_represented<T>` could have different meanings +// in different places. +// +// The Enable parameter is meaningless; it is provided as a convenience, +// to support certain SFINAE techniques when defining specializations. +template <typename T, typename Enable = void> +struct is_uniquely_represented : std::false_type {}; + +// is_uniquely_represented<unsigned char> +// +// unsigned char is a synonym for "byte", so it is guaranteed to be +// uniquely represented. +template <> +struct is_uniquely_represented<unsigned char> : std::true_type {}; + +// is_uniquely_represented for non-standard integral types +// +// Integral types other than bool should be uniquely represented on any +// platform that this will plausibly be ported to. +template <typename Integral> +struct is_uniquely_represented< + Integral, typename std::enable_if<std::is_integral<Integral>::value>::type> + : std::true_type {}; + +// is_uniquely_represented<bool> +// +// +template <> +struct is_uniquely_represented<bool> : std::false_type {}; + +// hash_bytes() +// +// Convenience function that combines `hash_state` with the byte representation +// of `value`. +template <typename H, typename T> +H hash_bytes(H hash_state, const T& value) { + const unsigned char* start = reinterpret_cast<const unsigned char*>(&value); + return H::combine_contiguous(std::move(hash_state), start, sizeof(value)); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Basic Types +// ----------------------------------------------------------------------------- + +// Note: Default `AbslHashValue` implementations live in `hash_internal`. This +// allows us to block lexical scope lookup when doing an unqualified call to +// `AbslHashValue` below. User-defined implementations of `AbslHashValue` can +// only be found via ADL. + +// AbslHashValue() for hashing bool values +// +// We use SFINAE to ensure that this overload only accepts bool, not types that +// are convertible to bool. +template <typename H, typename B> +typename std::enable_if<std::is_same<B, bool>::value, H>::type AbslHashValue( + H hash_state, B value) { + return H::combine(std::move(hash_state), + static_cast<unsigned char>(value ? 1 : 0)); +} + +// AbslHashValue() for hashing enum values +template <typename H, typename Enum> +typename std::enable_if<std::is_enum<Enum>::value, H>::type AbslHashValue( + H hash_state, Enum e) { + // In practice, we could almost certainly just invoke hash_bytes directly, + // but it's possible that a sanitizer might one day want to + // store data in the unused bits of an enum. To avoid that risk, we + // convert to the underlying type before hashing. Hopefully this will get + // optimized away; if not, we can reopen discussion with c-toolchain-team. + return H::combine(std::move(hash_state), + static_cast<typename std::underlying_type<Enum>::type>(e)); +} +// AbslHashValue() for hashing floating-point values +template <typename H, typename Float> +typename std::enable_if<std::is_same<Float, float>::value || + std::is_same<Float, double>::value, + H>::type +AbslHashValue(H hash_state, Float value) { + return hash_internal::hash_bytes(std::move(hash_state), + value == 0 ? 0 : value); +} + +// Long double has the property that it might have extra unused bytes in it. +// For example, in x86 sizeof(long double)==16 but it only really uses 80-bits +// of it. This means we can't use hash_bytes on a long double and have to +// convert it to something else first. +template <typename H, typename LongDouble> +typename std::enable_if<std::is_same<LongDouble, long double>::value, H>::type +AbslHashValue(H hash_state, LongDouble value) { + const int category = std::fpclassify(value); + switch (category) { + case FP_INFINITE: + // Add the sign bit to differentiate between +Inf and -Inf + hash_state = H::combine(std::move(hash_state), std::signbit(value)); + break; + + case FP_NAN: + case FP_ZERO: + default: + // Category is enough for these. + break; + + case FP_NORMAL: + case FP_SUBNORMAL: + // We can't convert `value` directly to double because this would have + // undefined behavior if the value is out of range. + // std::frexp gives us a value in the range (-1, -.5] or [.5, 1) that is + // guaranteed to be in range for `double`. The truncation is + // implementation defined, but that works as long as it is deterministic. + int exp; + auto mantissa = static_cast<double>(std::frexp(value, &exp)); + hash_state = H::combine(std::move(hash_state), mantissa, exp); + } + + return H::combine(std::move(hash_state), category); +} + +// AbslHashValue() for hashing pointers +template <typename H, typename T> +H AbslHashValue(H hash_state, T* ptr) { + auto v = reinterpret_cast<uintptr_t>(ptr); + // Due to alignment, pointers tend to have low bits as zero, and the next few + // bits follow a pattern since they are also multiples of some base value. + // Mixing the pointer twice helps prevent stuck low bits for certain alignment + // values. + return H::combine(std::move(hash_state), v, v); +} + +// AbslHashValue() for hashing nullptr_t +template <typename H> +H AbslHashValue(H hash_state, std::nullptr_t) { + return H::combine(std::move(hash_state), static_cast<void*>(nullptr)); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Composite Types +// ----------------------------------------------------------------------------- + +// is_hashable() +// +// Trait class which returns true if T is hashable by the absl::Hash framework. +// Used for the AbslHashValue implementations for composite types below. +template <typename T> +struct is_hashable; + +// AbslHashValue() for hashing pairs +template <typename H, typename T1, typename T2> +typename std::enable_if<is_hashable<T1>::value && is_hashable<T2>::value, + H>::type +AbslHashValue(H hash_state, const std::pair<T1, T2>& p) { + return H::combine(std::move(hash_state), p.first, p.second); +} + +// hash_tuple() +// +// Helper function for hashing a tuple. The third argument should +// be an index_sequence running from 0 to tuple_size<Tuple> - 1. +template <typename H, typename Tuple, size_t... Is> +H hash_tuple(H hash_state, const Tuple& t, absl::index_sequence<Is...>) { + return H::combine(std::move(hash_state), std::get<Is>(t)...); +} + +// AbslHashValue for hashing tuples +template <typename H, typename... Ts> +#if defined(_MSC_VER) +// This SFINAE gets MSVC confused under some conditions. Let's just disable it +// for now. +H +#else // _MSC_VER +typename std::enable_if<absl::conjunction<is_hashable<Ts>...>::value, H>::type +#endif // _MSC_VER +AbslHashValue(H hash_state, const std::tuple<Ts...>& t) { + return hash_internal::hash_tuple(std::move(hash_state), t, + absl::make_index_sequence<sizeof...(Ts)>()); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Pointers +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing unique_ptr +template <typename H, typename T, typename D> +H AbslHashValue(H hash_state, const std::unique_ptr<T, D>& ptr) { + return H::combine(std::move(hash_state), ptr.get()); +} + +// AbslHashValue for hashing shared_ptr +template <typename H, typename T> +H AbslHashValue(H hash_state, const std::shared_ptr<T>& ptr) { + return H::combine(std::move(hash_state), ptr.get()); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for String-Like Types +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing strings +// +// All the string-like types supported here provide the same hash expansion for +// the same character sequence. These types are: +// +// - `absl::Cord` +// - `std::string` (and std::basic_string<char, std::char_traits<char>, A> for +// any allocator A) +// - `absl::string_view` and `std::string_view` +// +// For simplicity, we currently support only `char` strings. This support may +// be broadened, if necessary, but with some caution - this overload would +// misbehave in cases where the traits' `eq()` member isn't equivalent to `==` +// on the underlying character type. +template <typename H> +H AbslHashValue(H hash_state, absl::string_view str) { + return H::combine( + H::combine_contiguous(std::move(hash_state), str.data(), str.size()), + str.size()); +} + +// Support std::wstring, std::u16string and std::u32string. +template <typename Char, typename Alloc, typename H, + typename = absl::enable_if_t<std::is_same<Char, wchar_t>::value || + std::is_same<Char, char16_t>::value || + std::is_same<Char, char32_t>::value>> +H AbslHashValue( + H hash_state, + const std::basic_string<Char, std::char_traits<Char>, Alloc>& str) { + return H::combine( + H::combine_contiguous(std::move(hash_state), str.data(), str.size()), + str.size()); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Sequence Containers +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing std::array +template <typename H, typename T, size_t N> +typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( + H hash_state, const std::array<T, N>& array) { + return H::combine_contiguous(std::move(hash_state), array.data(), + array.size()); +} + +// AbslHashValue for hashing std::deque +template <typename H, typename T, typename Allocator> +typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( + H hash_state, const std::deque<T, Allocator>& deque) { + // TODO(gromer): investigate a more efficient implementation taking + // advantage of the chunk structure. + for (const auto& t : deque) { + hash_state = H::combine(std::move(hash_state), t); + } + return H::combine(std::move(hash_state), deque.size()); +} + +// AbslHashValue for hashing std::forward_list +template <typename H, typename T, typename Allocator> +typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( + H hash_state, const std::forward_list<T, Allocator>& list) { + size_t size = 0; + for (const T& t : list) { + hash_state = H::combine(std::move(hash_state), t); + ++size; + } + return H::combine(std::move(hash_state), size); +} + +// AbslHashValue for hashing std::list +template <typename H, typename T, typename Allocator> +typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( + H hash_state, const std::list<T, Allocator>& list) { + for (const auto& t : list) { + hash_state = H::combine(std::move(hash_state), t); + } + return H::combine(std::move(hash_state), list.size()); +} + +// AbslHashValue for hashing std::vector +// +// Do not use this for vector<bool>. It does not have a .data(), and a fallback +// for std::hash<> is most likely faster. +template <typename H, typename T, typename Allocator> +typename std::enable_if<is_hashable<T>::value && !std::is_same<T, bool>::value, + H>::type +AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { + return H::combine(H::combine_contiguous(std::move(hash_state), vector.data(), + vector.size()), + vector.size()); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Ordered Associative Containers +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing std::map +template <typename H, typename Key, typename T, typename Compare, + typename Allocator> +typename std::enable_if<is_hashable<Key>::value && is_hashable<T>::value, + H>::type +AbslHashValue(H hash_state, const std::map<Key, T, Compare, Allocator>& map) { + for (const auto& t : map) { + hash_state = H::combine(std::move(hash_state), t); + } + return H::combine(std::move(hash_state), map.size()); +} + +// AbslHashValue for hashing std::multimap +template <typename H, typename Key, typename T, typename Compare, + typename Allocator> +typename std::enable_if<is_hashable<Key>::value && is_hashable<T>::value, + H>::type +AbslHashValue(H hash_state, + const std::multimap<Key, T, Compare, Allocator>& map) { + for (const auto& t : map) { + hash_state = H::combine(std::move(hash_state), t); + } + return H::combine(std::move(hash_state), map.size()); +} + +// AbslHashValue for hashing std::set +template <typename H, typename Key, typename Compare, typename Allocator> +typename std::enable_if<is_hashable<Key>::value, H>::type AbslHashValue( + H hash_state, const std::set<Key, Compare, Allocator>& set) { + for (const auto& t : set) { + hash_state = H::combine(std::move(hash_state), t); + } + return H::combine(std::move(hash_state), set.size()); +} + +// AbslHashValue for hashing std::multiset +template <typename H, typename Key, typename Compare, typename Allocator> +typename std::enable_if<is_hashable<Key>::value, H>::type AbslHashValue( + H hash_state, const std::multiset<Key, Compare, Allocator>& set) { + for (const auto& t : set) { + hash_state = H::combine(std::move(hash_state), t); + } + return H::combine(std::move(hash_state), set.size()); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Wrapper Types +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing std::reference_wrapper +template <typename H, typename T> +typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( + H hash_state, std::reference_wrapper<T> opt) { + return H::combine(std::move(hash_state), opt.get()); +} + +// AbslHashValue for hashing absl::optional +template <typename H, typename T> +typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( + H hash_state, const absl::optional<T>& opt) { + if (opt) hash_state = H::combine(std::move(hash_state), *opt); + return H::combine(std::move(hash_state), opt.has_value()); +} + +// VariantVisitor +template <typename H> +struct VariantVisitor { + H&& hash_state; + template <typename T> + H operator()(const T& t) const { + return H::combine(std::move(hash_state), t); + } +}; + +// AbslHashValue for hashing absl::variant +template <typename H, typename... T> +typename std::enable_if<conjunction<is_hashable<T>...>::value, H>::type +AbslHashValue(H hash_state, const absl::variant<T...>& v) { + if (!v.valueless_by_exception()) { + hash_state = absl::visit(VariantVisitor<H>{std::move(hash_state)}, v); + } + return H::combine(std::move(hash_state), v.index()); +} + +// ----------------------------------------------------------------------------- +// AbslHashValue for Other Types +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing std::bitset is not defined, for the same reason as +// for vector<bool> (see std::vector above): It does not expose the raw bytes, +// and a fallback to std::hash<> is most likely faster. + +// ----------------------------------------------------------------------------- + +// hash_range_or_bytes() +// +// Mixes all values in the range [data, data+size) into the hash state. +// This overload accepts only uniquely-represented types, and hashes them by +// hashing the entire range of bytes. +template <typename H, typename T> +typename std::enable_if<is_uniquely_represented<T>::value, H>::type +hash_range_or_bytes(H hash_state, const T* data, size_t size) { + const auto* bytes = reinterpret_cast<const unsigned char*>(data); + return H::combine_contiguous(std::move(hash_state), bytes, sizeof(T) * size); +} + +// hash_range_or_bytes() +template <typename H, typename T> +typename std::enable_if<!is_uniquely_represented<T>::value, H>::type +hash_range_or_bytes(H hash_state, const T* data, size_t size) { + for (const auto end = data + size; data < end; ++data) { + hash_state = H::combine(std::move(hash_state), *data); + } + return hash_state; +} + +#if defined(ABSL_INTERNAL_LEGACY_HASH_NAMESPACE) && \ + ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ +#define ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ 1 +#else +#define ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ 0 +#endif + +// HashSelect +// +// Type trait to select the appropriate hash implementation to use. +// HashSelect::type<T> will give the proper hash implementation, to be invoked +// as: +// HashSelect::type<T>::Invoke(state, value) +// Also, HashSelect::type<T>::value is a boolean equal to `true` if there is a +// valid `Invoke` function. Types that are not hashable will have a ::value of +// `false`. +struct HashSelect { + private: + struct State : HashStateBase<State> { + static State combine_contiguous(State hash_state, const unsigned char*, + size_t); + using State::HashStateBase::combine_contiguous; + }; + + struct UniquelyRepresentedProbe { + template <typename H, typename T> + static auto Invoke(H state, const T& value) + -> absl::enable_if_t<is_uniquely_represented<T>::value, H> { + return hash_internal::hash_bytes(std::move(state), value); + } + }; + + struct HashValueProbe { + template <typename H, typename T> + static auto Invoke(H state, const T& value) -> absl::enable_if_t< + std::is_same<H, + decltype(AbslHashValue(std::move(state), value))>::value, + H> { + return AbslHashValue(std::move(state), value); + } + }; + + struct LegacyHashProbe { +#if ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ + template <typename H, typename T> + static auto Invoke(H state, const T& value) -> absl::enable_if_t< + std::is_convertible< + decltype(ABSL_INTERNAL_LEGACY_HASH_NAMESPACE::hash<T>()(value)), + size_t>::value, + H> { + return hash_internal::hash_bytes( + std::move(state), + ABSL_INTERNAL_LEGACY_HASH_NAMESPACE::hash<T>{}(value)); + } +#endif // ABSL_HASH_INTERNAL_SUPPORT_LEGACY_HASH_ + }; + + struct StdHashProbe { + template <typename H, typename T> + static auto Invoke(H state, const T& value) + -> absl::enable_if_t<type_traits_internal::IsHashable<T>::value, H> { + return hash_internal::hash_bytes(std::move(state), std::hash<T>{}(value)); + } + }; + + template <typename Hash, typename T> + struct Probe : Hash { + private: + template <typename H, typename = decltype(H::Invoke( + std::declval<State>(), std::declval<const T&>()))> + static std::true_type Test(int); + template <typename U> + static std::false_type Test(char); + + public: + static constexpr bool value = decltype(Test<Hash>(0))::value; + }; + + public: + // Probe each implementation in order. + // disjunction provides short circuiting wrt instantiation. + template <typename T> + using Apply = absl::disjunction< // + Probe<UniquelyRepresentedProbe, T>, // + Probe<HashValueProbe, T>, // + Probe<LegacyHashProbe, T>, // + Probe<StdHashProbe, T>, // + std::false_type>; +}; + +template <typename T> +struct is_hashable + : std::integral_constant<bool, HashSelect::template Apply<T>::value> {}; + +// CityHashState +class ABSL_DLL CityHashState + : public HashStateBase<CityHashState> { + // absl::uint128 is not an alias or a thin wrapper around the intrinsic. + // We use the intrinsic when available to improve performance. +#ifdef ABSL_HAVE_INTRINSIC_INT128 + using uint128 = __uint128_t; +#else // ABSL_HAVE_INTRINSIC_INT128 + using uint128 = absl::uint128; +#endif // ABSL_HAVE_INTRINSIC_INT128 + + static constexpr uint64_t kMul = + sizeof(size_t) == 4 ? uint64_t{0xcc9e2d51} + : uint64_t{0x9ddfea08eb382d69}; + + template <typename T> + using IntegralFastPath = + conjunction<std::is_integral<T>, is_uniquely_represented<T>>; + + public: + // Move only + CityHashState(CityHashState&&) = default; + CityHashState& operator=(CityHashState&&) = default; + + // CityHashState::combine_contiguous() + // + // Fundamental base case for hash recursion: mixes the given range of bytes + // into the hash state. + static CityHashState combine_contiguous(CityHashState hash_state, + const unsigned char* first, + size_t size) { + return CityHashState( + CombineContiguousImpl(hash_state.state_, first, size, + std::integral_constant<int, sizeof(size_t)>{})); + } + using CityHashState::HashStateBase::combine_contiguous; + + // CityHashState::hash() + // + // For performance reasons in non-opt mode, we specialize this for + // integral types. + // Otherwise we would be instantiating and calling dozens of functions for + // something that is just one multiplication and a couple xor's. + // The result should be the same as running the whole algorithm, but faster. + template <typename T, absl::enable_if_t<IntegralFastPath<T>::value, int> = 0> + static size_t hash(T value) { + return static_cast<size_t>(Mix(Seed(), static_cast<uint64_t>(value))); + } + + // Overload of CityHashState::hash() + template <typename T, absl::enable_if_t<!IntegralFastPath<T>::value, int> = 0> + static size_t hash(const T& value) { + return static_cast<size_t>(combine(CityHashState{}, value).state_); + } + + private: + // Invoked only once for a given argument; that plus the fact that this is + // move-only ensures that there is only one non-moved-from object. + CityHashState() : state_(Seed()) {} + + // Workaround for MSVC bug. + // We make the type copyable to fix the calling convention, even though we + // never actually copy it. Keep it private to not affect the public API of the + // type. + CityHashState(const CityHashState&) = default; + + explicit CityHashState(uint64_t state) : state_(state) {} + + // Implementation of the base case for combine_contiguous where we actually + // mix the bytes into the state. + // Dispatch to different implementations of the combine_contiguous depending + // on the value of `sizeof(size_t)`. + static uint64_t CombineContiguousImpl(uint64_t state, + const unsigned char* first, size_t len, + std::integral_constant<int, 4> + /* sizeof_size_t */); + static uint64_t CombineContiguousImpl(uint64_t state, + const unsigned char* first, size_t len, + std::integral_constant<int, 8> + /* sizeof_size_t*/); + + // Slow dispatch path for calls to CombineContiguousImpl with a size argument + // larger than PiecewiseChunkSize(). Has the same effect as calling + // CombineContiguousImpl() repeatedly with the chunk stride size. + static uint64_t CombineLargeContiguousImpl32(uint64_t state, + const unsigned char* first, + size_t len); + static uint64_t CombineLargeContiguousImpl64(uint64_t state, + const unsigned char* first, + size_t len); + + // Reads 9 to 16 bytes from p. + // The first 8 bytes are in .first, the rest (zero padded) bytes are in + // .second. + static std::pair<uint64_t, uint64_t> Read9To16(const unsigned char* p, + size_t len) { + uint64_t high = little_endian::Load64(p + len - 8); + return {little_endian::Load64(p), high >> (128 - len * 8)}; + } + + // Reads 4 to 8 bytes from p. Zero pads to fill uint64_t. + static uint64_t Read4To8(const unsigned char* p, size_t len) { + return (static_cast<uint64_t>(little_endian::Load32(p + len - 4)) + << (len - 4) * 8) | + little_endian::Load32(p); + } + + // Reads 1 to 3 bytes from p. Zero pads to fill uint32_t. + static uint32_t Read1To3(const unsigned char* p, size_t len) { + return static_cast<uint32_t>((p[0]) | // + (p[len / 2] << (len / 2 * 8)) | // + (p[len - 1] << ((len - 1) * 8))); + } + + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Mix(uint64_t state, uint64_t v) { + using MultType = + absl::conditional_t<sizeof(size_t) == 4, uint64_t, uint128>; + // We do the addition in 64-bit space to make sure the 128-bit + // multiplication is fast. If we were to do it as MultType the compiler has + // to assume that the high word is non-zero and needs to perform 2 + // multiplications instead of one. + MultType m = state + v; + m *= kMul; + return static_cast<uint64_t>(m ^ (m >> (sizeof(m) * 8 / 2))); + } + + // Seed() + // + // A non-deterministic seed. + // + // The current purpose of this seed is to generate non-deterministic results + // and prevent having users depend on the particular hash values. + // It is not meant as a security feature right now, but it leaves the door + // open to upgrade it to a true per-process random seed. A true random seed + // costs more and we don't need to pay for that right now. + // + // On platforms with ASLR, we take advantage of it to make a per-process + // random value. + // See https://en.wikipedia.org/wiki/Address_space_layout_randomization + // + // On other platforms this is still going to be non-deterministic but most + // probably per-build and not per-process. + ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Seed() { +#if (!defined(__clang__) || __clang_major__ > 11) && \ + !defined(__apple_build_version__) + return static_cast<uint64_t>(reinterpret_cast<uintptr_t>(&kSeed)); +#else + // Workaround the absence of + // https://github.com/llvm/llvm-project/commit/bc15bf66dcca76cc06fe71fca35b74dc4d521021. + return static_cast<uint64_t>(reinterpret_cast<uintptr_t>(kSeed)); +#endif + } + static const void* const kSeed; + + uint64_t state_; +}; + +// CityHashState::CombineContiguousImpl() +inline uint64_t CityHashState::CombineContiguousImpl( + uint64_t state, const unsigned char* first, size_t len, + std::integral_constant<int, 4> /* sizeof_size_t */) { + // For large values we use CityHash, for small ones we just use a + // multiplicative hash. + uint64_t v; + if (len > 8) { + if (ABSL_PREDICT_FALSE(len > PiecewiseChunkSize())) { + return CombineLargeContiguousImpl32(state, first, len); + } + v = absl::hash_internal::CityHash32(reinterpret_cast<const char*>(first), len); + } else if (len >= 4) { + v = Read4To8(first, len); + } else if (len > 0) { + v = Read1To3(first, len); + } else { + // Empty ranges have no effect. + return state; + } + return Mix(state, v); +} + +// Overload of CityHashState::CombineContiguousImpl() +inline uint64_t CityHashState::CombineContiguousImpl( + uint64_t state, const unsigned char* first, size_t len, + std::integral_constant<int, 8> /* sizeof_size_t */) { + // For large values we use CityHash, for small ones we just use a + // multiplicative hash. + uint64_t v; + if (len > 16) { + if (ABSL_PREDICT_FALSE(len > PiecewiseChunkSize())) { + return CombineLargeContiguousImpl64(state, first, len); + } + v = absl::hash_internal::CityHash64(reinterpret_cast<const char*>(first), len); + } else if (len > 8) { + auto p = Read9To16(first, len); + state = Mix(state, p.first); + v = p.second; + } else if (len >= 4) { + v = Read4To8(first, len); + } else if (len > 0) { + v = Read1To3(first, len); + } else { + // Empty ranges have no effect. + return state; + } + return Mix(state, v); +} + +struct AggregateBarrier {}; + +// HashImpl + +// Add a private base class to make sure this type is not an aggregate. +// Aggregates can be aggregate initialized even if the default constructor is +// deleted. +struct PoisonedHash : private AggregateBarrier { + PoisonedHash() = delete; + PoisonedHash(const PoisonedHash&) = delete; + PoisonedHash& operator=(const PoisonedHash&) = delete; +}; + +template <typename T> +struct HashImpl { + size_t operator()(const T& value) const { return CityHashState::hash(value); } +}; + +template <typename T> +struct Hash + : absl::conditional_t<is_hashable<T>::value, HashImpl<T>, PoisonedHash> {}; + +template <typename H> +template <typename T, typename... Ts> +H HashStateBase<H>::combine(H state, const T& value, const Ts&... values) { + return H::combine(hash_internal::HashSelect::template Apply<T>::Invoke( + std::move(state), value), + values...); +} + +// HashStateBase::combine_contiguous() +template <typename H> +template <typename T> +H HashStateBase<H>::combine_contiguous(H state, const T* data, size_t size) { + return hash_internal::hash_range_or_bytes(std::move(state), data, size); +} + +// HashStateBase::PiecewiseCombiner::add_buffer() +template <typename H> +H PiecewiseCombiner::add_buffer(H state, const unsigned char* data, + size_t size) { + if (position_ + size < PiecewiseChunkSize()) { + // This partial chunk does not fill our existing buffer + memcpy(buf_ + position_, data, size); + position_ += size; + return state; + } + + // If the buffer is partially filled we need to complete the buffer + // and hash it. + if (position_ != 0) { + const size_t bytes_needed = PiecewiseChunkSize() - position_; + memcpy(buf_ + position_, data, bytes_needed); + state = H::combine_contiguous(std::move(state), buf_, PiecewiseChunkSize()); + data += bytes_needed; + size -= bytes_needed; + } + + // Hash whatever chunks we can without copying + while (size >= PiecewiseChunkSize()) { + state = H::combine_contiguous(std::move(state), data, PiecewiseChunkSize()); + data += PiecewiseChunkSize(); + size -= PiecewiseChunkSize(); + } + // Fill the buffer with the remainder + memcpy(buf_, data, size); + position_ = size; + return state; +} + +// HashStateBase::PiecewiseCombiner::finalize() +template <typename H> +H PiecewiseCombiner::finalize(H state) { + // Hash the remainder left in the buffer, which may be empty + return H::combine_contiguous(std::move(state), buf_, position_); +} + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HASH_INTERNAL_HASH_H_ diff --git a/third_party/abseil_cpp/absl/hash/internal/print_hash_of.cc b/third_party/abseil_cpp/absl/hash/internal/print_hash_of.cc new file mode 100644 index 000000000000..c392125a69fa --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/print_hash_of.cc @@ -0,0 +1,23 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdlib> + +#include "absl/hash/hash.h" + +// Prints the hash of argv[1]. +int main(int argc, char** argv) { + if (argc < 2) return 1; + printf("%zu\n", absl::Hash<int>{}(std::atoi(argv[1]))); // NOLINT +} diff --git a/third_party/abseil_cpp/absl/hash/internal/spy_hash_state.h b/third_party/abseil_cpp/absl/hash/internal/spy_hash_state.h new file mode 100644 index 000000000000..c08312081180 --- /dev/null +++ b/third_party/abseil_cpp/absl/hash/internal/spy_hash_state.h @@ -0,0 +1,231 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_HASH_INTERNAL_SPY_HASH_STATE_H_ +#define ABSL_HASH_INTERNAL_SPY_HASH_STATE_H_ + +#include <ostream> +#include <string> +#include <vector> + +#include "absl/hash/hash.h" +#include "absl/strings/match.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace hash_internal { + +// SpyHashState is an implementation of the HashState API that simply +// accumulates all input bytes in an internal buffer. This makes it useful +// for testing AbslHashValue overloads (so long as they are templated on the +// HashState parameter), since it can report the exact hash representation +// that the AbslHashValue overload produces. +// +// Sample usage: +// EXPECT_EQ(SpyHashState::combine(SpyHashState(), foo), +// SpyHashState::combine(SpyHashState(), bar)); +template <typename T> +class SpyHashStateImpl : public HashStateBase<SpyHashStateImpl<T>> { + public: + SpyHashStateImpl() : error_(std::make_shared<absl::optional<std::string>>()) { + static_assert(std::is_void<T>::value, ""); + } + + // Move-only + SpyHashStateImpl(const SpyHashStateImpl&) = delete; + SpyHashStateImpl& operator=(const SpyHashStateImpl&) = delete; + + SpyHashStateImpl(SpyHashStateImpl&& other) noexcept { + *this = std::move(other); + } + + SpyHashStateImpl& operator=(SpyHashStateImpl&& other) noexcept { + hash_representation_ = std::move(other.hash_representation_); + error_ = other.error_; + moved_from_ = other.moved_from_; + other.moved_from_ = true; + return *this; + } + + template <typename U> + SpyHashStateImpl(SpyHashStateImpl<U>&& other) { // NOLINT + hash_representation_ = std::move(other.hash_representation_); + error_ = other.error_; + moved_from_ = other.moved_from_; + other.moved_from_ = true; + } + + template <typename A, typename... Args> + static SpyHashStateImpl combine(SpyHashStateImpl s, const A& a, + const Args&... args) { + // Pass an instance of SpyHashStateImpl<A> when trying to combine `A`. This + // allows us to test that the user only uses this instance for combine calls + // and does not call AbslHashValue directly. + // See AbslHashValue implementation at the bottom. + s = SpyHashStateImpl<A>::HashStateBase::combine(std::move(s), a); + return SpyHashStateImpl::combine(std::move(s), args...); + } + static SpyHashStateImpl combine(SpyHashStateImpl s) { + if (direct_absl_hash_value_error_) { + *s.error_ = "AbslHashValue should not be invoked directly."; + } else if (s.moved_from_) { + *s.error_ = "Used moved-from instance of the hash state object."; + } + return s; + } + + static void SetDirectAbslHashValueError() { + direct_absl_hash_value_error_ = true; + } + + // Two SpyHashStateImpl objects are equal if they hold equal hash + // representations. + friend bool operator==(const SpyHashStateImpl& lhs, + const SpyHashStateImpl& rhs) { + return lhs.hash_representation_ == rhs.hash_representation_; + } + + friend bool operator!=(const SpyHashStateImpl& lhs, + const SpyHashStateImpl& rhs) { + return !(lhs == rhs); + } + + enum class CompareResult { + kEqual, + kASuffixB, + kBSuffixA, + kUnequal, + }; + + static CompareResult Compare(const SpyHashStateImpl& a, + const SpyHashStateImpl& b) { + const std::string a_flat = absl::StrJoin(a.hash_representation_, ""); + const std::string b_flat = absl::StrJoin(b.hash_representation_, ""); + if (a_flat == b_flat) return CompareResult::kEqual; + if (absl::EndsWith(a_flat, b_flat)) return CompareResult::kBSuffixA; + if (absl::EndsWith(b_flat, a_flat)) return CompareResult::kASuffixB; + return CompareResult::kUnequal; + } + + // operator<< prints the hash representation as a hex and ASCII dump, to + // facilitate debugging. + friend std::ostream& operator<<(std::ostream& out, + const SpyHashStateImpl& hash_state) { + out << "[\n"; + for (auto& s : hash_state.hash_representation_) { + size_t offset = 0; + for (char c : s) { + if (offset % 16 == 0) { + out << absl::StreamFormat("\n0x%04x: ", offset); + } + if (offset % 2 == 0) { + out << " "; + } + out << absl::StreamFormat("%02x", c); + ++offset; + } + out << "\n"; + } + return out << "]"; + } + + // The base case of the combine recursion, which writes raw bytes into the + // internal buffer. + static SpyHashStateImpl combine_contiguous(SpyHashStateImpl hash_state, + const unsigned char* begin, + size_t size) { + const size_t large_chunk_stride = PiecewiseChunkSize(); + if (size > large_chunk_stride) { + // Combining a large contiguous buffer must have the same effect as + // doing it piecewise by the stride length, followed by the (possibly + // empty) remainder. + while (size >= large_chunk_stride) { + hash_state = SpyHashStateImpl::combine_contiguous( + std::move(hash_state), begin, large_chunk_stride); + begin += large_chunk_stride; + size -= large_chunk_stride; + } + } + + hash_state.hash_representation_.emplace_back( + reinterpret_cast<const char*>(begin), size); + return hash_state; + } + + using SpyHashStateImpl::HashStateBase::combine_contiguous; + + absl::optional<std::string> error() const { + if (moved_from_) { + return "Returned a moved-from instance of the hash state object."; + } + return *error_; + } + + private: + template <typename U> + friend class SpyHashStateImpl; + + // This is true if SpyHashStateImpl<T> has been passed to a call of + // AbslHashValue with the wrong type. This detects that the user called + // AbslHashValue directly (because the hash state type does not match). + static bool direct_absl_hash_value_error_; + + std::vector<std::string> hash_representation_; + // This is a shared_ptr because we want all instances of the particular + // SpyHashState run to share the field. This way we can set the error for + // use-after-move and all the copies will see it. + std::shared_ptr<absl::optional<std::string>> error_; + bool moved_from_ = false; +}; + +template <typename T> +bool SpyHashStateImpl<T>::direct_absl_hash_value_error_; + +template <bool& B> +struct OdrUse { + constexpr OdrUse() {} + bool& b = B; +}; + +template <void (*)()> +struct RunOnStartup { + static bool run; + static constexpr OdrUse<run> kOdrUse{}; +}; + +template <void (*f)()> +bool RunOnStartup<f>::run = (f(), true); + +template < + typename T, typename U, + // Only trigger for when (T != U), + typename = absl::enable_if_t<!std::is_same<T, U>::value>, + // This statement works in two ways: + // - First, it instantiates RunOnStartup and forces the initialization of + // `run`, which set the global variable. + // - Second, it triggers a SFINAE error disabling the overload to prevent + // compile time errors. If we didn't disable the overload we would get + // ambiguous overload errors, which we don't want. + int = RunOnStartup<SpyHashStateImpl<T>::SetDirectAbslHashValueError>::run> +void AbslHashValue(SpyHashStateImpl<T>, const U&); + +using SpyHashState = SpyHashStateImpl<void>; + +} // namespace hash_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HASH_INTERNAL_SPY_HASH_STATE_H_ diff --git a/third_party/abseil_cpp/absl/memory/BUILD.bazel b/third_party/abseil_cpp/absl/memory/BUILD.bazel new file mode 100644 index 000000000000..d2824a05adda --- /dev/null +++ b/third_party/abseil_cpp/absl/memory/BUILD.bazel @@ -0,0 +1,65 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "memory", + hdrs = ["memory.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:core_headers", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "memory_test", + srcs = ["memory_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":memory", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "memory_exception_safety_test", + srcs = [ + "memory_exception_safety_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":memory", + "//absl/base:config", + "//absl/base:exception_safety_testing", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/memory/CMakeLists.txt b/third_party/abseil_cpp/absl/memory/CMakeLists.txt new file mode 100644 index 000000000000..78fb7e1b316b --- /dev/null +++ b/third_party/abseil_cpp/absl/memory/CMakeLists.txt @@ -0,0 +1,55 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + memory + HDRS + "memory.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::meta + PUBLIC +) + +absl_cc_test( + NAME + memory_test + SRCS + "memory_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::memory + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + memory_exception_safety_test + SRCS + "memory_exception_safety_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::memory + absl::config + absl::exception_safety_testing + gmock_main +) diff --git a/third_party/abseil_cpp/absl/memory/memory.h b/third_party/abseil_cpp/absl/memory/memory.h new file mode 100644 index 000000000000..2b5ff623d430 --- /dev/null +++ b/third_party/abseil_cpp/absl/memory/memory.h @@ -0,0 +1,699 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: memory.h +// ----------------------------------------------------------------------------- +// +// This header file contains utility functions for managing the creation and +// conversion of smart pointers. This file is an extension to the C++ +// standard <memory> library header file. + +#ifndef ABSL_MEMORY_MEMORY_H_ +#define ABSL_MEMORY_MEMORY_H_ + +#include <cstddef> +#include <limits> +#include <memory> +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// Function Template: WrapUnique() +// ----------------------------------------------------------------------------- +// +// Adopts ownership from a raw pointer and transfers it to the returned +// `std::unique_ptr`, whose type is deduced. Because of this deduction, *do not* +// specify the template type `T` when calling `WrapUnique`. +// +// Example: +// X* NewX(int, int); +// auto x = WrapUnique(NewX(1, 2)); // 'x' is std::unique_ptr<X>. +// +// Do not call WrapUnique with an explicit type, as in +// `WrapUnique<X>(NewX(1, 2))`. The purpose of WrapUnique is to automatically +// deduce the pointer type. If you wish to make the type explicit, just use +// `std::unique_ptr` directly. +// +// auto x = std::unique_ptr<X>(NewX(1, 2)); +// - or - +// std::unique_ptr<X> x(NewX(1, 2)); +// +// While `absl::WrapUnique` is useful for capturing the output of a raw +// pointer factory, prefer 'absl::make_unique<T>(args...)' over +// 'absl::WrapUnique(new T(args...))'. +// +// auto x = WrapUnique(new X(1, 2)); // works, but nonideal. +// auto x = make_unique<X>(1, 2); // safer, standard, avoids raw 'new'. +// +// Note that `absl::WrapUnique(p)` is valid only if `delete p` is a valid +// expression. In particular, `absl::WrapUnique()` cannot wrap pointers to +// arrays, functions or void, and it must not be used to capture pointers +// obtained from array-new expressions (even though that would compile!). +template <typename T> +std::unique_ptr<T> WrapUnique(T* ptr) { + static_assert(!std::is_array<T>::value, "array types are unsupported"); + static_assert(std::is_object<T>::value, "non-object types are unsupported"); + return std::unique_ptr<T>(ptr); +} + +namespace memory_internal { + +// Traits to select proper overload and return type for `absl::make_unique<>`. +template <typename T> +struct MakeUniqueResult { + using scalar = std::unique_ptr<T>; +}; +template <typename T> +struct MakeUniqueResult<T[]> { + using array = std::unique_ptr<T[]>; +}; +template <typename T, size_t N> +struct MakeUniqueResult<T[N]> { + using invalid = void; +}; + +} // namespace memory_internal + +// gcc 4.8 has __cplusplus at 201301 but the libstdc++ shipped with it doesn't +// define make_unique. Other supported compilers either just define __cplusplus +// as 201103 but have make_unique (msvc), or have make_unique whenever +// __cplusplus > 201103 (clang). +#if (__cplusplus > 201103L || defined(_MSC_VER)) && \ + !(defined(__GLIBCXX__) && !defined(__cpp_lib_make_unique)) +using std::make_unique; +#else +// ----------------------------------------------------------------------------- +// Function Template: make_unique<T>() +// ----------------------------------------------------------------------------- +// +// Creates a `std::unique_ptr<>`, while avoiding issues creating temporaries +// during the construction process. `absl::make_unique<>` also avoids redundant +// type declarations, by avoiding the need to explicitly use the `new` operator. +// +// This implementation of `absl::make_unique<>` is designed for C++11 code and +// will be replaced in C++14 by the equivalent `std::make_unique<>` abstraction. +// `absl::make_unique<>` is designed to be 100% compatible with +// `std::make_unique<>` so that the eventual migration will involve a simple +// rename operation. +// +// For more background on why `std::unique_ptr<T>(new T(a,b))` is problematic, +// see Herb Sutter's explanation on +// (Exception-Safe Function Calls)[https://herbsutter.com/gotw/_102/]. +// (In general, reviewers should treat `new T(a,b)` with scrutiny.) +// +// Example usage: +// +// auto p = make_unique<X>(args...); // 'p' is a std::unique_ptr<X> +// auto pa = make_unique<X[]>(5); // 'pa' is a std::unique_ptr<X[]> +// +// Three overloads of `absl::make_unique` are required: +// +// - For non-array T: +// +// Allocates a T with `new T(std::forward<Args> args...)`, +// forwarding all `args` to T's constructor. +// Returns a `std::unique_ptr<T>` owning that object. +// +// - For an array of unknown bounds T[]: +// +// `absl::make_unique<>` will allocate an array T of type U[] with +// `new U[n]()` and return a `std::unique_ptr<U[]>` owning that array. +// +// Note that 'U[n]()' is different from 'U[n]', and elements will be +// value-initialized. Note as well that `std::unique_ptr` will perform its +// own destruction of the array elements upon leaving scope, even though +// the array [] does not have a default destructor. +// +// NOTE: an array of unknown bounds T[] may still be (and often will be) +// initialized to have a size, and will still use this overload. E.g: +// +// auto my_array = absl::make_unique<int[]>(10); +// +// - For an array of known bounds T[N]: +// +// `absl::make_unique<>` is deleted (like with `std::make_unique<>`) as +// this overload is not useful. +// +// NOTE: an array of known bounds T[N] is not considered a useful +// construction, and may cause undefined behavior in templates. E.g: +// +// auto my_array = absl::make_unique<int[10]>(); +// +// In those cases, of course, you can still use the overload above and +// simply initialize it to its desired size: +// +// auto my_array = absl::make_unique<int[]>(10); + +// `absl::make_unique` overload for non-array types. +template <typename T, typename... Args> +typename memory_internal::MakeUniqueResult<T>::scalar make_unique( + Args&&... args) { + return std::unique_ptr<T>(new T(std::forward<Args>(args)...)); +} + +// `absl::make_unique` overload for an array T[] of unknown bounds. +// The array allocation needs to use the `new T[size]` form and cannot take +// element constructor arguments. The `std::unique_ptr` will manage destructing +// these array elements. +template <typename T> +typename memory_internal::MakeUniqueResult<T>::array make_unique(size_t n) { + return std::unique_ptr<T>(new typename absl::remove_extent_t<T>[n]()); +} + +// `absl::make_unique` overload for an array T[N] of known bounds. +// This construction will be rejected. +template <typename T, typename... Args> +typename memory_internal::MakeUniqueResult<T>::invalid make_unique( + Args&&... /* args */) = delete; +#endif + +// ----------------------------------------------------------------------------- +// Function Template: RawPtr() +// ----------------------------------------------------------------------------- +// +// Extracts the raw pointer from a pointer-like value `ptr`. `absl::RawPtr` is +// useful within templates that need to handle a complement of raw pointers, +// `std::nullptr_t`, and smart pointers. +template <typename T> +auto RawPtr(T&& ptr) -> decltype(std::addressof(*ptr)) { + // ptr is a forwarding reference to support Ts with non-const operators. + return (ptr != nullptr) ? std::addressof(*ptr) : nullptr; +} +inline std::nullptr_t RawPtr(std::nullptr_t) { return nullptr; } + +// ----------------------------------------------------------------------------- +// Function Template: ShareUniquePtr() +// ----------------------------------------------------------------------------- +// +// Adopts a `std::unique_ptr` rvalue and returns a `std::shared_ptr` of deduced +// type. Ownership (if any) of the held value is transferred to the returned +// shared pointer. +// +// Example: +// +// auto up = absl::make_unique<int>(10); +// auto sp = absl::ShareUniquePtr(std::move(up)); // shared_ptr<int> +// CHECK_EQ(*sp, 10); +// CHECK(up == nullptr); +// +// Note that this conversion is correct even when T is an array type, and more +// generally it works for *any* deleter of the `unique_ptr` (single-object +// deleter, array deleter, or any custom deleter), since the deleter is adopted +// by the shared pointer as well. The deleter is copied (unless it is a +// reference). +// +// Implements the resolution of [LWG 2415](http://wg21.link/lwg2415), by which a +// null shared pointer does not attempt to call the deleter. +template <typename T, typename D> +std::shared_ptr<T> ShareUniquePtr(std::unique_ptr<T, D>&& ptr) { + return ptr ? std::shared_ptr<T>(std::move(ptr)) : std::shared_ptr<T>(); +} + +// ----------------------------------------------------------------------------- +// Function Template: WeakenPtr() +// ----------------------------------------------------------------------------- +// +// Creates a weak pointer associated with a given shared pointer. The returned +// value is a `std::weak_ptr` of deduced type. +// +// Example: +// +// auto sp = std::make_shared<int>(10); +// auto wp = absl::WeakenPtr(sp); +// CHECK_EQ(sp.get(), wp.lock().get()); +// sp.reset(); +// CHECK(wp.lock() == nullptr); +// +template <typename T> +std::weak_ptr<T> WeakenPtr(const std::shared_ptr<T>& ptr) { + return std::weak_ptr<T>(ptr); +} + +namespace memory_internal { + +// ExtractOr<E, O, D>::type evaluates to E<O> if possible. Otherwise, D. +template <template <typename> class Extract, typename Obj, typename Default, + typename> +struct ExtractOr { + using type = Default; +}; + +template <template <typename> class Extract, typename Obj, typename Default> +struct ExtractOr<Extract, Obj, Default, void_t<Extract<Obj>>> { + using type = Extract<Obj>; +}; + +template <template <typename> class Extract, typename Obj, typename Default> +using ExtractOrT = typename ExtractOr<Extract, Obj, Default, void>::type; + +// Extractors for the features of allocators. +template <typename T> +using GetPointer = typename T::pointer; + +template <typename T> +using GetConstPointer = typename T::const_pointer; + +template <typename T> +using GetVoidPointer = typename T::void_pointer; + +template <typename T> +using GetConstVoidPointer = typename T::const_void_pointer; + +template <typename T> +using GetDifferenceType = typename T::difference_type; + +template <typename T> +using GetSizeType = typename T::size_type; + +template <typename T> +using GetPropagateOnContainerCopyAssignment = + typename T::propagate_on_container_copy_assignment; + +template <typename T> +using GetPropagateOnContainerMoveAssignment = + typename T::propagate_on_container_move_assignment; + +template <typename T> +using GetPropagateOnContainerSwap = typename T::propagate_on_container_swap; + +template <typename T> +using GetIsAlwaysEqual = typename T::is_always_equal; + +template <typename T> +struct GetFirstArg; + +template <template <typename...> class Class, typename T, typename... Args> +struct GetFirstArg<Class<T, Args...>> { + using type = T; +}; + +template <typename Ptr, typename = void> +struct ElementType { + using type = typename GetFirstArg<Ptr>::type; +}; + +template <typename T> +struct ElementType<T, void_t<typename T::element_type>> { + using type = typename T::element_type; +}; + +template <typename T, typename U> +struct RebindFirstArg; + +template <template <typename...> class Class, typename T, typename... Args, + typename U> +struct RebindFirstArg<Class<T, Args...>, U> { + using type = Class<U, Args...>; +}; + +template <typename T, typename U, typename = void> +struct RebindPtr { + using type = typename RebindFirstArg<T, U>::type; +}; + +template <typename T, typename U> +struct RebindPtr<T, U, void_t<typename T::template rebind<U>>> { + using type = typename T::template rebind<U>; +}; + +template <typename T, typename U> +constexpr bool HasRebindAlloc(...) { + return false; +} + +template <typename T, typename U> +constexpr bool HasRebindAlloc(typename T::template rebind<U>::other*) { + return true; +} + +template <typename T, typename U, bool = HasRebindAlloc<T, U>(nullptr)> +struct RebindAlloc { + using type = typename RebindFirstArg<T, U>::type; +}; + +template <typename T, typename U> +struct RebindAlloc<T, U, true> { + using type = typename T::template rebind<U>::other; +}; + +} // namespace memory_internal + +// ----------------------------------------------------------------------------- +// Class Template: pointer_traits +// ----------------------------------------------------------------------------- +// +// An implementation of C++11's std::pointer_traits. +// +// Provided for portability on toolchains that have a working C++11 compiler, +// but the standard library is lacking in C++11 support. For example, some +// version of the Android NDK. +// + +template <typename Ptr> +struct pointer_traits { + using pointer = Ptr; + + // element_type: + // Ptr::element_type if present. Otherwise T if Ptr is a template + // instantiation Template<T, Args...> + using element_type = typename memory_internal::ElementType<Ptr>::type; + + // difference_type: + // Ptr::difference_type if present, otherwise std::ptrdiff_t + using difference_type = + memory_internal::ExtractOrT<memory_internal::GetDifferenceType, Ptr, + std::ptrdiff_t>; + + // rebind: + // Ptr::rebind<U> if exists, otherwise Template<U, Args...> if Ptr is a + // template instantiation Template<T, Args...> + template <typename U> + using rebind = typename memory_internal::RebindPtr<Ptr, U>::type; + + // pointer_to: + // Calls Ptr::pointer_to(r) + static pointer pointer_to(element_type& r) { // NOLINT(runtime/references) + return Ptr::pointer_to(r); + } +}; + +// Specialization for T*. +template <typename T> +struct pointer_traits<T*> { + using pointer = T*; + using element_type = T; + using difference_type = std::ptrdiff_t; + + template <typename U> + using rebind = U*; + + // pointer_to: + // Calls std::addressof(r) + static pointer pointer_to( + element_type& r) noexcept { // NOLINT(runtime/references) + return std::addressof(r); + } +}; + +// ----------------------------------------------------------------------------- +// Class Template: allocator_traits +// ----------------------------------------------------------------------------- +// +// A C++11 compatible implementation of C++17's std::allocator_traits. +// +#if __cplusplus >= 201703L +using std::allocator_traits; +#else // __cplusplus >= 201703L +template <typename Alloc> +struct allocator_traits { + using allocator_type = Alloc; + + // value_type: + // Alloc::value_type + using value_type = typename Alloc::value_type; + + // pointer: + // Alloc::pointer if present, otherwise value_type* + using pointer = memory_internal::ExtractOrT<memory_internal::GetPointer, + Alloc, value_type*>; + + // const_pointer: + // Alloc::const_pointer if present, otherwise + // absl::pointer_traits<pointer>::rebind<const value_type> + using const_pointer = + memory_internal::ExtractOrT<memory_internal::GetConstPointer, Alloc, + typename absl::pointer_traits<pointer>:: + template rebind<const value_type>>; + + // void_pointer: + // Alloc::void_pointer if present, otherwise + // absl::pointer_traits<pointer>::rebind<void> + using void_pointer = memory_internal::ExtractOrT< + memory_internal::GetVoidPointer, Alloc, + typename absl::pointer_traits<pointer>::template rebind<void>>; + + // const_void_pointer: + // Alloc::const_void_pointer if present, otherwise + // absl::pointer_traits<pointer>::rebind<const void> + using const_void_pointer = memory_internal::ExtractOrT< + memory_internal::GetConstVoidPointer, Alloc, + typename absl::pointer_traits<pointer>::template rebind<const void>>; + + // difference_type: + // Alloc::difference_type if present, otherwise + // absl::pointer_traits<pointer>::difference_type + using difference_type = memory_internal::ExtractOrT< + memory_internal::GetDifferenceType, Alloc, + typename absl::pointer_traits<pointer>::difference_type>; + + // size_type: + // Alloc::size_type if present, otherwise + // std::make_unsigned<difference_type>::type + using size_type = memory_internal::ExtractOrT< + memory_internal::GetSizeType, Alloc, + typename std::make_unsigned<difference_type>::type>; + + // propagate_on_container_copy_assignment: + // Alloc::propagate_on_container_copy_assignment if present, otherwise + // std::false_type + using propagate_on_container_copy_assignment = memory_internal::ExtractOrT< + memory_internal::GetPropagateOnContainerCopyAssignment, Alloc, + std::false_type>; + + // propagate_on_container_move_assignment: + // Alloc::propagate_on_container_move_assignment if present, otherwise + // std::false_type + using propagate_on_container_move_assignment = memory_internal::ExtractOrT< + memory_internal::GetPropagateOnContainerMoveAssignment, Alloc, + std::false_type>; + + // propagate_on_container_swap: + // Alloc::propagate_on_container_swap if present, otherwise std::false_type + using propagate_on_container_swap = + memory_internal::ExtractOrT<memory_internal::GetPropagateOnContainerSwap, + Alloc, std::false_type>; + + // is_always_equal: + // Alloc::is_always_equal if present, otherwise std::is_empty<Alloc>::type + using is_always_equal = + memory_internal::ExtractOrT<memory_internal::GetIsAlwaysEqual, Alloc, + typename std::is_empty<Alloc>::type>; + + // rebind_alloc: + // Alloc::rebind<T>::other if present, otherwise Alloc<T, Args> if this Alloc + // is Alloc<U, Args> + template <typename T> + using rebind_alloc = typename memory_internal::RebindAlloc<Alloc, T>::type; + + // rebind_traits: + // absl::allocator_traits<rebind_alloc<T>> + template <typename T> + using rebind_traits = absl::allocator_traits<rebind_alloc<T>>; + + // allocate(Alloc& a, size_type n): + // Calls a.allocate(n) + static pointer allocate(Alloc& a, // NOLINT(runtime/references) + size_type n) { + return a.allocate(n); + } + + // allocate(Alloc& a, size_type n, const_void_pointer hint): + // Calls a.allocate(n, hint) if possible. + // If not possible, calls a.allocate(n) + static pointer allocate(Alloc& a, size_type n, // NOLINT(runtime/references) + const_void_pointer hint) { + return allocate_impl(0, a, n, hint); + } + + // deallocate(Alloc& a, pointer p, size_type n): + // Calls a.deallocate(p, n) + static void deallocate(Alloc& a, pointer p, // NOLINT(runtime/references) + size_type n) { + a.deallocate(p, n); + } + + // construct(Alloc& a, T* p, Args&&... args): + // Calls a.construct(p, std::forward<Args>(args)...) if possible. + // If not possible, calls + // ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...) + template <typename T, typename... Args> + static void construct(Alloc& a, T* p, // NOLINT(runtime/references) + Args&&... args) { + construct_impl(0, a, p, std::forward<Args>(args)...); + } + + // destroy(Alloc& a, T* p): + // Calls a.destroy(p) if possible. If not possible, calls p->~T(). + template <typename T> + static void destroy(Alloc& a, T* p) { // NOLINT(runtime/references) + destroy_impl(0, a, p); + } + + // max_size(const Alloc& a): + // Returns a.max_size() if possible. If not possible, returns + // std::numeric_limits<size_type>::max() / sizeof(value_type) + static size_type max_size(const Alloc& a) { return max_size_impl(0, a); } + + // select_on_container_copy_construction(const Alloc& a): + // Returns a.select_on_container_copy_construction() if possible. + // If not possible, returns a. + static Alloc select_on_container_copy_construction(const Alloc& a) { + return select_on_container_copy_construction_impl(0, a); + } + + private: + template <typename A> + static auto allocate_impl(int, A& a, // NOLINT(runtime/references) + size_type n, const_void_pointer hint) + -> decltype(a.allocate(n, hint)) { + return a.allocate(n, hint); + } + static pointer allocate_impl(char, Alloc& a, // NOLINT(runtime/references) + size_type n, const_void_pointer) { + return a.allocate(n); + } + + template <typename A, typename... Args> + static auto construct_impl(int, A& a, // NOLINT(runtime/references) + Args&&... args) + -> decltype(a.construct(std::forward<Args>(args)...)) { + a.construct(std::forward<Args>(args)...); + } + + template <typename T, typename... Args> + static void construct_impl(char, Alloc&, T* p, Args&&... args) { + ::new (static_cast<void*>(p)) T(std::forward<Args>(args)...); + } + + template <typename A, typename T> + static auto destroy_impl(int, A& a, // NOLINT(runtime/references) + T* p) -> decltype(a.destroy(p)) { + a.destroy(p); + } + template <typename T> + static void destroy_impl(char, Alloc&, T* p) { + p->~T(); + } + + template <typename A> + static auto max_size_impl(int, const A& a) -> decltype(a.max_size()) { + return a.max_size(); + } + static size_type max_size_impl(char, const Alloc&) { + return (std::numeric_limits<size_type>::max)() / sizeof(value_type); + } + + template <typename A> + static auto select_on_container_copy_construction_impl(int, const A& a) + -> decltype(a.select_on_container_copy_construction()) { + return a.select_on_container_copy_construction(); + } + static Alloc select_on_container_copy_construction_impl(char, + const Alloc& a) { + return a; + } +}; +#endif // __cplusplus >= 201703L + +namespace memory_internal { + +// This template alias transforms Alloc::is_nothrow into a metafunction with +// Alloc as a parameter so it can be used with ExtractOrT<>. +template <typename Alloc> +using GetIsNothrow = typename Alloc::is_nothrow; + +} // namespace memory_internal + +// ABSL_ALLOCATOR_NOTHROW is a build time configuration macro for user to +// specify whether the default allocation function can throw or never throws. +// If the allocation function never throws, user should define it to a non-zero +// value (e.g. via `-DABSL_ALLOCATOR_NOTHROW`). +// If the allocation function can throw, user should leave it undefined or +// define it to zero. +// +// allocator_is_nothrow<Alloc> is a traits class that derives from +// Alloc::is_nothrow if present, otherwise std::false_type. It's specialized +// for Alloc = std::allocator<T> for any type T according to the state of +// ABSL_ALLOCATOR_NOTHROW. +// +// default_allocator_is_nothrow is a class that derives from std::true_type +// when the default allocator (global operator new) never throws, and +// std::false_type when it can throw. It is a convenience shorthand for writing +// allocator_is_nothrow<std::allocator<T>> (T can be any type). +// NOTE: allocator_is_nothrow<std::allocator<T>> is guaranteed to derive from +// the same type for all T, because users should specialize neither +// allocator_is_nothrow nor std::allocator. +template <typename Alloc> +struct allocator_is_nothrow + : memory_internal::ExtractOrT<memory_internal::GetIsNothrow, Alloc, + std::false_type> {}; + +#if defined(ABSL_ALLOCATOR_NOTHROW) && ABSL_ALLOCATOR_NOTHROW +template <typename T> +struct allocator_is_nothrow<std::allocator<T>> : std::true_type {}; +struct default_allocator_is_nothrow : std::true_type {}; +#else +struct default_allocator_is_nothrow : std::false_type {}; +#endif + +namespace memory_internal { +template <typename Allocator, typename Iterator, typename... Args> +void ConstructRange(Allocator& alloc, Iterator first, Iterator last, + const Args&... args) { + for (Iterator cur = first; cur != last; ++cur) { + ABSL_INTERNAL_TRY { + std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), + args...); + } + ABSL_INTERNAL_CATCH_ANY { + while (cur != first) { + --cur; + std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); + } + ABSL_INTERNAL_RETHROW; + } + } +} + +template <typename Allocator, typename Iterator, typename InputIterator> +void CopyRange(Allocator& alloc, Iterator destination, InputIterator first, + InputIterator last) { + for (Iterator cur = destination; first != last; + static_cast<void>(++cur), static_cast<void>(++first)) { + ABSL_INTERNAL_TRY { + std::allocator_traits<Allocator>::construct(alloc, std::addressof(*cur), + *first); + } + ABSL_INTERNAL_CATCH_ANY { + while (cur != destination) { + --cur; + std::allocator_traits<Allocator>::destroy(alloc, std::addressof(*cur)); + } + ABSL_INTERNAL_RETHROW; + } + } +} +} // namespace memory_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_MEMORY_MEMORY_H_ diff --git a/third_party/abseil_cpp/absl/memory/memory_exception_safety_test.cc b/third_party/abseil_cpp/absl/memory/memory_exception_safety_test.cc new file mode 100644 index 000000000000..1df72614c00d --- /dev/null +++ b/third_party/abseil_cpp/absl/memory/memory_exception_safety_test.cc @@ -0,0 +1,57 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/memory/memory.h" + +#include "absl/base/config.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + +#include "gtest/gtest.h" +#include "absl/base/internal/exception_safety_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +constexpr int kLength = 50; +using Thrower = testing::ThrowingValue<testing::TypeSpec::kEverythingThrows>; + +TEST(MakeUnique, CheckForLeaks) { + constexpr int kValue = 321; + auto tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(Thrower(kValue)) + // Ensures make_unique does not modify the input. The real + // test, though, is ConstructorTracker checking for leaks. + .WithContracts(testing::strong_guarantee); + + EXPECT_TRUE(tester.Test([](Thrower* thrower) { + static_cast<void>(absl::make_unique<Thrower>(*thrower)); + })); + + EXPECT_TRUE(tester.Test([](Thrower* thrower) { + static_cast<void>(absl::make_unique<Thrower>(std::move(*thrower))); + })); + + // Test T[n] overload + EXPECT_TRUE(tester.Test([&](Thrower*) { + static_cast<void>(absl::make_unique<Thrower[]>(kLength)); + })); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_HAVE_EXCEPTIONS diff --git a/third_party/abseil_cpp/absl/memory/memory_test.cc b/third_party/abseil_cpp/absl/memory/memory_test.cc new file mode 100644 index 000000000000..1990c7ba4728 --- /dev/null +++ b/third_party/abseil_cpp/absl/memory/memory_test.cc @@ -0,0 +1,650 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for pointer utilities. + +#include "absl/memory/memory.h" + +#include <sys/types.h> + +#include <cstddef> +#include <memory> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { + +using ::testing::ElementsAre; +using ::testing::Return; + +// This class creates observable behavior to verify that a destructor has +// been called, via the instance_count variable. +class DestructorVerifier { + public: + DestructorVerifier() { ++instance_count_; } + DestructorVerifier(const DestructorVerifier&) = delete; + DestructorVerifier& operator=(const DestructorVerifier&) = delete; + ~DestructorVerifier() { --instance_count_; } + + // The number of instances of this class currently active. + static int instance_count() { return instance_count_; } + + private: + // The number of instances of this class currently active. + static int instance_count_; +}; + +int DestructorVerifier::instance_count_ = 0; + +TEST(WrapUniqueTest, WrapUnique) { + // Test that the unique_ptr is constructed properly by verifying that the + // destructor for its payload gets called at the proper time. + { + auto dv = new DestructorVerifier; + EXPECT_EQ(1, DestructorVerifier::instance_count()); + std::unique_ptr<DestructorVerifier> ptr = absl::WrapUnique(dv); + EXPECT_EQ(1, DestructorVerifier::instance_count()); + } + EXPECT_EQ(0, DestructorVerifier::instance_count()); +} +TEST(MakeUniqueTest, Basic) { + std::unique_ptr<std::string> p = absl::make_unique<std::string>(); + EXPECT_EQ("", *p); + p = absl::make_unique<std::string>("hi"); + EXPECT_EQ("hi", *p); +} + +// InitializationVerifier fills in a pattern when allocated so we can +// distinguish between its default and value initialized states (without +// accessing truly uninitialized memory). +struct InitializationVerifier { + static constexpr int kDefaultScalar = 0x43; + static constexpr int kDefaultArray = 0x4B; + + static void* operator new(size_t n) { + void* ret = ::operator new(n); + memset(ret, kDefaultScalar, n); + return ret; + } + + static void* operator new[](size_t n) { + void* ret = ::operator new[](n); + memset(ret, kDefaultArray, n); + return ret; + } + + int a; + int b; +}; + +TEST(Initialization, MakeUnique) { + auto p = absl::make_unique<InitializationVerifier>(); + + EXPECT_EQ(0, p->a); + EXPECT_EQ(0, p->b); +} + +TEST(Initialization, MakeUniqueArray) { + auto p = absl::make_unique<InitializationVerifier[]>(2); + + EXPECT_EQ(0, p[0].a); + EXPECT_EQ(0, p[0].b); + EXPECT_EQ(0, p[1].a); + EXPECT_EQ(0, p[1].b); +} + +struct MoveOnly { + MoveOnly() = default; + explicit MoveOnly(int i1) : ip1{new int{i1}} {} + MoveOnly(int i1, int i2) : ip1{new int{i1}}, ip2{new int{i2}} {} + std::unique_ptr<int> ip1; + std::unique_ptr<int> ip2; +}; + +struct AcceptMoveOnly { + explicit AcceptMoveOnly(MoveOnly m) : m_(std::move(m)) {} + MoveOnly m_; +}; + +TEST(MakeUniqueTest, MoveOnlyTypeAndValue) { + using ExpectedType = std::unique_ptr<MoveOnly>; + { + auto p = absl::make_unique<MoveOnly>(); + static_assert(std::is_same<decltype(p), ExpectedType>::value, + "unexpected return type"); + EXPECT_TRUE(!p->ip1); + EXPECT_TRUE(!p->ip2); + } + { + auto p = absl::make_unique<MoveOnly>(1); + static_assert(std::is_same<decltype(p), ExpectedType>::value, + "unexpected return type"); + EXPECT_TRUE(p->ip1 && *p->ip1 == 1); + EXPECT_TRUE(!p->ip2); + } + { + auto p = absl::make_unique<MoveOnly>(1, 2); + static_assert(std::is_same<decltype(p), ExpectedType>::value, + "unexpected return type"); + EXPECT_TRUE(p->ip1 && *p->ip1 == 1); + EXPECT_TRUE(p->ip2 && *p->ip2 == 2); + } +} + +TEST(MakeUniqueTest, AcceptMoveOnly) { + auto p = absl::make_unique<AcceptMoveOnly>(MoveOnly()); + p = std::unique_ptr<AcceptMoveOnly>(new AcceptMoveOnly(MoveOnly())); +} + +struct ArrayWatch { + void* operator new[](size_t n) { + allocs().push_back(n); + return ::operator new[](n); + } + void operator delete[](void* p) { return ::operator delete[](p); } + static std::vector<size_t>& allocs() { + static auto& v = *new std::vector<size_t>; + return v; + } +}; + +TEST(Make_UniqueTest, Array) { + // Ensure state is clean before we start so that these tests + // are order-agnostic. + ArrayWatch::allocs().clear(); + + auto p = absl::make_unique<ArrayWatch[]>(5); + static_assert(std::is_same<decltype(p), std::unique_ptr<ArrayWatch[]>>::value, + "unexpected return type"); + EXPECT_THAT(ArrayWatch::allocs(), ElementsAre(5 * sizeof(ArrayWatch))); +} + +TEST(Make_UniqueTest, NotAmbiguousWithStdMakeUnique) { + // Ensure that absl::make_unique is not ambiguous with std::make_unique. + // In C++14 mode, the below call to make_unique has both types as candidates. + struct TakesStdType { + explicit TakesStdType(const std::vector<int>& vec) {} + }; + using absl::make_unique; + (void)make_unique<TakesStdType>(std::vector<int>()); +} + +#if 0 +// These tests shouldn't compile. +TEST(MakeUniqueTestNC, AcceptMoveOnlyLvalue) { + auto m = MoveOnly(); + auto p = absl::make_unique<AcceptMoveOnly>(m); +} +TEST(MakeUniqueTestNC, KnownBoundArray) { + auto p = absl::make_unique<ArrayWatch[5]>(); +} +#endif + +TEST(RawPtrTest, RawPointer) { + int i = 5; + EXPECT_EQ(&i, absl::RawPtr(&i)); +} + +TEST(RawPtrTest, SmartPointer) { + int* o = new int(5); + std::unique_ptr<int> p(o); + EXPECT_EQ(o, absl::RawPtr(p)); +} + +class IntPointerNonConstDeref { + public: + explicit IntPointerNonConstDeref(int* p) : p_(p) {} + friend bool operator!=(const IntPointerNonConstDeref& a, std::nullptr_t) { + return a.p_ != nullptr; + } + int& operator*() { return *p_; } + + private: + std::unique_ptr<int> p_; +}; + +TEST(RawPtrTest, SmartPointerNonConstDereference) { + int* o = new int(5); + IntPointerNonConstDeref p(o); + EXPECT_EQ(o, absl::RawPtr(p)); +} + +TEST(RawPtrTest, NullValuedRawPointer) { + int* p = nullptr; + EXPECT_EQ(nullptr, absl::RawPtr(p)); +} + +TEST(RawPtrTest, NullValuedSmartPointer) { + std::unique_ptr<int> p; + EXPECT_EQ(nullptr, absl::RawPtr(p)); +} + +TEST(RawPtrTest, Nullptr) { + auto p = absl::RawPtr(nullptr); + EXPECT_TRUE((std::is_same<std::nullptr_t, decltype(p)>::value)); + EXPECT_EQ(nullptr, p); +} + +TEST(RawPtrTest, Null) { + auto p = absl::RawPtr(nullptr); + EXPECT_TRUE((std::is_same<std::nullptr_t, decltype(p)>::value)); + EXPECT_EQ(nullptr, p); +} + +TEST(RawPtrTest, Zero) { + auto p = absl::RawPtr(nullptr); + EXPECT_TRUE((std::is_same<std::nullptr_t, decltype(p)>::value)); + EXPECT_EQ(nullptr, p); +} + +TEST(ShareUniquePtrTest, Share) { + auto up = absl::make_unique<int>(); + int* rp = up.get(); + auto sp = absl::ShareUniquePtr(std::move(up)); + EXPECT_EQ(sp.get(), rp); +} + +TEST(ShareUniquePtrTest, ShareNull) { + struct NeverDie { + using pointer = void*; + void operator()(pointer) { + ASSERT_TRUE(false) << "Deleter should not have been called."; + } + }; + + std::unique_ptr<void, NeverDie> up; + auto sp = absl::ShareUniquePtr(std::move(up)); +} + +TEST(WeakenPtrTest, Weak) { + auto sp = std::make_shared<int>(); + auto wp = absl::WeakenPtr(sp); + EXPECT_EQ(sp.get(), wp.lock().get()); + sp.reset(); + EXPECT_TRUE(wp.expired()); +} + +// Should not compile. +/* +TEST(RawPtrTest, NotAPointer) { + absl::RawPtr(1.5); +} +*/ + +template <typename T> +struct SmartPointer { + using difference_type = char; +}; + +struct PointerWith { + using element_type = int32_t; + using difference_type = int16_t; + template <typename U> + using rebind = SmartPointer<U>; + + static PointerWith pointer_to( + element_type& r) { // NOLINT(runtime/references) + return PointerWith{&r}; + } + + element_type* ptr; +}; + +template <typename... Args> +struct PointerWithout {}; + +TEST(PointerTraits, Types) { + using TraitsWith = absl::pointer_traits<PointerWith>; + EXPECT_TRUE((std::is_same<TraitsWith::pointer, PointerWith>::value)); + EXPECT_TRUE((std::is_same<TraitsWith::element_type, int32_t>::value)); + EXPECT_TRUE((std::is_same<TraitsWith::difference_type, int16_t>::value)); + EXPECT_TRUE(( + std::is_same<TraitsWith::rebind<int64_t>, SmartPointer<int64_t>>::value)); + + using TraitsWithout = absl::pointer_traits<PointerWithout<double, int>>; + EXPECT_TRUE((std::is_same<TraitsWithout::pointer, + PointerWithout<double, int>>::value)); + EXPECT_TRUE((std::is_same<TraitsWithout::element_type, double>::value)); + EXPECT_TRUE( + (std::is_same<TraitsWithout ::difference_type, std::ptrdiff_t>::value)); + EXPECT_TRUE((std::is_same<TraitsWithout::rebind<int64_t>, + PointerWithout<int64_t, int>>::value)); + + using TraitsRawPtr = absl::pointer_traits<char*>; + EXPECT_TRUE((std::is_same<TraitsRawPtr::pointer, char*>::value)); + EXPECT_TRUE((std::is_same<TraitsRawPtr::element_type, char>::value)); + EXPECT_TRUE( + (std::is_same<TraitsRawPtr::difference_type, std::ptrdiff_t>::value)); + EXPECT_TRUE((std::is_same<TraitsRawPtr::rebind<int64_t>, int64_t*>::value)); +} + +TEST(PointerTraits, Functions) { + int i; + EXPECT_EQ(&i, absl::pointer_traits<PointerWith>::pointer_to(i).ptr); + EXPECT_EQ(&i, absl::pointer_traits<int*>::pointer_to(i)); +} + +TEST(AllocatorTraits, Typedefs) { + struct A { + struct value_type {}; + }; + EXPECT_TRUE(( + std::is_same<A, + typename absl::allocator_traits<A>::allocator_type>::value)); + EXPECT_TRUE( + (std::is_same<A::value_type, + typename absl::allocator_traits<A>::value_type>::value)); + + struct X {}; + struct HasPointer { + using value_type = X; + using pointer = SmartPointer<X>; + }; + EXPECT_TRUE((std::is_same<SmartPointer<X>, typename absl::allocator_traits< + HasPointer>::pointer>::value)); + EXPECT_TRUE( + (std::is_same<A::value_type*, + typename absl::allocator_traits<A>::pointer>::value)); + + EXPECT_TRUE( + (std::is_same< + SmartPointer<const X>, + typename absl::allocator_traits<HasPointer>::const_pointer>::value)); + EXPECT_TRUE( + (std::is_same<const A::value_type*, + typename absl::allocator_traits<A>::const_pointer>::value)); + + struct HasVoidPointer { + using value_type = X; + struct void_pointer {}; + }; + + EXPECT_TRUE((std::is_same<HasVoidPointer::void_pointer, + typename absl::allocator_traits< + HasVoidPointer>::void_pointer>::value)); + EXPECT_TRUE( + (std::is_same<SmartPointer<void>, typename absl::allocator_traits< + HasPointer>::void_pointer>::value)); + + struct HasConstVoidPointer { + using value_type = X; + struct const_void_pointer {}; + }; + + EXPECT_TRUE( + (std::is_same<HasConstVoidPointer::const_void_pointer, + typename absl::allocator_traits< + HasConstVoidPointer>::const_void_pointer>::value)); + EXPECT_TRUE((std::is_same<SmartPointer<const void>, + typename absl::allocator_traits< + HasPointer>::const_void_pointer>::value)); + + struct HasDifferenceType { + using value_type = X; + using difference_type = int; + }; + EXPECT_TRUE( + (std::is_same<int, typename absl::allocator_traits< + HasDifferenceType>::difference_type>::value)); + EXPECT_TRUE((std::is_same<char, typename absl::allocator_traits< + HasPointer>::difference_type>::value)); + + struct HasSizeType { + using value_type = X; + using size_type = unsigned int; + }; + EXPECT_TRUE((std::is_same<unsigned int, typename absl::allocator_traits< + HasSizeType>::size_type>::value)); + EXPECT_TRUE((std::is_same<unsigned char, typename absl::allocator_traits< + HasPointer>::size_type>::value)); + + struct HasPropagateOnCopy { + using value_type = X; + struct propagate_on_container_copy_assignment {}; + }; + + EXPECT_TRUE( + (std::is_same<HasPropagateOnCopy::propagate_on_container_copy_assignment, + typename absl::allocator_traits<HasPropagateOnCopy>:: + propagate_on_container_copy_assignment>::value)); + EXPECT_TRUE( + (std::is_same<std::false_type, + typename absl::allocator_traits< + A>::propagate_on_container_copy_assignment>::value)); + + struct HasPropagateOnMove { + using value_type = X; + struct propagate_on_container_move_assignment {}; + }; + + EXPECT_TRUE( + (std::is_same<HasPropagateOnMove::propagate_on_container_move_assignment, + typename absl::allocator_traits<HasPropagateOnMove>:: + propagate_on_container_move_assignment>::value)); + EXPECT_TRUE( + (std::is_same<std::false_type, + typename absl::allocator_traits< + A>::propagate_on_container_move_assignment>::value)); + + struct HasPropagateOnSwap { + using value_type = X; + struct propagate_on_container_swap {}; + }; + + EXPECT_TRUE( + (std::is_same<HasPropagateOnSwap::propagate_on_container_swap, + typename absl::allocator_traits<HasPropagateOnSwap>:: + propagate_on_container_swap>::value)); + EXPECT_TRUE( + (std::is_same<std::false_type, typename absl::allocator_traits<A>:: + propagate_on_container_swap>::value)); + + struct HasIsAlwaysEqual { + using value_type = X; + struct is_always_equal {}; + }; + + EXPECT_TRUE((std::is_same<HasIsAlwaysEqual::is_always_equal, + typename absl::allocator_traits< + HasIsAlwaysEqual>::is_always_equal>::value)); + EXPECT_TRUE((std::is_same<std::true_type, typename absl::allocator_traits< + A>::is_always_equal>::value)); + struct NonEmpty { + using value_type = X; + int i; + }; + EXPECT_TRUE( + (std::is_same<std::false_type, + absl::allocator_traits<NonEmpty>::is_always_equal>::value)); +} + +template <typename T> +struct AllocWithPrivateInheritance : private std::allocator<T> { + using value_type = T; +}; + +TEST(AllocatorTraits, RebindWithPrivateInheritance) { + // Regression test for some versions of gcc that do not like the sfinae we + // used in combination with private inheritance. + EXPECT_TRUE( + (std::is_same<AllocWithPrivateInheritance<int>, + absl::allocator_traits<AllocWithPrivateInheritance<char>>:: + rebind_alloc<int>>::value)); +} + +template <typename T> +struct Rebound {}; + +struct AllocWithRebind { + using value_type = int; + template <typename T> + struct rebind { + using other = Rebound<T>; + }; +}; + +template <typename T, typename U> +struct AllocWithoutRebind { + using value_type = int; +}; + +TEST(AllocatorTraits, Rebind) { + EXPECT_TRUE( + (std::is_same<Rebound<int>, + typename absl::allocator_traits< + AllocWithRebind>::template rebind_alloc<int>>::value)); + EXPECT_TRUE( + (std::is_same<absl::allocator_traits<Rebound<int>>, + typename absl::allocator_traits< + AllocWithRebind>::template rebind_traits<int>>::value)); + + EXPECT_TRUE( + (std::is_same<AllocWithoutRebind<double, char>, + typename absl::allocator_traits<AllocWithoutRebind< + int, char>>::template rebind_alloc<double>>::value)); + EXPECT_TRUE( + (std::is_same<absl::allocator_traits<AllocWithoutRebind<double, char>>, + typename absl::allocator_traits<AllocWithoutRebind< + int, char>>::template rebind_traits<double>>::value)); +} + +struct TestValue { + TestValue() {} + explicit TestValue(int* trace) : trace(trace) { ++*trace; } + ~TestValue() { + if (trace) --*trace; + } + int* trace = nullptr; +}; + +struct MinimalMockAllocator { + MinimalMockAllocator() : value(0) {} + explicit MinimalMockAllocator(int value) : value(value) {} + MinimalMockAllocator(const MinimalMockAllocator& other) + : value(other.value) {} + using value_type = TestValue; + MOCK_METHOD(value_type*, allocate, (size_t)); + MOCK_METHOD(void, deallocate, (value_type*, size_t)); + + int value; +}; + +TEST(AllocatorTraits, FunctionsMinimal) { + int trace = 0; + int hint; + TestValue x(&trace); + MinimalMockAllocator mock; + using Traits = absl::allocator_traits<MinimalMockAllocator>; + EXPECT_CALL(mock, allocate(7)).WillRepeatedly(Return(&x)); + EXPECT_CALL(mock, deallocate(&x, 7)); + + EXPECT_EQ(&x, Traits::allocate(mock, 7)); + static_cast<void>(Traits::allocate(mock, 7, static_cast<const void*>(&hint))); + EXPECT_EQ(&x, Traits::allocate(mock, 7, static_cast<const void*>(&hint))); + Traits::deallocate(mock, &x, 7); + + EXPECT_EQ(1, trace); + Traits::construct(mock, &x, &trace); + EXPECT_EQ(2, trace); + Traits::destroy(mock, &x); + EXPECT_EQ(1, trace); + + EXPECT_EQ(std::numeric_limits<size_t>::max() / sizeof(TestValue), + Traits::max_size(mock)); + + EXPECT_EQ(0, mock.value); + EXPECT_EQ(0, Traits::select_on_container_copy_construction(mock).value); +} + +struct FullMockAllocator { + FullMockAllocator() : value(0) {} + explicit FullMockAllocator(int value) : value(value) {} + FullMockAllocator(const FullMockAllocator& other) : value(other.value) {} + using value_type = TestValue; + MOCK_METHOD(value_type*, allocate, (size_t)); + MOCK_METHOD(value_type*, allocate, (size_t, const void*)); + MOCK_METHOD(void, construct, (value_type*, int*)); + MOCK_METHOD(void, destroy, (value_type*)); + MOCK_METHOD(size_t, max_size, (), + (const)); + MOCK_METHOD(FullMockAllocator, select_on_container_copy_construction, (), + (const)); + + int value; +}; + +TEST(AllocatorTraits, FunctionsFull) { + int trace = 0; + int hint; + TestValue x(&trace), y; + FullMockAllocator mock; + using Traits = absl::allocator_traits<FullMockAllocator>; + EXPECT_CALL(mock, allocate(7)).WillRepeatedly(Return(&x)); + EXPECT_CALL(mock, allocate(13, &hint)).WillRepeatedly(Return(&y)); + EXPECT_CALL(mock, construct(&x, &trace)); + EXPECT_CALL(mock, destroy(&x)); + EXPECT_CALL(mock, max_size()).WillRepeatedly(Return(17)); + EXPECT_CALL(mock, select_on_container_copy_construction()) + .WillRepeatedly(Return(FullMockAllocator(23))); + + EXPECT_EQ(&x, Traits::allocate(mock, 7)); + EXPECT_EQ(&y, Traits::allocate(mock, 13, static_cast<const void*>(&hint))); + + EXPECT_EQ(1, trace); + Traits::construct(mock, &x, &trace); + EXPECT_EQ(1, trace); + Traits::destroy(mock, &x); + EXPECT_EQ(1, trace); + + EXPECT_EQ(17, Traits::max_size(mock)); + + EXPECT_EQ(0, mock.value); + EXPECT_EQ(23, Traits::select_on_container_copy_construction(mock).value); +} + +TEST(AllocatorNoThrowTest, DefaultAllocator) { +#if defined(ABSL_ALLOCATOR_NOTHROW) && ABSL_ALLOCATOR_NOTHROW + EXPECT_TRUE(absl::default_allocator_is_nothrow::value); +#else + EXPECT_FALSE(absl::default_allocator_is_nothrow::value); +#endif +} + +TEST(AllocatorNoThrowTest, StdAllocator) { +#if defined(ABSL_ALLOCATOR_NOTHROW) && ABSL_ALLOCATOR_NOTHROW + EXPECT_TRUE(absl::allocator_is_nothrow<std::allocator<int>>::value); +#else + EXPECT_FALSE(absl::allocator_is_nothrow<std::allocator<int>>::value); +#endif +} + +TEST(AllocatorNoThrowTest, CustomAllocator) { + struct NoThrowAllocator { + using is_nothrow = std::true_type; + }; + struct CanThrowAllocator { + using is_nothrow = std::false_type; + }; + struct UnspecifiedAllocator {}; + EXPECT_TRUE(absl::allocator_is_nothrow<NoThrowAllocator>::value); + EXPECT_FALSE(absl::allocator_is_nothrow<CanThrowAllocator>::value); + EXPECT_FALSE(absl::allocator_is_nothrow<UnspecifiedAllocator>::value); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/meta/BUILD.bazel b/third_party/abseil_cpp/absl/meta/BUILD.bazel new file mode 100644 index 000000000000..5585fcca794a --- /dev/null +++ b/third_party/abseil_cpp/absl/meta/BUILD.bazel @@ -0,0 +1,48 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "type_traits", + hdrs = ["type_traits.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + ], +) + +cc_test( + name = "type_traits_test", + srcs = ["type_traits_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":type_traits", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/meta/CMakeLists.txt b/third_party/abseil_cpp/absl/meta/CMakeLists.txt new file mode 100644 index 000000000000..672ead2fd0a5 --- /dev/null +++ b/third_party/abseil_cpp/absl/meta/CMakeLists.txt @@ -0,0 +1,50 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + type_traits + HDRS + "type_traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + PUBLIC +) + +absl_cc_test( + NAME + type_traits_test + SRCS + "type_traits_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::type_traits + gmock_main +) + +# component target +absl_cc_library( + NAME + meta + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::type_traits + PUBLIC +) diff --git a/third_party/abseil_cpp/absl/meta/type_traits.h b/third_party/abseil_cpp/absl/meta/type_traits.h new file mode 100644 index 000000000000..d5cb5f3be39d --- /dev/null +++ b/third_party/abseil_cpp/absl/meta/type_traits.h @@ -0,0 +1,767 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// type_traits.h +// ----------------------------------------------------------------------------- +// +// This file contains C++11-compatible versions of standard <type_traits> API +// functions for determining the characteristics of types. Such traits can +// support type inference, classification, and transformation, as well as +// make it easier to write templates based on generic type behavior. +// +// See https://en.cppreference.com/w/cpp/header/type_traits +// +// WARNING: use of many of the constructs in this header will count as "complex +// template metaprogramming", so before proceeding, please carefully consider +// https://google.github.io/styleguide/cppguide.html#Template_metaprogramming +// +// WARNING: using template metaprogramming to detect or depend on API +// features is brittle and not guaranteed. Neither the standard library nor +// Abseil provides any guarantee that APIs are stable in the face of template +// metaprogramming. Use with caution. +#ifndef ABSL_META_TYPE_TRAITS_H_ +#define ABSL_META_TYPE_TRAITS_H_ + +#include <stddef.h> +#include <functional> +#include <type_traits> + +#include "absl/base/config.h" + +// MSVC constructibility traits do not detect destructor properties and so our +// implementations should not use them as a source-of-truth. +#if defined(_MSC_VER) && !defined(__clang__) && !defined(__GNUC__) +#define ABSL_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Defined and documented later on in this file. +template <typename T> +struct is_trivially_destructible; + +// Defined and documented later on in this file. +template <typename T> +struct is_trivially_move_assignable; + +namespace type_traits_internal { + +// Silence MSVC warnings about the destructor being defined as deleted. +#if defined(_MSC_VER) && !defined(__GNUC__) +#pragma warning(push) +#pragma warning(disable : 4624) +#endif // defined(_MSC_VER) && !defined(__GNUC__) + +template <class T> +union SingleMemberUnion { + T t; +}; + +// Restore the state of the destructor warning that was silenced above. +#if defined(_MSC_VER) && !defined(__GNUC__) +#pragma warning(pop) +#endif // defined(_MSC_VER) && !defined(__GNUC__) + +template <class T> +struct IsTriviallyMoveConstructibleObject + : std::integral_constant< + bool, std::is_move_constructible< + type_traits_internal::SingleMemberUnion<T>>::value && + absl::is_trivially_destructible<T>::value> {}; + +template <class T> +struct IsTriviallyCopyConstructibleObject + : std::integral_constant< + bool, std::is_copy_constructible< + type_traits_internal::SingleMemberUnion<T>>::value && + absl::is_trivially_destructible<T>::value> {}; + +template <class T> +struct IsTriviallyMoveAssignableReference : std::false_type {}; + +template <class T> +struct IsTriviallyMoveAssignableReference<T&> + : absl::is_trivially_move_assignable<T>::type {}; + +template <class T> +struct IsTriviallyMoveAssignableReference<T&&> + : absl::is_trivially_move_assignable<T>::type {}; + +template <typename... Ts> +struct VoidTImpl { + using type = void; +}; + +// This trick to retrieve a default alignment is necessary for our +// implementation of aligned_storage_t to be consistent with any implementation +// of std::aligned_storage. +template <size_t Len, typename T = std::aligned_storage<Len>> +struct default_alignment_of_aligned_storage; + +template <size_t Len, size_t Align> +struct default_alignment_of_aligned_storage<Len, + std::aligned_storage<Len, Align>> { + static constexpr size_t value = Align; +}; + +//////////////////////////////// +// Library Fundamentals V2 TS // +//////////////////////////////// + +// NOTE: The `is_detected` family of templates here differ from the library +// fundamentals specification in that for library fundamentals, `Op<Args...>` is +// evaluated as soon as the type `is_detected<Op, Args...>` undergoes +// substitution, regardless of whether or not the `::value` is accessed. That +// is inconsistent with all other standard traits and prevents lazy evaluation +// in larger contexts (such as if the `is_detected` check is a trailing argument +// of a `conjunction`. This implementation opts to instead be lazy in the same +// way that the standard traits are (this "defect" of the detection idiom +// specifications has been reported). + +template <class Enabler, template <class...> class Op, class... Args> +struct is_detected_impl { + using type = std::false_type; +}; + +template <template <class...> class Op, class... Args> +struct is_detected_impl<typename VoidTImpl<Op<Args...>>::type, Op, Args...> { + using type = std::true_type; +}; + +template <template <class...> class Op, class... Args> +struct is_detected : is_detected_impl<void, Op, Args...>::type {}; + +template <class Enabler, class To, template <class...> class Op, class... Args> +struct is_detected_convertible_impl { + using type = std::false_type; +}; + +template <class To, template <class...> class Op, class... Args> +struct is_detected_convertible_impl< + typename std::enable_if<std::is_convertible<Op<Args...>, To>::value>::type, + To, Op, Args...> { + using type = std::true_type; +}; + +template <class To, template <class...> class Op, class... Args> +struct is_detected_convertible + : is_detected_convertible_impl<void, To, Op, Args...>::type {}; + +template <typename T> +using IsCopyAssignableImpl = + decltype(std::declval<T&>() = std::declval<const T&>()); + +template <typename T> +using IsMoveAssignableImpl = decltype(std::declval<T&>() = std::declval<T&&>()); + +} // namespace type_traits_internal + +// MSVC 19.20 has a regression that causes our workarounds to fail, but their +// std forms now appear to be compliant. +#if defined(_MSC_VER) && !defined(__clang__) && (_MSC_VER >= 1920) + +template <typename T> +using is_copy_assignable = std::is_copy_assignable<T>; + +template <typename T> +using is_move_assignable = std::is_move_assignable<T>; + +#else + +template <typename T> +struct is_copy_assignable : type_traits_internal::is_detected< + type_traits_internal::IsCopyAssignableImpl, T> { +}; + +template <typename T> +struct is_move_assignable : type_traits_internal::is_detected< + type_traits_internal::IsMoveAssignableImpl, T> { +}; + +#endif + +// void_t() +// +// Ignores the type of any its arguments and returns `void`. In general, this +// metafunction allows you to create a general case that maps to `void` while +// allowing specializations that map to specific types. +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::void_t` metafunction. +// +// NOTE: `absl::void_t` does not use the standard-specified implementation so +// that it can remain compatible with gcc < 5.1. This can introduce slightly +// different behavior, such as when ordering partial specializations. +template <typename... Ts> +using void_t = typename type_traits_internal::VoidTImpl<Ts...>::type; + +// conjunction +// +// Performs a compile-time logical AND operation on the passed types (which +// must have `::value` members convertible to `bool`. Short-circuits if it +// encounters any `false` members (and does not compare the `::value` members +// of any remaining arguments). +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::conjunction` metafunction. +template <typename... Ts> +struct conjunction : std::true_type {}; + +template <typename T, typename... Ts> +struct conjunction<T, Ts...> + : std::conditional<T::value, conjunction<Ts...>, T>::type {}; + +template <typename T> +struct conjunction<T> : T {}; + +// disjunction +// +// Performs a compile-time logical OR operation on the passed types (which +// must have `::value` members convertible to `bool`. Short-circuits if it +// encounters any `true` members (and does not compare the `::value` members +// of any remaining arguments). +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::disjunction` metafunction. +template <typename... Ts> +struct disjunction : std::false_type {}; + +template <typename T, typename... Ts> +struct disjunction<T, Ts...> : + std::conditional<T::value, T, disjunction<Ts...>>::type {}; + +template <typename T> +struct disjunction<T> : T {}; + +// negation +// +// Performs a compile-time logical NOT operation on the passed type (which +// must have `::value` members convertible to `bool`. +// +// This metafunction is designed to be a drop-in replacement for the C++17 +// `std::negation` metafunction. +template <typename T> +struct negation : std::integral_constant<bool, !T::value> {}; + +// is_function() +// +// Determines whether the passed type `T` is a function type. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_function()` metafunction for platforms that have incomplete C++11 +// support (such as libstdc++ 4.x). +// +// This metafunction works because appending `const` to a type does nothing to +// function types and reference types (and forms a const-qualified type +// otherwise). +template <typename T> +struct is_function + : std::integral_constant< + bool, !(std::is_reference<T>::value || + std::is_const<typename std::add_const<T>::type>::value)> {}; + +// is_trivially_destructible() +// +// Determines whether the passed type `T` is trivially destructible. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_destructible()` metafunction for platforms that have +// incomplete C++11 support (such as libstdc++ 4.x). On any platforms that do +// fully support C++11, we check whether this yields the same result as the std +// implementation. +// +// NOTE: the extensions (__has_trivial_xxx) are implemented in gcc (version >= +// 4.3) and clang. Since we are supporting libstdc++ > 4.7, they should always +// be present. These extensions are documented at +// https://gcc.gnu.org/onlinedocs/gcc/Type-Traits.html#Type-Traits. +template <typename T> +struct is_trivially_destructible + : std::integral_constant<bool, __has_trivial_destructor(T) && + std::is_destructible<T>::value> { +#ifdef ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE + private: + static constexpr bool compliant = std::is_trivially_destructible<T>::value == + is_trivially_destructible::value; + static_assert(compliant || std::is_trivially_destructible<T>::value, + "Not compliant with std::is_trivially_destructible; " + "Standard: false, Implementation: true"); + static_assert(compliant || !std::is_trivially_destructible<T>::value, + "Not compliant with std::is_trivially_destructible; " + "Standard: true, Implementation: false"); +#endif // ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE +}; + +// is_trivially_default_constructible() +// +// Determines whether the passed type `T` is trivially default constructible. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_default_constructible()` metafunction for platforms that +// have incomplete C++11 support (such as libstdc++ 4.x). On any platforms that +// do fully support C++11, we check whether this yields the same result as the +// std implementation. +// +// NOTE: according to the C++ standard, Section: 20.15.4.3 [meta.unary.prop] +// "The predicate condition for a template specialization is_constructible<T, +// Args...> shall be satisfied if and only if the following variable +// definition would be well-formed for some invented variable t: +// +// T t(declval<Args>()...); +// +// is_trivially_constructible<T, Args...> additionally requires that the +// variable definition does not call any operation that is not trivial. +// For the purposes of this check, the call to std::declval is considered +// trivial." +// +// Notes from https://en.cppreference.com/w/cpp/types/is_constructible: +// In many implementations, is_nothrow_constructible also checks if the +// destructor throws because it is effectively noexcept(T(arg)). Same +// applies to is_trivially_constructible, which, in these implementations, also +// requires that the destructor is trivial. +// GCC bug 51452: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=51452 +// LWG issue 2116: http://cplusplus.github.io/LWG/lwg-active.html#2116. +// +// "T obj();" need to be well-formed and not call any nontrivial operation. +// Nontrivially destructible types will cause the expression to be nontrivial. +template <typename T> +struct is_trivially_default_constructible + : std::integral_constant<bool, __has_trivial_constructor(T) && + std::is_default_constructible<T>::value && + is_trivially_destructible<T>::value> { +#if defined(ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) && \ + !defined( \ + ABSL_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION) + private: + static constexpr bool compliant = + std::is_trivially_default_constructible<T>::value == + is_trivially_default_constructible::value; + static_assert(compliant || std::is_trivially_default_constructible<T>::value, + "Not compliant with std::is_trivially_default_constructible; " + "Standard: false, Implementation: true"); + static_assert(compliant || !std::is_trivially_default_constructible<T>::value, + "Not compliant with std::is_trivially_default_constructible; " + "Standard: true, Implementation: false"); +#endif // ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE +}; + +// is_trivially_move_constructible() +// +// Determines whether the passed type `T` is trivially move constructible. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_move_constructible()` metafunction for platforms that have +// incomplete C++11 support (such as libstdc++ 4.x). On any platforms that do +// fully support C++11, we check whether this yields the same result as the std +// implementation. +// +// NOTE: `T obj(declval<T>());` needs to be well-formed and not call any +// nontrivial operation. Nontrivially destructible types will cause the +// expression to be nontrivial. +template <typename T> +struct is_trivially_move_constructible + : std::conditional< + std::is_object<T>::value && !std::is_array<T>::value, + type_traits_internal::IsTriviallyMoveConstructibleObject<T>, + std::is_reference<T>>::type::type { +#if defined(ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) && \ + !defined( \ + ABSL_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION) + private: + static constexpr bool compliant = + std::is_trivially_move_constructible<T>::value == + is_trivially_move_constructible::value; + static_assert(compliant || std::is_trivially_move_constructible<T>::value, + "Not compliant with std::is_trivially_move_constructible; " + "Standard: false, Implementation: true"); + static_assert(compliant || !std::is_trivially_move_constructible<T>::value, + "Not compliant with std::is_trivially_move_constructible; " + "Standard: true, Implementation: false"); +#endif // ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE +}; + +// is_trivially_copy_constructible() +// +// Determines whether the passed type `T` is trivially copy constructible. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_copy_constructible()` metafunction for platforms that have +// incomplete C++11 support (such as libstdc++ 4.x). On any platforms that do +// fully support C++11, we check whether this yields the same result as the std +// implementation. +// +// NOTE: `T obj(declval<const T&>());` needs to be well-formed and not call any +// nontrivial operation. Nontrivially destructible types will cause the +// expression to be nontrivial. +template <typename T> +struct is_trivially_copy_constructible + : std::conditional< + std::is_object<T>::value && !std::is_array<T>::value, + type_traits_internal::IsTriviallyCopyConstructibleObject<T>, + std::is_lvalue_reference<T>>::type::type { +#if defined(ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) && \ + !defined( \ + ABSL_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION) + private: + static constexpr bool compliant = + std::is_trivially_copy_constructible<T>::value == + is_trivially_copy_constructible::value; + static_assert(compliant || std::is_trivially_copy_constructible<T>::value, + "Not compliant with std::is_trivially_copy_constructible; " + "Standard: false, Implementation: true"); + static_assert(compliant || !std::is_trivially_copy_constructible<T>::value, + "Not compliant with std::is_trivially_copy_constructible; " + "Standard: true, Implementation: false"); +#endif // ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE +}; + +// is_trivially_move_assignable() +// +// Determines whether the passed type `T` is trivially move assignable. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_move_assignable()` metafunction for platforms that have +// incomplete C++11 support (such as libstdc++ 4.x). On any platforms that do +// fully support C++11, we check whether this yields the same result as the std +// implementation. +// +// NOTE: `is_assignable<T, U>::value` is `true` if the expression +// `declval<T>() = declval<U>()` is well-formed when treated as an unevaluated +// operand. `is_trivially_assignable<T, U>` requires the assignment to call no +// operation that is not trivial. `is_trivially_copy_assignable<T>` is simply +// `is_trivially_assignable<T&, T>`. +template <typename T> +struct is_trivially_move_assignable + : std::conditional< + std::is_object<T>::value && !std::is_array<T>::value && + std::is_move_assignable<T>::value, + std::is_move_assignable<type_traits_internal::SingleMemberUnion<T>>, + type_traits_internal::IsTriviallyMoveAssignableReference<T>>::type:: + type { +#ifdef ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE + private: + static constexpr bool compliant = + std::is_trivially_move_assignable<T>::value == + is_trivially_move_assignable::value; + static_assert(compliant || std::is_trivially_move_assignable<T>::value, + "Not compliant with std::is_trivially_move_assignable; " + "Standard: false, Implementation: true"); + static_assert(compliant || !std::is_trivially_move_assignable<T>::value, + "Not compliant with std::is_trivially_move_assignable; " + "Standard: true, Implementation: false"); +#endif // ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE +}; + +// is_trivially_copy_assignable() +// +// Determines whether the passed type `T` is trivially copy assignable. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_copy_assignable()` metafunction for platforms that have +// incomplete C++11 support (such as libstdc++ 4.x). On any platforms that do +// fully support C++11, we check whether this yields the same result as the std +// implementation. +// +// NOTE: `is_assignable<T, U>::value` is `true` if the expression +// `declval<T>() = declval<U>()` is well-formed when treated as an unevaluated +// operand. `is_trivially_assignable<T, U>` requires the assignment to call no +// operation that is not trivial. `is_trivially_copy_assignable<T>` is simply +// `is_trivially_assignable<T&, const T&>`. +template <typename T> +struct is_trivially_copy_assignable + : std::integral_constant< + bool, __has_trivial_assign(typename std::remove_reference<T>::type) && + absl::is_copy_assignable<T>::value> { +#ifdef ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE + private: + static constexpr bool compliant = + std::is_trivially_copy_assignable<T>::value == + is_trivially_copy_assignable::value; + static_assert(compliant || std::is_trivially_copy_assignable<T>::value, + "Not compliant with std::is_trivially_copy_assignable; " + "Standard: false, Implementation: true"); + static_assert(compliant || !std::is_trivially_copy_assignable<T>::value, + "Not compliant with std::is_trivially_copy_assignable; " + "Standard: true, Implementation: false"); +#endif // ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE +}; + +namespace type_traits_internal { +// is_trivially_copyable() +// +// Determines whether the passed type `T` is trivially copyable. +// +// This metafunction is designed to be a drop-in replacement for the C++11 +// `std::is_trivially_copyable()` metafunction for platforms that have +// incomplete C++11 support (such as libstdc++ 4.x). We use the C++17 definition +// of TriviallyCopyable. +// +// NOTE: `is_trivially_copyable<T>::value` is `true` if all of T's copy/move +// constructors/assignment operators are trivial or deleted, T has at least +// one non-deleted copy/move constructor/assignment operator, and T is trivially +// destructible. Arrays of trivially copyable types are trivially copyable. +// +// We expose this metafunction only for internal use within absl. +template <typename T> +class is_trivially_copyable_impl { + using ExtentsRemoved = typename std::remove_all_extents<T>::type; + static constexpr bool kIsCopyOrMoveConstructible = + std::is_copy_constructible<ExtentsRemoved>::value || + std::is_move_constructible<ExtentsRemoved>::value; + static constexpr bool kIsCopyOrMoveAssignable = + absl::is_copy_assignable<ExtentsRemoved>::value || + absl::is_move_assignable<ExtentsRemoved>::value; + + public: + static constexpr bool kValue = + (__has_trivial_copy(ExtentsRemoved) || !kIsCopyOrMoveConstructible) && + (__has_trivial_assign(ExtentsRemoved) || !kIsCopyOrMoveAssignable) && + (kIsCopyOrMoveConstructible || kIsCopyOrMoveAssignable) && + is_trivially_destructible<ExtentsRemoved>::value && + // We need to check for this explicitly because otherwise we'll say + // references are trivial copyable when compiled by MSVC. + !std::is_reference<ExtentsRemoved>::value; +}; + +template <typename T> +struct is_trivially_copyable + : std::integral_constant< + bool, type_traits_internal::is_trivially_copyable_impl<T>::kValue> {}; +} // namespace type_traits_internal + +// ----------------------------------------------------------------------------- +// C++14 "_t" trait aliases +// ----------------------------------------------------------------------------- + +template <typename T> +using remove_cv_t = typename std::remove_cv<T>::type; + +template <typename T> +using remove_const_t = typename std::remove_const<T>::type; + +template <typename T> +using remove_volatile_t = typename std::remove_volatile<T>::type; + +template <typename T> +using add_cv_t = typename std::add_cv<T>::type; + +template <typename T> +using add_const_t = typename std::add_const<T>::type; + +template <typename T> +using add_volatile_t = typename std::add_volatile<T>::type; + +template <typename T> +using remove_reference_t = typename std::remove_reference<T>::type; + +template <typename T> +using add_lvalue_reference_t = typename std::add_lvalue_reference<T>::type; + +template <typename T> +using add_rvalue_reference_t = typename std::add_rvalue_reference<T>::type; + +template <typename T> +using remove_pointer_t = typename std::remove_pointer<T>::type; + +template <typename T> +using add_pointer_t = typename std::add_pointer<T>::type; + +template <typename T> +using make_signed_t = typename std::make_signed<T>::type; + +template <typename T> +using make_unsigned_t = typename std::make_unsigned<T>::type; + +template <typename T> +using remove_extent_t = typename std::remove_extent<T>::type; + +template <typename T> +using remove_all_extents_t = typename std::remove_all_extents<T>::type; + +template <size_t Len, size_t Align = type_traits_internal:: + default_alignment_of_aligned_storage<Len>::value> +using aligned_storage_t = typename std::aligned_storage<Len, Align>::type; + +template <typename T> +using decay_t = typename std::decay<T>::type; + +template <bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; + +template <bool B, typename T, typename F> +using conditional_t = typename std::conditional<B, T, F>::type; + +template <typename... T> +using common_type_t = typename std::common_type<T...>::type; + +template <typename T> +using underlying_type_t = typename std::underlying_type<T>::type; + + +namespace type_traits_internal { + +#if __cplusplus >= 201703L +// std::result_of is deprecated (C++17) or removed (C++20) +template<typename> struct result_of; +template<typename F, typename... Args> +struct result_of<F(Args...)> : std::invoke_result<F, Args...> {}; +#else +template<typename F> using result_of = std::result_of<F>; +#endif + +} // namespace type_traits_internal + +template<typename F> +using result_of_t = typename type_traits_internal::result_of<F>::type; + +namespace type_traits_internal { +// In MSVC we can't probe std::hash or stdext::hash because it triggers a +// static_assert instead of failing substitution. Libc++ prior to 4.0 +// also used a static_assert. +// +#if defined(_MSC_VER) || (defined(_LIBCPP_VERSION) && \ + _LIBCPP_VERSION < 4000 && _LIBCPP_STD_VER > 11) +#define ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 0 +#else +#define ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ 1 +#endif + +#if !ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ +template <typename Key, typename = size_t> +struct IsHashable : std::true_type {}; +#else // ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ +template <typename Key, typename = void> +struct IsHashable : std::false_type {}; + +template <typename Key> +struct IsHashable< + Key, + absl::enable_if_t<std::is_convertible< + decltype(std::declval<std::hash<Key>&>()(std::declval<Key const&>())), + std::size_t>::value>> : std::true_type {}; +#endif // !ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + +struct AssertHashEnabledHelper { + private: + static void Sink(...) {} + struct NAT {}; + + template <class Key> + static auto GetReturnType(int) + -> decltype(std::declval<std::hash<Key>>()(std::declval<Key const&>())); + template <class Key> + static NAT GetReturnType(...); + + template <class Key> + static std::nullptr_t DoIt() { + static_assert(IsHashable<Key>::value, + "std::hash<Key> does not provide a call operator"); + static_assert( + std::is_default_constructible<std::hash<Key>>::value, + "std::hash<Key> must be default constructible when it is enabled"); + static_assert( + std::is_copy_constructible<std::hash<Key>>::value, + "std::hash<Key> must be copy constructible when it is enabled"); + static_assert(absl::is_copy_assignable<std::hash<Key>>::value, + "std::hash<Key> must be copy assignable when it is enabled"); + // is_destructible is unchecked as it's implied by each of the + // is_constructible checks. + using ReturnType = decltype(GetReturnType<Key>(0)); + static_assert(std::is_same<ReturnType, NAT>::value || + std::is_same<ReturnType, size_t>::value, + "std::hash<Key> must return size_t"); + return nullptr; + } + + template <class... Ts> + friend void AssertHashEnabled(); +}; + +template <class... Ts> +inline void AssertHashEnabled() { + using Helper = AssertHashEnabledHelper; + Helper::Sink(Helper::DoIt<Ts>()...); +} + +} // namespace type_traits_internal + +// An internal namespace that is required to implement the C++17 swap traits. +// It is not further nested in type_traits_internal to avoid long symbol names. +namespace swap_internal { + +// Necessary for the traits. +using std::swap; + +// This declaration prevents global `swap` and `absl::swap` overloads from being +// considered unless ADL picks them up. +void swap(); + +template <class T> +using IsSwappableImpl = decltype(swap(std::declval<T&>(), std::declval<T&>())); + +// NOTE: This dance with the default template parameter is for MSVC. +template <class T, + class IsNoexcept = std::integral_constant< + bool, noexcept(swap(std::declval<T&>(), std::declval<T&>()))>> +using IsNothrowSwappableImpl = typename std::enable_if<IsNoexcept::value>::type; + +// IsSwappable +// +// Determines whether the standard swap idiom is a valid expression for +// arguments of type `T`. +template <class T> +struct IsSwappable + : absl::type_traits_internal::is_detected<IsSwappableImpl, T> {}; + +// IsNothrowSwappable +// +// Determines whether the standard swap idiom is a valid expression for +// arguments of type `T` and is noexcept. +template <class T> +struct IsNothrowSwappable + : absl::type_traits_internal::is_detected<IsNothrowSwappableImpl, T> {}; + +// Swap() +// +// Performs the swap idiom from a namespace where valid candidates may only be +// found in `std` or via ADL. +template <class T, absl::enable_if_t<IsSwappable<T>::value, int> = 0> +void Swap(T& lhs, T& rhs) noexcept(IsNothrowSwappable<T>::value) { + swap(lhs, rhs); +} + +// StdSwapIsUnconstrained +// +// Some standard library implementations are broken in that they do not +// constrain `std::swap`. This will effectively tell us if we are dealing with +// one of those implementations. +using StdSwapIsUnconstrained = IsSwappable<void()>; + +} // namespace swap_internal + +namespace type_traits_internal { + +// Make the swap-related traits/function accessible from this namespace. +using swap_internal::IsNothrowSwappable; +using swap_internal::IsSwappable; +using swap_internal::Swap; +using swap_internal::StdSwapIsUnconstrained; + +} // namespace type_traits_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_META_TYPE_TRAITS_H_ diff --git a/third_party/abseil_cpp/absl/meta/type_traits_test.cc b/third_party/abseil_cpp/absl/meta/type_traits_test.cc new file mode 100644 index 000000000000..1aafd0d49a83 --- /dev/null +++ b/third_party/abseil_cpp/absl/meta/type_traits_test.cc @@ -0,0 +1,1368 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/meta/type_traits.h" + +#include <cstdint> +#include <string> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gtest/gtest.h" + +namespace { + +using ::testing::StaticAssertTypeEq; + +template <class T, class U> +struct simple_pair { + T first; + U second; +}; + +struct Dummy {}; + +struct ReturnType {}; +struct ConvertibleToReturnType { + operator ReturnType() const; // NOLINT +}; + +// Unique types used as parameter types for testing the detection idiom. +struct StructA {}; +struct StructB {}; +struct StructC {}; + +struct TypeWithBarFunction { + template <class T, + absl::enable_if_t<std::is_same<T&&, StructA&>::value, int> = 0> + ReturnType bar(T&&, const StructB&, StructC&&) &&; // NOLINT +}; + +struct TypeWithBarFunctionAndConvertibleReturnType { + template <class T, + absl::enable_if_t<std::is_same<T&&, StructA&>::value, int> = 0> + ConvertibleToReturnType bar(T&&, const StructB&, StructC&&) &&; // NOLINT +}; + +template <class Class, class... Ts> +using BarIsCallableImpl = + decltype(std::declval<Class>().bar(std::declval<Ts>()...)); + +template <class Class, class... T> +using BarIsCallable = + absl::type_traits_internal::is_detected<BarIsCallableImpl, Class, T...>; + +template <class Class, class... T> +using BarIsCallableConv = absl::type_traits_internal::is_detected_convertible< + ReturnType, BarIsCallableImpl, Class, T...>; + +// NOTE: Test of detail type_traits_internal::is_detected. +TEST(IsDetectedTest, BasicUsage) { + EXPECT_TRUE((BarIsCallable<TypeWithBarFunction, StructA&, const StructB&, + StructC>::value)); + EXPECT_TRUE( + (BarIsCallable<TypeWithBarFunction, StructA&, StructB&, StructC>::value)); + EXPECT_TRUE( + (BarIsCallable<TypeWithBarFunction, StructA&, StructB, StructC>::value)); + + EXPECT_FALSE((BarIsCallable<int, StructA&, const StructB&, StructC>::value)); + EXPECT_FALSE((BarIsCallable<TypeWithBarFunction&, StructA&, const StructB&, + StructC>::value)); + EXPECT_FALSE((BarIsCallable<TypeWithBarFunction, StructA, const StructB&, + StructC>::value)); +} + +// NOTE: Test of detail type_traits_internal::is_detected_convertible. +TEST(IsDetectedConvertibleTest, BasicUsage) { + EXPECT_TRUE((BarIsCallableConv<TypeWithBarFunction, StructA&, const StructB&, + StructC>::value)); + EXPECT_TRUE((BarIsCallableConv<TypeWithBarFunction, StructA&, StructB&, + StructC>::value)); + EXPECT_TRUE((BarIsCallableConv<TypeWithBarFunction, StructA&, StructB, + StructC>::value)); + EXPECT_TRUE((BarIsCallableConv<TypeWithBarFunctionAndConvertibleReturnType, + StructA&, const StructB&, StructC>::value)); + EXPECT_TRUE((BarIsCallableConv<TypeWithBarFunctionAndConvertibleReturnType, + StructA&, StructB&, StructC>::value)); + EXPECT_TRUE((BarIsCallableConv<TypeWithBarFunctionAndConvertibleReturnType, + StructA&, StructB, StructC>::value)); + + EXPECT_FALSE( + (BarIsCallableConv<int, StructA&, const StructB&, StructC>::value)); + EXPECT_FALSE((BarIsCallableConv<TypeWithBarFunction&, StructA&, + const StructB&, StructC>::value)); + EXPECT_FALSE((BarIsCallableConv<TypeWithBarFunction, StructA, const StructB&, + StructC>::value)); + EXPECT_FALSE((BarIsCallableConv<TypeWithBarFunctionAndConvertibleReturnType&, + StructA&, const StructB&, StructC>::value)); + EXPECT_FALSE((BarIsCallableConv<TypeWithBarFunctionAndConvertibleReturnType, + StructA, const StructB&, StructC>::value)); +} + +TEST(VoidTTest, BasicUsage) { + StaticAssertTypeEq<void, absl::void_t<Dummy>>(); + StaticAssertTypeEq<void, absl::void_t<Dummy, Dummy, Dummy>>(); +} + +TEST(ConjunctionTest, BasicBooleanLogic) { + EXPECT_TRUE(absl::conjunction<>::value); + EXPECT_TRUE(absl::conjunction<std::true_type>::value); + EXPECT_TRUE((absl::conjunction<std::true_type, std::true_type>::value)); + EXPECT_FALSE((absl::conjunction<std::true_type, std::false_type>::value)); + EXPECT_FALSE((absl::conjunction<std::false_type, std::true_type>::value)); + EXPECT_FALSE((absl::conjunction<std::false_type, std::false_type>::value)); +} + +struct MyTrueType { + static constexpr bool value = true; +}; + +struct MyFalseType { + static constexpr bool value = false; +}; + +TEST(ConjunctionTest, ShortCircuiting) { + EXPECT_FALSE( + (absl::conjunction<std::true_type, std::false_type, Dummy>::value)); + EXPECT_TRUE((std::is_base_of<MyFalseType, + absl::conjunction<std::true_type, MyFalseType, + std::false_type>>::value)); + EXPECT_TRUE( + (std::is_base_of<MyTrueType, + absl::conjunction<std::true_type, MyTrueType>>::value)); +} + +TEST(DisjunctionTest, BasicBooleanLogic) { + EXPECT_FALSE(absl::disjunction<>::value); + EXPECT_FALSE(absl::disjunction<std::false_type>::value); + EXPECT_TRUE((absl::disjunction<std::true_type, std::true_type>::value)); + EXPECT_TRUE((absl::disjunction<std::true_type, std::false_type>::value)); + EXPECT_TRUE((absl::disjunction<std::false_type, std::true_type>::value)); + EXPECT_FALSE((absl::disjunction<std::false_type, std::false_type>::value)); +} + +TEST(DisjunctionTest, ShortCircuiting) { + EXPECT_TRUE( + (absl::disjunction<std::false_type, std::true_type, Dummy>::value)); + EXPECT_TRUE(( + std::is_base_of<MyTrueType, absl::disjunction<std::false_type, MyTrueType, + std::true_type>>::value)); + EXPECT_TRUE(( + std::is_base_of<MyFalseType, + absl::disjunction<std::false_type, MyFalseType>>::value)); +} + +TEST(NegationTest, BasicBooleanLogic) { + EXPECT_FALSE(absl::negation<std::true_type>::value); + EXPECT_FALSE(absl::negation<MyTrueType>::value); + EXPECT_TRUE(absl::negation<std::false_type>::value); + EXPECT_TRUE(absl::negation<MyFalseType>::value); +} + +// all member functions are trivial +class Trivial { + int n_; +}; + +struct TrivialDestructor { + ~TrivialDestructor() = default; +}; + +struct NontrivialDestructor { + ~NontrivialDestructor() {} +}; + +struct DeletedDestructor { + ~DeletedDestructor() = delete; +}; + +class TrivialDefaultCtor { + public: + TrivialDefaultCtor() = default; + explicit TrivialDefaultCtor(int n) : n_(n) {} + + private: + int n_; +}; + +class NontrivialDefaultCtor { + public: + NontrivialDefaultCtor() : n_(1) {} + + private: + int n_; +}; + +class DeletedDefaultCtor { + public: + DeletedDefaultCtor() = delete; + explicit DeletedDefaultCtor(int n) : n_(n) {} + + private: + int n_; +}; + +class TrivialMoveCtor { + public: + explicit TrivialMoveCtor(int n) : n_(n) {} + TrivialMoveCtor(TrivialMoveCtor&&) = default; + TrivialMoveCtor& operator=(const TrivialMoveCtor& t) { + n_ = t.n_; + return *this; + } + + private: + int n_; +}; + +class NontrivialMoveCtor { + public: + explicit NontrivialMoveCtor(int n) : n_(n) {} + NontrivialMoveCtor(NontrivialMoveCtor&& t) noexcept : n_(t.n_) {} + NontrivialMoveCtor& operator=(const NontrivialMoveCtor&) = default; + + private: + int n_; +}; + +class TrivialCopyCtor { + public: + explicit TrivialCopyCtor(int n) : n_(n) {} + TrivialCopyCtor(const TrivialCopyCtor&) = default; + TrivialCopyCtor& operator=(const TrivialCopyCtor& t) { + n_ = t.n_; + return *this; + } + + private: + int n_; +}; + +class NontrivialCopyCtor { + public: + explicit NontrivialCopyCtor(int n) : n_(n) {} + NontrivialCopyCtor(const NontrivialCopyCtor& t) : n_(t.n_) {} + NontrivialCopyCtor& operator=(const NontrivialCopyCtor&) = default; + + private: + int n_; +}; + +class DeletedCopyCtor { + public: + explicit DeletedCopyCtor(int n) : n_(n) {} + DeletedCopyCtor(const DeletedCopyCtor&) = delete; + DeletedCopyCtor& operator=(const DeletedCopyCtor&) = default; + + private: + int n_; +}; + +class TrivialMoveAssign { + public: + explicit TrivialMoveAssign(int n) : n_(n) {} + TrivialMoveAssign(const TrivialMoveAssign& t) : n_(t.n_) {} + TrivialMoveAssign& operator=(TrivialMoveAssign&&) = default; + ~TrivialMoveAssign() {} // can have nontrivial destructor + private: + int n_; +}; + +class NontrivialMoveAssign { + public: + explicit NontrivialMoveAssign(int n) : n_(n) {} + NontrivialMoveAssign(const NontrivialMoveAssign&) = default; + NontrivialMoveAssign& operator=(NontrivialMoveAssign&& t) noexcept { + n_ = t.n_; + return *this; + } + + private: + int n_; +}; + +class TrivialCopyAssign { + public: + explicit TrivialCopyAssign(int n) : n_(n) {} + TrivialCopyAssign(const TrivialCopyAssign& t) : n_(t.n_) {} + TrivialCopyAssign& operator=(const TrivialCopyAssign& t) = default; + ~TrivialCopyAssign() {} // can have nontrivial destructor + private: + int n_; +}; + +class NontrivialCopyAssign { + public: + explicit NontrivialCopyAssign(int n) : n_(n) {} + NontrivialCopyAssign(const NontrivialCopyAssign&) = default; + NontrivialCopyAssign& operator=(const NontrivialCopyAssign& t) { + n_ = t.n_; + return *this; + } + + private: + int n_; +}; + +class DeletedCopyAssign { + public: + explicit DeletedCopyAssign(int n) : n_(n) {} + DeletedCopyAssign(const DeletedCopyAssign&) = default; + DeletedCopyAssign& operator=(const DeletedCopyAssign&) = delete; + + private: + int n_; +}; + +struct MovableNonCopyable { + MovableNonCopyable() = default; + MovableNonCopyable(const MovableNonCopyable&) = delete; + MovableNonCopyable(MovableNonCopyable&&) = default; + MovableNonCopyable& operator=(const MovableNonCopyable&) = delete; + MovableNonCopyable& operator=(MovableNonCopyable&&) = default; +}; + +struct NonCopyableOrMovable { + NonCopyableOrMovable() = default; + NonCopyableOrMovable(const NonCopyableOrMovable&) = delete; + NonCopyableOrMovable(NonCopyableOrMovable&&) = delete; + NonCopyableOrMovable& operator=(const NonCopyableOrMovable&) = delete; + NonCopyableOrMovable& operator=(NonCopyableOrMovable&&) = delete; +}; + +class Base { + public: + virtual ~Base() {} +}; + +// Old versions of libc++, around Clang 3.5 to 3.6, consider deleted destructors +// as also being trivial. With the resolution of CWG 1928 and CWG 1734, this +// is no longer considered true and has thus been amended. +// Compiler Explorer: https://godbolt.org/g/zT59ZL +// CWG issue 1734: http://open-std.org/JTC1/SC22/WG21/docs/cwg_defects.html#1734 +// CWG issue 1928: http://open-std.org/JTC1/SC22/WG21/docs/cwg_closed.html#1928 +#if !defined(_LIBCPP_VERSION) || _LIBCPP_VERSION >= 3700 +#define ABSL_TRIVIALLY_DESTRUCTIBLE_CONSIDER_DELETED_DESTRUCTOR_NOT_TRIVIAL 1 +#endif + +// As of the moment, GCC versions >5.1 have a problem compiling for +// std::is_trivially_default_constructible<NontrivialDestructor[10]>, where +// NontrivialDestructor is a struct with a custom nontrivial destructor. Note +// that this problem only occurs for arrays of a known size, so something like +// std::is_trivially_default_constructible<NontrivialDestructor[]> does not +// have any problems. +// Compiler Explorer: https://godbolt.org/g/dXRbdK +// GCC bug 83689: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=83689 +#if defined(__clang__) || defined(_MSC_VER) || \ + (defined(__GNUC__) && __GNUC__ < 5) +#define ABSL_GCC_BUG_TRIVIALLY_CONSTRUCTIBLE_ON_ARRAY_OF_NONTRIVIAL 1 +#endif + +TEST(TypeTraitsTest, TestIsFunction) { + struct Callable { + void operator()() {} + }; + EXPECT_TRUE(absl::is_function<void()>::value); + EXPECT_TRUE(absl::is_function<void()&>::value); + EXPECT_TRUE(absl::is_function<void() const>::value); + EXPECT_TRUE(absl::is_function<void() noexcept>::value); + EXPECT_TRUE(absl::is_function<void(...) noexcept>::value); + + EXPECT_FALSE(absl::is_function<void(*)()>::value); + EXPECT_FALSE(absl::is_function<void(&)()>::value); + EXPECT_FALSE(absl::is_function<int>::value); + EXPECT_FALSE(absl::is_function<Callable>::value); +} + +TEST(TypeTraitsTest, TestTrivialDestructor) { + // Verify that arithmetic types and pointers have trivial destructors. + EXPECT_TRUE(absl::is_trivially_destructible<bool>::value); + EXPECT_TRUE(absl::is_trivially_destructible<char>::value); + EXPECT_TRUE(absl::is_trivially_destructible<unsigned char>::value); + EXPECT_TRUE(absl::is_trivially_destructible<signed char>::value); + EXPECT_TRUE(absl::is_trivially_destructible<wchar_t>::value); + EXPECT_TRUE(absl::is_trivially_destructible<int>::value); + EXPECT_TRUE(absl::is_trivially_destructible<unsigned int>::value); + EXPECT_TRUE(absl::is_trivially_destructible<int16_t>::value); + EXPECT_TRUE(absl::is_trivially_destructible<uint16_t>::value); + EXPECT_TRUE(absl::is_trivially_destructible<int64_t>::value); + EXPECT_TRUE(absl::is_trivially_destructible<uint64_t>::value); + EXPECT_TRUE(absl::is_trivially_destructible<float>::value); + EXPECT_TRUE(absl::is_trivially_destructible<double>::value); + EXPECT_TRUE(absl::is_trivially_destructible<long double>::value); + EXPECT_TRUE(absl::is_trivially_destructible<std::string*>::value); + EXPECT_TRUE(absl::is_trivially_destructible<Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_destructible<const std::string*>::value); + EXPECT_TRUE(absl::is_trivially_destructible<const Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_destructible<std::string**>::value); + EXPECT_TRUE(absl::is_trivially_destructible<Trivial**>::value); + + // classes with destructors + EXPECT_TRUE(absl::is_trivially_destructible<Trivial>::value); + EXPECT_TRUE(absl::is_trivially_destructible<TrivialDestructor>::value); + + // Verify that types with a nontrivial or deleted destructor + // are marked as such. + EXPECT_FALSE(absl::is_trivially_destructible<NontrivialDestructor>::value); +#ifdef ABSL_TRIVIALLY_DESTRUCTIBLE_CONSIDER_DELETED_DESTRUCTOR_NOT_TRIVIAL + EXPECT_FALSE(absl::is_trivially_destructible<DeletedDestructor>::value); +#endif + + // simple_pair of such types is trivial + EXPECT_TRUE((absl::is_trivially_destructible<simple_pair<int, int>>::value)); + EXPECT_TRUE((absl::is_trivially_destructible< + simple_pair<Trivial, TrivialDestructor>>::value)); + + // Verify that types without trivial destructors are correctly marked as such. + EXPECT_FALSE(absl::is_trivially_destructible<std::string>::value); + EXPECT_FALSE(absl::is_trivially_destructible<std::vector<int>>::value); + + // Verify that simple_pairs of types without trivial destructors + // are not marked as trivial. + EXPECT_FALSE((absl::is_trivially_destructible< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::is_trivially_destructible< + simple_pair<std::string, int>>::value)); + + // array of such types is trivial + using int10 = int[10]; + EXPECT_TRUE(absl::is_trivially_destructible<int10>::value); + using Trivial10 = Trivial[10]; + EXPECT_TRUE(absl::is_trivially_destructible<Trivial10>::value); + using TrivialDestructor10 = TrivialDestructor[10]; + EXPECT_TRUE(absl::is_trivially_destructible<TrivialDestructor10>::value); + + // Conversely, the opposite also holds. + using NontrivialDestructor10 = NontrivialDestructor[10]; + EXPECT_FALSE(absl::is_trivially_destructible<NontrivialDestructor10>::value); +} + +TEST(TypeTraitsTest, TestTrivialDefaultCtor) { + // arithmetic types and pointers have trivial default constructors. + EXPECT_TRUE(absl::is_trivially_default_constructible<bool>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<char>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<unsigned char>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<signed char>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<wchar_t>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<int>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<unsigned int>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<int16_t>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<uint16_t>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<int64_t>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<uint64_t>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<float>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<double>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<long double>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<std::string*>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<Trivial*>::value); + EXPECT_TRUE( + absl::is_trivially_default_constructible<const std::string*>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<const Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<std::string**>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<Trivial**>::value); + + // types with compiler generated default ctors + EXPECT_TRUE(absl::is_trivially_default_constructible<Trivial>::value); + EXPECT_TRUE( + absl::is_trivially_default_constructible<TrivialDefaultCtor>::value); + + // Verify that types without them are not. + EXPECT_FALSE( + absl::is_trivially_default_constructible<NontrivialDefaultCtor>::value); + EXPECT_FALSE( + absl::is_trivially_default_constructible<DeletedDefaultCtor>::value); + + // types with nontrivial destructor are nontrivial + EXPECT_FALSE( + absl::is_trivially_default_constructible<NontrivialDestructor>::value); + + // types with vtables + EXPECT_FALSE(absl::is_trivially_default_constructible<Base>::value); + + // Verify that simple_pair has trivial constructors where applicable. + EXPECT_TRUE((absl::is_trivially_default_constructible< + simple_pair<int, char*>>::value)); + EXPECT_TRUE((absl::is_trivially_default_constructible< + simple_pair<int, Trivial>>::value)); + EXPECT_TRUE((absl::is_trivially_default_constructible< + simple_pair<int, TrivialDefaultCtor>>::value)); + + // Verify that types without trivial constructors are + // correctly marked as such. + EXPECT_FALSE(absl::is_trivially_default_constructible<std::string>::value); + EXPECT_FALSE( + absl::is_trivially_default_constructible<std::vector<int>>::value); + + // Verify that simple_pairs of types without trivial constructors + // are not marked as trivial. + EXPECT_FALSE((absl::is_trivially_default_constructible< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::is_trivially_default_constructible< + simple_pair<std::string, int>>::value)); + + // Verify that arrays of such types are trivially default constructible + using int10 = int[10]; + EXPECT_TRUE(absl::is_trivially_default_constructible<int10>::value); + using Trivial10 = Trivial[10]; + EXPECT_TRUE(absl::is_trivially_default_constructible<Trivial10>::value); + using TrivialDefaultCtor10 = TrivialDefaultCtor[10]; + EXPECT_TRUE( + absl::is_trivially_default_constructible<TrivialDefaultCtor10>::value); + + // Conversely, the opposite also holds. +#ifdef ABSL_GCC_BUG_TRIVIALLY_CONSTRUCTIBLE_ON_ARRAY_OF_NONTRIVIAL + using NontrivialDefaultCtor10 = NontrivialDefaultCtor[10]; + EXPECT_FALSE( + absl::is_trivially_default_constructible<NontrivialDefaultCtor10>::value); +#endif +} + +// GCC prior to 7.4 had a bug in its trivially-constructible traits +// (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80654). +// This test makes sure that we do not depend on the trait in these cases when +// implementing absl triviality traits. + +template <class T> +struct BadConstructors { + BadConstructors() { static_assert(T::value, ""); } + + BadConstructors(BadConstructors&&) { static_assert(T::value, ""); } + + BadConstructors(const BadConstructors&) { static_assert(T::value, ""); } +}; + +TEST(TypeTraitsTest, TestTrivialityBadConstructors) { + using BadType = BadConstructors<int>; + + EXPECT_FALSE(absl::is_trivially_default_constructible<BadType>::value); + EXPECT_FALSE(absl::is_trivially_move_constructible<BadType>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<BadType>::value); +} + +TEST(TypeTraitsTest, TestTrivialMoveCtor) { + // Verify that arithmetic types and pointers have trivial move + // constructors. + EXPECT_TRUE(absl::is_trivially_move_constructible<bool>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<char>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<unsigned char>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<signed char>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<wchar_t>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<int>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<unsigned int>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<int16_t>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<uint16_t>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<int64_t>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<uint64_t>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<float>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<double>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<long double>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<std::string*>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<const std::string*>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<const Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<std::string**>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<Trivial**>::value); + + // Reference types + EXPECT_TRUE(absl::is_trivially_move_constructible<int&>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<int&&>::value); + + // types with compiler generated move ctors + EXPECT_TRUE(absl::is_trivially_move_constructible<Trivial>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<TrivialMoveCtor>::value); + + // Verify that types without them (i.e. nontrivial or deleted) are not. + EXPECT_FALSE( + absl::is_trivially_move_constructible<NontrivialCopyCtor>::value); + EXPECT_FALSE(absl::is_trivially_move_constructible<DeletedCopyCtor>::value); + EXPECT_FALSE( + absl::is_trivially_move_constructible<NonCopyableOrMovable>::value); + + // type with nontrivial destructor are nontrivial move construbtible + EXPECT_FALSE( + absl::is_trivially_move_constructible<NontrivialDestructor>::value); + + // types with vtables + EXPECT_FALSE(absl::is_trivially_move_constructible<Base>::value); + + // Verify that simple_pair of such types is trivially move constructible + EXPECT_TRUE( + (absl::is_trivially_move_constructible<simple_pair<int, char*>>::value)); + EXPECT_TRUE(( + absl::is_trivially_move_constructible<simple_pair<int, Trivial>>::value)); + EXPECT_TRUE((absl::is_trivially_move_constructible< + simple_pair<int, TrivialMoveCtor>>::value)); + + // Verify that types without trivial move constructors are + // correctly marked as such. + EXPECT_FALSE(absl::is_trivially_move_constructible<std::string>::value); + EXPECT_FALSE(absl::is_trivially_move_constructible<std::vector<int>>::value); + + // Verify that simple_pairs of types without trivial move constructors + // are not marked as trivial. + EXPECT_FALSE((absl::is_trivially_move_constructible< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::is_trivially_move_constructible< + simple_pair<std::string, int>>::value)); + + // Verify that arrays are not + using int10 = int[10]; + EXPECT_FALSE(absl::is_trivially_move_constructible<int10>::value); +} + +TEST(TypeTraitsTest, TestTrivialCopyCtor) { + // Verify that arithmetic types and pointers have trivial copy + // constructors. + EXPECT_TRUE(absl::is_trivially_copy_constructible<bool>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<char>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<unsigned char>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<signed char>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<wchar_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<int>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<unsigned int>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<int16_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<uint16_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<int64_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<uint64_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<float>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<double>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<long double>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<std::string*>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<const std::string*>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<const Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<std::string**>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<Trivial**>::value); + + // Reference types + EXPECT_TRUE(absl::is_trivially_copy_constructible<int&>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<int&&>::value); + + // types with compiler generated copy ctors + EXPECT_TRUE(absl::is_trivially_copy_constructible<Trivial>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<TrivialCopyCtor>::value); + + // Verify that types without them (i.e. nontrivial or deleted) are not. + EXPECT_FALSE( + absl::is_trivially_copy_constructible<NontrivialCopyCtor>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<DeletedCopyCtor>::value); + EXPECT_FALSE( + absl::is_trivially_copy_constructible<MovableNonCopyable>::value); + EXPECT_FALSE( + absl::is_trivially_copy_constructible<NonCopyableOrMovable>::value); + + // type with nontrivial destructor are nontrivial copy construbtible + EXPECT_FALSE( + absl::is_trivially_copy_constructible<NontrivialDestructor>::value); + + // types with vtables + EXPECT_FALSE(absl::is_trivially_copy_constructible<Base>::value); + + // Verify that simple_pair of such types is trivially copy constructible + EXPECT_TRUE( + (absl::is_trivially_copy_constructible<simple_pair<int, char*>>::value)); + EXPECT_TRUE(( + absl::is_trivially_copy_constructible<simple_pair<int, Trivial>>::value)); + EXPECT_TRUE((absl::is_trivially_copy_constructible< + simple_pair<int, TrivialCopyCtor>>::value)); + + // Verify that types without trivial copy constructors are + // correctly marked as such. + EXPECT_FALSE(absl::is_trivially_copy_constructible<std::string>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<std::vector<int>>::value); + + // Verify that simple_pairs of types without trivial copy constructors + // are not marked as trivial. + EXPECT_FALSE((absl::is_trivially_copy_constructible< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::is_trivially_copy_constructible< + simple_pair<std::string, int>>::value)); + + // Verify that arrays are not + using int10 = int[10]; + EXPECT_FALSE(absl::is_trivially_copy_constructible<int10>::value); +} + +TEST(TypeTraitsTest, TestTrivialMoveAssign) { + // Verify that arithmetic types and pointers have trivial move + // assignment operators. + EXPECT_TRUE(absl::is_trivially_move_assignable<bool>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<char>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<unsigned char>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<signed char>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<wchar_t>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<int>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<unsigned int>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<int16_t>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<uint16_t>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<int64_t>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<uint64_t>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<float>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<double>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<long double>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<std::string*>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<const std::string*>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<const Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<std::string**>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<Trivial**>::value); + + // const qualified types are not assignable + EXPECT_FALSE(absl::is_trivially_move_assignable<const int>::value); + + // types with compiler generated move assignment + EXPECT_TRUE(absl::is_trivially_move_assignable<Trivial>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<TrivialMoveAssign>::value); + + // Verify that types without them (i.e. nontrivial or deleted) are not. + EXPECT_FALSE(absl::is_trivially_move_assignable<NontrivialCopyAssign>::value); + EXPECT_FALSE(absl::is_trivially_move_assignable<DeletedCopyAssign>::value); + EXPECT_FALSE(absl::is_trivially_move_assignable<NonCopyableOrMovable>::value); + + // types with vtables + EXPECT_FALSE(absl::is_trivially_move_assignable<Base>::value); + + // Verify that simple_pair is trivially assignable + EXPECT_TRUE( + (absl::is_trivially_move_assignable<simple_pair<int, char*>>::value)); + EXPECT_TRUE( + (absl::is_trivially_move_assignable<simple_pair<int, Trivial>>::value)); + EXPECT_TRUE((absl::is_trivially_move_assignable< + simple_pair<int, TrivialMoveAssign>>::value)); + + // Verify that types not trivially move assignable are + // correctly marked as such. + EXPECT_FALSE(absl::is_trivially_move_assignable<std::string>::value); + EXPECT_FALSE(absl::is_trivially_move_assignable<std::vector<int>>::value); + + // Verify that simple_pairs of types not trivially move assignable + // are not marked as trivial. + EXPECT_FALSE((absl::is_trivially_move_assignable< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::is_trivially_move_assignable< + simple_pair<std::string, int>>::value)); + + // Verify that arrays are not trivially move assignable + using int10 = int[10]; + EXPECT_FALSE(absl::is_trivially_move_assignable<int10>::value); + + // Verify that references are handled correctly + EXPECT_TRUE(absl::is_trivially_move_assignable<Trivial&&>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<Trivial&>::value); +} + +TEST(TypeTraitsTest, TestTrivialCopyAssign) { + // Verify that arithmetic types and pointers have trivial copy + // assignment operators. + EXPECT_TRUE(absl::is_trivially_copy_assignable<bool>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<char>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<unsigned char>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<signed char>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<wchar_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<int>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<unsigned int>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<int16_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<uint16_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<int64_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<uint64_t>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<float>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<double>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<long double>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<std::string*>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<const std::string*>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<const Trivial*>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<std::string**>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<Trivial**>::value); + + // const qualified types are not assignable + EXPECT_FALSE(absl::is_trivially_copy_assignable<const int>::value); + + // types with compiler generated copy assignment + EXPECT_TRUE(absl::is_trivially_copy_assignable<Trivial>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<TrivialCopyAssign>::value); + + // Verify that types without them (i.e. nontrivial or deleted) are not. + EXPECT_FALSE(absl::is_trivially_copy_assignable<NontrivialCopyAssign>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<DeletedCopyAssign>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<MovableNonCopyable>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<NonCopyableOrMovable>::value); + + // types with vtables + EXPECT_FALSE(absl::is_trivially_copy_assignable<Base>::value); + + // Verify that simple_pair is trivially assignable + EXPECT_TRUE( + (absl::is_trivially_copy_assignable<simple_pair<int, char*>>::value)); + EXPECT_TRUE( + (absl::is_trivially_copy_assignable<simple_pair<int, Trivial>>::value)); + EXPECT_TRUE((absl::is_trivially_copy_assignable< + simple_pair<int, TrivialCopyAssign>>::value)); + + // Verify that types not trivially copy assignable are + // correctly marked as such. + EXPECT_FALSE(absl::is_trivially_copy_assignable<std::string>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<std::vector<int>>::value); + + // Verify that simple_pairs of types not trivially copy assignable + // are not marked as trivial. + EXPECT_FALSE((absl::is_trivially_copy_assignable< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::is_trivially_copy_assignable< + simple_pair<std::string, int>>::value)); + + // Verify that arrays are not trivially copy assignable + using int10 = int[10]; + EXPECT_FALSE(absl::is_trivially_copy_assignable<int10>::value); + + // Verify that references are handled correctly + EXPECT_TRUE(absl::is_trivially_copy_assignable<Trivial&&>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<Trivial&>::value); +} + +TEST(TypeTraitsTest, TestTriviallyCopyable) { + // Verify that arithmetic types and pointers are trivially copyable. + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable<bool>::value); + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable<char>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<unsigned char>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<signed char>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<wchar_t>::value); + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable<int>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<unsigned int>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<int16_t>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<uint16_t>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<int64_t>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<uint64_t>::value); + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable<float>::value); + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable<double>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<long double>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<std::string*>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<Trivial*>::value); + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable< + const std::string*>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<const Trivial*>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<std::string**>::value); + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<Trivial**>::value); + + // const qualified types are not assignable but are constructible + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<const int>::value); + + // Trivial copy constructor/assignment and destructor. + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<Trivial>::value); + // Trivial copy assignment, but non-trivial copy constructor/destructor. + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable< + TrivialCopyAssign>::value); + // Trivial copy constructor, but non-trivial assignment. + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable< + TrivialCopyCtor>::value); + + // Types with a non-trivial copy constructor/assignment + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable< + NontrivialCopyCtor>::value); + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable< + NontrivialCopyAssign>::value); + + // Types without copy constructor/assignment, but with move + // MSVC disagrees with other compilers about this: + // EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable< + // MovableNonCopyable>::value); + + // Types without copy/move constructor/assignment + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable< + NonCopyableOrMovable>::value); + + // No copy assign, but has trivial copy constructor. + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable< + DeletedCopyAssign>::value); + + // types with vtables + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable<Base>::value); + + // Verify that simple_pair is trivially copyable if members are + EXPECT_TRUE((absl::type_traits_internal::is_trivially_copyable< + simple_pair<int, char*>>::value)); + EXPECT_TRUE((absl::type_traits_internal::is_trivially_copyable< + simple_pair<int, Trivial>>::value)); + + // Verify that types not trivially copyable are + // correctly marked as such. + EXPECT_FALSE( + absl::type_traits_internal::is_trivially_copyable<std::string>::value); + EXPECT_FALSE(absl::type_traits_internal::is_trivially_copyable< + std::vector<int>>::value); + + // Verify that simple_pairs of types not trivially copyable + // are not marked as trivial. + EXPECT_FALSE((absl::type_traits_internal::is_trivially_copyable< + simple_pair<int, std::string>>::value)); + EXPECT_FALSE((absl::type_traits_internal::is_trivially_copyable< + simple_pair<std::string, int>>::value)); + EXPECT_FALSE((absl::type_traits_internal::is_trivially_copyable< + simple_pair<int, TrivialCopyAssign>>::value)); + + // Verify that arrays of trivially copyable types are trivially copyable + using int10 = int[10]; + EXPECT_TRUE(absl::type_traits_internal::is_trivially_copyable<int10>::value); + using int10x10 = int[10][10]; + EXPECT_TRUE( + absl::type_traits_internal::is_trivially_copyable<int10x10>::value); + + // Verify that references are handled correctly + EXPECT_FALSE( + absl::type_traits_internal::is_trivially_copyable<Trivial&&>::value); + EXPECT_FALSE( + absl::type_traits_internal::is_trivially_copyable<Trivial&>::value); +} + +#define ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(trait_name, ...) \ + EXPECT_TRUE((std::is_same<typename std::trait_name<__VA_ARGS__>::type, \ + absl::trait_name##_t<__VA_ARGS__>>::value)) + +TEST(TypeTraitsTest, TestRemoveCVAliases) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_cv, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_cv, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_cv, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_cv, const volatile int); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_const, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_const, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_const, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_const, const volatile int); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_volatile, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_volatile, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_volatile, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_volatile, const volatile int); +} + +TEST(TypeTraitsTest, TestAddCVAliases) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_cv, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_cv, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_cv, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_cv, const volatile int); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_const, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_const, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_const, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_const, const volatile int); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_volatile, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_volatile, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_volatile, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_volatile, const volatile int); +} + +TEST(TypeTraitsTest, TestReferenceAliases) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_reference, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_reference, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_reference, int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_reference, volatile int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_reference, int&&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_reference, volatile int&&); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_lvalue_reference, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_lvalue_reference, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_lvalue_reference, int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_lvalue_reference, volatile int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_lvalue_reference, int&&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_lvalue_reference, volatile int&&); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_rvalue_reference, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_rvalue_reference, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_rvalue_reference, int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_rvalue_reference, volatile int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_rvalue_reference, int&&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_rvalue_reference, volatile int&&); +} + +TEST(TypeTraitsTest, TestPointerAliases) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_pointer, int*); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_pointer, volatile int*); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_pointer, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(add_pointer, volatile int); +} + +TEST(TypeTraitsTest, TestSignednessAliases) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_signed, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_signed, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_signed, unsigned); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_signed, volatile unsigned); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_unsigned, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_unsigned, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_unsigned, unsigned); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(make_unsigned, volatile unsigned); +} + +TEST(TypeTraitsTest, TestExtentAliases) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_extent, int[]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_extent, int[1]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_extent, int[1][1]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_extent, int[][1]); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_all_extents, int[]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_all_extents, int[1]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_all_extents, int[1][1]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(remove_all_extents, int[][1]); +} + +TEST(TypeTraitsTest, TestAlignedStorageAlias) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 1); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 2); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 3); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 4); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 5); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 6); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 7); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 8); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 9); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 10); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 11); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 12); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 13); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 14); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 15); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 16); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 17); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 18); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 19); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 20); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 21); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 22); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 23); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 24); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 25); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 26); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 27); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 28); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 29); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 30); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 31); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 32); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 33); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 1, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 2, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 3, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 4, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 5, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 6, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 7, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 8, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 9, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 10, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 11, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 12, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 13, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 14, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 15, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 16, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 17, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 18, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 19, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 20, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 21, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 22, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 23, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 24, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 25, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 26, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 27, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 28, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 29, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 30, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 31, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 32, 128); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(aligned_storage, 33, 128); +} + +TEST(TypeTraitsTest, TestDecay) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, const int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, volatile int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, const volatile int); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, const int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, volatile int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, const volatile int&); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, const int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, volatile int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, const volatile int&); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int[1]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int[1][1]); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int[][1]); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int()); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int(float)); // NOLINT + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(decay, int(char, ...)); // NOLINT +} + +struct TypeA {}; +struct TypeB {}; +struct TypeC {}; +struct TypeD {}; + +template <typename T> +struct Wrap {}; + +enum class TypeEnum { A, B, C, D }; + +struct GetTypeT { + template <typename T, + absl::enable_if_t<std::is_same<T, TypeA>::value, int> = 0> + TypeEnum operator()(Wrap<T>) const { + return TypeEnum::A; + } + + template <typename T, + absl::enable_if_t<std::is_same<T, TypeB>::value, int> = 0> + TypeEnum operator()(Wrap<T>) const { + return TypeEnum::B; + } + + template <typename T, + absl::enable_if_t<std::is_same<T, TypeC>::value, int> = 0> + TypeEnum operator()(Wrap<T>) const { + return TypeEnum::C; + } + + // NOTE: TypeD is intentionally not handled +} constexpr GetType = {}; + +TEST(TypeTraitsTest, TestEnableIf) { + EXPECT_EQ(TypeEnum::A, GetType(Wrap<TypeA>())); + EXPECT_EQ(TypeEnum::B, GetType(Wrap<TypeB>())); + EXPECT_EQ(TypeEnum::C, GetType(Wrap<TypeC>())); +} + +TEST(TypeTraitsTest, TestConditional) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(conditional, true, int, char); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(conditional, false, int, char); +} + +// TODO(calabrese) Check with specialized std::common_type +TEST(TypeTraitsTest, TestCommonType) { + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(common_type, int); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(common_type, int, char); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(common_type, int, char, int); + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(common_type, int&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(common_type, int, char&); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(common_type, int, char, int&); +} + +TEST(TypeTraitsTest, TestUnderlyingType) { + enum class enum_char : char {}; + enum class enum_long_long : long long {}; // NOLINT(runtime/int) + + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(underlying_type, enum_char); + ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(underlying_type, enum_long_long); +} + +struct GetTypeExtT { + template <typename T> + absl::result_of_t<const GetTypeT&(T)> operator()(T&& arg) const { + return GetType(std::forward<T>(arg)); + } + + TypeEnum operator()(Wrap<TypeD>) const { return TypeEnum::D; } +} constexpr GetTypeExt = {}; + +TEST(TypeTraitsTest, TestResultOf) { + EXPECT_EQ(TypeEnum::A, GetTypeExt(Wrap<TypeA>())); + EXPECT_EQ(TypeEnum::B, GetTypeExt(Wrap<TypeB>())); + EXPECT_EQ(TypeEnum::C, GetTypeExt(Wrap<TypeC>())); + EXPECT_EQ(TypeEnum::D, GetTypeExt(Wrap<TypeD>())); +} + +template <typename T> +bool TestCopyAssign() { + return absl::is_copy_assignable<T>::value == + std::is_copy_assignable<T>::value; +} + +TEST(TypeTraitsTest, IsCopyAssignable) { + EXPECT_TRUE(TestCopyAssign<int>()); + EXPECT_TRUE(TestCopyAssign<int&>()); + EXPECT_TRUE(TestCopyAssign<int&&>()); + + struct S {}; + EXPECT_TRUE(TestCopyAssign<S>()); + EXPECT_TRUE(TestCopyAssign<S&>()); + EXPECT_TRUE(TestCopyAssign<S&&>()); + + class C { + public: + explicit C(C* c) : c_(c) {} + ~C() { delete c_; } + + private: + C* c_; + }; + EXPECT_TRUE(TestCopyAssign<C>()); + EXPECT_TRUE(TestCopyAssign<C&>()); + EXPECT_TRUE(TestCopyAssign<C&&>()); + + // Reason for ifndef: add_lvalue_reference<T> in libc++ breaks for these cases +#ifndef _LIBCPP_VERSION + EXPECT_TRUE(TestCopyAssign<int()>()); + EXPECT_TRUE(TestCopyAssign<int(int) const>()); + EXPECT_TRUE(TestCopyAssign<int(...) volatile&>()); + EXPECT_TRUE(TestCopyAssign<int(int, ...) const volatile&&>()); +#endif // _LIBCPP_VERSION +} + +template <typename T> +bool TestMoveAssign() { + return absl::is_move_assignable<T>::value == + std::is_move_assignable<T>::value; +} + +TEST(TypeTraitsTest, IsMoveAssignable) { + EXPECT_TRUE(TestMoveAssign<int>()); + EXPECT_TRUE(TestMoveAssign<int&>()); + EXPECT_TRUE(TestMoveAssign<int&&>()); + + struct S {}; + EXPECT_TRUE(TestMoveAssign<S>()); + EXPECT_TRUE(TestMoveAssign<S&>()); + EXPECT_TRUE(TestMoveAssign<S&&>()); + + class C { + public: + explicit C(C* c) : c_(c) {} + ~C() { delete c_; } + void operator=(const C&) = delete; + void operator=(C&&) = delete; + + private: + C* c_; + }; + EXPECT_TRUE(TestMoveAssign<C>()); + EXPECT_TRUE(TestMoveAssign<C&>()); + EXPECT_TRUE(TestMoveAssign<C&&>()); + + // Reason for ifndef: add_lvalue_reference<T> in libc++ breaks for these cases +#ifndef _LIBCPP_VERSION + EXPECT_TRUE(TestMoveAssign<int()>()); + EXPECT_TRUE(TestMoveAssign<int(int) const>()); + EXPECT_TRUE(TestMoveAssign<int(...) volatile&>()); + EXPECT_TRUE(TestMoveAssign<int(int, ...) const volatile&&>()); +#endif // _LIBCPP_VERSION +} + +namespace adl_namespace { + +struct DeletedSwap { +}; + +void swap(DeletedSwap&, DeletedSwap&) = delete; + +struct SpecialNoexceptSwap { + SpecialNoexceptSwap(SpecialNoexceptSwap&&) {} + SpecialNoexceptSwap& operator=(SpecialNoexceptSwap&&) { return *this; } + ~SpecialNoexceptSwap() = default; +}; + +void swap(SpecialNoexceptSwap&, SpecialNoexceptSwap&) noexcept {} + +} // namespace adl_namespace + +TEST(TypeTraitsTest, IsSwappable) { + using absl::type_traits_internal::IsSwappable; + using absl::type_traits_internal::StdSwapIsUnconstrained; + + EXPECT_TRUE(IsSwappable<int>::value); + + struct S {}; + EXPECT_TRUE(IsSwappable<S>::value); + + struct NoConstruct { + NoConstruct(NoConstruct&&) = delete; + NoConstruct& operator=(NoConstruct&&) { return *this; } + ~NoConstruct() = default; + }; + + EXPECT_EQ(IsSwappable<NoConstruct>::value, StdSwapIsUnconstrained::value); + struct NoAssign { + NoAssign(NoAssign&&) {} + NoAssign& operator=(NoAssign&&) = delete; + ~NoAssign() = default; + }; + + EXPECT_EQ(IsSwappable<NoAssign>::value, StdSwapIsUnconstrained::value); + + EXPECT_FALSE(IsSwappable<adl_namespace::DeletedSwap>::value); + + EXPECT_TRUE(IsSwappable<adl_namespace::SpecialNoexceptSwap>::value); +} + +TEST(TypeTraitsTest, IsNothrowSwappable) { + using absl::type_traits_internal::IsNothrowSwappable; + using absl::type_traits_internal::StdSwapIsUnconstrained; + + EXPECT_TRUE(IsNothrowSwappable<int>::value); + + struct NonNoexceptMoves { + NonNoexceptMoves(NonNoexceptMoves&&) {} + NonNoexceptMoves& operator=(NonNoexceptMoves&&) { return *this; } + ~NonNoexceptMoves() = default; + }; + + EXPECT_FALSE(IsNothrowSwappable<NonNoexceptMoves>::value); + + struct NoConstruct { + NoConstruct(NoConstruct&&) = delete; + NoConstruct& operator=(NoConstruct&&) { return *this; } + ~NoConstruct() = default; + }; + + EXPECT_FALSE(IsNothrowSwappable<NoConstruct>::value); + + struct NoAssign { + NoAssign(NoAssign&&) {} + NoAssign& operator=(NoAssign&&) = delete; + ~NoAssign() = default; + }; + + EXPECT_FALSE(IsNothrowSwappable<NoAssign>::value); + + EXPECT_FALSE(IsNothrowSwappable<adl_namespace::DeletedSwap>::value); + + EXPECT_TRUE(IsNothrowSwappable<adl_namespace::SpecialNoexceptSwap>::value); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/numeric/BUILD.bazel b/third_party/abseil_cpp/absl/numeric/BUILD.bazel new file mode 100644 index 000000000000..f808f5dab06e --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/BUILD.bazel @@ -0,0 +1,74 @@ +# Copyright 2018 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "int128", + srcs = [ + "int128.cc", + "int128_have_intrinsic.inc", + "int128_no_intrinsic.inc", + ], + hdrs = ["int128.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:bits", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "int128_test", + size = "small", + srcs = [ + "int128_stream_test.cc", + "int128_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":int128", + "//absl/base", + "//absl/base:core_headers", + "//absl/hash:hash_testing", + "//absl/meta:type_traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "int128_benchmark", + srcs = ["int128_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + deps = [ + ":int128", + "//absl/base:config", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/third_party/abseil_cpp/absl/numeric/CMakeLists.txt b/third_party/abseil_cpp/absl/numeric/CMakeLists.txt new file mode 100644 index 000000000000..1e12d80f7ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/CMakeLists.txt @@ -0,0 +1,61 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + int128 + HDRS + "int128.h" + SRCS + "int128.cc" + "int128_have_intrinsic.inc" + "int128_no_intrinsic.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::config + absl::core_headers + PUBLIC +) + +absl_cc_test( + NAME + int128_test + SRCS + "int128_stream_test.cc" + "int128_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::int128 + absl::base + absl::core_headers + absl::hash_testing + absl::type_traits + gmock_main +) + +# component target +absl_cc_library( + NAME + numeric + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::int128 + PUBLIC +) diff --git a/third_party/abseil_cpp/absl/numeric/int128.cc b/third_party/abseil_cpp/absl/numeric/int128.cc new file mode 100644 index 000000000000..e21e5e9a4ad4 --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128.cc @@ -0,0 +1,390 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/numeric/int128.h" + +#include <stddef.h> + +#include <cassert> +#include <iomanip> +#include <ostream> // NOLINT(readability/streams) +#include <sstream> +#include <string> +#include <type_traits> + +#include "absl/base/internal/bits.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_DLL const uint128 kuint128max = MakeUint128( + std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max()); + +namespace { + +// Returns the 0-based position of the last set bit (i.e., most significant bit) +// in the given uint128. The argument is not 0. +// +// For example: +// Given: 5 (decimal) == 101 (binary) +// Returns: 2 +inline ABSL_ATTRIBUTE_ALWAYS_INLINE int Fls128(uint128 n) { + if (uint64_t hi = Uint128High64(n)) { + ABSL_INTERNAL_ASSUME(hi != 0); + return 127 - base_internal::CountLeadingZeros64(hi); + } + const uint64_t low = Uint128Low64(n); + ABSL_INTERNAL_ASSUME(low != 0); + return 63 - base_internal::CountLeadingZeros64(low); +} + +// Long division/modulo for uint128 implemented using the shift-subtract +// division algorithm adapted from: +// https://stackoverflow.com/questions/5386377/division-without-using +inline void DivModImpl(uint128 dividend, uint128 divisor, uint128* quotient_ret, + uint128* remainder_ret) { + assert(divisor != 0); + + if (divisor > dividend) { + *quotient_ret = 0; + *remainder_ret = dividend; + return; + } + + if (divisor == dividend) { + *quotient_ret = 1; + *remainder_ret = 0; + return; + } + + uint128 denominator = divisor; + uint128 quotient = 0; + + // Left aligns the MSB of the denominator and the dividend. + const int shift = Fls128(dividend) - Fls128(denominator); + denominator <<= shift; + + // Uses shift-subtract algorithm to divide dividend by denominator. The + // remainder will be left in dividend. + for (int i = 0; i <= shift; ++i) { + quotient <<= 1; + if (dividend >= denominator) { + dividend -= denominator; + quotient |= 1; + } + denominator >>= 1; + } + + *quotient_ret = quotient; + *remainder_ret = dividend; +} + +template <typename T> +uint128 MakeUint128FromFloat(T v) { + static_assert(std::is_floating_point<T>::value, ""); + + // Rounding behavior is towards zero, same as for built-in types. + + // Undefined behavior if v is NaN or cannot fit into uint128. + assert(std::isfinite(v) && v > -1 && + (std::numeric_limits<T>::max_exponent <= 128 || + v < std::ldexp(static_cast<T>(1), 128))); + + if (v >= std::ldexp(static_cast<T>(1), 64)) { + uint64_t hi = static_cast<uint64_t>(std::ldexp(v, -64)); + uint64_t lo = static_cast<uint64_t>(v - std::ldexp(static_cast<T>(hi), 64)); + return MakeUint128(hi, lo); + } + + return MakeUint128(0, static_cast<uint64_t>(v)); +} + +#if defined(__clang__) && !defined(__SSE3__) +// Workaround for clang bug: https://bugs.llvm.org/show_bug.cgi?id=38289 +// Casting from long double to uint64_t is miscompiled and drops bits. +// It is more work, so only use when we need the workaround. +uint128 MakeUint128FromFloat(long double v) { + // Go 50 bits at a time, that fits in a double + static_assert(std::numeric_limits<double>::digits >= 50, ""); + static_assert(std::numeric_limits<long double>::digits <= 150, ""); + // Undefined behavior if v is not finite or cannot fit into uint128. + assert(std::isfinite(v) && v > -1 && v < std::ldexp(1.0L, 128)); + + v = std::ldexp(v, -100); + uint64_t w0 = static_cast<uint64_t>(static_cast<double>(std::trunc(v))); + v = std::ldexp(v - static_cast<double>(w0), 50); + uint64_t w1 = static_cast<uint64_t>(static_cast<double>(std::trunc(v))); + v = std::ldexp(v - static_cast<double>(w1), 50); + uint64_t w2 = static_cast<uint64_t>(static_cast<double>(std::trunc(v))); + return (static_cast<uint128>(w0) << 100) | (static_cast<uint128>(w1) << 50) | + static_cast<uint128>(w2); +} +#endif // __clang__ && !__SSE3__ +} // namespace + +uint128::uint128(float v) : uint128(MakeUint128FromFloat(v)) {} +uint128::uint128(double v) : uint128(MakeUint128FromFloat(v)) {} +uint128::uint128(long double v) : uint128(MakeUint128FromFloat(v)) {} + +uint128 operator/(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) / + static_cast<unsigned __int128>(rhs); +#else // ABSL_HAVE_INTRINSIC_INT128 + uint128 quotient = 0; + uint128 remainder = 0; + DivModImpl(lhs, rhs, "ient, &remainder); + return quotient; +#endif // ABSL_HAVE_INTRINSIC_INT128 +} +uint128 operator%(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) % + static_cast<unsigned __int128>(rhs); +#else // ABSL_HAVE_INTRINSIC_INT128 + uint128 quotient = 0; + uint128 remainder = 0; + DivModImpl(lhs, rhs, "ient, &remainder); + return remainder; +#endif // ABSL_HAVE_INTRINSIC_INT128 +} + +namespace { + +std::string Uint128ToFormattedString(uint128 v, std::ios_base::fmtflags flags) { + // Select a divisor which is the largest power of the base < 2^64. + uint128 div; + int div_base_log; + switch (flags & std::ios::basefield) { + case std::ios::hex: + div = 0x1000000000000000; // 16^15 + div_base_log = 15; + break; + case std::ios::oct: + div = 01000000000000000000000; // 8^21 + div_base_log = 21; + break; + default: // std::ios::dec + div = 10000000000000000000u; // 10^19 + div_base_log = 19; + break; + } + + // Now piece together the uint128 representation from three chunks of the + // original value, each less than "div" and therefore representable as a + // uint64_t. + std::ostringstream os; + std::ios_base::fmtflags copy_mask = + std::ios::basefield | std::ios::showbase | std::ios::uppercase; + os.setf(flags & copy_mask, copy_mask); + uint128 high = v; + uint128 low; + DivModImpl(high, div, &high, &low); + uint128 mid; + DivModImpl(high, div, &high, &mid); + if (Uint128Low64(high) != 0) { + os << Uint128Low64(high); + os << std::noshowbase << std::setfill('0') << std::setw(div_base_log); + os << Uint128Low64(mid); + os << std::setw(div_base_log); + } else if (Uint128Low64(mid) != 0) { + os << Uint128Low64(mid); + os << std::noshowbase << std::setfill('0') << std::setw(div_base_log); + } + os << Uint128Low64(low); + return os.str(); +} + +} // namespace + +std::ostream& operator<<(std::ostream& os, uint128 v) { + std::ios_base::fmtflags flags = os.flags(); + std::string rep = Uint128ToFormattedString(v, flags); + + // Add the requisite padding. + std::streamsize width = os.width(0); + if (static_cast<size_t>(width) > rep.size()) { + std::ios::fmtflags adjustfield = flags & std::ios::adjustfield; + if (adjustfield == std::ios::left) { + rep.append(width - rep.size(), os.fill()); + } else if (adjustfield == std::ios::internal && + (flags & std::ios::showbase) && + (flags & std::ios::basefield) == std::ios::hex && v != 0) { + rep.insert(2, width - rep.size(), os.fill()); + } else { + rep.insert(0, width - rep.size(), os.fill()); + } + } + + return os << rep; +} + +namespace { + +uint128 UnsignedAbsoluteValue(int128 v) { + // Cast to uint128 before possibly negating because -Int128Min() is undefined. + return Int128High64(v) < 0 ? -uint128(v) : uint128(v); +} + +} // namespace + +#if !defined(ABSL_HAVE_INTRINSIC_INT128) +namespace { + +template <typename T> +int128 MakeInt128FromFloat(T v) { + // Conversion when v is NaN or cannot fit into int128 would be undefined + // behavior if using an intrinsic 128-bit integer. + assert(std::isfinite(v) && (std::numeric_limits<T>::max_exponent <= 127 || + (v >= -std::ldexp(static_cast<T>(1), 127) && + v < std::ldexp(static_cast<T>(1), 127)))); + + // We must convert the absolute value and then negate as needed, because + // floating point types are typically sign-magnitude. Otherwise, the + // difference between the high and low 64 bits when interpreted as two's + // complement overwhelms the precision of the mantissa. + uint128 result = v < 0 ? -MakeUint128FromFloat(-v) : MakeUint128FromFloat(v); + return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(result)), + Uint128Low64(result)); +} + +} // namespace + +int128::int128(float v) : int128(MakeInt128FromFloat(v)) {} +int128::int128(double v) : int128(MakeInt128FromFloat(v)) {} +int128::int128(long double v) : int128(MakeInt128FromFloat(v)) {} + +int128 operator/(int128 lhs, int128 rhs) { + assert(lhs != Int128Min() || rhs != -1); // UB on two's complement. + + uint128 quotient = 0; + uint128 remainder = 0; + DivModImpl(UnsignedAbsoluteValue(lhs), UnsignedAbsoluteValue(rhs), + "ient, &remainder); + if ((Int128High64(lhs) < 0) != (Int128High64(rhs) < 0)) quotient = -quotient; + return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(quotient)), + Uint128Low64(quotient)); +} + +int128 operator%(int128 lhs, int128 rhs) { + assert(lhs != Int128Min() || rhs != -1); // UB on two's complement. + + uint128 quotient = 0; + uint128 remainder = 0; + DivModImpl(UnsignedAbsoluteValue(lhs), UnsignedAbsoluteValue(rhs), + "ient, &remainder); + if (Int128High64(lhs) < 0) remainder = -remainder; + return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(remainder)), + Uint128Low64(remainder)); +} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +std::ostream& operator<<(std::ostream& os, int128 v) { + std::ios_base::fmtflags flags = os.flags(); + std::string rep; + + // Add the sign if needed. + bool print_as_decimal = + (flags & std::ios::basefield) == std::ios::dec || + (flags & std::ios::basefield) == std::ios_base::fmtflags(); + if (print_as_decimal) { + if (Int128High64(v) < 0) { + rep = "-"; + } else if (flags & std::ios::showpos) { + rep = "+"; + } + } + + rep.append(Uint128ToFormattedString( + print_as_decimal ? UnsignedAbsoluteValue(v) : uint128(v), os.flags())); + + // Add the requisite padding. + std::streamsize width = os.width(0); + if (static_cast<size_t>(width) > rep.size()) { + switch (flags & std::ios::adjustfield) { + case std::ios::left: + rep.append(width - rep.size(), os.fill()); + break; + case std::ios::internal: + if (print_as_decimal && (rep[0] == '+' || rep[0] == '-')) { + rep.insert(1, width - rep.size(), os.fill()); + } else if ((flags & std::ios::basefield) == std::ios::hex && + (flags & std::ios::showbase) && v != 0) { + rep.insert(2, width - rep.size(), os.fill()); + } else { + rep.insert(0, width - rep.size(), os.fill()); + } + break; + default: // std::ios::right + rep.insert(0, width - rep.size(), os.fill()); + break; + } + } + + return os << rep; +} + +ABSL_NAMESPACE_END +} // namespace absl + +namespace std { +constexpr bool numeric_limits<absl::uint128>::is_specialized; +constexpr bool numeric_limits<absl::uint128>::is_signed; +constexpr bool numeric_limits<absl::uint128>::is_integer; +constexpr bool numeric_limits<absl::uint128>::is_exact; +constexpr bool numeric_limits<absl::uint128>::has_infinity; +constexpr bool numeric_limits<absl::uint128>::has_quiet_NaN; +constexpr bool numeric_limits<absl::uint128>::has_signaling_NaN; +constexpr float_denorm_style numeric_limits<absl::uint128>::has_denorm; +constexpr bool numeric_limits<absl::uint128>::has_denorm_loss; +constexpr float_round_style numeric_limits<absl::uint128>::round_style; +constexpr bool numeric_limits<absl::uint128>::is_iec559; +constexpr bool numeric_limits<absl::uint128>::is_bounded; +constexpr bool numeric_limits<absl::uint128>::is_modulo; +constexpr int numeric_limits<absl::uint128>::digits; +constexpr int numeric_limits<absl::uint128>::digits10; +constexpr int numeric_limits<absl::uint128>::max_digits10; +constexpr int numeric_limits<absl::uint128>::radix; +constexpr int numeric_limits<absl::uint128>::min_exponent; +constexpr int numeric_limits<absl::uint128>::min_exponent10; +constexpr int numeric_limits<absl::uint128>::max_exponent; +constexpr int numeric_limits<absl::uint128>::max_exponent10; +constexpr bool numeric_limits<absl::uint128>::traps; +constexpr bool numeric_limits<absl::uint128>::tinyness_before; + +constexpr bool numeric_limits<absl::int128>::is_specialized; +constexpr bool numeric_limits<absl::int128>::is_signed; +constexpr bool numeric_limits<absl::int128>::is_integer; +constexpr bool numeric_limits<absl::int128>::is_exact; +constexpr bool numeric_limits<absl::int128>::has_infinity; +constexpr bool numeric_limits<absl::int128>::has_quiet_NaN; +constexpr bool numeric_limits<absl::int128>::has_signaling_NaN; +constexpr float_denorm_style numeric_limits<absl::int128>::has_denorm; +constexpr bool numeric_limits<absl::int128>::has_denorm_loss; +constexpr float_round_style numeric_limits<absl::int128>::round_style; +constexpr bool numeric_limits<absl::int128>::is_iec559; +constexpr bool numeric_limits<absl::int128>::is_bounded; +constexpr bool numeric_limits<absl::int128>::is_modulo; +constexpr int numeric_limits<absl::int128>::digits; +constexpr int numeric_limits<absl::int128>::digits10; +constexpr int numeric_limits<absl::int128>::max_digits10; +constexpr int numeric_limits<absl::int128>::radix; +constexpr int numeric_limits<absl::int128>::min_exponent; +constexpr int numeric_limits<absl::int128>::min_exponent10; +constexpr int numeric_limits<absl::int128>::max_exponent; +constexpr int numeric_limits<absl::int128>::max_exponent10; +constexpr bool numeric_limits<absl::int128>::traps; +constexpr bool numeric_limits<absl::int128>::tinyness_before; +} // namespace std diff --git a/third_party/abseil_cpp/absl/numeric/int128.h b/third_party/abseil_cpp/absl/numeric/int128.h new file mode 100644 index 000000000000..0dd814a890e7 --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128.h @@ -0,0 +1,1092 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: int128.h +// ----------------------------------------------------------------------------- +// +// This header file defines 128-bit integer types, `uint128` and `int128`. + +#ifndef ABSL_NUMERIC_INT128_H_ +#define ABSL_NUMERIC_INT128_H_ + +#include <cassert> +#include <cmath> +#include <cstdint> +#include <cstring> +#include <iosfwd> +#include <limits> +#include <utility> + +#include "absl/base/config.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" + +#if defined(_MSC_VER) +// In very old versions of MSVC and when the /Zc:wchar_t flag is off, wchar_t is +// a typedef for unsigned short. Otherwise wchar_t is mapped to the __wchar_t +// builtin type. We need to make sure not to define operator wchar_t() +// alongside operator unsigned short() in these instances. +#define ABSL_INTERNAL_WCHAR_T __wchar_t +#if defined(_M_X64) +#include <intrin.h> +#pragma intrinsic(_umul128) +#endif // defined(_M_X64) +#else // defined(_MSC_VER) +#define ABSL_INTERNAL_WCHAR_T wchar_t +#endif // defined(_MSC_VER) + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class int128; + +// uint128 +// +// An unsigned 128-bit integer type. The API is meant to mimic an intrinsic type +// as closely as is practical, including exhibiting undefined behavior in +// analogous cases (e.g. division by zero). This type is intended to be a +// drop-in replacement once C++ supports an intrinsic `uint128_t` type; when +// that occurs, existing well-behaved uses of `uint128` will continue to work +// using that new type. +// +// Note: code written with this type will continue to compile once `uint128_t` +// is introduced, provided the replacement helper functions +// `Uint128(Low|High)64()` and `MakeUint128()` are made. +// +// A `uint128` supports the following: +// +// * Implicit construction from integral types +// * Explicit conversion to integral types +// +// Additionally, if your compiler supports `__int128`, `uint128` is +// interoperable with that type. (Abseil checks for this compatibility through +// the `ABSL_HAVE_INTRINSIC_INT128` macro.) +// +// However, a `uint128` differs from intrinsic integral types in the following +// ways: +// +// * Errors on implicit conversions that do not preserve value (such as +// loss of precision when converting to float values). +// * Requires explicit construction from and conversion to floating point +// types. +// * Conversion to integral types requires an explicit static_cast() to +// mimic use of the `-Wnarrowing` compiler flag. +// * The alignment requirement of `uint128` may differ from that of an +// intrinsic 128-bit integer type depending on platform and build +// configuration. +// +// Example: +// +// float y = absl::Uint128Max(); // Error. uint128 cannot be implicitly +// // converted to float. +// +// absl::uint128 v; +// uint64_t i = v; // Error +// uint64_t i = static_cast<uint64_t>(v); // OK +// +class +#if defined(ABSL_HAVE_INTRINSIC_INT128) + alignas(unsigned __int128) +#endif // ABSL_HAVE_INTRINSIC_INT128 + uint128 { + public: + uint128() = default; + + // Constructors from arithmetic types + constexpr uint128(int v); // NOLINT(runtime/explicit) + constexpr uint128(unsigned int v); // NOLINT(runtime/explicit) + constexpr uint128(long v); // NOLINT(runtime/int) + constexpr uint128(unsigned long v); // NOLINT(runtime/int) + constexpr uint128(long long v); // NOLINT(runtime/int) + constexpr uint128(unsigned long long v); // NOLINT(runtime/int) +#ifdef ABSL_HAVE_INTRINSIC_INT128 + constexpr uint128(__int128 v); // NOLINT(runtime/explicit) + constexpr uint128(unsigned __int128 v); // NOLINT(runtime/explicit) +#endif // ABSL_HAVE_INTRINSIC_INT128 + constexpr uint128(int128 v); // NOLINT(runtime/explicit) + explicit uint128(float v); + explicit uint128(double v); + explicit uint128(long double v); + + // Assignment operators from arithmetic types + uint128& operator=(int v); + uint128& operator=(unsigned int v); + uint128& operator=(long v); // NOLINT(runtime/int) + uint128& operator=(unsigned long v); // NOLINT(runtime/int) + uint128& operator=(long long v); // NOLINT(runtime/int) + uint128& operator=(unsigned long long v); // NOLINT(runtime/int) +#ifdef ABSL_HAVE_INTRINSIC_INT128 + uint128& operator=(__int128 v); + uint128& operator=(unsigned __int128 v); +#endif // ABSL_HAVE_INTRINSIC_INT128 + uint128& operator=(int128 v); + + // Conversion operators to other arithmetic types + constexpr explicit operator bool() const; + constexpr explicit operator char() const; + constexpr explicit operator signed char() const; + constexpr explicit operator unsigned char() const; + constexpr explicit operator char16_t() const; + constexpr explicit operator char32_t() const; + constexpr explicit operator ABSL_INTERNAL_WCHAR_T() const; + constexpr explicit operator short() const; // NOLINT(runtime/int) + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator unsigned short() const; + constexpr explicit operator int() const; + constexpr explicit operator unsigned int() const; + constexpr explicit operator long() const; // NOLINT(runtime/int) + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator unsigned long() const; + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator long long() const; + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator unsigned long long() const; +#ifdef ABSL_HAVE_INTRINSIC_INT128 + constexpr explicit operator __int128() const; + constexpr explicit operator unsigned __int128() const; +#endif // ABSL_HAVE_INTRINSIC_INT128 + explicit operator float() const; + explicit operator double() const; + explicit operator long double() const; + + // Trivial copy constructor, assignment operator and destructor. + + // Arithmetic operators. + uint128& operator+=(uint128 other); + uint128& operator-=(uint128 other); + uint128& operator*=(uint128 other); + // Long division/modulo for uint128. + uint128& operator/=(uint128 other); + uint128& operator%=(uint128 other); + uint128 operator++(int); + uint128 operator--(int); + uint128& operator<<=(int); + uint128& operator>>=(int); + uint128& operator&=(uint128 other); + uint128& operator|=(uint128 other); + uint128& operator^=(uint128 other); + uint128& operator++(); + uint128& operator--(); + + // Uint128Low64() + // + // Returns the lower 64-bit value of a `uint128` value. + friend constexpr uint64_t Uint128Low64(uint128 v); + + // Uint128High64() + // + // Returns the higher 64-bit value of a `uint128` value. + friend constexpr uint64_t Uint128High64(uint128 v); + + // MakeUInt128() + // + // Constructs a `uint128` numeric value from two 64-bit unsigned integers. + // Note that this factory function is the only way to construct a `uint128` + // from integer values greater than 2^64. + // + // Example: + // + // absl::uint128 big = absl::MakeUint128(1, 0); + friend constexpr uint128 MakeUint128(uint64_t high, uint64_t low); + + // Uint128Max() + // + // Returns the highest value for a 128-bit unsigned integer. + friend constexpr uint128 Uint128Max(); + + // Support for absl::Hash. + template <typename H> + friend H AbslHashValue(H h, uint128 v) { + return H::combine(std::move(h), Uint128High64(v), Uint128Low64(v)); + } + + private: + constexpr uint128(uint64_t high, uint64_t low); + + // TODO(strel) Update implementation to use __int128 once all users of + // uint128 are fixed to not depend on alignof(uint128) == 8. Also add + // alignas(16) to class definition to keep alignment consistent across + // platforms. +#if defined(ABSL_IS_LITTLE_ENDIAN) + uint64_t lo_; + uint64_t hi_; +#elif defined(ABSL_IS_BIG_ENDIAN) + uint64_t hi_; + uint64_t lo_; +#else // byte order +#error "Unsupported byte order: must be little-endian or big-endian." +#endif // byte order +}; + +// Prefer to use the constexpr `Uint128Max()`. +// +// TODO(absl-team) deprecate kuint128max once migration tool is released. +ABSL_DLL extern const uint128 kuint128max; + +// allow uint128 to be logged +std::ostream& operator<<(std::ostream& os, uint128 v); + +// TODO(strel) add operator>>(std::istream&, uint128) + +constexpr uint128 Uint128Max() { + return uint128((std::numeric_limits<uint64_t>::max)(), + (std::numeric_limits<uint64_t>::max)()); +} + +ABSL_NAMESPACE_END +} // namespace absl + +// Specialized numeric_limits for uint128. +namespace std { +template <> +class numeric_limits<absl::uint128> { + public: + static constexpr bool is_specialized = true; + static constexpr bool is_signed = false; + static constexpr bool is_integer = true; + static constexpr bool is_exact = true; + static constexpr bool has_infinity = false; + static constexpr bool has_quiet_NaN = false; + static constexpr bool has_signaling_NaN = false; + static constexpr float_denorm_style has_denorm = denorm_absent; + static constexpr bool has_denorm_loss = false; + static constexpr float_round_style round_style = round_toward_zero; + static constexpr bool is_iec559 = false; + static constexpr bool is_bounded = true; + static constexpr bool is_modulo = true; + static constexpr int digits = 128; + static constexpr int digits10 = 38; + static constexpr int max_digits10 = 0; + static constexpr int radix = 2; + static constexpr int min_exponent = 0; + static constexpr int min_exponent10 = 0; + static constexpr int max_exponent = 0; + static constexpr int max_exponent10 = 0; +#ifdef ABSL_HAVE_INTRINSIC_INT128 + static constexpr bool traps = numeric_limits<unsigned __int128>::traps; +#else // ABSL_HAVE_INTRINSIC_INT128 + static constexpr bool traps = numeric_limits<uint64_t>::traps; +#endif // ABSL_HAVE_INTRINSIC_INT128 + static constexpr bool tinyness_before = false; + + static constexpr absl::uint128 (min)() { return 0; } + static constexpr absl::uint128 lowest() { return 0; } + static constexpr absl::uint128 (max)() { return absl::Uint128Max(); } + static constexpr absl::uint128 epsilon() { return 0; } + static constexpr absl::uint128 round_error() { return 0; } + static constexpr absl::uint128 infinity() { return 0; } + static constexpr absl::uint128 quiet_NaN() { return 0; } + static constexpr absl::uint128 signaling_NaN() { return 0; } + static constexpr absl::uint128 denorm_min() { return 0; } +}; +} // namespace std + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// int128 +// +// A signed 128-bit integer type. The API is meant to mimic an intrinsic +// integral type as closely as is practical, including exhibiting undefined +// behavior in analogous cases (e.g. division by zero). +// +// An `int128` supports the following: +// +// * Implicit construction from integral types +// * Explicit conversion to integral types +// +// However, an `int128` differs from intrinsic integral types in the following +// ways: +// +// * It is not implicitly convertible to other integral types. +// * Requires explicit construction from and conversion to floating point +// types. + +// Additionally, if your compiler supports `__int128`, `int128` is +// interoperable with that type. (Abseil checks for this compatibility through +// the `ABSL_HAVE_INTRINSIC_INT128` macro.) +// +// The design goal for `int128` is that it will be compatible with a future +// `int128_t`, if that type becomes a part of the standard. +// +// Example: +// +// float y = absl::int128(17); // Error. int128 cannot be implicitly +// // converted to float. +// +// absl::int128 v; +// int64_t i = v; // Error +// int64_t i = static_cast<int64_t>(v); // OK +// +class int128 { + public: + int128() = default; + + // Constructors from arithmetic types + constexpr int128(int v); // NOLINT(runtime/explicit) + constexpr int128(unsigned int v); // NOLINT(runtime/explicit) + constexpr int128(long v); // NOLINT(runtime/int) + constexpr int128(unsigned long v); // NOLINT(runtime/int) + constexpr int128(long long v); // NOLINT(runtime/int) + constexpr int128(unsigned long long v); // NOLINT(runtime/int) +#ifdef ABSL_HAVE_INTRINSIC_INT128 + constexpr int128(__int128 v); // NOLINT(runtime/explicit) + constexpr explicit int128(unsigned __int128 v); +#endif // ABSL_HAVE_INTRINSIC_INT128 + constexpr explicit int128(uint128 v); + explicit int128(float v); + explicit int128(double v); + explicit int128(long double v); + + // Assignment operators from arithmetic types + int128& operator=(int v); + int128& operator=(unsigned int v); + int128& operator=(long v); // NOLINT(runtime/int) + int128& operator=(unsigned long v); // NOLINT(runtime/int) + int128& operator=(long long v); // NOLINT(runtime/int) + int128& operator=(unsigned long long v); // NOLINT(runtime/int) +#ifdef ABSL_HAVE_INTRINSIC_INT128 + int128& operator=(__int128 v); +#endif // ABSL_HAVE_INTRINSIC_INT128 + + // Conversion operators to other arithmetic types + constexpr explicit operator bool() const; + constexpr explicit operator char() const; + constexpr explicit operator signed char() const; + constexpr explicit operator unsigned char() const; + constexpr explicit operator char16_t() const; + constexpr explicit operator char32_t() const; + constexpr explicit operator ABSL_INTERNAL_WCHAR_T() const; + constexpr explicit operator short() const; // NOLINT(runtime/int) + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator unsigned short() const; + constexpr explicit operator int() const; + constexpr explicit operator unsigned int() const; + constexpr explicit operator long() const; // NOLINT(runtime/int) + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator unsigned long() const; + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator long long() const; + // NOLINTNEXTLINE(runtime/int) + constexpr explicit operator unsigned long long() const; +#ifdef ABSL_HAVE_INTRINSIC_INT128 + constexpr explicit operator __int128() const; + constexpr explicit operator unsigned __int128() const; +#endif // ABSL_HAVE_INTRINSIC_INT128 + explicit operator float() const; + explicit operator double() const; + explicit operator long double() const; + + // Trivial copy constructor, assignment operator and destructor. + + // Arithmetic operators + int128& operator+=(int128 other); + int128& operator-=(int128 other); + int128& operator*=(int128 other); + int128& operator/=(int128 other); + int128& operator%=(int128 other); + int128 operator++(int); // postfix increment: i++ + int128 operator--(int); // postfix decrement: i-- + int128& operator++(); // prefix increment: ++i + int128& operator--(); // prefix decrement: --i + int128& operator&=(int128 other); + int128& operator|=(int128 other); + int128& operator^=(int128 other); + int128& operator<<=(int amount); + int128& operator>>=(int amount); + + // Int128Low64() + // + // Returns the lower 64-bit value of a `int128` value. + friend constexpr uint64_t Int128Low64(int128 v); + + // Int128High64() + // + // Returns the higher 64-bit value of a `int128` value. + friend constexpr int64_t Int128High64(int128 v); + + // MakeInt128() + // + // Constructs a `int128` numeric value from two 64-bit integers. Note that + // signedness is conveyed in the upper `high` value. + // + // (absl::int128(1) << 64) * high + low + // + // Note that this factory function is the only way to construct a `int128` + // from integer values greater than 2^64 or less than -2^64. + // + // Example: + // + // absl::int128 big = absl::MakeInt128(1, 0); + // absl::int128 big_n = absl::MakeInt128(-1, 0); + friend constexpr int128 MakeInt128(int64_t high, uint64_t low); + + // Int128Max() + // + // Returns the maximum value for a 128-bit signed integer. + friend constexpr int128 Int128Max(); + + // Int128Min() + // + // Returns the minimum value for a 128-bit signed integer. + friend constexpr int128 Int128Min(); + + // Support for absl::Hash. + template <typename H> + friend H AbslHashValue(H h, int128 v) { + return H::combine(std::move(h), Int128High64(v), Int128Low64(v)); + } + + private: + constexpr int128(int64_t high, uint64_t low); + +#if defined(ABSL_HAVE_INTRINSIC_INT128) + __int128 v_; +#else // ABSL_HAVE_INTRINSIC_INT128 +#if defined(ABSL_IS_LITTLE_ENDIAN) + uint64_t lo_; + int64_t hi_; +#elif defined(ABSL_IS_BIG_ENDIAN) + int64_t hi_; + uint64_t lo_; +#else // byte order +#error "Unsupported byte order: must be little-endian or big-endian." +#endif // byte order +#endif // ABSL_HAVE_INTRINSIC_INT128 +}; + +std::ostream& operator<<(std::ostream& os, int128 v); + +// TODO(absl-team) add operator>>(std::istream&, int128) + +constexpr int128 Int128Max() { + return int128((std::numeric_limits<int64_t>::max)(), + (std::numeric_limits<uint64_t>::max)()); +} + +constexpr int128 Int128Min() { + return int128((std::numeric_limits<int64_t>::min)(), 0); +} + +ABSL_NAMESPACE_END +} // namespace absl + +// Specialized numeric_limits for int128. +namespace std { +template <> +class numeric_limits<absl::int128> { + public: + static constexpr bool is_specialized = true; + static constexpr bool is_signed = true; + static constexpr bool is_integer = true; + static constexpr bool is_exact = true; + static constexpr bool has_infinity = false; + static constexpr bool has_quiet_NaN = false; + static constexpr bool has_signaling_NaN = false; + static constexpr float_denorm_style has_denorm = denorm_absent; + static constexpr bool has_denorm_loss = false; + static constexpr float_round_style round_style = round_toward_zero; + static constexpr bool is_iec559 = false; + static constexpr bool is_bounded = true; + static constexpr bool is_modulo = false; + static constexpr int digits = 127; + static constexpr int digits10 = 38; + static constexpr int max_digits10 = 0; + static constexpr int radix = 2; + static constexpr int min_exponent = 0; + static constexpr int min_exponent10 = 0; + static constexpr int max_exponent = 0; + static constexpr int max_exponent10 = 0; +#ifdef ABSL_HAVE_INTRINSIC_INT128 + static constexpr bool traps = numeric_limits<__int128>::traps; +#else // ABSL_HAVE_INTRINSIC_INT128 + static constexpr bool traps = numeric_limits<uint64_t>::traps; +#endif // ABSL_HAVE_INTRINSIC_INT128 + static constexpr bool tinyness_before = false; + + static constexpr absl::int128 (min)() { return absl::Int128Min(); } + static constexpr absl::int128 lowest() { return absl::Int128Min(); } + static constexpr absl::int128 (max)() { return absl::Int128Max(); } + static constexpr absl::int128 epsilon() { return 0; } + static constexpr absl::int128 round_error() { return 0; } + static constexpr absl::int128 infinity() { return 0; } + static constexpr absl::int128 quiet_NaN() { return 0; } + static constexpr absl::int128 signaling_NaN() { return 0; } + static constexpr absl::int128 denorm_min() { return 0; } +}; +} // namespace std + +// -------------------------------------------------------------------------- +// Implementation details follow +// -------------------------------------------------------------------------- +namespace absl { +ABSL_NAMESPACE_BEGIN + +constexpr uint128 MakeUint128(uint64_t high, uint64_t low) { + return uint128(high, low); +} + +// Assignment from integer types. + +inline uint128& uint128::operator=(int v) { return *this = uint128(v); } + +inline uint128& uint128::operator=(unsigned int v) { + return *this = uint128(v); +} + +inline uint128& uint128::operator=(long v) { // NOLINT(runtime/int) + return *this = uint128(v); +} + +// NOLINTNEXTLINE(runtime/int) +inline uint128& uint128::operator=(unsigned long v) { + return *this = uint128(v); +} + +// NOLINTNEXTLINE(runtime/int) +inline uint128& uint128::operator=(long long v) { + return *this = uint128(v); +} + +// NOLINTNEXTLINE(runtime/int) +inline uint128& uint128::operator=(unsigned long long v) { + return *this = uint128(v); +} + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +inline uint128& uint128::operator=(__int128 v) { + return *this = uint128(v); +} + +inline uint128& uint128::operator=(unsigned __int128 v) { + return *this = uint128(v); +} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +inline uint128& uint128::operator=(int128 v) { + return *this = uint128(v); +} + +// Arithmetic operators. + +uint128 operator<<(uint128 lhs, int amount); +uint128 operator>>(uint128 lhs, int amount); +uint128 operator+(uint128 lhs, uint128 rhs); +uint128 operator-(uint128 lhs, uint128 rhs); +uint128 operator*(uint128 lhs, uint128 rhs); +uint128 operator/(uint128 lhs, uint128 rhs); +uint128 operator%(uint128 lhs, uint128 rhs); + +inline uint128& uint128::operator<<=(int amount) { + *this = *this << amount; + return *this; +} + +inline uint128& uint128::operator>>=(int amount) { + *this = *this >> amount; + return *this; +} + +inline uint128& uint128::operator+=(uint128 other) { + *this = *this + other; + return *this; +} + +inline uint128& uint128::operator-=(uint128 other) { + *this = *this - other; + return *this; +} + +inline uint128& uint128::operator*=(uint128 other) { + *this = *this * other; + return *this; +} + +inline uint128& uint128::operator/=(uint128 other) { + *this = *this / other; + return *this; +} + +inline uint128& uint128::operator%=(uint128 other) { + *this = *this % other; + return *this; +} + +constexpr uint64_t Uint128Low64(uint128 v) { return v.lo_; } + +constexpr uint64_t Uint128High64(uint128 v) { return v.hi_; } + +// Constructors from integer types. + +#if defined(ABSL_IS_LITTLE_ENDIAN) + +constexpr uint128::uint128(uint64_t high, uint64_t low) + : lo_{low}, hi_{high} {} + +constexpr uint128::uint128(int v) + : lo_{static_cast<uint64_t>(v)}, + hi_{v < 0 ? (std::numeric_limits<uint64_t>::max)() : 0} {} +constexpr uint128::uint128(long v) // NOLINT(runtime/int) + : lo_{static_cast<uint64_t>(v)}, + hi_{v < 0 ? (std::numeric_limits<uint64_t>::max)() : 0} {} +constexpr uint128::uint128(long long v) // NOLINT(runtime/int) + : lo_{static_cast<uint64_t>(v)}, + hi_{v < 0 ? (std::numeric_limits<uint64_t>::max)() : 0} {} + +constexpr uint128::uint128(unsigned int v) : lo_{v}, hi_{0} {} +// NOLINTNEXTLINE(runtime/int) +constexpr uint128::uint128(unsigned long v) : lo_{v}, hi_{0} {} +// NOLINTNEXTLINE(runtime/int) +constexpr uint128::uint128(unsigned long long v) : lo_{v}, hi_{0} {} + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +constexpr uint128::uint128(__int128 v) + : lo_{static_cast<uint64_t>(v & ~uint64_t{0})}, + hi_{static_cast<uint64_t>(static_cast<unsigned __int128>(v) >> 64)} {} +constexpr uint128::uint128(unsigned __int128 v) + : lo_{static_cast<uint64_t>(v & ~uint64_t{0})}, + hi_{static_cast<uint64_t>(v >> 64)} {} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +constexpr uint128::uint128(int128 v) + : lo_{Int128Low64(v)}, hi_{static_cast<uint64_t>(Int128High64(v))} {} + +#elif defined(ABSL_IS_BIG_ENDIAN) + +constexpr uint128::uint128(uint64_t high, uint64_t low) + : hi_{high}, lo_{low} {} + +constexpr uint128::uint128(int v) + : hi_{v < 0 ? (std::numeric_limits<uint64_t>::max)() : 0}, + lo_{static_cast<uint64_t>(v)} {} +constexpr uint128::uint128(long v) // NOLINT(runtime/int) + : hi_{v < 0 ? (std::numeric_limits<uint64_t>::max)() : 0}, + lo_{static_cast<uint64_t>(v)} {} +constexpr uint128::uint128(long long v) // NOLINT(runtime/int) + : hi_{v < 0 ? (std::numeric_limits<uint64_t>::max)() : 0}, + lo_{static_cast<uint64_t>(v)} {} + +constexpr uint128::uint128(unsigned int v) : hi_{0}, lo_{v} {} +// NOLINTNEXTLINE(runtime/int) +constexpr uint128::uint128(unsigned long v) : hi_{0}, lo_{v} {} +// NOLINTNEXTLINE(runtime/int) +constexpr uint128::uint128(unsigned long long v) : hi_{0}, lo_{v} {} + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +constexpr uint128::uint128(__int128 v) + : hi_{static_cast<uint64_t>(static_cast<unsigned __int128>(v) >> 64)}, + lo_{static_cast<uint64_t>(v & ~uint64_t{0})} {} +constexpr uint128::uint128(unsigned __int128 v) + : hi_{static_cast<uint64_t>(v >> 64)}, + lo_{static_cast<uint64_t>(v & ~uint64_t{0})} {} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +constexpr uint128::uint128(int128 v) + : hi_{static_cast<uint64_t>(Int128High64(v))}, lo_{Int128Low64(v)} {} + +#else // byte order +#error "Unsupported byte order: must be little-endian or big-endian." +#endif // byte order + +// Conversion operators to integer types. + +constexpr uint128::operator bool() const { return lo_ || hi_; } + +constexpr uint128::operator char() const { return static_cast<char>(lo_); } + +constexpr uint128::operator signed char() const { + return static_cast<signed char>(lo_); +} + +constexpr uint128::operator unsigned char() const { + return static_cast<unsigned char>(lo_); +} + +constexpr uint128::operator char16_t() const { + return static_cast<char16_t>(lo_); +} + +constexpr uint128::operator char32_t() const { + return static_cast<char32_t>(lo_); +} + +constexpr uint128::operator ABSL_INTERNAL_WCHAR_T() const { + return static_cast<ABSL_INTERNAL_WCHAR_T>(lo_); +} + +// NOLINTNEXTLINE(runtime/int) +constexpr uint128::operator short() const { return static_cast<short>(lo_); } + +constexpr uint128::operator unsigned short() const { // NOLINT(runtime/int) + return static_cast<unsigned short>(lo_); // NOLINT(runtime/int) +} + +constexpr uint128::operator int() const { return static_cast<int>(lo_); } + +constexpr uint128::operator unsigned int() const { + return static_cast<unsigned int>(lo_); +} + +// NOLINTNEXTLINE(runtime/int) +constexpr uint128::operator long() const { return static_cast<long>(lo_); } + +constexpr uint128::operator unsigned long() const { // NOLINT(runtime/int) + return static_cast<unsigned long>(lo_); // NOLINT(runtime/int) +} + +constexpr uint128::operator long long() const { // NOLINT(runtime/int) + return static_cast<long long>(lo_); // NOLINT(runtime/int) +} + +constexpr uint128::operator unsigned long long() const { // NOLINT(runtime/int) + return static_cast<unsigned long long>(lo_); // NOLINT(runtime/int) +} + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +constexpr uint128::operator __int128() const { + return (static_cast<__int128>(hi_) << 64) + lo_; +} + +constexpr uint128::operator unsigned __int128() const { + return (static_cast<unsigned __int128>(hi_) << 64) + lo_; +} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +// Conversion operators to floating point types. + +inline uint128::operator float() const { + return static_cast<float>(lo_) + std::ldexp(static_cast<float>(hi_), 64); +} + +inline uint128::operator double() const { + return static_cast<double>(lo_) + std::ldexp(static_cast<double>(hi_), 64); +} + +inline uint128::operator long double() const { + return static_cast<long double>(lo_) + + std::ldexp(static_cast<long double>(hi_), 64); +} + +// Comparison operators. + +inline bool operator==(uint128 lhs, uint128 rhs) { + return (Uint128Low64(lhs) == Uint128Low64(rhs) && + Uint128High64(lhs) == Uint128High64(rhs)); +} + +inline bool operator!=(uint128 lhs, uint128 rhs) { + return !(lhs == rhs); +} + +inline bool operator<(uint128 lhs, uint128 rhs) { +#ifdef ABSL_HAVE_INTRINSIC_INT128 + return static_cast<unsigned __int128>(lhs) < + static_cast<unsigned __int128>(rhs); +#else + return (Uint128High64(lhs) == Uint128High64(rhs)) + ? (Uint128Low64(lhs) < Uint128Low64(rhs)) + : (Uint128High64(lhs) < Uint128High64(rhs)); +#endif +} + +inline bool operator>(uint128 lhs, uint128 rhs) { return rhs < lhs; } + +inline bool operator<=(uint128 lhs, uint128 rhs) { return !(rhs < lhs); } + +inline bool operator>=(uint128 lhs, uint128 rhs) { return !(lhs < rhs); } + +// Unary operators. + +inline uint128 operator-(uint128 val) { + uint64_t hi = ~Uint128High64(val); + uint64_t lo = ~Uint128Low64(val) + 1; + if (lo == 0) ++hi; // carry + return MakeUint128(hi, lo); +} + +inline bool operator!(uint128 val) { + return !Uint128High64(val) && !Uint128Low64(val); +} + +// Logical operators. + +inline uint128 operator~(uint128 val) { + return MakeUint128(~Uint128High64(val), ~Uint128Low64(val)); +} + +inline uint128 operator|(uint128 lhs, uint128 rhs) { + return MakeUint128(Uint128High64(lhs) | Uint128High64(rhs), + Uint128Low64(lhs) | Uint128Low64(rhs)); +} + +inline uint128 operator&(uint128 lhs, uint128 rhs) { + return MakeUint128(Uint128High64(lhs) & Uint128High64(rhs), + Uint128Low64(lhs) & Uint128Low64(rhs)); +} + +inline uint128 operator^(uint128 lhs, uint128 rhs) { + return MakeUint128(Uint128High64(lhs) ^ Uint128High64(rhs), + Uint128Low64(lhs) ^ Uint128Low64(rhs)); +} + +inline uint128& uint128::operator|=(uint128 other) { + hi_ |= other.hi_; + lo_ |= other.lo_; + return *this; +} + +inline uint128& uint128::operator&=(uint128 other) { + hi_ &= other.hi_; + lo_ &= other.lo_; + return *this; +} + +inline uint128& uint128::operator^=(uint128 other) { + hi_ ^= other.hi_; + lo_ ^= other.lo_; + return *this; +} + +// Arithmetic operators. + +inline uint128 operator<<(uint128 lhs, int amount) { +#ifdef ABSL_HAVE_INTRINSIC_INT128 + return static_cast<unsigned __int128>(lhs) << amount; +#else + // uint64_t shifts of >= 64 are undefined, so we will need some + // special-casing. + if (amount < 64) { + if (amount != 0) { + return MakeUint128( + (Uint128High64(lhs) << amount) | (Uint128Low64(lhs) >> (64 - amount)), + Uint128Low64(lhs) << amount); + } + return lhs; + } + return MakeUint128(Uint128Low64(lhs) << (amount - 64), 0); +#endif +} + +inline uint128 operator>>(uint128 lhs, int amount) { +#ifdef ABSL_HAVE_INTRINSIC_INT128 + return static_cast<unsigned __int128>(lhs) >> amount; +#else + // uint64_t shifts of >= 64 are undefined, so we will need some + // special-casing. + if (amount < 64) { + if (amount != 0) { + return MakeUint128(Uint128High64(lhs) >> amount, + (Uint128Low64(lhs) >> amount) | + (Uint128High64(lhs) << (64 - amount))); + } + return lhs; + } + return MakeUint128(0, Uint128High64(lhs) >> (amount - 64)); +#endif +} + +inline uint128 operator+(uint128 lhs, uint128 rhs) { + uint128 result = MakeUint128(Uint128High64(lhs) + Uint128High64(rhs), + Uint128Low64(lhs) + Uint128Low64(rhs)); + if (Uint128Low64(result) < Uint128Low64(lhs)) { // check for carry + return MakeUint128(Uint128High64(result) + 1, Uint128Low64(result)); + } + return result; +} + +inline uint128 operator-(uint128 lhs, uint128 rhs) { + uint128 result = MakeUint128(Uint128High64(lhs) - Uint128High64(rhs), + Uint128Low64(lhs) - Uint128Low64(rhs)); + if (Uint128Low64(lhs) < Uint128Low64(rhs)) { // check for carry + return MakeUint128(Uint128High64(result) - 1, Uint128Low64(result)); + } + return result; +} + +inline uint128 operator*(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + // TODO(strel) Remove once alignment issues are resolved and unsigned __int128 + // can be used for uint128 storage. + return static_cast<unsigned __int128>(lhs) * + static_cast<unsigned __int128>(rhs); +#elif defined(_MSC_VER) && defined(_M_X64) + uint64_t carry; + uint64_t low = _umul128(Uint128Low64(lhs), Uint128Low64(rhs), &carry); + return MakeUint128(Uint128Low64(lhs) * Uint128High64(rhs) + + Uint128High64(lhs) * Uint128Low64(rhs) + carry, + low); +#else // ABSL_HAVE_INTRINSIC128 + uint64_t a32 = Uint128Low64(lhs) >> 32; + uint64_t a00 = Uint128Low64(lhs) & 0xffffffff; + uint64_t b32 = Uint128Low64(rhs) >> 32; + uint64_t b00 = Uint128Low64(rhs) & 0xffffffff; + uint128 result = + MakeUint128(Uint128High64(lhs) * Uint128Low64(rhs) + + Uint128Low64(lhs) * Uint128High64(rhs) + a32 * b32, + a00 * b00); + result += uint128(a32 * b00) << 32; + result += uint128(a00 * b32) << 32; + return result; +#endif // ABSL_HAVE_INTRINSIC128 +} + +// Increment/decrement operators. + +inline uint128 uint128::operator++(int) { + uint128 tmp(*this); + *this += 1; + return tmp; +} + +inline uint128 uint128::operator--(int) { + uint128 tmp(*this); + *this -= 1; + return tmp; +} + +inline uint128& uint128::operator++() { + *this += 1; + return *this; +} + +inline uint128& uint128::operator--() { + *this -= 1; + return *this; +} + +constexpr int128 MakeInt128(int64_t high, uint64_t low) { + return int128(high, low); +} + +// Assignment from integer types. +inline int128& int128::operator=(int v) { + return *this = int128(v); +} + +inline int128& int128::operator=(unsigned int v) { + return *this = int128(v); +} + +inline int128& int128::operator=(long v) { // NOLINT(runtime/int) + return *this = int128(v); +} + +// NOLINTNEXTLINE(runtime/int) +inline int128& int128::operator=(unsigned long v) { + return *this = int128(v); +} + +// NOLINTNEXTLINE(runtime/int) +inline int128& int128::operator=(long long v) { + return *this = int128(v); +} + +// NOLINTNEXTLINE(runtime/int) +inline int128& int128::operator=(unsigned long long v) { + return *this = int128(v); +} + +// Arithmetic operators. + +int128 operator+(int128 lhs, int128 rhs); +int128 operator-(int128 lhs, int128 rhs); +int128 operator*(int128 lhs, int128 rhs); +int128 operator/(int128 lhs, int128 rhs); +int128 operator%(int128 lhs, int128 rhs); +int128 operator|(int128 lhs, int128 rhs); +int128 operator&(int128 lhs, int128 rhs); +int128 operator^(int128 lhs, int128 rhs); +int128 operator<<(int128 lhs, int amount); +int128 operator>>(int128 lhs, int amount); + +inline int128& int128::operator+=(int128 other) { + *this = *this + other; + return *this; +} + +inline int128& int128::operator-=(int128 other) { + *this = *this - other; + return *this; +} + +inline int128& int128::operator*=(int128 other) { + *this = *this * other; + return *this; +} + +inline int128& int128::operator/=(int128 other) { + *this = *this / other; + return *this; +} + +inline int128& int128::operator%=(int128 other) { + *this = *this % other; + return *this; +} + +inline int128& int128::operator|=(int128 other) { + *this = *this | other; + return *this; +} + +inline int128& int128::operator&=(int128 other) { + *this = *this & other; + return *this; +} + +inline int128& int128::operator^=(int128 other) { + *this = *this ^ other; + return *this; +} + +inline int128& int128::operator<<=(int amount) { + *this = *this << amount; + return *this; +} + +inline int128& int128::operator>>=(int amount) { + *this = *this >> amount; + return *this; +} + +namespace int128_internal { + +// Casts from unsigned to signed while preserving the underlying binary +// representation. +constexpr int64_t BitCastToSigned(uint64_t v) { + // Casting an unsigned integer to a signed integer of the same + // width is implementation defined behavior if the source value would not fit + // in the destination type. We step around it with a roundtrip bitwise not + // operation to make sure this function remains constexpr. Clang, GCC, and + // MSVC optimize this to a no-op on x86-64. + return v & (uint64_t{1} << 63) ? ~static_cast<int64_t>(~v) + : static_cast<int64_t>(v); +} + +} // namespace int128_internal + +#if defined(ABSL_HAVE_INTRINSIC_INT128) +#include "absl/numeric/int128_have_intrinsic.inc" // IWYU pragma: export +#else // ABSL_HAVE_INTRINSIC_INT128 +#include "absl/numeric/int128_no_intrinsic.inc" // IWYU pragma: export +#endif // ABSL_HAVE_INTRINSIC_INT128 + +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_INTERNAL_WCHAR_T + +#endif // ABSL_NUMERIC_INT128_H_ diff --git a/third_party/abseil_cpp/absl/numeric/int128_benchmark.cc b/third_party/abseil_cpp/absl/numeric/int128_benchmark.cc new file mode 100644 index 000000000000..eab1515c0ad8 --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128_benchmark.cc @@ -0,0 +1,282 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <algorithm> +#include <cstdint> +#include <limits> +#include <random> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/config.h" +#include "absl/numeric/int128.h" + +namespace { + +constexpr size_t kSampleSize = 1000000; + +std::mt19937 MakeRandomEngine() { + std::random_device r; + std::seed_seq seed({r(), r(), r(), r(), r(), r(), r(), r()}); + return std::mt19937(seed); +} + +template <typename T, + typename H = typename std::conditional< + std::numeric_limits<T>::is_signed, int64_t, uint64_t>::type> +std::vector<std::pair<T, T>> GetRandomClass128SampleUniformDivisor() { + std::vector<std::pair<T, T>> values; + std::mt19937 random = MakeRandomEngine(); + std::uniform_int_distribution<H> uniform_h; + values.reserve(kSampleSize); + for (size_t i = 0; i < kSampleSize; ++i) { + T a{absl::MakeUint128(uniform_h(random), uniform_h(random))}; + T b{absl::MakeUint128(uniform_h(random), uniform_h(random))}; + values.emplace_back(std::max(a, b), std::max(T(2), std::min(a, b))); + } + return values; +} + +template <typename T> +void BM_DivideClass128UniformDivisor(benchmark::State& state) { + auto values = GetRandomClass128SampleUniformDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first / pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_DivideClass128UniformDivisor, absl::uint128); +BENCHMARK_TEMPLATE(BM_DivideClass128UniformDivisor, absl::int128); + +template <typename T> +void BM_RemainderClass128UniformDivisor(benchmark::State& state) { + auto values = GetRandomClass128SampleUniformDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first % pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_RemainderClass128UniformDivisor, absl::uint128); +BENCHMARK_TEMPLATE(BM_RemainderClass128UniformDivisor, absl::int128); + +template <typename T, + typename H = typename std::conditional< + std::numeric_limits<T>::is_signed, int64_t, uint64_t>::type> +std::vector<std::pair<T, H>> GetRandomClass128SampleSmallDivisor() { + std::vector<std::pair<T, H>> values; + std::mt19937 random = MakeRandomEngine(); + std::uniform_int_distribution<H> uniform_h; + values.reserve(kSampleSize); + for (size_t i = 0; i < kSampleSize; ++i) { + T a{absl::MakeUint128(uniform_h(random), uniform_h(random))}; + H b{std::max(H{2}, uniform_h(random))}; + values.emplace_back(std::max(a, T(b)), b); + } + return values; +} + +template <typename T> +void BM_DivideClass128SmallDivisor(benchmark::State& state) { + auto values = GetRandomClass128SampleSmallDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first / pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_DivideClass128SmallDivisor, absl::uint128); +BENCHMARK_TEMPLATE(BM_DivideClass128SmallDivisor, absl::int128); + +template <typename T> +void BM_RemainderClass128SmallDivisor(benchmark::State& state) { + auto values = GetRandomClass128SampleSmallDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first % pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_RemainderClass128SmallDivisor, absl::uint128); +BENCHMARK_TEMPLATE(BM_RemainderClass128SmallDivisor, absl::int128); + +std::vector<std::pair<absl::uint128, absl::uint128>> GetRandomClass128Sample() { + std::vector<std::pair<absl::uint128, absl::uint128>> values; + std::mt19937 random = MakeRandomEngine(); + std::uniform_int_distribution<uint64_t> uniform_uint64; + values.reserve(kSampleSize); + for (size_t i = 0; i < kSampleSize; ++i) { + values.emplace_back( + absl::MakeUint128(uniform_uint64(random), uniform_uint64(random)), + absl::MakeUint128(uniform_uint64(random), uniform_uint64(random))); + } + return values; +} + +void BM_MultiplyClass128(benchmark::State& state) { + auto values = GetRandomClass128Sample(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first * pair.second); + } + } +} +BENCHMARK(BM_MultiplyClass128); + +void BM_AddClass128(benchmark::State& state) { + auto values = GetRandomClass128Sample(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first + pair.second); + } + } +} +BENCHMARK(BM_AddClass128); + +#ifdef ABSL_HAVE_INTRINSIC_INT128 + +// Some implementations of <random> do not support __int128 when it is +// available, so we make our own uniform_int_distribution-like type. +template <typename T, + typename H = typename std::conditional< + std::is_same<T, __int128>::value, int64_t, uint64_t>::type> +class UniformIntDistribution128 { + public: + // NOLINTNEXTLINE: mimicking std::uniform_int_distribution API + T operator()(std::mt19937& generator) { + return (static_cast<T>(dist64_(generator)) << 64) | dist64_(generator); + } + + private: + std::uniform_int_distribution<H> dist64_; +}; + +template <typename T, + typename H = typename std::conditional< + std::is_same<T, __int128>::value, int64_t, uint64_t>::type> +std::vector<std::pair<T, T>> GetRandomIntrinsic128SampleUniformDivisor() { + std::vector<std::pair<T, T>> values; + std::mt19937 random = MakeRandomEngine(); + UniformIntDistribution128<T> uniform_128; + values.reserve(kSampleSize); + for (size_t i = 0; i < kSampleSize; ++i) { + T a = uniform_128(random); + T b = uniform_128(random); + values.emplace_back(std::max(a, b), + std::max(static_cast<T>(2), std::min(a, b))); + } + return values; +} + +template <typename T> +void BM_DivideIntrinsic128UniformDivisor(benchmark::State& state) { + auto values = GetRandomIntrinsic128SampleUniformDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first / pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_DivideIntrinsic128UniformDivisor, unsigned __int128); +BENCHMARK_TEMPLATE(BM_DivideIntrinsic128UniformDivisor, __int128); + +template <typename T> +void BM_RemainderIntrinsic128UniformDivisor(benchmark::State& state) { + auto values = GetRandomIntrinsic128SampleUniformDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first % pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128UniformDivisor, unsigned __int128); +BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128UniformDivisor, __int128); + +template <typename T, + typename H = typename std::conditional< + std::is_same<T, __int128>::value, int64_t, uint64_t>::type> +std::vector<std::pair<T, H>> GetRandomIntrinsic128SampleSmallDivisor() { + std::vector<std::pair<T, H>> values; + std::mt19937 random = MakeRandomEngine(); + UniformIntDistribution128<T> uniform_int128; + std::uniform_int_distribution<H> uniform_int64; + values.reserve(kSampleSize); + for (size_t i = 0; i < kSampleSize; ++i) { + T a = uniform_int128(random); + H b = std::max(H{2}, uniform_int64(random)); + values.emplace_back(std::max(a, static_cast<T>(b)), b); + } + return values; +} + +template <typename T> +void BM_DivideIntrinsic128SmallDivisor(benchmark::State& state) { + auto values = GetRandomIntrinsic128SampleSmallDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first / pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_DivideIntrinsic128SmallDivisor, unsigned __int128); +BENCHMARK_TEMPLATE(BM_DivideIntrinsic128SmallDivisor, __int128); + +template <typename T> +void BM_RemainderIntrinsic128SmallDivisor(benchmark::State& state) { + auto values = GetRandomIntrinsic128SampleSmallDivisor<T>(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first % pair.second); + } + } +} +BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128SmallDivisor, unsigned __int128); +BENCHMARK_TEMPLATE(BM_RemainderIntrinsic128SmallDivisor, __int128); + +std::vector<std::pair<unsigned __int128, unsigned __int128>> + GetRandomIntrinsic128Sample() { + std::vector<std::pair<unsigned __int128, unsigned __int128>> values; + std::mt19937 random = MakeRandomEngine(); + UniformIntDistribution128<unsigned __int128> uniform_uint128; + values.reserve(kSampleSize); + for (size_t i = 0; i < kSampleSize; ++i) { + values.emplace_back(uniform_uint128(random), uniform_uint128(random)); + } + return values; +} + +void BM_MultiplyIntrinsic128(benchmark::State& state) { + auto values = GetRandomIntrinsic128Sample(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first * pair.second); + } + } +} +BENCHMARK(BM_MultiplyIntrinsic128); + +void BM_AddIntrinsic128(benchmark::State& state) { + auto values = GetRandomIntrinsic128Sample(); + while (state.KeepRunningBatch(values.size())) { + for (const auto& pair : values) { + benchmark::DoNotOptimize(pair.first + pair.second); + } + } +} +BENCHMARK(BM_AddIntrinsic128); + +#endif // ABSL_HAVE_INTRINSIC_INT128 + +} // namespace diff --git a/third_party/abseil_cpp/absl/numeric/int128_have_intrinsic.inc b/third_party/abseil_cpp/absl/numeric/int128_have_intrinsic.inc new file mode 100644 index 000000000000..d6c76dd320ce --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128_have_intrinsic.inc @@ -0,0 +1,302 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains :int128 implementation details that depend on internal +// representation when ABSL_HAVE_INTRINSIC_INT128 is defined. This file is +// included by int128.h and relies on ABSL_INTERNAL_WCHAR_T being defined. + +namespace int128_internal { + +// Casts from unsigned to signed while preserving the underlying binary +// representation. +constexpr __int128 BitCastToSigned(unsigned __int128 v) { + // Casting an unsigned integer to a signed integer of the same + // width is implementation defined behavior if the source value would not fit + // in the destination type. We step around it with a roundtrip bitwise not + // operation to make sure this function remains constexpr. Clang and GCC + // optimize this to a no-op on x86-64. + return v & (static_cast<unsigned __int128>(1) << 127) + ? ~static_cast<__int128>(~v) + : static_cast<__int128>(v); +} + +} // namespace int128_internal + +inline int128& int128::operator=(__int128 v) { + v_ = v; + return *this; +} + +constexpr uint64_t Int128Low64(int128 v) { + return static_cast<uint64_t>(v.v_ & ~uint64_t{0}); +} + +constexpr int64_t Int128High64(int128 v) { + // Initially cast to unsigned to prevent a right shift on a negative value. + return int128_internal::BitCastToSigned( + static_cast<uint64_t>(static_cast<unsigned __int128>(v.v_) >> 64)); +} + +constexpr int128::int128(int64_t high, uint64_t low) + // Initially cast to unsigned to prevent a left shift that overflows. + : v_(int128_internal::BitCastToSigned(static_cast<unsigned __int128>(high) + << 64) | + low) {} + + +constexpr int128::int128(int v) : v_{v} {} + +constexpr int128::int128(long v) : v_{v} {} // NOLINT(runtime/int) + +constexpr int128::int128(long long v) : v_{v} {} // NOLINT(runtime/int) + +constexpr int128::int128(__int128 v) : v_{v} {} + +constexpr int128::int128(unsigned int v) : v_{v} {} + +constexpr int128::int128(unsigned long v) : v_{v} {} // NOLINT(runtime/int) + +// NOLINTNEXTLINE(runtime/int) +constexpr int128::int128(unsigned long long v) : v_{v} {} + +constexpr int128::int128(unsigned __int128 v) : v_{static_cast<__int128>(v)} {} + +inline int128::int128(float v) { + v_ = static_cast<__int128>(v); +} + +inline int128::int128(double v) { + v_ = static_cast<__int128>(v); +} + +inline int128::int128(long double v) { + v_ = static_cast<__int128>(v); +} + +constexpr int128::int128(uint128 v) : v_{static_cast<__int128>(v)} {} + +constexpr int128::operator bool() const { return static_cast<bool>(v_); } + +constexpr int128::operator char() const { return static_cast<char>(v_); } + +constexpr int128::operator signed char() const { + return static_cast<signed char>(v_); +} + +constexpr int128::operator unsigned char() const { + return static_cast<unsigned char>(v_); +} + +constexpr int128::operator char16_t() const { + return static_cast<char16_t>(v_); +} + +constexpr int128::operator char32_t() const { + return static_cast<char32_t>(v_); +} + +constexpr int128::operator ABSL_INTERNAL_WCHAR_T() const { + return static_cast<ABSL_INTERNAL_WCHAR_T>(v_); +} + +constexpr int128::operator short() const { // NOLINT(runtime/int) + return static_cast<short>(v_); // NOLINT(runtime/int) +} + +constexpr int128::operator unsigned short() const { // NOLINT(runtime/int) + return static_cast<unsigned short>(v_); // NOLINT(runtime/int) +} + +constexpr int128::operator int() const { + return static_cast<int>(v_); +} + +constexpr int128::operator unsigned int() const { + return static_cast<unsigned int>(v_); +} + +constexpr int128::operator long() const { // NOLINT(runtime/int) + return static_cast<long>(v_); // NOLINT(runtime/int) +} + +constexpr int128::operator unsigned long() const { // NOLINT(runtime/int) + return static_cast<unsigned long>(v_); // NOLINT(runtime/int) +} + +constexpr int128::operator long long() const { // NOLINT(runtime/int) + return static_cast<long long>(v_); // NOLINT(runtime/int) +} + +constexpr int128::operator unsigned long long() const { // NOLINT(runtime/int) + return static_cast<unsigned long long>(v_); // NOLINT(runtime/int) +} + +constexpr int128::operator __int128() const { return v_; } + +constexpr int128::operator unsigned __int128() const { + return static_cast<unsigned __int128>(v_); +} + +// Clang on PowerPC sometimes produces incorrect __int128 to floating point +// conversions. In that case, we do the conversion with a similar implementation +// to the conversion operators in int128_no_intrinsic.inc. +#if defined(__clang__) && !defined(__ppc64__) +inline int128::operator float() const { return static_cast<float>(v_); } + +inline int128::operator double () const { return static_cast<double>(v_); } + +inline int128::operator long double() const { + return static_cast<long double>(v_); +} + +#else // Clang on PowerPC +// Forward declaration for conversion operators to floating point types. +int128 operator-(int128 v); +bool operator!=(int128 lhs, int128 rhs); + +inline int128::operator float() const { + // We must convert the absolute value and then negate as needed, because + // floating point types are typically sign-magnitude. Otherwise, the + // difference between the high and low 64 bits when interpreted as two's + // complement overwhelms the precision of the mantissa. + // + // Also check to make sure we don't negate Int128Min() + return v_ < 0 && *this != Int128Min() + ? -static_cast<float>(-*this) + : static_cast<float>(Int128Low64(*this)) + + std::ldexp(static_cast<float>(Int128High64(*this)), 64); +} + +inline int128::operator double() const { + // See comment in int128::operator float() above. + return v_ < 0 && *this != Int128Min() + ? -static_cast<double>(-*this) + : static_cast<double>(Int128Low64(*this)) + + std::ldexp(static_cast<double>(Int128High64(*this)), 64); +} + +inline int128::operator long double() const { + // See comment in int128::operator float() above. + return v_ < 0 && *this != Int128Min() + ? -static_cast<long double>(-*this) + : static_cast<long double>(Int128Low64(*this)) + + std::ldexp(static_cast<long double>(Int128High64(*this)), + 64); +} +#endif // Clang on PowerPC + +// Comparison operators. + +inline bool operator==(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) == static_cast<__int128>(rhs); +} + +inline bool operator!=(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) != static_cast<__int128>(rhs); +} + +inline bool operator<(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) < static_cast<__int128>(rhs); +} + +inline bool operator>(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) > static_cast<__int128>(rhs); +} + +inline bool operator<=(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) <= static_cast<__int128>(rhs); +} + +inline bool operator>=(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) >= static_cast<__int128>(rhs); +} + +// Unary operators. + +inline int128 operator-(int128 v) { + return -static_cast<__int128>(v); +} + +inline bool operator!(int128 v) { + return !static_cast<__int128>(v); +} + +inline int128 operator~(int128 val) { + return ~static_cast<__int128>(val); +} + +// Arithmetic operators. + +inline int128 operator+(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) + static_cast<__int128>(rhs); +} + +inline int128 operator-(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) - static_cast<__int128>(rhs); +} + +inline int128 operator*(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) * static_cast<__int128>(rhs); +} + +inline int128 operator/(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) / static_cast<__int128>(rhs); +} + +inline int128 operator%(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) % static_cast<__int128>(rhs); +} + +inline int128 int128::operator++(int) { + int128 tmp(*this); + ++v_; + return tmp; +} + +inline int128 int128::operator--(int) { + int128 tmp(*this); + --v_; + return tmp; +} + +inline int128& int128::operator++() { + ++v_; + return *this; +} + +inline int128& int128::operator--() { + --v_; + return *this; +} + +inline int128 operator|(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) | static_cast<__int128>(rhs); +} + +inline int128 operator&(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) & static_cast<__int128>(rhs); +} + +inline int128 operator^(int128 lhs, int128 rhs) { + return static_cast<__int128>(lhs) ^ static_cast<__int128>(rhs); +} + +inline int128 operator<<(int128 lhs, int amount) { + return static_cast<__int128>(lhs) << amount; +} + +inline int128 operator>>(int128 lhs, int amount) { + return static_cast<__int128>(lhs) >> amount; +} diff --git a/third_party/abseil_cpp/absl/numeric/int128_no_intrinsic.inc b/third_party/abseil_cpp/absl/numeric/int128_no_intrinsic.inc new file mode 100644 index 000000000000..c753771ae73a --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128_no_intrinsic.inc @@ -0,0 +1,308 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains :int128 implementation details that depend on internal +// representation when ABSL_HAVE_INTRINSIC_INT128 is *not* defined. This file +// is included by int128.h and relies on ABSL_INTERNAL_WCHAR_T being defined. + +constexpr uint64_t Int128Low64(int128 v) { return v.lo_; } + +constexpr int64_t Int128High64(int128 v) { return v.hi_; } + +#if defined(ABSL_IS_LITTLE_ENDIAN) + +constexpr int128::int128(int64_t high, uint64_t low) : + lo_(low), hi_(high) {} + +constexpr int128::int128(int v) + : lo_{static_cast<uint64_t>(v)}, hi_{v < 0 ? ~int64_t{0} : 0} {} +constexpr int128::int128(long v) // NOLINT(runtime/int) + : lo_{static_cast<uint64_t>(v)}, hi_{v < 0 ? ~int64_t{0} : 0} {} +constexpr int128::int128(long long v) // NOLINT(runtime/int) + : lo_{static_cast<uint64_t>(v)}, hi_{v < 0 ? ~int64_t{0} : 0} {} + +constexpr int128::int128(unsigned int v) : lo_{v}, hi_{0} {} +// NOLINTNEXTLINE(runtime/int) +constexpr int128::int128(unsigned long v) : lo_{v}, hi_{0} {} +// NOLINTNEXTLINE(runtime/int) +constexpr int128::int128(unsigned long long v) : lo_{v}, hi_{0} {} + +constexpr int128::int128(uint128 v) + : lo_{Uint128Low64(v)}, hi_{static_cast<int64_t>(Uint128High64(v))} {} + +#elif defined(ABSL_IS_BIG_ENDIAN) + +constexpr int128::int128(int64_t high, uint64_t low) : + hi_{high}, lo_{low} {} + +constexpr int128::int128(int v) + : hi_{v < 0 ? ~int64_t{0} : 0}, lo_{static_cast<uint64_t>(v)} {} +constexpr int128::int128(long v) // NOLINT(runtime/int) + : hi_{v < 0 ? ~int64_t{0} : 0}, lo_{static_cast<uint64_t>(v)} {} +constexpr int128::int128(long long v) // NOLINT(runtime/int) + : hi_{v < 0 ? ~int64_t{0} : 0}, lo_{static_cast<uint64_t>(v)} {} + +constexpr int128::int128(unsigned int v) : hi_{0}, lo_{v} {} +// NOLINTNEXTLINE(runtime/int) +constexpr int128::int128(unsigned long v) : hi_{0}, lo_{v} {} +// NOLINTNEXTLINE(runtime/int) +constexpr int128::int128(unsigned long long v) : hi_{0}, lo_{v} {} + +constexpr int128::int128(uint128 v) + : hi_{static_cast<int64_t>(Uint128High64(v))}, lo_{Uint128Low64(v)} {} + +#else // byte order +#error "Unsupported byte order: must be little-endian or big-endian." +#endif // byte order + +constexpr int128::operator bool() const { return lo_ || hi_; } + +constexpr int128::operator char() const { + // NOLINTNEXTLINE(runtime/int) + return static_cast<char>(static_cast<long long>(*this)); +} + +constexpr int128::operator signed char() const { + // NOLINTNEXTLINE(runtime/int) + return static_cast<signed char>(static_cast<long long>(*this)); +} + +constexpr int128::operator unsigned char() const { + return static_cast<unsigned char>(lo_); +} + +constexpr int128::operator char16_t() const { + return static_cast<char16_t>(lo_); +} + +constexpr int128::operator char32_t() const { + return static_cast<char32_t>(lo_); +} + +constexpr int128::operator ABSL_INTERNAL_WCHAR_T() const { + // NOLINTNEXTLINE(runtime/int) + return static_cast<ABSL_INTERNAL_WCHAR_T>(static_cast<long long>(*this)); +} + +constexpr int128::operator short() const { // NOLINT(runtime/int) + // NOLINTNEXTLINE(runtime/int) + return static_cast<short>(static_cast<long long>(*this)); +} + +constexpr int128::operator unsigned short() const { // NOLINT(runtime/int) + return static_cast<unsigned short>(lo_); // NOLINT(runtime/int) +} + +constexpr int128::operator int() const { + // NOLINTNEXTLINE(runtime/int) + return static_cast<int>(static_cast<long long>(*this)); +} + +constexpr int128::operator unsigned int() const { + return static_cast<unsigned int>(lo_); +} + +constexpr int128::operator long() const { // NOLINT(runtime/int) + // NOLINTNEXTLINE(runtime/int) + return static_cast<long>(static_cast<long long>(*this)); +} + +constexpr int128::operator unsigned long() const { // NOLINT(runtime/int) + return static_cast<unsigned long>(lo_); // NOLINT(runtime/int) +} + +constexpr int128::operator long long() const { // NOLINT(runtime/int) + // We don't bother checking the value of hi_. If *this < 0, lo_'s high bit + // must be set in order for the value to fit into a long long. Conversely, if + // lo_'s high bit is set, *this must be < 0 for the value to fit. + return int128_internal::BitCastToSigned(lo_); +} + +constexpr int128::operator unsigned long long() const { // NOLINT(runtime/int) + return static_cast<unsigned long long>(lo_); // NOLINT(runtime/int) +} + +// Forward declaration for conversion operators to floating point types. +int128 operator-(int128 v); +bool operator!=(int128 lhs, int128 rhs); + +inline int128::operator float() const { + // We must convert the absolute value and then negate as needed, because + // floating point types are typically sign-magnitude. Otherwise, the + // difference between the high and low 64 bits when interpreted as two's + // complement overwhelms the precision of the mantissa. + // + // Also check to make sure we don't negate Int128Min() + return hi_ < 0 && *this != Int128Min() + ? -static_cast<float>(-*this) + : static_cast<float>(lo_) + + std::ldexp(static_cast<float>(hi_), 64); +} + +inline int128::operator double() const { + // See comment in int128::operator float() above. + return hi_ < 0 && *this != Int128Min() + ? -static_cast<double>(-*this) + : static_cast<double>(lo_) + + std::ldexp(static_cast<double>(hi_), 64); +} + +inline int128::operator long double() const { + // See comment in int128::operator float() above. + return hi_ < 0 && *this != Int128Min() + ? -static_cast<long double>(-*this) + : static_cast<long double>(lo_) + + std::ldexp(static_cast<long double>(hi_), 64); +} + +// Comparison operators. + +inline bool operator==(int128 lhs, int128 rhs) { + return (Int128Low64(lhs) == Int128Low64(rhs) && + Int128High64(lhs) == Int128High64(rhs)); +} + +inline bool operator!=(int128 lhs, int128 rhs) { + return !(lhs == rhs); +} + +inline bool operator<(int128 lhs, int128 rhs) { + return (Int128High64(lhs) == Int128High64(rhs)) + ? (Int128Low64(lhs) < Int128Low64(rhs)) + : (Int128High64(lhs) < Int128High64(rhs)); +} + +inline bool operator>(int128 lhs, int128 rhs) { + return (Int128High64(lhs) == Int128High64(rhs)) + ? (Int128Low64(lhs) > Int128Low64(rhs)) + : (Int128High64(lhs) > Int128High64(rhs)); +} + +inline bool operator<=(int128 lhs, int128 rhs) { + return !(lhs > rhs); +} + +inline bool operator>=(int128 lhs, int128 rhs) { + return !(lhs < rhs); +} + +// Unary operators. + +inline int128 operator-(int128 v) { + int64_t hi = ~Int128High64(v); + uint64_t lo = ~Int128Low64(v) + 1; + if (lo == 0) ++hi; // carry + return MakeInt128(hi, lo); +} + +inline bool operator!(int128 v) { + return !Int128Low64(v) && !Int128High64(v); +} + +inline int128 operator~(int128 val) { + return MakeInt128(~Int128High64(val), ~Int128Low64(val)); +} + +// Arithmetic operators. + +inline int128 operator+(int128 lhs, int128 rhs) { + int128 result = MakeInt128(Int128High64(lhs) + Int128High64(rhs), + Int128Low64(lhs) + Int128Low64(rhs)); + if (Int128Low64(result) < Int128Low64(lhs)) { // check for carry + return MakeInt128(Int128High64(result) + 1, Int128Low64(result)); + } + return result; +} + +inline int128 operator-(int128 lhs, int128 rhs) { + int128 result = MakeInt128(Int128High64(lhs) - Int128High64(rhs), + Int128Low64(lhs) - Int128Low64(rhs)); + if (Int128Low64(lhs) < Int128Low64(rhs)) { // check for carry + return MakeInt128(Int128High64(result) - 1, Int128Low64(result)); + } + return result; +} + +inline int128 operator*(int128 lhs, int128 rhs) { + uint128 result = uint128(lhs) * rhs; + return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(result)), + Uint128Low64(result)); +} + +inline int128 int128::operator++(int) { + int128 tmp(*this); + *this += 1; + return tmp; +} + +inline int128 int128::operator--(int) { + int128 tmp(*this); + *this -= 1; + return tmp; +} + +inline int128& int128::operator++() { + *this += 1; + return *this; +} + +inline int128& int128::operator--() { + *this -= 1; + return *this; +} + +inline int128 operator|(int128 lhs, int128 rhs) { + return MakeInt128(Int128High64(lhs) | Int128High64(rhs), + Int128Low64(lhs) | Int128Low64(rhs)); +} + +inline int128 operator&(int128 lhs, int128 rhs) { + return MakeInt128(Int128High64(lhs) & Int128High64(rhs), + Int128Low64(lhs) & Int128Low64(rhs)); +} + +inline int128 operator^(int128 lhs, int128 rhs) { + return MakeInt128(Int128High64(lhs) ^ Int128High64(rhs), + Int128Low64(lhs) ^ Int128Low64(rhs)); +} + +inline int128 operator<<(int128 lhs, int amount) { + // uint64_t shifts of >= 64 are undefined, so we need some special-casing. + if (amount < 64) { + if (amount != 0) { + return MakeInt128( + (Int128High64(lhs) << amount) | + static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)), + Int128Low64(lhs) << amount); + } + return lhs; + } + return MakeInt128(static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0); +} + +inline int128 operator>>(int128 lhs, int amount) { + // uint64_t shifts of >= 64 are undefined, so we need some special-casing. + if (amount < 64) { + if (amount != 0) { + return MakeInt128( + Int128High64(lhs) >> amount, + (Int128Low64(lhs) >> amount) | + (static_cast<uint64_t>(Int128High64(lhs)) << (64 - amount))); + } + return lhs; + } + return MakeInt128(0, + static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64))); +} diff --git a/third_party/abseil_cpp/absl/numeric/int128_stream_test.cc b/third_party/abseil_cpp/absl/numeric/int128_stream_test.cc new file mode 100644 index 000000000000..479ad66cf4d5 --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128_stream_test.cc @@ -0,0 +1,1395 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/numeric/int128.h" + +#include <sstream> +#include <string> + +#include "gtest/gtest.h" + +namespace { + +struct Uint128TestCase { + absl::uint128 value; + std::ios_base::fmtflags flags; + std::streamsize width; + const char* expected; +}; + +constexpr char kFill = '_'; + +std::string StreamFormatToString(std::ios_base::fmtflags flags, + std::streamsize width) { + std::vector<const char*> flagstr; + switch (flags & std::ios::basefield) { + case std::ios::dec: + flagstr.push_back("std::ios::dec"); + break; + case std::ios::oct: + flagstr.push_back("std::ios::oct"); + break; + case std::ios::hex: + flagstr.push_back("std::ios::hex"); + break; + default: // basefield not specified + break; + } + switch (flags & std::ios::adjustfield) { + case std::ios::left: + flagstr.push_back("std::ios::left"); + break; + case std::ios::internal: + flagstr.push_back("std::ios::internal"); + break; + case std::ios::right: + flagstr.push_back("std::ios::right"); + break; + default: // adjustfield not specified + break; + } + if (flags & std::ios::uppercase) flagstr.push_back("std::ios::uppercase"); + if (flags & std::ios::showbase) flagstr.push_back("std::ios::showbase"); + if (flags & std::ios::showpos) flagstr.push_back("std::ios::showpos"); + + std::ostringstream msg; + msg << "\n StreamFormatToString(test_case.flags, test_case.width)\n " + "flags: "; + if (!flagstr.empty()) { + for (size_t i = 0; i < flagstr.size() - 1; ++i) msg << flagstr[i] << " | "; + msg << flagstr.back(); + } else { + msg << "(default)"; + } + msg << "\n width: " << width << "\n fill: '" << kFill << "'"; + return msg.str(); +} + +void CheckUint128Case(const Uint128TestCase& test_case) { + std::ostringstream os; + os.flags(test_case.flags); + os.width(test_case.width); + os.fill(kFill); + os << test_case.value; + SCOPED_TRACE(StreamFormatToString(test_case.flags, test_case.width)); + EXPECT_EQ(test_case.expected, os.str()); +} + +constexpr std::ios::fmtflags kDec = std::ios::dec; +constexpr std::ios::fmtflags kOct = std::ios::oct; +constexpr std::ios::fmtflags kHex = std::ios::hex; +constexpr std::ios::fmtflags kLeft = std::ios::left; +constexpr std::ios::fmtflags kInt = std::ios::internal; +constexpr std::ios::fmtflags kRight = std::ios::right; +constexpr std::ios::fmtflags kUpper = std::ios::uppercase; +constexpr std::ios::fmtflags kBase = std::ios::showbase; +constexpr std::ios::fmtflags kPos = std::ios::showpos; + +TEST(Uint128, OStreamValueTest) { + CheckUint128Case({1, kDec, /*width = */ 0, "1"}); + CheckUint128Case({1, kOct, /*width = */ 0, "1"}); + CheckUint128Case({1, kHex, /*width = */ 0, "1"}); + CheckUint128Case({9, kDec, /*width = */ 0, "9"}); + CheckUint128Case({9, kOct, /*width = */ 0, "11"}); + CheckUint128Case({9, kHex, /*width = */ 0, "9"}); + CheckUint128Case({12345, kDec, /*width = */ 0, "12345"}); + CheckUint128Case({12345, kOct, /*width = */ 0, "30071"}); + CheckUint128Case({12345, kHex, /*width = */ 0, "3039"}); + CheckUint128Case( + {0x8000000000000000, kDec, /*width = */ 0, "9223372036854775808"}); + CheckUint128Case( + {0x8000000000000000, kOct, /*width = */ 0, "1000000000000000000000"}); + CheckUint128Case( + {0x8000000000000000, kHex, /*width = */ 0, "8000000000000000"}); + CheckUint128Case({std::numeric_limits<uint64_t>::max(), kDec, + /*width = */ 0, "18446744073709551615"}); + CheckUint128Case({std::numeric_limits<uint64_t>::max(), kOct, + /*width = */ 0, "1777777777777777777777"}); + CheckUint128Case({std::numeric_limits<uint64_t>::max(), kHex, + /*width = */ 0, "ffffffffffffffff"}); + CheckUint128Case( + {absl::MakeUint128(1, 0), kDec, /*width = */ 0, "18446744073709551616"}); + CheckUint128Case({absl::MakeUint128(1, 0), kOct, /*width = */ 0, + "2000000000000000000000"}); + CheckUint128Case( + {absl::MakeUint128(1, 0), kHex, /*width = */ 0, "10000000000000000"}); + CheckUint128Case({absl::MakeUint128(0x8000000000000000, 0), kDec, + /*width = */ 0, "170141183460469231731687303715884105728"}); + CheckUint128Case({absl::MakeUint128(0x8000000000000000, 0), kOct, + /*width = */ 0, + "2000000000000000000000000000000000000000000"}); + CheckUint128Case({absl::MakeUint128(0x8000000000000000, 0), kHex, + /*width = */ 0, "80000000000000000000000000000000"}); + CheckUint128Case({absl::kuint128max, kDec, /*width = */ 0, + "340282366920938463463374607431768211455"}); + CheckUint128Case({absl::kuint128max, kOct, /*width = */ 0, + "3777777777777777777777777777777777777777777"}); + CheckUint128Case({absl::kuint128max, kHex, /*width = */ 0, + "ffffffffffffffffffffffffffffffff"}); +} + +std::vector<Uint128TestCase> GetUint128FormatCases(); + +TEST(Uint128, OStreamFormatTest) { + for (const Uint128TestCase& test_case : GetUint128FormatCases()) { + CheckUint128Case(test_case); + } +} + +struct Int128TestCase { + absl::int128 value; + std::ios_base::fmtflags flags; + std::streamsize width; + const char* expected; +}; + +void CheckInt128Case(const Int128TestCase& test_case) { + std::ostringstream os; + os.flags(test_case.flags); + os.width(test_case.width); + os.fill(kFill); + os << test_case.value; + SCOPED_TRACE(StreamFormatToString(test_case.flags, test_case.width)); + EXPECT_EQ(test_case.expected, os.str()); +} + +TEST(Int128, OStreamValueTest) { + CheckInt128Case({1, kDec, /*width = */ 0, "1"}); + CheckInt128Case({1, kOct, /*width = */ 0, "1"}); + CheckInt128Case({1, kHex, /*width = */ 0, "1"}); + CheckInt128Case({9, kDec, /*width = */ 0, "9"}); + CheckInt128Case({9, kOct, /*width = */ 0, "11"}); + CheckInt128Case({9, kHex, /*width = */ 0, "9"}); + CheckInt128Case({12345, kDec, /*width = */ 0, "12345"}); + CheckInt128Case({12345, kOct, /*width = */ 0, "30071"}); + CheckInt128Case({12345, kHex, /*width = */ 0, "3039"}); + CheckInt128Case( + {0x8000000000000000, kDec, /*width = */ 0, "9223372036854775808"}); + CheckInt128Case( + {0x8000000000000000, kOct, /*width = */ 0, "1000000000000000000000"}); + CheckInt128Case( + {0x8000000000000000, kHex, /*width = */ 0, "8000000000000000"}); + CheckInt128Case({std::numeric_limits<uint64_t>::max(), kDec, + /*width = */ 0, "18446744073709551615"}); + CheckInt128Case({std::numeric_limits<uint64_t>::max(), kOct, + /*width = */ 0, "1777777777777777777777"}); + CheckInt128Case({std::numeric_limits<uint64_t>::max(), kHex, + /*width = */ 0, "ffffffffffffffff"}); + CheckInt128Case( + {absl::MakeInt128(1, 0), kDec, /*width = */ 0, "18446744073709551616"}); + CheckInt128Case( + {absl::MakeInt128(1, 0), kOct, /*width = */ 0, "2000000000000000000000"}); + CheckInt128Case( + {absl::MakeInt128(1, 0), kHex, /*width = */ 0, "10000000000000000"}); + CheckInt128Case({absl::MakeInt128(std::numeric_limits<int64_t>::max(), + std::numeric_limits<uint64_t>::max()), + std::ios::dec, /*width = */ 0, + "170141183460469231731687303715884105727"}); + CheckInt128Case({absl::MakeInt128(std::numeric_limits<int64_t>::max(), + std::numeric_limits<uint64_t>::max()), + std::ios::oct, /*width = */ 0, + "1777777777777777777777777777777777777777777"}); + CheckInt128Case({absl::MakeInt128(std::numeric_limits<int64_t>::max(), + std::numeric_limits<uint64_t>::max()), + std::ios::hex, /*width = */ 0, + "7fffffffffffffffffffffffffffffff"}); + CheckInt128Case({absl::MakeInt128(std::numeric_limits<int64_t>::min(), 0), + std::ios::dec, /*width = */ 0, + "-170141183460469231731687303715884105728"}); + CheckInt128Case({absl::MakeInt128(std::numeric_limits<int64_t>::min(), 0), + std::ios::oct, /*width = */ 0, + "2000000000000000000000000000000000000000000"}); + CheckInt128Case({absl::MakeInt128(std::numeric_limits<int64_t>::min(), 0), + std::ios::hex, /*width = */ 0, + "80000000000000000000000000000000"}); + CheckInt128Case({-1, std::ios::dec, /*width = */ 0, "-1"}); + CheckInt128Case({-1, std::ios::oct, /*width = */ 0, + "3777777777777777777777777777777777777777777"}); + CheckInt128Case( + {-1, std::ios::hex, /*width = */ 0, "ffffffffffffffffffffffffffffffff"}); + CheckInt128Case({-12345, std::ios::dec, /*width = */ 0, "-12345"}); + CheckInt128Case({-12345, std::ios::oct, /*width = */ 0, + "3777777777777777777777777777777777777747707"}); + CheckInt128Case({-12345, std::ios::hex, /*width = */ 0, + "ffffffffffffffffffffffffffffcfc7"}); +} + +std::vector<Int128TestCase> GetInt128FormatCases(); +TEST(Int128, OStreamFormatTest) { + for (const Int128TestCase& test_case : GetInt128FormatCases()) { + CheckInt128Case(test_case); + } +} + +std::vector<Int128TestCase> GetInt128FormatCases() { + return { + {0, std::ios_base::fmtflags(), /*width = */ 0, "0"}, + {0, std::ios_base::fmtflags(), /*width = */ 6, "_____0"}, + {0, kPos, /*width = */ 0, "+0"}, + {0, kPos, /*width = */ 6, "____+0"}, + {0, kBase, /*width = */ 0, "0"}, + {0, kBase, /*width = */ 6, "_____0"}, + {0, kBase | kPos, /*width = */ 0, "+0"}, + {0, kBase | kPos, /*width = */ 6, "____+0"}, + {0, kUpper, /*width = */ 0, "0"}, + {0, kUpper, /*width = */ 6, "_____0"}, + {0, kUpper | kPos, /*width = */ 0, "+0"}, + {0, kUpper | kPos, /*width = */ 6, "____+0"}, + {0, kUpper | kBase, /*width = */ 0, "0"}, + {0, kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kUpper | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kLeft, /*width = */ 0, "0"}, + {0, kLeft, /*width = */ 6, "0_____"}, + {0, kLeft | kPos, /*width = */ 0, "+0"}, + {0, kLeft | kPos, /*width = */ 6, "+0____"}, + {0, kLeft | kBase, /*width = */ 0, "0"}, + {0, kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kLeft | kBase | kPos, /*width = */ 0, "+0"}, + {0, kLeft | kBase | kPos, /*width = */ 6, "+0____"}, + {0, kLeft | kUpper, /*width = */ 0, "0"}, + {0, kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kLeft | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kLeft | kUpper | kPos, /*width = */ 6, "+0____"}, + {0, kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kLeft | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kLeft | kUpper | kBase | kPos, /*width = */ 6, "+0____"}, + {0, kInt, /*width = */ 0, "0"}, + {0, kInt, /*width = */ 6, "_____0"}, + {0, kInt | kPos, /*width = */ 0, "+0"}, + {0, kInt | kPos, /*width = */ 6, "+____0"}, + {0, kInt | kBase, /*width = */ 0, "0"}, + {0, kInt | kBase, /*width = */ 6, "_____0"}, + {0, kInt | kBase | kPos, /*width = */ 0, "+0"}, + {0, kInt | kBase | kPos, /*width = */ 6, "+____0"}, + {0, kInt | kUpper, /*width = */ 0, "0"}, + {0, kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kInt | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kInt | kUpper | kPos, /*width = */ 6, "+____0"}, + {0, kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kInt | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kInt | kUpper | kBase | kPos, /*width = */ 6, "+____0"}, + {0, kRight, /*width = */ 0, "0"}, + {0, kRight, /*width = */ 6, "_____0"}, + {0, kRight | kPos, /*width = */ 0, "+0"}, + {0, kRight | kPos, /*width = */ 6, "____+0"}, + {0, kRight | kBase, /*width = */ 0, "0"}, + {0, kRight | kBase, /*width = */ 6, "_____0"}, + {0, kRight | kBase | kPos, /*width = */ 0, "+0"}, + {0, kRight | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kRight | kUpper, /*width = */ 0, "0"}, + {0, kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kRight | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kRight | kUpper | kPos, /*width = */ 6, "____+0"}, + {0, kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kRight | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kRight | kUpper | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kDec, /*width = */ 0, "0"}, + {0, kDec, /*width = */ 6, "_____0"}, + {0, kDec | kPos, /*width = */ 0, "+0"}, + {0, kDec | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kBase, /*width = */ 0, "0"}, + {0, kDec | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kUpper, /*width = */ 0, "0"}, + {0, kDec | kUpper, /*width = */ 6, "_____0"}, + {0, kDec | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kDec | kUpper | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kUpper | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kLeft, /*width = */ 0, "0"}, + {0, kDec | kLeft, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kPos, /*width = */ 0, "+0"}, + {0, kDec | kLeft | kPos, /*width = */ 6, "+0____"}, + {0, kDec | kLeft | kBase, /*width = */ 0, "0"}, + {0, kDec | kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kLeft | kBase | kPos, /*width = */ 6, "+0____"}, + {0, kDec | kLeft | kUpper, /*width = */ 0, "0"}, + {0, kDec | kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kDec | kLeft | kUpper | kPos, /*width = */ 6, "+0____"}, + {0, kDec | kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 6, "+0____"}, + {0, kDec | kInt, /*width = */ 0, "0"}, + {0, kDec | kInt, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kPos, /*width = */ 0, "+0"}, + {0, kDec | kInt | kPos, /*width = */ 6, "+____0"}, + {0, kDec | kInt | kBase, /*width = */ 0, "0"}, + {0, kDec | kInt | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kInt | kBase | kPos, /*width = */ 6, "+____0"}, + {0, kDec | kInt | kUpper, /*width = */ 0, "0"}, + {0, kDec | kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kDec | kInt | kUpper | kPos, /*width = */ 6, "+____0"}, + {0, kDec | kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kInt | kUpper | kBase | kPos, /*width = */ 6, "+____0"}, + {0, kDec | kRight, /*width = */ 0, "0"}, + {0, kDec | kRight, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kPos, /*width = */ 0, "+0"}, + {0, kDec | kRight | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kRight | kBase, /*width = */ 0, "0"}, + {0, kDec | kRight | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kRight | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kRight | kUpper, /*width = */ 0, "0"}, + {0, kDec | kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kUpper | kPos, /*width = */ 0, "+0"}, + {0, kDec | kRight | kUpper | kPos, /*width = */ 6, "____+0"}, + {0, kDec | kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kUpper | kBase | kPos, /*width = */ 0, "+0"}, + {0, kDec | kRight | kUpper | kBase | kPos, /*width = */ 6, "____+0"}, + {0, kOct, /*width = */ 0, "0"}, + {0, kOct, /*width = */ 6, "_____0"}, + {0, kOct | kPos, /*width = */ 0, "0"}, + {0, kOct | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kBase, /*width = */ 0, "0"}, + {0, kOct | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kUpper, /*width = */ 0, "0"}, + {0, kOct | kUpper, /*width = */ 6, "_____0"}, + {0, kOct | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kLeft, /*width = */ 0, "0"}, + {0, kOct | kLeft, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kBase, /*width = */ 0, "0"}, + {0, kOct | kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kInt, /*width = */ 0, "0"}, + {0, kOct | kInt, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kBase, /*width = */ 0, "0"}, + {0, kOct | kInt | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight, /*width = */ 0, "0"}, + {0, kOct | kRight, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kBase, /*width = */ 0, "0"}, + {0, kOct | kRight | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex, /*width = */ 0, "0"}, + {0, kHex, /*width = */ 6, "_____0"}, + {0, kHex | kPos, /*width = */ 0, "0"}, + {0, kHex | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kBase, /*width = */ 0, "0"}, + {0, kHex | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kUpper, /*width = */ 0, "0"}, + {0, kHex | kUpper, /*width = */ 6, "_____0"}, + {0, kHex | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kLeft, /*width = */ 0, "0"}, + {0, kHex | kLeft, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kBase, /*width = */ 0, "0"}, + {0, kHex | kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kInt, /*width = */ 0, "0"}, + {0, kHex | kInt, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kBase, /*width = */ 0, "0"}, + {0, kHex | kInt | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight, /*width = */ 0, "0"}, + {0, kHex | kRight, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kBase, /*width = */ 0, "0"}, + {0, kHex | kRight | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {42, std::ios_base::fmtflags(), /*width = */ 0, "42"}, + {42, std::ios_base::fmtflags(), /*width = */ 6, "____42"}, + {42, kPos, /*width = */ 0, "+42"}, + {42, kPos, /*width = */ 6, "___+42"}, + {42, kBase, /*width = */ 0, "42"}, + {42, kBase, /*width = */ 6, "____42"}, + {42, kBase | kPos, /*width = */ 0, "+42"}, + {42, kBase | kPos, /*width = */ 6, "___+42"}, + {42, kUpper, /*width = */ 0, "42"}, + {42, kUpper, /*width = */ 6, "____42"}, + {42, kUpper | kPos, /*width = */ 0, "+42"}, + {42, kUpper | kPos, /*width = */ 6, "___+42"}, + {42, kUpper | kBase, /*width = */ 0, "42"}, + {42, kUpper | kBase, /*width = */ 6, "____42"}, + {42, kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kUpper | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kLeft, /*width = */ 0, "42"}, + {42, kLeft, /*width = */ 6, "42____"}, + {42, kLeft | kPos, /*width = */ 0, "+42"}, + {42, kLeft | kPos, /*width = */ 6, "+42___"}, + {42, kLeft | kBase, /*width = */ 0, "42"}, + {42, kLeft | kBase, /*width = */ 6, "42____"}, + {42, kLeft | kBase | kPos, /*width = */ 0, "+42"}, + {42, kLeft | kBase | kPos, /*width = */ 6, "+42___"}, + {42, kLeft | kUpper, /*width = */ 0, "42"}, + {42, kLeft | kUpper, /*width = */ 6, "42____"}, + {42, kLeft | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kLeft | kUpper | kPos, /*width = */ 6, "+42___"}, + {42, kLeft | kUpper | kBase, /*width = */ 0, "42"}, + {42, kLeft | kUpper | kBase, /*width = */ 6, "42____"}, + {42, kLeft | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kLeft | kUpper | kBase | kPos, /*width = */ 6, "+42___"}, + {42, kInt, /*width = */ 0, "42"}, + {42, kInt, /*width = */ 6, "____42"}, + {42, kInt | kPos, /*width = */ 0, "+42"}, + {42, kInt | kPos, /*width = */ 6, "+___42"}, + {42, kInt | kBase, /*width = */ 0, "42"}, + {42, kInt | kBase, /*width = */ 6, "____42"}, + {42, kInt | kBase | kPos, /*width = */ 0, "+42"}, + {42, kInt | kBase | kPos, /*width = */ 6, "+___42"}, + {42, kInt | kUpper, /*width = */ 0, "42"}, + {42, kInt | kUpper, /*width = */ 6, "____42"}, + {42, kInt | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kInt | kUpper | kPos, /*width = */ 6, "+___42"}, + {42, kInt | kUpper | kBase, /*width = */ 0, "42"}, + {42, kInt | kUpper | kBase, /*width = */ 6, "____42"}, + {42, kInt | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kInt | kUpper | kBase | kPos, /*width = */ 6, "+___42"}, + {42, kRight, /*width = */ 0, "42"}, + {42, kRight, /*width = */ 6, "____42"}, + {42, kRight | kPos, /*width = */ 0, "+42"}, + {42, kRight | kPos, /*width = */ 6, "___+42"}, + {42, kRight | kBase, /*width = */ 0, "42"}, + {42, kRight | kBase, /*width = */ 6, "____42"}, + {42, kRight | kBase | kPos, /*width = */ 0, "+42"}, + {42, kRight | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kRight | kUpper, /*width = */ 0, "42"}, + {42, kRight | kUpper, /*width = */ 6, "____42"}, + {42, kRight | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kRight | kUpper | kPos, /*width = */ 6, "___+42"}, + {42, kRight | kUpper | kBase, /*width = */ 0, "42"}, + {42, kRight | kUpper | kBase, /*width = */ 6, "____42"}, + {42, kRight | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kRight | kUpper | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kDec, /*width = */ 0, "42"}, + {42, kDec, /*width = */ 6, "____42"}, + {42, kDec | kPos, /*width = */ 0, "+42"}, + {42, kDec | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kBase, /*width = */ 0, "42"}, + {42, kDec | kBase, /*width = */ 6, "____42"}, + {42, kDec | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kUpper, /*width = */ 0, "42"}, + {42, kDec | kUpper, /*width = */ 6, "____42"}, + {42, kDec | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kDec | kUpper | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kUpper | kBase, /*width = */ 0, "42"}, + {42, kDec | kUpper | kBase, /*width = */ 6, "____42"}, + {42, kDec | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kUpper | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kLeft, /*width = */ 0, "42"}, + {42, kDec | kLeft, /*width = */ 6, "42____"}, + {42, kDec | kLeft | kPos, /*width = */ 0, "+42"}, + {42, kDec | kLeft | kPos, /*width = */ 6, "+42___"}, + {42, kDec | kLeft | kBase, /*width = */ 0, "42"}, + {42, kDec | kLeft | kBase, /*width = */ 6, "42____"}, + {42, kDec | kLeft | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kLeft | kBase | kPos, /*width = */ 6, "+42___"}, + {42, kDec | kLeft | kUpper, /*width = */ 0, "42"}, + {42, kDec | kLeft | kUpper, /*width = */ 6, "42____"}, + {42, kDec | kLeft | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kDec | kLeft | kUpper | kPos, /*width = */ 6, "+42___"}, + {42, kDec | kLeft | kUpper | kBase, /*width = */ 0, "42"}, + {42, kDec | kLeft | kUpper | kBase, /*width = */ 6, "42____"}, + {42, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 6, "+42___"}, + {42, kDec | kInt, /*width = */ 0, "42"}, + {42, kDec | kInt, /*width = */ 6, "____42"}, + {42, kDec | kInt | kPos, /*width = */ 0, "+42"}, + {42, kDec | kInt | kPos, /*width = */ 6, "+___42"}, + {42, kDec | kInt | kBase, /*width = */ 0, "42"}, + {42, kDec | kInt | kBase, /*width = */ 6, "____42"}, + {42, kDec | kInt | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kInt | kBase | kPos, /*width = */ 6, "+___42"}, + {42, kDec | kInt | kUpper, /*width = */ 0, "42"}, + {42, kDec | kInt | kUpper, /*width = */ 6, "____42"}, + {42, kDec | kInt | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kDec | kInt | kUpper | kPos, /*width = */ 6, "+___42"}, + {42, kDec | kInt | kUpper | kBase, /*width = */ 0, "42"}, + {42, kDec | kInt | kUpper | kBase, /*width = */ 6, "____42"}, + {42, kDec | kInt | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kInt | kUpper | kBase | kPos, /*width = */ 6, "+___42"}, + {42, kDec | kRight, /*width = */ 0, "42"}, + {42, kDec | kRight, /*width = */ 6, "____42"}, + {42, kDec | kRight | kPos, /*width = */ 0, "+42"}, + {42, kDec | kRight | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kRight | kBase, /*width = */ 0, "42"}, + {42, kDec | kRight | kBase, /*width = */ 6, "____42"}, + {42, kDec | kRight | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kRight | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kRight | kUpper, /*width = */ 0, "42"}, + {42, kDec | kRight | kUpper, /*width = */ 6, "____42"}, + {42, kDec | kRight | kUpper | kPos, /*width = */ 0, "+42"}, + {42, kDec | kRight | kUpper | kPos, /*width = */ 6, "___+42"}, + {42, kDec | kRight | kUpper | kBase, /*width = */ 0, "42"}, + {42, kDec | kRight | kUpper | kBase, /*width = */ 6, "____42"}, + {42, kDec | kRight | kUpper | kBase | kPos, /*width = */ 0, "+42"}, + {42, kDec | kRight | kUpper | kBase | kPos, /*width = */ 6, "___+42"}, + {42, kOct, /*width = */ 0, "52"}, + {42, kOct, /*width = */ 6, "____52"}, + {42, kOct | kPos, /*width = */ 0, "52"}, + {42, kOct | kPos, /*width = */ 6, "____52"}, + {42, kOct | kBase, /*width = */ 0, "052"}, + {42, kOct | kBase, /*width = */ 6, "___052"}, + {42, kOct | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kBase | kPos, /*width = */ 6, "___052"}, + {42, kOct | kUpper, /*width = */ 0, "52"}, + {42, kOct | kUpper, /*width = */ 6, "____52"}, + {42, kOct | kUpper | kPos, /*width = */ 0, "52"}, + {42, kOct | kUpper | kPos, /*width = */ 6, "____52"}, + {42, kOct | kUpper | kBase, /*width = */ 0, "052"}, + {42, kOct | kUpper | kBase, /*width = */ 6, "___052"}, + {42, kOct | kUpper | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kUpper | kBase | kPos, /*width = */ 6, "___052"}, + {42, kOct | kLeft, /*width = */ 0, "52"}, + {42, kOct | kLeft, /*width = */ 6, "52____"}, + {42, kOct | kLeft | kPos, /*width = */ 0, "52"}, + {42, kOct | kLeft | kPos, /*width = */ 6, "52____"}, + {42, kOct | kLeft | kBase, /*width = */ 0, "052"}, + {42, kOct | kLeft | kBase, /*width = */ 6, "052___"}, + {42, kOct | kLeft | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kLeft | kBase | kPos, /*width = */ 6, "052___"}, + {42, kOct | kLeft | kUpper, /*width = */ 0, "52"}, + {42, kOct | kLeft | kUpper, /*width = */ 6, "52____"}, + {42, kOct | kLeft | kUpper | kPos, /*width = */ 0, "52"}, + {42, kOct | kLeft | kUpper | kPos, /*width = */ 6, "52____"}, + {42, kOct | kLeft | kUpper | kBase, /*width = */ 0, "052"}, + {42, kOct | kLeft | kUpper | kBase, /*width = */ 6, "052___"}, + {42, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 6, "052___"}, + {42, kOct | kInt, /*width = */ 0, "52"}, + {42, kOct | kInt, /*width = */ 6, "____52"}, + {42, kOct | kInt | kPos, /*width = */ 0, "52"}, + {42, kOct | kInt | kPos, /*width = */ 6, "____52"}, + {42, kOct | kInt | kBase, /*width = */ 0, "052"}, + {42, kOct | kInt | kBase, /*width = */ 6, "___052"}, + {42, kOct | kInt | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kInt | kBase | kPos, /*width = */ 6, "___052"}, + {42, kOct | kInt | kUpper, /*width = */ 0, "52"}, + {42, kOct | kInt | kUpper, /*width = */ 6, "____52"}, + {42, kOct | kInt | kUpper | kPos, /*width = */ 0, "52"}, + {42, kOct | kInt | kUpper | kPos, /*width = */ 6, "____52"}, + {42, kOct | kInt | kUpper | kBase, /*width = */ 0, "052"}, + {42, kOct | kInt | kUpper | kBase, /*width = */ 6, "___052"}, + {42, kOct | kInt | kUpper | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kInt | kUpper | kBase | kPos, /*width = */ 6, "___052"}, + {42, kOct | kRight, /*width = */ 0, "52"}, + {42, kOct | kRight, /*width = */ 6, "____52"}, + {42, kOct | kRight | kPos, /*width = */ 0, "52"}, + {42, kOct | kRight | kPos, /*width = */ 6, "____52"}, + {42, kOct | kRight | kBase, /*width = */ 0, "052"}, + {42, kOct | kRight | kBase, /*width = */ 6, "___052"}, + {42, kOct | kRight | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kRight | kBase | kPos, /*width = */ 6, "___052"}, + {42, kOct | kRight | kUpper, /*width = */ 0, "52"}, + {42, kOct | kRight | kUpper, /*width = */ 6, "____52"}, + {42, kOct | kRight | kUpper | kPos, /*width = */ 0, "52"}, + {42, kOct | kRight | kUpper | kPos, /*width = */ 6, "____52"}, + {42, kOct | kRight | kUpper | kBase, /*width = */ 0, "052"}, + {42, kOct | kRight | kUpper | kBase, /*width = */ 6, "___052"}, + {42, kOct | kRight | kUpper | kBase | kPos, /*width = */ 0, "052"}, + {42, kOct | kRight | kUpper | kBase | kPos, /*width = */ 6, "___052"}, + {42, kHex, /*width = */ 0, "2a"}, + {42, kHex, /*width = */ 6, "____2a"}, + {42, kHex | kPos, /*width = */ 0, "2a"}, + {42, kHex | kPos, /*width = */ 6, "____2a"}, + {42, kHex | kBase, /*width = */ 0, "0x2a"}, + {42, kHex | kBase, /*width = */ 6, "__0x2a"}, + {42, kHex | kBase | kPos, /*width = */ 0, "0x2a"}, + {42, kHex | kBase | kPos, /*width = */ 6, "__0x2a"}, + {42, kHex | kUpper, /*width = */ 0, "2A"}, + {42, kHex | kUpper, /*width = */ 6, "____2A"}, + {42, kHex | kUpper | kPos, /*width = */ 0, "2A"}, + {42, kHex | kUpper | kPos, /*width = */ 6, "____2A"}, + {42, kHex | kUpper | kBase, /*width = */ 0, "0X2A"}, + {42, kHex | kUpper | kBase, /*width = */ 6, "__0X2A"}, + {42, kHex | kUpper | kBase | kPos, /*width = */ 0, "0X2A"}, + {42, kHex | kUpper | kBase | kPos, /*width = */ 6, "__0X2A"}, + {42, kHex | kLeft, /*width = */ 0, "2a"}, + {42, kHex | kLeft, /*width = */ 6, "2a____"}, + {42, kHex | kLeft | kPos, /*width = */ 0, "2a"}, + {42, kHex | kLeft | kPos, /*width = */ 6, "2a____"}, + {42, kHex | kLeft | kBase, /*width = */ 0, "0x2a"}, + {42, kHex | kLeft | kBase, /*width = */ 6, "0x2a__"}, + {42, kHex | kLeft | kBase | kPos, /*width = */ 0, "0x2a"}, + {42, kHex | kLeft | kBase | kPos, /*width = */ 6, "0x2a__"}, + {42, kHex | kLeft | kUpper, /*width = */ 0, "2A"}, + {42, kHex | kLeft | kUpper, /*width = */ 6, "2A____"}, + {42, kHex | kLeft | kUpper | kPos, /*width = */ 0, "2A"}, + {42, kHex | kLeft | kUpper | kPos, /*width = */ 6, "2A____"}, + {42, kHex | kLeft | kUpper | kBase, /*width = */ 0, "0X2A"}, + {42, kHex | kLeft | kUpper | kBase, /*width = */ 6, "0X2A__"}, + {42, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0X2A"}, + {42, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0X2A__"}, + {42, kHex | kInt, /*width = */ 0, "2a"}, + {42, kHex | kInt, /*width = */ 6, "____2a"}, + {42, kHex | kInt | kPos, /*width = */ 0, "2a"}, + {42, kHex | kInt | kPos, /*width = */ 6, "____2a"}, + {42, kHex | kInt | kBase, /*width = */ 0, "0x2a"}, + {42, kHex | kInt | kBase, /*width = */ 6, "0x__2a"}, + {42, kHex | kInt | kBase | kPos, /*width = */ 0, "0x2a"}, + {42, kHex | kInt | kBase | kPos, /*width = */ 6, "0x__2a"}, + {42, kHex | kInt | kUpper, /*width = */ 0, "2A"}, + {42, kHex | kInt | kUpper, /*width = */ 6, "____2A"}, + {42, kHex | kInt | kUpper | kPos, /*width = */ 0, "2A"}, + {42, kHex | kInt | kUpper | kPos, /*width = */ 6, "____2A"}, + {42, kHex | kInt | kUpper | kBase, /*width = */ 0, "0X2A"}, + {42, kHex | kInt | kUpper | kBase, /*width = */ 6, "0X__2A"}, + {42, kHex | kInt | kUpper | kBase | kPos, /*width = */ 0, "0X2A"}, + {42, kHex | kInt | kUpper | kBase | kPos, /*width = */ 6, "0X__2A"}, + {42, kHex | kRight, /*width = */ 0, "2a"}, + {42, kHex | kRight, /*width = */ 6, "____2a"}, + {42, kHex | kRight | kPos, /*width = */ 0, "2a"}, + {42, kHex | kRight | kPos, /*width = */ 6, "____2a"}, + {42, kHex | kRight | kBase, /*width = */ 0, "0x2a"}, + {42, kHex | kRight | kBase, /*width = */ 6, "__0x2a"}, + {42, kHex | kRight | kBase | kPos, /*width = */ 0, "0x2a"}, + {42, kHex | kRight | kBase | kPos, /*width = */ 6, "__0x2a"}, + {42, kHex | kRight | kUpper, /*width = */ 0, "2A"}, + {42, kHex | kRight | kUpper, /*width = */ 6, "____2A"}, + {42, kHex | kRight | kUpper | kPos, /*width = */ 0, "2A"}, + {42, kHex | kRight | kUpper | kPos, /*width = */ 6, "____2A"}, + {42, kHex | kRight | kUpper | kBase, /*width = */ 0, "0X2A"}, + {42, kHex | kRight | kUpper | kBase, /*width = */ 6, "__0X2A"}, + {42, kHex | kRight | kUpper | kBase | kPos, /*width = */ 0, "0X2A"}, + {42, kHex | kRight | kUpper | kBase | kPos, /*width = */ 6, "__0X2A"}, + {-321, std::ios_base::fmtflags(), /*width = */ 0, "-321"}, + {-321, std::ios_base::fmtflags(), /*width = */ 6, "__-321"}, + {-321, kPos, /*width = */ 0, "-321"}, + {-321, kPos, /*width = */ 6, "__-321"}, + {-321, kBase, /*width = */ 0, "-321"}, + {-321, kBase, /*width = */ 6, "__-321"}, + {-321, kBase | kPos, /*width = */ 0, "-321"}, + {-321, kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kUpper, /*width = */ 0, "-321"}, + {-321, kUpper, /*width = */ 6, "__-321"}, + {-321, kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kUpper | kPos, /*width = */ 6, "__-321"}, + {-321, kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kUpper | kBase, /*width = */ 6, "__-321"}, + {-321, kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kUpper | kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kLeft, /*width = */ 0, "-321"}, + {-321, kLeft, /*width = */ 6, "-321__"}, + {-321, kLeft | kPos, /*width = */ 0, "-321"}, + {-321, kLeft | kPos, /*width = */ 6, "-321__"}, + {-321, kLeft | kBase, /*width = */ 0, "-321"}, + {-321, kLeft | kBase, /*width = */ 6, "-321__"}, + {-321, kLeft | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kLeft | kBase | kPos, /*width = */ 6, "-321__"}, + {-321, kLeft | kUpper, /*width = */ 0, "-321"}, + {-321, kLeft | kUpper, /*width = */ 6, "-321__"}, + {-321, kLeft | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kLeft | kUpper | kPos, /*width = */ 6, "-321__"}, + {-321, kLeft | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kLeft | kUpper | kBase, /*width = */ 6, "-321__"}, + {-321, kLeft | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kLeft | kUpper | kBase | kPos, /*width = */ 6, "-321__"}, + {-321, kInt, /*width = */ 0, "-321"}, + {-321, kInt, /*width = */ 6, "-__321"}, + {-321, kInt | kPos, /*width = */ 0, "-321"}, + {-321, kInt | kPos, /*width = */ 6, "-__321"}, + {-321, kInt | kBase, /*width = */ 0, "-321"}, + {-321, kInt | kBase, /*width = */ 6, "-__321"}, + {-321, kInt | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kInt | kBase | kPos, /*width = */ 6, "-__321"}, + {-321, kInt | kUpper, /*width = */ 0, "-321"}, + {-321, kInt | kUpper, /*width = */ 6, "-__321"}, + {-321, kInt | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kInt | kUpper | kPos, /*width = */ 6, "-__321"}, + {-321, kInt | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kInt | kUpper | kBase, /*width = */ 6, "-__321"}, + {-321, kInt | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kInt | kUpper | kBase | kPos, /*width = */ 6, "-__321"}, + {-321, kRight, /*width = */ 0, "-321"}, + {-321, kRight, /*width = */ 6, "__-321"}, + {-321, kRight | kPos, /*width = */ 0, "-321"}, + {-321, kRight | kPos, /*width = */ 6, "__-321"}, + {-321, kRight | kBase, /*width = */ 0, "-321"}, + {-321, kRight | kBase, /*width = */ 6, "__-321"}, + {-321, kRight | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kRight | kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kRight | kUpper, /*width = */ 0, "-321"}, + {-321, kRight | kUpper, /*width = */ 6, "__-321"}, + {-321, kRight | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kRight | kUpper | kPos, /*width = */ 6, "__-321"}, + {-321, kRight | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kRight | kUpper | kBase, /*width = */ 6, "__-321"}, + {-321, kRight | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kRight | kUpper | kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kDec, /*width = */ 0, "-321"}, + {-321, kDec, /*width = */ 6, "__-321"}, + {-321, kDec | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kBase, /*width = */ 6, "__-321"}, + {-321, kDec | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kUpper, /*width = */ 0, "-321"}, + {-321, kDec | kUpper, /*width = */ 6, "__-321"}, + {-321, kDec | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kUpper | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kUpper | kBase, /*width = */ 6, "__-321"}, + {-321, kDec | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kUpper | kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kLeft, /*width = */ 0, "-321"}, + {-321, kDec | kLeft, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kPos, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kBase, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kBase | kPos, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kUpper, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kUpper, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kUpper | kPos, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kUpper | kBase, /*width = */ 6, "-321__"}, + {-321, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 6, "-321__"}, + {-321, kDec | kInt, /*width = */ 0, "-321"}, + {-321, kDec | kInt, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kPos, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kBase, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kBase | kPos, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kUpper, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kUpper, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kUpper | kPos, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kUpper | kBase, /*width = */ 6, "-__321"}, + {-321, kDec | kInt | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kInt | kUpper | kBase | kPos, /*width = */ 6, "-__321"}, + {-321, kDec | kRight, /*width = */ 0, "-321"}, + {-321, kDec | kRight, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kBase, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kBase | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kUpper, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kUpper, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kUpper | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kUpper | kPos, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kUpper | kBase, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kUpper | kBase, /*width = */ 6, "__-321"}, + {-321, kDec | kRight | kUpper | kBase | kPos, /*width = */ 0, "-321"}, + {-321, kDec | kRight | kUpper | kBase | kPos, /*width = */ 6, "__-321"}}; +} + +std::vector<Uint128TestCase> GetUint128FormatCases() { + return { + {0, std::ios_base::fmtflags(), /*width = */ 0, "0"}, + {0, std::ios_base::fmtflags(), /*width = */ 6, "_____0"}, + {0, kPos, /*width = */ 0, "0"}, + {0, kPos, /*width = */ 6, "_____0"}, + {0, kBase, /*width = */ 0, "0"}, + {0, kBase, /*width = */ 6, "_____0"}, + {0, kBase | kPos, /*width = */ 0, "0"}, + {0, kBase | kPos, /*width = */ 6, "_____0"}, + {0, kUpper, /*width = */ 0, "0"}, + {0, kUpper, /*width = */ 6, "_____0"}, + {0, kUpper | kPos, /*width = */ 0, "0"}, + {0, kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kUpper | kBase, /*width = */ 0, "0"}, + {0, kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kLeft, /*width = */ 0, "0"}, + {0, kLeft, /*width = */ 6, "0_____"}, + {0, kLeft | kPos, /*width = */ 0, "0"}, + {0, kLeft | kPos, /*width = */ 6, "0_____"}, + {0, kLeft | kBase, /*width = */ 0, "0"}, + {0, kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kLeft | kBase | kPos, /*width = */ 0, "0"}, + {0, kLeft | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kLeft | kUpper, /*width = */ 0, "0"}, + {0, kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kLeft | kUpper | kPos, /*width = */ 0, "0"}, + {0, kLeft | kUpper | kPos, /*width = */ 6, "0_____"}, + {0, kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kLeft | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kLeft | kUpper | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kInt, /*width = */ 0, "0"}, + {0, kInt, /*width = */ 6, "_____0"}, + {0, kInt | kPos, /*width = */ 0, "0"}, + {0, kInt | kPos, /*width = */ 6, "_____0"}, + {0, kInt | kBase, /*width = */ 0, "0"}, + {0, kInt | kBase, /*width = */ 6, "_____0"}, + {0, kInt | kBase | kPos, /*width = */ 0, "0"}, + {0, kInt | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kInt | kUpper, /*width = */ 0, "0"}, + {0, kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kInt | kUpper | kPos, /*width = */ 0, "0"}, + {0, kInt | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kInt | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kInt | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kRight, /*width = */ 0, "0"}, + {0, kRight, /*width = */ 6, "_____0"}, + {0, kRight | kPos, /*width = */ 0, "0"}, + {0, kRight | kPos, /*width = */ 6, "_____0"}, + {0, kRight | kBase, /*width = */ 0, "0"}, + {0, kRight | kBase, /*width = */ 6, "_____0"}, + {0, kRight | kBase | kPos, /*width = */ 0, "0"}, + {0, kRight | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kRight | kUpper, /*width = */ 0, "0"}, + {0, kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kRight | kUpper | kPos, /*width = */ 0, "0"}, + {0, kRight | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kRight | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kRight | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kDec, /*width = */ 0, "0"}, + {0, kDec, /*width = */ 6, "_____0"}, + {0, kDec | kPos, /*width = */ 0, "0"}, + {0, kDec | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kBase, /*width = */ 0, "0"}, + {0, kDec | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kUpper, /*width = */ 0, "0"}, + {0, kDec | kUpper, /*width = */ 6, "_____0"}, + {0, kDec | kUpper | kPos, /*width = */ 0, "0"}, + {0, kDec | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kLeft, /*width = */ 0, "0"}, + {0, kDec | kLeft, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kPos, /*width = */ 0, "0"}, + {0, kDec | kLeft | kPos, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kBase, /*width = */ 0, "0"}, + {0, kDec | kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kLeft | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kUpper, /*width = */ 0, "0"}, + {0, kDec | kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kUpper | kPos, /*width = */ 0, "0"}, + {0, kDec | kLeft | kUpper | kPos, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kDec | kInt, /*width = */ 0, "0"}, + {0, kDec | kInt, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kPos, /*width = */ 0, "0"}, + {0, kDec | kInt | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kBase, /*width = */ 0, "0"}, + {0, kDec | kInt | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kInt | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kUpper, /*width = */ 0, "0"}, + {0, kDec | kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kUpper | kPos, /*width = */ 0, "0"}, + {0, kDec | kInt | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kInt | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kInt | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kRight, /*width = */ 0, "0"}, + {0, kDec | kRight, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kPos, /*width = */ 0, "0"}, + {0, kDec | kRight | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kBase, /*width = */ 0, "0"}, + {0, kDec | kRight | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kRight | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kUpper, /*width = */ 0, "0"}, + {0, kDec | kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kUpper | kPos, /*width = */ 0, "0"}, + {0, kDec | kRight | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kDec | kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kDec | kRight | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kDec | kRight | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct, /*width = */ 0, "0"}, + {0, kOct, /*width = */ 6, "_____0"}, + {0, kOct | kPos, /*width = */ 0, "0"}, + {0, kOct | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kBase, /*width = */ 0, "0"}, + {0, kOct | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kUpper, /*width = */ 0, "0"}, + {0, kOct | kUpper, /*width = */ 6, "_____0"}, + {0, kOct | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kLeft, /*width = */ 0, "0"}, + {0, kOct | kLeft, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kBase, /*width = */ 0, "0"}, + {0, kOct | kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kOct | kInt, /*width = */ 0, "0"}, + {0, kOct | kInt, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kBase, /*width = */ 0, "0"}, + {0, kOct | kInt | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kInt | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kInt | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight, /*width = */ 0, "0"}, + {0, kOct | kRight, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kBase, /*width = */ 0, "0"}, + {0, kOct | kRight | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kOct | kRight | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kOct | kRight | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex, /*width = */ 0, "0"}, + {0, kHex, /*width = */ 6, "_____0"}, + {0, kHex | kPos, /*width = */ 0, "0"}, + {0, kHex | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kBase, /*width = */ 0, "0"}, + {0, kHex | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kUpper, /*width = */ 0, "0"}, + {0, kHex | kUpper, /*width = */ 6, "_____0"}, + {0, kHex | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kLeft, /*width = */ 0, "0"}, + {0, kHex | kLeft, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kBase, /*width = */ 0, "0"}, + {0, kHex | kLeft | kBase, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper | kBase, /*width = */ 6, "0_____"}, + {0, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0_____"}, + {0, kHex | kInt, /*width = */ 0, "0"}, + {0, kHex | kInt, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kBase, /*width = */ 0, "0"}, + {0, kHex | kInt | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kInt | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kInt | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight, /*width = */ 0, "0"}, + {0, kHex | kRight, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kBase, /*width = */ 0, "0"}, + {0, kHex | kRight | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kBase | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper | kPos, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper | kBase, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper | kBase, /*width = */ 6, "_____0"}, + {0, kHex | kRight | kUpper | kBase | kPos, /*width = */ 0, "0"}, + {0, kHex | kRight | kUpper | kBase | kPos, /*width = */ 6, "_____0"}, + {37, std::ios_base::fmtflags(), /*width = */ 0, "37"}, + {37, std::ios_base::fmtflags(), /*width = */ 6, "____37"}, + {37, kPos, /*width = */ 0, "37"}, + {37, kPos, /*width = */ 6, "____37"}, + {37, kBase, /*width = */ 0, "37"}, + {37, kBase, /*width = */ 6, "____37"}, + {37, kBase | kPos, /*width = */ 0, "37"}, + {37, kBase | kPos, /*width = */ 6, "____37"}, + {37, kUpper, /*width = */ 0, "37"}, + {37, kUpper, /*width = */ 6, "____37"}, + {37, kUpper | kPos, /*width = */ 0, "37"}, + {37, kUpper | kPos, /*width = */ 6, "____37"}, + {37, kUpper | kBase, /*width = */ 0, "37"}, + {37, kUpper | kBase, /*width = */ 6, "____37"}, + {37, kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kUpper | kBase | kPos, /*width = */ 6, "____37"}, + {37, kLeft, /*width = */ 0, "37"}, + {37, kLeft, /*width = */ 6, "37____"}, + {37, kLeft | kPos, /*width = */ 0, "37"}, + {37, kLeft | kPos, /*width = */ 6, "37____"}, + {37, kLeft | kBase, /*width = */ 0, "37"}, + {37, kLeft | kBase, /*width = */ 6, "37____"}, + {37, kLeft | kBase | kPos, /*width = */ 0, "37"}, + {37, kLeft | kBase | kPos, /*width = */ 6, "37____"}, + {37, kLeft | kUpper, /*width = */ 0, "37"}, + {37, kLeft | kUpper, /*width = */ 6, "37____"}, + {37, kLeft | kUpper | kPos, /*width = */ 0, "37"}, + {37, kLeft | kUpper | kPos, /*width = */ 6, "37____"}, + {37, kLeft | kUpper | kBase, /*width = */ 0, "37"}, + {37, kLeft | kUpper | kBase, /*width = */ 6, "37____"}, + {37, kLeft | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kLeft | kUpper | kBase | kPos, /*width = */ 6, "37____"}, + {37, kInt, /*width = */ 0, "37"}, + {37, kInt, /*width = */ 6, "____37"}, + {37, kInt | kPos, /*width = */ 0, "37"}, + {37, kInt | kPos, /*width = */ 6, "____37"}, + {37, kInt | kBase, /*width = */ 0, "37"}, + {37, kInt | kBase, /*width = */ 6, "____37"}, + {37, kInt | kBase | kPos, /*width = */ 0, "37"}, + {37, kInt | kBase | kPos, /*width = */ 6, "____37"}, + {37, kInt | kUpper, /*width = */ 0, "37"}, + {37, kInt | kUpper, /*width = */ 6, "____37"}, + {37, kInt | kUpper | kPos, /*width = */ 0, "37"}, + {37, kInt | kUpper | kPos, /*width = */ 6, "____37"}, + {37, kInt | kUpper | kBase, /*width = */ 0, "37"}, + {37, kInt | kUpper | kBase, /*width = */ 6, "____37"}, + {37, kInt | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kInt | kUpper | kBase | kPos, /*width = */ 6, "____37"}, + {37, kRight, /*width = */ 0, "37"}, + {37, kRight, /*width = */ 6, "____37"}, + {37, kRight | kPos, /*width = */ 0, "37"}, + {37, kRight | kPos, /*width = */ 6, "____37"}, + {37, kRight | kBase, /*width = */ 0, "37"}, + {37, kRight | kBase, /*width = */ 6, "____37"}, + {37, kRight | kBase | kPos, /*width = */ 0, "37"}, + {37, kRight | kBase | kPos, /*width = */ 6, "____37"}, + {37, kRight | kUpper, /*width = */ 0, "37"}, + {37, kRight | kUpper, /*width = */ 6, "____37"}, + {37, kRight | kUpper | kPos, /*width = */ 0, "37"}, + {37, kRight | kUpper | kPos, /*width = */ 6, "____37"}, + {37, kRight | kUpper | kBase, /*width = */ 0, "37"}, + {37, kRight | kUpper | kBase, /*width = */ 6, "____37"}, + {37, kRight | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kRight | kUpper | kBase | kPos, /*width = */ 6, "____37"}, + {37, kDec, /*width = */ 0, "37"}, + {37, kDec, /*width = */ 6, "____37"}, + {37, kDec | kPos, /*width = */ 0, "37"}, + {37, kDec | kPos, /*width = */ 6, "____37"}, + {37, kDec | kBase, /*width = */ 0, "37"}, + {37, kDec | kBase, /*width = */ 6, "____37"}, + {37, kDec | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kBase | kPos, /*width = */ 6, "____37"}, + {37, kDec | kUpper, /*width = */ 0, "37"}, + {37, kDec | kUpper, /*width = */ 6, "____37"}, + {37, kDec | kUpper | kPos, /*width = */ 0, "37"}, + {37, kDec | kUpper | kPos, /*width = */ 6, "____37"}, + {37, kDec | kUpper | kBase, /*width = */ 0, "37"}, + {37, kDec | kUpper | kBase, /*width = */ 6, "____37"}, + {37, kDec | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kUpper | kBase | kPos, /*width = */ 6, "____37"}, + {37, kDec | kLeft, /*width = */ 0, "37"}, + {37, kDec | kLeft, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kPos, /*width = */ 0, "37"}, + {37, kDec | kLeft | kPos, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kBase, /*width = */ 0, "37"}, + {37, kDec | kLeft | kBase, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kLeft | kBase | kPos, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kUpper, /*width = */ 0, "37"}, + {37, kDec | kLeft | kUpper, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kUpper | kPos, /*width = */ 0, "37"}, + {37, kDec | kLeft | kUpper | kPos, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kUpper | kBase, /*width = */ 0, "37"}, + {37, kDec | kLeft | kUpper | kBase, /*width = */ 6, "37____"}, + {37, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kLeft | kUpper | kBase | kPos, /*width = */ 6, "37____"}, + {37, kDec | kInt, /*width = */ 0, "37"}, + {37, kDec | kInt, /*width = */ 6, "____37"}, + {37, kDec | kInt | kPos, /*width = */ 0, "37"}, + {37, kDec | kInt | kPos, /*width = */ 6, "____37"}, + {37, kDec | kInt | kBase, /*width = */ 0, "37"}, + {37, kDec | kInt | kBase, /*width = */ 6, "____37"}, + {37, kDec | kInt | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kInt | kBase | kPos, /*width = */ 6, "____37"}, + {37, kDec | kInt | kUpper, /*width = */ 0, "37"}, + {37, kDec | kInt | kUpper, /*width = */ 6, "____37"}, + {37, kDec | kInt | kUpper | kPos, /*width = */ 0, "37"}, + {37, kDec | kInt | kUpper | kPos, /*width = */ 6, "____37"}, + {37, kDec | kInt | kUpper | kBase, /*width = */ 0, "37"}, + {37, kDec | kInt | kUpper | kBase, /*width = */ 6, "____37"}, + {37, kDec | kInt | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kInt | kUpper | kBase | kPos, /*width = */ 6, "____37"}, + {37, kDec | kRight, /*width = */ 0, "37"}, + {37, kDec | kRight, /*width = */ 6, "____37"}, + {37, kDec | kRight | kPos, /*width = */ 0, "37"}, + {37, kDec | kRight | kPos, /*width = */ 6, "____37"}, + {37, kDec | kRight | kBase, /*width = */ 0, "37"}, + {37, kDec | kRight | kBase, /*width = */ 6, "____37"}, + {37, kDec | kRight | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kRight | kBase | kPos, /*width = */ 6, "____37"}, + {37, kDec | kRight | kUpper, /*width = */ 0, "37"}, + {37, kDec | kRight | kUpper, /*width = */ 6, "____37"}, + {37, kDec | kRight | kUpper | kPos, /*width = */ 0, "37"}, + {37, kDec | kRight | kUpper | kPos, /*width = */ 6, "____37"}, + {37, kDec | kRight | kUpper | kBase, /*width = */ 0, "37"}, + {37, kDec | kRight | kUpper | kBase, /*width = */ 6, "____37"}, + {37, kDec | kRight | kUpper | kBase | kPos, /*width = */ 0, "37"}, + {37, kDec | kRight | kUpper | kBase | kPos, /*width = */ 6, "____37"}, + {37, kOct, /*width = */ 0, "45"}, + {37, kOct, /*width = */ 6, "____45"}, + {37, kOct | kPos, /*width = */ 0, "45"}, + {37, kOct | kPos, /*width = */ 6, "____45"}, + {37, kOct | kBase, /*width = */ 0, "045"}, + {37, kOct | kBase, /*width = */ 6, "___045"}, + {37, kOct | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kBase | kPos, /*width = */ 6, "___045"}, + {37, kOct | kUpper, /*width = */ 0, "45"}, + {37, kOct | kUpper, /*width = */ 6, "____45"}, + {37, kOct | kUpper | kPos, /*width = */ 0, "45"}, + {37, kOct | kUpper | kPos, /*width = */ 6, "____45"}, + {37, kOct | kUpper | kBase, /*width = */ 0, "045"}, + {37, kOct | kUpper | kBase, /*width = */ 6, "___045"}, + {37, kOct | kUpper | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kUpper | kBase | kPos, /*width = */ 6, "___045"}, + {37, kOct | kLeft, /*width = */ 0, "45"}, + {37, kOct | kLeft, /*width = */ 6, "45____"}, + {37, kOct | kLeft | kPos, /*width = */ 0, "45"}, + {37, kOct | kLeft | kPos, /*width = */ 6, "45____"}, + {37, kOct | kLeft | kBase, /*width = */ 0, "045"}, + {37, kOct | kLeft | kBase, /*width = */ 6, "045___"}, + {37, kOct | kLeft | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kLeft | kBase | kPos, /*width = */ 6, "045___"}, + {37, kOct | kLeft | kUpper, /*width = */ 0, "45"}, + {37, kOct | kLeft | kUpper, /*width = */ 6, "45____"}, + {37, kOct | kLeft | kUpper | kPos, /*width = */ 0, "45"}, + {37, kOct | kLeft | kUpper | kPos, /*width = */ 6, "45____"}, + {37, kOct | kLeft | kUpper | kBase, /*width = */ 0, "045"}, + {37, kOct | kLeft | kUpper | kBase, /*width = */ 6, "045___"}, + {37, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kLeft | kUpper | kBase | kPos, /*width = */ 6, "045___"}, + {37, kOct | kInt, /*width = */ 0, "45"}, + {37, kOct | kInt, /*width = */ 6, "____45"}, + {37, kOct | kInt | kPos, /*width = */ 0, "45"}, + {37, kOct | kInt | kPos, /*width = */ 6, "____45"}, + {37, kOct | kInt | kBase, /*width = */ 0, "045"}, + {37, kOct | kInt | kBase, /*width = */ 6, "___045"}, + {37, kOct | kInt | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kInt | kBase | kPos, /*width = */ 6, "___045"}, + {37, kOct | kInt | kUpper, /*width = */ 0, "45"}, + {37, kOct | kInt | kUpper, /*width = */ 6, "____45"}, + {37, kOct | kInt | kUpper | kPos, /*width = */ 0, "45"}, + {37, kOct | kInt | kUpper | kPos, /*width = */ 6, "____45"}, + {37, kOct | kInt | kUpper | kBase, /*width = */ 0, "045"}, + {37, kOct | kInt | kUpper | kBase, /*width = */ 6, "___045"}, + {37, kOct | kInt | kUpper | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kInt | kUpper | kBase | kPos, /*width = */ 6, "___045"}, + {37, kOct | kRight, /*width = */ 0, "45"}, + {37, kOct | kRight, /*width = */ 6, "____45"}, + {37, kOct | kRight | kPos, /*width = */ 0, "45"}, + {37, kOct | kRight | kPos, /*width = */ 6, "____45"}, + {37, kOct | kRight | kBase, /*width = */ 0, "045"}, + {37, kOct | kRight | kBase, /*width = */ 6, "___045"}, + {37, kOct | kRight | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kRight | kBase | kPos, /*width = */ 6, "___045"}, + {37, kOct | kRight | kUpper, /*width = */ 0, "45"}, + {37, kOct | kRight | kUpper, /*width = */ 6, "____45"}, + {37, kOct | kRight | kUpper | kPos, /*width = */ 0, "45"}, + {37, kOct | kRight | kUpper | kPos, /*width = */ 6, "____45"}, + {37, kOct | kRight | kUpper | kBase, /*width = */ 0, "045"}, + {37, kOct | kRight | kUpper | kBase, /*width = */ 6, "___045"}, + {37, kOct | kRight | kUpper | kBase | kPos, /*width = */ 0, "045"}, + {37, kOct | kRight | kUpper | kBase | kPos, /*width = */ 6, "___045"}, + {37, kHex, /*width = */ 0, "25"}, + {37, kHex, /*width = */ 6, "____25"}, + {37, kHex | kPos, /*width = */ 0, "25"}, + {37, kHex | kPos, /*width = */ 6, "____25"}, + {37, kHex | kBase, /*width = */ 0, "0x25"}, + {37, kHex | kBase, /*width = */ 6, "__0x25"}, + {37, kHex | kBase | kPos, /*width = */ 0, "0x25"}, + {37, kHex | kBase | kPos, /*width = */ 6, "__0x25"}, + {37, kHex | kUpper, /*width = */ 0, "25"}, + {37, kHex | kUpper, /*width = */ 6, "____25"}, + {37, kHex | kUpper | kPos, /*width = */ 0, "25"}, + {37, kHex | kUpper | kPos, /*width = */ 6, "____25"}, + {37, kHex | kUpper | kBase, /*width = */ 0, "0X25"}, + {37, kHex | kUpper | kBase, /*width = */ 6, "__0X25"}, + {37, kHex | kUpper | kBase | kPos, /*width = */ 0, "0X25"}, + {37, kHex | kUpper | kBase | kPos, /*width = */ 6, "__0X25"}, + {37, kHex | kLeft, /*width = */ 0, "25"}, + {37, kHex | kLeft, /*width = */ 6, "25____"}, + {37, kHex | kLeft | kPos, /*width = */ 0, "25"}, + {37, kHex | kLeft | kPos, /*width = */ 6, "25____"}, + {37, kHex | kLeft | kBase, /*width = */ 0, "0x25"}, + {37, kHex | kLeft | kBase, /*width = */ 6, "0x25__"}, + {37, kHex | kLeft | kBase | kPos, /*width = */ 0, "0x25"}, + {37, kHex | kLeft | kBase | kPos, /*width = */ 6, "0x25__"}, + {37, kHex | kLeft | kUpper, /*width = */ 0, "25"}, + {37, kHex | kLeft | kUpper, /*width = */ 6, "25____"}, + {37, kHex | kLeft | kUpper | kPos, /*width = */ 0, "25"}, + {37, kHex | kLeft | kUpper | kPos, /*width = */ 6, "25____"}, + {37, kHex | kLeft | kUpper | kBase, /*width = */ 0, "0X25"}, + {37, kHex | kLeft | kUpper | kBase, /*width = */ 6, "0X25__"}, + {37, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 0, "0X25"}, + {37, kHex | kLeft | kUpper | kBase | kPos, /*width = */ 6, "0X25__"}, + {37, kHex | kInt, /*width = */ 0, "25"}, + {37, kHex | kInt, /*width = */ 6, "____25"}, + {37, kHex | kInt | kPos, /*width = */ 0, "25"}, + {37, kHex | kInt | kPos, /*width = */ 6, "____25"}, + {37, kHex | kInt | kBase, /*width = */ 0, "0x25"}, + {37, kHex | kInt | kBase, /*width = */ 6, "0x__25"}, + {37, kHex | kInt | kBase | kPos, /*width = */ 0, "0x25"}, + {37, kHex | kInt | kBase | kPos, /*width = */ 6, "0x__25"}, + {37, kHex | kInt | kUpper, /*width = */ 0, "25"}, + {37, kHex | kInt | kUpper, /*width = */ 6, "____25"}, + {37, kHex | kInt | kUpper | kPos, /*width = */ 0, "25"}, + {37, kHex | kInt | kUpper | kPos, /*width = */ 6, "____25"}, + {37, kHex | kInt | kUpper | kBase, /*width = */ 0, "0X25"}, + {37, kHex | kInt | kUpper | kBase, /*width = */ 6, "0X__25"}, + {37, kHex | kInt | kUpper | kBase | kPos, /*width = */ 0, "0X25"}, + {37, kHex | kInt | kUpper | kBase | kPos, /*width = */ 6, "0X__25"}, + {37, kHex | kRight, /*width = */ 0, "25"}, + {37, kHex | kRight, /*width = */ 6, "____25"}, + {37, kHex | kRight | kPos, /*width = */ 0, "25"}, + {37, kHex | kRight | kPos, /*width = */ 6, "____25"}, + {37, kHex | kRight | kBase, /*width = */ 0, "0x25"}, + {37, kHex | kRight | kBase, /*width = */ 6, "__0x25"}, + {37, kHex | kRight | kBase | kPos, /*width = */ 0, "0x25"}, + {37, kHex | kRight | kBase | kPos, /*width = */ 6, "__0x25"}, + {37, kHex | kRight | kUpper, /*width = */ 0, "25"}, + {37, kHex | kRight | kUpper, /*width = */ 6, "____25"}, + {37, kHex | kRight | kUpper | kPos, /*width = */ 0, "25"}, + {37, kHex | kRight | kUpper | kPos, /*width = */ 6, "____25"}, + {37, kHex | kRight | kUpper | kBase, /*width = */ 0, "0X25"}, + {37, kHex | kRight | kUpper | kBase, /*width = */ 6, "__0X25"}, + {37, kHex | kRight | kUpper | kBase | kPos, /*width = */ 0, "0X25"}, + {37, kHex | kRight | kUpper | kBase | kPos, /*width = */ 6, "__0X25"}}; +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/numeric/int128_test.cc b/third_party/abseil_cpp/absl/numeric/int128_test.cc new file mode 100644 index 000000000000..bc86c714acce --- /dev/null +++ b/third_party/abseil_cpp/absl/numeric/int128_test.cc @@ -0,0 +1,1225 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/numeric/int128.h" + +#include <algorithm> +#include <limits> +#include <random> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/hash/hash_testing.h" +#include "absl/meta/type_traits.h" + +#if defined(_MSC_VER) && _MSC_VER == 1900 +// Disable "unary minus operator applied to unsigned type" warnings in Microsoft +// Visual C++ 14 (2015). +#pragma warning(disable:4146) +#endif + +namespace { + +template <typename T> +class Uint128IntegerTraitsTest : public ::testing::Test {}; +typedef ::testing::Types<bool, char, signed char, unsigned char, char16_t, + char32_t, wchar_t, + short, // NOLINT(runtime/int) + unsigned short, // NOLINT(runtime/int) + int, unsigned int, + long, // NOLINT(runtime/int) + unsigned long, // NOLINT(runtime/int) + long long, // NOLINT(runtime/int) + unsigned long long> // NOLINT(runtime/int) + IntegerTypes; + +template <typename T> +class Uint128FloatTraitsTest : public ::testing::Test {}; +typedef ::testing::Types<float, double, long double> FloatingPointTypes; + +TYPED_TEST_SUITE(Uint128IntegerTraitsTest, IntegerTypes); + +TYPED_TEST(Uint128IntegerTraitsTest, ConstructAssignTest) { + static_assert(std::is_constructible<absl::uint128, TypeParam>::value, + "absl::uint128 must be constructible from TypeParam"); + static_assert(std::is_assignable<absl::uint128&, TypeParam>::value, + "absl::uint128 must be assignable from TypeParam"); + static_assert(!std::is_assignable<TypeParam&, absl::uint128>::value, + "TypeParam must not be assignable from absl::uint128"); +} + +TYPED_TEST_SUITE(Uint128FloatTraitsTest, FloatingPointTypes); + +TYPED_TEST(Uint128FloatTraitsTest, ConstructAssignTest) { + static_assert(std::is_constructible<absl::uint128, TypeParam>::value, + "absl::uint128 must be constructible from TypeParam"); + static_assert(!std::is_assignable<absl::uint128&, TypeParam>::value, + "absl::uint128 must not be assignable from TypeParam"); + static_assert(!std::is_assignable<TypeParam&, absl::uint128>::value, + "TypeParam must not be assignable from absl::uint128"); +} + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +// These type traits done separately as TYPED_TEST requires typeinfo, and not +// all platforms have this for __int128 even though they define the type. +TEST(Uint128, IntrinsicTypeTraitsTest) { + static_assert(std::is_constructible<absl::uint128, __int128>::value, + "absl::uint128 must be constructible from __int128"); + static_assert(std::is_assignable<absl::uint128&, __int128>::value, + "absl::uint128 must be assignable from __int128"); + static_assert(!std::is_assignable<__int128&, absl::uint128>::value, + "__int128 must not be assignable from absl::uint128"); + + static_assert(std::is_constructible<absl::uint128, unsigned __int128>::value, + "absl::uint128 must be constructible from unsigned __int128"); + static_assert(std::is_assignable<absl::uint128&, unsigned __int128>::value, + "absl::uint128 must be assignable from unsigned __int128"); + static_assert(!std::is_assignable<unsigned __int128&, absl::uint128>::value, + "unsigned __int128 must not be assignable from absl::uint128"); +} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +TEST(Uint128, TrivialTraitsTest) { + static_assert(absl::is_trivially_default_constructible<absl::uint128>::value, + ""); + static_assert(absl::is_trivially_copy_constructible<absl::uint128>::value, + ""); + static_assert(absl::is_trivially_copy_assignable<absl::uint128>::value, ""); + static_assert(std::is_trivially_destructible<absl::uint128>::value, ""); +} + +TEST(Uint128, AllTests) { + absl::uint128 zero = 0; + absl::uint128 one = 1; + absl::uint128 one_2arg = absl::MakeUint128(0, 1); + absl::uint128 two = 2; + absl::uint128 three = 3; + absl::uint128 big = absl::MakeUint128(2000, 2); + absl::uint128 big_minus_one = absl::MakeUint128(2000, 1); + absl::uint128 bigger = absl::MakeUint128(2001, 1); + absl::uint128 biggest = absl::Uint128Max(); + absl::uint128 high_low = absl::MakeUint128(1, 0); + absl::uint128 low_high = + absl::MakeUint128(0, std::numeric_limits<uint64_t>::max()); + EXPECT_LT(one, two); + EXPECT_GT(two, one); + EXPECT_LT(one, big); + EXPECT_LT(one, big); + EXPECT_EQ(one, one_2arg); + EXPECT_NE(one, two); + EXPECT_GT(big, one); + EXPECT_GE(big, two); + EXPECT_GE(big, big_minus_one); + EXPECT_GT(big, big_minus_one); + EXPECT_LT(big_minus_one, big); + EXPECT_LE(big_minus_one, big); + EXPECT_NE(big_minus_one, big); + EXPECT_LT(big, biggest); + EXPECT_LE(big, biggest); + EXPECT_GT(biggest, big); + EXPECT_GE(biggest, big); + EXPECT_EQ(big, ~~big); + EXPECT_EQ(one, one | one); + EXPECT_EQ(big, big | big); + EXPECT_EQ(one, one | zero); + EXPECT_EQ(one, one & one); + EXPECT_EQ(big, big & big); + EXPECT_EQ(zero, one & zero); + EXPECT_EQ(zero, big & ~big); + EXPECT_EQ(zero, one ^ one); + EXPECT_EQ(zero, big ^ big); + EXPECT_EQ(one, one ^ zero); + + // Shift operators. + EXPECT_EQ(big, big << 0); + EXPECT_EQ(big, big >> 0); + EXPECT_GT(big << 1, big); + EXPECT_LT(big >> 1, big); + EXPECT_EQ(big, (big << 10) >> 10); + EXPECT_EQ(big, (big >> 1) << 1); + EXPECT_EQ(one, (one << 80) >> 80); + EXPECT_EQ(zero, (one >> 80) << 80); + + // Shift assignments. + absl::uint128 big_copy = big; + EXPECT_EQ(big << 0, big_copy <<= 0); + big_copy = big; + EXPECT_EQ(big >> 0, big_copy >>= 0); + big_copy = big; + EXPECT_EQ(big << 1, big_copy <<= 1); + big_copy = big; + EXPECT_EQ(big >> 1, big_copy >>= 1); + big_copy = big; + EXPECT_EQ(big << 10, big_copy <<= 10); + big_copy = big; + EXPECT_EQ(big >> 10, big_copy >>= 10); + big_copy = big; + EXPECT_EQ(big << 64, big_copy <<= 64); + big_copy = big; + EXPECT_EQ(big >> 64, big_copy >>= 64); + big_copy = big; + EXPECT_EQ(big << 73, big_copy <<= 73); + big_copy = big; + EXPECT_EQ(big >> 73, big_copy >>= 73); + + EXPECT_EQ(absl::Uint128High64(biggest), std::numeric_limits<uint64_t>::max()); + EXPECT_EQ(absl::Uint128Low64(biggest), std::numeric_limits<uint64_t>::max()); + EXPECT_EQ(zero + one, one); + EXPECT_EQ(one + one, two); + EXPECT_EQ(big_minus_one + one, big); + EXPECT_EQ(one - one, zero); + EXPECT_EQ(one - zero, one); + EXPECT_EQ(zero - one, biggest); + EXPECT_EQ(big - big, zero); + EXPECT_EQ(big - one, big_minus_one); + EXPECT_EQ(big + std::numeric_limits<uint64_t>::max(), bigger); + EXPECT_EQ(biggest + 1, zero); + EXPECT_EQ(zero - 1, biggest); + EXPECT_EQ(high_low - one, low_high); + EXPECT_EQ(low_high + one, high_low); + EXPECT_EQ(absl::Uint128High64((absl::uint128(1) << 64) - 1), 0); + EXPECT_EQ(absl::Uint128Low64((absl::uint128(1) << 64) - 1), + std::numeric_limits<uint64_t>::max()); + EXPECT_TRUE(!!one); + EXPECT_TRUE(!!high_low); + EXPECT_FALSE(!!zero); + EXPECT_FALSE(!one); + EXPECT_FALSE(!high_low); + EXPECT_TRUE(!zero); + EXPECT_TRUE(zero == 0); // NOLINT(readability/check) + EXPECT_FALSE(zero != 0); // NOLINT(readability/check) + EXPECT_FALSE(one == 0); // NOLINT(readability/check) + EXPECT_TRUE(one != 0); // NOLINT(readability/check) + EXPECT_FALSE(high_low == 0); // NOLINT(readability/check) + EXPECT_TRUE(high_low != 0); // NOLINT(readability/check) + + absl::uint128 test = zero; + EXPECT_EQ(++test, one); + EXPECT_EQ(test, one); + EXPECT_EQ(test++, one); + EXPECT_EQ(test, two); + EXPECT_EQ(test -= 2, zero); + EXPECT_EQ(test, zero); + EXPECT_EQ(test += 2, two); + EXPECT_EQ(test, two); + EXPECT_EQ(--test, one); + EXPECT_EQ(test, one); + EXPECT_EQ(test--, one); + EXPECT_EQ(test, zero); + EXPECT_EQ(test |= three, three); + EXPECT_EQ(test &= one, one); + EXPECT_EQ(test ^= three, two); + EXPECT_EQ(test >>= 1, one); + EXPECT_EQ(test <<= 1, two); + + EXPECT_EQ(big, -(-big)); + EXPECT_EQ(two, -((-one) - 1)); + EXPECT_EQ(absl::Uint128Max(), -one); + EXPECT_EQ(zero, -zero); + + EXPECT_EQ(absl::Uint128Max(), absl::kuint128max); +} + +TEST(Uint128, ConversionTests) { + EXPECT_TRUE(absl::MakeUint128(1, 0)); + +#ifdef ABSL_HAVE_INTRINSIC_INT128 + unsigned __int128 intrinsic = + (static_cast<unsigned __int128>(0x3a5b76c209de76f6) << 64) + + 0x1f25e1d63a2b46c5; + absl::uint128 custom = + absl::MakeUint128(0x3a5b76c209de76f6, 0x1f25e1d63a2b46c5); + + EXPECT_EQ(custom, absl::uint128(intrinsic)); + EXPECT_EQ(custom, absl::uint128(static_cast<__int128>(intrinsic))); + EXPECT_EQ(intrinsic, static_cast<unsigned __int128>(custom)); + EXPECT_EQ(intrinsic, static_cast<__int128>(custom)); +#endif // ABSL_HAVE_INTRINSIC_INT128 + + // verify that an integer greater than 2**64 that can be stored precisely + // inside a double is converted to a absl::uint128 without loss of + // information. + double precise_double = 0x530e * std::pow(2.0, 64.0) + 0xda74000000000000; + absl::uint128 from_precise_double(precise_double); + absl::uint128 from_precise_ints = + absl::MakeUint128(0x530e, 0xda74000000000000); + EXPECT_EQ(from_precise_double, from_precise_ints); + EXPECT_DOUBLE_EQ(static_cast<double>(from_precise_ints), precise_double); + + double approx_double = 0xffffeeeeddddcccc * std::pow(2.0, 64.0) + + 0xbbbbaaaa99998888; + absl::uint128 from_approx_double(approx_double); + EXPECT_DOUBLE_EQ(static_cast<double>(from_approx_double), approx_double); + + double round_to_zero = 0.7; + double round_to_five = 5.8; + double round_to_nine = 9.3; + EXPECT_EQ(static_cast<absl::uint128>(round_to_zero), 0); + EXPECT_EQ(static_cast<absl::uint128>(round_to_five), 5); + EXPECT_EQ(static_cast<absl::uint128>(round_to_nine), 9); + + absl::uint128 highest_precision_in_long_double = + ~absl::uint128{} >> (128 - std::numeric_limits<long double>::digits); + EXPECT_EQ(highest_precision_in_long_double, + static_cast<absl::uint128>( + static_cast<long double>(highest_precision_in_long_double))); + // Apply a mask just to make sure all the bits are the right place. + const absl::uint128 arbitrary_mask = + absl::MakeUint128(0xa29f622677ded751, 0xf8ca66add076f468); + EXPECT_EQ(highest_precision_in_long_double & arbitrary_mask, + static_cast<absl::uint128>(static_cast<long double>( + highest_precision_in_long_double & arbitrary_mask))); + + EXPECT_EQ(static_cast<absl::uint128>(-0.1L), 0); +} + +TEST(Uint128, OperatorAssignReturnRef) { + absl::uint128 v(1); + (v += 4) -= 3; + EXPECT_EQ(2, v); +} + +TEST(Uint128, Multiply) { + absl::uint128 a, b, c; + + // Zero test. + a = 0; + b = 0; + c = a * b; + EXPECT_EQ(0, c); + + // Max carries. + a = absl::uint128(0) - 1; + b = absl::uint128(0) - 1; + c = a * b; + EXPECT_EQ(1, c); + + // Self-operation with max carries. + c = absl::uint128(0) - 1; + c *= c; + EXPECT_EQ(1, c); + + // 1-bit x 1-bit. + for (int i = 0; i < 64; ++i) { + for (int j = 0; j < 64; ++j) { + a = absl::uint128(1) << i; + b = absl::uint128(1) << j; + c = a * b; + EXPECT_EQ(absl::uint128(1) << (i + j), c); + } + } + + // Verified with dc. + a = absl::MakeUint128(0xffffeeeeddddcccc, 0xbbbbaaaa99998888); + b = absl::MakeUint128(0x7777666655554444, 0x3333222211110000); + c = a * b; + EXPECT_EQ(absl::MakeUint128(0x530EDA741C71D4C3, 0xBF25975319080000), c); + EXPECT_EQ(0, c - b * a); + EXPECT_EQ(a*a - b*b, (a+b) * (a-b)); + + // Verified with dc. + a = absl::MakeUint128(0x0123456789abcdef, 0xfedcba9876543210); + b = absl::MakeUint128(0x02468ace13579bdf, 0xfdb97531eca86420); + c = a * b; + EXPECT_EQ(absl::MakeUint128(0x97a87f4f261ba3f2, 0x342d0bbf48948200), c); + EXPECT_EQ(0, c - b * a); + EXPECT_EQ(a*a - b*b, (a+b) * (a-b)); +} + +TEST(Uint128, AliasTests) { + absl::uint128 x1 = absl::MakeUint128(1, 2); + absl::uint128 x2 = absl::MakeUint128(2, 4); + x1 += x1; + EXPECT_EQ(x2, x1); + + absl::uint128 x3 = absl::MakeUint128(1, static_cast<uint64_t>(1) << 63); + absl::uint128 x4 = absl::MakeUint128(3, 0); + x3 += x3; + EXPECT_EQ(x4, x3); +} + +TEST(Uint128, DivideAndMod) { + using std::swap; + + // a := q * b + r + absl::uint128 a, b, q, r; + + // Zero test. + a = 0; + b = 123; + q = a / b; + r = a % b; + EXPECT_EQ(0, q); + EXPECT_EQ(0, r); + + a = absl::MakeUint128(0x530eda741c71d4c3, 0xbf25975319080000); + q = absl::MakeUint128(0x4de2cab081, 0x14c34ab4676e4bab); + b = absl::uint128(0x1110001); + r = absl::uint128(0x3eb455); + ASSERT_EQ(a, q * b + r); // Sanity-check. + + absl::uint128 result_q, result_r; + result_q = a / b; + result_r = a % b; + EXPECT_EQ(q, result_q); + EXPECT_EQ(r, result_r); + + // Try the other way around. + swap(q, b); + result_q = a / b; + result_r = a % b; + EXPECT_EQ(q, result_q); + EXPECT_EQ(r, result_r); + // Restore. + swap(b, q); + + // Dividend < divisor; result should be q:0 r:<dividend>. + swap(a, b); + result_q = a / b; + result_r = a % b; + EXPECT_EQ(0, result_q); + EXPECT_EQ(a, result_r); + // Try the other way around. + swap(a, q); + result_q = a / b; + result_r = a % b; + EXPECT_EQ(0, result_q); + EXPECT_EQ(a, result_r); + // Restore. + swap(q, a); + swap(b, a); + + // Try a large remainder. + b = a / 2 + 1; + absl::uint128 expected_r = + absl::MakeUint128(0x29876d3a0e38ea61, 0xdf92cba98c83ffff); + // Sanity checks. + ASSERT_EQ(a / 2 - 1, expected_r); + ASSERT_EQ(a, b + expected_r); + result_q = a / b; + result_r = a % b; + EXPECT_EQ(1, result_q); + EXPECT_EQ(expected_r, result_r); +} + +TEST(Uint128, DivideAndModRandomInputs) { + const int kNumIters = 1 << 18; + std::minstd_rand random(testing::UnitTest::GetInstance()->random_seed()); + std::uniform_int_distribution<uint64_t> uniform_uint64; + for (int i = 0; i < kNumIters; ++i) { + const absl::uint128 a = + absl::MakeUint128(uniform_uint64(random), uniform_uint64(random)); + const absl::uint128 b = + absl::MakeUint128(uniform_uint64(random), uniform_uint64(random)); + if (b == 0) { + continue; // Avoid a div-by-zero. + } + const absl::uint128 q = a / b; + const absl::uint128 r = a % b; + ASSERT_EQ(a, b * q + r); + } +} + +TEST(Uint128, ConstexprTest) { + constexpr absl::uint128 zero = absl::uint128(); + constexpr absl::uint128 one = 1; + constexpr absl::uint128 minus_two = -2; + EXPECT_EQ(zero, absl::uint128(0)); + EXPECT_EQ(one, absl::uint128(1)); + EXPECT_EQ(minus_two, absl::MakeUint128(-1, -2)); +} + +TEST(Uint128, NumericLimitsTest) { + static_assert(std::numeric_limits<absl::uint128>::is_specialized, ""); + static_assert(!std::numeric_limits<absl::uint128>::is_signed, ""); + static_assert(std::numeric_limits<absl::uint128>::is_integer, ""); + EXPECT_EQ(static_cast<int>(128 * std::log10(2)), + std::numeric_limits<absl::uint128>::digits10); + EXPECT_EQ(0, std::numeric_limits<absl::uint128>::min()); + EXPECT_EQ(0, std::numeric_limits<absl::uint128>::lowest()); + EXPECT_EQ(absl::Uint128Max(), std::numeric_limits<absl::uint128>::max()); +} + +TEST(Uint128, Hash) { + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly({ + // Some simple values + absl::uint128{0}, + absl::uint128{1}, + ~absl::uint128{}, + // 64 bit limits + absl::uint128{std::numeric_limits<int64_t>::max()}, + absl::uint128{std::numeric_limits<uint64_t>::max()} + 0, + absl::uint128{std::numeric_limits<uint64_t>::max()} + 1, + absl::uint128{std::numeric_limits<uint64_t>::max()} + 2, + // Keeping high same + absl::uint128{1} << 62, + absl::uint128{1} << 63, + // Keeping low same + absl::uint128{1} << 64, + absl::uint128{1} << 65, + // 128 bit limits + std::numeric_limits<absl::uint128>::max(), + std::numeric_limits<absl::uint128>::max() - 1, + std::numeric_limits<absl::uint128>::min() + 1, + std::numeric_limits<absl::uint128>::min(), + })); +} + + +TEST(Int128Uint128, ConversionTest) { + absl::int128 nonnegative_signed_values[] = { + 0, + 1, + 0xffeeddccbbaa9988, + absl::MakeInt128(0x7766554433221100, 0), + absl::MakeInt128(0x1234567890abcdef, 0xfedcba0987654321), + absl::Int128Max()}; + for (absl::int128 value : nonnegative_signed_values) { + EXPECT_EQ(value, absl::int128(absl::uint128(value))); + + absl::uint128 assigned_value; + assigned_value = value; + EXPECT_EQ(value, absl::int128(assigned_value)); + } + + absl::int128 negative_values[] = { + -1, -0x1234567890abcdef, + absl::MakeInt128(-0x5544332211ffeedd, 0), + -absl::MakeInt128(0x76543210fedcba98, 0xabcdef0123456789)}; + for (absl::int128 value : negative_values) { + EXPECT_EQ(absl::uint128(-value), -absl::uint128(value)); + + absl::uint128 assigned_value; + assigned_value = value; + EXPECT_EQ(absl::uint128(-value), -assigned_value); + } +} + +template <typename T> +class Int128IntegerTraitsTest : public ::testing::Test {}; + +TYPED_TEST_SUITE(Int128IntegerTraitsTest, IntegerTypes); + +TYPED_TEST(Int128IntegerTraitsTest, ConstructAssignTest) { + static_assert(std::is_constructible<absl::int128, TypeParam>::value, + "absl::int128 must be constructible from TypeParam"); + static_assert(std::is_assignable<absl::int128&, TypeParam>::value, + "absl::int128 must be assignable from TypeParam"); + static_assert(!std::is_assignable<TypeParam&, absl::int128>::value, + "TypeParam must not be assignable from absl::int128"); +} + +template <typename T> +class Int128FloatTraitsTest : public ::testing::Test {}; + +TYPED_TEST_SUITE(Int128FloatTraitsTest, FloatingPointTypes); + +TYPED_TEST(Int128FloatTraitsTest, ConstructAssignTest) { + static_assert(std::is_constructible<absl::int128, TypeParam>::value, + "absl::int128 must be constructible from TypeParam"); + static_assert(!std::is_assignable<absl::int128&, TypeParam>::value, + "absl::int128 must not be assignable from TypeParam"); + static_assert(!std::is_assignable<TypeParam&, absl::int128>::value, + "TypeParam must not be assignable from absl::int128"); +} + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +// These type traits done separately as TYPED_TEST requires typeinfo, and not +// all platforms have this for __int128 even though they define the type. +TEST(Int128, IntrinsicTypeTraitsTest) { + static_assert(std::is_constructible<absl::int128, __int128>::value, + "absl::int128 must be constructible from __int128"); + static_assert(std::is_assignable<absl::int128&, __int128>::value, + "absl::int128 must be assignable from __int128"); + static_assert(!std::is_assignable<__int128&, absl::int128>::value, + "__int128 must not be assignable from absl::int128"); + + static_assert(std::is_constructible<absl::int128, unsigned __int128>::value, + "absl::int128 must be constructible from unsigned __int128"); + static_assert(!std::is_assignable<absl::int128&, unsigned __int128>::value, + "absl::int128 must be assignable from unsigned __int128"); + static_assert(!std::is_assignable<unsigned __int128&, absl::int128>::value, + "unsigned __int128 must not be assignable from absl::int128"); +} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +TEST(Int128, TrivialTraitsTest) { + static_assert(absl::is_trivially_default_constructible<absl::int128>::value, + ""); + static_assert(absl::is_trivially_copy_constructible<absl::int128>::value, ""); + static_assert(absl::is_trivially_copy_assignable<absl::int128>::value, ""); + static_assert(std::is_trivially_destructible<absl::int128>::value, ""); +} + +TEST(Int128, BoolConversionTest) { + EXPECT_FALSE(absl::int128(0)); + for (int i = 0; i < 64; ++i) { + EXPECT_TRUE(absl::MakeInt128(0, uint64_t{1} << i)); + } + for (int i = 0; i < 63; ++i) { + EXPECT_TRUE(absl::MakeInt128(int64_t{1} << i, 0)); + } + EXPECT_TRUE(absl::Int128Min()); + + EXPECT_EQ(absl::int128(1), absl::int128(true)); + EXPECT_EQ(absl::int128(0), absl::int128(false)); +} + +template <typename T> +class Int128IntegerConversionTest : public ::testing::Test {}; + +TYPED_TEST_SUITE(Int128IntegerConversionTest, IntegerTypes); + +TYPED_TEST(Int128IntegerConversionTest, RoundTripTest) { + EXPECT_EQ(TypeParam{0}, static_cast<TypeParam>(absl::int128(0))); + EXPECT_EQ(std::numeric_limits<TypeParam>::min(), + static_cast<TypeParam>( + absl::int128(std::numeric_limits<TypeParam>::min()))); + EXPECT_EQ(std::numeric_limits<TypeParam>::max(), + static_cast<TypeParam>( + absl::int128(std::numeric_limits<TypeParam>::max()))); +} + +template <typename T> +class Int128FloatConversionTest : public ::testing::Test {}; + +TYPED_TEST_SUITE(Int128FloatConversionTest, FloatingPointTypes); + +TYPED_TEST(Int128FloatConversionTest, ConstructAndCastTest) { + // Conversions where the floating point values should be exactly the same. + // 0x9f5b is a randomly chosen small value. + for (int i = 0; i < 110; ++i) { // 110 = 126 - #bits in 0x9f5b + SCOPED_TRACE(::testing::Message() << "i = " << i); + + TypeParam float_value = std::ldexp(static_cast<TypeParam>(0x9f5b), i); + absl::int128 int_value = absl::int128(0x9f5b) << i; + + EXPECT_EQ(float_value, static_cast<TypeParam>(int_value)); + EXPECT_EQ(-float_value, static_cast<TypeParam>(-int_value)); + EXPECT_EQ(int_value, absl::int128(float_value)); + EXPECT_EQ(-int_value, absl::int128(-float_value)); + } + + // Round trip conversions with a small sample of randomly generated uint64_t + // values (less than int64_t max so that value * 2^64 fits into int128). + uint64_t values[] = {0x6d4492c24fb86199, 0x26ead65e4cb359b5, + 0x2c43407433ba3fd1, 0x3b574ec668df6b55, + 0x1c750e55a29f4f0f}; + for (uint64_t value : values) { + for (int i = 0; i <= 64; ++i) { + SCOPED_TRACE(::testing::Message() + << "value = " << value << "; i = " << i); + + TypeParam fvalue = std::ldexp(static_cast<TypeParam>(value), i); + EXPECT_DOUBLE_EQ(fvalue, static_cast<TypeParam>(absl::int128(fvalue))); + EXPECT_DOUBLE_EQ(-fvalue, static_cast<TypeParam>(-absl::int128(fvalue))); + EXPECT_DOUBLE_EQ(-fvalue, static_cast<TypeParam>(absl::int128(-fvalue))); + EXPECT_DOUBLE_EQ(fvalue, static_cast<TypeParam>(-absl::int128(-fvalue))); + } + } + + // Round trip conversions with a small sample of random large positive values. + absl::int128 large_values[] = { + absl::MakeInt128(0x5b0640d96c7b3d9f, 0xb7a7189e51d18622), + absl::MakeInt128(0x34bed042c6f65270, 0x73b236570669a089), + absl::MakeInt128(0x43deba9e6da12724, 0xf7f0f83da686797d), + absl::MakeInt128(0x71e8d383be4e5589, 0x75c3f96fb00752b6)}; + for (absl::int128 value : large_values) { + // Make value have as many significant bits as can be represented by + // the mantissa, also making sure the highest and lowest bit in the range + // are set. + value >>= (127 - std::numeric_limits<TypeParam>::digits); + value |= absl::int128(1) << (std::numeric_limits<TypeParam>::digits - 1); + value |= 1; + for (int i = 0; i < 127 - std::numeric_limits<TypeParam>::digits; ++i) { + absl::int128 int_value = value << i; + EXPECT_EQ(int_value, + static_cast<absl::int128>(static_cast<TypeParam>(int_value))); + EXPECT_EQ(-int_value, + static_cast<absl::int128>(static_cast<TypeParam>(-int_value))); + } + } + + // Small sample of checks that rounding is toward zero + EXPECT_EQ(0, absl::int128(TypeParam(0.1))); + EXPECT_EQ(17, absl::int128(TypeParam(17.8))); + EXPECT_EQ(0, absl::int128(TypeParam(-0.8))); + EXPECT_EQ(-53, absl::int128(TypeParam(-53.1))); + EXPECT_EQ(0, absl::int128(TypeParam(0.5))); + EXPECT_EQ(0, absl::int128(TypeParam(-0.5))); + TypeParam just_lt_one = std::nexttoward(TypeParam(1), TypeParam(0)); + EXPECT_EQ(0, absl::int128(just_lt_one)); + TypeParam just_gt_minus_one = std::nexttoward(TypeParam(-1), TypeParam(0)); + EXPECT_EQ(0, absl::int128(just_gt_minus_one)); + + // Check limits + EXPECT_DOUBLE_EQ(std::ldexp(static_cast<TypeParam>(1), 127), + static_cast<TypeParam>(absl::Int128Max())); + EXPECT_DOUBLE_EQ(-std::ldexp(static_cast<TypeParam>(1), 127), + static_cast<TypeParam>(absl::Int128Min())); +} + +TEST(Int128, FactoryTest) { + EXPECT_EQ(absl::int128(-1), absl::MakeInt128(-1, -1)); + EXPECT_EQ(absl::int128(-31), absl::MakeInt128(-1, -31)); + EXPECT_EQ(absl::int128(std::numeric_limits<int64_t>::min()), + absl::MakeInt128(-1, std::numeric_limits<int64_t>::min())); + EXPECT_EQ(absl::int128(0), absl::MakeInt128(0, 0)); + EXPECT_EQ(absl::int128(1), absl::MakeInt128(0, 1)); + EXPECT_EQ(absl::int128(std::numeric_limits<int64_t>::max()), + absl::MakeInt128(0, std::numeric_limits<int64_t>::max())); +} + +TEST(Int128, HighLowTest) { + struct HighLowPair { + int64_t high; + uint64_t low; + }; + HighLowPair values[]{{0, 0}, {0, 1}, {1, 0}, {123, 456}, {-654, 321}}; + for (const HighLowPair& pair : values) { + absl::int128 value = absl::MakeInt128(pair.high, pair.low); + EXPECT_EQ(pair.low, absl::Int128Low64(value)); + EXPECT_EQ(pair.high, absl::Int128High64(value)); + } +} + +TEST(Int128, LimitsTest) { + EXPECT_EQ(absl::MakeInt128(0x7fffffffffffffff, 0xffffffffffffffff), + absl::Int128Max()); + EXPECT_EQ(absl::Int128Max(), ~absl::Int128Min()); +} + +#if defined(ABSL_HAVE_INTRINSIC_INT128) +TEST(Int128, IntrinsicConversionTest) { + __int128 intrinsic = + (static_cast<__int128>(0x3a5b76c209de76f6) << 64) + 0x1f25e1d63a2b46c5; + absl::int128 custom = + absl::MakeInt128(0x3a5b76c209de76f6, 0x1f25e1d63a2b46c5); + + EXPECT_EQ(custom, absl::int128(intrinsic)); + EXPECT_EQ(intrinsic, static_cast<__int128>(custom)); +} +#endif // ABSL_HAVE_INTRINSIC_INT128 + +TEST(Int128, ConstexprTest) { + constexpr absl::int128 zero = absl::int128(); + constexpr absl::int128 one = 1; + constexpr absl::int128 minus_two = -2; + constexpr absl::int128 min = absl::Int128Min(); + constexpr absl::int128 max = absl::Int128Max(); + EXPECT_EQ(zero, absl::int128(0)); + EXPECT_EQ(one, absl::int128(1)); + EXPECT_EQ(minus_two, absl::MakeInt128(-1, -2)); + EXPECT_GT(max, one); + EXPECT_LT(min, minus_two); +} + +TEST(Int128, ComparisonTest) { + struct TestCase { + absl::int128 smaller; + absl::int128 larger; + }; + TestCase cases[] = { + {absl::int128(0), absl::int128(123)}, + {absl::MakeInt128(-12, 34), absl::MakeInt128(12, 34)}, + {absl::MakeInt128(1, 1000), absl::MakeInt128(1000, 1)}, + {absl::MakeInt128(-1000, 1000), absl::MakeInt128(-1, 1)}, + }; + for (const TestCase& pair : cases) { + SCOPED_TRACE(::testing::Message() << "pair.smaller = " << pair.smaller + << "; pair.larger = " << pair.larger); + + EXPECT_TRUE(pair.smaller == pair.smaller); // NOLINT(readability/check) + EXPECT_TRUE(pair.larger == pair.larger); // NOLINT(readability/check) + EXPECT_FALSE(pair.smaller == pair.larger); // NOLINT(readability/check) + + EXPECT_TRUE(pair.smaller != pair.larger); // NOLINT(readability/check) + EXPECT_FALSE(pair.smaller != pair.smaller); // NOLINT(readability/check) + EXPECT_FALSE(pair.larger != pair.larger); // NOLINT(readability/check) + + EXPECT_TRUE(pair.smaller < pair.larger); // NOLINT(readability/check) + EXPECT_FALSE(pair.larger < pair.smaller); // NOLINT(readability/check) + + EXPECT_TRUE(pair.larger > pair.smaller); // NOLINT(readability/check) + EXPECT_FALSE(pair.smaller > pair.larger); // NOLINT(readability/check) + + EXPECT_TRUE(pair.smaller <= pair.larger); // NOLINT(readability/check) + EXPECT_FALSE(pair.larger <= pair.smaller); // NOLINT(readability/check) + EXPECT_TRUE(pair.smaller <= pair.smaller); // NOLINT(readability/check) + EXPECT_TRUE(pair.larger <= pair.larger); // NOLINT(readability/check) + + EXPECT_TRUE(pair.larger >= pair.smaller); // NOLINT(readability/check) + EXPECT_FALSE(pair.smaller >= pair.larger); // NOLINT(readability/check) + EXPECT_TRUE(pair.smaller >= pair.smaller); // NOLINT(readability/check) + EXPECT_TRUE(pair.larger >= pair.larger); // NOLINT(readability/check) + } +} + +TEST(Int128, UnaryNegationTest) { + int64_t values64[] = {0, 1, 12345, 0x4000000000000000, + std::numeric_limits<int64_t>::max()}; + for (int64_t value : values64) { + SCOPED_TRACE(::testing::Message() << "value = " << value); + + EXPECT_EQ(absl::int128(-value), -absl::int128(value)); + EXPECT_EQ(absl::int128(value), -absl::int128(-value)); + EXPECT_EQ(absl::MakeInt128(-value, 0), -absl::MakeInt128(value, 0)); + EXPECT_EQ(absl::MakeInt128(value, 0), -absl::MakeInt128(-value, 0)); + } +} + +TEST(Int128, LogicalNotTest) { + EXPECT_TRUE(!absl::int128(0)); + for (int i = 0; i < 64; ++i) { + EXPECT_FALSE(!absl::MakeInt128(0, uint64_t{1} << i)); + } + for (int i = 0; i < 63; ++i) { + EXPECT_FALSE(!absl::MakeInt128(int64_t{1} << i, 0)); + } +} + +TEST(Int128, AdditionSubtractionTest) { + // 64 bit pairs that will not cause overflow / underflow. These test negative + // carry; positive carry must be checked separately. + std::pair<int64_t, int64_t> cases[]{ + {0, 0}, // 0, 0 + {0, 2945781290834}, // 0, + + {1908357619234, 0}, // +, 0 + {0, -1204895918245}, // 0, - + {-2957928523560, 0}, // -, 0 + {89023982312461, 98346012567134}, // +, + + {-63454234568239, -23456235230773}, // -, - + {98263457263502, -21428561935925}, // +, - + {-88235237438467, 15923659234573}, // -, + + }; + for (const auto& pair : cases) { + SCOPED_TRACE(::testing::Message() + << "pair = {" << pair.first << ", " << pair.second << '}'); + + EXPECT_EQ(absl::int128(pair.first + pair.second), + absl::int128(pair.first) + absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.second + pair.first), + absl::int128(pair.second) += absl::int128(pair.first)); + + EXPECT_EQ(absl::int128(pair.first - pair.second), + absl::int128(pair.first) - absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.second - pair.first), + absl::int128(pair.second) -= absl::int128(pair.first)); + + EXPECT_EQ( + absl::MakeInt128(pair.second + pair.first, 0), + absl::MakeInt128(pair.second, 0) + absl::MakeInt128(pair.first, 0)); + EXPECT_EQ( + absl::MakeInt128(pair.first + pair.second, 0), + absl::MakeInt128(pair.first, 0) += absl::MakeInt128(pair.second, 0)); + + EXPECT_EQ( + absl::MakeInt128(pair.second - pair.first, 0), + absl::MakeInt128(pair.second, 0) - absl::MakeInt128(pair.first, 0)); + EXPECT_EQ( + absl::MakeInt128(pair.first - pair.second, 0), + absl::MakeInt128(pair.first, 0) -= absl::MakeInt128(pair.second, 0)); + } + + // check positive carry + EXPECT_EQ(absl::MakeInt128(31, 0), + absl::MakeInt128(20, 1) + + absl::MakeInt128(10, std::numeric_limits<uint64_t>::max())); +} + +TEST(Int128, IncrementDecrementTest) { + absl::int128 value = 0; + EXPECT_EQ(0, value++); + EXPECT_EQ(1, value); + EXPECT_EQ(1, value--); + EXPECT_EQ(0, value); + EXPECT_EQ(-1, --value); + EXPECT_EQ(-1, value); + EXPECT_EQ(0, ++value); + EXPECT_EQ(0, value); +} + +TEST(Int128, MultiplicationTest) { + // 1 bit x 1 bit, and negative combinations + for (int i = 0; i < 64; ++i) { + for (int j = 0; j < 127 - i; ++j) { + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + absl::int128 a = absl::int128(1) << i; + absl::int128 b = absl::int128(1) << j; + absl::int128 c = absl::int128(1) << (i + j); + + EXPECT_EQ(c, a * b); + EXPECT_EQ(-c, -a * b); + EXPECT_EQ(-c, a * -b); + EXPECT_EQ(c, -a * -b); + + EXPECT_EQ(c, absl::int128(a) *= b); + EXPECT_EQ(-c, absl::int128(-a) *= b); + EXPECT_EQ(-c, absl::int128(a) *= -b); + EXPECT_EQ(c, absl::int128(-a) *= -b); + } + } + + // Pairs of random values that will not overflow signed 64-bit multiplication + std::pair<int64_t, int64_t> small_values[] = { + {0x5e61, 0xf29f79ca14b4}, // +, + + {0x3e033b, -0x612c0ee549}, // +, - + {-0x052ce7e8, 0x7c728f0f}, // -, + + {-0x3af7054626, -0xfb1e1d}, // -, - + }; + for (const std::pair<int64_t, int64_t>& pair : small_values) { + SCOPED_TRACE(::testing::Message() + << "pair = {" << pair.first << ", " << pair.second << '}'); + + EXPECT_EQ(absl::int128(pair.first * pair.second), + absl::int128(pair.first) * absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.first * pair.second), + absl::int128(pair.first) *= absl::int128(pair.second)); + + EXPECT_EQ(absl::MakeInt128(pair.first * pair.second, 0), + absl::MakeInt128(pair.first, 0) * absl::int128(pair.second)); + EXPECT_EQ(absl::MakeInt128(pair.first * pair.second, 0), + absl::MakeInt128(pair.first, 0) *= absl::int128(pair.second)); + } + + // Pairs of positive random values that will not overflow 64-bit + // multiplication and can be left shifted by 32 without overflow + std::pair<int64_t, int64_t> small_values2[] = { + {0x1bb0a110, 0x31487671}, + {0x4792784e, 0x28add7d7}, + {0x7b66553a, 0x11dff8ef}, + }; + for (const std::pair<int64_t, int64_t>& pair : small_values2) { + SCOPED_TRACE(::testing::Message() + << "pair = {" << pair.first << ", " << pair.second << '}'); + + absl::int128 a = absl::int128(pair.first << 32); + absl::int128 b = absl::int128(pair.second << 32); + absl::int128 c = absl::MakeInt128(pair.first * pair.second, 0); + + EXPECT_EQ(c, a * b); + EXPECT_EQ(-c, -a * b); + EXPECT_EQ(-c, a * -b); + EXPECT_EQ(c, -a * -b); + + EXPECT_EQ(c, absl::int128(a) *= b); + EXPECT_EQ(-c, absl::int128(-a) *= b); + EXPECT_EQ(-c, absl::int128(a) *= -b); + EXPECT_EQ(c, absl::int128(-a) *= -b); + } + + // check 0, 1, and -1 behavior with large values + absl::int128 large_values[] = { + {absl::MakeInt128(0xd66f061af02d0408, 0x727d2846cb475b53)}, + {absl::MakeInt128(0x27b8d5ed6104452d, 0x03f8a33b0ee1df4f)}, + {-absl::MakeInt128(0x621b6626b9e8d042, 0x27311ac99df00938)}, + {-absl::MakeInt128(0x34e0656f1e95fb60, 0x4281cfd731257a47)}, + }; + for (absl::int128 value : large_values) { + EXPECT_EQ(0, 0 * value); + EXPECT_EQ(0, value * 0); + EXPECT_EQ(0, absl::int128(0) *= value); + EXPECT_EQ(0, value *= 0); + + EXPECT_EQ(value, 1 * value); + EXPECT_EQ(value, value * 1); + EXPECT_EQ(value, absl::int128(1) *= value); + EXPECT_EQ(value, value *= 1); + + EXPECT_EQ(-value, -1 * value); + EXPECT_EQ(-value, value * -1); + EXPECT_EQ(-value, absl::int128(-1) *= value); + EXPECT_EQ(-value, value *= -1); + } + + // Manually calculated random large value cases + EXPECT_EQ(absl::MakeInt128(0xcd0efd3442219bb, 0xde47c05bcd9df6e1), + absl::MakeInt128(0x7c6448, 0x3bc4285c47a9d253) * 0x1a6037537b); + EXPECT_EQ(-absl::MakeInt128(0x1f8f149850b1e5e6, 0x1e50d6b52d272c3e), + -absl::MakeInt128(0x23, 0x2e68a513ca1b8859) * 0xe5a434cd14866e); + EXPECT_EQ(-absl::MakeInt128(0x55cae732029d1fce, 0xca6474b6423263e4), + 0xa9b98a8ddf66bc * -absl::MakeInt128(0x81, 0x672e58231e2469d7)); + EXPECT_EQ(absl::MakeInt128(0x19c8b7620b507dc4, 0xfec042b71a5f29a4), + -0x3e39341147 * -absl::MakeInt128(0x6a14b2, 0x5ed34cca42327b3c)); + + EXPECT_EQ(absl::MakeInt128(0xcd0efd3442219bb, 0xde47c05bcd9df6e1), + absl::MakeInt128(0x7c6448, 0x3bc4285c47a9d253) *= 0x1a6037537b); + EXPECT_EQ(-absl::MakeInt128(0x1f8f149850b1e5e6, 0x1e50d6b52d272c3e), + -absl::MakeInt128(0x23, 0x2e68a513ca1b8859) *= 0xe5a434cd14866e); + EXPECT_EQ(-absl::MakeInt128(0x55cae732029d1fce, 0xca6474b6423263e4), + absl::int128(0xa9b98a8ddf66bc) *= + -absl::MakeInt128(0x81, 0x672e58231e2469d7)); + EXPECT_EQ(absl::MakeInt128(0x19c8b7620b507dc4, 0xfec042b71a5f29a4), + absl::int128(-0x3e39341147) *= + -absl::MakeInt128(0x6a14b2, 0x5ed34cca42327b3c)); +} + +TEST(Int128, DivisionAndModuloTest) { + // Check against 64 bit division and modulo operators with a sample of + // randomly generated pairs. + std::pair<int64_t, int64_t> small_pairs[] = { + {0x15f2a64138, 0x67da05}, {0x5e56d194af43045f, 0xcf1543fb99}, + {0x15e61ed052036a, -0xc8e6}, {0x88125a341e85, -0xd23fb77683}, + {-0xc06e20, 0x5a}, {-0x4f100219aea3e85d, 0xdcc56cb4efe993}, + {-0x168d629105, -0xa7}, {-0x7b44e92f03ab2375, -0x6516}, + }; + for (const std::pair<int64_t, int64_t>& pair : small_pairs) { + SCOPED_TRACE(::testing::Message() + << "pair = {" << pair.first << ", " << pair.second << '}'); + + absl::int128 dividend = pair.first; + absl::int128 divisor = pair.second; + int64_t quotient = pair.first / pair.second; + int64_t remainder = pair.first % pair.second; + + EXPECT_EQ(quotient, dividend / divisor); + EXPECT_EQ(quotient, absl::int128(dividend) /= divisor); + EXPECT_EQ(remainder, dividend % divisor); + EXPECT_EQ(remainder, absl::int128(dividend) %= divisor); + } + + // Test behavior with 0, 1, and -1 with a sample of randomly generated large + // values. + absl::int128 values[] = { + absl::MakeInt128(0x63d26ee688a962b2, 0x9e1411abda5c1d70), + absl::MakeInt128(0x152f385159d6f986, 0xbf8d48ef63da395d), + -absl::MakeInt128(0x3098d7567030038c, 0x14e7a8a098dc2164), + -absl::MakeInt128(0x49a037aca35c809f, 0xa6a87525480ef330), + }; + for (absl::int128 value : values) { + SCOPED_TRACE(::testing::Message() << "value = " << value); + + EXPECT_EQ(0, 0 / value); + EXPECT_EQ(0, absl::int128(0) /= value); + EXPECT_EQ(0, 0 % value); + EXPECT_EQ(0, absl::int128(0) %= value); + + EXPECT_EQ(value, value / 1); + EXPECT_EQ(value, absl::int128(value) /= 1); + EXPECT_EQ(0, value % 1); + EXPECT_EQ(0, absl::int128(value) %= 1); + + EXPECT_EQ(-value, value / -1); + EXPECT_EQ(-value, absl::int128(value) /= -1); + EXPECT_EQ(0, value % -1); + EXPECT_EQ(0, absl::int128(value) %= -1); + } + + // Min and max values + EXPECT_EQ(0, absl::Int128Max() / absl::Int128Min()); + EXPECT_EQ(absl::Int128Max(), absl::Int128Max() % absl::Int128Min()); + EXPECT_EQ(-1, absl::Int128Min() / absl::Int128Max()); + EXPECT_EQ(-1, absl::Int128Min() % absl::Int128Max()); + + // Power of two division and modulo of random large dividends + absl::int128 positive_values[] = { + absl::MakeInt128(0x21e1a1cc69574620, 0xe7ac447fab2fc869), + absl::MakeInt128(0x32c2ff3ab89e66e8, 0x03379a613fd1ce74), + absl::MakeInt128(0x6f32ca786184dcaf, 0x046f9c9ecb3a9ce1), + absl::MakeInt128(0x1aeb469dd990e0ee, 0xda2740f243cd37eb), + }; + for (absl::int128 value : positive_values) { + for (int i = 0; i < 127; ++i) { + SCOPED_TRACE(::testing::Message() + << "value = " << value << "; i = " << i); + absl::int128 power_of_two = absl::int128(1) << i; + + EXPECT_EQ(value >> i, value / power_of_two); + EXPECT_EQ(value >> i, absl::int128(value) /= power_of_two); + EXPECT_EQ(value & (power_of_two - 1), value % power_of_two); + EXPECT_EQ(value & (power_of_two - 1), + absl::int128(value) %= power_of_two); + } + } + + // Manually calculated cases with random large dividends + struct DivisionModCase { + absl::int128 dividend; + absl::int128 divisor; + absl::int128 quotient; + absl::int128 remainder; + }; + DivisionModCase manual_cases[] = { + {absl::MakeInt128(0x6ada48d489007966, 0x3c9c5c98150d5d69), + absl::MakeInt128(0x8bc308fb, 0x8cb9cc9a3b803344), 0xc3b87e08, + absl::MakeInt128(0x1b7db5e1, 0xd9eca34b7af04b49)}, + {absl::MakeInt128(0xd6946511b5b, 0x4886c5c96546bf5f), + -absl::MakeInt128(0x263b, 0xfd516279efcfe2dc), -0x59cbabf0, + absl::MakeInt128(0x622, 0xf462909155651d1f)}, + {-absl::MakeInt128(0x33db734f9e8d1399, 0x8447ac92482bca4d), 0x37495078240, + -absl::MakeInt128(0xf01f1, 0xbc0368bf9a77eae8), -0x21a508f404d}, + {-absl::MakeInt128(0x13f837b409a07e7d, 0x7fc8e248a7d73560), -0x1b9f, + absl::MakeInt128(0xb9157556d724, 0xb14f635714d7563e), -0x1ade}, + }; + for (const DivisionModCase test_case : manual_cases) { + EXPECT_EQ(test_case.quotient, test_case.dividend / test_case.divisor); + EXPECT_EQ(test_case.quotient, + absl::int128(test_case.dividend) /= test_case.divisor); + EXPECT_EQ(test_case.remainder, test_case.dividend % test_case.divisor); + EXPECT_EQ(test_case.remainder, + absl::int128(test_case.dividend) %= test_case.divisor); + } +} + +TEST(Int128, BitwiseLogicTest) { + EXPECT_EQ(absl::int128(-1), ~absl::int128(0)); + + absl::int128 values[]{ + 0, -1, 0xde400bee05c3ff6b, absl::MakeInt128(0x7f32178dd81d634a, 0), + absl::MakeInt128(0xaf539057055613a9, 0x7d104d7d946c2e4d)}; + for (absl::int128 value : values) { + EXPECT_EQ(value, ~~value); + + EXPECT_EQ(value, value | value); + EXPECT_EQ(value, value & value); + EXPECT_EQ(0, value ^ value); + + EXPECT_EQ(value, absl::int128(value) |= value); + EXPECT_EQ(value, absl::int128(value) &= value); + EXPECT_EQ(0, absl::int128(value) ^= value); + + EXPECT_EQ(value, value | 0); + EXPECT_EQ(0, value & 0); + EXPECT_EQ(value, value ^ 0); + + EXPECT_EQ(absl::int128(-1), value | absl::int128(-1)); + EXPECT_EQ(value, value & absl::int128(-1)); + EXPECT_EQ(~value, value ^ absl::int128(-1)); + } + + // small sample of randomly generated int64_t's + std::pair<int64_t, int64_t> pairs64[]{ + {0x7f86797f5e991af4, 0x1ee30494fb007c97}, + {0x0b278282bacf01af, 0x58780e0a57a49e86}, + {0x059f266ccb93a666, 0x3d5b731bae9286f5}, + {0x63c0c4820f12108c, 0x58166713c12e1c3a}, + {0x381488bb2ed2a66e, 0x2220a3eb76a3698c}, + {0x2a0a0dfb81e06f21, 0x4b60585927f5523c}, + {0x555b1c3a03698537, 0x25478cd19d8e53cb}, + {0x4750f6f27d779225, 0x16397553c6ff05fc}, + }; + for (const std::pair<int64_t, int64_t>& pair : pairs64) { + SCOPED_TRACE(::testing::Message() + << "pair = {" << pair.first << ", " << pair.second << '}'); + + EXPECT_EQ(absl::MakeInt128(~pair.first, ~pair.second), + ~absl::MakeInt128(pair.first, pair.second)); + + EXPECT_EQ(absl::int128(pair.first & pair.second), + absl::int128(pair.first) & absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.first | pair.second), + absl::int128(pair.first) | absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.first ^ pair.second), + absl::int128(pair.first) ^ absl::int128(pair.second)); + + EXPECT_EQ(absl::int128(pair.first & pair.second), + absl::int128(pair.first) &= absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.first | pair.second), + absl::int128(pair.first) |= absl::int128(pair.second)); + EXPECT_EQ(absl::int128(pair.first ^ pair.second), + absl::int128(pair.first) ^= absl::int128(pair.second)); + + EXPECT_EQ( + absl::MakeInt128(pair.first & pair.second, 0), + absl::MakeInt128(pair.first, 0) & absl::MakeInt128(pair.second, 0)); + EXPECT_EQ( + absl::MakeInt128(pair.first | pair.second, 0), + absl::MakeInt128(pair.first, 0) | absl::MakeInt128(pair.second, 0)); + EXPECT_EQ( + absl::MakeInt128(pair.first ^ pair.second, 0), + absl::MakeInt128(pair.first, 0) ^ absl::MakeInt128(pair.second, 0)); + + EXPECT_EQ( + absl::MakeInt128(pair.first & pair.second, 0), + absl::MakeInt128(pair.first, 0) &= absl::MakeInt128(pair.second, 0)); + EXPECT_EQ( + absl::MakeInt128(pair.first | pair.second, 0), + absl::MakeInt128(pair.first, 0) |= absl::MakeInt128(pair.second, 0)); + EXPECT_EQ( + absl::MakeInt128(pair.first ^ pair.second, 0), + absl::MakeInt128(pair.first, 0) ^= absl::MakeInt128(pair.second, 0)); + } +} + +TEST(Int128, BitwiseShiftTest) { + for (int i = 0; i < 64; ++i) { + for (int j = 0; j <= i; ++j) { + // Left shift from j-th bit to i-th bit. + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + EXPECT_EQ(uint64_t{1} << i, absl::int128(uint64_t{1} << j) << (i - j)); + EXPECT_EQ(uint64_t{1} << i, absl::int128(uint64_t{1} << j) <<= (i - j)); + } + } + for (int i = 0; i < 63; ++i) { + for (int j = 0; j < 64; ++j) { + // Left shift from j-th bit to (i + 64)-th bit. + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + EXPECT_EQ(absl::MakeInt128(uint64_t{1} << i, 0), + absl::int128(uint64_t{1} << j) << (i + 64 - j)); + EXPECT_EQ(absl::MakeInt128(uint64_t{1} << i, 0), + absl::int128(uint64_t{1} << j) <<= (i + 64 - j)); + } + for (int j = 0; j <= i; ++j) { + // Left shift from (j + 64)-th bit to (i + 64)-th bit. + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + EXPECT_EQ(absl::MakeInt128(uint64_t{1} << i, 0), + absl::MakeInt128(uint64_t{1} << j, 0) << (i - j)); + EXPECT_EQ(absl::MakeInt128(uint64_t{1} << i, 0), + absl::MakeInt128(uint64_t{1} << j, 0) <<= (i - j)); + } + } + + for (int i = 0; i < 64; ++i) { + for (int j = i; j < 64; ++j) { + // Right shift from j-th bit to i-th bit. + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + EXPECT_EQ(uint64_t{1} << i, absl::int128(uint64_t{1} << j) >> (j - i)); + EXPECT_EQ(uint64_t{1} << i, absl::int128(uint64_t{1} << j) >>= (j - i)); + } + for (int j = 0; j < 63; ++j) { + // Right shift from (j + 64)-th bit to i-th bit. + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + EXPECT_EQ(uint64_t{1} << i, + absl::MakeInt128(uint64_t{1} << j, 0) >> (j + 64 - i)); + EXPECT_EQ(uint64_t{1} << i, + absl::MakeInt128(uint64_t{1} << j, 0) >>= (j + 64 - i)); + } + } + for (int i = 0; i < 63; ++i) { + for (int j = i; j < 63; ++j) { + // Right shift from (j + 64)-th bit to (i + 64)-th bit. + SCOPED_TRACE(::testing::Message() << "i = " << i << "; j = " << j); + EXPECT_EQ(absl::MakeInt128(uint64_t{1} << i, 0), + absl::MakeInt128(uint64_t{1} << j, 0) >> (j - i)); + EXPECT_EQ(absl::MakeInt128(uint64_t{1} << i, 0), + absl::MakeInt128(uint64_t{1} << j, 0) >>= (j - i)); + } + } +} + +TEST(Int128, NumericLimitsTest) { + static_assert(std::numeric_limits<absl::int128>::is_specialized, ""); + static_assert(std::numeric_limits<absl::int128>::is_signed, ""); + static_assert(std::numeric_limits<absl::int128>::is_integer, ""); + EXPECT_EQ(static_cast<int>(127 * std::log10(2)), + std::numeric_limits<absl::int128>::digits10); + EXPECT_EQ(absl::Int128Min(), std::numeric_limits<absl::int128>::min()); + EXPECT_EQ(absl::Int128Min(), std::numeric_limits<absl::int128>::lowest()); + EXPECT_EQ(absl::Int128Max(), std::numeric_limits<absl::int128>::max()); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/BUILD.bazel b/third_party/abseil_cpp/absl/random/BUILD.bazel new file mode 100644 index 000000000000..81e150e62e78 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/BUILD.bazel @@ -0,0 +1,509 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ABSL random-number generation libraries. + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "random", + hdrs = ["random.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":seed_sequences", + "//absl/random/internal:nonsecure_base", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:pool_urbg", + "//absl/random/internal:randen_engine", + ], +) + +cc_library( + name = "distributions", + srcs = [ + "discrete_distribution.cc", + "gaussian_distribution.cc", + ], + hdrs = [ + "bernoulli_distribution.h", + "beta_distribution.h", + "discrete_distribution.h", + "distributions.h", + "exponential_distribution.h", + "gaussian_distribution.h", + "log_uniform_int_distribution.h", + "poisson_distribution.h", + "uniform_int_distribution.h", + "uniform_real_distribution.h", + "zipf_distribution.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/random/internal:distribution_caller", + "//absl/random/internal:fast_uniform_bits", + "//absl/random/internal:fastmath", + "//absl/random/internal:generate_real", + "//absl/random/internal:iostream_state_saver", + "//absl/random/internal:traits", + "//absl/random/internal:uniform_helper", + "//absl/random/internal:wide_multiply", + "//absl/strings", + ], +) + +cc_library( + name = "seed_gen_exception", + srcs = ["seed_gen_exception.cc"], + hdrs = ["seed_gen_exception.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/base:config"], +) + +cc_library( + name = "seed_sequences", + srcs = ["seed_sequences.cc"], + hdrs = [ + "seed_sequences.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":seed_gen_exception", + "//absl/container:inlined_vector", + "//absl/random/internal:nonsecure_base", + "//absl/random/internal:pool_urbg", + "//absl/random/internal:salted_seed_seq", + "//absl/random/internal:seed_material", + "//absl/types:span", + ], +) + +cc_library( + name = "bit_gen_ref", + hdrs = ["bit_gen_ref.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":random", + "//absl/base:core_headers", + "//absl/base:fast_type_id", + "//absl/meta:type_traits", + "//absl/random/internal:distribution_caller", + "//absl/random/internal:fast_uniform_bits", + ], +) + +cc_library( + name = "mock_distributions", + testonly = 1, + hdrs = ["mock_distributions.h"], + deps = [ + ":distributions", + ":mocking_bit_gen", + "//absl/meta:type_traits", + "//absl/random/internal:mock_overload_set", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "mocking_bit_gen", + testonly = 1, + hdrs = [ + "mocking_bit_gen.h", + ], + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:fast_type_id", + "//absl/container:flat_hash_map", + "//absl/meta:type_traits", + "//absl/random/internal:distribution_caller", + "//absl/strings", + "//absl/types:span", + "//absl/types:variant", + "//absl/utility", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "bernoulli_distribution_test", + size = "small", + timeout = "eternal", # Android can take a very long time + srcs = ["bernoulli_distribution_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "beta_distribution_test", + size = "small", + timeout = "eternal", # Android can take a very long time + srcs = ["beta_distribution_test.cc"], + copts = ABSL_TEST_COPTS, + flaky = 1, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "distributions_test", + size = "small", + timeout = "moderate", + srcs = [ + "distributions_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/random/internal:distribution_test_util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "generators_test", + size = "small", + srcs = ["generators_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "log_uniform_int_distribution_test", + size = "medium", + srcs = [ + "log_uniform_int_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "discrete_distribution_test", + size = "medium", + srcs = [ + "discrete_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "poisson_distribution_test", + size = "small", + timeout = "eternal", # Android can take a very long time + srcs = [ + "poisson_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + # Too Slow. + "no_test_android_arm", + "no_test_loonix", + ], + deps = [ + ":distributions", + ":random", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/container:flat_hash_map", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "exponential_distribution_test", + size = "small", + srcs = ["exponential_distribution_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "gaussian_distribution_test", + size = "small", + timeout = "eternal", # Android can take a very long time + srcs = [ + "gaussian_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "//absl/strings:str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "uniform_int_distribution_test", + size = "medium", + timeout = "long", + srcs = [ + "uniform_int_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "uniform_real_distribution_test", + size = "medium", + srcs = [ + "uniform_real_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + ], + deps = [ + ":distributions", + ":random", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "zipf_distribution_test", + size = "medium", + srcs = [ + "zipf_distribution_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distributions", + ":random", + "//absl/base:raw_logging_internal", + "//absl/random/internal:distribution_test_util", + "//absl/random/internal:pcg_engine", + "//absl/random/internal:sequence_urbg", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "bit_gen_ref_test", + size = "small", + srcs = ["bit_gen_ref_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bit_gen_ref", + ":random", + "//absl/base:fast_type_id", + "//absl/random/internal:sequence_urbg", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "mocking_bit_gen_test", + size = "small", + srcs = ["mocking_bit_gen_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bit_gen_ref", + ":mock_distributions", + ":mocking_bit_gen", + ":random", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "mock_distributions_test", + size = "small", + srcs = ["mock_distributions_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":mock_distributions", + ":mocking_bit_gen", + ":random", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "examples_test", + size = "small", + srcs = ["examples_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":random", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "seed_sequences_test", + size = "small", + srcs = ["seed_sequences_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":random", + ":seed_sequences", + "//absl/random/internal:nonsecure_base", + "@com_google_googletest//:gtest_main", + ], +) + +BENCHMARK_TAGS = [ + "benchmark", + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + "no_test_darwin_x86_64", + "no_test_ios_x86_64", + "no_test_loonix", + "no_test_msvc_x64", + "no_test_wasm", +] + +# Benchmarks for various methods / test utilities +cc_binary( + name = "benchmarks", + testonly = 1, + srcs = [ + "benchmarks.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = BENCHMARK_TAGS, + deps = [ + ":distributions", + ":random", + ":seed_sequences", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/random/internal:fast_uniform_bits", + "//absl/random/internal:randen_engine", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/third_party/abseil_cpp/absl/random/CMakeLists.txt b/third_party/abseil_cpp/absl/random/CMakeLists.txt new file mode 100644 index 000000000000..7d7bec83d994 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/CMakeLists.txt @@ -0,0 +1,1207 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + random_random + HDRS + "random.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_nonsecure_base + absl::random_internal_pcg_engine + absl::random_internal_pool_urbg + absl::random_internal_randen_engine + absl::random_seed_sequences +) + +absl_cc_library( + NAME + random_bit_gen_ref + HDRS + "bit_gen_ref.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::random_internal_distribution_caller + absl::random_internal_fast_uniform_bits + absl::type_traits +) + +absl_cc_test( + NAME + random_bit_gen_ref_test + SRCS + "bit_gen_ref_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_bit_gen_ref + absl::random_random + absl::random_internal_sequence_urbg + absl::fast_type_id + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_mock_helpers + HDRS + "internal/mock_helpers.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::fast_type_id + absl::optional +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_mock_overload_set + HDRS + "internal/mock_overload_set.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_mocking_bit_gen + absl::random_internal_mock_helpers + TESTONLY +) + +absl_cc_library( + NAME + random_mocking_bit_gen + HDRS + "mock_distributions.h" + "mocking_bit_gen.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::flat_hash_map + absl::raw_logging_internal + absl::random_distributions + absl::random_internal_distribution_caller + absl::random_internal_mock_overload_set + absl::random_random + absl::strings + absl::span + absl::type_traits + absl::utility + absl::variant + gmock + gtest + TESTONLY +) + +absl_cc_test( + NAME + random_mock_distributions_test + SRCS + "mock_distributions_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_mocking_bit_gen + absl::random_random + gmock + gtest_main +) + +absl_cc_test( + NAME + random_mocking_bit_gen_test + SRCS + "mocking_bit_gen_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_bit_gen_ref + absl::random_mocking_bit_gen + absl::random_random + gmock + gtest_main +) + +absl_cc_library( + NAME + random_distributions + SRCS + "discrete_distribution.cc" + "gaussian_distribution.cc" + HDRS + "bernoulli_distribution.h" + "beta_distribution.h" + "discrete_distribution.h" + "distributions.h" + "exponential_distribution.h" + "gaussian_distribution.h" + "log_uniform_int_distribution.h" + "poisson_distribution.h" + "uniform_int_distribution.h" + "uniform_real_distribution.h" + "zipf_distribution.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base_internal + absl::config + absl::core_headers + absl::random_internal_generate_real + absl::random_internal_distribution_caller + absl::random_internal_fast_uniform_bits + absl::random_internal_fastmath + absl::random_internal_iostream_state_saver + absl::random_internal_traits + absl::random_internal_uniform_helper + absl::random_internal_wide_multiply + absl::strings + absl::type_traits +) + +absl_cc_library( + NAME + random_seed_gen_exception + SRCS + "seed_gen_exception.cc" + HDRS + "seed_gen_exception.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_library( + NAME + random_seed_sequences + SRCS + "seed_sequences.cc" + HDRS + "seed_sequences.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::inlined_vector + absl::random_internal_nonsecure_base + absl::random_internal_pool_urbg + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::random_seed_gen_exception + absl::span +) + +absl_cc_test( + NAME + random_bernoulli_distribution_test + SRCS + "bernoulli_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_random + absl::random_internal_sequence_urbg + absl::random_internal_pcg_engine + gmock + gtest_main +) + +absl_cc_test( + NAME + random_beta_distribution_test + SRCS + "beta_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_random + absl::random_internal_distribution_test_util + absl::random_internal_sequence_urbg + absl::random_internal_pcg_engine + absl::raw_logging_internal + absl::strings + absl::str_format + gmock + gtest_main +) + +absl_cc_test( + NAME + random_distributions_test + SRCS + "distributions_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_random + absl::random_internal_distribution_test_util + gmock + gtest_main +) + +absl_cc_test( + NAME + random_generators_test + SRCS + "generators_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + absl::random_distributions + absl::random_random + absl::raw_logging_internal + gmock + gtest_main +) + +absl_cc_test( + NAME + random_log_uniform_int_distribution_test + SRCS + "log_uniform_int_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::random_random + absl::raw_logging_internal + absl::strings + absl::str_format + gmock + gtest_main +) + +absl_cc_test( + NAME + random_discrete_distribution_test + SRCS + "discrete_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::random_random + absl::raw_logging_internal + absl::strings + gmock + gtest_main +) + +absl_cc_test( + NAME + random_poisson_distribution_test + SRCS + "poisson_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_random + absl::core_headers + absl::flat_hash_map + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::raw_logging_internal + absl::strings + absl::str_format + gmock + gtest_main +) + +absl_cc_test( + NAME + random_exponential_distribution_test + SRCS + "exponential_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::random_random + absl::raw_logging_internal + absl::strings + absl::str_format + gmock + gtest_main +) + +absl_cc_test( + NAME + random_gaussian_distribution_test + SRCS + "gaussian_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_sequence_urbg + absl::random_random + absl::raw_logging_internal + absl::strings + absl::str_format + gmock + gtest_main +) + +absl_cc_test( + NAME + random_uniform_int_distribution_test + SRCS + "uniform_int_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::random_random + absl::raw_logging_internal + absl::strings + gmock + gtest_main +) + +absl_cc_test( + NAME + random_uniform_real_distribution_test + SRCS + "uniform_real_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::random_random + absl::strings + gmock + gtest_main +) + +absl_cc_test( + NAME + random_zipf_distribution_test + SRCS + "zipf_distribution_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_distributions + absl::random_internal_distribution_test_util + absl::random_internal_pcg_engine + absl::random_internal_sequence_urbg + absl::random_random + absl::raw_logging_internal + absl::strings + gmock + gtest_main +) + +absl_cc_test( + NAME + random_examples_test + SRCS + "examples_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_random + gtest_main +) + +absl_cc_test( + NAME + random_seed_sequences_test + SRCS + "seed_sequences_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_seed_sequences + absl::random_internal_nonsecure_base + absl::random_random + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_traits + HDRS + "internal/traits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_distribution_caller + HDRS + "internal/distribution_caller.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::utility + absl::fast_type_id +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_fast_uniform_bits + HDRS + "internal/fast_uniform_bits.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_seed_material + SRCS + "internal/seed_material.cc" + HDRS + "internal/seed_material.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + $<$<BOOL:${MINGW}>:"bcrypt"> + DEPS + absl::core_headers + absl::optional + absl::random_internal_fast_uniform_bits + absl::raw_logging_internal + absl::span + absl::strings +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_pool_urbg + SRCS + "internal/pool_urbg.cc" + HDRS + "internal/pool_urbg.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::base + absl::config + absl::core_headers + absl::endian + absl::random_internal_randen + absl::random_internal_seed_material + absl::random_internal_traits + absl::random_seed_gen_exception + absl::raw_logging_internal + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_explicit_seed_seq + HDRS + "internal/random_internal_explicit_seed_seq.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + TESTONLY +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_sequence_urbg + HDRS + "internal/sequence_urbg.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + TESTONLY +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_salted_seed_seq + HDRS + "internal/salted_seed_seq.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::inlined_vector + absl::optional + absl::span + absl::random_internal_seed_material + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_iostream_state_saver + HDRS + "internal/iostream_state_saver.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::int128 + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_generate_real + HDRS + "internal/generate_real.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::random_internal_fastmath + absl::random_internal_traits + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_wide_multiply + HDRS + "internal/wide_multiply.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::config + absl::int128 +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_fastmath + HDRS + "internal/fastmath.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_nonsecure_base + HDRS + "internal/nonsecure_base.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::optional + absl::random_internal_pool_urbg + absl::random_internal_salted_seed_seq + absl::random_internal_seed_material + absl::span + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_pcg_engine + HDRS + "internal/pcg_engine.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::int128 + absl::random_internal_fastmath + absl::random_internal_iostream_state_saver + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_engine + HDRS + "internal/randen_engine.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_iostream_state_saver + absl::random_internal_randen + absl::raw_logging_internal + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_platform + HDRS + "internal/randen_traits.h" + "internal/platform.h" + SRCS + "internal/randen_round_keys.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen + SRCS + "internal/randen.cc" + HDRS + "internal/randen.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes + absl::random_internal_randen_slow +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_slow + SRCS + "internal/randen_slow.cc" + HDRS + "internal/randen_slow.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_hwaes + SRCS + "internal/randen_detect.cc" + HDRS + "internal/randen_detect.h" + "internal/randen_hwaes.h" + COPTS + ${ABSL_DEFAULT_COPTS} + ${ABSL_RANDOM_RANDEN_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes_impl + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_randen_hwaes_impl + SRCS + "internal/randen_hwaes.cc" + "internal/randen_hwaes.h" + COPTS + ${ABSL_DEFAULT_COPTS} + ${ABSL_RANDOM_RANDEN_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::config +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_distribution_test_util + SRCS + "internal/chi_square.cc" + "internal/distribution_test_util.cc" + HDRS + "internal/chi_square.h" + "internal/distribution_test_util.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::core_headers + absl::raw_logging_internal + absl::strings + absl::str_format + absl::span +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_traits_test + SRCS + "internal/traits_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_traits + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_generate_real_test + SRCS + "internal/generate_real_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::bits + absl::flags + absl::random_internal_generate_real + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_distribution_test_util_test + SRCS + "internal/distribution_test_util_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_distribution_test_util + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_fastmath_test + SRCS + "internal/fastmath_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_fastmath + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_explicit_seed_seq_test + SRCS + "internal/explicit_seed_seq_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_explicit_seed_seq + absl::random_seed_sequences + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_salted_seed_seq_test + SRCS + "internal/salted_seed_seq_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_salted_seed_seq + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_chi_square_test + SRCS + "internal/chi_square_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::core_headers + absl::random_internal_distribution_test_util + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_fast_uniform_bits_test + SRCS + "internal/fast_uniform_bits_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_fast_uniform_bits + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_nonsecure_base_test + SRCS + "internal/nonsecure_base_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_nonsecure_base + absl::random_random + absl::random_distributions + absl::random_seed_sequences + absl::strings + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_seed_material_test + SRCS + "internal/seed_material_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_seed_material + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_pool_urbg_test + SRCS + "internal/pool_urbg_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_pool_urbg + absl::span + absl::type_traits + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_pcg_engine_test + SRCS + "internal/pcg_engine_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_explicit_seed_seq + absl::random_internal_pcg_engine + absl::time + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_randen_engine_test + SRCS + "internal/randen_engine_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_explicit_seed_seq + absl::random_internal_randen_engine + absl::raw_logging_internal + absl::strings + absl::time + gmock + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_randen_test + SRCS + "internal/randen_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_randen + absl::type_traits + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_randen_slow_test + SRCS + "internal/randen_slow_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_randen_slow + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_randen_hwaes_test + SRCS + "internal/randen_hwaes_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_platform + absl::random_internal_randen_hwaes + absl::random_internal_randen_hwaes_impl + absl::raw_logging_internal + absl::str_format + gmock + gtest +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + random_internal_uniform_helper + HDRS + "internal/uniform_helper.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config + absl::random_internal_traits + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_uniform_helper_test + SRCS + "internal/uniform_helper_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_uniform_helper + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_iostream_state_saver_test + SRCS + "internal/iostream_state_saver_test.cc" + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_iostream_state_saver + gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_test( + NAME + random_internal_wide_multiply_test + SRCS + internal/wide_multiply_test.cc + COPTS + ${ABSL_TEST_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::random_internal_wide_multiply + absl::bits + absl::int128 + gtest_main +) diff --git a/third_party/abseil_cpp/absl/random/benchmarks.cc b/third_party/abseil_cpp/absl/random/benchmarks.cc new file mode 100644 index 000000000000..87bbb9810a77 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/benchmarks.cc @@ -0,0 +1,383 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Benchmarks for absl random distributions as well as a selection of the +// C++ standard library random distributions. + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <initializer_list> +#include <iterator> +#include <limits> +#include <random> +#include <type_traits> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" +#include "absl/random/bernoulli_distribution.h" +#include "absl/random/beta_distribution.h" +#include "absl/random/exponential_distribution.h" +#include "absl/random/gaussian_distribution.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/randen_engine.h" +#include "absl/random/log_uniform_int_distribution.h" +#include "absl/random/poisson_distribution.h" +#include "absl/random/random.h" +#include "absl/random/uniform_int_distribution.h" +#include "absl/random/uniform_real_distribution.h" +#include "absl/random/zipf_distribution.h" +#include "benchmark/benchmark.h" + +namespace { + +// Seed data to avoid reading random_device() for benchmarks. +uint32_t kSeedData[] = { + 0x1B510052, 0x9A532915, 0xD60F573F, 0xBC9BC6E4, 0x2B60A476, 0x81E67400, + 0x08BA6FB5, 0x571BE91F, 0xF296EC6B, 0x2A0DD915, 0xB6636521, 0xE7B9F9B6, + 0xFF34052E, 0xC5855664, 0x53B02D5D, 0xA99F8FA1, 0x08BA4799, 0x6E85076A, + 0x4B7A70E9, 0xB5B32944, 0xDB75092E, 0xC4192623, 0xAD6EA6B0, 0x49A7DF7D, + 0x9CEE60B8, 0x8FEDB266, 0xECAA8C71, 0x699A18FF, 0x5664526C, 0xC2B19EE1, + 0x193602A5, 0x75094C29, 0xA0591340, 0xE4183A3E, 0x3F54989A, 0x5B429D65, + 0x6B8FE4D6, 0x99F73FD6, 0xA1D29C07, 0xEFE830F5, 0x4D2D38E6, 0xF0255DC1, + 0x4CDD2086, 0x8470EB26, 0x6382E9C6, 0x021ECC5E, 0x09686B3F, 0x3EBAEFC9, + 0x3C971814, 0x6B6A70A1, 0x687F3584, 0x52A0E286, 0x13198A2E, 0x03707344, +}; + +// PrecompiledSeedSeq provides kSeedData to a conforming +// random engine to speed initialization in the benchmarks. +class PrecompiledSeedSeq { + public: + using result_type = uint32_t; + + PrecompiledSeedSeq() {} + + template <typename Iterator> + PrecompiledSeedSeq(Iterator begin, Iterator end) {} + + template <typename T> + PrecompiledSeedSeq(std::initializer_list<T> il) {} + + template <typename OutIterator> + void generate(OutIterator begin, OutIterator end) { + static size_t idx = 0; + for (; begin != end; begin++) { + *begin = kSeedData[idx++]; + if (idx >= ABSL_ARRAYSIZE(kSeedData)) { + idx = 0; + } + } + } + + size_t size() const { return ABSL_ARRAYSIZE(kSeedData); } + + template <typename OutIterator> + void param(OutIterator out) const { + std::copy(std::begin(kSeedData), std::end(kSeedData), out); + } +}; + +// use_default_initialization<T> indicates whether the random engine +// T must be default initialized, or whether we may initialize it using +// a seed sequence. This is used because some engines do not accept seed +// sequence-based initialization. +template <typename E> +using use_default_initialization = std::false_type; + +// make_engine<T, SSeq> returns a random_engine which is initialized, +// either via the default constructor, when use_default_initialization<T> +// is true, or via the indicated seed sequence, SSeq. +template <typename Engine, typename SSeq = PrecompiledSeedSeq> +typename absl::enable_if_t<!use_default_initialization<Engine>::value, Engine> +make_engine() { + // Initialize the random engine using the seed sequence SSeq, which + // is constructed from the precompiled seed data. + SSeq seq(std::begin(kSeedData), std::end(kSeedData)); + return Engine(seq); +} + +template <typename Engine, typename SSeq = PrecompiledSeedSeq> +typename absl::enable_if_t<use_default_initialization<Engine>::value, Engine> +make_engine() { + // Initialize the random engine using the default constructor. + return Engine(); +} + +template <typename Engine, typename SSeq> +void BM_Construct(benchmark::State& state) { + for (auto _ : state) { + auto rng = make_engine<Engine, SSeq>(); + benchmark::DoNotOptimize(rng()); + } +} + +template <typename Engine> +void BM_Direct(benchmark::State& state) { + using value_type = typename Engine::result_type; + // Direct use of the URBG. + auto rng = make_engine<Engine>(); + for (auto _ : state) { + benchmark::DoNotOptimize(rng()); + } + state.SetBytesProcessed(sizeof(value_type) * state.iterations()); +} + +template <typename Engine> +void BM_Generate(benchmark::State& state) { + // std::generate makes a copy of the RNG; thus this tests the + // copy-constructor efficiency. + using value_type = typename Engine::result_type; + std::vector<value_type> v(64); + auto rng = make_engine<Engine>(); + while (state.KeepRunningBatch(64)) { + std::generate(std::begin(v), std::end(v), rng); + } +} + +template <typename Engine, size_t elems> +void BM_Shuffle(benchmark::State& state) { + // Direct use of the Engine. + std::vector<uint32_t> v(elems); + while (state.KeepRunningBatch(elems)) { + auto rng = make_engine<Engine>(); + std::shuffle(std::begin(v), std::end(v), rng); + } +} + +template <typename Engine, size_t elems> +void BM_ShuffleReuse(benchmark::State& state) { + // Direct use of the Engine. + std::vector<uint32_t> v(elems); + auto rng = make_engine<Engine>(); + while (state.KeepRunningBatch(elems)) { + std::shuffle(std::begin(v), std::end(v), rng); + } +} + +template <typename Engine, typename Dist, typename... Args> +void BM_Dist(benchmark::State& state, Args&&... args) { + using value_type = typename Dist::result_type; + auto rng = make_engine<Engine>(); + Dist dis{std::forward<Args>(args)...}; + // Compare the following loop performance: + for (auto _ : state) { + benchmark::DoNotOptimize(dis(rng)); + } + state.SetBytesProcessed(sizeof(value_type) * state.iterations()); +} + +template <typename Engine, typename Dist> +void BM_Large(benchmark::State& state) { + using value_type = typename Dist::result_type; + volatile value_type kMin = 0; + volatile value_type kMax = std::numeric_limits<value_type>::max() / 2 + 1; + BM_Dist<Engine, Dist>(state, kMin, kMax); +} + +template <typename Engine, typename Dist> +void BM_Small(benchmark::State& state) { + using value_type = typename Dist::result_type; + volatile value_type kMin = 0; + volatile value_type kMax = std::numeric_limits<value_type>::max() / 64 + 1; + BM_Dist<Engine, Dist>(state, kMin, kMax); +} + +template <typename Engine, typename Dist, int A> +void BM_Bernoulli(benchmark::State& state) { + volatile double a = static_cast<double>(A) / 1000000; + BM_Dist<Engine, Dist>(state, a); +} + +template <typename Engine, typename Dist, int A, int B> +void BM_Beta(benchmark::State& state) { + using value_type = typename Dist::result_type; + volatile value_type a = static_cast<value_type>(A) / 100; + volatile value_type b = static_cast<value_type>(B) / 100; + BM_Dist<Engine, Dist>(state, a, b); +} + +template <typename Engine, typename Dist, int A> +void BM_Gamma(benchmark::State& state) { + using value_type = typename Dist::result_type; + volatile value_type a = static_cast<value_type>(A) / 100; + BM_Dist<Engine, Dist>(state, a); +} + +template <typename Engine, typename Dist, int A = 100> +void BM_Poisson(benchmark::State& state) { + volatile double a = static_cast<double>(A) / 100; + BM_Dist<Engine, Dist>(state, a); +} + +template <typename Engine, typename Dist, int Q = 2, int V = 1> +void BM_Zipf(benchmark::State& state) { + using value_type = typename Dist::result_type; + volatile double q = Q; + volatile double v = V; + BM_Dist<Engine, Dist>(state, std::numeric_limits<value_type>::max(), q, v); +} + +template <typename Engine, typename Dist> +void BM_Thread(benchmark::State& state) { + using value_type = typename Dist::result_type; + auto rng = make_engine<Engine>(); + Dist dis{}; + for (auto _ : state) { + benchmark::DoNotOptimize(dis(rng)); + } + state.SetBytesProcessed(sizeof(value_type) * state.iterations()); +} + +// NOTES: +// +// std::geometric_distribution is similar to the zipf distributions. +// The algorithm for the geometric_distribution is, basically, +// floor(log(1-X) / log(1-p)) + +// Normal benchmark suite +#define BM_BASIC(Engine) \ + BENCHMARK_TEMPLATE(BM_Construct, Engine, PrecompiledSeedSeq); \ + BENCHMARK_TEMPLATE(BM_Construct, Engine, std::seed_seq); \ + BENCHMARK_TEMPLATE(BM_Direct, Engine); \ + BENCHMARK_TEMPLATE(BM_Shuffle, Engine, 10); \ + BENCHMARK_TEMPLATE(BM_Shuffle, Engine, 100); \ + BENCHMARK_TEMPLATE(BM_Shuffle, Engine, 1000); \ + BENCHMARK_TEMPLATE(BM_ShuffleReuse, Engine, 100); \ + BENCHMARK_TEMPLATE(BM_ShuffleReuse, Engine, 1000); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, \ + absl::random_internal::FastUniformBits<uint32_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, \ + absl::random_internal::FastUniformBits<uint64_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, \ + absl::uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, \ + absl::uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Large, Engine, \ + std::uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Large, Engine, \ + std::uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Large, Engine, \ + absl::uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Large, Engine, \ + absl::uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::uniform_real_distribution<float>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::uniform_real_distribution<double>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, absl::uniform_real_distribution<float>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, absl::uniform_real_distribution<double>) + +#define BM_COPY(Engine) BENCHMARK_TEMPLATE(BM_Generate, Engine) + +#define BM_THREAD(Engine) \ + BENCHMARK_TEMPLATE(BM_Thread, Engine, \ + absl::uniform_int_distribution<int64_t>) \ + ->ThreadPerCpu(); \ + BENCHMARK_TEMPLATE(BM_Thread, Engine, \ + absl::uniform_real_distribution<double>) \ + ->ThreadPerCpu(); \ + BENCHMARK_TEMPLATE(BM_Shuffle, Engine, 100)->ThreadPerCpu(); \ + BENCHMARK_TEMPLATE(BM_Shuffle, Engine, 1000)->ThreadPerCpu(); \ + BENCHMARK_TEMPLATE(BM_ShuffleReuse, Engine, 100)->ThreadPerCpu(); \ + BENCHMARK_TEMPLATE(BM_ShuffleReuse, Engine, 1000)->ThreadPerCpu(); + +#define BM_EXTENDED(Engine) \ + /* -------------- Extended Uniform -----------------------*/ \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + std::uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + std::uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + absl::uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + absl::uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, std::uniform_real_distribution<float>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + std::uniform_real_distribution<double>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + absl::uniform_real_distribution<float>); \ + BENCHMARK_TEMPLATE(BM_Small, Engine, \ + absl::uniform_real_distribution<double>); \ + /* -------------- Other -----------------------*/ \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::normal_distribution<double>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, absl::gaussian_distribution<double>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::exponential_distribution<double>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, absl::exponential_distribution<double>); \ + BENCHMARK_TEMPLATE(BM_Poisson, Engine, std::poisson_distribution<int64_t>, \ + 100); \ + BENCHMARK_TEMPLATE(BM_Poisson, Engine, absl::poisson_distribution<int64_t>, \ + 100); \ + BENCHMARK_TEMPLATE(BM_Poisson, Engine, std::poisson_distribution<int64_t>, \ + 10 * 100); \ + BENCHMARK_TEMPLATE(BM_Poisson, Engine, absl::poisson_distribution<int64_t>, \ + 10 * 100); \ + BENCHMARK_TEMPLATE(BM_Poisson, Engine, std::poisson_distribution<int64_t>, \ + 13 * 100); \ + BENCHMARK_TEMPLATE(BM_Poisson, Engine, absl::poisson_distribution<int64_t>, \ + 13 * 100); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, \ + absl::log_uniform_int_distribution<int32_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, \ + absl::log_uniform_int_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Dist, Engine, std::geometric_distribution<int64_t>); \ + BENCHMARK_TEMPLATE(BM_Zipf, Engine, absl::zipf_distribution<uint64_t>); \ + BENCHMARK_TEMPLATE(BM_Zipf, Engine, absl::zipf_distribution<uint64_t>, 2, \ + 3); \ + BENCHMARK_TEMPLATE(BM_Bernoulli, Engine, std::bernoulli_distribution, \ + 257305); \ + BENCHMARK_TEMPLATE(BM_Bernoulli, Engine, absl::bernoulli_distribution, \ + 257305); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<double>, 65, \ + 41); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<double>, 99, \ + 330); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<double>, 150, \ + 150); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<double>, 410, \ + 580); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<float>, 65, 41); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<float>, 99, \ + 330); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<float>, 150, \ + 150); \ + BENCHMARK_TEMPLATE(BM_Beta, Engine, absl::beta_distribution<float>, 410, \ + 580); \ + BENCHMARK_TEMPLATE(BM_Gamma, Engine, std::gamma_distribution<float>, 199); \ + BENCHMARK_TEMPLATE(BM_Gamma, Engine, std::gamma_distribution<double>, 199); + +// ABSL Recommended interfaces. +BM_BASIC(absl::InsecureBitGen); // === pcg64_2018_engine +BM_BASIC(absl::BitGen); // === randen_engine<uint64_t>. +BM_THREAD(absl::BitGen); +BM_EXTENDED(absl::BitGen); + +// Instantiate benchmarks for multiple engines. +using randen_engine_64 = absl::random_internal::randen_engine<uint64_t>; +using randen_engine_32 = absl::random_internal::randen_engine<uint32_t>; + +// Comparison interfaces. +BM_BASIC(std::mt19937_64); +BM_COPY(std::mt19937_64); +BM_EXTENDED(std::mt19937_64); +BM_BASIC(randen_engine_64); +BM_COPY(randen_engine_64); +BM_EXTENDED(randen_engine_64); + +BM_BASIC(std::mt19937); +BM_COPY(std::mt19937); +BM_BASIC(randen_engine_32); +BM_COPY(randen_engine_32); + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/bernoulli_distribution.h b/third_party/abseil_cpp/absl/random/bernoulli_distribution.h new file mode 100644 index 000000000000..25bd0d5ca420 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/bernoulli_distribution.h @@ -0,0 +1,200 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_BERNOULLI_DISTRIBUTION_H_ +#define ABSL_RANDOM_BERNOULLI_DISTRIBUTION_H_ + +#include <cstdint> +#include <istream> +#include <limits> + +#include "absl/base/optimization.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::bernoulli_distribution is a drop in replacement for +// std::bernoulli_distribution. It guarantees that (given a perfect +// UniformRandomBitGenerator) the acceptance probability is *exactly* equal to +// the given double. +// +// The implementation assumes that double is IEEE754 +class bernoulli_distribution { + public: + using result_type = bool; + + class param_type { + public: + using distribution_type = bernoulli_distribution; + + explicit param_type(double p = 0.5) : prob_(p) { + assert(p >= 0.0 && p <= 1.0); + } + + double p() const { return prob_; } + + friend bool operator==(const param_type& p1, const param_type& p2) { + return p1.p() == p2.p(); + } + friend bool operator!=(const param_type& p1, const param_type& p2) { + return p1.p() != p2.p(); + } + + private: + double prob_; + }; + + bernoulli_distribution() : bernoulli_distribution(0.5) {} + + explicit bernoulli_distribution(double p) : param_(p) {} + + explicit bernoulli_distribution(param_type p) : param_(p) {} + + // no-op + void reset() {} + + template <typename URBG> + bool operator()(URBG& g) { // NOLINT(runtime/references) + return Generate(param_.p(), g); + } + + template <typename URBG> + bool operator()(URBG& g, // NOLINT(runtime/references) + const param_type& param) { + return Generate(param.p(), g); + } + + param_type param() const { return param_; } + void param(const param_type& param) { param_ = param; } + + double p() const { return param_.p(); } + + result_type(min)() const { return false; } + result_type(max)() const { return true; } + + friend bool operator==(const bernoulli_distribution& d1, + const bernoulli_distribution& d2) { + return d1.param_ == d2.param_; + } + + friend bool operator!=(const bernoulli_distribution& d1, + const bernoulli_distribution& d2) { + return d1.param_ != d2.param_; + } + + private: + static constexpr uint64_t kP32 = static_cast<uint64_t>(1) << 32; + + template <typename URBG> + static bool Generate(double p, URBG& g); // NOLINT(runtime/references) + + param_type param_; +}; + +template <typename CharT, typename Traits> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const bernoulli_distribution& x) { + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<double>::kPrecision); + os << x.p(); + return os; +} + +template <typename CharT, typename Traits> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + bernoulli_distribution& x) { // NOLINT(runtime/references) + auto saver = random_internal::make_istream_state_saver(is); + auto p = random_internal::read_floating_point<double>(is); + if (!is.fail()) { + x.param(bernoulli_distribution::param_type(p)); + } + return is; +} + +template <typename URBG> +bool bernoulli_distribution::Generate(double p, + URBG& g) { // NOLINT(runtime/references) + random_internal::FastUniformBits<uint32_t> fast_u32; + + while (true) { + // There are two aspects of the definition of `c` below that are worth + // commenting on. First, because `p` is in the range [0, 1], `c` is in the + // range [0, 2^32] which does not fit in a uint32_t and therefore requires + // 64 bits. + // + // Second, `c` is constructed by first casting explicitly to a signed + // integer and then converting implicitly to an unsigned integer of the same + // size. This is done because the hardware conversion instructions produce + // signed integers from double; if taken as a uint64_t the conversion would + // be wrong for doubles greater than 2^63 (not relevant in this use-case). + // If converted directly to an unsigned integer, the compiler would end up + // emitting code to handle such large values that are not relevant due to + // the known bounds on `c`. To avoid these extra instructions this + // implementation converts first to the signed type and then use the + // implicit conversion to unsigned (which is a no-op). + const uint64_t c = static_cast<int64_t>(p * kP32); + const uint32_t v = fast_u32(g); + // FAST PATH: this path fails with probability 1/2^32. Note that simply + // returning v <= c would approximate P very well (up to an absolute error + // of 1/2^32); the slow path (taken in that range of possible error, in the + // case of equality) eliminates the remaining error. + if (ABSL_PREDICT_TRUE(v != c)) return v < c; + + // It is guaranteed that `q` is strictly less than 1, because if `q` were + // greater than or equal to 1, the same would be true for `p`. Certainly `p` + // cannot be greater than 1, and if `p == 1`, then the fast path would + // necessary have been taken already. + const double q = static_cast<double>(c) / kP32; + + // The probability of acceptance on the fast path is `q` and so the + // probability of acceptance here should be `p - q`. + // + // Note that `q` is obtained from `p` via some shifts and conversions, the + // upshot of which is that `q` is simply `p` with some of the + // least-significant bits of its mantissa set to zero. This means that the + // difference `p - q` will not have any rounding errors. To see why, pretend + // that double has 10 bits of resolution and q is obtained from `p` in such + // a way that the 4 least-significant bits of its mantissa are set to zero. + // For example: + // p = 1.1100111011 * 2^-1 + // q = 1.1100110000 * 2^-1 + // p - q = 1.011 * 2^-8 + // The difference `p - q` has exactly the nonzero mantissa bits that were + // "lost" in `q` producing a number which is certainly representable in a + // double. + const double left = p - q; + + // By construction, the probability of being on this slow path is 1/2^32, so + // P(accept in slow path) = P(accept| in slow path) * P(slow path), + // which means the probability of acceptance here is `1 / (left * kP32)`: + const double here = left * kP32; + + // The simplest way to compute the result of this trial is to repeat the + // whole algorithm with the new probability. This terminates because even + // given arbitrarily unfriendly "random" bits, each iteration either + // multiplies a tiny probability by 2^32 (if c == 0) or strips off some + // number of nonzero mantissa bits. That process is bounded. + if (here == 0) return false; + p = here; + } +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_BERNOULLI_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/bernoulli_distribution_test.cc b/third_party/abseil_cpp/absl/random/bernoulli_distribution_test.cc new file mode 100644 index 000000000000..b250f8787c6e --- /dev/null +++ b/third_party/abseil_cpp/absl/random/bernoulli_distribution_test.cc @@ -0,0 +1,217 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/bernoulli_distribution.h" + +#include <cmath> +#include <cstddef> +#include <random> +#include <sstream> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" + +namespace { + +class BernoulliTest : public testing::TestWithParam<std::pair<double, size_t>> { +}; + +TEST_P(BernoulliTest, Serialize) { + const double d = GetParam().first; + absl::bernoulli_distribution before(d); + + { + absl::bernoulli_distribution via_param{ + absl::bernoulli_distribution::param_type(d)}; + EXPECT_EQ(via_param, before); + } + + std::stringstream ss; + ss << before; + absl::bernoulli_distribution after(0.6789); + + EXPECT_NE(before.p(), after.p()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before.p(), after.p()); + EXPECT_EQ(before.param(), after.param()); + EXPECT_EQ(before, after); +} + +TEST_P(BernoulliTest, Accuracy) { + // Sadly, the claim to fame for this implementation is precise accuracy, which + // is very, very hard to measure, the improvements come as trials approach the + // limit of double accuracy; thus the outcome differs from the + // std::bernoulli_distribution with a probability of approximately 1 in 2^-53. + const std::pair<double, size_t> para = GetParam(); + size_t trials = para.second; + double p = para.first; + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng(0x2B7E151628AED2A6); + + size_t yes = 0; + absl::bernoulli_distribution dist(p); + for (size_t i = 0; i < trials; ++i) { + if (dist(rng)) yes++; + } + + // Compute the distribution parameters for a binomial test, using a normal + // approximation for the confidence interval, as there are a sufficiently + // large number of trials that the central limit theorem applies. + const double stddev_p = std::sqrt((p * (1.0 - p)) / trials); + const double expected = trials * p; + const double stddev = trials * stddev_p; + + // 5 sigma, approved by Richard Feynman + EXPECT_NEAR(yes, expected, 5 * stddev) + << "@" << p << ", " + << std::abs(static_cast<double>(yes) - expected) / stddev << " stddev"; +} + +// There must be many more trials to make the mean approximately normal for `p` +// closes to 0 or 1. +INSTANTIATE_TEST_SUITE_P( + All, BernoulliTest, + ::testing::Values( + // Typical values. + std::make_pair(0, 30000), std::make_pair(1e-3, 30000000), + std::make_pair(0.1, 3000000), std::make_pair(0.5, 3000000), + std::make_pair(0.9, 30000000), std::make_pair(0.999, 30000000), + std::make_pair(1, 30000), + // Boundary cases. + std::make_pair(std::nextafter(1.0, 0.0), 1), // ~1 - epsilon + std::make_pair(std::numeric_limits<double>::epsilon(), 1), + std::make_pair(std::nextafter(std::numeric_limits<double>::min(), + 1.0), // min + epsilon + 1), + std::make_pair(std::numeric_limits<double>::min(), // smallest normal + 1), + std::make_pair( + std::numeric_limits<double>::denorm_min(), // smallest denorm + 1), + std::make_pair(std::numeric_limits<double>::min() / 2, 1), // denorm + std::make_pair(std::nextafter(std::numeric_limits<double>::min(), + 0.0), // denorm_max + 1))); + +// NOTE: absl::bernoulli_distribution is not guaranteed to be stable. +TEST(BernoulliTest, StabilityTest) { + // absl::bernoulli_distribution stability relies on FastUniformBits and + // integer arithmetic. + absl::random_internal::sequence_urbg urbg({ + 0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull, + 0x4864f22c059bf29eull, 0x247856d8b862665cull, 0xe46e86e9a1337e10ull, + 0xd8c8541f3519b133ull, 0xe75b5162c567b9e4ull, 0xf732e5ded7009c5bull, + 0xb170b98353121eacull, 0x1ec2e8986d2362caull, 0x814c8e35fe9a961aull, + 0x0c3cd59c9b638a02ull, 0xcb3bb6478a07715cull, 0x1224e62c978bbc7full, + 0x671ef2cb04e81f6eull, 0x3c1cbd811eaf1808ull, 0x1bbc23cfa8fac721ull, + 0xa4c2cda65e596a51ull, 0xb77216fad37adf91ull, 0x836d794457c08849ull, + 0xe083df03475f49d7ull, 0xbc9feb512e6b0d6cull, 0xb12d74fdd718c8c5ull, + 0x12ff09653bfbe4caull, 0x8dd03a105bc4ee7eull, 0x5738341045ba0d85ull, + 0xe3fd722dc65ad09eull, 0x5a14fd21ea2a5705ull, 0x14e6ea4d6edb0c73ull, + 0x275b0dc7e0a18acfull, 0x36cebe0d2653682eull, 0x0361e9b23861596bull, + }); + + // Generate a string of '0' and '1' for the distribution output. + auto generate = [&urbg](absl::bernoulli_distribution& dist) { + std::string output; + output.reserve(36); + urbg.reset(); + for (int i = 0; i < 35; i++) { + output.append(dist(urbg) ? "1" : "0"); + } + return output; + }; + + const double kP = 0.0331289862362; + { + absl::bernoulli_distribution dist(kP); + auto v = generate(dist); + EXPECT_EQ(35, urbg.invocations()); + EXPECT_EQ(v, "00000000000010000000000010000000000") << dist; + } + { + absl::bernoulli_distribution dist(kP * 10.0); + auto v = generate(dist); + EXPECT_EQ(35, urbg.invocations()); + EXPECT_EQ(v, "00000100010010010010000011000011010") << dist; + } + { + absl::bernoulli_distribution dist(kP * 20.0); + auto v = generate(dist); + EXPECT_EQ(35, urbg.invocations()); + EXPECT_EQ(v, "00011110010110110011011111110111011") << dist; + } + { + absl::bernoulli_distribution dist(1.0 - kP); + auto v = generate(dist); + EXPECT_EQ(35, urbg.invocations()); + EXPECT_EQ(v, "11111111111111111111011111111111111") << dist; + } +} + +TEST(BernoulliTest, StabilityTest2) { + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + // Generate a string of '0' and '1' for the distribution output. + auto generate = [&urbg](absl::bernoulli_distribution& dist) { + std::string output; + output.reserve(13); + urbg.reset(); + for (int i = 0; i < 12; i++) { + output.append(dist(urbg) ? "1" : "0"); + } + return output; + }; + + constexpr double b0 = 1.0 / 13.0 / 0.2; + constexpr double b1 = 2.0 / 13.0 / 0.2; + constexpr double b3 = (5.0 / 13.0 / 0.2) - ((1 - b0) + (1 - b1) + (1 - b1)); + { + absl::bernoulli_distribution dist(b0); + auto v = generate(dist); + EXPECT_EQ(12, urbg.invocations()); + EXPECT_EQ(v, "000011100101") << dist; + } + { + absl::bernoulli_distribution dist(b1); + auto v = generate(dist); + EXPECT_EQ(12, urbg.invocations()); + EXPECT_EQ(v, "001111101101") << dist; + } + { + absl::bernoulli_distribution dist(b3); + auto v = generate(dist); + EXPECT_EQ(12, urbg.invocations()); + EXPECT_EQ(v, "001111101111") << dist; + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/beta_distribution.h b/third_party/abseil_cpp/absl/random/beta_distribution.h new file mode 100644 index 000000000000..c154066fb813 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/beta_distribution.h @@ -0,0 +1,427 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_BETA_DISTRIBUTION_H_ +#define ABSL_RANDOM_BETA_DISTRIBUTION_H_ + +#include <cassert> +#include <cmath> +#include <istream> +#include <limits> +#include <ostream> +#include <type_traits> + +#include "absl/meta/type_traits.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/fastmath.h" +#include "absl/random/internal/generate_real.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::beta_distribution: +// Generate a floating-point variate conforming to a Beta distribution: +// pdf(x) \propto x^(alpha-1) * (1-x)^(beta-1), +// where the params alpha and beta are both strictly positive real values. +// +// The support is the open interval (0, 1), but the return value might be equal +// to 0 or 1, due to numerical errors when alpha and beta are very different. +// +// Usage note: One usage is that alpha and beta are counts of number of +// successes and failures. When the total number of trials are large, consider +// approximating a beta distribution with a Gaussian distribution with the same +// mean and variance. One could use the skewness, which depends only on the +// smaller of alpha and beta when the number of trials are sufficiently large, +// to quantify how far a beta distribution is from the normal distribution. +template <typename RealType = double> +class beta_distribution { + public: + using result_type = RealType; + + class param_type { + public: + using distribution_type = beta_distribution; + + explicit param_type(result_type alpha, result_type beta) + : alpha_(alpha), beta_(beta) { + assert(alpha >= 0); + assert(beta >= 0); + assert(alpha <= (std::numeric_limits<result_type>::max)()); + assert(beta <= (std::numeric_limits<result_type>::max)()); + if (alpha == 0 || beta == 0) { + method_ = DEGENERATE_SMALL; + x_ = (alpha >= beta) ? 1 : 0; + return; + } + // a_ = min(beta, alpha), b_ = max(beta, alpha). + if (beta < alpha) { + inverted_ = true; + a_ = beta; + b_ = alpha; + } else { + inverted_ = false; + a_ = alpha; + b_ = beta; + } + if (a_ <= 1 && b_ >= ThresholdForLargeA()) { + method_ = DEGENERATE_SMALL; + x_ = inverted_ ? result_type(1) : result_type(0); + return; + } + // For threshold values, see also: + // Evaluation of Beta Generation Algorithms, Ying-Chao Hung, et. al. + // February, 2009. + if ((b_ < 1.0 && a_ + b_ <= 1.2) || a_ <= ThresholdForSmallA()) { + // Choose Joehnk over Cheng when it's faster or when Cheng encounters + // numerical issues. + method_ = JOEHNK; + a_ = result_type(1) / alpha_; + b_ = result_type(1) / beta_; + if (std::isinf(a_) || std::isinf(b_)) { + method_ = DEGENERATE_SMALL; + x_ = inverted_ ? result_type(1) : result_type(0); + } + return; + } + if (a_ >= ThresholdForLargeA()) { + method_ = DEGENERATE_LARGE; + // Note: on PPC for long double, evaluating + // `std::numeric_limits::max() / ThresholdForLargeA` results in NaN. + result_type r = a_ / b_; + x_ = (inverted_ ? result_type(1) : r) / (1 + r); + return; + } + x_ = a_ + b_; + log_x_ = std::log(x_); + if (a_ <= 1) { + method_ = CHENG_BA; + y_ = result_type(1) / a_; + gamma_ = a_ + a_; + return; + } + method_ = CHENG_BB; + result_type r = (a_ - 1) / (b_ - 1); + y_ = std::sqrt((1 + r) / (b_ * r * 2 - r + 1)); + gamma_ = a_ + result_type(1) / y_; + } + + result_type alpha() const { return alpha_; } + result_type beta() const { return beta_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.alpha_ == b.alpha_ && a.beta_ == b.beta_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class beta_distribution; + +#ifdef _MSC_VER + // MSVC does not have constexpr implementations for std::log and std::exp + // so they are computed at runtime. +#define ABSL_RANDOM_INTERNAL_LOG_EXP_CONSTEXPR +#else +#define ABSL_RANDOM_INTERNAL_LOG_EXP_CONSTEXPR constexpr +#endif + + // The threshold for whether std::exp(1/a) is finite. + // Note that this value is quite large, and a smaller a_ is NOT abnormal. + static ABSL_RANDOM_INTERNAL_LOG_EXP_CONSTEXPR result_type + ThresholdForSmallA() { + return result_type(1) / + std::log((std::numeric_limits<result_type>::max)()); + } + + // The threshold for whether a * std::log(a) is finite. + static ABSL_RANDOM_INTERNAL_LOG_EXP_CONSTEXPR result_type + ThresholdForLargeA() { + return std::exp( + std::log((std::numeric_limits<result_type>::max)()) - + std::log(std::log((std::numeric_limits<result_type>::max)())) - + ThresholdPadding()); + } + +#undef ABSL_RANDOM_INTERNAL_LOG_EXP_CONSTEXPR + + // Pad the threshold for large A for long double on PPC. This is done via a + // template specialization below. + static constexpr result_type ThresholdPadding() { return 0; } + + enum Method { + JOEHNK, // Uses algorithm Joehnk + CHENG_BA, // Uses algorithm BA in Cheng + CHENG_BB, // Uses algorithm BB in Cheng + + // Note: See also: + // Hung et al. Evaluation of beta generation algorithms. Communications + // in Statistics-Simulation and Computation 38.4 (2009): 750-770. + // especially: + // Zechner, Heinz, and Ernst Stadlober. Generating beta variates via + // patchwork rejection. Computing 50.1 (1993): 1-18. + + DEGENERATE_SMALL, // a_ is abnormally small. + DEGENERATE_LARGE, // a_ is abnormally large. + }; + + result_type alpha_; + result_type beta_; + + result_type a_; // the smaller of {alpha, beta}, or 1.0/alpha_ in JOEHNK + result_type b_; // the larger of {alpha, beta}, or 1.0/beta_ in JOEHNK + result_type x_; // alpha + beta, or the result in degenerate cases + result_type log_x_; // log(x_) + result_type y_; // "beta" in Cheng + result_type gamma_; // "gamma" in Cheng + + Method method_; + + // Placing this last for optimal alignment. + // Whether alpha_ != a_, i.e. true iff alpha_ > beta_. + bool inverted_; + + static_assert(std::is_floating_point<RealType>::value, + "Class-template absl::beta_distribution<> must be " + "parameterized using a floating-point type."); + }; + + beta_distribution() : beta_distribution(1) {} + + explicit beta_distribution(result_type alpha, result_type beta = 1) + : param_(alpha, beta) {} + + explicit beta_distribution(const param_type& p) : param_(p) {} + + void reset() {} + + // Generating functions + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + param_type param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + result_type(min)() const { return 0; } + result_type(max)() const { return 1; } + + result_type alpha() const { return param_.alpha(); } + result_type beta() const { return param_.beta(); } + + friend bool operator==(const beta_distribution& a, + const beta_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const beta_distribution& a, + const beta_distribution& b) { + return a.param_ != b.param_; + } + + private: + template <typename URBG> + result_type AlgorithmJoehnk(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + template <typename URBG> + result_type AlgorithmCheng(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + template <typename URBG> + result_type DegenerateCase(URBG& g, // NOLINT(runtime/references) + const param_type& p) { + if (p.method_ == param_type::DEGENERATE_SMALL && p.alpha_ == p.beta_) { + // Returns 0 or 1 with equal probability. + random_internal::FastUniformBits<uint8_t> fast_u8; + return static_cast<result_type>((fast_u8(g) & 0x10) != + 0); // pick any single bit. + } + return p.x_; + } + + param_type param_; + random_internal::FastUniformBits<uint64_t> fast_u64_; +}; + +#if defined(__powerpc64__) || defined(__PPC64__) || defined(__powerpc__) || \ + defined(__ppc__) || defined(__PPC__) +// PPC needs a more stringent boundary for long double. +template <> +constexpr long double +beta_distribution<long double>::param_type::ThresholdPadding() { + return 10; +} +#endif + +template <typename RealType> +template <typename URBG> +typename beta_distribution<RealType>::result_type +beta_distribution<RealType>::AlgorithmJoehnk( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + using random_internal::GeneratePositiveTag; + using random_internal::GenerateRealFromBits; + using real_type = + absl::conditional_t<std::is_same<RealType, float>::value, float, double>; + + // Based on Joehnk, M. D. Erzeugung von betaverteilten und gammaverteilten + // Zufallszahlen. Metrika 8.1 (1964): 5-15. + // This method is described in Knuth, Vol 2 (Third Edition), pp 134. + + result_type u, v, x, y, z; + for (;;) { + u = GenerateRealFromBits<real_type, GeneratePositiveTag, false>( + fast_u64_(g)); + v = GenerateRealFromBits<real_type, GeneratePositiveTag, false>( + fast_u64_(g)); + + // Direct method. std::pow is slow for float, so rely on the optimizer to + // remove the std::pow() path for that case. + if (!std::is_same<float, result_type>::value) { + x = std::pow(u, p.a_); + y = std::pow(v, p.b_); + z = x + y; + if (z > 1) { + // Reject if and only if `x + y > 1.0` + continue; + } + if (z > 0) { + // When both alpha and beta are small, x and y are both close to 0, so + // divide by (x+y) directly may result in nan. + return x / z; + } + } + + // Log transform. + // x = log( pow(u, p.a_) ), y = log( pow(v, p.b_) ) + // since u, v <= 1.0, x, y < 0. + x = std::log(u) * p.a_; + y = std::log(v) * p.b_; + if (!std::isfinite(x) || !std::isfinite(y)) { + continue; + } + // z = log( pow(u, a) + pow(v, b) ) + z = x > y ? (x + std::log(1 + std::exp(y - x))) + : (y + std::log(1 + std::exp(x - y))); + // Reject iff log(x+y) > 0. + if (z > 0) { + continue; + } + return std::exp(x - z); + } +} + +template <typename RealType> +template <typename URBG> +typename beta_distribution<RealType>::result_type +beta_distribution<RealType>::AlgorithmCheng( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + using random_internal::GeneratePositiveTag; + using random_internal::GenerateRealFromBits; + using real_type = + absl::conditional_t<std::is_same<RealType, float>::value, float, double>; + + // Based on Cheng, Russell CH. Generating beta variates with nonintegral + // shape parameters. Communications of the ACM 21.4 (1978): 317-322. + // (https://dl.acm.org/citation.cfm?id=359482). + static constexpr result_type kLogFour = + result_type(1.3862943611198906188344642429163531361); // log(4) + static constexpr result_type kS = + result_type(2.6094379124341003746007593332261876); // 1+log(5) + + const bool use_algorithm_ba = (p.method_ == param_type::CHENG_BA); + result_type u1, u2, v, w, z, r, s, t, bw_inv, lhs; + for (;;) { + u1 = GenerateRealFromBits<real_type, GeneratePositiveTag, false>( + fast_u64_(g)); + u2 = GenerateRealFromBits<real_type, GeneratePositiveTag, false>( + fast_u64_(g)); + v = p.y_ * std::log(u1 / (1 - u1)); + w = p.a_ * std::exp(v); + bw_inv = result_type(1) / (p.b_ + w); + r = p.gamma_ * v - kLogFour; + s = p.a_ + r - w; + z = u1 * u1 * u2; + if (!use_algorithm_ba && s + kS >= 5 * z) { + break; + } + t = std::log(z); + if (!use_algorithm_ba && s >= t) { + break; + } + lhs = p.x_ * (p.log_x_ + std::log(bw_inv)) + r; + if (lhs >= t) { + break; + } + } + return p.inverted_ ? (1 - w * bw_inv) : w * bw_inv; +} + +template <typename RealType> +template <typename URBG> +typename beta_distribution<RealType>::result_type +beta_distribution<RealType>::operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p) { + switch (p.method_) { + case param_type::JOEHNK: + return AlgorithmJoehnk(g, p); + case param_type::CHENG_BA: + ABSL_FALLTHROUGH_INTENDED; + case param_type::CHENG_BB: + return AlgorithmCheng(g, p); + default: + return DegenerateCase(g, p); + } +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const beta_distribution<RealType>& x) { + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<RealType>::kPrecision); + os << x.alpha() << os.fill() << x.beta(); + return os; +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + beta_distribution<RealType>& x) { // NOLINT(runtime/references) + using result_type = typename beta_distribution<RealType>::result_type; + using param_type = typename beta_distribution<RealType>::param_type; + result_type alpha, beta; + + auto saver = random_internal::make_istream_state_saver(is); + alpha = random_internal::read_floating_point<result_type>(is); + if (is.fail()) return is; + beta = random_internal::read_floating_point<result_type>(is); + if (!is.fail()) { + x.param(param_type(alpha, beta)); + } + return is; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_BETA_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/beta_distribution_test.cc b/third_party/abseil_cpp/absl/random/beta_distribution_test.cc new file mode 100644 index 000000000000..277e4dc6eed7 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/beta_distribution_test.cc @@ -0,0 +1,619 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/beta_distribution.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <random> +#include <sstream> +#include <string> +#include <unordered_map> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/strip.h" + +namespace { + +template <typename IntType> +class BetaDistributionInterfaceTest : public ::testing::Test {}; + +using RealTypes = ::testing::Types<float, double, long double>; +TYPED_TEST_CASE(BetaDistributionInterfaceTest, RealTypes); + +TYPED_TEST(BetaDistributionInterfaceTest, SerializeTest) { + // The threshold for whether std::exp(1/a) is finite. + const TypeParam kSmallA = + 1.0f / std::log((std::numeric_limits<TypeParam>::max)()); + // The threshold for whether a * std::log(a) is finite. + const TypeParam kLargeA = + std::exp(std::log((std::numeric_limits<TypeParam>::max)()) - + std::log(std::log((std::numeric_limits<TypeParam>::max)()))); + const TypeParam kLargeAPPC = std::exp( + std::log((std::numeric_limits<TypeParam>::max)()) - + std::log(std::log((std::numeric_limits<TypeParam>::max)())) - 10.0f); + using param_type = typename absl::beta_distribution<TypeParam>::param_type; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + const TypeParam kValues[] = { + TypeParam(1e-20), TypeParam(1e-12), TypeParam(1e-8), TypeParam(1e-4), + TypeParam(1e-3), TypeParam(0.1), TypeParam(0.25), + std::nextafter(TypeParam(0.5), TypeParam(0)), // 0.5 - epsilon + std::nextafter(TypeParam(0.5), TypeParam(1)), // 0.5 + epsilon + TypeParam(0.5), TypeParam(1.0), // + std::nextafter(TypeParam(1), TypeParam(0)), // 1 - epsilon + std::nextafter(TypeParam(1), TypeParam(2)), // 1 + epsilon + TypeParam(12.5), TypeParam(1e2), TypeParam(1e8), TypeParam(1e12), + TypeParam(1e20), // + kSmallA, // + std::nextafter(kSmallA, TypeParam(0)), // + std::nextafter(kSmallA, TypeParam(1)), // + kLargeA, // + std::nextafter(kLargeA, TypeParam(0)), // + std::nextafter(kLargeA, std::numeric_limits<TypeParam>::max()), + kLargeAPPC, // + std::nextafter(kLargeAPPC, TypeParam(0)), + std::nextafter(kLargeAPPC, std::numeric_limits<TypeParam>::max()), + // Boundary cases. + std::numeric_limits<TypeParam>::max(), + std::numeric_limits<TypeParam>::epsilon(), + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(1)), // min + epsilon + std::numeric_limits<TypeParam>::min(), // smallest normal + std::numeric_limits<TypeParam>::denorm_min(), // smallest denorm + std::numeric_limits<TypeParam>::min() / 2, // denorm + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(0)), // denorm_max + }; + for (TypeParam alpha : kValues) { + for (TypeParam beta : kValues) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("Smoke test for Beta(%a, %a)", alpha, beta)); + + param_type param(alpha, beta); + absl::beta_distribution<TypeParam> before(alpha, beta); + EXPECT_EQ(before.alpha(), param.alpha()); + EXPECT_EQ(before.beta(), param.beta()); + + { + absl::beta_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + EXPECT_EQ(via_param.param(), before.param()); + } + + // Smoke test. + for (int i = 0; i < kCount; ++i) { + auto sample = before(gen); + EXPECT_TRUE(std::isfinite(sample)); + EXPECT_GE(sample, before.min()); + EXPECT_LE(sample, before.max()); + } + + // Validate stream serialization. + std::stringstream ss; + ss << before; + absl::beta_distribution<TypeParam> after(3.8f, 1.43f); + EXPECT_NE(before.alpha(), after.alpha()); + EXPECT_NE(before.beta(), after.beta()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + +#if defined(__powerpc64__) || defined(__PPC64__) || defined(__powerpc__) || \ + defined(__ppc__) || defined(__PPC__) + if (std::is_same<TypeParam, long double>::value) { + // Roundtripping floating point values requires sufficient precision + // to reconstruct the exact value. It turns out that long double + // has some errors doing this on ppc. + if (alpha <= std::numeric_limits<double>::max() && + alpha >= std::numeric_limits<double>::lowest()) { + EXPECT_EQ(static_cast<double>(before.alpha()), + static_cast<double>(after.alpha())) + << ss.str(); + } + if (beta <= std::numeric_limits<double>::max() && + beta >= std::numeric_limits<double>::lowest()) { + EXPECT_EQ(static_cast<double>(before.beta()), + static_cast<double>(after.beta())) + << ss.str(); + } + continue; + } +#endif + + EXPECT_EQ(before.alpha(), after.alpha()); + EXPECT_EQ(before.beta(), after.beta()); + EXPECT_EQ(before, after) // + << ss.str() << " " // + << (ss.good() ? "good " : "") // + << (ss.bad() ? "bad " : "") // + << (ss.eof() ? "eof " : "") // + << (ss.fail() ? "fail " : ""); + } + } +} + +TYPED_TEST(BetaDistributionInterfaceTest, DegenerateCases) { + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng(0x2B7E151628AED2A6); + + // Extreme cases when the params are abnormal. + constexpr int kCount = 1000; + const TypeParam kSmallValues[] = { + std::numeric_limits<TypeParam>::min(), + std::numeric_limits<TypeParam>::denorm_min(), + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(0)), // denorm_max + std::numeric_limits<TypeParam>::epsilon(), + }; + const TypeParam kLargeValues[] = { + std::numeric_limits<TypeParam>::max() * static_cast<TypeParam>(0.9999), + std::numeric_limits<TypeParam>::max() - 1, + std::numeric_limits<TypeParam>::max(), + }; + { + // Small alpha and beta. + // Useful WolframAlpha plots: + // * plot InverseBetaRegularized[x, 0.0001, 0.0001] from 0.495 to 0.505 + // * Beta[1.0, 0.0000001, 0.0000001] + // * Beta[0.9999, 0.0000001, 0.0000001] + for (TypeParam alpha : kSmallValues) { + for (TypeParam beta : kSmallValues) { + int zeros = 0; + int ones = 0; + absl::beta_distribution<TypeParam> d(alpha, beta); + for (int i = 0; i < kCount; ++i) { + TypeParam x = d(rng); + if (x == 0.0) { + zeros++; + } else if (x == 1.0) { + ones++; + } + } + EXPECT_EQ(ones + zeros, kCount); + if (alpha == beta) { + EXPECT_NE(ones, 0); + EXPECT_NE(zeros, 0); + } + } + } + } + { + // Small alpha, large beta. + // Useful WolframAlpha plots: + // * plot InverseBetaRegularized[x, 0.0001, 10000] from 0.995 to 1 + // * Beta[0, 0.0000001, 1000000] + // * Beta[0.001, 0.0000001, 1000000] + // * Beta[1, 0.0000001, 1000000] + for (TypeParam alpha : kSmallValues) { + for (TypeParam beta : kLargeValues) { + absl::beta_distribution<TypeParam> d(alpha, beta); + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(d(rng), 0.0); + } + } + } + } + { + // Large alpha, small beta. + // Useful WolframAlpha plots: + // * plot InverseBetaRegularized[x, 10000, 0.0001] from 0 to 0.001 + // * Beta[0.99, 1000000, 0.0000001] + // * Beta[1, 1000000, 0.0000001] + for (TypeParam alpha : kLargeValues) { + for (TypeParam beta : kSmallValues) { + absl::beta_distribution<TypeParam> d(alpha, beta); + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(d(rng), 1.0); + } + } + } + } + { + // Large alpha and beta. + absl::beta_distribution<TypeParam> d(std::numeric_limits<TypeParam>::max(), + std::numeric_limits<TypeParam>::max()); + for (int i = 0; i < kCount; ++i) { + EXPECT_EQ(d(rng), 0.5); + } + } + { + // Large alpha and beta but unequal. + absl::beta_distribution<TypeParam> d( + std::numeric_limits<TypeParam>::max(), + std::numeric_limits<TypeParam>::max() * 0.9999); + for (int i = 0; i < kCount; ++i) { + TypeParam x = d(rng); + EXPECT_NE(x, 0.5f); + EXPECT_FLOAT_EQ(x, 0.500025f); + } + } +} + +class BetaDistributionModel { + public: + explicit BetaDistributionModel(::testing::tuple<double, double> p) + : alpha_(::testing::get<0>(p)), beta_(::testing::get<1>(p)) {} + + double Mean() const { return alpha_ / (alpha_ + beta_); } + + double Variance() const { + return alpha_ * beta_ / (alpha_ + beta_ + 1) / (alpha_ + beta_) / + (alpha_ + beta_); + } + + double Kurtosis() const { + return 3 + 6 * + ((alpha_ - beta_) * (alpha_ - beta_) * (alpha_ + beta_ + 1) - + alpha_ * beta_ * (2 + alpha_ + beta_)) / + alpha_ / beta_ / (alpha_ + beta_ + 2) / (alpha_ + beta_ + 3); + } + + protected: + const double alpha_; + const double beta_; +}; + +class BetaDistributionTest + : public ::testing::TestWithParam<::testing::tuple<double, double>>, + public BetaDistributionModel { + public: + BetaDistributionTest() : BetaDistributionModel(GetParam()) {} + + protected: + template <class D> + bool SingleZTestOnMeanAndVariance(double p, size_t samples); + + template <class D> + bool SingleChiSquaredTest(double p, size_t samples, size_t buckets); + + absl::InsecureBitGen rng_; +}; + +template <class D> +bool BetaDistributionTest::SingleZTestOnMeanAndVariance(double p, + size_t samples) { + D dis(alpha_, beta_); + + std::vector<double> data; + data.reserve(samples); + for (size_t i = 0; i < samples; i++) { + const double variate = dis(rng_); + EXPECT_FALSE(std::isnan(variate)); + // Note that equality is allowed on both sides. + EXPECT_GE(variate, 0.0); + EXPECT_LE(variate, 1.0); + data.push_back(variate); + } + + // We validate that the sample mean and sample variance are indeed from a + // Beta distribution with the given shape parameters. + const auto m = absl::random_internal::ComputeDistributionMoments(data); + + // The variance of the sample mean is variance / n. + const double mean_stddev = std::sqrt(Variance() / static_cast<double>(m.n)); + + // The variance of the sample variance is (approximately): + // (kurtosis - 1) * variance^2 / n + const double variance_stddev = std::sqrt( + (Kurtosis() - 1) * Variance() * Variance() / static_cast<double>(m.n)); + // z score for the sample variance. + const double z_variance = (m.variance - Variance()) / variance_stddev; + + const double max_err = absl::random_internal::MaxErrorTolerance(p); + const double z_mean = absl::random_internal::ZScore(Mean(), m); + const bool pass = + absl::random_internal::Near("z", z_mean, 0.0, max_err) && + absl::random_internal::Near("z_variance", z_variance, 0.0, max_err); + if (!pass) { + ABSL_INTERNAL_LOG( + INFO, + absl::StrFormat( + "Beta(%f, %f), " + "mean: sample %f, expect %f, which is %f stddevs away, " + "variance: sample %f, expect %f, which is %f stddevs away.", + alpha_, beta_, m.mean, Mean(), + std::abs(m.mean - Mean()) / mean_stddev, m.variance, Variance(), + std::abs(m.variance - Variance()) / variance_stddev)); + } + return pass; +} + +template <class D> +bool BetaDistributionTest::SingleChiSquaredTest(double p, size_t samples, + size_t buckets) { + constexpr double kErr = 1e-7; + std::vector<double> cutoffs, expected; + const double bucket_width = 1.0 / static_cast<double>(buckets); + int i = 1; + int unmerged_buckets = 0; + for (; i < buckets; ++i) { + const double p = bucket_width * static_cast<double>(i); + const double boundary = + absl::random_internal::BetaIncompleteInv(alpha_, beta_, p); + // The intention is to add `boundary` to the list of `cutoffs`. It becomes + // problematic, however, when the boundary values are not monotone, due to + // numerical issues when computing the inverse regularized incomplete + // Beta function. In these cases, we merge that bucket with its previous + // neighbor and merge their expected counts. + if ((cutoffs.empty() && boundary < kErr) || + (!cutoffs.empty() && boundary <= cutoffs.back())) { + unmerged_buckets++; + continue; + } + if (boundary >= 1.0 - 1e-10) { + break; + } + cutoffs.push_back(boundary); + expected.push_back(static_cast<double>(1 + unmerged_buckets) * + bucket_width * static_cast<double>(samples)); + unmerged_buckets = 0; + } + cutoffs.push_back(std::numeric_limits<double>::infinity()); + // Merge all remaining buckets. + expected.push_back(static_cast<double>(buckets - i + 1) * bucket_width * + static_cast<double>(samples)); + // Make sure that we don't merge all the buckets, making this test + // meaningless. + EXPECT_GE(cutoffs.size(), 3) << alpha_ << ", " << beta_; + + D dis(alpha_, beta_); + + std::vector<int32_t> counts(cutoffs.size(), 0); + for (int i = 0; i < samples; i++) { + const double x = dis(rng_); + auto it = std::upper_bound(cutoffs.begin(), cutoffs.end(), x); + counts[std::distance(cutoffs.begin(), it)]++; + } + + // Null-hypothesis is that the distribution is beta distributed with the + // provided alpha, beta params (not estimated from the data). + const int dof = cutoffs.size() - 1; + + const double chi_square = absl::random_internal::ChiSquare( + counts.begin(), counts.end(), expected.begin(), expected.end()); + const bool pass = + (absl::random_internal::ChiSquarePValue(chi_square, dof) >= p); + if (!pass) { + for (int i = 0; i < cutoffs.size(); i++) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("cutoff[%d] = %f, actual count %d, expected %d", + i, cutoffs[i], counts[i], + static_cast<int>(expected[i]))); + } + + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat( + "Beta(%f, %f) %s %f, p = %f", alpha_, beta_, + absl::random_internal::kChiSquared, chi_square, + absl::random_internal::ChiSquarePValue(chi_square, dof))); + } + return pass; +} + +TEST_P(BetaDistributionTest, TestSampleStatistics) { + static constexpr int kRuns = 20; + static constexpr double kPFail = 0.02; + const double p = + absl::random_internal::RequiredSuccessProbability(kPFail, kRuns); + static constexpr int kSampleCount = 10000; + static constexpr int kBucketCount = 100; + int failed = 0; + for (int i = 0; i < kRuns; ++i) { + if (!SingleZTestOnMeanAndVariance<absl::beta_distribution<double>>( + p, kSampleCount)) { + failed++; + } + if (!SingleChiSquaredTest<absl::beta_distribution<double>>( + 0.005, kSampleCount, kBucketCount)) { + failed++; + } + } + // Set so that the test is not flaky at --runs_per_test=10000 + EXPECT_LE(failed, 5); +} + +std::string ParamName( + const ::testing::TestParamInfo<::testing::tuple<double, double>>& info) { + std::string name = absl::StrCat("alpha_", ::testing::get<0>(info.param), + "__beta_", ::testing::get<1>(info.param)); + return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); +} + +INSTANTIATE_TEST_CASE_P( + TestSampleStatisticsCombinations, BetaDistributionTest, + ::testing::Combine(::testing::Values(0.1, 0.2, 0.9, 1.1, 2.5, 10.0, 123.4), + ::testing::Values(0.1, 0.2, 0.9, 1.1, 2.5, 10.0, 123.4)), + ParamName); + +INSTANTIATE_TEST_CASE_P( + TestSampleStatistics_SelectedPairs, BetaDistributionTest, + ::testing::Values(std::make_pair(0.5, 1000), std::make_pair(1000, 0.5), + std::make_pair(900, 1000), std::make_pair(10000, 20000), + std::make_pair(4e5, 2e7), std::make_pair(1e7, 1e5)), + ParamName); + +// NOTE: absl::beta_distribution is not guaranteed to be stable. +TEST(BetaDistributionTest, StabilityTest) { + // absl::beta_distribution stability relies on the stability of + // absl::random_interna::RandU64ToDouble, std::exp, std::log, std::pow, + // and std::sqrt. + // + // This test also depends on the stability of std::frexp. + using testing::ElementsAre; + absl::random_internal::sequence_urbg urbg({ + 0xffff00000000e6c8ull, 0xffff0000000006c8ull, 0x800003766295CFA9ull, + 0x11C819684E734A41ull, 0x832603766295CFA9ull, 0x7fbe76c8b4395800ull, + 0xB3472DCA7B14A94Aull, 0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, + 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, 0x00035C904C70A239ull, + 0x00009E0BCBAADE14ull, 0x0000000000622CA7ull, 0x4864f22c059bf29eull, + 0x247856d8b862665cull, 0xe46e86e9a1337e10ull, 0xd8c8541f3519b133ull, + 0xffe75b52c567b9e4ull, 0xfffff732e5709c5bull, 0xff1f7f0b983532acull, + 0x1ec2e8986d2362caull, 0xC332DDEFBE6C5AA5ull, 0x6558218568AB9702ull, + 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, 0xECDD4775619F1510ull, + 0x814c8e35fe9a961aull, 0x0c3cd59c9b638a02ull, 0xcb3bb6478a07715cull, + 0x1224e62c978bbc7full, 0x671ef2cb04e81f6eull, 0x3c1cbd811eaf1808ull, + 0x1bbc23cfa8fac721ull, 0xa4c2cda65e596a51ull, 0xb77216fad37adf91ull, + 0x836d794457c08849ull, 0xe083df03475f49d7ull, 0xbc9feb512e6b0d6cull, + 0xb12d74fdd718c8c5ull, 0x12ff09653bfbe4caull, 0x8dd03a105bc4ee7eull, + 0x5738341045ba0d85ull, 0xf3fd722dc65ad09eull, 0xfa14fd21ea2a5705ull, + 0xffe6ea4d6edb0c73ull, 0xD07E9EFE2BF11FB4ull, 0x95DBDA4DAE909198ull, + 0xEAAD8E716B93D5A0ull, 0xD08ED1D0AFC725E0ull, 0x8E3C5B2F8E7594B7ull, + 0x8FF6E2FBF2122B64ull, 0x8888B812900DF01Cull, 0x4FAD5EA0688FC31Cull, + 0xD1CFF191B3A8C1ADull, 0x2F2F2218BE0E1777ull, 0xEA752DFE8B021FA1ull, + }); + + // Convert the real-valued result into a unit64 where we compare + // 5 (float) or 10 (double) decimal digits plus the base-2 exponent. + auto float_to_u64 = [](float d) { + int exp = 0; + auto f = std::frexp(d, &exp); + return (static_cast<uint64_t>(1e5 * f) * 10000) + std::abs(exp); + }; + auto double_to_u64 = [](double d) { + int exp = 0; + auto f = std::frexp(d, &exp); + return (static_cast<uint64_t>(1e10 * f) * 10000) + std::abs(exp); + }; + + std::vector<uint64_t> output(20); + { + // Algorithm Joehnk (float) + absl::beta_distribution<float> dist(0.1f, 0.2f); + std::generate(std::begin(output), std::end(output), + [&] { return float_to_u64(dist(urbg)); }); + EXPECT_EQ(44, urbg.invocations()); + EXPECT_THAT(output, // + testing::ElementsAre( + 998340000, 619030004, 500000001, 999990000, 996280000, + 500000001, 844740004, 847210001, 999970000, 872320000, + 585480007, 933280000, 869080042, 647670031, 528240004, + 969980004, 626050008, 915930002, 833440033, 878040015)); + } + + urbg.reset(); + { + // Algorithm Joehnk (double) + absl::beta_distribution<double> dist(0.1, 0.2); + std::generate(std::begin(output), std::end(output), + [&] { return double_to_u64(dist(urbg)); }); + EXPECT_EQ(44, urbg.invocations()); + EXPECT_THAT( + output, // + testing::ElementsAre( + 99834713000000, 61903356870004, 50000000000001, 99999721170000, + 99628374770000, 99999999990000, 84474397860004, 84721276240001, + 99997407490000, 87232528120000, 58548364780007, 93328932910000, + 86908237770042, 64767917930031, 52824581970004, 96998544140004, + 62605946270008, 91593604380002, 83345031740033, 87804397230015)); + } + + urbg.reset(); + { + // Algorithm Cheng 1 + absl::beta_distribution<double> dist(0.9, 2.0); + std::generate(std::begin(output), std::end(output), + [&] { return double_to_u64(dist(urbg)); }); + EXPECT_EQ(62, urbg.invocations()); + EXPECT_THAT( + output, // + testing::ElementsAre( + 62069004780001, 64433204450001, 53607416560000, 89644295430008, + 61434586310019, 55172615890002, 62187161490000, 56433684810003, + 80454622050005, 86418558710003, 92920514700001, 64645184680001, + 58549183380000, 84881283650005, 71078728590002, 69949694970000, + 73157461710001, 68592191300001, 70747623900000, 78584696930005)); + } + + urbg.reset(); + { + // Algorithm Cheng 2 + absl::beta_distribution<double> dist(1.5, 2.5); + std::generate(std::begin(output), std::end(output), + [&] { return double_to_u64(dist(urbg)); }); + EXPECT_EQ(54, urbg.invocations()); + EXPECT_THAT( + output, // + testing::ElementsAre( + 75000029250001, 76751482860001, 53264575220000, 69193133650005, + 78028324470013, 91573587560002, 59167523770000, 60658618560002, + 80075870540000, 94141320460004, 63196592770003, 78883906300002, + 96797992590001, 76907587800001, 56645167560000, 65408302280003, + 53401156320001, 64731238570000, 83065573750001, 79788333820001)); + } +} + +// This is an implementation-specific test. If any part of the implementation +// changes, then it is likely that this test will change as well. Also, if +// dependencies of the distribution change, such as RandU64ToDouble, then this +// is also likely to change. +TEST(BetaDistributionTest, AlgorithmBounds) { + { + absl::random_internal::sequence_urbg urbg( + {0x7fbe76c8b4395800ull, 0x8000000000000000ull}); + // u=0.499, v=0.5 + absl::beta_distribution<double> dist(1e-4, 1e-4); + double a = dist(urbg); + EXPECT_EQ(a, 2.0202860861567108529e-09); + EXPECT_EQ(2, urbg.invocations()); + } + + // Test that both the float & double algorithms appropriately reject the + // initial draw. + { + // 1/alpha = 1/beta = 2. + absl::beta_distribution<float> dist(0.5, 0.5); + + // first two outputs are close to 1.0 - epsilon, + // thus: (u ^ 2 + v ^ 2) > 1.0 + absl::random_internal::sequence_urbg urbg( + {0xffff00000006e6c8ull, 0xffff00000007c7c8ull, 0x800003766295CFA9ull, + 0x11C819684E734A41ull}); + { + double y = absl::beta_distribution<double>(0.5, 0.5)(urbg); + EXPECT_EQ(4, urbg.invocations()); + EXPECT_EQ(y, 0.9810668952633862) << y; + } + + // ...and: log(u) * a ~= log(v) * b ~= -0.02 + // thus z ~= -0.02 + log(1 + e(~0)) + // ~= -0.02 + 0.69 + // thus z > 0 + urbg.reset(); + { + float x = absl::beta_distribution<float>(0.5, 0.5)(urbg); + EXPECT_EQ(4, urbg.invocations()); + EXPECT_NEAR(0.98106688261032104, x, 0.0000005) << x << "f"; + } + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/bit_gen_ref.h b/third_party/abseil_cpp/absl/random/bit_gen_ref.h new file mode 100644 index 000000000000..9555460fd491 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/bit_gen_ref.h @@ -0,0 +1,181 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: bit_gen_ref.h +// ----------------------------------------------------------------------------- +// +// This header defines a bit generator "reference" class, for use in interfaces +// that take both Abseil (e.g. `absl::BitGen`) and standard library (e.g. +// `std::mt19937`) bit generators. + +#ifndef ABSL_RANDOM_BIT_GEN_REF_H_ +#define ABSL_RANDOM_BIT_GEN_REF_H_ + +#include "absl/base/internal/fast_type_id.h" +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" +#include "absl/random/internal/distribution_caller.h" +#include "absl/random/internal/fast_uniform_bits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +template <typename URBG, typename = void, typename = void, typename = void> +struct is_urbg : std::false_type {}; + +template <typename URBG> +struct is_urbg< + URBG, + absl::enable_if_t<std::is_same< + typename URBG::result_type, + typename std::decay<decltype((URBG::min)())>::type>::value>, + absl::enable_if_t<std::is_same< + typename URBG::result_type, + typename std::decay<decltype((URBG::max)())>::type>::value>, + absl::enable_if_t<std::is_same< + typename URBG::result_type, + typename std::decay<decltype(std::declval<URBG>()())>::type>::value>> + : std::true_type {}; + +template <typename> +struct DistributionCaller; +class MockHelpers; + +} // namespace random_internal + +// ----------------------------------------------------------------------------- +// absl::BitGenRef +// ----------------------------------------------------------------------------- +// +// `absl::BitGenRef` is a type-erasing class that provides a generator-agnostic +// non-owning "reference" interface for use in place of any specific uniform +// random bit generator (URBG). This class may be used for both Abseil +// (e.g. `absl::BitGen`, `absl::InsecureBitGen`) and Standard library (e.g +// `std::mt19937`, `std::minstd_rand`) bit generators. +// +// Like other reference classes, `absl::BitGenRef` does not own the +// underlying bit generator, and the underlying instance must outlive the +// `absl::BitGenRef`. +// +// `absl::BitGenRef` is particularly useful when used with an +// `absl::MockingBitGen` to test specific paths in functions which use random +// values. +// +// Example: +// void TakesBitGenRef(absl::BitGenRef gen) { +// int x = absl::Uniform<int>(gen, 0, 1000); +// } +// +class BitGenRef { + // SFINAE to detect whether the URBG type includes a member matching + // bool InvokeMock(base_internal::FastTypeIdType, void*, void*). + // + // These live inside BitGenRef so that they have friend access + // to MockingBitGen. (see similar methods in DistributionCaller). + template <template <class...> class Trait, class AlwaysVoid, class... Args> + struct detector : std::false_type {}; + template <template <class...> class Trait, class... Args> + struct detector<Trait, absl::void_t<Trait<Args...>>, Args...> + : std::true_type {}; + + template <class T> + using invoke_mock_t = decltype(std::declval<T*>()->InvokeMock( + std::declval<base_internal::FastTypeIdType>(), std::declval<void*>(), + std::declval<void*>())); + + template <typename T> + using HasInvokeMock = typename detector<invoke_mock_t, void, T>::type; + + public: + BitGenRef(const BitGenRef&) = default; + BitGenRef(BitGenRef&&) = default; + BitGenRef& operator=(const BitGenRef&) = default; + BitGenRef& operator=(BitGenRef&&) = default; + + template <typename URBG, typename absl::enable_if_t< + (!std::is_same<URBG, BitGenRef>::value && + random_internal::is_urbg<URBG>::value && + !HasInvokeMock<URBG>::value)>* = nullptr> + BitGenRef(URBG& gen) // NOLINT + : t_erased_gen_ptr_(reinterpret_cast<uintptr_t>(&gen)), + mock_call_(NotAMock), + generate_impl_fn_(ImplFn<URBG>) {} + + template <typename URBG, + typename absl::enable_if_t<(!std::is_same<URBG, BitGenRef>::value && + random_internal::is_urbg<URBG>::value && + HasInvokeMock<URBG>::value)>* = nullptr> + BitGenRef(URBG& gen) // NOLINT + : t_erased_gen_ptr_(reinterpret_cast<uintptr_t>(&gen)), + mock_call_(&MockCall<URBG>), + generate_impl_fn_(ImplFn<URBG>) {} + + using result_type = uint64_t; + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + result_type operator()() { return generate_impl_fn_(t_erased_gen_ptr_); } + + private: + using impl_fn = result_type (*)(uintptr_t); + using mock_call_fn = bool (*)(uintptr_t, base_internal::FastTypeIdType, void*, + void*); + + template <typename URBG> + static result_type ImplFn(uintptr_t ptr) { + // Ensure that the return values from operator() fill the entire + // range promised by result_type, min() and max(). + absl::random_internal::FastUniformBits<result_type> fast_uniform_bits; + return fast_uniform_bits(*reinterpret_cast<URBG*>(ptr)); + } + + // Get a type-erased InvokeMock pointer. + template <typename URBG> + static bool MockCall(uintptr_t gen_ptr, base_internal::FastTypeIdType type, + void* result, void* arg_tuple) { + return reinterpret_cast<URBG*>(gen_ptr)->InvokeMock(type, result, + arg_tuple); + } + static bool NotAMock(uintptr_t, base_internal::FastTypeIdType, void*, void*) { + return false; + } + + inline bool InvokeMock(base_internal::FastTypeIdType type, void* args_tuple, + void* result) { + if (mock_call_ == NotAMock) return false; // avoids an indirect call. + return mock_call_(t_erased_gen_ptr_, type, args_tuple, result); + } + + uintptr_t t_erased_gen_ptr_; + mock_call_fn mock_call_; + impl_fn generate_impl_fn_; + + template <typename> + friend struct ::absl::random_internal::DistributionCaller; // for InvokeMock + friend class ::absl::random_internal::MockHelpers; // for InvokeMock +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_BIT_GEN_REF_H_ diff --git a/third_party/abseil_cpp/absl/random/bit_gen_ref_test.cc b/third_party/abseil_cpp/absl/random/bit_gen_ref_test.cc new file mode 100644 index 000000000000..1135cf2da0bf --- /dev/null +++ b/third_party/abseil_cpp/absl/random/bit_gen_ref_test.cc @@ -0,0 +1,102 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include "absl/random/bit_gen_ref.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/fast_type_id.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class ConstBitGen { + public: + // URBG interface + using result_type = absl::BitGen::result_type; + + static constexpr result_type(min)() { return (absl::BitGen::min)(); } + static constexpr result_type(max)() { return (absl::BitGen::max)(); } + result_type operator()() { return 1; } + + // InvokeMock method + bool InvokeMock(base_internal::FastTypeIdType index, void*, void* result) { + *static_cast<int*>(result) = 42; + return true; + } +}; + +namespace { + +int FnTest(absl::BitGenRef gen_ref) { return absl::Uniform(gen_ref, 1, 7); } + +template <typename T> +class BitGenRefTest : public testing::Test {}; + +using BitGenTypes = + ::testing::Types<absl::BitGen, absl::InsecureBitGen, std::mt19937, + std::mt19937_64, std::minstd_rand>; +TYPED_TEST_SUITE(BitGenRefTest, BitGenTypes); + +TYPED_TEST(BitGenRefTest, BasicTest) { + TypeParam gen; + auto x = FnTest(gen); + EXPECT_NEAR(x, 4, 3); +} + +TYPED_TEST(BitGenRefTest, Copyable) { + TypeParam gen; + absl::BitGenRef gen_ref(gen); + FnTest(gen_ref); // Copy +} + +TEST(BitGenRefTest, PassThroughEquivalence) { + // sequence_urbg returns 64-bit results. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<uint64_t> output(12); + + { + absl::BitGenRef view(urbg); + for (auto& v : output) { + v = view(); + } + } + + std::vector<uint64_t> expected( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + EXPECT_THAT(output, testing::Eq(expected)); +} + +TEST(BitGenRefTest, MockingBitGenBaseOverrides) { + ConstBitGen const_gen; + EXPECT_EQ(FnTest(const_gen), 42); + + absl::BitGenRef gen_ref(const_gen); + EXPECT_EQ(FnTest(gen_ref), 42); // Copy +} +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/discrete_distribution.cc b/third_party/abseil_cpp/absl/random/discrete_distribution.cc new file mode 100644 index 000000000000..081accee52a4 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/discrete_distribution.cc @@ -0,0 +1,98 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/discrete_distribution.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Initializes the distribution table for Walker's Aliasing algorithm, described +// in Knuth, Vol 2. as well as in https://en.wikipedia.org/wiki/Alias_method +std::vector<std::pair<double, size_t>> InitDiscreteDistribution( + std::vector<double>* probabilities) { + // The empty-case should already be handled by the constructor. + assert(probabilities); + assert(!probabilities->empty()); + + // Step 1. Normalize the input probabilities to 1.0. + double sum = std::accumulate(std::begin(*probabilities), + std::end(*probabilities), 0.0); + if (std::fabs(sum - 1.0) > 1e-6) { + // Scale `probabilities` only when the sum is too far from 1.0. Scaling + // unconditionally will alter the probabilities slightly. + for (double& item : *probabilities) { + item = item / sum; + } + } + + // Step 2. At this point `probabilities` is set to the conditional + // probabilities of each element which sum to 1.0, to within reasonable error. + // These values are used to construct the proportional probability tables for + // the selection phases of Walker's Aliasing algorithm. + // + // To construct the table, pick an element which is under-full (i.e., an + // element for which `(*probabilities)[i] < 1.0/n`), and pair it with an + // element which is over-full (i.e., an element for which + // `(*probabilities)[i] > 1.0/n`). The smaller value can always be retired. + // The larger may still be greater than 1.0/n, or may now be less than 1.0/n, + // and put back onto the appropriate collection. + const size_t n = probabilities->size(); + std::vector<std::pair<double, size_t>> q; + q.reserve(n); + + std::vector<size_t> over; + std::vector<size_t> under; + size_t idx = 0; + for (const double item : *probabilities) { + assert(item >= 0); + const double v = item * n; + q.emplace_back(v, 0); + if (v < 1.0) { + under.push_back(idx++); + } else { + over.push_back(idx++); + } + } + while (!over.empty() && !under.empty()) { + auto lo = under.back(); + under.pop_back(); + auto hi = over.back(); + over.pop_back(); + + q[lo].second = hi; + const double r = q[hi].first - (1.0 - q[lo].first); + q[hi].first = r; + if (r < 1.0) { + under.push_back(hi); + } else { + over.push_back(hi); + } + } + + // Due to rounding errors, there may be un-paired elements in either + // collection; these should all be values near 1.0. For these values, set `q` + // to 1.0 and set the alternate to the identity. + for (auto i : over) { + q[i] = {1.0, i}; + } + for (auto i : under) { + q[i] = {1.0, i}; + } + return q; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/discrete_distribution.h b/third_party/abseil_cpp/absl/random/discrete_distribution.h new file mode 100644 index 000000000000..171aa11a1eb4 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/discrete_distribution.h @@ -0,0 +1,247 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_DISCRETE_DISTRIBUTION_H_ +#define ABSL_RANDOM_DISCRETE_DISTRIBUTION_H_ + +#include <cassert> +#include <cmath> +#include <istream> +#include <limits> +#include <numeric> +#include <type_traits> +#include <utility> +#include <vector> + +#include "absl/random/bernoulli_distribution.h" +#include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/uniform_int_distribution.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::discrete_distribution +// +// A discrete distribution produces random integers i, where 0 <= i < n +// distributed according to the discrete probability function: +// +// P(i|p0,...,pn−1)=pi +// +// This class is an implementation of discrete_distribution (see +// [rand.dist.samp.discrete]). +// +// The algorithm used is Walker's Aliasing algorithm, described in Knuth, Vol 2. +// absl::discrete_distribution takes O(N) time to precompute the probabilities +// (where N is the number of possible outcomes in the distribution) at +// construction, and then takes O(1) time for each variate generation. Many +// other implementations also take O(N) time to construct an ordered sequence of +// partial sums, plus O(log N) time per variate to binary search. +// +template <typename IntType = int> +class discrete_distribution { + public: + using result_type = IntType; + + class param_type { + public: + using distribution_type = discrete_distribution; + + param_type() { init(); } + + template <typename InputIterator> + explicit param_type(InputIterator begin, InputIterator end) + : p_(begin, end) { + init(); + } + + explicit param_type(std::initializer_list<double> weights) : p_(weights) { + init(); + } + + template <class UnaryOperation> + explicit param_type(size_t nw, double xmin, double xmax, + UnaryOperation fw) { + if (nw > 0) { + p_.reserve(nw); + double delta = (xmax - xmin) / static_cast<double>(nw); + assert(delta > 0); + double t = delta * 0.5; + for (size_t i = 0; i < nw; ++i) { + p_.push_back(fw(xmin + i * delta + t)); + } + } + init(); + } + + const std::vector<double>& probabilities() const { return p_; } + size_t n() const { return p_.size() - 1; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.probabilities() == b.probabilities(); + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class discrete_distribution; + + void init(); + + std::vector<double> p_; // normalized probabilities + std::vector<std::pair<double, size_t>> q_; // (acceptance, alternate) pairs + + static_assert(std::is_integral<result_type>::value, + "Class-template absl::discrete_distribution<> must be " + "parameterized using an integral type."); + }; + + discrete_distribution() : param_() {} + + explicit discrete_distribution(const param_type& p) : param_(p) {} + + template <typename InputIterator> + explicit discrete_distribution(InputIterator begin, InputIterator end) + : param_(begin, end) {} + + explicit discrete_distribution(std::initializer_list<double> weights) + : param_(weights) {} + + template <class UnaryOperation> + explicit discrete_distribution(size_t nw, double xmin, double xmax, + UnaryOperation fw) + : param_(nw, xmin, xmax, std::move(fw)) {} + + void reset() {} + + // generating functions + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + const param_type& param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + result_type(min)() const { return 0; } + result_type(max)() const { + return static_cast<result_type>(param_.n()); + } // inclusive + + // NOTE [rand.dist.sample.discrete] returns a std::vector<double> not a + // const std::vector<double>&. + const std::vector<double>& probabilities() const { + return param_.probabilities(); + } + + friend bool operator==(const discrete_distribution& a, + const discrete_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const discrete_distribution& a, + const discrete_distribution& b) { + return a.param_ != b.param_; + } + + private: + param_type param_; +}; + +// -------------------------------------------------------------------------- +// Implementation details only below +// -------------------------------------------------------------------------- + +namespace random_internal { + +// Using the vector `*probabilities`, whose values are the weights or +// probabilities of an element being selected, constructs the proportional +// probabilities used by the discrete distribution. `*probabilities` will be +// scaled, if necessary, so that its entries sum to a value sufficiently close +// to 1.0. +std::vector<std::pair<double, size_t>> InitDiscreteDistribution( + std::vector<double>* probabilities); + +} // namespace random_internal + +template <typename IntType> +void discrete_distribution<IntType>::param_type::init() { + if (p_.empty()) { + p_.push_back(1.0); + q_.emplace_back(1.0, 0); + } else { + assert(n() <= (std::numeric_limits<IntType>::max)()); + q_ = random_internal::InitDiscreteDistribution(&p_); + } +} + +template <typename IntType> +template <typename URBG> +typename discrete_distribution<IntType>::result_type +discrete_distribution<IntType>::operator()( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + const auto idx = absl::uniform_int_distribution<result_type>(0, p.n())(g); + const auto& q = p.q_[idx]; + const bool selected = absl::bernoulli_distribution(q.first)(g); + return selected ? idx : static_cast<result_type>(q.second); +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const discrete_distribution<IntType>& x) { + auto saver = random_internal::make_ostream_state_saver(os); + const auto& probabilities = x.param().probabilities(); + os << probabilities.size(); + + os.precision(random_internal::stream_precision_helper<double>::kPrecision); + for (const auto& p : probabilities) { + os << os.fill() << p; + } + return os; +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + discrete_distribution<IntType>& x) { // NOLINT(runtime/references) + using param_type = typename discrete_distribution<IntType>::param_type; + auto saver = random_internal::make_istream_state_saver(is); + + size_t n; + std::vector<double> p; + + is >> n; + if (is.fail()) return is; + if (n > 0) { + p.reserve(n); + for (IntType i = 0; i < n && !is.fail(); ++i) { + auto tmp = random_internal::read_floating_point<double>(is); + if (is.fail()) return is; + p.push_back(tmp); + } + } + x.param(param_type(p.begin(), p.end())); + return is; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_DISCRETE_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/discrete_distribution_test.cc b/third_party/abseil_cpp/absl/random/discrete_distribution_test.cc new file mode 100644 index 000000000000..6d007006ef48 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/discrete_distribution_test.cc @@ -0,0 +1,250 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/discrete_distribution.h" + +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <numeric> +#include <random> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/strip.h" + +namespace { + +template <typename IntType> +class DiscreteDistributionTypeTest : public ::testing::Test {}; + +using IntTypes = ::testing::Types<int8_t, uint8_t, int16_t, uint16_t, int32_t, + uint32_t, int64_t, uint64_t>; +TYPED_TEST_SUITE(DiscreteDistributionTypeTest, IntTypes); + +TYPED_TEST(DiscreteDistributionTypeTest, ParamSerializeTest) { + using param_type = + typename absl::discrete_distribution<TypeParam>::param_type; + + absl::discrete_distribution<TypeParam> empty; + EXPECT_THAT(empty.probabilities(), testing::ElementsAre(1.0)); + + absl::discrete_distribution<TypeParam> before({1.0, 2.0, 1.0}); + + // Validate that the probabilities sum to 1.0. We picked values which + // can be represented exactly to avoid floating-point roundoff error. + double s = 0; + for (const auto& x : before.probabilities()) { + s += x; + } + EXPECT_EQ(s, 1.0); + EXPECT_THAT(before.probabilities(), testing::ElementsAre(0.25, 0.5, 0.25)); + + // Validate the same data via an initializer list. + { + std::vector<double> data({1.0, 2.0, 1.0}); + + absl::discrete_distribution<TypeParam> via_param{ + param_type(std::begin(data), std::end(data))}; + + EXPECT_EQ(via_param, before); + } + + std::stringstream ss; + ss << before; + absl::discrete_distribution<TypeParam> after; + + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before, after); +} + +TYPED_TEST(DiscreteDistributionTypeTest, Constructor) { + auto fn = [](double x) { return x; }; + { + absl::discrete_distribution<int> unary(0, 1.0, 9.0, fn); + EXPECT_THAT(unary.probabilities(), testing::ElementsAre(1.0)); + } + + { + absl::discrete_distribution<int> unary(2, 1.0, 9.0, fn); + // => fn(1.0 + 0 * 4 + 2) => 3 + // => fn(1.0 + 1 * 4 + 2) => 7 + EXPECT_THAT(unary.probabilities(), testing::ElementsAre(0.3, 0.7)); + } +} + +TEST(DiscreteDistributionTest, InitDiscreteDistribution) { + using testing::Pair; + + { + std::vector<double> p({1.0, 2.0, 3.0}); + std::vector<std::pair<double, size_t>> q = + absl::random_internal::InitDiscreteDistribution(&p); + + EXPECT_THAT(p, testing::ElementsAre(1 / 6.0, 2 / 6.0, 3 / 6.0)); + + // Each bucket is p=1/3, so bucket 0 will send half it's traffic + // to bucket 2, while the rest will retain all of their traffic. + EXPECT_THAT(q, testing::ElementsAre(Pair(0.5, 2), // + Pair(1.0, 1), // + Pair(1.0, 2))); + } + + { + std::vector<double> p({1.0, 2.0, 3.0, 5.0, 2.0}); + + std::vector<std::pair<double, size_t>> q = + absl::random_internal::InitDiscreteDistribution(&p); + + EXPECT_THAT(p, testing::ElementsAre(1 / 13.0, 2 / 13.0, 3 / 13.0, 5 / 13.0, + 2 / 13.0)); + + // A more complex bucketing solution: Each bucket has p=0.2 + // So buckets 0, 1, 4 will send their alternate traffic elsewhere, which + // happens to be bucket 3. + // However, summing up that alternate traffic gives bucket 3 too much + // traffic, so it will send some traffic to bucket 2. + constexpr double b0 = 1.0 / 13.0 / 0.2; + constexpr double b1 = 2.0 / 13.0 / 0.2; + constexpr double b3 = (5.0 / 13.0 / 0.2) - ((1 - b0) + (1 - b1) + (1 - b1)); + + EXPECT_THAT(q, testing::ElementsAre(Pair(b0, 3), // + Pair(b1, 3), // + Pair(1.0, 2), // + Pair(b3, 2), // + Pair(b1, 3))); + } +} + +TEST(DiscreteDistributionTest, ChiSquaredTest50) { + using absl::random_internal::kChiSquared; + + constexpr size_t kTrials = 10000; + constexpr int kBuckets = 50; // inclusive, so actally +1 + + // 1-in-100000 threshold, but remember, there are about 8 tests + // in this file. And the test could fail for other reasons. + // Empirically validated with --runs_per_test=10000. + const int kThreshold = + absl::random_internal::ChiSquareValue(kBuckets, 0.99999); + + std::vector<double> weights(kBuckets, 0); + std::iota(std::begin(weights), std::end(weights), 1); + absl::discrete_distribution<int> dist(std::begin(weights), std::end(weights)); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng(0x2B7E151628AED2A6); + + std::vector<int32_t> counts(kBuckets, 0); + for (size_t i = 0; i < kTrials; i++) { + auto x = dist(rng); + counts[x]++; + } + + // Scale weights. + double sum = 0; + for (double x : weights) { + sum += x; + } + for (double& x : weights) { + x = kTrials * (x / sum); + } + + double chi_square = + absl::random_internal::ChiSquare(std::begin(counts), std::end(counts), + std::begin(weights), std::end(weights)); + + if (chi_square > kThreshold) { + double p_value = + absl::random_internal::ChiSquarePValue(chi_square, kBuckets); + + // Chi-squared test failed. Output does not appear to be uniform. + std::string msg; + for (size_t i = 0; i < counts.size(); i++) { + absl::StrAppend(&msg, i, ": ", counts[i], " vs ", weights[i], "\n"); + } + absl::StrAppend(&msg, kChiSquared, " p-value ", p_value, "\n"); + absl::StrAppend(&msg, "High ", kChiSquared, " value: ", chi_square, " > ", + kThreshold); + ABSL_RAW_LOG(INFO, "%s", msg.c_str()); + FAIL() << msg; + } +} + +TEST(DiscreteDistributionTest, StabilityTest) { + // absl::discrete_distribution stabilitiy relies on + // absl::uniform_int_distribution and absl::bernoulli_distribution. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(6); + + { + absl::discrete_distribution<int32_t> dist({1.0, 2.0, 3.0, 5.0, 2.0}); + EXPECT_EQ(0, dist.min()); + EXPECT_EQ(4, dist.max()); + for (auto& v : output) { + v = dist(urbg); + } + EXPECT_EQ(12, urbg.invocations()); + } + + // With 12 calls to urbg, each call into discrete_distribution consumes + // precisely 2 values: one for the uniform call, and a second for the + // bernoulli. + // + // Given the alt mapping: 0=>3, 1=>3, 2=>2, 3=>2, 4=>3, we can + // + // uniform: 443210143131 + // bernoulli: b0 000011100101 + // bernoulli: b1 001111101101 + // bernoulli: b2 111111111111 + // bernoulli: b3 001111101111 + // bernoulli: b4 001111101101 + // ... + EXPECT_THAT(output, testing::ElementsAre(3, 3, 1, 3, 3, 3)); + + { + urbg.reset(); + absl::discrete_distribution<int64_t> dist({1.0, 2.0, 3.0, 5.0, 2.0}); + EXPECT_EQ(0, dist.min()); + EXPECT_EQ(4, dist.max()); + for (auto& v : output) { + v = dist(urbg); + } + EXPECT_EQ(12, urbg.invocations()); + } + EXPECT_THAT(output, testing::ElementsAre(3, 3, 0, 3, 0, 4)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/distributions.h b/third_party/abseil_cpp/absl/random/distributions.h new file mode 100644 index 000000000000..31c79694e51b --- /dev/null +++ b/third_party/abseil_cpp/absl/random/distributions.h @@ -0,0 +1,452 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: distributions.h +// ----------------------------------------------------------------------------- +// +// This header defines functions representing distributions, which you use in +// combination with an Abseil random bit generator to produce random values +// according to the rules of that distribution. +// +// The Abseil random library defines the following distributions within this +// file: +// +// * `absl::Uniform` for uniform (constant) distributions having constant +// probability +// * `absl::Bernoulli` for discrete distributions having exactly two outcomes +// * `absl::Beta` for continuous distributions parameterized through two +// free parameters +// * `absl::Exponential` for discrete distributions of events occurring +// continuously and independently at a constant average rate +// * `absl::Gaussian` (also known as "normal distributions") for continuous +// distributions using an associated quadratic function +// * `absl::LogUniform` for continuous uniform distributions where the log +// to the given base of all values is uniform +// * `absl::Poisson` for discrete probability distributions that express the +// probability of a given number of events occurring within a fixed interval +// * `absl::Zipf` for discrete probability distributions commonly used for +// modelling of rare events +// +// Prefer use of these distribution function classes over manual construction of +// your own distribution classes, as it allows library maintainers greater +// flexibility to change the underlying implementation in the future. + +#ifndef ABSL_RANDOM_DISTRIBUTIONS_H_ +#define ABSL_RANDOM_DISTRIBUTIONS_H_ + +#include <algorithm> +#include <cmath> +#include <limits> +#include <random> +#include <type_traits> + +#include "absl/base/internal/inline_variable.h" +#include "absl/random/bernoulli_distribution.h" +#include "absl/random/beta_distribution.h" +#include "absl/random/exponential_distribution.h" +#include "absl/random/gaussian_distribution.h" +#include "absl/random/internal/distribution_caller.h" // IWYU pragma: export +#include "absl/random/internal/uniform_helper.h" // IWYU pragma: export +#include "absl/random/log_uniform_int_distribution.h" +#include "absl/random/poisson_distribution.h" +#include "absl/random/uniform_int_distribution.h" +#include "absl/random/uniform_real_distribution.h" +#include "absl/random/zipf_distribution.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_INTERNAL_INLINE_CONSTEXPR(IntervalClosedClosedTag, IntervalClosedClosed, + {}); +ABSL_INTERNAL_INLINE_CONSTEXPR(IntervalClosedClosedTag, IntervalClosed, {}); +ABSL_INTERNAL_INLINE_CONSTEXPR(IntervalClosedOpenTag, IntervalClosedOpen, {}); +ABSL_INTERNAL_INLINE_CONSTEXPR(IntervalOpenOpenTag, IntervalOpenOpen, {}); +ABSL_INTERNAL_INLINE_CONSTEXPR(IntervalOpenOpenTag, IntervalOpen, {}); +ABSL_INTERNAL_INLINE_CONSTEXPR(IntervalOpenClosedTag, IntervalOpenClosed, {}); + +// ----------------------------------------------------------------------------- +// absl::Uniform<T>(tag, bitgen, lo, hi) +// ----------------------------------------------------------------------------- +// +// `absl::Uniform()` produces random values of type `T` uniformly distributed in +// a defined interval {lo, hi}. The interval `tag` defines the type of interval +// which should be one of the following possible values: +// +// * `absl::IntervalOpenOpen` +// * `absl::IntervalOpenClosed` +// * `absl::IntervalClosedOpen` +// * `absl::IntervalClosedClosed` +// +// where "open" refers to an exclusive value (excluded) from the output, while +// "closed" refers to an inclusive value (included) from the output. +// +// In the absence of an explicit return type `T`, `absl::Uniform()` will deduce +// the return type based on the provided endpoint arguments {A lo, B hi}. +// Given these endpoints, one of {A, B} will be chosen as the return type, if +// a type can be implicitly converted into the other in a lossless way. The +// lack of any such implicit conversion between {A, B} will produce a +// compile-time error +// +// See https://en.wikipedia.org/wiki/Uniform_distribution_(continuous) +// +// Example: +// +// absl::BitGen bitgen; +// +// // Produce a random float value between 0.0 and 1.0, inclusive +// auto x = absl::Uniform(absl::IntervalClosedClosed, bitgen, 0.0f, 1.0f); +// +// // The most common interval of `absl::IntervalClosedOpen` is available by +// // default: +// +// auto x = absl::Uniform(bitgen, 0.0f, 1.0f); +// +// // Return-types are typically inferred from the arguments, however callers +// // can optionally provide an explicit return-type to the template. +// +// auto x = absl::Uniform<float>(bitgen, 0, 1); +// +template <typename R = void, typename TagType, typename URBG> +typename absl::enable_if_t<!std::is_same<R, void>::value, R> // +Uniform(TagType tag, + URBG&& urbg, // NOLINT(runtime/references) + R lo, R hi) { + using gen_t = absl::decay_t<URBG>; + using distribution_t = random_internal::UniformDistributionWrapper<R>; + + auto a = random_internal::uniform_lower_bound(tag, lo, hi); + auto b = random_internal::uniform_upper_bound(tag, lo, hi); + if (!random_internal::is_uniform_range_valid(a, b)) return lo; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, tag, lo, hi); +} + +// absl::Uniform<T>(bitgen, lo, hi) +// +// Overload of `Uniform()` using the default closed-open interval of [lo, hi), +// and returning values of type `T` +template <typename R = void, typename URBG> +typename absl::enable_if_t<!std::is_same<R, void>::value, R> // +Uniform(URBG&& urbg, // NOLINT(runtime/references) + R lo, R hi) { + using gen_t = absl::decay_t<URBG>; + using distribution_t = random_internal::UniformDistributionWrapper<R>; + constexpr auto tag = absl::IntervalClosedOpen; + + auto a = random_internal::uniform_lower_bound(tag, lo, hi); + auto b = random_internal::uniform_upper_bound(tag, lo, hi); + if (!random_internal::is_uniform_range_valid(a, b)) return lo; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, lo, hi); +} + +// absl::Uniform(tag, bitgen, lo, hi) +// +// Overload of `Uniform()` using different (but compatible) lo, hi types. Note +// that a compile-error will result if the return type cannot be deduced +// correctly from the passed types. +template <typename R = void, typename TagType, typename URBG, typename A, + typename B> +typename absl::enable_if_t<std::is_same<R, void>::value, + random_internal::uniform_inferred_return_t<A, B>> +Uniform(TagType tag, + URBG&& urbg, // NOLINT(runtime/references) + A lo, B hi) { + using gen_t = absl::decay_t<URBG>; + using return_t = typename random_internal::uniform_inferred_return_t<A, B>; + using distribution_t = random_internal::UniformDistributionWrapper<return_t>; + + auto a = random_internal::uniform_lower_bound<return_t>(tag, lo, hi); + auto b = random_internal::uniform_upper_bound<return_t>(tag, lo, hi); + if (!random_internal::is_uniform_range_valid(a, b)) return lo; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, tag, static_cast<return_t>(lo), + static_cast<return_t>(hi)); +} + +// absl::Uniform(bitgen, lo, hi) +// +// Overload of `Uniform()` using different (but compatible) lo, hi types and the +// default closed-open interval of [lo, hi). Note that a compile-error will +// result if the return type cannot be deduced correctly from the passed types. +template <typename R = void, typename URBG, typename A, typename B> +typename absl::enable_if_t<std::is_same<R, void>::value, + random_internal::uniform_inferred_return_t<A, B>> +Uniform(URBG&& urbg, // NOLINT(runtime/references) + A lo, B hi) { + using gen_t = absl::decay_t<URBG>; + using return_t = typename random_internal::uniform_inferred_return_t<A, B>; + using distribution_t = random_internal::UniformDistributionWrapper<return_t>; + + constexpr auto tag = absl::IntervalClosedOpen; + auto a = random_internal::uniform_lower_bound<return_t>(tag, lo, hi); + auto b = random_internal::uniform_upper_bound<return_t>(tag, lo, hi); + if (!random_internal::is_uniform_range_valid(a, b)) return lo; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, static_cast<return_t>(lo), + static_cast<return_t>(hi)); +} + +// absl::Uniform<unsigned T>(bitgen) +// +// Overload of Uniform() using the minimum and maximum values of a given type +// `T` (which must be unsigned), returning a value of type `unsigned T` +template <typename R, typename URBG> +typename absl::enable_if_t<!std::is_signed<R>::value, R> // +Uniform(URBG&& urbg) { // NOLINT(runtime/references) + using gen_t = absl::decay_t<URBG>; + using distribution_t = random_internal::UniformDistributionWrapper<R>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg); +} + +// ----------------------------------------------------------------------------- +// absl::Bernoulli(bitgen, p) +// ----------------------------------------------------------------------------- +// +// `absl::Bernoulli` produces a random boolean value, with probability `p` +// (where 0.0 <= p <= 1.0) equaling `true`. +// +// Prefer `absl::Bernoulli` to produce boolean values over other alternatives +// such as comparing an `absl::Uniform()` value to a specific output. +// +// See https://en.wikipedia.org/wiki/Bernoulli_distribution +// +// Example: +// +// absl::BitGen bitgen; +// ... +// if (absl::Bernoulli(bitgen, 1.0/3721.0)) { +// std::cout << "Asteroid field navigation successful."; +// } +// +template <typename URBG> +bool Bernoulli(URBG&& urbg, // NOLINT(runtime/references) + double p) { + using gen_t = absl::decay_t<URBG>; + using distribution_t = absl::bernoulli_distribution; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, p); +} + +// ----------------------------------------------------------------------------- +// absl::Beta<T>(bitgen, alpha, beta) +// ----------------------------------------------------------------------------- +// +// `absl::Beta` produces a floating point number distributed in the closed +// interval [0,1] and parameterized by two values `alpha` and `beta` as per a +// Beta distribution. `T` must be a floating point type, but may be inferred +// from the types of `alpha` and `beta`. +// +// See https://en.wikipedia.org/wiki/Beta_distribution. +// +// Example: +// +// absl::BitGen bitgen; +// ... +// double sample = absl::Beta(bitgen, 3.0, 2.0); +// +template <typename RealType, typename URBG> +RealType Beta(URBG&& urbg, // NOLINT(runtime/references) + RealType alpha, RealType beta) { + static_assert( + std::is_floating_point<RealType>::value, + "Template-argument 'RealType' must be a floating-point type, in " + "absl::Beta<RealType, URBG>(...)"); + + using gen_t = absl::decay_t<URBG>; + using distribution_t = typename absl::beta_distribution<RealType>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, alpha, beta); +} + +// ----------------------------------------------------------------------------- +// absl::Exponential<T>(bitgen, lambda = 1) +// ----------------------------------------------------------------------------- +// +// `absl::Exponential` produces a floating point number representing the +// distance (time) between two consecutive events in a point process of events +// occurring continuously and independently at a constant average rate. `T` must +// be a floating point type, but may be inferred from the type of `lambda`. +// +// See https://en.wikipedia.org/wiki/Exponential_distribution. +// +// Example: +// +// absl::BitGen bitgen; +// ... +// double call_length = absl::Exponential(bitgen, 7.0); +// +template <typename RealType, typename URBG> +RealType Exponential(URBG&& urbg, // NOLINT(runtime/references) + RealType lambda = 1) { + static_assert( + std::is_floating_point<RealType>::value, + "Template-argument 'RealType' must be a floating-point type, in " + "absl::Exponential<RealType, URBG>(...)"); + + using gen_t = absl::decay_t<URBG>; + using distribution_t = typename absl::exponential_distribution<RealType>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, lambda); +} + +// ----------------------------------------------------------------------------- +// absl::Gaussian<T>(bitgen, mean = 0, stddev = 1) +// ----------------------------------------------------------------------------- +// +// `absl::Gaussian` produces a floating point number selected from the Gaussian +// (ie. "Normal") distribution. `T` must be a floating point type, but may be +// inferred from the types of `mean` and `stddev`. +// +// See https://en.wikipedia.org/wiki/Normal_distribution +// +// Example: +// +// absl::BitGen bitgen; +// ... +// double giraffe_height = absl::Gaussian(bitgen, 16.3, 3.3); +// +template <typename RealType, typename URBG> +RealType Gaussian(URBG&& urbg, // NOLINT(runtime/references) + RealType mean = 0, RealType stddev = 1) { + static_assert( + std::is_floating_point<RealType>::value, + "Template-argument 'RealType' must be a floating-point type, in " + "absl::Gaussian<RealType, URBG>(...)"); + + using gen_t = absl::decay_t<URBG>; + using distribution_t = typename absl::gaussian_distribution<RealType>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, mean, stddev); +} + +// ----------------------------------------------------------------------------- +// absl::LogUniform<T>(bitgen, lo, hi, base = 2) +// ----------------------------------------------------------------------------- +// +// `absl::LogUniform` produces random values distributed where the log to a +// given base of all values is uniform in a closed interval [lo, hi]. `T` must +// be an integral type, but may be inferred from the types of `lo` and `hi`. +// +// I.e., `LogUniform(0, n, b)` is uniformly distributed across buckets +// [0], [1, b-1], [b, b^2-1] .. [b^(k-1), (b^k)-1] .. [b^floor(log(n, b)), n] +// and is uniformly distributed within each bucket. +// +// The resulting probability density is inversely related to bucket size, though +// values in the final bucket may be more likely than previous values. (In the +// extreme case where n = b^i the final value will be tied with zero as the most +// probable result. +// +// If `lo` is nonzero then this distribution is shifted to the desired interval, +// so LogUniform(lo, hi, b) is equivalent to LogUniform(0, hi-lo, b)+lo. +// +// See http://ecolego.facilia.se/ecolego/show/Log-Uniform%20Distribution +// +// Example: +// +// absl::BitGen bitgen; +// ... +// int v = absl::LogUniform(bitgen, 0, 1000); +// +template <typename IntType, typename URBG> +IntType LogUniform(URBG&& urbg, // NOLINT(runtime/references) + IntType lo, IntType hi, IntType base = 2) { + static_assert(std::is_integral<IntType>::value, + "Template-argument 'IntType' must be an integral type, in " + "absl::LogUniform<IntType, URBG>(...)"); + + using gen_t = absl::decay_t<URBG>; + using distribution_t = typename absl::log_uniform_int_distribution<IntType>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, lo, hi, base); +} + +// ----------------------------------------------------------------------------- +// absl::Poisson<T>(bitgen, mean = 1) +// ----------------------------------------------------------------------------- +// +// `absl::Poisson` produces discrete probabilities for a given number of events +// occurring within a fixed interval within the closed interval [0, max]. `T` +// must be an integral type. +// +// See https://en.wikipedia.org/wiki/Poisson_distribution +// +// Example: +// +// absl::BitGen bitgen; +// ... +// int requests_per_minute = absl::Poisson<int>(bitgen, 3.2); +// +template <typename IntType, typename URBG> +IntType Poisson(URBG&& urbg, // NOLINT(runtime/references) + double mean = 1.0) { + static_assert(std::is_integral<IntType>::value, + "Template-argument 'IntType' must be an integral type, in " + "absl::Poisson<IntType, URBG>(...)"); + + using gen_t = absl::decay_t<URBG>; + using distribution_t = typename absl::poisson_distribution<IntType>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, mean); +} + +// ----------------------------------------------------------------------------- +// absl::Zipf<T>(bitgen, hi = max, q = 2, v = 1) +// ----------------------------------------------------------------------------- +// +// `absl::Zipf` produces discrete probabilities commonly used for modelling of +// rare events over the closed interval [0, hi]. The parameters `v` and `q` +// determine the skew of the distribution. `T` must be an integral type, but +// may be inferred from the type of `hi`. +// +// See http://mathworld.wolfram.com/ZipfDistribution.html +// +// Example: +// +// absl::BitGen bitgen; +// ... +// int term_rank = absl::Zipf<int>(bitgen); +// +template <typename IntType, typename URBG> +IntType Zipf(URBG&& urbg, // NOLINT(runtime/references) + IntType hi = (std::numeric_limits<IntType>::max)(), double q = 2.0, + double v = 1.0) { + static_assert(std::is_integral<IntType>::value, + "Template-argument 'IntType' must be an integral type, in " + "absl::Zipf<IntType, URBG>(...)"); + + using gen_t = absl::decay_t<URBG>; + using distribution_t = typename absl::zipf_distribution<IntType>; + + return random_internal::DistributionCaller<gen_t>::template Call< + distribution_t>(&urbg, hi, q, v); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_DISTRIBUTIONS_H_ diff --git a/third_party/abseil_cpp/absl/random/distributions_test.cc b/third_party/abseil_cpp/absl/random/distributions_test.cc new file mode 100644 index 000000000000..5866a07257e5 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/distributions_test.cc @@ -0,0 +1,455 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/distributions.h" + +#include <cmath> +#include <cstdint> +#include <random> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/random.h" + +namespace { + +constexpr int kSize = 400000; + +class RandomDistributionsTest : public testing::Test {}; + + +struct Invalid {}; + +template <typename A, typename B> +auto InferredUniformReturnT(int) + -> decltype(absl::Uniform(std::declval<absl::InsecureBitGen&>(), + std::declval<A>(), std::declval<B>())); + +template <typename, typename> +Invalid InferredUniformReturnT(...); + +template <typename TagType, typename A, typename B> +auto InferredTaggedUniformReturnT(int) + -> decltype(absl::Uniform(std::declval<TagType>(), + std::declval<absl::InsecureBitGen&>(), + std::declval<A>(), std::declval<B>())); + +template <typename, typename, typename> +Invalid InferredTaggedUniformReturnT(...); + +// Given types <A, B, Expect>, CheckArgsInferType() verifies that +// +// absl::Uniform(gen, A{}, B{}) +// +// returns the type "Expect". +// +// This interface can also be used to assert that a given absl::Uniform() +// overload does not exist / will not compile. Given types <A, B>, the +// expression +// +// decltype(absl::Uniform(..., std::declval<A>(), std::declval<B>())) +// +// will not compile, leaving the definition of InferredUniformReturnT<A, B> to +// resolve (via SFINAE) to the overload which returns type "Invalid". This +// allows tests to assert that an invocation such as +// +// absl::Uniform(gen, 1.23f, std::numeric_limits<int>::max() - 1) +// +// should not compile, since neither type, float nor int, can precisely +// represent both endpoint-values. Writing: +// +// CheckArgsInferType<float, int, Invalid>() +// +// will assert that this overload does not exist. +template <typename A, typename B, typename Expect> +void CheckArgsInferType() { + static_assert( + absl::conjunction< + std::is_same<Expect, decltype(InferredUniformReturnT<A, B>(0))>, + std::is_same<Expect, + decltype(InferredUniformReturnT<B, A>(0))>>::value, + ""); + static_assert( + absl::conjunction< + std::is_same<Expect, decltype(InferredTaggedUniformReturnT< + absl::IntervalOpenOpenTag, A, B>(0))>, + std::is_same<Expect, + decltype(InferredTaggedUniformReturnT< + absl::IntervalOpenOpenTag, B, A>(0))>>::value, + ""); +} + +template <typename A, typename B, typename ExplicitRet> +auto ExplicitUniformReturnT(int) -> decltype( + absl::Uniform<ExplicitRet>(*std::declval<absl::InsecureBitGen*>(), + std::declval<A>(), std::declval<B>())); + +template <typename, typename, typename ExplicitRet> +Invalid ExplicitUniformReturnT(...); + +template <typename TagType, typename A, typename B, typename ExplicitRet> +auto ExplicitTaggedUniformReturnT(int) -> decltype(absl::Uniform<ExplicitRet>( + std::declval<TagType>(), *std::declval<absl::InsecureBitGen*>(), + std::declval<A>(), std::declval<B>())); + +template <typename, typename, typename, typename ExplicitRet> +Invalid ExplicitTaggedUniformReturnT(...); + +// Given types <A, B, Expect>, CheckArgsReturnExpectedType() verifies that +// +// absl::Uniform<Expect>(gen, A{}, B{}) +// +// returns the type "Expect", and that the function-overload has the signature +// +// Expect(URBG&, Expect, Expect) +template <typename A, typename B, typename Expect> +void CheckArgsReturnExpectedType() { + static_assert( + absl::conjunction< + std::is_same<Expect, + decltype(ExplicitUniformReturnT<A, B, Expect>(0))>, + std::is_same<Expect, decltype(ExplicitUniformReturnT<B, A, Expect>( + 0))>>::value, + ""); + static_assert( + absl::conjunction< + std::is_same<Expect, + decltype(ExplicitTaggedUniformReturnT< + absl::IntervalOpenOpenTag, A, B, Expect>(0))>, + std::is_same<Expect, decltype(ExplicitTaggedUniformReturnT< + absl::IntervalOpenOpenTag, B, A, + Expect>(0))>>::value, + ""); +} + +TEST_F(RandomDistributionsTest, UniformTypeInference) { + // Infers common types. + CheckArgsInferType<uint16_t, uint16_t, uint16_t>(); + CheckArgsInferType<uint32_t, uint32_t, uint32_t>(); + CheckArgsInferType<uint64_t, uint64_t, uint64_t>(); + CheckArgsInferType<int16_t, int16_t, int16_t>(); + CheckArgsInferType<int32_t, int32_t, int32_t>(); + CheckArgsInferType<int64_t, int64_t, int64_t>(); + CheckArgsInferType<float, float, float>(); + CheckArgsInferType<double, double, double>(); + + // Explicitly-specified return-values override inferences. + CheckArgsReturnExpectedType<int16_t, int16_t, int32_t>(); + CheckArgsReturnExpectedType<uint16_t, uint16_t, int32_t>(); + CheckArgsReturnExpectedType<int16_t, int16_t, int64_t>(); + CheckArgsReturnExpectedType<int16_t, int32_t, int64_t>(); + CheckArgsReturnExpectedType<int16_t, int32_t, double>(); + CheckArgsReturnExpectedType<float, float, double>(); + CheckArgsReturnExpectedType<int, int, int16_t>(); + + // Properly promotes uint16_t. + CheckArgsInferType<uint16_t, uint32_t, uint32_t>(); + CheckArgsInferType<uint16_t, uint64_t, uint64_t>(); + CheckArgsInferType<uint16_t, int32_t, int32_t>(); + CheckArgsInferType<uint16_t, int64_t, int64_t>(); + CheckArgsInferType<uint16_t, float, float>(); + CheckArgsInferType<uint16_t, double, double>(); + + // Properly promotes int16_t. + CheckArgsInferType<int16_t, int32_t, int32_t>(); + CheckArgsInferType<int16_t, int64_t, int64_t>(); + CheckArgsInferType<int16_t, float, float>(); + CheckArgsInferType<int16_t, double, double>(); + + // Invalid (u)int16_t-pairings do not compile. + // See "CheckArgsInferType" comments above, for how this is achieved. + CheckArgsInferType<uint16_t, int16_t, Invalid>(); + CheckArgsInferType<int16_t, uint32_t, Invalid>(); + CheckArgsInferType<int16_t, uint64_t, Invalid>(); + + // Properly promotes uint32_t. + CheckArgsInferType<uint32_t, uint64_t, uint64_t>(); + CheckArgsInferType<uint32_t, int64_t, int64_t>(); + CheckArgsInferType<uint32_t, double, double>(); + + // Properly promotes int32_t. + CheckArgsInferType<int32_t, int64_t, int64_t>(); + CheckArgsInferType<int32_t, double, double>(); + + // Invalid (u)int32_t-pairings do not compile. + CheckArgsInferType<uint32_t, int32_t, Invalid>(); + CheckArgsInferType<int32_t, uint64_t, Invalid>(); + CheckArgsInferType<int32_t, float, Invalid>(); + CheckArgsInferType<uint32_t, float, Invalid>(); + + // Invalid (u)int64_t-pairings do not compile. + CheckArgsInferType<uint64_t, int64_t, Invalid>(); + CheckArgsInferType<int64_t, float, Invalid>(); + CheckArgsInferType<int64_t, double, Invalid>(); + + // Properly promotes float. + CheckArgsInferType<float, double, double>(); +} + +TEST_F(RandomDistributionsTest, UniformExamples) { + // Examples. + absl::InsecureBitGen gen; + EXPECT_NE(1, absl::Uniform(gen, static_cast<uint16_t>(0), 1.0f)); + EXPECT_NE(1, absl::Uniform(gen, 0, 1.0)); + EXPECT_NE(1, absl::Uniform(absl::IntervalOpenOpen, gen, + static_cast<uint16_t>(0), 1.0f)); + EXPECT_NE(1, absl::Uniform(absl::IntervalOpenOpen, gen, 0, 1.0)); + EXPECT_NE(1, absl::Uniform(absl::IntervalOpenOpen, gen, -1, 1.0)); + EXPECT_NE(1, absl::Uniform<double>(absl::IntervalOpenOpen, gen, -1, 1)); + EXPECT_NE(1, absl::Uniform<float>(absl::IntervalOpenOpen, gen, 0, 1)); + EXPECT_NE(1, absl::Uniform<float>(gen, 0, 1)); +} + +TEST_F(RandomDistributionsTest, UniformNoBounds) { + absl::InsecureBitGen gen; + + absl::Uniform<uint8_t>(gen); + absl::Uniform<uint16_t>(gen); + absl::Uniform<uint32_t>(gen); + absl::Uniform<uint64_t>(gen); +} + +TEST_F(RandomDistributionsTest, UniformNonsenseRanges) { + // The ranges used in this test are undefined behavior. + // The results are arbitrary and subject to future changes. + absl::InsecureBitGen gen; + + // <uint> + EXPECT_EQ(0, absl::Uniform<uint64_t>(gen, 0, 0)); + EXPECT_EQ(1, absl::Uniform<uint64_t>(gen, 1, 0)); + EXPECT_EQ(0, absl::Uniform<uint64_t>(absl::IntervalOpenOpen, gen, 0, 0)); + EXPECT_EQ(1, absl::Uniform<uint64_t>(absl::IntervalOpenOpen, gen, 1, 0)); + + constexpr auto m = (std::numeric_limits<uint64_t>::max)(); + + EXPECT_EQ(m, absl::Uniform(gen, m, m)); + EXPECT_EQ(m, absl::Uniform(gen, m, m - 1)); + EXPECT_EQ(m - 1, absl::Uniform(gen, m - 1, m)); + EXPECT_EQ(m, absl::Uniform(absl::IntervalOpenOpen, gen, m, m)); + EXPECT_EQ(m, absl::Uniform(absl::IntervalOpenOpen, gen, m, m - 1)); + EXPECT_EQ(m - 1, absl::Uniform(absl::IntervalOpenOpen, gen, m - 1, m)); + + // <int> + EXPECT_EQ(0, absl::Uniform<int64_t>(gen, 0, 0)); + EXPECT_EQ(1, absl::Uniform<int64_t>(gen, 1, 0)); + EXPECT_EQ(0, absl::Uniform<int64_t>(absl::IntervalOpenOpen, gen, 0, 0)); + EXPECT_EQ(1, absl::Uniform<int64_t>(absl::IntervalOpenOpen, gen, 1, 0)); + + constexpr auto l = (std::numeric_limits<int64_t>::min)(); + constexpr auto r = (std::numeric_limits<int64_t>::max)(); + + EXPECT_EQ(l, absl::Uniform(gen, l, l)); + EXPECT_EQ(r, absl::Uniform(gen, r, r)); + EXPECT_EQ(r, absl::Uniform(gen, r, r - 1)); + EXPECT_EQ(r - 1, absl::Uniform(gen, r - 1, r)); + EXPECT_EQ(l, absl::Uniform(absl::IntervalOpenOpen, gen, l, l)); + EXPECT_EQ(r, absl::Uniform(absl::IntervalOpenOpen, gen, r, r)); + EXPECT_EQ(r, absl::Uniform(absl::IntervalOpenOpen, gen, r, r - 1)); + EXPECT_EQ(r - 1, absl::Uniform(absl::IntervalOpenOpen, gen, r - 1, r)); + + // <double> + const double e = std::nextafter(1.0, 2.0); // 1 + epsilon + const double f = std::nextafter(1.0, 0.0); // 1 - epsilon + const double g = std::numeric_limits<double>::denorm_min(); + + EXPECT_EQ(1.0, absl::Uniform(gen, 1.0, e)); + EXPECT_EQ(1.0, absl::Uniform(gen, 1.0, f)); + EXPECT_EQ(0.0, absl::Uniform(gen, 0.0, g)); + + EXPECT_EQ(e, absl::Uniform(absl::IntervalOpenOpen, gen, 1.0, e)); + EXPECT_EQ(f, absl::Uniform(absl::IntervalOpenOpen, gen, 1.0, f)); + EXPECT_EQ(g, absl::Uniform(absl::IntervalOpenOpen, gen, 0.0, g)); +} + +// TODO(lar): Validate properties of non-default interval-semantics. +TEST_F(RandomDistributionsTest, UniformReal) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Uniform(gen, 0, 1.0); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(0.5, moments.mean, 0.02); + EXPECT_NEAR(1 / 12.0, moments.variance, 0.02); + EXPECT_NEAR(0.0, moments.skewness, 0.02); + EXPECT_NEAR(9 / 5.0, moments.kurtosis, 0.02); +} + +TEST_F(RandomDistributionsTest, UniformInt) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + const int64_t kMax = 1000000000000ll; + int64_t j = absl::Uniform(absl::IntervalClosedClosed, gen, 0, kMax); + // convert to double. + values[i] = static_cast<double>(j) / static_cast<double>(kMax); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(0.5, moments.mean, 0.02); + EXPECT_NEAR(1 / 12.0, moments.variance, 0.02); + EXPECT_NEAR(0.0, moments.skewness, 0.02); + EXPECT_NEAR(9 / 5.0, moments.kurtosis, 0.02); + + /* + // NOTE: These are not supported by absl::Uniform, which is specialized + // on integer and real valued types. + + enum E { E0, E1 }; // enum + enum S : int { S0, S1 }; // signed enum + enum U : unsigned int { U0, U1 }; // unsigned enum + + absl::Uniform(gen, E0, E1); + absl::Uniform(gen, S0, S1); + absl::Uniform(gen, U0, U1); + */ +} + +TEST_F(RandomDistributionsTest, Exponential) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Exponential<double>(gen); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(1.0, moments.mean, 0.02); + EXPECT_NEAR(1.0, moments.variance, 0.025); + EXPECT_NEAR(2.0, moments.skewness, 0.1); + EXPECT_LT(5.0, moments.kurtosis); +} + +TEST_F(RandomDistributionsTest, PoissonDefault) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Poisson<int64_t>(gen); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(1.0, moments.mean, 0.02); + EXPECT_NEAR(1.0, moments.variance, 0.02); + EXPECT_NEAR(1.0, moments.skewness, 0.025); + EXPECT_LT(2.0, moments.kurtosis); +} + +TEST_F(RandomDistributionsTest, PoissonLarge) { + constexpr double kMean = 100000000.0; + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Poisson<int64_t>(gen, kMean); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(kMean, moments.mean, kMean * 0.015); + EXPECT_NEAR(kMean, moments.variance, kMean * 0.015); + EXPECT_NEAR(std::sqrt(kMean), moments.skewness, kMean * 0.02); + EXPECT_LT(2.0, moments.kurtosis); +} + +TEST_F(RandomDistributionsTest, Bernoulli) { + constexpr double kP = 0.5151515151; + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Bernoulli(gen, kP); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(kP, moments.mean, 0.01); +} + +TEST_F(RandomDistributionsTest, Beta) { + constexpr double kAlpha = 2.0; + constexpr double kBeta = 3.0; + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Beta(gen, kAlpha, kBeta); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(0.4, moments.mean, 0.01); +} + +TEST_F(RandomDistributionsTest, Zipf) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Zipf<int64_t>(gen, 100); + } + + // The mean of a zipf distribution is: H(N, s-1) / H(N,s). + // Given the parameter v = 1, this gives the following function: + // (Hn(100, 1) - Hn(1,1)) / (Hn(100,2) - Hn(1,2)) = 6.5944 + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(6.5944, moments.mean, 2000) << moments; +} + +TEST_F(RandomDistributionsTest, Gaussian) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::Gaussian<double>(gen); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(0.0, moments.mean, 0.02); + EXPECT_NEAR(1.0, moments.variance, 0.04); + EXPECT_NEAR(0, moments.skewness, 0.2); + EXPECT_NEAR(3.0, moments.kurtosis, 0.5); +} + +TEST_F(RandomDistributionsTest, LogUniform) { + std::vector<double> values(kSize); + + absl::InsecureBitGen gen; + for (int i = 0; i < kSize; i++) { + values[i] = absl::LogUniform<int64_t>(gen, 0, (1 << 10) - 1); + } + + // The mean is the sum of the fractional means of the uniform distributions: + // [0..0][1..1][2..3][4..7][8..15][16..31][32..63] + // [64..127][128..255][256..511][512..1023] + const double mean = (0 + 1 + 1 + 2 + 3 + 4 + 7 + 8 + 15 + 16 + 31 + 32 + 63 + + 64 + 127 + 128 + 255 + 256 + 511 + 512 + 1023) / + (2.0 * 11.0); + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(mean, moments.mean, 2) << moments; +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/examples_test.cc b/third_party/abseil_cpp/absl/random/examples_test.cc new file mode 100644 index 000000000000..1dcb5146f3d0 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/examples_test.cc @@ -0,0 +1,99 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cinttypes> +#include <random> +#include <sstream> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/random/random.h" + +template <typename T> +void Use(T) {} + +TEST(Examples, Basic) { + absl::BitGen gen; + std::vector<int> objs = {10, 20, 30, 40, 50}; + + // Choose an element from a set. + auto elem = objs[absl::Uniform(gen, 0u, objs.size())]; + Use(elem); + + // Generate a uniform value between 1 and 6. + auto dice_roll = absl::Uniform<int>(absl::IntervalClosedClosed, gen, 1, 6); + Use(dice_roll); + + // Generate a random byte. + auto byte = absl::Uniform<uint8_t>(gen); + Use(byte); + + // Generate a fractional value from [0f, 1f). + auto fraction = absl::Uniform<float>(gen, 0, 1); + Use(fraction); + + // Toss a fair coin; 50/50 probability. + bool coin_toss = absl::Bernoulli(gen, 0.5); + Use(coin_toss); + + // Select a file size between 1k and 10MB, biased towards smaller file sizes. + auto file_size = absl::LogUniform<size_t>(gen, 1000, 10 * 1000 * 1000); + Use(file_size); + + // Randomize (shuffle) a collection. + std::shuffle(std::begin(objs), std::end(objs), gen); +} + +TEST(Examples, CreateingCorrelatedVariateSequences) { + // Unexpected PRNG correlation is often a source of bugs, + // so when using absl::BitGen it must be an intentional choice. + // NOTE: All of these only exhibit process-level stability. + + // Create a correlated sequence from system entropy. + { + auto my_seed = absl::MakeSeedSeq(); + + absl::BitGen gen_1(my_seed); + absl::BitGen gen_2(my_seed); // Produces same variates as gen_1. + + EXPECT_EQ(absl::Bernoulli(gen_1, 0.5), absl::Bernoulli(gen_2, 0.5)); + EXPECT_EQ(absl::Uniform<uint32_t>(gen_1), absl::Uniform<uint32_t>(gen_2)); + } + + // Create a correlated sequence from an existing URBG. + { + absl::BitGen gen; + + auto my_seed = absl::CreateSeedSeqFrom(&gen); + absl::BitGen gen_1(my_seed); + absl::BitGen gen_2(my_seed); + + EXPECT_EQ(absl::Bernoulli(gen_1, 0.5), absl::Bernoulli(gen_2, 0.5)); + EXPECT_EQ(absl::Uniform<uint32_t>(gen_1), absl::Uniform<uint32_t>(gen_2)); + } + + // An alternate construction which uses user-supplied data + // instead of a random seed. + { + const char kData[] = "A simple seed string"; + std::seed_seq my_seed(std::begin(kData), std::end(kData)); + + absl::BitGen gen_1(my_seed); + absl::BitGen gen_2(my_seed); + + EXPECT_EQ(absl::Bernoulli(gen_1, 0.5), absl::Bernoulli(gen_2, 0.5)); + EXPECT_EQ(absl::Uniform<uint32_t>(gen_1), absl::Uniform<uint32_t>(gen_2)); + } +} + diff --git a/third_party/abseil_cpp/absl/random/exponential_distribution.h b/third_party/abseil_cpp/absl/random/exponential_distribution.h new file mode 100644 index 000000000000..b5caf8a1e1f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/exponential_distribution.h @@ -0,0 +1,165 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_EXPONENTIAL_DISTRIBUTION_H_ +#define ABSL_RANDOM_EXPONENTIAL_DISTRIBUTION_H_ + +#include <cassert> +#include <cmath> +#include <istream> +#include <limits> +#include <type_traits> + +#include "absl/meta/type_traits.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/generate_real.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::exponential_distribution: +// Generates a number conforming to an exponential distribution and is +// equivalent to the standard [rand.dist.pois.exp] distribution. +template <typename RealType = double> +class exponential_distribution { + public: + using result_type = RealType; + + class param_type { + public: + using distribution_type = exponential_distribution; + + explicit param_type(result_type lambda = 1) : lambda_(lambda) { + assert(lambda > 0); + neg_inv_lambda_ = -result_type(1) / lambda_; + } + + result_type lambda() const { return lambda_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.lambda_ == b.lambda_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class exponential_distribution; + + result_type lambda_; + result_type neg_inv_lambda_; + + static_assert( + std::is_floating_point<RealType>::value, + "Class-template absl::exponential_distribution<> must be parameterized " + "using a floating-point type."); + }; + + exponential_distribution() : exponential_distribution(1) {} + + explicit exponential_distribution(result_type lambda) : param_(lambda) {} + + explicit exponential_distribution(const param_type& p) : param_(p) {} + + void reset() {} + + // Generating functions + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + param_type param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + result_type(min)() const { return 0; } + result_type(max)() const { + return std::numeric_limits<result_type>::infinity(); + } + + result_type lambda() const { return param_.lambda(); } + + friend bool operator==(const exponential_distribution& a, + const exponential_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const exponential_distribution& a, + const exponential_distribution& b) { + return a.param_ != b.param_; + } + + private: + param_type param_; + random_internal::FastUniformBits<uint64_t> fast_u64_; +}; + +// -------------------------------------------------------------------------- +// Implementation details follow +// -------------------------------------------------------------------------- + +template <typename RealType> +template <typename URBG> +typename exponential_distribution<RealType>::result_type +exponential_distribution<RealType>::operator()( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + using random_internal::GenerateNegativeTag; + using random_internal::GenerateRealFromBits; + using real_type = + absl::conditional_t<std::is_same<RealType, float>::value, float, double>; + + const result_type u = GenerateRealFromBits<real_type, GenerateNegativeTag, + false>(fast_u64_(g)); // U(-1, 0) + + // log1p(-x) is mathematically equivalent to log(1 - x) but has more + // accuracy for x near zero. + return p.neg_inv_lambda_ * std::log1p(u); +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const exponential_distribution<RealType>& x) { + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<RealType>::kPrecision); + os << x.lambda(); + return os; +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + exponential_distribution<RealType>& x) { // NOLINT(runtime/references) + using result_type = typename exponential_distribution<RealType>::result_type; + using param_type = typename exponential_distribution<RealType>::param_type; + result_type lambda; + + auto saver = random_internal::make_istream_state_saver(is); + lambda = random_internal::read_floating_point<result_type>(is); + if (!is.fail()) { + x.param(param_type(lambda)); + } + return is; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_EXPONENTIAL_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/exponential_distribution_test.cc b/third_party/abseil_cpp/absl/random/exponential_distribution_test.cc new file mode 100644 index 000000000000..8e9e69b64b29 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/exponential_distribution_test.cc @@ -0,0 +1,430 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/exponential_distribution.h" + +#include <algorithm> +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <limits> +#include <random> +#include <sstream> +#include <string> +#include <type_traits> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/strip.h" + +namespace { + +using absl::random_internal::kChiSquared; + +template <typename RealType> +class ExponentialDistributionTypedTest : public ::testing::Test {}; + +#if defined(__EMSCRIPTEN__) +using RealTypes = ::testing::Types<float, double>; +#else +using RealTypes = ::testing::Types<float, double, long double>; +#endif // defined(__EMSCRIPTEN__) +TYPED_TEST_CASE(ExponentialDistributionTypedTest, RealTypes); + +TYPED_TEST(ExponentialDistributionTypedTest, SerializeTest) { + using param_type = + typename absl::exponential_distribution<TypeParam>::param_type; + + const TypeParam kParams[] = { + // Cases around 1. + 1, // + std::nextafter(TypeParam(1), TypeParam(0)), // 1 - epsilon + std::nextafter(TypeParam(1), TypeParam(2)), // 1 + epsilon + // Typical cases. + TypeParam(1e-8), TypeParam(1e-4), TypeParam(1), TypeParam(2), + TypeParam(1e4), TypeParam(1e8), TypeParam(1e20), TypeParam(2.5), + // Boundary cases. + std::numeric_limits<TypeParam>::max(), + std::numeric_limits<TypeParam>::epsilon(), + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(1)), // min + epsilon + std::numeric_limits<TypeParam>::min(), // smallest normal + // There are some errors dealing with denorms on apple platforms. + std::numeric_limits<TypeParam>::denorm_min(), // smallest denorm + std::numeric_limits<TypeParam>::min() / 2, // denorm + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(0)), // denorm_max + }; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + + for (const TypeParam lambda : kParams) { + // Some values may be invalid; skip those. + if (!std::isfinite(lambda)) continue; + ABSL_ASSERT(lambda > 0); + + const param_type param(lambda); + + absl::exponential_distribution<TypeParam> before(lambda); + EXPECT_EQ(before.lambda(), param.lambda()); + + { + absl::exponential_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + EXPECT_EQ(via_param.param(), before.param()); + } + + // Smoke test. + auto sample_min = before.max(); + auto sample_max = before.min(); + for (int i = 0; i < kCount; i++) { + auto sample = before(gen); + EXPECT_GE(sample, before.min()) << before; + EXPECT_LE(sample, before.max()) << before; + if (sample > sample_max) sample_max = sample; + if (sample < sample_min) sample_min = sample; + } + if (!std::is_same<TypeParam, long double>::value) { + ABSL_INTERNAL_LOG(INFO, + absl::StrFormat("Range {%f}: %f, %f, lambda=%f", lambda, + sample_min, sample_max, lambda)); + } + + std::stringstream ss; + ss << before; + + if (!std::isfinite(lambda)) { + // Streams do not deserialize inf/nan correctly. + continue; + } + // Validate stream serialization. + absl::exponential_distribution<TypeParam> after(34.56f); + + EXPECT_NE(before.lambda(), after.lambda()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + +#if defined(__powerpc64__) || defined(__PPC64__) || defined(__powerpc__) || \ + defined(__ppc__) || defined(__PPC__) + if (std::is_same<TypeParam, long double>::value) { + // Roundtripping floating point values requires sufficient precision to + // reconstruct the exact value. It turns out that long double has some + // errors doing this on ppc, particularly for values + // near {1.0 +/- epsilon}. + if (lambda <= std::numeric_limits<double>::max() && + lambda >= std::numeric_limits<double>::lowest()) { + EXPECT_EQ(static_cast<double>(before.lambda()), + static_cast<double>(after.lambda())) + << ss.str(); + } + continue; + } +#endif + + EXPECT_EQ(before.lambda(), after.lambda()) // + << ss.str() << " " // + << (ss.good() ? "good " : "") // + << (ss.bad() ? "bad " : "") // + << (ss.eof() ? "eof " : "") // + << (ss.fail() ? "fail " : ""); + } +} + +// http://www.itl.nist.gov/div898/handbook/eda/section3/eda3667.htm + +class ExponentialModel { + public: + explicit ExponentialModel(double lambda) + : lambda_(lambda), beta_(1.0 / lambda) {} + + double lambda() const { return lambda_; } + + double mean() const { return beta_; } + double variance() const { return beta_ * beta_; } + double stddev() const { return std::sqrt(variance()); } + double skew() const { return 2; } + double kurtosis() const { return 6.0; } + + double CDF(double x) { return 1.0 - std::exp(-lambda_ * x); } + + // The inverse CDF, or PercentPoint function of the distribution + double InverseCDF(double p) { + ABSL_ASSERT(p >= 0.0); + ABSL_ASSERT(p < 1.0); + return -beta_ * std::log(1.0 - p); + } + + private: + const double lambda_; + const double beta_; +}; + +struct Param { + double lambda; + double p_fail; + int trials; +}; + +class ExponentialDistributionTests : public testing::TestWithParam<Param>, + public ExponentialModel { + public: + ExponentialDistributionTests() : ExponentialModel(GetParam().lambda) {} + + // SingleZTest provides a basic z-squared test of the mean vs. expected + // mean for data generated by the poisson distribution. + template <typename D> + bool SingleZTest(const double p, const size_t samples); + + // SingleChiSquaredTest provides a basic chi-squared test of the normal + // distribution. + template <typename D> + double SingleChiSquaredTest(); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng_{0x2B7E151628AED2A6}; +}; + +template <typename D> +bool ExponentialDistributionTests::SingleZTest(const double p, + const size_t samples) { + D dis(lambda()); + + std::vector<double> data; + data.reserve(samples); + for (size_t i = 0; i < samples; i++) { + const double x = dis(rng_); + data.push_back(x); + } + + const auto m = absl::random_internal::ComputeDistributionMoments(data); + const double max_err = absl::random_internal::MaxErrorTolerance(p); + const double z = absl::random_internal::ZScore(mean(), m); + const bool pass = absl::random_internal::Near("z", z, 0.0, max_err); + + if (!pass) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("p=%f max_err=%f\n" + " lambda=%f\n" + " mean=%f vs. %f\n" + " stddev=%f vs. %f\n" + " skewness=%f vs. %f\n" + " kurtosis=%f vs. %f\n" + " z=%f vs. 0", + p, max_err, lambda(), m.mean, mean(), + std::sqrt(m.variance), stddev(), m.skewness, + skew(), m.kurtosis, kurtosis(), z)); + } + return pass; +} + +template <typename D> +double ExponentialDistributionTests::SingleChiSquaredTest() { + const size_t kSamples = 10000; + const int kBuckets = 50; + + // The InverseCDF is the percent point function of the distribution, and can + // be used to assign buckets roughly uniformly. + std::vector<double> cutoffs; + const double kInc = 1.0 / static_cast<double>(kBuckets); + for (double p = kInc; p < 1.0; p += kInc) { + cutoffs.push_back(InverseCDF(p)); + } + if (cutoffs.back() != std::numeric_limits<double>::infinity()) { + cutoffs.push_back(std::numeric_limits<double>::infinity()); + } + + D dis(lambda()); + + std::vector<int32_t> counts(cutoffs.size(), 0); + for (int j = 0; j < kSamples; j++) { + const double x = dis(rng_); + auto it = std::upper_bound(cutoffs.begin(), cutoffs.end(), x); + counts[std::distance(cutoffs.begin(), it)]++; + } + + // Null-hypothesis is that the distribution is exponentially distributed + // with the provided lambda (not estimated from the data). + const int dof = static_cast<int>(counts.size()) - 1; + + // Our threshold for logging is 1-in-50. + const double threshold = absl::random_internal::ChiSquareValue(dof, 0.98); + + const double expected = + static_cast<double>(kSamples) / static_cast<double>(counts.size()); + + double chi_square = absl::random_internal::ChiSquareWithExpected( + std::begin(counts), std::end(counts), expected); + double p = absl::random_internal::ChiSquarePValue(chi_square, dof); + + if (chi_square > threshold) { + for (int i = 0; i < cutoffs.size(); i++) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("%d : (%f) = %d", i, cutoffs[i], counts[i])); + } + + ABSL_INTERNAL_LOG(INFO, + absl::StrCat("lambda ", lambda(), "\n", // + " expected ", expected, "\n", // + kChiSquared, " ", chi_square, " (", p, ")\n", + kChiSquared, " @ 0.98 = ", threshold)); + } + return p; +} + +TEST_P(ExponentialDistributionTests, ZTest) { + const size_t kSamples = 10000; + const auto& param = GetParam(); + const int expected_failures = + std::max(1, static_cast<int>(std::ceil(param.trials * param.p_fail))); + const double p = absl::random_internal::RequiredSuccessProbability( + param.p_fail, param.trials); + + int failures = 0; + for (int i = 0; i < param.trials; i++) { + failures += SingleZTest<absl::exponential_distribution<double>>(p, kSamples) + ? 0 + : 1; + } + EXPECT_LE(failures, expected_failures); +} + +TEST_P(ExponentialDistributionTests, ChiSquaredTest) { + const int kTrials = 20; + int failures = 0; + + for (int i = 0; i < kTrials; i++) { + double p_value = + SingleChiSquaredTest<absl::exponential_distribution<double>>(); + if (p_value < 0.005) { // 1/200 + failures++; + } + } + + // There is a 0.10% chance of producing at least one failure, so raise the + // failure threshold high enough to allow for a flake rate < 10,000. + EXPECT_LE(failures, 4); +} + +std::vector<Param> GenParams() { + return { + Param{1.0, 0.02, 100}, + Param{2.5, 0.02, 100}, + Param{10, 0.02, 100}, + // large + Param{1e4, 0.02, 100}, + Param{1e9, 0.02, 100}, + // small + Param{0.1, 0.02, 100}, + Param{1e-3, 0.02, 100}, + Param{1e-5, 0.02, 100}, + }; +} + +std::string ParamName(const ::testing::TestParamInfo<Param>& info) { + const auto& p = info.param; + std::string name = absl::StrCat("lambda_", absl::SixDigits(p.lambda)); + return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); +} + +INSTANTIATE_TEST_CASE_P(All, ExponentialDistributionTests, + ::testing::ValuesIn(GenParams()), ParamName); + +// NOTE: absl::exponential_distribution is not guaranteed to be stable. +TEST(ExponentialDistributionTest, StabilityTest) { + // absl::exponential_distribution stability relies on std::log1p and + // absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(14); + + { + absl::exponential_distribution<double> dist; + std::generate(std::begin(output), std::end(output), + [&] { return static_cast<int>(10000.0 * dist(urbg)); }); + + EXPECT_EQ(14, urbg.invocations()); + EXPECT_THAT(output, + testing::ElementsAre(0, 71913, 14375, 5039, 1835, 861, 25936, + 804, 126, 12337, 17984, 27002, 0, 71913)); + } + + urbg.reset(); + { + absl::exponential_distribution<float> dist; + std::generate(std::begin(output), std::end(output), + [&] { return static_cast<int>(10000.0f * dist(urbg)); }); + + EXPECT_EQ(14, urbg.invocations()); + EXPECT_THAT(output, + testing::ElementsAre(0, 71913, 14375, 5039, 1835, 861, 25936, + 804, 126, 12337, 17984, 27002, 0, 71913)); + } +} + +TEST(ExponentialDistributionTest, AlgorithmBounds) { + // Relies on absl::uniform_real_distribution, so some of these comments + // reference that. + absl::exponential_distribution<double> dist; + + { + // This returns the smallest value >0 from absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg({0x0000000000000001ull}); + double a = dist(urbg); + EXPECT_EQ(a, 5.42101086242752217004e-20); + } + + { + // This returns a value very near 0.5 from absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg({0x7fffffffffffffefull}); + double a = dist(urbg); + EXPECT_EQ(a, 0.693147180559945175204); + } + + { + // This returns the largest value <1 from absl::uniform_real_distribution. + // WolframAlpha: ~39.1439465808987766283058547296341915292187253 + absl::random_internal::sequence_urbg urbg({0xFFFFFFFFFFFFFFeFull}); + double a = dist(urbg); + EXPECT_EQ(a, 36.7368005696771007251); + } + { + // This *ALSO* returns the largest value <1. + absl::random_internal::sequence_urbg urbg({0xFFFFFFFFFFFFFFFFull}); + double a = dist(urbg); + EXPECT_EQ(a, 36.7368005696771007251); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/gaussian_distribution.cc b/third_party/abseil_cpp/absl/random/gaussian_distribution.cc new file mode 100644 index 000000000000..c7a72cb2f61c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/gaussian_distribution.cc @@ -0,0 +1,104 @@ +// BEGIN GENERATED CODE; DO NOT EDIT +// clang-format off + +#include "absl/random/gaussian_distribution.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +const gaussian_distribution_base::Tables + gaussian_distribution_base::zg_ = { + {3.7130862467425505, 3.442619855899000214, 3.223084984581141565, + 3.083228858216868318, 2.978696252647779819, 2.894344007021528942, + 2.82312535054891045, 2.761169372387176857, 2.706113573121819549, + 2.656406411261359679, 2.610972248431847387, 2.56903362592493778, + 2.530009672388827457, 2.493454522095372106, 2.459018177411830486, + 2.426420645533749809, 2.395434278011062457, 2.365871370117638595, + 2.337575241339236776, 2.310413683698762988, 2.284274059677471769, + 2.25905957386919809, 2.234686395590979036, 2.21108140887870297, + 2.188180432076048731, 2.165926793748921497, 2.144270182360394905, + 2.123165708673976138, 2.102573135189237608, 2.082456237992015957, + 2.062782274508307978, 2.043521536655067194, 2.02464697337738464, + 2.006133869963471206, 1.987959574127619033, 1.970103260854325633, + 1.952545729553555764, 1.935269228296621957, 1.918257300864508963, + 1.901494653105150423, 1.884967035707758143, 1.868661140994487768, + 1.852564511728090002, 1.836665460258444904, 1.820952996596124418, + 1.805416764219227366, 1.790046982599857506, 1.77483439558606837, + 1.759770224899592339, 1.744846128113799244, 1.730054160563729182, + 1.71538674071366648, 1.700836618569915748, 1.686396846779167014, + 1.6720607540975998, 1.657821920954023254, 1.643674156862867441, + 1.629611479470633562, 1.615628095043159629, 1.601718380221376581, + 1.587876864890574558, 1.574098216022999264, 1.560377222366167382, + 1.546708779859908844, 1.533087877674041755, 1.519509584765938559, + 1.505969036863201937, 1.492461423781352714, 1.478981976989922842, + 1.465525957342709296, 1.452088642889222792, 1.438665316684561546, + 1.425251254514058319, 1.411841712447055919, 1.398431914131003539, + 1.385017037732650058, 1.371592202427340812, 1.358152454330141534, + 1.34469275175354519, 1.331207949665625279, 1.317692783209412299, + 1.304141850128615054, 1.290549591926194894, 1.27691027356015363, + 1.263217961454619287, 1.249466499573066436, 1.23564948326336066, + 1.221760230539994385, 1.207791750415947662, 1.193736707833126465, + 1.17958738466398616, 1.165335636164750222, 1.150972842148865416, + 1.136489852013158774, 1.121876922582540237, 1.107123647534034028, + 1.092218876907275371, 1.077150624892893482, 1.061905963694822042, + 1.046470900764042922, 1.030830236068192907, 1.014967395251327842, + 0.9988642334929808131, 0.9825008035154263464, 0.9658550794011470098, + 0.9489026255113034436, 0.9316161966151479401, 0.9139652510230292792, + 0.8959153525809346874, 0.8774274291129204872, 0.8584568431938099931, + 0.8389522142975741614, 0.8188539067003538507, 0.7980920606440534693, + 0.7765839878947563557, 0.7542306644540520688, 0.7309119106424850631, + 0.7064796113354325779, 0.6807479186691505202, 0.6534786387399710295, + 0.6243585973360461505, 0.5929629424714434327, 0.5586921784081798625, + 0.5206560387620546848, 0.4774378372966830431, 0.4265479863554152429, + 0.3628714310970211909, 0.2723208648139477384, 0}, + {0.001014352564120377413, 0.002669629083880922793, 0.005548995220771345792, + 0.008624484412859888607, 0.01183947865788486861, 0.01516729801054656976, + 0.01859210273701129151, 0.02210330461592709475, 0.02569329193593428151, + 0.02935631744000685023, 0.03308788614622575758, 0.03688438878665621645, + 0.04074286807444417458, 0.04466086220049143157, 0.04863629585986780496, + 0.05266740190305100461, 0.05675266348104984759, 0.06089077034804041277, + 0.06508058521306804567, 0.06932111739357792179, 0.07361150188411341722, + 0.07795098251397346301, 0.08233889824223575293, 0.08677467189478028919, + 0.09125780082683036809, 0.095787849121731522, 0.1003644410286559929, + 0.1049872554094214289, 0.1096560210148404546, 0.1143705124488661323, + 0.1191305467076509556, 0.1239359802028679736, 0.1287867061959434012, + 0.1336826525834396151, 0.1386237799845948804, 0.1436100800906280339, + 0.1486415742423425057, 0.1537183122081819397, 0.1588403711394795748, + 0.1640078546834206341, 0.1692208922373653057, 0.1744796383307898324, + 0.1797842721232958407, 0.1851349970089926078, 0.1905320403191375633, + 0.1959756531162781534, 0.2014661100743140865, 0.2070037094399269362, + 0.2125887730717307134, 0.2182216465543058426, 0.2239026993850088965, + 0.229632325232116602, 0.2354109422634795556, 0.2412389935454402889, + 0.2471169475123218551, 0.2530452985073261551, 0.2590245673962052742, + 0.2650553022555897087, 0.271138079138385224, 0.2772735029191887857, + 0.2834622082232336471, 0.2897048604429605656, 0.2960021568469337061, + 0.3023548277864842593, 0.3087636380061818397, 0.3152293880650116065, + 0.3217529158759855901, 0.3283350983728509642, 0.3349768533135899506, + 0.3416791412315512977, 0.3484429675463274756, 0.355269384847918035, + 0.3621594953693184626, 0.3691144536644731522, 0.376135469510563536, + 0.3832238110559021416, 0.3903808082373155797, 0.3976078564938743676, + 0.404906420807223999, 0.4122780401026620578, 0.4197243320495753771, + 0.4272469983049970721, 0.4348478302499918513, 0.4425287152754694975, + 0.4502916436820402768, 0.458138716267873114, 0.4660721526894572309, + 0.4740943006930180559, 0.4822076463294863724, 0.4904148252838453348, + 0.4987186354709807201, 0.5071220510755701794, 0.5156282382440030565, + 0.5242405726729852944, 0.5329626593838373561, 0.5417983550254266145, + 0.5507517931146057588, 0.5598274127040882009, 0.5690299910679523787, + 0.5783646811197646898, 0.5878370544347081283, 0.5974531509445183408, + 0.6072195366251219584, 0.6171433708188825973, 0.6272324852499290282, + 0.6374954773350440806, 0.6479418211102242475, 0.6585820000500898219, + 0.6694276673488921414, 0.6804918409973358395, 0.6917891434366769676, + 0.7033360990161600101, 0.7151515074105005976, 0.7272569183441868201, + 0.7396772436726493094, 0.7524415591746134169, 0.7655841738977066102, + 0.7791460859296898134, 0.7931770117713072832, 0.8077382946829627652, + 0.8229072113814113187, 0.8387836052959920519, 0.8555006078694531446, + 0.873243048910072206, 0.8922816507840289901, 0.9130436479717434217, + 0.9362826816850632339, 0.9635996931270905952, 1}}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +// clang-format on +// END GENERATED CODE diff --git a/third_party/abseil_cpp/absl/random/gaussian_distribution.h b/third_party/abseil_cpp/absl/random/gaussian_distribution.h new file mode 100644 index 000000000000..4b07a5c0af9d --- /dev/null +++ b/third_party/abseil_cpp/absl/random/gaussian_distribution.h @@ -0,0 +1,275 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_GAUSSIAN_DISTRIBUTION_H_ +#define ABSL_RANDOM_GAUSSIAN_DISTRIBUTION_H_ + +// absl::gaussian_distribution implements the Ziggurat algorithm +// for generating random gaussian numbers. +// +// Implementation based on "The Ziggurat Method for Generating Random Variables" +// by George Marsaglia and Wai Wan Tsang: http://www.jstatsoft.org/v05/i08/ +// + +#include <cmath> +#include <cstdint> +#include <istream> +#include <limits> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/generate_real.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// absl::gaussian_distribution_base implements the underlying ziggurat algorithm +// using the ziggurat tables generated by the gaussian_distribution_gentables +// binary. +// +// The specific algorithm has some of the improvements suggested by the +// 2005 paper, "An Improved Ziggurat Method to Generate Normal Random Samples", +// Jurgen A Doornik. (https://www.doornik.com/research/ziggurat.pdf) +class ABSL_DLL gaussian_distribution_base { + public: + template <typename URBG> + inline double zignor(URBG& g); // NOLINT(runtime/references) + + private: + friend class TableGenerator; + + template <typename URBG> + inline double zignor_fallback(URBG& g, // NOLINT(runtime/references) + bool neg); + + // Constants used for the gaussian distribution. + static constexpr double kR = 3.442619855899; // Start of the tail. + static constexpr double kRInv = 0.29047645161474317; // ~= (1.0 / kR) . + static constexpr double kV = 9.91256303526217e-3; + static constexpr uint64_t kMask = 0x07f; + + // The ziggurat tables store the pdf(f) and inverse-pdf(x) for equal-area + // points on one-half of the normal distribution, where the pdf function, + // pdf = e ^ (-1/2 *x^2), assumes that the mean = 0 & stddev = 1. + // + // These tables are just over 2kb in size; larger tables might improve the + // distributions, but also lead to more cache pollution. + // + // x = {3.71308, 3.44261, 3.22308, ..., 0} + // f = {0.00101, 0.00266, 0.00554, ..., 1} + struct Tables { + double x[kMask + 2]; + double f[kMask + 2]; + }; + static const Tables zg_; + random_internal::FastUniformBits<uint64_t> fast_u64_; +}; + +} // namespace random_internal + +// absl::gaussian_distribution: +// Generates a number conforming to a Gaussian distribution. +template <typename RealType = double> +class gaussian_distribution : random_internal::gaussian_distribution_base { + public: + using result_type = RealType; + + class param_type { + public: + using distribution_type = gaussian_distribution; + + explicit param_type(result_type mean = 0, result_type stddev = 1) + : mean_(mean), stddev_(stddev) {} + + // Returns the mean distribution parameter. The mean specifies the location + // of the peak. The default value is 0.0. + result_type mean() const { return mean_; } + + // Returns the deviation distribution parameter. The default value is 1.0. + result_type stddev() const { return stddev_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.mean_ == b.mean_ && a.stddev_ == b.stddev_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + result_type mean_; + result_type stddev_; + + static_assert( + std::is_floating_point<RealType>::value, + "Class-template absl::gaussian_distribution<> must be parameterized " + "using a floating-point type."); + }; + + gaussian_distribution() : gaussian_distribution(0) {} + + explicit gaussian_distribution(result_type mean, result_type stddev = 1) + : param_(mean, stddev) {} + + explicit gaussian_distribution(const param_type& p) : param_(p) {} + + void reset() {} + + // Generating functions + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + param_type param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + result_type(min)() const { + return -std::numeric_limits<result_type>::infinity(); + } + result_type(max)() const { + return std::numeric_limits<result_type>::infinity(); + } + + result_type mean() const { return param_.mean(); } + result_type stddev() const { return param_.stddev(); } + + friend bool operator==(const gaussian_distribution& a, + const gaussian_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const gaussian_distribution& a, + const gaussian_distribution& b) { + return a.param_ != b.param_; + } + + private: + param_type param_; +}; + +// -------------------------------------------------------------------------- +// Implementation details only below +// -------------------------------------------------------------------------- + +template <typename RealType> +template <typename URBG> +typename gaussian_distribution<RealType>::result_type +gaussian_distribution<RealType>::operator()( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + return p.mean() + p.stddev() * static_cast<result_type>(zignor(g)); +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const gaussian_distribution<RealType>& x) { + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<RealType>::kPrecision); + os << x.mean() << os.fill() << x.stddev(); + return os; +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + gaussian_distribution<RealType>& x) { // NOLINT(runtime/references) + using result_type = typename gaussian_distribution<RealType>::result_type; + using param_type = typename gaussian_distribution<RealType>::param_type; + + auto saver = random_internal::make_istream_state_saver(is); + auto mean = random_internal::read_floating_point<result_type>(is); + if (is.fail()) return is; + auto stddev = random_internal::read_floating_point<result_type>(is); + if (!is.fail()) { + x.param(param_type(mean, stddev)); + } + return is; +} + +namespace random_internal { + +template <typename URBG> +inline double gaussian_distribution_base::zignor_fallback(URBG& g, bool neg) { + using random_internal::GeneratePositiveTag; + using random_internal::GenerateRealFromBits; + + // This fallback path happens approximately 0.05% of the time. + double x, y; + do { + // kRInv = 1/r, U(0, 1) + x = kRInv * + std::log(GenerateRealFromBits<double, GeneratePositiveTag, false>( + fast_u64_(g))); + y = -std::log( + GenerateRealFromBits<double, GeneratePositiveTag, false>(fast_u64_(g))); + } while ((y + y) < (x * x)); + return neg ? (x - kR) : (kR - x); +} + +template <typename URBG> +inline double gaussian_distribution_base::zignor( + URBG& g) { // NOLINT(runtime/references) + using random_internal::GeneratePositiveTag; + using random_internal::GenerateRealFromBits; + using random_internal::GenerateSignedTag; + + while (true) { + // We use a single uint64_t to generate both a double and a strip. + // These bits are unused when the generated double is > 1/2^5. + // This may introduce some bias from the duplicated low bits of small + // values (those smaller than 1/2^5, which all end up on the left tail). + uint64_t bits = fast_u64_(g); + int i = static_cast<int>(bits & kMask); // pick a random strip + double j = GenerateRealFromBits<double, GenerateSignedTag, false>( + bits); // U(-1, 1) + const double x = j * zg_.x[i]; + + // Retangular box. Handles >97% of all cases. + // For any given box, this handles between 75% and 99% of values. + // Equivalent to U(01) < (x[i+1] / x[i]), and when i == 0, ~93.5% + if (std::abs(x) < zg_.x[i + 1]) { + return x; + } + + // i == 0: Base box. Sample using a ratio of uniforms. + if (i == 0) { + // This path happens about 0.05% of the time. + return zignor_fallback(g, j < 0); + } + + // i > 0: Wedge samples using precomputed values. + double v = GenerateRealFromBits<double, GeneratePositiveTag, false>( + fast_u64_(g)); // U(0, 1) + if ((zg_.f[i + 1] + v * (zg_.f[i] - zg_.f[i + 1])) < + std::exp(-0.5 * x * x)) { + return x; + } + + // The wedge was missed; reject the value and try again. + } +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_GAUSSIAN_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/gaussian_distribution_test.cc b/third_party/abseil_cpp/absl/random/gaussian_distribution_test.cc new file mode 100644 index 000000000000..02ac578a5c18 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/gaussian_distribution_test.cc @@ -0,0 +1,579 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/gaussian_distribution.h" + +#include <algorithm> +#include <cmath> +#include <cstddef> +#include <ios> +#include <iterator> +#include <random> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/strip.h" + +namespace { + +using absl::random_internal::kChiSquared; + +template <typename RealType> +class GaussianDistributionInterfaceTest : public ::testing::Test {}; + +using RealTypes = ::testing::Types<float, double, long double>; +TYPED_TEST_CASE(GaussianDistributionInterfaceTest, RealTypes); + +TYPED_TEST(GaussianDistributionInterfaceTest, SerializeTest) { + using param_type = + typename absl::gaussian_distribution<TypeParam>::param_type; + + const TypeParam kParams[] = { + // Cases around 1. + 1, // + std::nextafter(TypeParam(1), TypeParam(0)), // 1 - epsilon + std::nextafter(TypeParam(1), TypeParam(2)), // 1 + epsilon + // Arbitrary values. + TypeParam(1e-8), TypeParam(1e-4), TypeParam(2), TypeParam(1e4), + TypeParam(1e8), TypeParam(1e20), TypeParam(2.5), + // Boundary cases. + std::numeric_limits<TypeParam>::infinity(), + std::numeric_limits<TypeParam>::max(), + std::numeric_limits<TypeParam>::epsilon(), + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(1)), // min + epsilon + std::numeric_limits<TypeParam>::min(), // smallest normal + // There are some errors dealing with denorms on apple platforms. + std::numeric_limits<TypeParam>::denorm_min(), // smallest denorm + std::numeric_limits<TypeParam>::min() / 2, + std::nextafter(std::numeric_limits<TypeParam>::min(), + TypeParam(0)), // denorm_max + }; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + + // Use a loop to generate the combinations of {+/-x, +/-y}, and assign x, y to + // all values in kParams, + for (const auto mod : {0, 1, 2, 3}) { + for (const auto x : kParams) { + if (!std::isfinite(x)) continue; + for (const auto y : kParams) { + const TypeParam mean = (mod & 0x1) ? -x : x; + const TypeParam stddev = (mod & 0x2) ? -y : y; + const param_type param(mean, stddev); + + absl::gaussian_distribution<TypeParam> before(mean, stddev); + EXPECT_EQ(before.mean(), param.mean()); + EXPECT_EQ(before.stddev(), param.stddev()); + + { + absl::gaussian_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + EXPECT_EQ(via_param.param(), before.param()); + } + + // Smoke test. + auto sample_min = before.max(); + auto sample_max = before.min(); + for (int i = 0; i < kCount; i++) { + auto sample = before(gen); + if (sample > sample_max) sample_max = sample; + if (sample < sample_min) sample_min = sample; + EXPECT_GE(sample, before.min()) << before; + EXPECT_LE(sample, before.max()) << before; + } + if (!std::is_same<TypeParam, long double>::value) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("Range{%f, %f}: %f, %f", mean, stddev, + sample_min, sample_max)); + } + + std::stringstream ss; + ss << before; + + if (!std::isfinite(mean) || !std::isfinite(stddev)) { + // Streams do not parse inf/nan. + continue; + } + + // Validate stream serialization. + absl::gaussian_distribution<TypeParam> after(-0.53f, 2.3456f); + + EXPECT_NE(before.mean(), after.mean()); + EXPECT_NE(before.stddev(), after.stddev()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + +#if defined(__powerpc64__) || defined(__PPC64__) || defined(__powerpc__) || \ + defined(__ppc__) || defined(__PPC__) || defined(__EMSCRIPTEN__) + if (std::is_same<TypeParam, long double>::value) { + // Roundtripping floating point values requires sufficient precision + // to reconstruct the exact value. It turns out that long double + // has some errors doing this on ppc, particularly for values + // near {1.0 +/- epsilon}. + // + // Emscripten is even worse, implementing long double as a 128-bit + // type, but shipping with a strtold() that doesn't support that. + if (mean <= std::numeric_limits<double>::max() && + mean >= std::numeric_limits<double>::lowest()) { + EXPECT_EQ(static_cast<double>(before.mean()), + static_cast<double>(after.mean())) + << ss.str(); + } + if (stddev <= std::numeric_limits<double>::max() && + stddev >= std::numeric_limits<double>::lowest()) { + EXPECT_EQ(static_cast<double>(before.stddev()), + static_cast<double>(after.stddev())) + << ss.str(); + } + continue; + } +#endif + + EXPECT_EQ(before.mean(), after.mean()); + EXPECT_EQ(before.stddev(), after.stddev()) // + << ss.str() << " " // + << (ss.good() ? "good " : "") // + << (ss.bad() ? "bad " : "") // + << (ss.eof() ? "eof " : "") // + << (ss.fail() ? "fail " : ""); + } + } + } +} + +// http://www.itl.nist.gov/div898/handbook/eda/section3/eda3661.htm + +class GaussianModel { + public: + GaussianModel(double mean, double stddev) : mean_(mean), stddev_(stddev) {} + + double mean() const { return mean_; } + double variance() const { return stddev() * stddev(); } + double stddev() const { return stddev_; } + double skew() const { return 0; } + double kurtosis() const { return 3.0; } + + // The inverse CDF, or PercentPoint function. + double InverseCDF(double p) { + ABSL_ASSERT(p >= 0.0); + ABSL_ASSERT(p < 1.0); + return mean() + stddev() * -absl::random_internal::InverseNormalSurvival(p); + } + + private: + const double mean_; + const double stddev_; +}; + +struct Param { + double mean; + double stddev; + double p_fail; // Z-Test probability of failure. + int trials; // Z-Test trials. +}; + +// GaussianDistributionTests implements a z-test for the gaussian +// distribution. +class GaussianDistributionTests : public testing::TestWithParam<Param>, + public GaussianModel { + public: + GaussianDistributionTests() + : GaussianModel(GetParam().mean, GetParam().stddev) {} + + // SingleZTest provides a basic z-squared test of the mean vs. expected + // mean for data generated by the poisson distribution. + template <typename D> + bool SingleZTest(const double p, const size_t samples); + + // SingleChiSquaredTest provides a basic chi-squared test of the normal + // distribution. + template <typename D> + double SingleChiSquaredTest(); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng_{0x2B7E151628AED2A6}; +}; + +template <typename D> +bool GaussianDistributionTests::SingleZTest(const double p, + const size_t samples) { + D dis(mean(), stddev()); + + std::vector<double> data; + data.reserve(samples); + for (size_t i = 0; i < samples; i++) { + const double x = dis(rng_); + data.push_back(x); + } + + const double max_err = absl::random_internal::MaxErrorTolerance(p); + const auto m = absl::random_internal::ComputeDistributionMoments(data); + const double z = absl::random_internal::ZScore(mean(), m); + const bool pass = absl::random_internal::Near("z", z, 0.0, max_err); + + // NOTE: Informational statistical test: + // + // Compute the Jarque-Bera test statistic given the excess skewness + // and kurtosis. The statistic is drawn from a chi-square(2) distribution. + // https://en.wikipedia.org/wiki/Jarque%E2%80%93Bera_test + // + // The null-hypothesis (normal distribution) is rejected when + // (p = 0.05 => jb > 5.99) + // (p = 0.01 => jb > 9.21) + // NOTE: JB has a large type-I error rate, so it will reject the + // null-hypothesis even when it is true more often than the z-test. + // + const double jb = + static_cast<double>(m.n) / 6.0 * + (std::pow(m.skewness, 2.0) + std::pow(m.kurtosis - 3.0, 2.0) / 4.0); + + if (!pass || jb > 9.21) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("p=%f max_err=%f\n" + " mean=%f vs. %f\n" + " stddev=%f vs. %f\n" + " skewness=%f vs. %f\n" + " kurtosis=%f vs. %f\n" + " z=%f vs. 0\n" + " jb=%f vs. 9.21", + p, max_err, m.mean, mean(), std::sqrt(m.variance), + stddev(), m.skewness, skew(), m.kurtosis, + kurtosis(), z, jb)); + } + return pass; +} + +template <typename D> +double GaussianDistributionTests::SingleChiSquaredTest() { + const size_t kSamples = 10000; + const int kBuckets = 50; + + // The InverseCDF is the percent point function of the + // distribution, and can be used to assign buckets + // roughly uniformly. + std::vector<double> cutoffs; + const double kInc = 1.0 / static_cast<double>(kBuckets); + for (double p = kInc; p < 1.0; p += kInc) { + cutoffs.push_back(InverseCDF(p)); + } + if (cutoffs.back() != std::numeric_limits<double>::infinity()) { + cutoffs.push_back(std::numeric_limits<double>::infinity()); + } + + D dis(mean(), stddev()); + + std::vector<int32_t> counts(cutoffs.size(), 0); + for (int j = 0; j < kSamples; j++) { + const double x = dis(rng_); + auto it = std::upper_bound(cutoffs.begin(), cutoffs.end(), x); + counts[std::distance(cutoffs.begin(), it)]++; + } + + // Null-hypothesis is that the distribution is a gaussian distribution + // with the provided mean and stddev (not estimated from the data). + const int dof = static_cast<int>(counts.size()) - 1; + + // Our threshold for logging is 1-in-50. + const double threshold = absl::random_internal::ChiSquareValue(dof, 0.98); + + const double expected = + static_cast<double>(kSamples) / static_cast<double>(counts.size()); + + double chi_square = absl::random_internal::ChiSquareWithExpected( + std::begin(counts), std::end(counts), expected); + double p = absl::random_internal::ChiSquarePValue(chi_square, dof); + + // Log if the chi_square value is above the threshold. + if (chi_square > threshold) { + for (int i = 0; i < cutoffs.size(); i++) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("%d : (%f) = %d", i, cutoffs[i], counts[i])); + } + + ABSL_INTERNAL_LOG( + INFO, absl::StrCat("mean=", mean(), " stddev=", stddev(), "\n", // + " expected ", expected, "\n", // + kChiSquared, " ", chi_square, " (", p, ")\n", // + kChiSquared, " @ 0.98 = ", threshold)); + } + return p; +} + +TEST_P(GaussianDistributionTests, ZTest) { + // TODO(absl-team): Run these tests against std::normal_distribution<double> + // to validate outcomes are similar. + const size_t kSamples = 10000; + const auto& param = GetParam(); + const int expected_failures = + std::max(1, static_cast<int>(std::ceil(param.trials * param.p_fail))); + const double p = absl::random_internal::RequiredSuccessProbability( + param.p_fail, param.trials); + + int failures = 0; + for (int i = 0; i < param.trials; i++) { + failures += + SingleZTest<absl::gaussian_distribution<double>>(p, kSamples) ? 0 : 1; + } + EXPECT_LE(failures, expected_failures); +} + +TEST_P(GaussianDistributionTests, ChiSquaredTest) { + const int kTrials = 20; + int failures = 0; + + for (int i = 0; i < kTrials; i++) { + double p_value = + SingleChiSquaredTest<absl::gaussian_distribution<double>>(); + if (p_value < 0.0025) { // 1/400 + failures++; + } + } + // There is a 0.05% chance of producing at least one failure, so raise the + // failure threshold high enough to allow for a flake rate of less than one in + // 10,000. + EXPECT_LE(failures, 4); +} + +std::vector<Param> GenParams() { + return { + // Mean around 0. + Param{0.0, 1.0, 0.01, 100}, + Param{0.0, 1e2, 0.01, 100}, + Param{0.0, 1e4, 0.01, 100}, + Param{0.0, 1e8, 0.01, 100}, + Param{0.0, 1e16, 0.01, 100}, + Param{0.0, 1e-3, 0.01, 100}, + Param{0.0, 1e-5, 0.01, 100}, + Param{0.0, 1e-9, 0.01, 100}, + Param{0.0, 1e-17, 0.01, 100}, + + // Mean around 1. + Param{1.0, 1.0, 0.01, 100}, + Param{1.0, 1e2, 0.01, 100}, + Param{1.0, 1e-2, 0.01, 100}, + + // Mean around 100 / -100 + Param{1e2, 1.0, 0.01, 100}, + Param{-1e2, 1.0, 0.01, 100}, + Param{1e2, 1e6, 0.01, 100}, + Param{-1e2, 1e6, 0.01, 100}, + + // More extreme + Param{1e4, 1e4, 0.01, 100}, + Param{1e8, 1e4, 0.01, 100}, + Param{1e12, 1e4, 0.01, 100}, + }; +} + +std::string ParamName(const ::testing::TestParamInfo<Param>& info) { + const auto& p = info.param; + std::string name = absl::StrCat("mean_", absl::SixDigits(p.mean), "__stddev_", + absl::SixDigits(p.stddev)); + return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); +} + +INSTANTIATE_TEST_SUITE_P(All, GaussianDistributionTests, + ::testing::ValuesIn(GenParams()), ParamName); + +// NOTE: absl::gaussian_distribution is not guaranteed to be stable. +TEST(GaussianDistributionTest, StabilityTest) { + // absl::gaussian_distribution stability relies on the underlying zignor + // data, absl::random_interna::RandU64ToDouble, std::exp, std::log, and + // std::abs. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(11); + + { + absl::gaussian_distribution<double> dist; + std::generate(std::begin(output), std::end(output), + [&] { return static_cast<int>(10000000.0 * dist(urbg)); }); + + EXPECT_EQ(13, urbg.invocations()); + EXPECT_THAT(output, // + testing::ElementsAre(1494, 25518841, 9991550, 1351856, + -20373238, 3456682, 333530, -6804981, + -15279580, -16459654, 1494)); + } + + urbg.reset(); + { + absl::gaussian_distribution<float> dist; + std::generate(std::begin(output), std::end(output), + [&] { return static_cast<int>(1000000.0f * dist(urbg)); }); + + EXPECT_EQ(13, urbg.invocations()); + EXPECT_THAT( + output, // + testing::ElementsAre(149, 2551884, 999155, 135185, -2037323, 345668, + 33353, -680498, -1527958, -1645965, 149)); + } +} + +// This is an implementation-specific test. If any part of the implementation +// changes, then it is likely that this test will change as well. +// Also, if dependencies of the distribution change, such as RandU64ToDouble, +// then this is also likely to change. +TEST(GaussianDistributionTest, AlgorithmBounds) { + absl::gaussian_distribution<double> dist; + + // In ~95% of cases, a single value is used to generate the output. + // for all inputs where |x| < 0.750461021389 this should be the case. + // + // The exact constraints are based on the ziggurat tables, and any + // changes to the ziggurat tables may require adjusting these bounds. + // + // for i in range(0, len(X)-1): + // print i, X[i+1]/X[i], (X[i+1]/X[i] > 0.984375) + // + // 0.125 <= |values| <= 0.75 + const uint64_t kValues[] = { + 0x1000000000000100ull, 0x2000000000000100ull, 0x3000000000000100ull, + 0x4000000000000100ull, 0x5000000000000100ull, 0x6000000000000100ull, + // negative values + 0x9000000000000100ull, 0xa000000000000100ull, 0xb000000000000100ull, + 0xc000000000000100ull, 0xd000000000000100ull, 0xe000000000000100ull}; + + // 0.875 <= |values| <= 0.984375 + const uint64_t kExtraValues[] = { + 0x7000000000000100ull, 0x7800000000000100ull, // + 0x7c00000000000100ull, 0x7e00000000000100ull, // + // negative values + 0xf000000000000100ull, 0xf800000000000100ull, // + 0xfc00000000000100ull, 0xfe00000000000100ull}; + + auto make_box = [](uint64_t v, uint64_t box) { + return (v & 0xffffffffffffff80ull) | box; + }; + + // The box is the lower 7 bits of the value. When the box == 0, then + // the algorithm uses an escape hatch to select the result for large + // outputs. + for (uint64_t box = 0; box < 0x7f; box++) { + for (const uint64_t v : kValues) { + // Extra values are added to the sequence to attempt to avoid + // infinite loops from rejection sampling on bugs/errors. + absl::random_internal::sequence_urbg urbg( + {make_box(v, box), 0x0003eb76f6f7f755ull, 0x5FCEA50FDB2F953Bull}); + + auto a = dist(urbg); + EXPECT_EQ(1, urbg.invocations()) << box << " " << std::hex << v; + if (v & 0x8000000000000000ull) { + EXPECT_LT(a, 0.0) << box << " " << std::hex << v; + } else { + EXPECT_GT(a, 0.0) << box << " " << std::hex << v; + } + } + if (box > 10 && box < 100) { + // The center boxes use the fast algorithm for more + // than 98.4375% of values. + for (const uint64_t v : kExtraValues) { + absl::random_internal::sequence_urbg urbg( + {make_box(v, box), 0x0003eb76f6f7f755ull, 0x5FCEA50FDB2F953Bull}); + + auto a = dist(urbg); + EXPECT_EQ(1, urbg.invocations()) << box << " " << std::hex << v; + if (v & 0x8000000000000000ull) { + EXPECT_LT(a, 0.0) << box << " " << std::hex << v; + } else { + EXPECT_GT(a, 0.0) << box << " " << std::hex << v; + } + } + } + } + + // When the box == 0, the fallback algorithm uses a ratio of uniforms, + // which consumes 2 additional values from the urbg. + // Fallback also requires that the initial value be > 0.9271586026096681. + auto make_fallback = [](uint64_t v) { return (v & 0xffffffffffffff80ull); }; + + double tail[2]; + { + // 0.9375 + absl::random_internal::sequence_urbg urbg( + {make_fallback(0x7800000000000000ull), 0x13CCA830EB61BD96ull, + 0x00000076f6f7f755ull}); + tail[0] = dist(urbg); + EXPECT_EQ(3, urbg.invocations()); + EXPECT_GT(tail[0], 0); + } + { + // -0.9375 + absl::random_internal::sequence_urbg urbg( + {make_fallback(0xf800000000000000ull), 0x13CCA830EB61BD96ull, + 0x00000076f6f7f755ull}); + tail[1] = dist(urbg); + EXPECT_EQ(3, urbg.invocations()); + EXPECT_LT(tail[1], 0); + } + EXPECT_EQ(tail[0], -tail[1]); + EXPECT_EQ(418610, static_cast<int64_t>(tail[0] * 100000.0)); + + // When the box != 0, the fallback algorithm computes a wedge function. + // Depending on the box, the threshold for varies as high as + // 0.991522480228. + { + // 0.9921875, 0.875 + absl::random_internal::sequence_urbg urbg( + {make_box(0x7f00000000000000ull, 120), 0xe000000000000001ull, + 0x13CCA830EB61BD96ull}); + tail[0] = dist(urbg); + EXPECT_EQ(2, urbg.invocations()); + EXPECT_GT(tail[0], 0); + } + { + // -0.9921875, 0.875 + absl::random_internal::sequence_urbg urbg( + {make_box(0xff00000000000000ull, 120), 0xe000000000000001ull, + 0x13CCA830EB61BD96ull}); + tail[1] = dist(urbg); + EXPECT_EQ(2, urbg.invocations()); + EXPECT_LT(tail[1], 0); + } + EXPECT_EQ(tail[0], -tail[1]); + EXPECT_EQ(61948, static_cast<int64_t>(tail[0] * 100000.0)); + + // Fallback rejected, try again. + { + // -0.9921875, 0.0625 + absl::random_internal::sequence_urbg urbg( + {make_box(0xff00000000000000ull, 120), 0x1000000000000001, + make_box(0x1000000000000100ull, 50), 0x13CCA830EB61BD96ull}); + dist(urbg); + EXPECT_EQ(3, urbg.invocations()); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/generators_test.cc b/third_party/abseil_cpp/absl/random/generators_test.cc new file mode 100644 index 000000000000..41725f139cd3 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/generators_test.cc @@ -0,0 +1,179 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstddef> +#include <cstdint> +#include <random> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/random/distributions.h" +#include "absl/random/random.h" + +namespace { + +template <typename URBG> +void TestUniform(URBG* gen) { + // [a, b) default-semantics, inferred types. + absl::Uniform(*gen, 0, 100); // int + absl::Uniform(*gen, 0, 1.0); // Promoted to double + absl::Uniform(*gen, 0.0f, 1.0); // Promoted to double + absl::Uniform(*gen, 0.0, 1.0); // double + absl::Uniform(*gen, -1, 1L); // Promoted to long + + // Roll a die. + absl::Uniform(absl::IntervalClosedClosed, *gen, 1, 6); + + // Get a fraction. + absl::Uniform(absl::IntervalOpenOpen, *gen, 0.0, 1.0); + + // Assign a value to a random element. + std::vector<int> elems = {10, 20, 30, 40, 50}; + elems[absl::Uniform(*gen, 0u, elems.size())] = 5; + elems[absl::Uniform<size_t>(*gen, 0, elems.size())] = 3; + + // Choose some epsilon around zero. + absl::Uniform(absl::IntervalOpenOpen, *gen, -1.0, 1.0); + + // (a, b) semantics, inferred types. + absl::Uniform(absl::IntervalOpenOpen, *gen, 0, 1.0); // Promoted to double + + // Explict overriding of types. + absl::Uniform<int>(*gen, 0, 100); + absl::Uniform<int8_t>(*gen, 0, 100); + absl::Uniform<int16_t>(*gen, 0, 100); + absl::Uniform<uint16_t>(*gen, 0, 100); + absl::Uniform<int32_t>(*gen, 0, 1 << 10); + absl::Uniform<uint32_t>(*gen, 0, 1 << 10); + absl::Uniform<int64_t>(*gen, 0, 1 << 10); + absl::Uniform<uint64_t>(*gen, 0, 1 << 10); + + absl::Uniform<float>(*gen, 0.0, 1.0); + absl::Uniform<float>(*gen, 0, 1); + absl::Uniform<float>(*gen, -1, 1); + absl::Uniform<double>(*gen, 0.0, 1.0); + + absl::Uniform<float>(*gen, -1.0, 0); + absl::Uniform<double>(*gen, -1.0, 0); + + // Tagged + absl::Uniform<double>(absl::IntervalClosedClosed, *gen, 0, 1); + absl::Uniform<double>(absl::IntervalClosedOpen, *gen, 0, 1); + absl::Uniform<double>(absl::IntervalOpenOpen, *gen, 0, 1); + absl::Uniform<double>(absl::IntervalOpenClosed, *gen, 0, 1); + absl::Uniform<double>(absl::IntervalClosedClosed, *gen, 0, 1); + absl::Uniform<double>(absl::IntervalOpenOpen, *gen, 0, 1); + + absl::Uniform<int>(absl::IntervalClosedClosed, *gen, 0, 100); + absl::Uniform<int>(absl::IntervalClosedOpen, *gen, 0, 100); + absl::Uniform<int>(absl::IntervalOpenOpen, *gen, 0, 100); + absl::Uniform<int>(absl::IntervalOpenClosed, *gen, 0, 100); + absl::Uniform<int>(absl::IntervalClosedClosed, *gen, 0, 100); + absl::Uniform<int>(absl::IntervalOpenOpen, *gen, 0, 100); + + // With *generator as an R-value reference. + absl::Uniform<int>(URBG(), 0, 100); + absl::Uniform<double>(URBG(), 0.0, 1.0); +} + +template <typename URBG> +void TestExponential(URBG* gen) { + absl::Exponential<float>(*gen); + absl::Exponential<double>(*gen); + absl::Exponential<double>(URBG()); +} + +template <typename URBG> +void TestPoisson(URBG* gen) { + // [rand.dist.pois] Indicates that the std::poisson_distribution + // is parameterized by IntType, however MSVC does not allow 8-bit + // types. + absl::Poisson<int>(*gen); + absl::Poisson<int16_t>(*gen); + absl::Poisson<uint16_t>(*gen); + absl::Poisson<int32_t>(*gen); + absl::Poisson<uint32_t>(*gen); + absl::Poisson<int64_t>(*gen); + absl::Poisson<uint64_t>(*gen); + absl::Poisson<uint64_t>(URBG()); +} + +template <typename URBG> +void TestBernoulli(URBG* gen) { + absl::Bernoulli(*gen, 0.5); + absl::Bernoulli(*gen, 0.5); +} + +template <typename URBG> +void TestZipf(URBG* gen) { + absl::Zipf<int>(*gen, 100); + absl::Zipf<int8_t>(*gen, 100); + absl::Zipf<int16_t>(*gen, 100); + absl::Zipf<uint16_t>(*gen, 100); + absl::Zipf<int32_t>(*gen, 1 << 10); + absl::Zipf<uint32_t>(*gen, 1 << 10); + absl::Zipf<int64_t>(*gen, 1 << 10); + absl::Zipf<uint64_t>(*gen, 1 << 10); + absl::Zipf<uint64_t>(URBG(), 1 << 10); +} + +template <typename URBG> +void TestGaussian(URBG* gen) { + absl::Gaussian<float>(*gen, 1.0, 1.0); + absl::Gaussian<double>(*gen, 1.0, 1.0); + absl::Gaussian<double>(URBG(), 1.0, 1.0); +} + +template <typename URBG> +void TestLogNormal(URBG* gen) { + absl::LogUniform<int>(*gen, 0, 100); + absl::LogUniform<int8_t>(*gen, 0, 100); + absl::LogUniform<int16_t>(*gen, 0, 100); + absl::LogUniform<uint16_t>(*gen, 0, 100); + absl::LogUniform<int32_t>(*gen, 0, 1 << 10); + absl::LogUniform<uint32_t>(*gen, 0, 1 << 10); + absl::LogUniform<int64_t>(*gen, 0, 1 << 10); + absl::LogUniform<uint64_t>(*gen, 0, 1 << 10); + absl::LogUniform<uint64_t>(URBG(), 0, 1 << 10); +} + +template <typename URBG> +void CompatibilityTest() { + URBG gen; + + TestUniform(&gen); + TestExponential(&gen); + TestPoisson(&gen); + TestBernoulli(&gen); + TestZipf(&gen); + TestGaussian(&gen); + TestLogNormal(&gen); +} + +TEST(std_mt19937_64, Compatibility) { + // Validate with std::mt19937_64 + CompatibilityTest<std::mt19937_64>(); +} + +TEST(BitGen, Compatibility) { + // Validate with absl::BitGen + CompatibilityTest<absl::BitGen>(); +} + +TEST(InsecureBitGen, Compatibility) { + // Validate with absl::InsecureBitGen + CompatibilityTest<absl::InsecureBitGen>(); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/BUILD.bazel b/third_party/abseil_cpp/absl/random/internal/BUILD.bazel new file mode 100644 index 000000000000..8485e28b010b --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/BUILD.bazel @@ -0,0 +1,730 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") + +# Internal-only implementation classes for Abseil Random +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_RANDOM_RANDEN_COPTS", + "ABSL_TEST_COPTS", + "absl_random_randen_copts_init", +) + +package(default_visibility = [ + "//absl/random:__pkg__", +]) + +licenses(["notice"]) + +cc_library( + name = "traits", + hdrs = ["traits.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/base:config"], +) + +cc_library( + name = "distribution_caller", + hdrs = ["distribution_caller.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:fast_type_id", + "//absl/utility", + ], +) + +cc_library( + name = "fast_uniform_bits", + hdrs = [ + "fast_uniform_bits.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/meta:type_traits", + ], +) + +cc_library( + name = "seed_material", + srcs = [ + "seed_material.cc", + ], + hdrs = [ + "seed_material.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS + select({ + "//absl:windows": ["-DEFAULTLIB:bcrypt.lib"], + "//conditions:default": [], + }), + deps = [ + ":fast_uniform_bits", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/strings", + "//absl/types:optional", + "//absl/types:span", + ], +) + +cc_library( + name = "pool_urbg", + srcs = [ + "pool_urbg.cc", + ], + hdrs = [ + "pool_urbg.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = select({ + "//absl:windows": [], + "//absl:wasm": [], + "//conditions:default": ["-pthread"], + }) + ABSL_DEFAULT_LINKOPTS, + deps = [ + ":randen", + ":seed_material", + ":traits", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/base:raw_logging_internal", + "//absl/random:seed_gen_exception", + "//absl/types:span", + ], +) + +cc_library( + name = "explicit_seed_seq", + testonly = 1, + hdrs = [ + "explicit_seed_seq.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/base:config"], +) + +cc_library( + name = "sequence_urbg", + testonly = 1, + hdrs = [ + "sequence_urbg.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/base:config"], +) + +cc_library( + name = "salted_seed_seq", + hdrs = [ + "salted_seed_seq.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":seed_material", + "//absl/container:inlined_vector", + "//absl/meta:type_traits", + "//absl/types:optional", + "//absl/types:span", + ], +) + +cc_library( + name = "iostream_state_saver", + hdrs = ["iostream_state_saver.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/meta:type_traits", + "//absl/numeric:int128", + ], +) + +cc_library( + name = "generate_real", + hdrs = [ + "generate_real.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":fastmath", + ":traits", + "//absl/base:bits", + "//absl/meta:type_traits", + ], +) + +cc_library( + name = "fastmath", + hdrs = [ + "fastmath.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = ["//absl/base:bits"], +) + +cc_library( + name = "wide_multiply", + hdrs = ["wide_multiply.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":traits", + "//absl/base:bits", + "//absl/base:config", + "//absl/numeric:int128", + ], +) + +cc_library( + name = "nonsecure_base", + hdrs = ["nonsecure_base.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":pool_urbg", + ":salted_seed_seq", + ":seed_material", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/types:optional", + "//absl/types:span", + ], +) + +cc_library( + name = "pcg_engine", + hdrs = ["pcg_engine.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":fastmath", + ":iostream_state_saver", + "//absl/base:config", + "//absl/meta:type_traits", + "//absl/numeric:int128", + ], +) + +cc_library( + name = "randen_engine", + hdrs = ["randen_engine.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":iostream_state_saver", + ":randen", + "//absl/meta:type_traits", + ], +) + +cc_library( + name = "platform", + srcs = [ + "randen_round_keys.cc", + ], + hdrs = [ + "randen_traits.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + textual_hdrs = [ + "platform.h", + ], + deps = ["//absl/base:config"], +) + +cc_library( + name = "randen", + srcs = [ + "randen.cc", + ], + hdrs = [ + "randen.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":platform", + ":randen_hwaes", + ":randen_slow", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "randen_slow", + srcs = ["randen_slow.cc"], + hdrs = ["randen_slow.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":platform", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +absl_random_randen_copts_init() + +cc_library( + name = "randen_hwaes", + srcs = [ + "randen_detect.cc", + ], + hdrs = [ + "randen_detect.h", + "randen_hwaes.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":platform", + ":randen_hwaes_impl", + "//absl/base:config", + ], +) + +# build with --save_temps to see assembly language output. +cc_library( + name = "randen_hwaes_impl", + srcs = [ + "randen_hwaes.cc", + "randen_hwaes.h", + ], + copts = ABSL_DEFAULT_COPTS + ABSL_RANDOM_RANDEN_COPTS + select({ + "//absl:windows": [], + "//conditions:default": ["-Wno-pass-failed"], + }), + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":platform", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_binary( + name = "gaussian_distribution_gentables", + srcs = [ + "gaussian_distribution_gentables.cc", + ], + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:core_headers", + "//absl/random:distributions", + ], +) + +cc_library( + name = "distribution_test_util", + testonly = 1, + srcs = [ + "chi_square.cc", + "distribution_test_util.cc", + ], + hdrs = [ + "chi_square.h", + "distribution_test_util.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/strings", + "//absl/strings:str_format", + "//absl/types:span", + ], +) + +# Common tags for tests, etc. +ABSL_RANDOM_NONPORTABLE_TAGS = [ + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + "no_test_darwin_x86_64", + "no_test_ios_x86_64", + "no_test_loonix", + "no_test_msvc_x64", + "no_test_wasm", +] + +cc_test( + name = "traits_test", + size = "small", + srcs = ["traits_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "generate_real_test", + size = "small", + srcs = [ + "generate_real_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":generate_real", + "//absl/base:bits", + "//absl/flags:flag", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "distribution_test_util_test", + size = "small", + srcs = ["distribution_test_util_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distribution_test_util", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "fastmath_test", + size = "small", + srcs = ["fastmath_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":fastmath", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "explicit_seed_seq_test", + size = "small", + srcs = ["explicit_seed_seq_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":explicit_seed_seq", + "//absl/random:seed_sequences", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "salted_seed_seq_test", + size = "small", + srcs = ["salted_seed_seq_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":salted_seed_seq", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "chi_square_test", + size = "small", + srcs = [ + "chi_square_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":distribution_test_util", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "fast_uniform_bits_test", + size = "small", + srcs = [ + "fast_uniform_bits_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":fast_uniform_bits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "mock_helpers", + hdrs = ["mock_helpers.h"], + deps = [ + "//absl/base:fast_type_id", + "//absl/types:optional", + ], +) + +cc_library( + name = "mock_overload_set", + testonly = 1, + hdrs = ["mock_overload_set.h"], + deps = [ + ":mock_helpers", + "//absl/random:mocking_bit_gen", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "nonsecure_base_test", + size = "small", + srcs = [ + "nonsecure_base_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":nonsecure_base", + "//absl/random", + "//absl/random:distributions", + "//absl/random:seed_sequences", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "seed_material_test", + size = "small", + srcs = ["seed_material_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":seed_material", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "pool_urbg_test", + size = "small", + srcs = [ + "pool_urbg_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":pool_urbg", + "//absl/meta:type_traits", + "//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "pcg_engine_test", + size = "medium", # Trying to measure accuracy. + srcs = ["pcg_engine_test.cc"], + copts = ABSL_TEST_COPTS, + flaky = 1, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":explicit_seed_seq", + ":pcg_engine", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "randen_engine_test", + size = "medium", + srcs = [ + "randen_engine_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":explicit_seed_seq", + ":randen_engine", + "//absl/base:raw_logging_internal", + "//absl/strings", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "randen_test", + size = "small", + srcs = ["randen_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":randen", + "//absl/meta:type_traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "randen_slow_test", + size = "small", + srcs = ["randen_slow_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":platform", + ":randen_slow", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "randen_hwaes_test", + size = "small", + srcs = ["randen_hwaes_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ABSL_RANDOM_NONPORTABLE_TAGS, + deps = [ + ":platform", + ":randen_hwaes", + ":randen_hwaes_impl", # build_cleaner: keep + "//absl/base:raw_logging_internal", + "//absl/strings:str_format", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "wide_multiply_test", + size = "small", + srcs = ["wide_multiply_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":wide_multiply", + "//absl/base:bits", + "//absl/numeric:int128", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "nanobenchmark", + srcs = ["nanobenchmark.cc"], + linkopts = ABSL_DEFAULT_LINKOPTS, + textual_hdrs = ["nanobenchmark.h"], + deps = [ + ":platform", + ":randen_engine", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "uniform_helper", + hdrs = ["uniform_helper.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":traits", + "//absl/base:config", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "nanobenchmark_test", + size = "small", + srcs = ["nanobenchmark_test.cc"], + flaky = 1, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "benchmark", + "no_test_ios_x86_64", + "no_test_loonix", # Crashing. + ], + deps = [ + ":nanobenchmark", + "//absl/base:raw_logging_internal", + "//absl/strings", + ], +) + +cc_test( + name = "randen_benchmarks", + size = "medium", + timeout = "long", + srcs = ["randen_benchmarks.cc"], + copts = ABSL_TEST_COPTS + ABSL_RANDOM_RANDEN_COPTS, + flaky = 1, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ABSL_RANDOM_NONPORTABLE_TAGS + ["benchmark"], + deps = [ + ":nanobenchmark", + ":platform", + ":randen", + ":randen_engine", + ":randen_hwaes", + ":randen_hwaes_impl", + ":randen_slow", + "//absl/base:raw_logging_internal", + "//absl/strings", + ], +) + +cc_test( + name = "iostream_state_saver_test", + size = "small", + srcs = ["iostream_state_saver_test.cc"], + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":iostream_state_saver", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "uniform_helper_test", + size = "small", + srcs = ["uniform_helper_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":uniform_helper", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/random/internal/chi_square.cc b/third_party/abseil_cpp/absl/random/internal/chi_square.cc new file mode 100644 index 000000000000..640d48cea6f4 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/chi_square.cc @@ -0,0 +1,232 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/chi_square.h" + +#include <cmath> + +#include "absl/random/internal/distribution_test_util.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +#if defined(__EMSCRIPTEN__) +// Workaround __EMSCRIPTEN__ error: llvm_fma_f64 not found. +inline double fma(double x, double y, double z) { + return (x * y) + z; +} +#endif + +// Use Horner's method to evaluate a polynomial. +template <typename T, unsigned N> +inline T EvaluatePolynomial(T x, const T (&poly)[N]) { +#if !defined(__EMSCRIPTEN__) + using std::fma; +#endif + T p = poly[N - 1]; + for (unsigned i = 2; i <= N; i++) { + p = fma(p, x, poly[N - i]); + } + return p; +} + +static constexpr int kLargeDOF = 150; + +// Returns the probability of a normal z-value. +// +// Adapted from the POZ function in: +// Ibbetson D, Algorithm 209 +// Collected Algorithms of the CACM 1963 p. 616 +// +double POZ(double z) { + static constexpr double kP1[] = { + 0.797884560593, -0.531923007300, 0.319152932694, + -0.151968751364, 0.059054035642, -0.019198292004, + 0.005198775019, -0.001075204047, 0.000124818987, + }; + static constexpr double kP2[] = { + 0.999936657524, 0.000535310849, -0.002141268741, 0.005353579108, + -0.009279453341, 0.011630447319, -0.010557625006, 0.006549791214, + -0.002034254874, -0.000794620820, 0.001390604284, -0.000676904986, + -0.000019538132, 0.000152529290, -0.000045255659, + }; + + const double kZMax = 6.0; // Maximum meaningful z-value. + if (z == 0.0) { + return 0.5; + } + double x; + double y = 0.5 * std::fabs(z); + if (y >= (kZMax * 0.5)) { + x = 1.0; + } else if (y < 1.0) { + double w = y * y; + x = EvaluatePolynomial(w, kP1) * y * 2.0; + } else { + y -= 2.0; + x = EvaluatePolynomial(y, kP2); + } + return z > 0.0 ? ((x + 1.0) * 0.5) : ((1.0 - x) * 0.5); +} + +// Approximates the survival function of the normal distribution. +// +// Algorithm 26.2.18, from: +// [Abramowitz and Stegun, Handbook of Mathematical Functions,p.932] +// http://people.math.sfu.ca/~cbm/aands/abramowitz_and_stegun.pdf +// +double normal_survival(double z) { + // Maybe replace with the alternate formulation. + // 0.5 * erfc((x - mean)/(sqrt(2) * sigma)) + static constexpr double kR[] = { + 1.0, 0.196854, 0.115194, 0.000344, 0.019527, + }; + double r = EvaluatePolynomial(z, kR); + r *= r; + return 0.5 / (r * r); +} + +} // namespace + +// Calculates the critical chi-square value given degrees-of-freedom and a +// p-value, usually using bisection. Also known by the name CRITCHI. +double ChiSquareValue(int dof, double p) { + static constexpr double kChiEpsilon = + 0.000001; // Accuracy of the approximation. + static constexpr double kChiMax = + 99999.0; // Maximum chi-squared value. + + const double p_value = 1.0 - p; + if (dof < 1 || p_value > 1.0) { + return 0.0; + } + + if (dof > kLargeDOF) { + // For large degrees of freedom, use the normal approximation by + // Wilson, E. B. and Hilferty, M. M. (1931) + // chi^2 - mean + // Z = -------------- + // stddev + const double z = InverseNormalSurvival(p_value); + const double mean = 1 - 2.0 / (9 * dof); + const double variance = 2.0 / (9 * dof); + // Cannot use this method if the variance is 0. + if (variance != 0) { + return std::pow(z * std::sqrt(variance) + mean, 3.0) * dof; + } + } + + if (p_value <= 0.0) return kChiMax; + + // Otherwise search for the p value by bisection + double min_chisq = 0.0; + double max_chisq = kChiMax; + double current = dof / std::sqrt(p_value); + while ((max_chisq - min_chisq) > kChiEpsilon) { + if (ChiSquarePValue(current, dof) < p_value) { + max_chisq = current; + } else { + min_chisq = current; + } + current = (max_chisq + min_chisq) * 0.5; + } + return current; +} + +// Calculates the p-value (probability) of a given chi-square value +// and degrees of freedom. +// +// Adapted from the POCHISQ function from: +// Hill, I. D. and Pike, M. C. Algorithm 299 +// Collected Algorithms of the CACM 1963 p. 243 +// +double ChiSquarePValue(double chi_square, int dof) { + static constexpr double kLogSqrtPi = + 0.5723649429247000870717135; // Log[Sqrt[Pi]] + static constexpr double kInverseSqrtPi = + 0.5641895835477562869480795; // 1/(Sqrt[Pi]) + + // For large degrees of freedom, use the normal approximation by + // Wilson, E. B. and Hilferty, M. M. (1931) + // Via Wikipedia: + // By the Central Limit Theorem, because the chi-square distribution is the + // sum of k independent random variables with finite mean and variance, it + // converges to a normal distribution for large k. + if (dof > kLargeDOF) { + // Re-scale everything. + const double chi_square_scaled = std::pow(chi_square / dof, 1.0 / 3); + const double mean = 1 - 2.0 / (9 * dof); + const double variance = 2.0 / (9 * dof); + // If variance is 0, this method cannot be used. + if (variance != 0) { + const double z = (chi_square_scaled - mean) / std::sqrt(variance); + if (z > 0) { + return normal_survival(z); + } else if (z < 0) { + return 1.0 - normal_survival(-z); + } else { + return 0.5; + } + } + } + + // The chi square function is >= 0 for any degrees of freedom. + // In other words, probability that the chi square function >= 0 is 1. + if (chi_square <= 0.0) return 1.0; + + // If the degrees of freedom is zero, the chi square function is always 0 by + // definition. In other words, the probability that the chi square function + // is > 0 is zero (chi square values <= 0 have been filtered above). + if (dof < 1) return 0; + + auto capped_exp = [](double x) { return x < -20 ? 0.0 : std::exp(x); }; + static constexpr double kBigX = 20; + + double a = 0.5 * chi_square; + const bool even = !(dof & 1); // True if dof is an even number. + const double y = capped_exp(-a); + double s = even ? y : (2.0 * POZ(-std::sqrt(chi_square))); + + if (dof <= 2) { + return s; + } + + chi_square = 0.5 * (dof - 1.0); + double z = (even ? 1.0 : 0.5); + if (a > kBigX) { + double e = (even ? 0.0 : kLogSqrtPi); + double c = std::log(a); + while (z <= chi_square) { + e = std::log(z) + e; + s += capped_exp(c * z - a - e); + z += 1.0; + } + return s; + } + + double e = (even ? 1.0 : (kInverseSqrtPi / std::sqrt(a))); + double c = 0.0; + while (z <= chi_square) { + e = e * (a / z); + c = c + e; + z += 1.0; + } + return c * y + s; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/chi_square.h b/third_party/abseil_cpp/absl/random/internal/chi_square.h new file mode 100644 index 000000000000..07f4fbe52298 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/chi_square.h @@ -0,0 +1,89 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_CHI_SQUARE_H_ +#define ABSL_RANDOM_INTERNAL_CHI_SQUARE_H_ + +// The chi-square statistic. +// +// Useful for evaluating if `D` independent random variables are behaving as +// expected, or if two distributions are similar. (`D` is the degrees of +// freedom). +// +// Each bucket should have an expected count of 10 or more for the chi square to +// be meaningful. + +#include <cassert> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +constexpr const char kChiSquared[] = "chi-squared"; + +// Returns the measured chi square value, using a single expected value. This +// assumes that the values in [begin, end) are uniformly distributed. +template <typename Iterator> +double ChiSquareWithExpected(Iterator begin, Iterator end, double expected) { + // Compute the sum and the number of buckets. + assert(expected >= 10); // require at least 10 samples per bucket. + double chi_square = 0; + for (auto it = begin; it != end; it++) { + double d = static_cast<double>(*it) - expected; + chi_square += d * d; + } + chi_square = chi_square / expected; + return chi_square; +} + +// Returns the measured chi square value, taking the actual value of each bucket +// from the first set of iterators, and the expected value of each bucket from +// the second set of iterators. +template <typename Iterator, typename Expected> +double ChiSquare(Iterator it, Iterator end, Expected eit, Expected eend) { + double chi_square = 0; + for (; it != end && eit != eend; ++it, ++eit) { + if (*it > 0) { + assert(*eit > 0); + } + double e = static_cast<double>(*eit); + double d = static_cast<double>(*it - *eit); + if (d != 0) { + assert(e > 0); + chi_square += (d * d) / e; + } + } + assert(it == end && eit == eend); + return chi_square; +} + +// ====================================================================== +// The following methods can be used for an arbitrary significance level. +// + +// Calculates critical chi-square values to produce the given p-value using a +// bisection search for a value within epsilon, relying on the monotonicity of +// ChiSquarePValue(). +double ChiSquareValue(int dof, double p); + +// Calculates the p-value (probability) of a given chi-square value. +double ChiSquarePValue(double chi_square, int dof); + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_CHI_SQUARE_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/chi_square_test.cc b/third_party/abseil_cpp/absl/random/internal/chi_square_test.cc new file mode 100644 index 000000000000..5025defac12c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/chi_square_test.cc @@ -0,0 +1,365 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/chi_square.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <numeric> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/macros.h" + +using absl::random_internal::ChiSquare; +using absl::random_internal::ChiSquarePValue; +using absl::random_internal::ChiSquareValue; +using absl::random_internal::ChiSquareWithExpected; + +namespace { + +TEST(ChiSquare, Value) { + struct { + int line; + double chi_square; + int df; + double confidence; + } const specs[] = { + // Testing lookup at 1% confidence + {__LINE__, 0, 0, 0.01}, + {__LINE__, 0.00016, 1, 0.01}, + {__LINE__, 1.64650, 8, 0.01}, + {__LINE__, 5.81221, 16, 0.01}, + {__LINE__, 156.4319, 200, 0.01}, + {__LINE__, 1121.3784, 1234, 0.01}, + {__LINE__, 53557.1629, 54321, 0.01}, + {__LINE__, 651662.6647, 654321, 0.01}, + + // Testing lookup at 99% confidence + {__LINE__, 0, 0, 0.99}, + {__LINE__, 6.635, 1, 0.99}, + {__LINE__, 20.090, 8, 0.99}, + {__LINE__, 32.000, 16, 0.99}, + {__LINE__, 249.4456, 200, 0.99}, + {__LINE__, 1131.1573, 1023, 0.99}, + {__LINE__, 1352.5038, 1234, 0.99}, + {__LINE__, 55090.7356, 54321, 0.99}, + {__LINE__, 656985.1514, 654321, 0.99}, + + // Testing lookup at 99.9% confidence + {__LINE__, 16.2659, 3, 0.999}, + {__LINE__, 22.4580, 6, 0.999}, + {__LINE__, 267.5409, 200, 0.999}, + {__LINE__, 1168.5033, 1023, 0.999}, + {__LINE__, 55345.1741, 54321, 0.999}, + {__LINE__, 657861.7284, 654321, 0.999}, + {__LINE__, 51.1772, 24, 0.999}, + {__LINE__, 59.7003, 30, 0.999}, + {__LINE__, 37.6984, 15, 0.999}, + {__LINE__, 29.5898, 10, 0.999}, + {__LINE__, 27.8776, 9, 0.999}, + + // Testing lookup at random confidences + {__LINE__, 0.000157088, 1, 0.01}, + {__LINE__, 5.31852, 2, 0.93}, + {__LINE__, 1.92256, 4, 0.25}, + {__LINE__, 10.7709, 13, 0.37}, + {__LINE__, 26.2514, 17, 0.93}, + {__LINE__, 36.4799, 29, 0.84}, + {__LINE__, 25.818, 31, 0.27}, + {__LINE__, 63.3346, 64, 0.50}, + {__LINE__, 196.211, 128, 0.9999}, + {__LINE__, 215.21, 243, 0.10}, + {__LINE__, 285.393, 256, 0.90}, + {__LINE__, 984.504, 1024, 0.1923}, + {__LINE__, 2043.85, 2048, 0.4783}, + {__LINE__, 48004.6, 48273, 0.194}, + }; + for (const auto& spec : specs) { + SCOPED_TRACE(spec.line); + // Verify all values are have at most a 1% relative error. + const double val = ChiSquareValue(spec.df, spec.confidence); + const double err = std::max(5e-6, spec.chi_square / 5e3); // 1 part in 5000 + EXPECT_NEAR(spec.chi_square, val, err) << spec.line; + } + + // Relaxed test for extreme values, from + // http://www.ciphersbyritter.com/JAVASCRP/NORMCHIK.HTM#ChiSquare + EXPECT_NEAR(49.2680, ChiSquareValue(100, 1e-6), 5); // 0.000'005 mark + EXPECT_NEAR(123.499, ChiSquareValue(200, 1e-6), 5); // 0.000'005 mark + + EXPECT_NEAR(149.449, ChiSquareValue(100, 0.999), 0.01); + EXPECT_NEAR(161.318, ChiSquareValue(100, 0.9999), 0.01); + EXPECT_NEAR(172.098, ChiSquareValue(100, 0.99999), 0.01); + + EXPECT_NEAR(381.426, ChiSquareValue(300, 0.999), 0.05); + EXPECT_NEAR(399.756, ChiSquareValue(300, 0.9999), 0.1); + EXPECT_NEAR(416.126, ChiSquareValue(300, 0.99999), 0.2); +} + +TEST(ChiSquareTest, PValue) { + struct { + int line; + double pval; + double chi_square; + int df; + } static const specs[] = { + {__LINE__, 1, 0, 0}, + {__LINE__, 0, 0.001, 0}, + {__LINE__, 1.000, 0, 453}, + {__LINE__, 0.134471, 7972.52, 7834}, + {__LINE__, 0.203922, 28.32, 23}, + {__LINE__, 0.737171, 48274, 48472}, + {__LINE__, 0.444146, 583.1234, 579}, + {__LINE__, 0.294814, 138.2, 130}, + {__LINE__, 0.0816532, 12.63, 7}, + {__LINE__, 0, 682.32, 67}, + {__LINE__, 0.49405, 999, 999}, + {__LINE__, 1.000, 0, 9999}, + {__LINE__, 0.997477, 0.00001, 1}, + {__LINE__, 0, 5823.21, 5040}, + }; + for (const auto& spec : specs) { + SCOPED_TRACE(spec.line); + const double pval = ChiSquarePValue(spec.chi_square, spec.df); + EXPECT_NEAR(spec.pval, pval, 1e-3); + } +} + +TEST(ChiSquareTest, CalcChiSquare) { + struct { + int line; + std::vector<int> expected; + std::vector<int> actual; + } const specs[] = { + {__LINE__, + {56, 234, 76, 1, 546, 1, 87, 345, 1, 234}, + {2, 132, 4, 43, 234, 8, 345, 8, 236, 56}}, + {__LINE__, + {123, 36, 234, 367, 345, 2, 456, 567, 234, 567}, + {123, 56, 2345, 8, 345, 8, 2345, 23, 48, 267}}, + {__LINE__, + {123, 234, 345, 456, 567, 678, 789, 890, 98, 76}, + {123, 234, 345, 456, 567, 678, 789, 890, 98, 76}}, + {__LINE__, {3, 675, 23, 86, 2, 8, 2}, {456, 675, 23, 86, 23, 65, 2}}, + {__LINE__, {1}, {23}}, + }; + for (const auto& spec : specs) { + SCOPED_TRACE(spec.line); + double chi_square = 0; + for (int i = 0; i < spec.expected.size(); ++i) { + const double diff = spec.actual[i] - spec.expected[i]; + chi_square += (diff * diff) / spec.expected[i]; + } + EXPECT_NEAR(chi_square, + ChiSquare(std::begin(spec.actual), std::end(spec.actual), + std::begin(spec.expected), std::end(spec.expected)), + 1e-5); + } +} + +TEST(ChiSquareTest, CalcChiSquareInt64) { + const int64_t data[3] = {910293487, 910292491, 910216780}; + // $ python -c "import scipy.stats + // > print scipy.stats.chisquare([910293487, 910292491, 910216780])[0]" + // 4.25410123524 + double sum = std::accumulate(std::begin(data), std::end(data), double{0}); + size_t n = std::distance(std::begin(data), std::end(data)); + double a = ChiSquareWithExpected(std::begin(data), std::end(data), sum / n); + EXPECT_NEAR(4.254101, a, 1e-6); + + // ... Or with known values. + double b = + ChiSquareWithExpected(std::begin(data), std::end(data), 910267586.0); + EXPECT_NEAR(4.254101, b, 1e-6); +} + +TEST(ChiSquareTest, TableData) { + // Test data from + // http://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm + // 0.90 0.95 0.975 0.99 0.999 + const double data[100][5] = { + /* 1*/ {2.706, 3.841, 5.024, 6.635, 10.828}, + /* 2*/ {4.605, 5.991, 7.378, 9.210, 13.816}, + /* 3*/ {6.251, 7.815, 9.348, 11.345, 16.266}, + /* 4*/ {7.779, 9.488, 11.143, 13.277, 18.467}, + /* 5*/ {9.236, 11.070, 12.833, 15.086, 20.515}, + /* 6*/ {10.645, 12.592, 14.449, 16.812, 22.458}, + /* 7*/ {12.017, 14.067, 16.013, 18.475, 24.322}, + /* 8*/ {13.362, 15.507, 17.535, 20.090, 26.125}, + /* 9*/ {14.684, 16.919, 19.023, 21.666, 27.877}, + /*10*/ {15.987, 18.307, 20.483, 23.209, 29.588}, + /*11*/ {17.275, 19.675, 21.920, 24.725, 31.264}, + /*12*/ {18.549, 21.026, 23.337, 26.217, 32.910}, + /*13*/ {19.812, 22.362, 24.736, 27.688, 34.528}, + /*14*/ {21.064, 23.685, 26.119, 29.141, 36.123}, + /*15*/ {22.307, 24.996, 27.488, 30.578, 37.697}, + /*16*/ {23.542, 26.296, 28.845, 32.000, 39.252}, + /*17*/ {24.769, 27.587, 30.191, 33.409, 40.790}, + /*18*/ {25.989, 28.869, 31.526, 34.805, 42.312}, + /*19*/ {27.204, 30.144, 32.852, 36.191, 43.820}, + /*20*/ {28.412, 31.410, 34.170, 37.566, 45.315}, + /*21*/ {29.615, 32.671, 35.479, 38.932, 46.797}, + /*22*/ {30.813, 33.924, 36.781, 40.289, 48.268}, + /*23*/ {32.007, 35.172, 38.076, 41.638, 49.728}, + /*24*/ {33.196, 36.415, 39.364, 42.980, 51.179}, + /*25*/ {34.382, 37.652, 40.646, 44.314, 52.620}, + /*26*/ {35.563, 38.885, 41.923, 45.642, 54.052}, + /*27*/ {36.741, 40.113, 43.195, 46.963, 55.476}, + /*28*/ {37.916, 41.337, 44.461, 48.278, 56.892}, + /*29*/ {39.087, 42.557, 45.722, 49.588, 58.301}, + /*30*/ {40.256, 43.773, 46.979, 50.892, 59.703}, + /*31*/ {41.422, 44.985, 48.232, 52.191, 61.098}, + /*32*/ {42.585, 46.194, 49.480, 53.486, 62.487}, + /*33*/ {43.745, 47.400, 50.725, 54.776, 63.870}, + /*34*/ {44.903, 48.602, 51.966, 56.061, 65.247}, + /*35*/ {46.059, 49.802, 53.203, 57.342, 66.619}, + /*36*/ {47.212, 50.998, 54.437, 58.619, 67.985}, + /*37*/ {48.363, 52.192, 55.668, 59.893, 69.347}, + /*38*/ {49.513, 53.384, 56.896, 61.162, 70.703}, + /*39*/ {50.660, 54.572, 58.120, 62.428, 72.055}, + /*40*/ {51.805, 55.758, 59.342, 63.691, 73.402}, + /*41*/ {52.949, 56.942, 60.561, 64.950, 74.745}, + /*42*/ {54.090, 58.124, 61.777, 66.206, 76.084}, + /*43*/ {55.230, 59.304, 62.990, 67.459, 77.419}, + /*44*/ {56.369, 60.481, 64.201, 68.710, 78.750}, + /*45*/ {57.505, 61.656, 65.410, 69.957, 80.077}, + /*46*/ {58.641, 62.830, 66.617, 71.201, 81.400}, + /*47*/ {59.774, 64.001, 67.821, 72.443, 82.720}, + /*48*/ {60.907, 65.171, 69.023, 73.683, 84.037}, + /*49*/ {62.038, 66.339, 70.222, 74.919, 85.351}, + /*50*/ {63.167, 67.505, 71.420, 76.154, 86.661}, + /*51*/ {64.295, 68.669, 72.616, 77.386, 87.968}, + /*52*/ {65.422, 69.832, 73.810, 78.616, 89.272}, + /*53*/ {66.548, 70.993, 75.002, 79.843, 90.573}, + /*54*/ {67.673, 72.153, 76.192, 81.069, 91.872}, + /*55*/ {68.796, 73.311, 77.380, 82.292, 93.168}, + /*56*/ {69.919, 74.468, 78.567, 83.513, 94.461}, + /*57*/ {71.040, 75.624, 79.752, 84.733, 95.751}, + /*58*/ {72.160, 76.778, 80.936, 85.950, 97.039}, + /*59*/ {73.279, 77.931, 82.117, 87.166, 98.324}, + /*60*/ {74.397, 79.082, 83.298, 88.379, 99.607}, + /*61*/ {75.514, 80.232, 84.476, 89.591, 100.888}, + /*62*/ {76.630, 81.381, 85.654, 90.802, 102.166}, + /*63*/ {77.745, 82.529, 86.830, 92.010, 103.442}, + /*64*/ {78.860, 83.675, 88.004, 93.217, 104.716}, + /*65*/ {79.973, 84.821, 89.177, 94.422, 105.988}, + /*66*/ {81.085, 85.965, 90.349, 95.626, 107.258}, + /*67*/ {82.197, 87.108, 91.519, 96.828, 108.526}, + /*68*/ {83.308, 88.250, 92.689, 98.028, 109.791}, + /*69*/ {84.418, 89.391, 93.856, 99.228, 111.055}, + /*70*/ {85.527, 90.531, 95.023, 100.425, 112.317}, + /*71*/ {86.635, 91.670, 96.189, 101.621, 113.577}, + /*72*/ {87.743, 92.808, 97.353, 102.816, 114.835}, + /*73*/ {88.850, 93.945, 98.516, 104.010, 116.092}, + /*74*/ {89.956, 95.081, 99.678, 105.202, 117.346}, + /*75*/ {91.061, 96.217, 100.839, 106.393, 118.599}, + /*76*/ {92.166, 97.351, 101.999, 107.583, 119.850}, + /*77*/ {93.270, 98.484, 103.158, 108.771, 121.100}, + /*78*/ {94.374, 99.617, 104.316, 109.958, 122.348}, + /*79*/ {95.476, 100.749, 105.473, 111.144, 123.594}, + /*80*/ {96.578, 101.879, 106.629, 112.329, 124.839}, + /*81*/ {97.680, 103.010, 107.783, 113.512, 126.083}, + /*82*/ {98.780, 104.139, 108.937, 114.695, 127.324}, + /*83*/ {99.880, 105.267, 110.090, 115.876, 128.565}, + /*84*/ {100.980, 106.395, 111.242, 117.057, 129.804}, + /*85*/ {102.079, 107.522, 112.393, 118.236, 131.041}, + /*86*/ {103.177, 108.648, 113.544, 119.414, 132.277}, + /*87*/ {104.275, 109.773, 114.693, 120.591, 133.512}, + /*88*/ {105.372, 110.898, 115.841, 121.767, 134.746}, + /*89*/ {106.469, 112.022, 116.989, 122.942, 135.978}, + /*90*/ {107.565, 113.145, 118.136, 124.116, 137.208}, + /*91*/ {108.661, 114.268, 119.282, 125.289, 138.438}, + /*92*/ {109.756, 115.390, 120.427, 126.462, 139.666}, + /*93*/ {110.850, 116.511, 121.571, 127.633, 140.893}, + /*94*/ {111.944, 117.632, 122.715, 128.803, 142.119}, + /*95*/ {113.038, 118.752, 123.858, 129.973, 143.344}, + /*96*/ {114.131, 119.871, 125.000, 131.141, 144.567}, + /*97*/ {115.223, 120.990, 126.141, 132.309, 145.789}, + /*98*/ {116.315, 122.108, 127.282, 133.476, 147.010}, + /*99*/ {117.407, 123.225, 128.422, 134.642, 148.230}, + /*100*/ {118.498, 124.342, 129.561, 135.807, 149.449} + /**/}; + + // 0.90 0.95 0.975 0.99 0.999 + for (int i = 0; i < ABSL_ARRAYSIZE(data); i++) { + const double E = 0.0001; + EXPECT_NEAR(ChiSquarePValue(data[i][0], i + 1), 0.10, E) + << i << " " << data[i][0]; + EXPECT_NEAR(ChiSquarePValue(data[i][1], i + 1), 0.05, E) + << i << " " << data[i][1]; + EXPECT_NEAR(ChiSquarePValue(data[i][2], i + 1), 0.025, E) + << i << " " << data[i][2]; + EXPECT_NEAR(ChiSquarePValue(data[i][3], i + 1), 0.01, E) + << i << " " << data[i][3]; + EXPECT_NEAR(ChiSquarePValue(data[i][4], i + 1), 0.001, E) + << i << " " << data[i][4]; + + const double F = 0.1; + EXPECT_NEAR(ChiSquareValue(i + 1, 0.90), data[i][0], F) << i; + EXPECT_NEAR(ChiSquareValue(i + 1, 0.95), data[i][1], F) << i; + EXPECT_NEAR(ChiSquareValue(i + 1, 0.975), data[i][2], F) << i; + EXPECT_NEAR(ChiSquareValue(i + 1, 0.99), data[i][3], F) << i; + EXPECT_NEAR(ChiSquareValue(i + 1, 0.999), data[i][4], F) << i; + } +} + +TEST(ChiSquareTest, ChiSquareTwoIterator) { + // Test data from http://www.stat.yale.edu/Courses/1997-98/101/chigf.htm + // Null-hypothesis: This data is normally distributed. + const int counts[10] = {6, 6, 18, 33, 38, 38, 28, 21, 9, 3}; + const double expected[10] = {4.6, 8.8, 18.4, 30.0, 38.2, + 38.2, 30.0, 18.4, 8.8, 4.6}; + double chi_square = ChiSquare(std::begin(counts), std::end(counts), + std::begin(expected), std::end(expected)); + EXPECT_NEAR(chi_square, 2.69, 0.001); + + // Degrees of freedom: 10 bins. two estimated parameters. = 10 - 2 - 1. + const int dof = 7; + // The critical value of 7, 95% => 14.067 (see above test) + double p_value_05 = ChiSquarePValue(14.067, dof); + EXPECT_NEAR(p_value_05, 0.05, 0.001); // 95%-ile p-value + + double p_actual = ChiSquarePValue(chi_square, dof); + EXPECT_GT(p_actual, 0.05); // Accept the null hypothesis. +} + +TEST(ChiSquareTest, DiceRolls) { + // Assume we are testing 102 fair dice rolls. + // Null-hypothesis: This data is fairly distributed. + // + // The dof value of 4, @95% = 9.488 (see above test) + // The dof value of 5, @95% = 11.070 + const int rolls[6] = {22, 11, 17, 14, 20, 18}; + double sum = std::accumulate(std::begin(rolls), std::end(rolls), double{0}); + size_t n = std::distance(std::begin(rolls), std::end(rolls)); + + double a = ChiSquareWithExpected(std::begin(rolls), std::end(rolls), sum / n); + EXPECT_NEAR(a, 4.70588, 1e-5); + EXPECT_LT(a, ChiSquareValue(4, 0.95)); + + double p_a = ChiSquarePValue(a, 4); + EXPECT_NEAR(p_a, 0.318828, 1e-5); // Accept the null hypothesis. + + double b = ChiSquareWithExpected(std::begin(rolls), std::end(rolls), 17.0); + EXPECT_NEAR(b, 4.70588, 1e-5); + EXPECT_LT(b, ChiSquareValue(5, 0.95)); + + double p_b = ChiSquarePValue(b, 5); + EXPECT_NEAR(p_b, 0.4528180, 1e-5); // Accept the null hypothesis. +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/distribution_caller.h b/third_party/abseil_cpp/absl/random/internal/distribution_caller.h new file mode 100644 index 000000000000..fc81b787ebe2 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/distribution_caller.h @@ -0,0 +1,92 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_RANDOM_INTERNAL_DISTRIBUTION_CALLER_H_ +#define ABSL_RANDOM_INTERNAL_DISTRIBUTION_CALLER_H_ + +#include <utility> + +#include "absl/base/config.h" +#include "absl/base/internal/fast_type_id.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// DistributionCaller provides an opportunity to overload the general +// mechanism for calling a distribution, allowing for mock-RNG classes +// to intercept such calls. +template <typename URBG> +struct DistributionCaller { + // SFINAE to detect whether the URBG type includes a member matching + // bool InvokeMock(base_internal::FastTypeIdType, void*, void*). + // + // These live inside BitGenRef so that they have friend access + // to MockingBitGen. (see similar methods in DistributionCaller). + template <template <class...> class Trait, class AlwaysVoid, class... Args> + struct detector : std::false_type {}; + template <template <class...> class Trait, class... Args> + struct detector<Trait, absl::void_t<Trait<Args...>>, Args...> + : std::true_type {}; + + template <class T> + using invoke_mock_t = decltype(std::declval<T*>()->InvokeMock( + std::declval<::absl::base_internal::FastTypeIdType>(), + std::declval<void*>(), std::declval<void*>())); + + using HasInvokeMock = typename detector<invoke_mock_t, void, URBG>::type; + + // Default implementation of distribution caller. + template <typename DistrT, typename... Args> + static typename DistrT::result_type Impl(std::false_type, URBG* urbg, + Args&&... args) { + DistrT dist(std::forward<Args>(args)...); + return dist(*urbg); + } + + // Mock implementation of distribution caller. + // The underlying KeyT must match the KeyT constructed by MockOverloadSet. + template <typename DistrT, typename... Args> + static typename DistrT::result_type Impl(std::true_type, URBG* urbg, + Args&&... args) { + using ResultT = typename DistrT::result_type; + using ArgTupleT = std::tuple<absl::decay_t<Args>...>; + using KeyT = ResultT(DistrT, ArgTupleT); + + ArgTupleT arg_tuple(std::forward<Args>(args)...); + ResultT result; + if (!urbg->InvokeMock(::absl::base_internal::FastTypeId<KeyT>(), &arg_tuple, + &result)) { + auto dist = absl::make_from_tuple<DistrT>(arg_tuple); + result = dist(*urbg); + } + return result; + } + + // Default implementation of distribution caller. + template <typename DistrT, typename... Args> + static typename DistrT::result_type Call(URBG* urbg, Args&&... args) { + return Impl<DistrT, Args...>(HasInvokeMock{}, urbg, + std::forward<Args>(args)...); + } +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_DISTRIBUTION_CALLER_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/distribution_test_util.cc b/third_party/abseil_cpp/absl/random/internal/distribution_test_util.cc new file mode 100644 index 000000000000..e9005658c0a7 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/distribution_test_util.cc @@ -0,0 +1,418 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/distribution_test_util.h" + +#include <cassert> +#include <cmath> +#include <string> +#include <vector> + +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +#if defined(__EMSCRIPTEN__) +// Workaround __EMSCRIPTEN__ error: llvm_fma_f64 not found. +inline double fma(double x, double y, double z) { return (x * y) + z; } +#endif + +} // namespace + +DistributionMoments ComputeDistributionMoments( + absl::Span<const double> data_points) { + DistributionMoments result; + + // Compute m1 + for (double x : data_points) { + result.n++; + result.mean += x; + } + result.mean /= static_cast<double>(result.n); + + // Compute m2, m3, m4 + for (double x : data_points) { + double v = x - result.mean; + result.variance += v * v; + result.skewness += v * v * v; + result.kurtosis += v * v * v * v; + } + result.variance /= static_cast<double>(result.n - 1); + + result.skewness /= static_cast<double>(result.n); + result.skewness /= std::pow(result.variance, 1.5); + + result.kurtosis /= static_cast<double>(result.n); + result.kurtosis /= std::pow(result.variance, 2.0); + return result; + + // When validating the min/max count, the following confidence intervals may + // be of use: + // 3.291 * stddev = 99.9% CI + // 2.576 * stddev = 99% CI + // 1.96 * stddev = 95% CI + // 1.65 * stddev = 90% CI +} + +std::ostream& operator<<(std::ostream& os, const DistributionMoments& moments) { + return os << absl::StrFormat("mean=%f, stddev=%f, skewness=%f, kurtosis=%f", + moments.mean, std::sqrt(moments.variance), + moments.skewness, moments.kurtosis); +} + +double InverseNormalSurvival(double x) { + // inv_sf(u) = -sqrt(2) * erfinv(2u-1) + static constexpr double kSqrt2 = 1.4142135623730950488; + return -kSqrt2 * absl::random_internal::erfinv(2 * x - 1.0); +} + +bool Near(absl::string_view msg, double actual, double expected, double bound) { + assert(bound > 0.0); + double delta = fabs(expected - actual); + if (delta < bound) { + return true; + } + + std::string formatted = absl::StrCat( + msg, " actual=", actual, " expected=", expected, " err=", delta / bound); + ABSL_RAW_LOG(INFO, "%s", formatted.c_str()); + return false; +} + +// TODO(absl-team): Replace with an "ABSL_HAVE_SPECIAL_MATH" and try +// to use std::beta(). As of this writing P0226R1 is not implemented +// in libc++: http://libcxx.llvm.org/cxx1z_status.html +double beta(double p, double q) { + // Beta(x, y) = Gamma(x) * Gamma(y) / Gamma(x+y) + double lbeta = std::lgamma(p) + std::lgamma(q) - std::lgamma(p + q); + return std::exp(lbeta); +} + +// Approximation to inverse of the Error Function in double precision. +// (http://people.maths.ox.ac.uk/gilesm/files/gems_erfinv.pdf) +double erfinv(double x) { +#if !defined(__EMSCRIPTEN__) + using std::fma; +#endif + + double w = 0.0; + double p = 0.0; + w = -std::log((1.0 - x) * (1.0 + x)); + if (w < 6.250000) { + w = w - 3.125000; + p = -3.6444120640178196996e-21; + p = fma(p, w, -1.685059138182016589e-19); + p = fma(p, w, 1.2858480715256400167e-18); + p = fma(p, w, 1.115787767802518096e-17); + p = fma(p, w, -1.333171662854620906e-16); + p = fma(p, w, 2.0972767875968561637e-17); + p = fma(p, w, 6.6376381343583238325e-15); + p = fma(p, w, -4.0545662729752068639e-14); + p = fma(p, w, -8.1519341976054721522e-14); + p = fma(p, w, 2.6335093153082322977e-12); + p = fma(p, w, -1.2975133253453532498e-11); + p = fma(p, w, -5.4154120542946279317e-11); + p = fma(p, w, 1.051212273321532285e-09); + p = fma(p, w, -4.1126339803469836976e-09); + p = fma(p, w, -2.9070369957882005086e-08); + p = fma(p, w, 4.2347877827932403518e-07); + p = fma(p, w, -1.3654692000834678645e-06); + p = fma(p, w, -1.3882523362786468719e-05); + p = fma(p, w, 0.0001867342080340571352); + p = fma(p, w, -0.00074070253416626697512); + p = fma(p, w, -0.0060336708714301490533); + p = fma(p, w, 0.24015818242558961693); + p = fma(p, w, 1.6536545626831027356); + } else if (w < 16.000000) { + w = std::sqrt(w) - 3.250000; + p = 2.2137376921775787049e-09; + p = fma(p, w, 9.0756561938885390979e-08); + p = fma(p, w, -2.7517406297064545428e-07); + p = fma(p, w, 1.8239629214389227755e-08); + p = fma(p, w, 1.5027403968909827627e-06); + p = fma(p, w, -4.013867526981545969e-06); + p = fma(p, w, 2.9234449089955446044e-06); + p = fma(p, w, 1.2475304481671778723e-05); + p = fma(p, w, -4.7318229009055733981e-05); + p = fma(p, w, 6.8284851459573175448e-05); + p = fma(p, w, 2.4031110387097893999e-05); + p = fma(p, w, -0.0003550375203628474796); + p = fma(p, w, 0.00095328937973738049703); + p = fma(p, w, -0.0016882755560235047313); + p = fma(p, w, 0.0024914420961078508066); + p = fma(p, w, -0.0037512085075692412107); + p = fma(p, w, 0.005370914553590063617); + p = fma(p, w, 1.0052589676941592334); + p = fma(p, w, 3.0838856104922207635); + } else { + w = std::sqrt(w) - 5.000000; + p = -2.7109920616438573243e-11; + p = fma(p, w, -2.5556418169965252055e-10); + p = fma(p, w, 1.5076572693500548083e-09); + p = fma(p, w, -3.7894654401267369937e-09); + p = fma(p, w, 7.6157012080783393804e-09); + p = fma(p, w, -1.4960026627149240478e-08); + p = fma(p, w, 2.9147953450901080826e-08); + p = fma(p, w, -6.7711997758452339498e-08); + p = fma(p, w, 2.2900482228026654717e-07); + p = fma(p, w, -9.9298272942317002539e-07); + p = fma(p, w, 4.5260625972231537039e-06); + p = fma(p, w, -1.9681778105531670567e-05); + p = fma(p, w, 7.5995277030017761139e-05); + p = fma(p, w, -0.00021503011930044477347); + p = fma(p, w, -0.00013871931833623122026); + p = fma(p, w, 1.0103004648645343977); + p = fma(p, w, 4.8499064014085844221); + } + return p * x; +} + +namespace { + +// Direct implementation of AS63, BETAIN() +// https://www.jstor.org/stable/2346797?seq=3#page_scan_tab_contents. +// +// BETAIN(x, p, q, beta) +// x: the value of the upper limit x. +// p: the value of the parameter p. +// q: the value of the parameter q. +// beta: the value of ln B(p, q) +// +double BetaIncompleteImpl(const double x, const double p, const double q, + const double beta) { + if (p < (p + q) * x) { + // Incomplete beta function is symmetrical, so return the complement. + return 1. - BetaIncompleteImpl(1.0 - x, q, p, beta); + } + + double psq = p + q; + const double kErr = 1e-14; + const double xc = 1. - x; + const double pre = + std::exp(p * std::log(x) + (q - 1.) * std::log(xc) - beta) / p; + + double term = 1.; + double ai = 1.; + double result = 1.; + int ns = static_cast<int>(q + xc * psq); + + // Use the soper reduction forumla. + double rx = (ns == 0) ? x : x / xc; + double temp = q - ai; + for (;;) { + term = term * temp * rx / (p + ai); + result = result + term; + temp = std::fabs(term); + if (temp < kErr && temp < kErr * result) { + return result * pre; + } + ai = ai + 1.; + --ns; + if (ns >= 0) { + temp = q - ai; + if (ns == 0) { + rx = x; + } + } else { + temp = psq; + psq = psq + 1.; + } + } + + // NOTE: See also TOMS Alogrithm 708. + // http://www.netlib.org/toms/index.html + // + // NOTE: The NWSC library also includes BRATIO / ISUBX (p87) + // https://archive.org/details/DTIC_ADA261511/page/n75 +} + +// Direct implementation of AS109, XINBTA(p, q, beta, alpha) +// https://www.jstor.org/stable/2346798?read-now=1&seq=4#page_scan_tab_contents +// https://www.jstor.org/stable/2346887?seq=1#page_scan_tab_contents +// +// XINBTA(p, q, beta, alhpa) +// p: the value of the parameter p. +// q: the value of the parameter q. +// beta: the value of ln B(p, q) +// alpha: the value of the lower tail area. +// +double BetaIncompleteInvImpl(const double p, const double q, const double beta, + const double alpha) { + if (alpha < 0.5) { + // Inverse Incomplete beta function is symmetrical, return the complement. + return 1. - BetaIncompleteInvImpl(q, p, beta, 1. - alpha); + } + const double kErr = 1e-14; + double value = kErr; + + // Compute the initial estimate. + { + double r = std::sqrt(-std::log(alpha * alpha)); + double y = + r - fma(r, 0.27061, 2.30753) / fma(r, fma(r, 0.04481, 0.99229), 1.0); + if (p > 1. && q > 1.) { + r = (y * y - 3.) / 6.; + double s = 1. / (p + p - 1.); + double t = 1. / (q + q - 1.); + double h = 2. / s + t; + double w = + y * std::sqrt(h + r) / h - (t - s) * (r + 5. / 6. - t / (3. * h)); + value = p / (p + q * std::exp(w + w)); + } else { + r = q + q; + double t = 1.0 / (9. * q); + double u = 1.0 - t + y * std::sqrt(t); + t = r * (u * u * u); + if (t <= 0) { + value = 1.0 - std::exp((std::log((1.0 - alpha) * q) + beta) / q); + } else { + t = (4.0 * p + r - 2.0) / t; + if (t <= 1) { + value = std::exp((std::log(alpha * p) + beta) / p); + } else { + value = 1.0 - 2.0 / (t + 1.0); + } + } + } + } + + // Solve for x using a modified newton-raphson method using the function + // BetaIncomplete. + { + value = std::max(value, kErr); + value = std::min(value, 1.0 - kErr); + + const double r = 1.0 - p; + const double t = 1.0 - q; + double y; + double yprev = 0; + double sq = 1; + double prev = 1; + for (;;) { + if (value < 0 || value > 1.0) { + // Error case; value went infinite. + return std::numeric_limits<double>::infinity(); + } else if (value == 0 || value == 1) { + y = value; + } else { + y = BetaIncompleteImpl(value, p, q, beta); + if (!std::isfinite(y)) { + return y; + } + } + y = (y - alpha) * + std::exp(beta + r * std::log(value) + t * std::log(1.0 - value)); + if (y * yprev <= 0) { + prev = std::max(sq, std::numeric_limits<double>::min()); + } + double g = 1.0; + for (;;) { + const double adj = g * y; + const double adj_sq = adj * adj; + if (adj_sq >= prev) { + g = g / 3.0; + continue; + } + const double tx = value - adj; + if (tx < 0 || tx > 1) { + g = g / 3.0; + continue; + } + if (prev < kErr) { + return value; + } + if (y * y < kErr) { + return value; + } + if (tx == value) { + return value; + } + if (tx == 0 || tx == 1) { + g = g / 3.0; + continue; + } + value = tx; + yprev = y; + break; + } + } + } + + // NOTES: See also: Asymptotic inversion of the incomplete beta function. + // https://core.ac.uk/download/pdf/82140723.pdf + // + // NOTE: See the Boost library documentation as well: + // https://www.boost.org/doc/libs/1_52_0/libs/math/doc/sf_and_dist/html/math_toolkit/special/sf_beta/ibeta_function.html +} + +} // namespace + +double BetaIncomplete(const double x, const double p, const double q) { + // Error cases. + if (p < 0 || q < 0 || x < 0 || x > 1.0) { + return std::numeric_limits<double>::infinity(); + } + if (x == 0 || x == 1) { + return x; + } + // ln(Beta(p, q)) + double beta = std::lgamma(p) + std::lgamma(q) - std::lgamma(p + q); + return BetaIncompleteImpl(x, p, q, beta); +} + +double BetaIncompleteInv(const double p, const double q, const double alpha) { + // Error cases. + if (p < 0 || q < 0 || alpha < 0 || alpha > 1.0) { + return std::numeric_limits<double>::infinity(); + } + if (alpha == 0 || alpha == 1) { + return alpha; + } + // ln(Beta(p, q)) + double beta = std::lgamma(p) + std::lgamma(q) - std::lgamma(p + q); + return BetaIncompleteInvImpl(p, q, beta, alpha); +} + +// Given `num_trials` trials each with probability `p` of success, the +// probability of no failures is `p^k`. To ensure the probability of a failure +// is no more than `p_fail`, it must be that `p^k == 1 - p_fail`. This function +// computes `p` from that equation. +double RequiredSuccessProbability(const double p_fail, const int num_trials) { + double p = std::exp(std::log(1.0 - p_fail) / static_cast<double>(num_trials)); + ABSL_ASSERT(p > 0); + return p; +} + +double ZScore(double expected_mean, const DistributionMoments& moments) { + return (moments.mean - expected_mean) / + (std::sqrt(moments.variance) / + std::sqrt(static_cast<double>(moments.n))); +} + +double MaxErrorTolerance(double acceptance_probability) { + double one_sided_pvalue = 0.5 * (1.0 - acceptance_probability); + const double max_err = InverseNormalSurvival(one_sided_pvalue); + ABSL_ASSERT(max_err > 0); + return max_err; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/distribution_test_util.h b/third_party/abseil_cpp/absl/random/internal/distribution_test_util.h new file mode 100644 index 000000000000..6d94cf6c97bf --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/distribution_test_util.h @@ -0,0 +1,113 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_DISTRIBUTION_TEST_UTIL_H_ +#define ABSL_RANDOM_INTERNAL_DISTRIBUTION_TEST_UTIL_H_ + +#include <cstddef> +#include <iostream> +#include <vector> + +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +// NOTE: The functions in this file are test only, and are should not be used in +// non-test code. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// http://webspace.ship.edu/pgmarr/Geo441/Lectures/Lec%205%20-%20Normality%20Testing.pdf + +// Compute the 1st to 4th standard moments: +// mean, variance, skewness, and kurtosis. +// http://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm +struct DistributionMoments { + size_t n = 0; + double mean = 0.0; + double variance = 0.0; + double skewness = 0.0; + double kurtosis = 0.0; +}; +DistributionMoments ComputeDistributionMoments( + absl::Span<const double> data_points); + +std::ostream& operator<<(std::ostream& os, const DistributionMoments& moments); + +// Computes the Z-score for a set of data with the given distribution moments +// compared against `expected_mean`. +double ZScore(double expected_mean, const DistributionMoments& moments); + +// Returns the probability of success required for a single trial to ensure that +// after `num_trials` trials, the probability of at least one failure is no more +// than `p_fail`. +double RequiredSuccessProbability(double p_fail, int num_trials); + +// Computes the maximum distance from the mean tolerable, for Z-Tests that are +// expected to pass with `acceptance_probability`. Will terminate if the +// resulting tolerance is zero (due to passing in 0.0 for +// `acceptance_probability` or rounding errors). +// +// For example, +// MaxErrorTolerance(0.001) = 0.0 +// MaxErrorTolerance(0.5) = ~0.47 +// MaxErrorTolerance(1.0) = inf +double MaxErrorTolerance(double acceptance_probability); + +// Approximation to inverse of the Error Function in double precision. +// (http://people.maths.ox.ac.uk/gilesm/files/gems_erfinv.pdf) +double erfinv(double x); + +// Beta(p, q) = Gamma(p) * Gamma(q) / Gamma(p+q) +double beta(double p, double q); + +// The inverse of the normal survival function. +double InverseNormalSurvival(double x); + +// Returns whether actual is "near" expected, based on the bound. +bool Near(absl::string_view msg, double actual, double expected, double bound); + +// Implements the incomplete regularized beta function, AS63, BETAIN. +// https://www.jstor.org/stable/2346797 +// +// BetaIncomplete(x, p, q), where +// `x` is the value of the upper limit +// `p` is beta parameter p, `q` is beta parameter q. +// +// NOTE: This is a test-only function which is only accurate to within, at most, +// 1e-13 of the actual value. +// +double BetaIncomplete(double x, double p, double q); + +// Implements the inverse of the incomplete regularized beta function, AS109, +// XINBTA. +// https://www.jstor.org/stable/2346798 +// https://www.jstor.org/stable/2346887 +// +// BetaIncompleteInv(p, q, beta, alhpa) +// `p` is beta parameter p, `q` is beta parameter q. +// `alpha` is the value of the lower tail area. +// +// NOTE: This is a test-only function and, when successful, is only accurate to +// within ~1e-6 of the actual value; there are some cases where it diverges from +// the actual value by much more than that. The function uses Newton's method, +// and thus the runtime is highly variable. +double BetaIncompleteInv(double p, double q, double alpha); + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_DISTRIBUTION_TEST_UTIL_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/distribution_test_util_test.cc b/third_party/abseil_cpp/absl/random/internal/distribution_test_util_test.cc new file mode 100644 index 000000000000..c49d44fb4796 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/distribution_test_util_test.cc @@ -0,0 +1,193 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/distribution_test_util.h" + +#include "gtest/gtest.h" + +namespace { + +TEST(TestUtil, InverseErf) { + const struct { + const double z; + const double value; + } kErfInvTable[] = { + {0.0000001, 8.86227e-8}, + {0.00001, 8.86227e-6}, + {0.5, 0.4769362762044}, + {0.6, 0.5951160814499}, + {0.99999, 3.1234132743}, + {0.9999999, 3.7665625816}, + {0.999999944, 3.8403850690566985}, // = log((1-x) * (1+x)) =~ 16.004 + {0.999999999, 4.3200053849134452}, + }; + + for (const auto& data : kErfInvTable) { + auto value = absl::random_internal::erfinv(data.z); + + // Log using the Wolfram-alpha function name & parameters. + EXPECT_NEAR(value, data.value, 1e-8) + << " InverseErf[" << data.z << "] (expected=" << data.value << ") -> " + << value; + } +} + +const struct { + const double p; + const double q; + const double x; + const double alpha; +} kBetaTable[] = { + {0.5, 0.5, 0.01, 0.06376856085851985}, + {0.5, 0.5, 0.1, 0.2048327646991335}, + {0.5, 0.5, 1, 1}, + {1, 0.5, 0, 0}, + {1, 0.5, 0.01, 0.005012562893380045}, + {1, 0.5, 0.1, 0.0513167019494862}, + {1, 0.5, 0.5, 0.2928932188134525}, + {1, 1, 0.5, 0.5}, + {2, 2, 0.1, 0.028}, + {2, 2, 0.2, 0.104}, + {2, 2, 0.3, 0.216}, + {2, 2, 0.4, 0.352}, + {2, 2, 0.5, 0.5}, + {2, 2, 0.6, 0.648}, + {2, 2, 0.7, 0.784}, + {2, 2, 0.8, 0.896}, + {2, 2, 0.9, 0.972}, + {5.5, 5, 0.5, 0.4361908850559777}, + {10, 0.5, 0.9, 0.1516409096346979}, + {10, 5, 0.5, 0.08978271484375}, + {10, 5, 1, 1}, + {10, 10, 0.5, 0.5}, + {20, 5, 0.8, 0.4598773297575791}, + {20, 10, 0.6, 0.2146816102371739}, + {20, 10, 0.8, 0.9507364826957875}, + {20, 20, 0.5, 0.5}, + {20, 20, 0.6, 0.8979413687105918}, + {30, 10, 0.7, 0.2241297491808366}, + {30, 10, 0.8, 0.7586405487192086}, + {40, 20, 0.7, 0.7001783247477069}, + {1, 0.5, 0.1, 0.0513167019494862}, + {1, 0.5, 0.2, 0.1055728090000841}, + {1, 0.5, 0.3, 0.1633399734659245}, + {1, 0.5, 0.4, 0.2254033307585166}, + {1, 2, 0.2, 0.36}, + {1, 3, 0.2, 0.488}, + {1, 4, 0.2, 0.5904}, + {1, 5, 0.2, 0.67232}, + {2, 2, 0.3, 0.216}, + {3, 2, 0.3, 0.0837}, + {4, 2, 0.3, 0.03078}, + {5, 2, 0.3, 0.010935}, + + // These values test small & large points along the range of the Beta + // function. + // + // When selecting test points, remember that if BetaIncomplete(x, p, q) + // returns the same value to within the limits of precision over a large + // domain of the input, x, then BetaIncompleteInv(alpha, p, q) may return an + // essentially arbitrary value where BetaIncomplete(x, p, q) =~ alpha. + + // BetaRegularized[x, 0.00001, 0.00001], + // For x in {~0.001 ... ~0.999}, => ~0.5 + {1e-5, 1e-5, 1e-5, 0.4999424388184638311}, + {1e-5, 1e-5, (1.0 - 1e-8), 0.5000920948389232964}, + + // BetaRegularized[x, 0.00001, 10000]. + // For x in {~epsilon ... 1.0}, => ~1 + {1e-5, 1e5, 1e-6, 0.9999817708130066936}, + {1e-5, 1e5, (1.0 - 1e-7), 1.0}, + + // BetaRegularized[x, 10000, 0.00001]. + // For x in {0 .. 1-epsilon}, => ~0 + {1e5, 1e-5, 1e-6, 0}, + {1e5, 1e-5, (1.0 - 1e-6), 1.8229186993306369e-5}, +}; + +TEST(BetaTest, BetaIncomplete) { + for (const auto& data : kBetaTable) { + auto value = absl::random_internal::BetaIncomplete(data.x, data.p, data.q); + + // Log using the Wolfram-alpha function name & parameters. + EXPECT_NEAR(value, data.alpha, 1e-12) + << " BetaRegularized[" << data.x << ", " << data.p << ", " << data.q + << "] (expected=" << data.alpha << ") -> " << value; + } +} + +TEST(BetaTest, BetaIncompleteInv) { + for (const auto& data : kBetaTable) { + auto value = + absl::random_internal::BetaIncompleteInv(data.p, data.q, data.alpha); + + // Log using the Wolfram-alpha function name & parameters. + EXPECT_NEAR(value, data.x, 1e-6) + << " InverseBetaRegularized[" << data.alpha << ", " << data.p << ", " + << data.q << "] (expected=" << data.x << ") -> " << value; + } +} + +TEST(MaxErrorTolerance, MaxErrorTolerance) { + std::vector<std::pair<double, double>> cases = { + {0.0000001, 8.86227e-8 * 1.41421356237}, + {0.00001, 8.86227e-6 * 1.41421356237}, + {0.5, 0.4769362762044 * 1.41421356237}, + {0.6, 0.5951160814499 * 1.41421356237}, + {0.99999, 3.1234132743 * 1.41421356237}, + {0.9999999, 3.7665625816 * 1.41421356237}, + {0.999999944, 3.8403850690566985 * 1.41421356237}, + {0.999999999, 4.3200053849134452 * 1.41421356237}}; + for (auto entry : cases) { + EXPECT_NEAR(absl::random_internal::MaxErrorTolerance(entry.first), + entry.second, 1e-8); + } +} + +TEST(ZScore, WithSameMean) { + absl::random_internal::DistributionMoments m; + m.n = 100; + m.mean = 5; + m.variance = 1; + EXPECT_NEAR(absl::random_internal::ZScore(5, m), 0, 1e-12); + + m.n = 1; + m.mean = 0; + m.variance = 1; + EXPECT_NEAR(absl::random_internal::ZScore(0, m), 0, 1e-12); + + m.n = 10000; + m.mean = -5; + m.variance = 100; + EXPECT_NEAR(absl::random_internal::ZScore(-5, m), 0, 1e-12); +} + +TEST(ZScore, DifferentMean) { + absl::random_internal::DistributionMoments m; + m.n = 100; + m.mean = 5; + m.variance = 1; + EXPECT_NEAR(absl::random_internal::ZScore(4, m), 10, 1e-12); + + m.n = 1; + m.mean = 0; + m.variance = 1; + EXPECT_NEAR(absl::random_internal::ZScore(-1, m), 1, 1e-12); + + m.n = 10000; + m.mean = -5; + m.variance = 100; + EXPECT_NEAR(absl::random_internal::ZScore(-4, m), -10, 1e-12); +} +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/explicit_seed_seq.h b/third_party/abseil_cpp/absl/random/internal/explicit_seed_seq.h new file mode 100644 index 000000000000..6a743eaf46cc --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/explicit_seed_seq.h @@ -0,0 +1,91 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_EXPLICIT_SEED_SEQ_H_ +#define ABSL_RANDOM_INTERNAL_EXPLICIT_SEED_SEQ_H_ + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <initializer_list> +#include <iterator> +#include <vector> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// This class conforms to the C++ Standard "Seed Sequence" concept +// [rand.req.seedseq]. +// +// An "ExplicitSeedSeq" is meant to provide a conformant interface for +// forwarding pre-computed seed material to the constructor of a class +// conforming to the "Uniform Random Bit Generator" concept. This class makes no +// attempt to mutate the state provided by its constructor, and returns it +// directly via ExplicitSeedSeq::generate(). +// +// If this class is asked to generate more seed material than was provided to +// the constructor, then the remaining bytes will be filled with deterministic, +// nonrandom data. +class ExplicitSeedSeq { + public: + using result_type = uint32_t; + + ExplicitSeedSeq() : state_() {} + + // Copy and move both allowed. + ExplicitSeedSeq(const ExplicitSeedSeq& other) = default; + ExplicitSeedSeq& operator=(const ExplicitSeedSeq& other) = default; + ExplicitSeedSeq(ExplicitSeedSeq&& other) = default; + ExplicitSeedSeq& operator=(ExplicitSeedSeq&& other) = default; + + template <typename Iterator> + ExplicitSeedSeq(Iterator begin, Iterator end) { + for (auto it = begin; it != end; it++) { + state_.push_back(*it & 0xffffffff); + } + } + + template <typename T> + ExplicitSeedSeq(std::initializer_list<T> il) + : ExplicitSeedSeq(il.begin(), il.end()) {} + + size_t size() const { return state_.size(); } + + template <typename OutIterator> + void param(OutIterator out) const { + std::copy(std::begin(state_), std::end(state_), out); + } + + template <typename OutIterator> + void generate(OutIterator begin, OutIterator end) { + for (size_t index = 0; begin != end; begin++) { + *begin = state_.empty() ? 0 : state_[index++]; + if (index >= state_.size()) { + index = 0; + } + } + } + + protected: + std::vector<uint32_t> state_; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_EXPLICIT_SEED_SEQ_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/explicit_seed_seq_test.cc b/third_party/abseil_cpp/absl/random/internal/explicit_seed_seq_test.cc new file mode 100644 index 000000000000..a55ad73948b9 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/explicit_seed_seq_test.cc @@ -0,0 +1,204 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/explicit_seed_seq.h" + +#include <iterator> +#include <random> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/random/seed_sequences.h" + +namespace { + +template <typename Sseq> +bool ConformsToInterface() { + // Check that the SeedSequence can be default-constructed. + { Sseq default_constructed_seq; } + // Check that the SeedSequence can be constructed with two iterators. + { + uint32_t init_array[] = {1, 3, 5, 7, 9}; + Sseq iterator_constructed_seq(init_array, &init_array[5]); + } + // Check that the SeedSequence can be std::initializer_list-constructed. + { Sseq list_constructed_seq = {1, 3, 5, 7, 9, 11, 13}; } + // Check that param() and size() return state provided to constructor. + { + uint32_t init_array[] = {1, 2, 3, 4, 5}; + Sseq seq(init_array, &init_array[ABSL_ARRAYSIZE(init_array)]); + EXPECT_EQ(seq.size(), ABSL_ARRAYSIZE(init_array)); + + uint32_t state_array[ABSL_ARRAYSIZE(init_array)]; + seq.param(state_array); + + for (int i = 0; i < ABSL_ARRAYSIZE(state_array); i++) { + EXPECT_EQ(state_array[i], i + 1); + } + } + // Check for presence of generate() method. + { + Sseq seq; + uint32_t seeds[5]; + + seq.generate(seeds, &seeds[ABSL_ARRAYSIZE(seeds)]); + } + return true; +} +} // namespace + +TEST(SeedSequences, CheckInterfaces) { + // Control case + EXPECT_TRUE(ConformsToInterface<std::seed_seq>()); + + // Abseil classes + EXPECT_TRUE(ConformsToInterface<absl::random_internal::ExplicitSeedSeq>()); +} + +TEST(ExplicitSeedSeq, DefaultConstructorGeneratesZeros) { + const size_t kNumBlocks = 128; + + uint32_t outputs[kNumBlocks]; + absl::random_internal::ExplicitSeedSeq seq; + seq.generate(outputs, &outputs[kNumBlocks]); + + for (uint32_t& seed : outputs) { + EXPECT_EQ(seed, 0); + } +} + +TEST(ExplicitSeeqSeq, SeedMaterialIsForwardedIdentically) { + const size_t kNumBlocks = 128; + + uint32_t seed_material[kNumBlocks]; + std::random_device urandom{"/dev/urandom"}; + for (uint32_t& seed : seed_material) { + seed = urandom(); + } + absl::random_internal::ExplicitSeedSeq seq(seed_material, + &seed_material[kNumBlocks]); + + // Check that output is same as seed-material provided to constructor. + { + const size_t kNumGenerated = kNumBlocks / 2; + uint32_t outputs[kNumGenerated]; + seq.generate(outputs, &outputs[kNumGenerated]); + for (size_t i = 0; i < kNumGenerated; i++) { + EXPECT_EQ(outputs[i], seed_material[i]); + } + } + // Check that SeedSequence is stateless between invocations: Despite the last + // invocation of generate() only consuming half of the input-entropy, the same + // entropy will be recycled for the next invocation. + { + const size_t kNumGenerated = kNumBlocks; + uint32_t outputs[kNumGenerated]; + seq.generate(outputs, &outputs[kNumGenerated]); + for (size_t i = 0; i < kNumGenerated; i++) { + EXPECT_EQ(outputs[i], seed_material[i]); + } + } + // Check that when more seed-material is asked for than is provided, nonzero + // values are still written. + { + const size_t kNumGenerated = kNumBlocks * 2; + uint32_t outputs[kNumGenerated]; + seq.generate(outputs, &outputs[kNumGenerated]); + for (size_t i = 0; i < kNumGenerated; i++) { + EXPECT_EQ(outputs[i], seed_material[i % kNumBlocks]); + } + } +} + +TEST(ExplicitSeedSeq, CopyAndMoveConstructors) { + using testing::Each; + using testing::Eq; + using testing::Not; + using testing::Pointwise; + + uint32_t entropy[4]; + std::random_device urandom("/dev/urandom"); + for (uint32_t& entry : entropy) { + entry = urandom(); + } + absl::random_internal::ExplicitSeedSeq seq_from_entropy(std::begin(entropy), + std::end(entropy)); + // Copy constructor. + { + absl::random_internal::ExplicitSeedSeq seq_copy(seq_from_entropy); + EXPECT_EQ(seq_copy.size(), seq_from_entropy.size()); + + std::vector<uint32_t> seeds_1; + seeds_1.resize(1000, 0); + std::vector<uint32_t> seeds_2; + seeds_2.resize(1000, 1); + + seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); + seq_copy.generate(seeds_2.begin(), seeds_2.end()); + + EXPECT_THAT(seeds_1, Pointwise(Eq(), seeds_2)); + } + // Assignment operator. + { + for (uint32_t& entry : entropy) { + entry = urandom(); + } + absl::random_internal::ExplicitSeedSeq another_seq(std::begin(entropy), + std::end(entropy)); + + std::vector<uint32_t> seeds_1; + seeds_1.resize(1000, 0); + std::vector<uint32_t> seeds_2; + seeds_2.resize(1000, 0); + + seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); + another_seq.generate(seeds_2.begin(), seeds_2.end()); + + // Assert precondition: Sequences generated by seed-sequences are not equal. + EXPECT_THAT(seeds_1, Not(Pointwise(Eq(), seeds_2))); + + // Apply the assignment-operator. + another_seq = seq_from_entropy; + + // Re-generate seeds. + seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); + another_seq.generate(seeds_2.begin(), seeds_2.end()); + + // Seeds generated by seed-sequences should now be equal. + EXPECT_THAT(seeds_1, Pointwise(Eq(), seeds_2)); + } + // Move constructor. + { + // Get seeds from seed-sequence constructed from entropy. + std::vector<uint32_t> seeds_1; + seeds_1.resize(1000, 0); + seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); + + // Apply move-constructor move the sequence to another instance. + absl::random_internal::ExplicitSeedSeq moved_seq( + std::move(seq_from_entropy)); + std::vector<uint32_t> seeds_2; + seeds_2.resize(1000, 1); + moved_seq.generate(seeds_2.begin(), seeds_2.end()); + // Verify that seeds produced by moved-instance are the same as original. + EXPECT_THAT(seeds_1, Pointwise(Eq(), seeds_2)); + + // Verify that the moved-from instance now behaves like a + // default-constructed instance. + EXPECT_EQ(seq_from_entropy.size(), 0); + seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); + EXPECT_THAT(seeds_1, Each(Eq(0))); + } +} diff --git a/third_party/abseil_cpp/absl/random/internal/fast_uniform_bits.h b/third_party/abseil_cpp/absl/random/internal/fast_uniform_bits.h new file mode 100644 index 000000000000..425aaf7d830c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/fast_uniform_bits.h @@ -0,0 +1,268 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_FAST_UNIFORM_BITS_H_ +#define ABSL_RANDOM_INTERNAL_FAST_UNIFORM_BITS_H_ + +#include <cstddef> +#include <cstdint> +#include <limits> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +// Returns true if the input value is zero or a power of two. Useful for +// determining if the range of output values in a URBG +template <typename UIntType> +constexpr bool IsPowerOfTwoOrZero(UIntType n) { + return (n == 0) || ((n & (n - 1)) == 0); +} + +// Computes the length of the range of values producible by the URBG, or returns +// zero if that would encompass the entire range of representable values in +// URBG::result_type. +template <typename URBG> +constexpr typename URBG::result_type RangeSize() { + using result_type = typename URBG::result_type; + static_assert((URBG::max)() != (URBG::min)(), "URBG range cannot be 0."); + return ((URBG::max)() == (std::numeric_limits<result_type>::max)() && + (URBG::min)() == std::numeric_limits<result_type>::lowest()) + ? result_type{0} + : ((URBG::max)() - (URBG::min)() + result_type{1}); +} + +// Computes the floor of the log. (i.e., std::floor(std::log2(N)); +template <typename UIntType> +constexpr UIntType IntegerLog2(UIntType n) { + return (n <= 1) ? 0 : 1 + IntegerLog2(n >> 1); +} + +// Returns the number of bits of randomness returned through +// `PowerOfTwoVariate(urbg)`. +template <typename URBG> +constexpr size_t NumBits() { + return RangeSize<URBG>() == 0 + ? std::numeric_limits<typename URBG::result_type>::digits + : IntegerLog2(RangeSize<URBG>()); +} + +// Given a shift value `n`, constructs a mask with exactly the low `n` bits set. +// If `n == 0`, all bits are set. +template <typename UIntType> +constexpr UIntType MaskFromShift(size_t n) { + return ((n % std::numeric_limits<UIntType>::digits) == 0) + ? ~UIntType{0} + : (UIntType{1} << n) - UIntType{1}; +} + +// Tags used to dispatch FastUniformBits::generate to the simple or more complex +// entropy extraction algorithm. +struct SimplifiedLoopTag {}; +struct RejectionLoopTag {}; + +// FastUniformBits implements a fast path to acquire uniform independent bits +// from a type which conforms to the [rand.req.urbg] concept. +// Parameterized by: +// `UIntType`: the result (output) type +// +// The std::independent_bits_engine [rand.adapt.ibits] adaptor can be +// instantiated from an existing generator through a copy or a move. It does +// not, however, facilitate the production of pseudorandom bits from an un-owned +// generator that will outlive the std::independent_bits_engine instance. +template <typename UIntType = uint64_t> +class FastUniformBits { + public: + using result_type = UIntType; + + static constexpr result_type(min)() { return 0; } + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + template <typename URBG> + result_type operator()(URBG& g); // NOLINT(runtime/references) + + private: + static_assert(std::is_unsigned<UIntType>::value, + "Class-template FastUniformBits<> must be parameterized using " + "an unsigned type."); + + // Generate() generates a random value, dispatched on whether + // the underlying URBG must use rejection sampling to generate a value, + // or whether a simplified loop will suffice. + template <typename URBG> + result_type Generate(URBG& g, // NOLINT(runtime/references) + SimplifiedLoopTag); + + template <typename URBG> + result_type Generate(URBG& g, // NOLINT(runtime/references) + RejectionLoopTag); +}; + +template <typename UIntType> +template <typename URBG> +typename FastUniformBits<UIntType>::result_type +FastUniformBits<UIntType>::operator()(URBG& g) { // NOLINT(runtime/references) + // kRangeMask is the mask used when sampling variates from the URBG when the + // width of the URBG range is not a power of 2. + // Y = (2 ^ kRange) - 1 + static_assert((URBG::max)() > (URBG::min)(), + "URBG::max and URBG::min may not be equal."); + + using tag = absl::conditional_t<IsPowerOfTwoOrZero(RangeSize<URBG>()), + SimplifiedLoopTag, RejectionLoopTag>; + return Generate(g, tag{}); +} + +template <typename UIntType> +template <typename URBG> +typename FastUniformBits<UIntType>::result_type +FastUniformBits<UIntType>::Generate(URBG& g, // NOLINT(runtime/references) + SimplifiedLoopTag) { + // The simplified version of FastUniformBits works only on URBGs that have + // a range that is a power of 2. In this case we simply loop and shift without + // attempting to balance the bits across calls. + static_assert(IsPowerOfTwoOrZero(RangeSize<URBG>()), + "incorrect Generate tag for URBG instance"); + + static constexpr size_t kResultBits = + std::numeric_limits<result_type>::digits; + static constexpr size_t kUrbgBits = NumBits<URBG>(); + static constexpr size_t kIters = + (kResultBits / kUrbgBits) + (kResultBits % kUrbgBits != 0); + static constexpr size_t kShift = (kIters == 1) ? 0 : kUrbgBits; + static constexpr auto kMin = (URBG::min)(); + + result_type r = static_cast<result_type>(g() - kMin); + for (size_t n = 1; n < kIters; ++n) { + r = (r << kShift) + static_cast<result_type>(g() - kMin); + } + return r; +} + +template <typename UIntType> +template <typename URBG> +typename FastUniformBits<UIntType>::result_type +FastUniformBits<UIntType>::Generate(URBG& g, // NOLINT(runtime/references) + RejectionLoopTag) { + static_assert(!IsPowerOfTwoOrZero(RangeSize<URBG>()), + "incorrect Generate tag for URBG instance"); + using urbg_result_type = typename URBG::result_type; + + // See [rand.adapt.ibits] for more details on the constants calculated below. + // + // It is preferable to use roughly the same number of bits from each generator + // call, however this is only possible when the number of bits provided by the + // URBG is a divisor of the number of bits in `result_type`. In all other + // cases, the number of bits used cannot always be the same, but it can be + // guaranteed to be off by at most 1. Thus we run two loops, one with a + // smaller bit-width size (`kSmallWidth`) and one with a larger width size + // (satisfying `kLargeWidth == kSmallWidth + 1`). The loops are run + // `kSmallIters` and `kLargeIters` times respectively such + // that + // + // `kResultBits == kSmallIters * kSmallBits + // + kLargeIters * kLargeBits` + // + // where `kResultBits` is the total number of bits in `result_type`. + // + static constexpr size_t kResultBits = + std::numeric_limits<result_type>::digits; // w + static constexpr urbg_result_type kUrbgRange = RangeSize<URBG>(); // R + static constexpr size_t kUrbgBits = NumBits<URBG>(); // m + + // compute the initial estimate of the bits used. + // [rand.adapt.ibits] 2 (c) + static constexpr size_t kA = // ceil(w/m) + (kResultBits / kUrbgBits) + ((kResultBits % kUrbgBits) != 0); // n' + + static constexpr size_t kABits = kResultBits / kA; // w0' + static constexpr urbg_result_type kARejection = + ((kUrbgRange >> kABits) << kABits); // y0' + + // refine the selection to reduce the rejection frequency. + static constexpr size_t kTotalIters = + ((kUrbgRange - kARejection) <= (kARejection / kA)) ? kA : (kA + 1); // n + + // [rand.adapt.ibits] 2 (b) + static constexpr size_t kSmallIters = + kTotalIters - (kResultBits % kTotalIters); // n0 + static constexpr size_t kSmallBits = kResultBits / kTotalIters; // w0 + static constexpr urbg_result_type kSmallRejection = + ((kUrbgRange >> kSmallBits) << kSmallBits); // y0 + + static constexpr size_t kLargeBits = kSmallBits + 1; // w0+1 + static constexpr urbg_result_type kLargeRejection = + ((kUrbgRange >> kLargeBits) << kLargeBits); // y1 + + // + // Because `kLargeBits == kSmallBits + 1`, it follows that + // + // `kResultBits == kSmallIters * kSmallBits + kLargeIters` + // + // and therefore + // + // `kLargeIters == kTotalWidth % kSmallWidth` + // + // Intuitively, each iteration with the large width accounts for one unit + // of the remainder when `kTotalWidth` is divided by `kSmallWidth`. As + // mentioned above, if the URBG width is a divisor of `kTotalWidth`, then + // there would be no need for any large iterations (i.e., one loop would + // suffice), and indeed, in this case, `kLargeIters` would be zero. + static_assert(kResultBits == kSmallIters * kSmallBits + + (kTotalIters - kSmallIters) * kLargeBits, + "Error in looping constant calculations."); + + // The small shift is essentially small bits, but due to the potential + // of generating a smaller result_type from a larger urbg type, the actual + // shift might be 0. + static constexpr size_t kSmallShift = kSmallBits % kResultBits; + static constexpr auto kSmallMask = + MaskFromShift<urbg_result_type>(kSmallShift); + static constexpr size_t kLargeShift = kLargeBits % kResultBits; + static constexpr auto kLargeMask = + MaskFromShift<urbg_result_type>(kLargeShift); + + static constexpr auto kMin = (URBG::min)(); + + result_type s = 0; + for (size_t n = 0; n < kSmallIters; ++n) { + urbg_result_type v; + do { + v = g() - kMin; + } while (v >= kSmallRejection); + + s = (s << kSmallShift) + static_cast<result_type>(v & kSmallMask); + } + + for (size_t n = kSmallIters; n < kTotalIters; ++n) { + urbg_result_type v; + do { + v = g() - kMin; + } while (v >= kLargeRejection); + + s = (s << kLargeShift) + static_cast<result_type>(v & kLargeMask); + } + return s; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_FAST_UNIFORM_BITS_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/fast_uniform_bits_test.cc b/third_party/abseil_cpp/absl/random/internal/fast_uniform_bits_test.cc new file mode 100644 index 000000000000..cee702df852d --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/fast_uniform_bits_test.cc @@ -0,0 +1,336 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/fast_uniform_bits.h" + +#include <random> + +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +template <typename IntType> +class FastUniformBitsTypedTest : public ::testing::Test {}; + +using IntTypes = ::testing::Types<uint8_t, uint16_t, uint32_t, uint64_t>; + +TYPED_TEST_SUITE(FastUniformBitsTypedTest, IntTypes); + +TYPED_TEST(FastUniformBitsTypedTest, BasicTest) { + using Limits = std::numeric_limits<TypeParam>; + using FastBits = FastUniformBits<TypeParam>; + + EXPECT_EQ(0, (FastBits::min)()); + EXPECT_EQ((Limits::max)(), (FastBits::max)()); + + constexpr int kIters = 10000; + std::random_device rd; + std::mt19937 gen(rd()); + FastBits fast; + for (int i = 0; i < kIters; i++) { + const auto v = fast(gen); + EXPECT_LE(v, (FastBits::max)()); + EXPECT_GE(v, (FastBits::min)()); + } +} + +template <typename UIntType, UIntType Lo, UIntType Hi, UIntType Val = Lo> +struct FakeUrbg { + using result_type = UIntType; + + FakeUrbg() = default; + explicit FakeUrbg(bool r) : reject(r) {} + + static constexpr result_type(max)() { return Hi; } + static constexpr result_type(min)() { return Lo; } + result_type operator()() { + // when reject is set, return Hi half the time. + return ((++calls % 2) == 1 && reject) ? Hi : Val; + } + + bool reject = false; + size_t calls = 0; +}; + +TEST(FastUniformBitsTest, IsPowerOfTwoOrZero) { + EXPECT_TRUE(IsPowerOfTwoOrZero(uint8_t{0})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint8_t{1})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint8_t{2})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint8_t{3})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint8_t{4})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint8_t{16})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint8_t{17})); + EXPECT_FALSE(IsPowerOfTwoOrZero((std::numeric_limits<uint8_t>::max)())); + + EXPECT_TRUE(IsPowerOfTwoOrZero(uint16_t{0})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint16_t{1})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint16_t{2})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint16_t{3})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint16_t{4})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint16_t{16})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint16_t{17})); + EXPECT_FALSE(IsPowerOfTwoOrZero((std::numeric_limits<uint16_t>::max)())); + + EXPECT_TRUE(IsPowerOfTwoOrZero(uint32_t{0})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint32_t{1})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint32_t{2})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint32_t{3})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint32_t{32})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint32_t{17})); + EXPECT_FALSE(IsPowerOfTwoOrZero((std::numeric_limits<uint32_t>::max)())); + + EXPECT_TRUE(IsPowerOfTwoOrZero(uint64_t{0})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint64_t{1})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint64_t{2})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint64_t{3})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint64_t{4})); + EXPECT_TRUE(IsPowerOfTwoOrZero(uint64_t{64})); + EXPECT_FALSE(IsPowerOfTwoOrZero(uint64_t{17})); + EXPECT_FALSE(IsPowerOfTwoOrZero((std::numeric_limits<uint64_t>::max)())); +} + +TEST(FastUniformBitsTest, IntegerLog2) { + EXPECT_EQ(0, IntegerLog2(uint16_t{0})); + EXPECT_EQ(0, IntegerLog2(uint16_t{1})); + EXPECT_EQ(1, IntegerLog2(uint16_t{2})); + EXPECT_EQ(1, IntegerLog2(uint16_t{3})); + EXPECT_EQ(2, IntegerLog2(uint16_t{4})); + EXPECT_EQ(2, IntegerLog2(uint16_t{5})); + EXPECT_EQ(2, IntegerLog2(uint16_t{7})); + EXPECT_EQ(3, IntegerLog2(uint16_t{8})); + EXPECT_EQ(63, IntegerLog2((std::numeric_limits<uint64_t>::max)())); +} + +TEST(FastUniformBitsTest, RangeSize) { + EXPECT_EQ(2, (RangeSize<FakeUrbg<uint8_t, 0, 1>>())); + EXPECT_EQ(3, (RangeSize<FakeUrbg<uint8_t, 0, 2>>())); + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint8_t, 0, 3>>())); + // EXPECT_EQ(0, (RangeSize<FakeUrbg<uint8_t, 2, 2>>())); + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint8_t, 2, 5>>())); + EXPECT_EQ(5, (RangeSize<FakeUrbg<uint8_t, 2, 6>>())); + EXPECT_EQ(9, (RangeSize<FakeUrbg<uint8_t, 2, 10>>())); + EXPECT_EQ( + 0, (RangeSize< + FakeUrbg<uint8_t, 0, (std::numeric_limits<uint8_t>::max)()>>())); + + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint16_t, 0, 3>>())); + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint16_t, 2, 5>>())); + EXPECT_EQ(5, (RangeSize<FakeUrbg<uint16_t, 2, 6>>())); + EXPECT_EQ(18, (RangeSize<FakeUrbg<uint16_t, 1000, 1017>>())); + EXPECT_EQ( + 0, (RangeSize< + FakeUrbg<uint16_t, 0, (std::numeric_limits<uint16_t>::max)()>>())); + + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint32_t, 0, 3>>())); + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint32_t, 2, 5>>())); + EXPECT_EQ(5, (RangeSize<FakeUrbg<uint32_t, 2, 6>>())); + EXPECT_EQ(18, (RangeSize<FakeUrbg<uint32_t, 1000, 1017>>())); + EXPECT_EQ(0, (RangeSize<FakeUrbg<uint32_t, 0, 0xffffffff>>())); + EXPECT_EQ(0xffffffff, (RangeSize<FakeUrbg<uint32_t, 1, 0xffffffff>>())); + EXPECT_EQ(0xfffffffe, (RangeSize<FakeUrbg<uint32_t, 1, 0xfffffffe>>())); + EXPECT_EQ(0xfffffffd, (RangeSize<FakeUrbg<uint32_t, 2, 0xfffffffe>>())); + EXPECT_EQ( + 0, (RangeSize< + FakeUrbg<uint32_t, 0, (std::numeric_limits<uint32_t>::max)()>>())); + + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint64_t, 0, 3>>())); + EXPECT_EQ(4, (RangeSize<FakeUrbg<uint64_t, 2, 5>>())); + EXPECT_EQ(5, (RangeSize<FakeUrbg<uint64_t, 2, 6>>())); + EXPECT_EQ(18, (RangeSize<FakeUrbg<uint64_t, 1000, 1017>>())); + EXPECT_EQ(0x100000000, (RangeSize<FakeUrbg<uint64_t, 0, 0xffffffff>>())); + EXPECT_EQ(0xffffffff, (RangeSize<FakeUrbg<uint64_t, 1, 0xffffffff>>())); + EXPECT_EQ(0xfffffffe, (RangeSize<FakeUrbg<uint64_t, 1, 0xfffffffe>>())); + EXPECT_EQ(0xfffffffd, (RangeSize<FakeUrbg<uint64_t, 2, 0xfffffffe>>())); + EXPECT_EQ(0, (RangeSize<FakeUrbg<uint64_t, 0, 0xffffffffffffffff>>())); + EXPECT_EQ(0xffffffffffffffff, + (RangeSize<FakeUrbg<uint64_t, 1, 0xffffffffffffffff>>())); + EXPECT_EQ(0xfffffffffffffffe, + (RangeSize<FakeUrbg<uint64_t, 1, 0xfffffffffffffffe>>())); + EXPECT_EQ(0xfffffffffffffffd, + (RangeSize<FakeUrbg<uint64_t, 2, 0xfffffffffffffffe>>())); + EXPECT_EQ( + 0, (RangeSize< + FakeUrbg<uint64_t, 0, (std::numeric_limits<uint64_t>::max)()>>())); +} + +// The constants need to be choosen so that an infinite rejection loop doesn't +// happen... +using Urng1_5bit = FakeUrbg<uint8_t, 0, 2, 0>; // ~1.5 bits (range 3) +using Urng4bits = FakeUrbg<uint8_t, 1, 0x10, 2>; +using Urng22bits = FakeUrbg<uint32_t, 0, 0x3fffff, 0x301020>; +using Urng31bits = FakeUrbg<uint32_t, 1, 0xfffffffe, 0x60070f03>; // ~31.9 bits +using Urng32bits = FakeUrbg<uint32_t, 0, 0xffffffff, 0x74010f01>; +using Urng33bits = + FakeUrbg<uint64_t, 1, 0x1ffffffff, 0x013301033>; // ~32.9 bits +using Urng63bits = FakeUrbg<uint64_t, 1, 0xfffffffffffffffe, + 0xfedcba9012345678>; // ~63.9 bits +using Urng64bits = + FakeUrbg<uint64_t, 0, 0xffffffffffffffff, 0x123456780fedcba9>; + +TEST(FastUniformBitsTest, OutputsUpTo32Bits) { + // Tests that how values are composed; the single-bit deltas should be spread + // across each invocation. + Urng1_5bit urng1_5; + Urng4bits urng4; + Urng22bits urng22; + Urng31bits urng31; + Urng32bits urng32; + Urng33bits urng33; + Urng63bits urng63; + Urng64bits urng64; + + // 8-bit types + { + FastUniformBits<uint8_t> fast8; + EXPECT_EQ(0x0, fast8(urng1_5)); + EXPECT_EQ(0x11, fast8(urng4)); + EXPECT_EQ(0x20, fast8(urng22)); + EXPECT_EQ(0x2, fast8(urng31)); + EXPECT_EQ(0x1, fast8(urng32)); + EXPECT_EQ(0x32, fast8(urng33)); + EXPECT_EQ(0x77, fast8(urng63)); + EXPECT_EQ(0xa9, fast8(urng64)); + } + + // 16-bit types + { + FastUniformBits<uint16_t> fast16; + EXPECT_EQ(0x0, fast16(urng1_5)); + EXPECT_EQ(0x1111, fast16(urng4)); + EXPECT_EQ(0x1020, fast16(urng22)); + EXPECT_EQ(0x0f02, fast16(urng31)); + EXPECT_EQ(0x0f01, fast16(urng32)); + EXPECT_EQ(0x1032, fast16(urng33)); + EXPECT_EQ(0x5677, fast16(urng63)); + EXPECT_EQ(0xcba9, fast16(urng64)); + } + + // 32-bit types + { + FastUniformBits<uint32_t> fast32; + EXPECT_EQ(0x0, fast32(urng1_5)); + EXPECT_EQ(0x11111111, fast32(urng4)); + EXPECT_EQ(0x08301020, fast32(urng22)); + EXPECT_EQ(0x0f020f02, fast32(urng31)); + EXPECT_EQ(0x74010f01, fast32(urng32)); + EXPECT_EQ(0x13301032, fast32(urng33)); + EXPECT_EQ(0x12345677, fast32(urng63)); + EXPECT_EQ(0x0fedcba9, fast32(urng64)); + } +} + +TEST(FastUniformBitsTest, Outputs64Bits) { + // Tests that how values are composed; the single-bit deltas should be spread + // across each invocation. + FastUniformBits<uint64_t> fast64; + + { + FakeUrbg<uint8_t, 0, 1, 0> urng0; + FakeUrbg<uint8_t, 0, 1, 1> urng1; + Urng4bits urng4; + Urng22bits urng22; + Urng31bits urng31; + Urng32bits urng32; + Urng33bits urng33; + Urng63bits urng63; + Urng64bits urng64; + + // somewhat degenerate cases only create a single bit. + EXPECT_EQ(0x0, fast64(urng0)); + EXPECT_EQ(64, urng0.calls); + EXPECT_EQ(0xffffffffffffffff, fast64(urng1)); + EXPECT_EQ(64, urng1.calls); + + // less degenerate cases. + EXPECT_EQ(0x1111111111111111, fast64(urng4)); + EXPECT_EQ(16, urng4.calls); + EXPECT_EQ(0x01020c0408301020, fast64(urng22)); + EXPECT_EQ(3, urng22.calls); + EXPECT_EQ(0x387811c3c0870f02, fast64(urng31)); + EXPECT_EQ(3, urng31.calls); + EXPECT_EQ(0x74010f0174010f01, fast64(urng32)); + EXPECT_EQ(2, urng32.calls); + EXPECT_EQ(0x808194040cb01032, fast64(urng33)); + EXPECT_EQ(3, urng33.calls); + EXPECT_EQ(0x1234567712345677, fast64(urng63)); + EXPECT_EQ(2, urng63.calls); + EXPECT_EQ(0x123456780fedcba9, fast64(urng64)); + EXPECT_EQ(1, urng64.calls); + } + + // The 1.5 bit case is somewhat interesting in that the algorithm refinement + // causes one extra small sample. Comments here reference the names used in + // [rand.adapt.ibits] that correspond to this case. + { + Urng1_5bit urng1_5; + + // w = 64 + // R = 3 + // m = 1 + // n' = 64 + // w0' = 1 + // y0' = 2 + // n = (1 <= 0) > 64 : 65 = 65 + // n0 = 65 - (64%65) = 1 + // n1 = 64 + // w0 = 0 + // y0 = 3 + // w1 = 1 + // y1 = 2 + EXPECT_EQ(0x0, fast64(urng1_5)); + EXPECT_EQ(65, urng1_5.calls); + } + + // Validate rejections for non-power-of-2 cases. + { + Urng1_5bit urng1_5(true); + Urng31bits urng31(true); + Urng33bits urng33(true); + Urng63bits urng63(true); + + // For 1.5 bits, there would be 1+2*64, except the first + // value was accepted and shifted off the end. + EXPECT_EQ(0, fast64(urng1_5)); + EXPECT_EQ(128, urng1_5.calls); + EXPECT_EQ(0x387811c3c0870f02, fast64(urng31)); + EXPECT_EQ(6, urng31.calls); + EXPECT_EQ(0x808194040cb01032, fast64(urng33)); + EXPECT_EQ(6, urng33.calls); + EXPECT_EQ(0x1234567712345677, fast64(urng63)); + EXPECT_EQ(4, urng63.calls); + } +} + +TEST(FastUniformBitsTest, URBG32bitRegression) { + // Validate with deterministic 32-bit std::minstd_rand + // to ensure that operator() performs as expected. + + EXPECT_EQ(2147483646, RangeSize<std::minstd_rand>()); + EXPECT_EQ(30, IntegerLog2(RangeSize<std::minstd_rand>())); + + std::minstd_rand gen(1); + FastUniformBits<uint64_t> fast64; + + EXPECT_EQ(0x05e47095f8791f45, fast64(gen)); + EXPECT_EQ(0x028be17e3c07c122, fast64(gen)); + EXPECT_EQ(0x55d2847c1626e8c2, fast64(gen)); +} + +} // namespace +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/fastmath.h b/third_party/abseil_cpp/absl/random/internal/fastmath.h new file mode 100644 index 000000000000..6baeb5a7c917 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/fastmath.h @@ -0,0 +1,74 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_FASTMATH_H_ +#define ABSL_RANDOM_INTERNAL_FASTMATH_H_ + +// This file contains fast math functions (bitwise ops as well as some others) +// which are implementation details of various absl random number distributions. + +#include <cassert> +#include <cmath> +#include <cstdint> + +#include "absl/base/internal/bits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Returns the position of the first bit set. +inline int LeadingSetBit(uint64_t n) { + return 64 - base_internal::CountLeadingZeros64(n); +} + +// Compute log2(n) using integer operations. +// While std::log2 is more accurate than std::log(n) / std::log(2), for +// very large numbers--those close to std::numeric_limits<uint64_t>::max() - 2, +// for instance--std::log2 rounds up rather than down, which introduces +// definite skew in the results. +inline int IntLog2Floor(uint64_t n) { + return (n <= 1) ? 0 : (63 - base_internal::CountLeadingZeros64(n)); +} +inline int IntLog2Ceil(uint64_t n) { + return (n <= 1) ? 0 : (64 - base_internal::CountLeadingZeros64(n - 1)); +} + +inline double StirlingLogFactorial(double n) { + assert(n >= 1); + // Using Stirling's approximation. + constexpr double kLog2PI = 1.83787706640934548356; + const double logn = std::log(n); + const double ninv = 1.0 / static_cast<double>(n); + return n * logn - n + 0.5 * (kLog2PI + logn) + (1.0 / 12.0) * ninv - + (1.0 / 360.0) * ninv * ninv * ninv; +} + +// Rotate value right. +// +// We only implement the uint32_t / uint64_t versions because +// 1) those are the only ones we use, and +// 2) those are the only ones where clang detects the rotate idiom correctly. +inline constexpr uint32_t rotr(uint32_t value, uint8_t bits) { + return (value >> (bits & 31)) | (value << ((-bits) & 31)); +} +inline constexpr uint64_t rotr(uint64_t value, uint8_t bits) { + return (value >> (bits & 63)) | (value << ((-bits) & 63)); +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_FASTMATH_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/fastmath_test.cc b/third_party/abseil_cpp/absl/random/internal/fastmath_test.cc new file mode 100644 index 000000000000..65859c25de4c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/fastmath_test.cc @@ -0,0 +1,110 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/fastmath.h" + +#include "gtest/gtest.h" + +#if defined(__native_client__) || defined(__EMSCRIPTEN__) +// NACL has a less accurate implementation of std::log2 than most of +// the other platforms. For some values which should have integral results, +// sometimes NACL returns slightly larger values. +// +// The MUSL libc used by emscripten also has a similar bug. +#define ABSL_RANDOM_INACCURATE_LOG2 +#endif + +namespace { + +TEST(DistributionImplTest, LeadingSetBit) { + using absl::random_internal::LeadingSetBit; + constexpr uint64_t kZero = 0; + EXPECT_EQ(0, LeadingSetBit(kZero)); + EXPECT_EQ(64, LeadingSetBit(~kZero)); + + for (int index = 0; index < 64; index++) { + uint64_t x = static_cast<uint64_t>(1) << index; + EXPECT_EQ(index + 1, LeadingSetBit(x)) << index; + EXPECT_EQ(index + 1, LeadingSetBit(x + x - 1)) << index; + } +} + +TEST(FastMathTest, IntLog2FloorTest) { + using absl::random_internal::IntLog2Floor; + constexpr uint64_t kZero = 0; + EXPECT_EQ(0, IntLog2Floor(0)); // boundary. return 0. + EXPECT_EQ(0, IntLog2Floor(1)); + EXPECT_EQ(1, IntLog2Floor(2)); + EXPECT_EQ(63, IntLog2Floor(~kZero)); + + // A boundary case: Converting 0xffffffffffffffff requires > 53 + // bits of precision, so the conversion to double rounds up, + // and the result of std::log2(x) > IntLog2Floor(x). + EXPECT_LT(IntLog2Floor(~kZero), static_cast<int>(std::log2(~kZero))); + + for (int i = 0; i < 64; i++) { + const uint64_t i_pow_2 = static_cast<uint64_t>(1) << i; + EXPECT_EQ(i, IntLog2Floor(i_pow_2)); + EXPECT_EQ(i, static_cast<int>(std::log2(i_pow_2))); + + uint64_t y = i_pow_2; + for (int j = i - 1; j > 0; --j) { + y = y | (i_pow_2 >> j); + EXPECT_EQ(i, IntLog2Floor(y)); + } + } +} + +TEST(FastMathTest, IntLog2CeilTest) { + using absl::random_internal::IntLog2Ceil; + constexpr uint64_t kZero = 0; + EXPECT_EQ(0, IntLog2Ceil(0)); // boundary. return 0. + EXPECT_EQ(0, IntLog2Ceil(1)); + EXPECT_EQ(1, IntLog2Ceil(2)); + EXPECT_EQ(64, IntLog2Ceil(~kZero)); + + // A boundary case: Converting 0xffffffffffffffff requires > 53 + // bits of precision, so the conversion to double rounds up, + // and the result of std::log2(x) > IntLog2Floor(x). + EXPECT_LE(IntLog2Ceil(~kZero), static_cast<int>(std::log2(~kZero))); + + for (int i = 0; i < 64; i++) { + const uint64_t i_pow_2 = static_cast<uint64_t>(1) << i; + EXPECT_EQ(i, IntLog2Ceil(i_pow_2)); +#ifndef ABSL_RANDOM_INACCURATE_LOG2 + EXPECT_EQ(i, static_cast<int>(std::ceil(std::log2(i_pow_2)))); +#endif + + uint64_t y = i_pow_2; + for (int j = i - 1; j > 0; --j) { + y = y | (i_pow_2 >> j); + EXPECT_EQ(i + 1, IntLog2Ceil(y)); + } + } +} + +TEST(FastMathTest, StirlingLogFactorial) { + using absl::random_internal::StirlingLogFactorial; + + EXPECT_NEAR(StirlingLogFactorial(1.0), 0, 1e-3); + EXPECT_NEAR(StirlingLogFactorial(1.50), 0.284683, 1e-3); + EXPECT_NEAR(StirlingLogFactorial(2.0), 0.69314718056, 1e-4); + + for (int i = 2; i < 50; i++) { + double d = static_cast<double>(i); + EXPECT_NEAR(StirlingLogFactorial(d), std::lgamma(d + 1), 3e-5); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/gaussian_distribution_gentables.cc b/third_party/abseil_cpp/absl/random/internal/gaussian_distribution_gentables.cc new file mode 100644 index 000000000000..a95333d55f55 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/gaussian_distribution_gentables.cc @@ -0,0 +1,143 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generates gaussian_distribution.cc +// +// $ blaze run :gaussian_distribution_gentables > gaussian_distribution.cc +// +#include "absl/random/gaussian_distribution.h" + +#include <cmath> +#include <cstddef> +#include <iostream> +#include <limits> +#include <string> + +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +template <typename T, size_t N> +void FormatArrayContents(std::ostream* os, T (&data)[N]) { + if (!std::numeric_limits<T>::is_exact) { + // Note: T is either an integer or a float. + // float requires higher precision to ensure that values are + // reproduced exactly. + // Trivia: C99 has hexadecimal floating point literals, but C++11 does not. + // Using them would remove all concern of precision loss. + os->precision(std::numeric_limits<T>::max_digits10 + 2); + } + *os << " {"; + std::string separator = ""; + for (size_t i = 0; i < N; ++i) { + *os << separator << data[i]; + if ((i + 1) % 3 != 0) { + separator = ", "; + } else { + separator = ",\n "; + } + } + *os << "}"; +} + +} // namespace + +class TableGenerator : public gaussian_distribution_base { + public: + TableGenerator(); + void Print(std::ostream* os); + + using gaussian_distribution_base::kMask; + using gaussian_distribution_base::kR; + using gaussian_distribution_base::kV; + + private: + Tables tables_; +}; + +// Ziggurat gaussian initialization. For an explanation of the algorithm, see +// the Marsaglia paper, "The Ziggurat Method for Generating Random Variables". +// http://www.jstatsoft.org/v05/i08/ +// +// Further details are available in the Doornik paper +// https://www.doornik.com/research/ziggurat.pdf +// +TableGenerator::TableGenerator() { + // The constants here should match the values in gaussian_distribution.h + static constexpr int kC = kMask + 1; + + static_assert((ABSL_ARRAYSIZE(tables_.x) == kC + 1), + "xArray must be length kMask + 2"); + + static_assert((ABSL_ARRAYSIZE(tables_.x) == ABSL_ARRAYSIZE(tables_.f)), + "fx and x arrays must be identical length"); + + auto f = [](double x) { return std::exp(-0.5 * x * x); }; + auto f_inv = [](double x) { return std::sqrt(-2.0 * std::log(x)); }; + + tables_.x[0] = kV / f(kR); + tables_.f[0] = f(tables_.x[0]); + + tables_.x[1] = kR; + tables_.f[1] = f(tables_.x[1]); + + tables_.x[kC] = 0.0; + tables_.f[kC] = f(tables_.x[kC]); // 1.0 + + for (int i = 2; i < kC; i++) { + double v = (kV / tables_.x[i - 1]) + tables_.f[i - 1]; + tables_.x[i] = f_inv(v); + tables_.f[i] = v; + } +} + +void TableGenerator::Print(std::ostream* os) { + *os << "// BEGIN GENERATED CODE; DO NOT EDIT\n" + "// clang-format off\n" + "\n" + "#include \"absl/random/gaussian_distribution.h\"\n" + "\n" + "namespace absl {\n" + "ABSL_NAMESPACE_BEGIN\n" + "namespace random_internal {\n" + "\n" + "const gaussian_distribution_base::Tables\n" + " gaussian_distribution_base::zg_ = {\n"; + FormatArrayContents(os, tables_.x); + *os << ",\n"; + FormatArrayContents(os, tables_.f); + *os << "};\n" + "\n" + "} // namespace random_internal\n" + "ABSL_NAMESPACE_END\n" + "} // namespace absl\n" + "\n" + "// clang-format on\n" + "// END GENERATED CODE"; + *os << std::endl; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +int main(int, char**) { + std::cerr << "\nCopy the output to gaussian_distribution.cc" << std::endl; + absl::random_internal::TableGenerator generator; + generator.Print(&std::cout); + return 0; +} diff --git a/third_party/abseil_cpp/absl/random/internal/generate_real.h b/third_party/abseil_cpp/absl/random/internal/generate_real.h new file mode 100644 index 000000000000..20f6d20807d5 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/generate_real.h @@ -0,0 +1,146 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_GENERATE_REAL_H_ +#define ABSL_RANDOM_INTERNAL_GENERATE_REAL_H_ + +// This file contains some implementation details which are used by one or more +// of the absl random number distributions. + +#include <cstdint> +#include <cstring> +#include <limits> +#include <type_traits> + +#include "absl/base/internal/bits.h" +#include "absl/meta/type_traits.h" +#include "absl/random/internal/fastmath.h" +#include "absl/random/internal/traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Tristate tag types controlling the output of GenerateRealFromBits. +struct GeneratePositiveTag {}; +struct GenerateNegativeTag {}; +struct GenerateSignedTag {}; + +// GenerateRealFromBits generates a single real value from a single 64-bit +// `bits` with template fields controlling the output. +// +// The `SignedTag` parameter controls whether positive, negative, +// or either signed/unsigned may be returned. +// When SignedTag == GeneratePositiveTag, range is U(0, 1) +// When SignedTag == GenerateNegativeTag, range is U(-1, 0) +// When SignedTag == GenerateSignedTag, range is U(-1, 1) +// +// When the `IncludeZero` parameter is true, the function may return 0 for some +// inputs, otherwise it never returns 0. +// +// When a value in U(0,1) is required, use: +// Uniform64ToReal<double, PositiveValueT, true>; +// +// When a value in U(-1,1) is required, use: +// Uniform64ToReal<double, SignedValueT, false>; +// +// This generates more distinct values than the mathematical equivalent +// `U(0, 1) * 2.0 - 1.0`. +// +// Scaling the result by powers of 2 (and avoiding a multiply) is also possible: +// GenerateRealFromBits<double>(..., -1); => U(0, 0.5) +// GenerateRealFromBits<double>(..., 1); => U(0, 2) +// +template <typename RealType, // Real type, either float or double. + typename SignedTag = GeneratePositiveTag, // Whether a positive, + // negative, or signed + // value is generated. + bool IncludeZero = true> +inline RealType GenerateRealFromBits(uint64_t bits, int exp_bias = 0) { + using real_type = RealType; + using uint_type = absl::conditional_t<std::is_same<real_type, float>::value, + uint32_t, uint64_t>; + + static_assert( + (std::is_same<double, real_type>::value || + std::is_same<float, real_type>::value), + "GenerateRealFromBits must be parameterized by either float or double."); + + static_assert(sizeof(uint_type) == sizeof(real_type), + "Mismatched unsinged and real types."); + + static_assert((std::numeric_limits<real_type>::is_iec559 && + std::numeric_limits<real_type>::radix == 2), + "RealType representation is not IEEE 754 binary."); + + static_assert((std::is_same<SignedTag, GeneratePositiveTag>::value || + std::is_same<SignedTag, GenerateNegativeTag>::value || + std::is_same<SignedTag, GenerateSignedTag>::value), + ""); + + static constexpr int kExp = std::numeric_limits<real_type>::digits - 1; + static constexpr uint_type kMask = (static_cast<uint_type>(1) << kExp) - 1u; + static constexpr int kUintBits = sizeof(uint_type) * 8; + + int exp = exp_bias + int{std::numeric_limits<real_type>::max_exponent - 2}; + + // Determine the sign bit. + // Depending on the SignedTag, this may use the left-most bit + // or it may be a constant value. + uint_type sign = std::is_same<SignedTag, GenerateNegativeTag>::value + ? (static_cast<uint_type>(1) << (kUintBits - 1)) + : 0; + if (std::is_same<SignedTag, GenerateSignedTag>::value) { + if (std::is_same<uint_type, uint64_t>::value) { + sign = bits & uint64_t{0x8000000000000000}; + } + if (std::is_same<uint_type, uint32_t>::value) { + const uint64_t tmp = bits & uint64_t{0x8000000000000000}; + sign = static_cast<uint32_t>(tmp >> 32); + } + // adjust the bits and the exponent to account for removing + // the leading bit. + bits = bits & uint64_t{0x7FFFFFFFFFFFFFFF}; + exp++; + } + if (IncludeZero) { + if (bits == 0u) return 0; + } + + // Number of leading zeros is mapped to the exponent: 2^-clz + // bits is 0..01xxxxxx. After shifting, we're left with 1xxx...0..0 + int clz = base_internal::CountLeadingZeros64(bits); + bits <<= (IncludeZero ? clz : (clz & 63)); // remove 0-bits. + exp -= clz; // set the exponent. + bits >>= (63 - kExp); + + // Construct the 32-bit or 64-bit IEEE 754 floating-point value from + // the individual fields: sign, exp, mantissa(bits). + uint_type val = + (std::is_same<SignedTag, GeneratePositiveTag>::value ? 0u : sign) | + (static_cast<uint_type>(exp) << kExp) | + (static_cast<uint_type>(bits) & kMask); + + // bit_cast to the output-type + real_type result; + memcpy(static_cast<void*>(&result), static_cast<const void*>(&val), + sizeof(result)); + return result; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_GENERATE_REAL_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/generate_real_test.cc b/third_party/abseil_cpp/absl/random/internal/generate_real_test.cc new file mode 100644 index 000000000000..4bdc453483e4 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/generate_real_test.cc @@ -0,0 +1,497 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/generate_real.h" + +#include <cfloat> +#include <cstddef> +#include <cstdint> +#include <string> + +#include "gtest/gtest.h" +#include "absl/base/internal/bits.h" +#include "absl/flags/flag.h" + +ABSL_FLAG(int64_t, absl_random_test_trials, 50000, + "Number of trials for the probability tests."); + +using absl::random_internal::GenerateNegativeTag; +using absl::random_internal::GeneratePositiveTag; +using absl::random_internal::GenerateRealFromBits; +using absl::random_internal::GenerateSignedTag; + +namespace { + +TEST(GenerateRealTest, U64ToFloat_Positive_NoZero_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GeneratePositiveTag, false>(a); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), 2.710505431e-20f); + EXPECT_EQ(ToFloat(0x0000000000000001), 5.421010862e-20f); + EXPECT_EQ(ToFloat(0x8000000000000000), 0.5); + EXPECT_EQ(ToFloat(0x8000000000000001), 0.5); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), 0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloat_Positive_Zero_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GeneratePositiveTag, true>(a); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), 0.0); + EXPECT_EQ(ToFloat(0x0000000000000001), 5.421010862e-20f); + EXPECT_EQ(ToFloat(0x8000000000000000), 0.5); + EXPECT_EQ(ToFloat(0x8000000000000001), 0.5); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), 0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloat_Negative_NoZero_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GenerateNegativeTag, false>(a); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), -2.710505431e-20f); + EXPECT_EQ(ToFloat(0x0000000000000001), -5.421010862e-20f); + EXPECT_EQ(ToFloat(0x8000000000000000), -0.5); + EXPECT_EQ(ToFloat(0x8000000000000001), -0.5); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), -0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloat_Negative_Zero_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GenerateNegativeTag, true>(a); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), 0.0); + EXPECT_EQ(ToFloat(0x0000000000000001), -5.421010862e-20f); + EXPECT_EQ(ToFloat(0x8000000000000000), -0.5); + EXPECT_EQ(ToFloat(0x8000000000000001), -0.5); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), -0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloat_Signed_NoZero_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GenerateSignedTag, false>(a); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), 5.421010862e-20f); + EXPECT_EQ(ToFloat(0x0000000000000001), 1.084202172e-19f); + EXPECT_EQ(ToFloat(0x7FFFFFFFFFFFFFFF), 0.9999999404f); + EXPECT_EQ(ToFloat(0x8000000000000000), -5.421010862e-20f); + EXPECT_EQ(ToFloat(0x8000000000000001), -1.084202172e-19f); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), -0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloat_Signed_Zero_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GenerateSignedTag, true>(a); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), 0); + EXPECT_EQ(ToFloat(0x0000000000000001), 1.084202172e-19f); + EXPECT_EQ(ToFloat(0x7FFFFFFFFFFFFFFF), 0.9999999404f); + EXPECT_EQ(ToFloat(0x8000000000000000), 0); + EXPECT_EQ(ToFloat(0x8000000000000001), -1.084202172e-19f); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), -0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloat_Signed_Bias_Test) { + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GenerateSignedTag, true>(a, 1); + }; + EXPECT_EQ(ToFloat(0x0000000000000000), 0); + EXPECT_EQ(ToFloat(0x0000000000000001), 2 * 1.084202172e-19f); + EXPECT_EQ(ToFloat(0x7FFFFFFFFFFFFFFF), 2 * 0.9999999404f); + EXPECT_EQ(ToFloat(0x8000000000000000), 0); + EXPECT_EQ(ToFloat(0x8000000000000001), 2 * -1.084202172e-19f); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), 2 * -0.9999999404f); +} + +TEST(GenerateRealTest, U64ToFloatTest) { + auto ToFloat = [](uint64_t a) -> float { + return GenerateRealFromBits<float, GeneratePositiveTag, true>(a); + }; + + EXPECT_EQ(ToFloat(0x0000000000000000), 0.0f); + + EXPECT_EQ(ToFloat(0x8000000000000000), 0.5f); + EXPECT_EQ(ToFloat(0x8000000000000001), 0.5f); + EXPECT_EQ(ToFloat(0x800000FFFFFFFFFF), 0.5f); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), 0.9999999404f); + + EXPECT_GT(ToFloat(0x0000000000000001), 0.0f); + + EXPECT_NE(ToFloat(0x7FFFFF0000000000), ToFloat(0x7FFFFEFFFFFFFFFF)); + + EXPECT_LT(ToFloat(0xFFFFFFFFFFFFFFFF), 1.0f); + int32_t two_to_24 = 1 << 24; + EXPECT_EQ(static_cast<int32_t>(ToFloat(0xFFFFFFFFFFFFFFFF) * two_to_24), + two_to_24 - 1); + EXPECT_NE(static_cast<int32_t>(ToFloat(0xFFFFFFFFFFFFFFFF) * two_to_24 * 2), + two_to_24 * 2 - 1); + EXPECT_EQ(ToFloat(0xFFFFFFFFFFFFFFFF), ToFloat(0xFFFFFF0000000000)); + EXPECT_NE(ToFloat(0xFFFFFFFFFFFFFFFF), ToFloat(0xFFFFFEFFFFFFFFFF)); + EXPECT_EQ(ToFloat(0x7FFFFFFFFFFFFFFF), ToFloat(0x7FFFFF8000000000)); + EXPECT_NE(ToFloat(0x7FFFFFFFFFFFFFFF), ToFloat(0x7FFFFF7FFFFFFFFF)); + EXPECT_EQ(ToFloat(0x3FFFFFFFFFFFFFFF), ToFloat(0x3FFFFFC000000000)); + EXPECT_NE(ToFloat(0x3FFFFFFFFFFFFFFF), ToFloat(0x3FFFFFBFFFFFFFFF)); + + // For values where every bit counts, the values scale as multiples of the + // input. + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(i * ToFloat(0x0000000000000001), ToFloat(i)); + } + + // For each i: value generated from (1 << i). + float exp_values[64]; + exp_values[63] = 0.5f; + for (int i = 62; i >= 0; --i) exp_values[i] = 0.5f * exp_values[i + 1]; + constexpr uint64_t one = 1; + for (int i = 0; i < 64; ++i) { + EXPECT_EQ(ToFloat(one << i), exp_values[i]); + for (int j = 1; j < FLT_MANT_DIG && i - j >= 0; ++j) { + EXPECT_NE(exp_values[i] + exp_values[i - j], exp_values[i]); + EXPECT_EQ(ToFloat((one << i) + (one << (i - j))), + exp_values[i] + exp_values[i - j]); + } + for (int j = FLT_MANT_DIG; i - j >= 0; ++j) { + EXPECT_EQ(exp_values[i] + exp_values[i - j], exp_values[i]); + EXPECT_EQ(ToFloat((one << i) + (one << (i - j))), exp_values[i]); + } + } +} + +TEST(GenerateRealTest, U64ToDouble_Positive_NoZero_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GeneratePositiveTag, false>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), 2.710505431213761085e-20); + EXPECT_EQ(ToDouble(0x0000000000000001), 5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x0000000000000002), 1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0x8000000000000000), 0.5); + EXPECT_EQ(ToDouble(0x8000000000000001), 0.5); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), 0.999999999999999888978); +} + +TEST(GenerateRealTest, U64ToDouble_Positive_Zero_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GeneratePositiveTag, true>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), 0.0); + EXPECT_EQ(ToDouble(0x0000000000000001), 5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x8000000000000000), 0.5); + EXPECT_EQ(ToDouble(0x8000000000000001), 0.5); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), 0.999999999999999888978); +} + +TEST(GenerateRealTest, U64ToDouble_Negative_NoZero_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GenerateNegativeTag, false>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), -2.710505431213761085e-20); + EXPECT_EQ(ToDouble(0x0000000000000001), -5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x0000000000000002), -1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0x8000000000000000), -0.5); + EXPECT_EQ(ToDouble(0x8000000000000001), -0.5); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), -0.999999999999999888978); +} + +TEST(GenerateRealTest, U64ToDouble_Negative_Zero_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GenerateNegativeTag, true>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), 0.0); + EXPECT_EQ(ToDouble(0x0000000000000001), -5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x0000000000000002), -1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0x8000000000000000), -0.5); + EXPECT_EQ(ToDouble(0x8000000000000001), -0.5); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), -0.999999999999999888978); +} + +TEST(GenerateRealTest, U64ToDouble_Signed_NoZero_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GenerateSignedTag, false>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), 5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x0000000000000001), 1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0x7FFFFFFFFFFFFFFF), 0.999999999999999888978); + EXPECT_EQ(ToDouble(0x8000000000000000), -5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x8000000000000001), -1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), -0.999999999999999888978); +} + +TEST(GenerateRealTest, U64ToDouble_Signed_Zero_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GenerateSignedTag, true>(a); + }; + EXPECT_EQ(ToDouble(0x0000000000000000), 0); + EXPECT_EQ(ToDouble(0x0000000000000001), 1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0x7FFFFFFFFFFFFFFF), 0.999999999999999888978); + EXPECT_EQ(ToDouble(0x8000000000000000), 0); + EXPECT_EQ(ToDouble(0x8000000000000001), -1.084202172485504434e-19); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), -0.999999999999999888978); +} + +TEST(GenerateRealTest, U64ToDouble_GenerateSignedTag_Bias_Test) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GenerateSignedTag, true>(a, -1); + }; + EXPECT_EQ(ToDouble(0x0000000000000000), 0); + EXPECT_EQ(ToDouble(0x0000000000000001), 1.084202172485504434e-19 / 2); + EXPECT_EQ(ToDouble(0x7FFFFFFFFFFFFFFF), 0.999999999999999888978 / 2); + EXPECT_EQ(ToDouble(0x8000000000000000), 0); + EXPECT_EQ(ToDouble(0x8000000000000001), -1.084202172485504434e-19 / 2); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), -0.999999999999999888978 / 2); +} + +TEST(GenerateRealTest, U64ToDoubleTest) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GeneratePositiveTag, true>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), 0.0); + EXPECT_EQ(ToDouble(0x0000000000000000), 0.0); + + EXPECT_EQ(ToDouble(0x0000000000000001), 5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x7fffffffffffffef), 0.499999999999999944489); + EXPECT_EQ(ToDouble(0x8000000000000000), 0.5); + + // For values > 0.5, RandU64ToDouble discards up to 11 bits. (64-53). + EXPECT_EQ(ToDouble(0x8000000000000001), 0.5); + EXPECT_EQ(ToDouble(0x80000000000007FF), 0.5); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), 0.999999999999999888978); + EXPECT_NE(ToDouble(0x7FFFFFFFFFFFF800), ToDouble(0x7FFFFFFFFFFFF7FF)); + + EXPECT_LT(ToDouble(0xFFFFFFFFFFFFFFFF), 1.0); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFF), ToDouble(0xFFFFFFFFFFFFF800)); + EXPECT_NE(ToDouble(0xFFFFFFFFFFFFFFFF), ToDouble(0xFFFFFFFFFFFFF7FF)); + EXPECT_EQ(ToDouble(0x7FFFFFFFFFFFFFFF), ToDouble(0x7FFFFFFFFFFFFC00)); + EXPECT_NE(ToDouble(0x7FFFFFFFFFFFFFFF), ToDouble(0x7FFFFFFFFFFFFBFF)); + EXPECT_EQ(ToDouble(0x3FFFFFFFFFFFFFFF), ToDouble(0x3FFFFFFFFFFFFE00)); + EXPECT_NE(ToDouble(0x3FFFFFFFFFFFFFFF), ToDouble(0x3FFFFFFFFFFFFDFF)); + + EXPECT_EQ(ToDouble(0x1000000000000001), 0.0625); + EXPECT_EQ(ToDouble(0x2000000000000001), 0.125); + EXPECT_EQ(ToDouble(0x3000000000000001), 0.1875); + EXPECT_EQ(ToDouble(0x4000000000000001), 0.25); + EXPECT_EQ(ToDouble(0x5000000000000001), 0.3125); + EXPECT_EQ(ToDouble(0x6000000000000001), 0.375); + EXPECT_EQ(ToDouble(0x7000000000000001), 0.4375); + EXPECT_EQ(ToDouble(0x8000000000000001), 0.5); + EXPECT_EQ(ToDouble(0x9000000000000001), 0.5625); + EXPECT_EQ(ToDouble(0xa000000000000001), 0.625); + EXPECT_EQ(ToDouble(0xb000000000000001), 0.6875); + EXPECT_EQ(ToDouble(0xc000000000000001), 0.75); + EXPECT_EQ(ToDouble(0xd000000000000001), 0.8125); + EXPECT_EQ(ToDouble(0xe000000000000001), 0.875); + EXPECT_EQ(ToDouble(0xf000000000000001), 0.9375); + + // Large powers of 2. + int64_t two_to_53 = int64_t{1} << 53; + EXPECT_EQ(static_cast<int64_t>(ToDouble(0xFFFFFFFFFFFFFFFF) * two_to_53), + two_to_53 - 1); + EXPECT_NE(static_cast<int64_t>(ToDouble(0xFFFFFFFFFFFFFFFF) * two_to_53 * 2), + two_to_53 * 2 - 1); + + // For values where every bit counts, the values scale as multiples of the + // input. + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(i * ToDouble(0x0000000000000001), ToDouble(i)); + } + + // For each i: value generated from (1 << i). + double exp_values[64]; + exp_values[63] = 0.5; + for (int i = 62; i >= 0; --i) exp_values[i] = 0.5 * exp_values[i + 1]; + constexpr uint64_t one = 1; + for (int i = 0; i < 64; ++i) { + EXPECT_EQ(ToDouble(one << i), exp_values[i]); + for (int j = 1; j < DBL_MANT_DIG && i - j >= 0; ++j) { + EXPECT_NE(exp_values[i] + exp_values[i - j], exp_values[i]); + EXPECT_EQ(ToDouble((one << i) + (one << (i - j))), + exp_values[i] + exp_values[i - j]); + } + for (int j = DBL_MANT_DIG; i - j >= 0; ++j) { + EXPECT_EQ(exp_values[i] + exp_values[i - j], exp_values[i]); + EXPECT_EQ(ToDouble((one << i) + (one << (i - j))), exp_values[i]); + } + } +} + +TEST(GenerateRealTest, U64ToDoubleSignedTest) { + auto ToDouble = [](uint64_t a) { + return GenerateRealFromBits<double, GenerateSignedTag, false>(a); + }; + + EXPECT_EQ(ToDouble(0x0000000000000000), 5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x0000000000000001), 1.084202172485504434e-19); + + EXPECT_EQ(ToDouble(0x8000000000000000), -5.42101086242752217004e-20); + EXPECT_EQ(ToDouble(0x8000000000000001), -1.084202172485504434e-19); + + const double e_plus = ToDouble(0x0000000000000001); + const double e_minus = ToDouble(0x8000000000000001); + EXPECT_EQ(e_plus, 1.084202172485504434e-19); + EXPECT_EQ(e_minus, -1.084202172485504434e-19); + + EXPECT_EQ(ToDouble(0x3fffffffffffffef), 0.499999999999999944489); + EXPECT_EQ(ToDouble(0xbfffffffffffffef), -0.499999999999999944489); + + // For values > 0.5, RandU64ToDouble discards up to 10 bits. (63-53). + EXPECT_EQ(ToDouble(0x4000000000000000), 0.5); + EXPECT_EQ(ToDouble(0x4000000000000001), 0.5); + EXPECT_EQ(ToDouble(0x40000000000003FF), 0.5); + + EXPECT_EQ(ToDouble(0xC000000000000000), -0.5); + EXPECT_EQ(ToDouble(0xC000000000000001), -0.5); + EXPECT_EQ(ToDouble(0xC0000000000003FF), -0.5); + + EXPECT_EQ(ToDouble(0x7FFFFFFFFFFFFFFe), 0.999999999999999888978); + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFe), -0.999999999999999888978); + + EXPECT_NE(ToDouble(0x7FFFFFFFFFFFF800), ToDouble(0x7FFFFFFFFFFFF7FF)); + + EXPECT_LT(ToDouble(0x7FFFFFFFFFFFFFFF), 1.0); + EXPECT_GT(ToDouble(0x7FFFFFFFFFFFFFFF), 0.9999999999); + + EXPECT_GT(ToDouble(0xFFFFFFFFFFFFFFFe), -1.0); + EXPECT_LT(ToDouble(0xFFFFFFFFFFFFFFFe), -0.999999999); + + EXPECT_EQ(ToDouble(0xFFFFFFFFFFFFFFFe), ToDouble(0xFFFFFFFFFFFFFC00)); + EXPECT_EQ(ToDouble(0x7FFFFFFFFFFFFFFF), ToDouble(0x7FFFFFFFFFFFFC00)); + EXPECT_NE(ToDouble(0xFFFFFFFFFFFFFFFe), ToDouble(0xFFFFFFFFFFFFF3FF)); + EXPECT_NE(ToDouble(0x7FFFFFFFFFFFFFFF), ToDouble(0x7FFFFFFFFFFFF3FF)); + + EXPECT_EQ(ToDouble(0x1000000000000001), 0.125); + EXPECT_EQ(ToDouble(0x2000000000000001), 0.25); + EXPECT_EQ(ToDouble(0x3000000000000001), 0.375); + EXPECT_EQ(ToDouble(0x4000000000000001), 0.5); + EXPECT_EQ(ToDouble(0x5000000000000001), 0.625); + EXPECT_EQ(ToDouble(0x6000000000000001), 0.75); + EXPECT_EQ(ToDouble(0x7000000000000001), 0.875); + EXPECT_EQ(ToDouble(0x7800000000000001), 0.9375); + EXPECT_EQ(ToDouble(0x7c00000000000001), 0.96875); + EXPECT_EQ(ToDouble(0x7e00000000000001), 0.984375); + EXPECT_EQ(ToDouble(0x7f00000000000001), 0.9921875); + + // 0x8000000000000000 ~= 0 + EXPECT_EQ(ToDouble(0x9000000000000001), -0.125); + EXPECT_EQ(ToDouble(0xa000000000000001), -0.25); + EXPECT_EQ(ToDouble(0xb000000000000001), -0.375); + EXPECT_EQ(ToDouble(0xc000000000000001), -0.5); + EXPECT_EQ(ToDouble(0xd000000000000001), -0.625); + EXPECT_EQ(ToDouble(0xe000000000000001), -0.75); + EXPECT_EQ(ToDouble(0xf000000000000001), -0.875); + + // Large powers of 2. + int64_t two_to_53 = int64_t{1} << 53; + EXPECT_EQ(static_cast<int64_t>(ToDouble(0x7FFFFFFFFFFFFFFF) * two_to_53), + two_to_53 - 1); + EXPECT_EQ(static_cast<int64_t>(ToDouble(0xFFFFFFFFFFFFFFFF) * two_to_53), + -(two_to_53 - 1)); + + EXPECT_NE(static_cast<int64_t>(ToDouble(0x7FFFFFFFFFFFFFFF) * two_to_53 * 2), + two_to_53 * 2 - 1); + + // For values where every bit counts, the values scale as multiples of the + // input. + for (int i = 1; i < 100; ++i) { + EXPECT_EQ(i * e_plus, ToDouble(i)) << i; + EXPECT_EQ(i * e_minus, ToDouble(0x8000000000000000 | i)) << i; + } +} + +TEST(GenerateRealTest, ExhaustiveFloat) { + using absl::base_internal::CountLeadingZeros64; + auto ToFloat = [](uint64_t a) { + return GenerateRealFromBits<float, GeneratePositiveTag, true>(a); + }; + + // Rely on RandU64ToFloat generating values from greatest to least when + // supplied with uint64_t values from greatest (0xfff...) to least (0x0). + // Thus, this algorithm stores the previous value, and if the new value is at + // greater than or equal to the previous value, then there is a collision in + // the generation algorithm. + // + // Use the computation below to convert the random value into a result: + // double res = a() * (1.0f - sample) + b() * sample; + float last_f = 1.0, last_g = 2.0; + uint64_t f_collisions = 0, g_collisions = 0; + uint64_t f_unique = 0, g_unique = 0; + uint64_t total = 0; + auto count = [&](const float r) { + total++; + // `f` is mapped to the range [0, 1) (default) + const float f = 0.0f * (1.0f - r) + 1.0f * r; + if (f >= last_f) { + f_collisions++; + } else { + f_unique++; + last_f = f; + } + // `g` is mapped to the range [1, 2) + const float g = 1.0f * (1.0f - r) + 2.0f * r; + if (g >= last_g) { + g_collisions++; + } else { + g_unique++; + last_g = g; + } + }; + + size_t limit = absl::GetFlag(FLAGS_absl_random_test_trials); + + // Generate all uint64_t which have unique floating point values. + // Counting down from 0xFFFFFFFFFFFFFFFFu ... 0x0u + uint64_t x = ~uint64_t(0); + for (; x != 0 && limit > 0;) { + constexpr int kDig = (64 - FLT_MANT_DIG); + // Set a decrement value & the next point at which to change + // the decrement value. By default these are 1, 0. + uint64_t dec = 1; + uint64_t chk = 0; + + // Adjust decrement and check value based on how many leading 0 + // bits are set in the current value. + const int clz = CountLeadingZeros64(x); + if (clz < kDig) { + dec <<= (kDig - clz); + chk = (~uint64_t(0)) >> (clz + 1); + } + for (; x > chk && limit > 0; x -= dec) { + count(ToFloat(x)); + --limit; + } + } + + static_assert(FLT_MANT_DIG == 24, + "The float type is expected to have a 24 bit mantissa."); + + if (limit != 0) { + // There are between 2^28 and 2^29 unique values in the range [0, 1). For + // the low values of x, there are 2^24 -1 unique values. Once x > 2^24, + // there are 40 * 2^24 unique values. Thus: + // (2 + 4 + 8 ... + 2^23) + 40 * 2^23 + EXPECT_LT(1 << 28, f_unique); + EXPECT_EQ((1 << 24) + 40 * (1 << 23) - 1, f_unique); + EXPECT_EQ(total, f_unique); + EXPECT_EQ(0, f_collisions); + + // Expect at least 2^23 unique values for the range [1, 2) + EXPECT_LE(1 << 23, g_unique); + EXPECT_EQ(total - g_unique, g_collisions); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/iostream_state_saver.h b/third_party/abseil_cpp/absl/random/internal/iostream_state_saver.h new file mode 100644 index 000000000000..e6e242ee1e66 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/iostream_state_saver.h @@ -0,0 +1,245 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_IOSTREAM_STATE_SAVER_H_ +#define ABSL_RANDOM_INTERNAL_IOSTREAM_STATE_SAVER_H_ + +#include <cmath> +#include <iostream> +#include <limits> +#include <type_traits> + +#include "absl/meta/type_traits.h" +#include "absl/numeric/int128.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// The null_state_saver does nothing. +template <typename T> +class null_state_saver { + public: + using stream_type = T; + using flags_type = std::ios_base::fmtflags; + + null_state_saver(T&, flags_type) {} + ~null_state_saver() {} +}; + +// ostream_state_saver is a RAII object to save and restore the common +// basic_ostream flags used when implementing `operator <<()` on any of +// the absl random distributions. +template <typename OStream> +class ostream_state_saver { + public: + using ostream_type = OStream; + using flags_type = std::ios_base::fmtflags; + using fill_type = typename ostream_type::char_type; + using precision_type = std::streamsize; + + ostream_state_saver(ostream_type& os, // NOLINT(runtime/references) + flags_type flags, fill_type fill) + : os_(os), + flags_(os.flags(flags)), + fill_(os.fill(fill)), + precision_(os.precision()) { + // Save state in initialized variables. + } + + ~ostream_state_saver() { + // Restore saved state. + os_.precision(precision_); + os_.fill(fill_); + os_.flags(flags_); + } + + private: + ostream_type& os_; + const flags_type flags_; + const fill_type fill_; + const precision_type precision_; +}; + +#if defined(__NDK_MAJOR__) && __NDK_MAJOR__ < 16 +#define ABSL_RANDOM_INTERNAL_IOSTREAM_HEXFLOAT 1 +#else +#define ABSL_RANDOM_INTERNAL_IOSTREAM_HEXFLOAT 0 +#endif + +template <typename CharT, typename Traits> +ostream_state_saver<std::basic_ostream<CharT, Traits>> make_ostream_state_saver( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + std::ios_base::fmtflags flags = std::ios_base::dec | std::ios_base::left | +#if ABSL_RANDOM_INTERNAL_IOSTREAM_HEXFLOAT + std::ios_base::fixed | +#endif + std::ios_base::scientific) { + using result_type = ostream_state_saver<std::basic_ostream<CharT, Traits>>; + return result_type(os, flags, os.widen(' ')); +} + +template <typename T> +typename absl::enable_if_t<!std::is_base_of<std::ios_base, T>::value, + null_state_saver<T>> +make_ostream_state_saver(T& is, // NOLINT(runtime/references) + std::ios_base::fmtflags flags = std::ios_base::dec) { + std::cerr << "null_state_saver"; + using result_type = null_state_saver<T>; + return result_type(is, flags); +} + +// stream_precision_helper<type>::kPrecision returns the base 10 precision +// required to stream and reconstruct a real type exact binary value through +// a binary->decimal->binary transition. +template <typename T> +struct stream_precision_helper { + // max_digits10 may be 0 on MSVC; if so, use digits10 + 3. + static constexpr int kPrecision = + (std::numeric_limits<T>::max_digits10 > std::numeric_limits<T>::digits10) + ? std::numeric_limits<T>::max_digits10 + : (std::numeric_limits<T>::digits10 + 3); +}; + +template <> +struct stream_precision_helper<float> { + static constexpr int kPrecision = 9; +}; +template <> +struct stream_precision_helper<double> { + static constexpr int kPrecision = 17; +}; +template <> +struct stream_precision_helper<long double> { + static constexpr int kPrecision = 36; // assuming fp128 +}; + +// istream_state_saver is a RAII object to save and restore the common +// std::basic_istream<> flags used when implementing `operator >>()` on any of +// the absl random distributions. +template <typename IStream> +class istream_state_saver { + public: + using istream_type = IStream; + using flags_type = std::ios_base::fmtflags; + + istream_state_saver(istream_type& is, // NOLINT(runtime/references) + flags_type flags) + : is_(is), flags_(is.flags(flags)) {} + + ~istream_state_saver() { is_.flags(flags_); } + + private: + istream_type& is_; + flags_type flags_; +}; + +template <typename CharT, typename Traits> +istream_state_saver<std::basic_istream<CharT, Traits>> make_istream_state_saver( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + std::ios_base::fmtflags flags = std::ios_base::dec | + std::ios_base::scientific | + std::ios_base::skipws) { + using result_type = istream_state_saver<std::basic_istream<CharT, Traits>>; + return result_type(is, flags); +} + +template <typename T> +typename absl::enable_if_t<!std::is_base_of<std::ios_base, T>::value, + null_state_saver<T>> +make_istream_state_saver(T& is, // NOLINT(runtime/references) + std::ios_base::fmtflags flags = std::ios_base::dec) { + using result_type = null_state_saver<T>; + return result_type(is, flags); +} + +// stream_format_type<T> is a helper struct to convert types which +// basic_iostream cannot output as decimal numbers into types which +// basic_iostream can output as decimal numbers. Specifically: +// * signed/unsigned char-width types are converted to int. +// * TODO(lar): __int128 => uint128, except there is no operator << yet. +// +template <typename T> +struct stream_format_type + : public std::conditional<(sizeof(T) == sizeof(char)), int, T> {}; + +// stream_u128_helper allows us to write out either absl::uint128 or +// __uint128_t types in the same way, which enables their use as internal +// state of PRNG engines. +template <typename T> +struct stream_u128_helper; + +template <> +struct stream_u128_helper<absl::uint128> { + template <typename IStream> + inline absl::uint128 read(IStream& in) { + uint64_t h = 0; + uint64_t l = 0; + in >> h >> l; + return absl::MakeUint128(h, l); + } + + template <typename OStream> + inline void write(absl::uint128 val, OStream& out) { + uint64_t h = absl::Uint128High64(val); + uint64_t l = absl::Uint128Low64(val); + out << h << out.fill() << l; + } +}; + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +template <> +struct stream_u128_helper<__uint128_t> { + template <typename IStream> + inline __uint128_t read(IStream& in) { + uint64_t h = 0; + uint64_t l = 0; + in >> h >> l; + return (static_cast<__uint128_t>(h) << 64) | l; + } + + template <typename OStream> + inline void write(__uint128_t val, OStream& out) { + uint64_t h = static_cast<uint64_t>(val >> 64u); + uint64_t l = static_cast<uint64_t>(val); + out << h << out.fill() << l; + } +}; +#endif + +template <typename FloatType, typename IStream> +inline FloatType read_floating_point(IStream& is) { + static_assert(std::is_floating_point<FloatType>::value, ""); + FloatType dest; + is >> dest; + // Parsing a double value may report a subnormal value as an error + // despite being able to represent it. + // See https://stackoverflow.com/q/52410931/3286653 + // It may also report an underflow when parsing DOUBLE_MIN as an + // ERANGE error, as the parsed value may be smaller than DOUBLE_MIN + // and rounded up. + // See: https://stackoverflow.com/q/42005462 + if (is.fail() && + (std::fabs(dest) == (std::numeric_limits<FloatType>::min)() || + std::fpclassify(dest) == FP_SUBNORMAL)) { + is.clear(is.rdstate() & (~std::ios_base::failbit)); + } + return dest; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_IOSTREAM_STATE_SAVER_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/iostream_state_saver_test.cc b/third_party/abseil_cpp/absl/random/internal/iostream_state_saver_test.cc new file mode 100644 index 000000000000..7bb8ad959cf6 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/iostream_state_saver_test.cc @@ -0,0 +1,371 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/iostream_state_saver.h" + +#include <sstream> +#include <string> + +#include "gtest/gtest.h" + +namespace { + +using absl::random_internal::make_istream_state_saver; +using absl::random_internal::make_ostream_state_saver; +using absl::random_internal::stream_precision_helper; + +template <typename T> +typename absl::enable_if_t<std::is_integral<T>::value, T> // +StreamRoundTrip(T t) { + std::stringstream ss; + { + auto saver = make_ostream_state_saver(ss); + ss.precision(stream_precision_helper<T>::kPrecision); + ss << t; + } + T result = 0; + { + auto saver = make_istream_state_saver(ss); + ss >> result; + } + EXPECT_FALSE(ss.fail()) // + << ss.str() << " " // + << (ss.good() ? "good " : "") // + << (ss.bad() ? "bad " : "") // + << (ss.eof() ? "eof " : "") // + << (ss.fail() ? "fail " : ""); + + return result; +} + +template <typename T> +typename absl::enable_if_t<std::is_floating_point<T>::value, T> // +StreamRoundTrip(T t) { + std::stringstream ss; + { + auto saver = make_ostream_state_saver(ss); + ss.precision(stream_precision_helper<T>::kPrecision); + ss << t; + } + T result = 0; + { + auto saver = make_istream_state_saver(ss); + result = absl::random_internal::read_floating_point<T>(ss); + } + EXPECT_FALSE(ss.fail()) // + << ss.str() << " " // + << (ss.good() ? "good " : "") // + << (ss.bad() ? "bad " : "") // + << (ss.eof() ? "eof " : "") // + << (ss.fail() ? "fail " : ""); + + return result; +} + +TEST(IOStreamStateSaver, BasicSaverState) { + std::stringstream ss; + ss.precision(2); + ss.fill('x'); + ss.flags(std::ios_base::dec | std::ios_base::right); + + { + auto saver = make_ostream_state_saver(ss); + ss.precision(10); + EXPECT_NE('x', ss.fill()); + EXPECT_EQ(10, ss.precision()); + EXPECT_NE(std::ios_base::dec | std::ios_base::right, ss.flags()); + + ss << 1.23; + } + + EXPECT_EQ('x', ss.fill()); + EXPECT_EQ(2, ss.precision()); + EXPECT_EQ(std::ios_base::dec | std::ios_base::right, ss.flags()); +} + +TEST(IOStreamStateSaver, RoundTripInts) { + const uint64_t kUintValues[] = { + 0, + 1, + static_cast<uint64_t>(-1), + 2, + static_cast<uint64_t>(-2), + + 1 << 7, + 1 << 8, + 1 << 16, + 1ull << 32, + 1ull << 50, + 1ull << 62, + 1ull << 63, + + (1 << 7) - 1, + (1 << 8) - 1, + (1 << 16) - 1, + (1ull << 32) - 1, + (1ull << 50) - 1, + (1ull << 62) - 1, + (1ull << 63) - 1, + + static_cast<uint64_t>(-(1 << 8)), + static_cast<uint64_t>(-(1 << 16)), + static_cast<uint64_t>(-(1ll << 32)), + static_cast<uint64_t>(-(1ll << 50)), + static_cast<uint64_t>(-(1ll << 62)), + + static_cast<uint64_t>(-(1 << 8) - 1), + static_cast<uint64_t>(-(1 << 16) - 1), + static_cast<uint64_t>(-(1ll << 32) - 1), + static_cast<uint64_t>(-(1ll << 50) - 1), + static_cast<uint64_t>(-(1ll << 62) - 1), + }; + + for (const uint64_t u : kUintValues) { + EXPECT_EQ(u, StreamRoundTrip<uint64_t>(u)); + + int64_t x = static_cast<int64_t>(u); + EXPECT_EQ(x, StreamRoundTrip<int64_t>(x)); + + double d = static_cast<double>(x); + EXPECT_EQ(d, StreamRoundTrip<double>(d)); + + float f = d; + EXPECT_EQ(f, StreamRoundTrip<float>(f)); + } +} + +TEST(IOStreamStateSaver, RoundTripFloats) { + static_assert( + stream_precision_helper<float>::kPrecision >= 9, + "stream_precision_helper<float>::kPrecision should be at least 9"); + + const float kValues[] = { + 1, + std::nextafter(1.0f, 0.0f), // 1 - epsilon + std::nextafter(1.0f, 2.0f), // 1 + epsilon + + 1.0e+1f, + 1.0e-1f, + 1.0e+2f, + 1.0e-2f, + 1.0e+10f, + 1.0e-10f, + + 0.00000051110000111311111111f, + -0.00000051110000111211111111f, + + 1.234678912345678912345e+6f, + 1.234678912345678912345e-6f, + 1.234678912345678912345e+30f, + 1.234678912345678912345e-30f, + 1.234678912345678912345e+38f, + 1.0234678912345678912345e-38f, + + // Boundary cases. + std::numeric_limits<float>::max(), + std::numeric_limits<float>::lowest(), + std::numeric_limits<float>::epsilon(), + std::nextafter(std::numeric_limits<float>::min(), + 1.0f), // min + epsilon + std::numeric_limits<float>::min(), // smallest normal + // There are some errors dealing with denorms on apple platforms. + std::numeric_limits<float>::denorm_min(), // smallest denorm + std::numeric_limits<float>::min() / 2, + std::nextafter(std::numeric_limits<float>::min(), + 0.0f), // denorm_max + std::nextafter(std::numeric_limits<float>::denorm_min(), 1.0f), + }; + + for (const float f : kValues) { + EXPECT_EQ(f, StreamRoundTrip<float>(f)); + EXPECT_EQ(-f, StreamRoundTrip<float>(-f)); + + double d = f; + EXPECT_EQ(d, StreamRoundTrip<double>(d)); + EXPECT_EQ(-d, StreamRoundTrip<double>(-d)); + + // Avoid undefined behavior (overflow/underflow). + if (f <= static_cast<float>(std::numeric_limits<int64_t>::max()) && + f >= static_cast<float>(std::numeric_limits<int64_t>::lowest())) { + int64_t x = static_cast<int64_t>(f); + EXPECT_EQ(x, StreamRoundTrip<int64_t>(x)); + } + } +} + +TEST(IOStreamStateSaver, RoundTripDoubles) { + static_assert( + stream_precision_helper<double>::kPrecision >= 17, + "stream_precision_helper<double>::kPrecision should be at least 17"); + + const double kValues[] = { + 1, + std::nextafter(1.0, 0.0), // 1 - epsilon + std::nextafter(1.0, 2.0), // 1 + epsilon + + 1.0e+1, + 1.0e-1, + 1.0e+2, + 1.0e-2, + 1.0e+10, + 1.0e-10, + + 0.00000051110000111311111111, + -0.00000051110000111211111111, + + 1.234678912345678912345e+6, + 1.234678912345678912345e-6, + 1.234678912345678912345e+30, + 1.234678912345678912345e-30, + 1.234678912345678912345e+38, + 1.0234678912345678912345e-38, + + 1.0e+100, + 1.0e-100, + 1.234678912345678912345e+308, + 1.0234678912345678912345e-308, + 2.22507385850720138e-308, + + // Boundary cases. + std::numeric_limits<double>::max(), + std::numeric_limits<double>::lowest(), + std::numeric_limits<double>::epsilon(), + std::nextafter(std::numeric_limits<double>::min(), + 1.0), // min + epsilon + std::numeric_limits<double>::min(), // smallest normal + // There are some errors dealing with denorms on apple platforms. + std::numeric_limits<double>::denorm_min(), // smallest denorm + std::numeric_limits<double>::min() / 2, + std::nextafter(std::numeric_limits<double>::min(), + 0.0), // denorm_max + std::nextafter(std::numeric_limits<double>::denorm_min(), 1.0f), + }; + + for (const double d : kValues) { + EXPECT_EQ(d, StreamRoundTrip<double>(d)); + EXPECT_EQ(-d, StreamRoundTrip<double>(-d)); + + // Avoid undefined behavior (overflow/underflow). + if (d <= std::numeric_limits<float>::max() && + d >= std::numeric_limits<float>::lowest()) { + float f = static_cast<float>(d); + EXPECT_EQ(f, StreamRoundTrip<float>(f)); + } + + // Avoid undefined behavior (overflow/underflow). + if (d <= static_cast<double>(std::numeric_limits<int64_t>::max()) && + d >= static_cast<double>(std::numeric_limits<int64_t>::lowest())) { + int64_t x = static_cast<int64_t>(d); + EXPECT_EQ(x, StreamRoundTrip<int64_t>(x)); + } + } +} + +#if !defined(__EMSCRIPTEN__) +TEST(IOStreamStateSaver, RoundTripLongDoubles) { + // Technically, C++ only guarantees that long double is at least as large as a + // double. Practically it varies from 64-bits to 128-bits. + // + // So it is best to consider long double a best-effort extended precision + // type. + + static_assert( + stream_precision_helper<long double>::kPrecision >= 36, + "stream_precision_helper<long double>::kPrecision should be at least 36"); + + using real_type = long double; + const real_type kValues[] = { + 1, + std::nextafter(1.0, 0.0), // 1 - epsilon + std::nextafter(1.0, 2.0), // 1 + epsilon + + 1.0e+1, + 1.0e-1, + 1.0e+2, + 1.0e-2, + 1.0e+10, + 1.0e-10, + + 0.00000051110000111311111111, + -0.00000051110000111211111111, + + 1.2346789123456789123456789123456789e+6, + 1.2346789123456789123456789123456789e-6, + 1.2346789123456789123456789123456789e+30, + 1.2346789123456789123456789123456789e-30, + 1.2346789123456789123456789123456789e+38, + 1.2346789123456789123456789123456789e-38, + 1.2346789123456789123456789123456789e+308, + 1.2346789123456789123456789123456789e-308, + + 1.0e+100, + 1.0e-100, + 1.234678912345678912345e+308, + 1.0234678912345678912345e-308, + + // Boundary cases. + std::numeric_limits<real_type>::max(), + std::numeric_limits<real_type>::lowest(), + std::numeric_limits<real_type>::epsilon(), + std::nextafter(std::numeric_limits<real_type>::min(), + real_type(1)), // min + epsilon + std::numeric_limits<real_type>::min(), // smallest normal + // There are some errors dealing with denorms on apple platforms. + std::numeric_limits<real_type>::denorm_min(), // smallest denorm + std::numeric_limits<real_type>::min() / 2, + std::nextafter(std::numeric_limits<real_type>::min(), + 0.0), // denorm_max + std::nextafter(std::numeric_limits<real_type>::denorm_min(), 1.0f), + }; + + int index = -1; + for (const long double dd : kValues) { + index++; + EXPECT_EQ(dd, StreamRoundTrip<real_type>(dd)) << index; + EXPECT_EQ(-dd, StreamRoundTrip<real_type>(-dd)) << index; + + // Avoid undefined behavior (overflow/underflow). + if (dd <= std::numeric_limits<double>::max() && + dd >= std::numeric_limits<double>::lowest()) { + double d = static_cast<double>(dd); + EXPECT_EQ(d, StreamRoundTrip<double>(d)); + } + + // Avoid undefined behavior (overflow/underflow). + if (dd <= std::numeric_limits<int64_t>::max() && + dd >= std::numeric_limits<int64_t>::lowest()) { + int64_t x = static_cast<int64_t>(dd); + EXPECT_EQ(x, StreamRoundTrip<int64_t>(x)); + } + } +} +#endif // !defined(__EMSCRIPTEN__) + +TEST(StrToDTest, DoubleMin) { + const char kV[] = "2.22507385850720138e-308"; + char* end; + double x = std::strtod(kV, &end); + EXPECT_EQ(std::numeric_limits<double>::min(), x); + // errno may equal ERANGE. +} + +TEST(StrToDTest, DoubleDenormMin) { + const char kV[] = "4.94065645841246544e-324"; + char* end; + double x = std::strtod(kV, &end); + EXPECT_EQ(std::numeric_limits<double>::denorm_min(), x); + // errno may equal ERANGE. +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/mock_helpers.h b/third_party/abseil_cpp/absl/random/internal/mock_helpers.h new file mode 100644 index 000000000000..9af27ab3a210 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/mock_helpers.h @@ -0,0 +1,127 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_MOCK_HELPERS_H_ +#define ABSL_RANDOM_INTERNAL_MOCK_HELPERS_H_ + +#include <tuple> +#include <type_traits> + +#include "absl/base/internal/fast_type_id.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// MockHelpers works in conjunction with MockOverloadSet, MockingBitGen, and +// BitGenRef to enable the mocking capability for absl distribution functions. +// +// MockingBitGen registers mocks based on the typeid of a mock signature, KeyT, +// which is used to generate a unique id. +// +// KeyT is a signature of the form: +// result_type(discriminator_type, std::tuple<args...>) +// The mocked function signature will be composed from KeyT as: +// result_type(args...) +// +class MockHelpers { + using IdType = ::absl::base_internal::FastTypeIdType; + + // Given a key signature type used to index the mock, extract the components. + // KeyT is expected to have the form: + // result_type(discriminator_type, arg_tuple_type) + template <typename KeyT> + struct KeySignature; + + template <typename ResultT, typename DiscriminatorT, typename ArgTupleT> + struct KeySignature<ResultT(DiscriminatorT, ArgTupleT)> { + using result_type = ResultT; + using discriminator_type = DiscriminatorT; + using arg_tuple_type = ArgTupleT; + }; + + // Detector for InvokeMock. + template <class T> + using invoke_mock_t = decltype(std::declval<T*>()->InvokeMock( + std::declval<IdType>(), std::declval<void*>(), std::declval<void*>())); + + // Empty implementation of InvokeMock. + template <typename KeyT, typename ReturnT, typename ArgTupleT, typename URBG, + typename... Args> + static absl::optional<ReturnT> InvokeMockImpl(char, URBG*, Args&&...) { + return absl::nullopt; + } + + // Non-empty implementation of InvokeMock. + template <typename KeyT, typename ReturnT, typename ArgTupleT, typename URBG, + typename = invoke_mock_t<URBG>, typename... Args> + static absl::optional<ReturnT> InvokeMockImpl(int, URBG* urbg, + Args&&... args) { + ArgTupleT arg_tuple(std::forward<Args>(args)...); + ReturnT result; + if (urbg->InvokeMock(::absl::base_internal::FastTypeId<KeyT>(), &arg_tuple, + &result)) { + return result; + } + return absl::nullopt; + } + + public: + // Invoke a mock for the KeyT (may or may not be a signature). + // + // KeyT is used to generate a typeid-based lookup key for the mock. + // KeyT is a signature of the form: + // result_type(discriminator_type, std::tuple<args...>) + // The mocked function signature will be composed from KeyT as: + // result_type(args...) + // + // An instance of arg_tuple_type must be constructable from Args..., since + // the underlying mechanism requires a pointer to an argument tuple. + template <typename KeyT, typename URBG, typename... Args> + static auto MaybeInvokeMock(URBG* urbg, Args&&... args) + -> absl::optional<typename KeySignature<KeyT>::result_type> { + // Use function overloading to dispatch to the implemenation since + // more modern patterns (e.g. require + constexpr) are not supported in all + // compiler configurations. + return InvokeMockImpl<KeyT, typename KeySignature<KeyT>::result_type, + typename KeySignature<KeyT>::arg_tuple_type, URBG>( + 0, urbg, std::forward<Args>(args)...); + } + + // Acquire a mock for the KeyT (may or may not be a signature). + // + // KeyT is used to generate a typeid-based lookup for the mock. + // KeyT is a signature of the form: + // result_type(discriminator_type, std::tuple<args...>) + // The mocked function signature will be composed from KeyT as: + // result_type(args...) + template <typename KeyT, typename MockURBG> + static auto MockFor(MockURBG& m) -> decltype( + std::declval<MockURBG>() + .template RegisterMock<typename KeySignature<KeyT>::result_type, + typename KeySignature<KeyT>::arg_tuple_type>( + std::declval<IdType>())) { + return m.template RegisterMock<typename KeySignature<KeyT>::result_type, + typename KeySignature<KeyT>::arg_tuple_type>( + ::absl::base_internal::FastTypeId<KeyT>()); + } +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_MOCK_HELPERS_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/mock_overload_set.h b/third_party/abseil_cpp/absl/random/internal/mock_overload_set.h new file mode 100644 index 000000000000..dccc6cee679c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/mock_overload_set.h @@ -0,0 +1,97 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_MOCK_OVERLOAD_SET_H_ +#define ABSL_RANDOM_INTERNAL_MOCK_OVERLOAD_SET_H_ + +#include <type_traits> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/random/internal/mock_helpers.h" +#include "absl/random/mocking_bit_gen.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +template <typename DistrT, typename Fn> +struct MockSingleOverload; + +// MockSingleOverload +// +// MockSingleOverload hooks in to gMock's `ON_CALL` and `EXPECT_CALL` macros. +// EXPECT_CALL(mock_single_overload, Call(...))` will expand to a call to +// `mock_single_overload.gmock_Call(...)`. Because expectations are stored on +// the MockingBitGen (an argument passed inside `Call(...)`), this forwards to +// arguments to MockingBitGen::Register. +// +// The underlying KeyT must match the KeyT constructed by DistributionCaller. +template <typename DistrT, typename Ret, typename... Args> +struct MockSingleOverload<DistrT, Ret(MockingBitGen&, Args...)> { + static_assert(std::is_same<typename DistrT::result_type, Ret>::value, + "Overload signature must have return type matching the " + "distribution result_type."); + using KeyT = Ret(DistrT, std::tuple<Args...>); + auto gmock_Call( + absl::MockingBitGen& gen, // NOLINT(google-runtime-references) + const ::testing::Matcher<Args>&... matchers) + -> decltype(MockHelpers::MockFor<KeyT>(gen).gmock_Call(matchers...)) { + return MockHelpers::MockFor<KeyT>(gen).gmock_Call(matchers...); + } +}; + +template <typename DistrT, typename Ret, typename Arg, typename... Args> +struct MockSingleOverload<DistrT, Ret(Arg, MockingBitGen&, Args...)> { + static_assert(std::is_same<typename DistrT::result_type, Ret>::value, + "Overload signature must have return type matching the " + "distribution result_type."); + using KeyT = Ret(DistrT, std::tuple<Arg, Args...>); + auto gmock_Call( + const ::testing::Matcher<Arg>& matcher, + absl::MockingBitGen& gen, // NOLINT(google-runtime-references) + const ::testing::Matcher<Args>&... matchers) + -> decltype(MockHelpers::MockFor<KeyT>(gen).gmock_Call(matcher, + matchers...)) { + return MockHelpers::MockFor<KeyT>(gen).gmock_Call(matcher, matchers...); + } +}; + +// MockOverloadSet +// +// MockOverloadSet takes a distribution and a collection of signatures and +// performs overload resolution amongst all the overloads. This makes +// `EXPECT_CALL(mock_overload_set, Call(...))` expand and do overload resolution +// correctly. +template <typename DistrT, typename... Signatures> +struct MockOverloadSet; + +template <typename DistrT, typename Sig> +struct MockOverloadSet<DistrT, Sig> : public MockSingleOverload<DistrT, Sig> { + using MockSingleOverload<DistrT, Sig>::gmock_Call; +}; + +template <typename DistrT, typename FirstSig, typename... Rest> +struct MockOverloadSet<DistrT, FirstSig, Rest...> + : public MockSingleOverload<DistrT, FirstSig>, + public MockOverloadSet<DistrT, Rest...> { + using MockSingleOverload<DistrT, FirstSig>::gmock_Call; + using MockOverloadSet<DistrT, Rest...>::gmock_Call; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_RANDOM_INTERNAL_MOCK_OVERLOAD_SET_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/nanobenchmark.cc b/third_party/abseil_cpp/absl/random/internal/nanobenchmark.cc new file mode 100644 index 000000000000..c9181813f7f0 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/nanobenchmark.cc @@ -0,0 +1,804 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/nanobenchmark.h" + +#include <sys/types.h> + +#include <algorithm> // sort +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <cstdlib> +#include <cstring> // memcpy +#include <limits> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/attributes.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/platform.h" +#include "absl/random/internal/randen_engine.h" + +// OS +#if defined(_WIN32) || defined(_WIN64) +#define ABSL_OS_WIN +#include <windows.h> // NOLINT + +#elif defined(__ANDROID__) +#define ABSL_OS_ANDROID + +#elif defined(__linux__) +#define ABSL_OS_LINUX +#include <sched.h> // NOLINT +#include <sys/syscall.h> // NOLINT +#endif + +#if defined(ABSL_ARCH_X86_64) && !defined(ABSL_OS_WIN) +#include <cpuid.h> // NOLINT +#endif + +// __ppc_get_timebase_freq +#if defined(ABSL_ARCH_PPC) +#include <sys/platform/ppc.h> // NOLINT +#endif + +// clock_gettime +#if defined(ABSL_ARCH_ARM) || defined(ABSL_ARCH_AARCH64) +#include <time.h> // NOLINT +#endif + +// ABSL_RANDOM_INTERNAL_ATTRIBUTE_NEVER_INLINE prevents inlining of the method. +#if ABSL_HAVE_ATTRIBUTE(noinline) || (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_RANDOM_INTERNAL_ATTRIBUTE_NEVER_INLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +#define ABSL_RANDOM_INTERNAL_ATTRIBUTE_NEVER_INLINE __declspec(noinline) +#else +#define ABSL_RANDOM_INTERNAL_ATTRIBUTE_NEVER_INLINE +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal_nanobenchmark { +namespace { + +// For code folding. +namespace platform { +#if defined(ABSL_ARCH_X86_64) + +// TODO(janwas): Merge with the one in randen_hwaes.cc? +void Cpuid(const uint32_t level, const uint32_t count, + uint32_t* ABSL_RANDOM_INTERNAL_RESTRICT abcd) { +#if defined(ABSL_OS_WIN) + int regs[4]; + __cpuidex(regs, level, count); + for (int i = 0; i < 4; ++i) { + abcd[i] = regs[i]; + } +#else + uint32_t a, b, c, d; + __cpuid_count(level, count, a, b, c, d); + abcd[0] = a; + abcd[1] = b; + abcd[2] = c; + abcd[3] = d; +#endif +} + +std::string BrandString() { + char brand_string[49]; + uint32_t abcd[4]; + + // Check if brand string is supported (it is on all reasonable Intel/AMD) + Cpuid(0x80000000U, 0, abcd); + if (abcd[0] < 0x80000004U) { + return std::string(); + } + + for (int i = 0; i < 3; ++i) { + Cpuid(0x80000002U + i, 0, abcd); + memcpy(brand_string + i * 16, &abcd, sizeof(abcd)); + } + brand_string[48] = 0; + return brand_string; +} + +// Returns the frequency quoted inside the brand string. This does not +// account for throttling nor Turbo Boost. +double NominalClockRate() { + const std::string& brand_string = BrandString(); + // Brand strings include the maximum configured frequency. These prefixes are + // defined by Intel CPUID documentation. + const char* prefixes[3] = {"MHz", "GHz", "THz"}; + const double multipliers[3] = {1E6, 1E9, 1E12}; + for (size_t i = 0; i < 3; ++i) { + const size_t pos_prefix = brand_string.find(prefixes[i]); + if (pos_prefix != std::string::npos) { + const size_t pos_space = brand_string.rfind(' ', pos_prefix - 1); + if (pos_space != std::string::npos) { + const std::string digits = + brand_string.substr(pos_space + 1, pos_prefix - pos_space - 1); + return std::stod(digits) * multipliers[i]; + } + } + } + + return 0.0; +} + +#endif // ABSL_ARCH_X86_64 +} // namespace platform + +// Prevents the compiler from eliding the computations that led to "output". +template <class T> +inline void PreventElision(T&& output) { +#ifndef ABSL_OS_WIN + // Works by indicating to the compiler that "output" is being read and + // modified. The +r constraint avoids unnecessary writes to memory, but only + // works for built-in types (typically FuncOutput). + asm volatile("" : "+r"(output) : : "memory"); +#else + // MSVC does not support inline assembly anymore (and never supported GCC's + // RTL constraints). Self-assignment with #pragma optimize("off") might be + // expected to prevent elision, but it does not with MSVC 2015. Type-punning + // with volatile pointers generates inefficient code on MSVC 2017. + static std::atomic<T> dummy(T{}); + dummy.store(output, std::memory_order_relaxed); +#endif +} + +namespace timer { + +// Start/Stop return absolute timestamps and must be placed immediately before +// and after the region to measure. We provide separate Start/Stop functions +// because they use different fences. +// +// Background: RDTSC is not 'serializing'; earlier instructions may complete +// after it, and/or later instructions may complete before it. 'Fences' ensure +// regions' elapsed times are independent of such reordering. The only +// documented unprivileged serializing instruction is CPUID, which acts as a +// full fence (no reordering across it in either direction). Unfortunately +// the latency of CPUID varies wildly (perhaps made worse by not initializing +// its EAX input). Because it cannot reliably be deducted from the region's +// elapsed time, it must not be included in the region to measure (i.e. +// between the two RDTSC). +// +// The newer RDTSCP is sometimes described as serializing, but it actually +// only serves as a half-fence with release semantics. Although all +// instructions in the region will complete before the final timestamp is +// captured, subsequent instructions may leak into the region and increase the +// elapsed time. Inserting another fence after the final RDTSCP would prevent +// such reordering without affecting the measured region. +// +// Fortunately, such a fence exists. The LFENCE instruction is only documented +// to delay later loads until earlier loads are visible. However, Intel's +// reference manual says it acts as a full fence (waiting until all earlier +// instructions have completed, and delaying later instructions until it +// completes). AMD assigns the same behavior to MFENCE. +// +// We need a fence before the initial RDTSC to prevent earlier instructions +// from leaking into the region, and arguably another after RDTSC to avoid +// region instructions from completing before the timestamp is recorded. +// When surrounded by fences, the additional RDTSCP half-fence provides no +// benefit, so the initial timestamp can be recorded via RDTSC, which has +// lower overhead than RDTSCP because it does not read TSC_AUX. In summary, +// we define Start = LFENCE/RDTSC/LFENCE; Stop = RDTSCP/LFENCE. +// +// Using Start+Start leads to higher variance and overhead than Stop+Stop. +// However, Stop+Stop includes an LFENCE in the region measurements, which +// adds a delay dependent on earlier loads. The combination of Start+Stop +// is faster than Start+Start and more consistent than Stop+Stop because +// the first LFENCE already delayed subsequent loads before the measured +// region. This combination seems not to have been considered in prior work: +// http://akaros.cs.berkeley.edu/lxr/akaros/kern/arch/x86/rdtsc_test.c +// +// Note: performance counters can measure 'exact' instructions-retired or +// (unhalted) cycle counts. The RDPMC instruction is not serializing and also +// requires fences. Unfortunately, it is not accessible on all OSes and we +// prefer to avoid kernel-mode drivers. Performance counters are also affected +// by several under/over-count errata, so we use the TSC instead. + +// Returns a 64-bit timestamp in unit of 'ticks'; to convert to seconds, +// divide by InvariantTicksPerSecond. +inline uint64_t Start64() { + uint64_t t; +#if defined(ABSL_ARCH_PPC) + asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268)); +#elif defined(ABSL_ARCH_X86_64) +#if defined(ABSL_OS_WIN) + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); + t = __rdtsc(); + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); +#else + asm volatile( + "lfence\n\t" + "rdtsc\n\t" + "shl $32, %%rdx\n\t" + "or %%rdx, %0\n\t" + "lfence" + : "=a"(t) + : + // "memory" avoids reordering. rdx = TSC >> 32. + // "cc" = flags modified by SHL. + : "rdx", "memory", "cc"); +#endif +#else + // Fall back to OS - unsure how to reliably query cntvct_el0 frequency. + timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + t = ts.tv_sec * 1000000000LL + ts.tv_nsec; +#endif + return t; +} + +inline uint64_t Stop64() { + uint64_t t; +#if defined(ABSL_ARCH_X86_64) +#if defined(ABSL_OS_WIN) + _ReadWriteBarrier(); + unsigned aux; + t = __rdtscp(&aux); + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); +#else + // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx). + asm volatile( + "rdtscp\n\t" + "shl $32, %%rdx\n\t" + "or %%rdx, %0\n\t" + "lfence" + : "=a"(t) + : + // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32. + // "cc" = flags modified by SHL. + : "rcx", "rdx", "memory", "cc"); +#endif +#else + t = Start64(); +#endif + return t; +} + +// Returns a 32-bit timestamp with about 4 cycles less overhead than +// Start64. Only suitable for measuring very short regions because the +// timestamp overflows about once a second. +inline uint32_t Start32() { + uint32_t t; +#if defined(ABSL_ARCH_X86_64) +#if defined(ABSL_OS_WIN) + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); + t = static_cast<uint32_t>(__rdtsc()); + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); +#else + asm volatile( + "lfence\n\t" + "rdtsc\n\t" + "lfence" + : "=a"(t) + : + // "memory" avoids reordering. rdx = TSC >> 32. + : "rdx", "memory"); +#endif +#else + t = static_cast<uint32_t>(Start64()); +#endif + return t; +} + +inline uint32_t Stop32() { + uint32_t t; +#if defined(ABSL_ARCH_X86_64) +#if defined(ABSL_OS_WIN) + _ReadWriteBarrier(); + unsigned aux; + t = static_cast<uint32_t>(__rdtscp(&aux)); + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); +#else + // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx). + asm volatile( + "rdtscp\n\t" + "lfence" + : "=a"(t) + : + // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32. + : "rcx", "rdx", "memory"); +#endif +#else + t = static_cast<uint32_t>(Stop64()); +#endif + return t; +} + +} // namespace timer + +namespace robust_statistics { + +// Sorts integral values in ascending order (e.g. for Mode). About 3x faster +// than std::sort for input distributions with very few unique values. +template <class T> +void CountingSort(T* values, size_t num_values) { + // Unique values and their frequency (similar to flat_map). + using Unique = std::pair<T, int>; + std::vector<Unique> unique; + for (size_t i = 0; i < num_values; ++i) { + const T value = values[i]; + const auto pos = + std::find_if(unique.begin(), unique.end(), + [value](const Unique u) { return u.first == value; }); + if (pos == unique.end()) { + unique.push_back(std::make_pair(value, 1)); + } else { + ++pos->second; + } + } + + // Sort in ascending order of value (pair.first). + std::sort(unique.begin(), unique.end()); + + // Write that many copies of each unique value to the array. + T* ABSL_RANDOM_INTERNAL_RESTRICT p = values; + for (const auto& value_count : unique) { + std::fill(p, p + value_count.second, value_count.first); + p += value_count.second; + } + ABSL_RAW_CHECK(p == values + num_values, "Did not produce enough output"); +} + +// @return i in [idx_begin, idx_begin + half_count) that minimizes +// sorted[i + half_count] - sorted[i]. +template <typename T> +size_t MinRange(const T* const ABSL_RANDOM_INTERNAL_RESTRICT sorted, + const size_t idx_begin, const size_t half_count) { + T min_range = (std::numeric_limits<T>::max)(); + size_t min_idx = 0; + + for (size_t idx = idx_begin; idx < idx_begin + half_count; ++idx) { + ABSL_RAW_CHECK(sorted[idx] <= sorted[idx + half_count], "Not sorted"); + const T range = sorted[idx + half_count] - sorted[idx]; + if (range < min_range) { + min_range = range; + min_idx = idx; + } + } + + return min_idx; +} + +// Returns an estimate of the mode by calling MinRange on successively +// halved intervals. "sorted" must be in ascending order. This is the +// Half Sample Mode estimator proposed by Bickel in "On a fast, robust +// estimator of the mode", with complexity O(N log N). The mode is less +// affected by outliers in highly-skewed distributions than the median. +// The averaging operation below assumes "T" is an unsigned integer type. +template <typename T> +T ModeOfSorted(const T* const ABSL_RANDOM_INTERNAL_RESTRICT sorted, + const size_t num_values) { + size_t idx_begin = 0; + size_t half_count = num_values / 2; + while (half_count > 1) { + idx_begin = MinRange(sorted, idx_begin, half_count); + half_count >>= 1; + } + + const T x = sorted[idx_begin + 0]; + if (half_count == 0) { + return x; + } + ABSL_RAW_CHECK(half_count == 1, "Should stop at half_count=1"); + const T average = (x + sorted[idx_begin + 1] + 1) / 2; + return average; +} + +// Returns the mode. Side effect: sorts "values". +template <typename T> +T Mode(T* values, const size_t num_values) { + CountingSort(values, num_values); + return ModeOfSorted(values, num_values); +} + +template <typename T, size_t N> +T Mode(T (&values)[N]) { + return Mode(&values[0], N); +} + +// Returns the median value. Side effect: sorts "values". +template <typename T> +T Median(T* values, const size_t num_values) { + ABSL_RAW_CHECK(num_values != 0, "Empty input"); + std::sort(values, values + num_values); + const size_t half = num_values / 2; + // Odd count: return middle + if (num_values % 2) { + return values[half]; + } + // Even count: return average of middle two. + return (values[half] + values[half - 1] + 1) / 2; +} + +// Returns a robust measure of variability. +template <typename T> +T MedianAbsoluteDeviation(const T* values, const size_t num_values, + const T median) { + ABSL_RAW_CHECK(num_values != 0, "Empty input"); + std::vector<T> abs_deviations; + abs_deviations.reserve(num_values); + for (size_t i = 0; i < num_values; ++i) { + const int64_t abs = std::abs(int64_t(values[i]) - int64_t(median)); + abs_deviations.push_back(static_cast<T>(abs)); + } + return Median(abs_deviations.data(), num_values); +} + +} // namespace robust_statistics + +// Ticks := platform-specific timer values (CPU cycles on x86). Must be +// unsigned to guarantee wraparound on overflow. 32 bit timers are faster to +// read than 64 bit. +using Ticks = uint32_t; + +// Returns timer overhead / minimum measurable difference. +Ticks TimerResolution() { + // Nested loop avoids exceeding stack/L1 capacity. + Ticks repetitions[Params::kTimerSamples]; + for (size_t rep = 0; rep < Params::kTimerSamples; ++rep) { + Ticks samples[Params::kTimerSamples]; + for (size_t i = 0; i < Params::kTimerSamples; ++i) { + const Ticks t0 = timer::Start32(); + const Ticks t1 = timer::Stop32(); + samples[i] = t1 - t0; + } + repetitions[rep] = robust_statistics::Mode(samples); + } + return robust_statistics::Mode(repetitions); +} + +static const Ticks timer_resolution = TimerResolution(); + +// Estimates the expected value of "lambda" values with a variable number of +// samples until the variability "rel_mad" is less than "max_rel_mad". +template <class Lambda> +Ticks SampleUntilStable(const double max_rel_mad, double* rel_mad, + const Params& p, const Lambda& lambda) { + auto measure_duration = [&lambda]() -> Ticks { + const Ticks t0 = timer::Start32(); + lambda(); + const Ticks t1 = timer::Stop32(); + return t1 - t0; + }; + + // Choose initial samples_per_eval based on a single estimated duration. + Ticks est = measure_duration(); + static const double ticks_per_second = InvariantTicksPerSecond(); + const size_t ticks_per_eval = ticks_per_second * p.seconds_per_eval; + size_t samples_per_eval = ticks_per_eval / est; + samples_per_eval = (std::max)(samples_per_eval, p.min_samples_per_eval); + + std::vector<Ticks> samples; + samples.reserve(1 + samples_per_eval); + samples.push_back(est); + + // Percentage is too strict for tiny differences, so also allow a small + // absolute "median absolute deviation". + const Ticks max_abs_mad = (timer_resolution + 99) / 100; + *rel_mad = 0.0; // ensure initialized + + for (size_t eval = 0; eval < p.max_evals; ++eval, samples_per_eval *= 2) { + samples.reserve(samples.size() + samples_per_eval); + for (size_t i = 0; i < samples_per_eval; ++i) { + const Ticks r = measure_duration(); + samples.push_back(r); + } + + if (samples.size() >= p.min_mode_samples) { + est = robust_statistics::Mode(samples.data(), samples.size()); + } else { + // For "few" (depends also on the variance) samples, Median is safer. + est = robust_statistics::Median(samples.data(), samples.size()); + } + ABSL_RAW_CHECK(est != 0, "Estimator returned zero duration"); + + // Median absolute deviation (mad) is a robust measure of 'variability'. + const Ticks abs_mad = robust_statistics::MedianAbsoluteDeviation( + samples.data(), samples.size(), est); + *rel_mad = static_cast<double>(static_cast<int>(abs_mad)) / est; + + if (*rel_mad <= max_rel_mad || abs_mad <= max_abs_mad) { + if (p.verbose) { + ABSL_RAW_LOG(INFO, + "%6zu samples => %5u (abs_mad=%4u, rel_mad=%4.2f%%)\n", + samples.size(), est, abs_mad, *rel_mad * 100.0); + } + return est; + } + } + + if (p.verbose) { + ABSL_RAW_LOG(WARNING, + "rel_mad=%4.2f%% still exceeds %4.2f%% after %6zu samples.\n", + *rel_mad * 100.0, max_rel_mad * 100.0, samples.size()); + } + return est; +} + +using InputVec = std::vector<FuncInput>; + +// Returns vector of unique input values. +InputVec UniqueInputs(const FuncInput* inputs, const size_t num_inputs) { + InputVec unique(inputs, inputs + num_inputs); + std::sort(unique.begin(), unique.end()); + unique.erase(std::unique(unique.begin(), unique.end()), unique.end()); + return unique; +} + +// Returns how often we need to call func for sufficient precision, or zero +// on failure (e.g. the elapsed time is too long for a 32-bit tick count). +size_t NumSkip(const Func func, const void* arg, const InputVec& unique, + const Params& p) { + // Min elapsed ticks for any input. + Ticks min_duration = ~0u; + + for (const FuncInput input : unique) { + // Make sure a 32-bit timer is sufficient. + const uint64_t t0 = timer::Start64(); + PreventElision(func(arg, input)); + const uint64_t t1 = timer::Stop64(); + const uint64_t elapsed = t1 - t0; + if (elapsed >= (1ULL << 30)) { + ABSL_RAW_LOG(WARNING, + "Measurement failed: need 64-bit timer for input=%zu\n", + static_cast<size_t>(input)); + return 0; + } + + double rel_mad; + const Ticks total = SampleUntilStable( + p.target_rel_mad, &rel_mad, p, + [func, arg, input]() { PreventElision(func(arg, input)); }); + min_duration = (std::min)(min_duration, total - timer_resolution); + } + + // Number of repetitions required to reach the target resolution. + const size_t max_skip = p.precision_divisor; + // Number of repetitions given the estimated duration. + const size_t num_skip = + min_duration == 0 ? 0 : (max_skip + min_duration - 1) / min_duration; + if (p.verbose) { + ABSL_RAW_LOG(INFO, "res=%u max_skip=%zu min_dur=%u num_skip=%zu\n", + timer_resolution, max_skip, min_duration, num_skip); + } + return num_skip; +} + +// Replicates inputs until we can omit "num_skip" occurrences of an input. +InputVec ReplicateInputs(const FuncInput* inputs, const size_t num_inputs, + const size_t num_unique, const size_t num_skip, + const Params& p) { + InputVec full; + if (num_unique == 1) { + full.assign(p.subset_ratio * num_skip, inputs[0]); + return full; + } + + full.reserve(p.subset_ratio * num_skip * num_inputs); + for (size_t i = 0; i < p.subset_ratio * num_skip; ++i) { + full.insert(full.end(), inputs, inputs + num_inputs); + } + absl::random_internal::randen_engine<uint32_t> rng; + std::shuffle(full.begin(), full.end(), rng); + return full; +} + +// Copies the "full" to "subset" in the same order, but with "num_skip" +// randomly selected occurrences of "input_to_skip" removed. +void FillSubset(const InputVec& full, const FuncInput input_to_skip, + const size_t num_skip, InputVec* subset) { + const size_t count = std::count(full.begin(), full.end(), input_to_skip); + // Generate num_skip random indices: which occurrence to skip. + std::vector<uint32_t> omit; + // Replacement for std::iota, not yet available in MSVC builds. + omit.reserve(count); + for (size_t i = 0; i < count; ++i) { + omit.push_back(i); + } + // omit[] is the same on every call, but that's OK because they identify the + // Nth instance of input_to_skip, so the position within full[] differs. + absl::random_internal::randen_engine<uint32_t> rng; + std::shuffle(omit.begin(), omit.end(), rng); + omit.resize(num_skip); + std::sort(omit.begin(), omit.end()); + + uint32_t occurrence = ~0u; // 0 after preincrement + size_t idx_omit = 0; // cursor within omit[] + size_t idx_subset = 0; // cursor within *subset + for (const FuncInput next : full) { + if (next == input_to_skip) { + ++occurrence; + // Haven't removed enough already + if (idx_omit < num_skip) { + // This one is up for removal + if (occurrence == omit[idx_omit]) { + ++idx_omit; + continue; + } + } + } + if (idx_subset < subset->size()) { + (*subset)[idx_subset++] = next; + } + } + ABSL_RAW_CHECK(idx_subset == subset->size(), "idx_subset not at end"); + ABSL_RAW_CHECK(idx_omit == omit.size(), "idx_omit not at end"); + ABSL_RAW_CHECK(occurrence == count - 1, "occurrence not at end"); +} + +// Returns total ticks elapsed for all inputs. +Ticks TotalDuration(const Func func, const void* arg, const InputVec* inputs, + const Params& p, double* max_rel_mad) { + double rel_mad; + const Ticks duration = + SampleUntilStable(p.target_rel_mad, &rel_mad, p, [func, arg, inputs]() { + for (const FuncInput input : *inputs) { + PreventElision(func(arg, input)); + } + }); + *max_rel_mad = (std::max)(*max_rel_mad, rel_mad); + return duration; +} + +// (Nearly) empty Func for measuring timer overhead/resolution. +ABSL_RANDOM_INTERNAL_ATTRIBUTE_NEVER_INLINE FuncOutput +EmptyFunc(const void* arg, const FuncInput input) { + return input; +} + +// Returns overhead of accessing inputs[] and calling a function; this will +// be deducted from future TotalDuration return values. +Ticks Overhead(const void* arg, const InputVec* inputs, const Params& p) { + double rel_mad; + // Zero tolerance because repeatability is crucial and EmptyFunc is fast. + return SampleUntilStable(0.0, &rel_mad, p, [arg, inputs]() { + for (const FuncInput input : *inputs) { + PreventElision(EmptyFunc(arg, input)); + } + }); +} + +} // namespace + +void PinThreadToCPU(int cpu) { + // We might migrate to another CPU before pinning below, but at least cpu + // will be one of the CPUs on which this thread ran. +#if defined(ABSL_OS_WIN) + if (cpu < 0) { + cpu = static_cast<int>(GetCurrentProcessorNumber()); + ABSL_RAW_CHECK(cpu >= 0, "PinThreadToCPU detect failed"); + if (cpu >= 64) { + // NOTE: On wine, at least, GetCurrentProcessorNumber() sometimes returns + // a value > 64, which is out of range. When this happens, log a message + // and don't set a cpu affinity. + ABSL_RAW_LOG(ERROR, "Invalid CPU number: %d", cpu); + return; + } + } else if (cpu >= 64) { + // User specified an explicit CPU affinity > the valid range. + ABSL_RAW_LOG(FATAL, "Invalid CPU number: %d", cpu); + } + const DWORD_PTR prev = SetThreadAffinityMask(GetCurrentThread(), 1ULL << cpu); + ABSL_RAW_CHECK(prev != 0, "SetAffinity failed"); +#elif defined(ABSL_OS_LINUX) && !defined(ABSL_OS_ANDROID) + if (cpu < 0) { + cpu = sched_getcpu(); + ABSL_RAW_CHECK(cpu >= 0, "PinThreadToCPU detect failed"); + } + const pid_t pid = 0; // current thread + cpu_set_t set; + CPU_ZERO(&set); + CPU_SET(cpu, &set); + const int err = sched_setaffinity(pid, sizeof(set), &set); + ABSL_RAW_CHECK(err == 0, "SetAffinity failed"); +#endif +} + +// Returns tick rate. Invariant means the tick counter frequency is independent +// of CPU throttling or sleep. May be expensive, caller should cache the result. +double InvariantTicksPerSecond() { +#if defined(ABSL_ARCH_PPC) + return __ppc_get_timebase_freq(); +#elif defined(ABSL_ARCH_X86_64) + // We assume the TSC is invariant; it is on all recent Intel/AMD CPUs. + return platform::NominalClockRate(); +#else + // Fall back to clock_gettime nanoseconds. + return 1E9; +#endif +} + +size_t MeasureImpl(const Func func, const void* arg, const size_t num_skip, + const InputVec& unique, const InputVec& full, + const Params& p, Result* results) { + const float mul = 1.0f / static_cast<int>(num_skip); + + InputVec subset(full.size() - num_skip); + const Ticks overhead = Overhead(arg, &full, p); + const Ticks overhead_skip = Overhead(arg, &subset, p); + if (overhead < overhead_skip) { + ABSL_RAW_LOG(WARNING, "Measurement failed: overhead %u < %u\n", overhead, + overhead_skip); + return 0; + } + + if (p.verbose) { + ABSL_RAW_LOG(INFO, "#inputs=%5zu,%5zu overhead=%5u,%5u\n", full.size(), + subset.size(), overhead, overhead_skip); + } + + double max_rel_mad = 0.0; + const Ticks total = TotalDuration(func, arg, &full, p, &max_rel_mad); + + for (size_t i = 0; i < unique.size(); ++i) { + FillSubset(full, unique[i], num_skip, &subset); + const Ticks total_skip = TotalDuration(func, arg, &subset, p, &max_rel_mad); + + if (total < total_skip) { + ABSL_RAW_LOG(WARNING, "Measurement failed: total %u < %u\n", total, + total_skip); + return 0; + } + + const Ticks duration = (total - overhead) - (total_skip - overhead_skip); + results[i].input = unique[i]; + results[i].ticks = duration * mul; + results[i].variability = max_rel_mad; + } + + return unique.size(); +} + +size_t Measure(const Func func, const void* arg, const FuncInput* inputs, + const size_t num_inputs, Result* results, const Params& p) { + ABSL_RAW_CHECK(num_inputs != 0, "No inputs"); + + const InputVec unique = UniqueInputs(inputs, num_inputs); + const size_t num_skip = NumSkip(func, arg, unique, p); // never 0 + if (num_skip == 0) return 0; // NumSkip already printed error message + + const InputVec full = + ReplicateInputs(inputs, num_inputs, unique.size(), num_skip, p); + + // MeasureImpl may fail up to p.max_measure_retries times. + for (size_t i = 0; i < p.max_measure_retries; i++) { + auto result = MeasureImpl(func, arg, num_skip, unique, full, p, results); + if (result != 0) { + return result; + } + } + // All retries failed. (Unusual) + return 0; +} + +} // namespace random_internal_nanobenchmark +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/nanobenchmark.h b/third_party/abseil_cpp/absl/random/internal/nanobenchmark.h new file mode 100644 index 000000000000..a5097ba27b0f --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/nanobenchmark.h @@ -0,0 +1,172 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_NANOBENCHMARK_H_ +#define ABSL_RANDOM_INTERNAL_NANOBENCHMARK_H_ + +// Benchmarks functions of a single integer argument with realistic branch +// prediction hit rates. Uses a robust estimator to summarize the measurements. +// The precision is about 0.2%. +// +// Examples: see nanobenchmark_test.cc. +// +// Background: Microbenchmarks such as http://github.com/google/benchmark +// can measure elapsed times on the order of a microsecond. Shorter functions +// are typically measured by repeating them thousands of times and dividing +// the total elapsed time by this count. Unfortunately, repetition (especially +// with the same input parameter!) influences the runtime. In time-critical +// code, it is reasonable to expect warm instruction/data caches and TLBs, +// but a perfect record of which branches will be taken is unrealistic. +// Unless the application also repeatedly invokes the measured function with +// the same parameter, the benchmark is measuring something very different - +// a best-case result, almost as if the parameter were made a compile-time +// constant. This may lead to erroneous conclusions about branch-heavy +// algorithms outperforming branch-free alternatives. +// +// Our approach differs in three ways. Adding fences to the timer functions +// reduces variability due to instruction reordering, improving the timer +// resolution to about 40 CPU cycles. However, shorter functions must still +// be invoked repeatedly. For more realistic branch prediction performance, +// we vary the input parameter according to a user-specified distribution. +// Thus, instead of VaryInputs(Measure(Repeat(func))), we change the +// loop nesting to Measure(Repeat(VaryInputs(func))). We also estimate the +// central tendency of the measurement samples with the "half sample mode", +// which is more robust to outliers and skewed data than the mean or median. + +// NOTE: for compatibility with multiple translation units compiled with +// distinct flags, avoid #including headers that define functions. + +#include <stddef.h> +#include <stdint.h> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal_nanobenchmark { + +// Input influencing the function being measured (e.g. number of bytes to copy). +using FuncInput = size_t; + +// "Proof of work" returned by Func to ensure the compiler does not elide it. +using FuncOutput = uint64_t; + +// Function to measure: either 1) a captureless lambda or function with two +// arguments or 2) a lambda with capture, in which case the first argument +// is reserved for use by MeasureClosure. +using Func = FuncOutput (*)(const void*, FuncInput); + +// Internal parameters that determine precision/resolution/measuring time. +struct Params { + // For measuring timer overhead/resolution. Used in a nested loop => + // quadratic time, acceptable because we know timer overhead is "low". + // constexpr because this is used to define array bounds. + static constexpr size_t kTimerSamples = 256; + + // Best-case precision, expressed as a divisor of the timer resolution. + // Larger => more calls to Func and higher precision. + size_t precision_divisor = 1024; + + // Ratio between full and subset input distribution sizes. Cannot be less + // than 2; larger values increase measurement time but more faithfully + // model the given input distribution. + size_t subset_ratio = 2; + + // Together with the estimated Func duration, determines how many times to + // call Func before checking the sample variability. Larger values increase + // measurement time, memory/cache use and precision. + double seconds_per_eval = 4E-3; + + // The minimum number of samples before estimating the central tendency. + size_t min_samples_per_eval = 7; + + // The mode is better than median for estimating the central tendency of + // skewed/fat-tailed distributions, but it requires sufficient samples + // relative to the width of half-ranges. + size_t min_mode_samples = 64; + + // Maximum permissible variability (= median absolute deviation / center). + double target_rel_mad = 0.002; + + // Abort after this many evals without reaching target_rel_mad. This + // prevents infinite loops. + size_t max_evals = 9; + + // Retry the measure loop up to this many times. + size_t max_measure_retries = 2; + + // Whether to print additional statistics to stdout. + bool verbose = true; +}; + +// Measurement result for each unique input. +struct Result { + FuncInput input; + + // Robust estimate (mode or median) of duration. + float ticks; + + // Measure of variability (median absolute deviation relative to "ticks"). + float variability; +}; + +// Ensures the thread is running on the specified cpu, and no others. +// Reduces noise due to desynchronized socket RDTSC and context switches. +// If "cpu" is negative, pin to the currently running core. +void PinThreadToCPU(const int cpu = -1); + +// Returns tick rate, useful for converting measurements to seconds. Invariant +// means the tick counter frequency is independent of CPU throttling or sleep. +// This call may be expensive, callers should cache the result. +double InvariantTicksPerSecond(); + +// Precisely measures the number of ticks elapsed when calling "func" with the +// given inputs, shuffled to ensure realistic branch prediction hit rates. +// +// "func" returns a 'proof of work' to ensure its computations are not elided. +// "arg" is passed to Func, or reserved for internal use by MeasureClosure. +// "inputs" is an array of "num_inputs" (not necessarily unique) arguments to +// "func". The values should be chosen to maximize coverage of "func". This +// represents a distribution, so a value's frequency should reflect its +// probability in the real application. Order does not matter; for example, a +// uniform distribution over [0, 4) could be represented as {3,0,2,1}. +// Returns how many Result were written to "results": one per unique input, or +// zero if the measurement failed (an error message goes to stderr). +size_t Measure(const Func func, const void* arg, const FuncInput* inputs, + const size_t num_inputs, Result* results, + const Params& p = Params()); + +// Calls operator() of the given closure (lambda function). +template <class Closure> +static FuncOutput CallClosure(const void* f, const FuncInput input) { + return (*reinterpret_cast<const Closure*>(f))(input); +} + +// Same as Measure, except "closure" is typically a lambda function of +// FuncInput -> FuncOutput with a capture list. +template <class Closure> +static inline size_t MeasureClosure(const Closure& closure, + const FuncInput* inputs, + const size_t num_inputs, Result* results, + const Params& p = Params()) { + return Measure(reinterpret_cast<Func>(&CallClosure<Closure>), + reinterpret_cast<const void*>(&closure), inputs, num_inputs, + results, p); +} + +} // namespace random_internal_nanobenchmark +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_NANOBENCHMARK_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/nanobenchmark_test.cc b/third_party/abseil_cpp/absl/random/internal/nanobenchmark_test.cc new file mode 100644 index 000000000000..f1571e269ff5 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/nanobenchmark_test.cc @@ -0,0 +1,77 @@ +// Copyright 2017 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/nanobenchmark.h" + +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/numbers.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal_nanobenchmark { +namespace { + +uint64_t Div(const void*, FuncInput in) { + // Here we're measuring the throughput because benchmark invocations are + // independent. + const int64_t d1 = 0xFFFFFFFFFFll / int64_t(in); // IDIV + return d1; +} + +template <size_t N> +void MeasureDiv(const FuncInput (&inputs)[N]) { + Result results[N]; + Params params; + params.max_evals = 6; // avoid test timeout + const size_t num_results = Measure(&Div, nullptr, inputs, N, results, params); + if (num_results == 0) { + ABSL_RAW_LOG( + WARNING, + "WARNING: Measurement failed, should not happen when using " + "PinThreadToCPU unless the region to measure takes > 1 second.\n"); + return; + } + for (size_t i = 0; i < num_results; ++i) { + ABSL_RAW_LOG(INFO, "%5zu: %6.2f ticks; MAD=%4.2f%%\n", results[i].input, + results[i].ticks, results[i].variability * 100.0); + ABSL_RAW_CHECK(results[i].ticks != 0.0f, "Zero duration"); + } +} + +void RunAll(const int argc, char* argv[]) { + // Avoid migrating between cores - important on multi-socket systems. + int cpu = -1; + if (argc == 2) { + if (!absl::SimpleAtoi(argv[1], &cpu)) { + ABSL_RAW_LOG(FATAL, "The optional argument must be a CPU number >= 0.\n"); + } + } + PinThreadToCPU(cpu); + + // unpredictable == 1 but the compiler doesn't know that. + const FuncInput unpredictable = argc != 999; + static const FuncInput inputs[] = {unpredictable * 10, unpredictable * 100}; + + MeasureDiv(inputs); +} + +} // namespace +} // namespace random_internal_nanobenchmark +ABSL_NAMESPACE_END +} // namespace absl + +int main(int argc, char* argv[]) { + absl::random_internal_nanobenchmark::RunAll(argc, argv); + return 0; +} diff --git a/third_party/abseil_cpp/absl/random/internal/nonsecure_base.h b/third_party/abseil_cpp/absl/random/internal/nonsecure_base.h new file mode 100644 index 000000000000..730fa2ea12d2 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/nonsecure_base.h @@ -0,0 +1,150 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_NONSECURE_BASE_H_ +#define ABSL_RANDOM_INTERNAL_NONSECURE_BASE_H_ + +#include <algorithm> +#include <cstdint> +#include <iostream> +#include <iterator> +#include <random> +#include <string> +#include <type_traits> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" +#include "absl/random/internal/pool_urbg.h" +#include "absl/random/internal/salted_seed_seq.h" +#include "absl/random/internal/seed_material.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Each instance of NonsecureURBGBase<URBG> will be seeded by variates produced +// by a thread-unique URBG-instance. +template <typename URBG> +class NonsecureURBGBase { + public: + using result_type = typename URBG::result_type; + + // Default constructor + NonsecureURBGBase() : urbg_(ConstructURBG()) {} + + // Copy disallowed, move allowed. + NonsecureURBGBase(const NonsecureURBGBase&) = delete; + NonsecureURBGBase& operator=(const NonsecureURBGBase&) = delete; + NonsecureURBGBase(NonsecureURBGBase&&) = default; + NonsecureURBGBase& operator=(NonsecureURBGBase&&) = default; + + // Constructor using a seed + template <class SSeq, typename = typename absl::enable_if_t< + !std::is_same<SSeq, NonsecureURBGBase>::value>> + explicit NonsecureURBGBase(SSeq&& seq) + : urbg_(ConstructURBG(std::forward<SSeq>(seq))) {} + + // Note: on MSVC, min() or max() can be interpreted as MIN() or MAX(), so we + // enclose min() or max() in parens as (min)() and (max)(). + // Additionally, clang-format requires no space before this construction. + + // NonsecureURBGBase::min() + static constexpr result_type(min)() { return (URBG::min)(); } + + // NonsecureURBGBase::max() + static constexpr result_type(max)() { return (URBG::max)(); } + + // NonsecureURBGBase::operator()() + result_type operator()() { return urbg_(); } + + // NonsecureURBGBase::discard() + void discard(unsigned long long values) { // NOLINT(runtime/int) + urbg_.discard(values); + } + + bool operator==(const NonsecureURBGBase& other) const { + return urbg_ == other.urbg_; + } + + bool operator!=(const NonsecureURBGBase& other) const { + return !(urbg_ == other.urbg_); + } + + private: + // Seeder is a custom seed sequence type where generate() fills the provided + // buffer via the RandenPool entropy source. + struct Seeder { + using result_type = uint32_t; + + size_t size() { return 0; } + + template <typename OutIterator> + void param(OutIterator) const {} + + template <typename RandomAccessIterator> + void generate(RandomAccessIterator begin, RandomAccessIterator end) { + if (begin != end) { + // begin, end must be random access iterators assignable from uint32_t. + generate_impl( + std::integral_constant<bool, sizeof(*begin) == sizeof(uint32_t)>{}, + begin, end); + } + } + + // Commonly, generate is invoked with a pointer to a buffer which + // can be cast to a uint32_t. + template <typename RandomAccessIterator> + void generate_impl(std::integral_constant<bool, true>, + RandomAccessIterator begin, RandomAccessIterator end) { + auto buffer = absl::MakeSpan(begin, end); + auto target = absl::MakeSpan(reinterpret_cast<uint32_t*>(buffer.data()), + buffer.size()); + RandenPool<uint32_t>::Fill(target); + } + + // The non-uint32_t case should be uncommon, and involves an extra copy, + // filling the uint32_t buffer and then mixing into the output. + template <typename RandomAccessIterator> + void generate_impl(std::integral_constant<bool, false>, + RandomAccessIterator begin, RandomAccessIterator end) { + const size_t n = std::distance(begin, end); + absl::InlinedVector<uint32_t, 8> data(n, 0); + RandenPool<uint32_t>::Fill(absl::MakeSpan(data.begin(), data.end())); + std::copy(std::begin(data), std::end(data), begin); + } + }; + + static URBG ConstructURBG() { + Seeder seeder; + return URBG(seeder); + } + + template <typename SSeq> + static URBG ConstructURBG(SSeq&& seq) { // NOLINT(runtime/references) + auto salted_seq = + random_internal::MakeSaltedSeedSeq(std::forward<SSeq>(seq)); + return URBG(salted_seq); + } + + URBG urbg_; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_NONSECURE_BASE_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/nonsecure_base_test.cc b/third_party/abseil_cpp/absl/random/internal/nonsecure_base_test.cc new file mode 100644 index 000000000000..698027fc6e0d --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/nonsecure_base_test.cc @@ -0,0 +1,245 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/nonsecure_base.h" + +#include <algorithm> +#include <iostream> +#include <memory> +#include <random> +#include <sstream> + +#include "gtest/gtest.h" +#include "absl/random/distributions.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" + +namespace { + +using ExampleNonsecureURBG = + absl::random_internal::NonsecureURBGBase<std::mt19937>; + +template <typename T> +void Use(const T&) {} + +} // namespace + +TEST(NonsecureURBGBase, DefaultConstructorIsValid) { + ExampleNonsecureURBG urbg; +} + +// Ensure that the recommended template-instantiations are valid. +TEST(RecommendedTemplates, CanBeConstructed) { + absl::BitGen default_generator; + absl::InsecureBitGen insecure_generator; +} + +TEST(RecommendedTemplates, CanDiscardValues) { + absl::BitGen default_generator; + absl::InsecureBitGen insecure_generator; + + default_generator.discard(5); + insecure_generator.discard(5); +} + +TEST(NonsecureURBGBase, StandardInterface) { + // Names after definition of [rand.req.urbg] in C++ standard. + // e us a value of E + // v is a lvalue of E + // x, y are possibly const values of E + // s is a value of T + // q is a value satisfying requirements of seed_sequence + // z is a value of type unsigned long long + // os is a some specialization of basic_ostream + // is is a some specialization of basic_istream + + using E = absl::random_internal::NonsecureURBGBase<std::minstd_rand>; + + using T = typename E::result_type; + + static_assert(!std::is_copy_constructible<E>::value, + "NonsecureURBGBase should not be copy constructible"); + + static_assert(!absl::is_copy_assignable<E>::value, + "NonsecureURBGBase should not be copy assignable"); + + static_assert(std::is_move_constructible<E>::value, + "NonsecureURBGBase should be move constructible"); + + static_assert(absl::is_move_assignable<E>::value, + "NonsecureURBGBase should be move assignable"); + + static_assert(std::is_same<decltype(std::declval<E>()()), T>::value, + "return type of operator() must be result_type"); + + { + const E x, y; + Use(x); + Use(y); + + static_assert(std::is_same<decltype(x == y), bool>::value, + "return type of operator== must be bool"); + + static_assert(std::is_same<decltype(x != y), bool>::value, + "return type of operator== must be bool"); + } + + E e; + std::seed_seq q{1, 2, 3}; + + E{}; + E{q}; + + // Copy constructor not supported. + // E{x}; + + // result_type seed constructor not supported. + // E{T{1}}; + + // Move constructors are supported. + { + E tmp(q); + E m = std::move(tmp); + E n(std::move(m)); + EXPECT_TRUE(e != n); + } + + // Comparisons work. + { + // MSVC emits error 2718 when using EXPECT_EQ(e, x) + // * actual parameter with __declspec(align('#')) won't be aligned + E a(q); + E b(q); + + EXPECT_TRUE(a != e); + EXPECT_TRUE(a == b); + + a(); + EXPECT_TRUE(a != b); + } + + // e.seed(s) not supported. + + // [rand.req.eng] specifies the parameter as 'unsigned long long' + // e.discard(unsigned long long) is supported. + unsigned long long z = 1; // NOLINT(runtime/int) + e.discard(z); +} + +TEST(NonsecureURBGBase, SeedSeqConstructorIsValid) { + std::seed_seq seq; + ExampleNonsecureURBG rbg(seq); +} + +TEST(NonsecureURBGBase, CompatibleWithDistributionUtils) { + ExampleNonsecureURBG rbg; + + absl::Uniform(rbg, 0, 100); + absl::Uniform(rbg, 0.5, 0.7); + absl::Poisson<uint32_t>(rbg); + absl::Exponential<float>(rbg); +} + +TEST(NonsecureURBGBase, CompatibleWithStdDistributions) { + ExampleNonsecureURBG rbg; + + // Cast to void to suppress [[nodiscard]] warnings + static_cast<void>(std::uniform_int_distribution<uint32_t>(0, 100)(rbg)); + static_cast<void>(std::uniform_real_distribution<float>()(rbg)); + static_cast<void>(std::bernoulli_distribution(0.2)(rbg)); +} + +TEST(NonsecureURBGBase, ConsecutiveDefaultInstancesYieldUniqueVariates) { + const size_t kNumSamples = 128; + + ExampleNonsecureURBG rbg1; + ExampleNonsecureURBG rbg2; + + for (size_t i = 0; i < kNumSamples; i++) { + EXPECT_NE(rbg1(), rbg2()); + } +} + +TEST(NonsecureURBGBase, EqualSeedSequencesYieldEqualVariates) { + std::seed_seq seq; + + ExampleNonsecureURBG rbg1(seq); + ExampleNonsecureURBG rbg2(seq); + + // ExampleNonsecureURBG rbg3({1, 2, 3}); // Should not compile. + + for (uint32_t i = 0; i < 1000; i++) { + EXPECT_EQ(rbg1(), rbg2()); + } + + rbg1.discard(100); + rbg2.discard(100); + + // The sequences should continue after discarding + for (uint32_t i = 0; i < 1000; i++) { + EXPECT_EQ(rbg1(), rbg2()); + } +} + +// This is a PRNG-compatible type specifically designed to test +// that NonsecureURBGBase::Seeder can correctly handle iterators +// to arbitrary non-uint32_t size types. +template <typename T> +struct SeederTestEngine { + using result_type = T; + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + template <class SeedSequence, + typename = typename absl::enable_if_t< + !std::is_same<SeedSequence, SeederTestEngine>::value>> + explicit SeederTestEngine(SeedSequence&& seq) { + seed(seq); + } + + SeederTestEngine(const SeederTestEngine&) = default; + SeederTestEngine& operator=(const SeederTestEngine&) = default; + SeederTestEngine(SeederTestEngine&&) = default; + SeederTestEngine& operator=(SeederTestEngine&&) = default; + + result_type operator()() { return state[0]; } + + template <class SeedSequence> + void seed(SeedSequence&& seq) { + std::fill(std::begin(state), std::end(state), T(0)); + seq.generate(std::begin(state), std::end(state)); + } + + T state[2]; +}; + +TEST(NonsecureURBGBase, SeederWorksForU32) { + using U32 = + absl::random_internal::NonsecureURBGBase<SeederTestEngine<uint32_t>>; + U32 x; + EXPECT_NE(0, x()); +} + +TEST(NonsecureURBGBase, SeederWorksForU64) { + using U64 = + absl::random_internal::NonsecureURBGBase<SeederTestEngine<uint64_t>>; + + U64 x; + EXPECT_NE(0, x()); +} diff --git a/third_party/abseil_cpp/absl/random/internal/pcg_engine.h b/third_party/abseil_cpp/absl/random/internal/pcg_engine.h new file mode 100644 index 000000000000..53c23fe1b492 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/pcg_engine.h @@ -0,0 +1,307 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_PCG_ENGINE_H_ +#define ABSL_RANDOM_INTERNAL_PCG_ENGINE_H_ + +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/meta/type_traits.h" +#include "absl/numeric/int128.h" +#include "absl/random/internal/fastmath.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// pcg_engine is a simplified implementation of Melissa O'Neil's PCG engine in +// C++. PCG combines a linear congruential generator (LCG) with output state +// mixing functions to generate each random variate. pcg_engine supports only a +// single sequence (oneseq), and does not support streams. +// +// pcg_engine is parameterized by two types: +// Params, which provides the multiplier and increment values; +// Mix, which mixes the state into the result. +// +template <typename Params, typename Mix> +class pcg_engine { + static_assert(std::is_same<typename Params::state_type, + typename Mix::state_type>::value, + "Class-template absl::pcg_engine must be parameterized by " + "Params and Mix with identical state_type"); + + static_assert(std::is_unsigned<typename Mix::result_type>::value, + "Class-template absl::pcg_engine must be parameterized by " + "an unsigned Mix::result_type"); + + using params_type = Params; + using mix_type = Mix; + using state_type = typename Mix::state_type; + + public: + // C++11 URBG interface: + using result_type = typename Mix::result_type; + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + explicit pcg_engine(uint64_t seed_value = 0) { seed(seed_value); } + + template <class SeedSequence, + typename = typename absl::enable_if_t< + !std::is_same<SeedSequence, pcg_engine>::value>> + explicit pcg_engine(SeedSequence&& seq) { + seed(seq); + } + + pcg_engine(const pcg_engine&) = default; + pcg_engine& operator=(const pcg_engine&) = default; + pcg_engine(pcg_engine&&) = default; + pcg_engine& operator=(pcg_engine&&) = default; + + result_type operator()() { + // Advance the LCG state, always using the new value to generate the output. + state_ = lcg(state_); + return Mix{}(state_); + } + + void seed(uint64_t seed_value = 0) { + state_type tmp = seed_value; + state_ = lcg(tmp + Params::increment()); + } + + template <class SeedSequence> + typename absl::enable_if_t< + !std::is_convertible<SeedSequence, uint64_t>::value, void> + seed(SeedSequence&& seq) { + reseed(seq); + } + + void discard(uint64_t count) { state_ = advance(state_, count); } + + bool operator==(const pcg_engine& other) const { + return state_ == other.state_; + } + + bool operator!=(const pcg_engine& other) const { return !(*this == other); } + + template <class CharT, class Traits> + friend typename absl::enable_if_t<(sizeof(state_type) == 16), + std::basic_ostream<CharT, Traits>&> + operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const pcg_engine& engine) { + auto saver = random_internal::make_ostream_state_saver(os); + random_internal::stream_u128_helper<state_type> helper; + helper.write(pcg_engine::params_type::multiplier(), os); + os << os.fill(); + helper.write(pcg_engine::params_type::increment(), os); + os << os.fill(); + helper.write(engine.state_, os); + return os; + } + + template <class CharT, class Traits> + friend typename absl::enable_if_t<(sizeof(state_type) <= 8), + std::basic_ostream<CharT, Traits>&> + operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const pcg_engine& engine) { + auto saver = random_internal::make_ostream_state_saver(os); + os << pcg_engine::params_type::multiplier() << os.fill(); + os << pcg_engine::params_type::increment() << os.fill(); + os << engine.state_; + return os; + } + + template <class CharT, class Traits> + friend typename absl::enable_if_t<(sizeof(state_type) == 16), + std::basic_istream<CharT, Traits>&> + operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + pcg_engine& engine) { // NOLINT(runtime/references) + random_internal::stream_u128_helper<state_type> helper; + auto mult = helper.read(is); + auto inc = helper.read(is); + auto tmp = helper.read(is); + if (mult != pcg_engine::params_type::multiplier() || + inc != pcg_engine::params_type::increment()) { + // signal failure by setting the failbit. + is.setstate(is.rdstate() | std::ios_base::failbit); + } + if (!is.fail()) { + engine.state_ = tmp; + } + return is; + } + + template <class CharT, class Traits> + friend typename absl::enable_if_t<(sizeof(state_type) <= 8), + std::basic_istream<CharT, Traits>&> + operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + pcg_engine& engine) { // NOLINT(runtime/references) + state_type mult{}, inc{}, tmp{}; + is >> mult >> inc >> tmp; + if (mult != pcg_engine::params_type::multiplier() || + inc != pcg_engine::params_type::increment()) { + // signal failure by setting the failbit. + is.setstate(is.rdstate() | std::ios_base::failbit); + } + if (!is.fail()) { + engine.state_ = tmp; + } + return is; + } + + private: + state_type state_; + + // Returns the linear-congruential generator next state. + static inline constexpr state_type lcg(state_type s) { + return s * Params::multiplier() + Params::increment(); + } + + // Returns the linear-congruential arbitrary seek state. + inline state_type advance(state_type s, uint64_t n) const { + state_type mult = Params::multiplier(); + state_type inc = Params::increment(); + state_type m = 1; + state_type i = 0; + while (n > 0) { + if (n & 1) { + m *= mult; + i = i * mult + inc; + } + inc = (mult + 1) * inc; + mult *= mult; + n >>= 1; + } + return m * s + i; + } + + template <class SeedSequence> + void reseed(SeedSequence& seq) { + using sequence_result_type = typename SeedSequence::result_type; + constexpr size_t kBufferSize = + sizeof(state_type) / sizeof(sequence_result_type); + sequence_result_type buffer[kBufferSize]; + seq.generate(std::begin(buffer), std::end(buffer)); + // Convert the seed output to a single state value. + state_type tmp = buffer[0]; + for (size_t i = 1; i < kBufferSize; i++) { + tmp <<= (sizeof(sequence_result_type) * 8); + tmp |= buffer[i]; + } + state_ = lcg(tmp + params_type::increment()); + } +}; + +// Parameterized implementation of the PCG 128-bit oneseq state. +// This provides state_type, multiplier, and increment for pcg_engine. +template <uint64_t kMultA, uint64_t kMultB, uint64_t kIncA, uint64_t kIncB> +class pcg128_params { + public: +#if ABSL_HAVE_INTRINSIC_INT128 + using state_type = __uint128_t; + static inline constexpr state_type make_u128(uint64_t a, uint64_t b) { + return (static_cast<__uint128_t>(a) << 64) | b; + } +#else + using state_type = absl::uint128; + static inline constexpr state_type make_u128(uint64_t a, uint64_t b) { + return absl::MakeUint128(a, b); + } +#endif + + static inline constexpr state_type multiplier() { + return make_u128(kMultA, kMultB); + } + static inline constexpr state_type increment() { + return make_u128(kIncA, kIncB); + } +}; + +// Implementation of the PCG xsl_rr_128_64 128-bit mixing function, which +// accepts an input of state_type and mixes it into an output of result_type. +struct pcg_xsl_rr_128_64 { +#if ABSL_HAVE_INTRINSIC_INT128 + using state_type = __uint128_t; +#else + using state_type = absl::uint128; +#endif + using result_type = uint64_t; + + inline uint64_t operator()(state_type state) { + // This is equivalent to the xsl_rr_128_64 mixing function. +#if ABSL_HAVE_INTRINSIC_INT128 + uint64_t rotate = static_cast<uint64_t>(state >> 122u); + state ^= state >> 64; + uint64_t s = static_cast<uint64_t>(state); +#else + uint64_t h = Uint128High64(state); + uint64_t rotate = h >> 58u; + uint64_t s = Uint128Low64(state) ^ h; +#endif + return random_internal::rotr(s, rotate); + } +}; + +// Parameterized implementation of the PCG 64-bit oneseq state. +// This provides state_type, multiplier, and increment for pcg_engine. +template <uint64_t kMult, uint64_t kInc> +class pcg64_params { + public: + using state_type = uint64_t; + static inline constexpr state_type multiplier() { return kMult; } + static inline constexpr state_type increment() { return kInc; } +}; + +// Implementation of the PCG xsh_rr_64_32 64-bit mixing function, which accepts +// an input of state_type and mixes it into an output of result_type. +struct pcg_xsh_rr_64_32 { + using state_type = uint64_t; + using result_type = uint32_t; + inline uint32_t operator()(uint64_t state) { + return random_internal::rotr( + static_cast<uint32_t>(((state >> 18) ^ state) >> 27), state >> 59); + } +}; + +// Stable pcg_engine implementations: +// This is a 64-bit generator using 128-bits of state. +// The output sequence is equivalent to Melissa O'Neil's pcg64_oneseq. +using pcg64_2018_engine = pcg_engine< + random_internal::pcg128_params<0x2360ed051fc65da4ull, 0x4385df649fccf645ull, + 0x5851f42d4c957f2d, 0x14057b7ef767814f>, + random_internal::pcg_xsl_rr_128_64>; + +// This is a 32-bit generator using 64-bits of state. +// This is equivalent to Melissa O'Neil's pcg32_oneseq. +using pcg32_2018_engine = pcg_engine< + random_internal::pcg64_params<0x5851f42d4c957f2dull, 0x14057b7ef767814full>, + random_internal::pcg_xsh_rr_64_32>; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_PCG_ENGINE_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/pcg_engine_test.cc b/third_party/abseil_cpp/absl/random/internal/pcg_engine_test.cc new file mode 100644 index 000000000000..4d763e89eb58 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/pcg_engine_test.cc @@ -0,0 +1,638 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/pcg_engine.h" + +#include <algorithm> +#include <bitset> +#include <random> +#include <sstream> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/random/internal/explicit_seed_seq.h" +#include "absl/time/clock.h" + +#define UPDATE_GOLDEN 0 + +namespace { + +using absl::random_internal::ExplicitSeedSeq; +using absl::random_internal::pcg32_2018_engine; +using absl::random_internal::pcg64_2018_engine; + +template <typename EngineType> +class PCGEngineTest : public ::testing::Test {}; + +using EngineTypes = ::testing::Types<pcg64_2018_engine, pcg32_2018_engine>; + +TYPED_TEST_SUITE(PCGEngineTest, EngineTypes); + +TYPED_TEST(PCGEngineTest, VerifyReseedChangesAllValues) { + using engine_type = TypeParam; + using result_type = typename engine_type::result_type; + + const size_t kNumOutputs = 16; + engine_type engine; + + // MSVC emits error 2719 without the use of std::ref below. + // * formal parameter with __declspec(align('#')) won't be aligned + + { + std::seed_seq seq1{1, 2, 3, 4, 5, 6, 7}; + engine.seed(seq1); + } + result_type a[kNumOutputs]; + std::generate(std::begin(a), std::end(a), std::ref(engine)); + + { + std::random_device rd; + std::seed_seq seq2{rd(), rd(), rd()}; + engine.seed(seq2); + } + result_type b[kNumOutputs]; + std::generate(std::begin(b), std::end(b), std::ref(engine)); + + // Verify that two uncorrelated values have ~50% of there bits in common. Use + // a 10% margin-of-error to reduce flakiness. + size_t changed_bits = 0; + size_t unchanged_bits = 0; + size_t total_set = 0; + size_t total_bits = 0; + size_t equal_count = 0; + for (size_t i = 0; i < kNumOutputs; ++i) { + equal_count += (a[i] == b[i]) ? 1 : 0; + std::bitset<sizeof(result_type) * 8> bitset(a[i] ^ b[i]); + changed_bits += bitset.count(); + unchanged_bits += bitset.size() - bitset.count(); + + std::bitset<sizeof(result_type) * 8> a_set(a[i]); + std::bitset<sizeof(result_type) * 8> b_set(b[i]); + total_set += a_set.count() + b_set.count(); + total_bits += 2 * 8 * sizeof(result_type); + } + // On average, half the bits are changed between two calls. + EXPECT_LE(changed_bits, 0.60 * (changed_bits + unchanged_bits)); + EXPECT_GE(changed_bits, 0.40 * (changed_bits + unchanged_bits)); + + // verify using a quick normal-approximation to the binomial. + EXPECT_NEAR(total_set, total_bits * 0.5, 4 * std::sqrt(total_bits)) + << "@" << total_set / static_cast<double>(total_bits); + + // Also, A[i] == B[i] with probability (1/range) * N. + // Give this a pretty wide latitude, though. + const double kExpected = kNumOutputs / (1.0 * sizeof(result_type) * 8); + EXPECT_LE(equal_count, 1.0 + kExpected); +} + +// Number of values that needs to be consumed to clean two sizes of buffer +// and trigger third refresh. (slightly overestimates the actual state size). +constexpr size_t kTwoBufferValues = 16; + +TYPED_TEST(PCGEngineTest, VerifyDiscard) { + using engine_type = TypeParam; + + for (size_t num_used = 0; num_used < kTwoBufferValues; ++num_used) { + engine_type engine_used; + for (size_t i = 0; i < num_used; ++i) { + engine_used(); + } + + for (size_t num_discard = 0; num_discard < kTwoBufferValues; + ++num_discard) { + engine_type engine1 = engine_used; + engine_type engine2 = engine_used; + for (size_t i = 0; i < num_discard; ++i) { + engine1(); + } + engine2.discard(num_discard); + for (size_t i = 0; i < kTwoBufferValues; ++i) { + const auto r1 = engine1(); + const auto r2 = engine2(); + ASSERT_EQ(r1, r2) << "used=" << num_used << " discard=" << num_discard; + } + } + } +} + +TYPED_TEST(PCGEngineTest, StreamOperatorsResult) { + using engine_type = TypeParam; + + std::wostringstream os; + std::wistringstream is; + engine_type engine; + + EXPECT_EQ(&(os << engine), &os); + EXPECT_EQ(&(is >> engine), &is); +} + +TYPED_TEST(PCGEngineTest, StreamSerialization) { + using engine_type = TypeParam; + + for (size_t discard = 0; discard < kTwoBufferValues; ++discard) { + ExplicitSeedSeq seed_sequence{12, 34, 56}; + engine_type engine(seed_sequence); + engine.discard(discard); + + std::stringstream stream; + stream << engine; + + engine_type new_engine; + stream >> new_engine; + for (size_t i = 0; i < 64; ++i) { + EXPECT_EQ(engine(), new_engine()) << " " << i; + } + } +} + +constexpr size_t kNumGoldenOutputs = 127; + +// This test is checking if randen_engine is meets interface requirements +// defined in [rand.req.urbg]. +TYPED_TEST(PCGEngineTest, RandomNumberEngineInterface) { + using engine_type = TypeParam; + + using E = engine_type; + using T = typename E::result_type; + + static_assert(std::is_copy_constructible<E>::value, + "engine_type must be copy constructible"); + + static_assert(absl::is_copy_assignable<E>::value, + "engine_type must be copy assignable"); + + static_assert(std::is_move_constructible<E>::value, + "engine_type must be move constructible"); + + static_assert(absl::is_move_assignable<E>::value, + "engine_type must be move assignable"); + + static_assert(std::is_same<decltype(std::declval<E>()()), T>::value, + "return type of operator() must be result_type"); + + // Names after definition of [rand.req.urbg] in C++ standard. + // e us a value of E + // v is a lvalue of E + // x, y are possibly const values of E + // s is a value of T + // q is a value satisfying requirements of seed_sequence + // z is a value of type unsigned long long + // os is a some specialization of basic_ostream + // is is a some specialization of basic_istream + + E e, v; + const E x, y; + T s = 1; + std::seed_seq q{1, 2, 3}; + unsigned long long z = 1; // NOLINT(runtime/int) + std::wostringstream os; + std::wistringstream is; + + E{}; + E{x}; + E{s}; + E{q}; + + e.seed(); + + // MSVC emits error 2718 when using EXPECT_EQ(e, x) + // * actual parameter with __declspec(align('#')) won't be aligned + EXPECT_TRUE(e == x); + + e.seed(q); + { + E tmp(q); + EXPECT_TRUE(e == tmp); + } + + e(); + { + E tmp(q); + EXPECT_TRUE(e != tmp); + } + + e.discard(z); + + static_assert(std::is_same<decltype(x == y), bool>::value, + "return type of operator== must be bool"); + + static_assert(std::is_same<decltype(x != y), bool>::value, + "return type of operator== must be bool"); +} + +TYPED_TEST(PCGEngineTest, RandenEngineSFINAETest) { + using engine_type = TypeParam; + using result_type = typename engine_type::result_type; + + { + engine_type engine(result_type(1)); + engine.seed(result_type(1)); + } + + { + result_type n = 1; + engine_type engine(n); + engine.seed(n); + } + + { + engine_type engine(1); + engine.seed(1); + } + + { + int n = 1; + engine_type engine(n); + engine.seed(n); + } + + { + std::seed_seq seed_seq; + engine_type engine(seed_seq); + engine.seed(seed_seq); + } + + { + engine_type engine{std::seed_seq()}; + engine.seed(std::seed_seq()); + } +} + +// ------------------------------------------------------------------ +// Stability tests for pcg64_2018_engine +// ------------------------------------------------------------------ +TEST(PCG642018EngineTest, VerifyGolden) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0x01070196e695f8f1, 0x703ec840c59f4493, 0xe54954914b3a44fa, + 0x96130ff204b9285e, 0x7d9fdef535ceb21a, 0x666feed42e1219a0, + 0x981f685721c8326f, 0xad80710d6eab4dda, 0xe202c480b037a029, + 0x5d3390eaedd907e2, 0x0756befb39c6b8aa, 0x1fb44ba6634d62a3, + 0x8d20423662426642, 0x34ea910167a39fb4, 0x93010b43a80d0ab6, + 0x663db08a98fc568a, 0x720b0a1335956fae, 0x2c35483e31e1d3ba, + 0x429f39776337409d, 0xb46d99e638687344, 0x105370b96aedcaee, + 0x3999e92f811cff71, 0xd230f8bcb591cfc9, 0x0dce3db2ba7bdea5, + 0xcf2f52c91eec99af, 0x2bc7c24a8b998a39, 0xbd8af1b0d599a19c, + 0x56bc45abc66059f5, 0x170a46dc170f7f1e, 0xc25daf5277b85fad, + 0xe629c2e0c948eadb, 0x1720a796915542ed, 0x22fb0caa4f909951, + 0x7e0c0f4175acd83d, 0xd9fcab37ff2a860c, 0xab2280fb2054bad1, + 0x58e8a06f37fa9e99, 0xc3a52a30b06528c7, 0x0175f773a13fc1bd, + 0x731cfc584b00e840, 0x404cc7b2648069cb, 0x5bc29153b0b7f783, + 0x771310a38cc999d1, 0x766a572f0a71a916, 0x90f450fb4fc48348, + 0xf080ea3e1c7b1a0d, 0x15471a4507d66a44, 0x7d58e55a78f3df69, + 0x0130a094576ac99c, 0x46669cb2d04b1d87, 0x17ab5bed20191840, + 0x95b177d260adff3e, 0x025fb624b6ee4c07, 0xb35de4330154a95f, + 0xe8510fff67e24c79, 0x132c3cbcd76ed2d3, 0x35e7cc145a093904, + 0x9f5b5b5f81583b79, 0x3ee749a533966233, 0x4af85886cdeda8cd, + 0x0ca5380ecb3ef3aa, 0x4f674eb7661d3192, 0x88a29aad00cd7733, + 0x70b627ca045ffac6, 0x5912b43ea887623d, 0x95dc9fc6f62cf221, + 0x926081a12a5c905b, 0x9c57d4cd7dfce651, 0x85ab2cbf23e3bb5d, + 0xc5cd669f63023152, 0x3067be0fad5d898e, 0x12b56f444cb53d05, + 0xbc2e5a640c3434fc, 0x9280bff0e4613fe1, 0x98819094c528743e, + 0x999d1c98d829df33, 0x9ff82a012dc89242, 0xf99183ed39c8be94, + 0xf0f59161cd421c55, 0x3c705730c2f6c48d, 0x66ad85c6e9278a61, + 0x2a3428e4a428d5d0, 0x79207d68fd04940d, 0xea7f2b402edc8430, + 0xa06b419ac857f63b, 0xcb1dd0e6fbc47e1c, 0x4f55229200ada6a4, + 0x9647b5e6359c927f, 0x30bf8f9197c7efe5, 0xa79519529cc384d0, + 0xbb22c4f339ad6497, 0xd7b9782f59d14175, 0x0dff12fff2ec0118, + 0xa331ad8305343a7c, 0x48dad7e3f17e0862, 0x324c6fb3fd3c9665, + 0xf0e4350e7933dfc4, 0x7ccda2f30b8b03b6, 0xa0afc6179005de40, + 0xee65da6d063b3a30, 0xb9506f42f2bfe87a, 0xc9a2e26b0ef5baa0, + 0x39fa9d4f495011d6, 0xbecc21a45d023948, 0x6bf484c6593f737f, + 0x8065e0070cadc3b7, 0x9ef617ed8d419799, 0xac692cf8c233dd15, + 0xd2ed87583c4ebb98, 0xad95ba1bebfedc62, 0x9b60b160a8264e43, + 0x0bc8c45f71fcf25b, 0x4a78035cdf1c9931, 0x4602dc106667e029, + 0xb335a3c250498ac8, 0x0256ebc4df20cab8, 0x0c61efd153f0c8d9, + 0xe5d0150a4f806f88, 0x99d6521d351e7d87, 0x8d4888c9f80f4325, + 0x106c5735c1ba868d, 0x73414881b880a878, 0x808a9a58a3064751, + 0x339a29f3746de3d5, 0x5410d7fa4f873896, 0xd84623c81d7b8a03, + 0x1f7c7e7a7f47f462, + }; + + pcg64_2018_engine engine(0); +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%016lx, ", engine()); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(PCG642018EngineTest, VerifyGoldenSeeded) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0xb03988f1e39691ee, 0xbd2a1eb5ac31e97a, 0x8f00d6d433634d02, + 0x1823c28d483d5776, 0x000c3ee3e1aeb74a, 0xfa82ef27a4f3df9c, + 0xc6f382308654e454, 0x414afb1a238996c2, 0x4703a4bc252eb411, + 0x99d64f62c8f7f654, 0xbb07ebe11a34fa44, 0x79eb06a363c06131, + 0xf66ad3756f1c6b21, 0x130c01d5e869f457, 0x5ca2b9963aecbc81, + 0xfef7bebc1de27e6c, 0x1d174faa5ed2cdbf, 0xd75b7a773f2bb889, + 0xc35c872327a170a5, 0x46da6d88646a42fe, 0x4622985e0442dae2, + 0xbe3cbd67297f1f9b, 0xe7c37b4a4798bfd1, 0x173d5dfad15a25c3, + 0x0eb6849ba2961522, 0xb0ff7246e6700d73, 0x88cb9c42d3afa577, + 0xb609731dbd94d917, 0xd3941cda04b40081, 0x28d140f7409bea3a, + 0x3c96699a920a124a, 0xdb28be521958b2fd, 0x0a3f44db3d4c5124, + 0x7ac8e60ba13b70d2, 0x75f03a41ded5195a, 0xaed10ac7c4e4825d, + 0xb92a3b18aadb7adc, 0xda45e0081f2bca46, 0x74d39ab3753143fc, + 0xb686038018fac9ca, 0x4cc309fe99542dbb, 0xf3e1a4fcb311097c, + 0x58763d6fa698d69d, 0xd11c365dbecd8d60, 0x2c15d55725b1dee7, + 0x89805f254d85658c, 0x2374c44dfc62158b, 0x9a8350fa7995328d, + 0x198f838970cf91da, 0x96aff569562c0e53, 0xd76c8c52b7ec6e3f, + 0x23a01cd9ae4baa81, 0x3adb366b6d02a893, 0xb3313e2a4c5b333f, + 0x04c11230b96a5425, 0x1f7f7af04787d571, 0xaddb019365275ec7, + 0x5c960468ccb09f42, 0x8438db698c69a44a, 0x492be1e46111637e, + 0x9c6c01e18100c610, 0xbfe48e75b7d0aceb, 0xb5e0b89ec1ce6a00, + 0x9d280ecbc2fe8997, 0x290d9e991ba5fcab, 0xeec5bec7d9d2a4f0, + 0x726e81488f19150e, 0x1a6df7955a7e462c, 0x37a12d174ba46bb5, + 0x3cdcdffd96b1b5c5, 0x2c5d5ac10661a26e, 0xa742ed18f22e50c4, + 0x00e0ed88ff0d8a35, 0x3d3c1718cb1efc0b, 0x1d70c51ffbccbf11, + 0xfbbb895132a4092f, 0x619d27f2fb095f24, 0x69af68200985e5c4, + 0xbee4885f57373f8d, 0x10b7a6bfe0587e40, 0xa885e6cf2f7e5f0a, + 0x59f879464f767550, 0x24e805d69056990d, 0x860970b911095891, + 0xca3189954f84170d, 0x6652a5edd4590134, 0x5e1008cef76174bf, + 0xcbd417881f2bcfe5, 0xfd49fc9d706ecd17, 0xeebf540221ebd066, + 0x46af7679464504cb, 0xd4028486946956f1, 0xd4f41864b86c2103, + 0x7af090e751583372, 0x98cdaa09278cb642, 0xffd42b921215602f, + 0x1d05bec8466b1740, 0xf036fa78a0132044, 0x787880589d1ecc78, + 0x5644552cfef33230, 0x0a97e275fe06884b, 0x96d1b13333d470b5, + 0xc8b3cdad52d3b034, 0x091357b9db7376fd, 0xa5fe4232555edf8c, + 0x3371bc3b6ada76b5, 0x7deeb2300477c995, 0x6fc6d4244f2849c1, + 0x750e8cc797ca340a, 0x81728613cd79899f, 0x3467f4ee6f9aeb93, + 0x5ef0a905f58c640f, 0x432db85e5101c98a, 0x6488e96f46ac80c2, + 0x22fddb282625048c, 0x15b287a0bc2d4c5d, 0xa7e2343ef1f28bce, + 0xc87ee1aa89bed09e, 0x220610107812c5e9, 0xcbdab6fcd640f586, + 0x8d41047970928784, 0x1aa431509ec1ade0, 0xac3f0be53f518ddc, + 0x16f4428ad81d0cbb, 0x675b13c2736fc4bb, 0x6db073afdd87e32d, + 0x572f3ca2f1a078c6, + }; + + ExplicitSeedSeq seed_sequence{12, 34, 56}; + pcg64_2018_engine engine(seed_sequence); +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%016lx, ", engine()); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(seed_sequence); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(PCG642018EngineTest, VerifyGoldenFromDeserializedEngine) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0xdd425b47b4113dea, 0x1b07176479d444b0, 0x6b391027586f2e42, + 0xa166f2b15f4a2143, 0xffb6dbd7a179ee97, 0xb2c00035365bf0b1, + 0x8fbb518b45855521, 0xfc789a55ddf87c3b, 0x429531f0f17ff355, + 0xbe708560d603d283, 0x5bff415175c5cb6b, 0xe813491f4ad45394, + 0xa853f4506d55880d, 0x7e538453e568172e, 0xe101f1e098ddd0ec, + 0x6ee31266ee4c766d, 0xa8786d92d66b39d7, 0xfee622a2acf5e5b0, + 0x5fe8e82c102fa7b3, 0x01f10be4cdb53c9d, 0xbe0545366f857022, + 0x12e74f010a339bca, 0xb10d85ca40d5ce34, 0xe80d6feba5054875, + 0x2b7c1ee6d567d4ee, 0x2a9cd043bfd03b66, 0x5cfc531bd239f3f1, + 0x1c4734e4647d70f5, 0x85a8f60f006b5760, 0x6a4239ce76dca387, + 0x8da0f86d7339335c, 0xf055b0468551374d, 0x486e8567e9bea9a0, + 0x4cb531b8405192dd, 0xf813b1ee3157110b, 0x214c2a664a875d8e, + 0x74531237b29b35f7, 0xa6f0267bb77a771e, 0x64b552bff54184a4, + 0xa2d6f7af2d75b6fc, 0x460a10018e03b5ab, 0x76fd1fdcb81d0800, + 0x76f5f81805070d9d, 0x1fb75cb1a70b289a, 0x9dfd25a022c4b27f, + 0x9a31a14a80528e9e, 0x910dc565ddc25820, 0xd6aef8e2b0936c10, + 0xe1773c507fe70225, 0xe027fd7aadd632bc, 0xc1fecb427089c8b8, + 0xb5c74c69fa9dbf26, 0x71bf9b0e4670227d, 0x25f48fad205dcfdd, + 0x905248ec4d689c56, 0x5c2b7631b0de5c9d, 0x9f2ee0f8f485036c, + 0xfd6ce4ebb90bf7ea, 0xd435d20046085574, 0x6b7eadcb0625f986, + 0x679d7d44b48be89e, 0x49683b8e1cdc49de, 0x4366cf76e9a2f4ca, + 0x54026ec1cdad7bed, 0xa9a04385207f28d3, 0xc8e66de4eba074b2, + 0x40b08c42de0f4cc0, 0x1d4c5e0e93c5bbc0, 0x19b80792e470ae2d, + 0x6fcaaeaa4c2a5bd9, 0xa92cb07c4238438e, 0x8bb5c918a007e298, + 0x7cd671e944874cf4, 0x88166470b1ba3cac, 0xd013d476eaeeade6, + 0xcee416947189b3c3, 0x5d7c16ab0dce6088, 0xd3578a5c32b13d27, + 0x3875db5adc9cc973, 0xfbdaba01c5b5dc56, 0xffc4fdd391b231c3, + 0x2334520ecb164fec, 0x361c115e7b6de1fa, 0xeee58106cc3563d7, + 0x8b7f35a8db25ebb8, 0xb29d00211e2cafa6, 0x22a39fe4614b646b, + 0x92ca6de8b998506d, 0x40922fe3d388d1db, 0x9da47f1e540f802a, + 0x811dceebf16a25db, 0xf6524ae22e0e53a9, 0x52d9e780a16eb99d, + 0x4f504286bb830207, 0xf6654d4786bd5cc3, 0x00bd98316003a7e1, + 0xefda054a6ab8f5f3, 0x46cfb0f4c1872827, 0xc22b316965c0f3b2, + 0xd1a28087c7e7562a, 0xaa4f6a094b7f5cff, 0xfe2bc853a041f7da, + 0xe9d531402a83c3ba, 0xe545d8663d3ce4dd, 0xfa2dcd7d91a13fa8, + 0xda1a080e52a127b8, 0x19c98f1f809c3d84, 0x2cef109af4678c88, + 0x53462accab3b9132, 0x176b13a80415394e, 0xea70047ef6bc178b, + 0x57bca80506d6dcdf, 0xd853ba09ff09f5c4, 0x75f4df3a7ddd4775, + 0x209c367ade62f4fe, 0xa9a0bbc74d5f4682, 0x5dfe34bada86c21a, + 0xc2c05bbcd38566d1, 0x6de8088e348c916a, 0x6a7001c6000c2196, + 0xd9fb51865fc4a367, 0x12f320e444ece8ff, 0x6d56f7f793d65035, + 0x138f31b7a865f8aa, 0x58fc68b4026b9adf, 0xcd48954b79fb6436, + 0x27dfce4a0232af87, + }; + +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + std::seed_seq seed_sequence{1, 2, 3}; + pcg64_2018_engine engine(seed_sequence); + std::ostringstream stream; + stream << engine; + auto str = stream.str(); + printf("%s\n\n", str.c_str()); + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%016lx, ", engine()); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + pcg64_2018_engine engine; + std::istringstream stream( + "2549297995355413924 4865540595714422341 6364136223846793005 " + "1442695040888963407 18088519957565336995 4845369368158826708"); + stream >> engine; + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +// ------------------------------------------------------------------ +// Stability tests for pcg32_2018_engine +// ------------------------------------------------------------------ +TEST(PCG322018EngineTest, VerifyGolden) { + constexpr uint32_t kGolden[kNumGoldenOutputs] = { + 0x7a7ecbd9, 0x89fd6c06, 0xae646aa8, 0xcd3cf945, 0x6204b303, 0x198c8585, + 0x49fce611, 0xd1e9297a, 0x142d9440, 0xee75f56b, 0x473a9117, 0xe3a45903, + 0xbce807a1, 0xe54e5f4d, 0x497d6c51, 0x61829166, 0xa740474b, 0x031912a8, + 0x9de3defa, 0xd266dbf1, 0x0f38bebb, 0xec3c4f65, 0x07c5057d, 0xbbce03c8, + 0xfd2ac7a8, 0xffcf4773, 0x5b10affb, 0xede1c842, 0xe22b01b7, 0xda133c8c, + 0xaf89b0f4, 0x25d1b8bc, 0x9f625482, 0x7bfd6882, 0x2e2210c0, 0x2c8fb9a6, + 0x42cb3b83, 0x40ce0dab, 0x644a3510, 0x36230ef2, 0xe2cb6d43, 0x1012b343, + 0x746c6c9f, 0x36714cf8, 0xed1f5026, 0x8bbbf83e, 0xe98710f4, 0x8a2afa36, + 0x09035349, 0x6dc1a487, 0x682b634b, 0xc106794f, 0x7dd78beb, 0x628c262b, + 0x852fb232, 0xb153ac4c, 0x4f169d1b, 0xa69ab774, 0x4bd4b6f2, 0xdc351dd3, + 0x93ff3c8c, 0xa30819ab, 0xff07758c, 0x5ab13c62, 0xd16d7fb5, 0xc4950ffa, + 0xd309ae49, 0xb9677a87, 0x4464e317, 0x90dc44f1, 0xc694c1d4, 0x1d5e1168, + 0xadf37a2d, 0xda38990d, 0x1ec4bd33, 0x36ca25ce, 0xfa0dc76a, 0x968a9d43, + 0x6950ac39, 0xdd3276bc, 0x06d5a71e, 0x1f6f282d, 0x5c626c62, 0xdde3fc31, + 0x152194ce, 0xc35ed14c, 0xb1f7224e, 0x47f76bb8, 0xb34fdd08, 0x7011395e, + 0x162d2a49, 0x0d1bf09f, 0x9428a952, 0x03c5c344, 0xd3525616, 0x7816fff3, + 0x6bceb8a8, 0x8345a081, 0x366420fd, 0x182abeda, 0x70f82745, 0xaf15ded8, + 0xc7f52ca2, 0xa98db9c5, 0x919d99ba, 0x9c376c1c, 0xed8d34c2, 0x716ae9f5, + 0xef062fa5, 0xee3b6c56, 0x52325658, 0x61afa9c3, 0xfdaf02f0, 0x961cf3ab, + 0x9f291565, 0x4fbf3045, 0x0590c899, 0xde901385, 0x45005ffb, 0x509db162, + 0x262fa941, 0x4c421653, 0x4b17c21e, 0xea0d1530, 0xde803845, 0x61bfd515, + 0x438523ef, + }; + + pcg32_2018_engine engine(0); +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%08x, ", engine()); + if (i % 6 == 5) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(PCG322018EngineTest, VerifyGoldenSeeded) { + constexpr uint32_t kGolden[kNumGoldenOutputs] = { + 0x60b5a64c, 0x978502f9, 0x80a75f60, 0x241f1158, 0xa4cd1dbb, 0xe7284017, + 0x3b678da5, 0x5223ec99, 0xe4bdd5d9, 0x72190e6d, 0xe6e702c9, 0xff80c768, + 0xcf126ed3, 0x1fbd20ab, 0x60980489, 0xbc72bf89, 0x407ac6c0, 0x00bf3c51, + 0xf9087897, 0x172e4eb6, 0xe9e4f443, 0x1a6098bf, 0xbf44f8c2, 0xdd84a0e5, + 0xd9a52364, 0xc0e2e786, 0x061ae2ba, 0x9facb8e3, 0x6109432d, 0xd4e0a013, + 0xbd8eb9a6, 0x7e86c3b6, 0x629c0e68, 0x05337430, 0xb495b9f4, 0x11ccd65d, + 0xb578db25, 0x66f1246d, 0x6ef20a7f, 0x5e429812, 0x11772130, 0xb944b5c2, + 0x01624128, 0xa2385ab7, 0xd3e10d35, 0xbe570ec3, 0xc951656f, 0xbe8944a0, + 0x7be41062, 0x5709f919, 0xd745feda, 0x9870b9ae, 0xb44b8168, 0x19e7683b, + 0xded8017f, 0xc6e4d544, 0x91ae4225, 0xd6745fba, 0xb992f284, 0x65b12b33, + 0xa9d5fdb4, 0xf105ce1a, 0x35ca1a6e, 0x2ff70dd0, 0xd8335e49, 0xfb71ddf2, + 0xcaeabb89, 0x5c6f5f84, 0x9a811a7d, 0xbcecbbd1, 0x0f661ba0, 0x9ad93b9d, + 0xedd23e0b, 0x42062f48, 0xd38dd7e4, 0x6cd63c9c, 0x640b98ae, 0x4bff5653, + 0x12626371, 0x13266017, 0xe7a698d8, 0x39c74667, 0xe8fdf2e3, 0x52803bf8, + 0x2af6895b, 0x91335b7b, 0x699e4961, 0x00a40fff, 0x253ff2b6, 0x4a6cf672, + 0x9584e85f, 0xf2a5000c, 0x4d58aba8, 0xb8513e6a, 0x767fad65, 0x8e326f9e, + 0x182f15a1, 0x163dab52, 0xdf99c780, 0x047282a1, 0xee4f90dd, 0xd50394ae, + 0x6c9fd5f0, 0xb06a9194, 0x387e3840, 0x04a9487b, 0xf678a4c2, 0xd0a78810, + 0xd502c97e, 0xd6a9b12a, 0x4accc5dc, 0x416ed53e, 0x50411536, 0xeeb89c24, + 0x813a7902, 0x034ebca6, 0xffa52e7c, 0x7ecd3d0e, 0xfa37a0d2, 0xb1fbe2c1, + 0xb7efc6d1, 0xefa4ccee, 0xf6f80424, 0x2283f3d9, 0x68732284, 0x94f3b5c8, + 0xbbdeceb9, + }; + + ExplicitSeedSeq seed_sequence{12, 34, 56}; + pcg32_2018_engine engine(seed_sequence); +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%08x, ", engine()); + if (i % 6 == 5) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(seed_sequence); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(PCG322018EngineTest, VerifyGoldenFromDeserializedEngine) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0x780f7042, 0xba137215, 0x43ab6f22, 0x0cb55f46, 0x44b2627d, 0x835597af, + 0xea973ea1, 0x0d2abd35, 0x4fdd601c, 0xac4342fe, 0x7db7e93c, 0xe56ebcaf, + 0x3596470a, 0x7770a9ad, 0x9b893320, 0x57db3415, 0xb432de54, 0xa02baf71, + 0xa256aadb, 0x88921fc7, 0xa35fa6b3, 0xde3eca46, 0x605739a7, 0xa890b82b, + 0xe457b7ad, 0x335fb903, 0xeb06790c, 0xb3c54bf6, 0x6141e442, 0xa599a482, + 0xb78987cc, 0xc61dfe9d, 0x0f1d6ace, 0x17460594, 0x8f6a5061, 0x083dc354, + 0xe9c337fb, 0xcfd105f7, 0x926764b6, 0x638d24dc, 0xeaac650a, 0x67d2cb9c, + 0xd807733c, 0x205fc52e, 0xf5399e2e, 0x6c46ddcc, 0xb603e875, 0xce113a25, + 0x3c8d4813, 0xfb584db8, 0xf6d255ff, 0xea80954f, 0x42e8be85, 0xb2feee72, + 0x62bd8d16, 0x1be4a142, 0x97dca1a4, 0xdd6e7333, 0xb2caa20e, 0xa12b1588, + 0xeb3a5a1a, 0x6fa5ba89, 0x077ea931, 0x8ddb1713, 0x0dd03079, 0x2c2ba965, + 0xa77fac17, 0xc8325742, 0x8bb893bf, 0xc2315741, 0xeaceee92, 0x81dd2ee2, + 0xe5214216, 0x1b9b8fb2, 0x01646d03, 0x24facc25, 0xd8c0e0bb, 0xa33fe106, + 0xf34fe976, 0xb3b4b44e, 0x65618fed, 0x032c6192, 0xa9dd72ce, 0xf391887b, + 0xf41c6a6e, 0x05c4bd6d, 0x37fa260e, 0x46b05659, 0xb5f6348a, 0x62d26d89, + 0x39f6452d, 0xb17b30a2, 0xbdd82743, 0x38ecae3b, 0xfe90f0a2, 0xcb2d226d, + 0xcf8a0b1c, 0x0eed3d4d, 0xa1f69cfc, 0xd7ac3ba5, 0xce9d9a6b, 0x121deb4c, + 0x4a0d03f3, 0xc1821ed1, 0x59c249ac, 0xc0abb474, 0x28149985, 0xfd9a82ba, + 0x5960c3b2, 0xeff00cba, 0x6073aa17, 0x25dc0919, 0x9976626e, 0xdd2ccc33, + 0x39ecb6ec, 0xc6e15d13, 0xfac94cfd, 0x28cfd34f, 0xf2d2c32d, 0x51c23d08, + 0x4fdb2f48, 0x97baa807, 0xf2c1004c, 0xc4ae8136, 0x71f31c94, 0x8c92d601, + 0x36caf5cd, + }; + +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + std::seed_seq seed_sequence{1, 2, 3}; + pcg32_2018_engine engine(seed_sequence); + std::ostringstream stream; + stream << engine; + auto str = stream.str(); + printf("%s\n\n", str.c_str()); + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%08x, ", engine()); + if (i % 6 == 5) { + printf("\n"); + } + } + printf("\n\n\n"); + + EXPECT_FALSE(true); +#else + pcg32_2018_engine engine; + std::istringstream stream( + "6364136223846793005 1442695040888963407 6537028157270659894"); + stream >> engine; + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/platform.h b/third_party/abseil_cpp/absl/random/internal/platform.h new file mode 100644 index 000000000000..bbdb4e620816 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/platform.h @@ -0,0 +1,171 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_PLATFORM_H_ +#define ABSL_RANDOM_INTERNAL_PLATFORM_H_ + +// HERMETIC NOTE: The randen_hwaes target must not introduce duplicate +// symbols from arbitrary system and other headers, since it may be built +// with different flags from other targets, using different levels of +// optimization, potentially introducing ODR violations. + +// ----------------------------------------------------------------------------- +// Platform Feature Checks +// ----------------------------------------------------------------------------- + +// Currently supported operating systems and associated preprocessor +// symbols: +// +// Linux and Linux-derived __linux__ +// Android __ANDROID__ (implies __linux__) +// Linux (non-Android) __linux__ && !__ANDROID__ +// Darwin (macOS and iOS) __APPLE__ +// Akaros (http://akaros.org) __ros__ +// Windows _WIN32 +// NaCL __native_client__ +// AsmJS __asmjs__ +// WebAssembly __wasm__ +// Fuchsia __Fuchsia__ +// +// Note that since Android defines both __ANDROID__ and __linux__, one +// may probe for either Linux or Android by simply testing for __linux__. +// +// NOTE: For __APPLE__ platforms, we use #include <TargetConditionals.h> +// to distinguish os variants. +// +// http://nadeausoftware.com/articles/2012/01/c_c_tip_how_use_compiler_predefined_macros_detect_operating_system + +#if defined(__APPLE__) +#include <TargetConditionals.h> +#endif + +// ----------------------------------------------------------------------------- +// Architecture Checks +// ----------------------------------------------------------------------------- + +// These preprocessor directives are trying to determine CPU architecture, +// including necessary headers to support hardware AES. +// +// ABSL_ARCH_{X86/PPC/ARM} macros determine the platform. +#if defined(__x86_64__) || defined(__x86_64) || defined(_M_AMD64) || \ + defined(_M_X64) +#define ABSL_ARCH_X86_64 +#elif defined(__i386) || defined(_M_IX86) +#define ABSL_ARCH_X86_32 +#elif defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) +#define ABSL_ARCH_AARCH64 +#elif defined(__arm__) || defined(__ARMEL__) || defined(_M_ARM) +#define ABSL_ARCH_ARM +#elif defined(__powerpc64__) || defined(__PPC64__) || defined(__powerpc__) || \ + defined(__ppc__) || defined(__PPC__) +#define ABSL_ARCH_PPC +#else +// Unsupported architecture. +// * https://sourceforge.net/p/predef/wiki/Architectures/ +// * https://msdn.microsoft.com/en-us/library/b0084kay.aspx +// * for gcc, clang: "echo | gcc -E -dM -" +#endif + +// ----------------------------------------------------------------------------- +// Attribute Checks +// ----------------------------------------------------------------------------- + +// ABSL_RANDOM_INTERNAL_RESTRICT annotates whether pointers may be considered +// to be unaliased. +#if defined(__clang__) || defined(__GNUC__) +#define ABSL_RANDOM_INTERNAL_RESTRICT __restrict__ +#elif defined(_MSC_VER) +#define ABSL_RANDOM_INTERNAL_RESTRICT __restrict +#else +#define ABSL_RANDOM_INTERNAL_RESTRICT +#endif + +// ABSL_HAVE_ACCELERATED_AES indicates whether the currently active compiler +// flags (e.g. -maes) allow using hardware accelerated AES instructions, which +// implies us assuming that the target platform supports them. +#define ABSL_HAVE_ACCELERATED_AES 0 + +#if defined(ABSL_ARCH_X86_64) + +#if defined(__AES__) || defined(__AVX__) +#undef ABSL_HAVE_ACCELERATED_AES +#define ABSL_HAVE_ACCELERATED_AES 1 +#endif + +#elif defined(ABSL_ARCH_PPC) + +// Rely on VSX and CRYPTO extensions for vcipher on PowerPC. +#if (defined(__VEC__) || defined(__ALTIVEC__)) && defined(__VSX__) && \ + defined(__CRYPTO__) +#undef ABSL_HAVE_ACCELERATED_AES +#define ABSL_HAVE_ACCELERATED_AES 1 +#endif + +#elif defined(ABSL_ARCH_ARM) || defined(ABSL_ARCH_AARCH64) + +// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0053c/IHI0053C_acle_2_0.pdf +// Rely on NEON+CRYPTO extensions for ARM. +#if defined(__ARM_NEON) && defined(__ARM_FEATURE_CRYPTO) +#undef ABSL_HAVE_ACCELERATED_AES +#define ABSL_HAVE_ACCELERATED_AES 1 +#endif + +#endif + +// NaCl does not allow AES. +#if defined(__native_client__) +#undef ABSL_HAVE_ACCELERATED_AES +#define ABSL_HAVE_ACCELERATED_AES 0 +#endif + +// ABSL_RANDOM_INTERNAL_AES_DISPATCH indicates whether the currently active +// platform has, or should use run-time dispatch for selecting the +// acclerated Randen implementation. +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 0 + +#if defined(ABSL_ARCH_X86_64) +// Dispatch is available on x86_64 +#undef ABSL_RANDOM_INTERNAL_AES_DISPATCH +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 1 +#elif defined(__linux__) && defined(ABSL_ARCH_PPC) +// Or when running linux PPC +#undef ABSL_RANDOM_INTERNAL_AES_DISPATCH +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 1 +#elif defined(__linux__) && defined(ABSL_ARCH_AARCH64) +// Or when running linux AArch64 +#undef ABSL_RANDOM_INTERNAL_AES_DISPATCH +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 1 +#elif defined(__linux__) && defined(ABSL_ARCH_ARM) && (__ARM_ARCH >= 8) +// Or when running linux ARM v8 or higher. +// (This captures a lot of Android configurations.) +#undef ABSL_RANDOM_INTERNAL_AES_DISPATCH +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 1 +#endif + +// NaCl does not allow dispatch. +#if defined(__native_client__) +#undef ABSL_RANDOM_INTERNAL_AES_DISPATCH +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 0 +#endif + +// iOS does not support dispatch, even on x86, since applications +// should be bundled as fat binaries, with a different build tailored for +// each specific supported platform/architecture. +#if (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) || \ + (defined(TARGET_OS_IPHONE_SIMULATOR) && TARGET_OS_IPHONE_SIMULATOR) +#undef ABSL_RANDOM_INTERNAL_AES_DISPATCH +#define ABSL_RANDOM_INTERNAL_AES_DISPATCH 0 +#endif + +#endif // ABSL_RANDOM_INTERNAL_PLATFORM_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/pool_urbg.cc b/third_party/abseil_cpp/absl/random/internal/pool_urbg.cc new file mode 100644 index 000000000000..5bee530770eb --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/pool_urbg.cc @@ -0,0 +1,254 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/pool_urbg.h" + +#include <algorithm> +#include <atomic> +#include <cstdint> +#include <cstring> +#include <iterator> + +#include "absl/base/attributes.h" +#include "absl/base/call_once.h" +#include "absl/base/config.h" +#include "absl/base/internal/endian.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/internal/sysinfo.h" +#include "absl/base/internal/unaligned_access.h" +#include "absl/base/optimization.h" +#include "absl/random/internal/randen.h" +#include "absl/random/internal/seed_material.h" +#include "absl/random/seed_gen_exception.h" + +using absl::base_internal::SpinLock; +using absl::base_internal::SpinLockHolder; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +// RandenPoolEntry is a thread-safe pseudorandom bit generator, implementing a +// single generator within a RandenPool<T>. It is an internal implementation +// detail, and does not aim to conform to [rand.req.urng]. +// +// NOTE: There are alignment issues when used on ARM, for instance. +// See the allocation code in PoolAlignedAlloc(). +class RandenPoolEntry { + public: + static constexpr size_t kState = RandenTraits::kStateBytes / sizeof(uint32_t); + static constexpr size_t kCapacity = + RandenTraits::kCapacityBytes / sizeof(uint32_t); + + void Init(absl::Span<const uint32_t> data) { + SpinLockHolder l(&mu_); // Always uncontested. + std::copy(data.begin(), data.end(), std::begin(state_)); + next_ = kState; + } + + // Copy bytes into out. + void Fill(uint8_t* out, size_t bytes) ABSL_LOCKS_EXCLUDED(mu_); + + // Returns random bits from the buffer in units of T. + template <typename T> + inline T Generate() ABSL_LOCKS_EXCLUDED(mu_); + + inline void MaybeRefill() ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + if (next_ >= kState) { + next_ = kCapacity; + impl_.Generate(state_); + } + } + + private: + // Randen URBG state. + uint32_t state_[kState] ABSL_GUARDED_BY(mu_); // First to satisfy alignment. + SpinLock mu_; + const Randen impl_; + size_t next_ ABSL_GUARDED_BY(mu_); +}; + +template <> +inline uint8_t RandenPoolEntry::Generate<uint8_t>() { + SpinLockHolder l(&mu_); + MaybeRefill(); + return static_cast<uint8_t>(state_[next_++]); +} + +template <> +inline uint16_t RandenPoolEntry::Generate<uint16_t>() { + SpinLockHolder l(&mu_); + MaybeRefill(); + return static_cast<uint16_t>(state_[next_++]); +} + +template <> +inline uint32_t RandenPoolEntry::Generate<uint32_t>() { + SpinLockHolder l(&mu_); + MaybeRefill(); + return state_[next_++]; +} + +template <> +inline uint64_t RandenPoolEntry::Generate<uint64_t>() { + SpinLockHolder l(&mu_); + if (next_ >= kState - 1) { + next_ = kCapacity; + impl_.Generate(state_); + } + auto p = state_ + next_; + next_ += 2; + + uint64_t result; + std::memcpy(&result, p, sizeof(result)); + return result; +} + +void RandenPoolEntry::Fill(uint8_t* out, size_t bytes) { + SpinLockHolder l(&mu_); + while (bytes > 0) { + MaybeRefill(); + size_t remaining = (kState - next_) * sizeof(state_[0]); + size_t to_copy = std::min(bytes, remaining); + std::memcpy(out, &state_[next_], to_copy); + out += to_copy; + bytes -= to_copy; + next_ += (to_copy + sizeof(state_[0]) - 1) / sizeof(state_[0]); + } +} + +// Number of pooled urbg entries. +static constexpr int kPoolSize = 8; + +// Shared pool entries. +static absl::once_flag pool_once; +ABSL_CACHELINE_ALIGNED static RandenPoolEntry* shared_pools[kPoolSize]; + +// Returns an id in the range [0 ... kPoolSize), which indexes into the +// pool of random engines. +// +// Each thread to access the pool is assigned a sequential ID (without reuse) +// from the pool-id space; the id is cached in a thread_local variable. +// This id is assigned based on the arrival-order of the thread to the +// GetPoolID call; this has no binary, CL, or runtime stability because +// on subsequent runs the order within the same program may be significantly +// different. However, as other thread IDs are not assigned sequentially, +// this is not expected to matter. +int GetPoolID() { + static_assert(kPoolSize >= 1, + "At least one urbg instance is required for PoolURBG"); + + ABSL_CONST_INIT static std::atomic<int64_t> sequence{0}; + +#ifdef ABSL_HAVE_THREAD_LOCAL + static thread_local int my_pool_id = -1; + if (ABSL_PREDICT_FALSE(my_pool_id < 0)) { + my_pool_id = (sequence++ % kPoolSize); + } + return my_pool_id; +#else + static pthread_key_t tid_key = [] { + pthread_key_t tmp_key; + int err = pthread_key_create(&tmp_key, nullptr); + if (err) { + ABSL_RAW_LOG(FATAL, "pthread_key_create failed with %d", err); + } + return tmp_key; + }(); + + // Store the value in the pthread_{get/set}specific. However an uninitialized + // value is 0, so add +1 to distinguish from the null value. + intptr_t my_pool_id = + reinterpret_cast<intptr_t>(pthread_getspecific(tid_key)); + if (ABSL_PREDICT_FALSE(my_pool_id == 0)) { + // No allocated ID, allocate the next value, cache it, and return. + my_pool_id = (sequence++ % kPoolSize) + 1; + int err = pthread_setspecific(tid_key, reinterpret_cast<void*>(my_pool_id)); + if (err) { + ABSL_RAW_LOG(FATAL, "pthread_setspecific failed with %d", err); + } + } + return my_pool_id - 1; +#endif +} + +// Allocate a RandenPoolEntry with at least 32-byte alignment, which is required +// by ARM platform code. +RandenPoolEntry* PoolAlignedAlloc() { + constexpr size_t kAlignment = + ABSL_CACHELINE_SIZE > 32 ? ABSL_CACHELINE_SIZE : 32; + + // Not all the platforms that we build for have std::aligned_alloc, however + // since we never free these objects, we can over allocate and munge the + // pointers to the correct alignment. + void* memory = std::malloc(sizeof(RandenPoolEntry) + kAlignment); + auto x = reinterpret_cast<intptr_t>(memory); + auto y = x % kAlignment; + void* aligned = + (y == 0) ? memory : reinterpret_cast<void*>(x + kAlignment - y); + return new (aligned) RandenPoolEntry(); +} + +// Allocate and initialize kPoolSize objects of type RandenPoolEntry. +// +// The initialization strategy is to initialize one object directly from +// OS entropy, then to use that object to seed all of the individual +// pool instances. +void InitPoolURBG() { + static constexpr size_t kSeedSize = + RandenTraits::kStateBytes / sizeof(uint32_t); + // Read the seed data from OS entropy once. + uint32_t seed_material[kPoolSize * kSeedSize]; + if (!random_internal::ReadSeedMaterialFromOSEntropy( + absl::MakeSpan(seed_material))) { + random_internal::ThrowSeedGenException(); + } + for (int i = 0; i < kPoolSize; i++) { + shared_pools[i] = PoolAlignedAlloc(); + shared_pools[i]->Init( + absl::MakeSpan(&seed_material[i * kSeedSize], kSeedSize)); + } +} + +// Returns the pool entry for the current thread. +RandenPoolEntry* GetPoolForCurrentThread() { + absl::call_once(pool_once, InitPoolURBG); + return shared_pools[GetPoolID()]; +} + +} // namespace + +template <typename T> +typename RandenPool<T>::result_type RandenPool<T>::Generate() { + auto* pool = GetPoolForCurrentThread(); + return pool->Generate<T>(); +} + +template <typename T> +void RandenPool<T>::Fill(absl::Span<result_type> data) { + auto* pool = GetPoolForCurrentThread(); + pool->Fill(reinterpret_cast<uint8_t*>(data.data()), + data.size() * sizeof(result_type)); +} + +template class RandenPool<uint8_t>; +template class RandenPool<uint16_t>; +template class RandenPool<uint32_t>; +template class RandenPool<uint64_t>; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/pool_urbg.h b/third_party/abseil_cpp/absl/random/internal/pool_urbg.h new file mode 100644 index 000000000000..05721929f54e --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/pool_urbg.h @@ -0,0 +1,131 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_POOL_URBG_H_ +#define ABSL_RANDOM_INTERNAL_POOL_URBG_H_ + +#include <cinttypes> +#include <limits> + +#include "absl/random/internal/traits.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// RandenPool is a thread-safe random number generator [random.req.urbg] that +// uses an underlying pool of Randen generators to generate values. Each thread +// has affinity to one instance of the underlying pool generators. Concurrent +// access is guarded by a spin-lock. +template <typename T> +class RandenPool { + public: + using result_type = T; + static_assert(std::is_unsigned<result_type>::value, + "RandenPool template argument must be a built-in unsigned " + "integer type"); + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + RandenPool() {} + + // Returns a single value. + inline result_type operator()() { return Generate(); } + + // Fill data with random values. + static void Fill(absl::Span<result_type> data); + + protected: + // Generate returns a single value. + static result_type Generate(); +}; + +extern template class RandenPool<uint8_t>; +extern template class RandenPool<uint16_t>; +extern template class RandenPool<uint32_t>; +extern template class RandenPool<uint64_t>; + +// PoolURBG uses an underlying pool of random generators to implement a +// thread-compatible [random.req.urbg] interface with an internal cache of +// values. +template <typename T, size_t kBufferSize> +class PoolURBG { + // Inheritance to access the protected static members of RandenPool. + using unsigned_type = typename make_unsigned_bits<T>::type; + using PoolType = RandenPool<unsigned_type>; + using SpanType = absl::Span<unsigned_type>; + + static constexpr size_t kInitialBuffer = kBufferSize + 1; + static constexpr size_t kHalfBuffer = kBufferSize / 2; + + public: + using result_type = T; + + static_assert(std::is_unsigned<result_type>::value, + "PoolURBG must be parameterized by an unsigned integer type"); + + static_assert(kBufferSize > 1, + "PoolURBG must be parameterized by a buffer-size > 1"); + + static_assert(kBufferSize <= 256, + "PoolURBG must be parameterized by a buffer-size <= 256"); + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + PoolURBG() : next_(kInitialBuffer) {} + + // copy-constructor does not copy cache. + PoolURBG(const PoolURBG&) : next_(kInitialBuffer) {} + const PoolURBG& operator=(const PoolURBG&) { + next_ = kInitialBuffer; + return *this; + } + + // move-constructor does move cache. + PoolURBG(PoolURBG&&) = default; + PoolURBG& operator=(PoolURBG&&) = default; + + inline result_type operator()() { + if (next_ >= kBufferSize) { + next_ = (kBufferSize > 2 && next_ > kBufferSize) ? kHalfBuffer : 0; + PoolType::Fill(SpanType(reinterpret_cast<unsigned_type*>(state_ + next_), + kBufferSize - next_)); + } + return state_[next_++]; + } + + private: + // Buffer size. + size_t next_; // index within state_ + result_type state_[kBufferSize]; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_POOL_URBG_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/pool_urbg_test.cc b/third_party/abseil_cpp/absl/random/internal/pool_urbg_test.cc new file mode 100644 index 000000000000..53f4eacf16fe --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/pool_urbg_test.cc @@ -0,0 +1,182 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/pool_urbg.h" + +#include <algorithm> +#include <bitset> +#include <cmath> +#include <cstdint> +#include <iterator> + +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" +#include "absl/types/span.h" + +using absl::random_internal::PoolURBG; +using absl::random_internal::RandenPool; + +namespace { + +// is_randen_pool trait is true when parameterized by an RandenPool +template <typename T> +using is_randen_pool = typename absl::disjunction< // + std::is_same<T, RandenPool<uint8_t>>, // + std::is_same<T, RandenPool<uint16_t>>, // + std::is_same<T, RandenPool<uint32_t>>, // + std::is_same<T, RandenPool<uint64_t>>>; // + +// MyFill either calls RandenPool::Fill() or std::generate(..., rng) +template <typename T, typename V> +typename absl::enable_if_t<absl::negation<is_randen_pool<T>>::value, void> // +MyFill(T& rng, absl::Span<V> data) { // NOLINT(runtime/references) + std::generate(std::begin(data), std::end(data), rng); +} + +template <typename T, typename V> +typename absl::enable_if_t<is_randen_pool<T>::value, void> // +MyFill(T& rng, absl::Span<V> data) { // NOLINT(runtime/references) + rng.Fill(data); +} + +template <typename EngineType> +class PoolURBGTypedTest : public ::testing::Test {}; + +using EngineTypes = ::testing::Types< // + RandenPool<uint8_t>, // + RandenPool<uint16_t>, // + RandenPool<uint32_t>, // + RandenPool<uint64_t>, // + PoolURBG<uint8_t, 2>, // + PoolURBG<uint16_t, 2>, // + PoolURBG<uint32_t, 2>, // + PoolURBG<uint64_t, 2>, // + PoolURBG<unsigned int, 8>, // NOLINT(runtime/int) + PoolURBG<unsigned long, 8>, // NOLINT(runtime/int) + PoolURBG<unsigned long int, 4>, // NOLINT(runtime/int) + PoolURBG<unsigned long long, 4>>; // NOLINT(runtime/int) + +TYPED_TEST_SUITE(PoolURBGTypedTest, EngineTypes); + +// This test is checks that the engines meet the URBG interface requirements +// defined in [rand.req.urbg]. +TYPED_TEST(PoolURBGTypedTest, URBGInterface) { + using E = TypeParam; + using T = typename E::result_type; + + static_assert(std::is_copy_constructible<E>::value, + "engine must be copy constructible"); + + static_assert(absl::is_copy_assignable<E>::value, + "engine must be copy assignable"); + + E e; + const E x; + + e(); + + static_assert(std::is_same<decltype(e()), T>::value, + "return type of operator() must be result_type"); + + E u0(x); + u0(); + + E u1 = e; + u1(); +} + +// This validates that sequences are independent. +TYPED_TEST(PoolURBGTypedTest, VerifySequences) { + using E = TypeParam; + using result_type = typename E::result_type; + + E rng; + (void)rng(); // Discard one value. + + constexpr int kNumOutputs = 64; + result_type a[kNumOutputs]; + result_type b[kNumOutputs]; + std::fill(std::begin(b), std::end(b), 0); + + // Fill a using Fill or generate, depending on the engine type. + { + E x = rng; + MyFill(x, absl::MakeSpan(a)); + } + + // Fill b using std::generate(). + { + E x = rng; + std::generate(std::begin(b), std::end(b), x); + } + + // Test that generated sequence changed as sequence of bits, i.e. if about + // half of the bites were flipped between two non-correlated values. + size_t changed_bits = 0; + size_t unchanged_bits = 0; + size_t total_set = 0; + size_t total_bits = 0; + size_t equal_count = 0; + for (size_t i = 0; i < kNumOutputs; ++i) { + equal_count += (a[i] == b[i]) ? 1 : 0; + std::bitset<sizeof(result_type) * 8> bitset(a[i] ^ b[i]); + changed_bits += bitset.count(); + unchanged_bits += bitset.size() - bitset.count(); + + std::bitset<sizeof(result_type) * 8> a_set(a[i]); + std::bitset<sizeof(result_type) * 8> b_set(b[i]); + total_set += a_set.count() + b_set.count(); + total_bits += 2 * 8 * sizeof(result_type); + } + // On average, half the bits are changed between two calls. + EXPECT_LE(changed_bits, 0.60 * (changed_bits + unchanged_bits)); + EXPECT_GE(changed_bits, 0.40 * (changed_bits + unchanged_bits)); + + // verify using a quick normal-approximation to the binomial. + EXPECT_NEAR(total_set, total_bits * 0.5, 4 * std::sqrt(total_bits)) + << "@" << total_set / static_cast<double>(total_bits); + + // Also, A[i] == B[i] with probability (1/range) * N. + // Give this a pretty wide latitude, though. + const double kExpected = kNumOutputs / (1.0 * sizeof(result_type) * 8); + EXPECT_LE(equal_count, 1.0 + kExpected); +} + +} // namespace + +/* +$ nanobenchmarks 1 RandenPool construct +$ nanobenchmarks 1 PoolURBG construct + +RandenPool<uint32_t> | 1 | 1000 | 48482.00 ticks | 48.48 ticks | 13.9 ns +RandenPool<uint32_t> | 10 | 2000 | 1028795.00 ticks | 51.44 ticks | 14.7 ns +RandenPool<uint32_t> | 100 | 1000 | 5119968.00 ticks | 51.20 ticks | 14.6 ns +RandenPool<uint32_t> | 1000 | 500 | 25867936.00 ticks | 51.74 ticks | 14.8 ns + +RandenPool<uint64_t> | 1 | 1000 | 49921.00 ticks | 49.92 ticks | 14.3 ns +RandenPool<uint64_t> | 10 | 2000 | 1208269.00 ticks | 60.41 ticks | 17.3 ns +RandenPool<uint64_t> | 100 | 1000 | 5844955.00 ticks | 58.45 ticks | 16.7 ns +RandenPool<uint64_t> | 1000 | 500 | 28767404.00 ticks | 57.53 ticks | 16.4 ns + +PoolURBG<uint32_t,8> | 1 | 1000 | 86431.00 ticks | 86.43 ticks | 24.7 ns +PoolURBG<uint32_t,8> | 10 | 1000 | 206191.00 ticks | 20.62 ticks | 5.9 ns +PoolURBG<uint32_t,8> | 100 | 1000 | 1516049.00 ticks | 15.16 ticks | 4.3 ns +PoolURBG<uint32_t,8> | 1000 | 500 | 7613936.00 ticks | 15.23 ticks | 4.4 ns + +PoolURBG<uint64_t,4> | 1 | 1000 | 96668.00 ticks | 96.67 ticks | 27.6 ns +PoolURBG<uint64_t,4> | 10 | 1000 | 282423.00 ticks | 28.24 ticks | 8.1 ns +PoolURBG<uint64_t,4> | 100 | 1000 | 2609587.00 ticks | 26.10 ticks | 7.5 ns +PoolURBG<uint64_t,4> | 1000 | 500 | 12408757.00 ticks | 24.82 ticks | 7.1 ns + +*/ diff --git a/third_party/abseil_cpp/absl/random/internal/randen.cc b/third_party/abseil_cpp/absl/random/internal/randen.cc new file mode 100644 index 000000000000..78a1e00c0832 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen.cc @@ -0,0 +1,91 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen.h" + +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/randen_detect.h" + +// RANDen = RANDom generator or beetroots in Swiss German. +// 'Strong' (well-distributed, unpredictable, backtracking-resistant) random +// generator, faster in some benchmarks than std::mt19937_64 and pcg64_c32. +// +// High-level summary: +// 1) Reverie (see "A Robust and Sponge-Like PRNG with Improved Efficiency") is +// a sponge-like random generator that requires a cryptographic permutation. +// It improves upon "Provably Robust Sponge-Based PRNGs and KDFs" by +// achieving backtracking resistance with only one Permute() per buffer. +// +// 2) "Simpira v2: A Family of Efficient Permutations Using the AES Round +// Function" constructs up to 1024-bit permutations using an improved +// Generalized Feistel network with 2-round AES-128 functions. This Feistel +// block shuffle achieves diffusion faster and is less vulnerable to +// sliced-biclique attacks than the Type-2 cyclic shuffle. +// +// 3) "Improving the Generalized Feistel" and "New criterion for diffusion +// property" extends the same kind of improved Feistel block shuffle to 16 +// branches, which enables a 2048-bit permutation. +// +// We combine these three ideas and also change Simpira's subround keys from +// structured/low-entropy counters to digits of Pi. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +struct RandenState { + const void* keys; + bool has_crypto; +}; + +RandenState GetRandenState() { + static const RandenState state = []() { + RandenState tmp; +#if ABSL_RANDOM_INTERNAL_AES_DISPATCH + // HW AES Dispatch. + if (HasRandenHwAesImplementation() && CPUSupportsRandenHwAes()) { + tmp.has_crypto = true; + tmp.keys = RandenHwAes::GetKeys(); + } else { + tmp.has_crypto = false; + tmp.keys = RandenSlow::GetKeys(); + } +#elif ABSL_HAVE_ACCELERATED_AES + // HW AES is enabled. + tmp.has_crypto = true; + tmp.keys = RandenHwAes::GetKeys(); +#else + // HW AES is disabled. + tmp.has_crypto = false; + tmp.keys = RandenSlow::GetKeys(); +#endif + return tmp; + }(); + return state; +} + +} // namespace + +Randen::Randen() { + auto tmp = GetRandenState(); + keys_ = tmp.keys; +#if ABSL_RANDOM_INTERNAL_AES_DISPATCH + has_crypto_ = tmp.has_crypto; +#endif +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/randen.h b/third_party/abseil_cpp/absl/random/internal/randen.h new file mode 100644 index 000000000000..c2834aaf3d2c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen.h @@ -0,0 +1,102 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_RANDEN_H_ +#define ABSL_RANDOM_INTERNAL_RANDEN_H_ + +#include <cstddef> + +#include "absl/random/internal/platform.h" +#include "absl/random/internal/randen_hwaes.h" +#include "absl/random/internal/randen_slow.h" +#include "absl/random/internal/randen_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// RANDen = RANDom generator or beetroots in Swiss German. +// 'Strong' (well-distributed, unpredictable, backtracking-resistant) random +// generator, faster in some benchmarks than std::mt19937_64 and pcg64_c32. +// +// Randen implements the basic state manipulation methods. +class Randen { + public: + static constexpr size_t kStateBytes = RandenTraits::kStateBytes; + static constexpr size_t kCapacityBytes = RandenTraits::kCapacityBytes; + static constexpr size_t kSeedBytes = RandenTraits::kSeedBytes; + + ~Randen() = default; + + Randen(); + + // Generate updates the randen sponge. The outer portion of the sponge + // (kCapacityBytes .. kStateBytes) may be consumed as PRNG state. + template <typename T, size_t N> + void Generate(T (&state)[N]) const { + static_assert(N * sizeof(T) == kStateBytes, + "Randen::Generate() requires kStateBytes of state"); +#if ABSL_RANDOM_INTERNAL_AES_DISPATCH + // HW AES Dispatch. + if (has_crypto_) { + RandenHwAes::Generate(keys_, state); + } else { + RandenSlow::Generate(keys_, state); + } +#elif ABSL_HAVE_ACCELERATED_AES + // HW AES is enabled. + RandenHwAes::Generate(keys_, state); +#else + // HW AES is disabled. + RandenSlow::Generate(keys_, state); +#endif + } + + // Absorb incorporates additional seed material into the randen sponge. After + // absorb returns, Generate must be called before the state may be consumed. + template <typename S, size_t M, typename T, size_t N> + void Absorb(const S (&seed)[M], T (&state)[N]) const { + static_assert(M * sizeof(S) == RandenTraits::kSeedBytes, + "Randen::Absorb() requires kSeedBytes of seed"); + + static_assert(N * sizeof(T) == RandenTraits::kStateBytes, + "Randen::Absorb() requires kStateBytes of state"); +#if ABSL_RANDOM_INTERNAL_AES_DISPATCH + // HW AES Dispatch. + if (has_crypto_) { + RandenHwAes::Absorb(seed, state); + } else { + RandenSlow::Absorb(seed, state); + } +#elif ABSL_HAVE_ACCELERATED_AES + // HW AES is enabled. + RandenHwAes::Absorb(seed, state); +#else + // HW AES is disabled. + RandenSlow::Absorb(seed, state); +#endif + } + + private: + const void* keys_; +#if ABSL_RANDOM_INTERNAL_AES_DISPATCH + bool has_crypto_; +#endif +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_RANDEN_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/randen_benchmarks.cc b/third_party/abseil_cpp/absl/random/internal/randen_benchmarks.cc new file mode 100644 index 000000000000..f589172c0466 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_benchmarks.cc @@ -0,0 +1,174 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include "absl/random/internal/randen.h" + +#include <cstdint> +#include <cstdio> +#include <cstring> + +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/nanobenchmark.h" +#include "absl/random/internal/platform.h" +#include "absl/random/internal/randen_engine.h" +#include "absl/random/internal/randen_hwaes.h" +#include "absl/random/internal/randen_slow.h" +#include "absl/strings/numbers.h" + +namespace { + +using absl::random_internal::Randen; +using absl::random_internal::RandenHwAes; +using absl::random_internal::RandenSlow; + +using absl::random_internal_nanobenchmark::FuncInput; +using absl::random_internal_nanobenchmark::FuncOutput; +using absl::random_internal_nanobenchmark::InvariantTicksPerSecond; +using absl::random_internal_nanobenchmark::MeasureClosure; +using absl::random_internal_nanobenchmark::Params; +using absl::random_internal_nanobenchmark::PinThreadToCPU; +using absl::random_internal_nanobenchmark::Result; + +// Local state parameters. +static constexpr size_t kStateSizeT = Randen::kStateBytes / sizeof(uint64_t); +static constexpr size_t kSeedSizeT = Randen::kSeedBytes / sizeof(uint32_t); + +// Randen implementation benchmarks. +template <typename T> +struct AbsorbFn : public T { + mutable uint64_t state[kStateSizeT] = {}; + mutable uint32_t seed[kSeedSizeT] = {}; + + static constexpr size_t bytes() { return sizeof(seed); } + + FuncOutput operator()(const FuncInput num_iters) const { + for (size_t i = 0; i < num_iters; ++i) { + this->Absorb(seed, state); + } + return state[0]; + } +}; + +template <typename T> +struct GenerateFn : public T { + mutable uint64_t state[kStateSizeT]; + GenerateFn() { std::memset(state, 0, sizeof(state)); } + + static constexpr size_t bytes() { return sizeof(state); } + + FuncOutput operator()(const FuncInput num_iters) const { + const auto* keys = this->GetKeys(); + for (size_t i = 0; i < num_iters; ++i) { + this->Generate(keys, state); + } + return state[0]; + } +}; + +template <typename UInt> +struct Engine { + mutable absl::random_internal::randen_engine<UInt> rng; + + static constexpr size_t bytes() { return sizeof(UInt); } + + FuncOutput operator()(const FuncInput num_iters) const { + for (size_t i = 0; i < num_iters - 1; ++i) { + rng(); + } + return rng(); + } +}; + +template <size_t N> +void Print(const char* name, const size_t n, const Result (&results)[N], + const size_t bytes) { + if (n == 0) { + ABSL_RAW_LOG( + WARNING, + "WARNING: Measurement failed, should not happen when using " + "PinThreadToCPU unless the region to measure takes > 1 second.\n"); + return; + } + + static const double ns_per_tick = 1e9 / InvariantTicksPerSecond(); + static constexpr const double kNsPerS = 1e9; // ns/s + static constexpr const double kMBPerByte = 1.0 / 1048576.0; // Mb / b + static auto header = [] { + return printf("%20s %8s: %12s ticks; %9s (%9s) %8s\n", "Name", "Count", + "Total", "Variance", "Time", "bytes/s"); + }(); + (void)header; + + for (size_t i = 0; i < n; ++i) { + const double ticks_per_call = results[i].ticks / results[i].input; + const double ns_per_call = ns_per_tick * ticks_per_call; + const double bytes_per_ns = bytes / ns_per_call; + const double mb_per_s = bytes_per_ns * kNsPerS * kMBPerByte; + // Output + printf("%20s %8zu: %12.2f ticks; MAD=%4.2f%% (%6.1f ns) %8.1f Mb/s\n", + name, results[i].input, results[i].ticks, + results[i].variability * 100.0, ns_per_call, mb_per_s); + } +} + +// Fails here +template <typename Op, size_t N> +void Measure(const char* name, const FuncInput (&inputs)[N]) { + Op op; + + Result results[N]; + Params params; + params.verbose = false; + params.max_evals = 6; // avoid test timeout + const size_t num_results = MeasureClosure(op, inputs, N, results, params); + Print(name, num_results, results, op.bytes()); +} + +// unpredictable == 1 but the compiler does not know that. +void RunAll(const int argc, char* argv[]) { + if (argc == 2) { + int cpu = -1; + if (!absl::SimpleAtoi(argv[1], &cpu)) { + ABSL_RAW_LOG(FATAL, "The optional argument must be a CPU number >= 0.\n"); + } + PinThreadToCPU(cpu); + } + + // The compiler cannot reduce this to a constant. + const FuncInput unpredictable = (argc != 999); + static const FuncInput inputs[] = {unpredictable * 100, unpredictable * 1000}; + +#if !defined(ABSL_INTERNAL_DISABLE_AES) && ABSL_HAVE_ACCELERATED_AES + Measure<AbsorbFn<RandenHwAes>>("Absorb (HwAes)", inputs); +#endif + Measure<AbsorbFn<RandenSlow>>("Absorb (Slow)", inputs); + +#if !defined(ABSL_INTERNAL_DISABLE_AES) && ABSL_HAVE_ACCELERATED_AES + Measure<GenerateFn<RandenHwAes>>("Generate (HwAes)", inputs); +#endif + Measure<GenerateFn<RandenSlow>>("Generate (Slow)", inputs); + + // Measure the production engine. + static const FuncInput inputs1[] = {unpredictable * 1000, + unpredictable * 10000}; + Measure<Engine<uint64_t>>("randen_engine<uint64_t>", inputs1); + Measure<Engine<uint32_t>>("randen_engine<uint32_t>", inputs1); +} + +} // namespace + +int main(int argc, char* argv[]) { + RunAll(argc, argv); + return 0; +} diff --git a/third_party/abseil_cpp/absl/random/internal/randen_detect.cc b/third_party/abseil_cpp/absl/random/internal/randen_detect.cc new file mode 100644 index 000000000000..bbe7b965329c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_detect.cc @@ -0,0 +1,221 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// HERMETIC NOTE: The randen_hwaes target must not introduce duplicate +// symbols from arbitrary system and other headers, since it may be built +// with different flags from other targets, using different levels of +// optimization, potentially introducing ODR violations. + +#include "absl/random/internal/randen_detect.h" + +#include <cstdint> +#include <cstring> + +#include "absl/random/internal/platform.h" + +#if defined(ABSL_ARCH_X86_64) +#define ABSL_INTERNAL_USE_X86_CPUID +#elif defined(ABSL_ARCH_PPC) || defined(ABSL_ARCH_ARM) || \ + defined(ABSL_ARCH_AARCH64) +#if defined(__ANDROID__) +#define ABSL_INTERNAL_USE_ANDROID_GETAUXVAL +#define ABSL_INTERNAL_USE_GETAUXVAL +#elif defined(__linux__) +#define ABSL_INTERNAL_USE_LINUX_GETAUXVAL +#define ABSL_INTERNAL_USE_GETAUXVAL +#endif +#endif + +#if defined(ABSL_INTERNAL_USE_X86_CPUID) +#if defined(_WIN32) || defined(_WIN64) +#include <intrin.h> // NOLINT(build/include_order) +#pragma intrinsic(__cpuid) +#else +// MSVC-equivalent __cpuid intrinsic function. +static void __cpuid(int cpu_info[4], int info_type) { + __asm__ volatile("cpuid \n\t" + : "=a"(cpu_info[0]), "=b"(cpu_info[1]), "=c"(cpu_info[2]), + "=d"(cpu_info[3]) + : "a"(info_type), "c"(0)); +} +#endif +#endif // ABSL_INTERNAL_USE_X86_CPUID + +// On linux, just use the c-library getauxval call. +#if defined(ABSL_INTERNAL_USE_LINUX_GETAUXVAL) + +extern "C" unsigned long getauxval(unsigned long type); // NOLINT(runtime/int) + +static uint32_t GetAuxval(uint32_t hwcap_type) { + return static_cast<uint32_t>(getauxval(hwcap_type)); +} + +#endif + +// On android, probe the system's C library for getauxval(). +// This is the same technique used by the android NDK cpu features library +// as well as the google open-source cpu_features library. +// +// TODO(absl-team): Consider implementing a fallback of directly reading +// /proc/self/auxval. +#if defined(ABSL_INTERNAL_USE_ANDROID_GETAUXVAL) +#include <dlfcn.h> + +static uint32_t GetAuxval(uint32_t hwcap_type) { + // NOLINTNEXTLINE(runtime/int) + typedef unsigned long (*getauxval_func_t)(unsigned long); + + dlerror(); // Cleaning error state before calling dlopen. + void* libc_handle = dlopen("libc.so", RTLD_NOW); + if (!libc_handle) { + return 0; + } + uint32_t result = 0; + void* sym = dlsym(libc_handle, "getauxval"); + if (sym) { + getauxval_func_t func; + memcpy(&func, &sym, sizeof(func)); + result = static_cast<uint32_t>((*func)(hwcap_type)); + } + dlclose(libc_handle); + return result; +} + +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// The default return at the end of the function might be unreachable depending +// on the configuration. Ignore that warning. +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunreachable-code-return" +#endif + +// CPUSupportsRandenHwAes returns whether the CPU is a microarchitecture +// which supports the crpyto/aes instructions or extensions necessary to use the +// accelerated RandenHwAes implementation. +// +// 1. For x86 it is sufficient to use the CPUID instruction to detect whether +// the cpu supports AES instructions. Done. +// +// Fon non-x86 it is much more complicated. +// +// 2. When ABSL_INTERNAL_USE_GETAUXVAL is defined, use getauxval() (either +// the direct c-library version, or the android probing version which loads +// libc), and read the hardware capability bits. +// This is based on the technique used by boringssl uses to detect +// cpu capabilities, and should allow us to enable crypto in the android +// builds where it is supported. +// +// 3. Use the default for the compiler architecture. +// + +bool CPUSupportsRandenHwAes() { +#if defined(ABSL_INTERNAL_USE_X86_CPUID) + // 1. For x86: Use CPUID to detect the required AES instruction set. + int regs[4]; + __cpuid(reinterpret_cast<int*>(regs), 1); + return regs[2] & (1 << 25); // AES + +#elif defined(ABSL_INTERNAL_USE_GETAUXVAL) + // 2. Use getauxval() to read the hardware bits and determine + // cpu capabilities. + +#define AT_HWCAP 16 +#define AT_HWCAP2 26 +#if defined(ABSL_ARCH_PPC) + // For Power / PPC: Expect that the cpu supports VCRYPTO + // See https://members.openpowerfoundation.org/document/dl/576 + // VCRYPTO should be present in POWER8 >= 2.07. + // Uses Linux kernel constants from arch/powerpc/include/uapi/asm/cputable.h + static const uint32_t kVCRYPTO = 0x02000000; + const uint32_t hwcap = GetAuxval(AT_HWCAP2); + return (hwcap & kVCRYPTO) != 0; + +#elif defined(ABSL_ARCH_ARM) + // For ARM: Require crypto+neon + // http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500f/CIHBIBBA.html + // Uses Linux kernel constants from arch/arm64/include/asm/hwcap.h + static const uint32_t kNEON = 1 << 12; + uint32_t hwcap = GetAuxval(AT_HWCAP); + if ((hwcap & kNEON) == 0) { + return false; + } + + // And use it again to detect AES. + static const uint32_t kAES = 1 << 0; + const uint32_t hwcap2 = GetAuxval(AT_HWCAP2); + return (hwcap2 & kAES) != 0; + +#elif defined(ABSL_ARCH_AARCH64) + // For AARCH64: Require crypto+neon + // http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.ddi0500f/CIHBIBBA.html + static const uint32_t kNEON = 1 << 1; + static const uint32_t kAES = 1 << 3; + const uint32_t hwcap = GetAuxval(AT_HWCAP); + return ((hwcap & kNEON) != 0) && ((hwcap & kAES) != 0); +#endif + +#else // ABSL_INTERNAL_USE_GETAUXVAL + // 3. By default, assume that the compiler default. + return ABSL_HAVE_ACCELERATED_AES ? true : false; + +#endif + // NOTE: There are some other techniques that may be worth trying: + // + // * Use an environment variable: ABSL_RANDOM_USE_HWAES + // + // * Rely on compiler-generated target-based dispatch. + // Using x86/gcc it might look something like this: + // + // int __attribute__((target("aes"))) HasAes() { return 1; } + // int __attribute__((target("default"))) HasAes() { return 0; } + // + // This does not work on all architecture/compiler combinations. + // + // * On Linux consider reading /proc/cpuinfo and/or /proc/self/auxv. + // These files have lines which are easy to parse; for ARM/AARCH64 it is quite + // easy to find the Features: line and extract aes / neon. Likewise for + // PPC. + // + // * Fork a process and test for SIGILL: + // + // * Many architectures have instructions to read the ISA. Unfortunately + // most of those require that the code is running in ring 0 / + // protected-mode. + // + // There are several examples. e.g. Valgrind detects PPC ISA 2.07: + // https://github.com/lu-zero/valgrind/blob/master/none/tests/ppc64/test_isa_2_07_part1.c + // + // MRS <Xt>, ID_AA64ISAR0_EL1 ; Read ID_AA64ISAR0_EL1 into Xt + // + // uint64_t val; + // __asm __volatile("mrs %0, id_aa64isar0_el1" :"=&r" (val)); + // + // * Use a CPUID-style heuristic database. + // + // * On Apple (__APPLE__), AES is available on Arm v8. + // https://stackoverflow.com/questions/45637888/how-to-determine-armv8-features-at-runtime-on-ios +} + +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/randen_detect.h b/third_party/abseil_cpp/absl/random/internal/randen_detect.h new file mode 100644 index 000000000000..f283f4322647 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_detect.h @@ -0,0 +1,33 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_RANDEN_DETECT_H_ +#define ABSL_RANDOM_INTERNAL_RANDEN_DETECT_H_ + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Returns whether the current CPU supports RandenHwAes implementation. +// This typically involves supporting cryptographic extensions on whichever +// platform is currently running. +bool CPUSupportsRandenHwAes(); + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_RANDEN_DETECT_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/randen_engine.h b/third_party/abseil_cpp/absl/random/internal/randen_engine.h new file mode 100644 index 000000000000..6b33731336e7 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_engine.h @@ -0,0 +1,230 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_RANDEN_ENGINE_H_ +#define ABSL_RANDOM_INTERNAL_RANDEN_ENGINE_H_ + +#include <algorithm> +#include <cinttypes> +#include <cstdlib> +#include <iostream> +#include <iterator> +#include <limits> +#include <type_traits> + +#include "absl/meta/type_traits.h" +#include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/internal/randen.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Deterministic pseudorandom byte generator with backtracking resistance +// (leaking the state does not compromise prior outputs). Based on Reverie +// (see "A Robust and Sponge-Like PRNG with Improved Efficiency") instantiated +// with an improved Simpira-like permutation. +// Returns values of type "T" (must be a built-in unsigned integer type). +// +// RANDen = RANDom generator or beetroots in Swiss High German. +// 'Strong' (well-distributed, unpredictable, backtracking-resistant) random +// generator, faster in some benchmarks than std::mt19937_64 and pcg64_c32. +template <typename T> +class alignas(16) randen_engine { + public: + // C++11 URBG interface: + using result_type = T; + static_assert(std::is_unsigned<result_type>::value, + "randen_engine template argument must be a built-in unsigned " + "integer type"); + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + explicit randen_engine(result_type seed_value = 0) { seed(seed_value); } + + template <class SeedSequence, + typename = typename absl::enable_if_t< + !std::is_same<SeedSequence, randen_engine>::value>> + explicit randen_engine(SeedSequence&& seq) { + seed(seq); + } + + randen_engine(const randen_engine&) = default; + + // Returns random bits from the buffer in units of result_type. + result_type operator()() { + // Refill the buffer if needed (unlikely). + if (next_ >= kStateSizeT) { + next_ = kCapacityT; + impl_.Generate(state_); + } + + return state_[next_++]; + } + + template <class SeedSequence> + typename absl::enable_if_t< + !std::is_convertible<SeedSequence, result_type>::value> + seed(SeedSequence&& seq) { + // Zeroes the state. + seed(); + reseed(seq); + } + + void seed(result_type seed_value = 0) { + next_ = kStateSizeT; + // Zeroes the inner state and fills the outer state with seed_value to + // mimics behaviour of reseed + std::fill(std::begin(state_), std::begin(state_) + kCapacityT, 0); + std::fill(std::begin(state_) + kCapacityT, std::end(state_), seed_value); + } + + // Inserts entropy into (part of) the state. Calling this periodically with + // sufficient entropy ensures prediction resistance (attackers cannot predict + // future outputs even if state is compromised). + template <class SeedSequence> + void reseed(SeedSequence& seq) { + using sequence_result_type = typename SeedSequence::result_type; + static_assert(sizeof(sequence_result_type) == 4, + "SeedSequence::result_type must be 32-bit"); + + constexpr size_t kBufferSize = + Randen::kSeedBytes / sizeof(sequence_result_type); + alignas(16) sequence_result_type buffer[kBufferSize]; + + // Randen::Absorb XORs the seed into state, which is then mixed by a call + // to Randen::Generate. Seeding with only the provided entropy is preferred + // to using an arbitrary generate() call, so use [rand.req.seed_seq] + // size as a proxy for the number of entropy units that can be generated + // without relying on seed sequence mixing... + const size_t entropy_size = seq.size(); + if (entropy_size < kBufferSize) { + // ... and only request that many values, or 256-bits, when unspecified. + const size_t requested_entropy = (entropy_size == 0) ? 8u : entropy_size; + std::fill(std::begin(buffer) + requested_entropy, std::end(buffer), 0); + seq.generate(std::begin(buffer), std::begin(buffer) + requested_entropy); + // The Randen paper suggests preferentially initializing even-numbered + // 128-bit vectors of the randen state (there are 16 such vectors). + // The seed data is merged into the state offset by 128-bits, which + // implies prefering seed bytes [16..31, ..., 208..223]. Since the + // buffer is 32-bit values, we swap the corresponding buffer positions in + // 128-bit chunks. + size_t dst = kBufferSize; + while (dst > 7) { + // leave the odd bucket as-is. + dst -= 4; + size_t src = dst >> 1; + // swap 128-bits into the even bucket + std::swap(buffer[--dst], buffer[--src]); + std::swap(buffer[--dst], buffer[--src]); + std::swap(buffer[--dst], buffer[--src]); + std::swap(buffer[--dst], buffer[--src]); + } + } else { + seq.generate(std::begin(buffer), std::end(buffer)); + } + impl_.Absorb(buffer, state_); + + // Generate will be called when operator() is called + next_ = kStateSizeT; + } + + void discard(uint64_t count) { + uint64_t step = std::min<uint64_t>(kStateSizeT - next_, count); + count -= step; + + constexpr uint64_t kRateT = kStateSizeT - kCapacityT; + while (count > 0) { + next_ = kCapacityT; + impl_.Generate(state_); + step = std::min<uint64_t>(kRateT, count); + count -= step; + } + next_ += step; + } + + bool operator==(const randen_engine& other) const { + return next_ == other.next_ && + std::equal(std::begin(state_), std::end(state_), + std::begin(other.state_)); + } + + bool operator!=(const randen_engine& other) const { + return !(*this == other); + } + + template <class CharT, class Traits> + friend std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const randen_engine<T>& engine) { // NOLINT(runtime/references) + using numeric_type = + typename random_internal::stream_format_type<result_type>::type; + auto saver = random_internal::make_ostream_state_saver(os); + for (const auto& elem : engine.state_) { + // In the case that `elem` is `uint8_t`, it must be cast to something + // larger so that it prints as an integer rather than a character. For + // simplicity, apply the cast all circumstances. + os << static_cast<numeric_type>(elem) << os.fill(); + } + os << engine.next_; + return os; + } + + template <class CharT, class Traits> + friend std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + randen_engine<T>& engine) { // NOLINT(runtime/references) + using numeric_type = + typename random_internal::stream_format_type<result_type>::type; + result_type state[kStateSizeT]; + size_t next; + for (auto& elem : state) { + // It is not possible to read uint8_t from wide streams, so it is + // necessary to read a wider type and then cast it to uint8_t. + numeric_type value; + is >> value; + elem = static_cast<result_type>(value); + } + is >> next; + if (is.fail()) { + return is; + } + std::memcpy(engine.state_, state, sizeof(engine.state_)); + engine.next_ = next; + return is; + } + + private: + static constexpr size_t kStateSizeT = + Randen::kStateBytes / sizeof(result_type); + static constexpr size_t kCapacityT = + Randen::kCapacityBytes / sizeof(result_type); + + // First kCapacityT are `inner', the others are accessible random bits. + alignas(16) result_type state_[kStateSizeT]; + size_t next_; // index within state_ + Randen impl_; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_RANDEN_ENGINE_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/randen_engine_test.cc b/third_party/abseil_cpp/absl/random/internal/randen_engine_test.cc new file mode 100644 index 000000000000..c8e7685bddad --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_engine_test.cc @@ -0,0 +1,656 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen_engine.h" + +#include <algorithm> +#include <bitset> +#include <random> +#include <sstream> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/explicit_seed_seq.h" +#include "absl/strings/str_cat.h" +#include "absl/time/clock.h" + +#define UPDATE_GOLDEN 0 + +using randen_u64 = absl::random_internal::randen_engine<uint64_t>; +using randen_u32 = absl::random_internal::randen_engine<uint32_t>; +using absl::random_internal::ExplicitSeedSeq; + +namespace { + +template <typename UIntType> +class RandenEngineTypedTest : public ::testing::Test {}; + +using UIntTypes = ::testing::Types<uint8_t, uint16_t, uint32_t, uint64_t>; + +TYPED_TEST_SUITE(RandenEngineTypedTest, UIntTypes); + +TYPED_TEST(RandenEngineTypedTest, VerifyReseedChangesAllValues) { + using randen = typename absl::random_internal::randen_engine<TypeParam>; + using result_type = typename randen::result_type; + + const size_t kNumOutputs = (sizeof(randen) * 2 / sizeof(TypeParam)) + 1; + randen engine; + + // MSVC emits error 2719 without the use of std::ref below. + // * formal parameter with __declspec(align('#')) won't be aligned + + { + std::seed_seq seq1{1, 2, 3, 4, 5, 6, 7}; + engine.seed(seq1); + } + result_type a[kNumOutputs]; + std::generate(std::begin(a), std::end(a), std::ref(engine)); + + { + std::random_device rd; + std::seed_seq seq2{rd(), rd(), rd()}; + engine.seed(seq2); + } + result_type b[kNumOutputs]; + std::generate(std::begin(b), std::end(b), std::ref(engine)); + + // Test that generated sequence changed as sequence of bits, i.e. if about + // half of the bites were flipped between two non-correlated values. + size_t changed_bits = 0; + size_t unchanged_bits = 0; + size_t total_set = 0; + size_t total_bits = 0; + size_t equal_count = 0; + for (size_t i = 0; i < kNumOutputs; ++i) { + equal_count += (a[i] == b[i]) ? 1 : 0; + std::bitset<sizeof(result_type) * 8> bitset(a[i] ^ b[i]); + changed_bits += bitset.count(); + unchanged_bits += bitset.size() - bitset.count(); + + std::bitset<sizeof(result_type) * 8> a_set(a[i]); + std::bitset<sizeof(result_type) * 8> b_set(b[i]); + total_set += a_set.count() + b_set.count(); + total_bits += 2 * 8 * sizeof(result_type); + } + // On average, half the bits are changed between two calls. + EXPECT_LE(changed_bits, 0.60 * (changed_bits + unchanged_bits)); + EXPECT_GE(changed_bits, 0.40 * (changed_bits + unchanged_bits)); + + // Verify using a quick normal-approximation to the binomial. + EXPECT_NEAR(total_set, total_bits * 0.5, 4 * std::sqrt(total_bits)) + << "@" << total_set / static_cast<double>(total_bits); + + // Also, A[i] == B[i] with probability (1/range) * N. + // Give this a pretty wide latitude, though. + const double kExpected = kNumOutputs / (1.0 * sizeof(result_type) * 8); + EXPECT_LE(equal_count, 1.0 + kExpected); +} + +// Number of values that needs to be consumed to clean two sizes of buffer +// and trigger third refresh. (slightly overestimates the actual state size). +constexpr size_t kTwoBufferValues = sizeof(randen_u64) / sizeof(uint16_t) + 1; + +TYPED_TEST(RandenEngineTypedTest, VerifyDiscard) { + using randen = typename absl::random_internal::randen_engine<TypeParam>; + + for (size_t num_used = 0; num_used < kTwoBufferValues; ++num_used) { + randen engine_used; + for (size_t i = 0; i < num_used; ++i) { + engine_used(); + } + + for (size_t num_discard = 0; num_discard < kTwoBufferValues; + ++num_discard) { + randen engine1 = engine_used; + randen engine2 = engine_used; + for (size_t i = 0; i < num_discard; ++i) { + engine1(); + } + engine2.discard(num_discard); + for (size_t i = 0; i < kTwoBufferValues; ++i) { + const auto r1 = engine1(); + const auto r2 = engine2(); + ASSERT_EQ(r1, r2) << "used=" << num_used << " discard=" << num_discard; + } + } + } +} + +TYPED_TEST(RandenEngineTypedTest, StreamOperatorsResult) { + using randen = typename absl::random_internal::randen_engine<TypeParam>; + std::wostringstream os; + std::wistringstream is; + randen engine; + + EXPECT_EQ(&(os << engine), &os); + EXPECT_EQ(&(is >> engine), &is); +} + +TYPED_TEST(RandenEngineTypedTest, StreamSerialization) { + using randen = typename absl::random_internal::randen_engine<TypeParam>; + + for (size_t discard = 0; discard < kTwoBufferValues; ++discard) { + ExplicitSeedSeq seed_sequence{12, 34, 56}; + randen engine(seed_sequence); + engine.discard(discard); + + std::stringstream stream; + stream << engine; + + randen new_engine; + stream >> new_engine; + for (size_t i = 0; i < 64; ++i) { + EXPECT_EQ(engine(), new_engine()) << " " << i; + } + } +} + +constexpr size_t kNumGoldenOutputs = 127; + +// This test is checking if randen_engine is meets interface requirements +// defined in [rand.req.urbg]. +TYPED_TEST(RandenEngineTypedTest, RandomNumberEngineInterface) { + using randen = typename absl::random_internal::randen_engine<TypeParam>; + + using E = randen; + using T = typename E::result_type; + + static_assert(std::is_copy_constructible<E>::value, + "randen_engine must be copy constructible"); + + static_assert(absl::is_copy_assignable<E>::value, + "randen_engine must be copy assignable"); + + static_assert(std::is_move_constructible<E>::value, + "randen_engine must be move constructible"); + + static_assert(absl::is_move_assignable<E>::value, + "randen_engine must be move assignable"); + + static_assert(std::is_same<decltype(std::declval<E>()()), T>::value, + "return type of operator() must be result_type"); + + // Names after definition of [rand.req.urbg] in C++ standard. + // e us a value of E + // v is a lvalue of E + // x, y are possibly const values of E + // s is a value of T + // q is a value satisfying requirements of seed_sequence + // z is a value of type unsigned long long + // os is a some specialization of basic_ostream + // is is a some specialization of basic_istream + + E e, v; + const E x, y; + T s = 1; + std::seed_seq q{1, 2, 3}; + unsigned long long z = 1; // NOLINT(runtime/int) + std::wostringstream os; + std::wistringstream is; + + E{}; + E{x}; + E{s}; + E{q}; + + e.seed(); + + // MSVC emits error 2718 when using EXPECT_EQ(e, x) + // * actual parameter with __declspec(align('#')) won't be aligned + EXPECT_TRUE(e == x); + + e.seed(q); + { + E tmp(q); + EXPECT_TRUE(e == tmp); + } + + e(); + { + E tmp(q); + EXPECT_TRUE(e != tmp); + } + + e.discard(z); + + static_assert(std::is_same<decltype(x == y), bool>::value, + "return type of operator== must be bool"); + + static_assert(std::is_same<decltype(x != y), bool>::value, + "return type of operator== must be bool"); +} + +TYPED_TEST(RandenEngineTypedTest, RandenEngineSFINAETest) { + using randen = typename absl::random_internal::randen_engine<TypeParam>; + using result_type = typename randen::result_type; + + { + randen engine(result_type(1)); + engine.seed(result_type(1)); + } + + { + result_type n = 1; + randen engine(n); + engine.seed(n); + } + + { + randen engine(1); + engine.seed(1); + } + + { + int n = 1; + randen engine(n); + engine.seed(n); + } + + { + std::seed_seq seed_seq; + randen engine(seed_seq); + engine.seed(seed_seq); + } + + { + randen engine{std::seed_seq()}; + engine.seed(std::seed_seq()); + } +} + +TEST(RandenTest, VerifyGoldenRanden64Default) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0xc3c14f134e433977, 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, + 0xf0b780f545c72912, 0x15dbb1d37696599f, 0x30ec63baff3c6d59, + 0xb29f73606f7f20a6, 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, + 0x9cbf605e3fd9de8a, 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, + 0xf4b327fe0fc73c37, 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, + 0xd5af05dd3eff9556, 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, + 0x58d3575834956d42, 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, + 0x497cabf3431154fc, 0x4e24370570029a8b, 0xd88b5749f090e5ea, + 0xc651a582a970692f, 0x78fcec2cbb6342f5, 0x463cb745612f55db, + 0x352ee4ad1816afe3, 0x026ff374c101da7e, 0x811ef0821c3de851, + 0x6f7e616704c4fa59, 0xa0660379992d58fc, 0x04b0a374a3b795c7, + 0x915f3445685da798, 0x26802a8ac76571ce, 0x4663352533ce1882, + 0xb9fdefb4a24dc738, 0x5588ba3a4d6e6c51, 0xa2101a42d35f1956, + 0x607195a5e200f5fd, 0x7e100308f3290764, 0xe1e5e03c759c0709, + 0x082572cc5da6606f, 0xcbcf585399e432f1, 0xe8a2be4f8335d8f1, + 0x0904469acbfee8f2, 0xf08bd31b6daecd51, 0x08e8a1f1a69da69a, + 0x6542a20aad57bff5, 0x2e9705bb053d6b46, 0xda2fc9db0713c391, + 0x78e3a810213b6ffb, 0xdc16a59cdd85f8a6, 0xc0932718cd55781f, + 0xb9bfb29c2b20bfe5, 0xb97289c1be0f2f9c, 0xc0a2a0e403a892d4, + 0x5524bb834771435b, 0x8265da3d39d1a750, 0xff4af3ab8d1b78c5, + 0xf0ec5f424bcad77f, 0x66e455f627495189, 0xc82d3120b57e3270, + 0x3424e47dc22596e3, 0xbc0c95129ccedcdd, 0xc191c595afc4dcbf, + 0x120392bd2bb70939, 0x7f90650ea6cd6ab4, 0x7287491832695ad3, + 0xa7c8fac5a7917eb0, 0xd088cb9418be0361, 0x7c1bf9839c7c1ce5, + 0xe2e991fa58e1e79e, 0x78565cdefd28c4ad, 0x7351b9fef98bafad, + 0x2a9eac28b08c96bf, 0x6c4f179696cb2225, 0x13a685861bab87e0, + 0x64c6de5aa0501971, 0x30537425cac70991, 0x01590d9dc6c532b7, + 0x7e05e3aa8ec720dc, 0x74a07d9c54e3e63f, 0x738184388f3bc1d2, + 0x26ffdc5067be3acb, 0x6bcdf185561f255f, 0xa0eaf2e1cf99b1c6, + 0x171df81934f68604, 0x7ea5a21665683e5a, 0x5d1cb02075ba1cea, + 0x957f38cbd2123fdf, 0xba6364eff80de02f, 0x606e0a0e41d452ee, + 0x892d8317de82f7a2, 0xe707b1db50f7b43e, 0x4eb28826766fcf5b, + 0x5a362d56e80a0951, 0x6ee217df16527d78, 0xf6737962ba6b23dd, + 0x443e63857d4076ca, 0x790d9a5f048adfeb, 0xd796b052151ee94d, + 0x033ed95c12b04a03, 0x8b833ff84893da5d, 0x3d6724b1bb15eab9, + 0x9877c4225061ca76, 0xd68d6810adf74fb3, 0x42e5352fe30ce989, + 0x265b565a7431fde7, 0x3cdbf7e358df4b8b, 0x2922a47f6d3e8779, + 0x52d2242f65b37f88, 0x5d836d6e2958d6b5, 0x29d40f00566d5e26, + 0x288db0e1124b14a0, 0x6c056608b7d9c1b6, 0x0b9471bdb8f19d32, + 0x8fb946504faa6c9d, 0x8943a9464540251c, 0xfd1fe27d144a09e0, + 0xea6ac458da141bda, 0x8048f217633fce36, 0xfeda1384ade74d31, + 0x4334b8b02ff7612f, 0xdbc8441f5227e216, 0x096d119a3605c85b, + 0x2b72b31c21b7d7d0}; + + randen_u64 engine; +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%016lx, ", engine()); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(RandenTest, VerifyGoldenRanden64Seeded) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0x83a9e58f94d3dcd5, 0x70bbdff3d97949fb, 0x0438481f7471c1b4, + 0x34fdc58ee5fb5930, 0xceee4f2d2a937d17, 0xb5a26a68e432aea9, + 0x8b64774a3fb51740, 0xd89ac1fc74249c74, 0x03910d1d23fc3fdf, + 0xd38f630878aa897f, 0x0ee8f0f5615f7e44, 0x98f5a53df8279d52, + 0xb403f52c25938d0e, 0x240072996ea6e838, 0xd3a791246190fa61, + 0xaaedd3df7a7b4f80, 0xc6eacabe05deaf6e, 0xb7967dd8790edf4d, + 0x9a0a8e67e049d279, 0x0494f606aebc23e7, 0x598dcd687bc3e0ee, + 0x010ac81802d452a1, 0x6407c87160aa2842, 0x5a56e276486f93a0, + 0xc887a399d46a8f02, 0x9e1e6100fe93b740, 0x12d02e330f8901f6, + 0xc39ca52b47e790b7, 0xb0b0a2fa11e82e61, 0x1542d841a303806a, + 0x1fe659fd7d6e9d86, 0xb8c90d80746541ac, 0x239d56a5669ddc94, + 0xd40db57c8123d13c, 0x3abc2414153a0db0, 0x9bad665630cb8d61, + 0x0bd1fb90ee3f4bbc, 0x8f0b4d7e079b4e42, 0xfa0fb0e0ee59e793, + 0x51080b283e071100, 0x2c4b9e715081cc15, 0xbe10ed49de4941df, + 0xf8eaac9d4b1b0d37, 0x4bcce4b54605e139, 0xa64722b76765dda6, + 0xb9377d738ca28ab5, 0x779fad81a8ccc1af, 0x65cb3ee61ffd3ba7, + 0xd74e79087862836f, 0xd05b9c584c3f25bf, 0x2ba93a4693579827, + 0xd81530aff05420ce, 0xec06cea215478621, 0x4b1798a6796d65ad, + 0xf142f3fb3a6f6fa6, 0x002b7bf7e237b560, 0xf47f2605ef65b4f8, + 0x9804ec5517effc18, 0xaed3d7f8b7d481cd, 0x5651c24c1ce338d1, + 0x3e7a38208bf0a3c6, 0x6796a7b614534aed, 0x0d0f3b848358460f, + 0x0fa5fe7600b19524, 0x2b0cf38253faaedc, 0x10df9188233a9fd6, + 0x3a10033880138b59, 0x5fb0b0d23948e80f, 0x9e76f7b02fbf5350, + 0x0816052304b1a985, 0x30c9880db41fd218, 0x14aa399b65e20f28, + 0xe1454a8cace787b4, 0x325ac971b6c6f0f5, 0x716b1aa2784f3d36, + 0x3d5ce14accfd144f, 0x6c0c97710f651792, 0xbc5b0f59fb333532, + 0x2a90a7d2140470bc, 0x8da269f55c1e1c8d, 0xcfc37143895792ca, + 0xbe21eab1f30b238f, 0x8c47229dee4d65fd, 0x5743614ed1ed7d54, + 0x351372a99e9c476e, 0x2bd5ea15e5db085f, 0x6925fde46e0af4ca, + 0xed3eda2bdc1f45bd, 0xdef68c68d460fa6e, 0xe42a0de76253e2b5, + 0x4e5176dcbc29c305, 0xbfd85fba9f810f6e, 0x76a5a2a9beb815c6, + 0x01edc4ddceaf414c, 0xa4e98904b4bb3b4b, 0x00bd63ac7d2f1ddd, + 0xb8491fe6e998ddbb, 0xb386a3463dda6800, 0x0081887688871619, + 0x33d394b3344e9a38, 0x815dba65a3a8baf9, 0x4232f6ec02c2fd1a, + 0xb5cff603edd20834, 0x580189243f687663, 0xa8d5a2cbdc27fe99, + 0x725d881693fa0131, 0xa2be2c13db2c7ac5, 0x7b6a9614b509fd78, + 0xb6b136d71e717636, 0x660f1a71aff046ea, 0x0ba10ae346c8ec9e, + 0xe66dde53e3145b41, 0x3b18288c88c26be6, 0x4d9d9d2ff02db933, + 0x4167da8c70f46e8a, 0xf183beef8c6318b4, 0x4d889e1e71eeeef1, + 0x7175c71ad6689b6b, 0xfb9e42beacd1b7dd, 0xc33d0e91b29b5e0d, + 0xd39b83291ce47922, 0xc4d570fb8493d12e, 0x23d5a5724f424ae6, + 0x5245f161876b6616, 0x38d77dbd21ab578d, 0x9c3423311f4ecbfe, + 0x76fe31389bacd9d5, + }; + + ExplicitSeedSeq seed_sequence{12, 34, 56}; + randen_u64 engine(seed_sequence); +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%016lx, ", engine()); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(seed_sequence); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(RandenTest, VerifyGoldenRanden32Default) { + constexpr uint64_t kGolden[2 * kNumGoldenOutputs] = { + 0x4e433977, 0xc3c14f13, 0xd90410ee, 0xdda9f47c, 0x7fd8ca10, 0x887bf308, + 0x45c72912, 0xf0b780f5, 0x7696599f, 0x15dbb1d3, 0xff3c6d59, 0x30ec63ba, + 0x6f7f20a6, 0xb29f7360, 0x6f49a54c, 0x02808a31, 0xd5c8e50e, 0x3b8feaf9, + 0x3fd9de8a, 0x9cbf605e, 0x78183bbb, 0xc970ae1a, 0x56301ed5, 0xd8b2ffd3, + 0x0fc73c37, 0xf4b327fe, 0xeb8f9a19, 0xcdfd8d76, 0x91420c9d, 0xc3a506eb, + 0x3eff9556, 0xd5af05dd, 0x8f83c4a1, 0x48db1bb7, 0x0d6bfe8c, 0x7023920e, + 0x34956d42, 0x58d35758, 0x6b87b840, 0xed1ef4c2, 0x3e0b2df3, 0x8eef32a2, + 0x431154fc, 0x497cabf3, 0x70029a8b, 0x4e243705, 0xf090e5ea, 0xd88b5749, + 0xa970692f, 0xc651a582, 0xbb6342f5, 0x78fcec2c, 0x612f55db, 0x463cb745, + 0x1816afe3, 0x352ee4ad, 0xc101da7e, 0x026ff374, 0x1c3de851, 0x811ef082, + 0x04c4fa59, 0x6f7e6167, 0x992d58fc, 0xa0660379, 0xa3b795c7, 0x04b0a374, + 0x685da798, 0x915f3445, 0xc76571ce, 0x26802a8a, 0x33ce1882, 0x46633525, + 0xa24dc738, 0xb9fdefb4, 0x4d6e6c51, 0x5588ba3a, 0xd35f1956, 0xa2101a42, + 0xe200f5fd, 0x607195a5, 0xf3290764, 0x7e100308, 0x759c0709, 0xe1e5e03c, + 0x5da6606f, 0x082572cc, 0x99e432f1, 0xcbcf5853, 0x8335d8f1, 0xe8a2be4f, + 0xcbfee8f2, 0x0904469a, 0x6daecd51, 0xf08bd31b, 0xa69da69a, 0x08e8a1f1, + 0xad57bff5, 0x6542a20a, 0x053d6b46, 0x2e9705bb, 0x0713c391, 0xda2fc9db, + 0x213b6ffb, 0x78e3a810, 0xdd85f8a6, 0xdc16a59c, 0xcd55781f, 0xc0932718, + 0x2b20bfe5, 0xb9bfb29c, 0xbe0f2f9c, 0xb97289c1, 0x03a892d4, 0xc0a2a0e4, + 0x4771435b, 0x5524bb83, 0x39d1a750, 0x8265da3d, 0x8d1b78c5, 0xff4af3ab, + 0x4bcad77f, 0xf0ec5f42, 0x27495189, 0x66e455f6, 0xb57e3270, 0xc82d3120, + 0xc22596e3, 0x3424e47d, 0x9ccedcdd, 0xbc0c9512, 0xafc4dcbf, 0xc191c595, + 0x2bb70939, 0x120392bd, 0xa6cd6ab4, 0x7f90650e, 0x32695ad3, 0x72874918, + 0xa7917eb0, 0xa7c8fac5, 0x18be0361, 0xd088cb94, 0x9c7c1ce5, 0x7c1bf983, + 0x58e1e79e, 0xe2e991fa, 0xfd28c4ad, 0x78565cde, 0xf98bafad, 0x7351b9fe, + 0xb08c96bf, 0x2a9eac28, 0x96cb2225, 0x6c4f1796, 0x1bab87e0, 0x13a68586, + 0xa0501971, 0x64c6de5a, 0xcac70991, 0x30537425, 0xc6c532b7, 0x01590d9d, + 0x8ec720dc, 0x7e05e3aa, 0x54e3e63f, 0x74a07d9c, 0x8f3bc1d2, 0x73818438, + 0x67be3acb, 0x26ffdc50, 0x561f255f, 0x6bcdf185, 0xcf99b1c6, 0xa0eaf2e1, + 0x34f68604, 0x171df819, 0x65683e5a, 0x7ea5a216, 0x75ba1cea, 0x5d1cb020, + 0xd2123fdf, 0x957f38cb, 0xf80de02f, 0xba6364ef, 0x41d452ee, 0x606e0a0e, + 0xde82f7a2, 0x892d8317, 0x50f7b43e, 0xe707b1db, 0x766fcf5b, 0x4eb28826, + 0xe80a0951, 0x5a362d56, 0x16527d78, 0x6ee217df, 0xba6b23dd, 0xf6737962, + 0x7d4076ca, 0x443e6385, 0x048adfeb, 0x790d9a5f, 0x151ee94d, 0xd796b052, + 0x12b04a03, 0x033ed95c, 0x4893da5d, 0x8b833ff8, 0xbb15eab9, 0x3d6724b1, + 0x5061ca76, 0x9877c422, 0xadf74fb3, 0xd68d6810, 0xe30ce989, 0x42e5352f, + 0x7431fde7, 0x265b565a, 0x58df4b8b, 0x3cdbf7e3, 0x6d3e8779, 0x2922a47f, + 0x65b37f88, 0x52d2242f, 0x2958d6b5, 0x5d836d6e, 0x566d5e26, 0x29d40f00, + 0x124b14a0, 0x288db0e1, 0xb7d9c1b6, 0x6c056608, 0xb8f19d32, 0x0b9471bd, + 0x4faa6c9d, 0x8fb94650, 0x4540251c, 0x8943a946, 0x144a09e0, 0xfd1fe27d, + 0xda141bda, 0xea6ac458, 0x633fce36, 0x8048f217, 0xade74d31, 0xfeda1384, + 0x2ff7612f, 0x4334b8b0, 0x5227e216, 0xdbc8441f, 0x3605c85b, 0x096d119a, + 0x21b7d7d0, 0x2b72b31c}; + + randen_u32 engine; +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < 2 * kNumGoldenOutputs; ++i) { + printf("0x%08x, ", engine()); + if (i % 6 == 5) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(RandenTest, VerifyGoldenRanden32Seeded) { + constexpr uint64_t kGolden[2 * kNumGoldenOutputs] = { + 0x94d3dcd5, 0x83a9e58f, 0xd97949fb, 0x70bbdff3, 0x7471c1b4, 0x0438481f, + 0xe5fb5930, 0x34fdc58e, 0x2a937d17, 0xceee4f2d, 0xe432aea9, 0xb5a26a68, + 0x3fb51740, 0x8b64774a, 0x74249c74, 0xd89ac1fc, 0x23fc3fdf, 0x03910d1d, + 0x78aa897f, 0xd38f6308, 0x615f7e44, 0x0ee8f0f5, 0xf8279d52, 0x98f5a53d, + 0x25938d0e, 0xb403f52c, 0x6ea6e838, 0x24007299, 0x6190fa61, 0xd3a79124, + 0x7a7b4f80, 0xaaedd3df, 0x05deaf6e, 0xc6eacabe, 0x790edf4d, 0xb7967dd8, + 0xe049d279, 0x9a0a8e67, 0xaebc23e7, 0x0494f606, 0x7bc3e0ee, 0x598dcd68, + 0x02d452a1, 0x010ac818, 0x60aa2842, 0x6407c871, 0x486f93a0, 0x5a56e276, + 0xd46a8f02, 0xc887a399, 0xfe93b740, 0x9e1e6100, 0x0f8901f6, 0x12d02e33, + 0x47e790b7, 0xc39ca52b, 0x11e82e61, 0xb0b0a2fa, 0xa303806a, 0x1542d841, + 0x7d6e9d86, 0x1fe659fd, 0x746541ac, 0xb8c90d80, 0x669ddc94, 0x239d56a5, + 0x8123d13c, 0xd40db57c, 0x153a0db0, 0x3abc2414, 0x30cb8d61, 0x9bad6656, + 0xee3f4bbc, 0x0bd1fb90, 0x079b4e42, 0x8f0b4d7e, 0xee59e793, 0xfa0fb0e0, + 0x3e071100, 0x51080b28, 0x5081cc15, 0x2c4b9e71, 0xde4941df, 0xbe10ed49, + 0x4b1b0d37, 0xf8eaac9d, 0x4605e139, 0x4bcce4b5, 0x6765dda6, 0xa64722b7, + 0x8ca28ab5, 0xb9377d73, 0xa8ccc1af, 0x779fad81, 0x1ffd3ba7, 0x65cb3ee6, + 0x7862836f, 0xd74e7908, 0x4c3f25bf, 0xd05b9c58, 0x93579827, 0x2ba93a46, + 0xf05420ce, 0xd81530af, 0x15478621, 0xec06cea2, 0x796d65ad, 0x4b1798a6, + 0x3a6f6fa6, 0xf142f3fb, 0xe237b560, 0x002b7bf7, 0xef65b4f8, 0xf47f2605, + 0x17effc18, 0x9804ec55, 0xb7d481cd, 0xaed3d7f8, 0x1ce338d1, 0x5651c24c, + 0x8bf0a3c6, 0x3e7a3820, 0x14534aed, 0x6796a7b6, 0x8358460f, 0x0d0f3b84, + 0x00b19524, 0x0fa5fe76, 0x53faaedc, 0x2b0cf382, 0x233a9fd6, 0x10df9188, + 0x80138b59, 0x3a100338, 0x3948e80f, 0x5fb0b0d2, 0x2fbf5350, 0x9e76f7b0, + 0x04b1a985, 0x08160523, 0xb41fd218, 0x30c9880d, 0x65e20f28, 0x14aa399b, + 0xace787b4, 0xe1454a8c, 0xb6c6f0f5, 0x325ac971, 0x784f3d36, 0x716b1aa2, + 0xccfd144f, 0x3d5ce14a, 0x0f651792, 0x6c0c9771, 0xfb333532, 0xbc5b0f59, + 0x140470bc, 0x2a90a7d2, 0x5c1e1c8d, 0x8da269f5, 0x895792ca, 0xcfc37143, + 0xf30b238f, 0xbe21eab1, 0xee4d65fd, 0x8c47229d, 0xd1ed7d54, 0x5743614e, + 0x9e9c476e, 0x351372a9, 0xe5db085f, 0x2bd5ea15, 0x6e0af4ca, 0x6925fde4, + 0xdc1f45bd, 0xed3eda2b, 0xd460fa6e, 0xdef68c68, 0x6253e2b5, 0xe42a0de7, + 0xbc29c305, 0x4e5176dc, 0x9f810f6e, 0xbfd85fba, 0xbeb815c6, 0x76a5a2a9, + 0xceaf414c, 0x01edc4dd, 0xb4bb3b4b, 0xa4e98904, 0x7d2f1ddd, 0x00bd63ac, + 0xe998ddbb, 0xb8491fe6, 0x3dda6800, 0xb386a346, 0x88871619, 0x00818876, + 0x344e9a38, 0x33d394b3, 0xa3a8baf9, 0x815dba65, 0x02c2fd1a, 0x4232f6ec, + 0xedd20834, 0xb5cff603, 0x3f687663, 0x58018924, 0xdc27fe99, 0xa8d5a2cb, + 0x93fa0131, 0x725d8816, 0xdb2c7ac5, 0xa2be2c13, 0xb509fd78, 0x7b6a9614, + 0x1e717636, 0xb6b136d7, 0xaff046ea, 0x660f1a71, 0x46c8ec9e, 0x0ba10ae3, + 0xe3145b41, 0xe66dde53, 0x88c26be6, 0x3b18288c, 0xf02db933, 0x4d9d9d2f, + 0x70f46e8a, 0x4167da8c, 0x8c6318b4, 0xf183beef, 0x71eeeef1, 0x4d889e1e, + 0xd6689b6b, 0x7175c71a, 0xacd1b7dd, 0xfb9e42be, 0xb29b5e0d, 0xc33d0e91, + 0x1ce47922, 0xd39b8329, 0x8493d12e, 0xc4d570fb, 0x4f424ae6, 0x23d5a572, + 0x876b6616, 0x5245f161, 0x21ab578d, 0x38d77dbd, 0x1f4ecbfe, 0x9c342331, + 0x9bacd9d5, 0x76fe3138, + }; + + ExplicitSeedSeq seed_sequence{12, 34, 56}; + randen_u32 engine(seed_sequence); +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < 2 * kNumGoldenOutputs; ++i) { + printf("0x%08x, ", engine()); + if (i % 6 == 5) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } + engine.seed(seed_sequence); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(RandenTest, VerifyGoldenFromDeserializedEngine) { + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0x067f9f9ab919657a, 0x0534605912988583, 0x8a303f72feaa673f, + 0x77b7fd747909185c, 0xd9af90403c56d891, 0xd939c6cb204d14b5, + 0x7fbe6b954a47b483, 0x8b31a47cc34c768d, 0x3a9e546da2701a9c, + 0x5246539046253e71, 0x417191ffb2a848a1, 0x7b1c7bf5a5001d09, + 0x9489b15d194f2361, 0xfcebdeea3bcd2461, 0xd643027c854cec97, + 0x5885397f91e0d21c, 0x53173b0efae30d58, 0x1c9c71168449fac1, + 0xe358202b711ed8aa, 0x94e3918ed1d8227c, 0x5bb4e251450144cf, + 0xb5c7a519b489af3b, 0x6f8b560b1f7b3469, 0xfde11dd4a1c74eef, + 0x33383d2f76457dcf, 0x3060c0ec6db9fce1, 0x18f451fcddeec766, + 0xe73c5d6b9f26da2a, 0x8d4cc566671b32a4, 0xb8189b73776bc9ff, + 0x497a70f9caf0bc23, 0x23afcc509791dcea, 0x18af70dc4b27d306, + 0xd3853f955a0ce5b9, 0x441db6c01a0afb17, 0xd0136c3fb8e1f13f, + 0x5e4fd6fc2f33783c, 0xe0d24548adb5da51, 0x0f4d8362a7d3485a, + 0x9f572d68270fa563, 0x6351fbc823024393, 0xa66dbfc61810e9ab, + 0x0ff17fc14b651af8, 0xd74c55dafb99e623, 0x36303bc1ad85c6c2, + 0x4920cd6a2af7e897, 0x0b8848addc30fecd, 0x9e1562eda6488e93, + 0x197553807d607828, 0xbef5eaeda5e21235, 0x18d91d2616aca527, + 0xb7821937f5c873cd, 0x2cd4ae5650dbeefc, 0xb35a64376f75ffdf, + 0x9226d414d647fe07, 0x663f3db455bbb35e, 0xa829eead6ae93247, + 0x7fd69c204dd0d25f, 0xbe1411f891c9acb1, 0xd476f34a506d5f11, + 0xf423d2831649c5ca, 0x1e503962951abd75, 0xeccc9e8b1e34b537, + 0xb11a147294044854, 0xc4cf27f0abf4929d, 0xe9193abf6fa24c8c, + 0xa94a259e3aba8808, 0x21dc414197deffa3, 0xa2ae211d1ff622ae, + 0xfe3995c46be5a4f4, 0xe9984c284bf11128, 0xcb1ce9d2f0851a80, + 0x42fee17971d87cd8, 0xac76a98d177adc88, 0xa0973b3dedc4af6f, + 0xdf56d6bbcb1b8e86, 0xf1e6485f407b11c9, 0x2c63de4deccb15c0, + 0x6fe69db32ed4fad7, 0xaa51a65f84bca1f1, 0x242f2ee81d608afc, + 0x8eb88b2b69fc153b, 0x22c20098baf73fd1, 0x57759466f576488c, + 0x075ca562cea1be9d, 0x9a74814d73d28891, 0x73d1555fc02f4d3d, + 0xc17f8f210ee89337, 0x46cca7999eaeafd4, 0x5db8d6a327a0d8ac, + 0xb79b4f93c738d7a1, 0x9994512f0036ded1, 0xd3883026f38747f4, + 0xf31f7458078d097c, 0x736ce4d480680669, 0x7a496f4c7e1033e3, + 0xecf85bf297fbc68c, 0x9e37e1d0f24f3c4e, 0x15b6e067ca0746fc, + 0xdd4a39905c5db81c, 0xb5dfafa7bcfdf7da, 0xca6646fb6f92a276, + 0x1c6b35f363ef0efd, 0x6a33d06037ad9f76, 0x45544241afd8f80f, + 0x83f8d83f859c90c5, 0x22aea9c5365e8c19, 0xfac35b11f20b6a6a, + 0xd1acf49d1a27dd2f, 0xf281cd09c4fed405, 0x076000a42cd38e4f, + 0x6ace300565070445, 0x463a62781bddc4db, 0x1477126b46b569ac, + 0x127f2bb15035fbb8, 0xdfa30946049c04a8, 0x89072a586ba8dd3e, + 0x62c809582bb7e74d, 0x22c0c3641406c28b, 0x9b66e36c47ff004d, + 0xb9cd2c7519653330, 0x18608d79cd7a598d, 0x92c0bd1323e53e32, + 0x887ff00de8524aa5, 0xa074410b787abd10, 0x18ab41b8057a2063, + 0x1560abf26bc5f987}; + +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + std::seed_seq seed_sequence{1, 2, 3, 4, 5}; + randen_u64 engine(seed_sequence); + std::ostringstream stream; + stream << engine; + auto str = stream.str(); + printf("%s\n\n", str.c_str()); + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + printf("0x%016lx, ", engine()); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); +#else + randen_u64 engine; + std::istringstream stream( + "0 0 9824501439887287479 3242284395352394785 243836530774933777 " + "4047941804708365596 17165468127298385802 949276103645889255 " + "10659970394998657921 1657570836810929787 11697746266668051452 " + "9967209969299905230 14140390331161524430 7383014124183271684 " + "13146719127702337852 13983155220295807171 11121125587542359264 " + "195757810993252695 17138580243103178492 11326030747260920501 " + "8585097322474965590 18342582839328350995 15052982824209724634 " + "7321861343874683609 1806786911778767826 10100850842665572955 " + "9249328950653985078 13600624835326909759 11137960060943860251 " + "10208781341792329629 9282723971471525577 16373271619486811032 32"); + stream >> engine; + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, engine()); + } +#endif +} + +TEST(RandenTest, IsFastOrSlow) { + // randen_engine typically costs ~5ns per value for the optimized code paths, + // and the ~1000ns per value for slow code paths. However when running under + // msan, asan, etc. it can take much longer. + // + // The estimated operation time is something like: + // + // linux, optimized ~5ns + // ppc, optimized ~7ns + // nacl (slow), ~1100ns + // + // `kCount` is chosen below so that, in debug builds and without hardware + // acceleration, the test (assuming ~1us per call) should finish in ~0.1s + static constexpr size_t kCount = 100000; + randen_u64 engine; + randen_u64::result_type sum = 0; + auto start = absl::GetCurrentTimeNanos(); + for (int i = 0; i < kCount; i++) { + sum += engine(); + } + auto duration = absl::GetCurrentTimeNanos() - start; + + ABSL_INTERNAL_LOG(INFO, absl::StrCat(static_cast<double>(duration) / + static_cast<double>(kCount), + "ns")); + + EXPECT_GT(sum, 0); + EXPECT_GE(duration, kCount); // Should be slower than 1ns per call. +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/randen_hwaes.cc b/third_party/abseil_cpp/absl/random/internal/randen_hwaes.cc new file mode 100644 index 000000000000..b5a3f90aee63 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_hwaes.cc @@ -0,0 +1,573 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// HERMETIC NOTE: The randen_hwaes target must not introduce duplicate +// symbols from arbitrary system and other headers, since it may be built +// with different flags from other targets, using different levels of +// optimization, potentially introducing ODR violations. + +#include "absl/random/internal/randen_hwaes.h" + +#include <cstdint> +#include <cstring> + +#include "absl/base/attributes.h" +#include "absl/random/internal/platform.h" +#include "absl/random/internal/randen_traits.h" + +// ABSL_RANDEN_HWAES_IMPL indicates whether this file will contain +// a hardware accelerated implementation of randen, or whether it +// will contain stubs that exit the process. +#if defined(ABSL_ARCH_X86_64) || defined(ABSL_ARCH_X86_32) +// The platform.h directives are sufficient to indicate whether +// we should build accelerated implementations for x86. +#if (ABSL_HAVE_ACCELERATED_AES || ABSL_RANDOM_INTERNAL_AES_DISPATCH) +#define ABSL_RANDEN_HWAES_IMPL 1 +#endif +#elif defined(ABSL_ARCH_PPC) +// The platform.h directives are sufficient to indicate whether +// we should build accelerated implementations for PPC. +// +// NOTE: This has mostly been tested on 64-bit Power variants, +// and not embedded cpus such as powerpc32-8540 +#if ABSL_HAVE_ACCELERATED_AES +#define ABSL_RANDEN_HWAES_IMPL 1 +#endif +#elif defined(ABSL_ARCH_ARM) || defined(ABSL_ARCH_AARCH64) +// ARM is somewhat more complicated. We might support crypto natively... +#if ABSL_HAVE_ACCELERATED_AES || \ + (defined(__ARM_NEON) && defined(__ARM_FEATURE_CRYPTO)) +#define ABSL_RANDEN_HWAES_IMPL 1 + +#elif ABSL_RANDOM_INTERNAL_AES_DISPATCH && !defined(__APPLE__) && \ + (defined(__GNUC__) && __GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ > 9) +// ...or, on GCC, we can use an ASM directive to +// instruct the assember to allow crypto instructions. +#define ABSL_RANDEN_HWAES_IMPL 1 +#define ABSL_RANDEN_HWAES_IMPL_CRYPTO_DIRECTIVE 1 +#endif +#else +// HWAES is unsupported by these architectures / platforms: +// __myriad2__ +// __mips__ +// +// Other architectures / platforms are unknown. +// +// See the Abseil documentation on supported macros at: +// https://abseil.io/docs/cpp/platforms/macros +#endif + +#if !defined(ABSL_RANDEN_HWAES_IMPL) +// No accelerated implementation is supported. +// The RandenHwAes functions are stubs that print an error and exit. + +#include <cstdio> +#include <cstdlib> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// No accelerated implementation. +bool HasRandenHwAesImplementation() { return false; } + +// NOLINTNEXTLINE +const void* RandenHwAes::GetKeys() { + // Attempted to dispatch to an unsupported dispatch target. + const int d = ABSL_RANDOM_INTERNAL_AES_DISPATCH; + fprintf(stderr, "AES Hardware detection failed (%d).\n", d); + exit(1); + return nullptr; +} + +// NOLINTNEXTLINE +void RandenHwAes::Absorb(const void*, void*) { + // Attempted to dispatch to an unsupported dispatch target. + const int d = ABSL_RANDOM_INTERNAL_AES_DISPATCH; + fprintf(stderr, "AES Hardware detection failed (%d).\n", d); + exit(1); +} + +// NOLINTNEXTLINE +void RandenHwAes::Generate(const void*, void*) { + // Attempted to dispatch to an unsupported dispatch target. + const int d = ABSL_RANDOM_INTERNAL_AES_DISPATCH; + fprintf(stderr, "AES Hardware detection failed (%d).\n", d); + exit(1); +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#else // defined(ABSL_RANDEN_HWAES_IMPL) +// +// Accelerated implementations are supported. +// We need the per-architecture includes and defines. +// +namespace { + +using absl::random_internal::RandenTraits; + +// Randen operates on 128-bit vectors. +struct alignas(16) u64x2 { + uint64_t data[2]; +}; + +} // namespace + +// TARGET_CRYPTO defines a crypto attribute for each architecture. +// +// NOTE: Evaluate whether we should eliminate ABSL_TARGET_CRYPTO. +#if (defined(__clang__) || defined(__GNUC__)) +#if defined(ABSL_ARCH_X86_64) || defined(ABSL_ARCH_X86_32) +#define ABSL_TARGET_CRYPTO __attribute__((target("aes"))) +#elif defined(ABSL_ARCH_PPC) +#define ABSL_TARGET_CRYPTO __attribute__((target("crypto"))) +#else +#define ABSL_TARGET_CRYPTO +#endif +#else +#define ABSL_TARGET_CRYPTO +#endif + +#if defined(ABSL_ARCH_PPC) +// NOTE: Keep in mind that PPC can operate in little-endian or big-endian mode, +// however the PPC altivec vector registers (and thus the AES instructions) +// always operate in big-endian mode. + +#include <altivec.h> +// <altivec.h> #defines vector __vector; in C++, this is bad form. +#undef vector +#undef bool + +// Rely on the PowerPC AltiVec vector operations for accelerated AES +// instructions. GCC support of the PPC vector types is described in: +// https://gcc.gnu.org/onlinedocs/gcc-4.9.0/gcc/PowerPC-AltiVec_002fVSX-Built-in-Functions.html +// +// Already provides operator^=. +using Vector128 = __vector unsigned long long; // NOLINT(runtime/int) + +namespace { +inline ABSL_TARGET_CRYPTO Vector128 ReverseBytes(const Vector128& v) { + // Reverses the bytes of the vector. + const __vector unsigned char perm = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; + return vec_perm(v, v, perm); +} + +// WARNING: these load/store in native byte order. It is OK to load and then +// store an unchanged vector, but interpreting the bits as a number or input +// to AES will have undefined results. +inline ABSL_TARGET_CRYPTO Vector128 Vector128Load(const void* from) { + return vec_vsx_ld(0, reinterpret_cast<const Vector128*>(from)); +} + +inline ABSL_TARGET_CRYPTO void Vector128Store(const Vector128& v, void* to) { + vec_vsx_st(v, 0, reinterpret_cast<Vector128*>(to)); +} + +// One round of AES. "round_key" is a public constant for breaking the +// symmetry of AES (ensures previously equal columns differ afterwards). +inline ABSL_TARGET_CRYPTO Vector128 AesRound(const Vector128& state, + const Vector128& round_key) { + return Vector128(__builtin_crypto_vcipher(state, round_key)); +} + +// Enables native loads in the round loop by pre-swapping. +inline ABSL_TARGET_CRYPTO void SwapEndian(u64x2* state) { + for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) { + Vector128Store(ReverseBytes(Vector128Load(state + block)), state + block); + } +} + +} // namespace + +#elif defined(ABSL_ARCH_ARM) || defined(ABSL_ARCH_AARCH64) + +// This asm directive will cause the file to be compiled with crypto extensions +// whether or not the cpu-architecture supports it. +#if ABSL_RANDEN_HWAES_IMPL_CRYPTO_DIRECTIVE +asm(".arch_extension crypto\n"); + +// Override missing defines. +#if !defined(__ARM_NEON) +#define __ARM_NEON 1 +#endif + +#if !defined(__ARM_FEATURE_CRYPTO) +#define __ARM_FEATURE_CRYPTO 1 +#endif + +#endif + +// Rely on the ARM NEON+Crypto advanced simd types, defined in <arm_neon.h>. +// uint8x16_t is the user alias for underlying __simd128_uint8_t type. +// http://infocenter.arm.com/help/topic/com.arm.doc.ihi0073a/IHI0073A_arm_neon_intrinsics_ref.pdf +// +// <arm_neon> defines the following +// +// typedef __attribute__((neon_vector_type(16))) uint8_t uint8x16_t; +// typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t; +// typedef __attribute__((neon_polyvector_type(16))) int8_t poly8x16_t; +// +// vld1q_v +// vst1q_v +// vaeseq_v +// vaesmcq_v +#include <arm_neon.h> + +// Already provides operator^=. +using Vector128 = uint8x16_t; + +namespace { + +inline ABSL_TARGET_CRYPTO Vector128 Vector128Load(const void* from) { + return vld1q_u8(reinterpret_cast<const uint8_t*>(from)); +} + +inline ABSL_TARGET_CRYPTO void Vector128Store(const Vector128& v, void* to) { + vst1q_u8(reinterpret_cast<uint8_t*>(to), v); +} + +// One round of AES. "round_key" is a public constant for breaking the +// symmetry of AES (ensures previously equal columns differ afterwards). +inline ABSL_TARGET_CRYPTO Vector128 AesRound(const Vector128& state, + const Vector128& round_key) { + // It is important to always use the full round function - omitting the + // final MixColumns reduces security [https://eprint.iacr.org/2010/041.pdf] + // and does not help because we never decrypt. + // + // Note that ARM divides AES instructions differently than x86 / PPC, + // And we need to skip the first AddRoundKey step and add an extra + // AddRoundKey step to the end. Lucky for us this is just XOR. + return vaesmcq_u8(vaeseq_u8(state, uint8x16_t{})) ^ round_key; +} + +inline ABSL_TARGET_CRYPTO void SwapEndian(void*) {} + +} // namespace + +#elif defined(ABSL_ARCH_X86_64) || defined(ABSL_ARCH_X86_32) +// On x86 we rely on the aesni instructions +#include <wmmintrin.h> + +namespace { + +// Vector128 class is only wrapper for __m128i, benchmark indicates that it's +// faster than using __m128i directly. +class Vector128 { + public: + // Convert from/to intrinsics. + inline explicit Vector128(const __m128i& Vector128) : data_(Vector128) {} + + inline __m128i data() const { return data_; } + + inline Vector128& operator^=(const Vector128& other) { + data_ = _mm_xor_si128(data_, other.data()); + return *this; + } + + private: + __m128i data_; +}; + +inline ABSL_TARGET_CRYPTO Vector128 Vector128Load(const void* from) { + return Vector128(_mm_load_si128(reinterpret_cast<const __m128i*>(from))); +} + +inline ABSL_TARGET_CRYPTO void Vector128Store(const Vector128& v, void* to) { + _mm_store_si128(reinterpret_cast<__m128i*>(to), v.data()); +} + +// One round of AES. "round_key" is a public constant for breaking the +// symmetry of AES (ensures previously equal columns differ afterwards). +inline ABSL_TARGET_CRYPTO Vector128 AesRound(const Vector128& state, + const Vector128& round_key) { + // It is important to always use the full round function - omitting the + // final MixColumns reduces security [https://eprint.iacr.org/2010/041.pdf] + // and does not help because we never decrypt. + return Vector128(_mm_aesenc_si128(state.data(), round_key.data())); +} + +inline ABSL_TARGET_CRYPTO void SwapEndian(void*) {} + +} // namespace + +#endif + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunknown-pragmas" +#endif + +// At this point, all of the platform-specific features have been defined / +// implemented. +// +// REQUIRES: using Vector128 = ... +// REQUIRES: Vector128 Vector128Load(void*) {...} +// REQUIRES: void Vector128Store(Vector128, void*) {...} +// REQUIRES: Vector128 AesRound(Vector128, Vector128) {...} +// REQUIRES: void SwapEndian(uint64_t*) {...} +// +// PROVIDES: absl::random_internal::RandenHwAes::Absorb +// PROVIDES: absl::random_internal::RandenHwAes::Generate +namespace { + +// Block shuffles applies a shuffle to the entire state between AES rounds. +// Improved odd-even shuffle from "New criterion for diffusion property". +inline ABSL_TARGET_CRYPTO void BlockShuffle(u64x2* state) { + static_assert(RandenTraits::kFeistelBlocks == 16, + "Expecting 16 FeistelBlocks."); + + constexpr size_t shuffle[RandenTraits::kFeistelBlocks] = { + 7, 2, 13, 4, 11, 8, 3, 6, 15, 0, 9, 10, 1, 14, 5, 12}; + + const Vector128 v0 = Vector128Load(state + shuffle[0]); + const Vector128 v1 = Vector128Load(state + shuffle[1]); + const Vector128 v2 = Vector128Load(state + shuffle[2]); + const Vector128 v3 = Vector128Load(state + shuffle[3]); + const Vector128 v4 = Vector128Load(state + shuffle[4]); + const Vector128 v5 = Vector128Load(state + shuffle[5]); + const Vector128 v6 = Vector128Load(state + shuffle[6]); + const Vector128 v7 = Vector128Load(state + shuffle[7]); + const Vector128 w0 = Vector128Load(state + shuffle[8]); + const Vector128 w1 = Vector128Load(state + shuffle[9]); + const Vector128 w2 = Vector128Load(state + shuffle[10]); + const Vector128 w3 = Vector128Load(state + shuffle[11]); + const Vector128 w4 = Vector128Load(state + shuffle[12]); + const Vector128 w5 = Vector128Load(state + shuffle[13]); + const Vector128 w6 = Vector128Load(state + shuffle[14]); + const Vector128 w7 = Vector128Load(state + shuffle[15]); + + Vector128Store(v0, state + 0); + Vector128Store(v1, state + 1); + Vector128Store(v2, state + 2); + Vector128Store(v3, state + 3); + Vector128Store(v4, state + 4); + Vector128Store(v5, state + 5); + Vector128Store(v6, state + 6); + Vector128Store(v7, state + 7); + Vector128Store(w0, state + 8); + Vector128Store(w1, state + 9); + Vector128Store(w2, state + 10); + Vector128Store(w3, state + 11); + Vector128Store(w4, state + 12); + Vector128Store(w5, state + 13); + Vector128Store(w6, state + 14); + Vector128Store(w7, state + 15); +} + +// Feistel round function using two AES subrounds. Very similar to F() +// from Simpira v2, but with independent subround keys. Uses 17 AES rounds +// per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in +// parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel +// XORs are 'free' (included in the second AES instruction). +inline ABSL_TARGET_CRYPTO const u64x2* FeistelRound( + u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { + static_assert(RandenTraits::kFeistelBlocks == 16, + "Expecting 16 FeistelBlocks."); + + // MSVC does a horrible job at unrolling loops. + // So we unroll the loop by hand to improve the performance. + const Vector128 s0 = Vector128Load(state + 0); + const Vector128 s1 = Vector128Load(state + 1); + const Vector128 s2 = Vector128Load(state + 2); + const Vector128 s3 = Vector128Load(state + 3); + const Vector128 s4 = Vector128Load(state + 4); + const Vector128 s5 = Vector128Load(state + 5); + const Vector128 s6 = Vector128Load(state + 6); + const Vector128 s7 = Vector128Load(state + 7); + const Vector128 s8 = Vector128Load(state + 8); + const Vector128 s9 = Vector128Load(state + 9); + const Vector128 s10 = Vector128Load(state + 10); + const Vector128 s11 = Vector128Load(state + 11); + const Vector128 s12 = Vector128Load(state + 12); + const Vector128 s13 = Vector128Load(state + 13); + const Vector128 s14 = Vector128Load(state + 14); + const Vector128 s15 = Vector128Load(state + 15); + + // Encode even blocks with keys. + const Vector128 e0 = AesRound(s0, Vector128Load(keys + 0)); + const Vector128 e2 = AesRound(s2, Vector128Load(keys + 1)); + const Vector128 e4 = AesRound(s4, Vector128Load(keys + 2)); + const Vector128 e6 = AesRound(s6, Vector128Load(keys + 3)); + const Vector128 e8 = AesRound(s8, Vector128Load(keys + 4)); + const Vector128 e10 = AesRound(s10, Vector128Load(keys + 5)); + const Vector128 e12 = AesRound(s12, Vector128Load(keys + 6)); + const Vector128 e14 = AesRound(s14, Vector128Load(keys + 7)); + + // Encode odd blocks with even output from above. + const Vector128 o1 = AesRound(e0, s1); + const Vector128 o3 = AesRound(e2, s3); + const Vector128 o5 = AesRound(e4, s5); + const Vector128 o7 = AesRound(e6, s7); + const Vector128 o9 = AesRound(e8, s9); + const Vector128 o11 = AesRound(e10, s11); + const Vector128 o13 = AesRound(e12, s13); + const Vector128 o15 = AesRound(e14, s15); + + // Store odd blocks. (These will be shuffled later). + Vector128Store(o1, state + 1); + Vector128Store(o3, state + 3); + Vector128Store(o5, state + 5); + Vector128Store(o7, state + 7); + Vector128Store(o9, state + 9); + Vector128Store(o11, state + 11); + Vector128Store(o13, state + 13); + Vector128Store(o15, state + 15); + + return keys + 8; +} + +// Cryptographic permutation based via type-2 Generalized Feistel Network. +// Indistinguishable from ideal by chosen-ciphertext adversaries using less than +// 2^64 queries if the round function is a PRF. This is similar to the b=8 case +// of Simpira v2, but more efficient than its generic construction for b=16. +inline ABSL_TARGET_CRYPTO void Permute( + u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { + // (Successfully unrolled; the first iteration jumps into the second half) +#ifdef __clang__ +#pragma clang loop unroll_count(2) +#endif + for (size_t round = 0; round < RandenTraits::kFeistelRounds; ++round) { + keys = FeistelRound(state, keys); + BlockShuffle(state); + } +} + +} // namespace + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +bool HasRandenHwAesImplementation() { return true; } + +const void* ABSL_TARGET_CRYPTO RandenHwAes::GetKeys() { + // Round keys for one AES per Feistel round and branch. + // The canonical implementation uses first digits of Pi. +#if defined(ABSL_ARCH_PPC) + return kRandenRoundKeysBE; +#else + return kRandenRoundKeys; +#endif +} + +// NOLINTNEXTLINE +void ABSL_TARGET_CRYPTO RandenHwAes::Absorb(const void* seed_void, + void* state_void) { + static_assert(RandenTraits::kCapacityBytes / sizeof(Vector128) == 1, + "Unexpected Randen kCapacityBlocks"); + static_assert(RandenTraits::kStateBytes / sizeof(Vector128) == 16, + "Unexpected Randen kStateBlocks"); + + auto* state = + reinterpret_cast<u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(state_void); + const auto* seed = + reinterpret_cast<const u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(seed_void); + + Vector128 b1 = Vector128Load(state + 1); + b1 ^= Vector128Load(seed + 0); + Vector128Store(b1, state + 1); + + Vector128 b2 = Vector128Load(state + 2); + b2 ^= Vector128Load(seed + 1); + Vector128Store(b2, state + 2); + + Vector128 b3 = Vector128Load(state + 3); + b3 ^= Vector128Load(seed + 2); + Vector128Store(b3, state + 3); + + Vector128 b4 = Vector128Load(state + 4); + b4 ^= Vector128Load(seed + 3); + Vector128Store(b4, state + 4); + + Vector128 b5 = Vector128Load(state + 5); + b5 ^= Vector128Load(seed + 4); + Vector128Store(b5, state + 5); + + Vector128 b6 = Vector128Load(state + 6); + b6 ^= Vector128Load(seed + 5); + Vector128Store(b6, state + 6); + + Vector128 b7 = Vector128Load(state + 7); + b7 ^= Vector128Load(seed + 6); + Vector128Store(b7, state + 7); + + Vector128 b8 = Vector128Load(state + 8); + b8 ^= Vector128Load(seed + 7); + Vector128Store(b8, state + 8); + + Vector128 b9 = Vector128Load(state + 9); + b9 ^= Vector128Load(seed + 8); + Vector128Store(b9, state + 9); + + Vector128 b10 = Vector128Load(state + 10); + b10 ^= Vector128Load(seed + 9); + Vector128Store(b10, state + 10); + + Vector128 b11 = Vector128Load(state + 11); + b11 ^= Vector128Load(seed + 10); + Vector128Store(b11, state + 11); + + Vector128 b12 = Vector128Load(state + 12); + b12 ^= Vector128Load(seed + 11); + Vector128Store(b12, state + 12); + + Vector128 b13 = Vector128Load(state + 13); + b13 ^= Vector128Load(seed + 12); + Vector128Store(b13, state + 13); + + Vector128 b14 = Vector128Load(state + 14); + b14 ^= Vector128Load(seed + 13); + Vector128Store(b14, state + 14); + + Vector128 b15 = Vector128Load(state + 15); + b15 ^= Vector128Load(seed + 14); + Vector128Store(b15, state + 15); +} + +// NOLINTNEXTLINE +void ABSL_TARGET_CRYPTO RandenHwAes::Generate(const void* keys_void, + void* state_void) { + static_assert(RandenTraits::kCapacityBytes == sizeof(Vector128), + "Capacity mismatch"); + + auto* state = reinterpret_cast<u64x2*>(state_void); + const auto* keys = reinterpret_cast<const u64x2*>(keys_void); + + const Vector128 prev_inner = Vector128Load(state); + + SwapEndian(state); + + Permute(state, keys); + + SwapEndian(state); + + // Ensure backtracking resistance. + Vector128 inner = Vector128Load(state); + inner ^= prev_inner; + Vector128Store(inner, state); +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // (ABSL_RANDEN_HWAES_IMPL) diff --git a/third_party/abseil_cpp/absl/random/internal/randen_hwaes.h b/third_party/abseil_cpp/absl/random/internal/randen_hwaes.h new file mode 100644 index 000000000000..bce36b522609 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_hwaes.h @@ -0,0 +1,50 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_RANDEN_HWAES_H_ +#define ABSL_RANDOM_INTERNAL_RANDEN_HWAES_H_ + +#include "absl/base/config.h" + +// HERMETIC NOTE: The randen_hwaes target must not introduce duplicate +// symbols from arbitrary system and other headers, since it may be built +// with different flags from other targets, using different levels of +// optimization, potentially introducing ODR violations. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// RANDen = RANDom generator or beetroots in Swiss German. +// 'Strong' (well-distributed, unpredictable, backtracking-resistant) random +// generator, faster in some benchmarks than std::mt19937_64 and pcg64_c32. +// +// RandenHwAes implements the basic state manipulation methods. +class RandenHwAes { + public: + static void Generate(const void* keys, void* state_void); + static void Absorb(const void* seed_void, void* state_void); + static const void* GetKeys(); +}; + +// HasRandenHwAesImplementation returns true when there is an accelerated +// implementation, and false otherwise. If there is no implementation, +// then attempting to use it will abort the program. +bool HasRandenHwAesImplementation(); + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_RANDEN_HWAES_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/randen_hwaes_test.cc b/third_party/abseil_cpp/absl/random/internal/randen_hwaes_test.cc new file mode 100644 index 000000000000..66ddb43fd6a7 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_hwaes_test.cc @@ -0,0 +1,104 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen_hwaes.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/platform.h" +#include "absl/random/internal/randen_detect.h" +#include "absl/random/internal/randen_traits.h" +#include "absl/strings/str_format.h" + +namespace { + +using absl::random_internal::RandenHwAes; +using absl::random_internal::RandenTraits; + +// Local state parameters. +constexpr size_t kSeedBytes = + RandenTraits::kStateBytes - RandenTraits::kCapacityBytes; +constexpr size_t kStateSizeT = RandenTraits::kStateBytes / sizeof(uint64_t); +constexpr size_t kSeedSizeT = kSeedBytes / sizeof(uint32_t); + +struct alignas(16) randen { + uint64_t state[kStateSizeT]; + uint32_t seed[kSeedSizeT]; +}; + +TEST(RandenHwAesTest, Default) { + EXPECT_TRUE(absl::random_internal::CPUSupportsRandenHwAes()); + + constexpr uint64_t kGolden[] = { + 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, + 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, + 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, + 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, + 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, + 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, + 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, + 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, + 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, + 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, + 0x026ff374c101da7e, 0x811ef0821c3de851, + }; + + alignas(16) randen d; + memset(d.state, 0, sizeof(d.state)); + RandenHwAes::Generate(RandenHwAes::GetKeys(), d.state); + + uint64_t* id = d.state; + for (const auto& elem : kGolden) { + auto a = absl::StrFormat("%#x", elem); + auto b = absl::StrFormat("%#x", *id++); + EXPECT_EQ(a, b); + } +} + +} // namespace + +int main(int argc, char* argv[]) { + testing::InitGoogleTest(&argc, argv); + + ABSL_RAW_LOG(INFO, "ABSL_HAVE_ACCELERATED_AES=%d", ABSL_HAVE_ACCELERATED_AES); + ABSL_RAW_LOG(INFO, "ABSL_RANDOM_INTERNAL_AES_DISPATCH=%d", + ABSL_RANDOM_INTERNAL_AES_DISPATCH); + +#if defined(ABSL_ARCH_X86_64) + ABSL_RAW_LOG(INFO, "ABSL_ARCH_X86_64"); +#elif defined(ABSL_ARCH_X86_32) + ABSL_RAW_LOG(INFO, "ABSL_ARCH_X86_32"); +#elif defined(ABSL_ARCH_AARCH64) + ABSL_RAW_LOG(INFO, "ABSL_ARCH_AARCH64"); +#elif defined(ABSL_ARCH_ARM) + ABSL_RAW_LOG(INFO, "ABSL_ARCH_ARM"); +#elif defined(ABSL_ARCH_PPC) + ABSL_RAW_LOG(INFO, "ABSL_ARCH_PPC"); +#else + ABSL_RAW_LOG(INFO, "ARCH Unknown"); +#endif + + int x = absl::random_internal::HasRandenHwAesImplementation(); + ABSL_RAW_LOG(INFO, "HasRandenHwAesImplementation = %d", x); + + int y = absl::random_internal::CPUSupportsRandenHwAes(); + ABSL_RAW_LOG(INFO, "CPUSupportsRandenHwAes = %d", x); + + if (!x || !y) { + ABSL_RAW_LOG(INFO, "Skipping Randen HWAES tests."); + return 0; + } + return RUN_ALL_TESTS(); +} diff --git a/third_party/abseil_cpp/absl/random/internal/randen_round_keys.cc b/third_party/abseil_cpp/absl/random/internal/randen_round_keys.cc new file mode 100644 index 000000000000..5fb3ca556db6 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_round_keys.cc @@ -0,0 +1,462 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen_traits.h" + +// This file contains only the round keys for randen. +// +// "Nothing up my sleeve" numbers from the first hex digits of Pi, obtained +// from http://hexpi.sourceforge.net/. The array was generated by following +// Python script: + +/* +python >tmp.cc << EOF +"""Generates Randen round keys array from pi-hex.62500.txt file.""" +import binascii + +KEYS = 17 * 8 + +def chunks(l, n): + """Yield successive n-sized chunks from l.""" + for i in range(0, len(l), n): + yield l[i:i + n] + +def pairwise(t): + """Transforms sequence into sequence of pairs.""" + it = iter(t) + return zip(it,it) + +def digits_from_pi(): + """Reads digits from hexpi.sourceforge.net file.""" + with open("pi-hex.62500.txt") as file: + return file.read() + +def digits_from_urandom(): + """Reads digits from /dev/urandom.""" + with open("/dev/urandom") as file: + return binascii.hexlify(file.read(KEYS * 16)) + +def print_row(b) + print(" 0x{0}, 0x{1}, 0x{2}, 0x{3}, 0x{4}, 0x{5}, 0x{6}, 0x{7}, 0x{8}, 0x{9}, +0x{10}, 0x{11}, 0x{12}, 0x{13}, 0x{14}, 0x{15},".format(*b)) + + +digits = digits_from_pi() +#digits = digits_from_urandom() + +print("namespace {") +print("static constexpr size_t kKeyBytes = {0};\n".format(KEYS * 16)) +print("}") + +print("alignas(16) const unsigned char kRandenRoundKeysBE[kKeyBytes] = {") + +for i, u16 in zip(range(KEYS), chunks(digits, 32)): + b = list(chunks(u16, 2)) + print_row(b) + +print("};") + +print("alignas(16) const unsigned char kRandenRoundKeys[kKeyBytes] = {") + +for i, u16 in zip(range(KEYS), chunks(digits, 32)): + b = list(chunks(u16, 2)) + b.reverse() + print_row(b) + +print("};") + +EOF + +*/ + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { +static constexpr size_t kKeyBytes = 2176; +} + +alignas(16) const unsigned char kRandenRoundKeysBE[kKeyBytes] = { + 0x24, 0x3F, 0x6A, 0x88, 0x85, 0xA3, 0x08, 0xD3, 0x13, 0x19, 0x8A, 0x2E, + 0x03, 0x70, 0x73, 0x44, 0xA4, 0x09, 0x38, 0x22, 0x29, 0x9F, 0x31, 0xD0, + 0x08, 0x2E, 0xFA, 0x98, 0xEC, 0x4E, 0x6C, 0x89, 0x45, 0x28, 0x21, 0xE6, + 0x38, 0xD0, 0x13, 0x77, 0xBE, 0x54, 0x66, 0xCF, 0x34, 0xE9, 0x0C, 0x6C, + 0xC0, 0xAC, 0x29, 0xB7, 0xC9, 0x7C, 0x50, 0xDD, 0x3F, 0x84, 0xD5, 0xB5, + 0xB5, 0x47, 0x09, 0x17, 0x92, 0x16, 0xD5, 0xD9, 0x89, 0x79, 0xFB, 0x1B, + 0xD1, 0x31, 0x0B, 0xA6, 0x98, 0xDF, 0xB5, 0xAC, 0x2F, 0xFD, 0x72, 0xDB, + 0xD0, 0x1A, 0xDF, 0xB7, 0xB8, 0xE1, 0xAF, 0xED, 0x6A, 0x26, 0x7E, 0x96, + 0xBA, 0x7C, 0x90, 0x45, 0xF1, 0x2C, 0x7F, 0x99, 0x24, 0xA1, 0x99, 0x47, + 0xB3, 0x91, 0x6C, 0xF7, 0x08, 0x01, 0xF2, 0xE2, 0x85, 0x8E, 0xFC, 0x16, + 0x63, 0x69, 0x20, 0xD8, 0x71, 0x57, 0x4E, 0x69, 0xA4, 0x58, 0xFE, 0xA3, + 0xF4, 0x93, 0x3D, 0x7E, 0x0D, 0x95, 0x74, 0x8F, 0x72, 0x8E, 0xB6, 0x58, + 0x71, 0x8B, 0xCD, 0x58, 0x82, 0x15, 0x4A, 0xEE, 0x7B, 0x54, 0xA4, 0x1D, + 0xC2, 0x5A, 0x59, 0xB5, 0x9C, 0x30, 0xD5, 0x39, 0x2A, 0xF2, 0x60, 0x13, + 0xC5, 0xD1, 0xB0, 0x23, 0x28, 0x60, 0x85, 0xF0, 0xCA, 0x41, 0x79, 0x18, + 0xB8, 0xDB, 0x38, 0xEF, 0x8E, 0x79, 0xDC, 0xB0, 0x60, 0x3A, 0x18, 0x0E, + 0x6C, 0x9E, 0x0E, 0x8B, 0xB0, 0x1E, 0x8A, 0x3E, 0xD7, 0x15, 0x77, 0xC1, + 0xBD, 0x31, 0x4B, 0x27, 0x78, 0xAF, 0x2F, 0xDA, 0x55, 0x60, 0x5C, 0x60, + 0xE6, 0x55, 0x25, 0xF3, 0xAA, 0x55, 0xAB, 0x94, 0x57, 0x48, 0x98, 0x62, + 0x63, 0xE8, 0x14, 0x40, 0x55, 0xCA, 0x39, 0x6A, 0x2A, 0xAB, 0x10, 0xB6, + 0xB4, 0xCC, 0x5C, 0x34, 0x11, 0x41, 0xE8, 0xCE, 0xA1, 0x54, 0x86, 0xAF, + 0x7C, 0x72, 0xE9, 0x93, 0xB3, 0xEE, 0x14, 0x11, 0x63, 0x6F, 0xBC, 0x2A, + 0x2B, 0xA9, 0xC5, 0x5D, 0x74, 0x18, 0x31, 0xF6, 0xCE, 0x5C, 0x3E, 0x16, + 0x9B, 0x87, 0x93, 0x1E, 0xAF, 0xD6, 0xBA, 0x33, 0x6C, 0x24, 0xCF, 0x5C, + 0x7A, 0x32, 0x53, 0x81, 0x28, 0x95, 0x86, 0x77, 0x3B, 0x8F, 0x48, 0x98, + 0x6B, 0x4B, 0xB9, 0xAF, 0xC4, 0xBF, 0xE8, 0x1B, 0x66, 0x28, 0x21, 0x93, + 0x61, 0xD8, 0x09, 0xCC, 0xFB, 0x21, 0xA9, 0x91, 0x48, 0x7C, 0xAC, 0x60, + 0x5D, 0xEC, 0x80, 0x32, 0xEF, 0x84, 0x5D, 0x5D, 0xE9, 0x85, 0x75, 0xB1, + 0xDC, 0x26, 0x23, 0x02, 0xEB, 0x65, 0x1B, 0x88, 0x23, 0x89, 0x3E, 0x81, + 0xD3, 0x96, 0xAC, 0xC5, 0x0F, 0x6D, 0x6F, 0xF3, 0x83, 0xF4, 0x42, 0x39, + 0x2E, 0x0B, 0x44, 0x82, 0xA4, 0x84, 0x20, 0x04, 0x69, 0xC8, 0xF0, 0x4A, + 0x9E, 0x1F, 0x9B, 0x5E, 0x21, 0xC6, 0x68, 0x42, 0xF6, 0xE9, 0x6C, 0x9A, + 0x67, 0x0C, 0x9C, 0x61, 0xAB, 0xD3, 0x88, 0xF0, 0x6A, 0x51, 0xA0, 0xD2, + 0xD8, 0x54, 0x2F, 0x68, 0x96, 0x0F, 0xA7, 0x28, 0xAB, 0x51, 0x33, 0xA3, + 0x6E, 0xEF, 0x0B, 0x6C, 0x13, 0x7A, 0x3B, 0xE4, 0xBA, 0x3B, 0xF0, 0x50, + 0x7E, 0xFB, 0x2A, 0x98, 0xA1, 0xF1, 0x65, 0x1D, 0x39, 0xAF, 0x01, 0x76, + 0x66, 0xCA, 0x59, 0x3E, 0x82, 0x43, 0x0E, 0x88, 0x8C, 0xEE, 0x86, 0x19, + 0x45, 0x6F, 0x9F, 0xB4, 0x7D, 0x84, 0xA5, 0xC3, 0x3B, 0x8B, 0x5E, 0xBE, + 0xE0, 0x6F, 0x75, 0xD8, 0x85, 0xC1, 0x20, 0x73, 0x40, 0x1A, 0x44, 0x9F, + 0x56, 0xC1, 0x6A, 0xA6, 0x4E, 0xD3, 0xAA, 0x62, 0x36, 0x3F, 0x77, 0x06, + 0x1B, 0xFE, 0xDF, 0x72, 0x42, 0x9B, 0x02, 0x3D, 0x37, 0xD0, 0xD7, 0x24, + 0xD0, 0x0A, 0x12, 0x48, 0xDB, 0x0F, 0xEA, 0xD3, 0x49, 0xF1, 0xC0, 0x9B, + 0x07, 0x53, 0x72, 0xC9, 0x80, 0x99, 0x1B, 0x7B, 0x25, 0xD4, 0x79, 0xD8, + 0xF6, 0xE8, 0xDE, 0xF7, 0xE3, 0xFE, 0x50, 0x1A, 0xB6, 0x79, 0x4C, 0x3B, + 0x97, 0x6C, 0xE0, 0xBD, 0x04, 0xC0, 0x06, 0xBA, 0xC1, 0xA9, 0x4F, 0xB6, + 0x40, 0x9F, 0x60, 0xC4, 0x5E, 0x5C, 0x9E, 0xC2, 0x19, 0x6A, 0x24, 0x63, + 0x68, 0xFB, 0x6F, 0xAF, 0x3E, 0x6C, 0x53, 0xB5, 0x13, 0x39, 0xB2, 0xEB, + 0x3B, 0x52, 0xEC, 0x6F, 0x6D, 0xFC, 0x51, 0x1F, 0x9B, 0x30, 0x95, 0x2C, + 0xCC, 0x81, 0x45, 0x44, 0xAF, 0x5E, 0xBD, 0x09, 0xBE, 0xE3, 0xD0, 0x04, + 0xDE, 0x33, 0x4A, 0xFD, 0x66, 0x0F, 0x28, 0x07, 0x19, 0x2E, 0x4B, 0xB3, + 0xC0, 0xCB, 0xA8, 0x57, 0x45, 0xC8, 0x74, 0x0F, 0xD2, 0x0B, 0x5F, 0x39, + 0xB9, 0xD3, 0xFB, 0xDB, 0x55, 0x79, 0xC0, 0xBD, 0x1A, 0x60, 0x32, 0x0A, + 0xD6, 0xA1, 0x00, 0xC6, 0x40, 0x2C, 0x72, 0x79, 0x67, 0x9F, 0x25, 0xFE, + 0xFB, 0x1F, 0xA3, 0xCC, 0x8E, 0xA5, 0xE9, 0xF8, 0xDB, 0x32, 0x22, 0xF8, + 0x3C, 0x75, 0x16, 0xDF, 0xFD, 0x61, 0x6B, 0x15, 0x2F, 0x50, 0x1E, 0xC8, + 0xAD, 0x05, 0x52, 0xAB, 0x32, 0x3D, 0xB5, 0xFA, 0xFD, 0x23, 0x87, 0x60, + 0x53, 0x31, 0x7B, 0x48, 0x3E, 0x00, 0xDF, 0x82, 0x9E, 0x5C, 0x57, 0xBB, + 0xCA, 0x6F, 0x8C, 0xA0, 0x1A, 0x87, 0x56, 0x2E, 0xDF, 0x17, 0x69, 0xDB, + 0xD5, 0x42, 0xA8, 0xF6, 0x28, 0x7E, 0xFF, 0xC3, 0xAC, 0x67, 0x32, 0xC6, + 0x8C, 0x4F, 0x55, 0x73, 0x69, 0x5B, 0x27, 0xB0, 0xBB, 0xCA, 0x58, 0xC8, + 0xE1, 0xFF, 0xA3, 0x5D, 0xB8, 0xF0, 0x11, 0xA0, 0x10, 0xFA, 0x3D, 0x98, + 0xFD, 0x21, 0x83, 0xB8, 0x4A, 0xFC, 0xB5, 0x6C, 0x2D, 0xD1, 0xD3, 0x5B, + 0x9A, 0x53, 0xE4, 0x79, 0xB6, 0xF8, 0x45, 0x65, 0xD2, 0x8E, 0x49, 0xBC, + 0x4B, 0xFB, 0x97, 0x90, 0xE1, 0xDD, 0xF2, 0xDA, 0xA4, 0xCB, 0x7E, 0x33, + 0x62, 0xFB, 0x13, 0x41, 0xCE, 0xE4, 0xC6, 0xE8, 0xEF, 0x20, 0xCA, 0xDA, + 0x36, 0x77, 0x4C, 0x01, 0xD0, 0x7E, 0x9E, 0xFE, 0x2B, 0xF1, 0x1F, 0xB4, + 0x95, 0xDB, 0xDA, 0x4D, 0xAE, 0x90, 0x91, 0x98, 0xEA, 0xAD, 0x8E, 0x71, + 0x6B, 0x93, 0xD5, 0xA0, 0xD0, 0x8E, 0xD1, 0xD0, 0xAF, 0xC7, 0x25, 0xE0, + 0x8E, 0x3C, 0x5B, 0x2F, 0x8E, 0x75, 0x94, 0xB7, 0x8F, 0xF6, 0xE2, 0xFB, + 0xF2, 0x12, 0x2B, 0x64, 0x88, 0x88, 0xB8, 0x12, 0x90, 0x0D, 0xF0, 0x1C, + 0x4F, 0xAD, 0x5E, 0xA0, 0x68, 0x8F, 0xC3, 0x1C, 0xD1, 0xCF, 0xF1, 0x91, + 0xB3, 0xA8, 0xC1, 0xAD, 0x2F, 0x2F, 0x22, 0x18, 0xBE, 0x0E, 0x17, 0x77, + 0xEA, 0x75, 0x2D, 0xFE, 0x8B, 0x02, 0x1F, 0xA1, 0xE5, 0xA0, 0xCC, 0x0F, + 0xB5, 0x6F, 0x74, 0xE8, 0x18, 0xAC, 0xF3, 0xD6, 0xCE, 0x89, 0xE2, 0x99, + 0xB4, 0xA8, 0x4F, 0xE0, 0xFD, 0x13, 0xE0, 0xB7, 0x7C, 0xC4, 0x3B, 0x81, + 0xD2, 0xAD, 0xA8, 0xD9, 0x16, 0x5F, 0xA2, 0x66, 0x80, 0x95, 0x77, 0x05, + 0x93, 0xCC, 0x73, 0x14, 0x21, 0x1A, 0x14, 0x77, 0xE6, 0xAD, 0x20, 0x65, + 0x77, 0xB5, 0xFA, 0x86, 0xC7, 0x54, 0x42, 0xF5, 0xFB, 0x9D, 0x35, 0xCF, + 0xEB, 0xCD, 0xAF, 0x0C, 0x7B, 0x3E, 0x89, 0xA0, 0xD6, 0x41, 0x1B, 0xD3, + 0xAE, 0x1E, 0x7E, 0x49, 0x00, 0x25, 0x0E, 0x2D, 0x20, 0x71, 0xB3, 0x5E, + 0x22, 0x68, 0x00, 0xBB, 0x57, 0xB8, 0xE0, 0xAF, 0x24, 0x64, 0x36, 0x9B, + 0xF0, 0x09, 0xB9, 0x1E, 0x55, 0x63, 0x91, 0x1D, 0x59, 0xDF, 0xA6, 0xAA, + 0x78, 0xC1, 0x43, 0x89, 0xD9, 0x5A, 0x53, 0x7F, 0x20, 0x7D, 0x5B, 0xA2, + 0x02, 0xE5, 0xB9, 0xC5, 0x83, 0x26, 0x03, 0x76, 0x62, 0x95, 0xCF, 0xA9, + 0x11, 0xC8, 0x19, 0x68, 0x4E, 0x73, 0x4A, 0x41, 0xB3, 0x47, 0x2D, 0xCA, + 0x7B, 0x14, 0xA9, 0x4A, 0x1B, 0x51, 0x00, 0x52, 0x9A, 0x53, 0x29, 0x15, + 0xD6, 0x0F, 0x57, 0x3F, 0xBC, 0x9B, 0xC6, 0xE4, 0x2B, 0x60, 0xA4, 0x76, + 0x81, 0xE6, 0x74, 0x00, 0x08, 0xBA, 0x6F, 0xB5, 0x57, 0x1B, 0xE9, 0x1F, + 0xF2, 0x96, 0xEC, 0x6B, 0x2A, 0x0D, 0xD9, 0x15, 0xB6, 0x63, 0x65, 0x21, + 0xE7, 0xB9, 0xF9, 0xB6, 0xFF, 0x34, 0x05, 0x2E, 0xC5, 0x85, 0x56, 0x64, + 0x53, 0xB0, 0x2D, 0x5D, 0xA9, 0x9F, 0x8F, 0xA1, 0x08, 0xBA, 0x47, 0x99, + 0x6E, 0x85, 0x07, 0x6A, 0x4B, 0x7A, 0x70, 0xE9, 0xB5, 0xB3, 0x29, 0x44, + 0xDB, 0x75, 0x09, 0x2E, 0xC4, 0x19, 0x26, 0x23, 0xAD, 0x6E, 0xA6, 0xB0, + 0x49, 0xA7, 0xDF, 0x7D, 0x9C, 0xEE, 0x60, 0xB8, 0x8F, 0xED, 0xB2, 0x66, + 0xEC, 0xAA, 0x8C, 0x71, 0x69, 0x9A, 0x18, 0xFF, 0x56, 0x64, 0x52, 0x6C, + 0xC2, 0xB1, 0x9E, 0xE1, 0x19, 0x36, 0x02, 0xA5, 0x75, 0x09, 0x4C, 0x29, + 0xA0, 0x59, 0x13, 0x40, 0xE4, 0x18, 0x3A, 0x3E, 0x3F, 0x54, 0x98, 0x9A, + 0x5B, 0x42, 0x9D, 0x65, 0x6B, 0x8F, 0xE4, 0xD6, 0x99, 0xF7, 0x3F, 0xD6, + 0xA1, 0xD2, 0x9C, 0x07, 0xEF, 0xE8, 0x30, 0xF5, 0x4D, 0x2D, 0x38, 0xE6, + 0xF0, 0x25, 0x5D, 0xC1, 0x4C, 0xDD, 0x20, 0x86, 0x84, 0x70, 0xEB, 0x26, + 0x63, 0x82, 0xE9, 0xC6, 0x02, 0x1E, 0xCC, 0x5E, 0x09, 0x68, 0x6B, 0x3F, + 0x3E, 0xBA, 0xEF, 0xC9, 0x3C, 0x97, 0x18, 0x14, 0x6B, 0x6A, 0x70, 0xA1, + 0x68, 0x7F, 0x35, 0x84, 0x52, 0xA0, 0xE2, 0x86, 0xB7, 0x9C, 0x53, 0x05, + 0xAA, 0x50, 0x07, 0x37, 0x3E, 0x07, 0x84, 0x1C, 0x7F, 0xDE, 0xAE, 0x5C, + 0x8E, 0x7D, 0x44, 0xEC, 0x57, 0x16, 0xF2, 0xB8, 0xB0, 0x3A, 0xDA, 0x37, + 0xF0, 0x50, 0x0C, 0x0D, 0xF0, 0x1C, 0x1F, 0x04, 0x02, 0x00, 0xB3, 0xFF, + 0xAE, 0x0C, 0xF5, 0x1A, 0x3C, 0xB5, 0x74, 0xB2, 0x25, 0x83, 0x7A, 0x58, + 0xDC, 0x09, 0x21, 0xBD, 0xD1, 0x91, 0x13, 0xF9, 0x7C, 0xA9, 0x2F, 0xF6, + 0x94, 0x32, 0x47, 0x73, 0x22, 0xF5, 0x47, 0x01, 0x3A, 0xE5, 0xE5, 0x81, + 0x37, 0xC2, 0xDA, 0xDC, 0xC8, 0xB5, 0x76, 0x34, 0x9A, 0xF3, 0xDD, 0xA7, + 0xA9, 0x44, 0x61, 0x46, 0x0F, 0xD0, 0x03, 0x0E, 0xEC, 0xC8, 0xC7, 0x3E, + 0xA4, 0x75, 0x1E, 0x41, 0xE2, 0x38, 0xCD, 0x99, 0x3B, 0xEA, 0x0E, 0x2F, + 0x32, 0x80, 0xBB, 0xA1, 0x18, 0x3E, 0xB3, 0x31, 0x4E, 0x54, 0x8B, 0x38, + 0x4F, 0x6D, 0xB9, 0x08, 0x6F, 0x42, 0x0D, 0x03, 0xF6, 0x0A, 0x04, 0xBF, + 0x2C, 0xB8, 0x12, 0x90, 0x24, 0x97, 0x7C, 0x79, 0x56, 0x79, 0xB0, 0x72, + 0xBC, 0xAF, 0x89, 0xAF, 0xDE, 0x9A, 0x77, 0x1F, 0xD9, 0x93, 0x08, 0x10, + 0xB3, 0x8B, 0xAE, 0x12, 0xDC, 0xCF, 0x3F, 0x2E, 0x55, 0x12, 0x72, 0x1F, + 0x2E, 0x6B, 0x71, 0x24, 0x50, 0x1A, 0xDD, 0xE6, 0x9F, 0x84, 0xCD, 0x87, + 0x7A, 0x58, 0x47, 0x18, 0x74, 0x08, 0xDA, 0x17, 0xBC, 0x9F, 0x9A, 0xBC, + 0xE9, 0x4B, 0x7D, 0x8C, 0xEC, 0x7A, 0xEC, 0x3A, 0xDB, 0x85, 0x1D, 0xFA, + 0x63, 0x09, 0x43, 0x66, 0xC4, 0x64, 0xC3, 0xD2, 0xEF, 0x1C, 0x18, 0x47, + 0x32, 0x15, 0xD8, 0x08, 0xDD, 0x43, 0x3B, 0x37, 0x24, 0xC2, 0xBA, 0x16, + 0x12, 0xA1, 0x4D, 0x43, 0x2A, 0x65, 0xC4, 0x51, 0x50, 0x94, 0x00, 0x02, + 0x13, 0x3A, 0xE4, 0xDD, 0x71, 0xDF, 0xF8, 0x9E, 0x10, 0x31, 0x4E, 0x55, + 0x81, 0xAC, 0x77, 0xD6, 0x5F, 0x11, 0x19, 0x9B, 0x04, 0x35, 0x56, 0xF1, + 0xD7, 0xA3, 0xC7, 0x6B, 0x3C, 0x11, 0x18, 0x3B, 0x59, 0x24, 0xA5, 0x09, + 0xF2, 0x8F, 0xE6, 0xED, 0x97, 0xF1, 0xFB, 0xFA, 0x9E, 0xBA, 0xBF, 0x2C, + 0x1E, 0x15, 0x3C, 0x6E, 0x86, 0xE3, 0x45, 0x70, 0xEA, 0xE9, 0x6F, 0xB1, + 0x86, 0x0E, 0x5E, 0x0A, 0x5A, 0x3E, 0x2A, 0xB3, 0x77, 0x1F, 0xE7, 0x1C, + 0x4E, 0x3D, 0x06, 0xFA, 0x29, 0x65, 0xDC, 0xB9, 0x99, 0xE7, 0x1D, 0x0F, + 0x80, 0x3E, 0x89, 0xD6, 0x52, 0x66, 0xC8, 0x25, 0x2E, 0x4C, 0xC9, 0x78, + 0x9C, 0x10, 0xB3, 0x6A, 0xC6, 0x15, 0x0E, 0xBA, 0x94, 0xE2, 0xEA, 0x78, + 0xA6, 0xFC, 0x3C, 0x53, 0x1E, 0x0A, 0x2D, 0xF4, 0xF2, 0xF7, 0x4E, 0xA7, + 0x36, 0x1D, 0x2B, 0x3D, 0x19, 0x39, 0x26, 0x0F, 0x19, 0xC2, 0x79, 0x60, + 0x52, 0x23, 0xA7, 0x08, 0xF7, 0x13, 0x12, 0xB6, 0xEB, 0xAD, 0xFE, 0x6E, + 0xEA, 0xC3, 0x1F, 0x66, 0xE3, 0xBC, 0x45, 0x95, 0xA6, 0x7B, 0xC8, 0x83, + 0xB1, 0x7F, 0x37, 0xD1, 0x01, 0x8C, 0xFF, 0x28, 0xC3, 0x32, 0xDD, 0xEF, + 0xBE, 0x6C, 0x5A, 0xA5, 0x65, 0x58, 0x21, 0x85, 0x68, 0xAB, 0x97, 0x02, + 0xEE, 0xCE, 0xA5, 0x0F, 0xDB, 0x2F, 0x95, 0x3B, 0x2A, 0xEF, 0x7D, 0xAD, + 0x5B, 0x6E, 0x2F, 0x84, 0x15, 0x21, 0xB6, 0x28, 0x29, 0x07, 0x61, 0x70, + 0xEC, 0xDD, 0x47, 0x75, 0x61, 0x9F, 0x15, 0x10, 0x13, 0xCC, 0xA8, 0x30, + 0xEB, 0x61, 0xBD, 0x96, 0x03, 0x34, 0xFE, 0x1E, 0xAA, 0x03, 0x63, 0xCF, + 0xB5, 0x73, 0x5C, 0x90, 0x4C, 0x70, 0xA2, 0x39, 0xD5, 0x9E, 0x9E, 0x0B, + 0xCB, 0xAA, 0xDE, 0x14, 0xEE, 0xCC, 0x86, 0xBC, 0x60, 0x62, 0x2C, 0xA7, + 0x9C, 0xAB, 0x5C, 0xAB, 0xB2, 0xF3, 0x84, 0x6E, 0x64, 0x8B, 0x1E, 0xAF, + 0x19, 0xBD, 0xF0, 0xCA, 0xA0, 0x23, 0x69, 0xB9, 0x65, 0x5A, 0xBB, 0x50, + 0x40, 0x68, 0x5A, 0x32, 0x3C, 0x2A, 0xB4, 0xB3, 0x31, 0x9E, 0xE9, 0xD5, + 0xC0, 0x21, 0xB8, 0xF7, 0x9B, 0x54, 0x0B, 0x19, 0x87, 0x5F, 0xA0, 0x99, + 0x95, 0xF7, 0x99, 0x7E, 0x62, 0x3D, 0x7D, 0xA8, 0xF8, 0x37, 0x88, 0x9A, + 0x97, 0xE3, 0x2D, 0x77, 0x11, 0xED, 0x93, 0x5F, 0x16, 0x68, 0x12, 0x81, + 0x0E, 0x35, 0x88, 0x29, 0xC7, 0xE6, 0x1F, 0xD6, 0x96, 0xDE, 0xDF, 0xA1, + 0x78, 0x58, 0xBA, 0x99, 0x57, 0xF5, 0x84, 0xA5, 0x1B, 0x22, 0x72, 0x63, + 0x9B, 0x83, 0xC3, 0xFF, 0x1A, 0xC2, 0x46, 0x96, 0xCD, 0xB3, 0x0A, 0xEB, + 0x53, 0x2E, 0x30, 0x54, 0x8F, 0xD9, 0x48, 0xE4, 0x6D, 0xBC, 0x31, 0x28, + 0x58, 0xEB, 0xF2, 0xEF, 0x34, 0xC6, 0xFF, 0xEA, 0xFE, 0x28, 0xED, 0x61, + 0xEE, 0x7C, 0x3C, 0x73, 0x5D, 0x4A, 0x14, 0xD9, 0xE8, 0x64, 0xB7, 0xE3, + 0x42, 0x10, 0x5D, 0x14, 0x20, 0x3E, 0x13, 0xE0, 0x45, 0xEE, 0xE2, 0xB6, + 0xA3, 0xAA, 0xAB, 0xEA, 0xDB, 0x6C, 0x4F, 0x15, 0xFA, 0xCB, 0x4F, 0xD0, + 0xC7, 0x42, 0xF4, 0x42, 0xEF, 0x6A, 0xBB, 0xB5, 0x65, 0x4F, 0x3B, 0x1D, + 0x41, 0xCD, 0x21, 0x05, 0xD8, 0x1E, 0x79, 0x9E, 0x86, 0x85, 0x4D, 0xC7, + 0xE4, 0x4B, 0x47, 0x6A, 0x3D, 0x81, 0x62, 0x50, 0xCF, 0x62, 0xA1, 0xF2, + 0x5B, 0x8D, 0x26, 0x46, 0xFC, 0x88, 0x83, 0xA0, 0xC1, 0xC7, 0xB6, 0xA3, + 0x7F, 0x15, 0x24, 0xC3, 0x69, 0xCB, 0x74, 0x92, 0x47, 0x84, 0x8A, 0x0B, + 0x56, 0x92, 0xB2, 0x85, 0x09, 0x5B, 0xBF, 0x00, 0xAD, 0x19, 0x48, 0x9D, + 0x14, 0x62, 0xB1, 0x74, 0x23, 0x82, 0x0D, 0x00, 0x58, 0x42, 0x8D, 0x2A, + 0x0C, 0x55, 0xF5, 0xEA, 0x1D, 0xAD, 0xF4, 0x3E, 0x23, 0x3F, 0x70, 0x61, + 0x33, 0x72, 0xF0, 0x92, 0x8D, 0x93, 0x7E, 0x41, 0xD6, 0x5F, 0xEC, 0xF1, + 0x6C, 0x22, 0x3B, 0xDB, 0x7C, 0xDE, 0x37, 0x59, 0xCB, 0xEE, 0x74, 0x60, + 0x40, 0x85, 0xF2, 0xA7, 0xCE, 0x77, 0x32, 0x6E, 0xA6, 0x07, 0x80, 0x84, + 0x19, 0xF8, 0x50, 0x9E, 0xE8, 0xEF, 0xD8, 0x55, 0x61, 0xD9, 0x97, 0x35, + 0xA9, 0x69, 0xA7, 0xAA, 0xC5, 0x0C, 0x06, 0xC2, 0x5A, 0x04, 0xAB, 0xFC, + 0x80, 0x0B, 0xCA, 0xDC, 0x9E, 0x44, 0x7A, 0x2E, 0xC3, 0x45, 0x34, 0x84, + 0xFD, 0xD5, 0x67, 0x05, 0x0E, 0x1E, 0x9E, 0xC9, 0xDB, 0x73, 0xDB, 0xD3, + 0x10, 0x55, 0x88, 0xCD, 0x67, 0x5F, 0xDA, 0x79, 0xE3, 0x67, 0x43, 0x40, + 0xC5, 0xC4, 0x34, 0x65, 0x71, 0x3E, 0x38, 0xD8, 0x3D, 0x28, 0xF8, 0x9E, + 0xF1, 0x6D, 0xFF, 0x20, 0x15, 0x3E, 0x21, 0xE7, 0x8F, 0xB0, 0x3D, 0x4A, + 0xE6, 0xE3, 0x9F, 0x2B, 0xDB, 0x83, 0xAD, 0xF7, 0xE9, 0x3D, 0x5A, 0x68, + 0x94, 0x81, 0x40, 0xF7, 0xF6, 0x4C, 0x26, 0x1C, 0x94, 0x69, 0x29, 0x34, + 0x41, 0x15, 0x20, 0xF7, 0x76, 0x02, 0xD4, 0xF7, 0xBC, 0xF4, 0x6B, 0x2E, + 0xD4, 0xA1, 0x00, 0x68, 0xD4, 0x08, 0x24, 0x71, 0x33, 0x20, 0xF4, 0x6A, + 0x43, 0xB7, 0xD4, 0xB7, 0x50, 0x00, 0x61, 0xAF, 0x1E, 0x39, 0xF6, 0x2E, + 0x97, 0x24, 0x45, 0x46, +}; + +alignas(16) const unsigned char kRandenRoundKeys[kKeyBytes] = { + 0x44, 0x73, 0x70, 0x03, 0x2E, 0x8A, 0x19, 0x13, 0xD3, 0x08, 0xA3, 0x85, + 0x88, 0x6A, 0x3F, 0x24, 0x89, 0x6C, 0x4E, 0xEC, 0x98, 0xFA, 0x2E, 0x08, + 0xD0, 0x31, 0x9F, 0x29, 0x22, 0x38, 0x09, 0xA4, 0x6C, 0x0C, 0xE9, 0x34, + 0xCF, 0x66, 0x54, 0xBE, 0x77, 0x13, 0xD0, 0x38, 0xE6, 0x21, 0x28, 0x45, + 0x17, 0x09, 0x47, 0xB5, 0xB5, 0xD5, 0x84, 0x3F, 0xDD, 0x50, 0x7C, 0xC9, + 0xB7, 0x29, 0xAC, 0xC0, 0xAC, 0xB5, 0xDF, 0x98, 0xA6, 0x0B, 0x31, 0xD1, + 0x1B, 0xFB, 0x79, 0x89, 0xD9, 0xD5, 0x16, 0x92, 0x96, 0x7E, 0x26, 0x6A, + 0xED, 0xAF, 0xE1, 0xB8, 0xB7, 0xDF, 0x1A, 0xD0, 0xDB, 0x72, 0xFD, 0x2F, + 0xF7, 0x6C, 0x91, 0xB3, 0x47, 0x99, 0xA1, 0x24, 0x99, 0x7F, 0x2C, 0xF1, + 0x45, 0x90, 0x7C, 0xBA, 0x69, 0x4E, 0x57, 0x71, 0xD8, 0x20, 0x69, 0x63, + 0x16, 0xFC, 0x8E, 0x85, 0xE2, 0xF2, 0x01, 0x08, 0x58, 0xB6, 0x8E, 0x72, + 0x8F, 0x74, 0x95, 0x0D, 0x7E, 0x3D, 0x93, 0xF4, 0xA3, 0xFE, 0x58, 0xA4, + 0xB5, 0x59, 0x5A, 0xC2, 0x1D, 0xA4, 0x54, 0x7B, 0xEE, 0x4A, 0x15, 0x82, + 0x58, 0xCD, 0x8B, 0x71, 0xF0, 0x85, 0x60, 0x28, 0x23, 0xB0, 0xD1, 0xC5, + 0x13, 0x60, 0xF2, 0x2A, 0x39, 0xD5, 0x30, 0x9C, 0x0E, 0x18, 0x3A, 0x60, + 0xB0, 0xDC, 0x79, 0x8E, 0xEF, 0x38, 0xDB, 0xB8, 0x18, 0x79, 0x41, 0xCA, + 0x27, 0x4B, 0x31, 0xBD, 0xC1, 0x77, 0x15, 0xD7, 0x3E, 0x8A, 0x1E, 0xB0, + 0x8B, 0x0E, 0x9E, 0x6C, 0x94, 0xAB, 0x55, 0xAA, 0xF3, 0x25, 0x55, 0xE6, + 0x60, 0x5C, 0x60, 0x55, 0xDA, 0x2F, 0xAF, 0x78, 0xB6, 0x10, 0xAB, 0x2A, + 0x6A, 0x39, 0xCA, 0x55, 0x40, 0x14, 0xE8, 0x63, 0x62, 0x98, 0x48, 0x57, + 0x93, 0xE9, 0x72, 0x7C, 0xAF, 0x86, 0x54, 0xA1, 0xCE, 0xE8, 0x41, 0x11, + 0x34, 0x5C, 0xCC, 0xB4, 0xF6, 0x31, 0x18, 0x74, 0x5D, 0xC5, 0xA9, 0x2B, + 0x2A, 0xBC, 0x6F, 0x63, 0x11, 0x14, 0xEE, 0xB3, 0x5C, 0xCF, 0x24, 0x6C, + 0x33, 0xBA, 0xD6, 0xAF, 0x1E, 0x93, 0x87, 0x9B, 0x16, 0x3E, 0x5C, 0xCE, + 0xAF, 0xB9, 0x4B, 0x6B, 0x98, 0x48, 0x8F, 0x3B, 0x77, 0x86, 0x95, 0x28, + 0x81, 0x53, 0x32, 0x7A, 0x91, 0xA9, 0x21, 0xFB, 0xCC, 0x09, 0xD8, 0x61, + 0x93, 0x21, 0x28, 0x66, 0x1B, 0xE8, 0xBF, 0xC4, 0xB1, 0x75, 0x85, 0xE9, + 0x5D, 0x5D, 0x84, 0xEF, 0x32, 0x80, 0xEC, 0x5D, 0x60, 0xAC, 0x7C, 0x48, + 0xC5, 0xAC, 0x96, 0xD3, 0x81, 0x3E, 0x89, 0x23, 0x88, 0x1B, 0x65, 0xEB, + 0x02, 0x23, 0x26, 0xDC, 0x04, 0x20, 0x84, 0xA4, 0x82, 0x44, 0x0B, 0x2E, + 0x39, 0x42, 0xF4, 0x83, 0xF3, 0x6F, 0x6D, 0x0F, 0x9A, 0x6C, 0xE9, 0xF6, + 0x42, 0x68, 0xC6, 0x21, 0x5E, 0x9B, 0x1F, 0x9E, 0x4A, 0xF0, 0xC8, 0x69, + 0x68, 0x2F, 0x54, 0xD8, 0xD2, 0xA0, 0x51, 0x6A, 0xF0, 0x88, 0xD3, 0xAB, + 0x61, 0x9C, 0x0C, 0x67, 0xE4, 0x3B, 0x7A, 0x13, 0x6C, 0x0B, 0xEF, 0x6E, + 0xA3, 0x33, 0x51, 0xAB, 0x28, 0xA7, 0x0F, 0x96, 0x76, 0x01, 0xAF, 0x39, + 0x1D, 0x65, 0xF1, 0xA1, 0x98, 0x2A, 0xFB, 0x7E, 0x50, 0xF0, 0x3B, 0xBA, + 0xB4, 0x9F, 0x6F, 0x45, 0x19, 0x86, 0xEE, 0x8C, 0x88, 0x0E, 0x43, 0x82, + 0x3E, 0x59, 0xCA, 0x66, 0x73, 0x20, 0xC1, 0x85, 0xD8, 0x75, 0x6F, 0xE0, + 0xBE, 0x5E, 0x8B, 0x3B, 0xC3, 0xA5, 0x84, 0x7D, 0x06, 0x77, 0x3F, 0x36, + 0x62, 0xAA, 0xD3, 0x4E, 0xA6, 0x6A, 0xC1, 0x56, 0x9F, 0x44, 0x1A, 0x40, + 0x48, 0x12, 0x0A, 0xD0, 0x24, 0xD7, 0xD0, 0x37, 0x3D, 0x02, 0x9B, 0x42, + 0x72, 0xDF, 0xFE, 0x1B, 0x7B, 0x1B, 0x99, 0x80, 0xC9, 0x72, 0x53, 0x07, + 0x9B, 0xC0, 0xF1, 0x49, 0xD3, 0xEA, 0x0F, 0xDB, 0x3B, 0x4C, 0x79, 0xB6, + 0x1A, 0x50, 0xFE, 0xE3, 0xF7, 0xDE, 0xE8, 0xF6, 0xD8, 0x79, 0xD4, 0x25, + 0xC4, 0x60, 0x9F, 0x40, 0xB6, 0x4F, 0xA9, 0xC1, 0xBA, 0x06, 0xC0, 0x04, + 0xBD, 0xE0, 0x6C, 0x97, 0xB5, 0x53, 0x6C, 0x3E, 0xAF, 0x6F, 0xFB, 0x68, + 0x63, 0x24, 0x6A, 0x19, 0xC2, 0x9E, 0x5C, 0x5E, 0x2C, 0x95, 0x30, 0x9B, + 0x1F, 0x51, 0xFC, 0x6D, 0x6F, 0xEC, 0x52, 0x3B, 0xEB, 0xB2, 0x39, 0x13, + 0xFD, 0x4A, 0x33, 0xDE, 0x04, 0xD0, 0xE3, 0xBE, 0x09, 0xBD, 0x5E, 0xAF, + 0x44, 0x45, 0x81, 0xCC, 0x0F, 0x74, 0xC8, 0x45, 0x57, 0xA8, 0xCB, 0xC0, + 0xB3, 0x4B, 0x2E, 0x19, 0x07, 0x28, 0x0F, 0x66, 0x0A, 0x32, 0x60, 0x1A, + 0xBD, 0xC0, 0x79, 0x55, 0xDB, 0xFB, 0xD3, 0xB9, 0x39, 0x5F, 0x0B, 0xD2, + 0xCC, 0xA3, 0x1F, 0xFB, 0xFE, 0x25, 0x9F, 0x67, 0x79, 0x72, 0x2C, 0x40, + 0xC6, 0x00, 0xA1, 0xD6, 0x15, 0x6B, 0x61, 0xFD, 0xDF, 0x16, 0x75, 0x3C, + 0xF8, 0x22, 0x32, 0xDB, 0xF8, 0xE9, 0xA5, 0x8E, 0x60, 0x87, 0x23, 0xFD, + 0xFA, 0xB5, 0x3D, 0x32, 0xAB, 0x52, 0x05, 0xAD, 0xC8, 0x1E, 0x50, 0x2F, + 0xA0, 0x8C, 0x6F, 0xCA, 0xBB, 0x57, 0x5C, 0x9E, 0x82, 0xDF, 0x00, 0x3E, + 0x48, 0x7B, 0x31, 0x53, 0xC3, 0xFF, 0x7E, 0x28, 0xF6, 0xA8, 0x42, 0xD5, + 0xDB, 0x69, 0x17, 0xDF, 0x2E, 0x56, 0x87, 0x1A, 0xC8, 0x58, 0xCA, 0xBB, + 0xB0, 0x27, 0x5B, 0x69, 0x73, 0x55, 0x4F, 0x8C, 0xC6, 0x32, 0x67, 0xAC, + 0xB8, 0x83, 0x21, 0xFD, 0x98, 0x3D, 0xFA, 0x10, 0xA0, 0x11, 0xF0, 0xB8, + 0x5D, 0xA3, 0xFF, 0xE1, 0x65, 0x45, 0xF8, 0xB6, 0x79, 0xE4, 0x53, 0x9A, + 0x5B, 0xD3, 0xD1, 0x2D, 0x6C, 0xB5, 0xFC, 0x4A, 0x33, 0x7E, 0xCB, 0xA4, + 0xDA, 0xF2, 0xDD, 0xE1, 0x90, 0x97, 0xFB, 0x4B, 0xBC, 0x49, 0x8E, 0xD2, + 0x01, 0x4C, 0x77, 0x36, 0xDA, 0xCA, 0x20, 0xEF, 0xE8, 0xC6, 0xE4, 0xCE, + 0x41, 0x13, 0xFB, 0x62, 0x98, 0x91, 0x90, 0xAE, 0x4D, 0xDA, 0xDB, 0x95, + 0xB4, 0x1F, 0xF1, 0x2B, 0xFE, 0x9E, 0x7E, 0xD0, 0xE0, 0x25, 0xC7, 0xAF, + 0xD0, 0xD1, 0x8E, 0xD0, 0xA0, 0xD5, 0x93, 0x6B, 0x71, 0x8E, 0xAD, 0xEA, + 0x64, 0x2B, 0x12, 0xF2, 0xFB, 0xE2, 0xF6, 0x8F, 0xB7, 0x94, 0x75, 0x8E, + 0x2F, 0x5B, 0x3C, 0x8E, 0x1C, 0xC3, 0x8F, 0x68, 0xA0, 0x5E, 0xAD, 0x4F, + 0x1C, 0xF0, 0x0D, 0x90, 0x12, 0xB8, 0x88, 0x88, 0x77, 0x17, 0x0E, 0xBE, + 0x18, 0x22, 0x2F, 0x2F, 0xAD, 0xC1, 0xA8, 0xB3, 0x91, 0xF1, 0xCF, 0xD1, + 0xE8, 0x74, 0x6F, 0xB5, 0x0F, 0xCC, 0xA0, 0xE5, 0xA1, 0x1F, 0x02, 0x8B, + 0xFE, 0x2D, 0x75, 0xEA, 0xB7, 0xE0, 0x13, 0xFD, 0xE0, 0x4F, 0xA8, 0xB4, + 0x99, 0xE2, 0x89, 0xCE, 0xD6, 0xF3, 0xAC, 0x18, 0x05, 0x77, 0x95, 0x80, + 0x66, 0xA2, 0x5F, 0x16, 0xD9, 0xA8, 0xAD, 0xD2, 0x81, 0x3B, 0xC4, 0x7C, + 0x86, 0xFA, 0xB5, 0x77, 0x65, 0x20, 0xAD, 0xE6, 0x77, 0x14, 0x1A, 0x21, + 0x14, 0x73, 0xCC, 0x93, 0xA0, 0x89, 0x3E, 0x7B, 0x0C, 0xAF, 0xCD, 0xEB, + 0xCF, 0x35, 0x9D, 0xFB, 0xF5, 0x42, 0x54, 0xC7, 0x5E, 0xB3, 0x71, 0x20, + 0x2D, 0x0E, 0x25, 0x00, 0x49, 0x7E, 0x1E, 0xAE, 0xD3, 0x1B, 0x41, 0xD6, + 0x1E, 0xB9, 0x09, 0xF0, 0x9B, 0x36, 0x64, 0x24, 0xAF, 0xE0, 0xB8, 0x57, + 0xBB, 0x00, 0x68, 0x22, 0x7F, 0x53, 0x5A, 0xD9, 0x89, 0x43, 0xC1, 0x78, + 0xAA, 0xA6, 0xDF, 0x59, 0x1D, 0x91, 0x63, 0x55, 0xA9, 0xCF, 0x95, 0x62, + 0x76, 0x03, 0x26, 0x83, 0xC5, 0xB9, 0xE5, 0x02, 0xA2, 0x5B, 0x7D, 0x20, + 0x4A, 0xA9, 0x14, 0x7B, 0xCA, 0x2D, 0x47, 0xB3, 0x41, 0x4A, 0x73, 0x4E, + 0x68, 0x19, 0xC8, 0x11, 0xE4, 0xC6, 0x9B, 0xBC, 0x3F, 0x57, 0x0F, 0xD6, + 0x15, 0x29, 0x53, 0x9A, 0x52, 0x00, 0x51, 0x1B, 0x1F, 0xE9, 0x1B, 0x57, + 0xB5, 0x6F, 0xBA, 0x08, 0x00, 0x74, 0xE6, 0x81, 0x76, 0xA4, 0x60, 0x2B, + 0xB6, 0xF9, 0xB9, 0xE7, 0x21, 0x65, 0x63, 0xB6, 0x15, 0xD9, 0x0D, 0x2A, + 0x6B, 0xEC, 0x96, 0xF2, 0xA1, 0x8F, 0x9F, 0xA9, 0x5D, 0x2D, 0xB0, 0x53, + 0x64, 0x56, 0x85, 0xC5, 0x2E, 0x05, 0x34, 0xFF, 0x44, 0x29, 0xB3, 0xB5, + 0xE9, 0x70, 0x7A, 0x4B, 0x6A, 0x07, 0x85, 0x6E, 0x99, 0x47, 0xBA, 0x08, + 0x7D, 0xDF, 0xA7, 0x49, 0xB0, 0xA6, 0x6E, 0xAD, 0x23, 0x26, 0x19, 0xC4, + 0x2E, 0x09, 0x75, 0xDB, 0xFF, 0x18, 0x9A, 0x69, 0x71, 0x8C, 0xAA, 0xEC, + 0x66, 0xB2, 0xED, 0x8F, 0xB8, 0x60, 0xEE, 0x9C, 0x29, 0x4C, 0x09, 0x75, + 0xA5, 0x02, 0x36, 0x19, 0xE1, 0x9E, 0xB1, 0xC2, 0x6C, 0x52, 0x64, 0x56, + 0x65, 0x9D, 0x42, 0x5B, 0x9A, 0x98, 0x54, 0x3F, 0x3E, 0x3A, 0x18, 0xE4, + 0x40, 0x13, 0x59, 0xA0, 0xF5, 0x30, 0xE8, 0xEF, 0x07, 0x9C, 0xD2, 0xA1, + 0xD6, 0x3F, 0xF7, 0x99, 0xD6, 0xE4, 0x8F, 0x6B, 0x26, 0xEB, 0x70, 0x84, + 0x86, 0x20, 0xDD, 0x4C, 0xC1, 0x5D, 0x25, 0xF0, 0xE6, 0x38, 0x2D, 0x4D, + 0xC9, 0xEF, 0xBA, 0x3E, 0x3F, 0x6B, 0x68, 0x09, 0x5E, 0xCC, 0x1E, 0x02, + 0xC6, 0xE9, 0x82, 0x63, 0x86, 0xE2, 0xA0, 0x52, 0x84, 0x35, 0x7F, 0x68, + 0xA1, 0x70, 0x6A, 0x6B, 0x14, 0x18, 0x97, 0x3C, 0x5C, 0xAE, 0xDE, 0x7F, + 0x1C, 0x84, 0x07, 0x3E, 0x37, 0x07, 0x50, 0xAA, 0x05, 0x53, 0x9C, 0xB7, + 0x0D, 0x0C, 0x50, 0xF0, 0x37, 0xDA, 0x3A, 0xB0, 0xB8, 0xF2, 0x16, 0x57, + 0xEC, 0x44, 0x7D, 0x8E, 0xB2, 0x74, 0xB5, 0x3C, 0x1A, 0xF5, 0x0C, 0xAE, + 0xFF, 0xB3, 0x00, 0x02, 0x04, 0x1F, 0x1C, 0xF0, 0xF6, 0x2F, 0xA9, 0x7C, + 0xF9, 0x13, 0x91, 0xD1, 0xBD, 0x21, 0x09, 0xDC, 0x58, 0x7A, 0x83, 0x25, + 0xDC, 0xDA, 0xC2, 0x37, 0x81, 0xE5, 0xE5, 0x3A, 0x01, 0x47, 0xF5, 0x22, + 0x73, 0x47, 0x32, 0x94, 0x0E, 0x03, 0xD0, 0x0F, 0x46, 0x61, 0x44, 0xA9, + 0xA7, 0xDD, 0xF3, 0x9A, 0x34, 0x76, 0xB5, 0xC8, 0x2F, 0x0E, 0xEA, 0x3B, + 0x99, 0xCD, 0x38, 0xE2, 0x41, 0x1E, 0x75, 0xA4, 0x3E, 0xC7, 0xC8, 0xEC, + 0x08, 0xB9, 0x6D, 0x4F, 0x38, 0x8B, 0x54, 0x4E, 0x31, 0xB3, 0x3E, 0x18, + 0xA1, 0xBB, 0x80, 0x32, 0x79, 0x7C, 0x97, 0x24, 0x90, 0x12, 0xB8, 0x2C, + 0xBF, 0x04, 0x0A, 0xF6, 0x03, 0x0D, 0x42, 0x6F, 0x10, 0x08, 0x93, 0xD9, + 0x1F, 0x77, 0x9A, 0xDE, 0xAF, 0x89, 0xAF, 0xBC, 0x72, 0xB0, 0x79, 0x56, + 0x24, 0x71, 0x6B, 0x2E, 0x1F, 0x72, 0x12, 0x55, 0x2E, 0x3F, 0xCF, 0xDC, + 0x12, 0xAE, 0x8B, 0xB3, 0x17, 0xDA, 0x08, 0x74, 0x18, 0x47, 0x58, 0x7A, + 0x87, 0xCD, 0x84, 0x9F, 0xE6, 0xDD, 0x1A, 0x50, 0xFA, 0x1D, 0x85, 0xDB, + 0x3A, 0xEC, 0x7A, 0xEC, 0x8C, 0x7D, 0x4B, 0xE9, 0xBC, 0x9A, 0x9F, 0xBC, + 0x08, 0xD8, 0x15, 0x32, 0x47, 0x18, 0x1C, 0xEF, 0xD2, 0xC3, 0x64, 0xC4, + 0x66, 0x43, 0x09, 0x63, 0x51, 0xC4, 0x65, 0x2A, 0x43, 0x4D, 0xA1, 0x12, + 0x16, 0xBA, 0xC2, 0x24, 0x37, 0x3B, 0x43, 0xDD, 0x55, 0x4E, 0x31, 0x10, + 0x9E, 0xF8, 0xDF, 0x71, 0xDD, 0xE4, 0x3A, 0x13, 0x02, 0x00, 0x94, 0x50, + 0x6B, 0xC7, 0xA3, 0xD7, 0xF1, 0x56, 0x35, 0x04, 0x9B, 0x19, 0x11, 0x5F, + 0xD6, 0x77, 0xAC, 0x81, 0xFA, 0xFB, 0xF1, 0x97, 0xED, 0xE6, 0x8F, 0xF2, + 0x09, 0xA5, 0x24, 0x59, 0x3B, 0x18, 0x11, 0x3C, 0xB1, 0x6F, 0xE9, 0xEA, + 0x70, 0x45, 0xE3, 0x86, 0x6E, 0x3C, 0x15, 0x1E, 0x2C, 0xBF, 0xBA, 0x9E, + 0xFA, 0x06, 0x3D, 0x4E, 0x1C, 0xE7, 0x1F, 0x77, 0xB3, 0x2A, 0x3E, 0x5A, + 0x0A, 0x5E, 0x0E, 0x86, 0x25, 0xC8, 0x66, 0x52, 0xD6, 0x89, 0x3E, 0x80, + 0x0F, 0x1D, 0xE7, 0x99, 0xB9, 0xDC, 0x65, 0x29, 0x78, 0xEA, 0xE2, 0x94, + 0xBA, 0x0E, 0x15, 0xC6, 0x6A, 0xB3, 0x10, 0x9C, 0x78, 0xC9, 0x4C, 0x2E, + 0x3D, 0x2B, 0x1D, 0x36, 0xA7, 0x4E, 0xF7, 0xF2, 0xF4, 0x2D, 0x0A, 0x1E, + 0x53, 0x3C, 0xFC, 0xA6, 0xB6, 0x12, 0x13, 0xF7, 0x08, 0xA7, 0x23, 0x52, + 0x60, 0x79, 0xC2, 0x19, 0x0F, 0x26, 0x39, 0x19, 0x83, 0xC8, 0x7B, 0xA6, + 0x95, 0x45, 0xBC, 0xE3, 0x66, 0x1F, 0xC3, 0xEA, 0x6E, 0xFE, 0xAD, 0xEB, + 0xA5, 0x5A, 0x6C, 0xBE, 0xEF, 0xDD, 0x32, 0xC3, 0x28, 0xFF, 0x8C, 0x01, + 0xD1, 0x37, 0x7F, 0xB1, 0x3B, 0x95, 0x2F, 0xDB, 0x0F, 0xA5, 0xCE, 0xEE, + 0x02, 0x97, 0xAB, 0x68, 0x85, 0x21, 0x58, 0x65, 0x70, 0x61, 0x07, 0x29, + 0x28, 0xB6, 0x21, 0x15, 0x84, 0x2F, 0x6E, 0x5B, 0xAD, 0x7D, 0xEF, 0x2A, + 0x96, 0xBD, 0x61, 0xEB, 0x30, 0xA8, 0xCC, 0x13, 0x10, 0x15, 0x9F, 0x61, + 0x75, 0x47, 0xDD, 0xEC, 0x39, 0xA2, 0x70, 0x4C, 0x90, 0x5C, 0x73, 0xB5, + 0xCF, 0x63, 0x03, 0xAA, 0x1E, 0xFE, 0x34, 0x03, 0xA7, 0x2C, 0x62, 0x60, + 0xBC, 0x86, 0xCC, 0xEE, 0x14, 0xDE, 0xAA, 0xCB, 0x0B, 0x9E, 0x9E, 0xD5, + 0xCA, 0xF0, 0xBD, 0x19, 0xAF, 0x1E, 0x8B, 0x64, 0x6E, 0x84, 0xF3, 0xB2, + 0xAB, 0x5C, 0xAB, 0x9C, 0xB3, 0xB4, 0x2A, 0x3C, 0x32, 0x5A, 0x68, 0x40, + 0x50, 0xBB, 0x5A, 0x65, 0xB9, 0x69, 0x23, 0xA0, 0x99, 0xA0, 0x5F, 0x87, + 0x19, 0x0B, 0x54, 0x9B, 0xF7, 0xB8, 0x21, 0xC0, 0xD5, 0xE9, 0x9E, 0x31, + 0x77, 0x2D, 0xE3, 0x97, 0x9A, 0x88, 0x37, 0xF8, 0xA8, 0x7D, 0x3D, 0x62, + 0x7E, 0x99, 0xF7, 0x95, 0xD6, 0x1F, 0xE6, 0xC7, 0x29, 0x88, 0x35, 0x0E, + 0x81, 0x12, 0x68, 0x16, 0x5F, 0x93, 0xED, 0x11, 0x63, 0x72, 0x22, 0x1B, + 0xA5, 0x84, 0xF5, 0x57, 0x99, 0xBA, 0x58, 0x78, 0xA1, 0xDF, 0xDE, 0x96, + 0x54, 0x30, 0x2E, 0x53, 0xEB, 0x0A, 0xB3, 0xCD, 0x96, 0x46, 0xC2, 0x1A, + 0xFF, 0xC3, 0x83, 0x9B, 0xEA, 0xFF, 0xC6, 0x34, 0xEF, 0xF2, 0xEB, 0x58, + 0x28, 0x31, 0xBC, 0x6D, 0xE4, 0x48, 0xD9, 0x8F, 0xE3, 0xB7, 0x64, 0xE8, + 0xD9, 0x14, 0x4A, 0x5D, 0x73, 0x3C, 0x7C, 0xEE, 0x61, 0xED, 0x28, 0xFE, + 0xEA, 0xAB, 0xAA, 0xA3, 0xB6, 0xE2, 0xEE, 0x45, 0xE0, 0x13, 0x3E, 0x20, + 0x14, 0x5D, 0x10, 0x42, 0xB5, 0xBB, 0x6A, 0xEF, 0x42, 0xF4, 0x42, 0xC7, + 0xD0, 0x4F, 0xCB, 0xFA, 0x15, 0x4F, 0x6C, 0xDB, 0xC7, 0x4D, 0x85, 0x86, + 0x9E, 0x79, 0x1E, 0xD8, 0x05, 0x21, 0xCD, 0x41, 0x1D, 0x3B, 0x4F, 0x65, + 0x46, 0x26, 0x8D, 0x5B, 0xF2, 0xA1, 0x62, 0xCF, 0x50, 0x62, 0x81, 0x3D, + 0x6A, 0x47, 0x4B, 0xE4, 0x92, 0x74, 0xCB, 0x69, 0xC3, 0x24, 0x15, 0x7F, + 0xA3, 0xB6, 0xC7, 0xC1, 0xA0, 0x83, 0x88, 0xFC, 0x9D, 0x48, 0x19, 0xAD, + 0x00, 0xBF, 0x5B, 0x09, 0x85, 0xB2, 0x92, 0x56, 0x0B, 0x8A, 0x84, 0x47, + 0xEA, 0xF5, 0x55, 0x0C, 0x2A, 0x8D, 0x42, 0x58, 0x00, 0x0D, 0x82, 0x23, + 0x74, 0xB1, 0x62, 0x14, 0x41, 0x7E, 0x93, 0x8D, 0x92, 0xF0, 0x72, 0x33, + 0x61, 0x70, 0x3F, 0x23, 0x3E, 0xF4, 0xAD, 0x1D, 0x60, 0x74, 0xEE, 0xCB, + 0x59, 0x37, 0xDE, 0x7C, 0xDB, 0x3B, 0x22, 0x6C, 0xF1, 0xEC, 0x5F, 0xD6, + 0x9E, 0x50, 0xF8, 0x19, 0x84, 0x80, 0x07, 0xA6, 0x6E, 0x32, 0x77, 0xCE, + 0xA7, 0xF2, 0x85, 0x40, 0xC2, 0x06, 0x0C, 0xC5, 0xAA, 0xA7, 0x69, 0xA9, + 0x35, 0x97, 0xD9, 0x61, 0x55, 0xD8, 0xEF, 0xE8, 0x84, 0x34, 0x45, 0xC3, + 0x2E, 0x7A, 0x44, 0x9E, 0xDC, 0xCA, 0x0B, 0x80, 0xFC, 0xAB, 0x04, 0x5A, + 0xCD, 0x88, 0x55, 0x10, 0xD3, 0xDB, 0x73, 0xDB, 0xC9, 0x9E, 0x1E, 0x0E, + 0x05, 0x67, 0xD5, 0xFD, 0xD8, 0x38, 0x3E, 0x71, 0x65, 0x34, 0xC4, 0xC5, + 0x40, 0x43, 0x67, 0xE3, 0x79, 0xDA, 0x5F, 0x67, 0x4A, 0x3D, 0xB0, 0x8F, + 0xE7, 0x21, 0x3E, 0x15, 0x20, 0xFF, 0x6D, 0xF1, 0x9E, 0xF8, 0x28, 0x3D, + 0xF7, 0x40, 0x81, 0x94, 0x68, 0x5A, 0x3D, 0xE9, 0xF7, 0xAD, 0x83, 0xDB, + 0x2B, 0x9F, 0xE3, 0xE6, 0xF7, 0xD4, 0x02, 0x76, 0xF7, 0x20, 0x15, 0x41, + 0x34, 0x29, 0x69, 0x94, 0x1C, 0x26, 0x4C, 0xF6, 0x6A, 0xF4, 0x20, 0x33, + 0x71, 0x24, 0x08, 0xD4, 0x68, 0x00, 0xA1, 0xD4, 0x2E, 0x6B, 0xF4, 0xBC, + 0x46, 0x45, 0x24, 0x97, 0x2E, 0xF6, 0x39, 0x1E, 0xAF, 0x61, 0x00, 0x50, + 0xB7, 0xD4, 0xB7, 0x43, +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/randen_slow.cc b/third_party/abseil_cpp/absl/random/internal/randen_slow.cc new file mode 100644 index 000000000000..4e5f3dc1c7b7 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_slow.cc @@ -0,0 +1,457 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen_slow.h" + +#include <cstddef> +#include <cstdint> +#include <cstring> + +#include "absl/base/attributes.h" +#include "absl/random/internal/platform.h" +#include "absl/random/internal/randen_traits.h" + +#if ABSL_HAVE_ATTRIBUTE(always_inline) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE \ + __attribute__((always_inline)) +#elif defined(_MSC_VER) +// We can achieve something similar to attribute((always_inline)) with MSVC by +// using the __forceinline keyword, however this is not perfect. MSVC is +// much less aggressive about inlining, and even with the __forceinline keyword. +#define ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE __forceinline +#else +#define ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE +#endif + +namespace { + +// AES portions based on rijndael-alg-fst.c, +// https://fastcrypto.org/front/misc/rijndael-alg-fst.c +// +// Implementation of +// http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf +constexpr uint32_t te0[256] = { + 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, + 0xde6f6fb1, 0x91c5c554, 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, + 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, 0x8fcaca45, 0x1f82829d, + 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, + 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, + 0xe4727296, 0x9bc0c05b, 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, + 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, 0x6834345c, 0x51a5a5f4, + 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, + 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, + 0x0a05050f, 0x2f9a9ab5, 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, + 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, 0x1209091b, 0x1d83839e, + 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, + 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, + 0x5e2f2f71, 0x13848497, 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, + 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, 0xd46a6abe, 0x8dcbcb46, + 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, + 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, + 0x66333355, 0x11858594, 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, + 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, 0xa25151f3, 0x5da3a3fe, + 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, + 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, + 0xfdf3f30e, 0xbfd2d26d, 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, + 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, 0x93c4c457, 0x55a7a7f2, + 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, + 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, + 0x3b9090ab, 0x0b888883, 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, + 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, 0xdbe0e03b, 0x64323256, + 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, + 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, + 0xd3e4e437, 0xf279798b, 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, + 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, 0xd86c6cb4, 0xac5656fa, + 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, + 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, + 0x73b4b4c7, 0x97c6c651, 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, + 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, 0xe0707090, 0x7c3e3e42, + 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, + 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, + 0x3a1d1d27, 0x279e9eb9, 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, + 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, 0x2d9b9bb6, 0x3c1e1e22, + 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, + 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, + 0x844242c6, 0xd06868b8, 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, + 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, +}; + +constexpr uint32_t te1[256] = { + 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, + 0xb1de6f6f, 0x5491c5c5, 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, + 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, 0x458fcaca, 0x9d1f8282, + 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, + 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, + 0x96e47272, 0x5b9bc0c0, 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, + 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, 0x5c683434, 0xf451a5a5, + 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, + 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, + 0x0f0a0505, 0xb52f9a9a, 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, + 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, 0x1b120909, 0x9e1d8383, + 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, + 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, + 0x715e2f2f, 0x97138484, 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, + 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, 0xbed46a6a, 0x468dcbcb, + 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, + 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, + 0x55663333, 0x94118585, 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, + 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, 0xf3a25151, 0xfe5da3a3, + 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, + 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, + 0x0efdf3f3, 0x6dbfd2d2, 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, + 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, 0x5793c4c4, 0xf255a7a7, + 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, + 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, + 0xab3b9090, 0x830b8888, 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, + 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, 0x3bdbe0e0, 0x56643232, + 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, + 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, + 0x37d3e4e4, 0x8bf27979, 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, + 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, 0xb4d86c6c, 0xfaac5656, + 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, + 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, + 0xc773b4b4, 0x5197c6c6, 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, + 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, 0x90e07070, 0x427c3e3e, + 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, + 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, + 0x273a1d1d, 0xb9279e9e, 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, + 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, 0xb62d9b9b, 0x223c1e1e, + 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, + 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, + 0xc6844242, 0xb8d06868, 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, + 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, +}; + +constexpr uint32_t te2[256] = { + 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, + 0x6fb1de6f, 0xc55491c5, 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, + 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, 0xca458fca, 0x829d1f82, + 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, + 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, + 0x7296e472, 0xc05b9bc0, 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, + 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, 0x345c6834, 0xa5f451a5, + 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, + 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, + 0x050f0a05, 0x9ab52f9a, 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, + 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, 0x091b1209, 0x839e1d83, + 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, + 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, + 0x2f715e2f, 0x84971384, 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, + 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, 0x6abed46a, 0xcb468dcb, + 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, + 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, + 0x33556633, 0x85941185, 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, + 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, 0x51f3a251, 0xa3fe5da3, + 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, + 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, + 0xf30efdf3, 0xd26dbfd2, 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, + 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, 0xc45793c4, 0xa7f255a7, + 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, + 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, + 0x90ab3b90, 0x88830b88, 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, + 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, 0xe03bdbe0, 0x32566432, + 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, + 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, + 0xe437d3e4, 0x798bf279, 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, + 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, 0x6cb4d86c, 0x56faac56, + 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, + 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, + 0xb4c773b4, 0xc65197c6, 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, + 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, 0x7090e070, 0x3e427c3e, + 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, + 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, + 0x1d273a1d, 0x9eb9279e, 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, + 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, 0x9bb62d9b, 0x1e223c1e, + 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, + 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, + 0x42c68442, 0x68b8d068, 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, + 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, +}; + +constexpr uint32_t te3[256] = { + 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, + 0x6f6fb1de, 0xc5c55491, 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, + 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, 0xcaca458f, 0x82829d1f, + 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, + 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, + 0x727296e4, 0xc0c05b9b, 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, + 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, 0x34345c68, 0xa5a5f451, + 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, + 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, + 0x05050f0a, 0x9a9ab52f, 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, + 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, 0x09091b12, 0x83839e1d, + 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, + 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, + 0x2f2f715e, 0x84849713, 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, + 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, 0x6a6abed4, 0xcbcb468d, + 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, + 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, + 0x33335566, 0x85859411, 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, + 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, 0x5151f3a2, 0xa3a3fe5d, + 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, + 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, + 0xf3f30efd, 0xd2d26dbf, 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, + 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, 0xc4c45793, 0xa7a7f255, + 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, + 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, + 0x9090ab3b, 0x8888830b, 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, + 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, 0xe0e03bdb, 0x32325664, + 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, + 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, + 0xe4e437d3, 0x79798bf2, 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, + 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, 0x6c6cb4d8, 0x5656faac, + 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, + 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, + 0xb4b4c773, 0xc6c65197, 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, + 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, 0x707090e0, 0x3e3e427c, + 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, + 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, + 0x1d1d273a, 0x9e9eb927, 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, + 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, 0x9b9bb62d, 0x1e1e223c, + 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, + 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, + 0x4242c684, 0x6868b8d0, 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, + 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, +}; + +// Software implementation of the Vector128 class, using uint32_t +// as an underlying vector register. +struct alignas(16) Vector128 { + uint32_t s[4]; +}; + +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE Vector128 +Vector128Load(const void* from) { + Vector128 result; + const uint8_t* src = reinterpret_cast<const uint8_t*>(from); + result.s[0] = static_cast<uint32_t>(src[0]) << 24 | + static_cast<uint32_t>(src[1]) << 16 | + static_cast<uint32_t>(src[2]) << 8 | + static_cast<uint32_t>(src[3]); + result.s[1] = static_cast<uint32_t>(src[4]) << 24 | + static_cast<uint32_t>(src[5]) << 16 | + static_cast<uint32_t>(src[6]) << 8 | + static_cast<uint32_t>(src[7]); + result.s[2] = static_cast<uint32_t>(src[8]) << 24 | + static_cast<uint32_t>(src[9]) << 16 | + static_cast<uint32_t>(src[10]) << 8 | + static_cast<uint32_t>(src[11]); + result.s[3] = static_cast<uint32_t>(src[12]) << 24 | + static_cast<uint32_t>(src[13]) << 16 | + static_cast<uint32_t>(src[14]) << 8 | + static_cast<uint32_t>(src[15]); + return result; +} + +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Vector128Store( + const Vector128& v, void* to) { + uint8_t* dst = reinterpret_cast<uint8_t*>(to); + dst[0] = static_cast<uint8_t>(v.s[0] >> 24); + dst[1] = static_cast<uint8_t>(v.s[0] >> 16); + dst[2] = static_cast<uint8_t>(v.s[0] >> 8); + dst[3] = static_cast<uint8_t>(v.s[0]); + dst[4] = static_cast<uint8_t>(v.s[1] >> 24); + dst[5] = static_cast<uint8_t>(v.s[1] >> 16); + dst[6] = static_cast<uint8_t>(v.s[1] >> 8); + dst[7] = static_cast<uint8_t>(v.s[1]); + dst[8] = static_cast<uint8_t>(v.s[2] >> 24); + dst[9] = static_cast<uint8_t>(v.s[2] >> 16); + dst[10] = static_cast<uint8_t>(v.s[2] >> 8); + dst[11] = static_cast<uint8_t>(v.s[2]); + dst[12] = static_cast<uint8_t>(v.s[3] >> 24); + dst[13] = static_cast<uint8_t>(v.s[3] >> 16); + dst[14] = static_cast<uint8_t>(v.s[3] >> 8); + dst[15] = static_cast<uint8_t>(v.s[3]); +} + +// One round of AES. "round_key" is a public constant for breaking the +// symmetry of AES (ensures previously equal columns differ afterwards). +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE Vector128 +AesRound(const Vector128& state, const Vector128& round_key) { + Vector128 result; + result.s[0] = round_key.s[0] ^ // + te0[uint8_t(state.s[0] >> 24)] ^ // + te1[uint8_t(state.s[1] >> 16)] ^ // + te2[uint8_t(state.s[2] >> 8)] ^ // + te3[uint8_t(state.s[3])]; + result.s[1] = round_key.s[1] ^ // + te0[uint8_t(state.s[1] >> 24)] ^ // + te1[uint8_t(state.s[2] >> 16)] ^ // + te2[uint8_t(state.s[3] >> 8)] ^ // + te3[uint8_t(state.s[0])]; + result.s[2] = round_key.s[2] ^ // + te0[uint8_t(state.s[2] >> 24)] ^ // + te1[uint8_t(state.s[3] >> 16)] ^ // + te2[uint8_t(state.s[0] >> 8)] ^ // + te3[uint8_t(state.s[1])]; + result.s[3] = round_key.s[3] ^ // + te0[uint8_t(state.s[3] >> 24)] ^ // + te1[uint8_t(state.s[0] >> 16)] ^ // + te2[uint8_t(state.s[1] >> 8)] ^ // + te3[uint8_t(state.s[2])]; + return result; +} + +using ::absl::random_internal::RandenTraits; + +// Randen operates on 128-bit vectors. +struct alignas(16) u64x2 { + uint64_t data[2]; +}; + +// The improved Feistel block shuffle function for 16 blocks. +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void BlockShuffle( + u64x2* state) { + static_assert(RandenTraits::kFeistelBlocks == 16, + "Feistel block shuffle only works for 16 blocks."); + + constexpr size_t shuffle[RandenTraits::kFeistelBlocks] = { + 7, 2, 13, 4, 11, 8, 3, 6, 15, 0, 9, 10, 1, 14, 5, 12}; + + // The fully unrolled loop without the memcpy improves the speed by about + // 30% over the equivalent: +#if 0 + u64x2 source[RandenTraits::kFeistelBlocks]; + std::memcpy(source, state, sizeof(source)); + for (size_t i = 0; i < RandenTraits::kFeistelBlocks; i++) { + const u64x2 v0 = source[shuffle[i]]; + state[i] = v0; + } + return; +#endif + + const u64x2 v0 = state[shuffle[0]]; + const u64x2 v1 = state[shuffle[1]]; + const u64x2 v2 = state[shuffle[2]]; + const u64x2 v3 = state[shuffle[3]]; + const u64x2 v4 = state[shuffle[4]]; + const u64x2 v5 = state[shuffle[5]]; + const u64x2 v6 = state[shuffle[6]]; + const u64x2 v7 = state[shuffle[7]]; + const u64x2 w0 = state[shuffle[8]]; + const u64x2 w1 = state[shuffle[9]]; + const u64x2 w2 = state[shuffle[10]]; + const u64x2 w3 = state[shuffle[11]]; + const u64x2 w4 = state[shuffle[12]]; + const u64x2 w5 = state[shuffle[13]]; + const u64x2 w6 = state[shuffle[14]]; + const u64x2 w7 = state[shuffle[15]]; + state[0] = v0; + state[1] = v1; + state[2] = v2; + state[3] = v3; + state[4] = v4; + state[5] = v5; + state[6] = v6; + state[7] = v7; + state[8] = w0; + state[9] = w1; + state[10] = w2; + state[11] = w3; + state[12] = w4; + state[13] = w5; + state[14] = w6; + state[15] = w7; +} + +// Feistel round function using two AES subrounds. Very similar to F() +// from Simpira v2, but with independent subround keys. Uses 17 AES rounds +// per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in +// parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel +// XORs are 'free' (included in the second AES instruction). +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE const u64x2* FeistelRound( + u64x2* ABSL_RANDOM_INTERNAL_RESTRICT state, + const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { + for (size_t branch = 0; branch < RandenTraits::kFeistelBlocks; branch += 4) { + const Vector128 s0 = Vector128Load(state + branch); + const Vector128 s1 = Vector128Load(state + branch + 1); + const Vector128 f0 = AesRound(s0, Vector128Load(keys)); + keys++; + const Vector128 o1 = AesRound(f0, s1); + Vector128Store(o1, state + branch + 1); + + // Manually unroll this loop once. about 10% better than not unrolled. + const Vector128 s2 = Vector128Load(state + branch + 2); + const Vector128 s3 = Vector128Load(state + branch + 3); + const Vector128 f2 = AesRound(s2, Vector128Load(keys)); + keys++; + const Vector128 o3 = AesRound(f2, s3); + Vector128Store(o3, state + branch + 3); + } + return keys; +} + +// Cryptographic permutation based via type-2 Generalized Feistel Network. +// Indistinguishable from ideal by chosen-ciphertext adversaries using less than +// 2^64 queries if the round function is a PRF. This is similar to the b=8 case +// of Simpira v2, but more efficient than its generic construction for b=16. +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Permute( + u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { + for (size_t round = 0; round < RandenTraits::kFeistelRounds; ++round) { + keys = FeistelRound(state, keys); + BlockShuffle(state); + } +} + +} // namespace + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +const void* RandenSlow::GetKeys() { + // Round keys for one AES per Feistel round and branch. + // The canonical implementation uses first digits of Pi. + return kRandenRoundKeys; +} + +void RandenSlow::Absorb(const void* seed_void, void* state_void) { + auto* state = + reinterpret_cast<uint64_t * ABSL_RANDOM_INTERNAL_RESTRICT>(state_void); + const auto* seed = + reinterpret_cast<const uint64_t * ABSL_RANDOM_INTERNAL_RESTRICT>( + seed_void); + + constexpr size_t kCapacityBlocks = + RandenTraits::kCapacityBytes / sizeof(uint64_t); + static_assert( + kCapacityBlocks * sizeof(uint64_t) == RandenTraits::kCapacityBytes, + "Not i*V"); + + for (size_t i = kCapacityBlocks; + i < RandenTraits::kStateBytes / sizeof(uint64_t); ++i) { + state[i] ^= seed[i - kCapacityBlocks]; + } +} + +void RandenSlow::Generate(const void* keys_void, void* state_void) { + static_assert(RandenTraits::kCapacityBytes == sizeof(u64x2), + "Capacity mismatch"); + + auto* state = reinterpret_cast<u64x2*>(state_void); + const auto* keys = reinterpret_cast<const u64x2*>(keys_void); + + const u64x2 prev_inner = state[0]; + + Permute(state, keys); + + // Ensure backtracking resistance. + state[0].data[0] ^= prev_inner.data[0]; + state[0].data[1] ^= prev_inner.data[1]; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/randen_slow.h b/third_party/abseil_cpp/absl/random/internal/randen_slow.h new file mode 100644 index 000000000000..b6f137eb94b1 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_slow.h @@ -0,0 +1,40 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_RANDEN_SLOW_H_ +#define ABSL_RANDOM_INTERNAL_RANDEN_SLOW_H_ + +#include <cstddef> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// RANDen = RANDom generator or beetroots in Swiss German. +// RandenSlow implements the basic state manipulation methods for +// architectures lacking AES hardware acceleration intrinsics. +class RandenSlow { + public: + static void Generate(const void* keys, void* state_void); + static void Absorb(const void* seed_void, void* state_void); + static const void* GetKeys(); +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_RANDEN_SLOW_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/randen_slow_test.cc b/third_party/abseil_cpp/absl/random/internal/randen_slow_test.cc new file mode 100644 index 000000000000..4a5358370586 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_slow_test.cc @@ -0,0 +1,63 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen_slow.h" + +#include <cstring> + +#include "gtest/gtest.h" +#include "absl/random/internal/randen_traits.h" + +namespace { + +using absl::random_internal::RandenSlow; +using absl::random_internal::RandenTraits; + +// Local state parameters. +constexpr size_t kSeedBytes = + RandenTraits::kStateBytes - RandenTraits::kCapacityBytes; +constexpr size_t kStateSizeT = RandenTraits::kStateBytes / sizeof(uint64_t); +constexpr size_t kSeedSizeT = kSeedBytes / sizeof(uint32_t); + +struct alignas(16) randen { + uint64_t state[kStateSizeT]; + uint32_t seed[kSeedSizeT]; +}; + +TEST(RandenSlowTest, Default) { + constexpr uint64_t kGolden[] = { + 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, + 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, + 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, + 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, + 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, + 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, + 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, + 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, + 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, + 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, + 0x026ff374c101da7e, 0x811ef0821c3de851, + }; + + alignas(16) randen d; + std::memset(d.state, 0, sizeof(d.state)); + RandenSlow::Generate(RandenSlow::GetKeys(), d.state); + + uint64_t* id = d.state; + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, *id++); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/randen_test.cc b/third_party/abseil_cpp/absl/random/internal/randen_test.cc new file mode 100644 index 000000000000..c186fe0d686b --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_test.cc @@ -0,0 +1,70 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/randen.h" + +#include <cstring> + +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" + +namespace { + +using absl::random_internal::Randen; + +// Local state parameters. +constexpr size_t kStateSizeT = Randen::kStateBytes / sizeof(uint64_t); + +TEST(RandenTest, CopyAndMove) { + static_assert(std::is_copy_constructible<Randen>::value, + "Randen must be copy constructible"); + + static_assert(absl::is_copy_assignable<Randen>::value, + "Randen must be copy assignable"); + + static_assert(std::is_move_constructible<Randen>::value, + "Randen must be move constructible"); + + static_assert(absl::is_move_assignable<Randen>::value, + "Randen must be move assignable"); +} + +TEST(RandenTest, Default) { + constexpr uint64_t kGolden[] = { + 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, + 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, + 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, + 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, + 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, + 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, + 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, + 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, + 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, + 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, + 0x026ff374c101da7e, 0x811ef0821c3de851, + }; + + alignas(16) uint64_t state[kStateSizeT]; + std::memset(state, 0, sizeof(state)); + + Randen r; + r.Generate(state); + + auto id = std::begin(state); + for (const auto& elem : kGolden) { + EXPECT_EQ(elem, *id++); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/randen_traits.h b/third_party/abseil_cpp/absl/random/internal/randen_traits.h new file mode 100644 index 000000000000..53caa936140a --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/randen_traits.h @@ -0,0 +1,88 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_RANDEN_TRAITS_H_ +#define ABSL_RANDOM_INTERNAL_RANDEN_TRAITS_H_ + +// HERMETIC NOTE: The randen_hwaes target must not introduce duplicate +// symbols from arbitrary system and other headers, since it may be built +// with different flags from other targets, using different levels of +// optimization, potentially introducing ODR violations. + +#include <cstddef> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// RANDen = RANDom generator or beetroots in Swiss German. +// 'Strong' (well-distributed, unpredictable, backtracking-resistant) random +// generator, faster in some benchmarks than std::mt19937_64 and pcg64_c32. +// +// High-level summary: +// 1) Reverie (see "A Robust and Sponge-Like PRNG with Improved Efficiency") is +// a sponge-like random generator that requires a cryptographic permutation. +// It improves upon "Provably Robust Sponge-Based PRNGs and KDFs" by +// achieving backtracking resistance with only one Permute() per buffer. +// +// 2) "Simpira v2: A Family of Efficient Permutations Using the AES Round +// Function" constructs up to 1024-bit permutations using an improved +// Generalized Feistel network with 2-round AES-128 functions. This Feistel +// block shuffle achieves diffusion faster and is less vulnerable to +// sliced-biclique attacks than the Type-2 cyclic shuffle. +// +// 3) "Improving the Generalized Feistel" and "New criterion for diffusion +// property" extends the same kind of improved Feistel block shuffle to 16 +// branches, which enables a 2048-bit permutation. +// +// Combine these three ideas and also change Simpira's subround keys from +// structured/low-entropy counters to digits of Pi (or other random source). + +// RandenTraits contains the basic algorithm traits, such as the size of the +// state, seed, sponge, etc. +struct RandenTraits { + // Size of the entire sponge / state for the randen PRNG. + static constexpr size_t kStateBytes = 256; // 2048-bit + + // Size of the 'inner' (inaccessible) part of the sponge. Larger values would + // require more frequent calls to RandenGenerate. + static constexpr size_t kCapacityBytes = 16; // 128-bit + + // Size of the default seed consumed by the sponge. + static constexpr size_t kSeedBytes = kStateBytes - kCapacityBytes; + + // Assuming 128-bit blocks, the number of blocks in the state. + // Largest size for which security proofs are known. + static constexpr size_t kFeistelBlocks = 16; + + // Ensures SPRP security and two full subblock diffusions. + // Must be > 4 * log2(kFeistelBlocks). + static constexpr size_t kFeistelRounds = 16 + 1; + + // Size of the key. A 128-bit key block is used for every-other + // feistel block (Type-2 generalized Feistel network) in each round. + static constexpr size_t kKeyBytes = 16 * kFeistelRounds * kFeistelBlocks / 2; +}; + +// Randen key arrays. In randen_round_keys.cc +extern const unsigned char kRandenRoundKeys[RandenTraits::kKeyBytes]; +extern const unsigned char kRandenRoundKeysBE[RandenTraits::kKeyBytes]; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_RANDEN_TRAITS_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/salted_seed_seq.h b/third_party/abseil_cpp/absl/random/internal/salted_seed_seq.h new file mode 100644 index 000000000000..5953a090f80c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/salted_seed_seq.h @@ -0,0 +1,167 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_SALTED_SEED_SEQ_H_ +#define ABSL_RANDOM_INTERNAL_SALTED_SEED_SEQ_H_ + +#include <cstdint> +#include <cstdlib> +#include <initializer_list> +#include <iterator> +#include <memory> +#include <type_traits> +#include <utility> + +#include "absl/container/inlined_vector.h" +#include "absl/meta/type_traits.h" +#include "absl/random/internal/seed_material.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// This class conforms to the C++ Standard "Seed Sequence" concept +// [rand.req.seedseq]. +// +// A `SaltedSeedSeq` is meant to wrap an existing seed sequence and modify +// generated sequence by mixing with extra entropy. This entropy may be +// build-dependent or process-dependent. The implementation may change to be +// have either or both kinds of entropy. If salt is not available sequence is +// not modified. +template <typename SSeq> +class SaltedSeedSeq { + public: + using inner_sequence_type = SSeq; + using result_type = typename SSeq::result_type; + + SaltedSeedSeq() : seq_(absl::make_unique<SSeq>()) {} + + template <typename Iterator> + SaltedSeedSeq(Iterator begin, Iterator end) + : seq_(absl::make_unique<SSeq>(begin, end)) {} + + template <typename T> + SaltedSeedSeq(std::initializer_list<T> il) + : SaltedSeedSeq(il.begin(), il.end()) {} + + SaltedSeedSeq(const SaltedSeedSeq&) = delete; + SaltedSeedSeq& operator=(const SaltedSeedSeq&) = delete; + + SaltedSeedSeq(SaltedSeedSeq&&) = default; + SaltedSeedSeq& operator=(SaltedSeedSeq&&) = default; + + template <typename RandomAccessIterator> + void generate(RandomAccessIterator begin, RandomAccessIterator end) { + // The common case is that generate is called with ContiguousIterators + // to uint arrays. Such contiguous memory regions may be optimized, + // which we detect here. + using tag = absl::conditional_t< + (std::is_pointer<RandomAccessIterator>::value && + std::is_same<absl::decay_t<decltype(*begin)>, uint32_t>::value), + ContiguousAndUint32Tag, DefaultTag>; + if (begin != end) { + generate_impl(begin, end, tag{}); + } + } + + template <typename OutIterator> + void param(OutIterator out) const { + seq_->param(out); + } + + size_t size() const { return seq_->size(); } + + private: + struct ContiguousAndUint32Tag {}; + struct DefaultTag {}; + + // Generate which requires the iterators are contiguous pointers to uint32_t. + void generate_impl(uint32_t* begin, uint32_t* end, ContiguousAndUint32Tag) { + generate_contiguous(absl::MakeSpan(begin, end)); + } + + // The uncommon case for generate is that it is called with iterators over + // some other buffer type which is assignable from a 32-bit value. In this + // case we allocate a temporary 32-bit buffer and then copy-assign back + // to the initial inputs. + template <typename RandomAccessIterator> + void generate_impl(RandomAccessIterator begin, RandomAccessIterator end, + DefaultTag) { + return generate_and_copy(std::distance(begin, end), begin); + } + + // Fills the initial seed buffer the underlying SSeq::generate() call, + // mixing in the salt material. + void generate_contiguous(absl::Span<uint32_t> buffer) { + seq_->generate(buffer.begin(), buffer.end()); + const uint32_t salt = absl::random_internal::GetSaltMaterial().value_or(0); + MixIntoSeedMaterial(absl::MakeConstSpan(&salt, 1), buffer); + } + + // Allocates a seed buffer of `n` elements, generates the seed, then + // copies the result into the `out` iterator. + template <typename Iterator> + void generate_and_copy(size_t n, Iterator out) { + // Allocate a temporary buffer, generate, and then copy. + absl::InlinedVector<uint32_t, 8> data(n, 0); + generate_contiguous(absl::MakeSpan(data.data(), data.size())); + std::copy(data.begin(), data.end(), out); + } + + // Because [rand.req.seedseq] is not required to be copy-constructible, + // copy-assignable nor movable, we wrap it with unique pointer to be able + // to move SaltedSeedSeq. + std::unique_ptr<SSeq> seq_; +}; + +// is_salted_seed_seq indicates whether the type is a SaltedSeedSeq. +template <typename T, typename = void> +struct is_salted_seed_seq : public std::false_type {}; + +template <typename T> +struct is_salted_seed_seq< + T, typename std::enable_if<std::is_same< + T, SaltedSeedSeq<typename T::inner_sequence_type>>::value>::type> + : public std::true_type {}; + +// MakeSaltedSeedSeq returns a salted variant of the seed sequence. +// When provided with an existing SaltedSeedSeq, returns the input parameter, +// otherwise constructs a new SaltedSeedSeq which embodies the original +// non-salted seed parameters. +template < + typename SSeq, // + typename EnableIf = absl::enable_if_t<is_salted_seed_seq<SSeq>::value>> +SSeq MakeSaltedSeedSeq(SSeq&& seq) { + return SSeq(std::forward<SSeq>(seq)); +} + +template < + typename SSeq, // + typename EnableIf = absl::enable_if_t<!is_salted_seed_seq<SSeq>::value>> +SaltedSeedSeq<typename std::decay<SSeq>::type> MakeSaltedSeedSeq(SSeq&& seq) { + using sseq_type = typename std::decay<SSeq>::type; + using result_type = typename sseq_type::result_type; + + absl::InlinedVector<result_type, 8> data; + seq.param(std::back_inserter(data)); + return SaltedSeedSeq<sseq_type>(data.begin(), data.end()); +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_SALTED_SEED_SEQ_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/salted_seed_seq_test.cc b/third_party/abseil_cpp/absl/random/internal/salted_seed_seq_test.cc new file mode 100644 index 000000000000..0bf19a63ef8c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/salted_seed_seq_test.cc @@ -0,0 +1,168 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/salted_seed_seq.h" + +#include <iterator> +#include <random> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using absl::random_internal::GetSaltMaterial; +using absl::random_internal::MakeSaltedSeedSeq; +using absl::random_internal::SaltedSeedSeq; +using testing::Eq; +using testing::Pointwise; + +namespace { + +template <typename Sseq> +void ConformsToInterface() { + // Check that the SeedSequence can be default-constructed. + { Sseq default_constructed_seq; } + // Check that the SeedSequence can be constructed with two iterators. + { + uint32_t init_array[] = {1, 3, 5, 7, 9}; + Sseq iterator_constructed_seq(std::begin(init_array), std::end(init_array)); + } + // Check that the SeedSequence can be std::initializer_list-constructed. + { Sseq list_constructed_seq = {1, 3, 5, 7, 9, 11, 13}; } + // Check that param() and size() return state provided to constructor. + { + uint32_t init_array[] = {1, 2, 3, 4, 5}; + Sseq seq(std::begin(init_array), std::end(init_array)); + EXPECT_EQ(seq.size(), ABSL_ARRAYSIZE(init_array)); + + std::vector<uint32_t> state_vector; + seq.param(std::back_inserter(state_vector)); + + EXPECT_EQ(state_vector.size(), ABSL_ARRAYSIZE(init_array)); + for (int i = 0; i < state_vector.size(); i++) { + EXPECT_EQ(state_vector[i], i + 1); + } + } + // Check for presence of generate() method. + { + Sseq seq; + uint32_t seeds[5]; + + seq.generate(std::begin(seeds), std::end(seeds)); + } +} + +TEST(SaltedSeedSeq, CheckInterfaces) { + // Control case + ConformsToInterface<std::seed_seq>(); + + // Abseil classes + ConformsToInterface<SaltedSeedSeq<std::seed_seq>>(); +} + +TEST(SaltedSeedSeq, CheckConstructingFromOtherSequence) { + std::vector<uint32_t> seed_values(10, 1); + std::seed_seq seq(seed_values.begin(), seed_values.end()); + auto salted_seq = MakeSaltedSeedSeq(std::move(seq)); + + EXPECT_EQ(seq.size(), salted_seq.size()); + + std::vector<uint32_t> param_result; + seq.param(std::back_inserter(param_result)); + + EXPECT_EQ(seed_values, param_result); +} + +TEST(SaltedSeedSeq, SaltedSaltedSeedSeqIsNotDoubleSalted) { + uint32_t init[] = {1, 3, 5, 7, 9}; + + std::seed_seq seq(std::begin(init), std::end(init)); + + // The first salting. + SaltedSeedSeq<std::seed_seq> salted_seq = MakeSaltedSeedSeq(std::move(seq)); + uint32_t a[16]; + salted_seq.generate(std::begin(a), std::end(a)); + + // The second salting. + SaltedSeedSeq<std::seed_seq> salted_salted_seq = + MakeSaltedSeedSeq(std::move(salted_seq)); + uint32_t b[16]; + salted_salted_seq.generate(std::begin(b), std::end(b)); + + // ... both should be equal. + EXPECT_THAT(b, Pointwise(Eq(), a)) << "a[0] " << a[0]; +} + +TEST(SaltedSeedSeq, SeedMaterialIsSalted) { + const size_t kNumBlocks = 16; + + uint32_t seed_material[kNumBlocks]; + std::random_device urandom{"/dev/urandom"}; + for (uint32_t& seed : seed_material) { + seed = urandom(); + } + + std::seed_seq seq(std::begin(seed_material), std::end(seed_material)); + SaltedSeedSeq<std::seed_seq> salted_seq(std::begin(seed_material), + std::end(seed_material)); + + bool salt_is_available = GetSaltMaterial().has_value(); + + // If salt is available generated sequence should be different. + if (salt_is_available) { + uint32_t outputs[kNumBlocks]; + uint32_t salted_outputs[kNumBlocks]; + + seq.generate(std::begin(outputs), std::end(outputs)); + salted_seq.generate(std::begin(salted_outputs), std::end(salted_outputs)); + + EXPECT_THAT(outputs, Pointwise(testing::Ne(), salted_outputs)); + } +} + +TEST(SaltedSeedSeq, GenerateAcceptsDifferentTypes) { + const size_t kNumBlocks = 4; + + SaltedSeedSeq<std::seed_seq> seq({1, 2, 3}); + + uint32_t expected[kNumBlocks]; + seq.generate(std::begin(expected), std::end(expected)); + + // 32-bit outputs + { + unsigned long seed_material[kNumBlocks]; // NOLINT(runtime/int) + seq.generate(std::begin(seed_material), std::end(seed_material)); + EXPECT_THAT(seed_material, Pointwise(Eq(), expected)); + } + { + unsigned int seed_material[kNumBlocks]; // NOLINT(runtime/int) + seq.generate(std::begin(seed_material), std::end(seed_material)); + EXPECT_THAT(seed_material, Pointwise(Eq(), expected)); + } + + // 64-bit outputs. + { + uint64_t seed_material[kNumBlocks]; + seq.generate(std::begin(seed_material), std::end(seed_material)); + EXPECT_THAT(seed_material, Pointwise(Eq(), expected)); + } + { + int64_t seed_material[kNumBlocks]; + seq.generate(std::begin(seed_material), std::end(seed_material)); + EXPECT_THAT(seed_material, Pointwise(Eq(), expected)); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/seed_material.cc b/third_party/abseil_cpp/absl/random/internal/seed_material.cc new file mode 100644 index 000000000000..4d38a57419ec --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/seed_material.cc @@ -0,0 +1,219 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/seed_material.h" + +#include <fcntl.h> + +#ifndef _WIN32 +#include <unistd.h> +#else +#include <io.h> +#endif + +#include <algorithm> +#include <cerrno> +#include <cstdint> +#include <cstdlib> +#include <cstring> + +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/ascii.h" +#include "absl/strings/escaping.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" + +#if defined(__native_client__) + +#include <nacl/nacl_random.h> +#define ABSL_RANDOM_USE_NACL_SECURE_RANDOM 1 + +#elif defined(_WIN32) + +#include <windows.h> +#define ABSL_RANDOM_USE_BCRYPT 1 +#pragma comment(lib, "bcrypt.lib") + +#elif defined(__Fuchsia__) +#include <zircon/syscalls.h> + +#endif + +#if defined(ABSL_RANDOM_USE_BCRYPT) +#include <bcrypt.h> + +#ifndef BCRYPT_SUCCESS +#define BCRYPT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) +#endif +// Also link bcrypt; this can be done via linker options or: +// #pragma comment(lib, "bcrypt.lib") +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { +namespace { + +// Read OS Entropy for random number seeds. +// TODO(absl-team): Possibly place a cap on how much entropy may be read at a +// time. + +#if defined(ABSL_RANDOM_USE_BCRYPT) + +// On Windows potentially use the BCRYPT CNG API to read available entropy. +bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { + BCRYPT_ALG_HANDLE hProvider; + NTSTATUS ret; + ret = BCryptOpenAlgorithmProvider(&hProvider, BCRYPT_RNG_ALGORITHM, + MS_PRIMITIVE_PROVIDER, 0); + if (!(BCRYPT_SUCCESS(ret))) { + ABSL_RAW_LOG(ERROR, "Failed to open crypto provider."); + return false; + } + ret = BCryptGenRandom( + hProvider, // provider + reinterpret_cast<UCHAR*>(values.data()), // buffer + static_cast<ULONG>(sizeof(uint32_t) * values.size()), // bytes + 0); // flags + BCryptCloseAlgorithmProvider(hProvider, 0); + return BCRYPT_SUCCESS(ret); +} + +#elif defined(ABSL_RANDOM_USE_NACL_SECURE_RANDOM) + +// On NaCL use nacl_secure_random to acquire bytes. +bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { + auto buffer = reinterpret_cast<uint8_t*>(values.data()); + size_t buffer_size = sizeof(uint32_t) * values.size(); + + uint8_t* output_ptr = buffer; + while (buffer_size > 0) { + size_t nread = 0; + const int error = nacl_secure_random(output_ptr, buffer_size, &nread); + if (error != 0 || nread > buffer_size) { + ABSL_RAW_LOG(ERROR, "Failed to read secure_random seed data: %d", error); + return false; + } + output_ptr += nread; + buffer_size -= nread; + } + return true; +} + +#elif defined(__Fuchsia__) + +bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { + auto buffer = reinterpret_cast<uint8_t*>(values.data()); + size_t buffer_size = sizeof(uint32_t) * values.size(); + zx_cprng_draw(buffer, buffer_size); + return true; +} + +#else + +// On *nix, read entropy from /dev/urandom. +bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { + const char kEntropyFile[] = "/dev/urandom"; + + auto buffer = reinterpret_cast<uint8_t*>(values.data()); + size_t buffer_size = sizeof(uint32_t) * values.size(); + + int dev_urandom = open(kEntropyFile, O_RDONLY); + bool success = (-1 != dev_urandom); + if (!success) { + return false; + } + + while (success && buffer_size > 0) { + int bytes_read = read(dev_urandom, buffer, buffer_size); + int read_error = errno; + success = (bytes_read > 0); + if (success) { + buffer += bytes_read; + buffer_size -= bytes_read; + } else if (bytes_read == -1 && read_error == EINTR) { + success = true; // Need to try again. + } + } + close(dev_urandom); + return success; +} + +#endif + +} // namespace + +bool ReadSeedMaterialFromOSEntropy(absl::Span<uint32_t> values) { + assert(values.data() != nullptr); + if (values.data() == nullptr) { + return false; + } + if (values.empty()) { + return true; + } + return ReadSeedMaterialFromOSEntropyImpl(values); +} + +void MixIntoSeedMaterial(absl::Span<const uint32_t> sequence, + absl::Span<uint32_t> seed_material) { + // Algorithm is based on code available at + // https://gist.github.com/imneme/540829265469e673d045 + constexpr uint32_t kInitVal = 0x43b0d7e5; + constexpr uint32_t kHashMul = 0x931e8875; + constexpr uint32_t kMixMulL = 0xca01f9dd; + constexpr uint32_t kMixMulR = 0x4973f715; + constexpr uint32_t kShiftSize = sizeof(uint32_t) * 8 / 2; + + uint32_t hash_const = kInitVal; + auto hash = [&](uint32_t value) { + value ^= hash_const; + hash_const *= kHashMul; + value *= hash_const; + value ^= value >> kShiftSize; + return value; + }; + + auto mix = [&](uint32_t x, uint32_t y) { + uint32_t result = kMixMulL * x - kMixMulR * y; + result ^= result >> kShiftSize; + return result; + }; + + for (const auto& seq_val : sequence) { + for (auto& elem : seed_material) { + elem = mix(elem, hash(seq_val)); + } + } +} + +absl::optional<uint32_t> GetSaltMaterial() { + // Salt must be common for all generators within the same process so read it + // only once and store in static variable. + static const auto salt_material = []() -> absl::optional<uint32_t> { + uint32_t salt_value = 0; + + if (random_internal::ReadSeedMaterialFromOSEntropy( + MakeSpan(&salt_value, 1))) { + return salt_value; + } + + return absl::nullopt; + }(); + + return salt_material; +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/internal/seed_material.h b/third_party/abseil_cpp/absl/random/internal/seed_material.h new file mode 100644 index 000000000000..4be10e925685 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/seed_material.h @@ -0,0 +1,104 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_SEED_MATERIAL_H_ +#define ABSL_RANDOM_INTERNAL_SEED_MATERIAL_H_ + +#include <cassert> +#include <cstdint> +#include <cstdlib> +#include <string> +#include <vector> + +#include "absl/base/attributes.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Returns the number of 32-bit blocks needed to contain the given number of +// bits. +constexpr size_t SeedBitsToBlocks(size_t seed_size) { + return (seed_size + 31) / 32; +} + +// Amount of entropy (measured in bits) used to instantiate a Seed Sequence, +// with which to create a URBG. +constexpr size_t kEntropyBitsNeeded = 256; + +// Amount of entropy (measured in 32-bit blocks) used to instantiate a Seed +// Sequence, with which to create a URBG. +constexpr size_t kEntropyBlocksNeeded = + random_internal::SeedBitsToBlocks(kEntropyBitsNeeded); + +static_assert(kEntropyBlocksNeeded > 0, + "Entropy used to seed URBGs must be nonzero."); + +// Attempts to fill a span of uint32_t-values using an OS-provided source of +// true entropy (eg. /dev/urandom) into an array of uint32_t blocks of data. The +// resulting array may be used to initialize an instance of a class conforming +// to the C++ Standard "Seed Sequence" concept [rand.req.seedseq]. +// +// If values.data() == nullptr, the behavior is undefined. +ABSL_MUST_USE_RESULT +bool ReadSeedMaterialFromOSEntropy(absl::Span<uint32_t> values); + +// Attempts to fill a span of uint32_t-values using variates generated by an +// existing instance of a class conforming to the C++ Standard "Uniform Random +// Bit Generator" concept [rand.req.urng]. The resulting data may be used to +// initialize an instance of a class conforming to the C++ Standard +// "Seed Sequence" concept [rand.req.seedseq]. +// +// If urbg == nullptr or values.data() == nullptr, the behavior is undefined. +template <typename URBG> +ABSL_MUST_USE_RESULT bool ReadSeedMaterialFromURBG( + URBG* urbg, absl::Span<uint32_t> values) { + random_internal::FastUniformBits<uint32_t> distr; + + assert(urbg != nullptr && values.data() != nullptr); + if (urbg == nullptr || values.data() == nullptr) { + return false; + } + + for (uint32_t& seed_value : values) { + seed_value = distr(*urbg); + } + return true; +} + +// Mixes given sequence of values with into given sequence of seed material. +// Time complexity of this function is O(sequence.size() * +// seed_material.size()). +// +// Algorithm is based on code available at +// https://gist.github.com/imneme/540829265469e673d045 +// by Melissa O'Neill. +void MixIntoSeedMaterial(absl::Span<const uint32_t> sequence, + absl::Span<uint32_t> seed_material); + +// Returns salt value. +// +// Salt is obtained only once and stored in static variable. +// +// May return empty value if optaining the salt was not possible. +absl::optional<uint32_t> GetSaltMaterial(); + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_SEED_MATERIAL_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/seed_material_test.cc b/third_party/abseil_cpp/absl/random/internal/seed_material_test.cc new file mode 100644 index 000000000000..6db2820ec707 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/seed_material_test.cc @@ -0,0 +1,202 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/seed_material.h" + +#include <bitset> +#include <cstdlib> +#include <cstring> +#include <random> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#ifdef __ANDROID__ +// Android assert messages only go to system log, so death tests cannot inspect +// the message for matching. +#define ABSL_EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH_IF_SUPPORTED(statement, ".*") +#else +#define ABSL_EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH_IF_SUPPORTED(statement, regex) +#endif + +namespace { + +using testing::Each; +using testing::ElementsAre; +using testing::Eq; +using testing::Ne; +using testing::Pointwise; + +TEST(SeedBitsToBlocks, VerifyCases) { + EXPECT_EQ(0, absl::random_internal::SeedBitsToBlocks(0)); + EXPECT_EQ(1, absl::random_internal::SeedBitsToBlocks(1)); + EXPECT_EQ(1, absl::random_internal::SeedBitsToBlocks(31)); + EXPECT_EQ(1, absl::random_internal::SeedBitsToBlocks(32)); + EXPECT_EQ(2, absl::random_internal::SeedBitsToBlocks(33)); + EXPECT_EQ(4, absl::random_internal::SeedBitsToBlocks(127)); + EXPECT_EQ(4, absl::random_internal::SeedBitsToBlocks(128)); + EXPECT_EQ(5, absl::random_internal::SeedBitsToBlocks(129)); +} + +TEST(ReadSeedMaterialFromOSEntropy, SuccessiveReadsAreDistinct) { + constexpr size_t kSeedMaterialSize = 64; + uint32_t seed_material_1[kSeedMaterialSize] = {}; + uint32_t seed_material_2[kSeedMaterialSize] = {}; + + EXPECT_TRUE(absl::random_internal::ReadSeedMaterialFromOSEntropy( + absl::Span<uint32_t>(seed_material_1, kSeedMaterialSize))); + EXPECT_TRUE(absl::random_internal::ReadSeedMaterialFromOSEntropy( + absl::Span<uint32_t>(seed_material_2, kSeedMaterialSize))); + + EXPECT_THAT(seed_material_1, Pointwise(Ne(), seed_material_2)); +} + +TEST(ReadSeedMaterialFromOSEntropy, ReadZeroBytesIsNoOp) { + uint32_t seed_material[32] = {}; + std::memset(seed_material, 0xAA, sizeof(seed_material)); + EXPECT_TRUE(absl::random_internal::ReadSeedMaterialFromOSEntropy( + absl::Span<uint32_t>(seed_material, 0))); + + EXPECT_THAT(seed_material, Each(Eq(0xAAAAAAAA))); +} + +TEST(ReadSeedMaterialFromOSEntropy, NullPtrVectorArgument) { +#ifdef NDEBUG + EXPECT_FALSE(absl::random_internal::ReadSeedMaterialFromOSEntropy( + absl::Span<uint32_t>(nullptr, 32))); +#else + bool result; + ABSL_EXPECT_DEATH_IF_SUPPORTED( + result = absl::random_internal::ReadSeedMaterialFromOSEntropy( + absl::Span<uint32_t>(nullptr, 32)), + "!= nullptr"); + (void)result; // suppress unused-variable warning +#endif +} + +TEST(ReadSeedMaterialFromURBG, SeedMaterialEqualsVariateSequence) { + // Two default-constructed instances of std::mt19937_64 are guaranteed to + // produce equal variate-sequences. + std::mt19937 urbg_1; + std::mt19937 urbg_2; + constexpr size_t kSeedMaterialSize = 1024; + uint32_t seed_material[kSeedMaterialSize] = {}; + + EXPECT_TRUE(absl::random_internal::ReadSeedMaterialFromURBG( + &urbg_1, absl::Span<uint32_t>(seed_material, kSeedMaterialSize))); + for (uint32_t seed : seed_material) { + EXPECT_EQ(seed, urbg_2()); + } +} + +TEST(ReadSeedMaterialFromURBG, ReadZeroBytesIsNoOp) { + std::mt19937_64 urbg; + uint32_t seed_material[32]; + std::memset(seed_material, 0xAA, sizeof(seed_material)); + EXPECT_TRUE(absl::random_internal::ReadSeedMaterialFromURBG( + &urbg, absl::Span<uint32_t>(seed_material, 0))); + + EXPECT_THAT(seed_material, Each(Eq(0xAAAAAAAA))); +} + +TEST(ReadSeedMaterialFromURBG, NullUrbgArgument) { + constexpr size_t kSeedMaterialSize = 32; + uint32_t seed_material[kSeedMaterialSize]; +#ifdef NDEBUG + EXPECT_FALSE(absl::random_internal::ReadSeedMaterialFromURBG<std::mt19937_64>( + nullptr, absl::Span<uint32_t>(seed_material, kSeedMaterialSize))); +#else + bool result; + ABSL_EXPECT_DEATH_IF_SUPPORTED( + result = absl::random_internal::ReadSeedMaterialFromURBG<std::mt19937_64>( + nullptr, absl::Span<uint32_t>(seed_material, kSeedMaterialSize)), + "!= nullptr"); + (void)result; // suppress unused-variable warning +#endif +} + +TEST(ReadSeedMaterialFromURBG, NullPtrVectorArgument) { + std::mt19937_64 urbg; +#ifdef NDEBUG + EXPECT_FALSE(absl::random_internal::ReadSeedMaterialFromURBG( + &urbg, absl::Span<uint32_t>(nullptr, 32))); +#else + bool result; + ABSL_EXPECT_DEATH_IF_SUPPORTED( + result = absl::random_internal::ReadSeedMaterialFromURBG( + &urbg, absl::Span<uint32_t>(nullptr, 32)), + "!= nullptr"); + (void)result; // suppress unused-variable warning +#endif +} + +// The avalanche effect is a desirable cryptographic property of hashes in which +// changing a single bit in the input causes each bit of the output to be +// changed with probability near 50%. +// +// https://en.wikipedia.org/wiki/Avalanche_effect + +TEST(MixSequenceIntoSeedMaterial, AvalancheEffectTestOneBitLong) { + std::vector<uint32_t> seed_material = {1, 2, 3, 4, 5, 6, 7, 8}; + + // For every 32-bit number with exactly one bit set, verify the avalanche + // effect holds. In order to reduce flakiness of tests, accept values + // anywhere in the range of 30%-70%. + for (uint32_t v = 1; v != 0; v <<= 1) { + std::vector<uint32_t> seed_material_copy = seed_material; + absl::random_internal::MixIntoSeedMaterial( + absl::Span<uint32_t>(&v, 1), + absl::Span<uint32_t>(seed_material_copy.data(), + seed_material_copy.size())); + + uint32_t changed_bits = 0; + for (size_t i = 0; i < seed_material.size(); i++) { + std::bitset<sizeof(uint32_t) * 8> bitset(seed_material[i] ^ + seed_material_copy[i]); + changed_bits += bitset.count(); + } + + EXPECT_LE(changed_bits, 0.7 * sizeof(uint32_t) * 8 * seed_material.size()); + EXPECT_GE(changed_bits, 0.3 * sizeof(uint32_t) * 8 * seed_material.size()); + } +} + +TEST(MixSequenceIntoSeedMaterial, AvalancheEffectTestOneBitShort) { + std::vector<uint32_t> seed_material = {1}; + + // For every 32-bit number with exactly one bit set, verify the avalanche + // effect holds. In order to reduce flakiness of tests, accept values + // anywhere in the range of 30%-70%. + for (uint32_t v = 1; v != 0; v <<= 1) { + std::vector<uint32_t> seed_material_copy = seed_material; + absl::random_internal::MixIntoSeedMaterial( + absl::Span<uint32_t>(&v, 1), + absl::Span<uint32_t>(seed_material_copy.data(), + seed_material_copy.size())); + + uint32_t changed_bits = 0; + for (size_t i = 0; i < seed_material.size(); i++) { + std::bitset<sizeof(uint32_t) * 8> bitset(seed_material[i] ^ + seed_material_copy[i]); + changed_bits += bitset.count(); + } + + EXPECT_LE(changed_bits, 0.7 * sizeof(uint32_t) * 8 * seed_material.size()); + EXPECT_GE(changed_bits, 0.3 * sizeof(uint32_t) * 8 * seed_material.size()); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/sequence_urbg.h b/third_party/abseil_cpp/absl/random/internal/sequence_urbg.h new file mode 100644 index 000000000000..bc96a12cd29c --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/sequence_urbg.h @@ -0,0 +1,60 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_SEQUENCE_URBG_H_ +#define ABSL_RANDOM_INTERNAL_SEQUENCE_URBG_H_ + +#include <cstdint> +#include <cstring> +#include <limits> +#include <type_traits> +#include <vector> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// `sequence_urbg` is a simple random number generator which meets the +// requirements of [rand.req.urbg], and is solely for testing absl +// distributions. +class sequence_urbg { + public: + using result_type = uint64_t; + + static constexpr result_type(min)() { + return (std::numeric_limits<result_type>::min)(); + } + static constexpr result_type(max)() { + return (std::numeric_limits<result_type>::max)(); + } + + sequence_urbg(std::initializer_list<result_type> data) : i_(0), data_(data) {} + void reset() { i_ = 0; } + + result_type operator()() { return data_[i_++ % data_.size()]; } + + size_t invocations() const { return i_; } + + private: + size_t i_; + std::vector<result_type> data_; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_SEQUENCE_URBG_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/traits.h b/third_party/abseil_cpp/absl/random/internal/traits.h new file mode 100644 index 000000000000..75772bd9abb4 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/traits.h @@ -0,0 +1,101 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_TRAITS_H_ +#define ABSL_RANDOM_INTERNAL_TRAITS_H_ + +#include <cstdint> +#include <limits> +#include <type_traits> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// random_internal::is_widening_convertible<A, B> +// +// Returns whether a type A is widening-convertible to a type B. +// +// A is widening-convertible to B means: +// A a = <any number>; +// B b = a; +// A c = b; +// EXPECT_EQ(a, c); +template <typename A, typename B> +class is_widening_convertible { + // As long as there are enough bits in the exact part of a number: + // - unsigned can fit in float, signed, unsigned + // - signed can fit in float, signed + // - float can fit in float + // So we define rank to be: + // - rank(float) -> 2 + // - rank(signed) -> 1 + // - rank(unsigned) -> 0 + template <class T> + static constexpr int rank() { + return !std::numeric_limits<T>::is_integer + + std::numeric_limits<T>::is_signed; + } + + public: + // If an arithmetic-type B can represent at least as many digits as a type A, + // and B belongs to a rank no lower than A, then A can be safely represented + // by B through a widening-conversion. + static constexpr bool value = + std::numeric_limits<A>::digits <= std::numeric_limits<B>::digits && + rank<A>() <= rank<B>(); +}; + +// unsigned_bits<N>::type returns the unsigned int type with the indicated +// number of bits. +template <size_t N> +struct unsigned_bits; + +template <> +struct unsigned_bits<8> { + using type = uint8_t; +}; +template <> +struct unsigned_bits<16> { + using type = uint16_t; +}; +template <> +struct unsigned_bits<32> { + using type = uint32_t; +}; +template <> +struct unsigned_bits<64> { + using type = uint64_t; +}; + +#ifdef ABSL_HAVE_INTRINSIC_INT128 +template <> +struct unsigned_bits<128> { + using type = __uint128_t; +}; +#endif + +template <typename IntType> +struct make_unsigned_bits { + using type = typename unsigned_bits<std::numeric_limits< + typename std::make_unsigned<IntType>::type>::digits>::type; +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_TRAITS_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/traits_test.cc b/third_party/abseil_cpp/absl/random/internal/traits_test.cc new file mode 100644 index 000000000000..a844887d3e44 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/traits_test.cc @@ -0,0 +1,126 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/traits.h" + +#include <cstdint> +#include <type_traits> + +#include "gtest/gtest.h" + +namespace { + +using absl::random_internal::is_widening_convertible; + +// CheckWideningConvertsToSelf<T1, T2, ...>() +// +// For each type T, checks: +// - T IS widening-convertible to itself. +// +template <typename T> +void CheckWideningConvertsToSelf() { + static_assert(is_widening_convertible<T, T>::value, + "Type is not convertible to self!"); +} + +template <typename T, typename Next, typename... Args> +void CheckWideningConvertsToSelf() { + CheckWideningConvertsToSelf<T>(); + CheckWideningConvertsToSelf<Next, Args...>(); +} + +// CheckNotWideningConvertibleWithSigned<T1, T2, ...>() +// +// For each unsigned-type T, checks that: +// - T is NOT widening-convertible to Signed(T) +// - Signed(T) is NOT widening-convertible to T +// +template <typename T> +void CheckNotWideningConvertibleWithSigned() { + using signed_t = typename std::make_signed<T>::type; + + static_assert(!is_widening_convertible<T, signed_t>::value, + "Unsigned type is convertible to same-sized signed-type!"); + static_assert(!is_widening_convertible<signed_t, T>::value, + "Signed type is convertible to same-sized unsigned-type!"); +} + +template <typename T, typename Next, typename... Args> +void CheckNotWideningConvertibleWithSigned() { + CheckNotWideningConvertibleWithSigned<T>(); + CheckWideningConvertsToSelf<Next, Args...>(); +} + +// CheckWideningConvertsToLargerType<T1, T2, ...>() +// +// For each successive unsigned-types {Ti, Ti+1}, checks that: +// - Ti IS widening-convertible to Ti+1 +// - Ti IS widening-convertible to Signed(Ti+1) +// - Signed(Ti) is NOT widening-convertible to Ti +// - Signed(Ti) IS widening-convertible to Ti+1 +template <typename T, typename Higher> +void CheckWideningConvertsToLargerTypes() { + using signed_t = typename std::make_signed<T>::type; + using higher_t = Higher; + using signed_higher_t = typename std::make_signed<Higher>::type; + + static_assert(is_widening_convertible<T, higher_t>::value, + "Type not embeddable into larger type!"); + static_assert(is_widening_convertible<T, signed_higher_t>::value, + "Type not embeddable into larger signed type!"); + static_assert(!is_widening_convertible<signed_t, higher_t>::value, + "Signed type is embeddable into larger unsigned type!"); + static_assert(is_widening_convertible<signed_t, signed_higher_t>::value, + "Signed type not embeddable into larger signed type!"); +} + +template <typename T, typename Higher, typename Next, typename... Args> +void CheckWideningConvertsToLargerTypes() { + CheckWideningConvertsToLargerTypes<T, Higher>(); + CheckWideningConvertsToLargerTypes<Higher, Next, Args...>(); +} + +// CheckWideningConvertsTo<T, U, [expect]> +// +// Checks that T DOES widening-convert to U. +// If "expect" is false, then asserts that T does NOT widening-convert to U. +template <typename T, typename U, bool expect = true> +void CheckWideningConvertsTo() { + static_assert(is_widening_convertible<T, U>::value == expect, + "Unexpected result for is_widening_convertible<T, U>!"); +} + +TEST(TraitsTest, IsWideningConvertibleTest) { + constexpr bool kInvalid = false; + + CheckWideningConvertsToSelf< + uint8_t, uint16_t, uint32_t, uint64_t, + int8_t, int16_t, int32_t, int64_t, + float, double>(); + CheckNotWideningConvertibleWithSigned< + uint8_t, uint16_t, uint32_t, uint64_t>(); + CheckWideningConvertsToLargerTypes< + uint8_t, uint16_t, uint32_t, uint64_t>(); + + CheckWideningConvertsTo<float, double>(); + CheckWideningConvertsTo<uint16_t, float>(); + CheckWideningConvertsTo<uint32_t, double>(); + CheckWideningConvertsTo<uint64_t, double, kInvalid>(); + CheckWideningConvertsTo<double, float, kInvalid>(); + + CheckWideningConvertsTo<bool, int>(); + CheckWideningConvertsTo<bool, float>(); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/uniform_helper.h b/third_party/abseil_cpp/absl/random/internal/uniform_helper.h new file mode 100644 index 000000000000..1243bc1c62ab --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/uniform_helper.h @@ -0,0 +1,244 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef ABSL_RANDOM_INTERNAL_UNIFORM_HELPER_H_ +#define ABSL_RANDOM_INTERNAL_UNIFORM_HELPER_H_ + +#include <cmath> +#include <limits> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/meta/type_traits.h" +#include "absl/random/internal/traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +template <typename IntType> +class uniform_int_distribution; + +template <typename RealType> +class uniform_real_distribution; + +// Interval tag types which specify whether the interval is open or closed +// on either boundary. + +namespace random_internal { +template <typename T> +struct TagTypeCompare {}; + +template <typename T> +constexpr bool operator==(TagTypeCompare<T>, TagTypeCompare<T>) { + // Tags are mono-states. They always compare equal. + return true; +} +template <typename T> +constexpr bool operator!=(TagTypeCompare<T>, TagTypeCompare<T>) { + return false; +} + +} // namespace random_internal + +struct IntervalClosedClosedTag + : public random_internal::TagTypeCompare<IntervalClosedClosedTag> {}; +struct IntervalClosedOpenTag + : public random_internal::TagTypeCompare<IntervalClosedOpenTag> {}; +struct IntervalOpenClosedTag + : public random_internal::TagTypeCompare<IntervalOpenClosedTag> {}; +struct IntervalOpenOpenTag + : public random_internal::TagTypeCompare<IntervalOpenOpenTag> {}; + +namespace random_internal { + +// In the absence of an explicitly provided return-type, the template +// "uniform_inferred_return_t<A, B>" is used to derive a suitable type, based on +// the data-types of the endpoint-arguments {A lo, B hi}. +// +// Given endpoints {A lo, B hi}, one of {A, B} will be chosen as the +// return-type, if one type can be implicitly converted into the other, in a +// lossless way. The template "is_widening_convertible" implements the +// compile-time logic for deciding if such a conversion is possible. +// +// If no such conversion between {A, B} exists, then the overload for +// absl::Uniform() will be discarded, and the call will be ill-formed. +// Return-type for absl::Uniform() when the return-type is inferred. +template <typename A, typename B> +using uniform_inferred_return_t = + absl::enable_if_t<absl::disjunction<is_widening_convertible<A, B>, + is_widening_convertible<B, A>>::value, + typename std::conditional< + is_widening_convertible<A, B>::value, B, A>::type>; + +// The functions +// uniform_lower_bound(tag, a, b) +// and +// uniform_upper_bound(tag, a, b) +// are used as implementation-details for absl::Uniform(). +// +// Conceptually, +// [a, b] == [uniform_lower_bound(IntervalClosedClosed, a, b), +// uniform_upper_bound(IntervalClosedClosed, a, b)] +// (a, b) == [uniform_lower_bound(IntervalOpenOpen, a, b), +// uniform_upper_bound(IntervalOpenOpen, a, b)] +// [a, b) == [uniform_lower_bound(IntervalClosedOpen, a, b), +// uniform_upper_bound(IntervalClosedOpen, a, b)] +// (a, b] == [uniform_lower_bound(IntervalOpenClosed, a, b), +// uniform_upper_bound(IntervalOpenClosed, a, b)] +// +template <typename IntType, typename Tag> +typename absl::enable_if_t< + absl::conjunction< + std::is_integral<IntType>, + absl::disjunction<std::is_same<Tag, IntervalOpenClosedTag>, + std::is_same<Tag, IntervalOpenOpenTag>>>::value, + IntType> +uniform_lower_bound(Tag, IntType a, IntType) { + return a < (std::numeric_limits<IntType>::max)() ? (a + 1) : a; +} + +template <typename FloatType, typename Tag> +typename absl::enable_if_t< + absl::conjunction< + std::is_floating_point<FloatType>, + absl::disjunction<std::is_same<Tag, IntervalOpenClosedTag>, + std::is_same<Tag, IntervalOpenOpenTag>>>::value, + FloatType> +uniform_lower_bound(Tag, FloatType a, FloatType b) { + return std::nextafter(a, b); +} + +template <typename NumType, typename Tag> +typename absl::enable_if_t< + absl::disjunction<std::is_same<Tag, IntervalClosedClosedTag>, + std::is_same<Tag, IntervalClosedOpenTag>>::value, + NumType> +uniform_lower_bound(Tag, NumType a, NumType) { + return a; +} + +template <typename IntType, typename Tag> +typename absl::enable_if_t< + absl::conjunction< + std::is_integral<IntType>, + absl::disjunction<std::is_same<Tag, IntervalClosedOpenTag>, + std::is_same<Tag, IntervalOpenOpenTag>>>::value, + IntType> +uniform_upper_bound(Tag, IntType, IntType b) { + return b > (std::numeric_limits<IntType>::min)() ? (b - 1) : b; +} + +template <typename FloatType, typename Tag> +typename absl::enable_if_t< + absl::conjunction< + std::is_floating_point<FloatType>, + absl::disjunction<std::is_same<Tag, IntervalClosedOpenTag>, + std::is_same<Tag, IntervalOpenOpenTag>>>::value, + FloatType> +uniform_upper_bound(Tag, FloatType, FloatType b) { + return b; +} + +template <typename IntType, typename Tag> +typename absl::enable_if_t< + absl::conjunction< + std::is_integral<IntType>, + absl::disjunction<std::is_same<Tag, IntervalClosedClosedTag>, + std::is_same<Tag, IntervalOpenClosedTag>>>::value, + IntType> +uniform_upper_bound(Tag, IntType, IntType b) { + return b; +} + +template <typename FloatType, typename Tag> +typename absl::enable_if_t< + absl::conjunction< + std::is_floating_point<FloatType>, + absl::disjunction<std::is_same<Tag, IntervalClosedClosedTag>, + std::is_same<Tag, IntervalOpenClosedTag>>>::value, + FloatType> +uniform_upper_bound(Tag, FloatType, FloatType b) { + return std::nextafter(b, (std::numeric_limits<FloatType>::max)()); +} + +// Returns whether the bounds are valid for the underlying distribution. +// Inputs must have already been resolved via uniform_*_bound calls. +// +// The c++ standard constraints in [rand.dist.uni.int] are listed as: +// requires: lo <= hi. +// +// In the uniform_int_distrubtion, {lo, hi} are closed, closed. Thus: +// [0, 0] is legal. +// [0, 0) is not legal, but [0, 1) is, which translates to [0, 0]. +// (0, 1) is not legal, but (0, 2) is, which translates to [1, 1]. +// (0, 0] is not legal, but (0, 1] is, which translates to [1, 1]. +// +// The c++ standard constraints in [rand.dist.uni.real] are listed as: +// requires: lo <= hi. +// requires: (hi - lo) <= numeric_limits<T>::max() +// +// In the uniform_real_distribution, {lo, hi} are closed, open, Thus: +// [0, 0] is legal, which is [0, 0+epsilon). +// [0, 0) is legal. +// (0, 0) is not legal, but (0-epsilon, 0+epsilon) is. +// (0, 0] is not legal, but (0, 0+epsilon] is. +// +template <typename FloatType> +absl::enable_if_t<std::is_floating_point<FloatType>::value, bool> +is_uniform_range_valid(FloatType a, FloatType b) { + return a <= b && std::isfinite(b - a); +} + +template <typename IntType> +absl::enable_if_t<std::is_integral<IntType>::value, bool> +is_uniform_range_valid(IntType a, IntType b) { + return a <= b; +} + +// UniformDistribution selects either absl::uniform_int_distribution +// or absl::uniform_real_distribution depending on the NumType parameter. +template <typename NumType> +using UniformDistribution = + typename std::conditional<std::is_integral<NumType>::value, + absl::uniform_int_distribution<NumType>, + absl::uniform_real_distribution<NumType>>::type; + +// UniformDistributionWrapper is used as the underlying distribution type +// by the absl::Uniform template function. It selects the proper Abseil +// uniform distribution and provides constructor overloads that match the +// expected parameter order as well as adjusting distribtuion bounds based +// on the tag. +template <typename NumType> +struct UniformDistributionWrapper : public UniformDistribution<NumType> { + template <typename TagType> + explicit UniformDistributionWrapper(TagType, NumType lo, NumType hi) + : UniformDistribution<NumType>( + uniform_lower_bound<NumType>(TagType{}, lo, hi), + uniform_upper_bound<NumType>(TagType{}, lo, hi)) {} + + explicit UniformDistributionWrapper(NumType lo, NumType hi) + : UniformDistribution<NumType>( + uniform_lower_bound<NumType>(IntervalClosedOpenTag(), lo, hi), + uniform_upper_bound<NumType>(IntervalClosedOpenTag(), lo, hi)) {} + + explicit UniformDistributionWrapper() + : UniformDistribution<NumType>(std::numeric_limits<NumType>::lowest(), + (std::numeric_limits<NumType>::max)()) {} +}; + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_UNIFORM_HELPER_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/uniform_helper_test.cc b/third_party/abseil_cpp/absl/random/internal/uniform_helper_test.cc new file mode 100644 index 000000000000..173c49b0b7f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/uniform_helper_test.cc @@ -0,0 +1,279 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/uniform_helper.h" + +#include <cmath> +#include <cstdint> +#include <random> + +#include "gtest/gtest.h" + +namespace { + +using absl::IntervalClosedClosedTag; +using absl::IntervalClosedOpenTag; +using absl::IntervalOpenClosedTag; +using absl::IntervalOpenOpenTag; +using absl::random_internal::uniform_inferred_return_t; +using absl::random_internal::uniform_lower_bound; +using absl::random_internal::uniform_upper_bound; + +class UniformHelperTest : public testing::Test {}; + +TEST_F(UniformHelperTest, UniformBoundFunctionsGeneral) { + constexpr IntervalClosedClosedTag IntervalClosedClosed; + constexpr IntervalClosedOpenTag IntervalClosedOpen; + constexpr IntervalOpenClosedTag IntervalOpenClosed; + constexpr IntervalOpenOpenTag IntervalOpenOpen; + + // absl::uniform_int_distribution natively assumes IntervalClosedClosed + // absl::uniform_real_distribution natively assumes IntervalClosedOpen + + EXPECT_EQ(uniform_lower_bound(IntervalOpenClosed, 0, 100), 1); + EXPECT_EQ(uniform_lower_bound(IntervalOpenOpen, 0, 100), 1); + EXPECT_GT(uniform_lower_bound<float>(IntervalOpenClosed, 0, 1.0), 0); + EXPECT_GT(uniform_lower_bound<float>(IntervalOpenOpen, 0, 1.0), 0); + EXPECT_GT(uniform_lower_bound<double>(IntervalOpenClosed, 0, 1.0), 0); + EXPECT_GT(uniform_lower_bound<double>(IntervalOpenOpen, 0, 1.0), 0); + + EXPECT_EQ(uniform_lower_bound(IntervalClosedClosed, 0, 100), 0); + EXPECT_EQ(uniform_lower_bound(IntervalClosedOpen, 0, 100), 0); + EXPECT_EQ(uniform_lower_bound<float>(IntervalClosedClosed, 0, 1.0), 0); + EXPECT_EQ(uniform_lower_bound<float>(IntervalClosedOpen, 0, 1.0), 0); + EXPECT_EQ(uniform_lower_bound<double>(IntervalClosedClosed, 0, 1.0), 0); + EXPECT_EQ(uniform_lower_bound<double>(IntervalClosedOpen, 0, 1.0), 0); + + EXPECT_EQ(uniform_upper_bound(IntervalOpenOpen, 0, 100), 99); + EXPECT_EQ(uniform_upper_bound(IntervalClosedOpen, 0, 100), 99); + EXPECT_EQ(uniform_upper_bound<float>(IntervalOpenOpen, 0, 1.0), 1.0); + EXPECT_EQ(uniform_upper_bound<float>(IntervalClosedOpen, 0, 1.0), 1.0); + EXPECT_EQ(uniform_upper_bound<double>(IntervalOpenOpen, 0, 1.0), 1.0); + EXPECT_EQ(uniform_upper_bound<double>(IntervalClosedOpen, 0, 1.0), 1.0); + + EXPECT_EQ(uniform_upper_bound(IntervalOpenClosed, 0, 100), 100); + EXPECT_EQ(uniform_upper_bound(IntervalClosedClosed, 0, 100), 100); + EXPECT_GT(uniform_upper_bound<float>(IntervalOpenClosed, 0, 1.0), 1.0); + EXPECT_GT(uniform_upper_bound<float>(IntervalClosedClosed, 0, 1.0), 1.0); + EXPECT_GT(uniform_upper_bound<double>(IntervalOpenClosed, 0, 1.0), 1.0); + EXPECT_GT(uniform_upper_bound<double>(IntervalClosedClosed, 0, 1.0), 1.0); + + // Negative value tests + EXPECT_EQ(uniform_lower_bound(IntervalOpenClosed, -100, -1), -99); + EXPECT_EQ(uniform_lower_bound(IntervalOpenOpen, -100, -1), -99); + EXPECT_GT(uniform_lower_bound<float>(IntervalOpenClosed, -2.0, -1.0), -2.0); + EXPECT_GT(uniform_lower_bound<float>(IntervalOpenOpen, -2.0, -1.0), -2.0); + EXPECT_GT(uniform_lower_bound<double>(IntervalOpenClosed, -2.0, -1.0), -2.0); + EXPECT_GT(uniform_lower_bound<double>(IntervalOpenOpen, -2.0, -1.0), -2.0); + + EXPECT_EQ(uniform_lower_bound(IntervalClosedClosed, -100, -1), -100); + EXPECT_EQ(uniform_lower_bound(IntervalClosedOpen, -100, -1), -100); + EXPECT_EQ(uniform_lower_bound<float>(IntervalClosedClosed, -2.0, -1.0), -2.0); + EXPECT_EQ(uniform_lower_bound<float>(IntervalClosedOpen, -2.0, -1.0), -2.0); + EXPECT_EQ(uniform_lower_bound<double>(IntervalClosedClosed, -2.0, -1.0), + -2.0); + EXPECT_EQ(uniform_lower_bound<double>(IntervalClosedOpen, -2.0, -1.0), -2.0); + + EXPECT_EQ(uniform_upper_bound(IntervalOpenOpen, -100, -1), -2); + EXPECT_EQ(uniform_upper_bound(IntervalClosedOpen, -100, -1), -2); + EXPECT_EQ(uniform_upper_bound<float>(IntervalOpenOpen, -2.0, -1.0), -1.0); + EXPECT_EQ(uniform_upper_bound<float>(IntervalClosedOpen, -2.0, -1.0), -1.0); + EXPECT_EQ(uniform_upper_bound<double>(IntervalOpenOpen, -2.0, -1.0), -1.0); + EXPECT_EQ(uniform_upper_bound<double>(IntervalClosedOpen, -2.0, -1.0), -1.0); + + EXPECT_EQ(uniform_upper_bound(IntervalOpenClosed, -100, -1), -1); + EXPECT_EQ(uniform_upper_bound(IntervalClosedClosed, -100, -1), -1); + EXPECT_GT(uniform_upper_bound<float>(IntervalOpenClosed, -2.0, -1.0), -1.0); + EXPECT_GT(uniform_upper_bound<float>(IntervalClosedClosed, -2.0, -1.0), -1.0); + EXPECT_GT(uniform_upper_bound<double>(IntervalOpenClosed, -2.0, -1.0), -1.0); + EXPECT_GT(uniform_upper_bound<double>(IntervalClosedClosed, -2.0, -1.0), + -1.0); + + EXPECT_GT(uniform_lower_bound(IntervalOpenClosed, 1.0, 2.0), 1.0); + EXPECT_LT(uniform_lower_bound(IntervalOpenClosed, 1.0, +0.0), 1.0); + EXPECT_LT(uniform_lower_bound(IntervalOpenClosed, 1.0, -0.0), 1.0); + EXPECT_LT(uniform_lower_bound(IntervalOpenClosed, 1.0, -1.0), 1.0); +} + +TEST_F(UniformHelperTest, UniformBoundFunctionsIntBounds) { + // Verifies the saturating nature of uniform_lower_bound and + // uniform_upper_bound + constexpr IntervalOpenOpenTag IntervalOpenOpen; + + // uint max. + constexpr auto m = (std::numeric_limits<uint64_t>::max)(); + + EXPECT_EQ(1, uniform_lower_bound(IntervalOpenOpen, 0u, 0u)); + EXPECT_EQ(m, uniform_lower_bound(IntervalOpenOpen, m, m)); + EXPECT_EQ(m, uniform_lower_bound(IntervalOpenOpen, m - 1, m - 1)); + EXPECT_EQ(0, uniform_upper_bound(IntervalOpenOpen, 0u, 0u)); + EXPECT_EQ(m - 1, uniform_upper_bound(IntervalOpenOpen, m, m)); + + // int min/max + constexpr auto l = (std::numeric_limits<int64_t>::min)(); + constexpr auto r = (std::numeric_limits<int64_t>::max)(); + EXPECT_EQ(1, uniform_lower_bound(IntervalOpenOpen, 0, 0)); + EXPECT_EQ(l + 1, uniform_lower_bound(IntervalOpenOpen, l, l)); + EXPECT_EQ(r, uniform_lower_bound(IntervalOpenOpen, r - 1, r - 1)); + EXPECT_EQ(r, uniform_lower_bound(IntervalOpenOpen, r, r)); + EXPECT_EQ(-1, uniform_upper_bound(IntervalOpenOpen, 0, 0)); + EXPECT_EQ(l, uniform_upper_bound(IntervalOpenOpen, l, l)); + EXPECT_EQ(r - 1, uniform_upper_bound(IntervalOpenOpen, r, r)); +} + +TEST_F(UniformHelperTest, UniformBoundFunctionsRealBounds) { + // absl::uniform_real_distribution natively assumes IntervalClosedOpen; + // use the inverse here so each bound has to change. + constexpr IntervalOpenClosedTag IntervalOpenClosed; + + // Edge cases: the next value toward itself is itself. + EXPECT_EQ(1.0, uniform_lower_bound(IntervalOpenClosed, 1.0, 1.0)); + EXPECT_EQ(1.0f, uniform_lower_bound(IntervalOpenClosed, 1.0f, 1.0f)); + + // rightmost and leftmost finite values. + constexpr auto r = (std::numeric_limits<double>::max)(); + const auto re = std::nexttoward(r, 0.0); + constexpr auto l = -r; + const auto le = std::nexttoward(l, 0.0); + + EXPECT_EQ(l, uniform_lower_bound(IntervalOpenClosed, l, l)); // (l,l) + EXPECT_EQ(r, uniform_lower_bound(IntervalOpenClosed, r, r)); // (r,r) + EXPECT_EQ(le, uniform_lower_bound(IntervalOpenClosed, l, r)); // (l,r) + EXPECT_EQ(le, uniform_lower_bound(IntervalOpenClosed, l, 0.0)); // (l, 0) + EXPECT_EQ(le, uniform_lower_bound(IntervalOpenClosed, l, le)); // (l, le) + EXPECT_EQ(r, uniform_lower_bound(IntervalOpenClosed, re, r)); // (re, r) + + EXPECT_EQ(le, uniform_upper_bound(IntervalOpenClosed, l, l)); // (l,l) + EXPECT_EQ(r, uniform_upper_bound(IntervalOpenClosed, r, r)); // (r,r) + EXPECT_EQ(r, uniform_upper_bound(IntervalOpenClosed, l, r)); // (l,r) + EXPECT_EQ(r, uniform_upper_bound(IntervalOpenClosed, l, re)); // (l,re) + EXPECT_EQ(r, uniform_upper_bound(IntervalOpenClosed, 0.0, r)); // (0, r) + EXPECT_EQ(r, uniform_upper_bound(IntervalOpenClosed, re, r)); // (re, r) + EXPECT_EQ(r, uniform_upper_bound(IntervalOpenClosed, le, re)); // (le, re) + + const double e = std::nextafter(1.0, 2.0); // 1 + epsilon + const double f = std::nextafter(1.0, 0.0); // 1 - epsilon + + // (1.0, 1.0 + epsilon) + EXPECT_EQ(e, uniform_lower_bound(IntervalOpenClosed, 1.0, e)); + EXPECT_EQ(std::nextafter(e, 2.0), + uniform_upper_bound(IntervalOpenClosed, 1.0, e)); + + // (1.0-epsilon, 1.0) + EXPECT_EQ(1.0, uniform_lower_bound(IntervalOpenClosed, f, 1.0)); + EXPECT_EQ(e, uniform_upper_bound(IntervalOpenClosed, f, 1.0)); + + // denorm cases. + const double g = std::numeric_limits<double>::denorm_min(); + const double h = std::nextafter(g, 1.0); + + // (0, denorm_min) + EXPECT_EQ(g, uniform_lower_bound(IntervalOpenClosed, 0.0, g)); + EXPECT_EQ(h, uniform_upper_bound(IntervalOpenClosed, 0.0, g)); + + // (denorm_min, 1.0) + EXPECT_EQ(h, uniform_lower_bound(IntervalOpenClosed, g, 1.0)); + EXPECT_EQ(e, uniform_upper_bound(IntervalOpenClosed, g, 1.0)); + + // Edge cases: invalid bounds. + EXPECT_EQ(f, uniform_lower_bound(IntervalOpenClosed, 1.0, -1.0)); +} + +struct Invalid {}; + +template <typename A, typename B> +auto InferredUniformReturnT(int) -> uniform_inferred_return_t<A, B>; + +template <typename, typename> +Invalid InferredUniformReturnT(...); + +// Given types <A, B, Expect>, CheckArgsInferType() verifies that +// +// uniform_inferred_return_t<A, B> and +// uniform_inferred_return_t<B, A> +// +// returns the type "Expect". +// +// This interface can also be used to assert that a given inferred return types +// are invalid. Writing: +// +// CheckArgsInferType<float, int, Invalid>() +// +// will assert that this overload does not exist. +template <typename A, typename B, typename Expect> +void CheckArgsInferType() { + static_assert( + absl::conjunction< + std::is_same<Expect, decltype(InferredUniformReturnT<A, B>(0))>, + std::is_same<Expect, + decltype(InferredUniformReturnT<B, A>(0))>>::value, + ""); +} + +TEST_F(UniformHelperTest, UniformTypeInference) { + // Infers common types. + CheckArgsInferType<uint16_t, uint16_t, uint16_t>(); + CheckArgsInferType<uint32_t, uint32_t, uint32_t>(); + CheckArgsInferType<uint64_t, uint64_t, uint64_t>(); + CheckArgsInferType<int16_t, int16_t, int16_t>(); + CheckArgsInferType<int32_t, int32_t, int32_t>(); + CheckArgsInferType<int64_t, int64_t, int64_t>(); + CheckArgsInferType<float, float, float>(); + CheckArgsInferType<double, double, double>(); + + // Properly promotes uint16_t. + CheckArgsInferType<uint16_t, uint32_t, uint32_t>(); + CheckArgsInferType<uint16_t, uint64_t, uint64_t>(); + CheckArgsInferType<uint16_t, int32_t, int32_t>(); + CheckArgsInferType<uint16_t, int64_t, int64_t>(); + CheckArgsInferType<uint16_t, float, float>(); + CheckArgsInferType<uint16_t, double, double>(); + + // Properly promotes int16_t. + CheckArgsInferType<int16_t, int32_t, int32_t>(); + CheckArgsInferType<int16_t, int64_t, int64_t>(); + CheckArgsInferType<int16_t, float, float>(); + CheckArgsInferType<int16_t, double, double>(); + + // Invalid (u)int16_t-pairings do not compile. + // See "CheckArgsInferType" comments above, for how this is achieved. + CheckArgsInferType<uint16_t, int16_t, Invalid>(); + CheckArgsInferType<int16_t, uint32_t, Invalid>(); + CheckArgsInferType<int16_t, uint64_t, Invalid>(); + + // Properly promotes uint32_t. + CheckArgsInferType<uint32_t, uint64_t, uint64_t>(); + CheckArgsInferType<uint32_t, int64_t, int64_t>(); + CheckArgsInferType<uint32_t, double, double>(); + + // Properly promotes int32_t. + CheckArgsInferType<int32_t, int64_t, int64_t>(); + CheckArgsInferType<int32_t, double, double>(); + + // Invalid (u)int32_t-pairings do not compile. + CheckArgsInferType<uint32_t, int32_t, Invalid>(); + CheckArgsInferType<int32_t, uint64_t, Invalid>(); + CheckArgsInferType<int32_t, float, Invalid>(); + CheckArgsInferType<uint32_t, float, Invalid>(); + + // Invalid (u)int64_t-pairings do not compile. + CheckArgsInferType<uint64_t, int64_t, Invalid>(); + CheckArgsInferType<int64_t, float, Invalid>(); + CheckArgsInferType<int64_t, double, Invalid>(); + + // Properly promotes float. + CheckArgsInferType<float, double, double>(); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/internal/wide_multiply.h b/third_party/abseil_cpp/absl/random/internal/wide_multiply.h new file mode 100644 index 000000000000..0afcbe08e229 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/wide_multiply.h @@ -0,0 +1,111 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_INTERNAL_WIDE_MULTIPLY_H_ +#define ABSL_RANDOM_INTERNAL_WIDE_MULTIPLY_H_ + +#include <cstdint> +#include <limits> +#include <type_traits> + +#if (defined(_WIN32) || defined(_WIN64)) && defined(_M_IA64) +#include <intrin.h> // NOLINT(build/include_order) +#pragma intrinsic(_umul128) +#define ABSL_INTERNAL_USE_UMUL128 1 +#endif + +#include "absl/base/config.h" +#include "absl/base/internal/bits.h" +#include "absl/numeric/int128.h" +#include "absl/random/internal/traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace random_internal { + +// Helper object to multiply two 64-bit values to a 128-bit value. +// MultiplyU64ToU128 multiplies two 64-bit values to a 128-bit value. +// If an intrinsic is available, it is used, otherwise use native 32-bit +// multiplies to construct the result. +inline absl::uint128 MultiplyU64ToU128(uint64_t a, uint64_t b) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return absl::uint128(static_cast<__uint128_t>(a) * b); +#elif defined(ABSL_INTERNAL_USE_UMUL128) + // uint64_t * uint64_t => uint128 multiply using imul intrinsic on MSVC. + uint64_t high = 0; + const uint64_t low = _umul128(a, b, &high); + return absl::MakeUint128(high, low); +#else + // uint128(a) * uint128(b) in emulated mode computes a full 128-bit x 128-bit + // multiply. However there are many cases where that is not necessary, and it + // is only necessary to support a 64-bit x 64-bit = 128-bit multiply. This is + // for those cases. + const uint64_t a00 = static_cast<uint32_t>(a); + const uint64_t a32 = a >> 32; + const uint64_t b00 = static_cast<uint32_t>(b); + const uint64_t b32 = b >> 32; + + const uint64_t c00 = a00 * b00; + const uint64_t c32a = a00 * b32; + const uint64_t c32b = a32 * b00; + const uint64_t c64 = a32 * b32; + + const uint32_t carry = + static_cast<uint32_t>(((c00 >> 32) + static_cast<uint32_t>(c32a) + + static_cast<uint32_t>(c32b)) >> + 32); + + return absl::MakeUint128(c64 + (c32a >> 32) + (c32b >> 32) + carry, + c00 + (c32a << 32) + (c32b << 32)); +#endif +} + +// wide_multiply<T> multiplies two N-bit values to a 2N-bit result. +template <typename UIntType> +struct wide_multiply { + static constexpr size_t kN = std::numeric_limits<UIntType>::digits; + using input_type = UIntType; + using result_type = typename random_internal::unsigned_bits<kN * 2>::type; + + static result_type multiply(input_type a, input_type b) { + return static_cast<result_type>(a) * b; + } + + static input_type hi(result_type r) { return r >> kN; } + static input_type lo(result_type r) { return r; } + + static_assert(std::is_unsigned<UIntType>::value, + "Class-template wide_multiply<> argument must be unsigned."); +}; + +#ifndef ABSL_HAVE_INTRINSIC_INT128 +template <> +struct wide_multiply<uint64_t> { + using input_type = uint64_t; + using result_type = absl::uint128; + + static result_type multiply(uint64_t a, uint64_t b) { + return MultiplyU64ToU128(a, b); + } + + static uint64_t hi(result_type r) { return absl::Uint128High64(r); } + static uint64_t lo(result_type r) { return absl::Uint128Low64(r); } +}; +#endif + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_INTERNAL_WIDE_MULTIPLY_H_ diff --git a/third_party/abseil_cpp/absl/random/internal/wide_multiply_test.cc b/third_party/abseil_cpp/absl/random/internal/wide_multiply_test.cc new file mode 100644 index 000000000000..ca8ce923b710 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/internal/wide_multiply_test.cc @@ -0,0 +1,66 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/internal/wide_multiply.h" + +#include "gtest/gtest.h" +#include "absl/base/internal/bits.h" +#include "absl/numeric/int128.h" + +using absl::random_internal::MultiplyU64ToU128; + +namespace { + +TEST(WideMultiplyTest, MultiplyU64ToU128Test) { + constexpr uint64_t k1 = 1; + constexpr uint64_t kMax = ~static_cast<uint64_t>(0); + + EXPECT_EQ(absl::uint128(0), MultiplyU64ToU128(0, 0)); + + // Max uint64_t + EXPECT_EQ(MultiplyU64ToU128(kMax, kMax), + absl::MakeUint128(0xfffffffffffffffe, 0x0000000000000001)); + EXPECT_EQ(absl::MakeUint128(0, kMax), MultiplyU64ToU128(kMax, 1)); + EXPECT_EQ(absl::MakeUint128(0, kMax), MultiplyU64ToU128(1, kMax)); + for (int i = 0; i < 64; ++i) { + EXPECT_EQ(absl::MakeUint128(0, kMax) << i, + MultiplyU64ToU128(kMax, k1 << i)); + EXPECT_EQ(absl::MakeUint128(0, kMax) << i, + MultiplyU64ToU128(k1 << i, kMax)); + } + + // 1-bit x 1-bit. + for (int i = 0; i < 64; ++i) { + for (int j = 0; j < 64; ++j) { + EXPECT_EQ(absl::MakeUint128(0, 1) << (i + j), + MultiplyU64ToU128(k1 << i, k1 << j)); + EXPECT_EQ(absl::MakeUint128(0, 1) << (i + j), + MultiplyU64ToU128(k1 << i, k1 << j)); + } + } + + // Verified multiplies + EXPECT_EQ(MultiplyU64ToU128(0xffffeeeeddddcccc, 0xbbbbaaaa99998888), + absl::MakeUint128(0xbbbb9e2692c5dddc, 0xc28f7531048d2c60)); + EXPECT_EQ(MultiplyU64ToU128(0x0123456789abcdef, 0xfedcba9876543210), + absl::MakeUint128(0x0121fa00ad77d742, 0x2236d88fe5618cf0)); + EXPECT_EQ(MultiplyU64ToU128(0x0123456789abcdef, 0xfdb97531eca86420), + absl::MakeUint128(0x0120ae99d26725fc, 0xce197f0ecac319e0)); + EXPECT_EQ(MultiplyU64ToU128(0x97a87f4f261ba3f2, 0xfedcba9876543210), + absl::MakeUint128(0x96fbf1a8ae78d0ba, 0x5a6dd4b71f278320)); + EXPECT_EQ(MultiplyU64ToU128(0xfedcba9876543210, 0xfdb97531eca86420), + absl::MakeUint128(0xfc98c6981a413e22, 0x342d0bbf48948200)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/log_uniform_int_distribution.h b/third_party/abseil_cpp/absl/random/log_uniform_int_distribution.h new file mode 100644 index 000000000000..960816e2f8ff --- /dev/null +++ b/third_party/abseil_cpp/absl/random/log_uniform_int_distribution.h @@ -0,0 +1,254 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_LOG_UNIFORM_INT_DISTRIBUTION_H_ +#define ABSL_RANDOM_LOG_UNIFORM_INT_DISTRIBUTION_H_ + +#include <algorithm> +#include <cassert> +#include <cmath> +#include <istream> +#include <limits> +#include <ostream> +#include <type_traits> + +#include "absl/random/internal/fastmath.h" +#include "absl/random/internal/generate_real.h" +#include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/internal/traits.h" +#include "absl/random/uniform_int_distribution.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// log_uniform_int_distribution: +// +// Returns a random variate R in range [min, max] such that +// floor(log(R-min, base)) is uniformly distributed. +// We ensure uniformity by discretization using the +// boundary sets [0, 1, base, base * base, ... min(base*n, max)] +// +template <typename IntType = int> +class log_uniform_int_distribution { + private: + using unsigned_type = + typename random_internal::make_unsigned_bits<IntType>::type; + + public: + using result_type = IntType; + + class param_type { + public: + using distribution_type = log_uniform_int_distribution; + + explicit param_type( + result_type min = 0, + result_type max = (std::numeric_limits<result_type>::max)(), + result_type base = 2) + : min_(min), + max_(max), + base_(base), + range_(static_cast<unsigned_type>(max_) - + static_cast<unsigned_type>(min_)), + log_range_(0) { + assert(max_ >= min_); + assert(base_ > 1); + + if (base_ == 2) { + // Determine where the first set bit is on range(), giving a log2(range) + // value which can be used to construct bounds. + log_range_ = (std::min)(random_internal::LeadingSetBit(range()), + std::numeric_limits<unsigned_type>::digits); + } else { + // NOTE: Computing the logN(x) introduces error from 2 sources: + // 1. Conversion of int to double loses precision for values >= + // 2^53, which may cause some log() computations to operate on + // different values. + // 2. The error introduced by the division will cause the result + // to differ from the expected value. + // + // Thus a result which should equal K may equal K +/- epsilon, + // which can eliminate some values depending on where the bounds fall. + const double inv_log_base = 1.0 / std::log(base_); + const double log_range = std::log(static_cast<double>(range()) + 0.5); + log_range_ = static_cast<int>(std::ceil(inv_log_base * log_range)); + } + } + + result_type(min)() const { return min_; } + result_type(max)() const { return max_; } + result_type base() const { return base_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.min_ == b.min_ && a.max_ == b.max_ && a.base_ == b.base_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class log_uniform_int_distribution; + + int log_range() const { return log_range_; } + unsigned_type range() const { return range_; } + + result_type min_; + result_type max_; + result_type base_; + unsigned_type range_; // max - min + int log_range_; // ceil(logN(range_)) + + static_assert(std::is_integral<IntType>::value, + "Class-template absl::log_uniform_int_distribution<> must be " + "parameterized using an integral type."); + }; + + log_uniform_int_distribution() : log_uniform_int_distribution(0) {} + + explicit log_uniform_int_distribution( + result_type min, + result_type max = (std::numeric_limits<result_type>::max)(), + result_type base = 2) + : param_(min, max, base) {} + + explicit log_uniform_int_distribution(const param_type& p) : param_(p) {} + + void reset() {} + + // generating functions + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p) { + return (p.min)() + Generate(g, p); + } + + result_type(min)() const { return (param_.min)(); } + result_type(max)() const { return (param_.max)(); } + result_type base() const { return param_.base(); } + + param_type param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + friend bool operator==(const log_uniform_int_distribution& a, + const log_uniform_int_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const log_uniform_int_distribution& a, + const log_uniform_int_distribution& b) { + return a.param_ != b.param_; + } + + private: + // Returns a log-uniform variate in the range [0, p.range()]. The caller + // should add min() to shift the result to the correct range. + template <typename URNG> + unsigned_type Generate(URNG& g, // NOLINT(runtime/references) + const param_type& p); + + param_type param_; +}; + +template <typename IntType> +template <typename URBG> +typename log_uniform_int_distribution<IntType>::unsigned_type +log_uniform_int_distribution<IntType>::Generate( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + // sample e over [0, log_range]. Map the results of e to this: + // 0 => 0 + // 1 => [1, b-1] + // 2 => [b, (b^2)-1] + // n => [b^(n-1)..(b^n)-1] + const int e = absl::uniform_int_distribution<int>(0, p.log_range())(g); + if (e == 0) { + return 0; + } + const int d = e - 1; + + unsigned_type base_e, top_e; + if (p.base() == 2) { + base_e = static_cast<unsigned_type>(1) << d; + + top_e = (e >= std::numeric_limits<unsigned_type>::digits) + ? (std::numeric_limits<unsigned_type>::max)() + : (static_cast<unsigned_type>(1) << e) - 1; + } else { + const double r = std::pow(p.base(), d); + const double s = (r * p.base()) - 1.0; + + base_e = + (r > static_cast<double>((std::numeric_limits<unsigned_type>::max)())) + ? (std::numeric_limits<unsigned_type>::max)() + : static_cast<unsigned_type>(r); + + top_e = + (s > static_cast<double>((std::numeric_limits<unsigned_type>::max)())) + ? (std::numeric_limits<unsigned_type>::max)() + : static_cast<unsigned_type>(s); + } + + const unsigned_type lo = (base_e >= p.range()) ? p.range() : base_e; + const unsigned_type hi = (top_e >= p.range()) ? p.range() : top_e; + + // choose uniformly over [lo, hi] + return absl::uniform_int_distribution<result_type>(lo, hi)(g); +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const log_uniform_int_distribution<IntType>& x) { + using stream_type = + typename random_internal::stream_format_type<IntType>::type; + auto saver = random_internal::make_ostream_state_saver(os); + os << static_cast<stream_type>((x.min)()) << os.fill() + << static_cast<stream_type>((x.max)()) << os.fill() + << static_cast<stream_type>(x.base()); + return os; +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + log_uniform_int_distribution<IntType>& x) { // NOLINT(runtime/references) + using param_type = typename log_uniform_int_distribution<IntType>::param_type; + using result_type = + typename log_uniform_int_distribution<IntType>::result_type; + using stream_type = + typename random_internal::stream_format_type<IntType>::type; + + stream_type min; + stream_type max; + stream_type base; + + auto saver = random_internal::make_istream_state_saver(is); + is >> min >> max >> base; + if (!is.fail()) { + x.param(param_type(static_cast<result_type>(min), + static_cast<result_type>(max), + static_cast<result_type>(base))); + } + return is; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_LOG_UNIFORM_INT_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/log_uniform_int_distribution_test.cc b/third_party/abseil_cpp/absl/random/log_uniform_int_distribution_test.cc new file mode 100644 index 000000000000..5e780d96d306 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/log_uniform_int_distribution_test.cc @@ -0,0 +1,280 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/log_uniform_int_distribution.h" + +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <random> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/strip.h" + +namespace { + +template <typename IntType> +class LogUniformIntDistributionTypeTest : public ::testing::Test {}; + +using IntTypes = ::testing::Types<int8_t, int16_t, int32_t, int64_t, // + uint8_t, uint16_t, uint32_t, uint64_t>; +TYPED_TEST_CASE(LogUniformIntDistributionTypeTest, IntTypes); + +TYPED_TEST(LogUniformIntDistributionTypeTest, SerializeTest) { + using param_type = + typename absl::log_uniform_int_distribution<TypeParam>::param_type; + using Limits = std::numeric_limits<TypeParam>; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + for (const auto& param : { + param_type(0, 1), // + param_type(0, 2), // + param_type(0, 2, 10), // + param_type(9, 32, 4), // + param_type(1, 101, 10), // + param_type(1, Limits::max() / 2), // + param_type(0, Limits::max() - 1), // + param_type(0, Limits::max(), 2), // + param_type(0, Limits::max(), 10), // + param_type(Limits::min(), 0), // + param_type(Limits::lowest(), Limits::max()), // + param_type(Limits::min(), Limits::max()), // + }) { + // Validate parameters. + const auto min = param.min(); + const auto max = param.max(); + const auto base = param.base(); + absl::log_uniform_int_distribution<TypeParam> before(min, max, base); + EXPECT_EQ(before.min(), param.min()); + EXPECT_EQ(before.max(), param.max()); + EXPECT_EQ(before.base(), param.base()); + + { + absl::log_uniform_int_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + } + + // Validate stream serialization. + std::stringstream ss; + ss << before; + + absl::log_uniform_int_distribution<TypeParam> after(3, 6, 17); + + EXPECT_NE(before.max(), after.max()); + EXPECT_NE(before.base(), after.base()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before.min(), after.min()); + EXPECT_EQ(before.max(), after.max()); + EXPECT_EQ(before.base(), after.base()); + EXPECT_EQ(before.param(), after.param()); + EXPECT_EQ(before, after); + + // Smoke test. + auto sample_min = after.max(); + auto sample_max = after.min(); + for (int i = 0; i < kCount; i++) { + auto sample = after(gen); + EXPECT_GE(sample, after.min()); + EXPECT_LE(sample, after.max()); + if (sample > sample_max) sample_max = sample; + if (sample < sample_min) sample_min = sample; + } + ABSL_INTERNAL_LOG(INFO, + absl::StrCat("Range: ", +sample_min, ", ", +sample_max)); + } +} + +using log_uniform_i32 = absl::log_uniform_int_distribution<int32_t>; + +class LogUniformIntChiSquaredTest + : public testing::TestWithParam<log_uniform_i32::param_type> { + public: + // The ChiSquaredTestImpl provides a chi-squared goodness of fit test for + // data generated by the log-uniform-int distribution. + double ChiSquaredTestImpl(); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng_{0x2B7E151628AED2A6}; +}; + +double LogUniformIntChiSquaredTest::ChiSquaredTestImpl() { + using absl::random_internal::kChiSquared; + + const auto& param = GetParam(); + + // Check the distribution of L=log(log_uniform_int_distribution, base), + // expecting that L is roughly uniformly distributed, that is: + // + // P[L=0] ~= P[L=1] ~= ... ~= P[L=log(max)] + // + // For a total of X entries, each bucket should contain some number of samples + // in the interval [X/k - a, X/k + a]. + // + // Where `a` is approximately sqrt(X/k). This is validated by bucketing + // according to the log function and using a chi-squared test for uniformity. + + const bool is_2 = (param.base() == 2); + const double base_log = 1.0 / std::log(param.base()); + const auto bucket_index = [base_log, is_2, ¶m](int32_t x) { + uint64_t y = static_cast<uint64_t>(x) - param.min(); + return (y == 0) ? 0 + : is_2 ? static_cast<int>(1 + std::log2(y)) + : static_cast<int>(1 + std::log(y) * base_log); + }; + const int max_bucket = bucket_index(param.max()); // inclusive + const size_t trials = 15 + (max_bucket + 1) * 10; + + log_uniform_i32 dist(param); + + std::vector<int64_t> buckets(max_bucket + 1); + for (size_t i = 0; i < trials; ++i) { + const auto sample = dist(rng_); + // Check the bounds. + ABSL_ASSERT(sample <= dist.max()); + ABSL_ASSERT(sample >= dist.min()); + // Convert the output of the generator to one of num_bucket buckets. + int bucket = bucket_index(sample); + ABSL_ASSERT(bucket <= max_bucket); + ++buckets[bucket]; + } + + // The null-hypothesis is that the distribution is uniform with respect to + // log-uniform-int bucketization. + const int dof = buckets.size() - 1; + const double expected = trials / static_cast<double>(buckets.size()); + + const double threshold = absl::random_internal::ChiSquareValue(dof, 0.98); + + double chi_square = absl::random_internal::ChiSquareWithExpected( + std::begin(buckets), std::end(buckets), expected); + + const double p = absl::random_internal::ChiSquarePValue(chi_square, dof); + + if (chi_square > threshold) { + ABSL_INTERNAL_LOG(INFO, "values"); + for (size_t i = 0; i < buckets.size(); i++) { + ABSL_INTERNAL_LOG(INFO, absl::StrCat(i, ": ", buckets[i])); + } + ABSL_INTERNAL_LOG(INFO, + absl::StrFormat("trials=%d\n" + "%s(data, %d) = %f (%f)\n" + "%s @ 0.98 = %f", + trials, kChiSquared, dof, chi_square, p, + kChiSquared, threshold)); + } + return p; +} + +TEST_P(LogUniformIntChiSquaredTest, MultiTest) { + const int kTrials = 5; + int failures = 0; + for (int i = 0; i < kTrials; i++) { + double p_value = ChiSquaredTestImpl(); + if (p_value < 0.005) { + failures++; + } + } + + // There is a 0.10% chance of producing at least one failure, so raise the + // failure threshold high enough to allow for a flake rate < 10,000. + EXPECT_LE(failures, 4); +} + +// Generate the parameters for the test. +std::vector<log_uniform_i32::param_type> GenParams() { + using Param = log_uniform_i32::param_type; + using Limits = std::numeric_limits<int32_t>; + + return std::vector<Param>{ + Param{0, 1, 2}, + Param{1, 1, 2}, + Param{0, 2, 2}, + Param{0, 3, 2}, + Param{0, 4, 2}, + Param{0, 9, 10}, + Param{0, 10, 10}, + Param{0, 11, 10}, + Param{1, 10, 10}, + Param{0, (1 << 8) - 1, 2}, + Param{0, (1 << 8), 2}, + Param{0, (1 << 30) - 1, 2}, + Param{-1000, 1000, 10}, + Param{0, Limits::max(), 2}, + Param{0, Limits::max(), 3}, + Param{0, Limits::max(), 10}, + Param{Limits::min(), 0}, + Param{Limits::min(), Limits::max(), 2}, + }; +} + +std::string ParamName( + const ::testing::TestParamInfo<log_uniform_i32::param_type>& info) { + const auto& p = info.param; + std::string name = + absl::StrCat("min_", p.min(), "__max_", p.max(), "__base_", p.base()); + return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); +} + +INSTANTIATE_TEST_SUITE_P(All, LogUniformIntChiSquaredTest, + ::testing::ValuesIn(GenParams()), ParamName); + +// NOTE: absl::log_uniform_int_distribution is not guaranteed to be stable. +TEST(LogUniformIntDistributionTest, StabilityTest) { + using testing::ElementsAre; + // absl::uniform_int_distribution stability relies on + // absl::random_internal::LeadingSetBit, std::log, std::pow. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(6); + + { + absl::log_uniform_int_distribution<int32_t> dist(0, 256); + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + EXPECT_THAT(output, ElementsAre(256, 66, 4, 6, 57, 103)); + } + urbg.reset(); + { + absl::log_uniform_int_distribution<int32_t> dist(0, 256, 10); + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + EXPECT_THAT(output, ElementsAre(8, 4, 0, 0, 0, 69)); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/mock_distributions.h b/third_party/abseil_cpp/absl/random/mock_distributions.h new file mode 100644 index 000000000000..764ab370abeb --- /dev/null +++ b/third_party/abseil_cpp/absl/random/mock_distributions.h @@ -0,0 +1,266 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: mock_distributions.h +// ----------------------------------------------------------------------------- +// +// This file contains mock distribution functions for use alongside an +// `absl::MockingBitGen` object within the Googletest testing framework. Such +// mocks are useful to provide deterministic values as return values within +// (otherwise random) Abseil distribution functions. +// +// The return type of each function is a mock expectation object which +// is used to set the match result. +// +// More information about the Googletest testing framework is available at +// https://github.com/google/googletest +// +// EXPECT_CALL and ON_CALL need to be made within the same DLL component as +// the call to absl::Uniform and related methods, otherwise mocking will fail +// since the underlying implementation creates a type-specific pointer which +// will be distinct across different DLL boundaries. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockUniform<int>(), Call(mock, 1, 1000)) +// .WillRepeatedly(testing::ReturnRoundRobin({20, 40})); +// +// EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000), 20); +// EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000), 40); +// EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000), 20); +// EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000), 40); + +#ifndef ABSL_RANDOM_MOCK_DISTRIBUTIONS_H_ +#define ABSL_RANDOM_MOCK_DISTRIBUTIONS_H_ + +#include <limits> +#include <type_traits> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" +#include "absl/random/distributions.h" +#include "absl/random/internal/mock_overload_set.h" +#include "absl/random/mocking_bit_gen.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// absl::MockUniform +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Uniform. +// +// `absl::MockUniform` is a class template used in conjunction with Googletest's +// `ON_CALL()` and `EXPECT_CALL()` macros. To use it, default-construct an +// instance of it inside `ON_CALL()` or `EXPECT_CALL()`, and use `Call(...)` the +// same way one would define mocks on a Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockUniform<uint32_t>(), Call(mock)) +// .WillOnce(Return(123456)); +// auto x = absl::Uniform<uint32_t>(mock); +// assert(x == 123456) +// +template <typename R> +using MockUniform = random_internal::MockOverloadSet< + random_internal::UniformDistributionWrapper<R>, + R(IntervalClosedOpenTag, MockingBitGen&, R, R), + R(IntervalClosedClosedTag, MockingBitGen&, R, R), + R(IntervalOpenOpenTag, MockingBitGen&, R, R), + R(IntervalOpenClosedTag, MockingBitGen&, R, R), R(MockingBitGen&, R, R), + R(MockingBitGen&)>; + +// ----------------------------------------------------------------------------- +// absl::MockBernoulli +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Bernoulli. +// +// `absl::MockBernoulli` is a class used in conjunction with Googletest's +// `ON_CALL()` and `EXPECT_CALL()` macros. To use it, default-construct an +// instance of it inside `ON_CALL()` or `EXPECT_CALL()`, and use `Call(...)` the +// same way one would define mocks on a Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockBernoulli(), Call(mock, testing::_)) +// .WillOnce(Return(false)); +// assert(absl::Bernoulli(mock, 0.5) == false); +// +using MockBernoulli = + random_internal::MockOverloadSet<absl::bernoulli_distribution, + bool(MockingBitGen&, double)>; + +// ----------------------------------------------------------------------------- +// absl::MockBeta +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Beta. +// +// `absl::MockBeta` is a class used in conjunction with Googletest's `ON_CALL()` +// and `EXPECT_CALL()` macros. To use it, default-construct an instance of it +// inside `ON_CALL()` or `EXPECT_CALL()`, and use `Call(...)` the same way one +// would define mocks on a Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockBeta(), Call(mock, 3.0, 2.0)) +// .WillOnce(Return(0.567)); +// auto x = absl::Beta<double>(mock, 3.0, 2.0); +// assert(x == 0.567); +// +template <typename RealType> +using MockBeta = + random_internal::MockOverloadSet<absl::beta_distribution<RealType>, + RealType(MockingBitGen&, RealType, + RealType)>; + +// ----------------------------------------------------------------------------- +// absl::MockExponential +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Exponential. +// +// `absl::MockExponential` is a class template used in conjunction with +// Googletest's `ON_CALL()` and `EXPECT_CALL()` macros. To use it, +// default-construct an instance of it inside `ON_CALL()` or `EXPECT_CALL()`, +// and use `Call(...)` the same way one would define mocks on a +// Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockExponential<double>(), Call(mock, 0.5)) +// .WillOnce(Return(12.3456789)); +// auto x = absl::Exponential<double>(mock, 0.5); +// assert(x == 12.3456789) +// +template <typename RealType> +using MockExponential = + random_internal::MockOverloadSet<absl::exponential_distribution<RealType>, + RealType(MockingBitGen&, RealType)>; + +// ----------------------------------------------------------------------------- +// absl::MockGaussian +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Gaussian. +// +// `absl::MockGaussian` is a class template used in conjunction with +// Googletest's `ON_CALL()` and `EXPECT_CALL()` macros. To use it, +// default-construct an instance of it inside `ON_CALL()` or `EXPECT_CALL()`, +// and use `Call(...)` the same way one would define mocks on a +// Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockGaussian<double>(), Call(mock, 16.3, 3.3)) +// .WillOnce(Return(12.3456789)); +// auto x = absl::Gaussian<double>(mock, 16.3, 3.3); +// assert(x == 12.3456789) +// +template <typename RealType> +using MockGaussian = + random_internal::MockOverloadSet<absl::gaussian_distribution<RealType>, + RealType(MockingBitGen&, RealType, + RealType)>; + +// ----------------------------------------------------------------------------- +// absl::MockLogUniform +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::LogUniform. +// +// `absl::MockLogUniform` is a class template used in conjunction with +// Googletest's `ON_CALL()` and `EXPECT_CALL()` macros. To use it, +// default-construct an instance of it inside `ON_CALL()` or `EXPECT_CALL()`, +// and use `Call(...)` the same way one would define mocks on a +// Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockLogUniform<int>(), Call(mock, 10, 10000, 10)) +// .WillOnce(Return(1221)); +// auto x = absl::LogUniform<int>(mock, 10, 10000, 10); +// assert(x == 1221) +// +template <typename IntType> +using MockLogUniform = random_internal::MockOverloadSet< + absl::log_uniform_int_distribution<IntType>, + IntType(MockingBitGen&, IntType, IntType, IntType)>; + +// ----------------------------------------------------------------------------- +// absl::MockPoisson +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Poisson. +// +// `absl::MockPoisson` is a class template used in conjunction with Googletest's +// `ON_CALL()` and `EXPECT_CALL()` macros. To use it, default-construct an +// instance of it inside `ON_CALL()` or `EXPECT_CALL()`, and use `Call(...)` the +// same way one would define mocks on a Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockPoisson<int>(), Call(mock, 2.0)) +// .WillOnce(Return(1221)); +// auto x = absl::Poisson<int>(mock, 2.0); +// assert(x == 1221) +// +template <typename IntType> +using MockPoisson = + random_internal::MockOverloadSet<absl::poisson_distribution<IntType>, + IntType(MockingBitGen&, double)>; + +// ----------------------------------------------------------------------------- +// absl::MockZipf +// ----------------------------------------------------------------------------- +// +// Matches calls to absl::Zipf. +// +// `absl::MockZipf` is a class template used in conjunction with Googletest's +// `ON_CALL()` and `EXPECT_CALL()` macros. To use it, default-construct an +// instance of it inside `ON_CALL()` or `EXPECT_CALL()`, and use `Call(...)` the +// same way one would define mocks on a Googletest `MockFunction()`. +// +// Example: +// +// absl::MockingBitGen mock; +// EXPECT_CALL(absl::MockZipf<int>(), Call(mock, 1000000, 2.0, 1.0)) +// .WillOnce(Return(1221)); +// auto x = absl::Zipf<int>(mock, 1000000, 2.0, 1.0); +// assert(x == 1221) +// +template <typename IntType> +using MockZipf = + random_internal::MockOverloadSet<absl::zipf_distribution<IntType>, + IntType(MockingBitGen&, IntType, double, + double)>; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_MOCK_DISTRIBUTIONS_H_ diff --git a/third_party/abseil_cpp/absl/random/mock_distributions_test.cc b/third_party/abseil_cpp/absl/random/mock_distributions_test.cc new file mode 100644 index 000000000000..de23bafe1e55 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/mock_distributions_test.cc @@ -0,0 +1,72 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/mock_distributions.h" + +#include "gtest/gtest.h" +#include "absl/random/mocking_bit_gen.h" +#include "absl/random/random.h" + +namespace { +using ::testing::Return; + +TEST(MockDistributions, Examples) { + absl::MockingBitGen gen; + + EXPECT_NE(absl::Uniform<int>(gen, 1, 1000000), 20); + EXPECT_CALL(absl::MockUniform<int>(), Call(gen, 1, 1000000)) + .WillOnce(Return(20)); + EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000000), 20); + + EXPECT_NE(absl::Uniform<double>(gen, 0.0, 100.0), 5.0); + EXPECT_CALL(absl::MockUniform<double>(), Call(gen, 0.0, 100.0)) + .WillOnce(Return(5.0)); + EXPECT_EQ(absl::Uniform<double>(gen, 0.0, 100.0), 5.0); + + EXPECT_NE(absl::Exponential<double>(gen, 1.0), 42); + EXPECT_CALL(absl::MockExponential<double>(), Call(gen, 1.0)) + .WillOnce(Return(42)); + EXPECT_EQ(absl::Exponential<double>(gen, 1.0), 42); + + EXPECT_NE(absl::Poisson<int>(gen, 1.0), 500); + EXPECT_CALL(absl::MockPoisson<int>(), Call(gen, 1.0)).WillOnce(Return(500)); + EXPECT_EQ(absl::Poisson<int>(gen, 1.0), 500); + + EXPECT_NE(absl::Bernoulli(gen, 0.000001), true); + EXPECT_CALL(absl::MockBernoulli(), Call(gen, 0.000001)) + .WillOnce(Return(true)); + EXPECT_EQ(absl::Bernoulli(gen, 0.000001), true); + + EXPECT_NE(absl::Beta<double>(gen, 3.0, 2.0), 0.567); + EXPECT_CALL(absl::MockBeta<double>(), Call(gen, 3.0, 2.0)) + .WillOnce(Return(0.567)); + EXPECT_EQ(absl::Beta<double>(gen, 3.0, 2.0), 0.567); + + EXPECT_NE(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + EXPECT_CALL(absl::MockZipf<int>(), Call(gen, 1000000, 2.0, 1.0)) + .WillOnce(Return(1221)); + EXPECT_EQ(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + + EXPECT_NE(absl::Gaussian<double>(gen, 0.0, 1.0), 0.001); + EXPECT_CALL(absl::MockGaussian<double>(), Call(gen, 0.0, 1.0)) + .WillOnce(Return(0.001)); + EXPECT_EQ(absl::Gaussian<double>(gen, 0.0, 1.0), 0.001); + + EXPECT_NE(absl::LogUniform<int>(gen, 0, 1000000, 2), 2040); + EXPECT_CALL(absl::MockLogUniform<int>(), Call(gen, 0, 1000000, 2)) + .WillOnce(Return(2040)); + EXPECT_EQ(absl::LogUniform<int>(gen, 0, 1000000, 2), 2040); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/mocking_bit_gen.h b/third_party/abseil_cpp/absl/random/mocking_bit_gen.h new file mode 100644 index 000000000000..6d2f2c836245 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/mocking_bit_gen.h @@ -0,0 +1,228 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// mocking_bit_gen.h +// ----------------------------------------------------------------------------- +// +// This file includes an `absl::MockingBitGen` class to use as a mock within the +// Googletest testing framework. Such a mock is useful to provide deterministic +// values as return values within (otherwise random) Abseil distribution +// functions. Such determinism within a mock is useful within testing frameworks +// to test otherwise indeterminate APIs. +// +// More information about the Googletest testing framework is available at +// https://github.com/google/googletest + +#ifndef ABSL_RANDOM_MOCKING_BIT_GEN_H_ +#define ABSL_RANDOM_MOCKING_BIT_GEN_H_ + +#include <iterator> +#include <limits> +#include <memory> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/fast_type_id.h" +#include "absl/container/flat_hash_map.h" +#include "absl/meta/type_traits.h" +#include "absl/random/distributions.h" +#include "absl/random/internal/distribution_caller.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/types/span.h" +#include "absl/types/variant.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace random_internal { +template <typename> +struct DistributionCaller; +class MockHelpers; + +} // namespace random_internal +class BitGenRef; + +// MockingBitGen +// +// `absl::MockingBitGen` is a mock Uniform Random Bit Generator (URBG) class +// which can act in place of an `absl::BitGen` URBG within tests using the +// Googletest testing framework. +// +// Usage: +// +// Use an `absl::MockingBitGen` along with a mock distribution object (within +// mock_distributions.h) inside Googletest constructs such as ON_CALL(), +// EXPECT_TRUE(), etc. to produce deterministic results conforming to the +// distribution's API contract. +// +// Example: +// +// // Mock a call to an `absl::Bernoulli` distribution using Googletest +// absl::MockingBitGen bitgen; +// +// ON_CALL(absl::MockBernoulli(), Call(bitgen, 0.5)) +// .WillByDefault(testing::Return(true)); +// EXPECT_TRUE(absl::Bernoulli(bitgen, 0.5)); +// +// // Mock a call to an `absl::Uniform` distribution within Googletest +// absl::MockingBitGen bitgen; +// +// ON_CALL(absl::MockUniform<int>(), Call(bitgen, testing::_, testing::_)) +// .WillByDefault([] (int low, int high) { +// return (low + high) / 2; +// }); +// +// EXPECT_EQ(absl::Uniform<int>(gen, 0, 10), 5); +// EXPECT_EQ(absl::Uniform<int>(gen, 30, 40), 35); +// +// At this time, only mock distributions supplied within the Abseil random +// library are officially supported. +// +// EXPECT_CALL and ON_CALL need to be made within the same DLL component as +// the call to absl::Uniform and related methods, otherwise mocking will fail +// since the underlying implementation creates a type-specific pointer which +// will be distinct across different DLL boundaries. +// +class MockingBitGen { + public: + MockingBitGen() = default; + + ~MockingBitGen() { + for (const auto& del : deleters_) del(); + } + + // URBG interface + using result_type = absl::BitGen::result_type; + + static constexpr result_type(min)() { return (absl::BitGen::min)(); } + static constexpr result_type(max)() { return (absl::BitGen::max)(); } + result_type operator()() { return gen_(); } + + private: + using match_impl_fn = void (*)(void* mock_fn, void* t_erased_arg_tuple, + void* t_erased_result); + + struct MockData { + void* mock_fn = nullptr; + match_impl_fn match_impl = nullptr; + }; + + // GetMockFnType returns the testing::MockFunction for a result and tuple. + // This method only exists for type deduction and is otherwise unimplemented. + template <typename ResultT, typename... Args> + static auto GetMockFnType(ResultT, std::tuple<Args...>) + -> ::testing::MockFunction<ResultT(Args...)>; + + // MockFnCaller is a helper method for use with absl::apply to + // apply an ArgTupleT to a compatible MockFunction. + // NOTE: MockFnCaller is essentially equivalent to the lambda: + // [fn](auto... args) { return fn->Call(std::move(args)...)} + // however that fails to build on some supported platforms. + template <typename ResultT, typename MockFnType, typename Tuple> + struct MockFnCaller; + // specialization for std::tuple. + template <typename ResultT, typename MockFnType, typename... Args> + struct MockFnCaller<ResultT, MockFnType, std::tuple<Args...>> { + MockFnType* fn; + inline ResultT operator()(Args... args) { + return fn->Call(std::move(args)...); + } + }; + + // MockingBitGen::RegisterMock + // + // RegisterMock<ResultT, ArgTupleT>(FastTypeIdType) is the main extension + // point for extending the MockingBitGen framework. It provides a mechanism to + // install a mock expectation for a function like ResultT(Args...) keyed by + // type_idex onto the MockingBitGen context. The key is that the type_index + // used to register must match the type index used to call the mock. + // + // The returned MockFunction<...> type can be used to setup additional + // distribution parameters of the expectation. + template <typename ResultT, typename ArgTupleT> + auto RegisterMock(base_internal::FastTypeIdType type) + -> decltype(GetMockFnType(std::declval<ResultT>(), + std::declval<ArgTupleT>()))& { + using MockFnType = decltype( + GetMockFnType(std::declval<ResultT>(), std::declval<ArgTupleT>())); + auto& mock = mocks_[type]; + if (!mock.mock_fn) { + auto* mock_fn = new MockFnType; + mock.mock_fn = mock_fn; + mock.match_impl = &MatchImpl<ResultT, ArgTupleT>; + deleters_.emplace_back([mock_fn] { delete mock_fn; }); + } + return *static_cast<MockFnType*>(mock.mock_fn); + } + + // MockingBitGen::MatchImpl<> is a dispatch function which converts the + // generic type-erased parameters into a specific mock invocation call. + // Requires tuple_args to point to a ArgTupleT, which is a std::tuple<Args...> + // used to invoke the mock function. + // Requires result to point to a ResultT, which is the result of the call. + template <typename ResultT, typename ArgTupleT> + static void MatchImpl(/*MockFnType<ResultT, Args...>*/ void* mock_fn, + /*ArgTupleT*/ void* args_tuple, + /*ResultT*/ void* result) { + using MockFnType = decltype( + GetMockFnType(std::declval<ResultT>(), std::declval<ArgTupleT>())); + *static_cast<ResultT*>(result) = absl::apply( + MockFnCaller<ResultT, MockFnType, ArgTupleT>{ + static_cast<MockFnType*>(mock_fn)}, + *static_cast<ArgTupleT*>(args_tuple)); + } + + // MockingBitGen::InvokeMock + // + // InvokeMock(FastTypeIdType, args, result) is the entrypoint for invoking + // mocks registered on MockingBitGen. + // + // When no mocks are registered on the provided FastTypeIdType, returns false. + // Otherwise attempts to invoke the mock function ResultT(Args...) that + // was previously registered via the type_index. + // Requires tuple_args to point to a ArgTupleT, which is a std::tuple<Args...> + // used to invoke the mock function. + // Requires result to point to a ResultT, which is the result of the call. + inline bool InvokeMock(base_internal::FastTypeIdType type, void* args_tuple, + void* result) { + // Trigger a mock, if there exists one that matches `param`. + auto it = mocks_.find(type); + if (it == mocks_.end()) return false; + auto* mock_data = static_cast<MockData*>(&it->second); + mock_data->match_impl(mock_data->mock_fn, args_tuple, result); + return true; + } + + absl::flat_hash_map<base_internal::FastTypeIdType, MockData> mocks_; + std::vector<std::function<void()>> deleters_; + absl::BitGen gen_; + + template <typename> + friend struct ::absl::random_internal::DistributionCaller; // for InvokeMock + friend class ::absl::BitGenRef; // for InvokeMock + friend class ::absl::random_internal::MockHelpers; // for RegisterMock, + // InvokeMock +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_MOCKING_BIT_GEN_H_ diff --git a/third_party/abseil_cpp/absl/random/mocking_bit_gen_test.cc b/third_party/abseil_cpp/absl/random/mocking_bit_gen_test.cc new file mode 100644 index 000000000000..f0ffc9ac9283 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/mocking_bit_gen_test.cc @@ -0,0 +1,347 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#include "absl/random/mocking_bit_gen.h" + +#include <numeric> +#include <random> + +#include "gmock/gmock.h" +#include "gtest/gtest-spi.h" +#include "gtest/gtest.h" +#include "absl/random/bit_gen_ref.h" +#include "absl/random/mock_distributions.h" +#include "absl/random/random.h" + +namespace { +using ::testing::Ne; +using ::testing::Return; + +TEST(BasicMocking, AllDistributionsAreOverridable) { + absl::MockingBitGen gen; + + EXPECT_NE(absl::Uniform<int>(gen, 1, 1000000), 20); + EXPECT_CALL(absl::MockUniform<int>(), Call(gen, 1, 1000000)) + .WillOnce(Return(20)); + EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000000), 20); + + EXPECT_NE(absl::Uniform<double>(gen, 0.0, 100.0), 5.0); + EXPECT_CALL(absl::MockUniform<double>(), Call(gen, 0.0, 100.0)) + .WillOnce(Return(5.0)); + EXPECT_EQ(absl::Uniform<double>(gen, 0.0, 100.0), 5.0); + + EXPECT_NE(absl::Exponential<double>(gen, 1.0), 42); + EXPECT_CALL(absl::MockExponential<double>(), Call(gen, 1.0)) + .WillOnce(Return(42)); + EXPECT_EQ(absl::Exponential<double>(gen, 1.0), 42); + + EXPECT_NE(absl::Poisson<int>(gen, 1.0), 500); + EXPECT_CALL(absl::MockPoisson<int>(), Call(gen, 1.0)).WillOnce(Return(500)); + EXPECT_EQ(absl::Poisson<int>(gen, 1.0), 500); + + EXPECT_NE(absl::Bernoulli(gen, 0.000001), true); + EXPECT_CALL(absl::MockBernoulli(), Call(gen, 0.000001)) + .WillOnce(Return(true)); + EXPECT_EQ(absl::Bernoulli(gen, 0.000001), true); + + EXPECT_NE(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + EXPECT_CALL(absl::MockZipf<int>(), Call(gen, 1000000, 2.0, 1.0)) + .WillOnce(Return(1221)); + EXPECT_EQ(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + + EXPECT_NE(absl::Gaussian<double>(gen, 0.0, 1.0), 0.001); + EXPECT_CALL(absl::MockGaussian<double>(), Call(gen, 0.0, 1.0)) + .WillOnce(Return(0.001)); + EXPECT_EQ(absl::Gaussian<double>(gen, 0.0, 1.0), 0.001); + + EXPECT_NE(absl::LogUniform<int>(gen, 0, 1000000, 2), 500000); + EXPECT_CALL(absl::MockLogUniform<int>(), Call(gen, 0, 1000000, 2)) + .WillOnce(Return(500000)); + EXPECT_EQ(absl::LogUniform<int>(gen, 0, 1000000, 2), 500000); +} + +TEST(BasicMocking, OnDistribution) { + absl::MockingBitGen gen; + + EXPECT_NE(absl::Uniform<int>(gen, 1, 1000000), 20); + ON_CALL(absl::MockUniform<int>(), Call(gen, 1, 1000000)) + .WillByDefault(Return(20)); + EXPECT_EQ(absl::Uniform<int>(gen, 1, 1000000), 20); + + EXPECT_NE(absl::Uniform<double>(gen, 0.0, 100.0), 5.0); + ON_CALL(absl::MockUniform<double>(), Call(gen, 0.0, 100.0)) + .WillByDefault(Return(5.0)); + EXPECT_EQ(absl::Uniform<double>(gen, 0.0, 100.0), 5.0); + + EXPECT_NE(absl::Exponential<double>(gen, 1.0), 42); + ON_CALL(absl::MockExponential<double>(), Call(gen, 1.0)) + .WillByDefault(Return(42)); + EXPECT_EQ(absl::Exponential<double>(gen, 1.0), 42); + + EXPECT_NE(absl::Poisson<int>(gen, 1.0), 500); + ON_CALL(absl::MockPoisson<int>(), Call(gen, 1.0)).WillByDefault(Return(500)); + EXPECT_EQ(absl::Poisson<int>(gen, 1.0), 500); + + EXPECT_NE(absl::Bernoulli(gen, 0.000001), true); + ON_CALL(absl::MockBernoulli(), Call(gen, 0.000001)) + .WillByDefault(Return(true)); + EXPECT_EQ(absl::Bernoulli(gen, 0.000001), true); + + EXPECT_NE(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + ON_CALL(absl::MockZipf<int>(), Call(gen, 1000000, 2.0, 1.0)) + .WillByDefault(Return(1221)); + EXPECT_EQ(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + + EXPECT_NE(absl::Gaussian<double>(gen, 0.0, 1.0), 0.001); + ON_CALL(absl::MockGaussian<double>(), Call(gen, 0.0, 1.0)) + .WillByDefault(Return(0.001)); + EXPECT_EQ(absl::Gaussian<double>(gen, 0.0, 1.0), 0.001); + + EXPECT_NE(absl::LogUniform<int>(gen, 0, 1000000, 2), 2040); + ON_CALL(absl::MockLogUniform<int>(), Call(gen, 0, 1000000, 2)) + .WillByDefault(Return(2040)); + EXPECT_EQ(absl::LogUniform<int>(gen, 0, 1000000, 2), 2040); +} + +TEST(BasicMocking, GMockMatchers) { + absl::MockingBitGen gen; + + EXPECT_NE(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); + ON_CALL(absl::MockZipf<int>(), Call(gen, 1000000, 2.0, 1.0)) + .WillByDefault(Return(1221)); + EXPECT_EQ(absl::Zipf<int>(gen, 1000000, 2.0, 1.0), 1221); +} + +TEST(BasicMocking, OverridesWithMultipleGMockExpectations) { + absl::MockingBitGen gen; + + EXPECT_CALL(absl::MockUniform<int>(), Call(gen, 1, 10000)) + .WillOnce(Return(20)) + .WillOnce(Return(40)) + .WillOnce(Return(60)); + EXPECT_EQ(absl::Uniform(gen, 1, 10000), 20); + EXPECT_EQ(absl::Uniform(gen, 1, 10000), 40); + EXPECT_EQ(absl::Uniform(gen, 1, 10000), 60); +} + +TEST(BasicMocking, DefaultArgument) { + absl::MockingBitGen gen; + + ON_CALL(absl::MockExponential<double>(), Call(gen, 1.0)) + .WillByDefault(Return(200)); + + EXPECT_EQ(absl::Exponential<double>(gen), 200); + EXPECT_EQ(absl::Exponential<double>(gen, 1.0), 200); +} + +TEST(BasicMocking, MultipleGenerators) { + auto get_value = [](absl::BitGenRef gen_ref) { + return absl::Uniform(gen_ref, 1, 1000000); + }; + absl::MockingBitGen unmocked_generator; + absl::MockingBitGen mocked_with_3; + absl::MockingBitGen mocked_with_11; + + EXPECT_CALL(absl::MockUniform<int>(), Call(mocked_with_3, 1, 1000000)) + .WillOnce(Return(3)) + .WillRepeatedly(Return(17)); + EXPECT_CALL(absl::MockUniform<int>(), Call(mocked_with_11, 1, 1000000)) + .WillOnce(Return(11)) + .WillRepeatedly(Return(17)); + + // Ensure that unmocked generator generates neither value. + int unmocked_value = get_value(unmocked_generator); + EXPECT_NE(unmocked_value, 3); + EXPECT_NE(unmocked_value, 11); + // Mocked generators should generate their mocked values. + EXPECT_EQ(get_value(mocked_with_3), 3); + EXPECT_EQ(get_value(mocked_with_11), 11); + // Ensure that the mocks have expired. + EXPECT_NE(get_value(mocked_with_3), 3); + EXPECT_NE(get_value(mocked_with_11), 11); +} + +TEST(BasicMocking, MocksNotTrigeredForIncorrectTypes) { + absl::MockingBitGen gen; + EXPECT_CALL(absl::MockUniform<uint32_t>(), Call(gen)).WillOnce(Return(42)); + + EXPECT_NE(absl::Uniform<uint16_t>(gen), 42); // Not mocked + EXPECT_EQ(absl::Uniform<uint32_t>(gen), 42); // Mock triggered +} + +TEST(BasicMocking, FailsOnUnsatisfiedMocks) { + EXPECT_NONFATAL_FAILURE( + []() { + absl::MockingBitGen gen; + EXPECT_CALL(absl::MockExponential<double>(), Call(gen, 1.0)) + .WillOnce(Return(3.0)); + // Does not call absl::Exponential(). + }(), + "unsatisfied and active"); +} + +TEST(OnUniform, RespectsUniformIntervalSemantics) { + absl::MockingBitGen gen; + + EXPECT_CALL(absl::MockUniform<int>(), + Call(absl::IntervalClosed, gen, 1, 1000000)) + .WillOnce(Return(301)); + EXPECT_NE(absl::Uniform(gen, 1, 1000000), 301); // Not mocked + EXPECT_EQ(absl::Uniform(absl::IntervalClosed, gen, 1, 1000000), 301); +} + +TEST(OnUniform, RespectsNoArgUnsignedShorthand) { + absl::MockingBitGen gen; + EXPECT_CALL(absl::MockUniform<uint32_t>(), Call(gen)).WillOnce(Return(42)); + EXPECT_EQ(absl::Uniform<uint32_t>(gen), 42); +} + +TEST(RepeatedlyModifier, ForceSnakeEyesForManyDice) { + auto roll_some_dice = [](absl::BitGenRef gen_ref) { + std::vector<int> results(16); + for (auto& r : results) { + r = absl::Uniform(absl::IntervalClosed, gen_ref, 1, 6); + } + return results; + }; + std::vector<int> results; + absl::MockingBitGen gen; + + // Without any mocked calls, not all dice roll a "6". + results = roll_some_dice(gen); + EXPECT_LT(std::accumulate(std::begin(results), std::end(results), 0), + results.size() * 6); + + // Verify that we can force all "6"-rolls, with mocking. + ON_CALL(absl::MockUniform<int>(), Call(absl::IntervalClosed, gen, 1, 6)) + .WillByDefault(Return(6)); + results = roll_some_dice(gen); + EXPECT_EQ(std::accumulate(std::begin(results), std::end(results), 0), + results.size() * 6); +} + +TEST(WillOnce, DistinctCounters) { + absl::MockingBitGen gen; + EXPECT_CALL(absl::MockUniform<int>(), Call(gen, 1, 1000000)) + .Times(3) + .WillRepeatedly(Return(0)); + EXPECT_CALL(absl::MockUniform<int>(), Call(gen, 1000001, 2000000)) + .Times(3) + .WillRepeatedly(Return(1)); + EXPECT_EQ(absl::Uniform(gen, 1000001, 2000000), 1); + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 0); + EXPECT_EQ(absl::Uniform(gen, 1000001, 2000000), 1); + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 0); + EXPECT_EQ(absl::Uniform(gen, 1000001, 2000000), 1); + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 0); +} + +TEST(TimesModifier, ModifierSaturatesAndExpires) { + EXPECT_NONFATAL_FAILURE( + []() { + absl::MockingBitGen gen; + EXPECT_CALL(absl::MockUniform<int>(), Call(gen, 1, 1000000)) + .Times(3) + .WillRepeatedly(Return(15)) + .RetiresOnSaturation(); + + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 15); + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 15); + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 15); + // Times(3) has expired - Should get a different value now. + + EXPECT_NE(absl::Uniform(gen, 1, 1000000), 15); + }(), + ""); +} + +TEST(TimesModifier, Times0) { + absl::MockingBitGen gen; + EXPECT_CALL(absl::MockBernoulli(), Call(gen, 0.0)).Times(0); + EXPECT_CALL(absl::MockPoisson<int>(), Call(gen, 1.0)).Times(0); +} + +TEST(AnythingMatcher, MatchesAnyArgument) { + using testing::_; + + { + absl::MockingBitGen gen; + ON_CALL(absl::MockUniform<int>(), Call(absl::IntervalClosed, gen, _, 1000)) + .WillByDefault(Return(11)); + ON_CALL(absl::MockUniform<int>(), + Call(absl::IntervalClosed, gen, _, Ne(1000))) + .WillByDefault(Return(99)); + + EXPECT_EQ(absl::Uniform(absl::IntervalClosed, gen, 10, 1000000), 99); + EXPECT_EQ(absl::Uniform(absl::IntervalClosed, gen, 10, 1000), 11); + } + + { + absl::MockingBitGen gen; + ON_CALL(absl::MockUniform<int>(), Call(gen, 1, _)) + .WillByDefault(Return(25)); + ON_CALL(absl::MockUniform<int>(), Call(gen, Ne(1), _)) + .WillByDefault(Return(99)); + EXPECT_EQ(absl::Uniform(gen, 3, 1000000), 99); + EXPECT_EQ(absl::Uniform(gen, 1, 1000000), 25); + } + + { + absl::MockingBitGen gen; + ON_CALL(absl::MockUniform<int>(), Call(gen, _, _)) + .WillByDefault(Return(145)); + EXPECT_EQ(absl::Uniform(gen, 1, 1000), 145); + EXPECT_EQ(absl::Uniform(gen, 10, 1000), 145); + EXPECT_EQ(absl::Uniform(gen, 100, 1000), 145); + } +} + +TEST(AnythingMatcher, WithWillByDefault) { + using testing::_; + absl::MockingBitGen gen; + std::vector<int> values = {11, 22, 33, 44, 55, 66, 77, 88, 99, 1010}; + + ON_CALL(absl::MockUniform<size_t>(), Call(gen, 0, _)) + .WillByDefault(Return(0)); + for (int i = 0; i < 100; i++) { + auto& elem = values[absl::Uniform(gen, 0u, values.size())]; + EXPECT_EQ(elem, 11); + } +} + +TEST(BasicMocking, WillByDefaultWithArgs) { + using testing::_; + + absl::MockingBitGen gen; + ON_CALL(absl::MockPoisson<int>(), Call(gen, _)) + .WillByDefault( + [](double lambda) { return static_cast<int>(lambda * 10); }); + EXPECT_EQ(absl::Poisson<int>(gen, 1.7), 17); + EXPECT_EQ(absl::Poisson<int>(gen, 0.03), 0); +} + +TEST(MockingBitGen, InSequenceSucceedsInOrder) { + absl::MockingBitGen gen; + + testing::InSequence seq; + + EXPECT_CALL(absl::MockPoisson<int>(), Call(gen, 1.0)).WillOnce(Return(3)); + EXPECT_CALL(absl::MockPoisson<int>(), Call(gen, 2.0)).WillOnce(Return(4)); + + EXPECT_EQ(absl::Poisson<int>(gen, 1.0), 3); + EXPECT_EQ(absl::Poisson<int>(gen, 2.0), 4); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/poisson_distribution.h b/third_party/abseil_cpp/absl/random/poisson_distribution.h new file mode 100644 index 000000000000..cb5f5d5d0ff7 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/poisson_distribution.h @@ -0,0 +1,258 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_POISSON_DISTRIBUTION_H_ +#define ABSL_RANDOM_POISSON_DISTRIBUTION_H_ + +#include <cassert> +#include <cmath> +#include <istream> +#include <limits> +#include <ostream> +#include <type_traits> + +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/fastmath.h" +#include "absl/random/internal/generate_real.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::poisson_distribution: +// Generates discrete variates conforming to a Poisson distribution. +// p(n) = (mean^n / n!) exp(-mean) +// +// Depending on the parameter, the distribution selects one of the following +// algorithms: +// * The standard algorithm, attributed to Knuth, extended using a split method +// for larger values +// * The "Ratio of Uniforms as a convenient method for sampling from classical +// discrete distributions", Stadlober, 1989. +// http://www.sciencedirect.com/science/article/pii/0377042790903495 +// +// NOTE: param_type.mean() is a double, which permits values larger than +// poisson_distribution<IntType>::max(), however this should be avoided and +// the distribution results are limited to the max() value. +// +// The goals of this implementation are to provide good performance while still +// beig thread-safe: This limits the implementation to not using lgamma provided +// by <math.h>. +// +template <typename IntType = int> +class poisson_distribution { + public: + using result_type = IntType; + + class param_type { + public: + using distribution_type = poisson_distribution; + explicit param_type(double mean = 1.0); + + double mean() const { return mean_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.mean_ == b.mean_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class poisson_distribution; + + double mean_; + double emu_; // e ^ -mean_ + double lmu_; // ln(mean_) + double s_; + double log_k_; + int split_; + + static_assert(std::is_integral<IntType>::value, + "Class-template absl::poisson_distribution<> must be " + "parameterized using an integral type."); + }; + + poisson_distribution() : poisson_distribution(1.0) {} + + explicit poisson_distribution(double mean) : param_(mean) {} + + explicit poisson_distribution(const param_type& p) : param_(p) {} + + void reset() {} + + // generating functions + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + param_type param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + result_type(min)() const { return 0; } + result_type(max)() const { return (std::numeric_limits<result_type>::max)(); } + + double mean() const { return param_.mean(); } + + friend bool operator==(const poisson_distribution& a, + const poisson_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const poisson_distribution& a, + const poisson_distribution& b) { + return a.param_ != b.param_; + } + + private: + param_type param_; + random_internal::FastUniformBits<uint64_t> fast_u64_; +}; + +// ----------------------------------------------------------------------------- +// Implementation details follow +// ----------------------------------------------------------------------------- + +template <typename IntType> +poisson_distribution<IntType>::param_type::param_type(double mean) + : mean_(mean), split_(0) { + assert(mean >= 0); + assert(mean <= (std::numeric_limits<result_type>::max)()); + // As a defensive measure, avoid large values of the mean. The rejection + // algorithm used does not support very large values well. It my be worth + // changing algorithms to better deal with these cases. + assert(mean <= 1e10); + if (mean_ < 10) { + // For small lambda, use the knuth method. + split_ = 1; + emu_ = std::exp(-mean_); + } else if (mean_ <= 50) { + // Use split-knuth method. + split_ = 1 + static_cast<int>(mean_ / 10.0); + emu_ = std::exp(-mean_ / static_cast<double>(split_)); + } else { + // Use ratio of uniforms method. + constexpr double k2E = 0.7357588823428846; + constexpr double kSA = 0.4494580810294493; + + lmu_ = std::log(mean_); + double a = mean_ + 0.5; + s_ = kSA + std::sqrt(k2E * a); + const double mode = std::ceil(mean_) - 1; + log_k_ = lmu_ * mode - absl::random_internal::StirlingLogFactorial(mode); + } +} + +template <typename IntType> +template <typename URBG> +typename poisson_distribution<IntType>::result_type +poisson_distribution<IntType>::operator()( + URBG& g, // NOLINT(runtime/references) + const param_type& p) { + using random_internal::GeneratePositiveTag; + using random_internal::GenerateRealFromBits; + using random_internal::GenerateSignedTag; + + if (p.split_ != 0) { + // Use Knuth's algorithm with range splitting to avoid floating-point + // errors. Knuth's algorithm is: Ui is a sequence of uniform variates on + // (0,1); return the number of variates required for product(Ui) < + // exp(-lambda). + // + // The expected number of variates required for Knuth's method can be + // computed as follows: + // The expected value of U is 0.5, so solving for 0.5^n < exp(-lambda) gives + // the expected number of uniform variates + // required for a given lambda, which is: + // lambda = [2, 5, 9, 10, 11, 12, 13, 14, 15, 16, 17] + // n = [3, 8, 13, 15, 16, 18, 19, 21, 22, 24, 25] + // + result_type n = 0; + for (int split = p.split_; split > 0; --split) { + double r = 1.0; + do { + r *= GenerateRealFromBits<double, GeneratePositiveTag, true>( + fast_u64_(g)); // U(-1, 0) + ++n; + } while (r > p.emu_); + --n; + } + return n; + } + + // Use ratio of uniforms method. + // + // Let u ~ Uniform(0, 1), v ~ Uniform(-1, 1), + // a = lambda + 1/2, + // s = 1.5 - sqrt(3/e) + sqrt(2(lambda + 1/2)/e), + // x = s * v/u + a. + // P(floor(x) = k | u^2 < f(floor(x))/k), where + // f(m) = lambda^m exp(-lambda)/ m!, for 0 <= m, and f(m) = 0 otherwise, + // and k = max(f). + const double a = p.mean_ + 0.5; + for (;;) { + const double u = GenerateRealFromBits<double, GeneratePositiveTag, false>( + fast_u64_(g)); // U(0, 1) + const double v = GenerateRealFromBits<double, GenerateSignedTag, false>( + fast_u64_(g)); // U(-1, 1) + + const double x = std::floor(p.s_ * v / u + a); + if (x < 0) continue; // f(negative) = 0 + const double rhs = x * p.lmu_; + // clang-format off + double s = (x <= 1.0) ? 0.0 + : (x == 2.0) ? 0.693147180559945 + : absl::random_internal::StirlingLogFactorial(x); + // clang-format on + const double lhs = 2.0 * std::log(u) + p.log_k_ + s; + if (lhs < rhs) { + return x > (max)() ? (max)() + : static_cast<result_type>(x); // f(x)/k >= u^2 + } + } +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const poisson_distribution<IntType>& x) { + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<double>::kPrecision); + os << x.mean(); + return os; +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + poisson_distribution<IntType>& x) { // NOLINT(runtime/references) + using param_type = typename poisson_distribution<IntType>::param_type; + + auto saver = random_internal::make_istream_state_saver(is); + double mean = random_internal::read_floating_point<double>(is); + if (!is.fail()) { + x.param(param_type(mean)); + } + return is; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_POISSON_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/poisson_distribution_test.cc b/third_party/abseil_cpp/absl/random/poisson_distribution_test.cc new file mode 100644 index 000000000000..8baabd111892 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/poisson_distribution_test.cc @@ -0,0 +1,573 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/poisson_distribution.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <random> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/container/flat_hash_map.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/strip.h" + +// Notes about generating poisson variates: +// +// It is unlikely that any implementation of std::poisson_distribution +// will be stable over time and across library implementations. For instance +// the three different poisson variate generators listed below all differ: +// +// https://github.com/ampl/gsl/tree/master/randist/poisson.c +// * GSL uses a gamma + binomial + knuth method to compute poisson variates. +// +// https://github.com/gcc-mirror/gcc/blob/master/libstdc%2B%2B-v3/include/bits/random.tcc +// * GCC uses the Devroye rejection algorithm, based on +// Devroye, L. Non-Uniform Random Variates Generation. Springer-Verlag, +// New York, 1986, Ch. X, Sects. 3.3 & 3.4 (+ Errata!), ~p.511 +// http://www.nrbook.com/devroye/ +// +// https://github.com/llvm-mirror/libcxx/blob/master/include/random +// * CLANG uses a different rejection method, which appears to include a +// normal-distribution approximation and an exponential distribution to +// compute the threshold, including a similar factorial approximation to this +// one, but it is unclear where the algorithm comes from, exactly. +// + +namespace { + +using absl::random_internal::kChiSquared; + +// The PoissonDistributionInterfaceTest provides a basic test that +// absl::poisson_distribution conforms to the interface and serialization +// requirements imposed by [rand.req.dist] for the common integer types. + +template <typename IntType> +class PoissonDistributionInterfaceTest : public ::testing::Test {}; + +using IntTypes = ::testing::Types<int, int8_t, int16_t, int32_t, int64_t, + uint8_t, uint16_t, uint32_t, uint64_t>; +TYPED_TEST_CASE(PoissonDistributionInterfaceTest, IntTypes); + +TYPED_TEST(PoissonDistributionInterfaceTest, SerializeTest) { + using param_type = typename absl::poisson_distribution<TypeParam>::param_type; + const double kMax = + std::min(1e10 /* assertion limit */, + static_cast<double>(std::numeric_limits<TypeParam>::max())); + + const double kParams[] = { + // Cases around 1. + 1, // + std::nextafter(1.0, 0.0), // 1 - epsilon + std::nextafter(1.0, 2.0), // 1 + epsilon + // Arbitrary values. + 1e-8, 1e-4, + 0.0000005, // ~7.2e-7 + 0.2, // ~0.2x + 0.5, // 0.72 + 2, // ~2.8 + 20, // 3x ~9.6 + 100, 1e4, 1e8, 1.5e9, 1e20, + // Boundary cases. + std::numeric_limits<double>::max(), + std::numeric_limits<double>::epsilon(), + std::nextafter(std::numeric_limits<double>::min(), + 1.0), // min + epsilon + std::numeric_limits<double>::min(), // smallest normal + std::numeric_limits<double>::denorm_min(), // smallest denorm + std::numeric_limits<double>::min() / 2, // denorm + std::nextafter(std::numeric_limits<double>::min(), + 0.0), // denorm_max + }; + + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + for (const double m : kParams) { + const double mean = std::min(kMax, m); + const param_type param(mean); + + // Validate parameters. + absl::poisson_distribution<TypeParam> before(mean); + EXPECT_EQ(before.mean(), param.mean()); + + { + absl::poisson_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + EXPECT_EQ(via_param.param(), before.param()); + } + + // Smoke test. + auto sample_min = before.max(); + auto sample_max = before.min(); + for (int i = 0; i < kCount; i++) { + auto sample = before(gen); + EXPECT_GE(sample, before.min()); + EXPECT_LE(sample, before.max()); + if (sample > sample_max) sample_max = sample; + if (sample < sample_min) sample_min = sample; + } + + ABSL_INTERNAL_LOG(INFO, absl::StrCat("Range {", param.mean(), "}: ", + +sample_min, ", ", +sample_max)); + + // Validate stream serialization. + std::stringstream ss; + ss << before; + + absl::poisson_distribution<TypeParam> after(3.8); + + EXPECT_NE(before.mean(), after.mean()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before.mean(), after.mean()) // + << ss.str() << " " // + << (ss.good() ? "good " : "") // + << (ss.bad() ? "bad " : "") // + << (ss.eof() ? "eof " : "") // + << (ss.fail() ? "fail " : ""); + } +} + +// See http://www.itl.nist.gov/div898/handbook/eda/section3/eda366j.htm + +class PoissonModel { + public: + explicit PoissonModel(double mean) : mean_(mean) {} + + double mean() const { return mean_; } + double variance() const { return mean_; } + double stddev() const { return std::sqrt(variance()); } + double skew() const { return 1.0 / mean_; } + double kurtosis() const { return 3.0 + 1.0 / mean_; } + + // InitCDF() initializes the CDF for the distribution parameters. + void InitCDF(); + + // The InverseCDF, or the Percent-point function returns x, P(x) < v. + struct CDF { + size_t index; + double pmf; + double cdf; + }; + CDF InverseCDF(double p) { + CDF target{0, 0, p}; + auto it = std::upper_bound( + std::begin(cdf_), std::end(cdf_), target, + [](const CDF& a, const CDF& b) { return a.cdf < b.cdf; }); + return *it; + } + + void LogCDF() { + ABSL_INTERNAL_LOG(INFO, absl::StrCat("CDF (mean = ", mean_, ")")); + for (const auto c : cdf_) { + ABSL_INTERNAL_LOG(INFO, + absl::StrCat(c.index, ": pmf=", c.pmf, " cdf=", c.cdf)); + } + } + + private: + const double mean_; + + std::vector<CDF> cdf_; +}; + +// The goal is to compute an InverseCDF function, or percent point function for +// the poisson distribution, and use that to partition our output into equal +// range buckets. However there is no closed form solution for the inverse cdf +// for poisson distributions (the closest is the incomplete gamma function). +// Instead, `InitCDF` iteratively computes the PMF and the CDF. This enables +// searching for the bucket points. +void PoissonModel::InitCDF() { + if (!cdf_.empty()) { + // State already initialized. + return; + } + ABSL_ASSERT(mean_ < 201.0); + + const size_t max_i = 50 * stddev() + mean(); + const double e_neg_mean = std::exp(-mean()); + ABSL_ASSERT(e_neg_mean > 0); + + double d = 1; + double last_result = e_neg_mean; + double cumulative = e_neg_mean; + if (e_neg_mean > 1e-10) { + cdf_.push_back({0, e_neg_mean, cumulative}); + } + for (size_t i = 1; i < max_i; i++) { + d *= (mean() / i); + double result = e_neg_mean * d; + cumulative += result; + if (result < 1e-10 && result < last_result && cumulative > 0.999999) { + break; + } + if (result > 1e-7) { + cdf_.push_back({i, result, cumulative}); + } + last_result = result; + } + ABSL_ASSERT(!cdf_.empty()); +} + +// PoissonDistributionZTest implements a z-test for the poisson distribution. + +struct ZParam { + double mean; + double p_fail; // Z-Test probability of failure. + int trials; // Z-Test trials. + size_t samples; // Z-Test samples. +}; + +class PoissonDistributionZTest : public testing::TestWithParam<ZParam>, + public PoissonModel { + public: + PoissonDistributionZTest() : PoissonModel(GetParam().mean) {} + + // ZTestImpl provides a basic z-squared test of the mean vs. expected + // mean for data generated by the poisson distribution. + template <typename D> + bool SingleZTest(const double p, const size_t samples); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng_{0x2B7E151628AED2A6}; +}; + +template <typename D> +bool PoissonDistributionZTest::SingleZTest(const double p, + const size_t samples) { + D dis(mean()); + + absl::flat_hash_map<int32_t, int> buckets; + std::vector<double> data; + data.reserve(samples); + for (int j = 0; j < samples; j++) { + const auto x = dis(rng_); + buckets[x]++; + data.push_back(x); + } + + // The null-hypothesis is that the distribution is a poisson distribution with + // the provided mean (not estimated from the data). + const auto m = absl::random_internal::ComputeDistributionMoments(data); + const double max_err = absl::random_internal::MaxErrorTolerance(p); + const double z = absl::random_internal::ZScore(mean(), m); + const bool pass = absl::random_internal::Near("z", z, 0.0, max_err); + + if (!pass) { + ABSL_INTERNAL_LOG( + INFO, absl::StrFormat("p=%f max_err=%f\n" + " mean=%f vs. %f\n" + " stddev=%f vs. %f\n" + " skewness=%f vs. %f\n" + " kurtosis=%f vs. %f\n" + " z=%f", + p, max_err, m.mean, mean(), std::sqrt(m.variance), + stddev(), m.skewness, skew(), m.kurtosis, + kurtosis(), z)); + } + return pass; +} + +TEST_P(PoissonDistributionZTest, AbslPoissonDistribution) { + const auto& param = GetParam(); + const int expected_failures = + std::max(1, static_cast<int>(std::ceil(param.trials * param.p_fail))); + const double p = absl::random_internal::RequiredSuccessProbability( + param.p_fail, param.trials); + + int failures = 0; + for (int i = 0; i < param.trials; i++) { + failures += + SingleZTest<absl::poisson_distribution<int32_t>>(p, param.samples) ? 0 + : 1; + } + EXPECT_LE(failures, expected_failures); +} + +std::vector<ZParam> GetZParams() { + // These values have been adjusted from the "exact" computed values to reduce + // failure rates. + // + // It turns out that the actual values are not as close to the expected values + // as would be ideal. + return std::vector<ZParam>({ + // Knuth method. + ZParam{0.5, 0.01, 100, 1000}, + ZParam{1.0, 0.01, 100, 1000}, + ZParam{10.0, 0.01, 100, 5000}, + // Split-knuth method. + ZParam{20.0, 0.01, 100, 10000}, + ZParam{50.0, 0.01, 100, 10000}, + // Ratio of gaussians method. + ZParam{51.0, 0.01, 100, 10000}, + ZParam{200.0, 0.05, 10, 100000}, + ZParam{100000.0, 0.05, 10, 1000000}, + }); +} + +std::string ZParamName(const ::testing::TestParamInfo<ZParam>& info) { + const auto& p = info.param; + std::string name = absl::StrCat("mean_", absl::SixDigits(p.mean)); + return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); +} + +INSTANTIATE_TEST_SUITE_P(All, PoissonDistributionZTest, + ::testing::ValuesIn(GetZParams()), ZParamName); + +// The PoissonDistributionChiSquaredTest class provides a basic test framework +// for variates generated by a conforming poisson_distribution. +class PoissonDistributionChiSquaredTest : public testing::TestWithParam<double>, + public PoissonModel { + public: + PoissonDistributionChiSquaredTest() : PoissonModel(GetParam()) {} + + // The ChiSquaredTestImpl provides a chi-squared goodness of fit test for data + // generated by the poisson distribution. + template <typename D> + double ChiSquaredTestImpl(); + + private: + void InitChiSquaredTest(const double buckets); + + std::vector<size_t> cutoffs_; + std::vector<double> expected_; + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng_{0x2B7E151628AED2A6}; +}; + +void PoissonDistributionChiSquaredTest::InitChiSquaredTest( + const double buckets) { + if (!cutoffs_.empty() && !expected_.empty()) { + return; + } + InitCDF(); + + // The code below finds cuttoffs that yield approximately equally-sized + // buckets to the extent that it is possible. However for poisson + // distributions this is particularly challenging for small mean parameters. + // Track the expected proportion of items in each bucket. + double last_cdf = 0; + const double inc = 1.0 / buckets; + for (double p = inc; p <= 1.0; p += inc) { + auto result = InverseCDF(p); + if (!cutoffs_.empty() && cutoffs_.back() == result.index) { + continue; + } + double d = result.cdf - last_cdf; + cutoffs_.push_back(result.index); + expected_.push_back(d); + last_cdf = result.cdf; + } + cutoffs_.push_back(std::numeric_limits<size_t>::max()); + expected_.push_back(std::max(0.0, 1.0 - last_cdf)); +} + +template <typename D> +double PoissonDistributionChiSquaredTest::ChiSquaredTestImpl() { + const int kSamples = 2000; + const int kBuckets = 50; + + // The poisson CDF fails for large mean values, since e^-mean exceeds the + // machine precision. For these cases, using a normal approximation would be + // appropriate. + ABSL_ASSERT(mean() <= 200); + InitChiSquaredTest(kBuckets); + + D dis(mean()); + + std::vector<int32_t> counts(cutoffs_.size(), 0); + for (int j = 0; j < kSamples; j++) { + const size_t x = dis(rng_); + auto it = std::lower_bound(std::begin(cutoffs_), std::end(cutoffs_), x); + counts[std::distance(cutoffs_.begin(), it)]++; + } + + // Normalize the counts. + std::vector<int32_t> e(expected_.size(), 0); + for (int i = 0; i < e.size(); i++) { + e[i] = kSamples * expected_[i]; + } + + // The null-hypothesis is that the distribution is a poisson distribution with + // the provided mean (not estimated from the data). + const int dof = static_cast<int>(counts.size()) - 1; + + // The threshold for logging is 1-in-50. + const double threshold = absl::random_internal::ChiSquareValue(dof, 0.98); + + const double chi_square = absl::random_internal::ChiSquare( + std::begin(counts), std::end(counts), std::begin(e), std::end(e)); + + const double p = absl::random_internal::ChiSquarePValue(chi_square, dof); + + // Log if the chi_squared value is above the threshold. + if (chi_square > threshold) { + LogCDF(); + + ABSL_INTERNAL_LOG(INFO, absl::StrCat("VALUES buckets=", counts.size(), + " samples=", kSamples)); + for (size_t i = 0; i < counts.size(); i++) { + ABSL_INTERNAL_LOG( + INFO, absl::StrCat(cutoffs_[i], ": ", counts[i], " vs. E=", e[i])); + } + + ABSL_INTERNAL_LOG( + INFO, + absl::StrCat(kChiSquared, "(data, dof=", dof, ") = ", chi_square, " (", + p, ")\n", " vs.\n", kChiSquared, " @ 0.98 = ", threshold)); + } + return p; +} + +TEST_P(PoissonDistributionChiSquaredTest, AbslPoissonDistribution) { + const int kTrials = 20; + + // Large values are not yet supported -- this requires estimating the cdf + // using the normal distribution instead of the poisson in this case. + ASSERT_LE(mean(), 200.0); + if (mean() > 200.0) { + return; + } + + int failures = 0; + for (int i = 0; i < kTrials; i++) { + double p_value = ChiSquaredTestImpl<absl::poisson_distribution<int32_t>>(); + if (p_value < 0.005) { + failures++; + } + } + // There is a 0.10% chance of producing at least one failure, so raise the + // failure threshold high enough to allow for a flake rate < 10,000. + EXPECT_LE(failures, 4); +} + +INSTANTIATE_TEST_SUITE_P(All, PoissonDistributionChiSquaredTest, + ::testing::Values(0.5, 1.0, 2.0, 10.0, 50.0, 51.0, + 200.0)); + +// NOTE: absl::poisson_distribution is not guaranteed to be stable. +TEST(PoissonDistributionTest, StabilityTest) { + using testing::ElementsAre; + // absl::poisson_distribution stability relies on stability of + // std::exp, std::log, std::sqrt, std::ceil, std::floor, and + // absl::FastUniformBits, absl::StirlingLogFactorial, absl::RandU64ToDouble. + absl::random_internal::sequence_urbg urbg({ + 0x035b0dc7e0a18acfull, 0x06cebe0d2653682eull, 0x0061e9b23861596bull, + 0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull, + 0x4864f22c059bf29eull, 0x247856d8b862665cull, 0xe46e86e9a1337e10ull, + 0xd8c8541f3519b133ull, 0xe75b5162c567b9e4ull, 0xf732e5ded7009c5bull, + 0xb170b98353121eacull, 0x1ec2e8986d2362caull, 0x814c8e35fe9a961aull, + 0x0c3cd59c9b638a02ull, 0xcb3bb6478a07715cull, 0x1224e62c978bbc7full, + 0x671ef2cb04e81f6eull, 0x3c1cbd811eaf1808ull, 0x1bbc23cfa8fac721ull, + 0xa4c2cda65e596a51ull, 0xb77216fad37adf91ull, 0x836d794457c08849ull, + 0xe083df03475f49d7ull, 0xbc9feb512e6b0d6cull, 0xb12d74fdd718c8c5ull, + 0x12ff09653bfbe4caull, 0x8dd03a105bc4ee7eull, 0x5738341045ba0d85ull, + 0xf3fd722dc65ad09eull, 0xfa14fd21ea2a5705ull, 0xffe6ea4d6edb0c73ull, + 0xD07E9EFE2BF11FB4ull, 0x95DBDA4DAE909198ull, 0xEAAD8E716B93D5A0ull, + 0xD08ED1D0AFC725E0ull, 0x8E3C5B2F8E7594B7ull, 0x8FF6E2FBF2122B64ull, + 0x8888B812900DF01Cull, 0x4FAD5EA0688FC31Cull, 0xD1CFF191B3A8C1ADull, + 0x2F2F2218BE0E1777ull, 0xEA752DFE8B021FA1ull, 0xE5A0CC0FB56F74E8ull, + 0x18ACF3D6CE89E299ull, 0xB4A84FE0FD13E0B7ull, 0x7CC43B81D2ADA8D9ull, + 0x165FA26680957705ull, 0x93CC7314211A1477ull, 0xE6AD206577B5FA86ull, + 0xC75442F5FB9D35CFull, 0xEBCDAF0C7B3E89A0ull, 0xD6411BD3AE1E7E49ull, + 0x00250E2D2071B35Eull, 0x226800BB57B8E0AFull, 0x2464369BF009B91Eull, + 0x5563911D59DFA6AAull, 0x78C14389D95A537Full, 0x207D5BA202E5B9C5ull, + 0x832603766295CFA9ull, 0x11C819684E734A41ull, 0xB3472DCA7B14A94Aull, + }); + + std::vector<int> output(10); + + // Method 1. + { + absl::poisson_distribution<int> dist(5); + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + } + EXPECT_THAT(output, // mean = 4.2 + ElementsAre(1, 0, 0, 4, 2, 10, 3, 3, 7, 12)); + + // Method 2. + { + urbg.reset(); + absl::poisson_distribution<int> dist(25); + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + } + EXPECT_THAT(output, // mean = 19.8 + ElementsAre(9, 35, 18, 10, 35, 18, 10, 35, 18, 10)); + + // Method 3. + { + urbg.reset(); + absl::poisson_distribution<int> dist(121); + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + } + EXPECT_THAT(output, // mean = 124.1 + ElementsAre(161, 122, 129, 124, 112, 112, 117, 120, 130, 114)); +} + +TEST(PoissonDistributionTest, AlgorithmExpectedValue_1) { + // This tests small values of the Knuth method. + // The underlying uniform distribution will generate exactly 0.5. + absl::random_internal::sequence_urbg urbg({0x8000000000000001ull}); + absl::poisson_distribution<int> dist(5); + EXPECT_EQ(7, dist(urbg)); +} + +TEST(PoissonDistributionTest, AlgorithmExpectedValue_2) { + // This tests larger values of the Knuth method. + // The underlying uniform distribution will generate exactly 0.5. + absl::random_internal::sequence_urbg urbg({0x8000000000000001ull}); + absl::poisson_distribution<int> dist(25); + EXPECT_EQ(36, dist(urbg)); +} + +TEST(PoissonDistributionTest, AlgorithmExpectedValue_3) { + // This variant uses the ratio of uniforms method. + absl::random_internal::sequence_urbg urbg( + {0x7fffffffffffffffull, 0x8000000000000000ull}); + + absl::poisson_distribution<int> dist(121); + EXPECT_EQ(121, dist(urbg)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/random.h b/third_party/abseil_cpp/absl/random/random.h new file mode 100644 index 000000000000..71b6309288e6 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/random.h @@ -0,0 +1,189 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: random.h +// ----------------------------------------------------------------------------- +// +// This header defines the recommended Uniform Random Bit Generator (URBG) +// types for use within the Abseil Random library. These types are not +// suitable for security-related use-cases, but should suffice for most other +// uses of generating random values. +// +// The Abseil random library provides the following URBG types: +// +// * BitGen, a good general-purpose bit generator, optimized for generating +// random (but not cryptographically secure) values +// * InsecureBitGen, a slightly faster, though less random, bit generator, for +// cases where the existing BitGen is a drag on performance. + +#ifndef ABSL_RANDOM_RANDOM_H_ +#define ABSL_RANDOM_RANDOM_H_ + +#include <random> + +#include "absl/random/distributions.h" // IWYU pragma: export +#include "absl/random/internal/nonsecure_base.h" // IWYU pragma: export +#include "absl/random/internal/pcg_engine.h" // IWYU pragma: export +#include "absl/random/internal/pool_urbg.h" +#include "absl/random/internal/randen_engine.h" +#include "absl/random/seed_sequences.h" // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// absl::BitGen +// ----------------------------------------------------------------------------- +// +// `absl::BitGen` is a general-purpose random bit generator for generating +// random values for use within the Abseil random library. Typically, you use a +// bit generator in combination with a distribution to provide random values. +// +// Example: +// +// // Create an absl::BitGen. There is no need to seed this bit generator. +// absl::BitGen gen; +// +// // Generate an integer value in the closed interval [1,6] +// int die_roll = absl::uniform_int_distribution<int>(1, 6)(gen); +// +// `absl::BitGen` is seeded by default with non-deterministic data to produce +// different sequences of random values across different instances, including +// different binary invocations. This behavior is different than the standard +// library bit generators, which use golden values as their seeds. Default +// construction intentionally provides no stability guarantees, to avoid +// accidental dependence on such a property. +// +// `absl::BitGen` may be constructed with an optional seed sequence type, +// conforming to [rand.req.seed_seq], which will be mixed with additional +// non-deterministic data. +// +// Example: +// +// // Create an absl::BitGen using an std::seed_seq seed sequence +// std::seed_seq seq{1,2,3}; +// absl::BitGen gen_with_seed(seq); +// +// // Generate an integer value in the closed interval [1,6] +// int die_roll2 = absl::uniform_int_distribution<int>(1, 6)(gen_with_seed); +// +// `absl::BitGen` meets the requirements of the Uniform Random Bit Generator +// (URBG) concept as per the C++17 standard [rand.req.urng] though differs +// slightly with [rand.req.eng]. Like its standard library equivalents (e.g. +// `std::mersenne_twister_engine`) `absl::BitGen` is not cryptographically +// secure. +// +// Constructing two `absl::BitGen`s with the same seed sequence in the same +// binary will produce the same sequence of variates within the same binary, but +// need not do so across multiple binary invocations. +// +// This type has been optimized to perform better than Mersenne Twister +// (https://en.wikipedia.org/wiki/Mersenne_Twister) and many other complex URBG +// types on modern x86, ARM, and PPC architectures. +// +// This type is thread-compatible, but not thread-safe. + +// --------------------------------------------------------------------------- +// absl::BitGen member functions +// --------------------------------------------------------------------------- + +// absl::BitGen::operator()() +// +// Calls the BitGen, returning a generated value. + +// absl::BitGen::min() +// +// Returns the smallest possible value from this bit generator. + +// absl::BitGen::max() +// +// Returns the largest possible value from this bit generator. + +// absl::BitGen::discard(num) +// +// Advances the internal state of this bit generator by `num` times, and +// discards the intermediate results. +// --------------------------------------------------------------------------- + +using BitGen = random_internal::NonsecureURBGBase< + random_internal::randen_engine<uint64_t>>; + +// ----------------------------------------------------------------------------- +// absl::InsecureBitGen +// ----------------------------------------------------------------------------- +// +// `absl::InsecureBitGen` is an efficient random bit generator for generating +// random values, recommended only for performance-sensitive use cases where +// `absl::BitGen` is not satisfactory when compute-bounded by bit generation +// costs. +// +// Example: +// +// // Create an absl::InsecureBitGen +// absl::InsecureBitGen gen; +// for (size_t i = 0; i < 1000000; i++) { +// +// // Generate a bunch of random values from some complex distribution +// auto my_rnd = some_distribution(gen, 1, 1000); +// } +// +// Like `absl::BitGen`, `absl::InsecureBitGen` is seeded by default with +// non-deterministic data to produce different sequences of random values across +// different instances, including different binary invocations. (This behavior +// is different than the standard library bit generators, which use golden +// values as their seeds.) +// +// `absl::InsecureBitGen` may be constructed with an optional seed sequence +// type, conforming to [rand.req.seed_seq], which will be mixed with additional +// non-deterministic data. (See std_seed_seq.h for more information.) +// +// `absl::InsecureBitGen` meets the requirements of the Uniform Random Bit +// Generator (URBG) concept as per the C++17 standard [rand.req.urng] though +// its implementation differs slightly with [rand.req.eng]. Like its standard +// library equivalents (e.g. `std::mersenne_twister_engine`) +// `absl::InsecureBitGen` is not cryptographically secure. +// +// Prefer `absl::BitGen` over `absl::InsecureBitGen` as the general type is +// often fast enough for the vast majority of applications. + +using InsecureBitGen = + random_internal::NonsecureURBGBase<random_internal::pcg64_2018_engine>; + +// --------------------------------------------------------------------------- +// absl::InsecureBitGen member functions +// --------------------------------------------------------------------------- + +// absl::InsecureBitGen::operator()() +// +// Calls the InsecureBitGen, returning a generated value. + +// absl::InsecureBitGen::min() +// +// Returns the smallest possible value from this bit generator. + +// absl::InsecureBitGen::max() +// +// Returns the largest possible value from this bit generator. + +// absl::InsecureBitGen::discard(num) +// +// Advances the internal state of this bit generator by `num` times, and +// discards the intermediate results. +// --------------------------------------------------------------------------- + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_RANDOM_H_ diff --git a/third_party/abseil_cpp/absl/random/seed_gen_exception.cc b/third_party/abseil_cpp/absl/random/seed_gen_exception.cc new file mode 100644 index 000000000000..fdcb54a86cdd --- /dev/null +++ b/third_party/abseil_cpp/absl/random/seed_gen_exception.cc @@ -0,0 +1,46 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/seed_gen_exception.h" + +#include <iostream> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +static constexpr const char kExceptionMessage[] = + "Failed generating seed-material for URBG."; + +SeedGenException::~SeedGenException() = default; + +const char* SeedGenException::what() const noexcept { + return kExceptionMessage; +} + +namespace random_internal { + +void ThrowSeedGenException() { +#ifdef ABSL_HAVE_EXCEPTIONS + throw absl::SeedGenException(); +#else + std::cerr << kExceptionMessage << std::endl; + std::terminate(); +#endif +} + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/seed_gen_exception.h b/third_party/abseil_cpp/absl/random/seed_gen_exception.h new file mode 100644 index 000000000000..535390056437 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/seed_gen_exception.h @@ -0,0 +1,55 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: seed_gen_exception.h +// ----------------------------------------------------------------------------- +// +// This header defines an exception class which may be thrown if unpredictable +// events prevent the derivation of suitable seed-material for constructing a +// bit generator conforming to [rand.req.urng] (eg. entropy cannot be read from +// /dev/urandom on a Unix-based system). +// +// Note: if exceptions are disabled, `std::terminate()` is called instead. + +#ifndef ABSL_RANDOM_SEED_GEN_EXCEPTION_H_ +#define ABSL_RANDOM_SEED_GEN_EXCEPTION_H_ + +#include <exception> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +//------------------------------------------------------------------------------ +// SeedGenException +//------------------------------------------------------------------------------ +class SeedGenException : public std::exception { + public: + SeedGenException() = default; + ~SeedGenException() override; + const char* what() const noexcept override; +}; + +namespace random_internal { + +// throw delegator +[[noreturn]] void ThrowSeedGenException(); + +} // namespace random_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_SEED_GEN_EXCEPTION_H_ diff --git a/third_party/abseil_cpp/absl/random/seed_sequences.cc b/third_party/abseil_cpp/absl/random/seed_sequences.cc new file mode 100644 index 000000000000..426eafd3c8de --- /dev/null +++ b/third_party/abseil_cpp/absl/random/seed_sequences.cc @@ -0,0 +1,29 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/seed_sequences.h" + +#include "absl/random/internal/pool_urbg.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +SeedSeq MakeSeedSeq() { + SeedSeq::result_type seed_material[8]; + random_internal::RandenPool<uint32_t>::Fill(absl::MakeSpan(seed_material)); + return SeedSeq(std::begin(seed_material), std::end(seed_material)); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/random/seed_sequences.h b/third_party/abseil_cpp/absl/random/seed_sequences.h new file mode 100644 index 000000000000..ff1340cc8eed --- /dev/null +++ b/third_party/abseil_cpp/absl/random/seed_sequences.h @@ -0,0 +1,110 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: seed_sequences.h +// ----------------------------------------------------------------------------- +// +// This header contains utilities for creating and working with seed sequences +// conforming to [rand.req.seedseq]. In general, direct construction of seed +// sequences is discouraged, but use-cases for construction of identical bit +// generators (using the same seed sequence) may be helpful (e.g. replaying a +// simulation whose state is derived from variates of a bit generator). + +#ifndef ABSL_RANDOM_SEED_SEQUENCES_H_ +#define ABSL_RANDOM_SEED_SEQUENCES_H_ + +#include <iterator> +#include <random> + +#include "absl/random/internal/salted_seed_seq.h" +#include "absl/random/internal/seed_material.h" +#include "absl/random/seed_gen_exception.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// absl::SeedSeq +// ----------------------------------------------------------------------------- +// +// `absl::SeedSeq` constructs a seed sequence according to [rand.req.seedseq] +// for use within bit generators. `absl::SeedSeq`, unlike `std::seed_seq` +// additionally salts the generated seeds with extra implementation-defined +// entropy. For that reason, you can use `absl::SeedSeq` in combination with +// standard library bit generators (e.g. `std::mt19937`) to introduce +// non-determinism in your seeds. +// +// Example: +// +// absl::SeedSeq my_seed_seq({a, b, c}); +// std::mt19937 my_bitgen(my_seed_seq); +// +using SeedSeq = random_internal::SaltedSeedSeq<std::seed_seq>; + +// ----------------------------------------------------------------------------- +// absl::CreateSeedSeqFrom(bitgen*) +// ----------------------------------------------------------------------------- +// +// Constructs a seed sequence conforming to [rand.req.seedseq] using variates +// produced by a provided bit generator. +// +// You should generally avoid direct construction of seed sequences, but +// use-cases for reuse of a seed sequence to construct identical bit generators +// may be helpful (eg. replaying a simulation whose state is derived from bit +// generator values). +// +// If bitgen == nullptr, then behavior is undefined. +// +// Example: +// +// absl::BitGen my_bitgen; +// auto seed_seq = absl::CreateSeedSeqFrom(&my_bitgen); +// absl::BitGen new_engine(seed_seq); // derived from my_bitgen, but not +// // correlated. +// +template <typename URBG> +SeedSeq CreateSeedSeqFrom(URBG* urbg) { + SeedSeq::result_type + seed_material[random_internal::kEntropyBlocksNeeded]; + + if (!random_internal::ReadSeedMaterialFromURBG( + urbg, absl::MakeSpan(seed_material))) { + random_internal::ThrowSeedGenException(); + } + return SeedSeq(std::begin(seed_material), std::end(seed_material)); +} + +// ----------------------------------------------------------------------------- +// absl::MakeSeedSeq() +// ----------------------------------------------------------------------------- +// +// Constructs an `absl::SeedSeq` salting the generated values using +// implementation-defined entropy. The returned sequence can be used to create +// equivalent bit generators correlated using this sequence. +// +// Example: +// +// auto my_seed_seq = absl::MakeSeedSeq(); +// std::mt19937 rng1(my_seed_seq); +// std::mt19937 rng2(my_seed_seq); +// EXPECT_EQ(rng1(), rng2()); +// +SeedSeq MakeSeedSeq(); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_SEED_SEQUENCES_H_ diff --git a/third_party/abseil_cpp/absl/random/seed_sequences_test.cc b/third_party/abseil_cpp/absl/random/seed_sequences_test.cc new file mode 100644 index 000000000000..fe1100bda0d5 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/seed_sequences_test.cc @@ -0,0 +1,126 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/seed_sequences.h" + +#include <iterator> +#include <random> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/random/internal/nonsecure_base.h" +#include "absl/random/random.h" +namespace { + +TEST(SeedSequences, Examples) { + { + absl::SeedSeq seed_seq({1, 2, 3}); + absl::BitGen bitgen(seed_seq); + + EXPECT_NE(0, bitgen()); + } + { + absl::BitGen engine; + auto seed_seq = absl::CreateSeedSeqFrom(&engine); + absl::BitGen bitgen(seed_seq); + + EXPECT_NE(engine(), bitgen()); + } + { + auto seed_seq = absl::MakeSeedSeq(); + std::mt19937 random(seed_seq); + + EXPECT_NE(0, random()); + } +} + +TEST(CreateSeedSeqFrom, CompatibleWithStdTypes) { + using ExampleNonsecureURBG = + absl::random_internal::NonsecureURBGBase<std::minstd_rand0>; + + // Construct a URBG instance. + ExampleNonsecureURBG rng; + + // Construct a Seed Sequence from its variates. + auto seq_from_rng = absl::CreateSeedSeqFrom(&rng); + + // Ensure that another URBG can be validly constructed from the Seed Sequence. + std::mt19937_64{seq_from_rng}; +} + +TEST(CreateSeedSeqFrom, CompatibleWithBitGenerator) { + // Construct a URBG instance. + absl::BitGen rng; + + // Construct a Seed Sequence from its variates. + auto seq_from_rng = absl::CreateSeedSeqFrom(&rng); + + // Ensure that another URBG can be validly constructed from the Seed Sequence. + std::mt19937_64{seq_from_rng}; +} + +TEST(CreateSeedSeqFrom, CompatibleWithInsecureBitGen) { + // Construct a URBG instance. + absl::InsecureBitGen rng; + + // Construct a Seed Sequence from its variates. + auto seq_from_rng = absl::CreateSeedSeqFrom(&rng); + + // Ensure that another URBG can be validly constructed from the Seed Sequence. + std::mt19937_64{seq_from_rng}; +} + +TEST(CreateSeedSeqFrom, CompatibleWithRawURBG) { + // Construct a URBG instance. + std::random_device urandom; + + // Construct a Seed Sequence from its variates, using 64b of seed-material. + auto seq_from_rng = absl::CreateSeedSeqFrom(&urandom); + + // Ensure that another URBG can be validly constructed from the Seed Sequence. + std::mt19937_64{seq_from_rng}; +} + +template <typename URBG> +void TestReproducibleVariateSequencesForNonsecureURBG() { + const size_t kNumVariates = 1000; + + URBG rng; + // Reused for both RNG instances. + auto reusable_seed = absl::CreateSeedSeqFrom(&rng); + + typename URBG::result_type variates[kNumVariates]; + { + URBG child(reusable_seed); + for (auto& variate : variates) { + variate = child(); + } + } + // Ensure that variate-sequence can be "replayed" by identical RNG. + { + URBG child(reusable_seed); + for (auto& variate : variates) { + ASSERT_EQ(variate, child()); + } + } +} + +TEST(CreateSeedSeqFrom, ReproducesVariateSequencesForInsecureBitGen) { + TestReproducibleVariateSequencesForNonsecureURBG<absl::InsecureBitGen>(); +} + +TEST(CreateSeedSeqFrom, ReproducesVariateSequencesForBitGenerator) { + TestReproducibleVariateSequencesForNonsecureURBG<absl::BitGen>(); +} +} // namespace diff --git a/third_party/abseil_cpp/absl/random/uniform_int_distribution.h b/third_party/abseil_cpp/absl/random/uniform_int_distribution.h new file mode 100644 index 000000000000..c1f54ccebce5 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/uniform_int_distribution.h @@ -0,0 +1,275 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: uniform_int_distribution.h +// ----------------------------------------------------------------------------- +// +// This header defines a class for representing a uniform integer distribution +// over the closed (inclusive) interval [a,b]. You use this distribution in +// combination with an Abseil random bit generator to produce random values +// according to the rules of the distribution. +// +// `absl::uniform_int_distribution` is a drop-in replacement for the C++11 +// `std::uniform_int_distribution` [rand.dist.uni.int] but is considerably +// faster than the libstdc++ implementation. + +#ifndef ABSL_RANDOM_UNIFORM_INT_DISTRIBUTION_H_ +#define ABSL_RANDOM_UNIFORM_INT_DISTRIBUTION_H_ + +#include <cassert> +#include <istream> +#include <limits> +#include <type_traits> + +#include "absl/base/optimization.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/internal/traits.h" +#include "absl/random/internal/wide_multiply.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::uniform_int_distribution<T> +// +// This distribution produces random integer values uniformly distributed in the +// closed (inclusive) interval [a, b]. +// +// Example: +// +// absl::BitGen gen; +// +// // Use the distribution to produce a value between 1 and 6, inclusive. +// int die_roll = absl::uniform_int_distribution<int>(1, 6)(gen); +// +template <typename IntType = int> +class uniform_int_distribution { + private: + using unsigned_type = + typename random_internal::make_unsigned_bits<IntType>::type; + + public: + using result_type = IntType; + + class param_type { + public: + using distribution_type = uniform_int_distribution; + + explicit param_type( + result_type lo = 0, + result_type hi = (std::numeric_limits<result_type>::max)()) + : lo_(lo), + range_(static_cast<unsigned_type>(hi) - + static_cast<unsigned_type>(lo)) { + // [rand.dist.uni.int] precondition 2 + assert(lo <= hi); + } + + result_type a() const { return lo_; } + result_type b() const { + return static_cast<result_type>(static_cast<unsigned_type>(lo_) + range_); + } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.lo_ == b.lo_ && a.range_ == b.range_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class uniform_int_distribution; + unsigned_type range() const { return range_; } + + result_type lo_; + unsigned_type range_; + + static_assert(std::is_integral<result_type>::value, + "Class-template absl::uniform_int_distribution<> must be " + "parameterized using an integral type."); + }; // param_type + + uniform_int_distribution() : uniform_int_distribution(0) {} + + explicit uniform_int_distribution( + result_type lo, + result_type hi = (std::numeric_limits<result_type>::max)()) + : param_(lo, hi) {} + + explicit uniform_int_distribution(const param_type& param) : param_(param) {} + + // uniform_int_distribution<T>::reset() + // + // Resets the uniform int distribution. Note that this function has no effect + // because the distribution already produces independent values. + void reset() {} + + template <typename URBG> + result_type operator()(URBG& gen) { // NOLINT(runtime/references) + return (*this)(gen, param()); + } + + template <typename URBG> + result_type operator()( + URBG& gen, const param_type& param) { // NOLINT(runtime/references) + return param.a() + Generate(gen, param.range()); + } + + result_type a() const { return param_.a(); } + result_type b() const { return param_.b(); } + + param_type param() const { return param_; } + void param(const param_type& params) { param_ = params; } + + result_type(min)() const { return a(); } + result_type(max)() const { return b(); } + + friend bool operator==(const uniform_int_distribution& a, + const uniform_int_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const uniform_int_distribution& a, + const uniform_int_distribution& b) { + return !(a == b); + } + + private: + // Generates a value in the *closed* interval [0, R] + template <typename URBG> + unsigned_type Generate(URBG& g, // NOLINT(runtime/references) + unsigned_type R); + param_type param_; +}; + +// ----------------------------------------------------------------------------- +// Implementation details follow +// ----------------------------------------------------------------------------- +template <typename CharT, typename Traits, typename IntType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, + const uniform_int_distribution<IntType>& x) { + using stream_type = + typename random_internal::stream_format_type<IntType>::type; + auto saver = random_internal::make_ostream_state_saver(os); + os << static_cast<stream_type>(x.a()) << os.fill() + << static_cast<stream_type>(x.b()); + return os; +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, + uniform_int_distribution<IntType>& x) { + using param_type = typename uniform_int_distribution<IntType>::param_type; + using result_type = typename uniform_int_distribution<IntType>::result_type; + using stream_type = + typename random_internal::stream_format_type<IntType>::type; + + stream_type a; + stream_type b; + + auto saver = random_internal::make_istream_state_saver(is); + is >> a >> b; + if (!is.fail()) { + x.param( + param_type(static_cast<result_type>(a), static_cast<result_type>(b))); + } + return is; +} + +template <typename IntType> +template <typename URBG> +typename random_internal::make_unsigned_bits<IntType>::type +uniform_int_distribution<IntType>::Generate( + URBG& g, // NOLINT(runtime/references) + typename random_internal::make_unsigned_bits<IntType>::type R) { + random_internal::FastUniformBits<unsigned_type> fast_bits; + unsigned_type bits = fast_bits(g); + const unsigned_type Lim = R + 1; + if ((R & Lim) == 0) { + // If the interval's length is a power of two range, just take the low bits. + return bits & R; + } + + // Generates a uniform variate on [0, Lim) using fixed-point multiplication. + // The above fast-path guarantees that Lim is representable in unsigned_type. + // + // Algorithm adapted from + // http://lemire.me/blog/2016/06/30/fast-random-shuffling/, with added + // explanation. + // + // The algorithm creates a uniform variate `bits` in the interval [0, 2^N), + // and treats it as the fractional part of a fixed-point real value in [0, 1), + // multiplied by 2^N. For example, 0.25 would be represented as 2^(N - 2), + // because 2^N * 0.25 == 2^(N - 2). + // + // Next, `bits` and `Lim` are multiplied with a wide-multiply to bring the + // value into the range [0, Lim). The integral part (the high word of the + // multiplication result) is then very nearly the desired result. However, + // this is not quite accurate; viewing the multiplication result as one + // double-width integer, the resulting values for the sample are mapped as + // follows: + // + // If the result lies in this interval: Return this value: + // [0, 2^N) 0 + // [2^N, 2 * 2^N) 1 + // ... ... + // [K * 2^N, (K + 1) * 2^N) K + // ... ... + // [(Lim - 1) * 2^N, Lim * 2^N) Lim - 1 + // + // While all of these intervals have the same size, the result of `bits * Lim` + // must be a multiple of `Lim`, and not all of these intervals contain the + // same number of multiples of `Lim`. In particular, some contain + // `F = floor(2^N / Lim)` and some contain `F + 1 = ceil(2^N / Lim)`. This + // difference produces a small nonuniformity, which is corrected by applying + // rejection sampling to one of the values in the "larger intervals" (i.e., + // the intervals containing `F + 1` multiples of `Lim`. + // + // An interval contains `F + 1` multiples of `Lim` if and only if its smallest + // value modulo 2^N is less than `2^N % Lim`. The unique value satisfying + // this property is used as the one for rejection. That is, a value of + // `bits * Lim` is rejected if `(bit * Lim) % 2^N < (2^N % Lim)`. + + using helper = random_internal::wide_multiply<unsigned_type>; + auto product = helper::multiply(bits, Lim); + + // Two optimizations here: + // * Rejection occurs with some probability less than 1/2, and for reasonable + // ranges considerably less (in particular, less than 1/(F+1)), so + // ABSL_PREDICT_FALSE is apt. + // * `Lim` is an overestimate of `threshold`, and doesn't require a divide. + if (ABSL_PREDICT_FALSE(helper::lo(product) < Lim)) { + // This quantity is exactly equal to `2^N % Lim`, but does not require high + // precision calculations: `2^N % Lim` is congruent to `(2^N - Lim) % Lim`. + // Ideally this could be expressed simply as `-X` rather than `2^N - X`, but + // for types smaller than int, this calculation is incorrect due to integer + // promotion rules. + const unsigned_type threshold = + ((std::numeric_limits<unsigned_type>::max)() - Lim + 1) % Lim; + while (helper::lo(product) < threshold) { + bits = fast_bits(g); + product = helper::multiply(bits, Lim); + } + } + + return helper::hi(product); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_UNIFORM_INT_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/uniform_int_distribution_test.cc b/third_party/abseil_cpp/absl/random/uniform_int_distribution_test.cc new file mode 100644 index 000000000000..276d72ad2045 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/uniform_int_distribution_test.cc @@ -0,0 +1,259 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/uniform_int_distribution.h" + +#include <cmath> +#include <cstdint> +#include <iterator> +#include <random> +#include <sstream> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" + +namespace { + +template <typename IntType> +class UniformIntDistributionTest : public ::testing::Test {}; + +using IntTypes = ::testing::Types<int8_t, uint8_t, int16_t, uint16_t, int32_t, + uint32_t, int64_t, uint64_t>; +TYPED_TEST_SUITE(UniformIntDistributionTest, IntTypes); + +TYPED_TEST(UniformIntDistributionTest, ParamSerializeTest) { + // This test essentially ensures that the parameters serialize, + // not that the values generated cover the full range. + using Limits = std::numeric_limits<TypeParam>; + using param_type = + typename absl::uniform_int_distribution<TypeParam>::param_type; + const TypeParam kMin = std::is_unsigned<TypeParam>::value ? 37 : -105; + const TypeParam kNegOneOrZero = std::is_unsigned<TypeParam>::value ? 0 : -1; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + for (const auto& param : { + param_type(), + param_type(2, 2), // Same + param_type(9, 32), + param_type(kMin, 115), + param_type(kNegOneOrZero, Limits::max()), + param_type(Limits::min(), Limits::max()), + param_type(Limits::lowest(), Limits::max()), + param_type(Limits::min() + 1, Limits::max() - 1), + }) { + const auto a = param.a(); + const auto b = param.b(); + absl::uniform_int_distribution<TypeParam> before(a, b); + EXPECT_EQ(before.a(), param.a()); + EXPECT_EQ(before.b(), param.b()); + + { + // Initialize via param_type + absl::uniform_int_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + } + + // Initialize via iostreams + std::stringstream ss; + ss << before; + + absl::uniform_int_distribution<TypeParam> after(Limits::min() + 3, + Limits::max() - 5); + + EXPECT_NE(before.a(), after.a()); + EXPECT_NE(before.b(), after.b()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before.a(), after.a()); + EXPECT_EQ(before.b(), after.b()); + EXPECT_EQ(before.param(), after.param()); + EXPECT_EQ(before, after); + + // Smoke test. + auto sample_min = after.max(); + auto sample_max = after.min(); + for (int i = 0; i < kCount; i++) { + auto sample = after(gen); + EXPECT_GE(sample, after.min()); + EXPECT_LE(sample, after.max()); + if (sample > sample_max) { + sample_max = sample; + } + if (sample < sample_min) { + sample_min = sample; + } + } + std::string msg = absl::StrCat("Range: ", +sample_min, ", ", +sample_max); + ABSL_RAW_LOG(INFO, "%s", msg.c_str()); + } +} + +TYPED_TEST(UniformIntDistributionTest, ViolatesPreconditionsDeathTest) { +#if GTEST_HAS_DEATH_TEST + // Hi < Lo + EXPECT_DEBUG_DEATH({ absl::uniform_int_distribution<TypeParam> dist(10, 1); }, + ""); +#endif // GTEST_HAS_DEATH_TEST +#if defined(NDEBUG) + // opt-mode, for invalid parameters, will generate a garbage value, + // but should not enter an infinite loop. + absl::InsecureBitGen gen; + absl::uniform_int_distribution<TypeParam> dist(10, 1); + auto x = dist(gen); + + // Any value will generate a non-empty string. + EXPECT_FALSE(absl::StrCat(+x).empty()) << x; +#endif // NDEBUG +} + +TYPED_TEST(UniformIntDistributionTest, TestMoments) { + constexpr int kSize = 100000; + using Limits = std::numeric_limits<TypeParam>; + using param_type = + typename absl::uniform_int_distribution<TypeParam>::param_type; + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng{0x2B7E151628AED2A6}; + + std::vector<double> values(kSize); + for (const auto& param : + {param_type(0, Limits::max()), param_type(13, 127)}) { + absl::uniform_int_distribution<TypeParam> dist(param); + for (int i = 0; i < kSize; i++) { + const auto sample = dist(rng); + ASSERT_LE(dist.param().a(), sample); + ASSERT_GE(dist.param().b(), sample); + values[i] = sample; + } + + auto moments = absl::random_internal::ComputeDistributionMoments(values); + const double a = dist.param().a(); + const double b = dist.param().b(); + const double n = (b - a + 1); + const double mean = (a + b) / 2; + const double var = ((b - a + 1) * (b - a + 1) - 1) / 12; + const double kurtosis = 3 - 6 * (n * n + 1) / (5 * (n * n - 1)); + + // TODO(ahh): this is not the right bound + // empirically validated with --runs_per_test=10000. + EXPECT_NEAR(mean, moments.mean, 0.01 * var); + EXPECT_NEAR(var, moments.variance, 0.015 * var); + EXPECT_NEAR(0.0, moments.skewness, 0.025); + EXPECT_NEAR(kurtosis, moments.kurtosis, 0.02 * kurtosis); + } +} + +TYPED_TEST(UniformIntDistributionTest, ChiSquaredTest50) { + using absl::random_internal::kChiSquared; + + constexpr size_t kTrials = 1000; + constexpr int kBuckets = 50; // inclusive, so actally +1 + constexpr double kExpected = + static_cast<double>(kTrials) / static_cast<double>(kBuckets); + + // Empirically validated with --runs_per_test=10000. + const int kThreshold = + absl::random_internal::ChiSquareValue(kBuckets, 0.999999); + + const TypeParam min = std::is_unsigned<TypeParam>::value ? 37 : -37; + const TypeParam max = min + kBuckets; + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng{0x2B7E151628AED2A6}; + + absl::uniform_int_distribution<TypeParam> dist(min, max); + + std::vector<int32_t> counts(kBuckets + 1, 0); + for (size_t i = 0; i < kTrials; i++) { + auto x = dist(rng); + counts[x - min]++; + } + double chi_square = absl::random_internal::ChiSquareWithExpected( + std::begin(counts), std::end(counts), kExpected); + if (chi_square > kThreshold) { + double p_value = + absl::random_internal::ChiSquarePValue(chi_square, kBuckets); + + // Chi-squared test failed. Output does not appear to be uniform. + std::string msg; + for (const auto& a : counts) { + absl::StrAppend(&msg, a, "\n"); + } + absl::StrAppend(&msg, kChiSquared, " p-value ", p_value, "\n"); + absl::StrAppend(&msg, "High ", kChiSquared, " value: ", chi_square, " > ", + kThreshold); + ABSL_RAW_LOG(INFO, "%s", msg.c_str()); + FAIL() << msg; + } +} + +TEST(UniformIntDistributionTest, StabilityTest) { + // absl::uniform_int_distribution stability relies only on integer operations. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(12); + + { + absl::uniform_int_distribution<int32_t> dist(0, 4); + for (auto& v : output) { + v = dist(urbg); + } + } + EXPECT_EQ(12, urbg.invocations()); + EXPECT_THAT(output, testing::ElementsAre(4, 4, 3, 2, 1, 0, 1, 4, 3, 1, 3, 1)); + + { + urbg.reset(); + absl::uniform_int_distribution<int32_t> dist(0, 100); + for (auto& v : output) { + v = dist(urbg); + } + } + EXPECT_EQ(12, urbg.invocations()); + EXPECT_THAT(output, testing::ElementsAre(97, 86, 75, 41, 36, 16, 38, 92, 67, + 30, 80, 38)); + + { + urbg.reset(); + absl::uniform_int_distribution<int32_t> dist(0, 10000); + for (auto& v : output) { + v = dist(urbg); + } + } + EXPECT_EQ(12, urbg.invocations()); + EXPECT_THAT(output, testing::ElementsAre(9648, 8562, 7439, 4089, 3571, 1602, + 3813, 9195, 6641, 2986, 7956, 3765)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/uniform_real_distribution.h b/third_party/abseil_cpp/absl/random/uniform_real_distribution.h new file mode 100644 index 000000000000..5ba17b2341cf --- /dev/null +++ b/third_party/abseil_cpp/absl/random/uniform_real_distribution.h @@ -0,0 +1,202 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: uniform_real_distribution.h +// ----------------------------------------------------------------------------- +// +// This header defines a class for representing a uniform floating-point +// distribution over a half-open interval [a,b). You use this distribution in +// combination with an Abseil random bit generator to produce random values +// according to the rules of the distribution. +// +// `absl::uniform_real_distribution` is a drop-in replacement for the C++11 +// `std::uniform_real_distribution` [rand.dist.uni.real] but is considerably +// faster than the libstdc++ implementation. +// +// Note: the standard-library version may occasionally return `1.0` when +// default-initialized. See https://bugs.llvm.org//show_bug.cgi?id=18767 +// `absl::uniform_real_distribution` does not exhibit this behavior. + +#ifndef ABSL_RANDOM_UNIFORM_REAL_DISTRIBUTION_H_ +#define ABSL_RANDOM_UNIFORM_REAL_DISTRIBUTION_H_ + +#include <cassert> +#include <cmath> +#include <cstdint> +#include <istream> +#include <limits> +#include <type_traits> + +#include "absl/meta/type_traits.h" +#include "absl/random/internal/fast_uniform_bits.h" +#include "absl/random/internal/generate_real.h" +#include "absl/random/internal/iostream_state_saver.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::uniform_real_distribution<T> +// +// This distribution produces random floating-point values uniformly distributed +// over the half-open interval [a, b). +// +// Example: +// +// absl::BitGen gen; +// +// // Use the distribution to produce a value between 0.0 (inclusive) +// // and 1.0 (exclusive). +// double value = absl::uniform_real_distribution<double>(0, 1)(gen); +// +template <typename RealType = double> +class uniform_real_distribution { + public: + using result_type = RealType; + + class param_type { + public: + using distribution_type = uniform_real_distribution; + + explicit param_type(result_type lo = 0, result_type hi = 1) + : lo_(lo), hi_(hi), range_(hi - lo) { + // [rand.dist.uni.real] preconditions 2 & 3 + assert(lo <= hi); + // NOTE: For integral types, we can promote the range to an unsigned type, + // which gives full width of the range. However for real (fp) types, this + // is not possible, so value generation cannot use the full range of the + // real type. + assert(range_ <= (std::numeric_limits<result_type>::max)()); + assert(std::isfinite(range_)); + } + + result_type a() const { return lo_; } + result_type b() const { return hi_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.lo_ == b.lo_ && a.hi_ == b.hi_; + } + + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class uniform_real_distribution; + result_type lo_, hi_, range_; + + static_assert(std::is_floating_point<RealType>::value, + "Class-template absl::uniform_real_distribution<> must be " + "parameterized using a floating-point type."); + }; + + uniform_real_distribution() : uniform_real_distribution(0) {} + + explicit uniform_real_distribution(result_type lo, result_type hi = 1) + : param_(lo, hi) {} + + explicit uniform_real_distribution(const param_type& param) : param_(param) {} + + // uniform_real_distribution<T>::reset() + // + // Resets the uniform real distribution. Note that this function has no effect + // because the distribution already produces independent values. + void reset() {} + + template <typename URBG> + result_type operator()(URBG& gen) { // NOLINT(runtime/references) + return operator()(gen, param_); + } + + template <typename URBG> + result_type operator()(URBG& gen, // NOLINT(runtime/references) + const param_type& p); + + result_type a() const { return param_.a(); } + result_type b() const { return param_.b(); } + + param_type param() const { return param_; } + void param(const param_type& params) { param_ = params; } + + result_type(min)() const { return a(); } + result_type(max)() const { return b(); } + + friend bool operator==(const uniform_real_distribution& a, + const uniform_real_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const uniform_real_distribution& a, + const uniform_real_distribution& b) { + return a.param_ != b.param_; + } + + private: + param_type param_; + random_internal::FastUniformBits<uint64_t> fast_u64_; +}; + +// ----------------------------------------------------------------------------- +// Implementation details follow +// ----------------------------------------------------------------------------- +template <typename RealType> +template <typename URBG> +typename uniform_real_distribution<RealType>::result_type +uniform_real_distribution<RealType>::operator()( + URBG& gen, const param_type& p) { // NOLINT(runtime/references) + using random_internal::GeneratePositiveTag; + using random_internal::GenerateRealFromBits; + using real_type = + absl::conditional_t<std::is_same<RealType, float>::value, float, double>; + + while (true) { + const result_type sample = + GenerateRealFromBits<real_type, GeneratePositiveTag, true>( + fast_u64_(gen)); + const result_type res = p.a() + (sample * p.range_); + if (res < p.b() || p.range_ <= 0 || !std::isfinite(p.range_)) { + return res; + } + // else sample rejected, try again. + } +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const uniform_real_distribution<RealType>& x) { + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<RealType>::kPrecision); + os << x.a() << os.fill() << x.b(); + return os; +} + +template <typename CharT, typename Traits, typename RealType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + uniform_real_distribution<RealType>& x) { // NOLINT(runtime/references) + using param_type = typename uniform_real_distribution<RealType>::param_type; + using result_type = typename uniform_real_distribution<RealType>::result_type; + auto saver = random_internal::make_istream_state_saver(is); + auto a = random_internal::read_floating_point<result_type>(is); + if (is.fail()) return is; + auto b = random_internal::read_floating_point<result_type>(is); + if (!is.fail()) { + x.param(param_type(a, b)); + } + return is; +} +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_UNIFORM_REAL_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/uniform_real_distribution_test.cc b/third_party/abseil_cpp/absl/random/uniform_real_distribution_test.cc new file mode 100644 index 000000000000..be107cdde467 --- /dev/null +++ b/third_party/abseil_cpp/absl/random/uniform_real_distribution_test.cc @@ -0,0 +1,343 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/uniform_real_distribution.h" + +#include <cmath> +#include <cstdint> +#include <iterator> +#include <random> +#include <sstream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/distribution_test_util.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" + +// NOTES: +// * Some documentation on generating random real values suggests that +// it is possible to use std::nextafter(b, DBL_MAX) to generate a value on +// the closed range [a, b]. Unfortunately, that technique is not universally +// reliable due to floating point quantization. +// +// * absl::uniform_real_distribution<float> generates between 2^28 and 2^29 +// distinct floating point values in the range [0, 1). +// +// * absl::uniform_real_distribution<float> generates at least 2^23 distinct +// floating point values in the range [1, 2). This should be the same as +// any other range covered by a single exponent in IEEE 754. +// +// * absl::uniform_real_distribution<double> generates more than 2^52 distinct +// values in the range [0, 1), and should generate at least 2^52 distinct +// values in the range of [1, 2). +// + +namespace { + +template <typename RealType> +class UniformRealDistributionTest : public ::testing::Test {}; + +#if defined(__EMSCRIPTEN__) +using RealTypes = ::testing::Types<float, double>; +#else +using RealTypes = ::testing::Types<float, double, long double>; +#endif // defined(__EMSCRIPTEN__) + +TYPED_TEST_SUITE(UniformRealDistributionTest, RealTypes); + +TYPED_TEST(UniformRealDistributionTest, ParamSerializeTest) { + using param_type = + typename absl::uniform_real_distribution<TypeParam>::param_type; + + constexpr const TypeParam a{1152921504606846976}; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + for (const auto& param : { + param_type(), + param_type(TypeParam(2.0), TypeParam(2.0)), // Same + param_type(TypeParam(-0.1), TypeParam(0.1)), + param_type(TypeParam(0.05), TypeParam(0.12)), + param_type(TypeParam(-0.05), TypeParam(0.13)), + param_type(TypeParam(-0.05), TypeParam(-0.02)), + // double range = 0 + // 2^60 , 2^60 + 2^6 + param_type(a, TypeParam(1152921504606847040)), + // 2^60 , 2^60 + 2^7 + param_type(a, TypeParam(1152921504606847104)), + // double range = 2^8 + // 2^60 , 2^60 + 2^8 + param_type(a, TypeParam(1152921504606847232)), + // float range = 0 + // 2^60 , 2^60 + 2^36 + param_type(a, TypeParam(1152921573326323712)), + // 2^60 , 2^60 + 2^37 + param_type(a, TypeParam(1152921642045800448)), + // float range = 2^38 + // 2^60 , 2^60 + 2^38 + param_type(a, TypeParam(1152921779484753920)), + // Limits + param_type(0, std::numeric_limits<TypeParam>::max()), + param_type(std::numeric_limits<TypeParam>::lowest(), 0), + param_type(0, std::numeric_limits<TypeParam>::epsilon()), + param_type(-std::numeric_limits<TypeParam>::epsilon(), + std::numeric_limits<TypeParam>::epsilon()), + param_type(std::numeric_limits<TypeParam>::epsilon(), + 2 * std::numeric_limits<TypeParam>::epsilon()), + }) { + // Validate parameters. + const auto a = param.a(); + const auto b = param.b(); + absl::uniform_real_distribution<TypeParam> before(a, b); + EXPECT_EQ(before.a(), param.a()); + EXPECT_EQ(before.b(), param.b()); + + { + absl::uniform_real_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + } + + std::stringstream ss; + ss << before; + absl::uniform_real_distribution<TypeParam> after(TypeParam(1.0), + TypeParam(3.1)); + + EXPECT_NE(before.a(), after.a()); + EXPECT_NE(before.b(), after.b()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before.a(), after.a()); + EXPECT_EQ(before.b(), after.b()); + EXPECT_EQ(before.param(), after.param()); + EXPECT_EQ(before, after); + + // Smoke test. + auto sample_min = after.max(); + auto sample_max = after.min(); + for (int i = 0; i < kCount; i++) { + auto sample = after(gen); + // Failure here indicates a bug in uniform_real_distribution::operator(), + // or bad parameters--range too large, etc. + if (after.min() == after.max()) { + EXPECT_EQ(sample, after.min()); + } else { + EXPECT_GE(sample, after.min()); + EXPECT_LT(sample, after.max()); + } + if (sample > sample_max) { + sample_max = sample; + } + if (sample < sample_min) { + sample_min = sample; + } + } + + if (!std::is_same<TypeParam, long double>::value) { + // static_cast<double>(long double) can overflow. + std::string msg = absl::StrCat("Range: ", static_cast<double>(sample_min), + ", ", static_cast<double>(sample_max)); + ABSL_RAW_LOG(INFO, "%s", msg.c_str()); + } + } +} + +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable:4756) // Constant arithmetic overflow. +#endif +TYPED_TEST(UniformRealDistributionTest, ViolatesPreconditionsDeathTest) { +#if GTEST_HAS_DEATH_TEST + // Hi < Lo + EXPECT_DEBUG_DEATH( + { absl::uniform_real_distribution<TypeParam> dist(10.0, 1.0); }, ""); + + // Hi - Lo > numeric_limits<>::max() + EXPECT_DEBUG_DEATH( + { + absl::uniform_real_distribution<TypeParam> dist( + std::numeric_limits<TypeParam>::lowest(), + std::numeric_limits<TypeParam>::max()); + }, + ""); +#endif // GTEST_HAS_DEATH_TEST +#if defined(NDEBUG) + // opt-mode, for invalid parameters, will generate a garbage value, + // but should not enter an infinite loop. + absl::InsecureBitGen gen; + { + absl::uniform_real_distribution<TypeParam> dist(10.0, 1.0); + auto x = dist(gen); + EXPECT_FALSE(std::isnan(x)) << x; + } + { + absl::uniform_real_distribution<TypeParam> dist( + std::numeric_limits<TypeParam>::lowest(), + std::numeric_limits<TypeParam>::max()); + auto x = dist(gen); + // Infinite result. + EXPECT_FALSE(std::isfinite(x)) << x; + } +#endif // NDEBUG +} +#ifdef _MSC_VER +#pragma warning(pop) // warning(disable:4756) +#endif + +TYPED_TEST(UniformRealDistributionTest, TestMoments) { + constexpr int kSize = 1000000; + std::vector<double> values(kSize); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng{0x2B7E151628AED2A6}; + + absl::uniform_real_distribution<TypeParam> dist; + for (int i = 0; i < kSize; i++) { + values[i] = dist(rng); + } + + const auto moments = + absl::random_internal::ComputeDistributionMoments(values); + EXPECT_NEAR(0.5, moments.mean, 0.01); + EXPECT_NEAR(1 / 12.0, moments.variance, 0.015); + EXPECT_NEAR(0.0, moments.skewness, 0.02); + EXPECT_NEAR(9 / 5.0, moments.kurtosis, 0.015); +} + +TYPED_TEST(UniformRealDistributionTest, ChiSquaredTest50) { + using absl::random_internal::kChiSquared; + using param_type = + typename absl::uniform_real_distribution<TypeParam>::param_type; + + constexpr size_t kTrials = 100000; + constexpr int kBuckets = 50; + constexpr double kExpected = + static_cast<double>(kTrials) / static_cast<double>(kBuckets); + + // 1-in-100000 threshold, but remember, there are about 8 tests + // in this file. And the test could fail for other reasons. + // Empirically validated with --runs_per_test=10000. + const int kThreshold = + absl::random_internal::ChiSquareValue(kBuckets - 1, 0.999999); + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng{0x2B7E151628AED2A6}; + + for (const auto& param : {param_type(0, 1), param_type(5, 12), + param_type(-5, 13), param_type(-5, -2)}) { + const double min_val = param.a(); + const double max_val = param.b(); + const double factor = kBuckets / (max_val - min_val); + + std::vector<int32_t> counts(kBuckets, 0); + absl::uniform_real_distribution<TypeParam> dist(param); + for (size_t i = 0; i < kTrials; i++) { + auto x = dist(rng); + auto bucket = static_cast<size_t>((x - min_val) * factor); + counts[bucket]++; + } + + double chi_square = absl::random_internal::ChiSquareWithExpected( + std::begin(counts), std::end(counts), kExpected); + if (chi_square > kThreshold) { + double p_value = + absl::random_internal::ChiSquarePValue(chi_square, kBuckets); + + // Chi-squared test failed. Output does not appear to be uniform. + std::string msg; + for (const auto& a : counts) { + absl::StrAppend(&msg, a, "\n"); + } + absl::StrAppend(&msg, kChiSquared, " p-value ", p_value, "\n"); + absl::StrAppend(&msg, "High ", kChiSquared, " value: ", chi_square, " > ", + kThreshold); + ABSL_RAW_LOG(INFO, "%s", msg.c_str()); + FAIL() << msg; + } + } +} + +TYPED_TEST(UniformRealDistributionTest, StabilityTest) { + // absl::uniform_real_distribution stability relies only on + // random_internal::RandU64ToDouble and random_internal::RandU64ToFloat. + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(12); + + absl::uniform_real_distribution<TypeParam> dist; + std::generate(std::begin(output), std::end(output), [&] { + return static_cast<int>(TypeParam(1000000) * dist(urbg)); + }); + + EXPECT_THAT( + output, // + testing::ElementsAre(59, 999246, 762494, 395876, 167716, 82545, 925251, + 77341, 12527, 708791, 834451, 932808)); +} + +TEST(UniformRealDistributionTest, AlgorithmBounds) { + absl::uniform_real_distribution<double> dist; + + { + // This returns the smallest value >0 from absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg({0x0000000000000001ull}); + double a = dist(urbg); + EXPECT_EQ(a, 5.42101086242752217004e-20); + } + + { + // This returns a value very near 0.5 from absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg({0x7fffffffffffffefull}); + double a = dist(urbg); + EXPECT_EQ(a, 0.499999999999999944489); + } + { + // This returns a value very near 0.5 from absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg({0x8000000000000000ull}); + double a = dist(urbg); + EXPECT_EQ(a, 0.5); + } + + { + // This returns the largest value <1 from absl::uniform_real_distribution. + absl::random_internal::sequence_urbg urbg({0xFFFFFFFFFFFFFFEFull}); + double a = dist(urbg); + EXPECT_EQ(a, 0.999999999999999888978); + } + { + // This *ALSO* returns the largest value <1. + absl::random_internal::sequence_urbg urbg({0xFFFFFFFFFFFFFFFFull}); + double a = dist(urbg); + EXPECT_EQ(a, 0.999999999999999888978); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/random/zipf_distribution.h b/third_party/abseil_cpp/absl/random/zipf_distribution.h new file mode 100644 index 000000000000..22ebc756cfef --- /dev/null +++ b/third_party/abseil_cpp/absl/random/zipf_distribution.h @@ -0,0 +1,271 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_RANDOM_ZIPF_DISTRIBUTION_H_ +#define ABSL_RANDOM_ZIPF_DISTRIBUTION_H_ + +#include <cassert> +#include <cmath> +#include <istream> +#include <limits> +#include <ostream> +#include <type_traits> + +#include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/uniform_real_distribution.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::zipf_distribution produces random integer-values in the range [0, k], +// distributed according to the discrete probability function: +// +// P(x) = (v + x) ^ -q +// +// The parameter `v` must be greater than 0 and the parameter `q` must be +// greater than 1. If either of these parameters take invalid values then the +// behavior is undefined. +// +// IntType is the result_type generated by the generator. It must be of integral +// type; a static_assert ensures this is the case. +// +// The implementation is based on W.Hormann, G.Derflinger: +// +// "Rejection-Inversion to Generate Variates from Monotone Discrete +// Distributions" +// +// http://eeyore.wu-wien.ac.at/papers/96-04-04.wh-der.ps.gz +// +template <typename IntType = int> +class zipf_distribution { + public: + using result_type = IntType; + + class param_type { + public: + using distribution_type = zipf_distribution; + + // Preconditions: k > 0, v > 0, q > 1 + // The precondidtions are validated when NDEBUG is not defined via + // a pair of assert() directives. + // If NDEBUG is defined and either or both of these parameters take invalid + // values, the behavior of the class is undefined. + explicit param_type(result_type k = (std::numeric_limits<IntType>::max)(), + double q = 2.0, double v = 1.0); + + result_type k() const { return k_; } + double q() const { return q_; } + double v() const { return v_; } + + friend bool operator==(const param_type& a, const param_type& b) { + return a.k_ == b.k_ && a.q_ == b.q_ && a.v_ == b.v_; + } + friend bool operator!=(const param_type& a, const param_type& b) { + return !(a == b); + } + + private: + friend class zipf_distribution; + inline double h(double x) const; + inline double hinv(double x) const; + inline double compute_s() const; + inline double pow_negative_q(double x) const; + + // Parameters here are exactly the same as the parameters of Algorithm ZRI + // in the paper. + IntType k_; + double q_; + double v_; + + double one_minus_q_; // 1-q + double s_; + double one_minus_q_inv_; // 1 / 1-q + double hxm_; // h(k + 0.5) + double hx0_minus_hxm_; // h(x0) - h(k + 0.5) + + static_assert(std::is_integral<IntType>::value, + "Class-template absl::zipf_distribution<> must be " + "parameterized using an integral type."); + }; + + zipf_distribution() + : zipf_distribution((std::numeric_limits<IntType>::max)()) {} + + explicit zipf_distribution(result_type k, double q = 2.0, double v = 1.0) + : param_(k, q, v) {} + + explicit zipf_distribution(const param_type& p) : param_(p) {} + + void reset() {} + + template <typename URBG> + result_type operator()(URBG& g) { // NOLINT(runtime/references) + return (*this)(g, param_); + } + + template <typename URBG> + result_type operator()(URBG& g, // NOLINT(runtime/references) + const param_type& p); + + result_type k() const { return param_.k(); } + double q() const { return param_.q(); } + double v() const { return param_.v(); } + + param_type param() const { return param_; } + void param(const param_type& p) { param_ = p; } + + result_type(min)() const { return 0; } + result_type(max)() const { return k(); } + + friend bool operator==(const zipf_distribution& a, + const zipf_distribution& b) { + return a.param_ == b.param_; + } + friend bool operator!=(const zipf_distribution& a, + const zipf_distribution& b) { + return a.param_ != b.param_; + } + + private: + param_type param_; +}; + +// -------------------------------------------------------------------------- +// Implementation details follow +// -------------------------------------------------------------------------- + +template <typename IntType> +zipf_distribution<IntType>::param_type::param_type( + typename zipf_distribution<IntType>::result_type k, double q, double v) + : k_(k), q_(q), v_(v), one_minus_q_(1 - q) { + assert(q > 1); + assert(v > 0); + assert(k > 0); + one_minus_q_inv_ = 1 / one_minus_q_; + + // Setup for the ZRI algorithm (pg 17 of the paper). + // Compute: h(i max) => h(k + 0.5) + constexpr double kMax = 18446744073709549568.0; + double kd = static_cast<double>(k); + // TODO(absl-team): Determine if this check is needed, and if so, add a test + // that fails for k > kMax + if (kd > kMax) { + // Ensure that our maximum value is capped to a value which will + // round-trip back through double. + kd = kMax; + } + hxm_ = h(kd + 0.5); + + // Compute: h(0) + const bool use_precomputed = (v == 1.0 && q == 2.0); + const double h0x5 = use_precomputed ? (-1.0 / 1.5) // exp(-log(1.5)) + : h(0.5); + const double elogv_q = (v_ == 1.0) ? 1 : pow_negative_q(v_); + + // h(0) = h(0.5) - exp(log(v) * -q) + hx0_minus_hxm_ = (h0x5 - elogv_q) - hxm_; + + // And s + s_ = use_precomputed ? 0.46153846153846123 : compute_s(); +} + +template <typename IntType> +double zipf_distribution<IntType>::param_type::h(double x) const { + // std::exp(one_minus_q_ * std::log(v_ + x)) * one_minus_q_inv_; + x += v_; + return (one_minus_q_ == -1.0) + ? (-1.0 / x) // -exp(-log(x)) + : (std::exp(std::log(x) * one_minus_q_) * one_minus_q_inv_); +} + +template <typename IntType> +double zipf_distribution<IntType>::param_type::hinv(double x) const { + // std::exp(one_minus_q_inv_ * std::log(one_minus_q_ * x)) - v_; + return -v_ + ((one_minus_q_ == -1.0) + ? (-1.0 / x) // exp(-log(-x)) + : std::exp(one_minus_q_inv_ * std::log(one_minus_q_ * x))); +} + +template <typename IntType> +double zipf_distribution<IntType>::param_type::compute_s() const { + // 1 - hinv(h(1.5) - std::exp(std::log(v_ + 1) * -q_)); + return 1.0 - hinv(h(1.5) - pow_negative_q(v_ + 1.0)); +} + +template <typename IntType> +double zipf_distribution<IntType>::param_type::pow_negative_q(double x) const { + // std::exp(std::log(x) * -q_); + return q_ == 2.0 ? (1.0 / (x * x)) : std::exp(std::log(x) * -q_); +} + +template <typename IntType> +template <typename URBG> +typename zipf_distribution<IntType>::result_type +zipf_distribution<IntType>::operator()( + URBG& g, const param_type& p) { // NOLINT(runtime/references) + absl::uniform_real_distribution<double> uniform_double; + double k; + for (;;) { + const double v = uniform_double(g); + const double u = p.hxm_ + v * p.hx0_minus_hxm_; + const double x = p.hinv(u); + k = rint(x); // std::floor(x + 0.5); + if (k > p.k()) continue; // reject k > max_k + if (k - x <= p.s_) break; + const double h = p.h(k + 0.5); + const double r = p.pow_negative_q(p.v_ + k); + if (u >= h - r) break; + } + IntType ki = static_cast<IntType>(k); + assert(ki <= p.k_); + return ki; +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_ostream<CharT, Traits>& operator<<( + std::basic_ostream<CharT, Traits>& os, // NOLINT(runtime/references) + const zipf_distribution<IntType>& x) { + using stream_type = + typename random_internal::stream_format_type<IntType>::type; + auto saver = random_internal::make_ostream_state_saver(os); + os.precision(random_internal::stream_precision_helper<double>::kPrecision); + os << static_cast<stream_type>(x.k()) << os.fill() << x.q() << os.fill() + << x.v(); + return os; +} + +template <typename CharT, typename Traits, typename IntType> +std::basic_istream<CharT, Traits>& operator>>( + std::basic_istream<CharT, Traits>& is, // NOLINT(runtime/references) + zipf_distribution<IntType>& x) { // NOLINT(runtime/references) + using result_type = typename zipf_distribution<IntType>::result_type; + using param_type = typename zipf_distribution<IntType>::param_type; + using stream_type = + typename random_internal::stream_format_type<IntType>::type; + stream_type k; + double q; + double v; + + auto saver = random_internal::make_istream_state_saver(is); + is >> k >> q >> v; + if (!is.fail()) { + x.param(param_type(static_cast<result_type>(k), q, v)); + } + return is; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_RANDOM_ZIPF_DISTRIBUTION_H_ diff --git a/third_party/abseil_cpp/absl/random/zipf_distribution_test.cc b/third_party/abseil_cpp/absl/random/zipf_distribution_test.cc new file mode 100644 index 000000000000..f8cf70e0dddf --- /dev/null +++ b/third_party/abseil_cpp/absl/random/zipf_distribution_test.cc @@ -0,0 +1,427 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/random/zipf_distribution.h" + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <iterator> +#include <random> +#include <string> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/internal/chi_square.h" +#include "absl/random/internal/pcg_engine.h" +#include "absl/random/internal/sequence_urbg.h" +#include "absl/random/random.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_replace.h" +#include "absl/strings/strip.h" + +namespace { + +using ::absl::random_internal::kChiSquared; +using ::testing::ElementsAre; + +template <typename IntType> +class ZipfDistributionTypedTest : public ::testing::Test {}; + +using IntTypes = ::testing::Types<int, int8_t, int16_t, int32_t, int64_t, + uint8_t, uint16_t, uint32_t, uint64_t>; +TYPED_TEST_CASE(ZipfDistributionTypedTest, IntTypes); + +TYPED_TEST(ZipfDistributionTypedTest, SerializeTest) { + using param_type = typename absl::zipf_distribution<TypeParam>::param_type; + + constexpr int kCount = 1000; + absl::InsecureBitGen gen; + for (const auto& param : { + param_type(), + param_type(32), + param_type(100, 3, 2), + param_type(std::numeric_limits<TypeParam>::max(), 4, 3), + param_type(std::numeric_limits<TypeParam>::max() / 2), + }) { + // Validate parameters. + const auto k = param.k(); + const auto q = param.q(); + const auto v = param.v(); + + absl::zipf_distribution<TypeParam> before(k, q, v); + EXPECT_EQ(before.k(), param.k()); + EXPECT_EQ(before.q(), param.q()); + EXPECT_EQ(before.v(), param.v()); + + { + absl::zipf_distribution<TypeParam> via_param(param); + EXPECT_EQ(via_param, before); + } + + // Validate stream serialization. + std::stringstream ss; + ss << before; + absl::zipf_distribution<TypeParam> after(4, 5.5, 4.4); + + EXPECT_NE(before.k(), after.k()); + EXPECT_NE(before.q(), after.q()); + EXPECT_NE(before.v(), after.v()); + EXPECT_NE(before.param(), after.param()); + EXPECT_NE(before, after); + + ss >> after; + + EXPECT_EQ(before.k(), after.k()); + EXPECT_EQ(before.q(), after.q()); + EXPECT_EQ(before.v(), after.v()); + EXPECT_EQ(before.param(), after.param()); + EXPECT_EQ(before, after); + + // Smoke test. + auto sample_min = after.max(); + auto sample_max = after.min(); + for (int i = 0; i < kCount; i++) { + auto sample = after(gen); + EXPECT_GE(sample, after.min()); + EXPECT_LE(sample, after.max()); + if (sample > sample_max) sample_max = sample; + if (sample < sample_min) sample_min = sample; + } + ABSL_INTERNAL_LOG(INFO, + absl::StrCat("Range: ", +sample_min, ", ", +sample_max)); + } +} + +class ZipfModel { + public: + ZipfModel(size_t k, double q, double v) : k_(k), q_(q), v_(v) {} + + double mean() const { return mean_; } + + // For the other moments of the Zipf distribution, see, for example, + // http://mathworld.wolfram.com/ZipfDistribution.html + + // PMF(k) = (1 / k^s) / H(N,s) + // Returns the probability that any single invocation returns k. + double PMF(size_t i) { return i >= hnq_.size() ? 0.0 : hnq_[i] / sum_hnq_; } + + // CDF = H(k, s) / H(N,s) + double CDF(size_t i) { + if (i >= hnq_.size()) { + return 1.0; + } + auto it = std::begin(hnq_); + double h = 0.0; + for (const auto end = it; it != end; it++) { + h += *it; + } + return h / sum_hnq_; + } + + // The InverseCDF returns the k values which bound p on the upper and lower + // bound. Since there is no closed-form solution, this is implemented as a + // bisction of the cdf. + std::pair<size_t, size_t> InverseCDF(double p) { + size_t min = 0; + size_t max = hnq_.size(); + while (max > min + 1) { + size_t target = (max + min) >> 1; + double x = CDF(target); + if (x > p) { + max = target; + } else { + min = target; + } + } + return {min, max}; + } + + // Compute the probability totals, which are based on the generalized harmonic + // number, H(N,s). + // H(N,s) == SUM(k=1..N, 1 / k^s) + // + // In the limit, H(N,s) == zetac(s) + 1. + // + // NOTE: The mean of a zipf distribution could be computed here as well. + // Mean := H(N, s-1) / H(N,s). + // Given the parameter v = 1, this gives the following function: + // (Hn(100, 1) - Hn(1,1)) / (Hn(100,2) - Hn(1,2)) = 6.5944 + // + void Init() { + if (!hnq_.empty()) { + return; + } + hnq_.clear(); + hnq_.reserve(std::min(k_, size_t{1000})); + + sum_hnq_ = 0; + double qm1 = q_ - 1.0; + double sum_hnq_m1 = 0; + for (size_t i = 0; i < k_; i++) { + // Partial n-th generalized harmonic number + const double x = v_ + i; + + // H(n, q-1) + const double hnqm1 = + (q_ == 2.0) ? (1.0 / x) + : (q_ == 3.0) ? (1.0 / (x * x)) : std::pow(x, -qm1); + sum_hnq_m1 += hnqm1; + + // H(n, q) + const double hnq = + (q_ == 2.0) ? (1.0 / (x * x)) + : (q_ == 3.0) ? (1.0 / (x * x * x)) : std::pow(x, -q_); + sum_hnq_ += hnq; + hnq_.push_back(hnq); + if (i > 1000 && hnq <= 1e-10) { + // The harmonic number is too small. + break; + } + } + assert(sum_hnq_ > 0); + mean_ = sum_hnq_m1 / sum_hnq_; + } + + private: + const size_t k_; + const double q_; + const double v_; + + double mean_; + std::vector<double> hnq_; + double sum_hnq_; +}; + +using zipf_u64 = absl::zipf_distribution<uint64_t>; + +class ZipfTest : public testing::TestWithParam<zipf_u64::param_type>, + public ZipfModel { + public: + ZipfTest() : ZipfModel(GetParam().k(), GetParam().q(), GetParam().v()) {} + + // We use a fixed bit generator for distribution accuracy tests. This allows + // these tests to be deterministic, while still testing the qualify of the + // implementation. + absl::random_internal::pcg64_2018_engine rng_{0x2B7E151628AED2A6}; +}; + +TEST_P(ZipfTest, ChiSquaredTest) { + const auto& param = GetParam(); + Init(); + + size_t trials = 10000; + + // Find the split-points for the buckets. + std::vector<size_t> points; + std::vector<double> expected; + { + double last_cdf = 0.0; + double min_p = 1.0; + for (double p = 0.01; p < 1.0; p += 0.01) { + auto x = InverseCDF(p); + if (points.empty() || points.back() < x.second) { + const double p = CDF(x.second); + points.push_back(x.second); + double q = p - last_cdf; + expected.push_back(q); + last_cdf = p; + if (q < min_p) { + min_p = q; + } + } + } + if (last_cdf < 0.999) { + points.push_back(std::numeric_limits<size_t>::max()); + double q = 1.0 - last_cdf; + expected.push_back(q); + if (q < min_p) { + min_p = q; + } + } else { + points.back() = std::numeric_limits<size_t>::max(); + expected.back() += (1.0 - last_cdf); + } + // The Chi-Squared score is not completely scale-invariant; it works best + // when the small values are in the small digits. + trials = static_cast<size_t>(8.0 / min_p); + } + ASSERT_GT(points.size(), 0); + + // Generate n variates and fill the counts vector with the count of their + // occurrences. + std::vector<int64_t> buckets(points.size(), 0); + double avg = 0; + { + zipf_u64 dis(param); + for (size_t i = 0; i < trials; i++) { + uint64_t x = dis(rng_); + ASSERT_LE(x, dis.max()); + ASSERT_GE(x, dis.min()); + avg += static_cast<double>(x); + auto it = std::upper_bound(std::begin(points), std::end(points), + static_cast<size_t>(x)); + buckets[std::distance(std::begin(points), it)]++; + } + avg = avg / static_cast<double>(trials); + } + + // Validate the output using the Chi-Squared test. + for (auto& e : expected) { + e *= trials; + } + + // The null-hypothesis is that the distribution is a poisson distribution with + // the provided mean (not estimated from the data). + const int dof = static_cast<int>(expected.size()) - 1; + + // NOTE: This test runs about 15x per invocation, so a value of 0.9995 is + // approximately correct for a test suite failure rate of 1 in 100. In + // practice we see failures slightly higher than that. + const double threshold = absl::random_internal::ChiSquareValue(dof, 0.9999); + + const double chi_square = absl::random_internal::ChiSquare( + std::begin(buckets), std::end(buckets), std::begin(expected), + std::end(expected)); + + const double p_actual = + absl::random_internal::ChiSquarePValue(chi_square, dof); + + // Log if the chi_squared value is above the threshold. + if (chi_square > threshold) { + ABSL_INTERNAL_LOG(INFO, "values"); + for (size_t i = 0; i < expected.size(); i++) { + ABSL_INTERNAL_LOG(INFO, absl::StrCat(points[i], ": ", buckets[i], + " vs. E=", expected[i])); + } + ABSL_INTERNAL_LOG(INFO, absl::StrCat("trials ", trials)); + ABSL_INTERNAL_LOG(INFO, + absl::StrCat("mean ", avg, " vs. expected ", mean())); + ABSL_INTERNAL_LOG(INFO, absl::StrCat(kChiSquared, "(data, ", dof, ") = ", + chi_square, " (", p_actual, ")")); + ABSL_INTERNAL_LOG(INFO, + absl::StrCat(kChiSquared, " @ 0.9995 = ", threshold)); + FAIL() << kChiSquared << " value of " << chi_square + << " is above the threshold."; + } +} + +std::vector<zipf_u64::param_type> GenParams() { + using param = zipf_u64::param_type; + const auto k = param().k(); + const auto q = param().q(); + const auto v = param().v(); + const uint64_t k2 = 1 << 10; + return std::vector<zipf_u64::param_type>{ + // Default + param(k, q, v), + // vary K + param(4, q, v), param(1 << 4, q, v), param(k2, q, v), + // vary V + param(k2, q, 0.5), param(k2, q, 1.5), param(k2, q, 2.5), param(k2, q, 10), + // vary Q + param(k2, 1.5, v), param(k2, 3, v), param(k2, 5, v), param(k2, 10, v), + // Vary V & Q + param(k2, 1.5, 0.5), param(k2, 3, 1.5), param(k, 10, 10)}; +} + +std::string ParamName( + const ::testing::TestParamInfo<zipf_u64::param_type>& info) { + const auto& p = info.param; + std::string name = absl::StrCat("k_", p.k(), "__q_", absl::SixDigits(p.q()), + "__v_", absl::SixDigits(p.v())); + return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); +} + +INSTANTIATE_TEST_SUITE_P(All, ZipfTest, ::testing::ValuesIn(GenParams()), + ParamName); + +// NOTE: absl::zipf_distribution is not guaranteed to be stable. +TEST(ZipfDistributionTest, StabilityTest) { + // absl::zipf_distribution stability relies on + // absl::uniform_real_distribution, std::log, std::exp, std::log1p + absl::random_internal::sequence_urbg urbg( + {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, + 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, + 0xECDD4775619F1510ull, 0x13CCA830EB61BD96ull, 0x0334FE1EAA0363CFull, + 0xB5735C904C70A239ull, 0xD59E9E0BCBAADE14ull, 0xEECC86BC60622CA7ull}); + + std::vector<int> output(10); + + { + absl::zipf_distribution<int32_t> dist; + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + EXPECT_THAT(output, ElementsAre(10031, 0, 0, 3, 6, 0, 7, 47, 0, 0)); + } + urbg.reset(); + { + absl::zipf_distribution<int32_t> dist(std::numeric_limits<int32_t>::max(), + 3.3); + std::generate(std::begin(output), std::end(output), + [&] { return dist(urbg); }); + EXPECT_THAT(output, ElementsAre(44, 0, 0, 0, 0, 1, 0, 1, 3, 0)); + } +} + +TEST(ZipfDistributionTest, AlgorithmBounds) { + absl::zipf_distribution<int32_t> dist; + + // Small values from absl::uniform_real_distribution map to larger Zipf + // distribution values. + const std::pair<uint64_t, int32_t> kInputs[] = { + {0xffffffffffffffff, 0x0}, {0x7fffffffffffffff, 0x0}, + {0x3ffffffffffffffb, 0x1}, {0x1ffffffffffffffd, 0x4}, + {0xffffffffffffffe, 0x9}, {0x7ffffffffffffff, 0x12}, + {0x3ffffffffffffff, 0x25}, {0x1ffffffffffffff, 0x4c}, + {0xffffffffffffff, 0x99}, {0x7fffffffffffff, 0x132}, + {0x3fffffffffffff, 0x265}, {0x1fffffffffffff, 0x4cc}, + {0xfffffffffffff, 0x999}, {0x7ffffffffffff, 0x1332}, + {0x3ffffffffffff, 0x2665}, {0x1ffffffffffff, 0x4ccc}, + {0xffffffffffff, 0x9998}, {0x7fffffffffff, 0x1332f}, + {0x3fffffffffff, 0x2665a}, {0x1fffffffffff, 0x4cc9e}, + {0xfffffffffff, 0x998e0}, {0x7ffffffffff, 0x133051}, + {0x3ffffffffff, 0x265ae4}, {0x1ffffffffff, 0x4c9ed3}, + {0xffffffffff, 0x98e223}, {0x7fffffffff, 0x13058c4}, + {0x3fffffffff, 0x25b178e}, {0x1fffffffff, 0x4a062b2}, + {0xfffffffff, 0x8ee23b8}, {0x7ffffffff, 0x10b21642}, + {0x3ffffffff, 0x1d89d89d}, {0x1ffffffff, 0x2fffffff}, + {0xffffffff, 0x45d1745d}, {0x7fffffff, 0x5a5a5a5a}, + {0x3fffffff, 0x69ee5846}, {0x1fffffff, 0x73ecade3}, + {0xfffffff, 0x79a9d260}, {0x7ffffff, 0x7cc0532b}, + {0x3ffffff, 0x7e5ad146}, {0x1ffffff, 0x7f2c0bec}, + {0xffffff, 0x7f95adef}, {0x7fffff, 0x7fcac0da}, + {0x3fffff, 0x7fe55ae2}, {0x1fffff, 0x7ff2ac0e}, + {0xfffff, 0x7ff955ae}, {0x7ffff, 0x7ffcaac1}, + {0x3ffff, 0x7ffe555b}, {0x1ffff, 0x7fff2aac}, + {0xffff, 0x7fff9556}, {0x7fff, 0x7fffcaab}, + {0x3fff, 0x7fffe555}, {0x1fff, 0x7ffff2ab}, + {0xfff, 0x7ffff955}, {0x7ff, 0x7ffffcab}, + {0x3ff, 0x7ffffe55}, {0x1ff, 0x7fffff2b}, + {0xff, 0x7fffff95}, {0x7f, 0x7fffffcb}, + {0x3f, 0x7fffffe5}, {0x1f, 0x7ffffff3}, + {0xf, 0x7ffffff9}, {0x7, 0x7ffffffd}, + {0x3, 0x7ffffffe}, {0x1, 0x7fffffff}, + }; + + for (const auto& instance : kInputs) { + absl::random_internal::sequence_urbg urbg({instance.first}); + EXPECT_EQ(instance.second, dist(urbg)); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/status/BUILD.bazel b/third_party/abseil_cpp/absl/status/BUILD.bazel new file mode 100644 index 000000000000..189bd73d0e46 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/BUILD.bazel @@ -0,0 +1,103 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This package contains `absl::Status`. +# It will expand later to have utilities around `Status` like `StatusOr`, +# `StatusBuilder` and macros. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "status", + srcs = [ + "internal/status_internal.h", + "status.cc", + "status_payload_printer.cc", + ], + hdrs = [ + "status.h", + "status_payload_printer.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + "//absl/base:atomic_hook", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/container:inlined_vector", + "//absl/debugging:stacktrace", + "//absl/debugging:symbolize", + "//absl/strings", + "//absl/strings:cord", + "//absl/strings:str_format", + "//absl/types:optional", + ], +) + +cc_test( + name = "status_test", + srcs = ["status_test.cc"], + copts = ABSL_TEST_COPTS, + deps = [ + ":status", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "statusor", + srcs = [ + "internal/statusor_internal.h", + "statusor.cc", + ], + hdrs = [ + "statusor.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":status", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/types:variant", + "//absl/utility", + ], +) + +cc_test( + name = "statusor_test", + size = "small", + srcs = ["statusor_test.cc"], + deps = [ + ":status", + ":statusor", + "//absl/base", + "//absl/memory", + "//absl/types:any", + "//absl/utility", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/status/CMakeLists.txt b/third_party/abseil_cpp/absl/status/CMakeLists.txt new file mode 100644 index 000000000000..f0d798a3732d --- /dev/null +++ b/third_party/abseil_cpp/absl/status/CMakeLists.txt @@ -0,0 +1,88 @@ +# +# Copyright 2020 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +absl_cc_library( + NAME + status + HDRS + "status.h" + SRCS + "internal/status_internal.h" + "status.cc" + "status_payload_printer.h" + "status_payload_printer.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::atomic_hook + absl::config + absl::core_headers + absl::raw_logging_internal + absl::inlined_vector + absl::stacktrace + absl::symbolize + absl::strings + absl::cord + absl::str_format + absl::optional + PUBLIC +) + +absl_cc_test( + NAME + status_test + SRCS + "status_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::status + absl::strings + gmock_main +) + +absl_cc_library( + NAME + statusor + HDRS + "statusor.h" + SRCS + "statusor.cc" + "internal/statusor_internal.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::status + absl::core_headers + absl::raw_logging_internal + absl::type_traits + absl::strings + absl::utility + absl::variant + PUBLIC +) + +absl_cc_test( + NAME + statusor_test + SRCS + "statusor_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::status + absl::statusor + gmock_main +) diff --git a/third_party/abseil_cpp/absl/status/internal/status_internal.h b/third_party/abseil_cpp/absl/status/internal/status_internal.h new file mode 100644 index 000000000000..279f8f55bef0 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/internal/status_internal.h @@ -0,0 +1,58 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef ABSL_STATUS_INTERNAL_STATUS_INTERNAL_H_ +#define ABSL_STATUS_INTERNAL_STATUS_INTERNAL_H_ + +#include <string> + +#include "absl/container/inlined_vector.h" +#include "absl/strings/cord.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +enum class StatusCode : int; + +namespace status_internal { + +// Container for status payloads. +struct Payload { + std::string type_url; + absl::Cord payload; +}; + +using Payloads = absl::InlinedVector<Payload, 1>; + +// Reference-counted representation of Status data. +struct StatusRep { + StatusRep(absl::StatusCode code, std::string message, + std::unique_ptr<status_internal::Payloads> payloads) + : ref(int32_t{1}), + code(code), + message(std::move(message)), + payloads(std::move(payloads)) {} + + std::atomic<int32_t> ref; + absl::StatusCode code; + std::string message; + std::unique_ptr<status_internal::Payloads> payloads; +}; + +absl::StatusCode MapToLocalCode(int value); +} // namespace status_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STATUS_INTERNAL_STATUS_INTERNAL_H_ diff --git a/third_party/abseil_cpp/absl/status/internal/statusor_internal.h b/third_party/abseil_cpp/absl/status/internal/statusor_internal.h new file mode 100644 index 000000000000..eaac2c0b14c6 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/internal/statusor_internal.h @@ -0,0 +1,396 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef ABSL_STATUS_INTERNAL_STATUSOR_INTERNAL_H_ +#define ABSL_STATUS_INTERNAL_STATUSOR_INTERNAL_H_ + +#include <type_traits> +#include <utility> + +#include "absl/base/attributes.h" +#include "absl/meta/type_traits.h" +#include "absl/status/status.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +template <typename T> +class ABSL_MUST_USE_RESULT StatusOr; + +namespace internal_statusor { + +// Detects whether `U` has conversion operator to `StatusOr<T>`, i.e. `operator +// StatusOr<T>()`. +template <typename T, typename U, typename = void> +struct HasConversionOperatorToStatusOr : std::false_type {}; + +template <typename T, typename U> +void test(char (*)[sizeof(std::declval<U>().operator absl::StatusOr<T>())]); + +template <typename T, typename U> +struct HasConversionOperatorToStatusOr<T, U, decltype(test<T, U>(0))> + : std::true_type {}; + +// Detects whether `T` is constructible or convertible from `StatusOr<U>`. +template <typename T, typename U> +using IsConstructibleOrConvertibleFromStatusOr = + absl::disjunction<std::is_constructible<T, StatusOr<U>&>, + std::is_constructible<T, const StatusOr<U>&>, + std::is_constructible<T, StatusOr<U>&&>, + std::is_constructible<T, const StatusOr<U>&&>, + std::is_convertible<StatusOr<U>&, T>, + std::is_convertible<const StatusOr<U>&, T>, + std::is_convertible<StatusOr<U>&&, T>, + std::is_convertible<const StatusOr<U>&&, T>>; + +// Detects whether `T` is constructible or convertible or assignable from +// `StatusOr<U>`. +template <typename T, typename U> +using IsConstructibleOrConvertibleOrAssignableFromStatusOr = + absl::disjunction<IsConstructibleOrConvertibleFromStatusOr<T, U>, + std::is_assignable<T&, StatusOr<U>&>, + std::is_assignable<T&, const StatusOr<U>&>, + std::is_assignable<T&, StatusOr<U>&&>, + std::is_assignable<T&, const StatusOr<U>&&>>; + +// Detects whether direct initializing `StatusOr<T>` from `U` is ambiguous, i.e. +// when `U` is `StatusOr<V>` and `T` is constructible or convertible from `V`. +template <typename T, typename U> +struct IsDirectInitializationAmbiguous + : public absl::conditional_t< + std::is_same<absl::remove_cv_t<absl::remove_reference_t<U>>, + U>::value, + std::false_type, + IsDirectInitializationAmbiguous< + T, absl::remove_cv_t<absl::remove_reference_t<U>>>> {}; + +template <typename T, typename V> +struct IsDirectInitializationAmbiguous<T, absl::StatusOr<V>> + : public IsConstructibleOrConvertibleFromStatusOr<T, V> {}; + +// Checks against the constraints of the direction initialization, i.e. when +// `StatusOr<T>::StatusOr(U&&)` should participate in overload resolution. +template <typename T, typename U> +using IsDirectInitializationValid = absl::disjunction< + // Short circuits if T is basically U. + std::is_same<T, absl::remove_cv_t<absl::remove_reference_t<U>>>, + absl::negation<absl::disjunction< + std::is_same<absl::StatusOr<T>, + absl::remove_cv_t<absl::remove_reference_t<U>>>, + std::is_same<absl::Status, + absl::remove_cv_t<absl::remove_reference_t<U>>>, + std::is_same<absl::in_place_t, + absl::remove_cv_t<absl::remove_reference_t<U>>>, + IsDirectInitializationAmbiguous<T, U>>>>; + +// This trait detects whether `StatusOr<T>::operator=(U&&)` is ambiguous, which +// is equivalent to whether all the following conditions are met: +// 1. `U` is `StatusOr<V>`. +// 2. `T` is constructible and assignable from `V`. +// 3. `T` is constructible and assignable from `U` (i.e. `StatusOr<V>`). +// For example, the following code is considered ambiguous: +// (`T` is `bool`, `U` is `StatusOr<bool>`, `V` is `bool`) +// StatusOr<bool> s1 = true; // s1.ok() && s1.ValueOrDie() == true +// StatusOr<bool> s2 = false; // s2.ok() && s2.ValueOrDie() == false +// s1 = s2; // ambiguous, `s1 = s2.ValueOrDie()` or `s1 = bool(s2)`? +template <typename T, typename U> +struct IsForwardingAssignmentAmbiguous + : public absl::conditional_t< + std::is_same<absl::remove_cv_t<absl::remove_reference_t<U>>, + U>::value, + std::false_type, + IsForwardingAssignmentAmbiguous< + T, absl::remove_cv_t<absl::remove_reference_t<U>>>> {}; + +template <typename T, typename U> +struct IsForwardingAssignmentAmbiguous<T, absl::StatusOr<U>> + : public IsConstructibleOrConvertibleOrAssignableFromStatusOr<T, U> {}; + +// Checks against the constraints of the forwarding assignment, i.e. whether +// `StatusOr<T>::operator(U&&)` should participate in overload resolution. +template <typename T, typename U> +using IsForwardingAssignmentValid = absl::disjunction< + // Short circuits if T is basically U. + std::is_same<T, absl::remove_cv_t<absl::remove_reference_t<U>>>, + absl::negation<absl::disjunction< + std::is_same<absl::StatusOr<T>, + absl::remove_cv_t<absl::remove_reference_t<U>>>, + std::is_same<absl::Status, + absl::remove_cv_t<absl::remove_reference_t<U>>>, + std::is_same<absl::in_place_t, + absl::remove_cv_t<absl::remove_reference_t<U>>>, + IsForwardingAssignmentAmbiguous<T, U>>>>; + +class Helper { + public: + // Move type-agnostic error handling to the .cc. + static void HandleInvalidStatusCtorArg(Status*); + ABSL_ATTRIBUTE_NORETURN static void Crash(const absl::Status& status); +}; + +// Construct an instance of T in `p` through placement new, passing Args... to +// the constructor. +// This abstraction is here mostly for the gcc performance fix. +template <typename T, typename... Args> +ABSL_ATTRIBUTE_NONNULL(1) void PlacementNew(void* p, Args&&... args) { + new (p) T(std::forward<Args>(args)...); +} + +// Helper base class to hold the data and all operations. +// We move all this to a base class to allow mixing with the appropriate +// TraitsBase specialization. +template <typename T> +class StatusOrData { + template <typename U> + friend class StatusOrData; + + public: + StatusOrData() = delete; + + StatusOrData(const StatusOrData& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + StatusOrData(StatusOrData&& other) noexcept { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(std::move(other.status_)); + } + } + + template <typename U> + explicit StatusOrData(const StatusOrData<U>& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + template <typename U> + explicit StatusOrData(StatusOrData<U>&& other) { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(std::move(other.status_)); + } + } + + template <typename... Args> + explicit StatusOrData(absl::in_place_t, Args&&... args) + : data_(std::forward<Args>(args)...) { + MakeStatus(); + } + + explicit StatusOrData(const T& value) : data_(value) { + MakeStatus(); + } + explicit StatusOrData(T&& value) : data_(std::move(value)) { + MakeStatus(); + } + + template <typename U, + absl::enable_if_t<std::is_constructible<absl::Status, U&&>::value, + int> = 0> + explicit StatusOrData(U&& v) : status_(std::forward<U>(v)) { + EnsureNotOk(); + } + + StatusOrData& operator=(const StatusOrData& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(other.data_); + else + AssignStatus(other.status_); + return *this; + } + + StatusOrData& operator=(StatusOrData&& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(std::move(other.data_)); + else + AssignStatus(std::move(other.status_)); + return *this; + } + + ~StatusOrData() { + if (ok()) { + status_.~Status(); + data_.~T(); + } else { + status_.~Status(); + } + } + + template <typename U> + void Assign(U&& value) { + if (ok()) { + data_ = std::forward<U>(value); + } else { + MakeValue(std::forward<U>(value)); + status_ = OkStatus(); + } + } + + template <typename U> + void AssignStatus(U&& v) { + Clear(); + status_ = static_cast<absl::Status>(std::forward<U>(v)); + EnsureNotOk(); + } + + bool ok() const { return status_.ok(); } + + protected: + // status_ will always be active after the constructor. + // We make it a union to be able to initialize exactly how we need without + // waste. + // Eg. in the copy constructor we use the default constructor of Status in + // the ok() path to avoid an extra Ref call. + union { + Status status_; + }; + + // data_ is active iff status_.ok()==true + struct Dummy {}; + union { + // When T is const, we need some non-const object we can cast to void* for + // the placement new. dummy_ is that object. + Dummy dummy_; + T data_; + }; + + void Clear() { + if (ok()) data_.~T(); + } + + void EnsureOk() const { + if (ABSL_PREDICT_FALSE(!ok())) Helper::Crash(status_); + } + + void EnsureNotOk() { + if (ABSL_PREDICT_FALSE(ok())) Helper::HandleInvalidStatusCtorArg(&status_); + } + + // Construct the value (ie. data_) through placement new with the passed + // argument. + template <typename... Arg> + void MakeValue(Arg&&... arg) { + internal_statusor::PlacementNew<T>(&dummy_, std::forward<Arg>(arg)...); + } + + // Construct the status (ie. status_) through placement new with the passed + // argument. + template <typename... Args> + void MakeStatus(Args&&... args) { + internal_statusor::PlacementNew<Status>(&status_, + std::forward<Args>(args)...); + } +}; + +// Helper base classes to allow implicitly deleted constructors and assignment +// operators in `StatusOr`. For example, `CopyCtorBase` will explicitly delete +// the copy constructor when T is not copy constructible and `StatusOr` will +// inherit that behavior implicitly. +template <typename T, bool = std::is_copy_constructible<T>::value> +struct CopyCtorBase { + CopyCtorBase() = default; + CopyCtorBase(const CopyCtorBase&) = default; + CopyCtorBase(CopyCtorBase&&) = default; + CopyCtorBase& operator=(const CopyCtorBase&) = default; + CopyCtorBase& operator=(CopyCtorBase&&) = default; +}; + +template <typename T> +struct CopyCtorBase<T, false> { + CopyCtorBase() = default; + CopyCtorBase(const CopyCtorBase&) = delete; + CopyCtorBase(CopyCtorBase&&) = default; + CopyCtorBase& operator=(const CopyCtorBase&) = default; + CopyCtorBase& operator=(CopyCtorBase&&) = default; +}; + +template <typename T, bool = std::is_move_constructible<T>::value> +struct MoveCtorBase { + MoveCtorBase() = default; + MoveCtorBase(const MoveCtorBase&) = default; + MoveCtorBase(MoveCtorBase&&) = default; + MoveCtorBase& operator=(const MoveCtorBase&) = default; + MoveCtorBase& operator=(MoveCtorBase&&) = default; +}; + +template <typename T> +struct MoveCtorBase<T, false> { + MoveCtorBase() = default; + MoveCtorBase(const MoveCtorBase&) = default; + MoveCtorBase(MoveCtorBase&&) = delete; + MoveCtorBase& operator=(const MoveCtorBase&) = default; + MoveCtorBase& operator=(MoveCtorBase&&) = default; +}; + +template <typename T, bool = std::is_copy_constructible<T>::value&& + std::is_copy_assignable<T>::value> +struct CopyAssignBase { + CopyAssignBase() = default; + CopyAssignBase(const CopyAssignBase&) = default; + CopyAssignBase(CopyAssignBase&&) = default; + CopyAssignBase& operator=(const CopyAssignBase&) = default; + CopyAssignBase& operator=(CopyAssignBase&&) = default; +}; + +template <typename T> +struct CopyAssignBase<T, false> { + CopyAssignBase() = default; + CopyAssignBase(const CopyAssignBase&) = default; + CopyAssignBase(CopyAssignBase&&) = default; + CopyAssignBase& operator=(const CopyAssignBase&) = delete; + CopyAssignBase& operator=(CopyAssignBase&&) = default; +}; + +template <typename T, bool = std::is_move_constructible<T>::value&& + std::is_move_assignable<T>::value> +struct MoveAssignBase { + MoveAssignBase() = default; + MoveAssignBase(const MoveAssignBase&) = default; + MoveAssignBase(MoveAssignBase&&) = default; + MoveAssignBase& operator=(const MoveAssignBase&) = default; + MoveAssignBase& operator=(MoveAssignBase&&) = default; +}; + +template <typename T> +struct MoveAssignBase<T, false> { + MoveAssignBase() = default; + MoveAssignBase(const MoveAssignBase&) = default; + MoveAssignBase(MoveAssignBase&&) = default; + MoveAssignBase& operator=(const MoveAssignBase&) = default; + MoveAssignBase& operator=(MoveAssignBase&&) = delete; +}; + +ABSL_ATTRIBUTE_NORETURN void ThrowBadStatusOrAccess(absl::Status status); + +} // namespace internal_statusor +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STATUS_INTERNAL_STATUSOR_INTERNAL_H_ diff --git a/third_party/abseil_cpp/absl/status/status.cc b/third_party/abseil_cpp/absl/status/status.cc new file mode 100644 index 000000000000..c71de846827c --- /dev/null +++ b/third_party/abseil_cpp/absl/status/status.cc @@ -0,0 +1,442 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/status/status.h" + +#include <cassert> + +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/status/status_payload_printer.h" +#include "absl/strings/escaping.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_split.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +std::string StatusCodeToString(StatusCode code) { + switch (code) { + case StatusCode::kOk: + return "OK"; + case StatusCode::kCancelled: + return "CANCELLED"; + case StatusCode::kUnknown: + return "UNKNOWN"; + case StatusCode::kInvalidArgument: + return "INVALID_ARGUMENT"; + case StatusCode::kDeadlineExceeded: + return "DEADLINE_EXCEEDED"; + case StatusCode::kNotFound: + return "NOT_FOUND"; + case StatusCode::kAlreadyExists: + return "ALREADY_EXISTS"; + case StatusCode::kPermissionDenied: + return "PERMISSION_DENIED"; + case StatusCode::kUnauthenticated: + return "UNAUTHENTICATED"; + case StatusCode::kResourceExhausted: + return "RESOURCE_EXHAUSTED"; + case StatusCode::kFailedPrecondition: + return "FAILED_PRECONDITION"; + case StatusCode::kAborted: + return "ABORTED"; + case StatusCode::kOutOfRange: + return "OUT_OF_RANGE"; + case StatusCode::kUnimplemented: + return "UNIMPLEMENTED"; + case StatusCode::kInternal: + return "INTERNAL"; + case StatusCode::kUnavailable: + return "UNAVAILABLE"; + case StatusCode::kDataLoss: + return "DATA_LOSS"; + default: + return ""; + } +} + +std::ostream& operator<<(std::ostream& os, StatusCode code) { + return os << StatusCodeToString(code); +} + +namespace status_internal { + +static int FindPayloadIndexByUrl(const Payloads* payloads, + absl::string_view type_url) { + if (payloads == nullptr) return -1; + + for (size_t i = 0; i < payloads->size(); ++i) { + if ((*payloads)[i].type_url == type_url) return i; + } + + return -1; +} + +// Convert canonical code to a value known to this binary. +absl::StatusCode MapToLocalCode(int value) { + absl::StatusCode code = static_cast<absl::StatusCode>(value); + switch (code) { + case absl::StatusCode::kOk: + case absl::StatusCode::kCancelled: + case absl::StatusCode::kUnknown: + case absl::StatusCode::kInvalidArgument: + case absl::StatusCode::kDeadlineExceeded: + case absl::StatusCode::kNotFound: + case absl::StatusCode::kAlreadyExists: + case absl::StatusCode::kPermissionDenied: + case absl::StatusCode::kResourceExhausted: + case absl::StatusCode::kFailedPrecondition: + case absl::StatusCode::kAborted: + case absl::StatusCode::kOutOfRange: + case absl::StatusCode::kUnimplemented: + case absl::StatusCode::kInternal: + case absl::StatusCode::kUnavailable: + case absl::StatusCode::kDataLoss: + case absl::StatusCode::kUnauthenticated: + return code; + default: + return absl::StatusCode::kUnknown; + } +} +} // namespace status_internal + +absl::optional<absl::Cord> Status::GetPayload( + absl::string_view type_url) const { + const auto* payloads = GetPayloads(); + int index = status_internal::FindPayloadIndexByUrl(payloads, type_url); + if (index != -1) return (*payloads)[index].payload; + + return absl::nullopt; +} + +void Status::SetPayload(absl::string_view type_url, absl::Cord payload) { + if (ok()) return; + + PrepareToModify(); + + status_internal::StatusRep* rep = RepToPointer(rep_); + if (!rep->payloads) { + rep->payloads = absl::make_unique<status_internal::Payloads>(); + } + + int index = + status_internal::FindPayloadIndexByUrl(rep->payloads.get(), type_url); + if (index != -1) { + (*rep->payloads)[index].payload = std::move(payload); + return; + } + + rep->payloads->push_back({std::string(type_url), std::move(payload)}); +} + +bool Status::ErasePayload(absl::string_view type_url) { + int index = status_internal::FindPayloadIndexByUrl(GetPayloads(), type_url); + if (index != -1) { + PrepareToModify(); + GetPayloads()->erase(GetPayloads()->begin() + index); + if (GetPayloads()->empty() && message().empty()) { + // Special case: If this can be represented inlined, it MUST be + // inlined (EqualsSlow depends on this behavior). + StatusCode c = static_cast<StatusCode>(raw_code()); + Unref(rep_); + rep_ = CodeToInlinedRep(c); + } + return true; + } + + return false; +} + +void Status::ForEachPayload( + const std::function<void(absl::string_view, const absl::Cord&)>& visitor) + const { + if (auto* payloads = GetPayloads()) { + bool in_reverse = + payloads->size() > 1 && reinterpret_cast<uintptr_t>(payloads) % 13 > 6; + + for (size_t index = 0; index < payloads->size(); ++index) { + const auto& elem = + (*payloads)[in_reverse ? payloads->size() - 1 - index : index]; + +#ifdef NDEBUG + visitor(elem.type_url, elem.payload); +#else + // In debug mode invalidate the type url to prevent users from relying on + // this string lifetime. + + // NOLINTNEXTLINE intentional extra conversion to force temporary. + visitor(std::string(elem.type_url), elem.payload); +#endif // NDEBUG + } + } +} + +const std::string* Status::EmptyString() { + static std::string* empty_string = new std::string(); + return empty_string; +} + +constexpr const char Status::kMovedFromString[]; + +const std::string* Status::MovedFromString() { + static std::string* moved_from_string = new std::string(kMovedFromString); + return moved_from_string; +} + +void Status::UnrefNonInlined(uintptr_t rep) { + status_internal::StatusRep* r = RepToPointer(rep); + // Fast path: if ref==1, there is no need for a RefCountDec (since + // this is the only reference and therefore no other thread is + // allowed to be mucking with r). + if (r->ref.load(std::memory_order_acquire) == 1 || + r->ref.fetch_sub(1, std::memory_order_acq_rel) - 1 == 0) { + delete r; + } +} + +uintptr_t Status::NewRep(absl::StatusCode code, absl::string_view msg, + std::unique_ptr<status_internal::Payloads> payloads) { + status_internal::StatusRep* rep = new status_internal::StatusRep( + code, std::string(msg.data(), msg.size()), std::move(payloads)); + return PointerToRep(rep); +} + +Status::Status(absl::StatusCode code, absl::string_view msg) + : rep_(CodeToInlinedRep(code)) { + if (code != absl::StatusCode::kOk && !msg.empty()) { + rep_ = NewRep(code, msg, nullptr); + } +} + +int Status::raw_code() const { + if (IsInlined(rep_)) { + return static_cast<int>(InlinedRepToCode(rep_)); + } + status_internal::StatusRep* rep = RepToPointer(rep_); + return static_cast<int>(rep->code); +} + +absl::StatusCode Status::code() const { + return status_internal::MapToLocalCode(raw_code()); +} + +void Status::PrepareToModify() { + ABSL_RAW_CHECK(!ok(), "PrepareToModify shouldn't be called on OK status."); + if (IsInlined(rep_)) { + rep_ = NewRep(static_cast<absl::StatusCode>(raw_code()), + absl::string_view(), nullptr); + return; + } + + uintptr_t rep_i = rep_; + status_internal::StatusRep* rep = RepToPointer(rep_); + if (rep->ref.load(std::memory_order_acquire) != 1) { + std::unique_ptr<status_internal::Payloads> payloads; + if (rep->payloads) { + payloads = absl::make_unique<status_internal::Payloads>(*rep->payloads); + } + rep_ = NewRep(rep->code, message(), std::move(payloads)); + UnrefNonInlined(rep_i); + } +} + +bool Status::EqualsSlow(const absl::Status& a, const absl::Status& b) { + if (IsInlined(a.rep_) != IsInlined(b.rep_)) return false; + if (a.message() != b.message()) return false; + if (a.raw_code() != b.raw_code()) return false; + if (a.GetPayloads() == b.GetPayloads()) return true; + + const status_internal::Payloads no_payloads; + const status_internal::Payloads* larger_payloads = + a.GetPayloads() ? a.GetPayloads() : &no_payloads; + const status_internal::Payloads* smaller_payloads = + b.GetPayloads() ? b.GetPayloads() : &no_payloads; + if (larger_payloads->size() < smaller_payloads->size()) { + std::swap(larger_payloads, smaller_payloads); + } + if ((larger_payloads->size() - smaller_payloads->size()) > 1) return false; + // Payloads can be ordered differently, so we can't just compare payload + // vectors. + for (const auto& payload : *larger_payloads) { + + bool found = false; + for (const auto& other_payload : *smaller_payloads) { + if (payload.type_url == other_payload.type_url) { + if (payload.payload != other_payload.payload) { + return false; + } + found = true; + break; + } + } + if (!found) return false; + } + return true; +} + +std::string Status::ToStringSlow() const { + std::string text; + absl::StrAppend(&text, absl::StatusCodeToString(code()), ": ", message()); + status_internal::StatusPayloadPrinter printer = + status_internal::GetStatusPayloadPrinter(); + this->ForEachPayload([&](absl::string_view type_url, + const absl::Cord& payload) { + absl::optional<std::string> result; + if (printer) result = printer(type_url, payload); + absl::StrAppend( + &text, " [", type_url, "='", + result.has_value() ? *result : absl::CHexEscape(std::string(payload)), + "']"); + }); + + return text; +} + +std::ostream& operator<<(std::ostream& os, const Status& x) { + os << x.ToString(); + return os; +} + +Status AbortedError(absl::string_view message) { + return Status(absl::StatusCode::kAborted, message); +} + +Status AlreadyExistsError(absl::string_view message) { + return Status(absl::StatusCode::kAlreadyExists, message); +} + +Status CancelledError(absl::string_view message) { + return Status(absl::StatusCode::kCancelled, message); +} + +Status DataLossError(absl::string_view message) { + return Status(absl::StatusCode::kDataLoss, message); +} + +Status DeadlineExceededError(absl::string_view message) { + return Status(absl::StatusCode::kDeadlineExceeded, message); +} + +Status FailedPreconditionError(absl::string_view message) { + return Status(absl::StatusCode::kFailedPrecondition, message); +} + +Status InternalError(absl::string_view message) { + return Status(absl::StatusCode::kInternal, message); +} + +Status InvalidArgumentError(absl::string_view message) { + return Status(absl::StatusCode::kInvalidArgument, message); +} + +Status NotFoundError(absl::string_view message) { + return Status(absl::StatusCode::kNotFound, message); +} + +Status OutOfRangeError(absl::string_view message) { + return Status(absl::StatusCode::kOutOfRange, message); +} + +Status PermissionDeniedError(absl::string_view message) { + return Status(absl::StatusCode::kPermissionDenied, message); +} + +Status ResourceExhaustedError(absl::string_view message) { + return Status(absl::StatusCode::kResourceExhausted, message); +} + +Status UnauthenticatedError(absl::string_view message) { + return Status(absl::StatusCode::kUnauthenticated, message); +} + +Status UnavailableError(absl::string_view message) { + return Status(absl::StatusCode::kUnavailable, message); +} + +Status UnimplementedError(absl::string_view message) { + return Status(absl::StatusCode::kUnimplemented, message); +} + +Status UnknownError(absl::string_view message) { + return Status(absl::StatusCode::kUnknown, message); +} + +bool IsAborted(const Status& status) { + return status.code() == absl::StatusCode::kAborted; +} + +bool IsAlreadyExists(const Status& status) { + return status.code() == absl::StatusCode::kAlreadyExists; +} + +bool IsCancelled(const Status& status) { + return status.code() == absl::StatusCode::kCancelled; +} + +bool IsDataLoss(const Status& status) { + return status.code() == absl::StatusCode::kDataLoss; +} + +bool IsDeadlineExceeded(const Status& status) { + return status.code() == absl::StatusCode::kDeadlineExceeded; +} + +bool IsFailedPrecondition(const Status& status) { + return status.code() == absl::StatusCode::kFailedPrecondition; +} + +bool IsInternal(const Status& status) { + return status.code() == absl::StatusCode::kInternal; +} + +bool IsInvalidArgument(const Status& status) { + return status.code() == absl::StatusCode::kInvalidArgument; +} + +bool IsNotFound(const Status& status) { + return status.code() == absl::StatusCode::kNotFound; +} + +bool IsOutOfRange(const Status& status) { + return status.code() == absl::StatusCode::kOutOfRange; +} + +bool IsPermissionDenied(const Status& status) { + return status.code() == absl::StatusCode::kPermissionDenied; +} + +bool IsResourceExhausted(const Status& status) { + return status.code() == absl::StatusCode::kResourceExhausted; +} + +bool IsUnauthenticated(const Status& status) { + return status.code() == absl::StatusCode::kUnauthenticated; +} + +bool IsUnavailable(const Status& status) { + return status.code() == absl::StatusCode::kUnavailable; +} + +bool IsUnimplemented(const Status& status) { + return status.code() == absl::StatusCode::kUnimplemented; +} + +bool IsUnknown(const Status& status) { + return status.code() == absl::StatusCode::kUnknown; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/status/status.h b/third_party/abseil_cpp/absl/status/status.h new file mode 100644 index 000000000000..c4d6fce09050 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/status.h @@ -0,0 +1,817 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: status.h +// ----------------------------------------------------------------------------- +// +// This header file defines the Abseil `status` library, consisting of: +// +// * An `absl::Status` class for holding error handling information +// * A set of canonical `absl::StatusCode` error codes, and associated +// utilities for generating and propagating status codes. +// * A set of helper functions for creating status codes and checking their +// values +// +// Within Google, `absl::Status` is the primary mechanism for gracefully +// handling errors across API boundaries (and in particular across RPC +// boundaries). Some of these errors may be recoverable, but others may not. +// Most functions that can produce a recoverable error should be designed to +// return an `absl::Status` (or `absl::StatusOr`). +// +// Example: +// +// absl::Status myFunction(absl::string_view fname, ...) { +// ... +// // encounter error +// if (error condition) { +// return absl::InvalidArgumentError("bad mode"); +// } +// // else, return OK +// return absl::OkStatus(); +// } +// +// An `absl::Status` is designed to either return "OK" or one of a number of +// different error codes, corresponding to typical error conditions. +// In almost all cases, when using `absl::Status` you should use the canonical +// error codes (of type `absl::StatusCode`) enumerated in this header file. +// These canonical codes are understood across the codebase and will be +// accepted across all API and RPC boundaries. +#ifndef ABSL_STATUS_STATUS_H_ +#define ABSL_STATUS_STATUS_H_ + +#include <iostream> +#include <string> + +#include "absl/container/inlined_vector.h" +#include "absl/status/internal/status_internal.h" +#include "absl/strings/cord.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::StatusCode +// +// An `absl::StatusCode` is an enumerated type indicating either no error ("OK") +// or an error condition. In most cases, an `absl::Status` indicates a +// recoverable error, and the purpose of signalling an error is to indicate what +// action to take in response to that error. These error codes map to the proto +// RPC error codes indicated in https://cloud.google.com/apis/design/errors. +// +// The errors listed below are the canonical errors associated with +// `absl::Status` and are used throughout the codebase. As a result, these +// error codes are somewhat generic. +// +// In general, try to return the most specific error that applies if more than +// one error may pertain. For example, prefer `kOutOfRange` over +// `kFailedPrecondition` if both codes apply. Similarly prefer `kNotFound` or +// `kAlreadyExists` over `kFailedPrecondition`. +// +// Because these errors may travel RPC boundaries, these codes are tied to the +// `google.rpc.Code` definitions within +// https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto +// The string value of these RPC codes is denoted within each enum below. +// +// If your error handling code requires more context, you can attach payloads +// to your status. See `absl::Status::SetPayload()` and +// `absl::Status::GetPayload()` below. +enum class StatusCode : int { + // StatusCode::kOk + // + // kOK (gRPC code "OK") does not indicate an error; this value is returned on + // success. It is typical to check for this value before proceeding on any + // given call across an API or RPC boundary. To check this value, use the + // `absl::Status::ok()` member function rather than inspecting the raw code. + kOk = 0, + + // StatusCode::kCancelled + // + // kCancelled (gRPC code "CANCELLED") indicates the operation was cancelled, + // typically by the caller. + kCancelled = 1, + + // StatusCode::kUnknown + // + // kUnknown (gRPC code "UNKNOWN") indicates an unknown error occurred. In + // general, more specific errors should be raised, if possible. Errors raised + // by APIs that do not return enough error information may be converted to + // this error. + kUnknown = 2, + + // StatusCode::kInvalidArgument + // + // kInvalidArgument (gRPC code "INVALID_ARGUMENT") indicates the caller + // specified an invalid argument, such a malformed filename. Note that such + // errors should be narrowly limited to indicate to the invalid nature of the + // arguments themselves. Errors with validly formed arguments that may cause + // errors with the state of the receiving system should be denoted with + // `kFailedPrecondition` instead. + kInvalidArgument = 3, + + // StatusCode::kDeadlineExceeded + // + // kDeadlineExceeded (gRPC code "DEADLINE_EXCEEDED") indicates a deadline + // expired before the operation could complete. For operations that may change + // state within a system, this error may be returned even if the operation has + // completed successfully. For example, a successful response from a server + // could have been delayed long enough for the deadline to expire. + kDeadlineExceeded = 4, + + // StatusCode::kNotFound + // + // kNotFound (gRPC code "NOT_FOUND") indicates some requested entity (such as + // a file or directory) was not found. + // + // `kNotFound` is useful if a request should be denied for an entire class of + // users, such as during a gradual feature rollout or undocumented allow list. + // If, instead, a request should be denied for specific sets of users, such as + // through user-based access control, use `kPermissionDenied` instead. + kNotFound = 5, + + // StatusCode::kAlreadyExists + // + // kAlreadyExists (gRPC code "ALREADY_EXISTS") indicates the entity that a + // caller attempted to create (such as file or directory) is already present. + kAlreadyExists = 6, + + // StatusCode::kPermissionDenied + // + // kPermissionDenied (gRPC code "PERMISSION_DENIED") indicates that the caller + // does not have permission to execute the specified operation. Note that this + // error is different than an error due to an *un*authenticated user. This + // error code does not imply the request is valid or the requested entity + // exists or satisfies any other pre-conditions. + // + // `kPermissionDenied` must not be used for rejections caused by exhausting + // some resource. Instead, use `kResourceExhausted` for those errors. + // `kPermissionDenied` must not be used if the caller cannot be identified. + // Instead, use `kUnauthenticated` for those errors. + kPermissionDenied = 7, + + // StatusCode::kResourceExhausted + // + // kResourceExhausted (gRPC code "RESOURCE_EXHAUSTED") indicates some resource + // has been exhausted, perhaps a per-user quota, or perhaps the entire file + // system is out of space. + kResourceExhausted = 8, + + // StatusCode::kFailedPrecondition + // + // kFailedPrecondition (gRPC code "FAILED_PRECONDITION") indicates that the + // operation was rejected because the system is not in a state required for + // the operation's execution. For example, a directory to be deleted may be + // non-empty, an "rmdir" operation is applied to a non-directory, etc. + // + // Some guidelines that may help a service implementer in deciding between + // `kFailedPrecondition`, `kAborted`, and `kUnavailable`: + // + // (a) Use `kUnavailable` if the client can retry just the failing call. + // (b) Use `kAborted` if the client should retry at a higher transaction + // level (such as when a client-specified test-and-set fails, indicating + // the client should restart a read-modify-write sequence). + // (c) Use `kFailedPrecondition` if the client should not retry until + // the system state has been explicitly fixed. For example, if an "rmdir" + // fails because the directory is non-empty, `kFailedPrecondition` + // should be returned since the client should not retry unless + // the files are deleted from the directory. + kFailedPrecondition = 9, + + // StatusCode::kAborted + // + // kAborted (gRPC code "ABORTED") indicates the operation was aborted, + // typically due to a concurrency issue such as a sequencer check failure or a + // failed transaction. + // + // See the guidelines above for deciding between `kFailedPrecondition`, + // `kAborted`, and `kUnavailable`. + kAborted = 10, + + // StatusCode::kOutOfRange + // + // kOutOfRange (gRPC code "OUT_OF_RANGE") indicates the operation was + // attempted past the valid range, such as seeking or reading past an + // end-of-file. + // + // Unlike `kInvalidArgument`, this error indicates a problem that may + // be fixed if the system state changes. For example, a 32-bit file + // system will generate `kInvalidArgument` if asked to read at an + // offset that is not in the range [0,2^32-1], but it will generate + // `kOutOfRange` if asked to read from an offset past the current + // file size. + // + // There is a fair bit of overlap between `kFailedPrecondition` and + // `kOutOfRange`. We recommend using `kOutOfRange` (the more specific + // error) when it applies so that callers who are iterating through + // a space can easily look for an `kOutOfRange` error to detect when + // they are done. + kOutOfRange = 11, + + // StatusCode::kUnimplemented + // + // kUnimplemented (gRPC code "UNIMPLEMENTED") indicates the operation is not + // implemented or supported in this service. In this case, the operation + // should not be re-attempted. + kUnimplemented = 12, + + // StatusCode::kInternal + // + // kInternal (gRPC code "INTERNAL") indicates an internal error has occurred + // and some invariants expected by the underlying system have not been + // satisfied. This error code is reserved for serious errors. + kInternal = 13, + + // StatusCode::kUnavailable + // + // kUnavailable (gRPC code "UNAVAILABLE") indicates the service is currently + // unavailable and that this is most likely a transient condition. An error + // such as this can be corrected by retrying with a backoff scheme. Note that + // it is not always safe to retry non-idempotent operations. + // + // See the guidelines above for deciding between `kFailedPrecondition`, + // `kAborted`, and `kUnavailable`. + kUnavailable = 14, + + // StatusCode::kDataLoss + // + // kDataLoss (gRPC code "DATA_LOSS") indicates that unrecoverable data loss or + // corruption has occurred. As this error is serious, proper alerting should + // be attached to errors such as this. + kDataLoss = 15, + + // StatusCode::kUnauthenticated + // + // kUnauthenticated (gRPC code "UNAUTHENTICATED") indicates that the request + // does not have valid authentication credentials for the operation. Correct + // the authentication and try again. + kUnauthenticated = 16, + + // StatusCode::DoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ + // + // NOTE: this error code entry should not be used and you should not rely on + // its value, which may change. + // + // The purpose of this enumerated value is to force people who handle status + // codes with `switch()` statements to *not* simply enumerate all possible + // values, but instead provide a "default:" case. Providing such a default + // case ensures that code will compile when new codes are added. + kDoNotUseReservedForFutureExpansionUseDefaultInSwitchInstead_ = 20 +}; + +// StatusCodeToString() +// +// Returns the name for the status code, or "" if it is an unknown value. +std::string StatusCodeToString(StatusCode code); + +// operator<< +// +// Streams StatusCodeToString(code) to `os`. +std::ostream& operator<<(std::ostream& os, StatusCode code); + +// absl::Status +// +// The `absl::Status` class is generally used to gracefully handle errors +// across API boundaries (and in particular across RPC boundaries). Some of +// these errors may be recoverable, but others may not. Most +// functions which can produce a recoverable error should be designed to return +// either an `absl::Status` (or the similar `absl::StatusOr<T>`, which holds +// either an object of type `T` or an error). +// +// API developers should construct their functions to return `absl::OkStatus()` +// upon success, or an `absl::StatusCode` upon another type of error (e.g +// an `absl::StatusCode::kInvalidArgument` error). The API provides convenience +// functions to constuct each status code. +// +// Example: +// +// absl::Status myFunction(absl::string_view fname, ...) { +// ... +// // encounter error +// if (error condition) { +// // Construct an absl::StatusCode::kInvalidArgument error +// return absl::InvalidArgumentError("bad mode"); +// } +// // else, return OK +// return absl::OkStatus(); +// } +// +// Users handling status error codes should prefer checking for an OK status +// using the `ok()` member function. Handling multiple error codes may justify +// use of switch statement, but only check for error codes you know how to +// handle; do not try to exhaustively match against all canonical error codes. +// Errors that cannot be handled should be logged and/or propagated for higher +// levels to deal with. If you do use a switch statement, make sure that you +// also provide a `default:` switch case, so that code does not break as other +// canonical codes are added to the API. +// +// Example: +// +// absl::Status result = DoSomething(); +// if (!result.ok()) { +// LOG(ERROR) << result; +// } +// +// // Provide a default if switching on multiple error codes +// switch (result.code()) { +// // The user hasn't authenticated. Ask them to reauth +// case absl::StatusCode::kUnauthenticated: +// DoReAuth(); +// break; +// // The user does not have permission. Log an error. +// case absl::StatusCode::kPermissionDenied: +// LOG(ERROR) << result; +// break; +// // Propagate the error otherwise. +// default: +// return true; +// } +// +// An `absl::Status` can optionally include a payload with more information +// about the error. Typically, this payload serves one of several purposes: +// +// * It may provide more fine-grained semantic information about the error to +// facilitate actionable remedies. +// * It may provide human-readable contexual information that is more +// appropriate to display to an end user. +// +// Example: +// +// absl::Status result = DoSomething(); +// // Inform user to retry after 30 seconds +// // See more error details in googleapis/google/rpc/error_details.proto +// if (absl::IsResourceExhausted(result)) { +// google::rpc::RetryInfo info; +// info.retry_delay().seconds() = 30; +// // Payloads require a unique key (a URL to ensure no collisions with +// // other payloads), and an `absl::Cord` to hold the encoded data. +// absl::string_view url = "type.googleapis.com/google.rpc.RetryInfo"; +// result.SetPayload(url, info.SerializeAsCord()); +// return result; +// } +// +class ABSL_MUST_USE_RESULT Status final { + public: + // Constructors + + // This default constructor creates an OK status with no message or payload. + // Avoid this constructor and prefer explicit construction of an OK status + // with `absl::OkStatus()`. + Status(); + + // Creates a status in the canonical error space with the specified + // `absl::StatusCode` and error message. If `code == absl::StatusCode::kOk`, + // `msg` is ignored and an object identical to an OK status is constructed. + // + // The `msg` string must be in UTF-8. The implementation may complain (e.g., + // by printing a warning) if it is not. + Status(absl::StatusCode code, absl::string_view msg); + + Status(const Status&); + Status& operator=(const Status& x); + + // Move operators + + // The moved-from state is valid but unspecified. + Status(Status&&) noexcept; + Status& operator=(Status&&); + + ~Status(); + + // Status::Update() + // + // Updates the existing status with `new_status` provided that `this->ok()`. + // If the existing status already contains a non-OK error, this update has no + // effect and preserves the current data. Note that this behavior may change + // in the future to augment a current non-ok status with additional + // information about `new_status`. + // + // `Update()` provides a convenient way of keeping track of the first error + // encountered. + // + // Example: + // // Instead of "if (overall_status.ok()) overall_status = new_status" + // overall_status.Update(new_status); + // + void Update(const Status& new_status); + void Update(Status&& new_status); + + // Status::ok() + // + // Returns `true` if `this->ok()`. Prefer checking for an OK status using this + // member function. + ABSL_MUST_USE_RESULT bool ok() const; + + // Status::code() + // + // Returns the canonical error code of type `absl::StatusCode` of this status. + absl::StatusCode code() const; + + // Status::raw_code() + // + // Returns a raw (canonical) error code corresponding to the enum value of + // `google.rpc.Code` definitions within + // https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto. + // These values could be out of the range of canonical `absl::StatusCode` + // enum values. + // + // NOTE: This function should only be called when converting to an associated + // wire format. Use `Status::code()` for error handling. + int raw_code() const; + + // Status::message() + // + // Returns the error message associated with this error code, if available. + // Note that this message rarely describes the error code. It is not unusual + // for the error message to be the empty string. As a result, prefer + // `Status::ToString()` for debug logging. + absl::string_view message() const; + + friend bool operator==(const Status&, const Status&); + friend bool operator!=(const Status&, const Status&); + + // Status::ToString() + // + // Returns a combination of the error code name, the message and any + // associated payload messages. This string is designed simply to be human + // readable and its exact format should not be load bearing. Do not depend on + // the exact format of the result of `ToString()` which is subject to change. + // + // The printed code name and the message are generally substrings of the + // result, and the payloads to be printed use the status payload printer + // mechanism (which is internal). + std::string ToString() const; + + // Status::IgnoreError() + // + // Ignores any errors. This method does nothing except potentially suppress + // complaints from any tools that are checking that errors are not dropped on + // the floor. + void IgnoreError() const; + + // swap() + // + // Swap the contents of one status with another. + friend void swap(Status& a, Status& b); + + //---------------------------------------------------------------------------- + // Payload Management APIs + //---------------------------------------------------------------------------- + + // A payload may be attached to a status to provide additional context to an + // error that may not be satisifed by an existing `absl::StatusCode`. + // Typically, this payload serves one of several purposes: + // + // * It may provide more fine-grained semantic information about the error + // to facilitate actionable remedies. + // * It may provide human-readable contexual information that is more + // appropriate to display to an end user. + // + // A payload consists of a [key,value] pair, where the key is a string + // referring to a unique "type URL" and the value is an object of type + // `absl::Cord` to hold the contextual data. + // + // The "type URL" should be unique and follow the format of a URL + // (https://en.wikipedia.org/wiki/URL) and, ideally, provide some + // documentation or schema on how to interpret its associated data. For + // example, the default type URL for a protobuf message type is + // "type.googleapis.com/packagename.messagename". Other custom wire formats + // should define the format of type URL in a similar practice so as to + // minimize the chance of conflict between type URLs. + // Users should ensure that the type URL can be mapped to a concrete + // C++ type if they want to deserialize the payload and read it effectively. + // + // To attach a payload to a status object, call `Status::SetPayload()`, + // passing it the type URL and an `absl::Cord` of associated data. Similarly, + // to extract the payload from a status, call `Status::GetPayload()`. You + // may attach multiple payloads (with differing type URLs) to any given + // status object, provided that the status is currently exhibiting an error + // code (i.e. is not OK). + + // Status::GetPayload() + // + // Gets the payload of a status given its unique `type_url` key, if present. + absl::optional<absl::Cord> GetPayload(absl::string_view type_url) const; + + // Status::SetPayload() + // + // Sets the payload for a non-ok status using a `type_url` key, overwriting + // any existing payload for that `type_url`. + // + // NOTE: This function does nothing if the Status is ok. + void SetPayload(absl::string_view type_url, absl::Cord payload); + + // Status::ErasePayload() + // + // Erases the payload corresponding to the `type_url` key. Returns `true` if + // the payload was present. + bool ErasePayload(absl::string_view type_url); + + // Status::ForEachPayload() + // + // Iterates over the stored payloads and calls the + // `visitor(type_key, payload)` callable for each one. + // + // NOTE: The order of calls to `visitor()` is not specified and may change at + // any time. + // + // NOTE: Any mutation on the same 'absl::Status' object during visitation is + // forbidden and could result in undefined behavior. + void ForEachPayload( + const std::function<void(absl::string_view, const absl::Cord&)>& visitor) + const; + + private: + friend Status CancelledError(); + + // Creates a status in the canonical error space with the specified + // code, and an empty error message. + explicit Status(absl::StatusCode code); + + static void UnrefNonInlined(uintptr_t rep); + static void Ref(uintptr_t rep); + static void Unref(uintptr_t rep); + + // REQUIRES: !ok() + // Ensures rep_ is not shared with any other Status. + void PrepareToModify(); + + const status_internal::Payloads* GetPayloads() const; + status_internal::Payloads* GetPayloads(); + + // Takes ownership of payload. + static uintptr_t NewRep(absl::StatusCode code, absl::string_view msg, + std::unique_ptr<status_internal::Payloads> payload); + static bool EqualsSlow(const absl::Status& a, const absl::Status& b); + + // MSVC 14.0 limitation requires the const. + static constexpr const char kMovedFromString[] = + "Status accessed after move."; + + static const std::string* EmptyString(); + static const std::string* MovedFromString(); + + // Returns whether rep contains an inlined representation. + // See rep_ for details. + static bool IsInlined(uintptr_t rep); + + // Indicates whether this Status was the rhs of a move operation. See rep_ + // for details. + static bool IsMovedFrom(uintptr_t rep); + static uintptr_t MovedFromRep(); + + // Convert between error::Code and the inlined uintptr_t representation used + // by rep_. See rep_ for details. + static uintptr_t CodeToInlinedRep(absl::StatusCode code); + static absl::StatusCode InlinedRepToCode(uintptr_t rep); + + // Converts between StatusRep* and the external uintptr_t representation used + // by rep_. See rep_ for details. + static uintptr_t PointerToRep(status_internal::StatusRep* r); + static status_internal::StatusRep* RepToPointer(uintptr_t r); + + // Returns string for non-ok Status. + std::string ToStringSlow() const; + + // Status supports two different representations. + // - When the low bit is off it is an inlined representation. + // It uses the canonical error space, no message or payload. + // The error code is (rep_ >> 2). + // The (rep_ & 2) bit is the "moved from" indicator, used in IsMovedFrom(). + // - When the low bit is on it is an external representation. + // In this case all the data comes from a heap allocated Rep object. + // (rep_ - 1) is a status_internal::StatusRep* pointer to that structure. + uintptr_t rep_; +}; + +// OkStatus() +// +// Returns an OK status, equivalent to a default constructed instance. Prefer +// usage of `absl::OkStatus()` when constructing such an OK status. +Status OkStatus(); + +// operator<<() +// +// Prints a human-readable representation of `x` to `os`. +std::ostream& operator<<(std::ostream& os, const Status& x); + +// IsAborted() +// IsAlreadyExists() +// IsCancelled() +// IsDataLoss() +// IsDeadlineExceeded() +// IsFailedPrecondition() +// IsInternal() +// IsInvalidArgument() +// IsNotFound() +// IsOutOfRange() +// IsPermissionDenied() +// IsResourceExhausted() +// IsUnauthenticated() +// IsUnavailable() +// IsUnimplemented() +// IsUnknown() +// +// These convenience functions return `true` if a given status matches the +// `absl::StatusCode` error code of its associated function. +ABSL_MUST_USE_RESULT bool IsAborted(const Status& status); +ABSL_MUST_USE_RESULT bool IsAlreadyExists(const Status& status); +ABSL_MUST_USE_RESULT bool IsCancelled(const Status& status); +ABSL_MUST_USE_RESULT bool IsDataLoss(const Status& status); +ABSL_MUST_USE_RESULT bool IsDeadlineExceeded(const Status& status); +ABSL_MUST_USE_RESULT bool IsFailedPrecondition(const Status& status); +ABSL_MUST_USE_RESULT bool IsInternal(const Status& status); +ABSL_MUST_USE_RESULT bool IsInvalidArgument(const Status& status); +ABSL_MUST_USE_RESULT bool IsNotFound(const Status& status); +ABSL_MUST_USE_RESULT bool IsOutOfRange(const Status& status); +ABSL_MUST_USE_RESULT bool IsPermissionDenied(const Status& status); +ABSL_MUST_USE_RESULT bool IsResourceExhausted(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnauthenticated(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnavailable(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnimplemented(const Status& status); +ABSL_MUST_USE_RESULT bool IsUnknown(const Status& status); + +// AbortedError() +// AlreadyExistsError() +// CancelledError() +// DataLossError() +// DeadlineExceededError() +// FailedPreconditionError() +// InternalError() +// InvalidArgumentError() +// NotFoundError() +// OutOfRangeError() +// PermissionDeniedError() +// ResourceExhaustedError() +// UnauthenticatedError() +// UnavailableError() +// UnimplementedError() +// UnknownError() +// +// These convenience functions create an `absl::Status` object with an error +// code as indicated by the associated function name, using the error message +// passed in `message`. +Status AbortedError(absl::string_view message); +Status AlreadyExistsError(absl::string_view message); +Status CancelledError(absl::string_view message); +Status DataLossError(absl::string_view message); +Status DeadlineExceededError(absl::string_view message); +Status FailedPreconditionError(absl::string_view message); +Status InternalError(absl::string_view message); +Status InvalidArgumentError(absl::string_view message); +Status NotFoundError(absl::string_view message); +Status OutOfRangeError(absl::string_view message); +Status PermissionDeniedError(absl::string_view message); +Status ResourceExhaustedError(absl::string_view message); +Status UnauthenticatedError(absl::string_view message); +Status UnavailableError(absl::string_view message); +Status UnimplementedError(absl::string_view message); +Status UnknownError(absl::string_view message); + +//------------------------------------------------------------------------------ +// Implementation details follow +//------------------------------------------------------------------------------ + +inline Status::Status() : rep_(CodeToInlinedRep(absl::StatusCode::kOk)) {} + +inline Status::Status(absl::StatusCode code) : rep_(CodeToInlinedRep(code)) {} + +inline Status::Status(const Status& x) : rep_(x.rep_) { Ref(rep_); } + +inline Status& Status::operator=(const Status& x) { + uintptr_t old_rep = rep_; + if (x.rep_ != old_rep) { + Ref(x.rep_); + rep_ = x.rep_; + Unref(old_rep); + } + return *this; +} + +inline Status::Status(Status&& x) noexcept : rep_(x.rep_) { + x.rep_ = MovedFromRep(); +} + +inline Status& Status::operator=(Status&& x) { + uintptr_t old_rep = rep_; + rep_ = x.rep_; + x.rep_ = MovedFromRep(); + Unref(old_rep); + return *this; +} + +inline void Status::Update(const Status& new_status) { + if (ok()) { + *this = new_status; + } +} + +inline void Status::Update(Status&& new_status) { + if (ok()) { + *this = std::move(new_status); + } +} + +inline Status::~Status() { Unref(rep_); } + +inline bool Status::ok() const { + return rep_ == CodeToInlinedRep(absl::StatusCode::kOk); +} + +inline absl::string_view Status::message() const { + return !IsInlined(rep_) + ? RepToPointer(rep_)->message + : (IsMovedFrom(rep_) ? absl::string_view(kMovedFromString) + : absl::string_view()); +} + +inline bool operator==(const Status& lhs, const Status& rhs) { + return lhs.rep_ == rhs.rep_ || Status::EqualsSlow(lhs, rhs); +} + +inline bool operator!=(const Status& lhs, const Status& rhs) { + return !(lhs == rhs); +} + +inline std::string Status::ToString() const { + return ok() ? "OK" : ToStringSlow(); +} + +inline void Status::IgnoreError() const { + // no-op +} + +inline void swap(absl::Status& a, absl::Status& b) { + using std::swap; + swap(a.rep_, b.rep_); +} + +inline const status_internal::Payloads* Status::GetPayloads() const { + return IsInlined(rep_) ? nullptr : RepToPointer(rep_)->payloads.get(); +} + +inline status_internal::Payloads* Status::GetPayloads() { + return IsInlined(rep_) ? nullptr : RepToPointer(rep_)->payloads.get(); +} + +inline bool Status::IsInlined(uintptr_t rep) { return (rep & 1) == 0; } + +inline bool Status::IsMovedFrom(uintptr_t rep) { + return IsInlined(rep) && (rep & 2) != 0; +} + +inline uintptr_t Status::MovedFromRep() { + return CodeToInlinedRep(absl::StatusCode::kInternal) | 2; +} + +inline uintptr_t Status::CodeToInlinedRep(absl::StatusCode code) { + return static_cast<uintptr_t>(code) << 2; +} + +inline absl::StatusCode Status::InlinedRepToCode(uintptr_t rep) { + assert(IsInlined(rep)); + return static_cast<absl::StatusCode>(rep >> 2); +} + +inline status_internal::StatusRep* Status::RepToPointer(uintptr_t rep) { + assert(!IsInlined(rep)); + return reinterpret_cast<status_internal::StatusRep*>(rep - 1); +} + +inline uintptr_t Status::PointerToRep(status_internal::StatusRep* rep) { + return reinterpret_cast<uintptr_t>(rep) + 1; +} + +inline void Status::Ref(uintptr_t rep) { + if (!IsInlined(rep)) { + RepToPointer(rep)->ref.fetch_add(1, std::memory_order_relaxed); + } +} + +inline void Status::Unref(uintptr_t rep) { + if (!IsInlined(rep)) { + UnrefNonInlined(rep); + } +} + +inline Status OkStatus() { return Status(); } + +// Creates a `Status` object with the `absl::StatusCode::kCancelled` error code +// and an empty message. It is provided only for efficiency, given that +// message-less kCancelled errors are common in the infrastructure. +inline Status CancelledError() { return Status(absl::StatusCode::kCancelled); } + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STATUS_STATUS_H_ diff --git a/third_party/abseil_cpp/absl/status/status_payload_printer.cc b/third_party/abseil_cpp/absl/status/status_payload_printer.cc new file mode 100644 index 000000000000..a47aea11c2d3 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/status_payload_printer.cc @@ -0,0 +1,38 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/status/status_payload_printer.h" + +#include <atomic> + +#include "absl/base/attributes.h" +#include "absl/base/internal/atomic_hook.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace status_internal { + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +static absl::base_internal::AtomicHook<StatusPayloadPrinter> storage; + +void SetStatusPayloadPrinter(StatusPayloadPrinter printer) { + storage.Store(printer); +} + +StatusPayloadPrinter GetStatusPayloadPrinter() { + return storage.Load(); +} + +} // namespace status_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/status/status_payload_printer.h b/third_party/abseil_cpp/absl/status/status_payload_printer.h new file mode 100644 index 000000000000..5e0937f67da8 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/status_payload_printer.h @@ -0,0 +1,51 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef ABSL_STATUS_STATUS_PAYLOAD_PRINTER_H_ +#define ABSL_STATUS_STATUS_PAYLOAD_PRINTER_H_ + +#include <string> + +#include "absl/strings/cord.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace status_internal { + +// By default, `Status::ToString` and `operator<<(Status)` print a payload by +// dumping the type URL and the raw bytes. To help debugging, we provide an +// extension point, which is a global printer function that can be set by users +// to specify how to print payloads. The function takes the type URL and the +// payload as input, and should return a valid human-readable string on success +// or `absl::nullopt` on failure (in which case it falls back to the default +// approach of printing the raw bytes). +// NOTE: This is an internal API and the design is subject to change in the +// future in a non-backward-compatible way. Since it's only meant for debugging +// purpose, you should not rely on it in any critical logic. +using StatusPayloadPrinter = absl::optional<std::string> (*)(absl::string_view, + const absl::Cord&); + +// Sets the global payload printer. Only one printer should be set per process. +// If multiple printers are set, it's undefined which one will be used. +void SetStatusPayloadPrinter(StatusPayloadPrinter); + +// Returns the global payload printer if previously set, otherwise `nullptr`. +StatusPayloadPrinter GetStatusPayloadPrinter(); + +} // namespace status_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STATUS_STATUS_PAYLOAD_PRINTER_H_ diff --git a/third_party/abseil_cpp/absl/status/status_test.cc b/third_party/abseil_cpp/absl/status/status_test.cc new file mode 100644 index 000000000000..ca9488ad228c --- /dev/null +++ b/third_party/abseil_cpp/absl/status/status_test.cc @@ -0,0 +1,458 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/status/status.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" + +namespace { + +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Optional; +using ::testing::UnorderedElementsAreArray; + +TEST(StatusCode, InsertionOperator) { + const absl::StatusCode code = absl::StatusCode::kUnknown; + std::ostringstream oss; + oss << code; + EXPECT_EQ(oss.str(), absl::StatusCodeToString(code)); +} + +// This structure holds the details for testing a single error code, +// its creator, and its classifier. +struct ErrorTest { + absl::StatusCode code; + using Creator = absl::Status (*)(absl::string_view); + using Classifier = bool (*)(const absl::Status&); + Creator creator; + Classifier classifier; +}; + +constexpr ErrorTest kErrorTests[]{ + {absl::StatusCode::kCancelled, absl::CancelledError, absl::IsCancelled}, + {absl::StatusCode::kUnknown, absl::UnknownError, absl::IsUnknown}, + {absl::StatusCode::kInvalidArgument, absl::InvalidArgumentError, + absl::IsInvalidArgument}, + {absl::StatusCode::kDeadlineExceeded, absl::DeadlineExceededError, + absl::IsDeadlineExceeded}, + {absl::StatusCode::kNotFound, absl::NotFoundError, absl::IsNotFound}, + {absl::StatusCode::kAlreadyExists, absl::AlreadyExistsError, + absl::IsAlreadyExists}, + {absl::StatusCode::kPermissionDenied, absl::PermissionDeniedError, + absl::IsPermissionDenied}, + {absl::StatusCode::kResourceExhausted, absl::ResourceExhaustedError, + absl::IsResourceExhausted}, + {absl::StatusCode::kFailedPrecondition, absl::FailedPreconditionError, + absl::IsFailedPrecondition}, + {absl::StatusCode::kAborted, absl::AbortedError, absl::IsAborted}, + {absl::StatusCode::kOutOfRange, absl::OutOfRangeError, absl::IsOutOfRange}, + {absl::StatusCode::kUnimplemented, absl::UnimplementedError, + absl::IsUnimplemented}, + {absl::StatusCode::kInternal, absl::InternalError, absl::IsInternal}, + {absl::StatusCode::kUnavailable, absl::UnavailableError, + absl::IsUnavailable}, + {absl::StatusCode::kDataLoss, absl::DataLossError, absl::IsDataLoss}, + {absl::StatusCode::kUnauthenticated, absl::UnauthenticatedError, + absl::IsUnauthenticated}, +}; + +TEST(Status, CreateAndClassify) { + for (const auto& test : kErrorTests) { + SCOPED_TRACE(absl::StatusCodeToString(test.code)); + + // Ensure that the creator does, in fact, create status objects with the + // expected error code and message. + std::string message = + absl::StrCat("error code ", test.code, " test message"); + absl::Status status = test.creator(message); + EXPECT_EQ(test.code, status.code()); + EXPECT_EQ(message, status.message()); + + // Ensure that the classifier returns true for a status produced by the + // creator. + EXPECT_TRUE(test.classifier(status)); + + // Ensure that the classifier returns false for status with a different + // code. + for (const auto& other : kErrorTests) { + if (other.code != test.code) { + EXPECT_FALSE(test.classifier(absl::Status(other.code, ""))) + << " other.code = " << other.code; + } + } + } +} + +TEST(Status, DefaultConstructor) { + absl::Status status; + EXPECT_TRUE(status.ok()); + EXPECT_EQ(absl::StatusCode::kOk, status.code()); + EXPECT_EQ("", status.message()); +} + +TEST(Status, OkStatus) { + absl::Status status = absl::OkStatus(); + EXPECT_TRUE(status.ok()); + EXPECT_EQ(absl::StatusCode::kOk, status.code()); + EXPECT_EQ("", status.message()); +} + +TEST(Status, ConstructorWithCodeMessage) { + { + absl::Status status(absl::StatusCode::kCancelled, ""); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(absl::StatusCode::kCancelled, status.code()); + EXPECT_EQ("", status.message()); + } + { + absl::Status status(absl::StatusCode::kInternal, "message"); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(absl::StatusCode::kInternal, status.code()); + EXPECT_EQ("message", status.message()); + } +} + +TEST(Status, ConstructOutOfRangeCode) { + const int kRawCode = 9999; + absl::Status status(static_cast<absl::StatusCode>(kRawCode), ""); + EXPECT_EQ(absl::StatusCode::kUnknown, status.code()); + EXPECT_EQ(kRawCode, status.raw_code()); +} + +constexpr char kUrl1[] = "url.payload.1"; +constexpr char kUrl2[] = "url.payload.2"; +constexpr char kUrl3[] = "url.payload.3"; +constexpr char kUrl4[] = "url.payload.xx"; + +constexpr char kPayload1[] = "aaaaa"; +constexpr char kPayload2[] = "bbbbb"; +constexpr char kPayload3[] = "ccccc"; + +using PayloadsVec = std::vector<std::pair<std::string, absl::Cord>>; + +TEST(Status, TestGetSetPayload) { + absl::Status ok_status = absl::OkStatus(); + ok_status.SetPayload(kUrl1, absl::Cord(kPayload1)); + ok_status.SetPayload(kUrl2, absl::Cord(kPayload2)); + + EXPECT_FALSE(ok_status.GetPayload(kUrl1)); + EXPECT_FALSE(ok_status.GetPayload(kUrl2)); + + absl::Status bad_status(absl::StatusCode::kInternal, "fail"); + bad_status.SetPayload(kUrl1, absl::Cord(kPayload1)); + bad_status.SetPayload(kUrl2, absl::Cord(kPayload2)); + + EXPECT_THAT(bad_status.GetPayload(kUrl1), Optional(Eq(kPayload1))); + EXPECT_THAT(bad_status.GetPayload(kUrl2), Optional(Eq(kPayload2))); + + EXPECT_FALSE(bad_status.GetPayload(kUrl3)); + + bad_status.SetPayload(kUrl1, absl::Cord(kPayload3)); + EXPECT_THAT(bad_status.GetPayload(kUrl1), Optional(Eq(kPayload3))); + + // Testing dynamically generated type_url + bad_status.SetPayload(absl::StrCat(kUrl1, ".1"), absl::Cord(kPayload1)); + EXPECT_THAT(bad_status.GetPayload(absl::StrCat(kUrl1, ".1")), + Optional(Eq(kPayload1))); +} + +TEST(Status, TestErasePayload) { + absl::Status bad_status(absl::StatusCode::kInternal, "fail"); + bad_status.SetPayload(kUrl1, absl::Cord(kPayload1)); + bad_status.SetPayload(kUrl2, absl::Cord(kPayload2)); + bad_status.SetPayload(kUrl3, absl::Cord(kPayload3)); + + EXPECT_FALSE(bad_status.ErasePayload(kUrl4)); + + EXPECT_TRUE(bad_status.GetPayload(kUrl2)); + EXPECT_TRUE(bad_status.ErasePayload(kUrl2)); + EXPECT_FALSE(bad_status.GetPayload(kUrl2)); + EXPECT_FALSE(bad_status.ErasePayload(kUrl2)); + + EXPECT_TRUE(bad_status.ErasePayload(kUrl1)); + EXPECT_TRUE(bad_status.ErasePayload(kUrl3)); + + bad_status.SetPayload(kUrl1, absl::Cord(kPayload1)); + EXPECT_TRUE(bad_status.ErasePayload(kUrl1)); +} + +TEST(Status, TestComparePayloads) { + absl::Status bad_status1(absl::StatusCode::kInternal, "fail"); + bad_status1.SetPayload(kUrl1, absl::Cord(kPayload1)); + bad_status1.SetPayload(kUrl2, absl::Cord(kPayload2)); + bad_status1.SetPayload(kUrl3, absl::Cord(kPayload3)); + + absl::Status bad_status2(absl::StatusCode::kInternal, "fail"); + bad_status2.SetPayload(kUrl2, absl::Cord(kPayload2)); + bad_status2.SetPayload(kUrl3, absl::Cord(kPayload3)); + bad_status2.SetPayload(kUrl1, absl::Cord(kPayload1)); + + EXPECT_EQ(bad_status1, bad_status2); +} + +TEST(Status, TestComparePayloadsAfterErase) { + absl::Status payload_status(absl::StatusCode::kInternal, ""); + payload_status.SetPayload(kUrl1, absl::Cord(kPayload1)); + payload_status.SetPayload(kUrl2, absl::Cord(kPayload2)); + + absl::Status empty_status(absl::StatusCode::kInternal, ""); + + // Different payloads, not equal + EXPECT_NE(payload_status, empty_status); + EXPECT_TRUE(payload_status.ErasePayload(kUrl1)); + + // Still Different payloads, still not equal. + EXPECT_NE(payload_status, empty_status); + EXPECT_TRUE(payload_status.ErasePayload(kUrl2)); + + // Both empty payloads, should be equal + EXPECT_EQ(payload_status, empty_status); +} + +PayloadsVec AllVisitedPayloads(const absl::Status& s) { + PayloadsVec result; + + s.ForEachPayload([&](absl::string_view type_url, const absl::Cord& payload) { + result.push_back(std::make_pair(std::string(type_url), payload)); + }); + + return result; +} + +TEST(Status, TestForEachPayload) { + absl::Status bad_status(absl::StatusCode::kInternal, "fail"); + bad_status.SetPayload(kUrl1, absl::Cord(kPayload1)); + bad_status.SetPayload(kUrl2, absl::Cord(kPayload2)); + bad_status.SetPayload(kUrl3, absl::Cord(kPayload3)); + + int count = 0; + + bad_status.ForEachPayload( + [&count](absl::string_view, const absl::Cord&) { ++count; }); + + EXPECT_EQ(count, 3); + + PayloadsVec expected_payloads = {{kUrl1, absl::Cord(kPayload1)}, + {kUrl2, absl::Cord(kPayload2)}, + {kUrl3, absl::Cord(kPayload3)}}; + + // Test that we visit all the payloads in the status. + PayloadsVec visited_payloads = AllVisitedPayloads(bad_status); + EXPECT_THAT(visited_payloads, UnorderedElementsAreArray(expected_payloads)); + + // Test that visitation order is not consistent between run. + std::vector<absl::Status> scratch; + while (true) { + scratch.emplace_back(absl::StatusCode::kInternal, "fail"); + + scratch.back().SetPayload(kUrl1, absl::Cord(kPayload1)); + scratch.back().SetPayload(kUrl2, absl::Cord(kPayload2)); + scratch.back().SetPayload(kUrl3, absl::Cord(kPayload3)); + + if (AllVisitedPayloads(scratch.back()) != visited_payloads) { + break; + } + } +} + +TEST(Status, ToString) { + absl::Status s(absl::StatusCode::kInternal, "fail"); + EXPECT_EQ("INTERNAL: fail", s.ToString()); + s.SetPayload("foo", absl::Cord("bar")); + EXPECT_EQ("INTERNAL: fail [foo='bar']", s.ToString()); + s.SetPayload("bar", absl::Cord("\377")); + EXPECT_THAT(s.ToString(), + AllOf(HasSubstr("INTERNAL: fail"), HasSubstr("[foo='bar']"), + HasSubstr("[bar='\\xff']"))); +} + +absl::Status EraseAndReturn(const absl::Status& base) { + absl::Status copy = base; + EXPECT_TRUE(copy.ErasePayload(kUrl1)); + return copy; +} + +TEST(Status, CopyOnWriteForErasePayload) { + { + absl::Status base(absl::StatusCode::kInvalidArgument, "fail"); + base.SetPayload(kUrl1, absl::Cord(kPayload1)); + EXPECT_TRUE(base.GetPayload(kUrl1).has_value()); + absl::Status copy = EraseAndReturn(base); + EXPECT_TRUE(base.GetPayload(kUrl1).has_value()); + EXPECT_FALSE(copy.GetPayload(kUrl1).has_value()); + } + { + absl::Status base(absl::StatusCode::kInvalidArgument, "fail"); + base.SetPayload(kUrl1, absl::Cord(kPayload1)); + absl::Status copy = base; + + EXPECT_TRUE(base.GetPayload(kUrl1).has_value()); + EXPECT_TRUE(copy.GetPayload(kUrl1).has_value()); + + EXPECT_TRUE(base.ErasePayload(kUrl1)); + + EXPECT_FALSE(base.GetPayload(kUrl1).has_value()); + EXPECT_TRUE(copy.GetPayload(kUrl1).has_value()); + } +} + +TEST(Status, CopyConstructor) { + { + absl::Status status; + absl::Status copy(status); + EXPECT_EQ(copy, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + absl::Status copy(status); + EXPECT_EQ(copy, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + status.SetPayload(kUrl1, absl::Cord(kPayload1)); + absl::Status copy(status); + EXPECT_EQ(copy, status); + } +} + +TEST(Status, CopyAssignment) { + absl::Status assignee; + { + absl::Status status; + assignee = status; + EXPECT_EQ(assignee, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + assignee = status; + EXPECT_EQ(assignee, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + status.SetPayload(kUrl1, absl::Cord(kPayload1)); + assignee = status; + EXPECT_EQ(assignee, status); + } +} + +TEST(Status, CopyAssignmentIsNotRef) { + const absl::Status status_orig(absl::StatusCode::kInvalidArgument, "message"); + absl::Status status_copy = status_orig; + EXPECT_EQ(status_orig, status_copy); + status_copy.SetPayload(kUrl1, absl::Cord(kPayload1)); + EXPECT_NE(status_orig, status_copy); +} + +TEST(Status, MoveConstructor) { + { + absl::Status status; + absl::Status copy(absl::Status{}); + EXPECT_EQ(copy, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + absl::Status copy( + absl::Status(absl::StatusCode::kInvalidArgument, "message")); + EXPECT_EQ(copy, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + status.SetPayload(kUrl1, absl::Cord(kPayload1)); + absl::Status copy1(status); + absl::Status copy2(std::move(status)); + EXPECT_EQ(copy1, copy2); + } +} + +TEST(Status, MoveAssignment) { + absl::Status assignee; + { + absl::Status status; + assignee = absl::Status(); + EXPECT_EQ(assignee, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + assignee = absl::Status(absl::StatusCode::kInvalidArgument, "message"); + EXPECT_EQ(assignee, status); + } + { + absl::Status status(absl::StatusCode::kInvalidArgument, "message"); + status.SetPayload(kUrl1, absl::Cord(kPayload1)); + absl::Status copy(status); + assignee = std::move(status); + EXPECT_EQ(assignee, copy); + } +} + +TEST(Status, Update) { + absl::Status s; + s.Update(absl::OkStatus()); + EXPECT_TRUE(s.ok()); + const absl::Status a(absl::StatusCode::kCancelled, "message"); + s.Update(a); + EXPECT_EQ(s, a); + const absl::Status b(absl::StatusCode::kInternal, "other message"); + s.Update(b); + EXPECT_EQ(s, a); + s.Update(absl::OkStatus()); + EXPECT_EQ(s, a); + EXPECT_FALSE(s.ok()); +} + +TEST(Status, Equality) { + absl::Status ok; + absl::Status no_payload = absl::CancelledError("no payload"); + absl::Status one_payload = absl::InvalidArgumentError("one payload"); + one_payload.SetPayload(kUrl1, absl::Cord(kPayload1)); + absl::Status two_payloads = one_payload; + two_payloads.SetPayload(kUrl2, absl::Cord(kPayload2)); + const std::array<absl::Status, 4> status_arr = {ok, no_payload, one_payload, + two_payloads}; + for (int i = 0; i < status_arr.size(); i++) { + for (int j = 0; j < status_arr.size(); j++) { + if (i == j) { + EXPECT_TRUE(status_arr[i] == status_arr[j]); + EXPECT_FALSE(status_arr[i] != status_arr[j]); + } else { + EXPECT_TRUE(status_arr[i] != status_arr[j]); + EXPECT_FALSE(status_arr[i] == status_arr[j]); + } + } + } +} + +TEST(Status, Swap) { + auto test_swap = [](const absl::Status& s1, const absl::Status& s2) { + absl::Status copy1 = s1, copy2 = s2; + swap(copy1, copy2); + EXPECT_EQ(copy1, s2); + EXPECT_EQ(copy2, s1); + }; + const absl::Status ok; + const absl::Status no_payload(absl::StatusCode::kAlreadyExists, "no payload"); + absl::Status with_payload(absl::StatusCode::kInternal, "with payload"); + with_payload.SetPayload(kUrl1, absl::Cord(kPayload1)); + test_swap(ok, no_payload); + test_swap(no_payload, ok); + test_swap(ok, with_payload); + test_swap(with_payload, ok); + test_swap(no_payload, with_payload); + test_swap(with_payload, no_payload); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/status/statusor.cc b/third_party/abseil_cpp/absl/status/statusor.cc new file mode 100644 index 000000000000..b954b45e32c7 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/statusor.cc @@ -0,0 +1,71 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/status/statusor.h" + +#include <cstdlib> +#include <utility> + +#include "absl/base/internal/raw_logging.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +BadStatusOrAccess::BadStatusOrAccess(absl::Status status) + : status_(std::move(status)) {} + +BadStatusOrAccess::~BadStatusOrAccess() = default; +const char* BadStatusOrAccess::what() const noexcept { + return "Bad StatusOr access"; +} + +const absl::Status& BadStatusOrAccess::status() const { return status_; } + +namespace internal_statusor { + +void Helper::HandleInvalidStatusCtorArg(absl::Status* status) { + const char* kMessage = + "An OK status is not a valid constructor argument to StatusOr<T>"; +#ifdef NDEBUG + ABSL_INTERNAL_LOG(ERROR, kMessage); +#else + ABSL_INTERNAL_LOG(FATAL, kMessage); +#endif + // In optimized builds, we will fall back to InternalError. + *status = absl::InternalError(kMessage); +} + +void Helper::Crash(const absl::Status& status) { + ABSL_INTERNAL_LOG( + FATAL, + absl::StrCat("Attempting to fetch value instead of handling error ", + status.ToString())); +} + +void ThrowBadStatusOrAccess(absl::Status status) { +#ifdef ABSL_HAVE_EXCEPTIONS + throw absl::BadStatusOrAccess(std::move(status)); +#else + ABSL_INTERNAL_LOG( + FATAL, + absl::StrCat("Attempting to fetch value instead of handling error ", + status.ToString())); + std::abort(); +#endif +} + +} // namespace internal_statusor +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/status/statusor.h b/third_party/abseil_cpp/absl/status/statusor.h new file mode 100644 index 000000000000..469d486fdd2e --- /dev/null +++ b/third_party/abseil_cpp/absl/status/statusor.h @@ -0,0 +1,760 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: statusor.h +// ----------------------------------------------------------------------------- +// +// An `absl::StatusOr<T>` represents a union of an `absl::Status` object +// and an object of type `T`. The `absl::StatusOr<T>` will either contain an +// object of type `T` (indicating a successful operation), or an error (of type +// `absl::Status`) explaining why such a value is not present. +// +// In general, check the success of an operation returning an +// `absl::StatusOr<T>` like you would an `absl::Status` by using the `ok()` +// member function. +// +// Example: +// +// StatusOr<Foo> result = Calculation(); +// if (result.ok()) { +// result->DoSomethingCool(); +// } else { +// LOG(ERROR) << result.status(); +// } +#ifndef ABSL_STATUS_STATUSOR_H_ +#define ABSL_STATUS_STATUSOR_H_ + +#include <exception> +#include <initializer_list> +#include <new> +#include <string> +#include <type_traits> +#include <utility> + +#include "absl/base/attributes.h" +#include "absl/meta/type_traits.h" +#include "absl/status/internal/statusor_internal.h" +#include "absl/status/status.h" +#include "absl/types/variant.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// BadStatusOrAccess +// +// This class defines the type of object to throw (if exceptions are enabled), +// when accessing the value of an `absl::StatusOr<T>` object that does not +// contain a value. This behavior is analogous to that of +// `std::bad_optional_access` in the case of accessing an invalid +// `std::optional` value. +// +// Example: +// +// try { +// absl::StatusOr<int> v = FetchInt(); +// DoWork(v.value()); // Accessing value() when not "OK" may throw +// } catch (absl::BadStatusOrAccess& ex) { +// LOG(ERROR) << ex.status(); +// } +class BadStatusOrAccess : public std::exception { + public: + explicit BadStatusOrAccess(absl::Status status); + ~BadStatusOrAccess() override; + + // BadStatusOrAccess::what() + // + // Returns the associated explanatory string of the `absl::StatusOr<T>` + // object's error code. This function only returns the string literal "Bad + // StatusOr Access" for cases when evaluating general exceptions. + // + // The pointer of this string is guaranteed to be valid until any non-const + // function is invoked on the exception object. + const char* what() const noexcept override; + + // BadStatusOrAccess::status() + // + // Returns the associated `absl::Status` of the `absl::StatusOr<T>` object's + // error. + const absl::Status& status() const; + + private: + absl::Status status_; +}; + +// Returned StatusOr objects may not be ignored. +template <typename T> +class ABSL_MUST_USE_RESULT StatusOr; + +// absl::StatusOr<T> +// +// The `absl::StatusOr<T>` class template is a union of an `absl::Status` object +// and an object of type `T`. The `absl::StatusOr<T>` models an object that is +// either a usable object, or an error (of type `absl::Status`) explaining why +// such an object is not present. An `absl::StatusOr<T>` is typically the return +// value of a function which may fail. +// +// An `absl::StatusOr<T>` can never hold an "OK" status (an +// `absl::StatusCode::kOk` value); instead, the presence of an object of type +// `T` indicates success. Instead of checking for a `kOk` value, use the +// `absl::StatusOr<T>::ok()` member function. (It is for this reason, and code +// readability, that using the `ok()` function is preferred for `absl::Status` +// as well.) +// +// Example: +// +// StatusOr<Foo> result = DoBigCalculationThatCouldFail(); +// if (result.ok()) { +// result->DoSomethingCool(); +// } else { +// LOG(ERROR) << result.status(); +// } +// +// Accessing the object held by an `absl::StatusOr<T>` should be performed via +// `operator*` or `operator->`, after a call to `ok()` confirms that the +// `absl::StatusOr<T>` holds an object of type `T`: +// +// Example: +// +// absl::StatusOr<int> i = GetCount(); +// if (i.ok()) { +// updated_total += *i +// } +// +// NOTE: using `absl::StatusOr<T>::value()` when no valid value is present will +// throw an exception if exceptions are enabled or terminate the process when +// execeptions are not enabled. +// +// Example: +// +// StatusOr<Foo> result = DoBigCalculationThatCouldFail(); +// const Foo& foo = result.value(); // Crash/exception if no value present +// foo.DoSomethingCool(); +// +// A `absl::StatusOr<T*>` can be constructed from a null pointer like any other +// pointer value, and the result will be that `ok()` returns `true` and +// `value()` returns `nullptr`. Checking the value of pointer in an +// `absl::StatusOr<T>` generally requires a bit more care, to ensure both that a +// value is present and that value is not null: +// +// StatusOr<std::unique_ptr<Foo>> result = FooFactory::MakeNewFoo(arg); +// if (!result.ok()) { +// LOG(ERROR) << result.status(); +// } else if (*result == nullptr) { +// LOG(ERROR) << "Unexpected null pointer"; +// } else { +// (*result)->DoSomethingCool(); +// } +// +// Example factory implementation returning StatusOr<T>: +// +// StatusOr<Foo> FooFactory::MakeFoo(int arg) { +// if (arg <= 0) { +// return absl::Status(absl::StatusCode::kInvalidArgument, +// "Arg must be positive"); +// } +// return Foo(arg); +// } +template <typename T> +class StatusOr : private internal_statusor::StatusOrData<T>, + private internal_statusor::CopyCtorBase<T>, + private internal_statusor::MoveCtorBase<T>, + private internal_statusor::CopyAssignBase<T>, + private internal_statusor::MoveAssignBase<T> { + template <typename U> + friend class StatusOr; + + typedef internal_statusor::StatusOrData<T> Base; + + public: + // StatusOr<T>::value_type + // + // This instance data provides a generic `value_type` member for use within + // generic programming. This usage is analogous to that of + // `optional::value_type` in the case of `std::optional`. + typedef T value_type; + + // Constructors + + // Constructs a new `absl::StatusOr` with an `absl::StatusCode::kUnknown` + // status. This constructor is marked 'explicit' to prevent usages in return + // values such as 'return {};', under the misconception that + // `absl::StatusOr<std::vector<int>>` will be initialized with an empty + // vector, instead of an `absl::StatusCode::kUnknown` error code. + explicit StatusOr(); + + // `StatusOr<T>` is copy constructible if `T` is copy constructible. + StatusOr(const StatusOr&) = default; + // `StatusOr<T>` is copy assignable if `T` is copy constructible and copy + // assignable. + StatusOr& operator=(const StatusOr&) = default; + + // `StatusOr<T>` is move constructible if `T` is move constructible. + StatusOr(StatusOr&&) = default; + // `StatusOr<T>` is moveAssignable if `T` is move constructible and move + // assignable. + StatusOr& operator=(StatusOr&&) = default; + + // Converting Constructors + + // Constructs a new `absl::StatusOr<T>` from an `absl::StatusOr<U>`, when `T` + // is constructible from `U`. To avoid ambiguity, these constructors are + // disabled if `T` is also constructible from `StatusOr<U>.`. This constructor + // is explicit if and only if the corresponding construction of `T` from `U` + // is explicit. (This constructor inherits its explicitness from the + // underlying constructor.) + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, + std::is_convertible<const U&, T>, + absl::negation< + internal_statusor::IsConstructibleOrConvertibleFromStatusOr< + T, U>>>::value, + int> = 0> + StatusOr(const StatusOr<U>& other) // NOLINT + : Base(static_cast<const typename StatusOr<U>::Base&>(other)) {} + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, + absl::negation<std::is_convertible<const U&, T>>, + absl::negation< + internal_statusor::IsConstructibleOrConvertibleFromStatusOr< + T, U>>>::value, + int> = 0> + explicit StatusOr(const StatusOr<U>& other) + : Base(static_cast<const typename StatusOr<U>::Base&>(other)) {} + + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, + std::is_convertible<U&&, T>, + absl::negation< + internal_statusor::IsConstructibleOrConvertibleFromStatusOr< + T, U>>>::value, + int> = 0> + StatusOr(StatusOr<U>&& other) // NOLINT + : Base(static_cast<typename StatusOr<U>::Base&&>(other)) {} + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, + absl::negation<std::is_convertible<U&&, T>>, + absl::negation< + internal_statusor::IsConstructibleOrConvertibleFromStatusOr< + T, U>>>::value, + int> = 0> + explicit StatusOr(StatusOr<U>&& other) + : Base(static_cast<typename StatusOr<U>::Base&&>(other)) {} + + // Converting Assignment Operators + + // Creates an `absl::StatusOr<T>` through assignment from an + // `absl::StatusOr<U>` when: + // + // * Both `absl::StatusOr<T>` and `absl::StatusOr<U>` are OK by assigning + // `U` to `T` directly. + // * `absl::StatusOr<T>` is OK and `absl::StatusOr<U>` contains an error + // code by destroying `absl::StatusOr<T>`'s value and assigning from + // `absl::StatusOr<U>' + // * `absl::StatusOr<T>` contains an error code and `absl::StatusOr<U>` is + // OK by directly initializing `T` from `U`. + // * Both `absl::StatusOr<T>` and `absl::StatusOr<U>` contain an error + // code by assigning the `Status` in `absl::StatusOr<U>` to + // `absl::StatusOr<T>` + // + // These overloads only apply if `absl::StatusOr<T>` is constructible and + // assignable from `absl::StatusOr<U>` and `StatusOr<T>` cannot be directly + // assigned from `StatusOr<U>`. + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, + std::is_assignable<T, const U&>, + absl::negation< + internal_statusor:: + IsConstructibleOrConvertibleOrAssignableFromStatusOr< + T, U>>>::value, + int> = 0> + StatusOr& operator=(const StatusOr<U>& other) { + this->Assign(other); + return *this; + } + template < + typename U, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, + std::is_assignable<T, U&&>, + absl::negation< + internal_statusor:: + IsConstructibleOrConvertibleOrAssignableFromStatusOr< + T, U>>>::value, + int> = 0> + StatusOr& operator=(StatusOr<U>&& other) { + this->Assign(std::move(other)); + return *this; + } + + // Constructs a new `absl::StatusOr<T>` with a non-ok status. After calling + // this constructor, `this->ok()` will be `false` and calls to `value()` will + // crash, or produce an exception if exceptions are enabled. + // + // The constructor also takes any type `U` that is convertible to + // `absl::Status`. This constructor is explicit if an only if `U` is not of + // type `absl::Status` and the conversion from `U` to `Status` is explicit. + // + // REQUIRES: !Status(std::forward<U>(v)).ok(). This requirement is DCHECKed. + // In optimized builds, passing absl::OkStatus() here will have the effect + // of passing absl::StatusCode::kInternal as a fallback. + template < + typename U = absl::Status, + absl::enable_if_t< + absl::conjunction< + std::is_convertible<U&&, absl::Status>, + std::is_constructible<absl::Status, U&&>, + absl::negation<std::is_same<absl::decay_t<U>, absl::StatusOr<T>>>, + absl::negation<std::is_same<absl::decay_t<U>, T>>, + absl::negation<std::is_same<absl::decay_t<U>, absl::in_place_t>>, + absl::negation<internal_statusor::HasConversionOperatorToStatusOr< + T, U&&>>>::value, + int> = 0> + StatusOr(U&& v) : Base(std::forward<U>(v)) {} + + template < + typename U = absl::Status, + absl::enable_if_t< + absl::conjunction< + absl::negation<std::is_convertible<U&&, absl::Status>>, + std::is_constructible<absl::Status, U&&>, + absl::negation<std::is_same<absl::decay_t<U>, absl::StatusOr<T>>>, + absl::negation<std::is_same<absl::decay_t<U>, T>>, + absl::negation<std::is_same<absl::decay_t<U>, absl::in_place_t>>, + absl::negation<internal_statusor::HasConversionOperatorToStatusOr< + T, U&&>>>::value, + int> = 0> + explicit StatusOr(U&& v) : Base(std::forward<U>(v)) {} + + template < + typename U = absl::Status, + absl::enable_if_t< + absl::conjunction< + std::is_convertible<U&&, absl::Status>, + std::is_constructible<absl::Status, U&&>, + absl::negation<std::is_same<absl::decay_t<U>, absl::StatusOr<T>>>, + absl::negation<std::is_same<absl::decay_t<U>, T>>, + absl::negation<std::is_same<absl::decay_t<U>, absl::in_place_t>>, + absl::negation<internal_statusor::HasConversionOperatorToStatusOr< + T, U&&>>>::value, + int> = 0> + StatusOr& operator=(U&& v) { + this->AssignStatus(std::forward<U>(v)); + return *this; + } + + // Perfect-forwarding value assignment operator. + + // If `*this` contains a `T` value before the call, the contained value is + // assigned from `std::forward<U>(v)`; Otherwise, it is directly-initialized + // from `std::forward<U>(v)`. + // This function does not participate in overload unless: + // 1. `std::is_constructible_v<T, U>` is true, + // 2. `std::is_assignable_v<T&, U>` is true. + // 3. `std::is_same_v<StatusOr<T>, std::remove_cvref_t<U>>` is false. + // 4. Assigning `U` to `T` is not ambiguous: + // If `U` is `StatusOr<V>` and `T` is constructible and assignable from + // both `StatusOr<V>` and `V`, the assignment is considered bug-prone and + // ambiguous thus will fail to compile. For example: + // StatusOr<bool> s1 = true; // s1.ok() && *s1 == true + // StatusOr<bool> s2 = false; // s2.ok() && *s2 == false + // s1 = s2; // ambiguous, `s1 = *s2` or `s1 = bool(s2)`? + template < + typename U = T, + typename = typename std::enable_if<absl::conjunction< + std::is_constructible<T, U&&>, std::is_assignable<T&, U&&>, + absl::disjunction< + std::is_same<absl::remove_cv_t<absl::remove_reference_t<U>>, T>, + absl::conjunction< + absl::negation<std::is_convertible<U&&, absl::Status>>, + absl::negation<internal_statusor:: + HasConversionOperatorToStatusOr<T, U&&>>>>, + internal_statusor::IsForwardingAssignmentValid<T, U&&>>::value>::type> + StatusOr& operator=(U&& v) { + this->Assign(std::forward<U>(v)); + return *this; + } + + // Constructs the inner value `T` in-place using the provided args, using the + // `T(args...)` constructor. + template <typename... Args> + explicit StatusOr(absl::in_place_t, Args&&... args); + template <typename U, typename... Args> + explicit StatusOr(absl::in_place_t, std::initializer_list<U> ilist, + Args&&... args); + + // Constructs the inner value `T` in-place using the provided args, using the + // `T(U)` (direct-initialization) constructor. This constructor is only valid + // if `T` can be constructed from a `U`. Can accept move or copy constructors. + // + // This constructor is explicit if `U` is not convertible to `T`. To avoid + // ambiguity, this constuctor is disabled if `U` is a `StatusOr<J>`, where `J` + // is convertible to `T`. + template < + typename U = T, + absl::enable_if_t< + absl::conjunction< + internal_statusor::IsDirectInitializationValid<T, U&&>, + std::is_constructible<T, U&&>, std::is_convertible<U&&, T>, + absl::disjunction< + std::is_same<absl::remove_cv_t<absl::remove_reference_t<U>>, + T>, + absl::conjunction< + absl::negation<std::is_convertible<U&&, absl::Status>>, + absl::negation< + internal_statusor::HasConversionOperatorToStatusOr< + T, U&&>>>>>::value, + int> = 0> + StatusOr(U&& u) // NOLINT + : StatusOr(absl::in_place, std::forward<U>(u)) { + } + + template < + typename U = T, + absl::enable_if_t< + absl::conjunction< + internal_statusor::IsDirectInitializationValid<T, U&&>, + absl::disjunction< + std::is_same<absl::remove_cv_t<absl::remove_reference_t<U>>, + T>, + absl::conjunction< + absl::negation<std::is_constructible<absl::Status, U&&>>, + absl::negation< + internal_statusor::HasConversionOperatorToStatusOr< + T, U&&>>>>, + std::is_constructible<T, U&&>, + absl::negation<std::is_convertible<U&&, T>>>::value, + int> = 0> + explicit StatusOr(U&& u) // NOLINT + : StatusOr(absl::in_place, std::forward<U>(u)) { + } + + // StatusOr<T>::ok() + // + // Returns whether or not this `absl::StatusOr<T>` holds a `T` value. This + // member function is analagous to `absl::Status::ok()` and should be used + // similarly to check the status of return values. + // + // Example: + // + // StatusOr<Foo> result = DoBigCalculationThatCouldFail(); + // if (result.ok()) { + // // Handle result + // else { + // // Handle error + // } + ABSL_MUST_USE_RESULT bool ok() const { return this->status_.ok(); } + + // StatusOr<T>::status() + // + // Returns a reference to the current `absl::Status` contained within the + // `absl::StatusOr<T>`. If `absl::StatusOr<T>` contains a `T`, then this + // function returns `absl::OkStatus()`. + const Status& status() const &; + Status status() &&; + + // StatusOr<T>::value() + // + // Returns a reference to the held value if `this->ok()`. Otherwise, throws + // `absl::BadStatusOrAccess` if exceptions are enabled, or is guaranteed to + // terminate the process if exceptions are disabled. + // + // If you have already checked the status using `this->ok()`, you probably + // want to use `operator*()` or `operator->()` to access the value instead of + // `value`. + // + // Note: for value types that are cheap to copy, prefer simple code: + // + // T value = statusor.value(); + // + // Otherwise, if the value type is expensive to copy, but can be left + // in the StatusOr, simply assign to a reference: + // + // T& value = statusor.value(); // or `const T&` + // + // Otherwise, if the value type supports an efficient move, it can be + // used as follows: + // + // T value = std::move(statusor).value(); + // + // The `std::move` on statusor instead of on the whole expression enables + // warnings about possible uses of the statusor object after the move. + const T& value() const&; + T& value() &; + const T&& value() const&&; + T&& value() &&; + + // StatusOr<T>:: operator*() + // + // Returns a reference to the current value. + // + // REQUIRES: `this->ok() == true`, otherwise the behavior is undefined. + // + // Use `this->ok()` to verify that there is a current value within the + // `absl::StatusOr<T>`. Alternatively, see the `value()` member function for a + // similar API that guarantees crashing or throwing an exception if there is + // no current value. + const T& operator*() const&; + T& operator*() &; + const T&& operator*() const&&; + T&& operator*() &&; + + // StatusOr<T>::operator->() + // + // Returns a pointer to the current value. + // + // REQUIRES: `this->ok() == true`, otherwise the behavior is undefined. + // + // Use `this->ok()` to verify that there is a current value. + const T* operator->() const; + T* operator->(); + + // StatusOr<T>::value_or() + // + // Returns the current value if `this->ok() == true`. Otherwise constructs a + // value using the provided `default_value`. + // + // Unlike `value`, this function returns by value, copying the current value + // if necessary. If the value type supports an efficient move, it can be used + // as follows: + // + // T value = std::move(statusor).value_or(def); + // + // Unlike with `value`, calling `std::move()` on the result of `value_or` will + // still trigger a copy. + template <typename U> + T value_or(U&& default_value) const&; + template <typename U> + T value_or(U&& default_value) &&; + + // StatusOr<T>::IgnoreError() + // + // Ignores any errors. This method does nothing except potentially suppress + // complaints from any tools that are checking that errors are not dropped on + // the floor. + void IgnoreError() const; + + // StatusOr<T>::emplace() + // + // Reconstructs the inner value T in-place using the provided args, using the + // T(args...) constructor. Returns reference to the reconstructed `T`. + template <typename... Args> + T& emplace(Args&&... args) { + if (ok()) { + this->Clear(); + this->MakeValue(std::forward<Args>(args)...); + } else { + this->MakeValue(std::forward<Args>(args)...); + this->status_ = absl::OkStatus(); + } + return this->data_; + } + + template < + typename U, typename... Args, + absl::enable_if_t< + std::is_constructible<T, std::initializer_list<U>&, Args&&...>::value, + int> = 0> + T& emplace(std::initializer_list<U> ilist, Args&&... args) { + if (ok()) { + this->Clear(); + this->MakeValue(ilist, std::forward<Args>(args)...); + } else { + this->MakeValue(ilist, std::forward<Args>(args)...); + this->status_ = absl::OkStatus(); + } + return this->data_; + } + + private: + using internal_statusor::StatusOrData<T>::Assign; + template <typename U> + void Assign(const absl::StatusOr<U>& other); + template <typename U> + void Assign(absl::StatusOr<U>&& other); +}; + +// operator==() +// +// This operator checks the equality of two `absl::StatusOr<T>` objects. +template <typename T> +bool operator==(const StatusOr<T>& lhs, const StatusOr<T>& rhs) { + if (lhs.ok() && rhs.ok()) return *lhs == *rhs; + return lhs.status() == rhs.status(); +} + +// operator!=() +// +// This operator checks the inequality of two `absl::StatusOr<T>` objects. +template <typename T> +bool operator!=(const StatusOr<T>& lhs, const StatusOr<T>& rhs) { + return !(lhs == rhs); +} + +//------------------------------------------------------------------------------ +// Implementation details for StatusOr<T> +//------------------------------------------------------------------------------ + +// TODO(sbenza): avoid the string here completely. +template <typename T> +StatusOr<T>::StatusOr() : Base(Status(absl::StatusCode::kUnknown, "")) {} + +template <typename T> +template <typename U> +inline void StatusOr<T>::Assign(const StatusOr<U>& other) { + if (other.ok()) { + this->Assign(*other); + } else { + this->AssignStatus(other.status()); + } +} + +template <typename T> +template <typename U> +inline void StatusOr<T>::Assign(StatusOr<U>&& other) { + if (other.ok()) { + this->Assign(*std::move(other)); + } else { + this->AssignStatus(std::move(other).status()); + } +} +template <typename T> +template <typename... Args> +StatusOr<T>::StatusOr(absl::in_place_t, Args&&... args) + : Base(absl::in_place, std::forward<Args>(args)...) {} + +template <typename T> +template <typename U, typename... Args> +StatusOr<T>::StatusOr(absl::in_place_t, std::initializer_list<U> ilist, + Args&&... args) + : Base(absl::in_place, ilist, std::forward<Args>(args)...) {} + +template <typename T> +const Status& StatusOr<T>::status() const & { return this->status_; } +template <typename T> +Status StatusOr<T>::status() && { + return ok() ? OkStatus() : std::move(this->status_); +} + +template <typename T> +const T& StatusOr<T>::value() const& { + if (!this->ok()) internal_statusor::ThrowBadStatusOrAccess(this->status_); + return this->data_; +} + +template <typename T> +T& StatusOr<T>::value() & { + if (!this->ok()) internal_statusor::ThrowBadStatusOrAccess(this->status_); + return this->data_; +} + +template <typename T> +const T&& StatusOr<T>::value() const&& { + if (!this->ok()) { + internal_statusor::ThrowBadStatusOrAccess(std::move(this->status_)); + } + return std::move(this->data_); +} + +template <typename T> +T&& StatusOr<T>::value() && { + if (!this->ok()) { + internal_statusor::ThrowBadStatusOrAccess(std::move(this->status_)); + } + return std::move(this->data_); +} + +template <typename T> +const T& StatusOr<T>::operator*() const& { + this->EnsureOk(); + return this->data_; +} + +template <typename T> +T& StatusOr<T>::operator*() & { + this->EnsureOk(); + return this->data_; +} + +template <typename T> +const T&& StatusOr<T>::operator*() const&& { + this->EnsureOk(); + return std::move(this->data_); +} + +template <typename T> +T&& StatusOr<T>::operator*() && { + this->EnsureOk(); + return std::move(this->data_); +} + +template <typename T> +const T* StatusOr<T>::operator->() const { + this->EnsureOk(); + return &this->data_; +} + +template <typename T> +T* StatusOr<T>::operator->() { + this->EnsureOk(); + return &this->data_; +} + +template <typename T> +template <typename U> +T StatusOr<T>::value_or(U&& default_value) const& { + if (ok()) { + return this->data_; + } + return std::forward<U>(default_value); +} + +template <typename T> +template <typename U> +T StatusOr<T>::value_or(U&& default_value) && { + if (ok()) { + return std::move(this->data_); + } + return std::forward<U>(default_value); +} + +template <typename T> +void StatusOr<T>::IgnoreError() const { + // no-op +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STATUS_STATUSOR_H_ diff --git a/third_party/abseil_cpp/absl/status/statusor_internals.h b/third_party/abseil_cpp/absl/status/statusor_internals.h new file mode 100644 index 000000000000..5366c2840c37 --- /dev/null +++ b/third_party/abseil_cpp/absl/status/statusor_internals.h @@ -0,0 +1,250 @@ +/* Copyright 2017 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef ABSL_STATUS_STATUSOR_INTERNALS_H_ +#define ABSL_STATUS_STATUSOR_INTERNALS_H_ + +#include "absl/status/status.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace internal_statusor { + +class Helper { + public: + // Move type-agnostic error handling to the .cc. + static void HandleInvalidStatusCtorArg(Status*); + ABSL_ATTRIBUTE_NORETURN static void Crash(const Status& status); +}; + +// Construct an instance of T in `p` through placement new, passing Args... to +// the constructor. +// This abstraction is here mostly for the gcc performance fix. +template <typename T, typename... Args> +void PlacementNew(void* p, Args&&... args) { +#if defined(__GNUC__) && !defined(__clang__) + // Teach gcc that 'p' cannot be null, fixing code size issues. + if (p == nullptr) __builtin_unreachable(); +#endif + new (p) T(std::forward<Args>(args)...); +} + +// Helper base class to hold the data and all operations. +// We move all this to a base class to allow mixing with the appropriate +// TraitsBase specialization. +template <typename T> +class StatusOrData { + template <typename U> + friend class StatusOrData; + + public: + StatusOrData() = delete; + + StatusOrData(const StatusOrData& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + StatusOrData(StatusOrData&& other) noexcept { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + template <typename U> + StatusOrData(const StatusOrData<U>& other) { + if (other.ok()) { + MakeValue(other.data_); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + template <typename U> + StatusOrData(StatusOrData<U>&& other) { + if (other.ok()) { + MakeValue(std::move(other.data_)); + MakeStatus(); + } else { + MakeStatus(other.status_); + } + } + + explicit StatusOrData(const T& value) : data_(value) { MakeStatus(); } + explicit StatusOrData(T&& value) : data_(std::move(value)) { MakeStatus(); } + + explicit StatusOrData(const Status& status) : status_(status) { + EnsureNotOk(); + } + explicit StatusOrData(Status&& status) : status_(std::move(status)) { + EnsureNotOk(); + } + + StatusOrData& operator=(const StatusOrData& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(other.data_); + else + Assign(other.status_); + return *this; + } + + StatusOrData& operator=(StatusOrData&& other) { + if (this == &other) return *this; + if (other.ok()) + Assign(std::move(other.data_)); + else + Assign(std::move(other.status_)); + return *this; + } + + ~StatusOrData() { + if (ok()) { + status_.~Status(); + data_.~T(); + } else { + status_.~Status(); + } + } + + void Assign(const T& value) { + if (ok()) { + data_.~T(); + MakeValue(value); + } else { + MakeValue(value); + status_ = OkStatus(); + } + } + + void Assign(T&& value) { + if (ok()) { + data_.~T(); + MakeValue(std::move(value)); + } else { + MakeValue(std::move(value)); + status_ = OkStatus(); + } + } + + void Assign(const Status& status) { + Clear(); + status_ = status; + EnsureNotOk(); + } + + void Assign(Status&& status) { + Clear(); + // Note that we copy instead of moving the status here so that + // status.~StatusOrData() can call ok() without invoking UB. + status_ = status; + EnsureNotOk(); + } + + bool ok() const { return status_.ok(); } + + protected: + // status_ will always be active after the constructor. + // We make it a union to be able to initialize exactly how we need without + // waste. + // Eg. in the copy constructor we use the default constructor of Status in + // the ok() path to avoid an extra Ref call. + union { + Status status_; + }; + + // data_ is active iff status_.ok()==true + struct Dummy {}; + union { + // When T is const, we need some non-const object we can cast to void* for + // the placement new. dummy_ is that object. + Dummy dummy_; + T data_; + }; + + void Clear() { + if (ok()) data_.~T(); + } + + void EnsureOk() const { + if (!ok()) Helper::Crash(status_); + } + + void EnsureNotOk() { + if (ok()) Helper::HandleInvalidStatusCtorArg(&status_); + } + + // Construct the value (ie. data_) through placement new with the passed + // argument. + template <typename Arg> + void MakeValue(Arg&& arg) { + internal_statusor::PlacementNew<T>(&dummy_, std::forward<Arg>(arg)); + } + + // Construct the status (ie. status_) through placement new with the passed + // argument. + template <typename... Args> + void MakeStatus(Args&&... args) { + internal_statusor::PlacementNew<Status>(&status_, + std::forward<Args>(args)...); + } +}; + +// Helper base class to allow implicitly deleted constructors and assignment +// operations in StatusOr. +// TraitsBase will explicitly delete what it can't support and StatusOr will +// inherit that behavior implicitly. +template <bool Copy, bool Move> +struct TraitsBase { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = default; + TraitsBase(TraitsBase&&) = default; + TraitsBase& operator=(const TraitsBase&) = default; + TraitsBase& operator=(TraitsBase&&) = default; +}; + +template <> +struct TraitsBase<false, true> { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = delete; + TraitsBase(TraitsBase&&) = default; + TraitsBase& operator=(const TraitsBase&) = delete; + TraitsBase& operator=(TraitsBase&&) = default; +}; + +template <> +struct TraitsBase<false, false> { + TraitsBase() = default; + TraitsBase(const TraitsBase&) = delete; + TraitsBase(TraitsBase&&) = delete; + TraitsBase& operator=(const TraitsBase&) = delete; + TraitsBase& operator=(TraitsBase&&) = delete; +}; + +} // namespace internal_statusor + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STATUS_STATUSOR_INTERNALS_H_ diff --git a/third_party/abseil_cpp/absl/status/statusor_test.cc b/third_party/abseil_cpp/absl/status/statusor_test.cc new file mode 100644 index 000000000000..c2e8fb7e359e --- /dev/null +++ b/third_party/abseil_cpp/absl/status/statusor_test.cc @@ -0,0 +1,1811 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/status/statusor.h" + +#include <array> +#include <initializer_list> +#include <memory> +#include <type_traits> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/casts.h" +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/types/any.h" +#include "absl/utility/utility.h" + +namespace { + +using ::testing::AllOf; +using ::testing::AnyWith; +using ::testing::ElementsAre; +using ::testing::Field; +using ::testing::Ne; +using ::testing::Not; +using ::testing::Pointee; +using ::testing::VariantWith; + +#ifdef GTEST_HAS_STATUS_MATCHERS +using ::testing::status::IsOk; +using ::testing::status::IsOkAndHolds; +#else // GTEST_HAS_STATUS_MATCHERS +inline const ::absl::Status& GetStatus(const ::absl::Status& status) { + return status; +} + +template <typename T> +inline const ::absl::Status& GetStatus(const ::absl::StatusOr<T>& status) { + return status.status(); +} + +// Monomorphic implementation of matcher IsOkAndHolds(m). StatusOrType is a +// reference to StatusOr<T>. +template <typename StatusOrType> +class IsOkAndHoldsMatcherImpl + : public ::testing::MatcherInterface<StatusOrType> { + public: + typedef + typename std::remove_reference<StatusOrType>::type::value_type value_type; + + template <typename InnerMatcher> + explicit IsOkAndHoldsMatcherImpl(InnerMatcher&& inner_matcher) + : inner_matcher_(::testing::SafeMatcherCast<const value_type&>( + std::forward<InnerMatcher>(inner_matcher))) {} + + void DescribeTo(std::ostream* os) const override { + *os << "is OK and has a value that "; + inner_matcher_.DescribeTo(os); + } + + void DescribeNegationTo(std::ostream* os) const override { + *os << "isn't OK or has a value that "; + inner_matcher_.DescribeNegationTo(os); + } + + bool MatchAndExplain( + StatusOrType actual_value, + ::testing::MatchResultListener* result_listener) const override { + if (!actual_value.ok()) { + *result_listener << "which has status " << actual_value.status(); + return false; + } + + ::testing::StringMatchResultListener inner_listener; + const bool matches = + inner_matcher_.MatchAndExplain(*actual_value, &inner_listener); + const std::string inner_explanation = inner_listener.str(); + if (!inner_explanation.empty()) { + *result_listener << "which contains value " + << ::testing::PrintToString(*actual_value) << ", " + << inner_explanation; + } + return matches; + } + + private: + const ::testing::Matcher<const value_type&> inner_matcher_; +}; + +// Implements IsOkAndHolds(m) as a polymorphic matcher. +template <typename InnerMatcher> +class IsOkAndHoldsMatcher { + public: + explicit IsOkAndHoldsMatcher(InnerMatcher inner_matcher) + : inner_matcher_(std::move(inner_matcher)) {} + + // Converts this polymorphic matcher to a monomorphic matcher of the + // given type. StatusOrType can be either StatusOr<T> or a + // reference to StatusOr<T>. + template <typename StatusOrType> + operator ::testing::Matcher<StatusOrType>() const { // NOLINT + return ::testing::Matcher<StatusOrType>( + new IsOkAndHoldsMatcherImpl<const StatusOrType&>(inner_matcher_)); + } + + private: + const InnerMatcher inner_matcher_; +}; + +// Monomorphic implementation of matcher IsOk() for a given type T. +// T can be Status, StatusOr<>, or a reference to either of them. +template <typename T> +class MonoIsOkMatcherImpl : public ::testing::MatcherInterface<T> { + public: + void DescribeTo(std::ostream* os) const override { *os << "is OK"; } + void DescribeNegationTo(std::ostream* os) const override { + *os << "is not OK"; + } + bool MatchAndExplain(T actual_value, + ::testing::MatchResultListener*) const override { + return GetStatus(actual_value).ok(); + } +}; + +// Implements IsOk() as a polymorphic matcher. +class IsOkMatcher { + public: + template <typename T> + operator ::testing::Matcher<T>() const { // NOLINT + return ::testing::Matcher<T>(new MonoIsOkMatcherImpl<T>()); + } +}; + +// Macros for testing the results of functions that return absl::Status or +// absl::StatusOr<T> (for any type T). +#define EXPECT_OK(expression) EXPECT_THAT(expression, IsOk()) + +// Returns a gMock matcher that matches a StatusOr<> whose status is +// OK and whose value matches the inner matcher. +template <typename InnerMatcher> +IsOkAndHoldsMatcher<typename std::decay<InnerMatcher>::type> IsOkAndHolds( + InnerMatcher&& inner_matcher) { + return IsOkAndHoldsMatcher<typename std::decay<InnerMatcher>::type>( + std::forward<InnerMatcher>(inner_matcher)); +} + +// Returns a gMock matcher that matches a Status or StatusOr<> which is OK. +inline IsOkMatcher IsOk() { return IsOkMatcher(); } +#endif // GTEST_HAS_STATUS_MATCHERS + +struct CopyDetector { + CopyDetector() = default; + explicit CopyDetector(int xx) : x(xx) {} + CopyDetector(CopyDetector&& d) noexcept + : x(d.x), copied(false), moved(true) {} + CopyDetector(const CopyDetector& d) : x(d.x), copied(true), moved(false) {} + CopyDetector& operator=(const CopyDetector& c) { + x = c.x; + copied = true; + moved = false; + return *this; + } + CopyDetector& operator=(CopyDetector&& c) noexcept { + x = c.x; + copied = false; + moved = true; + return *this; + } + int x = 0; + bool copied = false; + bool moved = false; +}; + +testing::Matcher<const CopyDetector&> CopyDetectorHas(int a, bool b, bool c) { + return AllOf(Field(&CopyDetector::x, a), Field(&CopyDetector::moved, b), + Field(&CopyDetector::copied, c)); +} + +class Base1 { + public: + virtual ~Base1() {} + int pad; +}; + +class Base2 { + public: + virtual ~Base2() {} + int yetotherpad; +}; + +class Derived : public Base1, public Base2 { + public: + virtual ~Derived() {} + int evenmorepad; +}; + +class CopyNoAssign { + public: + explicit CopyNoAssign(int value) : foo(value) {} + CopyNoAssign(const CopyNoAssign& other) : foo(other.foo) {} + int foo; + + private: + const CopyNoAssign& operator=(const CopyNoAssign&); +}; + +absl::StatusOr<std::unique_ptr<int>> ReturnUniquePtr() { + // Uses implicit constructor from T&& + return absl::make_unique<int>(0); +} + +TEST(StatusOr, ElementType) { + static_assert(std::is_same<absl::StatusOr<int>::value_type, int>(), ""); + static_assert(std::is_same<absl::StatusOr<char>::value_type, char>(), ""); +} + +TEST(StatusOr, TestMoveOnlyInitialization) { + absl::StatusOr<std::unique_ptr<int>> thing(ReturnUniquePtr()); + ASSERT_TRUE(thing.ok()); + EXPECT_EQ(0, **thing); + int* previous = thing->get(); + + thing = ReturnUniquePtr(); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(0, **thing); + EXPECT_NE(previous, thing->get()); +} + +TEST(StatusOr, TestMoveOnlyValueExtraction) { + absl::StatusOr<std::unique_ptr<int>> thing(ReturnUniquePtr()); + ASSERT_TRUE(thing.ok()); + std::unique_ptr<int> ptr = *std::move(thing); + EXPECT_EQ(0, *ptr); + + thing = std::move(ptr); + ptr = std::move(*thing); + EXPECT_EQ(0, *ptr); +} + +TEST(StatusOr, TestMoveOnlyInitializationFromTemporaryByValueOrDie) { + std::unique_ptr<int> ptr(*ReturnUniquePtr()); + EXPECT_EQ(0, *ptr); +} + +TEST(StatusOr, TestValueOrDieOverloadForConstTemporary) { + static_assert( + std::is_same<const int&&, + decltype( + std::declval<const absl::StatusOr<int>&&>().value())>(), + "value() for const temporaries should return const T&&"); +} + +TEST(StatusOr, TestMoveOnlyConversion) { + absl::StatusOr<std::unique_ptr<const int>> const_thing(ReturnUniquePtr()); + EXPECT_TRUE(const_thing.ok()); + EXPECT_EQ(0, **const_thing); + + // Test rvalue converting assignment + const int* const_previous = const_thing->get(); + const_thing = ReturnUniquePtr(); + EXPECT_TRUE(const_thing.ok()); + EXPECT_EQ(0, **const_thing); + EXPECT_NE(const_previous, const_thing->get()); +} + +TEST(StatusOr, TestMoveOnlyVector) { + // Sanity check that absl::StatusOr<MoveOnly> works in vector. + std::vector<absl::StatusOr<std::unique_ptr<int>>> vec; + vec.push_back(ReturnUniquePtr()); + vec.resize(2); + auto another_vec = std::move(vec); + EXPECT_EQ(0, **another_vec[0]); + EXPECT_EQ(absl::UnknownError(""), another_vec[1].status()); +} + +TEST(StatusOr, TestDefaultCtor) { + absl::StatusOr<int> thing; + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), absl::StatusCode::kUnknown); +} + +TEST(StatusOr, StatusCtorForwards) { + absl::Status status(absl::StatusCode::kInternal, "Some error"); + + EXPECT_EQ(absl::StatusOr<int>(status).status().message(), "Some error"); + EXPECT_EQ(status.message(), "Some error"); + + EXPECT_EQ(absl::StatusOr<int>(std::move(status)).status().message(), + "Some error"); + EXPECT_NE(status.message(), "Some error"); +} + +// Define `EXPECT_DEATH_OR_THROW` to test the behavior of `StatusOr::value`, +// which either throws `BadStatusOrAccess` or `LOG(FATAL)` based on whether +// exceptions are enabled. +#ifdef ABSL_HAVE_EXCEPTIONS +#define EXPECT_DEATH_OR_THROW(statement, status_) \ + EXPECT_THROW( \ + { \ + try { \ + statement; \ + } catch (const absl::BadStatusOrAccess& e) { \ + EXPECT_EQ(e.status(), status_); \ + throw; \ + } \ + }, \ + absl::BadStatusOrAccess); +#else // ABSL_HAVE_EXCEPTIONS +#define EXPECT_DEATH_OR_THROW(statement, status) \ + EXPECT_DEATH_IF_SUPPORTED(statement, status.ToString()); +#endif // ABSL_HAVE_EXCEPTIONS + +TEST(StatusOrDeathTest, TestDefaultCtorValue) { + absl::StatusOr<int> thing; + EXPECT_DEATH_OR_THROW(thing.value(), absl::UnknownError("")); + const absl::StatusOr<int> thing2; + EXPECT_DEATH_OR_THROW(thing2.value(), absl::UnknownError("")); +} + +TEST(StatusOrDeathTest, TestValueNotOk) { + absl::StatusOr<int> thing(absl::CancelledError()); + EXPECT_DEATH_OR_THROW(thing.value(), absl::CancelledError()); +} + +TEST(StatusOrDeathTest, TestValueNotOkConst) { + const absl::StatusOr<int> thing(absl::UnknownError("")); + EXPECT_DEATH_OR_THROW(thing.value(), absl::UnknownError("")); +} + +TEST(StatusOrDeathTest, TestPointerDefaultCtorValue) { + absl::StatusOr<int*> thing; + EXPECT_DEATH_OR_THROW(thing.value(), absl::UnknownError("")); +} + +TEST(StatusOrDeathTest, TestPointerValueNotOk) { + absl::StatusOr<int*> thing(absl::CancelledError()); + EXPECT_DEATH_OR_THROW(thing.value(), absl::CancelledError()); +} + +TEST(StatusOrDeathTest, TestPointerValueNotOkConst) { + const absl::StatusOr<int*> thing(absl::CancelledError()); + EXPECT_DEATH_OR_THROW(thing.value(), absl::CancelledError()); +} + +#if GTEST_HAS_DEATH_TEST +TEST(StatusOrDeathTest, TestStatusCtorStatusOk) { + EXPECT_DEBUG_DEATH( + { + // This will DCHECK + absl::StatusOr<int> thing(absl::OkStatus()); + // In optimized mode, we are actually going to get error::INTERNAL for + // status here, rather than crashing, so check that. + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), absl::StatusCode::kInternal); + }, + "An OK status is not a valid constructor argument"); +} + +TEST(StatusOrDeathTest, TestPointerStatusCtorStatusOk) { + EXPECT_DEBUG_DEATH( + { + absl::StatusOr<int*> thing(absl::OkStatus()); + // In optimized mode, we are actually going to get error::INTERNAL for + // status here, rather than crashing, so check that. + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), absl::StatusCode::kInternal); + }, + "An OK status is not a valid constructor argument"); +} +#endif + +TEST(StatusOr, ValueAccessor) { + const int kIntValue = 110; + { + absl::StatusOr<int> status_or(kIntValue); + EXPECT_EQ(kIntValue, status_or.value()); + EXPECT_EQ(kIntValue, std::move(status_or).value()); + } + { + absl::StatusOr<CopyDetector> status_or(kIntValue); + EXPECT_THAT(status_or, + IsOkAndHolds(CopyDetectorHas(kIntValue, false, false))); + CopyDetector copy_detector = status_or.value(); + EXPECT_THAT(copy_detector, CopyDetectorHas(kIntValue, false, true)); + copy_detector = std::move(status_or).value(); + EXPECT_THAT(copy_detector, CopyDetectorHas(kIntValue, true, false)); + } +} + +TEST(StatusOr, BadValueAccess) { + const absl::Status kError = absl::CancelledError("message"); + absl::StatusOr<int> status_or(kError); + EXPECT_DEATH_OR_THROW(status_or.value(), kError); +} + +TEST(StatusOr, TestStatusCtor) { + absl::StatusOr<int> thing(absl::CancelledError()); + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), absl::StatusCode::kCancelled); +} + + + +TEST(StatusOr, TestValueCtor) { + const int kI = 4; + const absl::StatusOr<int> thing(kI); + EXPECT_TRUE(thing.ok()); + EXPECT_EQ(kI, *thing); +} + +struct Foo { + const int x; + explicit Foo(int y) : x(y) {} +}; + +TEST(StatusOr, InPlaceConstruction) { + EXPECT_THAT(absl::StatusOr<Foo>(absl::in_place, 10), + IsOkAndHolds(Field(&Foo::x, 10))); +} + +struct InPlaceHelper { + InPlaceHelper(std::initializer_list<int> xs, std::unique_ptr<int> yy) + : x(xs), y(std::move(yy)) {} + const std::vector<int> x; + std::unique_ptr<int> y; +}; + +TEST(StatusOr, InPlaceInitListConstruction) { + absl::StatusOr<InPlaceHelper> status_or(absl::in_place, {10, 11, 12}, + absl::make_unique<int>(13)); + EXPECT_THAT(status_or, IsOkAndHolds(AllOf( + Field(&InPlaceHelper::x, ElementsAre(10, 11, 12)), + Field(&InPlaceHelper::y, Pointee(13))))); +} + +TEST(StatusOr, Emplace) { + absl::StatusOr<Foo> status_or_foo(10); + status_or_foo.emplace(20); + EXPECT_THAT(status_or_foo, IsOkAndHolds(Field(&Foo::x, 20))); + status_or_foo = absl::InvalidArgumentError("msg"); + EXPECT_FALSE(status_or_foo.ok()); + EXPECT_EQ(status_or_foo.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_EQ(status_or_foo.status().message(), "msg"); + status_or_foo.emplace(20); + EXPECT_THAT(status_or_foo, IsOkAndHolds(Field(&Foo::x, 20))); +} + +TEST(StatusOr, EmplaceInitializerList) { + absl::StatusOr<InPlaceHelper> status_or(absl::in_place, {10, 11, 12}, + absl::make_unique<int>(13)); + status_or.emplace({1, 2, 3}, absl::make_unique<int>(4)); + EXPECT_THAT(status_or, + IsOkAndHolds(AllOf(Field(&InPlaceHelper::x, ElementsAre(1, 2, 3)), + Field(&InPlaceHelper::y, Pointee(4))))); + status_or = absl::InvalidArgumentError("msg"); + EXPECT_FALSE(status_or.ok()); + EXPECT_EQ(status_or.status().code(), absl::StatusCode::kInvalidArgument); + EXPECT_EQ(status_or.status().message(), "msg"); + status_or.emplace({1, 2, 3}, absl::make_unique<int>(4)); + EXPECT_THAT(status_or, + IsOkAndHolds(AllOf(Field(&InPlaceHelper::x, ElementsAre(1, 2, 3)), + Field(&InPlaceHelper::y, Pointee(4))))); +} + +TEST(StatusOr, TestCopyCtorStatusOk) { + const int kI = 4; + const absl::StatusOr<int> original(kI); + const absl::StatusOr<int> copy(original); + EXPECT_OK(copy.status()); + EXPECT_EQ(*original, *copy); +} + +TEST(StatusOr, TestCopyCtorStatusNotOk) { + absl::StatusOr<int> original(absl::CancelledError()); + absl::StatusOr<int> copy(original); + EXPECT_EQ(copy.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, TestCopyCtorNonAssignable) { + const int kI = 4; + CopyNoAssign value(kI); + absl::StatusOr<CopyNoAssign> original(value); + absl::StatusOr<CopyNoAssign> copy(original); + EXPECT_OK(copy.status()); + EXPECT_EQ(original->foo, copy->foo); +} + +TEST(StatusOr, TestCopyCtorStatusOKConverting) { + const int kI = 4; + absl::StatusOr<int> original(kI); + absl::StatusOr<double> copy(original); + EXPECT_OK(copy.status()); + EXPECT_DOUBLE_EQ(*original, *copy); +} + +TEST(StatusOr, TestCopyCtorStatusNotOkConverting) { + absl::StatusOr<int> original(absl::CancelledError()); + absl::StatusOr<double> copy(original); + EXPECT_EQ(copy.status(), original.status()); +} + +TEST(StatusOr, TestAssignmentStatusOk) { + // Copy assignmment + { + const auto p = std::make_shared<int>(17); + absl::StatusOr<std::shared_ptr<int>> source(p); + + absl::StatusOr<std::shared_ptr<int>> target; + target = source; + + ASSERT_TRUE(target.ok()); + EXPECT_OK(target.status()); + EXPECT_EQ(p, *target); + + ASSERT_TRUE(source.ok()); + EXPECT_OK(source.status()); + EXPECT_EQ(p, *source); + } + + // Move asssignment + { + const auto p = std::make_shared<int>(17); + absl::StatusOr<std::shared_ptr<int>> source(p); + + absl::StatusOr<std::shared_ptr<int>> target; + target = std::move(source); + + ASSERT_TRUE(target.ok()); + EXPECT_OK(target.status()); + EXPECT_EQ(p, *target); + + ASSERT_TRUE(source.ok()); + EXPECT_OK(source.status()); + EXPECT_EQ(nullptr, *source); + } +} + +TEST(StatusOr, TestAssignmentStatusNotOk) { + // Copy assignment + { + const absl::Status expected = absl::CancelledError(); + absl::StatusOr<int> source(expected); + + absl::StatusOr<int> target; + target = source; + + EXPECT_FALSE(target.ok()); + EXPECT_EQ(expected, target.status()); + + EXPECT_FALSE(source.ok()); + EXPECT_EQ(expected, source.status()); + } + + // Move assignment + { + const absl::Status expected = absl::CancelledError(); + absl::StatusOr<int> source(expected); + + absl::StatusOr<int> target; + target = std::move(source); + + EXPECT_FALSE(target.ok()); + EXPECT_EQ(expected, target.status()); + + EXPECT_FALSE(source.ok()); + EXPECT_EQ(source.status().code(), absl::StatusCode::kInternal); + } +} + +TEST(StatusOr, TestAssignmentStatusOKConverting) { + // Copy assignment + { + const int kI = 4; + absl::StatusOr<int> source(kI); + + absl::StatusOr<double> target; + target = source; + + ASSERT_TRUE(target.ok()); + EXPECT_OK(target.status()); + EXPECT_DOUBLE_EQ(kI, *target); + + ASSERT_TRUE(source.ok()); + EXPECT_OK(source.status()); + EXPECT_DOUBLE_EQ(kI, *source); + } + + // Move assignment + { + const auto p = new int(17); + absl::StatusOr<std::unique_ptr<int>> source(absl::WrapUnique(p)); + + absl::StatusOr<std::shared_ptr<int>> target; + target = std::move(source); + + ASSERT_TRUE(target.ok()); + EXPECT_OK(target.status()); + EXPECT_EQ(p, target->get()); + + ASSERT_TRUE(source.ok()); + EXPECT_OK(source.status()); + EXPECT_EQ(nullptr, source->get()); + } +} + +struct A { + int x; +}; + +struct ImplicitConstructibleFromA { + int x; + bool moved; + ImplicitConstructibleFromA(const A& a) // NOLINT + : x(a.x), moved(false) {} + ImplicitConstructibleFromA(A&& a) // NOLINT + : x(a.x), moved(true) {} +}; + +TEST(StatusOr, ImplicitConvertingConstructor) { + EXPECT_THAT( + absl::implicit_cast<absl::StatusOr<ImplicitConstructibleFromA>>( + absl::StatusOr<A>(A{11})), + IsOkAndHolds(AllOf(Field(&ImplicitConstructibleFromA::x, 11), + Field(&ImplicitConstructibleFromA::moved, true)))); + absl::StatusOr<A> a(A{12}); + EXPECT_THAT( + absl::implicit_cast<absl::StatusOr<ImplicitConstructibleFromA>>(a), + IsOkAndHolds(AllOf(Field(&ImplicitConstructibleFromA::x, 12), + Field(&ImplicitConstructibleFromA::moved, false)))); +} + +struct ExplicitConstructibleFromA { + int x; + bool moved; + explicit ExplicitConstructibleFromA(const A& a) : x(a.x), moved(false) {} + explicit ExplicitConstructibleFromA(A&& a) : x(a.x), moved(true) {} +}; + +TEST(StatusOr, ExplicitConvertingConstructor) { + EXPECT_FALSE( + (std::is_convertible<const absl::StatusOr<A>&, + absl::StatusOr<ExplicitConstructibleFromA>>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::StatusOr<A>&&, + absl::StatusOr<ExplicitConstructibleFromA>>::value)); + EXPECT_THAT( + absl::StatusOr<ExplicitConstructibleFromA>(absl::StatusOr<A>(A{11})), + IsOkAndHolds(AllOf(Field(&ExplicitConstructibleFromA::x, 11), + Field(&ExplicitConstructibleFromA::moved, true)))); + absl::StatusOr<A> a(A{12}); + EXPECT_THAT( + absl::StatusOr<ExplicitConstructibleFromA>(a), + IsOkAndHolds(AllOf(Field(&ExplicitConstructibleFromA::x, 12), + Field(&ExplicitConstructibleFromA::moved, false)))); +} + +struct ImplicitConstructibleFromBool { + ImplicitConstructibleFromBool(bool y) : x(y) {} // NOLINT + bool x = false; +}; + +struct ConvertibleToBool { + explicit ConvertibleToBool(bool y) : x(y) {} + operator bool() const { return x; } // NOLINT + bool x = false; +}; + +TEST(StatusOr, ImplicitBooleanConstructionWithImplicitCasts) { + EXPECT_THAT(absl::StatusOr<bool>(absl::StatusOr<ConvertibleToBool>(true)), + IsOkAndHolds(true)); + EXPECT_THAT(absl::StatusOr<bool>(absl::StatusOr<ConvertibleToBool>(false)), + IsOkAndHolds(false)); + EXPECT_THAT( + absl::implicit_cast<absl::StatusOr<ImplicitConstructibleFromBool>>( + absl::StatusOr<bool>(false)), + IsOkAndHolds(Field(&ImplicitConstructibleFromBool::x, false))); + EXPECT_FALSE((std::is_convertible< + absl::StatusOr<ConvertibleToBool>, + absl::StatusOr<ImplicitConstructibleFromBool>>::value)); +} + +TEST(StatusOr, BooleanConstructionWithImplicitCasts) { + EXPECT_THAT(absl::StatusOr<bool>(absl::StatusOr<ConvertibleToBool>(true)), + IsOkAndHolds(true)); + EXPECT_THAT(absl::StatusOr<bool>(absl::StatusOr<ConvertibleToBool>(false)), + IsOkAndHolds(false)); + EXPECT_THAT( + absl::StatusOr<ImplicitConstructibleFromBool>{ + absl::StatusOr<bool>(false)}, + IsOkAndHolds(Field(&ImplicitConstructibleFromBool::x, false))); + EXPECT_THAT( + absl::StatusOr<ImplicitConstructibleFromBool>{ + absl::StatusOr<bool>(absl::InvalidArgumentError(""))}, + Not(IsOk())); + + EXPECT_THAT( + absl::StatusOr<ImplicitConstructibleFromBool>{ + absl::StatusOr<ConvertibleToBool>(ConvertibleToBool{false})}, + IsOkAndHolds(Field(&ImplicitConstructibleFromBool::x, false))); + EXPECT_THAT( + absl::StatusOr<ImplicitConstructibleFromBool>{ + absl::StatusOr<ConvertibleToBool>(absl::InvalidArgumentError(""))}, + Not(IsOk())); +} + +TEST(StatusOr, ConstImplicitCast) { + EXPECT_THAT(absl::implicit_cast<absl::StatusOr<bool>>( + absl::StatusOr<const bool>(true)), + IsOkAndHolds(true)); + EXPECT_THAT(absl::implicit_cast<absl::StatusOr<bool>>( + absl::StatusOr<const bool>(false)), + IsOkAndHolds(false)); + EXPECT_THAT(absl::implicit_cast<absl::StatusOr<const bool>>( + absl::StatusOr<bool>(true)), + IsOkAndHolds(true)); + EXPECT_THAT(absl::implicit_cast<absl::StatusOr<const bool>>( + absl::StatusOr<bool>(false)), + IsOkAndHolds(false)); + EXPECT_THAT(absl::implicit_cast<absl::StatusOr<const std::string>>( + absl::StatusOr<std::string>("foo")), + IsOkAndHolds("foo")); + EXPECT_THAT(absl::implicit_cast<absl::StatusOr<std::string>>( + absl::StatusOr<const std::string>("foo")), + IsOkAndHolds("foo")); + EXPECT_THAT( + absl::implicit_cast<absl::StatusOr<std::shared_ptr<const std::string>>>( + absl::StatusOr<std::shared_ptr<std::string>>( + std::make_shared<std::string>("foo"))), + IsOkAndHolds(Pointee(std::string("foo")))); +} + +TEST(StatusOr, ConstExplicitConstruction) { + EXPECT_THAT(absl::StatusOr<bool>(absl::StatusOr<const bool>(true)), + IsOkAndHolds(true)); + EXPECT_THAT(absl::StatusOr<bool>(absl::StatusOr<const bool>(false)), + IsOkAndHolds(false)); + EXPECT_THAT(absl::StatusOr<const bool>(absl::StatusOr<bool>(true)), + IsOkAndHolds(true)); + EXPECT_THAT(absl::StatusOr<const bool>(absl::StatusOr<bool>(false)), + IsOkAndHolds(false)); +} + +struct ExplicitConstructibleFromInt { + int x; + explicit ExplicitConstructibleFromInt(int y) : x(y) {} +}; + +TEST(StatusOr, ExplicitConstruction) { + EXPECT_THAT(absl::StatusOr<ExplicitConstructibleFromInt>(10), + IsOkAndHolds(Field(&ExplicitConstructibleFromInt::x, 10))); +} + +TEST(StatusOr, ImplicitConstruction) { + // Check implicit casting works. + auto status_or = + absl::implicit_cast<absl::StatusOr<absl::variant<int, std::string>>>(10); + EXPECT_THAT(status_or, IsOkAndHolds(VariantWith<int>(10))); +} + +TEST(StatusOr, ImplicitConstructionFromInitliazerList) { + // Note: dropping the explicit std::initializer_list<int> is not supported + // by absl::StatusOr or absl::optional. + auto status_or = + absl::implicit_cast<absl::StatusOr<std::vector<int>>>({{10, 20, 30}}); + EXPECT_THAT(status_or, IsOkAndHolds(ElementsAre(10, 20, 30))); +} + +TEST(StatusOr, UniquePtrImplicitConstruction) { + auto status_or = absl::implicit_cast<absl::StatusOr<std::unique_ptr<Base1>>>( + absl::make_unique<Derived>()); + EXPECT_THAT(status_or, IsOkAndHolds(Ne(nullptr))); +} + +TEST(StatusOr, NestedStatusOrCopyAndMoveConstructorTests) { + absl::StatusOr<absl::StatusOr<CopyDetector>> status_or = CopyDetector(10); + absl::StatusOr<absl::StatusOr<CopyDetector>> status_error = + absl::InvalidArgumentError("foo"); + EXPECT_THAT(status_or, + IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, true, false)))); + absl::StatusOr<absl::StatusOr<CopyDetector>> a = status_or; + EXPECT_THAT(a, IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, false, true)))); + absl::StatusOr<absl::StatusOr<CopyDetector>> a_err = status_error; + EXPECT_THAT(a_err, Not(IsOk())); + + const absl::StatusOr<absl::StatusOr<CopyDetector>>& cref = status_or; + absl::StatusOr<absl::StatusOr<CopyDetector>> b = cref; // NOLINT + EXPECT_THAT(b, IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, false, true)))); + const absl::StatusOr<absl::StatusOr<CopyDetector>>& cref_err = status_error; + absl::StatusOr<absl::StatusOr<CopyDetector>> b_err = cref_err; // NOLINT + EXPECT_THAT(b_err, Not(IsOk())); + + absl::StatusOr<absl::StatusOr<CopyDetector>> c = std::move(status_or); + EXPECT_THAT(c, IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, true, false)))); + absl::StatusOr<absl::StatusOr<CopyDetector>> c_err = std::move(status_error); + EXPECT_THAT(c_err, Not(IsOk())); +} + +TEST(StatusOr, NestedStatusOrCopyAndMoveAssignment) { + absl::StatusOr<absl::StatusOr<CopyDetector>> status_or = CopyDetector(10); + absl::StatusOr<absl::StatusOr<CopyDetector>> status_error = + absl::InvalidArgumentError("foo"); + absl::StatusOr<absl::StatusOr<CopyDetector>> a; + a = status_or; + EXPECT_THAT(a, IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, false, true)))); + a = status_error; + EXPECT_THAT(a, Not(IsOk())); + + const absl::StatusOr<absl::StatusOr<CopyDetector>>& cref = status_or; + a = cref; + EXPECT_THAT(a, IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, false, true)))); + const absl::StatusOr<absl::StatusOr<CopyDetector>>& cref_err = status_error; + a = cref_err; + EXPECT_THAT(a, Not(IsOk())); + a = std::move(status_or); + EXPECT_THAT(a, IsOkAndHolds(IsOkAndHolds(CopyDetectorHas(10, true, false)))); + a = std::move(status_error); + EXPECT_THAT(a, Not(IsOk())); +} + +struct Copyable { + Copyable() {} + Copyable(const Copyable&) {} + Copyable& operator=(const Copyable&) { return *this; } +}; + +struct MoveOnly { + MoveOnly() {} + MoveOnly(MoveOnly&&) {} + MoveOnly& operator=(MoveOnly&&) { return *this; } +}; + +struct NonMovable { + NonMovable() {} + NonMovable(const NonMovable&) = delete; + NonMovable(NonMovable&&) = delete; + NonMovable& operator=(const NonMovable&) = delete; + NonMovable& operator=(NonMovable&&) = delete; +}; + +TEST(StatusOr, CopyAndMoveAbility) { + EXPECT_TRUE(std::is_copy_constructible<Copyable>::value); + EXPECT_TRUE(std::is_copy_assignable<Copyable>::value); + EXPECT_TRUE(std::is_move_constructible<Copyable>::value); + EXPECT_TRUE(std::is_move_assignable<Copyable>::value); + EXPECT_FALSE(std::is_copy_constructible<MoveOnly>::value); + EXPECT_FALSE(std::is_copy_assignable<MoveOnly>::value); + EXPECT_TRUE(std::is_move_constructible<MoveOnly>::value); + EXPECT_TRUE(std::is_move_assignable<MoveOnly>::value); + EXPECT_FALSE(std::is_copy_constructible<NonMovable>::value); + EXPECT_FALSE(std::is_copy_assignable<NonMovable>::value); + EXPECT_FALSE(std::is_move_constructible<NonMovable>::value); + EXPECT_FALSE(std::is_move_assignable<NonMovable>::value); +} + +TEST(StatusOr, StatusOrAnyCopyAndMoveConstructorTests) { + absl::StatusOr<absl::any> status_or = CopyDetector(10); + absl::StatusOr<absl::any> status_error = absl::InvalidArgumentError("foo"); + EXPECT_THAT( + status_or, + IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, true, false)))); + absl::StatusOr<absl::any> a = status_or; + EXPECT_THAT( + a, IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, false, true)))); + absl::StatusOr<absl::any> a_err = status_error; + EXPECT_THAT(a_err, Not(IsOk())); + + const absl::StatusOr<absl::any>& cref = status_or; + // No lint for no-change copy. + absl::StatusOr<absl::any> b = cref; // NOLINT + EXPECT_THAT( + b, IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, false, true)))); + const absl::StatusOr<absl::any>& cref_err = status_error; + // No lint for no-change copy. + absl::StatusOr<absl::any> b_err = cref_err; // NOLINT + EXPECT_THAT(b_err, Not(IsOk())); + + absl::StatusOr<absl::any> c = std::move(status_or); + EXPECT_THAT( + c, IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, true, false)))); + absl::StatusOr<absl::any> c_err = std::move(status_error); + EXPECT_THAT(c_err, Not(IsOk())); +} + +TEST(StatusOr, StatusOrAnyCopyAndMoveAssignment) { + absl::StatusOr<absl::any> status_or = CopyDetector(10); + absl::StatusOr<absl::any> status_error = absl::InvalidArgumentError("foo"); + absl::StatusOr<absl::any> a; + a = status_or; + EXPECT_THAT( + a, IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, false, true)))); + a = status_error; + EXPECT_THAT(a, Not(IsOk())); + + const absl::StatusOr<absl::any>& cref = status_or; + a = cref; + EXPECT_THAT( + a, IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, false, true)))); + const absl::StatusOr<absl::any>& cref_err = status_error; + a = cref_err; + EXPECT_THAT(a, Not(IsOk())); + a = std::move(status_or); + EXPECT_THAT( + a, IsOkAndHolds(AnyWith<CopyDetector>(CopyDetectorHas(10, true, false)))); + a = std::move(status_error); + EXPECT_THAT(a, Not(IsOk())); +} + +TEST(StatusOr, StatusOrCopyAndMoveTestsConstructor) { + absl::StatusOr<CopyDetector> status_or(10); + ASSERT_THAT(status_or, IsOkAndHolds(CopyDetectorHas(10, false, false))); + absl::StatusOr<CopyDetector> a(status_or); + EXPECT_THAT(a, IsOkAndHolds(CopyDetectorHas(10, false, true))); + const absl::StatusOr<CopyDetector>& cref = status_or; + absl::StatusOr<CopyDetector> b(cref); // NOLINT + EXPECT_THAT(b, IsOkAndHolds(CopyDetectorHas(10, false, true))); + absl::StatusOr<CopyDetector> c(std::move(status_or)); + EXPECT_THAT(c, IsOkAndHolds(CopyDetectorHas(10, true, false))); +} + +TEST(StatusOr, StatusOrCopyAndMoveTestsAssignment) { + absl::StatusOr<CopyDetector> status_or(10); + ASSERT_THAT(status_or, IsOkAndHolds(CopyDetectorHas(10, false, false))); + absl::StatusOr<CopyDetector> a; + a = status_or; + EXPECT_THAT(a, IsOkAndHolds(CopyDetectorHas(10, false, true))); + const absl::StatusOr<CopyDetector>& cref = status_or; + absl::StatusOr<CopyDetector> b; + b = cref; + EXPECT_THAT(b, IsOkAndHolds(CopyDetectorHas(10, false, true))); + absl::StatusOr<CopyDetector> c; + c = std::move(status_or); + EXPECT_THAT(c, IsOkAndHolds(CopyDetectorHas(10, true, false))); +} + +TEST(StatusOr, AbslAnyAssignment) { + EXPECT_FALSE((std::is_assignable<absl::StatusOr<absl::any>, + absl::StatusOr<int>>::value)); + absl::StatusOr<absl::any> status_or; + status_or = absl::InvalidArgumentError("foo"); + EXPECT_THAT(status_or, Not(IsOk())); +} + +TEST(StatusOr, ImplicitAssignment) { + absl::StatusOr<absl::variant<int, std::string>> status_or; + status_or = 10; + EXPECT_THAT(status_or, IsOkAndHolds(VariantWith<int>(10))); +} + +TEST(StatusOr, SelfDirectInitAssignment) { + absl::StatusOr<std::vector<int>> status_or = {{10, 20, 30}}; + status_or = *status_or; + EXPECT_THAT(status_or, IsOkAndHolds(ElementsAre(10, 20, 30))); +} + +TEST(StatusOr, ImplicitCastFromInitializerList) { + absl::StatusOr<std::vector<int>> status_or = {{10, 20, 30}}; + EXPECT_THAT(status_or, IsOkAndHolds(ElementsAre(10, 20, 30))); +} + +TEST(StatusOr, UniquePtrImplicitAssignment) { + absl::StatusOr<std::unique_ptr<Base1>> status_or; + status_or = absl::make_unique<Derived>(); + EXPECT_THAT(status_or, IsOkAndHolds(Ne(nullptr))); +} + +TEST(StatusOr, Pointer) { + struct A {}; + struct B : public A {}; + struct C : private A {}; + + EXPECT_TRUE((std::is_constructible<absl::StatusOr<A*>, B*>::value)); + EXPECT_TRUE((std::is_convertible<B*, absl::StatusOr<A*>>::value)); + EXPECT_FALSE((std::is_constructible<absl::StatusOr<A*>, C*>::value)); + EXPECT_FALSE((std::is_convertible<C*, absl::StatusOr<A*>>::value)); +} + +TEST(StatusOr, TestAssignmentStatusNotOkConverting) { + // Copy assignment + { + const absl::Status expected = absl::CancelledError(); + absl::StatusOr<int> source(expected); + + absl::StatusOr<double> target; + target = source; + + EXPECT_FALSE(target.ok()); + EXPECT_EQ(expected, target.status()); + + EXPECT_FALSE(source.ok()); + EXPECT_EQ(expected, source.status()); + } + + // Move assignment + { + const absl::Status expected = absl::CancelledError(); + absl::StatusOr<int> source(expected); + + absl::StatusOr<double> target; + target = std::move(source); + + EXPECT_FALSE(target.ok()); + EXPECT_EQ(expected, target.status()); + + EXPECT_FALSE(source.ok()); + EXPECT_EQ(source.status().code(), absl::StatusCode::kInternal); + } +} + +TEST(StatusOr, SelfAssignment) { + // Copy-assignment, status OK + { + // A string long enough that it's likely to defeat any inline representation + // optimization. + const std::string long_str(128, 'a'); + + absl::StatusOr<std::string> so = long_str; + so = *&so; + + ASSERT_TRUE(so.ok()); + EXPECT_OK(so.status()); + EXPECT_EQ(long_str, *so); + } + + // Copy-assignment, error status + { + absl::StatusOr<int> so = absl::NotFoundError("taco"); + so = *&so; + + EXPECT_FALSE(so.ok()); + EXPECT_EQ(so.status().code(), absl::StatusCode::kNotFound); + EXPECT_EQ(so.status().message(), "taco"); + } + + // Move-assignment with copyable type, status OK + { + absl::StatusOr<int> so = 17; + + // Fool the compiler, which otherwise complains. + auto& same = so; + so = std::move(same); + + ASSERT_TRUE(so.ok()); + EXPECT_OK(so.status()); + EXPECT_EQ(17, *so); + } + + // Move-assignment with copyable type, error status + { + absl::StatusOr<int> so = absl::NotFoundError("taco"); + + // Fool the compiler, which otherwise complains. + auto& same = so; + so = std::move(same); + + EXPECT_FALSE(so.ok()); + EXPECT_EQ(so.status().code(), absl::StatusCode::kNotFound); + EXPECT_EQ(so.status().message(), "taco"); + } + + // Move-assignment with non-copyable type, status OK + { + const auto raw = new int(17); + absl::StatusOr<std::unique_ptr<int>> so = absl::WrapUnique(raw); + + // Fool the compiler, which otherwise complains. + auto& same = so; + so = std::move(same); + + ASSERT_TRUE(so.ok()); + EXPECT_OK(so.status()); + EXPECT_EQ(raw, so->get()); + } + + // Move-assignment with non-copyable type, error status + { + absl::StatusOr<std::unique_ptr<int>> so = absl::NotFoundError("taco"); + + // Fool the compiler, which otherwise complains. + auto& same = so; + so = std::move(same); + + EXPECT_FALSE(so.ok()); + EXPECT_EQ(so.status().code(), absl::StatusCode::kNotFound); + EXPECT_EQ(so.status().message(), "taco"); + } +} + +// These types form the overload sets of the constructors and the assignment +// operators of `MockValue`. They distinguish construction from assignment, +// lvalue from rvalue. +struct FromConstructibleAssignableLvalue {}; +struct FromConstructibleAssignableRvalue {}; +struct FromImplicitConstructibleOnly {}; +struct FromAssignableOnly {}; + +// This class is for testing the forwarding value assignments of `StatusOr`. +// `from_rvalue` indicates whether the constructor or the assignment taking +// rvalue reference is called. `from_assignment` indicates whether any +// assignment is called. +struct MockValue { + // Constructs `MockValue` from `FromConstructibleAssignableLvalue`. + MockValue(const FromConstructibleAssignableLvalue&) // NOLINT + : from_rvalue(false), assigned(false) {} + // Constructs `MockValue` from `FromConstructibleAssignableRvalue`. + MockValue(FromConstructibleAssignableRvalue&&) // NOLINT + : from_rvalue(true), assigned(false) {} + // Constructs `MockValue` from `FromImplicitConstructibleOnly`. + // `MockValue` is not assignable from `FromImplicitConstructibleOnly`. + MockValue(const FromImplicitConstructibleOnly&) // NOLINT + : from_rvalue(false), assigned(false) {} + // Assigns `FromConstructibleAssignableLvalue`. + MockValue& operator=(const FromConstructibleAssignableLvalue&) { + from_rvalue = false; + assigned = true; + return *this; + } + // Assigns `FromConstructibleAssignableRvalue` (rvalue only). + MockValue& operator=(FromConstructibleAssignableRvalue&&) { + from_rvalue = true; + assigned = true; + return *this; + } + // Assigns `FromAssignableOnly`, but not constructible from + // `FromAssignableOnly`. + MockValue& operator=(const FromAssignableOnly&) { + from_rvalue = false; + assigned = true; + return *this; + } + bool from_rvalue; + bool assigned; +}; + +// operator=(U&&) +TEST(StatusOr, PerfectForwardingAssignment) { + // U == T + constexpr int kValue1 = 10, kValue2 = 20; + absl::StatusOr<CopyDetector> status_or; + CopyDetector lvalue(kValue1); + status_or = lvalue; + EXPECT_THAT(status_or, IsOkAndHolds(CopyDetectorHas(kValue1, false, true))); + status_or = CopyDetector(kValue2); + EXPECT_THAT(status_or, IsOkAndHolds(CopyDetectorHas(kValue2, true, false))); + + // U != T + EXPECT_TRUE( + (std::is_assignable<absl::StatusOr<MockValue>&, + const FromConstructibleAssignableLvalue&>::value)); + EXPECT_TRUE((std::is_assignable<absl::StatusOr<MockValue>&, + FromConstructibleAssignableLvalue&&>::value)); + EXPECT_FALSE( + (std::is_assignable<absl::StatusOr<MockValue>&, + const FromConstructibleAssignableRvalue&>::value)); + EXPECT_TRUE((std::is_assignable<absl::StatusOr<MockValue>&, + FromConstructibleAssignableRvalue&&>::value)); + EXPECT_TRUE( + (std::is_assignable<absl::StatusOr<MockValue>&, + const FromImplicitConstructibleOnly&>::value)); + EXPECT_FALSE((std::is_assignable<absl::StatusOr<MockValue>&, + const FromAssignableOnly&>::value)); + + absl::StatusOr<MockValue> from_lvalue(FromConstructibleAssignableLvalue{}); + EXPECT_FALSE(from_lvalue->from_rvalue); + EXPECT_FALSE(from_lvalue->assigned); + from_lvalue = FromConstructibleAssignableLvalue{}; + EXPECT_FALSE(from_lvalue->from_rvalue); + EXPECT_TRUE(from_lvalue->assigned); + + absl::StatusOr<MockValue> from_rvalue(FromConstructibleAssignableRvalue{}); + EXPECT_TRUE(from_rvalue->from_rvalue); + EXPECT_FALSE(from_rvalue->assigned); + from_rvalue = FromConstructibleAssignableRvalue{}; + EXPECT_TRUE(from_rvalue->from_rvalue); + EXPECT_TRUE(from_rvalue->assigned); + + absl::StatusOr<MockValue> from_implicit_constructible( + FromImplicitConstructibleOnly{}); + EXPECT_FALSE(from_implicit_constructible->from_rvalue); + EXPECT_FALSE(from_implicit_constructible->assigned); + // construct a temporary `StatusOr` object and invoke the `StatusOr` move + // assignment operator. + from_implicit_constructible = FromImplicitConstructibleOnly{}; + EXPECT_FALSE(from_implicit_constructible->from_rvalue); + EXPECT_FALSE(from_implicit_constructible->assigned); +} + +TEST(StatusOr, TestStatus) { + absl::StatusOr<int> good(4); + EXPECT_TRUE(good.ok()); + absl::StatusOr<int> bad(absl::CancelledError()); + EXPECT_FALSE(bad.ok()); + EXPECT_EQ(bad.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, OperatorStarRefQualifiers) { + static_assert( + std::is_same<const int&, + decltype(*std::declval<const absl::StatusOr<int>&>())>(), + "Unexpected ref-qualifiers"); + static_assert( + std::is_same<int&, decltype(*std::declval<absl::StatusOr<int>&>())>(), + "Unexpected ref-qualifiers"); + static_assert( + std::is_same<const int&&, + decltype(*std::declval<const absl::StatusOr<int>&&>())>(), + "Unexpected ref-qualifiers"); + static_assert( + std::is_same<int&&, decltype(*std::declval<absl::StatusOr<int>&&>())>(), + "Unexpected ref-qualifiers"); +} + +TEST(StatusOr, OperatorStar) { + const absl::StatusOr<std::string> const_lvalue("hello"); + EXPECT_EQ("hello", *const_lvalue); + + absl::StatusOr<std::string> lvalue("hello"); + EXPECT_EQ("hello", *lvalue); + + // Note: Recall that std::move() is equivalent to a static_cast to an rvalue + // reference type. + const absl::StatusOr<std::string> const_rvalue("hello"); + EXPECT_EQ("hello", *std::move(const_rvalue)); // NOLINT + + absl::StatusOr<std::string> rvalue("hello"); + EXPECT_EQ("hello", *std::move(rvalue)); +} + +TEST(StatusOr, OperatorArrowQualifiers) { + static_assert( + std::is_same< + const int*, + decltype(std::declval<const absl::StatusOr<int>&>().operator->())>(), + "Unexpected qualifiers"); + static_assert( + std::is_same< + int*, decltype(std::declval<absl::StatusOr<int>&>().operator->())>(), + "Unexpected qualifiers"); + static_assert( + std::is_same< + const int*, + decltype(std::declval<const absl::StatusOr<int>&&>().operator->())>(), + "Unexpected qualifiers"); + static_assert( + std::is_same< + int*, decltype(std::declval<absl::StatusOr<int>&&>().operator->())>(), + "Unexpected qualifiers"); +} + +TEST(StatusOr, OperatorArrow) { + const absl::StatusOr<std::string> const_lvalue("hello"); + EXPECT_EQ(std::string("hello"), const_lvalue->c_str()); + + absl::StatusOr<std::string> lvalue("hello"); + EXPECT_EQ(std::string("hello"), lvalue->c_str()); +} + +TEST(StatusOr, RValueStatus) { + absl::StatusOr<int> so(absl::NotFoundError("taco")); + const absl::Status s = std::move(so).status(); + + EXPECT_EQ(s.code(), absl::StatusCode::kNotFound); + EXPECT_EQ(s.message(), "taco"); + + // Check that !ok() still implies !status().ok(), even after moving out of the + // object. See the note on the rvalue ref-qualified status method. + EXPECT_FALSE(so.ok()); // NOLINT + EXPECT_FALSE(so.status().ok()); + EXPECT_EQ(so.status().code(), absl::StatusCode::kInternal); + EXPECT_EQ(so.status().message(), "Status accessed after move."); +} + +TEST(StatusOr, TestValue) { + const int kI = 4; + absl::StatusOr<int> thing(kI); + EXPECT_EQ(kI, *thing); +} + +TEST(StatusOr, TestValueConst) { + const int kI = 4; + const absl::StatusOr<int> thing(kI); + EXPECT_EQ(kI, *thing); +} + +TEST(StatusOr, TestPointerDefaultCtor) { + absl::StatusOr<int*> thing; + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), absl::StatusCode::kUnknown); +} + + + +TEST(StatusOr, TestPointerStatusCtor) { + absl::StatusOr<int*> thing(absl::CancelledError()); + EXPECT_FALSE(thing.ok()); + EXPECT_EQ(thing.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, TestPointerValueCtor) { + const int kI = 4; + + // Construction from a non-null pointer + { + absl::StatusOr<const int*> so(&kI); + EXPECT_TRUE(so.ok()); + EXPECT_OK(so.status()); + EXPECT_EQ(&kI, *so); + } + + // Construction from a null pointer constant + { + absl::StatusOr<const int*> so(nullptr); + EXPECT_TRUE(so.ok()); + EXPECT_OK(so.status()); + EXPECT_EQ(nullptr, *so); + } + + // Construction from a non-literal null pointer + { + const int* const p = nullptr; + + absl::StatusOr<const int*> so(p); + EXPECT_TRUE(so.ok()); + EXPECT_OK(so.status()); + EXPECT_EQ(nullptr, *so); + } +} + +TEST(StatusOr, TestPointerCopyCtorStatusOk) { + const int kI = 0; + absl::StatusOr<const int*> original(&kI); + absl::StatusOr<const int*> copy(original); + EXPECT_OK(copy.status()); + EXPECT_EQ(*original, *copy); +} + +TEST(StatusOr, TestPointerCopyCtorStatusNotOk) { + absl::StatusOr<int*> original(absl::CancelledError()); + absl::StatusOr<int*> copy(original); + EXPECT_EQ(copy.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, TestPointerCopyCtorStatusOKConverting) { + Derived derived; + absl::StatusOr<Derived*> original(&derived); + absl::StatusOr<Base2*> copy(original); + EXPECT_OK(copy.status()); + EXPECT_EQ(static_cast<const Base2*>(*original), *copy); +} + +TEST(StatusOr, TestPointerCopyCtorStatusNotOkConverting) { + absl::StatusOr<Derived*> original(absl::CancelledError()); + absl::StatusOr<Base2*> copy(original); + EXPECT_EQ(copy.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, TestPointerAssignmentStatusOk) { + const int kI = 0; + absl::StatusOr<const int*> source(&kI); + absl::StatusOr<const int*> target; + target = source; + EXPECT_OK(target.status()); + EXPECT_EQ(*source, *target); +} + +TEST(StatusOr, TestPointerAssignmentStatusNotOk) { + absl::StatusOr<int*> source(absl::CancelledError()); + absl::StatusOr<int*> target; + target = source; + EXPECT_EQ(target.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, TestPointerAssignmentStatusOKConverting) { + Derived derived; + absl::StatusOr<Derived*> source(&derived); + absl::StatusOr<Base2*> target; + target = source; + EXPECT_OK(target.status()); + EXPECT_EQ(static_cast<const Base2*>(*source), *target); +} + +TEST(StatusOr, TestPointerAssignmentStatusNotOkConverting) { + absl::StatusOr<Derived*> source(absl::CancelledError()); + absl::StatusOr<Base2*> target; + target = source; + EXPECT_EQ(target.status(), source.status()); +} + +TEST(StatusOr, TestPointerStatus) { + const int kI = 0; + absl::StatusOr<const int*> good(&kI); + EXPECT_TRUE(good.ok()); + absl::StatusOr<const int*> bad(absl::CancelledError()); + EXPECT_EQ(bad.status().code(), absl::StatusCode::kCancelled); +} + +TEST(StatusOr, TestPointerValue) { + const int kI = 0; + absl::StatusOr<const int*> thing(&kI); + EXPECT_EQ(&kI, *thing); +} + +TEST(StatusOr, TestPointerValueConst) { + const int kI = 0; + const absl::StatusOr<const int*> thing(&kI); + EXPECT_EQ(&kI, *thing); +} + +TEST(StatusOr, StatusOrVectorOfUniquePointerCanReserveAndResize) { + using EvilType = std::vector<std::unique_ptr<int>>; + static_assert(std::is_copy_constructible<EvilType>::value, ""); + std::vector<::absl::StatusOr<EvilType>> v(5); + v.reserve(v.capacity() + 10); + v.resize(v.capacity() + 10); +} + +TEST(StatusOr, ConstPayload) { + // A reduced version of a problematic type found in the wild. All of the + // operations below should compile. + absl::StatusOr<const int> a; + + // Copy-construction + absl::StatusOr<const int> b(a); + + // Copy-assignment + EXPECT_FALSE(std::is_copy_assignable<absl::StatusOr<const int>>::value); + + // Move-construction + absl::StatusOr<const int> c(std::move(a)); + + // Move-assignment + EXPECT_FALSE(std::is_move_assignable<absl::StatusOr<const int>>::value); +} + +TEST(StatusOr, MapToStatusOrUniquePtr) { + // A reduced version of a problematic type found in the wild. All of the + // operations below should compile. + using MapType = std::map<std::string, absl::StatusOr<std::unique_ptr<int>>>; + + MapType a; + + // Move-construction + MapType b(std::move(a)); + + // Move-assignment + a = std::move(b); +} + +TEST(StatusOr, ValueOrOk) { + const absl::StatusOr<int> status_or = 0; + EXPECT_EQ(status_or.value_or(-1), 0); +} + +TEST(StatusOr, ValueOrDefault) { + const absl::StatusOr<int> status_or = absl::CancelledError(); + EXPECT_EQ(status_or.value_or(-1), -1); +} + +TEST(StatusOr, MoveOnlyValueOrOk) { + EXPECT_THAT(absl::StatusOr<std::unique_ptr<int>>(absl::make_unique<int>(0)) + .value_or(absl::make_unique<int>(-1)), + Pointee(0)); +} + +TEST(StatusOr, MoveOnlyValueOrDefault) { + EXPECT_THAT(absl::StatusOr<std::unique_ptr<int>>(absl::CancelledError()) + .value_or(absl::make_unique<int>(-1)), + Pointee(-1)); +} + +static absl::StatusOr<int> MakeStatus() { return 100; } + +TEST(StatusOr, TestIgnoreError) { MakeStatus().IgnoreError(); } + +TEST(StatusOr, EqualityOperator) { + constexpr int kNumCases = 4; + std::array<absl::StatusOr<int>, kNumCases> group1 = { + absl::StatusOr<int>(1), absl::StatusOr<int>(2), + absl::StatusOr<int>(absl::InvalidArgumentError("msg")), + absl::StatusOr<int>(absl::InternalError("msg"))}; + std::array<absl::StatusOr<int>, kNumCases> group2 = { + absl::StatusOr<int>(1), absl::StatusOr<int>(2), + absl::StatusOr<int>(absl::InvalidArgumentError("msg")), + absl::StatusOr<int>(absl::InternalError("msg"))}; + for (int i = 0; i < kNumCases; ++i) { + for (int j = 0; j < kNumCases; ++j) { + if (i == j) { + EXPECT_TRUE(group1[i] == group2[j]); + EXPECT_FALSE(group1[i] != group2[j]); + } else { + EXPECT_FALSE(group1[i] == group2[j]); + EXPECT_TRUE(group1[i] != group2[j]); + } + } + } +} + +struct MyType { + bool operator==(const MyType&) const { return true; } +}; + +enum class ConvTraits { kNone = 0, kImplicit = 1, kExplicit = 2 }; + +// This class has conversion operator to `StatusOr<T>` based on value of +// `conv_traits`. +template <typename T, ConvTraits conv_traits = ConvTraits::kNone> +struct StatusOrConversionBase {}; + +template <typename T> +struct StatusOrConversionBase<T, ConvTraits::kImplicit> { + operator absl::StatusOr<T>() const& { // NOLINT + return absl::InvalidArgumentError("conversion to absl::StatusOr"); + } + operator absl::StatusOr<T>() && { // NOLINT + return absl::InvalidArgumentError("conversion to absl::StatusOr"); + } +}; + +template <typename T> +struct StatusOrConversionBase<T, ConvTraits::kExplicit> { + explicit operator absl::StatusOr<T>() const& { + return absl::InvalidArgumentError("conversion to absl::StatusOr"); + } + explicit operator absl::StatusOr<T>() && { + return absl::InvalidArgumentError("conversion to absl::StatusOr"); + } +}; + +// This class has conversion operator to `T` based on the value of +// `conv_traits`. +template <typename T, ConvTraits conv_traits = ConvTraits::kNone> +struct ConversionBase {}; + +template <typename T> +struct ConversionBase<T, ConvTraits::kImplicit> { + operator T() const& { return t; } // NOLINT + operator T() && { return std::move(t); } // NOLINT + T t; +}; + +template <typename T> +struct ConversionBase<T, ConvTraits::kExplicit> { + explicit operator T() const& { return t; } + explicit operator T() && { return std::move(t); } + T t; +}; + +// This class has conversion operator to `absl::Status` based on the value of +// `conv_traits`. +template <ConvTraits conv_traits = ConvTraits::kNone> +struct StatusConversionBase {}; + +template <> +struct StatusConversionBase<ConvTraits::kImplicit> { + operator absl::Status() const& { // NOLINT + return absl::InternalError("conversion to Status"); + } + operator absl::Status() && { // NOLINT + return absl::InternalError("conversion to Status"); + } +}; + +template <> +struct StatusConversionBase<ConvTraits::kExplicit> { + explicit operator absl::Status() const& { // NOLINT + return absl::InternalError("conversion to Status"); + } + explicit operator absl::Status() && { // NOLINT + return absl::InternalError("conversion to Status"); + } +}; + +static constexpr int kConvToStatus = 1; +static constexpr int kConvToStatusOr = 2; +static constexpr int kConvToT = 4; +static constexpr int kConvExplicit = 8; + +constexpr ConvTraits GetConvTraits(int bit, int config) { + return (config & bit) == 0 + ? ConvTraits::kNone + : ((config & kConvExplicit) == 0 ? ConvTraits::kImplicit + : ConvTraits::kExplicit); +} + +// This class conditionally has conversion operator to `absl::Status`, `T`, +// `StatusOr<T>`, based on values of the template parameters. +template <typename T, int config> +struct CustomType + : StatusOrConversionBase<T, GetConvTraits(kConvToStatusOr, config)>, + ConversionBase<T, GetConvTraits(kConvToT, config)>, + StatusConversionBase<GetConvTraits(kConvToStatus, config)> {}; + +struct ConvertibleToAnyStatusOr { + template <typename T> + operator absl::StatusOr<T>() const { // NOLINT + return absl::InvalidArgumentError("Conversion to absl::StatusOr"); + } +}; + +// Test the rank of overload resolution for `StatusOr<T>` constructor and +// assignment, from highest to lowest: +// 1. T/Status +// 2. U that has conversion operator to absl::StatusOr<T> +// 3. U that is convertible to Status +// 4. U that is convertible to T +TEST(StatusOr, ConstructionFromT) { + // Construct absl::StatusOr<T> from T when T is convertible to + // absl::StatusOr<T> + { + ConvertibleToAnyStatusOr v; + absl::StatusOr<ConvertibleToAnyStatusOr> statusor(v); + EXPECT_TRUE(statusor.ok()); + } + { + ConvertibleToAnyStatusOr v; + absl::StatusOr<ConvertibleToAnyStatusOr> statusor = v; + EXPECT_TRUE(statusor.ok()); + } + // Construct absl::StatusOr<T> from T when T is explicitly convertible to + // Status + { + CustomType<MyType, kConvToStatus | kConvExplicit> v; + absl::StatusOr<CustomType<MyType, kConvToStatus | kConvExplicit>> statusor( + v); + EXPECT_TRUE(statusor.ok()); + } + { + CustomType<MyType, kConvToStatus | kConvExplicit> v; + absl::StatusOr<CustomType<MyType, kConvToStatus | kConvExplicit>> statusor = + v; + EXPECT_TRUE(statusor.ok()); + } +} + +// Construct absl::StatusOr<T> from U when U is explicitly convertible to T +TEST(StatusOr, ConstructionFromTypeConvertibleToT) { + { + CustomType<MyType, kConvToT | kConvExplicit> v; + absl::StatusOr<MyType> statusor(v); + EXPECT_TRUE(statusor.ok()); + } + { + CustomType<MyType, kConvToT> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_TRUE(statusor.ok()); + } +} + +// Construct absl::StatusOr<T> from U when U has explicit conversion operator to +// absl::StatusOr<T> +TEST(StatusOr, ConstructionFromTypeWithConversionOperatorToStatusOrT) { + { + CustomType<MyType, kConvToStatusOr | kConvExplicit> v; + absl::StatusOr<MyType> statusor(v); + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToT | kConvToStatusOr | kConvExplicit> v; + absl::StatusOr<MyType> statusor(v); + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToStatusOr | kConvToStatus | kConvExplicit> v; + absl::StatusOr<MyType> statusor(v); + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, + kConvToT | kConvToStatusOr | kConvToStatus | kConvExplicit> + v; + absl::StatusOr<MyType> statusor(v); + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToStatusOr> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToT | kConvToStatusOr> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToStatusOr | kConvToStatus> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToT | kConvToStatusOr | kConvToStatus> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } +} + +TEST(StatusOr, ConstructionFromTypeConvertibleToStatus) { + // Construction fails because conversion to `Status` is explicit. + { + CustomType<MyType, kConvToStatus | kConvExplicit> v; + absl::StatusOr<MyType> statusor(v); + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status(), static_cast<absl::Status>(v)); + } + { + CustomType<MyType, kConvToT | kConvToStatus | kConvExplicit> v; + absl::StatusOr<MyType> statusor(v); + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status(), static_cast<absl::Status>(v)); + } + { + CustomType<MyType, kConvToStatus> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status(), static_cast<absl::Status>(v)); + } + { + CustomType<MyType, kConvToT | kConvToStatus> v; + absl::StatusOr<MyType> statusor = v; + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status(), static_cast<absl::Status>(v)); + } +} + +TEST(StatusOr, AssignmentFromT) { + // Assign to absl::StatusOr<T> from T when T is convertible to + // absl::StatusOr<T> + { + ConvertibleToAnyStatusOr v; + absl::StatusOr<ConvertibleToAnyStatusOr> statusor; + statusor = v; + EXPECT_TRUE(statusor.ok()); + } + // Assign to absl::StatusOr<T> from T when T is convertible to Status + { + CustomType<MyType, kConvToStatus> v; + absl::StatusOr<CustomType<MyType, kConvToStatus>> statusor; + statusor = v; + EXPECT_TRUE(statusor.ok()); + } +} + +TEST(StatusOr, AssignmentFromTypeConvertibleToT) { + // Assign to absl::StatusOr<T> from U when U is convertible to T + { + CustomType<MyType, kConvToT> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_TRUE(statusor.ok()); + } +} + +TEST(StatusOr, AssignmentFromTypeWithConversionOperatortoStatusOrT) { + // Assign to absl::StatusOr<T> from U when U has conversion operator to + // absl::StatusOr<T> + { + CustomType<MyType, kConvToStatusOr> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToT | kConvToStatusOr> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToStatusOr | kConvToStatus> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } + { + CustomType<MyType, kConvToT | kConvToStatusOr | kConvToStatus> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_EQ(statusor, v.operator absl::StatusOr<MyType>()); + } +} + +TEST(StatusOr, AssignmentFromTypeConvertibleToStatus) { + // Assign to absl::StatusOr<T> from U when U is convertible to Status + { + CustomType<MyType, kConvToStatus> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status(), static_cast<absl::Status>(v)); + } + { + CustomType<MyType, kConvToT | kConvToStatus> v; + absl::StatusOr<MyType> statusor; + statusor = v; + EXPECT_FALSE(statusor.ok()); + EXPECT_EQ(statusor.status(), static_cast<absl::Status>(v)); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/BUILD.bazel b/third_party/abseil_cpp/absl/strings/BUILD.bazel new file mode 100644 index 000000000000..30a8dd28b2d1 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/BUILD.bazel @@ -0,0 +1,788 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_TEST_COPTS", +) + +package( + default_visibility = ["//visibility:public"], + features = ["parse_headers"], +) + +licenses(["notice"]) + +cc_library( + name = "strings", + srcs = [ + "ascii.cc", + "charconv.cc", + "escaping.cc", + "internal/charconv_bigint.cc", + "internal/charconv_bigint.h", + "internal/charconv_parse.cc", + "internal/charconv_parse.h", + "internal/memutil.cc", + "internal/memutil.h", + "internal/stl_type_traits.h", + "internal/str_join_internal.h", + "internal/str_split_internal.h", + "match.cc", + "numbers.cc", + "str_cat.cc", + "str_replace.cc", + "str_split.cc", + "string_view.cc", + "substitute.cc", + ], + hdrs = [ + "ascii.h", + "charconv.h", + "escaping.h", + "internal/string_constant.h", + "match.h", + "numbers.h", + "str_cat.h", + "str_join.h", + "str_replace.h", + "str_split.h", + "string_view.h", + "strip.h", + "substitute.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":internal", + "//absl/base", + "//absl/base:bits", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/base:raw_logging_internal", + "//absl/base:throw_delegate", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/numeric:int128", + ], +) + +cc_library( + name = "internal", + srcs = [ + "internal/escaping.cc", + "internal/ostringstream.cc", + "internal/utf8.cc", + ], + hdrs = [ + "internal/char_map.h", + "internal/escaping.h", + "internal/ostringstream.h", + "internal/resize_uninitialized.h", + "internal/utf8.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/base:raw_logging_internal", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "match_test", + size = "small", + srcs = ["match_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "escaping_test", + size = "small", + srcs = [ + "escaping_test.cc", + "internal/escaping_test_common.h", + ], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":strings", + "//absl/base:core_headers", + "//absl/container:fixed_array", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "escaping_benchmark", + srcs = [ + "escaping_benchmark.cc", + "internal/escaping_test_common.h", + ], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:raw_logging_internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "ascii_test", + size = "small", + srcs = ["ascii_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "ascii_benchmark", + srcs = ["ascii_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "memutil_benchmark", + srcs = [ + "internal/memutil.h", + "internal/memutil_benchmark.cc", + ], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "memutil_test", + size = "small", + srcs = [ + "internal/memutil.h", + "internal/memutil_test.cc", + ], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "utf8_test", + size = "small", + srcs = [ + "internal/utf8_test.cc", + ], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":internal", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "string_constant_test", + size = "small", + srcs = ["internal/string_constant_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/meta:type_traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "string_view_benchmark", + srcs = ["string_view_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "string_view_test", + size = "small", + srcs = ["string_view_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:dynamic_annotations", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "cord_internal", + hdrs = ["internal/cord_internal.h"], + copts = ABSL_DEFAULT_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:base_internal", + "//absl/container:compressed_tuple", + "//absl/meta:type_traits", + ], +) + +cc_library( + name = "cord", + srcs = [ + "cord.cc", + ], + hdrs = [ + "cord.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord_internal", + ":internal", + ":str_format", + ":strings", + "//absl/base", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/base:raw_logging_internal", + "//absl/container:fixed_array", + "//absl/container:inlined_vector", + "//absl/functional:function_ref", + "//absl/meta:type_traits", + "//absl/types:optional", + ], +) + +cc_library( + name = "cord_test_helpers", + testonly = 1, + hdrs = [ + "cord_test_helpers.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord", + ], +) + +cc_test( + name = "cord_test", + size = "medium", + srcs = ["cord_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":cord_test_helpers", + ":str_format", + ":strings", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:endian", + "//absl/base:raw_logging_internal", + "//absl/container:fixed_array", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "substitute_test", + size = "small", + srcs = ["substitute_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_replace_benchmark", + srcs = ["str_replace_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:raw_logging_internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "str_replace_test", + size = "small", + srcs = ["str_replace_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_split_test", + srcs = ["str_split_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "//absl/base:dynamic_annotations", + "//absl/container:flat_hash_map", + "//absl/container:node_hash_map", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_split_benchmark", + srcs = ["str_split_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:raw_logging_internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "ostringstream_test", + size = "small", + srcs = ["internal/ostringstream_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "ostringstream_benchmark", + srcs = ["internal/ostringstream_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "resize_uninitialized_test", + size = "small", + srcs = [ + "internal/resize_uninitialized.h", + "internal/resize_uninitialized_test.cc", + ], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + "//absl/base:core_headers", + "//absl/meta:type_traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_join_test", + size = "small", + srcs = ["str_join_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_join_benchmark", + srcs = ["str_join_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "str_cat_test", + size = "small", + srcs = ["str_cat_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_cat_benchmark", + srcs = ["str_cat_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "numbers_test", + size = "medium", + srcs = [ + "internal/numbers_test_common.h", + "numbers_test.cc", + ], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":internal", + ":pow10_helper", + ":strings", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/random", + "//absl/random:distributions", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "numbers_benchmark", + srcs = ["numbers_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:raw_logging_internal", + "//absl/random", + "//absl/random:distributions", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "strip_test", + size = "small", + srcs = ["strip_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "char_map_test", + srcs = ["internal/char_map_test.cc"], + copts = ABSL_TEST_COPTS, + deps = [ + ":internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "char_map_benchmark", + srcs = ["internal/char_map_benchmark.cc"], + copts = ABSL_TEST_COPTS, + tags = ["benchmark"], + deps = [ + ":internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "charconv_test", + srcs = ["charconv_test.cc"], + copts = ABSL_TEST_COPTS, + deps = [ + ":pow10_helper", + ":str_format", + ":strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "charconv_parse_test", + srcs = [ + "internal/charconv_parse.h", + "internal/charconv_parse_test.cc", + ], + copts = ABSL_TEST_COPTS, + deps = [ + ":strings", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "charconv_bigint_test", + srcs = [ + "internal/charconv_bigint.h", + "internal/charconv_bigint_test.cc", + "internal/charconv_parse.h", + ], + copts = ABSL_TEST_COPTS, + deps = [ + ":strings", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "charconv_benchmark", + srcs = [ + "charconv_benchmark.cc", + ], + tags = [ + "benchmark", + ], + deps = [ + ":strings", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "str_format", + hdrs = [ + "str_format.h", + ], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":str_format_internal", + ], +) + +cc_library( + name = "str_format_internal", + srcs = [ + "internal/str_format/arg.cc", + "internal/str_format/bind.cc", + "internal/str_format/extension.cc", + "internal/str_format/float_conversion.cc", + "internal/str_format/output.cc", + "internal/str_format/parser.cc", + ], + hdrs = [ + "internal/str_format/arg.h", + "internal/str_format/bind.h", + "internal/str_format/checker.h", + "internal/str_format/extension.h", + "internal/str_format/float_conversion.h", + "internal/str_format/output.h", + "internal/str_format/parser.h", + ], + copts = ABSL_DEFAULT_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":strings", + "//absl/base:bits", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/functional:function_ref", + "//absl/meta:type_traits", + "//absl/numeric:int128", + "//absl/types:optional", + "//absl/types:span", + ], +) + +cc_test( + name = "str_format_test", + srcs = ["str_format_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":str_format", + ":strings", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_extension_test", + srcs = [ + "internal/str_format/extension_test.cc", + ], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":str_format", + ":str_format_internal", + ":strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_arg_test", + srcs = ["internal/str_format/arg_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":str_format", + ":str_format_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_bind_test", + srcs = ["internal/str_format/bind_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":str_format_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_checker_test", + srcs = ["internal/str_format/checker_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":str_format", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_convert_test", + size = "medium", + srcs = ["internal/str_format/convert_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":str_format_internal", + ":strings", + "//absl/base:raw_logging_internal", + "//absl/types:optional", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_output_test", + srcs = ["internal/str_format/output_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":str_format_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "str_format_parser_test", + srcs = ["internal/str_format/parser_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":str_format_internal", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "pow10_helper", + testonly = True, + srcs = ["internal/pow10_helper.cc"], + hdrs = ["internal/pow10_helper.h"], + visibility = ["//visibility:private"], + deps = ["//absl/base:config"], +) + +cc_test( + name = "pow10_helper_test", + srcs = ["internal/pow10_helper_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":pow10_helper", + ":str_format", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/strings/CMakeLists.txt b/third_party/abseil_cpp/absl/strings/CMakeLists.txt new file mode 100644 index 000000000000..2b994a71c07e --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/CMakeLists.txt @@ -0,0 +1,609 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + strings + HDRS + "ascii.h" + "charconv.h" + "escaping.h" + "internal/string_constant.h" + "match.h" + "numbers.h" + "str_cat.h" + "str_join.h" + "str_replace.h" + "str_split.h" + "string_view.h" + "strip.h" + "substitute.h" + SRCS + "ascii.cc" + "charconv.cc" + "escaping.cc" + "internal/charconv_bigint.cc" + "internal/charconv_bigint.h" + "internal/charconv_parse.cc" + "internal/charconv_parse.h" + "internal/memutil.cc" + "internal/memutil.h" + "internal/stl_type_traits.h" + "internal/str_join_internal.h" + "internal/str_split_internal.h" + "match.cc" + "numbers.cc" + "str_cat.cc" + "str_replace.cc" + "str_split.cc" + "string_view.cc" + "substitute.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::strings_internal + absl::base + absl::bits + absl::config + absl::core_headers + absl::endian + absl::int128 + absl::memory + absl::raw_logging_internal + absl::throw_delegate + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + strings_internal + HDRS + "internal/char_map.h" + "internal/escaping.cc" + "internal/escaping.h" + "internal/ostringstream.h" + "internal/resize_uninitialized.h" + "internal/utf8.h" + SRCS + "internal/ostringstream.cc" + "internal/utf8.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::endian + absl::raw_logging_internal + absl::type_traits +) + +absl_cc_test( + NAME + match_test + SRCS + "match_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::base + gmock_main +) + +absl_cc_test( + NAME + escaping_test + SRCS + "escaping_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::core_headers + absl::fixed_array + gmock_main +) + +absl_cc_test( + NAME + ascii_test + SRCS + "ascii_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + memutil_test + SRCS + "internal/memutil.h" + "internal/memutil_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + utf8_test + SRCS + "internal/utf8_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings_internal + absl::base + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + string_constant_test + SRCS + "internal/string_constant_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::type_traits + gmock_main +) + +absl_cc_test( + NAME + string_view_test + SRCS + "string_view_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::config + absl::core_headers + absl::dynamic_annotations + gmock_main +) + +absl_cc_test( + NAME + substitute_test + SRCS + "substitute_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + str_replace_test + SRCS + "str_replace_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + gmock_main +) + +absl_cc_test( + NAME + str_split_test + SRCS + "str_split_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::base + absl::core_headers + absl::dynamic_annotations + absl::flat_hash_map + absl::node_hash_map + gmock_main +) + +absl_cc_test( + NAME + ostringstream_test + SRCS + "internal/ostringstream_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings_internal + gmock_main +) + +absl_cc_test( + NAME + resize_uninitialized_test + SRCS + "internal/resize_uninitialized.h" + "internal/resize_uninitialized_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::core_headers + absl::type_traits + gmock_main +) + +absl_cc_test( + NAME + str_join_test + SRCS + "str_join_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::base + absl::core_headers + absl::memory + gmock_main +) + +absl_cc_test( + NAME + str_cat_test + SRCS + "str_cat_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + numbers_test + SRCS + "internal/numbers_test_common.h" + "numbers_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::core_headers + absl::pow10_helper + absl::config + absl::raw_logging_internal + absl::random_random + absl::random_distributions + absl::strings_internal + gmock_main +) + +absl_cc_test( + NAME + strip_test + SRCS + "strip_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::base + gmock_main +) + +absl_cc_test( + NAME + char_map_test + SRCS + "internal/char_map_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings_internal + gmock_main +) + +absl_cc_test( + NAME + charconv_test + SRCS + "charconv_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::str_format + absl::pow10_helper + gmock_main +) + +absl_cc_test( + NAME + charconv_parse_test + SRCS + "internal/charconv_parse.h" + "internal/charconv_parse_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::config + absl::raw_logging_internal + gmock_main +) + +absl_cc_test( + NAME + charconv_bigint_test + SRCS + "internal/charconv_bigint.h" + "internal/charconv_bigint_test.cc" + "internal/charconv_parse.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::config + gmock_main +) + +absl_cc_library( + NAME + str_format + HDRS + "str_format.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::str_format_internal + PUBLIC +) + +absl_cc_library( + NAME + str_format_internal + HDRS + "internal/str_format/arg.h" + "internal/str_format/bind.h" + "internal/str_format/checker.h" + "internal/str_format/extension.h" + "internal/str_format/float_conversion.h" + "internal/str_format/output.h" + "internal/str_format/parser.h" + SRCS + "internal/str_format/arg.cc" + "internal/str_format/bind.cc" + "internal/str_format/extension.cc" + "internal/str_format/float_conversion.cc" + "internal/str_format/output.cc" + "internal/str_format/parser.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bits + absl::strings + absl::config + absl::core_headers + absl::type_traits + absl::int128 + absl::span +) + +absl_cc_test( + NAME + str_format_test + SRCS + "str_format_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format + absl::cord + absl::strings + absl::core_headers + gmock_main +) + +absl_cc_test( + NAME + str_format_extension_test + SRCS + "internal/str_format/extension_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format + absl::str_format_internal + absl::strings + gmock_main +) + +absl_cc_test( + NAME + str_format_arg_test + SRCS + "internal/str_format/arg_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format + absl::str_format_internal + gmock_main +) + +absl_cc_test( + NAME + str_format_bind_test + SRCS + "internal/str_format/bind_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format_internal + gmock_main +) + +absl_cc_test( + NAME + str_format_checker_test + SRCS + "internal/str_format/checker_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format + gmock_main +) + +absl_cc_test( + NAME + str_format_convert_test + SRCS + "internal/str_format/convert_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::strings + absl::str_format_internal + absl::raw_logging_internal + absl::int128 + gmock_main +) + +absl_cc_test( + NAME + str_format_output_test + SRCS + "internal/str_format/output_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format_internal + absl::cord + gmock_main +) + +absl_cc_test( + NAME + str_format_parser_test + SRCS + "internal/str_format/parser_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::str_format_internal + absl::core_headers + gmock_main +) + +absl_cc_library( + NAME + pow10_helper + HDRS + "internal/pow10_helper.h" + SRCS + "internal/pow10_helper.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + TESTONLY +) + +absl_cc_test( + NAME + pow10_helper_test + SRCS + "internal/pow10_helper_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::pow10_helper + absl::str_format + gmock_main +) + +absl_cc_library( + NAME + cord + HDRS + "cord.h" + SRCS + "cord.cc" + "internal/cord_internal.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::compressed_tuple + absl::core_headers + absl::endian + absl::fixed_array + absl::function_ref + absl::inlined_vector + absl::optional + absl::raw_logging_internal + absl::strings + absl::strings_internal + absl::type_traits + PUBLIC +) + +absl_cc_library( + NAME + cord_test_helpers + HDRS + "cord_test_helpers.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::cord + TESTONLY +) + +absl_cc_test( + NAME + cord_test + SRCS + "cord_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::cord + absl::str_format + absl::strings + absl::base + absl::config + absl::core_headers + absl::endian + absl::raw_logging_internal + absl::fixed_array + gmock_main +) diff --git a/third_party/abseil_cpp/absl/strings/ascii.cc b/third_party/abseil_cpp/absl/strings/ascii.cc new file mode 100644 index 000000000000..93bb03e95815 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/ascii.cc @@ -0,0 +1,200 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/ascii.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace ascii_internal { + +// # Table generated by this Python code (bit 0x02 is currently unused): +// TODO(mbar) Move Python code for generation of table to BUILD and link here. + +// NOTE: The kAsciiPropertyBits table used within this code was generated by +// Python code of the following form. (Bit 0x02 is currently unused and +// available.) +// +// def Hex2(n): +// return '0x' + hex(n/16)[2:] + hex(n%16)[2:] +// def IsPunct(ch): +// return (ord(ch) >= 32 and ord(ch) < 127 and +// not ch.isspace() and not ch.isalnum()) +// def IsBlank(ch): +// return ch in ' \t' +// def IsCntrl(ch): +// return ord(ch) < 32 or ord(ch) == 127 +// def IsXDigit(ch): +// return ch.isdigit() or ch.lower() in 'abcdef' +// for i in range(128): +// ch = chr(i) +// mask = ((ch.isalpha() and 0x01 or 0) | +// (ch.isalnum() and 0x04 or 0) | +// (ch.isspace() and 0x08 or 0) | +// (IsPunct(ch) and 0x10 or 0) | +// (IsBlank(ch) and 0x20 or 0) | +// (IsCntrl(ch) and 0x40 or 0) | +// (IsXDigit(ch) and 0x80 or 0)) +// print Hex2(mask) + ',', +// if i % 16 == 7: +// print ' //', Hex2(i & 0x78) +// elif i % 16 == 15: +// print + +// clang-format off +// Array of bitfields holding character information. Each bit value corresponds +// to a particular character feature. For readability, and because the value +// of these bits is tightly coupled to this implementation, the individual bits +// are not named. Note that bitfields for all characters above ASCII 127 are +// zero-initialized. +ABSL_DLL const unsigned char kPropertyBits[256] = { + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x00 + 0x40, 0x68, 0x48, 0x48, 0x48, 0x48, 0x40, 0x40, + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, // 0x10 + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, + 0x28, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, // 0x20 + 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, 0x84, // 0x30 + 0x84, 0x84, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x05, // 0x40 + 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, + 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, // 0x50 + 0x05, 0x05, 0x05, 0x10, 0x10, 0x10, 0x10, 0x10, + 0x10, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x05, // 0x60 + 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, + 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, // 0x70 + 0x05, 0x05, 0x05, 0x10, 0x10, 0x10, 0x10, 0x40, +}; + +// Array of characters for the ascii_tolower() function. For values 'A' +// through 'Z', return the lower-case character; otherwise, return the +// identity of the passed character. +ABSL_DLL const char kToLower[256] = { + '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', + '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f', + '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', + '\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', + '\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27', + '\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f', + '\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37', + '\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f', + '\x40', 'a', 'b', 'c', 'd', 'e', 'f', 'g', + 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', + 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', + 'x', 'y', 'z', '\x5b', '\x5c', '\x5d', '\x5e', '\x5f', + '\x60', '\x61', '\x62', '\x63', '\x64', '\x65', '\x66', '\x67', + '\x68', '\x69', '\x6a', '\x6b', '\x6c', '\x6d', '\x6e', '\x6f', + '\x70', '\x71', '\x72', '\x73', '\x74', '\x75', '\x76', '\x77', + '\x78', '\x79', '\x7a', '\x7b', '\x7c', '\x7d', '\x7e', '\x7f', + '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', + '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', + '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', + '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', + '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', + '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', + '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', + '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', + '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', + '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', + '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', + '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', + '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', + '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', + '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', + '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', +}; + +// Array of characters for the ascii_toupper() function. For values 'a' +// through 'z', return the upper-case character; otherwise, return the +// identity of the passed character. +ABSL_DLL const char kToUpper[256] = { + '\x00', '\x01', '\x02', '\x03', '\x04', '\x05', '\x06', '\x07', + '\x08', '\x09', '\x0a', '\x0b', '\x0c', '\x0d', '\x0e', '\x0f', + '\x10', '\x11', '\x12', '\x13', '\x14', '\x15', '\x16', '\x17', + '\x18', '\x19', '\x1a', '\x1b', '\x1c', '\x1d', '\x1e', '\x1f', + '\x20', '\x21', '\x22', '\x23', '\x24', '\x25', '\x26', '\x27', + '\x28', '\x29', '\x2a', '\x2b', '\x2c', '\x2d', '\x2e', '\x2f', + '\x30', '\x31', '\x32', '\x33', '\x34', '\x35', '\x36', '\x37', + '\x38', '\x39', '\x3a', '\x3b', '\x3c', '\x3d', '\x3e', '\x3f', + '\x40', '\x41', '\x42', '\x43', '\x44', '\x45', '\x46', '\x47', + '\x48', '\x49', '\x4a', '\x4b', '\x4c', '\x4d', '\x4e', '\x4f', + '\x50', '\x51', '\x52', '\x53', '\x54', '\x55', '\x56', '\x57', + '\x58', '\x59', '\x5a', '\x5b', '\x5c', '\x5d', '\x5e', '\x5f', + '\x60', 'A', 'B', 'C', 'D', 'E', 'F', 'G', + 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', + 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', + 'X', 'Y', 'Z', '\x7b', '\x7c', '\x7d', '\x7e', '\x7f', + '\x80', '\x81', '\x82', '\x83', '\x84', '\x85', '\x86', '\x87', + '\x88', '\x89', '\x8a', '\x8b', '\x8c', '\x8d', '\x8e', '\x8f', + '\x90', '\x91', '\x92', '\x93', '\x94', '\x95', '\x96', '\x97', + '\x98', '\x99', '\x9a', '\x9b', '\x9c', '\x9d', '\x9e', '\x9f', + '\xa0', '\xa1', '\xa2', '\xa3', '\xa4', '\xa5', '\xa6', '\xa7', + '\xa8', '\xa9', '\xaa', '\xab', '\xac', '\xad', '\xae', '\xaf', + '\xb0', '\xb1', '\xb2', '\xb3', '\xb4', '\xb5', '\xb6', '\xb7', + '\xb8', '\xb9', '\xba', '\xbb', '\xbc', '\xbd', '\xbe', '\xbf', + '\xc0', '\xc1', '\xc2', '\xc3', '\xc4', '\xc5', '\xc6', '\xc7', + '\xc8', '\xc9', '\xca', '\xcb', '\xcc', '\xcd', '\xce', '\xcf', + '\xd0', '\xd1', '\xd2', '\xd3', '\xd4', '\xd5', '\xd6', '\xd7', + '\xd8', '\xd9', '\xda', '\xdb', '\xdc', '\xdd', '\xde', '\xdf', + '\xe0', '\xe1', '\xe2', '\xe3', '\xe4', '\xe5', '\xe6', '\xe7', + '\xe8', '\xe9', '\xea', '\xeb', '\xec', '\xed', '\xee', '\xef', + '\xf0', '\xf1', '\xf2', '\xf3', '\xf4', '\xf5', '\xf6', '\xf7', + '\xf8', '\xf9', '\xfa', '\xfb', '\xfc', '\xfd', '\xfe', '\xff', +}; +// clang-format on + +} // namespace ascii_internal + +void AsciiStrToLower(std::string* s) { + for (auto& ch : *s) { + ch = absl::ascii_tolower(ch); + } +} + +void AsciiStrToUpper(std::string* s) { + for (auto& ch : *s) { + ch = absl::ascii_toupper(ch); + } +} + +void RemoveExtraAsciiWhitespace(std::string* str) { + auto stripped = StripAsciiWhitespace(*str); + + if (stripped.empty()) { + str->clear(); + return; + } + + auto input_it = stripped.begin(); + auto input_end = stripped.end(); + auto output_it = &(*str)[0]; + bool is_ws = false; + + for (; input_it < input_end; ++input_it) { + if (is_ws) { + // Consecutive whitespace? Keep only the last. + is_ws = absl::ascii_isspace(*input_it); + if (is_ws) --output_it; + } else { + is_ws = absl::ascii_isspace(*input_it); + } + + *output_it = *input_it; + ++output_it; + } + + str->erase(output_it - &(*str)[0]); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/ascii.h b/third_party/abseil_cpp/absl/strings/ascii.h new file mode 100644 index 000000000000..b46bc71f35b9 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/ascii.h @@ -0,0 +1,242 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: ascii.h +// ----------------------------------------------------------------------------- +// +// This package contains functions operating on characters and strings +// restricted to standard ASCII. These include character classification +// functions analogous to those found in the ANSI C Standard Library <ctype.h> +// header file. +// +// C++ implementations provide <ctype.h> functionality based on their +// C environment locale. In general, reliance on such a locale is not ideal, as +// the locale standard is problematic (and may not return invariant information +// for the same character set, for example). These `ascii_*()` functions are +// hard-wired for standard ASCII, much faster, and guaranteed to behave +// consistently. They will never be overloaded, nor will their function +// signature change. +// +// `ascii_isalnum()`, `ascii_isalpha()`, `ascii_isascii()`, `ascii_isblank()`, +// `ascii_iscntrl()`, `ascii_isdigit()`, `ascii_isgraph()`, `ascii_islower()`, +// `ascii_isprint()`, `ascii_ispunct()`, `ascii_isspace()`, `ascii_isupper()`, +// `ascii_isxdigit()` +// Analogous to the <ctype.h> functions with similar names, these +// functions take an unsigned char and return a bool, based on whether the +// character matches the condition specified. +// +// If the input character has a numerical value greater than 127, these +// functions return `false`. +// +// `ascii_tolower()`, `ascii_toupper()` +// Analogous to the <ctype.h> functions with similar names, these functions +// take an unsigned char and return a char. +// +// If the input character is not an ASCII {lower,upper}-case letter (including +// numerical values greater than 127) then the functions return the same value +// as the input character. + +#ifndef ABSL_STRINGS_ASCII_H_ +#define ABSL_STRINGS_ASCII_H_ + +#include <algorithm> +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace ascii_internal { + +// Declaration for an array of bitfields holding character information. +ABSL_DLL extern const unsigned char kPropertyBits[256]; + +// Declaration for the array of characters to upper-case characters. +ABSL_DLL extern const char kToUpper[256]; + +// Declaration for the array of characters to lower-case characters. +ABSL_DLL extern const char kToLower[256]; + +} // namespace ascii_internal + +// ascii_isalpha() +// +// Determines whether the given character is an alphabetic character. +inline bool ascii_isalpha(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x01) != 0; +} + +// ascii_isalnum() +// +// Determines whether the given character is an alphanumeric character. +inline bool ascii_isalnum(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x04) != 0; +} + +// ascii_isspace() +// +// Determines whether the given character is a whitespace character (space, +// tab, vertical tab, formfeed, linefeed, or carriage return). +inline bool ascii_isspace(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x08) != 0; +} + +// ascii_ispunct() +// +// Determines whether the given character is a punctuation character. +inline bool ascii_ispunct(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x10) != 0; +} + +// ascii_isblank() +// +// Determines whether the given character is a blank character (tab or space). +inline bool ascii_isblank(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x20) != 0; +} + +// ascii_iscntrl() +// +// Determines whether the given character is a control character. +inline bool ascii_iscntrl(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x40) != 0; +} + +// ascii_isxdigit() +// +// Determines whether the given character can be represented as a hexadecimal +// digit character (i.e. {0-9} or {A-F}). +inline bool ascii_isxdigit(unsigned char c) { + return (ascii_internal::kPropertyBits[c] & 0x80) != 0; +} + +// ascii_isdigit() +// +// Determines whether the given character can be represented as a decimal +// digit character (i.e. {0-9}). +inline bool ascii_isdigit(unsigned char c) { return c >= '0' && c <= '9'; } + +// ascii_isprint() +// +// Determines whether the given character is printable, including whitespace. +inline bool ascii_isprint(unsigned char c) { return c >= 32 && c < 127; } + +// ascii_isgraph() +// +// Determines whether the given character has a graphical representation. +inline bool ascii_isgraph(unsigned char c) { return c > 32 && c < 127; } + +// ascii_isupper() +// +// Determines whether the given character is uppercase. +inline bool ascii_isupper(unsigned char c) { return c >= 'A' && c <= 'Z'; } + +// ascii_islower() +// +// Determines whether the given character is lowercase. +inline bool ascii_islower(unsigned char c) { return c >= 'a' && c <= 'z'; } + +// ascii_isascii() +// +// Determines whether the given character is ASCII. +inline bool ascii_isascii(unsigned char c) { return c < 128; } + +// ascii_tolower() +// +// Returns an ASCII character, converting to lowercase if uppercase is +// passed. Note that character values > 127 are simply returned. +inline char ascii_tolower(unsigned char c) { + return ascii_internal::kToLower[c]; +} + +// Converts the characters in `s` to lowercase, changing the contents of `s`. +void AsciiStrToLower(std::string* s); + +// Creates a lowercase string from a given absl::string_view. +ABSL_MUST_USE_RESULT inline std::string AsciiStrToLower(absl::string_view s) { + std::string result(s); + absl::AsciiStrToLower(&result); + return result; +} + +// ascii_toupper() +// +// Returns the ASCII character, converting to upper-case if lower-case is +// passed. Note that characters values > 127 are simply returned. +inline char ascii_toupper(unsigned char c) { + return ascii_internal::kToUpper[c]; +} + +// Converts the characters in `s` to uppercase, changing the contents of `s`. +void AsciiStrToUpper(std::string* s); + +// Creates an uppercase string from a given absl::string_view. +ABSL_MUST_USE_RESULT inline std::string AsciiStrToUpper(absl::string_view s) { + std::string result(s); + absl::AsciiStrToUpper(&result); + return result; +} + +// Returns absl::string_view with whitespace stripped from the beginning of the +// given string_view. +ABSL_MUST_USE_RESULT inline absl::string_view StripLeadingAsciiWhitespace( + absl::string_view str) { + auto it = std::find_if_not(str.begin(), str.end(), absl::ascii_isspace); + return str.substr(it - str.begin()); +} + +// Strips in place whitespace from the beginning of the given string. +inline void StripLeadingAsciiWhitespace(std::string* str) { + auto it = std::find_if_not(str->begin(), str->end(), absl::ascii_isspace); + str->erase(str->begin(), it); +} + +// Returns absl::string_view with whitespace stripped from the end of the given +// string_view. +ABSL_MUST_USE_RESULT inline absl::string_view StripTrailingAsciiWhitespace( + absl::string_view str) { + auto it = std::find_if_not(str.rbegin(), str.rend(), absl::ascii_isspace); + return str.substr(0, str.rend() - it); +} + +// Strips in place whitespace from the end of the given string +inline void StripTrailingAsciiWhitespace(std::string* str) { + auto it = std::find_if_not(str->rbegin(), str->rend(), absl::ascii_isspace); + str->erase(str->rend() - it); +} + +// Returns absl::string_view with whitespace stripped from both ends of the +// given string_view. +ABSL_MUST_USE_RESULT inline absl::string_view StripAsciiWhitespace( + absl::string_view str) { + return StripTrailingAsciiWhitespace(StripLeadingAsciiWhitespace(str)); +} + +// Strips in place whitespace from both ends of the given string +inline void StripAsciiWhitespace(std::string* str) { + StripTrailingAsciiWhitespace(str); + StripLeadingAsciiWhitespace(str); +} + +// Removes leading, trailing, and consecutive internal whitespace. +void RemoveExtraAsciiWhitespace(std::string*); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_ASCII_H_ diff --git a/third_party/abseil_cpp/absl/strings/ascii_benchmark.cc b/third_party/abseil_cpp/absl/strings/ascii_benchmark.cc new file mode 100644 index 000000000000..aca458c8042f --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/ascii_benchmark.cc @@ -0,0 +1,120 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/ascii.h" + +#include <cctype> +#include <string> +#include <array> +#include <random> + +#include "benchmark/benchmark.h" + +namespace { + +std::array<unsigned char, 256> MakeShuffledBytes() { + std::array<unsigned char, 256> bytes; + for (size_t i = 0; i < 256; ++i) bytes[i] = static_cast<unsigned char>(i); + std::random_device rd; + std::seed_seq seed({rd(), rd(), rd(), rd(), rd(), rd(), rd(), rd()}); + std::mt19937 g(seed); + std::shuffle(bytes.begin(), bytes.end(), g); + return bytes; +} + +template <typename Function> +void AsciiBenchmark(benchmark::State& state, Function f) { + std::array<unsigned char, 256> bytes = MakeShuffledBytes(); + size_t sum = 0; + for (auto _ : state) { + for (unsigned char b : bytes) sum += f(b) ? 1 : 0; + } + // Make a copy of `sum` before calling `DoNotOptimize` to make sure that `sum` + // can be put in a CPU register and not degrade performance in the loop above. + size_t sum2 = sum; + benchmark::DoNotOptimize(sum2); + state.SetBytesProcessed(state.iterations() * bytes.size()); +} + +using StdAsciiFunction = int (*)(int); +template <StdAsciiFunction f> +void BM_Ascii(benchmark::State& state) { + AsciiBenchmark(state, f); +} + +using AbslAsciiIsFunction = bool (*)(unsigned char); +template <AbslAsciiIsFunction f> +void BM_Ascii(benchmark::State& state) { + AsciiBenchmark(state, f); +} + +using AbslAsciiToFunction = char (*)(unsigned char); +template <AbslAsciiToFunction f> +void BM_Ascii(benchmark::State& state) { + AsciiBenchmark(state, f); +} + +inline char Noop(unsigned char b) { return static_cast<char>(b); } + +BENCHMARK_TEMPLATE(BM_Ascii, Noop); +BENCHMARK_TEMPLATE(BM_Ascii, std::isalpha); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isalpha); +BENCHMARK_TEMPLATE(BM_Ascii, std::isdigit); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isdigit); +BENCHMARK_TEMPLATE(BM_Ascii, std::isalnum); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isalnum); +BENCHMARK_TEMPLATE(BM_Ascii, std::isspace); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isspace); +BENCHMARK_TEMPLATE(BM_Ascii, std::ispunct); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_ispunct); +BENCHMARK_TEMPLATE(BM_Ascii, std::isblank); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isblank); +BENCHMARK_TEMPLATE(BM_Ascii, std::iscntrl); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_iscntrl); +BENCHMARK_TEMPLATE(BM_Ascii, std::isxdigit); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isxdigit); +BENCHMARK_TEMPLATE(BM_Ascii, std::isprint); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isprint); +BENCHMARK_TEMPLATE(BM_Ascii, std::isgraph); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isgraph); +BENCHMARK_TEMPLATE(BM_Ascii, std::isupper); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isupper); +BENCHMARK_TEMPLATE(BM_Ascii, std::islower); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_islower); +BENCHMARK_TEMPLATE(BM_Ascii, isascii); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_isascii); +BENCHMARK_TEMPLATE(BM_Ascii, std::tolower); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_tolower); +BENCHMARK_TEMPLATE(BM_Ascii, std::toupper); +BENCHMARK_TEMPLATE(BM_Ascii, absl::ascii_toupper); + +static void BM_StrToLower(benchmark::State& state) { + const int size = state.range(0); + std::string s(size, 'X'); + for (auto _ : state) { + benchmark::DoNotOptimize(absl::AsciiStrToLower(s)); + } +} +BENCHMARK(BM_StrToLower)->Range(1, 1 << 20); + +static void BM_StrToUpper(benchmark::State& state) { + const int size = state.range(0); + std::string s(size, 'x'); + for (auto _ : state) { + benchmark::DoNotOptimize(absl::AsciiStrToUpper(s)); + } +} +BENCHMARK(BM_StrToUpper)->Range(1, 1 << 20); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/ascii_test.cc b/third_party/abseil_cpp/absl/strings/ascii_test.cc new file mode 100644 index 000000000000..5ecd23f8697d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/ascii_test.cc @@ -0,0 +1,361 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/ascii.h" + +#include <cctype> +#include <clocale> +#include <cstring> +#include <string> + +#include "gtest/gtest.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" + +namespace { + +TEST(AsciiIsFoo, All) { + for (int i = 0; i < 256; i++) { + if ((i >= 'a' && i <= 'z') || (i >= 'A' && i <= 'Z')) + EXPECT_TRUE(absl::ascii_isalpha(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isalpha(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if ((i >= '0' && i <= '9')) + EXPECT_TRUE(absl::ascii_isdigit(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isdigit(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (absl::ascii_isalpha(i) || absl::ascii_isdigit(i)) + EXPECT_TRUE(absl::ascii_isalnum(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isalnum(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i != '\0' && strchr(" \r\n\t\v\f", i)) + EXPECT_TRUE(absl::ascii_isspace(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isspace(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i >= 32 && i < 127) + EXPECT_TRUE(absl::ascii_isprint(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isprint(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (absl::ascii_isprint(i) && !absl::ascii_isspace(i) && + !absl::ascii_isalnum(i)) + EXPECT_TRUE(absl::ascii_ispunct(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_ispunct(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i == ' ' || i == '\t') + EXPECT_TRUE(absl::ascii_isblank(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isblank(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i < 32 || i == 127) + EXPECT_TRUE(absl::ascii_iscntrl(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_iscntrl(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (absl::ascii_isdigit(i) || (i >= 'A' && i <= 'F') || + (i >= 'a' && i <= 'f')) + EXPECT_TRUE(absl::ascii_isxdigit(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isxdigit(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i > 32 && i < 127) + EXPECT_TRUE(absl::ascii_isgraph(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isgraph(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i >= 'A' && i <= 'Z') + EXPECT_TRUE(absl::ascii_isupper(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_isupper(i)) << ": failed on " << i; + } + for (int i = 0; i < 256; i++) { + if (i >= 'a' && i <= 'z') + EXPECT_TRUE(absl::ascii_islower(i)) << ": failed on " << i; + else + EXPECT_TRUE(!absl::ascii_islower(i)) << ": failed on " << i; + } + for (int i = 0; i < 128; i++) { + EXPECT_TRUE(absl::ascii_isascii(i)) << ": failed on " << i; + } + for (int i = 128; i < 256; i++) { + EXPECT_TRUE(!absl::ascii_isascii(i)) << ": failed on " << i; + } + + // The official is* functions don't accept negative signed chars, but + // our absl::ascii_is* functions do. + for (int i = 0; i < 256; i++) { + signed char sc = static_cast<signed char>(static_cast<unsigned char>(i)); + EXPECT_EQ(absl::ascii_isalpha(i), absl::ascii_isalpha(sc)) << i; + EXPECT_EQ(absl::ascii_isdigit(i), absl::ascii_isdigit(sc)) << i; + EXPECT_EQ(absl::ascii_isalnum(i), absl::ascii_isalnum(sc)) << i; + EXPECT_EQ(absl::ascii_isspace(i), absl::ascii_isspace(sc)) << i; + EXPECT_EQ(absl::ascii_ispunct(i), absl::ascii_ispunct(sc)) << i; + EXPECT_EQ(absl::ascii_isblank(i), absl::ascii_isblank(sc)) << i; + EXPECT_EQ(absl::ascii_iscntrl(i), absl::ascii_iscntrl(sc)) << i; + EXPECT_EQ(absl::ascii_isxdigit(i), absl::ascii_isxdigit(sc)) << i; + EXPECT_EQ(absl::ascii_isprint(i), absl::ascii_isprint(sc)) << i; + EXPECT_EQ(absl::ascii_isgraph(i), absl::ascii_isgraph(sc)) << i; + EXPECT_EQ(absl::ascii_isupper(i), absl::ascii_isupper(sc)) << i; + EXPECT_EQ(absl::ascii_islower(i), absl::ascii_islower(sc)) << i; + EXPECT_EQ(absl::ascii_isascii(i), absl::ascii_isascii(sc)) << i; + } +} + +// Checks that absl::ascii_isfoo returns the same value as isfoo in the C +// locale. +TEST(AsciiIsFoo, SameAsIsFoo) { +#ifndef __ANDROID__ + // temporarily change locale to C. It should already be C, but just for safety + const char* old_locale = setlocale(LC_CTYPE, "C"); + ASSERT_TRUE(old_locale != nullptr); +#endif + + for (int i = 0; i < 256; i++) { + EXPECT_EQ(isalpha(i) != 0, absl::ascii_isalpha(i)) << i; + EXPECT_EQ(isdigit(i) != 0, absl::ascii_isdigit(i)) << i; + EXPECT_EQ(isalnum(i) != 0, absl::ascii_isalnum(i)) << i; + EXPECT_EQ(isspace(i) != 0, absl::ascii_isspace(i)) << i; + EXPECT_EQ(ispunct(i) != 0, absl::ascii_ispunct(i)) << i; + EXPECT_EQ(isblank(i) != 0, absl::ascii_isblank(i)) << i; + EXPECT_EQ(iscntrl(i) != 0, absl::ascii_iscntrl(i)) << i; + EXPECT_EQ(isxdigit(i) != 0, absl::ascii_isxdigit(i)) << i; + EXPECT_EQ(isprint(i) != 0, absl::ascii_isprint(i)) << i; + EXPECT_EQ(isgraph(i) != 0, absl::ascii_isgraph(i)) << i; + EXPECT_EQ(isupper(i) != 0, absl::ascii_isupper(i)) << i; + EXPECT_EQ(islower(i) != 0, absl::ascii_islower(i)) << i; + EXPECT_EQ(isascii(i) != 0, absl::ascii_isascii(i)) << i; + } + +#ifndef __ANDROID__ + // restore the old locale. + ASSERT_TRUE(setlocale(LC_CTYPE, old_locale)); +#endif +} + +TEST(AsciiToFoo, All) { +#ifndef __ANDROID__ + // temporarily change locale to C. It should already be C, but just for safety + const char* old_locale = setlocale(LC_CTYPE, "C"); + ASSERT_TRUE(old_locale != nullptr); +#endif + + for (int i = 0; i < 256; i++) { + if (absl::ascii_islower(i)) + EXPECT_EQ(absl::ascii_toupper(i), 'A' + (i - 'a')) << i; + else + EXPECT_EQ(absl::ascii_toupper(i), static_cast<char>(i)) << i; + + if (absl::ascii_isupper(i)) + EXPECT_EQ(absl::ascii_tolower(i), 'a' + (i - 'A')) << i; + else + EXPECT_EQ(absl::ascii_tolower(i), static_cast<char>(i)) << i; + + // These CHECKs only hold in a C locale. + EXPECT_EQ(static_cast<char>(tolower(i)), absl::ascii_tolower(i)) << i; + EXPECT_EQ(static_cast<char>(toupper(i)), absl::ascii_toupper(i)) << i; + + // The official to* functions don't accept negative signed chars, but + // our absl::ascii_to* functions do. + signed char sc = static_cast<signed char>(static_cast<unsigned char>(i)); + EXPECT_EQ(absl::ascii_tolower(i), absl::ascii_tolower(sc)) << i; + EXPECT_EQ(absl::ascii_toupper(i), absl::ascii_toupper(sc)) << i; + } +#ifndef __ANDROID__ + // restore the old locale. + ASSERT_TRUE(setlocale(LC_CTYPE, old_locale)); +#endif +} + +TEST(AsciiStrTo, Lower) { + const char buf[] = "ABCDEF"; + const std::string str("GHIJKL"); + const std::string str2("MNOPQR"); + const absl::string_view sp(str2); + + EXPECT_EQ("abcdef", absl::AsciiStrToLower(buf)); + EXPECT_EQ("ghijkl", absl::AsciiStrToLower(str)); + EXPECT_EQ("mnopqr", absl::AsciiStrToLower(sp)); + + char mutable_buf[] = "Mutable"; + std::transform(mutable_buf, mutable_buf + strlen(mutable_buf), + mutable_buf, absl::ascii_tolower); + EXPECT_STREQ("mutable", mutable_buf); +} + +TEST(AsciiStrTo, Upper) { + const char buf[] = "abcdef"; + const std::string str("ghijkl"); + const std::string str2("mnopqr"); + const absl::string_view sp(str2); + + EXPECT_EQ("ABCDEF", absl::AsciiStrToUpper(buf)); + EXPECT_EQ("GHIJKL", absl::AsciiStrToUpper(str)); + EXPECT_EQ("MNOPQR", absl::AsciiStrToUpper(sp)); + + char mutable_buf[] = "Mutable"; + std::transform(mutable_buf, mutable_buf + strlen(mutable_buf), + mutable_buf, absl::ascii_toupper); + EXPECT_STREQ("MUTABLE", mutable_buf); +} + +TEST(StripLeadingAsciiWhitespace, FromStringView) { + EXPECT_EQ(absl::string_view{}, + absl::StripLeadingAsciiWhitespace(absl::string_view{})); + EXPECT_EQ("foo", absl::StripLeadingAsciiWhitespace({"foo"})); + EXPECT_EQ("foo", absl::StripLeadingAsciiWhitespace({"\t \n\f\r\n\vfoo"})); + EXPECT_EQ("foo foo\n ", + absl::StripLeadingAsciiWhitespace({"\t \n\f\r\n\vfoo foo\n "})); + EXPECT_EQ(absl::string_view{}, absl::StripLeadingAsciiWhitespace( + {"\t \n\f\r\v\n\t \n\f\r\v\n"})); +} + +TEST(StripLeadingAsciiWhitespace, InPlace) { + std::string str; + + absl::StripLeadingAsciiWhitespace(&str); + EXPECT_EQ("", str); + + str = "foo"; + absl::StripLeadingAsciiWhitespace(&str); + EXPECT_EQ("foo", str); + + str = "\t \n\f\r\n\vfoo"; + absl::StripLeadingAsciiWhitespace(&str); + EXPECT_EQ("foo", str); + + str = "\t \n\f\r\n\vfoo foo\n "; + absl::StripLeadingAsciiWhitespace(&str); + EXPECT_EQ("foo foo\n ", str); + + str = "\t \n\f\r\v\n\t \n\f\r\v\n"; + absl::StripLeadingAsciiWhitespace(&str); + EXPECT_EQ(absl::string_view{}, str); +} + +TEST(StripTrailingAsciiWhitespace, FromStringView) { + EXPECT_EQ(absl::string_view{}, + absl::StripTrailingAsciiWhitespace(absl::string_view{})); + EXPECT_EQ("foo", absl::StripTrailingAsciiWhitespace({"foo"})); + EXPECT_EQ("foo", absl::StripTrailingAsciiWhitespace({"foo\t \n\f\r\n\v"})); + EXPECT_EQ(" \nfoo foo", + absl::StripTrailingAsciiWhitespace({" \nfoo foo\t \n\f\r\n\v"})); + EXPECT_EQ(absl::string_view{}, absl::StripTrailingAsciiWhitespace( + {"\t \n\f\r\v\n\t \n\f\r\v\n"})); +} + +TEST(StripTrailingAsciiWhitespace, InPlace) { + std::string str; + + absl::StripTrailingAsciiWhitespace(&str); + EXPECT_EQ("", str); + + str = "foo"; + absl::StripTrailingAsciiWhitespace(&str); + EXPECT_EQ("foo", str); + + str = "foo\t \n\f\r\n\v"; + absl::StripTrailingAsciiWhitespace(&str); + EXPECT_EQ("foo", str); + + str = " \nfoo foo\t \n\f\r\n\v"; + absl::StripTrailingAsciiWhitespace(&str); + EXPECT_EQ(" \nfoo foo", str); + + str = "\t \n\f\r\v\n\t \n\f\r\v\n"; + absl::StripTrailingAsciiWhitespace(&str); + EXPECT_EQ(absl::string_view{}, str); +} + +TEST(StripAsciiWhitespace, FromStringView) { + EXPECT_EQ(absl::string_view{}, + absl::StripAsciiWhitespace(absl::string_view{})); + EXPECT_EQ("foo", absl::StripAsciiWhitespace({"foo"})); + EXPECT_EQ("foo", + absl::StripAsciiWhitespace({"\t \n\f\r\n\vfoo\t \n\f\r\n\v"})); + EXPECT_EQ("foo foo", absl::StripAsciiWhitespace( + {"\t \n\f\r\n\vfoo foo\t \n\f\r\n\v"})); + EXPECT_EQ(absl::string_view{}, + absl::StripAsciiWhitespace({"\t \n\f\r\v\n\t \n\f\r\v\n"})); +} + +TEST(StripAsciiWhitespace, InPlace) { + std::string str; + + absl::StripAsciiWhitespace(&str); + EXPECT_EQ("", str); + + str = "foo"; + absl::StripAsciiWhitespace(&str); + EXPECT_EQ("foo", str); + + str = "\t \n\f\r\n\vfoo\t \n\f\r\n\v"; + absl::StripAsciiWhitespace(&str); + EXPECT_EQ("foo", str); + + str = "\t \n\f\r\n\vfoo foo\t \n\f\r\n\v"; + absl::StripAsciiWhitespace(&str); + EXPECT_EQ("foo foo", str); + + str = "\t \n\f\r\v\n\t \n\f\r\v\n"; + absl::StripAsciiWhitespace(&str); + EXPECT_EQ(absl::string_view{}, str); +} + +TEST(RemoveExtraAsciiWhitespace, InPlace) { + const char* inputs[] = {"No extra space", + " Leading whitespace", + "Trailing whitespace ", + " Leading and trailing ", + " Whitespace \t in\v middle ", + "'Eeeeep! \n Newlines!\n", + "nospaces", + "", + "\n\t a\t\n\nb \t\n"}; + + const char* outputs[] = { + "No extra space", + "Leading whitespace", + "Trailing whitespace", + "Leading and trailing", + "Whitespace in middle", + "'Eeeeep! Newlines!", + "nospaces", + "", + "a\nb", + }; + const int NUM_TESTS = ABSL_ARRAYSIZE(inputs); + + for (int i = 0; i < NUM_TESTS; i++) { + std::string s(inputs[i]); + absl::RemoveExtraAsciiWhitespace(&s); + EXPECT_EQ(outputs[i], s); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/charconv.cc b/third_party/abseil_cpp/absl/strings/charconv.cc new file mode 100644 index 000000000000..3613a6528665 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/charconv.cc @@ -0,0 +1,984 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/charconv.h" + +#include <algorithm> +#include <cassert> +#include <cmath> +#include <cstring> + +#include "absl/base/casts.h" +#include "absl/base/internal/bits.h" +#include "absl/numeric/int128.h" +#include "absl/strings/internal/charconv_bigint.h" +#include "absl/strings/internal/charconv_parse.h" + +// The macro ABSL_BIT_PACK_FLOATS is defined on x86-64, where IEEE floating +// point numbers have the same endianness in memory as a bitfield struct +// containing the corresponding parts. +// +// When set, we replace calls to ldexp() with manual bit packing, which is +// faster and is unaffected by floating point environment. +#ifdef ABSL_BIT_PACK_FLOATS +#error ABSL_BIT_PACK_FLOATS cannot be directly set +#elif defined(__x86_64__) || defined(_M_X64) +#define ABSL_BIT_PACK_FLOATS 1 +#endif + +// A note about subnormals: +// +// The code below talks about "normals" and "subnormals". A normal IEEE float +// has a fixed-width mantissa and power of two exponent. For example, a normal +// `double` has a 53-bit mantissa. Because the high bit is always 1, it is not +// stored in the representation. The implicit bit buys an extra bit of +// resolution in the datatype. +// +// The downside of this scheme is that there is a large gap between DBL_MIN and +// zero. (Large, at least, relative to the different between DBL_MIN and the +// next representable number). This gap is softened by the "subnormal" numbers, +// which have the same power-of-two exponent as DBL_MIN, but no implicit 53rd +// bit. An all-bits-zero exponent in the encoding represents subnormals. (Zero +// is represented as a subnormal with an all-bits-zero mantissa.) +// +// The code below, in calculations, represents the mantissa as a uint64_t. The +// end result normally has the 53rd bit set. It represents subnormals by using +// narrower mantissas. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +template <typename FloatType> +struct FloatTraits; + +template <> +struct FloatTraits<double> { + // The number of mantissa bits in the given float type. This includes the + // implied high bit. + static constexpr int kTargetMantissaBits = 53; + + // The largest supported IEEE exponent, in our integral mantissa + // representation. + // + // If `m` is the largest possible int kTargetMantissaBits bits wide, then + // m * 2**kMaxExponent is exactly equal to DBL_MAX. + static constexpr int kMaxExponent = 971; + + // The smallest supported IEEE normal exponent, in our integral mantissa + // representation. + // + // If `m` is the smallest possible int kTargetMantissaBits bits wide, then + // m * 2**kMinNormalExponent is exactly equal to DBL_MIN. + static constexpr int kMinNormalExponent = -1074; + + static double MakeNan(const char* tagp) { + // Support nan no matter which namespace it's in. Some platforms + // incorrectly don't put it in namespace std. + using namespace std; // NOLINT + return nan(tagp); + } + + // Builds a nonzero floating point number out of the provided parts. + // + // This is intended to do the same operation as ldexp(mantissa, exponent), + // but using purely integer math, to avoid -ffastmath and floating + // point environment issues. Using type punning is also faster. We fall back + // to ldexp on a per-platform basis for portability. + // + // `exponent` must be between kMinNormalExponent and kMaxExponent. + // + // `mantissa` must either be exactly kTargetMantissaBits wide, in which case + // a normal value is made, or it must be less narrow than that, in which case + // `exponent` must be exactly kMinNormalExponent, and a subnormal value is + // made. + static double Make(uint64_t mantissa, int exponent, bool sign) { +#ifndef ABSL_BIT_PACK_FLOATS + // Support ldexp no matter which namespace it's in. Some platforms + // incorrectly don't put it in namespace std. + using namespace std; // NOLINT + return sign ? -ldexp(mantissa, exponent) : ldexp(mantissa, exponent); +#else + constexpr uint64_t kMantissaMask = + (uint64_t(1) << (kTargetMantissaBits - 1)) - 1; + uint64_t dbl = static_cast<uint64_t>(sign) << 63; + if (mantissa > kMantissaMask) { + // Normal value. + // Adjust by 1023 for the exponent representation bias, and an additional + // 52 due to the implied decimal point in the IEEE mantissa represenation. + dbl += uint64_t{exponent + 1023u + kTargetMantissaBits - 1} << 52; + mantissa &= kMantissaMask; + } else { + // subnormal value + assert(exponent == kMinNormalExponent); + } + dbl += mantissa; + return absl::bit_cast<double>(dbl); +#endif // ABSL_BIT_PACK_FLOATS + } +}; + +// Specialization of floating point traits for the `float` type. See the +// FloatTraits<double> specialization above for meaning of each of the following +// members and methods. +template <> +struct FloatTraits<float> { + static constexpr int kTargetMantissaBits = 24; + static constexpr int kMaxExponent = 104; + static constexpr int kMinNormalExponent = -149; + static float MakeNan(const char* tagp) { + // Support nanf no matter which namespace it's in. Some platforms + // incorrectly don't put it in namespace std. + using namespace std; // NOLINT + return nanf(tagp); + } + static float Make(uint32_t mantissa, int exponent, bool sign) { +#ifndef ABSL_BIT_PACK_FLOATS + // Support ldexpf no matter which namespace it's in. Some platforms + // incorrectly don't put it in namespace std. + using namespace std; // NOLINT + return sign ? -ldexpf(mantissa, exponent) : ldexpf(mantissa, exponent); +#else + constexpr uint32_t kMantissaMask = + (uint32_t(1) << (kTargetMantissaBits - 1)) - 1; + uint32_t flt = static_cast<uint32_t>(sign) << 31; + if (mantissa > kMantissaMask) { + // Normal value. + // Adjust by 127 for the exponent representation bias, and an additional + // 23 due to the implied decimal point in the IEEE mantissa represenation. + flt += uint32_t{exponent + 127u + kTargetMantissaBits - 1} << 23; + mantissa &= kMantissaMask; + } else { + // subnormal value + assert(exponent == kMinNormalExponent); + } + flt += mantissa; + return absl::bit_cast<float>(flt); +#endif // ABSL_BIT_PACK_FLOATS + } +}; + +// Decimal-to-binary conversions require coercing powers of 10 into a mantissa +// and a power of 2. The two helper functions Power10Mantissa(n) and +// Power10Exponent(n) perform this task. Together, these represent a hand- +// rolled floating point value which is equal to or just less than 10**n. +// +// The return values satisfy two range guarantees: +// +// Power10Mantissa(n) * 2**Power10Exponent(n) <= 10**n +// < (Power10Mantissa(n) + 1) * 2**Power10Exponent(n) +// +// 2**63 <= Power10Mantissa(n) < 2**64. +// +// Lookups into the power-of-10 table must first check the Power10Overflow() and +// Power10Underflow() functions, to avoid out-of-bounds table access. +// +// Indexes into these tables are biased by -kPower10TableMin, and the table has +// values in the range [kPower10TableMin, kPower10TableMax]. +extern const uint64_t kPower10MantissaTable[]; +extern const int16_t kPower10ExponentTable[]; + +// The smallest allowed value for use with the Power10Mantissa() and +// Power10Exponent() functions below. (If a smaller exponent is needed in +// calculations, the end result is guaranteed to underflow.) +constexpr int kPower10TableMin = -342; + +// The largest allowed value for use with the Power10Mantissa() and +// Power10Exponent() functions below. (If a smaller exponent is needed in +// calculations, the end result is guaranteed to overflow.) +constexpr int kPower10TableMax = 308; + +uint64_t Power10Mantissa(int n) { + return kPower10MantissaTable[n - kPower10TableMin]; +} + +int Power10Exponent(int n) { + return kPower10ExponentTable[n - kPower10TableMin]; +} + +// Returns true if n is large enough that 10**n always results in an IEEE +// overflow. +bool Power10Overflow(int n) { return n > kPower10TableMax; } + +// Returns true if n is small enough that 10**n times a ParsedFloat mantissa +// always results in an IEEE underflow. +bool Power10Underflow(int n) { return n < kPower10TableMin; } + +// Returns true if Power10Mantissa(n) * 2**Power10Exponent(n) is exactly equal +// to 10**n numerically. Put another way, this returns true if there is no +// truncation error in Power10Mantissa(n). +bool Power10Exact(int n) { return n >= 0 && n <= 27; } + +// Sentinel exponent values for representing numbers too large or too close to +// zero to represent in a double. +constexpr int kOverflow = 99999; +constexpr int kUnderflow = -99999; + +// Struct representing the calculated conversion result of a positive (nonzero) +// floating point number. +// +// The calculated number is mantissa * 2**exponent (mantissa is treated as an +// integer.) `mantissa` is chosen to be the correct width for the IEEE float +// representation being calculated. (`mantissa` will always have the same bit +// width for normal values, and narrower bit widths for subnormals.) +// +// If the result of conversion was an underflow or overflow, exponent is set +// to kUnderflow or kOverflow. +struct CalculatedFloat { + uint64_t mantissa = 0; + int exponent = 0; +}; + +// Returns the bit width of the given uint128. (Equivalently, returns 128 +// minus the number of leading zero bits.) +int BitWidth(uint128 value) { + if (Uint128High64(value) == 0) { + return 64 - base_internal::CountLeadingZeros64(Uint128Low64(value)); + } + return 128 - base_internal::CountLeadingZeros64(Uint128High64(value)); +} + +// Calculates how far to the right a mantissa needs to be shifted to create a +// properly adjusted mantissa for an IEEE floating point number. +// +// `mantissa_width` is the bit width of the mantissa to be shifted, and +// `binary_exponent` is the exponent of the number before the shift. +// +// This accounts for subnormal values, and will return a larger-than-normal +// shift if binary_exponent would otherwise be too low. +template <typename FloatType> +int NormalizedShiftSize(int mantissa_width, int binary_exponent) { + const int normal_shift = + mantissa_width - FloatTraits<FloatType>::kTargetMantissaBits; + const int minimum_shift = + FloatTraits<FloatType>::kMinNormalExponent - binary_exponent; + return std::max(normal_shift, minimum_shift); +} + +// Right shifts a uint128 so that it has the requested bit width. (The +// resulting value will have 128 - bit_width leading zeroes.) The initial +// `value` must be wider than the requested bit width. +// +// Returns the number of bits shifted. +int TruncateToBitWidth(int bit_width, uint128* value) { + const int current_bit_width = BitWidth(*value); + const int shift = current_bit_width - bit_width; + *value >>= shift; + return shift; +} + +// Checks if the given ParsedFloat represents one of the edge cases that are +// not dependent on number base: zero, infinity, or NaN. If so, sets *value +// the appropriate double, and returns true. +template <typename FloatType> +bool HandleEdgeCase(const strings_internal::ParsedFloat& input, bool negative, + FloatType* value) { + if (input.type == strings_internal::FloatType::kNan) { + // A bug in both clang and gcc would cause the compiler to optimize away the + // buffer we are building below. Declaring the buffer volatile avoids the + // issue, and has no measurable performance impact in microbenchmarks. + // + // https://bugs.llvm.org/show_bug.cgi?id=37778 + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86113 + constexpr ptrdiff_t kNanBufferSize = 128; + volatile char n_char_sequence[kNanBufferSize]; + if (input.subrange_begin == nullptr) { + n_char_sequence[0] = '\0'; + } else { + ptrdiff_t nan_size = input.subrange_end - input.subrange_begin; + nan_size = std::min(nan_size, kNanBufferSize - 1); + std::copy_n(input.subrange_begin, nan_size, n_char_sequence); + n_char_sequence[nan_size] = '\0'; + } + char* nan_argument = const_cast<char*>(n_char_sequence); + *value = negative ? -FloatTraits<FloatType>::MakeNan(nan_argument) + : FloatTraits<FloatType>::MakeNan(nan_argument); + return true; + } + if (input.type == strings_internal::FloatType::kInfinity) { + *value = negative ? -std::numeric_limits<FloatType>::infinity() + : std::numeric_limits<FloatType>::infinity(); + return true; + } + if (input.mantissa == 0) { + *value = negative ? -0.0 : 0.0; + return true; + } + return false; +} + +// Given a CalculatedFloat result of a from_chars conversion, generate the +// correct output values. +// +// CalculatedFloat can represent an underflow or overflow, in which case the +// error code in *result is set. Otherwise, the calculated floating point +// number is stored in *value. +template <typename FloatType> +void EncodeResult(const CalculatedFloat& calculated, bool negative, + absl::from_chars_result* result, FloatType* value) { + if (calculated.exponent == kOverflow) { + result->ec = std::errc::result_out_of_range; + *value = negative ? -std::numeric_limits<FloatType>::max() + : std::numeric_limits<FloatType>::max(); + return; + } else if (calculated.mantissa == 0 || calculated.exponent == kUnderflow) { + result->ec = std::errc::result_out_of_range; + *value = negative ? -0.0 : 0.0; + return; + } + *value = FloatTraits<FloatType>::Make(calculated.mantissa, + calculated.exponent, negative); +} + +// Returns the given uint128 shifted to the right by `shift` bits, and rounds +// the remaining bits using round_to_nearest logic. The value is returned as a +// uint64_t, since this is the type used by this library for storing calculated +// floating point mantissas. +// +// It is expected that the width of the input value shifted by `shift` will +// be the correct bit-width for the target mantissa, which is strictly narrower +// than a uint64_t. +// +// If `input_exact` is false, then a nonzero error epsilon is assumed. For +// rounding purposes, the true value being rounded is strictly greater than the +// input value. The error may represent a single lost carry bit. +// +// When input_exact, shifted bits of the form 1000000... represent a tie, which +// is broken by rounding to even -- the rounding direction is chosen so the low +// bit of the returned value is 0. +// +// When !input_exact, shifted bits of the form 10000000... represent a value +// strictly greater than one half (due to the error epsilon), and so ties are +// always broken by rounding up. +// +// When !input_exact, shifted bits of the form 01111111... are uncertain; +// the true value may or may not be greater than 10000000..., due to the +// possible lost carry bit. The correct rounding direction is unknown. In this +// case, the result is rounded down, and `output_exact` is set to false. +// +// Zero and negative values of `shift` are accepted, in which case the word is +// shifted left, as necessary. +uint64_t ShiftRightAndRound(uint128 value, int shift, bool input_exact, + bool* output_exact) { + if (shift <= 0) { + *output_exact = input_exact; + return static_cast<uint64_t>(value << -shift); + } + if (shift >= 128) { + // Exponent is so small that we are shifting away all significant bits. + // Answer will not be representable, even as a subnormal, so return a zero + // mantissa (which represents underflow). + *output_exact = true; + return 0; + } + + *output_exact = true; + const uint128 shift_mask = (uint128(1) << shift) - 1; + const uint128 halfway_point = uint128(1) << (shift - 1); + + const uint128 shifted_bits = value & shift_mask; + value >>= shift; + if (shifted_bits > halfway_point) { + // Shifted bits greater than 10000... require rounding up. + return static_cast<uint64_t>(value + 1); + } + if (shifted_bits == halfway_point) { + // In exact mode, shifted bits of 10000... mean we're exactly halfway + // between two numbers, and we must round to even. So only round up if + // the low bit of `value` is set. + // + // In inexact mode, the nonzero error means the actual value is greater + // than the halfway point and we must alway round up. + if ((value & 1) == 1 || !input_exact) { + ++value; + } + return static_cast<uint64_t>(value); + } + if (!input_exact && shifted_bits == halfway_point - 1) { + // Rounding direction is unclear, due to error. + *output_exact = false; + } + // Otherwise, round down. + return static_cast<uint64_t>(value); +} + +// Checks if a floating point guess needs to be rounded up, using high precision +// math. +// +// `guess_mantissa` and `guess_exponent` represent a candidate guess for the +// number represented by `parsed_decimal`. +// +// The exact number represented by `parsed_decimal` must lie between the two +// numbers: +// A = `guess_mantissa * 2**guess_exponent` +// B = `(guess_mantissa + 1) * 2**guess_exponent` +// +// This function returns false if `A` is the better guess, and true if `B` is +// the better guess, with rounding ties broken by rounding to even. +bool MustRoundUp(uint64_t guess_mantissa, int guess_exponent, + const strings_internal::ParsedFloat& parsed_decimal) { + // 768 is the number of digits needed in the worst case. We could determine a + // better limit dynamically based on the value of parsed_decimal.exponent. + // This would optimize pathological input cases only. (Sane inputs won't have + // hundreds of digits of mantissa.) + absl::strings_internal::BigUnsigned<84> exact_mantissa; + int exact_exponent = exact_mantissa.ReadFloatMantissa(parsed_decimal, 768); + + // Adjust the `guess` arguments to be halfway between A and B. + guess_mantissa = guess_mantissa * 2 + 1; + guess_exponent -= 1; + + // In our comparison: + // lhs = exact = exact_mantissa * 10**exact_exponent + // = exact_mantissa * 5**exact_exponent * 2**exact_exponent + // rhs = guess = guess_mantissa * 2**guess_exponent + // + // Because we are doing integer math, we can't directly deal with negative + // exponents. We instead move these to the other side of the inequality. + absl::strings_internal::BigUnsigned<84>& lhs = exact_mantissa; + int comparison; + if (exact_exponent >= 0) { + lhs.MultiplyByFiveToTheNth(exact_exponent); + absl::strings_internal::BigUnsigned<84> rhs(guess_mantissa); + // There are powers of 2 on both sides of the inequality; reduce this to + // a single bit-shift. + if (exact_exponent > guess_exponent) { + lhs.ShiftLeft(exact_exponent - guess_exponent); + } else { + rhs.ShiftLeft(guess_exponent - exact_exponent); + } + comparison = Compare(lhs, rhs); + } else { + // Move the power of 5 to the other side of the equation, giving us: + // lhs = exact_mantissa * 2**exact_exponent + // rhs = guess_mantissa * 5**(-exact_exponent) * 2**guess_exponent + absl::strings_internal::BigUnsigned<84> rhs = + absl::strings_internal::BigUnsigned<84>::FiveToTheNth(-exact_exponent); + rhs.MultiplyBy(guess_mantissa); + if (exact_exponent > guess_exponent) { + lhs.ShiftLeft(exact_exponent - guess_exponent); + } else { + rhs.ShiftLeft(guess_exponent - exact_exponent); + } + comparison = Compare(lhs, rhs); + } + if (comparison < 0) { + return false; + } else if (comparison > 0) { + return true; + } else { + // When lhs == rhs, the decimal input is exactly between A and B. + // Round towards even -- round up only if the low bit of the initial + // `guess_mantissa` was a 1. We shifted guess_mantissa left 1 bit at + // the beginning of this function, so test the 2nd bit here. + return (guess_mantissa & 2) == 2; + } +} + +// Constructs a CalculatedFloat from a given mantissa and exponent, but +// with the following normalizations applied: +// +// If rounding has caused mantissa to increase just past the allowed bit +// width, shift and adjust exponent. +// +// If exponent is too high, sets kOverflow. +// +// If mantissa is zero (representing a non-zero value not representable, even +// as a subnormal), sets kUnderflow. +template <typename FloatType> +CalculatedFloat CalculatedFloatFromRawValues(uint64_t mantissa, int exponent) { + CalculatedFloat result; + if (mantissa == uint64_t(1) << FloatTraits<FloatType>::kTargetMantissaBits) { + mantissa >>= 1; + exponent += 1; + } + if (exponent > FloatTraits<FloatType>::kMaxExponent) { + result.exponent = kOverflow; + } else if (mantissa == 0) { + result.exponent = kUnderflow; + } else { + result.exponent = exponent; + result.mantissa = mantissa; + } + return result; +} + +template <typename FloatType> +CalculatedFloat CalculateFromParsedHexadecimal( + const strings_internal::ParsedFloat& parsed_hex) { + uint64_t mantissa = parsed_hex.mantissa; + int exponent = parsed_hex.exponent; + int mantissa_width = 64 - base_internal::CountLeadingZeros64(mantissa); + const int shift = NormalizedShiftSize<FloatType>(mantissa_width, exponent); + bool result_exact; + exponent += shift; + mantissa = ShiftRightAndRound(mantissa, shift, + /* input exact= */ true, &result_exact); + // ParseFloat handles rounding in the hexadecimal case, so we don't have to + // check `result_exact` here. + return CalculatedFloatFromRawValues<FloatType>(mantissa, exponent); +} + +template <typename FloatType> +CalculatedFloat CalculateFromParsedDecimal( + const strings_internal::ParsedFloat& parsed_decimal) { + CalculatedFloat result; + + // Large or small enough decimal exponents will always result in overflow + // or underflow. + if (Power10Underflow(parsed_decimal.exponent)) { + result.exponent = kUnderflow; + return result; + } else if (Power10Overflow(parsed_decimal.exponent)) { + result.exponent = kOverflow; + return result; + } + + // Otherwise convert our power of 10 into a power of 2 times an integer + // mantissa, and multiply this by our parsed decimal mantissa. + uint128 wide_binary_mantissa = parsed_decimal.mantissa; + wide_binary_mantissa *= Power10Mantissa(parsed_decimal.exponent); + int binary_exponent = Power10Exponent(parsed_decimal.exponent); + + // Discard bits that are inaccurate due to truncation error. The magic + // `mantissa_width` constants below are justified in + // https://abseil.io/about/design/charconv. They represent the number of bits + // in `wide_binary_mantissa` that are guaranteed to be unaffected by error + // propagation. + bool mantissa_exact; + int mantissa_width; + if (parsed_decimal.subrange_begin) { + // Truncated mantissa + mantissa_width = 58; + mantissa_exact = false; + binary_exponent += + TruncateToBitWidth(mantissa_width, &wide_binary_mantissa); + } else if (!Power10Exact(parsed_decimal.exponent)) { + // Exact mantissa, truncated power of ten + mantissa_width = 63; + mantissa_exact = false; + binary_exponent += + TruncateToBitWidth(mantissa_width, &wide_binary_mantissa); + } else { + // Product is exact + mantissa_width = BitWidth(wide_binary_mantissa); + mantissa_exact = true; + } + + // Shift into an FloatType-sized mantissa, and round to nearest. + const int shift = + NormalizedShiftSize<FloatType>(mantissa_width, binary_exponent); + bool result_exact; + binary_exponent += shift; + uint64_t binary_mantissa = ShiftRightAndRound(wide_binary_mantissa, shift, + mantissa_exact, &result_exact); + if (!result_exact) { + // We could not determine the rounding direction using int128 math. Use + // full resolution math instead. + if (MustRoundUp(binary_mantissa, binary_exponent, parsed_decimal)) { + binary_mantissa += 1; + } + } + + return CalculatedFloatFromRawValues<FloatType>(binary_mantissa, + binary_exponent); +} + +template <typename FloatType> +from_chars_result FromCharsImpl(const char* first, const char* last, + FloatType& value, chars_format fmt_flags) { + from_chars_result result; + result.ptr = first; // overwritten on successful parse + result.ec = std::errc(); + + bool negative = false; + if (first != last && *first == '-') { + ++first; + negative = true; + } + // If the `hex` flag is *not* set, then we will accept a 0x prefix and try + // to parse a hexadecimal float. + if ((fmt_flags & chars_format::hex) == chars_format{} && last - first >= 2 && + *first == '0' && (first[1] == 'x' || first[1] == 'X')) { + const char* hex_first = first + 2; + strings_internal::ParsedFloat hex_parse = + strings_internal::ParseFloat<16>(hex_first, last, fmt_flags); + if (hex_parse.end == nullptr || + hex_parse.type != strings_internal::FloatType::kNumber) { + // Either we failed to parse a hex float after the "0x", or we read + // "0xinf" or "0xnan" which we don't want to match. + // + // However, a string that begins with "0x" also begins with "0", which + // is normally a valid match for the number zero. So we want these + // strings to match zero unless fmt_flags is `scientific`. (This flag + // means an exponent is required, which the string "0" does not have.) + if (fmt_flags == chars_format::scientific) { + result.ec = std::errc::invalid_argument; + } else { + result.ptr = first + 1; + value = negative ? -0.0 : 0.0; + } + return result; + } + // We matched a value. + result.ptr = hex_parse.end; + if (HandleEdgeCase(hex_parse, negative, &value)) { + return result; + } + CalculatedFloat calculated = + CalculateFromParsedHexadecimal<FloatType>(hex_parse); + EncodeResult(calculated, negative, &result, &value); + return result; + } + // Otherwise, we choose the number base based on the flags. + if ((fmt_flags & chars_format::hex) == chars_format::hex) { + strings_internal::ParsedFloat hex_parse = + strings_internal::ParseFloat<16>(first, last, fmt_flags); + if (hex_parse.end == nullptr) { + result.ec = std::errc::invalid_argument; + return result; + } + result.ptr = hex_parse.end; + if (HandleEdgeCase(hex_parse, negative, &value)) { + return result; + } + CalculatedFloat calculated = + CalculateFromParsedHexadecimal<FloatType>(hex_parse); + EncodeResult(calculated, negative, &result, &value); + return result; + } else { + strings_internal::ParsedFloat decimal_parse = + strings_internal::ParseFloat<10>(first, last, fmt_flags); + if (decimal_parse.end == nullptr) { + result.ec = std::errc::invalid_argument; + return result; + } + result.ptr = decimal_parse.end; + if (HandleEdgeCase(decimal_parse, negative, &value)) { + return result; + } + CalculatedFloat calculated = + CalculateFromParsedDecimal<FloatType>(decimal_parse); + EncodeResult(calculated, negative, &result, &value); + return result; + } +} +} // namespace + +from_chars_result from_chars(const char* first, const char* last, double& value, + chars_format fmt) { + return FromCharsImpl(first, last, value, fmt); +} + +from_chars_result from_chars(const char* first, const char* last, float& value, + chars_format fmt) { + return FromCharsImpl(first, last, value, fmt); +} + +namespace { + +// Table of powers of 10, from kPower10TableMin to kPower10TableMax. +// +// kPower10MantissaTable[i - kPower10TableMin] stores the 64-bit mantissa (high +// bit always on), and kPower10ExponentTable[i - kPower10TableMin] stores the +// power-of-two exponent. For a given number i, this gives the unique mantissa +// and exponent such that mantissa * 2**exponent <= 10**i < (mantissa + 1) * +// 2**exponent. + +const uint64_t kPower10MantissaTable[] = { + 0xeef453d6923bd65aU, 0x9558b4661b6565f8U, 0xbaaee17fa23ebf76U, + 0xe95a99df8ace6f53U, 0x91d8a02bb6c10594U, 0xb64ec836a47146f9U, + 0xe3e27a444d8d98b7U, 0x8e6d8c6ab0787f72U, 0xb208ef855c969f4fU, + 0xde8b2b66b3bc4723U, 0x8b16fb203055ac76U, 0xaddcb9e83c6b1793U, + 0xd953e8624b85dd78U, 0x87d4713d6f33aa6bU, 0xa9c98d8ccb009506U, + 0xd43bf0effdc0ba48U, 0x84a57695fe98746dU, 0xa5ced43b7e3e9188U, + 0xcf42894a5dce35eaU, 0x818995ce7aa0e1b2U, 0xa1ebfb4219491a1fU, + 0xca66fa129f9b60a6U, 0xfd00b897478238d0U, 0x9e20735e8cb16382U, + 0xc5a890362fddbc62U, 0xf712b443bbd52b7bU, 0x9a6bb0aa55653b2dU, + 0xc1069cd4eabe89f8U, 0xf148440a256e2c76U, 0x96cd2a865764dbcaU, + 0xbc807527ed3e12bcU, 0xeba09271e88d976bU, 0x93445b8731587ea3U, + 0xb8157268fdae9e4cU, 0xe61acf033d1a45dfU, 0x8fd0c16206306babU, + 0xb3c4f1ba87bc8696U, 0xe0b62e2929aba83cU, 0x8c71dcd9ba0b4925U, + 0xaf8e5410288e1b6fU, 0xdb71e91432b1a24aU, 0x892731ac9faf056eU, + 0xab70fe17c79ac6caU, 0xd64d3d9db981787dU, 0x85f0468293f0eb4eU, + 0xa76c582338ed2621U, 0xd1476e2c07286faaU, 0x82cca4db847945caU, + 0xa37fce126597973cU, 0xcc5fc196fefd7d0cU, 0xff77b1fcbebcdc4fU, + 0x9faacf3df73609b1U, 0xc795830d75038c1dU, 0xf97ae3d0d2446f25U, + 0x9becce62836ac577U, 0xc2e801fb244576d5U, 0xf3a20279ed56d48aU, + 0x9845418c345644d6U, 0xbe5691ef416bd60cU, 0xedec366b11c6cb8fU, + 0x94b3a202eb1c3f39U, 0xb9e08a83a5e34f07U, 0xe858ad248f5c22c9U, + 0x91376c36d99995beU, 0xb58547448ffffb2dU, 0xe2e69915b3fff9f9U, + 0x8dd01fad907ffc3bU, 0xb1442798f49ffb4aU, 0xdd95317f31c7fa1dU, + 0x8a7d3eef7f1cfc52U, 0xad1c8eab5ee43b66U, 0xd863b256369d4a40U, + 0x873e4f75e2224e68U, 0xa90de3535aaae202U, 0xd3515c2831559a83U, + 0x8412d9991ed58091U, 0xa5178fff668ae0b6U, 0xce5d73ff402d98e3U, + 0x80fa687f881c7f8eU, 0xa139029f6a239f72U, 0xc987434744ac874eU, + 0xfbe9141915d7a922U, 0x9d71ac8fada6c9b5U, 0xc4ce17b399107c22U, + 0xf6019da07f549b2bU, 0x99c102844f94e0fbU, 0xc0314325637a1939U, + 0xf03d93eebc589f88U, 0x96267c7535b763b5U, 0xbbb01b9283253ca2U, + 0xea9c227723ee8bcbU, 0x92a1958a7675175fU, 0xb749faed14125d36U, + 0xe51c79a85916f484U, 0x8f31cc0937ae58d2U, 0xb2fe3f0b8599ef07U, + 0xdfbdcece67006ac9U, 0x8bd6a141006042bdU, 0xaecc49914078536dU, + 0xda7f5bf590966848U, 0x888f99797a5e012dU, 0xaab37fd7d8f58178U, + 0xd5605fcdcf32e1d6U, 0x855c3be0a17fcd26U, 0xa6b34ad8c9dfc06fU, + 0xd0601d8efc57b08bU, 0x823c12795db6ce57U, 0xa2cb1717b52481edU, + 0xcb7ddcdda26da268U, 0xfe5d54150b090b02U, 0x9efa548d26e5a6e1U, + 0xc6b8e9b0709f109aU, 0xf867241c8cc6d4c0U, 0x9b407691d7fc44f8U, + 0xc21094364dfb5636U, 0xf294b943e17a2bc4U, 0x979cf3ca6cec5b5aU, + 0xbd8430bd08277231U, 0xece53cec4a314ebdU, 0x940f4613ae5ed136U, + 0xb913179899f68584U, 0xe757dd7ec07426e5U, 0x9096ea6f3848984fU, + 0xb4bca50b065abe63U, 0xe1ebce4dc7f16dfbU, 0x8d3360f09cf6e4bdU, + 0xb080392cc4349decU, 0xdca04777f541c567U, 0x89e42caaf9491b60U, + 0xac5d37d5b79b6239U, 0xd77485cb25823ac7U, 0x86a8d39ef77164bcU, + 0xa8530886b54dbdebU, 0xd267caa862a12d66U, 0x8380dea93da4bc60U, + 0xa46116538d0deb78U, 0xcd795be870516656U, 0x806bd9714632dff6U, + 0xa086cfcd97bf97f3U, 0xc8a883c0fdaf7df0U, 0xfad2a4b13d1b5d6cU, + 0x9cc3a6eec6311a63U, 0xc3f490aa77bd60fcU, 0xf4f1b4d515acb93bU, + 0x991711052d8bf3c5U, 0xbf5cd54678eef0b6U, 0xef340a98172aace4U, + 0x9580869f0e7aac0eU, 0xbae0a846d2195712U, 0xe998d258869facd7U, + 0x91ff83775423cc06U, 0xb67f6455292cbf08U, 0xe41f3d6a7377eecaU, + 0x8e938662882af53eU, 0xb23867fb2a35b28dU, 0xdec681f9f4c31f31U, + 0x8b3c113c38f9f37eU, 0xae0b158b4738705eU, 0xd98ddaee19068c76U, + 0x87f8a8d4cfa417c9U, 0xa9f6d30a038d1dbcU, 0xd47487cc8470652bU, + 0x84c8d4dfd2c63f3bU, 0xa5fb0a17c777cf09U, 0xcf79cc9db955c2ccU, + 0x81ac1fe293d599bfU, 0xa21727db38cb002fU, 0xca9cf1d206fdc03bU, + 0xfd442e4688bd304aU, 0x9e4a9cec15763e2eU, 0xc5dd44271ad3cdbaU, + 0xf7549530e188c128U, 0x9a94dd3e8cf578b9U, 0xc13a148e3032d6e7U, + 0xf18899b1bc3f8ca1U, 0x96f5600f15a7b7e5U, 0xbcb2b812db11a5deU, + 0xebdf661791d60f56U, 0x936b9fcebb25c995U, 0xb84687c269ef3bfbU, + 0xe65829b3046b0afaU, 0x8ff71a0fe2c2e6dcU, 0xb3f4e093db73a093U, + 0xe0f218b8d25088b8U, 0x8c974f7383725573U, 0xafbd2350644eeacfU, + 0xdbac6c247d62a583U, 0x894bc396ce5da772U, 0xab9eb47c81f5114fU, + 0xd686619ba27255a2U, 0x8613fd0145877585U, 0xa798fc4196e952e7U, + 0xd17f3b51fca3a7a0U, 0x82ef85133de648c4U, 0xa3ab66580d5fdaf5U, + 0xcc963fee10b7d1b3U, 0xffbbcfe994e5c61fU, 0x9fd561f1fd0f9bd3U, + 0xc7caba6e7c5382c8U, 0xf9bd690a1b68637bU, 0x9c1661a651213e2dU, + 0xc31bfa0fe5698db8U, 0xf3e2f893dec3f126U, 0x986ddb5c6b3a76b7U, + 0xbe89523386091465U, 0xee2ba6c0678b597fU, 0x94db483840b717efU, + 0xba121a4650e4ddebU, 0xe896a0d7e51e1566U, 0x915e2486ef32cd60U, + 0xb5b5ada8aaff80b8U, 0xe3231912d5bf60e6U, 0x8df5efabc5979c8fU, + 0xb1736b96b6fd83b3U, 0xddd0467c64bce4a0U, 0x8aa22c0dbef60ee4U, + 0xad4ab7112eb3929dU, 0xd89d64d57a607744U, 0x87625f056c7c4a8bU, + 0xa93af6c6c79b5d2dU, 0xd389b47879823479U, 0x843610cb4bf160cbU, + 0xa54394fe1eedb8feU, 0xce947a3da6a9273eU, 0x811ccc668829b887U, + 0xa163ff802a3426a8U, 0xc9bcff6034c13052U, 0xfc2c3f3841f17c67U, + 0x9d9ba7832936edc0U, 0xc5029163f384a931U, 0xf64335bcf065d37dU, + 0x99ea0196163fa42eU, 0xc06481fb9bcf8d39U, 0xf07da27a82c37088U, + 0x964e858c91ba2655U, 0xbbe226efb628afeaU, 0xeadab0aba3b2dbe5U, + 0x92c8ae6b464fc96fU, 0xb77ada0617e3bbcbU, 0xe55990879ddcaabdU, + 0x8f57fa54c2a9eab6U, 0xb32df8e9f3546564U, 0xdff9772470297ebdU, + 0x8bfbea76c619ef36U, 0xaefae51477a06b03U, 0xdab99e59958885c4U, + 0x88b402f7fd75539bU, 0xaae103b5fcd2a881U, 0xd59944a37c0752a2U, + 0x857fcae62d8493a5U, 0xa6dfbd9fb8e5b88eU, 0xd097ad07a71f26b2U, + 0x825ecc24c873782fU, 0xa2f67f2dfa90563bU, 0xcbb41ef979346bcaU, + 0xfea126b7d78186bcU, 0x9f24b832e6b0f436U, 0xc6ede63fa05d3143U, + 0xf8a95fcf88747d94U, 0x9b69dbe1b548ce7cU, 0xc24452da229b021bU, + 0xf2d56790ab41c2a2U, 0x97c560ba6b0919a5U, 0xbdb6b8e905cb600fU, + 0xed246723473e3813U, 0x9436c0760c86e30bU, 0xb94470938fa89bceU, + 0xe7958cb87392c2c2U, 0x90bd77f3483bb9b9U, 0xb4ecd5f01a4aa828U, + 0xe2280b6c20dd5232U, 0x8d590723948a535fU, 0xb0af48ec79ace837U, + 0xdcdb1b2798182244U, 0x8a08f0f8bf0f156bU, 0xac8b2d36eed2dac5U, + 0xd7adf884aa879177U, 0x86ccbb52ea94baeaU, 0xa87fea27a539e9a5U, + 0xd29fe4b18e88640eU, 0x83a3eeeef9153e89U, 0xa48ceaaab75a8e2bU, + 0xcdb02555653131b6U, 0x808e17555f3ebf11U, 0xa0b19d2ab70e6ed6U, + 0xc8de047564d20a8bU, 0xfb158592be068d2eU, 0x9ced737bb6c4183dU, + 0xc428d05aa4751e4cU, 0xf53304714d9265dfU, 0x993fe2c6d07b7fabU, + 0xbf8fdb78849a5f96U, 0xef73d256a5c0f77cU, 0x95a8637627989aadU, + 0xbb127c53b17ec159U, 0xe9d71b689dde71afU, 0x9226712162ab070dU, + 0xb6b00d69bb55c8d1U, 0xe45c10c42a2b3b05U, 0x8eb98a7a9a5b04e3U, + 0xb267ed1940f1c61cU, 0xdf01e85f912e37a3U, 0x8b61313bbabce2c6U, + 0xae397d8aa96c1b77U, 0xd9c7dced53c72255U, 0x881cea14545c7575U, + 0xaa242499697392d2U, 0xd4ad2dbfc3d07787U, 0x84ec3c97da624ab4U, + 0xa6274bbdd0fadd61U, 0xcfb11ead453994baU, 0x81ceb32c4b43fcf4U, + 0xa2425ff75e14fc31U, 0xcad2f7f5359a3b3eU, 0xfd87b5f28300ca0dU, + 0x9e74d1b791e07e48U, 0xc612062576589ddaU, 0xf79687aed3eec551U, + 0x9abe14cd44753b52U, 0xc16d9a0095928a27U, 0xf1c90080baf72cb1U, + 0x971da05074da7beeU, 0xbce5086492111aeaU, 0xec1e4a7db69561a5U, + 0x9392ee8e921d5d07U, 0xb877aa3236a4b449U, 0xe69594bec44de15bU, + 0x901d7cf73ab0acd9U, 0xb424dc35095cd80fU, 0xe12e13424bb40e13U, + 0x8cbccc096f5088cbU, 0xafebff0bcb24aafeU, 0xdbe6fecebdedd5beU, + 0x89705f4136b4a597U, 0xabcc77118461cefcU, 0xd6bf94d5e57a42bcU, + 0x8637bd05af6c69b5U, 0xa7c5ac471b478423U, 0xd1b71758e219652bU, + 0x83126e978d4fdf3bU, 0xa3d70a3d70a3d70aU, 0xccccccccccccccccU, + 0x8000000000000000U, 0xa000000000000000U, 0xc800000000000000U, + 0xfa00000000000000U, 0x9c40000000000000U, 0xc350000000000000U, + 0xf424000000000000U, 0x9896800000000000U, 0xbebc200000000000U, + 0xee6b280000000000U, 0x9502f90000000000U, 0xba43b74000000000U, + 0xe8d4a51000000000U, 0x9184e72a00000000U, 0xb5e620f480000000U, + 0xe35fa931a0000000U, 0x8e1bc9bf04000000U, 0xb1a2bc2ec5000000U, + 0xde0b6b3a76400000U, 0x8ac7230489e80000U, 0xad78ebc5ac620000U, + 0xd8d726b7177a8000U, 0x878678326eac9000U, 0xa968163f0a57b400U, + 0xd3c21bcecceda100U, 0x84595161401484a0U, 0xa56fa5b99019a5c8U, + 0xcecb8f27f4200f3aU, 0x813f3978f8940984U, 0xa18f07d736b90be5U, + 0xc9f2c9cd04674edeU, 0xfc6f7c4045812296U, 0x9dc5ada82b70b59dU, + 0xc5371912364ce305U, 0xf684df56c3e01bc6U, 0x9a130b963a6c115cU, + 0xc097ce7bc90715b3U, 0xf0bdc21abb48db20U, 0x96769950b50d88f4U, + 0xbc143fa4e250eb31U, 0xeb194f8e1ae525fdU, 0x92efd1b8d0cf37beU, + 0xb7abc627050305adU, 0xe596b7b0c643c719U, 0x8f7e32ce7bea5c6fU, + 0xb35dbf821ae4f38bU, 0xe0352f62a19e306eU, 0x8c213d9da502de45U, + 0xaf298d050e4395d6U, 0xdaf3f04651d47b4cU, 0x88d8762bf324cd0fU, + 0xab0e93b6efee0053U, 0xd5d238a4abe98068U, 0x85a36366eb71f041U, + 0xa70c3c40a64e6c51U, 0xd0cf4b50cfe20765U, 0x82818f1281ed449fU, + 0xa321f2d7226895c7U, 0xcbea6f8ceb02bb39U, 0xfee50b7025c36a08U, + 0x9f4f2726179a2245U, 0xc722f0ef9d80aad6U, 0xf8ebad2b84e0d58bU, + 0x9b934c3b330c8577U, 0xc2781f49ffcfa6d5U, 0xf316271c7fc3908aU, + 0x97edd871cfda3a56U, 0xbde94e8e43d0c8ecU, 0xed63a231d4c4fb27U, + 0x945e455f24fb1cf8U, 0xb975d6b6ee39e436U, 0xe7d34c64a9c85d44U, + 0x90e40fbeea1d3a4aU, 0xb51d13aea4a488ddU, 0xe264589a4dcdab14U, + 0x8d7eb76070a08aecU, 0xb0de65388cc8ada8U, 0xdd15fe86affad912U, + 0x8a2dbf142dfcc7abU, 0xacb92ed9397bf996U, 0xd7e77a8f87daf7fbU, + 0x86f0ac99b4e8dafdU, 0xa8acd7c0222311bcU, 0xd2d80db02aabd62bU, + 0x83c7088e1aab65dbU, 0xa4b8cab1a1563f52U, 0xcde6fd5e09abcf26U, + 0x80b05e5ac60b6178U, 0xa0dc75f1778e39d6U, 0xc913936dd571c84cU, + 0xfb5878494ace3a5fU, 0x9d174b2dcec0e47bU, 0xc45d1df942711d9aU, + 0xf5746577930d6500U, 0x9968bf6abbe85f20U, 0xbfc2ef456ae276e8U, + 0xefb3ab16c59b14a2U, 0x95d04aee3b80ece5U, 0xbb445da9ca61281fU, + 0xea1575143cf97226U, 0x924d692ca61be758U, 0xb6e0c377cfa2e12eU, + 0xe498f455c38b997aU, 0x8edf98b59a373fecU, 0xb2977ee300c50fe7U, + 0xdf3d5e9bc0f653e1U, 0x8b865b215899f46cU, 0xae67f1e9aec07187U, + 0xda01ee641a708de9U, 0x884134fe908658b2U, 0xaa51823e34a7eedeU, + 0xd4e5e2cdc1d1ea96U, 0x850fadc09923329eU, 0xa6539930bf6bff45U, + 0xcfe87f7cef46ff16U, 0x81f14fae158c5f6eU, 0xa26da3999aef7749U, + 0xcb090c8001ab551cU, 0xfdcb4fa002162a63U, 0x9e9f11c4014dda7eU, + 0xc646d63501a1511dU, 0xf7d88bc24209a565U, 0x9ae757596946075fU, + 0xc1a12d2fc3978937U, 0xf209787bb47d6b84U, 0x9745eb4d50ce6332U, + 0xbd176620a501fbffU, 0xec5d3fa8ce427affU, 0x93ba47c980e98cdfU, + 0xb8a8d9bbe123f017U, 0xe6d3102ad96cec1dU, 0x9043ea1ac7e41392U, + 0xb454e4a179dd1877U, 0xe16a1dc9d8545e94U, 0x8ce2529e2734bb1dU, + 0xb01ae745b101e9e4U, 0xdc21a1171d42645dU, 0x899504ae72497ebaU, + 0xabfa45da0edbde69U, 0xd6f8d7509292d603U, 0x865b86925b9bc5c2U, + 0xa7f26836f282b732U, 0xd1ef0244af2364ffU, 0x8335616aed761f1fU, + 0xa402b9c5a8d3a6e7U, 0xcd036837130890a1U, 0x802221226be55a64U, + 0xa02aa96b06deb0fdU, 0xc83553c5c8965d3dU, 0xfa42a8b73abbf48cU, + 0x9c69a97284b578d7U, 0xc38413cf25e2d70dU, 0xf46518c2ef5b8cd1U, + 0x98bf2f79d5993802U, 0xbeeefb584aff8603U, 0xeeaaba2e5dbf6784U, + 0x952ab45cfa97a0b2U, 0xba756174393d88dfU, 0xe912b9d1478ceb17U, + 0x91abb422ccb812eeU, 0xb616a12b7fe617aaU, 0xe39c49765fdf9d94U, + 0x8e41ade9fbebc27dU, 0xb1d219647ae6b31cU, 0xde469fbd99a05fe3U, + 0x8aec23d680043beeU, 0xada72ccc20054ae9U, 0xd910f7ff28069da4U, + 0x87aa9aff79042286U, 0xa99541bf57452b28U, 0xd3fa922f2d1675f2U, + 0x847c9b5d7c2e09b7U, 0xa59bc234db398c25U, 0xcf02b2c21207ef2eU, + 0x8161afb94b44f57dU, 0xa1ba1ba79e1632dcU, 0xca28a291859bbf93U, + 0xfcb2cb35e702af78U, 0x9defbf01b061adabU, 0xc56baec21c7a1916U, + 0xf6c69a72a3989f5bU, 0x9a3c2087a63f6399U, 0xc0cb28a98fcf3c7fU, + 0xf0fdf2d3f3c30b9fU, 0x969eb7c47859e743U, 0xbc4665b596706114U, + 0xeb57ff22fc0c7959U, 0x9316ff75dd87cbd8U, 0xb7dcbf5354e9beceU, + 0xe5d3ef282a242e81U, 0x8fa475791a569d10U, 0xb38d92d760ec4455U, + 0xe070f78d3927556aU, 0x8c469ab843b89562U, 0xaf58416654a6babbU, + 0xdb2e51bfe9d0696aU, 0x88fcf317f22241e2U, 0xab3c2fddeeaad25aU, + 0xd60b3bd56a5586f1U, 0x85c7056562757456U, 0xa738c6bebb12d16cU, + 0xd106f86e69d785c7U, 0x82a45b450226b39cU, 0xa34d721642b06084U, + 0xcc20ce9bd35c78a5U, 0xff290242c83396ceU, 0x9f79a169bd203e41U, + 0xc75809c42c684dd1U, 0xf92e0c3537826145U, 0x9bbcc7a142b17ccbU, + 0xc2abf989935ddbfeU, 0xf356f7ebf83552feU, 0x98165af37b2153deU, + 0xbe1bf1b059e9a8d6U, 0xeda2ee1c7064130cU, 0x9485d4d1c63e8be7U, + 0xb9a74a0637ce2ee1U, 0xe8111c87c5c1ba99U, 0x910ab1d4db9914a0U, + 0xb54d5e4a127f59c8U, 0xe2a0b5dc971f303aU, 0x8da471a9de737e24U, + 0xb10d8e1456105dadU, 0xdd50f1996b947518U, 0x8a5296ffe33cc92fU, + 0xace73cbfdc0bfb7bU, 0xd8210befd30efa5aU, 0x8714a775e3e95c78U, + 0xa8d9d1535ce3b396U, 0xd31045a8341ca07cU, 0x83ea2b892091e44dU, + 0xa4e4b66b68b65d60U, 0xce1de40642e3f4b9U, 0x80d2ae83e9ce78f3U, + 0xa1075a24e4421730U, 0xc94930ae1d529cfcU, 0xfb9b7cd9a4a7443cU, + 0x9d412e0806e88aa5U, 0xc491798a08a2ad4eU, 0xf5b5d7ec8acb58a2U, + 0x9991a6f3d6bf1765U, 0xbff610b0cc6edd3fU, 0xeff394dcff8a948eU, + 0x95f83d0a1fb69cd9U, 0xbb764c4ca7a4440fU, 0xea53df5fd18d5513U, + 0x92746b9be2f8552cU, 0xb7118682dbb66a77U, 0xe4d5e82392a40515U, + 0x8f05b1163ba6832dU, 0xb2c71d5bca9023f8U, 0xdf78e4b2bd342cf6U, + 0x8bab8eefb6409c1aU, 0xae9672aba3d0c320U, 0xda3c0f568cc4f3e8U, + 0x8865899617fb1871U, 0xaa7eebfb9df9de8dU, 0xd51ea6fa85785631U, + 0x8533285c936b35deU, 0xa67ff273b8460356U, 0xd01fef10a657842cU, + 0x8213f56a67f6b29bU, 0xa298f2c501f45f42U, 0xcb3f2f7642717713U, + 0xfe0efb53d30dd4d7U, 0x9ec95d1463e8a506U, 0xc67bb4597ce2ce48U, + 0xf81aa16fdc1b81daU, 0x9b10a4e5e9913128U, 0xc1d4ce1f63f57d72U, + 0xf24a01a73cf2dccfU, 0x976e41088617ca01U, 0xbd49d14aa79dbc82U, + 0xec9c459d51852ba2U, 0x93e1ab8252f33b45U, 0xb8da1662e7b00a17U, + 0xe7109bfba19c0c9dU, 0x906a617d450187e2U, 0xb484f9dc9641e9daU, + 0xe1a63853bbd26451U, 0x8d07e33455637eb2U, 0xb049dc016abc5e5fU, + 0xdc5c5301c56b75f7U, 0x89b9b3e11b6329baU, 0xac2820d9623bf429U, + 0xd732290fbacaf133U, 0x867f59a9d4bed6c0U, 0xa81f301449ee8c70U, + 0xd226fc195c6a2f8cU, 0x83585d8fd9c25db7U, 0xa42e74f3d032f525U, + 0xcd3a1230c43fb26fU, 0x80444b5e7aa7cf85U, 0xa0555e361951c366U, + 0xc86ab5c39fa63440U, 0xfa856334878fc150U, 0x9c935e00d4b9d8d2U, + 0xc3b8358109e84f07U, 0xf4a642e14c6262c8U, 0x98e7e9cccfbd7dbdU, + 0xbf21e44003acdd2cU, 0xeeea5d5004981478U, 0x95527a5202df0ccbU, + 0xbaa718e68396cffdU, 0xe950df20247c83fdU, 0x91d28b7416cdd27eU, + 0xb6472e511c81471dU, 0xe3d8f9e563a198e5U, 0x8e679c2f5e44ff8fU, +}; + +const int16_t kPower10ExponentTable[] = { + -1200, -1196, -1193, -1190, -1186, -1183, -1180, -1176, -1173, -1170, -1166, + -1163, -1160, -1156, -1153, -1150, -1146, -1143, -1140, -1136, -1133, -1130, + -1127, -1123, -1120, -1117, -1113, -1110, -1107, -1103, -1100, -1097, -1093, + -1090, -1087, -1083, -1080, -1077, -1073, -1070, -1067, -1063, -1060, -1057, + -1053, -1050, -1047, -1043, -1040, -1037, -1034, -1030, -1027, -1024, -1020, + -1017, -1014, -1010, -1007, -1004, -1000, -997, -994, -990, -987, -984, + -980, -977, -974, -970, -967, -964, -960, -957, -954, -950, -947, + -944, -940, -937, -934, -931, -927, -924, -921, -917, -914, -911, + -907, -904, -901, -897, -894, -891, -887, -884, -881, -877, -874, + -871, -867, -864, -861, -857, -854, -851, -847, -844, -841, -838, + -834, -831, -828, -824, -821, -818, -814, -811, -808, -804, -801, + -798, -794, -791, -788, -784, -781, -778, -774, -771, -768, -764, + -761, -758, -754, -751, -748, -744, -741, -738, -735, -731, -728, + -725, -721, -718, -715, -711, -708, -705, -701, -698, -695, -691, + -688, -685, -681, -678, -675, -671, -668, -665, -661, -658, -655, + -651, -648, -645, -642, -638, -635, -632, -628, -625, -622, -618, + -615, -612, -608, -605, -602, -598, -595, -592, -588, -585, -582, + -578, -575, -572, -568, -565, -562, -558, -555, -552, -549, -545, + -542, -539, -535, -532, -529, -525, -522, -519, -515, -512, -509, + -505, -502, -499, -495, -492, -489, -485, -482, -479, -475, -472, + -469, -465, -462, -459, -455, -452, -449, -446, -442, -439, -436, + -432, -429, -426, -422, -419, -416, -412, -409, -406, -402, -399, + -396, -392, -389, -386, -382, -379, -376, -372, -369, -366, -362, + -359, -356, -353, -349, -346, -343, -339, -336, -333, -329, -326, + -323, -319, -316, -313, -309, -306, -303, -299, -296, -293, -289, + -286, -283, -279, -276, -273, -269, -266, -263, -259, -256, -253, + -250, -246, -243, -240, -236, -233, -230, -226, -223, -220, -216, + -213, -210, -206, -203, -200, -196, -193, -190, -186, -183, -180, + -176, -173, -170, -166, -163, -160, -157, -153, -150, -147, -143, + -140, -137, -133, -130, -127, -123, -120, -117, -113, -110, -107, + -103, -100, -97, -93, -90, -87, -83, -80, -77, -73, -70, + -67, -63, -60, -57, -54, -50, -47, -44, -40, -37, -34, + -30, -27, -24, -20, -17, -14, -10, -7, -4, 0, 3, + 6, 10, 13, 16, 20, 23, 26, 30, 33, 36, 39, + 43, 46, 49, 53, 56, 59, 63, 66, 69, 73, 76, + 79, 83, 86, 89, 93, 96, 99, 103, 106, 109, 113, + 116, 119, 123, 126, 129, 132, 136, 139, 142, 146, 149, + 152, 156, 159, 162, 166, 169, 172, 176, 179, 182, 186, + 189, 192, 196, 199, 202, 206, 209, 212, 216, 219, 222, + 226, 229, 232, 235, 239, 242, 245, 249, 252, 255, 259, + 262, 265, 269, 272, 275, 279, 282, 285, 289, 292, 295, + 299, 302, 305, 309, 312, 315, 319, 322, 325, 328, 332, + 335, 338, 342, 345, 348, 352, 355, 358, 362, 365, 368, + 372, 375, 378, 382, 385, 388, 392, 395, 398, 402, 405, + 408, 412, 415, 418, 422, 425, 428, 431, 435, 438, 441, + 445, 448, 451, 455, 458, 461, 465, 468, 471, 475, 478, + 481, 485, 488, 491, 495, 498, 501, 505, 508, 511, 515, + 518, 521, 524, 528, 531, 534, 538, 541, 544, 548, 551, + 554, 558, 561, 564, 568, 571, 574, 578, 581, 584, 588, + 591, 594, 598, 601, 604, 608, 611, 614, 617, 621, 624, + 627, 631, 634, 637, 641, 644, 647, 651, 654, 657, 661, + 664, 667, 671, 674, 677, 681, 684, 687, 691, 694, 697, + 701, 704, 707, 711, 714, 717, 720, 724, 727, 730, 734, + 737, 740, 744, 747, 750, 754, 757, 760, 764, 767, 770, + 774, 777, 780, 784, 787, 790, 794, 797, 800, 804, 807, + 810, 813, 817, 820, 823, 827, 830, 833, 837, 840, 843, + 847, 850, 853, 857, 860, 863, 867, 870, 873, 877, 880, + 883, 887, 890, 893, 897, 900, 903, 907, 910, 913, 916, + 920, 923, 926, 930, 933, 936, 940, 943, 946, 950, 953, + 956, 960, +}; + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/charconv.h b/third_party/abseil_cpp/absl/strings/charconv.h new file mode 100644 index 000000000000..e04be32f9514 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/charconv.h @@ -0,0 +1,119 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CHARCONV_H_ +#define ABSL_STRINGS_CHARCONV_H_ + +#include <system_error> // NOLINT(build/c++11) + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Workalike compatibilty version of std::chars_format from C++17. +// +// This is an bitfield enumerator which can be passed to absl::from_chars to +// configure the string-to-float conversion. +enum class chars_format { + scientific = 1, + fixed = 2, + hex = 4, + general = fixed | scientific, +}; + +// The return result of a string-to-number conversion. +// +// `ec` will be set to `invalid_argument` if a well-formed number was not found +// at the start of the input range, `result_out_of_range` if a well-formed +// number was found, but it was out of the representable range of the requested +// type, or to std::errc() otherwise. +// +// If a well-formed number was found, `ptr` is set to one past the sequence of +// characters that were successfully parsed. If none was found, `ptr` is set +// to the `first` argument to from_chars. +struct from_chars_result { + const char* ptr; + std::errc ec; +}; + +// Workalike compatibilty version of std::from_chars from C++17. Currently +// this only supports the `double` and `float` types. +// +// This interface incorporates the proposed resolutions for library issues +// DR 3080 and DR 3081. If these are adopted with different wording, +// Abseil's behavior will change to match the standard. (The behavior most +// likely to change is for DR 3081, which says what `value` will be set to in +// the case of overflow and underflow. Code that wants to avoid possible +// breaking changes in this area should not depend on `value` when the returned +// from_chars_result indicates a range error.) +// +// Searches the range [first, last) for the longest matching pattern beginning +// at `first` that represents a floating point number. If one is found, store +// the result in `value`. +// +// The matching pattern format is almost the same as that of strtod(), except +// that C locale is not respected, and an initial '+' character in the input +// range will never be matched. +// +// If `fmt` is set, it must be one of the enumerator values of the chars_format. +// (This is despite the fact that chars_format is a bitmask type.) If set to +// `scientific`, a matching number must contain an exponent. If set to `fixed`, +// then an exponent will never match. (For example, the string "1e5" will be +// parsed as "1".) If set to `hex`, then a hexadecimal float is parsed in the +// format that strtod() accepts, except that a "0x" prefix is NOT matched. +// (In particular, in `hex` mode, the input "0xff" results in the largest +// matching pattern "0".) +absl::from_chars_result from_chars(const char* first, const char* last, + double& value, // NOLINT + chars_format fmt = chars_format::general); + +absl::from_chars_result from_chars(const char* first, const char* last, + float& value, // NOLINT + chars_format fmt = chars_format::general); + +// std::chars_format is specified as a bitmask type, which means the following +// operations must be provided: +inline constexpr chars_format operator&(chars_format lhs, chars_format rhs) { + return static_cast<chars_format>(static_cast<int>(lhs) & + static_cast<int>(rhs)); +} +inline constexpr chars_format operator|(chars_format lhs, chars_format rhs) { + return static_cast<chars_format>(static_cast<int>(lhs) | + static_cast<int>(rhs)); +} +inline constexpr chars_format operator^(chars_format lhs, chars_format rhs) { + return static_cast<chars_format>(static_cast<int>(lhs) ^ + static_cast<int>(rhs)); +} +inline constexpr chars_format operator~(chars_format arg) { + return static_cast<chars_format>(~static_cast<int>(arg)); +} +inline chars_format& operator&=(chars_format& lhs, chars_format rhs) { + lhs = lhs & rhs; + return lhs; +} +inline chars_format& operator|=(chars_format& lhs, chars_format rhs) { + lhs = lhs | rhs; + return lhs; +} +inline chars_format& operator^=(chars_format& lhs, chars_format rhs) { + lhs = lhs ^ rhs; + return lhs; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CHARCONV_H_ diff --git a/third_party/abseil_cpp/absl/strings/charconv_benchmark.cc b/third_party/abseil_cpp/absl/strings/charconv_benchmark.cc new file mode 100644 index 000000000000..e8c7371d6586 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/charconv_benchmark.cc @@ -0,0 +1,204 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/charconv.h" + +#include <cstdlib> +#include <cstring> +#include <string> + +#include "benchmark/benchmark.h" + +namespace { + +void BM_Strtod_Pi(benchmark::State& state) { + const char* pi = "3.14159"; + for (auto s : state) { + benchmark::DoNotOptimize(pi); + benchmark::DoNotOptimize(strtod(pi, nullptr)); + } +} +BENCHMARK(BM_Strtod_Pi); + +void BM_Absl_Pi(benchmark::State& state) { + const char* pi = "3.14159"; + const char* pi_end = pi + strlen(pi); + for (auto s : state) { + benchmark::DoNotOptimize(pi); + double v; + absl::from_chars(pi, pi_end, v); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_Absl_Pi); + +void BM_Strtod_Pi_float(benchmark::State& state) { + const char* pi = "3.14159"; + for (auto s : state) { + benchmark::DoNotOptimize(pi); + benchmark::DoNotOptimize(strtof(pi, nullptr)); + } +} +BENCHMARK(BM_Strtod_Pi_float); + +void BM_Absl_Pi_float(benchmark::State& state) { + const char* pi = "3.14159"; + const char* pi_end = pi + strlen(pi); + for (auto s : state) { + benchmark::DoNotOptimize(pi); + float v; + absl::from_chars(pi, pi_end, v); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_Absl_Pi_float); + +void BM_Strtod_HardLarge(benchmark::State& state) { + const char* num = "272104041512242479.e200"; + for (auto s : state) { + benchmark::DoNotOptimize(num); + benchmark::DoNotOptimize(strtod(num, nullptr)); + } +} +BENCHMARK(BM_Strtod_HardLarge); + +void BM_Absl_HardLarge(benchmark::State& state) { + const char* numstr = "272104041512242479.e200"; + const char* numstr_end = numstr + strlen(numstr); + for (auto s : state) { + benchmark::DoNotOptimize(numstr); + double v; + absl::from_chars(numstr, numstr_end, v); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_Absl_HardLarge); + +void BM_Strtod_HardSmall(benchmark::State& state) { + const char* num = "94080055902682397.e-242"; + for (auto s : state) { + benchmark::DoNotOptimize(num); + benchmark::DoNotOptimize(strtod(num, nullptr)); + } +} +BENCHMARK(BM_Strtod_HardSmall); + +void BM_Absl_HardSmall(benchmark::State& state) { + const char* numstr = "94080055902682397.e-242"; + const char* numstr_end = numstr + strlen(numstr); + for (auto s : state) { + benchmark::DoNotOptimize(numstr); + double v; + absl::from_chars(numstr, numstr_end, v); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_Absl_HardSmall); + +void BM_Strtod_HugeMantissa(benchmark::State& state) { + std::string huge(200, '3'); + const char* num = huge.c_str(); + for (auto s : state) { + benchmark::DoNotOptimize(num); + benchmark::DoNotOptimize(strtod(num, nullptr)); + } +} +BENCHMARK(BM_Strtod_HugeMantissa); + +void BM_Absl_HugeMantissa(benchmark::State& state) { + std::string huge(200, '3'); + const char* num = huge.c_str(); + const char* num_end = num + 200; + for (auto s : state) { + benchmark::DoNotOptimize(num); + double v; + absl::from_chars(num, num_end, v); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_Absl_HugeMantissa); + +std::string MakeHardCase(int length) { + // The number 1.1521...e-297 is exactly halfway between 12345 * 2**-1000 and + // the next larger representable number. The digits of this number are in + // the string below. + const std::string digits = + "1." + "152113937042223790993097181572444900347587985074226836242307364987727724" + "831384300183638649152607195040591791364113930628852279348613864894524591" + "272746490313676832900762939595690019745859128071117417798540258114233761" + "012939937017879509401007964861774960297319002612457273148497158989073482" + "171377406078223015359818300988676687994537274548940612510414856761641652" + "513434981938564294004070500716200446656421722229202383105446378511678258" + "370570631774499359748259931676320916632111681001853983492795053244971606" + "922718923011680846577744433974087653954904214152517799883551075537146316" + "168973685866425605046988661997658648354773076621610279716804960009043764" + "038392994055171112475093876476783502487512538082706095923790634572014823" + "78877699375152587890625" + + std::string(5000, '0'); + // generate the hard cases on either side for the given length. + // Lengths between 3 and 1000 are reasonable. + return digits.substr(0, length) + "1e-297"; +} + +void BM_Strtod_Big_And_Difficult(benchmark::State& state) { + std::string testcase = MakeHardCase(state.range(0)); + const char* begin = testcase.c_str(); + for (auto s : state) { + benchmark::DoNotOptimize(begin); + benchmark::DoNotOptimize(strtod(begin, nullptr)); + } +} +BENCHMARK(BM_Strtod_Big_And_Difficult)->Range(3, 5000); + +void BM_Absl_Big_And_Difficult(benchmark::State& state) { + std::string testcase = MakeHardCase(state.range(0)); + const char* begin = testcase.c_str(); + const char* end = begin + testcase.size(); + for (auto s : state) { + benchmark::DoNotOptimize(begin); + double v; + absl::from_chars(begin, end, v); + benchmark::DoNotOptimize(v); + } +} +BENCHMARK(BM_Absl_Big_And_Difficult)->Range(3, 5000); + +} // namespace + +// ------------------------------------------------------------------------ +// Benchmark Time CPU Iterations +// ------------------------------------------------------------------------ +// BM_Strtod_Pi 96 ns 96 ns 6337454 +// BM_Absl_Pi 35 ns 35 ns 20031996 +// BM_Strtod_Pi_float 91 ns 91 ns 7745851 +// BM_Absl_Pi_float 35 ns 35 ns 20430298 +// BM_Strtod_HardLarge 133 ns 133 ns 5288341 +// BM_Absl_HardLarge 181 ns 181 ns 3855615 +// BM_Strtod_HardSmall 279 ns 279 ns 2517243 +// BM_Absl_HardSmall 287 ns 287 ns 2458744 +// BM_Strtod_HugeMantissa 433 ns 433 ns 1604293 +// BM_Absl_HugeMantissa 160 ns 160 ns 4403671 +// BM_Strtod_Big_And_Difficult/3 236 ns 236 ns 2942496 +// BM_Strtod_Big_And_Difficult/8 232 ns 232 ns 2983796 +// BM_Strtod_Big_And_Difficult/64 437 ns 437 ns 1591951 +// BM_Strtod_Big_And_Difficult/512 1738 ns 1738 ns 402519 +// BM_Strtod_Big_And_Difficult/4096 3943 ns 3943 ns 176128 +// BM_Strtod_Big_And_Difficult/5000 4397 ns 4397 ns 157878 +// BM_Absl_Big_And_Difficult/3 39 ns 39 ns 17799583 +// BM_Absl_Big_And_Difficult/8 43 ns 43 ns 16096859 +// BM_Absl_Big_And_Difficult/64 550 ns 550 ns 1259717 +// BM_Absl_Big_And_Difficult/512 4167 ns 4167 ns 171414 +// BM_Absl_Big_And_Difficult/4096 9160 ns 9159 ns 76297 +// BM_Absl_Big_And_Difficult/5000 9738 ns 9738 ns 70140 diff --git a/third_party/abseil_cpp/absl/strings/charconv_test.cc b/third_party/abseil_cpp/absl/strings/charconv_test.cc new file mode 100644 index 000000000000..9090e9c89c50 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/charconv_test.cc @@ -0,0 +1,780 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/charconv.h" + +#include <cstdlib> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/internal/pow10_helper.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" + +#ifdef _MSC_FULL_VER +#define ABSL_COMPILER_DOES_EXACT_ROUNDING 0 +#define ABSL_STRTOD_HANDLES_NAN_CORRECTLY 0 +#else +#define ABSL_COMPILER_DOES_EXACT_ROUNDING 1 +#define ABSL_STRTOD_HANDLES_NAN_CORRECTLY 1 +#endif + +namespace { + +using absl::strings_internal::Pow10; + +#if ABSL_COMPILER_DOES_EXACT_ROUNDING + +// Tests that the given string is accepted by absl::from_chars, and that it +// converts exactly equal to the given number. +void TestDoubleParse(absl::string_view str, double expected_number) { + SCOPED_TRACE(str); + double actual_number = 0.0; + absl::from_chars_result result = + absl::from_chars(str.data(), str.data() + str.length(), actual_number); + EXPECT_EQ(result.ec, std::errc()); + EXPECT_EQ(result.ptr, str.data() + str.length()); + EXPECT_EQ(actual_number, expected_number); +} + +void TestFloatParse(absl::string_view str, float expected_number) { + SCOPED_TRACE(str); + float actual_number = 0.0; + absl::from_chars_result result = + absl::from_chars(str.data(), str.data() + str.length(), actual_number); + EXPECT_EQ(result.ec, std::errc()); + EXPECT_EQ(result.ptr, str.data() + str.length()); + EXPECT_EQ(actual_number, expected_number); +} + +// Tests that the given double or single precision floating point literal is +// parsed correctly by absl::from_chars. +// +// These convenience macros assume that the C++ compiler being used also does +// fully correct decimal-to-binary conversions. +#define FROM_CHARS_TEST_DOUBLE(number) \ + { \ + TestDoubleParse(#number, number); \ + TestDoubleParse("-" #number, -number); \ + } + +#define FROM_CHARS_TEST_FLOAT(number) \ + { \ + TestFloatParse(#number, number##f); \ + TestFloatParse("-" #number, -number##f); \ + } + +TEST(FromChars, NearRoundingCases) { + // Cases from "A Program for Testing IEEE Decimal-Binary Conversion" + // by Vern Paxson. + + // Forms that should round towards zero. (These are the hardest cases for + // each decimal mantissa size.) + FROM_CHARS_TEST_DOUBLE(5.e125); + FROM_CHARS_TEST_DOUBLE(69.e267); + FROM_CHARS_TEST_DOUBLE(999.e-026); + FROM_CHARS_TEST_DOUBLE(7861.e-034); + FROM_CHARS_TEST_DOUBLE(75569.e-254); + FROM_CHARS_TEST_DOUBLE(928609.e-261); + FROM_CHARS_TEST_DOUBLE(9210917.e080); + FROM_CHARS_TEST_DOUBLE(84863171.e114); + FROM_CHARS_TEST_DOUBLE(653777767.e273); + FROM_CHARS_TEST_DOUBLE(5232604057.e-298); + FROM_CHARS_TEST_DOUBLE(27235667517.e-109); + FROM_CHARS_TEST_DOUBLE(653532977297.e-123); + FROM_CHARS_TEST_DOUBLE(3142213164987.e-294); + FROM_CHARS_TEST_DOUBLE(46202199371337.e-072); + FROM_CHARS_TEST_DOUBLE(231010996856685.e-073); + FROM_CHARS_TEST_DOUBLE(9324754620109615.e212); + FROM_CHARS_TEST_DOUBLE(78459735791271921.e049); + FROM_CHARS_TEST_DOUBLE(272104041512242479.e200); + FROM_CHARS_TEST_DOUBLE(6802601037806061975.e198); + FROM_CHARS_TEST_DOUBLE(20505426358836677347.e-221); + FROM_CHARS_TEST_DOUBLE(836168422905420598437.e-234); + FROM_CHARS_TEST_DOUBLE(4891559871276714924261.e222); + FROM_CHARS_TEST_FLOAT(5.e-20); + FROM_CHARS_TEST_FLOAT(67.e14); + FROM_CHARS_TEST_FLOAT(985.e15); + FROM_CHARS_TEST_FLOAT(7693.e-42); + FROM_CHARS_TEST_FLOAT(55895.e-16); + FROM_CHARS_TEST_FLOAT(996622.e-44); + FROM_CHARS_TEST_FLOAT(7038531.e-32); + FROM_CHARS_TEST_FLOAT(60419369.e-46); + FROM_CHARS_TEST_FLOAT(702990899.e-20); + FROM_CHARS_TEST_FLOAT(6930161142.e-48); + FROM_CHARS_TEST_FLOAT(25933168707.e-13); + FROM_CHARS_TEST_FLOAT(596428896559.e20); + + // Similarly, forms that should round away from zero. + FROM_CHARS_TEST_DOUBLE(9.e-265); + FROM_CHARS_TEST_DOUBLE(85.e-037); + FROM_CHARS_TEST_DOUBLE(623.e100); + FROM_CHARS_TEST_DOUBLE(3571.e263); + FROM_CHARS_TEST_DOUBLE(81661.e153); + FROM_CHARS_TEST_DOUBLE(920657.e-023); + FROM_CHARS_TEST_DOUBLE(4603285.e-024); + FROM_CHARS_TEST_DOUBLE(87575437.e-309); + FROM_CHARS_TEST_DOUBLE(245540327.e122); + FROM_CHARS_TEST_DOUBLE(6138508175.e120); + FROM_CHARS_TEST_DOUBLE(83356057653.e193); + FROM_CHARS_TEST_DOUBLE(619534293513.e124); + FROM_CHARS_TEST_DOUBLE(2335141086879.e218); + FROM_CHARS_TEST_DOUBLE(36167929443327.e-159); + FROM_CHARS_TEST_DOUBLE(609610927149051.e-255); + FROM_CHARS_TEST_DOUBLE(3743626360493413.e-165); + FROM_CHARS_TEST_DOUBLE(94080055902682397.e-242); + FROM_CHARS_TEST_DOUBLE(899810892172646163.e283); + FROM_CHARS_TEST_DOUBLE(7120190517612959703.e120); + FROM_CHARS_TEST_DOUBLE(25188282901709339043.e-252); + FROM_CHARS_TEST_DOUBLE(308984926168550152811.e-052); + FROM_CHARS_TEST_DOUBLE(6372891218502368041059.e064); + FROM_CHARS_TEST_FLOAT(3.e-23); + FROM_CHARS_TEST_FLOAT(57.e18); + FROM_CHARS_TEST_FLOAT(789.e-35); + FROM_CHARS_TEST_FLOAT(2539.e-18); + FROM_CHARS_TEST_FLOAT(76173.e28); + FROM_CHARS_TEST_FLOAT(887745.e-11); + FROM_CHARS_TEST_FLOAT(5382571.e-37); + FROM_CHARS_TEST_FLOAT(82381273.e-35); + FROM_CHARS_TEST_FLOAT(750486563.e-38); + FROM_CHARS_TEST_FLOAT(3752432815.e-39); + FROM_CHARS_TEST_FLOAT(75224575729.e-45); + FROM_CHARS_TEST_FLOAT(459926601011.e15); +} + +#undef FROM_CHARS_TEST_DOUBLE +#undef FROM_CHARS_TEST_FLOAT +#endif + +float ToFloat(absl::string_view s) { + float f; + absl::from_chars(s.data(), s.data() + s.size(), f); + return f; +} + +double ToDouble(absl::string_view s) { + double d; + absl::from_chars(s.data(), s.data() + s.size(), d); + return d; +} + +// A duplication of the test cases in "NearRoundingCases" above, but with +// expected values expressed with integers, using ldexp/ldexpf. These test +// cases will work even on compilers that do not accurately round floating point +// literals. +TEST(FromChars, NearRoundingCasesExplicit) { + EXPECT_EQ(ToDouble("5.e125"), ldexp(6653062250012735, 365)); + EXPECT_EQ(ToDouble("69.e267"), ldexp(4705683757438170, 841)); + EXPECT_EQ(ToDouble("999.e-026"), ldexp(6798841691080350, -129)); + EXPECT_EQ(ToDouble("7861.e-034"), ldexp(8975675289889240, -153)); + EXPECT_EQ(ToDouble("75569.e-254"), ldexp(6091718967192243, -880)); + EXPECT_EQ(ToDouble("928609.e-261"), ldexp(7849264900213743, -900)); + EXPECT_EQ(ToDouble("9210917.e080"), ldexp(8341110837370930, 236)); + EXPECT_EQ(ToDouble("84863171.e114"), ldexp(4625202867375927, 353)); + EXPECT_EQ(ToDouble("653777767.e273"), ldexp(5068902999763073, 884)); + EXPECT_EQ(ToDouble("5232604057.e-298"), ldexp(5741343011915040, -1010)); + EXPECT_EQ(ToDouble("27235667517.e-109"), ldexp(6707124626673586, -380)); + EXPECT_EQ(ToDouble("653532977297.e-123"), ldexp(7078246407265384, -422)); + EXPECT_EQ(ToDouble("3142213164987.e-294"), ldexp(8219991337640559, -988)); + EXPECT_EQ(ToDouble("46202199371337.e-072"), ldexp(5224462102115359, -246)); + EXPECT_EQ(ToDouble("231010996856685.e-073"), ldexp(5224462102115359, -247)); + EXPECT_EQ(ToDouble("9324754620109615.e212"), ldexp(5539753864394442, 705)); + EXPECT_EQ(ToDouble("78459735791271921.e049"), ldexp(8388176519442766, 166)); + EXPECT_EQ(ToDouble("272104041512242479.e200"), ldexp(5554409530847367, 670)); + EXPECT_EQ(ToDouble("6802601037806061975.e198"), ldexp(5554409530847367, 668)); + EXPECT_EQ(ToDouble("20505426358836677347.e-221"), + ldexp(4524032052079546, -722)); + EXPECT_EQ(ToDouble("836168422905420598437.e-234"), + ldexp(5070963299887562, -760)); + EXPECT_EQ(ToDouble("4891559871276714924261.e222"), + ldexp(6452687840519111, 757)); + EXPECT_EQ(ToFloat("5.e-20"), ldexpf(15474250, -88)); + EXPECT_EQ(ToFloat("67.e14"), ldexpf(12479722, 29)); + EXPECT_EQ(ToFloat("985.e15"), ldexpf(14333636, 36)); + EXPECT_EQ(ToFloat("7693.e-42"), ldexpf(10979816, -150)); + EXPECT_EQ(ToFloat("55895.e-16"), ldexpf(12888509, -61)); + EXPECT_EQ(ToFloat("996622.e-44"), ldexpf(14224264, -150)); + EXPECT_EQ(ToFloat("7038531.e-32"), ldexpf(11420669, -107)); + EXPECT_EQ(ToFloat("60419369.e-46"), ldexpf(8623340, -150)); + EXPECT_EQ(ToFloat("702990899.e-20"), ldexpf(16209866, -61)); + EXPECT_EQ(ToFloat("6930161142.e-48"), ldexpf(9891056, -150)); + EXPECT_EQ(ToFloat("25933168707.e-13"), ldexpf(11138211, -32)); + EXPECT_EQ(ToFloat("596428896559.e20"), ldexpf(12333860, 82)); + + + EXPECT_EQ(ToDouble("9.e-265"), ldexp(8168427841980010, -930)); + EXPECT_EQ(ToDouble("85.e-037"), ldexp(6360455125664090, -169)); + EXPECT_EQ(ToDouble("623.e100"), ldexp(6263531988747231, 289)); + EXPECT_EQ(ToDouble("3571.e263"), ldexp(6234526311072170, 833)); + EXPECT_EQ(ToDouble("81661.e153"), ldexp(6696636728760206, 472)); + EXPECT_EQ(ToDouble("920657.e-023"), ldexp(5975405561110124, -109)); + EXPECT_EQ(ToDouble("4603285.e-024"), ldexp(5975405561110124, -110)); + EXPECT_EQ(ToDouble("87575437.e-309"), ldexp(8452160731874668, -1053)); + EXPECT_EQ(ToDouble("245540327.e122"), ldexp(4985336549131723, 381)); + EXPECT_EQ(ToDouble("6138508175.e120"), ldexp(4985336549131723, 379)); + EXPECT_EQ(ToDouble("83356057653.e193"), ldexp(5986732817132056, 625)); + EXPECT_EQ(ToDouble("619534293513.e124"), ldexp(4798406992060657, 399)); + EXPECT_EQ(ToDouble("2335141086879.e218"), ldexp(5419088166961646, 713)); + EXPECT_EQ(ToDouble("36167929443327.e-159"), ldexp(8135819834632444, -536)); + EXPECT_EQ(ToDouble("609610927149051.e-255"), ldexp(4576664294594737, -850)); + EXPECT_EQ(ToDouble("3743626360493413.e-165"), ldexp(6898586531774201, -549)); + EXPECT_EQ(ToDouble("94080055902682397.e-242"), ldexp(6273271706052298, -800)); + EXPECT_EQ(ToDouble("899810892172646163.e283"), ldexp(7563892574477827, 947)); + EXPECT_EQ(ToDouble("7120190517612959703.e120"), ldexp(5385467232557565, 409)); + EXPECT_EQ(ToDouble("25188282901709339043.e-252"), + ldexp(5635662608542340, -825)); + EXPECT_EQ(ToDouble("308984926168550152811.e-052"), + ldexp(5644774693823803, -157)); + EXPECT_EQ(ToDouble("6372891218502368041059.e064"), + ldexp(4616868614322430, 233)); + + EXPECT_EQ(ToFloat("3.e-23"), ldexpf(9507380, -98)); + EXPECT_EQ(ToFloat("57.e18"), ldexpf(12960300, 42)); + EXPECT_EQ(ToFloat("789.e-35"), ldexpf(10739312, -130)); + EXPECT_EQ(ToFloat("2539.e-18"), ldexpf(11990089, -72)); + EXPECT_EQ(ToFloat("76173.e28"), ldexpf(9845130, 86)); + EXPECT_EQ(ToFloat("887745.e-11"), ldexpf(9760860, -40)); + EXPECT_EQ(ToFloat("5382571.e-37"), ldexpf(11447463, -124)); + EXPECT_EQ(ToFloat("82381273.e-35"), ldexpf(8554961, -113)); + EXPECT_EQ(ToFloat("750486563.e-38"), ldexpf(9975678, -120)); + EXPECT_EQ(ToFloat("3752432815.e-39"), ldexpf(9975678, -121)); + EXPECT_EQ(ToFloat("75224575729.e-45"), ldexpf(13105970, -137)); + EXPECT_EQ(ToFloat("459926601011.e15"), ldexpf(12466336, 65)); +} + +// Common test logic for converting a string which lies exactly halfway between +// two target floats. +// +// mantissa and exponent represent the precise value between two floating point +// numbers, `expected_low` and `expected_high`. The floating point +// representation to parse in `StrCat(mantissa, "e", exponent)`. +// +// This function checks that an input just slightly less than the exact value +// is rounded down to `expected_low`, and an input just slightly greater than +// the exact value is rounded up to `expected_high`. +// +// The exact value should round to `expected_half`, which must be either +// `expected_low` or `expected_high`. +template <typename FloatType> +void TestHalfwayValue(const std::string& mantissa, int exponent, + FloatType expected_low, FloatType expected_high, + FloatType expected_half) { + std::string low_rep = mantissa; + low_rep[low_rep.size() - 1] -= 1; + absl::StrAppend(&low_rep, std::string(1000, '9'), "e", exponent); + + FloatType actual_low = 0; + absl::from_chars(low_rep.data(), low_rep.data() + low_rep.size(), actual_low); + EXPECT_EQ(expected_low, actual_low); + + std::string high_rep = + absl::StrCat(mantissa, std::string(1000, '0'), "1e", exponent); + FloatType actual_high = 0; + absl::from_chars(high_rep.data(), high_rep.data() + high_rep.size(), + actual_high); + EXPECT_EQ(expected_high, actual_high); + + std::string halfway_rep = absl::StrCat(mantissa, "e", exponent); + FloatType actual_half = 0; + absl::from_chars(halfway_rep.data(), halfway_rep.data() + halfway_rep.size(), + actual_half); + EXPECT_EQ(expected_half, actual_half); +} + +TEST(FromChars, DoubleRounding) { + const double zero = 0.0; + const double first_subnormal = nextafter(zero, 1.0); + const double second_subnormal = nextafter(first_subnormal, 1.0); + + const double first_normal = DBL_MIN; + const double last_subnormal = nextafter(first_normal, 0.0); + const double second_normal = nextafter(first_normal, 1.0); + + const double last_normal = DBL_MAX; + const double penultimate_normal = nextafter(last_normal, 0.0); + + // Various test cases for numbers between two representable floats. Each + // call to TestHalfwayValue tests a number just below and just above the + // halfway point, as well as the number exactly between them. + + // Test between zero and first_subnormal. Round-to-even tie rounds down. + TestHalfwayValue( + "2." + "470328229206232720882843964341106861825299013071623822127928412503377536" + "351043759326499181808179961898982823477228588654633283551779698981993873" + "980053909390631503565951557022639229085839244910518443593180284993653615" + "250031937045767824921936562366986365848075700158576926990370631192827955" + "855133292783433840935197801553124659726357957462276646527282722005637400" + "648549997709659947045402082816622623785739345073633900796776193057750674" + "017632467360096895134053553745851666113422376667860416215968046191446729" + "184030053005753084904876539171138659164623952491262365388187963623937328" + "042389101867234849766823508986338858792562830275599565752445550725518931" + "369083625477918694866799496832404970582102851318545139621383772282614543" + "7693412532098591327667236328125", + -324, zero, first_subnormal, zero); + + // first_subnormal and second_subnormal. Round-to-even tie rounds up. + TestHalfwayValue( + "7." + "410984687618698162648531893023320585475897039214871466383785237510132609" + "053131277979497545424539885696948470431685765963899850655339096945981621" + "940161728171894510697854671067917687257517734731555330779540854980960845" + "750095811137303474765809687100959097544227100475730780971111893578483867" + "565399878350301522805593404659373979179073872386829939581848166016912201" + "945649993128979841136206248449867871357218035220901702390328579173252022" + "052897402080290685402160661237554998340267130003581248647904138574340187" + "552090159017259254714629617513415977493871857473787096164563890871811984" + "127167305601704549300470526959016576377688490826798697257336652176556794" + "107250876433756084600398490497214911746308553955635418864151316847843631" + "3080237596295773983001708984375", + -324, first_subnormal, second_subnormal, second_subnormal); + + // last_subnormal and first_normal. Round-to-even tie rounds up. + TestHalfwayValue( + "2." + "225073858507201136057409796709131975934819546351645648023426109724822222" + "021076945516529523908135087914149158913039621106870086438694594645527657" + "207407820621743379988141063267329253552286881372149012981122451451889849" + "057222307285255133155755015914397476397983411801999323962548289017107081" + "850690630666655994938275772572015763062690663332647565300009245888316433" + "037779791869612049497390377829704905051080609940730262937128958950003583" + "799967207254304360284078895771796150945516748243471030702609144621572289" + "880258182545180325707018860872113128079512233426288368622321503775666622" + "503982534335974568884423900265498198385487948292206894721689831099698365" + "846814022854243330660339850886445804001034933970427567186443383770486037" + "86162277173854562306587467901408672332763671875", + -308, last_subnormal, first_normal, first_normal); + + // first_normal and second_normal. Round-to-even tie rounds down. + TestHalfwayValue( + "2." + "225073858507201630123055637955676152503612414573018013083228724049586647" + "606759446192036794116886953213985520549032000903434781884412325572184367" + "563347617020518175998922941393629966742598285899994830148971433555578567" + "693279306015978183162142425067962460785295885199272493577688320732492479" + "924816869232247165964934329258783950102250973957579510571600738343645738" + "494324192997092179207389919761694314131497173265255020084997973676783743" + "155205818804439163810572367791175177756227497413804253387084478193655533" + "073867420834526162513029462022730109054820067654020201547112002028139700" + "141575259123440177362244273712468151750189745559978653234255886219611516" + "335924167958029604477064946470184777360934300451421683607013647479513962" + "13837722826145437693412532098591327667236328125", + -308, first_normal, second_normal, first_normal); + + // penultimate_normal and last_normal. Round-to-even rounds down. + TestHalfwayValue( + "1." + "797693134862315608353258760581052985162070023416521662616611746258695532" + "672923265745300992879465492467506314903358770175220871059269879629062776" + "047355692132901909191523941804762171253349609463563872612866401980290377" + "995141836029815117562837277714038305214839639239356331336428021390916694" + "57927874464075218944", + 308, penultimate_normal, last_normal, penultimate_normal); +} + +// Same test cases as DoubleRounding, now with new and improved Much Smaller +// Precision! +TEST(FromChars, FloatRounding) { + const float zero = 0.0; + const float first_subnormal = nextafterf(zero, 1.0); + const float second_subnormal = nextafterf(first_subnormal, 1.0); + + const float first_normal = FLT_MIN; + const float last_subnormal = nextafterf(first_normal, 0.0); + const float second_normal = nextafterf(first_normal, 1.0); + + const float last_normal = FLT_MAX; + const float penultimate_normal = nextafterf(last_normal, 0.0); + + // Test between zero and first_subnormal. Round-to-even tie rounds down. + TestHalfwayValue( + "7." + "006492321624085354618647916449580656401309709382578858785341419448955413" + "42930300743319094181060791015625", + -46, zero, first_subnormal, zero); + + // first_subnormal and second_subnormal. Round-to-even tie rounds up. + TestHalfwayValue( + "2." + "101947696487225606385594374934874196920392912814773657635602425834686624" + "028790902229957282543182373046875", + -45, first_subnormal, second_subnormal, second_subnormal); + + // last_subnormal and first_normal. Round-to-even tie rounds up. + TestHalfwayValue( + "1." + "175494280757364291727882991035766513322858992758990427682963118425003064" + "9651730385585324256680905818939208984375", + -38, last_subnormal, first_normal, first_normal); + + // first_normal and second_normal. Round-to-even tie rounds down. + TestHalfwayValue( + "1." + "175494420887210724209590083408724842314472120785184615334540294131831453" + "9442813071445925743319094181060791015625", + -38, first_normal, second_normal, first_normal); + + // penultimate_normal and last_normal. Round-to-even rounds down. + TestHalfwayValue("3.40282336497324057985868971510891282432", 38, + penultimate_normal, last_normal, penultimate_normal); +} + +TEST(FromChars, Underflow) { + // Check that underflow is handled correctly, according to the specification + // in DR 3081. + double d; + float f; + absl::from_chars_result result; + + std::string negative_underflow = "-1e-1000"; + const char* begin = negative_underflow.data(); + const char* end = begin + negative_underflow.size(); + d = 100.0; + result = absl::from_chars(begin, end, d); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_TRUE(std::signbit(d)); // negative + EXPECT_GE(d, -std::numeric_limits<double>::min()); + f = 100.0; + result = absl::from_chars(begin, end, f); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_TRUE(std::signbit(f)); // negative + EXPECT_GE(f, -std::numeric_limits<float>::min()); + + std::string positive_underflow = "1e-1000"; + begin = positive_underflow.data(); + end = begin + positive_underflow.size(); + d = -100.0; + result = absl::from_chars(begin, end, d); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_FALSE(std::signbit(d)); // positive + EXPECT_LE(d, std::numeric_limits<double>::min()); + f = -100.0; + result = absl::from_chars(begin, end, f); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_FALSE(std::signbit(f)); // positive + EXPECT_LE(f, std::numeric_limits<float>::min()); +} + +TEST(FromChars, Overflow) { + // Check that overflow is handled correctly, according to the specification + // in DR 3081. + double d; + float f; + absl::from_chars_result result; + + std::string negative_overflow = "-1e1000"; + const char* begin = negative_overflow.data(); + const char* end = begin + negative_overflow.size(); + d = 100.0; + result = absl::from_chars(begin, end, d); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_TRUE(std::signbit(d)); // negative + EXPECT_EQ(d, -std::numeric_limits<double>::max()); + f = 100.0; + result = absl::from_chars(begin, end, f); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_TRUE(std::signbit(f)); // negative + EXPECT_EQ(f, -std::numeric_limits<float>::max()); + + std::string positive_overflow = "1e1000"; + begin = positive_overflow.data(); + end = begin + positive_overflow.size(); + d = -100.0; + result = absl::from_chars(begin, end, d); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_FALSE(std::signbit(d)); // positive + EXPECT_EQ(d, std::numeric_limits<double>::max()); + f = -100.0; + result = absl::from_chars(begin, end, f); + EXPECT_EQ(result.ptr, end); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_FALSE(std::signbit(f)); // positive + EXPECT_EQ(f, std::numeric_limits<float>::max()); +} + +TEST(FromChars, RegressionTestsFromFuzzer) { + absl::string_view src = "0x21900000p00000000099"; + float f; + auto result = absl::from_chars(src.data(), src.data() + src.size(), f); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); +} + +TEST(FromChars, ReturnValuePtr) { + // Check that `ptr` points one past the number scanned, even if that number + // is not representable. + double d; + absl::from_chars_result result; + + std::string normal = "3.14@#$%@#$%"; + result = absl::from_chars(normal.data(), normal.data() + normal.size(), d); + EXPECT_EQ(result.ec, std::errc()); + EXPECT_EQ(result.ptr - normal.data(), 4); + + std::string overflow = "1e1000@#$%@#$%"; + result = absl::from_chars(overflow.data(), + overflow.data() + overflow.size(), d); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_EQ(result.ptr - overflow.data(), 6); + + std::string garbage = "#$%@#$%"; + result = absl::from_chars(garbage.data(), + garbage.data() + garbage.size(), d); + EXPECT_EQ(result.ec, std::errc::invalid_argument); + EXPECT_EQ(result.ptr - garbage.data(), 0); +} + +// Check for a wide range of inputs that strtod() and absl::from_chars() exactly +// agree on the conversion amount. +// +// This test assumes the platform's strtod() uses perfect round_to_nearest +// rounding. +TEST(FromChars, TestVersusStrtod) { + for (int mantissa = 1000000; mantissa <= 9999999; mantissa += 501) { + for (int exponent = -300; exponent < 300; ++exponent) { + std::string candidate = absl::StrCat(mantissa, "e", exponent); + double strtod_value = strtod(candidate.c_str(), nullptr); + double absl_value = 0; + absl::from_chars(candidate.data(), candidate.data() + candidate.size(), + absl_value); + ASSERT_EQ(strtod_value, absl_value) << candidate; + } + } +} + +// Check for a wide range of inputs that strtof() and absl::from_chars() exactly +// agree on the conversion amount. +// +// This test assumes the platform's strtof() uses perfect round_to_nearest +// rounding. +TEST(FromChars, TestVersusStrtof) { + for (int mantissa = 1000000; mantissa <= 9999999; mantissa += 501) { + for (int exponent = -43; exponent < 32; ++exponent) { + std::string candidate = absl::StrCat(mantissa, "e", exponent); + float strtod_value = strtof(candidate.c_str(), nullptr); + float absl_value = 0; + absl::from_chars(candidate.data(), candidate.data() + candidate.size(), + absl_value); + ASSERT_EQ(strtod_value, absl_value) << candidate; + } + } +} + +// Tests if two floating point values have identical bit layouts. (EXPECT_EQ +// is not suitable for NaN testing, since NaNs are never equal.) +template <typename Float> +bool Identical(Float a, Float b) { + return 0 == memcmp(&a, &b, sizeof(Float)); +} + +// Check that NaNs are parsed correctly. The spec requires that +// std::from_chars on "NaN(123abc)" return the same value as std::nan("123abc"). +// How such an n-char-sequence affects the generated NaN is unspecified, so we +// just test for symmetry with std::nan and strtod here. +// +// (In Linux, this parses the value as a number and stuffs that number into the +// free bits of a quiet NaN.) +TEST(FromChars, NaNDoubles) { + for (std::string n_char_sequence : + {"", "1", "2", "3", "fff", "FFF", "200000", "400000", "4000000000000", + "8000000000000", "abc123", "legal_but_unexpected", + "99999999999999999999999", "_"}) { + std::string input = absl::StrCat("nan(", n_char_sequence, ")"); + SCOPED_TRACE(input); + double from_chars_double; + absl::from_chars(input.data(), input.data() + input.size(), + from_chars_double); + double std_nan_double = std::nan(n_char_sequence.c_str()); + EXPECT_TRUE(Identical(from_chars_double, std_nan_double)); + + // Also check that we match strtod()'s behavior. This test assumes that the + // platform has a compliant strtod(). +#if ABSL_STRTOD_HANDLES_NAN_CORRECTLY + double strtod_double = strtod(input.c_str(), nullptr); + EXPECT_TRUE(Identical(from_chars_double, strtod_double)); +#endif // ABSL_STRTOD_HANDLES_NAN_CORRECTLY + + // Check that we can parse a negative NaN + std::string negative_input = "-" + input; + double negative_from_chars_double; + absl::from_chars(negative_input.data(), + negative_input.data() + negative_input.size(), + negative_from_chars_double); + EXPECT_TRUE(std::signbit(negative_from_chars_double)); + EXPECT_FALSE(Identical(negative_from_chars_double, from_chars_double)); + from_chars_double = std::copysign(from_chars_double, -1.0); + EXPECT_TRUE(Identical(negative_from_chars_double, from_chars_double)); + } +} + +TEST(FromChars, NaNFloats) { + for (std::string n_char_sequence : + {"", "1", "2", "3", "fff", "FFF", "200000", "400000", "4000000000000", + "8000000000000", "abc123", "legal_but_unexpected", + "99999999999999999999999", "_"}) { + std::string input = absl::StrCat("nan(", n_char_sequence, ")"); + SCOPED_TRACE(input); + float from_chars_float; + absl::from_chars(input.data(), input.data() + input.size(), + from_chars_float); + float std_nan_float = std::nanf(n_char_sequence.c_str()); + EXPECT_TRUE(Identical(from_chars_float, std_nan_float)); + + // Also check that we match strtof()'s behavior. This test assumes that the + // platform has a compliant strtof(). +#if ABSL_STRTOD_HANDLES_NAN_CORRECTLY + float strtof_float = strtof(input.c_str(), nullptr); + EXPECT_TRUE(Identical(from_chars_float, strtof_float)); +#endif // ABSL_STRTOD_HANDLES_NAN_CORRECTLY + + // Check that we can parse a negative NaN + std::string negative_input = "-" + input; + float negative_from_chars_float; + absl::from_chars(negative_input.data(), + negative_input.data() + negative_input.size(), + negative_from_chars_float); + EXPECT_TRUE(std::signbit(negative_from_chars_float)); + EXPECT_FALSE(Identical(negative_from_chars_float, from_chars_float)); + from_chars_float = std::copysign(from_chars_float, -1.0); + EXPECT_TRUE(Identical(negative_from_chars_float, from_chars_float)); + } +} + +// Returns an integer larger than step. The values grow exponentially. +int NextStep(int step) { + return step + (step >> 2) + 1; +} + +// Test a conversion on a family of input strings, checking that the calculation +// is correct for in-bounds values, and that overflow and underflow are done +// correctly for out-of-bounds values. +// +// input_generator maps from an integer index to a string to test. +// expected_generator maps from an integer index to an expected Float value. +// from_chars conversion of input_generator(i) should result in +// expected_generator(i). +// +// lower_bound and upper_bound denote the smallest and largest values for which +// the conversion is expected to succeed. +template <typename Float> +void TestOverflowAndUnderflow( + const std::function<std::string(int)>& input_generator, + const std::function<Float(int)>& expected_generator, int lower_bound, + int upper_bound) { + // test legal values near lower_bound + int index, step; + for (index = lower_bound, step = 1; index < upper_bound; + index += step, step = NextStep(step)) { + std::string input = input_generator(index); + SCOPED_TRACE(input); + Float expected = expected_generator(index); + Float actual; + auto result = + absl::from_chars(input.data(), input.data() + input.size(), actual); + EXPECT_EQ(result.ec, std::errc()); + EXPECT_EQ(expected, actual) + << absl::StrFormat("%a vs %a", expected, actual); + } + // test legal values near upper_bound + for (index = upper_bound, step = 1; index > lower_bound; + index -= step, step = NextStep(step)) { + std::string input = input_generator(index); + SCOPED_TRACE(input); + Float expected = expected_generator(index); + Float actual; + auto result = + absl::from_chars(input.data(), input.data() + input.size(), actual); + EXPECT_EQ(result.ec, std::errc()); + EXPECT_EQ(expected, actual) + << absl::StrFormat("%a vs %a", expected, actual); + } + // Test underflow values below lower_bound + for (index = lower_bound - 1, step = 1; index > -1000000; + index -= step, step = NextStep(step)) { + std::string input = input_generator(index); + SCOPED_TRACE(input); + Float actual; + auto result = + absl::from_chars(input.data(), input.data() + input.size(), actual); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_LT(actual, 1.0); // check for underflow + } + // Test overflow values above upper_bound + for (index = upper_bound + 1, step = 1; index < 1000000; + index += step, step = NextStep(step)) { + std::string input = input_generator(index); + SCOPED_TRACE(input); + Float actual; + auto result = + absl::from_chars(input.data(), input.data() + input.size(), actual); + EXPECT_EQ(result.ec, std::errc::result_out_of_range); + EXPECT_GT(actual, 1.0); // check for overflow + } +} + +// Check that overflow and underflow are caught correctly for hex doubles. +// +// The largest representable double is 0x1.fffffffffffffp+1023, and the +// smallest representable subnormal is 0x0.0000000000001p-1022, which equals +// 0x1p-1074. Therefore 1023 and -1074 are the limits of acceptable exponents +// in this test. +TEST(FromChars, HexdecimalDoubleLimits) { + auto input_gen = [](int index) { return absl::StrCat("0x1.0p", index); }; + auto expected_gen = [](int index) { return std::ldexp(1.0, index); }; + TestOverflowAndUnderflow<double>(input_gen, expected_gen, -1074, 1023); +} + +// Check that overflow and underflow are caught correctly for hex floats. +// +// The largest representable float is 0x1.fffffep+127, and the smallest +// representable subnormal is 0x0.000002p-126, which equals 0x1p-149. +// Therefore 127 and -149 are the limits of acceptable exponents in this test. +TEST(FromChars, HexdecimalFloatLimits) { + auto input_gen = [](int index) { return absl::StrCat("0x1.0p", index); }; + auto expected_gen = [](int index) { return std::ldexp(1.0f, index); }; + TestOverflowAndUnderflow<float>(input_gen, expected_gen, -149, 127); +} + +// Check that overflow and underflow are caught correctly for decimal doubles. +// +// The largest representable double is about 1.8e308, and the smallest +// representable subnormal is about 5e-324. '1e-324' therefore rounds away from +// the smallest representable positive value. -323 and 308 are the limits of +// acceptable exponents in this test. +TEST(FromChars, DecimalDoubleLimits) { + auto input_gen = [](int index) { return absl::StrCat("1.0e", index); }; + auto expected_gen = [](int index) { return Pow10(index); }; + TestOverflowAndUnderflow<double>(input_gen, expected_gen, -323, 308); +} + +// Check that overflow and underflow are caught correctly for decimal floats. +// +// The largest representable float is about 3.4e38, and the smallest +// representable subnormal is about 1.45e-45. '1e-45' therefore rounds towards +// the smallest representable positive value. -45 and 38 are the limits of +// acceptable exponents in this test. +TEST(FromChars, DecimalFloatLimits) { + auto input_gen = [](int index) { return absl::StrCat("1.0e", index); }; + auto expected_gen = [](int index) { return Pow10(index); }; + TestOverflowAndUnderflow<float>(input_gen, expected_gen, -45, 38); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/cord.cc b/third_party/abseil_cpp/absl/strings/cord.cc new file mode 100644 index 000000000000..9efd13575039 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/cord.cc @@ -0,0 +1,1995 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/cord.h" + +#include <algorithm> +#include <atomic> +#include <cstddef> +#include <cstdio> +#include <cstdlib> +#include <iomanip> +#include <iostream> +#include <limits> +#include <ostream> +#include <sstream> +#include <type_traits> +#include <unordered_set> +#include <vector> + +#include "absl/base/casts.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/container/fixed_array.h" +#include "absl/container/inlined_vector.h" +#include "absl/strings/escaping.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +using ::absl::cord_internal::CordRep; +using ::absl::cord_internal::CordRepConcat; +using ::absl::cord_internal::CordRepExternal; +using ::absl::cord_internal::CordRepSubstring; + +using ::absl::cord_internal::CONCAT; +using ::absl::cord_internal::EXTERNAL; +using ::absl::cord_internal::FLAT; +using ::absl::cord_internal::SUBSTRING; + +namespace cord_internal { + +inline CordRepConcat* CordRep::concat() { + assert(tag == CONCAT); + return static_cast<CordRepConcat*>(this); +} + +inline const CordRepConcat* CordRep::concat() const { + assert(tag == CONCAT); + return static_cast<const CordRepConcat*>(this); +} + +inline CordRepSubstring* CordRep::substring() { + assert(tag == SUBSTRING); + return static_cast<CordRepSubstring*>(this); +} + +inline const CordRepSubstring* CordRep::substring() const { + assert(tag == SUBSTRING); + return static_cast<const CordRepSubstring*>(this); +} + +inline CordRepExternal* CordRep::external() { + assert(tag == EXTERNAL); + return static_cast<CordRepExternal*>(this); +} + +inline const CordRepExternal* CordRep::external() const { + assert(tag == EXTERNAL); + return static_cast<const CordRepExternal*>(this); +} + +} // namespace cord_internal + +static const size_t kFlatOverhead = offsetof(CordRep, data); + +// Largest and smallest flat node lengths we are willing to allocate +// Flat allocation size is stored in tag, which currently can encode sizes up +// to 4K, encoded as multiple of either 8 or 32 bytes. +// If we allow for larger sizes, we need to change this to 8/64, 16/128, etc. +static constexpr size_t kMaxFlatSize = 4096; +static constexpr size_t kMaxFlatLength = kMaxFlatSize - kFlatOverhead; +static constexpr size_t kMinFlatLength = 32 - kFlatOverhead; + +// Prefer copying blocks of at most this size, otherwise reference count. +static const size_t kMaxBytesToCopy = 511; + +// Helper functions for rounded div, and rounding to exact sizes. +static size_t DivUp(size_t n, size_t m) { return (n + m - 1) / m; } +static size_t RoundUp(size_t n, size_t m) { return DivUp(n, m) * m; } + +// Returns the size to the nearest equal or larger value that can be +// expressed exactly as a tag value. +static size_t RoundUpForTag(size_t size) { + return RoundUp(size, (size <= 1024) ? 8 : 32); +} + +// Converts the allocated size to a tag, rounding down if the size +// does not exactly match a 'tag expressible' size value. The result is +// undefined if the size exceeds the maximum size that can be encoded in +// a tag, i.e., if size is larger than TagToAllocatedSize(<max tag>). +static uint8_t AllocatedSizeToTag(size_t size) { + const size_t tag = (size <= 1024) ? size / 8 : 128 + size / 32 - 1024 / 32; + assert(tag <= std::numeric_limits<uint8_t>::max()); + return tag; +} + +// Converts the provided tag to the corresponding allocated size +static constexpr size_t TagToAllocatedSize(uint8_t tag) { + return (tag <= 128) ? (tag * 8) : (1024 + (tag - 128) * 32); +} + +// Converts the provided tag to the corresponding available data length +static constexpr size_t TagToLength(uint8_t tag) { + return TagToAllocatedSize(tag) - kFlatOverhead; +} + +// Enforce that kMaxFlatSize maps to a well-known exact tag value. +static_assert(TagToAllocatedSize(224) == kMaxFlatSize, "Bad tag logic"); + +constexpr uint64_t Fibonacci(unsigned char n, uint64_t a = 0, uint64_t b = 1) { + return n == 0 ? a : Fibonacci(n - 1, b, a + b); +} + +static_assert(Fibonacci(63) == 6557470319842, + "Fibonacci values computed incorrectly"); + +// Minimum length required for a given depth tree -- a tree is considered +// balanced if +// length(t) >= min_length[depth(t)] +// The root node depth is allowed to become twice as large to reduce rebalancing +// for larger strings (see IsRootBalanced). +static constexpr uint64_t min_length[] = { + Fibonacci(2), Fibonacci(3), Fibonacci(4), Fibonacci(5), + Fibonacci(6), Fibonacci(7), Fibonacci(8), Fibonacci(9), + Fibonacci(10), Fibonacci(11), Fibonacci(12), Fibonacci(13), + Fibonacci(14), Fibonacci(15), Fibonacci(16), Fibonacci(17), + Fibonacci(18), Fibonacci(19), Fibonacci(20), Fibonacci(21), + Fibonacci(22), Fibonacci(23), Fibonacci(24), Fibonacci(25), + Fibonacci(26), Fibonacci(27), Fibonacci(28), Fibonacci(29), + Fibonacci(30), Fibonacci(31), Fibonacci(32), Fibonacci(33), + Fibonacci(34), Fibonacci(35), Fibonacci(36), Fibonacci(37), + Fibonacci(38), Fibonacci(39), Fibonacci(40), Fibonacci(41), + Fibonacci(42), Fibonacci(43), Fibonacci(44), Fibonacci(45), + Fibonacci(46), Fibonacci(47), + 0xffffffffffffffffull, // Avoid overflow +}; + +static const int kMinLengthSize = ABSL_ARRAYSIZE(min_length); + +// The inlined size to use with absl::InlinedVector. +// +// Note: The InlinedVectors in this file (and in cord.h) do not need to use +// the same value for their inlined size. The fact that they do is historical. +// It may be desirable for each to use a different inlined size optimized for +// that InlinedVector's usage. +// +// TODO(jgm): Benchmark to see if there's a more optimal value than 47 for +// the inlined vector size (47 exists for backward compatibility). +static const int kInlinedVectorSize = 47; + +static inline bool IsRootBalanced(CordRep* node) { + if (node->tag != CONCAT) { + return true; + } else if (node->concat()->depth() <= 15) { + return true; + } else if (node->concat()->depth() > kMinLengthSize) { + return false; + } else { + // Allow depth to become twice as large as implied by fibonacci rule to + // reduce rebalancing for larger strings. + return (node->length >= min_length[node->concat()->depth() / 2]); + } +} + +static CordRep* Rebalance(CordRep* node); +static void DumpNode(CordRep* rep, bool include_data, std::ostream* os); +static bool VerifyNode(CordRep* root, CordRep* start_node, + bool full_validation); + +static inline CordRep* VerifyTree(CordRep* node) { + // Verification is expensive, so only do it in debug mode. + // Even in debug mode we normally do only light validation. + // If you are debugging Cord itself, you should define the + // macro EXTRA_CORD_VALIDATION, e.g. by adding + // --copt=-DEXTRA_CORD_VALIDATION to the blaze line. +#ifdef EXTRA_CORD_VALIDATION + assert(node == nullptr || VerifyNode(node, node, /*full_validation=*/true)); +#else // EXTRA_CORD_VALIDATION + assert(node == nullptr || VerifyNode(node, node, /*full_validation=*/false)); +#endif // EXTRA_CORD_VALIDATION + static_cast<void>(&VerifyNode); + + return node; +} + +// -------------------------------------------------------------------- +// Memory management + +inline CordRep* Ref(CordRep* rep) { + if (rep != nullptr) { + rep->refcount.Increment(); + } + return rep; +} + +// This internal routine is called from the cold path of Unref below. Keeping it +// in a separate routine allows good inlining of Unref into many profitable call +// sites. However, the call to this function can be highly disruptive to the +// register pressure in those callers. To minimize the cost to callers, we use +// a special LLVM calling convention that preserves most registers. This allows +// the call to this routine in cold paths to not disrupt the caller's register +// pressure. This calling convention is not available on all platforms; we +// intentionally allow LLVM to ignore the attribute rather than attempting to +// hardcode the list of supported platforms. +#if defined(__clang__) && !defined(__i386__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wattributes" +__attribute__((preserve_most)) +#pragma clang diagnostic pop +#endif +static void UnrefInternal(CordRep* rep) { + assert(rep != nullptr); + + absl::InlinedVector<CordRep*, kInlinedVectorSize> pending; + while (true) { + assert(!rep->refcount.IsImmortal()); + if (rep->tag == CONCAT) { + CordRepConcat* rep_concat = rep->concat(); + CordRep* right = rep_concat->right; + if (!right->refcount.Decrement()) { + pending.push_back(right); + } + CordRep* left = rep_concat->left; + delete rep_concat; + rep = nullptr; + if (!left->refcount.Decrement()) { + rep = left; + continue; + } + } else if (rep->tag == EXTERNAL) { + CordRepExternal* rep_external = rep->external(); + assert(rep_external->releaser_invoker != nullptr); + rep_external->releaser_invoker(rep_external); + rep = nullptr; + } else if (rep->tag == SUBSTRING) { + CordRepSubstring* rep_substring = rep->substring(); + CordRep* child = rep_substring->child; + delete rep_substring; + rep = nullptr; + if (!child->refcount.Decrement()) { + rep = child; + continue; + } + } else { + // Flat CordReps are allocated and constructed with raw ::operator new + // and placement new, and must be destructed and deallocated + // accordingly. +#if defined(__cpp_sized_deallocation) + size_t size = TagToAllocatedSize(rep->tag); + rep->~CordRep(); + ::operator delete(rep, size); +#else + rep->~CordRep(); + ::operator delete(rep); +#endif + rep = nullptr; + } + + if (!pending.empty()) { + rep = pending.back(); + pending.pop_back(); + } else { + break; + } + } +} + +inline void Unref(CordRep* rep) { + // Fast-path for two common, hot cases: a null rep and a shared root. + if (ABSL_PREDICT_TRUE(rep == nullptr || + rep->refcount.DecrementExpectHighRefcount())) { + return; + } + + UnrefInternal(rep); +} + +// Return the depth of a node +static int Depth(const CordRep* rep) { + if (rep->tag == CONCAT) { + return rep->concat()->depth(); + } else { + return 0; + } +} + +static void SetConcatChildren(CordRepConcat* concat, CordRep* left, + CordRep* right) { + concat->left = left; + concat->right = right; + + concat->length = left->length + right->length; + concat->set_depth(1 + std::max(Depth(left), Depth(right))); +} + +// Create a concatenation of the specified nodes. +// Does not change the refcounts of "left" and "right". +// The returned node has a refcount of 1. +static CordRep* RawConcat(CordRep* left, CordRep* right) { + // Avoid making degenerate concat nodes (one child is empty) + if (left == nullptr || left->length == 0) { + Unref(left); + return right; + } + if (right == nullptr || right->length == 0) { + Unref(right); + return left; + } + + CordRepConcat* rep = new CordRepConcat(); + rep->tag = CONCAT; + SetConcatChildren(rep, left, right); + + return rep; +} + +static CordRep* Concat(CordRep* left, CordRep* right) { + CordRep* rep = RawConcat(left, right); + if (rep != nullptr && !IsRootBalanced(rep)) { + rep = Rebalance(rep); + } + return VerifyTree(rep); +} + +// Make a balanced tree out of an array of leaf nodes. +static CordRep* MakeBalancedTree(CordRep** reps, size_t n) { + // Make repeated passes over the array, merging adjacent pairs + // until we are left with just a single node. + while (n > 1) { + size_t dst = 0; + for (size_t src = 0; src < n; src += 2) { + if (src + 1 < n) { + reps[dst] = Concat(reps[src], reps[src + 1]); + } else { + reps[dst] = reps[src]; + } + dst++; + } + n = dst; + } + + return reps[0]; +} + +// Create a new flat node. +static CordRep* NewFlat(size_t length_hint) { + if (length_hint <= kMinFlatLength) { + length_hint = kMinFlatLength; + } else if (length_hint > kMaxFlatLength) { + length_hint = kMaxFlatLength; + } + + // Round size up so it matches a size we can exactly express in a tag. + const size_t size = RoundUpForTag(length_hint + kFlatOverhead); + void* const raw_rep = ::operator new(size); + CordRep* rep = new (raw_rep) CordRep(); + rep->tag = AllocatedSizeToTag(size); + return VerifyTree(rep); +} + +// Create a new tree out of the specified array. +// The returned node has a refcount of 1. +static CordRep* NewTree(const char* data, + size_t length, + size_t alloc_hint) { + if (length == 0) return nullptr; + absl::FixedArray<CordRep*> reps((length - 1) / kMaxFlatLength + 1); + size_t n = 0; + do { + const size_t len = std::min(length, kMaxFlatLength); + CordRep* rep = NewFlat(len + alloc_hint); + rep->length = len; + memcpy(rep->data, data, len); + reps[n++] = VerifyTree(rep); + data += len; + length -= len; + } while (length != 0); + return MakeBalancedTree(reps.data(), n); +} + +namespace cord_internal { + +void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep) { + assert(!data.empty()); + rep->length = data.size(); + rep->tag = EXTERNAL; + rep->base = data.data(); + VerifyTree(rep); +} + +} // namespace cord_internal + +static CordRep* NewSubstring(CordRep* child, size_t offset, size_t length) { + // Never create empty substring nodes + if (length == 0) { + Unref(child); + return nullptr; + } else { + CordRepSubstring* rep = new CordRepSubstring(); + assert((offset + length) <= child->length); + rep->length = length; + rep->tag = SUBSTRING; + rep->start = offset; + rep->child = child; + return VerifyTree(rep); + } +} + +// -------------------------------------------------------------------- +// Cord::InlineRep functions + +constexpr unsigned char Cord::InlineRep::kMaxInline; + +inline void Cord::InlineRep::set_data(const char* data, size_t n, + bool nullify_tail) { + static_assert(kMaxInline == 15, "set_data is hard-coded for a length of 15"); + + cord_internal::SmallMemmove(data_.as_chars, data, n, nullify_tail); + set_tagged_size(static_cast<char>(n)); +} + +inline char* Cord::InlineRep::set_data(size_t n) { + assert(n <= kMaxInline); + ResetToEmpty(); + set_tagged_size(static_cast<char>(n)); + return data_.as_chars; +} + +inline CordRep* Cord::InlineRep::force_tree(size_t extra_hint) { + size_t len = tagged_size(); + if (len > kMaxInline) { + return data_.as_tree.rep; + } + + CordRep* result = NewFlat(len + extra_hint); + result->length = len; + static_assert(kMinFlatLength >= sizeof(data_.as_chars), ""); + memcpy(result->data, data_.as_chars, sizeof(data_.as_chars)); + set_tree(result); + return result; +} + +inline void Cord::InlineRep::reduce_size(size_t n) { + size_t tag = tagged_size(); + assert(tag <= kMaxInline); + assert(tag >= n); + tag -= n; + memset(data_.as_chars + tag, 0, n); + set_tagged_size(static_cast<char>(tag)); +} + +inline void Cord::InlineRep::remove_prefix(size_t n) { + cord_internal::SmallMemmove(data_.as_chars, data_.as_chars + n, + tagged_size() - n); + reduce_size(n); +} + +void Cord::InlineRep::AppendTree(CordRep* tree) { + if (tree == nullptr) return; + size_t len = tagged_size(); + if (len == 0) { + set_tree(tree); + } else { + set_tree(Concat(force_tree(0), tree)); + } +} + +void Cord::InlineRep::PrependTree(CordRep* tree) { + assert(tree != nullptr); + size_t len = tagged_size(); + if (len == 0) { + set_tree(tree); + } else { + set_tree(Concat(tree, force_tree(0))); + } +} + +// Searches for a non-full flat node at the rightmost leaf of the tree. If a +// suitable leaf is found, the function will update the length field for all +// nodes to account for the size increase. The append region address will be +// written to region and the actual size increase will be written to size. +static inline bool PrepareAppendRegion(CordRep* root, char** region, + size_t* size, size_t max_length) { + // Search down the right-hand path for a non-full FLAT node. + CordRep* dst = root; + while (dst->tag == CONCAT && dst->refcount.IsOne()) { + dst = dst->concat()->right; + } + + if (dst->tag < FLAT || !dst->refcount.IsOne()) { + *region = nullptr; + *size = 0; + return false; + } + + const size_t in_use = dst->length; + const size_t capacity = TagToLength(dst->tag); + if (in_use == capacity) { + *region = nullptr; + *size = 0; + return false; + } + + size_t size_increase = std::min(capacity - in_use, max_length); + + // We need to update the length fields for all nodes, including the leaf node. + for (CordRep* rep = root; rep != dst; rep = rep->concat()->right) { + rep->length += size_increase; + } + dst->length += size_increase; + + *region = dst->data + in_use; + *size = size_increase; + return true; +} + +void Cord::InlineRep::GetAppendRegion(char** region, size_t* size, + size_t max_length) { + if (max_length == 0) { + *region = nullptr; + *size = 0; + return; + } + + // Try to fit in the inline buffer if possible. + size_t inline_length = tagged_size(); + if (inline_length < kMaxInline && max_length <= kMaxInline - inline_length) { + *region = data_.as_chars + inline_length; + *size = max_length; + set_tagged_size(static_cast<char>(inline_length + max_length)); + return; + } + + CordRep* root = force_tree(max_length); + + if (PrepareAppendRegion(root, region, size, max_length)) { + return; + } + + // Allocate new node. + CordRep* new_node = + NewFlat(std::max(static_cast<size_t>(root->length), max_length)); + new_node->length = + std::min(static_cast<size_t>(TagToLength(new_node->tag)), max_length); + *region = new_node->data; + *size = new_node->length; + replace_tree(Concat(root, new_node)); +} + +void Cord::InlineRep::GetAppendRegion(char** region, size_t* size) { + const size_t max_length = std::numeric_limits<size_t>::max(); + + // Try to fit in the inline buffer if possible. + size_t inline_length = tagged_size(); + if (inline_length < kMaxInline) { + *region = data_.as_chars + inline_length; + *size = kMaxInline - inline_length; + set_tagged_size(kMaxInline); + return; + } + + CordRep* root = force_tree(max_length); + + if (PrepareAppendRegion(root, region, size, max_length)) { + return; + } + + // Allocate new node. + CordRep* new_node = NewFlat(root->length); + new_node->length = TagToLength(new_node->tag); + *region = new_node->data; + *size = new_node->length; + replace_tree(Concat(root, new_node)); +} + +// If the rep is a leaf, this will increment the value at total_mem_usage and +// will return true. +static bool RepMemoryUsageLeaf(const CordRep* rep, size_t* total_mem_usage) { + if (rep->tag >= FLAT) { + *total_mem_usage += TagToAllocatedSize(rep->tag); + return true; + } + if (rep->tag == EXTERNAL) { + *total_mem_usage += sizeof(CordRepConcat) + rep->length; + return true; + } + return false; +} + +void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { + ClearSlow(); + + data_ = src.data_; + if (is_tree()) { + Ref(tree()); + } +} + +void Cord::InlineRep::ClearSlow() { + if (is_tree()) { + Unref(tree()); + } + ResetToEmpty(); +} + +// -------------------------------------------------------------------- +// Constructors and destructors + +Cord::Cord(const Cord& src) : contents_(src.contents_) { + Ref(contents_.tree()); // Does nothing if contents_ has embedded data +} + +Cord::Cord(absl::string_view src) { + const size_t n = src.size(); + if (n <= InlineRep::kMaxInline) { + contents_.set_data(src.data(), n, false); + } else { + contents_.set_tree(NewTree(src.data(), n, 0)); + } +} + +template <typename T, Cord::EnableIfString<T>> +Cord::Cord(T&& src) { + if ( + // String is short: copy data to avoid external block overhead. + src.size() <= kMaxBytesToCopy || + // String is wasteful: copy data to avoid pinning too much unused memory. + src.size() < src.capacity() / 2 + ) { + if (src.size() <= InlineRep::kMaxInline) { + contents_.set_data(src.data(), src.size(), false); + } else { + contents_.set_tree(NewTree(src.data(), src.size(), 0)); + } + } else { + struct StringReleaser { + void operator()(absl::string_view /* data */) {} + std::string data; + }; + const absl::string_view original_data = src; + auto* rep = static_cast< + ::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>( + absl::cord_internal::NewExternalRep( + original_data, StringReleaser{std::forward<T>(src)})); + // Moving src may have invalidated its data pointer, so adjust it. + rep->base = rep->template get<0>().data.data(); + contents_.set_tree(rep); + } +} + +template Cord::Cord(std::string&& src); + +// The destruction code is separate so that the compiler can determine +// that it does not need to call the destructor on a moved-from Cord. +void Cord::DestroyCordSlow() { + Unref(VerifyTree(contents_.tree())); +} + +// -------------------------------------------------------------------- +// Mutators + +void Cord::Clear() { + Unref(contents_.clear()); +} + +Cord& Cord::operator=(absl::string_view src) { + + const char* data = src.data(); + size_t length = src.size(); + CordRep* tree = contents_.tree(); + if (length <= InlineRep::kMaxInline) { + // Embed into this->contents_ + contents_.set_data(data, length, true); + Unref(tree); + return *this; + } + if (tree != nullptr && tree->tag >= FLAT && + TagToLength(tree->tag) >= length && tree->refcount.IsOne()) { + // Copy in place if the existing FLAT node is reusable. + memmove(tree->data, data, length); + tree->length = length; + VerifyTree(tree); + return *this; + } + contents_.set_tree(NewTree(data, length, 0)); + Unref(tree); + return *this; +} + +template <typename T, Cord::EnableIfString<T>> +Cord& Cord::operator=(T&& src) { + if (src.size() <= kMaxBytesToCopy) { + *this = absl::string_view(src); + } else { + *this = Cord(std::forward<T>(src)); + } + return *this; +} + +template Cord& Cord::operator=(std::string&& src); + +// TODO(sanjay): Move to Cord::InlineRep section of file. For now, +// we keep it here to make diffs easier. +void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) { + if (src_size == 0) return; // memcpy(_, nullptr, 0) is undefined. + // Try to fit in the inline buffer if possible. + size_t inline_length = tagged_size(); + if (inline_length < kMaxInline && src_size <= kMaxInline - inline_length) { + // Append new data to embedded array + set_tagged_size(static_cast<char>(inline_length + src_size)); + memcpy(data_.as_chars + inline_length, src_data, src_size); + return; + } + + CordRep* root = tree(); + + size_t appended = 0; + if (root) { + char* region; + if (PrepareAppendRegion(root, ®ion, &appended, src_size)) { + memcpy(region, src_data, appended); + } + } else { + // It is possible that src_data == data_, but when we transition from an + // InlineRep to a tree we need to assign data_ = root via set_tree. To + // avoid corrupting the source data before we copy it, delay calling + // set_tree until after we've copied data. + // We are going from an inline size to beyond inline size. Make the new size + // either double the inlined size, or the added size + 10%. + const size_t size1 = inline_length * 2 + src_size; + const size_t size2 = inline_length + src_size / 10; + root = NewFlat(std::max<size_t>(size1, size2)); + appended = std::min(src_size, TagToLength(root->tag) - inline_length); + memcpy(root->data, data_.as_chars, inline_length); + memcpy(root->data + inline_length, src_data, appended); + root->length = inline_length + appended; + set_tree(root); + } + + src_data += appended; + src_size -= appended; + if (src_size == 0) { + return; + } + + // Use new block(s) for any remaining bytes that were not handled above. + // Alloc extra memory only if the right child of the root of the new tree is + // going to be a FLAT node, which will permit further inplace appends. + size_t length = src_size; + if (src_size < kMaxFlatLength) { + // The new length is either + // - old size + 10% + // - old_size + src_size + // This will cause a reasonable conservative step-up in size that is still + // large enough to avoid excessive amounts of small fragments being added. + length = std::max<size_t>(root->length / 10, src_size); + } + set_tree(Concat(root, NewTree(src_data, src_size, length - src_size))); +} + +inline CordRep* Cord::TakeRep() const& { + return Ref(contents_.tree()); +} + +inline CordRep* Cord::TakeRep() && { + CordRep* rep = contents_.tree(); + contents_.clear(); + return rep; +} + +template <typename C> +inline void Cord::AppendImpl(C&& src) { + if (empty()) { + // In case of an empty destination avoid allocating a new node, do not copy + // data. + *this = std::forward<C>(src); + return; + } + + // For short cords, it is faster to copy data if there is room in dst. + const size_t src_size = src.contents_.size(); + if (src_size <= kMaxBytesToCopy) { + CordRep* src_tree = src.contents_.tree(); + if (src_tree == nullptr) { + // src has embedded data. + contents_.AppendArray(src.contents_.data(), src_size); + return; + } + if (src_tree->tag >= FLAT) { + // src tree just has one flat node. + contents_.AppendArray(src_tree->data, src_size); + return; + } + if (&src == this) { + // ChunkIterator below assumes that src is not modified during traversal. + Append(Cord(src)); + return; + } + // TODO(mec): Should we only do this if "dst" has space? + for (absl::string_view chunk : src.Chunks()) { + Append(chunk); + } + return; + } + + contents_.AppendTree(std::forward<C>(src).TakeRep()); +} + +void Cord::Append(const Cord& src) { AppendImpl(src); } + +void Cord::Append(Cord&& src) { AppendImpl(std::move(src)); } + +template <typename T, Cord::EnableIfString<T>> +void Cord::Append(T&& src) { + if (src.size() <= kMaxBytesToCopy) { + Append(absl::string_view(src)); + } else { + Append(Cord(std::forward<T>(src))); + } +} + +template void Cord::Append(std::string&& src); + +void Cord::Prepend(const Cord& src) { + CordRep* src_tree = src.contents_.tree(); + if (src_tree != nullptr) { + Ref(src_tree); + contents_.PrependTree(src_tree); + return; + } + + // `src` cord is inlined. + absl::string_view src_contents(src.contents_.data(), src.contents_.size()); + return Prepend(src_contents); +} + +void Cord::Prepend(absl::string_view src) { + if (src.empty()) return; // memcpy(_, nullptr, 0) is undefined. + size_t cur_size = contents_.size(); + if (!contents_.is_tree() && cur_size + src.size() <= InlineRep::kMaxInline) { + // Use embedded storage. + char data[InlineRep::kMaxInline + 1] = {0}; + data[InlineRep::kMaxInline] = cur_size + src.size(); // set size + memcpy(data, src.data(), src.size()); + memcpy(data + src.size(), contents_.data(), cur_size); + memcpy(reinterpret_cast<void*>(&contents_), data, + InlineRep::kMaxInline + 1); + } else { + contents_.PrependTree(NewTree(src.data(), src.size(), 0)); + } +} + +template <typename T, Cord::EnableIfString<T>> +inline void Cord::Prepend(T&& src) { + if (src.size() <= kMaxBytesToCopy) { + Prepend(absl::string_view(src)); + } else { + Prepend(Cord(std::forward<T>(src))); + } +} + +template void Cord::Prepend(std::string&& src); + +static CordRep* RemovePrefixFrom(CordRep* node, size_t n) { + if (n >= node->length) return nullptr; + if (n == 0) return Ref(node); + absl::InlinedVector<CordRep*, kInlinedVectorSize> rhs_stack; + + while (node->tag == CONCAT) { + assert(n <= node->length); + if (n < node->concat()->left->length) { + // Push right to stack, descend left. + rhs_stack.push_back(node->concat()->right); + node = node->concat()->left; + } else { + // Drop left, descend right. + n -= node->concat()->left->length; + node = node->concat()->right; + } + } + assert(n <= node->length); + + if (n == 0) { + Ref(node); + } else { + size_t start = n; + size_t len = node->length - n; + if (node->tag == SUBSTRING) { + // Consider in-place update of node, similar to in RemoveSuffixFrom(). + start += node->substring()->start; + node = node->substring()->child; + } + node = NewSubstring(Ref(node), start, len); + } + while (!rhs_stack.empty()) { + node = Concat(node, Ref(rhs_stack.back())); + rhs_stack.pop_back(); + } + return node; +} + +// RemoveSuffixFrom() is very similar to RemovePrefixFrom(), with the +// exception that removing a suffix has an optimization where a node may be +// edited in place iff that node and all its ancestors have a refcount of 1. +static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) { + if (n >= node->length) return nullptr; + if (n == 0) return Ref(node); + absl::InlinedVector<CordRep*, kInlinedVectorSize> lhs_stack; + bool inplace_ok = node->refcount.IsOne(); + + while (node->tag == CONCAT) { + assert(n <= node->length); + if (n < node->concat()->right->length) { + // Push left to stack, descend right. + lhs_stack.push_back(node->concat()->left); + node = node->concat()->right; + } else { + // Drop right, descend left. + n -= node->concat()->right->length; + node = node->concat()->left; + } + inplace_ok = inplace_ok && node->refcount.IsOne(); + } + assert(n <= node->length); + + if (n == 0) { + Ref(node); + } else if (inplace_ok && node->tag != EXTERNAL) { + // Consider making a new buffer if the current node capacity is much + // larger than the new length. + Ref(node); + node->length -= n; + } else { + size_t start = 0; + size_t len = node->length - n; + if (node->tag == SUBSTRING) { + start = node->substring()->start; + node = node->substring()->child; + } + node = NewSubstring(Ref(node), start, len); + } + while (!lhs_stack.empty()) { + node = Concat(Ref(lhs_stack.back()), node); + lhs_stack.pop_back(); + } + return node; +} + +void Cord::RemovePrefix(size_t n) { + ABSL_INTERNAL_CHECK(n <= size(), + absl::StrCat("Requested prefix size ", n, + " exceeds Cord's size ", size())); + CordRep* tree = contents_.tree(); + if (tree == nullptr) { + contents_.remove_prefix(n); + } else { + CordRep* newrep = RemovePrefixFrom(tree, n); + Unref(tree); + contents_.replace_tree(VerifyTree(newrep)); + } +} + +void Cord::RemoveSuffix(size_t n) { + ABSL_INTERNAL_CHECK(n <= size(), + absl::StrCat("Requested suffix size ", n, + " exceeds Cord's size ", size())); + CordRep* tree = contents_.tree(); + if (tree == nullptr) { + contents_.reduce_size(n); + } else { + CordRep* newrep = RemoveSuffixFrom(tree, n); + Unref(tree); + contents_.replace_tree(VerifyTree(newrep)); + } +} + +// Work item for NewSubRange(). +struct SubRange { + SubRange(CordRep* a_node, size_t a_pos, size_t a_n) + : node(a_node), pos(a_pos), n(a_n) {} + CordRep* node; // nullptr means concat last 2 results. + size_t pos; + size_t n; +}; + +static CordRep* NewSubRange(CordRep* node, size_t pos, size_t n) { + absl::InlinedVector<CordRep*, kInlinedVectorSize> results; + absl::InlinedVector<SubRange, kInlinedVectorSize> todo; + todo.push_back(SubRange(node, pos, n)); + do { + const SubRange& sr = todo.back(); + node = sr.node; + pos = sr.pos; + n = sr.n; + todo.pop_back(); + + if (node == nullptr) { + assert(results.size() >= 2); + CordRep* right = results.back(); + results.pop_back(); + CordRep* left = results.back(); + results.pop_back(); + results.push_back(Concat(left, right)); + } else if (pos == 0 && n == node->length) { + results.push_back(Ref(node)); + } else if (node->tag != CONCAT) { + if (node->tag == SUBSTRING) { + pos += node->substring()->start; + node = node->substring()->child; + } + results.push_back(NewSubstring(Ref(node), pos, n)); + } else if (pos + n <= node->concat()->left->length) { + todo.push_back(SubRange(node->concat()->left, pos, n)); + } else if (pos >= node->concat()->left->length) { + pos -= node->concat()->left->length; + todo.push_back(SubRange(node->concat()->right, pos, n)); + } else { + size_t left_n = node->concat()->left->length - pos; + todo.push_back(SubRange(nullptr, 0, 0)); // Concat() + todo.push_back(SubRange(node->concat()->right, 0, n - left_n)); + todo.push_back(SubRange(node->concat()->left, pos, left_n)); + } + } while (!todo.empty()); + assert(results.size() == 1); + return results[0]; +} + +Cord Cord::Subcord(size_t pos, size_t new_size) const { + Cord sub_cord; + size_t length = size(); + if (pos > length) pos = length; + if (new_size > length - pos) new_size = length - pos; + CordRep* tree = contents_.tree(); + if (tree == nullptr) { + // sub_cord is newly constructed, no need to re-zero-out the tail of + // contents_ memory. + sub_cord.contents_.set_data(contents_.data() + pos, new_size, false); + } else if (new_size == 0) { + // We want to return empty subcord, so nothing to do. + } else if (new_size <= InlineRep::kMaxInline) { + Cord::ChunkIterator it = chunk_begin(); + it.AdvanceBytes(pos); + char* dest = sub_cord.contents_.data_.as_chars; + size_t remaining_size = new_size; + while (remaining_size > it->size()) { + cord_internal::SmallMemmove(dest, it->data(), it->size()); + remaining_size -= it->size(); + dest += it->size(); + ++it; + } + cord_internal::SmallMemmove(dest, it->data(), remaining_size); + sub_cord.contents_.set_tagged_size(new_size); + } else { + sub_cord.contents_.set_tree(NewSubRange(tree, pos, new_size)); + } + return sub_cord; +} + +// -------------------------------------------------------------------- +// Balancing + +class CordForest { + public: + explicit CordForest(size_t length) + : root_length_(length), trees_(kMinLengthSize, nullptr) {} + + void Build(CordRep* cord_root) { + std::vector<CordRep*> pending = {cord_root}; + + while (!pending.empty()) { + CordRep* node = pending.back(); + pending.pop_back(); + CheckNode(node); + if (ABSL_PREDICT_FALSE(node->tag != CONCAT)) { + AddNode(node); + continue; + } + + CordRepConcat* concat_node = node->concat(); + if (concat_node->depth() >= kMinLengthSize || + concat_node->length < min_length[concat_node->depth()]) { + pending.push_back(concat_node->right); + pending.push_back(concat_node->left); + + if (concat_node->refcount.IsOne()) { + concat_node->left = concat_freelist_; + concat_freelist_ = concat_node; + } else { + Ref(concat_node->right); + Ref(concat_node->left); + Unref(concat_node); + } + } else { + AddNode(node); + } + } + } + + CordRep* ConcatNodes() { + CordRep* sum = nullptr; + for (auto* node : trees_) { + if (node == nullptr) continue; + + sum = PrependNode(node, sum); + root_length_ -= node->length; + if (root_length_ == 0) break; + } + ABSL_INTERNAL_CHECK(sum != nullptr, "Failed to locate sum node"); + return VerifyTree(sum); + } + + private: + CordRep* AppendNode(CordRep* node, CordRep* sum) { + return (sum == nullptr) ? node : MakeConcat(sum, node); + } + + CordRep* PrependNode(CordRep* node, CordRep* sum) { + return (sum == nullptr) ? node : MakeConcat(node, sum); + } + + void AddNode(CordRep* node) { + CordRep* sum = nullptr; + + // Collect together everything with which we will merge with node + int i = 0; + for (; node->length > min_length[i + 1]; ++i) { + auto& tree_at_i = trees_[i]; + + if (tree_at_i == nullptr) continue; + sum = PrependNode(tree_at_i, sum); + tree_at_i = nullptr; + } + + sum = AppendNode(node, sum); + + // Insert sum into appropriate place in the forest + for (; sum->length >= min_length[i]; ++i) { + auto& tree_at_i = trees_[i]; + if (tree_at_i == nullptr) continue; + + sum = MakeConcat(tree_at_i, sum); + tree_at_i = nullptr; + } + + // min_length[0] == 1, which means sum->length >= min_length[0] + assert(i > 0); + trees_[i - 1] = sum; + } + + // Make concat node trying to resue existing CordRepConcat nodes we + // already collected in the concat_freelist_. + CordRep* MakeConcat(CordRep* left, CordRep* right) { + if (concat_freelist_ == nullptr) return RawConcat(left, right); + + CordRepConcat* rep = concat_freelist_; + if (concat_freelist_->left == nullptr) { + concat_freelist_ = nullptr; + } else { + concat_freelist_ = concat_freelist_->left->concat(); + } + SetConcatChildren(rep, left, right); + + return rep; + } + + static void CheckNode(CordRep* node) { + ABSL_INTERNAL_CHECK(node->length != 0u, ""); + if (node->tag == CONCAT) { + ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, ""); + ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, ""); + ABSL_INTERNAL_CHECK(node->length == (node->concat()->left->length + + node->concat()->right->length), + ""); + } + } + + size_t root_length_; + + // use an inlined vector instead of a flat array to get bounds checking + absl::InlinedVector<CordRep*, kInlinedVectorSize> trees_; + + // List of concat nodes we can re-use for Cord balancing. + CordRepConcat* concat_freelist_ = nullptr; +}; + +static CordRep* Rebalance(CordRep* node) { + VerifyTree(node); + assert(node->tag == CONCAT); + + if (node->length == 0) { + return nullptr; + } + + CordForest forest(node->length); + forest.Build(node); + return forest.ConcatNodes(); +} + +// -------------------------------------------------------------------- +// Comparators + +namespace { + +int ClampResult(int memcmp_res) { + return static_cast<int>(memcmp_res > 0) - static_cast<int>(memcmp_res < 0); +} + +int CompareChunks(absl::string_view* lhs, absl::string_view* rhs, + size_t* size_to_compare) { + size_t compared_size = std::min(lhs->size(), rhs->size()); + assert(*size_to_compare >= compared_size); + *size_to_compare -= compared_size; + + int memcmp_res = ::memcmp(lhs->data(), rhs->data(), compared_size); + if (memcmp_res != 0) return memcmp_res; + + lhs->remove_prefix(compared_size); + rhs->remove_prefix(compared_size); + + return 0; +} + +// This overload set computes comparison results from memcmp result. This +// interface is used inside GenericCompare below. Differet implementations +// are specialized for int and bool. For int we clamp result to {-1, 0, 1} +// set. For bool we just interested in "value == 0". +template <typename ResultType> +ResultType ComputeCompareResult(int memcmp_res) { + return ClampResult(memcmp_res); +} +template <> +bool ComputeCompareResult<bool>(int memcmp_res) { + return memcmp_res == 0; +} + +} // namespace + +// Helper routine. Locates the first flat chunk of the Cord without +// initializing the iterator. +inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const { + size_t n = tagged_size(); + if (n <= kMaxInline) { + return absl::string_view(data_.as_chars, n); + } + + CordRep* node = tree(); + if (node->tag >= FLAT) { + return absl::string_view(node->data, node->length); + } + + if (node->tag == EXTERNAL) { + return absl::string_view(node->external()->base, node->length); + } + + // Walk down the left branches until we hit a non-CONCAT node. + while (node->tag == CONCAT) { + node = node->concat()->left; + } + + // Get the child node if we encounter a SUBSTRING. + size_t offset = 0; + size_t length = node->length; + assert(length != 0); + + if (node->tag == SUBSTRING) { + offset = node->substring()->start; + node = node->substring()->child; + } + + if (node->tag >= FLAT) { + return absl::string_view(node->data + offset, length); + } + + assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here"); + + return absl::string_view(node->external()->base + offset, length); +} + +inline int Cord::CompareSlowPath(absl::string_view rhs, size_t compared_size, + size_t size_to_compare) const { + auto advance = [](Cord::ChunkIterator* it, absl::string_view* chunk) { + if (!chunk->empty()) return true; + ++*it; + if (it->bytes_remaining_ == 0) return false; + *chunk = **it; + return true; + }; + + Cord::ChunkIterator lhs_it = chunk_begin(); + + // compared_size is inside first chunk. + absl::string_view lhs_chunk = + (lhs_it.bytes_remaining_ != 0) ? *lhs_it : absl::string_view(); + assert(compared_size <= lhs_chunk.size()); + assert(compared_size <= rhs.size()); + lhs_chunk.remove_prefix(compared_size); + rhs.remove_prefix(compared_size); + size_to_compare -= compared_size; // skip already compared size. + + while (advance(&lhs_it, &lhs_chunk) && !rhs.empty()) { + int comparison_result = CompareChunks(&lhs_chunk, &rhs, &size_to_compare); + if (comparison_result != 0) return comparison_result; + if (size_to_compare == 0) return 0; + } + + return static_cast<int>(rhs.empty()) - static_cast<int>(lhs_chunk.empty()); +} + +inline int Cord::CompareSlowPath(const Cord& rhs, size_t compared_size, + size_t size_to_compare) const { + auto advance = [](Cord::ChunkIterator* it, absl::string_view* chunk) { + if (!chunk->empty()) return true; + ++*it; + if (it->bytes_remaining_ == 0) return false; + *chunk = **it; + return true; + }; + + Cord::ChunkIterator lhs_it = chunk_begin(); + Cord::ChunkIterator rhs_it = rhs.chunk_begin(); + + // compared_size is inside both first chunks. + absl::string_view lhs_chunk = + (lhs_it.bytes_remaining_ != 0) ? *lhs_it : absl::string_view(); + absl::string_view rhs_chunk = + (rhs_it.bytes_remaining_ != 0) ? *rhs_it : absl::string_view(); + assert(compared_size <= lhs_chunk.size()); + assert(compared_size <= rhs_chunk.size()); + lhs_chunk.remove_prefix(compared_size); + rhs_chunk.remove_prefix(compared_size); + size_to_compare -= compared_size; // skip already compared size. + + while (advance(&lhs_it, &lhs_chunk) && advance(&rhs_it, &rhs_chunk)) { + int memcmp_res = CompareChunks(&lhs_chunk, &rhs_chunk, &size_to_compare); + if (memcmp_res != 0) return memcmp_res; + if (size_to_compare == 0) return 0; + } + + return static_cast<int>(rhs_chunk.empty()) - + static_cast<int>(lhs_chunk.empty()); +} + +inline absl::string_view Cord::GetFirstChunk(const Cord& c) { + return c.contents_.FindFlatStartPiece(); +} +inline absl::string_view Cord::GetFirstChunk(absl::string_view sv) { + return sv; +} + +// Compares up to 'size_to_compare' bytes of 'lhs' with 'rhs'. It is assumed +// that 'size_to_compare' is greater that size of smallest of first chunks. +template <typename ResultType, typename RHS> +ResultType GenericCompare(const Cord& lhs, const RHS& rhs, + size_t size_to_compare) { + absl::string_view lhs_chunk = Cord::GetFirstChunk(lhs); + absl::string_view rhs_chunk = Cord::GetFirstChunk(rhs); + + size_t compared_size = std::min(lhs_chunk.size(), rhs_chunk.size()); + assert(size_to_compare >= compared_size); + int memcmp_res = ::memcmp(lhs_chunk.data(), rhs_chunk.data(), compared_size); + if (compared_size == size_to_compare || memcmp_res != 0) { + return ComputeCompareResult<ResultType>(memcmp_res); + } + + return ComputeCompareResult<ResultType>( + lhs.CompareSlowPath(rhs, compared_size, size_to_compare)); +} + +bool Cord::EqualsImpl(absl::string_view rhs, size_t size_to_compare) const { + return GenericCompare<bool>(*this, rhs, size_to_compare); +} + +bool Cord::EqualsImpl(const Cord& rhs, size_t size_to_compare) const { + return GenericCompare<bool>(*this, rhs, size_to_compare); +} + +template <typename RHS> +inline int SharedCompareImpl(const Cord& lhs, const RHS& rhs) { + size_t lhs_size = lhs.size(); + size_t rhs_size = rhs.size(); + if (lhs_size == rhs_size) { + return GenericCompare<int>(lhs, rhs, lhs_size); + } + if (lhs_size < rhs_size) { + auto data_comp_res = GenericCompare<int>(lhs, rhs, lhs_size); + return data_comp_res == 0 ? -1 : data_comp_res; + } + + auto data_comp_res = GenericCompare<int>(lhs, rhs, rhs_size); + return data_comp_res == 0 ? +1 : data_comp_res; +} + +int Cord::Compare(absl::string_view rhs) const { + return SharedCompareImpl(*this, rhs); +} + +int Cord::CompareImpl(const Cord& rhs) const { + return SharedCompareImpl(*this, rhs); +} + +bool Cord::EndsWith(absl::string_view rhs) const { + size_t my_size = size(); + size_t rhs_size = rhs.size(); + + if (my_size < rhs_size) return false; + + Cord tmp(*this); + tmp.RemovePrefix(my_size - rhs_size); + return tmp.EqualsImpl(rhs, rhs_size); +} + +bool Cord::EndsWith(const Cord& rhs) const { + size_t my_size = size(); + size_t rhs_size = rhs.size(); + + if (my_size < rhs_size) return false; + + Cord tmp(*this); + tmp.RemovePrefix(my_size - rhs_size); + return tmp.EqualsImpl(rhs, rhs_size); +} + +// -------------------------------------------------------------------- +// Misc. + +Cord::operator std::string() const { + std::string s; + absl::CopyCordToString(*this, &s); + return s; +} + +void CopyCordToString(const Cord& src, std::string* dst) { + if (!src.contents_.is_tree()) { + src.contents_.CopyTo(dst); + } else { + absl::strings_internal::STLStringResizeUninitialized(dst, src.size()); + src.CopyToArraySlowPath(&(*dst)[0]); + } +} + +void Cord::CopyToArraySlowPath(char* dst) const { + assert(contents_.is_tree()); + absl::string_view fragment; + if (GetFlatAux(contents_.tree(), &fragment)) { + memcpy(dst, fragment.data(), fragment.size()); + return; + } + for (absl::string_view chunk : Chunks()) { + memcpy(dst, chunk.data(), chunk.size()); + dst += chunk.size(); + } +} + +Cord::ChunkIterator& Cord::ChunkIterator::operator++() { + ABSL_HARDENING_ASSERT(bytes_remaining_ > 0 && + "Attempted to iterate past `end()`"); + assert(bytes_remaining_ >= current_chunk_.size()); + bytes_remaining_ -= current_chunk_.size(); + + if (stack_of_right_children_.empty()) { + assert(!current_chunk_.empty()); // Called on invalid iterator. + // We have reached the end of the Cord. + return *this; + } + + // Process the next node on the stack. + CordRep* node = stack_of_right_children_.back(); + stack_of_right_children_.pop_back(); + + // Walk down the left branches until we hit a non-CONCAT node. Save the + // right children to the stack for subsequent traversal. + while (node->tag == CONCAT) { + stack_of_right_children_.push_back(node->concat()->right); + node = node->concat()->left; + } + + // Get the child node if we encounter a SUBSTRING. + size_t offset = 0; + size_t length = node->length; + if (node->tag == SUBSTRING) { + offset = node->substring()->start; + node = node->substring()->child; + } + + assert(node->tag == EXTERNAL || node->tag >= FLAT); + assert(length != 0); + const char* data = + node->tag == EXTERNAL ? node->external()->base : node->data; + current_chunk_ = absl::string_view(data + offset, length); + current_leaf_ = node; + return *this; +} + +Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) { + ABSL_HARDENING_ASSERT(bytes_remaining_ >= n && + "Attempted to iterate past `end()`"); + Cord subcord; + + if (n <= InlineRep::kMaxInline) { + // Range to read fits in inline data. Flatten it. + char* data = subcord.contents_.set_data(n); + while (n > current_chunk_.size()) { + memcpy(data, current_chunk_.data(), current_chunk_.size()); + data += current_chunk_.size(); + n -= current_chunk_.size(); + ++*this; + } + memcpy(data, current_chunk_.data(), n); + if (n < current_chunk_.size()) { + RemoveChunkPrefix(n); + } else if (n > 0) { + ++*this; + } + return subcord; + } + if (n < current_chunk_.size()) { + // Range to read is a proper subrange of the current chunk. + assert(current_leaf_ != nullptr); + CordRep* subnode = Ref(current_leaf_); + const char* data = + subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data; + subnode = NewSubstring(subnode, current_chunk_.data() - data, n); + subcord.contents_.set_tree(VerifyTree(subnode)); + RemoveChunkPrefix(n); + return subcord; + } + + // Range to read begins with a proper subrange of the current chunk. + assert(!current_chunk_.empty()); + assert(current_leaf_ != nullptr); + CordRep* subnode = Ref(current_leaf_); + if (current_chunk_.size() < subnode->length) { + const char* data = + subnode->tag == EXTERNAL ? subnode->external()->base : subnode->data; + subnode = NewSubstring(subnode, current_chunk_.data() - data, + current_chunk_.size()); + } + n -= current_chunk_.size(); + bytes_remaining_ -= current_chunk_.size(); + + // Process the next node(s) on the stack, reading whole subtrees depending on + // their length and how many bytes we are advancing. + CordRep* node = nullptr; + while (!stack_of_right_children_.empty()) { + node = stack_of_right_children_.back(); + stack_of_right_children_.pop_back(); + if (node->length > n) break; + // TODO(qrczak): This might unnecessarily recreate existing concat nodes. + // Avoiding that would need pretty complicated logic (instead of + // current_leaf_, keep current_subtree_ which points to the highest node + // such that the current leaf can be found on the path of left children + // starting from current_subtree_; delay creating subnode while node is + // below current_subtree_; find the proper node along the path of left + // children starting from current_subtree_ if this loop exits while staying + // below current_subtree_; etc.; alternatively, push parents instead of + // right children on the stack). + subnode = Concat(subnode, Ref(node)); + n -= node->length; + bytes_remaining_ -= node->length; + node = nullptr; + } + + if (node == nullptr) { + // We have reached the end of the Cord. + assert(bytes_remaining_ == 0); + subcord.contents_.set_tree(VerifyTree(subnode)); + return subcord; + } + + // Walk down the appropriate branches until we hit a non-CONCAT node. Save the + // right children to the stack for subsequent traversal. + while (node->tag == CONCAT) { + if (node->concat()->left->length > n) { + // Push right, descend left. + stack_of_right_children_.push_back(node->concat()->right); + node = node->concat()->left; + } else { + // Read left, descend right. + subnode = Concat(subnode, Ref(node->concat()->left)); + n -= node->concat()->left->length; + bytes_remaining_ -= node->concat()->left->length; + node = node->concat()->right; + } + } + + // Get the child node if we encounter a SUBSTRING. + size_t offset = 0; + size_t length = node->length; + if (node->tag == SUBSTRING) { + offset = node->substring()->start; + node = node->substring()->child; + } + + // Range to read ends with a proper (possibly empty) subrange of the current + // chunk. + assert(node->tag == EXTERNAL || node->tag >= FLAT); + assert(length > n); + if (n > 0) subnode = Concat(subnode, NewSubstring(Ref(node), offset, n)); + const char* data = + node->tag == EXTERNAL ? node->external()->base : node->data; + current_chunk_ = absl::string_view(data + offset + n, length - n); + current_leaf_ = node; + bytes_remaining_ -= n; + subcord.contents_.set_tree(VerifyTree(subnode)); + return subcord; +} + +void Cord::ChunkIterator::AdvanceBytesSlowPath(size_t n) { + assert(bytes_remaining_ >= n && "Attempted to iterate past `end()`"); + assert(n >= current_chunk_.size()); // This should only be called when + // iterating to a new node. + + n -= current_chunk_.size(); + bytes_remaining_ -= current_chunk_.size(); + + // Process the next node(s) on the stack, skipping whole subtrees depending on + // their length and how many bytes we are advancing. + CordRep* node = nullptr; + while (!stack_of_right_children_.empty()) { + node = stack_of_right_children_.back(); + stack_of_right_children_.pop_back(); + if (node->length > n) break; + n -= node->length; + bytes_remaining_ -= node->length; + node = nullptr; + } + + if (node == nullptr) { + // We have reached the end of the Cord. + assert(bytes_remaining_ == 0); + return; + } + + // Walk down the appropriate branches until we hit a non-CONCAT node. Save the + // right children to the stack for subsequent traversal. + while (node->tag == CONCAT) { + if (node->concat()->left->length > n) { + // Push right, descend left. + stack_of_right_children_.push_back(node->concat()->right); + node = node->concat()->left; + } else { + // Skip left, descend right. + n -= node->concat()->left->length; + bytes_remaining_ -= node->concat()->left->length; + node = node->concat()->right; + } + } + + // Get the child node if we encounter a SUBSTRING. + size_t offset = 0; + size_t length = node->length; + if (node->tag == SUBSTRING) { + offset = node->substring()->start; + node = node->substring()->child; + } + + assert(node->tag == EXTERNAL || node->tag >= FLAT); + assert(length > n); + const char* data = + node->tag == EXTERNAL ? node->external()->base : node->data; + current_chunk_ = absl::string_view(data + offset + n, length - n); + current_leaf_ = node; + bytes_remaining_ -= n; +} + +char Cord::operator[](size_t i) const { + ABSL_HARDENING_ASSERT(i < size()); + size_t offset = i; + const CordRep* rep = contents_.tree(); + if (rep == nullptr) { + return contents_.data()[i]; + } + while (true) { + assert(rep != nullptr); + assert(offset < rep->length); + if (rep->tag >= FLAT) { + // Get the "i"th character directly from the flat array. + return rep->data[offset]; + } else if (rep->tag == EXTERNAL) { + // Get the "i"th character from the external array. + return rep->external()->base[offset]; + } else if (rep->tag == CONCAT) { + // Recursively branch to the side of the concatenation that the "i"th + // character is on. + size_t left_length = rep->concat()->left->length; + if (offset < left_length) { + rep = rep->concat()->left; + } else { + offset -= left_length; + rep = rep->concat()->right; + } + } else { + // This must be a substring a node, so bypass it to get to the child. + assert(rep->tag == SUBSTRING); + offset += rep->substring()->start; + rep = rep->substring()->child; + } + } +} + +absl::string_view Cord::FlattenSlowPath() { + size_t total_size = size(); + CordRep* new_rep; + char* new_buffer; + + // Try to put the contents into a new flat rep. If they won't fit in the + // biggest possible flat node, use an external rep instead. + if (total_size <= kMaxFlatLength) { + new_rep = NewFlat(total_size); + new_rep->length = total_size; + new_buffer = new_rep->data; + CopyToArraySlowPath(new_buffer); + } else { + new_buffer = std::allocator<char>().allocate(total_size); + CopyToArraySlowPath(new_buffer); + new_rep = absl::cord_internal::NewExternalRep( + absl::string_view(new_buffer, total_size), [](absl::string_view s) { + std::allocator<char>().deallocate(const_cast<char*>(s.data()), + s.size()); + }); + } + Unref(contents_.tree()); + contents_.set_tree(new_rep); + return absl::string_view(new_buffer, total_size); +} + +/* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) { + assert(rep != nullptr); + if (rep->tag >= FLAT) { + *fragment = absl::string_view(rep->data, rep->length); + return true; + } else if (rep->tag == EXTERNAL) { + *fragment = absl::string_view(rep->external()->base, rep->length); + return true; + } else if (rep->tag == SUBSTRING) { + CordRep* child = rep->substring()->child; + if (child->tag >= FLAT) { + *fragment = + absl::string_view(child->data + rep->substring()->start, rep->length); + return true; + } else if (child->tag == EXTERNAL) { + *fragment = absl::string_view( + child->external()->base + rep->substring()->start, rep->length); + return true; + } + } + return false; +} + +/* static */ void Cord::ForEachChunkAux( + absl::cord_internal::CordRep* rep, + absl::FunctionRef<void(absl::string_view)> callback) { + assert(rep != nullptr); + int stack_pos = 0; + constexpr int stack_max = 128; + // Stack of right branches for tree traversal + absl::cord_internal::CordRep* stack[stack_max]; + absl::cord_internal::CordRep* current_node = rep; + while (true) { + if (current_node->tag == CONCAT) { + if (stack_pos == stack_max) { + // There's no more room on our stack array to add another right branch, + // and the idea is to avoid allocations, so call this function + // recursively to navigate this subtree further. (This is not something + // we expect to happen in practice). + ForEachChunkAux(current_node, callback); + + // Pop the next right branch and iterate. + current_node = stack[--stack_pos]; + continue; + } else { + // Save the right branch for later traversal and continue down the left + // branch. + stack[stack_pos++] = current_node->concat()->right; + current_node = current_node->concat()->left; + continue; + } + } + // This is a leaf node, so invoke our callback. + absl::string_view chunk; + bool success = GetFlatAux(current_node, &chunk); + assert(success); + if (success) { + callback(chunk); + } + if (stack_pos == 0) { + // end of traversal + return; + } + current_node = stack[--stack_pos]; + } +} + +static void DumpNode(CordRep* rep, bool include_data, std::ostream* os) { + const int kIndentStep = 1; + int indent = 0; + absl::InlinedVector<CordRep*, kInlinedVectorSize> stack; + absl::InlinedVector<int, kInlinedVectorSize> indents; + for (;;) { + *os << std::setw(3) << rep->refcount.Get(); + *os << " " << std::setw(7) << rep->length; + *os << " ["; + if (include_data) *os << static_cast<void*>(rep); + *os << "]"; + *os << " " << (IsRootBalanced(rep) ? 'b' : 'u'); + *os << " " << std::setw(indent) << ""; + if (rep->tag == CONCAT) { + *os << "CONCAT depth=" << Depth(rep) << "\n"; + indent += kIndentStep; + indents.push_back(indent); + stack.push_back(rep->concat()->right); + rep = rep->concat()->left; + } else if (rep->tag == SUBSTRING) { + *os << "SUBSTRING @ " << rep->substring()->start << "\n"; + indent += kIndentStep; + rep = rep->substring()->child; + } else { // Leaf + if (rep->tag == EXTERNAL) { + *os << "EXTERNAL ["; + if (include_data) + *os << absl::CEscape(std::string(rep->external()->base, rep->length)); + *os << "]\n"; + } else { + *os << "FLAT cap=" << TagToLength(rep->tag) << " ["; + if (include_data) + *os << absl::CEscape(std::string(rep->data, rep->length)); + *os << "]\n"; + } + if (stack.empty()) break; + rep = stack.back(); + stack.pop_back(); + indent = indents.back(); + indents.pop_back(); + } + } + ABSL_INTERNAL_CHECK(indents.empty(), ""); +} + +static std::string ReportError(CordRep* root, CordRep* node) { + std::ostringstream buf; + buf << "Error at node " << node << " in:"; + DumpNode(root, true, &buf); + return buf.str(); +} + +static bool VerifyNode(CordRep* root, CordRep* start_node, + bool full_validation) { + absl::InlinedVector<CordRep*, 2> worklist; + worklist.push_back(start_node); + do { + CordRep* node = worklist.back(); + worklist.pop_back(); + + ABSL_INTERNAL_CHECK(node != nullptr, ReportError(root, node)); + if (node != root) { + ABSL_INTERNAL_CHECK(node->length != 0, ReportError(root, node)); + } + + if (node->tag == CONCAT) { + ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, + ReportError(root, node)); + ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, + ReportError(root, node)); + ABSL_INTERNAL_CHECK((node->length == node->concat()->left->length + + node->concat()->right->length), + ReportError(root, node)); + if (full_validation) { + worklist.push_back(node->concat()->right); + worklist.push_back(node->concat()->left); + } + } else if (node->tag >= FLAT) { + ABSL_INTERNAL_CHECK(node->length <= TagToLength(node->tag), + ReportError(root, node)); + } else if (node->tag == EXTERNAL) { + ABSL_INTERNAL_CHECK(node->external()->base != nullptr, + ReportError(root, node)); + } else if (node->tag == SUBSTRING) { + ABSL_INTERNAL_CHECK( + node->substring()->start < node->substring()->child->length, + ReportError(root, node)); + ABSL_INTERNAL_CHECK(node->substring()->start + node->length <= + node->substring()->child->length, + ReportError(root, node)); + } + } while (!worklist.empty()); + return true; +} + +// Traverses the tree and computes the total memory allocated. +/* static */ size_t Cord::MemoryUsageAux(const CordRep* rep) { + size_t total_mem_usage = 0; + + // Allow a quick exit for the common case that the root is a leaf. + if (RepMemoryUsageLeaf(rep, &total_mem_usage)) { + return total_mem_usage; + } + + // Iterate over the tree. cur_node is never a leaf node and leaf nodes will + // never be appended to tree_stack. This reduces overhead from manipulating + // tree_stack. + absl::InlinedVector<const CordRep*, kInlinedVectorSize> tree_stack; + const CordRep* cur_node = rep; + while (true) { + const CordRep* next_node = nullptr; + + if (cur_node->tag == CONCAT) { + total_mem_usage += sizeof(CordRepConcat); + const CordRep* left = cur_node->concat()->left; + if (!RepMemoryUsageLeaf(left, &total_mem_usage)) { + next_node = left; + } + + const CordRep* right = cur_node->concat()->right; + if (!RepMemoryUsageLeaf(right, &total_mem_usage)) { + if (next_node) { + tree_stack.push_back(next_node); + } + next_node = right; + } + } else { + // Since cur_node is not a leaf or a concat node it must be a substring. + assert(cur_node->tag == SUBSTRING); + total_mem_usage += sizeof(CordRepSubstring); + next_node = cur_node->substring()->child; + if (RepMemoryUsageLeaf(next_node, &total_mem_usage)) { + next_node = nullptr; + } + } + + if (!next_node) { + if (tree_stack.empty()) { + return total_mem_usage; + } + next_node = tree_stack.back(); + tree_stack.pop_back(); + } + cur_node = next_node; + } +} + +std::ostream& operator<<(std::ostream& out, const Cord& cord) { + for (absl::string_view chunk : cord.Chunks()) { + out.write(chunk.data(), chunk.size()); + } + return out; +} + +namespace strings_internal { +size_t CordTestAccess::FlatOverhead() { return kFlatOverhead; } +size_t CordTestAccess::MaxFlatLength() { return kMaxFlatLength; } +size_t CordTestAccess::FlatTagToLength(uint8_t tag) { + return TagToLength(tag); +} +uint8_t CordTestAccess::LengthToTag(size_t s) { + ABSL_INTERNAL_CHECK(s <= kMaxFlatLength, absl::StrCat("Invalid length ", s)); + return AllocatedSizeToTag(s + kFlatOverhead); +} +size_t CordTestAccess::SizeofCordRepConcat() { return sizeof(CordRepConcat); } +size_t CordTestAccess::SizeofCordRepExternal() { + return sizeof(CordRepExternal); +} +size_t CordTestAccess::SizeofCordRepSubstring() { + return sizeof(CordRepSubstring); +} +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/cord.h b/third_party/abseil_cpp/absl/strings/cord.h new file mode 100644 index 000000000000..5d5c897e663c --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/cord.h @@ -0,0 +1,1299 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: cord.h +// ----------------------------------------------------------------------------- +// +// This file defines the `absl::Cord` data structure and operations on that data +// structure. A Cord is a string-like sequence of characters optimized for +// specific use cases. Unlike a `std::string`, which stores an array of +// contiguous characters, Cord data is stored in a structure consisting of +// separate, reference-counted "chunks." (Currently, this implementation is a +// tree structure, though that implementation may change.) +// +// Because a Cord consists of these chunks, data can be added to or removed from +// a Cord during its lifetime. Chunks may also be shared between Cords. Unlike a +// `std::string`, a Cord can therefore accomodate data that changes over its +// lifetime, though it's not quite "mutable"; it can change only in the +// attachment, detachment, or rearrangement of chunks of its constituent data. +// +// A Cord provides some benefit over `std::string` under the following (albeit +// narrow) circumstances: +// +// * Cord data is designed to grow and shrink over a Cord's lifetime. Cord +// provides efficient insertions and deletions at the start and end of the +// character sequences, avoiding copies in those cases. Static data should +// generally be stored as strings. +// * External memory consisting of string-like data can be directly added to +// a Cord without requiring copies or allocations. +// * Cord data may be shared and copied cheaply. Cord provides a copy-on-write +// implementation and cheap sub-Cord operations. Copying a Cord is an O(1) +// operation. +// +// As a consequence to the above, Cord data is generally large. Small data +// should generally use strings, as construction of a Cord requires some +// overhead. Small Cords (<= 15 bytes) are represented inline, but most small +// Cords are expected to grow over their lifetimes. +// +// Note that because a Cord is made up of separate chunked data, random access +// to character data within a Cord is slower than within a `std::string`. +// +// Thread Safety +// +// Cord has the same thread-safety properties as many other types like +// std::string, std::vector<>, int, etc -- it is thread-compatible. In +// particular, if threads do not call non-const methods, then it is safe to call +// const methods without synchronization. Copying a Cord produces a new instance +// that can be used concurrently with the original in arbitrary ways. + +#ifndef ABSL_STRINGS_CORD_H_ +#define ABSL_STRINGS_CORD_H_ + +#include <algorithm> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <iosfwd> +#include <iterator> +#include <string> +#include <type_traits> + +#include "absl/base/internal/endian.h" +#include "absl/base/internal/per_thread_tls.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/container/inlined_vector.h" +#include "absl/functional/function_ref.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/internal/string_constant.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +class Cord; +class CordTestPeer; +template <typename Releaser> +Cord MakeCordFromExternal(absl::string_view, Releaser&&); +void CopyCordToString(const Cord& src, std::string* dst); + +// Cord +// +// A Cord is a sequence of characters, designed to be more efficient than a +// `std::string` in certain circumstances: namely, large string data that needs +// to change over its lifetime or shared, especially when such data is shared +// across API boundaries. +// +// A Cord stores its character data in a structure that allows efficient prepend +// and append operations. This makes a Cord useful for large string data sent +// over in a wire format that may need to be prepended or appended at some point +// during the data exchange (e.g. HTTP, protocol buffers). For example, a +// Cord is useful for storing an HTTP request, and prepending an HTTP header to +// such a request. +// +// Cords should not be used for storing general string data, however. They +// require overhead to construct and are slower than strings for random access. +// +// The Cord API provides the following common API operations: +// +// * Create or assign Cords out of existing string data, memory, or other Cords +// * Append and prepend data to an existing Cord +// * Create new Sub-Cords from existing Cord data +// * Swap Cord data and compare Cord equality +// * Write out Cord data by constructing a `std::string` +// +// Additionally, the API provides iterator utilities to iterate through Cord +// data via chunks or character bytes. +// +class Cord { + private: + template <typename T> + using EnableIfString = + absl::enable_if_t<std::is_same<T, std::string>::value, int>; + + public: + // Cord::Cord() Constructors. + + // Creates an empty Cord. + constexpr Cord() noexcept; + + // Creates a Cord from an existing Cord. Cord is copyable and efficiently + // movable. The moved-from state is valid but unspecified. + Cord(const Cord& src); + Cord(Cord&& src) noexcept; + Cord& operator=(const Cord& x); + Cord& operator=(Cord&& x) noexcept; + + // Creates a Cord from a `src` string. This constructor is marked explicit to + // prevent implicit Cord constructions from arguments convertible to an + // `absl::string_view`. + explicit Cord(absl::string_view src); + Cord& operator=(absl::string_view src); + + // Creates a Cord from a `std::string&&` rvalue. These constructors are + // templated to avoid ambiguities for types that are convertible to both + // `absl::string_view` and `std::string`, such as `const char*`. + template <typename T, EnableIfString<T> = 0> + explicit Cord(T&& src); + template <typename T, EnableIfString<T> = 0> + Cord& operator=(T&& src); + + // Cord::~Cord() + // + // Destructs the Cord. + ~Cord() { + if (contents_.is_tree()) DestroyCordSlow(); + } + + // MakeCordFromExternal() + // + // Creates a Cord that takes ownership of external string memory. The + // contents of `data` are not copied to the Cord; instead, the external + // memory is added to the Cord and reference-counted. This data may not be + // changed for the life of the Cord, though it may be prepended or appended + // to. + // + // `MakeCordFromExternal()` takes a callable "releaser" that is invoked when + // the reference count for `data` reaches zero. As noted above, this data must + // remain live until the releaser is invoked. The callable releaser also must: + // + // * be move constructible + // * support `void operator()(absl::string_view) const` or `void operator()` + // + // Example: + // + // Cord MakeCord(BlockPool* pool) { + // Block* block = pool->NewBlock(); + // FillBlock(block); + // return absl::MakeCordFromExternal( + // block->ToStringView(), + // [pool, block](absl::string_view v) { + // pool->FreeBlock(block, v); + // }); + // } + // + // WARNING: Because a Cord can be reference-counted, it's likely a bug if your + // releaser doesn't do anything. For example, consider the following: + // + // void Foo(const char* buffer, int len) { + // auto c = absl::MakeCordFromExternal(absl::string_view(buffer, len), + // [](absl::string_view) {}); + // + // // BUG: If Bar() copies its cord for any reason, including keeping a + // // substring of it, the lifetime of buffer might be extended beyond + // // when Foo() returns. + // Bar(c); + // } + template <typename Releaser> + friend Cord MakeCordFromExternal(absl::string_view data, Releaser&& releaser); + + // Cord::Clear() + // + // Releases the Cord data. Any nodes that share data with other Cords, if + // applicable, will have their reference counts reduced by 1. + void Clear(); + + // Cord::Append() + // + // Appends data to the Cord, which may come from another Cord or other string + // data. + void Append(const Cord& src); + void Append(Cord&& src); + void Append(absl::string_view src); + template <typename T, EnableIfString<T> = 0> + void Append(T&& src); + + // Cord::Prepend() + // + // Prepends data to the Cord, which may come from another Cord or other string + // data. + void Prepend(const Cord& src); + void Prepend(absl::string_view src); + template <typename T, EnableIfString<T> = 0> + void Prepend(T&& src); + + // Cord::RemovePrefix() + // + // Removes the first `n` bytes of a Cord. + void RemovePrefix(size_t n); + void RemoveSuffix(size_t n); + + // Cord::Subcord() + // + // Returns a new Cord representing the subrange [pos, pos + new_size) of + // *this. If pos >= size(), the result is empty(). If + // (pos + new_size) >= size(), the result is the subrange [pos, size()). + Cord Subcord(size_t pos, size_t new_size) const; + + // Cord::swap() + // + // Swaps the contents of the Cord with `other`. + void swap(Cord& other) noexcept; + + // swap() + // + // Swaps the contents of two Cords. + friend void swap(Cord& x, Cord& y) noexcept { + x.swap(y); + } + + // Cord::size() + // + // Returns the size of the Cord. + size_t size() const; + + // Cord::empty() + // + // Determines whether the given Cord is empty, returning `true` is so. + bool empty() const; + + // Cord::EstimatedMemoryUsage() + // + // Returns the *approximate* number of bytes held in full or in part by this + // Cord (which may not remain the same between invocations). Note that Cords + // that share memory could each be "charged" independently for the same shared + // memory. + size_t EstimatedMemoryUsage() const; + + // Cord::Compare() + // + // Compares 'this' Cord with rhs. This function and its relatives treat Cords + // as sequences of unsigned bytes. The comparison is a straightforward + // lexicographic comparison. `Cord::Compare()` returns values as follows: + // + // -1 'this' Cord is smaller + // 0 two Cords are equal + // 1 'this' Cord is larger + int Compare(absl::string_view rhs) const; + int Compare(const Cord& rhs) const; + + // Cord::StartsWith() + // + // Determines whether the Cord starts with the passed string data `rhs`. + bool StartsWith(const Cord& rhs) const; + bool StartsWith(absl::string_view rhs) const; + + // Cord::EndsWidth() + // + // Determines whether the Cord ends with the passed string data `rhs`. + bool EndsWith(absl::string_view rhs) const; + bool EndsWith(const Cord& rhs) const; + + // Cord::operator std::string() + // + // Converts a Cord into a `std::string()`. This operator is marked explicit to + // prevent unintended Cord usage in functions that take a string. + explicit operator std::string() const; + + // CopyCordToString() + // + // Copies the contents of a `src` Cord into a `*dst` string. + // + // This function optimizes the case of reusing the destination string since it + // can reuse previously allocated capacity. However, this function does not + // guarantee that pointers previously returned by `dst->data()` remain valid + // even if `*dst` had enough capacity to hold `src`. If `*dst` is a new + // object, prefer to simply use the conversion operator to `std::string`. + friend void CopyCordToString(const Cord& src, std::string* dst); + + class CharIterator; + + //---------------------------------------------------------------------------- + // Cord::ChunkIterator + //---------------------------------------------------------------------------- + // + // A `Cord::ChunkIterator` allows iteration over the constituent chunks of its + // Cord. Such iteration allows you to perform non-const operatons on the data + // of a Cord without modifying it. + // + // Generally, you do not instantiate a `Cord::ChunkIterator` directly; + // instead, you create one implicitly through use of the `Cord::Chunks()` + // member function. + // + // The `Cord::ChunkIterator` has the following properties: + // + // * The iterator is invalidated after any non-const operation on the + // Cord object over which it iterates. + // * The `string_view` returned by dereferencing a valid, non-`end()` + // iterator is guaranteed to be non-empty. + // * Two `ChunkIterator` objects can be compared equal if and only if they + // remain valid and iterate over the same Cord. + // * The iterator in this case is a proxy iterator; the `string_view` + // returned by the iterator does not live inside the Cord, and its + // lifetime is limited to the lifetime of the iterator itself. To help + // prevent lifetime issues, `ChunkIterator::reference` is not a true + // reference type and is equivalent to `value_type`. + // * The iterator keeps state that can grow for Cords that contain many + // nodes and are imbalanced due to sharing. Prefer to pass this type by + // const reference instead of by value. + class ChunkIterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = absl::string_view; + using difference_type = ptrdiff_t; + using pointer = const value_type*; + using reference = value_type; + + ChunkIterator() = default; + + ChunkIterator& operator++(); + ChunkIterator operator++(int); + bool operator==(const ChunkIterator& other) const; + bool operator!=(const ChunkIterator& other) const; + reference operator*() const; + pointer operator->() const; + + friend class Cord; + friend class CharIterator; + + private: + // Constructs a `begin()` iterator from `cord`. + explicit ChunkIterator(const Cord* cord); + + // Removes `n` bytes from `current_chunk_`. Expects `n` to be smaller than + // `current_chunk_.size()`. + void RemoveChunkPrefix(size_t n); + Cord AdvanceAndReadBytes(size_t n); + void AdvanceBytes(size_t n); + // Iterates `n` bytes, where `n` is expected to be greater than or equal to + // `current_chunk_.size()`. + void AdvanceBytesSlowPath(size_t n); + + // A view into bytes of the current `CordRep`. It may only be a view to a + // suffix of bytes if this is being used by `CharIterator`. + absl::string_view current_chunk_; + // The current leaf, or `nullptr` if the iterator points to short data. + // If the current chunk is a substring node, current_leaf_ points to the + // underlying flat or external node. + absl::cord_internal::CordRep* current_leaf_ = nullptr; + // The number of bytes left in the `Cord` over which we are iterating. + size_t bytes_remaining_ = 0; + absl::InlinedVector<absl::cord_internal::CordRep*, 4> + stack_of_right_children_; + }; + + // Cord::ChunkIterator::chunk_begin() + // + // Returns an iterator to the first chunk of the `Cord`. + // + // Generally, prefer using `Cord::Chunks()` within a range-based for loop for + // iterating over the chunks of a Cord. This method may be useful for getting + // a `ChunkIterator` where range-based for-loops are not useful. + // + // Example: + // + // absl::Cord::ChunkIterator FindAsChunk(const absl::Cord& c, + // absl::string_view s) { + // return std::find(c.chunk_begin(), c.chunk_end(), s); + // } + ChunkIterator chunk_begin() const; + + // Cord::ChunkItertator::chunk_end() + // + // Returns an iterator one increment past the last chunk of the `Cord`. + // + // Generally, prefer using `Cord::Chunks()` within a range-based for loop for + // iterating over the chunks of a Cord. This method may be useful for getting + // a `ChunkIterator` where range-based for-loops may not be available. + ChunkIterator chunk_end() const; + + //---------------------------------------------------------------------------- + // Cord::ChunkIterator::ChunkRange + //---------------------------------------------------------------------------- + // + // `ChunkRange` is a helper class for iterating over the chunks of the `Cord`, + // producing an iterator which can be used within a range-based for loop. + // Construction of a `ChunkRange` will return an iterator pointing to the + // first chunk of the Cord. Generally, do not construct a `ChunkRange` + // directly; instead, prefer to use the `Cord::Chunks()` method. + // + // Implementation note: `ChunkRange` is simply a convenience wrapper over + // `Cord::chunk_begin()` and `Cord::chunk_end()`. + class ChunkRange { + public: + explicit ChunkRange(const Cord* cord) : cord_(cord) {} + + ChunkIterator begin() const; + ChunkIterator end() const; + + private: + const Cord* cord_; + }; + + // Cord::Chunks() + // + // Returns a `Cord::ChunkIterator::ChunkRange` for iterating over the chunks + // of a `Cord` with a range-based for-loop. For most iteration tasks on a + // Cord, use `Cord::Chunks()` to retrieve this iterator. + // + // Example: + // + // void ProcessChunks(const Cord& cord) { + // for (absl::string_view chunk : cord.Chunks()) { ... } + // } + // + // Note that the ordinary caveats of temporary lifetime extension apply: + // + // void Process() { + // for (absl::string_view chunk : CordFactory().Chunks()) { + // // The temporary Cord returned by CordFactory has been destroyed! + // } + // } + ChunkRange Chunks() const; + + //---------------------------------------------------------------------------- + // Cord::CharIterator + //---------------------------------------------------------------------------- + // + // A `Cord::CharIterator` allows iteration over the constituent characters of + // a `Cord`. + // + // Generally, you do not instantiate a `Cord::CharIterator` directly; instead, + // you create one implicitly through use of the `Cord::Chars()` member + // function. + // + // A `Cord::CharIterator` has the following properties: + // + // * The iterator is invalidated after any non-const operation on the + // Cord object over which it iterates. + // * Two `CharIterator` objects can be compared equal if and only if they + // remain valid and iterate over the same Cord. + // * The iterator keeps state that can grow for Cords that contain many + // nodes and are imbalanced due to sharing. Prefer to pass this type by + // const reference instead of by value. + // * This type cannot act as a forward iterator because a `Cord` can reuse + // sections of memory. This fact violates the requirement for forward + // iterators to compare equal if dereferencing them returns the same + // object. + class CharIterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = char; + using difference_type = ptrdiff_t; + using pointer = const char*; + using reference = const char&; + + CharIterator() = default; + + CharIterator& operator++(); + CharIterator operator++(int); + bool operator==(const CharIterator& other) const; + bool operator!=(const CharIterator& other) const; + reference operator*() const; + pointer operator->() const; + + friend Cord; + + private: + explicit CharIterator(const Cord* cord) : chunk_iterator_(cord) {} + + ChunkIterator chunk_iterator_; + }; + + // Cord::CharIterator::AdvanceAndRead() + // + // Advances the `Cord::CharIterator` by `n_bytes` and returns the bytes + // advanced as a separate `Cord`. `n_bytes` must be less than or equal to the + // number of bytes within the Cord; otherwise, behavior is undefined. It is + // valid to pass `char_end()` and `0`. + static Cord AdvanceAndRead(CharIterator* it, size_t n_bytes); + + // Cord::CharIterator::Advance() + // + // Advances the `Cord::CharIterator` by `n_bytes`. `n_bytes` must be less than + // or equal to the number of bytes remaining within the Cord; otherwise, + // behavior is undefined. It is valid to pass `char_end()` and `0`. + static void Advance(CharIterator* it, size_t n_bytes); + + // Cord::CharIterator::ChunkRemaining() + // + // Returns the longest contiguous view starting at the iterator's position. + // + // `it` must be dereferenceable. + static absl::string_view ChunkRemaining(const CharIterator& it); + + // Cord::CharIterator::char_begin() + // + // Returns an iterator to the first character of the `Cord`. + // + // Generally, prefer using `Cord::Chars()` within a range-based for loop for + // iterating over the chunks of a Cord. This method may be useful for getting + // a `CharIterator` where range-based for-loops may not be available. + CharIterator char_begin() const; + + // Cord::CharIterator::char_end() + // + // Returns an iterator to one past the last character of the `Cord`. + // + // Generally, prefer using `Cord::Chars()` within a range-based for loop for + // iterating over the chunks of a Cord. This method may be useful for getting + // a `CharIterator` where range-based for-loops are not useful. + CharIterator char_end() const; + + // Cord::CharIterator::CharRange + // + // `CharRange` is a helper class for iterating over the characters of a + // producing an iterator which can be used within a range-based for loop. + // Construction of a `CharRange` will return an iterator pointing to the first + // character of the Cord. Generally, do not construct a `CharRange` directly; + // instead, prefer to use the `Cord::Chars()` method show below. + // + // Implementation note: `CharRange` is simply a convenience wrapper over + // `Cord::char_begin()` and `Cord::char_end()`. + class CharRange { + public: + explicit CharRange(const Cord* cord) : cord_(cord) {} + + CharIterator begin() const; + CharIterator end() const; + + private: + const Cord* cord_; + }; + + // Cord::CharIterator::Chars() + // + // Returns a `Cord::CharIterator` for iterating over the characters of a + // `Cord` with a range-based for-loop. For most character-based iteration + // tasks on a Cord, use `Cord::Chars()` to retrieve this iterator. + // + // Example: + // + // void ProcessCord(const Cord& cord) { + // for (char c : cord.Chars()) { ... } + // } + // + // Note that the ordinary caveats of temporary lifetime extension apply: + // + // void Process() { + // for (char c : CordFactory().Chars()) { + // // The temporary Cord returned by CordFactory has been destroyed! + // } + // } + CharRange Chars() const; + + // Cord::operator[] + // + // Gets the "i"th character of the Cord and returns it, provided that + // 0 <= i < Cord.size(). + // + // NOTE: This routine is reasonably efficient. It is roughly + // logarithmic based on the number of chunks that make up the cord. Still, + // if you need to iterate over the contents of a cord, you should + // use a CharIterator/ChunkIterator rather than call operator[] or Get() + // repeatedly in a loop. + char operator[](size_t i) const; + + // Cord::TryFlat() + // + // If this cord's representation is a single flat array, returns a + // string_view referencing that array. Otherwise returns nullopt. + absl::optional<absl::string_view> TryFlat() const; + + // Cord::Flatten() + // + // Flattens the cord into a single array and returns a view of the data. + // + // If the cord was already flat, the contents are not modified. + absl::string_view Flatten(); + + // Supports absl::Cord as a sink object for absl::Format(). + friend void AbslFormatFlush(absl::Cord* cord, absl::string_view part) { + cord->Append(part); + } + + template <typename H> + friend H AbslHashValue(H hash_state, const absl::Cord& c) { + absl::optional<absl::string_view> maybe_flat = c.TryFlat(); + if (maybe_flat.has_value()) { + return H::combine(std::move(hash_state), *maybe_flat); + } + return c.HashFragmented(std::move(hash_state)); + } + + // Create a Cord with the contents of StringConstant<T>::value. + // No allocations will be done and no data will be copied. + // This is an INTERNAL API and subject to change or removal. This API can only + // be used by spelling absl::strings_internal::MakeStringConstant, which is + // also an internal API. + template <typename T> + explicit constexpr Cord(strings_internal::StringConstant<T>); + + private: + friend class CordTestPeer; + friend bool operator==(const Cord& lhs, const Cord& rhs); + friend bool operator==(const Cord& lhs, absl::string_view rhs); + + // Calls the provided function once for each cord chunk, in order. Unlike + // Chunks(), this API will not allocate memory. + void ForEachChunk(absl::FunctionRef<void(absl::string_view)>) const; + + // Allocates new contiguous storage for the contents of the cord. This is + // called by Flatten() when the cord was not already flat. + absl::string_view FlattenSlowPath(); + + // Actual cord contents are hidden inside the following simple + // class so that we can isolate the bulk of cord.cc from changes + // to the representation. + // + // InlineRep holds either a tree pointer, or an array of kMaxInline bytes. + class InlineRep { + public: + static constexpr unsigned char kMaxInline = cord_internal::kMaxInline; + static_assert(kMaxInline >= sizeof(absl::cord_internal::CordRep*), ""); + static constexpr unsigned char kTreeFlag = cord_internal::kTreeFlag; + static constexpr unsigned char kProfiledFlag = cord_internal::kProfiledFlag; + + constexpr InlineRep() : data_() {} + InlineRep(const InlineRep& src); + InlineRep(InlineRep&& src); + InlineRep& operator=(const InlineRep& src); + InlineRep& operator=(InlineRep&& src) noexcept; + + explicit constexpr InlineRep(cord_internal::InlineData data); + + void Swap(InlineRep* rhs); + bool empty() const; + size_t size() const; + const char* data() const; // Returns nullptr if holding pointer + void set_data(const char* data, size_t n, + bool nullify_tail); // Discards pointer, if any + char* set_data(size_t n); // Write data to the result + // Returns nullptr if holding bytes + absl::cord_internal::CordRep* tree() const; + // Discards old pointer, if any + void set_tree(absl::cord_internal::CordRep* rep); + // Replaces a tree with a new root. This is faster than set_tree, but it + // should only be used when it's clear that the old rep was a tree. + void replace_tree(absl::cord_internal::CordRep* rep); + // Returns non-null iff was holding a pointer + absl::cord_internal::CordRep* clear(); + // Converts to pointer if necessary. + absl::cord_internal::CordRep* force_tree(size_t extra_hint); + void reduce_size(size_t n); // REQUIRES: holding data + void remove_prefix(size_t n); // REQUIRES: holding data + void AppendArray(const char* src_data, size_t src_size); + absl::string_view FindFlatStartPiece() const; + void AppendTree(absl::cord_internal::CordRep* tree); + void PrependTree(absl::cord_internal::CordRep* tree); + void GetAppendRegion(char** region, size_t* size, size_t max_length); + void GetAppendRegion(char** region, size_t* size); + bool IsSame(const InlineRep& other) const { + return memcmp(&data_, &other.data_, sizeof(data_)) == 0; + } + int BitwiseCompare(const InlineRep& other) const { + uint64_t x, y; + // Use memcpy to avoid aliasing issues. + memcpy(&x, &data_, sizeof(x)); + memcpy(&y, &other.data_, sizeof(y)); + if (x == y) { + memcpy(&x, reinterpret_cast<const char*>(&data_) + 8, sizeof(x)); + memcpy(&y, reinterpret_cast<const char*>(&other.data_) + 8, sizeof(y)); + if (x == y) return 0; + } + return absl::big_endian::FromHost64(x) < absl::big_endian::FromHost64(y) + ? -1 + : 1; + } + void CopyTo(std::string* dst) const { + // memcpy is much faster when operating on a known size. On most supported + // platforms, the small string optimization is large enough that resizing + // to 15 bytes does not cause a memory allocation. + absl::strings_internal::STLStringResizeUninitialized(dst, + sizeof(data_) - 1); + memcpy(&(*dst)[0], &data_, sizeof(data_) - 1); + // erase is faster than resize because the logic for memory allocation is + // not needed. + dst->erase(tagged_size()); + } + + // Copies the inline contents into `dst`. Assumes the cord is not empty. + void CopyToArray(char* dst) const; + + bool is_tree() const { return tagged_size() > kMaxInline; } + + private: + friend class Cord; + + void AssignSlow(const InlineRep& src); + // Unrefs the tree, stops profiling, and zeroes the contents + void ClearSlow(); + + void ResetToEmpty() { data_ = {}; } + + // This uses reinterpret_cast instead of the union to avoid accessing the + // inactive union element. The tagged size is not a common prefix. + void set_tagged_size(char new_tag) { + reinterpret_cast<char*>(&data_)[kMaxInline] = new_tag; + } + char tagged_size() const { + return reinterpret_cast<const char*>(&data_)[kMaxInline]; + } + + cord_internal::InlineData data_; + }; + InlineRep contents_; + + // Helper for MemoryUsage(). + static size_t MemoryUsageAux(const absl::cord_internal::CordRep* rep); + + // Helper for GetFlat() and TryFlat(). + static bool GetFlatAux(absl::cord_internal::CordRep* rep, + absl::string_view* fragment); + + // Helper for ForEachChunk(). + static void ForEachChunkAux( + absl::cord_internal::CordRep* rep, + absl::FunctionRef<void(absl::string_view)> callback); + + // The destructor for non-empty Cords. + void DestroyCordSlow(); + + // Out-of-line implementation of slower parts of logic. + void CopyToArraySlowPath(char* dst) const; + int CompareSlowPath(absl::string_view rhs, size_t compared_size, + size_t size_to_compare) const; + int CompareSlowPath(const Cord& rhs, size_t compared_size, + size_t size_to_compare) const; + bool EqualsImpl(absl::string_view rhs, size_t size_to_compare) const; + bool EqualsImpl(const Cord& rhs, size_t size_to_compare) const; + int CompareImpl(const Cord& rhs) const; + + template <typename ResultType, typename RHS> + friend ResultType GenericCompare(const Cord& lhs, const RHS& rhs, + size_t size_to_compare); + static absl::string_view GetFirstChunk(const Cord& c); + static absl::string_view GetFirstChunk(absl::string_view sv); + + // Returns a new reference to contents_.tree(), or steals an existing + // reference if called on an rvalue. + absl::cord_internal::CordRep* TakeRep() const&; + absl::cord_internal::CordRep* TakeRep() &&; + + // Helper for Append(). + template <typename C> + void AppendImpl(C&& src); + + // Helper for AbslHashValue(). + template <typename H> + H HashFragmented(H hash_state) const { + typename H::AbslInternalPiecewiseCombiner combiner; + ForEachChunk([&combiner, &hash_state](absl::string_view chunk) { + hash_state = combiner.add_buffer(std::move(hash_state), chunk.data(), + chunk.size()); + }); + return H::combine(combiner.finalize(std::move(hash_state)), size()); + } +}; + +ABSL_NAMESPACE_END +} // namespace absl + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// allow a Cord to be logged +extern std::ostream& operator<<(std::ostream& out, const Cord& cord); + +// ------------------------------------------------------------------ +// Internal details follow. Clients should ignore. + +namespace cord_internal { + +// Fast implementation of memmove for up to 15 bytes. This implementation is +// safe for overlapping regions. If nullify_tail is true, the destination is +// padded with '\0' up to 16 bytes. +inline void SmallMemmove(char* dst, const char* src, size_t n, + bool nullify_tail = false) { + if (n >= 8) { + assert(n <= 16); + uint64_t buf1; + uint64_t buf2; + memcpy(&buf1, src, 8); + memcpy(&buf2, src + n - 8, 8); + if (nullify_tail) { + memset(dst + 8, 0, 8); + } + memcpy(dst, &buf1, 8); + memcpy(dst + n - 8, &buf2, 8); + } else if (n >= 4) { + uint32_t buf1; + uint32_t buf2; + memcpy(&buf1, src, 4); + memcpy(&buf2, src + n - 4, 4); + if (nullify_tail) { + memset(dst + 4, 0, 4); + memset(dst + 8, 0, 8); + } + memcpy(dst, &buf1, 4); + memcpy(dst + n - 4, &buf2, 4); + } else { + if (n != 0) { + dst[0] = src[0]; + dst[n / 2] = src[n / 2]; + dst[n - 1] = src[n - 1]; + } + if (nullify_tail) { + memset(dst + 8, 0, 8); + memset(dst + n, 0, 8); + } + } +} + +// Does non-template-specific `CordRepExternal` initialization. +// Expects `data` to be non-empty. +void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep); + +// Creates a new `CordRep` that owns `data` and `releaser` and returns a pointer +// to it, or `nullptr` if `data` was empty. +template <typename Releaser> +// NOLINTNEXTLINE - suppress clang-tidy raw pointer return. +CordRep* NewExternalRep(absl::string_view data, Releaser&& releaser) { + using ReleaserType = absl::decay_t<Releaser>; + if (data.empty()) { + // Never create empty external nodes. + InvokeReleaser(Rank0{}, ReleaserType(std::forward<Releaser>(releaser)), + data); + return nullptr; + } + + CordRepExternal* rep = new CordRepExternalImpl<ReleaserType>( + std::forward<Releaser>(releaser), 0); + InitializeCordRepExternal(data, rep); + return rep; +} + +// Overload for function reference types that dispatches using a function +// pointer because there are no `alignof()` or `sizeof()` a function reference. +// NOLINTNEXTLINE - suppress clang-tidy raw pointer return. +inline CordRep* NewExternalRep(absl::string_view data, + void (&releaser)(absl::string_view)) { + return NewExternalRep(data, &releaser); +} + +} // namespace cord_internal + +template <typename Releaser> +Cord MakeCordFromExternal(absl::string_view data, Releaser&& releaser) { + Cord cord; + cord.contents_.set_tree(::absl::cord_internal::NewExternalRep( + data, std::forward<Releaser>(releaser))); + return cord; +} + +constexpr Cord::InlineRep::InlineRep(cord_internal::InlineData data) + : data_(data) {} + +inline Cord::InlineRep::InlineRep(const Cord::InlineRep& src) { + data_ = src.data_; +} + +inline Cord::InlineRep::InlineRep(Cord::InlineRep&& src) { + data_ = src.data_; + src.ResetToEmpty(); +} + +inline Cord::InlineRep& Cord::InlineRep::operator=(const Cord::InlineRep& src) { + if (this == &src) { + return *this; + } + if (!is_tree() && !src.is_tree()) { + data_ = src.data_; + return *this; + } + AssignSlow(src); + return *this; +} + +inline Cord::InlineRep& Cord::InlineRep::operator=( + Cord::InlineRep&& src) noexcept { + if (is_tree()) { + ClearSlow(); + } + data_ = src.data_; + src.ResetToEmpty(); + return *this; +} + +inline void Cord::InlineRep::Swap(Cord::InlineRep* rhs) { + if (rhs == this) { + return; + } + + std::swap(data_, rhs->data_); +} + +inline const char* Cord::InlineRep::data() const { + return is_tree() ? nullptr : data_.as_chars; +} + +inline absl::cord_internal::CordRep* Cord::InlineRep::tree() const { + if (is_tree()) { + return data_.as_tree.rep; + } else { + return nullptr; + } +} + +inline bool Cord::InlineRep::empty() const { return tagged_size() == 0; } + +inline size_t Cord::InlineRep::size() const { + const char tag = tagged_size(); + if (tag <= kMaxInline) return tag; + return static_cast<size_t>(tree()->length); +} + +inline void Cord::InlineRep::set_tree(absl::cord_internal::CordRep* rep) { + if (rep == nullptr) { + ResetToEmpty(); + } else { + bool was_tree = is_tree(); + data_.as_tree = {rep, {}, tagged_size()}; + if (!was_tree) { + // If we were not a tree already, set the tag. + // Otherwise, leave it alone because it might have the profile bit on. + set_tagged_size(kTreeFlag); + } + } +} + +inline void Cord::InlineRep::replace_tree(absl::cord_internal::CordRep* rep) { + ABSL_ASSERT(is_tree()); + if (ABSL_PREDICT_FALSE(rep == nullptr)) { + set_tree(rep); + return; + } + data_.as_tree = {rep, {}, tagged_size()}; +} + +inline absl::cord_internal::CordRep* Cord::InlineRep::clear() { + absl::cord_internal::CordRep* result = tree(); + ResetToEmpty(); + return result; +} + +inline void Cord::InlineRep::CopyToArray(char* dst) const { + assert(!is_tree()); + size_t n = tagged_size(); + assert(n != 0); + cord_internal::SmallMemmove(dst, data_.as_chars, n); +} + +constexpr inline Cord::Cord() noexcept {} + +template <typename T> +constexpr Cord::Cord(strings_internal::StringConstant<T>) + : contents_(strings_internal::StringConstant<T>::value.size() <= + cord_internal::kMaxInline + ? cord_internal::InlineData( + strings_internal::StringConstant<T>::value) + : cord_internal::InlineData(cord_internal::AsTree{ + &cord_internal::ConstInitExternalStorage< + strings_internal::StringConstant<T>>::value, + {}, + cord_internal::kTreeFlag})) {} + +inline Cord& Cord::operator=(const Cord& x) { + contents_ = x.contents_; + return *this; +} + +inline Cord::Cord(Cord&& src) noexcept : contents_(std::move(src.contents_)) {} + +inline void Cord::swap(Cord& other) noexcept { + contents_.Swap(&other.contents_); +} + +inline Cord& Cord::operator=(Cord&& x) noexcept { + contents_ = std::move(x.contents_); + return *this; +} + +extern template Cord::Cord(std::string&& src); +extern template Cord& Cord::operator=(std::string&& src); + +inline size_t Cord::size() const { + // Length is 1st field in str.rep_ + return contents_.size(); +} + +inline bool Cord::empty() const { return contents_.empty(); } + +inline size_t Cord::EstimatedMemoryUsage() const { + size_t result = sizeof(Cord); + if (const absl::cord_internal::CordRep* rep = contents_.tree()) { + result += MemoryUsageAux(rep); + } + return result; +} + +inline absl::optional<absl::string_view> Cord::TryFlat() const { + absl::cord_internal::CordRep* rep = contents_.tree(); + if (rep == nullptr) { + return absl::string_view(contents_.data(), contents_.size()); + } + absl::string_view fragment; + if (GetFlatAux(rep, &fragment)) { + return fragment; + } + return absl::nullopt; +} + +inline absl::string_view Cord::Flatten() { + absl::cord_internal::CordRep* rep = contents_.tree(); + if (rep == nullptr) { + return absl::string_view(contents_.data(), contents_.size()); + } else { + absl::string_view already_flat_contents; + if (GetFlatAux(rep, &already_flat_contents)) { + return already_flat_contents; + } + } + return FlattenSlowPath(); +} + +inline void Cord::Append(absl::string_view src) { + contents_.AppendArray(src.data(), src.size()); +} + +extern template void Cord::Append(std::string&& src); +extern template void Cord::Prepend(std::string&& src); + +inline int Cord::Compare(const Cord& rhs) const { + if (!contents_.is_tree() && !rhs.contents_.is_tree()) { + return contents_.BitwiseCompare(rhs.contents_); + } + + return CompareImpl(rhs); +} + +// Does 'this' cord start/end with rhs +inline bool Cord::StartsWith(const Cord& rhs) const { + if (contents_.IsSame(rhs.contents_)) return true; + size_t rhs_size = rhs.size(); + if (size() < rhs_size) return false; + return EqualsImpl(rhs, rhs_size); +} + +inline bool Cord::StartsWith(absl::string_view rhs) const { + size_t rhs_size = rhs.size(); + if (size() < rhs_size) return false; + return EqualsImpl(rhs, rhs_size); +} + +inline Cord::ChunkIterator::ChunkIterator(const Cord* cord) + : bytes_remaining_(cord->size()) { + if (cord->empty()) return; + if (cord->contents_.is_tree()) { + stack_of_right_children_.push_back(cord->contents_.tree()); + operator++(); + } else { + current_chunk_ = absl::string_view(cord->contents_.data(), cord->size()); + } +} + +inline Cord::ChunkIterator Cord::ChunkIterator::operator++(int) { + ChunkIterator tmp(*this); + operator++(); + return tmp; +} + +inline bool Cord::ChunkIterator::operator==(const ChunkIterator& other) const { + return bytes_remaining_ == other.bytes_remaining_; +} + +inline bool Cord::ChunkIterator::operator!=(const ChunkIterator& other) const { + return !(*this == other); +} + +inline Cord::ChunkIterator::reference Cord::ChunkIterator::operator*() const { + ABSL_HARDENING_ASSERT(bytes_remaining_ != 0); + return current_chunk_; +} + +inline Cord::ChunkIterator::pointer Cord::ChunkIterator::operator->() const { + ABSL_HARDENING_ASSERT(bytes_remaining_ != 0); + return ¤t_chunk_; +} + +inline void Cord::ChunkIterator::RemoveChunkPrefix(size_t n) { + assert(n < current_chunk_.size()); + current_chunk_.remove_prefix(n); + bytes_remaining_ -= n; +} + +inline void Cord::ChunkIterator::AdvanceBytes(size_t n) { + if (ABSL_PREDICT_TRUE(n < current_chunk_.size())) { + RemoveChunkPrefix(n); + } else if (n != 0) { + AdvanceBytesSlowPath(n); + } +} + +inline Cord::ChunkIterator Cord::chunk_begin() const { + return ChunkIterator(this); +} + +inline Cord::ChunkIterator Cord::chunk_end() const { return ChunkIterator(); } + +inline Cord::ChunkIterator Cord::ChunkRange::begin() const { + return cord_->chunk_begin(); +} + +inline Cord::ChunkIterator Cord::ChunkRange::end() const { + return cord_->chunk_end(); +} + +inline Cord::ChunkRange Cord::Chunks() const { return ChunkRange(this); } + +inline Cord::CharIterator& Cord::CharIterator::operator++() { + if (ABSL_PREDICT_TRUE(chunk_iterator_->size() > 1)) { + chunk_iterator_.RemoveChunkPrefix(1); + } else { + ++chunk_iterator_; + } + return *this; +} + +inline Cord::CharIterator Cord::CharIterator::operator++(int) { + CharIterator tmp(*this); + operator++(); + return tmp; +} + +inline bool Cord::CharIterator::operator==(const CharIterator& other) const { + return chunk_iterator_ == other.chunk_iterator_; +} + +inline bool Cord::CharIterator::operator!=(const CharIterator& other) const { + return !(*this == other); +} + +inline Cord::CharIterator::reference Cord::CharIterator::operator*() const { + return *chunk_iterator_->data(); +} + +inline Cord::CharIterator::pointer Cord::CharIterator::operator->() const { + return chunk_iterator_->data(); +} + +inline Cord Cord::AdvanceAndRead(CharIterator* it, size_t n_bytes) { + assert(it != nullptr); + return it->chunk_iterator_.AdvanceAndReadBytes(n_bytes); +} + +inline void Cord::Advance(CharIterator* it, size_t n_bytes) { + assert(it != nullptr); + it->chunk_iterator_.AdvanceBytes(n_bytes); +} + +inline absl::string_view Cord::ChunkRemaining(const CharIterator& it) { + return *it.chunk_iterator_; +} + +inline Cord::CharIterator Cord::char_begin() const { + return CharIterator(this); +} + +inline Cord::CharIterator Cord::char_end() const { return CharIterator(); } + +inline Cord::CharIterator Cord::CharRange::begin() const { + return cord_->char_begin(); +} + +inline Cord::CharIterator Cord::CharRange::end() const { + return cord_->char_end(); +} + +inline Cord::CharRange Cord::Chars() const { return CharRange(this); } + +inline void Cord::ForEachChunk( + absl::FunctionRef<void(absl::string_view)> callback) const { + absl::cord_internal::CordRep* rep = contents_.tree(); + if (rep == nullptr) { + callback(absl::string_view(contents_.data(), contents_.size())); + } else { + return ForEachChunkAux(rep, callback); + } +} + +// Nonmember Cord-to-Cord relational operarators. +inline bool operator==(const Cord& lhs, const Cord& rhs) { + if (lhs.contents_.IsSame(rhs.contents_)) return true; + size_t rhs_size = rhs.size(); + if (lhs.size() != rhs_size) return false; + return lhs.EqualsImpl(rhs, rhs_size); +} + +inline bool operator!=(const Cord& x, const Cord& y) { return !(x == y); } +inline bool operator<(const Cord& x, const Cord& y) { + return x.Compare(y) < 0; +} +inline bool operator>(const Cord& x, const Cord& y) { + return x.Compare(y) > 0; +} +inline bool operator<=(const Cord& x, const Cord& y) { + return x.Compare(y) <= 0; +} +inline bool operator>=(const Cord& x, const Cord& y) { + return x.Compare(y) >= 0; +} + +// Nonmember Cord-to-absl::string_view relational operators. +// +// Due to implicit conversions, these also enable comparisons of Cord with +// with std::string, ::string, and const char*. +inline bool operator==(const Cord& lhs, absl::string_view rhs) { + size_t lhs_size = lhs.size(); + size_t rhs_size = rhs.size(); + if (lhs_size != rhs_size) return false; + return lhs.EqualsImpl(rhs, rhs_size); +} + +inline bool operator==(absl::string_view x, const Cord& y) { return y == x; } +inline bool operator!=(const Cord& x, absl::string_view y) { return !(x == y); } +inline bool operator!=(absl::string_view x, const Cord& y) { return !(x == y); } +inline bool operator<(const Cord& x, absl::string_view y) { + return x.Compare(y) < 0; +} +inline bool operator<(absl::string_view x, const Cord& y) { + return y.Compare(x) > 0; +} +inline bool operator>(const Cord& x, absl::string_view y) { return y < x; } +inline bool operator>(absl::string_view x, const Cord& y) { return y < x; } +inline bool operator<=(const Cord& x, absl::string_view y) { return !(y < x); } +inline bool operator<=(absl::string_view x, const Cord& y) { return !(y < x); } +inline bool operator>=(const Cord& x, absl::string_view y) { return !(x < y); } +inline bool operator>=(absl::string_view x, const Cord& y) { return !(x < y); } + +// Some internals exposed to test code. +namespace strings_internal { +class CordTestAccess { + public: + static size_t FlatOverhead(); + static size_t MaxFlatLength(); + static size_t SizeofCordRepConcat(); + static size_t SizeofCordRepExternal(); + static size_t SizeofCordRepSubstring(); + static size_t FlatTagToLength(uint8_t tag); + static uint8_t LengthToTag(size_t s); +}; +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORD_H_ diff --git a/third_party/abseil_cpp/absl/strings/cord_test.cc b/third_party/abseil_cpp/absl/strings/cord_test.cc new file mode 100644 index 000000000000..7942bfc03c49 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/cord_test.cc @@ -0,0 +1,1711 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/cord.h" + +#include <algorithm> +#include <climits> +#include <cstdio> +#include <iterator> +#include <map> +#include <numeric> +#include <random> +#include <sstream> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/casts.h" +#include "absl/base/config.h" +#include "absl/base/internal/endian.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/container/fixed_array.h" +#include "absl/strings/cord_test_helpers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" + +typedef std::mt19937_64 RandomEngine; + +static std::string RandomLowercaseString(RandomEngine* rng); +static std::string RandomLowercaseString(RandomEngine* rng, size_t length); + +static int GetUniformRandomUpTo(RandomEngine* rng, int upper_bound) { + if (upper_bound > 0) { + std::uniform_int_distribution<int> uniform(0, upper_bound - 1); + return uniform(*rng); + } else { + return 0; + } +} + +static size_t GetUniformRandomUpTo(RandomEngine* rng, size_t upper_bound) { + if (upper_bound > 0) { + std::uniform_int_distribution<size_t> uniform(0, upper_bound - 1); + return uniform(*rng); + } else { + return 0; + } +} + +static int32_t GenerateSkewedRandom(RandomEngine* rng, int max_log) { + const uint32_t base = (*rng)() % (max_log + 1); + const uint32_t mask = ((base < 32) ? (1u << base) : 0u) - 1u; + return (*rng)() & mask; +} + +static std::string RandomLowercaseString(RandomEngine* rng) { + int length; + std::bernoulli_distribution one_in_1k(0.001); + std::bernoulli_distribution one_in_10k(0.0001); + // With low probability, make a large fragment + if (one_in_10k(*rng)) { + length = GetUniformRandomUpTo(rng, 1048576); + } else if (one_in_1k(*rng)) { + length = GetUniformRandomUpTo(rng, 10000); + } else { + length = GenerateSkewedRandom(rng, 10); + } + return RandomLowercaseString(rng, length); +} + +static std::string RandomLowercaseString(RandomEngine* rng, size_t length) { + std::string result(length, '\0'); + std::uniform_int_distribution<int> chars('a', 'z'); + std::generate(result.begin(), result.end(), + [&]() { return static_cast<char>(chars(*rng)); }); + return result; +} + +static void DoNothing(absl::string_view /* data */, void* /* arg */) {} + +static void DeleteExternalString(absl::string_view data, void* arg) { + std::string* s = reinterpret_cast<std::string*>(arg); + EXPECT_EQ(data, *s); + delete s; +} + +// Add "s" to *dst via `MakeCordFromExternal` +static void AddExternalMemory(absl::string_view s, absl::Cord* dst) { + std::string* str = new std::string(s.data(), s.size()); + dst->Append(absl::MakeCordFromExternal(*str, [str](absl::string_view data) { + DeleteExternalString(data, str); + })); +} + +static void DumpGrowth() { + absl::Cord str; + for (int i = 0; i < 1000; i++) { + char c = 'a' + i % 26; + str.Append(absl::string_view(&c, 1)); + } +} + +// Make a Cord with some number of fragments. Return the size (in bytes) +// of the smallest fragment. +static size_t AppendWithFragments(const std::string& s, RandomEngine* rng, + absl::Cord* cord) { + size_t j = 0; + const size_t max_size = s.size() / 5; // Make approx. 10 fragments + size_t min_size = max_size; // size of smallest fragment + while (j < s.size()) { + size_t N = 1 + GetUniformRandomUpTo(rng, max_size); + if (N > (s.size() - j)) { + N = s.size() - j; + } + if (N < min_size) { + min_size = N; + } + + std::bernoulli_distribution coin_flip(0.5); + if (coin_flip(*rng)) { + // Grow by adding an external-memory. + AddExternalMemory(absl::string_view(s.data() + j, N), cord); + } else { + cord->Append(absl::string_view(s.data() + j, N)); + } + j += N; + } + return min_size; +} + +// Add an external memory that contains the specified std::string to cord +static void AddNewStringBlock(const std::string& str, absl::Cord* dst) { + char* data = new char[str.size()]; + memcpy(data, str.data(), str.size()); + dst->Append(absl::MakeCordFromExternal( + absl::string_view(data, str.size()), + [](absl::string_view s) { delete[] s.data(); })); +} + +// Make a Cord out of many different types of nodes. +static absl::Cord MakeComposite() { + absl::Cord cord; + cord.Append("the"); + AddExternalMemory(" quick brown", &cord); + AddExternalMemory(" fox jumped", &cord); + + absl::Cord full(" over"); + AddExternalMemory(" the lazy", &full); + AddNewStringBlock(" dog slept the whole day away", &full); + absl::Cord substring = full.Subcord(0, 18); + + // Make substring long enough to defeat the copying fast path in Append. + substring.Append(std::string(1000, '.')); + cord.Append(substring); + cord = cord.Subcord(0, cord.size() - 998); // Remove most of extra junk + + return cord; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class CordTestPeer { + public: + static void ForEachChunk( + const Cord& c, absl::FunctionRef<void(absl::string_view)> callback) { + c.ForEachChunk(callback); + } + + static bool IsTree(const Cord& c) { return c.contents_.is_tree(); } +}; + +ABSL_NAMESPACE_END +} // namespace absl + +TEST(Cord, AllFlatSizes) { + using absl::strings_internal::CordTestAccess; + + for (size_t s = 0; s < CordTestAccess::MaxFlatLength(); s++) { + // Make a string of length s. + std::string src; + while (src.size() < s) { + src.push_back('a' + (src.size() % 26)); + } + + absl::Cord dst(src); + EXPECT_EQ(std::string(dst), src) << s; + } +} + +// We create a Cord at least 128GB in size using the fact that Cords can +// internally reference-count; thus the Cord is enormous without actually +// consuming very much memory. +TEST(GigabyteCord, FromExternal) { + const size_t one_gig = 1024U * 1024U * 1024U; + size_t max_size = 2 * one_gig; + if (sizeof(max_size) > 4) max_size = 128 * one_gig; + + size_t length = 128 * 1024; + char* data = new char[length]; + absl::Cord from = absl::MakeCordFromExternal( + absl::string_view(data, length), + [](absl::string_view sv) { delete[] sv.data(); }); + + // This loop may seem odd due to its combination of exponential doubling of + // size and incremental size increases. We do it incrementally to be sure the + // Cord will need rebalancing and will exercise code that, in the past, has + // caused crashes in production. We grow exponentially so that the code will + // execute in a reasonable amount of time. + absl::Cord c; + ABSL_RAW_LOG(INFO, "Made a Cord with %zu bytes!", c.size()); + c.Append(from); + while (c.size() < max_size) { + c.Append(c); + c.Append(from); + c.Append(from); + c.Append(from); + c.Append(from); + } + + for (int i = 0; i < 1024; ++i) { + c.Append(from); + } + ABSL_RAW_LOG(INFO, "Made a Cord with %zu bytes!", c.size()); + // Note: on a 32-bit build, this comes out to 2,818,048,000 bytes. + // Note: on a 64-bit build, this comes out to 171,932,385,280 bytes. +} + +static absl::Cord MakeExternalCord(int size) { + char* buffer = new char[size]; + memset(buffer, 'x', size); + absl::Cord cord; + cord.Append(absl::MakeCordFromExternal( + absl::string_view(buffer, size), + [](absl::string_view s) { delete[] s.data(); })); + return cord; +} + +// Extern to fool clang that this is not constant. Needed to suppress +// a warning of unsafe code we want to test. +extern bool my_unique_true_boolean; +bool my_unique_true_boolean = true; + +TEST(Cord, Assignment) { + absl::Cord x(absl::string_view("hi there")); + absl::Cord y(x); + ASSERT_EQ(std::string(x), "hi there"); + ASSERT_EQ(std::string(y), "hi there"); + ASSERT_TRUE(x == y); + ASSERT_TRUE(x <= y); + ASSERT_TRUE(y <= x); + + x = absl::string_view("foo"); + ASSERT_EQ(std::string(x), "foo"); + ASSERT_EQ(std::string(y), "hi there"); + ASSERT_TRUE(x < y); + ASSERT_TRUE(y > x); + ASSERT_TRUE(x != y); + ASSERT_TRUE(x <= y); + ASSERT_TRUE(y >= x); + + x = "foo"; + ASSERT_EQ(x, "foo"); + + // Test that going from inline rep to tree we don't leak memory. + std::vector<std::pair<absl::string_view, absl::string_view>> + test_string_pairs = {{"hi there", "foo"}, + {"loooooong coooooord", "short cord"}, + {"short cord", "loooooong coooooord"}, + {"loooooong coooooord1", "loooooong coooooord2"}}; + for (std::pair<absl::string_view, absl::string_view> test_strings : + test_string_pairs) { + absl::Cord tmp(test_strings.first); + absl::Cord z(std::move(tmp)); + ASSERT_EQ(std::string(z), test_strings.first); + tmp = test_strings.second; + z = std::move(tmp); + ASSERT_EQ(std::string(z), test_strings.second); + } + { + // Test that self-move assignment doesn't crash/leak. + // Do not write such code! + absl::Cord my_small_cord("foo"); + absl::Cord my_big_cord("loooooong coooooord"); + // Bypass clang's warning on self move-assignment. + absl::Cord* my_small_alias = + my_unique_true_boolean ? &my_small_cord : &my_big_cord; + absl::Cord* my_big_alias = + !my_unique_true_boolean ? &my_small_cord : &my_big_cord; + + *my_small_alias = std::move(my_small_cord); + *my_big_alias = std::move(my_big_cord); + // my_small_cord and my_big_cord are in an unspecified but valid + // state, and will be correctly destroyed here. + } +} + +TEST(Cord, StartsEndsWith) { + absl::Cord x(absl::string_view("abcde")); + absl::Cord empty(""); + + ASSERT_TRUE(x.StartsWith(absl::Cord("abcde"))); + ASSERT_TRUE(x.StartsWith(absl::Cord("abc"))); + ASSERT_TRUE(x.StartsWith(absl::Cord(""))); + ASSERT_TRUE(empty.StartsWith(absl::Cord(""))); + ASSERT_TRUE(x.EndsWith(absl::Cord("abcde"))); + ASSERT_TRUE(x.EndsWith(absl::Cord("cde"))); + ASSERT_TRUE(x.EndsWith(absl::Cord(""))); + ASSERT_TRUE(empty.EndsWith(absl::Cord(""))); + + ASSERT_TRUE(!x.StartsWith(absl::Cord("xyz"))); + ASSERT_TRUE(!empty.StartsWith(absl::Cord("xyz"))); + ASSERT_TRUE(!x.EndsWith(absl::Cord("xyz"))); + ASSERT_TRUE(!empty.EndsWith(absl::Cord("xyz"))); + + ASSERT_TRUE(x.StartsWith("abcde")); + ASSERT_TRUE(x.StartsWith("abc")); + ASSERT_TRUE(x.StartsWith("")); + ASSERT_TRUE(empty.StartsWith("")); + ASSERT_TRUE(x.EndsWith("abcde")); + ASSERT_TRUE(x.EndsWith("cde")); + ASSERT_TRUE(x.EndsWith("")); + ASSERT_TRUE(empty.EndsWith("")); + + ASSERT_TRUE(!x.StartsWith("xyz")); + ASSERT_TRUE(!empty.StartsWith("xyz")); + ASSERT_TRUE(!x.EndsWith("xyz")); + ASSERT_TRUE(!empty.EndsWith("xyz")); +} + +TEST(Cord, Subcord) { + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + const std::string s = RandomLowercaseString(&rng, 1024); + + absl::Cord a; + AppendWithFragments(s, &rng, &a); + ASSERT_EQ(s.size(), a.size()); + + // Check subcords of a, from a variety of interesting points. + std::set<size_t> positions; + for (int i = 0; i <= 32; ++i) { + positions.insert(i); + positions.insert(i * 32 - 1); + positions.insert(i * 32); + positions.insert(i * 32 + 1); + positions.insert(a.size() - i); + } + positions.insert(237); + positions.insert(732); + for (size_t pos : positions) { + if (pos > a.size()) continue; + for (size_t end_pos : positions) { + if (end_pos < pos || end_pos > a.size()) continue; + absl::Cord sa = a.Subcord(pos, end_pos - pos); + EXPECT_EQ(absl::string_view(s).substr(pos, end_pos - pos), + std::string(sa)) + << a; + } + } + + // Do the same thing for an inline cord. + const std::string sh = "short"; + absl::Cord c(sh); + for (size_t pos = 0; pos <= sh.size(); ++pos) { + for (size_t n = 0; n <= sh.size() - pos; ++n) { + absl::Cord sc = c.Subcord(pos, n); + EXPECT_EQ(sh.substr(pos, n), std::string(sc)) << c; + } + } + + // Check subcords of subcords. + absl::Cord sa = a.Subcord(0, a.size()); + std::string ss = s.substr(0, s.size()); + while (sa.size() > 1) { + sa = sa.Subcord(1, sa.size() - 2); + ss = ss.substr(1, ss.size() - 2); + EXPECT_EQ(ss, std::string(sa)) << a; + if (HasFailure()) break; // halt cascade + } + + // It is OK to ask for too much. + sa = a.Subcord(0, a.size() + 1); + EXPECT_EQ(s, std::string(sa)); + + // It is OK to ask for something beyond the end. + sa = a.Subcord(a.size() + 1, 0); + EXPECT_TRUE(sa.empty()); + sa = a.Subcord(a.size() + 1, 1); + EXPECT_TRUE(sa.empty()); +} + +TEST(Cord, Swap) { + absl::string_view a("Dexter"); + absl::string_view b("Mandark"); + absl::Cord x(a); + absl::Cord y(b); + swap(x, y); + ASSERT_EQ(x, absl::Cord(b)); + ASSERT_EQ(y, absl::Cord(a)); + x.swap(y); + ASSERT_EQ(x, absl::Cord(a)); + ASSERT_EQ(y, absl::Cord(b)); +} + +static void VerifyCopyToString(const absl::Cord& cord) { + std::string initially_empty; + absl::CopyCordToString(cord, &initially_empty); + EXPECT_EQ(initially_empty, cord); + + constexpr size_t kInitialLength = 1024; + std::string has_initial_contents(kInitialLength, 'x'); + const char* address_before_copy = has_initial_contents.data(); + absl::CopyCordToString(cord, &has_initial_contents); + EXPECT_EQ(has_initial_contents, cord); + + if (cord.size() <= kInitialLength) { + EXPECT_EQ(has_initial_contents.data(), address_before_copy) + << "CopyCordToString allocated new string storage; " + "has_initial_contents = \"" + << has_initial_contents << "\""; + } +} + +TEST(Cord, CopyToString) { + VerifyCopyToString(absl::Cord()); + VerifyCopyToString(absl::Cord("small cord")); + VerifyCopyToString( + absl::MakeFragmentedCord({"fragmented ", "cord ", "to ", "test ", + "copying ", "to ", "a ", "string."})); +} + +TEST(TryFlat, Empty) { + absl::Cord c; + EXPECT_EQ(c.TryFlat(), ""); +} + +TEST(TryFlat, Flat) { + absl::Cord c("hello"); + EXPECT_EQ(c.TryFlat(), "hello"); +} + +TEST(TryFlat, SubstrInlined) { + absl::Cord c("hello"); + c.RemovePrefix(1); + EXPECT_EQ(c.TryFlat(), "ello"); +} + +TEST(TryFlat, SubstrFlat) { + absl::Cord c("longer than 15 bytes"); + c.RemovePrefix(1); + EXPECT_EQ(c.TryFlat(), "onger than 15 bytes"); +} + +TEST(TryFlat, Concat) { + absl::Cord c = absl::MakeFragmentedCord({"hel", "lo"}); + EXPECT_EQ(c.TryFlat(), absl::nullopt); +} + +TEST(TryFlat, External) { + absl::Cord c = absl::MakeCordFromExternal("hell", [](absl::string_view) {}); + EXPECT_EQ(c.TryFlat(), "hell"); +} + +TEST(TryFlat, SubstrExternal) { + absl::Cord c = absl::MakeCordFromExternal("hell", [](absl::string_view) {}); + c.RemovePrefix(1); + EXPECT_EQ(c.TryFlat(), "ell"); +} + +TEST(TryFlat, SubstrConcat) { + absl::Cord c = absl::MakeFragmentedCord({"hello", " world"}); + c.RemovePrefix(1); + EXPECT_EQ(c.TryFlat(), absl::nullopt); +} + +static bool IsFlat(const absl::Cord& c) { + return c.chunk_begin() == c.chunk_end() || ++c.chunk_begin() == c.chunk_end(); +} + +static void VerifyFlatten(absl::Cord c) { + std::string old_contents(c); + absl::string_view old_flat; + bool already_flat_and_non_empty = IsFlat(c) && !c.empty(); + if (already_flat_and_non_empty) { + old_flat = *c.chunk_begin(); + } + absl::string_view new_flat = c.Flatten(); + + // Verify that the contents of the flattened Cord are correct. + EXPECT_EQ(new_flat, old_contents); + EXPECT_EQ(std::string(c), old_contents); + + // If the Cord contained data and was already flat, verify that the data + // wasn't copied. + if (already_flat_and_non_empty) { + EXPECT_EQ(old_flat.data(), new_flat.data()) + << "Allocated new memory even though the Cord was already flat."; + } + + // Verify that the flattened Cord is in fact flat. + EXPECT_TRUE(IsFlat(c)); +} + +TEST(Cord, Flatten) { + VerifyFlatten(absl::Cord()); + VerifyFlatten(absl::Cord("small cord")); + VerifyFlatten(absl::Cord("larger than small buffer optimization")); + VerifyFlatten(absl::MakeFragmentedCord({"small ", "fragmented ", "cord"})); + + // Test with a cord that is longer than the largest flat buffer + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + VerifyFlatten(absl::Cord(RandomLowercaseString(&rng, 8192))); +} + +// Test data +namespace { +class TestData { + private: + std::vector<std::string> data_; + + // Return a std::string of the specified length. + static std::string MakeString(int length) { + std::string result; + char buf[30]; + snprintf(buf, sizeof(buf), "(%d)", length); + while (result.size() < length) { + result += buf; + } + result.resize(length); + return result; + } + + public: + TestData() { + // short strings increasing in length by one + for (int i = 0; i < 30; i++) { + data_.push_back(MakeString(i)); + } + + // strings around half kMaxFlatLength + static const int kMaxFlatLength = 4096 - 9; + static const int kHalf = kMaxFlatLength / 2; + + for (int i = -10; i <= +10; i++) { + data_.push_back(MakeString(kHalf + i)); + } + + for (int i = -10; i <= +10; i++) { + data_.push_back(MakeString(kMaxFlatLength + i)); + } + } + + size_t size() const { return data_.size(); } + const std::string& data(size_t i) const { return data_[i]; } +}; +} // namespace + +TEST(Cord, MultipleLengths) { + TestData d; + for (size_t i = 0; i < d.size(); i++) { + std::string a = d.data(i); + + { // Construct from Cord + absl::Cord tmp(a); + absl::Cord x(tmp); + EXPECT_EQ(a, std::string(x)) << "'" << a << "'"; + } + + { // Construct from absl::string_view + absl::Cord x(a); + EXPECT_EQ(a, std::string(x)) << "'" << a << "'"; + } + + { // Append cord to self + absl::Cord self(a); + self.Append(self); + EXPECT_EQ(a + a, std::string(self)) << "'" << a << "' + '" << a << "'"; + } + + { // Prepend cord to self + absl::Cord self(a); + self.Prepend(self); + EXPECT_EQ(a + a, std::string(self)) << "'" << a << "' + '" << a << "'"; + } + + // Try to append/prepend others + for (size_t j = 0; j < d.size(); j++) { + std::string b = d.data(j); + + { // CopyFrom Cord + absl::Cord x(a); + absl::Cord y(b); + x = y; + EXPECT_EQ(b, std::string(x)) << "'" << a << "' + '" << b << "'"; + } + + { // CopyFrom absl::string_view + absl::Cord x(a); + x = b; + EXPECT_EQ(b, std::string(x)) << "'" << a << "' + '" << b << "'"; + } + + { // Cord::Append(Cord) + absl::Cord x(a); + absl::Cord y(b); + x.Append(y); + EXPECT_EQ(a + b, std::string(x)) << "'" << a << "' + '" << b << "'"; + } + + { // Cord::Append(absl::string_view) + absl::Cord x(a); + x.Append(b); + EXPECT_EQ(a + b, std::string(x)) << "'" << a << "' + '" << b << "'"; + } + + { // Cord::Prepend(Cord) + absl::Cord x(a); + absl::Cord y(b); + x.Prepend(y); + EXPECT_EQ(b + a, std::string(x)) << "'" << b << "' + '" << a << "'"; + } + + { // Cord::Prepend(absl::string_view) + absl::Cord x(a); + x.Prepend(b); + EXPECT_EQ(b + a, std::string(x)) << "'" << b << "' + '" << a << "'"; + } + } + } +} + +namespace { + +TEST(Cord, RemoveSuffixWithExternalOrSubstring) { + absl::Cord cord = absl::MakeCordFromExternal( + "foo bar baz", [](absl::string_view s) { DoNothing(s, nullptr); }); + + EXPECT_EQ("foo bar baz", std::string(cord)); + + // This RemoveSuffix() will wrap the EXTERNAL node in a SUBSTRING node. + cord.RemoveSuffix(4); + EXPECT_EQ("foo bar", std::string(cord)); + + // This RemoveSuffix() will adjust the SUBSTRING node in-place. + cord.RemoveSuffix(4); + EXPECT_EQ("foo", std::string(cord)); +} + +TEST(Cord, RemoveSuffixMakesZeroLengthNode) { + absl::Cord c; + c.Append(absl::Cord(std::string(100, 'x'))); + absl::Cord other_ref = c; // Prevent inplace appends + c.Append(absl::Cord(std::string(200, 'y'))); + c.RemoveSuffix(200); + EXPECT_EQ(std::string(100, 'x'), std::string(c)); +} + +} // namespace + +// CordSpliceTest contributed by hendrie. +namespace { + +// Create a cord with an external memory block filled with 'z' +absl::Cord CordWithZedBlock(size_t size) { + char* data = new char[size]; + if (size > 0) { + memset(data, 'z', size); + } + absl::Cord cord = absl::MakeCordFromExternal( + absl::string_view(data, size), + [](absl::string_view s) { delete[] s.data(); }); + return cord; +} + +// Establish that ZedBlock does what we think it does. +TEST(CordSpliceTest, ZedBlock) { + absl::Cord blob = CordWithZedBlock(10); + EXPECT_EQ(10, blob.size()); + std::string s; + absl::CopyCordToString(blob, &s); + EXPECT_EQ("zzzzzzzzzz", s); +} + +TEST(CordSpliceTest, ZedBlock0) { + absl::Cord blob = CordWithZedBlock(0); + EXPECT_EQ(0, blob.size()); + std::string s; + absl::CopyCordToString(blob, &s); + EXPECT_EQ("", s); +} + +TEST(CordSpliceTest, ZedBlockSuffix1) { + absl::Cord blob = CordWithZedBlock(10); + EXPECT_EQ(10, blob.size()); + absl::Cord suffix(blob); + suffix.RemovePrefix(9); + EXPECT_EQ(1, suffix.size()); + std::string s; + absl::CopyCordToString(suffix, &s); + EXPECT_EQ("z", s); +} + +// Remove all of a prefix block +TEST(CordSpliceTest, ZedBlockSuffix0) { + absl::Cord blob = CordWithZedBlock(10); + EXPECT_EQ(10, blob.size()); + absl::Cord suffix(blob); + suffix.RemovePrefix(10); + EXPECT_EQ(0, suffix.size()); + std::string s; + absl::CopyCordToString(suffix, &s); + EXPECT_EQ("", s); +} + +absl::Cord BigCord(size_t len, char v) { + std::string s(len, v); + return absl::Cord(s); +} + +// Splice block into cord. +absl::Cord SpliceCord(const absl::Cord& blob, int64_t offset, + const absl::Cord& block) { + ABSL_RAW_CHECK(offset >= 0, ""); + ABSL_RAW_CHECK(offset + block.size() <= blob.size(), ""); + absl::Cord result(blob); + result.RemoveSuffix(blob.size() - offset); + result.Append(block); + absl::Cord suffix(blob); + suffix.RemovePrefix(offset + block.size()); + result.Append(suffix); + ABSL_RAW_CHECK(blob.size() == result.size(), ""); + return result; +} + +// Taking an empty suffix of a block breaks appending. +TEST(CordSpliceTest, RemoveEntireBlock1) { + absl::Cord zero = CordWithZedBlock(10); + absl::Cord suffix(zero); + suffix.RemovePrefix(10); + absl::Cord result; + result.Append(suffix); +} + +TEST(CordSpliceTest, RemoveEntireBlock2) { + absl::Cord zero = CordWithZedBlock(10); + absl::Cord prefix(zero); + prefix.RemoveSuffix(10); + absl::Cord suffix(zero); + suffix.RemovePrefix(10); + absl::Cord result(prefix); + result.Append(suffix); +} + +TEST(CordSpliceTest, RemoveEntireBlock3) { + absl::Cord blob = CordWithZedBlock(10); + absl::Cord block = BigCord(10, 'b'); + blob = SpliceCord(blob, 0, block); +} + +struct CordCompareTestCase { + template <typename LHS, typename RHS> + CordCompareTestCase(const LHS& lhs, const RHS& rhs) + : lhs_cord(lhs), rhs_cord(rhs) {} + + absl::Cord lhs_cord; + absl::Cord rhs_cord; +}; + +const auto sign = [](int x) { return x == 0 ? 0 : (x > 0 ? 1 : -1); }; + +void VerifyComparison(const CordCompareTestCase& test_case) { + std::string lhs_string(test_case.lhs_cord); + std::string rhs_string(test_case.rhs_cord); + int expected = sign(lhs_string.compare(rhs_string)); + EXPECT_EQ(expected, test_case.lhs_cord.Compare(test_case.rhs_cord)) + << "LHS=" << lhs_string << "; RHS=" << rhs_string; + EXPECT_EQ(expected, test_case.lhs_cord.Compare(rhs_string)) + << "LHS=" << lhs_string << "; RHS=" << rhs_string; + EXPECT_EQ(-expected, test_case.rhs_cord.Compare(test_case.lhs_cord)) + << "LHS=" << rhs_string << "; RHS=" << lhs_string; + EXPECT_EQ(-expected, test_case.rhs_cord.Compare(lhs_string)) + << "LHS=" << rhs_string << "; RHS=" << lhs_string; +} + +TEST(Cord, Compare) { + absl::Cord subcord("aaaaaBBBBBcccccDDDDD"); + subcord = subcord.Subcord(3, 10); + + absl::Cord tmp("aaaaaaaaaaaaaaaa"); + tmp.Append("BBBBBBBBBBBBBBBB"); + absl::Cord concat = absl::Cord("cccccccccccccccc"); + concat.Append("DDDDDDDDDDDDDDDD"); + concat.Prepend(tmp); + + absl::Cord concat2("aaaaaaaaaaaaa"); + concat2.Append("aaaBBBBBBBBBBBBBBBBccccc"); + concat2.Append("cccccccccccDDDDDDDDDDDDDD"); + concat2.Append("DD"); + + std::vector<CordCompareTestCase> test_cases = {{ + // Inline cords + {"abcdef", "abcdef"}, + {"abcdef", "abcdee"}, + {"abcdef", "abcdeg"}, + {"bbcdef", "abcdef"}, + {"bbcdef", "abcdeg"}, + {"abcdefa", "abcdef"}, + {"abcdef", "abcdefa"}, + + // Small flat cords + {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBcccccDDDDD"}, + {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBxccccDDDDD"}, + {"aaaaaBBBBBcxcccDDDDD", "aaaaaBBBBBcccccDDDDD"}, + {"aaaaaBBBBBxccccDDDDD", "aaaaaBBBBBcccccDDDDX"}, + {"aaaaaBBBBBcccccDDDDDa", "aaaaaBBBBBcccccDDDDD"}, + {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBcccccDDDDDa"}, + + // Subcords + {subcord, subcord}, + {subcord, "aaBBBBBccc"}, + {subcord, "aaBBBBBccd"}, + {subcord, "aaBBBBBccb"}, + {subcord, "aaBBBBBxcb"}, + {subcord, "aaBBBBBccca"}, + {subcord, "aaBBBBBcc"}, + + // Concats + {concat, concat}, + {concat, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDDD"}, + {concat, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBcccccccccccccccxDDDDDDDDDDDDDDDD"}, + {concat, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBacccccccccccccccDDDDDDDDDDDDDDDD"}, + {concat, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDD"}, + {concat, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDDDe"}, + + {concat, concat2}, + }}; + + for (const auto& tc : test_cases) { + VerifyComparison(tc); + } +} + +TEST(Cord, CompareAfterAssign) { + absl::Cord a("aaaaaa1111111"); + absl::Cord b("aaaaaa2222222"); + a = "cccccc"; + b = "cccccc"; + EXPECT_EQ(a, b); + EXPECT_FALSE(a < b); + + a = "aaaa"; + b = "bbbbb"; + a = ""; + b = ""; + EXPECT_EQ(a, b); + EXPECT_FALSE(a < b); +} + +// Test CompareTo() and ComparePrefix() against string and substring +// comparison methods from basic_string. +static void TestCompare(const absl::Cord& c, const absl::Cord& d, + RandomEngine* rng) { + typedef std::basic_string<uint8_t> ustring; + ustring cs(reinterpret_cast<const uint8_t*>(std::string(c).data()), c.size()); + ustring ds(reinterpret_cast<const uint8_t*>(std::string(d).data()), d.size()); + // ustring comparison is ideal because we expect Cord comparisons to be + // based on unsigned byte comparisons regardless of whether char is signed. + int expected = sign(cs.compare(ds)); + EXPECT_EQ(expected, sign(c.Compare(d))) << c << ", " << d; +} + +TEST(Compare, ComparisonIsUnsigned) { + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + std::uniform_int_distribution<uint32_t> uniform_uint8(0, 255); + char x = static_cast<char>(uniform_uint8(rng)); + TestCompare( + absl::Cord(std::string(GetUniformRandomUpTo(&rng, 100), x)), + absl::Cord(std::string(GetUniformRandomUpTo(&rng, 100), x ^ 0x80)), &rng); +} + +TEST(Compare, RandomComparisons) { + const int kIters = 5000; + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + + int n = GetUniformRandomUpTo(&rng, 5000); + absl::Cord a[] = {MakeExternalCord(n), + absl::Cord("ant"), + absl::Cord("elephant"), + absl::Cord("giraffe"), + absl::Cord(std::string(GetUniformRandomUpTo(&rng, 100), + GetUniformRandomUpTo(&rng, 100))), + absl::Cord(""), + absl::Cord("x"), + absl::Cord("A"), + absl::Cord("B"), + absl::Cord("C")}; + for (int i = 0; i < kIters; i++) { + absl::Cord c, d; + for (int j = 0; j < (i % 7) + 1; j++) { + c.Append(a[GetUniformRandomUpTo(&rng, ABSL_ARRAYSIZE(a))]); + d.Append(a[GetUniformRandomUpTo(&rng, ABSL_ARRAYSIZE(a))]); + } + std::bernoulli_distribution coin_flip(0.5); + TestCompare(coin_flip(rng) ? c : absl::Cord(std::string(c)), + coin_flip(rng) ? d : absl::Cord(std::string(d)), &rng); + } +} + +template <typename T1, typename T2> +void CompareOperators() { + const T1 a("a"); + const T2 b("b"); + + EXPECT_TRUE(a == a); + // For pointer type (i.e. `const char*`), operator== compares the address + // instead of the string, so `a == const char*("a")` isn't necessarily true. + EXPECT_TRUE(std::is_pointer<T1>::value || a == T1("a")); + EXPECT_TRUE(std::is_pointer<T2>::value || a == T2("a")); + EXPECT_FALSE(a == b); + + EXPECT_TRUE(a != b); + EXPECT_FALSE(a != a); + + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); + + EXPECT_TRUE(b > a); + EXPECT_FALSE(a > b); + + EXPECT_TRUE(a >= a); + EXPECT_TRUE(b >= a); + EXPECT_FALSE(a >= b); + + EXPECT_TRUE(a <= a); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(b <= a); +} + +TEST(ComparisonOperators, Cord_Cord) { + CompareOperators<absl::Cord, absl::Cord>(); +} + +TEST(ComparisonOperators, Cord_StringPiece) { + CompareOperators<absl::Cord, absl::string_view>(); +} + +TEST(ComparisonOperators, StringPiece_Cord) { + CompareOperators<absl::string_view, absl::Cord>(); +} + +TEST(ComparisonOperators, Cord_string) { + CompareOperators<absl::Cord, std::string>(); +} + +TEST(ComparisonOperators, string_Cord) { + CompareOperators<std::string, absl::Cord>(); +} + +TEST(ComparisonOperators, stdstring_Cord) { + CompareOperators<std::string, absl::Cord>(); +} + +TEST(ComparisonOperators, Cord_stdstring) { + CompareOperators<absl::Cord, std::string>(); +} + +TEST(ComparisonOperators, charstar_Cord) { + CompareOperators<const char*, absl::Cord>(); +} + +TEST(ComparisonOperators, Cord_charstar) { + CompareOperators<absl::Cord, const char*>(); +} + +TEST(ConstructFromExternal, ReleaserInvoked) { + // Empty external memory means the releaser should be called immediately. + { + bool invoked = false; + auto releaser = [&invoked](absl::string_view) { invoked = true; }; + { + auto c = absl::MakeCordFromExternal("", releaser); + EXPECT_TRUE(invoked); + } + } + + // If the size of the data is small enough, a future constructor + // implementation may copy the bytes and immediately invoke the releaser + // instead of creating an external node. We make a large dummy std::string to + // make this test independent of such an optimization. + std::string large_dummy(2048, 'c'); + { + bool invoked = false; + auto releaser = [&invoked](absl::string_view) { invoked = true; }; + { + auto c = absl::MakeCordFromExternal(large_dummy, releaser); + EXPECT_FALSE(invoked); + } + EXPECT_TRUE(invoked); + } + + { + bool invoked = false; + auto releaser = [&invoked](absl::string_view) { invoked = true; }; + { + absl::Cord copy; + { + auto c = absl::MakeCordFromExternal(large_dummy, releaser); + copy = c; + EXPECT_FALSE(invoked); + } + EXPECT_FALSE(invoked); + } + EXPECT_TRUE(invoked); + } +} + +TEST(ConstructFromExternal, CompareContents) { + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + + for (int length = 1; length <= 2048; length *= 2) { + std::string data = RandomLowercaseString(&rng, length); + auto* external = new std::string(data); + auto cord = + absl::MakeCordFromExternal(*external, [external](absl::string_view sv) { + EXPECT_EQ(external->data(), sv.data()); + EXPECT_EQ(external->size(), sv.size()); + delete external; + }); + EXPECT_EQ(data, cord); + } +} + +TEST(ConstructFromExternal, LargeReleaser) { + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + constexpr size_t kLength = 256; + std::string data = RandomLowercaseString(&rng, kLength); + std::array<char, kLength> data_array; + for (size_t i = 0; i < kLength; ++i) data_array[i] = data[i]; + bool invoked = false; + auto releaser = [data_array, &invoked](absl::string_view data) { + EXPECT_EQ(data, absl::string_view(data_array.data(), data_array.size())); + invoked = true; + }; + (void)absl::MakeCordFromExternal(data, releaser); + EXPECT_TRUE(invoked); +} + +TEST(ConstructFromExternal, FunctionPointerReleaser) { + static absl::string_view data("hello world"); + static bool invoked; + auto* releaser = + static_cast<void (*)(absl::string_view)>([](absl::string_view sv) { + EXPECT_EQ(data, sv); + invoked = true; + }); + invoked = false; + (void)absl::MakeCordFromExternal(data, releaser); + EXPECT_TRUE(invoked); + + invoked = false; + (void)absl::MakeCordFromExternal(data, *releaser); + EXPECT_TRUE(invoked); +} + +TEST(ConstructFromExternal, MoveOnlyReleaser) { + struct Releaser { + explicit Releaser(bool* invoked) : invoked(invoked) {} + Releaser(Releaser&& other) noexcept : invoked(other.invoked) {} + void operator()(absl::string_view) const { *invoked = true; } + + bool* invoked; + }; + + bool invoked = false; + (void)absl::MakeCordFromExternal("dummy", Releaser(&invoked)); + EXPECT_TRUE(invoked); +} + +TEST(ConstructFromExternal, NoArgLambda) { + bool invoked = false; + (void)absl::MakeCordFromExternal("dummy", [&invoked]() { invoked = true; }); + EXPECT_TRUE(invoked); +} + +TEST(ConstructFromExternal, StringViewArgLambda) { + bool invoked = false; + (void)absl::MakeCordFromExternal( + "dummy", [&invoked](absl::string_view) { invoked = true; }); + EXPECT_TRUE(invoked); +} + +TEST(ConstructFromExternal, NonTrivialReleaserDestructor) { + struct Releaser { + explicit Releaser(bool* destroyed) : destroyed(destroyed) {} + ~Releaser() { *destroyed = true; } + void operator()(absl::string_view) const {} + + bool* destroyed; + }; + + bool destroyed = false; + Releaser releaser(&destroyed); + (void)absl::MakeCordFromExternal("dummy", releaser); + EXPECT_TRUE(destroyed); +} + +TEST(ConstructFromExternal, ReferenceQualifierOverloads) { + struct Releaser { + void operator()(absl::string_view) & { *lvalue_invoked = true; } + void operator()(absl::string_view) && { *rvalue_invoked = true; } + + bool* lvalue_invoked; + bool* rvalue_invoked; + }; + + bool lvalue_invoked = false; + bool rvalue_invoked = false; + Releaser releaser = {&lvalue_invoked, &rvalue_invoked}; + (void)absl::MakeCordFromExternal("", releaser); + EXPECT_FALSE(lvalue_invoked); + EXPECT_TRUE(rvalue_invoked); + rvalue_invoked = false; + + (void)absl::MakeCordFromExternal("dummy", releaser); + EXPECT_FALSE(lvalue_invoked); + EXPECT_TRUE(rvalue_invoked); + rvalue_invoked = false; + + // NOLINTNEXTLINE: suppress clang-tidy std::move on trivially copyable type. + (void)absl::MakeCordFromExternal("dummy", std::move(releaser)); + EXPECT_FALSE(lvalue_invoked); + EXPECT_TRUE(rvalue_invoked); +} + +TEST(ExternalMemory, BasicUsage) { + static const char* strings[] = {"", "hello", "there"}; + for (const char* str : strings) { + absl::Cord dst("(prefix)"); + AddExternalMemory(str, &dst); + dst.Append("(suffix)"); + EXPECT_EQ((std::string("(prefix)") + str + std::string("(suffix)")), + std::string(dst)); + } +} + +TEST(ExternalMemory, RemovePrefixSuffix) { + // Exhaustively try all sub-strings. + absl::Cord cord = MakeComposite(); + std::string s = std::string(cord); + for (int offset = 0; offset <= s.size(); offset++) { + for (int length = 0; length <= s.size() - offset; length++) { + absl::Cord result(cord); + result.RemovePrefix(offset); + result.RemoveSuffix(result.size() - length); + EXPECT_EQ(s.substr(offset, length), std::string(result)) + << offset << " " << length; + } + } +} + +TEST(ExternalMemory, Get) { + absl::Cord cord("hello"); + AddExternalMemory(" world!", &cord); + AddExternalMemory(" how are ", &cord); + cord.Append(" you?"); + std::string s = std::string(cord); + for (int i = 0; i < s.size(); i++) { + EXPECT_EQ(s[i], cord[i]); + } +} + +// CordMemoryUsage tests verify the correctness of the EstimatedMemoryUsage() +// These tests take into account that the reported memory usage is approximate +// and non-deterministic. For all tests, We verify that the reported memory +// usage is larger than `size()`, and less than `size() * 1.5` as a cord should +// never reserve more 'extra' capacity than half of its size as it grows. +// Additionally we have some whiteboxed expectations based on our knowledge of +// the layout and size of empty and inlined cords, and flat nodes. + +TEST(CordMemoryUsage, Empty) { + EXPECT_EQ(sizeof(absl::Cord), absl::Cord().EstimatedMemoryUsage()); +} + +TEST(CordMemoryUsage, Embedded) { + absl::Cord a("hello"); + EXPECT_EQ(a.EstimatedMemoryUsage(), sizeof(absl::Cord)); +} + +TEST(CordMemoryUsage, EmbeddedAppend) { + absl::Cord a("a"); + absl::Cord b("bcd"); + EXPECT_EQ(b.EstimatedMemoryUsage(), sizeof(absl::Cord)); + a.Append(b); + EXPECT_EQ(a.EstimatedMemoryUsage(), sizeof(absl::Cord)); +} + +TEST(CordMemoryUsage, ExternalMemory) { + static const int kLength = 1000; + absl::Cord cord; + AddExternalMemory(std::string(kLength, 'x'), &cord); + EXPECT_GT(cord.EstimatedMemoryUsage(), kLength); + EXPECT_LE(cord.EstimatedMemoryUsage(), kLength * 1.5); +} + +TEST(CordMemoryUsage, Flat) { + static const int kLength = 125; + absl::Cord a(std::string(kLength, 'a')); + EXPECT_GT(a.EstimatedMemoryUsage(), kLength); + EXPECT_LE(a.EstimatedMemoryUsage(), kLength * 1.5); +} + +TEST(CordMemoryUsage, AppendFlat) { + using absl::strings_internal::CordTestAccess; + absl::Cord a(std::string(CordTestAccess::MaxFlatLength(), 'a')); + size_t length = a.EstimatedMemoryUsage(); + a.Append(std::string(CordTestAccess::MaxFlatLength(), 'b')); + size_t delta = a.EstimatedMemoryUsage() - length; + EXPECT_GT(delta, CordTestAccess::MaxFlatLength()); + EXPECT_LE(delta, CordTestAccess::MaxFlatLength() * 1.5); +} + +// Regtest for a change that had to be rolled back because it expanded out +// of the InlineRep too soon, which was observable through MemoryUsage(). +TEST(CordMemoryUsage, InlineRep) { + constexpr size_t kMaxInline = 15; // Cord::InlineRep::N + const std::string small_string(kMaxInline, 'x'); + absl::Cord c1(small_string); + + absl::Cord c2; + c2.Append(small_string); + EXPECT_EQ(c1, c2); + EXPECT_EQ(c1.EstimatedMemoryUsage(), c2.EstimatedMemoryUsage()); +} + +} // namespace + +// Regtest for 7510292 (fix a bug introduced by 7465150) +TEST(Cord, Concat_Append) { + // Create a rep of type CONCAT + absl::Cord s1("foobarbarbarbarbar"); + s1.Append("abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefg"); + size_t size = s1.size(); + + // Create a copy of s1 and append to it. + absl::Cord s2 = s1; + s2.Append("x"); + + // 7465150 modifies s1 when it shouldn't. + EXPECT_EQ(s1.size(), size); + EXPECT_EQ(s2.size(), size + 1); +} + +TEST(MakeFragmentedCord, MakeFragmentedCordFromInitializerList) { + absl::Cord fragmented = + absl::MakeFragmentedCord({"A ", "fragmented ", "Cord"}); + + EXPECT_EQ("A fragmented Cord", fragmented); + + auto chunk_it = fragmented.chunk_begin(); + + ASSERT_TRUE(chunk_it != fragmented.chunk_end()); + EXPECT_EQ("A ", *chunk_it); + + ASSERT_TRUE(++chunk_it != fragmented.chunk_end()); + EXPECT_EQ("fragmented ", *chunk_it); + + ASSERT_TRUE(++chunk_it != fragmented.chunk_end()); + EXPECT_EQ("Cord", *chunk_it); + + ASSERT_TRUE(++chunk_it == fragmented.chunk_end()); +} + +TEST(MakeFragmentedCord, MakeFragmentedCordFromVector) { + std::vector<absl::string_view> chunks = {"A ", "fragmented ", "Cord"}; + absl::Cord fragmented = absl::MakeFragmentedCord(chunks); + + EXPECT_EQ("A fragmented Cord", fragmented); + + auto chunk_it = fragmented.chunk_begin(); + + ASSERT_TRUE(chunk_it != fragmented.chunk_end()); + EXPECT_EQ("A ", *chunk_it); + + ASSERT_TRUE(++chunk_it != fragmented.chunk_end()); + EXPECT_EQ("fragmented ", *chunk_it); + + ASSERT_TRUE(++chunk_it != fragmented.chunk_end()); + EXPECT_EQ("Cord", *chunk_it); + + ASSERT_TRUE(++chunk_it == fragmented.chunk_end()); +} + +TEST(CordChunkIterator, Traits) { + static_assert(std::is_copy_constructible<absl::Cord::ChunkIterator>::value, + ""); + static_assert(std::is_copy_assignable<absl::Cord::ChunkIterator>::value, ""); + + // Move semantics to satisfy swappable via std::swap + static_assert(std::is_move_constructible<absl::Cord::ChunkIterator>::value, + ""); + static_assert(std::is_move_assignable<absl::Cord::ChunkIterator>::value, ""); + + static_assert( + std::is_same< + std::iterator_traits<absl::Cord::ChunkIterator>::iterator_category, + std::input_iterator_tag>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<absl::Cord::ChunkIterator>::value_type, + absl::string_view>::value, + ""); + static_assert( + std::is_same< + std::iterator_traits<absl::Cord::ChunkIterator>::difference_type, + ptrdiff_t>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<absl::Cord::ChunkIterator>::pointer, + const absl::string_view*>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<absl::Cord::ChunkIterator>::reference, + absl::string_view>::value, + ""); +} + +static void VerifyChunkIterator(const absl::Cord& cord, + size_t expected_chunks) { + EXPECT_EQ(cord.chunk_begin() == cord.chunk_end(), cord.empty()) << cord; + EXPECT_EQ(cord.chunk_begin() != cord.chunk_end(), !cord.empty()); + + absl::Cord::ChunkRange range = cord.Chunks(); + EXPECT_EQ(range.begin() == range.end(), cord.empty()); + EXPECT_EQ(range.begin() != range.end(), !cord.empty()); + + std::string content(cord); + size_t pos = 0; + auto pre_iter = cord.chunk_begin(), post_iter = cord.chunk_begin(); + size_t n_chunks = 0; + while (pre_iter != cord.chunk_end() && post_iter != cord.chunk_end()) { + EXPECT_FALSE(pre_iter == cord.chunk_end()); // NOLINT: explicitly test == + EXPECT_FALSE(post_iter == cord.chunk_end()); // NOLINT + + EXPECT_EQ(pre_iter, post_iter); + EXPECT_EQ(*pre_iter, *post_iter); + + EXPECT_EQ(pre_iter->data(), (*pre_iter).data()); + EXPECT_EQ(pre_iter->size(), (*pre_iter).size()); + + absl::string_view chunk = *pre_iter; + EXPECT_FALSE(chunk.empty()); + EXPECT_LE(pos + chunk.size(), content.size()); + EXPECT_EQ(absl::string_view(content.c_str() + pos, chunk.size()), chunk); + + int n_equal_iterators = 0; + for (absl::Cord::ChunkIterator it = range.begin(); it != range.end(); + ++it) { + n_equal_iterators += static_cast<int>(it == pre_iter); + } + EXPECT_EQ(n_equal_iterators, 1); + + ++pre_iter; + EXPECT_EQ(*post_iter++, chunk); + + pos += chunk.size(); + ++n_chunks; + } + EXPECT_EQ(expected_chunks, n_chunks); + EXPECT_EQ(pos, content.size()); + EXPECT_TRUE(pre_iter == cord.chunk_end()); // NOLINT: explicitly test == + EXPECT_TRUE(post_iter == cord.chunk_end()); // NOLINT +} + +TEST(CordChunkIterator, Operations) { + absl::Cord empty_cord; + VerifyChunkIterator(empty_cord, 0); + + absl::Cord small_buffer_cord("small cord"); + VerifyChunkIterator(small_buffer_cord, 1); + + absl::Cord flat_node_cord("larger than small buffer optimization"); + VerifyChunkIterator(flat_node_cord, 1); + + VerifyChunkIterator( + absl::MakeFragmentedCord({"a ", "small ", "fragmented ", "cord ", "for ", + "testing ", "chunk ", "iterations."}), + 8); + + absl::Cord reused_nodes_cord(std::string(40, 'c')); + reused_nodes_cord.Prepend(absl::Cord(std::string(40, 'b'))); + reused_nodes_cord.Prepend(absl::Cord(std::string(40, 'a'))); + size_t expected_chunks = 3; + for (int i = 0; i < 8; ++i) { + reused_nodes_cord.Prepend(reused_nodes_cord); + expected_chunks *= 2; + VerifyChunkIterator(reused_nodes_cord, expected_chunks); + } + + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + absl::Cord flat_cord(RandomLowercaseString(&rng, 256)); + absl::Cord subcords; + for (int i = 0; i < 128; ++i) subcords.Prepend(flat_cord.Subcord(i, 128)); + VerifyChunkIterator(subcords, 128); +} + +TEST(CordCharIterator, Traits) { + static_assert(std::is_copy_constructible<absl::Cord::CharIterator>::value, + ""); + static_assert(std::is_copy_assignable<absl::Cord::CharIterator>::value, ""); + + // Move semantics to satisfy swappable via std::swap + static_assert(std::is_move_constructible<absl::Cord::CharIterator>::value, + ""); + static_assert(std::is_move_assignable<absl::Cord::CharIterator>::value, ""); + + static_assert( + std::is_same< + std::iterator_traits<absl::Cord::CharIterator>::iterator_category, + std::input_iterator_tag>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<absl::Cord::CharIterator>::value_type, + char>::value, + ""); + static_assert( + std::is_same< + std::iterator_traits<absl::Cord::CharIterator>::difference_type, + ptrdiff_t>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<absl::Cord::CharIterator>::pointer, + const char*>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<absl::Cord::CharIterator>::reference, + const char&>::value, + ""); +} + +static void VerifyCharIterator(const absl::Cord& cord) { + EXPECT_EQ(cord.char_begin() == cord.char_end(), cord.empty()); + EXPECT_EQ(cord.char_begin() != cord.char_end(), !cord.empty()); + + absl::Cord::CharRange range = cord.Chars(); + EXPECT_EQ(range.begin() == range.end(), cord.empty()); + EXPECT_EQ(range.begin() != range.end(), !cord.empty()); + + size_t i = 0; + absl::Cord::CharIterator pre_iter = cord.char_begin(); + absl::Cord::CharIterator post_iter = cord.char_begin(); + std::string content(cord); + while (pre_iter != cord.char_end() && post_iter != cord.char_end()) { + EXPECT_FALSE(pre_iter == cord.char_end()); // NOLINT: explicitly test == + EXPECT_FALSE(post_iter == cord.char_end()); // NOLINT + + EXPECT_LT(i, cord.size()); + EXPECT_EQ(content[i], *pre_iter); + + EXPECT_EQ(pre_iter, post_iter); + EXPECT_EQ(*pre_iter, *post_iter); + EXPECT_EQ(&*pre_iter, &*post_iter); + + EXPECT_EQ(&*pre_iter, pre_iter.operator->()); + + const char* character_address = &*pre_iter; + absl::Cord::CharIterator copy = pre_iter; + ++copy; + EXPECT_EQ(character_address, &*pre_iter); + + int n_equal_iterators = 0; + for (absl::Cord::CharIterator it = range.begin(); it != range.end(); ++it) { + n_equal_iterators += static_cast<int>(it == pre_iter); + } + EXPECT_EQ(n_equal_iterators, 1); + + absl::Cord::CharIterator advance_iter = range.begin(); + absl::Cord::Advance(&advance_iter, i); + EXPECT_EQ(pre_iter, advance_iter); + + advance_iter = range.begin(); + EXPECT_EQ(absl::Cord::AdvanceAndRead(&advance_iter, i), cord.Subcord(0, i)); + EXPECT_EQ(pre_iter, advance_iter); + + advance_iter = pre_iter; + absl::Cord::Advance(&advance_iter, cord.size() - i); + EXPECT_EQ(range.end(), advance_iter); + + advance_iter = pre_iter; + EXPECT_EQ(absl::Cord::AdvanceAndRead(&advance_iter, cord.size() - i), + cord.Subcord(i, cord.size() - i)); + EXPECT_EQ(range.end(), advance_iter); + + ++i; + ++pre_iter; + post_iter++; + } + EXPECT_EQ(i, cord.size()); + EXPECT_TRUE(pre_iter == cord.char_end()); // NOLINT: explicitly test == + EXPECT_TRUE(post_iter == cord.char_end()); // NOLINT + + absl::Cord::CharIterator zero_advanced_end = cord.char_end(); + absl::Cord::Advance(&zero_advanced_end, 0); + EXPECT_EQ(zero_advanced_end, cord.char_end()); + + absl::Cord::CharIterator it = cord.char_begin(); + for (absl::string_view chunk : cord.Chunks()) { + while (!chunk.empty()) { + EXPECT_EQ(absl::Cord::ChunkRemaining(it), chunk); + chunk.remove_prefix(1); + ++it; + } + } +} + +TEST(CordCharIterator, Operations) { + absl::Cord empty_cord; + VerifyCharIterator(empty_cord); + + absl::Cord small_buffer_cord("small cord"); + VerifyCharIterator(small_buffer_cord); + + absl::Cord flat_node_cord("larger than small buffer optimization"); + VerifyCharIterator(flat_node_cord); + + VerifyCharIterator( + absl::MakeFragmentedCord({"a ", "small ", "fragmented ", "cord ", "for ", + "testing ", "character ", "iteration."})); + + absl::Cord reused_nodes_cord("ghi"); + reused_nodes_cord.Prepend(absl::Cord("def")); + reused_nodes_cord.Prepend(absl::Cord("abc")); + for (int i = 0; i < 4; ++i) { + reused_nodes_cord.Prepend(reused_nodes_cord); + VerifyCharIterator(reused_nodes_cord); + } + + RandomEngine rng(testing::GTEST_FLAG(random_seed)); + absl::Cord flat_cord(RandomLowercaseString(&rng, 256)); + absl::Cord subcords; + for (int i = 0; i < 4; ++i) subcords.Prepend(flat_cord.Subcord(16 * i, 128)); + VerifyCharIterator(subcords); +} + +TEST(Cord, StreamingOutput) { + absl::Cord c = + absl::MakeFragmentedCord({"A ", "small ", "fragmented ", "Cord", "."}); + std::stringstream output; + output << c; + EXPECT_EQ("A small fragmented Cord.", output.str()); +} + +TEST(Cord, ForEachChunk) { + for (int num_elements : {1, 10, 200}) { + SCOPED_TRACE(num_elements); + std::vector<std::string> cord_chunks; + for (int i = 0; i < num_elements; ++i) { + cord_chunks.push_back(absl::StrCat("[", i, "]")); + } + absl::Cord c = absl::MakeFragmentedCord(cord_chunks); + + std::vector<std::string> iterated_chunks; + absl::CordTestPeer::ForEachChunk(c, + [&iterated_chunks](absl::string_view sv) { + iterated_chunks.emplace_back(sv); + }); + EXPECT_EQ(iterated_chunks, cord_chunks); + } +} + +TEST(Cord, SmallBufferAssignFromOwnData) { + constexpr size_t kMaxInline = 15; + std::string contents = "small buff cord"; + EXPECT_EQ(contents.size(), kMaxInline); + for (size_t pos = 0; pos < contents.size(); ++pos) { + for (size_t count = contents.size() - pos; count > 0; --count) { + absl::Cord c(contents); + absl::string_view flat = c.Flatten(); + c = flat.substr(pos, count); + EXPECT_EQ(c, contents.substr(pos, count)) + << "pos = " << pos << "; count = " << count; + } + } +} + +TEST(Cord, Format) { + absl::Cord c; + absl::Format(&c, "There were %04d little %s.", 3, "pigs"); + EXPECT_EQ(c, "There were 0003 little pigs."); + absl::Format(&c, "And %-3llx bad wolf!", 1); + EXPECT_EQ(c, "There were 0003 little pigs.And 1 bad wolf!"); +} + +TEST(CordDeathTest, Hardening) { + absl::Cord cord("hello"); + // These statement should abort the program in all builds modes. + EXPECT_DEATH_IF_SUPPORTED(cord.RemovePrefix(6), ""); + EXPECT_DEATH_IF_SUPPORTED(cord.RemoveSuffix(6), ""); + + bool test_hardening = false; + ABSL_HARDENING_ASSERT([&]() { + // This only runs when ABSL_HARDENING_ASSERT is active. + test_hardening = true; + return true; + }()); + if (!test_hardening) return; + + EXPECT_DEATH_IF_SUPPORTED(cord[5], ""); + EXPECT_DEATH_IF_SUPPORTED(*cord.chunk_end(), ""); + EXPECT_DEATH_IF_SUPPORTED(static_cast<void>(cord.chunk_end()->empty()), ""); + EXPECT_DEATH_IF_SUPPORTED(++cord.chunk_end(), ""); +} + +class AfterExitCordTester { + public: + bool Set(absl::Cord* cord, absl::string_view expected) { + cord_ = cord; + expected_ = expected; + return true; + } + + ~AfterExitCordTester() { + EXPECT_EQ(*cord_, expected_); + } + private: + absl::Cord* cord_; + absl::string_view expected_; +}; + +template <typename Str> +void TestConstinitConstructor(Str) { + const auto expected = Str::value; + // Defined before `cord` to be destroyed after it. + static AfterExitCordTester exit_tester; // NOLINT + ABSL_CONST_INIT static absl::Cord cord(Str{}); // NOLINT + static bool init_exit_tester = exit_tester.Set(&cord, expected); + (void)init_exit_tester; + + EXPECT_EQ(cord, expected); + // Copy the object and test the copy, and the original. + { + absl::Cord copy = cord; + EXPECT_EQ(copy, expected); + } + // The original still works + EXPECT_EQ(cord, expected); + + // Try making adding more structure to the tree. + { + absl::Cord copy = cord; + std::string expected_copy(expected); + for (int i = 0; i < 10; ++i) { + copy.Append(cord); + absl::StrAppend(&expected_copy, expected); + EXPECT_EQ(copy, expected_copy); + } + } + + // Make sure we are using the right branch during constant evaluation. + EXPECT_EQ(absl::CordTestPeer::IsTree(cord), cord.size() >= 16); + + for (int i = 0; i < 10; ++i) { + // Make a few more Cords from the same global rep. + // This tests what happens when the refcount for it gets below 1. + EXPECT_EQ(expected, absl::Cord(Str{})); + } +} + +constexpr int SimpleStrlen(const char* p) { + return *p ? 1 + SimpleStrlen(p + 1) : 0; +} + +struct ShortView { + constexpr absl::string_view operator()() const { + return absl::string_view("SSO string", SimpleStrlen("SSO string")); + } +}; + +struct LongView { + constexpr absl::string_view operator()() const { + return absl::string_view("String that does not fit SSO.", + SimpleStrlen("String that does not fit SSO.")); + } +}; + + +TEST(Cord, ConstinitConstructor) { + TestConstinitConstructor( + absl::strings_internal::MakeStringConstant(ShortView{})); + TestConstinitConstructor( + absl::strings_internal::MakeStringConstant(LongView{})); +} diff --git a/third_party/abseil_cpp/absl/strings/cord_test_helpers.h b/third_party/abseil_cpp/absl/strings/cord_test_helpers.h new file mode 100644 index 000000000000..f1036e3b1388 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/cord_test_helpers.h @@ -0,0 +1,60 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_STRINGS_CORD_TEST_HELPERS_H_ +#define ABSL_STRINGS_CORD_TEST_HELPERS_H_ + +#include "absl/strings/cord.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Creates a multi-segment Cord from an iterable container of strings. The +// resulting Cord is guaranteed to have one segment for every string in the +// container. This allows code to be unit tested with multi-segment Cord +// inputs. +// +// Example: +// +// absl::Cord c = absl::MakeFragmentedCord({"A ", "fragmented ", "Cord"}); +// EXPECT_FALSE(c.GetFlat(&unused)); +// +// The mechanism by which this Cord is created is an implementation detail. Any +// implementation that produces a multi-segment Cord may produce a flat Cord in +// the future as new optimizations are added to the Cord class. +// MakeFragmentedCord will, however, always be updated to return a multi-segment +// Cord. +template <typename Container> +Cord MakeFragmentedCord(const Container& c) { + Cord result; + for (const auto& s : c) { + auto* external = new std::string(s); + Cord tmp = absl::MakeCordFromExternal( + *external, [external](absl::string_view) { delete external; }); + tmp.Prepend(result); + result = tmp; + } + return result; +} + +inline Cord MakeFragmentedCord(std::initializer_list<absl::string_view> list) { + return MakeFragmentedCord<std::initializer_list<absl::string_view>>(list); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORD_TEST_HELPERS_H_ diff --git a/third_party/abseil_cpp/absl/strings/escaping.cc b/third_party/abseil_cpp/absl/strings/escaping.cc new file mode 100644 index 000000000000..18b20b83fd36 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/escaping.cc @@ -0,0 +1,949 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/escaping.h" + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <cstring> +#include <iterator> +#include <limits> +#include <string> + +#include "absl/base/internal/endian.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/unaligned_access.h" +#include "absl/strings/internal/char_map.h" +#include "absl/strings/internal/escaping.h" +#include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/internal/utf8.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +// These are used for the leave_nulls_escaped argument to CUnescapeInternal(). +constexpr bool kUnescapeNulls = false; + +inline bool is_octal_digit(char c) { return ('0' <= c) && (c <= '7'); } + +inline int hex_digit_to_int(char c) { + static_assert('0' == 0x30 && 'A' == 0x41 && 'a' == 0x61, + "Character set must be ASCII."); + assert(absl::ascii_isxdigit(c)); + int x = static_cast<unsigned char>(c); + if (x > '9') { + x += 9; + } + return x & 0xf; +} + +inline bool IsSurrogate(char32_t c, absl::string_view src, std::string* error) { + if (c >= 0xD800 && c <= 0xDFFF) { + if (error) { + *error = absl::StrCat("invalid surrogate character (0xD800-DFFF): \\", + src); + } + return true; + } + return false; +} + +// ---------------------------------------------------------------------- +// CUnescapeInternal() +// Implements both CUnescape() and CUnescapeForNullTerminatedString(). +// +// Unescapes C escape sequences and is the reverse of CEscape(). +// +// If 'source' is valid, stores the unescaped string and its size in +// 'dest' and 'dest_len' respectively, and returns true. Otherwise +// returns false and optionally stores the error description in +// 'error'. Set 'error' to nullptr to disable error reporting. +// +// 'dest' should point to a buffer that is at least as big as 'source'. +// 'source' and 'dest' may be the same. +// +// NOTE: any changes to this function must also be reflected in the older +// UnescapeCEscapeSequences(). +// ---------------------------------------------------------------------- +bool CUnescapeInternal(absl::string_view source, bool leave_nulls_escaped, + char* dest, ptrdiff_t* dest_len, std::string* error) { + char* d = dest; + const char* p = source.data(); + const char* end = p + source.size(); + const char* last_byte = end - 1; + + // Small optimization for case where source = dest and there's no escaping + while (p == d && p < end && *p != '\\') p++, d++; + + while (p < end) { + if (*p != '\\') { + *d++ = *p++; + } else { + if (++p > last_byte) { // skip past the '\\' + if (error) *error = "String cannot end with \\"; + return false; + } + switch (*p) { + case 'a': *d++ = '\a'; break; + case 'b': *d++ = '\b'; break; + case 'f': *d++ = '\f'; break; + case 'n': *d++ = '\n'; break; + case 'r': *d++ = '\r'; break; + case 't': *d++ = '\t'; break; + case 'v': *d++ = '\v'; break; + case '\\': *d++ = '\\'; break; + case '?': *d++ = '\?'; break; // \? Who knew? + case '\'': *d++ = '\''; break; + case '"': *d++ = '\"'; break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': { + // octal digit: 1 to 3 digits + const char* octal_start = p; + unsigned int ch = *p - '0'; + if (p < last_byte && is_octal_digit(p[1])) ch = ch * 8 + *++p - '0'; + if (p < last_byte && is_octal_digit(p[1])) + ch = ch * 8 + *++p - '0'; // now points at last digit + if (ch > 0xff) { + if (error) { + *error = "Value of \\" + + std::string(octal_start, p + 1 - octal_start) + + " exceeds 0xff"; + } + return false; + } + if ((ch == 0) && leave_nulls_escaped) { + // Copy the escape sequence for the null character + const ptrdiff_t octal_size = p + 1 - octal_start; + *d++ = '\\'; + memmove(d, octal_start, octal_size); + d += octal_size; + break; + } + *d++ = ch; + break; + } + case 'x': + case 'X': { + if (p >= last_byte) { + if (error) *error = "String cannot end with \\x"; + return false; + } else if (!absl::ascii_isxdigit(p[1])) { + if (error) *error = "\\x cannot be followed by a non-hex digit"; + return false; + } + unsigned int ch = 0; + const char* hex_start = p; + while (p < last_byte && absl::ascii_isxdigit(p[1])) + // Arbitrarily many hex digits + ch = (ch << 4) + hex_digit_to_int(*++p); + if (ch > 0xFF) { + if (error) { + *error = "Value of \\" + + std::string(hex_start, p + 1 - hex_start) + + " exceeds 0xff"; + } + return false; + } + if ((ch == 0) && leave_nulls_escaped) { + // Copy the escape sequence for the null character + const ptrdiff_t hex_size = p + 1 - hex_start; + *d++ = '\\'; + memmove(d, hex_start, hex_size); + d += hex_size; + break; + } + *d++ = ch; + break; + } + case 'u': { + // \uhhhh => convert 4 hex digits to UTF-8 + char32_t rune = 0; + const char* hex_start = p; + if (p + 4 >= end) { + if (error) { + *error = "\\u must be followed by 4 hex digits: \\" + + std::string(hex_start, p + 1 - hex_start); + } + return false; + } + for (int i = 0; i < 4; ++i) { + // Look one char ahead. + if (absl::ascii_isxdigit(p[1])) { + rune = (rune << 4) + hex_digit_to_int(*++p); // Advance p. + } else { + if (error) { + *error = "\\u must be followed by 4 hex digits: \\" + + std::string(hex_start, p + 1 - hex_start); + } + return false; + } + } + if ((rune == 0) && leave_nulls_escaped) { + // Copy the escape sequence for the null character + *d++ = '\\'; + memmove(d, hex_start, 5); // u0000 + d += 5; + break; + } + if (IsSurrogate(rune, absl::string_view(hex_start, 5), error)) { + return false; + } + d += strings_internal::EncodeUTF8Char(d, rune); + break; + } + case 'U': { + // \Uhhhhhhhh => convert 8 hex digits to UTF-8 + char32_t rune = 0; + const char* hex_start = p; + if (p + 8 >= end) { + if (error) { + *error = "\\U must be followed by 8 hex digits: \\" + + std::string(hex_start, p + 1 - hex_start); + } + return false; + } + for (int i = 0; i < 8; ++i) { + // Look one char ahead. + if (absl::ascii_isxdigit(p[1])) { + // Don't change rune until we're sure this + // is within the Unicode limit, but do advance p. + uint32_t newrune = (rune << 4) + hex_digit_to_int(*++p); + if (newrune > 0x10FFFF) { + if (error) { + *error = "Value of \\" + + std::string(hex_start, p + 1 - hex_start) + + " exceeds Unicode limit (0x10FFFF)"; + } + return false; + } else { + rune = newrune; + } + } else { + if (error) { + *error = "\\U must be followed by 8 hex digits: \\" + + std::string(hex_start, p + 1 - hex_start); + } + return false; + } + } + if ((rune == 0) && leave_nulls_escaped) { + // Copy the escape sequence for the null character + *d++ = '\\'; + memmove(d, hex_start, 9); // U00000000 + d += 9; + break; + } + if (IsSurrogate(rune, absl::string_view(hex_start, 9), error)) { + return false; + } + d += strings_internal::EncodeUTF8Char(d, rune); + break; + } + default: { + if (error) *error = std::string("Unknown escape sequence: \\") + *p; + return false; + } + } + p++; // read past letter we escaped + } + } + *dest_len = d - dest; + return true; +} + +// ---------------------------------------------------------------------- +// CUnescapeInternal() +// +// Same as above but uses a std::string for output. 'source' and 'dest' +// may be the same. +// ---------------------------------------------------------------------- +bool CUnescapeInternal(absl::string_view source, bool leave_nulls_escaped, + std::string* dest, std::string* error) { + strings_internal::STLStringResizeUninitialized(dest, source.size()); + + ptrdiff_t dest_size; + if (!CUnescapeInternal(source, + leave_nulls_escaped, + &(*dest)[0], + &dest_size, + error)) { + return false; + } + dest->erase(dest_size); + return true; +} + +// ---------------------------------------------------------------------- +// CEscape() +// CHexEscape() +// Utf8SafeCEscape() +// Utf8SafeCHexEscape() +// Escapes 'src' using C-style escape sequences. This is useful for +// preparing query flags. The 'Hex' version uses hexadecimal rather than +// octal sequences. The 'Utf8Safe' version does not touch UTF-8 bytes. +// +// Escaped chars: \n, \r, \t, ", ', \, and !absl::ascii_isprint(). +// ---------------------------------------------------------------------- +std::string CEscapeInternal(absl::string_view src, bool use_hex, + bool utf8_safe) { + std::string dest; + bool last_hex_escape = false; // true if last output char was \xNN. + + for (unsigned char c : src) { + bool is_hex_escape = false; + switch (c) { + case '\n': dest.append("\\" "n"); break; + case '\r': dest.append("\\" "r"); break; + case '\t': dest.append("\\" "t"); break; + case '\"': dest.append("\\" "\""); break; + case '\'': dest.append("\\" "'"); break; + case '\\': dest.append("\\" "\\"); break; + default: + // Note that if we emit \xNN and the src character after that is a hex + // digit then that digit must be escaped too to prevent it being + // interpreted as part of the character code by C. + if ((!utf8_safe || c < 0x80) && + (!absl::ascii_isprint(c) || + (last_hex_escape && absl::ascii_isxdigit(c)))) { + if (use_hex) { + dest.append("\\" "x"); + dest.push_back(numbers_internal::kHexChar[c / 16]); + dest.push_back(numbers_internal::kHexChar[c % 16]); + is_hex_escape = true; + } else { + dest.append("\\"); + dest.push_back(numbers_internal::kHexChar[c / 64]); + dest.push_back(numbers_internal::kHexChar[(c % 64) / 8]); + dest.push_back(numbers_internal::kHexChar[c % 8]); + } + } else { + dest.push_back(c); + break; + } + } + last_hex_escape = is_hex_escape; + } + + return dest; +} + +/* clang-format off */ +constexpr char c_escaped_len[256] = { + 4, 4, 4, 4, 4, 4, 4, 4, 4, 2, 2, 4, 4, 2, 4, 4, // \t, \n, \r + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 1, 1, 2, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // ", ' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // '0'..'9' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 'A'..'O' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, // 'P'..'Z', '\' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 'a'..'o' + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, // 'p'..'z', DEL + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, +}; +/* clang-format on */ + +// Calculates the length of the C-style escaped version of 'src'. +// Assumes that non-printable characters are escaped using octal sequences, and +// that UTF-8 bytes are not handled specially. +inline size_t CEscapedLength(absl::string_view src) { + size_t escaped_len = 0; + for (unsigned char c : src) escaped_len += c_escaped_len[c]; + return escaped_len; +} + +void CEscapeAndAppendInternal(absl::string_view src, std::string* dest) { + size_t escaped_len = CEscapedLength(src); + if (escaped_len == src.size()) { + dest->append(src.data(), src.size()); + return; + } + + size_t cur_dest_len = dest->size(); + strings_internal::STLStringResizeUninitialized(dest, + cur_dest_len + escaped_len); + char* append_ptr = &(*dest)[cur_dest_len]; + + for (unsigned char c : src) { + int char_len = c_escaped_len[c]; + if (char_len == 1) { + *append_ptr++ = c; + } else if (char_len == 2) { + switch (c) { + case '\n': + *append_ptr++ = '\\'; + *append_ptr++ = 'n'; + break; + case '\r': + *append_ptr++ = '\\'; + *append_ptr++ = 'r'; + break; + case '\t': + *append_ptr++ = '\\'; + *append_ptr++ = 't'; + break; + case '\"': + *append_ptr++ = '\\'; + *append_ptr++ = '\"'; + break; + case '\'': + *append_ptr++ = '\\'; + *append_ptr++ = '\''; + break; + case '\\': + *append_ptr++ = '\\'; + *append_ptr++ = '\\'; + break; + } + } else { + *append_ptr++ = '\\'; + *append_ptr++ = '0' + c / 64; + *append_ptr++ = '0' + (c % 64) / 8; + *append_ptr++ = '0' + c % 8; + } + } +} + +bool Base64UnescapeInternal(const char* src_param, size_t szsrc, char* dest, + size_t szdest, const signed char* unbase64, + size_t* len) { + static const char kPad64Equals = '='; + static const char kPad64Dot = '.'; + + size_t destidx = 0; + int decode = 0; + int state = 0; + unsigned int ch = 0; + unsigned int temp = 0; + + // If "char" is signed by default, using *src as an array index results in + // accessing negative array elements. Treat the input as a pointer to + // unsigned char to avoid this. + const unsigned char* src = reinterpret_cast<const unsigned char*>(src_param); + + // The GET_INPUT macro gets the next input character, skipping + // over any whitespace, and stopping when we reach the end of the + // string or when we read any non-data character. The arguments are + // an arbitrary identifier (used as a label for goto) and the number + // of data bytes that must remain in the input to avoid aborting the + // loop. +#define GET_INPUT(label, remain) \ + label: \ + --szsrc; \ + ch = *src++; \ + decode = unbase64[ch]; \ + if (decode < 0) { \ + if (absl::ascii_isspace(ch) && szsrc >= remain) goto label; \ + state = 4 - remain; \ + break; \ + } + + // if dest is null, we're just checking to see if it's legal input + // rather than producing output. (I suspect this could just be done + // with a regexp...). We duplicate the loop so this test can be + // outside it instead of in every iteration. + + if (dest) { + // This loop consumes 4 input bytes and produces 3 output bytes + // per iteration. We can't know at the start that there is enough + // data left in the string for a full iteration, so the loop may + // break out in the middle; if so 'state' will be set to the + // number of input bytes read. + + while (szsrc >= 4) { + // We'll start by optimistically assuming that the next four + // bytes of the string (src[0..3]) are four good data bytes + // (that is, no nulls, whitespace, padding chars, or illegal + // chars). We need to test src[0..2] for nulls individually + // before constructing temp to preserve the property that we + // never read past a null in the string (no matter how long + // szsrc claims the string is). + + if (!src[0] || !src[1] || !src[2] || + ((temp = ((unsigned(unbase64[src[0]]) << 18) | + (unsigned(unbase64[src[1]]) << 12) | + (unsigned(unbase64[src[2]]) << 6) | + (unsigned(unbase64[src[3]])))) & + 0x80000000)) { + // Iff any of those four characters was bad (null, illegal, + // whitespace, padding), then temp's high bit will be set + // (because unbase64[] is -1 for all bad characters). + // + // We'll back up and resort to the slower decoder, which knows + // how to handle those cases. + + GET_INPUT(first, 4); + temp = decode; + GET_INPUT(second, 3); + temp = (temp << 6) | decode; + GET_INPUT(third, 2); + temp = (temp << 6) | decode; + GET_INPUT(fourth, 1); + temp = (temp << 6) | decode; + } else { + // We really did have four good data bytes, so advance four + // characters in the string. + + szsrc -= 4; + src += 4; + } + + // temp has 24 bits of input, so write that out as three bytes. + + if (destidx + 3 > szdest) return false; + dest[destidx + 2] = temp; + temp >>= 8; + dest[destidx + 1] = temp; + temp >>= 8; + dest[destidx] = temp; + destidx += 3; + } + } else { + while (szsrc >= 4) { + if (!src[0] || !src[1] || !src[2] || + ((temp = ((unsigned(unbase64[src[0]]) << 18) | + (unsigned(unbase64[src[1]]) << 12) | + (unsigned(unbase64[src[2]]) << 6) | + (unsigned(unbase64[src[3]])))) & + 0x80000000)) { + GET_INPUT(first_no_dest, 4); + GET_INPUT(second_no_dest, 3); + GET_INPUT(third_no_dest, 2); + GET_INPUT(fourth_no_dest, 1); + } else { + szsrc -= 4; + src += 4; + } + destidx += 3; + } + } + +#undef GET_INPUT + + // if the loop terminated because we read a bad character, return + // now. + if (decode < 0 && ch != kPad64Equals && ch != kPad64Dot && + !absl::ascii_isspace(ch)) + return false; + + if (ch == kPad64Equals || ch == kPad64Dot) { + // if we stopped by hitting an '=' or '.', un-read that character -- we'll + // look at it again when we count to check for the proper number of + // equals signs at the end. + ++szsrc; + --src; + } else { + // This loop consumes 1 input byte per iteration. It's used to + // clean up the 0-3 input bytes remaining when the first, faster + // loop finishes. 'temp' contains the data from 'state' input + // characters read by the first loop. + while (szsrc > 0) { + --szsrc; + ch = *src++; + decode = unbase64[ch]; + if (decode < 0) { + if (absl::ascii_isspace(ch)) { + continue; + } else if (ch == kPad64Equals || ch == kPad64Dot) { + // back up one character; we'll read it again when we check + // for the correct number of pad characters at the end. + ++szsrc; + --src; + break; + } else { + return false; + } + } + + // Each input character gives us six bits of output. + temp = (temp << 6) | decode; + ++state; + if (state == 4) { + // If we've accumulated 24 bits of output, write that out as + // three bytes. + if (dest) { + if (destidx + 3 > szdest) return false; + dest[destidx + 2] = temp; + temp >>= 8; + dest[destidx + 1] = temp; + temp >>= 8; + dest[destidx] = temp; + } + destidx += 3; + state = 0; + temp = 0; + } + } + } + + // Process the leftover data contained in 'temp' at the end of the input. + int expected_equals = 0; + switch (state) { + case 0: + // Nothing left over; output is a multiple of 3 bytes. + break; + + case 1: + // Bad input; we have 6 bits left over. + return false; + + case 2: + // Produce one more output byte from the 12 input bits we have left. + if (dest) { + if (destidx + 1 > szdest) return false; + temp >>= 4; + dest[destidx] = temp; + } + ++destidx; + expected_equals = 2; + break; + + case 3: + // Produce two more output bytes from the 18 input bits we have left. + if (dest) { + if (destidx + 2 > szdest) return false; + temp >>= 2; + dest[destidx + 1] = temp; + temp >>= 8; + dest[destidx] = temp; + } + destidx += 2; + expected_equals = 1; + break; + + default: + // state should have no other values at this point. + ABSL_RAW_LOG(FATAL, "This can't happen; base64 decoder state = %d", + state); + } + + // The remainder of the string should be all whitespace, mixed with + // exactly 0 equals signs, or exactly 'expected_equals' equals + // signs. (Always accepting 0 equals signs is an Abseil extension + // not covered in the RFC, as is accepting dot as the pad character.) + + int equals = 0; + while (szsrc > 0) { + if (*src == kPad64Equals || *src == kPad64Dot) + ++equals; + else if (!absl::ascii_isspace(*src)) + return false; + --szsrc; + ++src; + } + + const bool ok = (equals == 0 || equals == expected_equals); + if (ok) *len = destidx; + return ok; +} + +// The arrays below were generated by the following code +// #include <sys/time.h> +// #include <stdlib.h> +// #include <string.h> +// main() +// { +// static const char Base64[] = +// "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +// char* pos; +// int idx, i, j; +// printf(" "); +// for (i = 0; i < 255; i += 8) { +// for (j = i; j < i + 8; j++) { +// pos = strchr(Base64, j); +// if ((pos == nullptr) || (j == 0)) +// idx = -1; +// else +// idx = pos - Base64; +// if (idx == -1) +// printf(" %2d, ", idx); +// else +// printf(" %2d/*%c*/,", idx, j); +// } +// printf("\n "); +// } +// } +// +// where the value of "Base64[]" was replaced by one of the base-64 conversion +// tables from the functions below. +/* clang-format off */ +constexpr signed char kUnBase64[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 62/*+*/, -1, -1, -1, 63/*/ */, + 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, + 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, + -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, + 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, + 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, -1, + -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, + 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/, + 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/, + 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 +}; + +constexpr signed char kUnWebSafeBase64[] = { + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 62/*-*/, -1, -1, + 52/*0*/, 53/*1*/, 54/*2*/, 55/*3*/, 56/*4*/, 57/*5*/, 58/*6*/, 59/*7*/, + 60/*8*/, 61/*9*/, -1, -1, -1, -1, -1, -1, + -1, 0/*A*/, 1/*B*/, 2/*C*/, 3/*D*/, 4/*E*/, 5/*F*/, 6/*G*/, + 07/*H*/, 8/*I*/, 9/*J*/, 10/*K*/, 11/*L*/, 12/*M*/, 13/*N*/, 14/*O*/, + 15/*P*/, 16/*Q*/, 17/*R*/, 18/*S*/, 19/*T*/, 20/*U*/, 21/*V*/, 22/*W*/, + 23/*X*/, 24/*Y*/, 25/*Z*/, -1, -1, -1, -1, 63/*_*/, + -1, 26/*a*/, 27/*b*/, 28/*c*/, 29/*d*/, 30/*e*/, 31/*f*/, 32/*g*/, + 33/*h*/, 34/*i*/, 35/*j*/, 36/*k*/, 37/*l*/, 38/*m*/, 39/*n*/, 40/*o*/, + 41/*p*/, 42/*q*/, 43/*r*/, 44/*s*/, 45/*t*/, 46/*u*/, 47/*v*/, 48/*w*/, + 49/*x*/, 50/*y*/, 51/*z*/, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1 +}; +/* clang-format on */ + +constexpr char kWebSafeBase64Chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +template <typename String> +bool Base64UnescapeInternal(const char* src, size_t slen, String* dest, + const signed char* unbase64) { + // Determine the size of the output string. Base64 encodes every 3 bytes into + // 4 characters. any leftover chars are added directly for good measure. + // This is documented in the base64 RFC: http://tools.ietf.org/html/rfc3548 + const size_t dest_len = 3 * (slen / 4) + (slen % 4); + + strings_internal::STLStringResizeUninitialized(dest, dest_len); + + // We are getting the destination buffer by getting the beginning of the + // string and converting it into a char *. + size_t len; + const bool ok = + Base64UnescapeInternal(src, slen, &(*dest)[0], dest_len, unbase64, &len); + if (!ok) { + dest->clear(); + return false; + } + + // could be shorter if there was padding + assert(len <= dest_len); + dest->erase(len); + + return true; +} + +/* clang-format off */ +constexpr char kHexValueLenient[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 0, 0, 0, 0, 0, // '0'..'9' + 0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 'A'..'F' + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 10, 11, 12, 13, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 'a'..'f' + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +/* clang-format on */ + +// This is a templated function so that T can be either a char* +// or a string. This works because we use the [] operator to access +// individual characters at a time. +template <typename T> +void HexStringToBytesInternal(const char* from, T to, ptrdiff_t num) { + for (int i = 0; i < num; i++) { + to[i] = (kHexValueLenient[from[i * 2] & 0xFF] << 4) + + (kHexValueLenient[from[i * 2 + 1] & 0xFF]); + } +} + +// This is a templated function so that T can be either a char* or a +// std::string. +template <typename T> +void BytesToHexStringInternal(const unsigned char* src, T dest, ptrdiff_t num) { + auto dest_ptr = &dest[0]; + for (auto src_ptr = src; src_ptr != (src + num); ++src_ptr, dest_ptr += 2) { + const char* hex_p = &numbers_internal::kHexTable[*src_ptr * 2]; + std::copy(hex_p, hex_p + 2, dest_ptr); + } +} + +} // namespace + +// ---------------------------------------------------------------------- +// CUnescape() +// +// See CUnescapeInternal() for implementation details. +// ---------------------------------------------------------------------- +bool CUnescape(absl::string_view source, std::string* dest, + std::string* error) { + return CUnescapeInternal(source, kUnescapeNulls, dest, error); +} + +std::string CEscape(absl::string_view src) { + std::string dest; + CEscapeAndAppendInternal(src, &dest); + return dest; +} + +std::string CHexEscape(absl::string_view src) { + return CEscapeInternal(src, true, false); +} + +std::string Utf8SafeCEscape(absl::string_view src) { + return CEscapeInternal(src, false, true); +} + +std::string Utf8SafeCHexEscape(absl::string_view src) { + return CEscapeInternal(src, true, true); +} + +// ---------------------------------------------------------------------- +// Base64Unescape() - base64 decoder +// Base64Escape() - base64 encoder +// WebSafeBase64Unescape() - Google's variation of base64 decoder +// WebSafeBase64Escape() - Google's variation of base64 encoder +// +// Check out +// http://tools.ietf.org/html/rfc2045 for formal description, but what we +// care about is that... +// Take the encoded stuff in groups of 4 characters and turn each +// character into a code 0 to 63 thus: +// A-Z map to 0 to 25 +// a-z map to 26 to 51 +// 0-9 map to 52 to 61 +// +(- for WebSafe) maps to 62 +// /(_ for WebSafe) maps to 63 +// There will be four numbers, all less than 64 which can be represented +// by a 6 digit binary number (aaaaaa, bbbbbb, cccccc, dddddd respectively). +// Arrange the 6 digit binary numbers into three bytes as such: +// aaaaaabb bbbbcccc ccdddddd +// Equals signs (one or two) are used at the end of the encoded block to +// indicate that the text was not an integer multiple of three bytes long. +// ---------------------------------------------------------------------- + +bool Base64Unescape(absl::string_view src, std::string* dest) { + return Base64UnescapeInternal(src.data(), src.size(), dest, kUnBase64); +} + +bool WebSafeBase64Unescape(absl::string_view src, std::string* dest) { + return Base64UnescapeInternal(src.data(), src.size(), dest, kUnWebSafeBase64); +} + +void Base64Escape(absl::string_view src, std::string* dest) { + strings_internal::Base64EscapeInternal( + reinterpret_cast<const unsigned char*>(src.data()), src.size(), dest, + true, strings_internal::kBase64Chars); +} + +void WebSafeBase64Escape(absl::string_view src, std::string* dest) { + strings_internal::Base64EscapeInternal( + reinterpret_cast<const unsigned char*>(src.data()), src.size(), dest, + false, kWebSafeBase64Chars); +} + +std::string Base64Escape(absl::string_view src) { + std::string dest; + strings_internal::Base64EscapeInternal( + reinterpret_cast<const unsigned char*>(src.data()), src.size(), &dest, + true, strings_internal::kBase64Chars); + return dest; +} + +std::string WebSafeBase64Escape(absl::string_view src) { + std::string dest; + strings_internal::Base64EscapeInternal( + reinterpret_cast<const unsigned char*>(src.data()), src.size(), &dest, + false, kWebSafeBase64Chars); + return dest; +} + +std::string HexStringToBytes(absl::string_view from) { + std::string result; + const auto num = from.size() / 2; + strings_internal::STLStringResizeUninitialized(&result, num); + absl::HexStringToBytesInternal<std::string&>(from.data(), result, num); + return result; +} + +std::string BytesToHexString(absl::string_view from) { + std::string result; + strings_internal::STLStringResizeUninitialized(&result, 2 * from.size()); + absl::BytesToHexStringInternal<std::string&>( + reinterpret_cast<const unsigned char*>(from.data()), result, from.size()); + return result; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/escaping.h b/third_party/abseil_cpp/absl/strings/escaping.h new file mode 100644 index 000000000000..f5ca26c5dadb --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/escaping.h @@ -0,0 +1,164 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: escaping.h +// ----------------------------------------------------------------------------- +// +// This header file contains string utilities involved in escaping and +// unescaping strings in various ways. + +#ifndef ABSL_STRINGS_ESCAPING_H_ +#define ABSL_STRINGS_ESCAPING_H_ + +#include <cstddef> +#include <string> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// CUnescape() +// +// Unescapes a `source` string and copies it into `dest`, rewriting C-style +// escape sequences (https://en.cppreference.com/w/cpp/language/escape) into +// their proper code point equivalents, returning `true` if successful. +// +// The following unescape sequences can be handled: +// +// * ASCII escape sequences ('\n','\r','\\', etc.) to their ASCII equivalents +// * Octal escape sequences ('\nnn') to byte nnn. The unescaped value must +// resolve to a single byte or an error will occur. E.g. values greater than +// 0xff will produce an error. +// * Hexadecimal escape sequences ('\xnn') to byte nn. While an arbitrary +// number of following digits are allowed, the unescaped value must resolve +// to a single byte or an error will occur. E.g. '\x0045' is equivalent to +// '\x45', but '\x1234' will produce an error. +// * Unicode escape sequences ('\unnnn' for exactly four hex digits or +// '\Unnnnnnnn' for exactly eight hex digits, which will be encoded in +// UTF-8. (E.g., `\u2019` unescapes to the three bytes 0xE2, 0x80, and +// 0x99). +// +// If any errors are encountered, this function returns `false`, leaving the +// `dest` output parameter in an unspecified state, and stores the first +// encountered error in `error`. To disable error reporting, set `error` to +// `nullptr` or use the overload with no error reporting below. +// +// Example: +// +// std::string s = "foo\\rbar\\nbaz\\t"; +// std::string unescaped_s; +// if (!absl::CUnescape(s, &unescaped_s) { +// ... +// } +// EXPECT_EQ(unescaped_s, "foo\rbar\nbaz\t"); +bool CUnescape(absl::string_view source, std::string* dest, std::string* error); + +// Overload of `CUnescape()` with no error reporting. +inline bool CUnescape(absl::string_view source, std::string* dest) { + return CUnescape(source, dest, nullptr); +} + +// CEscape() +// +// Escapes a 'src' string using C-style escapes sequences +// (https://en.cppreference.com/w/cpp/language/escape), escaping other +// non-printable/non-whitespace bytes as octal sequences (e.g. "\377"). +// +// Example: +// +// std::string s = "foo\rbar\tbaz\010\011\012\013\014\x0d\n"; +// std::string escaped_s = absl::CEscape(s); +// EXPECT_EQ(escaped_s, "foo\\rbar\\tbaz\\010\\t\\n\\013\\014\\r\\n"); +std::string CEscape(absl::string_view src); + +// CHexEscape() +// +// Escapes a 'src' string using C-style escape sequences, escaping +// other non-printable/non-whitespace bytes as hexadecimal sequences (e.g. +// "\xFF"). +// +// Example: +// +// std::string s = "foo\rbar\tbaz\010\011\012\013\014\x0d\n"; +// std::string escaped_s = absl::CHexEscape(s); +// EXPECT_EQ(escaped_s, "foo\\rbar\\tbaz\\x08\\t\\n\\x0b\\x0c\\r\\n"); +std::string CHexEscape(absl::string_view src); + +// Utf8SafeCEscape() +// +// Escapes a 'src' string using C-style escape sequences, escaping bytes as +// octal sequences, and passing through UTF-8 characters without conversion. +// I.e., when encountering any bytes with their high bit set, this function +// will not escape those values, whether or not they are valid UTF-8. +std::string Utf8SafeCEscape(absl::string_view src); + +// Utf8SafeCHexEscape() +// +// Escapes a 'src' string using C-style escape sequences, escaping bytes as +// hexadecimal sequences, and passing through UTF-8 characters without +// conversion. +std::string Utf8SafeCHexEscape(absl::string_view src); + +// Base64Unescape() +// +// Converts a `src` string encoded in Base64 to its binary equivalent, writing +// it to a `dest` buffer, returning `true` on success. If `src` contains invalid +// characters, `dest` is cleared and returns `false`. +bool Base64Unescape(absl::string_view src, std::string* dest); + +// WebSafeBase64Unescape() +// +// Converts a `src` string encoded in Base64 to its binary equivalent, writing +// it to a `dest` buffer, but using '-' instead of '+', and '_' instead of '/'. +// If `src` contains invalid characters, `dest` is cleared and returns `false`. +bool WebSafeBase64Unescape(absl::string_view src, std::string* dest); + +// Base64Escape() +// +// Encodes a `src` string into a base64-encoded string, with padding characters. +// This function conforms with RFC 4648 section 4 (base64). +void Base64Escape(absl::string_view src, std::string* dest); +std::string Base64Escape(absl::string_view src); + +// WebSafeBase64Escape() +// +// Encodes a `src` string into a base64-like string, using '-' instead of '+' +// and '_' instead of '/', and without padding. This function conforms with RFC +// 4648 section 5 (base64url). +void WebSafeBase64Escape(absl::string_view src, std::string* dest); +std::string WebSafeBase64Escape(absl::string_view src); + +// HexStringToBytes() +// +// Converts an ASCII hex string into bytes, returning binary data of length +// `from.size()/2`. +std::string HexStringToBytes(absl::string_view from); + +// BytesToHexString() +// +// Converts binary data into an ASCII text string, returning a string of size +// `2*from.size()`. +std::string BytesToHexString(absl::string_view from); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_ESCAPING_H_ diff --git a/third_party/abseil_cpp/absl/strings/escaping_benchmark.cc b/third_party/abseil_cpp/absl/strings/escaping_benchmark.cc new file mode 100644 index 000000000000..10d5b033c520 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/escaping_benchmark.cc @@ -0,0 +1,94 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/escaping.h" + +#include <cstdio> +#include <cstring> +#include <random> + +#include "benchmark/benchmark.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/internal/escaping_test_common.h" + +namespace { + +void BM_CUnescapeHexString(benchmark::State& state) { + std::string src; + for (int i = 0; i < 50; i++) { + src += "\\x55"; + } + std::string dest; + for (auto _ : state) { + absl::CUnescape(src, &dest); + } +} +BENCHMARK(BM_CUnescapeHexString); + +void BM_WebSafeBase64Escape_string(benchmark::State& state) { + std::string raw; + for (int i = 0; i < 10; ++i) { + for (const auto& test_set : absl::strings_internal::base64_strings()) { + raw += std::string(test_set.plaintext); + } + } + + // The actual benchmark loop is tiny... + std::string escaped; + for (auto _ : state) { + absl::WebSafeBase64Escape(raw, &escaped); + } + + // We want to be sure the compiler doesn't throw away the loop above, + // and the easiest way to ensure that is to round-trip the results and verify + // them. + std::string round_trip; + absl::WebSafeBase64Unescape(escaped, &round_trip); + ABSL_RAW_CHECK(round_trip == raw, ""); +} +BENCHMARK(BM_WebSafeBase64Escape_string); + +// Used for the CEscape benchmarks +const char kStringValueNoEscape[] = "1234567890"; +const char kStringValueSomeEscaped[] = "123\n56789\xA1"; +const char kStringValueMostEscaped[] = "\xA1\xA2\ny\xA4\xA5\xA6z\b\r"; + +void CEscapeBenchmarkHelper(benchmark::State& state, const char* string_value, + int max_len) { + std::string src; + while (src.size() < max_len) { + absl::StrAppend(&src, string_value); + } + + for (auto _ : state) { + absl::CEscape(src); + } +} + +void BM_CEscape_NoEscape(benchmark::State& state) { + CEscapeBenchmarkHelper(state, kStringValueNoEscape, state.range(0)); +} +BENCHMARK(BM_CEscape_NoEscape)->Range(1, 1 << 14); + +void BM_CEscape_SomeEscaped(benchmark::State& state) { + CEscapeBenchmarkHelper(state, kStringValueSomeEscaped, state.range(0)); +} +BENCHMARK(BM_CEscape_SomeEscaped)->Range(1, 1 << 14); + +void BM_CEscape_MostEscaped(benchmark::State& state) { + CEscapeBenchmarkHelper(state, kStringValueMostEscaped, state.range(0)); +} +BENCHMARK(BM_CEscape_MostEscaped)->Range(1, 1 << 14); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/escaping_test.cc b/third_party/abseil_cpp/absl/strings/escaping_test.cc new file mode 100644 index 000000000000..45671a0ed598 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/escaping_test.cc @@ -0,0 +1,664 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/escaping.h" + +#include <array> +#include <cstdio> +#include <cstring> +#include <memory> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/fixed_array.h" +#include "absl/strings/str_cat.h" + +#include "absl/strings/internal/escaping_test_common.h" + +namespace { + +struct epair { + std::string escaped; + std::string unescaped; +}; + +TEST(CEscape, EscapeAndUnescape) { + const std::string inputs[] = { + std::string("foo\nxx\r\b\0023"), + std::string(""), + std::string("abc"), + std::string("\1chad_rules"), + std::string("\1arnar_drools"), + std::string("xxxx\r\t'\"\\"), + std::string("\0xx\0", 4), + std::string("\x01\x31"), + std::string("abc\xb\x42\141bc"), + std::string("123\1\x31\x32\x33"), + std::string("\xc1\xca\x1b\x62\x19o\xcc\x04"), + std::string( + "\\\"\xe8\xb0\xb7\xe6\xad\x8c\\\" is Google\\\'s Chinese name"), + }; + // Do this twice, once for octal escapes and once for hex escapes. + for (int kind = 0; kind < 4; kind++) { + for (const std::string& original : inputs) { + std::string escaped; + switch (kind) { + case 0: + escaped = absl::CEscape(original); + break; + case 1: + escaped = absl::CHexEscape(original); + break; + case 2: + escaped = absl::Utf8SafeCEscape(original); + break; + case 3: + escaped = absl::Utf8SafeCHexEscape(original); + break; + } + std::string unescaped_str; + EXPECT_TRUE(absl::CUnescape(escaped, &unescaped_str)); + EXPECT_EQ(unescaped_str, original); + + unescaped_str.erase(); + std::string error; + EXPECT_TRUE(absl::CUnescape(escaped, &unescaped_str, &error)); + EXPECT_EQ(error, ""); + + // Check in-place unescaping + std::string s = escaped; + EXPECT_TRUE(absl::CUnescape(s, &s)); + ASSERT_EQ(s, original); + } + } + // Check that all possible two character strings can be escaped then + // unescaped successfully. + for (int char0 = 0; char0 < 256; char0++) { + for (int char1 = 0; char1 < 256; char1++) { + char chars[2]; + chars[0] = char0; + chars[1] = char1; + std::string s(chars, 2); + std::string escaped = absl::CHexEscape(s); + std::string unescaped; + EXPECT_TRUE(absl::CUnescape(escaped, &unescaped)); + EXPECT_EQ(s, unescaped); + } + } +} + +TEST(CEscape, BasicEscaping) { + epair oct_values[] = { + {"foo\\rbar\\nbaz\\t", "foo\rbar\nbaz\t"}, + {"\\'full of \\\"sound\\\" and \\\"fury\\\"\\'", + "'full of \"sound\" and \"fury\"'"}, + {"signi\\\\fying\\\\ nothing\\\\", "signi\\fying\\ nothing\\"}, + {"\\010\\t\\n\\013\\014\\r", "\010\011\012\013\014\015"} + }; + epair hex_values[] = { + {"ubik\\rubik\\nubik\\t", "ubik\rubik\nubik\t"}, + {"I\\\'ve just seen a \\\"face\\\"", + "I've just seen a \"face\""}, + {"hel\\\\ter\\\\skel\\\\ter\\\\", "hel\\ter\\skel\\ter\\"}, + {"\\x08\\t\\n\\x0b\\x0c\\r", "\010\011\012\013\014\015"} + }; + epair utf8_oct_values[] = { + {"\xe8\xb0\xb7\xe6\xad\x8c\\r\xe8\xb0\xb7\xe6\xad\x8c\\nbaz\\t", + "\xe8\xb0\xb7\xe6\xad\x8c\r\xe8\xb0\xb7\xe6\xad\x8c\nbaz\t"}, + {"\\\"\xe8\xb0\xb7\xe6\xad\x8c\\\" is Google\\\'s Chinese name", + "\"\xe8\xb0\xb7\xe6\xad\x8c\" is Google\'s Chinese name"}, + {"\xe3\x83\xa1\xe3\x83\xbc\xe3\x83\xab\\\\are\\\\Japanese\\\\chars\\\\", + "\xe3\x83\xa1\xe3\x83\xbc\xe3\x83\xab\\are\\Japanese\\chars\\"}, + {"\xed\x81\xac\xeb\xa1\xac\\010\\t\\n\\013\\014\\r", + "\xed\x81\xac\xeb\xa1\xac\010\011\012\013\014\015"} + }; + epair utf8_hex_values[] = { + {"\x20\xe4\xbd\xa0\\t\xe5\xa5\xbd,\\r!\\n", + "\x20\xe4\xbd\xa0\t\xe5\xa5\xbd,\r!\n"}, + {"\xe8\xa9\xa6\xe9\xa8\x93\\\' means \\\"test\\\"", + "\xe8\xa9\xa6\xe9\xa8\x93\' means \"test\""}, + {"\\\\\xe6\x88\x91\\\\:\\\\\xe6\x9d\xa8\xe6\xac\xa2\\\\", + "\\\xe6\x88\x91\\:\\\xe6\x9d\xa8\xe6\xac\xa2\\"}, + {"\xed\x81\xac\xeb\xa1\xac\\x08\\t\\n\\x0b\\x0c\\r", + "\xed\x81\xac\xeb\xa1\xac\010\011\012\013\014\015"} + }; + + for (const epair& val : oct_values) { + std::string escaped = absl::CEscape(val.unescaped); + EXPECT_EQ(escaped, val.escaped); + } + for (const epair& val : hex_values) { + std::string escaped = absl::CHexEscape(val.unescaped); + EXPECT_EQ(escaped, val.escaped); + } + for (const epair& val : utf8_oct_values) { + std::string escaped = absl::Utf8SafeCEscape(val.unescaped); + EXPECT_EQ(escaped, val.escaped); + } + for (const epair& val : utf8_hex_values) { + std::string escaped = absl::Utf8SafeCHexEscape(val.unescaped); + EXPECT_EQ(escaped, val.escaped); + } +} + +TEST(Unescape, BasicFunction) { + epair tests[] = + {{"", ""}, + {"\\u0030", "0"}, + {"\\u00A3", "\xC2\xA3"}, + {"\\u22FD", "\xE2\x8B\xBD"}, + {"\\U00010000", "\xF0\x90\x80\x80"}, + {"\\U0010FFFD", "\xF4\x8F\xBF\xBD"}}; + for (const epair& val : tests) { + std::string out; + EXPECT_TRUE(absl::CUnescape(val.escaped, &out)); + EXPECT_EQ(out, val.unescaped); + } + std::string bad[] = {"\\u1", // too short + "\\U1", // too short + "\\Uffffff", // exceeds 0x10ffff (largest Unicode) + "\\U00110000", // exceeds 0x10ffff (largest Unicode) + "\\uD835", // surrogate character (D800-DFFF) + "\\U0000DD04", // surrogate character (D800-DFFF) + "\\777", // exceeds 0xff + "\\xABCD"}; // exceeds 0xff + for (const std::string& e : bad) { + std::string error; + std::string out; + EXPECT_FALSE(absl::CUnescape(e, &out, &error)); + EXPECT_FALSE(error.empty()); + + out.erase(); + EXPECT_FALSE(absl::CUnescape(e, &out)); + } +} + +class CUnescapeTest : public testing::Test { + protected: + static const char kStringWithMultipleOctalNulls[]; + static const char kStringWithMultipleHexNulls[]; + static const char kStringWithMultipleUnicodeNulls[]; + + std::string result_string_; +}; + +const char CUnescapeTest::kStringWithMultipleOctalNulls[] = + "\\0\\n" // null escape \0 plus newline + "0\\n" // just a number 0 (not a null escape) plus newline + "\\00\\12" // null escape \00 plus octal newline code + "\\000"; // null escape \000 + +// This has the same ingredients as kStringWithMultipleOctalNulls +// but with \x hex escapes instead of octal escapes. +const char CUnescapeTest::kStringWithMultipleHexNulls[] = + "\\x0\\n" + "0\\n" + "\\x00\\xa" + "\\x000"; + +const char CUnescapeTest::kStringWithMultipleUnicodeNulls[] = + "\\u0000\\n" // short-form (4-digit) null escape plus newline + "0\\n" // just a number 0 (not a null escape) plus newline + "\\U00000000"; // long-form (8-digit) null escape + +TEST_F(CUnescapeTest, Unescapes1CharOctalNull) { + std::string original_string = "\\0"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes2CharOctalNull) { + std::string original_string = "\\00"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes3CharOctalNull) { + std::string original_string = "\\000"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes1CharHexNull) { + std::string original_string = "\\x0"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes2CharHexNull) { + std::string original_string = "\\x00"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes3CharHexNull) { + std::string original_string = "\\x000"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes4CharUnicodeNull) { + std::string original_string = "\\u0000"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, Unescapes8CharUnicodeNull) { + std::string original_string = "\\U00000000"; + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0", 1), result_string_); +} + +TEST_F(CUnescapeTest, UnescapesMultipleOctalNulls) { + std::string original_string(kStringWithMultipleOctalNulls); + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + // All escapes, including newlines and null escapes, should have been + // converted to the equivalent characters. + EXPECT_EQ(std::string("\0\n" + "0\n" + "\0\n" + "\0", + 7), + result_string_); +} + + +TEST_F(CUnescapeTest, UnescapesMultipleHexNulls) { + std::string original_string(kStringWithMultipleHexNulls); + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0\n" + "0\n" + "\0\n" + "\0", + 7), + result_string_); +} + +TEST_F(CUnescapeTest, UnescapesMultipleUnicodeNulls) { + std::string original_string(kStringWithMultipleUnicodeNulls); + EXPECT_TRUE(absl::CUnescape(original_string, &result_string_)); + EXPECT_EQ(std::string("\0\n" + "0\n" + "\0", + 5), + result_string_); +} + +static struct { + absl::string_view plaintext; + absl::string_view cyphertext; +} const base64_tests[] = { + // Empty string. + {{"", 0}, {"", 0}}, + {{nullptr, 0}, + {"", 0}}, // if length is zero, plaintext ptr must be ignored! + + // Basic bit patterns; + // values obtained with "echo -n '...' | uuencode -m test" + + {{"\000", 1}, "AA=="}, + {{"\001", 1}, "AQ=="}, + {{"\002", 1}, "Ag=="}, + {{"\004", 1}, "BA=="}, + {{"\010", 1}, "CA=="}, + {{"\020", 1}, "EA=="}, + {{"\040", 1}, "IA=="}, + {{"\100", 1}, "QA=="}, + {{"\200", 1}, "gA=="}, + + {{"\377", 1}, "/w=="}, + {{"\376", 1}, "/g=="}, + {{"\375", 1}, "/Q=="}, + {{"\373", 1}, "+w=="}, + {{"\367", 1}, "9w=="}, + {{"\357", 1}, "7w=="}, + {{"\337", 1}, "3w=="}, + {{"\277", 1}, "vw=="}, + {{"\177", 1}, "fw=="}, + {{"\000\000", 2}, "AAA="}, + {{"\000\001", 2}, "AAE="}, + {{"\000\002", 2}, "AAI="}, + {{"\000\004", 2}, "AAQ="}, + {{"\000\010", 2}, "AAg="}, + {{"\000\020", 2}, "ABA="}, + {{"\000\040", 2}, "ACA="}, + {{"\000\100", 2}, "AEA="}, + {{"\000\200", 2}, "AIA="}, + {{"\001\000", 2}, "AQA="}, + {{"\002\000", 2}, "AgA="}, + {{"\004\000", 2}, "BAA="}, + {{"\010\000", 2}, "CAA="}, + {{"\020\000", 2}, "EAA="}, + {{"\040\000", 2}, "IAA="}, + {{"\100\000", 2}, "QAA="}, + {{"\200\000", 2}, "gAA="}, + + {{"\377\377", 2}, "//8="}, + {{"\377\376", 2}, "//4="}, + {{"\377\375", 2}, "//0="}, + {{"\377\373", 2}, "//s="}, + {{"\377\367", 2}, "//c="}, + {{"\377\357", 2}, "/+8="}, + {{"\377\337", 2}, "/98="}, + {{"\377\277", 2}, "/78="}, + {{"\377\177", 2}, "/38="}, + {{"\376\377", 2}, "/v8="}, + {{"\375\377", 2}, "/f8="}, + {{"\373\377", 2}, "+/8="}, + {{"\367\377", 2}, "9/8="}, + {{"\357\377", 2}, "7/8="}, + {{"\337\377", 2}, "3/8="}, + {{"\277\377", 2}, "v/8="}, + {{"\177\377", 2}, "f/8="}, + + {{"\000\000\000", 3}, "AAAA"}, + {{"\000\000\001", 3}, "AAAB"}, + {{"\000\000\002", 3}, "AAAC"}, + {{"\000\000\004", 3}, "AAAE"}, + {{"\000\000\010", 3}, "AAAI"}, + {{"\000\000\020", 3}, "AAAQ"}, + {{"\000\000\040", 3}, "AAAg"}, + {{"\000\000\100", 3}, "AABA"}, + {{"\000\000\200", 3}, "AACA"}, + {{"\000\001\000", 3}, "AAEA"}, + {{"\000\002\000", 3}, "AAIA"}, + {{"\000\004\000", 3}, "AAQA"}, + {{"\000\010\000", 3}, "AAgA"}, + {{"\000\020\000", 3}, "ABAA"}, + {{"\000\040\000", 3}, "ACAA"}, + {{"\000\100\000", 3}, "AEAA"}, + {{"\000\200\000", 3}, "AIAA"}, + {{"\001\000\000", 3}, "AQAA"}, + {{"\002\000\000", 3}, "AgAA"}, + {{"\004\000\000", 3}, "BAAA"}, + {{"\010\000\000", 3}, "CAAA"}, + {{"\020\000\000", 3}, "EAAA"}, + {{"\040\000\000", 3}, "IAAA"}, + {{"\100\000\000", 3}, "QAAA"}, + {{"\200\000\000", 3}, "gAAA"}, + + {{"\377\377\377", 3}, "////"}, + {{"\377\377\376", 3}, "///+"}, + {{"\377\377\375", 3}, "///9"}, + {{"\377\377\373", 3}, "///7"}, + {{"\377\377\367", 3}, "///3"}, + {{"\377\377\357", 3}, "///v"}, + {{"\377\377\337", 3}, "///f"}, + {{"\377\377\277", 3}, "//+/"}, + {{"\377\377\177", 3}, "//9/"}, + {{"\377\376\377", 3}, "//7/"}, + {{"\377\375\377", 3}, "//3/"}, + {{"\377\373\377", 3}, "//v/"}, + {{"\377\367\377", 3}, "//f/"}, + {{"\377\357\377", 3}, "/+//"}, + {{"\377\337\377", 3}, "/9//"}, + {{"\377\277\377", 3}, "/7//"}, + {{"\377\177\377", 3}, "/3//"}, + {{"\376\377\377", 3}, "/v//"}, + {{"\375\377\377", 3}, "/f//"}, + {{"\373\377\377", 3}, "+///"}, + {{"\367\377\377", 3}, "9///"}, + {{"\357\377\377", 3}, "7///"}, + {{"\337\377\377", 3}, "3///"}, + {{"\277\377\377", 3}, "v///"}, + {{"\177\377\377", 3}, "f///"}, + + // Random numbers: values obtained with + // + // #! /bin/bash + // dd bs=$1 count=1 if=/dev/random of=/tmp/bar.random + // od -N $1 -t o1 /tmp/bar.random + // uuencode -m test < /tmp/bar.random + // + // where $1 is the number of bytes (2, 3) + + {{"\243\361", 2}, "o/E="}, + {{"\024\167", 2}, "FHc="}, + {{"\313\252", 2}, "y6o="}, + {{"\046\041", 2}, "JiE="}, + {{"\145\236", 2}, "ZZ4="}, + {{"\254\325", 2}, "rNU="}, + {{"\061\330", 2}, "Mdg="}, + {{"\245\032", 2}, "pRo="}, + {{"\006\000", 2}, "BgA="}, + {{"\375\131", 2}, "/Vk="}, + {{"\303\210", 2}, "w4g="}, + {{"\040\037", 2}, "IB8="}, + {{"\261\372", 2}, "sfo="}, + {{"\335\014", 2}, "3Qw="}, + {{"\233\217", 2}, "m48="}, + {{"\373\056", 2}, "+y4="}, + {{"\247\232", 2}, "p5o="}, + {{"\107\053", 2}, "Rys="}, + {{"\204\077", 2}, "hD8="}, + {{"\276\211", 2}, "vok="}, + {{"\313\110", 2}, "y0g="}, + {{"\363\376", 2}, "8/4="}, + {{"\251\234", 2}, "qZw="}, + {{"\103\262", 2}, "Q7I="}, + {{"\142\312", 2}, "Yso="}, + {{"\067\211", 2}, "N4k="}, + {{"\220\001", 2}, "kAE="}, + {{"\152\240", 2}, "aqA="}, + {{"\367\061", 2}, "9zE="}, + {{"\133\255", 2}, "W60="}, + {{"\176\035", 2}, "fh0="}, + {{"\032\231", 2}, "Gpk="}, + + {{"\013\007\144", 3}, "Cwdk"}, + {{"\030\112\106", 3}, "GEpG"}, + {{"\047\325\046", 3}, "J9Um"}, + {{"\310\160\022", 3}, "yHAS"}, + {{"\131\100\237", 3}, "WUCf"}, + {{"\064\342\134", 3}, "NOJc"}, + {{"\010\177\004", 3}, "CH8E"}, + {{"\345\147\205", 3}, "5WeF"}, + {{"\300\343\360", 3}, "wOPw"}, + {{"\061\240\201", 3}, "MaCB"}, + {{"\225\333\044", 3}, "ldsk"}, + {{"\215\137\352", 3}, "jV/q"}, + {{"\371\147\160", 3}, "+Wdw"}, + {{"\030\320\051", 3}, "GNAp"}, + {{"\044\174\241", 3}, "JHyh"}, + {{"\260\127\037", 3}, "sFcf"}, + {{"\111\045\033", 3}, "SSUb"}, + {{"\202\114\107", 3}, "gkxH"}, + {{"\057\371\042", 3}, "L/ki"}, + {{"\223\247\244", 3}, "k6ek"}, + {{"\047\216\144", 3}, "J45k"}, + {{"\203\070\327", 3}, "gzjX"}, + {{"\247\140\072", 3}, "p2A6"}, + {{"\124\115\116", 3}, "VE1O"}, + {{"\157\162\050", 3}, "b3Io"}, + {{"\357\223\004", 3}, "75ME"}, + {{"\052\117\156", 3}, "Kk9u"}, + {{"\347\154\000", 3}, "52wA"}, + {{"\303\012\142", 3}, "wwpi"}, + {{"\060\035\362", 3}, "MB3y"}, + {{"\130\226\361", 3}, "WJbx"}, + {{"\173\013\071", 3}, "ews5"}, + {{"\336\004\027", 3}, "3gQX"}, + {{"\357\366\234", 3}, "7/ac"}, + {{"\353\304\111", 3}, "68RJ"}, + {{"\024\264\131", 3}, "FLRZ"}, + {{"\075\114\251", 3}, "PUyp"}, + {{"\315\031\225", 3}, "zRmV"}, + {{"\154\201\276", 3}, "bIG+"}, + {{"\200\066\072", 3}, "gDY6"}, + {{"\142\350\267", 3}, "Yui3"}, + {{"\033\000\166", 3}, "GwB2"}, + {{"\210\055\077", 3}, "iC0/"}, + {{"\341\037\124", 3}, "4R9U"}, + {{"\161\103\152", 3}, "cUNq"}, + {{"\270\142\131", 3}, "uGJZ"}, + {{"\337\076\074", 3}, "3z48"}, + {{"\375\106\362", 3}, "/Uby"}, + {{"\227\301\127", 3}, "l8FX"}, + {{"\340\002\234", 3}, "4AKc"}, + {{"\121\064\033", 3}, "UTQb"}, + {{"\157\134\143", 3}, "b1xj"}, + {{"\247\055\327", 3}, "py3X"}, + {{"\340\142\005", 3}, "4GIF"}, + {{"\060\260\143", 3}, "MLBj"}, + {{"\075\203\170", 3}, "PYN4"}, + {{"\143\160\016", 3}, "Y3AO"}, + {{"\313\013\063", 3}, "ywsz"}, + {{"\174\236\135", 3}, "fJ5d"}, + {{"\103\047\026", 3}, "QycW"}, + {{"\365\005\343", 3}, "9QXj"}, + {{"\271\160\223", 3}, "uXCT"}, + {{"\362\255\172", 3}, "8q16"}, + {{"\113\012\015", 3}, "SwoN"}, + + // various lengths, generated by this python script: + // + // from std::string import lowercase as lc + // for i in range(27): + // print '{ %2d, "%s",%s "%s" },' % (i, lc[:i], ' ' * (26-i), + // lc[:i].encode('base64').strip()) + + {{"", 0}, {"", 0}}, + {"a", "YQ=="}, + {"ab", "YWI="}, + {"abc", "YWJj"}, + {"abcd", "YWJjZA=="}, + {"abcde", "YWJjZGU="}, + {"abcdef", "YWJjZGVm"}, + {"abcdefg", "YWJjZGVmZw=="}, + {"abcdefgh", "YWJjZGVmZ2g="}, + {"abcdefghi", "YWJjZGVmZ2hp"}, + {"abcdefghij", "YWJjZGVmZ2hpag=="}, + {"abcdefghijk", "YWJjZGVmZ2hpams="}, + {"abcdefghijkl", "YWJjZGVmZ2hpamts"}, + {"abcdefghijklm", "YWJjZGVmZ2hpamtsbQ=="}, + {"abcdefghijklmn", "YWJjZGVmZ2hpamtsbW4="}, + {"abcdefghijklmno", "YWJjZGVmZ2hpamtsbW5v"}, + {"abcdefghijklmnop", "YWJjZGVmZ2hpamtsbW5vcA=="}, + {"abcdefghijklmnopq", "YWJjZGVmZ2hpamtsbW5vcHE="}, + {"abcdefghijklmnopqr", "YWJjZGVmZ2hpamtsbW5vcHFy"}, + {"abcdefghijklmnopqrs", "YWJjZGVmZ2hpamtsbW5vcHFycw=="}, + {"abcdefghijklmnopqrst", "YWJjZGVmZ2hpamtsbW5vcHFyc3Q="}, + {"abcdefghijklmnopqrstu", "YWJjZGVmZ2hpamtsbW5vcHFyc3R1"}, + {"abcdefghijklmnopqrstuv", "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dg=="}, + {"abcdefghijklmnopqrstuvw", "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnc="}, + {"abcdefghijklmnopqrstuvwx", "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4"}, + {"abcdefghijklmnopqrstuvwxy", "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eQ=="}, + {"abcdefghijklmnopqrstuvwxyz", "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo="}, +}; + +template <typename StringType> +void TestEscapeAndUnescape() { + // Check the short strings; this tests the math (and boundaries) + for (const auto& tc : base64_tests) { + StringType encoded("this junk should be ignored"); + absl::Base64Escape(tc.plaintext, &encoded); + EXPECT_EQ(encoded, tc.cyphertext); + EXPECT_EQ(absl::Base64Escape(tc.plaintext), tc.cyphertext); + + StringType decoded("this junk should be ignored"); + EXPECT_TRUE(absl::Base64Unescape(encoded, &decoded)); + EXPECT_EQ(decoded, tc.plaintext); + + StringType websafe(tc.cyphertext); + for (int c = 0; c < websafe.size(); ++c) { + if ('+' == websafe[c]) websafe[c] = '-'; + if ('/' == websafe[c]) websafe[c] = '_'; + if ('=' == websafe[c]) { + websafe.resize(c); + break; + } + } + + encoded = "this junk should be ignored"; + absl::WebSafeBase64Escape(tc.plaintext, &encoded); + EXPECT_EQ(encoded, websafe); + EXPECT_EQ(absl::WebSafeBase64Escape(tc.plaintext), websafe); + + // Let's try the string version of the decoder + decoded = "this junk should be ignored"; + EXPECT_TRUE(absl::WebSafeBase64Unescape(websafe, &decoded)); + EXPECT_EQ(decoded, tc.plaintext); + } + + // Now try the long strings, this tests the streaming + for (const auto& tc : absl::strings_internal::base64_strings()) { + StringType buffer; + absl::WebSafeBase64Escape(tc.plaintext, &buffer); + EXPECT_EQ(tc.cyphertext, buffer); + EXPECT_EQ(absl::WebSafeBase64Escape(tc.plaintext), tc.cyphertext); + } + + // Verify the behavior when decoding bad data + { + absl::string_view data_set[] = {"ab-/", absl::string_view("\0bcd", 4), + absl::string_view("abc.\0", 5)}; + for (absl::string_view bad_data : data_set) { + StringType buf; + EXPECT_FALSE(absl::Base64Unescape(bad_data, &buf)); + EXPECT_FALSE(absl::WebSafeBase64Unescape(bad_data, &buf)); + EXPECT_TRUE(buf.empty()); + } + } +} + +TEST(Base64, EscapeAndUnescape) { + TestEscapeAndUnescape<std::string>(); +} + +TEST(Base64, DISABLED_HugeData) { + const size_t kSize = size_t(3) * 1000 * 1000 * 1000; + static_assert(kSize % 3 == 0, "kSize must be divisible by 3"); + const std::string huge(kSize, 'x'); + + std::string escaped; + absl::Base64Escape(huge, &escaped); + + // Generates the string that should match a base64 encoded "xxx..." string. + // "xxx" in base64 is "eHh4". + std::string expected_encoding; + expected_encoding.reserve(kSize / 3 * 4); + for (size_t i = 0; i < kSize / 3; ++i) { + expected_encoding.append("eHh4"); + } + EXPECT_EQ(expected_encoding, escaped); + + std::string unescaped; + EXPECT_TRUE(absl::Base64Unescape(escaped, &unescaped)); + EXPECT_EQ(huge, unescaped); +} + +TEST(HexAndBack, HexStringToBytes_and_BytesToHexString) { + std::string hex_mixed = "0123456789abcdefABCDEF"; + std::string bytes_expected = "\x01\x23\x45\x67\x89\xab\xcd\xef\xAB\xCD\xEF"; + std::string hex_only_lower = "0123456789abcdefabcdef"; + + std::string bytes_result = absl::HexStringToBytes(hex_mixed); + EXPECT_EQ(bytes_expected, bytes_result); + + std::string prefix_valid = hex_mixed + "?"; + std::string prefix_valid_result = absl::HexStringToBytes( + absl::string_view(prefix_valid.data(), prefix_valid.size() - 1)); + EXPECT_EQ(bytes_expected, prefix_valid_result); + + std::string infix_valid = "?" + hex_mixed + "???"; + std::string infix_valid_result = absl::HexStringToBytes( + absl::string_view(infix_valid.data() + 1, hex_mixed.size())); + EXPECT_EQ(bytes_expected, infix_valid_result); + + std::string hex_result = absl::BytesToHexString(bytes_expected); + EXPECT_EQ(hex_only_lower, hex_result); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/char_map.h b/third_party/abseil_cpp/absl/strings/internal/char_map.h new file mode 100644 index 000000000000..61484de0b795 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/char_map.h @@ -0,0 +1,156 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Character Map Class +// +// A fast, bit-vector map for 8-bit unsigned characters. +// This class is useful for non-character purposes as well. + +#ifndef ABSL_STRINGS_INTERNAL_CHAR_MAP_H_ +#define ABSL_STRINGS_INTERNAL_CHAR_MAP_H_ + +#include <cstddef> +#include <cstdint> +#include <cstring> + +#include "absl/base/macros.h" +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +class Charmap { + public: + constexpr Charmap() : m_() {} + + // Initializes with a given char*. Note that NUL is not treated as + // a terminator, but rather a char to be flicked. + Charmap(const char* str, int len) : m_() { + while (len--) SetChar(*str++); + } + + // Initializes with a given char*. NUL is treated as a terminator + // and will not be in the charmap. + explicit Charmap(const char* str) : m_() { + while (*str) SetChar(*str++); + } + + constexpr bool contains(unsigned char c) const { + return (m_[c / 64] >> (c % 64)) & 0x1; + } + + // Returns true if and only if a character exists in both maps. + bool IntersectsWith(const Charmap& c) const { + for (size_t i = 0; i < ABSL_ARRAYSIZE(m_); ++i) { + if ((m_[i] & c.m_[i]) != 0) return true; + } + return false; + } + + bool IsZero() const { + for (uint64_t c : m_) { + if (c != 0) return false; + } + return true; + } + + // Containing only a single specified char. + static constexpr Charmap Char(char x) { + return Charmap(CharMaskForWord(x, 0), CharMaskForWord(x, 1), + CharMaskForWord(x, 2), CharMaskForWord(x, 3)); + } + + // Containing all the chars in the C-string 's'. + // Note that this is expensively recursive because of the C++11 constexpr + // formulation. Use only in constexpr initializers. + static constexpr Charmap FromString(const char* s) { + return *s == 0 ? Charmap() : (Char(*s) | FromString(s + 1)); + } + + // Containing all the chars in the closed interval [lo,hi]. + static constexpr Charmap Range(char lo, char hi) { + return Charmap(RangeForWord(lo, hi, 0), RangeForWord(lo, hi, 1), + RangeForWord(lo, hi, 2), RangeForWord(lo, hi, 3)); + } + + friend constexpr Charmap operator&(const Charmap& a, const Charmap& b) { + return Charmap(a.m_[0] & b.m_[0], a.m_[1] & b.m_[1], a.m_[2] & b.m_[2], + a.m_[3] & b.m_[3]); + } + + friend constexpr Charmap operator|(const Charmap& a, const Charmap& b) { + return Charmap(a.m_[0] | b.m_[0], a.m_[1] | b.m_[1], a.m_[2] | b.m_[2], + a.m_[3] | b.m_[3]); + } + + friend constexpr Charmap operator~(const Charmap& a) { + return Charmap(~a.m_[0], ~a.m_[1], ~a.m_[2], ~a.m_[3]); + } + + private: + constexpr Charmap(uint64_t b0, uint64_t b1, uint64_t b2, uint64_t b3) + : m_{b0, b1, b2, b3} {} + + static constexpr uint64_t RangeForWord(unsigned char lo, unsigned char hi, + uint64_t word) { + return OpenRangeFromZeroForWord(hi + 1, word) & + ~OpenRangeFromZeroForWord(lo, word); + } + + // All the chars in the specified word of the range [0, upper). + static constexpr uint64_t OpenRangeFromZeroForWord(uint64_t upper, + uint64_t word) { + return (upper <= 64 * word) + ? 0 + : (upper >= 64 * (word + 1)) + ? ~static_cast<uint64_t>(0) + : (~static_cast<uint64_t>(0) >> (64 - upper % 64)); + } + + static constexpr uint64_t CharMaskForWord(unsigned char x, uint64_t word) { + return (x / 64 == word) ? (static_cast<uint64_t>(1) << (x % 64)) : 0; + } + + private: + void SetChar(unsigned char c) { + m_[c / 64] |= static_cast<uint64_t>(1) << (c % 64); + } + + uint64_t m_[4]; +}; + +// Mirror the char-classifying predicates in <cctype> +constexpr Charmap UpperCharmap() { return Charmap::Range('A', 'Z'); } +constexpr Charmap LowerCharmap() { return Charmap::Range('a', 'z'); } +constexpr Charmap DigitCharmap() { return Charmap::Range('0', '9'); } +constexpr Charmap AlphaCharmap() { return LowerCharmap() | UpperCharmap(); } +constexpr Charmap AlnumCharmap() { return DigitCharmap() | AlphaCharmap(); } +constexpr Charmap XDigitCharmap() { + return DigitCharmap() | Charmap::Range('A', 'F') | Charmap::Range('a', 'f'); +} +constexpr Charmap PrintCharmap() { return Charmap::Range(0x20, 0x7e); } +constexpr Charmap SpaceCharmap() { return Charmap::FromString("\t\n\v\f\r "); } +constexpr Charmap CntrlCharmap() { + return Charmap::Range(0, 0x7f) & ~PrintCharmap(); +} +constexpr Charmap BlankCharmap() { return Charmap::FromString("\t "); } +constexpr Charmap GraphCharmap() { return PrintCharmap() & ~SpaceCharmap(); } +constexpr Charmap PunctCharmap() { return GraphCharmap() & ~AlnumCharmap(); } + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CHAR_MAP_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/char_map_benchmark.cc b/third_party/abseil_cpp/absl/strings/internal/char_map_benchmark.cc new file mode 100644 index 000000000000..5cef967b3087 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/char_map_benchmark.cc @@ -0,0 +1,61 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/char_map.h" + +#include <cstdint> + +#include "benchmark/benchmark.h" + +namespace { + +absl::strings_internal::Charmap MakeBenchmarkMap() { + absl::strings_internal::Charmap m; + uint32_t x[] = {0x0, 0x1, 0x2, 0x3, 0xf, 0xe, 0xd, 0xc}; + for (uint32_t& t : x) t *= static_cast<uint32_t>(0x11111111UL); + for (uint32_t i = 0; i < 256; ++i) { + if ((x[i / 32] >> (i % 32)) & 1) + m = m | absl::strings_internal::Charmap::Char(i); + } + return m; +} + +// Micro-benchmark for Charmap::contains. +void BM_Contains(benchmark::State& state) { + // Loop-body replicated 10 times to increase time per iteration. + // Argument continuously changed to avoid generating common subexpressions. + const absl::strings_internal::Charmap benchmark_map = MakeBenchmarkMap(); + unsigned char c = 0; + int ops = 0; + for (auto _ : state) { + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + ops += benchmark_map.contains(c++); + } + benchmark::DoNotOptimize(ops); +} +BENCHMARK(BM_Contains); + +// We don't bother benchmarking Charmap::IsZero or Charmap::IntersectsWith; +// their running time is data-dependent and it is not worth characterizing +// "typical" data. + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/char_map_test.cc b/third_party/abseil_cpp/absl/strings/internal/char_map_test.cc new file mode 100644 index 000000000000..d3306241a404 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/char_map_test.cc @@ -0,0 +1,172 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/char_map.h" + +#include <cctype> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { + +constexpr absl::strings_internal::Charmap everything_map = + ~absl::strings_internal::Charmap(); +constexpr absl::strings_internal::Charmap nothing_map{}; + +TEST(Charmap, AllTests) { + const absl::strings_internal::Charmap also_nothing_map("", 0); + ASSERT_TRUE(everything_map.contains('\0')); + ASSERT_TRUE(!nothing_map.contains('\0')); + ASSERT_TRUE(!also_nothing_map.contains('\0')); + for (unsigned char ch = 1; ch != 0; ++ch) { + ASSERT_TRUE(everything_map.contains(ch)); + ASSERT_TRUE(!nothing_map.contains(ch)); + ASSERT_TRUE(!also_nothing_map.contains(ch)); + } + + const absl::strings_internal::Charmap symbols("&@#@^!@?", 5); + ASSERT_TRUE(symbols.contains('&')); + ASSERT_TRUE(symbols.contains('@')); + ASSERT_TRUE(symbols.contains('#')); + ASSERT_TRUE(symbols.contains('^')); + ASSERT_TRUE(!symbols.contains('!')); + ASSERT_TRUE(!symbols.contains('?')); + int cnt = 0; + for (unsigned char ch = 1; ch != 0; ++ch) + cnt += symbols.contains(ch); + ASSERT_EQ(cnt, 4); + + const absl::strings_internal::Charmap lets("^abcde", 3); + const absl::strings_internal::Charmap lets2("fghij\0klmnop", 10); + const absl::strings_internal::Charmap lets3("fghij\0klmnop"); + ASSERT_TRUE(lets2.contains('k')); + ASSERT_TRUE(!lets3.contains('k')); + + ASSERT_TRUE(symbols.IntersectsWith(lets)); + ASSERT_TRUE(!lets2.IntersectsWith(lets)); + ASSERT_TRUE(lets.IntersectsWith(symbols)); + ASSERT_TRUE(!lets.IntersectsWith(lets2)); + + ASSERT_TRUE(nothing_map.IsZero()); + ASSERT_TRUE(!lets.IsZero()); +} + +namespace { +std::string Members(const absl::strings_internal::Charmap& m) { + std::string r; + for (size_t i = 0; i < 256; ++i) + if (m.contains(i)) r.push_back(i); + return r; +} + +std::string ClosedRangeString(unsigned char lo, unsigned char hi) { + // Don't depend on lo<hi. Just increment until lo==hi. + std::string s; + while (true) { + s.push_back(lo); + if (lo == hi) break; + ++lo; + } + return s; +} + +} // namespace + +TEST(Charmap, Constexpr) { + constexpr absl::strings_internal::Charmap kEmpty = nothing_map; + EXPECT_THAT(Members(kEmpty), ""); + constexpr absl::strings_internal::Charmap kA = + absl::strings_internal::Charmap::Char('A'); + EXPECT_THAT(Members(kA), "A"); + constexpr absl::strings_internal::Charmap kAZ = + absl::strings_internal::Charmap::Range('A', 'Z'); + EXPECT_THAT(Members(kAZ), "ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + constexpr absl::strings_internal::Charmap kIdentifier = + absl::strings_internal::Charmap::Range('0', '9') | + absl::strings_internal::Charmap::Range('A', 'Z') | + absl::strings_internal::Charmap::Range('a', 'z') | + absl::strings_internal::Charmap::Char('_'); + EXPECT_THAT(Members(kIdentifier), + "0123456789" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "_" + "abcdefghijklmnopqrstuvwxyz"); + constexpr absl::strings_internal::Charmap kAll = everything_map; + for (size_t i = 0; i < 256; ++i) { + EXPECT_TRUE(kAll.contains(i)) << i; + } + constexpr absl::strings_internal::Charmap kHello = + absl::strings_internal::Charmap::FromString("Hello, world!"); + EXPECT_THAT(Members(kHello), " !,Hdelorw"); + + // test negation and intersection + constexpr absl::strings_internal::Charmap kABC = + absl::strings_internal::Charmap::Range('A', 'Z') & + ~absl::strings_internal::Charmap::Range('D', 'Z'); + EXPECT_THAT(Members(kABC), "ABC"); +} + +TEST(Charmap, Range) { + // Exhaustive testing takes too long, so test some of the boundaries that + // are perhaps going to cause trouble. + std::vector<size_t> poi = {0, 1, 2, 3, 4, 7, 8, 9, 15, + 16, 17, 30, 31, 32, 33, 63, 64, 65, + 127, 128, 129, 223, 224, 225, 254, 255}; + for (auto lo = poi.begin(); lo != poi.end(); ++lo) { + SCOPED_TRACE(*lo); + for (auto hi = lo; hi != poi.end(); ++hi) { + SCOPED_TRACE(*hi); + EXPECT_THAT(Members(absl::strings_internal::Charmap::Range(*lo, *hi)), + ClosedRangeString(*lo, *hi)); + } + } +} + +bool AsBool(int x) { return static_cast<bool>(x); } + +TEST(CharmapCtype, Match) { + for (int c = 0; c < 256; ++c) { + SCOPED_TRACE(c); + SCOPED_TRACE(static_cast<char>(c)); + EXPECT_EQ(AsBool(std::isupper(c)), + absl::strings_internal::UpperCharmap().contains(c)); + EXPECT_EQ(AsBool(std::islower(c)), + absl::strings_internal::LowerCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isdigit(c)), + absl::strings_internal::DigitCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isalpha(c)), + absl::strings_internal::AlphaCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isalnum(c)), + absl::strings_internal::AlnumCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isxdigit(c)), + absl::strings_internal::XDigitCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isprint(c)), + absl::strings_internal::PrintCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isspace(c)), + absl::strings_internal::SpaceCharmap().contains(c)); + EXPECT_EQ(AsBool(std::iscntrl(c)), + absl::strings_internal::CntrlCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isblank(c)), + absl::strings_internal::BlankCharmap().contains(c)); + EXPECT_EQ(AsBool(std::isgraph(c)), + absl::strings_internal::GraphCharmap().contains(c)); + EXPECT_EQ(AsBool(std::ispunct(c)), + absl::strings_internal::PunctCharmap().contains(c)); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/charconv_bigint.cc b/third_party/abseil_cpp/absl/strings/internal/charconv_bigint.cc new file mode 100644 index 000000000000..ebf8c0791af9 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/charconv_bigint.cc @@ -0,0 +1,359 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/charconv_bigint.h" + +#include <algorithm> +#include <cassert> +#include <string> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +namespace { + +// Table containing some large powers of 5, for fast computation. + +// Constant step size for entries in the kLargePowersOfFive table. Each entry +// is larger than the previous entry by a factor of 5**kLargePowerOfFiveStep +// (or 5**27). +// +// In other words, the Nth entry in the table is 5**(27*N). +// +// 5**27 is the largest power of 5 that fits in 64 bits. +constexpr int kLargePowerOfFiveStep = 27; + +// The largest legal index into the kLargePowersOfFive table. +// +// In other words, the largest precomputed power of 5 is 5**(27*20). +constexpr int kLargestPowerOfFiveIndex = 20; + +// Table of powers of (5**27), up to (5**27)**20 == 5**540. +// +// Used to generate large powers of 5 while limiting the number of repeated +// multiplications required. +// +// clang-format off +const uint32_t kLargePowersOfFive[] = { +// 5**27 (i=1), start=0, end=2 + 0xfa10079dU, 0x6765c793U, +// 5**54 (i=2), start=2, end=6 + 0x97d9f649U, 0x6664242dU, 0x29939b14U, 0x29c30f10U, +// 5**81 (i=3), start=6, end=12 + 0xc4f809c5U, 0x7bf3f22aU, 0x67bdae34U, 0xad340517U, 0x369d1b5fU, 0x10de1593U, +// 5**108 (i=4), start=12, end=20 + 0x92b260d1U, 0x9efff7c7U, 0x81de0ec6U, 0xaeba5d56U, 0x410664a4U, 0x4f40737aU, + 0x20d3846fU, 0x06d00f73U, +// 5**135 (i=5), start=20, end=30 + 0xff1b172dU, 0x13a1d71cU, 0xefa07617U, 0x7f682d3dU, 0xff8c90c0U, 0x3f0131e7U, + 0x3fdcb9feU, 0x917b0177U, 0x16c407a7U, 0x02c06b9dU, +// 5**162 (i=6), start=30, end=42 + 0x960f7199U, 0x056667ecU, 0xe07aefd8U, 0x80f2b9ccU, 0x8273f5e3U, 0xeb9a214aU, + 0x40b38005U, 0x0e477ad4U, 0x277d08e6U, 0xfa28b11eU, 0xd3f7d784U, 0x011c835bU, +// 5**189 (i=7), start=42, end=56 + 0xf723d9d5U, 0x3282d3f3U, 0xe00857d1U, 0x69659d25U, 0x2cf117cfU, 0x24da6d07U, + 0x954d1417U, 0x3e5d8cedU, 0x7a8bb766U, 0xfd785ae6U, 0x645436d2U, 0x40c78b34U, + 0x94151217U, 0x0072e9f7U, +// 5**216 (i=8), start=56, end=72 + 0x2b416aa1U, 0x7893c5a7U, 0xe37dc6d4U, 0x2bad2beaU, 0xf0fc846cU, 0x7575ae4bU, + 0x62587b14U, 0x83b67a34U, 0x02110cdbU, 0xf7992f55U, 0x00deb022U, 0xa4a23becU, + 0x8af5c5cdU, 0xb85b654fU, 0x818df38bU, 0x002e69d2U, +// 5**243 (i=9), start=72, end=90 + 0x3518cbbdU, 0x20b0c15fU, 0x38756c2fU, 0xfb5dc3ddU, 0x22ad2d94U, 0xbf35a952U, + 0xa699192aU, 0x9a613326U, 0xad2a9cedU, 0xd7f48968U, 0xe87dfb54U, 0xc8f05db6U, + 0x5ef67531U, 0x31c1ab49U, 0xe202ac9fU, 0x9b2957b5U, 0xa143f6d3U, 0x0012bf07U, +// 5**270 (i=10), start=90, end=110 + 0x8b971de9U, 0x21aba2e1U, 0x63944362U, 0x57172336U, 0xd9544225U, 0xfb534166U, + 0x08c563eeU, 0x14640ee2U, 0x24e40d31U, 0x02b06537U, 0x03887f14U, 0x0285e533U, + 0xb744ef26U, 0x8be3a6c4U, 0x266979b4U, 0x6761ece2U, 0xd9cb39e4U, 0xe67de319U, + 0x0d39e796U, 0x00079250U, +// 5**297 (i=11), start=110, end=132 + 0x260eb6e5U, 0xf414a796U, 0xee1a7491U, 0xdb9368ebU, 0xf50c105bU, 0x59157750U, + 0x9ed2fb5cU, 0xf6e56d8bU, 0xeaee8d23U, 0x0f319f75U, 0x2aa134d6U, 0xac2908e9U, + 0xd4413298U, 0x02f02a55U, 0x989d5a7aU, 0x70dde184U, 0xba8040a7U, 0x03200981U, + 0xbe03b11cU, 0x3c1c2a18U, 0xd60427a1U, 0x00030ee0U, +// 5**324 (i=12), start=132, end=156 + 0xce566d71U, 0xf1c4aa25U, 0x4e93ca53U, 0xa72283d0U, 0x551a73eaU, 0x3d0538e2U, + 0x8da4303fU, 0x6a58de60U, 0x0e660221U, 0x49cf61a6U, 0x8d058fc1U, 0xb9d1a14cU, + 0x4bab157dU, 0xc85c6932U, 0x518c8b9eU, 0x9b92b8d0U, 0x0d8a0e21U, 0xbd855df9U, + 0xb3ea59a1U, 0x8da29289U, 0x4584d506U, 0x3752d80fU, 0xb72569c6U, 0x00013c33U, +// 5**351 (i=13), start=156, end=182 + 0x190f354dU, 0x83695cfeU, 0xe5a4d0c7U, 0xb60fb7e8U, 0xee5bbcc4U, 0xb922054cU, + 0xbb4f0d85U, 0x48394028U, 0x1d8957dbU, 0x0d7edb14U, 0x4ecc7587U, 0x505e9e02U, + 0x4c87f36bU, 0x99e66bd6U, 0x44b9ed35U, 0x753037d4U, 0xe5fe5f27U, 0x2742c203U, + 0x13b2ed2bU, 0xdc525d2cU, 0xe6fde59aU, 0x77ffb18fU, 0x13c5752cU, 0x08a84bccU, + 0x859a4940U, 0x00007fb6U, +// 5**378 (i=14), start=182, end=210 + 0x4f98cb39U, 0xa60edbbcU, 0x83b5872eU, 0xa501acffU, 0x9cc76f78U, 0xbadd4c73U, + 0x43e989faU, 0xca7acf80U, 0x2e0c824fU, 0xb19f4ffcU, 0x092fd81cU, 0xe4eb645bU, + 0xa1ff84c2U, 0x8a5a83baU, 0xa8a1fae9U, 0x1db43609U, 0xb0fed50bU, 0x0dd7d2bdU, + 0x7d7accd8U, 0x91fa640fU, 0x37dcc6c5U, 0x1c417fd5U, 0xe4d462adU, 0xe8a43399U, + 0x131bf9a5U, 0x8df54d29U, 0x36547dc1U, 0x00003395U, +// 5**405 (i=15), start=210, end=240 + 0x5bd330f5U, 0x77d21967U, 0x1ac481b7U, 0x6be2f7ceU, 0x7f4792a9U, 0xe84c2c52U, + 0x84592228U, 0x9dcaf829U, 0xdab44ce1U, 0x3d0c311bU, 0x532e297dU, 0x4704e8b4U, + 0x9cdc32beU, 0x41e64d9dU, 0x7717bea1U, 0xa824c00dU, 0x08f50b27U, 0x0f198d77U, + 0x49bbfdf0U, 0x025c6c69U, 0xd4e55cd3U, 0xf083602bU, 0xb9f0fecdU, 0xc0864aeaU, + 0x9cb98681U, 0xaaf620e9U, 0xacb6df30U, 0x4faafe66U, 0x8af13c3bU, 0x000014d5U, +// 5**432 (i=16), start=240, end=272 + 0x682bb941U, 0x89a9f297U, 0xcba75d7bU, 0x404217b1U, 0xb4e519e9U, 0xa1bc162bU, + 0xf7f5910aU, 0x98715af5U, 0x2ff53e57U, 0xe3ef118cU, 0x490c4543U, 0xbc9b1734U, + 0x2affbe4dU, 0x4cedcb4cU, 0xfb14e99eU, 0x35e34212U, 0xece39c24U, 0x07673ab3U, + 0xe73115ddU, 0xd15d38e7U, 0x093eed3bU, 0xf8e7eac5U, 0x78a8cc80U, 0x25227aacU, + 0x3f590551U, 0x413da1cbU, 0xdf643a55U, 0xab65ad44U, 0xd70b23d7U, 0xc672cd76U, + 0x3364ea62U, 0x0000086aU, +// 5**459 (i=17), start=272, end=306 + 0x22f163ddU, 0x23cf07acU, 0xbe2af6c2U, 0xf412f6f6U, 0xc3ff541eU, 0x6eeaf7deU, + 0xa47047e0U, 0x408cda92U, 0x0f0eeb08U, 0x56deba9dU, 0xcfc6b090U, 0x8bbbdf04U, + 0x3933cdb3U, 0x9e7bb67dU, 0x9f297035U, 0x38946244U, 0xee1d37bbU, 0xde898174U, + 0x63f3559dU, 0x705b72fbU, 0x138d27d9U, 0xf8603a78U, 0x735eec44U, 0xe30987d5U, + 0xc6d38070U, 0x9cfe548eU, 0x9ff01422U, 0x7c564aa8U, 0x91cc60baU, 0xcbc3565dU, + 0x7550a50bU, 0x6909aeadU, 0x13234c45U, 0x00000366U, +// 5**486 (i=18), start=306, end=342 + 0x17954989U, 0x3a7d7709U, 0x98042de5U, 0xa9011443U, 0x45e723c2U, 0x269ffd6fU, + 0x58852a46U, 0xaaa1042aU, 0x2eee8153U, 0xb2b6c39eU, 0xaf845b65U, 0xf6c365d7U, + 0xe4cffb2bU, 0xc840e90cU, 0xabea8abbU, 0x5c58f8d2U, 0x5c19fa3aU, 0x4670910aU, + 0x4449f21cU, 0xefa645b3U, 0xcc427decU, 0x083c3d73U, 0x467cb413U, 0x6fe10ae4U, + 0x3caffc72U, 0x9f8da55eU, 0x5e5c8ea7U, 0x490594bbU, 0xf0871b0bU, 0xdd89816cU, + 0x8e931df8U, 0xe85ce1c9U, 0xcca090a5U, 0x575fa16bU, 0x6b9f106cU, 0x0000015fU, +// 5**513 (i=19), start=342, end=380 + 0xee20d805U, 0x57bc3c07U, 0xcdea624eU, 0xd3f0f52dU, 0x9924b4f4U, 0xcf968640U, + 0x61d41962U, 0xe87fb464U, 0xeaaf51c7U, 0x564c8b60U, 0xccda4028U, 0x529428bbU, + 0x313a1fa8U, 0x96bd0f94U, 0x7a82ebaaU, 0xad99e7e9U, 0xf2668cd4U, 0xbe33a45eU, + 0xfd0db669U, 0x87ee369fU, 0xd3ec20edU, 0x9c4d7db7U, 0xdedcf0d8U, 0x7cd2ca64U, + 0xe25a6577U, 0x61003fd4U, 0xe56f54ccU, 0x10b7c748U, 0x40526e5eU, 0x7300ae87U, + 0x5c439261U, 0x2c0ff469U, 0xbf723f12U, 0xb2379b61U, 0xbf59b4f5U, 0xc91b1c3fU, + 0xf0046d27U, 0x0000008dU, +// 5**540 (i=20), start=380, end=420 + 0x525c9e11U, 0xf4e0eb41U, 0xebb2895dU, 0x5da512f9U, 0x7d9b29d4U, 0x452f4edcU, + 0x0b90bc37U, 0x341777cbU, 0x63d269afU, 0x1da77929U, 0x0a5c1826U, 0x77991898U, + 0x5aeddf86U, 0xf853a877U, 0x538c31ccU, 0xe84896daU, 0xb7a0010bU, 0x17ef4de5U, + 0xa52a2adeU, 0x029fd81cU, 0x987ce701U, 0x27fefd77U, 0xdb46c66fU, 0x5d301900U, + 0x496998c0U, 0xbb6598b9U, 0x5eebb607U, 0xe547354aU, 0xdf4a2f7eU, 0xf06c4955U, + 0x96242ffaU, 0x1775fb27U, 0xbecc58ceU, 0xebf2a53bU, 0x3eaad82aU, 0xf41137baU, + 0x573e6fbaU, 0xfb4866b8U, 0x54002148U, 0x00000039U, +}; +// clang-format on + +// Returns a pointer to the big integer data for (5**27)**i. i must be +// between 1 and 20, inclusive. +const uint32_t* LargePowerOfFiveData(int i) { + return kLargePowersOfFive + i * (i - 1); +} + +// Returns the size of the big integer data for (5**27)**i, in words. i must be +// between 1 and 20, inclusive. +int LargePowerOfFiveSize(int i) { return 2 * i; } +} // namespace + +ABSL_DLL const uint32_t kFiveToNth[14] = { + 1, 5, 25, 125, 625, 3125, 15625, + 78125, 390625, 1953125, 9765625, 48828125, 244140625, 1220703125, +}; + +ABSL_DLL const uint32_t kTenToNth[10] = { + 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, +}; + +template <int max_words> +int BigUnsigned<max_words>::ReadFloatMantissa(const ParsedFloat& fp, + int significant_digits) { + SetToZero(); + assert(fp.type == FloatType::kNumber); + + if (fp.subrange_begin == nullptr) { + // We already exactly parsed the mantissa, so no more work is necessary. + words_[0] = fp.mantissa & 0xffffffffu; + words_[1] = fp.mantissa >> 32; + if (words_[1]) { + size_ = 2; + } else if (words_[0]) { + size_ = 1; + } + return fp.exponent; + } + int exponent_adjust = + ReadDigits(fp.subrange_begin, fp.subrange_end, significant_digits); + return fp.literal_exponent + exponent_adjust; +} + +template <int max_words> +int BigUnsigned<max_words>::ReadDigits(const char* begin, const char* end, + int significant_digits) { + assert(significant_digits <= Digits10() + 1); + SetToZero(); + + bool after_decimal_point = false; + // Discard any leading zeroes before the decimal point + while (begin < end && *begin == '0') { + ++begin; + } + int dropped_digits = 0; + // Discard any trailing zeroes. These may or may not be after the decimal + // point. + while (begin < end && *std::prev(end) == '0') { + --end; + ++dropped_digits; + } + if (begin < end && *std::prev(end) == '.') { + // If the string ends in '.', either before or after dropping zeroes, then + // drop the decimal point and look for more digits to drop. + dropped_digits = 0; + --end; + while (begin < end && *std::prev(end) == '0') { + --end; + ++dropped_digits; + } + } else if (dropped_digits) { + // We dropped digits, and aren't sure if they're before or after the decimal + // point. Figure that out now. + const char* dp = std::find(begin, end, '.'); + if (dp != end) { + // The dropped trailing digits were after the decimal point, so don't + // count them. + dropped_digits = 0; + } + } + // Any non-fraction digits we dropped need to be accounted for in our exponent + // adjustment. + int exponent_adjust = dropped_digits; + + uint32_t queued = 0; + int digits_queued = 0; + for (; begin != end && significant_digits > 0; ++begin) { + if (*begin == '.') { + after_decimal_point = true; + continue; + } + if (after_decimal_point) { + // For each fractional digit we emit in our parsed integer, adjust our + // decimal exponent to compensate. + --exponent_adjust; + } + int digit = (*begin - '0'); + --significant_digits; + if (significant_digits == 0 && std::next(begin) != end && + (digit == 0 || digit == 5)) { + // If this is the very last significant digit, but insignificant digits + // remain, we know that the last of those remaining significant digits is + // nonzero. (If it wasn't, we would have stripped it before we got here.) + // So if this final digit is a 0 or 5, adjust it upward by 1. + // + // This adjustment is what allows incredibly large mantissas ending in + // 500000...000000000001 to correctly round up, rather than to nearest. + ++digit; + } + queued = 10 * queued + digit; + ++digits_queued; + if (digits_queued == kMaxSmallPowerOfTen) { + MultiplyBy(kTenToNth[kMaxSmallPowerOfTen]); + AddWithCarry(0, queued); + queued = digits_queued = 0; + } + } + // Encode any remaining digits. + if (digits_queued) { + MultiplyBy(kTenToNth[digits_queued]); + AddWithCarry(0, queued); + } + + // If any insignificant digits remain, we will drop them. But if we have not + // yet read the decimal point, then we have to adjust the exponent to account + // for the dropped digits. + if (begin < end && !after_decimal_point) { + // This call to std::find will result in a pointer either to the decimal + // point, or to the end of our buffer if there was none. + // + // Either way, [begin, decimal_point) will contain the set of dropped digits + // that require an exponent adjustment. + const char* decimal_point = std::find(begin, end, '.'); + exponent_adjust += (decimal_point - begin); + } + return exponent_adjust; +} + +template <int max_words> +/* static */ BigUnsigned<max_words> BigUnsigned<max_words>::FiveToTheNth( + int n) { + BigUnsigned answer(1u); + + // Seed from the table of large powers, if possible. + bool first_pass = true; + while (n >= kLargePowerOfFiveStep) { + int big_power = + std::min(n / kLargePowerOfFiveStep, kLargestPowerOfFiveIndex); + if (first_pass) { + // just copy, rather than multiplying by 1 + std::copy( + LargePowerOfFiveData(big_power), + LargePowerOfFiveData(big_power) + LargePowerOfFiveSize(big_power), + answer.words_); + answer.size_ = LargePowerOfFiveSize(big_power); + first_pass = false; + } else { + answer.MultiplyBy(LargePowerOfFiveSize(big_power), + LargePowerOfFiveData(big_power)); + } + n -= kLargePowerOfFiveStep * big_power; + } + answer.MultiplyByFiveToTheNth(n); + return answer; +} + +template <int max_words> +void BigUnsigned<max_words>::MultiplyStep(int original_size, + const uint32_t* other_words, + int other_size, int step) { + int this_i = std::min(original_size - 1, step); + int other_i = step - this_i; + + uint64_t this_word = 0; + uint64_t carry = 0; + for (; this_i >= 0 && other_i < other_size; --this_i, ++other_i) { + uint64_t product = words_[this_i]; + product *= other_words[other_i]; + this_word += product; + carry += (this_word >> 32); + this_word &= 0xffffffff; + } + AddWithCarry(step + 1, carry); + words_[step] = this_word & 0xffffffff; + if (this_word > 0 && size_ <= step) { + size_ = step + 1; + } +} + +template <int max_words> +std::string BigUnsigned<max_words>::ToString() const { + BigUnsigned<max_words> copy = *this; + std::string result; + // Build result in reverse order + while (copy.size() > 0) { + int next_digit = copy.DivMod<10>(); + result.push_back('0' + next_digit); + } + if (result.empty()) { + result.push_back('0'); + } + std::reverse(result.begin(), result.end()); + return result; +} + +template class BigUnsigned<4>; +template class BigUnsigned<84>; + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/charconv_bigint.h b/third_party/abseil_cpp/absl/strings/internal/charconv_bigint.h new file mode 100644 index 000000000000..8f702976a80d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/charconv_bigint.h @@ -0,0 +1,423 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CHARCONV_BIGINT_H_ +#define ABSL_STRINGS_INTERNAL_CHARCONV_BIGINT_H_ + +#include <algorithm> +#include <cstdint> +#include <iostream> +#include <string> + +#include "absl/base/config.h" +#include "absl/strings/ascii.h" +#include "absl/strings/internal/charconv_parse.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// The largest power that 5 that can be raised to, and still fit in a uint32_t. +constexpr int kMaxSmallPowerOfFive = 13; +// The largest power that 10 that can be raised to, and still fit in a uint32_t. +constexpr int kMaxSmallPowerOfTen = 9; + +ABSL_DLL extern const uint32_t + kFiveToNth[kMaxSmallPowerOfFive + 1]; +ABSL_DLL extern const uint32_t kTenToNth[kMaxSmallPowerOfTen + 1]; + +// Large, fixed-width unsigned integer. +// +// Exact rounding for decimal-to-binary floating point conversion requires very +// large integer math, but a design goal of absl::from_chars is to avoid +// allocating memory. The integer precision needed for decimal-to-binary +// conversions is large but bounded, so a huge fixed-width integer class +// suffices. +// +// This is an intentionally limited big integer class. Only needed operations +// are implemented. All storage lives in an array data member, and all +// arithmetic is done in-place, to avoid requiring separate storage for operand +// and result. +// +// This is an internal class. Some methods live in the .cc file, and are +// instantiated only for the values of max_words we need. +template <int max_words> +class BigUnsigned { + public: + static_assert(max_words == 4 || max_words == 84, + "unsupported max_words value"); + + BigUnsigned() : size_(0), words_{} {} + explicit constexpr BigUnsigned(uint64_t v) + : size_((v >> 32) ? 2 : v ? 1 : 0), + words_{static_cast<uint32_t>(v & 0xffffffffu), + static_cast<uint32_t>(v >> 32)} {} + + // Constructs a BigUnsigned from the given string_view containing a decimal + // value. If the input string is not a decimal integer, constructs a 0 + // instead. + explicit BigUnsigned(absl::string_view sv) : size_(0), words_{} { + // Check for valid input, returning a 0 otherwise. This is reasonable + // behavior only because this constructor is for unit tests. + if (std::find_if_not(sv.begin(), sv.end(), ascii_isdigit) != sv.end() || + sv.empty()) { + return; + } + int exponent_adjust = + ReadDigits(sv.data(), sv.data() + sv.size(), Digits10() + 1); + if (exponent_adjust > 0) { + MultiplyByTenToTheNth(exponent_adjust); + } + } + + // Loads the mantissa value of a previously-parsed float. + // + // Returns the associated decimal exponent. The value of the parsed float is + // exactly *this * 10**exponent. + int ReadFloatMantissa(const ParsedFloat& fp, int significant_digits); + + // Returns the number of decimal digits of precision this type provides. All + // numbers with this many decimal digits or fewer are representable by this + // type. + // + // Analagous to std::numeric_limits<BigUnsigned>::digits10. + static constexpr int Digits10() { + // 9975007/1035508 is very slightly less than log10(2**32). + return static_cast<uint64_t>(max_words) * 9975007 / 1035508; + } + + // Shifts left by the given number of bits. + void ShiftLeft(int count) { + if (count > 0) { + const int word_shift = count / 32; + if (word_shift >= max_words) { + SetToZero(); + return; + } + size_ = (std::min)(size_ + word_shift, max_words); + count %= 32; + if (count == 0) { + std::copy_backward(words_, words_ + size_ - word_shift, words_ + size_); + } else { + for (int i = (std::min)(size_, max_words - 1); i > word_shift; --i) { + words_[i] = (words_[i - word_shift] << count) | + (words_[i - word_shift - 1] >> (32 - count)); + } + words_[word_shift] = words_[0] << count; + // Grow size_ if necessary. + if (size_ < max_words && words_[size_]) { + ++size_; + } + } + std::fill(words_, words_ + word_shift, 0u); + } + } + + + // Multiplies by v in-place. + void MultiplyBy(uint32_t v) { + if (size_ == 0 || v == 1) { + return; + } + if (v == 0) { + SetToZero(); + return; + } + const uint64_t factor = v; + uint64_t window = 0; + for (int i = 0; i < size_; ++i) { + window += factor * words_[i]; + words_[i] = window & 0xffffffff; + window >>= 32; + } + // If carry bits remain and there's space for them, grow size_. + if (window && size_ < max_words) { + words_[size_] = window & 0xffffffff; + ++size_; + } + } + + void MultiplyBy(uint64_t v) { + uint32_t words[2]; + words[0] = static_cast<uint32_t>(v); + words[1] = static_cast<uint32_t>(v >> 32); + if (words[1] == 0) { + MultiplyBy(words[0]); + } else { + MultiplyBy(2, words); + } + } + + // Multiplies in place by 5 to the power of n. n must be non-negative. + void MultiplyByFiveToTheNth(int n) { + while (n >= kMaxSmallPowerOfFive) { + MultiplyBy(kFiveToNth[kMaxSmallPowerOfFive]); + n -= kMaxSmallPowerOfFive; + } + if (n > 0) { + MultiplyBy(kFiveToNth[n]); + } + } + + // Multiplies in place by 10 to the power of n. n must be non-negative. + void MultiplyByTenToTheNth(int n) { + if (n > kMaxSmallPowerOfTen) { + // For large n, raise to a power of 5, then shift left by the same amount. + // (10**n == 5**n * 2**n.) This requires fewer multiplications overall. + MultiplyByFiveToTheNth(n); + ShiftLeft(n); + } else if (n > 0) { + // We can do this more quickly for very small N by using a single + // multiplication. + MultiplyBy(kTenToNth[n]); + } + } + + // Returns the value of 5**n, for non-negative n. This implementation uses + // a lookup table, and is faster then seeding a BigUnsigned with 1 and calling + // MultiplyByFiveToTheNth(). + static BigUnsigned FiveToTheNth(int n); + + // Multiplies by another BigUnsigned, in-place. + template <int M> + void MultiplyBy(const BigUnsigned<M>& other) { + MultiplyBy(other.size(), other.words()); + } + + void SetToZero() { + std::fill(words_, words_ + size_, 0u); + size_ = 0; + } + + // Returns the value of the nth word of this BigUnsigned. This is + // range-checked, and returns 0 on out-of-bounds accesses. + uint32_t GetWord(int index) const { + if (index < 0 || index >= size_) { + return 0; + } + return words_[index]; + } + + // Returns this integer as a decimal string. This is not used in the decimal- + // to-binary conversion; it is intended to aid in testing. + std::string ToString() const; + + int size() const { return size_; } + const uint32_t* words() const { return words_; } + + private: + // Reads the number between [begin, end), possibly containing a decimal point, + // into this BigUnsigned. + // + // Callers are required to ensure [begin, end) contains a valid number, with + // one or more decimal digits and at most one decimal point. This routine + // will behave unpredictably if these preconditions are not met. + // + // Only the first `significant_digits` digits are read. Digits beyond this + // limit are "sticky": If the final significant digit is 0 or 5, and if any + // dropped digit is nonzero, then that final significant digit is adjusted up + // to 1 or 6. This adjustment allows for precise rounding. + // + // Returns `exponent_adjustment`, a power-of-ten exponent adjustment to + // account for the decimal point and for dropped significant digits. After + // this function returns, + // actual_value_of_parsed_string ~= *this * 10**exponent_adjustment. + int ReadDigits(const char* begin, const char* end, int significant_digits); + + // Performs a step of big integer multiplication. This computes the full + // (64-bit-wide) values that should be added at the given index (step), and + // adds to that location in-place. + // + // Because our math all occurs in place, we must multiply starting from the + // highest word working downward. (This is a bit more expensive due to the + // extra carries involved.) + // + // This must be called in steps, for each word to be calculated, starting from + // the high end and working down to 0. The first value of `step` should be + // `std::min(original_size + other.size_ - 2, max_words - 1)`. + // The reason for this expression is that multiplying the i'th word from one + // multiplicand and the j'th word of another multiplicand creates a + // two-word-wide value to be stored at the (i+j)'th element. The highest + // word indices we will access are `original_size - 1` from this object, and + // `other.size_ - 1` from our operand. Therefore, + // `original_size + other.size_ - 2` is the first step we should calculate, + // but limited on an upper bound by max_words. + + // Working from high-to-low ensures that we do not overwrite the portions of + // the initial value of *this which are still needed for later steps. + // + // Once called with step == 0, *this contains the result of the + // multiplication. + // + // `original_size` is the size_ of *this before the first call to + // MultiplyStep(). `other_words` and `other_size` are the contents of our + // operand. `step` is the step to perform, as described above. + void MultiplyStep(int original_size, const uint32_t* other_words, + int other_size, int step); + + void MultiplyBy(int other_size, const uint32_t* other_words) { + const int original_size = size_; + const int first_step = + (std::min)(original_size + other_size - 2, max_words - 1); + for (int step = first_step; step >= 0; --step) { + MultiplyStep(original_size, other_words, other_size, step); + } + } + + // Adds a 32-bit value to the index'th word, with carry. + void AddWithCarry(int index, uint32_t value) { + if (value) { + while (index < max_words && value > 0) { + words_[index] += value; + // carry if we overflowed in this word: + if (value > words_[index]) { + value = 1; + ++index; + } else { + value = 0; + } + } + size_ = (std::min)(max_words, (std::max)(index + 1, size_)); + } + } + + void AddWithCarry(int index, uint64_t value) { + if (value && index < max_words) { + uint32_t high = value >> 32; + uint32_t low = value & 0xffffffff; + words_[index] += low; + if (words_[index] < low) { + ++high; + if (high == 0) { + // Carry from the low word caused our high word to overflow. + // Short circuit here to do the right thing. + AddWithCarry(index + 2, static_cast<uint32_t>(1)); + return; + } + } + if (high > 0) { + AddWithCarry(index + 1, high); + } else { + // Normally 32-bit AddWithCarry() sets size_, but since we don't call + // it when `high` is 0, do it ourselves here. + size_ = (std::min)(max_words, (std::max)(index + 1, size_)); + } + } + } + + // Divide this in place by a constant divisor. Returns the remainder of the + // division. + template <uint32_t divisor> + uint32_t DivMod() { + uint64_t accumulator = 0; + for (int i = size_ - 1; i >= 0; --i) { + accumulator <<= 32; + accumulator += words_[i]; + // accumulator / divisor will never overflow an int32_t in this loop + words_[i] = static_cast<uint32_t>(accumulator / divisor); + accumulator = accumulator % divisor; + } + while (size_ > 0 && words_[size_ - 1] == 0) { + --size_; + } + return static_cast<uint32_t>(accumulator); + } + + // The number of elements in words_ that may carry significant values. + // All elements beyond this point are 0. + // + // When size_ is 0, this BigUnsigned stores the value 0. + // When size_ is nonzero, is *not* guaranteed that words_[size_ - 1] is + // nonzero. This can occur due to overflow truncation. + // In particular, x.size_ != y.size_ does *not* imply x != y. + int size_; + uint32_t words_[max_words]; +}; + +// Compares two big integer instances. +// +// Returns -1 if lhs < rhs, 0 if lhs == rhs, and 1 if lhs > rhs. +template <int N, int M> +int Compare(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + int limit = (std::max)(lhs.size(), rhs.size()); + for (int i = limit - 1; i >= 0; --i) { + const uint32_t lhs_word = lhs.GetWord(i); + const uint32_t rhs_word = rhs.GetWord(i); + if (lhs_word < rhs_word) { + return -1; + } else if (lhs_word > rhs_word) { + return 1; + } + } + return 0; +} + +template <int N, int M> +bool operator==(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + int limit = (std::max)(lhs.size(), rhs.size()); + for (int i = 0; i < limit; ++i) { + if (lhs.GetWord(i) != rhs.GetWord(i)) { + return false; + } + } + return true; +} + +template <int N, int M> +bool operator!=(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + return !(lhs == rhs); +} + +template <int N, int M> +bool operator<(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + return Compare(lhs, rhs) == -1; +} + +template <int N, int M> +bool operator>(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + return rhs < lhs; +} +template <int N, int M> +bool operator<=(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + return !(rhs < lhs); +} +template <int N, int M> +bool operator>=(const BigUnsigned<N>& lhs, const BigUnsigned<M>& rhs) { + return !(lhs < rhs); +} + +// Output operator for BigUnsigned, for testing purposes only. +template <int N> +std::ostream& operator<<(std::ostream& os, const BigUnsigned<N>& num) { + return os << num.ToString(); +} + +// Explicit instantiation declarations for the sizes of BigUnsigned that we +// are using. +// +// For now, the choices of 4 and 84 are arbitrary; 4 is a small value that is +// still bigger than an int128, and 84 is a large value we will want to use +// in the from_chars implementation. +// +// Comments justifying the use of 84 belong in the from_chars implementation, +// and will be added in a follow-up CL. +extern template class BigUnsigned<4>; +extern template class BigUnsigned<84>; + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CHARCONV_BIGINT_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/charconv_bigint_test.cc b/third_party/abseil_cpp/absl/strings/internal/charconv_bigint_test.cc new file mode 100644 index 000000000000..a8b9945829e8 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/charconv_bigint_test.cc @@ -0,0 +1,260 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/charconv_bigint.h" + +#include <string> + +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +TEST(BigUnsigned, ShiftLeft) { + { + // Check that 3 * 2**100 is calculated correctly + BigUnsigned<4> num(3u); + num.ShiftLeft(100); + EXPECT_EQ(num, BigUnsigned<4>("3802951800684688204490109616128")); + } + { + // Test that overflow is truncated properly. + // 15 is 4 bits long, and BigUnsigned<4> is a 128-bit bigint. + // Shifting left by 125 bits should truncate off the high bit, so that + // 15 << 125 == 7 << 125 + // after truncation. + BigUnsigned<4> a(15u); + BigUnsigned<4> b(7u); + BigUnsigned<4> c(3u); + a.ShiftLeft(125); + b.ShiftLeft(125); + c.ShiftLeft(125); + EXPECT_EQ(a, b); + EXPECT_NE(a, c); + } + { + // Same test, larger bigint: + BigUnsigned<84> a(15u); + BigUnsigned<84> b(7u); + BigUnsigned<84> c(3u); + a.ShiftLeft(84 * 32 - 3); + b.ShiftLeft(84 * 32 - 3); + c.ShiftLeft(84 * 32 - 3); + EXPECT_EQ(a, b); + EXPECT_NE(a, c); + } + { + // Check that incrementally shifting has the same result as doing it all at + // once (attempting to capture corner cases.) + const std::string seed = "1234567890123456789012345678901234567890"; + BigUnsigned<84> a(seed); + for (int i = 1; i <= 84 * 32; ++i) { + a.ShiftLeft(1); + BigUnsigned<84> b(seed); + b.ShiftLeft(i); + EXPECT_EQ(a, b); + } + // And we should have fully rotated all bits off by now: + EXPECT_EQ(a, BigUnsigned<84>(0u)); + } + { + // Bit shifting large and small numbers by large and small offsets. + // Intended to exercise bounds-checking corner on ShiftLeft() (directly + // and under asan). + + // 2**(32*84)-1 + const BigUnsigned<84> all_bits_one( + "1474444211396924248063325089479706787923460402125687709454567433186613" + "6228083464060749874845919674257665016359189106695900028098437021384227" + "3285029708032466536084583113729486015826557532750465299832071590813090" + "2011853039837649252477307070509704043541368002938784757296893793903797" + "8180292336310543540677175225040919704702800559606097685920595947397024" + "8303316808753252115729411497720357971050627997031988036134171378490368" + "6008000778741115399296162550786288457245180872759047016734959330367829" + "5235612397427686310674725251378116268607113017720538636924549612987647" + "5767411074510311386444547332882472126067840027882117834454260409440463" + "9345147252664893456053258463203120637089916304618696601333953616715125" + "2115882482473279040772264257431663818610405673876655957323083702713344" + "4201105427930770976052393421467136557055"); + const BigUnsigned<84> zero(0u); + const BigUnsigned<84> one(1u); + // in bounds shifts + for (int i = 1; i < 84*32; ++i) { + // shifting all_bits_one to the left should result in a smaller number, + // since the high bits rotate off and the low bits are replaced with + // zeroes. + BigUnsigned<84> big_shifted = all_bits_one; + big_shifted.ShiftLeft(i); + EXPECT_GT(all_bits_one, big_shifted); + // Shifting 1 to the left should instead result in a larger number. + BigUnsigned<84> small_shifted = one; + small_shifted.ShiftLeft(i); + EXPECT_LT(one, small_shifted); + } + // Shifting by zero or a negative number has no effect + for (int no_op_shift : {0, -1, -84 * 32, std::numeric_limits<int>::min()}) { + BigUnsigned<84> big_shifted = all_bits_one; + big_shifted.ShiftLeft(no_op_shift); + EXPECT_EQ(all_bits_one, big_shifted); + BigUnsigned<84> small_shifted = one; + big_shifted.ShiftLeft(no_op_shift); + EXPECT_EQ(one, small_shifted); + } + // Shifting by an amount greater than the number of bits should result in + // zero. + for (int out_of_bounds_shift : + {84 * 32, 84 * 32 + 1, std::numeric_limits<int>::max()}) { + BigUnsigned<84> big_shifted = all_bits_one; + big_shifted.ShiftLeft(out_of_bounds_shift); + EXPECT_EQ(zero, big_shifted); + BigUnsigned<84> small_shifted = one; + small_shifted.ShiftLeft(out_of_bounds_shift); + EXPECT_EQ(zero, small_shifted); + } + } +} + +TEST(BigUnsigned, MultiplyByUint32) { + const BigUnsigned<84> factorial_100( + "933262154439441526816992388562667004907159682643816214685929638952175999" + "932299156089414639761565182862536979208272237582511852109168640000000000" + "00000000000000"); + BigUnsigned<84> a(1u); + for (uint32_t i = 1; i <= 100; ++i) { + a.MultiplyBy(i); + } + EXPECT_EQ(a, BigUnsigned<84>(factorial_100)); +} + +TEST(BigUnsigned, MultiplyByBigUnsigned) { + { + // Put the terms of factorial_200 into two bigints, and multiply them + // together. + const BigUnsigned<84> factorial_200( + "7886578673647905035523632139321850622951359776871732632947425332443594" + "4996340334292030428401198462390417721213891963883025764279024263710506" + "1926624952829931113462857270763317237396988943922445621451664240254033" + "2918641312274282948532775242424075739032403212574055795686602260319041" + "7032406235170085879617892222278962370389737472000000000000000000000000" + "0000000000000000000000000"); + BigUnsigned<84> evens(1u); + BigUnsigned<84> odds(1u); + for (uint32_t i = 1; i < 200; i += 2) { + odds.MultiplyBy(i); + evens.MultiplyBy(i + 1); + } + evens.MultiplyBy(odds); + EXPECT_EQ(evens, factorial_200); + } + { + // Multiply various powers of 10 together. + for (int a = 0 ; a < 700; a += 25) { + SCOPED_TRACE(a); + BigUnsigned<84> a_value("3" + std::string(a, '0')); + for (int b = 0; b < (700 - a); b += 25) { + SCOPED_TRACE(b); + BigUnsigned<84> b_value("2" + std::string(b, '0')); + BigUnsigned<84> expected_product("6" + std::string(a + b, '0')); + b_value.MultiplyBy(a_value); + EXPECT_EQ(b_value, expected_product); + } + } + } +} + +TEST(BigUnsigned, MultiplyByOverflow) { + { + // Check that multiplcation overflow predictably truncates. + + // A big int with all bits on. + BigUnsigned<4> all_bits_on("340282366920938463463374607431768211455"); + // Modulo 2**128, this is equal to -1. Therefore the square of this, + // modulo 2**128, should be 1. + all_bits_on.MultiplyBy(all_bits_on); + EXPECT_EQ(all_bits_on, BigUnsigned<4>(1u)); + } + { + // Try multiplying a large bigint by 2**50, and compare the result to + // shifting. + BigUnsigned<4> value_1("12345678901234567890123456789012345678"); + BigUnsigned<4> value_2("12345678901234567890123456789012345678"); + BigUnsigned<4> two_to_fiftieth(1u); + two_to_fiftieth.ShiftLeft(50); + + value_1.ShiftLeft(50); + value_2.MultiplyBy(two_to_fiftieth); + EXPECT_EQ(value_1, value_2); + } +} + +TEST(BigUnsigned, FiveToTheNth) { + { + // Sanity check that MultiplyByFiveToTheNth gives consistent answers, up to + // and including overflow. + for (int i = 0; i < 1160; ++i) { + SCOPED_TRACE(i); + BigUnsigned<84> value_1(123u); + BigUnsigned<84> value_2(123u); + value_1.MultiplyByFiveToTheNth(i); + for (int j = 0; j < i; j++) { + value_2.MultiplyBy(5u); + } + EXPECT_EQ(value_1, value_2); + } + } + { + // Check that the faster, table-lookup-based static method returns the same + // result that multiplying in-place would return, up to and including + // overflow. + for (int i = 0; i < 1160; ++i) { + SCOPED_TRACE(i); + BigUnsigned<84> value_1(1u); + value_1.MultiplyByFiveToTheNth(i); + BigUnsigned<84> value_2 = BigUnsigned<84>::FiveToTheNth(i); + EXPECT_EQ(value_1, value_2); + } + } +} + +TEST(BigUnsigned, TenToTheNth) { + { + // Sanity check MultiplyByTenToTheNth. + for (int i = 0; i < 800; ++i) { + SCOPED_TRACE(i); + BigUnsigned<84> value_1(123u); + BigUnsigned<84> value_2(123u); + value_1.MultiplyByTenToTheNth(i); + for (int j = 0; j < i; j++) { + value_2.MultiplyBy(10u); + } + EXPECT_EQ(value_1, value_2); + } + } + { + // Alternate testing approach, taking advantage of the decimal parser. + for (int i = 0; i < 200; ++i) { + SCOPED_TRACE(i); + BigUnsigned<84> value_1(135u); + value_1.MultiplyByTenToTheNth(i); + BigUnsigned<84> value_2("135" + std::string(i, '0')); + EXPECT_EQ(value_1, value_2); + } + } +} + + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/charconv_parse.cc b/third_party/abseil_cpp/absl/strings/internal/charconv_parse.cc new file mode 100644 index 000000000000..8b11868c887a --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/charconv_parse.cc @@ -0,0 +1,504 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/charconv_parse.h" +#include "absl/strings/charconv.h" + +#include <cassert> +#include <cstdint> +#include <limits> + +#include "absl/strings/internal/memutil.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +// ParseFloat<10> will read the first 19 significant digits of the mantissa. +// This number was chosen for multiple reasons. +// +// (a) First, for whatever integer type we choose to represent the mantissa, we +// want to choose the largest possible number of decimal digits for that integer +// type. We are using uint64_t, which can express any 19-digit unsigned +// integer. +// +// (b) Second, we need to parse enough digits that the binary value of any +// mantissa we capture has more bits of resolution than the mantissa +// representation in the target float. Our algorithm requires at least 3 bits +// of headway, but 19 decimal digits give a little more than that. +// +// The following static assertions verify the above comments: +constexpr int kDecimalMantissaDigitsMax = 19; + +static_assert(std::numeric_limits<uint64_t>::digits10 == + kDecimalMantissaDigitsMax, + "(a) above"); + +// IEEE doubles, which we assume in Abseil, have 53 binary bits of mantissa. +static_assert(std::numeric_limits<double>::is_iec559, "IEEE double assumed"); +static_assert(std::numeric_limits<double>::radix == 2, "IEEE double fact"); +static_assert(std::numeric_limits<double>::digits == 53, "IEEE double fact"); + +// The lowest valued 19-digit decimal mantissa we can read still contains +// sufficient information to reconstruct a binary mantissa. +static_assert(1000000000000000000u > (uint64_t(1) << (53 + 3)), "(b) above"); + +// ParseFloat<16> will read the first 15 significant digits of the mantissa. +// +// Because a base-16-to-base-2 conversion can be done exactly, we do not need +// to maximize the number of scanned hex digits to improve our conversion. What +// is required is to scan two more bits than the mantissa can represent, so that +// we always round correctly. +// +// (One extra bit does not suffice to perform correct rounding, since a number +// exactly halfway between two representable floats has unique rounding rules, +// so we need to differentiate between a "halfway between" number and a "closer +// to the larger value" number.) +constexpr int kHexadecimalMantissaDigitsMax = 15; + +// The minimum number of significant bits that will be read from +// kHexadecimalMantissaDigitsMax hex digits. We must subtract by three, since +// the most significant digit can be a "1", which only contributes a single +// significant bit. +constexpr int kGuaranteedHexadecimalMantissaBitPrecision = + 4 * kHexadecimalMantissaDigitsMax - 3; + +static_assert(kGuaranteedHexadecimalMantissaBitPrecision > + std::numeric_limits<double>::digits + 2, + "kHexadecimalMantissaDigitsMax too small"); + +// We also impose a limit on the number of significant digits we will read from +// an exponent, to avoid having to deal with integer overflow. We use 9 for +// this purpose. +// +// If we read a 9 digit exponent, the end result of the conversion will +// necessarily be infinity or zero, depending on the sign of the exponent. +// Therefore we can just drop extra digits on the floor without any extra +// logic. +constexpr int kDecimalExponentDigitsMax = 9; +static_assert(std::numeric_limits<int>::digits10 >= kDecimalExponentDigitsMax, + "int type too small"); + +// To avoid incredibly large inputs causing integer overflow for our exponent, +// we impose an arbitrary but very large limit on the number of significant +// digits we will accept. The implementation refuses to match a string with +// more consecutive significant mantissa digits than this. +constexpr int kDecimalDigitLimit = 50000000; + +// Corresponding limit for hexadecimal digit inputs. This is one fourth the +// amount of kDecimalDigitLimit, since each dropped hexadecimal digit requires +// a binary exponent adjustment of 4. +constexpr int kHexadecimalDigitLimit = kDecimalDigitLimit / 4; + +// The largest exponent we can read is 999999999 (per +// kDecimalExponentDigitsMax), and the largest exponent adjustment we can get +// from dropped mantissa digits is 2 * kDecimalDigitLimit, and the sum of these +// comfortably fits in an integer. +// +// We count kDecimalDigitLimit twice because there are independent limits for +// numbers before and after the decimal point. (In the case where there are no +// significant digits before the decimal point, there are independent limits for +// post-decimal-point leading zeroes and for significant digits.) +static_assert(999999999 + 2 * kDecimalDigitLimit < + std::numeric_limits<int>::max(), + "int type too small"); +static_assert(999999999 + 2 * (4 * kHexadecimalDigitLimit) < + std::numeric_limits<int>::max(), + "int type too small"); + +// Returns true if the provided bitfield allows parsing an exponent value +// (e.g., "1.5e100"). +bool AllowExponent(chars_format flags) { + bool fixed = (flags & chars_format::fixed) == chars_format::fixed; + bool scientific = + (flags & chars_format::scientific) == chars_format::scientific; + return scientific || !fixed; +} + +// Returns true if the provided bitfield requires an exponent value be present. +bool RequireExponent(chars_format flags) { + bool fixed = (flags & chars_format::fixed) == chars_format::fixed; + bool scientific = + (flags & chars_format::scientific) == chars_format::scientific; + return scientific && !fixed; +} + +const int8_t kAsciiToInt[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, + 9, -1, -1, -1, -1, -1, -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1}; + +// Returns true if `ch` is a digit in the given base +template <int base> +bool IsDigit(char ch); + +// Converts a valid `ch` to its digit value in the given base. +template <int base> +unsigned ToDigit(char ch); + +// Returns true if `ch` is the exponent delimiter for the given base. +template <int base> +bool IsExponentCharacter(char ch); + +// Returns the maximum number of significant digits we will read for a float +// in the given base. +template <int base> +constexpr int MantissaDigitsMax(); + +// Returns the largest consecutive run of digits we will accept when parsing a +// number in the given base. +template <int base> +constexpr int DigitLimit(); + +// Returns the amount the exponent must be adjusted by for each dropped digit. +// (For decimal this is 1, since the digits are in base 10 and the exponent base +// is also 10, but for hexadecimal this is 4, since the digits are base 16 but +// the exponent base is 2.) +template <int base> +constexpr int DigitMagnitude(); + +template <> +bool IsDigit<10>(char ch) { + return ch >= '0' && ch <= '9'; +} +template <> +bool IsDigit<16>(char ch) { + return kAsciiToInt[static_cast<unsigned char>(ch)] >= 0; +} + +template <> +unsigned ToDigit<10>(char ch) { + return ch - '0'; +} +template <> +unsigned ToDigit<16>(char ch) { + return kAsciiToInt[static_cast<unsigned char>(ch)]; +} + +template <> +bool IsExponentCharacter<10>(char ch) { + return ch == 'e' || ch == 'E'; +} + +template <> +bool IsExponentCharacter<16>(char ch) { + return ch == 'p' || ch == 'P'; +} + +template <> +constexpr int MantissaDigitsMax<10>() { + return kDecimalMantissaDigitsMax; +} +template <> +constexpr int MantissaDigitsMax<16>() { + return kHexadecimalMantissaDigitsMax; +} + +template <> +constexpr int DigitLimit<10>() { + return kDecimalDigitLimit; +} +template <> +constexpr int DigitLimit<16>() { + return kHexadecimalDigitLimit; +} + +template <> +constexpr int DigitMagnitude<10>() { + return 1; +} +template <> +constexpr int DigitMagnitude<16>() { + return 4; +} + +// Reads decimal digits from [begin, end) into *out. Returns the number of +// digits consumed. +// +// After max_digits has been read, keeps consuming characters, but no longer +// adjusts *out. If a nonzero digit is dropped this way, *dropped_nonzero_digit +// is set; otherwise, it is left unmodified. +// +// If no digits are matched, returns 0 and leaves *out unchanged. +// +// ConsumeDigits does not protect against overflow on *out; max_digits must +// be chosen with respect to type T to avoid the possibility of overflow. +template <int base, typename T> +int ConsumeDigits(const char* begin, const char* end, int max_digits, T* out, + bool* dropped_nonzero_digit) { + if (base == 10) { + assert(max_digits <= std::numeric_limits<T>::digits10); + } else if (base == 16) { + assert(max_digits * 4 <= std::numeric_limits<T>::digits); + } + const char* const original_begin = begin; + + // Skip leading zeros, but only if *out is zero. + // They don't cause an overflow so we don't have to count them for + // `max_digits`. + while (!*out && end != begin && *begin == '0') ++begin; + + T accumulator = *out; + const char* significant_digits_end = + (end - begin > max_digits) ? begin + max_digits : end; + while (begin < significant_digits_end && IsDigit<base>(*begin)) { + // Do not guard against *out overflow; max_digits was chosen to avoid this. + // Do assert against it, to detect problems in debug builds. + auto digit = static_cast<T>(ToDigit<base>(*begin)); + assert(accumulator * base >= accumulator); + accumulator *= base; + assert(accumulator + digit >= accumulator); + accumulator += digit; + ++begin; + } + bool dropped_nonzero = false; + while (begin < end && IsDigit<base>(*begin)) { + dropped_nonzero = dropped_nonzero || (*begin != '0'); + ++begin; + } + if (dropped_nonzero && dropped_nonzero_digit != nullptr) { + *dropped_nonzero_digit = true; + } + *out = accumulator; + return static_cast<int>(begin - original_begin); +} + +// Returns true if `v` is one of the chars allowed inside parentheses following +// a NaN. +bool IsNanChar(char v) { + return (v == '_') || (v >= '0' && v <= '9') || (v >= 'a' && v <= 'z') || + (v >= 'A' && v <= 'Z'); +} + +// Checks the range [begin, end) for a strtod()-formatted infinity or NaN. If +// one is found, sets `out` appropriately and returns true. +bool ParseInfinityOrNan(const char* begin, const char* end, + strings_internal::ParsedFloat* out) { + if (end - begin < 3) { + return false; + } + switch (*begin) { + case 'i': + case 'I': { + // An infinity string consists of the characters "inf" or "infinity", + // case insensitive. + if (strings_internal::memcasecmp(begin + 1, "nf", 2) != 0) { + return false; + } + out->type = strings_internal::FloatType::kInfinity; + if (end - begin >= 8 && + strings_internal::memcasecmp(begin + 3, "inity", 5) == 0) { + out->end = begin + 8; + } else { + out->end = begin + 3; + } + return true; + } + case 'n': + case 'N': { + // A NaN consists of the characters "nan", case insensitive, optionally + // followed by a parenthesized sequence of zero or more alphanumeric + // characters and/or underscores. + if (strings_internal::memcasecmp(begin + 1, "an", 2) != 0) { + return false; + } + out->type = strings_internal::FloatType::kNan; + out->end = begin + 3; + // NaN is allowed to be followed by a parenthesized string, consisting of + // only the characters [a-zA-Z0-9_]. Match that if it's present. + begin += 3; + if (begin < end && *begin == '(') { + const char* nan_begin = begin + 1; + while (nan_begin < end && IsNanChar(*nan_begin)) { + ++nan_begin; + } + if (nan_begin < end && *nan_begin == ')') { + // We found an extra NaN specifier range + out->subrange_begin = begin + 1; + out->subrange_end = nan_begin; + out->end = nan_begin + 1; + } + } + return true; + } + default: + return false; + } +} +} // namespace + +namespace strings_internal { + +template <int base> +strings_internal::ParsedFloat ParseFloat(const char* begin, const char* end, + chars_format format_flags) { + strings_internal::ParsedFloat result; + + // Exit early if we're given an empty range. + if (begin == end) return result; + + // Handle the infinity and NaN cases. + if (ParseInfinityOrNan(begin, end, &result)) { + return result; + } + + const char* const mantissa_begin = begin; + while (begin < end && *begin == '0') { + ++begin; // skip leading zeros + } + uint64_t mantissa = 0; + + int exponent_adjustment = 0; + bool mantissa_is_inexact = false; + int pre_decimal_digits = ConsumeDigits<base>( + begin, end, MantissaDigitsMax<base>(), &mantissa, &mantissa_is_inexact); + begin += pre_decimal_digits; + int digits_left; + if (pre_decimal_digits >= DigitLimit<base>()) { + // refuse to parse pathological inputs + return result; + } else if (pre_decimal_digits > MantissaDigitsMax<base>()) { + // We dropped some non-fraction digits on the floor. Adjust our exponent + // to compensate. + exponent_adjustment = + static_cast<int>(pre_decimal_digits - MantissaDigitsMax<base>()); + digits_left = 0; + } else { + digits_left = + static_cast<int>(MantissaDigitsMax<base>() - pre_decimal_digits); + } + if (begin < end && *begin == '.') { + ++begin; + if (mantissa == 0) { + // If we haven't seen any nonzero digits yet, keep skipping zeros. We + // have to adjust the exponent to reflect the changed place value. + const char* begin_zeros = begin; + while (begin < end && *begin == '0') { + ++begin; + } + int zeros_skipped = static_cast<int>(begin - begin_zeros); + if (zeros_skipped >= DigitLimit<base>()) { + // refuse to parse pathological inputs + return result; + } + exponent_adjustment -= static_cast<int>(zeros_skipped); + } + int post_decimal_digits = ConsumeDigits<base>( + begin, end, digits_left, &mantissa, &mantissa_is_inexact); + begin += post_decimal_digits; + + // Since `mantissa` is an integer, each significant digit we read after + // the decimal point requires an adjustment to the exponent. "1.23e0" will + // be stored as `mantissa` == 123 and `exponent` == -2 (that is, + // "123e-2"). + if (post_decimal_digits >= DigitLimit<base>()) { + // refuse to parse pathological inputs + return result; + } else if (post_decimal_digits > digits_left) { + exponent_adjustment -= digits_left; + } else { + exponent_adjustment -= post_decimal_digits; + } + } + // If we've found no mantissa whatsoever, this isn't a number. + if (mantissa_begin == begin) { + return result; + } + // A bare "." doesn't count as a mantissa either. + if (begin - mantissa_begin == 1 && *mantissa_begin == '.') { + return result; + } + + if (mantissa_is_inexact) { + // We dropped significant digits on the floor. Handle this appropriately. + if (base == 10) { + // If we truncated significant decimal digits, store the full range of the + // mantissa for future big integer math for exact rounding. + result.subrange_begin = mantissa_begin; + result.subrange_end = begin; + } else if (base == 16) { + // If we truncated hex digits, reflect this fact by setting the low + // ("sticky") bit. This allows for correct rounding in all cases. + mantissa |= 1; + } + } + result.mantissa = mantissa; + + const char* const exponent_begin = begin; + result.literal_exponent = 0; + bool found_exponent = false; + if (AllowExponent(format_flags) && begin < end && + IsExponentCharacter<base>(*begin)) { + bool negative_exponent = false; + ++begin; + if (begin < end && *begin == '-') { + negative_exponent = true; + ++begin; + } else if (begin < end && *begin == '+') { + ++begin; + } + const char* const exponent_digits_begin = begin; + // Exponent is always expressed in decimal, even for hexadecimal floats. + begin += ConsumeDigits<10>(begin, end, kDecimalExponentDigitsMax, + &result.literal_exponent, nullptr); + if (begin == exponent_digits_begin) { + // there were no digits where we expected an exponent. We failed to read + // an exponent and should not consume the 'e' after all. Rewind 'begin'. + found_exponent = false; + begin = exponent_begin; + } else { + found_exponent = true; + if (negative_exponent) { + result.literal_exponent = -result.literal_exponent; + } + } + } + + if (!found_exponent && RequireExponent(format_flags)) { + // Provided flags required an exponent, but none was found. This results + // in a failure to scan. + return result; + } + + // Success! + result.type = strings_internal::FloatType::kNumber; + if (result.mantissa > 0) { + result.exponent = result.literal_exponent + + (DigitMagnitude<base>() * exponent_adjustment); + } else { + result.exponent = 0; + } + result.end = begin; + return result; +} + +template ParsedFloat ParseFloat<10>(const char* begin, const char* end, + chars_format format_flags); +template ParsedFloat ParseFloat<16>(const char* begin, const char* end, + chars_format format_flags); + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/charconv_parse.h b/third_party/abseil_cpp/absl/strings/internal/charconv_parse.h new file mode 100644 index 000000000000..505998b5394a --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/charconv_parse.h @@ -0,0 +1,99 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CHARCONV_PARSE_H_ +#define ABSL_STRINGS_INTERNAL_CHARCONV_PARSE_H_ + +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/strings/charconv.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// Enum indicating whether a parsed float is a number or special value. +enum class FloatType { kNumber, kInfinity, kNan }; + +// The decomposed parts of a parsed `float` or `double`. +struct ParsedFloat { + // Representation of the parsed mantissa, with the decimal point adjusted to + // make it an integer. + // + // During decimal scanning, this contains 19 significant digits worth of + // mantissa value. If digits beyond this point are found, they + // are truncated, and if any of these dropped digits are nonzero, then + // `mantissa` is inexact, and the full mantissa is stored in [subrange_begin, + // subrange_end). + // + // During hexadecimal scanning, this contains 15 significant hex digits worth + // of mantissa value. Digits beyond this point are sticky -- they are + // truncated, but if any dropped digits are nonzero, the low bit of mantissa + // will be set. (This allows for precise rounding, and avoids the need + // to store the full mantissa in [subrange_begin, subrange_end).) + uint64_t mantissa = 0; + + // Floating point expontent. This reflects any decimal point adjustments and + // any truncated digits from the mantissa. The absolute value of the parsed + // number is represented by mantissa * (base ** exponent), where base==10 for + // decimal floats, and base==2 for hexadecimal floats. + int exponent = 0; + + // The literal exponent value scanned from the input, or 0 if none was + // present. This does not reflect any adjustments applied to mantissa. + int literal_exponent = 0; + + // The type of number scanned. + FloatType type = FloatType::kNumber; + + // When non-null, [subrange_begin, subrange_end) marks a range of characters + // that require further processing. The meaning is dependent on float type. + // If type == kNumber and this is set, this is a "wide input": the input + // mantissa contained more than 19 digits. The range contains the full + // mantissa. It plus `literal_exponent` need to be examined to find the best + // floating point match. + // If type == kNan and this is set, the range marks the contents of a + // matched parenthesized character region after the NaN. + const char* subrange_begin = nullptr; + const char* subrange_end = nullptr; + + // One-past-the-end of the successfully parsed region, or nullptr if no + // matching pattern was found. + const char* end = nullptr; +}; + +// Read the floating point number in the provided range, and populate +// ParsedFloat accordingly. +// +// format_flags is a bitmask value specifying what patterns this API will match. +// `scientific` and `fixed` are honored per std::from_chars rules +// ([utility.from.chars], C++17): if exactly one of these bits is set, then an +// exponent is required, or dislallowed, respectively. +// +// Template parameter `base` must be either 10 or 16. For base 16, a "0x" is +// *not* consumed. The `hex` bit from format_flags is ignored by ParseFloat. +template <int base> +ParsedFloat ParseFloat(const char* begin, const char* end, + absl::chars_format format_flags); + +extern template ParsedFloat ParseFloat<10>(const char* begin, const char* end, + absl::chars_format format_flags); +extern template ParsedFloat ParseFloat<16>(const char* begin, const char* end, + absl::chars_format format_flags); + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_STRINGS_INTERNAL_CHARCONV_PARSE_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/charconv_parse_test.cc b/third_party/abseil_cpp/absl/strings/internal/charconv_parse_test.cc new file mode 100644 index 000000000000..bc2d11187651 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/charconv_parse_test.cc @@ -0,0 +1,357 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/charconv_parse.h" + +#include <string> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/str_cat.h" + +using absl::chars_format; +using absl::strings_internal::FloatType; +using absl::strings_internal::ParsedFloat; +using absl::strings_internal::ParseFloat; + +namespace { + +// Check that a given string input is parsed to the expected mantissa and +// exponent. +// +// Input string `s` must contain a '$' character. It marks the end of the +// characters that should be consumed by the match. It is stripped from the +// input to ParseFloat. +// +// If input string `s` contains '[' and ']' characters, these mark the region +// of characters that should be marked as the "subrange". For NaNs, this is +// the location of the extended NaN string. For numbers, this is the location +// of the full, over-large mantissa. +template <int base> +void ExpectParsedFloat(std::string s, absl::chars_format format_flags, + FloatType expected_type, uint64_t expected_mantissa, + int expected_exponent, + int expected_literal_exponent = -999) { + SCOPED_TRACE(s); + + int begin_subrange = -1; + int end_subrange = -1; + // If s contains '[' and ']', then strip these characters and set the subrange + // indices appropriately. + std::string::size_type open_bracket_pos = s.find('['); + if (open_bracket_pos != std::string::npos) { + begin_subrange = static_cast<int>(open_bracket_pos); + s.replace(open_bracket_pos, 1, ""); + std::string::size_type close_bracket_pos = s.find(']'); + ABSL_RAW_CHECK(close_bracket_pos != absl::string_view::npos, + "Test input contains [ without matching ]"); + end_subrange = static_cast<int>(close_bracket_pos); + s.replace(close_bracket_pos, 1, ""); + } + const std::string::size_type expected_characters_matched = s.find('$'); + ABSL_RAW_CHECK(expected_characters_matched != std::string::npos, + "Input string must contain $"); + s.replace(expected_characters_matched, 1, ""); + + ParsedFloat parsed = + ParseFloat<base>(s.data(), s.data() + s.size(), format_flags); + + EXPECT_NE(parsed.end, nullptr); + if (parsed.end == nullptr) { + return; // The following tests are not useful if we fully failed to parse + } + EXPECT_EQ(parsed.type, expected_type); + if (begin_subrange == -1) { + EXPECT_EQ(parsed.subrange_begin, nullptr); + EXPECT_EQ(parsed.subrange_end, nullptr); + } else { + EXPECT_EQ(parsed.subrange_begin, s.data() + begin_subrange); + EXPECT_EQ(parsed.subrange_end, s.data() + end_subrange); + } + if (parsed.type == FloatType::kNumber) { + EXPECT_EQ(parsed.mantissa, expected_mantissa); + EXPECT_EQ(parsed.exponent, expected_exponent); + if (expected_literal_exponent != -999) { + EXPECT_EQ(parsed.literal_exponent, expected_literal_exponent); + } + } + auto characters_matched = static_cast<int>(parsed.end - s.data()); + EXPECT_EQ(characters_matched, expected_characters_matched); +} + +// Check that a given string input is parsed to the expected mantissa and +// exponent. +// +// Input string `s` must contain a '$' character. It marks the end of the +// characters that were consumed by the match. +template <int base> +void ExpectNumber(std::string s, absl::chars_format format_flags, + uint64_t expected_mantissa, int expected_exponent, + int expected_literal_exponent = -999) { + ExpectParsedFloat<base>(std::move(s), format_flags, FloatType::kNumber, + expected_mantissa, expected_exponent, + expected_literal_exponent); +} + +// Check that a given string input is parsed to the given special value. +// +// This tests against both number bases, since infinities and NaNs have +// identical representations in both modes. +void ExpectSpecial(const std::string& s, absl::chars_format format_flags, + FloatType type) { + ExpectParsedFloat<10>(s, format_flags, type, 0, 0); + ExpectParsedFloat<16>(s, format_flags, type, 0, 0); +} + +// Check that a given input string is not matched by Float. +template <int base> +void ExpectFailedParse(absl::string_view s, absl::chars_format format_flags) { + ParsedFloat parsed = + ParseFloat<base>(s.data(), s.data() + s.size(), format_flags); + EXPECT_EQ(parsed.end, nullptr); +} + +TEST(ParseFloat, SimpleValue) { + // Test that various forms of floating point numbers all parse correctly. + ExpectNumber<10>("1.23456789e5$", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789e+5$", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789E5$", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789e05$", chars_format::general, 123456789, -3); + ExpectNumber<10>("123.456789e3$", chars_format::general, 123456789, -3); + ExpectNumber<10>("0.000123456789e9$", chars_format::general, 123456789, -3); + ExpectNumber<10>("123456.789$", chars_format::general, 123456789, -3); + ExpectNumber<10>("123456789e-3$", chars_format::general, 123456789, -3); + + ExpectNumber<16>("1.234abcdefp28$", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("1.234abcdefp+28$", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("1.234ABCDEFp28$", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("1.234AbCdEfP0028$", chars_format::general, 0x1234abcdef, + -8); + ExpectNumber<16>("123.4abcdefp20$", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("0.0001234abcdefp44$", chars_format::general, 0x1234abcdef, + -8); + ExpectNumber<16>("1234abcd.ef$", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("1234abcdefp-8$", chars_format::general, 0x1234abcdef, -8); + + // ExpectNumber does not attempt to drop trailing zeroes. + ExpectNumber<10>("0001.2345678900e005$", chars_format::general, 12345678900, + -5); + ExpectNumber<16>("0001.234abcdef000p28$", chars_format::general, + 0x1234abcdef000, -20); + + // Ensure non-matching characters after a number are ignored, even when they + // look like potentially matching characters. + ExpectNumber<10>("1.23456789e5$ ", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789e5$e5e5", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789e5$.25", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789e5$-", chars_format::general, 123456789, -3); + ExpectNumber<10>("1.23456789e5$PUPPERS!!!", chars_format::general, 123456789, + -3); + ExpectNumber<10>("123456.789$efghij", chars_format::general, 123456789, -3); + ExpectNumber<10>("123456.789$e", chars_format::general, 123456789, -3); + ExpectNumber<10>("123456.789$p5", chars_format::general, 123456789, -3); + ExpectNumber<10>("123456.789$.10", chars_format::general, 123456789, -3); + + ExpectNumber<16>("1.234abcdefp28$ ", chars_format::general, 0x1234abcdef, + -8); + ExpectNumber<16>("1.234abcdefp28$p28", chars_format::general, 0x1234abcdef, + -8); + ExpectNumber<16>("1.234abcdefp28$.125", chars_format::general, 0x1234abcdef, + -8); + ExpectNumber<16>("1.234abcdefp28$-", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("1.234abcdefp28$KITTEHS!!!", chars_format::general, + 0x1234abcdef, -8); + ExpectNumber<16>("1234abcd.ef$ghijk", chars_format::general, 0x1234abcdef, + -8); + ExpectNumber<16>("1234abcd.ef$p", chars_format::general, 0x1234abcdef, -8); + ExpectNumber<16>("1234abcd.ef$.10", chars_format::general, 0x1234abcdef, -8); + + // Ensure we can read a full resolution mantissa without overflow. + ExpectNumber<10>("9999999999999999999$", chars_format::general, + 9999999999999999999u, 0); + ExpectNumber<16>("fffffffffffffff$", chars_format::general, + 0xfffffffffffffffu, 0); + + // Check that zero is consistently read. + ExpectNumber<10>("0$", chars_format::general, 0, 0); + ExpectNumber<16>("0$", chars_format::general, 0, 0); + ExpectNumber<10>("000000000000000000000000000000000000000$", + chars_format::general, 0, 0); + ExpectNumber<16>("000000000000000000000000000000000000000$", + chars_format::general, 0, 0); + ExpectNumber<10>("0000000000000000000000.000000000000000000$", + chars_format::general, 0, 0); + ExpectNumber<16>("0000000000000000000000.000000000000000000$", + chars_format::general, 0, 0); + ExpectNumber<10>("0.00000000000000000000000000000000e123456$", + chars_format::general, 0, 0); + ExpectNumber<16>("0.00000000000000000000000000000000p123456$", + chars_format::general, 0, 0); +} + +TEST(ParseFloat, LargeDecimalMantissa) { + // After 19 significant decimal digits in the mantissa, ParsedFloat will + // truncate additional digits. We need to test that: + // 1) the truncation to 19 digits happens + // 2) the returned exponent reflects the dropped significant digits + // 3) a correct literal_exponent is set + // + // If and only if a significant digit is found after 19 digits, then the + // entirety of the mantissa in case the exact value is needed to make a + // rounding decision. The [ and ] characters below denote where such a + // subregion was marked by by ParseFloat. They are not part of the input. + + // Mark a capture group only if a dropped digit is significant (nonzero). + ExpectNumber<10>("100000000000000000000000000$", chars_format::general, + 1000000000000000000, + /* adjusted exponent */ 8); + + ExpectNumber<10>("123456789123456789100000000$", chars_format::general, + 1234567891234567891, + /* adjusted exponent */ 8); + + ExpectNumber<10>("[123456789123456789123456789]$", chars_format::general, + 1234567891234567891, + /* adjusted exponent */ 8, + /* literal exponent */ 0); + + ExpectNumber<10>("[123456789123456789100000009]$", chars_format::general, + 1234567891234567891, + /* adjusted exponent */ 8, + /* literal exponent */ 0); + + ExpectNumber<10>("[123456789123456789120000000]$", chars_format::general, + 1234567891234567891, + /* adjusted exponent */ 8, + /* literal exponent */ 0); + + // Leading zeroes should not count towards the 19 significant digit limit + ExpectNumber<10>("[00000000123456789123456789123456789]$", + chars_format::general, 1234567891234567891, + /* adjusted exponent */ 8, + /* literal exponent */ 0); + + ExpectNumber<10>("00000000123456789123456789100000000$", + chars_format::general, 1234567891234567891, + /* adjusted exponent */ 8); + + // Truncated digits after the decimal point should not cause a further + // exponent adjustment. + ExpectNumber<10>("1.234567891234567891e123$", chars_format::general, + 1234567891234567891, 105); + ExpectNumber<10>("[1.23456789123456789123456789]e123$", chars_format::general, + 1234567891234567891, + /* adjusted exponent */ 105, + /* literal exponent */ 123); + + // Ensure we truncate, and not round. (The from_chars algorithm we use + // depends on our guess missing low, if it misses, so we need the rounding + // error to be downward.) + ExpectNumber<10>("[1999999999999999999999]$", chars_format::general, + 1999999999999999999, + /* adjusted exponent */ 3, + /* literal exponent */ 0); +} + +TEST(ParseFloat, LargeHexadecimalMantissa) { + // After 15 significant hex digits in the mantissa, ParsedFloat will treat + // additional digits as sticky, We need to test that: + // 1) The truncation to 15 digits happens + // 2) The returned exponent reflects the dropped significant digits + // 3) If a nonzero digit is dropped, the low bit of mantissa is set. + + ExpectNumber<16>("123456789abcdef123456789abcdef$", chars_format::general, + 0x123456789abcdef, 60); + + // Leading zeroes should not count towards the 15 significant digit limit + ExpectNumber<16>("000000123456789abcdef123456789abcdef$", + chars_format::general, 0x123456789abcdef, 60); + + // Truncated digits after the radix point should not cause a further + // exponent adjustment. + ExpectNumber<16>("1.23456789abcdefp100$", chars_format::general, + 0x123456789abcdef, 44); + ExpectNumber<16>("1.23456789abcdef123456789abcdefp100$", + chars_format::general, 0x123456789abcdef, 44); + + // test sticky digit behavior. The low bit should be set iff any dropped + // digit is nonzero. + ExpectNumber<16>("123456789abcdee123456789abcdee$", chars_format::general, + 0x123456789abcdef, 60); + ExpectNumber<16>("123456789abcdee000000000000001$", chars_format::general, + 0x123456789abcdef, 60); + ExpectNumber<16>("123456789abcdee000000000000000$", chars_format::general, + 0x123456789abcdee, 60); +} + +TEST(ParseFloat, ScientificVsFixed) { + // In fixed mode, an exponent is never matched (but the remainder of the + // number will be matched.) + ExpectNumber<10>("1.23456789$e5", chars_format::fixed, 123456789, -8); + ExpectNumber<10>("123456.789$", chars_format::fixed, 123456789, -3); + ExpectNumber<16>("1.234abcdef$p28", chars_format::fixed, 0x1234abcdef, -36); + ExpectNumber<16>("1234abcd.ef$", chars_format::fixed, 0x1234abcdef, -8); + + // In scientific mode, numbers don't match *unless* they have an exponent. + ExpectNumber<10>("1.23456789e5$", chars_format::scientific, 123456789, -3); + ExpectFailedParse<10>("-123456.789$", chars_format::scientific); + ExpectNumber<16>("1.234abcdefp28$", chars_format::scientific, 0x1234abcdef, + -8); + ExpectFailedParse<16>("1234abcd.ef$", chars_format::scientific); +} + +TEST(ParseFloat, Infinity) { + ExpectFailedParse<10>("in", chars_format::general); + ExpectFailedParse<16>("in", chars_format::general); + ExpectFailedParse<10>("inx", chars_format::general); + ExpectFailedParse<16>("inx", chars_format::general); + ExpectSpecial("inf$", chars_format::general, FloatType::kInfinity); + ExpectSpecial("Inf$", chars_format::general, FloatType::kInfinity); + ExpectSpecial("INF$", chars_format::general, FloatType::kInfinity); + ExpectSpecial("inf$inite", chars_format::general, FloatType::kInfinity); + ExpectSpecial("iNfInItY$", chars_format::general, FloatType::kInfinity); + ExpectSpecial("infinity$!!!", chars_format::general, FloatType::kInfinity); +} + +TEST(ParseFloat, NaN) { + ExpectFailedParse<10>("na", chars_format::general); + ExpectFailedParse<16>("na", chars_format::general); + ExpectFailedParse<10>("nah", chars_format::general); + ExpectFailedParse<16>("nah", chars_format::general); + ExpectSpecial("nan$", chars_format::general, FloatType::kNan); + ExpectSpecial("NaN$", chars_format::general, FloatType::kNan); + ExpectSpecial("nAn$", chars_format::general, FloatType::kNan); + ExpectSpecial("NAN$", chars_format::general, FloatType::kNan); + ExpectSpecial("NaN$aNaNaNaNaBatman!", chars_format::general, FloatType::kNan); + + // A parenthesized sequence of the characters [a-zA-Z0-9_] is allowed to + // appear after an NaN. Check that this is allowed, and that the correct + // characters are grouped. + // + // (The characters [ and ] in the pattern below delimit the expected matched + // subgroup; they are not part of the input passed to ParseFloat.) + ExpectSpecial("nan([0xabcdef])$", chars_format::general, FloatType::kNan); + ExpectSpecial("nan([0xabcdef])$...", chars_format::general, FloatType::kNan); + ExpectSpecial("nan([0xabcdef])$)...", chars_format::general, FloatType::kNan); + ExpectSpecial("nan([])$", chars_format::general, FloatType::kNan); + ExpectSpecial("nan([aAzZ09_])$", chars_format::general, FloatType::kNan); + // If the subgroup contains illegal characters, don't match it at all. + ExpectSpecial("nan$(bad-char)", chars_format::general, FloatType::kNan); + // Also cope with a missing close paren. + ExpectSpecial("nan$(0xabcdef", chars_format::general, FloatType::kNan); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/cord_internal.h b/third_party/abseil_cpp/absl/strings/internal/cord_internal.h new file mode 100644 index 000000000000..aa91a691b949 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/cord_internal.h @@ -0,0 +1,270 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_ +#define ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_ + +#include <atomic> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <type_traits> + +#include "absl/base/internal/invoke.h" +#include "absl/container/internal/compressed_tuple.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Wraps std::atomic for reference counting. +class Refcount { + public: + constexpr Refcount() : count_{kRefIncrement} {} + struct Immortal {}; + explicit constexpr Refcount(Immortal) : count_(kImmortalTag) {} + + // Increments the reference count. Imposes no memory ordering. + inline void Increment() { + count_.fetch_add(kRefIncrement, std::memory_order_relaxed); + } + + // Asserts that the current refcount is greater than 0. If the refcount is + // greater than 1, decrements the reference count. + // + // Returns false if there are no references outstanding; true otherwise. + // Inserts barriers to ensure that state written before this method returns + // false will be visible to a thread that just observed this method returning + // false. + inline bool Decrement() { + int32_t refcount = count_.load(std::memory_order_acquire); + assert(refcount > 0 || refcount & kImmortalTag); + return refcount != kRefIncrement && + count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) != + kRefIncrement; + } + + // Same as Decrement but expect that refcount is greater than 1. + inline bool DecrementExpectHighRefcount() { + int32_t refcount = + count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel); + assert(refcount > 0 || refcount & kImmortalTag); + return refcount != kRefIncrement; + } + + // Returns the current reference count using acquire semantics. + inline int32_t Get() const { + return count_.load(std::memory_order_acquire) >> kImmortalShift; + } + + // Returns whether the atomic integer is 1. + // If the reference count is used in the conventional way, a + // reference count of 1 implies that the current thread owns the + // reference and no other thread shares it. + // This call performs the test for a reference count of one, and + // performs the memory barrier needed for the owning thread + // to act on the object, knowing that it has exclusive access to the + // object. + inline bool IsOne() { + return count_.load(std::memory_order_acquire) == kRefIncrement; + } + + bool IsImmortal() const { + return (count_.load(std::memory_order_relaxed) & kImmortalTag) != 0; + } + + private: + // We reserve the bottom bit to tag a reference count as immortal. + // By making it `1` we ensure that we never reach `0` when adding/subtracting + // `2`, thus it never looks as if it should be destroyed. + // These are used for the StringConstant constructor where we do not increase + // the refcount at construction time (due to constinit requirements) but we + // will still decrease it at destruction time to avoid branching on Unref. + enum { + kImmortalShift = 1, + kRefIncrement = 1 << kImmortalShift, + kImmortalTag = kRefIncrement - 1 + }; + + std::atomic<int32_t> count_; +}; + +// The overhead of a vtable is too much for Cord, so we roll our own subclasses +// using only a single byte to differentiate classes from each other - the "tag" +// byte. Define the subclasses first so we can provide downcasting helper +// functions in the base class. + +struct CordRepConcat; +struct CordRepSubstring; +struct CordRepExternal; + +// Various representations that we allow +enum CordRepKind { + CONCAT = 0, + EXTERNAL = 1, + SUBSTRING = 2, + + // We have different tags for different sized flat arrays, + // starting with FLAT + FLAT = 3, +}; + +struct CordRep { + CordRep() = default; + constexpr CordRep(Refcount::Immortal immortal, size_t l) + : length(l), refcount(immortal), tag(EXTERNAL), data{} {} + + // The following three fields have to be less than 32 bytes since + // that is the smallest supported flat node size. + size_t length; + Refcount refcount; + // If tag < FLAT, it represents CordRepKind and indicates the type of node. + // Otherwise, the node type is CordRepFlat and the tag is the encoded size. + uint8_t tag; + char data[1]; // Starting point for flat array: MUST BE LAST FIELD of CordRep + + inline CordRepConcat* concat(); + inline const CordRepConcat* concat() const; + inline CordRepSubstring* substring(); + inline const CordRepSubstring* substring() const; + inline CordRepExternal* external(); + inline const CordRepExternal* external() const; +}; + +struct CordRepConcat : public CordRep { + CordRep* left; + CordRep* right; + + uint8_t depth() const { return static_cast<uint8_t>(data[0]); } + void set_depth(uint8_t depth) { data[0] = static_cast<char>(depth); } +}; + +struct CordRepSubstring : public CordRep { + size_t start; // Starting offset of substring in child + CordRep* child; +}; + +// Type for function pointer that will invoke the releaser function and also +// delete the `CordRepExternalImpl` corresponding to the passed in +// `CordRepExternal`. +using ExternalReleaserInvoker = void (*)(CordRepExternal*); + +// External CordReps are allocated together with a type erased releaser. The +// releaser is stored in the memory directly following the CordRepExternal. +struct CordRepExternal : public CordRep { + CordRepExternal() = default; + explicit constexpr CordRepExternal(absl::string_view str) + : CordRep(Refcount::Immortal{}, str.size()), + base(str.data()), + releaser_invoker(nullptr) {} + + const char* base; + // Pointer to function that knows how to call and destroy the releaser. + ExternalReleaserInvoker releaser_invoker; +}; + +struct Rank1 {}; +struct Rank0 : Rank1 {}; + +template <typename Releaser, typename = ::absl::base_internal::invoke_result_t< + Releaser, absl::string_view>> +void InvokeReleaser(Rank0, Releaser&& releaser, absl::string_view data) { + ::absl::base_internal::invoke(std::forward<Releaser>(releaser), data); +} + +template <typename Releaser, + typename = ::absl::base_internal::invoke_result_t<Releaser>> +void InvokeReleaser(Rank1, Releaser&& releaser, absl::string_view) { + ::absl::base_internal::invoke(std::forward<Releaser>(releaser)); +} + +// We use CompressedTuple so that we can benefit from EBCO. +template <typename Releaser> +struct CordRepExternalImpl + : public CordRepExternal, + public ::absl::container_internal::CompressedTuple<Releaser> { + // The extra int arg is so that we can avoid interfering with copy/move + // constructors while still benefitting from perfect forwarding. + template <typename T> + CordRepExternalImpl(T&& releaser, int) + : CordRepExternalImpl::CompressedTuple(std::forward<T>(releaser)) { + this->releaser_invoker = &Release; + } + + ~CordRepExternalImpl() { + InvokeReleaser(Rank0{}, std::move(this->template get<0>()), + absl::string_view(base, length)); + } + + static void Release(CordRepExternal* rep) { + delete static_cast<CordRepExternalImpl*>(rep); + } +}; + +template <typename Str> +struct ConstInitExternalStorage { + ABSL_CONST_INIT static CordRepExternal value; +}; + +template <typename Str> +CordRepExternal ConstInitExternalStorage<Str>::value(Str::value); + +enum { + kMaxInline = 15, + // Tag byte & kMaxInline means we are storing a pointer. + kTreeFlag = 1 << 4, + // Tag byte & kProfiledFlag means we are profiling the Cord. + kProfiledFlag = 1 << 5 +}; + +// If the data has length <= kMaxInline, we store it in `as_chars`, and +// store the size in `tagged_size`. +// Else we store it in a tree and store a pointer to that tree in +// `as_tree.rep` and store a tag in `tagged_size`. +struct AsTree { + absl::cord_internal::CordRep* rep; + char padding[kMaxInline + 1 - sizeof(absl::cord_internal::CordRep*) - 1]; + char tagged_size; +}; + +constexpr char GetOrNull(absl::string_view data, size_t pos) { + return pos < data.size() ? data[pos] : '\0'; +} + +union InlineData { + constexpr InlineData() : as_chars{} {} + explicit constexpr InlineData(AsTree tree) : as_tree(tree) {} + explicit constexpr InlineData(absl::string_view chars) + : as_chars{GetOrNull(chars, 0), GetOrNull(chars, 1), + GetOrNull(chars, 2), GetOrNull(chars, 3), + GetOrNull(chars, 4), GetOrNull(chars, 5), + GetOrNull(chars, 6), GetOrNull(chars, 7), + GetOrNull(chars, 8), GetOrNull(chars, 9), + GetOrNull(chars, 10), GetOrNull(chars, 11), + GetOrNull(chars, 12), GetOrNull(chars, 13), + GetOrNull(chars, 14), static_cast<char>(chars.size())} {} + + AsTree as_tree; + char as_chars[kMaxInline + 1]; +}; +static_assert(sizeof(InlineData) == kMaxInline + 1, ""); +static_assert(sizeof(AsTree) == sizeof(InlineData), ""); +static_assert(offsetof(AsTree, tagged_size) == kMaxInline, ""); + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_STRINGS_INTERNAL_CORD_INTERNAL_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/escaping.cc b/third_party/abseil_cpp/absl/strings/internal/escaping.cc new file mode 100644 index 000000000000..c5271286ad00 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/escaping.cc @@ -0,0 +1,180 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/escaping.h" + +#include "absl/base/internal/endian.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +const char kBase64Chars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +size_t CalculateBase64EscapedLenInternal(size_t input_len, bool do_padding) { + // Base64 encodes three bytes of input at a time. If the input is not + // divisible by three, we pad as appropriate. + // + // (from https://tools.ietf.org/html/rfc3548) + // Special processing is performed if fewer than 24 bits are available + // at the end of the data being encoded. A full encoding quantum is + // always completed at the end of a quantity. When fewer than 24 input + // bits are available in an input group, zero bits are added (on the + // right) to form an integral number of 6-bit groups. Padding at the + // end of the data is performed using the '=' character. Since all base + // 64 input is an integral number of octets, only the following cases + // can arise: + + // Base64 encodes each three bytes of input into four bytes of output. + size_t len = (input_len / 3) * 4; + + if (input_len % 3 == 0) { + // (from https://tools.ietf.org/html/rfc3548) + // (1) the final quantum of encoding input is an integral multiple of 24 + // bits; here, the final unit of encoded output will be an integral + // multiple of 4 characters with no "=" padding, + } else if (input_len % 3 == 1) { + // (from https://tools.ietf.org/html/rfc3548) + // (2) the final quantum of encoding input is exactly 8 bits; here, the + // final unit of encoded output will be two characters followed by two + // "=" padding characters, or + len += 2; + if (do_padding) { + len += 2; + } + } else { // (input_len % 3 == 2) + // (from https://tools.ietf.org/html/rfc3548) + // (3) the final quantum of encoding input is exactly 16 bits; here, the + // final unit of encoded output will be three characters followed by one + // "=" padding character. + len += 3; + if (do_padding) { + len += 1; + } + } + + assert(len >= input_len); // make sure we didn't overflow + return len; +} + +size_t Base64EscapeInternal(const unsigned char* src, size_t szsrc, char* dest, + size_t szdest, const char* base64, + bool do_padding) { + static const char kPad64 = '='; + + if (szsrc * 4 > szdest * 3) return 0; + + char* cur_dest = dest; + const unsigned char* cur_src = src; + + char* const limit_dest = dest + szdest; + const unsigned char* const limit_src = src + szsrc; + + // Three bytes of data encodes to four characters of cyphertext. + // So we can pump through three-byte chunks atomically. + if (szsrc >= 3) { // "limit_src - 3" is UB if szsrc < 3. + while (cur_src < limit_src - 3) { // While we have >= 32 bits. + uint32_t in = absl::big_endian::Load32(cur_src) >> 8; + + cur_dest[0] = base64[in >> 18]; + in &= 0x3FFFF; + cur_dest[1] = base64[in >> 12]; + in &= 0xFFF; + cur_dest[2] = base64[in >> 6]; + in &= 0x3F; + cur_dest[3] = base64[in]; + + cur_dest += 4; + cur_src += 3; + } + } + // To save time, we didn't update szdest or szsrc in the loop. So do it now. + szdest = limit_dest - cur_dest; + szsrc = limit_src - cur_src; + + /* now deal with the tail (<=3 bytes) */ + switch (szsrc) { + case 0: + // Nothing left; nothing more to do. + break; + case 1: { + // One byte left: this encodes to two characters, and (optionally) + // two pad characters to round out the four-character cypherblock. + if (szdest < 2) return 0; + uint32_t in = cur_src[0]; + cur_dest[0] = base64[in >> 2]; + in &= 0x3; + cur_dest[1] = base64[in << 4]; + cur_dest += 2; + szdest -= 2; + if (do_padding) { + if (szdest < 2) return 0; + cur_dest[0] = kPad64; + cur_dest[1] = kPad64; + cur_dest += 2; + szdest -= 2; + } + break; + } + case 2: { + // Two bytes left: this encodes to three characters, and (optionally) + // one pad character to round out the four-character cypherblock. + if (szdest < 3) return 0; + uint32_t in = absl::big_endian::Load16(cur_src); + cur_dest[0] = base64[in >> 10]; + in &= 0x3FF; + cur_dest[1] = base64[in >> 4]; + in &= 0x00F; + cur_dest[2] = base64[in << 2]; + cur_dest += 3; + szdest -= 3; + if (do_padding) { + if (szdest < 1) return 0; + cur_dest[0] = kPad64; + cur_dest += 1; + szdest -= 1; + } + break; + } + case 3: { + // Three bytes left: same as in the big loop above. We can't do this in + // the loop because the loop above always reads 4 bytes, and the fourth + // byte is past the end of the input. + if (szdest < 4) return 0; + uint32_t in = (cur_src[0] << 16) + absl::big_endian::Load16(cur_src + 1); + cur_dest[0] = base64[in >> 18]; + in &= 0x3FFFF; + cur_dest[1] = base64[in >> 12]; + in &= 0xFFF; + cur_dest[2] = base64[in >> 6]; + in &= 0x3F; + cur_dest[3] = base64[in]; + cur_dest += 4; + szdest -= 4; + break; + } + default: + // Should not be reached: blocks of 4 bytes are handled + // in the while loop before this switch statement. + ABSL_RAW_LOG(FATAL, "Logic problem? szsrc = %zu", szsrc); + break; + } + return (cur_dest - dest); +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/escaping.h b/third_party/abseil_cpp/absl/strings/internal/escaping.h new file mode 100644 index 000000000000..6a9ce602d9ed --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/escaping.h @@ -0,0 +1,58 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_ESCAPING_H_ +#define ABSL_STRINGS_INTERNAL_ESCAPING_H_ + +#include <cassert> + +#include "absl/strings/internal/resize_uninitialized.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +ABSL_CONST_INIT extern const char kBase64Chars[]; + +// Calculates how long a string will be when it is base64 encoded given its +// length and whether or not the result should be padded. +size_t CalculateBase64EscapedLenInternal(size_t input_len, bool do_padding); + +// Base64-encodes `src` using the alphabet provided in `base64` and writes the +// result to `dest`. If `do_padding` is true, `dest` is padded with '=' chars +// until its length is a multiple of 3. Returns the length of `dest`. +size_t Base64EscapeInternal(const unsigned char* src, size_t szsrc, char* dest, + size_t szdest, const char* base64, bool do_padding); + +// Base64-encodes `src` using the alphabet provided in `base64` and writes the +// result to `dest`. If `do_padding` is true, `dest` is padded with '=' chars +// until its length is a multiple of 3. +template <typename String> +void Base64EscapeInternal(const unsigned char* src, size_t szsrc, String* dest, + bool do_padding, const char* base64_chars) { + const size_t calc_escaped_size = + CalculateBase64EscapedLenInternal(szsrc, do_padding); + STLStringResizeUninitialized(dest, calc_escaped_size); + + const size_t escaped_len = Base64EscapeInternal( + src, szsrc, &(*dest)[0], dest->size(), base64_chars, do_padding); + assert(calc_escaped_size == escaped_len); + dest->erase(escaped_len); +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_ESCAPING_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/escaping_test_common.h b/third_party/abseil_cpp/absl/strings/internal/escaping_test_common.h new file mode 100644 index 000000000000..7b18017a0890 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/escaping_test_common.h @@ -0,0 +1,133 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This test contains common things needed by both escaping_test.cc and +// escaping_benchmark.cc. + +#ifndef ABSL_STRINGS_INTERNAL_ESCAPING_TEST_COMMON_H_ +#define ABSL_STRINGS_INTERNAL_ESCAPING_TEST_COMMON_H_ + +#include <array> +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +struct base64_testcase { + absl::string_view plaintext; + absl::string_view cyphertext; +}; + +inline const std::array<base64_testcase, 5>& base64_strings() { + static const std::array<base64_testcase, 5> testcase{{ + // Some google quotes + // Cyphertext created with "uuencode (GNU sharutils) 4.6.3" + // (Note that we're testing the websafe encoding, though, so if + // you add messages, be sure to run "tr -- '+/' '-_'" on the output) + { "I was always good at math and science, and I never realized " + "that was unusual or somehow undesirable. So one of the things " + "I care a lot about is helping to remove that stigma, " + "to show girls that you can be feminine, you can like the things " + "that girls like, but you can also be really good at technology. " + "You can be really good at building things." + " - Marissa Meyer, Newsweek, 2010-12-22" "\n", + + "SSB3YXMgYWx3YXlzIGdvb2QgYXQgbWF0aCBhbmQgc2NpZW5jZSwgYW5kIEkg" + "bmV2ZXIgcmVhbGl6ZWQgdGhhdCB3YXMgdW51c3VhbCBvciBzb21laG93IHVu" + "ZGVzaXJhYmxlLiBTbyBvbmUgb2YgdGhlIHRoaW5ncyBJIGNhcmUgYSBsb3Qg" + "YWJvdXQgaXMgaGVscGluZyB0byByZW1vdmUgdGhhdCBzdGlnbWEsIHRvIHNo" + "b3cgZ2lybHMgdGhhdCB5b3UgY2FuIGJlIGZlbWluaW5lLCB5b3UgY2FuIGxp" + "a2UgdGhlIHRoaW5ncyB0aGF0IGdpcmxzIGxpa2UsIGJ1dCB5b3UgY2FuIGFs" + "c28gYmUgcmVhbGx5IGdvb2QgYXQgdGVjaG5vbG9neS4gWW91IGNhbiBiZSBy" + "ZWFsbHkgZ29vZCBhdCBidWlsZGluZyB0aGluZ3MuIC0gTWFyaXNzYSBNZXll" + "ciwgTmV3c3dlZWssIDIwMTAtMTItMjIK" }, + + { "Typical first year for a new cluster: " + "~0.5 overheating " + "~1 PDU failure " + "~1 rack-move " + "~1 network rewiring " + "~20 rack failures " + "~5 racks go wonky " + "~8 network maintenances " + "~12 router reloads " + "~3 router failures " + "~dozens of minor 30-second blips for dns " + "~1000 individual machine failures " + "~thousands of hard drive failures " + "slow disks, bad memory, misconfigured machines, flaky machines, etc." + " - Jeff Dean, The Joys of Real Hardware" "\n", + + "VHlwaWNhbCBmaXJzdCB5ZWFyIGZvciBhIG5ldyBjbHVzdGVyOiB-MC41IG92" + "ZXJoZWF0aW5nIH4xIFBEVSBmYWlsdXJlIH4xIHJhY2stbW92ZSB-MSBuZXR3" + "b3JrIHJld2lyaW5nIH4yMCByYWNrIGZhaWx1cmVzIH41IHJhY2tzIGdvIHdv" + "bmt5IH44IG5ldHdvcmsgbWFpbnRlbmFuY2VzIH4xMiByb3V0ZXIgcmVsb2Fk" + "cyB-MyByb3V0ZXIgZmFpbHVyZXMgfmRvemVucyBvZiBtaW5vciAzMC1zZWNv" + "bmQgYmxpcHMgZm9yIGRucyB-MTAwMCBpbmRpdmlkdWFsIG1hY2hpbmUgZmFp" + "bHVyZXMgfnRob3VzYW5kcyBvZiBoYXJkIGRyaXZlIGZhaWx1cmVzIHNsb3cg" + "ZGlza3MsIGJhZCBtZW1vcnksIG1pc2NvbmZpZ3VyZWQgbWFjaGluZXMsIGZs" + "YWt5IG1hY2hpbmVzLCBldGMuIC0gSmVmZiBEZWFuLCBUaGUgSm95cyBvZiBS" + "ZWFsIEhhcmR3YXJlCg" }, + + { "I'm the head of the webspam team at Google. " + "That means that if you type your name into Google and get porn back, " + "it's my fault. Unless you're a porn star, in which case porn is a " + "completely reasonable response." + " - Matt Cutts, Google Plus" "\n", + + "SSdtIHRoZSBoZWFkIG9mIHRoZSB3ZWJzcGFtIHRlYW0gYXQgR29vZ2xlLiAg" + "VGhhdCBtZWFucyB0aGF0IGlmIHlvdSB0eXBlIHlvdXIgbmFtZSBpbnRvIEdv" + "b2dsZSBhbmQgZ2V0IHBvcm4gYmFjaywgaXQncyBteSBmYXVsdC4gVW5sZXNz" + "IHlvdSdyZSBhIHBvcm4gc3RhciwgaW4gd2hpY2ggY2FzZSBwb3JuIGlzIGEg" + "Y29tcGxldGVseSByZWFzb25hYmxlIHJlc3BvbnNlLiAtIE1hdHQgQ3V0dHMs" + "IEdvb2dsZSBQbHVzCg" }, + + { "It will still be a long time before machines approach human " + "intelligence. " + "But luckily, machines don't actually have to be intelligent; " + "they just have to fake it. Access to a wealth of information, " + "combined with a rudimentary decision-making capacity, " + "can often be almost as useful. Of course, the results are better yet " + "when coupled with intelligence. A reference librarian with access to " + "a good search engine is a formidable tool." + " - Craig Silverstein, Siemens Pictures of the Future, Spring 2004" + "\n", + + "SXQgd2lsbCBzdGlsbCBiZSBhIGxvbmcgdGltZSBiZWZvcmUgbWFjaGluZXMg" + "YXBwcm9hY2ggaHVtYW4gaW50ZWxsaWdlbmNlLiBCdXQgbHVja2lseSwgbWFj" + "aGluZXMgZG9uJ3QgYWN0dWFsbHkgaGF2ZSB0byBiZSBpbnRlbGxpZ2VudDsg" + "dGhleSBqdXN0IGhhdmUgdG8gZmFrZSBpdC4gQWNjZXNzIHRvIGEgd2VhbHRo" + "IG9mIGluZm9ybWF0aW9uLCBjb21iaW5lZCB3aXRoIGEgcnVkaW1lbnRhcnkg" + "ZGVjaXNpb24tbWFraW5nIGNhcGFjaXR5LCBjYW4gb2Z0ZW4gYmUgYWxtb3N0" + "IGFzIHVzZWZ1bC4gT2YgY291cnNlLCB0aGUgcmVzdWx0cyBhcmUgYmV0dGVy" + "IHlldCB3aGVuIGNvdXBsZWQgd2l0aCBpbnRlbGxpZ2VuY2UuIEEgcmVmZXJl" + "bmNlIGxpYnJhcmlhbiB3aXRoIGFjY2VzcyB0byBhIGdvb2Qgc2VhcmNoIGVu" + "Z2luZSBpcyBhIGZvcm1pZGFibGUgdG9vbC4gLSBDcmFpZyBTaWx2ZXJzdGVp" + "biwgU2llbWVucyBQaWN0dXJlcyBvZiB0aGUgRnV0dXJlLCBTcHJpbmcgMjAw" + "NAo" }, + + // Degenerate edge case + { "", + "" }, + }}; + + return testcase; +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_ESCAPING_TEST_COMMON_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/memutil.cc b/third_party/abseil_cpp/absl/strings/internal/memutil.cc new file mode 100644 index 000000000000..2519c6881e35 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/memutil.cc @@ -0,0 +1,112 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/memutil.h" + +#include <cstdlib> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +int memcasecmp(const char* s1, const char* s2, size_t len) { + const unsigned char* us1 = reinterpret_cast<const unsigned char*>(s1); + const unsigned char* us2 = reinterpret_cast<const unsigned char*>(s2); + + for (size_t i = 0; i < len; i++) { + const int diff = + int{static_cast<unsigned char>(absl::ascii_tolower(us1[i]))} - + int{static_cast<unsigned char>(absl::ascii_tolower(us2[i]))}; + if (diff != 0) return diff; + } + return 0; +} + +char* memdup(const char* s, size_t slen) { + void* copy; + if ((copy = malloc(slen)) == nullptr) return nullptr; + memcpy(copy, s, slen); + return reinterpret_cast<char*>(copy); +} + +char* memrchr(const char* s, int c, size_t slen) { + for (const char* e = s + slen - 1; e >= s; e--) { + if (*e == c) return const_cast<char*>(e); + } + return nullptr; +} + +size_t memspn(const char* s, size_t slen, const char* accept) { + const char* p = s; + const char* spanp; + char c, sc; + +cont: + c = *p++; + if (slen-- == 0) return p - 1 - s; + for (spanp = accept; (sc = *spanp++) != '\0';) + if (sc == c) goto cont; + return p - 1 - s; +} + +size_t memcspn(const char* s, size_t slen, const char* reject) { + const char* p = s; + const char* spanp; + char c, sc; + + while (slen-- != 0) { + c = *p++; + for (spanp = reject; (sc = *spanp++) != '\0';) + if (sc == c) return p - 1 - s; + } + return p - s; +} + +char* mempbrk(const char* s, size_t slen, const char* accept) { + const char* scanp; + int sc; + + for (; slen; ++s, --slen) { + for (scanp = accept; (sc = *scanp++) != '\0';) + if (sc == *s) return const_cast<char*>(s); + } + return nullptr; +} + +// This is significantly faster for case-sensitive matches with very +// few possible matches. See unit test for benchmarks. +const char* memmatch(const char* phaystack, size_t haylen, const char* pneedle, + size_t neelen) { + if (0 == neelen) { + return phaystack; // even if haylen is 0 + } + if (haylen < neelen) return nullptr; + + const char* match; + const char* hayend = phaystack + haylen - neelen + 1; + // A static cast is used here to work around the fact that memchr returns + // a void* on Posix-compliant systems and const void* on Windows. + while ((match = static_cast<const char*>( + memchr(phaystack, pneedle[0], hayend - phaystack)))) { + if (memcmp(match, pneedle, neelen) == 0) + return match; + else + phaystack = match + 1; + } + return nullptr; +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/memutil.h b/third_party/abseil_cpp/absl/strings/internal/memutil.h new file mode 100644 index 000000000000..9ad05358086c --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/memutil.h @@ -0,0 +1,148 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// These routines provide mem versions of standard C string routines, +// such as strpbrk. They function exactly the same as the str versions, +// so if you wonder what they are, replace the word "mem" by +// "str" and check out the man page. I could return void*, as the +// strutil.h mem*() routines tend to do, but I return char* instead +// since this is by far the most common way these functions are called. +// +// The difference between the mem and str versions is the mem version +// takes a pointer and a length, rather than a '\0'-terminated string. +// The memcase* routines defined here assume the locale is "C" +// (they use absl::ascii_tolower instead of tolower). +// +// These routines are based on the BSD library. +// +// Here's a list of routines from string.h, and their mem analogues. +// Functions in lowercase are defined in string.h; those in UPPERCASE +// are defined here: +// +// strlen -- +// strcat strncat MEMCAT +// strcpy strncpy memcpy +// -- memccpy (very cool function, btw) +// -- memmove +// -- memset +// strcmp strncmp memcmp +// strcasecmp strncasecmp MEMCASECMP +// strchr memchr +// strcoll -- +// strxfrm -- +// strdup strndup MEMDUP +// strrchr MEMRCHR +// strspn MEMSPN +// strcspn MEMCSPN +// strpbrk MEMPBRK +// strstr MEMSTR MEMMEM +// (g)strcasestr MEMCASESTR MEMCASEMEM +// strtok -- +// strprefix MEMPREFIX (strprefix is from strutil.h) +// strcaseprefix MEMCASEPREFIX (strcaseprefix is from strutil.h) +// strsuffix MEMSUFFIX (strsuffix is from strutil.h) +// strcasesuffix MEMCASESUFFIX (strcasesuffix is from strutil.h) +// -- MEMIS +// -- MEMCASEIS +// strcount MEMCOUNT (strcount is from strutil.h) + +#ifndef ABSL_STRINGS_INTERNAL_MEMUTIL_H_ +#define ABSL_STRINGS_INTERNAL_MEMUTIL_H_ + +#include <cstddef> +#include <cstring> + +#include "absl/base/port.h" // disable some warnings on Windows +#include "absl/strings/ascii.h" // for absl::ascii_tolower + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +inline char* memcat(char* dest, size_t destlen, const char* src, + size_t srclen) { + return reinterpret_cast<char*>(memcpy(dest + destlen, src, srclen)); +} + +int memcasecmp(const char* s1, const char* s2, size_t len); +char* memdup(const char* s, size_t slen); +char* memrchr(const char* s, int c, size_t slen); +size_t memspn(const char* s, size_t slen, const char* accept); +size_t memcspn(const char* s, size_t slen, const char* reject); +char* mempbrk(const char* s, size_t slen, const char* accept); + +// This is for internal use only. Don't call this directly +template <bool case_sensitive> +const char* int_memmatch(const char* haystack, size_t haylen, + const char* needle, size_t neelen) { + if (0 == neelen) { + return haystack; // even if haylen is 0 + } + const char* hayend = haystack + haylen; + const char* needlestart = needle; + const char* needleend = needlestart + neelen; + + for (; haystack < hayend; ++haystack) { + char hay = case_sensitive + ? *haystack + : absl::ascii_tolower(static_cast<unsigned char>(*haystack)); + char nee = case_sensitive + ? *needle + : absl::ascii_tolower(static_cast<unsigned char>(*needle)); + if (hay == nee) { + if (++needle == needleend) { + return haystack + 1 - neelen; + } + } else if (needle != needlestart) { + // must back up haystack in case a prefix matched (find "aab" in "aaab") + haystack -= needle - needlestart; // for loop will advance one more + needle = needlestart; + } + } + return nullptr; +} + +// These are the guys you can call directly +inline const char* memstr(const char* phaystack, size_t haylen, + const char* pneedle) { + return int_memmatch<true>(phaystack, haylen, pneedle, strlen(pneedle)); +} + +inline const char* memcasestr(const char* phaystack, size_t haylen, + const char* pneedle) { + return int_memmatch<false>(phaystack, haylen, pneedle, strlen(pneedle)); +} + +inline const char* memmem(const char* phaystack, size_t haylen, + const char* pneedle, size_t needlelen) { + return int_memmatch<true>(phaystack, haylen, pneedle, needlelen); +} + +inline const char* memcasemem(const char* phaystack, size_t haylen, + const char* pneedle, size_t needlelen) { + return int_memmatch<false>(phaystack, haylen, pneedle, needlelen); +} + +// This is significantly faster for case-sensitive matches with very +// few possible matches. See unit test for benchmarks. +const char* memmatch(const char* phaystack, size_t haylen, const char* pneedle, + size_t neelen); + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_MEMUTIL_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/memutil_benchmark.cc b/third_party/abseil_cpp/absl/strings/internal/memutil_benchmark.cc new file mode 100644 index 000000000000..dc95c3e5e55a --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/memutil_benchmark.cc @@ -0,0 +1,323 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/memutil.h" + +#include <algorithm> +#include <cstdlib> + +#include "benchmark/benchmark.h" +#include "absl/strings/ascii.h" + +// We fill the haystack with aaaaaaaaaaaaaaaaaa...aaaab. +// That gives us: +// - an easy search: 'b' +// - a medium search: 'ab'. That means every letter is a possible match. +// - a pathological search: 'aaaaaa.......aaaaab' (half as many a's as haytack) +// We benchmark case-sensitive and case-insensitive versions of +// three memmem implementations: +// - memmem() from memutil.h +// - search() from STL +// - memmatch(), a custom implementation using memchr and memcmp. +// Here are sample results: +// +// Run on (12 X 3800 MHz CPU s) +// CPU Caches: +// L1 Data 32K (x6) +// L1 Instruction 32K (x6) +// L2 Unified 256K (x6) +// L3 Unified 15360K (x1) +// ---------------------------------------------------------------- +// Benchmark Time CPU Iterations +// ---------------------------------------------------------------- +// BM_Memmem 3583 ns 3582 ns 196469 2.59966GB/s +// BM_MemmemMedium 13743 ns 13742 ns 50901 693.986MB/s +// BM_MemmemPathological 13695030 ns 13693977 ns 51 713.133kB/s +// BM_Memcasemem 3299 ns 3299 ns 212942 2.82309GB/s +// BM_MemcasememMedium 16407 ns 16406 ns 42170 581.309MB/s +// BM_MemcasememPathological 17267745 ns 17266030 ns 41 565.598kB/s +// BM_Search 1610 ns 1609 ns 431321 5.78672GB/s +// BM_SearchMedium 11111 ns 11110 ns 63001 858.414MB/s +// BM_SearchPathological 12117390 ns 12116397 ns 58 805.984kB/s +// BM_Searchcase 3081 ns 3081 ns 229949 3.02313GB/s +// BM_SearchcaseMedium 16003 ns 16001 ns 44170 595.998MB/s +// BM_SearchcasePathological 15823413 ns 15821909 ns 44 617.222kB/s +// BM_Memmatch 197 ns 197 ns 3584225 47.2951GB/s +// BM_MemmatchMedium 52333 ns 52329 ns 13280 182.244MB/s +// BM_MemmatchPathological 659799 ns 659727 ns 1058 14.4556MB/s +// BM_Memcasematch 5460 ns 5460 ns 127606 1.70586GB/s +// BM_MemcasematchMedium 32861 ns 32857 ns 21258 290.248MB/s +// BM_MemcasematchPathological 15154243 ns 15153089 ns 46 644.464kB/s +// BM_MemmemStartup 5 ns 5 ns 150821500 +// BM_SearchStartup 5 ns 5 ns 150644203 +// BM_MemmatchStartup 7 ns 7 ns 97068802 +// +// Conclusions: +// +// The following recommendations are based on the sample results above. However, +// we have found that the performance of STL search can vary significantly +// depending on compiler and standard library implementation. We recommend you +// run the benchmarks for yourself on relevant platforms. +// +// If you need case-insensitive, STL search is slightly better than memmem for +// all cases. +// +// Case-sensitive is more subtle: +// Custom memmatch is _very_ fast at scanning, so if you have very few possible +// matches in your haystack, that's the way to go. Performance drops +// significantly with more matches. +// +// STL search is slightly faster than memmem in the medium and pathological +// benchmarks. However, the performance of memmem is currently more dependable +// across platforms and build configurations. + +namespace { + +constexpr int kHaystackSize = 10000; +constexpr int64_t kHaystackSize64 = kHaystackSize; +const char* MakeHaystack() { + char* haystack = new char[kHaystackSize]; + for (int i = 0; i < kHaystackSize - 1; ++i) haystack[i] = 'a'; + haystack[kHaystackSize - 1] = 'b'; + return haystack; +} +const char* const kHaystack = MakeHaystack(); + +void BM_Memmem(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::strings_internal::memmem(kHaystack, kHaystackSize, "b", 1)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_Memmem); + +void BM_MemmemMedium(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::strings_internal::memmem(kHaystack, kHaystackSize, "ab", 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemmemMedium); + +void BM_MemmemPathological(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(absl::strings_internal::memmem( + kHaystack, kHaystackSize, kHaystack + kHaystackSize / 2, + kHaystackSize - kHaystackSize / 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemmemPathological); + +void BM_Memcasemem(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::strings_internal::memcasemem(kHaystack, kHaystackSize, "b", 1)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_Memcasemem); + +void BM_MemcasememMedium(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::strings_internal::memcasemem(kHaystack, kHaystackSize, "ab", 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemcasememMedium); + +void BM_MemcasememPathological(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(absl::strings_internal::memcasemem( + kHaystack, kHaystackSize, kHaystack + kHaystackSize / 2, + kHaystackSize - kHaystackSize / 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemcasememPathological); + +bool case_eq(const char a, const char b) { + return absl::ascii_tolower(a) == absl::ascii_tolower(b); +} + +void BM_Search(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(std::search(kHaystack, kHaystack + kHaystackSize, + kHaystack + kHaystackSize - 1, + kHaystack + kHaystackSize)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_Search); + +void BM_SearchMedium(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(std::search(kHaystack, kHaystack + kHaystackSize, + kHaystack + kHaystackSize - 2, + kHaystack + kHaystackSize)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_SearchMedium); + +void BM_SearchPathological(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(std::search(kHaystack, kHaystack + kHaystackSize, + kHaystack + kHaystackSize / 2, + kHaystack + kHaystackSize)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_SearchPathological); + +void BM_Searchcase(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(std::search(kHaystack, kHaystack + kHaystackSize, + kHaystack + kHaystackSize - 1, + kHaystack + kHaystackSize, case_eq)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_Searchcase); + +void BM_SearchcaseMedium(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(std::search(kHaystack, kHaystack + kHaystackSize, + kHaystack + kHaystackSize - 2, + kHaystack + kHaystackSize, case_eq)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_SearchcaseMedium); + +void BM_SearchcasePathological(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(std::search(kHaystack, kHaystack + kHaystackSize, + kHaystack + kHaystackSize / 2, + kHaystack + kHaystackSize, case_eq)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_SearchcasePathological); + +char* memcasechr(const char* s, int c, size_t slen) { + c = absl::ascii_tolower(c); + for (; slen; ++s, --slen) { + if (absl::ascii_tolower(*s) == c) return const_cast<char*>(s); + } + return nullptr; +} + +const char* memcasematch(const char* phaystack, size_t haylen, + const char* pneedle, size_t neelen) { + if (0 == neelen) { + return phaystack; // even if haylen is 0 + } + if (haylen < neelen) return nullptr; + + const char* match; + const char* hayend = phaystack + haylen - neelen + 1; + while ((match = static_cast<char*>( + memcasechr(phaystack, pneedle[0], hayend - phaystack)))) { + if (absl::strings_internal::memcasecmp(match, pneedle, neelen) == 0) + return match; + else + phaystack = match + 1; + } + return nullptr; +} + +void BM_Memmatch(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::strings_internal::memmatch(kHaystack, kHaystackSize, "b", 1)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_Memmatch); + +void BM_MemmatchMedium(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::strings_internal::memmatch(kHaystack, kHaystackSize, "ab", 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemmatchMedium); + +void BM_MemmatchPathological(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(absl::strings_internal::memmatch( + kHaystack, kHaystackSize, kHaystack + kHaystackSize / 2, + kHaystackSize - kHaystackSize / 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemmatchPathological); + +void BM_Memcasematch(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(memcasematch(kHaystack, kHaystackSize, "b", 1)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_Memcasematch); + +void BM_MemcasematchMedium(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(memcasematch(kHaystack, kHaystackSize, "ab", 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemcasematchMedium); + +void BM_MemcasematchPathological(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(memcasematch(kHaystack, kHaystackSize, + kHaystack + kHaystackSize / 2, + kHaystackSize - kHaystackSize / 2)); + } + state.SetBytesProcessed(kHaystackSize64 * state.iterations()); +} +BENCHMARK(BM_MemcasematchPathological); + +void BM_MemmemStartup(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(absl::strings_internal::memmem( + kHaystack + kHaystackSize - 10, 10, kHaystack + kHaystackSize - 1, 1)); + } +} +BENCHMARK(BM_MemmemStartup); + +void BM_SearchStartup(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize( + std::search(kHaystack + kHaystackSize - 10, kHaystack + kHaystackSize, + kHaystack + kHaystackSize - 1, kHaystack + kHaystackSize)); + } +} +BENCHMARK(BM_SearchStartup); + +void BM_MemmatchStartup(benchmark::State& state) { + for (auto _ : state) { + benchmark::DoNotOptimize(absl::strings_internal::memmatch( + kHaystack + kHaystackSize - 10, 10, kHaystack + kHaystackSize - 1, 1)); + } +} +BENCHMARK(BM_MemmatchStartup); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/memutil_test.cc b/third_party/abseil_cpp/absl/strings/internal/memutil_test.cc new file mode 100644 index 000000000000..d8681ddf4e3b --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/memutil_test.cc @@ -0,0 +1,179 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit test for memutil.cc + +#include "absl/strings/internal/memutil.h" + +#include <cstdlib> + +#include "gtest/gtest.h" +#include "absl/strings/ascii.h" + +namespace { + +static char* memcasechr(const char* s, int c, size_t slen) { + c = absl::ascii_tolower(c); + for (; slen; ++s, --slen) { + if (absl::ascii_tolower(*s) == c) return const_cast<char*>(s); + } + return nullptr; +} + +static const char* memcasematch(const char* phaystack, size_t haylen, + const char* pneedle, size_t neelen) { + if (0 == neelen) { + return phaystack; // even if haylen is 0 + } + if (haylen < neelen) return nullptr; + + const char* match; + const char* hayend = phaystack + haylen - neelen + 1; + while ((match = static_cast<char*>( + memcasechr(phaystack, pneedle[0], hayend - phaystack)))) { + if (absl::strings_internal::memcasecmp(match, pneedle, neelen) == 0) + return match; + else + phaystack = match + 1; + } + return nullptr; +} + +TEST(MemUtilTest, AllTests) { + // check memutil functions + char a[1000]; + absl::strings_internal::memcat(a, 0, "hello", sizeof("hello") - 1); + absl::strings_internal::memcat(a, 5, " there", sizeof(" there") - 1); + + EXPECT_EQ(absl::strings_internal::memcasecmp(a, "heLLO there", + sizeof("hello there") - 1), + 0); + EXPECT_EQ(absl::strings_internal::memcasecmp(a, "heLLO therf", + sizeof("hello there") - 1), + -1); + EXPECT_EQ(absl::strings_internal::memcasecmp(a, "heLLO therf", + sizeof("hello there") - 2), + 0); + EXPECT_EQ(absl::strings_internal::memcasecmp(a, "whatever", 0), 0); + + char* p = absl::strings_internal::memdup("hello", 5); + free(p); + + p = absl::strings_internal::memrchr("hello there", 'e', + sizeof("hello there") - 1); + EXPECT_TRUE(p && p[-1] == 'r'); + p = absl::strings_internal::memrchr("hello there", 'e', + sizeof("hello there") - 2); + EXPECT_TRUE(p && p[-1] == 'h'); + p = absl::strings_internal::memrchr("hello there", 'u', + sizeof("hello there") - 1); + EXPECT_TRUE(p == nullptr); + + int len = absl::strings_internal::memspn("hello there", + sizeof("hello there") - 1, "hole"); + EXPECT_EQ(len, sizeof("hello") - 1); + len = absl::strings_internal::memspn("hello there", sizeof("hello there") - 1, + "u"); + EXPECT_EQ(len, 0); + len = absl::strings_internal::memspn("hello there", sizeof("hello there") - 1, + ""); + EXPECT_EQ(len, 0); + len = absl::strings_internal::memspn("hello there", sizeof("hello there") - 1, + "trole h"); + EXPECT_EQ(len, sizeof("hello there") - 1); + len = absl::strings_internal::memspn("hello there!", + sizeof("hello there!") - 1, "trole h"); + EXPECT_EQ(len, sizeof("hello there") - 1); + len = absl::strings_internal::memspn("hello there!", + sizeof("hello there!") - 2, "trole h!"); + EXPECT_EQ(len, sizeof("hello there!") - 2); + + len = absl::strings_internal::memcspn("hello there", + sizeof("hello there") - 1, "leho"); + EXPECT_EQ(len, 0); + len = absl::strings_internal::memcspn("hello there", + sizeof("hello there") - 1, "u"); + EXPECT_EQ(len, sizeof("hello there") - 1); + len = absl::strings_internal::memcspn("hello there", + sizeof("hello there") - 1, ""); + EXPECT_EQ(len, sizeof("hello there") - 1); + len = absl::strings_internal::memcspn("hello there", + sizeof("hello there") - 1, " "); + EXPECT_EQ(len, 5); + + p = absl::strings_internal::mempbrk("hello there", sizeof("hello there") - 1, + "leho"); + EXPECT_TRUE(p && p[1] == 'e' && p[2] == 'l'); + p = absl::strings_internal::mempbrk("hello there", sizeof("hello there") - 1, + "nu"); + EXPECT_TRUE(p == nullptr); + p = absl::strings_internal::mempbrk("hello there!", + sizeof("hello there!") - 2, "!"); + EXPECT_TRUE(p == nullptr); + p = absl::strings_internal::mempbrk("hello there", sizeof("hello there") - 1, + " t "); + EXPECT_TRUE(p && p[-1] == 'o' && p[1] == 't'); + + { + const char kHaystack[] = "0123456789"; + EXPECT_EQ(absl::strings_internal::memmem(kHaystack, 0, "", 0), kHaystack); + EXPECT_EQ(absl::strings_internal::memmem(kHaystack, 10, "012", 3), + kHaystack); + EXPECT_EQ(absl::strings_internal::memmem(kHaystack, 10, "0xx", 1), + kHaystack); + EXPECT_EQ(absl::strings_internal::memmem(kHaystack, 10, "789", 3), + kHaystack + 7); + EXPECT_EQ(absl::strings_internal::memmem(kHaystack, 10, "9xx", 1), + kHaystack + 9); + EXPECT_TRUE(absl::strings_internal::memmem(kHaystack, 10, "9xx", 3) == + nullptr); + EXPECT_TRUE(absl::strings_internal::memmem(kHaystack, 10, "xxx", 1) == + nullptr); + } + { + const char kHaystack[] = "aBcDeFgHiJ"; + EXPECT_EQ(absl::strings_internal::memcasemem(kHaystack, 0, "", 0), + kHaystack); + EXPECT_EQ(absl::strings_internal::memcasemem(kHaystack, 10, "Abc", 3), + kHaystack); + EXPECT_EQ(absl::strings_internal::memcasemem(kHaystack, 10, "Axx", 1), + kHaystack); + EXPECT_EQ(absl::strings_internal::memcasemem(kHaystack, 10, "hIj", 3), + kHaystack + 7); + EXPECT_EQ(absl::strings_internal::memcasemem(kHaystack, 10, "jxx", 1), + kHaystack + 9); + EXPECT_TRUE(absl::strings_internal::memcasemem(kHaystack, 10, "jxx", 3) == + nullptr); + EXPECT_TRUE(absl::strings_internal::memcasemem(kHaystack, 10, "xxx", 1) == + nullptr); + } + { + const char kHaystack[] = "0123456789"; + EXPECT_EQ(absl::strings_internal::memmatch(kHaystack, 0, "", 0), kHaystack); + EXPECT_EQ(absl::strings_internal::memmatch(kHaystack, 10, "012", 3), + kHaystack); + EXPECT_EQ(absl::strings_internal::memmatch(kHaystack, 10, "0xx", 1), + kHaystack); + EXPECT_EQ(absl::strings_internal::memmatch(kHaystack, 10, "789", 3), + kHaystack + 7); + EXPECT_EQ(absl::strings_internal::memmatch(kHaystack, 10, "9xx", 1), + kHaystack + 9); + EXPECT_TRUE(absl::strings_internal::memmatch(kHaystack, 10, "9xx", 3) == + nullptr); + EXPECT_TRUE(absl::strings_internal::memmatch(kHaystack, 10, "xxx", 1) == + nullptr); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/numbers_test_common.h b/third_party/abseil_cpp/absl/strings/internal/numbers_test_common.h new file mode 100644 index 000000000000..eaa88a88975b --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/numbers_test_common.h @@ -0,0 +1,184 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This file contains common things needed by numbers_test.cc, +// numbers_legacy_test.cc and numbers_benchmark.cc. + +#ifndef ABSL_STRINGS_INTERNAL_NUMBERS_TEST_COMMON_H_ +#define ABSL_STRINGS_INTERNAL_NUMBERS_TEST_COMMON_H_ + +#include <array> +#include <cstdint> +#include <limits> +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +template <typename IntType> +inline bool Itoa(IntType value, int base, std::string* destination) { + destination->clear(); + if (base <= 1 || base > 36) { + return false; + } + + if (value == 0) { + destination->push_back('0'); + return true; + } + + bool negative = value < 0; + while (value != 0) { + const IntType next_value = value / base; + // Can't use std::abs here because of problems when IntType is unsigned. + int remainder = + static_cast<int>(value > next_value * base ? value - next_value * base + : next_value * base - value); + char c = remainder < 10 ? '0' + remainder : 'A' + remainder - 10; + destination->insert(0, 1, c); + value = next_value; + } + + if (negative) { + destination->insert(0, 1, '-'); + } + return true; +} + +struct uint32_test_case { + const char* str; + bool expect_ok; + int base; // base to pass to the conversion function + uint32_t expected; +}; + +inline const std::array<uint32_test_case, 27>& strtouint32_test_cases() { + static const std::array<uint32_test_case, 27> test_cases{{ + {"0xffffffff", true, 16, (std::numeric_limits<uint32_t>::max)()}, + {"0x34234324", true, 16, 0x34234324}, + {"34234324", true, 16, 0x34234324}, + {"0", true, 16, 0}, + {" \t\n 0xffffffff", true, 16, (std::numeric_limits<uint32_t>::max)()}, + {" \f\v 46", true, 10, 46}, // must accept weird whitespace + {" \t\n 72717222", true, 8, 072717222}, + {" \t\n 072717222", true, 8, 072717222}, + {" \t\n 072717228", false, 8, 07271722}, + {"0", true, 0, 0}, + + // Base-10 version. + {"34234324", true, 0, 34234324}, + {"4294967295", true, 0, (std::numeric_limits<uint32_t>::max)()}, + {"34234324 \n\t", true, 10, 34234324}, + + // Unusual base + {"0", true, 3, 0}, + {"2", true, 3, 2}, + {"11", true, 3, 4}, + + // Invalid uints. + {"", false, 0, 0}, + {" ", false, 0, 0}, + {"abc", false, 0, 0}, // would be valid hex, but prefix is missing + {"34234324a", false, 0, 34234324}, + {"34234.3", false, 0, 34234}, + {"-1", false, 0, 0}, + {" -123", false, 0, 0}, + {" \t\n -123", false, 0, 0}, + + // Out of bounds. + {"4294967296", false, 0, (std::numeric_limits<uint32_t>::max)()}, + {"0x100000000", false, 0, (std::numeric_limits<uint32_t>::max)()}, + {nullptr, false, 0, 0}, + }}; + return test_cases; +} + +struct uint64_test_case { + const char* str; + bool expect_ok; + int base; + uint64_t expected; +}; + +inline const std::array<uint64_test_case, 34>& strtouint64_test_cases() { + static const std::array<uint64_test_case, 34> test_cases{{ + {"0x3423432448783446", true, 16, int64_t{0x3423432448783446}}, + {"3423432448783446", true, 16, int64_t{0x3423432448783446}}, + + {"0", true, 16, 0}, + {"000", true, 0, 0}, + {"0", true, 0, 0}, + {" \t\n 0xffffffffffffffff", true, 16, + (std::numeric_limits<uint64_t>::max)()}, + + {"012345670123456701234", true, 8, int64_t{012345670123456701234}}, + {"12345670123456701234", true, 8, int64_t{012345670123456701234}}, + + {"12845670123456701234", false, 8, 0}, + + // Base-10 version. + {"34234324487834466", true, 0, int64_t{34234324487834466}}, + + {" \t\n 18446744073709551615", true, 0, + (std::numeric_limits<uint64_t>::max)()}, + + {"34234324487834466 \n\t ", true, 0, int64_t{34234324487834466}}, + + {" \f\v 46", true, 10, 46}, // must accept weird whitespace + + // Unusual base + {"0", true, 3, 0}, + {"2", true, 3, 2}, + {"11", true, 3, 4}, + + {"0", true, 0, 0}, + + // Invalid uints. + {"", false, 0, 0}, + {" ", false, 0, 0}, + {"abc", false, 0, 0}, + {"34234324487834466a", false, 0, 0}, + {"34234487834466.3", false, 0, 0}, + {"-1", false, 0, 0}, + {" -123", false, 0, 0}, + {" \t\n -123", false, 0, 0}, + + // Out of bounds. + {"18446744073709551616", false, 10, 0}, + {"18446744073709551616", false, 0, 0}, + {"0x10000000000000000", false, 16, + (std::numeric_limits<uint64_t>::max)()}, + {"0X10000000000000000", false, 16, + (std::numeric_limits<uint64_t>::max)()}, // 0X versus 0x. + {"0x10000000000000000", false, 0, (std::numeric_limits<uint64_t>::max)()}, + {"0X10000000000000000", false, 0, + (std::numeric_limits<uint64_t>::max)()}, // 0X versus 0x. + + {"0x1234", true, 16, 0x1234}, + + // Base-10 string version. + {"1234", true, 0, 1234}, + {nullptr, false, 0, 0}, + }}; + return test_cases; +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_NUMBERS_TEST_COMMON_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/ostringstream.cc b/third_party/abseil_cpp/absl/strings/internal/ostringstream.cc new file mode 100644 index 000000000000..05324c780c60 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/ostringstream.cc @@ -0,0 +1,36 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/ostringstream.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +OStringStream::Buf::int_type OStringStream::overflow(int c) { + assert(s_); + if (!Buf::traits_type::eq_int_type(c, Buf::traits_type::eof())) + s_->push_back(static_cast<char>(c)); + return 1; +} + +std::streamsize OStringStream::xsputn(const char* s, std::streamsize n) { + assert(s_); + s_->append(s, n); + return n; +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/ostringstream.h b/third_party/abseil_cpp/absl/strings/internal/ostringstream.h new file mode 100644 index 000000000000..d25d60473f6d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/ostringstream.h @@ -0,0 +1,89 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_OSTRINGSTREAM_H_ +#define ABSL_STRINGS_INTERNAL_OSTRINGSTREAM_H_ + +#include <cassert> +#include <ostream> +#include <streambuf> +#include <string> + +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// The same as std::ostringstream but appends to a user-specified std::string, +// and is faster. It is ~70% faster to create, ~50% faster to write to, and +// completely free to extract the result std::string. +// +// std::string s; +// OStringStream strm(&s); +// strm << 42 << ' ' << 3.14; // appends to `s` +// +// The stream object doesn't have to be named. Starting from C++11 operator<< +// works with rvalues of std::ostream. +// +// std::string s; +// OStringStream(&s) << 42 << ' ' << 3.14; // appends to `s` +// +// OStringStream is faster to create than std::ostringstream but it's still +// relatively slow. Avoid creating multiple streams where a single stream will +// do. +// +// Creates unnecessary instances of OStringStream: slow. +// +// std::string s; +// OStringStream(&s) << 42; +// OStringStream(&s) << ' '; +// OStringStream(&s) << 3.14; +// +// Creates a single instance of OStringStream and reuses it: fast. +// +// std::string s; +// OStringStream strm(&s); +// strm << 42; +// strm << ' '; +// strm << 3.14; +// +// Note: flush() has no effect. No reason to call it. +class OStringStream : private std::basic_streambuf<char>, public std::ostream { + public: + // The argument can be null, in which case you'll need to call str(p) with a + // non-null argument before you can write to the stream. + // + // The destructor of OStringStream doesn't use the std::string. It's OK to + // destroy the std::string before the stream. + explicit OStringStream(std::string* s) : std::ostream(this), s_(s) {} + + std::string* str() { return s_; } + const std::string* str() const { return s_; } + void str(std::string* s) { s_ = s; } + + private: + using Buf = std::basic_streambuf<char>; + + Buf::int_type overflow(int c) override; + std::streamsize xsputn(const char* s, std::streamsize n) override; + + std::string* s_; +}; + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_OSTRINGSTREAM_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/ostringstream_benchmark.cc b/third_party/abseil_cpp/absl/strings/internal/ostringstream_benchmark.cc new file mode 100644 index 000000000000..5979f18236e3 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/ostringstream_benchmark.cc @@ -0,0 +1,106 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/ostringstream.h" + +#include <sstream> +#include <string> + +#include "benchmark/benchmark.h" + +namespace { + +enum StringType { + kNone, + kStdString, +}; + +// Benchmarks for std::ostringstream. +template <StringType kOutput> +void BM_StdStream(benchmark::State& state) { + const int num_writes = state.range(0); + const int bytes_per_write = state.range(1); + const std::string payload(bytes_per_write, 'x'); + for (auto _ : state) { + std::ostringstream strm; + benchmark::DoNotOptimize(strm); + for (int i = 0; i != num_writes; ++i) { + strm << payload; + } + switch (kOutput) { + case kNone: { + break; + } + case kStdString: { + std::string s = strm.str(); + benchmark::DoNotOptimize(s); + break; + } + } + } +} + +// Create the stream, optionally write to it, then destroy it. +BENCHMARK_TEMPLATE(BM_StdStream, kNone) + ->ArgPair(0, 0) + ->ArgPair(1, 16) // 16 bytes is small enough for SSO + ->ArgPair(1, 256) // 256 bytes requires heap allocation + ->ArgPair(1024, 256); +// Create the stream, write to it, get std::string out, then destroy. +BENCHMARK_TEMPLATE(BM_StdStream, kStdString) + ->ArgPair(1, 16) // 16 bytes is small enough for SSO + ->ArgPair(1, 256) // 256 bytes requires heap allocation + ->ArgPair(1024, 256); + +// Benchmarks for OStringStream. +template <StringType kOutput> +void BM_CustomStream(benchmark::State& state) { + const int num_writes = state.range(0); + const int bytes_per_write = state.range(1); + const std::string payload(bytes_per_write, 'x'); + for (auto _ : state) { + std::string out; + absl::strings_internal::OStringStream strm(&out); + benchmark::DoNotOptimize(strm); + for (int i = 0; i != num_writes; ++i) { + strm << payload; + } + switch (kOutput) { + case kNone: { + break; + } + case kStdString: { + std::string s = out; + benchmark::DoNotOptimize(s); + break; + } + } + } +} + +// Create the stream, optionally write to it, then destroy it. +BENCHMARK_TEMPLATE(BM_CustomStream, kNone) + ->ArgPair(0, 0) + ->ArgPair(1, 16) // 16 bytes is small enough for SSO + ->ArgPair(1, 256) // 256 bytes requires heap allocation + ->ArgPair(1024, 256); +// Create the stream, write to it, get std::string out, then destroy. +// It's not useful in practice to extract std::string from OStringStream; we +// measure it for completeness. +BENCHMARK_TEMPLATE(BM_CustomStream, kStdString) + ->ArgPair(1, 16) // 16 bytes is small enough for SSO + ->ArgPair(1, 256) // 256 bytes requires heap allocation + ->ArgPair(1024, 256); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/ostringstream_test.cc b/third_party/abseil_cpp/absl/strings/internal/ostringstream_test.cc new file mode 100644 index 000000000000..2879e50eb38d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/ostringstream_test.cc @@ -0,0 +1,102 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/ostringstream.h" + +#include <memory> +#include <ostream> +#include <string> +#include <type_traits> + +#include "gtest/gtest.h" + +namespace { + +TEST(OStringStream, IsOStream) { + static_assert( + std::is_base_of<std::ostream, absl::strings_internal::OStringStream>(), + ""); +} + +TEST(OStringStream, ConstructDestroy) { + { + absl::strings_internal::OStringStream strm(nullptr); + EXPECT_EQ(nullptr, strm.str()); + } + { + std::string s = "abc"; + { + absl::strings_internal::OStringStream strm(&s); + EXPECT_EQ(&s, strm.str()); + } + EXPECT_EQ("abc", s); + } + { + std::unique_ptr<std::string> s(new std::string); + absl::strings_internal::OStringStream strm(s.get()); + s.reset(); + } +} + +TEST(OStringStream, Str) { + std::string s1; + absl::strings_internal::OStringStream strm(&s1); + const absl::strings_internal::OStringStream& c_strm(strm); + + static_assert(std::is_same<decltype(strm.str()), std::string*>(), ""); + static_assert(std::is_same<decltype(c_strm.str()), const std::string*>(), ""); + + EXPECT_EQ(&s1, strm.str()); + EXPECT_EQ(&s1, c_strm.str()); + + strm.str(&s1); + EXPECT_EQ(&s1, strm.str()); + EXPECT_EQ(&s1, c_strm.str()); + + std::string s2; + strm.str(&s2); + EXPECT_EQ(&s2, strm.str()); + EXPECT_EQ(&s2, c_strm.str()); + + strm.str(nullptr); + EXPECT_EQ(nullptr, strm.str()); + EXPECT_EQ(nullptr, c_strm.str()); +} + +TEST(OStreamStream, WriteToLValue) { + std::string s = "abc"; + { + absl::strings_internal::OStringStream strm(&s); + EXPECT_EQ("abc", s); + strm << ""; + EXPECT_EQ("abc", s); + strm << 42; + EXPECT_EQ("abc42", s); + strm << 'x' << 'y'; + EXPECT_EQ("abc42xy", s); + } + EXPECT_EQ("abc42xy", s); +} + +TEST(OStreamStream, WriteToRValue) { + std::string s = "abc"; + absl::strings_internal::OStringStream(&s) << ""; + EXPECT_EQ("abc", s); + absl::strings_internal::OStringStream(&s) << 42; + EXPECT_EQ("abc42", s); + absl::strings_internal::OStringStream(&s) << 'x' << 'y'; + EXPECT_EQ("abc42xy", s); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/pow10_helper.cc b/third_party/abseil_cpp/absl/strings/internal/pow10_helper.cc new file mode 100644 index 000000000000..42e96c3425d2 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/pow10_helper.cc @@ -0,0 +1,122 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/pow10_helper.h" + +#include <cmath> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +namespace { + +// The exact value of 1e23 falls precisely halfway between two representable +// doubles. Furthermore, the rounding rules we prefer (break ties by rounding +// to the nearest even) dictate in this case that the number should be rounded +// down, but this is not completely specified for floating-point literals in +// C++. (It just says to use the default rounding mode of the standard +// library.) We ensure the result we want by using a number that has an +// unambiguous correctly rounded answer. +constexpr double k1e23 = 9999999999999999e7; + +constexpr double kPowersOfTen[] = { + 0.0, 1e-323, 1e-322, 1e-321, 1e-320, 1e-319, 1e-318, 1e-317, 1e-316, + 1e-315, 1e-314, 1e-313, 1e-312, 1e-311, 1e-310, 1e-309, 1e-308, 1e-307, + 1e-306, 1e-305, 1e-304, 1e-303, 1e-302, 1e-301, 1e-300, 1e-299, 1e-298, + 1e-297, 1e-296, 1e-295, 1e-294, 1e-293, 1e-292, 1e-291, 1e-290, 1e-289, + 1e-288, 1e-287, 1e-286, 1e-285, 1e-284, 1e-283, 1e-282, 1e-281, 1e-280, + 1e-279, 1e-278, 1e-277, 1e-276, 1e-275, 1e-274, 1e-273, 1e-272, 1e-271, + 1e-270, 1e-269, 1e-268, 1e-267, 1e-266, 1e-265, 1e-264, 1e-263, 1e-262, + 1e-261, 1e-260, 1e-259, 1e-258, 1e-257, 1e-256, 1e-255, 1e-254, 1e-253, + 1e-252, 1e-251, 1e-250, 1e-249, 1e-248, 1e-247, 1e-246, 1e-245, 1e-244, + 1e-243, 1e-242, 1e-241, 1e-240, 1e-239, 1e-238, 1e-237, 1e-236, 1e-235, + 1e-234, 1e-233, 1e-232, 1e-231, 1e-230, 1e-229, 1e-228, 1e-227, 1e-226, + 1e-225, 1e-224, 1e-223, 1e-222, 1e-221, 1e-220, 1e-219, 1e-218, 1e-217, + 1e-216, 1e-215, 1e-214, 1e-213, 1e-212, 1e-211, 1e-210, 1e-209, 1e-208, + 1e-207, 1e-206, 1e-205, 1e-204, 1e-203, 1e-202, 1e-201, 1e-200, 1e-199, + 1e-198, 1e-197, 1e-196, 1e-195, 1e-194, 1e-193, 1e-192, 1e-191, 1e-190, + 1e-189, 1e-188, 1e-187, 1e-186, 1e-185, 1e-184, 1e-183, 1e-182, 1e-181, + 1e-180, 1e-179, 1e-178, 1e-177, 1e-176, 1e-175, 1e-174, 1e-173, 1e-172, + 1e-171, 1e-170, 1e-169, 1e-168, 1e-167, 1e-166, 1e-165, 1e-164, 1e-163, + 1e-162, 1e-161, 1e-160, 1e-159, 1e-158, 1e-157, 1e-156, 1e-155, 1e-154, + 1e-153, 1e-152, 1e-151, 1e-150, 1e-149, 1e-148, 1e-147, 1e-146, 1e-145, + 1e-144, 1e-143, 1e-142, 1e-141, 1e-140, 1e-139, 1e-138, 1e-137, 1e-136, + 1e-135, 1e-134, 1e-133, 1e-132, 1e-131, 1e-130, 1e-129, 1e-128, 1e-127, + 1e-126, 1e-125, 1e-124, 1e-123, 1e-122, 1e-121, 1e-120, 1e-119, 1e-118, + 1e-117, 1e-116, 1e-115, 1e-114, 1e-113, 1e-112, 1e-111, 1e-110, 1e-109, + 1e-108, 1e-107, 1e-106, 1e-105, 1e-104, 1e-103, 1e-102, 1e-101, 1e-100, + 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92, 1e-91, + 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, 1e-83, 1e-82, + 1e-81, 1e-80, 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, 1e-74, 1e-73, + 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65, 1e-64, + 1e-63, 1e-62, 1e-61, 1e-60, 1e-59, 1e-58, 1e-57, 1e-56, 1e-55, + 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, 1e-46, + 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 1e-39, 1e-38, 1e-37, + 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, 1e-28, + 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, 1e-19, + 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, + 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, + 1e+0, 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, + 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, + 1e+18, 1e+19, 1e+20, 1e+21, 1e+22, k1e23, 1e+24, 1e+25, 1e+26, + 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, + 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 1e+41, 1e+42, 1e+43, 1e+44, + 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, + 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 1e+61, 1e+62, + 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, + 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, + 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, + 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, + 1e+99, 1e+100, 1e+101, 1e+102, 1e+103, 1e+104, 1e+105, 1e+106, 1e+107, + 1e+108, 1e+109, 1e+110, 1e+111, 1e+112, 1e+113, 1e+114, 1e+115, 1e+116, + 1e+117, 1e+118, 1e+119, 1e+120, 1e+121, 1e+122, 1e+123, 1e+124, 1e+125, + 1e+126, 1e+127, 1e+128, 1e+129, 1e+130, 1e+131, 1e+132, 1e+133, 1e+134, + 1e+135, 1e+136, 1e+137, 1e+138, 1e+139, 1e+140, 1e+141, 1e+142, 1e+143, + 1e+144, 1e+145, 1e+146, 1e+147, 1e+148, 1e+149, 1e+150, 1e+151, 1e+152, + 1e+153, 1e+154, 1e+155, 1e+156, 1e+157, 1e+158, 1e+159, 1e+160, 1e+161, + 1e+162, 1e+163, 1e+164, 1e+165, 1e+166, 1e+167, 1e+168, 1e+169, 1e+170, + 1e+171, 1e+172, 1e+173, 1e+174, 1e+175, 1e+176, 1e+177, 1e+178, 1e+179, + 1e+180, 1e+181, 1e+182, 1e+183, 1e+184, 1e+185, 1e+186, 1e+187, 1e+188, + 1e+189, 1e+190, 1e+191, 1e+192, 1e+193, 1e+194, 1e+195, 1e+196, 1e+197, + 1e+198, 1e+199, 1e+200, 1e+201, 1e+202, 1e+203, 1e+204, 1e+205, 1e+206, + 1e+207, 1e+208, 1e+209, 1e+210, 1e+211, 1e+212, 1e+213, 1e+214, 1e+215, + 1e+216, 1e+217, 1e+218, 1e+219, 1e+220, 1e+221, 1e+222, 1e+223, 1e+224, + 1e+225, 1e+226, 1e+227, 1e+228, 1e+229, 1e+230, 1e+231, 1e+232, 1e+233, + 1e+234, 1e+235, 1e+236, 1e+237, 1e+238, 1e+239, 1e+240, 1e+241, 1e+242, + 1e+243, 1e+244, 1e+245, 1e+246, 1e+247, 1e+248, 1e+249, 1e+250, 1e+251, + 1e+252, 1e+253, 1e+254, 1e+255, 1e+256, 1e+257, 1e+258, 1e+259, 1e+260, + 1e+261, 1e+262, 1e+263, 1e+264, 1e+265, 1e+266, 1e+267, 1e+268, 1e+269, + 1e+270, 1e+271, 1e+272, 1e+273, 1e+274, 1e+275, 1e+276, 1e+277, 1e+278, + 1e+279, 1e+280, 1e+281, 1e+282, 1e+283, 1e+284, 1e+285, 1e+286, 1e+287, + 1e+288, 1e+289, 1e+290, 1e+291, 1e+292, 1e+293, 1e+294, 1e+295, 1e+296, + 1e+297, 1e+298, 1e+299, 1e+300, 1e+301, 1e+302, 1e+303, 1e+304, 1e+305, + 1e+306, 1e+307, 1e+308, +}; + +} // namespace + +double Pow10(int exp) { + if (exp < -324) { + return 0.0; + } else if (exp > 308) { + return INFINITY; + } else { + return kPowersOfTen[exp + 324]; + } +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/pow10_helper.h b/third_party/abseil_cpp/absl/strings/internal/pow10_helper.h new file mode 100644 index 000000000000..c37c2c3ffee5 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/pow10_helper.h @@ -0,0 +1,40 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This test helper library contains a table of powers of 10, to guarantee +// precise values are computed across the full range of doubles. We can't rely +// on the pow() function, because not all standard libraries ship a version +// that is precise. +#ifndef ABSL_STRINGS_INTERNAL_POW10_HELPER_H_ +#define ABSL_STRINGS_INTERNAL_POW10_HELPER_H_ + +#include <vector> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// Computes the precise value of 10^exp. (I.e. the nearest representable +// double to the exact value, rounding to nearest-even in the (single) case of +// being exactly halfway between.) +double Pow10(int exp); + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_POW10_HELPER_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/pow10_helper_test.cc b/third_party/abseil_cpp/absl/strings/internal/pow10_helper_test.cc new file mode 100644 index 000000000000..a4ff76d31e10 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/pow10_helper_test.cc @@ -0,0 +1,122 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/pow10_helper.h" + +#include <cmath> + +#include "gtest/gtest.h" +#include "absl/strings/str_format.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +namespace { + +struct TestCase { + int power; // Testing Pow10(power) + uint64_t significand; // Raw bits of the expected value + int radix; // significand is adjusted by 2^radix +}; + +TEST(Pow10HelperTest, Works) { + // The logic in pow10_helper.cc is so simple that theoretically we don't even + // need a test. However, we're paranoid and believe that there may be + // compilers that don't round floating-point literals correctly, even though + // it is specified by the standard. We check various edge cases, just to be + // sure. + constexpr TestCase kTestCases[] = { + // Subnormals + {-323, 0x2, -1074}, + {-322, 0x14, -1074}, + {-321, 0xca, -1074}, + {-320, 0x7e8, -1074}, + {-319, 0x4f10, -1074}, + {-318, 0x316a2, -1074}, + {-317, 0x1ee257, -1074}, + {-316, 0x134d761, -1074}, + {-315, 0xc1069cd, -1074}, + {-314, 0x78a42205, -1074}, + {-313, 0x4b6695433, -1074}, + {-312, 0x2f201d49fb, -1074}, + {-311, 0x1d74124e3d1, -1074}, + {-310, 0x12688b70e62b, -1074}, + {-309, 0xb8157268fdaf, -1074}, + {-308, 0x730d67819e8d2, -1074}, + // Values that are very close to rounding the other way. + // Comment shows difference of significand from the true value. + {-307, 0x11fa182c40c60d, -1072}, // -.4588 + {-290, 0x18f2b061aea072, -1016}, // .4854 + {-276, 0x11BA03F5B21000, -969}, // .4709 + {-259, 0x1899C2F6732210, -913}, // .4830 + {-252, 0x1D53844EE47DD1, -890}, // -.4743 + {-227, 0x1E5297287C2F45, -807}, // -.4708 + {-198, 0x1322E220A5B17E, -710}, // -.4714 + {-195, 0x12B010D3E1CF56, -700}, // .4928 + {-192, 0x123FF06EEA847A, -690}, // .4968 + {-163, 0x1708D0F84D3DE7, -594}, // -.4977 + {-145, 0x13FAAC3E3FA1F3, -534}, // -.4785 + {-111, 0x133D4032C2C7F5, -421}, // .4774 + {-106, 0x1D5B561574765B, -405}, // -.4869 + {-104, 0x16EF5B40C2FC77, -398}, // -.4741 + {-88, 0x197683DF2F268D, -345}, // -.4738 + {-86, 0x13E497065CD61F, -338}, // .4736 + {-76, 0x17288E1271F513, -305}, // -.4761 + {-63, 0x1A53FC9631D10D, -262}, // .4929 + {-30, 0x14484BFEEBC2A0, -152}, // .4758 + {-21, 0x12E3B40A0E9B4F, -122}, // -.4916 + {-5, 0x14F8B588E368F1, -69}, // .4829 + {23, 0x152D02C7E14AF6, 24}, // -.5000 (exactly, round-to-even) + {29, 0x1431E0FAE6D721, 44}, // -.4870 + {34, 0x1ED09BEAD87C03, 60}, // -.4721 + {70, 0x172EBAD6DDC73D, 180}, // .4733 + {105, 0x1BE7ABD3781ECA, 296}, // -.4850 + {126, 0x17A2ECC414A03F, 366}, // -.4999 + {130, 0x1CDA62055B2D9E, 379}, // .4855 + {165, 0x115D847AD00087, 496}, // -.4913 + {172, 0x14B378469B6732, 519}, // .4818 + {187, 0x1262DFEEBBB0F9, 569}, // -.4805 + {210, 0x18557F31326BBB, 645}, // -.4992 + {212, 0x1302CB5E6F642A, 652}, // -.4838 + {215, 0x1290BA9A38C7D1, 662}, // -.4881 + {236, 0x1F736F9B3494E9, 731}, // .4707 + {244, 0x176EC98994F489, 758}, // .4924 + {250, 0x1658E3AB795204, 778}, // -.4963 + {252, 0x117571DDF6C814, 785}, // .4873 + {254, 0x1B4781EAD1989E, 791}, // -.4887 + {260, 0x1A03FDE214CAF1, 811}, // .4784 + {284, 0x1585041B2C477F, 891}, // .4798 + {304, 0x1D2A1BE4048F90, 957}, // -.4987 + // Out-of-range values + {-324, 0x0, 0}, + {-325, 0x0, 0}, + {-326, 0x0, 0}, + {309, 1, 2000}, + {310, 1, 2000}, + {311, 1, 2000}, + }; + for (const TestCase& test_case : kTestCases) { + EXPECT_EQ(Pow10(test_case.power), + std::ldexp(test_case.significand, test_case.radix)) + << absl::StrFormat("Failure for Pow10(%d): %a vs %a", test_case.power, + Pow10(test_case.power), + std::ldexp(test_case.significand, test_case.radix)); + } +} + +} // namespace +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/resize_uninitialized.h b/third_party/abseil_cpp/absl/strings/internal/resize_uninitialized.h new file mode 100644 index 000000000000..e42628e3949a --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/resize_uninitialized.h @@ -0,0 +1,73 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_STRINGS_INTERNAL_RESIZE_UNINITIALIZED_H_ +#define ABSL_STRINGS_INTERNAL_RESIZE_UNINITIALIZED_H_ + +#include <string> +#include <type_traits> +#include <utility> + +#include "absl/base/port.h" +#include "absl/meta/type_traits.h" // for void_t + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// Is a subclass of true_type or false_type, depending on whether or not +// T has a __resize_default_init member. +template <typename string_type, typename = void> +struct ResizeUninitializedTraits { + using HasMember = std::false_type; + static void Resize(string_type* s, size_t new_size) { s->resize(new_size); } +}; + +// __resize_default_init is provided by libc++ >= 8.0 +template <typename string_type> +struct ResizeUninitializedTraits< + string_type, absl::void_t<decltype(std::declval<string_type&>() + .__resize_default_init(237))> > { + using HasMember = std::true_type; + static void Resize(string_type* s, size_t new_size) { + s->__resize_default_init(new_size); + } +}; + +// Returns true if the std::string implementation supports a resize where +// the new characters added to the std::string are left untouched. +// +// (A better name might be "STLStringSupportsUninitializedResize", alluding to +// the previous function.) +template <typename string_type> +inline constexpr bool STLStringSupportsNontrashingResize(string_type*) { + return ResizeUninitializedTraits<string_type>::HasMember::value; +} + +// Like str->resize(new_size), except any new characters added to "*str" as a +// result of resizing may be left uninitialized, rather than being filled with +// '0' bytes. Typically used when code is then going to overwrite the backing +// store of the std::string with known data. +template <typename string_type, typename = void> +inline void STLStringResizeUninitialized(string_type* s, size_t new_size) { + ResizeUninitializedTraits<string_type>::Resize(s, new_size); +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_RESIZE_UNINITIALIZED_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/resize_uninitialized_test.cc b/third_party/abseil_cpp/absl/strings/internal/resize_uninitialized_test.cc new file mode 100644 index 000000000000..0f8b3c2a95b8 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/resize_uninitialized_test.cc @@ -0,0 +1,82 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/resize_uninitialized.h" + +#include "gtest/gtest.h" + +namespace { + +int resize_call_count = 0; + +// A mock string class whose only purpose is to track how many times its +// resize() method has been called. +struct resizable_string { + size_t size() const { return 0; } + char& operator[](size_t) { + static char c = '\0'; + return c; + } + void resize(size_t) { resize_call_count += 1; } +}; + +int resize_default_init_call_count = 0; + +// A mock string class whose only purpose is to track how many times its +// resize() and __resize_default_init() methods have been called. +struct resize_default_init_string { + size_t size() const { return 0; } + char& operator[](size_t) { + static char c = '\0'; + return c; + } + void resize(size_t) { resize_call_count += 1; } + void __resize_default_init(size_t) { resize_default_init_call_count += 1; } +}; + +TEST(ResizeUninit, WithAndWithout) { + resize_call_count = 0; + resize_default_init_call_count = 0; + { + resizable_string rs; + + EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_FALSE( + absl::strings_internal::STLStringSupportsNontrashingResize(&rs)); + EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(resize_default_init_call_count, 0); + absl::strings_internal::STLStringResizeUninitialized(&rs, 237); + EXPECT_EQ(resize_call_count, 1); + EXPECT_EQ(resize_default_init_call_count, 0); + } + + resize_call_count = 0; + resize_default_init_call_count = 0; + { + resize_default_init_string rus; + + EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_TRUE( + absl::strings_internal::STLStringSupportsNontrashingResize(&rus)); + EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(resize_default_init_call_count, 0); + absl::strings_internal::STLStringResizeUninitialized(&rus, 237); + EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(resize_default_init_call_count, 1); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/stl_type_traits.h b/third_party/abseil_cpp/absl/strings/internal/stl_type_traits.h new file mode 100644 index 000000000000..6035ca45cbd2 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/stl_type_traits.h @@ -0,0 +1,248 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Thie file provides the IsStrictlyBaseOfAndConvertibleToSTLContainer type +// trait metafunction to assist in working with the _GLIBCXX_DEBUG debug +// wrappers of STL containers. +// +// DO NOT INCLUDE THIS FILE DIRECTLY. Use this file by including +// absl/strings/str_split.h. +// +// IWYU pragma: private, include "absl/strings/str_split.h" + +#ifndef ABSL_STRINGS_INTERNAL_STL_TYPE_TRAITS_H_ +#define ABSL_STRINGS_INTERNAL_STL_TYPE_TRAITS_H_ + +#include <array> +#include <bitset> +#include <deque> +#include <forward_list> +#include <list> +#include <map> +#include <set> +#include <type_traits> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +template <typename C, template <typename...> class T> +struct IsSpecializationImpl : std::false_type {}; +template <template <typename...> class T, typename... Args> +struct IsSpecializationImpl<T<Args...>, T> : std::true_type {}; +template <typename C, template <typename...> class T> +using IsSpecialization = IsSpecializationImpl<absl::decay_t<C>, T>; + +template <typename C> +struct IsArrayImpl : std::false_type {}; +template <template <typename, size_t> class A, typename T, size_t N> +struct IsArrayImpl<A<T, N>> : std::is_same<A<T, N>, std::array<T, N>> {}; +template <typename C> +using IsArray = IsArrayImpl<absl::decay_t<C>>; + +template <typename C> +struct IsBitsetImpl : std::false_type {}; +template <template <size_t> class B, size_t N> +struct IsBitsetImpl<B<N>> : std::is_same<B<N>, std::bitset<N>> {}; +template <typename C> +using IsBitset = IsBitsetImpl<absl::decay_t<C>>; + +template <typename C> +struct IsSTLContainer + : absl::disjunction< + IsArray<C>, IsBitset<C>, IsSpecialization<C, std::deque>, + IsSpecialization<C, std::forward_list>, + IsSpecialization<C, std::list>, IsSpecialization<C, std::map>, + IsSpecialization<C, std::multimap>, IsSpecialization<C, std::set>, + IsSpecialization<C, std::multiset>, + IsSpecialization<C, std::unordered_map>, + IsSpecialization<C, std::unordered_multimap>, + IsSpecialization<C, std::unordered_set>, + IsSpecialization<C, std::unordered_multiset>, + IsSpecialization<C, std::vector>> {}; + +template <typename C, template <typename...> class T, typename = void> +struct IsBaseOfSpecializationImpl : std::false_type {}; +// IsBaseOfSpecializationImpl needs multiple partial specializations to SFINAE +// on the existence of container dependent types and plug them into the STL +// template. +template <typename C, template <typename, typename> class T> +struct IsBaseOfSpecializationImpl< + C, T, absl::void_t<typename C::value_type, typename C::allocator_type>> + : std::is_base_of<C, + T<typename C::value_type, typename C::allocator_type>> {}; +template <typename C, template <typename, typename, typename> class T> +struct IsBaseOfSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::key_compare, + typename C::allocator_type>> + : std::is_base_of<C, T<typename C::key_type, typename C::key_compare, + typename C::allocator_type>> {}; +template <typename C, template <typename, typename, typename, typename> class T> +struct IsBaseOfSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::mapped_type, + typename C::key_compare, typename C::allocator_type>> + : std::is_base_of<C, + T<typename C::key_type, typename C::mapped_type, + typename C::key_compare, typename C::allocator_type>> { +}; +template <typename C, template <typename, typename, typename, typename> class T> +struct IsBaseOfSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::hasher, + typename C::key_equal, typename C::allocator_type>> + : std::is_base_of<C, T<typename C::key_type, typename C::hasher, + typename C::key_equal, typename C::allocator_type>> { +}; +template <typename C, + template <typename, typename, typename, typename, typename> class T> +struct IsBaseOfSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::mapped_type, + typename C::hasher, typename C::key_equal, + typename C::allocator_type>> + : std::is_base_of<C, T<typename C::key_type, typename C::mapped_type, + typename C::hasher, typename C::key_equal, + typename C::allocator_type>> {}; +template <typename C, template <typename...> class T> +using IsBaseOfSpecialization = IsBaseOfSpecializationImpl<absl::decay_t<C>, T>; + +template <typename C> +struct IsBaseOfArrayImpl : std::false_type {}; +template <template <typename, size_t> class A, typename T, size_t N> +struct IsBaseOfArrayImpl<A<T, N>> : std::is_base_of<A<T, N>, std::array<T, N>> { +}; +template <typename C> +using IsBaseOfArray = IsBaseOfArrayImpl<absl::decay_t<C>>; + +template <typename C> +struct IsBaseOfBitsetImpl : std::false_type {}; +template <template <size_t> class B, size_t N> +struct IsBaseOfBitsetImpl<B<N>> : std::is_base_of<B<N>, std::bitset<N>> {}; +template <typename C> +using IsBaseOfBitset = IsBaseOfBitsetImpl<absl::decay_t<C>>; + +template <typename C> +struct IsBaseOfSTLContainer + : absl::disjunction<IsBaseOfArray<C>, IsBaseOfBitset<C>, + IsBaseOfSpecialization<C, std::deque>, + IsBaseOfSpecialization<C, std::forward_list>, + IsBaseOfSpecialization<C, std::list>, + IsBaseOfSpecialization<C, std::map>, + IsBaseOfSpecialization<C, std::multimap>, + IsBaseOfSpecialization<C, std::set>, + IsBaseOfSpecialization<C, std::multiset>, + IsBaseOfSpecialization<C, std::unordered_map>, + IsBaseOfSpecialization<C, std::unordered_multimap>, + IsBaseOfSpecialization<C, std::unordered_set>, + IsBaseOfSpecialization<C, std::unordered_multiset>, + IsBaseOfSpecialization<C, std::vector>> {}; + +template <typename C, template <typename...> class T, typename = void> +struct IsConvertibleToSpecializationImpl : std::false_type {}; +// IsConvertibleToSpecializationImpl needs multiple partial specializations to +// SFINAE on the existence of container dependent types and plug them into the +// STL template. +template <typename C, template <typename, typename> class T> +struct IsConvertibleToSpecializationImpl< + C, T, absl::void_t<typename C::value_type, typename C::allocator_type>> + : std::is_convertible< + C, T<typename C::value_type, typename C::allocator_type>> {}; +template <typename C, template <typename, typename, typename> class T> +struct IsConvertibleToSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::key_compare, + typename C::allocator_type>> + : std::is_convertible<C, T<typename C::key_type, typename C::key_compare, + typename C::allocator_type>> {}; +template <typename C, template <typename, typename, typename, typename> class T> +struct IsConvertibleToSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::mapped_type, + typename C::key_compare, typename C::allocator_type>> + : std::is_convertible< + C, T<typename C::key_type, typename C::mapped_type, + typename C::key_compare, typename C::allocator_type>> {}; +template <typename C, template <typename, typename, typename, typename> class T> +struct IsConvertibleToSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::hasher, + typename C::key_equal, typename C::allocator_type>> + : std::is_convertible< + C, T<typename C::key_type, typename C::hasher, typename C::key_equal, + typename C::allocator_type>> {}; +template <typename C, + template <typename, typename, typename, typename, typename> class T> +struct IsConvertibleToSpecializationImpl< + C, T, + absl::void_t<typename C::key_type, typename C::mapped_type, + typename C::hasher, typename C::key_equal, + typename C::allocator_type>> + : std::is_convertible<C, T<typename C::key_type, typename C::mapped_type, + typename C::hasher, typename C::key_equal, + typename C::allocator_type>> {}; +template <typename C, template <typename...> class T> +using IsConvertibleToSpecialization = + IsConvertibleToSpecializationImpl<absl::decay_t<C>, T>; + +template <typename C> +struct IsConvertibleToArrayImpl : std::false_type {}; +template <template <typename, size_t> class A, typename T, size_t N> +struct IsConvertibleToArrayImpl<A<T, N>> + : std::is_convertible<A<T, N>, std::array<T, N>> {}; +template <typename C> +using IsConvertibleToArray = IsConvertibleToArrayImpl<absl::decay_t<C>>; + +template <typename C> +struct IsConvertibleToBitsetImpl : std::false_type {}; +template <template <size_t> class B, size_t N> +struct IsConvertibleToBitsetImpl<B<N>> + : std::is_convertible<B<N>, std::bitset<N>> {}; +template <typename C> +using IsConvertibleToBitset = IsConvertibleToBitsetImpl<absl::decay_t<C>>; + +template <typename C> +struct IsConvertibleToSTLContainer + : absl::disjunction< + IsConvertibleToArray<C>, IsConvertibleToBitset<C>, + IsConvertibleToSpecialization<C, std::deque>, + IsConvertibleToSpecialization<C, std::forward_list>, + IsConvertibleToSpecialization<C, std::list>, + IsConvertibleToSpecialization<C, std::map>, + IsConvertibleToSpecialization<C, std::multimap>, + IsConvertibleToSpecialization<C, std::set>, + IsConvertibleToSpecialization<C, std::multiset>, + IsConvertibleToSpecialization<C, std::unordered_map>, + IsConvertibleToSpecialization<C, std::unordered_multimap>, + IsConvertibleToSpecialization<C, std::unordered_set>, + IsConvertibleToSpecialization<C, std::unordered_multiset>, + IsConvertibleToSpecialization<C, std::vector>> {}; + +template <typename C> +struct IsStrictlyBaseOfAndConvertibleToSTLContainer + : absl::conjunction<absl::negation<IsSTLContainer<C>>, + IsBaseOfSTLContainer<C>, + IsConvertibleToSTLContainer<C>> {}; + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_STRINGS_INTERNAL_STL_TYPE_TRAITS_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/arg.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/arg.cc new file mode 100644 index 000000000000..e28a29b17169 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/arg.cc @@ -0,0 +1,488 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +// POSIX spec: +// http://pubs.opengroup.org/onlinepubs/009695399/functions/fprintf.html +// +#include "absl/strings/internal/str_format/arg.h" + +#include <cassert> +#include <cerrno> +#include <cstdlib> +#include <string> +#include <type_traits> + +#include "absl/base/port.h" +#include "absl/strings/internal/str_format/float_conversion.h" +#include "absl/strings/numbers.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { +namespace { + +// Reduce *capacity by s.size(), clipped to a 0 minimum. +void ReducePadding(string_view s, size_t *capacity) { + *capacity = Excess(s.size(), *capacity); +} + +// Reduce *capacity by n, clipped to a 0 minimum. +void ReducePadding(size_t n, size_t *capacity) { + *capacity = Excess(n, *capacity); +} + +template <typename T> +struct MakeUnsigned : std::make_unsigned<T> {}; +template <> +struct MakeUnsigned<absl::int128> { + using type = absl::uint128; +}; +template <> +struct MakeUnsigned<absl::uint128> { + using type = absl::uint128; +}; + +template <typename T> +struct IsSigned : std::is_signed<T> {}; +template <> +struct IsSigned<absl::int128> : std::true_type {}; +template <> +struct IsSigned<absl::uint128> : std::false_type {}; + +// Integral digit printer. +// Call one of the PrintAs* routines after construction once. +// Use with_neg_and_zero/without_neg_or_zero/is_negative to access the results. +class IntDigits { + public: + // Print the unsigned integer as octal. + // Supports unsigned integral types and uint128. + template <typename T> + void PrintAsOct(T v) { + static_assert(!IsSigned<T>::value, ""); + char *p = storage_ + sizeof(storage_); + do { + *--p = static_cast<char>('0' + (static_cast<size_t>(v) & 7)); + v >>= 3; + } while (v); + start_ = p; + size_ = storage_ + sizeof(storage_) - p; + } + + // Print the signed or unsigned integer as decimal. + // Supports all integral types. + template <typename T> + void PrintAsDec(T v) { + static_assert(std::is_integral<T>::value, ""); + start_ = storage_; + size_ = numbers_internal::FastIntToBuffer(v, storage_) - storage_; + } + + void PrintAsDec(int128 v) { + auto u = static_cast<uint128>(v); + bool add_neg = false; + if (v < 0) { + add_neg = true; + u = uint128{} - u; + } + PrintAsDec(u, add_neg); + } + + void PrintAsDec(uint128 v, bool add_neg = false) { + // This function can be sped up if needed. We can call FastIntToBuffer + // twice, or fix FastIntToBuffer to support uint128. + char *p = storage_ + sizeof(storage_); + do { + p -= 2; + numbers_internal::PutTwoDigits(static_cast<size_t>(v % 100), p); + v /= 100; + } while (v); + if (p[0] == '0') { + // We printed one too many hexits. + ++p; + } + if (add_neg) { + *--p = '-'; + } + size_ = storage_ + sizeof(storage_) - p; + start_ = p; + } + + // Print the unsigned integer as hex using lowercase. + // Supports unsigned integral types and uint128. + template <typename T> + void PrintAsHexLower(T v) { + static_assert(!IsSigned<T>::value, ""); + char *p = storage_ + sizeof(storage_); + + do { + p -= 2; + constexpr const char* table = numbers_internal::kHexTable; + std::memcpy(p, table + 2 * (static_cast<size_t>(v) & 0xFF), 2); + if (sizeof(T) == 1) break; + v >>= 8; + } while (v); + if (p[0] == '0') { + // We printed one too many digits. + ++p; + } + start_ = p; + size_ = storage_ + sizeof(storage_) - p; + } + + // Print the unsigned integer as hex using uppercase. + // Supports unsigned integral types and uint128. + template <typename T> + void PrintAsHexUpper(T v) { + static_assert(!IsSigned<T>::value, ""); + char *p = storage_ + sizeof(storage_); + + // kHexTable is only lowercase, so do it manually for uppercase. + do { + *--p = "0123456789ABCDEF"[static_cast<size_t>(v) & 15]; + v >>= 4; + } while (v); + start_ = p; + size_ = storage_ + sizeof(storage_) - p; + } + + // The printed value including the '-' sign if available. + // For inputs of value `0`, this will return "0" + string_view with_neg_and_zero() const { return {start_, size_}; } + + // The printed value not including the '-' sign. + // For inputs of value `0`, this will return "". + string_view without_neg_or_zero() const { + static_assert('-' < '0', "The check below verifies both."); + size_t advance = start_[0] <= '0' ? 1 : 0; + return {start_ + advance, size_ - advance}; + } + + bool is_negative() const { return start_[0] == '-'; } + + private: + const char *start_; + size_t size_; + // Max size: 128 bit value as octal -> 43 digits, plus sign char + char storage_[128 / 3 + 1 + 1]; +}; + +// Note: 'o' conversions do not have a base indicator, it's just that +// the '#' flag is specified to modify the precision for 'o' conversions. +string_view BaseIndicator(const IntDigits &as_digits, + const FormatConversionSpecImpl conv) { + // always show 0x for %p. + bool alt = conv.has_alt_flag() || + conv.conversion_char() == FormatConversionCharInternal::p; + bool hex = (conv.conversion_char() == FormatConversionCharInternal::x || + conv.conversion_char() == FormatConversionCharInternal::X || + conv.conversion_char() == FormatConversionCharInternal::p); + // From the POSIX description of '#' flag: + // "For x or X conversion specifiers, a non-zero result shall have + // 0x (or 0X) prefixed to it." + if (alt && hex && !as_digits.without_neg_or_zero().empty()) { + return conv.conversion_char() == FormatConversionCharInternal::X ? "0X" + : "0x"; + } + return {}; +} + +string_view SignColumn(bool neg, const FormatConversionSpecImpl conv) { + if (conv.conversion_char() == FormatConversionCharInternal::d || + conv.conversion_char() == FormatConversionCharInternal::i) { + if (neg) return "-"; + if (conv.has_show_pos_flag()) return "+"; + if (conv.has_sign_col_flag()) return " "; + } + return {}; +} + +bool ConvertCharImpl(unsigned char v, const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + size_t fill = 0; + if (conv.width() >= 0) fill = conv.width(); + ReducePadding(1, &fill); + if (!conv.has_left_flag()) sink->Append(fill, ' '); + sink->Append(1, v); + if (conv.has_left_flag()) sink->Append(fill, ' '); + return true; +} + +bool ConvertIntImplInnerSlow(const IntDigits &as_digits, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + // Print as a sequence of Substrings: + // [left_spaces][sign][base_indicator][zeroes][formatted][right_spaces] + size_t fill = 0; + if (conv.width() >= 0) fill = conv.width(); + + string_view formatted = as_digits.without_neg_or_zero(); + ReducePadding(formatted, &fill); + + string_view sign = SignColumn(as_digits.is_negative(), conv); + ReducePadding(sign, &fill); + + string_view base_indicator = BaseIndicator(as_digits, conv); + ReducePadding(base_indicator, &fill); + + int precision = conv.precision(); + bool precision_specified = precision >= 0; + if (!precision_specified) + precision = 1; + + if (conv.has_alt_flag() && + conv.conversion_char() == FormatConversionCharInternal::o) { + // From POSIX description of the '#' (alt) flag: + // "For o conversion, it increases the precision (if necessary) to + // force the first digit of the result to be zero." + if (formatted.empty() || *formatted.begin() != '0') { + int needed = static_cast<int>(formatted.size()) + 1; + precision = std::max(precision, needed); + } + } + + size_t num_zeroes = Excess(formatted.size(), precision); + ReducePadding(num_zeroes, &fill); + + size_t num_left_spaces = !conv.has_left_flag() ? fill : 0; + size_t num_right_spaces = conv.has_left_flag() ? fill : 0; + + // From POSIX description of the '0' (zero) flag: + // "For d, i, o, u, x, and X conversion specifiers, if a precision + // is specified, the '0' flag is ignored." + if (!precision_specified && conv.has_zero_flag()) { + num_zeroes += num_left_spaces; + num_left_spaces = 0; + } + + sink->Append(num_left_spaces, ' '); + sink->Append(sign); + sink->Append(base_indicator); + sink->Append(num_zeroes, '0'); + sink->Append(formatted); + sink->Append(num_right_spaces, ' '); + return true; +} + +template <typename T> +bool ConvertIntArg(T v, const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + using U = typename MakeUnsigned<T>::type; + IntDigits as_digits; + + // This odd casting is due to a bug in -Wswitch behavior in gcc49 which causes + // it to complain about a switch/case type mismatch, even though both are + // FormatConverionChar. Likely this is because at this point + // FormatConversionChar is declared, but not defined. + switch (static_cast<uint8_t>(conv.conversion_char())) { + case static_cast<uint8_t>(FormatConversionCharInternal::c): + return ConvertCharImpl(static_cast<unsigned char>(v), conv, sink); + + case static_cast<uint8_t>(FormatConversionCharInternal::o): + as_digits.PrintAsOct(static_cast<U>(v)); + break; + + case static_cast<uint8_t>(FormatConversionCharInternal::x): + as_digits.PrintAsHexLower(static_cast<U>(v)); + break; + case static_cast<uint8_t>(FormatConversionCharInternal::X): + as_digits.PrintAsHexUpper(static_cast<U>(v)); + break; + + case static_cast<uint8_t>(FormatConversionCharInternal::u): + as_digits.PrintAsDec(static_cast<U>(v)); + break; + + case static_cast<uint8_t>(FormatConversionCharInternal::d): + case static_cast<uint8_t>(FormatConversionCharInternal::i): + as_digits.PrintAsDec(v); + break; + + case static_cast<uint8_t>(FormatConversionCharInternal::a): + case static_cast<uint8_t>(FormatConversionCharInternal::e): + case static_cast<uint8_t>(FormatConversionCharInternal::f): + case static_cast<uint8_t>(FormatConversionCharInternal::g): + case static_cast<uint8_t>(FormatConversionCharInternal::A): + case static_cast<uint8_t>(FormatConversionCharInternal::E): + case static_cast<uint8_t>(FormatConversionCharInternal::F): + case static_cast<uint8_t>(FormatConversionCharInternal::G): + return ConvertFloatImpl(static_cast<double>(v), conv, sink); + + default: + ABSL_INTERNAL_ASSUME(false); + } + + if (conv.is_basic()) { + sink->Append(as_digits.with_neg_and_zero()); + return true; + } + return ConvertIntImplInnerSlow(as_digits, conv, sink); +} + +template <typename T> +bool ConvertFloatArg(T v, const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return FormatConversionCharIsFloat(conv.conversion_char()) && + ConvertFloatImpl(v, conv, sink); +} + +inline bool ConvertStringArg(string_view v, const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + if (conv.is_basic()) { + sink->Append(v); + return true; + } + return sink->PutPaddedString(v, conv.width(), conv.precision(), + conv.has_left_flag()); +} + +} // namespace + +// ==================== Strings ==================== +StringConvertResult FormatConvertImpl(const std::string &v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertStringArg(v, conv, sink)}; +} + +StringConvertResult FormatConvertImpl(string_view v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertStringArg(v, conv, sink)}; +} + +ArgConvertResult<FormatConversionCharSetUnion( + FormatConversionCharSetInternal::s, FormatConversionCharSetInternal::p)> +FormatConvertImpl(const char *v, const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + if (conv.conversion_char() == FormatConversionCharInternal::p) + return {FormatConvertImpl(VoidPtr(v), conv, sink).value}; + size_t len; + if (v == nullptr) { + len = 0; + } else if (conv.precision() < 0) { + len = std::strlen(v); + } else { + // If precision is set, we look for the NUL-terminator on the valid range. + len = std::find(v, v + conv.precision(), '\0') - v; + } + return {ConvertStringArg(string_view(v, len), conv, sink)}; +} + +// ==================== Raw pointers ==================== +ArgConvertResult<FormatConversionCharSetInternal::p> FormatConvertImpl( + VoidPtr v, const FormatConversionSpecImpl conv, FormatSinkImpl *sink) { + if (!v.value) { + sink->Append("(nil)"); + return {true}; + } + IntDigits as_digits; + as_digits.PrintAsHexLower(v.value); + return {ConvertIntImplInnerSlow(as_digits, conv, sink)}; +} + +// ==================== Floats ==================== +FloatingConvertResult FormatConvertImpl(float v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertFloatArg(v, conv, sink)}; +} +FloatingConvertResult FormatConvertImpl(double v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertFloatArg(v, conv, sink)}; +} +FloatingConvertResult FormatConvertImpl(long double v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertFloatArg(v, conv, sink)}; +} + +// ==================== Chars ==================== +IntegralConvertResult FormatConvertImpl(char v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(signed char v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(unsigned char v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} + +// ==================== Ints ==================== +IntegralConvertResult FormatConvertImpl(short v, // NOLINT + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(unsigned short v, // NOLINT + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(int v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(unsigned v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(long v, // NOLINT + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(unsigned long v, // NOLINT + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(long long v, // NOLINT + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(unsigned long long v, // NOLINT + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(absl::int128 v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} +IntegralConvertResult FormatConvertImpl(absl::uint128 v, + const FormatConversionSpecImpl conv, + FormatSinkImpl *sink) { + return {ConvertIntArg(v, conv, sink)}; +} + +ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(); + + + +} // namespace str_format_internal + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/arg.h b/third_party/abseil_cpp/absl/strings/internal/str_format/arg.h new file mode 100644 index 000000000000..7040c866778e --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/arg.h @@ -0,0 +1,518 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_ARG_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_ARG_H_ + +#include <string.h> +#include <wchar.h> + +#include <cstdio> +#include <iomanip> +#include <limits> +#include <memory> +#include <sstream> +#include <string> +#include <type_traits> + +#include "absl/base/port.h" +#include "absl/meta/type_traits.h" +#include "absl/numeric/int128.h" +#include "absl/strings/internal/str_format/extension.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class Cord; +class FormatCountCapture; +class FormatSink; + +template <absl::FormatConversionCharSet C> +struct FormatConvertResult; +class FormatConversionSpec; + +namespace str_format_internal { + +template <typename T, typename = void> +struct HasUserDefinedConvert : std::false_type {}; + +template <typename T> +struct HasUserDefinedConvert<T, void_t<decltype(AbslFormatConvert( + std::declval<const T&>(), + std::declval<const FormatConversionSpec&>(), + std::declval<FormatSink*>()))>> + : std::true_type {}; + +void AbslFormatConvert(); // Stops the lexical name lookup +template <typename T> +auto FormatConvertImpl(const T& v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink) + -> decltype(AbslFormatConvert(v, + std::declval<const FormatConversionSpec&>(), + std::declval<FormatSink*>())) { + using FormatConversionSpecT = + absl::enable_if_t<sizeof(const T& (*)()) != 0, FormatConversionSpec>; + using FormatSinkT = + absl::enable_if_t<sizeof(const T& (*)()) != 0, FormatSink>; + auto fcs = conv.Wrap<FormatConversionSpecT>(); + auto fs = sink->Wrap<FormatSinkT>(); + return AbslFormatConvert(v, fcs, &fs); +} + +template <typename T> +class StreamedWrapper; + +// If 'v' can be converted (in the printf sense) according to 'conv', +// then convert it, appending to `sink` and return `true`. +// Otherwise fail and return `false`. + +// AbslFormatConvert(v, conv, sink) is intended to be found by ADL on 'v' +// as an extension mechanism. These FormatConvertImpl functions are the default +// implementations. +// The ADL search is augmented via the 'Sink*' parameter, which also +// serves as a disambiguator to reject possible unintended 'AbslFormatConvert' +// functions in the namespaces associated with 'v'. + +// Raw pointers. +struct VoidPtr { + VoidPtr() = default; + template <typename T, + decltype(reinterpret_cast<uintptr_t>(std::declval<T*>())) = 0> + VoidPtr(T* ptr) // NOLINT + : value(ptr ? reinterpret_cast<uintptr_t>(ptr) : 0) {} + uintptr_t value; +}; + +template <FormatConversionCharSet C> +struct ArgConvertResult { + bool value; +}; + +template <FormatConversionCharSet C> +constexpr FormatConversionCharSet ExtractCharSet(FormatConvertResult<C>) { + return C; +} + +template <FormatConversionCharSet C> +constexpr FormatConversionCharSet ExtractCharSet(ArgConvertResult<C>) { + return C; +} + +using StringConvertResult = + ArgConvertResult<FormatConversionCharSetInternal::s>; +ArgConvertResult<FormatConversionCharSetInternal::p> FormatConvertImpl( + VoidPtr v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); + +// Strings. +StringConvertResult FormatConvertImpl(const std::string& v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +StringConvertResult FormatConvertImpl(string_view v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +ArgConvertResult<FormatConversionCharSetUnion( + FormatConversionCharSetInternal::s, FormatConversionCharSetInternal::p)> +FormatConvertImpl(const char* v, const FormatConversionSpecImpl conv, + FormatSinkImpl* sink); + +template <class AbslCord, typename std::enable_if<std::is_same< + AbslCord, absl::Cord>::value>::type* = nullptr> +StringConvertResult FormatConvertImpl(const AbslCord& value, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + bool is_left = conv.has_left_flag(); + size_t space_remaining = 0; + + int width = conv.width(); + if (width >= 0) space_remaining = width; + + size_t to_write = value.size(); + + int precision = conv.precision(); + if (precision >= 0) + to_write = (std::min)(to_write, static_cast<size_t>(precision)); + + space_remaining = Excess(to_write, space_remaining); + + if (space_remaining > 0 && !is_left) sink->Append(space_remaining, ' '); + + for (string_view piece : value.Chunks()) { + if (piece.size() > to_write) { + piece.remove_suffix(piece.size() - to_write); + to_write = 0; + } else { + to_write -= piece.size(); + } + sink->Append(piece); + if (to_write == 0) { + break; + } + } + + if (space_remaining > 0 && is_left) sink->Append(space_remaining, ' '); + return {true}; +} + +using IntegralConvertResult = ArgConvertResult<FormatConversionCharSetUnion( + FormatConversionCharSetInternal::c, + FormatConversionCharSetInternal::kNumeric, + FormatConversionCharSetInternal::kStar)>; +using FloatingConvertResult = + ArgConvertResult<FormatConversionCharSetInternal::kFloating>; + +// Floats. +FloatingConvertResult FormatConvertImpl(float v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +FloatingConvertResult FormatConvertImpl(double v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +FloatingConvertResult FormatConvertImpl(long double v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); + +// Chars. +IntegralConvertResult FormatConvertImpl(char v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(signed char v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(unsigned char v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); + +// Ints. +IntegralConvertResult FormatConvertImpl(short v, // NOLINT + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(unsigned short v, // NOLINT + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(int v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(unsigned v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(long v, // NOLINT + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(unsigned long v, // NOLINT + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(long long v, // NOLINT + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(unsigned long long v, // NOLINT + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(int128 v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +IntegralConvertResult FormatConvertImpl(uint128 v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink); +template <typename T, enable_if_t<std::is_same<T, bool>::value, int> = 0> +IntegralConvertResult FormatConvertImpl(T v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + return FormatConvertImpl(static_cast<int>(v), conv, sink); +} + +// We provide this function to help the checker, but it is never defined. +// FormatArgImpl will use the underlying Convert functions instead. +template <typename T> +typename std::enable_if<std::is_enum<T>::value && + !HasUserDefinedConvert<T>::value, + IntegralConvertResult>::type +FormatConvertImpl(T v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); + +template <typename T> +StringConvertResult FormatConvertImpl(const StreamedWrapper<T>& v, + FormatConversionSpecImpl conv, + FormatSinkImpl* out) { + std::ostringstream oss; + oss << v.v_; + if (!oss) return {false}; + return str_format_internal::FormatConvertImpl(oss.str(), conv, out); +} + +// Use templates and dependent types to delay evaluation of the function +// until after FormatCountCapture is fully defined. +struct FormatCountCaptureHelper { + template <class T = int> + static ArgConvertResult<FormatConversionCharSetInternal::n> ConvertHelper( + const FormatCountCapture& v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + const absl::enable_if_t<sizeof(T) != 0, FormatCountCapture>& v2 = v; + + if (conv.conversion_char() != + str_format_internal::FormatConversionCharInternal::n) { + return {false}; + } + *v2.p_ = static_cast<int>(sink->size()); + return {true}; + } +}; + +template <class T = int> +ArgConvertResult<FormatConversionCharSetInternal::n> FormatConvertImpl( + const FormatCountCapture& v, FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + return FormatCountCaptureHelper::ConvertHelper(v, conv, sink); +} + +// Helper friend struct to hide implementation details from the public API of +// FormatArgImpl. +struct FormatArgImplFriend { + template <typename Arg> + static bool ToInt(Arg arg, int* out) { + // A value initialized FormatConversionSpecImpl has a `none` conv, which + // tells the dispatcher to run the `int` conversion. + return arg.dispatcher_(arg.data_, {}, out); + } + + template <typename Arg> + static bool Convert(Arg arg, FormatConversionSpecImpl conv, + FormatSinkImpl* out) { + return arg.dispatcher_(arg.data_, conv, out); + } + + template <typename Arg> + static typename Arg::Dispatcher GetVTablePtrForTest(Arg arg) { + return arg.dispatcher_; + } +}; + +template <typename Arg> +constexpr FormatConversionCharSet ArgumentToConv() { + return absl::str_format_internal::ExtractCharSet( + decltype(str_format_internal::FormatConvertImpl( + std::declval<const Arg&>(), + std::declval<const FormatConversionSpecImpl&>(), + std::declval<FormatSinkImpl*>())){}); +} + +// A type-erased handle to a format argument. +class FormatArgImpl { + private: + enum { kInlinedSpace = 8 }; + + using VoidPtr = str_format_internal::VoidPtr; + + union Data { + const void* ptr; + const volatile void* volatile_ptr; + char buf[kInlinedSpace]; + }; + + using Dispatcher = bool (*)(Data, FormatConversionSpecImpl, void* out); + + template <typename T> + struct store_by_value + : std::integral_constant<bool, (sizeof(T) <= kInlinedSpace) && + (std::is_integral<T>::value || + std::is_floating_point<T>::value || + std::is_pointer<T>::value || + std::is_same<VoidPtr, T>::value)> {}; + + enum StoragePolicy { ByPointer, ByVolatilePointer, ByValue }; + template <typename T> + struct storage_policy + : std::integral_constant<StoragePolicy, + (std::is_volatile<T>::value + ? ByVolatilePointer + : (store_by_value<T>::value ? ByValue + : ByPointer))> { + }; + + // To reduce the number of vtables we will decay values before hand. + // Anything with a user-defined Convert will get its own vtable. + // For everything else: + // - Decay char* and char arrays into `const char*` + // - Decay any other pointer to `const void*` + // - Decay all enums to their underlying type. + // - Decay function pointers to void*. + template <typename T, typename = void> + struct DecayType { + static constexpr bool kHasUserDefined = + str_format_internal::HasUserDefinedConvert<T>::value; + using type = typename std::conditional< + !kHasUserDefined && std::is_convertible<T, const char*>::value, + const char*, + typename std::conditional<!kHasUserDefined && + std::is_convertible<T, VoidPtr>::value, + VoidPtr, const T&>::type>::type; + }; + template <typename T> + struct DecayType<T, + typename std::enable_if< + !str_format_internal::HasUserDefinedConvert<T>::value && + std::is_enum<T>::value>::type> { + using type = typename std::underlying_type<T>::type; + }; + + public: + template <typename T> + explicit FormatArgImpl(const T& value) { + using D = typename DecayType<T>::type; + static_assert( + std::is_same<D, const T&>::value || storage_policy<D>::value == ByValue, + "Decayed types must be stored by value"); + Init(static_cast<D>(value)); + } + + private: + friend struct str_format_internal::FormatArgImplFriend; + template <typename T, StoragePolicy = storage_policy<T>::value> + struct Manager; + + template <typename T> + struct Manager<T, ByPointer> { + static Data SetValue(const T& value) { + Data data; + data.ptr = std::addressof(value); + return data; + } + + static const T& Value(Data arg) { return *static_cast<const T*>(arg.ptr); } + }; + + template <typename T> + struct Manager<T, ByVolatilePointer> { + static Data SetValue(const T& value) { + Data data; + data.volatile_ptr = &value; + return data; + } + + static const T& Value(Data arg) { + return *static_cast<const T*>(arg.volatile_ptr); + } + }; + + template <typename T> + struct Manager<T, ByValue> { + static Data SetValue(const T& value) { + Data data; + memcpy(data.buf, &value, sizeof(value)); + return data; + } + + static T Value(Data arg) { + T value; + memcpy(&value, arg.buf, sizeof(T)); + return value; + } + }; + + template <typename T> + void Init(const T& value) { + data_ = Manager<T>::SetValue(value); + dispatcher_ = &Dispatch<T>; + } + + template <typename T> + static int ToIntVal(const T& val) { + using CommonType = typename std::conditional<std::is_signed<T>::value, + int64_t, uint64_t>::type; + if (static_cast<CommonType>(val) > + static_cast<CommonType>((std::numeric_limits<int>::max)())) { + return (std::numeric_limits<int>::max)(); + } else if (std::is_signed<T>::value && + static_cast<CommonType>(val) < + static_cast<CommonType>((std::numeric_limits<int>::min)())) { + return (std::numeric_limits<int>::min)(); + } + return static_cast<int>(val); + } + + template <typename T> + static bool ToInt(Data arg, int* out, std::true_type /* is_integral */, + std::false_type) { + *out = ToIntVal(Manager<T>::Value(arg)); + return true; + } + + template <typename T> + static bool ToInt(Data arg, int* out, std::false_type, + std::true_type /* is_enum */) { + *out = ToIntVal(static_cast<typename std::underlying_type<T>::type>( + Manager<T>::Value(arg))); + return true; + } + + template <typename T> + static bool ToInt(Data, int*, std::false_type, std::false_type) { + return false; + } + + template <typename T> + static bool Dispatch(Data arg, FormatConversionSpecImpl spec, void* out) { + // A `none` conv indicates that we want the `int` conversion. + if (ABSL_PREDICT_FALSE(spec.conversion_char() == + FormatConversionCharInternal::kNone)) { + return ToInt<T>(arg, static_cast<int*>(out), std::is_integral<T>(), + std::is_enum<T>()); + } + if (ABSL_PREDICT_FALSE(!Contains(ArgumentToConv<T>(), + spec.conversion_char()))) { + return false; + } + return str_format_internal::FormatConvertImpl( + Manager<T>::Value(arg), spec, + static_cast<FormatSinkImpl*>(out)) + .value; + } + + Data data_; + Dispatcher dispatcher_; +}; + +#define ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(T, E) \ + E template bool FormatArgImpl::Dispatch<T>(Data, FormatConversionSpecImpl, \ + void*) + +#define ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(...) \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(str_format_internal::VoidPtr, \ + __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(bool, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(char, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(signed char, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(unsigned char, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(short, __VA_ARGS__); /* NOLINT */ \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(unsigned short, /* NOLINT */ \ + __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(int, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(unsigned int, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(long, __VA_ARGS__); /* NOLINT */ \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(unsigned long, /* NOLINT */ \ + __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(long long, /* NOLINT */ \ + __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(unsigned long long, /* NOLINT */ \ + __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(int128, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(uint128, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(float, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(double, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(long double, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(const char*, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(std::string, __VA_ARGS__); \ + ABSL_INTERNAL_FORMAT_DISPATCH_INSTANTIATE_(string_view, __VA_ARGS__) + +ABSL_INTERNAL_FORMAT_DISPATCH_OVERLOADS_EXPAND_(extern); + + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_ARG_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/arg_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/arg_test.cc new file mode 100644 index 000000000000..1261937c3097 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/arg_test.cc @@ -0,0 +1,130 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/arg.h" + +#include <ostream> +#include <string> +#include "gtest/gtest.h" +#include "absl/strings/str_format.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { +namespace { + +class FormatArgImplTest : public ::testing::Test { + public: + enum Color { kRed, kGreen, kBlue }; + + static const char *hi() { return "hi"; } + + struct X {}; + + X x_; +}; + +inline FormatConvertResult<FormatConversionCharSet{}> AbslFormatConvert( + const FormatArgImplTest::X &, const FormatConversionSpec &, FormatSink *) { + return {false}; +} + +TEST_F(FormatArgImplTest, ToInt) { + int out = 0; + EXPECT_TRUE(FormatArgImplFriend::ToInt(FormatArgImpl(1), &out)); + EXPECT_EQ(1, out); + EXPECT_TRUE(FormatArgImplFriend::ToInt(FormatArgImpl(-1), &out)); + EXPECT_EQ(-1, out); + EXPECT_TRUE( + FormatArgImplFriend::ToInt(FormatArgImpl(static_cast<char>(64)), &out)); + EXPECT_EQ(64, out); + EXPECT_TRUE(FormatArgImplFriend::ToInt( + FormatArgImpl(static_cast<unsigned long long>(123456)), &out)); // NOLINT + EXPECT_EQ(123456, out); + EXPECT_TRUE(FormatArgImplFriend::ToInt( + FormatArgImpl(static_cast<unsigned long long>( // NOLINT + std::numeric_limits<int>::max()) + + 1), + &out)); + EXPECT_EQ(std::numeric_limits<int>::max(), out); + EXPECT_TRUE(FormatArgImplFriend::ToInt( + FormatArgImpl(static_cast<long long>( // NOLINT + std::numeric_limits<int>::min()) - + 10), + &out)); + EXPECT_EQ(std::numeric_limits<int>::min(), out); + EXPECT_TRUE(FormatArgImplFriend::ToInt(FormatArgImpl(false), &out)); + EXPECT_EQ(0, out); + EXPECT_TRUE(FormatArgImplFriend::ToInt(FormatArgImpl(true), &out)); + EXPECT_EQ(1, out); + EXPECT_FALSE(FormatArgImplFriend::ToInt(FormatArgImpl(2.2), &out)); + EXPECT_FALSE(FormatArgImplFriend::ToInt(FormatArgImpl(3.2f), &out)); + EXPECT_FALSE(FormatArgImplFriend::ToInt( + FormatArgImpl(static_cast<int *>(nullptr)), &out)); + EXPECT_FALSE(FormatArgImplFriend::ToInt(FormatArgImpl(hi()), &out)); + EXPECT_FALSE(FormatArgImplFriend::ToInt(FormatArgImpl("hi"), &out)); + EXPECT_FALSE(FormatArgImplFriend::ToInt(FormatArgImpl(x_), &out)); + EXPECT_TRUE(FormatArgImplFriend::ToInt(FormatArgImpl(kBlue), &out)); + EXPECT_EQ(2, out); +} + +extern const char kMyArray[]; + +TEST_F(FormatArgImplTest, CharArraysDecayToCharPtr) { + const char* a = ""; + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(a)), + FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(""))); + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(a)), + FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl("A"))); + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(a)), + FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl("ABC"))); + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(a)), + FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(kMyArray))); +} + +TEST_F(FormatArgImplTest, OtherPtrDecayToVoidPtr) { + auto expected = FormatArgImplFriend::GetVTablePtrForTest( + FormatArgImpl(static_cast<void *>(nullptr))); + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest( + FormatArgImpl(static_cast<int *>(nullptr))), + expected); + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest( + FormatArgImpl(static_cast<volatile int *>(nullptr))), + expected); + + auto p = static_cast<void (*)()>([] {}); + EXPECT_EQ(FormatArgImplFriend::GetVTablePtrForTest(FormatArgImpl(p)), + expected); +} + +TEST_F(FormatArgImplTest, WorksWithCharArraysOfUnknownSize) { + std::string s; + FormatSinkImpl sink(&s); + FormatConversionSpecImpl conv; + FormatConversionSpecImplFriend::SetConversionChar( + FormatConversionCharInternal::s, &conv); + FormatConversionSpecImplFriend::SetFlags(Flags(), &conv); + FormatConversionSpecImplFriend::SetWidth(-1, &conv); + FormatConversionSpecImplFriend::SetPrecision(-1, &conv); + EXPECT_TRUE( + FormatArgImplFriend::Convert(FormatArgImpl(kMyArray), conv, &sink)); + sink.Flush(); + EXPECT_EQ("ABCDE", s); +} +const char kMyArray[] = "ABCDE"; + +} // namespace +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/bind.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/bind.cc new file mode 100644 index 000000000000..4e68b90b5ce8 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/bind.cc @@ -0,0 +1,259 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/bind.h" + +#include <cerrno> +#include <limits> +#include <sstream> +#include <string> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +namespace { + +inline bool BindFromPosition(int position, int* value, + absl::Span<const FormatArgImpl> pack) { + assert(position > 0); + if (static_cast<size_t>(position) > pack.size()) { + return false; + } + // -1 because positions are 1-based + return FormatArgImplFriend::ToInt(pack[position - 1], value); +} + +class ArgContext { + public: + explicit ArgContext(absl::Span<const FormatArgImpl> pack) : pack_(pack) {} + + // Fill 'bound' with the results of applying the context's argument pack + // to the specified 'unbound'. We synthesize a BoundConversion by + // lining up a UnboundConversion with a user argument. We also + // resolve any '*' specifiers for width and precision, so after + // this call, 'bound' has all the information it needs to be formatted. + // Returns false on failure. + bool Bind(const UnboundConversion* unbound, BoundConversion* bound); + + private: + absl::Span<const FormatArgImpl> pack_; +}; + +inline bool ArgContext::Bind(const UnboundConversion* unbound, + BoundConversion* bound) { + const FormatArgImpl* arg = nullptr; + int arg_position = unbound->arg_position; + if (static_cast<size_t>(arg_position - 1) >= pack_.size()) return false; + arg = &pack_[arg_position - 1]; // 1-based + + if (!unbound->flags.basic) { + int width = unbound->width.value(); + bool force_left = false; + if (unbound->width.is_from_arg()) { + if (!BindFromPosition(unbound->width.get_from_arg(), &width, pack_)) + return false; + if (width < 0) { + // "A negative field width is taken as a '-' flag followed by a + // positive field width." + force_left = true; + // Make sure we don't overflow the width when negating it. + width = -std::max(width, -std::numeric_limits<int>::max()); + } + } + + int precision = unbound->precision.value(); + if (unbound->precision.is_from_arg()) { + if (!BindFromPosition(unbound->precision.get_from_arg(), &precision, + pack_)) + return false; + } + + FormatConversionSpecImplFriend::SetWidth(width, bound); + FormatConversionSpecImplFriend::SetPrecision(precision, bound); + + if (force_left) { + Flags flags = unbound->flags; + flags.left = true; + FormatConversionSpecImplFriend::SetFlags(flags, bound); + } else { + FormatConversionSpecImplFriend::SetFlags(unbound->flags, bound); + } + } else { + FormatConversionSpecImplFriend::SetFlags(unbound->flags, bound); + FormatConversionSpecImplFriend::SetWidth(-1, bound); + FormatConversionSpecImplFriend::SetPrecision(-1, bound); + } + FormatConversionSpecImplFriend::SetConversionChar(unbound->conv, bound); + bound->set_arg(arg); + return true; +} + +template <typename Converter> +class ConverterConsumer { + public: + ConverterConsumer(Converter converter, absl::Span<const FormatArgImpl> pack) + : converter_(converter), arg_context_(pack) {} + + bool Append(string_view s) { + converter_.Append(s); + return true; + } + bool ConvertOne(const UnboundConversion& conv, string_view conv_string) { + BoundConversion bound; + if (!arg_context_.Bind(&conv, &bound)) return false; + return converter_.ConvertOne(bound, conv_string); + } + + private: + Converter converter_; + ArgContext arg_context_; +}; + +template <typename Converter> +bool ConvertAll(const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args, Converter converter) { + if (format.has_parsed_conversion()) { + return format.parsed_conversion()->ProcessFormat( + ConverterConsumer<Converter>(converter, args)); + } else { + return ParseFormatString(format.str(), + ConverterConsumer<Converter>(converter, args)); + } +} + +class DefaultConverter { + public: + explicit DefaultConverter(FormatSinkImpl* sink) : sink_(sink) {} + + void Append(string_view s) const { sink_->Append(s); } + + bool ConvertOne(const BoundConversion& bound, string_view /*conv*/) const { + return FormatArgImplFriend::Convert(*bound.arg(), bound, sink_); + } + + private: + FormatSinkImpl* sink_; +}; + +class SummarizingConverter { + public: + explicit SummarizingConverter(FormatSinkImpl* sink) : sink_(sink) {} + + void Append(string_view s) const { sink_->Append(s); } + + bool ConvertOne(const BoundConversion& bound, string_view /*conv*/) const { + UntypedFormatSpecImpl spec("%d"); + + std::ostringstream ss; + ss << "{" << Streamable(spec, {*bound.arg()}) << ":" + << FormatConversionSpecImplFriend::FlagsToString(bound); + if (bound.width() >= 0) ss << bound.width(); + if (bound.precision() >= 0) ss << "." << bound.precision(); + ss << bound.conversion_char() << "}"; + Append(ss.str()); + return true; + } + + private: + FormatSinkImpl* sink_; +}; + +} // namespace + +bool BindWithPack(const UnboundConversion* props, + absl::Span<const FormatArgImpl> pack, + BoundConversion* bound) { + return ArgContext(pack).Bind(props, bound); +} + +std::string Summarize(const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args) { + typedef SummarizingConverter Converter; + std::string out; + { + // inner block to destroy sink before returning out. It ensures a last + // flush. + FormatSinkImpl sink(&out); + if (!ConvertAll(format, args, Converter(&sink))) { + return ""; + } + } + return out; +} + +bool FormatUntyped(FormatRawSinkImpl raw_sink, + const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args) { + FormatSinkImpl sink(raw_sink); + using Converter = DefaultConverter; + return ConvertAll(format, args, Converter(&sink)); +} + +std::ostream& Streamable::Print(std::ostream& os) const { + if (!FormatUntyped(&os, format_, args_)) os.setstate(std::ios::failbit); + return os; +} + +std::string& AppendPack(std::string* out, const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args) { + size_t orig = out->size(); + if (ABSL_PREDICT_FALSE(!FormatUntyped(out, format, args))) { + out->erase(orig); + } + return *out; +} + +std::string FormatPack(const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args) { + std::string out; + if (ABSL_PREDICT_FALSE(!FormatUntyped(&out, format, args))) { + out.clear(); + } + return out; +} + +int FprintF(std::FILE* output, const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args) { + FILERawSink sink(output); + if (!FormatUntyped(&sink, format, args)) { + errno = EINVAL; + return -1; + } + if (sink.error()) { + errno = sink.error(); + return -1; + } + if (sink.count() > static_cast<size_t>(std::numeric_limits<int>::max())) { + errno = EFBIG; + return -1; + } + return static_cast<int>(sink.count()); +} + +int SnprintF(char* output, size_t size, const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args) { + BufferRawSink sink(output, size ? size - 1 : 0); + if (!FormatUntyped(&sink, format, args)) { + errno = EINVAL; + return -1; + } + size_t total = sink.total_written(); + if (size) output[std::min(total, size - 1)] = 0; + return static_cast<int>(total); +} + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/bind.h b/third_party/abseil_cpp/absl/strings/internal/str_format/bind.h new file mode 100644 index 000000000000..267cc0ef6928 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/bind.h @@ -0,0 +1,217 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_BIND_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_BIND_H_ + +#include <array> +#include <cstdio> +#include <sstream> +#include <string> + +#include "absl/base/port.h" +#include "absl/strings/internal/str_format/arg.h" +#include "absl/strings/internal/str_format/checker.h" +#include "absl/strings/internal/str_format/parser.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class UntypedFormatSpec; + +namespace str_format_internal { + +class BoundConversion : public FormatConversionSpecImpl { + public: + const FormatArgImpl* arg() const { return arg_; } + void set_arg(const FormatArgImpl* a) { arg_ = a; } + + private: + const FormatArgImpl* arg_; +}; + +// This is the type-erased class that the implementation uses. +class UntypedFormatSpecImpl { + public: + UntypedFormatSpecImpl() = delete; + + explicit UntypedFormatSpecImpl(string_view s) + : data_(s.data()), size_(s.size()) {} + explicit UntypedFormatSpecImpl( + const str_format_internal::ParsedFormatBase* pc) + : data_(pc), size_(~size_t{}) {} + + bool has_parsed_conversion() const { return size_ == ~size_t{}; } + + string_view str() const { + assert(!has_parsed_conversion()); + return string_view(static_cast<const char*>(data_), size_); + } + const str_format_internal::ParsedFormatBase* parsed_conversion() const { + assert(has_parsed_conversion()); + return static_cast<const str_format_internal::ParsedFormatBase*>(data_); + } + + template <typename T> + static const UntypedFormatSpecImpl& Extract(const T& s) { + return s.spec_; + } + + private: + const void* data_; + size_t size_; +}; + +template <typename T, FormatConversionCharSet...> +struct MakeDependent { + using type = T; +}; + +// Implicitly convertible from `const char*`, `string_view`, and the +// `ExtendedParsedFormat` type. This abstraction allows all format functions to +// operate on any without providing too many overloads. +template <FormatConversionCharSet... Args> +class FormatSpecTemplate + : public MakeDependent<UntypedFormatSpec, Args...>::type { + using Base = typename MakeDependent<UntypedFormatSpec, Args...>::type; + + public: +#ifdef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + + // Honeypot overload for when the string is not constexpr. + // We use the 'unavailable' attribute to give a better compiler error than + // just 'method is deleted'. + FormatSpecTemplate(...) // NOLINT + __attribute__((unavailable("Format string is not constexpr."))); + + // Honeypot overload for when the format is constexpr and invalid. + // We use the 'unavailable' attribute to give a better compiler error than + // just 'method is deleted'. + // To avoid checking the format twice, we just check that the format is + // constexpr. If is it valid, then the overload below will kick in. + // We add the template here to make this overload have lower priority. + template <typename = void> + FormatSpecTemplate(const char* s) // NOLINT + __attribute__(( + enable_if(str_format_internal::EnsureConstexpr(s), "constexpr trap"), + unavailable( + "Format specified does not match the arguments passed."))); + + template <typename T = void> + FormatSpecTemplate(string_view s) // NOLINT + __attribute__((enable_if(str_format_internal::EnsureConstexpr(s), + "constexpr trap"))) { + static_assert(sizeof(T*) == 0, + "Format specified does not match the arguments passed."); + } + + // Good format overload. + FormatSpecTemplate(const char* s) // NOLINT + __attribute__((enable_if(ValidFormatImpl<Args...>(s), "bad format trap"))) + : Base(s) {} + + FormatSpecTemplate(string_view s) // NOLINT + __attribute__((enable_if(ValidFormatImpl<Args...>(s), "bad format trap"))) + : Base(s) {} + +#else // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + + FormatSpecTemplate(const char* s) : Base(s) {} // NOLINT + FormatSpecTemplate(string_view s) : Base(s) {} // NOLINT + +#endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + + template < + FormatConversionCharSet... C, + typename = typename std::enable_if<sizeof...(C) == sizeof...(Args)>::type, + typename = typename std::enable_if<AllOf(Contains(Args, + C)...)>::type> + FormatSpecTemplate(const ExtendedParsedFormat<C...>& pc) // NOLINT + : Base(&pc) {} +}; + +class Streamable { + public: + Streamable(const UntypedFormatSpecImpl& format, + absl::Span<const FormatArgImpl> args) + : format_(format) { + if (args.size() <= ABSL_ARRAYSIZE(few_args_)) { + for (size_t i = 0; i < args.size(); ++i) { + few_args_[i] = args[i]; + } + args_ = absl::MakeSpan(few_args_, args.size()); + } else { + many_args_.assign(args.begin(), args.end()); + args_ = many_args_; + } + } + + std::ostream& Print(std::ostream& os) const; + + friend std::ostream& operator<<(std::ostream& os, const Streamable& l) { + return l.Print(os); + } + + private: + const UntypedFormatSpecImpl& format_; + absl::Span<const FormatArgImpl> args_; + // if args_.size() is 4 or less: + FormatArgImpl few_args_[4] = {FormatArgImpl(0), FormatArgImpl(0), + FormatArgImpl(0), FormatArgImpl(0)}; + // if args_.size() is more than 4: + std::vector<FormatArgImpl> many_args_; +}; + +// for testing +std::string Summarize(UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args); +bool BindWithPack(const UnboundConversion* props, + absl::Span<const FormatArgImpl> pack, BoundConversion* bound); + +bool FormatUntyped(FormatRawSinkImpl raw_sink, + UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args); + +std::string& AppendPack(std::string* out, UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args); + +std::string FormatPack(const UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args); + +int FprintF(std::FILE* output, UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args); +int SnprintF(char* output, size_t size, UntypedFormatSpecImpl format, + absl::Span<const FormatArgImpl> args); + +// Returned by Streamed(v). Converts via '%s' to the std::string created +// by std::ostream << v. +template <typename T> +class StreamedWrapper { + public: + explicit StreamedWrapper(const T& v) : v_(v) { } + + private: + template <typename S> + friend ArgConvertResult<FormatConversionCharSetInternal::s> FormatConvertImpl( + const StreamedWrapper<S>& v, FormatConversionSpecImpl conv, + FormatSinkImpl* out); + const T& v_; +}; + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_BIND_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/bind_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/bind_test.cc new file mode 100644 index 000000000000..1eef9c4326e2 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/bind_test.cc @@ -0,0 +1,157 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/bind.h" + +#include <string.h> +#include <limits> + +#include "gtest/gtest.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { +namespace { + +class FormatBindTest : public ::testing::Test { + public: + bool Extract(const char *s, UnboundConversion *props, int *next) const { + return ConsumeUnboundConversion(s, s + strlen(s), props, next) == + s + strlen(s); + } +}; + +TEST_F(FormatBindTest, BindSingle) { + struct Expectation { + int line; + const char *fmt; + int ok_phases; + const FormatArgImpl *arg; + int width; + int precision; + int next_arg; + }; + const int no = -1; + const int ia[] = { 10, 20, 30, 40}; + const FormatArgImpl args[] = {FormatArgImpl(ia[0]), FormatArgImpl(ia[1]), + FormatArgImpl(ia[2]), FormatArgImpl(ia[3])}; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmissing-field-initializers" + const Expectation kExpect[] = { + {__LINE__, "d", 2, &args[0], no, no, 2}, + {__LINE__, "4d", 2, &args[0], 4, no, 2}, + {__LINE__, ".5d", 2, &args[0], no, 5, 2}, + {__LINE__, "4.5d", 2, &args[0], 4, 5, 2}, + {__LINE__, "*d", 2, &args[1], 10, no, 3}, + {__LINE__, ".*d", 2, &args[1], no, 10, 3}, + {__LINE__, "*.*d", 2, &args[2], 10, 20, 4}, + {__LINE__, "1$d", 2, &args[0], no, no, 0}, + {__LINE__, "2$d", 2, &args[1], no, no, 0}, + {__LINE__, "3$d", 2, &args[2], no, no, 0}, + {__LINE__, "4$d", 2, &args[3], no, no, 0}, + {__LINE__, "2$*1$d", 2, &args[1], 10, no, 0}, + {__LINE__, "2$*2$d", 2, &args[1], 20, no, 0}, + {__LINE__, "2$*3$d", 2, &args[1], 30, no, 0}, + {__LINE__, "2$.*1$d", 2, &args[1], no, 10, 0}, + {__LINE__, "2$.*2$d", 2, &args[1], no, 20, 0}, + {__LINE__, "2$.*3$d", 2, &args[1], no, 30, 0}, + {__LINE__, "2$*3$.*1$d", 2, &args[1], 30, 10, 0}, + {__LINE__, "2$*2$.*2$d", 2, &args[1], 20, 20, 0}, + {__LINE__, "2$*1$.*3$d", 2, &args[1], 10, 30, 0}, + {__LINE__, "2$*3$.*1$d", 2, &args[1], 30, 10, 0}, + {__LINE__, "1$*d", 0}, // indexed, then positional + {__LINE__, "*2$d", 0}, // positional, then indexed + {__LINE__, "6$d", 1}, // arg position out of bounds + {__LINE__, "1$6$d", 0}, // width position incorrectly specified + {__LINE__, "1$.6$d", 0}, // precision position incorrectly specified + {__LINE__, "1$*6$d", 1}, // width position out of bounds + {__LINE__, "1$.*6$d", 1}, // precision position out of bounds + }; +#pragma GCC diagnostic pop + for (const Expectation &e : kExpect) { + SCOPED_TRACE(e.line); + SCOPED_TRACE(e.fmt); + UnboundConversion props; + BoundConversion bound; + int ok_phases = 0; + int next = 0; + if (Extract(e.fmt, &props, &next)) { + ++ok_phases; + if (BindWithPack(&props, args, &bound)) { + ++ok_phases; + } + } + EXPECT_EQ(e.ok_phases, ok_phases); + if (e.ok_phases < 2) continue; + if (e.arg != nullptr) { + EXPECT_EQ(e.arg, bound.arg()); + } + EXPECT_EQ(e.width, bound.width()); + EXPECT_EQ(e.precision, bound.precision()); + } +} + +TEST_F(FormatBindTest, WidthUnderflowRegression) { + UnboundConversion props; + BoundConversion bound; + int next = 0; + const int args_i[] = {std::numeric_limits<int>::min(), 17}; + const FormatArgImpl args[] = {FormatArgImpl(args_i[0]), + FormatArgImpl(args_i[1])}; + ASSERT_TRUE(Extract("*d", &props, &next)); + ASSERT_TRUE(BindWithPack(&props, args, &bound)); + + EXPECT_EQ(bound.width(), std::numeric_limits<int>::max()); + EXPECT_EQ(bound.arg(), args + 1); +} + +TEST_F(FormatBindTest, FormatPack) { + struct Expectation { + int line; + const char *fmt; + const char *summary; + }; + const int ia[] = { 10, 20, 30, 40, -10 }; + const FormatArgImpl args[] = {FormatArgImpl(ia[0]), FormatArgImpl(ia[1]), + FormatArgImpl(ia[2]), FormatArgImpl(ia[3]), + FormatArgImpl(ia[4])}; + const Expectation kExpect[] = { + {__LINE__, "a%4db%dc", "a{10:4d}b{20:d}c"}, + {__LINE__, "a%.4db%dc", "a{10:.4d}b{20:d}c"}, + {__LINE__, "a%4.5db%dc", "a{10:4.5d}b{20:d}c"}, + {__LINE__, "a%db%4.5dc", "a{10:d}b{20:4.5d}c"}, + {__LINE__, "a%db%*.*dc", "a{10:d}b{40:20.30d}c"}, + {__LINE__, "a%.*fb", "a{20:.10f}b"}, + {__LINE__, "a%1$db%2$*3$.*4$dc", "a{10:d}b{20:30.40d}c"}, + {__LINE__, "a%4$db%3$*2$.*1$dc", "a{40:d}b{30:20.10d}c"}, + {__LINE__, "a%04ldb", "a{10:04d}b"}, + {__LINE__, "a%-#04lldb", "a{10:-#04d}b"}, + {__LINE__, "a%1$*5$db", "a{10:-10d}b"}, + {__LINE__, "a%1$.*5$db", "a{10:d}b"}, + }; + for (const Expectation &e : kExpect) { + absl::string_view fmt = e.fmt; + SCOPED_TRACE(e.line); + SCOPED_TRACE(e.fmt); + UntypedFormatSpecImpl format(fmt); + EXPECT_EQ(e.summary, + str_format_internal::Summarize(format, absl::MakeSpan(args))) + << "line:" << e.line; + } +} + +} // namespace +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/checker.h b/third_party/abseil_cpp/absl/strings/internal/str_format/checker.h new file mode 100644 index 000000000000..2a2601eccfd8 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/checker.h @@ -0,0 +1,333 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_CHECKER_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_CHECKER_H_ + +#include "absl/base/attributes.h" +#include "absl/strings/internal/str_format/arg.h" +#include "absl/strings/internal/str_format/extension.h" + +// Compile time check support for entry points. + +#ifndef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER +#if ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(__native_client__) +#define ABSL_INTERNAL_ENABLE_FORMAT_CHECKER 1 +#endif // ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(__native_client__) +#endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +constexpr bool AllOf() { return true; } + +template <typename... T> +constexpr bool AllOf(bool b, T... t) { + return b && AllOf(t...); +} + +#ifdef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + +constexpr bool ContainsChar(const char* chars, char c) { + return *chars == c || (*chars && ContainsChar(chars + 1, c)); +} + +// A constexpr compatible list of Convs. +struct ConvList { + const FormatConversionCharSet* array; + int count; + + // We do the bound check here to avoid having to do it on the callers. + // Returning an empty FormatConversionCharSet has the same effect as + // short circuiting because it will never match any conversion. + constexpr FormatConversionCharSet operator[](int i) const { + return i < count ? array[i] : FormatConversionCharSet{}; + } + + constexpr ConvList without_front() const { + return count != 0 ? ConvList{array + 1, count - 1} : *this; + } +}; + +template <size_t count> +struct ConvListT { + // Make sure the array has size > 0. + FormatConversionCharSet list[count ? count : 1]; +}; + +constexpr char GetChar(string_view str, size_t index) { + return index < str.size() ? str[index] : char{}; +} + +constexpr string_view ConsumeFront(string_view str, size_t len = 1) { + return len <= str.size() ? string_view(str.data() + len, str.size() - len) + : string_view(); +} + +constexpr string_view ConsumeAnyOf(string_view format, const char* chars) { + return ContainsChar(chars, GetChar(format, 0)) + ? ConsumeAnyOf(ConsumeFront(format), chars) + : format; +} + +constexpr bool IsDigit(char c) { return c >= '0' && c <= '9'; } + +// Helper class for the ParseDigits function. +// It encapsulates the two return values we need there. +struct Integer { + string_view format; + int value; + + // If the next character is a '$', consume it. + // Otherwise, make `this` an invalid positional argument. + constexpr Integer ConsumePositionalDollar() const { + return GetChar(format, 0) == '$' ? Integer{ConsumeFront(format), value} + : Integer{format, 0}; + } +}; + +constexpr Integer ParseDigits(string_view format, int value = 0) { + return IsDigit(GetChar(format, 0)) + ? ParseDigits(ConsumeFront(format), + 10 * value + GetChar(format, 0) - '0') + : Integer{format, value}; +} + +// Parse digits for a positional argument. +// The parsing also consumes the '$'. +constexpr Integer ParsePositional(string_view format) { + return ParseDigits(format).ConsumePositionalDollar(); +} + +// Parses a single conversion specifier. +// See ConvParser::Run() for post conditions. +class ConvParser { + constexpr ConvParser SetFormat(string_view format) const { + return ConvParser(format, args_, error_, arg_position_, is_positional_); + } + + constexpr ConvParser SetArgs(ConvList args) const { + return ConvParser(format_, args, error_, arg_position_, is_positional_); + } + + constexpr ConvParser SetError(bool error) const { + return ConvParser(format_, args_, error_ || error, arg_position_, + is_positional_); + } + + constexpr ConvParser SetArgPosition(int arg_position) const { + return ConvParser(format_, args_, error_, arg_position, is_positional_); + } + + // Consumes the next arg and verifies that it matches `conv`. + // `error_` is set if there is no next arg or if it doesn't match `conv`. + constexpr ConvParser ConsumeNextArg(char conv) const { + return SetArgs(args_.without_front()).SetError(!Contains(args_[0], conv)); + } + + // Verify that positional argument `i.value` matches `conv`. + // `error_` is set if `i.value` is not a valid argument or if it doesn't + // match. + constexpr ConvParser VerifyPositional(Integer i, char conv) const { + return SetFormat(i.format).SetError(!Contains(args_[i.value - 1], conv)); + } + + // Parse the position of the arg and store it in `arg_position_`. + constexpr ConvParser ParseArgPosition(Integer arg) const { + return SetFormat(arg.format).SetArgPosition(arg.value); + } + + // Consume the flags. + constexpr ConvParser ParseFlags() const { + return SetFormat(ConsumeAnyOf(format_, "-+ #0")); + } + + // Consume the width. + // If it is '*', we verify that it matches `args_`. `error_` is set if it + // doesn't match. + constexpr ConvParser ParseWidth() const { + return IsDigit(GetChar(format_, 0)) + ? SetFormat(ParseDigits(format_).format) + : GetChar(format_, 0) == '*' + ? is_positional_ + ? VerifyPositional( + ParsePositional(ConsumeFront(format_)), '*') + : SetFormat(ConsumeFront(format_)) + .ConsumeNextArg('*') + : *this; + } + + // Consume the precision. + // If it is '*', we verify that it matches `args_`. `error_` is set if it + // doesn't match. + constexpr ConvParser ParsePrecision() const { + return GetChar(format_, 0) != '.' + ? *this + : GetChar(format_, 1) == '*' + ? is_positional_ + ? VerifyPositional( + ParsePositional(ConsumeFront(format_, 2)), '*') + : SetFormat(ConsumeFront(format_, 2)) + .ConsumeNextArg('*') + : SetFormat(ParseDigits(ConsumeFront(format_)).format); + } + + // Consume the length characters. + constexpr ConvParser ParseLength() const { + return SetFormat(ConsumeAnyOf(format_, "lLhjztq")); + } + + // Consume the conversion character and verify that it matches `args_`. + // `error_` is set if it doesn't match. + constexpr ConvParser ParseConversion() const { + return is_positional_ + ? VerifyPositional({ConsumeFront(format_), arg_position_}, + GetChar(format_, 0)) + : ConsumeNextArg(GetChar(format_, 0)) + .SetFormat(ConsumeFront(format_)); + } + + constexpr ConvParser(string_view format, ConvList args, bool error, + int arg_position, bool is_positional) + : format_(format), + args_(args), + error_(error), + arg_position_(arg_position), + is_positional_(is_positional) {} + + public: + constexpr ConvParser(string_view format, ConvList args, bool is_positional) + : format_(format), + args_(args), + error_(false), + arg_position_(0), + is_positional_(is_positional) {} + + // Consume the whole conversion specifier. + // `format()` will be set to the character after the conversion character. + // `error()` will be set if any of the arguments do not match. + constexpr ConvParser Run() const { + return (is_positional_ ? ParseArgPosition(ParsePositional(format_)) : *this) + .ParseFlags() + .ParseWidth() + .ParsePrecision() + .ParseLength() + .ParseConversion(); + } + + constexpr string_view format() const { return format_; } + constexpr ConvList args() const { return args_; } + constexpr bool error() const { return error_; } + constexpr bool is_positional() const { return is_positional_; } + + private: + string_view format_; + // Current list of arguments. If we are not in positional mode we will consume + // from the front. + ConvList args_; + bool error_; + // Holds the argument position of the conversion character, if we are in + // positional mode. Otherwise, it is unspecified. + int arg_position_; + // Whether we are in positional mode. + // It changes the behavior of '*' and where to find the converted argument. + bool is_positional_; +}; + +// Parses a whole format expression. +// See FormatParser::Run(). +class FormatParser { + static constexpr bool FoundPercent(string_view format) { + return format.empty() || + (GetChar(format, 0) == '%' && GetChar(format, 1) != '%'); + } + + // We use an inner function to increase the recursion limit. + // The inner function consumes up to `limit` characters on every run. + // This increases the limit from 512 to ~512*limit. + static constexpr string_view ConsumeNonPercentInner(string_view format, + int limit = 20) { + return FoundPercent(format) || !limit + ? format + : ConsumeNonPercentInner( + ConsumeFront(format, GetChar(format, 0) == '%' && + GetChar(format, 1) == '%' + ? 2 + : 1), + limit - 1); + } + + // Consume characters until the next conversion spec %. + // It skips %%. + static constexpr string_view ConsumeNonPercent(string_view format) { + return FoundPercent(format) + ? format + : ConsumeNonPercent(ConsumeNonPercentInner(format)); + } + + static constexpr bool IsPositional(string_view format) { + return IsDigit(GetChar(format, 0)) ? IsPositional(ConsumeFront(format)) + : GetChar(format, 0) == '$'; + } + + constexpr bool RunImpl(bool is_positional) const { + // In non-positional mode we require all arguments to be consumed. + // In positional mode just reaching the end of the format without errors is + // enough. + return (format_.empty() && (is_positional || args_.count == 0)) || + (!format_.empty() && + ValidateArg( + ConvParser(ConsumeFront(format_), args_, is_positional).Run())); + } + + constexpr bool ValidateArg(ConvParser conv) const { + return !conv.error() && FormatParser(conv.format(), conv.args()) + .RunImpl(conv.is_positional()); + } + + public: + constexpr FormatParser(string_view format, ConvList args) + : format_(ConsumeNonPercent(format)), args_(args) {} + + // Runs the parser for `format` and `args`. + // It verifies that the format is valid and that all conversion specifiers + // match the arguments passed. + // In non-positional mode it also verfies that all arguments are consumed. + constexpr bool Run() const { + return RunImpl(!format_.empty() && IsPositional(ConsumeFront(format_))); + } + + private: + string_view format_; + // Current list of arguments. + // If we are not in positional mode we will consume from the front and will + // have to be empty in the end. + ConvList args_; +}; + +template <FormatConversionCharSet... C> +constexpr bool ValidFormatImpl(string_view format) { + return FormatParser(format, + {ConvListT<sizeof...(C)>{{C...}}.list, sizeof...(C)}) + .Run(); +} + +#endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_CHECKER_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/checker_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/checker_test.cc new file mode 100644 index 000000000000..7c70f47d682a --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/checker_test.cc @@ -0,0 +1,170 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/str_format.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { +namespace { + +std::string ConvToString(FormatConversionCharSet conv) { + std::string out; +#define CONV_SET_CASE(c) \ + if (Contains(conv, FormatConversionCharSetInternal::c)) { \ + out += #c; \ + } + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(CONV_SET_CASE, ) +#undef CONV_SET_CASE + if (Contains(conv, FormatConversionCharSetInternal::kStar)) { + out += "*"; + } + return out; +} + +TEST(StrFormatChecker, ArgumentToConv) { + FormatConversionCharSet conv = ArgumentToConv<std::string>(); + EXPECT_EQ(ConvToString(conv), "s"); + + conv = ArgumentToConv<const char*>(); + EXPECT_EQ(ConvToString(conv), "sp"); + + conv = ArgumentToConv<double>(); + EXPECT_EQ(ConvToString(conv), "fFeEgGaA"); + + conv = ArgumentToConv<int>(); + EXPECT_EQ(ConvToString(conv), "cdiouxXfFeEgGaA*"); + + conv = ArgumentToConv<std::string*>(); + EXPECT_EQ(ConvToString(conv), "p"); +} + +#ifdef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + +struct Case { + bool result; + const char* format; +}; + +template <typename... Args> +constexpr Case ValidFormat(const char* format) { + return {ValidFormatImpl<ArgumentToConv<Args>()...>(format), format}; +} + +TEST(StrFormatChecker, ValidFormat) { + // We want to make sure these expressions are constexpr and they have the + // expected value. + // If they are not constexpr the attribute will just ignore them and not give + // a compile time error. + enum e {}; + enum class e2 {}; + constexpr Case trues[] = { + ValidFormat<>("abc"), // + + ValidFormat<e>("%d"), // + ValidFormat<e2>("%d"), // + ValidFormat<int>("%% %d"), // + ValidFormat<int>("%ld"), // + ValidFormat<int>("%lld"), // + ValidFormat<std::string>("%s"), // + ValidFormat<std::string>("%10s"), // + ValidFormat<int>("%.10x"), // + ValidFormat<int, int>("%*.3x"), // + ValidFormat<int>("%1.d"), // + ValidFormat<int>("%.d"), // + ValidFormat<int, double>("%d %g"), // + ValidFormat<int, std::string>("%*s"), // + ValidFormat<int, double>("%.*f"), // + ValidFormat<void (*)(), volatile int*>("%p %p"), // + ValidFormat<string_view, const char*, double, void*>( + "string_view=%s const char*=%s double=%f void*=%p)"), + + ValidFormat<int>("%% %1$d"), // + ValidFormat<int>("%1$ld"), // + ValidFormat<int>("%1$lld"), // + ValidFormat<std::string>("%1$s"), // + ValidFormat<std::string>("%1$10s"), // + ValidFormat<int>("%1$.10x"), // + ValidFormat<int>("%1$*1$.*1$d"), // + ValidFormat<int, int>("%1$*2$.3x"), // + ValidFormat<int>("%1$1.d"), // + ValidFormat<int>("%1$.d"), // + ValidFormat<double, int>("%2$d %1$g"), // + ValidFormat<int, std::string>("%2$*1$s"), // + ValidFormat<int, double>("%2$.*1$f"), // + ValidFormat<void*, string_view, const char*, double>( + "string_view=%2$s const char*=%3$s double=%4$f void*=%1$p " + "repeat=%3$s)")}; + + for (Case c : trues) { + EXPECT_TRUE(c.result) << c.format; + } + + constexpr Case falses[] = { + ValidFormat<int>(""), // + + ValidFormat<e>("%s"), // + ValidFormat<e2>("%s"), // + ValidFormat<>("%s"), // + ValidFormat<>("%r"), // + ValidFormat<int>("%s"), // + ValidFormat<int>("%.1.d"), // + ValidFormat<int>("%*1d"), // + ValidFormat<int>("%1-d"), // + ValidFormat<std::string, int>("%*s"), // + ValidFormat<int>("%*d"), // + ValidFormat<std::string>("%p"), // + ValidFormat<int (*)(int)>("%d"), // + + ValidFormat<>("%3$d"), // + ValidFormat<>("%1$r"), // + ValidFormat<int>("%1$s"), // + ValidFormat<int>("%1$.1.d"), // + ValidFormat<int>("%1$*2$1d"), // + ValidFormat<int>("%1$1-d"), // + ValidFormat<std::string, int>("%2$*1$s"), // + ValidFormat<std::string>("%1$p"), + + ValidFormat<int, int>("%d %2$d"), // + }; + + for (Case c : falses) { + EXPECT_FALSE(c.result) << c.format; + } +} + +TEST(StrFormatChecker, LongFormat) { +#define CHARS_X_40 "1234567890123456789012345678901234567890" +#define CHARS_X_400 \ + CHARS_X_40 CHARS_X_40 CHARS_X_40 CHARS_X_40 CHARS_X_40 CHARS_X_40 CHARS_X_40 \ + CHARS_X_40 CHARS_X_40 CHARS_X_40 +#define CHARS_X_4000 \ + CHARS_X_400 CHARS_X_400 CHARS_X_400 CHARS_X_400 CHARS_X_400 CHARS_X_400 \ + CHARS_X_400 CHARS_X_400 CHARS_X_400 CHARS_X_400 + constexpr char long_format[] = + CHARS_X_4000 "%d" CHARS_X_4000 "%s" CHARS_X_4000; + constexpr bool is_valid = ValidFormat<int, std::string>(long_format).result; + EXPECT_TRUE(is_valid); +} + +#endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + +} // namespace +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/convert_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/convert_test.cc new file mode 100644 index 000000000000..375db0a0592c --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/convert_test.cc @@ -0,0 +1,1242 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <errno.h> +#include <stdarg.h> +#include <stdio.h> + +#include <cctype> +#include <cmath> +#include <limits> +#include <string> +#include <thread> // NOLINT + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/internal/str_format/bind.h" +#include "absl/strings/match.h" +#include "absl/types/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { +namespace { + +struct NativePrintfTraits { + bool hex_float_has_glibc_rounding; + bool hex_float_prefers_denormal_repr; + bool hex_float_uses_minimal_precision_when_not_specified; + bool hex_float_optimizes_leading_digit_bit_count; +}; + +template <typename T, size_t N> +size_t ArraySize(T (&)[N]) { + return N; +} + +std::string LengthModFor(float) { return ""; } +std::string LengthModFor(double) { return ""; } +std::string LengthModFor(long double) { return "L"; } +std::string LengthModFor(char) { return "hh"; } +std::string LengthModFor(signed char) { return "hh"; } +std::string LengthModFor(unsigned char) { return "hh"; } +std::string LengthModFor(short) { return "h"; } // NOLINT +std::string LengthModFor(unsigned short) { return "h"; } // NOLINT +std::string LengthModFor(int) { return ""; } +std::string LengthModFor(unsigned) { return ""; } +std::string LengthModFor(long) { return "l"; } // NOLINT +std::string LengthModFor(unsigned long) { return "l"; } // NOLINT +std::string LengthModFor(long long) { return "ll"; } // NOLINT +std::string LengthModFor(unsigned long long) { return "ll"; } // NOLINT + +std::string EscCharImpl(int v) { + if (std::isprint(static_cast<unsigned char>(v))) { + return std::string(1, static_cast<char>(v)); + } + char buf[64]; + int n = snprintf(buf, sizeof(buf), "\\%#.2x", + static_cast<unsigned>(v & 0xff)); + assert(n > 0 && n < sizeof(buf)); + return std::string(buf, n); +} + +std::string Esc(char v) { return EscCharImpl(v); } +std::string Esc(signed char v) { return EscCharImpl(v); } +std::string Esc(unsigned char v) { return EscCharImpl(v); } + +template <typename T> +std::string Esc(const T &v) { + std::ostringstream oss; + oss << v; + return oss.str(); +} + +void StrAppendV(std::string *dst, const char *format, va_list ap) { + // First try with a small fixed size buffer + static const int kSpaceLength = 1024; + char space[kSpaceLength]; + + // It's possible for methods that use a va_list to invalidate + // the data in it upon use. The fix is to make a copy + // of the structure before using it and use that copy instead. + va_list backup_ap; + va_copy(backup_ap, ap); + int result = vsnprintf(space, kSpaceLength, format, backup_ap); + va_end(backup_ap); + if (result < kSpaceLength) { + if (result >= 0) { + // Normal case -- everything fit. + dst->append(space, result); + return; + } + if (result < 0) { + // Just an error. + return; + } + } + + // Increase the buffer size to the size requested by vsnprintf, + // plus one for the closing \0. + int length = result + 1; + char *buf = new char[length]; + + // Restore the va_list before we use it again + va_copy(backup_ap, ap); + result = vsnprintf(buf, length, format, backup_ap); + va_end(backup_ap); + + if (result >= 0 && result < length) { + // It fit + dst->append(buf, result); + } + delete[] buf; +} + +void StrAppend(std::string *out, const char *format, ...) { + va_list ap; + va_start(ap, format); + StrAppendV(out, format, ap); + va_end(ap); +} + +std::string StrPrint(const char *format, ...) { + va_list ap; + va_start(ap, format); + std::string result; + StrAppendV(&result, format, ap); + va_end(ap); + return result; +} + +NativePrintfTraits VerifyNativeImplementationImpl() { + NativePrintfTraits result; + + // >>> hex_float_has_glibc_rounding. To have glibc's rounding behavior we need + // to meet three requirements: + // + // - The threshold for rounding up is 8 (for e.g. MSVC uses 9). + // - If the digits lower than than the 8 are non-zero then we round up. + // - If the digits lower than the 8 are all zero then we round toward even. + // + // The numbers below represent all the cases covering {below,at,above} the + // threshold (8) with both {zero,non-zero} lower bits and both {even,odd} + // preceding digits. + const double d0079 = 65657.0; // 0x1.0079p+16 + const double d0179 = 65913.0; // 0x1.0179p+16 + const double d0080 = 65664.0; // 0x1.0080p+16 + const double d0180 = 65920.0; // 0x1.0180p+16 + const double d0081 = 65665.0; // 0x1.0081p+16 + const double d0181 = 65921.0; // 0x1.0181p+16 + result.hex_float_has_glibc_rounding = + StartsWith(StrPrint("%.2a", d0079), "0x1.00") && + StartsWith(StrPrint("%.2a", d0179), "0x1.01") && + StartsWith(StrPrint("%.2a", d0080), "0x1.00") && + StartsWith(StrPrint("%.2a", d0180), "0x1.02") && + StartsWith(StrPrint("%.2a", d0081), "0x1.01") && + StartsWith(StrPrint("%.2a", d0181), "0x1.02"); + + // >>> hex_float_prefers_denormal_repr. Formatting `denormal` on glibc yields + // "0x0.0000000000001p-1022", whereas on std libs that don't use denormal + // representation it would either be 0x1p-1074 or 0x1.0000000000000-1074. + const double denormal = std::numeric_limits<double>::denorm_min(); + result.hex_float_prefers_denormal_repr = + StartsWith(StrPrint("%a", denormal), "0x0.0000000000001"); + + // >>> hex_float_uses_minimal_precision_when_not_specified. Some (non-glibc) + // libs will format the following as "0x1.0079000000000p+16". + result.hex_float_uses_minimal_precision_when_not_specified = + (StrPrint("%a", d0079) == "0x1.0079p+16"); + + // >>> hex_float_optimizes_leading_digit_bit_count. The number 1.5, when + // formatted by glibc should yield "0x1.8p+0" for `double` and "0xcp-3" for + // `long double`, i.e., number of bits in the leading digit is adapted to the + // number of bits in the mantissa. + const double d_15 = 1.5; + const long double ld_15 = 1.5; + result.hex_float_optimizes_leading_digit_bit_count = + StartsWith(StrPrint("%a", d_15), "0x1.8") && + StartsWith(StrPrint("%La", ld_15), "0xc"); + + return result; +} + +const NativePrintfTraits &VerifyNativeImplementation() { + static NativePrintfTraits native_traits = VerifyNativeImplementationImpl(); + return native_traits; +} + +class FormatConvertTest : public ::testing::Test { }; + +template <typename T> +void TestStringConvert(const T& str) { + const FormatArgImpl args[] = {FormatArgImpl(str)}; + struct Expectation { + const char *out; + const char *fmt; + }; + const Expectation kExpect[] = { + {"hello", "%1$s" }, + {"", "%1$.s" }, + {"", "%1$.0s" }, + {"h", "%1$.1s" }, + {"he", "%1$.2s" }, + {"hello", "%1$.10s" }, + {" hello", "%1$6s" }, + {" he", "%1$5.2s" }, + {"he ", "%1$-5.2s" }, + {"hello ", "%1$-6.10s" }, + }; + for (const Expectation &e : kExpect) { + UntypedFormatSpecImpl format(e.fmt); + EXPECT_EQ(e.out, FormatPack(format, absl::MakeSpan(args))); + } +} + +TEST_F(FormatConvertTest, BasicString) { + TestStringConvert("hello"); // As char array. + TestStringConvert(static_cast<const char*>("hello")); + TestStringConvert(std::string("hello")); + TestStringConvert(string_view("hello")); +} + +TEST_F(FormatConvertTest, NullString) { + const char* p = nullptr; + UntypedFormatSpecImpl format("%s"); + EXPECT_EQ("", FormatPack(format, {FormatArgImpl(p)})); +} + +TEST_F(FormatConvertTest, StringPrecision) { + // We cap at the precision. + char c = 'a'; + const char* p = &c; + UntypedFormatSpecImpl format("%.1s"); + EXPECT_EQ("a", FormatPack(format, {FormatArgImpl(p)})); + + // We cap at the NUL-terminator. + p = "ABC"; + UntypedFormatSpecImpl format2("%.10s"); + EXPECT_EQ("ABC", FormatPack(format2, {FormatArgImpl(p)})); +} + +// Pointer formatting is implementation defined. This checks that the argument +// can be matched to `ptr`. +MATCHER_P(MatchesPointerString, ptr, "") { + if (ptr == nullptr && arg == "(nil)") { + return true; + } + void* parsed = nullptr; + if (sscanf(arg.c_str(), "%p", &parsed) != 1) { + ABSL_RAW_LOG(FATAL, "Could not parse %s", arg.c_str()); + } + return ptr == parsed; +} + +TEST_F(FormatConvertTest, Pointer) { + static int x = 0; + const int *xp = &x; + char c = 'h'; + char *mcp = &c; + const char *cp = "hi"; + const char *cnil = nullptr; + const int *inil = nullptr; + using VoidF = void (*)(); + VoidF fp = [] {}, fnil = nullptr; + volatile char vc; + volatile char *vcp = &vc; + volatile char *vcnil = nullptr; + const FormatArgImpl args_array[] = { + FormatArgImpl(xp), FormatArgImpl(cp), FormatArgImpl(inil), + FormatArgImpl(cnil), FormatArgImpl(mcp), FormatArgImpl(fp), + FormatArgImpl(fnil), FormatArgImpl(vcp), FormatArgImpl(vcnil), + }; + auto args = absl::MakeConstSpan(args_array); + + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%20p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%.1p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%.20p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%30.20p"), args), + MatchesPointerString(&x)); + + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%-p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%-20p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%-.1p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%.20p"), args), + MatchesPointerString(&x)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%-30.20p"), args), + MatchesPointerString(&x)); + + // const char* + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%2$p"), args), + MatchesPointerString(cp)); + // null const int* + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%3$p"), args), + MatchesPointerString(nullptr)); + // null const char* + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%4$p"), args), + MatchesPointerString(nullptr)); + // nonconst char* + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%5$p"), args), + MatchesPointerString(mcp)); + + // function pointers + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%6$p"), args), + MatchesPointerString(reinterpret_cast<const void*>(fp))); + EXPECT_THAT( + FormatPack(UntypedFormatSpecImpl("%8$p"), args), + MatchesPointerString(reinterpret_cast<volatile const void *>(vcp))); + + // null function pointers + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%7$p"), args), + MatchesPointerString(nullptr)); + EXPECT_THAT(FormatPack(UntypedFormatSpecImpl("%9$p"), args), + MatchesPointerString(nullptr)); +} + +struct Cardinal { + enum Pos { k1 = 1, k2 = 2, k3 = 3 }; + enum Neg { kM1 = -1, kM2 = -2, kM3 = -3 }; +}; + +TEST_F(FormatConvertTest, Enum) { + const Cardinal::Pos k3 = Cardinal::k3; + const Cardinal::Neg km3 = Cardinal::kM3; + const FormatArgImpl args[] = {FormatArgImpl(k3), FormatArgImpl(km3)}; + UntypedFormatSpecImpl format("%1$d"); + UntypedFormatSpecImpl format2("%2$d"); + EXPECT_EQ("3", FormatPack(format, absl::MakeSpan(args))); + EXPECT_EQ("-3", FormatPack(format2, absl::MakeSpan(args))); +} + +template <typename T> +class TypedFormatConvertTest : public FormatConvertTest { }; + +TYPED_TEST_SUITE_P(TypedFormatConvertTest); + +std::vector<std::string> AllFlagCombinations() { + const char kFlags[] = {'-', '#', '0', '+', ' '}; + std::vector<std::string> result; + for (size_t fsi = 0; fsi < (1ull << ArraySize(kFlags)); ++fsi) { + std::string flag_set; + for (size_t fi = 0; fi < ArraySize(kFlags); ++fi) + if (fsi & (1ull << fi)) + flag_set += kFlags[fi]; + result.push_back(flag_set); + } + return result; +} + +TYPED_TEST_P(TypedFormatConvertTest, AllIntsWithFlags) { + typedef TypeParam T; + typedef typename std::make_unsigned<T>::type UnsignedT; + using remove_volatile_t = typename std::remove_volatile<T>::type; + const T kMin = std::numeric_limits<remove_volatile_t>::min(); + const T kMax = std::numeric_limits<remove_volatile_t>::max(); + const T kVals[] = { + remove_volatile_t(1), + remove_volatile_t(2), + remove_volatile_t(3), + remove_volatile_t(123), + remove_volatile_t(-1), + remove_volatile_t(-2), + remove_volatile_t(-3), + remove_volatile_t(-123), + remove_volatile_t(0), + kMax - remove_volatile_t(1), + kMax, + kMin + remove_volatile_t(1), + kMin, + }; + const char kConvChars[] = {'d', 'i', 'u', 'o', 'x', 'X'}; + const std::string kWid[] = {"", "4", "10"}; + const std::string kPrec[] = {"", ".", ".0", ".4", ".10"}; + + const std::vector<std::string> flag_sets = AllFlagCombinations(); + + for (size_t vi = 0; vi < ArraySize(kVals); ++vi) { + const T val = kVals[vi]; + SCOPED_TRACE(Esc(val)); + const FormatArgImpl args[] = {FormatArgImpl(val)}; + for (size_t ci = 0; ci < ArraySize(kConvChars); ++ci) { + const char conv_char = kConvChars[ci]; + for (size_t fsi = 0; fsi < flag_sets.size(); ++fsi) { + const std::string &flag_set = flag_sets[fsi]; + for (size_t wi = 0; wi < ArraySize(kWid); ++wi) { + const std::string &wid = kWid[wi]; + for (size_t pi = 0; pi < ArraySize(kPrec); ++pi) { + const std::string &prec = kPrec[pi]; + + const bool is_signed_conv = (conv_char == 'd' || conv_char == 'i'); + const bool is_unsigned_to_signed = + !std::is_signed<T>::value && is_signed_conv; + // Don't consider sign-related flags '+' and ' ' when doing + // unsigned to signed conversions. + if (is_unsigned_to_signed && + flag_set.find_first_of("+ ") != std::string::npos) { + continue; + } + + std::string new_fmt("%"); + new_fmt += flag_set; + new_fmt += wid; + new_fmt += prec; + // old and new always agree up to here. + std::string old_fmt = new_fmt; + new_fmt += conv_char; + std::string old_result; + if (is_unsigned_to_signed) { + // don't expect agreement on unsigned formatted as signed, + // as printf can't do that conversion properly. For those + // cases, we do expect agreement with printf with a "%u" + // and the unsigned equivalent of 'val'. + UnsignedT uval = val; + old_fmt += LengthModFor(uval); + old_fmt += "u"; + old_result = StrPrint(old_fmt.c_str(), uval); + } else { + old_fmt += LengthModFor(val); + old_fmt += conv_char; + old_result = StrPrint(old_fmt.c_str(), val); + } + + SCOPED_TRACE(std::string() + " old_fmt: \"" + old_fmt + + "\"'" + " new_fmt: \"" + + new_fmt + "\""); + UntypedFormatSpecImpl format(new_fmt); + EXPECT_EQ(old_result, FormatPack(format, absl::MakeSpan(args))); + } + } + } + } + } +} + +TYPED_TEST_P(TypedFormatConvertTest, Char) { + typedef TypeParam T; + using remove_volatile_t = typename std::remove_volatile<T>::type; + static const T kMin = std::numeric_limits<remove_volatile_t>::min(); + static const T kMax = std::numeric_limits<remove_volatile_t>::max(); + T kVals[] = { + remove_volatile_t(1), remove_volatile_t(2), remove_volatile_t(10), + remove_volatile_t(-1), remove_volatile_t(-2), remove_volatile_t(-10), + remove_volatile_t(0), + kMin + remove_volatile_t(1), kMin, + kMax - remove_volatile_t(1), kMax + }; + for (const T &c : kVals) { + const FormatArgImpl args[] = {FormatArgImpl(c)}; + UntypedFormatSpecImpl format("%c"); + EXPECT_EQ(StrPrint("%c", c), FormatPack(format, absl::MakeSpan(args))); + } +} + +REGISTER_TYPED_TEST_CASE_P(TypedFormatConvertTest, AllIntsWithFlags, Char); + +typedef ::testing::Types< + int, unsigned, volatile int, + short, unsigned short, + long, unsigned long, + long long, unsigned long long, + signed char, unsigned char, char> + AllIntTypes; +INSTANTIATE_TYPED_TEST_CASE_P(TypedFormatConvertTestWithAllIntTypes, + TypedFormatConvertTest, AllIntTypes); +TEST_F(FormatConvertTest, VectorBool) { + // Make sure vector<bool>'s values behave as bools. + std::vector<bool> v = {true, false}; + const std::vector<bool> cv = {true, false}; + EXPECT_EQ("1,0,1,0", + FormatPack(UntypedFormatSpecImpl("%d,%d,%d,%d"), + absl::Span<const FormatArgImpl>( + {FormatArgImpl(v[0]), FormatArgImpl(v[1]), + FormatArgImpl(cv[0]), FormatArgImpl(cv[1])}))); +} + + +TEST_F(FormatConvertTest, Int128) { + absl::int128 positive = static_cast<absl::int128>(0x1234567890abcdef) * 1979; + absl::int128 negative = -positive; + absl::int128 max = absl::Int128Max(), min = absl::Int128Min(); + const FormatArgImpl args[] = {FormatArgImpl(positive), + FormatArgImpl(negative), FormatArgImpl(max), + FormatArgImpl(min)}; + + struct Case { + const char* format; + const char* expected; + } cases[] = { + {"%1$d", "2595989796776606496405"}, + {"%1$30d", " 2595989796776606496405"}, + {"%1$-30d", "2595989796776606496405 "}, + {"%1$u", "2595989796776606496405"}, + {"%1$x", "8cba9876066020f695"}, + {"%2$d", "-2595989796776606496405"}, + {"%2$30d", " -2595989796776606496405"}, + {"%2$-30d", "-2595989796776606496405 "}, + {"%2$u", "340282366920938460867384810655161715051"}, + {"%2$x", "ffffffffffffff73456789f99fdf096b"}, + {"%3$d", "170141183460469231731687303715884105727"}, + {"%3$u", "170141183460469231731687303715884105727"}, + {"%3$x", "7fffffffffffffffffffffffffffffff"}, + {"%4$d", "-170141183460469231731687303715884105728"}, + {"%4$x", "80000000000000000000000000000000"}, + }; + + for (auto c : cases) { + UntypedFormatSpecImpl format(c.format); + EXPECT_EQ(c.expected, FormatPack(format, absl::MakeSpan(args))); + } +} + +TEST_F(FormatConvertTest, Uint128) { + absl::uint128 v = static_cast<absl::uint128>(0x1234567890abcdef) * 1979; + absl::uint128 max = absl::Uint128Max(); + const FormatArgImpl args[] = {FormatArgImpl(v), FormatArgImpl(max)}; + + struct Case { + const char* format; + const char* expected; + } cases[] = { + {"%1$d", "2595989796776606496405"}, + {"%1$30d", " 2595989796776606496405"}, + {"%1$-30d", "2595989796776606496405 "}, + {"%1$u", "2595989796776606496405"}, + {"%1$x", "8cba9876066020f695"}, + {"%2$d", "340282366920938463463374607431768211455"}, + {"%2$u", "340282366920938463463374607431768211455"}, + {"%2$x", "ffffffffffffffffffffffffffffffff"}, + }; + + for (auto c : cases) { + UntypedFormatSpecImpl format(c.format); + EXPECT_EQ(c.expected, FormatPack(format, absl::MakeSpan(args))); + } +} + +template <typename Floating> +void TestWithMultipleFormatsHelper(const std::vector<Floating> &floats) { + const NativePrintfTraits &native_traits = VerifyNativeImplementation(); + // Reserve the space to ensure we don't allocate memory in the output itself. + std::string str_format_result; + str_format_result.reserve(1 << 20); + std::string string_printf_result; + string_printf_result.reserve(1 << 20); + + const char *const kFormats[] = { + "%", "%.3", "%8.5", "%500", "%.5000", "%.60", "%.30", "%03", + "%+", "% ", "%-10", "%#15.3", "%#.0", "%.0", "%1$*2$", "%1$.*2$"}; + + for (const char *fmt : kFormats) { + for (char f : {'f', 'F', // + 'g', 'G', // + 'a', 'A', // + 'e', 'E'}) { + std::string fmt_str = std::string(fmt) + f; + + if (fmt == absl::string_view("%.5000") && f != 'f' && f != 'F' && + f != 'a' && f != 'A') { + // This particular test takes way too long with snprintf. + // Disable for the case we are not implementing natively. + continue; + } + + if ((f == 'a' || f == 'A') && + !native_traits.hex_float_has_glibc_rounding) { + continue; + } + + for (Floating d : floats) { + if (!native_traits.hex_float_prefers_denormal_repr && + (f == 'a' || f == 'A') && std::fpclassify(d) == FP_SUBNORMAL) { + continue; + } + int i = -10; + FormatArgImpl args[2] = {FormatArgImpl(d), FormatArgImpl(i)}; + UntypedFormatSpecImpl format(fmt_str); + + string_printf_result.clear(); + StrAppend(&string_printf_result, fmt_str.c_str(), d, i); + str_format_result.clear(); + + { + AppendPack(&str_format_result, format, absl::MakeSpan(args)); + } + + if (string_printf_result != str_format_result) { + // We use ASSERT_EQ here because failures are usually correlated and a + // bug would print way too many failed expectations causing the test + // to time out. + ASSERT_EQ(string_printf_result, str_format_result) + << fmt_str << " " << StrPrint("%.18g", d) << " " + << StrPrint("%a", d) << " " << StrPrint("%.50f", d); + } + } + } + } +} + +TEST_F(FormatConvertTest, Float) { +#ifdef _MSC_VER + // MSVC has a different rounding policy than us so we can't test our + // implementation against the native one there. + return; +#endif // _MSC_VER + + std::vector<float> floats = {0.0f, + -0.0f, + .9999999f, + 9999999.f, + std::numeric_limits<float>::max(), + -std::numeric_limits<float>::max(), + std::numeric_limits<float>::min(), + -std::numeric_limits<float>::min(), + std::numeric_limits<float>::lowest(), + -std::numeric_limits<float>::lowest(), + std::numeric_limits<float>::epsilon(), + std::numeric_limits<float>::epsilon() + 1.0f, + std::numeric_limits<float>::infinity(), + -std::numeric_limits<float>::infinity()}; + + // Some regression tests. + floats.push_back(0.999999989f); + + if (std::numeric_limits<float>::has_denorm != std::denorm_absent) { + floats.push_back(std::numeric_limits<float>::denorm_min()); + floats.push_back(-std::numeric_limits<float>::denorm_min()); + } + + for (float base : + {1.f, 12.f, 123.f, 1234.f, 12345.f, 123456.f, 1234567.f, 12345678.f, + 123456789.f, 1234567890.f, 12345678901.f, 12345678.f, 12345678.f}) { + for (int exp = -123; exp <= 123; ++exp) { + for (int sign : {1, -1}) { + floats.push_back(sign * std::ldexp(base, exp)); + } + } + } + + for (int exp = -300; exp <= 300; ++exp) { + const float all_ones_mantissa = 0xffffff; + floats.push_back(std::ldexp(all_ones_mantissa, exp)); + } + + // Remove duplicates to speed up the logic below. + std::sort(floats.begin(), floats.end()); + floats.erase(std::unique(floats.begin(), floats.end()), floats.end()); + +#ifndef __APPLE__ + // Apple formats NaN differently (+nan) vs. (nan) + floats.push_back(std::nan("")); +#endif + + TestWithMultipleFormatsHelper(floats); +} + +TEST_F(FormatConvertTest, Double) { +#ifdef _MSC_VER + // MSVC has a different rounding policy than us so we can't test our + // implementation against the native one there. + return; +#endif // _MSC_VER + + std::vector<double> doubles = {0.0, + -0.0, + .99999999999999, + 99999999999999., + std::numeric_limits<double>::max(), + -std::numeric_limits<double>::max(), + std::numeric_limits<double>::min(), + -std::numeric_limits<double>::min(), + std::numeric_limits<double>::lowest(), + -std::numeric_limits<double>::lowest(), + std::numeric_limits<double>::epsilon(), + std::numeric_limits<double>::epsilon() + 1, + std::numeric_limits<double>::infinity(), + -std::numeric_limits<double>::infinity()}; + + // Some regression tests. + doubles.push_back(0.99999999999999989); + + if (std::numeric_limits<double>::has_denorm != std::denorm_absent) { + doubles.push_back(std::numeric_limits<double>::denorm_min()); + doubles.push_back(-std::numeric_limits<double>::denorm_min()); + } + + for (double base : + {1., 12., 123., 1234., 12345., 123456., 1234567., 12345678., 123456789., + 1234567890., 12345678901., 123456789012., 1234567890123.}) { + for (int exp = -123; exp <= 123; ++exp) { + for (int sign : {1, -1}) { + doubles.push_back(sign * std::ldexp(base, exp)); + } + } + } + + // Workaround libc bug. + // https://sourceware.org/bugzilla/show_bug.cgi?id=22142 + const bool gcc_bug_22142 = + StrPrint("%f", std::numeric_limits<double>::max()) != + "1797693134862315708145274237317043567980705675258449965989174768031" + "5726078002853876058955863276687817154045895351438246423432132688946" + "4182768467546703537516986049910576551282076245490090389328944075868" + "5084551339423045832369032229481658085593321233482747978262041447231" + "68738177180919299881250404026184124858368.000000"; + + if (!gcc_bug_22142) { + for (int exp = -300; exp <= 300; ++exp) { + const double all_ones_mantissa = 0x1fffffffffffff; + doubles.push_back(std::ldexp(all_ones_mantissa, exp)); + } + } + + if (gcc_bug_22142) { + for (auto &d : doubles) { + using L = std::numeric_limits<double>; + double d2 = std::abs(d); + if (d2 == L::max() || d2 == L::min() || d2 == L::denorm_min()) { + d = 0; + } + } + } + + // Remove duplicates to speed up the logic below. + std::sort(doubles.begin(), doubles.end()); + doubles.erase(std::unique(doubles.begin(), doubles.end()), doubles.end()); + +#ifndef __APPLE__ + // Apple formats NaN differently (+nan) vs. (nan) + doubles.push_back(std::nan("")); +#endif + + TestWithMultipleFormatsHelper(doubles); +} + +TEST_F(FormatConvertTest, DoubleRound) { + std::string s; + const auto format = [&](const char *fmt, double d) -> std::string & { + s.clear(); + FormatArgImpl args[1] = {FormatArgImpl(d)}; + AppendPack(&s, UntypedFormatSpecImpl(fmt), absl::MakeSpan(args)); +#if !defined(_MSC_VER) + // MSVC has a different rounding policy than us so we can't test our + // implementation against the native one there. + EXPECT_EQ(StrPrint(fmt, d), s); +#endif // _MSC_VER + + return s; + }; + // All of these values have to be exactly represented. + // Otherwise we might not be testing what we think we are testing. + + // These values can fit in a 64bit "fast" representation. + const double exact_value = 0.00000000000005684341886080801486968994140625; + assert(exact_value == std::pow(2, -44)); + // Round up at a 5xx. + EXPECT_EQ(format("%.13f", exact_value), "0.0000000000001"); + // Round up at a >5 + EXPECT_EQ(format("%.14f", exact_value), "0.00000000000006"); + // Round down at a <5 + EXPECT_EQ(format("%.16f", exact_value), "0.0000000000000568"); + // Nine handling + EXPECT_EQ(format("%.35f", exact_value), + "0.00000000000005684341886080801486969"); + EXPECT_EQ(format("%.36f", exact_value), + "0.000000000000056843418860808014869690"); + // Round down the last nine. + EXPECT_EQ(format("%.37f", exact_value), + "0.0000000000000568434188608080148696899"); + EXPECT_EQ(format("%.10f", 0.000003814697265625), "0.0000038147"); + // Round up the last nine + EXPECT_EQ(format("%.11f", 0.000003814697265625), "0.00000381470"); + EXPECT_EQ(format("%.12f", 0.000003814697265625), "0.000003814697"); + + // Round to even (down) + EXPECT_EQ(format("%.43f", exact_value), + "0.0000000000000568434188608080148696899414062"); + // Exact + EXPECT_EQ(format("%.44f", exact_value), + "0.00000000000005684341886080801486968994140625"); + // Round to even (up), let make the last digits 75 instead of 25 + EXPECT_EQ(format("%.43f", exact_value + std::pow(2, -43)), + "0.0000000000001705302565824240446090698242188"); + // Exact, just to check. + EXPECT_EQ(format("%.44f", exact_value + std::pow(2, -43)), + "0.00000000000017053025658242404460906982421875"); + + // This value has to be small enough that it won't fit in the uint128 + // representation for printing. + const double small_exact_value = + 0.000000000000000000000000000000000000752316384526264005099991383822237233803945956334136013765601092018187046051025390625; // NOLINT + assert(small_exact_value == std::pow(2, -120)); + // Round up at a 5xx. + EXPECT_EQ(format("%.37f", small_exact_value), + "0.0000000000000000000000000000000000008"); + // Round down at a <5 + EXPECT_EQ(format("%.38f", small_exact_value), + "0.00000000000000000000000000000000000075"); + // Round up at a >5 + EXPECT_EQ(format("%.41f", small_exact_value), + "0.00000000000000000000000000000000000075232"); + // Nine handling + EXPECT_EQ(format("%.55f", small_exact_value), + "0.0000000000000000000000000000000000007523163845262640051"); + EXPECT_EQ(format("%.56f", small_exact_value), + "0.00000000000000000000000000000000000075231638452626400510"); + EXPECT_EQ(format("%.57f", small_exact_value), + "0.000000000000000000000000000000000000752316384526264005100"); + EXPECT_EQ(format("%.58f", small_exact_value), + "0.0000000000000000000000000000000000007523163845262640051000"); + // Round down the last nine + EXPECT_EQ(format("%.59f", small_exact_value), + "0.00000000000000000000000000000000000075231638452626400509999"); + // Round up the last nine + EXPECT_EQ(format("%.79f", small_exact_value), + "0.000000000000000000000000000000000000" + "7523163845262640050999913838222372338039460"); + + // Round to even (down) + EXPECT_EQ(format("%.119f", small_exact_value), + "0.000000000000000000000000000000000000" + "75231638452626400509999138382223723380" + "394595633413601376560109201818704605102539062"); + // Exact + EXPECT_EQ(format("%.120f", small_exact_value), + "0.000000000000000000000000000000000000" + "75231638452626400509999138382223723380" + "3945956334136013765601092018187046051025390625"); + // Round to even (up), let make the last digits 75 instead of 25 + EXPECT_EQ(format("%.119f", small_exact_value + std::pow(2, -119)), + "0.000000000000000000000000000000000002" + "25694915357879201529997415146671170141" + "183786900240804129680327605456113815307617188"); + // Exact, just to check. + EXPECT_EQ(format("%.120f", small_exact_value + std::pow(2, -119)), + "0.000000000000000000000000000000000002" + "25694915357879201529997415146671170141" + "1837869002408041296803276054561138153076171875"); +} + +TEST_F(FormatConvertTest, DoubleRoundA) { + const NativePrintfTraits &native_traits = VerifyNativeImplementation(); + std::string s; + const auto format = [&](const char *fmt, double d) -> std::string & { + s.clear(); + FormatArgImpl args[1] = {FormatArgImpl(d)}; + AppendPack(&s, UntypedFormatSpecImpl(fmt), absl::MakeSpan(args)); + if (native_traits.hex_float_has_glibc_rounding) { + EXPECT_EQ(StrPrint(fmt, d), s); + } + return s; + }; + + // 0x1.00018000p+100 + const double on_boundary_odd = 1267679614447900152596896153600.0; + EXPECT_EQ(format("%.0a", on_boundary_odd), "0x1p+100"); + EXPECT_EQ(format("%.1a", on_boundary_odd), "0x1.0p+100"); + EXPECT_EQ(format("%.2a", on_boundary_odd), "0x1.00p+100"); + EXPECT_EQ(format("%.3a", on_boundary_odd), "0x1.000p+100"); + EXPECT_EQ(format("%.4a", on_boundary_odd), "0x1.0002p+100"); // round + EXPECT_EQ(format("%.5a", on_boundary_odd), "0x1.00018p+100"); + EXPECT_EQ(format("%.6a", on_boundary_odd), "0x1.000180p+100"); + + // 0x1.00028000p-2 + const double on_boundary_even = 0.250009536743164062500; + EXPECT_EQ(format("%.0a", on_boundary_even), "0x1p-2"); + EXPECT_EQ(format("%.1a", on_boundary_even), "0x1.0p-2"); + EXPECT_EQ(format("%.2a", on_boundary_even), "0x1.00p-2"); + EXPECT_EQ(format("%.3a", on_boundary_even), "0x1.000p-2"); + EXPECT_EQ(format("%.4a", on_boundary_even), "0x1.0002p-2"); // no round + EXPECT_EQ(format("%.5a", on_boundary_even), "0x1.00028p-2"); + EXPECT_EQ(format("%.6a", on_boundary_even), "0x1.000280p-2"); + + // 0x1.00018001p+1 + const double slightly_over = 2.00004577683284878730773925781250; + EXPECT_EQ(format("%.0a", slightly_over), "0x1p+1"); + EXPECT_EQ(format("%.1a", slightly_over), "0x1.0p+1"); + EXPECT_EQ(format("%.2a", slightly_over), "0x1.00p+1"); + EXPECT_EQ(format("%.3a", slightly_over), "0x1.000p+1"); + EXPECT_EQ(format("%.4a", slightly_over), "0x1.0002p+1"); + EXPECT_EQ(format("%.5a", slightly_over), "0x1.00018p+1"); + EXPECT_EQ(format("%.6a", slightly_over), "0x1.000180p+1"); + + // 0x1.00017fffp+0 + const double slightly_under = 1.000022887950763106346130371093750; + EXPECT_EQ(format("%.0a", slightly_under), "0x1p+0"); + EXPECT_EQ(format("%.1a", slightly_under), "0x1.0p+0"); + EXPECT_EQ(format("%.2a", slightly_under), "0x1.00p+0"); + EXPECT_EQ(format("%.3a", slightly_under), "0x1.000p+0"); + EXPECT_EQ(format("%.4a", slightly_under), "0x1.0001p+0"); + EXPECT_EQ(format("%.5a", slightly_under), "0x1.00018p+0"); + EXPECT_EQ(format("%.6a", slightly_under), "0x1.000180p+0"); + EXPECT_EQ(format("%.7a", slightly_under), "0x1.0001800p+0"); + + // 0x1.1b3829ac28058p+3 + const double hex_value = 8.85060580848964661981881363317370414733886718750; + EXPECT_EQ(format("%.0a", hex_value), "0x1p+3"); + EXPECT_EQ(format("%.1a", hex_value), "0x1.2p+3"); + EXPECT_EQ(format("%.2a", hex_value), "0x1.1bp+3"); + EXPECT_EQ(format("%.3a", hex_value), "0x1.1b4p+3"); + EXPECT_EQ(format("%.4a", hex_value), "0x1.1b38p+3"); + EXPECT_EQ(format("%.5a", hex_value), "0x1.1b383p+3"); + EXPECT_EQ(format("%.6a", hex_value), "0x1.1b382ap+3"); + EXPECT_EQ(format("%.7a", hex_value), "0x1.1b3829bp+3"); + EXPECT_EQ(format("%.8a", hex_value), "0x1.1b3829acp+3"); + EXPECT_EQ(format("%.9a", hex_value), "0x1.1b3829ac3p+3"); + EXPECT_EQ(format("%.10a", hex_value), "0x1.1b3829ac28p+3"); + EXPECT_EQ(format("%.11a", hex_value), "0x1.1b3829ac280p+3"); + EXPECT_EQ(format("%.12a", hex_value), "0x1.1b3829ac2806p+3"); + EXPECT_EQ(format("%.13a", hex_value), "0x1.1b3829ac28058p+3"); + EXPECT_EQ(format("%.14a", hex_value), "0x1.1b3829ac280580p+3"); + EXPECT_EQ(format("%.15a", hex_value), "0x1.1b3829ac2805800p+3"); + EXPECT_EQ(format("%.16a", hex_value), "0x1.1b3829ac28058000p+3"); + EXPECT_EQ(format("%.17a", hex_value), "0x1.1b3829ac280580000p+3"); + EXPECT_EQ(format("%.18a", hex_value), "0x1.1b3829ac2805800000p+3"); + EXPECT_EQ(format("%.19a", hex_value), "0x1.1b3829ac28058000000p+3"); + EXPECT_EQ(format("%.20a", hex_value), "0x1.1b3829ac280580000000p+3"); + EXPECT_EQ(format("%.21a", hex_value), "0x1.1b3829ac2805800000000p+3"); + + // 0x1.0818283848586p+3 + const double hex_value2 = 8.2529488658208371987257123691961169242858886718750; + EXPECT_EQ(format("%.0a", hex_value2), "0x1p+3"); + EXPECT_EQ(format("%.1a", hex_value2), "0x1.1p+3"); + EXPECT_EQ(format("%.2a", hex_value2), "0x1.08p+3"); + EXPECT_EQ(format("%.3a", hex_value2), "0x1.082p+3"); + EXPECT_EQ(format("%.4a", hex_value2), "0x1.0818p+3"); + EXPECT_EQ(format("%.5a", hex_value2), "0x1.08183p+3"); + EXPECT_EQ(format("%.6a", hex_value2), "0x1.081828p+3"); + EXPECT_EQ(format("%.7a", hex_value2), "0x1.0818284p+3"); + EXPECT_EQ(format("%.8a", hex_value2), "0x1.08182838p+3"); + EXPECT_EQ(format("%.9a", hex_value2), "0x1.081828385p+3"); + EXPECT_EQ(format("%.10a", hex_value2), "0x1.0818283848p+3"); + EXPECT_EQ(format("%.11a", hex_value2), "0x1.08182838486p+3"); + EXPECT_EQ(format("%.12a", hex_value2), "0x1.081828384858p+3"); + EXPECT_EQ(format("%.13a", hex_value2), "0x1.0818283848586p+3"); + EXPECT_EQ(format("%.14a", hex_value2), "0x1.08182838485860p+3"); + EXPECT_EQ(format("%.15a", hex_value2), "0x1.081828384858600p+3"); + EXPECT_EQ(format("%.16a", hex_value2), "0x1.0818283848586000p+3"); + EXPECT_EQ(format("%.17a", hex_value2), "0x1.08182838485860000p+3"); + EXPECT_EQ(format("%.18a", hex_value2), "0x1.081828384858600000p+3"); + EXPECT_EQ(format("%.19a", hex_value2), "0x1.0818283848586000000p+3"); + EXPECT_EQ(format("%.20a", hex_value2), "0x1.08182838485860000000p+3"); + EXPECT_EQ(format("%.21a", hex_value2), "0x1.081828384858600000000p+3"); +} + +TEST_F(FormatConvertTest, LongDoubleRoundA) { + if (std::numeric_limits<long double>::digits % 4 != 0) { + // This test doesn't really make sense to run on platforms where a long + // double has a different mantissa size (mod 4) than Prod, since then the + // leading digit will be formatted differently. + return; + } + const NativePrintfTraits &native_traits = VerifyNativeImplementation(); + std::string s; + const auto format = [&](const char *fmt, long double d) -> std::string & { + s.clear(); + FormatArgImpl args[1] = {FormatArgImpl(d)}; + AppendPack(&s, UntypedFormatSpecImpl(fmt), absl::MakeSpan(args)); + if (native_traits.hex_float_has_glibc_rounding && + native_traits.hex_float_optimizes_leading_digit_bit_count) { + EXPECT_EQ(StrPrint(fmt, d), s); + } + return s; + }; + + // 0x8.8p+4 + const long double on_boundary_even = 136.0; + EXPECT_EQ(format("%.0La", on_boundary_even), "0x8p+4"); + EXPECT_EQ(format("%.1La", on_boundary_even), "0x8.8p+4"); + EXPECT_EQ(format("%.2La", on_boundary_even), "0x8.80p+4"); + EXPECT_EQ(format("%.3La", on_boundary_even), "0x8.800p+4"); + EXPECT_EQ(format("%.4La", on_boundary_even), "0x8.8000p+4"); + EXPECT_EQ(format("%.5La", on_boundary_even), "0x8.80000p+4"); + EXPECT_EQ(format("%.6La", on_boundary_even), "0x8.800000p+4"); + + // 0x9.8p+4 + const long double on_boundary_odd = 152.0; + EXPECT_EQ(format("%.0La", on_boundary_odd), "0xap+4"); + EXPECT_EQ(format("%.1La", on_boundary_odd), "0x9.8p+4"); + EXPECT_EQ(format("%.2La", on_boundary_odd), "0x9.80p+4"); + EXPECT_EQ(format("%.3La", on_boundary_odd), "0x9.800p+4"); + EXPECT_EQ(format("%.4La", on_boundary_odd), "0x9.8000p+4"); + EXPECT_EQ(format("%.5La", on_boundary_odd), "0x9.80000p+4"); + EXPECT_EQ(format("%.6La", on_boundary_odd), "0x9.800000p+4"); + + // 0x8.80001p+24 + const long double slightly_over = 142606352.0; + EXPECT_EQ(format("%.0La", slightly_over), "0x9p+24"); + EXPECT_EQ(format("%.1La", slightly_over), "0x8.8p+24"); + EXPECT_EQ(format("%.2La", slightly_over), "0x8.80p+24"); + EXPECT_EQ(format("%.3La", slightly_over), "0x8.800p+24"); + EXPECT_EQ(format("%.4La", slightly_over), "0x8.8000p+24"); + EXPECT_EQ(format("%.5La", slightly_over), "0x8.80001p+24"); + EXPECT_EQ(format("%.6La", slightly_over), "0x8.800010p+24"); + + // 0x8.7ffffp+24 + const long double slightly_under = 142606320.0; + EXPECT_EQ(format("%.0La", slightly_under), "0x8p+24"); + EXPECT_EQ(format("%.1La", slightly_under), "0x8.8p+24"); + EXPECT_EQ(format("%.2La", slightly_under), "0x8.80p+24"); + EXPECT_EQ(format("%.3La", slightly_under), "0x8.800p+24"); + EXPECT_EQ(format("%.4La", slightly_under), "0x8.8000p+24"); + EXPECT_EQ(format("%.5La", slightly_under), "0x8.7ffffp+24"); + EXPECT_EQ(format("%.6La", slightly_under), "0x8.7ffff0p+24"); + EXPECT_EQ(format("%.7La", slightly_under), "0x8.7ffff00p+24"); + + // 0xc.0828384858688000p+128 + const long double eights = 4094231060438608800781871108094404067328.0; + EXPECT_EQ(format("%.0La", eights), "0xcp+128"); + EXPECT_EQ(format("%.1La", eights), "0xc.1p+128"); + EXPECT_EQ(format("%.2La", eights), "0xc.08p+128"); + EXPECT_EQ(format("%.3La", eights), "0xc.083p+128"); + EXPECT_EQ(format("%.4La", eights), "0xc.0828p+128"); + EXPECT_EQ(format("%.5La", eights), "0xc.08284p+128"); + EXPECT_EQ(format("%.6La", eights), "0xc.082838p+128"); + EXPECT_EQ(format("%.7La", eights), "0xc.0828385p+128"); + EXPECT_EQ(format("%.8La", eights), "0xc.08283848p+128"); + EXPECT_EQ(format("%.9La", eights), "0xc.082838486p+128"); + EXPECT_EQ(format("%.10La", eights), "0xc.0828384858p+128"); + EXPECT_EQ(format("%.11La", eights), "0xc.08283848587p+128"); + EXPECT_EQ(format("%.12La", eights), "0xc.082838485868p+128"); + EXPECT_EQ(format("%.13La", eights), "0xc.0828384858688p+128"); + EXPECT_EQ(format("%.14La", eights), "0xc.08283848586880p+128"); + EXPECT_EQ(format("%.15La", eights), "0xc.082838485868800p+128"); + EXPECT_EQ(format("%.16La", eights), "0xc.0828384858688000p+128"); +} + +// We don't actually store the results. This is just to exercise the rest of the +// machinery. +struct NullSink { + friend void AbslFormatFlush(NullSink *sink, string_view str) {} +}; + +template <typename... T> +bool FormatWithNullSink(absl::string_view fmt, const T &... a) { + NullSink sink; + FormatArgImpl args[] = {FormatArgImpl(a)...}; + return FormatUntyped(&sink, UntypedFormatSpecImpl(fmt), absl::MakeSpan(args)); +} + +TEST_F(FormatConvertTest, ExtremeWidthPrecision) { + for (const char *fmt : {"f"}) { + for (double d : {1e-100, 1.0, 1e100}) { + constexpr int max = std::numeric_limits<int>::max(); + EXPECT_TRUE(FormatWithNullSink(std::string("%.*") + fmt, max, d)); + EXPECT_TRUE(FormatWithNullSink(std::string("%1.*") + fmt, max, d)); + EXPECT_TRUE(FormatWithNullSink(std::string("%*") + fmt, max, d)); + EXPECT_TRUE(FormatWithNullSink(std::string("%*.*") + fmt, max, max, d)); + } + } +} + +TEST_F(FormatConvertTest, LongDouble) { +#ifdef _MSC_VER + // MSVC has a different rounding policy than us so we can't test our + // implementation against the native one there. + return; +#endif // _MSC_VER + const NativePrintfTraits &native_traits = VerifyNativeImplementation(); + const char *const kFormats[] = {"%", "%.3", "%8.5", "%9", "%.5000", + "%.60", "%+", "% ", "%-10"}; + + std::vector<long double> doubles = { + 0.0, + -0.0, + std::numeric_limits<long double>::max(), + -std::numeric_limits<long double>::max(), + std::numeric_limits<long double>::min(), + -std::numeric_limits<long double>::min(), + std::numeric_limits<long double>::infinity(), + -std::numeric_limits<long double>::infinity()}; + + for (long double base : {1.L, 12.L, 123.L, 1234.L, 12345.L, 123456.L, + 1234567.L, 12345678.L, 123456789.L, 1234567890.L, + 12345678901.L, 123456789012.L, 1234567890123.L, + // This value is not representable in double, but it + // is in long double that uses the extended format. + // This is to verify that we are not truncating the + // value mistakenly through a double. + 10000000000000000.25L}) { + for (int exp : {-1000, -500, 0, 500, 1000}) { + for (int sign : {1, -1}) { + doubles.push_back(sign * std::ldexp(base, exp)); + doubles.push_back(sign / std::ldexp(base, exp)); + } + } + } + + // Regression tests + // + // Using a string literal because not all platforms support hex literals or it + // might be out of range. + doubles.push_back(std::strtold("-0xf.ffffffb5feafffbp-16324L", nullptr)); + + for (const char *fmt : kFormats) { + for (char f : {'f', 'F', // + 'g', 'G', // + 'a', 'A', // + 'e', 'E'}) { + std::string fmt_str = std::string(fmt) + 'L' + f; + + if (fmt == absl::string_view("%.5000") && f != 'f' && f != 'F' && + f != 'a' && f != 'A') { + // This particular test takes way too long with snprintf. + // Disable for the case we are not implementing natively. + continue; + } + + if (f == 'a' || f == 'A') { + if (!native_traits.hex_float_has_glibc_rounding || + !native_traits.hex_float_optimizes_leading_digit_bit_count) { + continue; + } + } + + for (auto d : doubles) { + FormatArgImpl arg(d); + UntypedFormatSpecImpl format(fmt_str); + // We use ASSERT_EQ here because failures are usually correlated and a + // bug would print way too many failed expectations causing the test to + // time out. + ASSERT_EQ(StrPrint(fmt_str.c_str(), d), FormatPack(format, {&arg, 1})) + << fmt_str << " " << StrPrint("%.18Lg", d) << " " + << StrPrint("%La", d) << " " << StrPrint("%.1080Lf", d); + } + } + } +} + +TEST_F(FormatConvertTest, IntAsDouble) { + const NativePrintfTraits &native_traits = VerifyNativeImplementation(); + const int kMin = std::numeric_limits<int>::min(); + const int kMax = std::numeric_limits<int>::max(); + const int ia[] = { + 1, 2, 3, 123, + -1, -2, -3, -123, + 0, kMax - 1, kMax, kMin + 1, kMin }; + for (const int fx : ia) { + SCOPED_TRACE(fx); + const FormatArgImpl args[] = {FormatArgImpl(fx)}; + struct Expectation { + int line; + std::string out; + const char *fmt; + }; + const double dx = static_cast<double>(fx); + std::vector<Expectation> expect = { + {__LINE__, StrPrint("%f", dx), "%f"}, + {__LINE__, StrPrint("%12f", dx), "%12f"}, + {__LINE__, StrPrint("%.12f", dx), "%.12f"}, + {__LINE__, StrPrint("%.12a", dx), "%.12a"}, + }; + if (native_traits.hex_float_uses_minimal_precision_when_not_specified) { + Expectation ex = {__LINE__, StrPrint("%12a", dx), "%12a"}; + expect.push_back(ex); + } + for (const Expectation &e : expect) { + SCOPED_TRACE(e.line); + SCOPED_TRACE(e.fmt); + UntypedFormatSpecImpl format(e.fmt); + EXPECT_EQ(e.out, FormatPack(format, absl::MakeSpan(args))); + } + } +} + +template <typename T> +bool FormatFails(const char* test_format, T value) { + std::string format_string = std::string("<<") + test_format + ">>"; + UntypedFormatSpecImpl format(format_string); + + int one = 1; + const FormatArgImpl args[] = {FormatArgImpl(value), FormatArgImpl(one)}; + EXPECT_EQ(FormatPack(format, absl::MakeSpan(args)), "") + << "format=" << test_format << " value=" << value; + return FormatPack(format, absl::MakeSpan(args)).empty(); +} + +TEST_F(FormatConvertTest, ExpectedFailures) { + // Int input + EXPECT_TRUE(FormatFails("%p", 1)); + EXPECT_TRUE(FormatFails("%s", 1)); + EXPECT_TRUE(FormatFails("%n", 1)); + + // Double input + EXPECT_TRUE(FormatFails("%p", 1.)); + EXPECT_TRUE(FormatFails("%s", 1.)); + EXPECT_TRUE(FormatFails("%n", 1.)); + EXPECT_TRUE(FormatFails("%c", 1.)); + EXPECT_TRUE(FormatFails("%d", 1.)); + EXPECT_TRUE(FormatFails("%x", 1.)); + EXPECT_TRUE(FormatFails("%*d", 1.)); + + // String input + EXPECT_TRUE(FormatFails("%n", "")); + EXPECT_TRUE(FormatFails("%c", "")); + EXPECT_TRUE(FormatFails("%d", "")); + EXPECT_TRUE(FormatFails("%x", "")); + EXPECT_TRUE(FormatFails("%f", "")); + EXPECT_TRUE(FormatFails("%*d", "")); +} + +// Sanity check to make sure that we are testing what we think we're testing on +// e.g. the x86_64+glibc platform. +TEST_F(FormatConvertTest, GlibcHasCorrectTraits) { +#if !defined(__GLIBC__) || !defined(__x86_64__) + return; +#endif + const NativePrintfTraits &native_traits = VerifyNativeImplementation(); + // If one of the following tests break then it is either because the above PP + // macro guards failed to exclude a new platform (likely) or because something + // has changed in the implemention of glibc sprintf float formatting behavior. + // If the latter, then the code that computes these flags needs to be + // revisited and/or possibly the StrFormat implementation. + EXPECT_TRUE(native_traits.hex_float_has_glibc_rounding); + EXPECT_TRUE(native_traits.hex_float_prefers_denormal_repr); + EXPECT_TRUE( + native_traits.hex_float_uses_minimal_precision_when_not_specified); + EXPECT_TRUE(native_traits.hex_float_optimizes_leading_digit_bit_count); +} + +} // namespace +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/extension.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/extension.cc new file mode 100644 index 000000000000..bb0d96cf3216 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/extension.cc @@ -0,0 +1,75 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/extension.h" + +#include <errno.h> +#include <algorithm> +#include <string> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +std::string Flags::ToString() const { + std::string s; + s.append(left ? "-" : ""); + s.append(show_pos ? "+" : ""); + s.append(sign_col ? " " : ""); + s.append(alt ? "#" : ""); + s.append(zero ? "0" : ""); + return s; +} + +#define ABSL_INTERNAL_X_VAL(id) \ + constexpr absl::FormatConversionChar FormatConversionCharInternal::id; +ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_X_VAL, ) +#undef ABSL_INTERNAL_X_VAL +// NOLINTNEXTLINE(readability-redundant-declaration) +constexpr absl::FormatConversionChar FormatConversionCharInternal::kNone; + +#define ABSL_INTERNAL_CHAR_SET_CASE(c) \ + constexpr FormatConversionCharSet FormatConversionCharSetInternal::c; +ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_CHAR_SET_CASE, ) +#undef ABSL_INTERNAL_CHAR_SET_CASE + +// NOLINTNEXTLINE(readability-redundant-declaration) +constexpr FormatConversionCharSet FormatConversionCharSetInternal::kStar; +// NOLINTNEXTLINE(readability-redundant-declaration) +constexpr FormatConversionCharSet FormatConversionCharSetInternal::kIntegral; +// NOLINTNEXTLINE(readability-redundant-declaration) +constexpr FormatConversionCharSet FormatConversionCharSetInternal::kFloating; +// NOLINTNEXTLINE(readability-redundant-declaration) +constexpr FormatConversionCharSet FormatConversionCharSetInternal::kNumeric; +// NOLINTNEXTLINE(readability-redundant-declaration) +constexpr FormatConversionCharSet FormatConversionCharSetInternal::kPointer; + +bool FormatSinkImpl::PutPaddedString(string_view value, int width, + int precision, bool left) { + size_t space_remaining = 0; + if (width >= 0) space_remaining = width; + size_t n = value.size(); + if (precision >= 0) n = std::min(n, static_cast<size_t>(precision)); + string_view shown(value.data(), n); + space_remaining = Excess(shown.size(), space_remaining); + if (!left) Append(space_remaining, ' '); + Append(shown); + if (left) Append(space_remaining, ' '); + return true; +} + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/extension.h b/third_party/abseil_cpp/absl/strings/internal/str_format/extension.h new file mode 100644 index 000000000000..a9b9e137deb2 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/extension.h @@ -0,0 +1,427 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_EXTENSION_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_EXTENSION_H_ + +#include <limits.h> + +#include <cstddef> +#include <cstring> +#include <ostream> + +#include "absl/base/config.h" +#include "absl/base/port.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/internal/str_format/output.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +enum class FormatConversionChar : uint8_t; +enum class FormatConversionCharSet : uint64_t; + +namespace str_format_internal { + +class FormatRawSinkImpl { + public: + // Implicitly convert from any type that provides the hook function as + // described above. + template <typename T, decltype(str_format_internal::InvokeFlush( + std::declval<T*>(), string_view()))* = nullptr> + FormatRawSinkImpl(T* raw) // NOLINT + : sink_(raw), write_(&FormatRawSinkImpl::Flush<T>) {} + + void Write(string_view s) { write_(sink_, s); } + + template <typename T> + static FormatRawSinkImpl Extract(T s) { + return s.sink_; + } + + private: + template <typename T> + static void Flush(void* r, string_view s) { + str_format_internal::InvokeFlush(static_cast<T*>(r), s); + } + + void* sink_; + void (*write_)(void*, string_view); +}; + +// An abstraction to which conversions write their string data. +class FormatSinkImpl { + public: + explicit FormatSinkImpl(FormatRawSinkImpl raw) : raw_(raw) {} + + ~FormatSinkImpl() { Flush(); } + + void Flush() { + raw_.Write(string_view(buf_, pos_ - buf_)); + pos_ = buf_; + } + + void Append(size_t n, char c) { + if (n == 0) return; + size_ += n; + auto raw_append = [&](size_t count) { + memset(pos_, c, count); + pos_ += count; + }; + while (n > Avail()) { + n -= Avail(); + if (Avail() > 0) { + raw_append(Avail()); + } + Flush(); + } + raw_append(n); + } + + void Append(string_view v) { + size_t n = v.size(); + if (n == 0) return; + size_ += n; + if (n >= Avail()) { + Flush(); + raw_.Write(v); + return; + } + memcpy(pos_, v.data(), n); + pos_ += n; + } + + size_t size() const { return size_; } + + // Put 'v' to 'sink' with specified width, precision, and left flag. + bool PutPaddedString(string_view v, int width, int precision, bool left); + + template <typename T> + T Wrap() { + return T(this); + } + + template <typename T> + static FormatSinkImpl* Extract(T* s) { + return s->sink_; + } + + private: + size_t Avail() const { return buf_ + sizeof(buf_) - pos_; } + + FormatRawSinkImpl raw_; + size_t size_ = 0; + char* pos_ = buf_; + char buf_[1024]; +}; + +struct Flags { + bool basic : 1; // fastest conversion: no flags, width, or precision + bool left : 1; // "-" + bool show_pos : 1; // "+" + bool sign_col : 1; // " " + bool alt : 1; // "#" + bool zero : 1; // "0" + std::string ToString() const; + friend std::ostream& operator<<(std::ostream& os, const Flags& v) { + return os << v.ToString(); + } +}; + +// clang-format off +#define ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(X_VAL, X_SEP) \ + /* text */ \ + X_VAL(c) X_SEP X_VAL(s) X_SEP \ + /* ints */ \ + X_VAL(d) X_SEP X_VAL(i) X_SEP X_VAL(o) X_SEP \ + X_VAL(u) X_SEP X_VAL(x) X_SEP X_VAL(X) X_SEP \ + /* floats */ \ + X_VAL(f) X_SEP X_VAL(F) X_SEP X_VAL(e) X_SEP X_VAL(E) X_SEP \ + X_VAL(g) X_SEP X_VAL(G) X_SEP X_VAL(a) X_SEP X_VAL(A) X_SEP \ + /* misc */ \ + X_VAL(n) X_SEP X_VAL(p) +// clang-format on + +// This type should not be referenced, it exists only to provide labels +// internally that match the values declared in FormatConversionChar in +// str_format.h. This is meant to allow internal libraries to use the same +// declared interface type as the public interface +// (absl::StrFormatConversionChar) while keeping the definition in a public +// header. +// Internal libraries should use the form +// `FormatConversionCharInternal::c`, `FormatConversionCharInternal::kNone` for +// comparisons. Use in switch statements is not recommended due to a bug in how +// gcc 4.9 -Wswitch handles declared but undefined enums. +struct FormatConversionCharInternal { + FormatConversionCharInternal() = delete; + + private: + // clang-format off + enum class Enum : uint8_t { + c, s, // text + d, i, o, u, x, X, // int + f, F, e, E, g, G, a, A, // float + n, p, // misc + kNone + }; + // clang-format on + public: +#define ABSL_INTERNAL_X_VAL(id) \ + static constexpr FormatConversionChar id = \ + static_cast<FormatConversionChar>(Enum::id); + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_X_VAL, ) +#undef ABSL_INTERNAL_X_VAL + static constexpr FormatConversionChar kNone = + static_cast<FormatConversionChar>(Enum::kNone); +}; +// clang-format on + +inline FormatConversionChar FormatConversionCharFromChar(char c) { + switch (c) { +#define ABSL_INTERNAL_X_VAL(id) \ + case #id[0]: \ + return FormatConversionCharInternal::id; + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_X_VAL, ) +#undef ABSL_INTERNAL_X_VAL + } + return FormatConversionCharInternal::kNone; +} + +inline bool FormatConversionCharIsUpper(FormatConversionChar c) { + if (c == FormatConversionCharInternal::X || + c == FormatConversionCharInternal::F || + c == FormatConversionCharInternal::E || + c == FormatConversionCharInternal::G || + c == FormatConversionCharInternal::A) { + return true; + } else { + return false; + } +} + +inline bool FormatConversionCharIsFloat(FormatConversionChar c) { + if (c == FormatConversionCharInternal::a || + c == FormatConversionCharInternal::e || + c == FormatConversionCharInternal::f || + c == FormatConversionCharInternal::g || + c == FormatConversionCharInternal::A || + c == FormatConversionCharInternal::E || + c == FormatConversionCharInternal::F || + c == FormatConversionCharInternal::G) { + return true; + } else { + return false; + } +} + +inline char FormatConversionCharToChar(FormatConversionChar c) { + if (c == FormatConversionCharInternal::kNone) { + return '\0'; + +#define ABSL_INTERNAL_X_VAL(e) \ + } else if (c == FormatConversionCharInternal::e) { \ + return #e[0]; +#define ABSL_INTERNAL_X_SEP + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_X_VAL, + ABSL_INTERNAL_X_SEP) + } else { + return '\0'; + } + +#undef ABSL_INTERNAL_X_VAL +#undef ABSL_INTERNAL_X_SEP +} + +// The associated char. +inline std::ostream& operator<<(std::ostream& os, FormatConversionChar v) { + char c = FormatConversionCharToChar(v); + if (!c) c = '?'; + return os << c; +} + +struct FormatConversionSpecImplFriend; + +class FormatConversionSpecImpl { + public: + // Width and precison are not specified, no flags are set. + bool is_basic() const { return flags_.basic; } + bool has_left_flag() const { return flags_.left; } + bool has_show_pos_flag() const { return flags_.show_pos; } + bool has_sign_col_flag() const { return flags_.sign_col; } + bool has_alt_flag() const { return flags_.alt; } + bool has_zero_flag() const { return flags_.zero; } + + FormatConversionChar conversion_char() const { + // Keep this field first in the struct . It generates better code when + // accessing it when ConversionSpec is passed by value in registers. + static_assert(offsetof(FormatConversionSpecImpl, conv_) == 0, ""); + return conv_; + } + + // Returns the specified width. If width is unspecfied, it returns a negative + // value. + int width() const { return width_; } + // Returns the specified precision. If precision is unspecfied, it returns a + // negative value. + int precision() const { return precision_; } + + template <typename T> + T Wrap() { + return T(*this); + } + + private: + friend struct str_format_internal::FormatConversionSpecImplFriend; + FormatConversionChar conv_ = FormatConversionCharInternal::kNone; + Flags flags_; + int width_; + int precision_; +}; + +struct FormatConversionSpecImplFriend final { + static void SetFlags(Flags f, FormatConversionSpecImpl* conv) { + conv->flags_ = f; + } + static void SetConversionChar(FormatConversionChar c, + FormatConversionSpecImpl* conv) { + conv->conv_ = c; + } + static void SetWidth(int w, FormatConversionSpecImpl* conv) { + conv->width_ = w; + } + static void SetPrecision(int p, FormatConversionSpecImpl* conv) { + conv->precision_ = p; + } + static std::string FlagsToString(const FormatConversionSpecImpl& spec) { + return spec.flags_.ToString(); + } +}; + +// Type safe OR operator. +// We need this for two reasons: +// 1. operator| on enums makes them decay to integers and the result is an +// integer. We need the result to stay as an enum. +// 2. We use "enum class" which would not work even if we accepted the decay. +constexpr FormatConversionCharSet FormatConversionCharSetUnion( + FormatConversionCharSet a) { + return a; +} + +template <typename... CharSet> +constexpr FormatConversionCharSet FormatConversionCharSetUnion( + FormatConversionCharSet a, CharSet... rest) { + return static_cast<FormatConversionCharSet>( + static_cast<uint64_t>(a) | + static_cast<uint64_t>(FormatConversionCharSetUnion(rest...))); +} + +constexpr uint64_t FormatConversionCharToConvInt(FormatConversionChar c) { + return uint64_t{1} << (1 + static_cast<uint8_t>(c)); +} + +constexpr uint64_t FormatConversionCharToConvInt(char conv) { + return +#define ABSL_INTERNAL_CHAR_SET_CASE(c) \ + conv == #c[0] \ + ? FormatConversionCharToConvInt(FormatConversionCharInternal::c) \ + : + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_CHAR_SET_CASE, ) +#undef ABSL_INTERNAL_CHAR_SET_CASE + conv == '*' + ? 1 + : 0; +} + +constexpr FormatConversionCharSet FormatConversionCharToConvValue(char conv) { + return static_cast<FormatConversionCharSet>( + FormatConversionCharToConvInt(conv)); +} + +struct FormatConversionCharSetInternal { +#define ABSL_INTERNAL_CHAR_SET_CASE(c) \ + static constexpr FormatConversionCharSet c = \ + FormatConversionCharToConvValue(#c[0]); + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_CHAR_SET_CASE, ) +#undef ABSL_INTERNAL_CHAR_SET_CASE + + // Used for width/precision '*' specification. + static constexpr FormatConversionCharSet kStar = + FormatConversionCharToConvValue('*'); + + static constexpr FormatConversionCharSet kIntegral = + FormatConversionCharSetUnion(d, i, u, o, x, X); + static constexpr FormatConversionCharSet kFloating = + FormatConversionCharSetUnion(a, e, f, g, A, E, F, G); + static constexpr FormatConversionCharSet kNumeric = + FormatConversionCharSetUnion(kIntegral, kFloating); + static constexpr FormatConversionCharSet kPointer = p; +}; + +// Type safe OR operator. +// We need this for two reasons: +// 1. operator| on enums makes them decay to integers and the result is an +// integer. We need the result to stay as an enum. +// 2. We use "enum class" which would not work even if we accepted the decay. +constexpr FormatConversionCharSet operator|(FormatConversionCharSet a, + FormatConversionCharSet b) { + return FormatConversionCharSetUnion(a, b); +} + +// Overloaded conversion functions to support absl::ParsedFormat. +// Get a conversion with a single character in it. +constexpr FormatConversionCharSet ToFormatConversionCharSet(char c) { + return static_cast<FormatConversionCharSet>( + FormatConversionCharToConvValue(c)); +} + +// Get a conversion with a single character in it. +constexpr FormatConversionCharSet ToFormatConversionCharSet( + FormatConversionCharSet c) { + return c; +} + +template <typename T> +void ToFormatConversionCharSet(T) = delete; + +// Checks whether `c` exists in `set`. +constexpr bool Contains(FormatConversionCharSet set, char c) { + return (static_cast<uint64_t>(set) & + static_cast<uint64_t>(FormatConversionCharToConvValue(c))) != 0; +} + +// Checks whether all the characters in `c` are contained in `set` +constexpr bool Contains(FormatConversionCharSet set, + FormatConversionCharSet c) { + return (static_cast<uint64_t>(set) & static_cast<uint64_t>(c)) == + static_cast<uint64_t>(c); +} + +// Checks whether all the characters in `c` are contained in `set` +constexpr bool Contains(FormatConversionCharSet set, FormatConversionChar c) { + return (static_cast<uint64_t>(set) & FormatConversionCharToConvInt(c)) != 0; +} + +// Return capacity - used, clipped to a minimum of 0. +inline size_t Excess(size_t used, size_t capacity) { + return used < capacity ? capacity - used : 0; +} + +} // namespace str_format_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_EXTENSION_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/extension_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/extension_test.cc new file mode 100644 index 000000000000..1c93fdb1c75b --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/extension_test.cc @@ -0,0 +1,98 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include "absl/strings/internal/str_format/extension.h" + +#include <random> +#include <string> + +#include "gtest/gtest.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" + +namespace my_namespace { +class UserDefinedType { + public: + UserDefinedType() = default; + + void Append(absl::string_view str) { value_.append(str.data(), str.size()); } + const std::string& Value() const { return value_; } + + friend void AbslFormatFlush(UserDefinedType* x, absl::string_view str) { + x->Append(str); + } + + private: + std::string value_; +}; +} // namespace my_namespace + +namespace { + +std::string MakeRandomString(size_t len) { + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> dis('a', 'z'); + std::string s(len, '0'); + for (char& c : s) { + c = dis(gen); + } + return s; +} + +TEST(FormatExtensionTest, SinkAppendSubstring) { + for (size_t chunk_size : {1, 10, 100, 1000, 10000}) { + std::string expected, actual; + absl::str_format_internal::FormatSinkImpl sink(&actual); + for (size_t chunks = 0; chunks < 10; ++chunks) { + std::string rand = MakeRandomString(chunk_size); + expected += rand; + sink.Append(rand); + } + sink.Flush(); + EXPECT_EQ(actual, expected); + } +} + +TEST(FormatExtensionTest, SinkAppendChars) { + for (size_t chunk_size : {1, 10, 100, 1000, 10000}) { + std::string expected, actual; + absl::str_format_internal::FormatSinkImpl sink(&actual); + for (size_t chunks = 0; chunks < 10; ++chunks) { + std::string rand = MakeRandomString(1); + expected.append(chunk_size, rand[0]); + sink.Append(chunk_size, rand[0]); + } + sink.Flush(); + EXPECT_EQ(actual, expected); + } +} + +TEST(FormatExtensionTest, VerifyEnumEquality) { +#define X_VAL(id) \ + EXPECT_EQ(absl::FormatConversionChar::id, \ + absl::str_format_internal::FormatConversionCharInternal::id); + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(X_VAL, ); +#undef X_VAL + +#define X_VAL(id) \ + EXPECT_EQ(absl::FormatConversionCharSet::id, \ + absl::str_format_internal::FormatConversionCharSetInternal::id); + ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(X_VAL, ); +#undef X_VAL +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/float_conversion.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/float_conversion.cc new file mode 100644 index 000000000000..0ded0a66afa9 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/float_conversion.cc @@ -0,0 +1,1419 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/float_conversion.h" + +#include <string.h> + +#include <algorithm> +#include <cassert> +#include <cmath> +#include <limits> +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/bits.h" +#include "absl/base/optimization.h" +#include "absl/functional/function_ref.h" +#include "absl/meta/type_traits.h" +#include "absl/numeric/int128.h" +#include "absl/strings/numbers.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +namespace { + +// The code below wants to avoid heap allocations. +// To do so it needs to allocate memory on the stack. +// `StackArray` will allocate memory on the stack in the form of a uint32_t +// array and call the provided callback with said memory. +// It will allocate memory in increments of 512 bytes. We could allocate the +// largest needed unconditionally, but that is more than we need in most of +// cases. This way we use less stack in the common cases. +class StackArray { + using Func = absl::FunctionRef<void(absl::Span<uint32_t>)>; + static constexpr size_t kStep = 512 / sizeof(uint32_t); + // 5 steps is 2560 bytes, which is enough to hold a long double with the + // largest/smallest exponents. + // The operations below will static_assert their particular maximum. + static constexpr size_t kNumSteps = 5; + + // We do not want this function to be inlined. + // Otherwise the caller will allocate the stack space unnecessarily for all + // the variants even though it only calls one. + template <size_t steps> + ABSL_ATTRIBUTE_NOINLINE static void RunWithCapacityImpl(Func f) { + uint32_t values[steps * kStep]{}; + f(absl::MakeSpan(values)); + } + + public: + static constexpr size_t kMaxCapacity = kStep * kNumSteps; + + static void RunWithCapacity(size_t capacity, Func f) { + assert(capacity <= kMaxCapacity); + const size_t step = (capacity + kStep - 1) / kStep; + assert(step <= kNumSteps); + switch (step) { + case 1: + return RunWithCapacityImpl<1>(f); + case 2: + return RunWithCapacityImpl<2>(f); + case 3: + return RunWithCapacityImpl<3>(f); + case 4: + return RunWithCapacityImpl<4>(f); + case 5: + return RunWithCapacityImpl<5>(f); + } + + assert(false && "Invalid capacity"); + } +}; + +// Calculates `10 * (*v) + carry` and stores the result in `*v` and returns +// the carry. +template <typename Int> +inline Int MultiplyBy10WithCarry(Int *v, Int carry) { + using BiggerInt = absl::conditional_t<sizeof(Int) == 4, uint64_t, uint128>; + BiggerInt tmp = 10 * static_cast<BiggerInt>(*v) + carry; + *v = static_cast<Int>(tmp); + return static_cast<Int>(tmp >> (sizeof(Int) * 8)); +} + +// Calculates `(2^64 * carry + *v) / 10`. +// Stores the quotient in `*v` and returns the remainder. +// Requires: `0 <= carry <= 9` +inline uint64_t DivideBy10WithCarry(uint64_t *v, uint64_t carry) { + constexpr uint64_t divisor = 10; + // 2^64 / divisor = chunk_quotient + chunk_remainder / divisor + constexpr uint64_t chunk_quotient = (uint64_t{1} << 63) / (divisor / 2); + constexpr uint64_t chunk_remainder = uint64_t{} - chunk_quotient * divisor; + + const uint64_t mod = *v % divisor; + const uint64_t next_carry = chunk_remainder * carry + mod; + *v = *v / divisor + carry * chunk_quotient + next_carry / divisor; + return next_carry % divisor; +} + +// Generates the decimal representation for an integer of the form `v * 2^exp`, +// where `v` and `exp` are both positive integers. +// It generates the digits from the left (ie the most significant digit first) +// to allow for direct printing into the sink. +// +// Requires `0 <= exp` and `exp <= numeric_limits<long double>::max_exponent`. +class BinaryToDecimal { + static constexpr int ChunksNeeded(int exp) { + // We will left shift a uint128 by `exp` bits, so we need `128+exp` total + // bits. Round up to 32. + // See constructor for details about adding `10%` to the value. + return (128 + exp + 31) / 32 * 11 / 10; + } + + public: + // Run the conversion for `v * 2^exp` and call `f(binary_to_decimal)`. + // This function will allocate enough stack space to perform the conversion. + static void RunConversion(uint128 v, int exp, + absl::FunctionRef<void(BinaryToDecimal)> f) { + assert(exp > 0); + assert(exp <= std::numeric_limits<long double>::max_exponent); + static_assert( + static_cast<int>(StackArray::kMaxCapacity) >= + ChunksNeeded(std::numeric_limits<long double>::max_exponent), + ""); + + StackArray::RunWithCapacity( + ChunksNeeded(exp), + [=](absl::Span<uint32_t> input) { f(BinaryToDecimal(input, v, exp)); }); + } + + int TotalDigits() const { + return static_cast<int>((decimal_end_ - decimal_start_) * kDigitsPerChunk + + CurrentDigits().size()); + } + + // See the current block of digits. + absl::string_view CurrentDigits() const { + return absl::string_view(digits_ + kDigitsPerChunk - size_, size_); + } + + // Advance the current view of digits. + // Returns `false` when no more digits are available. + bool AdvanceDigits() { + if (decimal_start_ >= decimal_end_) return false; + + uint32_t w = data_[decimal_start_++]; + for (size_ = 0; size_ < kDigitsPerChunk; w /= 10) { + digits_[kDigitsPerChunk - ++size_] = w % 10 + '0'; + } + return true; + } + + private: + BinaryToDecimal(absl::Span<uint32_t> data, uint128 v, int exp) : data_(data) { + // We need to print the digits directly into the sink object without + // buffering them all first. To do this we need two things: + // - to know the total number of digits to do padding when necessary + // - to generate the decimal digits from the left. + // + // In order to do this, we do a two pass conversion. + // On the first pass we convert the binary representation of the value into + // a decimal representation in which each uint32_t chunk holds up to 9 + // decimal digits. In the second pass we take each decimal-holding-uint32_t + // value and generate the ascii decimal digits into `digits_`. + // + // The binary and decimal representations actually share the same memory + // region. As we go converting the chunks from binary to decimal we free + // them up and reuse them for the decimal representation. One caveat is that + // the decimal representation is around 7% less efficient in space than the + // binary one. We allocate an extra 10% memory to account for this. See + // ChunksNeeded for this calculation. + int chunk_index = exp / 32; + decimal_start_ = decimal_end_ = ChunksNeeded(exp); + const int offset = exp % 32; + // Left shift v by exp bits. + data_[chunk_index] = static_cast<uint32_t>(v << offset); + for (v >>= (32 - offset); v; v >>= 32) + data_[++chunk_index] = static_cast<uint32_t>(v); + + while (chunk_index >= 0) { + // While we have more than one chunk available, go in steps of 1e9. + // `data_[chunk_index]` holds the highest non-zero binary chunk, so keep + // the variable updated. + uint32_t carry = 0; + for (int i = chunk_index; i >= 0; --i) { + uint64_t tmp = uint64_t{data_[i]} + (uint64_t{carry} << 32); + data_[i] = static_cast<uint32_t>(tmp / uint64_t{1000000000}); + carry = static_cast<uint32_t>(tmp % uint64_t{1000000000}); + } + + // If the highest chunk is now empty, remove it from view. + if (data_[chunk_index] == 0) --chunk_index; + + --decimal_start_; + assert(decimal_start_ != chunk_index); + data_[decimal_start_] = carry; + } + + // Fill the first set of digits. The first chunk might not be complete, so + // handle differently. + for (uint32_t first = data_[decimal_start_++]; first != 0; first /= 10) { + digits_[kDigitsPerChunk - ++size_] = first % 10 + '0'; + } + } + + private: + static constexpr int kDigitsPerChunk = 9; + + int decimal_start_; + int decimal_end_; + + char digits_[kDigitsPerChunk]; + int size_ = 0; + + absl::Span<uint32_t> data_; +}; + +// Converts a value of the form `x * 2^-exp` into a sequence of decimal digits. +// Requires `-exp < 0` and +// `-exp >= limits<long double>::min_exponent - limits<long double>::digits`. +class FractionalDigitGenerator { + public: + // Run the conversion for `v * 2^exp` and call `f(generator)`. + // This function will allocate enough stack space to perform the conversion. + static void RunConversion( + uint128 v, int exp, absl::FunctionRef<void(FractionalDigitGenerator)> f) { + using Limits = std::numeric_limits<long double>; + assert(-exp < 0); + assert(-exp >= Limits::min_exponent - 128); + static_assert(StackArray::kMaxCapacity >= + (Limits::digits + 128 - Limits::min_exponent + 31) / 32, + ""); + StackArray::RunWithCapacity((Limits::digits + exp + 31) / 32, + [=](absl::Span<uint32_t> input) { + f(FractionalDigitGenerator(input, v, exp)); + }); + } + + // Returns true if there are any more non-zero digits left. + bool HasMoreDigits() const { return next_digit_ != 0 || chunk_index_ >= 0; } + + // Returns true if the remainder digits are greater than 5000... + bool IsGreaterThanHalf() const { + return next_digit_ > 5 || (next_digit_ == 5 && chunk_index_ >= 0); + } + // Returns true if the remainder digits are exactly 5000... + bool IsExactlyHalf() const { return next_digit_ == 5 && chunk_index_ < 0; } + + struct Digits { + int digit_before_nine; + int num_nines; + }; + + // Get the next set of digits. + // They are composed by a non-9 digit followed by a runs of zero or more 9s. + Digits GetDigits() { + Digits digits{next_digit_, 0}; + + next_digit_ = GetOneDigit(); + while (next_digit_ == 9) { + ++digits.num_nines; + next_digit_ = GetOneDigit(); + } + + return digits; + } + + private: + // Return the next digit. + int GetOneDigit() { + if (chunk_index_ < 0) return 0; + + uint32_t carry = 0; + for (int i = chunk_index_; i >= 0; --i) { + carry = MultiplyBy10WithCarry(&data_[i], carry); + } + // If the lowest chunk is now empty, remove it from view. + if (data_[chunk_index_] == 0) --chunk_index_; + return carry; + } + + FractionalDigitGenerator(absl::Span<uint32_t> data, uint128 v, int exp) + : chunk_index_(exp / 32), data_(data) { + const int offset = exp % 32; + // Right shift `v` by `exp` bits. + data_[chunk_index_] = static_cast<uint32_t>(v << (32 - offset)); + v >>= offset; + // Make sure we don't overflow the data. We already calculated that + // non-zero bits fit, so we might not have space for leading zero bits. + for (int pos = chunk_index_; v; v >>= 32) + data_[--pos] = static_cast<uint32_t>(v); + + // Fill next_digit_, as GetDigits expects it to be populated always. + next_digit_ = GetOneDigit(); + } + + int next_digit_; + int chunk_index_; + absl::Span<uint32_t> data_; +}; + +// Count the number of leading zero bits. +int LeadingZeros(uint64_t v) { return base_internal::CountLeadingZeros64(v); } +int LeadingZeros(uint128 v) { + auto high = static_cast<uint64_t>(v >> 64); + auto low = static_cast<uint64_t>(v); + return high != 0 ? base_internal::CountLeadingZeros64(high) + : 64 + base_internal::CountLeadingZeros64(low); +} + +// Round up the text digits starting at `p`. +// The buffer must have an extra digit that is known to not need rounding. +// This is done below by having an extra '0' digit on the left. +void RoundUp(char *p) { + while (*p == '9' || *p == '.') { + if (*p == '9') *p = '0'; + --p; + } + ++*p; +} + +// Check the previous digit and round up or down to follow the round-to-even +// policy. +void RoundToEven(char *p) { + if (*p == '.') --p; + if (*p % 2 == 1) RoundUp(p); +} + +// Simple integral decimal digit printing for values that fit in 64-bits. +// Returns the pointer to the last written digit. +char *PrintIntegralDigitsFromRightFast(uint64_t v, char *p) { + do { + *--p = DivideBy10WithCarry(&v, 0) + '0'; + } while (v != 0); + return p; +} + +// Simple integral decimal digit printing for values that fit in 128-bits. +// Returns the pointer to the last written digit. +char *PrintIntegralDigitsFromRightFast(uint128 v, char *p) { + auto high = static_cast<uint64_t>(v >> 64); + auto low = static_cast<uint64_t>(v); + + while (high != 0) { + uint64_t carry = DivideBy10WithCarry(&high, 0); + carry = DivideBy10WithCarry(&low, carry); + *--p = carry + '0'; + } + return PrintIntegralDigitsFromRightFast(low, p); +} + +// Simple fractional decimal digit printing for values that fir in 64-bits after +// shifting. +// Performs rounding if necessary to fit within `precision`. +// Returns the pointer to one after the last character written. +char *PrintFractionalDigitsFast(uint64_t v, char *start, int exp, + int precision) { + char *p = start; + v <<= (64 - exp); + while (precision > 0) { + if (!v) return p; + *p++ = MultiplyBy10WithCarry(&v, uint64_t{0}) + '0'; + --precision; + } + + // We need to round. + if (v < 0x8000000000000000) { + // We round down, so nothing to do. + } else if (v > 0x8000000000000000) { + // We round up. + RoundUp(p - 1); + } else { + RoundToEven(p - 1); + } + + assert(precision == 0); + // Precision can only be zero here. + return p; +} + +// Simple fractional decimal digit printing for values that fir in 128-bits +// after shifting. +// Performs rounding if necessary to fit within `precision`. +// Returns the pointer to one after the last character written. +char *PrintFractionalDigitsFast(uint128 v, char *start, int exp, + int precision) { + char *p = start; + v <<= (128 - exp); + auto high = static_cast<uint64_t>(v >> 64); + auto low = static_cast<uint64_t>(v); + + // While we have digits to print and `low` is not empty, do the long + // multiplication. + while (precision > 0 && low != 0) { + uint64_t carry = MultiplyBy10WithCarry(&low, uint64_t{0}); + carry = MultiplyBy10WithCarry(&high, carry); + + *p++ = carry + '0'; + --precision; + } + + // Now `low` is empty, so use a faster approach for the rest of the digits. + // This block is pretty much the same as the main loop for the 64-bit case + // above. + while (precision > 0) { + if (!high) return p; + *p++ = MultiplyBy10WithCarry(&high, uint64_t{0}) + '0'; + --precision; + } + + // We need to round. + if (high < 0x8000000000000000) { + // We round down, so nothing to do. + } else if (high > 0x8000000000000000 || low != 0) { + // We round up. + RoundUp(p - 1); + } else { + RoundToEven(p - 1); + } + + assert(precision == 0); + // Precision can only be zero here. + return p; +} + +struct FormatState { + char sign_char; + int precision; + const FormatConversionSpecImpl &conv; + FormatSinkImpl *sink; + + // In `alt` mode (flag #) we keep the `.` even if there are no fractional + // digits. In non-alt mode, we strip it. + bool ShouldPrintDot() const { return precision != 0 || conv.has_alt_flag(); } +}; + +struct Padding { + int left_spaces; + int zeros; + int right_spaces; +}; + +Padding ExtraWidthToPadding(size_t total_size, const FormatState &state) { + if (state.conv.width() < 0 || + static_cast<size_t>(state.conv.width()) <= total_size) { + return {0, 0, 0}; + } + int missing_chars = state.conv.width() - total_size; + if (state.conv.has_left_flag()) { + return {0, 0, missing_chars}; + } else if (state.conv.has_zero_flag()) { + return {0, missing_chars, 0}; + } else { + return {missing_chars, 0, 0}; + } +} + +void FinalPrint(const FormatState &state, absl::string_view data, + int padding_offset, int trailing_zeros, + absl::string_view data_postfix) { + if (state.conv.width() < 0) { + // No width specified. Fast-path. + if (state.sign_char != '\0') state.sink->Append(1, state.sign_char); + state.sink->Append(data); + state.sink->Append(trailing_zeros, '0'); + state.sink->Append(data_postfix); + return; + } + + auto padding = ExtraWidthToPadding((state.sign_char != '\0' ? 1 : 0) + + data.size() + data_postfix.size() + + static_cast<size_t>(trailing_zeros), + state); + + state.sink->Append(padding.left_spaces, ' '); + if (state.sign_char != '\0') state.sink->Append(1, state.sign_char); + // Padding in general needs to be inserted somewhere in the middle of `data`. + state.sink->Append(data.substr(0, padding_offset)); + state.sink->Append(padding.zeros, '0'); + state.sink->Append(data.substr(padding_offset)); + state.sink->Append(trailing_zeros, '0'); + state.sink->Append(data_postfix); + state.sink->Append(padding.right_spaces, ' '); +} + +// Fastpath %f formatter for when the shifted value fits in a simple integral +// type. +// Prints `v*2^exp` with the options from `state`. +template <typename Int> +void FormatFFast(Int v, int exp, const FormatState &state) { + constexpr int input_bits = sizeof(Int) * 8; + + static constexpr size_t integral_size = + /* in case we need to round up an extra digit */ 1 + + /* decimal digits for uint128 */ 40 + 1; + char buffer[integral_size + /* . */ 1 + /* max digits uint128 */ 128]; + buffer[integral_size] = '.'; + char *const integral_digits_end = buffer + integral_size; + char *integral_digits_start; + char *const fractional_digits_start = buffer + integral_size + 1; + char *fractional_digits_end = fractional_digits_start; + + if (exp >= 0) { + const int total_bits = input_bits - LeadingZeros(v) + exp; + integral_digits_start = + total_bits <= 64 + ? PrintIntegralDigitsFromRightFast(static_cast<uint64_t>(v) << exp, + integral_digits_end) + : PrintIntegralDigitsFromRightFast(static_cast<uint128>(v) << exp, + integral_digits_end); + } else { + exp = -exp; + + integral_digits_start = PrintIntegralDigitsFromRightFast( + exp < input_bits ? v >> exp : 0, integral_digits_end); + // PrintFractionalDigits may pull a carried 1 all the way up through the + // integral portion. + integral_digits_start[-1] = '0'; + + fractional_digits_end = + exp <= 64 ? PrintFractionalDigitsFast(v, fractional_digits_start, exp, + state.precision) + : PrintFractionalDigitsFast(static_cast<uint128>(v), + fractional_digits_start, exp, + state.precision); + // There was a carry, so include the first digit too. + if (integral_digits_start[-1] != '0') --integral_digits_start; + } + + size_t size = fractional_digits_end - integral_digits_start; + + // In `alt` mode (flag #) we keep the `.` even if there are no fractional + // digits. In non-alt mode, we strip it. + if (!state.ShouldPrintDot()) --size; + FinalPrint(state, absl::string_view(integral_digits_start, size), + /*padding_offset=*/0, + static_cast<int>(state.precision - (fractional_digits_end - + fractional_digits_start)), + /*data_postfix=*/""); +} + +// Slow %f formatter for when the shifted value does not fit in a uint128, and +// `exp > 0`. +// Prints `v*2^exp` with the options from `state`. +// This one is guaranteed to not have fractional digits, so we don't have to +// worry about anything after the `.`. +void FormatFPositiveExpSlow(uint128 v, int exp, const FormatState &state) { + BinaryToDecimal::RunConversion(v, exp, [&](BinaryToDecimal btd) { + const size_t total_digits = + btd.TotalDigits() + + (state.ShouldPrintDot() ? static_cast<size_t>(state.precision) + 1 : 0); + + const auto padding = ExtraWidthToPadding( + total_digits + (state.sign_char != '\0' ? 1 : 0), state); + + state.sink->Append(padding.left_spaces, ' '); + if (state.sign_char != '\0') state.sink->Append(1, state.sign_char); + state.sink->Append(padding.zeros, '0'); + + do { + state.sink->Append(btd.CurrentDigits()); + } while (btd.AdvanceDigits()); + + if (state.ShouldPrintDot()) state.sink->Append(1, '.'); + state.sink->Append(state.precision, '0'); + state.sink->Append(padding.right_spaces, ' '); + }); +} + +// Slow %f formatter for when the shifted value does not fit in a uint128, and +// `exp < 0`. +// Prints `v*2^exp` with the options from `state`. +// This one is guaranteed to be < 1.0, so we don't have to worry about integral +// digits. +void FormatFNegativeExpSlow(uint128 v, int exp, const FormatState &state) { + const size_t total_digits = + /* 0 */ 1 + + (state.ShouldPrintDot() ? static_cast<size_t>(state.precision) + 1 : 0); + auto padding = + ExtraWidthToPadding(total_digits + (state.sign_char ? 1 : 0), state); + padding.zeros += 1; + state.sink->Append(padding.left_spaces, ' '); + if (state.sign_char != '\0') state.sink->Append(1, state.sign_char); + state.sink->Append(padding.zeros, '0'); + + if (state.ShouldPrintDot()) state.sink->Append(1, '.'); + + // Print digits + int digits_to_go = state.precision; + + FractionalDigitGenerator::RunConversion( + v, exp, [&](FractionalDigitGenerator digit_gen) { + // There are no digits to print here. + if (state.precision == 0) return; + + // We go one digit at a time, while keeping track of runs of nines. + // The runs of nines are used to perform rounding when necessary. + + while (digits_to_go > 0 && digit_gen.HasMoreDigits()) { + auto digits = digit_gen.GetDigits(); + + // Now we have a digit and a run of nines. + // See if we can print them all. + if (digits.num_nines + 1 < digits_to_go) { + // We don't have to round yet, so print them. + state.sink->Append(1, digits.digit_before_nine + '0'); + state.sink->Append(digits.num_nines, '9'); + digits_to_go -= digits.num_nines + 1; + + } else { + // We can't print all the nines, see where we have to truncate. + + bool round_up = false; + if (digits.num_nines + 1 > digits_to_go) { + // We round up at a nine. No need to print them. + round_up = true; + } else { + // We can fit all the nines, but truncate just after it. + if (digit_gen.IsGreaterThanHalf()) { + round_up = true; + } else if (digit_gen.IsExactlyHalf()) { + // Round to even + round_up = + digits.num_nines != 0 || digits.digit_before_nine % 2 == 1; + } + } + + if (round_up) { + state.sink->Append(1, digits.digit_before_nine + '1'); + --digits_to_go; + // The rest will be zeros. + } else { + state.sink->Append(1, digits.digit_before_nine + '0'); + state.sink->Append(digits_to_go - 1, '9'); + digits_to_go = 0; + } + return; + } + } + }); + + state.sink->Append(digits_to_go, '0'); + state.sink->Append(padding.right_spaces, ' '); +} + +template <typename Int> +void FormatF(Int mantissa, int exp, const FormatState &state) { + if (exp >= 0) { + const int total_bits = sizeof(Int) * 8 - LeadingZeros(mantissa) + exp; + + // Fallback to the slow stack-based approach if we can't do it in a 64 or + // 128 bit state. + if (ABSL_PREDICT_FALSE(total_bits > 128)) { + return FormatFPositiveExpSlow(mantissa, exp, state); + } + } else { + // Fallback to the slow stack-based approach if we can't do it in a 64 or + // 128 bit state. + if (ABSL_PREDICT_FALSE(exp < -128)) { + return FormatFNegativeExpSlow(mantissa, -exp, state); + } + } + return FormatFFast(mantissa, exp, state); +} + +// Grab the group of four bits (nibble) from `n`. E.g., nibble 1 corresponds to +// bits 4-7. +template <typename Int> +uint8_t GetNibble(Int n, int nibble_index) { + constexpr Int mask_low_nibble = Int{0xf}; + int shift = nibble_index * 4; + n &= mask_low_nibble << shift; + return static_cast<uint8_t>((n >> shift) & 0xf); +} + +// Add one to the given nibble, applying carry to higher nibbles. Returns true +// if overflow, false otherwise. +template <typename Int> +bool IncrementNibble(int nibble_index, Int *n) { + constexpr int kShift = sizeof(Int) * 8 - 1; + constexpr int kNumNibbles = sizeof(Int) * 8 / 4; + Int before = *n >> kShift; + // Here we essentially want to take the number 1 and move it into the requsted + // nibble, then add it to *n to effectively increment the nibble. However, + // ASan will complain if we try to shift the 1 beyond the limits of the Int, + // i.e., if the nibble_index is out of range. So therefore we check for this + // and if we are out of range we just add 0 which leaves *n unchanged, which + // seems like the reasonable thing to do in that case. + *n += ((nibble_index >= kNumNibbles) ? 0 : (Int{1} << (nibble_index * 4))); + Int after = *n >> kShift; + return (before && !after) || (nibble_index >= kNumNibbles); +} + +// Return a mask with 1's in the given nibble and all lower nibbles. +template <typename Int> +Int MaskUpToNibbleInclusive(int nibble_index) { + constexpr int kNumNibbles = sizeof(Int) * 8 / 4; + static const Int ones = ~Int{0}; + return ones >> std::max(0, 4 * (kNumNibbles - nibble_index - 1)); +} + +// Return a mask with 1's below the given nibble. +template <typename Int> +Int MaskUpToNibbleExclusive(int nibble_index) { + return nibble_index <= 0 ? 0 : MaskUpToNibbleInclusive<Int>(nibble_index - 1); +} + +template <typename Int> +Int MoveToNibble(uint8_t nibble, int nibble_index) { + return Int{nibble} << (4 * nibble_index); +} + +// Given mantissa size, find optimal # of mantissa bits to put in initial digit. +// +// In the hex representation we keep a single hex digit to the left of the dot. +// However, the question as to how many bits of the mantissa should be put into +// that hex digit in theory is arbitrary, but in practice it is optimal to +// choose based on the size of the mantissa. E.g., for a `double`, there are 53 +// mantissa bits, so that means that we should put 1 bit to the left of the dot, +// thereby leaving 52 bits to the right, which is evenly divisible by four and +// thus all fractional digits represent actual precision. For a `long double`, +// on the other hand, there are 64 bits of mantissa, thus we can use all four +// bits for the initial hex digit and still have a number left over (60) that is +// a multiple of four. Once again, the goal is to have all fractional digits +// represent real precision. +template <typename Float> +constexpr int HexFloatLeadingDigitSizeInBits() { + return std::numeric_limits<Float>::digits % 4 > 0 + ? std::numeric_limits<Float>::digits % 4 + : 4; +} + +// This function captures the rounding behavior of glibc for hex float +// representations. E.g. when rounding 0x1.ab800000 to a precision of .2 +// ("%.2a") glibc will round up because it rounds toward the even number (since +// 0xb is an odd number, it will round up to 0xc). However, when rounding at a +// point that is not followed by 800000..., it disregards the parity and rounds +// up if > 8 and rounds down if < 8. +template <typename Int> +bool HexFloatNeedsRoundUp(Int mantissa, int final_nibble_displayed, + uint8_t leading) { + // If the last nibble (hex digit) to be displayed is the lowest on in the + // mantissa then that means that we don't have any further nibbles to inform + // rounding, so don't round. + if (final_nibble_displayed <= 0) { + return false; + } + int rounding_nibble_idx = final_nibble_displayed - 1; + constexpr int kTotalNibbles = sizeof(Int) * 8 / 4; + assert(final_nibble_displayed <= kTotalNibbles); + Int mantissa_up_to_rounding_nibble_inclusive = + mantissa & MaskUpToNibbleInclusive<Int>(rounding_nibble_idx); + Int eight = MoveToNibble<Int>(8, rounding_nibble_idx); + if (mantissa_up_to_rounding_nibble_inclusive != eight) { + return mantissa_up_to_rounding_nibble_inclusive > eight; + } + // Nibble in question == 8. + uint8_t round_if_odd = (final_nibble_displayed == kTotalNibbles) + ? leading + : GetNibble(mantissa, final_nibble_displayed); + return round_if_odd % 2 == 1; +} + +// Stores values associated with a Float type needed by the FormatA +// implementation in order to avoid templatizing that function by the Float +// type. +struct HexFloatTypeParams { + template <typename Float> + explicit HexFloatTypeParams(Float) + : min_exponent(std::numeric_limits<Float>::min_exponent - 1), + leading_digit_size_bits(HexFloatLeadingDigitSizeInBits<Float>()) { + assert(leading_digit_size_bits >= 1 && leading_digit_size_bits <= 4); + } + + int min_exponent; + int leading_digit_size_bits; +}; + +// Hex Float Rounding. First check if we need to round; if so, then we do that +// by manipulating (incrementing) the mantissa, that way we can later print the +// mantissa digits by iterating through them in the same way regardless of +// whether a rounding happened. +template <typename Int> +void FormatARound(bool precision_specified, const FormatState &state, + uint8_t *leading, Int *mantissa, int *exp) { + constexpr int kTotalNibbles = sizeof(Int) * 8 / 4; + // Index of the last nibble that we could display given precision. + int final_nibble_displayed = + precision_specified ? std::max(0, (kTotalNibbles - state.precision)) : 0; + if (HexFloatNeedsRoundUp(*mantissa, final_nibble_displayed, *leading)) { + // Need to round up. + bool overflow = IncrementNibble(final_nibble_displayed, mantissa); + *leading += (overflow ? 1 : 0); + if (ABSL_PREDICT_FALSE(*leading > 15)) { + // We have overflowed the leading digit. This would mean that we would + // need two hex digits to the left of the dot, which is not allowed. So + // adjust the mantissa and exponent so that the result is always 1.0eXXX. + *leading = 1; + *mantissa = 0; + *exp += 4; + } + } + // Now that we have handled a possible round-up we can go ahead and zero out + // all the nibbles of the mantissa that we won't need. + if (precision_specified) { + *mantissa &= ~MaskUpToNibbleExclusive<Int>(final_nibble_displayed); + } +} + +template <typename Int> +void FormatANormalize(const HexFloatTypeParams float_traits, uint8_t *leading, + Int *mantissa, int *exp) { + constexpr int kIntBits = sizeof(Int) * 8; + static const Int kHighIntBit = Int{1} << (kIntBits - 1); + const int kLeadDigitBitsCount = float_traits.leading_digit_size_bits; + // Normalize mantissa so that highest bit set is in MSB position, unless we + // get interrupted by the exponent threshold. + while (*mantissa && !(*mantissa & kHighIntBit)) { + if (ABSL_PREDICT_FALSE(*exp - 1 < float_traits.min_exponent)) { + *mantissa >>= (float_traits.min_exponent - *exp); + *exp = float_traits.min_exponent; + return; + } + *mantissa <<= 1; + --*exp; + } + // Extract bits for leading digit then shift them away leaving the + // fractional part. + *leading = + static_cast<uint8_t>(*mantissa >> (kIntBits - kLeadDigitBitsCount)); + *exp -= (*mantissa != 0) ? kLeadDigitBitsCount : *exp; + *mantissa <<= kLeadDigitBitsCount; +} + +template <typename Int> +void FormatA(const HexFloatTypeParams float_traits, Int mantissa, int exp, + bool uppercase, const FormatState &state) { + // Int properties. + constexpr int kIntBits = sizeof(Int) * 8; + constexpr int kTotalNibbles = sizeof(Int) * 8 / 4; + // Did the user specify a precision explicitly? + const bool precision_specified = state.conv.precision() >= 0; + + // ========== Normalize/Denormalize ========== + exp += kIntBits; // make all digits fractional digits. + // This holds the (up to four) bits of leading digit, i.e., the '1' in the + // number 0x1.e6fp+2. It's always > 0 unless number is zero or denormal. + uint8_t leading = 0; + FormatANormalize(float_traits, &leading, &mantissa, &exp); + + // =============== Rounding ================== + // Check if we need to round; if so, then we do that by manipulating + // (incrementing) the mantissa before beginning to print characters. + FormatARound(precision_specified, state, &leading, &mantissa, &exp); + + // ============= Format Result =============== + // This buffer holds the "0x1.ab1de3" portion of "0x1.ab1de3pe+2". Compute the + // size with long double which is the largest of the floats. + constexpr size_t kBufSizeForHexFloatRepr = + 2 // 0x + + std::numeric_limits<long double>::digits / 4 // number of hex digits + + 1 // round up + + 1; // "." (dot) + char digits_buffer[kBufSizeForHexFloatRepr]; + char *digits_iter = digits_buffer; + const char *const digits = + static_cast<const char *>("0123456789ABCDEF0123456789abcdef") + + (uppercase ? 0 : 16); + + // =============== Hex Prefix ================ + *digits_iter++ = '0'; + *digits_iter++ = uppercase ? 'X' : 'x'; + + // ========== Non-Fractional Digit =========== + *digits_iter++ = digits[leading]; + + // ================== Dot ==================== + // There are three reasons we might need a dot. Keep in mind that, at this + // point, the mantissa holds only the fractional part. + if ((precision_specified && state.precision > 0) || + (!precision_specified && mantissa > 0) || state.conv.has_alt_flag()) { + *digits_iter++ = '.'; + } + + // ============ Fractional Digits ============ + int digits_emitted = 0; + while (mantissa > 0) { + *digits_iter++ = digits[GetNibble(mantissa, kTotalNibbles - 1)]; + mantissa <<= 4; + ++digits_emitted; + } + int trailing_zeros = + precision_specified ? state.precision - digits_emitted : 0; + assert(trailing_zeros >= 0); + auto digits_result = string_view(digits_buffer, digits_iter - digits_buffer); + + // =============== Exponent ================== + constexpr size_t kBufSizeForExpDecRepr = + numbers_internal::kFastToBufferSize // requred for FastIntToBuffer + + 1 // 'p' or 'P' + + 1; // '+' or '-' + char exp_buffer[kBufSizeForExpDecRepr]; + exp_buffer[0] = uppercase ? 'P' : 'p'; + exp_buffer[1] = exp >= 0 ? '+' : '-'; + numbers_internal::FastIntToBuffer(exp < 0 ? -exp : exp, exp_buffer + 2); + + // ============ Assemble Result ============== + FinalPrint(state, // + digits_result, // 0xN.NNN... + 2, // offset in `data` to start padding if needed. + trailing_zeros, // num remaining mantissa padding zeros + exp_buffer); // exponent +} + +char *CopyStringTo(absl::string_view v, char *out) { + std::memcpy(out, v.data(), v.size()); + return out + v.size(); +} + +template <typename Float> +bool FallbackToSnprintf(const Float v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + int w = conv.width() >= 0 ? conv.width() : 0; + int p = conv.precision() >= 0 ? conv.precision() : -1; + char fmt[32]; + { + char *fp = fmt; + *fp++ = '%'; + fp = CopyStringTo(FormatConversionSpecImplFriend::FlagsToString(conv), fp); + fp = CopyStringTo("*.*", fp); + if (std::is_same<long double, Float>()) { + *fp++ = 'L'; + } + *fp++ = FormatConversionCharToChar(conv.conversion_char()); + *fp = 0; + assert(fp < fmt + sizeof(fmt)); + } + std::string space(512, '\0'); + absl::string_view result; + while (true) { + int n = snprintf(&space[0], space.size(), fmt, w, p, v); + if (n < 0) return false; + if (static_cast<size_t>(n) < space.size()) { + result = absl::string_view(space.data(), n); + break; + } + space.resize(n + 1); + } + sink->Append(result); + return true; +} + +// 128-bits in decimal: ceil(128*log(2)/log(10)) +// or std::numeric_limits<__uint128_t>::digits10 +constexpr int kMaxFixedPrecision = 39; + +constexpr int kBufferLength = /*sign*/ 1 + + /*integer*/ kMaxFixedPrecision + + /*point*/ 1 + + /*fraction*/ kMaxFixedPrecision + + /*exponent e+123*/ 5; + +struct Buffer { + void push_front(char c) { + assert(begin > data); + *--begin = c; + } + void push_back(char c) { + assert(end < data + sizeof(data)); + *end++ = c; + } + void pop_back() { + assert(begin < end); + --end; + } + + char &back() { + assert(begin < end); + return end[-1]; + } + + char last_digit() const { return end[-1] == '.' ? end[-2] : end[-1]; } + + int size() const { return static_cast<int>(end - begin); } + + char data[kBufferLength]; + char *begin; + char *end; +}; + +enum class FormatStyle { Fixed, Precision }; + +// If the value is Inf or Nan, print it and return true. +// Otherwise, return false. +template <typename Float> +bool ConvertNonNumericFloats(char sign_char, Float v, + const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + char text[4], *ptr = text; + if (sign_char != '\0') *ptr++ = sign_char; + if (std::isnan(v)) { + ptr = std::copy_n( + FormatConversionCharIsUpper(conv.conversion_char()) ? "NAN" : "nan", 3, + ptr); + } else if (std::isinf(v)) { + ptr = std::copy_n( + FormatConversionCharIsUpper(conv.conversion_char()) ? "INF" : "inf", 3, + ptr); + } else { + return false; + } + + return sink->PutPaddedString(string_view(text, ptr - text), conv.width(), -1, + conv.has_left_flag()); +} + +// Round up the last digit of the value. +// It will carry over and potentially overflow. 'exp' will be adjusted in that +// case. +template <FormatStyle mode> +void RoundUp(Buffer *buffer, int *exp) { + char *p = &buffer->back(); + while (p >= buffer->begin && (*p == '9' || *p == '.')) { + if (*p == '9') *p = '0'; + --p; + } + + if (p < buffer->begin) { + *p = '1'; + buffer->begin = p; + if (mode == FormatStyle::Precision) { + std::swap(p[1], p[2]); // move the . + ++*exp; + buffer->pop_back(); + } + } else { + ++*p; + } +} + +void PrintExponent(int exp, char e, Buffer *out) { + out->push_back(e); + if (exp < 0) { + out->push_back('-'); + exp = -exp; + } else { + out->push_back('+'); + } + // Exponent digits. + if (exp > 99) { + out->push_back(exp / 100 + '0'); + out->push_back(exp / 10 % 10 + '0'); + out->push_back(exp % 10 + '0'); + } else { + out->push_back(exp / 10 + '0'); + out->push_back(exp % 10 + '0'); + } +} + +template <typename Float, typename Int> +constexpr bool CanFitMantissa() { + return +#if defined(__clang__) && !defined(__SSE3__) + // Workaround for clang bug: https://bugs.llvm.org/show_bug.cgi?id=38289 + // Casting from long double to uint64_t is miscompiled and drops bits. + (!std::is_same<Float, long double>::value || + !std::is_same<Int, uint64_t>::value) && +#endif + std::numeric_limits<Float>::digits <= std::numeric_limits<Int>::digits; +} + +template <typename Float> +struct Decomposed { + using MantissaType = + absl::conditional_t<std::is_same<long double, Float>::value, uint128, + uint64_t>; + static_assert(std::numeric_limits<Float>::digits <= sizeof(MantissaType) * 8, + ""); + MantissaType mantissa; + int exponent; +}; + +// Decompose the double into an integer mantissa and an exponent. +template <typename Float> +Decomposed<Float> Decompose(Float v) { + int exp; + Float m = std::frexp(v, &exp); + m = std::ldexp(m, std::numeric_limits<Float>::digits); + exp -= std::numeric_limits<Float>::digits; + + return {static_cast<typename Decomposed<Float>::MantissaType>(m), exp}; +} + +// Print 'digits' as decimal. +// In Fixed mode, we add a '.' at the end. +// In Precision mode, we add a '.' after the first digit. +template <FormatStyle mode, typename Int> +int PrintIntegralDigits(Int digits, Buffer *out) { + int printed = 0; + if (digits) { + for (; digits; digits /= 10) out->push_front(digits % 10 + '0'); + printed = out->size(); + if (mode == FormatStyle::Precision) { + out->push_front(*out->begin); + out->begin[1] = '.'; + } else { + out->push_back('.'); + } + } else if (mode == FormatStyle::Fixed) { + out->push_front('0'); + out->push_back('.'); + printed = 1; + } + return printed; +} + +// Back out 'extra_digits' digits and round up if necessary. +bool RemoveExtraPrecision(int extra_digits, bool has_leftover_value, + Buffer *out, int *exp_out) { + if (extra_digits <= 0) return false; + + // Back out the extra digits + out->end -= extra_digits; + + bool needs_to_round_up = [&] { + // We look at the digit just past the end. + // There must be 'extra_digits' extra valid digits after end. + if (*out->end > '5') return true; + if (*out->end < '5') return false; + if (has_leftover_value || std::any_of(out->end + 1, out->end + extra_digits, + [](char c) { return c != '0'; })) + return true; + + // Ends in ...50*, round to even. + return out->last_digit() % 2 == 1; + }(); + + if (needs_to_round_up) { + RoundUp<FormatStyle::Precision>(out, exp_out); + } + return true; +} + +// Print the value into the buffer. +// This will not include the exponent, which will be returned in 'exp_out' for +// Precision mode. +template <typename Int, typename Float, FormatStyle mode> +bool FloatToBufferImpl(Int int_mantissa, int exp, int precision, Buffer *out, + int *exp_out) { + assert((CanFitMantissa<Float, Int>())); + + const int int_bits = std::numeric_limits<Int>::digits; + + // In precision mode, we start printing one char to the right because it will + // also include the '.' + // In fixed mode we put the dot afterwards on the right. + out->begin = out->end = + out->data + 1 + kMaxFixedPrecision + (mode == FormatStyle::Precision); + + if (exp >= 0) { + if (std::numeric_limits<Float>::digits + exp > int_bits) { + // The value will overflow the Int + return false; + } + int digits_printed = PrintIntegralDigits<mode>(int_mantissa << exp, out); + int digits_to_zero_pad = precision; + if (mode == FormatStyle::Precision) { + *exp_out = digits_printed - 1; + digits_to_zero_pad -= digits_printed - 1; + if (RemoveExtraPrecision(-digits_to_zero_pad, false, out, exp_out)) { + return true; + } + } + for (; digits_to_zero_pad-- > 0;) out->push_back('0'); + return true; + } + + exp = -exp; + // We need at least 4 empty bits for the next decimal digit. + // We will multiply by 10. + if (exp > int_bits - 4) return false; + + const Int mask = (Int{1} << exp) - 1; + + // Print the integral part first. + int digits_printed = PrintIntegralDigits<mode>(int_mantissa >> exp, out); + int_mantissa &= mask; + + int fractional_count = precision; + if (mode == FormatStyle::Precision) { + if (digits_printed == 0) { + // Find the first non-zero digit, when in Precision mode. + *exp_out = 0; + if (int_mantissa) { + while (int_mantissa <= mask) { + int_mantissa *= 10; + --*exp_out; + } + } + out->push_front(static_cast<char>(int_mantissa >> exp) + '0'); + out->push_back('.'); + int_mantissa &= mask; + } else { + // We already have a digit, and a '.' + *exp_out = digits_printed - 1; + fractional_count -= *exp_out; + if (RemoveExtraPrecision(-fractional_count, int_mantissa != 0, out, + exp_out)) { + // If we had enough digits, return right away. + // The code below will try to round again otherwise. + return true; + } + } + } + + auto get_next_digit = [&] { + int_mantissa *= 10; + int digit = static_cast<int>(int_mantissa >> exp); + int_mantissa &= mask; + return digit; + }; + + // Print fractional_count more digits, if available. + for (; fractional_count > 0; --fractional_count) { + out->push_back(get_next_digit() + '0'); + } + + int next_digit = get_next_digit(); + if (next_digit > 5 || + (next_digit == 5 && (int_mantissa || out->last_digit() % 2 == 1))) { + RoundUp<mode>(out, exp_out); + } + + return true; +} + +template <FormatStyle mode, typename Float> +bool FloatToBuffer(Decomposed<Float> decomposed, int precision, Buffer *out, + int *exp) { + if (precision > kMaxFixedPrecision) return false; + + // Try with uint64_t. + if (CanFitMantissa<Float, std::uint64_t>() && + FloatToBufferImpl<std::uint64_t, Float, mode>( + static_cast<std::uint64_t>(decomposed.mantissa), + static_cast<std::uint64_t>(decomposed.exponent), precision, out, exp)) + return true; + +#if defined(ABSL_HAVE_INTRINSIC_INT128) + // If that is not enough, try with __uint128_t. + return CanFitMantissa<Float, __uint128_t>() && + FloatToBufferImpl<__uint128_t, Float, mode>( + static_cast<__uint128_t>(decomposed.mantissa), + static_cast<__uint128_t>(decomposed.exponent), precision, out, + exp); +#endif + return false; +} + +void WriteBufferToSink(char sign_char, absl::string_view str, + const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + int left_spaces = 0, zeros = 0, right_spaces = 0; + int missing_chars = + conv.width() >= 0 ? std::max(conv.width() - static_cast<int>(str.size()) - + static_cast<int>(sign_char != 0), + 0) + : 0; + if (conv.has_left_flag()) { + right_spaces = missing_chars; + } else if (conv.has_zero_flag()) { + zeros = missing_chars; + } else { + left_spaces = missing_chars; + } + + sink->Append(left_spaces, ' '); + if (sign_char != '\0') sink->Append(1, sign_char); + sink->Append(zeros, '0'); + sink->Append(str); + sink->Append(right_spaces, ' '); +} + +template <typename Float> +bool FloatToSink(const Float v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + // Print the sign or the sign column. + Float abs_v = v; + char sign_char = 0; + if (std::signbit(abs_v)) { + sign_char = '-'; + abs_v = -abs_v; + } else if (conv.has_show_pos_flag()) { + sign_char = '+'; + } else if (conv.has_sign_col_flag()) { + sign_char = ' '; + } + + // Print nan/inf. + if (ConvertNonNumericFloats(sign_char, abs_v, conv, sink)) { + return true; + } + + int precision = conv.precision() < 0 ? 6 : conv.precision(); + + int exp = 0; + + auto decomposed = Decompose(abs_v); + + Buffer buffer; + + FormatConversionChar c = conv.conversion_char(); + + if (c == FormatConversionCharInternal::f || + c == FormatConversionCharInternal::F) { + FormatF(decomposed.mantissa, decomposed.exponent, + {sign_char, precision, conv, sink}); + return true; + } else if (c == FormatConversionCharInternal::e || + c == FormatConversionCharInternal::E) { + if (!FloatToBuffer<FormatStyle::Precision>(decomposed, precision, &buffer, + &exp)) { + return FallbackToSnprintf(v, conv, sink); + } + if (!conv.has_alt_flag() && buffer.back() == '.') buffer.pop_back(); + PrintExponent( + exp, FormatConversionCharIsUpper(conv.conversion_char()) ? 'E' : 'e', + &buffer); + } else if (c == FormatConversionCharInternal::g || + c == FormatConversionCharInternal::G) { + precision = std::max(0, precision - 1); + if (!FloatToBuffer<FormatStyle::Precision>(decomposed, precision, &buffer, + &exp)) { + return FallbackToSnprintf(v, conv, sink); + } + if (precision + 1 > exp && exp >= -4) { + if (exp < 0) { + // Have 1.23456, needs 0.00123456 + // Move the first digit + buffer.begin[1] = *buffer.begin; + // Add some zeros + for (; exp < -1; ++exp) *buffer.begin-- = '0'; + *buffer.begin-- = '.'; + *buffer.begin = '0'; + } else if (exp > 0) { + // Have 1.23456, needs 1234.56 + // Move the '.' exp positions to the right. + std::rotate(buffer.begin + 1, buffer.begin + 2, buffer.begin + exp + 2); + } + exp = 0; + } + if (!conv.has_alt_flag()) { + while (buffer.back() == '0') buffer.pop_back(); + if (buffer.back() == '.') buffer.pop_back(); + } + if (exp) { + PrintExponent( + exp, FormatConversionCharIsUpper(conv.conversion_char()) ? 'E' : 'e', + &buffer); + } + } else if (c == FormatConversionCharInternal::a || + c == FormatConversionCharInternal::A) { + bool uppercase = (c == FormatConversionCharInternal::A); + FormatA(HexFloatTypeParams(Float{}), decomposed.mantissa, + decomposed.exponent, uppercase, {sign_char, precision, conv, sink}); + return true; + } else { + return false; + } + + WriteBufferToSink(sign_char, + absl::string_view(buffer.begin, buffer.end - buffer.begin), + conv, sink); + + return true; +} + +} // namespace + +bool ConvertFloatImpl(long double v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + if (std::numeric_limits<long double>::digits == + 2 * std::numeric_limits<double>::digits) { + // This is the `double-double` representation of `long double`. + // We do not handle it natively. Fallback to snprintf. + return FallbackToSnprintf(v, conv, sink); + } + + return FloatToSink(v, conv, sink); +} + +bool ConvertFloatImpl(float v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + return FloatToSink(static_cast<double>(v), conv, sink); +} + +bool ConvertFloatImpl(double v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink) { + return FloatToSink(v, conv, sink); +} + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/float_conversion.h b/third_party/abseil_cpp/absl/strings/internal/str_format/float_conversion.h new file mode 100644 index 000000000000..71100e714257 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/float_conversion.h @@ -0,0 +1,37 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_FLOAT_CONVERSION_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_FLOAT_CONVERSION_H_ + +#include "absl/strings/internal/str_format/extension.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +bool ConvertFloatImpl(float v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink); + +bool ConvertFloatImpl(double v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink); + +bool ConvertFloatImpl(long double v, const FormatConversionSpecImpl &conv, + FormatSinkImpl *sink); + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_FLOAT_CONVERSION_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/output.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/output.cc new file mode 100644 index 000000000000..c4b24706132c --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/output.cc @@ -0,0 +1,72 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/output.h" + +#include <errno.h> +#include <cstring> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +namespace { +struct ClearErrnoGuard { + ClearErrnoGuard() : old_value(errno) { errno = 0; } + ~ClearErrnoGuard() { + if (!errno) errno = old_value; + } + int old_value; +}; +} // namespace + +void BufferRawSink::Write(string_view v) { + size_t to_write = std::min(v.size(), size_); + std::memcpy(buffer_, v.data(), to_write); + buffer_ += to_write; + size_ -= to_write; + total_written_ += v.size(); +} + +void FILERawSink::Write(string_view v) { + while (!v.empty() && !error_) { + // Reset errno to zero in case the libc implementation doesn't set errno + // when a failure occurs. + ClearErrnoGuard guard; + + if (size_t result = std::fwrite(v.data(), 1, v.size(), output_)) { + // Some progress was made. + count_ += result; + v.remove_prefix(result); + } else { + if (errno == EINTR) { + continue; + } else if (errno) { + error_ = errno; + } else if (std::ferror(output_)) { + // Non-POSIX compliant libc implementations may not set errno, so we + // have check the streams error indicator. + error_ = EBADF; + } else { + // We're likely on a non-POSIX system that encountered EINTR but had no + // way of reporting it. + continue; + } + } + } +} + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/output.h b/third_party/abseil_cpp/absl/strings/internal/str_format/output.h new file mode 100644 index 000000000000..8030dae00f4f --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/output.h @@ -0,0 +1,96 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Output extension hooks for the Format library. +// `internal::InvokeFlush` calls the appropriate flush function for the +// specified output argument. +// `BufferRawSink` is a simple output sink for a char buffer. Used by SnprintF. +// `FILERawSink` is a std::FILE* based sink. Used by PrintF and FprintF. + +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_OUTPUT_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_OUTPUT_H_ + +#include <cstdio> +#include <ostream> +#include <string> + +#include "absl/base/port.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +// RawSink implementation that writes into a char* buffer. +// It will not overflow the buffer, but will keep the total count of chars +// that would have been written. +class BufferRawSink { + public: + BufferRawSink(char* buffer, size_t size) : buffer_(buffer), size_(size) {} + + size_t total_written() const { return total_written_; } + void Write(string_view v); + + private: + char* buffer_; + size_t size_; + size_t total_written_ = 0; +}; + +// RawSink implementation that writes into a FILE*. +// It keeps track of the total number of bytes written and any error encountered +// during the writes. +class FILERawSink { + public: + explicit FILERawSink(std::FILE* output) : output_(output) {} + + void Write(string_view v); + + size_t count() const { return count_; } + int error() const { return error_; } + + private: + std::FILE* output_; + int error_ = 0; + size_t count_ = 0; +}; + +// Provide RawSink integration with common types from the STL. +inline void AbslFormatFlush(std::string* out, string_view s) { + out->append(s.data(), s.size()); +} +inline void AbslFormatFlush(std::ostream* out, string_view s) { + out->write(s.data(), s.size()); +} + +inline void AbslFormatFlush(FILERawSink* sink, string_view v) { + sink->Write(v); +} + +inline void AbslFormatFlush(BufferRawSink* sink, string_view v) { + sink->Write(v); +} + +// This is a SFINAE to get a better compiler error message when the type +// is not supported. +template <typename T> +auto InvokeFlush(T* out, string_view s) -> decltype(AbslFormatFlush(out, s)) { + AbslFormatFlush(out, s); +} + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_OUTPUT_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/output_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/output_test.cc new file mode 100644 index 000000000000..ce2e91a0bbe8 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/output_test.cc @@ -0,0 +1,79 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/output.h" + +#include <sstream> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/cord.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +TEST(InvokeFlush, String) { + std::string str = "ABC"; + str_format_internal::InvokeFlush(&str, "DEF"); + EXPECT_EQ(str, "ABCDEF"); +} + +TEST(InvokeFlush, Stream) { + std::stringstream str; + str << "ABC"; + str_format_internal::InvokeFlush(&str, "DEF"); + EXPECT_EQ(str.str(), "ABCDEF"); +} + +TEST(InvokeFlush, Cord) { + absl::Cord str("ABC"); + str_format_internal::InvokeFlush(&str, "DEF"); + EXPECT_EQ(str, "ABCDEF"); +} + +TEST(BufferRawSink, Limits) { + char buf[16]; + { + std::fill(std::begin(buf), std::end(buf), 'x'); + str_format_internal::BufferRawSink bufsink(buf, sizeof(buf) - 1); + str_format_internal::InvokeFlush(&bufsink, "Hello World237"); + EXPECT_EQ(std::string(buf, sizeof(buf)), "Hello World237xx"); + } + { + std::fill(std::begin(buf), std::end(buf), 'x'); + str_format_internal::BufferRawSink bufsink(buf, sizeof(buf) - 1); + str_format_internal::InvokeFlush(&bufsink, "Hello World237237"); + EXPECT_EQ(std::string(buf, sizeof(buf)), "Hello World2372x"); + } + { + std::fill(std::begin(buf), std::end(buf), 'x'); + str_format_internal::BufferRawSink bufsink(buf, sizeof(buf) - 1); + str_format_internal::InvokeFlush(&bufsink, "Hello World"); + str_format_internal::InvokeFlush(&bufsink, "237"); + EXPECT_EQ(std::string(buf, sizeof(buf)), "Hello World237xx"); + } + { + std::fill(std::begin(buf), std::end(buf), 'x'); + str_format_internal::BufferRawSink bufsink(buf, sizeof(buf) - 1); + str_format_internal::InvokeFlush(&bufsink, "Hello World"); + str_format_internal::InvokeFlush(&bufsink, "237237"); + EXPECT_EQ(std::string(buf, sizeof(buf)), "Hello World2372x"); + } +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/parser.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/parser.cc new file mode 100644 index 000000000000..f308d0235120 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/parser.cc @@ -0,0 +1,350 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/parser.h" + +#include <assert.h> +#include <string.h> +#include <wchar.h> +#include <cctype> +#include <cstdint> + +#include <algorithm> +#include <initializer_list> +#include <limits> +#include <ostream> +#include <string> +#include <unordered_set> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +using CC = FormatConversionCharInternal; +using LM = LengthMod; + +ABSL_CONST_INIT const ConvTag kTags[256] = { + {}, {}, {}, {}, {}, {}, {}, {}, // 00-07 + {}, {}, {}, {}, {}, {}, {}, {}, // 08-0f + {}, {}, {}, {}, {}, {}, {}, {}, // 10-17 + {}, {}, {}, {}, {}, {}, {}, {}, // 18-1f + {}, {}, {}, {}, {}, {}, {}, {}, // 20-27 + {}, {}, {}, {}, {}, {}, {}, {}, // 28-2f + {}, {}, {}, {}, {}, {}, {}, {}, // 30-37 + {}, {}, {}, {}, {}, {}, {}, {}, // 38-3f + {}, CC::A, {}, {}, {}, CC::E, CC::F, CC::G, // @ABCDEFG + {}, {}, {}, {}, LM::L, {}, {}, {}, // HIJKLMNO + {}, {}, {}, {}, {}, {}, {}, {}, // PQRSTUVW + CC::X, {}, {}, {}, {}, {}, {}, {}, // XYZ[\]^_ + {}, CC::a, {}, CC::c, CC::d, CC::e, CC::f, CC::g, // `abcdefg + LM::h, CC::i, LM::j, {}, LM::l, {}, CC::n, CC::o, // hijklmno + CC::p, LM::q, {}, CC::s, LM::t, CC::u, {}, {}, // pqrstuvw + CC::x, {}, LM::z, {}, {}, {}, {}, {}, // xyz{|}! + {}, {}, {}, {}, {}, {}, {}, {}, // 80-87 + {}, {}, {}, {}, {}, {}, {}, {}, // 88-8f + {}, {}, {}, {}, {}, {}, {}, {}, // 90-97 + {}, {}, {}, {}, {}, {}, {}, {}, // 98-9f + {}, {}, {}, {}, {}, {}, {}, {}, // a0-a7 + {}, {}, {}, {}, {}, {}, {}, {}, // a8-af + {}, {}, {}, {}, {}, {}, {}, {}, // b0-b7 + {}, {}, {}, {}, {}, {}, {}, {}, // b8-bf + {}, {}, {}, {}, {}, {}, {}, {}, // c0-c7 + {}, {}, {}, {}, {}, {}, {}, {}, // c8-cf + {}, {}, {}, {}, {}, {}, {}, {}, // d0-d7 + {}, {}, {}, {}, {}, {}, {}, {}, // d8-df + {}, {}, {}, {}, {}, {}, {}, {}, // e0-e7 + {}, {}, {}, {}, {}, {}, {}, {}, // e8-ef + {}, {}, {}, {}, {}, {}, {}, {}, // f0-f7 + {}, {}, {}, {}, {}, {}, {}, {}, // f8-ff +}; + +namespace { + +bool CheckFastPathSetting(const UnboundConversion& conv) { + bool should_be_basic = !conv.flags.left && // + !conv.flags.show_pos && // + !conv.flags.sign_col && // + !conv.flags.alt && // + !conv.flags.zero && // + (conv.width.value() == -1) && + (conv.precision.value() == -1); + if (should_be_basic != conv.flags.basic) { + fprintf(stderr, + "basic=%d left=%d show_pos=%d sign_col=%d alt=%d zero=%d " + "width=%d precision=%d\n", + conv.flags.basic, conv.flags.left, conv.flags.show_pos, + conv.flags.sign_col, conv.flags.alt, conv.flags.zero, + conv.width.value(), conv.precision.value()); + } + return should_be_basic == conv.flags.basic; +} + +template <bool is_positional> +const char *ConsumeConversion(const char *pos, const char *const end, + UnboundConversion *conv, int *next_arg) { + const char* const original_pos = pos; + char c; + // Read the next char into `c` and update `pos`. Returns false if there are + // no more chars to read. +#define ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR() \ + do { \ + if (ABSL_PREDICT_FALSE(pos == end)) return nullptr; \ + c = *pos++; \ + } while (0) + + const auto parse_digits = [&] { + int digits = c - '0'; + // We do not want to overflow `digits` so we consume at most digits10 + // digits. If there are more digits the parsing will fail later on when the + // digit doesn't match the expected characters. + int num_digits = std::numeric_limits<int>::digits10; + for (;;) { + if (ABSL_PREDICT_FALSE(pos == end)) break; + c = *pos++; + if (!std::isdigit(c)) break; + --num_digits; + if (ABSL_PREDICT_FALSE(!num_digits)) break; + digits = 10 * digits + c - '0'; + } + return digits; + }; + + if (is_positional) { + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + if (ABSL_PREDICT_FALSE(c < '1' || c > '9')) return nullptr; + conv->arg_position = parse_digits(); + assert(conv->arg_position > 0); + if (ABSL_PREDICT_FALSE(c != '$')) return nullptr; + } + + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + + // We should start with the basic flag on. + assert(conv->flags.basic); + + // Any non alpha character makes this conversion not basic. + // This includes flags (-+ #0), width (1-9, *) or precision (.). + // All conversion characters and length modifiers are alpha characters. + if (c < 'A') { + conv->flags.basic = false; + + for (; c <= '0';) { + // FIXME: We might be able to speed this up reusing the lookup table from + // above. It might require changing Flags to be a plain integer where we + // can |= a value. + switch (c) { + case '-': + conv->flags.left = true; + break; + case '+': + conv->flags.show_pos = true; + break; + case ' ': + conv->flags.sign_col = true; + break; + case '#': + conv->flags.alt = true; + break; + case '0': + conv->flags.zero = true; + break; + default: + goto flags_done; + } + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + } +flags_done: + + if (c <= '9') { + if (c >= '0') { + int maybe_width = parse_digits(); + if (!is_positional && c == '$') { + if (ABSL_PREDICT_FALSE(*next_arg != 0)) return nullptr; + // Positional conversion. + *next_arg = -1; + conv->flags = Flags(); + conv->flags.basic = true; + return ConsumeConversion<true>(original_pos, end, conv, next_arg); + } + conv->width.set_value(maybe_width); + } else if (c == '*') { + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + if (is_positional) { + if (ABSL_PREDICT_FALSE(c < '1' || c > '9')) return nullptr; + conv->width.set_from_arg(parse_digits()); + if (ABSL_PREDICT_FALSE(c != '$')) return nullptr; + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + } else { + conv->width.set_from_arg(++*next_arg); + } + } + } + + if (c == '.') { + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + if (std::isdigit(c)) { + conv->precision.set_value(parse_digits()); + } else if (c == '*') { + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + if (is_positional) { + if (ABSL_PREDICT_FALSE(c < '1' || c > '9')) return nullptr; + conv->precision.set_from_arg(parse_digits()); + if (c != '$') return nullptr; + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + } else { + conv->precision.set_from_arg(++*next_arg); + } + } else { + conv->precision.set_value(0); + } + } + } + + auto tag = GetTagForChar(c); + + if (ABSL_PREDICT_FALSE(!tag.is_conv())) { + if (ABSL_PREDICT_FALSE(!tag.is_length())) return nullptr; + + // It is a length modifier. + using str_format_internal::LengthMod; + LengthMod length_mod = tag.as_length(); + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + if (c == 'h' && length_mod == LengthMod::h) { + conv->length_mod = LengthMod::hh; + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + } else if (c == 'l' && length_mod == LengthMod::l) { + conv->length_mod = LengthMod::ll; + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + } else { + conv->length_mod = length_mod; + } + tag = GetTagForChar(c); + if (ABSL_PREDICT_FALSE(!tag.is_conv())) return nullptr; + } + + assert(CheckFastPathSetting(*conv)); + (void)(&CheckFastPathSetting); + + conv->conv = tag.as_conv(); + if (!is_positional) conv->arg_position = ++*next_arg; + return pos; +} + +} // namespace + +std::string LengthModToString(LengthMod v) { + switch (v) { + case LengthMod::h: + return "h"; + case LengthMod::hh: + return "hh"; + case LengthMod::l: + return "l"; + case LengthMod::ll: + return "ll"; + case LengthMod::L: + return "L"; + case LengthMod::j: + return "j"; + case LengthMod::z: + return "z"; + case LengthMod::t: + return "t"; + case LengthMod::q: + return "q"; + case LengthMod::none: + return ""; + } + return ""; +} + +const char *ConsumeUnboundConversion(const char *p, const char *end, + UnboundConversion *conv, int *next_arg) { + if (*next_arg < 0) return ConsumeConversion<true>(p, end, conv, next_arg); + return ConsumeConversion<false>(p, end, conv, next_arg); +} + +struct ParsedFormatBase::ParsedFormatConsumer { + explicit ParsedFormatConsumer(ParsedFormatBase *parsedformat) + : parsed(parsedformat), data_pos(parsedformat->data_.get()) {} + + bool Append(string_view s) { + if (s.empty()) return true; + + size_t text_end = AppendText(s); + + if (!parsed->items_.empty() && !parsed->items_.back().is_conversion) { + // Let's extend the existing text run. + parsed->items_.back().text_end = text_end; + } else { + // Let's make a new text run. + parsed->items_.push_back({false, text_end, {}}); + } + return true; + } + + bool ConvertOne(const UnboundConversion &conv, string_view s) { + size_t text_end = AppendText(s); + parsed->items_.push_back({true, text_end, conv}); + return true; + } + + size_t AppendText(string_view s) { + memcpy(data_pos, s.data(), s.size()); + data_pos += s.size(); + return static_cast<size_t>(data_pos - parsed->data_.get()); + } + + ParsedFormatBase *parsed; + char* data_pos; +}; + +ParsedFormatBase::ParsedFormatBase( + string_view format, bool allow_ignored, + std::initializer_list<FormatConversionCharSet> convs) + : data_(format.empty() ? nullptr : new char[format.size()]) { + has_error_ = !ParseFormatString(format, ParsedFormatConsumer(this)) || + !MatchesConversions(allow_ignored, convs); +} + +bool ParsedFormatBase::MatchesConversions( + bool allow_ignored, + std::initializer_list<FormatConversionCharSet> convs) const { + std::unordered_set<int> used; + auto add_if_valid_conv = [&](int pos, char c) { + if (static_cast<size_t>(pos) > convs.size() || + !Contains(convs.begin()[pos - 1], c)) + return false; + used.insert(pos); + return true; + }; + for (const ConversionItem &item : items_) { + if (!item.is_conversion) continue; + auto &conv = item.conv; + if (conv.precision.is_from_arg() && + !add_if_valid_conv(conv.precision.get_from_arg(), '*')) + return false; + if (conv.width.is_from_arg() && + !add_if_valid_conv(conv.width.get_from_arg(), '*')) + return false; + if (!add_if_valid_conv(conv.arg_position, + FormatConversionCharToChar(conv.conv))) + return false; + } + return used.size() == convs.size() || allow_ignored; +} + +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/parser.h b/third_party/abseil_cpp/absl/strings/internal/str_format/parser.h new file mode 100644 index 000000000000..6504dd3ddc20 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/parser.h @@ -0,0 +1,349 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_STR_FORMAT_PARSER_H_ +#define ABSL_STRINGS_INTERNAL_STR_FORMAT_PARSER_H_ + +#include <limits.h> +#include <stddef.h> +#include <stdlib.h> + +#include <cassert> +#include <cstdint> +#include <initializer_list> +#include <iosfwd> +#include <iterator> +#include <memory> +#include <string> +#include <vector> + +#include "absl/strings/internal/str_format/checker.h" +#include "absl/strings/internal/str_format/extension.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +enum class LengthMod : std::uint8_t { h, hh, l, ll, L, j, z, t, q, none }; + +std::string LengthModToString(LengthMod v); + +// The analyzed properties of a single specified conversion. +struct UnboundConversion { + UnboundConversion() + : flags() /* This is required to zero all the fields of flags. */ { + flags.basic = true; + } + + class InputValue { + public: + void set_value(int value) { + assert(value >= 0); + value_ = value; + } + int value() const { return value_; } + + // Marks the value as "from arg". aka the '*' format. + // Requires `value >= 1`. + // When set, is_from_arg() return true and get_from_arg() returns the + // original value. + // `value()`'s return value is unspecfied in this state. + void set_from_arg(int value) { + assert(value > 0); + value_ = -value - 1; + } + bool is_from_arg() const { return value_ < -1; } + int get_from_arg() const { + assert(is_from_arg()); + return -value_ - 1; + } + + private: + int value_ = -1; + }; + + // No need to initialize. It will always be set in the parser. + int arg_position; + + InputValue width; + InputValue precision; + + Flags flags; + LengthMod length_mod = LengthMod::none; + FormatConversionChar conv = FormatConversionCharInternal::kNone; +}; + +// Consume conversion spec prefix (not including '%') of [p, end) if valid. +// Examples of valid specs would be e.g.: "s", "d", "-12.6f". +// If valid, it returns the first character following the conversion spec, +// and the spec part is broken down and returned in 'conv'. +// If invalid, returns nullptr. +const char* ConsumeUnboundConversion(const char* p, const char* end, + UnboundConversion* conv, int* next_arg); + +// Helper tag class for the table below. +// It allows fast `char -> ConversionChar/LengthMod` checking and +// conversions. +class ConvTag { + public: + constexpr ConvTag(FormatConversionChar conversion_char) // NOLINT + : tag_(static_cast<int8_t>(conversion_char)) {} + // We invert the length modifiers to make them negative so that we can easily + // test for them. + constexpr ConvTag(LengthMod length_mod) // NOLINT + : tag_(~static_cast<std::int8_t>(length_mod)) {} + // Everything else is -128, which is negative to make is_conv() simpler. + constexpr ConvTag() : tag_(-128) {} + + bool is_conv() const { return tag_ >= 0; } + bool is_length() const { return tag_ < 0 && tag_ != -128; } + FormatConversionChar as_conv() const { + assert(is_conv()); + return static_cast<FormatConversionChar>(tag_); + } + LengthMod as_length() const { + assert(is_length()); + return static_cast<LengthMod>(~tag_); + } + + private: + std::int8_t tag_; +}; + +extern const ConvTag kTags[256]; +// Keep a single table for all the conversion chars and length modifiers. +inline ConvTag GetTagForChar(char c) { + return kTags[static_cast<unsigned char>(c)]; +} + +// Parse the format string provided in 'src' and pass the identified items into +// 'consumer'. +// Text runs will be passed by calling +// Consumer::Append(string_view); +// ConversionItems will be passed by calling +// Consumer::ConvertOne(UnboundConversion, string_view); +// In the case of ConvertOne, the string_view that is passed is the +// portion of the format string corresponding to the conversion, not including +// the leading %. On success, it returns true. On failure, it stops and returns +// false. +template <typename Consumer> +bool ParseFormatString(string_view src, Consumer consumer) { + int next_arg = 0; + const char* p = src.data(); + const char* const end = p + src.size(); + while (p != end) { + const char* percent = static_cast<const char*>(memchr(p, '%', end - p)); + if (!percent) { + // We found the last substring. + return consumer.Append(string_view(p, end - p)); + } + // We found a percent, so push the text run then process the percent. + if (ABSL_PREDICT_FALSE(!consumer.Append(string_view(p, percent - p)))) { + return false; + } + if (ABSL_PREDICT_FALSE(percent + 1 >= end)) return false; + + auto tag = GetTagForChar(percent[1]); + if (tag.is_conv()) { + if (ABSL_PREDICT_FALSE(next_arg < 0)) { + // This indicates an error in the format string. + // The only way to get `next_arg < 0` here is to have a positional + // argument first which sets next_arg to -1 and then a non-positional + // argument. + return false; + } + p = percent + 2; + + // Keep this case separate from the one below. + // ConvertOne is more efficient when the compiler can see that the `basic` + // flag is set. + UnboundConversion conv; + conv.conv = tag.as_conv(); + conv.arg_position = ++next_arg; + if (ABSL_PREDICT_FALSE( + !consumer.ConvertOne(conv, string_view(percent + 1, 1)))) { + return false; + } + } else if (percent[1] != '%') { + UnboundConversion conv; + p = ConsumeUnboundConversion(percent + 1, end, &conv, &next_arg); + if (ABSL_PREDICT_FALSE(p == nullptr)) return false; + if (ABSL_PREDICT_FALSE(!consumer.ConvertOne( + conv, string_view(percent + 1, p - (percent + 1))))) { + return false; + } + } else { + if (ABSL_PREDICT_FALSE(!consumer.Append("%"))) return false; + p = percent + 2; + continue; + } + } + return true; +} + +// Always returns true, or fails to compile in a constexpr context if s does not +// point to a constexpr char array. +constexpr bool EnsureConstexpr(string_view s) { + return s.empty() || s[0] == s[0]; +} + +class ParsedFormatBase { + public: + explicit ParsedFormatBase( + string_view format, bool allow_ignored, + std::initializer_list<FormatConversionCharSet> convs); + + ParsedFormatBase(const ParsedFormatBase& other) { *this = other; } + + ParsedFormatBase(ParsedFormatBase&& other) { *this = std::move(other); } + + ParsedFormatBase& operator=(const ParsedFormatBase& other) { + if (this == &other) return *this; + has_error_ = other.has_error_; + items_ = other.items_; + size_t text_size = items_.empty() ? 0 : items_.back().text_end; + data_.reset(new char[text_size]); + memcpy(data_.get(), other.data_.get(), text_size); + return *this; + } + + ParsedFormatBase& operator=(ParsedFormatBase&& other) { + if (this == &other) return *this; + has_error_ = other.has_error_; + data_ = std::move(other.data_); + items_ = std::move(other.items_); + // Reset the vector to make sure the invariants hold. + other.items_.clear(); + return *this; + } + + template <typename Consumer> + bool ProcessFormat(Consumer consumer) const { + const char* const base = data_.get(); + string_view text(base, 0); + for (const auto& item : items_) { + const char* const end = text.data() + text.size(); + text = string_view(end, (base + item.text_end) - end); + if (item.is_conversion) { + if (!consumer.ConvertOne(item.conv, text)) return false; + } else { + if (!consumer.Append(text)) return false; + } + } + return !has_error_; + } + + bool has_error() const { return has_error_; } + + private: + // Returns whether the conversions match and if !allow_ignored it verifies + // that all conversions are used by the format. + bool MatchesConversions( + bool allow_ignored, + std::initializer_list<FormatConversionCharSet> convs) const; + + struct ParsedFormatConsumer; + + struct ConversionItem { + bool is_conversion; + // Points to the past-the-end location of this element in the data_ array. + size_t text_end; + UnboundConversion conv; + }; + + bool has_error_; + std::unique_ptr<char[]> data_; + std::vector<ConversionItem> items_; +}; + + +// A value type representing a preparsed format. These can be created, copied +// around, and reused to speed up formatting loops. +// The user must specify through the template arguments the conversion +// characters used in the format. This will be checked at compile time. +// +// This class uses Conv enum values to specify each argument. +// This allows for more flexibility as you can specify multiple possible +// conversion characters for each argument. +// ParsedFormat<char...> is a simplified alias for when the user only +// needs to specify a single conversion character for each argument. +// +// Example: +// // Extended format supports multiple characters per argument: +// using MyFormat = ExtendedParsedFormat<Conv::d | Conv::x>; +// MyFormat GetFormat(bool use_hex) { +// if (use_hex) return MyFormat("foo %x bar"); +// return MyFormat("foo %d bar"); +// } +// // 'format' can be used with any value that supports 'd' and 'x', +// // like `int`. +// auto format = GetFormat(use_hex); +// value = StringF(format, i); +// +// This class also supports runtime format checking with the ::New() and +// ::NewAllowIgnored() factory functions. +// This is the only API that allows the user to pass a runtime specified format +// string. These factory functions will return NULL if the format does not match +// the conversions requested by the user. +template <FormatConversionCharSet... C> +class ExtendedParsedFormat : public str_format_internal::ParsedFormatBase { + public: + explicit ExtendedParsedFormat(string_view format) +#ifdef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + __attribute__(( + enable_if(str_format_internal::EnsureConstexpr(format), + "Format string is not constexpr."), + enable_if(str_format_internal::ValidFormatImpl<C...>(format), + "Format specified does not match the template arguments."))) +#endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER + : ExtendedParsedFormat(format, false) { + } + + // ExtendedParsedFormat factory function. + // The user still has to specify the conversion characters, but they will not + // be checked at compile time. Instead, it will be checked at runtime. + // This delays the checking to runtime, but allows the user to pass + // dynamically sourced formats. + // It returns NULL if the format does not match the conversion characters. + // The user is responsible for checking the return value before using it. + // + // The 'New' variant will check that all the specified arguments are being + // consumed by the format and return NULL if any argument is being ignored. + // The 'NewAllowIgnored' variant will not verify this and will allow formats + // that ignore arguments. + static std::unique_ptr<ExtendedParsedFormat> New(string_view format) { + return New(format, false); + } + static std::unique_ptr<ExtendedParsedFormat> NewAllowIgnored( + string_view format) { + return New(format, true); + } + + private: + static std::unique_ptr<ExtendedParsedFormat> New(string_view format, + bool allow_ignored) { + std::unique_ptr<ExtendedParsedFormat> conv( + new ExtendedParsedFormat(format, allow_ignored)); + if (conv->has_error()) return nullptr; + return conv; + } + + ExtendedParsedFormat(string_view s, bool allow_ignored) + : ParsedFormatBase(s, allow_ignored, {C...}) {} +}; +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_FORMAT_PARSER_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_format/parser_test.cc b/third_party/abseil_cpp/absl/strings/internal/str_format/parser_test.cc new file mode 100644 index 000000000000..a5fa1c79aaf4 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_format/parser_test.cc @@ -0,0 +1,427 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/str_format/parser.h" + +#include <string.h> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace str_format_internal { + +namespace { + +using testing::Pair; + +TEST(LengthModTest, Names) { + struct Expectation { + int line; + LengthMod mod; + const char *name; + }; + const Expectation kExpect[] = { + {__LINE__, LengthMod::none, "" }, + {__LINE__, LengthMod::h, "h" }, + {__LINE__, LengthMod::hh, "hh"}, + {__LINE__, LengthMod::l, "l" }, + {__LINE__, LengthMod::ll, "ll"}, + {__LINE__, LengthMod::L, "L" }, + {__LINE__, LengthMod::j, "j" }, + {__LINE__, LengthMod::z, "z" }, + {__LINE__, LengthMod::t, "t" }, + {__LINE__, LengthMod::q, "q" }, + }; + EXPECT_EQ(ABSL_ARRAYSIZE(kExpect), 10); + for (auto e : kExpect) { + SCOPED_TRACE(e.line); + EXPECT_EQ(e.name, LengthModToString(e.mod)); + } +} + +TEST(ConversionCharTest, Names) { + struct Expectation { + FormatConversionChar id; + char name; + }; + // clang-format off + const Expectation kExpect[] = { +#define X(c) {FormatConversionCharInternal::c, #c[0]} + X(c), X(s), // text + X(d), X(i), X(o), X(u), X(x), X(X), // int + X(f), X(F), X(e), X(E), X(g), X(G), X(a), X(A), // float + X(n), X(p), // misc +#undef X + {FormatConversionCharInternal::kNone, '\0'}, + }; + // clang-format on + for (auto e : kExpect) { + SCOPED_TRACE(e.name); + FormatConversionChar v = e.id; + EXPECT_EQ(e.name, FormatConversionCharToChar(v)); + } +} + +class ConsumeUnboundConversionTest : public ::testing::Test { + public: + std::pair<string_view, string_view> Consume(string_view src) { + int next = 0; + o = UnboundConversion(); // refresh + const char* p = ConsumeUnboundConversion( + src.data(), src.data() + src.size(), &o, &next); + if (!p) return {{}, src}; + return {string_view(src.data(), p - src.data()), + string_view(p, src.data() + src.size() - p)}; + } + + bool Run(const char *fmt, bool force_positional = false) { + int next = force_positional ? -1 : 0; + o = UnboundConversion(); // refresh + return ConsumeUnboundConversion(fmt, fmt + strlen(fmt), &o, &next) == + fmt + strlen(fmt); + } + UnboundConversion o; +}; + +TEST_F(ConsumeUnboundConversionTest, ConsumeSpecification) { + struct Expectation { + int line; + string_view src; + string_view out; + string_view src_post; + }; + const Expectation kExpect[] = { + {__LINE__, "", "", "" }, + {__LINE__, "b", "", "b" }, // 'b' is invalid + {__LINE__, "ba", "", "ba"}, // 'b' is invalid + {__LINE__, "l", "", "l" }, // just length mod isn't okay + {__LINE__, "d", "d", "" }, // basic + {__LINE__, "d ", "d", " " }, // leave suffix + {__LINE__, "dd", "d", "d" }, // don't be greedy + {__LINE__, "d9", "d", "9" }, // leave non-space suffix + {__LINE__, "dzz", "d", "zz"}, // length mod as suffix + {__LINE__, "1$*2$d", "1$*2$d", "" }, // arg indexing and * allowed. + {__LINE__, "0-14.3hhd", "0-14.3hhd", ""}, // precision, width + {__LINE__, " 0-+#14.3hhd", " 0-+#14.3hhd", ""}, // flags + }; + for (const auto& e : kExpect) { + SCOPED_TRACE(e.line); + EXPECT_THAT(Consume(e.src), Pair(e.out, e.src_post)); + } +} + +TEST_F(ConsumeUnboundConversionTest, BasicConversion) { + EXPECT_FALSE(Run("")); + EXPECT_FALSE(Run("z")); + + EXPECT_FALSE(Run("dd")); // no excess allowed + + EXPECT_TRUE(Run("d")); + EXPECT_EQ('d', FormatConversionCharToChar(o.conv)); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_LT(o.width.value(), 0); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_LT(o.precision.value(), 0); + EXPECT_EQ(1, o.arg_position); +} + +TEST_F(ConsumeUnboundConversionTest, ArgPosition) { + EXPECT_TRUE(Run("d")); + EXPECT_EQ(1, o.arg_position); + EXPECT_TRUE(Run("3$d")); + EXPECT_EQ(3, o.arg_position); + EXPECT_TRUE(Run("1$d")); + EXPECT_EQ(1, o.arg_position); + EXPECT_TRUE(Run("1$d", true)); + EXPECT_EQ(1, o.arg_position); + EXPECT_TRUE(Run("123$d")); + EXPECT_EQ(123, o.arg_position); + EXPECT_TRUE(Run("123$d", true)); + EXPECT_EQ(123, o.arg_position); + EXPECT_TRUE(Run("10$d")); + EXPECT_EQ(10, o.arg_position); + EXPECT_TRUE(Run("10$d", true)); + EXPECT_EQ(10, o.arg_position); + + // Position can't be zero. + EXPECT_FALSE(Run("0$d")); + EXPECT_FALSE(Run("0$d", true)); + EXPECT_FALSE(Run("1$*0$d")); + EXPECT_FALSE(Run("1$.*0$d")); + + // Position can't start with a zero digit at all. That is not a 'decimal'. + EXPECT_FALSE(Run("01$p")); + EXPECT_FALSE(Run("01$p", true)); + EXPECT_FALSE(Run("1$*01$p")); + EXPECT_FALSE(Run("1$.*01$p")); +} + +TEST_F(ConsumeUnboundConversionTest, WidthAndPrecision) { + EXPECT_TRUE(Run("14d")); + EXPECT_EQ('d', FormatConversionCharToChar(o.conv)); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_EQ(14, o.width.value()); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_LT(o.precision.value(), 0); + + EXPECT_TRUE(Run("14.d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_EQ(14, o.width.value()); + EXPECT_EQ(0, o.precision.value()); + + EXPECT_TRUE(Run(".d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_LT(o.width.value(), 0); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_EQ(0, o.precision.value()); + + EXPECT_TRUE(Run(".5d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_LT(o.width.value(), 0); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_EQ(5, o.precision.value()); + + EXPECT_TRUE(Run(".0d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_LT(o.width.value(), 0); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_EQ(0, o.precision.value()); + + EXPECT_TRUE(Run("14.5d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_EQ(14, o.width.value()); + EXPECT_EQ(5, o.precision.value()); + + EXPECT_TRUE(Run("*.*d")); + EXPECT_TRUE(o.width.is_from_arg()); + EXPECT_EQ(1, o.width.get_from_arg()); + EXPECT_TRUE(o.precision.is_from_arg()); + EXPECT_EQ(2, o.precision.get_from_arg()); + EXPECT_EQ(3, o.arg_position); + + EXPECT_TRUE(Run("*d")); + EXPECT_TRUE(o.width.is_from_arg()); + EXPECT_EQ(1, o.width.get_from_arg()); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_LT(o.precision.value(), 0); + EXPECT_EQ(2, o.arg_position); + + EXPECT_TRUE(Run(".*d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_LT(o.width.value(), 0); + EXPECT_TRUE(o.precision.is_from_arg()); + EXPECT_EQ(1, o.precision.get_from_arg()); + EXPECT_EQ(2, o.arg_position); + + // mixed implicit and explicit: didn't specify arg position. + EXPECT_FALSE(Run("*23$.*34$d")); + + EXPECT_TRUE(Run("12$*23$.*34$d")); + EXPECT_EQ(12, o.arg_position); + EXPECT_TRUE(o.width.is_from_arg()); + EXPECT_EQ(23, o.width.get_from_arg()); + EXPECT_TRUE(o.precision.is_from_arg()); + EXPECT_EQ(34, o.precision.get_from_arg()); + + EXPECT_TRUE(Run("2$*5$.*9$d")); + EXPECT_EQ(2, o.arg_position); + EXPECT_TRUE(o.width.is_from_arg()); + EXPECT_EQ(5, o.width.get_from_arg()); + EXPECT_TRUE(o.precision.is_from_arg()); + EXPECT_EQ(9, o.precision.get_from_arg()); + + EXPECT_FALSE(Run(".*0$d")) << "no arg 0"; + + // Large values + EXPECT_TRUE(Run("999999999.999999999d")); + EXPECT_FALSE(o.width.is_from_arg()); + EXPECT_EQ(999999999, o.width.value()); + EXPECT_FALSE(o.precision.is_from_arg()); + EXPECT_EQ(999999999, o.precision.value()); + + EXPECT_FALSE(Run("1000000000.999999999d")); + EXPECT_FALSE(Run("999999999.1000000000d")); + EXPECT_FALSE(Run("9999999999d")); + EXPECT_FALSE(Run(".9999999999d")); +} + +TEST_F(ConsumeUnboundConversionTest, Flags) { + static const char kAllFlags[] = "-+ #0"; + static const int kNumFlags = ABSL_ARRAYSIZE(kAllFlags) - 1; + for (int rev = 0; rev < 2; ++rev) { + for (int i = 0; i < 1 << kNumFlags; ++i) { + std::string fmt; + for (int k = 0; k < kNumFlags; ++k) + if ((i >> k) & 1) fmt += kAllFlags[k]; + // flag order shouldn't matter + if (rev == 1) { std::reverse(fmt.begin(), fmt.end()); } + fmt += 'd'; + SCOPED_TRACE(fmt); + EXPECT_TRUE(Run(fmt.c_str())); + EXPECT_EQ(fmt.find('-') == std::string::npos, !o.flags.left); + EXPECT_EQ(fmt.find('+') == std::string::npos, !o.flags.show_pos); + EXPECT_EQ(fmt.find(' ') == std::string::npos, !o.flags.sign_col); + EXPECT_EQ(fmt.find('#') == std::string::npos, !o.flags.alt); + EXPECT_EQ(fmt.find('0') == std::string::npos, !o.flags.zero); + } + } +} + +TEST_F(ConsumeUnboundConversionTest, BasicFlag) { + // Flag is on + for (const char* fmt : {"d", "llx", "G", "1$X"}) { + SCOPED_TRACE(fmt); + EXPECT_TRUE(Run(fmt)); + EXPECT_TRUE(o.flags.basic); + } + + // Flag is off + for (const char* fmt : {"3d", ".llx", "-G", "1$#X"}) { + SCOPED_TRACE(fmt); + EXPECT_TRUE(Run(fmt)); + EXPECT_FALSE(o.flags.basic); + } +} + +TEST_F(ConsumeUnboundConversionTest, LengthMod) { + EXPECT_TRUE(Run("d")); + EXPECT_EQ(LengthMod::none, o.length_mod); + EXPECT_TRUE(Run("hd")); + EXPECT_EQ(LengthMod::h, o.length_mod); + EXPECT_TRUE(Run("hhd")); + EXPECT_EQ(LengthMod::hh, o.length_mod); + EXPECT_TRUE(Run("ld")); + EXPECT_EQ(LengthMod::l, o.length_mod); + EXPECT_TRUE(Run("lld")); + EXPECT_EQ(LengthMod::ll, o.length_mod); + EXPECT_TRUE(Run("Lf")); + EXPECT_EQ(LengthMod::L, o.length_mod); + EXPECT_TRUE(Run("qf")); + EXPECT_EQ(LengthMod::q, o.length_mod); + EXPECT_TRUE(Run("jd")); + EXPECT_EQ(LengthMod::j, o.length_mod); + EXPECT_TRUE(Run("zd")); + EXPECT_EQ(LengthMod::z, o.length_mod); + EXPECT_TRUE(Run("td")); + EXPECT_EQ(LengthMod::t, o.length_mod); +} + +struct SummarizeConsumer { + std::string* out; + explicit SummarizeConsumer(std::string* out) : out(out) {} + + bool Append(string_view s) { + *out += "[" + std::string(s) + "]"; + return true; + } + + bool ConvertOne(const UnboundConversion& conv, string_view s) { + *out += "{"; + *out += std::string(s); + *out += ":"; + *out += std::to_string(conv.arg_position) + "$"; + if (conv.width.is_from_arg()) { + *out += std::to_string(conv.width.get_from_arg()) + "$*"; + } + if (conv.precision.is_from_arg()) { + *out += "." + std::to_string(conv.precision.get_from_arg()) + "$*"; + } + *out += FormatConversionCharToChar(conv.conv); + *out += "}"; + return true; + } +}; + +std::string SummarizeParsedFormat(const ParsedFormatBase& pc) { + std::string out; + if (!pc.ProcessFormat(SummarizeConsumer(&out))) out += "!"; + return out; +} + +class ParsedFormatTest : public testing::Test {}; + +TEST_F(ParsedFormatTest, ValueSemantics) { + ParsedFormatBase p1({}, true, {}); // empty format + EXPECT_EQ("", SummarizeParsedFormat(p1)); + + ParsedFormatBase p2 = p1; // copy construct (empty) + EXPECT_EQ(SummarizeParsedFormat(p1), SummarizeParsedFormat(p2)); + + p1 = ParsedFormatBase("hello%s", true, + {FormatConversionCharSetInternal::s}); // move assign + EXPECT_EQ("[hello]{s:1$s}", SummarizeParsedFormat(p1)); + + ParsedFormatBase p3 = p1; // copy construct (nonempty) + EXPECT_EQ(SummarizeParsedFormat(p1), SummarizeParsedFormat(p3)); + + using std::swap; + swap(p1, p2); + EXPECT_EQ("", SummarizeParsedFormat(p1)); + EXPECT_EQ("[hello]{s:1$s}", SummarizeParsedFormat(p2)); + swap(p1, p2); // undo + + p2 = p1; // copy assign + EXPECT_EQ(SummarizeParsedFormat(p1), SummarizeParsedFormat(p2)); +} + +struct ExpectParse { + const char* in; + std::initializer_list<FormatConversionCharSet> conv_set; + const char* out; +}; + +TEST_F(ParsedFormatTest, Parsing) { + // Parse should be equivalent to that obtained by ConversionParseIterator. + // No need to retest the parsing edge cases here. + const ExpectParse kExpect[] = { + {"", {}, ""}, + {"ab", {}, "[ab]"}, + {"a%d", {FormatConversionCharSetInternal::d}, "[a]{d:1$d}"}, + {"a%+d", {FormatConversionCharSetInternal::d}, "[a]{+d:1$d}"}, + {"a% d", {FormatConversionCharSetInternal::d}, "[a]{ d:1$d}"}, + {"a%b %d", {}, "[a]!"}, // stop after error + }; + for (const auto& e : kExpect) { + SCOPED_TRACE(e.in); + EXPECT_EQ(e.out, + SummarizeParsedFormat(ParsedFormatBase(e.in, false, e.conv_set))); + } +} + +TEST_F(ParsedFormatTest, ParsingFlagOrder) { + const ExpectParse kExpect[] = { + {"a%+ 0d", {FormatConversionCharSetInternal::d}, "[a]{+ 0d:1$d}"}, + {"a%+0 d", {FormatConversionCharSetInternal::d}, "[a]{+0 d:1$d}"}, + {"a%0+ d", {FormatConversionCharSetInternal::d}, "[a]{0+ d:1$d}"}, + {"a% +0d", {FormatConversionCharSetInternal::d}, "[a]{ +0d:1$d}"}, + {"a%0 +d", {FormatConversionCharSetInternal::d}, "[a]{0 +d:1$d}"}, + {"a% 0+d", {FormatConversionCharSetInternal::d}, "[a]{ 0+d:1$d}"}, + {"a%+ 0+d", {FormatConversionCharSetInternal::d}, "[a]{+ 0+d:1$d}"}, + }; + for (const auto& e : kExpect) { + SCOPED_TRACE(e.in); + EXPECT_EQ(e.out, + SummarizeParsedFormat(ParsedFormatBase(e.in, false, e.conv_set))); + } +} + +} // namespace +} // namespace str_format_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/str_join_internal.h b/third_party/abseil_cpp/absl/strings/internal/str_join_internal.h new file mode 100644 index 000000000000..31dbf672f0b6 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_join_internal.h @@ -0,0 +1,314 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// This file declares INTERNAL parts of the Join API that are inlined/templated +// or otherwise need to be available at compile time. The main abstractions +// defined in this file are: +// +// - A handful of default Formatters +// - JoinAlgorithm() overloads +// - JoinRange() overloads +// - JoinTuple() +// +// DO NOT INCLUDE THIS FILE DIRECTLY. Use this file by including +// absl/strings/str_join.h +// +// IWYU pragma: private, include "absl/strings/str_join.h" + +#ifndef ABSL_STRINGS_INTERNAL_STR_JOIN_INTERNAL_H_ +#define ABSL_STRINGS_INTERNAL_STR_JOIN_INTERNAL_H_ + +#include <cstring> +#include <iterator> +#include <memory> +#include <string> +#include <type_traits> +#include <utility> + +#include "absl/strings/internal/ostringstream.h" +#include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/str_cat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// +// Formatter objects +// +// The following are implementation classes for standard Formatter objects. The +// factory functions that users will call to create and use these formatters are +// defined and documented in strings/join.h. +// + +// The default formatter. Converts alpha-numeric types to strings. +struct AlphaNumFormatterImpl { + // This template is needed in order to support passing in a dereferenced + // vector<bool>::iterator + template <typename T> + void operator()(std::string* out, const T& t) const { + StrAppend(out, AlphaNum(t)); + } + + void operator()(std::string* out, const AlphaNum& t) const { + StrAppend(out, t); + } +}; + +// A type that's used to overload the JoinAlgorithm() function (defined below) +// for ranges that do not require additional formatting (e.g., a range of +// strings). + +struct NoFormatter : public AlphaNumFormatterImpl {}; + +// Formats types to strings using the << operator. +class StreamFormatterImpl { + public: + // The method isn't const because it mutates state. Making it const will + // render StreamFormatterImpl thread-hostile. + template <typename T> + void operator()(std::string* out, const T& t) { + // The stream is created lazily to avoid paying the relatively high cost + // of its construction when joining an empty range. + if (strm_) { + strm_->clear(); // clear the bad, fail and eof bits in case they were set + strm_->str(out); + } else { + strm_.reset(new strings_internal::OStringStream(out)); + } + *strm_ << t; + } + + private: + std::unique_ptr<strings_internal::OStringStream> strm_; +}; + +// Formats a std::pair<>. The 'first' member is formatted using f1_ and the +// 'second' member is formatted using f2_. sep_ is the separator. +template <typename F1, typename F2> +class PairFormatterImpl { + public: + PairFormatterImpl(F1 f1, absl::string_view sep, F2 f2) + : f1_(std::move(f1)), sep_(sep), f2_(std::move(f2)) {} + + template <typename T> + void operator()(std::string* out, const T& p) { + f1_(out, p.first); + out->append(sep_); + f2_(out, p.second); + } + + template <typename T> + void operator()(std::string* out, const T& p) const { + f1_(out, p.first); + out->append(sep_); + f2_(out, p.second); + } + + private: + F1 f1_; + std::string sep_; + F2 f2_; +}; + +// Wraps another formatter and dereferences the argument to operator() then +// passes the dereferenced argument to the wrapped formatter. This can be +// useful, for example, to join a std::vector<int*>. +template <typename Formatter> +class DereferenceFormatterImpl { + public: + DereferenceFormatterImpl() : f_() {} + explicit DereferenceFormatterImpl(Formatter&& f) + : f_(std::forward<Formatter>(f)) {} + + template <typename T> + void operator()(std::string* out, const T& t) { + f_(out, *t); + } + + template <typename T> + void operator()(std::string* out, const T& t) const { + f_(out, *t); + } + + private: + Formatter f_; +}; + +// DefaultFormatter<T> is a traits class that selects a default Formatter to use +// for the given type T. The ::Type member names the Formatter to use. This is +// used by the strings::Join() functions that do NOT take a Formatter argument, +// in which case a default Formatter must be chosen. +// +// AlphaNumFormatterImpl is the default in the base template, followed by +// specializations for other types. +template <typename ValueType> +struct DefaultFormatter { + typedef AlphaNumFormatterImpl Type; +}; +template <> +struct DefaultFormatter<const char*> { + typedef AlphaNumFormatterImpl Type; +}; +template <> +struct DefaultFormatter<char*> { + typedef AlphaNumFormatterImpl Type; +}; +template <> +struct DefaultFormatter<std::string> { + typedef NoFormatter Type; +}; +template <> +struct DefaultFormatter<absl::string_view> { + typedef NoFormatter Type; +}; +template <typename ValueType> +struct DefaultFormatter<ValueType*> { + typedef DereferenceFormatterImpl<typename DefaultFormatter<ValueType>::Type> + Type; +}; + +template <typename ValueType> +struct DefaultFormatter<std::unique_ptr<ValueType>> + : public DefaultFormatter<ValueType*> {}; + +// +// JoinAlgorithm() functions +// + +// The main joining algorithm. This simply joins the elements in the given +// iterator range, each separated by the given separator, into an output string, +// and formats each element using the provided Formatter object. +template <typename Iterator, typename Formatter> +std::string JoinAlgorithm(Iterator start, Iterator end, absl::string_view s, + Formatter&& f) { + std::string result; + absl::string_view sep(""); + for (Iterator it = start; it != end; ++it) { + result.append(sep.data(), sep.size()); + f(&result, *it); + sep = s; + } + return result; +} + +// A joining algorithm that's optimized for a forward iterator range of +// string-like objects that do not need any additional formatting. This is to +// optimize the common case of joining, say, a std::vector<string> or a +// std::vector<absl::string_view>. +// +// This is an overload of the previous JoinAlgorithm() function. Here the +// Formatter argument is of type NoFormatter. Since NoFormatter is an internal +// type, this overload is only invoked when strings::Join() is called with a +// range of string-like objects (e.g., std::string, absl::string_view), and an +// explicit Formatter argument was NOT specified. +// +// The optimization is that the needed space will be reserved in the output +// string to avoid the need to resize while appending. To do this, the iterator +// range will be traversed twice: once to calculate the total needed size, and +// then again to copy the elements and delimiters to the output string. +template <typename Iterator, + typename = typename std::enable_if<std::is_convertible< + typename std::iterator_traits<Iterator>::iterator_category, + std::forward_iterator_tag>::value>::type> +std::string JoinAlgorithm(Iterator start, Iterator end, absl::string_view s, + NoFormatter) { + std::string result; + if (start != end) { + // Sums size + size_t result_size = start->size(); + for (Iterator it = start; ++it != end;) { + result_size += s.size(); + result_size += it->size(); + } + + if (result_size > 0) { + STLStringResizeUninitialized(&result, result_size); + + // Joins strings + char* result_buf = &*result.begin(); + memcpy(result_buf, start->data(), start->size()); + result_buf += start->size(); + for (Iterator it = start; ++it != end;) { + memcpy(result_buf, s.data(), s.size()); + result_buf += s.size(); + memcpy(result_buf, it->data(), it->size()); + result_buf += it->size(); + } + } + } + + return result; +} + +// JoinTupleLoop implements a loop over the elements of a std::tuple, which +// are heterogeneous. The primary template matches the tuple interior case. It +// continues the iteration after appending a separator (for nonzero indices) +// and formatting an element of the tuple. The specialization for the I=N case +// matches the end-of-tuple, and terminates the iteration. +template <size_t I, size_t N> +struct JoinTupleLoop { + template <typename Tup, typename Formatter> + void operator()(std::string* out, const Tup& tup, absl::string_view sep, + Formatter&& fmt) { + if (I > 0) out->append(sep.data(), sep.size()); + fmt(out, std::get<I>(tup)); + JoinTupleLoop<I + 1, N>()(out, tup, sep, fmt); + } +}; +template <size_t N> +struct JoinTupleLoop<N, N> { + template <typename Tup, typename Formatter> + void operator()(std::string*, const Tup&, absl::string_view, Formatter&&) {} +}; + +template <typename... T, typename Formatter> +std::string JoinAlgorithm(const std::tuple<T...>& tup, absl::string_view sep, + Formatter&& fmt) { + std::string result; + JoinTupleLoop<0, sizeof...(T)>()(&result, tup, sep, fmt); + return result; +} + +template <typename Iterator> +std::string JoinRange(Iterator first, Iterator last, + absl::string_view separator) { + // No formatter was explicitly given, so a default must be chosen. + typedef typename std::iterator_traits<Iterator>::value_type ValueType; + typedef typename DefaultFormatter<ValueType>::Type Formatter; + return JoinAlgorithm(first, last, separator, Formatter()); +} + +template <typename Range, typename Formatter> +std::string JoinRange(const Range& range, absl::string_view separator, + Formatter&& fmt) { + using std::begin; + using std::end; + return JoinAlgorithm(begin(range), end(range), separator, fmt); +} + +template <typename Range> +std::string JoinRange(const Range& range, absl::string_view separator) { + using std::begin; + using std::end; + return JoinRange(begin(range), end(range), separator); +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_JOIN_INTERNAL_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/str_split_internal.h b/third_party/abseil_cpp/absl/strings/internal/str_split_internal.h new file mode 100644 index 000000000000..a2f41c153131 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/str_split_internal.h @@ -0,0 +1,430 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// This file declares INTERNAL parts of the Split API that are inline/templated +// or otherwise need to be available at compile time. The main abstractions +// defined in here are +// +// - ConvertibleToStringView +// - SplitIterator<> +// - Splitter<> +// +// DO NOT INCLUDE THIS FILE DIRECTLY. Use this file by including +// absl/strings/str_split.h. +// +// IWYU pragma: private, include "absl/strings/str_split.h" + +#ifndef ABSL_STRINGS_INTERNAL_STR_SPLIT_INTERNAL_H_ +#define ABSL_STRINGS_INTERNAL_STR_SPLIT_INTERNAL_H_ + +#include <array> +#include <initializer_list> +#include <iterator> +#include <map> +#include <type_traits> +#include <utility> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" + +#ifdef _GLIBCXX_DEBUG +#include "absl/strings/internal/stl_type_traits.h" +#endif // _GLIBCXX_DEBUG + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// This class is implicitly constructible from everything that absl::string_view +// is implicitly constructible from, except for rvalue strings. This means it +// can be used as a function parameter in places where passing a temporary +// string might cause memory lifetime issues. +class ConvertibleToStringView { + public: + ConvertibleToStringView(const char* s) // NOLINT(runtime/explicit) + : value_(s) {} + ConvertibleToStringView(char* s) : value_(s) {} // NOLINT(runtime/explicit) + ConvertibleToStringView(absl::string_view s) // NOLINT(runtime/explicit) + : value_(s) {} + ConvertibleToStringView(const std::string& s) // NOLINT(runtime/explicit) + : value_(s) {} + + // Matches rvalue strings and moves their data to a member. + ConvertibleToStringView(std::string&& s) = delete; + ConvertibleToStringView(const std::string&& s) = delete; + + absl::string_view value() const { return value_; } + + private: + absl::string_view value_; +}; + +// An iterator that enumerates the parts of a string from a Splitter. The text +// to be split, the Delimiter, and the Predicate are all taken from the given +// Splitter object. Iterators may only be compared if they refer to the same +// Splitter instance. +// +// This class is NOT part of the public splitting API. +template <typename Splitter> +class SplitIterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = absl::string_view; + using difference_type = ptrdiff_t; + using pointer = const value_type*; + using reference = const value_type&; + + enum State { kInitState, kLastState, kEndState }; + SplitIterator(State state, const Splitter* splitter) + : pos_(0), + state_(state), + splitter_(splitter), + delimiter_(splitter->delimiter()), + predicate_(splitter->predicate()) { + // Hack to maintain backward compatibility. This one block makes it so an + // empty absl::string_view whose .data() happens to be nullptr behaves + // *differently* from an otherwise empty absl::string_view whose .data() is + // not nullptr. This is an undesirable difference in general, but this + // behavior is maintained to avoid breaking existing code that happens to + // depend on this old behavior/bug. Perhaps it will be fixed one day. The + // difference in behavior is as follows: + // Split(absl::string_view(""), '-'); // {""} + // Split(absl::string_view(), '-'); // {} + if (splitter_->text().data() == nullptr) { + state_ = kEndState; + pos_ = splitter_->text().size(); + return; + } + + if (state_ == kEndState) { + pos_ = splitter_->text().size(); + } else { + ++(*this); + } + } + + bool at_end() const { return state_ == kEndState; } + + reference operator*() const { return curr_; } + pointer operator->() const { return &curr_; } + + SplitIterator& operator++() { + do { + if (state_ == kLastState) { + state_ = kEndState; + return *this; + } + const absl::string_view text = splitter_->text(); + const absl::string_view d = delimiter_.Find(text, pos_); + if (d.data() == text.data() + text.size()) state_ = kLastState; + curr_ = text.substr(pos_, d.data() - (text.data() + pos_)); + pos_ += curr_.size() + d.size(); + } while (!predicate_(curr_)); + return *this; + } + + SplitIterator operator++(int) { + SplitIterator old(*this); + ++(*this); + return old; + } + + friend bool operator==(const SplitIterator& a, const SplitIterator& b) { + return a.state_ == b.state_ && a.pos_ == b.pos_; + } + + friend bool operator!=(const SplitIterator& a, const SplitIterator& b) { + return !(a == b); + } + + private: + size_t pos_; + State state_; + absl::string_view curr_; + const Splitter* splitter_; + typename Splitter::DelimiterType delimiter_; + typename Splitter::PredicateType predicate_; +}; + +// HasMappedType<T>::value is true iff there exists a type T::mapped_type. +template <typename T, typename = void> +struct HasMappedType : std::false_type {}; +template <typename T> +struct HasMappedType<T, absl::void_t<typename T::mapped_type>> + : std::true_type {}; + +// HasValueType<T>::value is true iff there exists a type T::value_type. +template <typename T, typename = void> +struct HasValueType : std::false_type {}; +template <typename T> +struct HasValueType<T, absl::void_t<typename T::value_type>> : std::true_type { +}; + +// HasConstIterator<T>::value is true iff there exists a type T::const_iterator. +template <typename T, typename = void> +struct HasConstIterator : std::false_type {}; +template <typename T> +struct HasConstIterator<T, absl::void_t<typename T::const_iterator>> + : std::true_type {}; + +// IsInitializerList<T>::value is true iff T is an std::initializer_list. More +// details below in Splitter<> where this is used. +std::false_type IsInitializerListDispatch(...); // default: No +template <typename T> +std::true_type IsInitializerListDispatch(std::initializer_list<T>*); +template <typename T> +struct IsInitializerList + : decltype(IsInitializerListDispatch(static_cast<T*>(nullptr))) {}; + +// A SplitterIsConvertibleTo<C>::type alias exists iff the specified condition +// is true for type 'C'. +// +// Restricts conversion to container-like types (by testing for the presence of +// a const_iterator member type) and also to disable conversion to an +// std::initializer_list (which also has a const_iterator). Otherwise, code +// compiled in C++11 will get an error due to ambiguous conversion paths (in +// C++11 std::vector<T>::operator= is overloaded to take either a std::vector<T> +// or an std::initializer_list<T>). + +template <typename C, bool has_value_type, bool has_mapped_type> +struct SplitterIsConvertibleToImpl : std::false_type {}; + +template <typename C> +struct SplitterIsConvertibleToImpl<C, true, false> + : std::is_constructible<typename C::value_type, absl::string_view> {}; + +template <typename C> +struct SplitterIsConvertibleToImpl<C, true, true> + : absl::conjunction< + std::is_constructible<typename C::key_type, absl::string_view>, + std::is_constructible<typename C::mapped_type, absl::string_view>> {}; + +template <typename C> +struct SplitterIsConvertibleTo + : SplitterIsConvertibleToImpl< + C, +#ifdef _GLIBCXX_DEBUG + !IsStrictlyBaseOfAndConvertibleToSTLContainer<C>::value && +#endif // _GLIBCXX_DEBUG + !IsInitializerList< + typename std::remove_reference<C>::type>::value && + HasValueType<C>::value && HasConstIterator<C>::value, + HasMappedType<C>::value> { +}; + +// This class implements the range that is returned by absl::StrSplit(). This +// class has templated conversion operators that allow it to be implicitly +// converted to a variety of types that the caller may have specified on the +// left-hand side of an assignment. +// +// The main interface for interacting with this class is through its implicit +// conversion operators. However, this class may also be used like a container +// in that it has .begin() and .end() member functions. It may also be used +// within a range-for loop. +// +// Output containers can be collections of any type that is constructible from +// an absl::string_view. +// +// An Predicate functor may be supplied. This predicate will be used to filter +// the split strings: only strings for which the predicate returns true will be +// kept. A Predicate object is any unary functor that takes an absl::string_view +// and returns bool. +// +// The StringType parameter can be either string_view or string, depending on +// whether the Splitter refers to a string stored elsewhere, or if the string +// resides inside the Splitter itself. +template <typename Delimiter, typename Predicate, typename StringType> +class Splitter { + public: + using DelimiterType = Delimiter; + using PredicateType = Predicate; + using const_iterator = strings_internal::SplitIterator<Splitter>; + using value_type = typename std::iterator_traits<const_iterator>::value_type; + + Splitter(StringType input_text, Delimiter d, Predicate p) + : text_(std::move(input_text)), + delimiter_(std::move(d)), + predicate_(std::move(p)) {} + + absl::string_view text() const { return text_; } + const Delimiter& delimiter() const { return delimiter_; } + const Predicate& predicate() const { return predicate_; } + + // Range functions that iterate the split substrings as absl::string_view + // objects. These methods enable a Splitter to be used in a range-based for + // loop. + const_iterator begin() const { return {const_iterator::kInitState, this}; } + const_iterator end() const { return {const_iterator::kEndState, this}; } + + // An implicit conversion operator that is restricted to only those containers + // that the splitter is convertible to. + template <typename Container, + typename = typename std::enable_if< + SplitterIsConvertibleTo<Container>::value>::type> + operator Container() const { // NOLINT(runtime/explicit) + return ConvertToContainer<Container, typename Container::value_type, + HasMappedType<Container>::value>()(*this); + } + + // Returns a pair with its .first and .second members set to the first two + // strings returned by the begin() iterator. Either/both of .first and .second + // will be constructed with empty strings if the iterator doesn't have a + // corresponding value. + template <typename First, typename Second> + operator std::pair<First, Second>() const { // NOLINT(runtime/explicit) + absl::string_view first, second; + auto it = begin(); + if (it != end()) { + first = *it; + if (++it != end()) { + second = *it; + } + } + return {First(first), Second(second)}; + } + + private: + // ConvertToContainer is a functor converting a Splitter to the requested + // Container of ValueType. It is specialized below to optimize splitting to + // certain combinations of Container and ValueType. + // + // This base template handles the generic case of storing the split results in + // the requested non-map-like container and converting the split substrings to + // the requested type. + template <typename Container, typename ValueType, bool is_map = false> + struct ConvertToContainer { + Container operator()(const Splitter& splitter) const { + Container c; + auto it = std::inserter(c, c.end()); + for (const auto& sp : splitter) { + *it++ = ValueType(sp); + } + return c; + } + }; + + // Partial specialization for a std::vector<absl::string_view>. + // + // Optimized for the common case of splitting to a + // std::vector<absl::string_view>. In this case we first split the results to + // a small array of absl::string_view on the stack, to reduce reallocations. + template <typename A> + struct ConvertToContainer<std::vector<absl::string_view, A>, + absl::string_view, false> { + std::vector<absl::string_view, A> operator()( + const Splitter& splitter) const { + struct raw_view { + const char* data; + size_t size; + operator absl::string_view() const { // NOLINT(runtime/explicit) + return {data, size}; + } + }; + std::vector<absl::string_view, A> v; + std::array<raw_view, 16> ar; + for (auto it = splitter.begin(); !it.at_end();) { + size_t index = 0; + do { + ar[index].data = it->data(); + ar[index].size = it->size(); + ++it; + } while (++index != ar.size() && !it.at_end()); + v.insert(v.end(), ar.begin(), ar.begin() + index); + } + return v; + } + }; + + // Partial specialization for a std::vector<std::string>. + // + // Optimized for the common case of splitting to a std::vector<std::string>. + // In this case we first split the results to a std::vector<absl::string_view> + // so the returned std::vector<std::string> can have space reserved to avoid + // std::string moves. + template <typename A> + struct ConvertToContainer<std::vector<std::string, A>, std::string, false> { + std::vector<std::string, A> operator()(const Splitter& splitter) const { + const std::vector<absl::string_view> v = splitter; + return std::vector<std::string, A>(v.begin(), v.end()); + } + }; + + // Partial specialization for containers of pairs (e.g., maps). + // + // The algorithm is to insert a new pair into the map for each even-numbered + // item, with the even-numbered item as the key with a default-constructed + // value. Each odd-numbered item will then be assigned to the last pair's + // value. + template <typename Container, typename First, typename Second> + struct ConvertToContainer<Container, std::pair<const First, Second>, true> { + Container operator()(const Splitter& splitter) const { + Container m; + typename Container::iterator it; + bool insert = true; + for (const auto& sp : splitter) { + if (insert) { + it = Inserter<Container>::Insert(&m, First(sp), Second()); + } else { + it->second = Second(sp); + } + insert = !insert; + } + return m; + } + + // Inserts the key and value into the given map, returning an iterator to + // the inserted item. Specialized for std::map and std::multimap to use + // emplace() and adapt emplace()'s return value. + template <typename Map> + struct Inserter { + using M = Map; + template <typename... Args> + static typename M::iterator Insert(M* m, Args&&... args) { + return m->insert(std::make_pair(std::forward<Args>(args)...)).first; + } + }; + + template <typename... Ts> + struct Inserter<std::map<Ts...>> { + using M = std::map<Ts...>; + template <typename... Args> + static typename M::iterator Insert(M* m, Args&&... args) { + return m->emplace(std::make_pair(std::forward<Args>(args)...)).first; + } + }; + + template <typename... Ts> + struct Inserter<std::multimap<Ts...>> { + using M = std::multimap<Ts...>; + template <typename... Args> + static typename M::iterator Insert(M* m, Args&&... args) { + return m->emplace(std::make_pair(std::forward<Args>(args)...)); + } + }; + }; + + StringType text_; + Delimiter delimiter_; + Predicate predicate_; +}; + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STR_SPLIT_INTERNAL_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/string_constant.h b/third_party/abseil_cpp/absl/strings/internal/string_constant.h new file mode 100644 index 000000000000..b15f1d9bcfac --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/string_constant.h @@ -0,0 +1,70 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_STRING_CONSTANT_H_ +#define ABSL_STRINGS_INTERNAL_STRING_CONSTANT_H_ + +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// StringConstant<T> represents a compile time string constant. +// It can be accessed via its `absl::string_view value` static member. +// It is guaranteed that the `string_view` returned has constant `.data()`, +// constant `.size()` and constant `value[i]` for all `0 <= i < .size()` +// +// The `T` is an opaque type. It is guaranteed that different string constants +// will have different values of `T`. This allows users to associate the string +// constant with other static state at compile time. +// +// Instances should be made using the `MakeStringConstant()` factory function +// below. +template <typename T> +struct StringConstant { + private: + // Returns true if `view` points to constant data. + // Otherwise, it can't be constant evaluated. + static constexpr bool ValidateConstant(absl::string_view view) { + return view.empty() || 2 * view[0] != 1; + } + + public: + static constexpr absl::string_view value = T{}(); + constexpr absl::string_view operator()() const { return value; } + + static_assert(ValidateConstant(value), + "The input string_view must point to constant data."); +}; + +template <typename T> +constexpr absl::string_view StringConstant<T>::value; // NOLINT + +// Factory function for `StringConstant` instances. +// It supports callables that have a constexpr default constructor and a +// constexpr operator(). +// It must return an `absl::string_view` or `const char*` pointing to constant +// data. This is validated at compile time. +template <typename T> +constexpr StringConstant<T> MakeStringConstant(T) { + return {}; +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_STRING_CONSTANT_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/string_constant_test.cc b/third_party/abseil_cpp/absl/strings/internal/string_constant_test.cc new file mode 100644 index 000000000000..392833cf1592 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/string_constant_test.cc @@ -0,0 +1,60 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/string_constant.h" + +#include "absl/meta/type_traits.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +namespace { + +using absl::strings_internal::MakeStringConstant; + +struct Callable { + constexpr absl::string_view operator()() const { + return absl::string_view("Callable", 8); + } +}; + +TEST(StringConstant, Traits) { + constexpr auto str = MakeStringConstant(Callable{}); + using T = decltype(str); + + EXPECT_TRUE(std::is_empty<T>::value); + EXPECT_TRUE(std::is_trivial<T>::value); + EXPECT_TRUE(absl::is_trivially_default_constructible<T>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<T>::value); + EXPECT_TRUE(absl::is_trivially_move_constructible<T>::value); + EXPECT_TRUE(absl::is_trivially_destructible<T>::value); +} + +TEST(StringConstant, MakeFromCallable) { + constexpr auto str = MakeStringConstant(Callable{}); + using T = decltype(str); + EXPECT_EQ(Callable{}(), T::value); + EXPECT_EQ(Callable{}(), str()); +} + +TEST(StringConstant, MakeFromStringConstant) { + // We want to make sure the StringConstant itself is a valid input to the + // factory function. + constexpr auto str = MakeStringConstant(Callable{}); + constexpr auto str2 = MakeStringConstant(str); + using T = decltype(str2); + EXPECT_EQ(Callable{}(), T::value); + EXPECT_EQ(Callable{}(), str2()); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/internal/utf8.cc b/third_party/abseil_cpp/absl/strings/internal/utf8.cc new file mode 100644 index 000000000000..8fd8edc1ec6f --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/utf8.cc @@ -0,0 +1,53 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// UTF8 utilities, implemented to reduce dependencies. + +#include "absl/strings/internal/utf8.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +size_t EncodeUTF8Char(char *buffer, char32_t utf8_char) { + if (utf8_char <= 0x7F) { + *buffer = static_cast<char>(utf8_char); + return 1; + } else if (utf8_char <= 0x7FF) { + buffer[1] = 0x80 | (utf8_char & 0x3F); + utf8_char >>= 6; + buffer[0] = 0xC0 | utf8_char; + return 2; + } else if (utf8_char <= 0xFFFF) { + buffer[2] = 0x80 | (utf8_char & 0x3F); + utf8_char >>= 6; + buffer[1] = 0x80 | (utf8_char & 0x3F); + utf8_char >>= 6; + buffer[0] = 0xE0 | utf8_char; + return 3; + } else { + buffer[3] = 0x80 | (utf8_char & 0x3F); + utf8_char >>= 6; + buffer[2] = 0x80 | (utf8_char & 0x3F); + utf8_char >>= 6; + buffer[1] = 0x80 | (utf8_char & 0x3F); + utf8_char >>= 6; + buffer[0] = 0xF0 | utf8_char; + return 4; + } +} + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/internal/utf8.h b/third_party/abseil_cpp/absl/strings/internal/utf8.h new file mode 100644 index 000000000000..32fb1093bea3 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/utf8.h @@ -0,0 +1,50 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// UTF8 utilities, implemented to reduce dependencies. + +#ifndef ABSL_STRINGS_INTERNAL_UTF8_H_ +#define ABSL_STRINGS_INTERNAL_UTF8_H_ + +#include <cstddef> +#include <cstdint> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +// For Unicode code points 0 through 0x10FFFF, EncodeUTF8Char writes +// out the UTF-8 encoding into buffer, and returns the number of chars +// it wrote. +// +// As described in https://tools.ietf.org/html/rfc3629#section-3 , the encodings +// are: +// 00 - 7F : 0xxxxxxx +// 80 - 7FF : 110xxxxx 10xxxxxx +// 800 - FFFF : 1110xxxx 10xxxxxx 10xxxxxx +// 10000 - 10FFFF : 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx +// +// Values greater than 0x10FFFF are not supported and may or may not write +// characters into buffer, however never will more than kMaxEncodedUTF8Size +// bytes be written, regardless of the value of utf8_char. +enum { kMaxEncodedUTF8Size = 4 }; +size_t EncodeUTF8Char(char *buffer, char32_t utf8_char); + +} // namespace strings_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_UTF8_H_ diff --git a/third_party/abseil_cpp/absl/strings/internal/utf8_test.cc b/third_party/abseil_cpp/absl/strings/internal/utf8_test.cc new file mode 100644 index 000000000000..88dd5036e3da --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/internal/utf8_test.cc @@ -0,0 +1,66 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/utf8.h" + +#include <cstdint> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/base/port.h" + +namespace { + +#if !defined(__cpp_char8_t) +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++2a-compat" +#endif +TEST(EncodeUTF8Char, BasicFunction) { + std::pair<char32_t, std::string> tests[] = {{0x0030, u8"\u0030"}, + {0x00A3, u8"\u00A3"}, + {0x00010000, u8"\U00010000"}, + {0x0000FFFF, u8"\U0000FFFF"}, + {0x0010FFFD, u8"\U0010FFFD"}}; + for (auto &test : tests) { + char buf0[7] = {'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}; + char buf1[7] = {'\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF', '\xFF'}; + char *buf0_written = + &buf0[absl::strings_internal::EncodeUTF8Char(buf0, test.first)]; + char *buf1_written = + &buf1[absl::strings_internal::EncodeUTF8Char(buf1, test.first)]; + int apparent_length = 7; + while (buf0[apparent_length - 1] == '\x00' && + buf1[apparent_length - 1] == '\xFF') { + if (--apparent_length == 0) break; + } + EXPECT_EQ(apparent_length, buf0_written - buf0); + EXPECT_EQ(apparent_length, buf1_written - buf1); + EXPECT_EQ(apparent_length, test.second.length()); + EXPECT_EQ(std::string(buf0, apparent_length), test.second); + EXPECT_EQ(std::string(buf1, apparent_length), test.second); + } + char buf[32] = "Don't Tread On Me"; + EXPECT_LE(absl::strings_internal::EncodeUTF8Char(buf, 0x00110000), + absl::strings_internal::kMaxEncodedUTF8Size); + char buf2[32] = "Negative is invalid but sane"; + EXPECT_LE(absl::strings_internal::EncodeUTF8Char(buf2, -1), + absl::strings_internal::kMaxEncodedUTF8Size); +} +#if defined(__clang__) +#pragma clang diagnostic pop +#endif +#endif // !defined(__cpp_char8_t) + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/match.cc b/third_party/abseil_cpp/absl/strings/match.cc new file mode 100644 index 000000000000..8127cb0c5e77 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/match.cc @@ -0,0 +1,40 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/match.h" + +#include "absl/strings/internal/memutil.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +bool EqualsIgnoreCase(absl::string_view piece1, absl::string_view piece2) { + return (piece1.size() == piece2.size() && + 0 == absl::strings_internal::memcasecmp(piece1.data(), piece2.data(), + piece1.size())); + // memcasecmp uses absl::ascii_tolower(). +} + +bool StartsWithIgnoreCase(absl::string_view text, absl::string_view prefix) { + return (text.size() >= prefix.size()) && + EqualsIgnoreCase(text.substr(0, prefix.size()), prefix); +} + +bool EndsWithIgnoreCase(absl::string_view text, absl::string_view suffix) { + return (text.size() >= suffix.size()) && + EqualsIgnoreCase(text.substr(text.size() - suffix.size()), suffix); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/match.h b/third_party/abseil_cpp/absl/strings/match.h new file mode 100644 index 000000000000..90fca98ad2cf --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/match.h @@ -0,0 +1,90 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: match.h +// ----------------------------------------------------------------------------- +// +// This file contains simple utilities for performing string matching checks. +// All of these function parameters are specified as `absl::string_view`, +// meaning that these functions can accept `std::string`, `absl::string_view` or +// NUL-terminated C-style strings. +// +// Examples: +// std::string s = "foo"; +// absl::string_view sv = "f"; +// assert(absl::StrContains(s, sv)); +// +// Note: The order of parameters in these functions is designed to mimic the +// order an equivalent member function would exhibit; +// e.g. `s.Contains(x)` ==> `absl::StrContains(s, x). +#ifndef ABSL_STRINGS_MATCH_H_ +#define ABSL_STRINGS_MATCH_H_ + +#include <cstring> + +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// StrContains() +// +// Returns whether a given string `haystack` contains the substring `needle`. +inline bool StrContains(absl::string_view haystack, absl::string_view needle) { + return haystack.find(needle, 0) != haystack.npos; +} + +// StartsWith() +// +// Returns whether a given string `text` begins with `prefix`. +inline bool StartsWith(absl::string_view text, absl::string_view prefix) { + return prefix.empty() || + (text.size() >= prefix.size() && + memcmp(text.data(), prefix.data(), prefix.size()) == 0); +} + +// EndsWith() +// +// Returns whether a given string `text` ends with `suffix`. +inline bool EndsWith(absl::string_view text, absl::string_view suffix) { + return suffix.empty() || + (text.size() >= suffix.size() && + memcmp(text.data() + (text.size() - suffix.size()), suffix.data(), + suffix.size()) == 0); +} + +// EqualsIgnoreCase() +// +// Returns whether given ASCII strings `piece1` and `piece2` are equal, ignoring +// case in the comparison. +bool EqualsIgnoreCase(absl::string_view piece1, absl::string_view piece2); + +// StartsWithIgnoreCase() +// +// Returns whether a given ASCII string `text` starts with `prefix`, +// ignoring case in the comparison. +bool StartsWithIgnoreCase(absl::string_view text, absl::string_view prefix); + +// EndsWithIgnoreCase() +// +// Returns whether a given ASCII string `text` ends with `suffix`, ignoring +// case in the comparison. +bool EndsWithIgnoreCase(absl::string_view text, absl::string_view suffix); + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_MATCH_H_ diff --git a/third_party/abseil_cpp/absl/strings/match_test.cc b/third_party/abseil_cpp/absl/strings/match_test.cc new file mode 100644 index 000000000000..4c313dda14e3 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/match_test.cc @@ -0,0 +1,110 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/match.h" + +#include "gtest/gtest.h" + +namespace { + +TEST(MatchTest, StartsWith) { + const std::string s1("123\0abc", 7); + const absl::string_view a("foobar"); + const absl::string_view b(s1); + const absl::string_view e; + EXPECT_TRUE(absl::StartsWith(a, a)); + EXPECT_TRUE(absl::StartsWith(a, "foo")); + EXPECT_TRUE(absl::StartsWith(a, e)); + EXPECT_TRUE(absl::StartsWith(b, s1)); + EXPECT_TRUE(absl::StartsWith(b, b)); + EXPECT_TRUE(absl::StartsWith(b, e)); + EXPECT_TRUE(absl::StartsWith(e, "")); + EXPECT_FALSE(absl::StartsWith(a, b)); + EXPECT_FALSE(absl::StartsWith(b, a)); + EXPECT_FALSE(absl::StartsWith(e, a)); +} + +TEST(MatchTest, EndsWith) { + const std::string s1("123\0abc", 7); + const absl::string_view a("foobar"); + const absl::string_view b(s1); + const absl::string_view e; + EXPECT_TRUE(absl::EndsWith(a, a)); + EXPECT_TRUE(absl::EndsWith(a, "bar")); + EXPECT_TRUE(absl::EndsWith(a, e)); + EXPECT_TRUE(absl::EndsWith(b, s1)); + EXPECT_TRUE(absl::EndsWith(b, b)); + EXPECT_TRUE(absl::EndsWith(b, e)); + EXPECT_TRUE(absl::EndsWith(e, "")); + EXPECT_FALSE(absl::EndsWith(a, b)); + EXPECT_FALSE(absl::EndsWith(b, a)); + EXPECT_FALSE(absl::EndsWith(e, a)); +} + +TEST(MatchTest, Contains) { + absl::string_view a("abcdefg"); + absl::string_view b("abcd"); + absl::string_view c("efg"); + absl::string_view d("gh"); + EXPECT_TRUE(absl::StrContains(a, a)); + EXPECT_TRUE(absl::StrContains(a, b)); + EXPECT_TRUE(absl::StrContains(a, c)); + EXPECT_FALSE(absl::StrContains(a, d)); + EXPECT_TRUE(absl::StrContains("", "")); + EXPECT_TRUE(absl::StrContains("abc", "")); + EXPECT_FALSE(absl::StrContains("", "a")); +} + +TEST(MatchTest, ContainsNull) { + const std::string s = "foo"; + const char* cs = "foo"; + const absl::string_view sv("foo"); + const absl::string_view sv2("foo\0bar", 4); + EXPECT_EQ(s, "foo"); + EXPECT_EQ(sv, "foo"); + EXPECT_NE(sv2, "foo"); + EXPECT_TRUE(absl::EndsWith(s, sv)); + EXPECT_TRUE(absl::StartsWith(cs, sv)); + EXPECT_TRUE(absl::StrContains(cs, sv)); + EXPECT_FALSE(absl::StrContains(cs, sv2)); +} + +TEST(MatchTest, EqualsIgnoreCase) { + std::string text = "the"; + absl::string_view data(text); + + EXPECT_TRUE(absl::EqualsIgnoreCase(data, "The")); + EXPECT_TRUE(absl::EqualsIgnoreCase(data, "THE")); + EXPECT_TRUE(absl::EqualsIgnoreCase(data, "the")); + EXPECT_FALSE(absl::EqualsIgnoreCase(data, "Quick")); + EXPECT_FALSE(absl::EqualsIgnoreCase(data, "then")); +} + +TEST(MatchTest, StartsWithIgnoreCase) { + EXPECT_TRUE(absl::StartsWithIgnoreCase("foo", "foo")); + EXPECT_TRUE(absl::StartsWithIgnoreCase("foo", "Fo")); + EXPECT_TRUE(absl::StartsWithIgnoreCase("foo", "")); + EXPECT_FALSE(absl::StartsWithIgnoreCase("foo", "fooo")); + EXPECT_FALSE(absl::StartsWithIgnoreCase("", "fo")); +} + +TEST(MatchTest, EndsWithIgnoreCase) { + EXPECT_TRUE(absl::EndsWithIgnoreCase("foo", "foo")); + EXPECT_TRUE(absl::EndsWithIgnoreCase("foo", "Oo")); + EXPECT_TRUE(absl::EndsWithIgnoreCase("foo", "")); + EXPECT_FALSE(absl::EndsWithIgnoreCase("foo", "fooo")); + EXPECT_FALSE(absl::EndsWithIgnoreCase("", "fo")); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/numbers.cc b/third_party/abseil_cpp/absl/strings/numbers.cc new file mode 100644 index 000000000000..3da1059c908d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/numbers.cc @@ -0,0 +1,1083 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains string processing functions related to +// numeric values. + +#include "absl/strings/numbers.h" + +#include <algorithm> +#include <cassert> +#include <cfloat> // for DBL_DIG and FLT_DIG +#include <cmath> // for HUGE_VAL +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> +#include <utility> + +#include "absl/base/attributes.h" +#include "absl/base/internal/bits.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/ascii.h" +#include "absl/strings/charconv.h" +#include "absl/strings/escaping.h" +#include "absl/strings/internal/memutil.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +bool SimpleAtof(absl::string_view str, float* out) { + *out = 0.0; + str = StripAsciiWhitespace(str); + if (!str.empty() && str[0] == '+') { + str.remove_prefix(1); + } + auto result = absl::from_chars(str.data(), str.data() + str.size(), *out); + if (result.ec == std::errc::invalid_argument) { + return false; + } + if (result.ptr != str.data() + str.size()) { + // not all non-whitespace characters consumed + return false; + } + // from_chars() with DR 3081's current wording will return max() on + // overflow. SimpleAtof returns infinity instead. + if (result.ec == std::errc::result_out_of_range) { + if (*out > 1.0) { + *out = std::numeric_limits<float>::infinity(); + } else if (*out < -1.0) { + *out = -std::numeric_limits<float>::infinity(); + } + } + return true; +} + +bool SimpleAtod(absl::string_view str, double* out) { + *out = 0.0; + str = StripAsciiWhitespace(str); + if (!str.empty() && str[0] == '+') { + str.remove_prefix(1); + } + auto result = absl::from_chars(str.data(), str.data() + str.size(), *out); + if (result.ec == std::errc::invalid_argument) { + return false; + } + if (result.ptr != str.data() + str.size()) { + // not all non-whitespace characters consumed + return false; + } + // from_chars() with DR 3081's current wording will return max() on + // overflow. SimpleAtod returns infinity instead. + if (result.ec == std::errc::result_out_of_range) { + if (*out > 1.0) { + *out = std::numeric_limits<double>::infinity(); + } else if (*out < -1.0) { + *out = -std::numeric_limits<double>::infinity(); + } + } + return true; +} + +bool SimpleAtob(absl::string_view str, bool* out) { + ABSL_RAW_CHECK(out != nullptr, "Output pointer must not be nullptr."); + if (EqualsIgnoreCase(str, "true") || EqualsIgnoreCase(str, "t") || + EqualsIgnoreCase(str, "yes") || EqualsIgnoreCase(str, "y") || + EqualsIgnoreCase(str, "1")) { + *out = true; + return true; + } + if (EqualsIgnoreCase(str, "false") || EqualsIgnoreCase(str, "f") || + EqualsIgnoreCase(str, "no") || EqualsIgnoreCase(str, "n") || + EqualsIgnoreCase(str, "0")) { + *out = false; + return true; + } + return false; +} + +// ---------------------------------------------------------------------- +// FastIntToBuffer() overloads +// +// Like the Fast*ToBuffer() functions above, these are intended for speed. +// Unlike the Fast*ToBuffer() functions, however, these functions write +// their output to the beginning of the buffer. The caller is responsible +// for ensuring that the buffer has enough space to hold the output. +// +// Returns a pointer to the end of the string (i.e. the null character +// terminating the string). +// ---------------------------------------------------------------------- + +namespace { + +// Used to optimize printing a decimal number's final digit. +const char one_ASCII_final_digits[10][2] { + {'0', 0}, {'1', 0}, {'2', 0}, {'3', 0}, {'4', 0}, + {'5', 0}, {'6', 0}, {'7', 0}, {'8', 0}, {'9', 0}, +}; + +} // namespace + +char* numbers_internal::FastIntToBuffer(uint32_t i, char* buffer) { + uint32_t digits; + // The idea of this implementation is to trim the number of divides to as few + // as possible, and also reducing memory stores and branches, by going in + // steps of two digits at a time rather than one whenever possible. + // The huge-number case is first, in the hopes that the compiler will output + // that case in one branch-free block of code, and only output conditional + // branches into it from below. + if (i >= 1000000000) { // >= 1,000,000,000 + digits = i / 100000000; // 100,000,000 + i -= digits * 100000000; + PutTwoDigits(digits, buffer); + buffer += 2; + lt100_000_000: + digits = i / 1000000; // 1,000,000 + i -= digits * 1000000; + PutTwoDigits(digits, buffer); + buffer += 2; + lt1_000_000: + digits = i / 10000; // 10,000 + i -= digits * 10000; + PutTwoDigits(digits, buffer); + buffer += 2; + lt10_000: + digits = i / 100; + i -= digits * 100; + PutTwoDigits(digits, buffer); + buffer += 2; + lt100: + digits = i; + PutTwoDigits(digits, buffer); + buffer += 2; + *buffer = 0; + return buffer; + } + + if (i < 100) { + digits = i; + if (i >= 10) goto lt100; + memcpy(buffer, one_ASCII_final_digits[i], 2); + return buffer + 1; + } + if (i < 10000) { // 10,000 + if (i >= 1000) goto lt10_000; + digits = i / 100; + i -= digits * 100; + *buffer++ = '0' + digits; + goto lt100; + } + if (i < 1000000) { // 1,000,000 + if (i >= 100000) goto lt1_000_000; + digits = i / 10000; // 10,000 + i -= digits * 10000; + *buffer++ = '0' + digits; + goto lt10_000; + } + if (i < 100000000) { // 100,000,000 + if (i >= 10000000) goto lt100_000_000; + digits = i / 1000000; // 1,000,000 + i -= digits * 1000000; + *buffer++ = '0' + digits; + goto lt1_000_000; + } + // we already know that i < 1,000,000,000 + digits = i / 100000000; // 100,000,000 + i -= digits * 100000000; + *buffer++ = '0' + digits; + goto lt100_000_000; +} + +char* numbers_internal::FastIntToBuffer(int32_t i, char* buffer) { + uint32_t u = i; + if (i < 0) { + *buffer++ = '-'; + // We need to do the negation in modular (i.e., "unsigned") + // arithmetic; MSVC++ apprently warns for plain "-u", so + // we write the equivalent expression "0 - u" instead. + u = 0 - u; + } + return numbers_internal::FastIntToBuffer(u, buffer); +} + +char* numbers_internal::FastIntToBuffer(uint64_t i, char* buffer) { + uint32_t u32 = static_cast<uint32_t>(i); + if (u32 == i) return numbers_internal::FastIntToBuffer(u32, buffer); + + // Here we know i has at least 10 decimal digits. + uint64_t top_1to11 = i / 1000000000; + u32 = static_cast<uint32_t>(i - top_1to11 * 1000000000); + uint32_t top_1to11_32 = static_cast<uint32_t>(top_1to11); + + if (top_1to11_32 == top_1to11) { + buffer = numbers_internal::FastIntToBuffer(top_1to11_32, buffer); + } else { + // top_1to11 has more than 32 bits too; print it in two steps. + uint32_t top_8to9 = static_cast<uint32_t>(top_1to11 / 100); + uint32_t mid_2 = static_cast<uint32_t>(top_1to11 - top_8to9 * 100); + buffer = numbers_internal::FastIntToBuffer(top_8to9, buffer); + PutTwoDigits(mid_2, buffer); + buffer += 2; + } + + // We have only 9 digits now, again the maximum uint32_t can handle fully. + uint32_t digits = u32 / 10000000; // 10,000,000 + u32 -= digits * 10000000; + PutTwoDigits(digits, buffer); + buffer += 2; + digits = u32 / 100000; // 100,000 + u32 -= digits * 100000; + PutTwoDigits(digits, buffer); + buffer += 2; + digits = u32 / 1000; // 1,000 + u32 -= digits * 1000; + PutTwoDigits(digits, buffer); + buffer += 2; + digits = u32 / 10; + u32 -= digits * 10; + PutTwoDigits(digits, buffer); + buffer += 2; + memcpy(buffer, one_ASCII_final_digits[u32], 2); + return buffer + 1; +} + +char* numbers_internal::FastIntToBuffer(int64_t i, char* buffer) { + uint64_t u = i; + if (i < 0) { + *buffer++ = '-'; + u = 0 - u; + } + return numbers_internal::FastIntToBuffer(u, buffer); +} + +// Given a 128-bit number expressed as a pair of uint64_t, high half first, +// return that number multiplied by the given 32-bit value. If the result is +// too large to fit in a 128-bit number, divide it by 2 until it fits. +static std::pair<uint64_t, uint64_t> Mul32(std::pair<uint64_t, uint64_t> num, + uint32_t mul) { + uint64_t bits0_31 = num.second & 0xFFFFFFFF; + uint64_t bits32_63 = num.second >> 32; + uint64_t bits64_95 = num.first & 0xFFFFFFFF; + uint64_t bits96_127 = num.first >> 32; + + // The picture so far: each of these 64-bit values has only the lower 32 bits + // filled in. + // bits96_127: [ 00000000 xxxxxxxx ] + // bits64_95: [ 00000000 xxxxxxxx ] + // bits32_63: [ 00000000 xxxxxxxx ] + // bits0_31: [ 00000000 xxxxxxxx ] + + bits0_31 *= mul; + bits32_63 *= mul; + bits64_95 *= mul; + bits96_127 *= mul; + + // Now the top halves may also have value, though all 64 of their bits will + // never be set at the same time, since they are a result of a 32x32 bit + // multiply. This makes the carry calculation slightly easier. + // bits96_127: [ mmmmmmmm | mmmmmmmm ] + // bits64_95: [ | mmmmmmmm mmmmmmmm | ] + // bits32_63: | [ mmmmmmmm | mmmmmmmm ] + // bits0_31: | [ | mmmmmmmm mmmmmmmm ] + // eventually: [ bits128_up | ...bits64_127.... | ..bits0_63... ] + + uint64_t bits0_63 = bits0_31 + (bits32_63 << 32); + uint64_t bits64_127 = bits64_95 + (bits96_127 << 32) + (bits32_63 >> 32) + + (bits0_63 < bits0_31); + uint64_t bits128_up = (bits96_127 >> 32) + (bits64_127 < bits64_95); + if (bits128_up == 0) return {bits64_127, bits0_63}; + + int shift = 64 - base_internal::CountLeadingZeros64(bits128_up); + uint64_t lo = (bits0_63 >> shift) + (bits64_127 << (64 - shift)); + uint64_t hi = (bits64_127 >> shift) + (bits128_up << (64 - shift)); + return {hi, lo}; +} + +// Compute num * 5 ^ expfive, and return the first 128 bits of the result, +// where the first bit is always a one. So PowFive(1, 0) starts 0b100000, +// PowFive(1, 1) starts 0b101000, PowFive(1, 2) starts 0b110010, etc. +static std::pair<uint64_t, uint64_t> PowFive(uint64_t num, int expfive) { + std::pair<uint64_t, uint64_t> result = {num, 0}; + while (expfive >= 13) { + // 5^13 is the highest power of five that will fit in a 32-bit integer. + result = Mul32(result, 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5); + expfive -= 13; + } + constexpr int powers_of_five[13] = { + 1, + 5, + 5 * 5, + 5 * 5 * 5, + 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5, + 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5 * 5}; + result = Mul32(result, powers_of_five[expfive & 15]); + int shift = base_internal::CountLeadingZeros64(result.first); + if (shift != 0) { + result.first = (result.first << shift) + (result.second >> (64 - shift)); + result.second = (result.second << shift); + } + return result; +} + +struct ExpDigits { + int32_t exponent; + char digits[6]; +}; + +// SplitToSix converts value, a positive double-precision floating-point number, +// into a base-10 exponent and 6 ASCII digits, where the first digit is never +// zero. For example, SplitToSix(1) returns an exponent of zero and a digits +// array of {'1', '0', '0', '0', '0', '0'}. If value is exactly halfway between +// two possible representations, e.g. value = 100000.5, then "round to even" is +// performed. +static ExpDigits SplitToSix(const double value) { + ExpDigits exp_dig; + int exp = 5; + double d = value; + // First step: calculate a close approximation of the output, where the + // value d will be between 100,000 and 999,999, representing the digits + // in the output ASCII array, and exp is the base-10 exponent. It would be + // faster to use a table here, and to look up the base-2 exponent of value, + // however value is an IEEE-754 64-bit number, so the table would have 2,000 + // entries, which is not cache-friendly. + if (d >= 999999.5) { + if (d >= 1e+261) exp += 256, d *= 1e-256; + if (d >= 1e+133) exp += 128, d *= 1e-128; + if (d >= 1e+69) exp += 64, d *= 1e-64; + if (d >= 1e+37) exp += 32, d *= 1e-32; + if (d >= 1e+21) exp += 16, d *= 1e-16; + if (d >= 1e+13) exp += 8, d *= 1e-8; + if (d >= 1e+9) exp += 4, d *= 1e-4; + if (d >= 1e+7) exp += 2, d *= 1e-2; + if (d >= 1e+6) exp += 1, d *= 1e-1; + } else { + if (d < 1e-250) exp -= 256, d *= 1e256; + if (d < 1e-122) exp -= 128, d *= 1e128; + if (d < 1e-58) exp -= 64, d *= 1e64; + if (d < 1e-26) exp -= 32, d *= 1e32; + if (d < 1e-10) exp -= 16, d *= 1e16; + if (d < 1e-2) exp -= 8, d *= 1e8; + if (d < 1e+2) exp -= 4, d *= 1e4; + if (d < 1e+4) exp -= 2, d *= 1e2; + if (d < 1e+5) exp -= 1, d *= 1e1; + } + // At this point, d is in the range [99999.5..999999.5) and exp is in the + // range [-324..308]. Since we need to round d up, we want to add a half + // and truncate. + // However, the technique above may have lost some precision, due to its + // repeated multiplication by constants that each may be off by half a bit + // of precision. This only matters if we're close to the edge though. + // Since we'd like to know if the fractional part of d is close to a half, + // we multiply it by 65536 and see if the fractional part is close to 32768. + // (The number doesn't have to be a power of two,but powers of two are faster) + uint64_t d64k = d * 65536; + int dddddd; // A 6-digit decimal integer. + if ((d64k % 65536) == 32767 || (d64k % 65536) == 32768) { + // OK, it's fairly likely that precision was lost above, which is + // not a surprise given only 52 mantissa bits are available. Therefore + // redo the calculation using 128-bit numbers. (64 bits are not enough). + + // Start out with digits rounded down; maybe add one below. + dddddd = static_cast<int>(d64k / 65536); + + // mantissa is a 64-bit integer representing M.mmm... * 2^63. The actual + // value we're representing, of course, is M.mmm... * 2^exp2. + int exp2; + double m = std::frexp(value, &exp2); + uint64_t mantissa = m * (32768.0 * 65536.0 * 65536.0 * 65536.0); + // std::frexp returns an m value in the range [0.5, 1.0), however we + // can't multiply it by 2^64 and convert to an integer because some FPUs + // throw an exception when converting an number higher than 2^63 into an + // integer - even an unsigned 64-bit integer! Fortunately it doesn't matter + // since m only has 52 significant bits anyway. + mantissa <<= 1; + exp2 -= 64; // not needed, but nice for debugging + + // OK, we are here to compare: + // (dddddd + 0.5) * 10^(exp-5) vs. mantissa * 2^exp2 + // so we can round up dddddd if appropriate. Those values span the full + // range of 600 orders of magnitude of IEE 64-bit floating-point. + // Fortunately, we already know they are very close, so we don't need to + // track the base-2 exponent of both sides. This greatly simplifies the + // the math since the 2^exp2 calculation is unnecessary and the power-of-10 + // calculation can become a power-of-5 instead. + + std::pair<uint64_t, uint64_t> edge, val; + if (exp >= 6) { + // Compare (dddddd + 0.5) * 5 ^ (exp - 5) to mantissa + // Since we're tossing powers of two, 2 * dddddd + 1 is the + // same as dddddd + 0.5 + edge = PowFive(2 * dddddd + 1, exp - 5); + + val.first = mantissa; + val.second = 0; + } else { + // We can't compare (dddddd + 0.5) * 5 ^ (exp - 5) to mantissa as we did + // above because (exp - 5) is negative. So we compare (dddddd + 0.5) to + // mantissa * 5 ^ (5 - exp) + edge = PowFive(2 * dddddd + 1, 0); + + val = PowFive(mantissa, 5 - exp); + } + // printf("exp=%d %016lx %016lx vs %016lx %016lx\n", exp, val.first, + // val.second, edge.first, edge.second); + if (val > edge) { + dddddd++; + } else if (val == edge) { + dddddd += (dddddd & 1); + } + } else { + // Here, we are not close to the edge. + dddddd = static_cast<int>((d64k + 32768) / 65536); + } + if (dddddd == 1000000) { + dddddd = 100000; + exp += 1; + } + exp_dig.exponent = exp; + + int two_digits = dddddd / 10000; + dddddd -= two_digits * 10000; + numbers_internal::PutTwoDigits(two_digits, &exp_dig.digits[0]); + + two_digits = dddddd / 100; + dddddd -= two_digits * 100; + numbers_internal::PutTwoDigits(two_digits, &exp_dig.digits[2]); + + numbers_internal::PutTwoDigits(dddddd, &exp_dig.digits[4]); + return exp_dig; +} + +// Helper function for fast formatting of floating-point. +// The result is the same as "%g", a.k.a. "%.6g". +size_t numbers_internal::SixDigitsToBuffer(double d, char* const buffer) { + static_assert(std::numeric_limits<float>::is_iec559, + "IEEE-754/IEC-559 support only"); + + char* out = buffer; // we write data to out, incrementing as we go, but + // FloatToBuffer always returns the address of the buffer + // passed in. + + if (std::isnan(d)) { + strcpy(out, "nan"); // NOLINT(runtime/printf) + return 3; + } + if (d == 0) { // +0 and -0 are handled here + if (std::signbit(d)) *out++ = '-'; + *out++ = '0'; + *out = 0; + return out - buffer; + } + if (d < 0) { + *out++ = '-'; + d = -d; + } + if (std::isinf(d)) { + strcpy(out, "inf"); // NOLINT(runtime/printf) + return out + 3 - buffer; + } + + auto exp_dig = SplitToSix(d); + int exp = exp_dig.exponent; + const char* digits = exp_dig.digits; + out[0] = '0'; + out[1] = '.'; + switch (exp) { + case 5: + memcpy(out, &digits[0], 6), out += 6; + *out = 0; + return out - buffer; + case 4: + memcpy(out, &digits[0], 5), out += 5; + if (digits[5] != '0') { + *out++ = '.'; + *out++ = digits[5]; + } + *out = 0; + return out - buffer; + case 3: + memcpy(out, &digits[0], 4), out += 4; + if ((digits[5] | digits[4]) != '0') { + *out++ = '.'; + *out++ = digits[4]; + if (digits[5] != '0') *out++ = digits[5]; + } + *out = 0; + return out - buffer; + case 2: + memcpy(out, &digits[0], 3), out += 3; + *out++ = '.'; + memcpy(out, &digits[3], 3); + out += 3; + while (out[-1] == '0') --out; + if (out[-1] == '.') --out; + *out = 0; + return out - buffer; + case 1: + memcpy(out, &digits[0], 2), out += 2; + *out++ = '.'; + memcpy(out, &digits[2], 4); + out += 4; + while (out[-1] == '0') --out; + if (out[-1] == '.') --out; + *out = 0; + return out - buffer; + case 0: + memcpy(out, &digits[0], 1), out += 1; + *out++ = '.'; + memcpy(out, &digits[1], 5); + out += 5; + while (out[-1] == '0') --out; + if (out[-1] == '.') --out; + *out = 0; + return out - buffer; + case -4: + out[2] = '0'; + ++out; + ABSL_FALLTHROUGH_INTENDED; + case -3: + out[2] = '0'; + ++out; + ABSL_FALLTHROUGH_INTENDED; + case -2: + out[2] = '0'; + ++out; + ABSL_FALLTHROUGH_INTENDED; + case -1: + out += 2; + memcpy(out, &digits[0], 6); + out += 6; + while (out[-1] == '0') --out; + *out = 0; + return out - buffer; + } + assert(exp < -4 || exp >= 6); + out[0] = digits[0]; + assert(out[1] == '.'); + out += 2; + memcpy(out, &digits[1], 5), out += 5; + while (out[-1] == '0') --out; + if (out[-1] == '.') --out; + *out++ = 'e'; + if (exp > 0) { + *out++ = '+'; + } else { + *out++ = '-'; + exp = -exp; + } + if (exp > 99) { + int dig1 = exp / 100; + exp -= dig1 * 100; + *out++ = '0' + dig1; + } + PutTwoDigits(exp, out); + out += 2; + *out = 0; + return out - buffer; +} + +namespace { +// Represents integer values of digits. +// Uses 36 to indicate an invalid character since we support +// bases up to 36. +static const int8_t kAsciiToInt[256] = { + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, // 16 36s. + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 36, 36, 36, 36, 36, 36, 36, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + 36, 36, 36, 36, 36, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, + 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36, 36}; + +// Parse the sign and optional hex or oct prefix in text. +inline bool safe_parse_sign_and_base(absl::string_view* text /*inout*/, + int* base_ptr /*inout*/, + bool* negative_ptr /*output*/) { + if (text->data() == nullptr) { + return false; + } + + const char* start = text->data(); + const char* end = start + text->size(); + int base = *base_ptr; + + // Consume whitespace. + while (start < end && absl::ascii_isspace(start[0])) { + ++start; + } + while (start < end && absl::ascii_isspace(end[-1])) { + --end; + } + if (start >= end) { + return false; + } + + // Consume sign. + *negative_ptr = (start[0] == '-'); + if (*negative_ptr || start[0] == '+') { + ++start; + if (start >= end) { + return false; + } + } + + // Consume base-dependent prefix. + // base 0: "0x" -> base 16, "0" -> base 8, default -> base 10 + // base 16: "0x" -> base 16 + // Also validate the base. + if (base == 0) { + if (end - start >= 2 && start[0] == '0' && + (start[1] == 'x' || start[1] == 'X')) { + base = 16; + start += 2; + if (start >= end) { + // "0x" with no digits after is invalid. + return false; + } + } else if (end - start >= 1 && start[0] == '0') { + base = 8; + start += 1; + } else { + base = 10; + } + } else if (base == 16) { + if (end - start >= 2 && start[0] == '0' && + (start[1] == 'x' || start[1] == 'X')) { + start += 2; + if (start >= end) { + // "0x" with no digits after is invalid. + return false; + } + } + } else if (base >= 2 && base <= 36) { + // okay + } else { + return false; + } + *text = absl::string_view(start, end - start); + *base_ptr = base; + return true; +} + +// Consume digits. +// +// The classic loop: +// +// for each digit +// value = value * base + digit +// value *= sign +// +// The classic loop needs overflow checking. It also fails on the most +// negative integer, -2147483648 in 32-bit two's complement representation. +// +// My improved loop: +// +// if (!negative) +// for each digit +// value = value * base +// value = value + digit +// else +// for each digit +// value = value * base +// value = value - digit +// +// Overflow checking becomes simple. + +// Lookup tables per IntType: +// vmax/base and vmin/base are precomputed because division costs at least 8ns. +// TODO(junyer): Doing this per base instead (i.e. an array of structs, not a +// struct of arrays) would probably be better in terms of d-cache for the most +// commonly used bases. +template <typename IntType> +struct LookupTables { + ABSL_CONST_INIT static const IntType kVmaxOverBase[]; + ABSL_CONST_INIT static const IntType kVminOverBase[]; +}; + +// An array initializer macro for X/base where base in [0, 36]. +// However, note that lookups for base in [0, 1] should never happen because +// base has been validated to be in [2, 36] by safe_parse_sign_and_base(). +#define X_OVER_BASE_INITIALIZER(X) \ + { \ + 0, 0, X / 2, X / 3, X / 4, X / 5, X / 6, X / 7, X / 8, X / 9, X / 10, \ + X / 11, X / 12, X / 13, X / 14, X / 15, X / 16, X / 17, X / 18, \ + X / 19, X / 20, X / 21, X / 22, X / 23, X / 24, X / 25, X / 26, \ + X / 27, X / 28, X / 29, X / 30, X / 31, X / 32, X / 33, X / 34, \ + X / 35, X / 36, \ + } + +// This kVmaxOverBase is generated with +// for (int base = 2; base < 37; ++base) { +// absl::uint128 max = std::numeric_limits<absl::uint128>::max(); +// auto result = max / base; +// std::cout << " MakeUint128(" << absl::Uint128High64(result) << "u, " +// << absl::Uint128Low64(result) << "u),\n"; +// } +// See https://godbolt.org/z/aneYsb +// +// uint128& operator/=(uint128) is not constexpr, so hardcode the resulting +// array to avoid a static initializer. +template<> +const uint128 LookupTables<uint128>::kVmaxOverBase[] = { + 0, + 0, + MakeUint128(9223372036854775807u, 18446744073709551615u), + MakeUint128(6148914691236517205u, 6148914691236517205u), + MakeUint128(4611686018427387903u, 18446744073709551615u), + MakeUint128(3689348814741910323u, 3689348814741910323u), + MakeUint128(3074457345618258602u, 12297829382473034410u), + MakeUint128(2635249153387078802u, 5270498306774157604u), + MakeUint128(2305843009213693951u, 18446744073709551615u), + MakeUint128(2049638230412172401u, 14347467612885206812u), + MakeUint128(1844674407370955161u, 11068046444225730969u), + MakeUint128(1676976733973595601u, 8384883669867978007u), + MakeUint128(1537228672809129301u, 6148914691236517205u), + MakeUint128(1418980313362273201u, 4256940940086819603u), + MakeUint128(1317624576693539401u, 2635249153387078802u), + MakeUint128(1229782938247303441u, 1229782938247303441u), + MakeUint128(1152921504606846975u, 18446744073709551615u), + MakeUint128(1085102592571150095u, 1085102592571150095u), + MakeUint128(1024819115206086200u, 16397105843297379214u), + MakeUint128(970881267037344821u, 16504981539634861972u), + MakeUint128(922337203685477580u, 14757395258967641292u), + MakeUint128(878416384462359600u, 14054662151397753612u), + MakeUint128(838488366986797800u, 13415813871788764811u), + MakeUint128(802032351030850070u, 4812194106185100421u), + MakeUint128(768614336404564650u, 12297829382473034410u), + MakeUint128(737869762948382064u, 11805916207174113034u), + MakeUint128(709490156681136600u, 11351842506898185609u), + MakeUint128(683212743470724133u, 17080318586768103348u), + MakeUint128(658812288346769700u, 10540996613548315209u), + MakeUint128(636094623231363848u, 15266270957552732371u), + MakeUint128(614891469123651720u, 9838263505978427528u), + MakeUint128(595056260442243600u, 9520900167075897608u), + MakeUint128(576460752303423487u, 18446744073709551615u), + MakeUint128(558992244657865200u, 8943875914525843207u), + MakeUint128(542551296285575047u, 9765923333140350855u), + MakeUint128(527049830677415760u, 8432797290838652167u), + MakeUint128(512409557603043100u, 8198552921648689607u), +}; + +// This kVmaxOverBase generated with +// for (int base = 2; base < 37; ++base) { +// absl::int128 max = std::numeric_limits<absl::int128>::max(); +// auto result = max / base; +// std::cout << "\tMakeInt128(" << absl::Int128High64(result) << ", " +// << absl::Int128Low64(result) << "u),\n"; +// } +// See https://godbolt.org/z/7djYWz +// +// int128& operator/=(int128) is not constexpr, so hardcode the resulting array +// to avoid a static initializer. +template<> +const int128 LookupTables<int128>::kVmaxOverBase[] = { + 0, + 0, + MakeInt128(4611686018427387903, 18446744073709551615u), + MakeInt128(3074457345618258602, 12297829382473034410u), + MakeInt128(2305843009213693951, 18446744073709551615u), + MakeInt128(1844674407370955161, 11068046444225730969u), + MakeInt128(1537228672809129301, 6148914691236517205u), + MakeInt128(1317624576693539401, 2635249153387078802u), + MakeInt128(1152921504606846975, 18446744073709551615u), + MakeInt128(1024819115206086200, 16397105843297379214u), + MakeInt128(922337203685477580, 14757395258967641292u), + MakeInt128(838488366986797800, 13415813871788764811u), + MakeInt128(768614336404564650, 12297829382473034410u), + MakeInt128(709490156681136600, 11351842506898185609u), + MakeInt128(658812288346769700, 10540996613548315209u), + MakeInt128(614891469123651720, 9838263505978427528u), + MakeInt128(576460752303423487, 18446744073709551615u), + MakeInt128(542551296285575047, 9765923333140350855u), + MakeInt128(512409557603043100, 8198552921648689607u), + MakeInt128(485440633518672410, 17475862806672206794u), + MakeInt128(461168601842738790, 7378697629483820646u), + MakeInt128(439208192231179800, 7027331075698876806u), + MakeInt128(419244183493398900, 6707906935894382405u), + MakeInt128(401016175515425035, 2406097053092550210u), + MakeInt128(384307168202282325, 6148914691236517205u), + MakeInt128(368934881474191032, 5902958103587056517u), + MakeInt128(354745078340568300, 5675921253449092804u), + MakeInt128(341606371735362066, 17763531330238827482u), + MakeInt128(329406144173384850, 5270498306774157604u), + MakeInt128(318047311615681924, 7633135478776366185u), + MakeInt128(307445734561825860, 4919131752989213764u), + MakeInt128(297528130221121800, 4760450083537948804u), + MakeInt128(288230376151711743, 18446744073709551615u), + MakeInt128(279496122328932600, 4471937957262921603u), + MakeInt128(271275648142787523, 14106333703424951235u), + MakeInt128(263524915338707880, 4216398645419326083u), + MakeInt128(256204778801521550, 4099276460824344803u), +}; + +// This kVminOverBase generated with +// for (int base = 2; base < 37; ++base) { +// absl::int128 min = std::numeric_limits<absl::int128>::min(); +// auto result = min / base; +// std::cout << "\tMakeInt128(" << absl::Int128High64(result) << ", " +// << absl::Int128Low64(result) << "u),\n"; +// } +// +// See https://godbolt.org/z/7djYWz +// +// int128& operator/=(int128) is not constexpr, so hardcode the resulting array +// to avoid a static initializer. +template<> +const int128 LookupTables<int128>::kVminOverBase[] = { + 0, + 0, + MakeInt128(-4611686018427387904, 0u), + MakeInt128(-3074457345618258603, 6148914691236517206u), + MakeInt128(-2305843009213693952, 0u), + MakeInt128(-1844674407370955162, 7378697629483820647u), + MakeInt128(-1537228672809129302, 12297829382473034411u), + MakeInt128(-1317624576693539402, 15811494920322472814u), + MakeInt128(-1152921504606846976, 0u), + MakeInt128(-1024819115206086201, 2049638230412172402u), + MakeInt128(-922337203685477581, 3689348814741910324u), + MakeInt128(-838488366986797801, 5030930201920786805u), + MakeInt128(-768614336404564651, 6148914691236517206u), + MakeInt128(-709490156681136601, 7094901566811366007u), + MakeInt128(-658812288346769701, 7905747460161236407u), + MakeInt128(-614891469123651721, 8608480567731124088u), + MakeInt128(-576460752303423488, 0u), + MakeInt128(-542551296285575048, 8680820740569200761u), + MakeInt128(-512409557603043101, 10248191152060862009u), + MakeInt128(-485440633518672411, 970881267037344822u), + MakeInt128(-461168601842738791, 11068046444225730970u), + MakeInt128(-439208192231179801, 11419412998010674810u), + MakeInt128(-419244183493398901, 11738837137815169211u), + MakeInt128(-401016175515425036, 16040647020617001406u), + MakeInt128(-384307168202282326, 12297829382473034411u), + MakeInt128(-368934881474191033, 12543785970122495099u), + MakeInt128(-354745078340568301, 12770822820260458812u), + MakeInt128(-341606371735362067, 683212743470724134u), + MakeInt128(-329406144173384851, 13176245766935394012u), + MakeInt128(-318047311615681925, 10813608594933185431u), + MakeInt128(-307445734561825861, 13527612320720337852u), + MakeInt128(-297528130221121801, 13686293990171602812u), + MakeInt128(-288230376151711744, 0u), + MakeInt128(-279496122328932601, 13974806116446630013u), + MakeInt128(-271275648142787524, 4340410370284600381u), + MakeInt128(-263524915338707881, 14230345428290225533u), + MakeInt128(-256204778801521551, 14347467612885206813u), +}; + +template <typename IntType> +const IntType LookupTables<IntType>::kVmaxOverBase[] = + X_OVER_BASE_INITIALIZER(std::numeric_limits<IntType>::max()); + +template <typename IntType> +const IntType LookupTables<IntType>::kVminOverBase[] = + X_OVER_BASE_INITIALIZER(std::numeric_limits<IntType>::min()); + +#undef X_OVER_BASE_INITIALIZER + +template <typename IntType> +inline bool safe_parse_positive_int(absl::string_view text, int base, + IntType* value_p) { + IntType value = 0; + const IntType vmax = std::numeric_limits<IntType>::max(); + assert(vmax > 0); + assert(base >= 0); + assert(vmax >= static_cast<IntType>(base)); + const IntType vmax_over_base = LookupTables<IntType>::kVmaxOverBase[base]; + assert(base < 2 || + std::numeric_limits<IntType>::max() / base == vmax_over_base); + const char* start = text.data(); + const char* end = start + text.size(); + // loop over digits + for (; start < end; ++start) { + unsigned char c = static_cast<unsigned char>(start[0]); + int digit = kAsciiToInt[c]; + if (digit >= base) { + *value_p = value; + return false; + } + if (value > vmax_over_base) { + *value_p = vmax; + return false; + } + value *= base; + if (value > vmax - digit) { + *value_p = vmax; + return false; + } + value += digit; + } + *value_p = value; + return true; +} + +template <typename IntType> +inline bool safe_parse_negative_int(absl::string_view text, int base, + IntType* value_p) { + IntType value = 0; + const IntType vmin = std::numeric_limits<IntType>::min(); + assert(vmin < 0); + assert(vmin <= 0 - base); + IntType vmin_over_base = LookupTables<IntType>::kVminOverBase[base]; + assert(base < 2 || + std::numeric_limits<IntType>::min() / base == vmin_over_base); + // 2003 c++ standard [expr.mul] + // "... the sign of the remainder is implementation-defined." + // Although (vmin/base)*base + vmin%base is always vmin. + // 2011 c++ standard tightens the spec but we cannot rely on it. + // TODO(junyer): Handle this in the lookup table generation. + if (vmin % base > 0) { + vmin_over_base += 1; + } + const char* start = text.data(); + const char* end = start + text.size(); + // loop over digits + for (; start < end; ++start) { + unsigned char c = static_cast<unsigned char>(start[0]); + int digit = kAsciiToInt[c]; + if (digit >= base) { + *value_p = value; + return false; + } + if (value < vmin_over_base) { + *value_p = vmin; + return false; + } + value *= base; + if (value < vmin + digit) { + *value_p = vmin; + return false; + } + value -= digit; + } + *value_p = value; + return true; +} + +// Input format based on POSIX.1-2008 strtol +// http://pubs.opengroup.org/onlinepubs/9699919799/functions/strtol.html +template <typename IntType> +inline bool safe_int_internal(absl::string_view text, IntType* value_p, + int base) { + *value_p = 0; + bool negative; + if (!safe_parse_sign_and_base(&text, &base, &negative)) { + return false; + } + if (!negative) { + return safe_parse_positive_int(text, base, value_p); + } else { + return safe_parse_negative_int(text, base, value_p); + } +} + +template <typename IntType> +inline bool safe_uint_internal(absl::string_view text, IntType* value_p, + int base) { + *value_p = 0; + bool negative; + if (!safe_parse_sign_and_base(&text, &base, &negative) || negative) { + return false; + } + return safe_parse_positive_int(text, base, value_p); +} +} // anonymous namespace + +namespace numbers_internal { + +// Digit conversion. +ABSL_CONST_INIT ABSL_DLL const char kHexChar[] = + "0123456789abcdef"; + +ABSL_CONST_INIT ABSL_DLL const char kHexTable[513] = + "000102030405060708090a0b0c0d0e0f" + "101112131415161718191a1b1c1d1e1f" + "202122232425262728292a2b2c2d2e2f" + "303132333435363738393a3b3c3d3e3f" + "404142434445464748494a4b4c4d4e4f" + "505152535455565758595a5b5c5d5e5f" + "606162636465666768696a6b6c6d6e6f" + "707172737475767778797a7b7c7d7e7f" + "808182838485868788898a8b8c8d8e8f" + "909192939495969798999a9b9c9d9e9f" + "a0a1a2a3a4a5a6a7a8a9aaabacadaeaf" + "b0b1b2b3b4b5b6b7b8b9babbbcbdbebf" + "c0c1c2c3c4c5c6c7c8c9cacbcccdcecf" + "d0d1d2d3d4d5d6d7d8d9dadbdcdddedf" + "e0e1e2e3e4e5e6e7e8e9eaebecedeeef" + "f0f1f2f3f4f5f6f7f8f9fafbfcfdfeff"; + +ABSL_CONST_INIT ABSL_DLL const char two_ASCII_digits[100][2] = { + {'0', '0'}, {'0', '1'}, {'0', '2'}, {'0', '3'}, {'0', '4'}, {'0', '5'}, + {'0', '6'}, {'0', '7'}, {'0', '8'}, {'0', '9'}, {'1', '0'}, {'1', '1'}, + {'1', '2'}, {'1', '3'}, {'1', '4'}, {'1', '5'}, {'1', '6'}, {'1', '7'}, + {'1', '8'}, {'1', '9'}, {'2', '0'}, {'2', '1'}, {'2', '2'}, {'2', '3'}, + {'2', '4'}, {'2', '5'}, {'2', '6'}, {'2', '7'}, {'2', '8'}, {'2', '9'}, + {'3', '0'}, {'3', '1'}, {'3', '2'}, {'3', '3'}, {'3', '4'}, {'3', '5'}, + {'3', '6'}, {'3', '7'}, {'3', '8'}, {'3', '9'}, {'4', '0'}, {'4', '1'}, + {'4', '2'}, {'4', '3'}, {'4', '4'}, {'4', '5'}, {'4', '6'}, {'4', '7'}, + {'4', '8'}, {'4', '9'}, {'5', '0'}, {'5', '1'}, {'5', '2'}, {'5', '3'}, + {'5', '4'}, {'5', '5'}, {'5', '6'}, {'5', '7'}, {'5', '8'}, {'5', '9'}, + {'6', '0'}, {'6', '1'}, {'6', '2'}, {'6', '3'}, {'6', '4'}, {'6', '5'}, + {'6', '6'}, {'6', '7'}, {'6', '8'}, {'6', '9'}, {'7', '0'}, {'7', '1'}, + {'7', '2'}, {'7', '3'}, {'7', '4'}, {'7', '5'}, {'7', '6'}, {'7', '7'}, + {'7', '8'}, {'7', '9'}, {'8', '0'}, {'8', '1'}, {'8', '2'}, {'8', '3'}, + {'8', '4'}, {'8', '5'}, {'8', '6'}, {'8', '7'}, {'8', '8'}, {'8', '9'}, + {'9', '0'}, {'9', '1'}, {'9', '2'}, {'9', '3'}, {'9', '4'}, {'9', '5'}, + {'9', '6'}, {'9', '7'}, {'9', '8'}, {'9', '9'}}; + +bool safe_strto32_base(absl::string_view text, int32_t* value, int base) { + return safe_int_internal<int32_t>(text, value, base); +} + +bool safe_strto64_base(absl::string_view text, int64_t* value, int base) { + return safe_int_internal<int64_t>(text, value, base); +} + +bool safe_strto128_base(absl::string_view text, int128* value, int base) { + return safe_int_internal<absl::int128>(text, value, base); +} + +bool safe_strtou32_base(absl::string_view text, uint32_t* value, int base) { + return safe_uint_internal<uint32_t>(text, value, base); +} + +bool safe_strtou64_base(absl::string_view text, uint64_t* value, int base) { + return safe_uint_internal<uint64_t>(text, value, base); +} + +bool safe_strtou128_base(absl::string_view text, uint128* value, int base) { + return safe_uint_internal<absl::uint128>(text, value, base); +} + +} // namespace numbers_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/numbers.h b/third_party/abseil_cpp/absl/strings/numbers.h new file mode 100644 index 000000000000..2e004b44f887 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/numbers.h @@ -0,0 +1,273 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: numbers.h +// ----------------------------------------------------------------------------- +// +// This package contains functions for converting strings to numbers. For +// converting numbers to strings, use `StrCat()` or `StrAppend()` in str_cat.h, +// which automatically detect and convert most number values appropriately. + +#ifndef ABSL_STRINGS_NUMBERS_H_ +#define ABSL_STRINGS_NUMBERS_H_ + +#ifdef __SSE4_2__ +#include <x86intrin.h> +#endif + +#include <cstddef> +#include <cstdlib> +#include <cstring> +#include <ctime> +#include <limits> +#include <string> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/base/internal/bits.h" +#ifdef __SSE4_2__ +// TODO(jorg): Remove this when we figure out the right way +// to swap bytes on SSE 4.2 that works with the compilers +// we claim to support. Also, add tests for the compiler +// that doesn't support the Intel _bswap64 intrinsic but +// does support all the SSE 4.2 intrinsics +#include "absl/base/internal/endian.h" +#endif +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/numeric/int128.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// SimpleAtoi() +// +// Converts the given string (optionally followed or preceded by ASCII +// whitespace) into an integer value, returning `true` if successful. The string +// must reflect a base-10 integer whose value falls within the range of the +// integer type (optionally preceded by a `+` or `-`). If any errors are +// encountered, this function returns `false`, leaving `out` in an unspecified +// state. +template <typename int_type> +ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, int_type* out); + +// SimpleAtof() +// +// Converts the given string (optionally followed or preceded by ASCII +// whitespace) into a float, which may be rounded on overflow or underflow, +// returning `true` if successful. +// See https://en.cppreference.com/w/c/string/byte/strtof for details about the +// allowed formats for `str`, except SimpleAtof() is locale-independent and will +// always use the "C" locale. If any errors are encountered, this function +// returns `false`, leaving `out` in an unspecified state. +ABSL_MUST_USE_RESULT bool SimpleAtof(absl::string_view str, float* out); + +// SimpleAtod() +// +// Converts the given string (optionally followed or preceded by ASCII +// whitespace) into a double, which may be rounded on overflow or underflow, +// returning `true` if successful. +// See https://en.cppreference.com/w/c/string/byte/strtof for details about the +// allowed formats for `str`, except SimpleAtod is locale-independent and will +// always use the "C" locale. If any errors are encountered, this function +// returns `false`, leaving `out` in an unspecified state. +ABSL_MUST_USE_RESULT bool SimpleAtod(absl::string_view str, double* out); + +// SimpleAtob() +// +// Converts the given string into a boolean, returning `true` if successful. +// The following case-insensitive strings are interpreted as boolean `true`: +// "true", "t", "yes", "y", "1". The following case-insensitive strings +// are interpreted as boolean `false`: "false", "f", "no", "n", "0". If any +// errors are encountered, this function returns `false`, leaving `out` in an +// unspecified state. +ABSL_MUST_USE_RESULT bool SimpleAtob(absl::string_view str, bool* out); + +ABSL_NAMESPACE_END +} // namespace absl + +// End of public API. Implementation details follow. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace numbers_internal { + +// Digit conversion. +ABSL_DLL extern const char kHexChar[17]; // 0123456789abcdef +ABSL_DLL extern const char + kHexTable[513]; // 000102030405060708090a0b0c0d0e0f1011... +ABSL_DLL extern const char + two_ASCII_digits[100][2]; // 00, 01, 02, 03... + +// Writes a two-character representation of 'i' to 'buf'. 'i' must be in the +// range 0 <= i < 100, and buf must have space for two characters. Example: +// char buf[2]; +// PutTwoDigits(42, buf); +// // buf[0] == '4' +// // buf[1] == '2' +inline void PutTwoDigits(size_t i, char* buf) { + assert(i < 100); + memcpy(buf, two_ASCII_digits[i], 2); +} + +// safe_strto?() functions for implementing SimpleAtoi() +bool safe_strto32_base(absl::string_view text, int32_t* value, int base); +bool safe_strto64_base(absl::string_view text, int64_t* value, int base); +bool safe_strto128_base(absl::string_view text, absl::int128* value, + int base); +bool safe_strtou32_base(absl::string_view text, uint32_t* value, int base); +bool safe_strtou64_base(absl::string_view text, uint64_t* value, int base); +bool safe_strtou128_base(absl::string_view text, absl::uint128* value, + int base); + +static const int kFastToBufferSize = 32; +static const int kSixDigitsToBufferSize = 16; + +// Helper function for fast formatting of floating-point values. +// The result is the same as printf's "%g", a.k.a. "%.6g"; that is, six +// significant digits are returned, trailing zeros are removed, and numbers +// outside the range 0.0001-999999 are output using scientific notation +// (1.23456e+06). This routine is heavily optimized. +// Required buffer size is `kSixDigitsToBufferSize`. +size_t SixDigitsToBuffer(double d, char* buffer); + +// These functions are intended for speed. All functions take an output buffer +// as an argument and return a pointer to the last byte they wrote, which is the +// terminating '\0'. At most `kFastToBufferSize` bytes are written. +char* FastIntToBuffer(int32_t, char*); +char* FastIntToBuffer(uint32_t, char*); +char* FastIntToBuffer(int64_t, char*); +char* FastIntToBuffer(uint64_t, char*); + +// For enums and integer types that are not an exact match for the types above, +// use templates to call the appropriate one of the four overloads above. +template <typename int_type> +char* FastIntToBuffer(int_type i, char* buffer) { + static_assert(sizeof(i) <= 64 / 8, + "FastIntToBuffer works only with 64-bit-or-less integers."); + // TODO(jorg): This signed-ness check is used because it works correctly + // with enums, and it also serves to check that int_type is not a pointer. + // If one day something like std::is_signed<enum E> works, switch to it. + if (static_cast<int_type>(1) - 2 < 0) { // Signed + if (sizeof(i) > 32 / 8) { // 33-bit to 64-bit + return FastIntToBuffer(static_cast<int64_t>(i), buffer); + } else { // 32-bit or less + return FastIntToBuffer(static_cast<int32_t>(i), buffer); + } + } else { // Unsigned + if (sizeof(i) > 32 / 8) { // 33-bit to 64-bit + return FastIntToBuffer(static_cast<uint64_t>(i), buffer); + } else { // 32-bit or less + return FastIntToBuffer(static_cast<uint32_t>(i), buffer); + } + } +} + +// Implementation of SimpleAtoi, generalized to support arbitrary base (used +// with base different from 10 elsewhere in Abseil implementation). +template <typename int_type> +ABSL_MUST_USE_RESULT bool safe_strtoi_base(absl::string_view s, int_type* out, + int base) { + static_assert(sizeof(*out) == 4 || sizeof(*out) == 8, + "SimpleAtoi works only with 32-bit or 64-bit integers."); + static_assert(!std::is_floating_point<int_type>::value, + "Use SimpleAtof or SimpleAtod instead."); + bool parsed; + // TODO(jorg): This signed-ness check is used because it works correctly + // with enums, and it also serves to check that int_type is not a pointer. + // If one day something like std::is_signed<enum E> works, switch to it. + if (static_cast<int_type>(1) - 2 < 0) { // Signed + if (sizeof(*out) == 64 / 8) { // 64-bit + int64_t val; + parsed = numbers_internal::safe_strto64_base(s, &val, base); + *out = static_cast<int_type>(val); + } else { // 32-bit + int32_t val; + parsed = numbers_internal::safe_strto32_base(s, &val, base); + *out = static_cast<int_type>(val); + } + } else { // Unsigned + if (sizeof(*out) == 64 / 8) { // 64-bit + uint64_t val; + parsed = numbers_internal::safe_strtou64_base(s, &val, base); + *out = static_cast<int_type>(val); + } else { // 32-bit + uint32_t val; + parsed = numbers_internal::safe_strtou32_base(s, &val, base); + *out = static_cast<int_type>(val); + } + } + return parsed; +} + +// FastHexToBufferZeroPad16() +// +// Outputs `val` into `out` as if by `snprintf(out, 17, "%016x", val)` but +// without the terminating null character. Thus `out` must be of length >= 16. +// Returns the number of non-pad digits of the output (it can never be zero +// since 0 has one digit). +inline size_t FastHexToBufferZeroPad16(uint64_t val, char* out) { +#ifdef __SSE4_2__ + uint64_t be = absl::big_endian::FromHost64(val); + const auto kNibbleMask = _mm_set1_epi8(0xf); + const auto kHexDigits = _mm_setr_epi8('0', '1', '2', '3', '4', '5', '6', '7', + '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'); + auto v = _mm_loadl_epi64(reinterpret_cast<__m128i*>(&be)); // load lo dword + auto v4 = _mm_srli_epi64(v, 4); // shift 4 right + auto il = _mm_unpacklo_epi8(v4, v); // interleave bytes + auto m = _mm_and_si128(il, kNibbleMask); // mask out nibbles + auto hexchars = _mm_shuffle_epi8(kHexDigits, m); // hex chars + _mm_storeu_si128(reinterpret_cast<__m128i*>(out), hexchars); +#else + for (int i = 0; i < 8; ++i) { + auto byte = (val >> (56 - 8 * i)) & 0xFF; + auto* hex = &absl::numbers_internal::kHexTable[byte * 2]; + std::memcpy(out + 2 * i, hex, 2); + } +#endif + // | 0x1 so that even 0 has 1 digit. + return 16 - absl::base_internal::CountLeadingZeros64(val | 0x1) / 4; +} + +} // namespace numbers_internal + +// SimpleAtoi() +// +// Converts a string to an integer, using `safe_strto?()` functions for actual +// parsing, returning `true` if successful. The `safe_strto?()` functions apply +// strict checking; the string must be a base-10 integer, optionally followed or +// preceded by ASCII whitespace, with a value in the range of the corresponding +// integer type. +template <typename int_type> +ABSL_MUST_USE_RESULT bool SimpleAtoi(absl::string_view str, int_type* out) { + return numbers_internal::safe_strtoi_base(str, out, 10); +} + +ABSL_MUST_USE_RESULT inline bool SimpleAtoi(absl::string_view str, + absl::int128* out) { + return numbers_internal::safe_strto128_base(str, out, 10); +} + +ABSL_MUST_USE_RESULT inline bool SimpleAtoi(absl::string_view str, + absl::uint128* out) { + return numbers_internal::safe_strtou128_base(str, out, 10); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_NUMBERS_H_ diff --git a/third_party/abseil_cpp/absl/strings/numbers_benchmark.cc b/third_party/abseil_cpp/absl/strings/numbers_benchmark.cc new file mode 100644 index 000000000000..6e79b3e811ff --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/numbers_benchmark.cc @@ -0,0 +1,286 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <random> +#include <string> +#include <type_traits> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/distributions.h" +#include "absl/random/random.h" +#include "absl/strings/numbers.h" + +namespace { + +template <typename T> +void BM_FastIntToBuffer(benchmark::State& state) { + const int inc = state.range(0); + char buf[absl::numbers_internal::kFastToBufferSize]; + // Use the unsigned type to increment to take advantage of well-defined + // modular arithmetic. + typename std::make_unsigned<T>::type x = 0; + for (auto _ : state) { + absl::numbers_internal::FastIntToBuffer(static_cast<T>(x), buf); + x += inc; + } +} +BENCHMARK_TEMPLATE(BM_FastIntToBuffer, int32_t)->Range(0, 1 << 15); +BENCHMARK_TEMPLATE(BM_FastIntToBuffer, int64_t)->Range(0, 1 << 30); + +// Creates an integer that would be printed as `num_digits` repeated 7s in the +// given `base`. `base` must be greater than or equal to 8. +int64_t RepeatedSevens(int num_digits, int base) { + ABSL_RAW_CHECK(base >= 8, ""); + int64_t num = 7; + while (--num_digits) num = base * num + 7; + return num; +} + +void BM_safe_strto32_string(benchmark::State& state) { + const int digits = state.range(0); + const int base = state.range(1); + std::string str(digits, '7'); // valid in octal, decimal and hex + int32_t value = 0; + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::numbers_internal::safe_strto32_base(str, &value, base)); + } + ABSL_RAW_CHECK(value == RepeatedSevens(digits, base), ""); +} +BENCHMARK(BM_safe_strto32_string) + ->ArgPair(1, 8) + ->ArgPair(1, 10) + ->ArgPair(1, 16) + ->ArgPair(2, 8) + ->ArgPair(2, 10) + ->ArgPair(2, 16) + ->ArgPair(4, 8) + ->ArgPair(4, 10) + ->ArgPair(4, 16) + ->ArgPair(8, 8) + ->ArgPair(8, 10) + ->ArgPair(8, 16) + ->ArgPair(10, 8) + ->ArgPair(9, 10); + +void BM_safe_strto64_string(benchmark::State& state) { + const int digits = state.range(0); + const int base = state.range(1); + std::string str(digits, '7'); // valid in octal, decimal and hex + int64_t value = 0; + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::numbers_internal::safe_strto64_base(str, &value, base)); + } + ABSL_RAW_CHECK(value == RepeatedSevens(digits, base), ""); +} +BENCHMARK(BM_safe_strto64_string) + ->ArgPair(1, 8) + ->ArgPair(1, 10) + ->ArgPair(1, 16) + ->ArgPair(2, 8) + ->ArgPair(2, 10) + ->ArgPair(2, 16) + ->ArgPair(4, 8) + ->ArgPair(4, 10) + ->ArgPair(4, 16) + ->ArgPair(8, 8) + ->ArgPair(8, 10) + ->ArgPair(8, 16) + ->ArgPair(16, 8) + ->ArgPair(16, 10) + ->ArgPair(16, 16); + +void BM_safe_strtou32_string(benchmark::State& state) { + const int digits = state.range(0); + const int base = state.range(1); + std::string str(digits, '7'); // valid in octal, decimal and hex + uint32_t value = 0; + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::numbers_internal::safe_strtou32_base(str, &value, base)); + } + ABSL_RAW_CHECK(value == RepeatedSevens(digits, base), ""); +} +BENCHMARK(BM_safe_strtou32_string) + ->ArgPair(1, 8) + ->ArgPair(1, 10) + ->ArgPair(1, 16) + ->ArgPair(2, 8) + ->ArgPair(2, 10) + ->ArgPair(2, 16) + ->ArgPair(4, 8) + ->ArgPair(4, 10) + ->ArgPair(4, 16) + ->ArgPair(8, 8) + ->ArgPair(8, 10) + ->ArgPair(8, 16) + ->ArgPair(10, 8) + ->ArgPair(9, 10); + +void BM_safe_strtou64_string(benchmark::State& state) { + const int digits = state.range(0); + const int base = state.range(1); + std::string str(digits, '7'); // valid in octal, decimal and hex + uint64_t value = 0; + for (auto _ : state) { + benchmark::DoNotOptimize( + absl::numbers_internal::safe_strtou64_base(str, &value, base)); + } + ABSL_RAW_CHECK(value == RepeatedSevens(digits, base), ""); +} +BENCHMARK(BM_safe_strtou64_string) + ->ArgPair(1, 8) + ->ArgPair(1, 10) + ->ArgPair(1, 16) + ->ArgPair(2, 8) + ->ArgPair(2, 10) + ->ArgPair(2, 16) + ->ArgPair(4, 8) + ->ArgPair(4, 10) + ->ArgPair(4, 16) + ->ArgPair(8, 8) + ->ArgPair(8, 10) + ->ArgPair(8, 16) + ->ArgPair(16, 8) + ->ArgPair(16, 10) + ->ArgPair(16, 16); + +// Returns a vector of `num_strings` strings. Each string represents a +// floating point number with `num_digits` digits before the decimal point and +// another `num_digits` digits after. +std::vector<std::string> MakeFloatStrings(int num_strings, int num_digits) { + // For convenience, use a random number generator to generate the test data. + // We don't actually need random properties, so use a fixed seed. + std::minstd_rand0 rng(1); + std::uniform_int_distribution<int> random_digit('0', '9'); + + std::vector<std::string> float_strings(num_strings); + for (std::string& s : float_strings) { + s.reserve(2 * num_digits + 1); + for (int i = 0; i < num_digits; ++i) { + s.push_back(static_cast<char>(random_digit(rng))); + } + s.push_back('.'); + for (int i = 0; i < num_digits; ++i) { + s.push_back(static_cast<char>(random_digit(rng))); + } + } + return float_strings; +} + +template <typename StringType> +StringType GetStringAs(const std::string& s) { + return static_cast<StringType>(s); +} +template <> +const char* GetStringAs<const char*>(const std::string& s) { + return s.c_str(); +} + +template <typename StringType> +std::vector<StringType> GetStringsAs(const std::vector<std::string>& strings) { + std::vector<StringType> result; + result.reserve(strings.size()); + for (const std::string& s : strings) { + result.push_back(GetStringAs<StringType>(s)); + } + return result; +} + +template <typename T> +void BM_SimpleAtof(benchmark::State& state) { + const int num_strings = state.range(0); + const int num_digits = state.range(1); + std::vector<std::string> backing_strings = + MakeFloatStrings(num_strings, num_digits); + std::vector<T> inputs = GetStringsAs<T>(backing_strings); + float value; + for (auto _ : state) { + for (const T& input : inputs) { + benchmark::DoNotOptimize(absl::SimpleAtof(input, &value)); + } + } +} +BENCHMARK_TEMPLATE(BM_SimpleAtof, absl::string_view) + ->ArgPair(10, 1) + ->ArgPair(10, 2) + ->ArgPair(10, 4) + ->ArgPair(10, 8); +BENCHMARK_TEMPLATE(BM_SimpleAtof, const char*) + ->ArgPair(10, 1) + ->ArgPair(10, 2) + ->ArgPair(10, 4) + ->ArgPair(10, 8); +BENCHMARK_TEMPLATE(BM_SimpleAtof, std::string) + ->ArgPair(10, 1) + ->ArgPair(10, 2) + ->ArgPair(10, 4) + ->ArgPair(10, 8); + +template <typename T> +void BM_SimpleAtod(benchmark::State& state) { + const int num_strings = state.range(0); + const int num_digits = state.range(1); + std::vector<std::string> backing_strings = + MakeFloatStrings(num_strings, num_digits); + std::vector<T> inputs = GetStringsAs<T>(backing_strings); + double value; + for (auto _ : state) { + for (const T& input : inputs) { + benchmark::DoNotOptimize(absl::SimpleAtod(input, &value)); + } + } +} +BENCHMARK_TEMPLATE(BM_SimpleAtod, absl::string_view) + ->ArgPair(10, 1) + ->ArgPair(10, 2) + ->ArgPair(10, 4) + ->ArgPair(10, 8); +BENCHMARK_TEMPLATE(BM_SimpleAtod, const char*) + ->ArgPair(10, 1) + ->ArgPair(10, 2) + ->ArgPair(10, 4) + ->ArgPair(10, 8); +BENCHMARK_TEMPLATE(BM_SimpleAtod, std::string) + ->ArgPair(10, 1) + ->ArgPair(10, 2) + ->ArgPair(10, 4) + ->ArgPair(10, 8); + +void BM_FastHexToBufferZeroPad16(benchmark::State& state) { + absl::BitGen rng; + std::vector<uint64_t> nums; + nums.resize(1000); + auto min = std::numeric_limits<uint64_t>::min(); + auto max = std::numeric_limits<uint64_t>::max(); + for (auto& num : nums) { + num = absl::LogUniform(rng, min, max); + } + + char buf[16]; + while (state.KeepRunningBatch(nums.size())) { + for (auto num : nums) { + auto digits = absl::numbers_internal::FastHexToBufferZeroPad16(num, buf); + benchmark::DoNotOptimize(digits); + benchmark::DoNotOptimize(buf); + } + } +} +BENCHMARK(BM_FastHexToBufferZeroPad16); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/numbers_test.cc b/third_party/abseil_cpp/absl/strings/numbers_test.cc new file mode 100644 index 000000000000..4ab67fb669be --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/numbers_test.cc @@ -0,0 +1,1356 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file tests string processing functions related to numeric values. + +#include "absl/strings/numbers.h" + +#include <sys/types.h> + +#include <cfenv> // NOLINT(build/c++11) +#include <cinttypes> +#include <climits> +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <limits> +#include <numeric> +#include <random> +#include <set> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/random/distributions.h" +#include "absl/random/random.h" +#include "absl/strings/internal/numbers_test_common.h" +#include "absl/strings/internal/ostringstream.h" +#include "absl/strings/internal/pow10_helper.h" +#include "absl/strings/str_cat.h" + +namespace { + +using absl::numbers_internal::kSixDigitsToBufferSize; +using absl::numbers_internal::safe_strto32_base; +using absl::numbers_internal::safe_strto64_base; +using absl::numbers_internal::safe_strtou32_base; +using absl::numbers_internal::safe_strtou64_base; +using absl::numbers_internal::SixDigitsToBuffer; +using absl::strings_internal::Itoa; +using absl::strings_internal::strtouint32_test_cases; +using absl::strings_internal::strtouint64_test_cases; +using absl::SimpleAtoi; +using testing::Eq; +using testing::MatchesRegex; + +// Number of floats to test with. +// 5,000,000 is a reasonable default for a test that only takes a few seconds. +// 1,000,000,000+ triggers checking for all possible mantissa values for +// double-precision tests. 2,000,000,000+ triggers checking for every possible +// single-precision float. +const int kFloatNumCases = 5000000; + +// This is a slow, brute-force routine to compute the exact base-10 +// representation of a double-precision floating-point number. It +// is useful for debugging only. +std::string PerfectDtoa(double d) { + if (d == 0) return "0"; + if (d < 0) return "-" + PerfectDtoa(-d); + + // Basic theory: decompose d into mantissa and exp, where + // d = mantissa * 2^exp, and exp is as close to zero as possible. + int64_t mantissa, exp = 0; + while (d >= 1ULL << 63) ++exp, d *= 0.5; + while ((mantissa = d) != d) --exp, d *= 2.0; + + // Then convert mantissa to ASCII, and either double it (if + // exp > 0) or halve it (if exp < 0) repeatedly. "halve it" + // in this case means multiplying it by five and dividing by 10. + constexpr int maxlen = 1100; // worst case is actually 1030 or so. + char buf[maxlen + 5]; + for (int64_t num = mantissa, pos = maxlen; --pos >= 0;) { + buf[pos] = '0' + (num % 10); + num /= 10; + } + char* begin = &buf[0]; + char* end = buf + maxlen; + for (int i = 0; i != exp; i += (exp > 0) ? 1 : -1) { + int carry = 0; + for (char* p = end; --p != begin;) { + int dig = *p - '0'; + dig = dig * (exp > 0 ? 2 : 5) + carry; + carry = dig / 10; + dig %= 10; + *p = '0' + dig; + } + } + if (exp < 0) { + // "dividing by 10" above means we have to add the decimal point. + memmove(end + 1 + exp, end + exp, 1 - exp); + end[exp] = '.'; + ++end; + } + while (*begin == '0' && begin[1] != '.') ++begin; + return {begin, end}; +} + +TEST(ToString, PerfectDtoa) { + EXPECT_THAT(PerfectDtoa(1), Eq("1")); + EXPECT_THAT(PerfectDtoa(0.1), + Eq("0.1000000000000000055511151231257827021181583404541015625")); + EXPECT_THAT(PerfectDtoa(1e24), Eq("999999999999999983222784")); + EXPECT_THAT(PerfectDtoa(5e-324), MatchesRegex("0.0000.*625")); + for (int i = 0; i < 100; ++i) { + for (double multiplier : + {1e-300, 1e-200, 1e-100, 0.1, 1.0, 10.0, 1e100, 1e300}) { + double d = multiplier * i; + std::string s = PerfectDtoa(d); + EXPECT_DOUBLE_EQ(d, strtod(s.c_str(), nullptr)); + } + } +} + +template <typename integer> +struct MyInteger { + integer i; + explicit constexpr MyInteger(integer i) : i(i) {} + constexpr operator integer() const { return i; } + + constexpr MyInteger operator+(MyInteger other) const { return i + other.i; } + constexpr MyInteger operator-(MyInteger other) const { return i - other.i; } + constexpr MyInteger operator*(MyInteger other) const { return i * other.i; } + constexpr MyInteger operator/(MyInteger other) const { return i / other.i; } + + constexpr bool operator<(MyInteger other) const { return i < other.i; } + constexpr bool operator<=(MyInteger other) const { return i <= other.i; } + constexpr bool operator==(MyInteger other) const { return i == other.i; } + constexpr bool operator>=(MyInteger other) const { return i >= other.i; } + constexpr bool operator>(MyInteger other) const { return i > other.i; } + constexpr bool operator!=(MyInteger other) const { return i != other.i; } + + integer as_integer() const { return i; } +}; + +typedef MyInteger<int64_t> MyInt64; +typedef MyInteger<uint64_t> MyUInt64; + +void CheckInt32(int32_t x) { + char buffer[absl::numbers_internal::kFastToBufferSize]; + char* actual = absl::numbers_internal::FastIntToBuffer(x, buffer); + std::string expected = std::to_string(x); + EXPECT_EQ(expected, std::string(buffer, actual)) << " Input " << x; + + char* generic_actual = absl::numbers_internal::FastIntToBuffer(x, buffer); + EXPECT_EQ(expected, std::string(buffer, generic_actual)) << " Input " << x; +} + +void CheckInt64(int64_t x) { + char buffer[absl::numbers_internal::kFastToBufferSize + 3]; + buffer[0] = '*'; + buffer[23] = '*'; + buffer[24] = '*'; + char* actual = absl::numbers_internal::FastIntToBuffer(x, &buffer[1]); + std::string expected = std::to_string(x); + EXPECT_EQ(expected, std::string(&buffer[1], actual)) << " Input " << x; + EXPECT_EQ(buffer[0], '*'); + EXPECT_EQ(buffer[23], '*'); + EXPECT_EQ(buffer[24], '*'); + + char* my_actual = + absl::numbers_internal::FastIntToBuffer(MyInt64(x), &buffer[1]); + EXPECT_EQ(expected, std::string(&buffer[1], my_actual)) << " Input " << x; +} + +void CheckUInt32(uint32_t x) { + char buffer[absl::numbers_internal::kFastToBufferSize]; + char* actual = absl::numbers_internal::FastIntToBuffer(x, buffer); + std::string expected = std::to_string(x); + EXPECT_EQ(expected, std::string(buffer, actual)) << " Input " << x; + + char* generic_actual = absl::numbers_internal::FastIntToBuffer(x, buffer); + EXPECT_EQ(expected, std::string(buffer, generic_actual)) << " Input " << x; +} + +void CheckUInt64(uint64_t x) { + char buffer[absl::numbers_internal::kFastToBufferSize + 1]; + char* actual = absl::numbers_internal::FastIntToBuffer(x, &buffer[1]); + std::string expected = std::to_string(x); + EXPECT_EQ(expected, std::string(&buffer[1], actual)) << " Input " << x; + + char* generic_actual = absl::numbers_internal::FastIntToBuffer(x, &buffer[1]); + EXPECT_EQ(expected, std::string(&buffer[1], generic_actual)) + << " Input " << x; + + char* my_actual = + absl::numbers_internal::FastIntToBuffer(MyUInt64(x), &buffer[1]); + EXPECT_EQ(expected, std::string(&buffer[1], my_actual)) << " Input " << x; +} + +void CheckHex64(uint64_t v) { + char expected[16 + 1]; + std::string actual = absl::StrCat(absl::Hex(v, absl::kZeroPad16)); + snprintf(expected, sizeof(expected), "%016" PRIx64, static_cast<uint64_t>(v)); + EXPECT_EQ(expected, actual) << " Input " << v; + actual = absl::StrCat(absl::Hex(v, absl::kSpacePad16)); + snprintf(expected, sizeof(expected), "%16" PRIx64, static_cast<uint64_t>(v)); + EXPECT_EQ(expected, actual) << " Input " << v; +} + +TEST(Numbers, TestFastPrints) { + for (int i = -100; i <= 100; i++) { + CheckInt32(i); + CheckInt64(i); + } + for (int i = 0; i <= 100; i++) { + CheckUInt32(i); + CheckUInt64(i); + } + // Test min int to make sure that works + CheckInt32(INT_MIN); + CheckInt32(INT_MAX); + CheckInt64(LONG_MIN); + CheckInt64(uint64_t{1000000000}); + CheckInt64(uint64_t{9999999999}); + CheckInt64(uint64_t{100000000000000}); + CheckInt64(uint64_t{999999999999999}); + CheckInt64(uint64_t{1000000000000000000}); + CheckInt64(uint64_t{1199999999999999999}); + CheckInt64(int64_t{-700000000000000000}); + CheckInt64(LONG_MAX); + CheckUInt32(std::numeric_limits<uint32_t>::max()); + CheckUInt64(uint64_t{1000000000}); + CheckUInt64(uint64_t{9999999999}); + CheckUInt64(uint64_t{100000000000000}); + CheckUInt64(uint64_t{999999999999999}); + CheckUInt64(uint64_t{1000000000000000000}); + CheckUInt64(uint64_t{1199999999999999999}); + CheckUInt64(std::numeric_limits<uint64_t>::max()); + + for (int i = 0; i < 10000; i++) { + CheckHex64(i); + } + CheckHex64(uint64_t{0x123456789abcdef0}); +} + +template <typename int_type, typename in_val_type> +void VerifySimpleAtoiGood(in_val_type in_value, int_type exp_value) { + std::string s; + // (u)int128 can be streamed but not StrCat'd. + absl::strings_internal::OStringStream(&s) << in_value; + int_type x = static_cast<int_type>(~exp_value); + EXPECT_TRUE(SimpleAtoi(s, &x)) + << "in_value=" << in_value << " s=" << s << " x=" << x; + EXPECT_EQ(exp_value, x); + x = static_cast<int_type>(~exp_value); + EXPECT_TRUE(SimpleAtoi(s.c_str(), &x)); + EXPECT_EQ(exp_value, x); +} + +template <typename int_type, typename in_val_type> +void VerifySimpleAtoiBad(in_val_type in_value) { + std::string s; + // (u)int128 can be streamed but not StrCat'd. + absl::strings_internal::OStringStream(&s) << in_value; + int_type x; + EXPECT_FALSE(SimpleAtoi(s, &x)); + EXPECT_FALSE(SimpleAtoi(s.c_str(), &x)); +} + +TEST(NumbersTest, Atoi) { + // SimpleAtoi(absl::string_view, int32_t) + VerifySimpleAtoiGood<int32_t>(0, 0); + VerifySimpleAtoiGood<int32_t>(42, 42); + VerifySimpleAtoiGood<int32_t>(-42, -42); + + VerifySimpleAtoiGood<int32_t>(std::numeric_limits<int32_t>::min(), + std::numeric_limits<int32_t>::min()); + VerifySimpleAtoiGood<int32_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + + // SimpleAtoi(absl::string_view, uint32_t) + VerifySimpleAtoiGood<uint32_t>(0, 0); + VerifySimpleAtoiGood<uint32_t>(42, 42); + VerifySimpleAtoiBad<uint32_t>(-42); + + VerifySimpleAtoiBad<uint32_t>(std::numeric_limits<int32_t>::min()); + VerifySimpleAtoiGood<uint32_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleAtoiGood<uint32_t>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleAtoiBad<uint32_t>(std::numeric_limits<int64_t>::min()); + VerifySimpleAtoiBad<uint32_t>(std::numeric_limits<int64_t>::max()); + VerifySimpleAtoiBad<uint32_t>(std::numeric_limits<uint64_t>::max()); + + // SimpleAtoi(absl::string_view, int64_t) + VerifySimpleAtoiGood<int64_t>(0, 0); + VerifySimpleAtoiGood<int64_t>(42, 42); + VerifySimpleAtoiGood<int64_t>(-42, -42); + + VerifySimpleAtoiGood<int64_t>(std::numeric_limits<int32_t>::min(), + std::numeric_limits<int32_t>::min()); + VerifySimpleAtoiGood<int64_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleAtoiGood<int64_t>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleAtoiGood<int64_t>(std::numeric_limits<int64_t>::min(), + std::numeric_limits<int64_t>::min()); + VerifySimpleAtoiGood<int64_t>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleAtoiBad<int64_t>(std::numeric_limits<uint64_t>::max()); + + // SimpleAtoi(absl::string_view, uint64_t) + VerifySimpleAtoiGood<uint64_t>(0, 0); + VerifySimpleAtoiGood<uint64_t>(42, 42); + VerifySimpleAtoiBad<uint64_t>(-42); + + VerifySimpleAtoiBad<uint64_t>(std::numeric_limits<int32_t>::min()); + VerifySimpleAtoiGood<uint64_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleAtoiGood<uint64_t>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleAtoiBad<uint64_t>(std::numeric_limits<int64_t>::min()); + VerifySimpleAtoiGood<uint64_t>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleAtoiGood<uint64_t>(std::numeric_limits<uint64_t>::max(), + std::numeric_limits<uint64_t>::max()); + + // SimpleAtoi(absl::string_view, absl::uint128) + VerifySimpleAtoiGood<absl::uint128>(0, 0); + VerifySimpleAtoiGood<absl::uint128>(42, 42); + VerifySimpleAtoiBad<absl::uint128>(-42); + + VerifySimpleAtoiBad<absl::uint128>(std::numeric_limits<int32_t>::min()); + VerifySimpleAtoiGood<absl::uint128>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleAtoiGood<absl::uint128>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleAtoiBad<absl::uint128>(std::numeric_limits<int64_t>::min()); + VerifySimpleAtoiGood<absl::uint128>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleAtoiGood<absl::uint128>(std::numeric_limits<uint64_t>::max(), + std::numeric_limits<uint64_t>::max()); + VerifySimpleAtoiGood<absl::uint128>( + std::numeric_limits<absl::uint128>::max(), + std::numeric_limits<absl::uint128>::max()); + + // SimpleAtoi(absl::string_view, absl::int128) + VerifySimpleAtoiGood<absl::int128>(0, 0); + VerifySimpleAtoiGood<absl::int128>(42, 42); + VerifySimpleAtoiGood<absl::int128>(-42, -42); + + VerifySimpleAtoiGood<absl::int128>(std::numeric_limits<int32_t>::min(), + std::numeric_limits<int32_t>::min()); + VerifySimpleAtoiGood<absl::int128>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleAtoiGood<absl::int128>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleAtoiGood<absl::int128>(std::numeric_limits<int64_t>::min(), + std::numeric_limits<int64_t>::min()); + VerifySimpleAtoiGood<absl::int128>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleAtoiGood<absl::int128>(std::numeric_limits<uint64_t>::max(), + std::numeric_limits<uint64_t>::max()); + VerifySimpleAtoiGood<absl::int128>( + std::numeric_limits<absl::int128>::min(), + std::numeric_limits<absl::int128>::min()); + VerifySimpleAtoiGood<absl::int128>( + std::numeric_limits<absl::int128>::max(), + std::numeric_limits<absl::int128>::max()); + VerifySimpleAtoiBad<absl::int128>(std::numeric_limits<absl::uint128>::max()); + + // Some other types + VerifySimpleAtoiGood<int>(-42, -42); + VerifySimpleAtoiGood<int32_t>(-42, -42); + VerifySimpleAtoiGood<uint32_t>(42, 42); + VerifySimpleAtoiGood<unsigned int>(42, 42); + VerifySimpleAtoiGood<int64_t>(-42, -42); + VerifySimpleAtoiGood<long>(-42, -42); // NOLINT(runtime/int) + VerifySimpleAtoiGood<uint64_t>(42, 42); + VerifySimpleAtoiGood<size_t>(42, 42); + VerifySimpleAtoiGood<std::string::size_type>(42, 42); +} + +TEST(NumbersTest, Atod) { + double d; + EXPECT_TRUE(absl::SimpleAtod("nan", &d)); + EXPECT_TRUE(std::isnan(d)); +} + +TEST(NumbersTest, Atoenum) { + enum E01 { + E01_zero = 0, + E01_one = 1, + }; + + VerifySimpleAtoiGood<E01>(E01_zero, E01_zero); + VerifySimpleAtoiGood<E01>(E01_one, E01_one); + + enum E_101 { + E_101_minusone = -1, + E_101_zero = 0, + E_101_one = 1, + }; + + VerifySimpleAtoiGood<E_101>(E_101_minusone, E_101_minusone); + VerifySimpleAtoiGood<E_101>(E_101_zero, E_101_zero); + VerifySimpleAtoiGood<E_101>(E_101_one, E_101_one); + + enum E_bigint { + E_bigint_zero = 0, + E_bigint_one = 1, + E_bigint_max31 = static_cast<int32_t>(0x7FFFFFFF), + }; + + VerifySimpleAtoiGood<E_bigint>(E_bigint_zero, E_bigint_zero); + VerifySimpleAtoiGood<E_bigint>(E_bigint_one, E_bigint_one); + VerifySimpleAtoiGood<E_bigint>(E_bigint_max31, E_bigint_max31); + + enum E_fullint { + E_fullint_zero = 0, + E_fullint_one = 1, + E_fullint_max31 = static_cast<int32_t>(0x7FFFFFFF), + E_fullint_min32 = INT32_MIN, + }; + + VerifySimpleAtoiGood<E_fullint>(E_fullint_zero, E_fullint_zero); + VerifySimpleAtoiGood<E_fullint>(E_fullint_one, E_fullint_one); + VerifySimpleAtoiGood<E_fullint>(E_fullint_max31, E_fullint_max31); + VerifySimpleAtoiGood<E_fullint>(E_fullint_min32, E_fullint_min32); + + enum E_biguint { + E_biguint_zero = 0, + E_biguint_one = 1, + E_biguint_max31 = static_cast<uint32_t>(0x7FFFFFFF), + E_biguint_max32 = static_cast<uint32_t>(0xFFFFFFFF), + }; + + VerifySimpleAtoiGood<E_biguint>(E_biguint_zero, E_biguint_zero); + VerifySimpleAtoiGood<E_biguint>(E_biguint_one, E_biguint_one); + VerifySimpleAtoiGood<E_biguint>(E_biguint_max31, E_biguint_max31); + VerifySimpleAtoiGood<E_biguint>(E_biguint_max32, E_biguint_max32); +} + +TEST(stringtest, safe_strto32_base) { + int32_t value; + EXPECT_TRUE(safe_strto32_base("0x34234324", &value, 16)); + EXPECT_EQ(0x34234324, value); + + EXPECT_TRUE(safe_strto32_base("0X34234324", &value, 16)); + EXPECT_EQ(0x34234324, value); + + EXPECT_TRUE(safe_strto32_base("34234324", &value, 16)); + EXPECT_EQ(0x34234324, value); + + EXPECT_TRUE(safe_strto32_base("0", &value, 16)); + EXPECT_EQ(0, value); + + EXPECT_TRUE(safe_strto32_base(" \t\n -0x34234324", &value, 16)); + EXPECT_EQ(-0x34234324, value); + + EXPECT_TRUE(safe_strto32_base(" \t\n -34234324", &value, 16)); + EXPECT_EQ(-0x34234324, value); + + EXPECT_TRUE(safe_strto32_base("7654321", &value, 8)); + EXPECT_EQ(07654321, value); + + EXPECT_TRUE(safe_strto32_base("-01234", &value, 8)); + EXPECT_EQ(-01234, value); + + EXPECT_FALSE(safe_strto32_base("1834", &value, 8)); + + // Autodetect base. + EXPECT_TRUE(safe_strto32_base("0", &value, 0)); + EXPECT_EQ(0, value); + + EXPECT_TRUE(safe_strto32_base("077", &value, 0)); + EXPECT_EQ(077, value); // Octal interpretation + + // Leading zero indicates octal, but then followed by invalid digit. + EXPECT_FALSE(safe_strto32_base("088", &value, 0)); + + // Leading 0x indicated hex, but then followed by invalid digit. + EXPECT_FALSE(safe_strto32_base("0xG", &value, 0)); + + // Base-10 version. + EXPECT_TRUE(safe_strto32_base("34234324", &value, 10)); + EXPECT_EQ(34234324, value); + + EXPECT_TRUE(safe_strto32_base("0", &value, 10)); + EXPECT_EQ(0, value); + + EXPECT_TRUE(safe_strto32_base(" \t\n -34234324", &value, 10)); + EXPECT_EQ(-34234324, value); + + EXPECT_TRUE(safe_strto32_base("34234324 \n\t ", &value, 10)); + EXPECT_EQ(34234324, value); + + // Invalid ints. + EXPECT_FALSE(safe_strto32_base("", &value, 10)); + EXPECT_FALSE(safe_strto32_base(" ", &value, 10)); + EXPECT_FALSE(safe_strto32_base("abc", &value, 10)); + EXPECT_FALSE(safe_strto32_base("34234324a", &value, 10)); + EXPECT_FALSE(safe_strto32_base("34234.3", &value, 10)); + + // Out of bounds. + EXPECT_FALSE(safe_strto32_base("2147483648", &value, 10)); + EXPECT_FALSE(safe_strto32_base("-2147483649", &value, 10)); + + // String version. + EXPECT_TRUE(safe_strto32_base(std::string("0x1234"), &value, 16)); + EXPECT_EQ(0x1234, value); + + // Base-10 string version. + EXPECT_TRUE(safe_strto32_base("1234", &value, 10)); + EXPECT_EQ(1234, value); +} + +TEST(stringtest, safe_strto32_range) { + // These tests verify underflow/overflow behaviour. + int32_t value; + EXPECT_FALSE(safe_strto32_base("2147483648", &value, 10)); + EXPECT_EQ(std::numeric_limits<int32_t>::max(), value); + + EXPECT_TRUE(safe_strto32_base("-2147483648", &value, 10)); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), value); + + EXPECT_FALSE(safe_strto32_base("-2147483649", &value, 10)); + EXPECT_EQ(std::numeric_limits<int32_t>::min(), value); +} + +TEST(stringtest, safe_strto64_range) { + // These tests verify underflow/overflow behaviour. + int64_t value; + EXPECT_FALSE(safe_strto64_base("9223372036854775808", &value, 10)); + EXPECT_EQ(std::numeric_limits<int64_t>::max(), value); + + EXPECT_TRUE(safe_strto64_base("-9223372036854775808", &value, 10)); + EXPECT_EQ(std::numeric_limits<int64_t>::min(), value); + + EXPECT_FALSE(safe_strto64_base("-9223372036854775809", &value, 10)); + EXPECT_EQ(std::numeric_limits<int64_t>::min(), value); +} + +TEST(stringtest, safe_strto32_leading_substring) { + // These tests verify this comment in numbers.h: + // On error, returns false, and sets *value to: [...] + // conversion of leading substring if available ("123@@@" -> 123) + // 0 if no leading substring available + int32_t value; + EXPECT_FALSE(safe_strto32_base("04069@@@", &value, 10)); + EXPECT_EQ(4069, value); + + EXPECT_FALSE(safe_strto32_base("04069@@@", &value, 8)); + EXPECT_EQ(0406, value); + + EXPECT_FALSE(safe_strto32_base("04069balloons", &value, 10)); + EXPECT_EQ(4069, value); + + EXPECT_FALSE(safe_strto32_base("04069balloons", &value, 16)); + EXPECT_EQ(0x4069ba, value); + + EXPECT_FALSE(safe_strto32_base("@@@", &value, 10)); + EXPECT_EQ(0, value); // there was no leading substring +} + +TEST(stringtest, safe_strto64_leading_substring) { + // These tests verify this comment in numbers.h: + // On error, returns false, and sets *value to: [...] + // conversion of leading substring if available ("123@@@" -> 123) + // 0 if no leading substring available + int64_t value; + EXPECT_FALSE(safe_strto64_base("04069@@@", &value, 10)); + EXPECT_EQ(4069, value); + + EXPECT_FALSE(safe_strto64_base("04069@@@", &value, 8)); + EXPECT_EQ(0406, value); + + EXPECT_FALSE(safe_strto64_base("04069balloons", &value, 10)); + EXPECT_EQ(4069, value); + + EXPECT_FALSE(safe_strto64_base("04069balloons", &value, 16)); + EXPECT_EQ(0x4069ba, value); + + EXPECT_FALSE(safe_strto64_base("@@@", &value, 10)); + EXPECT_EQ(0, value); // there was no leading substring +} + +TEST(stringtest, safe_strto64_base) { + int64_t value; + EXPECT_TRUE(safe_strto64_base("0x3423432448783446", &value, 16)); + EXPECT_EQ(int64_t{0x3423432448783446}, value); + + EXPECT_TRUE(safe_strto64_base("3423432448783446", &value, 16)); + EXPECT_EQ(int64_t{0x3423432448783446}, value); + + EXPECT_TRUE(safe_strto64_base("0", &value, 16)); + EXPECT_EQ(0, value); + + EXPECT_TRUE(safe_strto64_base(" \t\n -0x3423432448783446", &value, 16)); + EXPECT_EQ(int64_t{-0x3423432448783446}, value); + + EXPECT_TRUE(safe_strto64_base(" \t\n -3423432448783446", &value, 16)); + EXPECT_EQ(int64_t{-0x3423432448783446}, value); + + EXPECT_TRUE(safe_strto64_base("123456701234567012", &value, 8)); + EXPECT_EQ(int64_t{0123456701234567012}, value); + + EXPECT_TRUE(safe_strto64_base("-017777777777777", &value, 8)); + EXPECT_EQ(int64_t{-017777777777777}, value); + + EXPECT_FALSE(safe_strto64_base("19777777777777", &value, 8)); + + // Autodetect base. + EXPECT_TRUE(safe_strto64_base("0", &value, 0)); + EXPECT_EQ(0, value); + + EXPECT_TRUE(safe_strto64_base("077", &value, 0)); + EXPECT_EQ(077, value); // Octal interpretation + + // Leading zero indicates octal, but then followed by invalid digit. + EXPECT_FALSE(safe_strto64_base("088", &value, 0)); + + // Leading 0x indicated hex, but then followed by invalid digit. + EXPECT_FALSE(safe_strto64_base("0xG", &value, 0)); + + // Base-10 version. + EXPECT_TRUE(safe_strto64_base("34234324487834466", &value, 10)); + EXPECT_EQ(int64_t{34234324487834466}, value); + + EXPECT_TRUE(safe_strto64_base("0", &value, 10)); + EXPECT_EQ(0, value); + + EXPECT_TRUE(safe_strto64_base(" \t\n -34234324487834466", &value, 10)); + EXPECT_EQ(int64_t{-34234324487834466}, value); + + EXPECT_TRUE(safe_strto64_base("34234324487834466 \n\t ", &value, 10)); + EXPECT_EQ(int64_t{34234324487834466}, value); + + // Invalid ints. + EXPECT_FALSE(safe_strto64_base("", &value, 10)); + EXPECT_FALSE(safe_strto64_base(" ", &value, 10)); + EXPECT_FALSE(safe_strto64_base("abc", &value, 10)); + EXPECT_FALSE(safe_strto64_base("34234324487834466a", &value, 10)); + EXPECT_FALSE(safe_strto64_base("34234487834466.3", &value, 10)); + + // Out of bounds. + EXPECT_FALSE(safe_strto64_base("9223372036854775808", &value, 10)); + EXPECT_FALSE(safe_strto64_base("-9223372036854775809", &value, 10)); + + // String version. + EXPECT_TRUE(safe_strto64_base(std::string("0x1234"), &value, 16)); + EXPECT_EQ(0x1234, value); + + // Base-10 string version. + EXPECT_TRUE(safe_strto64_base("1234", &value, 10)); + EXPECT_EQ(1234, value); +} + +const size_t kNumRandomTests = 10000; + +template <typename IntType> +void test_random_integer_parse_base(bool (*parse_func)(absl::string_view, + IntType* value, + int base)) { + using RandomEngine = std::minstd_rand0; + std::random_device rd; + RandomEngine rng(rd()); + std::uniform_int_distribution<IntType> random_int( + std::numeric_limits<IntType>::min()); + std::uniform_int_distribution<int> random_base(2, 35); + for (size_t i = 0; i < kNumRandomTests; i++) { + IntType value = random_int(rng); + int base = random_base(rng); + std::string str_value; + EXPECT_TRUE(Itoa<IntType>(value, base, &str_value)); + IntType parsed_value; + + // Test successful parse + EXPECT_TRUE(parse_func(str_value, &parsed_value, base)); + EXPECT_EQ(parsed_value, value); + + // Test overflow + EXPECT_FALSE( + parse_func(absl::StrCat(std::numeric_limits<IntType>::max(), value), + &parsed_value, base)); + + // Test underflow + if (std::numeric_limits<IntType>::min() < 0) { + EXPECT_FALSE( + parse_func(absl::StrCat(std::numeric_limits<IntType>::min(), value), + &parsed_value, base)); + } else { + EXPECT_FALSE(parse_func(absl::StrCat("-", value), &parsed_value, base)); + } + } +} + +TEST(stringtest, safe_strto32_random) { + test_random_integer_parse_base<int32_t>(&safe_strto32_base); +} +TEST(stringtest, safe_strto64_random) { + test_random_integer_parse_base<int64_t>(&safe_strto64_base); +} +TEST(stringtest, safe_strtou32_random) { + test_random_integer_parse_base<uint32_t>(&safe_strtou32_base); +} +TEST(stringtest, safe_strtou64_random) { + test_random_integer_parse_base<uint64_t>(&safe_strtou64_base); +} +TEST(stringtest, safe_strtou128_random) { + // random number generators don't work for uint128, and + // uint128 can be streamed but not StrCat'd, so this code must be custom + // implemented for uint128, but is generally the same as what's above. + // test_random_integer_parse_base<absl::uint128>( + // &absl::numbers_internal::safe_strtou128_base); + using RandomEngine = std::minstd_rand0; + using IntType = absl::uint128; + constexpr auto parse_func = &absl::numbers_internal::safe_strtou128_base; + + std::random_device rd; + RandomEngine rng(rd()); + std::uniform_int_distribution<uint64_t> random_uint64( + std::numeric_limits<uint64_t>::min()); + std::uniform_int_distribution<int> random_base(2, 35); + + for (size_t i = 0; i < kNumRandomTests; i++) { + IntType value = random_uint64(rng); + value = (value << 64) + random_uint64(rng); + int base = random_base(rng); + std::string str_value; + EXPECT_TRUE(Itoa<IntType>(value, base, &str_value)); + IntType parsed_value; + + // Test successful parse + EXPECT_TRUE(parse_func(str_value, &parsed_value, base)); + EXPECT_EQ(parsed_value, value); + + // Test overflow + std::string s; + absl::strings_internal::OStringStream(&s) + << std::numeric_limits<IntType>::max() << value; + EXPECT_FALSE(parse_func(s, &parsed_value, base)); + + // Test underflow + s.clear(); + absl::strings_internal::OStringStream(&s) << "-" << value; + EXPECT_FALSE(parse_func(s, &parsed_value, base)); + } +} +TEST(stringtest, safe_strto128_random) { + // random number generators don't work for int128, and + // int128 can be streamed but not StrCat'd, so this code must be custom + // implemented for int128, but is generally the same as what's above. + // test_random_integer_parse_base<absl::int128>( + // &absl::numbers_internal::safe_strto128_base); + using RandomEngine = std::minstd_rand0; + using IntType = absl::int128; + constexpr auto parse_func = &absl::numbers_internal::safe_strto128_base; + + std::random_device rd; + RandomEngine rng(rd()); + std::uniform_int_distribution<int64_t> random_int64( + std::numeric_limits<int64_t>::min()); + std::uniform_int_distribution<uint64_t> random_uint64( + std::numeric_limits<uint64_t>::min()); + std::uniform_int_distribution<int> random_base(2, 35); + + for (size_t i = 0; i < kNumRandomTests; ++i) { + int64_t high = random_int64(rng); + uint64_t low = random_uint64(rng); + IntType value = absl::MakeInt128(high, low); + + int base = random_base(rng); + std::string str_value; + EXPECT_TRUE(Itoa<IntType>(value, base, &str_value)); + IntType parsed_value; + + // Test successful parse + EXPECT_TRUE(parse_func(str_value, &parsed_value, base)); + EXPECT_EQ(parsed_value, value); + + // Test overflow + std::string s; + absl::strings_internal::OStringStream(&s) + << std::numeric_limits<IntType>::max() << value; + EXPECT_FALSE(parse_func(s, &parsed_value, base)); + + // Test underflow + s.clear(); + absl::strings_internal::OStringStream(&s) + << std::numeric_limits<IntType>::min() << value; + EXPECT_FALSE(parse_func(s, &parsed_value, base)); + } +} + +TEST(stringtest, safe_strtou32_base) { + for (int i = 0; strtouint32_test_cases()[i].str != nullptr; ++i) { + const auto& e = strtouint32_test_cases()[i]; + uint32_t value; + EXPECT_EQ(e.expect_ok, safe_strtou32_base(e.str, &value, e.base)) + << "str=\"" << e.str << "\" base=" << e.base; + if (e.expect_ok) { + EXPECT_EQ(e.expected, value) << "i=" << i << " str=\"" << e.str + << "\" base=" << e.base; + } + } +} + +TEST(stringtest, safe_strtou32_base_length_delimited) { + for (int i = 0; strtouint32_test_cases()[i].str != nullptr; ++i) { + const auto& e = strtouint32_test_cases()[i]; + std::string tmp(e.str); + tmp.append("12"); // Adds garbage at the end. + + uint32_t value; + EXPECT_EQ(e.expect_ok, + safe_strtou32_base(absl::string_view(tmp.data(), strlen(e.str)), + &value, e.base)) + << "str=\"" << e.str << "\" base=" << e.base; + if (e.expect_ok) { + EXPECT_EQ(e.expected, value) << "i=" << i << " str=" << e.str + << " base=" << e.base; + } + } +} + +TEST(stringtest, safe_strtou64_base) { + for (int i = 0; strtouint64_test_cases()[i].str != nullptr; ++i) { + const auto& e = strtouint64_test_cases()[i]; + uint64_t value; + EXPECT_EQ(e.expect_ok, safe_strtou64_base(e.str, &value, e.base)) + << "str=\"" << e.str << "\" base=" << e.base; + if (e.expect_ok) { + EXPECT_EQ(e.expected, value) << "str=" << e.str << " base=" << e.base; + } + } +} + +TEST(stringtest, safe_strtou64_base_length_delimited) { + for (int i = 0; strtouint64_test_cases()[i].str != nullptr; ++i) { + const auto& e = strtouint64_test_cases()[i]; + std::string tmp(e.str); + tmp.append("12"); // Adds garbage at the end. + + uint64_t value; + EXPECT_EQ(e.expect_ok, + safe_strtou64_base(absl::string_view(tmp.data(), strlen(e.str)), + &value, e.base)) + << "str=\"" << e.str << "\" base=" << e.base; + if (e.expect_ok) { + EXPECT_EQ(e.expected, value) << "str=\"" << e.str << "\" base=" << e.base; + } + } +} + +// feenableexcept() and fedisableexcept() are extensions supported by some libc +// implementations. +#if defined(__GLIBC__) || defined(__BIONIC__) +#define ABSL_HAVE_FEENABLEEXCEPT 1 +#define ABSL_HAVE_FEDISABLEEXCEPT 1 +#endif + +class SimpleDtoaTest : public testing::Test { + protected: + void SetUp() override { + // Store the current floating point env & clear away any pending exceptions. + feholdexcept(&fp_env_); +#ifdef ABSL_HAVE_FEENABLEEXCEPT + // Turn on floating point exceptions. + feenableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW); +#endif + } + + void TearDown() override { + // Restore the floating point environment to the original state. + // In theory fedisableexcept is unnecessary; fesetenv will also do it. + // In practice, our toolchains have subtle bugs. +#ifdef ABSL_HAVE_FEDISABLEEXCEPT + fedisableexcept(FE_DIVBYZERO | FE_INVALID | FE_OVERFLOW); +#endif + fesetenv(&fp_env_); + } + + std::string ToNineDigits(double value) { + char buffer[16]; // more than enough for %.9g + snprintf(buffer, sizeof(buffer), "%.9g", value); + return buffer; + } + + fenv_t fp_env_; +}; + +// Run the given runnable functor for "cases" test cases, chosen over the +// available range of float. pi and e and 1/e are seeded, and then all +// available integer powers of 2 and 10 are multiplied against them. In +// addition to trying all those values, we try the next higher and next lower +// float, and then we add additional test cases evenly distributed between them. +// Each test case is passed to runnable as both a positive and negative value. +template <typename R> +void ExhaustiveFloat(uint32_t cases, R&& runnable) { + runnable(0.0f); + runnable(-0.0f); + if (cases >= 2e9) { // more than 2 billion? Might as well run them all. + for (float f = 0; f < std::numeric_limits<float>::max(); ) { + f = nextafterf(f, std::numeric_limits<float>::max()); + runnable(-f); + runnable(f); + } + return; + } + std::set<float> floats = {3.4028234e38f}; + for (float f : {1.0, 3.14159265, 2.718281828, 1 / 2.718281828}) { + for (float testf = f; testf != 0; testf *= 0.1f) floats.insert(testf); + for (float testf = f; testf != 0; testf *= 0.5f) floats.insert(testf); + for (float testf = f; testf < 3e38f / 2; testf *= 2.0f) + floats.insert(testf); + for (float testf = f; testf < 3e38f / 10; testf *= 10) floats.insert(testf); + } + + float last = *floats.begin(); + + runnable(last); + runnable(-last); + int iters_per_float = cases / floats.size(); + if (iters_per_float == 0) iters_per_float = 1; + for (float f : floats) { + if (f == last) continue; + float testf = std::nextafter(last, std::numeric_limits<float>::max()); + runnable(testf); + runnable(-testf); + last = testf; + if (f == last) continue; + double step = (double{f} - last) / iters_per_float; + for (double d = last + step; d < f; d += step) { + testf = d; + if (testf != last) { + runnable(testf); + runnable(-testf); + last = testf; + } + } + testf = std::nextafter(f, 0.0f); + if (testf > last) { + runnable(testf); + runnable(-testf); + last = testf; + } + if (f != last) { + runnable(f); + runnable(-f); + last = f; + } + } +} + +TEST_F(SimpleDtoaTest, ExhaustiveDoubleToSixDigits) { + uint64_t test_count = 0; + std::vector<double> mismatches; + auto checker = [&](double d) { + if (d != d) return; // rule out NaNs + ++test_count; + char sixdigitsbuf[kSixDigitsToBufferSize] = {0}; + SixDigitsToBuffer(d, sixdigitsbuf); + char snprintfbuf[kSixDigitsToBufferSize] = {0}; + snprintf(snprintfbuf, kSixDigitsToBufferSize, "%g", d); + if (strcmp(sixdigitsbuf, snprintfbuf) != 0) { + mismatches.push_back(d); + if (mismatches.size() < 10) { + ABSL_RAW_LOG(ERROR, "%s", + absl::StrCat("Six-digit failure with double. ", "d=", d, + "=", d, " sixdigits=", sixdigitsbuf, + " printf(%g)=", snprintfbuf) + .c_str()); + } + } + }; + // Some quick sanity checks... + checker(5e-324); + checker(1e-308); + checker(1.0); + checker(1.000005); + checker(1.7976931348623157e308); + checker(0.00390625); +#ifndef _MSC_VER + // on MSVC, snprintf() rounds it to 0.00195313. SixDigitsToBuffer() rounds it + // to 0.00195312 (round half to even). + checker(0.001953125); +#endif + checker(0.005859375); + // Some cases where the rounding is very very close + checker(1.089095e-15); + checker(3.274195e-55); + checker(6.534355e-146); + checker(2.920845e+234); + + if (mismatches.empty()) { + test_count = 0; + ExhaustiveFloat(kFloatNumCases, checker); + + test_count = 0; + std::vector<int> digit_testcases{ + 100000, 100001, 100002, 100005, 100010, 100020, 100050, 100100, // misc + 195312, 195313, // 1.953125 is a case where we round down, just barely. + 200000, 500000, 800000, // misc mid-range cases + 585937, 585938, // 5.859375 is a case where we round up, just barely. + 900000, 990000, 999000, 999900, 999990, 999996, 999997, 999998, 999999}; + if (kFloatNumCases >= 1e9) { + // If at least 1 billion test cases were requested, user wants an + // exhaustive test. So let's test all mantissas, too. + constexpr int min_mantissa = 100000, max_mantissa = 999999; + digit_testcases.resize(max_mantissa - min_mantissa + 1); + std::iota(digit_testcases.begin(), digit_testcases.end(), min_mantissa); + } + + for (int exponent = -324; exponent <= 308; ++exponent) { + double powten = absl::strings_internal::Pow10(exponent); + if (powten == 0) powten = 5e-324; + if (kFloatNumCases >= 1e9) { + // The exhaustive test takes a very long time, so log progress. + char buf[kSixDigitsToBufferSize]; + ABSL_RAW_LOG( + INFO, "%s", + absl::StrCat("Exp ", exponent, " powten=", powten, "(", powten, + ") (", + std::string(buf, SixDigitsToBuffer(powten, buf)), ")") + .c_str()); + } + for (int digits : digit_testcases) { + if (exponent == 308 && digits >= 179769) break; // don't overflow! + double digiform = (digits + 0.5) * 0.00001; + double testval = digiform * powten; + double pretestval = nextafter(testval, 0); + double posttestval = nextafter(testval, 1.7976931348623157e308); + checker(testval); + checker(pretestval); + checker(posttestval); + } + } + } else { + EXPECT_EQ(mismatches.size(), 0); + for (size_t i = 0; i < mismatches.size(); ++i) { + if (i > 100) i = mismatches.size() - 1; + double d = mismatches[i]; + char sixdigitsbuf[kSixDigitsToBufferSize] = {0}; + SixDigitsToBuffer(d, sixdigitsbuf); + char snprintfbuf[kSixDigitsToBufferSize] = {0}; + snprintf(snprintfbuf, kSixDigitsToBufferSize, "%g", d); + double before = nextafter(d, 0.0); + double after = nextafter(d, 1.7976931348623157e308); + char b1[32], b2[kSixDigitsToBufferSize]; + ABSL_RAW_LOG( + ERROR, "%s", + absl::StrCat( + "Mismatch #", i, " d=", d, " (", ToNineDigits(d), ")", + " sixdigits='", sixdigitsbuf, "'", " snprintf='", snprintfbuf, + "'", " Before.=", PerfectDtoa(before), " ", + (SixDigitsToBuffer(before, b2), b2), + " vs snprintf=", (snprintf(b1, sizeof(b1), "%g", before), b1), + " Perfect=", PerfectDtoa(d), " ", (SixDigitsToBuffer(d, b2), b2), + " vs snprintf=", (snprintf(b1, sizeof(b1), "%g", d), b1), + " After.=.", PerfectDtoa(after), " ", + (SixDigitsToBuffer(after, b2), b2), + " vs snprintf=", (snprintf(b1, sizeof(b1), "%g", after), b1)) + .c_str()); + } + } +} + +TEST(StrToInt32, Partial) { + struct Int32TestLine { + std::string input; + bool status; + int32_t value; + }; + const int32_t int32_min = std::numeric_limits<int32_t>::min(); + const int32_t int32_max = std::numeric_limits<int32_t>::max(); + Int32TestLine int32_test_line[] = { + {"", false, 0}, + {" ", false, 0}, + {"-", false, 0}, + {"123@@@", false, 123}, + {absl::StrCat(int32_min, int32_max), false, int32_min}, + {absl::StrCat(int32_max, int32_max), false, int32_max}, + }; + + for (const Int32TestLine& test_line : int32_test_line) { + int32_t value = -2; + bool status = safe_strto32_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = -2; + status = safe_strto32_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = -2; + status = safe_strto32_base(absl::string_view(test_line.input), &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + } +} + +TEST(StrToUint32, Partial) { + struct Uint32TestLine { + std::string input; + bool status; + uint32_t value; + }; + const uint32_t uint32_max = std::numeric_limits<uint32_t>::max(); + Uint32TestLine uint32_test_line[] = { + {"", false, 0}, + {" ", false, 0}, + {"-", false, 0}, + {"123@@@", false, 123}, + {absl::StrCat(uint32_max, uint32_max), false, uint32_max}, + }; + + for (const Uint32TestLine& test_line : uint32_test_line) { + uint32_t value = 2; + bool status = safe_strtou32_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = 2; + status = safe_strtou32_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = 2; + status = safe_strtou32_base(absl::string_view(test_line.input), &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + } +} + +TEST(StrToInt64, Partial) { + struct Int64TestLine { + std::string input; + bool status; + int64_t value; + }; + const int64_t int64_min = std::numeric_limits<int64_t>::min(); + const int64_t int64_max = std::numeric_limits<int64_t>::max(); + Int64TestLine int64_test_line[] = { + {"", false, 0}, + {" ", false, 0}, + {"-", false, 0}, + {"123@@@", false, 123}, + {absl::StrCat(int64_min, int64_max), false, int64_min}, + {absl::StrCat(int64_max, int64_max), false, int64_max}, + }; + + for (const Int64TestLine& test_line : int64_test_line) { + int64_t value = -2; + bool status = safe_strto64_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = -2; + status = safe_strto64_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = -2; + status = safe_strto64_base(absl::string_view(test_line.input), &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + } +} + +TEST(StrToUint64, Partial) { + struct Uint64TestLine { + std::string input; + bool status; + uint64_t value; + }; + const uint64_t uint64_max = std::numeric_limits<uint64_t>::max(); + Uint64TestLine uint64_test_line[] = { + {"", false, 0}, + {" ", false, 0}, + {"-", false, 0}, + {"123@@@", false, 123}, + {absl::StrCat(uint64_max, uint64_max), false, uint64_max}, + }; + + for (const Uint64TestLine& test_line : uint64_test_line) { + uint64_t value = 2; + bool status = safe_strtou64_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = 2; + status = safe_strtou64_base(test_line.input, &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + value = 2; + status = safe_strtou64_base(absl::string_view(test_line.input), &value, 10); + EXPECT_EQ(test_line.status, status) << test_line.input; + EXPECT_EQ(test_line.value, value) << test_line.input; + } +} + +TEST(StrToInt32Base, PrefixOnly) { + struct Int32TestLine { + std::string input; + bool status; + int32_t value; + }; + Int32TestLine int32_test_line[] = { + { "", false, 0 }, + { "-", false, 0 }, + { "-0", true, 0 }, + { "0", true, 0 }, + { "0x", false, 0 }, + { "-0x", false, 0 }, + }; + const int base_array[] = { 0, 2, 8, 10, 16 }; + + for (const Int32TestLine& line : int32_test_line) { + for (const int base : base_array) { + int32_t value = 2; + bool status = safe_strto32_base(line.input.c_str(), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strto32_base(line.input, &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strto32_base(absl::string_view(line.input), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + } + } +} + +TEST(StrToUint32Base, PrefixOnly) { + struct Uint32TestLine { + std::string input; + bool status; + uint32_t value; + }; + Uint32TestLine uint32_test_line[] = { + { "", false, 0 }, + { "0", true, 0 }, + { "0x", false, 0 }, + }; + const int base_array[] = { 0, 2, 8, 10, 16 }; + + for (const Uint32TestLine& line : uint32_test_line) { + for (const int base : base_array) { + uint32_t value = 2; + bool status = safe_strtou32_base(line.input.c_str(), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strtou32_base(line.input, &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strtou32_base(absl::string_view(line.input), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + } + } +} + +TEST(StrToInt64Base, PrefixOnly) { + struct Int64TestLine { + std::string input; + bool status; + int64_t value; + }; + Int64TestLine int64_test_line[] = { + { "", false, 0 }, + { "-", false, 0 }, + { "-0", true, 0 }, + { "0", true, 0 }, + { "0x", false, 0 }, + { "-0x", false, 0 }, + }; + const int base_array[] = { 0, 2, 8, 10, 16 }; + + for (const Int64TestLine& line : int64_test_line) { + for (const int base : base_array) { + int64_t value = 2; + bool status = safe_strto64_base(line.input.c_str(), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strto64_base(line.input, &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strto64_base(absl::string_view(line.input), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + } + } +} + +TEST(StrToUint64Base, PrefixOnly) { + struct Uint64TestLine { + std::string input; + bool status; + uint64_t value; + }; + Uint64TestLine uint64_test_line[] = { + { "", false, 0 }, + { "0", true, 0 }, + { "0x", false, 0 }, + }; + const int base_array[] = { 0, 2, 8, 10, 16 }; + + for (const Uint64TestLine& line : uint64_test_line) { + for (const int base : base_array) { + uint64_t value = 2; + bool status = safe_strtou64_base(line.input.c_str(), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strtou64_base(line.input, &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + value = 2; + status = safe_strtou64_base(absl::string_view(line.input), &value, base); + EXPECT_EQ(line.status, status) << line.input << " " << base; + EXPECT_EQ(line.value, value) << line.input << " " << base; + } + } +} + +void TestFastHexToBufferZeroPad16(uint64_t v) { + char buf[16]; + auto digits = absl::numbers_internal::FastHexToBufferZeroPad16(v, buf); + absl::string_view res(buf, 16); + char buf2[17]; + snprintf(buf2, sizeof(buf2), "%016" PRIx64, v); + EXPECT_EQ(res, buf2) << v; + size_t expected_digits = snprintf(buf2, sizeof(buf2), "%" PRIx64, v); + EXPECT_EQ(digits, expected_digits) << v; +} + +TEST(FastHexToBufferZeroPad16, Smoke) { + TestFastHexToBufferZeroPad16(std::numeric_limits<uint64_t>::min()); + TestFastHexToBufferZeroPad16(std::numeric_limits<uint64_t>::max()); + TestFastHexToBufferZeroPad16(std::numeric_limits<int64_t>::min()); + TestFastHexToBufferZeroPad16(std::numeric_limits<int64_t>::max()); + absl::BitGen rng; + for (int i = 0; i < 100000; ++i) { + TestFastHexToBufferZeroPad16( + absl::LogUniform(rng, std::numeric_limits<uint64_t>::min(), + std::numeric_limits<uint64_t>::max())); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_cat.cc b/third_party/abseil_cpp/absl/strings/str_cat.cc new file mode 100644 index 000000000000..dd5d25b0d6df --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_cat.cc @@ -0,0 +1,246 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_cat.h" + +#include <assert.h> + +#include <algorithm> +#include <cstdint> +#include <cstring> + +#include "absl/strings/ascii.h" +#include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/numbers.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +AlphaNum::AlphaNum(Hex hex) { + static_assert(numbers_internal::kFastToBufferSize >= 32, + "This function only works when output buffer >= 32 bytes long"); + char* const end = &digits_[numbers_internal::kFastToBufferSize]; + auto real_width = + absl::numbers_internal::FastHexToBufferZeroPad16(hex.value, end - 16); + if (real_width >= hex.width) { + piece_ = absl::string_view(end - real_width, real_width); + } else { + // Pad first 16 chars because FastHexToBufferZeroPad16 pads only to 16 and + // max pad width can be up to 20. + std::memset(end - 32, hex.fill, 16); + // Patch up everything else up to the real_width. + std::memset(end - real_width - 16, hex.fill, 16); + piece_ = absl::string_view(end - hex.width, hex.width); + } +} + +AlphaNum::AlphaNum(Dec dec) { + assert(dec.width <= numbers_internal::kFastToBufferSize); + char* const end = &digits_[numbers_internal::kFastToBufferSize]; + char* const minfill = end - dec.width; + char* writer = end; + uint64_t value = dec.value; + bool neg = dec.neg; + while (value > 9) { + *--writer = '0' + (value % 10); + value /= 10; + } + *--writer = '0' + value; + if (neg) *--writer = '-'; + + ptrdiff_t fillers = writer - minfill; + if (fillers > 0) { + // Tricky: if the fill character is ' ', then it's <fill><+/-><digits> + // But...: if the fill character is '0', then it's <+/-><fill><digits> + bool add_sign_again = false; + if (neg && dec.fill == '0') { // If filling with '0', + ++writer; // ignore the sign we just added + add_sign_again = true; // and re-add the sign later. + } + writer -= fillers; + std::fill_n(writer, fillers, dec.fill); + if (add_sign_again) *--writer = '-'; + } + + piece_ = absl::string_view(writer, end - writer); +} + +// ---------------------------------------------------------------------- +// StrCat() +// This merges the given strings or integers, with no delimiter. This +// is designed to be the fastest possible way to construct a string out +// of a mix of raw C strings, string_views, strings, and integer values. +// ---------------------------------------------------------------------- + +// Append is merely a version of memcpy that returns the address of the byte +// after the area just overwritten. +static char* Append(char* out, const AlphaNum& x) { + // memcpy is allowed to overwrite arbitrary memory, so doing this after the + // call would force an extra fetch of x.size(). + char* after = out + x.size(); + if (x.size() != 0) { + memcpy(out, x.data(), x.size()); + } + return after; +} + +std::string StrCat(const AlphaNum& a, const AlphaNum& b) { + std::string result; + absl::strings_internal::STLStringResizeUninitialized(&result, + a.size() + b.size()); + char* const begin = &result[0]; + char* out = begin; + out = Append(out, a); + out = Append(out, b); + assert(out == begin + result.size()); + return result; +} + +std::string StrCat(const AlphaNum& a, const AlphaNum& b, const AlphaNum& c) { + std::string result; + strings_internal::STLStringResizeUninitialized( + &result, a.size() + b.size() + c.size()); + char* const begin = &result[0]; + char* out = begin; + out = Append(out, a); + out = Append(out, b); + out = Append(out, c); + assert(out == begin + result.size()); + return result; +} + +std::string StrCat(const AlphaNum& a, const AlphaNum& b, const AlphaNum& c, + const AlphaNum& d) { + std::string result; + strings_internal::STLStringResizeUninitialized( + &result, a.size() + b.size() + c.size() + d.size()); + char* const begin = &result[0]; + char* out = begin; + out = Append(out, a); + out = Append(out, b); + out = Append(out, c); + out = Append(out, d); + assert(out == begin + result.size()); + return result; +} + +namespace strings_internal { + +// Do not call directly - these are not part of the public API. +std::string CatPieces(std::initializer_list<absl::string_view> pieces) { + std::string result; + size_t total_size = 0; + for (const absl::string_view& piece : pieces) total_size += piece.size(); + strings_internal::STLStringResizeUninitialized(&result, total_size); + + char* const begin = &result[0]; + char* out = begin; + for (const absl::string_view& piece : pieces) { + const size_t this_size = piece.size(); + if (this_size != 0) { + memcpy(out, piece.data(), this_size); + out += this_size; + } + } + assert(out == begin + result.size()); + return result; +} + +// It's possible to call StrAppend with an absl::string_view that is itself a +// fragment of the string we're appending to. However the results of this are +// random. Therefore, check for this in debug mode. Use unsigned math so we +// only have to do one comparison. Note, there's an exception case: appending an +// empty string is always allowed. +#define ASSERT_NO_OVERLAP(dest, src) \ + assert(((src).size() == 0) || \ + (uintptr_t((src).data() - (dest).data()) > uintptr_t((dest).size()))) + +void AppendPieces(std::string* dest, + std::initializer_list<absl::string_view> pieces) { + size_t old_size = dest->size(); + size_t total_size = old_size; + for (const absl::string_view& piece : pieces) { + ASSERT_NO_OVERLAP(*dest, piece); + total_size += piece.size(); + } + strings_internal::STLStringResizeUninitialized(dest, total_size); + + char* const begin = &(*dest)[0]; + char* out = begin + old_size; + for (const absl::string_view& piece : pieces) { + const size_t this_size = piece.size(); + if (this_size != 0) { + memcpy(out, piece.data(), this_size); + out += this_size; + } + } + assert(out == begin + dest->size()); +} + +} // namespace strings_internal + +void StrAppend(std::string* dest, const AlphaNum& a) { + ASSERT_NO_OVERLAP(*dest, a); + dest->append(a.data(), a.size()); +} + +void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b) { + ASSERT_NO_OVERLAP(*dest, a); + ASSERT_NO_OVERLAP(*dest, b); + std::string::size_type old_size = dest->size(); + strings_internal::STLStringResizeUninitialized( + dest, old_size + a.size() + b.size()); + char* const begin = &(*dest)[0]; + char* out = begin + old_size; + out = Append(out, a); + out = Append(out, b); + assert(out == begin + dest->size()); +} + +void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c) { + ASSERT_NO_OVERLAP(*dest, a); + ASSERT_NO_OVERLAP(*dest, b); + ASSERT_NO_OVERLAP(*dest, c); + std::string::size_type old_size = dest->size(); + strings_internal::STLStringResizeUninitialized( + dest, old_size + a.size() + b.size() + c.size()); + char* const begin = &(*dest)[0]; + char* out = begin + old_size; + out = Append(out, a); + out = Append(out, b); + out = Append(out, c); + assert(out == begin + dest->size()); +} + +void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d) { + ASSERT_NO_OVERLAP(*dest, a); + ASSERT_NO_OVERLAP(*dest, b); + ASSERT_NO_OVERLAP(*dest, c); + ASSERT_NO_OVERLAP(*dest, d); + std::string::size_type old_size = dest->size(); + strings_internal::STLStringResizeUninitialized( + dest, old_size + a.size() + b.size() + c.size() + d.size()); + char* const begin = &(*dest)[0]; + char* out = begin + old_size; + out = Append(out, a); + out = Append(out, b); + out = Append(out, c); + out = Append(out, d); + assert(out == begin + dest->size()); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/str_cat.h b/third_party/abseil_cpp/absl/strings/str_cat.h new file mode 100644 index 000000000000..a8a85c7322b2 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_cat.h @@ -0,0 +1,408 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: str_cat.h +// ----------------------------------------------------------------------------- +// +// This package contains functions for efficiently concatenating and appending +// strings: `StrCat()` and `StrAppend()`. Most of the work within these routines +// is actually handled through use of a special AlphaNum type, which was +// designed to be used as a parameter type that efficiently manages conversion +// to strings and avoids copies in the above operations. +// +// Any routine accepting either a string or a number may accept `AlphaNum`. +// The basic idea is that by accepting a `const AlphaNum &` as an argument +// to your function, your callers will automagically convert bools, integers, +// and floating point values to strings for you. +// +// NOTE: Use of `AlphaNum` outside of the //absl/strings package is unsupported +// except for the specific case of function parameters of type `AlphaNum` or +// `const AlphaNum &`. In particular, instantiating `AlphaNum` directly as a +// stack variable is not supported. +// +// Conversion from 8-bit values is not accepted because, if it were, then an +// attempt to pass ':' instead of ":" might result in a 58 ending up in your +// result. +// +// Bools convert to "0" or "1". Pointers to types other than `char *` are not +// valid inputs. No output is generated for null `char *` pointers. +// +// Floating point numbers are formatted with six-digit precision, which is +// the default for "std::cout <<" or printf "%g" (the same as "%.6g"). +// +// You can convert to hexadecimal output rather than decimal output using the +// `Hex` type contained here. To do so, pass `Hex(my_int)` as a parameter to +// `StrCat()` or `StrAppend()`. You may specify a minimum hex field width using +// a `PadSpec` enum. +// +// ----------------------------------------------------------------------------- + +#ifndef ABSL_STRINGS_STR_CAT_H_ +#define ABSL_STRINGS_STR_CAT_H_ + +#include <array> +#include <cstdint> +#include <string> +#include <type_traits> +#include <vector> + +#include "absl/base/port.h" +#include "absl/strings/numbers.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace strings_internal { +// AlphaNumBuffer allows a way to pass a string to StrCat without having to do +// memory allocation. It is simply a pair of a fixed-size character array, and +// a size. Please don't use outside of absl, yet. +template <size_t max_size> +struct AlphaNumBuffer { + std::array<char, max_size> data; + size_t size; +}; + +} // namespace strings_internal + +// Enum that specifies the number of significant digits to return in a `Hex` or +// `Dec` conversion and fill character to use. A `kZeroPad2` value, for example, +// would produce hexadecimal strings such as "0a","0f" and a 'kSpacePad5' value +// would produce hexadecimal strings such as " a"," f". +enum PadSpec : uint8_t { + kNoPad = 1, + kZeroPad2, + kZeroPad3, + kZeroPad4, + kZeroPad5, + kZeroPad6, + kZeroPad7, + kZeroPad8, + kZeroPad9, + kZeroPad10, + kZeroPad11, + kZeroPad12, + kZeroPad13, + kZeroPad14, + kZeroPad15, + kZeroPad16, + kZeroPad17, + kZeroPad18, + kZeroPad19, + kZeroPad20, + + kSpacePad2 = kZeroPad2 + 64, + kSpacePad3, + kSpacePad4, + kSpacePad5, + kSpacePad6, + kSpacePad7, + kSpacePad8, + kSpacePad9, + kSpacePad10, + kSpacePad11, + kSpacePad12, + kSpacePad13, + kSpacePad14, + kSpacePad15, + kSpacePad16, + kSpacePad17, + kSpacePad18, + kSpacePad19, + kSpacePad20, +}; + +// ----------------------------------------------------------------------------- +// Hex +// ----------------------------------------------------------------------------- +// +// `Hex` stores a set of hexadecimal string conversion parameters for use +// within `AlphaNum` string conversions. +struct Hex { + uint64_t value; + uint8_t width; + char fill; + + template <typename Int> + explicit Hex( + Int v, PadSpec spec = absl::kNoPad, + typename std::enable_if<sizeof(Int) == 1 && + !std::is_pointer<Int>::value>::type* = nullptr) + : Hex(spec, static_cast<uint8_t>(v)) {} + template <typename Int> + explicit Hex( + Int v, PadSpec spec = absl::kNoPad, + typename std::enable_if<sizeof(Int) == 2 && + !std::is_pointer<Int>::value>::type* = nullptr) + : Hex(spec, static_cast<uint16_t>(v)) {} + template <typename Int> + explicit Hex( + Int v, PadSpec spec = absl::kNoPad, + typename std::enable_if<sizeof(Int) == 4 && + !std::is_pointer<Int>::value>::type* = nullptr) + : Hex(spec, static_cast<uint32_t>(v)) {} + template <typename Int> + explicit Hex( + Int v, PadSpec spec = absl::kNoPad, + typename std::enable_if<sizeof(Int) == 8 && + !std::is_pointer<Int>::value>::type* = nullptr) + : Hex(spec, static_cast<uint64_t>(v)) {} + template <typename Pointee> + explicit Hex(Pointee* v, PadSpec spec = absl::kNoPad) + : Hex(spec, reinterpret_cast<uintptr_t>(v)) {} + + private: + Hex(PadSpec spec, uint64_t v) + : value(v), + width(spec == absl::kNoPad + ? 1 + : spec >= absl::kSpacePad2 ? spec - absl::kSpacePad2 + 2 + : spec - absl::kZeroPad2 + 2), + fill(spec >= absl::kSpacePad2 ? ' ' : '0') {} +}; + +// ----------------------------------------------------------------------------- +// Dec +// ----------------------------------------------------------------------------- +// +// `Dec` stores a set of decimal string conversion parameters for use +// within `AlphaNum` string conversions. Dec is slower than the default +// integer conversion, so use it only if you need padding. +struct Dec { + uint64_t value; + uint8_t width; + char fill; + bool neg; + + template <typename Int> + explicit Dec(Int v, PadSpec spec = absl::kNoPad, + typename std::enable_if<(sizeof(Int) <= 8)>::type* = nullptr) + : value(v >= 0 ? static_cast<uint64_t>(v) + : uint64_t{0} - static_cast<uint64_t>(v)), + width(spec == absl::kNoPad + ? 1 + : spec >= absl::kSpacePad2 ? spec - absl::kSpacePad2 + 2 + : spec - absl::kZeroPad2 + 2), + fill(spec >= absl::kSpacePad2 ? ' ' : '0'), + neg(v < 0) {} +}; + +// ----------------------------------------------------------------------------- +// AlphaNum +// ----------------------------------------------------------------------------- +// +// The `AlphaNum` class acts as the main parameter type for `StrCat()` and +// `StrAppend()`, providing efficient conversion of numeric, boolean, and +// hexadecimal values (through the `Hex` type) into strings. + +class AlphaNum { + public: + // No bool ctor -- bools convert to an integral type. + // A bool ctor would also convert incoming pointers (bletch). + + AlphaNum(int x) // NOLINT(runtime/explicit) + : piece_(digits_, + numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + AlphaNum(unsigned int x) // NOLINT(runtime/explicit) + : piece_(digits_, + numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + AlphaNum(long x) // NOLINT(*) + : piece_(digits_, + numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + AlphaNum(unsigned long x) // NOLINT(*) + : piece_(digits_, + numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + AlphaNum(long long x) // NOLINT(*) + : piece_(digits_, + numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + AlphaNum(unsigned long long x) // NOLINT(*) + : piece_(digits_, + numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + + AlphaNum(float f) // NOLINT(runtime/explicit) + : piece_(digits_, numbers_internal::SixDigitsToBuffer(f, digits_)) {} + AlphaNum(double f) // NOLINT(runtime/explicit) + : piece_(digits_, numbers_internal::SixDigitsToBuffer(f, digits_)) {} + + AlphaNum(Hex hex); // NOLINT(runtime/explicit) + AlphaNum(Dec dec); // NOLINT(runtime/explicit) + + template <size_t size> + AlphaNum( // NOLINT(runtime/explicit) + const strings_internal::AlphaNumBuffer<size>& buf) + : piece_(&buf.data[0], buf.size) {} + + AlphaNum(const char* c_str) : piece_(c_str) {} // NOLINT(runtime/explicit) + AlphaNum(absl::string_view pc) : piece_(pc) {} // NOLINT(runtime/explicit) + + template <typename Allocator> + AlphaNum( // NOLINT(runtime/explicit) + const std::basic_string<char, std::char_traits<char>, Allocator>& str) + : piece_(str) {} + + // Use string literals ":" instead of character literals ':'. + AlphaNum(char c) = delete; // NOLINT(runtime/explicit) + + AlphaNum(const AlphaNum&) = delete; + AlphaNum& operator=(const AlphaNum&) = delete; + + absl::string_view::size_type size() const { return piece_.size(); } + const char* data() const { return piece_.data(); } + absl::string_view Piece() const { return piece_; } + + // Normal enums are already handled by the integer formatters. + // This overload matches only scoped enums. + template <typename T, + typename = typename std::enable_if< + std::is_enum<T>{} && !std::is_convertible<T, int>{}>::type> + AlphaNum(T e) // NOLINT(runtime/explicit) + : AlphaNum(static_cast<typename std::underlying_type<T>::type>(e)) {} + + // vector<bool>::reference and const_reference require special help to + // convert to `AlphaNum` because it requires two user defined conversions. + template < + typename T, + typename std::enable_if< + std::is_class<T>::value && + (std::is_same<T, std::vector<bool>::reference>::value || + std::is_same<T, std::vector<bool>::const_reference>::value)>::type* = + nullptr> + AlphaNum(T e) : AlphaNum(static_cast<bool>(e)) {} // NOLINT(runtime/explicit) + + private: + absl::string_view piece_; + char digits_[numbers_internal::kFastToBufferSize]; +}; + +// ----------------------------------------------------------------------------- +// StrCat() +// ----------------------------------------------------------------------------- +// +// Merges given strings or numbers, using no delimiter(s), returning the merged +// result as a string. +// +// `StrCat()` is designed to be the fastest possible way to construct a string +// out of a mix of raw C strings, string_views, strings, bool values, +// and numeric values. +// +// Don't use `StrCat()` for user-visible strings. The localization process +// works poorly on strings built up out of fragments. +// +// For clarity and performance, don't use `StrCat()` when appending to a +// string. Use `StrAppend()` instead. In particular, avoid using any of these +// (anti-)patterns: +// +// str.append(StrCat(...)) +// str += StrCat(...) +// str = StrCat(str, ...) +// +// The last case is the worst, with a potential to change a loop +// from a linear time operation with O(1) dynamic allocations into a +// quadratic time operation with O(n) dynamic allocations. +// +// See `StrAppend()` below for more information. + +namespace strings_internal { + +// Do not call directly - this is not part of the public API. +std::string CatPieces(std::initializer_list<absl::string_view> pieces); +void AppendPieces(std::string* dest, + std::initializer_list<absl::string_view> pieces); + +} // namespace strings_internal + +ABSL_MUST_USE_RESULT inline std::string StrCat() { return std::string(); } + +ABSL_MUST_USE_RESULT inline std::string StrCat(const AlphaNum& a) { + return std::string(a.data(), a.size()); +} + +ABSL_MUST_USE_RESULT std::string StrCat(const AlphaNum& a, const AlphaNum& b); +ABSL_MUST_USE_RESULT std::string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c); +ABSL_MUST_USE_RESULT std::string StrCat(const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d); + +// Support 5 or more arguments +template <typename... AV> +ABSL_MUST_USE_RESULT inline std::string StrCat( + const AlphaNum& a, const AlphaNum& b, const AlphaNum& c, const AlphaNum& d, + const AlphaNum& e, const AV&... args) { + return strings_internal::CatPieces( + {a.Piece(), b.Piece(), c.Piece(), d.Piece(), e.Piece(), + static_cast<const AlphaNum&>(args).Piece()...}); +} + +// ----------------------------------------------------------------------------- +// StrAppend() +// ----------------------------------------------------------------------------- +// +// Appends a string or set of strings to an existing string, in a similar +// fashion to `StrCat()`. +// +// WARNING: `StrAppend(&str, a, b, c, ...)` requires that none of the +// a, b, c, parameters be a reference into str. For speed, `StrAppend()` does +// not try to check each of its input arguments to be sure that they are not +// a subset of the string being appended to. That is, while this will work: +// +// std::string s = "foo"; +// s += s; +// +// This output is undefined: +// +// std::string s = "foo"; +// StrAppend(&s, s); +// +// This output is undefined as well, since `absl::string_view` does not own its +// data: +// +// std::string s = "foobar"; +// absl::string_view p = s; +// StrAppend(&s, p); + +inline void StrAppend(std::string*) {} +void StrAppend(std::string* dest, const AlphaNum& a); +void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b); +void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c); +void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d); + +// Support 5 or more arguments +template <typename... AV> +inline void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, + const AlphaNum& c, const AlphaNum& d, const AlphaNum& e, + const AV&... args) { + strings_internal::AppendPieces( + dest, {a.Piece(), b.Piece(), c.Piece(), d.Piece(), e.Piece(), + static_cast<const AlphaNum&>(args).Piece()...}); +} + +// Helper function for the future StrCat default floating-point format, %.6g +// This is fast. +inline strings_internal::AlphaNumBuffer< + numbers_internal::kSixDigitsToBufferSize> +SixDigits(double d) { + strings_internal::AlphaNumBuffer<numbers_internal::kSixDigitsToBufferSize> + result; + result.size = numbers_internal::SixDigitsToBuffer(d, &result.data[0]); + return result; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STR_CAT_H_ diff --git a/third_party/abseil_cpp/absl/strings/str_cat_benchmark.cc b/third_party/abseil_cpp/absl/strings/str_cat_benchmark.cc new file mode 100644 index 000000000000..02c4dbe6d8f1 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_cat_benchmark.cc @@ -0,0 +1,187 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_cat.h" + +#include <cstdint> +#include <string> + +#include "benchmark/benchmark.h" +#include "absl/strings/substitute.h" + +namespace { + +const char kStringOne[] = "Once Upon A Time, "; +const char kStringTwo[] = "There was a string benchmark"; + +// We want to include negative numbers in the benchmark, so this function +// is used to count 0, 1, -1, 2, -2, 3, -3, ... +inline int IncrementAlternatingSign(int i) { + return i > 0 ? -i : 1 - i; +} + +void BM_Sum_By_StrCat(benchmark::State& state) { + int i = 0; + char foo[100]; + for (auto _ : state) { + // NOLINTNEXTLINE(runtime/printf) + strcpy(foo, absl::StrCat(kStringOne, i, kStringTwo, i * 65536ULL).c_str()); + int sum = 0; + for (char* f = &foo[0]; *f != 0; ++f) { + sum += *f; + } + benchmark::DoNotOptimize(sum); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_Sum_By_StrCat); + +void BM_StrCat_By_snprintf(benchmark::State& state) { + int i = 0; + char on_stack[1000]; + for (auto _ : state) { + snprintf(on_stack, sizeof(on_stack), "%s %s:%d", kStringOne, kStringTwo, i); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_StrCat_By_snprintf); + +void BM_StrCat_By_Strings(benchmark::State& state) { + int i = 0; + for (auto _ : state) { + std::string result = + std::string(kStringOne) + " " + kStringTwo + ":" + absl::StrCat(i); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_StrCat_By_Strings); + +void BM_StrCat_By_StringOpPlus(benchmark::State& state) { + int i = 0; + for (auto _ : state) { + std::string result = kStringOne; + result += " "; + result += kStringTwo; + result += ":"; + result += absl::StrCat(i); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_StrCat_By_StringOpPlus); + +void BM_StrCat_By_StrCat(benchmark::State& state) { + int i = 0; + for (auto _ : state) { + std::string result = absl::StrCat(kStringOne, " ", kStringTwo, ":", i); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_StrCat_By_StrCat); + +void BM_HexCat_By_StrCat(benchmark::State& state) { + int i = 0; + for (auto _ : state) { + std::string result = + absl::StrCat(kStringOne, " ", absl::Hex(int64_t{i} + 0x10000000)); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_HexCat_By_StrCat); + +void BM_HexCat_By_Substitute(benchmark::State& state) { + int i = 0; + for (auto _ : state) { + std::string result = absl::Substitute( + "$0 $1", kStringOne, reinterpret_cast<void*>(int64_t{i} + 0x10000000)); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_HexCat_By_Substitute); + +void BM_FloatToString_By_StrCat(benchmark::State& state) { + int i = 0; + float foo = 0.0f; + for (auto _ : state) { + std::string result = absl::StrCat(foo += 1.001f, " != ", int64_t{i}); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_FloatToString_By_StrCat); + +void BM_DoubleToString_By_SixDigits(benchmark::State& state) { + int i = 0; + double foo = 0.0; + for (auto _ : state) { + std::string result = + absl::StrCat(absl::SixDigits(foo += 1.001), " != ", int64_t{i}); + benchmark::DoNotOptimize(result); + i = IncrementAlternatingSign(i); + } +} +BENCHMARK(BM_DoubleToString_By_SixDigits); + +template <typename... Chunks> +void BM_StrAppendImpl(benchmark::State& state, size_t total_bytes, + Chunks... chunks) { + for (auto s : state) { + std::string result; + while (result.size() < total_bytes) { + absl::StrAppend(&result, chunks...); + benchmark::DoNotOptimize(result); + } + } +} + +void BM_StrAppend(benchmark::State& state) { + const int total_bytes = state.range(0); + const int chunks_at_a_time = state.range(1); + const absl::string_view kChunk = "0123456789"; + + switch (chunks_at_a_time) { + case 1: + return BM_StrAppendImpl(state, total_bytes, kChunk); + case 2: + return BM_StrAppendImpl(state, total_bytes, kChunk, kChunk); + case 4: + return BM_StrAppendImpl(state, total_bytes, kChunk, kChunk, kChunk, + kChunk); + case 8: + return BM_StrAppendImpl(state, total_bytes, kChunk, kChunk, kChunk, + kChunk, kChunk, kChunk, kChunk, kChunk); + default: + std::abort(); + } +} + +template <typename B> +void StrAppendConfig(B* benchmark) { + for (int bytes : {10, 100, 1000, 10000}) { + for (int chunks : {1, 2, 4, 8}) { + // Only add the ones that divide properly. Otherwise we are over counting. + if (bytes % (10 * chunks) == 0) { + benchmark->Args({bytes, chunks}); + } + } + } +} + +BENCHMARK(BM_StrAppend)->Apply(StrAppendConfig); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_cat_test.cc b/third_party/abseil_cpp/absl/strings/str_cat_test.cc new file mode 100644 index 000000000000..f3770dc076f0 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_cat_test.cc @@ -0,0 +1,610 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit tests for all str_cat.h functions + +#include "absl/strings/str_cat.h" + +#include <cstdint> +#include <string> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/strings/substitute.h" + +#ifdef __ANDROID__ +// Android assert messages only go to system log, so death tests cannot inspect +// the message for matching. +#define ABSL_EXPECT_DEBUG_DEATH(statement, regex) \ + EXPECT_DEBUG_DEATH(statement, ".*") +#else +#define ABSL_EXPECT_DEBUG_DEATH(statement, regex) \ + EXPECT_DEBUG_DEATH(statement, regex) +#endif + +namespace { + +// Test absl::StrCat of ints and longs of various sizes and signdedness. +TEST(StrCat, Ints) { + const short s = -1; // NOLINT(runtime/int) + const uint16_t us = 2; + const int i = -3; + const unsigned int ui = 4; + const long l = -5; // NOLINT(runtime/int) + const unsigned long ul = 6; // NOLINT(runtime/int) + const long long ll = -7; // NOLINT(runtime/int) + const unsigned long long ull = 8; // NOLINT(runtime/int) + const ptrdiff_t ptrdiff = -9; + const size_t size = 10; + const intptr_t intptr = -12; + const uintptr_t uintptr = 13; + std::string answer; + answer = absl::StrCat(s, us); + EXPECT_EQ(answer, "-12"); + answer = absl::StrCat(i, ui); + EXPECT_EQ(answer, "-34"); + answer = absl::StrCat(l, ul); + EXPECT_EQ(answer, "-56"); + answer = absl::StrCat(ll, ull); + EXPECT_EQ(answer, "-78"); + answer = absl::StrCat(ptrdiff, size); + EXPECT_EQ(answer, "-910"); + answer = absl::StrCat(ptrdiff, intptr); + EXPECT_EQ(answer, "-9-12"); + answer = absl::StrCat(uintptr, 0); + EXPECT_EQ(answer, "130"); +} + +TEST(StrCat, Enums) { + enum SmallNumbers { One = 1, Ten = 10 } e = Ten; + EXPECT_EQ("10", absl::StrCat(e)); + EXPECT_EQ("-5", absl::StrCat(SmallNumbers(-5))); + + enum class Option { Boxers = 1, Briefs = -1 }; + + EXPECT_EQ("-1", absl::StrCat(Option::Briefs)); + + enum class Airplane : uint64_t { + Airbus = 1, + Boeing = 1000, + Canary = 10000000000 // too big for "int" + }; + + EXPECT_EQ("10000000000", absl::StrCat(Airplane::Canary)); + + enum class TwoGig : int32_t { + TwoToTheZero = 1, + TwoToTheSixteenth = 1 << 16, + TwoToTheThirtyFirst = INT32_MIN + }; + EXPECT_EQ("65536", absl::StrCat(TwoGig::TwoToTheSixteenth)); + EXPECT_EQ("-2147483648", absl::StrCat(TwoGig::TwoToTheThirtyFirst)); + EXPECT_EQ("-1", absl::StrCat(static_cast<TwoGig>(-1))); + + enum class FourGig : uint32_t { + TwoToTheZero = 1, + TwoToTheSixteenth = 1 << 16, + TwoToTheThirtyFirst = 1U << 31 // too big for "int" + }; + EXPECT_EQ("65536", absl::StrCat(FourGig::TwoToTheSixteenth)); + EXPECT_EQ("2147483648", absl::StrCat(FourGig::TwoToTheThirtyFirst)); + EXPECT_EQ("4294967295", absl::StrCat(static_cast<FourGig>(-1))); + + EXPECT_EQ("10000000000", absl::StrCat(Airplane::Canary)); +} + +TEST(StrCat, Basics) { + std::string result; + + std::string strs[] = {"Hello", "Cruel", "World"}; + + std::string stdstrs[] = { + "std::Hello", + "std::Cruel", + "std::World" + }; + + absl::string_view pieces[] = {"Hello", "Cruel", "World"}; + + const char* c_strs[] = { + "Hello", + "Cruel", + "World" + }; + + int32_t i32s[] = {'H', 'C', 'W'}; + uint64_t ui64s[] = {12345678910LL, 10987654321LL}; + + EXPECT_EQ(absl::StrCat(), ""); + + result = absl::StrCat(false, true, 2, 3); + EXPECT_EQ(result, "0123"); + + result = absl::StrCat(-1); + EXPECT_EQ(result, "-1"); + + result = absl::StrCat(absl::SixDigits(0.5)); + EXPECT_EQ(result, "0.5"); + + result = absl::StrCat(strs[1], pieces[2]); + EXPECT_EQ(result, "CruelWorld"); + + result = absl::StrCat(stdstrs[1], " ", stdstrs[2]); + EXPECT_EQ(result, "std::Cruel std::World"); + + result = absl::StrCat(strs[0], ", ", pieces[2]); + EXPECT_EQ(result, "Hello, World"); + + result = absl::StrCat(strs[0], ", ", strs[1], " ", strs[2], "!"); + EXPECT_EQ(result, "Hello, Cruel World!"); + + result = absl::StrCat(pieces[0], ", ", pieces[1], " ", pieces[2]); + EXPECT_EQ(result, "Hello, Cruel World"); + + result = absl::StrCat(c_strs[0], ", ", c_strs[1], " ", c_strs[2]); + EXPECT_EQ(result, "Hello, Cruel World"); + + result = absl::StrCat("ASCII ", i32s[0], ", ", i32s[1], " ", i32s[2], "!"); + EXPECT_EQ(result, "ASCII 72, 67 87!"); + + result = absl::StrCat(ui64s[0], ", ", ui64s[1], "!"); + EXPECT_EQ(result, "12345678910, 10987654321!"); + + std::string one = + "1"; // Actually, it's the size of this string that we want; a + // 64-bit build distinguishes between size_t and uint64_t, + // even though they're both unsigned 64-bit values. + result = absl::StrCat("And a ", one.size(), " and a ", + &result[2] - &result[0], " and a ", one, " 2 3 4", "!"); + EXPECT_EQ(result, "And a 1 and a 2 and a 1 2 3 4!"); + + // result = absl::StrCat("Single chars won't compile", '!'); + // result = absl::StrCat("Neither will nullptrs", nullptr); + result = + absl::StrCat("To output a char by ASCII/numeric value, use +: ", '!' + 0); + EXPECT_EQ(result, "To output a char by ASCII/numeric value, use +: 33"); + + float f = 100000.5; + result = absl::StrCat("A hundred K and a half is ", absl::SixDigits(f)); + EXPECT_EQ(result, "A hundred K and a half is 100000"); + + f = 100001.5; + result = + absl::StrCat("A hundred K and one and a half is ", absl::SixDigits(f)); + EXPECT_EQ(result, "A hundred K and one and a half is 100002"); + + double d = 100000.5; + d *= d; + result = + absl::StrCat("A hundred K and a half squared is ", absl::SixDigits(d)); + EXPECT_EQ(result, "A hundred K and a half squared is 1.00001e+10"); + + result = absl::StrCat(1, 2, 333, 4444, 55555, 666666, 7777777, 88888888, + 999999999); + EXPECT_EQ(result, "12333444455555666666777777788888888999999999"); +} + +TEST(StrCat, CornerCases) { + std::string result; + + result = absl::StrCat(""); // NOLINT + EXPECT_EQ(result, ""); + result = absl::StrCat("", ""); + EXPECT_EQ(result, ""); + result = absl::StrCat("", "", ""); + EXPECT_EQ(result, ""); + result = absl::StrCat("", "", "", ""); + EXPECT_EQ(result, ""); + result = absl::StrCat("", "", "", "", ""); + EXPECT_EQ(result, ""); +} + +// A minimal allocator that uses malloc(). +template <typename T> +struct Mallocator { + typedef T value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + + size_type max_size() const { + return size_t(std::numeric_limits<size_type>::max()) / sizeof(value_type); + } + template <typename U> + struct rebind { + typedef Mallocator<U> other; + }; + Mallocator() = default; + template <class U> + Mallocator(const Mallocator<U>&) {} // NOLINT(runtime/explicit) + + T* allocate(size_t n) { return static_cast<T*>(std::malloc(n * sizeof(T))); } + void deallocate(T* p, size_t) { std::free(p); } +}; +template <typename T, typename U> +bool operator==(const Mallocator<T>&, const Mallocator<U>&) { + return true; +} +template <typename T, typename U> +bool operator!=(const Mallocator<T>&, const Mallocator<U>&) { + return false; +} + +TEST(StrCat, CustomAllocator) { + using mstring = + std::basic_string<char, std::char_traits<char>, Mallocator<char>>; + const mstring str1("PARACHUTE OFF A BLIMP INTO MOSCONE!!"); + + const mstring str2("Read this book about coffee tables"); + + std::string result = absl::StrCat(str1, str2); + EXPECT_EQ(result, + "PARACHUTE OFF A BLIMP INTO MOSCONE!!" + "Read this book about coffee tables"); +} + +TEST(StrCat, MaxArgs) { + std::string result; + // Test 10 up to 26 arguments, the old maximum + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a"); + EXPECT_EQ(result, "123456789a"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b"); + EXPECT_EQ(result, "123456789ab"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c"); + EXPECT_EQ(result, "123456789abc"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d"); + EXPECT_EQ(result, "123456789abcd"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e"); + EXPECT_EQ(result, "123456789abcde"); + result = + absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f"); + EXPECT_EQ(result, "123456789abcdef"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g"); + EXPECT_EQ(result, "123456789abcdefg"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h"); + EXPECT_EQ(result, "123456789abcdefgh"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i"); + EXPECT_EQ(result, "123456789abcdefghi"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j"); + EXPECT_EQ(result, "123456789abcdefghij"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k"); + EXPECT_EQ(result, "123456789abcdefghijk"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l"); + EXPECT_EQ(result, "123456789abcdefghijkl"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m"); + EXPECT_EQ(result, "123456789abcdefghijklm"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m", "n"); + EXPECT_EQ(result, "123456789abcdefghijklmn"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m", "n", "o"); + EXPECT_EQ(result, "123456789abcdefghijklmno"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m", "n", "o", "p"); + EXPECT_EQ(result, "123456789abcdefghijklmnop"); + result = absl::StrCat(1, 2, 3, 4, 5, 6, 7, 8, 9, "a", "b", "c", "d", "e", "f", + "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q"); + EXPECT_EQ(result, "123456789abcdefghijklmnopq"); + // No limit thanks to C++11's variadic templates + result = absl::StrCat( + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, "a", "b", "c", "d", "e", "f", "g", "h", + "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", + "x", "y", "z", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", + "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"); + EXPECT_EQ(result, + "12345678910abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"); +} + +TEST(StrAppend, Basics) { + std::string result = "existing text"; + + std::string strs[] = {"Hello", "Cruel", "World"}; + + std::string stdstrs[] = { + "std::Hello", + "std::Cruel", + "std::World" + }; + + absl::string_view pieces[] = {"Hello", "Cruel", "World"}; + + const char* c_strs[] = { + "Hello", + "Cruel", + "World" + }; + + int32_t i32s[] = {'H', 'C', 'W'}; + uint64_t ui64s[] = {12345678910LL, 10987654321LL}; + + std::string::size_type old_size = result.size(); + absl::StrAppend(&result); + EXPECT_EQ(result.size(), old_size); + + old_size = result.size(); + absl::StrAppend(&result, strs[0]); + EXPECT_EQ(result.substr(old_size), "Hello"); + + old_size = result.size(); + absl::StrAppend(&result, strs[1], pieces[2]); + EXPECT_EQ(result.substr(old_size), "CruelWorld"); + + old_size = result.size(); + absl::StrAppend(&result, stdstrs[0], ", ", pieces[2]); + EXPECT_EQ(result.substr(old_size), "std::Hello, World"); + + old_size = result.size(); + absl::StrAppend(&result, strs[0], ", ", stdstrs[1], " ", strs[2], "!"); + EXPECT_EQ(result.substr(old_size), "Hello, std::Cruel World!"); + + old_size = result.size(); + absl::StrAppend(&result, pieces[0], ", ", pieces[1], " ", pieces[2]); + EXPECT_EQ(result.substr(old_size), "Hello, Cruel World"); + + old_size = result.size(); + absl::StrAppend(&result, c_strs[0], ", ", c_strs[1], " ", c_strs[2]); + EXPECT_EQ(result.substr(old_size), "Hello, Cruel World"); + + old_size = result.size(); + absl::StrAppend(&result, "ASCII ", i32s[0], ", ", i32s[1], " ", i32s[2], "!"); + EXPECT_EQ(result.substr(old_size), "ASCII 72, 67 87!"); + + old_size = result.size(); + absl::StrAppend(&result, ui64s[0], ", ", ui64s[1], "!"); + EXPECT_EQ(result.substr(old_size), "12345678910, 10987654321!"); + + std::string one = + "1"; // Actually, it's the size of this string that we want; a + // 64-bit build distinguishes between size_t and uint64_t, + // even though they're both unsigned 64-bit values. + old_size = result.size(); + absl::StrAppend(&result, "And a ", one.size(), " and a ", + &result[2] - &result[0], " and a ", one, " 2 3 4", "!"); + EXPECT_EQ(result.substr(old_size), "And a 1 and a 2 and a 1 2 3 4!"); + + // result = absl::StrCat("Single chars won't compile", '!'); + // result = absl::StrCat("Neither will nullptrs", nullptr); + old_size = result.size(); + absl::StrAppend(&result, + "To output a char by ASCII/numeric value, use +: ", '!' + 0); + EXPECT_EQ(result.substr(old_size), + "To output a char by ASCII/numeric value, use +: 33"); + + // Test 9 arguments, the old maximum + old_size = result.size(); + absl::StrAppend(&result, 1, 22, 333, 4444, 55555, 666666, 7777777, 88888888, + 9); + EXPECT_EQ(result.substr(old_size), "1223334444555556666667777777888888889"); + + // No limit thanks to C++11's variadic templates + old_size = result.size(); + absl::StrAppend( + &result, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, // + "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", // + "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", // + "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", // + "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", // + "No limit thanks to C++11's variadic templates"); + EXPECT_EQ(result.substr(old_size), + "12345678910abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + "No limit thanks to C++11's variadic templates"); +} + +TEST(StrCat, VectorBoolReferenceTypes) { + std::vector<bool> v; + v.push_back(true); + v.push_back(false); + std::vector<bool> const& cv = v; + // Test that vector<bool>::reference and vector<bool>::const_reference + // are handled as if the were really bool types and not the proxy types + // they really are. + std::string result = absl::StrCat(v[0], v[1], cv[0], cv[1]); // NOLINT + EXPECT_EQ(result, "1010"); +} + +// Passing nullptr to memcpy is undefined behavior and this test +// provides coverage of codepaths that handle empty strings with nullptrs. +TEST(StrCat, AvoidsMemcpyWithNullptr) { + EXPECT_EQ(absl::StrCat(42, absl::string_view{}), "42"); + + // Cover CatPieces code. + EXPECT_EQ(absl::StrCat(1, 2, 3, 4, 5, absl::string_view{}), "12345"); + + // Cover AppendPieces. + std::string result; + absl::StrAppend(&result, 1, 2, 3, 4, 5, absl::string_view{}); + EXPECT_EQ(result, "12345"); +} + +#ifdef GTEST_HAS_DEATH_TEST +TEST(StrAppend, Death) { + std::string s = "self"; + // on linux it's "assertion", on mac it's "Assertion", + // on chromiumos it's "Assertion ... failed". + ABSL_EXPECT_DEBUG_DEATH(absl::StrAppend(&s, s.c_str() + 1), + "ssertion.*failed"); + ABSL_EXPECT_DEBUG_DEATH(absl::StrAppend(&s, s), "ssertion.*failed"); +} +#endif // GTEST_HAS_DEATH_TEST + +TEST(StrAppend, CornerCases) { + std::string result; + absl::StrAppend(&result, ""); + EXPECT_EQ(result, ""); + absl::StrAppend(&result, "", ""); + EXPECT_EQ(result, ""); + absl::StrAppend(&result, "", "", ""); + EXPECT_EQ(result, ""); + absl::StrAppend(&result, "", "", "", ""); + EXPECT_EQ(result, ""); + absl::StrAppend(&result, "", "", "", "", ""); + EXPECT_EQ(result, ""); +} + +TEST(StrAppend, CornerCasesNonEmptyAppend) { + for (std::string result : {"hello", "a string too long to fit in the SSO"}) { + const std::string expected = result; + absl::StrAppend(&result, ""); + EXPECT_EQ(result, expected); + absl::StrAppend(&result, "", ""); + EXPECT_EQ(result, expected); + absl::StrAppend(&result, "", "", ""); + EXPECT_EQ(result, expected); + absl::StrAppend(&result, "", "", "", ""); + EXPECT_EQ(result, expected); + absl::StrAppend(&result, "", "", "", "", ""); + EXPECT_EQ(result, expected); + } +} + +template <typename IntType> +void CheckHex(IntType v, const char* nopad_format, const char* zeropad_format, + const char* spacepad_format) { + char expected[256]; + + std::string actual = absl::StrCat(absl::Hex(v, absl::kNoPad)); + snprintf(expected, sizeof(expected), nopad_format, v); + EXPECT_EQ(expected, actual) << " decimal value " << v; + + for (int spec = absl::kZeroPad2; spec <= absl::kZeroPad20; ++spec) { + std::string actual = + absl::StrCat(absl::Hex(v, static_cast<absl::PadSpec>(spec))); + snprintf(expected, sizeof(expected), zeropad_format, + spec - absl::kZeroPad2 + 2, v); + EXPECT_EQ(expected, actual) << " decimal value " << v; + } + + for (int spec = absl::kSpacePad2; spec <= absl::kSpacePad20; ++spec) { + std::string actual = + absl::StrCat(absl::Hex(v, static_cast<absl::PadSpec>(spec))); + snprintf(expected, sizeof(expected), spacepad_format, + spec - absl::kSpacePad2 + 2, v); + EXPECT_EQ(expected, actual) << " decimal value " << v; + } +} + +template <typename IntType> +void CheckDec(IntType v, const char* nopad_format, const char* zeropad_format, + const char* spacepad_format) { + char expected[256]; + + std::string actual = absl::StrCat(absl::Dec(v, absl::kNoPad)); + snprintf(expected, sizeof(expected), nopad_format, v); + EXPECT_EQ(expected, actual) << " decimal value " << v; + + for (int spec = absl::kZeroPad2; spec <= absl::kZeroPad20; ++spec) { + std::string actual = + absl::StrCat(absl::Dec(v, static_cast<absl::PadSpec>(spec))); + snprintf(expected, sizeof(expected), zeropad_format, + spec - absl::kZeroPad2 + 2, v); + EXPECT_EQ(expected, actual) + << " decimal value " << v << " format '" << zeropad_format + << "' digits " << (spec - absl::kZeroPad2 + 2); + } + + for (int spec = absl::kSpacePad2; spec <= absl::kSpacePad20; ++spec) { + std::string actual = + absl::StrCat(absl::Dec(v, static_cast<absl::PadSpec>(spec))); + snprintf(expected, sizeof(expected), spacepad_format, + spec - absl::kSpacePad2 + 2, v); + EXPECT_EQ(expected, actual) + << " decimal value " << v << " format '" << spacepad_format + << "' digits " << (spec - absl::kSpacePad2 + 2); + } +} + +void CheckHexDec64(uint64_t v) { + unsigned long long ullv = v; // NOLINT(runtime/int) + + CheckHex(ullv, "%llx", "%0*llx", "%*llx"); + CheckDec(ullv, "%llu", "%0*llu", "%*llu"); + + long long llv = static_cast<long long>(ullv); // NOLINT(runtime/int) + CheckDec(llv, "%lld", "%0*lld", "%*lld"); + + if (sizeof(v) == sizeof(&v)) { + auto uintptr = static_cast<uintptr_t>(v); + void* ptr = reinterpret_cast<void*>(uintptr); + CheckHex(ptr, "%llx", "%0*llx", "%*llx"); + } +} + +void CheckHexDec32(uint32_t uv) { + CheckHex(uv, "%x", "%0*x", "%*x"); + CheckDec(uv, "%u", "%0*u", "%*u"); + int32_t v = static_cast<int32_t>(uv); + CheckDec(v, "%d", "%0*d", "%*d"); + + if (sizeof(v) == sizeof(&v)) { + auto uintptr = static_cast<uintptr_t>(v); + void* ptr = reinterpret_cast<void*>(uintptr); + CheckHex(ptr, "%x", "%0*x", "%*x"); + } +} + +void CheckAll(uint64_t v) { + CheckHexDec64(v); + CheckHexDec32(static_cast<uint32_t>(v)); +} + +void TestFastPrints() { + // Test all small ints; there aren't many and they're common. + for (int i = 0; i < 10000; i++) { + CheckAll(i); + } + + CheckAll(std::numeric_limits<uint64_t>::max()); + CheckAll(std::numeric_limits<uint64_t>::max() - 1); + CheckAll(std::numeric_limits<int64_t>::min()); + CheckAll(std::numeric_limits<int64_t>::min() + 1); + CheckAll(std::numeric_limits<uint32_t>::max()); + CheckAll(std::numeric_limits<uint32_t>::max() - 1); + CheckAll(std::numeric_limits<int32_t>::min()); + CheckAll(std::numeric_limits<int32_t>::min() + 1); + CheckAll(999999999); // fits in 32 bits + CheckAll(1000000000); // fits in 32 bits + CheckAll(9999999999); // doesn't fit in 32 bits + CheckAll(10000000000); // doesn't fit in 32 bits + CheckAll(999999999999999999); // fits in signed 64-bit + CheckAll(9999999999999999999u); // fits in unsigned 64-bit, but not signed. + CheckAll(1000000000000000000); // fits in signed 64-bit + CheckAll(10000000000000000000u); // fits in unsigned 64-bit, but not signed. + + CheckAll(999999999876543210); // check all decimal digits, signed + CheckAll(9999999999876543210u); // check all decimal digits, unsigned. + CheckAll(0x123456789abcdef0); // check all hex digits + CheckAll(0x12345678); + + int8_t minus_one_8bit = -1; + EXPECT_EQ("ff", absl::StrCat(absl::Hex(minus_one_8bit))); + + int16_t minus_one_16bit = -1; + EXPECT_EQ("ffff", absl::StrCat(absl::Hex(minus_one_16bit))); +} + +TEST(Numbers, TestFunctionsMovedOverFromNumbersMain) { + TestFastPrints(); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_format.h b/third_party/abseil_cpp/absl/strings/str_format.h new file mode 100644 index 000000000000..01465107e105 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_format.h @@ -0,0 +1,813 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: str_format.h +// ----------------------------------------------------------------------------- +// +// The `str_format` library is a typesafe replacement for the family of +// `printf()` string formatting routines within the `<cstdio>` standard library +// header. Like the `printf` family, `str_format` uses a "format string" to +// perform argument substitutions based on types. See the `FormatSpec` section +// below for format string documentation. +// +// Example: +// +// std::string s = absl::StrFormat( +// "%s %s You have $%d!", "Hello", name, dollars); +// +// The library consists of the following basic utilities: +// +// * `absl::StrFormat()`, a type-safe replacement for `std::sprintf()`, to +// write a format string to a `string` value. +// * `absl::StrAppendFormat()` to append a format string to a `string` +// * `absl::StreamFormat()` to more efficiently write a format string to a +// stream, such as`std::cout`. +// * `absl::PrintF()`, `absl::FPrintF()` and `absl::SNPrintF()` as +// replacements for `std::printf()`, `std::fprintf()` and `std::snprintf()`. +// +// Note: a version of `std::sprintf()` is not supported as it is +// generally unsafe due to buffer overflows. +// +// Additionally, you can provide a format string (and its associated arguments) +// using one of the following abstractions: +// +// * A `FormatSpec` class template fully encapsulates a format string and its +// type arguments and is usually provided to `str_format` functions as a +// variadic argument of type `FormatSpec<Arg...>`. The `FormatSpec<Args...>` +// template is evaluated at compile-time, providing type safety. +// * A `ParsedFormat` instance, which encapsulates a specific, pre-compiled +// format string for a specific set of type(s), and which can be passed +// between API boundaries. (The `FormatSpec` type should not be used +// directly except as an argument type for wrapper functions.) +// +// The `str_format` library provides the ability to output its format strings to +// arbitrary sink types: +// +// * A generic `Format()` function to write outputs to arbitrary sink types, +// which must implement a `FormatRawSink` interface. +// +// * A `FormatUntyped()` function that is similar to `Format()` except it is +// loosely typed. `FormatUntyped()` is not a template and does not perform +// any compile-time checking of the format string; instead, it returns a +// boolean from a runtime check. +// +// In addition, the `str_format` library provides extension points for +// augmenting formatting to new types. See "StrFormat Extensions" below. + +#ifndef ABSL_STRINGS_STR_FORMAT_H_ +#define ABSL_STRINGS_STR_FORMAT_H_ + +#include <cstdio> +#include <string> + +#include "absl/strings/internal/str_format/arg.h" // IWYU pragma: export +#include "absl/strings/internal/str_format/bind.h" // IWYU pragma: export +#include "absl/strings/internal/str_format/checker.h" // IWYU pragma: export +#include "absl/strings/internal/str_format/extension.h" // IWYU pragma: export +#include "absl/strings/internal/str_format/parser.h" // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// UntypedFormatSpec +// +// A type-erased class that can be used directly within untyped API entry +// points. An `UntypedFormatSpec` is specifically used as an argument to +// `FormatUntyped()`. +// +// Example: +// +// absl::UntypedFormatSpec format("%d"); +// std::string out; +// CHECK(absl::FormatUntyped(&out, format, {absl::FormatArg(1)})); +class UntypedFormatSpec { + public: + UntypedFormatSpec() = delete; + UntypedFormatSpec(const UntypedFormatSpec&) = delete; + UntypedFormatSpec& operator=(const UntypedFormatSpec&) = delete; + + explicit UntypedFormatSpec(string_view s) : spec_(s) {} + + protected: + explicit UntypedFormatSpec(const str_format_internal::ParsedFormatBase* pc) + : spec_(pc) {} + + private: + friend str_format_internal::UntypedFormatSpecImpl; + str_format_internal::UntypedFormatSpecImpl spec_; +}; + +// FormatStreamed() +// +// Takes a streamable argument and returns an object that can print it +// with '%s'. Allows printing of types that have an `operator<<` but no +// intrinsic type support within `StrFormat()` itself. +// +// Example: +// +// absl::StrFormat("%s", absl::FormatStreamed(obj)); +template <typename T> +str_format_internal::StreamedWrapper<T> FormatStreamed(const T& v) { + return str_format_internal::StreamedWrapper<T>(v); +} + +// FormatCountCapture +// +// This class provides a way to safely wrap `StrFormat()` captures of `%n` +// conversions, which denote the number of characters written by a formatting +// operation to this point, into an integer value. +// +// This wrapper is designed to allow safe usage of `%n` within `StrFormat(); in +// the `printf()` family of functions, `%n` is not safe to use, as the `int *` +// buffer can be used to capture arbitrary data. +// +// Example: +// +// int n = 0; +// std::string s = absl::StrFormat("%s%d%n", "hello", 123, +// absl::FormatCountCapture(&n)); +// EXPECT_EQ(8, n); +class FormatCountCapture { + public: + explicit FormatCountCapture(int* p) : p_(p) {} + + private: + // FormatCountCaptureHelper is used to define FormatConvertImpl() for this + // class. + friend struct str_format_internal::FormatCountCaptureHelper; + // Unused() is here because of the false positive from -Wunused-private-field + // p_ is used in the templated function of the friend FormatCountCaptureHelper + // class. + int* Unused() { return p_; } + int* p_; +}; + +// FormatSpec +// +// The `FormatSpec` type defines the makeup of a format string within the +// `str_format` library. It is a variadic class template that is evaluated at +// compile-time, according to the format string and arguments that are passed to +// it. +// +// You should not need to manipulate this type directly. You should only name it +// if you are writing wrapper functions which accept format arguments that will +// be provided unmodified to functions in this library. Such a wrapper function +// might be a class method that provides format arguments and/or internally uses +// the result of formatting. +// +// For a `FormatSpec` to be valid at compile-time, it must be provided as +// either: +// +// * A `constexpr` literal or `absl::string_view`, which is how it most often +// used. +// * A `ParsedFormat` instantiation, which ensures the format string is +// valid before use. (See below.) +// +// Example: +// +// // Provided as a string literal. +// absl::StrFormat("Welcome to %s, Number %d!", "The Village", 6); +// +// // Provided as a constexpr absl::string_view. +// constexpr absl::string_view formatString = "Welcome to %s, Number %d!"; +// absl::StrFormat(formatString, "The Village", 6); +// +// // Provided as a pre-compiled ParsedFormat object. +// // Note that this example is useful only for illustration purposes. +// absl::ParsedFormat<'s', 'd'> formatString("Welcome to %s, Number %d!"); +// absl::StrFormat(formatString, "TheVillage", 6); +// +// A format string generally follows the POSIX syntax as used within the POSIX +// `printf` specification. +// +// (See http://pubs.opengroup.org/onlinepubs/9699919799/functions/fprintf.html.) +// +// In specific, the `FormatSpec` supports the following type specifiers: +// * `c` for characters +// * `s` for strings +// * `d` or `i` for integers +// * `o` for unsigned integer conversions into octal +// * `x` or `X` for unsigned integer conversions into hex +// * `u` for unsigned integers +// * `f` or `F` for floating point values into decimal notation +// * `e` or `E` for floating point values into exponential notation +// * `a` or `A` for floating point values into hex exponential notation +// * `g` or `G` for floating point values into decimal or exponential +// notation based on their precision +// * `p` for pointer address values +// * `n` for the special case of writing out the number of characters +// written to this point. The resulting value must be captured within an +// `absl::FormatCountCapture` type. +// +// Implementation-defined behavior: +// * A null pointer provided to "%s" or "%p" is output as "(nil)". +// * A non-null pointer provided to "%p" is output in hex as if by %#x or +// %#lx. +// +// NOTE: `o`, `x\X` and `u` will convert signed values to their unsigned +// counterpart before formatting. +// +// Examples: +// "%c", 'a' -> "a" +// "%c", 32 -> " " +// "%s", "C" -> "C" +// "%s", std::string("C++") -> "C++" +// "%d", -10 -> "-10" +// "%o", 10 -> "12" +// "%x", 16 -> "10" +// "%f", 123456789 -> "123456789.000000" +// "%e", .01 -> "1.00000e-2" +// "%a", -3.0 -> "-0x1.8p+1" +// "%g", .01 -> "1e-2" +// "%p", (void*)&value -> "0x7ffdeb6ad2a4" +// +// int n = 0; +// std::string s = absl::StrFormat( +// "%s%d%n", "hello", 123, absl::FormatCountCapture(&n)); +// EXPECT_EQ(8, n); +// +// The `FormatSpec` intrinsically supports all of these fundamental C++ types: +// +// * Characters: `char`, `signed char`, `unsigned char` +// * Integers: `int`, `short`, `unsigned short`, `unsigned`, `long`, +// `unsigned long`, `long long`, `unsigned long long` +// * Floating-point: `float`, `double`, `long double` +// +// However, in the `str_format` library, a format conversion specifies a broader +// C++ conceptual category instead of an exact type. For example, `%s` binds to +// any string-like argument, so `std::string`, `absl::string_view`, and +// `const char*` are all accepted. Likewise, `%d` accepts any integer-like +// argument, etc. + +template <typename... Args> +using FormatSpec = str_format_internal::FormatSpecTemplate< + str_format_internal::ArgumentToConv<Args>()...>; + +// ParsedFormat +// +// A `ParsedFormat` is a class template representing a preparsed `FormatSpec`, +// with template arguments specifying the conversion characters used within the +// format string. Such characters must be valid format type specifiers, and +// these type specifiers are checked at compile-time. +// +// Instances of `ParsedFormat` can be created, copied, and reused to speed up +// formatting loops. A `ParsedFormat` may either be constructed statically, or +// dynamically through its `New()` factory function, which only constructs a +// runtime object if the format is valid at that time. +// +// Example: +// +// // Verified at compile time. +// absl::ParsedFormat<'s', 'd'> formatString("Welcome to %s, Number %d!"); +// absl::StrFormat(formatString, "TheVillage", 6); +// +// // Verified at runtime. +// auto format_runtime = absl::ParsedFormat<'d'>::New(format_string); +// if (format_runtime) { +// value = absl::StrFormat(*format_runtime, i); +// } else { +// ... error case ... +// } + +#if defined(__cpp_nontype_template_parameter_auto) +// If C++17 is available, an 'extended' format is also allowed that can specify +// multiple conversion characters per format argument, using a combination of +// `absl::FormatConversionCharSet` enum values (logically a set union) +// via the `|` operator. (Single character-based arguments are still accepted, +// but cannot be combined). Some common conversions also have predefined enum +// values, such as `absl::FormatConversionCharSet::kIntegral`. +// +// Example: +// // Extended format supports multiple conversion characters per argument, +// // specified via a combination of `FormatConversionCharSet` enums. +// using MyFormat = absl::ParsedFormat<absl::FormatConversionCharSet::d | +// absl::FormatConversionCharSet::x>; +// MyFormat GetFormat(bool use_hex) { +// if (use_hex) return MyFormat("foo %x bar"); +// return MyFormat("foo %d bar"); +// } +// // `format` can be used with any value that supports 'd' and 'x', +// // like `int`. +// auto format = GetFormat(use_hex); +// value = StringF(format, i); +template <auto... Conv> +using ParsedFormat = absl::str_format_internal::ExtendedParsedFormat< + absl::str_format_internal::ToFormatConversionCharSet(Conv)...>; +#else +template <char... Conv> +using ParsedFormat = str_format_internal::ExtendedParsedFormat< + absl::str_format_internal::ToFormatConversionCharSet(Conv)...>; +#endif // defined(__cpp_nontype_template_parameter_auto) + +// StrFormat() +// +// Returns a `string` given a `printf()`-style format string and zero or more +// additional arguments. Use it as you would `sprintf()`. `StrFormat()` is the +// primary formatting function within the `str_format` library, and should be +// used in most cases where you need type-safe conversion of types into +// formatted strings. +// +// The format string generally consists of ordinary character data along with +// one or more format conversion specifiers (denoted by the `%` character). +// Ordinary character data is returned unchanged into the result string, while +// each conversion specification performs a type substitution from +// `StrFormat()`'s other arguments. See the comments for `FormatSpec` for full +// information on the makeup of this format string. +// +// Example: +// +// std::string s = absl::StrFormat( +// "Welcome to %s, Number %d!", "The Village", 6); +// EXPECT_EQ("Welcome to The Village, Number 6!", s); +// +// Returns an empty string in case of error. +template <typename... Args> +ABSL_MUST_USE_RESULT std::string StrFormat(const FormatSpec<Args...>& format, + const Args&... args) { + return str_format_internal::FormatPack( + str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// StrAppendFormat() +// +// Appends to a `dst` string given a format string, and zero or more additional +// arguments, returning `*dst` as a convenience for chaining purposes. Appends +// nothing in case of error (but possibly alters its capacity). +// +// Example: +// +// std::string orig("For example PI is approximately "); +// std::cout << StrAppendFormat(&orig, "%12.6f", 3.14); +template <typename... Args> +std::string& StrAppendFormat(std::string* dst, + const FormatSpec<Args...>& format, + const Args&... args) { + return str_format_internal::AppendPack( + dst, str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// StreamFormat() +// +// Writes to an output stream given a format string and zero or more arguments, +// generally in a manner that is more efficient than streaming the result of +// `absl:: StrFormat()`. The returned object must be streamed before the full +// expression ends. +// +// Example: +// +// std::cout << StreamFormat("%12.6f", 3.14); +template <typename... Args> +ABSL_MUST_USE_RESULT str_format_internal::Streamable StreamFormat( + const FormatSpec<Args...>& format, const Args&... args) { + return str_format_internal::Streamable( + str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// PrintF() +// +// Writes to stdout given a format string and zero or more arguments. This +// function is functionally equivalent to `std::printf()` (and type-safe); +// prefer `absl::PrintF()` over `std::printf()`. +// +// Example: +// +// std::string_view s = "Ulaanbaatar"; +// absl::PrintF("The capital of Mongolia is %s", s); +// +// Outputs: "The capital of Mongolia is Ulaanbaatar" +// +template <typename... Args> +int PrintF(const FormatSpec<Args...>& format, const Args&... args) { + return str_format_internal::FprintF( + stdout, str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// FPrintF() +// +// Writes to a file given a format string and zero or more arguments. This +// function is functionally equivalent to `std::fprintf()` (and type-safe); +// prefer `absl::FPrintF()` over `std::fprintf()`. +// +// Example: +// +// std::string_view s = "Ulaanbaatar"; +// absl::FPrintF(stdout, "The capital of Mongolia is %s", s); +// +// Outputs: "The capital of Mongolia is Ulaanbaatar" +// +template <typename... Args> +int FPrintF(std::FILE* output, const FormatSpec<Args...>& format, + const Args&... args) { + return str_format_internal::FprintF( + output, str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// SNPrintF() +// +// Writes to a sized buffer given a format string and zero or more arguments. +// This function is functionally equivalent to `std::snprintf()` (and +// type-safe); prefer `absl::SNPrintF()` over `std::snprintf()`. +// +// In particular, a successful call to `absl::SNPrintF()` writes at most `size` +// bytes of the formatted output to `output`, including a NUL-terminator, and +// returns the number of bytes that would have been written if truncation did +// not occur. In the event of an error, a negative value is returned and `errno` +// is set. +// +// Example: +// +// std::string_view s = "Ulaanbaatar"; +// char output[128]; +// absl::SNPrintF(output, sizeof(output), +// "The capital of Mongolia is %s", s); +// +// Post-condition: output == "The capital of Mongolia is Ulaanbaatar" +// +template <typename... Args> +int SNPrintF(char* output, std::size_t size, const FormatSpec<Args...>& format, + const Args&... args) { + return str_format_internal::SnprintF( + output, size, str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// ----------------------------------------------------------------------------- +// Custom Output Formatting Functions +// ----------------------------------------------------------------------------- + +// FormatRawSink +// +// FormatRawSink is a type erased wrapper around arbitrary sink objects +// specifically used as an argument to `Format()`. +// +// All the object has to do define an overload of `AbslFormatFlush()` for the +// sink, usually by adding a ADL-based free function in the same namespace as +// the sink: +// +// void AbslFormatFlush(MySink* dest, absl::string_view part); +// +// where `dest` is the pointer passed to `absl::Format()`. The function should +// append `part` to `dest`. +// +// FormatRawSink does not own the passed sink object. The passed object must +// outlive the FormatRawSink. +class FormatRawSink { + public: + // Implicitly convert from any type that provides the hook function as + // described above. + template <typename T, + typename = typename std::enable_if<std::is_constructible< + str_format_internal::FormatRawSinkImpl, T*>::value>::type> + FormatRawSink(T* raw) // NOLINT + : sink_(raw) {} + + private: + friend str_format_internal::FormatRawSinkImpl; + str_format_internal::FormatRawSinkImpl sink_; +}; + +// Format() +// +// Writes a formatted string to an arbitrary sink object (implementing the +// `absl::FormatRawSink` interface), using a format string and zero or more +// additional arguments. +// +// By default, `std::string`, `std::ostream`, and `absl::Cord` are supported as +// destination objects. If a `std::string` is used the formatted string is +// appended to it. +// +// `absl::Format()` is a generic version of `absl::StrAppendFormat()`, for +// custom sinks. The format string, like format strings for `StrFormat()`, is +// checked at compile-time. +// +// On failure, this function returns `false` and the state of the sink is +// unspecified. +template <typename... Args> +bool Format(FormatRawSink raw_sink, const FormatSpec<Args...>& format, + const Args&... args) { + return str_format_internal::FormatUntyped( + str_format_internal::FormatRawSinkImpl::Extract(raw_sink), + str_format_internal::UntypedFormatSpecImpl::Extract(format), + {str_format_internal::FormatArgImpl(args)...}); +} + +// FormatArg +// +// A type-erased handle to a format argument specifically used as an argument to +// `FormatUntyped()`. You may construct `FormatArg` by passing +// reference-to-const of any printable type. `FormatArg` is both copyable and +// assignable. The source data must outlive the `FormatArg` instance. See +// example below. +// +using FormatArg = str_format_internal::FormatArgImpl; + +// FormatUntyped() +// +// Writes a formatted string to an arbitrary sink object (implementing the +// `absl::FormatRawSink` interface), using an `UntypedFormatSpec` and zero or +// more additional arguments. +// +// This function acts as the most generic formatting function in the +// `str_format` library. The caller provides a raw sink, an unchecked format +// string, and (usually) a runtime specified list of arguments; no compile-time +// checking of formatting is performed within this function. As a result, a +// caller should check the return value to verify that no error occurred. +// On failure, this function returns `false` and the state of the sink is +// unspecified. +// +// The arguments are provided in an `absl::Span<const absl::FormatArg>`. +// Each `absl::FormatArg` object binds to a single argument and keeps a +// reference to it. The values used to create the `FormatArg` objects must +// outlive this function call. (See `str_format_arg.h` for information on +// the `FormatArg` class.)_ +// +// Example: +// +// std::optional<std::string> FormatDynamic( +// const std::string& in_format, +// const vector<std::string>& in_args) { +// std::string out; +// std::vector<absl::FormatArg> args; +// for (const auto& v : in_args) { +// // It is important that 'v' is a reference to the objects in in_args. +// // The values we pass to FormatArg must outlive the call to +// // FormatUntyped. +// args.emplace_back(v); +// } +// absl::UntypedFormatSpec format(in_format); +// if (!absl::FormatUntyped(&out, format, args)) { +// return std::nullopt; +// } +// return std::move(out); +// } +// +ABSL_MUST_USE_RESULT inline bool FormatUntyped( + FormatRawSink raw_sink, const UntypedFormatSpec& format, + absl::Span<const FormatArg> args) { + return str_format_internal::FormatUntyped( + str_format_internal::FormatRawSinkImpl::Extract(raw_sink), + str_format_internal::UntypedFormatSpecImpl::Extract(format), args); +} + +//------------------------------------------------------------------------------ +// StrFormat Extensions +//------------------------------------------------------------------------------ +// +// AbslFormatConvert() +// +// The StrFormat library provides a customization API for formatting +// user-defined types using absl::StrFormat(). The API relies on detecting an +// overload in the user-defined type's namespace of a free (non-member) +// `AbslFormatConvert()` function, usually as a friend definition with the +// following signature: +// +// absl::FormatConvertResult<...> AbslFormatConvert( +// const X& value, +// const absl::FormatConversionSpec& spec, +// absl::FormatSink *sink); +// +// An `AbslFormatConvert()` overload for a type should only be declared in the +// same file and namespace as said type. +// +// The abstractions within this definition include: +// +// * An `absl::FormatConversionSpec` to specify the fields to pull from a +// user-defined type's format string +// * An `absl::FormatSink` to hold the converted string data during the +// conversion process. +// * An `absl::FormatConvertResult` to hold the status of the returned +// formatting operation +// +// The return type encodes all the conversion characters that your +// AbslFormatConvert() routine accepts. The return value should be {true}. +// A return value of {false} will result in `StrFormat()` returning +// an empty string. This result will be propagated to the result of +// `FormatUntyped`. +// +// Example: +// +// struct Point { +// // To add formatting support to `Point`, we simply need to add a free +// // (non-member) function `AbslFormatConvert()`. This method interprets +// // `spec` to print in the request format. The allowed conversion characters +// // can be restricted via the type of the result, in this example +// // string and integral formatting are allowed (but not, for instance +// // floating point characters like "%f"). You can add such a free function +// // using a friend declaration within the body of the class: +// friend absl::FormatConvertResult<absl::FormatConversionCharSet::kString | +// absl::FormatConversionCharSet::kIntegral> +// AbslFormatConvert(const Point& p, const absl::FormatConversionSpec& spec, +// absl::FormatSink* s) { +// if (spec.conversion_char() == absl::FormatConversionChar::s) { +// s->Append(absl::StrCat("x=", p.x, " y=", p.y)); +// } else { +// s->Append(absl::StrCat(p.x, ",", p.y)); +// } +// return {true}; +// } +// +// int x; +// int y; +// }; + +// clang-format off + +// FormatConversionChar +// +// Specifies the formatting character provided in the format string +// passed to `StrFormat()`. +enum class FormatConversionChar : uint8_t { + c, s, // text + d, i, o, u, x, X, // int + f, F, e, E, g, G, a, A, // float + n, p // misc +}; +// clang-format on + +// FormatConversionSpec +// +// Specifies modifications to the conversion of the format string, through use +// of one or more format flags in the source format string. +class FormatConversionSpec { + public: + // FormatConversionSpec::is_basic() + // + // Indicates that width and precision are not specified, and no additional + // flags are set for this conversion character in the format string. + bool is_basic() const { return impl_.is_basic(); } + + // FormatConversionSpec::has_left_flag() + // + // Indicates whether the result should be left justified for this conversion + // character in the format string. This flag is set through use of a '-' + // character in the format string. E.g. "%-s" + bool has_left_flag() const { return impl_.has_left_flag(); } + + // FormatConversionSpec::has_show_pos_flag() + // + // Indicates whether a sign column is prepended to the result for this + // conversion character in the format string, even if the result is positive. + // This flag is set through use of a '+' character in the format string. + // E.g. "%+d" + bool has_show_pos_flag() const { return impl_.has_show_pos_flag(); } + + // FormatConversionSpec::has_sign_col_flag() + // + // Indicates whether a mandatory sign column is added to the result for this + // conversion character. This flag is set through use of a space character + // (' ') in the format string. E.g. "% i" + bool has_sign_col_flag() const { return impl_.has_sign_col_flag(); } + + // FormatConversionSpec::has_alt_flag() + // + // Indicates whether an "alternate" format is applied to the result for this + // conversion character. Alternative forms depend on the type of conversion + // character, and unallowed alternatives are undefined. This flag is set + // through use of a '#' character in the format string. E.g. "%#h" + bool has_alt_flag() const { return impl_.has_alt_flag(); } + + // FormatConversionSpec::has_zero_flag() + // + // Indicates whether zeroes should be prepended to the result for this + // conversion character instead of spaces. This flag is set through use of the + // '0' character in the format string. E.g. "%0f" + bool has_zero_flag() const { return impl_.has_zero_flag(); } + + // FormatConversionSpec::conversion_char() + // + // Returns the underlying conversion character. + FormatConversionChar conversion_char() const { + return impl_.conversion_char(); + } + + // FormatConversionSpec::width() + // + // Returns the specified width (indicated through use of a non-zero integer + // value or '*' character) of the conversion character. If width is + // unspecified, it returns a negative value. + int width() const { return impl_.width(); } + + // FormatConversionSpec::precision() + // + // Returns the specified precision (through use of the '.' character followed + // by a non-zero integer value or '*' character) of the conversion character. + // If precision is unspecified, it returns a negative value. + int precision() const { return impl_.precision(); } + + private: + explicit FormatConversionSpec( + str_format_internal::FormatConversionSpecImpl impl) + : impl_(impl) {} + + friend str_format_internal::FormatConversionSpecImpl; + + absl::str_format_internal::FormatConversionSpecImpl impl_; +}; + +// Type safe OR operator for FormatConversionCharSet to allow accepting multiple +// conversion chars in custom format converters. +constexpr FormatConversionCharSet operator|(FormatConversionCharSet a, + FormatConversionCharSet b) { + return static_cast<FormatConversionCharSet>(static_cast<uint64_t>(a) | + static_cast<uint64_t>(b)); +} + +// FormatConversionCharSet +// +// Specifies the _accepted_ conversion types as a template parameter to +// FormatConvertResult for custom implementations of `AbslFormatConvert`. +// Note the helper predefined alias definitions (kIntegral, etc.) below. +enum class FormatConversionCharSet : uint64_t { + // text + c = str_format_internal::FormatConversionCharToConvInt('c'), + s = str_format_internal::FormatConversionCharToConvInt('s'), + // integer + d = str_format_internal::FormatConversionCharToConvInt('d'), + i = str_format_internal::FormatConversionCharToConvInt('i'), + o = str_format_internal::FormatConversionCharToConvInt('o'), + u = str_format_internal::FormatConversionCharToConvInt('u'), + x = str_format_internal::FormatConversionCharToConvInt('x'), + X = str_format_internal::FormatConversionCharToConvInt('X'), + // Float + f = str_format_internal::FormatConversionCharToConvInt('f'), + F = str_format_internal::FormatConversionCharToConvInt('F'), + e = str_format_internal::FormatConversionCharToConvInt('e'), + E = str_format_internal::FormatConversionCharToConvInt('E'), + g = str_format_internal::FormatConversionCharToConvInt('g'), + G = str_format_internal::FormatConversionCharToConvInt('G'), + a = str_format_internal::FormatConversionCharToConvInt('a'), + A = str_format_internal::FormatConversionCharToConvInt('A'), + // misc + n = str_format_internal::FormatConversionCharToConvInt('n'), + p = str_format_internal::FormatConversionCharToConvInt('p'), + + // Used for width/precision '*' specification. + kStar = static_cast<uint64_t>( + absl::str_format_internal::FormatConversionCharSetInternal::kStar), + // Some predefined values: + kIntegral = d | i | u | o | x | X, + kFloating = a | e | f | g | A | E | F | G, + kNumeric = kIntegral | kFloating, + kString = s, + kPointer = p, +}; + +// FormatSink +// +// An abstraction to which conversions write their string data. +// +class FormatSink { + public: + // Appends `count` copies of `ch`. + void Append(size_t count, char ch) { sink_->Append(count, ch); } + + void Append(string_view v) { sink_->Append(v); } + + // Appends the first `precision` bytes of `v`. If this is less than + // `width`, spaces will be appended first (if `left` is false), or + // after (if `left` is true) to ensure the total amount appended is + // at least `width`. + bool PutPaddedString(string_view v, int width, int precision, bool left) { + return sink_->PutPaddedString(v, width, precision, left); + } + + private: + friend str_format_internal::FormatSinkImpl; + explicit FormatSink(str_format_internal::FormatSinkImpl* s) : sink_(s) {} + str_format_internal::FormatSinkImpl* sink_; +}; + +// FormatConvertResult +// +// Indicates whether a call to AbslFormatConvert() was successful. +// This return type informs the StrFormat extension framework (through +// ADL but using the return type) of what conversion characters are supported. +// It is strongly discouraged to return {false}, as this will result in an +// empty string in StrFormat. +template <FormatConversionCharSet C> +struct FormatConvertResult { + bool value; +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STR_FORMAT_H_ diff --git a/third_party/abseil_cpp/absl/strings/str_format_test.cc b/third_party/abseil_cpp/absl/strings/str_format_test.cc new file mode 100644 index 000000000000..c60027ad297d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_format_test.cc @@ -0,0 +1,774 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_format.h" + +#include <cstdarg> +#include <cstdint> +#include <cstdio> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/cord.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { +using str_format_internal::FormatArgImpl; + +using FormatEntryPointTest = ::testing::Test; + +TEST_F(FormatEntryPointTest, Format) { + std::string sink; + EXPECT_TRUE(Format(&sink, "A format %d", 123)); + EXPECT_EQ("A format 123", sink); + sink.clear(); + + ParsedFormat<'d'> pc("A format %d"); + EXPECT_TRUE(Format(&sink, pc, 123)); + EXPECT_EQ("A format 123", sink); +} +TEST_F(FormatEntryPointTest, UntypedFormat) { + constexpr const char* formats[] = { + "", + "a", + "%80d", +#if !defined(_MSC_VER) && !defined(__ANDROID__) && !defined(__native_client__) + // MSVC, NaCL and Android don't support positional syntax. + "complicated multipart %% %1$d format %1$0999d", +#endif // _MSC_VER + }; + for (const char* fmt : formats) { + std::string actual; + int i = 123; + FormatArgImpl arg_123(i); + absl::Span<const FormatArgImpl> args(&arg_123, 1); + UntypedFormatSpec format(fmt); + + EXPECT_TRUE(FormatUntyped(&actual, format, args)); + char buf[4096]{}; + snprintf(buf, sizeof(buf), fmt, 123); + EXPECT_EQ( + str_format_internal::FormatPack( + str_format_internal::UntypedFormatSpecImpl::Extract(format), args), + buf); + EXPECT_EQ(actual, buf); + } + // The internal version works with a preparsed format. + ParsedFormat<'d'> pc("A format %d"); + int i = 345; + FormatArg arg(i); + std::string out; + EXPECT_TRUE(str_format_internal::FormatUntyped( + &out, str_format_internal::UntypedFormatSpecImpl(&pc), {&arg, 1})); + EXPECT_EQ("A format 345", out); +} + +TEST_F(FormatEntryPointTest, StringFormat) { + EXPECT_EQ("123", StrFormat("%d", 123)); + constexpr absl::string_view view("=%d=", 4); + EXPECT_EQ("=123=", StrFormat(view, 123)); +} + +TEST_F(FormatEntryPointTest, AppendFormat) { + std::string s; + std::string& r = StrAppendFormat(&s, "%d", 123); + EXPECT_EQ(&s, &r); // should be same object + EXPECT_EQ("123", r); +} + +TEST_F(FormatEntryPointTest, AppendFormatFail) { + std::string s = "orig"; + + UntypedFormatSpec format(" more %d"); + FormatArgImpl arg("not an int"); + + EXPECT_EQ("orig", + str_format_internal::AppendPack( + &s, str_format_internal::UntypedFormatSpecImpl::Extract(format), + {&arg, 1})); +} + + +TEST_F(FormatEntryPointTest, ManyArgs) { + EXPECT_EQ("24", StrFormat("%24$d", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24)); + EXPECT_EQ("60", StrFormat("%60$d", 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + 53, 54, 55, 56, 57, 58, 59, 60)); +} + +TEST_F(FormatEntryPointTest, Preparsed) { + ParsedFormat<'d'> pc("%d"); + EXPECT_EQ("123", StrFormat(pc, 123)); + // rvalue ok? + EXPECT_EQ("123", StrFormat(ParsedFormat<'d'>("%d"), 123)); + constexpr absl::string_view view("=%d=", 4); + EXPECT_EQ("=123=", StrFormat(ParsedFormat<'d'>(view), 123)); +} + +TEST_F(FormatEntryPointTest, FormatCountCapture) { + int n = 0; + EXPECT_EQ("", StrFormat("%n", FormatCountCapture(&n))); + EXPECT_EQ(0, n); + EXPECT_EQ("123", StrFormat("%d%n", 123, FormatCountCapture(&n))); + EXPECT_EQ(3, n); +} + +TEST_F(FormatEntryPointTest, FormatCountCaptureWrongType) { + // Should reject int*. + int n = 0; + UntypedFormatSpec format("%d%n"); + int i = 123, *ip = &n; + FormatArgImpl args[2] = {FormatArgImpl(i), FormatArgImpl(ip)}; + + EXPECT_EQ("", str_format_internal::FormatPack( + str_format_internal::UntypedFormatSpecImpl::Extract(format), + absl::MakeSpan(args))); +} + +TEST_F(FormatEntryPointTest, FormatCountCaptureMultiple) { + int n1 = 0; + int n2 = 0; + EXPECT_EQ(" 1 2", + StrFormat("%5d%n%10d%n", 1, FormatCountCapture(&n1), 2, + FormatCountCapture(&n2))); + EXPECT_EQ(5, n1); + EXPECT_EQ(15, n2); +} + +TEST_F(FormatEntryPointTest, FormatCountCaptureExample) { + int n; + std::string s; + StrAppendFormat(&s, "%s: %n%s\n", "(1,1)", FormatCountCapture(&n), "(1,2)"); + StrAppendFormat(&s, "%*s%s\n", n, "", "(2,2)"); + EXPECT_EQ(7, n); + EXPECT_EQ( + "(1,1): (1,2)\n" + " (2,2)\n", + s); +} + +TEST_F(FormatEntryPointTest, Stream) { + const std::string formats[] = { + "", + "a", + "%80d", + "%d %u %c %s %f %g", +#if !defined(_MSC_VER) && !defined(__ANDROID__) && !defined(__native_client__) + // MSVC, NaCL and Android don't support positional syntax. + "complicated multipart %% %1$d format %1$080d", +#endif // _MSC_VER + }; + std::string buf(4096, '\0'); + for (const auto& fmt : formats) { + const auto parsed = + ParsedFormat<'d', 'u', 'c', 's', 'f', 'g'>::NewAllowIgnored(fmt); + std::ostringstream oss; + oss << StreamFormat(*parsed, 123, 3, 49, "multistreaming!!!", 1.01, 1.01); + int fmt_result = snprintf(&*buf.begin(), buf.size(), fmt.c_str(), // + 123, 3, 49, "multistreaming!!!", 1.01, 1.01); + ASSERT_TRUE(oss) << fmt; + ASSERT_TRUE(fmt_result >= 0 && static_cast<size_t>(fmt_result) < buf.size()) + << fmt_result; + EXPECT_EQ(buf.c_str(), oss.str()); + } +} + +TEST_F(FormatEntryPointTest, StreamOk) { + std::ostringstream oss; + oss << StreamFormat("hello %d", 123); + EXPECT_EQ("hello 123", oss.str()); + EXPECT_TRUE(oss.good()); +} + +TEST_F(FormatEntryPointTest, StreamFail) { + std::ostringstream oss; + UntypedFormatSpec format("hello %d"); + FormatArgImpl arg("non-numeric"); + oss << str_format_internal::Streamable( + str_format_internal::UntypedFormatSpecImpl::Extract(format), {&arg, 1}); + EXPECT_EQ("hello ", oss.str()); // partial write + EXPECT_TRUE(oss.fail()); +} + +std::string WithSnprintf(const char* fmt, ...) { + std::string buf; + buf.resize(128); + va_list va; + va_start(va, fmt); + int r = vsnprintf(&*buf.begin(), buf.size(), fmt, va); + va_end(va); + EXPECT_GE(r, 0); + EXPECT_LT(r, buf.size()); + buf.resize(r); + return buf; +} + +TEST_F(FormatEntryPointTest, FloatPrecisionArg) { + // Test that positional parameters for width and precision + // are indexed to precede the value. + // Also sanity check the same formats against snprintf. + EXPECT_EQ("0.1", StrFormat("%.1f", 0.1)); + EXPECT_EQ("0.1", WithSnprintf("%.1f", 0.1)); + EXPECT_EQ(" 0.1", StrFormat("%*.1f", 5, 0.1)); + EXPECT_EQ(" 0.1", WithSnprintf("%*.1f", 5, 0.1)); + EXPECT_EQ("0.1", StrFormat("%.*f", 1, 0.1)); + EXPECT_EQ("0.1", WithSnprintf("%.*f", 1, 0.1)); + EXPECT_EQ(" 0.1", StrFormat("%*.*f", 5, 1, 0.1)); + EXPECT_EQ(" 0.1", WithSnprintf("%*.*f", 5, 1, 0.1)); +} +namespace streamed_test { +struct X {}; +std::ostream& operator<<(std::ostream& os, const X&) { + return os << "X"; +} +} // streamed_test + +TEST_F(FormatEntryPointTest, FormatStreamed) { + EXPECT_EQ("123", StrFormat("%s", FormatStreamed(123))); + EXPECT_EQ(" 123", StrFormat("%5s", FormatStreamed(123))); + EXPECT_EQ("123 ", StrFormat("%-5s", FormatStreamed(123))); + EXPECT_EQ("X", StrFormat("%s", FormatStreamed(streamed_test::X()))); + EXPECT_EQ("123", StrFormat("%s", FormatStreamed(StreamFormat("%d", 123)))); +} + +// Helper class that creates a temporary file and exposes a FILE* to it. +// It will close the file on destruction. +class TempFile { + public: + TempFile() : file_(std::tmpfile()) {} + ~TempFile() { std::fclose(file_); } + + std::FILE* file() const { return file_; } + + // Read the file into a string. + std::string ReadFile() { + std::fseek(file_, 0, SEEK_END); + int size = std::ftell(file_); + EXPECT_GT(size, 0); + std::rewind(file_); + std::string str(2 * size, ' '); + int read_bytes = std::fread(&str[0], 1, str.size(), file_); + EXPECT_EQ(read_bytes, size); + str.resize(read_bytes); + EXPECT_TRUE(std::feof(file_)); + return str; + } + + private: + std::FILE* file_; +}; + +TEST_F(FormatEntryPointTest, FPrintF) { + TempFile tmp; + int result = + FPrintF(tmp.file(), "STRING: %s NUMBER: %010d", std::string("ABC"), -19); + EXPECT_EQ(result, 30); + EXPECT_EQ(tmp.ReadFile(), "STRING: ABC NUMBER: -000000019"); +} + +TEST_F(FormatEntryPointTest, FPrintFError) { + errno = 0; + int result = FPrintF(stdin, "ABC"); + EXPECT_LT(result, 0); + EXPECT_EQ(errno, EBADF); +} + +#ifdef __GLIBC__ +TEST_F(FormatEntryPointTest, FprintfTooLarge) { + std::FILE* f = std::fopen("/dev/null", "w"); + int width = 2000000000; + errno = 0; + int result = FPrintF(f, "%*d %*d", width, 0, width, 0); + EXPECT_LT(result, 0); + EXPECT_EQ(errno, EFBIG); + std::fclose(f); +} + +TEST_F(FormatEntryPointTest, PrintF) { + int stdout_tmp = dup(STDOUT_FILENO); + + TempFile tmp; + std::fflush(stdout); + dup2(fileno(tmp.file()), STDOUT_FILENO); + + int result = PrintF("STRING: %s NUMBER: %010d", std::string("ABC"), -19); + + std::fflush(stdout); + dup2(stdout_tmp, STDOUT_FILENO); + close(stdout_tmp); + + EXPECT_EQ(result, 30); + EXPECT_EQ(tmp.ReadFile(), "STRING: ABC NUMBER: -000000019"); +} +#endif // __GLIBC__ + +TEST_F(FormatEntryPointTest, SNPrintF) { + char buffer[16]; + int result = + SNPrintF(buffer, sizeof(buffer), "STRING: %s", std::string("ABC")); + EXPECT_EQ(result, 11); + EXPECT_EQ(std::string(buffer), "STRING: ABC"); + + result = SNPrintF(buffer, sizeof(buffer), "NUMBER: %d", 123456); + EXPECT_EQ(result, 14); + EXPECT_EQ(std::string(buffer), "NUMBER: 123456"); + + result = SNPrintF(buffer, sizeof(buffer), "NUMBER: %d", 1234567); + EXPECT_EQ(result, 15); + EXPECT_EQ(std::string(buffer), "NUMBER: 1234567"); + + result = SNPrintF(buffer, sizeof(buffer), "NUMBER: %d", 12345678); + EXPECT_EQ(result, 16); + EXPECT_EQ(std::string(buffer), "NUMBER: 1234567"); + + result = SNPrintF(buffer, sizeof(buffer), "NUMBER: %d", 123456789); + EXPECT_EQ(result, 17); + EXPECT_EQ(std::string(buffer), "NUMBER: 1234567"); + + result = SNPrintF(nullptr, 0, "Just checking the %s of the output.", "size"); + EXPECT_EQ(result, 37); +} + +TEST(StrFormat, BehavesAsDocumented) { + std::string s = absl::StrFormat("%s, %d!", "Hello", 123); + EXPECT_EQ("Hello, 123!", s); + // The format of a replacement is + // '%'[position][flags][width['.'precision]][length_modifier][format] + EXPECT_EQ(absl::StrFormat("%1$+3.2Lf", 1.1), "+1.10"); + // Text conversion: + // "c" - Character. Eg: 'a' -> "A", 20 -> " " + EXPECT_EQ(StrFormat("%c", 'a'), "a"); + EXPECT_EQ(StrFormat("%c", 0x20), " "); + // Formats char and integral types: int, long, uint64_t, etc. + EXPECT_EQ(StrFormat("%c", int{'a'}), "a"); + EXPECT_EQ(StrFormat("%c", long{'a'}), "a"); // NOLINT + EXPECT_EQ(StrFormat("%c", uint64_t{'a'}), "a"); + // "s" - string Eg: "C" -> "C", std::string("C++") -> "C++" + // Formats std::string, char*, string_view, and Cord. + EXPECT_EQ(StrFormat("%s", "C"), "C"); + EXPECT_EQ(StrFormat("%s", std::string("C++")), "C++"); + EXPECT_EQ(StrFormat("%s", string_view("view")), "view"); + EXPECT_EQ(StrFormat("%s", absl::Cord("cord")), "cord"); + // Integral Conversion + // These format integral types: char, int, long, uint64_t, etc. + EXPECT_EQ(StrFormat("%d", char{10}), "10"); + EXPECT_EQ(StrFormat("%d", int{10}), "10"); + EXPECT_EQ(StrFormat("%d", long{10}), "10"); // NOLINT + EXPECT_EQ(StrFormat("%d", uint64_t{10}), "10"); + // d,i - signed decimal Eg: -10 -> "-10" + EXPECT_EQ(StrFormat("%d", -10), "-10"); + EXPECT_EQ(StrFormat("%i", -10), "-10"); + // o - octal Eg: 10 -> "12" + EXPECT_EQ(StrFormat("%o", 10), "12"); + // u - unsigned decimal Eg: 10 -> "10" + EXPECT_EQ(StrFormat("%u", 10), "10"); + // x/X - lower,upper case hex Eg: 10 -> "a"/"A" + EXPECT_EQ(StrFormat("%x", 10), "a"); + EXPECT_EQ(StrFormat("%X", 10), "A"); + // Floating-point, with upper/lower-case output. + // These format floating points types: float, double, long double, etc. + EXPECT_EQ(StrFormat("%.1f", float{1}), "1.0"); + EXPECT_EQ(StrFormat("%.1f", double{1}), "1.0"); + const long double long_double = 1.0; + EXPECT_EQ(StrFormat("%.1f", long_double), "1.0"); + // These also format integral types: char, int, long, uint64_t, etc.: + EXPECT_EQ(StrFormat("%.1f", char{1}), "1.0"); + EXPECT_EQ(StrFormat("%.1f", int{1}), "1.0"); + EXPECT_EQ(StrFormat("%.1f", long{1}), "1.0"); // NOLINT + EXPECT_EQ(StrFormat("%.1f", uint64_t{1}), "1.0"); + // f/F - decimal. Eg: 123456789 -> "123456789.000000" + EXPECT_EQ(StrFormat("%f", 123456789), "123456789.000000"); + EXPECT_EQ(StrFormat("%F", 123456789), "123456789.000000"); + // e/E - exponentiated Eg: .01 -> "1.00000e-2"/"1.00000E-2" + EXPECT_EQ(StrFormat("%e", .01), "1.000000e-02"); + EXPECT_EQ(StrFormat("%E", .01), "1.000000E-02"); + // g/G - exponentiate to fit Eg: .01 -> "0.01", 1e10 ->"1e+10"/"1E+10" + EXPECT_EQ(StrFormat("%g", .01), "0.01"); + EXPECT_EQ(StrFormat("%g", 1e10), "1e+10"); + EXPECT_EQ(StrFormat("%G", 1e10), "1E+10"); + // a/A - lower,upper case hex Eg: -3.0 -> "-0x1.8p+1"/"-0X1.8P+1" + +// On Android platform <=21, there is a regression in hexfloat formatting. +#if !defined(__ANDROID_API__) || __ANDROID_API__ > 21 + EXPECT_EQ(StrFormat("%.1a", -3.0), "-0x1.8p+1"); // .1 to fix MSVC output + EXPECT_EQ(StrFormat("%.1A", -3.0), "-0X1.8P+1"); // .1 to fix MSVC output +#endif + + // Other conversion + int64_t value = 0x7ffdeb4; + auto ptr_value = static_cast<uintptr_t>(value); + const int& something = *reinterpret_cast<const int*>(ptr_value); + EXPECT_EQ(StrFormat("%p", &something), StrFormat("0x%x", ptr_value)); + + // Output widths are supported, with optional flags. + EXPECT_EQ(StrFormat("%3d", 1), " 1"); + EXPECT_EQ(StrFormat("%3d", 123456), "123456"); + EXPECT_EQ(StrFormat("%06.2f", 1.234), "001.23"); + EXPECT_EQ(StrFormat("%+d", 1), "+1"); + EXPECT_EQ(StrFormat("% d", 1), " 1"); + EXPECT_EQ(StrFormat("%-4d", -1), "-1 "); + EXPECT_EQ(StrFormat("%#o", 10), "012"); + EXPECT_EQ(StrFormat("%#x", 15), "0xf"); + EXPECT_EQ(StrFormat("%04d", 8), "0008"); + // Posix positional substitution. + EXPECT_EQ(absl::StrFormat("%2$s, %3$s, %1$s!", "vici", "veni", "vidi"), + "veni, vidi, vici!"); + // Length modifiers are ignored. + EXPECT_EQ(StrFormat("%hhd", int{1}), "1"); + EXPECT_EQ(StrFormat("%hd", int{1}), "1"); + EXPECT_EQ(StrFormat("%ld", int{1}), "1"); + EXPECT_EQ(StrFormat("%lld", int{1}), "1"); + EXPECT_EQ(StrFormat("%Ld", int{1}), "1"); + EXPECT_EQ(StrFormat("%jd", int{1}), "1"); + EXPECT_EQ(StrFormat("%zd", int{1}), "1"); + EXPECT_EQ(StrFormat("%td", int{1}), "1"); + EXPECT_EQ(StrFormat("%qd", int{1}), "1"); +} + +using str_format_internal::ExtendedParsedFormat; +using str_format_internal::ParsedFormatBase; + +struct SummarizeConsumer { + std::string* out; + explicit SummarizeConsumer(std::string* out) : out(out) {} + + bool Append(string_view s) { + *out += "[" + std::string(s) + "]"; + return true; + } + + bool ConvertOne(const str_format_internal::UnboundConversion& conv, + string_view s) { + *out += "{"; + *out += std::string(s); + *out += ":"; + *out += std::to_string(conv.arg_position) + "$"; + if (conv.width.is_from_arg()) { + *out += std::to_string(conv.width.get_from_arg()) + "$*"; + } + if (conv.precision.is_from_arg()) { + *out += "." + std::to_string(conv.precision.get_from_arg()) + "$*"; + } + *out += str_format_internal::FormatConversionCharToChar(conv.conv); + *out += "}"; + return true; + } +}; + +std::string SummarizeParsedFormat(const ParsedFormatBase& pc) { + std::string out; + if (!pc.ProcessFormat(SummarizeConsumer(&out))) out += "!"; + return out; +} + +using ParsedFormatTest = ::testing::Test; + +TEST_F(ParsedFormatTest, SimpleChecked) { + EXPECT_EQ("[ABC]{d:1$d}[DEF]", + SummarizeParsedFormat(ParsedFormat<'d'>("ABC%dDEF"))); + EXPECT_EQ("{s:1$s}[FFF]{d:2$d}[ZZZ]{f:3$f}", + SummarizeParsedFormat(ParsedFormat<'s', 'd', 'f'>("%sFFF%dZZZ%f"))); + EXPECT_EQ("{s:1$s}[ ]{.*d:3$.2$*d}", + SummarizeParsedFormat(ParsedFormat<'s', '*', 'd'>("%s %.*d"))); +} + +TEST_F(ParsedFormatTest, SimpleUncheckedCorrect) { + auto f = ParsedFormat<'d'>::New("ABC%dDEF"); + ASSERT_TRUE(f); + EXPECT_EQ("[ABC]{d:1$d}[DEF]", SummarizeParsedFormat(*f)); + + std::string format = "%sFFF%dZZZ%f"; + auto f2 = ParsedFormat<'s', 'd', 'f'>::New(format); + + ASSERT_TRUE(f2); + EXPECT_EQ("{s:1$s}[FFF]{d:2$d}[ZZZ]{f:3$f}", SummarizeParsedFormat(*f2)); + + f2 = ParsedFormat<'s', 'd', 'f'>::New("%s %d %f"); + + ASSERT_TRUE(f2); + EXPECT_EQ("{s:1$s}[ ]{d:2$d}[ ]{f:3$f}", SummarizeParsedFormat(*f2)); + + auto star = ParsedFormat<'*', 'd'>::New("%*d"); + ASSERT_TRUE(star); + EXPECT_EQ("{*d:2$1$*d}", SummarizeParsedFormat(*star)); + + auto dollar = ParsedFormat<'d', 's'>::New("%2$s %1$d"); + ASSERT_TRUE(dollar); + EXPECT_EQ("{2$s:2$s}[ ]{1$d:1$d}", SummarizeParsedFormat(*dollar)); + // with reuse + dollar = ParsedFormat<'d', 's'>::New("%2$s %1$d %1$d"); + ASSERT_TRUE(dollar); + EXPECT_EQ("{2$s:2$s}[ ]{1$d:1$d}[ ]{1$d:1$d}", + SummarizeParsedFormat(*dollar)); +} + +TEST_F(ParsedFormatTest, SimpleUncheckedIgnoredArgs) { + EXPECT_FALSE((ParsedFormat<'d', 's'>::New("ABC"))); + EXPECT_FALSE((ParsedFormat<'d', 's'>::New("%dABC"))); + EXPECT_FALSE((ParsedFormat<'d', 's'>::New("ABC%2$s"))); + auto f = ParsedFormat<'d', 's'>::NewAllowIgnored("ABC"); + ASSERT_TRUE(f); + EXPECT_EQ("[ABC]", SummarizeParsedFormat(*f)); + f = ParsedFormat<'d', 's'>::NewAllowIgnored("%dABC"); + ASSERT_TRUE(f); + EXPECT_EQ("{d:1$d}[ABC]", SummarizeParsedFormat(*f)); + f = ParsedFormat<'d', 's'>::NewAllowIgnored("ABC%2$s"); + ASSERT_TRUE(f); + EXPECT_EQ("[ABC]{2$s:2$s}", SummarizeParsedFormat(*f)); +} + +TEST_F(ParsedFormatTest, SimpleUncheckedUnsupported) { + EXPECT_FALSE(ParsedFormat<'d'>::New("%1$d %1$x")); + EXPECT_FALSE(ParsedFormat<'x'>::New("%1$d %1$x")); +} + +TEST_F(ParsedFormatTest, SimpleUncheckedIncorrect) { + EXPECT_FALSE(ParsedFormat<'d'>::New("")); + + EXPECT_FALSE(ParsedFormat<'d'>::New("ABC%dDEF%d")); + + std::string format = "%sFFF%dZZZ%f"; + EXPECT_FALSE((ParsedFormat<'s', 'd', 'g'>::New(format))); +} + +#if defined(__cpp_nontype_template_parameter_auto) + +template <auto T> +std::true_type IsValidParsedFormatArgTest(ParsedFormat<T>*); + +template <auto T> +std::false_type IsValidParsedFormatArgTest(...); + +template <auto T> +using IsValidParsedFormatArg = decltype(IsValidParsedFormatArgTest<T>(nullptr)); + +TEST_F(ParsedFormatTest, OnlyValidTypesAllowed) { + ASSERT_TRUE(IsValidParsedFormatArg<'c'>::value); + + ASSERT_TRUE(IsValidParsedFormatArg<FormatConversionCharSet::d>::value); + + ASSERT_TRUE(IsValidParsedFormatArg<absl::FormatConversionCharSet::d | + absl::FormatConversionCharSet::x>::value); + ASSERT_TRUE( + IsValidParsedFormatArg<absl::FormatConversionCharSet::kIntegral>::value); + + // This is an easy mistake to make, however, this will reduce to an integer + // which has no meaning, so we need to ensure it doesn't compile. + ASSERT_FALSE(IsValidParsedFormatArg<'x' | 'd'>::value); + + // For now, we disallow construction based on ConversionChar (rather than + // CharSet) + ASSERT_FALSE(IsValidParsedFormatArg<absl::FormatConversionChar::d>::value); +} + +TEST_F(ParsedFormatTest, ExtendedTyping) { + EXPECT_FALSE(ParsedFormat<FormatConversionCharSet::d>::New("")); + ASSERT_TRUE(ParsedFormat<absl::FormatConversionCharSet::d>::New("%d")); + auto v1 = ParsedFormat<'d', absl::FormatConversionCharSet::s>::New("%d%s"); + ASSERT_TRUE(v1); + auto v2 = ParsedFormat<absl::FormatConversionCharSet::d, 's'>::New("%d%s"); + ASSERT_TRUE(v2); + auto v3 = ParsedFormat<absl::FormatConversionCharSet::d | + absl::FormatConversionCharSet::s, + 's'>::New("%d%s"); + ASSERT_TRUE(v3); + auto v4 = ParsedFormat<absl::FormatConversionCharSet::d | + absl::FormatConversionCharSet::s, + 's'>::New("%s%s"); + ASSERT_TRUE(v4); +} +#endif + +TEST_F(ParsedFormatTest, UncheckedCorrect) { + auto f = + ExtendedParsedFormat<absl::FormatConversionCharSet::d>::New("ABC%dDEF"); + ASSERT_TRUE(f); + EXPECT_EQ("[ABC]{d:1$d}[DEF]", SummarizeParsedFormat(*f)); + + std::string format = "%sFFF%dZZZ%f"; + auto f2 = ExtendedParsedFormat< + absl::FormatConversionCharSet::kString, absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::kFloating>::New(format); + + ASSERT_TRUE(f2); + EXPECT_EQ("{s:1$s}[FFF]{d:2$d}[ZZZ]{f:3$f}", SummarizeParsedFormat(*f2)); + + f2 = ExtendedParsedFormat< + absl::FormatConversionCharSet::kString, absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::kFloating>::New("%s %d %f"); + + ASSERT_TRUE(f2); + EXPECT_EQ("{s:1$s}[ ]{d:2$d}[ ]{f:3$f}", SummarizeParsedFormat(*f2)); + + auto star = + ExtendedParsedFormat<absl::FormatConversionCharSet::kStar, + absl::FormatConversionCharSet::d>::New("%*d"); + ASSERT_TRUE(star); + EXPECT_EQ("{*d:2$1$*d}", SummarizeParsedFormat(*star)); + + auto dollar = + ExtendedParsedFormat<absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::New("%2$s %1$d"); + ASSERT_TRUE(dollar); + EXPECT_EQ("{2$s:2$s}[ ]{1$d:1$d}", SummarizeParsedFormat(*dollar)); + // with reuse + dollar = ExtendedParsedFormat< + absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::New("%2$s %1$d %1$d"); + ASSERT_TRUE(dollar); + EXPECT_EQ("{2$s:2$s}[ ]{1$d:1$d}[ ]{1$d:1$d}", + SummarizeParsedFormat(*dollar)); +} + +TEST_F(ParsedFormatTest, UncheckedIgnoredArgs) { + EXPECT_FALSE( + (ExtendedParsedFormat<absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::New("ABC"))); + EXPECT_FALSE( + (ExtendedParsedFormat<absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::New("%dABC"))); + EXPECT_FALSE( + (ExtendedParsedFormat<absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::New("ABC%2$s"))); + auto f = ExtendedParsedFormat< + absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::NewAllowIgnored("ABC"); + ASSERT_TRUE(f); + EXPECT_EQ("[ABC]", SummarizeParsedFormat(*f)); + f = ExtendedParsedFormat< + absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::NewAllowIgnored("%dABC"); + ASSERT_TRUE(f); + EXPECT_EQ("{d:1$d}[ABC]", SummarizeParsedFormat(*f)); + f = ExtendedParsedFormat< + absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::s>::NewAllowIgnored("ABC%2$s"); + ASSERT_TRUE(f); + EXPECT_EQ("[ABC]{2$s:2$s}", SummarizeParsedFormat(*f)); +} + +TEST_F(ParsedFormatTest, UncheckedMultipleTypes) { + auto dx = + ExtendedParsedFormat<absl::FormatConversionCharSet::d | + absl::FormatConversionCharSet::x>::New("%1$d %1$x"); + EXPECT_TRUE(dx); + EXPECT_EQ("{1$d:1$d}[ ]{1$x:1$x}", SummarizeParsedFormat(*dx)); + + dx = ExtendedParsedFormat<absl::FormatConversionCharSet::d | + absl::FormatConversionCharSet::x>::New("%1$d"); + EXPECT_TRUE(dx); + EXPECT_EQ("{1$d:1$d}", SummarizeParsedFormat(*dx)); +} + +TEST_F(ParsedFormatTest, UncheckedIncorrect) { + EXPECT_FALSE(ExtendedParsedFormat<absl::FormatConversionCharSet::d>::New("")); + + EXPECT_FALSE(ExtendedParsedFormat<absl::FormatConversionCharSet::d>::New( + "ABC%dDEF%d")); + + std::string format = "%sFFF%dZZZ%f"; + EXPECT_FALSE( + (ExtendedParsedFormat<absl::FormatConversionCharSet::s, + absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::g>::New(format))); +} + +TEST_F(ParsedFormatTest, RegressionMixPositional) { + EXPECT_FALSE( + (ExtendedParsedFormat<absl::FormatConversionCharSet::d, + absl::FormatConversionCharSet::o>::New("%1$d %o"))); +} + +using FormatWrapperTest = ::testing::Test; + +// Plain wrapper for StrFormat. +template <typename... Args> +std::string WrappedFormat(const absl::FormatSpec<Args...>& format, + const Args&... args) { + return StrFormat(format, args...); +} + +TEST_F(FormatWrapperTest, ConstexprStringFormat) { + EXPECT_EQ(WrappedFormat("%s there", "hello"), "hello there"); +} + +TEST_F(FormatWrapperTest, ParsedFormat) { + ParsedFormat<'s'> format("%s there"); + EXPECT_EQ(WrappedFormat(format, "hello"), "hello there"); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl + +using FormatExtensionTest = ::testing::Test; + +struct Point { + friend absl::FormatConvertResult<absl::FormatConversionCharSet::kString | + absl::FormatConversionCharSet::kIntegral> + AbslFormatConvert(const Point& p, const absl::FormatConversionSpec& spec, + absl::FormatSink* s) { + if (spec.conversion_char() == absl::FormatConversionChar::s) { + s->Append(absl::StrCat("x=", p.x, " y=", p.y)); + } else { + s->Append(absl::StrCat(p.x, ",", p.y)); + } + return {true}; + } + + int x = 10; + int y = 20; +}; + +TEST_F(FormatExtensionTest, AbslFormatConvertExample) { + Point p; + EXPECT_EQ(absl::StrFormat("a %s z", p), "a x=10 y=20 z"); + EXPECT_EQ(absl::StrFormat("a %d z", p), "a 10,20 z"); + + // Typed formatting will fail to compile an invalid format. + // StrFormat("%f", p); // Does not compile. + std::string actual; + absl::UntypedFormatSpec f1("%f"); + // FormatUntyped will return false for bad character. + EXPECT_FALSE(absl::FormatUntyped(&actual, f1, {absl::FormatArg(p)})); +} + +// Some codegen thunks that we can use to easily dump the generated assembly for +// different StrFormat calls. + +std::string CodegenAbslStrFormatInt(int i) { // NOLINT + return absl::StrFormat("%d", i); +} + +std::string CodegenAbslStrFormatIntStringInt64(int i, const std::string& s, + int64_t i64) { // NOLINT + return absl::StrFormat("%d %s %d", i, s, i64); +} + +void CodegenAbslStrAppendFormatInt(std::string* out, int i) { // NOLINT + absl::StrAppendFormat(out, "%d", i); +} + +void CodegenAbslStrAppendFormatIntStringInt64(std::string* out, int i, + const std::string& s, + int64_t i64) { // NOLINT + absl::StrAppendFormat(out, "%d %s %d", i, s, i64); +} diff --git a/third_party/abseil_cpp/absl/strings/str_join.h b/third_party/abseil_cpp/absl/strings/str_join.h new file mode 100644 index 000000000000..ae5731a42b3d --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_join.h @@ -0,0 +1,293 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: str_join.h +// ----------------------------------------------------------------------------- +// +// This header file contains functions for joining a range of elements and +// returning the result as a std::string. StrJoin operations are specified by +// passing a range, a separator string to use between the elements joined, and +// an optional Formatter responsible for converting each argument in the range +// to a string. If omitted, a default `AlphaNumFormatter()` is called on the +// elements to be joined, using the same formatting that `absl::StrCat()` uses. +// This package defines a number of default formatters, and you can define your +// own implementations. +// +// Ranges are specified by passing a container with `std::begin()` and +// `std::end()` iterators, container-specific `begin()` and `end()` iterators, a +// brace-initialized `std::initializer_list`, or a `std::tuple` of heterogeneous +// objects. The separator string is specified as an `absl::string_view`. +// +// Because the default formatter uses the `absl::AlphaNum` class, +// `absl::StrJoin()`, like `absl::StrCat()`, will work out-of-the-box on +// collections of strings, ints, floats, doubles, etc. +// +// Example: +// +// std::vector<std::string> v = {"foo", "bar", "baz"}; +// std::string s = absl::StrJoin(v, "-"); +// EXPECT_EQ("foo-bar-baz", s); +// +// See comments on the `absl::StrJoin()` function for more examples. + +#ifndef ABSL_STRINGS_STR_JOIN_H_ +#define ABSL_STRINGS_STR_JOIN_H_ + +#include <cstdio> +#include <cstring> +#include <initializer_list> +#include <iterator> +#include <string> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/base/macros.h" +#include "absl/strings/internal/str_join_internal.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// Concept: Formatter +// ----------------------------------------------------------------------------- +// +// A Formatter is a function object that is responsible for formatting its +// argument as a string and appending it to a given output std::string. +// Formatters may be implemented as function objects, lambdas, or normal +// functions. You may provide your own Formatter to enable `absl::StrJoin()` to +// work with arbitrary types. +// +// The following is an example of a custom Formatter that simply uses +// `std::to_string()` to format an integer as a std::string. +// +// struct MyFormatter { +// void operator()(std::string* out, int i) const { +// out->append(std::to_string(i)); +// } +// }; +// +// You would use the above formatter by passing an instance of it as the final +// argument to `absl::StrJoin()`: +// +// std::vector<int> v = {1, 2, 3, 4}; +// std::string s = absl::StrJoin(v, "-", MyFormatter()); +// EXPECT_EQ("1-2-3-4", s); +// +// The following standard formatters are provided within this file: +// +// - `AlphaNumFormatter()` (the default) +// - `StreamFormatter()` +// - `PairFormatter()` +// - `DereferenceFormatter()` + +// AlphaNumFormatter() +// +// Default formatter used if none is specified. Uses `absl::AlphaNum` to convert +// numeric arguments to strings. +inline strings_internal::AlphaNumFormatterImpl AlphaNumFormatter() { + return strings_internal::AlphaNumFormatterImpl(); +} + +// StreamFormatter() +// +// Formats its argument using the << operator. +inline strings_internal::StreamFormatterImpl StreamFormatter() { + return strings_internal::StreamFormatterImpl(); +} + +// Function Template: PairFormatter(Formatter, absl::string_view, Formatter) +// +// Formats a `std::pair` by putting a given separator between the pair's +// `.first` and `.second` members. This formatter allows you to specify +// custom Formatters for both the first and second member of each pair. +template <typename FirstFormatter, typename SecondFormatter> +inline strings_internal::PairFormatterImpl<FirstFormatter, SecondFormatter> +PairFormatter(FirstFormatter f1, absl::string_view sep, SecondFormatter f2) { + return strings_internal::PairFormatterImpl<FirstFormatter, SecondFormatter>( + std::move(f1), sep, std::move(f2)); +} + +// Function overload of PairFormatter() for using a default +// `AlphaNumFormatter()` for each Formatter in the pair. +inline strings_internal::PairFormatterImpl< + strings_internal::AlphaNumFormatterImpl, + strings_internal::AlphaNumFormatterImpl> +PairFormatter(absl::string_view sep) { + return PairFormatter(AlphaNumFormatter(), sep, AlphaNumFormatter()); +} + +// Function Template: DereferenceFormatter(Formatter) +// +// Formats its argument by dereferencing it and then applying the given +// formatter. This formatter is useful for formatting a container of +// pointer-to-T. This pattern often shows up when joining repeated fields in +// protocol buffers. +template <typename Formatter> +strings_internal::DereferenceFormatterImpl<Formatter> DereferenceFormatter( + Formatter&& f) { + return strings_internal::DereferenceFormatterImpl<Formatter>( + std::forward<Formatter>(f)); +} + +// Function overload of `DererefenceFormatter()` for using a default +// `AlphaNumFormatter()`. +inline strings_internal::DereferenceFormatterImpl< + strings_internal::AlphaNumFormatterImpl> +DereferenceFormatter() { + return strings_internal::DereferenceFormatterImpl< + strings_internal::AlphaNumFormatterImpl>(AlphaNumFormatter()); +} + +// ----------------------------------------------------------------------------- +// StrJoin() +// ----------------------------------------------------------------------------- +// +// Joins a range of elements and returns the result as a std::string. +// `absl::StrJoin()` takes a range, a separator string to use between the +// elements joined, and an optional Formatter responsible for converting each +// argument in the range to a string. +// +// If omitted, the default `AlphaNumFormatter()` is called on the elements to be +// joined. +// +// Example 1: +// // Joins a collection of strings. This pattern also works with a collection +// // of `absl::string_view` or even `const char*`. +// std::vector<std::string> v = {"foo", "bar", "baz"}; +// std::string s = absl::StrJoin(v, "-"); +// EXPECT_EQ("foo-bar-baz", s); +// +// Example 2: +// // Joins the values in the given `std::initializer_list<>` specified using +// // brace initialization. This pattern also works with an initializer_list +// // of ints or `absl::string_view` -- any `AlphaNum`-compatible type. +// std::string s = absl::StrJoin({"foo", "bar", "baz"}, "-"); +// EXPECT_EQ("foo-bar-baz", s); +// +// Example 3: +// // Joins a collection of ints. This pattern also works with floats, +// // doubles, int64s -- any `StrCat()`-compatible type. +// std::vector<int> v = {1, 2, 3, -4}; +// std::string s = absl::StrJoin(v, "-"); +// EXPECT_EQ("1-2-3--4", s); +// +// Example 4: +// // Joins a collection of pointer-to-int. By default, pointers are +// // dereferenced and the pointee is formatted using the default format for +// // that type; such dereferencing occurs for all levels of indirection, so +// // this pattern works just as well for `std::vector<int**>` as for +// // `std::vector<int*>`. +// int x = 1, y = 2, z = 3; +// std::vector<int*> v = {&x, &y, &z}; +// std::string s = absl::StrJoin(v, "-"); +// EXPECT_EQ("1-2-3", s); +// +// Example 5: +// // Dereferencing of `std::unique_ptr<>` is also supported: +// std::vector<std::unique_ptr<int>> v +// v.emplace_back(new int(1)); +// v.emplace_back(new int(2)); +// v.emplace_back(new int(3)); +// std::string s = absl::StrJoin(v, "-"); +// EXPECT_EQ("1-2-3", s); +// +// Example 6: +// // Joins a `std::map`, with each key-value pair separated by an equals +// // sign. This pattern would also work with, say, a +// // `std::vector<std::pair<>>`. +// std::map<std::string, int> m = { +// std::make_pair("a", 1), +// std::make_pair("b", 2), +// std::make_pair("c", 3)}; +// std::string s = absl::StrJoin(m, ",", absl::PairFormatter("=")); +// EXPECT_EQ("a=1,b=2,c=3", s); +// +// Example 7: +// // These examples show how `absl::StrJoin()` handles a few common edge +// // cases: +// std::vector<std::string> v_empty; +// EXPECT_EQ("", absl::StrJoin(v_empty, "-")); +// +// std::vector<std::string> v_one_item = {"foo"}; +// EXPECT_EQ("foo", absl::StrJoin(v_one_item, "-")); +// +// std::vector<std::string> v_empty_string = {""}; +// EXPECT_EQ("", absl::StrJoin(v_empty_string, "-")); +// +// std::vector<std::string> v_one_item_empty_string = {"a", ""}; +// EXPECT_EQ("a-", absl::StrJoin(v_one_item_empty_string, "-")); +// +// std::vector<std::string> v_two_empty_string = {"", ""}; +// EXPECT_EQ("-", absl::StrJoin(v_two_empty_string, "-")); +// +// Example 8: +// // Joins a `std::tuple<T...>` of heterogeneous types, converting each to +// // a std::string using the `absl::AlphaNum` class. +// std::string s = absl::StrJoin(std::make_tuple(123, "abc", 0.456), "-"); +// EXPECT_EQ("123-abc-0.456", s); + +template <typename Iterator, typename Formatter> +std::string StrJoin(Iterator start, Iterator end, absl::string_view sep, + Formatter&& fmt) { + return strings_internal::JoinAlgorithm(start, end, sep, fmt); +} + +template <typename Range, typename Formatter> +std::string StrJoin(const Range& range, absl::string_view separator, + Formatter&& fmt) { + return strings_internal::JoinRange(range, separator, fmt); +} + +template <typename T, typename Formatter> +std::string StrJoin(std::initializer_list<T> il, absl::string_view separator, + Formatter&& fmt) { + return strings_internal::JoinRange(il, separator, fmt); +} + +template <typename... T, typename Formatter> +std::string StrJoin(const std::tuple<T...>& value, absl::string_view separator, + Formatter&& fmt) { + return strings_internal::JoinAlgorithm(value, separator, fmt); +} + +template <typename Iterator> +std::string StrJoin(Iterator start, Iterator end, absl::string_view separator) { + return strings_internal::JoinRange(start, end, separator); +} + +template <typename Range> +std::string StrJoin(const Range& range, absl::string_view separator) { + return strings_internal::JoinRange(range, separator); +} + +template <typename T> +std::string StrJoin(std::initializer_list<T> il, + absl::string_view separator) { + return strings_internal::JoinRange(il, separator); +} + +template <typename... T> +std::string StrJoin(const std::tuple<T...>& value, + absl::string_view separator) { + return strings_internal::JoinAlgorithm(value, separator, AlphaNumFormatter()); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STR_JOIN_H_ diff --git a/third_party/abseil_cpp/absl/strings/str_join_benchmark.cc b/third_party/abseil_cpp/absl/strings/str_join_benchmark.cc new file mode 100644 index 000000000000..d6f689ff300b --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_join_benchmark.cc @@ -0,0 +1,97 @@ +// +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_join.h" + +#include <string> +#include <vector> +#include <utility> + +#include "benchmark/benchmark.h" + +namespace { + +void BM_Join2_Strings(benchmark::State& state) { + const int string_len = state.range(0); + const int num_strings = state.range(1); + const std::string s(string_len, 'x'); + const std::vector<std::string> v(num_strings, s); + for (auto _ : state) { + std::string s = absl::StrJoin(v, "-"); + benchmark::DoNotOptimize(s); + } +} +BENCHMARK(BM_Join2_Strings) + ->ArgPair(1 << 0, 1 << 3) + ->ArgPair(1 << 10, 1 << 3) + ->ArgPair(1 << 13, 1 << 3) + ->ArgPair(1 << 0, 1 << 10) + ->ArgPair(1 << 10, 1 << 10) + ->ArgPair(1 << 13, 1 << 10) + ->ArgPair(1 << 0, 1 << 13) + ->ArgPair(1 << 10, 1 << 13) + ->ArgPair(1 << 13, 1 << 13); + +void BM_Join2_Ints(benchmark::State& state) { + const int num_ints = state.range(0); + const std::vector<int> v(num_ints, 42); + for (auto _ : state) { + std::string s = absl::StrJoin(v, "-"); + benchmark::DoNotOptimize(s); + } +} +BENCHMARK(BM_Join2_Ints)->Range(0, 1 << 13); + +void BM_Join2_KeysAndValues(benchmark::State& state) { + const int string_len = state.range(0); + const int num_pairs = state.range(1); + const std::string s(string_len, 'x'); + const std::vector<std::pair<std::string, int>> v(num_pairs, + std::make_pair(s, 42)); + for (auto _ : state) { + std::string s = absl::StrJoin(v, ",", absl::PairFormatter("=")); + benchmark::DoNotOptimize(s); + } +} +BENCHMARK(BM_Join2_KeysAndValues) + ->ArgPair(1 << 0, 1 << 3) + ->ArgPair(1 << 10, 1 << 3) + ->ArgPair(1 << 13, 1 << 3) + ->ArgPair(1 << 0, 1 << 10) + ->ArgPair(1 << 10, 1 << 10) + ->ArgPair(1 << 13, 1 << 10) + ->ArgPair(1 << 0, 1 << 13) + ->ArgPair(1 << 10, 1 << 13) + ->ArgPair(1 << 13, 1 << 13); + +void BM_JoinStreamable(benchmark::State& state) { + const int string_len = state.range(0); + const int num_strings = state.range(1); + const std::vector<std::string> v(num_strings, std::string(string_len, 'x')); + for (auto _ : state) { + std::string s = absl::StrJoin(v, "", absl::StreamFormatter()); + benchmark::DoNotOptimize(s); + } +} +BENCHMARK(BM_JoinStreamable) + ->ArgPair(0, 0) + ->ArgPair(16, 1) + ->ArgPair(256, 1) + ->ArgPair(16, 16) + ->ArgPair(256, 16) + ->ArgPair(16, 256) + ->ArgPair(256, 256); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_join_test.cc b/third_party/abseil_cpp/absl/strings/str_join_test.cc new file mode 100644 index 000000000000..2be6256e43f7 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_join_test.cc @@ -0,0 +1,474 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit tests for all join.h functions + +#include "absl/strings/str_join.h" + +#include <cstddef> +#include <cstdint> +#include <cstdio> +#include <functional> +#include <initializer_list> +#include <map> +#include <memory> +#include <ostream> +#include <tuple> +#include <type_traits> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/macros.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" + +namespace { + +TEST(StrJoin, APIExamples) { + { + // Collection of strings + std::vector<std::string> v = {"foo", "bar", "baz"}; + EXPECT_EQ("foo-bar-baz", absl::StrJoin(v, "-")); + } + + { + // Collection of absl::string_view + std::vector<absl::string_view> v = {"foo", "bar", "baz"}; + EXPECT_EQ("foo-bar-baz", absl::StrJoin(v, "-")); + } + + { + // Collection of const char* + std::vector<const char*> v = {"foo", "bar", "baz"}; + EXPECT_EQ("foo-bar-baz", absl::StrJoin(v, "-")); + } + + { + // Collection of non-const char* + std::string a = "foo", b = "bar", c = "baz"; + std::vector<char*> v = {&a[0], &b[0], &c[0]}; + EXPECT_EQ("foo-bar-baz", absl::StrJoin(v, "-")); + } + + { + // Collection of ints + std::vector<int> v = {1, 2, 3, -4}; + EXPECT_EQ("1-2-3--4", absl::StrJoin(v, "-")); + } + + { + // Literals passed as a std::initializer_list + std::string s = absl::StrJoin({"a", "b", "c"}, "-"); + EXPECT_EQ("a-b-c", s); + } + { + // Join a std::tuple<T...>. + std::string s = absl::StrJoin(std::make_tuple(123, "abc", 0.456), "-"); + EXPECT_EQ("123-abc-0.456", s); + } + + { + // Collection of unique_ptrs + std::vector<std::unique_ptr<int>> v; + v.emplace_back(new int(1)); + v.emplace_back(new int(2)); + v.emplace_back(new int(3)); + EXPECT_EQ("1-2-3", absl::StrJoin(v, "-")); + } + + { + // Array of ints + const int a[] = {1, 2, 3, -4}; + EXPECT_EQ("1-2-3--4", absl::StrJoin(a, a + ABSL_ARRAYSIZE(a), "-")); + } + + { + // Collection of pointers + int x = 1, y = 2, z = 3; + std::vector<int*> v = {&x, &y, &z}; + EXPECT_EQ("1-2-3", absl::StrJoin(v, "-")); + } + + { + // Collection of pointers to pointers + int x = 1, y = 2, z = 3; + int *px = &x, *py = &y, *pz = &z; + std::vector<int**> v = {&px, &py, &pz}; + EXPECT_EQ("1-2-3", absl::StrJoin(v, "-")); + } + + { + // Collection of pointers to std::string + std::string a("a"), b("b"); + std::vector<std::string*> v = {&a, &b}; + EXPECT_EQ("a-b", absl::StrJoin(v, "-")); + } + + { + // A std::map, which is a collection of std::pair<>s. + std::map<std::string, int> m = {{"a", 1}, {"b", 2}, {"c", 3}}; + EXPECT_EQ("a=1,b=2,c=3", absl::StrJoin(m, ",", absl::PairFormatter("="))); + } + + { + // Shows absl::StrSplit and absl::StrJoin working together. This example is + // equivalent to s/=/-/g. + const std::string s = "a=b=c=d"; + EXPECT_EQ("a-b-c-d", absl::StrJoin(absl::StrSplit(s, "="), "-")); + } + + // + // A few examples of edge cases + // + + { + // Empty range yields an empty string. + std::vector<std::string> v; + EXPECT_EQ("", absl::StrJoin(v, "-")); + } + + { + // A range of 1 element gives a string with that element but no + // separator. + std::vector<std::string> v = {"foo"}; + EXPECT_EQ("foo", absl::StrJoin(v, "-")); + } + + { + // A range with a single empty string element + std::vector<std::string> v = {""}; + EXPECT_EQ("", absl::StrJoin(v, "-")); + } + + { + // A range with 2 elements, one of which is an empty string + std::vector<std::string> v = {"a", ""}; + EXPECT_EQ("a-", absl::StrJoin(v, "-")); + } + + { + // A range with 2 empty elements. + std::vector<std::string> v = {"", ""}; + EXPECT_EQ("-", absl::StrJoin(v, "-")); + } + + { + // A std::vector of bool. + std::vector<bool> v = {true, false, true}; + EXPECT_EQ("1-0-1", absl::StrJoin(v, "-")); + } +} + +TEST(StrJoin, CustomFormatter) { + std::vector<std::string> v{"One", "Two", "Three"}; + { + std::string joined = + absl::StrJoin(v, "", [](std::string* out, const std::string& in) { + absl::StrAppend(out, "(", in, ")"); + }); + EXPECT_EQ("(One)(Two)(Three)", joined); + } + { + class ImmovableFormatter { + public: + void operator()(std::string* out, const std::string& in) { + absl::StrAppend(out, "(", in, ")"); + } + ImmovableFormatter() {} + ImmovableFormatter(const ImmovableFormatter&) = delete; + }; + EXPECT_EQ("(One)(Two)(Three)", absl::StrJoin(v, "", ImmovableFormatter())); + } + { + class OverloadedFormatter { + public: + void operator()(std::string* out, const std::string& in) { + absl::StrAppend(out, "(", in, ")"); + } + void operator()(std::string* out, const std::string& in) const { + absl::StrAppend(out, "[", in, "]"); + } + }; + EXPECT_EQ("(One)(Two)(Three)", absl::StrJoin(v, "", OverloadedFormatter())); + const OverloadedFormatter fmt = {}; + EXPECT_EQ("[One][Two][Three]", absl::StrJoin(v, "", fmt)); + } +} + +// +// Tests the Formatters +// + +TEST(AlphaNumFormatter, FormatterAPI) { + // Not an exhaustive test. See strings/strcat_test.h for the exhaustive test + // of what AlphaNum can convert. + auto f = absl::AlphaNumFormatter(); + std::string s; + f(&s, "Testing: "); + f(&s, static_cast<int>(1)); + f(&s, static_cast<int16_t>(2)); + f(&s, static_cast<int64_t>(3)); + f(&s, static_cast<float>(4)); + f(&s, static_cast<double>(5)); + f(&s, static_cast<unsigned>(6)); + f(&s, static_cast<size_t>(7)); + f(&s, absl::string_view(" OK")); + EXPECT_EQ("Testing: 1234567 OK", s); +} + +// Make sure people who are mistakenly using std::vector<bool> even though +// they're not memory-constrained can use absl::AlphaNumFormatter(). +TEST(AlphaNumFormatter, VectorOfBool) { + auto f = absl::AlphaNumFormatter(); + std::string s; + std::vector<bool> v = {true, false, true}; + f(&s, *v.cbegin()); + f(&s, *v.begin()); + f(&s, v[1]); + EXPECT_EQ("110", s); +} + +TEST(AlphaNumFormatter, AlphaNum) { + auto f = absl::AlphaNumFormatter(); + std::string s; + f(&s, absl::AlphaNum("hello")); + EXPECT_EQ("hello", s); +} + +struct StreamableType { + std::string contents; +}; +inline std::ostream& operator<<(std::ostream& os, const StreamableType& t) { + os << "Streamable:" << t.contents; + return os; +} + +TEST(StreamFormatter, FormatterAPI) { + auto f = absl::StreamFormatter(); + std::string s; + f(&s, "Testing: "); + f(&s, static_cast<int>(1)); + f(&s, static_cast<int16_t>(2)); + f(&s, static_cast<int64_t>(3)); + f(&s, static_cast<float>(4)); + f(&s, static_cast<double>(5)); + f(&s, static_cast<unsigned>(6)); + f(&s, static_cast<size_t>(7)); + f(&s, absl::string_view(" OK ")); + StreamableType streamable = {"object"}; + f(&s, streamable); + EXPECT_EQ("Testing: 1234567 OK Streamable:object", s); +} + +// A dummy formatter that wraps each element in parens. Used in some tests +// below. +struct TestingParenFormatter { + template <typename T> + void operator()(std::string* s, const T& t) { + absl::StrAppend(s, "(", t, ")"); + } +}; + +TEST(PairFormatter, FormatterAPI) { + { + // Tests default PairFormatter(sep) that uses AlphaNumFormatter for the + // 'first' and 'second' members. + const auto f = absl::PairFormatter("="); + std::string s; + f(&s, std::make_pair("a", "b")); + f(&s, std::make_pair(1, 2)); + EXPECT_EQ("a=b1=2", s); + } + + { + // Tests using a custom formatter for the 'first' and 'second' members. + auto f = absl::PairFormatter(TestingParenFormatter(), "=", + TestingParenFormatter()); + std::string s; + f(&s, std::make_pair("a", "b")); + f(&s, std::make_pair(1, 2)); + EXPECT_EQ("(a)=(b)(1)=(2)", s); + } +} + +TEST(DereferenceFormatter, FormatterAPI) { + { + // Tests wrapping the default AlphaNumFormatter. + const absl::strings_internal::DereferenceFormatterImpl< + absl::strings_internal::AlphaNumFormatterImpl> + f; + int x = 1, y = 2, z = 3; + std::string s; + f(&s, &x); + f(&s, &y); + f(&s, &z); + EXPECT_EQ("123", s); + } + + { + // Tests wrapping std::string's default formatter. + absl::strings_internal::DereferenceFormatterImpl< + absl::strings_internal::DefaultFormatter<std::string>::Type> + f; + + std::string x = "x"; + std::string y = "y"; + std::string z = "z"; + std::string s; + f(&s, &x); + f(&s, &y); + f(&s, &z); + EXPECT_EQ(s, "xyz"); + } + + { + // Tests wrapping a custom formatter. + auto f = absl::DereferenceFormatter(TestingParenFormatter()); + int x = 1, y = 2, z = 3; + std::string s; + f(&s, &x); + f(&s, &y); + f(&s, &z); + EXPECT_EQ("(1)(2)(3)", s); + } + + { + absl::strings_internal::DereferenceFormatterImpl< + absl::strings_internal::AlphaNumFormatterImpl> + f; + auto x = std::unique_ptr<int>(new int(1)); + auto y = std::unique_ptr<int>(new int(2)); + auto z = std::unique_ptr<int>(new int(3)); + std::string s; + f(&s, x); + f(&s, y); + f(&s, z); + EXPECT_EQ("123", s); + } +} + +// +// Tests the interfaces for the 4 public Join function overloads. The semantics +// of the algorithm is covered in the above APIExamples test. +// +TEST(StrJoin, PublicAPIOverloads) { + std::vector<std::string> v = {"a", "b", "c"}; + + // Iterators + formatter + EXPECT_EQ("a-b-c", + absl::StrJoin(v.begin(), v.end(), "-", absl::AlphaNumFormatter())); + // Range + formatter + EXPECT_EQ("a-b-c", absl::StrJoin(v, "-", absl::AlphaNumFormatter())); + // Iterators, no formatter + EXPECT_EQ("a-b-c", absl::StrJoin(v.begin(), v.end(), "-")); + // Range, no formatter + EXPECT_EQ("a-b-c", absl::StrJoin(v, "-")); +} + +TEST(StrJoin, Array) { + const absl::string_view a[] = {"a", "b", "c"}; + EXPECT_EQ("a-b-c", absl::StrJoin(a, "-")); +} + +TEST(StrJoin, InitializerList) { + { EXPECT_EQ("a-b-c", absl::StrJoin({"a", "b", "c"}, "-")); } + + { + auto a = {"a", "b", "c"}; + EXPECT_EQ("a-b-c", absl::StrJoin(a, "-")); + } + + { + std::initializer_list<const char*> a = {"a", "b", "c"}; + EXPECT_EQ("a-b-c", absl::StrJoin(a, "-")); + } + + { + std::initializer_list<std::string> a = {"a", "b", "c"}; + EXPECT_EQ("a-b-c", absl::StrJoin(a, "-")); + } + + { + std::initializer_list<absl::string_view> a = {"a", "b", "c"}; + EXPECT_EQ("a-b-c", absl::StrJoin(a, "-")); + } + + { + // Tests initializer_list with a non-default formatter + auto a = {"a", "b", "c"}; + TestingParenFormatter f; + EXPECT_EQ("(a)-(b)-(c)", absl::StrJoin(a, "-", f)); + } + + { + // initializer_list of ints + EXPECT_EQ("1-2-3", absl::StrJoin({1, 2, 3}, "-")); + } + + { + // Tests initializer_list of ints with a non-default formatter + auto a = {1, 2, 3}; + TestingParenFormatter f; + EXPECT_EQ("(1)-(2)-(3)", absl::StrJoin(a, "-", f)); + } +} + +TEST(StrJoin, Tuple) { + EXPECT_EQ("", absl::StrJoin(std::make_tuple(), "-")); + EXPECT_EQ("hello", absl::StrJoin(std::make_tuple("hello"), "-")); + + int x(10); + std::string y("hello"); + double z(3.14); + EXPECT_EQ("10-hello-3.14", absl::StrJoin(std::make_tuple(x, y, z), "-")); + + // Faster! Faster!! + EXPECT_EQ("10-hello-3.14", + absl::StrJoin(std::make_tuple(x, std::cref(y), z), "-")); + + struct TestFormatter { + char buffer[128]; + void operator()(std::string* out, int v) { + snprintf(buffer, sizeof(buffer), "%#.8x", v); + out->append(buffer); + } + void operator()(std::string* out, double v) { + snprintf(buffer, sizeof(buffer), "%#.0f", v); + out->append(buffer); + } + void operator()(std::string* out, const std::string& v) { + snprintf(buffer, sizeof(buffer), "%.4s", v.c_str()); + out->append(buffer); + } + }; + EXPECT_EQ("0x0000000a-hell-3.", + absl::StrJoin(std::make_tuple(x, y, z), "-", TestFormatter())); + EXPECT_EQ( + "0x0000000a-hell-3.", + absl::StrJoin(std::make_tuple(x, std::cref(y), z), "-", TestFormatter())); + EXPECT_EQ("0x0000000a-hell-3.", + absl::StrJoin(std::make_tuple(&x, &y, &z), "-", + absl::DereferenceFormatter(TestFormatter()))); + EXPECT_EQ("0x0000000a-hell-3.", + absl::StrJoin(std::make_tuple(absl::make_unique<int>(x), + absl::make_unique<std::string>(y), + absl::make_unique<double>(z)), + "-", absl::DereferenceFormatter(TestFormatter()))); + EXPECT_EQ("0x0000000a-hell-3.", + absl::StrJoin(std::make_tuple(absl::make_unique<int>(x), &y, &z), + "-", absl::DereferenceFormatter(TestFormatter()))); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_replace.cc b/third_party/abseil_cpp/absl/strings/str_replace.cc new file mode 100644 index 000000000000..2bd5fa98218c --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_replace.cc @@ -0,0 +1,82 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_replace.h" + +#include "absl/strings/str_cat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace strings_internal { + +using FixedMapping = + std::initializer_list<std::pair<absl::string_view, absl::string_view>>; + +// Applies the ViableSubstitutions in subs_ptr to the absl::string_view s, and +// stores the result in *result_ptr. Returns the number of substitutions that +// occurred. +int ApplySubstitutions( + absl::string_view s, + std::vector<strings_internal::ViableSubstitution>* subs_ptr, + std::string* result_ptr) { + auto& subs = *subs_ptr; + int substitutions = 0; + size_t pos = 0; + while (!subs.empty()) { + auto& sub = subs.back(); + if (sub.offset >= pos) { + if (pos <= s.size()) { + StrAppend(result_ptr, s.substr(pos, sub.offset - pos), sub.replacement); + } + pos = sub.offset + sub.old.size(); + substitutions += 1; + } + sub.offset = s.find(sub.old, pos); + if (sub.offset == s.npos) { + subs.pop_back(); + } else { + // Insertion sort to ensure the last ViableSubstitution continues to be + // before all the others. + size_t index = subs.size(); + while (--index && subs[index - 1].OccursBefore(subs[index])) { + std::swap(subs[index], subs[index - 1]); + } + } + } + result_ptr->append(s.data() + pos, s.size() - pos); + return substitutions; +} + +} // namespace strings_internal + +// We can implement this in terms of the generic StrReplaceAll, but +// we must specify the template overload because C++ cannot deduce the type +// of an initializer_list parameter to a function, and also if we don't specify +// the type, we just call ourselves. +// +// Note that we implement them here, rather than in the header, so that they +// aren't inlined. + +std::string StrReplaceAll(absl::string_view s, + strings_internal::FixedMapping replacements) { + return StrReplaceAll<strings_internal::FixedMapping>(s, replacements); +} + +int StrReplaceAll(strings_internal::FixedMapping replacements, + std::string* target) { + return StrReplaceAll<strings_internal::FixedMapping>(replacements, target); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/str_replace.h b/third_party/abseil_cpp/absl/strings/str_replace.h new file mode 100644 index 000000000000..273c7077353e --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_replace.h @@ -0,0 +1,219 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: str_replace.h +// ----------------------------------------------------------------------------- +// +// This file defines `absl::StrReplaceAll()`, a general-purpose string +// replacement function designed for large, arbitrary text substitutions, +// especially on strings which you are receiving from some other system for +// further processing (e.g. processing regular expressions, escaping HTML +// entities, etc.). `StrReplaceAll` is designed to be efficient even when only +// one substitution is being performed, or when substitution is rare. +// +// If the string being modified is known at compile-time, and the substitutions +// vary, `absl::Substitute()` may be a better choice. +// +// Example: +// +// std::string html_escaped = absl::StrReplaceAll(user_input, { +// {"&", "&"}, +// {"<", "<"}, +// {">", ">"}, +// {"\"", """}, +// {"'", "'"}}); +#ifndef ABSL_STRINGS_STR_REPLACE_H_ +#define ABSL_STRINGS_STR_REPLACE_H_ + +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/attributes.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// StrReplaceAll() +// +// Replaces character sequences within a given string with replacements provided +// within an initializer list of key/value pairs. Candidate replacements are +// considered in order as they occur within the string, with earlier matches +// taking precedence, and longer matches taking precedence for candidates +// starting at the same position in the string. Once a substitution is made, the +// replaced text is not considered for any further substitutions. +// +// Example: +// +// std::string s = absl::StrReplaceAll( +// "$who bought $count #Noun. Thanks $who!", +// {{"$count", absl::StrCat(5)}, +// {"$who", "Bob"}, +// {"#Noun", "Apples"}}); +// EXPECT_EQ("Bob bought 5 Apples. Thanks Bob!", s); +ABSL_MUST_USE_RESULT std::string StrReplaceAll( + absl::string_view s, + std::initializer_list<std::pair<absl::string_view, absl::string_view>> + replacements); + +// Overload of `StrReplaceAll()` to accept a container of key/value replacement +// pairs (typically either an associative map or a `std::vector` of `std::pair` +// elements). A vector of pairs is generally more efficient. +// +// Examples: +// +// std::map<const absl::string_view, const absl::string_view> replacements; +// replacements["$who"] = "Bob"; +// replacements["$count"] = "5"; +// replacements["#Noun"] = "Apples"; +// std::string s = absl::StrReplaceAll( +// "$who bought $count #Noun. Thanks $who!", +// replacements); +// EXPECT_EQ("Bob bought 5 Apples. Thanks Bob!", s); +// +// // A std::vector of std::pair elements can be more efficient. +// std::vector<std::pair<const absl::string_view, std::string>> replacements; +// replacements.push_back({"&", "&"}); +// replacements.push_back({"<", "<"}); +// replacements.push_back({">", ">"}); +// std::string s = absl::StrReplaceAll("if (ptr < &foo)", +// replacements); +// EXPECT_EQ("if (ptr < &foo)", s); +template <typename StrToStrMapping> +std::string StrReplaceAll(absl::string_view s, + const StrToStrMapping& replacements); + +// Overload of `StrReplaceAll()` to replace character sequences within a given +// output string *in place* with replacements provided within an initializer +// list of key/value pairs, returning the number of substitutions that occurred. +// +// Example: +// +// std::string s = std::string("$who bought $count #Noun. Thanks $who!"); +// int count; +// count = absl::StrReplaceAll({{"$count", absl::StrCat(5)}, +// {"$who", "Bob"}, +// {"#Noun", "Apples"}}, &s); +// EXPECT_EQ(count, 4); +// EXPECT_EQ("Bob bought 5 Apples. Thanks Bob!", s); +int StrReplaceAll( + std::initializer_list<std::pair<absl::string_view, absl::string_view>> + replacements, + std::string* target); + +// Overload of `StrReplaceAll()` to replace patterns within a given output +// string *in place* with replacements provided within a container of key/value +// pairs. +// +// Example: +// +// std::string s = std::string("if (ptr < &foo)"); +// int count = absl::StrReplaceAll({{"&", "&"}, +// {"<", "<"}, +// {">", ">"}}, &s); +// EXPECT_EQ(count, 2); +// EXPECT_EQ("if (ptr < &foo)", s); +template <typename StrToStrMapping> +int StrReplaceAll(const StrToStrMapping& replacements, std::string* target); + +// Implementation details only, past this point. +namespace strings_internal { + +struct ViableSubstitution { + absl::string_view old; + absl::string_view replacement; + size_t offset; + + ViableSubstitution(absl::string_view old_str, + absl::string_view replacement_str, size_t offset_val) + : old(old_str), replacement(replacement_str), offset(offset_val) {} + + // One substitution occurs "before" another (takes priority) if either + // it has the lowest offset, or it has the same offset but a larger size. + bool OccursBefore(const ViableSubstitution& y) const { + if (offset != y.offset) return offset < y.offset; + return old.size() > y.old.size(); + } +}; + +// Build a vector of ViableSubstitutions based on the given list of +// replacements. subs can be implemented as a priority_queue. However, it turns +// out that most callers have small enough a list of substitutions that the +// overhead of such a queue isn't worth it. +template <typename StrToStrMapping> +std::vector<ViableSubstitution> FindSubstitutions( + absl::string_view s, const StrToStrMapping& replacements) { + std::vector<ViableSubstitution> subs; + subs.reserve(replacements.size()); + + for (const auto& rep : replacements) { + using std::get; + absl::string_view old(get<0>(rep)); + + size_t pos = s.find(old); + if (pos == s.npos) continue; + + // Ignore attempts to replace "". This condition is almost never true, + // but above condition is frequently true. That's why we test for this + // now and not before. + if (old.empty()) continue; + + subs.emplace_back(old, get<1>(rep), pos); + + // Insertion sort to ensure the last ViableSubstitution comes before + // all the others. + size_t index = subs.size(); + while (--index && subs[index - 1].OccursBefore(subs[index])) { + std::swap(subs[index], subs[index - 1]); + } + } + return subs; +} + +int ApplySubstitutions(absl::string_view s, + std::vector<ViableSubstitution>* subs_ptr, + std::string* result_ptr); + +} // namespace strings_internal + +template <typename StrToStrMapping> +std::string StrReplaceAll(absl::string_view s, + const StrToStrMapping& replacements) { + auto subs = strings_internal::FindSubstitutions(s, replacements); + std::string result; + result.reserve(s.size()); + strings_internal::ApplySubstitutions(s, &subs, &result); + return result; +} + +template <typename StrToStrMapping> +int StrReplaceAll(const StrToStrMapping& replacements, std::string* target) { + auto subs = strings_internal::FindSubstitutions(*target, replacements); + if (subs.empty()) return 0; + + std::string result; + result.reserve(target->size()); + int substitutions = + strings_internal::ApplySubstitutions(*target, &subs, &result); + target->swap(result); + return substitutions; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STR_REPLACE_H_ diff --git a/third_party/abseil_cpp/absl/strings/str_replace_benchmark.cc b/third_party/abseil_cpp/absl/strings/str_replace_benchmark.cc new file mode 100644 index 000000000000..01331da29f6e --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_replace_benchmark.cc @@ -0,0 +1,122 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_replace.h" + +#include <cstring> +#include <string> + +#include "benchmark/benchmark.h" +#include "absl/base/internal/raw_logging.h" + +namespace { + +std::string* big_string; +std::string* after_replacing_the; +std::string* after_replacing_many; + +struct Replacement { + const char* needle; + const char* replacement; +} replacements[] = { + {"the", "box"}, // + {"brown", "quick"}, // + {"jumped", "liquored"}, // + {"dozen", "brown"}, // + {"lazy", "pack"}, // + {"liquor", "shakes"}, // +}; + +// Here, we set up a string for use in global-replace benchmarks. +// We started with a million blanks, and then deterministically insert +// 10,000 copies each of two pangrams. The result is a string that is +// 40% blank space and 60% these words. 'the' occurs 18,247 times and +// all the substitutions together occur 49,004 times. +// +// We then create "after_replacing_the" to be a string that is a result of +// replacing "the" with "box" in big_string. +// +// And then we create "after_replacing_many" to be a string that is result +// of preferring several substitutions. +void SetUpStrings() { + if (big_string == nullptr) { + size_t r = 0; + big_string = new std::string(1000 * 1000, ' '); + for (std::string phrase : {"the quick brown fox jumped over the lazy dogs", + "pack my box with the five dozen liquor jugs"}) { + for (int i = 0; i < 10 * 1000; ++i) { + r = r * 237 + 41; // not very random. + memcpy(&(*big_string)[r % (big_string->size() - phrase.size())], + phrase.data(), phrase.size()); + } + } + // big_string->resize(50); + // OK, we've set up the string, now let's set up expectations - first by + // just replacing "the" with "box" + after_replacing_the = new std::string(*big_string); + for (size_t pos = 0; + (pos = after_replacing_the->find("the", pos)) != std::string::npos;) { + memcpy(&(*after_replacing_the)[pos], "box", 3); + } + // And then with all the replacements. + after_replacing_many = new std::string(*big_string); + for (size_t pos = 0;;) { + size_t next_pos = static_cast<size_t>(-1); + const char* needle_string = nullptr; + const char* replacement_string = nullptr; + for (const auto& r : replacements) { + auto needlepos = after_replacing_many->find(r.needle, pos); + if (needlepos != std::string::npos && needlepos < next_pos) { + next_pos = needlepos; + needle_string = r.needle; + replacement_string = r.replacement; + } + } + if (next_pos > after_replacing_many->size()) break; + after_replacing_many->replace(next_pos, strlen(needle_string), + replacement_string); + next_pos += strlen(replacement_string); + pos = next_pos; + } + } +} + +void BM_StrReplaceAllOneReplacement(benchmark::State& state) { + SetUpStrings(); + std::string src = *big_string; + for (auto _ : state) { + std::string dest = absl::StrReplaceAll(src, {{"the", "box"}}); + ABSL_RAW_CHECK(dest == *after_replacing_the, + "not benchmarking intended behavior"); + } +} +BENCHMARK(BM_StrReplaceAllOneReplacement); + +void BM_StrReplaceAll(benchmark::State& state) { + SetUpStrings(); + std::string src = *big_string; + for (auto _ : state) { + std::string dest = absl::StrReplaceAll(src, {{"the", "box"}, + {"brown", "quick"}, + {"jumped", "liquored"}, + {"dozen", "brown"}, + {"lazy", "pack"}, + {"liquor", "shakes"}}); + ABSL_RAW_CHECK(dest == *after_replacing_many, + "not benchmarking intended behavior"); + } +} +BENCHMARK(BM_StrReplaceAll); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_replace_test.cc b/third_party/abseil_cpp/absl/strings/str_replace_test.cc new file mode 100644 index 000000000000..9d8c7f75b57e --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_replace_test.cc @@ -0,0 +1,341 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_replace.h" + +#include <list> +#include <map> +#include <tuple> + +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" + +TEST(StrReplaceAll, OneReplacement) { + std::string s; + + // Empty string. + s = absl::StrReplaceAll(s, {{"", ""}}); + EXPECT_EQ(s, ""); + s = absl::StrReplaceAll(s, {{"x", ""}}); + EXPECT_EQ(s, ""); + s = absl::StrReplaceAll(s, {{"", "y"}}); + EXPECT_EQ(s, ""); + s = absl::StrReplaceAll(s, {{"x", "y"}}); + EXPECT_EQ(s, ""); + + // Empty substring. + s = absl::StrReplaceAll("abc", {{"", ""}}); + EXPECT_EQ(s, "abc"); + s = absl::StrReplaceAll("abc", {{"", "y"}}); + EXPECT_EQ(s, "abc"); + s = absl::StrReplaceAll("abc", {{"x", ""}}); + EXPECT_EQ(s, "abc"); + + // Substring not found. + s = absl::StrReplaceAll("abc", {{"xyz", "123"}}); + EXPECT_EQ(s, "abc"); + + // Replace entire string. + s = absl::StrReplaceAll("abc", {{"abc", "xyz"}}); + EXPECT_EQ(s, "xyz"); + + // Replace once at the start. + s = absl::StrReplaceAll("abc", {{"a", "x"}}); + EXPECT_EQ(s, "xbc"); + + // Replace once in the middle. + s = absl::StrReplaceAll("abc", {{"b", "x"}}); + EXPECT_EQ(s, "axc"); + + // Replace once at the end. + s = absl::StrReplaceAll("abc", {{"c", "x"}}); + EXPECT_EQ(s, "abx"); + + // Replace multiple times with varying lengths of original/replacement. + s = absl::StrReplaceAll("ababa", {{"a", "xxx"}}); + EXPECT_EQ(s, "xxxbxxxbxxx"); + + s = absl::StrReplaceAll("ababa", {{"b", "xxx"}}); + EXPECT_EQ(s, "axxxaxxxa"); + + s = absl::StrReplaceAll("aaabaaabaaa", {{"aaa", "x"}}); + EXPECT_EQ(s, "xbxbx"); + + s = absl::StrReplaceAll("abbbabbba", {{"bbb", "x"}}); + EXPECT_EQ(s, "axaxa"); + + // Overlapping matches are replaced greedily. + s = absl::StrReplaceAll("aaa", {{"aa", "x"}}); + EXPECT_EQ(s, "xa"); + + // The replacements are not recursive. + s = absl::StrReplaceAll("aaa", {{"aa", "a"}}); + EXPECT_EQ(s, "aa"); +} + +TEST(StrReplaceAll, ManyReplacements) { + std::string s; + + // Empty string. + s = absl::StrReplaceAll("", {{"", ""}, {"x", ""}, {"", "y"}, {"x", "y"}}); + EXPECT_EQ(s, ""); + + // Empty substring. + s = absl::StrReplaceAll("abc", {{"", ""}, {"", "y"}, {"x", ""}}); + EXPECT_EQ(s, "abc"); + + // Replace entire string, one char at a time + s = absl::StrReplaceAll("abc", {{"a", "x"}, {"b", "y"}, {"c", "z"}}); + EXPECT_EQ(s, "xyz"); + s = absl::StrReplaceAll("zxy", {{"z", "x"}, {"x", "y"}, {"y", "z"}}); + EXPECT_EQ(s, "xyz"); + + // Replace once at the start (longer matches take precedence) + s = absl::StrReplaceAll("abc", {{"a", "x"}, {"ab", "xy"}, {"abc", "xyz"}}); + EXPECT_EQ(s, "xyz"); + + // Replace once in the middle. + s = absl::StrReplaceAll( + "Abc!", {{"a", "x"}, {"ab", "xy"}, {"b", "y"}, {"bc", "yz"}, {"c", "z"}}); + EXPECT_EQ(s, "Ayz!"); + + // Replace once at the end. + s = absl::StrReplaceAll( + "Abc!", + {{"a", "x"}, {"ab", "xy"}, {"b", "y"}, {"bc!", "yz?"}, {"c!", "z;"}}); + EXPECT_EQ(s, "Ayz?"); + + // Replace multiple times with varying lengths of original/replacement. + s = absl::StrReplaceAll("ababa", {{"a", "xxx"}, {"b", "XXXX"}}); + EXPECT_EQ(s, "xxxXXXXxxxXXXXxxx"); + + // Overlapping matches are replaced greedily. + s = absl::StrReplaceAll("aaa", {{"aa", "x"}, {"a", "X"}}); + EXPECT_EQ(s, "xX"); + s = absl::StrReplaceAll("aaa", {{"a", "X"}, {"aa", "x"}}); + EXPECT_EQ(s, "xX"); + + // Two well-known sentences + s = absl::StrReplaceAll("the quick brown fox jumped over the lazy dogs", + { + {"brown", "box"}, + {"dogs", "jugs"}, + {"fox", "with"}, + {"jumped", "five"}, + {"over", "dozen"}, + {"quick", "my"}, + {"the", "pack"}, + {"the lazy", "liquor"}, + }); + EXPECT_EQ(s, "pack my box with five dozen liquor jugs"); +} + +TEST(StrReplaceAll, ManyReplacementsInMap) { + std::map<const char *, const char *> replacements; + replacements["$who"] = "Bob"; + replacements["$count"] = "5"; + replacements["#Noun"] = "Apples"; + std::string s = absl::StrReplaceAll("$who bought $count #Noun. Thanks $who!", + replacements); + EXPECT_EQ("Bob bought 5 Apples. Thanks Bob!", s); +} + +TEST(StrReplaceAll, ReplacementsInPlace) { + std::string s = std::string("$who bought $count #Noun. Thanks $who!"); + int count; + count = absl::StrReplaceAll({{"$count", absl::StrCat(5)}, + {"$who", "Bob"}, + {"#Noun", "Apples"}}, &s); + EXPECT_EQ(count, 4); + EXPECT_EQ("Bob bought 5 Apples. Thanks Bob!", s); +} + +TEST(StrReplaceAll, ReplacementsInPlaceInMap) { + std::string s = std::string("$who bought $count #Noun. Thanks $who!"); + std::map<absl::string_view, absl::string_view> replacements; + replacements["$who"] = "Bob"; + replacements["$count"] = "5"; + replacements["#Noun"] = "Apples"; + int count; + count = absl::StrReplaceAll(replacements, &s); + EXPECT_EQ(count, 4); + EXPECT_EQ("Bob bought 5 Apples. Thanks Bob!", s); +} + +struct Cont { + Cont() {} + explicit Cont(absl::string_view src) : data(src) {} + + absl::string_view data; +}; + +template <int index> +absl::string_view get(const Cont& c) { + auto splitter = absl::StrSplit(c.data, ':'); + auto it = splitter.begin(); + for (int i = 0; i < index; ++i) ++it; + + return *it; +} + +TEST(StrReplaceAll, VariableNumber) { + std::string s; + { + std::vector<std::pair<std::string, std::string>> replacements; + + s = "abc"; + EXPECT_EQ(0, absl::StrReplaceAll(replacements, &s)); + EXPECT_EQ("abc", s); + + s = "abc"; + replacements.push_back({"a", "A"}); + EXPECT_EQ(1, absl::StrReplaceAll(replacements, &s)); + EXPECT_EQ("Abc", s); + + s = "abc"; + replacements.push_back({"b", "B"}); + EXPECT_EQ(2, absl::StrReplaceAll(replacements, &s)); + EXPECT_EQ("ABc", s); + + s = "abc"; + replacements.push_back({"d", "D"}); + EXPECT_EQ(2, absl::StrReplaceAll(replacements, &s)); + EXPECT_EQ("ABc", s); + + EXPECT_EQ("ABcABc", absl::StrReplaceAll("abcabc", replacements)); + } + + { + std::map<const char*, const char*> replacements; + replacements["aa"] = "x"; + replacements["a"] = "X"; + s = "aaa"; + EXPECT_EQ(2, absl::StrReplaceAll(replacements, &s)); + EXPECT_EQ("xX", s); + + EXPECT_EQ("xxX", absl::StrReplaceAll("aaaaa", replacements)); + } + + { + std::list<std::pair<absl::string_view, absl::string_view>> replacements = { + {"a", "x"}, {"b", "y"}, {"c", "z"}}; + + std::string s = absl::StrReplaceAll("abc", replacements); + EXPECT_EQ(s, "xyz"); + } + + { + using X = std::tuple<absl::string_view, std::string, int>; + std::vector<X> replacements(3); + replacements[0] = X{"a", "x", 1}; + replacements[1] = X{"b", "y", 0}; + replacements[2] = X{"c", "z", -1}; + + std::string s = absl::StrReplaceAll("abc", replacements); + EXPECT_EQ(s, "xyz"); + } + + { + std::vector<Cont> replacements(3); + replacements[0] = Cont{"a:x"}; + replacements[1] = Cont{"b:y"}; + replacements[2] = Cont{"c:z"}; + + std::string s = absl::StrReplaceAll("abc", replacements); + EXPECT_EQ(s, "xyz"); + } +} + +// Same as above, but using the in-place variant of absl::StrReplaceAll, +// that returns the # of replacements performed. +TEST(StrReplaceAll, Inplace) { + std::string s; + int reps; + + // Empty string. + s = ""; + reps = absl::StrReplaceAll({{"", ""}, {"x", ""}, {"", "y"}, {"x", "y"}}, &s); + EXPECT_EQ(reps, 0); + EXPECT_EQ(s, ""); + + // Empty substring. + s = "abc"; + reps = absl::StrReplaceAll({{"", ""}, {"", "y"}, {"x", ""}}, &s); + EXPECT_EQ(reps, 0); + EXPECT_EQ(s, "abc"); + + // Replace entire string, one char at a time + s = "abc"; + reps = absl::StrReplaceAll({{"a", "x"}, {"b", "y"}, {"c", "z"}}, &s); + EXPECT_EQ(reps, 3); + EXPECT_EQ(s, "xyz"); + s = "zxy"; + reps = absl::StrReplaceAll({{"z", "x"}, {"x", "y"}, {"y", "z"}}, &s); + EXPECT_EQ(reps, 3); + EXPECT_EQ(s, "xyz"); + + // Replace once at the start (longer matches take precedence) + s = "abc"; + reps = absl::StrReplaceAll({{"a", "x"}, {"ab", "xy"}, {"abc", "xyz"}}, &s); + EXPECT_EQ(reps, 1); + EXPECT_EQ(s, "xyz"); + + // Replace once in the middle. + s = "Abc!"; + reps = absl::StrReplaceAll( + {{"a", "x"}, {"ab", "xy"}, {"b", "y"}, {"bc", "yz"}, {"c", "z"}}, &s); + EXPECT_EQ(reps, 1); + EXPECT_EQ(s, "Ayz!"); + + // Replace once at the end. + s = "Abc!"; + reps = absl::StrReplaceAll( + {{"a", "x"}, {"ab", "xy"}, {"b", "y"}, {"bc!", "yz?"}, {"c!", "z;"}}, &s); + EXPECT_EQ(reps, 1); + EXPECT_EQ(s, "Ayz?"); + + // Replace multiple times with varying lengths of original/replacement. + s = "ababa"; + reps = absl::StrReplaceAll({{"a", "xxx"}, {"b", "XXXX"}}, &s); + EXPECT_EQ(reps, 5); + EXPECT_EQ(s, "xxxXXXXxxxXXXXxxx"); + + // Overlapping matches are replaced greedily. + s = "aaa"; + reps = absl::StrReplaceAll({{"aa", "x"}, {"a", "X"}}, &s); + EXPECT_EQ(reps, 2); + EXPECT_EQ(s, "xX"); + s = "aaa"; + reps = absl::StrReplaceAll({{"a", "X"}, {"aa", "x"}}, &s); + EXPECT_EQ(reps, 2); + EXPECT_EQ(s, "xX"); + + // Two well-known sentences + s = "the quick brown fox jumped over the lazy dogs"; + reps = absl::StrReplaceAll( + { + {"brown", "box"}, + {"dogs", "jugs"}, + {"fox", "with"}, + {"jumped", "five"}, + {"over", "dozen"}, + {"quick", "my"}, + {"the", "pack"}, + {"the lazy", "liquor"}, + }, + &s); + EXPECT_EQ(reps, 8); + EXPECT_EQ(s, "pack my box with five dozen liquor jugs"); +} diff --git a/third_party/abseil_cpp/absl/strings/str_split.cc b/third_party/abseil_cpp/absl/strings/str_split.cc new file mode 100644 index 000000000000..e08c26b6bbbe --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_split.cc @@ -0,0 +1,139 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_split.h" + +#include <algorithm> +#include <cassert> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <iterator> +#include <limits> +#include <memory> + +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/ascii.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +// This GenericFind() template function encapsulates the finding algorithm +// shared between the ByString and ByAnyChar delimiters. The FindPolicy +// template parameter allows each delimiter to customize the actual find +// function to use and the length of the found delimiter. For example, the +// Literal delimiter will ultimately use absl::string_view::find(), and the +// AnyOf delimiter will use absl::string_view::find_first_of(). +template <typename FindPolicy> +absl::string_view GenericFind(absl::string_view text, + absl::string_view delimiter, size_t pos, + FindPolicy find_policy) { + if (delimiter.empty() && text.length() > 0) { + // Special case for empty string delimiters: always return a zero-length + // absl::string_view referring to the item at position 1 past pos. + return absl::string_view(text.data() + pos + 1, 0); + } + size_t found_pos = absl::string_view::npos; + absl::string_view found(text.data() + text.size(), + 0); // By default, not found + found_pos = find_policy.Find(text, delimiter, pos); + if (found_pos != absl::string_view::npos) { + found = absl::string_view(text.data() + found_pos, + find_policy.Length(delimiter)); + } + return found; +} + +// Finds using absl::string_view::find(), therefore the length of the found +// delimiter is delimiter.length(). +struct LiteralPolicy { + size_t Find(absl::string_view text, absl::string_view delimiter, size_t pos) { + return text.find(delimiter, pos); + } + size_t Length(absl::string_view delimiter) { return delimiter.length(); } +}; + +// Finds using absl::string_view::find_first_of(), therefore the length of the +// found delimiter is 1. +struct AnyOfPolicy { + size_t Find(absl::string_view text, absl::string_view delimiter, size_t pos) { + return text.find_first_of(delimiter, pos); + } + size_t Length(absl::string_view /* delimiter */) { return 1; } +}; + +} // namespace + +// +// ByString +// + +ByString::ByString(absl::string_view sp) : delimiter_(sp) {} + +absl::string_view ByString::Find(absl::string_view text, size_t pos) const { + if (delimiter_.length() == 1) { + // Much faster to call find on a single character than on an + // absl::string_view. + size_t found_pos = text.find(delimiter_[0], pos); + if (found_pos == absl::string_view::npos) + return absl::string_view(text.data() + text.size(), 0); + return text.substr(found_pos, 1); + } + return GenericFind(text, delimiter_, pos, LiteralPolicy()); +} + +// +// ByChar +// + +absl::string_view ByChar::Find(absl::string_view text, size_t pos) const { + size_t found_pos = text.find(c_, pos); + if (found_pos == absl::string_view::npos) + return absl::string_view(text.data() + text.size(), 0); + return text.substr(found_pos, 1); +} + +// +// ByAnyChar +// + +ByAnyChar::ByAnyChar(absl::string_view sp) : delimiters_(sp) {} + +absl::string_view ByAnyChar::Find(absl::string_view text, size_t pos) const { + return GenericFind(text, delimiters_, pos, AnyOfPolicy()); +} + +// +// ByLength +// +ByLength::ByLength(ptrdiff_t length) : length_(length) { + ABSL_RAW_CHECK(length > 0, ""); +} + +absl::string_view ByLength::Find(absl::string_view text, + size_t pos) const { + pos = std::min(pos, text.size()); // truncate `pos` + absl::string_view substr = text.substr(pos); + // If the string is shorter than the chunk size we say we + // "can't find the delimiter" so this will be the last chunk. + if (substr.length() <= static_cast<size_t>(length_)) + return absl::string_view(text.data() + text.size(), 0); + + return absl::string_view(substr.data() + length_, 0); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/str_split.h b/third_party/abseil_cpp/absl/strings/str_split.h new file mode 100644 index 000000000000..bfbca422a8dc --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_split.h @@ -0,0 +1,548 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: str_split.h +// ----------------------------------------------------------------------------- +// +// This file contains functions for splitting strings. It defines the main +// `StrSplit()` function, several delimiters for determining the boundaries on +// which to split the string, and predicates for filtering delimited results. +// `StrSplit()` adapts the returned collection to the type specified by the +// caller. +// +// Example: +// +// // Splits the given string on commas. Returns the results in a +// // vector of strings. +// std::vector<std::string> v = absl::StrSplit("a,b,c", ','); +// // Can also use "," +// // v[0] == "a", v[1] == "b", v[2] == "c" +// +// See StrSplit() below for more information. +#ifndef ABSL_STRINGS_STR_SPLIT_H_ +#define ABSL_STRINGS_STR_SPLIT_H_ + +#include <algorithm> +#include <cstddef> +#include <map> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/strings/internal/str_split_internal.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +//------------------------------------------------------------------------------ +// Delimiters +//------------------------------------------------------------------------------ +// +// `StrSplit()` uses delimiters to define the boundaries between elements in the +// provided input. Several `Delimiter` types are defined below. If a string +// (`const char*`, `std::string`, or `absl::string_view`) is passed in place of +// an explicit `Delimiter` object, `StrSplit()` treats it the same way as if it +// were passed a `ByString` delimiter. +// +// A `Delimiter` is an object with a `Find()` function that knows how to find +// the first occurrence of itself in a given `absl::string_view`. +// +// The following `Delimiter` types are available for use within `StrSplit()`: +// +// - `ByString` (default for string arguments) +// - `ByChar` (default for a char argument) +// - `ByAnyChar` +// - `ByLength` +// - `MaxSplits` +// +// A Delimiter's `Find()` member function will be passed an input `text` that is +// to be split and a position (`pos`) to begin searching for the next delimiter +// in `text`. The returned absl::string_view should refer to the next occurrence +// (after `pos`) of the represented delimiter; this returned absl::string_view +// represents the next location where the input `text` should be broken. +// +// The returned absl::string_view may be zero-length if the Delimiter does not +// represent a part of the string (e.g., a fixed-length delimiter). If no +// delimiter is found in the input `text`, a zero-length absl::string_view +// referring to `text.end()` should be returned (e.g., +// `text.substr(text.size())`). It is important that the returned +// absl::string_view always be within the bounds of the input `text` given as an +// argument--it must not refer to a string that is physically located outside of +// the given string. +// +// The following example is a simple Delimiter object that is created with a +// single char and will look for that char in the text passed to the `Find()` +// function: +// +// struct SimpleDelimiter { +// const char c_; +// explicit SimpleDelimiter(char c) : c_(c) {} +// absl::string_view Find(absl::string_view text, size_t pos) { +// auto found = text.find(c_, pos); +// if (found == absl::string_view::npos) +// return text.substr(text.size()); +// +// return text.substr(found, 1); +// } +// }; + +// ByString +// +// A sub-string delimiter. If `StrSplit()` is passed a string in place of a +// `Delimiter` object, the string will be implicitly converted into a +// `ByString` delimiter. +// +// Example: +// +// // Because a string literal is converted to an `absl::ByString`, +// // the following two splits are equivalent. +// +// std::vector<std::string> v1 = absl::StrSplit("a, b, c", ", "); +// +// using absl::ByString; +// std::vector<std::string> v2 = absl::StrSplit("a, b, c", +// ByString(", ")); +// // v[0] == "a", v[1] == "b", v[2] == "c" +class ByString { + public: + explicit ByString(absl::string_view sp); + absl::string_view Find(absl::string_view text, size_t pos) const; + + private: + const std::string delimiter_; +}; + +// ByChar +// +// A single character delimiter. `ByChar` is functionally equivalent to a +// 1-char string within a `ByString` delimiter, but slightly more efficient. +// +// Example: +// +// // Because a char literal is converted to a absl::ByChar, +// // the following two splits are equivalent. +// std::vector<std::string> v1 = absl::StrSplit("a,b,c", ','); +// using absl::ByChar; +// std::vector<std::string> v2 = absl::StrSplit("a,b,c", ByChar(',')); +// // v[0] == "a", v[1] == "b", v[2] == "c" +// +// `ByChar` is also the default delimiter if a single character is given +// as the delimiter to `StrSplit()`. For example, the following calls are +// equivalent: +// +// std::vector<std::string> v = absl::StrSplit("a-b", '-'); +// +// using absl::ByChar; +// std::vector<std::string> v = absl::StrSplit("a-b", ByChar('-')); +// +class ByChar { + public: + explicit ByChar(char c) : c_(c) {} + absl::string_view Find(absl::string_view text, size_t pos) const; + + private: + char c_; +}; + +// ByAnyChar +// +// A delimiter that will match any of the given byte-sized characters within +// its provided string. +// +// Note: this delimiter works with single-byte string data, but does not work +// with variable-width encodings, such as UTF-8. +// +// Example: +// +// using absl::ByAnyChar; +// std::vector<std::string> v = absl::StrSplit("a,b=c", ByAnyChar(",=")); +// // v[0] == "a", v[1] == "b", v[2] == "c" +// +// If `ByAnyChar` is given the empty string, it behaves exactly like +// `ByString` and matches each individual character in the input string. +// +class ByAnyChar { + public: + explicit ByAnyChar(absl::string_view sp); + absl::string_view Find(absl::string_view text, size_t pos) const; + + private: + const std::string delimiters_; +}; + +// ByLength +// +// A delimiter for splitting into equal-length strings. The length argument to +// the constructor must be greater than 0. +// +// Note: this delimiter works with single-byte string data, but does not work +// with variable-width encodings, such as UTF-8. +// +// Example: +// +// using absl::ByLength; +// std::vector<std::string> v = absl::StrSplit("123456789", ByLength(3)); + +// // v[0] == "123", v[1] == "456", v[2] == "789" +// +// Note that the string does not have to be a multiple of the fixed split +// length. In such a case, the last substring will be shorter. +// +// using absl::ByLength; +// std::vector<std::string> v = absl::StrSplit("12345", ByLength(2)); +// +// // v[0] == "12", v[1] == "34", v[2] == "5" +class ByLength { + public: + explicit ByLength(ptrdiff_t length); + absl::string_view Find(absl::string_view text, size_t pos) const; + + private: + const ptrdiff_t length_; +}; + +namespace strings_internal { + +// A traits-like metafunction for selecting the default Delimiter object type +// for a particular Delimiter type. The base case simply exposes type Delimiter +// itself as the delimiter's Type. However, there are specializations for +// string-like objects that map them to the ByString delimiter object. +// This allows functions like absl::StrSplit() and absl::MaxSplits() to accept +// string-like objects (e.g., ',') as delimiter arguments but they will be +// treated as if a ByString delimiter was given. +template <typename Delimiter> +struct SelectDelimiter { + using type = Delimiter; +}; + +template <> +struct SelectDelimiter<char> { + using type = ByChar; +}; +template <> +struct SelectDelimiter<char*> { + using type = ByString; +}; +template <> +struct SelectDelimiter<const char*> { + using type = ByString; +}; +template <> +struct SelectDelimiter<absl::string_view> { + using type = ByString; +}; +template <> +struct SelectDelimiter<std::string> { + using type = ByString; +}; + +// Wraps another delimiter and sets a max number of matches for that delimiter. +template <typename Delimiter> +class MaxSplitsImpl { + public: + MaxSplitsImpl(Delimiter delimiter, int limit) + : delimiter_(delimiter), limit_(limit), count_(0) {} + absl::string_view Find(absl::string_view text, size_t pos) { + if (count_++ == limit_) { + return absl::string_view(text.data() + text.size(), + 0); // No more matches. + } + return delimiter_.Find(text, pos); + } + + private: + Delimiter delimiter_; + const int limit_; + int count_; +}; + +} // namespace strings_internal + +// MaxSplits() +// +// A delimiter that limits the number of matches which can occur to the passed +// `limit`. The last element in the returned collection will contain all +// remaining unsplit pieces, which may contain instances of the delimiter. +// The collection will contain at most `limit` + 1 elements. +// Example: +// +// using absl::MaxSplits; +// std::vector<std::string> v = absl::StrSplit("a,b,c", MaxSplits(',', 1)); +// +// // v[0] == "a", v[1] == "b,c" +template <typename Delimiter> +inline strings_internal::MaxSplitsImpl< + typename strings_internal::SelectDelimiter<Delimiter>::type> +MaxSplits(Delimiter delimiter, int limit) { + typedef + typename strings_internal::SelectDelimiter<Delimiter>::type DelimiterType; + return strings_internal::MaxSplitsImpl<DelimiterType>( + DelimiterType(delimiter), limit); +} + +//------------------------------------------------------------------------------ +// Predicates +//------------------------------------------------------------------------------ +// +// Predicates filter the results of a `StrSplit()` by determining whether or not +// a resultant element is included in the result set. A predicate may be passed +// as an optional third argument to the `StrSplit()` function. +// +// Predicates are unary functions (or functors) that take a single +// `absl::string_view` argument and return a bool indicating whether the +// argument should be included (`true`) or excluded (`false`). +// +// Predicates are useful when filtering out empty substrings. By default, empty +// substrings may be returned by `StrSplit()`, which is similar to the way split +// functions work in other programming languages. + +// AllowEmpty() +// +// Always returns `true`, indicating that all strings--including empty +// strings--should be included in the split output. This predicate is not +// strictly needed because this is the default behavior of `StrSplit()`; +// however, it might be useful at some call sites to make the intent explicit. +// +// Example: +// +// std::vector<std::string> v = absl::StrSplit(" a , ,,b,", ',', AllowEmpty()); +// +// // v[0] == " a ", v[1] == " ", v[2] == "", v[3] = "b", v[4] == "" +struct AllowEmpty { + bool operator()(absl::string_view) const { return true; } +}; + +// SkipEmpty() +// +// Returns `false` if the given `absl::string_view` is empty, indicating that +// `StrSplit()` should omit the empty string. +// +// Example: +// +// std::vector<std::string> v = absl::StrSplit(",a,,b,", ',', SkipEmpty()); +// +// // v[0] == "a", v[1] == "b" +// +// Note: `SkipEmpty()` does not consider a string containing only whitespace +// to be empty. To skip such whitespace as well, use the `SkipWhitespace()` +// predicate. +struct SkipEmpty { + bool operator()(absl::string_view sp) const { return !sp.empty(); } +}; + +// SkipWhitespace() +// +// Returns `false` if the given `absl::string_view` is empty *or* contains only +// whitespace, indicating that `StrSplit()` should omit the string. +// +// Example: +// +// std::vector<std::string> v = absl::StrSplit(" a , ,,b,", +// ',', SkipWhitespace()); +// // v[0] == " a ", v[1] == "b" +// +// // SkipEmpty() would return whitespace elements +// std::vector<std::string> v = absl::StrSplit(" a , ,,b,", ',', SkipEmpty()); +// // v[0] == " a ", v[1] == " ", v[2] == "b" +struct SkipWhitespace { + bool operator()(absl::string_view sp) const { + sp = absl::StripAsciiWhitespace(sp); + return !sp.empty(); + } +}; + +template <typename T> +using EnableSplitIfString = + typename std::enable_if<std::is_same<T, std::string>::value || + std::is_same<T, const std::string>::value, + int>::type; + +//------------------------------------------------------------------------------ +// StrSplit() +//------------------------------------------------------------------------------ + +// StrSplit() +// +// Splits a given string based on the provided `Delimiter` object, returning the +// elements within the type specified by the caller. Optionally, you may pass a +// `Predicate` to `StrSplit()` indicating whether to include or exclude the +// resulting element within the final result set. (See the overviews for +// Delimiters and Predicates above.) +// +// Example: +// +// std::vector<std::string> v = absl::StrSplit("a,b,c,d", ','); +// // v[0] == "a", v[1] == "b", v[2] == "c", v[3] == "d" +// +// You can also provide an explicit `Delimiter` object: +// +// Example: +// +// using absl::ByAnyChar; +// std::vector<std::string> v = absl::StrSplit("a,b=c", ByAnyChar(",=")); +// // v[0] == "a", v[1] == "b", v[2] == "c" +// +// See above for more information on delimiters. +// +// By default, empty strings are included in the result set. You can optionally +// include a third `Predicate` argument to apply a test for whether the +// resultant element should be included in the result set: +// +// Example: +// +// std::vector<std::string> v = absl::StrSplit(" a , ,,b,", +// ',', SkipWhitespace()); +// // v[0] == " a ", v[1] == "b" +// +// See above for more information on predicates. +// +//------------------------------------------------------------------------------ +// StrSplit() Return Types +//------------------------------------------------------------------------------ +// +// The `StrSplit()` function adapts the returned collection to the collection +// specified by the caller (e.g. `std::vector` above). The returned collections +// may contain `std::string`, `absl::string_view` (in which case the original +// string being split must ensure that it outlives the collection), or any +// object that can be explicitly created from an `absl::string_view`. This +// behavior works for: +// +// 1) All standard STL containers including `std::vector`, `std::list`, +// `std::deque`, `std::set`,`std::multiset`, 'std::map`, and `std::multimap` +// 2) `std::pair` (which is not actually a container). See below. +// +// Example: +// +// // The results are returned as `absl::string_view` objects. Note that we +// // have to ensure that the input string outlives any results. +// std::vector<absl::string_view> v = absl::StrSplit("a,b,c", ','); +// +// // Stores results in a std::set<std::string>, which also performs +// // de-duplication and orders the elements in ascending order. +// std::set<std::string> a = absl::StrSplit("b,a,c,a,b", ','); +// // v[0] == "a", v[1] == "b", v[2] = "c" +// +// // `StrSplit()` can be used within a range-based for loop, in which case +// // each element will be of type `absl::string_view`. +// std::vector<std::string> v; +// for (const auto sv : absl::StrSplit("a,b,c", ',')) { +// if (sv != "b") v.emplace_back(sv); +// } +// // v[0] == "a", v[1] == "c" +// +// // Stores results in a map. The map implementation assumes that the input +// // is provided as a series of key/value pairs. For example, the 0th element +// // resulting from the split will be stored as a key to the 1st element. If +// // an odd number of elements are resolved, the last element is paired with +// // a default-constructed value (e.g., empty string). +// std::map<std::string, std::string> m = absl::StrSplit("a,b,c", ','); +// // m["a"] == "b", m["c"] == "" // last component value equals "" +// +// Splitting to `std::pair` is an interesting case because it can hold only two +// elements and is not a collection type. When splitting to a `std::pair` the +// first two split strings become the `std::pair` `.first` and `.second` +// members, respectively. The remaining split substrings are discarded. If there +// are less than two split substrings, the empty string is used for the +// corresponding +// `std::pair` member. +// +// Example: +// +// // Stores first two split strings as the members in a std::pair. +// std::pair<std::string, std::string> p = absl::StrSplit("a,b,c", ','); +// // p.first == "a", p.second == "b" // "c" is omitted. +// +// The `StrSplit()` function can be used multiple times to perform more +// complicated splitting logic, such as intelligently parsing key-value pairs. +// +// Example: +// +// // The input string "a=b=c,d=e,f=,g" becomes +// // { "a" => "b=c", "d" => "e", "f" => "", "g" => "" } +// std::map<std::string, std::string> m; +// for (absl::string_view sp : absl::StrSplit("a=b=c,d=e,f=,g", ',')) { +// m.insert(absl::StrSplit(sp, absl::MaxSplits('=', 1))); +// } +// EXPECT_EQ("b=c", m.find("a")->second); +// EXPECT_EQ("e", m.find("d")->second); +// EXPECT_EQ("", m.find("f")->second); +// EXPECT_EQ("", m.find("g")->second); +// +// WARNING: Due to a legacy bug that is maintained for backward compatibility, +// splitting the following empty string_views produces different results: +// +// absl::StrSplit(absl::string_view(""), '-'); // {""} +// absl::StrSplit(absl::string_view(), '-'); // {}, but should be {""} +// +// Try not to depend on this distinction because the bug may one day be fixed. +template <typename Delimiter> +strings_internal::Splitter< + typename strings_internal::SelectDelimiter<Delimiter>::type, AllowEmpty, + absl::string_view> +StrSplit(strings_internal::ConvertibleToStringView text, Delimiter d) { + using DelimiterType = + typename strings_internal::SelectDelimiter<Delimiter>::type; + return strings_internal::Splitter<DelimiterType, AllowEmpty, + absl::string_view>( + text.value(), DelimiterType(d), AllowEmpty()); +} + +template <typename Delimiter, typename StringType, + EnableSplitIfString<StringType> = 0> +strings_internal::Splitter< + typename strings_internal::SelectDelimiter<Delimiter>::type, AllowEmpty, + std::string> +StrSplit(StringType&& text, Delimiter d) { + using DelimiterType = + typename strings_internal::SelectDelimiter<Delimiter>::type; + return strings_internal::Splitter<DelimiterType, AllowEmpty, std::string>( + std::move(text), DelimiterType(d), AllowEmpty()); +} + +template <typename Delimiter, typename Predicate> +strings_internal::Splitter< + typename strings_internal::SelectDelimiter<Delimiter>::type, Predicate, + absl::string_view> +StrSplit(strings_internal::ConvertibleToStringView text, Delimiter d, + Predicate p) { + using DelimiterType = + typename strings_internal::SelectDelimiter<Delimiter>::type; + return strings_internal::Splitter<DelimiterType, Predicate, + absl::string_view>( + text.value(), DelimiterType(d), std::move(p)); +} + +template <typename Delimiter, typename Predicate, typename StringType, + EnableSplitIfString<StringType> = 0> +strings_internal::Splitter< + typename strings_internal::SelectDelimiter<Delimiter>::type, Predicate, + std::string> +StrSplit(StringType&& text, Delimiter d, Predicate p) { + using DelimiterType = + typename strings_internal::SelectDelimiter<Delimiter>::type; + return strings_internal::Splitter<DelimiterType, Predicate, std::string>( + std::move(text), DelimiterType(d), std::move(p)); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STR_SPLIT_H_ diff --git a/third_party/abseil_cpp/absl/strings/str_split_benchmark.cc b/third_party/abseil_cpp/absl/strings/str_split_benchmark.cc new file mode 100644 index 000000000000..f38dfcfe5af7 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_split_benchmark.cc @@ -0,0 +1,180 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_split.h" + +#include <iterator> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/string_view.h" + +namespace { + +std::string MakeTestString(int desired_length) { + static const int kAverageValueLen = 25; + std::string test(desired_length * kAverageValueLen, 'x'); + for (int i = 1; i < test.size(); i += kAverageValueLen) { + test[i] = ';'; + } + return test; +} + +void BM_Split2StringView(benchmark::State& state) { + std::string test = MakeTestString(state.range(0)); + for (auto _ : state) { + std::vector<absl::string_view> result = absl::StrSplit(test, ';'); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_Split2StringView, 0, 1 << 20); + +static const absl::string_view kDelimiters = ";:,."; + +std::string MakeMultiDelimiterTestString(int desired_length) { + static const int kAverageValueLen = 25; + std::string test(desired_length * kAverageValueLen, 'x'); + for (int i = 0; i * kAverageValueLen < test.size(); ++i) { + // Cycle through a variety of delimiters. + test[i * kAverageValueLen] = kDelimiters[i % kDelimiters.size()]; + } + return test; +} + +// Measure StrSplit with ByAnyChar with four delimiters to choose from. +void BM_Split2StringViewByAnyChar(benchmark::State& state) { + std::string test = MakeMultiDelimiterTestString(state.range(0)); + for (auto _ : state) { + std::vector<absl::string_view> result = + absl::StrSplit(test, absl::ByAnyChar(kDelimiters)); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_Split2StringViewByAnyChar, 0, 1 << 20); + +void BM_Split2StringViewLifted(benchmark::State& state) { + std::string test = MakeTestString(state.range(0)); + std::vector<absl::string_view> result; + for (auto _ : state) { + result = absl::StrSplit(test, ';'); + } + benchmark::DoNotOptimize(result); +} +BENCHMARK_RANGE(BM_Split2StringViewLifted, 0, 1 << 20); + +void BM_Split2String(benchmark::State& state) { + std::string test = MakeTestString(state.range(0)); + for (auto _ : state) { + std::vector<std::string> result = absl::StrSplit(test, ';'); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_Split2String, 0, 1 << 20); + +// This benchmark is for comparing Split2 to Split1 (SplitStringUsing). In +// particular, this benchmark uses SkipEmpty() to match SplitStringUsing's +// behavior. +void BM_Split2SplitStringUsing(benchmark::State& state) { + std::string test = MakeTestString(state.range(0)); + for (auto _ : state) { + std::vector<std::string> result = + absl::StrSplit(test, ';', absl::SkipEmpty()); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_Split2SplitStringUsing, 0, 1 << 20); + +void BM_SplitStringToUnorderedSet(benchmark::State& state) { + const int len = state.range(0); + std::string test(len, 'x'); + for (int i = 1; i < len; i += 2) { + test[i] = ';'; + } + for (auto _ : state) { + std::unordered_set<std::string> result = + absl::StrSplit(test, ':', absl::SkipEmpty()); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_SplitStringToUnorderedSet, 0, 1 << 20); + +void BM_SplitStringToUnorderedMap(benchmark::State& state) { + const int len = state.range(0); + std::string test(len, 'x'); + for (int i = 1; i < len; i += 2) { + test[i] = ';'; + } + for (auto _ : state) { + std::unordered_map<std::string, std::string> result = + absl::StrSplit(test, ':', absl::SkipEmpty()); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_SplitStringToUnorderedMap, 0, 1 << 20); + +void BM_SplitStringAllowEmpty(benchmark::State& state) { + const int len = state.range(0); + std::string test(len, 'x'); + for (int i = 1; i < len; i += 2) { + test[i] = ';'; + } + for (auto _ : state) { + std::vector<std::string> result = absl::StrSplit(test, ';'); + benchmark::DoNotOptimize(result); + } +} +BENCHMARK_RANGE(BM_SplitStringAllowEmpty, 0, 1 << 20); + +struct OneCharLiteral { + char operator()() const { return 'X'; } +}; + +struct OneCharStringLiteral { + const char* operator()() const { return "X"; } +}; + +template <typename DelimiterFactory> +void BM_SplitStringWithOneChar(benchmark::State& state) { + const auto delimiter = DelimiterFactory()(); + std::vector<absl::string_view> pieces; + size_t v = 0; + for (auto _ : state) { + pieces = absl::StrSplit("The quick brown fox jumps over the lazy dog", + delimiter); + v += pieces.size(); + } + ABSL_RAW_CHECK(v == state.iterations(), ""); +} +BENCHMARK_TEMPLATE(BM_SplitStringWithOneChar, OneCharLiteral); +BENCHMARK_TEMPLATE(BM_SplitStringWithOneChar, OneCharStringLiteral); + +template <typename DelimiterFactory> +void BM_SplitStringWithOneCharNoVector(benchmark::State& state) { + const auto delimiter = DelimiterFactory()(); + size_t v = 0; + for (auto _ : state) { + auto splitter = absl::StrSplit( + "The quick brown fox jumps over the lazy dog", delimiter); + v += std::distance(splitter.begin(), splitter.end()); + } + ABSL_RAW_CHECK(v == state.iterations(), ""); +} +BENCHMARK_TEMPLATE(BM_SplitStringWithOneCharNoVector, OneCharLiteral); +BENCHMARK_TEMPLATE(BM_SplitStringWithOneCharNoVector, OneCharStringLiteral); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/str_split_test.cc b/third_party/abseil_cpp/absl/strings/str_split_test.cc new file mode 100644 index 000000000000..7f7c097faee2 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/str_split_test.cc @@ -0,0 +1,953 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/str_split.h" + +#include <deque> +#include <initializer_list> +#include <list> +#include <map> +#include <memory> +#include <string> +#include <type_traits> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/dynamic_annotations.h" +#include "absl/base/macros.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/node_hash_map.h" +#include "absl/strings/numbers.h" + +namespace { + +using ::testing::ElementsAre; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +TEST(Split, TraitsTest) { + static_assert(!absl::strings_internal::SplitterIsConvertibleTo<int>::value, + ""); + static_assert( + !absl::strings_internal::SplitterIsConvertibleTo<std::string>::value, ""); + static_assert(absl::strings_internal::SplitterIsConvertibleTo< + std::vector<std::string>>::value, + ""); + static_assert( + !absl::strings_internal::SplitterIsConvertibleTo<std::vector<int>>::value, + ""); + static_assert(absl::strings_internal::SplitterIsConvertibleTo< + std::vector<absl::string_view>>::value, + ""); + static_assert(absl::strings_internal::SplitterIsConvertibleTo< + std::map<std::string, std::string>>::value, + ""); + static_assert(absl::strings_internal::SplitterIsConvertibleTo< + std::map<absl::string_view, absl::string_view>>::value, + ""); + static_assert(!absl::strings_internal::SplitterIsConvertibleTo< + std::map<int, std::string>>::value, + ""); + static_assert(!absl::strings_internal::SplitterIsConvertibleTo< + std::map<std::string, int>>::value, + ""); +} + +// This tests the overall split API, which is made up of the absl::StrSplit() +// function and the Delimiter objects in the absl:: namespace. +// This TEST macro is outside of any namespace to require full specification of +// namespaces just like callers will need to use. +TEST(Split, APIExamples) { + { + // Passes string delimiter. Assumes the default of ByString. + std::vector<std::string> v = absl::StrSplit("a,b,c", ","); // NOLINT + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + + // Equivalent to... + using absl::ByString; + v = absl::StrSplit("a,b,c", ByString(",")); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + + // Equivalent to... + EXPECT_THAT(absl::StrSplit("a,b,c", ByString(",")), + ElementsAre("a", "b", "c")); + } + + { + // Same as above, but using a single character as the delimiter. + std::vector<std::string> v = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + + // Equivalent to... + using absl::ByChar; + v = absl::StrSplit("a,b,c", ByChar(',')); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Uses the Literal string "=>" as the delimiter. + const std::vector<std::string> v = absl::StrSplit("a=>b=>c", "=>"); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // The substrings are returned as string_views, eliminating copying. + std::vector<absl::string_view> v = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Leading and trailing empty substrings. + std::vector<std::string> v = absl::StrSplit(",a,b,c,", ','); + EXPECT_THAT(v, ElementsAre("", "a", "b", "c", "")); + } + + { + // Splits on a delimiter that is not found. + std::vector<std::string> v = absl::StrSplit("abc", ','); + EXPECT_THAT(v, ElementsAre("abc")); + } + + { + // Splits the input string into individual characters by using an empty + // string as the delimiter. + std::vector<std::string> v = absl::StrSplit("abc", ""); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Splits string data with embedded NUL characters, using NUL as the + // delimiter. A simple delimiter of "\0" doesn't work because strlen() will + // say that's the empty string when constructing the absl::string_view + // delimiter. Instead, a non-empty string containing NUL can be used as the + // delimiter. + std::string embedded_nulls("a\0b\0c", 5); + std::string null_delim("\0", 1); + std::vector<std::string> v = absl::StrSplit(embedded_nulls, null_delim); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Stores first two split strings as the members in a std::pair. + std::pair<std::string, std::string> p = absl::StrSplit("a,b,c", ','); + EXPECT_EQ("a", p.first); + EXPECT_EQ("b", p.second); + // "c" is omitted because std::pair can hold only two elements. + } + + { + // Results stored in std::set<std::string> + std::set<std::string> v = absl::StrSplit("a,b,c,a,b,c,a,b,c", ','); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Uses a non-const char* delimiter. + char a[] = ","; + char* d = a + 0; + std::vector<std::string> v = absl::StrSplit("a,b,c", d); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Results split using either of , or ; + using absl::ByAnyChar; + std::vector<std::string> v = absl::StrSplit("a,b;c", ByAnyChar(",;")); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Uses the SkipWhitespace predicate. + using absl::SkipWhitespace; + std::vector<std::string> v = + absl::StrSplit(" a , ,,b,", ',', SkipWhitespace()); + EXPECT_THAT(v, ElementsAre(" a ", "b")); + } + + { + // Uses the ByLength delimiter. + using absl::ByLength; + std::vector<std::string> v = absl::StrSplit("abcdefg", ByLength(3)); + EXPECT_THAT(v, ElementsAre("abc", "def", "g")); + } + + { + // Different forms of initialization / conversion. + std::vector<std::string> v1 = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(v1, ElementsAre("a", "b", "c")); + std::vector<std::string> v2(absl::StrSplit("a,b,c", ',')); + EXPECT_THAT(v2, ElementsAre("a", "b", "c")); + auto v3 = std::vector<std::string>(absl::StrSplit("a,b,c", ',')); + EXPECT_THAT(v3, ElementsAre("a", "b", "c")); + v3 = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(v3, ElementsAre("a", "b", "c")); + } + + { + // Results stored in a std::map. + std::map<std::string, std::string> m = absl::StrSplit("a,1,b,2,a,3", ','); + EXPECT_EQ(2, m.size()); + EXPECT_EQ("3", m["a"]); + EXPECT_EQ("2", m["b"]); + } + + { + // Results stored in a std::multimap. + std::multimap<std::string, std::string> m = + absl::StrSplit("a,1,b,2,a,3", ','); + EXPECT_EQ(3, m.size()); + auto it = m.find("a"); + EXPECT_EQ("1", it->second); + ++it; + EXPECT_EQ("3", it->second); + it = m.find("b"); + EXPECT_EQ("2", it->second); + } + + { + // Demonstrates use in a range-based for loop in C++11. + std::string s = "x,x,x,x,x,x,x"; + for (absl::string_view sp : absl::StrSplit(s, ',')) { + EXPECT_EQ("x", sp); + } + } + + { + // Demonstrates use with a Predicate in a range-based for loop. + using absl::SkipWhitespace; + std::string s = " ,x,,x,,x,x,x,,"; + for (absl::string_view sp : absl::StrSplit(s, ',', SkipWhitespace())) { + EXPECT_EQ("x", sp); + } + } + + { + // Demonstrates a "smart" split to std::map using two separate calls to + // absl::StrSplit. One call to split the records, and another call to split + // the keys and values. This also uses the Limit delimiter so that the + // std::string "a=b=c" will split to "a" -> "b=c". + std::map<std::string, std::string> m; + for (absl::string_view sp : absl::StrSplit("a=b=c,d=e,f=,g", ',')) { + m.insert(absl::StrSplit(sp, absl::MaxSplits('=', 1))); + } + EXPECT_EQ("b=c", m.find("a")->second); + EXPECT_EQ("e", m.find("d")->second); + EXPECT_EQ("", m.find("f")->second); + EXPECT_EQ("", m.find("g")->second); + } +} + +// +// Tests for SplitIterator +// + +TEST(SplitIterator, Basics) { + auto splitter = absl::StrSplit("a,b", ','); + auto it = splitter.begin(); + auto end = splitter.end(); + + EXPECT_NE(it, end); + EXPECT_EQ("a", *it); // tests dereference + ++it; // tests preincrement + EXPECT_NE(it, end); + EXPECT_EQ("b", + std::string(it->data(), it->size())); // tests dereference as ptr + it++; // tests postincrement + EXPECT_EQ(it, end); +} + +// Simple Predicate to skip a particular string. +class Skip { + public: + explicit Skip(const std::string& s) : s_(s) {} + bool operator()(absl::string_view sp) { return sp != s_; } + + private: + std::string s_; +}; + +TEST(SplitIterator, Predicate) { + auto splitter = absl::StrSplit("a,b,c", ',', Skip("b")); + auto it = splitter.begin(); + auto end = splitter.end(); + + EXPECT_NE(it, end); + EXPECT_EQ("a", *it); // tests dereference + ++it; // tests preincrement -- "b" should be skipped here. + EXPECT_NE(it, end); + EXPECT_EQ("c", + std::string(it->data(), it->size())); // tests dereference as ptr + it++; // tests postincrement + EXPECT_EQ(it, end); +} + +TEST(SplitIterator, EdgeCases) { + // Expected input and output, assuming a delimiter of ',' + struct { + std::string in; + std::vector<std::string> expect; + } specs[] = { + {"", {""}}, + {"foo", {"foo"}}, + {",", {"", ""}}, + {",foo", {"", "foo"}}, + {"foo,", {"foo", ""}}, + {",foo,", {"", "foo", ""}}, + {"foo,bar", {"foo", "bar"}}, + }; + + for (const auto& spec : specs) { + SCOPED_TRACE(spec.in); + auto splitter = absl::StrSplit(spec.in, ','); + auto it = splitter.begin(); + auto end = splitter.end(); + for (const auto& expected : spec.expect) { + EXPECT_NE(it, end); + EXPECT_EQ(expected, *it++); + } + EXPECT_EQ(it, end); + } +} + +TEST(Splitter, Const) { + const auto splitter = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(splitter, ElementsAre("a", "b", "c")); +} + +TEST(Split, EmptyAndNull) { + // Attention: Splitting a null absl::string_view is different than splitting + // an empty absl::string_view even though both string_views are considered + // equal. This behavior is likely surprising and undesirable. However, to + // maintain backward compatibility, there is a small "hack" in + // str_split_internal.h that preserves this behavior. If that behavior is ever + // changed/fixed, this test will need to be updated. + EXPECT_THAT(absl::StrSplit(absl::string_view(""), '-'), ElementsAre("")); + EXPECT_THAT(absl::StrSplit(absl::string_view(), '-'), ElementsAre()); +} + +TEST(SplitIterator, EqualityAsEndCondition) { + auto splitter = absl::StrSplit("a,b,c", ','); + auto it = splitter.begin(); + auto it2 = it; + + // Increments it2 twice to point to "c" in the input text. + ++it2; + ++it2; + EXPECT_EQ("c", *it2); + + // This test uses a non-end SplitIterator as the terminating condition in a + // for loop. This relies on SplitIterator equality for non-end SplitIterators + // working correctly. At this point it2 points to "c", and we use that as the + // "end" condition in this test. + std::vector<absl::string_view> v; + for (; it != it2; ++it) { + v.push_back(*it); + } + EXPECT_THAT(v, ElementsAre("a", "b")); +} + +// +// Tests for Splitter +// + +TEST(Splitter, RangeIterators) { + auto splitter = absl::StrSplit("a,b,c", ','); + std::vector<absl::string_view> output; + for (const absl::string_view& p : splitter) { + output.push_back(p); + } + EXPECT_THAT(output, ElementsAre("a", "b", "c")); +} + +// Some template functions for use in testing conversion operators +template <typename ContainerType, typename Splitter> +void TestConversionOperator(const Splitter& splitter) { + ContainerType output = splitter; + EXPECT_THAT(output, UnorderedElementsAre("a", "b", "c", "d")); +} + +template <typename MapType, typename Splitter> +void TestMapConversionOperator(const Splitter& splitter) { + MapType m = splitter; + EXPECT_THAT(m, UnorderedElementsAre(Pair("a", "b"), Pair("c", "d"))); +} + +template <typename FirstType, typename SecondType, typename Splitter> +void TestPairConversionOperator(const Splitter& splitter) { + std::pair<FirstType, SecondType> p = splitter; + EXPECT_EQ(p, (std::pair<FirstType, SecondType>("a", "b"))); +} + +TEST(Splitter, ConversionOperator) { + auto splitter = absl::StrSplit("a,b,c,d", ','); + + TestConversionOperator<std::vector<absl::string_view>>(splitter); + TestConversionOperator<std::vector<std::string>>(splitter); + TestConversionOperator<std::list<absl::string_view>>(splitter); + TestConversionOperator<std::list<std::string>>(splitter); + TestConversionOperator<std::deque<absl::string_view>>(splitter); + TestConversionOperator<std::deque<std::string>>(splitter); + TestConversionOperator<std::set<absl::string_view>>(splitter); + TestConversionOperator<std::set<std::string>>(splitter); + TestConversionOperator<std::multiset<absl::string_view>>(splitter); + TestConversionOperator<std::multiset<std::string>>(splitter); + TestConversionOperator<std::unordered_set<std::string>>(splitter); + + // Tests conversion to map-like objects. + + TestMapConversionOperator<std::map<absl::string_view, absl::string_view>>( + splitter); + TestMapConversionOperator<std::map<absl::string_view, std::string>>(splitter); + TestMapConversionOperator<std::map<std::string, absl::string_view>>(splitter); + TestMapConversionOperator<std::map<std::string, std::string>>(splitter); + TestMapConversionOperator< + std::multimap<absl::string_view, absl::string_view>>(splitter); + TestMapConversionOperator<std::multimap<absl::string_view, std::string>>( + splitter); + TestMapConversionOperator<std::multimap<std::string, absl::string_view>>( + splitter); + TestMapConversionOperator<std::multimap<std::string, std::string>>(splitter); + TestMapConversionOperator<std::unordered_map<std::string, std::string>>( + splitter); + TestMapConversionOperator< + absl::node_hash_map<absl::string_view, absl::string_view>>(splitter); + TestMapConversionOperator< + absl::node_hash_map<absl::string_view, std::string>>(splitter); + TestMapConversionOperator< + absl::node_hash_map<std::string, absl::string_view>>(splitter); + TestMapConversionOperator< + absl::flat_hash_map<absl::string_view, absl::string_view>>(splitter); + TestMapConversionOperator< + absl::flat_hash_map<absl::string_view, std::string>>(splitter); + TestMapConversionOperator< + absl::flat_hash_map<std::string, absl::string_view>>(splitter); + + // Tests conversion to std::pair + + TestPairConversionOperator<absl::string_view, absl::string_view>(splitter); + TestPairConversionOperator<absl::string_view, std::string>(splitter); + TestPairConversionOperator<std::string, absl::string_view>(splitter); + TestPairConversionOperator<std::string, std::string>(splitter); +} + +// A few additional tests for conversion to std::pair. This conversion is +// different from others because a std::pair always has exactly two elements: +// .first and .second. The split has to work even when the split has +// less-than, equal-to, and more-than 2 strings. +TEST(Splitter, ToPair) { + { + // Empty string + std::pair<std::string, std::string> p = absl::StrSplit("", ','); + EXPECT_EQ("", p.first); + EXPECT_EQ("", p.second); + } + + { + // Only first + std::pair<std::string, std::string> p = absl::StrSplit("a", ','); + EXPECT_EQ("a", p.first); + EXPECT_EQ("", p.second); + } + + { + // Only second + std::pair<std::string, std::string> p = absl::StrSplit(",b", ','); + EXPECT_EQ("", p.first); + EXPECT_EQ("b", p.second); + } + + { + // First and second. + std::pair<std::string, std::string> p = absl::StrSplit("a,b", ','); + EXPECT_EQ("a", p.first); + EXPECT_EQ("b", p.second); + } + + { + // First and second and then more stuff that will be ignored. + std::pair<std::string, std::string> p = absl::StrSplit("a,b,c", ','); + EXPECT_EQ("a", p.first); + EXPECT_EQ("b", p.second); + // "c" is omitted. + } +} + +TEST(Splitter, Predicates) { + static const char kTestChars[] = ",a, ,b,"; + using absl::AllowEmpty; + using absl::SkipEmpty; + using absl::SkipWhitespace; + + { + // No predicate. Does not skip empties. + auto splitter = absl::StrSplit(kTestChars, ','); + std::vector<std::string> v = splitter; + EXPECT_THAT(v, ElementsAre("", "a", " ", "b", "")); + } + + { + // Allows empty strings. Same behavior as no predicate at all. + auto splitter = absl::StrSplit(kTestChars, ',', AllowEmpty()); + std::vector<std::string> v_allowempty = splitter; + EXPECT_THAT(v_allowempty, ElementsAre("", "a", " ", "b", "")); + + // Ensures AllowEmpty equals the behavior with no predicate. + auto splitter_nopredicate = absl::StrSplit(kTestChars, ','); + std::vector<std::string> v_nopredicate = splitter_nopredicate; + EXPECT_EQ(v_allowempty, v_nopredicate); + } + + { + // Skips empty strings. + auto splitter = absl::StrSplit(kTestChars, ',', SkipEmpty()); + std::vector<std::string> v = splitter; + EXPECT_THAT(v, ElementsAre("a", " ", "b")); + } + + { + // Skips empty and all-whitespace strings. + auto splitter = absl::StrSplit(kTestChars, ',', SkipWhitespace()); + std::vector<std::string> v = splitter; + EXPECT_THAT(v, ElementsAre("a", "b")); + } +} + +// +// Tests for StrSplit() +// + +TEST(Split, Basics) { + { + // Doesn't really do anything useful because the return value is ignored, + // but it should work. + absl::StrSplit("a,b,c", ','); + } + + { + std::vector<absl::string_view> v = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + std::vector<std::string> v = absl::StrSplit("a,b,c", ','); + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + } + + { + // Ensures that assignment works. This requires a little extra work with + // C++11 because of overloads with initializer_list. + std::vector<std::string> v; + v = absl::StrSplit("a,b,c", ','); + + EXPECT_THAT(v, ElementsAre("a", "b", "c")); + std::map<std::string, std::string> m; + m = absl::StrSplit("a,b,c", ','); + EXPECT_EQ(2, m.size()); + std::unordered_map<std::string, std::string> hm; + hm = absl::StrSplit("a,b,c", ','); + EXPECT_EQ(2, hm.size()); + } +} + +absl::string_view ReturnStringView() { return "Hello World"; } +const char* ReturnConstCharP() { return "Hello World"; } +char* ReturnCharP() { return const_cast<char*>("Hello World"); } + +TEST(Split, AcceptsCertainTemporaries) { + std::vector<std::string> v; + v = absl::StrSplit(ReturnStringView(), ' '); + EXPECT_THAT(v, ElementsAre("Hello", "World")); + v = absl::StrSplit(ReturnConstCharP(), ' '); + EXPECT_THAT(v, ElementsAre("Hello", "World")); + v = absl::StrSplit(ReturnCharP(), ' '); + EXPECT_THAT(v, ElementsAre("Hello", "World")); +} + +TEST(Split, Temporary) { + // Use a std::string longer than the SSO length, so that when the temporary is + // destroyed, if the splitter keeps a reference to the string's contents, + // it'll reference freed memory instead of just dead on-stack memory. + const char input[] = "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u"; + EXPECT_LT(sizeof(std::string), ABSL_ARRAYSIZE(input)) + << "Input should be larger than fits on the stack."; + + // This happens more often in C++11 as part of a range-based for loop. + auto splitter = absl::StrSplit(std::string(input), ','); + std::string expected = "a"; + for (absl::string_view letter : splitter) { + EXPECT_EQ(expected, letter); + ++expected[0]; + } + EXPECT_EQ("v", expected); + + // This happens more often in C++11 as part of a range-based for loop. + auto std_splitter = absl::StrSplit(std::string(input), ','); + expected = "a"; + for (absl::string_view letter : std_splitter) { + EXPECT_EQ(expected, letter); + ++expected[0]; + } + EXPECT_EQ("v", expected); +} + +template <typename T> +static std::unique_ptr<T> CopyToHeap(const T& value) { + return std::unique_ptr<T>(new T(value)); +} + +TEST(Split, LvalueCaptureIsCopyable) { + std::string input = "a,b"; + auto heap_splitter = CopyToHeap(absl::StrSplit(input, ',')); + auto stack_splitter = *heap_splitter; + heap_splitter.reset(); + std::vector<std::string> result = stack_splitter; + EXPECT_THAT(result, testing::ElementsAre("a", "b")); +} + +TEST(Split, TemporaryCaptureIsCopyable) { + auto heap_splitter = CopyToHeap(absl::StrSplit(std::string("a,b"), ',')); + auto stack_splitter = *heap_splitter; + heap_splitter.reset(); + std::vector<std::string> result = stack_splitter; + EXPECT_THAT(result, testing::ElementsAre("a", "b")); +} + +TEST(Split, SplitterIsCopyableAndMoveable) { + auto a = absl::StrSplit("foo", '-'); + + // Ensures that the following expressions compile. + auto b = a; // Copy construct + auto c = std::move(a); // Move construct + b = c; // Copy assign + c = std::move(b); // Move assign + + EXPECT_THAT(c, ElementsAre("foo")); +} + +TEST(Split, StringDelimiter) { + { + std::vector<absl::string_view> v = absl::StrSplit("a,b", ','); + EXPECT_THAT(v, ElementsAre("a", "b")); + } + + { + std::vector<absl::string_view> v = absl::StrSplit("a,b", std::string(",")); + EXPECT_THAT(v, ElementsAre("a", "b")); + } + + { + std::vector<absl::string_view> v = + absl::StrSplit("a,b", absl::string_view(",")); + EXPECT_THAT(v, ElementsAre("a", "b")); + } +} + +#if !defined(__cpp_char8_t) +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++2a-compat" +#endif +TEST(Split, UTF8) { + // Tests splitting utf8 strings and utf8 delimiters. + std::string utf8_string = u8"\u03BA\u1F79\u03C3\u03BC\u03B5"; + { + // A utf8 input string with an ascii delimiter. + std::string to_split = "a," + utf8_string; + std::vector<absl::string_view> v = absl::StrSplit(to_split, ','); + EXPECT_THAT(v, ElementsAre("a", utf8_string)); + } + + { + // A utf8 input string and a utf8 delimiter. + std::string to_split = "a," + utf8_string + ",b"; + std::string unicode_delimiter = "," + utf8_string + ","; + std::vector<absl::string_view> v = + absl::StrSplit(to_split, unicode_delimiter); + EXPECT_THAT(v, ElementsAre("a", "b")); + } + + { + // A utf8 input string and ByAnyChar with ascii chars. + std::vector<absl::string_view> v = + absl::StrSplit(u8"Foo h\u00E4llo th\u4E1Ere", absl::ByAnyChar(" \t")); + EXPECT_THAT(v, ElementsAre("Foo", u8"h\u00E4llo", u8"th\u4E1Ere")); + } +} +#if defined(__clang__) +#pragma clang diagnostic pop +#endif +#endif // !defined(__cpp_char8_t) + +TEST(Split, EmptyStringDelimiter) { + { + std::vector<std::string> v = absl::StrSplit("", ""); + EXPECT_THAT(v, ElementsAre("")); + } + + { + std::vector<std::string> v = absl::StrSplit("a", ""); + EXPECT_THAT(v, ElementsAre("a")); + } + + { + std::vector<std::string> v = absl::StrSplit("ab", ""); + EXPECT_THAT(v, ElementsAre("a", "b")); + } + + { + std::vector<std::string> v = absl::StrSplit("a b", ""); + EXPECT_THAT(v, ElementsAre("a", " ", "b")); + } +} + +TEST(Split, SubstrDelimiter) { + std::vector<absl::string_view> results; + absl::string_view delim("//"); + + results = absl::StrSplit("", delim); + EXPECT_THAT(results, ElementsAre("")); + + results = absl::StrSplit("//", delim); + EXPECT_THAT(results, ElementsAre("", "")); + + results = absl::StrSplit("ab", delim); + EXPECT_THAT(results, ElementsAre("ab")); + + results = absl::StrSplit("ab//", delim); + EXPECT_THAT(results, ElementsAre("ab", "")); + + results = absl::StrSplit("ab/", delim); + EXPECT_THAT(results, ElementsAre("ab/")); + + results = absl::StrSplit("a/b", delim); + EXPECT_THAT(results, ElementsAre("a/b")); + + results = absl::StrSplit("a//b", delim); + EXPECT_THAT(results, ElementsAre("a", "b")); + + results = absl::StrSplit("a///b", delim); + EXPECT_THAT(results, ElementsAre("a", "/b")); + + results = absl::StrSplit("a////b", delim); + EXPECT_THAT(results, ElementsAre("a", "", "b")); +} + +TEST(Split, EmptyResults) { + std::vector<absl::string_view> results; + + results = absl::StrSplit("", '#'); + EXPECT_THAT(results, ElementsAre("")); + + results = absl::StrSplit("#", '#'); + EXPECT_THAT(results, ElementsAre("", "")); + + results = absl::StrSplit("#cd", '#'); + EXPECT_THAT(results, ElementsAre("", "cd")); + + results = absl::StrSplit("ab#cd#", '#'); + EXPECT_THAT(results, ElementsAre("ab", "cd", "")); + + results = absl::StrSplit("ab##cd", '#'); + EXPECT_THAT(results, ElementsAre("ab", "", "cd")); + + results = absl::StrSplit("ab##", '#'); + EXPECT_THAT(results, ElementsAre("ab", "", "")); + + results = absl::StrSplit("ab#ab#", '#'); + EXPECT_THAT(results, ElementsAre("ab", "ab", "")); + + results = absl::StrSplit("aaaa", 'a'); + EXPECT_THAT(results, ElementsAre("", "", "", "", "")); + + results = absl::StrSplit("", '#', absl::SkipEmpty()); + EXPECT_THAT(results, ElementsAre()); +} + +template <typename Delimiter> +static bool IsFoundAtStartingPos(absl::string_view text, Delimiter d, + size_t starting_pos, int expected_pos) { + absl::string_view found = d.Find(text, starting_pos); + return found.data() != text.data() + text.size() && + expected_pos == found.data() - text.data(); +} + +// Helper function for testing Delimiter objects. Returns true if the given +// Delimiter is found in the given string at the given position. This function +// tests two cases: +// 1. The actual text given, staring at position 0 +// 2. The text given with leading padding that should be ignored +template <typename Delimiter> +static bool IsFoundAt(absl::string_view text, Delimiter d, int expected_pos) { + const std::string leading_text = ",x,y,z,"; + return IsFoundAtStartingPos(text, d, 0, expected_pos) && + IsFoundAtStartingPos(leading_text + std::string(text), d, + leading_text.length(), + expected_pos + leading_text.length()); +} + +// +// Tests for ByString +// + +// Tests using any delimiter that represents a single comma. +template <typename Delimiter> +void TestComma(Delimiter d) { + EXPECT_TRUE(IsFoundAt(",", d, 0)); + EXPECT_TRUE(IsFoundAt("a,", d, 1)); + EXPECT_TRUE(IsFoundAt(",b", d, 0)); + EXPECT_TRUE(IsFoundAt("a,b", d, 1)); + EXPECT_TRUE(IsFoundAt("a,b,", d, 1)); + EXPECT_TRUE(IsFoundAt("a,b,c", d, 1)); + EXPECT_FALSE(IsFoundAt("", d, -1)); + EXPECT_FALSE(IsFoundAt(" ", d, -1)); + EXPECT_FALSE(IsFoundAt("a", d, -1)); + EXPECT_FALSE(IsFoundAt("a b c", d, -1)); + EXPECT_FALSE(IsFoundAt("a;b;c", d, -1)); + EXPECT_FALSE(IsFoundAt(";", d, -1)); +} + +TEST(Delimiter, ByString) { + using absl::ByString; + TestComma(ByString(",")); + + // Works as named variable. + ByString comma_string(","); + TestComma(comma_string); + + // The first occurrence of empty string ("") in a string is at position 0. + // There is a test below that demonstrates this for absl::string_view::find(). + // If the ByString delimiter returned position 0 for this, there would + // be an infinite loop in the SplitIterator code. To avoid this, empty string + // is a special case in that it always returns the item at position 1. + absl::string_view abc("abc"); + EXPECT_EQ(0, abc.find("")); // "" is found at position 0 + ByString empty(""); + EXPECT_FALSE(IsFoundAt("", empty, 0)); + EXPECT_FALSE(IsFoundAt("a", empty, 0)); + EXPECT_TRUE(IsFoundAt("ab", empty, 1)); + EXPECT_TRUE(IsFoundAt("abc", empty, 1)); +} + +TEST(Split, ByChar) { + using absl::ByChar; + TestComma(ByChar(',')); + + // Works as named variable. + ByChar comma_char(','); + TestComma(comma_char); +} + +// +// Tests for ByAnyChar +// + +TEST(Delimiter, ByAnyChar) { + using absl::ByAnyChar; + ByAnyChar one_delim(","); + // Found + EXPECT_TRUE(IsFoundAt(",", one_delim, 0)); + EXPECT_TRUE(IsFoundAt("a,", one_delim, 1)); + EXPECT_TRUE(IsFoundAt("a,b", one_delim, 1)); + EXPECT_TRUE(IsFoundAt(",b", one_delim, 0)); + // Not found + EXPECT_FALSE(IsFoundAt("", one_delim, -1)); + EXPECT_FALSE(IsFoundAt(" ", one_delim, -1)); + EXPECT_FALSE(IsFoundAt("a", one_delim, -1)); + EXPECT_FALSE(IsFoundAt("a;b;c", one_delim, -1)); + EXPECT_FALSE(IsFoundAt(";", one_delim, -1)); + + ByAnyChar two_delims(",;"); + // Found + EXPECT_TRUE(IsFoundAt(",", two_delims, 0)); + EXPECT_TRUE(IsFoundAt(";", two_delims, 0)); + EXPECT_TRUE(IsFoundAt(",;", two_delims, 0)); + EXPECT_TRUE(IsFoundAt(";,", two_delims, 0)); + EXPECT_TRUE(IsFoundAt(",;b", two_delims, 0)); + EXPECT_TRUE(IsFoundAt(";,b", two_delims, 0)); + EXPECT_TRUE(IsFoundAt("a;,", two_delims, 1)); + EXPECT_TRUE(IsFoundAt("a,;", two_delims, 1)); + EXPECT_TRUE(IsFoundAt("a;,b", two_delims, 1)); + EXPECT_TRUE(IsFoundAt("a,;b", two_delims, 1)); + // Not found + EXPECT_FALSE(IsFoundAt("", two_delims, -1)); + EXPECT_FALSE(IsFoundAt(" ", two_delims, -1)); + EXPECT_FALSE(IsFoundAt("a", two_delims, -1)); + EXPECT_FALSE(IsFoundAt("a=b=c", two_delims, -1)); + EXPECT_FALSE(IsFoundAt("=", two_delims, -1)); + + // ByAnyChar behaves just like ByString when given a delimiter of empty + // string. That is, it always returns a zero-length absl::string_view + // referring to the item at position 1, not position 0. + ByAnyChar empty(""); + EXPECT_FALSE(IsFoundAt("", empty, 0)); + EXPECT_FALSE(IsFoundAt("a", empty, 0)); + EXPECT_TRUE(IsFoundAt("ab", empty, 1)); + EXPECT_TRUE(IsFoundAt("abc", empty, 1)); +} + +// +// Tests for ByLength +// + +TEST(Delimiter, ByLength) { + using absl::ByLength; + + ByLength four_char_delim(4); + + // Found + EXPECT_TRUE(IsFoundAt("abcde", four_char_delim, 4)); + EXPECT_TRUE(IsFoundAt("abcdefghijklmnopqrstuvwxyz", four_char_delim, 4)); + EXPECT_TRUE(IsFoundAt("a b,c\nd", four_char_delim, 4)); + // Not found + EXPECT_FALSE(IsFoundAt("", four_char_delim, 0)); + EXPECT_FALSE(IsFoundAt("a", four_char_delim, 0)); + EXPECT_FALSE(IsFoundAt("ab", four_char_delim, 0)); + EXPECT_FALSE(IsFoundAt("abc", four_char_delim, 0)); + EXPECT_FALSE(IsFoundAt("abcd", four_char_delim, 0)); +} + +TEST(Split, WorksWithLargeStrings) { + if (sizeof(size_t) > 4) { + std::string s((uint32_t{1} << 31) + 1, 'x'); // 2G + 1 byte + s.back() = '-'; + std::vector<absl::string_view> v = absl::StrSplit(s, '-'); + EXPECT_EQ(2, v.size()); + // The first element will contain 2G of 'x's. + // testing::StartsWith is too slow with a 2G string. + EXPECT_EQ('x', v[0][0]); + EXPECT_EQ('x', v[0][1]); + EXPECT_EQ('x', v[0][3]); + EXPECT_EQ("", v[1]); + } +} + +TEST(SplitInternalTest, TypeTraits) { + EXPECT_FALSE(absl::strings_internal::HasMappedType<int>::value); + EXPECT_TRUE( + (absl::strings_internal::HasMappedType<std::map<int, int>>::value)); + EXPECT_FALSE(absl::strings_internal::HasValueType<int>::value); + EXPECT_TRUE( + (absl::strings_internal::HasValueType<std::map<int, int>>::value)); + EXPECT_FALSE(absl::strings_internal::HasConstIterator<int>::value); + EXPECT_TRUE( + (absl::strings_internal::HasConstIterator<std::map<int, int>>::value)); + EXPECT_FALSE(absl::strings_internal::IsInitializerList<int>::value); + EXPECT_TRUE((absl::strings_internal::IsInitializerList< + std::initializer_list<int>>::value)); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/string_view.cc b/third_party/abseil_cpp/absl/strings/string_view.cc new file mode 100644 index 000000000000..c5f5de936deb --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/string_view.cc @@ -0,0 +1,235 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/string_view.h" + +#ifndef ABSL_USES_STD_STRING_VIEW + +#include <algorithm> +#include <climits> +#include <cstring> +#include <ostream> + +#include "absl/strings/internal/memutil.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { +void WritePadding(std::ostream& o, size_t pad) { + char fill_buf[32]; + memset(fill_buf, o.fill(), sizeof(fill_buf)); + while (pad) { + size_t n = std::min(pad, sizeof(fill_buf)); + o.write(fill_buf, n); + pad -= n; + } +} + +class LookupTable { + public: + // For each character in wanted, sets the index corresponding + // to the ASCII code of that character. This is used by + // the find_.*_of methods below to tell whether or not a character is in + // the lookup table in constant time. + explicit LookupTable(string_view wanted) { + for (char c : wanted) { + table_[Index(c)] = true; + } + } + bool operator[](char c) const { return table_[Index(c)]; } + + private: + static unsigned char Index(char c) { return static_cast<unsigned char>(c); } + bool table_[UCHAR_MAX + 1] = {}; +}; + +} // namespace + +std::ostream& operator<<(std::ostream& o, string_view piece) { + std::ostream::sentry sentry(o); + if (sentry) { + size_t lpad = 0; + size_t rpad = 0; + if (static_cast<size_t>(o.width()) > piece.size()) { + size_t pad = o.width() - piece.size(); + if ((o.flags() & o.adjustfield) == o.left) { + rpad = pad; + } else { + lpad = pad; + } + } + if (lpad) WritePadding(o, lpad); + o.write(piece.data(), piece.size()); + if (rpad) WritePadding(o, rpad); + o.width(0); + } + return o; +} + +string_view::size_type string_view::find(string_view s, size_type pos) const + noexcept { + if (empty() || pos > length_) { + if (empty() && pos == 0 && s.empty()) return 0; + return npos; + } + const char* result = + strings_internal::memmatch(ptr_ + pos, length_ - pos, s.ptr_, s.length_); + return result ? result - ptr_ : npos; +} + +string_view::size_type string_view::find(char c, size_type pos) const noexcept { + if (empty() || pos >= length_) { + return npos; + } + const char* result = + static_cast<const char*>(memchr(ptr_ + pos, c, length_ - pos)); + return result != nullptr ? result - ptr_ : npos; +} + +string_view::size_type string_view::rfind(string_view s, size_type pos) const + noexcept { + if (length_ < s.length_) return npos; + if (s.empty()) return std::min(length_, pos); + const char* last = ptr_ + std::min(length_ - s.length_, pos) + s.length_; + const char* result = std::find_end(ptr_, last, s.ptr_, s.ptr_ + s.length_); + return result != last ? result - ptr_ : npos; +} + +// Search range is [0..pos] inclusive. If pos == npos, search everything. +string_view::size_type string_view::rfind(char c, size_type pos) const + noexcept { + // Note: memrchr() is not available on Windows. + if (empty()) return npos; + for (size_type i = std::min(pos, length_ - 1);; --i) { + if (ptr_[i] == c) { + return i; + } + if (i == 0) break; + } + return npos; +} + +string_view::size_type string_view::find_first_of(string_view s, + size_type pos) const + noexcept { + if (empty() || s.empty()) { + return npos; + } + // Avoid the cost of LookupTable() for a single-character search. + if (s.length_ == 1) return find_first_of(s.ptr_[0], pos); + LookupTable tbl(s); + for (size_type i = pos; i < length_; ++i) { + if (tbl[ptr_[i]]) { + return i; + } + } + return npos; +} + +string_view::size_type string_view::find_first_not_of(string_view s, + size_type pos) const + noexcept { + if (empty()) return npos; + // Avoid the cost of LookupTable() for a single-character search. + if (s.length_ == 1) return find_first_not_of(s.ptr_[0], pos); + LookupTable tbl(s); + for (size_type i = pos; i < length_; ++i) { + if (!tbl[ptr_[i]]) { + return i; + } + } + return npos; +} + +string_view::size_type string_view::find_first_not_of(char c, + size_type pos) const + noexcept { + if (empty()) return npos; + for (; pos < length_; ++pos) { + if (ptr_[pos] != c) { + return pos; + } + } + return npos; +} + +string_view::size_type string_view::find_last_of(string_view s, + size_type pos) const noexcept { + if (empty() || s.empty()) return npos; + // Avoid the cost of LookupTable() for a single-character search. + if (s.length_ == 1) return find_last_of(s.ptr_[0], pos); + LookupTable tbl(s); + for (size_type i = std::min(pos, length_ - 1);; --i) { + if (tbl[ptr_[i]]) { + return i; + } + if (i == 0) break; + } + return npos; +} + +string_view::size_type string_view::find_last_not_of(string_view s, + size_type pos) const + noexcept { + if (empty()) return npos; + size_type i = std::min(pos, length_ - 1); + if (s.empty()) return i; + // Avoid the cost of LookupTable() for a single-character search. + if (s.length_ == 1) return find_last_not_of(s.ptr_[0], pos); + LookupTable tbl(s); + for (;; --i) { + if (!tbl[ptr_[i]]) { + return i; + } + if (i == 0) break; + } + return npos; +} + +string_view::size_type string_view::find_last_not_of(char c, + size_type pos) const + noexcept { + if (empty()) return npos; + size_type i = std::min(pos, length_ - 1); + for (;; --i) { + if (ptr_[i] != c) { + return i; + } + if (i == 0) break; + } + return npos; +} + +// MSVC has non-standard behavior that implicitly creates definitions for static +// const members. These implicit definitions conflict with explicit out-of-class +// member definitions that are required by the C++ standard, resulting in +// LNK1169 "multiply defined" errors at link time. __declspec(selectany) asks +// MSVC to choose only one definition for the symbol it decorates. See details +// at https://msdn.microsoft.com/en-us/library/34h23df8(v=vs.100).aspx +#ifdef _MSC_VER +#define ABSL_STRING_VIEW_SELECTANY __declspec(selectany) +#else +#define ABSL_STRING_VIEW_SELECTANY +#endif + +ABSL_STRING_VIEW_SELECTANY +constexpr string_view::size_type string_view::npos; +ABSL_STRING_VIEW_SELECTANY +constexpr string_view::size_type string_view::kMaxSize; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_STRING_VIEW diff --git a/third_party/abseil_cpp/absl/strings/string_view.h b/third_party/abseil_cpp/absl/strings/string_view.h new file mode 100644 index 000000000000..5260b5b73f47 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/string_view.h @@ -0,0 +1,629 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: string_view.h +// ----------------------------------------------------------------------------- +// +// This file contains the definition of the `absl::string_view` class. A +// `string_view` points to a contiguous span of characters, often part or all of +// another `std::string`, double-quoted string literal, character array, or even +// another `string_view`. +// +// This `absl::string_view` abstraction is designed to be a drop-in +// replacement for the C++17 `std::string_view` abstraction. +#ifndef ABSL_STRINGS_STRING_VIEW_H_ +#define ABSL_STRINGS_STRING_VIEW_H_ + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstring> +#include <iosfwd> +#include <iterator> +#include <limits> +#include <string> + +#include "absl/base/config.h" +#include "absl/base/internal/throw_delegate.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" + +#ifdef ABSL_USES_STD_STRING_VIEW + +#include <string_view> // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN +using string_view = std::string_view; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_STRING_VIEW + +#if ABSL_HAVE_BUILTIN(__builtin_memcmp) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_INTERNAL_STRING_VIEW_MEMCMP __builtin_memcmp +#else // ABSL_HAVE_BUILTIN(__builtin_memcmp) +#define ABSL_INTERNAL_STRING_VIEW_MEMCMP memcmp +#endif // ABSL_HAVE_BUILTIN(__builtin_memcmp) + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::string_view +// +// A `string_view` provides a lightweight view into the string data provided by +// a `std::string`, double-quoted string literal, character array, or even +// another `string_view`. A `string_view` does *not* own the string to which it +// points, and that data cannot be modified through the view. +// +// You can use `string_view` as a function or method parameter anywhere a +// parameter can receive a double-quoted string literal, `const char*`, +// `std::string`, or another `absl::string_view` argument with no need to copy +// the string data. Systematic use of `string_view` within function arguments +// reduces data copies and `strlen()` calls. +// +// Because of its small size, prefer passing `string_view` by value: +// +// void MyFunction(absl::string_view arg); +// +// If circumstances require, you may also pass one by const reference: +// +// void MyFunction(const absl::string_view& arg); // not preferred +// +// Passing by value generates slightly smaller code for many architectures. +// +// In either case, the source data of the `string_view` must outlive the +// `string_view` itself. +// +// A `string_view` is also suitable for local variables if you know that the +// lifetime of the underlying object is longer than the lifetime of your +// `string_view` variable. However, beware of binding a `string_view` to a +// temporary value: +// +// // BAD use of string_view: lifetime problem +// absl::string_view sv = obj.ReturnAString(); +// +// // GOOD use of string_view: str outlives sv +// std::string str = obj.ReturnAString(); +// absl::string_view sv = str; +// +// Due to lifetime issues, a `string_view` is sometimes a poor choice for a +// return value and usually a poor choice for a data member. If you do use a +// `string_view` this way, it is your responsibility to ensure that the object +// pointed to by the `string_view` outlives the `string_view`. +// +// A `string_view` may represent a whole string or just part of a string. For +// example, when splitting a string, `std::vector<absl::string_view>` is a +// natural data type for the output. +// +// For another example, a Cord is a non-contiguous, potentially very +// long string-like object. The Cord class has an interface that iteratively +// provides string_view objects that point to the successive pieces of a Cord +// object. +// +// When constructed from a source which is NUL-terminated, the `string_view` +// itself will not include the NUL-terminator unless a specific size (including +// the NUL) is passed to the constructor. As a result, common idioms that work +// on NUL-terminated strings do not work on `string_view` objects. If you write +// code that scans a `string_view`, you must check its length rather than test +// for nul, for example. Note, however, that nuls may still be embedded within +// a `string_view` explicitly. +// +// You may create a null `string_view` in two ways: +// +// absl::string_view sv; +// absl::string_view sv(nullptr, 0); +// +// For the above, `sv.data() == nullptr`, `sv.length() == 0`, and +// `sv.empty() == true`. Also, if you create a `string_view` with a non-null +// pointer then `sv.data() != nullptr`. Thus, you can use `string_view()` to +// signal an undefined value that is different from other `string_view` values +// in a similar fashion to how `const char* p1 = nullptr;` is different from +// `const char* p2 = "";`. However, in practice, it is not recommended to rely +// on this behavior. +// +// Be careful not to confuse a null `string_view` with an empty one. A null +// `string_view` is an empty `string_view`, but some empty `string_view`s are +// not null. Prefer checking for emptiness over checking for null. +// +// There are many ways to create an empty string_view: +// +// const char* nullcp = nullptr; +// // string_view.size() will return 0 in all cases. +// absl::string_view(); +// absl::string_view(nullcp, 0); +// absl::string_view(""); +// absl::string_view("", 0); +// absl::string_view("abcdef", 0); +// absl::string_view("abcdef" + 6, 0); +// +// All empty `string_view` objects whether null or not, are equal: +// +// absl::string_view() == absl::string_view("", 0) +// absl::string_view(nullptr, 0) == absl::string_view("abcdef"+6, 0) +class string_view { + public: + using traits_type = std::char_traits<char>; + using value_type = char; + using pointer = char*; + using const_pointer = const char*; + using reference = char&; + using const_reference = const char&; + using const_iterator = const char*; + using iterator = const_iterator; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using reverse_iterator = const_reverse_iterator; + using size_type = size_t; + using difference_type = std::ptrdiff_t; + + static constexpr size_type npos = static_cast<size_type>(-1); + + // Null `string_view` constructor + constexpr string_view() noexcept : ptr_(nullptr), length_(0) {} + + // Implicit constructors + + template <typename Allocator> + string_view( // NOLINT(runtime/explicit) + const std::basic_string<char, std::char_traits<char>, Allocator>& + str) noexcept + // This is implemented in terms of `string_view(p, n)` so `str.size()` + // doesn't need to be reevaluated after `ptr_` is set. + : string_view(str.data(), str.size()) {} + + // Implicit constructor of a `string_view` from NUL-terminated `str`. When + // accepting possibly null strings, use `absl::NullSafeStringView(str)` + // instead (see below). + constexpr string_view(const char* str) // NOLINT(runtime/explicit) + : ptr_(str), + length_(str ? CheckLengthInternal(StrlenInternal(str)) : 0) {} + + // Implicit constructor of a `string_view` from a `const char*` and length. + constexpr string_view(const char* data, size_type len) + : ptr_(data), length_(CheckLengthInternal(len)) {} + + // NOTE: Harmlessly omitted to work around gdb bug. + // constexpr string_view(const string_view&) noexcept = default; + // string_view& operator=(const string_view&) noexcept = default; + + // Iterators + + // string_view::begin() + // + // Returns an iterator pointing to the first character at the beginning of the + // `string_view`, or `end()` if the `string_view` is empty. + constexpr const_iterator begin() const noexcept { return ptr_; } + + // string_view::end() + // + // Returns an iterator pointing just beyond the last character at the end of + // the `string_view`. This iterator acts as a placeholder; attempting to + // access it results in undefined behavior. + constexpr const_iterator end() const noexcept { return ptr_ + length_; } + + // string_view::cbegin() + // + // Returns a const iterator pointing to the first character at the beginning + // of the `string_view`, or `end()` if the `string_view` is empty. + constexpr const_iterator cbegin() const noexcept { return begin(); } + + // string_view::cend() + // + // Returns a const iterator pointing just beyond the last character at the end + // of the `string_view`. This pointer acts as a placeholder; attempting to + // access its element results in undefined behavior. + constexpr const_iterator cend() const noexcept { return end(); } + + // string_view::rbegin() + // + // Returns a reverse iterator pointing to the last character at the end of the + // `string_view`, or `rend()` if the `string_view` is empty. + const_reverse_iterator rbegin() const noexcept { + return const_reverse_iterator(end()); + } + + // string_view::rend() + // + // Returns a reverse iterator pointing just before the first character at the + // beginning of the `string_view`. This pointer acts as a placeholder; + // attempting to access its element results in undefined behavior. + const_reverse_iterator rend() const noexcept { + return const_reverse_iterator(begin()); + } + + // string_view::crbegin() + // + // Returns a const reverse iterator pointing to the last character at the end + // of the `string_view`, or `crend()` if the `string_view` is empty. + const_reverse_iterator crbegin() const noexcept { return rbegin(); } + + // string_view::crend() + // + // Returns a const reverse iterator pointing just before the first character + // at the beginning of the `string_view`. This pointer acts as a placeholder; + // attempting to access its element results in undefined behavior. + const_reverse_iterator crend() const noexcept { return rend(); } + + // Capacity Utilities + + // string_view::size() + // + // Returns the number of characters in the `string_view`. + constexpr size_type size() const noexcept { + return length_; + } + + // string_view::length() + // + // Returns the number of characters in the `string_view`. Alias for `size()`. + constexpr size_type length() const noexcept { return size(); } + + // string_view::max_size() + // + // Returns the maximum number of characters the `string_view` can hold. + constexpr size_type max_size() const noexcept { return kMaxSize; } + + // string_view::empty() + // + // Checks if the `string_view` is empty (refers to no characters). + constexpr bool empty() const noexcept { return length_ == 0; } + + // string_view::operator[] + // + // Returns the ith element of the `string_view` using the array operator. + // Note that this operator does not perform any bounds checking. + constexpr const_reference operator[](size_type i) const { + return ABSL_HARDENING_ASSERT(i < size()), ptr_[i]; + } + + // string_view::at() + // + // Returns the ith element of the `string_view`. Bounds checking is performed, + // and an exception of type `std::out_of_range` will be thrown on invalid + // access. + constexpr const_reference at(size_type i) const { + return ABSL_PREDICT_TRUE(i < size()) + ? ptr_[i] + : ((void)base_internal::ThrowStdOutOfRange( + "absl::string_view::at"), + ptr_[i]); + } + + // string_view::front() + // + // Returns the first element of a `string_view`. + constexpr const_reference front() const { + return ABSL_HARDENING_ASSERT(!empty()), ptr_[0]; + } + + // string_view::back() + // + // Returns the last element of a `string_view`. + constexpr const_reference back() const { + return ABSL_HARDENING_ASSERT(!empty()), ptr_[size() - 1]; + } + + // string_view::data() + // + // Returns a pointer to the underlying character array (which is of course + // stored elsewhere). Note that `string_view::data()` may contain embedded nul + // characters, but the returned buffer may or may not be NUL-terminated; + // therefore, do not pass `data()` to a routine that expects a NUL-terminated + // string. + constexpr const_pointer data() const noexcept { return ptr_; } + + // Modifiers + + // string_view::remove_prefix() + // + // Removes the first `n` characters from the `string_view`. Note that the + // underlying string is not changed, only the view. + void remove_prefix(size_type n) { + ABSL_HARDENING_ASSERT(n <= length_); + ptr_ += n; + length_ -= n; + } + + // string_view::remove_suffix() + // + // Removes the last `n` characters from the `string_view`. Note that the + // underlying string is not changed, only the view. + void remove_suffix(size_type n) { + ABSL_HARDENING_ASSERT(n <= length_); + length_ -= n; + } + + // string_view::swap() + // + // Swaps this `string_view` with another `string_view`. + void swap(string_view& s) noexcept { + auto t = *this; + *this = s; + s = t; + } + + // Explicit conversion operators + + // Converts to `std::basic_string`. + template <typename A> + explicit operator std::basic_string<char, traits_type, A>() const { + if (!data()) return {}; + return std::basic_string<char, traits_type, A>(data(), size()); + } + + // string_view::copy() + // + // Copies the contents of the `string_view` at offset `pos` and length `n` + // into `buf`. + size_type copy(char* buf, size_type n, size_type pos = 0) const { + if (ABSL_PREDICT_FALSE(pos > length_)) { + base_internal::ThrowStdOutOfRange("absl::string_view::copy"); + } + size_type rlen = (std::min)(length_ - pos, n); + if (rlen > 0) { + const char* start = ptr_ + pos; + traits_type::copy(buf, start, rlen); + } + return rlen; + } + + // string_view::substr() + // + // Returns a "substring" of the `string_view` (at offset `pos` and length + // `n`) as another string_view. This function throws `std::out_of_bounds` if + // `pos > size`. + // Use absl::ClippedSubstr if you need a truncating substr operation. + constexpr string_view substr(size_type pos, size_type n = npos) const { + return ABSL_PREDICT_FALSE(pos > length_) + ? (base_internal::ThrowStdOutOfRange( + "absl::string_view::substr"), + string_view()) + : string_view(ptr_ + pos, Min(n, length_ - pos)); + } + + // string_view::compare() + // + // Performs a lexicographical comparison between the `string_view` and + // another `absl::string_view`, returning -1 if `this` is less than, 0 if + // `this` is equal to, and 1 if `this` is greater than the passed string + // view. Note that in the case of data equality, a further comparison is made + // on the respective sizes of the two `string_view`s to determine which is + // smaller, equal, or greater. + constexpr int compare(string_view x) const noexcept { + return CompareImpl(length_, x.length_, + Min(length_, x.length_) == 0 + ? 0 + : ABSL_INTERNAL_STRING_VIEW_MEMCMP( + ptr_, x.ptr_, Min(length_, x.length_))); + } + + // Overload of `string_view::compare()` for comparing a substring of the + // 'string_view` and another `absl::string_view`. + int compare(size_type pos1, size_type count1, string_view v) const { + return substr(pos1, count1).compare(v); + } + + // Overload of `string_view::compare()` for comparing a substring of the + // `string_view` and a substring of another `absl::string_view`. + int compare(size_type pos1, size_type count1, string_view v, size_type pos2, + size_type count2) const { + return substr(pos1, count1).compare(v.substr(pos2, count2)); + } + + // Overload of `string_view::compare()` for comparing a `string_view` and a + // a different C-style string `s`. + int compare(const char* s) const { return compare(string_view(s)); } + + // Overload of `string_view::compare()` for comparing a substring of the + // `string_view` and a different string C-style string `s`. + int compare(size_type pos1, size_type count1, const char* s) const { + return substr(pos1, count1).compare(string_view(s)); + } + + // Overload of `string_view::compare()` for comparing a substring of the + // `string_view` and a substring of a different C-style string `s`. + int compare(size_type pos1, size_type count1, const char* s, + size_type count2) const { + return substr(pos1, count1).compare(string_view(s, count2)); + } + + // Find Utilities + + // string_view::find() + // + // Finds the first occurrence of the substring `s` within the `string_view`, + // returning the position of the first character's match, or `npos` if no + // match was found. + size_type find(string_view s, size_type pos = 0) const noexcept; + + // Overload of `string_view::find()` for finding the given character `c` + // within the `string_view`. + size_type find(char c, size_type pos = 0) const noexcept; + + // string_view::rfind() + // + // Finds the last occurrence of a substring `s` within the `string_view`, + // returning the position of the first character's match, or `npos` if no + // match was found. + size_type rfind(string_view s, size_type pos = npos) const + noexcept; + + // Overload of `string_view::rfind()` for finding the last given character `c` + // within the `string_view`. + size_type rfind(char c, size_type pos = npos) const noexcept; + + // string_view::find_first_of() + // + // Finds the first occurrence of any of the characters in `s` within the + // `string_view`, returning the start position of the match, or `npos` if no + // match was found. + size_type find_first_of(string_view s, size_type pos = 0) const + noexcept; + + // Overload of `string_view::find_first_of()` for finding a character `c` + // within the `string_view`. + size_type find_first_of(char c, size_type pos = 0) const + noexcept { + return find(c, pos); + } + + // string_view::find_last_of() + // + // Finds the last occurrence of any of the characters in `s` within the + // `string_view`, returning the start position of the match, or `npos` if no + // match was found. + size_type find_last_of(string_view s, size_type pos = npos) const + noexcept; + + // Overload of `string_view::find_last_of()` for finding a character `c` + // within the `string_view`. + size_type find_last_of(char c, size_type pos = npos) const + noexcept { + return rfind(c, pos); + } + + // string_view::find_first_not_of() + // + // Finds the first occurrence of any of the characters not in `s` within the + // `string_view`, returning the start position of the first non-match, or + // `npos` if no non-match was found. + size_type find_first_not_of(string_view s, size_type pos = 0) const noexcept; + + // Overload of `string_view::find_first_not_of()` for finding a character + // that is not `c` within the `string_view`. + size_type find_first_not_of(char c, size_type pos = 0) const noexcept; + + // string_view::find_last_not_of() + // + // Finds the last occurrence of any of the characters not in `s` within the + // `string_view`, returning the start position of the last non-match, or + // `npos` if no non-match was found. + size_type find_last_not_of(string_view s, + size_type pos = npos) const noexcept; + + // Overload of `string_view::find_last_not_of()` for finding a character + // that is not `c` within the `string_view`. + size_type find_last_not_of(char c, size_type pos = npos) const + noexcept; + + private: + static constexpr size_type kMaxSize = + (std::numeric_limits<difference_type>::max)(); + + static constexpr size_type CheckLengthInternal(size_type len) { + return ABSL_HARDENING_ASSERT(len <= kMaxSize), len; + } + + static constexpr size_type StrlenInternal(const char* str) { +#if defined(_MSC_VER) && _MSC_VER >= 1910 && !defined(__clang__) + // MSVC 2017+ can evaluate this at compile-time. + const char* begin = str; + while (*str != '\0') ++str; + return str - begin; +#elif ABSL_HAVE_BUILTIN(__builtin_strlen) || \ + (defined(__GNUC__) && !defined(__clang__)) + // GCC has __builtin_strlen according to + // https://gcc.gnu.org/onlinedocs/gcc-4.7.0/gcc/Other-Builtins.html, but + // ABSL_HAVE_BUILTIN doesn't detect that, so we use the extra checks above. + // __builtin_strlen is constexpr. + return __builtin_strlen(str); +#else + return str ? strlen(str) : 0; +#endif + } + + static constexpr size_t Min(size_type length_a, size_type length_b) { + return length_a < length_b ? length_a : length_b; + } + + static constexpr int CompareImpl(size_type length_a, size_type length_b, + int compare_result) { + return compare_result == 0 ? static_cast<int>(length_a > length_b) - + static_cast<int>(length_a < length_b) + : (compare_result < 0 ? -1 : 1); + } + + const char* ptr_; + size_type length_; +}; + +// This large function is defined inline so that in a fairly common case where +// one of the arguments is a literal, the compiler can elide a lot of the +// following comparisons. +constexpr bool operator==(string_view x, string_view y) noexcept { + return x.size() == y.size() && + (x.empty() || + ABSL_INTERNAL_STRING_VIEW_MEMCMP(x.data(), y.data(), x.size()) == 0); +} + +constexpr bool operator!=(string_view x, string_view y) noexcept { + return !(x == y); +} + +constexpr bool operator<(string_view x, string_view y) noexcept { + return x.compare(y) < 0; +} + +constexpr bool operator>(string_view x, string_view y) noexcept { + return y < x; +} + +constexpr bool operator<=(string_view x, string_view y) noexcept { + return !(y < x); +} + +constexpr bool operator>=(string_view x, string_view y) noexcept { + return !(x < y); +} + +// IO Insertion Operator +std::ostream& operator<<(std::ostream& o, string_view piece); + +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_INTERNAL_STRING_VIEW_MEMCMP + +#endif // ABSL_USES_STD_STRING_VIEW + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ClippedSubstr() +// +// Like `s.substr(pos, n)`, but clips `pos` to an upper bound of `s.size()`. +// Provided because std::string_view::substr throws if `pos > size()` +inline string_view ClippedSubstr(string_view s, size_t pos, + size_t n = string_view::npos) { + pos = (std::min)(pos, static_cast<size_t>(s.size())); + return s.substr(pos, n); +} + +// NullSafeStringView() +// +// Creates an `absl::string_view` from a pointer `p` even if it's null-valued. +// This function should be used where an `absl::string_view` can be created from +// a possibly-null pointer. +constexpr string_view NullSafeStringView(const char* p) { + return p ? string_view(p) : string_view(); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STRING_VIEW_H_ diff --git a/third_party/abseil_cpp/absl/strings/string_view_benchmark.cc b/third_party/abseil_cpp/absl/strings/string_view_benchmark.cc new file mode 100644 index 000000000000..0d74e23e2fc9 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/string_view_benchmark.cc @@ -0,0 +1,381 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/string_view.h" + +#include <algorithm> +#include <cstdint> +#include <map> +#include <random> +#include <string> +#include <unordered_set> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/attributes.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/strings/str_cat.h" + +namespace { + +void BM_StringViewFromString(benchmark::State& state) { + std::string s(state.range(0), 'x'); + std::string* ps = &s; + struct SV { + SV() = default; + explicit SV(const std::string& s) : sv(s) {} + absl::string_view sv; + } sv; + SV* psv = &sv; + benchmark::DoNotOptimize(ps); + benchmark::DoNotOptimize(psv); + for (auto _ : state) { + new (psv) SV(*ps); + benchmark::DoNotOptimize(sv); + } +} +BENCHMARK(BM_StringViewFromString)->Arg(12)->Arg(128); + +// Provide a forcibly out-of-line wrapper for operator== that can be used in +// benchmarks to measure the impact of inlining. +ABSL_ATTRIBUTE_NOINLINE +bool NonInlinedEq(absl::string_view a, absl::string_view b) { return a == b; } + +// We use functions that cannot be inlined to perform the comparison loops so +// that inlining of the operator== can't optimize away *everything*. +ABSL_ATTRIBUTE_NOINLINE +void DoEqualityComparisons(benchmark::State& state, absl::string_view a, + absl::string_view b) { + for (auto _ : state) { + benchmark::DoNotOptimize(a == b); + } +} + +void BM_EqualIdentical(benchmark::State& state) { + std::string x(state.range(0), 'a'); + DoEqualityComparisons(state, x, x); +} +BENCHMARK(BM_EqualIdentical)->DenseRange(0, 3)->Range(4, 1 << 10); + +void BM_EqualSame(benchmark::State& state) { + std::string x(state.range(0), 'a'); + std::string y = x; + DoEqualityComparisons(state, x, y); +} +BENCHMARK(BM_EqualSame) + ->DenseRange(0, 10) + ->Arg(20) + ->Arg(40) + ->Arg(70) + ->Arg(110) + ->Range(160, 4096); + +void BM_EqualDifferent(benchmark::State& state) { + const int len = state.range(0); + std::string x(len, 'a'); + std::string y = x; + if (len > 0) { + y[len - 1] = 'b'; + } + DoEqualityComparisons(state, x, y); +} +BENCHMARK(BM_EqualDifferent)->DenseRange(0, 3)->Range(4, 1 << 10); + +// This benchmark is intended to check that important simplifications can be +// made with absl::string_view comparisons against constant strings. The idea is +// that if constant strings cause redundant components of the comparison, the +// compiler should detect and eliminate them. Here we use 8 different strings, +// each with the same size. Provided our comparison makes the implementation +// inline-able by the compiler, it should fold all of these away into a single +// size check once per loop iteration. +ABSL_ATTRIBUTE_NOINLINE +void DoConstantSizeInlinedEqualityComparisons(benchmark::State& state, + absl::string_view a) { + for (auto _ : state) { + benchmark::DoNotOptimize(a == "aaa"); + benchmark::DoNotOptimize(a == "bbb"); + benchmark::DoNotOptimize(a == "ccc"); + benchmark::DoNotOptimize(a == "ddd"); + benchmark::DoNotOptimize(a == "eee"); + benchmark::DoNotOptimize(a == "fff"); + benchmark::DoNotOptimize(a == "ggg"); + benchmark::DoNotOptimize(a == "hhh"); + } +} +void BM_EqualConstantSizeInlined(benchmark::State& state) { + std::string x(state.range(0), 'a'); + DoConstantSizeInlinedEqualityComparisons(state, x); +} +// We only need to check for size of 3, and <> 3 as this benchmark only has to +// do with size differences. +BENCHMARK(BM_EqualConstantSizeInlined)->DenseRange(2, 4); + +// This benchmark exists purely to give context to the above timings: this is +// what they would look like if the compiler is completely unable to simplify +// between two comparisons when they are comparing against constant strings. +ABSL_ATTRIBUTE_NOINLINE +void DoConstantSizeNonInlinedEqualityComparisons(benchmark::State& state, + absl::string_view a) { + for (auto _ : state) { + // Force these out-of-line to compare with the above function. + benchmark::DoNotOptimize(NonInlinedEq(a, "aaa")); + benchmark::DoNotOptimize(NonInlinedEq(a, "bbb")); + benchmark::DoNotOptimize(NonInlinedEq(a, "ccc")); + benchmark::DoNotOptimize(NonInlinedEq(a, "ddd")); + benchmark::DoNotOptimize(NonInlinedEq(a, "eee")); + benchmark::DoNotOptimize(NonInlinedEq(a, "fff")); + benchmark::DoNotOptimize(NonInlinedEq(a, "ggg")); + benchmark::DoNotOptimize(NonInlinedEq(a, "hhh")); + } +} + +void BM_EqualConstantSizeNonInlined(benchmark::State& state) { + std::string x(state.range(0), 'a'); + DoConstantSizeNonInlinedEqualityComparisons(state, x); +} +// We only need to check for size of 3, and <> 3 as this benchmark only has to +// do with size differences. +BENCHMARK(BM_EqualConstantSizeNonInlined)->DenseRange(2, 4); + +void BM_CompareSame(benchmark::State& state) { + const int len = state.range(0); + std::string x; + for (int i = 0; i < len; i++) { + x += 'a'; + } + std::string y = x; + absl::string_view a = x; + absl::string_view b = y; + + for (auto _ : state) { + benchmark::DoNotOptimize(a); + benchmark::DoNotOptimize(b); + benchmark::DoNotOptimize(a.compare(b)); + } +} +BENCHMARK(BM_CompareSame)->DenseRange(0, 3)->Range(4, 1 << 10); + +void BM_CompareFirstOneLess(benchmark::State& state) { + const int len = state.range(0); + std::string x(len, 'a'); + std::string y = x; + y.back() = 'b'; + absl::string_view a = x; + absl::string_view b = y; + + for (auto _ : state) { + benchmark::DoNotOptimize(a); + benchmark::DoNotOptimize(b); + benchmark::DoNotOptimize(a.compare(b)); + } +} +BENCHMARK(BM_CompareFirstOneLess)->DenseRange(1, 3)->Range(4, 1 << 10); + +void BM_CompareSecondOneLess(benchmark::State& state) { + const int len = state.range(0); + std::string x(len, 'a'); + std::string y = x; + x.back() = 'b'; + absl::string_view a = x; + absl::string_view b = y; + + for (auto _ : state) { + benchmark::DoNotOptimize(a); + benchmark::DoNotOptimize(b); + benchmark::DoNotOptimize(a.compare(b)); + } +} +BENCHMARK(BM_CompareSecondOneLess)->DenseRange(1, 3)->Range(4, 1 << 10); + +void BM_find_string_view_len_one(benchmark::State& state) { + std::string haystack(state.range(0), '0'); + absl::string_view s(haystack); + for (auto _ : state) { + benchmark::DoNotOptimize(s.find("x")); // not present; length 1 + } +} +BENCHMARK(BM_find_string_view_len_one)->Range(1, 1 << 20); + +void BM_find_string_view_len_two(benchmark::State& state) { + std::string haystack(state.range(0), '0'); + absl::string_view s(haystack); + for (auto _ : state) { + benchmark::DoNotOptimize(s.find("xx")); // not present; length 2 + } +} +BENCHMARK(BM_find_string_view_len_two)->Range(1, 1 << 20); + +void BM_find_one_char(benchmark::State& state) { + std::string haystack(state.range(0), '0'); + absl::string_view s(haystack); + for (auto _ : state) { + benchmark::DoNotOptimize(s.find('x')); // not present + } +} +BENCHMARK(BM_find_one_char)->Range(1, 1 << 20); + +void BM_rfind_one_char(benchmark::State& state) { + std::string haystack(state.range(0), '0'); + absl::string_view s(haystack); + for (auto _ : state) { + benchmark::DoNotOptimize(s.rfind('x')); // not present + } +} +BENCHMARK(BM_rfind_one_char)->Range(1, 1 << 20); + +void BM_worst_case_find_first_of(benchmark::State& state, int haystack_len) { + const int needle_len = state.range(0); + std::string needle; + for (int i = 0; i < needle_len; ++i) { + needle += 'a' + i; + } + std::string haystack(haystack_len, '0'); // 1000 zeros. + + absl::string_view s(haystack); + for (auto _ : state) { + benchmark::DoNotOptimize(s.find_first_of(needle)); + } +} + +void BM_find_first_of_short(benchmark::State& state) { + BM_worst_case_find_first_of(state, 10); +} + +void BM_find_first_of_medium(benchmark::State& state) { + BM_worst_case_find_first_of(state, 100); +} + +void BM_find_first_of_long(benchmark::State& state) { + BM_worst_case_find_first_of(state, 1000); +} + +BENCHMARK(BM_find_first_of_short)->DenseRange(0, 4)->Arg(8)->Arg(16)->Arg(32); +BENCHMARK(BM_find_first_of_medium)->DenseRange(0, 4)->Arg(8)->Arg(16)->Arg(32); +BENCHMARK(BM_find_first_of_long)->DenseRange(0, 4)->Arg(8)->Arg(16)->Arg(32); + +struct EasyMap : public std::map<absl::string_view, uint64_t> { + explicit EasyMap(size_t) {} +}; + +// This templated benchmark helper function is intended to stress operator== or +// operator< in a realistic test. It surely isn't entirely realistic, but it's +// a start. The test creates a map of type Map, a template arg, and populates +// it with table_size key/value pairs. Each key has WordsPerKey words. After +// creating the map, a number of lookups are done in random order. Some keys +// are used much more frequently than others in this phase of the test. +template <typename Map, int WordsPerKey> +void StringViewMapBenchmark(benchmark::State& state) { + const int table_size = state.range(0); + const double kFractionOfKeysThatAreHot = 0.2; + const int kNumLookupsOfHotKeys = 20; + const int kNumLookupsOfColdKeys = 1; + const char* words[] = {"the", "quick", "brown", "fox", "jumped", + "over", "the", "lazy", "dog", "and", + "found", "a", "large", "mushroom", "and", + "a", "couple", "crickets", "eating", "pie"}; + // Create some keys that consist of words in random order. + std::random_device r; + std::seed_seq seed({r(), r(), r(), r(), r(), r(), r(), r()}); + std::mt19937 rng(seed); + std::vector<std::string> keys(table_size); + std::vector<int> all_indices; + const int kBlockSize = 1 << 12; + std::unordered_set<std::string> t(kBlockSize); + std::uniform_int_distribution<int> uniform(0, ABSL_ARRAYSIZE(words) - 1); + for (int i = 0; i < table_size; i++) { + all_indices.push_back(i); + do { + keys[i].clear(); + for (int j = 0; j < WordsPerKey; j++) { + absl::StrAppend(&keys[i], j > 0 ? " " : "", words[uniform(rng)]); + } + } while (!t.insert(keys[i]).second); + } + + // Create a list of strings to lookup: a permutation of the array of + // keys we just created, with repeats. "Hot" keys get repeated more. + std::shuffle(all_indices.begin(), all_indices.end(), rng); + const int num_hot = table_size * kFractionOfKeysThatAreHot; + const int num_cold = table_size - num_hot; + std::vector<int> hot_indices(all_indices.begin(), + all_indices.begin() + num_hot); + std::vector<int> indices; + for (int i = 0; i < kNumLookupsOfColdKeys; i++) { + indices.insert(indices.end(), all_indices.begin(), all_indices.end()); + } + for (int i = 0; i < kNumLookupsOfHotKeys - kNumLookupsOfColdKeys; i++) { + indices.insert(indices.end(), hot_indices.begin(), hot_indices.end()); + } + std::shuffle(indices.begin(), indices.end(), rng); + ABSL_RAW_CHECK( + num_cold * kNumLookupsOfColdKeys + num_hot * kNumLookupsOfHotKeys == + indices.size(), + ""); + // After constructing the array we probe it with absl::string_views built from + // test_strings. This means operator== won't see equal pointers, so + // it'll have to check for equal lengths and equal characters. + std::vector<std::string> test_strings(indices.size()); + for (int i = 0; i < indices.size(); i++) { + test_strings[i] = keys[indices[i]]; + } + + // Run the benchmark. It includes map construction but is mostly + // map lookups. + for (auto _ : state) { + Map h(table_size); + for (int i = 0; i < table_size; i++) { + h[keys[i]] = i * 2; + } + ABSL_RAW_CHECK(h.size() == table_size, ""); + uint64_t sum = 0; + for (int i = 0; i < indices.size(); i++) { + sum += h[test_strings[i]]; + } + benchmark::DoNotOptimize(sum); + } +} + +void BM_StdMap_4(benchmark::State& state) { + StringViewMapBenchmark<EasyMap, 4>(state); +} +BENCHMARK(BM_StdMap_4)->Range(1 << 10, 1 << 16); + +void BM_StdMap_8(benchmark::State& state) { + StringViewMapBenchmark<EasyMap, 8>(state); +} +BENCHMARK(BM_StdMap_8)->Range(1 << 10, 1 << 16); + +void BM_CopyToStringNative(benchmark::State& state) { + std::string src(state.range(0), 'x'); + absl::string_view sv(src); + std::string dst; + for (auto _ : state) { + dst.assign(sv.begin(), sv.end()); + } +} +BENCHMARK(BM_CopyToStringNative)->Range(1 << 3, 1 << 12); + +void BM_AppendToStringNative(benchmark::State& state) { + std::string src(state.range(0), 'x'); + absl::string_view sv(src); + std::string dst; + for (auto _ : state) { + dst.clear(); + dst.insert(dst.end(), sv.begin(), sv.end()); + } +} +BENCHMARK(BM_AppendToStringNative)->Range(1 << 3, 1 << 12); + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/string_view_test.cc b/third_party/abseil_cpp/absl/strings/string_view_test.cc new file mode 100644 index 000000000000..dcebb1500100 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/string_view_test.cc @@ -0,0 +1,1264 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/string_view.h" + +#include <stdlib.h> +#include <iomanip> +#include <iterator> +#include <limits> +#include <map> +#include <sstream> +#include <stdexcept> +#include <string> +#include <type_traits> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/dynamic_annotations.h" +#include "absl/base/options.h" + +#if defined(ABSL_HAVE_STD_STRING_VIEW) || defined(__ANDROID__) +// We don't control the death messaging when using std::string_view. +// Android assert messages only go to system log, so death tests cannot inspect +// the message for matching. +#define ABSL_EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH_IF_SUPPORTED(statement, ".*") +#else +#define ABSL_EXPECT_DEATH_IF_SUPPORTED(statement, regex) \ + EXPECT_DEATH_IF_SUPPORTED(statement, regex) +#endif + +namespace { + +// A minimal allocator that uses malloc(). +template <typename T> +struct Mallocator { + typedef T value_type; + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + + size_type max_size() const { + return size_t(std::numeric_limits<size_type>::max()) / sizeof(value_type); + } + template <typename U> + struct rebind { + typedef Mallocator<U> other; + }; + Mallocator() = default; + template <class U> + Mallocator(const Mallocator<U>&) {} // NOLINT(runtime/explicit) + + T* allocate(size_t n) { return static_cast<T*>(std::malloc(n * sizeof(T))); } + void deallocate(T* p, size_t) { std::free(p); } +}; +template <typename T, typename U> +bool operator==(const Mallocator<T>&, const Mallocator<U>&) { + return true; +} +template <typename T, typename U> +bool operator!=(const Mallocator<T>&, const Mallocator<U>&) { + return false; +} + +TEST(StringViewTest, Ctor) { + { + // Null. + absl::string_view s10; + EXPECT_TRUE(s10.data() == nullptr); + EXPECT_EQ(0, s10.length()); + } + + { + // const char* without length. + const char* hello = "hello"; + absl::string_view s20(hello); + EXPECT_TRUE(s20.data() == hello); + EXPECT_EQ(5, s20.length()); + + // const char* with length. + absl::string_view s21(hello, 4); + EXPECT_TRUE(s21.data() == hello); + EXPECT_EQ(4, s21.length()); + + // Not recommended, but valid C++ + absl::string_view s22(hello, 6); + EXPECT_TRUE(s22.data() == hello); + EXPECT_EQ(6, s22.length()); + } + + { + // std::string. + std::string hola = "hola"; + absl::string_view s30(hola); + EXPECT_TRUE(s30.data() == hola.data()); + EXPECT_EQ(4, s30.length()); + + // std::string with embedded '\0'. + hola.push_back('\0'); + hola.append("h2"); + hola.push_back('\0'); + absl::string_view s31(hola); + EXPECT_TRUE(s31.data() == hola.data()); + EXPECT_EQ(8, s31.length()); + } + + { + using mstring = + std::basic_string<char, std::char_traits<char>, Mallocator<char>>; + mstring str1("BUNGIE-JUMPING!"); + const mstring str2("SLEEPING!"); + + absl::string_view s1(str1); + s1.remove_prefix(strlen("BUNGIE-JUM")); + + absl::string_view s2(str2); + s2.remove_prefix(strlen("SLEE")); + + EXPECT_EQ(s1, s2); + EXPECT_EQ(s1, "PING!"); + } + + // TODO(mec): absl::string_view(const absl::string_view&); +} + +TEST(StringViewTest, Swap) { + absl::string_view a("a"); + absl::string_view b("bbb"); + EXPECT_TRUE(noexcept(a.swap(b))); + a.swap(b); + EXPECT_EQ(a, "bbb"); + EXPECT_EQ(b, "a"); + a.swap(b); + EXPECT_EQ(a, "a"); + EXPECT_EQ(b, "bbb"); +} + +TEST(StringViewTest, STLComparator) { + std::string s1("foo"); + std::string s2("bar"); + std::string s3("baz"); + + absl::string_view p1(s1); + absl::string_view p2(s2); + absl::string_view p3(s3); + + typedef std::map<absl::string_view, int> TestMap; + TestMap map; + + map.insert(std::make_pair(p1, 0)); + map.insert(std::make_pair(p2, 1)); + map.insert(std::make_pair(p3, 2)); + EXPECT_EQ(map.size(), 3); + + TestMap::const_iterator iter = map.begin(); + EXPECT_EQ(iter->second, 1); + ++iter; + EXPECT_EQ(iter->second, 2); + ++iter; + EXPECT_EQ(iter->second, 0); + ++iter; + EXPECT_TRUE(iter == map.end()); + + TestMap::iterator new_iter = map.find("zot"); + EXPECT_TRUE(new_iter == map.end()); + + new_iter = map.find("bar"); + EXPECT_TRUE(new_iter != map.end()); + + map.erase(new_iter); + EXPECT_EQ(map.size(), 2); + + iter = map.begin(); + EXPECT_EQ(iter->second, 2); + ++iter; + EXPECT_EQ(iter->second, 0); + ++iter; + EXPECT_TRUE(iter == map.end()); +} + +#define COMPARE(result, op, x, y) \ + EXPECT_EQ(result, absl::string_view((x)) op absl::string_view((y))); \ + EXPECT_EQ(result, absl::string_view((x)).compare(absl::string_view((y))) op 0) + +TEST(StringViewTest, ComparisonOperators) { + COMPARE(true, ==, "", ""); + COMPARE(true, ==, "", absl::string_view()); + COMPARE(true, ==, absl::string_view(), ""); + COMPARE(true, ==, "a", "a"); + COMPARE(true, ==, "aa", "aa"); + COMPARE(false, ==, "a", ""); + COMPARE(false, ==, "", "a"); + COMPARE(false, ==, "a", "b"); + COMPARE(false, ==, "a", "aa"); + COMPARE(false, ==, "aa", "a"); + + COMPARE(false, !=, "", ""); + COMPARE(false, !=, "a", "a"); + COMPARE(false, !=, "aa", "aa"); + COMPARE(true, !=, "a", ""); + COMPARE(true, !=, "", "a"); + COMPARE(true, !=, "a", "b"); + COMPARE(true, !=, "a", "aa"); + COMPARE(true, !=, "aa", "a"); + + COMPARE(true, <, "a", "b"); + COMPARE(true, <, "a", "aa"); + COMPARE(true, <, "aa", "b"); + COMPARE(true, <, "aa", "bb"); + COMPARE(false, <, "a", "a"); + COMPARE(false, <, "b", "a"); + COMPARE(false, <, "aa", "a"); + COMPARE(false, <, "b", "aa"); + COMPARE(false, <, "bb", "aa"); + + COMPARE(true, <=, "a", "a"); + COMPARE(true, <=, "a", "b"); + COMPARE(true, <=, "a", "aa"); + COMPARE(true, <=, "aa", "b"); + COMPARE(true, <=, "aa", "bb"); + COMPARE(false, <=, "b", "a"); + COMPARE(false, <=, "aa", "a"); + COMPARE(false, <=, "b", "aa"); + COMPARE(false, <=, "bb", "aa"); + + COMPARE(false, >=, "a", "b"); + COMPARE(false, >=, "a", "aa"); + COMPARE(false, >=, "aa", "b"); + COMPARE(false, >=, "aa", "bb"); + COMPARE(true, >=, "a", "a"); + COMPARE(true, >=, "b", "a"); + COMPARE(true, >=, "aa", "a"); + COMPARE(true, >=, "b", "aa"); + COMPARE(true, >=, "bb", "aa"); + + COMPARE(false, >, "a", "a"); + COMPARE(false, >, "a", "b"); + COMPARE(false, >, "a", "aa"); + COMPARE(false, >, "aa", "b"); + COMPARE(false, >, "aa", "bb"); + COMPARE(true, >, "b", "a"); + COMPARE(true, >, "aa", "a"); + COMPARE(true, >, "b", "aa"); + COMPARE(true, >, "bb", "aa"); +} + +TEST(StringViewTest, ComparisonOperatorsByCharacterPosition) { + std::string x; + for (int i = 0; i < 256; i++) { + x += 'a'; + std::string y = x; + COMPARE(true, ==, x, y); + for (int j = 0; j < i; j++) { + std::string z = x; + z[j] = 'b'; // Differs in position 'j' + COMPARE(false, ==, x, z); + COMPARE(true, <, x, z); + COMPARE(true, >, z, x); + if (j + 1 < i) { + z[j + 1] = 'A'; // Differs in position 'j+1' as well + COMPARE(false, ==, x, z); + COMPARE(true, <, x, z); + COMPARE(true, >, z, x); + z[j + 1] = 'z'; // Differs in position 'j+1' as well + COMPARE(false, ==, x, z); + COMPARE(true, <, x, z); + COMPARE(true, >, z, x); + } + } + } +} +#undef COMPARE + +// Sadly, our users often confuse std::string::npos with +// absl::string_view::npos; So much so that we test here that they are the same. +// They need to both be unsigned, and both be the maximum-valued integer of +// their type. + +template <typename T> +struct is_type { + template <typename U> + static bool same(U) { + return false; + } + static bool same(T) { return true; } +}; + +TEST(StringViewTest, NposMatchesStdStringView) { + EXPECT_EQ(absl::string_view::npos, std::string::npos); + + EXPECT_TRUE(is_type<size_t>::same(absl::string_view::npos)); + EXPECT_FALSE(is_type<size_t>::same("")); + + // Make sure absl::string_view::npos continues to be a header constant. + char test[absl::string_view::npos & 1] = {0}; + EXPECT_EQ(0, test[0]); +} + +TEST(StringViewTest, STL1) { + const absl::string_view a("abcdefghijklmnopqrstuvwxyz"); + const absl::string_view b("abc"); + const absl::string_view c("xyz"); + const absl::string_view d("foobar"); + const absl::string_view e; + std::string temp("123"); + temp += '\0'; + temp += "456"; + const absl::string_view f(temp); + + EXPECT_EQ(a[6], 'g'); + EXPECT_EQ(b[0], 'a'); + EXPECT_EQ(c[2], 'z'); + EXPECT_EQ(f[3], '\0'); + EXPECT_EQ(f[5], '5'); + + EXPECT_EQ(*d.data(), 'f'); + EXPECT_EQ(d.data()[5], 'r'); + EXPECT_TRUE(e.data() == nullptr); + + EXPECT_EQ(*a.begin(), 'a'); + EXPECT_EQ(*(b.begin() + 2), 'c'); + EXPECT_EQ(*(c.end() - 1), 'z'); + + EXPECT_EQ(*a.rbegin(), 'z'); + EXPECT_EQ(*(b.rbegin() + 2), 'a'); + EXPECT_EQ(*(c.rend() - 1), 'x'); + EXPECT_TRUE(a.rbegin() + 26 == a.rend()); + + EXPECT_EQ(a.size(), 26); + EXPECT_EQ(b.size(), 3); + EXPECT_EQ(c.size(), 3); + EXPECT_EQ(d.size(), 6); + EXPECT_EQ(e.size(), 0); + EXPECT_EQ(f.size(), 7); + + EXPECT_TRUE(!d.empty()); + EXPECT_TRUE(d.begin() != d.end()); + EXPECT_TRUE(d.begin() + 6 == d.end()); + + EXPECT_TRUE(e.empty()); + EXPECT_TRUE(e.begin() == e.end()); + + char buf[4] = { '%', '%', '%', '%' }; + EXPECT_EQ(a.copy(buf, 4), 4); + EXPECT_EQ(buf[0], a[0]); + EXPECT_EQ(buf[1], a[1]); + EXPECT_EQ(buf[2], a[2]); + EXPECT_EQ(buf[3], a[3]); + EXPECT_EQ(a.copy(buf, 3, 7), 3); + EXPECT_EQ(buf[0], a[7]); + EXPECT_EQ(buf[1], a[8]); + EXPECT_EQ(buf[2], a[9]); + EXPECT_EQ(buf[3], a[3]); + EXPECT_EQ(c.copy(buf, 99), 3); + EXPECT_EQ(buf[0], c[0]); + EXPECT_EQ(buf[1], c[1]); + EXPECT_EQ(buf[2], c[2]); + EXPECT_EQ(buf[3], a[3]); +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(a.copy(buf, 1, 27), std::out_of_range); +#else + ABSL_EXPECT_DEATH_IF_SUPPORTED(a.copy(buf, 1, 27), "absl::string_view::copy"); +#endif +} + +// Separated from STL1() because some compilers produce an overly +// large stack frame for the combined function. +TEST(StringViewTest, STL2) { + const absl::string_view a("abcdefghijklmnopqrstuvwxyz"); + const absl::string_view b("abc"); + const absl::string_view c("xyz"); + absl::string_view d("foobar"); + const absl::string_view e; + const absl::string_view f( + "123" + "\0" + "456", + 7); + + d = absl::string_view(); + EXPECT_EQ(d.size(), 0); + EXPECT_TRUE(d.empty()); + EXPECT_TRUE(d.data() == nullptr); + EXPECT_TRUE(d.begin() == d.end()); + + EXPECT_EQ(a.find(b), 0); + EXPECT_EQ(a.find(b, 1), absl::string_view::npos); + EXPECT_EQ(a.find(c), 23); + EXPECT_EQ(a.find(c, 9), 23); + EXPECT_EQ(a.find(c, absl::string_view::npos), absl::string_view::npos); + EXPECT_EQ(b.find(c), absl::string_view::npos); + EXPECT_EQ(b.find(c, absl::string_view::npos), absl::string_view::npos); + EXPECT_EQ(a.find(d), 0); + EXPECT_EQ(a.find(e), 0); + EXPECT_EQ(a.find(d, 12), 12); + EXPECT_EQ(a.find(e, 17), 17); + absl::string_view g("xx not found bb"); + EXPECT_EQ(a.find(g), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(d.find(b), absl::string_view::npos); + EXPECT_EQ(e.find(b), absl::string_view::npos); + EXPECT_EQ(d.find(b, 4), absl::string_view::npos); + EXPECT_EQ(e.find(b, 7), absl::string_view::npos); + + size_t empty_search_pos = std::string().find(std::string()); + EXPECT_EQ(d.find(d), empty_search_pos); + EXPECT_EQ(d.find(e), empty_search_pos); + EXPECT_EQ(e.find(d), empty_search_pos); + EXPECT_EQ(e.find(e), empty_search_pos); + EXPECT_EQ(d.find(d, 4), std::string().find(std::string(), 4)); + EXPECT_EQ(d.find(e, 4), std::string().find(std::string(), 4)); + EXPECT_EQ(e.find(d, 4), std::string().find(std::string(), 4)); + EXPECT_EQ(e.find(e, 4), std::string().find(std::string(), 4)); + + EXPECT_EQ(a.find('a'), 0); + EXPECT_EQ(a.find('c'), 2); + EXPECT_EQ(a.find('z'), 25); + EXPECT_EQ(a.find('$'), absl::string_view::npos); + EXPECT_EQ(a.find('\0'), absl::string_view::npos); + EXPECT_EQ(f.find('\0'), 3); + EXPECT_EQ(f.find('3'), 2); + EXPECT_EQ(f.find('5'), 5); + EXPECT_EQ(g.find('o'), 4); + EXPECT_EQ(g.find('o', 4), 4); + EXPECT_EQ(g.find('o', 5), 8); + EXPECT_EQ(a.find('b', 5), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(d.find('\0'), absl::string_view::npos); + EXPECT_EQ(e.find('\0'), absl::string_view::npos); + EXPECT_EQ(d.find('\0', 4), absl::string_view::npos); + EXPECT_EQ(e.find('\0', 7), absl::string_view::npos); + EXPECT_EQ(d.find('x'), absl::string_view::npos); + EXPECT_EQ(e.find('x'), absl::string_view::npos); + EXPECT_EQ(d.find('x', 4), absl::string_view::npos); + EXPECT_EQ(e.find('x', 7), absl::string_view::npos); + + EXPECT_EQ(a.rfind(b), 0); + EXPECT_EQ(a.rfind(b, 1), 0); + EXPECT_EQ(a.rfind(c), 23); + EXPECT_EQ(a.rfind(c, 22), absl::string_view::npos); + EXPECT_EQ(a.rfind(c, 1), absl::string_view::npos); + EXPECT_EQ(a.rfind(c, 0), absl::string_view::npos); + EXPECT_EQ(b.rfind(c), absl::string_view::npos); + EXPECT_EQ(b.rfind(c, 0), absl::string_view::npos); + EXPECT_EQ(a.rfind(d), std::string(a).rfind(std::string())); + EXPECT_EQ(a.rfind(e), std::string(a).rfind(std::string())); + EXPECT_EQ(a.rfind(d, 12), 12); + EXPECT_EQ(a.rfind(e, 17), 17); + EXPECT_EQ(a.rfind(g), absl::string_view::npos); + EXPECT_EQ(d.rfind(b), absl::string_view::npos); + EXPECT_EQ(e.rfind(b), absl::string_view::npos); + EXPECT_EQ(d.rfind(b, 4), absl::string_view::npos); + EXPECT_EQ(e.rfind(b, 7), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(d.rfind(d, 4), std::string().rfind(std::string())); + EXPECT_EQ(e.rfind(d, 7), std::string().rfind(std::string())); + EXPECT_EQ(d.rfind(e, 4), std::string().rfind(std::string())); + EXPECT_EQ(e.rfind(e, 7), std::string().rfind(std::string())); + EXPECT_EQ(d.rfind(d), std::string().rfind(std::string())); + EXPECT_EQ(e.rfind(d), std::string().rfind(std::string())); + EXPECT_EQ(d.rfind(e), std::string().rfind(std::string())); + EXPECT_EQ(e.rfind(e), std::string().rfind(std::string())); + + EXPECT_EQ(g.rfind('o'), 8); + EXPECT_EQ(g.rfind('q'), absl::string_view::npos); + EXPECT_EQ(g.rfind('o', 8), 8); + EXPECT_EQ(g.rfind('o', 7), 4); + EXPECT_EQ(g.rfind('o', 3), absl::string_view::npos); + EXPECT_EQ(f.rfind('\0'), 3); + EXPECT_EQ(f.rfind('\0', 12), 3); + EXPECT_EQ(f.rfind('3'), 2); + EXPECT_EQ(f.rfind('5'), 5); + // empty string nonsense + EXPECT_EQ(d.rfind('o'), absl::string_view::npos); + EXPECT_EQ(e.rfind('o'), absl::string_view::npos); + EXPECT_EQ(d.rfind('o', 4), absl::string_view::npos); + EXPECT_EQ(e.rfind('o', 7), absl::string_view::npos); +} + +// Continued from STL2 +TEST(StringViewTest, STL2FindFirst) { + const absl::string_view a("abcdefghijklmnopqrstuvwxyz"); + const absl::string_view b("abc"); + const absl::string_view c("xyz"); + absl::string_view d("foobar"); + const absl::string_view e; + const absl::string_view f( + "123" + "\0" + "456", + 7); + absl::string_view g("xx not found bb"); + + d = absl::string_view(); + EXPECT_EQ(a.find_first_of(b), 0); + EXPECT_EQ(a.find_first_of(b, 0), 0); + EXPECT_EQ(a.find_first_of(b, 1), 1); + EXPECT_EQ(a.find_first_of(b, 2), 2); + EXPECT_EQ(a.find_first_of(b, 3), absl::string_view::npos); + EXPECT_EQ(a.find_first_of(c), 23); + EXPECT_EQ(a.find_first_of(c, 23), 23); + EXPECT_EQ(a.find_first_of(c, 24), 24); + EXPECT_EQ(a.find_first_of(c, 25), 25); + EXPECT_EQ(a.find_first_of(c, 26), absl::string_view::npos); + EXPECT_EQ(g.find_first_of(b), 13); + EXPECT_EQ(g.find_first_of(c), 0); + EXPECT_EQ(a.find_first_of(f), absl::string_view::npos); + EXPECT_EQ(f.find_first_of(a), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(a.find_first_of(d), absl::string_view::npos); + EXPECT_EQ(a.find_first_of(e), absl::string_view::npos); + EXPECT_EQ(d.find_first_of(b), absl::string_view::npos); + EXPECT_EQ(e.find_first_of(b), absl::string_view::npos); + EXPECT_EQ(d.find_first_of(d), absl::string_view::npos); + EXPECT_EQ(e.find_first_of(d), absl::string_view::npos); + EXPECT_EQ(d.find_first_of(e), absl::string_view::npos); + EXPECT_EQ(e.find_first_of(e), absl::string_view::npos); + + EXPECT_EQ(a.find_first_not_of(b), 3); + EXPECT_EQ(a.find_first_not_of(c), 0); + EXPECT_EQ(b.find_first_not_of(a), absl::string_view::npos); + EXPECT_EQ(c.find_first_not_of(a), absl::string_view::npos); + EXPECT_EQ(f.find_first_not_of(a), 0); + EXPECT_EQ(a.find_first_not_of(f), 0); + EXPECT_EQ(a.find_first_not_of(d), 0); + EXPECT_EQ(a.find_first_not_of(e), 0); + // empty string nonsense + EXPECT_EQ(a.find_first_not_of(d), 0); + EXPECT_EQ(a.find_first_not_of(e), 0); + EXPECT_EQ(a.find_first_not_of(d, 1), 1); + EXPECT_EQ(a.find_first_not_of(e, 1), 1); + EXPECT_EQ(a.find_first_not_of(d, a.size() - 1), a.size() - 1); + EXPECT_EQ(a.find_first_not_of(e, a.size() - 1), a.size() - 1); + EXPECT_EQ(a.find_first_not_of(d, a.size()), absl::string_view::npos); + EXPECT_EQ(a.find_first_not_of(e, a.size()), absl::string_view::npos); + EXPECT_EQ(a.find_first_not_of(d, absl::string_view::npos), + absl::string_view::npos); + EXPECT_EQ(a.find_first_not_of(e, absl::string_view::npos), + absl::string_view::npos); + EXPECT_EQ(d.find_first_not_of(a), absl::string_view::npos); + EXPECT_EQ(e.find_first_not_of(a), absl::string_view::npos); + EXPECT_EQ(d.find_first_not_of(d), absl::string_view::npos); + EXPECT_EQ(e.find_first_not_of(d), absl::string_view::npos); + EXPECT_EQ(d.find_first_not_of(e), absl::string_view::npos); + EXPECT_EQ(e.find_first_not_of(e), absl::string_view::npos); + + absl::string_view h("===="); + EXPECT_EQ(h.find_first_not_of('='), absl::string_view::npos); + EXPECT_EQ(h.find_first_not_of('=', 3), absl::string_view::npos); + EXPECT_EQ(h.find_first_not_of('\0'), 0); + EXPECT_EQ(g.find_first_not_of('x'), 2); + EXPECT_EQ(f.find_first_not_of('\0'), 0); + EXPECT_EQ(f.find_first_not_of('\0', 3), 4); + EXPECT_EQ(f.find_first_not_of('\0', 2), 2); + // empty string nonsense + EXPECT_EQ(d.find_first_not_of('x'), absl::string_view::npos); + EXPECT_EQ(e.find_first_not_of('x'), absl::string_view::npos); + EXPECT_EQ(d.find_first_not_of('\0'), absl::string_view::npos); + EXPECT_EQ(e.find_first_not_of('\0'), absl::string_view::npos); +} + +// Continued from STL2 +TEST(StringViewTest, STL2FindLast) { + const absl::string_view a("abcdefghijklmnopqrstuvwxyz"); + const absl::string_view b("abc"); + const absl::string_view c("xyz"); + absl::string_view d("foobar"); + const absl::string_view e; + const absl::string_view f( + "123" + "\0" + "456", + 7); + absl::string_view g("xx not found bb"); + absl::string_view h("===="); + absl::string_view i("56"); + + d = absl::string_view(); + EXPECT_EQ(h.find_last_of(a), absl::string_view::npos); + EXPECT_EQ(g.find_last_of(a), g.size()-1); + EXPECT_EQ(a.find_last_of(b), 2); + EXPECT_EQ(a.find_last_of(c), a.size()-1); + EXPECT_EQ(f.find_last_of(i), 6); + EXPECT_EQ(a.find_last_of('a'), 0); + EXPECT_EQ(a.find_last_of('b'), 1); + EXPECT_EQ(a.find_last_of('z'), 25); + EXPECT_EQ(a.find_last_of('a', 5), 0); + EXPECT_EQ(a.find_last_of('b', 5), 1); + EXPECT_EQ(a.find_last_of('b', 0), absl::string_view::npos); + EXPECT_EQ(a.find_last_of('z', 25), 25); + EXPECT_EQ(a.find_last_of('z', 24), absl::string_view::npos); + EXPECT_EQ(f.find_last_of(i, 5), 5); + EXPECT_EQ(f.find_last_of(i, 6), 6); + EXPECT_EQ(f.find_last_of(a, 4), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(f.find_last_of(d), absl::string_view::npos); + EXPECT_EQ(f.find_last_of(e), absl::string_view::npos); + EXPECT_EQ(f.find_last_of(d, 4), absl::string_view::npos); + EXPECT_EQ(f.find_last_of(e, 4), absl::string_view::npos); + EXPECT_EQ(d.find_last_of(d), absl::string_view::npos); + EXPECT_EQ(d.find_last_of(e), absl::string_view::npos); + EXPECT_EQ(e.find_last_of(d), absl::string_view::npos); + EXPECT_EQ(e.find_last_of(e), absl::string_view::npos); + EXPECT_EQ(d.find_last_of(f), absl::string_view::npos); + EXPECT_EQ(e.find_last_of(f), absl::string_view::npos); + EXPECT_EQ(d.find_last_of(d, 4), absl::string_view::npos); + EXPECT_EQ(d.find_last_of(e, 4), absl::string_view::npos); + EXPECT_EQ(e.find_last_of(d, 4), absl::string_view::npos); + EXPECT_EQ(e.find_last_of(e, 4), absl::string_view::npos); + EXPECT_EQ(d.find_last_of(f, 4), absl::string_view::npos); + EXPECT_EQ(e.find_last_of(f, 4), absl::string_view::npos); + + EXPECT_EQ(a.find_last_not_of(b), a.size()-1); + EXPECT_EQ(a.find_last_not_of(c), 22); + EXPECT_EQ(b.find_last_not_of(a), absl::string_view::npos); + EXPECT_EQ(b.find_last_not_of(b), absl::string_view::npos); + EXPECT_EQ(f.find_last_not_of(i), 4); + EXPECT_EQ(a.find_last_not_of(c, 24), 22); + EXPECT_EQ(a.find_last_not_of(b, 3), 3); + EXPECT_EQ(a.find_last_not_of(b, 2), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(f.find_last_not_of(d), f.size()-1); + EXPECT_EQ(f.find_last_not_of(e), f.size()-1); + EXPECT_EQ(f.find_last_not_of(d, 4), 4); + EXPECT_EQ(f.find_last_not_of(e, 4), 4); + EXPECT_EQ(d.find_last_not_of(d), absl::string_view::npos); + EXPECT_EQ(d.find_last_not_of(e), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of(d), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of(e), absl::string_view::npos); + EXPECT_EQ(d.find_last_not_of(f), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of(f), absl::string_view::npos); + EXPECT_EQ(d.find_last_not_of(d, 4), absl::string_view::npos); + EXPECT_EQ(d.find_last_not_of(e, 4), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of(d, 4), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of(e, 4), absl::string_view::npos); + EXPECT_EQ(d.find_last_not_of(f, 4), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of(f, 4), absl::string_view::npos); + + EXPECT_EQ(h.find_last_not_of('x'), h.size() - 1); + EXPECT_EQ(h.find_last_not_of('='), absl::string_view::npos); + EXPECT_EQ(b.find_last_not_of('c'), 1); + EXPECT_EQ(h.find_last_not_of('x', 2), 2); + EXPECT_EQ(h.find_last_not_of('=', 2), absl::string_view::npos); + EXPECT_EQ(b.find_last_not_of('b', 1), 0); + // empty string nonsense + EXPECT_EQ(d.find_last_not_of('x'), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of('x'), absl::string_view::npos); + EXPECT_EQ(d.find_last_not_of('\0'), absl::string_view::npos); + EXPECT_EQ(e.find_last_not_of('\0'), absl::string_view::npos); +} + +// Continued from STL2 +TEST(StringViewTest, STL2Substr) { + const absl::string_view a("abcdefghijklmnopqrstuvwxyz"); + const absl::string_view b("abc"); + const absl::string_view c("xyz"); + absl::string_view d("foobar"); + const absl::string_view e; + + d = absl::string_view(); + EXPECT_EQ(a.substr(0, 3), b); + EXPECT_EQ(a.substr(23), c); + EXPECT_EQ(a.substr(23, 3), c); + EXPECT_EQ(a.substr(23, 99), c); + EXPECT_EQ(a.substr(0), a); + EXPECT_EQ(a.substr(3, 2), "de"); + // empty string nonsense + EXPECT_EQ(d.substr(0, 99), e); + // use of npos + EXPECT_EQ(a.substr(0, absl::string_view::npos), a); + EXPECT_EQ(a.substr(23, absl::string_view::npos), c); + // throw exception +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW((void)a.substr(99, 2), std::out_of_range); +#else + ABSL_EXPECT_DEATH_IF_SUPPORTED((void)a.substr(99, 2), + "absl::string_view::substr"); +#endif +} + +TEST(StringViewTest, TruncSubstr) { + const absl::string_view hi("hi"); + EXPECT_EQ("", absl::ClippedSubstr(hi, 0, 0)); + EXPECT_EQ("h", absl::ClippedSubstr(hi, 0, 1)); + EXPECT_EQ("hi", absl::ClippedSubstr(hi, 0)); + EXPECT_EQ("i", absl::ClippedSubstr(hi, 1)); + EXPECT_EQ("", absl::ClippedSubstr(hi, 2)); + EXPECT_EQ("", absl::ClippedSubstr(hi, 3)); // truncation + EXPECT_EQ("", absl::ClippedSubstr(hi, 3, 2)); // truncation +} + +TEST(StringViewTest, UTF8) { + std::string utf8 = "\u00E1"; + std::string utf8_twice = utf8 + " " + utf8; + int utf8_len = strlen(utf8.data()); + EXPECT_EQ(utf8_len, absl::string_view(utf8_twice).find_first_of(" ")); + EXPECT_EQ(utf8_len, absl::string_view(utf8_twice).find_first_of(" \t")); +} + +TEST(StringViewTest, FindConformance) { + struct { + std::string haystack; + std::string needle; + } specs[] = { + {"", ""}, + {"", "a"}, + {"a", ""}, + {"a", "a"}, + {"a", "b"}, + {"aa", ""}, + {"aa", "a"}, + {"aa", "b"}, + {"ab", "a"}, + {"ab", "b"}, + {"abcd", ""}, + {"abcd", "a"}, + {"abcd", "d"}, + {"abcd", "ab"}, + {"abcd", "bc"}, + {"abcd", "cd"}, + {"abcd", "abcd"}, + }; + for (const auto& s : specs) { + SCOPED_TRACE(s.haystack); + SCOPED_TRACE(s.needle); + std::string st = s.haystack; + absl::string_view sp = s.haystack; + for (size_t i = 0; i <= sp.size(); ++i) { + size_t pos = (i == sp.size()) ? absl::string_view::npos : i; + SCOPED_TRACE(pos); + EXPECT_EQ(sp.find(s.needle, pos), + st.find(s.needle, pos)); + EXPECT_EQ(sp.rfind(s.needle, pos), + st.rfind(s.needle, pos)); + EXPECT_EQ(sp.find_first_of(s.needle, pos), + st.find_first_of(s.needle, pos)); + EXPECT_EQ(sp.find_first_not_of(s.needle, pos), + st.find_first_not_of(s.needle, pos)); + EXPECT_EQ(sp.find_last_of(s.needle, pos), + st.find_last_of(s.needle, pos)); + EXPECT_EQ(sp.find_last_not_of(s.needle, pos), + st.find_last_not_of(s.needle, pos)); + } + } +} + +TEST(StringViewTest, Remove) { + absl::string_view a("foobar"); + std::string s1("123"); + s1 += '\0'; + s1 += "456"; + absl::string_view e; + std::string s2; + + // remove_prefix + absl::string_view c(a); + c.remove_prefix(3); + EXPECT_EQ(c, "bar"); + c = a; + c.remove_prefix(0); + EXPECT_EQ(c, a); + c.remove_prefix(c.size()); + EXPECT_EQ(c, e); + + // remove_suffix + c = a; + c.remove_suffix(3); + EXPECT_EQ(c, "foo"); + c = a; + c.remove_suffix(0); + EXPECT_EQ(c, a); + c.remove_suffix(c.size()); + EXPECT_EQ(c, e); +} + +TEST(StringViewTest, Set) { + absl::string_view a("foobar"); + absl::string_view empty; + absl::string_view b; + + // set + b = absl::string_view("foobar", 6); + EXPECT_EQ(b, a); + b = absl::string_view("foobar", 0); + EXPECT_EQ(b, empty); + b = absl::string_view("foobar", 7); + EXPECT_NE(b, a); + + b = absl::string_view("foobar"); + EXPECT_EQ(b, a); +} + +TEST(StringViewTest, FrontBack) { + static const char arr[] = "abcd"; + const absl::string_view csp(arr, 4); + EXPECT_EQ(&arr[0], &csp.front()); + EXPECT_EQ(&arr[3], &csp.back()); +} + +TEST(StringViewTest, FrontBackSingleChar) { + static const char c = 'a'; + const absl::string_view csp(&c, 1); + EXPECT_EQ(&c, &csp.front()); + EXPECT_EQ(&c, &csp.back()); +} + +TEST(StringViewTest, FrontBackEmpty) { +#ifndef ABSL_USES_STD_STRING_VIEW +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + // Abseil's string_view implementation has debug assertions that check that + // front() and back() are not called on an empty string_view. + absl::string_view sv; + ABSL_EXPECT_DEATH_IF_SUPPORTED(sv.front(), ""); + ABSL_EXPECT_DEATH_IF_SUPPORTED(sv.back(), ""); +#endif +#endif +} + +// `std::string_view::string_view(const char*)` calls +// `std::char_traits<char>::length(const char*)` to get the string length. In +// libc++, it doesn't allow `nullptr` in the constexpr context, with the error +// "read of dereferenced null pointer is not allowed in a constant expression". +// At run time, the behavior of `std::char_traits::length()` on `nullptr` is +// undefined by the standard and usually results in crash with libc++. +// GCC also started rejected this in libstdc++ starting in GCC9. +// In MSVC, creating a constexpr string_view from nullptr also triggers an +// "unevaluable pointer value" error. This compiler implementation conforms +// to the standard, but `absl::string_view` implements a different +// behavior for historical reasons. We work around tests that construct +// `string_view` from `nullptr` when using libc++. +#if !defined(ABSL_USES_STD_STRING_VIEW) || \ + (!(defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE >= 9) && \ + !defined(_LIBCPP_VERSION) && !defined(_MSC_VER)) +#define ABSL_HAVE_STRING_VIEW_FROM_NULLPTR 1 +#endif + +TEST(StringViewTest, NULLInput) { + absl::string_view s; + EXPECT_EQ(s.data(), nullptr); + EXPECT_EQ(s.size(), 0); + +#ifdef ABSL_HAVE_STRING_VIEW_FROM_NULLPTR + s = absl::string_view(nullptr); + EXPECT_EQ(s.data(), nullptr); + EXPECT_EQ(s.size(), 0); + + // .ToString() on a absl::string_view with nullptr should produce the empty + // string. + EXPECT_EQ("", std::string(s)); +#endif // ABSL_HAVE_STRING_VIEW_FROM_NULLPTR +} + +TEST(StringViewTest, Comparisons2) { + // The `compare` member has 6 overloads (v: string_view, s: const char*): + // (1) compare(v) + // (2) compare(pos1, count1, v) + // (3) compare(pos1, count1, v, pos2, count2) + // (4) compare(s) + // (5) compare(pos1, count1, s) + // (6) compare(pos1, count1, s, count2) + + absl::string_view abc("abcdefghijklmnopqrstuvwxyz"); + + // check comparison operations on strings longer than 4 bytes. + EXPECT_EQ(abc, absl::string_view("abcdefghijklmnopqrstuvwxyz")); + EXPECT_EQ(abc.compare(absl::string_view("abcdefghijklmnopqrstuvwxyz")), 0); + + EXPECT_LT(abc, absl::string_view("abcdefghijklmnopqrstuvwxzz")); + EXPECT_LT(abc.compare(absl::string_view("abcdefghijklmnopqrstuvwxzz")), 0); + + EXPECT_GT(abc, absl::string_view("abcdefghijklmnopqrstuvwxyy")); + EXPECT_GT(abc.compare(absl::string_view("abcdefghijklmnopqrstuvwxyy")), 0); + + // The "substr" variants of `compare`. + absl::string_view digits("0123456789"); + auto npos = absl::string_view::npos; + + // Taking string_view + EXPECT_EQ(digits.compare(3, npos, absl::string_view("3456789")), 0); // 2 + EXPECT_EQ(digits.compare(3, 4, absl::string_view("3456")), 0); // 2 + EXPECT_EQ(digits.compare(10, 0, absl::string_view()), 0); // 2 + EXPECT_EQ(digits.compare(3, 4, absl::string_view("0123456789"), 3, 4), + 0); // 3 + EXPECT_LT(digits.compare(3, 4, absl::string_view("0123456789"), 3, 5), + 0); // 3 + EXPECT_LT(digits.compare(0, npos, absl::string_view("0123456789"), 3, 5), + 0); // 3 + // Taking const char* + EXPECT_EQ(digits.compare(3, 4, "3456"), 0); // 5 + EXPECT_EQ(digits.compare(3, npos, "3456789"), 0); // 5 + EXPECT_EQ(digits.compare(10, 0, ""), 0); // 5 + EXPECT_EQ(digits.compare(3, 4, "0123456789", 3, 4), 0); // 6 + EXPECT_LT(digits.compare(3, 4, "0123456789", 3, 5), 0); // 6 + EXPECT_LT(digits.compare(0, npos, "0123456789", 3, 5), 0); // 6 +} + +TEST(StringViewTest, At) { + absl::string_view abc = "abc"; + EXPECT_EQ(abc.at(0), 'a'); + EXPECT_EQ(abc.at(1), 'b'); + EXPECT_EQ(abc.at(2), 'c'); +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(abc.at(3), std::out_of_range); +#else + ABSL_EXPECT_DEATH_IF_SUPPORTED(abc.at(3), "absl::string_view::at"); +#endif +} + +struct MyCharAlloc : std::allocator<char> {}; + +TEST(StringViewTest, ExplicitConversionOperator) { + absl::string_view sp = "hi"; + EXPECT_EQ(sp, std::string(sp)); +} + +TEST(StringViewTest, NullSafeStringView) { + { + absl::string_view s = absl::NullSafeStringView(nullptr); + EXPECT_EQ(nullptr, s.data()); + EXPECT_EQ(0, s.size()); + EXPECT_EQ(absl::string_view(), s); + } + { + static const char kHi[] = "hi"; + absl::string_view s = absl::NullSafeStringView(kHi); + EXPECT_EQ(kHi, s.data()); + EXPECT_EQ(strlen(kHi), s.size()); + EXPECT_EQ(absl::string_view("hi"), s); + } +} + +TEST(StringViewTest, ConstexprNullSafeStringView) { + { + constexpr absl::string_view s = absl::NullSafeStringView(nullptr); + EXPECT_EQ(nullptr, s.data()); + EXPECT_EQ(0, s.size()); + EXPECT_EQ(absl::string_view(), s); + } +#if !defined(_MSC_VER) || _MSC_VER >= 1910 + // MSVC 2017+ is required for good constexpr string_view support. + // See the implementation of `absl::string_view::StrlenInternal()`. + { + static constexpr char kHi[] = "hi"; + absl::string_view s = absl::NullSafeStringView(kHi); + EXPECT_EQ(kHi, s.data()); + EXPECT_EQ(strlen(kHi), s.size()); + EXPECT_EQ(absl::string_view("hi"), s); + } + { + constexpr absl::string_view s = absl::NullSafeStringView("hello"); + EXPECT_EQ(s.size(), 5); + EXPECT_EQ("hello", s); + } +#endif +} + +TEST(StringViewTest, ConstexprCompiles) { + constexpr absl::string_view sp; +#ifdef ABSL_HAVE_STRING_VIEW_FROM_NULLPTR + constexpr absl::string_view cstr(nullptr); +#endif + constexpr absl::string_view cstr_len("cstr", 4); + +#if defined(ABSL_USES_STD_STRING_VIEW) + // In libstdc++ (as of 7.2), `std::string_view::string_view(const char*)` + // calls `std::char_traits<char>::length(const char*)` to get the string + // length, but it is not marked constexpr yet. See GCC bug: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78156 + // Also, there is a LWG issue that adds constexpr to length() which was just + // resolved 2017-06-02. See + // http://www.open-std.org/jtc1/sc22/wg21/docs/lwg-defects.html#2232 + // TODO(zhangxy): Update the condition when libstdc++ adopts the constexpr + // length(). +#if !defined(__GLIBCXX__) +#define ABSL_HAVE_CONSTEXPR_STRING_VIEW_FROM_CSTR 1 +#endif // !__GLIBCXX__ + +#else // ABSL_USES_STD_STRING_VIEW + +// This duplicates the check for __builtin_strlen in the header. +#if ABSL_HAVE_BUILTIN(__builtin_strlen) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_HAVE_CONSTEXPR_STRING_VIEW_FROM_CSTR 1 +#elif defined(__GNUC__) // GCC or clang +#error GCC/clang should have constexpr string_view. +#endif + +// MSVC 2017+ should be able to construct a constexpr string_view from a cstr. +#if defined(_MSC_VER) && _MSC_VER >= 1910 +#define ABSL_HAVE_CONSTEXPR_STRING_VIEW_FROM_CSTR 1 +#endif + +#endif // ABSL_USES_STD_STRING_VIEW + +#ifdef ABSL_HAVE_CONSTEXPR_STRING_VIEW_FROM_CSTR + constexpr absl::string_view cstr_strlen("foo"); + EXPECT_EQ(cstr_strlen.length(), 3); + constexpr absl::string_view cstr_strlen2 = "bar"; + EXPECT_EQ(cstr_strlen2, "bar"); + +#if ABSL_HAVE_BUILTIN(__builtin_memcmp) || \ + (defined(__GNUC__) && !defined(__clang__)) +#define ABSL_HAVE_CONSTEXPR_STRING_VIEW_COMPARISON 1 +#endif +#ifdef ABSL_HAVE_CONSTEXPR_STRING_VIEW_COMPARISON + constexpr absl::string_view foo = "foo"; + constexpr absl::string_view bar = "bar"; + constexpr bool foo_eq_bar = foo == bar; + constexpr bool foo_ne_bar = foo != bar; + constexpr bool foo_lt_bar = foo < bar; + constexpr bool foo_le_bar = foo <= bar; + constexpr bool foo_gt_bar = foo > bar; + constexpr bool foo_ge_bar = foo >= bar; + constexpr int foo_compare_bar = foo.compare(bar); + EXPECT_FALSE(foo_eq_bar); + EXPECT_TRUE(foo_ne_bar); + EXPECT_FALSE(foo_lt_bar); + EXPECT_FALSE(foo_le_bar); + EXPECT_TRUE(foo_gt_bar); + EXPECT_TRUE(foo_ge_bar); + EXPECT_GT(foo_compare_bar, 0); +#endif +#endif + +#if !defined(__clang__) || 3 < __clang_major__ || \ + (3 == __clang_major__ && 4 < __clang_minor__) + // older clang versions (< 3.5) complain that: + // "cannot perform pointer arithmetic on null pointer" + constexpr absl::string_view::iterator const_begin_empty = sp.begin(); + constexpr absl::string_view::iterator const_end_empty = sp.end(); + EXPECT_EQ(const_begin_empty, const_end_empty); + +#ifdef ABSL_HAVE_STRING_VIEW_FROM_NULLPTR + constexpr absl::string_view::iterator const_begin_nullptr = cstr.begin(); + constexpr absl::string_view::iterator const_end_nullptr = cstr.end(); + EXPECT_EQ(const_begin_nullptr, const_end_nullptr); +#endif // ABSL_HAVE_STRING_VIEW_FROM_NULLPTR +#endif // !defined(__clang__) || ... + + constexpr absl::string_view::iterator const_begin = cstr_len.begin(); + constexpr absl::string_view::iterator const_end = cstr_len.end(); + constexpr absl::string_view::size_type const_size = cstr_len.size(); + constexpr absl::string_view::size_type const_length = cstr_len.length(); + static_assert(const_begin + const_size == const_end, + "pointer arithmetic check"); + static_assert(const_begin + const_length == const_end, + "pointer arithmetic check"); +#ifndef _MSC_VER + // MSVC has bugs doing constexpr pointer arithmetic. + // https://developercommunity.visualstudio.com/content/problem/482192/bad-pointer-arithmetic-in-constepxr-2019-rc1-svc1.html + EXPECT_EQ(const_begin + const_size, const_end); + EXPECT_EQ(const_begin + const_length, const_end); +#endif + + constexpr bool isempty = sp.empty(); + EXPECT_TRUE(isempty); + + constexpr const char c = cstr_len[2]; + EXPECT_EQ(c, 't'); + + constexpr const char cfront = cstr_len.front(); + constexpr const char cback = cstr_len.back(); + EXPECT_EQ(cfront, 'c'); + EXPECT_EQ(cback, 'r'); + + constexpr const char* np = sp.data(); + constexpr const char* cstr_ptr = cstr_len.data(); + EXPECT_EQ(np, nullptr); + EXPECT_NE(cstr_ptr, nullptr); + + constexpr size_t sp_npos = sp.npos; + EXPECT_EQ(sp_npos, -1); +} + +TEST(StringViewTest, ConstexprSubstr) { + constexpr absl::string_view foobar("foobar", 6); + constexpr absl::string_view foo = foobar.substr(0, 3); + constexpr absl::string_view bar = foobar.substr(3); + EXPECT_EQ(foo, "foo"); + EXPECT_EQ(bar, "bar"); +} + +TEST(StringViewTest, Noexcept) { + EXPECT_TRUE((std::is_nothrow_constructible<absl::string_view, + const std::string&>::value)); + EXPECT_TRUE((std::is_nothrow_constructible<absl::string_view, + const std::string&>::value)); + EXPECT_TRUE(std::is_nothrow_constructible<absl::string_view>::value); + constexpr absl::string_view sp; + EXPECT_TRUE(noexcept(sp.begin())); + EXPECT_TRUE(noexcept(sp.end())); + EXPECT_TRUE(noexcept(sp.cbegin())); + EXPECT_TRUE(noexcept(sp.cend())); + EXPECT_TRUE(noexcept(sp.rbegin())); + EXPECT_TRUE(noexcept(sp.rend())); + EXPECT_TRUE(noexcept(sp.crbegin())); + EXPECT_TRUE(noexcept(sp.crend())); + EXPECT_TRUE(noexcept(sp.size())); + EXPECT_TRUE(noexcept(sp.length())); + EXPECT_TRUE(noexcept(sp.empty())); + EXPECT_TRUE(noexcept(sp.data())); + EXPECT_TRUE(noexcept(sp.compare(sp))); + EXPECT_TRUE(noexcept(sp.find(sp))); + EXPECT_TRUE(noexcept(sp.find('f'))); + EXPECT_TRUE(noexcept(sp.rfind(sp))); + EXPECT_TRUE(noexcept(sp.rfind('f'))); + EXPECT_TRUE(noexcept(sp.find_first_of(sp))); + EXPECT_TRUE(noexcept(sp.find_first_of('f'))); + EXPECT_TRUE(noexcept(sp.find_last_of(sp))); + EXPECT_TRUE(noexcept(sp.find_last_of('f'))); + EXPECT_TRUE(noexcept(sp.find_first_not_of(sp))); + EXPECT_TRUE(noexcept(sp.find_first_not_of('f'))); + EXPECT_TRUE(noexcept(sp.find_last_not_of(sp))); + EXPECT_TRUE(noexcept(sp.find_last_not_of('f'))); +} + +TEST(StringViewTest, BoundsCheck) { +#ifndef ABSL_USES_STD_STRING_VIEW +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + // Abseil's string_view implementation has bounds-checking in debug mode. + absl::string_view h = "hello"; + ABSL_EXPECT_DEATH_IF_SUPPORTED(h[5], ""); + ABSL_EXPECT_DEATH_IF_SUPPORTED(h[-1], ""); +#endif +#endif +} + +TEST(ComparisonOpsTest, StringCompareNotAmbiguous) { + EXPECT_EQ("hello", std::string("hello")); + EXPECT_LT("hello", std::string("world")); +} + +TEST(ComparisonOpsTest, HeterogenousStringViewEquals) { + EXPECT_EQ(absl::string_view("hello"), std::string("hello")); + EXPECT_EQ("hello", absl::string_view("hello")); +} + +TEST(FindOneCharTest, EdgeCases) { + absl::string_view a("xxyyyxx"); + + // Set a = "xyyyx". + a.remove_prefix(1); + a.remove_suffix(1); + + EXPECT_EQ(0, a.find('x')); + EXPECT_EQ(0, a.find('x', 0)); + EXPECT_EQ(4, a.find('x', 1)); + EXPECT_EQ(4, a.find('x', 4)); + EXPECT_EQ(absl::string_view::npos, a.find('x', 5)); + + EXPECT_EQ(4, a.rfind('x')); + EXPECT_EQ(4, a.rfind('x', 5)); + EXPECT_EQ(4, a.rfind('x', 4)); + EXPECT_EQ(0, a.rfind('x', 3)); + EXPECT_EQ(0, a.rfind('x', 0)); + + // Set a = "yyy". + a.remove_prefix(1); + a.remove_suffix(1); + + EXPECT_EQ(absl::string_view::npos, a.find('x')); + EXPECT_EQ(absl::string_view::npos, a.rfind('x')); +} + +#ifndef ABSL_HAVE_THREAD_SANITIZER // Allocates too much memory for tsan. +TEST(HugeStringView, TwoPointTwoGB) { + if (sizeof(size_t) <= 4) + return; + // Try a huge string piece. + const size_t size = size_t{2200} * 1000 * 1000; + std::string s(size, 'a'); + absl::string_view sp(s); + EXPECT_EQ(size, sp.length()); + sp.remove_prefix(1); + EXPECT_EQ(size - 1, sp.length()); + sp.remove_suffix(2); + EXPECT_EQ(size - 1 - 2, sp.length()); +} +#endif // ABSL_HAVE_THREAD_SANITIZER + +#if !defined(NDEBUG) && !defined(ABSL_USES_STD_STRING_VIEW) +TEST(NonNegativeLenTest, NonNegativeLen) { + ABSL_EXPECT_DEATH_IF_SUPPORTED(absl::string_view("xyz", -1), + "len <= kMaxSize"); +} + +TEST(LenExceedsMaxSizeTest, LenExceedsMaxSize) { + auto max_size = absl::string_view().max_size(); + + // This should construct ok (although the view itself is obviously invalid). + absl::string_view ok_view("", max_size); + + // Adding one to the max should trigger an assertion. + ABSL_EXPECT_DEATH_IF_SUPPORTED(absl::string_view("", max_size + 1), + "len <= kMaxSize"); +} +#endif // !defined(NDEBUG) && !defined(ABSL_USES_STD_STRING_VIEW) + +class StringViewStreamTest : public ::testing::Test { + public: + // Set negative 'width' for right justification. + template <typename T> + std::string Pad(const T& s, int width, char fill = 0) { + std::ostringstream oss; + if (fill != 0) { + oss << std::setfill(fill); + } + if (width < 0) { + width = -width; + oss << std::right; + } + oss << std::setw(width) << s; + return oss.str(); + } +}; + +TEST_F(StringViewStreamTest, Padding) { + std::string s("hello"); + absl::string_view sp(s); + for (int w = -64; w < 64; ++w) { + SCOPED_TRACE(w); + EXPECT_EQ(Pad(s, w), Pad(sp, w)); + } + for (int w = -64; w < 64; ++w) { + SCOPED_TRACE(w); + EXPECT_EQ(Pad(s, w, '#'), Pad(sp, w, '#')); + } +} + +TEST_F(StringViewStreamTest, ResetsWidth) { + // Width should reset after one formatted write. + // If we weren't resetting width after formatting the string_view, + // we'd have width=5 carrying over to the printing of the "]", + // creating "[###hi####]". + std::string s = "hi"; + absl::string_view sp = s; + { + std::ostringstream oss; + oss << "[" << std::setfill('#') << std::setw(5) << s << "]"; + ASSERT_EQ("[###hi]", oss.str()); + } + { + std::ostringstream oss; + oss << "[" << std::setfill('#') << std::setw(5) << sp << "]"; + EXPECT_EQ("[###hi]", oss.str()); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/strip.h b/third_party/abseil_cpp/absl/strings/strip.h new file mode 100644 index 000000000000..111872ca54ba --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/strip.h @@ -0,0 +1,91 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: strip.h +// ----------------------------------------------------------------------------- +// +// This file contains various functions for stripping substrings from a string. +#ifndef ABSL_STRINGS_STRIP_H_ +#define ABSL_STRINGS_STRIP_H_ + +#include <cstddef> +#include <string> + +#include "absl/base/macros.h" +#include "absl/strings/ascii.h" +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ConsumePrefix() +// +// Strips the `expected` prefix from the start of the given string, returning +// `true` if the strip operation succeeded or false otherwise. +// +// Example: +// +// absl::string_view input("abc"); +// EXPECT_TRUE(absl::ConsumePrefix(&input, "a")); +// EXPECT_EQ(input, "bc"); +inline bool ConsumePrefix(absl::string_view* str, absl::string_view expected) { + if (!absl::StartsWith(*str, expected)) return false; + str->remove_prefix(expected.size()); + return true; +} +// ConsumeSuffix() +// +// Strips the `expected` suffix from the end of the given string, returning +// `true` if the strip operation succeeded or false otherwise. +// +// Example: +// +// absl::string_view input("abcdef"); +// EXPECT_TRUE(absl::ConsumeSuffix(&input, "def")); +// EXPECT_EQ(input, "abc"); +inline bool ConsumeSuffix(absl::string_view* str, absl::string_view expected) { + if (!absl::EndsWith(*str, expected)) return false; + str->remove_suffix(expected.size()); + return true; +} + +// StripPrefix() +// +// Returns a view into the input string 'str' with the given 'prefix' removed, +// but leaving the original string intact. If the prefix does not match at the +// start of the string, returns the original string instead. +ABSL_MUST_USE_RESULT inline absl::string_view StripPrefix( + absl::string_view str, absl::string_view prefix) { + if (absl::StartsWith(str, prefix)) str.remove_prefix(prefix.size()); + return str; +} + +// StripSuffix() +// +// Returns a view into the input string 'str' with the given 'suffix' removed, +// but leaving the original string intact. If the suffix does not match at the +// end of the string, returns the original string instead. +ABSL_MUST_USE_RESULT inline absl::string_view StripSuffix( + absl::string_view str, absl::string_view suffix) { + if (absl::EndsWith(str, suffix)) str.remove_suffix(suffix.size()); + return str; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_STRIP_H_ diff --git a/third_party/abseil_cpp/absl/strings/strip_test.cc b/third_party/abseil_cpp/absl/strings/strip_test.cc new file mode 100644 index 000000000000..e4e00cb66ebc --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/strip_test.cc @@ -0,0 +1,198 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains functions that remove a defined part from the string, +// i.e., strip the string. + +#include "absl/strings/strip.h" + +#include <cassert> +#include <cstdio> +#include <cstring> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/string_view.h" + +namespace { + +TEST(Strip, ConsumePrefixOneChar) { + absl::string_view input("abc"); + EXPECT_TRUE(absl::ConsumePrefix(&input, "a")); + EXPECT_EQ(input, "bc"); + + EXPECT_FALSE(absl::ConsumePrefix(&input, "x")); + EXPECT_EQ(input, "bc"); + + EXPECT_TRUE(absl::ConsumePrefix(&input, "b")); + EXPECT_EQ(input, "c"); + + EXPECT_TRUE(absl::ConsumePrefix(&input, "c")); + EXPECT_EQ(input, ""); + + EXPECT_FALSE(absl::ConsumePrefix(&input, "a")); + EXPECT_EQ(input, ""); +} + +TEST(Strip, ConsumePrefix) { + absl::string_view input("abcdef"); + EXPECT_FALSE(absl::ConsumePrefix(&input, "abcdefg")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_FALSE(absl::ConsumePrefix(&input, "abce")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_TRUE(absl::ConsumePrefix(&input, "")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_FALSE(absl::ConsumePrefix(&input, "abcdeg")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_TRUE(absl::ConsumePrefix(&input, "abcdef")); + EXPECT_EQ(input, ""); + + input = "abcdef"; + EXPECT_TRUE(absl::ConsumePrefix(&input, "abcde")); + EXPECT_EQ(input, "f"); +} + +TEST(Strip, ConsumeSuffix) { + absl::string_view input("abcdef"); + EXPECT_FALSE(absl::ConsumeSuffix(&input, "abcdefg")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_TRUE(absl::ConsumeSuffix(&input, "")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_TRUE(absl::ConsumeSuffix(&input, "def")); + EXPECT_EQ(input, "abc"); + + input = "abcdef"; + EXPECT_FALSE(absl::ConsumeSuffix(&input, "abcdeg")); + EXPECT_EQ(input, "abcdef"); + + EXPECT_TRUE(absl::ConsumeSuffix(&input, "f")); + EXPECT_EQ(input, "abcde"); + + EXPECT_TRUE(absl::ConsumeSuffix(&input, "abcde")); + EXPECT_EQ(input, ""); +} + +TEST(Strip, StripPrefix) { + const absl::string_view null_str; + + EXPECT_EQ(absl::StripPrefix("foobar", "foo"), "bar"); + EXPECT_EQ(absl::StripPrefix("foobar", ""), "foobar"); + EXPECT_EQ(absl::StripPrefix("foobar", null_str), "foobar"); + EXPECT_EQ(absl::StripPrefix("foobar", "foobar"), ""); + EXPECT_EQ(absl::StripPrefix("foobar", "bar"), "foobar"); + EXPECT_EQ(absl::StripPrefix("foobar", "foobarr"), "foobar"); + EXPECT_EQ(absl::StripPrefix("", ""), ""); +} + +TEST(Strip, StripSuffix) { + const absl::string_view null_str; + + EXPECT_EQ(absl::StripSuffix("foobar", "bar"), "foo"); + EXPECT_EQ(absl::StripSuffix("foobar", ""), "foobar"); + EXPECT_EQ(absl::StripSuffix("foobar", null_str), "foobar"); + EXPECT_EQ(absl::StripSuffix("foobar", "foobar"), ""); + EXPECT_EQ(absl::StripSuffix("foobar", "foo"), "foobar"); + EXPECT_EQ(absl::StripSuffix("foobar", "ffoobar"), "foobar"); + EXPECT_EQ(absl::StripSuffix("", ""), ""); +} + +TEST(Strip, RemoveExtraAsciiWhitespace) { + const char* inputs[] = { + "No extra space", + " Leading whitespace", + "Trailing whitespace ", + " Leading and trailing ", + " Whitespace \t in\v middle ", + "'Eeeeep! \n Newlines!\n", + "nospaces", + }; + const char* outputs[] = { + "No extra space", + "Leading whitespace", + "Trailing whitespace", + "Leading and trailing", + "Whitespace in middle", + "'Eeeeep! Newlines!", + "nospaces", + }; + int NUM_TESTS = 7; + + for (int i = 0; i < NUM_TESTS; i++) { + std::string s(inputs[i]); + absl::RemoveExtraAsciiWhitespace(&s); + EXPECT_STREQ(outputs[i], s.c_str()); + } + + // Test that absl::RemoveExtraAsciiWhitespace returns immediately for empty + // strings (It was adding the \0 character to the C++ std::string, which broke + // tests involving empty()) + std::string zero_string = ""; + assert(zero_string.empty()); + absl::RemoveExtraAsciiWhitespace(&zero_string); + EXPECT_EQ(zero_string.size(), 0); + EXPECT_TRUE(zero_string.empty()); +} + +TEST(Strip, StripTrailingAsciiWhitespace) { + std::string test = "foo "; + absl::StripTrailingAsciiWhitespace(&test); + EXPECT_EQ(test, "foo"); + + test = " "; + absl::StripTrailingAsciiWhitespace(&test); + EXPECT_EQ(test, ""); + + test = ""; + absl::StripTrailingAsciiWhitespace(&test); + EXPECT_EQ(test, ""); + + test = " abc\t"; + absl::StripTrailingAsciiWhitespace(&test); + EXPECT_EQ(test, " abc"); +} + +TEST(String, StripLeadingAsciiWhitespace) { + absl::string_view orig = "\t \n\f\r\n\vfoo"; + EXPECT_EQ("foo", absl::StripLeadingAsciiWhitespace(orig)); + orig = "\t \n\f\r\v\n\t \n\f\r\v\n"; + EXPECT_EQ(absl::string_view(), absl::StripLeadingAsciiWhitespace(orig)); +} + +TEST(Strip, StripAsciiWhitespace) { + std::string test2 = "\t \f\r\n\vfoo \t\f\r\v\n"; + absl::StripAsciiWhitespace(&test2); + EXPECT_EQ(test2, "foo"); + std::string test3 = "bar"; + absl::StripAsciiWhitespace(&test3); + EXPECT_EQ(test3, "bar"); + std::string test4 = "\t \f\r\n\vfoo"; + absl::StripAsciiWhitespace(&test4); + EXPECT_EQ(test4, "foo"); + std::string test5 = "foo \t\f\r\v\n"; + absl::StripAsciiWhitespace(&test5); + EXPECT_EQ(test5, "foo"); + absl::string_view test6("\t \f\r\n\vfoo \t\f\r\v\n"); + test6 = absl::StripAsciiWhitespace(test6); + EXPECT_EQ(test6, "foo"); + test6 = absl::StripAsciiWhitespace(test6); + EXPECT_EQ(test6, "foo"); // already stripped +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/strings/substitute.cc b/third_party/abseil_cpp/absl/strings/substitute.cc new file mode 100644 index 000000000000..1f3c7409abb6 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/substitute.cc @@ -0,0 +1,171 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/substitute.h" + +#include <algorithm> + +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/ascii.h" +#include "absl/strings/escaping.h" +#include "absl/strings/internal/resize_uninitialized.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace substitute_internal { + +void SubstituteAndAppendArray(std::string* output, absl::string_view format, + const absl::string_view* args_array, + size_t num_args) { + // Determine total size needed. + size_t size = 0; + for (size_t i = 0; i < format.size(); i++) { + if (format[i] == '$') { + if (i + 1 >= format.size()) { +#ifndef NDEBUG + ABSL_RAW_LOG(FATAL, + "Invalid absl::Substitute() format string: \"%s\".", + absl::CEscape(format).c_str()); +#endif + return; + } else if (absl::ascii_isdigit(format[i + 1])) { + int index = format[i + 1] - '0'; + if (static_cast<size_t>(index) >= num_args) { +#ifndef NDEBUG + ABSL_RAW_LOG( + FATAL, + "Invalid absl::Substitute() format string: asked for \"$" + "%d\", but only %d args were given. Full format string was: " + "\"%s\".", + index, static_cast<int>(num_args), absl::CEscape(format).c_str()); +#endif + return; + } + size += args_array[index].size(); + ++i; // Skip next char. + } else if (format[i + 1] == '$') { + ++size; + ++i; // Skip next char. + } else { +#ifndef NDEBUG + ABSL_RAW_LOG(FATAL, + "Invalid absl::Substitute() format string: \"%s\".", + absl::CEscape(format).c_str()); +#endif + return; + } + } else { + ++size; + } + } + + if (size == 0) return; + + // Build the string. + size_t original_size = output->size(); + strings_internal::STLStringResizeUninitialized(output, original_size + size); + char* target = &(*output)[original_size]; + for (size_t i = 0; i < format.size(); i++) { + if (format[i] == '$') { + if (absl::ascii_isdigit(format[i + 1])) { + const absl::string_view src = args_array[format[i + 1] - '0']; + target = std::copy(src.begin(), src.end(), target); + ++i; // Skip next char. + } else if (format[i + 1] == '$') { + *target++ = '$'; + ++i; // Skip next char. + } + } else { + *target++ = format[i]; + } + } + + assert(target == output->data() + output->size()); +} + +Arg::Arg(const void* value) { + static_assert(sizeof(scratch_) >= sizeof(value) * 2 + 2, + "fix sizeof(scratch_)"); + if (value == nullptr) { + piece_ = "NULL"; + } else { + char* ptr = scratch_ + sizeof(scratch_); + uintptr_t num = reinterpret_cast<uintptr_t>(value); + do { + *--ptr = absl::numbers_internal::kHexChar[num & 0xf]; + num >>= 4; + } while (num != 0); + *--ptr = 'x'; + *--ptr = '0'; + piece_ = absl::string_view(ptr, scratch_ + sizeof(scratch_) - ptr); + } +} + +// TODO(jorg): Don't duplicate so much code between here and str_cat.cc +Arg::Arg(Hex hex) { + char* const end = &scratch_[numbers_internal::kFastToBufferSize]; + char* writer = end; + uint64_t value = hex.value; + do { + *--writer = absl::numbers_internal::kHexChar[value & 0xF]; + value >>= 4; + } while (value != 0); + + char* beg; + if (end - writer < hex.width) { + beg = end - hex.width; + std::fill_n(beg, writer - beg, hex.fill); + } else { + beg = writer; + } + + piece_ = absl::string_view(beg, end - beg); +} + +// TODO(jorg): Don't duplicate so much code between here and str_cat.cc +Arg::Arg(Dec dec) { + assert(dec.width <= numbers_internal::kFastToBufferSize); + char* const end = &scratch_[numbers_internal::kFastToBufferSize]; + char* const minfill = end - dec.width; + char* writer = end; + uint64_t value = dec.value; + bool neg = dec.neg; + while (value > 9) { + *--writer = '0' + (value % 10); + value /= 10; + } + *--writer = '0' + value; + if (neg) *--writer = '-'; + + ptrdiff_t fillers = writer - minfill; + if (fillers > 0) { + // Tricky: if the fill character is ' ', then it's <fill><+/-><digits> + // But...: if the fill character is '0', then it's <+/-><fill><digits> + bool add_sign_again = false; + if (neg && dec.fill == '0') { // If filling with '0', + ++writer; // ignore the sign we just added + add_sign_again = true; // and re-add the sign later. + } + writer -= fillers; + std::fill_n(writer, fillers, dec.fill); + if (add_sign_again) *--writer = '-'; + } + + piece_ = absl::string_view(writer, end - writer); +} + +} // namespace substitute_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/strings/substitute.h b/third_party/abseil_cpp/absl/strings/substitute.h new file mode 100644 index 000000000000..c6da4dc6e722 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/substitute.h @@ -0,0 +1,696 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: substitute.h +// ----------------------------------------------------------------------------- +// +// This package contains functions for efficiently performing string +// substitutions using a format string with positional notation: +// `Substitute()` and `SubstituteAndAppend()`. +// +// Unlike printf-style format specifiers, `Substitute()` functions do not need +// to specify the type of the substitution arguments. Supported arguments +// following the format string, such as strings, string_views, ints, +// floats, and bools, are automatically converted to strings during the +// substitution process. (See below for a full list of supported types.) +// +// `Substitute()` does not allow you to specify *how* to format a value, beyond +// the default conversion to string. For example, you cannot format an integer +// in hex. +// +// The format string uses positional identifiers indicated by a dollar sign ($) +// and single digit positional ids to indicate which substitution arguments to +// use at that location within the format string. +// +// A '$$' sequence in the format string causes a literal '$' character to be +// output. +// +// Example 1: +// std::string s = Substitute("$1 purchased $0 $2 for $$10. Thanks $1!", +// 5, "Bob", "Apples"); +// EXPECT_EQ("Bob purchased 5 Apples for $10. Thanks Bob!", s); +// +// Example 2: +// std::string s = "Hi. "; +// SubstituteAndAppend(&s, "My name is $0 and I am $1 years old.", "Bob", 5); +// EXPECT_EQ("Hi. My name is Bob and I am 5 years old.", s); +// +// Supported types: +// * absl::string_view, std::string, const char* (null is equivalent to "") +// * int32_t, int64_t, uint32_t, uint64_t +// * float, double +// * bool (Printed as "true" or "false") +// * pointer types other than char* (Printed as "0x<lower case hex string>", +// except that null is printed as "NULL") +// +// If an invalid format string is provided, Substitute returns an empty string +// and SubstituteAndAppend does not change the provided output string. +// A format string is invalid if it: +// * ends in an unescaped $ character, +// e.g. "Hello $", or +// * calls for a position argument which is not provided, +// e.g. Substitute("Hello $2", "world"), or +// * specifies a non-digit, non-$ character after an unescaped $ character, +// e.g. "Hello $f". +// In debug mode, i.e. #ifndef NDEBUG, such errors terminate the program. + +#ifndef ABSL_STRINGS_SUBSTITUTE_H_ +#define ABSL_STRINGS_SUBSTITUTE_H_ + +#include <cstring> +#include <string> +#include <type_traits> +#include <vector> + +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/strings/ascii.h" +#include "absl/strings/escaping.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace substitute_internal { + +// Arg +// +// This class provides an argument type for `absl::Substitute()` and +// `absl::SubstituteAndAppend()`. `Arg` handles implicit conversion of various +// types to a string. (`Arg` is very similar to the `AlphaNum` class in +// `StrCat()`.) +// +// This class has implicit constructors. +class Arg { + public: + // Overloads for string-y things + // + // Explicitly overload `const char*` so the compiler doesn't cast to `bool`. + Arg(const char* value) // NOLINT(runtime/explicit) + : piece_(absl::NullSafeStringView(value)) {} + template <typename Allocator> + Arg( // NOLINT + const std::basic_string<char, std::char_traits<char>, Allocator>& + value) noexcept + : piece_(value) {} + Arg(absl::string_view value) // NOLINT(runtime/explicit) + : piece_(value) {} + + // Overloads for primitives + // + // No overloads are available for signed and unsigned char because if people + // are explicitly declaring their chars as signed or unsigned then they are + // probably using them as 8-bit integers and would probably prefer an integer + // representation. However, we can't really know, so we make the caller decide + // what to do. + Arg(char value) // NOLINT(runtime/explicit) + : piece_(scratch_, 1) { + scratch_[0] = value; + } + Arg(short value) // NOLINT(*) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(unsigned short value) // NOLINT(*) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(int value) // NOLINT(runtime/explicit) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(unsigned int value) // NOLINT(runtime/explicit) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(long value) // NOLINT(*) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(unsigned long value) // NOLINT(*) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(long long value) // NOLINT(*) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(unsigned long long value) // NOLINT(*) + : piece_(scratch_, + numbers_internal::FastIntToBuffer(value, scratch_) - scratch_) {} + Arg(float value) // NOLINT(runtime/explicit) + : piece_(scratch_, numbers_internal::SixDigitsToBuffer(value, scratch_)) { + } + Arg(double value) // NOLINT(runtime/explicit) + : piece_(scratch_, numbers_internal::SixDigitsToBuffer(value, scratch_)) { + } + Arg(bool value) // NOLINT(runtime/explicit) + : piece_(value ? "true" : "false") {} + + Arg(Hex hex); // NOLINT(runtime/explicit) + Arg(Dec dec); // NOLINT(runtime/explicit) + + // vector<bool>::reference and const_reference require special help to + // convert to `AlphaNum` because it requires two user defined conversions. + template <typename T, + absl::enable_if_t< + std::is_class<T>::value && + (std::is_same<T, std::vector<bool>::reference>::value || + std::is_same<T, std::vector<bool>::const_reference>::value)>* = + nullptr> + Arg(T value) // NOLINT(google-explicit-constructor) + : Arg(static_cast<bool>(value)) {} + + // `void*` values, with the exception of `char*`, are printed as + // "0x<hex value>". However, in the case of `nullptr`, "NULL" is printed. + Arg(const void* value); // NOLINT(runtime/explicit) + + Arg(const Arg&) = delete; + Arg& operator=(const Arg&) = delete; + + absl::string_view piece() const { return piece_; } + + private: + absl::string_view piece_; + char scratch_[numbers_internal::kFastToBufferSize]; +}; + +// Internal helper function. Don't call this from outside this implementation. +// This interface may change without notice. +void SubstituteAndAppendArray(std::string* output, absl::string_view format, + const absl::string_view* args_array, + size_t num_args); + +#if defined(ABSL_BAD_CALL_IF) +constexpr int CalculateOneBit(const char* format) { + // Returns: + // * 2^N for '$N' when N is in [0-9] + // * 0 for correct '$' escaping: '$$'. + // * -1 otherwise. + return (*format < '0' || *format > '9') ? (*format == '$' ? 0 : -1) + : (1 << (*format - '0')); +} + +constexpr const char* SkipNumber(const char* format) { + return !*format ? format : (format + 1); +} + +constexpr int PlaceholderBitmask(const char* format) { + return !*format + ? 0 + : *format != '$' ? PlaceholderBitmask(format + 1) + : (CalculateOneBit(format + 1) | + PlaceholderBitmask(SkipNumber(format + 1))); +} +#endif // ABSL_BAD_CALL_IF + +} // namespace substitute_internal + +// +// PUBLIC API +// + +// SubstituteAndAppend() +// +// Substitutes variables into a given format string and appends to a given +// output string. See file comments above for usage. +// +// The declarations of `SubstituteAndAppend()` below consist of overloads +// for passing 0 to 10 arguments, respectively. +// +// NOTE: A zero-argument `SubstituteAndAppend()` may be used within variadic +// templates to allow a variable number of arguments. +// +// Example: +// template <typename... Args> +// void VarMsg(std::string* boilerplate, absl::string_view format, +// const Args&... args) { +// absl::SubstituteAndAppend(boilerplate, format, args...); +// } +// +inline void SubstituteAndAppend(std::string* output, absl::string_view format) { + substitute_internal::SubstituteAndAppendArray(output, format, nullptr, 0); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0) { + const absl::string_view args[] = {a0.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1) { + const absl::string_view args[] = {a0.piece(), a1.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), + a3.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), + a3.piece(), a4.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), + a3.piece(), a4.piece(), a5.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend(std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), + a3.piece(), a4.piece(), a5.piece(), + a6.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend( + std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, const substitute_internal::Arg& a7) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), + a3.piece(), a4.piece(), a5.piece(), + a6.piece(), a7.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend( + std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, const substitute_internal::Arg& a7, + const substitute_internal::Arg& a8) { + const absl::string_view args[] = {a0.piece(), a1.piece(), a2.piece(), + a3.piece(), a4.piece(), a5.piece(), + a6.piece(), a7.piece(), a8.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +inline void SubstituteAndAppend( + std::string* output, absl::string_view format, + const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, const substitute_internal::Arg& a7, + const substitute_internal::Arg& a8, const substitute_internal::Arg& a9) { + const absl::string_view args[] = { + a0.piece(), a1.piece(), a2.piece(), a3.piece(), a4.piece(), + a5.piece(), a6.piece(), a7.piece(), a8.piece(), a9.piece()}; + substitute_internal::SubstituteAndAppendArray(output, format, args, + ABSL_ARRAYSIZE(args)); +} + +#if defined(ABSL_BAD_CALL_IF) +// This body of functions catches cases where the number of placeholders +// doesn't match the number of data arguments. +void SubstituteAndAppend(std::string* output, const char* format) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 0, + "There were no substitution arguments " + "but this format string has a $[0-9] in it"); + +void SubstituteAndAppend(std::string* output, const char* format, + const substitute_internal::Arg& a0) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1, + "There was 1 substitution argument given, but " + "this format string is either missing its $0, or " + "contains one of $1-$9"); + +void SubstituteAndAppend(std::string* output, const char* format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 3, + "There were 2 substitution arguments given, but " + "this format string is either missing its $0/$1, or " + "contains one of $2-$9"); + +void SubstituteAndAppend(std::string* output, const char* format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 7, + "There were 3 substitution arguments given, but " + "this format string is either missing its $0/$1/$2, or " + "contains one of $3-$9"); + +void SubstituteAndAppend(std::string* output, const char* format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 15, + "There were 4 substitution arguments given, but " + "this format string is either missing its $0-$3, or " + "contains one of $4-$9"); + +void SubstituteAndAppend(std::string* output, const char* format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 31, + "There were 5 substitution arguments given, but " + "this format string is either missing its $0-$4, or " + "contains one of $5-$9"); + +void SubstituteAndAppend(std::string* output, const char* format, + const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 63, + "There were 6 substitution arguments given, but " + "this format string is either missing its $0-$5, or " + "contains one of $6-$9"); + +void SubstituteAndAppend( + std::string* output, const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 127, + "There were 7 substitution arguments given, but " + "this format string is either missing its $0-$6, or " + "contains one of $7-$9"); + +void SubstituteAndAppend( + std::string* output, const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 255, + "There were 8 substitution arguments given, but " + "this format string is either missing its $0-$7, or " + "contains one of $8-$9"); + +void SubstituteAndAppend( + std::string* output, const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7, const substitute_internal::Arg& a8) + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 511, + "There were 9 substitution arguments given, but " + "this format string is either missing its $0-$8, or contains a $9"); + +void SubstituteAndAppend( + std::string* output, const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7, const substitute_internal::Arg& a8, + const substitute_internal::Arg& a9) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1023, + "There were 10 substitution arguments given, but this " + "format string doesn't contain all of $0 through $9"); +#endif // ABSL_BAD_CALL_IF + +// Substitute() +// +// Substitutes variables into a given format string. See file comments above +// for usage. +// +// The declarations of `Substitute()` below consist of overloads for passing 0 +// to 10 arguments, respectively. +// +// NOTE: A zero-argument `Substitute()` may be used within variadic templates to +// allow a variable number of arguments. +// +// Example: +// template <typename... Args> +// void VarMsg(absl::string_view format, const Args&... args) { +// std::string s = absl::Substitute(format, args...); + +ABSL_MUST_USE_RESULT inline std::string Substitute(absl::string_view format) { + std::string result; + SubstituteAndAppend(&result, format); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0) { + std::string result; + SubstituteAndAppend(&result, format, a0); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3, a4); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3, a4, a5); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3, a4, a5, a6); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3, a4, a5, a6, a7); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7, const substitute_internal::Arg& a8) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3, a4, a5, a6, a7, a8); + return result; +} + +ABSL_MUST_USE_RESULT inline std::string Substitute( + absl::string_view format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7, const substitute_internal::Arg& a8, + const substitute_internal::Arg& a9) { + std::string result; + SubstituteAndAppend(&result, format, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9); + return result; +} + +#if defined(ABSL_BAD_CALL_IF) +// This body of functions catches cases where the number of placeholders +// doesn't match the number of data arguments. +std::string Substitute(const char* format) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 0, + "There were no substitution arguments " + "but this format string has a $[0-9] in it"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1, + "There was 1 substitution argument given, but " + "this format string is either missing its $0, or " + "contains one of $1-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 3, + "There were 2 substitution arguments given, but " + "this format string is either missing its $0/$1, or " + "contains one of $2-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 7, + "There were 3 substitution arguments given, but " + "this format string is either missing its $0/$1/$2, or " + "contains one of $3-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 15, + "There were 4 substitution arguments given, but " + "this format string is either missing its $0-$3, or " + "contains one of $4-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 31, + "There were 5 substitution arguments given, but " + "this format string is either missing its $0-$4, or " + "contains one of $5-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 63, + "There were 6 substitution arguments given, but " + "this format string is either missing its $0-$5, or " + "contains one of $6-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 127, + "There were 7 substitution arguments given, but " + "this format string is either missing its $0-$6, or " + "contains one of $7-$9"); + +std::string Substitute(const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, + const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, + const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, + const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 255, + "There were 8 substitution arguments given, but " + "this format string is either missing its $0-$7, or " + "contains one of $8-$9"); + +std::string Substitute( + const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7, const substitute_internal::Arg& a8) + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 511, + "There were 9 substitution arguments given, but " + "this format string is either missing its $0-$8, or contains a $9"); + +std::string Substitute( + const char* format, const substitute_internal::Arg& a0, + const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, + const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, + const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, + const substitute_internal::Arg& a7, const substitute_internal::Arg& a8, + const substitute_internal::Arg& a9) + ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1023, + "There were 10 substitution arguments given, but this " + "format string doesn't contain all of $0 through $9"); +#endif // ABSL_BAD_CALL_IF + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_SUBSTITUTE_H_ diff --git a/third_party/abseil_cpp/absl/strings/substitute_test.cc b/third_party/abseil_cpp/absl/strings/substitute_test.cc new file mode 100644 index 000000000000..442c921528d8 --- /dev/null +++ b/third_party/abseil_cpp/absl/strings/substitute_test.cc @@ -0,0 +1,204 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/substitute.h" + +#include <cstdint> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/strings/str_cat.h" + +namespace { + +TEST(SubstituteTest, Substitute) { + // Basic. + EXPECT_EQ("Hello, world!", absl::Substitute("$0, $1!", "Hello", "world")); + + // Non-char* types. + EXPECT_EQ("123 0.2 0.1 foo true false x", + absl::Substitute("$0 $1 $2 $3 $4 $5 $6", 123, 0.2, 0.1f, + std::string("foo"), true, false, 'x')); + + // All int types. + EXPECT_EQ( + "-32767 65535 " + "-1234567890 3234567890 " + "-1234567890 3234567890 " + "-1234567890123456789 9234567890123456789", + absl::Substitute( + "$0 $1 $2 $3 $4 $5 $6 $7", + static_cast<short>(-32767), // NOLINT(runtime/int) + static_cast<unsigned short>(65535), // NOLINT(runtime/int) + -1234567890, 3234567890U, -1234567890L, 3234567890UL, + -int64_t{1234567890123456789}, uint64_t{9234567890123456789u})); + + // Hex format + EXPECT_EQ("0 1 f ffff0ffff 0123456789abcdef", + absl::Substitute("$0$1$2$3$4 $5", // + absl::Hex(0), absl::Hex(1, absl::kSpacePad2), + absl::Hex(0xf, absl::kSpacePad2), + absl::Hex(int16_t{-1}, absl::kSpacePad5), + absl::Hex(int16_t{-1}, absl::kZeroPad5), + absl::Hex(0x123456789abcdef, absl::kZeroPad16))); + + // Dec format + EXPECT_EQ("0 115 -1-0001 81985529216486895", + absl::Substitute("$0$1$2$3$4 $5", // + absl::Dec(0), absl::Dec(1, absl::kSpacePad2), + absl::Dec(0xf, absl::kSpacePad2), + absl::Dec(int16_t{-1}, absl::kSpacePad5), + absl::Dec(int16_t{-1}, absl::kZeroPad5), + absl::Dec(0x123456789abcdef, absl::kZeroPad16))); + + // Pointer. + const int* int_p = reinterpret_cast<const int*>(0x12345); + std::string str = absl::Substitute("$0", int_p); + EXPECT_EQ(absl::StrCat("0x", absl::Hex(int_p)), str); + + // Volatile Pointer. + // Like C++ streamed I/O, such pointers implicitly become bool + volatile int vol = 237; + volatile int *volatile volptr = &vol; + str = absl::Substitute("$0", volptr); + EXPECT_EQ("true", str); + + // null is special. StrCat prints 0x0. Substitute prints NULL. + const uint64_t* null_p = nullptr; + str = absl::Substitute("$0", null_p); + EXPECT_EQ("NULL", str); + + // char* is also special. + const char* char_p = "print me"; + str = absl::Substitute("$0", char_p); + EXPECT_EQ("print me", str); + + char char_buf[16]; + strncpy(char_buf, "print me too", sizeof(char_buf)); + str = absl::Substitute("$0", char_buf); + EXPECT_EQ("print me too", str); + + // null char* is "doubly" special. Represented as the empty string. + char_p = nullptr; + str = absl::Substitute("$0", char_p); + EXPECT_EQ("", str); + + // Out-of-order. + EXPECT_EQ("b, a, c, b", absl::Substitute("$1, $0, $2, $1", "a", "b", "c")); + + // Literal $ + EXPECT_EQ("$", absl::Substitute("$$")); + + EXPECT_EQ("$1", absl::Substitute("$$1")); + + // Test all overloads. + EXPECT_EQ("a", absl::Substitute("$0", "a")); + EXPECT_EQ("a b", absl::Substitute("$0 $1", "a", "b")); + EXPECT_EQ("a b c", absl::Substitute("$0 $1 $2", "a", "b", "c")); + EXPECT_EQ("a b c d", absl::Substitute("$0 $1 $2 $3", "a", "b", "c", "d")); + EXPECT_EQ("a b c d e", + absl::Substitute("$0 $1 $2 $3 $4", "a", "b", "c", "d", "e")); + EXPECT_EQ("a b c d e f", absl::Substitute("$0 $1 $2 $3 $4 $5", "a", "b", "c", + "d", "e", "f")); + EXPECT_EQ("a b c d e f g", absl::Substitute("$0 $1 $2 $3 $4 $5 $6", "a", "b", + "c", "d", "e", "f", "g")); + EXPECT_EQ("a b c d e f g h", + absl::Substitute("$0 $1 $2 $3 $4 $5 $6 $7", "a", "b", "c", "d", "e", + "f", "g", "h")); + EXPECT_EQ("a b c d e f g h i", + absl::Substitute("$0 $1 $2 $3 $4 $5 $6 $7 $8", "a", "b", "c", "d", + "e", "f", "g", "h", "i")); + EXPECT_EQ("a b c d e f g h i j", + absl::Substitute("$0 $1 $2 $3 $4 $5 $6 $7 $8 $9", "a", "b", "c", + "d", "e", "f", "g", "h", "i", "j")); + EXPECT_EQ("a b c d e f g h i j b0", + absl::Substitute("$0 $1 $2 $3 $4 $5 $6 $7 $8 $9 $10", "a", "b", "c", + "d", "e", "f", "g", "h", "i", "j")); + + const char* null_cstring = nullptr; + EXPECT_EQ("Text: ''", absl::Substitute("Text: '$0'", null_cstring)); +} + +TEST(SubstituteTest, SubstituteAndAppend) { + std::string str = "Hello"; + absl::SubstituteAndAppend(&str, ", $0!", "world"); + EXPECT_EQ("Hello, world!", str); + + // Test all overloads. + str.clear(); + absl::SubstituteAndAppend(&str, "$0", "a"); + EXPECT_EQ("a", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1", "a", "b"); + EXPECT_EQ("a b", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2", "a", "b", "c"); + EXPECT_EQ("a b c", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3", "a", "b", "c", "d"); + EXPECT_EQ("a b c d", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3 $4", "a", "b", "c", "d", "e"); + EXPECT_EQ("a b c d e", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3 $4 $5", "a", "b", "c", "d", "e", + "f"); + EXPECT_EQ("a b c d e f", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3 $4 $5 $6", "a", "b", "c", "d", + "e", "f", "g"); + EXPECT_EQ("a b c d e f g", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3 $4 $5 $6 $7", "a", "b", "c", "d", + "e", "f", "g", "h"); + EXPECT_EQ("a b c d e f g h", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3 $4 $5 $6 $7 $8", "a", "b", "c", + "d", "e", "f", "g", "h", "i"); + EXPECT_EQ("a b c d e f g h i", str); + str.clear(); + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3 $4 $5 $6 $7 $8 $9", "a", "b", + "c", "d", "e", "f", "g", "h", "i", "j"); + EXPECT_EQ("a b c d e f g h i j", str); +} + +TEST(SubstituteTest, VectorBoolRef) { + std::vector<bool> v = {true, false}; + const auto& cv = v; + EXPECT_EQ("true false true false", + absl::Substitute("$0 $1 $2 $3", v[0], v[1], cv[0], cv[1])); + + std::string str = "Logic be like: "; + absl::SubstituteAndAppend(&str, "$0 $1 $2 $3", v[0], v[1], cv[0], cv[1]); + EXPECT_EQ("Logic be like: true false true false", str); +} + +#ifdef GTEST_HAS_DEATH_TEST + +TEST(SubstituteDeathTest, SubstituteDeath) { + EXPECT_DEBUG_DEATH( + static_cast<void>(absl::Substitute(absl::string_view("-$2"), "a", "b")), + "Invalid absl::Substitute\\(\\) format string: asked for \"\\$2\", " + "but only 2 args were given."); + EXPECT_DEBUG_DEATH( + static_cast<void>(absl::Substitute(absl::string_view("-$z-"))), + "Invalid absl::Substitute\\(\\) format string: \"-\\$z-\""); + EXPECT_DEBUG_DEATH( + static_cast<void>(absl::Substitute(absl::string_view("-$"))), + "Invalid absl::Substitute\\(\\) format string: \"-\\$\""); +} + +#endif // GTEST_HAS_DEATH_TEST + +} // namespace diff --git a/third_party/abseil_cpp/absl/synchronization/BUILD.bazel b/third_party/abseil_cpp/absl/synchronization/BUILD.bazel new file mode 100644 index 000000000000..cd4009a15739 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/BUILD.bazel @@ -0,0 +1,288 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +# Internal data structure for efficiently detecting mutex dependency cycles +cc_library( + name = "graphcycles_internal", + srcs = [ + "internal/graphcycles.cc", + ], + hdrs = [ + "internal/graphcycles.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + "//absl/base", + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:malloc_internal", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "kernel_timeout_internal", + hdrs = ["internal/kernel_timeout.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/synchronization:__pkg__", + ], + deps = [ + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/time", + ], +) + +cc_library( + name = "synchronization", + srcs = [ + "barrier.cc", + "blocking_counter.cc", + "internal/create_thread_identity.cc", + "internal/per_thread_sem.cc", + "internal/waiter.cc", + "mutex.cc", + "notification.cc", + ], + hdrs = [ + "barrier.h", + "blocking_counter.h", + "internal/create_thread_identity.h", + "internal/futex.h", + "internal/per_thread_sem.h", + "internal/waiter.h", + "mutex.h", + "notification.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = select({ + "//absl:windows": [], + "//absl:wasm": [], + "//conditions:default": ["-pthread"], + }) + ABSL_DEFAULT_LINKOPTS, + deps = [ + ":graphcycles_internal", + ":kernel_timeout_internal", + "//absl/base", + "//absl/base:atomic_hook", + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:dynamic_annotations", + "//absl/base:malloc_internal", + "//absl/base:raw_logging_internal", + "//absl/debugging:stacktrace", + "//absl/debugging:symbolize", + "//absl/time", + ], +) + +cc_test( + name = "barrier_test", + size = "small", + srcs = ["barrier_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":synchronization", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "blocking_counter_test", + size = "small", + srcs = ["blocking_counter_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":synchronization", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "graphcycles_test", + size = "medium", + srcs = ["internal/graphcycles_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":graphcycles_internal", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "graphcycles_benchmark", + srcs = ["internal/graphcycles_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "benchmark", + ], + deps = [ + ":graphcycles_internal", + "//absl/base:raw_logging_internal", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_library( + name = "thread_pool", + testonly = 1, + hdrs = ["internal/thread_pool.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":synchronization", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "mutex_test", + size = "large", + srcs = ["mutex_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + shard_count = 25, + deps = [ + ":synchronization", + ":thread_pool", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/memory", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "mutex_benchmark_common", + testonly = 1, + srcs = ["mutex_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/synchronization:__pkg__", + ], + deps = [ + ":synchronization", + ":thread_pool", + "//absl/base", + "//absl/base:config", + "@com_github_google_benchmark//:benchmark_main", + ], + alwayslink = 1, +) + +cc_binary( + name = "mutex_benchmark", + testonly = 1, + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":mutex_benchmark_common", + ], +) + +cc_test( + name = "notification_test", + size = "small", + srcs = ["notification_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":synchronization", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "per_thread_sem_test_common", + testonly = 1, + srcs = ["internal/per_thread_sem_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":synchronization", + "//absl/base", + "//absl/base:config", + "//absl/strings", + "//absl/time", + "@com_google_googletest//:gtest", + ], + alwayslink = 1, +) + +cc_test( + name = "per_thread_sem_test", + size = "medium", + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":per_thread_sem_test_common", + ":synchronization", + "//absl/strings", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "lifetime_test", + srcs = [ + "lifetime_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["no_test_ios_x86_64"], + deps = [ + ":synchronization", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + ], +) diff --git a/third_party/abseil_cpp/absl/synchronization/CMakeLists.txt b/third_party/abseil_cpp/absl/synchronization/CMakeLists.txt new file mode 100644 index 000000000000..e633d0bf5312 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/CMakeLists.txt @@ -0,0 +1,216 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + graphcycles_internal + HDRS + "internal/graphcycles.h" + SRCS + "internal/graphcycles.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::malloc_internal + absl::raw_logging_internal +) + +absl_cc_library( + NAME + kernel_timeout_internal + HDRS + "internal/kernel_timeout.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::raw_logging_internal + absl::time +) + +absl_cc_library( + NAME + synchronization + HDRS + "barrier.h" + "blocking_counter.h" + "internal/create_thread_identity.h" + "internal/futex.h" + "internal/per_thread_sem.h" + "internal/waiter.h" + "mutex.h" + "notification.h" + SRCS + "barrier.cc" + "blocking_counter.cc" + "internal/create_thread_identity.cc" + "internal/per_thread_sem.cc" + "internal/waiter.cc" + "notification.cc" + "mutex.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::graphcycles_internal + absl::kernel_timeout_internal + absl::atomic_hook + absl::base + absl::base_internal + absl::config + absl::core_headers + absl::dynamic_annotations + absl::malloc_internal + absl::raw_logging_internal + absl::stacktrace + absl::symbolize + absl::time + Threads::Threads + PUBLIC +) + +absl_cc_test( + NAME + barrier_test + SRCS + "barrier_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::synchronization + absl::time + gmock_main +) + +absl_cc_test( + NAME + blocking_counter_test + SRCS + "blocking_counter_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::synchronization + absl::time + gmock_main +) + +absl_cc_test( + NAME + graphcycles_test + SRCS + "internal/graphcycles_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::graphcycles_internal + absl::core_headers + absl::raw_logging_internal + gmock_main +) + +absl_cc_library( + NAME + thread_pool + HDRS + "internal/thread_pool.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::synchronization + absl::core_headers + TESTONLY +) + +absl_cc_test( + NAME + mutex_test + SRCS + "mutex_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::synchronization + absl::thread_pool + absl::base + absl::config + absl::core_headers + absl::memory + absl::raw_logging_internal + absl::time + gmock_main +) + +absl_cc_test( + NAME + notification_test + SRCS + "notification_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::synchronization + absl::time + gmock_main +) + +absl_cc_library( + NAME + per_thread_sem_test_common + SRCS + "internal/per_thread_sem_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::synchronization + absl::base + absl::config + absl::strings + absl::time + gmock + TESTONLY +) + +absl_cc_test( + NAME + per_thread_sem_test + SRCS + "internal/per_thread_sem_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::per_thread_sem_test_common + absl::synchronization + absl::strings + absl::time + gmock_main +) + +absl_cc_test( + NAME + lifetime_test + SRCS + "lifetime_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::synchronization + absl::core_headers + absl::raw_logging_internal +) diff --git a/third_party/abseil_cpp/absl/synchronization/barrier.cc b/third_party/abseil_cpp/absl/synchronization/barrier.cc new file mode 100644 index 000000000000..0dfd795e7b5f --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/barrier.cc @@ -0,0 +1,52 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/barrier.h" + +#include "absl/base/internal/raw_logging.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Return whether int *arg is zero. +static bool IsZero(void *arg) { + return 0 == *reinterpret_cast<int *>(arg); +} + +bool Barrier::Block() { + MutexLock l(&this->lock_); + + this->num_to_block_--; + if (this->num_to_block_ < 0) { + ABSL_RAW_LOG( + FATAL, + "Block() called too many times. num_to_block_=%d out of total=%d", + this->num_to_block_, this->num_to_exit_); + } + + this->lock_.Await(Condition(IsZero, &this->num_to_block_)); + + // Determine which thread can safely delete this Barrier object + this->num_to_exit_--; + ABSL_RAW_CHECK(this->num_to_exit_ >= 0, "barrier underflow"); + + // If num_to_exit_ == 0 then all other threads in the barrier have + // exited the Wait() and have released the Mutex so this thread is + // free to delete the barrier. + return this->num_to_exit_ == 0; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/barrier.h b/third_party/abseil_cpp/absl/synchronization/barrier.h new file mode 100644 index 000000000000..d8e754406f4b --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/barrier.h @@ -0,0 +1,79 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// barrier.h +// ----------------------------------------------------------------------------- + +#ifndef ABSL_SYNCHRONIZATION_BARRIER_H_ +#define ABSL_SYNCHRONIZATION_BARRIER_H_ + +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Barrier +// +// This class creates a barrier which blocks threads until a prespecified +// threshold of threads (`num_threads`) utilizes the barrier. A thread utilizes +// the `Barrier` by calling `Block()` on the barrier, which will block that +// thread; no call to `Block()` will return until `num_threads` threads have +// called it. +// +// Exactly one call to `Block()` will return `true`, which is then responsible +// for destroying the barrier; because stack allocation will cause the barrier +// to be deleted when it is out of scope, barriers should not be stack +// allocated. +// +// Example: +// +// // Main thread creates a `Barrier`: +// barrier = new Barrier(num_threads); +// +// // Each participating thread could then call: +// if (barrier->Block()) delete barrier; // Exactly one call to `Block()` +// // returns `true`; that call +// // deletes the barrier. +class Barrier { + public: + // `num_threads` is the number of threads that will participate in the barrier + explicit Barrier(int num_threads) + : num_to_block_(num_threads), num_to_exit_(num_threads) {} + + Barrier(const Barrier&) = delete; + Barrier& operator=(const Barrier&) = delete; + + // Barrier::Block() + // + // Blocks the current thread, and returns only when the `num_threads` + // threshold of threads utilizing this barrier has been reached. `Block()` + // returns `true` for precisely one caller, which may then destroy the + // barrier. + // + // Memory ordering: For any threads X and Y, any action taken by X + // before X calls `Block()` will be visible to Y after Y returns from + // `Block()`. + bool Block(); + + private: + Mutex lock_; + int num_to_block_ ABSL_GUARDED_BY(lock_); + int num_to_exit_ ABSL_GUARDED_BY(lock_); +}; + +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_SYNCHRONIZATION_BARRIER_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/barrier_test.cc b/third_party/abseil_cpp/absl/synchronization/barrier_test.cc new file mode 100644 index 000000000000..bfc6cb1883b1 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/barrier_test.cc @@ -0,0 +1,75 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/barrier.h" + +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "gtest/gtest.h" +#include "absl/synchronization/mutex.h" +#include "absl/time/clock.h" + + +TEST(Barrier, SanityTest) { + constexpr int kNumThreads = 10; + absl::Barrier* barrier = new absl::Barrier(kNumThreads); + + absl::Mutex mutex; + int counter = 0; // Guarded by mutex. + + auto thread_func = [&] { + if (barrier->Block()) { + // This thread is the last thread to reach the barrier so it is + // responsible for deleting it. + delete barrier; + } + + // Increment the counter. + absl::MutexLock lock(&mutex); + ++counter; + }; + + // Start (kNumThreads - 1) threads running thread_func. + std::vector<std::thread> threads; + for (int i = 0; i < kNumThreads - 1; ++i) { + threads.push_back(std::thread(thread_func)); + } + + // Give (kNumThreads - 1) threads a chance to reach the barrier. + // This test assumes at least one thread will have run after the + // sleep has elapsed. Sleeping in a test is usually bad form, but we + // need to make sure that we are testing the barrier instead of some + // other synchronization method. + absl::SleepFor(absl::Seconds(1)); + + // The counter should still be zero since no thread should have + // been able to pass the barrier yet. + { + absl::MutexLock lock(&mutex); + EXPECT_EQ(counter, 0); + } + + // Start 1 more thread. This should make all threads pass the barrier. + threads.push_back(std::thread(thread_func)); + + // All threads should now be able to proceed and finish. + for (auto& thread : threads) { + thread.join(); + } + + // All threads should now have incremented the counter. + absl::MutexLock lock(&mutex); + EXPECT_EQ(counter, kNumThreads); +} diff --git a/third_party/abseil_cpp/absl/synchronization/blocking_counter.cc b/third_party/abseil_cpp/absl/synchronization/blocking_counter.cc new file mode 100644 index 000000000000..3cea7aed2441 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/blocking_counter.cc @@ -0,0 +1,57 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/blocking_counter.h" + +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Return whether int *arg is zero. +static bool IsZero(void *arg) { + return 0 == *reinterpret_cast<int *>(arg); +} + +bool BlockingCounter::DecrementCount() { + MutexLock l(&lock_); + count_--; + if (count_ < 0) { + ABSL_RAW_LOG( + FATAL, + "BlockingCounter::DecrementCount() called too many times. count=%d", + count_); + } + return count_ == 0; +} + +void BlockingCounter::Wait() { + MutexLock l(&this->lock_); + ABSL_RAW_CHECK(count_ >= 0, "BlockingCounter underflow"); + + // only one thread may call Wait(). To support more than one thread, + // implement a counter num_to_exit, like in the Barrier class. + ABSL_RAW_CHECK(num_waiting_ == 0, "multiple threads called Wait()"); + num_waiting_++; + + this->lock_.Await(Condition(IsZero, &this->count_)); + + // At this point, We know that all threads executing DecrementCount have + // released the lock, and so will not touch this object again. + // Therefore, the thread calling this method is free to delete the object + // after we return from this method. +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/blocking_counter.h b/third_party/abseil_cpp/absl/synchronization/blocking_counter.h new file mode 100644 index 000000000000..1f53f9f24045 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/blocking_counter.h @@ -0,0 +1,99 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// blocking_counter.h +// ----------------------------------------------------------------------------- + +#ifndef ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_ +#define ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_ + +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// BlockingCounter +// +// This class allows a thread to block for a pre-specified number of actions. +// `BlockingCounter` maintains a single non-negative abstract integer "count" +// with an initial value `initial_count`. A thread can then call `Wait()` on +// this blocking counter to block until the specified number of events occur; +// worker threads then call 'DecrementCount()` on the counter upon completion of +// their work. Once the counter's internal "count" reaches zero, the blocked +// thread unblocks. +// +// A `BlockingCounter` requires the following: +// - its `initial_count` is non-negative. +// - the number of calls to `DecrementCount()` on it is at most +// `initial_count`. +// - `Wait()` is called at most once on it. +// +// Given the above requirements, a `BlockingCounter` provides the following +// guarantees: +// - Once its internal "count" reaches zero, no legal action on the object +// can further change the value of "count". +// - When `Wait()` returns, it is legal to destroy the `BlockingCounter`. +// - When `Wait()` returns, the number of calls to `DecrementCount()` on +// this blocking counter exactly equals `initial_count`. +// +// Example: +// BlockingCounter bcount(N); // there are N items of work +// ... Allow worker threads to start. +// ... On completing each work item, workers do: +// ... bcount.DecrementCount(); // an item of work has been completed +// +// bcount.Wait(); // wait for all work to be complete +// +class BlockingCounter { + public: + explicit BlockingCounter(int initial_count) + : count_(initial_count), num_waiting_(0) {} + + BlockingCounter(const BlockingCounter&) = delete; + BlockingCounter& operator=(const BlockingCounter&) = delete; + + // BlockingCounter::DecrementCount() + // + // Decrements the counter's "count" by one, and return "count == 0". This + // function requires that "count != 0" when it is called. + // + // Memory ordering: For any threads X and Y, any action taken by X + // before it calls `DecrementCount()` is visible to thread Y after + // Y's call to `DecrementCount()`, provided Y's call returns `true`. + bool DecrementCount(); + + // BlockingCounter::Wait() + // + // Blocks until the counter reaches zero. This function may be called at most + // once. On return, `DecrementCount()` will have been called "initial_count" + // times and the blocking counter may be destroyed. + // + // Memory ordering: For any threads X and Y, any action taken by X + // before X calls `DecrementCount()` is visible to Y after Y returns + // from `Wait()`. + void Wait(); + + private: + Mutex lock_; + int count_ ABSL_GUARDED_BY(lock_); + int num_waiting_ ABSL_GUARDED_BY(lock_); +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/blocking_counter_test.cc b/third_party/abseil_cpp/absl/synchronization/blocking_counter_test.cc new file mode 100644 index 000000000000..2926224af750 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/blocking_counter_test.cc @@ -0,0 +1,68 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/blocking_counter.h" + +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "gtest/gtest.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +void PauseAndDecreaseCounter(BlockingCounter* counter, int* done) { + absl::SleepFor(absl::Seconds(1)); + *done = 1; + counter->DecrementCount(); +} + +TEST(BlockingCounterTest, BasicFunctionality) { + // This test verifies that BlockingCounter functions correctly. Starts a + // number of threads that just sleep for a second and decrement a counter. + + // Initialize the counter. + const int num_workers = 10; + BlockingCounter counter(num_workers); + + std::vector<std::thread> workers; + std::vector<int> done(num_workers, 0); + + // Start a number of parallel tasks that will just wait for a seconds and + // then decrement the count. + workers.reserve(num_workers); + for (int k = 0; k < num_workers; k++) { + workers.emplace_back( + [&counter, &done, k] { PauseAndDecreaseCounter(&counter, &done[k]); }); + } + + // Wait for the threads to have all finished. + counter.Wait(); + + // Check that all the workers have completed. + for (int k = 0; k < num_workers; k++) { + EXPECT_EQ(1, done[k]); + } + + for (std::thread& w : workers) { + w.join(); + } +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/internal/create_thread_identity.cc b/third_party/abseil_cpp/absl/synchronization/internal/create_thread_identity.cc new file mode 100644 index 000000000000..53a71b342b9f --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/create_thread_identity.cc @@ -0,0 +1,140 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdint.h> +#include <new> + +// This file is a no-op if the required LowLevelAlloc support is missing. +#include "absl/base/internal/low_level_alloc.h" +#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING + +#include <string.h> + +#include "absl/base/attributes.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/synchronization/internal/per_thread_sem.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// ThreadIdentity storage is persistent, we maintain a free-list of previously +// released ThreadIdentity objects. +ABSL_CONST_INIT static base_internal::SpinLock freelist_lock( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); +ABSL_CONST_INIT static base_internal::ThreadIdentity* thread_identity_freelist; + +// A per-thread destructor for reclaiming associated ThreadIdentity objects. +// Since we must preserve their storage we cache them for re-use. +void ReclaimThreadIdentity(void* v) { + base_internal::ThreadIdentity* identity = + static_cast<base_internal::ThreadIdentity*>(v); + + // all_locks might have been allocated by the Mutex implementation. + // We free it here when we are notified that our thread is dying. + if (identity->per_thread_synch.all_locks != nullptr) { + base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks); + } + + PerThreadSem::Destroy(identity); + + // We must explicitly clear the current thread's identity: + // (a) Subsequent (unrelated) per-thread destructors may require an identity. + // We must guarantee a new identity is used in this case (this instructor + // will be reinvoked up to PTHREAD_DESTRUCTOR_ITERATIONS in this case). + // (b) ThreadIdentity implementations may depend on memory that is not + // reinitialized before reuse. We must allow explicit clearing of the + // association state in this case. + base_internal::ClearCurrentThreadIdentity(); + { + base_internal::SpinLockHolder l(&freelist_lock); + identity->next = thread_identity_freelist; + thread_identity_freelist = identity; + } +} + +// Return value rounded up to next multiple of align. +// Align must be a power of two. +static intptr_t RoundUp(intptr_t addr, intptr_t align) { + return (addr + align - 1) & ~(align - 1); +} + +static void ResetThreadIdentity(base_internal::ThreadIdentity* identity) { + base_internal::PerThreadSynch* pts = &identity->per_thread_synch; + pts->next = nullptr; + pts->skip = nullptr; + pts->may_skip = false; + pts->waitp = nullptr; + pts->suppress_fatal_errors = false; + pts->readers = 0; + pts->priority = 0; + pts->next_priority_read_cycles = 0; + pts->state.store(base_internal::PerThreadSynch::State::kAvailable, + std::memory_order_relaxed); + pts->maybe_unlocking = false; + pts->wake = false; + pts->cond_waiter = false; + pts->all_locks = nullptr; + identity->blocked_count_ptr = nullptr; + identity->ticker.store(0, std::memory_order_relaxed); + identity->wait_start.store(0, std::memory_order_relaxed); + identity->is_idle.store(false, std::memory_order_relaxed); + identity->next = nullptr; +} + +static base_internal::ThreadIdentity* NewThreadIdentity() { + base_internal::ThreadIdentity* identity = nullptr; + + { + // Re-use a previously released object if possible. + base_internal::SpinLockHolder l(&freelist_lock); + if (thread_identity_freelist) { + identity = thread_identity_freelist; // Take list-head. + thread_identity_freelist = thread_identity_freelist->next; + } + } + + if (identity == nullptr) { + // Allocate enough space to align ThreadIdentity to a multiple of + // PerThreadSynch::kAlignment. This space is never released (it is + // added to a freelist by ReclaimThreadIdentity instead). + void* allocation = base_internal::LowLevelAlloc::Alloc( + sizeof(*identity) + base_internal::PerThreadSynch::kAlignment - 1); + // Round up the address to the required alignment. + identity = reinterpret_cast<base_internal::ThreadIdentity*>( + RoundUp(reinterpret_cast<intptr_t>(allocation), + base_internal::PerThreadSynch::kAlignment)); + } + ResetThreadIdentity(identity); + + return identity; +} + +// Allocates and attaches ThreadIdentity object for the calling thread. Returns +// the new identity. +// REQUIRES: CurrentThreadIdentity(false) == nullptr +base_internal::ThreadIdentity* CreateThreadIdentity() { + base_internal::ThreadIdentity* identity = NewThreadIdentity(); + PerThreadSem::Init(identity); + // Associate the value with the current thread, and attach our destructor. + base_internal::SetCurrentThreadIdentity(identity, ReclaimThreadIdentity); + return identity; +} + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/third_party/abseil_cpp/absl/synchronization/internal/create_thread_identity.h b/third_party/abseil_cpp/absl/synchronization/internal/create_thread_identity.h new file mode 100644 index 000000000000..e121f68377b1 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/create_thread_identity.h @@ -0,0 +1,60 @@ +/* + * Copyright 2017 The Abseil Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Interface for getting the current ThreadIdentity, creating one if necessary. +// See thread_identity.h. +// +// This file is separate from thread_identity.h because creating a new +// ThreadIdentity requires slightly higher level libraries (per_thread_sem +// and low_level_alloc) than accessing an existing one. This separation allows +// us to have a smaller //absl/base:base. + +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_ + +#include "absl/base/internal/thread_identity.h" +#include "absl/base/port.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// Allocates and attaches a ThreadIdentity object for the calling thread. +// For private use only. +base_internal::ThreadIdentity* CreateThreadIdentity(); + +// A per-thread destructor for reclaiming associated ThreadIdentity objects. +// For private use only. +void ReclaimThreadIdentity(void* v); + +// Returns the ThreadIdentity object representing the calling thread; guaranteed +// to be unique for its lifetime. The returned object will remain valid for the +// program's lifetime; although it may be re-assigned to a subsequent thread. +// If one does not exist for the calling thread, allocate it now. +inline base_internal::ThreadIdentity* GetOrCreateCurrentThreadIdentity() { + base_internal::ThreadIdentity* identity = + base_internal::CurrentThreadIdentityIfPresent(); + if (ABSL_PREDICT_FALSE(identity == nullptr)) { + return CreateThreadIdentity(); + } + return identity; +} + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_SYNCHRONIZATION_INTERNAL_CREATE_THREAD_IDENTITY_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/internal/futex.h b/third_party/abseil_cpp/absl/synchronization/internal/futex.h new file mode 100644 index 000000000000..06fbd6d072d1 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/futex.h @@ -0,0 +1,154 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ + +#include "absl/base/config.h" + +#ifdef _WIN32 +#include <windows.h> +#else +#include <sys/time.h> +#include <unistd.h> +#endif + +#ifdef __linux__ +#include <linux/futex.h> +#include <sys/syscall.h> +#endif + +#include <errno.h> +#include <stdio.h> +#include <time.h> + +#include <atomic> +#include <cstdint> + +#include "absl/base/optimization.h" +#include "absl/synchronization/internal/kernel_timeout.h" + +#ifdef ABSL_INTERNAL_HAVE_FUTEX +#error ABSL_INTERNAL_HAVE_FUTEX may not be set on the command line +#elif defined(__BIONIC__) +// Bionic supports all the futex operations we need even when some of the futex +// definitions are missing. +#define ABSL_INTERNAL_HAVE_FUTEX +#elif defined(__linux__) && defined(FUTEX_CLOCK_REALTIME) +// FUTEX_CLOCK_REALTIME requires Linux >= 2.6.28. +#define ABSL_INTERNAL_HAVE_FUTEX +#endif + +#ifdef ABSL_INTERNAL_HAVE_FUTEX + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// Some Android headers are missing these definitions even though they +// support these futex operations. +#ifdef __BIONIC__ +#ifndef SYS_futex +#define SYS_futex __NR_futex +#endif +#ifndef FUTEX_WAIT_BITSET +#define FUTEX_WAIT_BITSET 9 +#endif +#ifndef FUTEX_PRIVATE_FLAG +#define FUTEX_PRIVATE_FLAG 128 +#endif +#ifndef FUTEX_CLOCK_REALTIME +#define FUTEX_CLOCK_REALTIME 256 +#endif +#ifndef FUTEX_BITSET_MATCH_ANY +#define FUTEX_BITSET_MATCH_ANY 0xFFFFFFFF +#endif +#endif + +#if defined(__NR_futex_time64) && !defined(SYS_futex_time64) +#define SYS_futex_time64 __NR_futex_time64 +#endif + +#if defined(SYS_futex_time64) && !defined(SYS_futex) +#define SYS_futex SYS_futex_time64 +#endif + +class FutexImpl { + public: + static int WaitUntil(std::atomic<int32_t> *v, int32_t val, + KernelTimeout t) { + int err = 0; + if (t.has_timeout()) { + // https://locklessinc.com/articles/futex_cheat_sheet/ + // Unlike FUTEX_WAIT, FUTEX_WAIT_BITSET uses absolute time. + struct timespec abs_timeout = t.MakeAbsTimespec(); + // Atomically check that the futex value is still 0, and if it + // is, sleep until abs_timeout or until woken by FUTEX_WAKE. + err = syscall( + SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME, val, + &abs_timeout, nullptr, FUTEX_BITSET_MATCH_ANY); + } else { + // Atomically check that the futex value is still 0, and if it + // is, sleep until woken by FUTEX_WAKE. + err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAIT | FUTEX_PRIVATE_FLAG, val, nullptr); + } + if (ABSL_PREDICT_FALSE(err != 0)) { + err = -errno; + } + return err; + } + + static int WaitBitsetAbsoluteTimeout(std::atomic<int32_t> *v, int32_t val, + int32_t bits, + const struct timespec *abstime) { + int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG, val, abstime, + nullptr, bits); + if (ABSL_PREDICT_FALSE(err != 0)) { + err = -errno; + } + return err; + } + + static int Wake(std::atomic<int32_t> *v, int32_t count) { + int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAKE | FUTEX_PRIVATE_FLAG, count); + if (ABSL_PREDICT_FALSE(err < 0)) { + err = -errno; + } + return err; + } + + // FUTEX_WAKE_BITSET + static int WakeBitset(std::atomic<int32_t> *v, int32_t count, int32_t bits) { + int err = syscall(SYS_futex, reinterpret_cast<int32_t *>(v), + FUTEX_WAKE_BITSET | FUTEX_PRIVATE_FLAG, count, nullptr, + nullptr, bits); + if (ABSL_PREDICT_FALSE(err < 0)) { + err = -errno; + } + return err; + } +}; + +class Futex : public FutexImpl {}; + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_HAVE_FUTEX + +#endif // ABSL_SYNCHRONIZATION_INTERNAL_FUTEX_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/internal/graphcycles.cc b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles.cc new file mode 100644 index 000000000000..27fec21681dc --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles.cc @@ -0,0 +1,698 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// GraphCycles provides incremental cycle detection on a dynamic +// graph using the following algorithm: +// +// A dynamic topological sort algorithm for directed acyclic graphs +// David J. Pearce, Paul H. J. Kelly +// Journal of Experimental Algorithmics (JEA) JEA Homepage archive +// Volume 11, 2006, Article No. 1.7 +// +// Brief summary of the algorithm: +// +// (1) Maintain a rank for each node that is consistent +// with the topological sort of the graph. I.e., path from x to y +// implies rank[x] < rank[y]. +// (2) When a new edge (x->y) is inserted, do nothing if rank[x] < rank[y]. +// (3) Otherwise: adjust ranks in the neighborhood of x and y. + +#include "absl/base/attributes.h" +// This file is a no-op if the required LowLevelAlloc support is missing. +#include "absl/base/internal/low_level_alloc.h" +#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING + +#include "absl/synchronization/internal/graphcycles.h" + +#include <algorithm> +#include <array> +#include <limits> +#include "absl/base/internal/hide_ptr.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" + +// Do not use STL. This module does not use standard memory allocation. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +namespace { + +// Avoid LowLevelAlloc's default arena since it calls malloc hooks in +// which people are doing things like acquiring Mutexes. +ABSL_CONST_INIT static absl::base_internal::SpinLock arena_mu( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); +ABSL_CONST_INIT static base_internal::LowLevelAlloc::Arena* arena; + +static void InitArenaIfNecessary() { + arena_mu.Lock(); + if (arena == nullptr) { + arena = base_internal::LowLevelAlloc::NewArena(0); + } + arena_mu.Unlock(); +} + +// Number of inlined elements in Vec. Hash table implementation +// relies on this being a power of two. +static const uint32_t kInline = 8; + +// A simple LowLevelAlloc based resizable vector with inlined storage +// for a few elements. T must be a plain type since constructor +// and destructor are not run on elements of type T managed by Vec. +template <typename T> +class Vec { + public: + Vec() { Init(); } + ~Vec() { Discard(); } + + void clear() { + Discard(); + Init(); + } + + bool empty() const { return size_ == 0; } + uint32_t size() const { return size_; } + T* begin() { return ptr_; } + T* end() { return ptr_ + size_; } + const T& operator[](uint32_t i) const { return ptr_[i]; } + T& operator[](uint32_t i) { return ptr_[i]; } + const T& back() const { return ptr_[size_-1]; } + void pop_back() { size_--; } + + void push_back(const T& v) { + if (size_ == capacity_) Grow(size_ + 1); + ptr_[size_] = v; + size_++; + } + + void resize(uint32_t n) { + if (n > capacity_) Grow(n); + size_ = n; + } + + void fill(const T& val) { + for (uint32_t i = 0; i < size(); i++) { + ptr_[i] = val; + } + } + + // Guarantees src is empty at end. + // Provided for the hash table resizing code below. + void MoveFrom(Vec<T>* src) { + if (src->ptr_ == src->space_) { + // Need to actually copy + resize(src->size_); + std::copy(src->ptr_, src->ptr_ + src->size_, ptr_); + src->size_ = 0; + } else { + Discard(); + ptr_ = src->ptr_; + size_ = src->size_; + capacity_ = src->capacity_; + src->Init(); + } + } + + private: + T* ptr_; + T space_[kInline]; + uint32_t size_; + uint32_t capacity_; + + void Init() { + ptr_ = space_; + size_ = 0; + capacity_ = kInline; + } + + void Discard() { + if (ptr_ != space_) base_internal::LowLevelAlloc::Free(ptr_); + } + + void Grow(uint32_t n) { + while (capacity_ < n) { + capacity_ *= 2; + } + size_t request = static_cast<size_t>(capacity_) * sizeof(T); + T* copy = static_cast<T*>( + base_internal::LowLevelAlloc::AllocWithArena(request, arena)); + std::copy(ptr_, ptr_ + size_, copy); + Discard(); + ptr_ = copy; + } + + Vec(const Vec&) = delete; + Vec& operator=(const Vec&) = delete; +}; + +// A hash set of non-negative int32_t that uses Vec for its underlying storage. +class NodeSet { + public: + NodeSet() { Init(); } + + void clear() { Init(); } + bool contains(int32_t v) const { return table_[FindIndex(v)] == v; } + + bool insert(int32_t v) { + uint32_t i = FindIndex(v); + if (table_[i] == v) { + return false; + } + if (table_[i] == kEmpty) { + // Only inserting over an empty cell increases the number of occupied + // slots. + occupied_++; + } + table_[i] = v; + // Double when 75% full. + if (occupied_ >= table_.size() - table_.size()/4) Grow(); + return true; + } + + void erase(uint32_t v) { + uint32_t i = FindIndex(v); + if (static_cast<uint32_t>(table_[i]) == v) { + table_[i] = kDel; + } + } + + // Iteration: is done via HASH_FOR_EACH + // Example: + // HASH_FOR_EACH(elem, node->out) { ... } +#define HASH_FOR_EACH(elem, eset) \ + for (int32_t elem, _cursor = 0; (eset).Next(&_cursor, &elem); ) + bool Next(int32_t* cursor, int32_t* elem) { + while (static_cast<uint32_t>(*cursor) < table_.size()) { + int32_t v = table_[*cursor]; + (*cursor)++; + if (v >= 0) { + *elem = v; + return true; + } + } + return false; + } + + private: + enum : int32_t { kEmpty = -1, kDel = -2 }; + Vec<int32_t> table_; + uint32_t occupied_; // Count of non-empty slots (includes deleted slots) + + static uint32_t Hash(uint32_t a) { return a * 41; } + + // Return index for storing v. May return an empty index or deleted index + int FindIndex(int32_t v) const { + // Search starting at hash index. + const uint32_t mask = table_.size() - 1; + uint32_t i = Hash(v) & mask; + int deleted_index = -1; // If >= 0, index of first deleted element we see + while (true) { + int32_t e = table_[i]; + if (v == e) { + return i; + } else if (e == kEmpty) { + // Return any previously encountered deleted slot. + return (deleted_index >= 0) ? deleted_index : i; + } else if (e == kDel && deleted_index < 0) { + // Keep searching since v might be present later. + deleted_index = i; + } + i = (i + 1) & mask; // Linear probing; quadratic is slightly slower. + } + } + + void Init() { + table_.clear(); + table_.resize(kInline); + table_.fill(kEmpty); + occupied_ = 0; + } + + void Grow() { + Vec<int32_t> copy; + copy.MoveFrom(&table_); + occupied_ = 0; + table_.resize(copy.size() * 2); + table_.fill(kEmpty); + + for (const auto& e : copy) { + if (e >= 0) insert(e); + } + } + + NodeSet(const NodeSet&) = delete; + NodeSet& operator=(const NodeSet&) = delete; +}; + +// We encode a node index and a node version in GraphId. The version +// number is incremented when the GraphId is freed which automatically +// invalidates all copies of the GraphId. + +inline GraphId MakeId(int32_t index, uint32_t version) { + GraphId g; + g.handle = + (static_cast<uint64_t>(version) << 32) | static_cast<uint32_t>(index); + return g; +} + +inline int32_t NodeIndex(GraphId id) { + return static_cast<uint32_t>(id.handle & 0xfffffffful); +} + +inline uint32_t NodeVersion(GraphId id) { + return static_cast<uint32_t>(id.handle >> 32); +} + +struct Node { + int32_t rank; // rank number assigned by Pearce-Kelly algorithm + uint32_t version; // Current version number + int32_t next_hash; // Next entry in hash table + bool visited; // Temporary marker used by depth-first-search + uintptr_t masked_ptr; // User-supplied pointer + NodeSet in; // List of immediate predecessor nodes in graph + NodeSet out; // List of immediate successor nodes in graph + int priority; // Priority of recorded stack trace. + int nstack; // Depth of recorded stack trace. + void* stack[40]; // stack[0,nstack-1] holds stack trace for node. +}; + +// Hash table for pointer to node index lookups. +class PointerMap { + public: + explicit PointerMap(const Vec<Node*>* nodes) : nodes_(nodes) { + table_.fill(-1); + } + + int32_t Find(void* ptr) { + auto masked = base_internal::HidePtr(ptr); + for (int32_t i = table_[Hash(ptr)]; i != -1;) { + Node* n = (*nodes_)[i]; + if (n->masked_ptr == masked) return i; + i = n->next_hash; + } + return -1; + } + + void Add(void* ptr, int32_t i) { + int32_t* head = &table_[Hash(ptr)]; + (*nodes_)[i]->next_hash = *head; + *head = i; + } + + int32_t Remove(void* ptr) { + // Advance through linked list while keeping track of the + // predecessor slot that points to the current entry. + auto masked = base_internal::HidePtr(ptr); + for (int32_t* slot = &table_[Hash(ptr)]; *slot != -1; ) { + int32_t index = *slot; + Node* n = (*nodes_)[index]; + if (n->masked_ptr == masked) { + *slot = n->next_hash; // Remove n from linked list + n->next_hash = -1; + return index; + } + slot = &n->next_hash; + } + return -1; + } + + private: + // Number of buckets in hash table for pointer lookups. + static constexpr uint32_t kHashTableSize = 8171; // should be prime + + const Vec<Node*>* nodes_; + std::array<int32_t, kHashTableSize> table_; + + static uint32_t Hash(void* ptr) { + return reinterpret_cast<uintptr_t>(ptr) % kHashTableSize; + } +}; + +} // namespace + +struct GraphCycles::Rep { + Vec<Node*> nodes_; + Vec<int32_t> free_nodes_; // Indices for unused entries in nodes_ + PointerMap ptrmap_; + + // Temporary state. + Vec<int32_t> deltaf_; // Results of forward DFS + Vec<int32_t> deltab_; // Results of backward DFS + Vec<int32_t> list_; // All nodes to reprocess + Vec<int32_t> merged_; // Rank values to assign to list_ entries + Vec<int32_t> stack_; // Emulates recursion stack for depth-first searches + + Rep() : ptrmap_(&nodes_) {} +}; + +static Node* FindNode(GraphCycles::Rep* rep, GraphId id) { + Node* n = rep->nodes_[NodeIndex(id)]; + return (n->version == NodeVersion(id)) ? n : nullptr; +} + +GraphCycles::GraphCycles() { + InitArenaIfNecessary(); + rep_ = new (base_internal::LowLevelAlloc::AllocWithArena(sizeof(Rep), arena)) + Rep; +} + +GraphCycles::~GraphCycles() { + for (auto* node : rep_->nodes_) { + node->Node::~Node(); + base_internal::LowLevelAlloc::Free(node); + } + rep_->Rep::~Rep(); + base_internal::LowLevelAlloc::Free(rep_); +} + +bool GraphCycles::CheckInvariants() const { + Rep* r = rep_; + NodeSet ranks; // Set of ranks seen so far. + for (uint32_t x = 0; x < r->nodes_.size(); x++) { + Node* nx = r->nodes_[x]; + void* ptr = base_internal::UnhidePtr<void>(nx->masked_ptr); + if (ptr != nullptr && static_cast<uint32_t>(r->ptrmap_.Find(ptr)) != x) { + ABSL_RAW_LOG(FATAL, "Did not find live node in hash table %u %p", x, ptr); + } + if (nx->visited) { + ABSL_RAW_LOG(FATAL, "Did not clear visited marker on node %u", x); + } + if (!ranks.insert(nx->rank)) { + ABSL_RAW_LOG(FATAL, "Duplicate occurrence of rank %d", nx->rank); + } + HASH_FOR_EACH(y, nx->out) { + Node* ny = r->nodes_[y]; + if (nx->rank >= ny->rank) { + ABSL_RAW_LOG(FATAL, "Edge %u->%d has bad rank assignment %d->%d", x, y, + nx->rank, ny->rank); + } + } + } + return true; +} + +GraphId GraphCycles::GetId(void* ptr) { + int32_t i = rep_->ptrmap_.Find(ptr); + if (i != -1) { + return MakeId(i, rep_->nodes_[i]->version); + } else if (rep_->free_nodes_.empty()) { + Node* n = + new (base_internal::LowLevelAlloc::AllocWithArena(sizeof(Node), arena)) + Node; + n->version = 1; // Avoid 0 since it is used by InvalidGraphId() + n->visited = false; + n->rank = rep_->nodes_.size(); + n->masked_ptr = base_internal::HidePtr(ptr); + n->nstack = 0; + n->priority = 0; + rep_->nodes_.push_back(n); + rep_->ptrmap_.Add(ptr, n->rank); + return MakeId(n->rank, n->version); + } else { + // Preserve preceding rank since the set of ranks in use must be + // a permutation of [0,rep_->nodes_.size()-1]. + int32_t r = rep_->free_nodes_.back(); + rep_->free_nodes_.pop_back(); + Node* n = rep_->nodes_[r]; + n->masked_ptr = base_internal::HidePtr(ptr); + n->nstack = 0; + n->priority = 0; + rep_->ptrmap_.Add(ptr, r); + return MakeId(r, n->version); + } +} + +void GraphCycles::RemoveNode(void* ptr) { + int32_t i = rep_->ptrmap_.Remove(ptr); + if (i == -1) { + return; + } + Node* x = rep_->nodes_[i]; + HASH_FOR_EACH(y, x->out) { + rep_->nodes_[y]->in.erase(i); + } + HASH_FOR_EACH(y, x->in) { + rep_->nodes_[y]->out.erase(i); + } + x->in.clear(); + x->out.clear(); + x->masked_ptr = base_internal::HidePtr<void>(nullptr); + if (x->version == std::numeric_limits<uint32_t>::max()) { + // Cannot use x any more + } else { + x->version++; // Invalidates all copies of node. + rep_->free_nodes_.push_back(i); + } +} + +void* GraphCycles::Ptr(GraphId id) { + Node* n = FindNode(rep_, id); + return n == nullptr ? nullptr + : base_internal::UnhidePtr<void>(n->masked_ptr); +} + +bool GraphCycles::HasNode(GraphId node) { + return FindNode(rep_, node) != nullptr; +} + +bool GraphCycles::HasEdge(GraphId x, GraphId y) const { + Node* xn = FindNode(rep_, x); + return xn && FindNode(rep_, y) && xn->out.contains(NodeIndex(y)); +} + +void GraphCycles::RemoveEdge(GraphId x, GraphId y) { + Node* xn = FindNode(rep_, x); + Node* yn = FindNode(rep_, y); + if (xn && yn) { + xn->out.erase(NodeIndex(y)); + yn->in.erase(NodeIndex(x)); + // No need to update the rank assignment since a previous valid + // rank assignment remains valid after an edge deletion. + } +} + +static bool ForwardDFS(GraphCycles::Rep* r, int32_t n, int32_t upper_bound); +static void BackwardDFS(GraphCycles::Rep* r, int32_t n, int32_t lower_bound); +static void Reorder(GraphCycles::Rep* r); +static void Sort(const Vec<Node*>&, Vec<int32_t>* delta); +static void MoveToList( + GraphCycles::Rep* r, Vec<int32_t>* src, Vec<int32_t>* dst); + +bool GraphCycles::InsertEdge(GraphId idx, GraphId idy) { + Rep* r = rep_; + const int32_t x = NodeIndex(idx); + const int32_t y = NodeIndex(idy); + Node* nx = FindNode(r, idx); + Node* ny = FindNode(r, idy); + if (nx == nullptr || ny == nullptr) return true; // Expired ids + + if (nx == ny) return false; // Self edge + if (!nx->out.insert(y)) { + // Edge already exists. + return true; + } + + ny->in.insert(x); + + if (nx->rank <= ny->rank) { + // New edge is consistent with existing rank assignment. + return true; + } + + // Current rank assignments are incompatible with the new edge. Recompute. + // We only need to consider nodes that fall in the range [ny->rank,nx->rank]. + if (!ForwardDFS(r, y, nx->rank)) { + // Found a cycle. Undo the insertion and tell caller. + nx->out.erase(y); + ny->in.erase(x); + // Since we do not call Reorder() on this path, clear any visited + // markers left by ForwardDFS. + for (const auto& d : r->deltaf_) { + r->nodes_[d]->visited = false; + } + return false; + } + BackwardDFS(r, x, ny->rank); + Reorder(r); + return true; +} + +static bool ForwardDFS(GraphCycles::Rep* r, int32_t n, int32_t upper_bound) { + // Avoid recursion since stack space might be limited. + // We instead keep a stack of nodes to visit. + r->deltaf_.clear(); + r->stack_.clear(); + r->stack_.push_back(n); + while (!r->stack_.empty()) { + n = r->stack_.back(); + r->stack_.pop_back(); + Node* nn = r->nodes_[n]; + if (nn->visited) continue; + + nn->visited = true; + r->deltaf_.push_back(n); + + HASH_FOR_EACH(w, nn->out) { + Node* nw = r->nodes_[w]; + if (nw->rank == upper_bound) { + return false; // Cycle + } + if (!nw->visited && nw->rank < upper_bound) { + r->stack_.push_back(w); + } + } + } + return true; +} + +static void BackwardDFS(GraphCycles::Rep* r, int32_t n, int32_t lower_bound) { + r->deltab_.clear(); + r->stack_.clear(); + r->stack_.push_back(n); + while (!r->stack_.empty()) { + n = r->stack_.back(); + r->stack_.pop_back(); + Node* nn = r->nodes_[n]; + if (nn->visited) continue; + + nn->visited = true; + r->deltab_.push_back(n); + + HASH_FOR_EACH(w, nn->in) { + Node* nw = r->nodes_[w]; + if (!nw->visited && lower_bound < nw->rank) { + r->stack_.push_back(w); + } + } + } +} + +static void Reorder(GraphCycles::Rep* r) { + Sort(r->nodes_, &r->deltab_); + Sort(r->nodes_, &r->deltaf_); + + // Adds contents of delta lists to list_ (backwards deltas first). + r->list_.clear(); + MoveToList(r, &r->deltab_, &r->list_); + MoveToList(r, &r->deltaf_, &r->list_); + + // Produce sorted list of all ranks that will be reassigned. + r->merged_.resize(r->deltab_.size() + r->deltaf_.size()); + std::merge(r->deltab_.begin(), r->deltab_.end(), + r->deltaf_.begin(), r->deltaf_.end(), + r->merged_.begin()); + + // Assign the ranks in order to the collected list. + for (uint32_t i = 0; i < r->list_.size(); i++) { + r->nodes_[r->list_[i]]->rank = r->merged_[i]; + } +} + +static void Sort(const Vec<Node*>& nodes, Vec<int32_t>* delta) { + struct ByRank { + const Vec<Node*>* nodes; + bool operator()(int32_t a, int32_t b) const { + return (*nodes)[a]->rank < (*nodes)[b]->rank; + } + }; + ByRank cmp; + cmp.nodes = &nodes; + std::sort(delta->begin(), delta->end(), cmp); +} + +static void MoveToList( + GraphCycles::Rep* r, Vec<int32_t>* src, Vec<int32_t>* dst) { + for (auto& v : *src) { + int32_t w = v; + v = r->nodes_[w]->rank; // Replace v entry with its rank + r->nodes_[w]->visited = false; // Prepare for future DFS calls + dst->push_back(w); + } +} + +int GraphCycles::FindPath(GraphId idx, GraphId idy, int max_path_len, + GraphId path[]) const { + Rep* r = rep_; + if (FindNode(r, idx) == nullptr || FindNode(r, idy) == nullptr) return 0; + const int32_t x = NodeIndex(idx); + const int32_t y = NodeIndex(idy); + + // Forward depth first search starting at x until we hit y. + // As we descend into a node, we push it onto the path. + // As we leave a node, we remove it from the path. + int path_len = 0; + + NodeSet seen; + r->stack_.clear(); + r->stack_.push_back(x); + while (!r->stack_.empty()) { + int32_t n = r->stack_.back(); + r->stack_.pop_back(); + if (n < 0) { + // Marker to indicate that we are leaving a node + path_len--; + continue; + } + + if (path_len < max_path_len) { + path[path_len] = MakeId(n, rep_->nodes_[n]->version); + } + path_len++; + r->stack_.push_back(-1); // Will remove tentative path entry + + if (n == y) { + return path_len; + } + + HASH_FOR_EACH(w, r->nodes_[n]->out) { + if (seen.insert(w)) { + r->stack_.push_back(w); + } + } + } + + return 0; +} + +bool GraphCycles::IsReachable(GraphId x, GraphId y) const { + return FindPath(x, y, 0, nullptr) > 0; +} + +void GraphCycles::UpdateStackTrace(GraphId id, int priority, + int (*get_stack_trace)(void** stack, int)) { + Node* n = FindNode(rep_, id); + if (n == nullptr || n->priority >= priority) { + return; + } + n->nstack = (*get_stack_trace)(n->stack, ABSL_ARRAYSIZE(n->stack)); + n->priority = priority; +} + +int GraphCycles::GetStackTrace(GraphId id, void*** ptr) { + Node* n = FindNode(rep_, id); + if (n == nullptr) { + *ptr = nullptr; + return 0; + } else { + *ptr = n->stack; + return n->nstack; + } +} + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/third_party/abseil_cpp/absl/synchronization/internal/graphcycles.h b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles.h new file mode 100644 index 000000000000..ceba33e4de89 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles.h @@ -0,0 +1,141 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_GRAPHCYCLES_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_GRAPHCYCLES_H_ + +// GraphCycles detects the introduction of a cycle into a directed +// graph that is being built up incrementally. +// +// Nodes are identified by small integers. It is not possible to +// record multiple edges with the same (source, destination) pair; +// requests to add an edge where one already exists are silently +// ignored. +// +// It is also not possible to introduce a cycle; an attempt to insert +// an edge that would introduce a cycle fails and returns false. +// +// GraphCycles uses no internal locking; calls into it should be +// serialized externally. + +// Performance considerations: +// Works well on sparse graphs, poorly on dense graphs. +// Extra information is maintained incrementally to detect cycles quickly. +// InsertEdge() is very fast when the edge already exists, and reasonably fast +// otherwise. +// FindPath() is linear in the size of the graph. +// The current implementation uses O(|V|+|E|) space. + +#include <cstdint> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// Opaque identifier for a graph node. +struct GraphId { + uint64_t handle; + + bool operator==(const GraphId& x) const { return handle == x.handle; } + bool operator!=(const GraphId& x) const { return handle != x.handle; } +}; + +// Return an invalid graph id that will never be assigned by GraphCycles. +inline GraphId InvalidGraphId() { + return GraphId{0}; +} + +class GraphCycles { + public: + GraphCycles(); + ~GraphCycles(); + + // Return the id to use for ptr, assigning one if necessary. + // Subsequent calls with the same ptr value will return the same id + // until Remove(). + GraphId GetId(void* ptr); + + // Remove "ptr" from the graph. Its corresponding node and all + // edges to and from it are removed. + void RemoveNode(void* ptr); + + // Return the pointer associated with id, or nullptr if id is not + // currently in the graph. + void* Ptr(GraphId id); + + // Attempt to insert an edge from source_node to dest_node. If the + // edge would introduce a cycle, return false without making any + // changes. Otherwise add the edge and return true. + bool InsertEdge(GraphId source_node, GraphId dest_node); + + // Remove any edge that exists from source_node to dest_node. + void RemoveEdge(GraphId source_node, GraphId dest_node); + + // Return whether node exists in the graph. + bool HasNode(GraphId node); + + // Return whether there is an edge directly from source_node to dest_node. + bool HasEdge(GraphId source_node, GraphId dest_node) const; + + // Return whether dest_node is reachable from source_node + // by following edges. + bool IsReachable(GraphId source_node, GraphId dest_node) const; + + // Find a path from "source" to "dest". If such a path exists, + // place the nodes on the path in the array path[], and return + // the number of nodes on the path. If the path is longer than + // max_path_len nodes, only the first max_path_len nodes are placed + // in path[]. The client should compare the return value with + // max_path_len" to see when this occurs. If no path exists, return + // 0. Any valid path stored in path[] will start with "source" and + // end with "dest". There is no guarantee that the path is the + // shortest, but no node will appear twice in the path, except the + // source and destination node if they are identical; therefore, the + // return value is at most one greater than the number of nodes in + // the graph. + int FindPath(GraphId source, GraphId dest, int max_path_len, + GraphId path[]) const; + + // Update the stack trace recorded for id with the current stack + // trace if the last time it was updated had a smaller priority + // than the priority passed on this call. + // + // *get_stack_trace is called to get the stack trace. + void UpdateStackTrace(GraphId id, int priority, + int (*get_stack_trace)(void**, int)); + + // Set *ptr to the beginning of the array that holds the recorded + // stack trace for id and return the depth of the stack trace. + int GetStackTrace(GraphId id, void*** ptr); + + // Check internal invariants. Crashes on failure, returns true on success. + // Expensive: should only be called from graphcycles_test.cc. + bool CheckInvariants() const; + + // ---------------------------------------------------- + struct Rep; + private: + Rep *rep_; // opaque representation + GraphCycles(const GraphCycles&) = delete; + GraphCycles& operator=(const GraphCycles&) = delete; +}; + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif diff --git a/third_party/abseil_cpp/absl/synchronization/internal/graphcycles_benchmark.cc b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles_benchmark.cc new file mode 100644 index 000000000000..54823e0ba51a --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles_benchmark.cc @@ -0,0 +1,44 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/internal/graphcycles.h" + +#include <algorithm> +#include <cstdint> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/internal/raw_logging.h" + +namespace { + +void BM_StressTest(benchmark::State& state) { + const int num_nodes = state.range(0); + while (state.KeepRunningBatch(num_nodes)) { + absl::synchronization_internal::GraphCycles g; + std::vector<absl::synchronization_internal::GraphId> nodes(num_nodes); + for (int i = 0; i < num_nodes; i++) { + nodes[i] = g.GetId(reinterpret_cast<void*>(static_cast<uintptr_t>(i))); + } + for (int i = 0; i < num_nodes; i++) { + int end = std::min(num_nodes, i + 5); + for (int j = i + 1; j < end; j++) { + ABSL_RAW_CHECK(g.InsertEdge(nodes[i], nodes[j]), ""); + } + } + } +} +BENCHMARK(BM_StressTest)->Range(2048, 1048576); + +} // namespace diff --git a/third_party/abseil_cpp/absl/synchronization/internal/graphcycles_test.cc b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles_test.cc new file mode 100644 index 000000000000..74eaffe7a806 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/graphcycles_test.cc @@ -0,0 +1,464 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/internal/graphcycles.h" + +#include <map> +#include <random> +#include <unordered_set> +#include <utility> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// We emulate a GraphCycles object with a node vector and an edge vector. +// We then compare the two implementations. + +using Nodes = std::vector<int>; +struct Edge { + int from; + int to; +}; +using Edges = std::vector<Edge>; +using RandomEngine = std::mt19937_64; + +// Mapping from integer index to GraphId. +typedef std::map<int, GraphId> IdMap; +static GraphId Get(const IdMap& id, int num) { + auto iter = id.find(num); + return (iter == id.end()) ? InvalidGraphId() : iter->second; +} + +// Return whether "to" is reachable from "from". +static bool IsReachable(Edges *edges, int from, int to, + std::unordered_set<int> *seen) { + seen->insert(from); // we are investigating "from"; don't do it again + if (from == to) return true; + for (const auto &edge : *edges) { + if (edge.from == from) { + if (edge.to == to) { // success via edge directly + return true; + } else if (seen->find(edge.to) == seen->end() && // success via edge + IsReachable(edges, edge.to, to, seen)) { + return true; + } + } + } + return false; +} + +static void PrintEdges(Edges *edges) { + ABSL_RAW_LOG(INFO, "EDGES (%zu)", edges->size()); + for (const auto &edge : *edges) { + int a = edge.from; + int b = edge.to; + ABSL_RAW_LOG(INFO, "%d %d", a, b); + } + ABSL_RAW_LOG(INFO, "---"); +} + +static void PrintGCEdges(Nodes *nodes, const IdMap &id, GraphCycles *gc) { + ABSL_RAW_LOG(INFO, "GC EDGES"); + for (int a : *nodes) { + for (int b : *nodes) { + if (gc->HasEdge(Get(id, a), Get(id, b))) { + ABSL_RAW_LOG(INFO, "%d %d", a, b); + } + } + } + ABSL_RAW_LOG(INFO, "---"); +} + +static void PrintTransitiveClosure(Nodes *nodes, Edges *edges) { + ABSL_RAW_LOG(INFO, "Transitive closure"); + for (int a : *nodes) { + for (int b : *nodes) { + std::unordered_set<int> seen; + if (IsReachable(edges, a, b, &seen)) { + ABSL_RAW_LOG(INFO, "%d %d", a, b); + } + } + } + ABSL_RAW_LOG(INFO, "---"); +} + +static void PrintGCTransitiveClosure(Nodes *nodes, const IdMap &id, + GraphCycles *gc) { + ABSL_RAW_LOG(INFO, "GC Transitive closure"); + for (int a : *nodes) { + for (int b : *nodes) { + if (gc->IsReachable(Get(id, a), Get(id, b))) { + ABSL_RAW_LOG(INFO, "%d %d", a, b); + } + } + } + ABSL_RAW_LOG(INFO, "---"); +} + +static void CheckTransitiveClosure(Nodes *nodes, Edges *edges, const IdMap &id, + GraphCycles *gc) { + std::unordered_set<int> seen; + for (const auto &a : *nodes) { + for (const auto &b : *nodes) { + seen.clear(); + bool gc_reachable = gc->IsReachable(Get(id, a), Get(id, b)); + bool reachable = IsReachable(edges, a, b, &seen); + if (gc_reachable != reachable) { + PrintEdges(edges); + PrintGCEdges(nodes, id, gc); + PrintTransitiveClosure(nodes, edges); + PrintGCTransitiveClosure(nodes, id, gc); + ABSL_RAW_LOG(FATAL, "gc_reachable %s reachable %s a %d b %d", + gc_reachable ? "true" : "false", + reachable ? "true" : "false", a, b); + } + } + } +} + +static void CheckEdges(Nodes *nodes, Edges *edges, const IdMap &id, + GraphCycles *gc) { + int count = 0; + for (const auto &edge : *edges) { + int a = edge.from; + int b = edge.to; + if (!gc->HasEdge(Get(id, a), Get(id, b))) { + PrintEdges(edges); + PrintGCEdges(nodes, id, gc); + ABSL_RAW_LOG(FATAL, "!gc->HasEdge(%d, %d)", a, b); + } + } + for (const auto &a : *nodes) { + for (const auto &b : *nodes) { + if (gc->HasEdge(Get(id, a), Get(id, b))) { + count++; + } + } + } + if (count != edges->size()) { + PrintEdges(edges); + PrintGCEdges(nodes, id, gc); + ABSL_RAW_LOG(FATAL, "edges->size() %zu count %d", edges->size(), count); + } +} + +static void CheckInvariants(const GraphCycles &gc) { + if (ABSL_PREDICT_FALSE(!gc.CheckInvariants())) + ABSL_RAW_LOG(FATAL, "CheckInvariants"); +} + +// Returns the index of a randomly chosen node in *nodes. +// Requires *nodes be non-empty. +static int RandomNode(RandomEngine* rng, Nodes *nodes) { + std::uniform_int_distribution<int> uniform(0, nodes->size()-1); + return uniform(*rng); +} + +// Returns the index of a randomly chosen edge in *edges. +// Requires *edges be non-empty. +static int RandomEdge(RandomEngine* rng, Edges *edges) { + std::uniform_int_distribution<int> uniform(0, edges->size()-1); + return uniform(*rng); +} + +// Returns the index of edge (from, to) in *edges or -1 if it is not in *edges. +static int EdgeIndex(Edges *edges, int from, int to) { + int i = 0; + while (i != edges->size() && + ((*edges)[i].from != from || (*edges)[i].to != to)) { + i++; + } + return i == edges->size()? -1 : i; +} + +TEST(GraphCycles, RandomizedTest) { + int next_node = 0; + Nodes nodes; + Edges edges; // from, to + IdMap id; + GraphCycles graph_cycles; + static const int kMaxNodes = 7; // use <= 7 nodes to keep test short + static const int kDataOffset = 17; // an offset to the node-specific data + int n = 100000; + int op = 0; + RandomEngine rng(testing::UnitTest::GetInstance()->random_seed()); + std::uniform_int_distribution<int> uniform(0, 5); + + auto ptr = [](intptr_t i) { + return reinterpret_cast<void*>(i + kDataOffset); + }; + + for (int iter = 0; iter != n; iter++) { + for (const auto &node : nodes) { + ASSERT_EQ(graph_cycles.Ptr(Get(id, node)), ptr(node)) << " node " << node; + } + CheckEdges(&nodes, &edges, id, &graph_cycles); + CheckTransitiveClosure(&nodes, &edges, id, &graph_cycles); + op = uniform(rng); + switch (op) { + case 0: // Add a node + if (nodes.size() < kMaxNodes) { + int new_node = next_node++; + GraphId new_gnode = graph_cycles.GetId(ptr(new_node)); + ASSERT_NE(new_gnode, InvalidGraphId()); + id[new_node] = new_gnode; + ASSERT_EQ(ptr(new_node), graph_cycles.Ptr(new_gnode)); + nodes.push_back(new_node); + } + break; + + case 1: // Remove a node + if (nodes.size() > 0) { + int node_index = RandomNode(&rng, &nodes); + int node = nodes[node_index]; + nodes[node_index] = nodes.back(); + nodes.pop_back(); + graph_cycles.RemoveNode(ptr(node)); + ASSERT_EQ(graph_cycles.Ptr(Get(id, node)), nullptr); + id.erase(node); + int i = 0; + while (i != edges.size()) { + if (edges[i].from == node || edges[i].to == node) { + edges[i] = edges.back(); + edges.pop_back(); + } else { + i++; + } + } + } + break; + + case 2: // Add an edge + if (nodes.size() > 0) { + int from = RandomNode(&rng, &nodes); + int to = RandomNode(&rng, &nodes); + if (EdgeIndex(&edges, nodes[from], nodes[to]) == -1) { + if (graph_cycles.InsertEdge(id[nodes[from]], id[nodes[to]])) { + Edge new_edge; + new_edge.from = nodes[from]; + new_edge.to = nodes[to]; + edges.push_back(new_edge); + } else { + std::unordered_set<int> seen; + ASSERT_TRUE(IsReachable(&edges, nodes[to], nodes[from], &seen)) + << "Edge " << nodes[to] << "->" << nodes[from]; + } + } + } + break; + + case 3: // Remove an edge + if (edges.size() > 0) { + int i = RandomEdge(&rng, &edges); + int from = edges[i].from; + int to = edges[i].to; + ASSERT_EQ(i, EdgeIndex(&edges, from, to)); + edges[i] = edges.back(); + edges.pop_back(); + ASSERT_EQ(-1, EdgeIndex(&edges, from, to)); + graph_cycles.RemoveEdge(id[from], id[to]); + } + break; + + case 4: // Check a path + if (nodes.size() > 0) { + int from = RandomNode(&rng, &nodes); + int to = RandomNode(&rng, &nodes); + GraphId path[2*kMaxNodes]; + int path_len = graph_cycles.FindPath(id[nodes[from]], id[nodes[to]], + ABSL_ARRAYSIZE(path), path); + std::unordered_set<int> seen; + bool reachable = IsReachable(&edges, nodes[from], nodes[to], &seen); + bool gc_reachable = + graph_cycles.IsReachable(Get(id, nodes[from]), Get(id, nodes[to])); + ASSERT_EQ(path_len != 0, reachable); + ASSERT_EQ(path_len != 0, gc_reachable); + // In the following line, we add one because a node can appear + // twice, if the path is from that node to itself, perhaps via + // every other node. + ASSERT_LE(path_len, kMaxNodes + 1); + if (path_len != 0) { + ASSERT_EQ(id[nodes[from]], path[0]); + ASSERT_EQ(id[nodes[to]], path[path_len-1]); + for (int i = 1; i < path_len; i++) { + ASSERT_TRUE(graph_cycles.HasEdge(path[i-1], path[i])); + } + } + } + break; + + case 5: // Check invariants + CheckInvariants(graph_cycles); + break; + + default: + ABSL_RAW_LOG(FATAL, "op %d", op); + } + + // Very rarely, test graph expansion by adding then removing many nodes. + std::bernoulli_distribution one_in_1024(1.0 / 1024); + if (one_in_1024(rng)) { + CheckEdges(&nodes, &edges, id, &graph_cycles); + CheckTransitiveClosure(&nodes, &edges, id, &graph_cycles); + for (int i = 0; i != 256; i++) { + int new_node = next_node++; + GraphId new_gnode = graph_cycles.GetId(ptr(new_node)); + ASSERT_NE(InvalidGraphId(), new_gnode); + id[new_node] = new_gnode; + ASSERT_EQ(ptr(new_node), graph_cycles.Ptr(new_gnode)); + for (const auto &node : nodes) { + ASSERT_NE(node, new_node); + } + nodes.push_back(new_node); + } + for (int i = 0; i != 256; i++) { + ASSERT_GT(nodes.size(), 0); + int node_index = RandomNode(&rng, &nodes); + int node = nodes[node_index]; + nodes[node_index] = nodes.back(); + nodes.pop_back(); + graph_cycles.RemoveNode(ptr(node)); + id.erase(node); + int j = 0; + while (j != edges.size()) { + if (edges[j].from == node || edges[j].to == node) { + edges[j] = edges.back(); + edges.pop_back(); + } else { + j++; + } + } + } + CheckInvariants(graph_cycles); + } + } +} + +class GraphCyclesTest : public ::testing::Test { + public: + IdMap id_; + GraphCycles g_; + + static void* Ptr(int i) { + return reinterpret_cast<void*>(static_cast<uintptr_t>(i)); + } + + static int Num(void* ptr) { + return static_cast<int>(reinterpret_cast<uintptr_t>(ptr)); + } + + // Test relies on ith NewNode() call returning Node numbered i + GraphCyclesTest() { + for (int i = 0; i < 100; i++) { + id_[i] = g_.GetId(Ptr(i)); + } + CheckInvariants(g_); + } + + bool AddEdge(int x, int y) { + return g_.InsertEdge(Get(id_, x), Get(id_, y)); + } + + void AddMultiples() { + // For every node x > 0: add edge to 2*x, 3*x + for (int x = 1; x < 25; x++) { + EXPECT_TRUE(AddEdge(x, 2*x)) << x; + EXPECT_TRUE(AddEdge(x, 3*x)) << x; + } + CheckInvariants(g_); + } + + std::string Path(int x, int y) { + GraphId path[5]; + int np = g_.FindPath(Get(id_, x), Get(id_, y), ABSL_ARRAYSIZE(path), path); + std::string result; + for (int i = 0; i < np; i++) { + if (i >= ABSL_ARRAYSIZE(path)) { + result += " ..."; + break; + } + if (!result.empty()) result.push_back(' '); + char buf[20]; + snprintf(buf, sizeof(buf), "%d", Num(g_.Ptr(path[i]))); + result += buf; + } + return result; + } +}; + +TEST_F(GraphCyclesTest, NoCycle) { + AddMultiples(); + CheckInvariants(g_); +} + +TEST_F(GraphCyclesTest, SimpleCycle) { + AddMultiples(); + EXPECT_FALSE(AddEdge(8, 4)); + EXPECT_EQ("4 8", Path(4, 8)); + CheckInvariants(g_); +} + +TEST_F(GraphCyclesTest, IndirectCycle) { + AddMultiples(); + EXPECT_TRUE(AddEdge(16, 9)); + CheckInvariants(g_); + EXPECT_FALSE(AddEdge(9, 2)); + EXPECT_EQ("2 4 8 16 9", Path(2, 9)); + CheckInvariants(g_); +} + +TEST_F(GraphCyclesTest, LongPath) { + ASSERT_TRUE(AddEdge(2, 4)); + ASSERT_TRUE(AddEdge(4, 6)); + ASSERT_TRUE(AddEdge(6, 8)); + ASSERT_TRUE(AddEdge(8, 10)); + ASSERT_TRUE(AddEdge(10, 12)); + ASSERT_FALSE(AddEdge(12, 2)); + EXPECT_EQ("2 4 6 8 10 ...", Path(2, 12)); + CheckInvariants(g_); +} + +TEST_F(GraphCyclesTest, RemoveNode) { + ASSERT_TRUE(AddEdge(1, 2)); + ASSERT_TRUE(AddEdge(2, 3)); + ASSERT_TRUE(AddEdge(3, 4)); + ASSERT_TRUE(AddEdge(4, 5)); + g_.RemoveNode(g_.Ptr(id_[3])); + id_.erase(3); + ASSERT_TRUE(AddEdge(5, 1)); +} + +TEST_F(GraphCyclesTest, ManyEdges) { + const int N = 50; + for (int i = 0; i < N; i++) { + for (int j = 1; j < N; j++) { + ASSERT_TRUE(AddEdge(i, i+j)); + } + } + CheckInvariants(g_); + ASSERT_TRUE(AddEdge(2*N-1, 0)); + CheckInvariants(g_); + ASSERT_FALSE(AddEdge(10, 9)); + CheckInvariants(g_); +} + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/internal/kernel_timeout.h b/third_party/abseil_cpp/absl/synchronization/internal/kernel_timeout.h new file mode 100644 index 000000000000..bbd4d2d70f44 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/kernel_timeout.h @@ -0,0 +1,156 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// An optional absolute timeout, with nanosecond granularity, +// compatible with absl::Time. Suitable for in-register +// parameter-passing (e.g. syscalls.) +// Constructible from a absl::Time (for a timeout to be respected) or {} +// (for "no timeout".) +// This is a private low-level API for use by a handful of low-level +// components that are friends of this class. Higher-level components +// should build APIs based on absl::Time and absl::Duration. + +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_ + +#include <time.h> + +#include <algorithm> +#include <limits> + +#include "absl/base/internal/raw_logging.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +class Futex; +class Waiter; + +class KernelTimeout { + public: + // A timeout that should expire at <t>. Any value, in the full + // InfinitePast() to InfiniteFuture() range, is valid here and will be + // respected. + explicit KernelTimeout(absl::Time t) : ns_(MakeNs(t)) {} + // No timeout. + KernelTimeout() : ns_(0) {} + + // A more explicit factory for those who prefer it. Equivalent to {}. + static KernelTimeout Never() { return {}; } + + // We explicitly do not support other custom formats: timespec, int64_t nanos. + // Unify on this and absl::Time, please. + + bool has_timeout() const { return ns_ != 0; } + + // Convert to parameter for sem_timedwait/futex/similar. Only for approved + // users. Do not call if !has_timeout. + struct timespec MakeAbsTimespec(); + + private: + // internal rep, not user visible: ns after unix epoch. + // zero = no timeout. + // Negative we treat as an unlikely (and certainly expired!) but valid + // timeout. + int64_t ns_; + + static int64_t MakeNs(absl::Time t) { + // optimization--InfiniteFuture is common "no timeout" value + // and cheaper to compare than convert. + if (t == absl::InfiniteFuture()) return 0; + int64_t x = ToUnixNanos(t); + + // A timeout that lands exactly on the epoch (x=0) needs to be respected, + // so we alter it unnoticably to 1. Negative timeouts are in + // theory supported, but handled poorly by the kernel (long + // delays) so push them forward too; since all such times have + // already passed, it's indistinguishable. + if (x <= 0) x = 1; + // A time larger than what can be represented to the kernel is treated + // as no timeout. + if (x == (std::numeric_limits<int64_t>::max)()) x = 0; + return x; + } + +#ifdef _WIN32 + // Converts to milliseconds from now, or INFINITE when + // !has_timeout(). For use by SleepConditionVariableSRW on + // Windows. Callers should recognize that the return value is a + // relative duration (it should be recomputed by calling this method + // in the case of a spurious wakeup). + // This header file may be included transitively by public header files, + // so we define our own DWORD and INFINITE instead of getting them from + // <intsafe.h> and <WinBase.h>. + typedef unsigned long DWord; // NOLINT + DWord InMillisecondsFromNow() const { + constexpr DWord kInfinite = (std::numeric_limits<DWord>::max)(); + if (!has_timeout()) { + return kInfinite; + } + // The use of absl::Now() to convert from absolute time to + // relative time means that absl::Now() cannot use anything that + // depends on KernelTimeout (for example, Mutex) on Windows. + int64_t now = ToUnixNanos(absl::Now()); + if (ns_ >= now) { + // Round up so that Now() + ms_from_now >= ns_. + constexpr uint64_t max_nanos = + (std::numeric_limits<int64_t>::max)() - 999999u; + uint64_t ms_from_now = + (std::min<uint64_t>(max_nanos, ns_ - now) + 999999u) / 1000000u; + if (ms_from_now > kInfinite) { + return kInfinite; + } + return static_cast<DWord>(ms_from_now); + } + return 0; + } +#endif + + friend class Futex; + friend class Waiter; +}; + +inline struct timespec KernelTimeout::MakeAbsTimespec() { + int64_t n = ns_; + static const int64_t kNanosPerSecond = 1000 * 1000 * 1000; + if (n == 0) { + ABSL_RAW_LOG( + ERROR, "Tried to create a timespec from a non-timeout; never do this."); + // But we'll try to continue sanely. no-timeout ~= saturated timeout. + n = (std::numeric_limits<int64_t>::max)(); + } + + // Kernel APIs validate timespecs as being at or after the epoch, + // despite the kernel time type being signed. However, no one can + // tell the difference between a timeout at or before the epoch (since + // all such timeouts have expired!) + if (n < 0) n = 0; + + struct timespec abstime; + int64_t seconds = (std::min)(n / kNanosPerSecond, + int64_t{(std::numeric_limits<time_t>::max)()}); + abstime.tv_sec = static_cast<time_t>(seconds); + abstime.tv_nsec = static_cast<decltype(abstime.tv_nsec)>(n % kNanosPerSecond); + return abstime; +} + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_SYNCHRONIZATION_INTERNAL_KERNEL_TIMEOUT_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem.cc b/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem.cc new file mode 100644 index 000000000000..821ca9b4e9c3 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem.cc @@ -0,0 +1,106 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file is a no-op if the required LowLevelAlloc support is missing. +#include "absl/base/internal/low_level_alloc.h" +#ifndef ABSL_LOW_LEVEL_ALLOC_MISSING + +#include "absl/synchronization/internal/per_thread_sem.h" + +#include <atomic> + +#include "absl/base/attributes.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/synchronization/internal/waiter.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +void PerThreadSem::SetThreadBlockedCounter(std::atomic<int> *counter) { + base_internal::ThreadIdentity *identity; + identity = GetOrCreateCurrentThreadIdentity(); + identity->blocked_count_ptr = counter; +} + +std::atomic<int> *PerThreadSem::GetThreadBlockedCounter() { + base_internal::ThreadIdentity *identity; + identity = GetOrCreateCurrentThreadIdentity(); + return identity->blocked_count_ptr; +} + +void PerThreadSem::Init(base_internal::ThreadIdentity *identity) { + new (Waiter::GetWaiter(identity)) Waiter(); + identity->ticker.store(0, std::memory_order_relaxed); + identity->wait_start.store(0, std::memory_order_relaxed); + identity->is_idle.store(false, std::memory_order_relaxed); +} + +void PerThreadSem::Destroy(base_internal::ThreadIdentity *identity) { + Waiter::GetWaiter(identity)->~Waiter(); +} + +void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) { + const int ticker = + identity->ticker.fetch_add(1, std::memory_order_relaxed) + 1; + const int wait_start = identity->wait_start.load(std::memory_order_relaxed); + const bool is_idle = identity->is_idle.load(std::memory_order_relaxed); + if (wait_start && (ticker - wait_start > Waiter::kIdlePeriods) && !is_idle) { + // Wakeup the waiting thread since it is time for it to become idle. + Waiter::GetWaiter(identity)->Poke(); + } +} + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +extern "C" { + +ABSL_ATTRIBUTE_WEAK void AbslInternalPerThreadSemPost( + absl::base_internal::ThreadIdentity *identity) { + absl::synchronization_internal::Waiter::GetWaiter(identity)->Post(); +} + +ABSL_ATTRIBUTE_WEAK bool AbslInternalPerThreadSemWait( + absl::synchronization_internal::KernelTimeout t) { + bool timeout = false; + absl::base_internal::ThreadIdentity *identity; + identity = absl::synchronization_internal::GetOrCreateCurrentThreadIdentity(); + + // Ensure wait_start != 0. + int ticker = identity->ticker.load(std::memory_order_relaxed); + identity->wait_start.store(ticker ? ticker : 1, std::memory_order_relaxed); + identity->is_idle.store(false, std::memory_order_relaxed); + + if (identity->blocked_count_ptr != nullptr) { + // Increment count of threads blocked in a given thread pool. + identity->blocked_count_ptr->fetch_add(1, std::memory_order_relaxed); + } + + timeout = + !absl::synchronization_internal::Waiter::GetWaiter(identity)->Wait(t); + + if (identity->blocked_count_ptr != nullptr) { + identity->blocked_count_ptr->fetch_sub(1, std::memory_order_relaxed); + } + + identity->is_idle.store(false, std::memory_order_relaxed); + identity->wait_start.store(0, std::memory_order_relaxed); + return !timeout; +} + +} // extern "C" + +#endif // ABSL_LOW_LEVEL_ALLOC_MISSING diff --git a/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem.h b/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem.h new file mode 100644 index 000000000000..2228b6e8ea91 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem.h @@ -0,0 +1,115 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// PerThreadSem is a low-level synchronization primitive controlling the +// runnability of a single thread, used internally by Mutex and CondVar. +// +// This is NOT a general-purpose synchronization mechanism, and should not be +// used directly by applications. Applications should use Mutex and CondVar. +// +// The semantics of PerThreadSem are the same as that of a counting semaphore. +// Each thread maintains an abstract "count" value associated with its identity. + +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_PER_THREAD_SEM_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_PER_THREAD_SEM_H_ + +#include <atomic> + +#include "absl/base/internal/thread_identity.h" +#include "absl/synchronization/internal/create_thread_identity.h" +#include "absl/synchronization/internal/kernel_timeout.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class Mutex; + +namespace synchronization_internal { + +class PerThreadSem { + public: + PerThreadSem() = delete; + PerThreadSem(const PerThreadSem&) = delete; + PerThreadSem& operator=(const PerThreadSem&) = delete; + + // Routine invoked periodically (once a second) by a background thread. + // Has no effect on user-visible state. + static void Tick(base_internal::ThreadIdentity* identity); + + // --------------------------------------------------------------------------- + // Routines used by autosizing threadpools to detect when threads are + // blocked. Each thread has a counter pointer, initially zero. If non-zero, + // the implementation atomically increments the counter when it blocks on a + // semaphore, a decrements it again when it wakes. This allows a threadpool + // to keep track of how many of its threads are blocked. + // SetThreadBlockedCounter() should be used only by threadpool + // implementations. GetThreadBlockedCounter() should be used by modules that + // block threads; if the pointer returned is non-zero, the location should be + // incremented before the thread blocks, and decremented after it wakes. + static void SetThreadBlockedCounter(std::atomic<int> *counter); + static std::atomic<int> *GetThreadBlockedCounter(); + + private: + // Create the PerThreadSem associated with "identity". Initializes count=0. + // REQUIRES: May only be called by ThreadIdentity. + static void Init(base_internal::ThreadIdentity* identity); + + // Destroy the PerThreadSem associated with "identity". + // REQUIRES: May only be called by ThreadIdentity. + static void Destroy(base_internal::ThreadIdentity* identity); + + // Increments "identity"'s count. + static inline void Post(base_internal::ThreadIdentity* identity); + + // Waits until either our count > 0 or t has expired. + // If count > 0, decrements count and returns true. Otherwise returns false. + // !t.has_timeout() => Wait(t) will return true. + static inline bool Wait(KernelTimeout t); + + // Permitted callers. + friend class PerThreadSemTest; + friend class absl::Mutex; + friend absl::base_internal::ThreadIdentity* CreateThreadIdentity(); + friend void ReclaimThreadIdentity(void* v); +}; + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +// In some build configurations we pass --detect-odr-violations to the +// gold linker. This causes it to flag weak symbol overrides as ODR +// violations. Because ODR only applies to C++ and not C, +// --detect-odr-violations ignores symbols not mangled with C++ names. +// By changing our extension points to be extern "C", we dodge this +// check. +extern "C" { +void AbslInternalPerThreadSemPost( + absl::base_internal::ThreadIdentity* identity); +bool AbslInternalPerThreadSemWait( + absl::synchronization_internal::KernelTimeout t); +} // extern "C" + +void absl::synchronization_internal::PerThreadSem::Post( + absl::base_internal::ThreadIdentity* identity) { + AbslInternalPerThreadSemPost(identity); +} + +bool absl::synchronization_internal::PerThreadSem::Wait( + absl::synchronization_internal::KernelTimeout t) { + return AbslInternalPerThreadSemWait(t); +} + +#endif // ABSL_SYNCHRONIZATION_INTERNAL_PER_THREAD_SEM_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem_test.cc b/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem_test.cc new file mode 100644 index 000000000000..8cf59e64e961 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/per_thread_sem_test.cc @@ -0,0 +1,181 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/internal/per_thread_sem.h" + +#include <atomic> +#include <condition_variable> // NOLINT(build/c++11) +#include <functional> +#include <limits> +#include <mutex> // NOLINT(build/c++11) +#include <string> +#include <thread> // NOLINT(build/c++11) + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/strings/str_cat.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +// In this test we explicitly avoid the use of synchronization +// primitives which might use PerThreadSem, most notably absl::Mutex. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +class SimpleSemaphore { + public: + SimpleSemaphore() : count_(0) {} + + // Decrements (locks) the semaphore. If the semaphore's value is + // greater than zero, then the decrement proceeds, and the function + // returns, immediately. If the semaphore currently has the value + // zero, then the call blocks until it becomes possible to perform + // the decrement. + void Wait() { + std::unique_lock<std::mutex> lock(mu_); + cv_.wait(lock, [this]() { return count_ > 0; }); + --count_; + cv_.notify_one(); + } + + // Increments (unlocks) the semaphore. If the semaphore's value + // consequently becomes greater than zero, then another thread + // blocked Wait() call will be woken up and proceed to lock the + // semaphore. + void Post() { + std::lock_guard<std::mutex> lock(mu_); + ++count_; + cv_.notify_one(); + } + + private: + std::mutex mu_; + std::condition_variable cv_; + int count_; +}; + +struct ThreadData { + int num_iterations; // Number of replies to send. + SimpleSemaphore identity2_written; // Posted by thread writing identity2. + base_internal::ThreadIdentity *identity1; // First Post()-er. + base_internal::ThreadIdentity *identity2; // First Wait()-er. + KernelTimeout timeout; +}; + +// Need friendship with PerThreadSem. +class PerThreadSemTest : public testing::Test { + public: + static void TimingThread(ThreadData* t) { + t->identity2 = GetOrCreateCurrentThreadIdentity(); + t->identity2_written.Post(); + while (t->num_iterations--) { + Wait(t->timeout); + Post(t->identity1); + } + } + + void TestTiming(const char *msg, bool timeout) { + static const int kNumIterations = 100; + ThreadData t; + t.num_iterations = kNumIterations; + t.timeout = timeout ? + KernelTimeout(absl::Now() + absl::Seconds(10000)) // far in the future + : KernelTimeout::Never(); + t.identity1 = GetOrCreateCurrentThreadIdentity(); + + // We can't use the Thread class here because it uses the Mutex + // class which will invoke PerThreadSem, so we use std::thread instead. + std::thread partner_thread(std::bind(TimingThread, &t)); + + // Wait for our partner thread to register their identity. + t.identity2_written.Wait(); + + int64_t min_cycles = std::numeric_limits<int64_t>::max(); + int64_t total_cycles = 0; + for (int i = 0; i < kNumIterations; ++i) { + absl::SleepFor(absl::Milliseconds(20)); + int64_t cycles = base_internal::CycleClock::Now(); + Post(t.identity2); + Wait(t.timeout); + cycles = base_internal::CycleClock::Now() - cycles; + min_cycles = std::min(min_cycles, cycles); + total_cycles += cycles; + } + std::string out = StrCat( + msg, "min cycle count=", min_cycles, " avg cycle count=", + absl::SixDigits(static_cast<double>(total_cycles) / kNumIterations)); + printf("%s\n", out.c_str()); + + partner_thread.join(); + } + + protected: + static void Post(base_internal::ThreadIdentity *id) { + PerThreadSem::Post(id); + } + static bool Wait(KernelTimeout t) { + return PerThreadSem::Wait(t); + } + + // convenience overload + static bool Wait(absl::Time t) { + return Wait(KernelTimeout(t)); + } + + static void Tick(base_internal::ThreadIdentity *identity) { + PerThreadSem::Tick(identity); + } +}; + +namespace { + +TEST_F(PerThreadSemTest, WithoutTimeout) { + PerThreadSemTest::TestTiming("Without timeout: ", false); +} + +TEST_F(PerThreadSemTest, WithTimeout) { + PerThreadSemTest::TestTiming("With timeout: ", true); +} + +TEST_F(PerThreadSemTest, Timeouts) { + const absl::Duration delay = absl::Milliseconds(50); + const absl::Time start = absl::Now(); + EXPECT_FALSE(Wait(start + delay)); + const absl::Duration elapsed = absl::Now() - start; + // Allow for a slight early return, to account for quality of implementation + // issues on various platforms. + const absl::Duration slop = absl::Microseconds(200); + EXPECT_LE(delay - slop, elapsed) + << "Wait returned " << delay - elapsed + << " early (with " << slop << " slop), start time was " << start; + + absl::Time negative_timeout = absl::UnixEpoch() - absl::Milliseconds(100); + EXPECT_FALSE(Wait(negative_timeout)); + EXPECT_LE(negative_timeout, absl::Now() + slop); // trivially true :) + + Post(GetOrCreateCurrentThreadIdentity()); + // The wait here has an expired timeout, but we have a wake to consume, + // so this should succeed + EXPECT_TRUE(Wait(negative_timeout)); +} + +} // namespace + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/internal/thread_pool.h b/third_party/abseil_cpp/absl/synchronization/internal/thread_pool.h new file mode 100644 index 000000000000..0cb96dacde83 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/thread_pool.h @@ -0,0 +1,93 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_THREAD_POOL_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_THREAD_POOL_H_ + +#include <cassert> +#include <cstddef> +#include <functional> +#include <queue> +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// A simple ThreadPool implementation for tests. +class ThreadPool { + public: + explicit ThreadPool(int num_threads) { + for (int i = 0; i < num_threads; ++i) { + threads_.push_back(std::thread(&ThreadPool::WorkLoop, this)); + } + } + + ThreadPool(const ThreadPool &) = delete; + ThreadPool &operator=(const ThreadPool &) = delete; + + ~ThreadPool() { + { + absl::MutexLock l(&mu_); + for (size_t i = 0; i < threads_.size(); i++) { + queue_.push(nullptr); // Shutdown signal. + } + } + for (auto &t : threads_) { + t.join(); + } + } + + // Schedule a function to be run on a ThreadPool thread immediately. + void Schedule(std::function<void()> func) { + assert(func != nullptr); + absl::MutexLock l(&mu_); + queue_.push(std::move(func)); + } + + private: + bool WorkAvailable() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return !queue_.empty(); + } + + void WorkLoop() { + while (true) { + std::function<void()> func; + { + absl::MutexLock l(&mu_); + mu_.Await(absl::Condition(this, &ThreadPool::WorkAvailable)); + func = std::move(queue_.front()); + queue_.pop(); + } + if (func == nullptr) { // Shutdown signal. + break; + } + func(); + } + } + + absl::Mutex mu_; + std::queue<std::function<void()>> queue_ ABSL_GUARDED_BY(mu_); + std::vector<std::thread> threads_; +}; + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_SYNCHRONIZATION_INTERNAL_THREAD_POOL_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/internal/waiter.cc b/third_party/abseil_cpp/absl/synchronization/internal/waiter.cc new file mode 100644 index 000000000000..2123be60f54f --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/waiter.cc @@ -0,0 +1,428 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/internal/waiter.h" + +#include "absl/base/config.h" + +#ifdef _WIN32 +#include <windows.h> +#else +#include <pthread.h> +#include <sys/time.h> +#include <unistd.h> +#endif + +#ifdef __linux__ +#include <linux/futex.h> +#include <sys/syscall.h> +#endif + +#ifdef ABSL_HAVE_SEMAPHORE_H +#include <semaphore.h> +#endif + +#include <errno.h> +#include <stdio.h> +#include <time.h> + +#include <atomic> +#include <cassert> +#include <cstdint> +#include <new> +#include <type_traits> + +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/base/optimization.h" +#include "absl/synchronization/internal/kernel_timeout.h" + + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +static void MaybeBecomeIdle() { + base_internal::ThreadIdentity *identity = + base_internal::CurrentThreadIdentityIfPresent(); + assert(identity != nullptr); + const bool is_idle = identity->is_idle.load(std::memory_order_relaxed); + const int ticker = identity->ticker.load(std::memory_order_relaxed); + const int wait_start = identity->wait_start.load(std::memory_order_relaxed); + if (!is_idle && ticker - wait_start > Waiter::kIdlePeriods) { + identity->is_idle.store(true, std::memory_order_relaxed); + } +} + +#if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX + +Waiter::Waiter() { + futex_.store(0, std::memory_order_relaxed); +} + +Waiter::~Waiter() = default; + +bool Waiter::Wait(KernelTimeout t) { + // Loop until we can atomically decrement futex from a positive + // value, waiting on a futex while we believe it is zero. + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; + while (true) { + int32_t x = futex_.load(std::memory_order_relaxed); + while (x != 0) { + if (!futex_.compare_exchange_weak(x, x - 1, + std::memory_order_acquire, + std::memory_order_relaxed)) { + continue; // Raced with someone, retry. + } + return true; // Consumed a wakeup, we are done. + } + + + if (!first_pass) MaybeBecomeIdle(); + const int err = Futex::WaitUntil(&futex_, 0, t); + if (err != 0) { + if (err == -EINTR || err == -EWOULDBLOCK) { + // Do nothing, the loop will retry. + } else if (err == -ETIMEDOUT) { + return false; + } else { + ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); + } + } + first_pass = false; + } +} + +void Waiter::Post() { + if (futex_.fetch_add(1, std::memory_order_release) == 0) { + // We incremented from 0, need to wake a potential waiter. + Poke(); + } +} + +void Waiter::Poke() { + // Wake one thread waiting on the futex. + const int err = Futex::Wake(&futex_, 1); + if (ABSL_PREDICT_FALSE(err < 0)) { + ABSL_RAW_LOG(FATAL, "Futex operation failed with error %d\n", err); + } +} + +#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR + +class PthreadMutexHolder { + public: + explicit PthreadMutexHolder(pthread_mutex_t *mu) : mu_(mu) { + const int err = pthread_mutex_lock(mu_); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_mutex_lock failed: %d", err); + } + } + + PthreadMutexHolder(const PthreadMutexHolder &rhs) = delete; + PthreadMutexHolder &operator=(const PthreadMutexHolder &rhs) = delete; + + ~PthreadMutexHolder() { + const int err = pthread_mutex_unlock(mu_); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_mutex_unlock failed: %d", err); + } + } + + private: + pthread_mutex_t *mu_; +}; + +Waiter::Waiter() { + const int err = pthread_mutex_init(&mu_, 0); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_mutex_init failed: %d", err); + } + + const int err2 = pthread_cond_init(&cv_, 0); + if (err2 != 0) { + ABSL_RAW_LOG(FATAL, "pthread_cond_init failed: %d", err2); + } + + waiter_count_ = 0; + wakeup_count_ = 0; +} + +Waiter::~Waiter() { + const int err = pthread_mutex_destroy(&mu_); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_mutex_destroy failed: %d", err); + } + + const int err2 = pthread_cond_destroy(&cv_); + if (err2 != 0) { + ABSL_RAW_LOG(FATAL, "pthread_cond_destroy failed: %d", err2); + } +} + +bool Waiter::Wait(KernelTimeout t) { + struct timespec abs_timeout; + if (t.has_timeout()) { + abs_timeout = t.MakeAbsTimespec(); + } + + PthreadMutexHolder h(&mu_); + ++waiter_count_; + // Loop until we find a wakeup to consume or timeout. + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; + while (wakeup_count_ == 0) { + if (!first_pass) MaybeBecomeIdle(); + // No wakeups available, time to wait. + if (!t.has_timeout()) { + const int err = pthread_cond_wait(&cv_, &mu_); + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_cond_wait failed: %d", err); + } + } else { + const int err = pthread_cond_timedwait(&cv_, &mu_, &abs_timeout); + if (err == ETIMEDOUT) { + --waiter_count_; + return false; + } + if (err != 0) { + ABSL_RAW_LOG(FATAL, "pthread_cond_timedwait failed: %d", err); + } + } + first_pass = false; + } + // Consume a wakeup and we're done. + --wakeup_count_; + --waiter_count_; + return true; +} + +void Waiter::Post() { + PthreadMutexHolder h(&mu_); + ++wakeup_count_; + InternalCondVarPoke(); +} + +void Waiter::Poke() { + PthreadMutexHolder h(&mu_); + InternalCondVarPoke(); +} + +void Waiter::InternalCondVarPoke() { + if (waiter_count_ != 0) { + const int err = pthread_cond_signal(&cv_); + if (ABSL_PREDICT_FALSE(err != 0)) { + ABSL_RAW_LOG(FATAL, "pthread_cond_signal failed: %d", err); + } + } +} + +#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM + +Waiter::Waiter() { + if (sem_init(&sem_, 0, 0) != 0) { + ABSL_RAW_LOG(FATAL, "sem_init failed with errno %d\n", errno); + } + wakeups_.store(0, std::memory_order_relaxed); +} + +Waiter::~Waiter() { + if (sem_destroy(&sem_) != 0) { + ABSL_RAW_LOG(FATAL, "sem_destroy failed with errno %d\n", errno); + } +} + +bool Waiter::Wait(KernelTimeout t) { + struct timespec abs_timeout; + if (t.has_timeout()) { + abs_timeout = t.MakeAbsTimespec(); + } + + // Loop until we timeout or consume a wakeup. + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; + while (true) { + int x = wakeups_.load(std::memory_order_relaxed); + while (x != 0) { + if (!wakeups_.compare_exchange_weak(x, x - 1, + std::memory_order_acquire, + std::memory_order_relaxed)) { + continue; // Raced with someone, retry. + } + // Successfully consumed a wakeup, we're done. + return true; + } + + if (!first_pass) MaybeBecomeIdle(); + // Nothing to consume, wait (looping on EINTR). + while (true) { + if (!t.has_timeout()) { + if (sem_wait(&sem_) == 0) break; + if (errno == EINTR) continue; + ABSL_RAW_LOG(FATAL, "sem_wait failed: %d", errno); + } else { + if (sem_timedwait(&sem_, &abs_timeout) == 0) break; + if (errno == EINTR) continue; + if (errno == ETIMEDOUT) return false; + ABSL_RAW_LOG(FATAL, "sem_timedwait failed: %d", errno); + } + } + first_pass = false; + } +} + +void Waiter::Post() { + // Post a wakeup. + if (wakeups_.fetch_add(1, std::memory_order_release) == 0) { + // We incremented from 0, need to wake a potential waiter. + Poke(); + } +} + +void Waiter::Poke() { + if (sem_post(&sem_) != 0) { // Wake any semaphore waiter. + ABSL_RAW_LOG(FATAL, "sem_post failed with errno %d\n", errno); + } +} + +#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32 + +class Waiter::WinHelper { + public: + static SRWLOCK *GetLock(Waiter *w) { + return reinterpret_cast<SRWLOCK *>(&w->mu_storage_); + } + + static CONDITION_VARIABLE *GetCond(Waiter *w) { + return reinterpret_cast<CONDITION_VARIABLE *>(&w->cv_storage_); + } + + static_assert(sizeof(SRWLOCK) == sizeof(void *), + "`mu_storage_` does not have the same size as SRWLOCK"); + static_assert(alignof(SRWLOCK) == alignof(void *), + "`mu_storage_` does not have the same alignment as SRWLOCK"); + + static_assert(sizeof(CONDITION_VARIABLE) == sizeof(void *), + "`ABSL_CONDITION_VARIABLE_STORAGE` does not have the same size " + "as `CONDITION_VARIABLE`"); + static_assert( + alignof(CONDITION_VARIABLE) == alignof(void *), + "`cv_storage_` does not have the same alignment as `CONDITION_VARIABLE`"); + + // The SRWLOCK and CONDITION_VARIABLE types must be trivially constructible + // and destructible because we never call their constructors or destructors. + static_assert(std::is_trivially_constructible<SRWLOCK>::value, + "The `SRWLOCK` type must be trivially constructible"); + static_assert( + std::is_trivially_constructible<CONDITION_VARIABLE>::value, + "The `CONDITION_VARIABLE` type must be trivially constructible"); + static_assert(std::is_trivially_destructible<SRWLOCK>::value, + "The `SRWLOCK` type must be trivially destructible"); + static_assert(std::is_trivially_destructible<CONDITION_VARIABLE>::value, + "The `CONDITION_VARIABLE` type must be trivially destructible"); +}; + +class LockHolder { + public: + explicit LockHolder(SRWLOCK* mu) : mu_(mu) { + AcquireSRWLockExclusive(mu_); + } + + LockHolder(const LockHolder&) = delete; + LockHolder& operator=(const LockHolder&) = delete; + + ~LockHolder() { + ReleaseSRWLockExclusive(mu_); + } + + private: + SRWLOCK* mu_; +}; + +Waiter::Waiter() { + auto *mu = ::new (static_cast<void *>(&mu_storage_)) SRWLOCK; + auto *cv = ::new (static_cast<void *>(&cv_storage_)) CONDITION_VARIABLE; + InitializeSRWLock(mu); + InitializeConditionVariable(cv); + waiter_count_ = 0; + wakeup_count_ = 0; +} + +// SRW locks and condition variables do not need to be explicitly destroyed. +// https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock +// https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with +Waiter::~Waiter() = default; + +bool Waiter::Wait(KernelTimeout t) { + SRWLOCK *mu = WinHelper::GetLock(this); + CONDITION_VARIABLE *cv = WinHelper::GetCond(this); + + LockHolder h(mu); + ++waiter_count_; + + // Loop until we find a wakeup to consume or timeout. + // Note that, since the thread ticker is just reset, we don't need to check + // whether the thread is idle on the very first pass of the loop. + bool first_pass = true; + while (wakeup_count_ == 0) { + if (!first_pass) MaybeBecomeIdle(); + // No wakeups available, time to wait. + if (!SleepConditionVariableSRW(cv, mu, t.InMillisecondsFromNow(), 0)) { + // GetLastError() returns a Win32 DWORD, but we assign to + // unsigned long to simplify the ABSL_RAW_LOG case below. The uniform + // initialization guarantees this is not a narrowing conversion. + const unsigned long err{GetLastError()}; // NOLINT(runtime/int) + if (err == ERROR_TIMEOUT) { + --waiter_count_; + return false; + } else { + ABSL_RAW_LOG(FATAL, "SleepConditionVariableSRW failed: %lu", err); + } + } + first_pass = false; + } + // Consume a wakeup and we're done. + --wakeup_count_; + --waiter_count_; + return true; +} + +void Waiter::Post() { + LockHolder h(WinHelper::GetLock(this)); + ++wakeup_count_; + InternalCondVarPoke(); +} + +void Waiter::Poke() { + LockHolder h(WinHelper::GetLock(this)); + InternalCondVarPoke(); +} + +void Waiter::InternalCondVarPoke() { + if (waiter_count_ != 0) { + WakeConditionVariable(WinHelper::GetCond(this)); + } +} + +#else +#error Unknown ABSL_WAITER_MODE +#endif + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/internal/waiter.h b/third_party/abseil_cpp/absl/synchronization/internal/waiter.h new file mode 100644 index 000000000000..be3df180d4e2 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/internal/waiter.h @@ -0,0 +1,155 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_ +#define ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_ + +#include "absl/base/config.h" + +#ifdef _WIN32 +#include <sdkddkver.h> +#else +#include <pthread.h> +#endif + +#ifdef __linux__ +#include <linux/futex.h> +#endif + +#ifdef ABSL_HAVE_SEMAPHORE_H +#include <semaphore.h> +#endif + +#include <atomic> +#include <cstdint> + +#include "absl/base/internal/thread_identity.h" +#include "absl/synchronization/internal/futex.h" +#include "absl/synchronization/internal/kernel_timeout.h" + +// May be chosen at compile time via -DABSL_FORCE_WAITER_MODE=<index> +#define ABSL_WAITER_MODE_FUTEX 0 +#define ABSL_WAITER_MODE_SEM 1 +#define ABSL_WAITER_MODE_CONDVAR 2 +#define ABSL_WAITER_MODE_WIN32 3 + +#if defined(ABSL_FORCE_WAITER_MODE) +#define ABSL_WAITER_MODE ABSL_FORCE_WAITER_MODE +#elif defined(_WIN32) && _WIN32_WINNT >= _WIN32_WINNT_VISTA +#define ABSL_WAITER_MODE ABSL_WAITER_MODE_WIN32 +#elif defined(ABSL_INTERNAL_HAVE_FUTEX) +#define ABSL_WAITER_MODE ABSL_WAITER_MODE_FUTEX +#elif defined(ABSL_HAVE_SEMAPHORE_H) +#define ABSL_WAITER_MODE ABSL_WAITER_MODE_SEM +#else +#define ABSL_WAITER_MODE ABSL_WAITER_MODE_CONDVAR +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace synchronization_internal { + +// Waiter is an OS-specific semaphore. +class Waiter { + public: + // Prepare any data to track waits. + Waiter(); + + // Not copyable or movable + Waiter(const Waiter&) = delete; + Waiter& operator=(const Waiter&) = delete; + + // Destroy any data to track waits. + ~Waiter(); + + // Blocks the calling thread until a matching call to `Post()` or + // `t` has passed. Returns `true` if woken (`Post()` called), + // `false` on timeout. + bool Wait(KernelTimeout t); + + // Restart the caller of `Wait()` as with a normal semaphore. + void Post(); + + // If anyone is waiting, wake them up temporarily and cause them to + // call `MaybeBecomeIdle()`. They will then return to waiting for a + // `Post()` or timeout. + void Poke(); + + // Returns the Waiter associated with the identity. + static Waiter* GetWaiter(base_internal::ThreadIdentity* identity) { + static_assert( + sizeof(Waiter) <= sizeof(base_internal::ThreadIdentity::WaiterState), + "Insufficient space for Waiter"); + return reinterpret_cast<Waiter*>(identity->waiter_state.data); + } + + // How many periods to remain idle before releasing resources +#ifndef ABSL_HAVE_THREAD_SANITIZER + static constexpr int kIdlePeriods = 60; +#else + // Memory consumption under ThreadSanitizer is a serious concern, + // so we release resources sooner. The value of 1 leads to 1 to 2 second + // delay before marking a thread as idle. + static const int kIdlePeriods = 1; +#endif + + private: +#if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX + // Futexes are defined by specification to be 32-bits. + // Thus std::atomic<int32_t> must be just an int32_t with lockfree methods. + std::atomic<int32_t> futex_; + static_assert(sizeof(int32_t) == sizeof(futex_), "Wrong size for futex"); + +#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_CONDVAR + // REQUIRES: mu_ must be held. + void InternalCondVarPoke(); + + pthread_mutex_t mu_; + pthread_cond_t cv_; + int waiter_count_; + int wakeup_count_; // Unclaimed wakeups. + +#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_SEM + sem_t sem_; + // This seems superfluous, but for Poke() we need to cause spurious + // wakeups on the semaphore. Hence we can't actually use the + // semaphore's count. + std::atomic<int> wakeups_; + +#elif ABSL_WAITER_MODE == ABSL_WAITER_MODE_WIN32 + // WinHelper - Used to define utilities for accessing the lock and + // condition variable storage once the types are complete. + class WinHelper; + + // REQUIRES: WinHelper::GetLock(this) must be held. + void InternalCondVarPoke(); + + // We can't include Windows.h in our headers, so we use aligned charachter + // buffers to define the storage of SRWLOCK and CONDITION_VARIABLE. + alignas(void*) unsigned char mu_storage_[sizeof(void*)]; + alignas(void*) unsigned char cv_storage_[sizeof(void*)]; + int waiter_count_; + int wakeup_count_; + +#else + #error Unknown ABSL_WAITER_MODE +#endif +}; + +} // namespace synchronization_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_SYNCHRONIZATION_INTERNAL_WAITER_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/lifetime_test.cc b/third_party/abseil_cpp/absl/synchronization/lifetime_test.cc new file mode 100644 index 000000000000..cc973a329071 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/lifetime_test.cc @@ -0,0 +1,181 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdlib> +#include <thread> // NOLINT(build/c++11), Abseil test +#include <type_traits> + +#include "absl/base/attributes.h" +#include "absl/base/const_init.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" + +namespace { + +// A two-threaded test which checks that Mutex, CondVar, and Notification have +// correct basic functionality. The intent is to establish that they +// function correctly in various phases of construction and destruction. +// +// Thread one acquires a lock on 'mutex', wakes thread two via 'notification', +// then waits for 'state' to be set, as signalled by 'condvar'. +// +// Thread two waits on 'notification', then sets 'state' inside the 'mutex', +// signalling the change via 'condvar'. +// +// These tests use ABSL_RAW_CHECK to validate invariants, rather than EXPECT or +// ASSERT from gUnit, because we need to invoke them during global destructors, +// when gUnit teardown would have already begun. +void ThreadOne(absl::Mutex* mutex, absl::CondVar* condvar, + absl::Notification* notification, bool* state) { + // Test that the notification is in a valid initial state. + ABSL_RAW_CHECK(!notification->HasBeenNotified(), "invalid Notification"); + ABSL_RAW_CHECK(*state == false, "*state not initialized"); + + { + absl::MutexLock lock(mutex); + + notification->Notify(); + ABSL_RAW_CHECK(notification->HasBeenNotified(), "invalid Notification"); + + while (*state == false) { + condvar->Wait(mutex); + } + } +} + +void ThreadTwo(absl::Mutex* mutex, absl::CondVar* condvar, + absl::Notification* notification, bool* state) { + ABSL_RAW_CHECK(*state == false, "*state not initialized"); + + // Wake thread one + notification->WaitForNotification(); + ABSL_RAW_CHECK(notification->HasBeenNotified(), "invalid Notification"); + { + absl::MutexLock lock(mutex); + *state = true; + condvar->Signal(); + } +} + +// Launch thread 1 and thread 2, and block on their completion. +// If any of 'mutex', 'condvar', or 'notification' is nullptr, use a locally +// constructed instance instead. +void RunTests(absl::Mutex* mutex, absl::CondVar* condvar) { + absl::Mutex default_mutex; + absl::CondVar default_condvar; + absl::Notification notification; + if (!mutex) { + mutex = &default_mutex; + } + if (!condvar) { + condvar = &default_condvar; + } + bool state = false; + std::thread thread_one(ThreadOne, mutex, condvar, ¬ification, &state); + std::thread thread_two(ThreadTwo, mutex, condvar, ¬ification, &state); + thread_one.join(); + thread_two.join(); +} + +void TestLocals() { + absl::Mutex mutex; + absl::CondVar condvar; + RunTests(&mutex, &condvar); +} + +// Normal kConstInit usage +ABSL_CONST_INIT absl::Mutex const_init_mutex(absl::kConstInit); +void TestConstInitGlobal() { RunTests(&const_init_mutex, nullptr); } + +// Global variables during start and termination +// +// In a translation unit, static storage duration variables are initialized in +// the order of their definitions, and destroyed in the reverse order of their +// definitions. We can use this to arrange for tests to be run on these objects +// before they are created, and after they are destroyed. + +using Function = void (*)(); + +class OnConstruction { + public: + explicit OnConstruction(Function fn) { fn(); } +}; + +class OnDestruction { + public: + explicit OnDestruction(Function fn) : fn_(fn) {} + ~OnDestruction() { fn_(); } + private: + Function fn_; +}; + +// These tests require that the compiler correctly supports C++11 constant +// initialization... but MSVC has a known regression since v19.10: +// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html +// TODO(epastor): Limit the affected range once MSVC fixes this bug. +#if defined(__clang__) || !(defined(_MSC_VER) && _MSC_VER > 1900) +// kConstInit +// Test early usage. (Declaration comes first; definitions must appear after +// the test runner.) +extern absl::Mutex early_const_init_mutex; +// (Normally I'd write this +[], to make the cast-to-function-pointer explicit, +// but in some MSVC setups we support, lambdas provide conversion operators to +// different flavors of function pointers, making this trick ambiguous.) +OnConstruction test_early_const_init([] { + RunTests(&early_const_init_mutex, nullptr); +}); +// This definition appears before test_early_const_init, but it should be +// initialized first (due to constant initialization). Test that the object +// actually works when constructed this way. +ABSL_CONST_INIT absl::Mutex early_const_init_mutex(absl::kConstInit); + +// Furthermore, test that the const-init c'tor doesn't stomp over the state of +// a Mutex. Really, this is a test that the platform under test correctly +// supports C++11 constant initialization. (The constant-initialization +// constructors of globals "happen at link time"; memory is pre-initialized, +// before the constructors of either grab_lock or check_still_locked are run.) +extern absl::Mutex const_init_sanity_mutex; +OnConstruction grab_lock([]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + const_init_sanity_mutex.Lock(); +}); +ABSL_CONST_INIT absl::Mutex const_init_sanity_mutex(absl::kConstInit); +OnConstruction check_still_locked([]() ABSL_NO_THREAD_SAFETY_ANALYSIS { + const_init_sanity_mutex.AssertHeld(); + const_init_sanity_mutex.Unlock(); +}); +#endif // defined(__clang__) || !(defined(_MSC_VER) && _MSC_VER > 1900) + +// Test shutdown usage. (Declarations come first; definitions must appear after +// the test runner.) +extern absl::Mutex late_const_init_mutex; +// OnDestruction is being used here as a global variable, even though it has a +// non-trivial destructor. This is against the style guide. We're violating +// that rule here to check that the exception we allow for kConstInit is safe. +// NOLINTNEXTLINE +OnDestruction test_late_const_init([] { + RunTests(&late_const_init_mutex, nullptr); +}); +ABSL_CONST_INIT absl::Mutex late_const_init_mutex(absl::kConstInit); + +} // namespace + +int main() { + TestLocals(); + TestConstInitGlobal(); + // Explicitly call exit(0) here, to make it clear that we intend for the + // above global object destructors to run. + std::exit(0); +} diff --git a/third_party/abseil_cpp/absl/synchronization/mutex.cc b/third_party/abseil_cpp/absl/synchronization/mutex.cc new file mode 100644 index 000000000000..9e01393ca4df --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/mutex.cc @@ -0,0 +1,2740 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/mutex.h" + +#ifdef _WIN32 +#include <windows.h> +#ifdef ERROR +#undef ERROR +#endif +#else +#include <fcntl.h> +#include <pthread.h> +#include <sched.h> +#include <sys/time.h> +#endif + +#include <assert.h> +#include <errno.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> + +#include <algorithm> +#include <atomic> +#include <cinttypes> +#include <thread> // NOLINT(build/c++11) + +#include "absl/base/attributes.h" +#include "absl/base/call_once.h" +#include "absl/base/config.h" +#include "absl/base/dynamic_annotations.h" +#include "absl/base/internal/atomic_hook.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/hide_ptr.h" +#include "absl/base/internal/low_level_alloc.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/internal/sysinfo.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/base/internal/tsan_mutex_interface.h" +#include "absl/base/port.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/synchronization/internal/graphcycles.h" +#include "absl/synchronization/internal/per_thread_sem.h" +#include "absl/time/time.h" + +using absl::base_internal::CurrentThreadIdentityIfPresent; +using absl::base_internal::PerThreadSynch; +using absl::base_internal::SchedulingGuard; +using absl::base_internal::ThreadIdentity; +using absl::synchronization_internal::GetOrCreateCurrentThreadIdentity; +using absl::synchronization_internal::GraphCycles; +using absl::synchronization_internal::GraphId; +using absl::synchronization_internal::InvalidGraphId; +using absl::synchronization_internal::KernelTimeout; +using absl::synchronization_internal::PerThreadSem; + +extern "C" { +ABSL_ATTRIBUTE_WEAK void AbslInternalMutexYield() { std::this_thread::yield(); } +} // extern "C" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +#if defined(ABSL_HAVE_THREAD_SANITIZER) +constexpr OnDeadlockCycle kDeadlockDetectionDefault = OnDeadlockCycle::kIgnore; +#else +constexpr OnDeadlockCycle kDeadlockDetectionDefault = OnDeadlockCycle::kAbort; +#endif + +ABSL_CONST_INIT std::atomic<OnDeadlockCycle> synch_deadlock_detection( + kDeadlockDetectionDefault); +ABSL_CONST_INIT std::atomic<bool> synch_check_invariants(false); + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +absl::base_internal::AtomicHook<void (*)(int64_t wait_cycles)> + submit_profile_data; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook<void (*)( + const char *msg, const void *obj, int64_t wait_cycles)> + mutex_tracer; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES + absl::base_internal::AtomicHook<void (*)(const char *msg, const void *cv)> + cond_var_tracer; +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES absl::base_internal::AtomicHook< + bool (*)(const void *pc, char *out, int out_size)> + symbolizer(absl::Symbolize); + +} // namespace + +static inline bool EvalConditionAnnotated(const Condition *cond, Mutex *mu, + bool locking, bool trylock, + bool read_lock); + +void RegisterMutexProfiler(void (*fn)(int64_t wait_timestamp)) { + submit_profile_data.Store(fn); +} + +void RegisterMutexTracer(void (*fn)(const char *msg, const void *obj, + int64_t wait_cycles)) { + mutex_tracer.Store(fn); +} + +void RegisterCondVarTracer(void (*fn)(const char *msg, const void *cv)) { + cond_var_tracer.Store(fn); +} + +void RegisterSymbolizer(bool (*fn)(const void *pc, char *out, int out_size)) { + symbolizer.Store(fn); +} + +struct ABSL_CACHELINE_ALIGNED MutexGlobals { + absl::once_flag once; + int num_cpus = 0; + int spinloop_iterations = 0; +}; + +static const MutexGlobals& GetMutexGlobals() { + ABSL_CONST_INIT static MutexGlobals data; + absl::base_internal::LowLevelCallOnce(&data.once, [&]() { + data.num_cpus = absl::base_internal::NumCPUs(); + data.spinloop_iterations = data.num_cpus > 1 ? 1500 : 0; + }); + return data; +} + +// Spinlock delay on iteration c. Returns new c. +namespace { + enum DelayMode { AGGRESSIVE, GENTLE }; +}; + +namespace synchronization_internal { +int MutexDelay(int32_t c, int mode) { + // If this a uniprocessor, only yield/sleep. Otherwise, if the mode is + // aggressive then spin many times before yielding. If the mode is + // gentle then spin only a few times before yielding. Aggressive spinning is + // used to ensure that an Unlock() call, which must get the spin lock for + // any thread to make progress gets it without undue delay. + const int32_t limit = + GetMutexGlobals().num_cpus > 1 ? (mode == AGGRESSIVE ? 5000 : 250) : 0; + if (c < limit) { + // Spin. + c++; + } else { + SchedulingGuard::ScopedEnable enable_rescheduling; + ABSL_TSAN_MUTEX_PRE_DIVERT(nullptr, 0); + if (c == limit) { + // Yield once. + AbslInternalMutexYield(); + c++; + } else { + // Then wait. + absl::SleepFor(absl::Microseconds(10)); + c = 0; + } + ABSL_TSAN_MUTEX_POST_DIVERT(nullptr, 0); + } + return c; +} +} // namespace synchronization_internal + +// --------------------------Generic atomic ops +// Ensure that "(*pv & bits) == bits" by doing an atomic update of "*pv" to +// "*pv | bits" if necessary. Wait until (*pv & wait_until_clear)==0 +// before making any change. +// This is used to set flags in mutex and condition variable words. +static void AtomicSetBits(std::atomic<intptr_t>* pv, intptr_t bits, + intptr_t wait_until_clear) { + intptr_t v; + do { + v = pv->load(std::memory_order_relaxed); + } while ((v & bits) != bits && + ((v & wait_until_clear) != 0 || + !pv->compare_exchange_weak(v, v | bits, + std::memory_order_release, + std::memory_order_relaxed))); +} + +// Ensure that "(*pv & bits) == 0" by doing an atomic update of "*pv" to +// "*pv & ~bits" if necessary. Wait until (*pv & wait_until_clear)==0 +// before making any change. +// This is used to unset flags in mutex and condition variable words. +static void AtomicClearBits(std::atomic<intptr_t>* pv, intptr_t bits, + intptr_t wait_until_clear) { + intptr_t v; + do { + v = pv->load(std::memory_order_relaxed); + } while ((v & bits) != 0 && + ((v & wait_until_clear) != 0 || + !pv->compare_exchange_weak(v, v & ~bits, + std::memory_order_release, + std::memory_order_relaxed))); +} + +//------------------------------------------------------------------ + +// Data for doing deadlock detection. +ABSL_CONST_INIT static absl::base_internal::SpinLock deadlock_graph_mu( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); + +// Graph used to detect deadlocks. +ABSL_CONST_INIT static GraphCycles *deadlock_graph + ABSL_GUARDED_BY(deadlock_graph_mu) ABSL_PT_GUARDED_BY(deadlock_graph_mu); + +//------------------------------------------------------------------ +// An event mechanism for debugging mutex use. +// It also allows mutexes to be given names for those who can't handle +// addresses, and instead like to give their data structures names like +// "Henry", "Fido", or "Rupert IV, King of Yondavia". + +namespace { // to prevent name pollution +enum { // Mutex and CondVar events passed as "ev" to PostSynchEvent + // Mutex events + SYNCH_EV_TRYLOCK_SUCCESS, + SYNCH_EV_TRYLOCK_FAILED, + SYNCH_EV_READERTRYLOCK_SUCCESS, + SYNCH_EV_READERTRYLOCK_FAILED, + SYNCH_EV_LOCK, + SYNCH_EV_LOCK_RETURNING, + SYNCH_EV_READERLOCK, + SYNCH_EV_READERLOCK_RETURNING, + SYNCH_EV_UNLOCK, + SYNCH_EV_READERUNLOCK, + + // CondVar events + SYNCH_EV_WAIT, + SYNCH_EV_WAIT_RETURNING, + SYNCH_EV_SIGNAL, + SYNCH_EV_SIGNALALL, +}; + +enum { // Event flags + SYNCH_F_R = 0x01, // reader event + SYNCH_F_LCK = 0x02, // PostSynchEvent called with mutex held + SYNCH_F_TRY = 0x04, // TryLock or ReaderTryLock + SYNCH_F_UNLOCK = 0x08, // Unlock or ReaderUnlock + + SYNCH_F_LCK_W = SYNCH_F_LCK, + SYNCH_F_LCK_R = SYNCH_F_LCK | SYNCH_F_R, +}; +} // anonymous namespace + +// Properties of the events. +static const struct { + int flags; + const char *msg; +} event_properties[] = { + {SYNCH_F_LCK_W | SYNCH_F_TRY, "TryLock succeeded "}, + {0, "TryLock failed "}, + {SYNCH_F_LCK_R | SYNCH_F_TRY, "ReaderTryLock succeeded "}, + {0, "ReaderTryLock failed "}, + {0, "Lock blocking "}, + {SYNCH_F_LCK_W, "Lock returning "}, + {0, "ReaderLock blocking "}, + {SYNCH_F_LCK_R, "ReaderLock returning "}, + {SYNCH_F_LCK_W | SYNCH_F_UNLOCK, "Unlock "}, + {SYNCH_F_LCK_R | SYNCH_F_UNLOCK, "ReaderUnlock "}, + {0, "Wait on "}, + {0, "Wait unblocked "}, + {0, "Signal on "}, + {0, "SignalAll on "}, +}; + +ABSL_CONST_INIT static absl::base_internal::SpinLock synch_event_mu( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); + +// Hash table size; should be prime > 2. +// Can't be too small, as it's used for deadlock detection information. +static constexpr uint32_t kNSynchEvent = 1031; + +static struct SynchEvent { // this is a trivial hash table for the events + // struct is freed when refcount reaches 0 + int refcount ABSL_GUARDED_BY(synch_event_mu); + + // buckets have linear, 0-terminated chains + SynchEvent *next ABSL_GUARDED_BY(synch_event_mu); + + // Constant after initialization + uintptr_t masked_addr; // object at this address is called "name" + + // No explicit synchronization used. Instead we assume that the + // client who enables/disables invariants/logging on a Mutex does so + // while the Mutex is not being concurrently accessed by others. + void (*invariant)(void *arg); // called on each event + void *arg; // first arg to (*invariant)() + bool log; // logging turned on + + // Constant after initialization + char name[1]; // actually longer---NUL-terminated string +} * synch_event[kNSynchEvent] ABSL_GUARDED_BY(synch_event_mu); + +// Ensure that the object at "addr" has a SynchEvent struct associated with it, +// set "bits" in the word there (waiting until lockbit is clear before doing +// so), and return a refcounted reference that will remain valid until +// UnrefSynchEvent() is called. If a new SynchEvent is allocated, +// the string name is copied into it. +// When used with a mutex, the caller should also ensure that kMuEvent +// is set in the mutex word, and similarly for condition variables and kCVEvent. +static SynchEvent *EnsureSynchEvent(std::atomic<intptr_t> *addr, + const char *name, intptr_t bits, + intptr_t lockbit) { + uint32_t h = reinterpret_cast<intptr_t>(addr) % kNSynchEvent; + SynchEvent *e; + // first look for existing SynchEvent struct.. + synch_event_mu.Lock(); + for (e = synch_event[h]; + e != nullptr && e->masked_addr != base_internal::HidePtr(addr); + e = e->next) { + } + if (e == nullptr) { // no SynchEvent struct found; make one. + if (name == nullptr) { + name = ""; + } + size_t l = strlen(name); + e = reinterpret_cast<SynchEvent *>( + base_internal::LowLevelAlloc::Alloc(sizeof(*e) + l)); + e->refcount = 2; // one for return value, one for linked list + e->masked_addr = base_internal::HidePtr(addr); + e->invariant = nullptr; + e->arg = nullptr; + e->log = false; + strcpy(e->name, name); // NOLINT(runtime/printf) + e->next = synch_event[h]; + AtomicSetBits(addr, bits, lockbit); + synch_event[h] = e; + } else { + e->refcount++; // for return value + } + synch_event_mu.Unlock(); + return e; +} + +// Deallocate the SynchEvent *e, whose refcount has fallen to zero. +static void DeleteSynchEvent(SynchEvent *e) { + base_internal::LowLevelAlloc::Free(e); +} + +// Decrement the reference count of *e, or do nothing if e==null. +static void UnrefSynchEvent(SynchEvent *e) { + if (e != nullptr) { + synch_event_mu.Lock(); + bool del = (--(e->refcount) == 0); + synch_event_mu.Unlock(); + if (del) { + DeleteSynchEvent(e); + } + } +} + +// Forget the mapping from the object (Mutex or CondVar) at address addr +// to SynchEvent object, and clear "bits" in its word (waiting until lockbit +// is clear before doing so). +static void ForgetSynchEvent(std::atomic<intptr_t> *addr, intptr_t bits, + intptr_t lockbit) { + uint32_t h = reinterpret_cast<intptr_t>(addr) % kNSynchEvent; + SynchEvent **pe; + SynchEvent *e; + synch_event_mu.Lock(); + for (pe = &synch_event[h]; + (e = *pe) != nullptr && e->masked_addr != base_internal::HidePtr(addr); + pe = &e->next) { + } + bool del = false; + if (e != nullptr) { + *pe = e->next; + del = (--(e->refcount) == 0); + } + AtomicClearBits(addr, bits, lockbit); + synch_event_mu.Unlock(); + if (del) { + DeleteSynchEvent(e); + } +} + +// Return a refcounted reference to the SynchEvent of the object at address +// "addr", if any. The pointer returned is valid until the UnrefSynchEvent() is +// called. +static SynchEvent *GetSynchEvent(const void *addr) { + uint32_t h = reinterpret_cast<intptr_t>(addr) % kNSynchEvent; + SynchEvent *e; + synch_event_mu.Lock(); + for (e = synch_event[h]; + e != nullptr && e->masked_addr != base_internal::HidePtr(addr); + e = e->next) { + } + if (e != nullptr) { + e->refcount++; + } + synch_event_mu.Unlock(); + return e; +} + +// Called when an event "ev" occurs on a Mutex of CondVar "obj" +// if event recording is on +static void PostSynchEvent(void *obj, int ev) { + SynchEvent *e = GetSynchEvent(obj); + // logging is on if event recording is on and either there's no event struct, + // or it explicitly says to log + if (e == nullptr || e->log) { + void *pcs[40]; + int n = absl::GetStackTrace(pcs, ABSL_ARRAYSIZE(pcs), 1); + // A buffer with enough space for the ASCII for all the PCs, even on a + // 64-bit machine. + char buffer[ABSL_ARRAYSIZE(pcs) * 24]; + int pos = snprintf(buffer, sizeof (buffer), " @"); + for (int i = 0; i != n; i++) { + pos += snprintf(&buffer[pos], sizeof (buffer) - pos, " %p", pcs[i]); + } + ABSL_RAW_LOG(INFO, "%s%p %s %s", event_properties[ev].msg, obj, + (e == nullptr ? "" : e->name), buffer); + } + const int flags = event_properties[ev].flags; + if ((flags & SYNCH_F_LCK) != 0 && e != nullptr && e->invariant != nullptr) { + // Calling the invariant as is causes problems under ThreadSanitizer. + // We are currently inside of Mutex Lock/Unlock and are ignoring all + // memory accesses and synchronization. If the invariant transitively + // synchronizes something else and we ignore the synchronization, we will + // get false positive race reports later. + // Reuse EvalConditionAnnotated to properly call into user code. + struct local { + static bool pred(SynchEvent *ev) { + (*ev->invariant)(ev->arg); + return false; + } + }; + Condition cond(&local::pred, e); + Mutex *mu = static_cast<Mutex *>(obj); + const bool locking = (flags & SYNCH_F_UNLOCK) == 0; + const bool trylock = (flags & SYNCH_F_TRY) != 0; + const bool read_lock = (flags & SYNCH_F_R) != 0; + EvalConditionAnnotated(&cond, mu, locking, trylock, read_lock); + } + UnrefSynchEvent(e); +} + +//------------------------------------------------------------------ + +// The SynchWaitParams struct encapsulates the way in which a thread is waiting: +// whether it has a timeout, the condition, exclusive/shared, and whether a +// condition variable wait has an associated Mutex (as opposed to another +// type of lock). It also points to the PerThreadSynch struct of its thread. +// cv_word tells Enqueue() to enqueue on a CondVar using CondVarEnqueue(). +// +// This structure is held on the stack rather than directly in +// PerThreadSynch because a thread can be waiting on multiple Mutexes if, +// while waiting on one Mutex, the implementation calls a client callback +// (such as a Condition function) that acquires another Mutex. We don't +// strictly need to allow this, but programmers become confused if we do not +// allow them to use functions such a LOG() within Condition functions. The +// PerThreadSynch struct points at the most recent SynchWaitParams struct when +// the thread is on a Mutex's waiter queue. +struct SynchWaitParams { + SynchWaitParams(Mutex::MuHow how_arg, const Condition *cond_arg, + KernelTimeout timeout_arg, Mutex *cvmu_arg, + PerThreadSynch *thread_arg, + std::atomic<intptr_t> *cv_word_arg) + : how(how_arg), + cond(cond_arg), + timeout(timeout_arg), + cvmu(cvmu_arg), + thread(thread_arg), + cv_word(cv_word_arg), + contention_start_cycles(base_internal::CycleClock::Now()) {} + + const Mutex::MuHow how; // How this thread needs to wait. + const Condition *cond; // The condition that this thread is waiting for. + // In Mutex, this field is set to zero if a timeout + // expires. + KernelTimeout timeout; // timeout expiry---absolute time + // In Mutex, this field is set to zero if a timeout + // expires. + Mutex *const cvmu; // used for transfer from cond var to mutex + PerThreadSynch *const thread; // thread that is waiting + + // If not null, thread should be enqueued on the CondVar whose state + // word is cv_word instead of queueing normally on the Mutex. + std::atomic<intptr_t> *cv_word; + + int64_t contention_start_cycles; // Time (in cycles) when this thread started + // to contend for the mutex. +}; + +struct SynchLocksHeld { + int n; // number of valid entries in locks[] + bool overflow; // true iff we overflowed the array at some point + struct { + Mutex *mu; // lock acquired + int32_t count; // times acquired + GraphId id; // deadlock_graph id of acquired lock + } locks[40]; + // If a thread overfills the array during deadlock detection, we + // continue, discarding information as needed. If no overflow has + // taken place, we can provide more error checking, such as + // detecting when a thread releases a lock it does not hold. +}; + +// A sentinel value in lists that is not 0. +// A 0 value is used to mean "not on a list". +static PerThreadSynch *const kPerThreadSynchNull = + reinterpret_cast<PerThreadSynch *>(1); + +static SynchLocksHeld *LocksHeldAlloc() { + SynchLocksHeld *ret = reinterpret_cast<SynchLocksHeld *>( + base_internal::LowLevelAlloc::Alloc(sizeof(SynchLocksHeld))); + ret->n = 0; + ret->overflow = false; + return ret; +} + +// Return the PerThreadSynch-struct for this thread. +static PerThreadSynch *Synch_GetPerThread() { + ThreadIdentity *identity = GetOrCreateCurrentThreadIdentity(); + return &identity->per_thread_synch; +} + +static PerThreadSynch *Synch_GetPerThreadAnnotated(Mutex *mu) { + if (mu) { + ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); + } + PerThreadSynch *w = Synch_GetPerThread(); + if (mu) { + ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); + } + return w; +} + +static SynchLocksHeld *Synch_GetAllLocks() { + PerThreadSynch *s = Synch_GetPerThread(); + if (s->all_locks == nullptr) { + s->all_locks = LocksHeldAlloc(); // Freed by ReclaimThreadIdentity. + } + return s->all_locks; +} + +// Post on "w"'s associated PerThreadSem. +inline void Mutex::IncrementSynchSem(Mutex *mu, PerThreadSynch *w) { + if (mu) { + ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); + } + PerThreadSem::Post(w->thread_identity()); + if (mu) { + ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); + } +} + +// Wait on "w"'s associated PerThreadSem; returns false if timeout expired. +bool Mutex::DecrementSynchSem(Mutex *mu, PerThreadSynch *w, KernelTimeout t) { + if (mu) { + ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); + } + assert(w == Synch_GetPerThread()); + static_cast<void>(w); + bool res = PerThreadSem::Wait(t); + if (mu) { + ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); + } + return res; +} + +// We're in a fatal signal handler that hopes to use Mutex and to get +// lucky by not deadlocking. We try to improve its chances of success +// by effectively disabling some of the consistency checks. This will +// prevent certain ABSL_RAW_CHECK() statements from being triggered when +// re-rentry is detected. The ABSL_RAW_CHECK() statements are those in the +// Mutex code checking that the "waitp" field has not been reused. +void Mutex::InternalAttemptToUseMutexInFatalSignalHandler() { + // Fix the per-thread state only if it exists. + ThreadIdentity *identity = CurrentThreadIdentityIfPresent(); + if (identity != nullptr) { + identity->per_thread_synch.suppress_fatal_errors = true; + } + // Don't do deadlock detection when we are already failing. + synch_deadlock_detection.store(OnDeadlockCycle::kIgnore, + std::memory_order_release); +} + +// --------------------------time support + +// Return the current time plus the timeout. Use the same clock as +// PerThreadSem::Wait() for consistency. Unfortunately, we don't have +// such a choice when a deadline is given directly. +static absl::Time DeadlineFromTimeout(absl::Duration timeout) { +#ifndef _WIN32 + struct timeval tv; + gettimeofday(&tv, nullptr); + return absl::TimeFromTimeval(tv) + timeout; +#else + return absl::Now() + timeout; +#endif +} + +// --------------------------Mutexes + +// In the layout below, the msb of the bottom byte is currently unused. Also, +// the following constraints were considered in choosing the layout: +// o Both the debug allocator's "uninitialized" and "freed" patterns (0xab and +// 0xcd) are illegal: reader and writer lock both held. +// o kMuWriter and kMuEvent should exceed kMuDesig and kMuWait, to enable the +// bit-twiddling trick in Mutex::Unlock(). +// o kMuWriter / kMuReader == kMuWrWait / kMuWait, +// to enable the bit-twiddling trick in CheckForMutexCorruption(). +static const intptr_t kMuReader = 0x0001L; // a reader holds the lock +static const intptr_t kMuDesig = 0x0002L; // there's a designated waker +static const intptr_t kMuWait = 0x0004L; // threads are waiting +static const intptr_t kMuWriter = 0x0008L; // a writer holds the lock +static const intptr_t kMuEvent = 0x0010L; // record this mutex's events +// INVARIANT1: there's a thread that was blocked on the mutex, is +// no longer, yet has not yet acquired the mutex. If there's a +// designated waker, all threads can avoid taking the slow path in +// unlock because the designated waker will subsequently acquire +// the lock and wake someone. To maintain INVARIANT1 the bit is +// set when a thread is unblocked(INV1a), and threads that were +// unblocked reset the bit when they either acquire or re-block +// (INV1b). +static const intptr_t kMuWrWait = 0x0020L; // runnable writer is waiting + // for a reader +static const intptr_t kMuSpin = 0x0040L; // spinlock protects wait list +static const intptr_t kMuLow = 0x00ffL; // mask all mutex bits +static const intptr_t kMuHigh = ~kMuLow; // mask pointer/reader count + +// Hack to make constant values available to gdb pretty printer +enum { + kGdbMuSpin = kMuSpin, + kGdbMuEvent = kMuEvent, + kGdbMuWait = kMuWait, + kGdbMuWriter = kMuWriter, + kGdbMuDesig = kMuDesig, + kGdbMuWrWait = kMuWrWait, + kGdbMuReader = kMuReader, + kGdbMuLow = kMuLow, +}; + +// kMuWrWait implies kMuWait. +// kMuReader and kMuWriter are mutually exclusive. +// If kMuReader is zero, there are no readers. +// Otherwise, if kMuWait is zero, the high order bits contain a count of the +// number of readers. Otherwise, the reader count is held in +// PerThreadSynch::readers of the most recently queued waiter, again in the +// bits above kMuLow. +static const intptr_t kMuOne = 0x0100; // a count of one reader + +// flags passed to Enqueue and LockSlow{,WithTimeout,Loop} +static const int kMuHasBlocked = 0x01; // already blocked (MUST == 1) +static const int kMuIsCond = 0x02; // conditional waiter (CV or Condition) + +static_assert(PerThreadSynch::kAlignment > kMuLow, + "PerThreadSynch::kAlignment must be greater than kMuLow"); + +// This struct contains various bitmasks to be used in +// acquiring and releasing a mutex in a particular mode. +struct MuHowS { + // if all the bits in fast_need_zero are zero, the lock can be acquired by + // adding fast_add and oring fast_or. The bit kMuDesig should be reset iff + // this is the designated waker. + intptr_t fast_need_zero; + intptr_t fast_or; + intptr_t fast_add; + + intptr_t slow_need_zero; // fast_need_zero with events (e.g. logging) + + intptr_t slow_inc_need_zero; // if all the bits in slow_inc_need_zero are + // zero a reader can acquire a read share by + // setting the reader bit and incrementing + // the reader count (in last waiter since + // we're now slow-path). kMuWrWait be may + // be ignored if we already waited once. +}; + +static const MuHowS kSharedS = { + // shared or read lock + kMuWriter | kMuWait | kMuEvent, // fast_need_zero + kMuReader, // fast_or + kMuOne, // fast_add + kMuWriter | kMuWait, // slow_need_zero + kMuSpin | kMuWriter | kMuWrWait, // slow_inc_need_zero +}; +static const MuHowS kExclusiveS = { + // exclusive or write lock + kMuWriter | kMuReader | kMuEvent, // fast_need_zero + kMuWriter, // fast_or + 0, // fast_add + kMuWriter | kMuReader, // slow_need_zero + ~static_cast<intptr_t>(0), // slow_inc_need_zero +}; +static const Mutex::MuHow kShared = &kSharedS; // shared lock +static const Mutex::MuHow kExclusive = &kExclusiveS; // exclusive lock + +#ifdef NDEBUG +static constexpr bool kDebugMode = false; +#else +static constexpr bool kDebugMode = true; +#endif + +#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE +static unsigned TsanFlags(Mutex::MuHow how) { + return how == kShared ? __tsan_mutex_read_lock : 0; +} +#endif + +static bool DebugOnlyIsExiting() { + return false; +} + +Mutex::~Mutex() { + intptr_t v = mu_.load(std::memory_order_relaxed); + if ((v & kMuEvent) != 0 && !DebugOnlyIsExiting()) { + ForgetSynchEvent(&this->mu_, kMuEvent, kMuSpin); + } + if (kDebugMode) { + this->ForgetDeadlockInfo(); + } + ABSL_TSAN_MUTEX_DESTROY(this, __tsan_mutex_not_static); +} + +void Mutex::EnableDebugLog(const char *name) { + SynchEvent *e = EnsureSynchEvent(&this->mu_, name, kMuEvent, kMuSpin); + e->log = true; + UnrefSynchEvent(e); +} + +void EnableMutexInvariantDebugging(bool enabled) { + synch_check_invariants.store(enabled, std::memory_order_release); +} + +void Mutex::EnableInvariantDebugging(void (*invariant)(void *), + void *arg) { + if (synch_check_invariants.load(std::memory_order_acquire) && + invariant != nullptr) { + SynchEvent *e = EnsureSynchEvent(&this->mu_, nullptr, kMuEvent, kMuSpin); + e->invariant = invariant; + e->arg = arg; + UnrefSynchEvent(e); + } +} + +void SetMutexDeadlockDetectionMode(OnDeadlockCycle mode) { + synch_deadlock_detection.store(mode, std::memory_order_release); +} + +// Return true iff threads x and y are waiting on the same condition for the +// same type of lock. Requires that x and y be waiting on the same Mutex +// queue. +static bool MuSameCondition(PerThreadSynch *x, PerThreadSynch *y) { + return x->waitp->how == y->waitp->how && + Condition::GuaranteedEqual(x->waitp->cond, y->waitp->cond); +} + +// Given the contents of a mutex word containing a PerThreadSynch pointer, +// return the pointer. +static inline PerThreadSynch *GetPerThreadSynch(intptr_t v) { + return reinterpret_cast<PerThreadSynch *>(v & kMuHigh); +} + +// The next several routines maintain the per-thread next and skip fields +// used in the Mutex waiter queue. +// The queue is a circular singly-linked list, of which the "head" is the +// last element, and head->next if the first element. +// The skip field has the invariant: +// For thread x, x->skip is one of: +// - invalid (iff x is not in a Mutex wait queue), +// - null, or +// - a pointer to a distinct thread waiting later in the same Mutex queue +// such that all threads in [x, x->skip] have the same condition and +// lock type (MuSameCondition() is true for all pairs in [x, x->skip]). +// In addition, if x->skip is valid, (x->may_skip || x->skip == null) +// +// By the spec of MuSameCondition(), it is not necessary when removing the +// first runnable thread y from the front a Mutex queue to adjust the skip +// field of another thread x because if x->skip==y, x->skip must (have) become +// invalid before y is removed. The function TryRemove can remove a specified +// thread from an arbitrary position in the queue whether runnable or not, so +// it fixes up skip fields that would otherwise be left dangling. +// The statement +// if (x->may_skip && MuSameCondition(x, x->next)) { x->skip = x->next; } +// maintains the invariant provided x is not the last waiter in a Mutex queue +// The statement +// if (x->skip != null) { x->skip = x->skip->skip; } +// maintains the invariant. + +// Returns the last thread y in a mutex waiter queue such that all threads in +// [x, y] inclusive share the same condition. Sets skip fields of some threads +// in that range to optimize future evaluation of Skip() on x values in +// the range. Requires thread x is in a mutex waiter queue. +// The locking is unusual. Skip() is called under these conditions: +// - spinlock is held in call from Enqueue(), with maybe_unlocking == false +// - Mutex is held in call from UnlockSlow() by last unlocker, with +// maybe_unlocking == true +// - both Mutex and spinlock are held in call from DequeueAllWakeable() (from +// UnlockSlow()) and TryRemove() +// These cases are mutually exclusive, so Skip() never runs concurrently +// with itself on the same Mutex. The skip chain is used in these other places +// that cannot occur concurrently: +// - FixSkip() (from TryRemove()) - spinlock and Mutex are held) +// - Dequeue() (with spinlock and Mutex held) +// - UnlockSlow() (with spinlock and Mutex held) +// A more complex case is Enqueue() +// - Enqueue() (with spinlock held and maybe_unlocking == false) +// This is the first case in which Skip is called, above. +// - Enqueue() (without spinlock held; but queue is empty and being freshly +// formed) +// - Enqueue() (with spinlock held and maybe_unlocking == true) +// The first case has mutual exclusion, and the second isolation through +// working on an otherwise unreachable data structure. +// In the last case, Enqueue() is required to change no skip/next pointers +// except those in the added node and the former "head" node. This implies +// that the new node is added after head, and so must be the new head or the +// new front of the queue. +static PerThreadSynch *Skip(PerThreadSynch *x) { + PerThreadSynch *x0 = nullptr; + PerThreadSynch *x1 = x; + PerThreadSynch *x2 = x->skip; + if (x2 != nullptr) { + // Each iteration attempts to advance sequence (x0,x1,x2) to next sequence + // such that x1 == x0->skip && x2 == x1->skip + while ((x0 = x1, x1 = x2, x2 = x2->skip) != nullptr) { + x0->skip = x2; // short-circuit skip from x0 to x2 + } + x->skip = x1; // short-circuit skip from x to result + } + return x1; +} + +// "ancestor" appears before "to_be_removed" in the same Mutex waiter queue. +// The latter is going to be removed out of order, because of a timeout. +// Check whether "ancestor" has a skip field pointing to "to_be_removed", +// and fix it if it does. +static void FixSkip(PerThreadSynch *ancestor, PerThreadSynch *to_be_removed) { + if (ancestor->skip == to_be_removed) { // ancestor->skip left dangling + if (to_be_removed->skip != nullptr) { + ancestor->skip = to_be_removed->skip; // can skip past to_be_removed + } else if (ancestor->next != to_be_removed) { // they are not adjacent + ancestor->skip = ancestor->next; // can skip one past ancestor + } else { + ancestor->skip = nullptr; // can't skip at all + } + } +} + +static void CondVarEnqueue(SynchWaitParams *waitp); + +// Enqueue thread "waitp->thread" on a waiter queue. +// Called with mutex spinlock held if head != nullptr +// If head==nullptr and waitp->cv_word==nullptr, then Enqueue() is +// idempotent; it alters no state associated with the existing (empty) +// queue. +// +// If waitp->cv_word == nullptr, queue the thread at either the front or +// the end (according to its priority) of the circular mutex waiter queue whose +// head is "head", and return the new head. mu is the previous mutex state, +// which contains the reader count (perhaps adjusted for the operation in +// progress) if the list was empty and a read lock held, and the holder hint if +// the list was empty and a write lock held. (flags & kMuIsCond) indicates +// whether this thread was transferred from a CondVar or is waiting for a +// non-trivial condition. In this case, Enqueue() never returns nullptr +// +// If waitp->cv_word != nullptr, CondVarEnqueue() is called, and "head" is +// returned. This mechanism is used by CondVar to queue a thread on the +// condition variable queue instead of the mutex queue in implementing Wait(). +// In this case, Enqueue() can return nullptr (if head==nullptr). +static PerThreadSynch *Enqueue(PerThreadSynch *head, + SynchWaitParams *waitp, intptr_t mu, int flags) { + // If we have been given a cv_word, call CondVarEnqueue() and return + // the previous head of the Mutex waiter queue. + if (waitp->cv_word != nullptr) { + CondVarEnqueue(waitp); + return head; + } + + PerThreadSynch *s = waitp->thread; + ABSL_RAW_CHECK( + s->waitp == nullptr || // normal case + s->waitp == waitp || // Fer()---transfer from condition variable + s->suppress_fatal_errors, + "detected illegal recursion into Mutex code"); + s->waitp = waitp; + s->skip = nullptr; // maintain skip invariant (see above) + s->may_skip = true; // always true on entering queue + s->wake = false; // not being woken + s->cond_waiter = ((flags & kMuIsCond) != 0); + if (head == nullptr) { // s is the only waiter + s->next = s; // it's the only entry in the cycle + s->readers = mu; // reader count is from mu word + s->maybe_unlocking = false; // no one is searching an empty list + head = s; // s is new head + } else { + PerThreadSynch *enqueue_after = nullptr; // we'll put s after this element +#ifdef ABSL_HAVE_PTHREAD_GETSCHEDPARAM + int64_t now_cycles = base_internal::CycleClock::Now(); + if (s->next_priority_read_cycles < now_cycles) { + // Every so often, update our idea of the thread's priority. + // pthread_getschedparam() is 5% of the block/wakeup time; + // base_internal::CycleClock::Now() is 0.5%. + int policy; + struct sched_param param; + const int err = pthread_getschedparam(pthread_self(), &policy, ¶m); + if (err != 0) { + ABSL_RAW_LOG(ERROR, "pthread_getschedparam failed: %d", err); + } else { + s->priority = param.sched_priority; + s->next_priority_read_cycles = + now_cycles + + static_cast<int64_t>(base_internal::CycleClock::Frequency()); + } + } + if (s->priority > head->priority) { // s's priority is above head's + // try to put s in priority-fifo order, or failing that at the front. + if (!head->maybe_unlocking) { + // No unlocker can be scanning the queue, so we can insert between + // skip-chains, and within a skip-chain if it has the same condition as + // s. We insert in priority-fifo order, examining the end of every + // skip-chain, plus every element with the same condition as s. + PerThreadSynch *advance_to = head; // next value of enqueue_after + PerThreadSynch *cur; // successor of enqueue_after + do { + enqueue_after = advance_to; + cur = enqueue_after->next; // this advance ensures progress + advance_to = Skip(cur); // normally, advance to end of skip chain + // (side-effect: optimizes skip chain) + if (advance_to != cur && s->priority > advance_to->priority && + MuSameCondition(s, cur)) { + // but this skip chain is not a singleton, s has higher priority + // than its tail and has the same condition as the chain, + // so we can insert within the skip-chain + advance_to = cur; // advance by just one + } + } while (s->priority <= advance_to->priority); + // termination guaranteed because s->priority > head->priority + // and head is the end of a skip chain + } else if (waitp->how == kExclusive && + Condition::GuaranteedEqual(waitp->cond, nullptr)) { + // An unlocker could be scanning the queue, but we know it will recheck + // the queue front for writers that have no condition, which is what s + // is, so an insert at front is safe. + enqueue_after = head; // add after head, at front + } + } +#endif + if (enqueue_after != nullptr) { + s->next = enqueue_after->next; + enqueue_after->next = s; + + // enqueue_after can be: head, Skip(...), or cur. + // The first two imply enqueue_after->skip == nullptr, and + // the last is used only if MuSameCondition(s, cur). + // We require this because clearing enqueue_after->skip + // is impossible; enqueue_after's predecessors might also + // incorrectly skip over s if we were to allow other + // insertion points. + ABSL_RAW_CHECK( + enqueue_after->skip == nullptr || MuSameCondition(enqueue_after, s), + "Mutex Enqueue failure"); + + if (enqueue_after != head && enqueue_after->may_skip && + MuSameCondition(enqueue_after, enqueue_after->next)) { + // enqueue_after can skip to its new successor, s + enqueue_after->skip = enqueue_after->next; + } + if (MuSameCondition(s, s->next)) { // s->may_skip is known to be true + s->skip = s->next; // s may skip to its successor + } + } else { // enqueue not done any other way, so + // we're inserting s at the back + // s will become new head; copy data from head into it + s->next = head->next; // add s after head + head->next = s; + s->readers = head->readers; // reader count is from previous head + s->maybe_unlocking = head->maybe_unlocking; // same for unlock hint + if (head->may_skip && MuSameCondition(head, s)) { + // head now has successor; may skip + head->skip = s; + } + head = s; // s is new head + } + } + s->state.store(PerThreadSynch::kQueued, std::memory_order_relaxed); + return head; +} + +// Dequeue the successor pw->next of thread pw from the Mutex waiter queue +// whose last element is head. The new head element is returned, or null +// if the list is made empty. +// Dequeue is called with both spinlock and Mutex held. +static PerThreadSynch *Dequeue(PerThreadSynch *head, PerThreadSynch *pw) { + PerThreadSynch *w = pw->next; + pw->next = w->next; // snip w out of list + if (head == w) { // we removed the head + head = (pw == w) ? nullptr : pw; // either emptied list, or pw is new head + } else if (pw != head && MuSameCondition(pw, pw->next)) { + // pw can skip to its new successor + if (pw->next->skip != + nullptr) { // either skip to its successors skip target + pw->skip = pw->next->skip; + } else { // or to pw's successor + pw->skip = pw->next; + } + } + return head; +} + +// Traverse the elements [ pw->next, h] of the circular list whose last element +// is head. +// Remove all elements with wake==true and place them in the +// singly-linked list wake_list in the order found. Assumes that +// there is only one such element if the element has how == kExclusive. +// Return the new head. +static PerThreadSynch *DequeueAllWakeable(PerThreadSynch *head, + PerThreadSynch *pw, + PerThreadSynch **wake_tail) { + PerThreadSynch *orig_h = head; + PerThreadSynch *w = pw->next; + bool skipped = false; + do { + if (w->wake) { // remove this element + ABSL_RAW_CHECK(pw->skip == nullptr, "bad skip in DequeueAllWakeable"); + // we're removing pw's successor so either pw->skip is zero or we should + // already have removed pw since if pw->skip!=null, pw has the same + // condition as w. + head = Dequeue(head, pw); + w->next = *wake_tail; // keep list terminated + *wake_tail = w; // add w to wake_list; + wake_tail = &w->next; // next addition to end + if (w->waitp->how == kExclusive) { // wake at most 1 writer + break; + } + } else { // not waking this one; skip + pw = Skip(w); // skip as much as possible + skipped = true; + } + w = pw->next; + // We want to stop processing after we've considered the original head, + // orig_h. We can't test for w==orig_h in the loop because w may skip over + // it; we are guaranteed only that w's predecessor will not skip over + // orig_h. When we've considered orig_h, either we've processed it and + // removed it (so orig_h != head), or we considered it and skipped it (so + // skipped==true && pw == head because skipping from head always skips by + // just one, leaving pw pointing at head). So we want to + // continue the loop with the negation of that expression. + } while (orig_h == head && (pw != head || !skipped)); + return head; +} + +// Try to remove thread s from the list of waiters on this mutex. +// Does nothing if s is not on the waiter list. +void Mutex::TryRemove(PerThreadSynch *s) { + SchedulingGuard::ScopedDisable disable_rescheduling; + intptr_t v = mu_.load(std::memory_order_relaxed); + // acquire spinlock & lock + if ((v & (kMuWait | kMuSpin | kMuWriter | kMuReader)) == kMuWait && + mu_.compare_exchange_strong(v, v | kMuSpin | kMuWriter, + std::memory_order_acquire, + std::memory_order_relaxed)) { + PerThreadSynch *h = GetPerThreadSynch(v); + if (h != nullptr) { + PerThreadSynch *pw = h; // pw is w's predecessor + PerThreadSynch *w; + if ((w = pw->next) != s) { // search for thread, + do { // processing at least one element + if (!MuSameCondition(s, w)) { // seeking different condition + pw = Skip(w); // so skip all that won't match + // we don't have to worry about dangling skip fields + // in the threads we skipped; none can point to s + // because their condition differs from s + } else { // seeking same condition + FixSkip(w, s); // fix up any skip pointer from w to s + pw = w; + } + // don't search further if we found the thread, or we're about to + // process the first thread again. + } while ((w = pw->next) != s && pw != h); + } + if (w == s) { // found thread; remove it + // pw->skip may be non-zero here; the loop above ensured that + // no ancestor of s can skip to s, so removal is safe anyway. + h = Dequeue(h, pw); + s->next = nullptr; + s->state.store(PerThreadSynch::kAvailable, std::memory_order_release); + } + } + intptr_t nv; + do { // release spinlock and lock + v = mu_.load(std::memory_order_relaxed); + nv = v & (kMuDesig | kMuEvent); + if (h != nullptr) { + nv |= kMuWait | reinterpret_cast<intptr_t>(h); + h->readers = 0; // we hold writer lock + h->maybe_unlocking = false; // finished unlocking + } + } while (!mu_.compare_exchange_weak(v, nv, + std::memory_order_release, + std::memory_order_relaxed)); + } +} + +// Wait until thread "s", which must be the current thread, is removed from the +// this mutex's waiter queue. If "s->waitp->timeout" has a timeout, wake up +// if the wait extends past the absolute time specified, even if "s" is still +// on the mutex queue. In this case, remove "s" from the queue and return +// true, otherwise return false. +ABSL_XRAY_LOG_ARGS(1) void Mutex::Block(PerThreadSynch *s) { + while (s->state.load(std::memory_order_acquire) == PerThreadSynch::kQueued) { + if (!DecrementSynchSem(this, s, s->waitp->timeout)) { + // After a timeout, we go into a spin loop until we remove ourselves + // from the queue, or someone else removes us. We can't be sure to be + // able to remove ourselves in a single lock acquisition because this + // mutex may be held, and the holder has the right to read the centre + // of the waiter queue without holding the spinlock. + this->TryRemove(s); + int c = 0; + while (s->next != nullptr) { + c = synchronization_internal::MutexDelay(c, GENTLE); + this->TryRemove(s); + } + if (kDebugMode) { + // This ensures that we test the case that TryRemove() is called when s + // is not on the queue. + this->TryRemove(s); + } + s->waitp->timeout = KernelTimeout::Never(); // timeout is satisfied + s->waitp->cond = nullptr; // condition no longer relevant for wakeups + } + } + ABSL_RAW_CHECK(s->waitp != nullptr || s->suppress_fatal_errors, + "detected illegal recursion in Mutex code"); + s->waitp = nullptr; +} + +// Wake thread w, and return the next thread in the list. +PerThreadSynch *Mutex::Wakeup(PerThreadSynch *w) { + PerThreadSynch *next = w->next; + w->next = nullptr; + w->state.store(PerThreadSynch::kAvailable, std::memory_order_release); + IncrementSynchSem(this, w); + + return next; +} + +static GraphId GetGraphIdLocked(Mutex *mu) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(deadlock_graph_mu) { + if (!deadlock_graph) { // (re)create the deadlock graph. + deadlock_graph = + new (base_internal::LowLevelAlloc::Alloc(sizeof(*deadlock_graph))) + GraphCycles; + } + return deadlock_graph->GetId(mu); +} + +static GraphId GetGraphId(Mutex *mu) ABSL_LOCKS_EXCLUDED(deadlock_graph_mu) { + deadlock_graph_mu.Lock(); + GraphId id = GetGraphIdLocked(mu); + deadlock_graph_mu.Unlock(); + return id; +} + +// Record a lock acquisition. This is used in debug mode for deadlock +// detection. The held_locks pointer points to the relevant data +// structure for each case. +static void LockEnter(Mutex* mu, GraphId id, SynchLocksHeld *held_locks) { + int n = held_locks->n; + int i = 0; + while (i != n && held_locks->locks[i].id != id) { + i++; + } + if (i == n) { + if (n == ABSL_ARRAYSIZE(held_locks->locks)) { + held_locks->overflow = true; // lost some data + } else { // we have room for lock + held_locks->locks[i].mu = mu; + held_locks->locks[i].count = 1; + held_locks->locks[i].id = id; + held_locks->n = n + 1; + } + } else { + held_locks->locks[i].count++; + } +} + +// Record a lock release. Each call to LockEnter(mu, id, x) should be +// eventually followed by a call to LockLeave(mu, id, x) by the same thread. +// It does not process the event if is not needed when deadlock detection is +// disabled. +static void LockLeave(Mutex* mu, GraphId id, SynchLocksHeld *held_locks) { + int n = held_locks->n; + int i = 0; + while (i != n && held_locks->locks[i].id != id) { + i++; + } + if (i == n) { + if (!held_locks->overflow) { + // The deadlock id may have been reassigned after ForgetDeadlockInfo, + // but in that case mu should still be present. + i = 0; + while (i != n && held_locks->locks[i].mu != mu) { + i++; + } + if (i == n) { // mu missing means releasing unheld lock + SynchEvent *mu_events = GetSynchEvent(mu); + ABSL_RAW_LOG(FATAL, + "thread releasing lock it does not hold: %p %s; " + , + static_cast<void *>(mu), + mu_events == nullptr ? "" : mu_events->name); + } + } + } else if (held_locks->locks[i].count == 1) { + held_locks->n = n - 1; + held_locks->locks[i] = held_locks->locks[n - 1]; + held_locks->locks[n - 1].id = InvalidGraphId(); + held_locks->locks[n - 1].mu = + nullptr; // clear mu to please the leak detector. + } else { + assert(held_locks->locks[i].count > 0); + held_locks->locks[i].count--; + } +} + +// Call LockEnter() if in debug mode and deadlock detection is enabled. +static inline void DebugOnlyLockEnter(Mutex *mu) { + if (kDebugMode) { + if (synch_deadlock_detection.load(std::memory_order_acquire) != + OnDeadlockCycle::kIgnore) { + LockEnter(mu, GetGraphId(mu), Synch_GetAllLocks()); + } + } +} + +// Call LockEnter() if in debug mode and deadlock detection is enabled. +static inline void DebugOnlyLockEnter(Mutex *mu, GraphId id) { + if (kDebugMode) { + if (synch_deadlock_detection.load(std::memory_order_acquire) != + OnDeadlockCycle::kIgnore) { + LockEnter(mu, id, Synch_GetAllLocks()); + } + } +} + +// Call LockLeave() if in debug mode and deadlock detection is enabled. +static inline void DebugOnlyLockLeave(Mutex *mu) { + if (kDebugMode) { + if (synch_deadlock_detection.load(std::memory_order_acquire) != + OnDeadlockCycle::kIgnore) { + LockLeave(mu, GetGraphId(mu), Synch_GetAllLocks()); + } + } +} + +static char *StackString(void **pcs, int n, char *buf, int maxlen, + bool symbolize) { + static const int kSymLen = 200; + char sym[kSymLen]; + int len = 0; + for (int i = 0; i != n; i++) { + if (symbolize) { + if (!symbolizer(pcs[i], sym, kSymLen)) { + sym[0] = '\0'; + } + snprintf(buf + len, maxlen - len, "%s\t@ %p %s\n", + (i == 0 ? "\n" : ""), + pcs[i], sym); + } else { + snprintf(buf + len, maxlen - len, " %p", pcs[i]); + } + len += strlen(&buf[len]); + } + return buf; +} + +static char *CurrentStackString(char *buf, int maxlen, bool symbolize) { + void *pcs[40]; + return StackString(pcs, absl::GetStackTrace(pcs, ABSL_ARRAYSIZE(pcs), 2), buf, + maxlen, symbolize); +} + +namespace { +enum { kMaxDeadlockPathLen = 10 }; // maximum length of a deadlock cycle; + // a path this long would be remarkable +// Buffers required to report a deadlock. +// We do not allocate them on stack to avoid large stack frame. +struct DeadlockReportBuffers { + char buf[6100]; + GraphId path[kMaxDeadlockPathLen]; +}; + +struct ScopedDeadlockReportBuffers { + ScopedDeadlockReportBuffers() { + b = reinterpret_cast<DeadlockReportBuffers *>( + base_internal::LowLevelAlloc::Alloc(sizeof(*b))); + } + ~ScopedDeadlockReportBuffers() { base_internal::LowLevelAlloc::Free(b); } + DeadlockReportBuffers *b; +}; + +// Helper to pass to GraphCycles::UpdateStackTrace. +int GetStack(void** stack, int max_depth) { + return absl::GetStackTrace(stack, max_depth, 3); +} +} // anonymous namespace + +// Called in debug mode when a thread is about to acquire a lock in a way that +// may block. +static GraphId DeadlockCheck(Mutex *mu) { + if (synch_deadlock_detection.load(std::memory_order_acquire) == + OnDeadlockCycle::kIgnore) { + return InvalidGraphId(); + } + + SynchLocksHeld *all_locks = Synch_GetAllLocks(); + + absl::base_internal::SpinLockHolder lock(&deadlock_graph_mu); + const GraphId mu_id = GetGraphIdLocked(mu); + + if (all_locks->n == 0) { + // There are no other locks held. Return now so that we don't need to + // call GetSynchEvent(). This way we do not record the stack trace + // for this Mutex. It's ok, since if this Mutex is involved in a deadlock, + // it can't always be the first lock acquired by a thread. + return mu_id; + } + + // We prefer to keep stack traces that show a thread holding and acquiring + // as many locks as possible. This increases the chances that a given edge + // in the acquires-before graph will be represented in the stack traces + // recorded for the locks. + deadlock_graph->UpdateStackTrace(mu_id, all_locks->n + 1, GetStack); + + // For each other mutex already held by this thread: + for (int i = 0; i != all_locks->n; i++) { + const GraphId other_node_id = all_locks->locks[i].id; + const Mutex *other = + static_cast<const Mutex *>(deadlock_graph->Ptr(other_node_id)); + if (other == nullptr) { + // Ignore stale lock + continue; + } + + // Add the acquired-before edge to the graph. + if (!deadlock_graph->InsertEdge(other_node_id, mu_id)) { + ScopedDeadlockReportBuffers scoped_buffers; + DeadlockReportBuffers *b = scoped_buffers.b; + static int number_of_reported_deadlocks = 0; + number_of_reported_deadlocks++; + // Symbolize only 2 first deadlock report to avoid huge slowdowns. + bool symbolize = number_of_reported_deadlocks <= 2; + ABSL_RAW_LOG(ERROR, "Potential Mutex deadlock: %s", + CurrentStackString(b->buf, sizeof (b->buf), symbolize)); + int len = 0; + for (int j = 0; j != all_locks->n; j++) { + void* pr = deadlock_graph->Ptr(all_locks->locks[j].id); + if (pr != nullptr) { + snprintf(b->buf + len, sizeof (b->buf) - len, " %p", pr); + len += static_cast<int>(strlen(&b->buf[len])); + } + } + ABSL_RAW_LOG(ERROR, "Acquiring %p Mutexes held: %s", + static_cast<void *>(mu), b->buf); + ABSL_RAW_LOG(ERROR, "Cycle: "); + int path_len = deadlock_graph->FindPath( + mu_id, other_node_id, ABSL_ARRAYSIZE(b->path), b->path); + for (int j = 0; j != path_len; j++) { + GraphId id = b->path[j]; + Mutex *path_mu = static_cast<Mutex *>(deadlock_graph->Ptr(id)); + if (path_mu == nullptr) continue; + void** stack; + int depth = deadlock_graph->GetStackTrace(id, &stack); + snprintf(b->buf, sizeof(b->buf), + "mutex@%p stack: ", static_cast<void *>(path_mu)); + StackString(stack, depth, b->buf + strlen(b->buf), + static_cast<int>(sizeof(b->buf) - strlen(b->buf)), + symbolize); + ABSL_RAW_LOG(ERROR, "%s", b->buf); + } + if (synch_deadlock_detection.load(std::memory_order_acquire) == + OnDeadlockCycle::kAbort) { + deadlock_graph_mu.Unlock(); // avoid deadlock in fatal sighandler + ABSL_RAW_LOG(FATAL, "dying due to potential deadlock"); + return mu_id; + } + break; // report at most one potential deadlock per acquisition + } + } + + return mu_id; +} + +// Invoke DeadlockCheck() iff we're in debug mode and +// deadlock checking has been enabled. +static inline GraphId DebugOnlyDeadlockCheck(Mutex *mu) { + if (kDebugMode && synch_deadlock_detection.load(std::memory_order_acquire) != + OnDeadlockCycle::kIgnore) { + return DeadlockCheck(mu); + } else { + return InvalidGraphId(); + } +} + +void Mutex::ForgetDeadlockInfo() { + if (kDebugMode && synch_deadlock_detection.load(std::memory_order_acquire) != + OnDeadlockCycle::kIgnore) { + deadlock_graph_mu.Lock(); + if (deadlock_graph != nullptr) { + deadlock_graph->RemoveNode(this); + } + deadlock_graph_mu.Unlock(); + } +} + +void Mutex::AssertNotHeld() const { + // We have the data to allow this check only if in debug mode and deadlock + // detection is enabled. + if (kDebugMode && + (mu_.load(std::memory_order_relaxed) & (kMuWriter | kMuReader)) != 0 && + synch_deadlock_detection.load(std::memory_order_acquire) != + OnDeadlockCycle::kIgnore) { + GraphId id = GetGraphId(const_cast<Mutex *>(this)); + SynchLocksHeld *locks = Synch_GetAllLocks(); + for (int i = 0; i != locks->n; i++) { + if (locks->locks[i].id == id) { + SynchEvent *mu_events = GetSynchEvent(this); + ABSL_RAW_LOG(FATAL, "thread should not hold mutex %p %s", + static_cast<const void *>(this), + (mu_events == nullptr ? "" : mu_events->name)); + } + } + } +} + +// Attempt to acquire *mu, and return whether successful. The implementation +// may spin for a short while if the lock cannot be acquired immediately. +static bool TryAcquireWithSpinning(std::atomic<intptr_t>* mu) { + int c = GetMutexGlobals().spinloop_iterations; + do { // do/while somewhat faster on AMD + intptr_t v = mu->load(std::memory_order_relaxed); + if ((v & (kMuReader|kMuEvent)) != 0) { + return false; // a reader or tracing -> give up + } else if (((v & kMuWriter) == 0) && // no holder -> try to acquire + mu->compare_exchange_strong(v, kMuWriter | v, + std::memory_order_acquire, + std::memory_order_relaxed)) { + return true; + } + } while (--c > 0); + return false; +} + +ABSL_XRAY_LOG_ARGS(1) void Mutex::Lock() { + ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); + GraphId id = DebugOnlyDeadlockCheck(this); + intptr_t v = mu_.load(std::memory_order_relaxed); + // try fast acquire, then spin loop + if ((v & (kMuWriter | kMuReader | kMuEvent)) != 0 || + !mu_.compare_exchange_strong(v, kMuWriter | v, + std::memory_order_acquire, + std::memory_order_relaxed)) { + // try spin acquire, then slow loop + if (!TryAcquireWithSpinning(&this->mu_)) { + this->LockSlow(kExclusive, nullptr, 0); + } + } + DebugOnlyLockEnter(this, id); + ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); +} + +ABSL_XRAY_LOG_ARGS(1) void Mutex::ReaderLock() { + ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); + GraphId id = DebugOnlyDeadlockCheck(this); + intptr_t v = mu_.load(std::memory_order_relaxed); + // try fast acquire, then slow loop + if ((v & (kMuWriter | kMuWait | kMuEvent)) != 0 || + !mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, + std::memory_order_acquire, + std::memory_order_relaxed)) { + this->LockSlow(kShared, nullptr, 0); + } + DebugOnlyLockEnter(this, id); + ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); +} + +void Mutex::LockWhen(const Condition &cond) { + ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); + GraphId id = DebugOnlyDeadlockCheck(this); + this->LockSlow(kExclusive, &cond, 0); + DebugOnlyLockEnter(this, id); + ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); +} + +bool Mutex::LockWhenWithTimeout(const Condition &cond, absl::Duration timeout) { + return LockWhenWithDeadline(cond, DeadlineFromTimeout(timeout)); +} + +bool Mutex::LockWhenWithDeadline(const Condition &cond, absl::Time deadline) { + ABSL_TSAN_MUTEX_PRE_LOCK(this, 0); + GraphId id = DebugOnlyDeadlockCheck(this); + bool res = LockSlowWithDeadline(kExclusive, &cond, + KernelTimeout(deadline), 0); + DebugOnlyLockEnter(this, id); + ABSL_TSAN_MUTEX_POST_LOCK(this, 0, 0); + return res; +} + +void Mutex::ReaderLockWhen(const Condition &cond) { + ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); + GraphId id = DebugOnlyDeadlockCheck(this); + this->LockSlow(kShared, &cond, 0); + DebugOnlyLockEnter(this, id); + ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); +} + +bool Mutex::ReaderLockWhenWithTimeout(const Condition &cond, + absl::Duration timeout) { + return ReaderLockWhenWithDeadline(cond, DeadlineFromTimeout(timeout)); +} + +bool Mutex::ReaderLockWhenWithDeadline(const Condition &cond, + absl::Time deadline) { + ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_read_lock); + GraphId id = DebugOnlyDeadlockCheck(this); + bool res = LockSlowWithDeadline(kShared, &cond, KernelTimeout(deadline), 0); + DebugOnlyLockEnter(this, id); + ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_read_lock, 0); + return res; +} + +void Mutex::Await(const Condition &cond) { + if (cond.Eval()) { // condition already true; nothing to do + if (kDebugMode) { + this->AssertReaderHeld(); + } + } else { // normal case + ABSL_RAW_CHECK(this->AwaitCommon(cond, KernelTimeout::Never()), + "condition untrue on return from Await"); + } +} + +bool Mutex::AwaitWithTimeout(const Condition &cond, absl::Duration timeout) { + return AwaitWithDeadline(cond, DeadlineFromTimeout(timeout)); +} + +bool Mutex::AwaitWithDeadline(const Condition &cond, absl::Time deadline) { + if (cond.Eval()) { // condition already true; nothing to do + if (kDebugMode) { + this->AssertReaderHeld(); + } + return true; + } + + KernelTimeout t{deadline}; + bool res = this->AwaitCommon(cond, t); + ABSL_RAW_CHECK(res || t.has_timeout(), + "condition untrue on return from Await"); + return res; +} + +bool Mutex::AwaitCommon(const Condition &cond, KernelTimeout t) { + this->AssertReaderHeld(); + MuHow how = + (mu_.load(std::memory_order_relaxed) & kMuWriter) ? kExclusive : kShared; + ABSL_TSAN_MUTEX_PRE_UNLOCK(this, TsanFlags(how)); + SynchWaitParams waitp( + how, &cond, t, nullptr /*no cvmu*/, Synch_GetPerThreadAnnotated(this), + nullptr /*no cv_word*/); + int flags = kMuHasBlocked; + if (!Condition::GuaranteedEqual(&cond, nullptr)) { + flags |= kMuIsCond; + } + this->UnlockSlow(&waitp); + this->Block(waitp.thread); + ABSL_TSAN_MUTEX_POST_UNLOCK(this, TsanFlags(how)); + ABSL_TSAN_MUTEX_PRE_LOCK(this, TsanFlags(how)); + this->LockSlowLoop(&waitp, flags); + bool res = waitp.cond != nullptr || // => cond known true from LockSlowLoop + EvalConditionAnnotated(&cond, this, true, false, how == kShared); + ABSL_TSAN_MUTEX_POST_LOCK(this, TsanFlags(how), 0); + return res; +} + +ABSL_XRAY_LOG_ARGS(1) bool Mutex::TryLock() { + ABSL_TSAN_MUTEX_PRE_LOCK(this, __tsan_mutex_try_lock); + intptr_t v = mu_.load(std::memory_order_relaxed); + if ((v & (kMuWriter | kMuReader | kMuEvent)) == 0 && // try fast acquire + mu_.compare_exchange_strong(v, kMuWriter | v, + std::memory_order_acquire, + std::memory_order_relaxed)) { + DebugOnlyLockEnter(this); + ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0); + return true; + } + if ((v & kMuEvent) != 0) { // we're recording events + if ((v & kExclusive->slow_need_zero) == 0 && // try fast acquire + mu_.compare_exchange_strong( + v, (kExclusive->fast_or | v) + kExclusive->fast_add, + std::memory_order_acquire, std::memory_order_relaxed)) { + DebugOnlyLockEnter(this); + PostSynchEvent(this, SYNCH_EV_TRYLOCK_SUCCESS); + ABSL_TSAN_MUTEX_POST_LOCK(this, __tsan_mutex_try_lock, 0); + return true; + } else { + PostSynchEvent(this, SYNCH_EV_TRYLOCK_FAILED); + } + } + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_try_lock | __tsan_mutex_try_lock_failed, 0); + return false; +} + +ABSL_XRAY_LOG_ARGS(1) bool Mutex::ReaderTryLock() { + ABSL_TSAN_MUTEX_PRE_LOCK(this, + __tsan_mutex_read_lock | __tsan_mutex_try_lock); + intptr_t v = mu_.load(std::memory_order_relaxed); + // The while-loops (here and below) iterate only if the mutex word keeps + // changing (typically because the reader count changes) under the CAS. We + // limit the number of attempts to avoid having to think about livelock. + int loop_limit = 5; + while ((v & (kMuWriter|kMuWait|kMuEvent)) == 0 && loop_limit != 0) { + if (mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, + std::memory_order_acquire, + std::memory_order_relaxed)) { + DebugOnlyLockEnter(this); + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0); + return true; + } + loop_limit--; + v = mu_.load(std::memory_order_relaxed); + } + if ((v & kMuEvent) != 0) { // we're recording events + loop_limit = 5; + while ((v & kShared->slow_need_zero) == 0 && loop_limit != 0) { + if (mu_.compare_exchange_strong(v, (kMuReader | v) + kMuOne, + std::memory_order_acquire, + std::memory_order_relaxed)) { + DebugOnlyLockEnter(this); + PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_SUCCESS); + ABSL_TSAN_MUTEX_POST_LOCK( + this, __tsan_mutex_read_lock | __tsan_mutex_try_lock, 0); + return true; + } + loop_limit--; + v = mu_.load(std::memory_order_relaxed); + } + if ((v & kMuEvent) != 0) { + PostSynchEvent(this, SYNCH_EV_READERTRYLOCK_FAILED); + } + } + ABSL_TSAN_MUTEX_POST_LOCK(this, + __tsan_mutex_read_lock | __tsan_mutex_try_lock | + __tsan_mutex_try_lock_failed, + 0); + return false; +} + +ABSL_XRAY_LOG_ARGS(1) void Mutex::Unlock() { + ABSL_TSAN_MUTEX_PRE_UNLOCK(this, 0); + DebugOnlyLockLeave(this); + intptr_t v = mu_.load(std::memory_order_relaxed); + + if (kDebugMode && ((v & (kMuWriter | kMuReader)) != kMuWriter)) { + ABSL_RAW_LOG(FATAL, "Mutex unlocked when destroyed or not locked: v=0x%x", + static_cast<unsigned>(v)); + } + + // should_try_cas is whether we'll try a compare-and-swap immediately. + // NOTE: optimized out when kDebugMode is false. + bool should_try_cas = ((v & (kMuEvent | kMuWriter)) == kMuWriter && + (v & (kMuWait | kMuDesig)) != kMuWait); + // But, we can use an alternate computation of it, that compilers + // currently don't find on their own. When that changes, this function + // can be simplified. + intptr_t x = (v ^ (kMuWriter | kMuWait)) & (kMuWriter | kMuEvent); + intptr_t y = (v ^ (kMuWriter | kMuWait)) & (kMuWait | kMuDesig); + // Claim: "x == 0 && y > 0" is equal to should_try_cas. + // Also, because kMuWriter and kMuEvent exceed kMuDesig and kMuWait, + // all possible non-zero values for x exceed all possible values for y. + // Therefore, (x == 0 && y > 0) == (x < y). + if (kDebugMode && should_try_cas != (x < y)) { + // We would usually use PRIdPTR here, but is not correctly implemented + // within the android toolchain. + ABSL_RAW_LOG(FATAL, "internal logic error %llx %llx %llx\n", + static_cast<long long>(v), static_cast<long long>(x), + static_cast<long long>(y)); + } + if (x < y && + mu_.compare_exchange_strong(v, v & ~(kMuWrWait | kMuWriter), + std::memory_order_release, + std::memory_order_relaxed)) { + // fast writer release (writer with no waiters or with designated waker) + } else { + this->UnlockSlow(nullptr /*no waitp*/); // take slow path + } + ABSL_TSAN_MUTEX_POST_UNLOCK(this, 0); +} + +// Requires v to represent a reader-locked state. +static bool ExactlyOneReader(intptr_t v) { + assert((v & (kMuWriter|kMuReader)) == kMuReader); + assert((v & kMuHigh) != 0); + // The more straightforward "(v & kMuHigh) == kMuOne" also works, but + // on some architectures the following generates slightly smaller code. + // It may be faster too. + constexpr intptr_t kMuMultipleWaitersMask = kMuHigh ^ kMuOne; + return (v & kMuMultipleWaitersMask) == 0; +} + +ABSL_XRAY_LOG_ARGS(1) void Mutex::ReaderUnlock() { + ABSL_TSAN_MUTEX_PRE_UNLOCK(this, __tsan_mutex_read_lock); + DebugOnlyLockLeave(this); + intptr_t v = mu_.load(std::memory_order_relaxed); + assert((v & (kMuWriter|kMuReader)) == kMuReader); + if ((v & (kMuReader|kMuWait|kMuEvent)) == kMuReader) { + // fast reader release (reader with no waiters) + intptr_t clear = ExactlyOneReader(v) ? kMuReader|kMuOne : kMuOne; + if (mu_.compare_exchange_strong(v, v - clear, + std::memory_order_release, + std::memory_order_relaxed)) { + ABSL_TSAN_MUTEX_POST_UNLOCK(this, __tsan_mutex_read_lock); + return; + } + } + this->UnlockSlow(nullptr /*no waitp*/); // take slow path + ABSL_TSAN_MUTEX_POST_UNLOCK(this, __tsan_mutex_read_lock); +} + +// The zap_desig_waker bitmask is used to clear the designated waker flag in +// the mutex if this thread has blocked, and therefore may be the designated +// waker. +static const intptr_t zap_desig_waker[] = { + ~static_cast<intptr_t>(0), // not blocked + ~static_cast<intptr_t>( + kMuDesig) // blocked; turn off the designated waker bit +}; + +// The ignore_waiting_writers bitmask is used to ignore the existence +// of waiting writers if a reader that has already blocked once +// wakes up. +static const intptr_t ignore_waiting_writers[] = { + ~static_cast<intptr_t>(0), // not blocked + ~static_cast<intptr_t>( + kMuWrWait) // blocked; pretend there are no waiting writers +}; + +// Internal version of LockWhen(). See LockSlowWithDeadline() +ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlow(MuHow how, const Condition *cond, + int flags) { + ABSL_RAW_CHECK( + this->LockSlowWithDeadline(how, cond, KernelTimeout::Never(), flags), + "condition untrue on return from LockSlow"); +} + +// Compute cond->Eval() and tell race detectors that we do it under mutex mu. +static inline bool EvalConditionAnnotated(const Condition *cond, Mutex *mu, + bool locking, bool trylock, + bool read_lock) { + // Delicate annotation dance. + // We are currently inside of read/write lock/unlock operation. + // All memory accesses are ignored inside of mutex operations + for unlock + // operation tsan considers that we've already released the mutex. + bool res = false; +#ifdef ABSL_INTERNAL_HAVE_TSAN_INTERFACE + const int flags = read_lock ? __tsan_mutex_read_lock : 0; + const int tryflags = flags | (trylock ? __tsan_mutex_try_lock : 0); +#endif + if (locking) { + // For lock we pretend that we have finished the operation, + // evaluate the predicate, then unlock the mutex and start locking it again + // to match the annotation at the end of outer lock operation. + // Note: we can't simply do POST_LOCK, Eval, PRE_LOCK, because then tsan + // will think the lock acquisition is recursive which will trigger + // deadlock detector. + ABSL_TSAN_MUTEX_POST_LOCK(mu, tryflags, 0); + res = cond->Eval(); + // There is no "try" version of Unlock, so use flags instead of tryflags. + ABSL_TSAN_MUTEX_PRE_UNLOCK(mu, flags); + ABSL_TSAN_MUTEX_POST_UNLOCK(mu, flags); + ABSL_TSAN_MUTEX_PRE_LOCK(mu, tryflags); + } else { + // Similarly, for unlock we pretend that we have unlocked the mutex, + // lock the mutex, evaluate the predicate, and start unlocking it again + // to match the annotation at the end of outer unlock operation. + ABSL_TSAN_MUTEX_POST_UNLOCK(mu, flags); + ABSL_TSAN_MUTEX_PRE_LOCK(mu, flags); + ABSL_TSAN_MUTEX_POST_LOCK(mu, flags, 0); + res = cond->Eval(); + ABSL_TSAN_MUTEX_PRE_UNLOCK(mu, flags); + } + // Prevent unused param warnings in non-TSAN builds. + static_cast<void>(mu); + static_cast<void>(trylock); + static_cast<void>(read_lock); + return res; +} + +// Compute cond->Eval() hiding it from race detectors. +// We are hiding it because inside of UnlockSlow we can evaluate a predicate +// that was just added by a concurrent Lock operation; Lock adds the predicate +// to the internal Mutex list without actually acquiring the Mutex +// (it only acquires the internal spinlock, which is rightfully invisible for +// tsan). As the result there is no tsan-visible synchronization between the +// addition and this thread. So if we would enable race detection here, +// it would race with the predicate initialization. +static inline bool EvalConditionIgnored(Mutex *mu, const Condition *cond) { + // Memory accesses are already ignored inside of lock/unlock operations, + // but synchronization operations are also ignored. When we evaluate the + // predicate we must ignore only memory accesses but not synchronization, + // because missed synchronization can lead to false reports later. + // So we "divert" (which un-ignores both memory accesses and synchronization) + // and then separately turn on ignores of memory accesses. + ABSL_TSAN_MUTEX_PRE_DIVERT(mu, 0); + ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_BEGIN(); + bool res = cond->Eval(); + ABSL_ANNOTATE_IGNORE_READS_AND_WRITES_END(); + ABSL_TSAN_MUTEX_POST_DIVERT(mu, 0); + static_cast<void>(mu); // Prevent unused param warning in non-TSAN builds. + return res; +} + +// Internal equivalent of *LockWhenWithDeadline(), where +// "t" represents the absolute timeout; !t.has_timeout() means "forever". +// "how" is "kShared" (for ReaderLockWhen) or "kExclusive" (for LockWhen) +// In flags, bits are ored together: +// - kMuHasBlocked indicates that the client has already blocked on the call so +// the designated waker bit must be cleared and waiting writers should not +// obstruct this call +// - kMuIsCond indicates that this is a conditional acquire (condition variable, +// Await, LockWhen) so contention profiling should be suppressed. +bool Mutex::LockSlowWithDeadline(MuHow how, const Condition *cond, + KernelTimeout t, int flags) { + intptr_t v = mu_.load(std::memory_order_relaxed); + bool unlock = false; + if ((v & how->fast_need_zero) == 0 && // try fast acquire + mu_.compare_exchange_strong( + v, (how->fast_or | (v & zap_desig_waker[flags & kMuHasBlocked])) + + how->fast_add, + std::memory_order_acquire, std::memory_order_relaxed)) { + if (cond == nullptr || + EvalConditionAnnotated(cond, this, true, false, how == kShared)) { + return true; + } + unlock = true; + } + SynchWaitParams waitp( + how, cond, t, nullptr /*no cvmu*/, Synch_GetPerThreadAnnotated(this), + nullptr /*no cv_word*/); + if (!Condition::GuaranteedEqual(cond, nullptr)) { + flags |= kMuIsCond; + } + if (unlock) { + this->UnlockSlow(&waitp); + this->Block(waitp.thread); + flags |= kMuHasBlocked; + } + this->LockSlowLoop(&waitp, flags); + return waitp.cond != nullptr || // => cond known true from LockSlowLoop + cond == nullptr || + EvalConditionAnnotated(cond, this, true, false, how == kShared); +} + +// RAW_CHECK_FMT() takes a condition, a printf-style format string, and +// the printf-style argument list. The format string must be a literal. +// Arguments after the first are not evaluated unless the condition is true. +#define RAW_CHECK_FMT(cond, ...) \ + do { \ + if (ABSL_PREDICT_FALSE(!(cond))) { \ + ABSL_RAW_LOG(FATAL, "Check " #cond " failed: " __VA_ARGS__); \ + } \ + } while (0) + +static void CheckForMutexCorruption(intptr_t v, const char* label) { + // Test for either of two situations that should not occur in v: + // kMuWriter and kMuReader + // kMuWrWait and !kMuWait + const uintptr_t w = v ^ kMuWait; + // By flipping that bit, we can now test for: + // kMuWriter and kMuReader in w + // kMuWrWait and kMuWait in w + // We've chosen these two pairs of values to be so that they will overlap, + // respectively, when the word is left shifted by three. This allows us to + // save a branch in the common (correct) case of them not being coincident. + static_assert(kMuReader << 3 == kMuWriter, "must match"); + static_assert(kMuWait << 3 == kMuWrWait, "must match"); + if (ABSL_PREDICT_TRUE((w & (w << 3) & (kMuWriter | kMuWrWait)) == 0)) return; + RAW_CHECK_FMT((v & (kMuWriter | kMuReader)) != (kMuWriter | kMuReader), + "%s: Mutex corrupt: both reader and writer lock held: %p", + label, reinterpret_cast<void *>(v)); + RAW_CHECK_FMT((v & (kMuWait | kMuWrWait)) != kMuWrWait, + "%s: Mutex corrupt: waiting writer with no waiters: %p", + label, reinterpret_cast<void *>(v)); + assert(false); +} + +void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { + SchedulingGuard::ScopedDisable disable_rescheduling; + int c = 0; + intptr_t v = mu_.load(std::memory_order_relaxed); + if ((v & kMuEvent) != 0) { + PostSynchEvent(this, + waitp->how == kExclusive? SYNCH_EV_LOCK: SYNCH_EV_READERLOCK); + } + ABSL_RAW_CHECK( + waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors, + "detected illegal recursion into Mutex code"); + for (;;) { + v = mu_.load(std::memory_order_relaxed); + CheckForMutexCorruption(v, "Lock"); + if ((v & waitp->how->slow_need_zero) == 0) { + if (mu_.compare_exchange_strong( + v, (waitp->how->fast_or | + (v & zap_desig_waker[flags & kMuHasBlocked])) + + waitp->how->fast_add, + std::memory_order_acquire, std::memory_order_relaxed)) { + if (waitp->cond == nullptr || + EvalConditionAnnotated(waitp->cond, this, true, false, + waitp->how == kShared)) { + break; // we timed out, or condition true, so return + } + this->UnlockSlow(waitp); // got lock but condition false + this->Block(waitp->thread); + flags |= kMuHasBlocked; + c = 0; + } + } else { // need to access waiter list + bool dowait = false; + if ((v & (kMuSpin|kMuWait)) == 0) { // no waiters + // This thread tries to become the one and only waiter. + PerThreadSynch *new_h = Enqueue(nullptr, waitp, v, flags); + intptr_t nv = (v & zap_desig_waker[flags & kMuHasBlocked] & kMuLow) | + kMuWait; + ABSL_RAW_CHECK(new_h != nullptr, "Enqueue to empty list failed"); + if (waitp->how == kExclusive && (v & kMuReader) != 0) { + nv |= kMuWrWait; + } + if (mu_.compare_exchange_strong( + v, reinterpret_cast<intptr_t>(new_h) | nv, + std::memory_order_release, std::memory_order_relaxed)) { + dowait = true; + } else { // attempted Enqueue() failed + // zero out the waitp field set by Enqueue() + waitp->thread->waitp = nullptr; + } + } else if ((v & waitp->how->slow_inc_need_zero & + ignore_waiting_writers[flags & kMuHasBlocked]) == 0) { + // This is a reader that needs to increment the reader count, + // but the count is currently held in the last waiter. + if (mu_.compare_exchange_strong( + v, (v & zap_desig_waker[flags & kMuHasBlocked]) | kMuSpin | + kMuReader, + std::memory_order_acquire, std::memory_order_relaxed)) { + PerThreadSynch *h = GetPerThreadSynch(v); + h->readers += kMuOne; // inc reader count in waiter + do { // release spinlock + v = mu_.load(std::memory_order_relaxed); + } while (!mu_.compare_exchange_weak(v, (v & ~kMuSpin) | kMuReader, + std::memory_order_release, + std::memory_order_relaxed)); + if (waitp->cond == nullptr || + EvalConditionAnnotated(waitp->cond, this, true, false, + waitp->how == kShared)) { + break; // we timed out, or condition true, so return + } + this->UnlockSlow(waitp); // got lock but condition false + this->Block(waitp->thread); + flags |= kMuHasBlocked; + c = 0; + } + } else if ((v & kMuSpin) == 0 && // attempt to queue ourselves + mu_.compare_exchange_strong( + v, (v & zap_desig_waker[flags & kMuHasBlocked]) | kMuSpin | + kMuWait, + std::memory_order_acquire, std::memory_order_relaxed)) { + PerThreadSynch *h = GetPerThreadSynch(v); + PerThreadSynch *new_h = Enqueue(h, waitp, v, flags); + intptr_t wr_wait = 0; + ABSL_RAW_CHECK(new_h != nullptr, "Enqueue to list failed"); + if (waitp->how == kExclusive && (v & kMuReader) != 0) { + wr_wait = kMuWrWait; // give priority to a waiting writer + } + do { // release spinlock + v = mu_.load(std::memory_order_relaxed); + } while (!mu_.compare_exchange_weak( + v, (v & (kMuLow & ~kMuSpin)) | kMuWait | wr_wait | + reinterpret_cast<intptr_t>(new_h), + std::memory_order_release, std::memory_order_relaxed)); + dowait = true; + } + if (dowait) { + this->Block(waitp->thread); // wait until removed from list or timeout + flags |= kMuHasBlocked; + c = 0; + } + } + ABSL_RAW_CHECK( + waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors, + "detected illegal recursion into Mutex code"); + // delay, then try again + c = synchronization_internal::MutexDelay(c, GENTLE); + } + ABSL_RAW_CHECK( + waitp->thread->waitp == nullptr || waitp->thread->suppress_fatal_errors, + "detected illegal recursion into Mutex code"); + if ((v & kMuEvent) != 0) { + PostSynchEvent(this, + waitp->how == kExclusive? SYNCH_EV_LOCK_RETURNING : + SYNCH_EV_READERLOCK_RETURNING); + } +} + +// Unlock this mutex, which is held by the current thread. +// If waitp is non-zero, it must be the wait parameters for the current thread +// which holds the lock but is not runnable because its condition is false +// or it is in the process of blocking on a condition variable; it must requeue +// itself on the mutex/condvar to wait for its condition to become true. +ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams *waitp) { + SchedulingGuard::ScopedDisable disable_rescheduling; + intptr_t v = mu_.load(std::memory_order_relaxed); + this->AssertReaderHeld(); + CheckForMutexCorruption(v, "Unlock"); + if ((v & kMuEvent) != 0) { + PostSynchEvent(this, + (v & kMuWriter) != 0? SYNCH_EV_UNLOCK: SYNCH_EV_READERUNLOCK); + } + int c = 0; + // the waiter under consideration to wake, or zero + PerThreadSynch *w = nullptr; + // the predecessor to w or zero + PerThreadSynch *pw = nullptr; + // head of the list searched previously, or zero + PerThreadSynch *old_h = nullptr; + // a condition that's known to be false. + const Condition *known_false = nullptr; + PerThreadSynch *wake_list = kPerThreadSynchNull; // list of threads to wake + intptr_t wr_wait = 0; // set to kMuWrWait if we wake a reader and a + // later writer could have acquired the lock + // (starvation avoidance) + ABSL_RAW_CHECK(waitp == nullptr || waitp->thread->waitp == nullptr || + waitp->thread->suppress_fatal_errors, + "detected illegal recursion into Mutex code"); + // This loop finds threads wake_list to wakeup if any, and removes them from + // the list of waiters. In addition, it places waitp.thread on the queue of + // waiters if waitp is non-zero. + for (;;) { + v = mu_.load(std::memory_order_relaxed); + if ((v & kMuWriter) != 0 && (v & (kMuWait | kMuDesig)) != kMuWait && + waitp == nullptr) { + // fast writer release (writer with no waiters or with designated waker) + if (mu_.compare_exchange_strong(v, v & ~(kMuWrWait | kMuWriter), + std::memory_order_release, + std::memory_order_relaxed)) { + return; + } + } else if ((v & (kMuReader | kMuWait)) == kMuReader && waitp == nullptr) { + // fast reader release (reader with no waiters) + intptr_t clear = ExactlyOneReader(v) ? kMuReader | kMuOne : kMuOne; + if (mu_.compare_exchange_strong(v, v - clear, + std::memory_order_release, + std::memory_order_relaxed)) { + return; + } + } else if ((v & kMuSpin) == 0 && // attempt to get spinlock + mu_.compare_exchange_strong(v, v | kMuSpin, + std::memory_order_acquire, + std::memory_order_relaxed)) { + if ((v & kMuWait) == 0) { // no one to wake + intptr_t nv; + bool do_enqueue = true; // always Enqueue() the first time + ABSL_RAW_CHECK(waitp != nullptr, + "UnlockSlow is confused"); // about to sleep + do { // must loop to release spinlock as reader count may change + v = mu_.load(std::memory_order_relaxed); + // decrement reader count if there are readers + intptr_t new_readers = (v >= kMuOne)? v - kMuOne : v; + PerThreadSynch *new_h = nullptr; + if (do_enqueue) { + // If we are enqueuing on a CondVar (waitp->cv_word != nullptr) then + // we must not retry here. The initial attempt will always have + // succeeded, further attempts would enqueue us against *this due to + // Fer() handling. + do_enqueue = (waitp->cv_word == nullptr); + new_h = Enqueue(nullptr, waitp, new_readers, kMuIsCond); + } + intptr_t clear = kMuWrWait | kMuWriter; // by default clear write bit + if ((v & kMuWriter) == 0 && ExactlyOneReader(v)) { // last reader + clear = kMuWrWait | kMuReader; // clear read bit + } + nv = (v & kMuLow & ~clear & ~kMuSpin); + if (new_h != nullptr) { + nv |= kMuWait | reinterpret_cast<intptr_t>(new_h); + } else { // new_h could be nullptr if we queued ourselves on a + // CondVar + // In that case, we must place the reader count back in the mutex + // word, as Enqueue() did not store it in the new waiter. + nv |= new_readers & kMuHigh; + } + // release spinlock & our lock; retry if reader-count changed + // (writer count cannot change since we hold lock) + } while (!mu_.compare_exchange_weak(v, nv, + std::memory_order_release, + std::memory_order_relaxed)); + break; + } + + // There are waiters. + // Set h to the head of the circular waiter list. + PerThreadSynch *h = GetPerThreadSynch(v); + if ((v & kMuReader) != 0 && (h->readers & kMuHigh) > kMuOne) { + // a reader but not the last + h->readers -= kMuOne; // release our lock + intptr_t nv = v; // normally just release spinlock + if (waitp != nullptr) { // but waitp!=nullptr => must queue ourselves + PerThreadSynch *new_h = Enqueue(h, waitp, v, kMuIsCond); + ABSL_RAW_CHECK(new_h != nullptr, + "waiters disappeared during Enqueue()!"); + nv &= kMuLow; + nv |= kMuWait | reinterpret_cast<intptr_t>(new_h); + } + mu_.store(nv, std::memory_order_release); // release spinlock + // can release with a store because there were waiters + break; + } + + // Either we didn't search before, or we marked the queue + // as "maybe_unlocking" and no one else should have changed it. + ABSL_RAW_CHECK(old_h == nullptr || h->maybe_unlocking, + "Mutex queue changed beneath us"); + + // The lock is becoming free, and there's a waiter + if (old_h != nullptr && + !old_h->may_skip) { // we used old_h as a terminator + old_h->may_skip = true; // allow old_h to skip once more + ABSL_RAW_CHECK(old_h->skip == nullptr, "illegal skip from head"); + if (h != old_h && MuSameCondition(old_h, old_h->next)) { + old_h->skip = old_h->next; // old_h not head & can skip to successor + } + } + if (h->next->waitp->how == kExclusive && + Condition::GuaranteedEqual(h->next->waitp->cond, nullptr)) { + // easy case: writer with no condition; no need to search + pw = h; // wake w, the successor of h (=pw) + w = h->next; + w->wake = true; + // We are waking up a writer. This writer may be racing against + // an already awake reader for the lock. We want the + // writer to usually win this race, + // because if it doesn't, we can potentially keep taking a reader + // perpetually and writers will starve. Worse than + // that, this can also starve other readers if kMuWrWait gets set + // later. + wr_wait = kMuWrWait; + } else if (w != nullptr && (w->waitp->how == kExclusive || h == old_h)) { + // we found a waiter w to wake on a previous iteration and either it's + // a writer, or we've searched the entire list so we have all the + // readers. + if (pw == nullptr) { // if w's predecessor is unknown, it must be h + pw = h; + } + } else { + // At this point we don't know all the waiters to wake, and the first + // waiter has a condition or is a reader. We avoid searching over + // waiters we've searched on previous iterations by starting at + // old_h if it's set. If old_h==h, there's no one to wakeup at all. + if (old_h == h) { // we've searched before, and nothing's new + // so there's no one to wake. + intptr_t nv = (v & ~(kMuReader|kMuWriter|kMuWrWait)); + h->readers = 0; + h->maybe_unlocking = false; // finished unlocking + if (waitp != nullptr) { // we must queue ourselves and sleep + PerThreadSynch *new_h = Enqueue(h, waitp, v, kMuIsCond); + nv &= kMuLow; + if (new_h != nullptr) { + nv |= kMuWait | reinterpret_cast<intptr_t>(new_h); + } // else new_h could be nullptr if we queued ourselves on a + // CondVar + } + // release spinlock & lock + // can release with a store because there were waiters + mu_.store(nv, std::memory_order_release); + break; + } + + // set up to walk the list + PerThreadSynch *w_walk; // current waiter during list walk + PerThreadSynch *pw_walk; // previous waiter during list walk + if (old_h != nullptr) { // we've searched up to old_h before + pw_walk = old_h; + w_walk = old_h->next; + } else { // no prior search, start at beginning + pw_walk = + nullptr; // h->next's predecessor may change; don't record it + w_walk = h->next; + } + + h->may_skip = false; // ensure we never skip past h in future searches + // even if other waiters are queued after it. + ABSL_RAW_CHECK(h->skip == nullptr, "illegal skip from head"); + + h->maybe_unlocking = true; // we're about to scan the waiter list + // without the spinlock held. + // Enqueue must be conservative about + // priority queuing. + + // We must release the spinlock to evaluate the conditions. + mu_.store(v, std::memory_order_release); // release just spinlock + // can release with a store because there were waiters + + // h is the last waiter queued, and w_walk the first unsearched waiter. + // Without the spinlock, the locations mu_ and h->next may now change + // underneath us, but since we hold the lock itself, the only legal + // change is to add waiters between h and w_walk. Therefore, it's safe + // to walk the path from w_walk to h inclusive. (TryRemove() can remove + // a waiter anywhere, but it acquires both the spinlock and the Mutex) + + old_h = h; // remember we searched to here + + // Walk the path upto and including h looking for waiters we can wake. + while (pw_walk != h) { + w_walk->wake = false; + if (w_walk->waitp->cond == + nullptr || // no condition => vacuously true OR + (w_walk->waitp->cond != known_false && + // this thread's condition is not known false, AND + // is in fact true + EvalConditionIgnored(this, w_walk->waitp->cond))) { + if (w == nullptr) { + w_walk->wake = true; // can wake this waiter + w = w_walk; + pw = pw_walk; + if (w_walk->waitp->how == kExclusive) { + wr_wait = kMuWrWait; + break; // bail if waking this writer + } + } else if (w_walk->waitp->how == kShared) { // wake if a reader + w_walk->wake = true; + } else { // writer with true condition + wr_wait = kMuWrWait; + } + } else { // can't wake; condition false + known_false = w_walk->waitp->cond; // remember last false condition + } + if (w_walk->wake) { // we're waking reader w_walk + pw_walk = w_walk; // don't skip similar waiters + } else { // not waking; skip as much as possible + pw_walk = Skip(w_walk); + } + // If pw_walk == h, then load of pw_walk->next can race with + // concurrent write in Enqueue(). However, at the same time + // we do not need to do the load, because we will bail out + // from the loop anyway. + if (pw_walk != h) { + w_walk = pw_walk->next; + } + } + + continue; // restart for(;;)-loop to wakeup w or to find more waiters + } + ABSL_RAW_CHECK(pw->next == w, "pw not w's predecessor"); + // The first (and perhaps only) waiter we've chosen to wake is w, whose + // predecessor is pw. If w is a reader, we must wake all the other + // waiters with wake==true as well. We may also need to queue + // ourselves if waitp != null. The spinlock and the lock are still + // held. + + // This traverses the list in [ pw->next, h ], where h is the head, + // removing all elements with wake==true and placing them in the + // singly-linked list wake_list. Returns the new head. + h = DequeueAllWakeable(h, pw, &wake_list); + + intptr_t nv = (v & kMuEvent) | kMuDesig; + // assume no waiters left, + // set kMuDesig for INV1a + + if (waitp != nullptr) { // we must queue ourselves and sleep + h = Enqueue(h, waitp, v, kMuIsCond); + // h is new last waiter; could be null if we queued ourselves on a + // CondVar + } + + ABSL_RAW_CHECK(wake_list != kPerThreadSynchNull, + "unexpected empty wake list"); + + if (h != nullptr) { // there are waiters left + h->readers = 0; + h->maybe_unlocking = false; // finished unlocking + nv |= wr_wait | kMuWait | reinterpret_cast<intptr_t>(h); + } + + // release both spinlock & lock + // can release with a store because there were waiters + mu_.store(nv, std::memory_order_release); + break; // out of for(;;)-loop + } + // aggressive here; no one can proceed till we do + c = synchronization_internal::MutexDelay(c, AGGRESSIVE); + } // end of for(;;)-loop + + if (wake_list != kPerThreadSynchNull) { + int64_t enqueue_timestamp = wake_list->waitp->contention_start_cycles; + bool cond_waiter = wake_list->cond_waiter; + do { + wake_list = Wakeup(wake_list); // wake waiters + } while (wake_list != kPerThreadSynchNull); + if (!cond_waiter) { + // Sample lock contention events only if the (first) waiter was trying to + // acquire the lock, not waiting on a condition variable or Condition. + int64_t wait_cycles = + base_internal::CycleClock::Now() - enqueue_timestamp; + mutex_tracer("slow release", this, wait_cycles); + ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0); + submit_profile_data(enqueue_timestamp); + ABSL_TSAN_MUTEX_POST_DIVERT(this, 0); + } + } +} + +// Used by CondVar implementation to reacquire mutex after waking from +// condition variable. This routine is used instead of Lock() because the +// waiting thread may have been moved from the condition variable queue to the +// mutex queue without a wakeup, by Trans(). In that case, when the thread is +// finally woken, the woken thread will believe it has been woken from the +// condition variable (i.e. its PC will be in when in the CondVar code), when +// in fact it has just been woken from the mutex. Thus, it must enter the slow +// path of the mutex in the same state as if it had just woken from the mutex. +// That is, it must ensure to clear kMuDesig (INV1b). +void Mutex::Trans(MuHow how) { + this->LockSlow(how, nullptr, kMuHasBlocked | kMuIsCond); +} + +// Used by CondVar implementation to effectively wake thread w from the +// condition variable. If this mutex is free, we simply wake the thread. +// It will later acquire the mutex with high probability. Otherwise, we +// enqueue thread w on this mutex. +void Mutex::Fer(PerThreadSynch *w) { + SchedulingGuard::ScopedDisable disable_rescheduling; + int c = 0; + ABSL_RAW_CHECK(w->waitp->cond == nullptr, + "Mutex::Fer while waiting on Condition"); + ABSL_RAW_CHECK(!w->waitp->timeout.has_timeout(), + "Mutex::Fer while in timed wait"); + ABSL_RAW_CHECK(w->waitp->cv_word == nullptr, + "Mutex::Fer with pending CondVar queueing"); + for (;;) { + intptr_t v = mu_.load(std::memory_order_relaxed); + // Note: must not queue if the mutex is unlocked (nobody will wake it). + // For example, we can have only kMuWait (conditional) or maybe + // kMuWait|kMuWrWait. + // conflicting != 0 implies that the waking thread cannot currently take + // the mutex, which in turn implies that someone else has it and can wake + // us if we queue. + const intptr_t conflicting = + kMuWriter | (w->waitp->how == kShared ? 0 : kMuReader); + if ((v & conflicting) == 0) { + w->next = nullptr; + w->state.store(PerThreadSynch::kAvailable, std::memory_order_release); + IncrementSynchSem(this, w); + return; + } else { + if ((v & (kMuSpin|kMuWait)) == 0) { // no waiters + // This thread tries to become the one and only waiter. + PerThreadSynch *new_h = Enqueue(nullptr, w->waitp, v, kMuIsCond); + ABSL_RAW_CHECK(new_h != nullptr, + "Enqueue failed"); // we must queue ourselves + if (mu_.compare_exchange_strong( + v, reinterpret_cast<intptr_t>(new_h) | (v & kMuLow) | kMuWait, + std::memory_order_release, std::memory_order_relaxed)) { + return; + } + } else if ((v & kMuSpin) == 0 && + mu_.compare_exchange_strong(v, v | kMuSpin | kMuWait)) { + PerThreadSynch *h = GetPerThreadSynch(v); + PerThreadSynch *new_h = Enqueue(h, w->waitp, v, kMuIsCond); + ABSL_RAW_CHECK(new_h != nullptr, + "Enqueue failed"); // we must queue ourselves + do { + v = mu_.load(std::memory_order_relaxed); + } while (!mu_.compare_exchange_weak( + v, + (v & kMuLow & ~kMuSpin) | kMuWait | + reinterpret_cast<intptr_t>(new_h), + std::memory_order_release, std::memory_order_relaxed)); + return; + } + } + c = synchronization_internal::MutexDelay(c, GENTLE); + } +} + +void Mutex::AssertHeld() const { + if ((mu_.load(std::memory_order_relaxed) & kMuWriter) == 0) { + SynchEvent *e = GetSynchEvent(this); + ABSL_RAW_LOG(FATAL, "thread should hold write lock on Mutex %p %s", + static_cast<const void *>(this), + (e == nullptr ? "" : e->name)); + } +} + +void Mutex::AssertReaderHeld() const { + if ((mu_.load(std::memory_order_relaxed) & (kMuReader | kMuWriter)) == 0) { + SynchEvent *e = GetSynchEvent(this); + ABSL_RAW_LOG( + FATAL, "thread should hold at least a read lock on Mutex %p %s", + static_cast<const void *>(this), (e == nullptr ? "" : e->name)); + } +} + +// -------------------------------- condition variables +static const intptr_t kCvSpin = 0x0001L; // spinlock protects waiter list +static const intptr_t kCvEvent = 0x0002L; // record events + +static const intptr_t kCvLow = 0x0003L; // low order bits of CV + +// Hack to make constant values available to gdb pretty printer +enum { kGdbCvSpin = kCvSpin, kGdbCvEvent = kCvEvent, kGdbCvLow = kCvLow, }; + +static_assert(PerThreadSynch::kAlignment > kCvLow, + "PerThreadSynch::kAlignment must be greater than kCvLow"); + +void CondVar::EnableDebugLog(const char *name) { + SynchEvent *e = EnsureSynchEvent(&this->cv_, name, kCvEvent, kCvSpin); + e->log = true; + UnrefSynchEvent(e); +} + +CondVar::~CondVar() { + if ((cv_.load(std::memory_order_relaxed) & kCvEvent) != 0) { + ForgetSynchEvent(&this->cv_, kCvEvent, kCvSpin); + } +} + + +// Remove thread s from the list of waiters on this condition variable. +void CondVar::Remove(PerThreadSynch *s) { + SchedulingGuard::ScopedDisable disable_rescheduling; + intptr_t v; + int c = 0; + for (v = cv_.load(std::memory_order_relaxed);; + v = cv_.load(std::memory_order_relaxed)) { + if ((v & kCvSpin) == 0 && // attempt to acquire spinlock + cv_.compare_exchange_strong(v, v | kCvSpin, + std::memory_order_acquire, + std::memory_order_relaxed)) { + PerThreadSynch *h = reinterpret_cast<PerThreadSynch *>(v & ~kCvLow); + if (h != nullptr) { + PerThreadSynch *w = h; + while (w->next != s && w->next != h) { // search for thread + w = w->next; + } + if (w->next == s) { // found thread; remove it + w->next = s->next; + if (h == s) { + h = (w == s) ? nullptr : w; + } + s->next = nullptr; + s->state.store(PerThreadSynch::kAvailable, std::memory_order_release); + } + } + // release spinlock + cv_.store((v & kCvEvent) | reinterpret_cast<intptr_t>(h), + std::memory_order_release); + return; + } else { + // try again after a delay + c = synchronization_internal::MutexDelay(c, GENTLE); + } + } +} + +// Queue thread waitp->thread on condition variable word cv_word using +// wait parameters waitp. +// We split this into a separate routine, rather than simply doing it as part +// of WaitCommon(). If we were to queue ourselves on the condition variable +// before calling Mutex::UnlockSlow(), the Mutex code might be re-entered (via +// the logging code, or via a Condition function) and might potentially attempt +// to block this thread. That would be a problem if the thread were already on +// a the condition variable waiter queue. Thus, we use the waitp->cv_word +// to tell the unlock code to call CondVarEnqueue() to queue the thread on the +// condition variable queue just before the mutex is to be unlocked, and (most +// importantly) after any call to an external routine that might re-enter the +// mutex code. +static void CondVarEnqueue(SynchWaitParams *waitp) { + // This thread might be transferred to the Mutex queue by Fer() when + // we are woken. To make sure that is what happens, Enqueue() doesn't + // call CondVarEnqueue() again but instead uses its normal code. We + // must do this before we queue ourselves so that cv_word will be null + // when seen by the dequeuer, who may wish immediately to requeue + // this thread on another queue. + std::atomic<intptr_t> *cv_word = waitp->cv_word; + waitp->cv_word = nullptr; + + intptr_t v = cv_word->load(std::memory_order_relaxed); + int c = 0; + while ((v & kCvSpin) != 0 || // acquire spinlock + !cv_word->compare_exchange_weak(v, v | kCvSpin, + std::memory_order_acquire, + std::memory_order_relaxed)) { + c = synchronization_internal::MutexDelay(c, GENTLE); + v = cv_word->load(std::memory_order_relaxed); + } + ABSL_RAW_CHECK(waitp->thread->waitp == nullptr, "waiting when shouldn't be"); + waitp->thread->waitp = waitp; // prepare ourselves for waiting + PerThreadSynch *h = reinterpret_cast<PerThreadSynch *>(v & ~kCvLow); + if (h == nullptr) { // add this thread to waiter list + waitp->thread->next = waitp->thread; + } else { + waitp->thread->next = h->next; + h->next = waitp->thread; + } + waitp->thread->state.store(PerThreadSynch::kQueued, + std::memory_order_relaxed); + cv_word->store((v & kCvEvent) | reinterpret_cast<intptr_t>(waitp->thread), + std::memory_order_release); +} + +bool CondVar::WaitCommon(Mutex *mutex, KernelTimeout t) { + bool rc = false; // return value; true iff we timed-out + + intptr_t mutex_v = mutex->mu_.load(std::memory_order_relaxed); + Mutex::MuHow mutex_how = ((mutex_v & kMuWriter) != 0) ? kExclusive : kShared; + ABSL_TSAN_MUTEX_PRE_UNLOCK(mutex, TsanFlags(mutex_how)); + + // maybe trace this call + intptr_t v = cv_.load(std::memory_order_relaxed); + cond_var_tracer("Wait", this); + if ((v & kCvEvent) != 0) { + PostSynchEvent(this, SYNCH_EV_WAIT); + } + + // Release mu and wait on condition variable. + SynchWaitParams waitp(mutex_how, nullptr, t, mutex, + Synch_GetPerThreadAnnotated(mutex), &cv_); + // UnlockSlow() will call CondVarEnqueue() just before releasing the + // Mutex, thus queuing this thread on the condition variable. See + // CondVarEnqueue() for the reasons. + mutex->UnlockSlow(&waitp); + + // wait for signal + while (waitp.thread->state.load(std::memory_order_acquire) == + PerThreadSynch::kQueued) { + if (!Mutex::DecrementSynchSem(mutex, waitp.thread, t)) { + this->Remove(waitp.thread); + rc = true; + } + } + + ABSL_RAW_CHECK(waitp.thread->waitp != nullptr, "not waiting when should be"); + waitp.thread->waitp = nullptr; // cleanup + + // maybe trace this call + cond_var_tracer("Unwait", this); + if ((v & kCvEvent) != 0) { + PostSynchEvent(this, SYNCH_EV_WAIT_RETURNING); + } + + // From synchronization point of view Wait is unlock of the mutex followed + // by lock of the mutex. We've annotated start of unlock in the beginning + // of the function. Now, finish unlock and annotate lock of the mutex. + // (Trans is effectively lock). + ABSL_TSAN_MUTEX_POST_UNLOCK(mutex, TsanFlags(mutex_how)); + ABSL_TSAN_MUTEX_PRE_LOCK(mutex, TsanFlags(mutex_how)); + mutex->Trans(mutex_how); // Reacquire mutex + ABSL_TSAN_MUTEX_POST_LOCK(mutex, TsanFlags(mutex_how), 0); + return rc; +} + +bool CondVar::WaitWithTimeout(Mutex *mu, absl::Duration timeout) { + return WaitWithDeadline(mu, DeadlineFromTimeout(timeout)); +} + +bool CondVar::WaitWithDeadline(Mutex *mu, absl::Time deadline) { + return WaitCommon(mu, KernelTimeout(deadline)); +} + +void CondVar::Wait(Mutex *mu) { + WaitCommon(mu, KernelTimeout::Never()); +} + +// Wake thread w +// If it was a timed wait, w will be waiting on w->cv +// Otherwise, if it was not a Mutex mutex, w will be waiting on w->sem +// Otherwise, w is transferred to the Mutex mutex via Mutex::Fer(). +void CondVar::Wakeup(PerThreadSynch *w) { + if (w->waitp->timeout.has_timeout() || w->waitp->cvmu == nullptr) { + // The waiting thread only needs to observe "w->state == kAvailable" to be + // released, we must cache "cvmu" before clearing "next". + Mutex *mu = w->waitp->cvmu; + w->next = nullptr; + w->state.store(PerThreadSynch::kAvailable, std::memory_order_release); + Mutex::IncrementSynchSem(mu, w); + } else { + w->waitp->cvmu->Fer(w); + } +} + +void CondVar::Signal() { + SchedulingGuard::ScopedDisable disable_rescheduling; + ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0); + intptr_t v; + int c = 0; + for (v = cv_.load(std::memory_order_relaxed); v != 0; + v = cv_.load(std::memory_order_relaxed)) { + if ((v & kCvSpin) == 0 && // attempt to acquire spinlock + cv_.compare_exchange_strong(v, v | kCvSpin, + std::memory_order_acquire, + std::memory_order_relaxed)) { + PerThreadSynch *h = reinterpret_cast<PerThreadSynch *>(v & ~kCvLow); + PerThreadSynch *w = nullptr; + if (h != nullptr) { // remove first waiter + w = h->next; + if (w == h) { + h = nullptr; + } else { + h->next = w->next; + } + } + // release spinlock + cv_.store((v & kCvEvent) | reinterpret_cast<intptr_t>(h), + std::memory_order_release); + if (w != nullptr) { + CondVar::Wakeup(w); // wake waiter, if there was one + cond_var_tracer("Signal wakeup", this); + } + if ((v & kCvEvent) != 0) { + PostSynchEvent(this, SYNCH_EV_SIGNAL); + } + ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); + return; + } else { + c = synchronization_internal::MutexDelay(c, GENTLE); + } + } + ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); +} + +void CondVar::SignalAll () { + ABSL_TSAN_MUTEX_PRE_SIGNAL(nullptr, 0); + intptr_t v; + int c = 0; + for (v = cv_.load(std::memory_order_relaxed); v != 0; + v = cv_.load(std::memory_order_relaxed)) { + // empty the list if spinlock free + // We do this by simply setting the list to empty using + // compare and swap. We then have the entire list in our hands, + // which cannot be changing since we grabbed it while no one + // held the lock. + if ((v & kCvSpin) == 0 && + cv_.compare_exchange_strong(v, v & kCvEvent, std::memory_order_acquire, + std::memory_order_relaxed)) { + PerThreadSynch *h = reinterpret_cast<PerThreadSynch *>(v & ~kCvLow); + if (h != nullptr) { + PerThreadSynch *w; + PerThreadSynch *n = h->next; + do { // for every thread, wake it up + w = n; + n = n->next; + CondVar::Wakeup(w); + } while (w != h); + cond_var_tracer("SignalAll wakeup", this); + } + if ((v & kCvEvent) != 0) { + PostSynchEvent(this, SYNCH_EV_SIGNALALL); + } + ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); + return; + } else { + // try again after a delay + c = synchronization_internal::MutexDelay(c, GENTLE); + } + } + ABSL_TSAN_MUTEX_POST_SIGNAL(nullptr, 0); +} + +void ReleasableMutexLock::Release() { + ABSL_RAW_CHECK(this->mu_ != nullptr, + "ReleasableMutexLock::Release may only be called once"); + this->mu_->Unlock(); + this->mu_ = nullptr; +} + +#ifdef ABSL_HAVE_THREAD_SANITIZER +extern "C" void __tsan_read1(void *addr); +#else +#define __tsan_read1(addr) // do nothing if TSan not enabled +#endif + +// A function that just returns its argument, dereferenced +static bool Dereference(void *arg) { + // ThreadSanitizer does not instrument this file for memory accesses. + // This function dereferences a user variable that can participate + // in a data race, so we need to manually tell TSan about this memory access. + __tsan_read1(arg); + return *(static_cast<bool *>(arg)); +} + +Condition::Condition() {} // null constructor, used for kTrue only +const Condition Condition::kTrue; + +Condition::Condition(bool (*func)(void *), void *arg) + : eval_(&CallVoidPtrFunction), + function_(func), + method_(nullptr), + arg_(arg) {} + +bool Condition::CallVoidPtrFunction(const Condition *c) { + return (*c->function_)(c->arg_); +} + +Condition::Condition(const bool *cond) + : eval_(CallVoidPtrFunction), + function_(Dereference), + method_(nullptr), + // const_cast is safe since Dereference does not modify arg + arg_(const_cast<bool *>(cond)) {} + +bool Condition::Eval() const { + // eval_ == null for kTrue + return (this->eval_ == nullptr) || (*this->eval_)(this); +} + +bool Condition::GuaranteedEqual(const Condition *a, const Condition *b) { + if (a == nullptr) { + return b == nullptr || b->eval_ == nullptr; + } + if (b == nullptr || b->eval_ == nullptr) { + return a->eval_ == nullptr; + } + return a->eval_ == b->eval_ && a->function_ == b->function_ && + a->arg_ == b->arg_ && a->method_ == b->method_; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/mutex.h b/third_party/abseil_cpp/absl/synchronization/mutex.h new file mode 100644 index 000000000000..598d1e0617cf --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/mutex.h @@ -0,0 +1,1084 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// mutex.h +// ----------------------------------------------------------------------------- +// +// This header file defines a `Mutex` -- a mutually exclusive lock -- and the +// most common type of synchronization primitive for facilitating locks on +// shared resources. A mutex is used to prevent multiple threads from accessing +// and/or writing to a shared resource concurrently. +// +// Unlike a `std::mutex`, the Abseil `Mutex` provides the following additional +// features: +// * Conditional predicates intrinsic to the `Mutex` object +// * Shared/reader locks, in addition to standard exclusive/writer locks +// * Deadlock detection and debug support. +// +// The following helper classes are also defined within this file: +// +// MutexLock - An RAII wrapper to acquire and release a `Mutex` for exclusive/ +// write access within the current scope. +// +// ReaderMutexLock +// - An RAII wrapper to acquire and release a `Mutex` for shared/read +// access within the current scope. +// +// WriterMutexLock +// - Effectively an alias for `MutexLock` above, designed for use in +// distinguishing reader and writer locks within code. +// +// In addition to simple mutex locks, this file also defines ways to perform +// locking under certain conditions. +// +// Condition - (Preferred) Used to wait for a particular predicate that +// depends on state protected by the `Mutex` to become true. +// CondVar - A lower-level variant of `Condition` that relies on +// application code to explicitly signal the `CondVar` when +// a condition has been met. +// +// See below for more information on using `Condition` or `CondVar`. +// +// Mutexes and mutex behavior can be quite complicated. The information within +// this header file is limited, as a result. Please consult the Mutex guide for +// more complete information and examples. + +#ifndef ABSL_SYNCHRONIZATION_MUTEX_H_ +#define ABSL_SYNCHRONIZATION_MUTEX_H_ + +#include <atomic> +#include <cstdint> +#include <string> + +#include "absl/base/const_init.h" +#include "absl/base/internal/identity.h" +#include "absl/base/internal/low_level_alloc.h" +#include "absl/base/internal/thread_identity.h" +#include "absl/base/internal/tsan_mutex_interface.h" +#include "absl/base/port.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/internal/kernel_timeout.h" +#include "absl/synchronization/internal/per_thread_sem.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class Condition; +struct SynchWaitParams; + +// ----------------------------------------------------------------------------- +// Mutex +// ----------------------------------------------------------------------------- +// +// A `Mutex` is a non-reentrant (aka non-recursive) Mutually Exclusive lock +// on some resource, typically a variable or data structure with associated +// invariants. Proper usage of mutexes prevents concurrent access by different +// threads to the same resource. +// +// A `Mutex` has two basic operations: `Mutex::Lock()` and `Mutex::Unlock()`. +// The `Lock()` operation *acquires* a `Mutex` (in a state known as an +// *exclusive* -- or write -- lock), while the `Unlock()` operation *releases* a +// Mutex. During the span of time between the Lock() and Unlock() operations, +// a mutex is said to be *held*. By design all mutexes support exclusive/write +// locks, as this is the most common way to use a mutex. +// +// The `Mutex` state machine for basic lock/unlock operations is quite simple: +// +// | | Lock() | Unlock() | +// |----------------+------------+----------| +// | Free | Exclusive | invalid | +// | Exclusive | blocks | Free | +// +// Attempts to `Unlock()` must originate from the thread that performed the +// corresponding `Lock()` operation. +// +// An "invalid" operation is disallowed by the API. The `Mutex` implementation +// is allowed to do anything on an invalid call, including but not limited to +// crashing with a useful error message, silently succeeding, or corrupting +// data structures. In debug mode, the implementation attempts to crash with a +// useful error message. +// +// `Mutex` is not guaranteed to be "fair" in prioritizing waiting threads; it +// is, however, approximately fair over long periods, and starvation-free for +// threads at the same priority. +// +// The lock/unlock primitives are now annotated with lock annotations +// defined in (base/thread_annotations.h). When writing multi-threaded code, +// you should use lock annotations whenever possible to document your lock +// synchronization policy. Besides acting as documentation, these annotations +// also help compilers or static analysis tools to identify and warn about +// issues that could potentially result in race conditions and deadlocks. +// +// For more information about the lock annotations, please see +// [Thread Safety Analysis](http://clang.llvm.org/docs/ThreadSafetyAnalysis.html) +// in the Clang documentation. +// +// See also `MutexLock`, below, for scoped `Mutex` acquisition. + +class ABSL_LOCKABLE Mutex { + public: + // Creates a `Mutex` that is not held by anyone. This constructor is + // typically used for Mutexes allocated on the heap or the stack. + // + // To create `Mutex` instances with static storage duration + // (e.g. a namespace-scoped or global variable), see + // `Mutex::Mutex(absl::kConstInit)` below instead. + Mutex(); + + // Creates a mutex with static storage duration. A global variable + // constructed this way avoids the lifetime issues that can occur on program + // startup and shutdown. (See absl/base/const_init.h.) + // + // For Mutexes allocated on the heap and stack, instead use the default + // constructor, which can interact more fully with the thread sanitizer. + // + // Example usage: + // namespace foo { + // ABSL_CONST_INIT Mutex mu(absl::kConstInit); + // } + explicit constexpr Mutex(absl::ConstInitType); + + ~Mutex(); + + // Mutex::Lock() + // + // Blocks the calling thread, if necessary, until this `Mutex` is free, and + // then acquires it exclusively. (This lock is also known as a "write lock.") + void Lock() ABSL_EXCLUSIVE_LOCK_FUNCTION(); + + // Mutex::Unlock() + // + // Releases this `Mutex` and returns it from the exclusive/write state to the + // free state. Caller must hold the `Mutex` exclusively. + void Unlock() ABSL_UNLOCK_FUNCTION(); + + // Mutex::TryLock() + // + // If the mutex can be acquired without blocking, does so exclusively and + // returns `true`. Otherwise, returns `false`. Returns `true` with high + // probability if the `Mutex` was free. + bool TryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true); + + // Mutex::AssertHeld() + // + // Return immediately if this thread holds the `Mutex` exclusively (in write + // mode). Otherwise, may report an error (typically by crashing with a + // diagnostic), or may return immediately. + void AssertHeld() const ABSL_ASSERT_EXCLUSIVE_LOCK(); + + // --------------------------------------------------------------------------- + // Reader-Writer Locking + // --------------------------------------------------------------------------- + + // A Mutex can also be used as a starvation-free reader-writer lock. + // Neither read-locks nor write-locks are reentrant/recursive to avoid + // potential client programming errors. + // + // The Mutex API provides `Writer*()` aliases for the existing `Lock()`, + // `Unlock()` and `TryLock()` methods for use within applications mixing + // reader/writer locks. Using `Reader*()` and `Writer*()` operations in this + // manner can make locking behavior clearer when mixing read and write modes. + // + // Introducing reader locks necessarily complicates the `Mutex` state + // machine somewhat. The table below illustrates the allowed state transitions + // of a mutex in such cases. Note that ReaderLock() may block even if the lock + // is held in shared mode; this occurs when another thread is blocked on a + // call to WriterLock(). + // + // --------------------------------------------------------------------------- + // Operation: WriterLock() Unlock() ReaderLock() ReaderUnlock() + // --------------------------------------------------------------------------- + // State + // --------------------------------------------------------------------------- + // Free Exclusive invalid Shared(1) invalid + // Shared(1) blocks invalid Shared(2) or blocks Free + // Shared(n) n>1 blocks invalid Shared(n+1) or blocks Shared(n-1) + // Exclusive blocks Free blocks invalid + // --------------------------------------------------------------------------- + // + // In comments below, "shared" refers to a state of Shared(n) for any n > 0. + + // Mutex::ReaderLock() + // + // Blocks the calling thread, if necessary, until this `Mutex` is either free, + // or in shared mode, and then acquires a share of it. Note that + // `ReaderLock()` will block if some other thread has an exclusive/writer lock + // on the mutex. + + void ReaderLock() ABSL_SHARED_LOCK_FUNCTION(); + + // Mutex::ReaderUnlock() + // + // Releases a read share of this `Mutex`. `ReaderUnlock` may return a mutex to + // the free state if this thread holds the last reader lock on the mutex. Note + // that you cannot call `ReaderUnlock()` on a mutex held in write mode. + void ReaderUnlock() ABSL_UNLOCK_FUNCTION(); + + // Mutex::ReaderTryLock() + // + // If the mutex can be acquired without blocking, acquires this mutex for + // shared access and returns `true`. Otherwise, returns `false`. Returns + // `true` with high probability if the `Mutex` was free or shared. + bool ReaderTryLock() ABSL_SHARED_TRYLOCK_FUNCTION(true); + + // Mutex::AssertReaderHeld() + // + // Returns immediately if this thread holds the `Mutex` in at least shared + // mode (read mode). Otherwise, may report an error (typically by + // crashing with a diagnostic), or may return immediately. + void AssertReaderHeld() const ABSL_ASSERT_SHARED_LOCK(); + + // Mutex::WriterLock() + // Mutex::WriterUnlock() + // Mutex::WriterTryLock() + // + // Aliases for `Mutex::Lock()`, `Mutex::Unlock()`, and `Mutex::TryLock()`. + // + // These methods may be used (along with the complementary `Reader*()` + // methods) to distingish simple exclusive `Mutex` usage (`Lock()`, + // etc.) from reader/writer lock usage. + void WriterLock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->Lock(); } + + void WriterUnlock() ABSL_UNLOCK_FUNCTION() { this->Unlock(); } + + bool WriterTryLock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { + return this->TryLock(); + } + + // --------------------------------------------------------------------------- + // Conditional Critical Regions + // --------------------------------------------------------------------------- + + // Conditional usage of a `Mutex` can occur using two distinct paradigms: + // + // * Use of `Mutex` member functions with `Condition` objects. + // * Use of the separate `CondVar` abstraction. + // + // In general, prefer use of `Condition` and the `Mutex` member functions + // listed below over `CondVar`. When there are multiple threads waiting on + // distinctly different conditions, however, a battery of `CondVar`s may be + // more efficient. This section discusses use of `Condition` objects. + // + // `Mutex` contains member functions for performing lock operations only under + // certain conditions, of class `Condition`. For correctness, the `Condition` + // must return a boolean that is a pure function, only of state protected by + // the `Mutex`. The condition must be invariant w.r.t. environmental state + // such as thread, cpu id, or time, and must be `noexcept`. The condition will + // always be invoked with the mutex held in at least read mode, so you should + // not block it for long periods or sleep it on a timer. + // + // Since a condition must not depend directly on the current time, use + // `*WithTimeout()` member function variants to make your condition + // effectively true after a given duration, or `*WithDeadline()` variants to + // make your condition effectively true after a given time. + // + // The condition function should have no side-effects aside from debug + // logging; as a special exception, the function may acquire other mutexes + // provided it releases all those that it acquires. (This exception was + // required to allow logging.) + + // Mutex::Await() + // + // Unlocks this `Mutex` and blocks until simultaneously both `cond` is `true` + // and this `Mutex` can be reacquired, then reacquires this `Mutex` in the + // same mode in which it was previously held. If the condition is initially + // `true`, `Await()` *may* skip the release/re-acquire step. + // + // `Await()` requires that this thread holds this `Mutex` in some mode. + void Await(const Condition &cond); + + // Mutex::LockWhen() + // Mutex::ReaderLockWhen() + // Mutex::WriterLockWhen() + // + // Blocks until simultaneously both `cond` is `true` and this `Mutex` can + // be acquired, then atomically acquires this `Mutex`. `LockWhen()` is + // logically equivalent to `*Lock(); Await();` though they may have different + // performance characteristics. + void LockWhen(const Condition &cond) ABSL_EXCLUSIVE_LOCK_FUNCTION(); + + void ReaderLockWhen(const Condition &cond) ABSL_SHARED_LOCK_FUNCTION(); + + void WriterLockWhen(const Condition &cond) ABSL_EXCLUSIVE_LOCK_FUNCTION() { + this->LockWhen(cond); + } + + // --------------------------------------------------------------------------- + // Mutex Variants with Timeouts/Deadlines + // --------------------------------------------------------------------------- + + // Mutex::AwaitWithTimeout() + // Mutex::AwaitWithDeadline() + // + // Unlocks this `Mutex` and blocks until simultaneously: + // - either `cond` is true or the {timeout has expired, deadline has passed} + // and + // - this `Mutex` can be reacquired, + // then reacquire this `Mutex` in the same mode in which it was previously + // held, returning `true` iff `cond` is `true` on return. + // + // If the condition is initially `true`, the implementation *may* skip the + // release/re-acquire step and return immediately. + // + // Deadlines in the past are equivalent to an immediate deadline. + // Negative timeouts are equivalent to a zero timeout. + // + // This method requires that this thread holds this `Mutex` in some mode. + bool AwaitWithTimeout(const Condition &cond, absl::Duration timeout); + + bool AwaitWithDeadline(const Condition &cond, absl::Time deadline); + + // Mutex::LockWhenWithTimeout() + // Mutex::ReaderLockWhenWithTimeout() + // Mutex::WriterLockWhenWithTimeout() + // + // Blocks until simultaneously both: + // - either `cond` is `true` or the timeout has expired, and + // - this `Mutex` can be acquired, + // then atomically acquires this `Mutex`, returning `true` iff `cond` is + // `true` on return. + // + // Negative timeouts are equivalent to a zero timeout. + bool LockWhenWithTimeout(const Condition &cond, absl::Duration timeout) + ABSL_EXCLUSIVE_LOCK_FUNCTION(); + bool ReaderLockWhenWithTimeout(const Condition &cond, absl::Duration timeout) + ABSL_SHARED_LOCK_FUNCTION(); + bool WriterLockWhenWithTimeout(const Condition &cond, absl::Duration timeout) + ABSL_EXCLUSIVE_LOCK_FUNCTION() { + return this->LockWhenWithTimeout(cond, timeout); + } + + // Mutex::LockWhenWithDeadline() + // Mutex::ReaderLockWhenWithDeadline() + // Mutex::WriterLockWhenWithDeadline() + // + // Blocks until simultaneously both: + // - either `cond` is `true` or the deadline has been passed, and + // - this `Mutex` can be acquired, + // then atomically acquires this Mutex, returning `true` iff `cond` is `true` + // on return. + // + // Deadlines in the past are equivalent to an immediate deadline. + bool LockWhenWithDeadline(const Condition &cond, absl::Time deadline) + ABSL_EXCLUSIVE_LOCK_FUNCTION(); + bool ReaderLockWhenWithDeadline(const Condition &cond, absl::Time deadline) + ABSL_SHARED_LOCK_FUNCTION(); + bool WriterLockWhenWithDeadline(const Condition &cond, absl::Time deadline) + ABSL_EXCLUSIVE_LOCK_FUNCTION() { + return this->LockWhenWithDeadline(cond, deadline); + } + + // --------------------------------------------------------------------------- + // Debug Support: Invariant Checking, Deadlock Detection, Logging. + // --------------------------------------------------------------------------- + + // Mutex::EnableInvariantDebugging() + // + // If `invariant`!=null and if invariant debugging has been enabled globally, + // cause `(*invariant)(arg)` to be called at moments when the invariant for + // this `Mutex` should hold (for example: just after acquire, just before + // release). + // + // The routine `invariant` should have no side-effects since it is not + // guaranteed how many times it will be called; it should check the invariant + // and crash if it does not hold. Enabling global invariant debugging may + // substantially reduce `Mutex` performance; it should be set only for + // non-production runs. Optimization options may also disable invariant + // checks. + void EnableInvariantDebugging(void (*invariant)(void *), void *arg); + + // Mutex::EnableDebugLog() + // + // Cause all subsequent uses of this `Mutex` to be logged via + // `ABSL_RAW_LOG(INFO)`. Log entries are tagged with `name` if no previous + // call to `EnableInvariantDebugging()` or `EnableDebugLog()` has been made. + // + // Note: This method substantially reduces `Mutex` performance. + void EnableDebugLog(const char *name); + + // Deadlock detection + + // Mutex::ForgetDeadlockInfo() + // + // Forget any deadlock-detection information previously gathered + // about this `Mutex`. Call this method in debug mode when the lock ordering + // of a `Mutex` changes. + void ForgetDeadlockInfo(); + + // Mutex::AssertNotHeld() + // + // Return immediately if this thread does not hold this `Mutex` in any + // mode; otherwise, may report an error (typically by crashing with a + // diagnostic), or may return immediately. + // + // Currently this check is performed only if all of: + // - in debug mode + // - SetMutexDeadlockDetectionMode() has been set to kReport or kAbort + // - number of locks concurrently held by this thread is not large. + // are true. + void AssertNotHeld() const; + + // Special cases. + + // A `MuHow` is a constant that indicates how a lock should be acquired. + // Internal implementation detail. Clients should ignore. + typedef const struct MuHowS *MuHow; + + // Mutex::InternalAttemptToUseMutexInFatalSignalHandler() + // + // Causes the `Mutex` implementation to prepare itself for re-entry caused by + // future use of `Mutex` within a fatal signal handler. This method is + // intended for use only for last-ditch attempts to log crash information. + // It does not guarantee that attempts to use Mutexes within the handler will + // not deadlock; it merely makes other faults less likely. + // + // WARNING: This routine must be invoked from a signal handler, and the + // signal handler must either loop forever or terminate the process. + // Attempts to return from (or `longjmp` out of) the signal handler once this + // call has been made may cause arbitrary program behaviour including + // crashes and deadlocks. + static void InternalAttemptToUseMutexInFatalSignalHandler(); + + private: + std::atomic<intptr_t> mu_; // The Mutex state. + + // Post()/Wait() versus associated PerThreadSem; in class for required + // friendship with PerThreadSem. + static inline void IncrementSynchSem(Mutex *mu, + base_internal::PerThreadSynch *w); + static inline bool DecrementSynchSem( + Mutex *mu, base_internal::PerThreadSynch *w, + synchronization_internal::KernelTimeout t); + + // slow path acquire + void LockSlowLoop(SynchWaitParams *waitp, int flags); + // wrappers around LockSlowLoop() + bool LockSlowWithDeadline(MuHow how, const Condition *cond, + synchronization_internal::KernelTimeout t, + int flags); + void LockSlow(MuHow how, const Condition *cond, + int flags) ABSL_ATTRIBUTE_COLD; + // slow path release + void UnlockSlow(SynchWaitParams *waitp) ABSL_ATTRIBUTE_COLD; + // Common code between Await() and AwaitWithTimeout/Deadline() + bool AwaitCommon(const Condition &cond, + synchronization_internal::KernelTimeout t); + // Attempt to remove thread s from queue. + void TryRemove(base_internal::PerThreadSynch *s); + // Block a thread on mutex. + void Block(base_internal::PerThreadSynch *s); + // Wake a thread; return successor. + base_internal::PerThreadSynch *Wakeup(base_internal::PerThreadSynch *w); + + friend class CondVar; // for access to Trans()/Fer(). + void Trans(MuHow how); // used for CondVar->Mutex transfer + void Fer( + base_internal::PerThreadSynch *w); // used for CondVar->Mutex transfer + + // Catch the error of writing Mutex when intending MutexLock. + Mutex(const volatile Mutex * /*ignored*/) {} // NOLINT(runtime/explicit) + + Mutex(const Mutex&) = delete; + Mutex& operator=(const Mutex&) = delete; +}; + +// ----------------------------------------------------------------------------- +// Mutex RAII Wrappers +// ----------------------------------------------------------------------------- + +// MutexLock +// +// `MutexLock` is a helper class, which acquires and releases a `Mutex` via +// RAII. +// +// Example: +// +// Class Foo { +// public: +// Foo::Bar* Baz() { +// MutexLock lock(&mu_); +// ... +// return bar; +// } +// +// private: +// Mutex mu_; +// }; +class ABSL_SCOPED_LOCKABLE MutexLock { + public: + // Constructors + + // Calls `mu->Lock()` and returns when that call returns. That is, `*mu` is + // guaranteed to be locked when this object is constructed. Requires that + // `mu` be dereferenceable. + explicit MutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) : mu_(mu) { + this->mu_->Lock(); + } + + // Like above, but calls `mu->LockWhen(cond)` instead. That is, in addition to + // the above, the condition given by `cond` is also guaranteed to hold when + // this object is constructed. + explicit MutexLock(Mutex *mu, const Condition &cond) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + this->mu_->LockWhen(cond); + } + + MutexLock(const MutexLock &) = delete; // NOLINT(runtime/mutex) + MutexLock(MutexLock&&) = delete; // NOLINT(runtime/mutex) + MutexLock& operator=(const MutexLock&) = delete; + MutexLock& operator=(MutexLock&&) = delete; + + ~MutexLock() ABSL_UNLOCK_FUNCTION() { this->mu_->Unlock(); } + + private: + Mutex *const mu_; +}; + +// ReaderMutexLock +// +// The `ReaderMutexLock` is a helper class, like `MutexLock`, which acquires and +// releases a shared lock on a `Mutex` via RAII. +class ABSL_SCOPED_LOCKABLE ReaderMutexLock { + public: + explicit ReaderMutexLock(Mutex *mu) ABSL_SHARED_LOCK_FUNCTION(mu) : mu_(mu) { + mu->ReaderLock(); + } + + explicit ReaderMutexLock(Mutex *mu, const Condition &cond) + ABSL_SHARED_LOCK_FUNCTION(mu) + : mu_(mu) { + mu->ReaderLockWhen(cond); + } + + ReaderMutexLock(const ReaderMutexLock&) = delete; + ReaderMutexLock(ReaderMutexLock&&) = delete; + ReaderMutexLock& operator=(const ReaderMutexLock&) = delete; + ReaderMutexLock& operator=(ReaderMutexLock&&) = delete; + + ~ReaderMutexLock() ABSL_UNLOCK_FUNCTION() { this->mu_->ReaderUnlock(); } + + private: + Mutex *const mu_; +}; + +// WriterMutexLock +// +// The `WriterMutexLock` is a helper class, like `MutexLock`, which acquires and +// releases a write (exclusive) lock on a `Mutex` via RAII. +class ABSL_SCOPED_LOCKABLE WriterMutexLock { + public: + explicit WriterMutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + mu->WriterLock(); + } + + explicit WriterMutexLock(Mutex *mu, const Condition &cond) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + mu->WriterLockWhen(cond); + } + + WriterMutexLock(const WriterMutexLock&) = delete; + WriterMutexLock(WriterMutexLock&&) = delete; + WriterMutexLock& operator=(const WriterMutexLock&) = delete; + WriterMutexLock& operator=(WriterMutexLock&&) = delete; + + ~WriterMutexLock() ABSL_UNLOCK_FUNCTION() { this->mu_->WriterUnlock(); } + + private: + Mutex *const mu_; +}; + +// ----------------------------------------------------------------------------- +// Condition +// ----------------------------------------------------------------------------- +// +// As noted above, `Mutex` contains a number of member functions which take a +// `Condition` as an argument; clients can wait for conditions to become `true` +// before attempting to acquire the mutex. These sections are known as +// "condition critical" sections. To use a `Condition`, you simply need to +// construct it, and use within an appropriate `Mutex` member function; +// everything else in the `Condition` class is an implementation detail. +// +// A `Condition` is specified as a function pointer which returns a boolean. +// `Condition` functions should be pure functions -- their results should depend +// only on passed arguments, should not consult any external state (such as +// clocks), and should have no side-effects, aside from debug logging. Any +// objects that the function may access should be limited to those which are +// constant while the mutex is blocked on the condition (e.g. a stack variable), +// or objects of state protected explicitly by the mutex. +// +// No matter which construction is used for `Condition`, the underlying +// function pointer / functor / callable must not throw any +// exceptions. Correctness of `Mutex` / `Condition` is not guaranteed in +// the face of a throwing `Condition`. (When Abseil is allowed to depend +// on C++17, these function pointers will be explicitly marked +// `noexcept`; until then this requirement cannot be enforced in the +// type system.) +// +// Note: to use a `Condition`, you need only construct it and pass it to a +// suitable `Mutex' member function, such as `Mutex::Await()`, or to the +// constructor of one of the scope guard classes. +// +// Example using LockWhen/Unlock: +// +// // assume count_ is not internal reference count +// int count_ ABSL_GUARDED_BY(mu_); +// Condition count_is_zero(+[](int *count) { return *count == 0; }, &count_); +// +// mu_.LockWhen(count_is_zero); +// // ... +// mu_.Unlock(); +// +// Example using a scope guard: +// +// { +// MutexLock lock(&mu_, count_is_zero); +// // ... +// } +// +// When multiple threads are waiting on exactly the same condition, make sure +// that they are constructed with the same parameters (same pointer to function +// + arg, or same pointer to object + method), so that the mutex implementation +// can avoid redundantly evaluating the same condition for each thread. +class Condition { + public: + // A Condition that returns the result of "(*func)(arg)" + Condition(bool (*func)(void *), void *arg); + + // Templated version for people who are averse to casts. + // + // To use a lambda, prepend it with unary plus, which converts the lambda + // into a function pointer: + // Condition(+[](T* t) { return ...; }, arg). + // + // Note: lambdas in this case must contain no bound variables. + // + // See class comment for performance advice. + template<typename T> + Condition(bool (*func)(T *), T *arg); + + // Templated version for invoking a method that returns a `bool`. + // + // `Condition(object, &Class::Method)` constructs a `Condition` that evaluates + // `object->Method()`. + // + // Implementation Note: `absl::internal::identity` is used to allow methods to + // come from base classes. A simpler signature like + // `Condition(T*, bool (T::*)())` does not suffice. + template<typename T> + Condition(T *object, bool (absl::internal::identity<T>::type::* method)()); + + // Same as above, for const members + template<typename T> + Condition(const T *object, + bool (absl::internal::identity<T>::type::* method)() const); + + // A Condition that returns the value of `*cond` + explicit Condition(const bool *cond); + + // Templated version for invoking a functor that returns a `bool`. + // This approach accepts pointers to non-mutable lambdas, `std::function`, + // the result of` std::bind` and user-defined functors that define + // `bool F::operator()() const`. + // + // Example: + // + // auto reached = [this, current]() { + // mu_.AssertReaderHeld(); // For annotalysis. + // return processed_ >= current; + // }; + // mu_.Await(Condition(&reached)); + // + // NOTE: never use "mu_.AssertHeld()" instead of "mu_.AssertReaderHeld()" in + // the lambda as it may be called when the mutex is being unlocked from a + // scope holding only a reader lock, which will make the assertion not + // fulfilled and crash the binary. + + // See class comment for performance advice. In particular, if there + // might be more than one waiter for the same condition, make sure + // that all waiters construct the condition with the same pointers. + + // Implementation note: The second template parameter ensures that this + // constructor doesn't participate in overload resolution if T doesn't have + // `bool operator() const`. + template <typename T, typename E = decltype( + static_cast<bool (T::*)() const>(&T::operator()))> + explicit Condition(const T *obj) + : Condition(obj, static_cast<bool (T::*)() const>(&T::operator())) {} + + // A Condition that always returns `true`. + static const Condition kTrue; + + // Evaluates the condition. + bool Eval() const; + + // Returns `true` if the two conditions are guaranteed to return the same + // value if evaluated at the same time, `false` if the evaluation *may* return + // different results. + // + // Two `Condition` values are guaranteed equal if both their `func` and `arg` + // components are the same. A null pointer is equivalent to a `true` + // condition. + static bool GuaranteedEqual(const Condition *a, const Condition *b); + + private: + typedef bool (*InternalFunctionType)(void * arg); + typedef bool (Condition::*InternalMethodType)(); + typedef bool (*InternalMethodCallerType)(void * arg, + InternalMethodType internal_method); + + bool (*eval_)(const Condition*); // Actual evaluator + InternalFunctionType function_; // function taking pointer returning bool + InternalMethodType method_; // method returning bool + void *arg_; // arg of function_ or object of method_ + + Condition(); // null constructor used only to create kTrue + + // Various functions eval_ can point to: + static bool CallVoidPtrFunction(const Condition*); + template <typename T> static bool CastAndCallFunction(const Condition* c); + template <typename T> static bool CastAndCallMethod(const Condition* c); +}; + +// ----------------------------------------------------------------------------- +// CondVar +// ----------------------------------------------------------------------------- +// +// A condition variable, reflecting state evaluated separately outside of the +// `Mutex` object, which can be signaled to wake callers. +// This class is not normally needed; use `Mutex` member functions such as +// `Mutex::Await()` and intrinsic `Condition` abstractions. In rare cases +// with many threads and many conditions, `CondVar` may be faster. +// +// The implementation may deliver signals to any condition variable at +// any time, even when no call to `Signal()` or `SignalAll()` is made; as a +// result, upon being awoken, you must check the logical condition you have +// been waiting upon. +// +// Examples: +// +// Usage for a thread waiting for some condition C protected by mutex mu: +// mu.Lock(); +// while (!C) { cv->Wait(&mu); } // releases and reacquires mu +// // C holds; process data +// mu.Unlock(); +// +// Usage to wake T is: +// mu.Lock(); +// // process data, possibly establishing C +// if (C) { cv->Signal(); } +// mu.Unlock(); +// +// If C may be useful to more than one waiter, use `SignalAll()` instead of +// `Signal()`. +// +// With this implementation it is efficient to use `Signal()/SignalAll()` inside +// the locked region; this usage can make reasoning about your program easier. +// +class CondVar { + public: + // A `CondVar` allocated on the heap or on the stack can use the this + // constructor. + CondVar(); + ~CondVar(); + + // CondVar::Wait() + // + // Atomically releases a `Mutex` and blocks on this condition variable. + // Waits until awakened by a call to `Signal()` or `SignalAll()` (or a + // spurious wakeup), then reacquires the `Mutex` and returns. + // + // Requires and ensures that the current thread holds the `Mutex`. + void Wait(Mutex *mu); + + // CondVar::WaitWithTimeout() + // + // Atomically releases a `Mutex` and blocks on this condition variable. + // Waits until awakened by a call to `Signal()` or `SignalAll()` (or a + // spurious wakeup), or until the timeout has expired, then reacquires + // the `Mutex` and returns. + // + // Returns true if the timeout has expired without this `CondVar` + // being signalled in any manner. If both the timeout has expired + // and this `CondVar` has been signalled, the implementation is free + // to return `true` or `false`. + // + // Requires and ensures that the current thread holds the `Mutex`. + bool WaitWithTimeout(Mutex *mu, absl::Duration timeout); + + // CondVar::WaitWithDeadline() + // + // Atomically releases a `Mutex` and blocks on this condition variable. + // Waits until awakened by a call to `Signal()` or `SignalAll()` (or a + // spurious wakeup), or until the deadline has passed, then reacquires + // the `Mutex` and returns. + // + // Deadlines in the past are equivalent to an immediate deadline. + // + // Returns true if the deadline has passed without this `CondVar` + // being signalled in any manner. If both the deadline has passed + // and this `CondVar` has been signalled, the implementation is free + // to return `true` or `false`. + // + // Requires and ensures that the current thread holds the `Mutex`. + bool WaitWithDeadline(Mutex *mu, absl::Time deadline); + + // CondVar::Signal() + // + // Signal this `CondVar`; wake at least one waiter if one exists. + void Signal(); + + // CondVar::SignalAll() + // + // Signal this `CondVar`; wake all waiters. + void SignalAll(); + + // CondVar::EnableDebugLog() + // + // Causes all subsequent uses of this `CondVar` to be logged via + // `ABSL_RAW_LOG(INFO)`. Log entries are tagged with `name` if `name != 0`. + // Note: this method substantially reduces `CondVar` performance. + void EnableDebugLog(const char *name); + + private: + bool WaitCommon(Mutex *mutex, synchronization_internal::KernelTimeout t); + void Remove(base_internal::PerThreadSynch *s); + void Wakeup(base_internal::PerThreadSynch *w); + std::atomic<intptr_t> cv_; // Condition variable state. + CondVar(const CondVar&) = delete; + CondVar& operator=(const CondVar&) = delete; +}; + + +// Variants of MutexLock. +// +// If you find yourself using one of these, consider instead using +// Mutex::Unlock() and/or if-statements for clarity. + +// MutexLockMaybe +// +// MutexLockMaybe is like MutexLock, but is a no-op when mu is null. +class ABSL_SCOPED_LOCKABLE MutexLockMaybe { + public: + explicit MutexLockMaybe(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + if (this->mu_ != nullptr) { + this->mu_->Lock(); + } + } + + explicit MutexLockMaybe(Mutex *mu, const Condition &cond) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + if (this->mu_ != nullptr) { + this->mu_->LockWhen(cond); + } + } + + ~MutexLockMaybe() ABSL_UNLOCK_FUNCTION() { + if (this->mu_ != nullptr) { this->mu_->Unlock(); } + } + + private: + Mutex *const mu_; + MutexLockMaybe(const MutexLockMaybe&) = delete; + MutexLockMaybe(MutexLockMaybe&&) = delete; + MutexLockMaybe& operator=(const MutexLockMaybe&) = delete; + MutexLockMaybe& operator=(MutexLockMaybe&&) = delete; +}; + +// ReleasableMutexLock +// +// ReleasableMutexLock is like MutexLock, but permits `Release()` of its +// mutex before destruction. `Release()` may be called at most once. +class ABSL_SCOPED_LOCKABLE ReleasableMutexLock { + public: + explicit ReleasableMutexLock(Mutex *mu) ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + this->mu_->Lock(); + } + + explicit ReleasableMutexLock(Mutex *mu, const Condition &cond) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mu) + : mu_(mu) { + this->mu_->LockWhen(cond); + } + + ~ReleasableMutexLock() ABSL_UNLOCK_FUNCTION() { + if (this->mu_ != nullptr) { this->mu_->Unlock(); } + } + + void Release() ABSL_UNLOCK_FUNCTION(); + + private: + Mutex *mu_; + ReleasableMutexLock(const ReleasableMutexLock&) = delete; + ReleasableMutexLock(ReleasableMutexLock&&) = delete; + ReleasableMutexLock& operator=(const ReleasableMutexLock&) = delete; + ReleasableMutexLock& operator=(ReleasableMutexLock&&) = delete; +}; + +inline Mutex::Mutex() : mu_(0) { + ABSL_TSAN_MUTEX_CREATE(this, __tsan_mutex_not_static); +} + +inline constexpr Mutex::Mutex(absl::ConstInitType) : mu_(0) {} + +inline CondVar::CondVar() : cv_(0) {} + +// static +template <typename T> +bool Condition::CastAndCallMethod(const Condition *c) { + typedef bool (T::*MemberType)(); + MemberType rm = reinterpret_cast<MemberType>(c->method_); + T *x = static_cast<T *>(c->arg_); + return (x->*rm)(); +} + +// static +template <typename T> +bool Condition::CastAndCallFunction(const Condition *c) { + typedef bool (*FuncType)(T *); + FuncType fn = reinterpret_cast<FuncType>(c->function_); + T *x = static_cast<T *>(c->arg_); + return (*fn)(x); +} + +template <typename T> +inline Condition::Condition(bool (*func)(T *), T *arg) + : eval_(&CastAndCallFunction<T>), + function_(reinterpret_cast<InternalFunctionType>(func)), + method_(nullptr), + arg_(const_cast<void *>(static_cast<const void *>(arg))) {} + +template <typename T> +inline Condition::Condition(T *object, + bool (absl::internal::identity<T>::type::*method)()) + : eval_(&CastAndCallMethod<T>), + function_(nullptr), + method_(reinterpret_cast<InternalMethodType>(method)), + arg_(object) {} + +template <typename T> +inline Condition::Condition(const T *object, + bool (absl::internal::identity<T>::type::*method)() + const) + : eval_(&CastAndCallMethod<T>), + function_(nullptr), + method_(reinterpret_cast<InternalMethodType>(method)), + arg_(reinterpret_cast<void *>(const_cast<T *>(object))) {} + +// Register a hook for profiling support. +// +// The function pointer registered here will be called whenever a mutex is +// contended. The callback is given the absl/base/cycleclock.h timestamp when +// waiting began. +// +// Calls to this function do not race or block, but there is no ordering +// guaranteed between calls to this function and call to the provided hook. +// In particular, the previously registered hook may still be called for some +// time after this function returns. +void RegisterMutexProfiler(void (*fn)(int64_t wait_timestamp)); + +// Register a hook for Mutex tracing. +// +// The function pointer registered here will be called whenever a mutex is +// contended. The callback is given an opaque handle to the contended mutex, +// an event name, and the number of wait cycles (as measured by +// //absl/base/internal/cycleclock.h, and which may not be real +// "cycle" counts.) +// +// The only event name currently sent is "slow release". +// +// This has the same memory ordering concerns as RegisterMutexProfiler() above. +void RegisterMutexTracer(void (*fn)(const char *msg, const void *obj, + int64_t wait_cycles)); + +// TODO(gfalcon): Combine RegisterMutexProfiler() and RegisterMutexTracer() +// into a single interface, since they are only ever called in pairs. + +// Register a hook for CondVar tracing. +// +// The function pointer registered here will be called here on various CondVar +// events. The callback is given an opaque handle to the CondVar object and +// a string identifying the event. This is thread-safe, but only a single +// tracer can be registered. +// +// Events that can be sent are "Wait", "Unwait", "Signal wakeup", and +// "SignalAll wakeup". +// +// This has the same memory ordering concerns as RegisterMutexProfiler() above. +void RegisterCondVarTracer(void (*fn)(const char *msg, const void *cv)); + +// Register a hook for symbolizing stack traces in deadlock detector reports. +// +// 'pc' is the program counter being symbolized, 'out' is the buffer to write +// into, and 'out_size' is the size of the buffer. This function can return +// false if symbolizing failed, or true if a NUL-terminated symbol was written +// to 'out.' +// +// This has the same memory ordering concerns as RegisterMutexProfiler() above. +// +// DEPRECATED: The default symbolizer function is absl::Symbolize() and the +// ability to register a different hook for symbolizing stack traces will be +// removed on or after 2023-05-01. +ABSL_DEPRECATED("absl::RegisterSymbolizer() is deprecated and will be removed " + "on or after 2023-05-01") +void RegisterSymbolizer(bool (*fn)(const void *pc, char *out, int out_size)); + +// EnableMutexInvariantDebugging() +// +// Enable or disable global support for Mutex invariant debugging. If enabled, +// then invariant predicates can be registered per-Mutex for debug checking. +// See Mutex::EnableInvariantDebugging(). +void EnableMutexInvariantDebugging(bool enabled); + +// When in debug mode, and when the feature has been enabled globally, the +// implementation will keep track of lock ordering and complain (or optionally +// crash) if a cycle is detected in the acquired-before graph. + +// Possible modes of operation for the deadlock detector in debug mode. +enum class OnDeadlockCycle { + kIgnore, // Neither report on nor attempt to track cycles in lock ordering + kReport, // Report lock cycles to stderr when detected + kAbort, // Report lock cycles to stderr when detected, then abort +}; + +// SetMutexDeadlockDetectionMode() +// +// Enable or disable global support for detection of potential deadlocks +// due to Mutex lock ordering inversions. When set to 'kIgnore', tracking of +// lock ordering is disabled. Otherwise, in debug builds, a lock ordering graph +// will be maintained internally, and detected cycles will be reported in +// the manner chosen here. +void SetMutexDeadlockDetectionMode(OnDeadlockCycle mode); + +ABSL_NAMESPACE_END +} // namespace absl + +// In some build configurations we pass --detect-odr-violations to the +// gold linker. This causes it to flag weak symbol overrides as ODR +// violations. Because ODR only applies to C++ and not C, +// --detect-odr-violations ignores symbols not mangled with C++ names. +// By changing our extension points to be extern "C", we dodge this +// check. +extern "C" { +void AbslInternalMutexYield(); +} // extern "C" + +#endif // ABSL_SYNCHRONIZATION_MUTEX_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/mutex_benchmark.cc b/third_party/abseil_cpp/absl/synchronization/mutex_benchmark.cc new file mode 100644 index 000000000000..933ea14f8f1d --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/mutex_benchmark.cc @@ -0,0 +1,224 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <mutex> // NOLINT(build/c++11) +#include <vector> + +#include "absl/base/config.h" +#include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/spinlock.h" +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/mutex.h" +#include "benchmark/benchmark.h" + +namespace { + +void BM_Mutex(benchmark::State& state) { + static absl::Mutex* mu = new absl::Mutex; + for (auto _ : state) { + absl::MutexLock lock(mu); + } +} +BENCHMARK(BM_Mutex)->UseRealTime()->Threads(1)->ThreadPerCpu(); + +static void DelayNs(int64_t ns, int* data) { + int64_t end = absl::base_internal::CycleClock::Now() + + ns * absl::base_internal::CycleClock::Frequency() / 1e9; + while (absl::base_internal::CycleClock::Now() < end) { + ++(*data); + benchmark::DoNotOptimize(*data); + } +} + +template <typename MutexType> +class RaiiLocker { + public: + explicit RaiiLocker(MutexType* mu) : mu_(mu) { mu_->Lock(); } + ~RaiiLocker() { mu_->Unlock(); } + private: + MutexType* mu_; +}; + +template <> +class RaiiLocker<std::mutex> { + public: + explicit RaiiLocker(std::mutex* mu) : mu_(mu) { mu_->lock(); } + ~RaiiLocker() { mu_->unlock(); } + private: + std::mutex* mu_; +}; + +template <typename MutexType> +void BM_Contended(benchmark::State& state) { + struct Shared { + MutexType mu; + int data = 0; + }; + static auto* shared = new Shared; + int local = 0; + for (auto _ : state) { + // Here we model both local work outside of the critical section as well as + // some work inside of the critical section. The idea is to capture some + // more or less realisitic contention levels. + // If contention is too low, the benchmark won't measure anything useful. + // If contention is unrealistically high, the benchmark will favor + // bad mutex implementations that block and otherwise distract threads + // from the mutex and shared state for as much as possible. + // To achieve this amount of local work is multiplied by number of threads + // to keep ratio between local work and critical section approximately + // equal regardless of number of threads. + DelayNs(100 * state.threads, &local); + RaiiLocker<MutexType> locker(&shared->mu); + DelayNs(state.range(0), &shared->data); + } +} + +BENCHMARK_TEMPLATE(BM_Contended, absl::Mutex) + ->UseRealTime() + // ThreadPerCpu poorly handles non-power-of-two CPU counts. + ->Threads(1) + ->Threads(2) + ->Threads(4) + ->Threads(6) + ->Threads(8) + ->Threads(12) + ->Threads(16) + ->Threads(24) + ->Threads(32) + ->Threads(48) + ->Threads(64) + ->Threads(96) + ->Threads(128) + ->Threads(192) + ->Threads(256) + // Some empirically chosen amounts of work in critical section. + // 1 is low contention, 200 is high contention and few values in between. + ->Arg(1) + ->Arg(20) + ->Arg(50) + ->Arg(200); + +BENCHMARK_TEMPLATE(BM_Contended, absl::base_internal::SpinLock) + ->UseRealTime() + // ThreadPerCpu poorly handles non-power-of-two CPU counts. + ->Threads(1) + ->Threads(2) + ->Threads(4) + ->Threads(6) + ->Threads(8) + ->Threads(12) + ->Threads(16) + ->Threads(24) + ->Threads(32) + ->Threads(48) + ->Threads(64) + ->Threads(96) + ->Threads(128) + ->Threads(192) + ->Threads(256) + // Some empirically chosen amounts of work in critical section. + // 1 is low contention, 200 is high contention and few values in between. + ->Arg(1) + ->Arg(20) + ->Arg(50) + ->Arg(200); + +BENCHMARK_TEMPLATE(BM_Contended, std::mutex) + ->UseRealTime() + // ThreadPerCpu poorly handles non-power-of-two CPU counts. + ->Threads(1) + ->Threads(2) + ->Threads(4) + ->Threads(6) + ->Threads(8) + ->Threads(12) + ->Threads(16) + ->Threads(24) + ->Threads(32) + ->Threads(48) + ->Threads(64) + ->Threads(96) + ->Threads(128) + ->Threads(192) + ->Threads(256) + // Some empirically chosen amounts of work in critical section. + // 1 is low contention, 200 is high contention and few values in between. + ->Arg(1) + ->Arg(20) + ->Arg(50) + ->Arg(200); + +// Measure the overhead of conditions on mutex release (when they must be +// evaluated). Mutex has (some) support for equivalence classes allowing +// Conditions with the same function/argument to potentially not be multiply +// evaluated. +// +// num_classes==0 is used for the special case of every waiter being distinct. +void BM_ConditionWaiters(benchmark::State& state) { + int num_classes = state.range(0); + int num_waiters = state.range(1); + + struct Helper { + static void Waiter(absl::BlockingCounter* init, absl::Mutex* m, int* p) { + init->DecrementCount(); + m->LockWhen(absl::Condition( + static_cast<bool (*)(int*)>([](int* v) { return *v == 0; }), p)); + m->Unlock(); + } + }; + + if (num_classes == 0) { + // No equivalence classes. + num_classes = num_waiters; + } + + absl::BlockingCounter init(num_waiters); + absl::Mutex mu; + std::vector<int> equivalence_classes(num_classes, 1); + + // Must be declared last to be destroyed first. + absl::synchronization_internal::ThreadPool pool(num_waiters); + + for (int i = 0; i < num_waiters; i++) { + // Mutex considers Conditions with the same function and argument + // to be equivalent. + pool.Schedule([&, i] { + Helper::Waiter(&init, &mu, &equivalence_classes[i % num_classes]); + }); + } + init.Wait(); + + for (auto _ : state) { + mu.Lock(); + mu.Unlock(); // Each unlock requires Condition evaluation for our waiters. + } + + mu.Lock(); + for (int i = 0; i < num_classes; i++) { + equivalence_classes[i] = 0; + } + mu.Unlock(); +} + +// Some configurations have higher thread limits than others. +#if defined(__linux__) && !defined(ABSL_HAVE_THREAD_SANITIZER) +constexpr int kMaxConditionWaiters = 8192; +#else +constexpr int kMaxConditionWaiters = 1024; +#endif +BENCHMARK(BM_ConditionWaiters)->RangePair(0, 2, 1, kMaxConditionWaiters); + +} // namespace diff --git a/third_party/abseil_cpp/absl/synchronization/mutex_test.cc b/third_party/abseil_cpp/absl/synchronization/mutex_test.cc new file mode 100644 index 000000000000..058f757b482f --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/mutex_test.cc @@ -0,0 +1,1706 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/mutex.h" + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include <algorithm> +#include <atomic> +#include <cstdlib> +#include <functional> +#include <memory> +#include <random> +#include <string> +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/sysinfo.h" +#include "absl/memory/memory.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace { + +// TODO(dmauro): Replace with a commandline flag. +static constexpr bool kExtendedTest = false; + +std::unique_ptr<absl::synchronization_internal::ThreadPool> CreatePool( + int threads) { + return absl::make_unique<absl::synchronization_internal::ThreadPool>(threads); +} + +std::unique_ptr<absl::synchronization_internal::ThreadPool> +CreateDefaultPool() { + return CreatePool(kExtendedTest ? 32 : 10); +} + +// Hack to schedule a function to run on a thread pool thread after a +// duration has elapsed. +static void ScheduleAfter(absl::synchronization_internal::ThreadPool *tp, + absl::Duration after, + const std::function<void()> &func) { + tp->Schedule([func, after] { + absl::SleepFor(after); + func(); + }); +} + +struct TestContext { + int iterations; + int threads; + int g0; // global 0 + int g1; // global 1 + absl::Mutex mu; + absl::CondVar cv; +}; + +// To test whether the invariant check call occurs +static std::atomic<bool> invariant_checked; + +static bool GetInvariantChecked() { + return invariant_checked.load(std::memory_order_relaxed); +} + +static void SetInvariantChecked(bool new_value) { + invariant_checked.store(new_value, std::memory_order_relaxed); +} + +static void CheckSumG0G1(void *v) { + TestContext *cxt = static_cast<TestContext *>(v); + ABSL_RAW_CHECK(cxt->g0 == -cxt->g1, "Error in CheckSumG0G1"); + SetInvariantChecked(true); +} + +static void TestMu(TestContext *cxt, int c) { + for (int i = 0; i != cxt->iterations; i++) { + absl::MutexLock l(&cxt->mu); + int a = cxt->g0 + 1; + cxt->g0 = a; + cxt->g1--; + } +} + +static void TestTry(TestContext *cxt, int c) { + for (int i = 0; i != cxt->iterations; i++) { + do { + std::this_thread::yield(); + } while (!cxt->mu.TryLock()); + int a = cxt->g0 + 1; + cxt->g0 = a; + cxt->g1--; + cxt->mu.Unlock(); + } +} + +static void TestR20ms(TestContext *cxt, int c) { + for (int i = 0; i != cxt->iterations; i++) { + absl::ReaderMutexLock l(&cxt->mu); + absl::SleepFor(absl::Milliseconds(20)); + cxt->mu.AssertReaderHeld(); + } +} + +static void TestRW(TestContext *cxt, int c) { + if ((c & 1) == 0) { + for (int i = 0; i != cxt->iterations; i++) { + absl::WriterMutexLock l(&cxt->mu); + cxt->g0++; + cxt->g1--; + cxt->mu.AssertHeld(); + cxt->mu.AssertReaderHeld(); + } + } else { + for (int i = 0; i != cxt->iterations; i++) { + absl::ReaderMutexLock l(&cxt->mu); + ABSL_RAW_CHECK(cxt->g0 == -cxt->g1, "Error in TestRW"); + cxt->mu.AssertReaderHeld(); + } + } +} + +struct MyContext { + int target; + TestContext *cxt; + bool MyTurn(); +}; + +bool MyContext::MyTurn() { + TestContext *cxt = this->cxt; + return cxt->g0 == this->target || cxt->g0 == cxt->iterations; +} + +static void TestAwait(TestContext *cxt, int c) { + MyContext mc; + mc.target = c; + mc.cxt = cxt; + absl::MutexLock l(&cxt->mu); + cxt->mu.AssertHeld(); + while (cxt->g0 < cxt->iterations) { + cxt->mu.Await(absl::Condition(&mc, &MyContext::MyTurn)); + ABSL_RAW_CHECK(mc.MyTurn(), "Error in TestAwait"); + cxt->mu.AssertHeld(); + if (cxt->g0 < cxt->iterations) { + int a = cxt->g0 + 1; + cxt->g0 = a; + mc.target += cxt->threads; + } + } +} + +static void TestSignalAll(TestContext *cxt, int c) { + int target = c; + absl::MutexLock l(&cxt->mu); + cxt->mu.AssertHeld(); + while (cxt->g0 < cxt->iterations) { + while (cxt->g0 != target && cxt->g0 != cxt->iterations) { + cxt->cv.Wait(&cxt->mu); + } + if (cxt->g0 < cxt->iterations) { + int a = cxt->g0 + 1; + cxt->g0 = a; + cxt->cv.SignalAll(); + target += cxt->threads; + } + } +} + +static void TestSignal(TestContext *cxt, int c) { + ABSL_RAW_CHECK(cxt->threads == 2, "TestSignal should use 2 threads"); + int target = c; + absl::MutexLock l(&cxt->mu); + cxt->mu.AssertHeld(); + while (cxt->g0 < cxt->iterations) { + while (cxt->g0 != target && cxt->g0 != cxt->iterations) { + cxt->cv.Wait(&cxt->mu); + } + if (cxt->g0 < cxt->iterations) { + int a = cxt->g0 + 1; + cxt->g0 = a; + cxt->cv.Signal(); + target += cxt->threads; + } + } +} + +static void TestCVTimeout(TestContext *cxt, int c) { + int target = c; + absl::MutexLock l(&cxt->mu); + cxt->mu.AssertHeld(); + while (cxt->g0 < cxt->iterations) { + while (cxt->g0 != target && cxt->g0 != cxt->iterations) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(100)); + } + if (cxt->g0 < cxt->iterations) { + int a = cxt->g0 + 1; + cxt->g0 = a; + cxt->cv.SignalAll(); + target += cxt->threads; + } + } +} + +static bool G0GE2(TestContext *cxt) { return cxt->g0 >= 2; } + +static void TestTime(TestContext *cxt, int c, bool use_cv) { + ABSL_RAW_CHECK(cxt->iterations == 1, "TestTime should only use 1 iteration"); + ABSL_RAW_CHECK(cxt->threads > 2, "TestTime should use more than 2 threads"); + const bool kFalse = false; + absl::Condition false_cond(&kFalse); + absl::Condition g0ge2(G0GE2, cxt); + if (c == 0) { + absl::MutexLock l(&cxt->mu); + + absl::Time start = absl::Now(); + if (use_cv) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(1)); + } else { + ABSL_RAW_CHECK(!cxt->mu.AwaitWithTimeout(false_cond, absl::Seconds(1)), + "TestTime failed"); + } + absl::Duration elapsed = absl::Now() - start; + ABSL_RAW_CHECK( + absl::Seconds(0.9) <= elapsed && elapsed <= absl::Seconds(2.0), + "TestTime failed"); + ABSL_RAW_CHECK(cxt->g0 == 1, "TestTime failed"); + + start = absl::Now(); + if (use_cv) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(1)); + } else { + ABSL_RAW_CHECK(!cxt->mu.AwaitWithTimeout(false_cond, absl::Seconds(1)), + "TestTime failed"); + } + elapsed = absl::Now() - start; + ABSL_RAW_CHECK( + absl::Seconds(0.9) <= elapsed && elapsed <= absl::Seconds(2.0), + "TestTime failed"); + cxt->g0++; + if (use_cv) { + cxt->cv.Signal(); + } + + start = absl::Now(); + if (use_cv) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(4)); + } else { + ABSL_RAW_CHECK(!cxt->mu.AwaitWithTimeout(false_cond, absl::Seconds(4)), + "TestTime failed"); + } + elapsed = absl::Now() - start; + ABSL_RAW_CHECK( + absl::Seconds(3.9) <= elapsed && elapsed <= absl::Seconds(6.0), + "TestTime failed"); + ABSL_RAW_CHECK(cxt->g0 >= 3, "TestTime failed"); + + start = absl::Now(); + if (use_cv) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(1)); + } else { + ABSL_RAW_CHECK(!cxt->mu.AwaitWithTimeout(false_cond, absl::Seconds(1)), + "TestTime failed"); + } + elapsed = absl::Now() - start; + ABSL_RAW_CHECK( + absl::Seconds(0.9) <= elapsed && elapsed <= absl::Seconds(2.0), + "TestTime failed"); + if (use_cv) { + cxt->cv.SignalAll(); + } + + start = absl::Now(); + if (use_cv) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(1)); + } else { + ABSL_RAW_CHECK(!cxt->mu.AwaitWithTimeout(false_cond, absl::Seconds(1)), + "TestTime failed"); + } + elapsed = absl::Now() - start; + ABSL_RAW_CHECK(absl::Seconds(0.9) <= elapsed && + elapsed <= absl::Seconds(2.0), "TestTime failed"); + ABSL_RAW_CHECK(cxt->g0 == cxt->threads, "TestTime failed"); + + } else if (c == 1) { + absl::MutexLock l(&cxt->mu); + const absl::Time start = absl::Now(); + if (use_cv) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Milliseconds(500)); + } else { + ABSL_RAW_CHECK( + !cxt->mu.AwaitWithTimeout(false_cond, absl::Milliseconds(500)), + "TestTime failed"); + } + const absl::Duration elapsed = absl::Now() - start; + ABSL_RAW_CHECK( + absl::Seconds(0.4) <= elapsed && elapsed <= absl::Seconds(0.9), + "TestTime failed"); + cxt->g0++; + } else if (c == 2) { + absl::MutexLock l(&cxt->mu); + if (use_cv) { + while (cxt->g0 < 2) { + cxt->cv.WaitWithTimeout(&cxt->mu, absl::Seconds(100)); + } + } else { + ABSL_RAW_CHECK(cxt->mu.AwaitWithTimeout(g0ge2, absl::Seconds(100)), + "TestTime failed"); + } + cxt->g0++; + } else { + absl::MutexLock l(&cxt->mu); + if (use_cv) { + while (cxt->g0 < 2) { + cxt->cv.Wait(&cxt->mu); + } + } else { + cxt->mu.Await(g0ge2); + } + cxt->g0++; + } +} + +static void TestMuTime(TestContext *cxt, int c) { TestTime(cxt, c, false); } + +static void TestCVTime(TestContext *cxt, int c) { TestTime(cxt, c, true); } + +static void EndTest(int *c0, int *c1, absl::Mutex *mu, absl::CondVar *cv, + const std::function<void(int)>& cb) { + mu->Lock(); + int c = (*c0)++; + mu->Unlock(); + cb(c); + absl::MutexLock l(mu); + (*c1)++; + cv->Signal(); +} + +// Code common to RunTest() and RunTestWithInvariantDebugging(). +static int RunTestCommon(TestContext *cxt, void (*test)(TestContext *cxt, int), + int threads, int iterations, int operations) { + absl::Mutex mu2; + absl::CondVar cv2; + int c0 = 0; + int c1 = 0; + cxt->g0 = 0; + cxt->g1 = 0; + cxt->iterations = iterations; + cxt->threads = threads; + absl::synchronization_internal::ThreadPool tp(threads); + for (int i = 0; i != threads; i++) { + tp.Schedule(std::bind(&EndTest, &c0, &c1, &mu2, &cv2, + std::function<void(int)>( + std::bind(test, cxt, std::placeholders::_1)))); + } + mu2.Lock(); + while (c1 != threads) { + cv2.Wait(&mu2); + } + mu2.Unlock(); + return cxt->g0; +} + +// Basis for the parameterized tests configured below. +static int RunTest(void (*test)(TestContext *cxt, int), int threads, + int iterations, int operations) { + TestContext cxt; + return RunTestCommon(&cxt, test, threads, iterations, operations); +} + +// Like RunTest(), but sets an invariant on the tested Mutex and +// verifies that the invariant check happened. The invariant function +// will be passed the TestContext* as its arg and must call +// SetInvariantChecked(true); +#if !defined(ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED) +static int RunTestWithInvariantDebugging(void (*test)(TestContext *cxt, int), + int threads, int iterations, + int operations, + void (*invariant)(void *)) { + absl::EnableMutexInvariantDebugging(true); + SetInvariantChecked(false); + TestContext cxt; + cxt.mu.EnableInvariantDebugging(invariant, &cxt); + int ret = RunTestCommon(&cxt, test, threads, iterations, operations); + ABSL_RAW_CHECK(GetInvariantChecked(), "Invariant not checked"); + absl::EnableMutexInvariantDebugging(false); // Restore. + return ret; +} +#endif + +// -------------------------------------------------------- +// Test for fix of bug in TryRemove() +struct TimeoutBugStruct { + absl::Mutex mu; + bool a; + int a_waiter_count; +}; + +static void WaitForA(TimeoutBugStruct *x) { + x->mu.LockWhen(absl::Condition(&x->a)); + x->a_waiter_count--; + x->mu.Unlock(); +} + +static bool NoAWaiters(TimeoutBugStruct *x) { return x->a_waiter_count == 0; } + +// Test that a CondVar.Wait(&mutex) can un-block a call to mutex.Await() in +// another thread. +TEST(Mutex, CondVarWaitSignalsAwait) { + // Use a struct so the lock annotations apply. + struct { + absl::Mutex barrier_mu; + bool barrier ABSL_GUARDED_BY(barrier_mu) = false; + + absl::Mutex release_mu; + bool release ABSL_GUARDED_BY(release_mu) = false; + absl::CondVar released_cv; + } state; + + auto pool = CreateDefaultPool(); + + // Thread A. Sets barrier, waits for release using Mutex::Await, then + // signals released_cv. + pool->Schedule([&state] { + state.release_mu.Lock(); + + state.barrier_mu.Lock(); + state.barrier = true; + state.barrier_mu.Unlock(); + + state.release_mu.Await(absl::Condition(&state.release)); + state.released_cv.Signal(); + state.release_mu.Unlock(); + }); + + state.barrier_mu.LockWhen(absl::Condition(&state.barrier)); + state.barrier_mu.Unlock(); + state.release_mu.Lock(); + // Thread A is now blocked on release by way of Mutex::Await(). + + // Set release. Calling released_cv.Wait() should un-block thread A, + // which will signal released_cv. If not, the test will hang. + state.release = true; + state.released_cv.Wait(&state.release_mu); + state.release_mu.Unlock(); +} + +// Test that a CondVar.WaitWithTimeout(&mutex) can un-block a call to +// mutex.Await() in another thread. +TEST(Mutex, CondVarWaitWithTimeoutSignalsAwait) { + // Use a struct so the lock annotations apply. + struct { + absl::Mutex barrier_mu; + bool barrier ABSL_GUARDED_BY(barrier_mu) = false; + + absl::Mutex release_mu; + bool release ABSL_GUARDED_BY(release_mu) = false; + absl::CondVar released_cv; + } state; + + auto pool = CreateDefaultPool(); + + // Thread A. Sets barrier, waits for release using Mutex::Await, then + // signals released_cv. + pool->Schedule([&state] { + state.release_mu.Lock(); + + state.barrier_mu.Lock(); + state.barrier = true; + state.barrier_mu.Unlock(); + + state.release_mu.Await(absl::Condition(&state.release)); + state.released_cv.Signal(); + state.release_mu.Unlock(); + }); + + state.barrier_mu.LockWhen(absl::Condition(&state.barrier)); + state.barrier_mu.Unlock(); + state.release_mu.Lock(); + // Thread A is now blocked on release by way of Mutex::Await(). + + // Set release. Calling released_cv.Wait() should un-block thread A, + // which will signal released_cv. If not, the test will hang. + state.release = true; + EXPECT_TRUE( + !state.released_cv.WaitWithTimeout(&state.release_mu, absl::Seconds(10))) + << "; Unrecoverable test failure: CondVar::WaitWithTimeout did not " + "unblock the absl::Mutex::Await call in another thread."; + + state.release_mu.Unlock(); +} + +// Test for regression of a bug in loop of TryRemove() +TEST(Mutex, MutexTimeoutBug) { + auto tp = CreateDefaultPool(); + + TimeoutBugStruct x; + x.a = false; + x.a_waiter_count = 2; + tp->Schedule(std::bind(&WaitForA, &x)); + tp->Schedule(std::bind(&WaitForA, &x)); + absl::SleepFor(absl::Seconds(1)); // Allow first two threads to hang. + // The skip field of the second will point to the first because there are + // only two. + + // Now cause a thread waiting on an always-false to time out + // This would deadlock when the bug was present. + bool always_false = false; + x.mu.LockWhenWithTimeout(absl::Condition(&always_false), + absl::Milliseconds(500)); + + // if we get here, the bug is not present. Cleanup the state. + + x.a = true; // wakeup the two waiters on A + x.mu.Await(absl::Condition(&NoAWaiters, &x)); // wait for them to exit + x.mu.Unlock(); +} + +struct CondVarWaitDeadlock : testing::TestWithParam<int> { + absl::Mutex mu; + absl::CondVar cv; + bool cond1 = false; + bool cond2 = false; + bool read_lock1; + bool read_lock2; + bool signal_unlocked; + + CondVarWaitDeadlock() { + read_lock1 = GetParam() & (1 << 0); + read_lock2 = GetParam() & (1 << 1); + signal_unlocked = GetParam() & (1 << 2); + } + + void Waiter1() { + if (read_lock1) { + mu.ReaderLock(); + while (!cond1) { + cv.Wait(&mu); + } + mu.ReaderUnlock(); + } else { + mu.Lock(); + while (!cond1) { + cv.Wait(&mu); + } + mu.Unlock(); + } + } + + void Waiter2() { + if (read_lock2) { + mu.ReaderLockWhen(absl::Condition(&cond2)); + mu.ReaderUnlock(); + } else { + mu.LockWhen(absl::Condition(&cond2)); + mu.Unlock(); + } + } +}; + +// Test for a deadlock bug in Mutex::Fer(). +// The sequence of events that lead to the deadlock is: +// 1. waiter1 blocks on cv in read mode (mu bits = 0). +// 2. waiter2 blocks on mu in either mode (mu bits = kMuWait). +// 3. main thread locks mu, sets cond1, unlocks mu (mu bits = kMuWait). +// 4. main thread signals on cv and this eventually calls Mutex::Fer(). +// Currently Fer wakes waiter1 since mu bits = kMuWait (mutex is unlocked). +// Before the bug fix Fer neither woke waiter1 nor queued it on mutex, +// which resulted in deadlock. +TEST_P(CondVarWaitDeadlock, Test) { + auto waiter1 = CreatePool(1); + auto waiter2 = CreatePool(1); + waiter1->Schedule([this] { this->Waiter1(); }); + waiter2->Schedule([this] { this->Waiter2(); }); + + // Wait while threads block (best-effort is fine). + absl::SleepFor(absl::Milliseconds(100)); + + // Wake condwaiter. + mu.Lock(); + cond1 = true; + if (signal_unlocked) { + mu.Unlock(); + cv.Signal(); + } else { + cv.Signal(); + mu.Unlock(); + } + waiter1.reset(); // "join" waiter1 + + // Wake waiter. + mu.Lock(); + cond2 = true; + mu.Unlock(); + waiter2.reset(); // "join" waiter2 +} + +INSTANTIATE_TEST_SUITE_P(CondVarWaitDeadlockTest, CondVarWaitDeadlock, + ::testing::Range(0, 8), + ::testing::PrintToStringParamName()); + +// -------------------------------------------------------- +// Test for fix of bug in DequeueAllWakeable() +// Bug was that if there was more than one waiting reader +// and all should be woken, the most recently blocked one +// would not be. + +struct DequeueAllWakeableBugStruct { + absl::Mutex mu; + absl::Mutex mu2; // protects all fields below + int unfinished_count; // count of unfinished readers; under mu2 + bool done1; // unfinished_count == 0; under mu2 + int finished_count; // count of finished readers, under mu2 + bool done2; // finished_count == 0; under mu2 +}; + +// Test for regression of a bug in loop of DequeueAllWakeable() +static void AcquireAsReader(DequeueAllWakeableBugStruct *x) { + x->mu.ReaderLock(); + x->mu2.Lock(); + x->unfinished_count--; + x->done1 = (x->unfinished_count == 0); + x->mu2.Unlock(); + // make sure that both readers acquired mu before we release it. + absl::SleepFor(absl::Seconds(2)); + x->mu.ReaderUnlock(); + + x->mu2.Lock(); + x->finished_count--; + x->done2 = (x->finished_count == 0); + x->mu2.Unlock(); +} + +// Test for regression of a bug in loop of DequeueAllWakeable() +TEST(Mutex, MutexReaderWakeupBug) { + auto tp = CreateDefaultPool(); + + DequeueAllWakeableBugStruct x; + x.unfinished_count = 2; + x.done1 = false; + x.finished_count = 2; + x.done2 = false; + x.mu.Lock(); // acquire mu exclusively + // queue two thread that will block on reader locks on x.mu + tp->Schedule(std::bind(&AcquireAsReader, &x)); + tp->Schedule(std::bind(&AcquireAsReader, &x)); + absl::SleepFor(absl::Seconds(1)); // give time for reader threads to block + x.mu.Unlock(); // wake them up + + // both readers should finish promptly + EXPECT_TRUE( + x.mu2.LockWhenWithTimeout(absl::Condition(&x.done1), absl::Seconds(10))); + x.mu2.Unlock(); + + EXPECT_TRUE( + x.mu2.LockWhenWithTimeout(absl::Condition(&x.done2), absl::Seconds(10))); + x.mu2.Unlock(); +} + +struct LockWhenTestStruct { + absl::Mutex mu1; + bool cond = false; + + absl::Mutex mu2; + bool waiting = false; +}; + +static bool LockWhenTestIsCond(LockWhenTestStruct* s) { + s->mu2.Lock(); + s->waiting = true; + s->mu2.Unlock(); + return s->cond; +} + +static void LockWhenTestWaitForIsCond(LockWhenTestStruct* s) { + s->mu1.LockWhen(absl::Condition(&LockWhenTestIsCond, s)); + s->mu1.Unlock(); +} + +TEST(Mutex, LockWhen) { + LockWhenTestStruct s; + + std::thread t(LockWhenTestWaitForIsCond, &s); + s.mu2.LockWhen(absl::Condition(&s.waiting)); + s.mu2.Unlock(); + + s.mu1.Lock(); + s.cond = true; + s.mu1.Unlock(); + + t.join(); +} + +TEST(Mutex, LockWhenGuard) { + absl::Mutex mu; + int n = 30; + bool done = false; + + // We don't inline the lambda because the conversion is ambiguous in MSVC. + bool (*cond_eq_10)(int *) = [](int *p) { return *p == 10; }; + bool (*cond_lt_10)(int *) = [](int *p) { return *p < 10; }; + + std::thread t1([&mu, &n, &done, cond_eq_10]() { + absl::ReaderMutexLock lock(&mu, absl::Condition(cond_eq_10, &n)); + done = true; + }); + + std::thread t2[10]; + for (std::thread &t : t2) { + t = std::thread([&mu, &n, cond_lt_10]() { + absl::WriterMutexLock lock(&mu, absl::Condition(cond_lt_10, &n)); + ++n; + }); + } + + { + absl::MutexLock lock(&mu); + n = 0; + } + + for (std::thread &t : t2) t.join(); + t1.join(); + + EXPECT_TRUE(done); + EXPECT_EQ(n, 10); +} + +// -------------------------------------------------------- +// The following test requires Mutex::ReaderLock to be a real shared +// lock, which is not the case in all builds. +#if !defined(ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE) + +// Test for fix of bug in UnlockSlow() that incorrectly decremented the reader +// count when putting a thread to sleep waiting for a false condition when the +// lock was not held. + +// For this bug to strike, we make a thread wait on a free mutex with no +// waiters by causing its wakeup condition to be false. Then the +// next two acquirers must be readers. The bug causes the lock +// to be released when one reader unlocks, rather than both. + +struct ReaderDecrementBugStruct { + bool cond; // to delay first thread (under mu) + int done; // reference count (under mu) + absl::Mutex mu; + + bool waiting_on_cond; // under mu2 + bool have_reader_lock; // under mu2 + bool complete; // under mu2 + absl::Mutex mu2; // > mu +}; + +// L >= mu, L < mu_waiting_on_cond +static bool IsCond(void *v) { + ReaderDecrementBugStruct *x = reinterpret_cast<ReaderDecrementBugStruct *>(v); + x->mu2.Lock(); + x->waiting_on_cond = true; + x->mu2.Unlock(); + return x->cond; +} + +// L >= mu +static bool AllDone(void *v) { + ReaderDecrementBugStruct *x = reinterpret_cast<ReaderDecrementBugStruct *>(v); + return x->done == 0; +} + +// L={} +static void WaitForCond(ReaderDecrementBugStruct *x) { + absl::Mutex dummy; + absl::MutexLock l(&dummy); + x->mu.LockWhen(absl::Condition(&IsCond, x)); + x->done--; + x->mu.Unlock(); +} + +// L={} +static void GetReadLock(ReaderDecrementBugStruct *x) { + x->mu.ReaderLock(); + x->mu2.Lock(); + x->have_reader_lock = true; + x->mu2.Await(absl::Condition(&x->complete)); + x->mu2.Unlock(); + x->mu.ReaderUnlock(); + x->mu.Lock(); + x->done--; + x->mu.Unlock(); +} + +// Test for reader counter being decremented incorrectly by waiter +// with false condition. +TEST(Mutex, MutexReaderDecrementBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { + ReaderDecrementBugStruct x; + x.cond = false; + x.waiting_on_cond = false; + x.have_reader_lock = false; + x.complete = false; + x.done = 2; // initial ref count + + // Run WaitForCond() and wait for it to sleep + std::thread thread1(WaitForCond, &x); + x.mu2.LockWhen(absl::Condition(&x.waiting_on_cond)); + x.mu2.Unlock(); + + // Run GetReadLock(), and wait for it to get the read lock + std::thread thread2(GetReadLock, &x); + x.mu2.LockWhen(absl::Condition(&x.have_reader_lock)); + x.mu2.Unlock(); + + // Get the reader lock ourselves, and release it. + x.mu.ReaderLock(); + x.mu.ReaderUnlock(); + + // The lock should be held in read mode by GetReadLock(). + // If we have the bug, the lock will be free. + x.mu.AssertReaderHeld(); + + // Wake up all the threads. + x.mu2.Lock(); + x.complete = true; + x.mu2.Unlock(); + + // TODO(delesley): turn on analysis once lock upgrading is supported. + // (This call upgrades the lock from shared to exclusive.) + x.mu.Lock(); + x.cond = true; + x.mu.Await(absl::Condition(&AllDone, &x)); + x.mu.Unlock(); + + thread1.join(); + thread2.join(); +} +#endif // !ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE + +// Test that we correctly handle the situation when a lock is +// held and then destroyed (w/o unlocking). +#ifdef ABSL_HAVE_THREAD_SANITIZER +// TSAN reports errors when locked Mutexes are destroyed. +TEST(Mutex, DISABLED_LockedMutexDestructionBug) NO_THREAD_SAFETY_ANALYSIS { +#else +TEST(Mutex, LockedMutexDestructionBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { +#endif + for (int i = 0; i != 10; i++) { + // Create, lock and destroy 10 locks. + const int kNumLocks = 10; + auto mu = absl::make_unique<absl::Mutex[]>(kNumLocks); + for (int j = 0; j != kNumLocks; j++) { + if ((j % 2) == 0) { + mu[j].WriterLock(); + } else { + mu[j].ReaderLock(); + } + } + } +} + +// -------------------------------------------------------- +// Test for bug with pattern of readers using a condvar. The bug was that if a +// reader went to sleep on a condition variable while one or more other readers +// held the lock, but there were no waiters, the reader count (held in the +// mutex word) would be lost. (This is because Enqueue() had at one time +// always placed the thread on the Mutex queue. Later (CL 4075610), to +// tolerate re-entry into Mutex from a Condition predicate, Enqueue() was +// changed so that it could also place a thread on a condition-variable. This +// introduced the case where Enqueue() returned with an empty queue, and this +// case was handled incorrectly in one place.) + +static void ReaderForReaderOnCondVar(absl::Mutex *mu, absl::CondVar *cv, + int *running) { + std::random_device dev; + std::mt19937 gen(dev()); + std::uniform_int_distribution<int> random_millis(0, 15); + mu->ReaderLock(); + while (*running == 3) { + absl::SleepFor(absl::Milliseconds(random_millis(gen))); + cv->WaitWithTimeout(mu, absl::Milliseconds(random_millis(gen))); + } + mu->ReaderUnlock(); + mu->Lock(); + (*running)--; + mu->Unlock(); +} + +struct True { + template <class... Args> + bool operator()(Args...) const { + return true; + } +}; + +struct DerivedTrue : True {}; + +TEST(Mutex, FunctorCondition) { + { // Variadic + True f; + EXPECT_TRUE(absl::Condition(&f).Eval()); + } + + { // Inherited + DerivedTrue g; + EXPECT_TRUE(absl::Condition(&g).Eval()); + } + + { // lambda + int value = 3; + auto is_zero = [&value] { return value == 0; }; + absl::Condition c(&is_zero); + EXPECT_FALSE(c.Eval()); + value = 0; + EXPECT_TRUE(c.Eval()); + } + + { // bind + int value = 0; + auto is_positive = std::bind(std::less<int>(), 0, std::cref(value)); + absl::Condition c(&is_positive); + EXPECT_FALSE(c.Eval()); + value = 1; + EXPECT_TRUE(c.Eval()); + } + + { // std::function + int value = 3; + std::function<bool()> is_zero = [&value] { return value == 0; }; + absl::Condition c(&is_zero); + EXPECT_FALSE(c.Eval()); + value = 0; + EXPECT_TRUE(c.Eval()); + } +} + +static bool IntIsZero(int *x) { return *x == 0; } + +// Test for reader waiting condition variable when there are other readers +// but no waiters. +TEST(Mutex, TestReaderOnCondVar) { + auto tp = CreateDefaultPool(); + absl::Mutex mu; + absl::CondVar cv; + int running = 3; + tp->Schedule(std::bind(&ReaderForReaderOnCondVar, &mu, &cv, &running)); + tp->Schedule(std::bind(&ReaderForReaderOnCondVar, &mu, &cv, &running)); + absl::SleepFor(absl::Seconds(2)); + mu.Lock(); + running--; + mu.Await(absl::Condition(&IntIsZero, &running)); + mu.Unlock(); +} + +// -------------------------------------------------------- +struct AcquireFromConditionStruct { + absl::Mutex mu0; // protects value, done + int value; // times condition function is called; under mu0, + bool done; // done with test? under mu0 + absl::Mutex mu1; // used to attempt to mess up state of mu0 + absl::CondVar cv; // so the condition function can be invoked from + // CondVar::Wait(). +}; + +static bool ConditionWithAcquire(AcquireFromConditionStruct *x) { + x->value++; // count times this function is called + + if (x->value == 2 || x->value == 3) { + // On the second and third invocation of this function, sleep for 100ms, + // but with the side-effect of altering the state of a Mutex other than + // than one for which this is a condition. The spec now explicitly allows + // this side effect; previously it did not. it was illegal. + bool always_false = false; + x->mu1.LockWhenWithTimeout(absl::Condition(&always_false), + absl::Milliseconds(100)); + x->mu1.Unlock(); + } + ABSL_RAW_CHECK(x->value < 4, "should not be invoked a fourth time"); + + // We arrange for the condition to return true on only the 2nd and 3rd calls. + return x->value == 2 || x->value == 3; +} + +static void WaitForCond2(AcquireFromConditionStruct *x) { + // wait for cond0 to become true + x->mu0.LockWhen(absl::Condition(&ConditionWithAcquire, x)); + x->done = true; + x->mu0.Unlock(); +} + +// Test for Condition whose function acquires other Mutexes +TEST(Mutex, AcquireFromCondition) { + auto tp = CreateDefaultPool(); + + AcquireFromConditionStruct x; + x.value = 0; + x.done = false; + tp->Schedule( + std::bind(&WaitForCond2, &x)); // run WaitForCond2() in a thread T + // T will hang because the first invocation of ConditionWithAcquire() will + // return false. + absl::SleepFor(absl::Milliseconds(500)); // allow T time to hang + + x.mu0.Lock(); + x.cv.WaitWithTimeout(&x.mu0, absl::Milliseconds(500)); // wake T + // T will be woken because the Wait() will call ConditionWithAcquire() + // for the second time, and it will return true. + + x.mu0.Unlock(); + + // T will then acquire the lock and recheck its own condition. + // It will find the condition true, as this is the third invocation, + // but the use of another Mutex by the calling function will + // cause the old mutex implementation to think that the outer + // LockWhen() has timed out because the inner LockWhenWithTimeout() did. + // T will then check the condition a fourth time because it finds a + // timeout occurred. This should not happen in the new + // implementation that allows the Condition function to use Mutexes. + + // It should also succeed, even though the Condition function + // is being invoked from CondVar::Wait, and thus this thread + // is conceptually waiting both on the condition variable, and on mu2. + + x.mu0.LockWhen(absl::Condition(&x.done)); + x.mu0.Unlock(); +} + +TEST(Mutex, DeadlockDetector) { + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + + // check that we can call ForgetDeadlockInfo() on a lock with the lock held + absl::Mutex m1; + absl::Mutex m2; + absl::Mutex m3; + absl::Mutex m4; + + m1.Lock(); // m1 gets ID1 + m2.Lock(); // m2 gets ID2 + m3.Lock(); // m3 gets ID3 + m3.Unlock(); + m2.Unlock(); + // m1 still held + m1.ForgetDeadlockInfo(); // m1 loses ID + m2.Lock(); // m2 gets ID2 + m3.Lock(); // m3 gets ID3 + m4.Lock(); // m4 gets ID4 + m3.Unlock(); + m2.Unlock(); + m4.Unlock(); + m1.Unlock(); +} + +// Bazel has a test "warning" file that programs can write to if the +// test should pass with a warning. This class disables the warning +// file until it goes out of scope. +class ScopedDisableBazelTestWarnings { + public: + ScopedDisableBazelTestWarnings() { +#ifdef _WIN32 + char file[MAX_PATH]; + if (GetEnvironmentVariableA(kVarName, file, sizeof(file)) < sizeof(file)) { + warnings_output_file_ = file; + SetEnvironmentVariableA(kVarName, nullptr); + } +#else + const char *file = getenv(kVarName); + if (file != nullptr) { + warnings_output_file_ = file; + unsetenv(kVarName); + } +#endif + } + + ~ScopedDisableBazelTestWarnings() { + if (!warnings_output_file_.empty()) { +#ifdef _WIN32 + SetEnvironmentVariableA(kVarName, warnings_output_file_.c_str()); +#else + setenv(kVarName, warnings_output_file_.c_str(), 0); +#endif + } + } + + private: + static const char kVarName[]; + std::string warnings_output_file_; +}; +const char ScopedDisableBazelTestWarnings::kVarName[] = + "TEST_WARNINGS_OUTPUT_FILE"; + +#ifdef ABSL_HAVE_THREAD_SANITIZER +// This test intentionally creates deadlocks to test the deadlock detector. +TEST(Mutex, DISABLED_DeadlockDetectorBazelWarning) { +#else +TEST(Mutex, DeadlockDetectorBazelWarning) { +#endif + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kReport); + + // Cause deadlock detection to detect something, if it's + // compiled in and enabled. But turn off the bazel warning. + ScopedDisableBazelTestWarnings disable_bazel_test_warnings; + + absl::Mutex mu0; + absl::Mutex mu1; + bool got_mu0 = mu0.TryLock(); + mu1.Lock(); // acquire mu1 while holding mu0 + if (got_mu0) { + mu0.Unlock(); + } + if (mu0.TryLock()) { // try lock shouldn't cause deadlock detector to fire + mu0.Unlock(); + } + mu0.Lock(); // acquire mu0 while holding mu1; should get one deadlock + // report here + mu0.Unlock(); + mu1.Unlock(); + + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); +} + +// This test is tagged with NO_THREAD_SAFETY_ANALYSIS because the +// annotation-based static thread-safety analysis is not currently +// predicate-aware and cannot tell if the two for-loops that acquire and +// release the locks have the same predicates. +TEST(Mutex, DeadlockDetectorStressTest) ABSL_NO_THREAD_SAFETY_ANALYSIS { + // Stress test: Here we create a large number of locks and use all of them. + // If a deadlock detector keeps a full graph of lock acquisition order, + // it will likely be too slow for this test to pass. + const int n_locks = 1 << 17; + auto array_of_locks = absl::make_unique<absl::Mutex[]>(n_locks); + for (int i = 0; i < n_locks; i++) { + int end = std::min(n_locks, i + 5); + // acquire and then release locks i, i+1, ..., i+4 + for (int j = i; j < end; j++) { + array_of_locks[j].Lock(); + } + for (int j = i; j < end; j++) { + array_of_locks[j].Unlock(); + } + } +} + +#ifdef ABSL_HAVE_THREAD_SANITIZER +// TSAN reports errors when locked Mutexes are destroyed. +TEST(Mutex, DISABLED_DeadlockIdBug) NO_THREAD_SAFETY_ANALYSIS { +#else +TEST(Mutex, DeadlockIdBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { +#endif + // Test a scenario where a cached deadlock graph node id in the + // list of held locks is not invalidated when the corresponding + // mutex is deleted. + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + // Mutex that will be destroyed while being held + absl::Mutex *a = new absl::Mutex; + // Other mutexes needed by test + absl::Mutex b, c; + + // Hold mutex. + a->Lock(); + + // Force deadlock id assignment by acquiring another lock. + b.Lock(); + b.Unlock(); + + // Delete the mutex. The Mutex destructor tries to remove held locks, + // but the attempt isn't foolproof. It can fail if: + // (a) Deadlock detection is currently disabled. + // (b) The destruction is from another thread. + // We exploit (a) by temporarily disabling deadlock detection. + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kIgnore); + delete a; + absl::SetMutexDeadlockDetectionMode(absl::OnDeadlockCycle::kAbort); + + // Now acquire another lock which will force a deadlock id assignment. + // We should end up getting assigned the same deadlock id that was + // freed up when "a" was deleted, which will cause a spurious deadlock + // report if the held lock entry for "a" was not invalidated. + c.Lock(); + c.Unlock(); +} + +// -------------------------------------------------------- +// Test for timeouts/deadlines on condition waits that are specified using +// absl::Duration and absl::Time. For each waiting function we test with +// a timeout/deadline that has already expired/passed, one that is infinite +// and so never expires/passes, and one that will expire/pass in the near +// future. + +static absl::Duration TimeoutTestAllowedSchedulingDelay() { + // Note: we use a function here because Microsoft Visual Studio fails to + // properly initialize constexpr static absl::Duration variables. + return absl::Milliseconds(150); +} + +// Returns true if `actual_delay` is close enough to `expected_delay` to pass +// the timeouts/deadlines test. Otherwise, logs warnings and returns false. +ABSL_MUST_USE_RESULT +static bool DelayIsWithinBounds(absl::Duration expected_delay, + absl::Duration actual_delay) { + bool pass = true; + // Do not allow the observed delay to be less than expected. This may occur + // in practice due to clock skew or when the synchronization primitives use a + // different clock than absl::Now(), but these cases should be handled by the + // the retry mechanism in each TimeoutTest. + if (actual_delay < expected_delay) { + ABSL_RAW_LOG(WARNING, + "Actual delay %s was too short, expected %s (difference %s)", + absl::FormatDuration(actual_delay).c_str(), + absl::FormatDuration(expected_delay).c_str(), + absl::FormatDuration(actual_delay - expected_delay).c_str()); + pass = false; + } + // If the expected delay is <= zero then allow a small error tolerance, since + // we do not expect context switches to occur during test execution. + // Otherwise, thread scheduling delays may be substantial in rare cases, so + // tolerate up to kTimeoutTestAllowedSchedulingDelay of error. + absl::Duration tolerance = expected_delay <= absl::ZeroDuration() + ? absl::Milliseconds(10) + : TimeoutTestAllowedSchedulingDelay(); + if (actual_delay > expected_delay + tolerance) { + ABSL_RAW_LOG(WARNING, + "Actual delay %s was too long, expected %s (difference %s)", + absl::FormatDuration(actual_delay).c_str(), + absl::FormatDuration(expected_delay).c_str(), + absl::FormatDuration(actual_delay - expected_delay).c_str()); + pass = false; + } + return pass; +} + +// Parameters for TimeoutTest, below. +struct TimeoutTestParam { + // The file and line number (used for logging purposes only). + const char *from_file; + int from_line; + + // Should the absolute deadline API based on absl::Time be tested? If false, + // the relative deadline API based on absl::Duration is tested. + bool use_absolute_deadline; + + // The deadline/timeout used when calling the API being tested + // (e.g. Mutex::LockWhenWithDeadline). + absl::Duration wait_timeout; + + // The delay before the condition will be set true by the test code. If zero + // or negative, the condition is set true immediately (before calling the API + // being tested). Otherwise, if infinite, the condition is never set true. + // Otherwise a closure is scheduled for the future that sets the condition + // true. + absl::Duration satisfy_condition_delay; + + // The expected result of the condition after the call to the API being + // tested. Generally `true` means the condition was true when the API returns, + // `false` indicates an expected timeout. + bool expected_result; + + // The expected delay before the API under test returns. This is inherently + // flaky, so some slop is allowed (see `DelayIsWithinBounds` above), and the + // test keeps trying indefinitely until this constraint passes. + absl::Duration expected_delay; +}; + +// Print a `TimeoutTestParam` to a debug log. +std::ostream &operator<<(std::ostream &os, const TimeoutTestParam ¶m) { + return os << "from: " << param.from_file << ":" << param.from_line + << " use_absolute_deadline: " + << (param.use_absolute_deadline ? "true" : "false") + << " wait_timeout: " << param.wait_timeout + << " satisfy_condition_delay: " << param.satisfy_condition_delay + << " expected_result: " + << (param.expected_result ? "true" : "false") + << " expected_delay: " << param.expected_delay; +} + +std::string FormatString(const TimeoutTestParam ¶m) { + std::ostringstream os; + os << param; + return os.str(); +} + +// Like `thread::Executor::ScheduleAt` except: +// a) Delays zero or negative are executed immediately in the current thread. +// b) Infinite delays are never scheduled. +// c) Calls this test's `ScheduleAt` helper instead of using `pool` directly. +static void RunAfterDelay(absl::Duration delay, + absl::synchronization_internal::ThreadPool *pool, + const std::function<void()> &callback) { + if (delay <= absl::ZeroDuration()) { + callback(); // immediate + } else if (delay != absl::InfiniteDuration()) { + ScheduleAfter(pool, delay, callback); + } +} + +class TimeoutTest : public ::testing::Test, + public ::testing::WithParamInterface<TimeoutTestParam> {}; + +std::vector<TimeoutTestParam> MakeTimeoutTestParamValues() { + // The `finite` delay is a finite, relatively short, delay. We make it larger + // than our allowed scheduling delay (slop factor) to avoid confusion when + // diagnosing test failures. The other constants here have clear meanings. + const absl::Duration finite = 3 * TimeoutTestAllowedSchedulingDelay(); + const absl::Duration never = absl::InfiniteDuration(); + const absl::Duration negative = -absl::InfiniteDuration(); + const absl::Duration immediate = absl::ZeroDuration(); + + // Every test case is run twice; once using the absolute deadline API and once + // using the relative timeout API. + std::vector<TimeoutTestParam> values; + for (bool use_absolute_deadline : {false, true}) { + // Tests with a negative timeout (deadline in the past), which should + // immediately return current state of the condition. + + // The condition is already true: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + negative, // wait_timeout + immediate, // satisfy_condition_delay + true, // expected_result + immediate, // expected_delay + }); + + // The condition becomes true, but the timeout has already expired: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + negative, // wait_timeout + finite, // satisfy_condition_delay + false, // expected_result + immediate // expected_delay + }); + + // The condition never becomes true: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + negative, // wait_timeout + never, // satisfy_condition_delay + false, // expected_result + immediate // expected_delay + }); + + // Tests with an infinite timeout (deadline in the infinite future), which + // should only return when the condition becomes true. + + // The condition is already true: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + never, // wait_timeout + immediate, // satisfy_condition_delay + true, // expected_result + immediate // expected_delay + }); + + // The condition becomes true before the (infinite) expiry: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + never, // wait_timeout + finite, // satisfy_condition_delay + true, // expected_result + finite, // expected_delay + }); + + // Tests with a (small) finite timeout (deadline soon), with the condition + // becoming true both before and after its expiry. + + // The condition is already true: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + never, // wait_timeout + immediate, // satisfy_condition_delay + true, // expected_result + immediate // expected_delay + }); + + // The condition becomes true before the expiry: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + finite * 2, // wait_timeout + finite, // satisfy_condition_delay + true, // expected_result + finite // expected_delay + }); + + // The condition becomes true, but the timeout has already expired: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + finite, // wait_timeout + finite * 2, // satisfy_condition_delay + false, // expected_result + finite // expected_delay + }); + + // The condition never becomes true: + values.push_back(TimeoutTestParam{ + __FILE__, __LINE__, use_absolute_deadline, + finite, // wait_timeout + never, // satisfy_condition_delay + false, // expected_result + finite // expected_delay + }); + } + return values; +} + +// Instantiate `TimeoutTest` with `MakeTimeoutTestParamValues()`. +INSTANTIATE_TEST_SUITE_P(All, TimeoutTest, + testing::ValuesIn(MakeTimeoutTestParamValues())); + +TEST_P(TimeoutTest, Await) { + const TimeoutTestParam params = GetParam(); + ABSL_RAW_LOG(INFO, "Params: %s", FormatString(params).c_str()); + + // Because this test asserts bounds on scheduling delays it is flaky. To + // compensate it loops forever until it passes. Failures express as test + // timeouts, in which case the test log can be used to diagnose the issue. + for (int attempt = 1;; ++attempt) { + ABSL_RAW_LOG(INFO, "Attempt %d", attempt); + + absl::Mutex mu; + bool value = false; // condition value (under mu) + + std::unique_ptr<absl::synchronization_internal::ThreadPool> pool = + CreateDefaultPool(); + RunAfterDelay(params.satisfy_condition_delay, pool.get(), [&] { + absl::MutexLock l(&mu); + value = true; + }); + + absl::MutexLock lock(&mu); + absl::Time start_time = absl::Now(); + absl::Condition cond(&value); + bool result = + params.use_absolute_deadline + ? mu.AwaitWithDeadline(cond, start_time + params.wait_timeout) + : mu.AwaitWithTimeout(cond, params.wait_timeout); + if (DelayIsWithinBounds(params.expected_delay, absl::Now() - start_time)) { + EXPECT_EQ(params.expected_result, result); + break; + } + } +} + +TEST_P(TimeoutTest, LockWhen) { + const TimeoutTestParam params = GetParam(); + ABSL_RAW_LOG(INFO, "Params: %s", FormatString(params).c_str()); + + // Because this test asserts bounds on scheduling delays it is flaky. To + // compensate it loops forever until it passes. Failures express as test + // timeouts, in which case the test log can be used to diagnose the issue. + for (int attempt = 1;; ++attempt) { + ABSL_RAW_LOG(INFO, "Attempt %d", attempt); + + absl::Mutex mu; + bool value = false; // condition value (under mu) + + std::unique_ptr<absl::synchronization_internal::ThreadPool> pool = + CreateDefaultPool(); + RunAfterDelay(params.satisfy_condition_delay, pool.get(), [&] { + absl::MutexLock l(&mu); + value = true; + }); + + absl::Time start_time = absl::Now(); + absl::Condition cond(&value); + bool result = + params.use_absolute_deadline + ? mu.LockWhenWithDeadline(cond, start_time + params.wait_timeout) + : mu.LockWhenWithTimeout(cond, params.wait_timeout); + mu.Unlock(); + + if (DelayIsWithinBounds(params.expected_delay, absl::Now() - start_time)) { + EXPECT_EQ(params.expected_result, result); + break; + } + } +} + +TEST_P(TimeoutTest, ReaderLockWhen) { + const TimeoutTestParam params = GetParam(); + ABSL_RAW_LOG(INFO, "Params: %s", FormatString(params).c_str()); + + // Because this test asserts bounds on scheduling delays it is flaky. To + // compensate it loops forever until it passes. Failures express as test + // timeouts, in which case the test log can be used to diagnose the issue. + for (int attempt = 0;; ++attempt) { + ABSL_RAW_LOG(INFO, "Attempt %d", attempt); + + absl::Mutex mu; + bool value = false; // condition value (under mu) + + std::unique_ptr<absl::synchronization_internal::ThreadPool> pool = + CreateDefaultPool(); + RunAfterDelay(params.satisfy_condition_delay, pool.get(), [&] { + absl::MutexLock l(&mu); + value = true; + }); + + absl::Time start_time = absl::Now(); + bool result = + params.use_absolute_deadline + ? mu.ReaderLockWhenWithDeadline(absl::Condition(&value), + start_time + params.wait_timeout) + : mu.ReaderLockWhenWithTimeout(absl::Condition(&value), + params.wait_timeout); + mu.ReaderUnlock(); + + if (DelayIsWithinBounds(params.expected_delay, absl::Now() - start_time)) { + EXPECT_EQ(params.expected_result, result); + break; + } + } +} + +TEST_P(TimeoutTest, Wait) { + const TimeoutTestParam params = GetParam(); + ABSL_RAW_LOG(INFO, "Params: %s", FormatString(params).c_str()); + + // Because this test asserts bounds on scheduling delays it is flaky. To + // compensate it loops forever until it passes. Failures express as test + // timeouts, in which case the test log can be used to diagnose the issue. + for (int attempt = 0;; ++attempt) { + ABSL_RAW_LOG(INFO, "Attempt %d", attempt); + + absl::Mutex mu; + bool value = false; // condition value (under mu) + absl::CondVar cv; // signals a change of `value` + + std::unique_ptr<absl::synchronization_internal::ThreadPool> pool = + CreateDefaultPool(); + RunAfterDelay(params.satisfy_condition_delay, pool.get(), [&] { + absl::MutexLock l(&mu); + value = true; + cv.Signal(); + }); + + absl::MutexLock lock(&mu); + absl::Time start_time = absl::Now(); + absl::Duration timeout = params.wait_timeout; + absl::Time deadline = start_time + timeout; + while (!value) { + if (params.use_absolute_deadline ? cv.WaitWithDeadline(&mu, deadline) + : cv.WaitWithTimeout(&mu, timeout)) { + break; // deadline/timeout exceeded + } + timeout = deadline - absl::Now(); // recompute + } + bool result = value; // note: `mu` is still held + + if (DelayIsWithinBounds(params.expected_delay, absl::Now() - start_time)) { + EXPECT_EQ(params.expected_result, result); + break; + } + } +} + +TEST(Mutex, Logging) { + // Allow user to look at logging output + absl::Mutex logged_mutex; + logged_mutex.EnableDebugLog("fido_mutex"); + absl::CondVar logged_cv; + logged_cv.EnableDebugLog("rover_cv"); + logged_mutex.Lock(); + logged_cv.WaitWithTimeout(&logged_mutex, absl::Milliseconds(20)); + logged_mutex.Unlock(); + logged_mutex.ReaderLock(); + logged_mutex.ReaderUnlock(); + logged_mutex.Lock(); + logged_mutex.Unlock(); + logged_cv.Signal(); + logged_cv.SignalAll(); +} + +// -------------------------------------------------------- + +// Generate the vector of thread counts for tests parameterized on thread count. +static std::vector<int> AllThreadCountValues() { + if (kExtendedTest) { + return {2, 4, 8, 10, 16, 20, 24, 30, 32}; + } + return {2, 4, 10}; +} + +// A test fixture parameterized by thread count. +class MutexVariableThreadCountTest : public ::testing::TestWithParam<int> {}; + +// Instantiate the above with AllThreadCountOptions(). +INSTANTIATE_TEST_SUITE_P(ThreadCounts, MutexVariableThreadCountTest, + ::testing::ValuesIn(AllThreadCountValues()), + ::testing::PrintToStringParamName()); + +// Reduces iterations by some factor for slow platforms +// (determined empirically). +static int ScaleIterations(int x) { + // ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE is set in the implementation + // of Mutex that uses either std::mutex or pthread_mutex_t. Use + // these as keys to determine the slow implementation. +#if defined(ABSL_MUTEX_READER_LOCK_IS_EXCLUSIVE) + return x / 10; +#else + return x; +#endif +} + +TEST_P(MutexVariableThreadCountTest, Mutex) { + int threads = GetParam(); + int iterations = ScaleIterations(10000000) / threads; + int operations = threads * iterations; + EXPECT_EQ(RunTest(&TestMu, threads, iterations, operations), operations); +#if !defined(ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED) + iterations = std::min(iterations, 10); + operations = threads * iterations; + EXPECT_EQ(RunTestWithInvariantDebugging(&TestMu, threads, iterations, + operations, CheckSumG0G1), + operations); +#endif +} + +TEST_P(MutexVariableThreadCountTest, Try) { + int threads = GetParam(); + int iterations = 1000000 / threads; + int operations = iterations * threads; + EXPECT_EQ(RunTest(&TestTry, threads, iterations, operations), operations); +#if !defined(ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED) + iterations = std::min(iterations, 10); + operations = threads * iterations; + EXPECT_EQ(RunTestWithInvariantDebugging(&TestTry, threads, iterations, + operations, CheckSumG0G1), + operations); +#endif +} + +TEST_P(MutexVariableThreadCountTest, R20ms) { + int threads = GetParam(); + int iterations = 100; + int operations = iterations * threads; + EXPECT_EQ(RunTest(&TestR20ms, threads, iterations, operations), 0); +} + +TEST_P(MutexVariableThreadCountTest, RW) { + int threads = GetParam(); + int iterations = ScaleIterations(20000000) / threads; + int operations = iterations * threads; + EXPECT_EQ(RunTest(&TestRW, threads, iterations, operations), operations / 2); +#if !defined(ABSL_MUTEX_ENABLE_INVARIANT_DEBUGGING_NOT_IMPLEMENTED) + iterations = std::min(iterations, 10); + operations = threads * iterations; + EXPECT_EQ(RunTestWithInvariantDebugging(&TestRW, threads, iterations, + operations, CheckSumG0G1), + operations / 2); +#endif +} + +TEST_P(MutexVariableThreadCountTest, Await) { + int threads = GetParam(); + int iterations = ScaleIterations(500000); + int operations = iterations; + EXPECT_EQ(RunTest(&TestAwait, threads, iterations, operations), operations); +} + +TEST_P(MutexVariableThreadCountTest, SignalAll) { + int threads = GetParam(); + int iterations = 200000 / threads; + int operations = iterations; + EXPECT_EQ(RunTest(&TestSignalAll, threads, iterations, operations), + operations); +} + +TEST(Mutex, Signal) { + int threads = 2; // TestSignal must use two threads + int iterations = 200000; + int operations = iterations; + EXPECT_EQ(RunTest(&TestSignal, threads, iterations, operations), operations); +} + +TEST(Mutex, Timed) { + int threads = 10; // Use a fixed thread count of 10 + int iterations = 1000; + int operations = iterations; + EXPECT_EQ(RunTest(&TestCVTimeout, threads, iterations, operations), + operations); +} + +TEST(Mutex, CVTime) { + int threads = 10; // Use a fixed thread count of 10 + int iterations = 1; + EXPECT_EQ(RunTest(&TestCVTime, threads, iterations, 1), + threads * iterations); +} + +TEST(Mutex, MuTime) { + int threads = 10; // Use a fixed thread count of 10 + int iterations = 1; + EXPECT_EQ(RunTest(&TestMuTime, threads, iterations, 1), threads * iterations); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/synchronization/notification.cc b/third_party/abseil_cpp/absl/synchronization/notification.cc new file mode 100644 index 000000000000..e91b90382271 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/notification.cc @@ -0,0 +1,78 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/notification.h" + +#include <atomic> + +#include "absl/base/attributes.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +void Notification::Notify() { + MutexLock l(&this->mutex_); + +#ifndef NDEBUG + if (ABSL_PREDICT_FALSE(notified_yet_.load(std::memory_order_relaxed))) { + ABSL_RAW_LOG( + FATAL, + "Notify() method called more than once for Notification object %p", + static_cast<void *>(this)); + } +#endif + + notified_yet_.store(true, std::memory_order_release); +} + +Notification::~Notification() { + // Make sure that the thread running Notify() exits before the object is + // destructed. + MutexLock l(&this->mutex_); +} + +void Notification::WaitForNotification() const { + if (!HasBeenNotifiedInternal(&this->notified_yet_)) { + this->mutex_.LockWhen(Condition(&HasBeenNotifiedInternal, + &this->notified_yet_)); + this->mutex_.Unlock(); + } +} + +bool Notification::WaitForNotificationWithTimeout( + absl::Duration timeout) const { + bool notified = HasBeenNotifiedInternal(&this->notified_yet_); + if (!notified) { + notified = this->mutex_.LockWhenWithTimeout( + Condition(&HasBeenNotifiedInternal, &this->notified_yet_), timeout); + this->mutex_.Unlock(); + } + return notified; +} + +bool Notification::WaitForNotificationWithDeadline(absl::Time deadline) const { + bool notified = HasBeenNotifiedInternal(&this->notified_yet_); + if (!notified) { + notified = this->mutex_.LockWhenWithDeadline( + Condition(&HasBeenNotifiedInternal, &this->notified_yet_), deadline); + this->mutex_.Unlock(); + } + return notified; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/synchronization/notification.h b/third_party/abseil_cpp/absl/synchronization/notification.h new file mode 100644 index 000000000000..9a354ca2c023 --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/notification.h @@ -0,0 +1,123 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// notification.h +// ----------------------------------------------------------------------------- +// +// This header file defines a `Notification` abstraction, which allows threads +// to receive notification of a single occurrence of a single event. +// +// The `Notification` object maintains a private boolean "notified" state that +// transitions to `true` at most once. The `Notification` class provides the +// following primary member functions: +// * `HasBeenNotified() `to query its state +// * `WaitForNotification*()` to have threads wait until the "notified" state +// is `true`. +// * `Notify()` to set the notification's "notified" state to `true` and +// notify all waiting threads that the event has occurred. +// This method may only be called once. +// +// Note that while `Notify()` may only be called once, it is perfectly valid to +// call any of the `WaitForNotification*()` methods multiple times, from +// multiple threads -- even after the notification's "notified" state has been +// set -- in which case those methods will immediately return. +// +// Note that the lifetime of a `Notification` requires careful consideration; +// it might not be safe to destroy a notification after calling `Notify()` since +// it is still legal for other threads to call `WaitForNotification*()` methods +// on the notification. However, observers responding to a "notified" state of +// `true` can safely delete the notification without interfering with the call +// to `Notify()` in the other thread. +// +// Memory ordering: For any threads X and Y, if X calls `Notify()`, then any +// action taken by X before it calls `Notify()` is visible to thread Y after: +// * Y returns from `WaitForNotification()`, or +// * Y receives a `true` return value from either `HasBeenNotified()` or +// `WaitForNotificationWithTimeout()`. + +#ifndef ABSL_SYNCHRONIZATION_NOTIFICATION_H_ +#define ABSL_SYNCHRONIZATION_NOTIFICATION_H_ + +#include <atomic> + +#include "absl/base/macros.h" +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// Notification +// ----------------------------------------------------------------------------- +class Notification { + public: + // Initializes the "notified" state to unnotified. + Notification() : notified_yet_(false) {} + explicit Notification(bool prenotify) : notified_yet_(prenotify) {} + Notification(const Notification&) = delete; + Notification& operator=(const Notification&) = delete; + ~Notification(); + + // Notification::HasBeenNotified() + // + // Returns the value of the notification's internal "notified" state. + bool HasBeenNotified() const { + return HasBeenNotifiedInternal(&this->notified_yet_); + } + + // Notification::WaitForNotification() + // + // Blocks the calling thread until the notification's "notified" state is + // `true`. Note that if `Notify()` has been previously called on this + // notification, this function will immediately return. + void WaitForNotification() const; + + // Notification::WaitForNotificationWithTimeout() + // + // Blocks until either the notification's "notified" state is `true` (which + // may occur immediately) or the timeout has elapsed, returning the value of + // its "notified" state in either case. + bool WaitForNotificationWithTimeout(absl::Duration timeout) const; + + // Notification::WaitForNotificationWithDeadline() + // + // Blocks until either the notification's "notified" state is `true` (which + // may occur immediately) or the deadline has expired, returning the value of + // its "notified" state in either case. + bool WaitForNotificationWithDeadline(absl::Time deadline) const; + + // Notification::Notify() + // + // Sets the "notified" state of this notification to `true` and wakes waiting + // threads. Note: do not call `Notify()` multiple times on the same + // `Notification`; calling `Notify()` more than once on the same notification + // results in undefined behavior. + void Notify(); + + private: + static inline bool HasBeenNotifiedInternal( + const std::atomic<bool>* notified_yet) { + return notified_yet->load(std::memory_order_acquire); + } + + mutable Mutex mutex_; + std::atomic<bool> notified_yet_; // written under mutex_ +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_SYNCHRONIZATION_NOTIFICATION_H_ diff --git a/third_party/abseil_cpp/absl/synchronization/notification_test.cc b/third_party/abseil_cpp/absl/synchronization/notification_test.cc new file mode 100644 index 000000000000..100ea76f33ac --- /dev/null +++ b/third_party/abseil_cpp/absl/synchronization/notification_test.cc @@ -0,0 +1,133 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/synchronization/notification.h" + +#include <thread> // NOLINT(build/c++11) +#include <vector> + +#include "gtest/gtest.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// A thread-safe class that holds a counter. +class ThreadSafeCounter { + public: + ThreadSafeCounter() : count_(0) {} + + void Increment() { + MutexLock lock(&mutex_); + ++count_; + } + + int Get() const { + MutexLock lock(&mutex_); + return count_; + } + + void WaitUntilGreaterOrEqual(int n) { + MutexLock lock(&mutex_); + auto cond = [this, n]() { return count_ >= n; }; + mutex_.Await(Condition(&cond)); + } + + private: + mutable Mutex mutex_; + int count_; +}; + +// Runs the |i|'th worker thread for the tests in BasicTests(). Increments the +// |ready_counter|, waits on the |notification|, and then increments the +// |done_counter|. +static void RunWorker(int i, ThreadSafeCounter* ready_counter, + Notification* notification, + ThreadSafeCounter* done_counter) { + ready_counter->Increment(); + notification->WaitForNotification(); + done_counter->Increment(); +} + +// Tests that the |notification| properly blocks and awakens threads. Assumes +// that the |notification| is not yet triggered. If |notify_before_waiting| is +// true, the |notification| is triggered before any threads are created, so the +// threads never block in WaitForNotification(). Otherwise, the |notification| +// is triggered at a later point when most threads are likely to be blocking in +// WaitForNotification(). +static void BasicTests(bool notify_before_waiting, Notification* notification) { + EXPECT_FALSE(notification->HasBeenNotified()); + EXPECT_FALSE( + notification->WaitForNotificationWithTimeout(absl::Milliseconds(0))); + EXPECT_FALSE(notification->WaitForNotificationWithDeadline(absl::Now())); + + const absl::Duration delay = absl::Milliseconds(50); + const absl::Time start = absl::Now(); + EXPECT_FALSE(notification->WaitForNotificationWithTimeout(delay)); + const absl::Duration elapsed = absl::Now() - start; + + // Allow for a slight early return, to account for quality of implementation + // issues on various platforms. + const absl::Duration slop = absl::Microseconds(200); + EXPECT_LE(delay - slop, elapsed) + << "WaitForNotificationWithTimeout returned " << delay - elapsed + << " early (with " << slop << " slop), start time was " << start; + + ThreadSafeCounter ready_counter; + ThreadSafeCounter done_counter; + + if (notify_before_waiting) { + notification->Notify(); + } + + // Create a bunch of threads that increment the |done_counter| after being + // notified. + const int kNumThreads = 10; + std::vector<std::thread> workers; + for (int i = 0; i < kNumThreads; ++i) { + workers.push_back(std::thread(&RunWorker, i, &ready_counter, notification, + &done_counter)); + } + + if (!notify_before_waiting) { + ready_counter.WaitUntilGreaterOrEqual(kNumThreads); + + // Workers have not been notified yet, so the |done_counter| should be + // unmodified. + EXPECT_EQ(0, done_counter.Get()); + + notification->Notify(); + } + + // After notifying and then joining the workers, both counters should be + // fully incremented. + notification->WaitForNotification(); // should exit immediately + EXPECT_TRUE(notification->HasBeenNotified()); + EXPECT_TRUE(notification->WaitForNotificationWithTimeout(absl::Seconds(0))); + EXPECT_TRUE(notification->WaitForNotificationWithDeadline(absl::Now())); + for (std::thread& worker : workers) { + worker.join(); + } + EXPECT_EQ(kNumThreads, ready_counter.Get()); + EXPECT_EQ(kNumThreads, done_counter.Get()); +} + +TEST(NotificationTest, SanityTest) { + Notification local_notification1, local_notification2; + BasicTests(false, &local_notification1); + BasicTests(true, &local_notification2); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/BUILD.bazel b/third_party/abseil_cpp/absl/time/BUILD.bazel new file mode 100644 index 000000000000..991241a0df5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/BUILD.bazel @@ -0,0 +1,125 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "time", + srcs = [ + "civil_time.cc", + "clock.cc", + "duration.cc", + "format.cc", + "internal/get_current_time_chrono.inc", + "internal/get_current_time_posix.inc", + "time.cc", + ], + hdrs = [ + "civil_time.h", + "clock.h", + "time.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/numeric:int128", + "//absl/strings", + "//absl/time/internal/cctz:civil_time", + "//absl/time/internal/cctz:time_zone", + ], +) + +cc_library( + name = "test_util", + testonly = 1, + srcs = [ + "internal/test_util.cc", + "internal/zoneinfo.inc", + ], + hdrs = ["internal/test_util.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl/time:__pkg__", + ], + deps = [ + ":time", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/time/internal/cctz:time_zone", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "time_test", + srcs = [ + "civil_time_test.cc", + "clock_test.cc", + "duration_test.cc", + "format_test.cc", + "time_test.cc", + "time_zone_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":test_util", + ":time", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/numeric:int128", + "//absl/time/internal/cctz:time_zone", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "time_benchmark", + srcs = [ + "civil_time_benchmark.cc", + "clock_benchmark.cc", + "duration_benchmark.cc", + "format_benchmark.cc", + "time_benchmark.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "benchmark", + ], + deps = [ + ":test_util", + ":time", + "//absl/base", + "//absl/base:core_headers", + "//absl/hash", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/third_party/abseil_cpp/absl/time/CMakeLists.txt b/third_party/abseil_cpp/absl/time/CMakeLists.txt new file mode 100644 index 000000000000..00bdd499c1f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/CMakeLists.txt @@ -0,0 +1,128 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + time + HDRS + "civil_time.h" + "clock.h" + "time.h" + SRCS + "civil_time.cc" + "clock.cc" + "duration.cc" + "format.cc" + "internal/get_current_time_chrono.inc" + "internal/get_current_time_posix.inc" + "time.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::civil_time + absl::core_headers + absl::int128 + absl::raw_logging_internal + absl::strings + absl::time_zone + PUBLIC +) + +absl_cc_library( + NAME + civil_time + HDRS + "internal/cctz/include/cctz/civil_time.h" + "internal/cctz/include/cctz/civil_time_detail.h" + SRCS + "internal/cctz/src/civil_time_detail.cc" + COPTS + ${ABSL_DEFAULT_COPTS} +) + +if(APPLE) + find_library(CoreFoundation CoreFoundation) +endif() + +absl_cc_library( + NAME + time_zone + HDRS + "internal/cctz/include/cctz/time_zone.h" + "internal/cctz/include/cctz/zone_info_source.h" + SRCS + "internal/cctz/src/time_zone_fixed.cc" + "internal/cctz/src/time_zone_fixed.h" + "internal/cctz/src/time_zone_format.cc" + "internal/cctz/src/time_zone_if.cc" + "internal/cctz/src/time_zone_if.h" + "internal/cctz/src/time_zone_impl.cc" + "internal/cctz/src/time_zone_impl.h" + "internal/cctz/src/time_zone_info.cc" + "internal/cctz/src/time_zone_info.h" + "internal/cctz/src/time_zone_libc.cc" + "internal/cctz/src/time_zone_libc.h" + "internal/cctz/src/time_zone_lookup.cc" + "internal/cctz/src/time_zone_posix.cc" + "internal/cctz/src/time_zone_posix.h" + "internal/cctz/src/tzfile.h" + "internal/cctz/src/zone_info_source.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + $<$<PLATFORM_ID:Darwin>:${CoreFoundation}> +) + +absl_cc_library( + NAME + time_internal_test_util + HDRS + "internal/test_util.h" + SRCS + "internal/test_util.cc" + "internal/zoneinfo.inc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::time + absl::config + absl::raw_logging_internal + absl::time_zone + gmock + TESTONLY +) + +absl_cc_test( + NAME + time_test + SRCS + "civil_time_test.cc" + "clock_test.cc" + "duration_test.cc" + "format_test.cc" + "time_test.cc" + "time_zone_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::time_internal_test_util + absl::time + absl::config + absl::core_headers + absl::time_zone + gmock_main +) diff --git a/third_party/abseil_cpp/absl/time/civil_time.cc b/third_party/abseil_cpp/absl/time/civil_time.cc new file mode 100644 index 000000000000..bdfe9ce0efb3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/civil_time.cc @@ -0,0 +1,175 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/civil_time.h" + +#include <cstdlib> +#include <string> + +#include "absl/strings/str_cat.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +// Since a civil time has a larger year range than absl::Time (64-bit years vs +// 64-bit seconds, respectively) we normalize years to roughly +/- 400 years +// around the year 2400, which will produce an equivalent year in a range that +// absl::Time can handle. +inline civil_year_t NormalizeYear(civil_year_t year) { + return 2400 + year % 400; +} + +// Formats the given CivilSecond according to the given format. +std::string FormatYearAnd(string_view fmt, CivilSecond cs) { + const CivilSecond ncs(NormalizeYear(cs.year()), cs.month(), cs.day(), + cs.hour(), cs.minute(), cs.second()); + const TimeZone utc = UTCTimeZone(); + // TODO(absl-team): Avoid conversion of fmt string. + return StrCat(cs.year(), + FormatTime(std::string(fmt), FromCivil(ncs, utc), utc)); +} + +template <typename CivilT> +bool ParseYearAnd(string_view fmt, string_view s, CivilT* c) { + // Civil times support a larger year range than absl::Time, so we need to + // parse the year separately, normalize it, then use absl::ParseTime on the + // normalized string. + const std::string ss = std::string(s); // TODO(absl-team): Avoid conversion. + const char* const np = ss.c_str(); + char* endp; + errno = 0; + const civil_year_t y = + std::strtoll(np, &endp, 10); // NOLINT(runtime/deprecated_fn) + if (endp == np || errno == ERANGE) return false; + const std::string norm = StrCat(NormalizeYear(y), endp); + + const TimeZone utc = UTCTimeZone(); + Time t; + if (ParseTime(StrCat("%Y", fmt), norm, utc, &t, nullptr)) { + const auto cs = ToCivilSecond(t, utc); + *c = CivilT(y, cs.month(), cs.day(), cs.hour(), cs.minute(), cs.second()); + return true; + } + + return false; +} + +// Tries to parse the type as a CivilT1, but then assigns the result to the +// argument of type CivilT2. +template <typename CivilT1, typename CivilT2> +bool ParseAs(string_view s, CivilT2* c) { + CivilT1 t1; + if (ParseCivilTime(s, &t1)) { + *c = CivilT2(t1); + return true; + } + return false; +} + +template <typename CivilT> +bool ParseLenient(string_view s, CivilT* c) { + // A fastpath for when the given string data parses exactly into the given + // type T (e.g., s="YYYY-MM-DD" and CivilT=CivilDay). + if (ParseCivilTime(s, c)) return true; + // Try parsing as each of the 6 types, trying the most common types first + // (based on csearch results). + if (ParseAs<CivilDay>(s, c)) return true; + if (ParseAs<CivilSecond>(s, c)) return true; + if (ParseAs<CivilHour>(s, c)) return true; + if (ParseAs<CivilMonth>(s, c)) return true; + if (ParseAs<CivilMinute>(s, c)) return true; + if (ParseAs<CivilYear>(s, c)) return true; + return false; +} +} // namespace + +std::string FormatCivilTime(CivilSecond c) { + return FormatYearAnd("-%m-%d%ET%H:%M:%S", c); +} +std::string FormatCivilTime(CivilMinute c) { + return FormatYearAnd("-%m-%d%ET%H:%M", c); +} +std::string FormatCivilTime(CivilHour c) { + return FormatYearAnd("-%m-%d%ET%H", c); +} +std::string FormatCivilTime(CivilDay c) { return FormatYearAnd("-%m-%d", c); } +std::string FormatCivilTime(CivilMonth c) { return FormatYearAnd("-%m", c); } +std::string FormatCivilTime(CivilYear c) { return FormatYearAnd("", c); } + +bool ParseCivilTime(string_view s, CivilSecond* c) { + return ParseYearAnd("-%m-%d%ET%H:%M:%S", s, c); +} +bool ParseCivilTime(string_view s, CivilMinute* c) { + return ParseYearAnd("-%m-%d%ET%H:%M", s, c); +} +bool ParseCivilTime(string_view s, CivilHour* c) { + return ParseYearAnd("-%m-%d%ET%H", s, c); +} +bool ParseCivilTime(string_view s, CivilDay* c) { + return ParseYearAnd("-%m-%d", s, c); +} +bool ParseCivilTime(string_view s, CivilMonth* c) { + return ParseYearAnd("-%m", s, c); +} +bool ParseCivilTime(string_view s, CivilYear* c) { + return ParseYearAnd("", s, c); +} + +bool ParseLenientCivilTime(string_view s, CivilSecond* c) { + return ParseLenient(s, c); +} +bool ParseLenientCivilTime(string_view s, CivilMinute* c) { + return ParseLenient(s, c); +} +bool ParseLenientCivilTime(string_view s, CivilHour* c) { + return ParseLenient(s, c); +} +bool ParseLenientCivilTime(string_view s, CivilDay* c) { + return ParseLenient(s, c); +} +bool ParseLenientCivilTime(string_view s, CivilMonth* c) { + return ParseLenient(s, c); +} +bool ParseLenientCivilTime(string_view s, CivilYear* c) { + return ParseLenient(s, c); +} + +namespace time_internal { + +std::ostream& operator<<(std::ostream& os, CivilYear y) { + return os << FormatCivilTime(y); +} +std::ostream& operator<<(std::ostream& os, CivilMonth m) { + return os << FormatCivilTime(m); +} +std::ostream& operator<<(std::ostream& os, CivilDay d) { + return os << FormatCivilTime(d); +} +std::ostream& operator<<(std::ostream& os, CivilHour h) { + return os << FormatCivilTime(h); +} +std::ostream& operator<<(std::ostream& os, CivilMinute m) { + return os << FormatCivilTime(m); +} +std::ostream& operator<<(std::ostream& os, CivilSecond s) { + return os << FormatCivilTime(s); +} + +} // namespace time_internal + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/civil_time.h b/third_party/abseil_cpp/absl/time/civil_time.h new file mode 100644 index 000000000000..bb4600443445 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/civil_time.h @@ -0,0 +1,538 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: civil_time.h +// ----------------------------------------------------------------------------- +// +// This header file defines abstractions for computing with "civil time". +// The term "civil time" refers to the legally recognized human-scale time +// that is represented by the six fields `YYYY-MM-DD hh:mm:ss`. A "date" +// is perhaps the most common example of a civil time (represented here as +// an `absl::CivilDay`). +// +// Modern-day civil time follows the Gregorian Calendar and is a +// time-zone-independent concept: a civil time of "2015-06-01 12:00:00", for +// example, is not tied to a time zone. Put another way, a civil time does not +// map to a unique point in time; a civil time must be mapped to an absolute +// time *through* a time zone. +// +// Because a civil time is what most people think of as "time," it is common to +// map absolute times to civil times to present to users. +// +// Time zones define the relationship between absolute and civil times. Given an +// absolute or civil time and a time zone, you can compute the other time: +// +// Civil Time = F(Absolute Time, Time Zone) +// Absolute Time = G(Civil Time, Time Zone) +// +// The Abseil time library allows you to construct such civil times from +// absolute times; consult time.h for such functionality. +// +// This library provides six classes for constructing civil-time objects, and +// provides several helper functions for rounding, iterating, and performing +// arithmetic on civil-time objects, while avoiding complications like +// daylight-saving time (DST): +// +// * `absl::CivilSecond` +// * `absl::CivilMinute` +// * `absl::CivilHour` +// * `absl::CivilDay` +// * `absl::CivilMonth` +// * `absl::CivilYear` +// +// Example: +// +// // Construct a civil-time object for a specific day +// const absl::CivilDay cd(1969, 07, 20); +// +// // Construct a civil-time object for a specific second +// const absl::CivilSecond cd(2018, 8, 1, 12, 0, 1); +// +// Note: In C++14 and later, this library is usable in a constexpr context. +// +// Example: +// +// // Valid in C++14 +// constexpr absl::CivilDay cd(1969, 07, 20); + +#ifndef ABSL_TIME_CIVIL_TIME_H_ +#define ABSL_TIME_CIVIL_TIME_H_ + +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace time_internal { +struct second_tag : cctz::detail::second_tag {}; +struct minute_tag : second_tag, cctz::detail::minute_tag {}; +struct hour_tag : minute_tag, cctz::detail::hour_tag {}; +struct day_tag : hour_tag, cctz::detail::day_tag {}; +struct month_tag : day_tag, cctz::detail::month_tag {}; +struct year_tag : month_tag, cctz::detail::year_tag {}; +} // namespace time_internal + +// ----------------------------------------------------------------------------- +// CivilSecond, CivilMinute, CivilHour, CivilDay, CivilMonth, CivilYear +// ----------------------------------------------------------------------------- +// +// Each of these civil-time types is a simple value type with the same +// interface for construction and the same six accessors for each of the civil +// time fields (year, month, day, hour, minute, and second, aka YMDHMS). These +// classes differ only in their alignment, which is indicated by the type name +// and specifies the field on which arithmetic operates. +// +// CONSTRUCTION +// +// Each of the civil-time types can be constructed in two ways: by directly +// passing to the constructor up to six integers representing the YMDHMS fields, +// or by copying the YMDHMS fields from a differently aligned civil-time type. +// Omitted fields are assigned their minimum valid value. Hours, minutes, and +// seconds will be set to 0, month and day will be set to 1. Since there is no +// minimum year, the default is 1970. +// +// Examples: +// +// absl::CivilDay default_value; // 1970-01-01 00:00:00 +// +// absl::CivilDay a(2015, 2, 3); // 2015-02-03 00:00:00 +// absl::CivilDay b(2015, 2, 3, 4, 5, 6); // 2015-02-03 00:00:00 +// absl::CivilDay c(2015); // 2015-01-01 00:00:00 +// +// absl::CivilSecond ss(2015, 2, 3, 4, 5, 6); // 2015-02-03 04:05:06 +// absl::CivilMinute mm(ss); // 2015-02-03 04:05:00 +// absl::CivilHour hh(mm); // 2015-02-03 04:00:00 +// absl::CivilDay d(hh); // 2015-02-03 00:00:00 +// absl::CivilMonth m(d); // 2015-02-01 00:00:00 +// absl::CivilYear y(m); // 2015-01-01 00:00:00 +// +// m = absl::CivilMonth(y); // 2015-01-01 00:00:00 +// d = absl::CivilDay(m); // 2015-01-01 00:00:00 +// hh = absl::CivilHour(d); // 2015-01-01 00:00:00 +// mm = absl::CivilMinute(hh); // 2015-01-01 00:00:00 +// ss = absl::CivilSecond(mm); // 2015-01-01 00:00:00 +// +// Each civil-time class is aligned to the civil-time field indicated in the +// class's name after normalization. Alignment is performed by setting all the +// inferior fields to their minimum valid value (as described above). The +// following are examples of how each of the six types would align the fields +// representing November 22, 2015 at 12:34:56 in the afternoon. (Note: the +// string format used here is not important; it's just a shorthand way of +// showing the six YMDHMS fields.) +// +// absl::CivilSecond : 2015-11-22 12:34:56 +// absl::CivilMinute : 2015-11-22 12:34:00 +// absl::CivilHour : 2015-11-22 12:00:00 +// absl::CivilDay : 2015-11-22 00:00:00 +// absl::CivilMonth : 2015-11-01 00:00:00 +// absl::CivilYear : 2015-01-01 00:00:00 +// +// Each civil-time type performs arithmetic on the field to which it is +// aligned. This means that adding 1 to an absl::CivilDay increments the day +// field (normalizing as necessary), and subtracting 7 from an absl::CivilMonth +// operates on the month field (normalizing as necessary). All arithmetic +// produces a valid civil time. Difference requires two similarly aligned +// civil-time objects and returns the scalar answer in units of the objects' +// alignment. For example, the difference between two absl::CivilHour objects +// will give an answer in units of civil hours. +// +// ALIGNMENT CONVERSION +// +// The alignment of a civil-time object cannot change, but the object may be +// used to construct a new object with a different alignment. This is referred +// to as "realigning". When realigning to a type with the same or more +// precision (e.g., absl::CivilDay -> absl::CivilSecond), the conversion may be +// performed implicitly since no information is lost. However, if information +// could be discarded (e.g., CivilSecond -> CivilDay), the conversion must +// be explicit at the call site. +// +// Examples: +// +// void UseDay(absl::CivilDay day); +// +// absl::CivilSecond cs; +// UseDay(cs); // Won't compile because data may be discarded +// UseDay(absl::CivilDay(cs)); // OK: explicit conversion +// +// absl::CivilDay cd; +// UseDay(cd); // OK: no conversion needed +// +// absl::CivilMonth cm; +// UseDay(cm); // OK: implicit conversion to absl::CivilDay +// +// NORMALIZATION +// +// Normalization takes invalid values and adjusts them to produce valid values. +// Within the civil-time library, integer arguments passed to the Civil* +// constructors may be out-of-range, in which case they are normalized by +// carrying overflow into a field of courser granularity to produce valid +// civil-time objects. This normalization enables natural arithmetic on +// constructor arguments without worrying about the field's range. +// +// Examples: +// +// // Out-of-range; normalized to 2016-11-01 +// absl::CivilDay d(2016, 10, 32); +// // Out-of-range, negative: normalized to 2016-10-30T23 +// absl::CivilHour h1(2016, 10, 31, -1); +// // Normalization is cumulative: normalized to 2016-10-30T23 +// absl::CivilHour h2(2016, 10, 32, -25); +// +// Note: If normalization is undesired, you can signal an error by comparing +// the constructor arguments to the normalized values returned by the YMDHMS +// properties. +// +// COMPARISON +// +// Comparison between civil-time objects considers all six YMDHMS fields, +// regardless of the type's alignment. Comparison between differently aligned +// civil-time types is allowed. +// +// Examples: +// +// absl::CivilDay feb_3(2015, 2, 3); // 2015-02-03 00:00:00 +// absl::CivilDay mar_4(2015, 3, 4); // 2015-03-04 00:00:00 +// // feb_3 < mar_4 +// // absl::CivilYear(feb_3) == absl::CivilYear(mar_4) +// +// absl::CivilSecond feb_3_noon(2015, 2, 3, 12, 0, 0); // 2015-02-03 12:00:00 +// // feb_3 < feb_3_noon +// // feb_3 == absl::CivilDay(feb_3_noon) +// +// // Iterates all the days of February 2015. +// for (absl::CivilDay d(2015, 2, 1); d < absl::CivilMonth(2015, 3); ++d) { +// // ... +// } +// +// ARITHMETIC +// +// Civil-time types support natural arithmetic operators such as addition, +// subtraction, and difference. Arithmetic operates on the civil-time field +// indicated in the type's name. Difference operators require arguments with +// the same alignment and return the answer in units of the alignment. +// +// Example: +// +// absl::CivilDay a(2015, 2, 3); +// ++a; // 2015-02-04 00:00:00 +// --a; // 2015-02-03 00:00:00 +// absl::CivilDay b = a + 1; // 2015-02-04 00:00:00 +// absl::CivilDay c = 1 + b; // 2015-02-05 00:00:00 +// int n = c - a; // n = 2 (civil days) +// int m = c - absl::CivilMonth(c); // Won't compile: different types. +// +// ACCESSORS +// +// Each civil-time type has accessors for all six of the civil-time fields: +// year, month, day, hour, minute, and second. +// +// civil_year_t year() +// int month() +// int day() +// int hour() +// int minute() +// int second() +// +// Recall that fields inferior to the type's alignment will be set to their +// minimum valid value. +// +// Example: +// +// absl::CivilDay d(2015, 6, 28); +// // d.year() == 2015 +// // d.month() == 6 +// // d.day() == 28 +// // d.hour() == 0 +// // d.minute() == 0 +// // d.second() == 0 +// +// CASE STUDY: Adding a month to January 31. +// +// One of the classic questions that arises when considering a civil time +// library (or a date library or a date/time library) is this: +// "What is the result of adding a month to January 31?" +// This is an interesting question because it is unclear what is meant by a +// "month", and several different answers are possible, depending on context: +// +// 1. March 3 (or 2 if a leap year), if "add a month" means to add a month to +// the current month, and adjust the date to overflow the extra days into +// March. In this case the result of "February 31" would be normalized as +// within the civil-time library. +// 2. February 28 (or 29 if a leap year), if "add a month" means to add a +// month, and adjust the date while holding the resulting month constant. +// In this case, the result of "February 31" would be truncated to the last +// day in February. +// 3. An error. The caller may get some error, an exception, an invalid date +// object, or perhaps return `false`. This may make sense because there is +// no single unambiguously correct answer to the question. +// +// Practically speaking, any answer that is not what the programmer intended +// is the wrong answer. +// +// The Abseil time library avoids this problem by making it impossible to +// ask ambiguous questions. All civil-time objects are aligned to a particular +// civil-field boundary (such as aligned to a year, month, day, hour, minute, +// or second), and arithmetic operates on the field to which the object is +// aligned. This means that in order to "add a month" the object must first be +// aligned to a month boundary, which is equivalent to the first day of that +// month. +// +// Of course, there are ways to compute an answer the question at hand using +// this Abseil time library, but they require the programmer to be explicit +// about the answer they expect. To illustrate, let's see how to compute all +// three of the above possible answers to the question of "Jan 31 plus 1 +// month": +// +// Example: +// +// const absl::CivilDay d(2015, 1, 31); +// +// // Answer 1: +// // Add 1 to the month field in the constructor, and rely on normalization. +// const auto normalized = absl::CivilDay(d.year(), d.month() + 1, d.day()); +// // normalized == 2015-03-03 (aka Feb 31) +// +// // Answer 2: +// // Add 1 to month field, capping to the end of next month. +// const auto next_month = absl::CivilMonth(d) + 1; +// const auto last_day_of_next_month = absl::CivilDay(next_month + 1) - 1; +// const auto capped = std::min(normalized, last_day_of_next_month); +// // capped == 2015-02-28 +// +// // Answer 3: +// // Signal an error if the normalized answer is not in next month. +// if (absl::CivilMonth(normalized) != next_month) { +// // error, month overflow +// } +// +using CivilSecond = + time_internal::cctz::detail::civil_time<time_internal::second_tag>; +using CivilMinute = + time_internal::cctz::detail::civil_time<time_internal::minute_tag>; +using CivilHour = + time_internal::cctz::detail::civil_time<time_internal::hour_tag>; +using CivilDay = + time_internal::cctz::detail::civil_time<time_internal::day_tag>; +using CivilMonth = + time_internal::cctz::detail::civil_time<time_internal::month_tag>; +using CivilYear = + time_internal::cctz::detail::civil_time<time_internal::year_tag>; + +// civil_year_t +// +// Type alias of a civil-time year value. This type is guaranteed to (at least) +// support any year value supported by `time_t`. +// +// Example: +// +// absl::CivilSecond cs = ...; +// absl::civil_year_t y = cs.year(); +// cs = absl::CivilSecond(y, 1, 1, 0, 0, 0); // CivilSecond(CivilYear(cs)) +// +using civil_year_t = time_internal::cctz::year_t; + +// civil_diff_t +// +// Type alias of the difference between two civil-time values. +// This type is used to indicate arguments that are not +// normalized (such as parameters to the civil-time constructors), the results +// of civil-time subtraction, or the operand to civil-time addition. +// +// Example: +// +// absl::civil_diff_t n_sec = cs1 - cs2; // cs1 == cs2 + n_sec; +// +using civil_diff_t = time_internal::cctz::diff_t; + +// Weekday::monday, Weekday::tuesday, Weekday::wednesday, Weekday::thursday, +// Weekday::friday, Weekday::saturday, Weekday::sunday +// +// The Weekday enum class represents the civil-time concept of a "weekday" with +// members for all days of the week. +// +// absl::Weekday wd = absl::Weekday::thursday; +// +using Weekday = time_internal::cctz::weekday; + +// GetWeekday() +// +// Returns the absl::Weekday for the given (realigned) civil-time value. +// +// Example: +// +// absl::CivilDay a(2015, 8, 13); +// absl::Weekday wd = absl::GetWeekday(a); // wd == absl::Weekday::thursday +// +inline Weekday GetWeekday(CivilSecond cs) { + return time_internal::cctz::get_weekday(cs); +} + +// NextWeekday() +// PrevWeekday() +// +// Returns the absl::CivilDay that strictly follows or precedes a given +// absl::CivilDay, and that falls on the given absl::Weekday. +// +// Example, given the following month: +// +// August 2015 +// Su Mo Tu We Th Fr Sa +// 1 +// 2 3 4 5 6 7 8 +// 9 10 11 12 13 14 15 +// 16 17 18 19 20 21 22 +// 23 24 25 26 27 28 29 +// 30 31 +// +// absl::CivilDay a(2015, 8, 13); +// // absl::GetWeekday(a) == absl::Weekday::thursday +// absl::CivilDay b = absl::NextWeekday(a, absl::Weekday::thursday); +// // b = 2015-08-20 +// absl::CivilDay c = absl::PrevWeekday(a, absl::Weekday::thursday); +// // c = 2015-08-06 +// +// absl::CivilDay d = ... +// // Gets the following Thursday if d is not already Thursday +// absl::CivilDay thurs1 = absl::NextWeekday(d - 1, absl::Weekday::thursday); +// // Gets the previous Thursday if d is not already Thursday +// absl::CivilDay thurs2 = absl::PrevWeekday(d + 1, absl::Weekday::thursday); +// +inline CivilDay NextWeekday(CivilDay cd, Weekday wd) { + return CivilDay(time_internal::cctz::next_weekday(cd, wd)); +} +inline CivilDay PrevWeekday(CivilDay cd, Weekday wd) { + return CivilDay(time_internal::cctz::prev_weekday(cd, wd)); +} + +// GetYearDay() +// +// Returns the day-of-year for the given (realigned) civil-time value. +// +// Example: +// +// absl::CivilDay a(2015, 1, 1); +// int yd_jan_1 = absl::GetYearDay(a); // yd_jan_1 = 1 +// absl::CivilDay b(2015, 12, 31); +// int yd_dec_31 = absl::GetYearDay(b); // yd_dec_31 = 365 +// +inline int GetYearDay(CivilSecond cs) { + return time_internal::cctz::get_yearday(cs); +} + +// FormatCivilTime() +// +// Formats the given civil-time value into a string value of the following +// format: +// +// Type | Format +// --------------------------------- +// CivilSecond | YYYY-MM-DDTHH:MM:SS +// CivilMinute | YYYY-MM-DDTHH:MM +// CivilHour | YYYY-MM-DDTHH +// CivilDay | YYYY-MM-DD +// CivilMonth | YYYY-MM +// CivilYear | YYYY +// +// Example: +// +// absl::CivilDay d = absl::CivilDay(1969, 7, 20); +// std::string day_string = absl::FormatCivilTime(d); // "1969-07-20" +// +std::string FormatCivilTime(CivilSecond c); +std::string FormatCivilTime(CivilMinute c); +std::string FormatCivilTime(CivilHour c); +std::string FormatCivilTime(CivilDay c); +std::string FormatCivilTime(CivilMonth c); +std::string FormatCivilTime(CivilYear c); + +// absl::ParseCivilTime() +// +// Parses a civil-time value from the specified `absl::string_view` into the +// passed output parameter. Returns `true` upon successful parsing. +// +// The expected form of the input string is as follows: +// +// Type | Format +// --------------------------------- +// CivilSecond | YYYY-MM-DDTHH:MM:SS +// CivilMinute | YYYY-MM-DDTHH:MM +// CivilHour | YYYY-MM-DDTHH +// CivilDay | YYYY-MM-DD +// CivilMonth | YYYY-MM +// CivilYear | YYYY +// +// Example: +// +// absl::CivilDay d; +// bool ok = absl::ParseCivilTime("2018-01-02", &d); // OK +// +// Note that parsing will fail if the string's format does not match the +// expected type exactly. `ParseLenientCivilTime()` below is more lenient. +// +bool ParseCivilTime(absl::string_view s, CivilSecond* c); +bool ParseCivilTime(absl::string_view s, CivilMinute* c); +bool ParseCivilTime(absl::string_view s, CivilHour* c); +bool ParseCivilTime(absl::string_view s, CivilDay* c); +bool ParseCivilTime(absl::string_view s, CivilMonth* c); +bool ParseCivilTime(absl::string_view s, CivilYear* c); + +// ParseLenientCivilTime() +// +// Parses any of the formats accepted by `absl::ParseCivilTime()`, but is more +// lenient if the format of the string does not exactly match the associated +// type. +// +// Example: +// +// absl::CivilDay d; +// bool ok = absl::ParseLenientCivilTime("1969-07-20", &d); // OK +// ok = absl::ParseLenientCivilTime("1969-07-20T10", &d); // OK: T10 floored +// ok = absl::ParseLenientCivilTime("1969-07", &d); // OK: day defaults to 1 +// +bool ParseLenientCivilTime(absl::string_view s, CivilSecond* c); +bool ParseLenientCivilTime(absl::string_view s, CivilMinute* c); +bool ParseLenientCivilTime(absl::string_view s, CivilHour* c); +bool ParseLenientCivilTime(absl::string_view s, CivilDay* c); +bool ParseLenientCivilTime(absl::string_view s, CivilMonth* c); +bool ParseLenientCivilTime(absl::string_view s, CivilYear* c); + +namespace time_internal { // For functions found via ADL on civil-time tags. + +// Streaming Operators +// +// Each civil-time type may be sent to an output stream using operator<<(). +// The result matches the string produced by `FormatCivilTime()`. +// +// Example: +// +// absl::CivilDay d = absl::CivilDay(1969, 7, 20); +// std::cout << "Date is: " << d << "\n"; +// +std::ostream& operator<<(std::ostream& os, CivilYear y); +std::ostream& operator<<(std::ostream& os, CivilMonth m); +std::ostream& operator<<(std::ostream& os, CivilDay d); +std::ostream& operator<<(std::ostream& os, CivilHour h); +std::ostream& operator<<(std::ostream& os, CivilMinute m); +std::ostream& operator<<(std::ostream& os, CivilSecond s); + +} // namespace time_internal + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_CIVIL_TIME_H_ diff --git a/third_party/abseil_cpp/absl/time/civil_time_benchmark.cc b/third_party/abseil_cpp/absl/time/civil_time_benchmark.cc new file mode 100644 index 000000000000..f04dbe200ed2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/civil_time_benchmark.cc @@ -0,0 +1,127 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/civil_time.h" + +#include <numeric> +#include <vector> + +#include "absl/hash/hash.h" +#include "benchmark/benchmark.h" + +namespace { + +// Run on (12 X 3492 MHz CPUs); 2018-11-05T13:44:29.814239103-08:00 +// CPU: Intel Haswell with HyperThreading (6 cores) dL1:32KB dL2:256KB dL3:15MB +// Benchmark Time(ns) CPU(ns) Iterations +// ---------------------------------------------------------------- +// BM_Difference_Days 14.5 14.5 48531105 +// BM_Step_Days 12.6 12.6 54876006 +// BM_Format 587 587 1000000 +// BM_Parse 692 692 1000000 +// BM_RoundTripFormatParse 1309 1309 532075 +// BM_CivilYearAbslHash 0.710 0.710 976400000 +// BM_CivilMonthAbslHash 1.13 1.13 619500000 +// BM_CivilDayAbslHash 1.70 1.70 426000000 +// BM_CivilHourAbslHash 2.45 2.45 287600000 +// BM_CivilMinuteAbslHash 3.21 3.21 226200000 +// BM_CivilSecondAbslHash 4.10 4.10 171800000 + +void BM_Difference_Days(benchmark::State& state) { + const absl::CivilDay c(2014, 8, 22); + const absl::CivilDay epoch(1970, 1, 1); + while (state.KeepRunning()) { + const absl::civil_diff_t n = c - epoch; + benchmark::DoNotOptimize(n); + } +} +BENCHMARK(BM_Difference_Days); + +void BM_Step_Days(benchmark::State& state) { + const absl::CivilDay kStart(2014, 8, 22); + absl::CivilDay c = kStart; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(++c); + } +} +BENCHMARK(BM_Step_Days); + +void BM_Format(benchmark::State& state) { + const absl::CivilSecond c(2014, 1, 2, 3, 4, 5); + while (state.KeepRunning()) { + const std::string s = absl::FormatCivilTime(c); + benchmark::DoNotOptimize(s); + } +} +BENCHMARK(BM_Format); + +void BM_Parse(benchmark::State& state) { + const std::string f = "2014-01-02T03:04:05"; + absl::CivilSecond c; + while (state.KeepRunning()) { + const bool b = absl::ParseCivilTime(f, &c); + benchmark::DoNotOptimize(b); + } +} +BENCHMARK(BM_Parse); + +void BM_RoundTripFormatParse(benchmark::State& state) { + const absl::CivilSecond c(2014, 1, 2, 3, 4, 5); + absl::CivilSecond out; + while (state.KeepRunning()) { + const bool b = absl::ParseCivilTime(absl::FormatCivilTime(c), &out); + benchmark::DoNotOptimize(b); + } +} +BENCHMARK(BM_RoundTripFormatParse); + +template <typename T> +void BM_CivilTimeAbslHash(benchmark::State& state) { + const int kSize = 100000; + std::vector<T> civil_times(kSize); + std::iota(civil_times.begin(), civil_times.end(), T(2018)); + + absl::Hash<T> absl_hasher; + while (state.KeepRunningBatch(kSize)) { + for (const T civil_time : civil_times) { + benchmark::DoNotOptimize(absl_hasher(civil_time)); + } + } +} +void BM_CivilYearAbslHash(benchmark::State& state) { + BM_CivilTimeAbslHash<absl::CivilYear>(state); +} +void BM_CivilMonthAbslHash(benchmark::State& state) { + BM_CivilTimeAbslHash<absl::CivilMonth>(state); +} +void BM_CivilDayAbslHash(benchmark::State& state) { + BM_CivilTimeAbslHash<absl::CivilDay>(state); +} +void BM_CivilHourAbslHash(benchmark::State& state) { + BM_CivilTimeAbslHash<absl::CivilHour>(state); +} +void BM_CivilMinuteAbslHash(benchmark::State& state) { + BM_CivilTimeAbslHash<absl::CivilMinute>(state); +} +void BM_CivilSecondAbslHash(benchmark::State& state) { + BM_CivilTimeAbslHash<absl::CivilSecond>(state); +} +BENCHMARK(BM_CivilYearAbslHash); +BENCHMARK(BM_CivilMonthAbslHash); +BENCHMARK(BM_CivilDayAbslHash); +BENCHMARK(BM_CivilHourAbslHash); +BENCHMARK(BM_CivilMinuteAbslHash); +BENCHMARK(BM_CivilSecondAbslHash); + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/civil_time_test.cc b/third_party/abseil_cpp/absl/time/civil_time_test.cc new file mode 100644 index 000000000000..0ebd97adbc56 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/civil_time_test.cc @@ -0,0 +1,1243 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/civil_time.h" + +#include <limits> +#include <sstream> +#include <type_traits> + +#include "absl/base/macros.h" +#include "gtest/gtest.h" + +namespace { + +TEST(CivilTime, DefaultConstruction) { + absl::CivilSecond ss; + EXPECT_EQ("1970-01-01T00:00:00", absl::FormatCivilTime(ss)); + + absl::CivilMinute mm; + EXPECT_EQ("1970-01-01T00:00", absl::FormatCivilTime(mm)); + + absl::CivilHour hh; + EXPECT_EQ("1970-01-01T00", absl::FormatCivilTime(hh)); + + absl::CivilDay d; + EXPECT_EQ("1970-01-01", absl::FormatCivilTime(d)); + + absl::CivilMonth m; + EXPECT_EQ("1970-01", absl::FormatCivilTime(m)); + + absl::CivilYear y; + EXPECT_EQ("1970", absl::FormatCivilTime(y)); +} + +TEST(CivilTime, StructMember) { + struct S { + absl::CivilDay day; + }; + S s = {}; + EXPECT_EQ(absl::CivilDay{}, s.day); +} + +TEST(CivilTime, FieldsConstruction) { + EXPECT_EQ("2015-01-02T03:04:05", + absl::FormatCivilTime(absl::CivilSecond(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02T03:04:00", + absl::FormatCivilTime(absl::CivilSecond(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02T03:00:00", + absl::FormatCivilTime(absl::CivilSecond(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02T00:00:00", + absl::FormatCivilTime(absl::CivilSecond(2015, 1, 2))); + EXPECT_EQ("2015-01-01T00:00:00", + absl::FormatCivilTime(absl::CivilSecond(2015, 1))); + EXPECT_EQ("2015-01-01T00:00:00", + absl::FormatCivilTime(absl::CivilSecond(2015))); + + EXPECT_EQ("2015-01-02T03:04", + absl::FormatCivilTime(absl::CivilMinute(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02T03:04", + absl::FormatCivilTime(absl::CivilMinute(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02T03:00", + absl::FormatCivilTime(absl::CivilMinute(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02T00:00", + absl::FormatCivilTime(absl::CivilMinute(2015, 1, 2))); + EXPECT_EQ("2015-01-01T00:00", + absl::FormatCivilTime(absl::CivilMinute(2015, 1))); + EXPECT_EQ("2015-01-01T00:00", + absl::FormatCivilTime(absl::CivilMinute(2015))); + + EXPECT_EQ("2015-01-02T03", + absl::FormatCivilTime(absl::CivilHour(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02T03", + absl::FormatCivilTime(absl::CivilHour(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02T03", + absl::FormatCivilTime(absl::CivilHour(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02T00", + absl::FormatCivilTime(absl::CivilHour(2015, 1, 2))); + EXPECT_EQ("2015-01-01T00", + absl::FormatCivilTime(absl::CivilHour(2015, 1))); + EXPECT_EQ("2015-01-01T00", + absl::FormatCivilTime(absl::CivilHour(2015))); + + EXPECT_EQ("2015-01-02", + absl::FormatCivilTime(absl::CivilDay(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02", + absl::FormatCivilTime(absl::CivilDay(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02", + absl::FormatCivilTime(absl::CivilDay(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02", + absl::FormatCivilTime(absl::CivilDay(2015, 1, 2))); + EXPECT_EQ("2015-01-01", + absl::FormatCivilTime(absl::CivilDay(2015, 1))); + EXPECT_EQ("2015-01-01", + absl::FormatCivilTime(absl::CivilDay(2015))); + + EXPECT_EQ("2015-01", + absl::FormatCivilTime(absl::CivilMonth(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01", + absl::FormatCivilTime(absl::CivilMonth(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01", + absl::FormatCivilTime(absl::CivilMonth(2015, 1, 2, 3))); + EXPECT_EQ("2015-01", + absl::FormatCivilTime(absl::CivilMonth(2015, 1, 2))); + EXPECT_EQ("2015-01", + absl::FormatCivilTime(absl::CivilMonth(2015, 1))); + EXPECT_EQ("2015-01", + absl::FormatCivilTime(absl::CivilMonth(2015))); + + EXPECT_EQ("2015", + absl::FormatCivilTime(absl::CivilYear(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015", + absl::FormatCivilTime(absl::CivilYear(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015", + absl::FormatCivilTime(absl::CivilYear(2015, 1, 2, 3))); + EXPECT_EQ("2015", + absl::FormatCivilTime(absl::CivilYear(2015, 1, 2))); + EXPECT_EQ("2015", + absl::FormatCivilTime(absl::CivilYear(2015, 1))); + EXPECT_EQ("2015", + absl::FormatCivilTime(absl::CivilYear(2015))); +} + +TEST(CivilTime, FieldsConstructionLimits) { + const int kIntMax = std::numeric_limits<int>::max(); + EXPECT_EQ("2038-01-19T03:14:07", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, 1, 0, 0, kIntMax))); + EXPECT_EQ("6121-02-11T05:21:07", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, 1, 0, kIntMax, kIntMax))); + EXPECT_EQ("251104-11-20T12:21:07", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, 1, kIntMax, kIntMax, kIntMax))); + EXPECT_EQ("6130715-05-30T12:21:07", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, kIntMax, kIntMax, kIntMax, kIntMax))); + EXPECT_EQ("185087685-11-26T12:21:07", + absl::FormatCivilTime(absl::CivilSecond( + 1970, kIntMax, kIntMax, kIntMax, kIntMax, kIntMax))); + + const int kIntMin = std::numeric_limits<int>::min(); + EXPECT_EQ("1901-12-13T20:45:52", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, 1, 0, 0, kIntMin))); + EXPECT_EQ("-2182-11-20T18:37:52", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, 1, 0, kIntMin, kIntMin))); + EXPECT_EQ("-247165-02-11T10:37:52", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, 1, kIntMin, kIntMin, kIntMin))); + EXPECT_EQ("-6126776-08-01T10:37:52", + absl::FormatCivilTime(absl::CivilSecond( + 1970, 1, kIntMin, kIntMin, kIntMin, kIntMin))); + EXPECT_EQ("-185083747-10-31T10:37:52", + absl::FormatCivilTime(absl::CivilSecond( + 1970, kIntMin, kIntMin, kIntMin, kIntMin, kIntMin))); +} + +TEST(CivilTime, RangeLimits) { + const absl::civil_year_t kYearMax = + std::numeric_limits<absl::civil_year_t>::max(); + EXPECT_EQ(absl::CivilYear(kYearMax), + absl::CivilYear::max()); + EXPECT_EQ(absl::CivilMonth(kYearMax, 12), + absl::CivilMonth::max()); + EXPECT_EQ(absl::CivilDay(kYearMax, 12, 31), + absl::CivilDay::max()); + EXPECT_EQ(absl::CivilHour(kYearMax, 12, 31, 23), + absl::CivilHour::max()); + EXPECT_EQ(absl::CivilMinute(kYearMax, 12, 31, 23, 59), + absl::CivilMinute::max()); + EXPECT_EQ(absl::CivilSecond(kYearMax, 12, 31, 23, 59, 59), + absl::CivilSecond::max()); + + const absl::civil_year_t kYearMin = + std::numeric_limits<absl::civil_year_t>::min(); + EXPECT_EQ(absl::CivilYear(kYearMin), + absl::CivilYear::min()); + EXPECT_EQ(absl::CivilMonth(kYearMin, 1), + absl::CivilMonth::min()); + EXPECT_EQ(absl::CivilDay(kYearMin, 1, 1), + absl::CivilDay::min()); + EXPECT_EQ(absl::CivilHour(kYearMin, 1, 1, 0), + absl::CivilHour::min()); + EXPECT_EQ(absl::CivilMinute(kYearMin, 1, 1, 0, 0), + absl::CivilMinute::min()); + EXPECT_EQ(absl::CivilSecond(kYearMin, 1, 1, 0, 0, 0), + absl::CivilSecond::min()); +} + +TEST(CivilTime, ImplicitCrossAlignment) { + absl::CivilYear year(2015); + absl::CivilMonth month = year; + absl::CivilDay day = month; + absl::CivilHour hour = day; + absl::CivilMinute minute = hour; + absl::CivilSecond second = minute; + + second = year; + EXPECT_EQ(second, year); + second = month; + EXPECT_EQ(second, month); + second = day; + EXPECT_EQ(second, day); + second = hour; + EXPECT_EQ(second, hour); + second = minute; + EXPECT_EQ(second, minute); + + minute = year; + EXPECT_EQ(minute, year); + minute = month; + EXPECT_EQ(minute, month); + minute = day; + EXPECT_EQ(minute, day); + minute = hour; + EXPECT_EQ(minute, hour); + + hour = year; + EXPECT_EQ(hour, year); + hour = month; + EXPECT_EQ(hour, month); + hour = day; + EXPECT_EQ(hour, day); + + day = year; + EXPECT_EQ(day, year); + day = month; + EXPECT_EQ(day, month); + + month = year; + EXPECT_EQ(month, year); + + // Ensures unsafe conversions are not allowed. + EXPECT_FALSE( + (std::is_convertible<absl::CivilSecond, absl::CivilMinute>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilSecond, absl::CivilHour>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilSecond, absl::CivilDay>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilSecond, absl::CivilMonth>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilSecond, absl::CivilYear>::value)); + + EXPECT_FALSE( + (std::is_convertible<absl::CivilMinute, absl::CivilHour>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilMinute, absl::CivilDay>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilMinute, absl::CivilMonth>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilMinute, absl::CivilYear>::value)); + + EXPECT_FALSE( + (std::is_convertible<absl::CivilHour, absl::CivilDay>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilHour, absl::CivilMonth>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilHour, absl::CivilYear>::value)); + + EXPECT_FALSE( + (std::is_convertible<absl::CivilDay, absl::CivilMonth>::value)); + EXPECT_FALSE( + (std::is_convertible<absl::CivilDay, absl::CivilYear>::value)); + + EXPECT_FALSE( + (std::is_convertible<absl::CivilMonth, absl::CivilYear>::value)); +} + +TEST(CivilTime, ExplicitCrossAlignment) { + // + // Assign from smaller units -> larger units + // + + absl::CivilSecond second(2015, 1, 2, 3, 4, 5); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(second)); + + absl::CivilMinute minute(second); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(minute)); + + absl::CivilHour hour(minute); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(hour)); + + absl::CivilDay day(hour); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(day)); + + absl::CivilMonth month(day); + EXPECT_EQ("2015-01", absl::FormatCivilTime(month)); + + absl::CivilYear year(month); + EXPECT_EQ("2015", absl::FormatCivilTime(year)); + + // + // Now assign from larger units -> smaller units + // + + month = absl::CivilMonth(year); + EXPECT_EQ("2015-01", absl::FormatCivilTime(month)); + + day = absl::CivilDay(month); + EXPECT_EQ("2015-01-01", absl::FormatCivilTime(day)); + + hour = absl::CivilHour(day); + EXPECT_EQ("2015-01-01T00", absl::FormatCivilTime(hour)); + + minute = absl::CivilMinute(hour); + EXPECT_EQ("2015-01-01T00:00", absl::FormatCivilTime(minute)); + + second = absl::CivilSecond(minute); + EXPECT_EQ("2015-01-01T00:00:00", absl::FormatCivilTime(second)); +} + +// Metafunction to test whether difference is allowed between two types. +template <typename T1, typename T2> +struct HasDiff { + template <typename U1, typename U2> + static std::false_type test(...); + template <typename U1, typename U2> + static std::true_type test(decltype(std::declval<U1>() - std::declval<U2>())); + static constexpr bool value = decltype(test<T1, T2>(0))::value; +}; + +TEST(CivilTime, DisallowCrossAlignedDifference) { + // Difference is allowed between types with the same alignment. + static_assert(HasDiff<absl::CivilSecond, absl::CivilSecond>::value, ""); + static_assert(HasDiff<absl::CivilMinute, absl::CivilMinute>::value, ""); + static_assert(HasDiff<absl::CivilHour, absl::CivilHour>::value, ""); + static_assert(HasDiff<absl::CivilDay, absl::CivilDay>::value, ""); + static_assert(HasDiff<absl::CivilMonth, absl::CivilMonth>::value, ""); + static_assert(HasDiff<absl::CivilYear, absl::CivilYear>::value, ""); + + // Difference is disallowed between types with different alignments. + static_assert(!HasDiff<absl::CivilSecond, absl::CivilMinute>::value, ""); + static_assert(!HasDiff<absl::CivilSecond, absl::CivilHour>::value, ""); + static_assert(!HasDiff<absl::CivilSecond, absl::CivilDay>::value, ""); + static_assert(!HasDiff<absl::CivilSecond, absl::CivilMonth>::value, ""); + static_assert(!HasDiff<absl::CivilSecond, absl::CivilYear>::value, ""); + + static_assert(!HasDiff<absl::CivilMinute, absl::CivilHour>::value, ""); + static_assert(!HasDiff<absl::CivilMinute, absl::CivilDay>::value, ""); + static_assert(!HasDiff<absl::CivilMinute, absl::CivilMonth>::value, ""); + static_assert(!HasDiff<absl::CivilMinute, absl::CivilYear>::value, ""); + + static_assert(!HasDiff<absl::CivilHour, absl::CivilDay>::value, ""); + static_assert(!HasDiff<absl::CivilHour, absl::CivilMonth>::value, ""); + static_assert(!HasDiff<absl::CivilHour, absl::CivilYear>::value, ""); + + static_assert(!HasDiff<absl::CivilDay, absl::CivilMonth>::value, ""); + static_assert(!HasDiff<absl::CivilDay, absl::CivilYear>::value, ""); + + static_assert(!HasDiff<absl::CivilMonth, absl::CivilYear>::value, ""); +} + +TEST(CivilTime, ValueSemantics) { + const absl::CivilHour a(2015, 1, 2, 3); + const absl::CivilHour b = a; + const absl::CivilHour c(b); + absl::CivilHour d; + d = c; + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(d)); +} + +TEST(CivilTime, Relational) { + // Tests that the alignment unit is ignored in comparison. + const absl::CivilYear year(2014); + const absl::CivilMonth month(year); + EXPECT_EQ(year, month); + +#define TEST_RELATIONAL(OLDER, YOUNGER) \ + do { \ + EXPECT_FALSE(OLDER < OLDER); \ + EXPECT_FALSE(OLDER > OLDER); \ + EXPECT_TRUE(OLDER >= OLDER); \ + EXPECT_TRUE(OLDER <= OLDER); \ + EXPECT_FALSE(YOUNGER < YOUNGER); \ + EXPECT_FALSE(YOUNGER > YOUNGER); \ + EXPECT_TRUE(YOUNGER >= YOUNGER); \ + EXPECT_TRUE(YOUNGER <= YOUNGER); \ + EXPECT_EQ(OLDER, OLDER); \ + EXPECT_NE(OLDER, YOUNGER); \ + EXPECT_LT(OLDER, YOUNGER); \ + EXPECT_LE(OLDER, YOUNGER); \ + EXPECT_GT(YOUNGER, OLDER); \ + EXPECT_GE(YOUNGER, OLDER); \ + } while (0) + + // Alignment is ignored in comparison (verified above), so CivilSecond is + // used to test comparison in all field positions. + TEST_RELATIONAL(absl::CivilSecond(2014, 1, 1, 0, 0, 0), + absl::CivilSecond(2015, 1, 1, 0, 0, 0)); + TEST_RELATIONAL(absl::CivilSecond(2014, 1, 1, 0, 0, 0), + absl::CivilSecond(2014, 2, 1, 0, 0, 0)); + TEST_RELATIONAL(absl::CivilSecond(2014, 1, 1, 0, 0, 0), + absl::CivilSecond(2014, 1, 2, 0, 0, 0)); + TEST_RELATIONAL(absl::CivilSecond(2014, 1, 1, 0, 0, 0), + absl::CivilSecond(2014, 1, 1, 1, 0, 0)); + TEST_RELATIONAL(absl::CivilSecond(2014, 1, 1, 1, 0, 0), + absl::CivilSecond(2014, 1, 1, 1, 1, 0)); + TEST_RELATIONAL(absl::CivilSecond(2014, 1, 1, 1, 1, 0), + absl::CivilSecond(2014, 1, 1, 1, 1, 1)); + + // Tests the relational operators of two different civil-time types. + TEST_RELATIONAL(absl::CivilDay(2014, 1, 1), + absl::CivilMinute(2014, 1, 1, 1, 1)); + TEST_RELATIONAL(absl::CivilDay(2014, 1, 1), + absl::CivilMonth(2014, 2)); + +#undef TEST_RELATIONAL +} + +TEST(CivilTime, Arithmetic) { + absl::CivilSecond second(2015, 1, 2, 3, 4, 5); + EXPECT_EQ("2015-01-02T03:04:06", absl::FormatCivilTime(second += 1)); + EXPECT_EQ("2015-01-02T03:04:07", absl::FormatCivilTime(second + 1)); + EXPECT_EQ("2015-01-02T03:04:08", absl::FormatCivilTime(2 + second)); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(second - 1)); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(second -= 1)); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(second++)); + EXPECT_EQ("2015-01-02T03:04:07", absl::FormatCivilTime(++second)); + EXPECT_EQ("2015-01-02T03:04:07", absl::FormatCivilTime(second--)); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(--second)); + + absl::CivilMinute minute(2015, 1, 2, 3, 4); + EXPECT_EQ("2015-01-02T03:05", absl::FormatCivilTime(minute += 1)); + EXPECT_EQ("2015-01-02T03:06", absl::FormatCivilTime(minute + 1)); + EXPECT_EQ("2015-01-02T03:07", absl::FormatCivilTime(2 + minute)); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(minute - 1)); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(minute -= 1)); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(minute++)); + EXPECT_EQ("2015-01-02T03:06", absl::FormatCivilTime(++minute)); + EXPECT_EQ("2015-01-02T03:06", absl::FormatCivilTime(minute--)); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(--minute)); + + absl::CivilHour hour(2015, 1, 2, 3); + EXPECT_EQ("2015-01-02T04", absl::FormatCivilTime(hour += 1)); + EXPECT_EQ("2015-01-02T05", absl::FormatCivilTime(hour + 1)); + EXPECT_EQ("2015-01-02T06", absl::FormatCivilTime(2 + hour)); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(hour - 1)); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(hour -= 1)); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(hour++)); + EXPECT_EQ("2015-01-02T05", absl::FormatCivilTime(++hour)); + EXPECT_EQ("2015-01-02T05", absl::FormatCivilTime(hour--)); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(--hour)); + + absl::CivilDay day(2015, 1, 2); + EXPECT_EQ("2015-01-03", absl::FormatCivilTime(day += 1)); + EXPECT_EQ("2015-01-04", absl::FormatCivilTime(day + 1)); + EXPECT_EQ("2015-01-05", absl::FormatCivilTime(2 + day)); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(day - 1)); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(day -= 1)); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(day++)); + EXPECT_EQ("2015-01-04", absl::FormatCivilTime(++day)); + EXPECT_EQ("2015-01-04", absl::FormatCivilTime(day--)); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(--day)); + + absl::CivilMonth month(2015, 1); + EXPECT_EQ("2015-02", absl::FormatCivilTime(month += 1)); + EXPECT_EQ("2015-03", absl::FormatCivilTime(month + 1)); + EXPECT_EQ("2015-04", absl::FormatCivilTime(2 + month)); + EXPECT_EQ("2015-01", absl::FormatCivilTime(month - 1)); + EXPECT_EQ("2015-01", absl::FormatCivilTime(month -= 1)); + EXPECT_EQ("2015-01", absl::FormatCivilTime(month++)); + EXPECT_EQ("2015-03", absl::FormatCivilTime(++month)); + EXPECT_EQ("2015-03", absl::FormatCivilTime(month--)); + EXPECT_EQ("2015-01", absl::FormatCivilTime(--month)); + + absl::CivilYear year(2015); + EXPECT_EQ("2016", absl::FormatCivilTime(year += 1)); + EXPECT_EQ("2017", absl::FormatCivilTime(year + 1)); + EXPECT_EQ("2018", absl::FormatCivilTime(2 + year)); + EXPECT_EQ("2015", absl::FormatCivilTime(year - 1)); + EXPECT_EQ("2015", absl::FormatCivilTime(year -= 1)); + EXPECT_EQ("2015", absl::FormatCivilTime(year++)); + EXPECT_EQ("2017", absl::FormatCivilTime(++year)); + EXPECT_EQ("2017", absl::FormatCivilTime(year--)); + EXPECT_EQ("2015", absl::FormatCivilTime(--year)); +} + +TEST(CivilTime, ArithmeticLimits) { + const int kIntMax = std::numeric_limits<int>::max(); + const int kIntMin = std::numeric_limits<int>::min(); + + absl::CivilSecond second(1970, 1, 1, 0, 0, 0); + second += kIntMax; + EXPECT_EQ("2038-01-19T03:14:07", absl::FormatCivilTime(second)); + second -= kIntMax; + EXPECT_EQ("1970-01-01T00:00:00", absl::FormatCivilTime(second)); + second += kIntMin; + EXPECT_EQ("1901-12-13T20:45:52", absl::FormatCivilTime(second)); + second -= kIntMin; + EXPECT_EQ("1970-01-01T00:00:00", absl::FormatCivilTime(second)); + + absl::CivilMinute minute(1970, 1, 1, 0, 0); + minute += kIntMax; + EXPECT_EQ("6053-01-23T02:07", absl::FormatCivilTime(minute)); + minute -= kIntMax; + EXPECT_EQ("1970-01-01T00:00", absl::FormatCivilTime(minute)); + minute += kIntMin; + EXPECT_EQ("-2114-12-08T21:52", absl::FormatCivilTime(minute)); + minute -= kIntMin; + EXPECT_EQ("1970-01-01T00:00", absl::FormatCivilTime(minute)); + + absl::CivilHour hour(1970, 1, 1, 0); + hour += kIntMax; + EXPECT_EQ("246953-10-09T07", absl::FormatCivilTime(hour)); + hour -= kIntMax; + EXPECT_EQ("1970-01-01T00", absl::FormatCivilTime(hour)); + hour += kIntMin; + EXPECT_EQ("-243014-03-24T16", absl::FormatCivilTime(hour)); + hour -= kIntMin; + EXPECT_EQ("1970-01-01T00", absl::FormatCivilTime(hour)); + + absl::CivilDay day(1970, 1, 1); + day += kIntMax; + EXPECT_EQ("5881580-07-11", absl::FormatCivilTime(day)); + day -= kIntMax; + EXPECT_EQ("1970-01-01", absl::FormatCivilTime(day)); + day += kIntMin; + EXPECT_EQ("-5877641-06-23", absl::FormatCivilTime(day)); + day -= kIntMin; + EXPECT_EQ("1970-01-01", absl::FormatCivilTime(day)); + + absl::CivilMonth month(1970, 1); + month += kIntMax; + EXPECT_EQ("178958940-08", absl::FormatCivilTime(month)); + month -= kIntMax; + EXPECT_EQ("1970-01", absl::FormatCivilTime(month)); + month += kIntMin; + EXPECT_EQ("-178955001-05", absl::FormatCivilTime(month)); + month -= kIntMin; + EXPECT_EQ("1970-01", absl::FormatCivilTime(month)); + + absl::CivilYear year(0); + year += kIntMax; + EXPECT_EQ("2147483647", absl::FormatCivilTime(year)); + year -= kIntMax; + EXPECT_EQ("0", absl::FormatCivilTime(year)); + year += kIntMin; + EXPECT_EQ("-2147483648", absl::FormatCivilTime(year)); + year -= kIntMin; + EXPECT_EQ("0", absl::FormatCivilTime(year)); +} + +TEST(CivilTime, Difference) { + absl::CivilSecond second(2015, 1, 2, 3, 4, 5); + EXPECT_EQ(0, second - second); + EXPECT_EQ(10, (second + 10) - second); + EXPECT_EQ(-10, (second - 10) - second); + + absl::CivilMinute minute(2015, 1, 2, 3, 4); + EXPECT_EQ(0, minute - minute); + EXPECT_EQ(10, (minute + 10) - minute); + EXPECT_EQ(-10, (minute - 10) - minute); + + absl::CivilHour hour(2015, 1, 2, 3); + EXPECT_EQ(0, hour - hour); + EXPECT_EQ(10, (hour + 10) - hour); + EXPECT_EQ(-10, (hour - 10) - hour); + + absl::CivilDay day(2015, 1, 2); + EXPECT_EQ(0, day - day); + EXPECT_EQ(10, (day + 10) - day); + EXPECT_EQ(-10, (day - 10) - day); + + absl::CivilMonth month(2015, 1); + EXPECT_EQ(0, month - month); + EXPECT_EQ(10, (month + 10) - month); + EXPECT_EQ(-10, (month - 10) - month); + + absl::CivilYear year(2015); + EXPECT_EQ(0, year - year); + EXPECT_EQ(10, (year + 10) - year); + EXPECT_EQ(-10, (year - 10) - year); +} + +TEST(CivilTime, DifferenceLimits) { + const absl::civil_diff_t kDiffMax = + std::numeric_limits<absl::civil_diff_t>::max(); + const absl::civil_diff_t kDiffMin = + std::numeric_limits<absl::civil_diff_t>::min(); + + // Check day arithmetic at the end of the year range. + const absl::CivilDay max_day(kDiffMax, 12, 31); + EXPECT_EQ(1, max_day - (max_day - 1)); + EXPECT_EQ(-1, (max_day - 1) - max_day); + + // Check day arithmetic at the start of the year range. + const absl::CivilDay min_day(kDiffMin, 1, 1); + EXPECT_EQ(1, (min_day + 1) - min_day); + EXPECT_EQ(-1, min_day - (min_day + 1)); + + // Check the limits of the return value. + const absl::CivilDay d1(1970, 1, 1); + const absl::CivilDay d2(25252734927768524, 7, 27); + EXPECT_EQ(kDiffMax, d2 - d1); + EXPECT_EQ(kDiffMin, d1 - (d2 + 1)); +} + +TEST(CivilTime, Properties) { + absl::CivilSecond ss(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, ss.year()); + EXPECT_EQ(2, ss.month()); + EXPECT_EQ(3, ss.day()); + EXPECT_EQ(4, ss.hour()); + EXPECT_EQ(5, ss.minute()); + EXPECT_EQ(6, ss.second()); + EXPECT_EQ(absl::Weekday::tuesday, absl::GetWeekday(ss)); + EXPECT_EQ(34, absl::GetYearDay(ss)); + + absl::CivilMinute mm(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, mm.year()); + EXPECT_EQ(2, mm.month()); + EXPECT_EQ(3, mm.day()); + EXPECT_EQ(4, mm.hour()); + EXPECT_EQ(5, mm.minute()); + EXPECT_EQ(0, mm.second()); + EXPECT_EQ(absl::Weekday::tuesday, absl::GetWeekday(mm)); + EXPECT_EQ(34, absl::GetYearDay(mm)); + + absl::CivilHour hh(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, hh.year()); + EXPECT_EQ(2, hh.month()); + EXPECT_EQ(3, hh.day()); + EXPECT_EQ(4, hh.hour()); + EXPECT_EQ(0, hh.minute()); + EXPECT_EQ(0, hh.second()); + EXPECT_EQ(absl::Weekday::tuesday, absl::GetWeekday(hh)); + EXPECT_EQ(34, absl::GetYearDay(hh)); + + absl::CivilDay d(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, d.year()); + EXPECT_EQ(2, d.month()); + EXPECT_EQ(3, d.day()); + EXPECT_EQ(0, d.hour()); + EXPECT_EQ(0, d.minute()); + EXPECT_EQ(0, d.second()); + EXPECT_EQ(absl::Weekday::tuesday, absl::GetWeekday(d)); + EXPECT_EQ(34, absl::GetYearDay(d)); + + absl::CivilMonth m(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, m.year()); + EXPECT_EQ(2, m.month()); + EXPECT_EQ(1, m.day()); + EXPECT_EQ(0, m.hour()); + EXPECT_EQ(0, m.minute()); + EXPECT_EQ(0, m.second()); + EXPECT_EQ(absl::Weekday::sunday, absl::GetWeekday(m)); + EXPECT_EQ(32, absl::GetYearDay(m)); + + absl::CivilYear y(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, y.year()); + EXPECT_EQ(1, y.month()); + EXPECT_EQ(1, y.day()); + EXPECT_EQ(0, y.hour()); + EXPECT_EQ(0, y.minute()); + EXPECT_EQ(0, y.second()); + EXPECT_EQ(absl::Weekday::thursday, absl::GetWeekday(y)); + EXPECT_EQ(1, absl::GetYearDay(y)); +} + +TEST(CivilTime, Format) { + absl::CivilSecond ss; + EXPECT_EQ("1970-01-01T00:00:00", absl::FormatCivilTime(ss)); + + absl::CivilMinute mm; + EXPECT_EQ("1970-01-01T00:00", absl::FormatCivilTime(mm)); + + absl::CivilHour hh; + EXPECT_EQ("1970-01-01T00", absl::FormatCivilTime(hh)); + + absl::CivilDay d; + EXPECT_EQ("1970-01-01", absl::FormatCivilTime(d)); + + absl::CivilMonth m; + EXPECT_EQ("1970-01", absl::FormatCivilTime(m)); + + absl::CivilYear y; + EXPECT_EQ("1970", absl::FormatCivilTime(y)); +} + +TEST(CivilTime, Parse) { + absl::CivilSecond ss; + absl::CivilMinute mm; + absl::CivilHour hh; + absl::CivilDay d; + absl::CivilMonth m; + absl::CivilYear y; + + // CivilSecond OK; others fail + EXPECT_TRUE(absl::ParseCivilTime("2015-01-02T03:04:05", &ss)); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(ss)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04:05", &mm)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04:05", &hh)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04:05", &d)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04:05", &m)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04:05", &y)); + + // CivilMinute OK; others fail + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04", &ss)); + EXPECT_TRUE(absl::ParseCivilTime("2015-01-02T03:04", &mm)); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(mm)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04", &hh)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04", &d)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04", &m)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03:04", &y)); + + // CivilHour OK; others fail + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03", &ss)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03", &mm)); + EXPECT_TRUE(absl::ParseCivilTime("2015-01-02T03", &hh)); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(hh)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03", &d)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03", &m)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02T03", &y)); + + // CivilDay OK; others fail + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02", &ss)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02", &mm)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02", &hh)); + EXPECT_TRUE(absl::ParseCivilTime("2015-01-02", &d)); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(d)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02", &m)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01-02", &y)); + + // CivilMonth OK; others fail + EXPECT_FALSE(absl::ParseCivilTime("2015-01", &ss)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01", &mm)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01", &hh)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01", &d)); + EXPECT_TRUE(absl::ParseCivilTime("2015-01", &m)); + EXPECT_EQ("2015-01", absl::FormatCivilTime(m)); + EXPECT_FALSE(absl::ParseCivilTime("2015-01", &y)); + + // CivilYear OK; others fail + EXPECT_FALSE(absl::ParseCivilTime("2015", &ss)); + EXPECT_FALSE(absl::ParseCivilTime("2015", &mm)); + EXPECT_FALSE(absl::ParseCivilTime("2015", &hh)); + EXPECT_FALSE(absl::ParseCivilTime("2015", &d)); + EXPECT_FALSE(absl::ParseCivilTime("2015", &m)); + EXPECT_TRUE(absl::ParseCivilTime("2015", &y)); + EXPECT_EQ("2015", absl::FormatCivilTime(y)); +} + +TEST(CivilTime, FormatAndParseLenient) { + absl::CivilSecond ss; + EXPECT_EQ("1970-01-01T00:00:00", absl::FormatCivilTime(ss)); + + absl::CivilMinute mm; + EXPECT_EQ("1970-01-01T00:00", absl::FormatCivilTime(mm)); + + absl::CivilHour hh; + EXPECT_EQ("1970-01-01T00", absl::FormatCivilTime(hh)); + + absl::CivilDay d; + EXPECT_EQ("1970-01-01", absl::FormatCivilTime(d)); + + absl::CivilMonth m; + EXPECT_EQ("1970-01", absl::FormatCivilTime(m)); + + absl::CivilYear y; + EXPECT_EQ("1970", absl::FormatCivilTime(y)); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-01-02T03:04:05", &ss)); + EXPECT_EQ("2015-01-02T03:04:05", absl::FormatCivilTime(ss)); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-01-02T03:04:05", &mm)); + EXPECT_EQ("2015-01-02T03:04", absl::FormatCivilTime(mm)); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-01-02T03:04:05", &hh)); + EXPECT_EQ("2015-01-02T03", absl::FormatCivilTime(hh)); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-01-02T03:04:05", &d)); + EXPECT_EQ("2015-01-02", absl::FormatCivilTime(d)); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-01-02T03:04:05", &m)); + EXPECT_EQ("2015-01", absl::FormatCivilTime(m)); + + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-01-02T03:04:05", &y)); + EXPECT_EQ("2015", absl::FormatCivilTime(y)); +} + +TEST(CivilTime, ParseEdgeCases) { + absl::CivilSecond ss; + EXPECT_TRUE( + absl::ParseLenientCivilTime("9223372036854775807-12-31T23:59:59", &ss)); + EXPECT_EQ("9223372036854775807-12-31T23:59:59", absl::FormatCivilTime(ss)); + EXPECT_TRUE( + absl::ParseLenientCivilTime("-9223372036854775808-01-01T00:00:00", &ss)); + EXPECT_EQ("-9223372036854775808-01-01T00:00:00", absl::FormatCivilTime(ss)); + + absl::CivilMinute mm; + EXPECT_TRUE( + absl::ParseLenientCivilTime("9223372036854775807-12-31T23:59", &mm)); + EXPECT_EQ("9223372036854775807-12-31T23:59", absl::FormatCivilTime(mm)); + EXPECT_TRUE( + absl::ParseLenientCivilTime("-9223372036854775808-01-01T00:00", &mm)); + EXPECT_EQ("-9223372036854775808-01-01T00:00", absl::FormatCivilTime(mm)); + + absl::CivilHour hh; + EXPECT_TRUE( + absl::ParseLenientCivilTime("9223372036854775807-12-31T23", &hh)); + EXPECT_EQ("9223372036854775807-12-31T23", absl::FormatCivilTime(hh)); + EXPECT_TRUE( + absl::ParseLenientCivilTime("-9223372036854775808-01-01T00", &hh)); + EXPECT_EQ("-9223372036854775808-01-01T00", absl::FormatCivilTime(hh)); + + absl::CivilDay d; + EXPECT_TRUE(absl::ParseLenientCivilTime("9223372036854775807-12-31", &d)); + EXPECT_EQ("9223372036854775807-12-31", absl::FormatCivilTime(d)); + EXPECT_TRUE(absl::ParseLenientCivilTime("-9223372036854775808-01-01", &d)); + EXPECT_EQ("-9223372036854775808-01-01", absl::FormatCivilTime(d)); + + absl::CivilMonth m; + EXPECT_TRUE(absl::ParseLenientCivilTime("9223372036854775807-12", &m)); + EXPECT_EQ("9223372036854775807-12", absl::FormatCivilTime(m)); + EXPECT_TRUE(absl::ParseLenientCivilTime("-9223372036854775808-01", &m)); + EXPECT_EQ("-9223372036854775808-01", absl::FormatCivilTime(m)); + + absl::CivilYear y; + EXPECT_TRUE(absl::ParseLenientCivilTime("9223372036854775807", &y)); + EXPECT_EQ("9223372036854775807", absl::FormatCivilTime(y)); + EXPECT_TRUE(absl::ParseLenientCivilTime("-9223372036854775808", &y)); + EXPECT_EQ("-9223372036854775808", absl::FormatCivilTime(y)); + + // Tests some valid, but interesting, cases + EXPECT_TRUE(absl::ParseLenientCivilTime("0", &ss)) << ss; + EXPECT_EQ(absl::CivilYear(0), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime("0-1", &ss)) << ss; + EXPECT_EQ(absl::CivilMonth(0, 1), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime(" 2015 ", &ss)) << ss; + EXPECT_EQ(absl::CivilYear(2015), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime(" 2015-6 ", &ss)) << ss; + EXPECT_EQ(absl::CivilMonth(2015, 6), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-6-7", &ss)) << ss; + EXPECT_EQ(absl::CivilDay(2015, 6, 7), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime(" 2015-6-7 ", &ss)) << ss; + EXPECT_EQ(absl::CivilDay(2015, 6, 7), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime("2015-06-07T10:11:12 ", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(2015, 6, 7, 10, 11, 12), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime(" 2015-06-07T10:11:12 ", &ss)) << ss; + EXPECT_EQ(absl::CivilSecond(2015, 6, 7, 10, 11, 12), ss); + EXPECT_TRUE(absl::ParseLenientCivilTime("-01-01", &ss)) << ss; + EXPECT_EQ(absl::CivilMonth(-1, 1), ss); + + // Tests some invalid cases + EXPECT_FALSE(absl::ParseLenientCivilTime("01-01-2015", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015-", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("0xff-01", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015-02-30T04:05:06", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015-02-03T04:05:96", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("X2015-02-03T04:05:06", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015-02-03T04:05:003", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015 -02-03T04:05:06", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015-02-03-04:05:06", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("2015:02:03T04-05-06", &ss)) << ss; + EXPECT_FALSE(absl::ParseLenientCivilTime("9223372036854775808", &y)) << y; +} + +TEST(CivilTime, OutputStream) { + absl::CivilSecond cs(2016, 2, 3, 4, 5, 6); + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::CivilYear(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016.................X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::CivilMonth(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02..............X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::CivilDay(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03...........X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::CivilHour(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03T04........X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::CivilMinute(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03T04:05.....X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::CivilSecond(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03T04:05:06..X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << absl::Weekday::wednesday; + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..Wednesday............X..", ss.str()); + } +} + +TEST(CivilTime, Weekday) { + absl::CivilDay d(1970, 1, 1); + EXPECT_EQ(absl::Weekday::thursday, absl::GetWeekday(d)) << d; + + // We used to get this wrong for years < -30. + d = absl::CivilDay(-31, 12, 24); + EXPECT_EQ(absl::Weekday::wednesday, absl::GetWeekday(d)) << d; +} + +TEST(CivilTime, NextPrevWeekday) { + // Jan 1, 1970 was a Thursday. + const absl::CivilDay thursday(1970, 1, 1); + + // Thursday -> Thursday + absl::CivilDay d = absl::NextWeekday(thursday, absl::Weekday::thursday); + EXPECT_EQ(7, d - thursday) << d; + EXPECT_EQ(d - 14, absl::PrevWeekday(thursday, absl::Weekday::thursday)); + + // Thursday -> Friday + d = absl::NextWeekday(thursday, absl::Weekday::friday); + EXPECT_EQ(1, d - thursday) << d; + EXPECT_EQ(d - 7, absl::PrevWeekday(thursday, absl::Weekday::friday)); + + // Thursday -> Saturday + d = absl::NextWeekday(thursday, absl::Weekday::saturday); + EXPECT_EQ(2, d - thursday) << d; + EXPECT_EQ(d - 7, absl::PrevWeekday(thursday, absl::Weekday::saturday)); + + // Thursday -> Sunday + d = absl::NextWeekday(thursday, absl::Weekday::sunday); + EXPECT_EQ(3, d - thursday) << d; + EXPECT_EQ(d - 7, absl::PrevWeekday(thursday, absl::Weekday::sunday)); + + // Thursday -> Monday + d = absl::NextWeekday(thursday, absl::Weekday::monday); + EXPECT_EQ(4, d - thursday) << d; + EXPECT_EQ(d - 7, absl::PrevWeekday(thursday, absl::Weekday::monday)); + + // Thursday -> Tuesday + d = absl::NextWeekday(thursday, absl::Weekday::tuesday); + EXPECT_EQ(5, d - thursday) << d; + EXPECT_EQ(d - 7, absl::PrevWeekday(thursday, absl::Weekday::tuesday)); + + // Thursday -> Wednesday + d = absl::NextWeekday(thursday, absl::Weekday::wednesday); + EXPECT_EQ(6, d - thursday) << d; + EXPECT_EQ(d - 7, absl::PrevWeekday(thursday, absl::Weekday::wednesday)); +} + +// NOTE: Run this with --copt=-ftrapv to detect overflow problems. +TEST(CivilTime, DifferenceWithHugeYear) { + absl::CivilDay d1(9223372036854775807, 1, 1); + absl::CivilDay d2(9223372036854775807, 12, 31); + EXPECT_EQ(364, d2 - d1); + + d1 = absl::CivilDay(-9223372036854775807 - 1, 1, 1); + d2 = absl::CivilDay(-9223372036854775807 - 1, 12, 31); + EXPECT_EQ(365, d2 - d1); + + // Check the limits of the return value at the end of the year range. + d1 = absl::CivilDay(9223372036854775807, 1, 1); + d2 = absl::CivilDay(9198119301927009252, 6, 6); + EXPECT_EQ(9223372036854775807, d1 - d2); + d2 = d2 - 1; + EXPECT_EQ(-9223372036854775807 - 1, d2 - d1); + + // Check the limits of the return value at the start of the year range. + d1 = absl::CivilDay(-9223372036854775807 - 1, 1, 1); + d2 = absl::CivilDay(-9198119301927009254, 7, 28); + EXPECT_EQ(9223372036854775807, d2 - d1); + d2 = d2 + 1; + EXPECT_EQ(-9223372036854775807 - 1, d1 - d2); + + // Check the limits of the return value from either side of year 0. + d1 = absl::CivilDay(-12626367463883278, 9, 3); + d2 = absl::CivilDay(12626367463883277, 3, 28); + EXPECT_EQ(9223372036854775807, d2 - d1); + d2 = d2 + 1; + EXPECT_EQ(-9223372036854775807 - 1, d1 - d2); +} + +// NOTE: Run this with --copt=-ftrapv to detect overflow problems. +TEST(CivilTime, DifferenceNoIntermediateOverflow) { + // The difference up to the minute field would be below the minimum + // int64_t, but the 52 extra seconds brings us back to the minimum. + absl::CivilSecond s1(-292277022657, 1, 27, 8, 29 - 1, 52); + absl::CivilSecond s2(1970, 1, 1, 0, 0 - 1, 0); + EXPECT_EQ(-9223372036854775807 - 1, s1 - s2); + + // The difference up to the minute field would be above the maximum + // int64_t, but the -53 extra seconds brings us back to the maximum. + s1 = absl::CivilSecond(292277026596, 12, 4, 15, 30, 7 - 7); + s2 = absl::CivilSecond(1970, 1, 1, 0, 0, 0 - 7); + EXPECT_EQ(9223372036854775807, s1 - s2); +} + +TEST(CivilTime, NormalizeSimpleOverflow) { + absl::CivilSecond cs; + cs = absl::CivilSecond(2013, 11, 15, 16, 32, 59 + 1); + EXPECT_EQ("2013-11-15T16:33:00", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16, 59 + 1, 14); + EXPECT_EQ("2013-11-15T17:00:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 23 + 1, 32, 14); + EXPECT_EQ("2013-11-16T00:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 30 + 1, 16, 32, 14); + EXPECT_EQ("2013-12-01T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 12 + 1, 15, 16, 32, 14); + EXPECT_EQ("2014-01-15T16:32:14", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeSimpleUnderflow) { + absl::CivilSecond cs; + cs = absl::CivilSecond(2013, 11, 15, 16, 32, 0 - 1); + EXPECT_EQ("2013-11-15T16:31:59", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16, 0 - 1, 14); + EXPECT_EQ("2013-11-15T15:59:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 0 - 1, 32, 14); + EXPECT_EQ("2013-11-14T23:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 1 - 1, 16, 32, 14); + EXPECT_EQ("2013-10-31T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 1 - 1, 15, 16, 32, 14); + EXPECT_EQ("2012-12-15T16:32:14", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeMultipleOverflow) { + absl::CivilSecond cs(2013, 12, 31, 23, 59, 59 + 1); + EXPECT_EQ("2014-01-01T00:00:00", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeMultipleUnderflow) { + absl::CivilSecond cs(2014, 1, 1, 0, 0, 0 - 1); + EXPECT_EQ("2013-12-31T23:59:59", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeOverflowLimits) { + absl::CivilSecond cs; + + const int kintmax = std::numeric_limits<int>::max(); + cs = absl::CivilSecond(0, kintmax, kintmax, kintmax, kintmax, kintmax); + EXPECT_EQ("185085715-11-27T12:21:07", absl::FormatCivilTime(cs)); + + const int kintmin = std::numeric_limits<int>::min(); + cs = absl::CivilSecond(0, kintmin, kintmin, kintmin, kintmin, kintmin); + EXPECT_EQ("-185085717-10-31T10:37:52", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeComplexOverflow) { + absl::CivilSecond cs; + cs = absl::CivilSecond(2013, 11, 15, 16, 32, 14 + 123456789); + EXPECT_EQ("2017-10-14T14:05:23", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16, 32 + 1234567, 14); + EXPECT_EQ("2016-03-22T00:39:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16 + 123456, 32, 14); + EXPECT_EQ("2027-12-16T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15 + 1234, 16, 32, 14); + EXPECT_EQ("2017-04-02T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11 + 123, 15, 16, 32, 14); + EXPECT_EQ("2024-02-15T16:32:14", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeComplexUnderflow) { + absl::CivilSecond cs; + cs = absl::CivilSecond(1999, 3, 0, 0, 0, 0); // year 400 + EXPECT_EQ("1999-02-28T00:00:00", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16, 32, 14 - 123456789); + EXPECT_EQ("2009-12-17T18:59:05", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16, 32 - 1234567, 14); + EXPECT_EQ("2011-07-12T08:25:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15, 16 - 123456, 32, 14); + EXPECT_EQ("1999-10-16T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11, 15 - 1234, 16, 32, 14); + EXPECT_EQ("2010-06-30T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11 - 123, 15, 16, 32, 14); + EXPECT_EQ("2003-08-15T16:32:14", absl::FormatCivilTime(cs)); +} + +TEST(CivilTime, NormalizeMishmash) { + absl::CivilSecond cs; + cs = absl::CivilSecond(2013, 11 - 123, 15 + 1234, 16 - 123456, 32 + 1234567, + 14 - 123456789); + EXPECT_EQ("1991-05-09T03:06:05", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11 + 123, 15 - 1234, 16 + 123456, 32 - 1234567, + 14 + 123456789); + EXPECT_EQ("2036-05-24T05:58:23", absl::FormatCivilTime(cs)); + + cs = absl::CivilSecond(2013, 11, -146097 + 1, 16, 32, 14); + EXPECT_EQ("1613-11-01T16:32:14", absl::FormatCivilTime(cs)); + cs = absl::CivilSecond(2013, 11 + 400 * 12, -146097 + 1, 16, 32, 14); + EXPECT_EQ("2013-11-01T16:32:14", absl::FormatCivilTime(cs)); +} + +// Convert all the days from 1970-1-1 to 1970-1-146097 (aka 2369-12-31) +// and check that they normalize to the expected time. 146097 days span +// the 400-year Gregorian cycle used during normalization. +TEST(CivilTime, NormalizeAllTheDays) { + absl::CivilDay expected(1970, 1, 1); + for (int day = 1; day <= 146097; ++day) { + absl::CivilSecond cs(1970, 1, day, 0, 0, 0); + EXPECT_EQ(expected, cs); + ++expected; + } +} + +TEST(CivilTime, NormalizeWithHugeYear) { + absl::CivilMonth c(9223372036854775807, 1); + EXPECT_EQ("9223372036854775807-01", absl::FormatCivilTime(c)); + c = c - 1; // Causes normalization + EXPECT_EQ("9223372036854775806-12", absl::FormatCivilTime(c)); + + c = absl::CivilMonth(-9223372036854775807 - 1, 1); + EXPECT_EQ("-9223372036854775808-01", absl::FormatCivilTime(c)); + c = c + 12; // Causes normalization + EXPECT_EQ("-9223372036854775807-01", absl::FormatCivilTime(c)); +} + +TEST(CivilTime, LeapYears) { + const absl::CivilSecond s1(2013, 2, 28 + 1, 0, 0, 0); + EXPECT_EQ("2013-03-01T00:00:00", absl::FormatCivilTime(s1)); + + const absl::CivilSecond s2(2012, 2, 28 + 1, 0, 0, 0); + EXPECT_EQ("2012-02-29T00:00:00", absl::FormatCivilTime(s2)); + + const absl::CivilSecond s3(1900, 2, 28 + 1, 0, 0, 0); + EXPECT_EQ("1900-03-01T00:00:00", absl::FormatCivilTime(s3)); + + const struct { + int year; + int days; + struct { + int month; + int day; + } leap_day; // The date of the day after Feb 28. + } kLeapYearTable[]{ + {1900, 365, {3, 1}}, + {1999, 365, {3, 1}}, + {2000, 366, {2, 29}}, // leap year + {2001, 365, {3, 1}}, + {2002, 365, {3, 1}}, + {2003, 365, {3, 1}}, + {2004, 366, {2, 29}}, // leap year + {2005, 365, {3, 1}}, + {2006, 365, {3, 1}}, + {2007, 365, {3, 1}}, + {2008, 366, {2, 29}}, // leap year + {2009, 365, {3, 1}}, + {2100, 365, {3, 1}}, + }; + + for (int i = 0; i < ABSL_ARRAYSIZE(kLeapYearTable); ++i) { + const int y = kLeapYearTable[i].year; + const int m = kLeapYearTable[i].leap_day.month; + const int d = kLeapYearTable[i].leap_day.day; + const int n = kLeapYearTable[i].days; + + // Tests incrementing through the leap day. + const absl::CivilDay feb28(y, 2, 28); + const absl::CivilDay next_day = feb28 + 1; + EXPECT_EQ(m, next_day.month()); + EXPECT_EQ(d, next_day.day()); + + // Tests difference in days of leap years. + const absl::CivilYear year(feb28); + const absl::CivilYear next_year = year + 1; + EXPECT_EQ(n, absl::CivilDay(next_year) - absl::CivilDay(year)); + } +} + +TEST(CivilTime, FirstThursdayInMonth) { + const absl::CivilDay nov1(2014, 11, 1); + const absl::CivilDay thursday = + absl::NextWeekday(nov1 - 1, absl::Weekday::thursday); + EXPECT_EQ("2014-11-06", absl::FormatCivilTime(thursday)); + + // Bonus: Date of Thanksgiving in the United States + // Rule: Fourth Thursday of November + const absl::CivilDay thanksgiving = thursday + 7 * 3; + EXPECT_EQ("2014-11-27", absl::FormatCivilTime(thanksgiving)); +} + +TEST(CivilTime, DocumentationExample) { + absl::CivilSecond second(2015, 6, 28, 1, 2, 3); // 2015-06-28 01:02:03 + absl::CivilMinute minute(second); // 2015-06-28 01:02:00 + absl::CivilDay day(minute); // 2015-06-28 00:00:00 + + second -= 1; // 2015-06-28 01:02:02 + --second; // 2015-06-28 01:02:01 + EXPECT_EQ(minute, second - 1); // Comparison between types + EXPECT_LT(minute, second); + + // int diff = second - minute; // ERROR: Mixed types, won't compile + + absl::CivilDay june_1(2015, 6, 1); // Pass fields to c'tor. + int diff = day - june_1; // Num days between 'day' and June 1 + EXPECT_EQ(27, diff); + + // Fields smaller than alignment are floored to their minimum value. + absl::CivilDay day_floor(2015, 1, 2, 9, 9, 9); + EXPECT_EQ(0, day_floor.hour()); // 09:09:09 is floored + EXPECT_EQ(absl::CivilDay(2015, 1, 2), day_floor); + + // Unspecified fields default to their minium value + absl::CivilDay day_default(2015); // Defaults to Jan 1 + EXPECT_EQ(absl::CivilDay(2015, 1, 1), day_default); + + // Iterates all the days of June. + absl::CivilMonth june(day); // CivilDay -> CivilMonth + absl::CivilMonth july = june + 1; + for (absl::CivilDay day = june_1; day < july; ++day) { + // ... + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/clock.cc b/third_party/abseil_cpp/absl/time/clock.cc new file mode 100644 index 000000000000..6862e011be1f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/clock.cc @@ -0,0 +1,567 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/clock.h" + +#include "absl/base/attributes.h" + +#ifdef _WIN32 +#include <windows.h> +#endif + +#include <algorithm> +#include <atomic> +#include <cerrno> +#include <cstdint> +#include <ctime> +#include <limits> + +#include "absl/base/internal/spinlock.h" +#include "absl/base/internal/unscaledcycleclock.h" +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/base/thread_annotations.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +Time Now() { + // TODO(bww): Get a timespec instead so we don't have to divide. + int64_t n = absl::GetCurrentTimeNanos(); + if (n >= 0) { + return time_internal::FromUnixDuration( + time_internal::MakeDuration(n / 1000000000, n % 1000000000 * 4)); + } + return time_internal::FromUnixDuration(absl::Nanoseconds(n)); +} +ABSL_NAMESPACE_END +} // namespace absl + +// Decide if we should use the fast GetCurrentTimeNanos() algorithm +// based on the cyclecounter, otherwise just get the time directly +// from the OS on every call. This can be chosen at compile-time via +// -DABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS=[0|1] +#ifndef ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS +#if ABSL_USE_UNSCALED_CYCLECLOCK +#define ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS 1 +#else +#define ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS 0 +#endif +#endif + +#if defined(__APPLE__) || defined(_WIN32) +#include "absl/time/internal/get_current_time_chrono.inc" +#else +#include "absl/time/internal/get_current_time_posix.inc" +#endif + +// Allows override by test. +#ifndef GET_CURRENT_TIME_NANOS_FROM_SYSTEM +#define GET_CURRENT_TIME_NANOS_FROM_SYSTEM() \ + ::absl::time_internal::GetCurrentTimeNanosFromSystem() +#endif + +#if !ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS +namespace absl { +ABSL_NAMESPACE_BEGIN +int64_t GetCurrentTimeNanos() { return GET_CURRENT_TIME_NANOS_FROM_SYSTEM(); } +ABSL_NAMESPACE_END +} // namespace absl +#else // Use the cyclecounter-based implementation below. + +// Allows override by test. +#ifndef GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW +#define GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW() \ + ::absl::time_internal::UnscaledCycleClockWrapperForGetCurrentTime::Now() +#endif + +// The following counters are used only by the test code. +static int64_t stats_initializations; +static int64_t stats_reinitializations; +static int64_t stats_calibrations; +static int64_t stats_slow_paths; +static int64_t stats_fast_slow_paths; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +// This is a friend wrapper around UnscaledCycleClock::Now() +// (needed to access UnscaledCycleClock). +class UnscaledCycleClockWrapperForGetCurrentTime { + public: + static int64_t Now() { return base_internal::UnscaledCycleClock::Now(); } +}; +} // namespace time_internal + +// uint64_t is used in this module to provide an extra bit in multiplications + +// Return the time in ns as told by the kernel interface. Place in *cycleclock +// the value of the cycleclock at about the time of the syscall. +// This call represents the time base that this module synchronizes to. +// Ensures that *cycleclock does not step back by up to (1 << 16) from +// last_cycleclock, to discard small backward counter steps. (Larger steps are +// assumed to be complete resyncs, which shouldn't happen. If they do, a full +// reinitialization of the outer algorithm should occur.) +static int64_t GetCurrentTimeNanosFromKernel(uint64_t last_cycleclock, + uint64_t *cycleclock) { + // We try to read clock values at about the same time as the kernel clock. + // This value gets adjusted up or down as estimate of how long that should + // take, so we can reject attempts that take unusually long. + static std::atomic<uint64_t> approx_syscall_time_in_cycles{10 * 1000}; + + uint64_t local_approx_syscall_time_in_cycles = // local copy + approx_syscall_time_in_cycles.load(std::memory_order_relaxed); + + int64_t current_time_nanos_from_system; + uint64_t before_cycles; + uint64_t after_cycles; + uint64_t elapsed_cycles; + int loops = 0; + do { + before_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW(); + current_time_nanos_from_system = GET_CURRENT_TIME_NANOS_FROM_SYSTEM(); + after_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW(); + // elapsed_cycles is unsigned, so is large on overflow + elapsed_cycles = after_cycles - before_cycles; + if (elapsed_cycles >= local_approx_syscall_time_in_cycles && + ++loops == 20) { // clock changed frequencies? Back off. + loops = 0; + if (local_approx_syscall_time_in_cycles < 1000 * 1000) { + local_approx_syscall_time_in_cycles = + (local_approx_syscall_time_in_cycles + 1) << 1; + } + approx_syscall_time_in_cycles.store( + local_approx_syscall_time_in_cycles, + std::memory_order_relaxed); + } + } while (elapsed_cycles >= local_approx_syscall_time_in_cycles || + last_cycleclock - after_cycles < (static_cast<uint64_t>(1) << 16)); + + // Number of times in a row we've seen a kernel time call take substantially + // less than approx_syscall_time_in_cycles. + static std::atomic<uint32_t> seen_smaller{ 0 }; + + // Adjust approx_syscall_time_in_cycles to be within a factor of 2 + // of the typical time to execute one iteration of the loop above. + if ((local_approx_syscall_time_in_cycles >> 1) < elapsed_cycles) { + // measured time is no smaller than half current approximation + seen_smaller.store(0, std::memory_order_relaxed); + } else if (seen_smaller.fetch_add(1, std::memory_order_relaxed) >= 3) { + // smaller delays several times in a row; reduce approximation by 12.5% + const uint64_t new_approximation = + local_approx_syscall_time_in_cycles - + (local_approx_syscall_time_in_cycles >> 3); + approx_syscall_time_in_cycles.store(new_approximation, + std::memory_order_relaxed); + seen_smaller.store(0, std::memory_order_relaxed); + } + + *cycleclock = after_cycles; + return current_time_nanos_from_system; +} + + +// --------------------------------------------------------------------- +// An implementation of reader-write locks that use no atomic ops in the read +// case. This is a generalization of Lamport's method for reading a multiword +// clock. Increment a word on each write acquisition, using the low-order bit +// as a spinlock; the word is the high word of the "clock". Readers read the +// high word, then all other data, then the high word again, and repeat the +// read if the reads of the high words yields different answers, or an odd +// value (either case suggests possible interference from a writer). +// Here we use a spinlock to ensure only one writer at a time, rather than +// spinning on the bottom bit of the word to benefit from SpinLock +// spin-delay tuning. + +// Acquire seqlock (*seq) and return the value to be written to unlock. +static inline uint64_t SeqAcquire(std::atomic<uint64_t> *seq) { + uint64_t x = seq->fetch_add(1, std::memory_order_relaxed); + + // We put a release fence between update to *seq and writes to shared data. + // Thus all stores to shared data are effectively release operations and + // update to *seq above cannot be re-ordered past any of them. Note that + // this barrier is not for the fetch_add above. A release barrier for the + // fetch_add would be before it, not after. + std::atomic_thread_fence(std::memory_order_release); + + return x + 2; // original word plus 2 +} + +// Release seqlock (*seq) by writing x to it---a value previously returned by +// SeqAcquire. +static inline void SeqRelease(std::atomic<uint64_t> *seq, uint64_t x) { + // The unlock store to *seq must have release ordering so that all + // updates to shared data must finish before this store. + seq->store(x, std::memory_order_release); // release lock for readers +} + +// --------------------------------------------------------------------- + +// "nsscaled" is unit of time equal to a (2**kScale)th of a nanosecond. +enum { kScale = 30 }; + +// The minimum interval between samples of the time base. +// We pick enough time to amortize the cost of the sample, +// to get a reasonably accurate cycle counter rate reading, +// and not so much that calculations will overflow 64-bits. +static const uint64_t kMinNSBetweenSamples = 2000 << 20; + +// We require that kMinNSBetweenSamples shifted by kScale +// have at least a bit left over for 64-bit calculations. +static_assert(((kMinNSBetweenSamples << (kScale + 1)) >> (kScale + 1)) == + kMinNSBetweenSamples, + "cannot represent kMaxBetweenSamplesNSScaled"); + +// A reader-writer lock protecting the static locations below. +// See SeqAcquire() and SeqRelease() above. +ABSL_CONST_INIT static absl::base_internal::SpinLock lock( + absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY); +ABSL_CONST_INIT static std::atomic<uint64_t> seq(0); + +// data from a sample of the kernel's time value +struct TimeSampleAtomic { + std::atomic<uint64_t> raw_ns; // raw kernel time + std::atomic<uint64_t> base_ns; // our estimate of time + std::atomic<uint64_t> base_cycles; // cycle counter reading + std::atomic<uint64_t> nsscaled_per_cycle; // cycle period + // cycles before we'll sample again (a scaled reciprocal of the period, + // to avoid a division on the fast path). + std::atomic<uint64_t> min_cycles_per_sample; +}; +// Same again, but with non-atomic types +struct TimeSample { + uint64_t raw_ns; // raw kernel time + uint64_t base_ns; // our estimate of time + uint64_t base_cycles; // cycle counter reading + uint64_t nsscaled_per_cycle; // cycle period + uint64_t min_cycles_per_sample; // approx cycles before next sample +}; + +static struct TimeSampleAtomic last_sample; // the last sample; under seq + +static int64_t GetCurrentTimeNanosSlowPath() ABSL_ATTRIBUTE_COLD; + +// Read the contents of *atomic into *sample. +// Each field is read atomically, but to maintain atomicity between fields, +// the access must be done under a lock. +static void ReadTimeSampleAtomic(const struct TimeSampleAtomic *atomic, + struct TimeSample *sample) { + sample->base_ns = atomic->base_ns.load(std::memory_order_relaxed); + sample->base_cycles = atomic->base_cycles.load(std::memory_order_relaxed); + sample->nsscaled_per_cycle = + atomic->nsscaled_per_cycle.load(std::memory_order_relaxed); + sample->min_cycles_per_sample = + atomic->min_cycles_per_sample.load(std::memory_order_relaxed); + sample->raw_ns = atomic->raw_ns.load(std::memory_order_relaxed); +} + +// Public routine. +// Algorithm: We wish to compute real time from a cycle counter. In normal +// operation, we construct a piecewise linear approximation to the kernel time +// source, using the cycle counter value. The start of each line segment is at +// the same point as the end of the last, but may have a different slope (that +// is, a different idea of the cycle counter frequency). Every couple of +// seconds, the kernel time source is sampled and compared with the current +// approximation. A new slope is chosen that, if followed for another couple +// of seconds, will correct the error at the current position. The information +// for a sample is in the "last_sample" struct. The linear approximation is +// estimated_time = last_sample.base_ns + +// last_sample.ns_per_cycle * (counter_reading - last_sample.base_cycles) +// (ns_per_cycle is actually stored in different units and scaled, to avoid +// overflow). The base_ns of the next linear approximation is the +// estimated_time using the last approximation; the base_cycles is the cycle +// counter value at that time; the ns_per_cycle is the number of ns per cycle +// measured since the last sample, but adjusted so that most of the difference +// between the estimated_time and the kernel time will be corrected by the +// estimated time to the next sample. In normal operation, this algorithm +// relies on: +// - the cycle counter and kernel time rates not changing a lot in a few +// seconds. +// - the client calling into the code often compared to a couple of seconds, so +// the time to the next correction can be estimated. +// Any time ns_per_cycle is not known, a major error is detected, or the +// assumption about frequent calls is violated, the implementation returns the +// kernel time. It records sufficient data that a linear approximation can +// resume a little later. + +int64_t GetCurrentTimeNanos() { + // read the data from the "last_sample" struct (but don't need raw_ns yet) + // The reads of "seq" and test of the values emulate a reader lock. + uint64_t base_ns; + uint64_t base_cycles; + uint64_t nsscaled_per_cycle; + uint64_t min_cycles_per_sample; + uint64_t seq_read0; + uint64_t seq_read1; + + // If we have enough information to interpolate, the value returned will be + // derived from this cycleclock-derived time estimate. On some platforms + // (POWER) the function to retrieve this value has enough complexity to + // contribute to register pressure - reading it early before initializing + // the other pieces of the calculation minimizes spill/restore instructions, + // minimizing icache cost. + uint64_t now_cycles = GET_CURRENT_TIME_NANOS_CYCLECLOCK_NOW(); + + // Acquire pairs with the barrier in SeqRelease - if this load sees that + // store, the shared-data reads necessarily see that SeqRelease's updates + // to the same shared data. + seq_read0 = seq.load(std::memory_order_acquire); + + base_ns = last_sample.base_ns.load(std::memory_order_relaxed); + base_cycles = last_sample.base_cycles.load(std::memory_order_relaxed); + nsscaled_per_cycle = + last_sample.nsscaled_per_cycle.load(std::memory_order_relaxed); + min_cycles_per_sample = + last_sample.min_cycles_per_sample.load(std::memory_order_relaxed); + + // This acquire fence pairs with the release fence in SeqAcquire. Since it + // is sequenced between reads of shared data and seq_read1, the reads of + // shared data are effectively acquiring. + std::atomic_thread_fence(std::memory_order_acquire); + + // The shared-data reads are effectively acquire ordered, and the + // shared-data writes are effectively release ordered. Therefore if our + // shared-data reads see any of a particular update's shared-data writes, + // seq_read1 is guaranteed to see that update's SeqAcquire. + seq_read1 = seq.load(std::memory_order_relaxed); + + // Fast path. Return if min_cycles_per_sample has not yet elapsed since the + // last sample, and we read a consistent sample. The fast path activates + // only when min_cycles_per_sample is non-zero, which happens when we get an + // estimate for the cycle time. The predicate will fail if now_cycles < + // base_cycles, or if some other thread is in the slow path. + // + // Since we now read now_cycles before base_ns, it is possible for now_cycles + // to be less than base_cycles (if we were interrupted between those loads and + // last_sample was updated). This is harmless, because delta_cycles will wrap + // and report a time much much bigger than min_cycles_per_sample. In that case + // we will take the slow path. + uint64_t delta_cycles = now_cycles - base_cycles; + if (seq_read0 == seq_read1 && (seq_read0 & 1) == 0 && + delta_cycles < min_cycles_per_sample) { + return base_ns + ((delta_cycles * nsscaled_per_cycle) >> kScale); + } + return GetCurrentTimeNanosSlowPath(); +} + +// Return (a << kScale)/b. +// Zero is returned if b==0. Scaling is performed internally to +// preserve precision without overflow. +static uint64_t SafeDivideAndScale(uint64_t a, uint64_t b) { + // Find maximum safe_shift so that + // 0 <= safe_shift <= kScale and (a << safe_shift) does not overflow. + int safe_shift = kScale; + while (((a << safe_shift) >> safe_shift) != a) { + safe_shift--; + } + uint64_t scaled_b = b >> (kScale - safe_shift); + uint64_t quotient = 0; + if (scaled_b != 0) { + quotient = (a << safe_shift) / scaled_b; + } + return quotient; +} + +static uint64_t UpdateLastSample( + uint64_t now_cycles, uint64_t now_ns, uint64_t delta_cycles, + const struct TimeSample *sample) ABSL_ATTRIBUTE_COLD; + +// The slow path of GetCurrentTimeNanos(). This is taken while gathering +// initial samples, when enough time has elapsed since the last sample, and if +// any other thread is writing to last_sample. +// +// Manually mark this 'noinline' to minimize stack frame size of the fast +// path. Without this, sometimes a compiler may inline this big block of code +// into the fast path. That causes lots of register spills and reloads that +// are unnecessary unless the slow path is taken. +// +// TODO(absl-team): Remove this attribute when our compiler is smart enough +// to do the right thing. +ABSL_ATTRIBUTE_NOINLINE +static int64_t GetCurrentTimeNanosSlowPath() ABSL_LOCKS_EXCLUDED(lock) { + // Serialize access to slow-path. Fast-path readers are not blocked yet, and + // code below must not modify last_sample until the seqlock is acquired. + lock.Lock(); + + // Sample the kernel time base. This is the definition of + // "now" if we take the slow path. + static uint64_t last_now_cycles; // protected by lock + uint64_t now_cycles; + uint64_t now_ns = GetCurrentTimeNanosFromKernel(last_now_cycles, &now_cycles); + last_now_cycles = now_cycles; + + uint64_t estimated_base_ns; + + // ---------- + // Read the "last_sample" values again; this time holding the write lock. + struct TimeSample sample; + ReadTimeSampleAtomic(&last_sample, &sample); + + // ---------- + // Try running the fast path again; another thread may have updated the + // sample between our run of the fast path and the sample we just read. + uint64_t delta_cycles = now_cycles - sample.base_cycles; + if (delta_cycles < sample.min_cycles_per_sample) { + // Another thread updated the sample. This path does not take the seqlock + // so that blocked readers can make progress without blocking new readers. + estimated_base_ns = sample.base_ns + + ((delta_cycles * sample.nsscaled_per_cycle) >> kScale); + stats_fast_slow_paths++; + } else { + estimated_base_ns = + UpdateLastSample(now_cycles, now_ns, delta_cycles, &sample); + } + + lock.Unlock(); + + return estimated_base_ns; +} + +// Main part of the algorithm. Locks out readers, updates the approximation +// using the new sample from the kernel, and stores the result in last_sample +// for readers. Returns the new estimated time. +static uint64_t UpdateLastSample(uint64_t now_cycles, uint64_t now_ns, + uint64_t delta_cycles, + const struct TimeSample *sample) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(lock) { + uint64_t estimated_base_ns = now_ns; + uint64_t lock_value = SeqAcquire(&seq); // acquire seqlock to block readers + + // The 5s in the next if-statement limits the time for which we will trust + // the cycle counter and our last sample to give a reasonable result. + // Errors in the rate of the source clock can be multiplied by the ratio + // between this limit and kMinNSBetweenSamples. + if (sample->raw_ns == 0 || // no recent sample, or clock went backwards + sample->raw_ns + static_cast<uint64_t>(5) * 1000 * 1000 * 1000 < now_ns || + now_ns < sample->raw_ns || now_cycles < sample->base_cycles) { + // record this sample, and forget any previously known slope. + last_sample.raw_ns.store(now_ns, std::memory_order_relaxed); + last_sample.base_ns.store(estimated_base_ns, std::memory_order_relaxed); + last_sample.base_cycles.store(now_cycles, std::memory_order_relaxed); + last_sample.nsscaled_per_cycle.store(0, std::memory_order_relaxed); + last_sample.min_cycles_per_sample.store(0, std::memory_order_relaxed); + stats_initializations++; + } else if (sample->raw_ns + 500 * 1000 * 1000 < now_ns && + sample->base_cycles + 50 < now_cycles) { + // Enough time has passed to compute the cycle time. + if (sample->nsscaled_per_cycle != 0) { // Have a cycle time estimate. + // Compute time from counter reading, but avoiding overflow + // delta_cycles may be larger than on the fast path. + uint64_t estimated_scaled_ns; + int s = -1; + do { + s++; + estimated_scaled_ns = (delta_cycles >> s) * sample->nsscaled_per_cycle; + } while (estimated_scaled_ns / sample->nsscaled_per_cycle != + (delta_cycles >> s)); + estimated_base_ns = sample->base_ns + + (estimated_scaled_ns >> (kScale - s)); + } + + // Compute the assumed cycle time kMinNSBetweenSamples ns into the future + // assuming the cycle counter rate stays the same as the last interval. + uint64_t ns = now_ns - sample->raw_ns; + uint64_t measured_nsscaled_per_cycle = SafeDivideAndScale(ns, delta_cycles); + + uint64_t assumed_next_sample_delta_cycles = + SafeDivideAndScale(kMinNSBetweenSamples, measured_nsscaled_per_cycle); + + int64_t diff_ns = now_ns - estimated_base_ns; // estimate low by this much + + // We want to set nsscaled_per_cycle so that our estimate of the ns time + // at the assumed cycle time is the assumed ns time. + // That is, we want to set nsscaled_per_cycle so: + // kMinNSBetweenSamples + diff_ns == + // (assumed_next_sample_delta_cycles * nsscaled_per_cycle) >> kScale + // But we wish to damp oscillations, so instead correct only most + // of our current error, by solving: + // kMinNSBetweenSamples + diff_ns - (diff_ns / 16) == + // (assumed_next_sample_delta_cycles * nsscaled_per_cycle) >> kScale + ns = kMinNSBetweenSamples + diff_ns - (diff_ns / 16); + uint64_t new_nsscaled_per_cycle = + SafeDivideAndScale(ns, assumed_next_sample_delta_cycles); + if (new_nsscaled_per_cycle != 0 && + diff_ns < 100 * 1000 * 1000 && -diff_ns < 100 * 1000 * 1000) { + // record the cycle time measurement + last_sample.nsscaled_per_cycle.store( + new_nsscaled_per_cycle, std::memory_order_relaxed); + uint64_t new_min_cycles_per_sample = + SafeDivideAndScale(kMinNSBetweenSamples, new_nsscaled_per_cycle); + last_sample.min_cycles_per_sample.store( + new_min_cycles_per_sample, std::memory_order_relaxed); + stats_calibrations++; + } else { // something went wrong; forget the slope + last_sample.nsscaled_per_cycle.store(0, std::memory_order_relaxed); + last_sample.min_cycles_per_sample.store(0, std::memory_order_relaxed); + estimated_base_ns = now_ns; + stats_reinitializations++; + } + last_sample.raw_ns.store(now_ns, std::memory_order_relaxed); + last_sample.base_ns.store(estimated_base_ns, std::memory_order_relaxed); + last_sample.base_cycles.store(now_cycles, std::memory_order_relaxed); + } else { + // have a sample, but no slope; waiting for enough time for a calibration + stats_slow_paths++; + } + + SeqRelease(&seq, lock_value); // release the readers + + return estimated_base_ns; +} +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_USE_CYCLECLOCK_FOR_GET_CURRENT_TIME_NANOS + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +// Returns the maximum duration that SleepOnce() can sleep for. +constexpr absl::Duration MaxSleep() { +#ifdef _WIN32 + // Windows Sleep() takes unsigned long argument in milliseconds. + return absl::Milliseconds( + std::numeric_limits<unsigned long>::max()); // NOLINT(runtime/int) +#else + return absl::Seconds(std::numeric_limits<time_t>::max()); +#endif +} + +// Sleeps for the given duration. +// REQUIRES: to_sleep <= MaxSleep(). +void SleepOnce(absl::Duration to_sleep) { +#ifdef _WIN32 + Sleep(to_sleep / absl::Milliseconds(1)); +#else + struct timespec sleep_time = absl::ToTimespec(to_sleep); + while (nanosleep(&sleep_time, &sleep_time) != 0 && errno == EINTR) { + // Ignore signals and wait for the full interval to elapse. + } +#endif +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl + +extern "C" { + +ABSL_ATTRIBUTE_WEAK void AbslInternalSleepFor(absl::Duration duration) { + while (duration > absl::ZeroDuration()) { + absl::Duration to_sleep = std::min(duration, absl::MaxSleep()); + absl::SleepOnce(to_sleep); + duration -= to_sleep; + } +} + +} // extern "C" diff --git a/third_party/abseil_cpp/absl/time/clock.h b/third_party/abseil_cpp/absl/time/clock.h new file mode 100644 index 000000000000..27764a922d5e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/clock.h @@ -0,0 +1,74 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: clock.h +// ----------------------------------------------------------------------------- +// +// This header file contains utility functions for working with the system-wide +// realtime clock. For descriptions of the main time abstractions used within +// this header file, consult the time.h header file. +#ifndef ABSL_TIME_CLOCK_H_ +#define ABSL_TIME_CLOCK_H_ + +#include "absl/base/macros.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Now() +// +// Returns the current time, expressed as an `absl::Time` absolute time value. +absl::Time Now(); + +// GetCurrentTimeNanos() +// +// Returns the current time, expressed as a count of nanoseconds since the Unix +// Epoch (https://en.wikipedia.org/wiki/Unix_time). Prefer `absl::Now()` instead +// for all but the most performance-sensitive cases (i.e. when you are calling +// this function hundreds of thousands of times per second). +int64_t GetCurrentTimeNanos(); + +// SleepFor() +// +// Sleeps for the specified duration, expressed as an `absl::Duration`. +// +// Notes: +// * Signal interruptions will not reduce the sleep duration. +// * Returns immediately when passed a nonpositive duration. +void SleepFor(absl::Duration duration); + +ABSL_NAMESPACE_END +} // namespace absl + +// ----------------------------------------------------------------------------- +// Implementation Details +// ----------------------------------------------------------------------------- + +// In some build configurations we pass --detect-odr-violations to the +// gold linker. This causes it to flag weak symbol overrides as ODR +// violations. Because ODR only applies to C++ and not C, +// --detect-odr-violations ignores symbols not mangled with C++ names. +// By changing our extension points to be extern "C", we dodge this +// check. +extern "C" { +void AbslInternalSleepFor(absl::Duration duration); +} // extern "C" + +inline void absl::SleepFor(absl::Duration duration) { + AbslInternalSleepFor(duration); +} + +#endif // ABSL_TIME_CLOCK_H_ diff --git a/third_party/abseil_cpp/absl/time/clock_benchmark.cc b/third_party/abseil_cpp/absl/time/clock_benchmark.cc new file mode 100644 index 000000000000..c5c795ecbd23 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/clock_benchmark.cc @@ -0,0 +1,74 @@ +// Copyright 2018 The Abseil Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/clock.h" + +#if !defined(_WIN32) +#include <sys/time.h> +#else +#include <winsock2.h> +#endif // _WIN32 +#include <cstdio> + +#include "absl/base/internal/cycleclock.h" +#include "benchmark/benchmark.h" + +namespace { + +void BM_Clock_Now_AbslTime(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Now()); + } +} +BENCHMARK(BM_Clock_Now_AbslTime); + +void BM_Clock_Now_GetCurrentTimeNanos(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::GetCurrentTimeNanos()); + } +} +BENCHMARK(BM_Clock_Now_GetCurrentTimeNanos); + +void BM_Clock_Now_AbslTime_ToUnixNanos(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToUnixNanos(absl::Now())); + } +} +BENCHMARK(BM_Clock_Now_AbslTime_ToUnixNanos); + +void BM_Clock_Now_CycleClock(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::base_internal::CycleClock::Now()); + } +} +BENCHMARK(BM_Clock_Now_CycleClock); + +#if !defined(_WIN32) +static void BM_Clock_Now_gettimeofday(benchmark::State& state) { + struct timeval tv; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(gettimeofday(&tv, nullptr)); + } +} +BENCHMARK(BM_Clock_Now_gettimeofday); + +static void BM_Clock_Now_clock_gettime(benchmark::State& state) { + struct timespec ts; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(clock_gettime(CLOCK_REALTIME, &ts)); + } +} +BENCHMARK(BM_Clock_Now_clock_gettime); +#endif // _WIN32 + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/clock_test.cc b/third_party/abseil_cpp/absl/time/clock_test.cc new file mode 100644 index 000000000000..4bcfc6bc7272 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/clock_test.cc @@ -0,0 +1,118 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/clock.h" + +#include "absl/base/config.h" +#if defined(ABSL_HAVE_ALARM) +#include <signal.h> +#include <unistd.h> +#elif defined(__linux__) || defined(__APPLE__) +#error all known Linux and Apple targets have alarm +#endif + +#include "gtest/gtest.h" +#include "absl/time/time.h" + +namespace { + +TEST(Time, Now) { + const absl::Time before = absl::FromUnixNanos(absl::GetCurrentTimeNanos()); + const absl::Time now = absl::Now(); + const absl::Time after = absl::FromUnixNanos(absl::GetCurrentTimeNanos()); + EXPECT_GE(now, before); + EXPECT_GE(after, now); +} + +enum class AlarmPolicy { kWithoutAlarm, kWithAlarm }; + +#if defined(ABSL_HAVE_ALARM) +bool alarm_handler_invoked = false; + +void AlarmHandler(int signo) { + ASSERT_EQ(signo, SIGALRM); + alarm_handler_invoked = true; +} +#endif + +// Does SleepFor(d) take between lower_bound and upper_bound at least +// once between now and (now + timeout)? If requested (and supported), +// add an alarm for the middle of the sleep period and expect it to fire. +bool SleepForBounded(absl::Duration d, absl::Duration lower_bound, + absl::Duration upper_bound, absl::Duration timeout, + AlarmPolicy alarm_policy, int* attempts) { + const absl::Time deadline = absl::Now() + timeout; + while (absl::Now() < deadline) { +#if defined(ABSL_HAVE_ALARM) + sig_t old_alarm = SIG_DFL; + if (alarm_policy == AlarmPolicy::kWithAlarm) { + alarm_handler_invoked = false; + old_alarm = signal(SIGALRM, AlarmHandler); + alarm(absl::ToInt64Seconds(d / 2)); + } +#else + EXPECT_EQ(alarm_policy, AlarmPolicy::kWithoutAlarm); +#endif + ++*attempts; + absl::Time start = absl::Now(); + absl::SleepFor(d); + absl::Duration actual = absl::Now() - start; +#if defined(ABSL_HAVE_ALARM) + if (alarm_policy == AlarmPolicy::kWithAlarm) { + signal(SIGALRM, old_alarm); + if (!alarm_handler_invoked) continue; + } +#endif + if (lower_bound <= actual && actual <= upper_bound) { + return true; // yes, the SleepFor() was correctly bounded + } + } + return false; +} + +testing::AssertionResult AssertSleepForBounded(absl::Duration d, + absl::Duration early, + absl::Duration late, + absl::Duration timeout, + AlarmPolicy alarm_policy) { + const absl::Duration lower_bound = d - early; + const absl::Duration upper_bound = d + late; + int attempts = 0; + if (SleepForBounded(d, lower_bound, upper_bound, timeout, alarm_policy, + &attempts)) { + return testing::AssertionSuccess(); + } + return testing::AssertionFailure() + << "SleepFor(" << d << ") did not return within [" << lower_bound + << ":" << upper_bound << "] in " << attempts << " attempt" + << (attempts == 1 ? "" : "s") << " over " << timeout + << (alarm_policy == AlarmPolicy::kWithAlarm ? " with" : " without") + << " an alarm"; +} + +// Tests that SleepFor() returns neither too early nor too late. +TEST(SleepFor, Bounded) { + const absl::Duration d = absl::Milliseconds(2500); + const absl::Duration early = absl::Milliseconds(100); + const absl::Duration late = absl::Milliseconds(300); + const absl::Duration timeout = 48 * d; + EXPECT_TRUE(AssertSleepForBounded(d, early, late, timeout, + AlarmPolicy::kWithoutAlarm)); +#if defined(ABSL_HAVE_ALARM) + EXPECT_TRUE(AssertSleepForBounded(d, early, late, timeout, + AlarmPolicy::kWithAlarm)); +#endif +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/duration.cc b/third_party/abseil_cpp/absl/time/duration.cc new file mode 100644 index 000000000000..4443109a5121 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/duration.cc @@ -0,0 +1,954 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The implementation of the absl::Duration class, which is declared in +// //absl/time.h. This class behaves like a numeric type; it has no public +// methods and is used only through the operators defined here. +// +// Implementation notes: +// +// An absl::Duration is represented as +// +// rep_hi_ : (int64_t) Whole seconds +// rep_lo_ : (uint32_t) Fractions of a second +// +// The seconds value (rep_hi_) may be positive or negative as appropriate. +// The fractional seconds (rep_lo_) is always a positive offset from rep_hi_. +// The API for Duration guarantees at least nanosecond resolution, which +// means rep_lo_ could have a max value of 1B - 1 if it stored nanoseconds. +// However, to utilize more of the available 32 bits of space in rep_lo_, +// we instead store quarters of a nanosecond in rep_lo_ resulting in a max +// value of 4B - 1. This allows us to correctly handle calculations like +// 0.5 nanos + 0.5 nanos = 1 nano. The following example shows the actual +// Duration rep using quarters of a nanosecond. +// +// 2.5 sec = {rep_hi_=2, rep_lo_=2000000000} // lo = 4 * 500000000 +// -2.5 sec = {rep_hi_=-3, rep_lo_=2000000000} +// +// Infinite durations are represented as Durations with the rep_lo_ field set +// to all 1s. +// +// +InfiniteDuration: +// rep_hi_ : kint64max +// rep_lo_ : ~0U +// +// -InfiniteDuration: +// rep_hi_ : kint64min +// rep_lo_ : ~0U +// +// Arithmetic overflows/underflows to +/- infinity and saturates. + +#if defined(_MSC_VER) +#include <winsock2.h> // for timeval +#endif + +#include <algorithm> +#include <cassert> +#include <cctype> +#include <cerrno> +#include <cmath> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <ctime> +#include <functional> +#include <limits> +#include <string> + +#include "absl/base/casts.h" +#include "absl/base/macros.h" +#include "absl/numeric/int128.h" +#include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +using time_internal::kTicksPerNanosecond; +using time_internal::kTicksPerSecond; + +constexpr int64_t kint64max = std::numeric_limits<int64_t>::max(); +constexpr int64_t kint64min = std::numeric_limits<int64_t>::min(); + +// Can't use std::isinfinite() because it doesn't exist on windows. +inline bool IsFinite(double d) { + if (std::isnan(d)) return false; + return d != std::numeric_limits<double>::infinity() && + d != -std::numeric_limits<double>::infinity(); +} + +inline bool IsValidDivisor(double d) { + if (std::isnan(d)) return false; + return d != 0.0; +} + +// Can't use std::round() because it is only available in C++11. +// Note that we ignore the possibility of floating-point over/underflow. +template <typename Double> +inline double Round(Double d) { + return d < 0 ? std::ceil(d - 0.5) : std::floor(d + 0.5); +} + +// *sec may be positive or negative. *ticks must be in the range +// -kTicksPerSecond < *ticks < kTicksPerSecond. If *ticks is negative it +// will be normalized to a positive value by adjusting *sec accordingly. +inline void NormalizeTicks(int64_t* sec, int64_t* ticks) { + if (*ticks < 0) { + --*sec; + *ticks += kTicksPerSecond; + } +} + +// Makes a uint128 from the absolute value of the given scalar. +inline uint128 MakeU128(int64_t a) { + uint128 u128 = 0; + if (a < 0) { + ++u128; + ++a; // Makes it safe to negate 'a' + a = -a; + } + u128 += static_cast<uint64_t>(a); + return u128; +} + +// Makes a uint128 count of ticks out of the absolute value of the Duration. +inline uint128 MakeU128Ticks(Duration d) { + int64_t rep_hi = time_internal::GetRepHi(d); + uint32_t rep_lo = time_internal::GetRepLo(d); + if (rep_hi < 0) { + ++rep_hi; + rep_hi = -rep_hi; + rep_lo = kTicksPerSecond - rep_lo; + } + uint128 u128 = static_cast<uint64_t>(rep_hi); + u128 *= static_cast<uint64_t>(kTicksPerSecond); + u128 += rep_lo; + return u128; +} + +// Breaks a uint128 of ticks into a Duration. +inline Duration MakeDurationFromU128(uint128 u128, bool is_neg) { + int64_t rep_hi; + uint32_t rep_lo; + const uint64_t h64 = Uint128High64(u128); + const uint64_t l64 = Uint128Low64(u128); + if (h64 == 0) { // fastpath + const uint64_t hi = l64 / kTicksPerSecond; + rep_hi = static_cast<int64_t>(hi); + rep_lo = static_cast<uint32_t>(l64 - hi * kTicksPerSecond); + } else { + // kMaxRepHi64 is the high 64 bits of (2^63 * kTicksPerSecond). + // Any positive tick count whose high 64 bits are >= kMaxRepHi64 + // is not representable as a Duration. A negative tick count can + // have its high 64 bits == kMaxRepHi64 but only when the low 64 + // bits are all zero, otherwise it is not representable either. + const uint64_t kMaxRepHi64 = 0x77359400UL; + if (h64 >= kMaxRepHi64) { + if (is_neg && h64 == kMaxRepHi64 && l64 == 0) { + // Avoid trying to represent -kint64min below. + return time_internal::MakeDuration(kint64min); + } + return is_neg ? -InfiniteDuration() : InfiniteDuration(); + } + const uint128 kTicksPerSecond128 = static_cast<uint64_t>(kTicksPerSecond); + const uint128 hi = u128 / kTicksPerSecond128; + rep_hi = static_cast<int64_t>(Uint128Low64(hi)); + rep_lo = + static_cast<uint32_t>(Uint128Low64(u128 - hi * kTicksPerSecond128)); + } + if (is_neg) { + rep_hi = -rep_hi; + if (rep_lo != 0) { + --rep_hi; + rep_lo = kTicksPerSecond - rep_lo; + } + } + return time_internal::MakeDuration(rep_hi, rep_lo); +} + +// Convert between int64_t and uint64_t, preserving representation. This +// allows us to do arithmetic in the unsigned domain, where overflow has +// well-defined behavior. See operator+=() and operator-=(). +// +// C99 7.20.1.1.1, as referenced by C++11 18.4.1.2, says, "The typedef +// name intN_t designates a signed integer type with width N, no padding +// bits, and a two's complement representation." So, we can convert to +// and from the corresponding uint64_t value using a bit cast. +inline uint64_t EncodeTwosComp(int64_t v) { + return absl::bit_cast<uint64_t>(v); +} +inline int64_t DecodeTwosComp(uint64_t v) { return absl::bit_cast<int64_t>(v); } + +// Note: The overflow detection in this function is done using greater/less *or +// equal* because kint64max/min is too large to be represented exactly in a +// double (which only has 53 bits of precision). In order to avoid assigning to +// rep->hi a double value that is too large for an int64_t (and therefore is +// undefined), we must consider computations that equal kint64max/min as a +// double as overflow cases. +inline bool SafeAddRepHi(double a_hi, double b_hi, Duration* d) { + double c = a_hi + b_hi; + if (c >= static_cast<double>(kint64max)) { + *d = InfiniteDuration(); + return false; + } + if (c <= static_cast<double>(kint64min)) { + *d = -InfiniteDuration(); + return false; + } + *d = time_internal::MakeDuration(c, time_internal::GetRepLo(*d)); + return true; +} + +// A functor that's similar to std::multiplies<T>, except this returns the max +// T value instead of overflowing. This is only defined for uint128. +template <typename Ignored> +struct SafeMultiply { + uint128 operator()(uint128 a, uint128 b) const { + // b hi is always zero because it originated as an int64_t. + assert(Uint128High64(b) == 0); + // Fastpath to avoid the expensive overflow check with division. + if (Uint128High64(a) == 0) { + return (((Uint128Low64(a) | Uint128Low64(b)) >> 32) == 0) + ? static_cast<uint128>(Uint128Low64(a) * Uint128Low64(b)) + : a * b; + } + return b == 0 ? b : (a > kuint128max / b) ? kuint128max : a * b; + } +}; + +// Scales (i.e., multiplies or divides, depending on the Operation template) +// the Duration d by the int64_t r. +template <template <typename> class Operation> +inline Duration ScaleFixed(Duration d, int64_t r) { + const uint128 a = MakeU128Ticks(d); + const uint128 b = MakeU128(r); + const uint128 q = Operation<uint128>()(a, b); + const bool is_neg = (time_internal::GetRepHi(d) < 0) != (r < 0); + return MakeDurationFromU128(q, is_neg); +} + +// Scales (i.e., multiplies or divides, depending on the Operation template) +// the Duration d by the double r. +template <template <typename> class Operation> +inline Duration ScaleDouble(Duration d, double r) { + Operation<double> op; + double hi_doub = op(time_internal::GetRepHi(d), r); + double lo_doub = op(time_internal::GetRepLo(d), r); + + double hi_int = 0; + double hi_frac = std::modf(hi_doub, &hi_int); + + // Moves hi's fractional bits to lo. + lo_doub /= kTicksPerSecond; + lo_doub += hi_frac; + + double lo_int = 0; + double lo_frac = std::modf(lo_doub, &lo_int); + + // Rolls lo into hi if necessary. + int64_t lo64 = Round(lo_frac * kTicksPerSecond); + + Duration ans; + if (!SafeAddRepHi(hi_int, lo_int, &ans)) return ans; + int64_t hi64 = time_internal::GetRepHi(ans); + if (!SafeAddRepHi(hi64, lo64 / kTicksPerSecond, &ans)) return ans; + hi64 = time_internal::GetRepHi(ans); + lo64 %= kTicksPerSecond; + NormalizeTicks(&hi64, &lo64); + return time_internal::MakeDuration(hi64, lo64); +} + +// Tries to divide num by den as fast as possible by looking for common, easy +// cases. If the division was done, the quotient is in *q and the remainder is +// in *rem and true will be returned. +inline bool IDivFastPath(const Duration num, const Duration den, int64_t* q, + Duration* rem) { + // Bail if num or den is an infinity. + if (time_internal::IsInfiniteDuration(num) || + time_internal::IsInfiniteDuration(den)) + return false; + + int64_t num_hi = time_internal::GetRepHi(num); + uint32_t num_lo = time_internal::GetRepLo(num); + int64_t den_hi = time_internal::GetRepHi(den); + uint32_t den_lo = time_internal::GetRepLo(den); + + if (den_hi == 0 && den_lo == kTicksPerNanosecond) { + // Dividing by 1ns + if (num_hi >= 0 && num_hi < (kint64max - kTicksPerSecond) / 1000000000) { + *q = num_hi * 1000000000 + num_lo / kTicksPerNanosecond; + *rem = time_internal::MakeDuration(0, num_lo % den_lo); + return true; + } + } else if (den_hi == 0 && den_lo == 100 * kTicksPerNanosecond) { + // Dividing by 100ns (common when converting to Universal time) + if (num_hi >= 0 && num_hi < (kint64max - kTicksPerSecond) / 10000000) { + *q = num_hi * 10000000 + num_lo / (100 * kTicksPerNanosecond); + *rem = time_internal::MakeDuration(0, num_lo % den_lo); + return true; + } + } else if (den_hi == 0 && den_lo == 1000 * kTicksPerNanosecond) { + // Dividing by 1us + if (num_hi >= 0 && num_hi < (kint64max - kTicksPerSecond) / 1000000) { + *q = num_hi * 1000000 + num_lo / (1000 * kTicksPerNanosecond); + *rem = time_internal::MakeDuration(0, num_lo % den_lo); + return true; + } + } else if (den_hi == 0 && den_lo == 1000000 * kTicksPerNanosecond) { + // Dividing by 1ms + if (num_hi >= 0 && num_hi < (kint64max - kTicksPerSecond) / 1000) { + *q = num_hi * 1000 + num_lo / (1000000 * kTicksPerNanosecond); + *rem = time_internal::MakeDuration(0, num_lo % den_lo); + return true; + } + } else if (den_hi > 0 && den_lo == 0) { + // Dividing by positive multiple of 1s + if (num_hi >= 0) { + if (den_hi == 1) { + *q = num_hi; + *rem = time_internal::MakeDuration(0, num_lo); + return true; + } + *q = num_hi / den_hi; + *rem = time_internal::MakeDuration(num_hi % den_hi, num_lo); + return true; + } + if (num_lo != 0) { + num_hi += 1; + } + int64_t quotient = num_hi / den_hi; + int64_t rem_sec = num_hi % den_hi; + if (rem_sec > 0) { + rem_sec -= den_hi; + quotient += 1; + } + if (num_lo != 0) { + rem_sec -= 1; + } + *q = quotient; + *rem = time_internal::MakeDuration(rem_sec, num_lo); + return true; + } + + return false; +} + +} // namespace + +namespace time_internal { + +// The 'satq' argument indicates whether the quotient should saturate at the +// bounds of int64_t. If it does saturate, the difference will spill over to +// the remainder. If it does not saturate, the remainder remain accurate, +// but the returned quotient will over/underflow int64_t and should not be used. +int64_t IDivDuration(bool satq, const Duration num, const Duration den, + Duration* rem) { + int64_t q = 0; + if (IDivFastPath(num, den, &q, rem)) { + return q; + } + + const bool num_neg = num < ZeroDuration(); + const bool den_neg = den < ZeroDuration(); + const bool quotient_neg = num_neg != den_neg; + + if (time_internal::IsInfiniteDuration(num) || den == ZeroDuration()) { + *rem = num_neg ? -InfiniteDuration() : InfiniteDuration(); + return quotient_neg ? kint64min : kint64max; + } + if (time_internal::IsInfiniteDuration(den)) { + *rem = num; + return 0; + } + + const uint128 a = MakeU128Ticks(num); + const uint128 b = MakeU128Ticks(den); + uint128 quotient128 = a / b; + + if (satq) { + // Limits the quotient to the range of int64_t. + if (quotient128 > uint128(static_cast<uint64_t>(kint64max))) { + quotient128 = quotient_neg ? uint128(static_cast<uint64_t>(kint64min)) + : uint128(static_cast<uint64_t>(kint64max)); + } + } + + const uint128 remainder128 = a - quotient128 * b; + *rem = MakeDurationFromU128(remainder128, num_neg); + + if (!quotient_neg || quotient128 == 0) { + return Uint128Low64(quotient128) & kint64max; + } + // The quotient needs to be negated, but we need to carefully handle + // quotient128s with the top bit on. + return -static_cast<int64_t>(Uint128Low64(quotient128 - 1) & kint64max) - 1; +} + +} // namespace time_internal + +// +// Additive operators. +// + +Duration& Duration::operator+=(Duration rhs) { + if (time_internal::IsInfiniteDuration(*this)) return *this; + if (time_internal::IsInfiniteDuration(rhs)) return *this = rhs; + const int64_t orig_rep_hi = rep_hi_; + rep_hi_ = + DecodeTwosComp(EncodeTwosComp(rep_hi_) + EncodeTwosComp(rhs.rep_hi_)); + if (rep_lo_ >= kTicksPerSecond - rhs.rep_lo_) { + rep_hi_ = DecodeTwosComp(EncodeTwosComp(rep_hi_) + 1); + rep_lo_ -= kTicksPerSecond; + } + rep_lo_ += rhs.rep_lo_; + if (rhs.rep_hi_ < 0 ? rep_hi_ > orig_rep_hi : rep_hi_ < orig_rep_hi) { + return *this = rhs.rep_hi_ < 0 ? -InfiniteDuration() : InfiniteDuration(); + } + return *this; +} + +Duration& Duration::operator-=(Duration rhs) { + if (time_internal::IsInfiniteDuration(*this)) return *this; + if (time_internal::IsInfiniteDuration(rhs)) { + return *this = rhs.rep_hi_ >= 0 ? -InfiniteDuration() : InfiniteDuration(); + } + const int64_t orig_rep_hi = rep_hi_; + rep_hi_ = + DecodeTwosComp(EncodeTwosComp(rep_hi_) - EncodeTwosComp(rhs.rep_hi_)); + if (rep_lo_ < rhs.rep_lo_) { + rep_hi_ = DecodeTwosComp(EncodeTwosComp(rep_hi_) - 1); + rep_lo_ += kTicksPerSecond; + } + rep_lo_ -= rhs.rep_lo_; + if (rhs.rep_hi_ < 0 ? rep_hi_ < orig_rep_hi : rep_hi_ > orig_rep_hi) { + return *this = rhs.rep_hi_ >= 0 ? -InfiniteDuration() : InfiniteDuration(); + } + return *this; +} + +// +// Multiplicative operators. +// + +Duration& Duration::operator*=(int64_t r) { + if (time_internal::IsInfiniteDuration(*this)) { + const bool is_neg = (r < 0) != (rep_hi_ < 0); + return *this = is_neg ? -InfiniteDuration() : InfiniteDuration(); + } + return *this = ScaleFixed<SafeMultiply>(*this, r); +} + +Duration& Duration::operator*=(double r) { + if (time_internal::IsInfiniteDuration(*this) || !IsFinite(r)) { + const bool is_neg = (std::signbit(r) != 0) != (rep_hi_ < 0); + return *this = is_neg ? -InfiniteDuration() : InfiniteDuration(); + } + return *this = ScaleDouble<std::multiplies>(*this, r); +} + +Duration& Duration::operator/=(int64_t r) { + if (time_internal::IsInfiniteDuration(*this) || r == 0) { + const bool is_neg = (r < 0) != (rep_hi_ < 0); + return *this = is_neg ? -InfiniteDuration() : InfiniteDuration(); + } + return *this = ScaleFixed<std::divides>(*this, r); +} + +Duration& Duration::operator/=(double r) { + if (time_internal::IsInfiniteDuration(*this) || !IsValidDivisor(r)) { + const bool is_neg = (std::signbit(r) != 0) != (rep_hi_ < 0); + return *this = is_neg ? -InfiniteDuration() : InfiniteDuration(); + } + return *this = ScaleDouble<std::divides>(*this, r); +} + +Duration& Duration::operator%=(Duration rhs) { + time_internal::IDivDuration(false, *this, rhs, this); + return *this; +} + +double FDivDuration(Duration num, Duration den) { + // Arithmetic with infinity is sticky. + if (time_internal::IsInfiniteDuration(num) || den == ZeroDuration()) { + return (num < ZeroDuration()) == (den < ZeroDuration()) + ? std::numeric_limits<double>::infinity() + : -std::numeric_limits<double>::infinity(); + } + if (time_internal::IsInfiniteDuration(den)) return 0.0; + + double a = + static_cast<double>(time_internal::GetRepHi(num)) * kTicksPerSecond + + time_internal::GetRepLo(num); + double b = + static_cast<double>(time_internal::GetRepHi(den)) * kTicksPerSecond + + time_internal::GetRepLo(den); + return a / b; +} + +// +// Trunc/Floor/Ceil. +// + +Duration Trunc(Duration d, Duration unit) { + return d - (d % unit); +} + +Duration Floor(const Duration d, const Duration unit) { + const absl::Duration td = Trunc(d, unit); + return td <= d ? td : td - AbsDuration(unit); +} + +Duration Ceil(const Duration d, const Duration unit) { + const absl::Duration td = Trunc(d, unit); + return td >= d ? td : td + AbsDuration(unit); +} + +// +// Factory functions. +// + +Duration DurationFromTimespec(timespec ts) { + if (static_cast<uint64_t>(ts.tv_nsec) < 1000 * 1000 * 1000) { + int64_t ticks = ts.tv_nsec * kTicksPerNanosecond; + return time_internal::MakeDuration(ts.tv_sec, ticks); + } + return Seconds(ts.tv_sec) + Nanoseconds(ts.tv_nsec); +} + +Duration DurationFromTimeval(timeval tv) { + if (static_cast<uint64_t>(tv.tv_usec) < 1000 * 1000) { + int64_t ticks = tv.tv_usec * 1000 * kTicksPerNanosecond; + return time_internal::MakeDuration(tv.tv_sec, ticks); + } + return Seconds(tv.tv_sec) + Microseconds(tv.tv_usec); +} + +// +// Conversion to other duration types. +// + +int64_t ToInt64Nanoseconds(Duration d) { + if (time_internal::GetRepHi(d) >= 0 && + time_internal::GetRepHi(d) >> 33 == 0) { + return (time_internal::GetRepHi(d) * 1000 * 1000 * 1000) + + (time_internal::GetRepLo(d) / kTicksPerNanosecond); + } + return d / Nanoseconds(1); +} +int64_t ToInt64Microseconds(Duration d) { + if (time_internal::GetRepHi(d) >= 0 && + time_internal::GetRepHi(d) >> 43 == 0) { + return (time_internal::GetRepHi(d) * 1000 * 1000) + + (time_internal::GetRepLo(d) / (kTicksPerNanosecond * 1000)); + } + return d / Microseconds(1); +} +int64_t ToInt64Milliseconds(Duration d) { + if (time_internal::GetRepHi(d) >= 0 && + time_internal::GetRepHi(d) >> 53 == 0) { + return (time_internal::GetRepHi(d) * 1000) + + (time_internal::GetRepLo(d) / (kTicksPerNanosecond * 1000 * 1000)); + } + return d / Milliseconds(1); +} +int64_t ToInt64Seconds(Duration d) { + int64_t hi = time_internal::GetRepHi(d); + if (time_internal::IsInfiniteDuration(d)) return hi; + if (hi < 0 && time_internal::GetRepLo(d) != 0) ++hi; + return hi; +} +int64_t ToInt64Minutes(Duration d) { + int64_t hi = time_internal::GetRepHi(d); + if (time_internal::IsInfiniteDuration(d)) return hi; + if (hi < 0 && time_internal::GetRepLo(d) != 0) ++hi; + return hi / 60; +} +int64_t ToInt64Hours(Duration d) { + int64_t hi = time_internal::GetRepHi(d); + if (time_internal::IsInfiniteDuration(d)) return hi; + if (hi < 0 && time_internal::GetRepLo(d) != 0) ++hi; + return hi / (60 * 60); +} + +double ToDoubleNanoseconds(Duration d) { + return FDivDuration(d, Nanoseconds(1)); +} +double ToDoubleMicroseconds(Duration d) { + return FDivDuration(d, Microseconds(1)); +} +double ToDoubleMilliseconds(Duration d) { + return FDivDuration(d, Milliseconds(1)); +} +double ToDoubleSeconds(Duration d) { + return FDivDuration(d, Seconds(1)); +} +double ToDoubleMinutes(Duration d) { + return FDivDuration(d, Minutes(1)); +} +double ToDoubleHours(Duration d) { + return FDivDuration(d, Hours(1)); +} + +timespec ToTimespec(Duration d) { + timespec ts; + if (!time_internal::IsInfiniteDuration(d)) { + int64_t rep_hi = time_internal::GetRepHi(d); + uint32_t rep_lo = time_internal::GetRepLo(d); + if (rep_hi < 0) { + // Tweak the fields so that unsigned division of rep_lo + // maps to truncation (towards zero) for the timespec. + rep_lo += kTicksPerNanosecond - 1; + if (rep_lo >= kTicksPerSecond) { + rep_hi += 1; + rep_lo -= kTicksPerSecond; + } + } + ts.tv_sec = rep_hi; + if (ts.tv_sec == rep_hi) { // no time_t narrowing + ts.tv_nsec = rep_lo / kTicksPerNanosecond; + return ts; + } + } + if (d >= ZeroDuration()) { + ts.tv_sec = std::numeric_limits<time_t>::max(); + ts.tv_nsec = 1000 * 1000 * 1000 - 1; + } else { + ts.tv_sec = std::numeric_limits<time_t>::min(); + ts.tv_nsec = 0; + } + return ts; +} + +timeval ToTimeval(Duration d) { + timeval tv; + timespec ts = ToTimespec(d); + if (ts.tv_sec < 0) { + // Tweak the fields so that positive division of tv_nsec + // maps to truncation (towards zero) for the timeval. + ts.tv_nsec += 1000 - 1; + if (ts.tv_nsec >= 1000 * 1000 * 1000) { + ts.tv_sec += 1; + ts.tv_nsec -= 1000 * 1000 * 1000; + } + } + tv.tv_sec = ts.tv_sec; + if (tv.tv_sec != ts.tv_sec) { // narrowing + if (ts.tv_sec < 0) { + tv.tv_sec = std::numeric_limits<decltype(tv.tv_sec)>::min(); + tv.tv_usec = 0; + } else { + tv.tv_sec = std::numeric_limits<decltype(tv.tv_sec)>::max(); + tv.tv_usec = 1000 * 1000 - 1; + } + return tv; + } + tv.tv_usec = static_cast<int>(ts.tv_nsec / 1000); // suseconds_t + return tv; +} + +std::chrono::nanoseconds ToChronoNanoseconds(Duration d) { + return time_internal::ToChronoDuration<std::chrono::nanoseconds>(d); +} +std::chrono::microseconds ToChronoMicroseconds(Duration d) { + return time_internal::ToChronoDuration<std::chrono::microseconds>(d); +} +std::chrono::milliseconds ToChronoMilliseconds(Duration d) { + return time_internal::ToChronoDuration<std::chrono::milliseconds>(d); +} +std::chrono::seconds ToChronoSeconds(Duration d) { + return time_internal::ToChronoDuration<std::chrono::seconds>(d); +} +std::chrono::minutes ToChronoMinutes(Duration d) { + return time_internal::ToChronoDuration<std::chrono::minutes>(d); +} +std::chrono::hours ToChronoHours(Duration d) { + return time_internal::ToChronoDuration<std::chrono::hours>(d); +} + +// +// To/From string formatting. +// + +namespace { + +// Formats a positive 64-bit integer in the given field width. Note that +// it is up to the caller of Format64() to ensure that there is sufficient +// space before ep to hold the conversion. +char* Format64(char* ep, int width, int64_t v) { + do { + --width; + *--ep = '0' + (v % 10); // contiguous digits + } while (v /= 10); + while (--width >= 0) *--ep = '0'; // zero pad + return ep; +} + +// Helpers for FormatDuration() that format 'n' and append it to 'out' +// followed by the given 'unit'. If 'n' formats to "0", nothing is +// appended (not even the unit). + +// A type that encapsulates how to display a value of a particular unit. For +// values that are displayed with fractional parts, the precision indicates +// where to round the value. The precision varies with the display unit because +// a Duration can hold only quarters of a nanosecond, so displaying information +// beyond that is just noise. +// +// For example, a microsecond value of 42.00025xxxxx should not display beyond 5 +// fractional digits, because it is in the noise of what a Duration can +// represent. +struct DisplayUnit { + absl::string_view abbr; + int prec; + double pow10; +}; +ABSL_CONST_INIT const DisplayUnit kDisplayNano = {"ns", 2, 1e2}; +ABSL_CONST_INIT const DisplayUnit kDisplayMicro = {"us", 5, 1e5}; +ABSL_CONST_INIT const DisplayUnit kDisplayMilli = {"ms", 8, 1e8}; +ABSL_CONST_INIT const DisplayUnit kDisplaySec = {"s", 11, 1e11}; +ABSL_CONST_INIT const DisplayUnit kDisplayMin = {"m", -1, 0.0}; // prec ignored +ABSL_CONST_INIT const DisplayUnit kDisplayHour = {"h", -1, + 0.0}; // prec ignored + +void AppendNumberUnit(std::string* out, int64_t n, DisplayUnit unit) { + char buf[sizeof("2562047788015216")]; // hours in max duration + char* const ep = buf + sizeof(buf); + char* bp = Format64(ep, 0, n); + if (*bp != '0' || bp + 1 != ep) { + out->append(bp, ep - bp); + out->append(unit.abbr.data(), unit.abbr.size()); + } +} + +// Note: unit.prec is limited to double's digits10 value (typically 15) so it +// always fits in buf[]. +void AppendNumberUnit(std::string* out, double n, DisplayUnit unit) { + constexpr int kBufferSize = std::numeric_limits<double>::digits10; + const int prec = std::min(kBufferSize, unit.prec); + char buf[kBufferSize]; // also large enough to hold integer part + char* ep = buf + sizeof(buf); + double d = 0; + int64_t frac_part = Round(std::modf(n, &d) * unit.pow10); + int64_t int_part = d; + if (int_part != 0 || frac_part != 0) { + char* bp = Format64(ep, 0, int_part); // always < 1000 + out->append(bp, ep - bp); + if (frac_part != 0) { + out->push_back('.'); + bp = Format64(ep, prec, frac_part); + while (ep[-1] == '0') --ep; + out->append(bp, ep - bp); + } + out->append(unit.abbr.data(), unit.abbr.size()); + } +} + +} // namespace + +// From Go's doc at https://golang.org/pkg/time/#Duration.String +// [FormatDuration] returns a string representing the duration in the +// form "72h3m0.5s". Leading zero units are omitted. As a special +// case, durations less than one second format use a smaller unit +// (milli-, micro-, or nanoseconds) to ensure that the leading digit +// is non-zero. +// Unlike Go, we format the zero duration as 0, with no unit. +std::string FormatDuration(Duration d) { + const Duration min_duration = Seconds(kint64min); + if (d == min_duration) { + // Avoid needing to negate kint64min by directly returning what the + // following code should produce in that case. + return "-2562047788015215h30m8s"; + } + std::string s; + if (d < ZeroDuration()) { + s.append("-"); + d = -d; + } + if (d == InfiniteDuration()) { + s.append("inf"); + } else if (d < Seconds(1)) { + // Special case for durations with a magnitude < 1 second. The duration + // is printed as a fraction of a single unit, e.g., "1.2ms". + if (d < Microseconds(1)) { + AppendNumberUnit(&s, FDivDuration(d, Nanoseconds(1)), kDisplayNano); + } else if (d < Milliseconds(1)) { + AppendNumberUnit(&s, FDivDuration(d, Microseconds(1)), kDisplayMicro); + } else { + AppendNumberUnit(&s, FDivDuration(d, Milliseconds(1)), kDisplayMilli); + } + } else { + AppendNumberUnit(&s, IDivDuration(d, Hours(1), &d), kDisplayHour); + AppendNumberUnit(&s, IDivDuration(d, Minutes(1), &d), kDisplayMin); + AppendNumberUnit(&s, FDivDuration(d, Seconds(1)), kDisplaySec); + } + if (s.empty() || s == "-") { + s = "0"; + } + return s; +} + +namespace { + +// A helper for ParseDuration() that parses a leading number from the given +// string and stores the result in *int_part/*frac_part/*frac_scale. The +// given string pointer is modified to point to the first unconsumed char. +bool ConsumeDurationNumber(const char** dpp, const char* ep, int64_t* int_part, + int64_t* frac_part, int64_t* frac_scale) { + *int_part = 0; + *frac_part = 0; + *frac_scale = 1; // invariant: *frac_part < *frac_scale + const char* start = *dpp; + for (; *dpp != ep; *dpp += 1) { + const int d = **dpp - '0'; // contiguous digits + if (d < 0 || 10 <= d) break; + + if (*int_part > kint64max / 10) return false; + *int_part *= 10; + if (*int_part > kint64max - d) return false; + *int_part += d; + } + const bool int_part_empty = (*dpp == start); + if (*dpp == ep || **dpp != '.') return !int_part_empty; + + for (*dpp += 1; *dpp != ep; *dpp += 1) { + const int d = **dpp - '0'; // contiguous digits + if (d < 0 || 10 <= d) break; + if (*frac_scale <= kint64max / 10) { + *frac_part *= 10; + *frac_part += d; + *frac_scale *= 10; + } + } + return !int_part_empty || *frac_scale != 1; +} + +// A helper for ParseDuration() that parses a leading unit designator (e.g., +// ns, us, ms, s, m, h) from the given string and stores the resulting unit +// in "*unit". The given string pointer is modified to point to the first +// unconsumed char. +bool ConsumeDurationUnit(const char** start, const char* end, Duration* unit) { + size_t size = end - *start; + switch (size) { + case 0: + return false; + default: + switch (**start) { + case 'n': + if (*(*start + 1) == 's') { + *start += 2; + *unit = Nanoseconds(1); + return true; + } + break; + case 'u': + if (*(*start + 1) == 's') { + *start += 2; + *unit = Microseconds(1); + return true; + } + break; + case 'm': + if (*(*start + 1) == 's') { + *start += 2; + *unit = Milliseconds(1); + return true; + } + break; + default: + break; + } + ABSL_FALLTHROUGH_INTENDED; + case 1: + switch (**start) { + case 's': + *unit = Seconds(1); + *start += 1; + return true; + case 'm': + *unit = Minutes(1); + *start += 1; + return true; + case 'h': + *unit = Hours(1); + *start += 1; + return true; + default: + return false; + } + } +} + +} // namespace + +// From Go's doc at https://golang.org/pkg/time/#ParseDuration +// [ParseDuration] parses a duration string. A duration string is +// a possibly signed sequence of decimal numbers, each with optional +// fraction and a unit suffix, such as "300ms", "-1.5h" or "2h45m". +// Valid time units are "ns", "us" "ms", "s", "m", "h". +bool ParseDuration(absl::string_view dur_sv, Duration* d) { + int sign = 1; + if (absl::ConsumePrefix(&dur_sv, "-")) { + sign = -1; + } else { + absl::ConsumePrefix(&dur_sv, "+"); + } + if (dur_sv.empty()) return false; + + // Special case for a string of "0". + if (dur_sv == "0") { + *d = ZeroDuration(); + return true; + } + + if (dur_sv == "inf") { + *d = sign * InfiniteDuration(); + return true; + } + + const char* start = dur_sv.data(); + const char* end = start + dur_sv.size(); + + Duration dur; + while (start != end) { + int64_t int_part; + int64_t frac_part; + int64_t frac_scale; + Duration unit; + if (!ConsumeDurationNumber(&start, end, &int_part, &frac_part, + &frac_scale) || + !ConsumeDurationUnit(&start, end, &unit)) { + return false; + } + if (int_part != 0) dur += sign * int_part * unit; + if (frac_part != 0) dur += sign * frac_part * unit / frac_scale; + } + *d = dur; + return true; +} + +bool AbslParseFlag(absl::string_view text, Duration* dst, std::string*) { + return ParseDuration(text, dst); +} + +std::string AbslUnparseFlag(Duration d) { return FormatDuration(d); } +bool ParseFlag(const std::string& text, Duration* dst, std::string* ) { + return ParseDuration(text, dst); +} + +std::string UnparseFlag(Duration d) { return FormatDuration(d); } + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/duration_benchmark.cc b/third_party/abseil_cpp/absl/time/duration_benchmark.cc new file mode 100644 index 000000000000..83a836c8c8a2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/duration_benchmark.cc @@ -0,0 +1,428 @@ +// Copyright 2018 The Abseil Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cmath> +#include <cstddef> +#include <cstdint> +#include <ctime> +#include <string> + +#include "absl/base/attributes.h" +#include "absl/time/time.h" +#include "benchmark/benchmark.h" + +namespace { + +// +// Factory functions +// + +void BM_Duration_Factory_Nanoseconds(benchmark::State& state) { + int64_t i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Nanoseconds(i)); + i += 314159; + } +} +BENCHMARK(BM_Duration_Factory_Nanoseconds); + +void BM_Duration_Factory_Microseconds(benchmark::State& state) { + int64_t i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Microseconds(i)); + i += 314; + } +} +BENCHMARK(BM_Duration_Factory_Microseconds); + +void BM_Duration_Factory_Milliseconds(benchmark::State& state) { + int64_t i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Milliseconds(i)); + i += 1; + } +} +BENCHMARK(BM_Duration_Factory_Milliseconds); + +void BM_Duration_Factory_Seconds(benchmark::State& state) { + int64_t i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Seconds(i)); + i += 1; + } +} +BENCHMARK(BM_Duration_Factory_Seconds); + +void BM_Duration_Factory_Minutes(benchmark::State& state) { + int64_t i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Minutes(i)); + i += 1; + } +} +BENCHMARK(BM_Duration_Factory_Minutes); + +void BM_Duration_Factory_Hours(benchmark::State& state) { + int64_t i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Hours(i)); + i += 1; + } +} +BENCHMARK(BM_Duration_Factory_Hours); + +void BM_Duration_Factory_DoubleNanoseconds(benchmark::State& state) { + double d = 1; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Nanoseconds(d)); + d = d * 1.00000001 + 1; + } +} +BENCHMARK(BM_Duration_Factory_DoubleNanoseconds); + +void BM_Duration_Factory_DoubleMicroseconds(benchmark::State& state) { + double d = 1e-3; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Microseconds(d)); + d = d * 1.00000001 + 1e-3; + } +} +BENCHMARK(BM_Duration_Factory_DoubleMicroseconds); + +void BM_Duration_Factory_DoubleMilliseconds(benchmark::State& state) { + double d = 1e-6; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Milliseconds(d)); + d = d * 1.00000001 + 1e-6; + } +} +BENCHMARK(BM_Duration_Factory_DoubleMilliseconds); + +void BM_Duration_Factory_DoubleSeconds(benchmark::State& state) { + double d = 1e-9; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Seconds(d)); + d = d * 1.00000001 + 1e-9; + } +} +BENCHMARK(BM_Duration_Factory_DoubleSeconds); + +void BM_Duration_Factory_DoubleMinutes(benchmark::State& state) { + double d = 1e-9; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Minutes(d)); + d = d * 1.00000001 + 1e-9; + } +} +BENCHMARK(BM_Duration_Factory_DoubleMinutes); + +void BM_Duration_Factory_DoubleHours(benchmark::State& state) { + double d = 1e-9; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::Hours(d)); + d = d * 1.00000001 + 1e-9; + } +} +BENCHMARK(BM_Duration_Factory_DoubleHours); + +// +// Arithmetic +// + +void BM_Duration_Addition(benchmark::State& state) { + absl::Duration d = absl::Nanoseconds(1); + absl::Duration step = absl::Milliseconds(1); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(d += step); + } +} +BENCHMARK(BM_Duration_Addition); + +void BM_Duration_Subtraction(benchmark::State& state) { + absl::Duration d = absl::Seconds(std::numeric_limits<int64_t>::max()); + absl::Duration step = absl::Milliseconds(1); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(d -= step); + } +} +BENCHMARK(BM_Duration_Subtraction); + +void BM_Duration_Multiplication_Fixed(benchmark::State& state) { + absl::Duration d = absl::Milliseconds(1); + absl::Duration s; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(s += d * (i + 1)); + ++i; + } +} +BENCHMARK(BM_Duration_Multiplication_Fixed); + +void BM_Duration_Multiplication_Double(benchmark::State& state) { + absl::Duration d = absl::Milliseconds(1); + absl::Duration s; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(s += d * (i + 1.0)); + ++i; + } +} +BENCHMARK(BM_Duration_Multiplication_Double); + +void BM_Duration_Division_Fixed(benchmark::State& state) { + absl::Duration d = absl::Seconds(1); + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(d /= i + 1); + ++i; + } +} +BENCHMARK(BM_Duration_Division_Fixed); + +void BM_Duration_Division_Double(benchmark::State& state) { + absl::Duration d = absl::Seconds(1); + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(d /= i + 1.0); + ++i; + } +} +BENCHMARK(BM_Duration_Division_Double); + +void BM_Duration_FDivDuration_Nanoseconds(benchmark::State& state) { + double d = 1; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize( + d += absl::FDivDuration(absl::Milliseconds(i), absl::Nanoseconds(1))); + ++i; + } +} +BENCHMARK(BM_Duration_FDivDuration_Nanoseconds); + +void BM_Duration_IDivDuration_Nanoseconds(benchmark::State& state) { + int64_t a = 1; + absl::Duration ignore; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(a += + absl::IDivDuration(absl::Nanoseconds(i), + absl::Nanoseconds(1), &ignore)); + ++i; + } +} +BENCHMARK(BM_Duration_IDivDuration_Nanoseconds); + +void BM_Duration_IDivDuration_Microseconds(benchmark::State& state) { + int64_t a = 1; + absl::Duration ignore; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(a += absl::IDivDuration(absl::Microseconds(i), + absl::Microseconds(1), + &ignore)); + ++i; + } +} +BENCHMARK(BM_Duration_IDivDuration_Microseconds); + +void BM_Duration_IDivDuration_Milliseconds(benchmark::State& state) { + int64_t a = 1; + absl::Duration ignore; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(a += absl::IDivDuration(absl::Milliseconds(i), + absl::Milliseconds(1), + &ignore)); + ++i; + } +} +BENCHMARK(BM_Duration_IDivDuration_Milliseconds); + +void BM_Duration_IDivDuration_Seconds(benchmark::State& state) { + int64_t a = 1; + absl::Duration ignore; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize( + a += absl::IDivDuration(absl::Seconds(i), absl::Seconds(1), &ignore)); + ++i; + } +} +BENCHMARK(BM_Duration_IDivDuration_Seconds); + +void BM_Duration_IDivDuration_Minutes(benchmark::State& state) { + int64_t a = 1; + absl::Duration ignore; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize( + a += absl::IDivDuration(absl::Minutes(i), absl::Minutes(1), &ignore)); + ++i; + } +} +BENCHMARK(BM_Duration_IDivDuration_Minutes); + +void BM_Duration_IDivDuration_Hours(benchmark::State& state) { + int64_t a = 1; + absl::Duration ignore; + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize( + a += absl::IDivDuration(absl::Hours(i), absl::Hours(1), &ignore)); + ++i; + } +} +BENCHMARK(BM_Duration_IDivDuration_Hours); + +void BM_Duration_ToInt64Nanoseconds(benchmark::State& state) { + absl::Duration d = absl::Seconds(100000); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToInt64Nanoseconds(d)); + } +} +BENCHMARK(BM_Duration_ToInt64Nanoseconds); + +void BM_Duration_ToInt64Microseconds(benchmark::State& state) { + absl::Duration d = absl::Seconds(100000); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToInt64Microseconds(d)); + } +} +BENCHMARK(BM_Duration_ToInt64Microseconds); + +void BM_Duration_ToInt64Milliseconds(benchmark::State& state) { + absl::Duration d = absl::Seconds(100000); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToInt64Milliseconds(d)); + } +} +BENCHMARK(BM_Duration_ToInt64Milliseconds); + +void BM_Duration_ToInt64Seconds(benchmark::State& state) { + absl::Duration d = absl::Seconds(100000); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToInt64Seconds(d)); + } +} +BENCHMARK(BM_Duration_ToInt64Seconds); + +void BM_Duration_ToInt64Minutes(benchmark::State& state) { + absl::Duration d = absl::Seconds(100000); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToInt64Minutes(d)); + } +} +BENCHMARK(BM_Duration_ToInt64Minutes); + +void BM_Duration_ToInt64Hours(benchmark::State& state) { + absl::Duration d = absl::Seconds(100000); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToInt64Hours(d)); + } +} +BENCHMARK(BM_Duration_ToInt64Hours); + +// +// To/FromTimespec +// + +void BM_Duration_ToTimespec_AbslTime(benchmark::State& state) { + absl::Duration d = absl::Seconds(1); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToTimespec(d)); + } +} +BENCHMARK(BM_Duration_ToTimespec_AbslTime); + +ABSL_ATTRIBUTE_NOINLINE timespec DoubleToTimespec(double seconds) { + timespec ts; + ts.tv_sec = seconds; + ts.tv_nsec = (seconds - ts.tv_sec) * (1000 * 1000 * 1000); + return ts; +} + +void BM_Duration_ToTimespec_Double(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(DoubleToTimespec(1.0)); + } +} +BENCHMARK(BM_Duration_ToTimespec_Double); + +void BM_Duration_FromTimespec_AbslTime(benchmark::State& state) { + timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 0; + while (state.KeepRunning()) { + if (++ts.tv_nsec == 1000 * 1000 * 1000) { + ++ts.tv_sec; + ts.tv_nsec = 0; + } + benchmark::DoNotOptimize(absl::DurationFromTimespec(ts)); + } +} +BENCHMARK(BM_Duration_FromTimespec_AbslTime); + +ABSL_ATTRIBUTE_NOINLINE double TimespecToDouble(timespec ts) { + return ts.tv_sec + (ts.tv_nsec / (1000 * 1000 * 1000)); +} + +void BM_Duration_FromTimespec_Double(benchmark::State& state) { + timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 0; + while (state.KeepRunning()) { + if (++ts.tv_nsec == 1000 * 1000 * 1000) { + ++ts.tv_sec; + ts.tv_nsec = 0; + } + benchmark::DoNotOptimize(TimespecToDouble(ts)); + } +} +BENCHMARK(BM_Duration_FromTimespec_Double); + +// +// String conversions +// + +const char* const kDurations[] = { + "0", // 0 + "123ns", // 1 + "1h2m3s", // 2 + "-2h3m4.005006007s", // 3 + "2562047788015215h30m7.99999999975s", // 4 +}; +const int kNumDurations = sizeof(kDurations) / sizeof(kDurations[0]); + +void BM_Duration_FormatDuration(benchmark::State& state) { + const std::string s = kDurations[state.range(0)]; + state.SetLabel(s); + absl::Duration d; + absl::ParseDuration(kDurations[state.range(0)], &d); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::FormatDuration(d)); + } +} +BENCHMARK(BM_Duration_FormatDuration)->DenseRange(0, kNumDurations - 1); + +void BM_Duration_ParseDuration(benchmark::State& state) { + const std::string s = kDurations[state.range(0)]; + state.SetLabel(s); + absl::Duration d; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ParseDuration(s, &d)); + } +} +BENCHMARK(BM_Duration_ParseDuration)->DenseRange(0, kNumDurations - 1); + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/duration_test.cc b/third_party/abseil_cpp/absl/time/duration_test.cc new file mode 100644 index 000000000000..4d85a2c4f455 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/duration_test.cc @@ -0,0 +1,1808 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(_MSC_VER) +#include <winsock2.h> // for timeval +#endif + +#include <chrono> // NOLINT(build/c++11) +#include <cmath> +#include <cstdint> +#include <ctime> +#include <iomanip> +#include <limits> +#include <random> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/time/time.h" + +namespace { + +constexpr int64_t kint64max = std::numeric_limits<int64_t>::max(); +constexpr int64_t kint64min = std::numeric_limits<int64_t>::min(); + +// Approximates the given number of years. This is only used to make some test +// code more readable. +absl::Duration ApproxYears(int64_t n) { return absl::Hours(n) * 365 * 24; } + +// A gMock matcher to match timespec values. Use this matcher like: +// timespec ts1, ts2; +// EXPECT_THAT(ts1, TimespecMatcher(ts2)); +MATCHER_P(TimespecMatcher, ts, "") { + if (ts.tv_sec == arg.tv_sec && ts.tv_nsec == arg.tv_nsec) + return true; + *result_listener << "expected: {" << ts.tv_sec << ", " << ts.tv_nsec << "} "; + *result_listener << "actual: {" << arg.tv_sec << ", " << arg.tv_nsec << "}"; + return false; +} + +// A gMock matcher to match timeval values. Use this matcher like: +// timeval tv1, tv2; +// EXPECT_THAT(tv1, TimevalMatcher(tv2)); +MATCHER_P(TimevalMatcher, tv, "") { + if (tv.tv_sec == arg.tv_sec && tv.tv_usec == arg.tv_usec) + return true; + *result_listener << "expected: {" << tv.tv_sec << ", " << tv.tv_usec << "} "; + *result_listener << "actual: {" << arg.tv_sec << ", " << arg.tv_usec << "}"; + return false; +} + +TEST(Duration, ConstExpr) { + constexpr absl::Duration d0 = absl::ZeroDuration(); + static_assert(d0 == absl::ZeroDuration(), "ZeroDuration()"); + constexpr absl::Duration d1 = absl::Seconds(1); + static_assert(d1 == absl::Seconds(1), "Seconds(1)"); + static_assert(d1 != absl::ZeroDuration(), "Seconds(1)"); + constexpr absl::Duration d2 = absl::InfiniteDuration(); + static_assert(d2 == absl::InfiniteDuration(), "InfiniteDuration()"); + static_assert(d2 != absl::ZeroDuration(), "InfiniteDuration()"); +} + +TEST(Duration, ValueSemantics) { + // If this compiles, the test passes. + constexpr absl::Duration a; // Default construction + constexpr absl::Duration b = a; // Copy construction + constexpr absl::Duration c(b); // Copy construction (again) + + absl::Duration d; + d = c; // Assignment +} + +TEST(Duration, Factories) { + constexpr absl::Duration zero = absl::ZeroDuration(); + constexpr absl::Duration nano = absl::Nanoseconds(1); + constexpr absl::Duration micro = absl::Microseconds(1); + constexpr absl::Duration milli = absl::Milliseconds(1); + constexpr absl::Duration sec = absl::Seconds(1); + constexpr absl::Duration min = absl::Minutes(1); + constexpr absl::Duration hour = absl::Hours(1); + + EXPECT_EQ(zero, absl::Duration()); + EXPECT_EQ(zero, absl::Seconds(0)); + EXPECT_EQ(nano, absl::Nanoseconds(1)); + EXPECT_EQ(micro, absl::Nanoseconds(1000)); + EXPECT_EQ(milli, absl::Microseconds(1000)); + EXPECT_EQ(sec, absl::Milliseconds(1000)); + EXPECT_EQ(min, absl::Seconds(60)); + EXPECT_EQ(hour, absl::Minutes(60)); + + // Tests factory limits + const absl::Duration inf = absl::InfiniteDuration(); + + EXPECT_GT(inf, absl::Seconds(kint64max)); + EXPECT_LT(-inf, absl::Seconds(kint64min)); + EXPECT_LT(-inf, absl::Seconds(-kint64max)); + + EXPECT_EQ(inf, absl::Minutes(kint64max)); + EXPECT_EQ(-inf, absl::Minutes(kint64min)); + EXPECT_EQ(-inf, absl::Minutes(-kint64max)); + EXPECT_GT(inf, absl::Minutes(kint64max / 60)); + EXPECT_LT(-inf, absl::Minutes(kint64min / 60)); + EXPECT_LT(-inf, absl::Minutes(-kint64max / 60)); + + EXPECT_EQ(inf, absl::Hours(kint64max)); + EXPECT_EQ(-inf, absl::Hours(kint64min)); + EXPECT_EQ(-inf, absl::Hours(-kint64max)); + EXPECT_GT(inf, absl::Hours(kint64max / 3600)); + EXPECT_LT(-inf, absl::Hours(kint64min / 3600)); + EXPECT_LT(-inf, absl::Hours(-kint64max / 3600)); +} + +TEST(Duration, ToConversion) { +#define TEST_DURATION_CONVERSION(UNIT) \ + do { \ + const absl::Duration d = absl::UNIT(1.5); \ + constexpr absl::Duration z = absl::ZeroDuration(); \ + constexpr absl::Duration inf = absl::InfiniteDuration(); \ + constexpr double dbl_inf = std::numeric_limits<double>::infinity(); \ + EXPECT_EQ(kint64min, absl::ToInt64##UNIT(-inf)); \ + EXPECT_EQ(-1, absl::ToInt64##UNIT(-d)); \ + EXPECT_EQ(0, absl::ToInt64##UNIT(z)); \ + EXPECT_EQ(1, absl::ToInt64##UNIT(d)); \ + EXPECT_EQ(kint64max, absl::ToInt64##UNIT(inf)); \ + EXPECT_EQ(-dbl_inf, absl::ToDouble##UNIT(-inf)); \ + EXPECT_EQ(-1.5, absl::ToDouble##UNIT(-d)); \ + EXPECT_EQ(0, absl::ToDouble##UNIT(z)); \ + EXPECT_EQ(1.5, absl::ToDouble##UNIT(d)); \ + EXPECT_EQ(dbl_inf, absl::ToDouble##UNIT(inf)); \ + } while (0) + + TEST_DURATION_CONVERSION(Nanoseconds); + TEST_DURATION_CONVERSION(Microseconds); + TEST_DURATION_CONVERSION(Milliseconds); + TEST_DURATION_CONVERSION(Seconds); + TEST_DURATION_CONVERSION(Minutes); + TEST_DURATION_CONVERSION(Hours); + +#undef TEST_DURATION_CONVERSION +} + +template <int64_t N> +void TestToConversion() { + constexpr absl::Duration nano = absl::Nanoseconds(N); + EXPECT_EQ(N, absl::ToInt64Nanoseconds(nano)); + EXPECT_EQ(0, absl::ToInt64Microseconds(nano)); + EXPECT_EQ(0, absl::ToInt64Milliseconds(nano)); + EXPECT_EQ(0, absl::ToInt64Seconds(nano)); + EXPECT_EQ(0, absl::ToInt64Minutes(nano)); + EXPECT_EQ(0, absl::ToInt64Hours(nano)); + const absl::Duration micro = absl::Microseconds(N); + EXPECT_EQ(N * 1000, absl::ToInt64Nanoseconds(micro)); + EXPECT_EQ(N, absl::ToInt64Microseconds(micro)); + EXPECT_EQ(0, absl::ToInt64Milliseconds(micro)); + EXPECT_EQ(0, absl::ToInt64Seconds(micro)); + EXPECT_EQ(0, absl::ToInt64Minutes(micro)); + EXPECT_EQ(0, absl::ToInt64Hours(micro)); + const absl::Duration milli = absl::Milliseconds(N); + EXPECT_EQ(N * 1000 * 1000, absl::ToInt64Nanoseconds(milli)); + EXPECT_EQ(N * 1000, absl::ToInt64Microseconds(milli)); + EXPECT_EQ(N, absl::ToInt64Milliseconds(milli)); + EXPECT_EQ(0, absl::ToInt64Seconds(milli)); + EXPECT_EQ(0, absl::ToInt64Minutes(milli)); + EXPECT_EQ(0, absl::ToInt64Hours(milli)); + const absl::Duration sec = absl::Seconds(N); + EXPECT_EQ(N * 1000 * 1000 * 1000, absl::ToInt64Nanoseconds(sec)); + EXPECT_EQ(N * 1000 * 1000, absl::ToInt64Microseconds(sec)); + EXPECT_EQ(N * 1000, absl::ToInt64Milliseconds(sec)); + EXPECT_EQ(N, absl::ToInt64Seconds(sec)); + EXPECT_EQ(0, absl::ToInt64Minutes(sec)); + EXPECT_EQ(0, absl::ToInt64Hours(sec)); + const absl::Duration min = absl::Minutes(N); + EXPECT_EQ(N * 60 * 1000 * 1000 * 1000, absl::ToInt64Nanoseconds(min)); + EXPECT_EQ(N * 60 * 1000 * 1000, absl::ToInt64Microseconds(min)); + EXPECT_EQ(N * 60 * 1000, absl::ToInt64Milliseconds(min)); + EXPECT_EQ(N * 60, absl::ToInt64Seconds(min)); + EXPECT_EQ(N, absl::ToInt64Minutes(min)); + EXPECT_EQ(0, absl::ToInt64Hours(min)); + const absl::Duration hour = absl::Hours(N); + EXPECT_EQ(N * 60 * 60 * 1000 * 1000 * 1000, absl::ToInt64Nanoseconds(hour)); + EXPECT_EQ(N * 60 * 60 * 1000 * 1000, absl::ToInt64Microseconds(hour)); + EXPECT_EQ(N * 60 * 60 * 1000, absl::ToInt64Milliseconds(hour)); + EXPECT_EQ(N * 60 * 60, absl::ToInt64Seconds(hour)); + EXPECT_EQ(N * 60, absl::ToInt64Minutes(hour)); + EXPECT_EQ(N, absl::ToInt64Hours(hour)); +} + +TEST(Duration, ToConversionDeprecated) { + TestToConversion<43>(); + TestToConversion<1>(); + TestToConversion<0>(); + TestToConversion<-1>(); + TestToConversion<-43>(); +} + +template <int64_t N> +void TestFromChronoBasicEquality() { + using std::chrono::nanoseconds; + using std::chrono::microseconds; + using std::chrono::milliseconds; + using std::chrono::seconds; + using std::chrono::minutes; + using std::chrono::hours; + + static_assert(absl::Nanoseconds(N) == absl::FromChrono(nanoseconds(N)), ""); + static_assert(absl::Microseconds(N) == absl::FromChrono(microseconds(N)), ""); + static_assert(absl::Milliseconds(N) == absl::FromChrono(milliseconds(N)), ""); + static_assert(absl::Seconds(N) == absl::FromChrono(seconds(N)), ""); + static_assert(absl::Minutes(N) == absl::FromChrono(minutes(N)), ""); + static_assert(absl::Hours(N) == absl::FromChrono(hours(N)), ""); +} + +TEST(Duration, FromChrono) { + TestFromChronoBasicEquality<-123>(); + TestFromChronoBasicEquality<-1>(); + TestFromChronoBasicEquality<0>(); + TestFromChronoBasicEquality<1>(); + TestFromChronoBasicEquality<123>(); + + // Minutes (might, depending on the platform) saturate at +inf. + const auto chrono_minutes_max = std::chrono::minutes::max(); + const auto minutes_max = absl::FromChrono(chrono_minutes_max); + const int64_t minutes_max_count = chrono_minutes_max.count(); + if (minutes_max_count > kint64max / 60) { + EXPECT_EQ(absl::InfiniteDuration(), minutes_max); + } else { + EXPECT_EQ(absl::Minutes(minutes_max_count), minutes_max); + } + + // Minutes (might, depending on the platform) saturate at -inf. + const auto chrono_minutes_min = std::chrono::minutes::min(); + const auto minutes_min = absl::FromChrono(chrono_minutes_min); + const int64_t minutes_min_count = chrono_minutes_min.count(); + if (minutes_min_count < kint64min / 60) { + EXPECT_EQ(-absl::InfiniteDuration(), minutes_min); + } else { + EXPECT_EQ(absl::Minutes(minutes_min_count), minutes_min); + } + + // Hours (might, depending on the platform) saturate at +inf. + const auto chrono_hours_max = std::chrono::hours::max(); + const auto hours_max = absl::FromChrono(chrono_hours_max); + const int64_t hours_max_count = chrono_hours_max.count(); + if (hours_max_count > kint64max / 3600) { + EXPECT_EQ(absl::InfiniteDuration(), hours_max); + } else { + EXPECT_EQ(absl::Hours(hours_max_count), hours_max); + } + + // Hours (might, depending on the platform) saturate at -inf. + const auto chrono_hours_min = std::chrono::hours::min(); + const auto hours_min = absl::FromChrono(chrono_hours_min); + const int64_t hours_min_count = chrono_hours_min.count(); + if (hours_min_count < kint64min / 3600) { + EXPECT_EQ(-absl::InfiniteDuration(), hours_min); + } else { + EXPECT_EQ(absl::Hours(hours_min_count), hours_min); + } +} + +template <int64_t N> +void TestToChrono() { + using std::chrono::nanoseconds; + using std::chrono::microseconds; + using std::chrono::milliseconds; + using std::chrono::seconds; + using std::chrono::minutes; + using std::chrono::hours; + + EXPECT_EQ(nanoseconds(N), absl::ToChronoNanoseconds(absl::Nanoseconds(N))); + EXPECT_EQ(microseconds(N), absl::ToChronoMicroseconds(absl::Microseconds(N))); + EXPECT_EQ(milliseconds(N), absl::ToChronoMilliseconds(absl::Milliseconds(N))); + EXPECT_EQ(seconds(N), absl::ToChronoSeconds(absl::Seconds(N))); + + constexpr auto absl_minutes = absl::Minutes(N); + auto chrono_minutes = minutes(N); + if (absl_minutes == -absl::InfiniteDuration()) { + chrono_minutes = minutes::min(); + } else if (absl_minutes == absl::InfiniteDuration()) { + chrono_minutes = minutes::max(); + } + EXPECT_EQ(chrono_minutes, absl::ToChronoMinutes(absl_minutes)); + + constexpr auto absl_hours = absl::Hours(N); + auto chrono_hours = hours(N); + if (absl_hours == -absl::InfiniteDuration()) { + chrono_hours = hours::min(); + } else if (absl_hours == absl::InfiniteDuration()) { + chrono_hours = hours::max(); + } + EXPECT_EQ(chrono_hours, absl::ToChronoHours(absl_hours)); +} + +TEST(Duration, ToChrono) { + using std::chrono::nanoseconds; + using std::chrono::microseconds; + using std::chrono::milliseconds; + using std::chrono::seconds; + using std::chrono::minutes; + using std::chrono::hours; + + TestToChrono<kint64min>(); + TestToChrono<-1>(); + TestToChrono<0>(); + TestToChrono<1>(); + TestToChrono<kint64max>(); + + // Verify truncation toward zero. + const auto tick = absl::Nanoseconds(1) / 4; + EXPECT_EQ(nanoseconds(0), absl::ToChronoNanoseconds(tick)); + EXPECT_EQ(nanoseconds(0), absl::ToChronoNanoseconds(-tick)); + EXPECT_EQ(microseconds(0), absl::ToChronoMicroseconds(tick)); + EXPECT_EQ(microseconds(0), absl::ToChronoMicroseconds(-tick)); + EXPECT_EQ(milliseconds(0), absl::ToChronoMilliseconds(tick)); + EXPECT_EQ(milliseconds(0), absl::ToChronoMilliseconds(-tick)); + EXPECT_EQ(seconds(0), absl::ToChronoSeconds(tick)); + EXPECT_EQ(seconds(0), absl::ToChronoSeconds(-tick)); + EXPECT_EQ(minutes(0), absl::ToChronoMinutes(tick)); + EXPECT_EQ(minutes(0), absl::ToChronoMinutes(-tick)); + EXPECT_EQ(hours(0), absl::ToChronoHours(tick)); + EXPECT_EQ(hours(0), absl::ToChronoHours(-tick)); + + // Verifies +/- infinity saturation at max/min. + constexpr auto inf = absl::InfiniteDuration(); + EXPECT_EQ(nanoseconds::min(), absl::ToChronoNanoseconds(-inf)); + EXPECT_EQ(nanoseconds::max(), absl::ToChronoNanoseconds(inf)); + EXPECT_EQ(microseconds::min(), absl::ToChronoMicroseconds(-inf)); + EXPECT_EQ(microseconds::max(), absl::ToChronoMicroseconds(inf)); + EXPECT_EQ(milliseconds::min(), absl::ToChronoMilliseconds(-inf)); + EXPECT_EQ(milliseconds::max(), absl::ToChronoMilliseconds(inf)); + EXPECT_EQ(seconds::min(), absl::ToChronoSeconds(-inf)); + EXPECT_EQ(seconds::max(), absl::ToChronoSeconds(inf)); + EXPECT_EQ(minutes::min(), absl::ToChronoMinutes(-inf)); + EXPECT_EQ(minutes::max(), absl::ToChronoMinutes(inf)); + EXPECT_EQ(hours::min(), absl::ToChronoHours(-inf)); + EXPECT_EQ(hours::max(), absl::ToChronoHours(inf)); +} + +TEST(Duration, FactoryOverloads) { + enum E { kOne = 1 }; +#define TEST_FACTORY_OVERLOADS(NAME) \ + EXPECT_EQ(1, NAME(kOne) / NAME(kOne)); \ + EXPECT_EQ(1, NAME(static_cast<int8_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<int16_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<int32_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<int64_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<uint8_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<uint16_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<uint32_t>(1)) / NAME(1)); \ + EXPECT_EQ(1, NAME(static_cast<uint64_t>(1)) / NAME(1)); \ + EXPECT_EQ(NAME(1) / 2, NAME(static_cast<float>(0.5))); \ + EXPECT_EQ(NAME(1) / 2, NAME(static_cast<double>(0.5))); \ + EXPECT_EQ(1.5, absl::FDivDuration(NAME(static_cast<float>(1.5)), NAME(1))); \ + EXPECT_EQ(1.5, absl::FDivDuration(NAME(static_cast<double>(1.5)), NAME(1))); + + TEST_FACTORY_OVERLOADS(absl::Nanoseconds); + TEST_FACTORY_OVERLOADS(absl::Microseconds); + TEST_FACTORY_OVERLOADS(absl::Milliseconds); + TEST_FACTORY_OVERLOADS(absl::Seconds); + TEST_FACTORY_OVERLOADS(absl::Minutes); + TEST_FACTORY_OVERLOADS(absl::Hours); + +#undef TEST_FACTORY_OVERLOADS + + EXPECT_EQ(absl::Milliseconds(1500), absl::Seconds(1.5)); + EXPECT_LT(absl::Nanoseconds(1), absl::Nanoseconds(1.5)); + EXPECT_GT(absl::Nanoseconds(2), absl::Nanoseconds(1.5)); + + const double dbl_inf = std::numeric_limits<double>::infinity(); + EXPECT_EQ(absl::InfiniteDuration(), absl::Nanoseconds(dbl_inf)); + EXPECT_EQ(absl::InfiniteDuration(), absl::Microseconds(dbl_inf)); + EXPECT_EQ(absl::InfiniteDuration(), absl::Milliseconds(dbl_inf)); + EXPECT_EQ(absl::InfiniteDuration(), absl::Seconds(dbl_inf)); + EXPECT_EQ(absl::InfiniteDuration(), absl::Minutes(dbl_inf)); + EXPECT_EQ(absl::InfiniteDuration(), absl::Hours(dbl_inf)); + EXPECT_EQ(-absl::InfiniteDuration(), absl::Nanoseconds(-dbl_inf)); + EXPECT_EQ(-absl::InfiniteDuration(), absl::Microseconds(-dbl_inf)); + EXPECT_EQ(-absl::InfiniteDuration(), absl::Milliseconds(-dbl_inf)); + EXPECT_EQ(-absl::InfiniteDuration(), absl::Seconds(-dbl_inf)); + EXPECT_EQ(-absl::InfiniteDuration(), absl::Minutes(-dbl_inf)); + EXPECT_EQ(-absl::InfiniteDuration(), absl::Hours(-dbl_inf)); +} + +TEST(Duration, InfinityExamples) { + // These examples are used in the documentation in time.h. They are + // written so that they can be copy-n-pasted easily. + + constexpr absl::Duration inf = absl::InfiniteDuration(); + constexpr absl::Duration d = absl::Seconds(1); // Any finite duration + + EXPECT_TRUE(inf == inf + inf); + EXPECT_TRUE(inf == inf + d); + EXPECT_TRUE(inf == inf - inf); + EXPECT_TRUE(-inf == d - inf); + + EXPECT_TRUE(inf == d * 1e100); + EXPECT_TRUE(0 == d / inf); // NOLINT(readability/check) + + // Division by zero returns infinity, or kint64min/MAX where necessary. + EXPECT_TRUE(inf == d / 0); + EXPECT_TRUE(kint64max == d / absl::ZeroDuration()); +} + +TEST(Duration, InfinityComparison) { + const absl::Duration inf = absl::InfiniteDuration(); + const absl::Duration any_dur = absl::Seconds(1); + + // Equality + EXPECT_EQ(inf, inf); + EXPECT_EQ(-inf, -inf); + EXPECT_NE(inf, -inf); + EXPECT_NE(any_dur, inf); + EXPECT_NE(any_dur, -inf); + + // Relational + EXPECT_GT(inf, any_dur); + EXPECT_LT(-inf, any_dur); + EXPECT_LT(-inf, inf); + EXPECT_GT(inf, -inf); +} + +TEST(Duration, InfinityAddition) { + const absl::Duration sec_max = absl::Seconds(kint64max); + const absl::Duration sec_min = absl::Seconds(kint64min); + const absl::Duration any_dur = absl::Seconds(1); + const absl::Duration inf = absl::InfiniteDuration(); + + // Addition + EXPECT_EQ(inf, inf + inf); + EXPECT_EQ(inf, inf + -inf); + EXPECT_EQ(-inf, -inf + inf); + EXPECT_EQ(-inf, -inf + -inf); + + EXPECT_EQ(inf, inf + any_dur); + EXPECT_EQ(inf, any_dur + inf); + EXPECT_EQ(-inf, -inf + any_dur); + EXPECT_EQ(-inf, any_dur + -inf); + + // Interesting case + absl::Duration almost_inf = sec_max + absl::Nanoseconds(999999999); + EXPECT_GT(inf, almost_inf); + almost_inf += -absl::Nanoseconds(999999999); + EXPECT_GT(inf, almost_inf); + + // Addition overflow/underflow + EXPECT_EQ(inf, sec_max + absl::Seconds(1)); + EXPECT_EQ(inf, sec_max + sec_max); + EXPECT_EQ(-inf, sec_min + -absl::Seconds(1)); + EXPECT_EQ(-inf, sec_min + -sec_max); + + // For reference: IEEE 754 behavior + const double dbl_inf = std::numeric_limits<double>::infinity(); + EXPECT_TRUE(std::isinf(dbl_inf + dbl_inf)); + EXPECT_TRUE(std::isnan(dbl_inf + -dbl_inf)); // We return inf + EXPECT_TRUE(std::isnan(-dbl_inf + dbl_inf)); // We return inf + EXPECT_TRUE(std::isinf(-dbl_inf + -dbl_inf)); +} + +TEST(Duration, InfinitySubtraction) { + const absl::Duration sec_max = absl::Seconds(kint64max); + const absl::Duration sec_min = absl::Seconds(kint64min); + const absl::Duration any_dur = absl::Seconds(1); + const absl::Duration inf = absl::InfiniteDuration(); + + // Subtraction + EXPECT_EQ(inf, inf - inf); + EXPECT_EQ(inf, inf - -inf); + EXPECT_EQ(-inf, -inf - inf); + EXPECT_EQ(-inf, -inf - -inf); + + EXPECT_EQ(inf, inf - any_dur); + EXPECT_EQ(-inf, any_dur - inf); + EXPECT_EQ(-inf, -inf - any_dur); + EXPECT_EQ(inf, any_dur - -inf); + + // Subtraction overflow/underflow + EXPECT_EQ(inf, sec_max - -absl::Seconds(1)); + EXPECT_EQ(inf, sec_max - -sec_max); + EXPECT_EQ(-inf, sec_min - absl::Seconds(1)); + EXPECT_EQ(-inf, sec_min - sec_max); + + // Interesting case + absl::Duration almost_neg_inf = sec_min; + EXPECT_LT(-inf, almost_neg_inf); + almost_neg_inf -= -absl::Nanoseconds(1); + EXPECT_LT(-inf, almost_neg_inf); + + // For reference: IEEE 754 behavior + const double dbl_inf = std::numeric_limits<double>::infinity(); + EXPECT_TRUE(std::isnan(dbl_inf - dbl_inf)); // We return inf + EXPECT_TRUE(std::isinf(dbl_inf - -dbl_inf)); + EXPECT_TRUE(std::isinf(-dbl_inf - dbl_inf)); + EXPECT_TRUE(std::isnan(-dbl_inf - -dbl_inf)); // We return inf +} + +TEST(Duration, InfinityMultiplication) { + const absl::Duration sec_max = absl::Seconds(kint64max); + const absl::Duration sec_min = absl::Seconds(kint64min); + const absl::Duration inf = absl::InfiniteDuration(); + +#define TEST_INF_MUL_WITH_TYPE(T) \ + EXPECT_EQ(inf, inf * static_cast<T>(2)); \ + EXPECT_EQ(-inf, inf * static_cast<T>(-2)); \ + EXPECT_EQ(-inf, -inf * static_cast<T>(2)); \ + EXPECT_EQ(inf, -inf * static_cast<T>(-2)); \ + EXPECT_EQ(inf, inf * static_cast<T>(0)); \ + EXPECT_EQ(-inf, -inf * static_cast<T>(0)); \ + EXPECT_EQ(inf, sec_max * static_cast<T>(2)); \ + EXPECT_EQ(inf, sec_min * static_cast<T>(-2)); \ + EXPECT_EQ(inf, (sec_max / static_cast<T>(2)) * static_cast<T>(3)); \ + EXPECT_EQ(-inf, sec_max * static_cast<T>(-2)); \ + EXPECT_EQ(-inf, sec_min * static_cast<T>(2)); \ + EXPECT_EQ(-inf, (sec_min / static_cast<T>(2)) * static_cast<T>(3)); + + TEST_INF_MUL_WITH_TYPE(int64_t); // NOLINT(readability/function) + TEST_INF_MUL_WITH_TYPE(double); // NOLINT(readability/function) + +#undef TEST_INF_MUL_WITH_TYPE + + const double dbl_inf = std::numeric_limits<double>::infinity(); + EXPECT_EQ(inf, inf * dbl_inf); + EXPECT_EQ(-inf, -inf * dbl_inf); + EXPECT_EQ(-inf, inf * -dbl_inf); + EXPECT_EQ(inf, -inf * -dbl_inf); + + const absl::Duration any_dur = absl::Seconds(1); + EXPECT_EQ(inf, any_dur * dbl_inf); + EXPECT_EQ(-inf, -any_dur * dbl_inf); + EXPECT_EQ(-inf, any_dur * -dbl_inf); + EXPECT_EQ(inf, -any_dur * -dbl_inf); + + // Fixed-point multiplication will produce a finite value, whereas floating + // point fuzziness will overflow to inf. + EXPECT_NE(absl::InfiniteDuration(), absl::Seconds(1) * kint64max); + EXPECT_EQ(inf, absl::Seconds(1) * static_cast<double>(kint64max)); + EXPECT_NE(-absl::InfiniteDuration(), absl::Seconds(1) * kint64min); + EXPECT_EQ(-inf, absl::Seconds(1) * static_cast<double>(kint64min)); + + // Note that sec_max * or / by 1.0 overflows to inf due to the 53-bit + // limitations of double. + EXPECT_NE(inf, sec_max); + EXPECT_NE(inf, sec_max / 1); + EXPECT_EQ(inf, sec_max / 1.0); + EXPECT_NE(inf, sec_max * 1); + EXPECT_EQ(inf, sec_max * 1.0); +} + +TEST(Duration, InfinityDivision) { + const absl::Duration sec_max = absl::Seconds(kint64max); + const absl::Duration sec_min = absl::Seconds(kint64min); + const absl::Duration inf = absl::InfiniteDuration(); + + // Division of Duration by a double +#define TEST_INF_DIV_WITH_TYPE(T) \ + EXPECT_EQ(inf, inf / static_cast<T>(2)); \ + EXPECT_EQ(-inf, inf / static_cast<T>(-2)); \ + EXPECT_EQ(-inf, -inf / static_cast<T>(2)); \ + EXPECT_EQ(inf, -inf / static_cast<T>(-2)); + + TEST_INF_DIV_WITH_TYPE(int64_t); // NOLINT(readability/function) + TEST_INF_DIV_WITH_TYPE(double); // NOLINT(readability/function) + +#undef TEST_INF_DIV_WITH_TYPE + + // Division of Duration by a double overflow/underflow + EXPECT_EQ(inf, sec_max / 0.5); + EXPECT_EQ(inf, sec_min / -0.5); + EXPECT_EQ(inf, ((sec_max / 0.5) + absl::Seconds(1)) / 0.5); + EXPECT_EQ(-inf, sec_max / -0.5); + EXPECT_EQ(-inf, sec_min / 0.5); + EXPECT_EQ(-inf, ((sec_min / 0.5) - absl::Seconds(1)) / 0.5); + + const double dbl_inf = std::numeric_limits<double>::infinity(); + EXPECT_EQ(inf, inf / dbl_inf); + EXPECT_EQ(-inf, inf / -dbl_inf); + EXPECT_EQ(-inf, -inf / dbl_inf); + EXPECT_EQ(inf, -inf / -dbl_inf); + + const absl::Duration any_dur = absl::Seconds(1); + EXPECT_EQ(absl::ZeroDuration(), any_dur / dbl_inf); + EXPECT_EQ(absl::ZeroDuration(), any_dur / -dbl_inf); + EXPECT_EQ(absl::ZeroDuration(), -any_dur / dbl_inf); + EXPECT_EQ(absl::ZeroDuration(), -any_dur / -dbl_inf); +} + +TEST(Duration, InfinityModulus) { + const absl::Duration sec_max = absl::Seconds(kint64max); + const absl::Duration any_dur = absl::Seconds(1); + const absl::Duration inf = absl::InfiniteDuration(); + + EXPECT_EQ(inf, inf % inf); + EXPECT_EQ(inf, inf % -inf); + EXPECT_EQ(-inf, -inf % -inf); + EXPECT_EQ(-inf, -inf % inf); + + EXPECT_EQ(any_dur, any_dur % inf); + EXPECT_EQ(any_dur, any_dur % -inf); + EXPECT_EQ(-any_dur, -any_dur % inf); + EXPECT_EQ(-any_dur, -any_dur % -inf); + + EXPECT_EQ(inf, inf % -any_dur); + EXPECT_EQ(inf, inf % any_dur); + EXPECT_EQ(-inf, -inf % -any_dur); + EXPECT_EQ(-inf, -inf % any_dur); + + // Remainder isn't affected by overflow. + EXPECT_EQ(absl::ZeroDuration(), sec_max % absl::Seconds(1)); + EXPECT_EQ(absl::ZeroDuration(), sec_max % absl::Milliseconds(1)); + EXPECT_EQ(absl::ZeroDuration(), sec_max % absl::Microseconds(1)); + EXPECT_EQ(absl::ZeroDuration(), sec_max % absl::Nanoseconds(1)); + EXPECT_EQ(absl::ZeroDuration(), sec_max % absl::Nanoseconds(1) / 4); +} + +TEST(Duration, InfinityIDiv) { + const absl::Duration sec_max = absl::Seconds(kint64max); + const absl::Duration any_dur = absl::Seconds(1); + const absl::Duration inf = absl::InfiniteDuration(); + const double dbl_inf = std::numeric_limits<double>::infinity(); + + // IDivDuration (int64_t return value + a remainer) + absl::Duration rem = absl::ZeroDuration(); + EXPECT_EQ(kint64max, absl::IDivDuration(inf, inf, &rem)); + EXPECT_EQ(inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64max, absl::IDivDuration(-inf, -inf, &rem)); + EXPECT_EQ(-inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64max, absl::IDivDuration(inf, any_dur, &rem)); + EXPECT_EQ(inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(0, absl::IDivDuration(any_dur, inf, &rem)); + EXPECT_EQ(any_dur, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64max, absl::IDivDuration(-inf, -any_dur, &rem)); + EXPECT_EQ(-inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(0, absl::IDivDuration(-any_dur, -inf, &rem)); + EXPECT_EQ(-any_dur, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64min, absl::IDivDuration(-inf, inf, &rem)); + EXPECT_EQ(-inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64min, absl::IDivDuration(inf, -inf, &rem)); + EXPECT_EQ(inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64min, absl::IDivDuration(-inf, any_dur, &rem)); + EXPECT_EQ(-inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(0, absl::IDivDuration(-any_dur, inf, &rem)); + EXPECT_EQ(-any_dur, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(kint64min, absl::IDivDuration(inf, -any_dur, &rem)); + EXPECT_EQ(inf, rem); + + rem = absl::ZeroDuration(); + EXPECT_EQ(0, absl::IDivDuration(any_dur, -inf, &rem)); + EXPECT_EQ(any_dur, rem); + + // IDivDuration overflow/underflow + rem = any_dur; + EXPECT_EQ(kint64max, + absl::IDivDuration(sec_max, absl::Nanoseconds(1) / 4, &rem)); + EXPECT_EQ(sec_max - absl::Nanoseconds(kint64max) / 4, rem); + + rem = any_dur; + EXPECT_EQ(kint64max, + absl::IDivDuration(sec_max, absl::Milliseconds(1), &rem)); + EXPECT_EQ(sec_max - absl::Milliseconds(kint64max), rem); + + rem = any_dur; + EXPECT_EQ(kint64max, + absl::IDivDuration(-sec_max, -absl::Milliseconds(1), &rem)); + EXPECT_EQ(-sec_max + absl::Milliseconds(kint64max), rem); + + rem = any_dur; + EXPECT_EQ(kint64min, + absl::IDivDuration(-sec_max, absl::Milliseconds(1), &rem)); + EXPECT_EQ(-sec_max - absl::Milliseconds(kint64min), rem); + + rem = any_dur; + EXPECT_EQ(kint64min, + absl::IDivDuration(sec_max, -absl::Milliseconds(1), &rem)); + EXPECT_EQ(sec_max + absl::Milliseconds(kint64min), rem); + + // + // operator/(Duration, Duration) is a wrapper for IDivDuration(). + // + + // IEEE 754 says inf / inf should be nan, but int64_t doesn't have + // nan so we'll return kint64max/kint64min instead. + EXPECT_TRUE(std::isnan(dbl_inf / dbl_inf)); + EXPECT_EQ(kint64max, inf / inf); + EXPECT_EQ(kint64max, -inf / -inf); + EXPECT_EQ(kint64min, -inf / inf); + EXPECT_EQ(kint64min, inf / -inf); + + EXPECT_TRUE(std::isinf(dbl_inf / 2.0)); + EXPECT_EQ(kint64max, inf / any_dur); + EXPECT_EQ(kint64max, -inf / -any_dur); + EXPECT_EQ(kint64min, -inf / any_dur); + EXPECT_EQ(kint64min, inf / -any_dur); + + EXPECT_EQ(0.0, 2.0 / dbl_inf); + EXPECT_EQ(0, any_dur / inf); + EXPECT_EQ(0, any_dur / -inf); + EXPECT_EQ(0, -any_dur / inf); + EXPECT_EQ(0, -any_dur / -inf); + EXPECT_EQ(0, absl::ZeroDuration() / inf); + + // Division of Duration by a Duration overflow/underflow + EXPECT_EQ(kint64max, sec_max / absl::Milliseconds(1)); + EXPECT_EQ(kint64max, -sec_max / -absl::Milliseconds(1)); + EXPECT_EQ(kint64min, -sec_max / absl::Milliseconds(1)); + EXPECT_EQ(kint64min, sec_max / -absl::Milliseconds(1)); +} + +TEST(Duration, InfinityFDiv) { + const absl::Duration any_dur = absl::Seconds(1); + const absl::Duration inf = absl::InfiniteDuration(); + const double dbl_inf = std::numeric_limits<double>::infinity(); + + EXPECT_EQ(dbl_inf, absl::FDivDuration(inf, inf)); + EXPECT_EQ(dbl_inf, absl::FDivDuration(-inf, -inf)); + EXPECT_EQ(dbl_inf, absl::FDivDuration(inf, any_dur)); + EXPECT_EQ(0.0, absl::FDivDuration(any_dur, inf)); + EXPECT_EQ(dbl_inf, absl::FDivDuration(-inf, -any_dur)); + EXPECT_EQ(0.0, absl::FDivDuration(-any_dur, -inf)); + + EXPECT_EQ(-dbl_inf, absl::FDivDuration(-inf, inf)); + EXPECT_EQ(-dbl_inf, absl::FDivDuration(inf, -inf)); + EXPECT_EQ(-dbl_inf, absl::FDivDuration(-inf, any_dur)); + EXPECT_EQ(0.0, absl::FDivDuration(-any_dur, inf)); + EXPECT_EQ(-dbl_inf, absl::FDivDuration(inf, -any_dur)); + EXPECT_EQ(0.0, absl::FDivDuration(any_dur, -inf)); +} + +TEST(Duration, DivisionByZero) { + const absl::Duration zero = absl::ZeroDuration(); + const absl::Duration inf = absl::InfiniteDuration(); + const absl::Duration any_dur = absl::Seconds(1); + const double dbl_inf = std::numeric_limits<double>::infinity(); + const double dbl_denorm = std::numeric_limits<double>::denorm_min(); + + // Operator/(Duration, double) + EXPECT_EQ(inf, zero / 0.0); + EXPECT_EQ(-inf, zero / -0.0); + EXPECT_EQ(inf, any_dur / 0.0); + EXPECT_EQ(-inf, any_dur / -0.0); + EXPECT_EQ(-inf, -any_dur / 0.0); + EXPECT_EQ(inf, -any_dur / -0.0); + + // Tests dividing by a number very close to, but not quite zero. + EXPECT_EQ(zero, zero / dbl_denorm); + EXPECT_EQ(zero, zero / -dbl_denorm); + EXPECT_EQ(inf, any_dur / dbl_denorm); + EXPECT_EQ(-inf, any_dur / -dbl_denorm); + EXPECT_EQ(-inf, -any_dur / dbl_denorm); + EXPECT_EQ(inf, -any_dur / -dbl_denorm); + + // IDiv + absl::Duration rem = zero; + EXPECT_EQ(kint64max, absl::IDivDuration(zero, zero, &rem)); + EXPECT_EQ(inf, rem); + + rem = zero; + EXPECT_EQ(kint64max, absl::IDivDuration(any_dur, zero, &rem)); + EXPECT_EQ(inf, rem); + + rem = zero; + EXPECT_EQ(kint64min, absl::IDivDuration(-any_dur, zero, &rem)); + EXPECT_EQ(-inf, rem); + + // Operator/(Duration, Duration) + EXPECT_EQ(kint64max, zero / zero); + EXPECT_EQ(kint64max, any_dur / zero); + EXPECT_EQ(kint64min, -any_dur / zero); + + // FDiv + EXPECT_EQ(dbl_inf, absl::FDivDuration(zero, zero)); + EXPECT_EQ(dbl_inf, absl::FDivDuration(any_dur, zero)); + EXPECT_EQ(-dbl_inf, absl::FDivDuration(-any_dur, zero)); +} + +TEST(Duration, NaN) { + // Note that IEEE 754 does not define the behavior of a nan's sign when it is + // copied, so the code below allows for either + or - InfiniteDuration. +#define TEST_NAN_HANDLING(NAME, NAN) \ + do { \ + const auto inf = absl::InfiniteDuration(); \ + auto x = NAME(NAN); \ + EXPECT_TRUE(x == inf || x == -inf); \ + auto y = NAME(42); \ + y *= NAN; \ + EXPECT_TRUE(y == inf || y == -inf); \ + auto z = NAME(42); \ + z /= NAN; \ + EXPECT_TRUE(z == inf || z == -inf); \ + } while (0) + + const double nan = std::numeric_limits<double>::quiet_NaN(); + TEST_NAN_HANDLING(absl::Nanoseconds, nan); + TEST_NAN_HANDLING(absl::Microseconds, nan); + TEST_NAN_HANDLING(absl::Milliseconds, nan); + TEST_NAN_HANDLING(absl::Seconds, nan); + TEST_NAN_HANDLING(absl::Minutes, nan); + TEST_NAN_HANDLING(absl::Hours, nan); + + TEST_NAN_HANDLING(absl::Nanoseconds, -nan); + TEST_NAN_HANDLING(absl::Microseconds, -nan); + TEST_NAN_HANDLING(absl::Milliseconds, -nan); + TEST_NAN_HANDLING(absl::Seconds, -nan); + TEST_NAN_HANDLING(absl::Minutes, -nan); + TEST_NAN_HANDLING(absl::Hours, -nan); + +#undef TEST_NAN_HANDLING +} + +TEST(Duration, Range) { + const absl::Duration range = ApproxYears(100 * 1e9); + const absl::Duration range_future = range; + const absl::Duration range_past = -range; + + EXPECT_LT(range_future, absl::InfiniteDuration()); + EXPECT_GT(range_past, -absl::InfiniteDuration()); + + const absl::Duration full_range = range_future - range_past; + EXPECT_GT(full_range, absl::ZeroDuration()); + EXPECT_LT(full_range, absl::InfiniteDuration()); + + const absl::Duration neg_full_range = range_past - range_future; + EXPECT_LT(neg_full_range, absl::ZeroDuration()); + EXPECT_GT(neg_full_range, -absl::InfiniteDuration()); + + EXPECT_LT(neg_full_range, full_range); + EXPECT_EQ(neg_full_range, -full_range); +} + +TEST(Duration, RelationalOperators) { +#define TEST_REL_OPS(UNIT) \ + static_assert(UNIT(2) == UNIT(2), ""); \ + static_assert(UNIT(1) != UNIT(2), ""); \ + static_assert(UNIT(1) < UNIT(2), ""); \ + static_assert(UNIT(3) > UNIT(2), ""); \ + static_assert(UNIT(1) <= UNIT(2), ""); \ + static_assert(UNIT(2) <= UNIT(2), ""); \ + static_assert(UNIT(3) >= UNIT(2), ""); \ + static_assert(UNIT(2) >= UNIT(2), ""); + + TEST_REL_OPS(absl::Nanoseconds); + TEST_REL_OPS(absl::Microseconds); + TEST_REL_OPS(absl::Milliseconds); + TEST_REL_OPS(absl::Seconds); + TEST_REL_OPS(absl::Minutes); + TEST_REL_OPS(absl::Hours); + +#undef TEST_REL_OPS +} + +TEST(Duration, Addition) { +#define TEST_ADD_OPS(UNIT) \ + do { \ + EXPECT_EQ(UNIT(2), UNIT(1) + UNIT(1)); \ + EXPECT_EQ(UNIT(1), UNIT(2) - UNIT(1)); \ + EXPECT_EQ(UNIT(0), UNIT(2) - UNIT(2)); \ + EXPECT_EQ(UNIT(-1), UNIT(1) - UNIT(2)); \ + EXPECT_EQ(UNIT(-2), UNIT(0) - UNIT(2)); \ + EXPECT_EQ(UNIT(-2), UNIT(1) - UNIT(3)); \ + absl::Duration a = UNIT(1); \ + a += UNIT(1); \ + EXPECT_EQ(UNIT(2), a); \ + a -= UNIT(1); \ + EXPECT_EQ(UNIT(1), a); \ + } while (0) + + TEST_ADD_OPS(absl::Nanoseconds); + TEST_ADD_OPS(absl::Microseconds); + TEST_ADD_OPS(absl::Milliseconds); + TEST_ADD_OPS(absl::Seconds); + TEST_ADD_OPS(absl::Minutes); + TEST_ADD_OPS(absl::Hours); + +#undef TEST_ADD_OPS + + EXPECT_EQ(absl::Seconds(2), absl::Seconds(3) - 2 * absl::Milliseconds(500)); + EXPECT_EQ(absl::Seconds(2) + absl::Milliseconds(500), + absl::Seconds(3) - absl::Milliseconds(500)); + + EXPECT_EQ(absl::Seconds(1) + absl::Milliseconds(998), + absl::Milliseconds(999) + absl::Milliseconds(999)); + + EXPECT_EQ(absl::Milliseconds(-1), + absl::Milliseconds(998) - absl::Milliseconds(999)); + + // Tests fractions of a nanoseconds. These are implementation details only. + EXPECT_GT(absl::Nanoseconds(1), absl::Nanoseconds(1) / 2); + EXPECT_EQ(absl::Nanoseconds(1), + absl::Nanoseconds(1) / 2 + absl::Nanoseconds(1) / 2); + EXPECT_GT(absl::Nanoseconds(1) / 4, absl::Nanoseconds(0)); + EXPECT_EQ(absl::Nanoseconds(1) / 8, absl::Nanoseconds(0)); + + // Tests subtraction that will cause wrap around of the rep_lo_ bits. + absl::Duration d_7_5 = absl::Seconds(7) + absl::Milliseconds(500); + absl::Duration d_3_7 = absl::Seconds(3) + absl::Milliseconds(700); + absl::Duration ans_3_8 = absl::Seconds(3) + absl::Milliseconds(800); + EXPECT_EQ(ans_3_8, d_7_5 - d_3_7); + + // Subtracting min_duration + absl::Duration min_dur = absl::Seconds(kint64min); + EXPECT_EQ(absl::Seconds(0), min_dur - min_dur); + EXPECT_EQ(absl::Seconds(kint64max), absl::Seconds(-1) - min_dur); +} + +TEST(Duration, Negation) { + // By storing negations of various values in constexpr variables we + // verify that the initializers are constant expressions. + constexpr absl::Duration negated_zero_duration = -absl::ZeroDuration(); + EXPECT_EQ(negated_zero_duration, absl::ZeroDuration()); + + constexpr absl::Duration negated_infinite_duration = + -absl::InfiniteDuration(); + EXPECT_NE(negated_infinite_duration, absl::InfiniteDuration()); + EXPECT_EQ(-negated_infinite_duration, absl::InfiniteDuration()); + + // The public APIs to check if a duration is infinite depend on using + // -InfiniteDuration(), but we're trying to test operator- here, so we + // need to use the lower-level internal query IsInfiniteDuration. + EXPECT_TRUE( + absl::time_internal::IsInfiniteDuration(negated_infinite_duration)); + + // The largest Duration is kint64max seconds and kTicksPerSecond - 1 ticks. + // Using the absl::time_internal::MakeDuration API is the cleanest way to + // construct that Duration. + constexpr absl::Duration max_duration = absl::time_internal::MakeDuration( + kint64max, absl::time_internal::kTicksPerSecond - 1); + constexpr absl::Duration negated_max_duration = -max_duration; + // The largest negatable value is one tick above the minimum representable; + // it's the negation of max_duration. + constexpr absl::Duration nearly_min_duration = + absl::time_internal::MakeDuration(kint64min, int64_t{1}); + constexpr absl::Duration negated_nearly_min_duration = -nearly_min_duration; + + EXPECT_EQ(negated_max_duration, nearly_min_duration); + EXPECT_EQ(negated_nearly_min_duration, max_duration); + EXPECT_EQ(-(-max_duration), max_duration); + + constexpr absl::Duration min_duration = + absl::time_internal::MakeDuration(kint64min); + constexpr absl::Duration negated_min_duration = -min_duration; + EXPECT_EQ(negated_min_duration, absl::InfiniteDuration()); +} + +TEST(Duration, AbsoluteValue) { + EXPECT_EQ(absl::ZeroDuration(), AbsDuration(absl::ZeroDuration())); + EXPECT_EQ(absl::Seconds(1), AbsDuration(absl::Seconds(1))); + EXPECT_EQ(absl::Seconds(1), AbsDuration(absl::Seconds(-1))); + + EXPECT_EQ(absl::InfiniteDuration(), AbsDuration(absl::InfiniteDuration())); + EXPECT_EQ(absl::InfiniteDuration(), AbsDuration(-absl::InfiniteDuration())); + + absl::Duration max_dur = + absl::Seconds(kint64max) + (absl::Seconds(1) - absl::Nanoseconds(1) / 4); + EXPECT_EQ(max_dur, AbsDuration(max_dur)); + + absl::Duration min_dur = absl::Seconds(kint64min); + EXPECT_EQ(absl::InfiniteDuration(), AbsDuration(min_dur)); + EXPECT_EQ(max_dur, AbsDuration(min_dur + absl::Nanoseconds(1) / 4)); +} + +TEST(Duration, Multiplication) { +#define TEST_MUL_OPS(UNIT) \ + do { \ + EXPECT_EQ(UNIT(5), UNIT(2) * 2.5); \ + EXPECT_EQ(UNIT(2), UNIT(5) / 2.5); \ + EXPECT_EQ(UNIT(-5), UNIT(-2) * 2.5); \ + EXPECT_EQ(UNIT(-5), -UNIT(2) * 2.5); \ + EXPECT_EQ(UNIT(-5), UNIT(2) * -2.5); \ + EXPECT_EQ(UNIT(-2), UNIT(-5) / 2.5); \ + EXPECT_EQ(UNIT(-2), -UNIT(5) / 2.5); \ + EXPECT_EQ(UNIT(-2), UNIT(5) / -2.5); \ + EXPECT_EQ(UNIT(2), UNIT(11) % UNIT(3)); \ + absl::Duration a = UNIT(2); \ + a *= 2.5; \ + EXPECT_EQ(UNIT(5), a); \ + a /= 2.5; \ + EXPECT_EQ(UNIT(2), a); \ + a %= UNIT(1); \ + EXPECT_EQ(UNIT(0), a); \ + absl::Duration big = UNIT(1000000000); \ + big *= 3; \ + big /= 3; \ + EXPECT_EQ(UNIT(1000000000), big); \ + EXPECT_EQ(-UNIT(2), -UNIT(2)); \ + EXPECT_EQ(-UNIT(2), UNIT(2) * -1); \ + EXPECT_EQ(-UNIT(2), -1 * UNIT(2)); \ + EXPECT_EQ(-UNIT(-2), UNIT(2)); \ + EXPECT_EQ(2, UNIT(2) / UNIT(1)); \ + absl::Duration rem; \ + EXPECT_EQ(2, absl::IDivDuration(UNIT(2), UNIT(1), &rem)); \ + EXPECT_EQ(2.0, absl::FDivDuration(UNIT(2), UNIT(1))); \ + } while (0) + + TEST_MUL_OPS(absl::Nanoseconds); + TEST_MUL_OPS(absl::Microseconds); + TEST_MUL_OPS(absl::Milliseconds); + TEST_MUL_OPS(absl::Seconds); + TEST_MUL_OPS(absl::Minutes); + TEST_MUL_OPS(absl::Hours); + +#undef TEST_MUL_OPS + + // Ensures that multiplication and division by 1 with a maxed-out durations + // doesn't lose precision. + absl::Duration max_dur = + absl::Seconds(kint64max) + (absl::Seconds(1) - absl::Nanoseconds(1) / 4); + absl::Duration min_dur = absl::Seconds(kint64min); + EXPECT_EQ(max_dur, max_dur * 1); + EXPECT_EQ(max_dur, max_dur / 1); + EXPECT_EQ(min_dur, min_dur * 1); + EXPECT_EQ(min_dur, min_dur / 1); + + // Tests division on a Duration with a large number of significant digits. + // Tests when the digits span hi and lo as well as only in hi. + absl::Duration sigfigs = absl::Seconds(2000000000) + absl::Nanoseconds(3); + EXPECT_EQ(absl::Seconds(666666666) + absl::Nanoseconds(666666667) + + absl::Nanoseconds(1) / 2, + sigfigs / 3); + sigfigs = absl::Seconds(int64_t{7000000000}); + EXPECT_EQ(absl::Seconds(2333333333) + absl::Nanoseconds(333333333) + + absl::Nanoseconds(1) / 4, + sigfigs / 3); + + EXPECT_EQ(absl::Seconds(7) + absl::Milliseconds(500), absl::Seconds(3) * 2.5); + EXPECT_EQ(absl::Seconds(8) * -1 + absl::Milliseconds(300), + (absl::Seconds(2) + absl::Milliseconds(200)) * -3.5); + EXPECT_EQ(-absl::Seconds(8) + absl::Milliseconds(300), + (absl::Seconds(2) + absl::Milliseconds(200)) * -3.5); + EXPECT_EQ(absl::Seconds(1) + absl::Milliseconds(875), + (absl::Seconds(7) + absl::Milliseconds(500)) / 4); + EXPECT_EQ(absl::Seconds(30), + (absl::Seconds(7) + absl::Milliseconds(500)) / 0.25); + EXPECT_EQ(absl::Seconds(3), + (absl::Seconds(7) + absl::Milliseconds(500)) / 2.5); + + // Tests division remainder. + EXPECT_EQ(absl::Nanoseconds(0), absl::Nanoseconds(7) % absl::Nanoseconds(1)); + EXPECT_EQ(absl::Nanoseconds(0), absl::Nanoseconds(0) % absl::Nanoseconds(10)); + EXPECT_EQ(absl::Nanoseconds(2), absl::Nanoseconds(7) % absl::Nanoseconds(5)); + EXPECT_EQ(absl::Nanoseconds(2), absl::Nanoseconds(2) % absl::Nanoseconds(5)); + + EXPECT_EQ(absl::Nanoseconds(1), absl::Nanoseconds(10) % absl::Nanoseconds(3)); + EXPECT_EQ(absl::Nanoseconds(1), + absl::Nanoseconds(10) % absl::Nanoseconds(-3)); + EXPECT_EQ(absl::Nanoseconds(-1), + absl::Nanoseconds(-10) % absl::Nanoseconds(3)); + EXPECT_EQ(absl::Nanoseconds(-1), + absl::Nanoseconds(-10) % absl::Nanoseconds(-3)); + + EXPECT_EQ(absl::Milliseconds(100), + absl::Seconds(1) % absl::Milliseconds(300)); + EXPECT_EQ( + absl::Milliseconds(300), + (absl::Seconds(3) + absl::Milliseconds(800)) % absl::Milliseconds(500)); + + EXPECT_EQ(absl::Nanoseconds(1), absl::Nanoseconds(1) % absl::Seconds(1)); + EXPECT_EQ(absl::Nanoseconds(-1), absl::Nanoseconds(-1) % absl::Seconds(1)); + EXPECT_EQ(0, absl::Nanoseconds(-1) / absl::Seconds(1)); // Actual -1e-9 + + // Tests identity a = (a/b)*b + a%b +#define TEST_MOD_IDENTITY(a, b) \ + EXPECT_EQ((a), ((a) / (b))*(b) + ((a)%(b))) + + TEST_MOD_IDENTITY(absl::Seconds(0), absl::Seconds(2)); + TEST_MOD_IDENTITY(absl::Seconds(1), absl::Seconds(1)); + TEST_MOD_IDENTITY(absl::Seconds(1), absl::Seconds(2)); + TEST_MOD_IDENTITY(absl::Seconds(2), absl::Seconds(1)); + + TEST_MOD_IDENTITY(absl::Seconds(-2), absl::Seconds(1)); + TEST_MOD_IDENTITY(absl::Seconds(2), absl::Seconds(-1)); + TEST_MOD_IDENTITY(absl::Seconds(-2), absl::Seconds(-1)); + + TEST_MOD_IDENTITY(absl::Nanoseconds(0), absl::Nanoseconds(2)); + TEST_MOD_IDENTITY(absl::Nanoseconds(1), absl::Nanoseconds(1)); + TEST_MOD_IDENTITY(absl::Nanoseconds(1), absl::Nanoseconds(2)); + TEST_MOD_IDENTITY(absl::Nanoseconds(2), absl::Nanoseconds(1)); + + TEST_MOD_IDENTITY(absl::Nanoseconds(-2), absl::Nanoseconds(1)); + TEST_MOD_IDENTITY(absl::Nanoseconds(2), absl::Nanoseconds(-1)); + TEST_MOD_IDENTITY(absl::Nanoseconds(-2), absl::Nanoseconds(-1)); + + // Mixed seconds + subseconds + absl::Duration mixed_a = absl::Seconds(1) + absl::Nanoseconds(2); + absl::Duration mixed_b = absl::Seconds(1) + absl::Nanoseconds(3); + + TEST_MOD_IDENTITY(absl::Seconds(0), mixed_a); + TEST_MOD_IDENTITY(mixed_a, mixed_a); + TEST_MOD_IDENTITY(mixed_a, mixed_b); + TEST_MOD_IDENTITY(mixed_b, mixed_a); + + TEST_MOD_IDENTITY(-mixed_a, mixed_b); + TEST_MOD_IDENTITY(mixed_a, -mixed_b); + TEST_MOD_IDENTITY(-mixed_a, -mixed_b); + +#undef TEST_MOD_IDENTITY +} + +TEST(Duration, Truncation) { + const absl::Duration d = absl::Nanoseconds(1234567890); + const absl::Duration inf = absl::InfiniteDuration(); + for (int unit_sign : {1, -1}) { // sign shouldn't matter + EXPECT_EQ(absl::Nanoseconds(1234567890), + Trunc(d, unit_sign * absl::Nanoseconds(1))); + EXPECT_EQ(absl::Microseconds(1234567), + Trunc(d, unit_sign * absl::Microseconds(1))); + EXPECT_EQ(absl::Milliseconds(1234), + Trunc(d, unit_sign * absl::Milliseconds(1))); + EXPECT_EQ(absl::Seconds(1), Trunc(d, unit_sign * absl::Seconds(1))); + EXPECT_EQ(inf, Trunc(inf, unit_sign * absl::Seconds(1))); + + EXPECT_EQ(absl::Nanoseconds(-1234567890), + Trunc(-d, unit_sign * absl::Nanoseconds(1))); + EXPECT_EQ(absl::Microseconds(-1234567), + Trunc(-d, unit_sign * absl::Microseconds(1))); + EXPECT_EQ(absl::Milliseconds(-1234), + Trunc(-d, unit_sign * absl::Milliseconds(1))); + EXPECT_EQ(absl::Seconds(-1), Trunc(-d, unit_sign * absl::Seconds(1))); + EXPECT_EQ(-inf, Trunc(-inf, unit_sign * absl::Seconds(1))); + } +} + +TEST(Duration, Flooring) { + const absl::Duration d = absl::Nanoseconds(1234567890); + const absl::Duration inf = absl::InfiniteDuration(); + for (int unit_sign : {1, -1}) { // sign shouldn't matter + EXPECT_EQ(absl::Nanoseconds(1234567890), + absl::Floor(d, unit_sign * absl::Nanoseconds(1))); + EXPECT_EQ(absl::Microseconds(1234567), + absl::Floor(d, unit_sign * absl::Microseconds(1))); + EXPECT_EQ(absl::Milliseconds(1234), + absl::Floor(d, unit_sign * absl::Milliseconds(1))); + EXPECT_EQ(absl::Seconds(1), absl::Floor(d, unit_sign * absl::Seconds(1))); + EXPECT_EQ(inf, absl::Floor(inf, unit_sign * absl::Seconds(1))); + + EXPECT_EQ(absl::Nanoseconds(-1234567890), + absl::Floor(-d, unit_sign * absl::Nanoseconds(1))); + EXPECT_EQ(absl::Microseconds(-1234568), + absl::Floor(-d, unit_sign * absl::Microseconds(1))); + EXPECT_EQ(absl::Milliseconds(-1235), + absl::Floor(-d, unit_sign * absl::Milliseconds(1))); + EXPECT_EQ(absl::Seconds(-2), absl::Floor(-d, unit_sign * absl::Seconds(1))); + EXPECT_EQ(-inf, absl::Floor(-inf, unit_sign * absl::Seconds(1))); + } +} + +TEST(Duration, Ceiling) { + const absl::Duration d = absl::Nanoseconds(1234567890); + const absl::Duration inf = absl::InfiniteDuration(); + for (int unit_sign : {1, -1}) { // // sign shouldn't matter + EXPECT_EQ(absl::Nanoseconds(1234567890), + absl::Ceil(d, unit_sign * absl::Nanoseconds(1))); + EXPECT_EQ(absl::Microseconds(1234568), + absl::Ceil(d, unit_sign * absl::Microseconds(1))); + EXPECT_EQ(absl::Milliseconds(1235), + absl::Ceil(d, unit_sign * absl::Milliseconds(1))); + EXPECT_EQ(absl::Seconds(2), absl::Ceil(d, unit_sign * absl::Seconds(1))); + EXPECT_EQ(inf, absl::Ceil(inf, unit_sign * absl::Seconds(1))); + + EXPECT_EQ(absl::Nanoseconds(-1234567890), + absl::Ceil(-d, unit_sign * absl::Nanoseconds(1))); + EXPECT_EQ(absl::Microseconds(-1234567), + absl::Ceil(-d, unit_sign * absl::Microseconds(1))); + EXPECT_EQ(absl::Milliseconds(-1234), + absl::Ceil(-d, unit_sign * absl::Milliseconds(1))); + EXPECT_EQ(absl::Seconds(-1), absl::Ceil(-d, unit_sign * absl::Seconds(1))); + EXPECT_EQ(-inf, absl::Ceil(-inf, unit_sign * absl::Seconds(1))); + } +} + +TEST(Duration, RoundTripUnits) { + const int kRange = 100000; + +#define ROUND_TRIP_UNIT(U, LOW, HIGH) \ + do { \ + for (int64_t i = LOW; i < HIGH; ++i) { \ + absl::Duration d = absl::U(i); \ + if (d == absl::InfiniteDuration()) \ + EXPECT_EQ(kint64max, d / absl::U(1)); \ + else if (d == -absl::InfiniteDuration()) \ + EXPECT_EQ(kint64min, d / absl::U(1)); \ + else \ + EXPECT_EQ(i, absl::U(i) / absl::U(1)); \ + } \ + } while (0) + + ROUND_TRIP_UNIT(Nanoseconds, kint64min, kint64min + kRange); + ROUND_TRIP_UNIT(Nanoseconds, -kRange, kRange); + ROUND_TRIP_UNIT(Nanoseconds, kint64max - kRange, kint64max); + + ROUND_TRIP_UNIT(Microseconds, kint64min, kint64min + kRange); + ROUND_TRIP_UNIT(Microseconds, -kRange, kRange); + ROUND_TRIP_UNIT(Microseconds, kint64max - kRange, kint64max); + + ROUND_TRIP_UNIT(Milliseconds, kint64min, kint64min + kRange); + ROUND_TRIP_UNIT(Milliseconds, -kRange, kRange); + ROUND_TRIP_UNIT(Milliseconds, kint64max - kRange, kint64max); + + ROUND_TRIP_UNIT(Seconds, kint64min, kint64min + kRange); + ROUND_TRIP_UNIT(Seconds, -kRange, kRange); + ROUND_TRIP_UNIT(Seconds, kint64max - kRange, kint64max); + + ROUND_TRIP_UNIT(Minutes, kint64min / 60, kint64min / 60 + kRange); + ROUND_TRIP_UNIT(Minutes, -kRange, kRange); + ROUND_TRIP_UNIT(Minutes, kint64max / 60 - kRange, kint64max / 60); + + ROUND_TRIP_UNIT(Hours, kint64min / 3600, kint64min / 3600 + kRange); + ROUND_TRIP_UNIT(Hours, -kRange, kRange); + ROUND_TRIP_UNIT(Hours, kint64max / 3600 - kRange, kint64max / 3600); + +#undef ROUND_TRIP_UNIT +} + +TEST(Duration, TruncConversions) { + // Tests ToTimespec()/DurationFromTimespec() + const struct { + absl::Duration d; + timespec ts; + } to_ts[] = { + {absl::Seconds(1) + absl::Nanoseconds(1), {1, 1}}, + {absl::Seconds(1) + absl::Nanoseconds(1) / 2, {1, 0}}, + {absl::Seconds(1) + absl::Nanoseconds(0), {1, 0}}, + {absl::Seconds(0) + absl::Nanoseconds(0), {0, 0}}, + {absl::Seconds(0) - absl::Nanoseconds(1) / 2, {0, 0}}, + {absl::Seconds(0) - absl::Nanoseconds(1), {-1, 999999999}}, + {absl::Seconds(-1) + absl::Nanoseconds(1), {-1, 1}}, + {absl::Seconds(-1) + absl::Nanoseconds(1) / 2, {-1, 1}}, + {absl::Seconds(-1) + absl::Nanoseconds(0), {-1, 0}}, + {absl::Seconds(-1) - absl::Nanoseconds(1) / 2, {-1, 0}}, + }; + for (const auto& test : to_ts) { + EXPECT_THAT(absl::ToTimespec(test.d), TimespecMatcher(test.ts)); + } + const struct { + timespec ts; + absl::Duration d; + } from_ts[] = { + {{1, 1}, absl::Seconds(1) + absl::Nanoseconds(1)}, + {{1, 0}, absl::Seconds(1) + absl::Nanoseconds(0)}, + {{0, 0}, absl::Seconds(0) + absl::Nanoseconds(0)}, + {{0, -1}, absl::Seconds(0) - absl::Nanoseconds(1)}, + {{-1, 999999999}, absl::Seconds(0) - absl::Nanoseconds(1)}, + {{-1, 1}, absl::Seconds(-1) + absl::Nanoseconds(1)}, + {{-1, 0}, absl::Seconds(-1) + absl::Nanoseconds(0)}, + {{-1, -1}, absl::Seconds(-1) - absl::Nanoseconds(1)}, + {{-2, 999999999}, absl::Seconds(-1) - absl::Nanoseconds(1)}, + }; + for (const auto& test : from_ts) { + EXPECT_EQ(test.d, absl::DurationFromTimespec(test.ts)); + } + + // Tests ToTimeval()/DurationFromTimeval() (same as timespec above) + const struct { + absl::Duration d; + timeval tv; + } to_tv[] = { + {absl::Seconds(1) + absl::Microseconds(1), {1, 1}}, + {absl::Seconds(1) + absl::Microseconds(1) / 2, {1, 0}}, + {absl::Seconds(1) + absl::Microseconds(0), {1, 0}}, + {absl::Seconds(0) + absl::Microseconds(0), {0, 0}}, + {absl::Seconds(0) - absl::Microseconds(1) / 2, {0, 0}}, + {absl::Seconds(0) - absl::Microseconds(1), {-1, 999999}}, + {absl::Seconds(-1) + absl::Microseconds(1), {-1, 1}}, + {absl::Seconds(-1) + absl::Microseconds(1) / 2, {-1, 1}}, + {absl::Seconds(-1) + absl::Microseconds(0), {-1, 0}}, + {absl::Seconds(-1) - absl::Microseconds(1) / 2, {-1, 0}}, + }; + for (const auto& test : to_tv) { + EXPECT_THAT(absl::ToTimeval(test.d), TimevalMatcher(test.tv)); + } + const struct { + timeval tv; + absl::Duration d; + } from_tv[] = { + {{1, 1}, absl::Seconds(1) + absl::Microseconds(1)}, + {{1, 0}, absl::Seconds(1) + absl::Microseconds(0)}, + {{0, 0}, absl::Seconds(0) + absl::Microseconds(0)}, + {{0, -1}, absl::Seconds(0) - absl::Microseconds(1)}, + {{-1, 999999}, absl::Seconds(0) - absl::Microseconds(1)}, + {{-1, 1}, absl::Seconds(-1) + absl::Microseconds(1)}, + {{-1, 0}, absl::Seconds(-1) + absl::Microseconds(0)}, + {{-1, -1}, absl::Seconds(-1) - absl::Microseconds(1)}, + {{-2, 999999}, absl::Seconds(-1) - absl::Microseconds(1)}, + }; + for (const auto& test : from_tv) { + EXPECT_EQ(test.d, absl::DurationFromTimeval(test.tv)); + } +} + +TEST(Duration, SmallConversions) { + // Special tests for conversions of small durations. + + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0)); + // TODO(bww): Is the next one OK? + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0.124999999e-9)); + EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.125e-9)); + EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.250e-9)); + EXPECT_EQ(absl::Nanoseconds(1) / 2, absl::Seconds(0.375e-9)); + EXPECT_EQ(absl::Nanoseconds(1) / 2, absl::Seconds(0.500e-9)); + EXPECT_EQ(absl::Nanoseconds(3) / 4, absl::Seconds(0.625e-9)); + EXPECT_EQ(absl::Nanoseconds(3) / 4, absl::Seconds(0.750e-9)); + EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(0.875e-9)); + EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(1.000e-9)); + + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(-0.124999999e-9)); + EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.125e-9)); + EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.250e-9)); + EXPECT_EQ(-absl::Nanoseconds(1) / 2, absl::Seconds(-0.375e-9)); + EXPECT_EQ(-absl::Nanoseconds(1) / 2, absl::Seconds(-0.500e-9)); + EXPECT_EQ(-absl::Nanoseconds(3) / 4, absl::Seconds(-0.625e-9)); + EXPECT_EQ(-absl::Nanoseconds(3) / 4, absl::Seconds(-0.750e-9)); + EXPECT_EQ(-absl::Nanoseconds(1), absl::Seconds(-0.875e-9)); + EXPECT_EQ(-absl::Nanoseconds(1), absl::Seconds(-1.000e-9)); + + timespec ts; + ts.tv_sec = 0; + ts.tv_nsec = 0; + EXPECT_THAT(ToTimespec(absl::Nanoseconds(0)), TimespecMatcher(ts)); + // TODO(bww): Are the next three OK? + EXPECT_THAT(ToTimespec(absl::Nanoseconds(1) / 4), TimespecMatcher(ts)); + EXPECT_THAT(ToTimespec(absl::Nanoseconds(2) / 4), TimespecMatcher(ts)); + EXPECT_THAT(ToTimespec(absl::Nanoseconds(3) / 4), TimespecMatcher(ts)); + ts.tv_nsec = 1; + EXPECT_THAT(ToTimespec(absl::Nanoseconds(4) / 4), TimespecMatcher(ts)); + EXPECT_THAT(ToTimespec(absl::Nanoseconds(5) / 4), TimespecMatcher(ts)); + EXPECT_THAT(ToTimespec(absl::Nanoseconds(6) / 4), TimespecMatcher(ts)); + EXPECT_THAT(ToTimespec(absl::Nanoseconds(7) / 4), TimespecMatcher(ts)); + ts.tv_nsec = 2; + EXPECT_THAT(ToTimespec(absl::Nanoseconds(8) / 4), TimespecMatcher(ts)); + + timeval tv; + tv.tv_sec = 0; + tv.tv_usec = 0; + EXPECT_THAT(ToTimeval(absl::Nanoseconds(0)), TimevalMatcher(tv)); + // TODO(bww): Is the next one OK? + EXPECT_THAT(ToTimeval(absl::Nanoseconds(999)), TimevalMatcher(tv)); + tv.tv_usec = 1; + EXPECT_THAT(ToTimeval(absl::Nanoseconds(1000)), TimevalMatcher(tv)); + EXPECT_THAT(ToTimeval(absl::Nanoseconds(1999)), TimevalMatcher(tv)); + tv.tv_usec = 2; + EXPECT_THAT(ToTimeval(absl::Nanoseconds(2000)), TimevalMatcher(tv)); +} + +void VerifySameAsMul(double time_as_seconds, int* const misses) { + auto direct_seconds = absl::Seconds(time_as_seconds); + auto mul_by_one_second = time_as_seconds * absl::Seconds(1); + if (direct_seconds != mul_by_one_second) { + if (*misses > 10) return; + ASSERT_LE(++(*misses), 10) << "Too many errors, not reporting more."; + EXPECT_EQ(direct_seconds, mul_by_one_second) + << "given double time_as_seconds = " << std::setprecision(17) + << time_as_seconds; + } +} + +// For a variety of interesting durations, we find the exact point +// where one double converts to that duration, and the very next double +// converts to the next duration. For both of those points, verify that +// Seconds(point) returns the same duration as point * Seconds(1.0) +TEST(Duration, ToDoubleSecondsCheckEdgeCases) { + constexpr uint32_t kTicksPerSecond = absl::time_internal::kTicksPerSecond; + constexpr auto duration_tick = absl::time_internal::MakeDuration(0, 1u); + int misses = 0; + for (int64_t seconds = 0; seconds < 99; ++seconds) { + uint32_t tick_vals[] = {0, +999, +999999, +999999999, kTicksPerSecond - 1, + 0, 1000, 1000000, 1000000000, kTicksPerSecond, + 1, 1001, 1000001, 1000000001, kTicksPerSecond + 1, + 2, 1002, 1000002, 1000000002, kTicksPerSecond + 2, + 3, 1003, 1000003, 1000000003, kTicksPerSecond + 3, + 4, 1004, 1000004, 1000000004, kTicksPerSecond + 4, + 5, 6, 7, 8, 9}; + for (uint32_t ticks : tick_vals) { + absl::Duration s_plus_t = absl::Seconds(seconds) + ticks * duration_tick; + for (absl::Duration d : {s_plus_t, -s_plus_t}) { + absl::Duration after_d = d + duration_tick; + EXPECT_NE(d, after_d); + EXPECT_EQ(after_d - d, duration_tick); + + double low_edge = ToDoubleSeconds(d); + EXPECT_EQ(d, absl::Seconds(low_edge)); + + double high_edge = ToDoubleSeconds(after_d); + EXPECT_EQ(after_d, absl::Seconds(high_edge)); + + for (;;) { + double midpoint = low_edge + (high_edge - low_edge) / 2; + if (midpoint == low_edge || midpoint == high_edge) break; + absl::Duration mid_duration = absl::Seconds(midpoint); + if (mid_duration == d) { + low_edge = midpoint; + } else { + EXPECT_EQ(mid_duration, after_d); + high_edge = midpoint; + } + } + // Now low_edge is the highest double that converts to Duration d, + // and high_edge is the lowest double that converts to Duration after_d. + VerifySameAsMul(low_edge, &misses); + VerifySameAsMul(high_edge, &misses); + } + } + } +} + +TEST(Duration, ToDoubleSecondsCheckRandom) { + std::random_device rd; + std::seed_seq seed({rd(), rd(), rd(), rd(), rd(), rd(), rd(), rd()}); + std::mt19937_64 gen(seed); + // We want doubles distributed from 1/8ns up to 2^63, where + // as many values are tested from 1ns to 2ns as from 1sec to 2sec, + // so even distribute along a log-scale of those values, and + // exponentiate before using them. (9.223377e+18 is just slightly + // out of bounds for absl::Duration.) + std::uniform_real_distribution<double> uniform(std::log(0.125e-9), + std::log(9.223377e+18)); + int misses = 0; + for (int i = 0; i < 1000000; ++i) { + double d = std::exp(uniform(gen)); + VerifySameAsMul(d, &misses); + VerifySameAsMul(-d, &misses); + } +} + +TEST(Duration, ConversionSaturation) { + absl::Duration d; + + const auto max_timeval_sec = + std::numeric_limits<decltype(timeval::tv_sec)>::max(); + const auto min_timeval_sec = + std::numeric_limits<decltype(timeval::tv_sec)>::min(); + timeval tv; + tv.tv_sec = max_timeval_sec; + tv.tv_usec = 999998; + d = absl::DurationFromTimeval(tv); + tv = ToTimeval(d); + EXPECT_EQ(max_timeval_sec, tv.tv_sec); + EXPECT_EQ(999998, tv.tv_usec); + d += absl::Microseconds(1); + tv = ToTimeval(d); + EXPECT_EQ(max_timeval_sec, tv.tv_sec); + EXPECT_EQ(999999, tv.tv_usec); + d += absl::Microseconds(1); // no effect + tv = ToTimeval(d); + EXPECT_EQ(max_timeval_sec, tv.tv_sec); + EXPECT_EQ(999999, tv.tv_usec); + + tv.tv_sec = min_timeval_sec; + tv.tv_usec = 1; + d = absl::DurationFromTimeval(tv); + tv = ToTimeval(d); + EXPECT_EQ(min_timeval_sec, tv.tv_sec); + EXPECT_EQ(1, tv.tv_usec); + d -= absl::Microseconds(1); + tv = ToTimeval(d); + EXPECT_EQ(min_timeval_sec, tv.tv_sec); + EXPECT_EQ(0, tv.tv_usec); + d -= absl::Microseconds(1); // no effect + tv = ToTimeval(d); + EXPECT_EQ(min_timeval_sec, tv.tv_sec); + EXPECT_EQ(0, tv.tv_usec); + + const auto max_timespec_sec = + std::numeric_limits<decltype(timespec::tv_sec)>::max(); + const auto min_timespec_sec = + std::numeric_limits<decltype(timespec::tv_sec)>::min(); + timespec ts; + ts.tv_sec = max_timespec_sec; + ts.tv_nsec = 999999998; + d = absl::DurationFromTimespec(ts); + ts = absl::ToTimespec(d); + EXPECT_EQ(max_timespec_sec, ts.tv_sec); + EXPECT_EQ(999999998, ts.tv_nsec); + d += absl::Nanoseconds(1); + ts = absl::ToTimespec(d); + EXPECT_EQ(max_timespec_sec, ts.tv_sec); + EXPECT_EQ(999999999, ts.tv_nsec); + d += absl::Nanoseconds(1); // no effect + ts = absl::ToTimespec(d); + EXPECT_EQ(max_timespec_sec, ts.tv_sec); + EXPECT_EQ(999999999, ts.tv_nsec); + + ts.tv_sec = min_timespec_sec; + ts.tv_nsec = 1; + d = absl::DurationFromTimespec(ts); + ts = absl::ToTimespec(d); + EXPECT_EQ(min_timespec_sec, ts.tv_sec); + EXPECT_EQ(1, ts.tv_nsec); + d -= absl::Nanoseconds(1); + ts = absl::ToTimespec(d); + EXPECT_EQ(min_timespec_sec, ts.tv_sec); + EXPECT_EQ(0, ts.tv_nsec); + d -= absl::Nanoseconds(1); // no effect + ts = absl::ToTimespec(d); + EXPECT_EQ(min_timespec_sec, ts.tv_sec); + EXPECT_EQ(0, ts.tv_nsec); +} + +TEST(Duration, FormatDuration) { + // Example from Go's docs. + EXPECT_EQ("72h3m0.5s", + absl::FormatDuration(absl::Hours(72) + absl::Minutes(3) + + absl::Milliseconds(500))); + // Go's largest time: 2540400h10m10.000000000s + EXPECT_EQ("2540400h10m10s", + absl::FormatDuration(absl::Hours(2540400) + absl::Minutes(10) + + absl::Seconds(10))); + + EXPECT_EQ("0", absl::FormatDuration(absl::ZeroDuration())); + EXPECT_EQ("0", absl::FormatDuration(absl::Seconds(0))); + EXPECT_EQ("0", absl::FormatDuration(absl::Nanoseconds(0))); + + EXPECT_EQ("1ns", absl::FormatDuration(absl::Nanoseconds(1))); + EXPECT_EQ("1us", absl::FormatDuration(absl::Microseconds(1))); + EXPECT_EQ("1ms", absl::FormatDuration(absl::Milliseconds(1))); + EXPECT_EQ("1s", absl::FormatDuration(absl::Seconds(1))); + EXPECT_EQ("1m", absl::FormatDuration(absl::Minutes(1))); + EXPECT_EQ("1h", absl::FormatDuration(absl::Hours(1))); + + EXPECT_EQ("1h1m", absl::FormatDuration(absl::Hours(1) + absl::Minutes(1))); + EXPECT_EQ("1h1s", absl::FormatDuration(absl::Hours(1) + absl::Seconds(1))); + EXPECT_EQ("1m1s", absl::FormatDuration(absl::Minutes(1) + absl::Seconds(1))); + + EXPECT_EQ("1h0.25s", + absl::FormatDuration(absl::Hours(1) + absl::Milliseconds(250))); + EXPECT_EQ("1m0.25s", + absl::FormatDuration(absl::Minutes(1) + absl::Milliseconds(250))); + EXPECT_EQ("1h1m0.25s", + absl::FormatDuration(absl::Hours(1) + absl::Minutes(1) + + absl::Milliseconds(250))); + EXPECT_EQ("1h0.0005s", + absl::FormatDuration(absl::Hours(1) + absl::Microseconds(500))); + EXPECT_EQ("1h0.0000005s", + absl::FormatDuration(absl::Hours(1) + absl::Nanoseconds(500))); + + // Subsecond special case. + EXPECT_EQ("1.5ns", absl::FormatDuration(absl::Nanoseconds(1) + + absl::Nanoseconds(1) / 2)); + EXPECT_EQ("1.25ns", absl::FormatDuration(absl::Nanoseconds(1) + + absl::Nanoseconds(1) / 4)); + EXPECT_EQ("1ns", absl::FormatDuration(absl::Nanoseconds(1) + + absl::Nanoseconds(1) / 9)); + EXPECT_EQ("1.2us", absl::FormatDuration(absl::Microseconds(1) + + absl::Nanoseconds(200))); + EXPECT_EQ("1.2ms", absl::FormatDuration(absl::Milliseconds(1) + + absl::Microseconds(200))); + EXPECT_EQ("1.0002ms", absl::FormatDuration(absl::Milliseconds(1) + + absl::Nanoseconds(200))); + EXPECT_EQ("1.00001ms", absl::FormatDuration(absl::Milliseconds(1) + + absl::Nanoseconds(10))); + EXPECT_EQ("1.000001ms", + absl::FormatDuration(absl::Milliseconds(1) + absl::Nanoseconds(1))); + + // Negative durations. + EXPECT_EQ("-1ns", absl::FormatDuration(absl::Nanoseconds(-1))); + EXPECT_EQ("-1us", absl::FormatDuration(absl::Microseconds(-1))); + EXPECT_EQ("-1ms", absl::FormatDuration(absl::Milliseconds(-1))); + EXPECT_EQ("-1s", absl::FormatDuration(absl::Seconds(-1))); + EXPECT_EQ("-1m", absl::FormatDuration(absl::Minutes(-1))); + EXPECT_EQ("-1h", absl::FormatDuration(absl::Hours(-1))); + + EXPECT_EQ("-1h1m", + absl::FormatDuration(-(absl::Hours(1) + absl::Minutes(1)))); + EXPECT_EQ("-1h1s", + absl::FormatDuration(-(absl::Hours(1) + absl::Seconds(1)))); + EXPECT_EQ("-1m1s", + absl::FormatDuration(-(absl::Minutes(1) + absl::Seconds(1)))); + + EXPECT_EQ("-1ns", absl::FormatDuration(absl::Nanoseconds(-1))); + EXPECT_EQ("-1.2us", absl::FormatDuration( + -(absl::Microseconds(1) + absl::Nanoseconds(200)))); + EXPECT_EQ("-1.2ms", absl::FormatDuration( + -(absl::Milliseconds(1) + absl::Microseconds(200)))); + EXPECT_EQ("-1.0002ms", absl::FormatDuration(-(absl::Milliseconds(1) + + absl::Nanoseconds(200)))); + EXPECT_EQ("-1.00001ms", absl::FormatDuration(-(absl::Milliseconds(1) + + absl::Nanoseconds(10)))); + EXPECT_EQ("-1.000001ms", absl::FormatDuration(-(absl::Milliseconds(1) + + absl::Nanoseconds(1)))); + + // + // Interesting corner cases. + // + + const absl::Duration qns = absl::Nanoseconds(1) / 4; + const absl::Duration max_dur = + absl::Seconds(kint64max) + (absl::Seconds(1) - qns); + const absl::Duration min_dur = absl::Seconds(kint64min); + + EXPECT_EQ("0.25ns", absl::FormatDuration(qns)); + EXPECT_EQ("-0.25ns", absl::FormatDuration(-qns)); + EXPECT_EQ("2562047788015215h30m7.99999999975s", + absl::FormatDuration(max_dur)); + EXPECT_EQ("-2562047788015215h30m8s", absl::FormatDuration(min_dur)); + + // Tests printing full precision from units that print using FDivDuration + EXPECT_EQ("55.00000000025s", absl::FormatDuration(absl::Seconds(55) + qns)); + EXPECT_EQ("55.00000025ms", + absl::FormatDuration(absl::Milliseconds(55) + qns)); + EXPECT_EQ("55.00025us", absl::FormatDuration(absl::Microseconds(55) + qns)); + EXPECT_EQ("55.25ns", absl::FormatDuration(absl::Nanoseconds(55) + qns)); + + // Formatting infinity + EXPECT_EQ("inf", absl::FormatDuration(absl::InfiniteDuration())); + EXPECT_EQ("-inf", absl::FormatDuration(-absl::InfiniteDuration())); + + // Formatting approximately +/- 100 billion years + const absl::Duration huge_range = ApproxYears(100000000000); + EXPECT_EQ("876000000000000h", absl::FormatDuration(huge_range)); + EXPECT_EQ("-876000000000000h", absl::FormatDuration(-huge_range)); + + EXPECT_EQ("876000000000000h0.999999999s", + absl::FormatDuration(huge_range + + (absl::Seconds(1) - absl::Nanoseconds(1)))); + EXPECT_EQ("876000000000000h0.9999999995s", + absl::FormatDuration( + huge_range + (absl::Seconds(1) - absl::Nanoseconds(1) / 2))); + EXPECT_EQ("876000000000000h0.99999999975s", + absl::FormatDuration( + huge_range + (absl::Seconds(1) - absl::Nanoseconds(1) / 4))); + + EXPECT_EQ("-876000000000000h0.999999999s", + absl::FormatDuration(-huge_range - + (absl::Seconds(1) - absl::Nanoseconds(1)))); + EXPECT_EQ("-876000000000000h0.9999999995s", + absl::FormatDuration( + -huge_range - (absl::Seconds(1) - absl::Nanoseconds(1) / 2))); + EXPECT_EQ("-876000000000000h0.99999999975s", + absl::FormatDuration( + -huge_range - (absl::Seconds(1) - absl::Nanoseconds(1) / 4))); +} + +TEST(Duration, ParseDuration) { + absl::Duration d; + + // No specified unit. Should only work for zero and infinity. + EXPECT_TRUE(absl::ParseDuration("0", &d)); + EXPECT_EQ(absl::ZeroDuration(), d); + EXPECT_TRUE(absl::ParseDuration("+0", &d)); + EXPECT_EQ(absl::ZeroDuration(), d); + EXPECT_TRUE(absl::ParseDuration("-0", &d)); + EXPECT_EQ(absl::ZeroDuration(), d); + + EXPECT_TRUE(absl::ParseDuration("inf", &d)); + EXPECT_EQ(absl::InfiniteDuration(), d); + EXPECT_TRUE(absl::ParseDuration("+inf", &d)); + EXPECT_EQ(absl::InfiniteDuration(), d); + EXPECT_TRUE(absl::ParseDuration("-inf", &d)); + EXPECT_EQ(-absl::InfiniteDuration(), d); + EXPECT_FALSE(absl::ParseDuration("infBlah", &d)); + + // Illegal input forms. + EXPECT_FALSE(absl::ParseDuration("", &d)); + EXPECT_FALSE(absl::ParseDuration("0.0", &d)); + EXPECT_FALSE(absl::ParseDuration(".0", &d)); + EXPECT_FALSE(absl::ParseDuration(".", &d)); + EXPECT_FALSE(absl::ParseDuration("01", &d)); + EXPECT_FALSE(absl::ParseDuration("1", &d)); + EXPECT_FALSE(absl::ParseDuration("-1", &d)); + EXPECT_FALSE(absl::ParseDuration("2", &d)); + EXPECT_FALSE(absl::ParseDuration("2 s", &d)); + EXPECT_FALSE(absl::ParseDuration(".s", &d)); + EXPECT_FALSE(absl::ParseDuration("-.s", &d)); + EXPECT_FALSE(absl::ParseDuration("s", &d)); + EXPECT_FALSE(absl::ParseDuration(" 2s", &d)); + EXPECT_FALSE(absl::ParseDuration("2s ", &d)); + EXPECT_FALSE(absl::ParseDuration(" 2s ", &d)); + EXPECT_FALSE(absl::ParseDuration("2mt", &d)); + EXPECT_FALSE(absl::ParseDuration("1e3s", &d)); + + // One unit type. + EXPECT_TRUE(absl::ParseDuration("1ns", &d)); + EXPECT_EQ(absl::Nanoseconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1us", &d)); + EXPECT_EQ(absl::Microseconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1ms", &d)); + EXPECT_EQ(absl::Milliseconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1s", &d)); + EXPECT_EQ(absl::Seconds(1), d); + EXPECT_TRUE(absl::ParseDuration("2m", &d)); + EXPECT_EQ(absl::Minutes(2), d); + EXPECT_TRUE(absl::ParseDuration("2h", &d)); + EXPECT_EQ(absl::Hours(2), d); + + // Huge counts of a unit. + EXPECT_TRUE(absl::ParseDuration("9223372036854775807us", &d)); + EXPECT_EQ(absl::Microseconds(9223372036854775807), d); + EXPECT_TRUE(absl::ParseDuration("-9223372036854775807us", &d)); + EXPECT_EQ(absl::Microseconds(-9223372036854775807), d); + + // Multiple units. + EXPECT_TRUE(absl::ParseDuration("2h3m4s", &d)); + EXPECT_EQ(absl::Hours(2) + absl::Minutes(3) + absl::Seconds(4), d); + EXPECT_TRUE(absl::ParseDuration("3m4s5us", &d)); + EXPECT_EQ(absl::Minutes(3) + absl::Seconds(4) + absl::Microseconds(5), d); + EXPECT_TRUE(absl::ParseDuration("2h3m4s5ms6us7ns", &d)); + EXPECT_EQ(absl::Hours(2) + absl::Minutes(3) + absl::Seconds(4) + + absl::Milliseconds(5) + absl::Microseconds(6) + + absl::Nanoseconds(7), + d); + + // Multiple units out of order. + EXPECT_TRUE(absl::ParseDuration("2us3m4s5h", &d)); + EXPECT_EQ(absl::Hours(5) + absl::Minutes(3) + absl::Seconds(4) + + absl::Microseconds(2), + d); + + // Fractional values of units. + EXPECT_TRUE(absl::ParseDuration("1.5ns", &d)); + EXPECT_EQ(1.5 * absl::Nanoseconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1.5us", &d)); + EXPECT_EQ(1.5 * absl::Microseconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1.5ms", &d)); + EXPECT_EQ(1.5 * absl::Milliseconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1.5s", &d)); + EXPECT_EQ(1.5 * absl::Seconds(1), d); + EXPECT_TRUE(absl::ParseDuration("1.5m", &d)); + EXPECT_EQ(1.5 * absl::Minutes(1), d); + EXPECT_TRUE(absl::ParseDuration("1.5h", &d)); + EXPECT_EQ(1.5 * absl::Hours(1), d); + + // Huge fractional counts of a unit. + EXPECT_TRUE(absl::ParseDuration("0.4294967295s", &d)); + EXPECT_EQ(absl::Nanoseconds(429496729) + absl::Nanoseconds(1) / 2, d); + EXPECT_TRUE(absl::ParseDuration("0.429496729501234567890123456789s", &d)); + EXPECT_EQ(absl::Nanoseconds(429496729) + absl::Nanoseconds(1) / 2, d); + + // Negative durations. + EXPECT_TRUE(absl::ParseDuration("-1s", &d)); + EXPECT_EQ(absl::Seconds(-1), d); + EXPECT_TRUE(absl::ParseDuration("-1m", &d)); + EXPECT_EQ(absl::Minutes(-1), d); + EXPECT_TRUE(absl::ParseDuration("-1h", &d)); + EXPECT_EQ(absl::Hours(-1), d); + + EXPECT_TRUE(absl::ParseDuration("-1h2s", &d)); + EXPECT_EQ(-(absl::Hours(1) + absl::Seconds(2)), d); + EXPECT_FALSE(absl::ParseDuration("1h-2s", &d)); + EXPECT_FALSE(absl::ParseDuration("-1h-2s", &d)); + EXPECT_FALSE(absl::ParseDuration("-1h -2s", &d)); +} + +TEST(Duration, FormatParseRoundTrip) { +#define TEST_PARSE_ROUNDTRIP(d) \ + do { \ + std::string s = absl::FormatDuration(d); \ + absl::Duration dur; \ + EXPECT_TRUE(absl::ParseDuration(s, &dur)); \ + EXPECT_EQ(d, dur); \ + } while (0) + + TEST_PARSE_ROUNDTRIP(absl::Nanoseconds(1)); + TEST_PARSE_ROUNDTRIP(absl::Microseconds(1)); + TEST_PARSE_ROUNDTRIP(absl::Milliseconds(1)); + TEST_PARSE_ROUNDTRIP(absl::Seconds(1)); + TEST_PARSE_ROUNDTRIP(absl::Minutes(1)); + TEST_PARSE_ROUNDTRIP(absl::Hours(1)); + TEST_PARSE_ROUNDTRIP(absl::Hours(1) + absl::Nanoseconds(2)); + + TEST_PARSE_ROUNDTRIP(absl::Nanoseconds(-1)); + TEST_PARSE_ROUNDTRIP(absl::Microseconds(-1)); + TEST_PARSE_ROUNDTRIP(absl::Milliseconds(-1)); + TEST_PARSE_ROUNDTRIP(absl::Seconds(-1)); + TEST_PARSE_ROUNDTRIP(absl::Minutes(-1)); + TEST_PARSE_ROUNDTRIP(absl::Hours(-1)); + + TEST_PARSE_ROUNDTRIP(absl::Hours(-1) + absl::Nanoseconds(2)); + TEST_PARSE_ROUNDTRIP(absl::Hours(1) + absl::Nanoseconds(-2)); + TEST_PARSE_ROUNDTRIP(absl::Hours(-1) + absl::Nanoseconds(-2)); + + TEST_PARSE_ROUNDTRIP(absl::Nanoseconds(1) + + absl::Nanoseconds(1) / 4); // 1.25ns + + const absl::Duration huge_range = ApproxYears(100000000000); + TEST_PARSE_ROUNDTRIP(huge_range); + TEST_PARSE_ROUNDTRIP(huge_range + (absl::Seconds(1) - absl::Nanoseconds(1))); + +#undef TEST_PARSE_ROUNDTRIP +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/format.cc b/third_party/abseil_cpp/absl/time/format.cc new file mode 100644 index 000000000000..4005fb704cf6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/format.cc @@ -0,0 +1,160 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <string.h> + +#include <cctype> +#include <cstdint> + +#include "absl/strings/match.h" +#include "absl/strings/string_view.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" +#include "absl/time/time.h" + +namespace cctz = absl::time_internal::cctz; + +namespace absl { +ABSL_NAMESPACE_BEGIN + +ABSL_DLL extern const char RFC3339_full[] = "%Y-%m-%d%ET%H:%M:%E*S%Ez"; +ABSL_DLL extern const char RFC3339_sec[] = "%Y-%m-%d%ET%H:%M:%S%Ez"; + +ABSL_DLL extern const char RFC1123_full[] = "%a, %d %b %E4Y %H:%M:%S %z"; +ABSL_DLL extern const char RFC1123_no_wday[] = "%d %b %E4Y %H:%M:%S %z"; + +namespace { + +const char kInfiniteFutureStr[] = "infinite-future"; +const char kInfinitePastStr[] = "infinite-past"; + +struct cctz_parts { + cctz::time_point<cctz::seconds> sec; + cctz::detail::femtoseconds fem; +}; + +inline cctz::time_point<cctz::seconds> unix_epoch() { + return std::chrono::time_point_cast<cctz::seconds>( + std::chrono::system_clock::from_time_t(0)); +} + +// Splits a Time into seconds and femtoseconds, which can be used with CCTZ. +// Requires that 't' is finite. See duration.cc for details about rep_hi and +// rep_lo. +cctz_parts Split(absl::Time t) { + const auto d = time_internal::ToUnixDuration(t); + const int64_t rep_hi = time_internal::GetRepHi(d); + const int64_t rep_lo = time_internal::GetRepLo(d); + const auto sec = unix_epoch() + cctz::seconds(rep_hi); + const auto fem = cctz::detail::femtoseconds(rep_lo * (1000 * 1000 / 4)); + return {sec, fem}; +} + +// Joins the given seconds and femtoseconds into a Time. See duration.cc for +// details about rep_hi and rep_lo. +absl::Time Join(const cctz_parts& parts) { + const int64_t rep_hi = (parts.sec - unix_epoch()).count(); + const uint32_t rep_lo = parts.fem.count() / (1000 * 1000 / 4); + const auto d = time_internal::MakeDuration(rep_hi, rep_lo); + return time_internal::FromUnixDuration(d); +} + +} // namespace + +std::string FormatTime(absl::string_view format, absl::Time t, + absl::TimeZone tz) { + if (t == absl::InfiniteFuture()) return std::string(kInfiniteFutureStr); + if (t == absl::InfinitePast()) return std::string(kInfinitePastStr); + const auto parts = Split(t); + return cctz::detail::format(std::string(format), parts.sec, parts.fem, + cctz::time_zone(tz)); +} + +std::string FormatTime(absl::Time t, absl::TimeZone tz) { + return FormatTime(RFC3339_full, t, tz); +} + +std::string FormatTime(absl::Time t) { + return absl::FormatTime(RFC3339_full, t, absl::LocalTimeZone()); +} + +bool ParseTime(absl::string_view format, absl::string_view input, + absl::Time* time, std::string* err) { + return absl::ParseTime(format, input, absl::UTCTimeZone(), time, err); +} + +// If the input string does not contain an explicit UTC offset, interpret +// the fields with respect to the given TimeZone. +bool ParseTime(absl::string_view format, absl::string_view input, + absl::TimeZone tz, absl::Time* time, std::string* err) { + auto strip_leading_space = [](absl::string_view* sv) { + while (!sv->empty()) { + if (!std::isspace(sv->front())) return; + sv->remove_prefix(1); + } + }; + + // Portable toolchains means we don't get nice constexpr here. + struct Literal { + const char* name; + size_t size; + absl::Time value; + }; + static Literal literals[] = { + {kInfiniteFutureStr, strlen(kInfiniteFutureStr), InfiniteFuture()}, + {kInfinitePastStr, strlen(kInfinitePastStr), InfinitePast()}, + }; + strip_leading_space(&input); + for (const auto& lit : literals) { + if (absl::StartsWith(input, absl::string_view(lit.name, lit.size))) { + absl::string_view tail = input; + tail.remove_prefix(lit.size); + strip_leading_space(&tail); + if (tail.empty()) { + *time = lit.value; + return true; + } + } + } + + std::string error; + cctz_parts parts; + const bool b = + cctz::detail::parse(std::string(format), std::string(input), + cctz::time_zone(tz), &parts.sec, &parts.fem, &error); + if (b) { + *time = Join(parts); + } else if (err != nullptr) { + *err = error; + } + return b; +} + +// Functions required to support absl::Time flags. +bool AbslParseFlag(absl::string_view text, absl::Time* t, std::string* error) { + return absl::ParseTime(RFC3339_full, text, absl::UTCTimeZone(), t, error); +} + +std::string AbslUnparseFlag(absl::Time t) { + return absl::FormatTime(RFC3339_full, t, absl::UTCTimeZone()); +} +bool ParseFlag(const std::string& text, absl::Time* t, std::string* error) { + return absl::ParseTime(RFC3339_full, text, absl::UTCTimeZone(), t, error); +} + +std::string UnparseFlag(absl::Time t) { + return absl::FormatTime(RFC3339_full, t, absl::UTCTimeZone()); +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/format_benchmark.cc b/third_party/abseil_cpp/absl/time/format_benchmark.cc new file mode 100644 index 000000000000..19e481dbd1a5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/format_benchmark.cc @@ -0,0 +1,64 @@ +// Copyright 2018 The Abseil Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstddef> +#include <string> + +#include "absl/time/internal/test_util.h" +#include "absl/time/time.h" +#include "benchmark/benchmark.h" + +namespace { + +namespace { +const char* const kFormats[] = { + absl::RFC1123_full, // 0 + absl::RFC1123_no_wday, // 1 + absl::RFC3339_full, // 2 + absl::RFC3339_sec, // 3 + "%Y-%m-%d%ET%H:%M:%S", // 4 + "%Y-%m-%d", // 5 +}; +const int kNumFormats = sizeof(kFormats) / sizeof(kFormats[0]); +} // namespace + +void BM_Format_FormatTime(benchmark::State& state) { + const std::string fmt = kFormats[state.range(0)]; + state.SetLabel(fmt); + const absl::TimeZone lax = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + const absl::Time t = + absl::FromCivil(absl::CivilSecond(1977, 6, 28, 9, 8, 7), lax) + + absl::Nanoseconds(1); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::FormatTime(fmt, t, lax).length()); + } +} +BENCHMARK(BM_Format_FormatTime)->DenseRange(0, kNumFormats - 1); + +void BM_Format_ParseTime(benchmark::State& state) { + const std::string fmt = kFormats[state.range(0)]; + state.SetLabel(fmt); + const absl::TimeZone lax = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + absl::Time t = absl::FromCivil(absl::CivilSecond(1977, 6, 28, 9, 8, 7), lax) + + absl::Nanoseconds(1); + const std::string when = absl::FormatTime(fmt, t, lax); + std::string err; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ParseTime(fmt, when, lax, &t, &err)); + } +} +BENCHMARK(BM_Format_ParseTime)->DenseRange(0, kNumFormats - 1); + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/format_test.cc b/third_party/abseil_cpp/absl/time/format_test.cc new file mode 100644 index 000000000000..a9a1eb8ee1d6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/format_test.cc @@ -0,0 +1,441 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <limits> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/time/internal/test_util.h" +#include "absl/time/time.h" + +using testing::HasSubstr; + +namespace { + +// A helper that tests the given format specifier by itself, and with leading +// and trailing characters. For example: TestFormatSpecifier(t, "%a", "Thu"). +void TestFormatSpecifier(absl::Time t, absl::TimeZone tz, + const std::string& fmt, const std::string& ans) { + EXPECT_EQ(ans, absl::FormatTime(fmt, t, tz)); + EXPECT_EQ("xxx " + ans, absl::FormatTime("xxx " + fmt, t, tz)); + EXPECT_EQ(ans + " yyy", absl::FormatTime(fmt + " yyy", t, tz)); + EXPECT_EQ("xxx " + ans + " yyy", + absl::FormatTime("xxx " + fmt + " yyy", t, tz)); +} + +// +// Testing FormatTime() +// + +TEST(FormatTime, Basics) { + absl::TimeZone tz = absl::UTCTimeZone(); + absl::Time t = absl::FromTimeT(0); + + // Starts with a couple basic edge cases. + EXPECT_EQ("", absl::FormatTime("", t, tz)); + EXPECT_EQ(" ", absl::FormatTime(" ", t, tz)); + EXPECT_EQ(" ", absl::FormatTime(" ", t, tz)); + EXPECT_EQ("xxx", absl::FormatTime("xxx", t, tz)); + std::string big(128, 'x'); + EXPECT_EQ(big, absl::FormatTime(big, t, tz)); + // Cause the 1024-byte buffer to grow. + std::string bigger(100000, 'x'); + EXPECT_EQ(bigger, absl::FormatTime(bigger, t, tz)); + + t += absl::Hours(13) + absl::Minutes(4) + absl::Seconds(5); + t += absl::Milliseconds(6) + absl::Microseconds(7) + absl::Nanoseconds(8); + EXPECT_EQ("1970-01-01", absl::FormatTime("%Y-%m-%d", t, tz)); + EXPECT_EQ("13:04:05", absl::FormatTime("%H:%M:%S", t, tz)); + EXPECT_EQ("13:04:05.006", absl::FormatTime("%H:%M:%E3S", t, tz)); + EXPECT_EQ("13:04:05.006007", absl::FormatTime("%H:%M:%E6S", t, tz)); + EXPECT_EQ("13:04:05.006007008", absl::FormatTime("%H:%M:%E9S", t, tz)); +} + +TEST(FormatTime, LocaleSpecific) { + const absl::TimeZone tz = absl::UTCTimeZone(); + absl::Time t = absl::FromTimeT(0); + + TestFormatSpecifier(t, tz, "%a", "Thu"); + TestFormatSpecifier(t, tz, "%A", "Thursday"); + TestFormatSpecifier(t, tz, "%b", "Jan"); + TestFormatSpecifier(t, tz, "%B", "January"); + + // %c should at least produce the numeric year and time-of-day. + const std::string s = + absl::FormatTime("%c", absl::FromTimeT(0), absl::UTCTimeZone()); + EXPECT_THAT(s, HasSubstr("1970")); + EXPECT_THAT(s, HasSubstr("00:00:00")); + + TestFormatSpecifier(t, tz, "%p", "AM"); + TestFormatSpecifier(t, tz, "%x", "01/01/70"); + TestFormatSpecifier(t, tz, "%X", "00:00:00"); +} + +TEST(FormatTime, ExtendedSeconds) { + const absl::TimeZone tz = absl::UTCTimeZone(); + + // No subseconds. + absl::Time t = absl::FromTimeT(0) + absl::Seconds(5); + EXPECT_EQ("05", absl::FormatTime("%E*S", t, tz)); + EXPECT_EQ("05.000000000000000", absl::FormatTime("%E15S", t, tz)); + + // With subseconds. + t += absl::Milliseconds(6) + absl::Microseconds(7) + absl::Nanoseconds(8); + EXPECT_EQ("05.006007008", absl::FormatTime("%E*S", t, tz)); + EXPECT_EQ("05", absl::FormatTime("%E0S", t, tz)); + EXPECT_EQ("05.006007008000000", absl::FormatTime("%E15S", t, tz)); + + // Times before the Unix epoch. + t = absl::FromUnixMicros(-1); + EXPECT_EQ("1969-12-31 23:59:59.999999", + absl::FormatTime("%Y-%m-%d %H:%M:%E*S", t, tz)); + + // Here is a "%E*S" case we got wrong for a while. While the first + // instant below is correctly rendered as "...:07.333304", the second + // one used to appear as "...:07.33330499999999999". + t = absl::FromUnixMicros(1395024427333304); + EXPECT_EQ("2014-03-17 02:47:07.333304", + absl::FormatTime("%Y-%m-%d %H:%M:%E*S", t, tz)); + t += absl::Microseconds(1); + EXPECT_EQ("2014-03-17 02:47:07.333305", + absl::FormatTime("%Y-%m-%d %H:%M:%E*S", t, tz)); +} + +TEST(FormatTime, RFC1123FormatPadsYear) { // locale specific + absl::TimeZone tz = absl::UTCTimeZone(); + + // A year of 77 should be padded to 0077. + absl::Time t = absl::FromCivil(absl::CivilSecond(77, 6, 28, 9, 8, 7), tz); + EXPECT_EQ("Mon, 28 Jun 0077 09:08:07 +0000", + absl::FormatTime(absl::RFC1123_full, t, tz)); + EXPECT_EQ("28 Jun 0077 09:08:07 +0000", + absl::FormatTime(absl::RFC1123_no_wday, t, tz)); +} + +TEST(FormatTime, InfiniteTime) { + absl::TimeZone tz = absl::time_internal::LoadTimeZone("America/Los_Angeles"); + + // The format and timezone are ignored. + EXPECT_EQ("infinite-future", + absl::FormatTime("%H:%M blah", absl::InfiniteFuture(), tz)); + EXPECT_EQ("infinite-past", + absl::FormatTime("%H:%M blah", absl::InfinitePast(), tz)); +} + +// +// Testing ParseTime() +// + +TEST(ParseTime, Basics) { + absl::Time t = absl::FromTimeT(1234567890); + std::string err; + + // Simple edge cases. + EXPECT_TRUE(absl::ParseTime("", "", &t, &err)) << err; + EXPECT_EQ(absl::UnixEpoch(), t); // everything defaulted + EXPECT_TRUE(absl::ParseTime(" ", " ", &t, &err)) << err; + EXPECT_TRUE(absl::ParseTime(" ", " ", &t, &err)) << err; + EXPECT_TRUE(absl::ParseTime("x", "x", &t, &err)) << err; + EXPECT_TRUE(absl::ParseTime("xxx", "xxx", &t, &err)) << err; + + EXPECT_TRUE(absl::ParseTime("%Y-%m-%d %H:%M:%S %z", + "2013-06-28 19:08:09 -0800", &t, &err)) + << err; + const auto ci = absl::FixedTimeZone(-8 * 60 * 60).At(t); + EXPECT_EQ(absl::CivilSecond(2013, 6, 28, 19, 8, 9), ci.cs); + EXPECT_EQ(absl::ZeroDuration(), ci.subsecond); +} + +TEST(ParseTime, NullErrorString) { + absl::Time t; + EXPECT_FALSE(absl::ParseTime("%Q", "invalid format", &t, nullptr)); + EXPECT_FALSE(absl::ParseTime("%H", "12 trailing data", &t, nullptr)); + EXPECT_FALSE( + absl::ParseTime("%H out of range", "42 out of range", &t, nullptr)); +} + +TEST(ParseTime, WithTimeZone) { + const absl::TimeZone tz = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + absl::Time t; + std::string e; + + // We can parse a string without a UTC offset if we supply a timezone. + EXPECT_TRUE( + absl::ParseTime("%Y-%m-%d %H:%M:%S", "2013-06-28 19:08:09", tz, &t, &e)) + << e; + auto ci = tz.At(t); + EXPECT_EQ(absl::CivilSecond(2013, 6, 28, 19, 8, 9), ci.cs); + EXPECT_EQ(absl::ZeroDuration(), ci.subsecond); + + // But the timezone is ignored when a UTC offset is present. + EXPECT_TRUE(absl::ParseTime("%Y-%m-%d %H:%M:%S %z", + "2013-06-28 19:08:09 +0800", tz, &t, &e)) + << e; + ci = absl::FixedTimeZone(8 * 60 * 60).At(t); + EXPECT_EQ(absl::CivilSecond(2013, 6, 28, 19, 8, 9), ci.cs); + EXPECT_EQ(absl::ZeroDuration(), ci.subsecond); +} + +TEST(ParseTime, ErrorCases) { + absl::Time t = absl::FromTimeT(0); + std::string err; + + EXPECT_FALSE(absl::ParseTime("%S", "123", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); + + // Can't parse an illegal format specifier. + err.clear(); + EXPECT_FALSE(absl::ParseTime("%Q", "x", &t, &err)) << err; + // Exact contents of "err" are platform-dependent because of + // differences in the strptime implementation between macOS and Linux. + EXPECT_FALSE(err.empty()); + + // Fails because of trailing, unparsed data "blah". + EXPECT_FALSE(absl::ParseTime("%m-%d", "2-3 blah", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); + + // Feb 31 requires normalization. + EXPECT_FALSE(absl::ParseTime("%m-%d", "2-31", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Out-of-range")); + + // Check that we cannot have spaces in UTC offsets. + EXPECT_TRUE(absl::ParseTime("%z", "-0203", &t, &err)) << err; + EXPECT_FALSE(absl::ParseTime("%z", "- 2 3", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_TRUE(absl::ParseTime("%Ez", "-02:03", &t, &err)) << err; + EXPECT_FALSE(absl::ParseTime("%Ez", "- 2: 3", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + + // Check that we reject other malformed UTC offsets. + EXPECT_FALSE(absl::ParseTime("%Ez", "+-08:00", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%Ez", "-+08:00", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + + // Check that we do not accept "-0" in fields that allow zero. + EXPECT_FALSE(absl::ParseTime("%Y", "-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%E4Y", "-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%H", "-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%M", "-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%S", "-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%z", "+-000", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%Ez", "+-0:00", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + EXPECT_FALSE(absl::ParseTime("%z", "-00-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); + EXPECT_FALSE(absl::ParseTime("%Ez", "-00:-0", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); +} + +TEST(ParseTime, ExtendedSeconds) { + std::string err; + absl::Time t; + + // Here is a "%E*S" case we got wrong for a while. The fractional + // part of the first instant is less than 2^31 and was correctly + // parsed, while the second (and any subsecond field >=2^31) failed. + t = absl::UnixEpoch(); + EXPECT_TRUE(absl::ParseTime("%E*S", "0.2147483647", &t, &err)) << err; + EXPECT_EQ(absl::UnixEpoch() + absl::Nanoseconds(214748364) + + absl::Nanoseconds(1) / 2, + t); + t = absl::UnixEpoch(); + EXPECT_TRUE(absl::ParseTime("%E*S", "0.2147483648", &t, &err)) << err; + EXPECT_EQ(absl::UnixEpoch() + absl::Nanoseconds(214748364) + + absl::Nanoseconds(3) / 4, + t); + + // We should also be able to specify long strings of digits far + // beyond the current resolution and have them convert the same way. + t = absl::UnixEpoch(); + EXPECT_TRUE(absl::ParseTime( + "%E*S", "0.214748364801234567890123456789012345678901234567890123456789", + &t, &err)) + << err; + EXPECT_EQ(absl::UnixEpoch() + absl::Nanoseconds(214748364) + + absl::Nanoseconds(3) / 4, + t); +} + +TEST(ParseTime, ExtendedOffsetErrors) { + std::string err; + absl::Time t; + + // %z against +-HHMM. + EXPECT_FALSE(absl::ParseTime("%z", "-123", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); + + // %z against +-HH. + EXPECT_FALSE(absl::ParseTime("%z", "-1", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); + + // %Ez against +-HH:MM. + EXPECT_FALSE(absl::ParseTime("%Ez", "-12:3", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); + + // %Ez against +-HHMM. + EXPECT_FALSE(absl::ParseTime("%Ez", "-123", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Illegal trailing data")); + + // %Ez against +-HH. + EXPECT_FALSE(absl::ParseTime("%Ez", "-1", &t, &err)) << err; + EXPECT_THAT(err, HasSubstr("Failed to parse")); +} + +TEST(ParseTime, InfiniteTime) { + absl::Time t; + std::string err; + EXPECT_TRUE(absl::ParseTime("%H:%M blah", "infinite-future", &t, &err)); + EXPECT_EQ(absl::InfiniteFuture(), t); + + // Surrounding whitespace. + EXPECT_TRUE(absl::ParseTime("%H:%M blah", " infinite-future", &t, &err)); + EXPECT_EQ(absl::InfiniteFuture(), t); + EXPECT_TRUE(absl::ParseTime("%H:%M blah", "infinite-future ", &t, &err)); + EXPECT_EQ(absl::InfiniteFuture(), t); + EXPECT_TRUE(absl::ParseTime("%H:%M blah", " infinite-future ", &t, &err)); + EXPECT_EQ(absl::InfiniteFuture(), t); + + EXPECT_TRUE(absl::ParseTime("%H:%M blah", "infinite-past", &t, &err)); + EXPECT_EQ(absl::InfinitePast(), t); + + // Surrounding whitespace. + EXPECT_TRUE(absl::ParseTime("%H:%M blah", " infinite-past", &t, &err)); + EXPECT_EQ(absl::InfinitePast(), t); + EXPECT_TRUE(absl::ParseTime("%H:%M blah", "infinite-past ", &t, &err)); + EXPECT_EQ(absl::InfinitePast(), t); + EXPECT_TRUE(absl::ParseTime("%H:%M blah", " infinite-past ", &t, &err)); + EXPECT_EQ(absl::InfinitePast(), t); + + // "infinite-future" as literal string + absl::TimeZone tz = absl::UTCTimeZone(); + EXPECT_TRUE(absl::ParseTime("infinite-future %H:%M", "infinite-future 03:04", + &t, &err)); + EXPECT_NE(absl::InfiniteFuture(), t); + EXPECT_EQ(3, tz.At(t).cs.hour()); + EXPECT_EQ(4, tz.At(t).cs.minute()); + + // "infinite-past" as literal string + EXPECT_TRUE( + absl::ParseTime("infinite-past %H:%M", "infinite-past 03:04", &t, &err)); + EXPECT_NE(absl::InfinitePast(), t); + EXPECT_EQ(3, tz.At(t).cs.hour()); + EXPECT_EQ(4, tz.At(t).cs.minute()); + + // The input doesn't match the format. + EXPECT_FALSE(absl::ParseTime("infinite-future %H:%M", "03:04", &t, &err)); + EXPECT_FALSE(absl::ParseTime("infinite-past %H:%M", "03:04", &t, &err)); +} + +TEST(ParseTime, FailsOnUnrepresentableTime) { + const absl::TimeZone utc = absl::UTCTimeZone(); + absl::Time t; + EXPECT_FALSE( + absl::ParseTime("%Y-%m-%d", "-292277022657-01-27", utc, &t, nullptr)); + EXPECT_TRUE( + absl::ParseTime("%Y-%m-%d", "-292277022657-01-28", utc, &t, nullptr)); + EXPECT_TRUE( + absl::ParseTime("%Y-%m-%d", "292277026596-12-04", utc, &t, nullptr)); + EXPECT_FALSE( + absl::ParseTime("%Y-%m-%d", "292277026596-12-05", utc, &t, nullptr)); +} + +// +// Roundtrip test for FormatTime()/ParseTime(). +// + +TEST(FormatParse, RoundTrip) { + const absl::TimeZone lax = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + const absl::Time in = + absl::FromCivil(absl::CivilSecond(1977, 6, 28, 9, 8, 7), lax); + const absl::Duration subseconds = absl::Nanoseconds(654321); + std::string err; + + // RFC3339, which renders subseconds. + { + absl::Time out; + const std::string s = + absl::FormatTime(absl::RFC3339_full, in + subseconds, lax); + EXPECT_TRUE(absl::ParseTime(absl::RFC3339_full, s, &out, &err)) + << s << ": " << err; + EXPECT_EQ(in + subseconds, out); // RFC3339_full includes %Ez + } + + // RFC1123, which only does whole seconds. + { + absl::Time out; + const std::string s = absl::FormatTime(absl::RFC1123_full, in, lax); + EXPECT_TRUE(absl::ParseTime(absl::RFC1123_full, s, &out, &err)) + << s << ": " << err; + EXPECT_EQ(in, out); // RFC1123_full includes %z + } + + // `absl::FormatTime()` falls back to strftime() for "%c", which appears to + // work. On Windows, `absl::ParseTime()` falls back to std::get_time() which + // appears to fail on "%c" (or at least on the "%c" text produced by + // `strftime()`). This makes it fail the round-trip test. + // + // Under the emscripten compiler `absl::ParseTime() falls back to + // `strptime()`, but that ends up using a different definition for "%c" + // compared to `strftime()`, also causing the round-trip test to fail + // (see https://github.com/kripken/emscripten/pull/7491). +#if !defined(_MSC_VER) && !defined(__EMSCRIPTEN__) + // Even though we don't know what %c will produce, it should roundtrip, + // but only in the 0-offset timezone. + { + absl::Time out; + const std::string s = absl::FormatTime("%c", in, absl::UTCTimeZone()); + EXPECT_TRUE(absl::ParseTime("%c", s, &out, &err)) << s << ": " << err; + EXPECT_EQ(in, out); + } +#endif // !_MSC_VER && !__EMSCRIPTEN__ +} + +TEST(FormatParse, RoundTripDistantFuture) { + const absl::TimeZone tz = absl::UTCTimeZone(); + const absl::Time in = + absl::FromUnixSeconds(std::numeric_limits<int64_t>::max()); + std::string err; + + absl::Time out; + const std::string s = absl::FormatTime(absl::RFC3339_full, in, tz); + EXPECT_TRUE(absl::ParseTime(absl::RFC3339_full, s, &out, &err)) + << s << ": " << err; + EXPECT_EQ(in, out); +} + +TEST(FormatParse, RoundTripDistantPast) { + const absl::TimeZone tz = absl::UTCTimeZone(); + const absl::Time in = + absl::FromUnixSeconds(std::numeric_limits<int64_t>::min()); + std::string err; + + absl::Time out; + const std::string s = absl::FormatTime(absl::RFC3339_full, in, tz); + EXPECT_TRUE(absl::ParseTime(absl::RFC3339_full, s, &out, &err)) + << s << ": " << err; + EXPECT_EQ(in, out); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/BUILD.bazel b/third_party/abseil_cpp/absl/time/internal/cctz/BUILD.bazel new file mode 100644 index 000000000000..45a952924d98 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/BUILD.bazel @@ -0,0 +1,171 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") + +package(features = ["-parse_headers"]) + +licenses(["notice"]) + +filegroup( + name = "zoneinfo", + srcs = glob(["testdata/zoneinfo/**"]), +) + +config_setting( + name = "osx", + constraint_values = [ + "@bazel_tools//platforms:osx", + ], +) + +config_setting( + name = "ios", + constraint_values = [ + "@bazel_tools//platforms:ios", + ], +) + +### libraries + +cc_library( + name = "civil_time", + srcs = ["src/civil_time_detail.cc"], + hdrs = [ + "include/cctz/civil_time.h", + ], + textual_hdrs = ["include/cctz/civil_time_detail.h"], + visibility = ["//visibility:public"], + deps = ["//absl/base:config"], +) + +cc_library( + name = "time_zone", + srcs = [ + "src/time_zone_fixed.cc", + "src/time_zone_fixed.h", + "src/time_zone_format.cc", + "src/time_zone_if.cc", + "src/time_zone_if.h", + "src/time_zone_impl.cc", + "src/time_zone_impl.h", + "src/time_zone_info.cc", + "src/time_zone_info.h", + "src/time_zone_libc.cc", + "src/time_zone_libc.h", + "src/time_zone_lookup.cc", + "src/time_zone_posix.cc", + "src/time_zone_posix.h", + "src/tzfile.h", + "src/zone_info_source.cc", + ], + hdrs = [ + "include/cctz/time_zone.h", + "include/cctz/zone_info_source.h", + ], + linkopts = select({ + ":osx": [ + "-framework Foundation", + ], + ":ios": [ + "-framework Foundation", + ], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], + deps = [ + ":civil_time", + "//absl/base:config", + ], +) + +### tests + +test_suite( + name = "all_tests", + visibility = ["//visibility:public"], +) + +cc_test( + name = "civil_time_test", + size = "small", + srcs = ["src/civil_time_test.cc"], + deps = [ + ":civil_time", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "time_zone_format_test", + size = "small", + srcs = ["src/time_zone_format_test.cc"], + data = [":zoneinfo"], + tags = [ + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + ], + deps = [ + ":civil_time", + ":time_zone", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "time_zone_lookup_test", + size = "small", + timeout = "moderate", + srcs = ["src/time_zone_lookup_test.cc"], + data = [":zoneinfo"], + tags = [ + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + ], + deps = [ + ":civil_time", + ":time_zone", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +### benchmarks + +cc_test( + name = "cctz_benchmark", + srcs = [ + "src/cctz_benchmark.cc", + "src/time_zone_if.h", + "src/time_zone_impl.h", + "src/time_zone_info.h", + "src/tzfile.h", + ], + linkstatic = 1, + tags = ["benchmark"], + deps = [ + ":civil_time", + ":time_zone", + "//absl/base:config", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +### examples + +### binaries diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/civil_time.h b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/civil_time.h new file mode 100644 index 000000000000..d47ff86fe43e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/civil_time.h @@ -0,0 +1,332 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_CIVIL_TIME_H_ +#define ABSL_TIME_INTERNAL_CCTZ_CIVIL_TIME_H_ + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time_detail.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// The term "civil time" refers to the legally recognized human-scale time +// that is represented by the six fields YYYY-MM-DD hh:mm:ss. Modern-day civil +// time follows the Gregorian Calendar and is a time-zone-independent concept. +// A "date" is perhaps the most common example of a civil time (represented in +// this library as cctz::civil_day). This library provides six classes and a +// handful of functions that help with rounding, iterating, and arithmetic on +// civil times while avoiding complications like daylight-saving time (DST). +// +// The following six classes form the core of this civil-time library: +// +// * civil_second +// * civil_minute +// * civil_hour +// * civil_day +// * civil_month +// * civil_year +// +// Each class is a simple value type with the same interface for construction +// and the same six accessors for each of the civil fields (year, month, day, +// hour, minute, and second, aka YMDHMS). These classes differ only in their +// alignment, which is indicated by the type name and specifies the field on +// which arithmetic operates. +// +// Each class can be constructed by passing up to six optional integer +// arguments representing the YMDHMS fields (in that order) to the +// constructor. Omitted fields are assigned their minimum valid value. Hours, +// minutes, and seconds will be set to 0, month and day will be set to 1, and +// since there is no minimum valid year, it will be set to 1970. So, a +// default-constructed civil-time object will have YMDHMS fields representing +// "1970-01-01 00:00:00". Fields that are out-of-range are normalized (e.g., +// October 32 -> November 1) so that all civil-time objects represent valid +// values. +// +// Each civil-time class is aligned to the civil-time field indicated in the +// class's name after normalization. Alignment is performed by setting all the +// inferior fields to their minimum valid value (as described above). The +// following are examples of how each of the six types would align the fields +// representing November 22, 2015 at 12:34:56 in the afternoon. (Note: the +// string format used here is not important; it's just a shorthand way of +// showing the six YMDHMS fields.) +// +// civil_second 2015-11-22 12:34:56 +// civil_minute 2015-11-22 12:34:00 +// civil_hour 2015-11-22 12:00:00 +// civil_day 2015-11-22 00:00:00 +// civil_month 2015-11-01 00:00:00 +// civil_year 2015-01-01 00:00:00 +// +// Each civil-time type performs arithmetic on the field to which it is +// aligned. This means that adding 1 to a civil_day increments the day field +// (normalizing as necessary), and subtracting 7 from a civil_month operates +// on the month field (normalizing as necessary). All arithmetic produces a +// valid civil time. Difference requires two similarly aligned civil-time +// objects and returns the scalar answer in units of the objects' alignment. +// For example, the difference between two civil_hour objects will give an +// answer in units of civil hours. +// +// In addition to the six civil-time types just described, there are +// a handful of helper functions and algorithms for performing common +// calculations. These are described below. +// +// Note: In C++14 and later, this library is usable in a constexpr context. +// +// CONSTRUCTION: +// +// Each of the civil-time types can be constructed in two ways: by directly +// passing to the constructor up to six (optional) integers representing the +// YMDHMS fields, or by copying the YMDHMS fields from a differently aligned +// civil-time type. +// +// civil_day default_value; // 1970-01-01 00:00:00 +// +// civil_day a(2015, 2, 3); // 2015-02-03 00:00:00 +// civil_day b(2015, 2, 3, 4, 5, 6); // 2015-02-03 00:00:00 +// civil_day c(2015); // 2015-01-01 00:00:00 +// +// civil_second ss(2015, 2, 3, 4, 5, 6); // 2015-02-03 04:05:06 +// civil_minute mm(ss); // 2015-02-03 04:05:00 +// civil_hour hh(mm); // 2015-02-03 04:00:00 +// civil_day d(hh); // 2015-02-03 00:00:00 +// civil_month m(d); // 2015-02-01 00:00:00 +// civil_year y(m); // 2015-01-01 00:00:00 +// +// m = civil_month(y); // 2015-01-01 00:00:00 +// d = civil_day(m); // 2015-01-01 00:00:00 +// hh = civil_hour(d); // 2015-01-01 00:00:00 +// mm = civil_minute(hh); // 2015-01-01 00:00:00 +// ss = civil_second(mm); // 2015-01-01 00:00:00 +// +// ALIGNMENT CONVERSION: +// +// The alignment of a civil-time object cannot change, but the object may be +// used to construct a new object with a different alignment. This is referred +// to as "realigning". When realigning to a type with the same or more +// precision (e.g., civil_day -> civil_second), the conversion may be +// performed implicitly since no information is lost. However, if information +// could be discarded (e.g., civil_second -> civil_day), the conversion must +// be explicit at the call site. +// +// void fun(const civil_day& day); +// +// civil_second cs; +// fun(cs); // Won't compile because data may be discarded +// fun(civil_day(cs)); // OK: explicit conversion +// +// civil_day cd; +// fun(cd); // OK: no conversion needed +// +// civil_month cm; +// fun(cm); // OK: implicit conversion to civil_day +// +// NORMALIZATION: +// +// Integer arguments passed to the constructor may be out-of-range, in which +// case they are normalized to produce a valid civil-time object. This enables +// natural arithmetic on constructor arguments without worrying about the +// field's range. Normalization guarantees that there are no invalid +// civil-time objects. +// +// civil_day d(2016, 10, 32); // Out-of-range day; normalized to 2016-11-01 +// +// Note: If normalization is undesired, you can signal an error by comparing +// the constructor arguments to the normalized values returned by the YMDHMS +// properties. +// +// PROPERTIES: +// +// All civil-time types have accessors for all six of the civil-time fields: +// year, month, day, hour, minute, and second. Recall that fields inferior to +// the type's alignment will be set to their minimum valid value. +// +// civil_day d(2015, 6, 28); +// // d.year() == 2015 +// // d.month() == 6 +// // d.day() == 28 +// // d.hour() == 0 +// // d.minute() == 0 +// // d.second() == 0 +// +// COMPARISON: +// +// Comparison always considers all six YMDHMS fields, regardless of the type's +// alignment. Comparison between differently aligned civil-time types is +// allowed. +// +// civil_day feb_3(2015, 2, 3); // 2015-02-03 00:00:00 +// civil_day mar_4(2015, 3, 4); // 2015-03-04 00:00:00 +// // feb_3 < mar_4 +// // civil_year(feb_3) == civil_year(mar_4) +// +// civil_second feb_3_noon(2015, 2, 3, 12, 0, 0); // 2015-02-03 12:00:00 +// // feb_3 < feb_3_noon +// // feb_3 == civil_day(feb_3_noon) +// +// // Iterates all the days of February 2015. +// for (civil_day d(2015, 2, 1); d < civil_month(2015, 3); ++d) { +// // ... +// } +// +// STREAMING: +// +// Each civil-time type may be sent to an output stream using operator<<(). +// The output format follows the pattern "YYYY-MM-DDThh:mm:ss" where fields +// inferior to the type's alignment are omitted. +// +// civil_second cs(2015, 2, 3, 4, 5, 6); +// std::cout << cs << "\n"; // Outputs: 2015-02-03T04:05:06 +// +// civil_day cd(cs); +// std::cout << cd << "\n"; // Outputs: 2015-02-03 +// +// civil_year cy(cs); +// std::cout << cy << "\n"; // Outputs: 2015 +// +// ARITHMETIC: +// +// Civil-time types support natural arithmetic operators such as addition, +// subtraction, and difference. Arithmetic operates on the civil-time field +// indicated in the type's name. Difference requires arguments with the same +// alignment and returns the answer in units of the alignment. +// +// civil_day a(2015, 2, 3); +// ++a; // 2015-02-04 00:00:00 +// --a; // 2015-02-03 00:00:00 +// civil_day b = a + 1; // 2015-02-04 00:00:00 +// civil_day c = 1 + b; // 2015-02-05 00:00:00 +// int n = c - a; // n = 2 (civil days) +// int m = c - civil_month(c); // Won't compile: different types. +// +// EXAMPLE: Adding a month to January 31. +// +// One of the classic questions that arises when considering a civil-time +// library (or a date library or a date/time library) is this: "What happens +// when you add a month to January 31?" This is an interesting question +// because there could be a number of possible answers: +// +// 1. March 3 (or 2 if a leap year). This may make sense if the operation +// wants the equivalent of February 31. +// 2. February 28 (or 29 if a leap year). This may make sense if the operation +// wants the last day of January to go to the last day of February. +// 3. Error. The caller may get some error, an exception, an invalid date +// object, or maybe false is returned. This may make sense because there is +// no single unambiguously correct answer to the question. +// +// Practically speaking, any answer that is not what the programmer intended +// is the wrong answer. +// +// This civil-time library avoids the problem by making it impossible to ask +// ambiguous questions. All civil-time objects are aligned to a particular +// civil-field boundary (such as aligned to a year, month, day, hour, minute, +// or second), and arithmetic operates on the field to which the object is +// aligned. This means that in order to "add a month" the object must first be +// aligned to a month boundary, which is equivalent to the first day of that +// month. +// +// Of course, there are ways to compute an answer the question at hand using +// this civil-time library, but they require the programmer to be explicit +// about the answer they expect. To illustrate, let's see how to compute all +// three of the above possible answers to the question of "Jan 31 plus 1 +// month": +// +// const civil_day d(2015, 1, 31); +// +// // Answer 1: +// // Add 1 to the month field in the constructor, and rely on normalization. +// const auto ans_normalized = civil_day(d.year(), d.month() + 1, d.day()); +// // ans_normalized == 2015-03-03 (aka Feb 31) +// +// // Answer 2: +// // Add 1 to month field, capping to the end of next month. +// const auto next_month = civil_month(d) + 1; +// const auto last_day_of_next_month = civil_day(next_month + 1) - 1; +// const auto ans_capped = std::min(ans_normalized, last_day_of_next_month); +// // ans_capped == 2015-02-28 +// +// // Answer 3: +// // Signal an error if the normalized answer is not in next month. +// if (civil_month(ans_normalized) != next_month) { +// // error, month overflow +// } +// +using civil_year = detail::civil_year; +using civil_month = detail::civil_month; +using civil_day = detail::civil_day; +using civil_hour = detail::civil_hour; +using civil_minute = detail::civil_minute; +using civil_second = detail::civil_second; + +// An enum class with members monday, tuesday, wednesday, thursday, friday, +// saturday, and sunday. These enum values may be sent to an output stream +// using operator<<(). The result is the full weekday name in English with a +// leading capital letter. +// +// weekday wd = weekday::thursday; +// std::cout << wd << "\n"; // Outputs: Thursday +// +using detail::weekday; + +// Returns the weekday for the given civil-time value. +// +// civil_day a(2015, 8, 13); +// weekday wd = get_weekday(a); // wd == weekday::thursday +// +using detail::get_weekday; + +// Returns the civil_day that strictly follows or precedes the given +// civil_day, and that falls on the given weekday. +// +// For example, given: +// +// August 2015 +// Su Mo Tu We Th Fr Sa +// 1 +// 2 3 4 5 6 7 8 +// 9 10 11 12 13 14 15 +// 16 17 18 19 20 21 22 +// 23 24 25 26 27 28 29 +// 30 31 +// +// civil_day a(2015, 8, 13); // get_weekday(a) == weekday::thursday +// civil_day b = next_weekday(a, weekday::thursday); // b = 2015-08-20 +// civil_day c = prev_weekday(a, weekday::thursday); // c = 2015-08-06 +// +// civil_day d = ... +// // Gets the following Thursday if d is not already Thursday +// civil_day thurs1 = next_weekday(d - 1, weekday::thursday); +// // Gets the previous Thursday if d is not already Thursday +// civil_day thurs2 = prev_weekday(d + 1, weekday::thursday); +// +using detail::next_weekday; +using detail::prev_weekday; + +// Returns the day-of-year for the given civil-time value. +// +// civil_day a(2015, 1, 1); +// int yd_jan_1 = get_yearday(a); // yd_jan_1 = 1 +// civil_day b(2015, 12, 31); +// int yd_dec_31 = get_yearday(b); // yd_dec_31 = 365 +// +using detail::get_yearday; + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_CIVIL_TIME_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h new file mode 100644 index 000000000000..8aadde57ca72 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/civil_time_detail.h @@ -0,0 +1,628 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_CIVIL_TIME_DETAIL_H_ +#define ABSL_TIME_INTERNAL_CCTZ_CIVIL_TIME_DETAIL_H_ + +#include <cstdint> +#include <limits> +#include <ostream> +#include <type_traits> + +#include "absl/base/config.h" + +// Disable constexpr support unless we are in C++14 mode. +#if __cpp_constexpr >= 201304 || (defined(_MSC_VER) && _MSC_VER >= 1910) +#define CONSTEXPR_D constexpr // data +#define CONSTEXPR_F constexpr // function +#define CONSTEXPR_M constexpr // member +#else +#define CONSTEXPR_D const +#define CONSTEXPR_F inline +#define CONSTEXPR_M +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// Support years that at least span the range of 64-bit time_t values. +using year_t = std::int_fast64_t; + +// Type alias that indicates an argument is not normalized (e.g., the +// constructor parameters and operands/results of addition/subtraction). +using diff_t = std::int_fast64_t; + +namespace detail { + +// Type aliases that indicate normalized argument values. +using month_t = std::int_fast8_t; // [1:12] +using day_t = std::int_fast8_t; // [1:31] +using hour_t = std::int_fast8_t; // [0:23] +using minute_t = std::int_fast8_t; // [0:59] +using second_t = std::int_fast8_t; // [0:59] + +// Normalized civil-time fields: Y-M-D HH:MM:SS. +struct fields { + CONSTEXPR_M fields(year_t year, month_t month, day_t day, hour_t hour, + minute_t minute, second_t second) + : y(year), m(month), d(day), hh(hour), mm(minute), ss(second) {} + std::int_least64_t y; + std::int_least8_t m; + std::int_least8_t d; + std::int_least8_t hh; + std::int_least8_t mm; + std::int_least8_t ss; +}; + +struct second_tag {}; +struct minute_tag : second_tag {}; +struct hour_tag : minute_tag {}; +struct day_tag : hour_tag {}; +struct month_tag : day_tag {}; +struct year_tag : month_tag {}; + +//////////////////////////////////////////////////////////////////////// + +// Field normalization (without avoidable overflow). + +namespace impl { + +CONSTEXPR_F bool is_leap_year(year_t y) noexcept { + return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0); +} +CONSTEXPR_F int year_index(year_t y, month_t m) noexcept { + return (static_cast<int>((y + (m > 2)) % 400) + 400) % 400; +} +CONSTEXPR_F int days_per_century(year_t y, month_t m) noexcept { + const int yi = year_index(y, m); + return 36524 + (yi == 0 || yi > 300); +} +CONSTEXPR_F int days_per_4years(year_t y, month_t m) noexcept { + const int yi = year_index(y, m); + return 1460 + (yi == 0 || yi > 300 || (yi - 1) % 100 < 96); +} +CONSTEXPR_F int days_per_year(year_t y, month_t m) noexcept { + return is_leap_year(y + (m > 2)) ? 366 : 365; +} +CONSTEXPR_F int days_per_month(year_t y, month_t m) noexcept { + CONSTEXPR_D int k_days_per_month[1 + 12] = { + -1, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 // non leap year + }; + return k_days_per_month[m] + (m == 2 && is_leap_year(y)); +} + +CONSTEXPR_F fields n_day(year_t y, month_t m, diff_t d, diff_t cd, hour_t hh, + minute_t mm, second_t ss) noexcept { + year_t ey = y % 400; + const year_t oey = ey; + ey += (cd / 146097) * 400; + cd %= 146097; + if (cd < 0) { + ey -= 400; + cd += 146097; + } + ey += (d / 146097) * 400; + d = d % 146097 + cd; + if (d > 0) { + if (d > 146097) { + ey += 400; + d -= 146097; + } + } else { + if (d > -365) { + // We often hit the previous year when stepping a civil time backwards, + // so special case it to avoid counting up by 100/4/1-year chunks. + ey -= 1; + d += days_per_year(ey, m); + } else { + ey -= 400; + d += 146097; + } + } + if (d > 365) { + for (;;) { + int n = days_per_century(ey, m); + if (d <= n) break; + d -= n; + ey += 100; + } + for (;;) { + int n = days_per_4years(ey, m); + if (d <= n) break; + d -= n; + ey += 4; + } + for (;;) { + int n = days_per_year(ey, m); + if (d <= n) break; + d -= n; + ++ey; + } + } + if (d > 28) { + for (;;) { + int n = days_per_month(ey, m); + if (d <= n) break; + d -= n; + if (++m > 12) { + ++ey; + m = 1; + } + } + } + return fields(y + (ey - oey), m, static_cast<day_t>(d), hh, mm, ss); +} +CONSTEXPR_F fields n_mon(year_t y, diff_t m, diff_t d, diff_t cd, hour_t hh, + minute_t mm, second_t ss) noexcept { + if (m != 12) { + y += m / 12; + m %= 12; + if (m <= 0) { + y -= 1; + m += 12; + } + } + return n_day(y, static_cast<month_t>(m), d, cd, hh, mm, ss); +} +CONSTEXPR_F fields n_hour(year_t y, diff_t m, diff_t d, diff_t cd, diff_t hh, + minute_t mm, second_t ss) noexcept { + cd += hh / 24; + hh %= 24; + if (hh < 0) { + cd -= 1; + hh += 24; + } + return n_mon(y, m, d, cd, static_cast<hour_t>(hh), mm, ss); +} +CONSTEXPR_F fields n_min(year_t y, diff_t m, diff_t d, diff_t hh, diff_t ch, + diff_t mm, second_t ss) noexcept { + ch += mm / 60; + mm %= 60; + if (mm < 0) { + ch -= 1; + mm += 60; + } + return n_hour(y, m, d, hh / 24 + ch / 24, hh % 24 + ch % 24, + static_cast<minute_t>(mm), ss); +} +CONSTEXPR_F fields n_sec(year_t y, diff_t m, diff_t d, diff_t hh, diff_t mm, + diff_t ss) noexcept { + // Optimization for when (non-constexpr) fields are already normalized. + if (0 <= ss && ss < 60) { + const second_t nss = static_cast<second_t>(ss); + if (0 <= mm && mm < 60) { + const minute_t nmm = static_cast<minute_t>(mm); + if (0 <= hh && hh < 24) { + const hour_t nhh = static_cast<hour_t>(hh); + if (1 <= d && d <= 28 && 1 <= m && m <= 12) { + const day_t nd = static_cast<day_t>(d); + const month_t nm = static_cast<month_t>(m); + return fields(y, nm, nd, nhh, nmm, nss); + } + return n_mon(y, m, d, 0, nhh, nmm, nss); + } + return n_hour(y, m, d, hh / 24, hh % 24, nmm, nss); + } + return n_min(y, m, d, hh, mm / 60, mm % 60, nss); + } + diff_t cm = ss / 60; + ss %= 60; + if (ss < 0) { + cm -= 1; + ss += 60; + } + return n_min(y, m, d, hh, mm / 60 + cm / 60, mm % 60 + cm % 60, + static_cast<second_t>(ss)); +} + +} // namespace impl + +//////////////////////////////////////////////////////////////////////// + +// Increments the indicated (normalized) field by "n". +CONSTEXPR_F fields step(second_tag, fields f, diff_t n) noexcept { + return impl::n_sec(f.y, f.m, f.d, f.hh, f.mm + n / 60, f.ss + n % 60); +} +CONSTEXPR_F fields step(minute_tag, fields f, diff_t n) noexcept { + return impl::n_min(f.y, f.m, f.d, f.hh + n / 60, 0, f.mm + n % 60, f.ss); +} +CONSTEXPR_F fields step(hour_tag, fields f, diff_t n) noexcept { + return impl::n_hour(f.y, f.m, f.d + n / 24, 0, f.hh + n % 24, f.mm, f.ss); +} +CONSTEXPR_F fields step(day_tag, fields f, diff_t n) noexcept { + return impl::n_day(f.y, f.m, f.d, n, f.hh, f.mm, f.ss); +} +CONSTEXPR_F fields step(month_tag, fields f, diff_t n) noexcept { + return impl::n_mon(f.y + n / 12, f.m + n % 12, f.d, 0, f.hh, f.mm, f.ss); +} +CONSTEXPR_F fields step(year_tag, fields f, diff_t n) noexcept { + return fields(f.y + n, f.m, f.d, f.hh, f.mm, f.ss); +} + +//////////////////////////////////////////////////////////////////////// + +namespace impl { + +// Returns (v * f + a) but avoiding intermediate overflow when possible. +CONSTEXPR_F diff_t scale_add(diff_t v, diff_t f, diff_t a) noexcept { + return (v < 0) ? ((v + 1) * f + a) - f : ((v - 1) * f + a) + f; +} + +// Map a (normalized) Y/M/D to the number of days before/after 1970-01-01. +// Probably overflows for years outside [-292277022656:292277026595]. +CONSTEXPR_F diff_t ymd_ord(year_t y, month_t m, day_t d) noexcept { + const diff_t eyear = (m <= 2) ? y - 1 : y; + const diff_t era = (eyear >= 0 ? eyear : eyear - 399) / 400; + const diff_t yoe = eyear - era * 400; + const diff_t doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + const diff_t doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} + +// Returns the difference in days between two normalized Y-M-D tuples. +// ymd_ord() will encounter integer overflow given extreme year values, +// yet the difference between two such extreme values may actually be +// small, so we take a little care to avoid overflow when possible by +// exploiting the 146097-day cycle. +CONSTEXPR_F diff_t day_difference(year_t y1, month_t m1, day_t d1, year_t y2, + month_t m2, day_t d2) noexcept { + const diff_t a_c4_off = y1 % 400; + const diff_t b_c4_off = y2 % 400; + diff_t c4_diff = (y1 - a_c4_off) - (y2 - b_c4_off); + diff_t delta = ymd_ord(a_c4_off, m1, d1) - ymd_ord(b_c4_off, m2, d2); + if (c4_diff > 0 && delta < 0) { + delta += 2 * 146097; + c4_diff -= 2 * 400; + } else if (c4_diff < 0 && delta > 0) { + delta -= 2 * 146097; + c4_diff += 2 * 400; + } + return (c4_diff / 400 * 146097) + delta; +} + +} // namespace impl + +// Returns the difference between fields structs using the indicated unit. +CONSTEXPR_F diff_t difference(year_tag, fields f1, fields f2) noexcept { + return f1.y - f2.y; +} +CONSTEXPR_F diff_t difference(month_tag, fields f1, fields f2) noexcept { + return impl::scale_add(difference(year_tag{}, f1, f2), 12, (f1.m - f2.m)); +} +CONSTEXPR_F diff_t difference(day_tag, fields f1, fields f2) noexcept { + return impl::day_difference(f1.y, f1.m, f1.d, f2.y, f2.m, f2.d); +} +CONSTEXPR_F diff_t difference(hour_tag, fields f1, fields f2) noexcept { + return impl::scale_add(difference(day_tag{}, f1, f2), 24, (f1.hh - f2.hh)); +} +CONSTEXPR_F diff_t difference(minute_tag, fields f1, fields f2) noexcept { + return impl::scale_add(difference(hour_tag{}, f1, f2), 60, (f1.mm - f2.mm)); +} +CONSTEXPR_F diff_t difference(second_tag, fields f1, fields f2) noexcept { + return impl::scale_add(difference(minute_tag{}, f1, f2), 60, f1.ss - f2.ss); +} + +//////////////////////////////////////////////////////////////////////// + +// Aligns the (normalized) fields struct to the indicated field. +CONSTEXPR_F fields align(second_tag, fields f) noexcept { return f; } +CONSTEXPR_F fields align(minute_tag, fields f) noexcept { + return fields{f.y, f.m, f.d, f.hh, f.mm, 0}; +} +CONSTEXPR_F fields align(hour_tag, fields f) noexcept { + return fields{f.y, f.m, f.d, f.hh, 0, 0}; +} +CONSTEXPR_F fields align(day_tag, fields f) noexcept { + return fields{f.y, f.m, f.d, 0, 0, 0}; +} +CONSTEXPR_F fields align(month_tag, fields f) noexcept { + return fields{f.y, f.m, 1, 0, 0, 0}; +} +CONSTEXPR_F fields align(year_tag, fields f) noexcept { + return fields{f.y, 1, 1, 0, 0, 0}; +} + +//////////////////////////////////////////////////////////////////////// + +namespace impl { + +template <typename H> +H AbslHashValueImpl(second_tag, H h, fields f) { + return H::combine(std::move(h), f.y, f.m, f.d, f.hh, f.mm, f.ss); +} +template <typename H> +H AbslHashValueImpl(minute_tag, H h, fields f) { + return H::combine(std::move(h), f.y, f.m, f.d, f.hh, f.mm); +} +template <typename H> +H AbslHashValueImpl(hour_tag, H h, fields f) { + return H::combine(std::move(h), f.y, f.m, f.d, f.hh); +} +template <typename H> +H AbslHashValueImpl(day_tag, H h, fields f) { + return H::combine(std::move(h), f.y, f.m, f.d); +} +template <typename H> +H AbslHashValueImpl(month_tag, H h, fields f) { + return H::combine(std::move(h), f.y, f.m); +} +template <typename H> +H AbslHashValueImpl(year_tag, H h, fields f) { + return H::combine(std::move(h), f.y); +} + +} // namespace impl + +//////////////////////////////////////////////////////////////////////// + +template <typename T> +class civil_time { + public: + explicit CONSTEXPR_M civil_time(year_t y, diff_t m = 1, diff_t d = 1, + diff_t hh = 0, diff_t mm = 0, + diff_t ss = 0) noexcept + : civil_time(impl::n_sec(y, m, d, hh, mm, ss)) {} + + CONSTEXPR_M civil_time() noexcept : f_{1970, 1, 1, 0, 0, 0} {} + civil_time(const civil_time&) = default; + civil_time& operator=(const civil_time&) = default; + + // Conversion between civil times of different alignment. Conversion to + // a more precise alignment is allowed implicitly (e.g., day -> hour), + // but conversion where information is discarded must be explicit + // (e.g., second -> minute). + template <typename U, typename S> + using preserves_data = + typename std::enable_if<std::is_base_of<U, S>::value>::type; + template <typename U> + CONSTEXPR_M civil_time(const civil_time<U>& ct, + preserves_data<T, U>* = nullptr) noexcept + : civil_time(ct.f_) {} + template <typename U> + explicit CONSTEXPR_M civil_time(const civil_time<U>& ct, + preserves_data<U, T>* = nullptr) noexcept + : civil_time(ct.f_) {} + + // Factories for the maximum/minimum representable civil_time. + static CONSTEXPR_F civil_time(max)() { + const auto max_year = (std::numeric_limits<std::int_least64_t>::max)(); + return civil_time(max_year, 12, 31, 23, 59, 59); + } + static CONSTEXPR_F civil_time(min)() { + const auto min_year = (std::numeric_limits<std::int_least64_t>::min)(); + return civil_time(min_year, 1, 1, 0, 0, 0); + } + + // Field accessors. Note: All but year() return an int. + CONSTEXPR_M year_t year() const noexcept { return f_.y; } + CONSTEXPR_M int month() const noexcept { return f_.m; } + CONSTEXPR_M int day() const noexcept { return f_.d; } + CONSTEXPR_M int hour() const noexcept { return f_.hh; } + CONSTEXPR_M int minute() const noexcept { return f_.mm; } + CONSTEXPR_M int second() const noexcept { return f_.ss; } + + // Assigning arithmetic. + CONSTEXPR_M civil_time& operator+=(diff_t n) noexcept { + return *this = *this + n; + } + CONSTEXPR_M civil_time& operator-=(diff_t n) noexcept { + return *this = *this - n; + } + CONSTEXPR_M civil_time& operator++() noexcept { return *this += 1; } + CONSTEXPR_M civil_time operator++(int) noexcept { + const civil_time a = *this; + ++*this; + return a; + } + CONSTEXPR_M civil_time& operator--() noexcept { return *this -= 1; } + CONSTEXPR_M civil_time operator--(int) noexcept { + const civil_time a = *this; + --*this; + return a; + } + + // Binary arithmetic operators. + friend CONSTEXPR_F civil_time operator+(civil_time a, diff_t n) noexcept { + return civil_time(step(T{}, a.f_, n)); + } + friend CONSTEXPR_F civil_time operator+(diff_t n, civil_time a) noexcept { + return a + n; + } + friend CONSTEXPR_F civil_time operator-(civil_time a, diff_t n) noexcept { + return n != (std::numeric_limits<diff_t>::min)() + ? civil_time(step(T{}, a.f_, -n)) + : civil_time(step(T{}, step(T{}, a.f_, -(n + 1)), 1)); + } + friend CONSTEXPR_F diff_t operator-(civil_time lhs, civil_time rhs) noexcept { + return difference(T{}, lhs.f_, rhs.f_); + } + + template <typename H> + friend H AbslHashValue(H h, civil_time a) { + return impl::AbslHashValueImpl(T{}, std::move(h), a.f_); + } + + private: + // All instantiations of this template are allowed to call the following + // private constructor and access the private fields member. + template <typename U> + friend class civil_time; + + // The designated constructor that all others eventually call. + explicit CONSTEXPR_M civil_time(fields f) noexcept : f_(align(T{}, f)) {} + + fields f_; +}; + +// Disallows difference between differently aligned types. +// auto n = civil_day(...) - civil_hour(...); // would be confusing. +template <typename T, typename U> +CONSTEXPR_F diff_t operator-(civil_time<T>, civil_time<U>) = delete; + +using civil_year = civil_time<year_tag>; +using civil_month = civil_time<month_tag>; +using civil_day = civil_time<day_tag>; +using civil_hour = civil_time<hour_tag>; +using civil_minute = civil_time<minute_tag>; +using civil_second = civil_time<second_tag>; + +//////////////////////////////////////////////////////////////////////// + +// Relational operators that work with differently aligned objects. +// Always compares all six fields. +template <typename T1, typename T2> +CONSTEXPR_F bool operator<(const civil_time<T1>& lhs, + const civil_time<T2>& rhs) noexcept { + return ( + lhs.year() < rhs.year() || + (lhs.year() == rhs.year() && + (lhs.month() < rhs.month() || + (lhs.month() == rhs.month() && + (lhs.day() < rhs.day() || (lhs.day() == rhs.day() && + (lhs.hour() < rhs.hour() || + (lhs.hour() == rhs.hour() && + (lhs.minute() < rhs.minute() || + (lhs.minute() == rhs.minute() && + (lhs.second() < rhs.second()))))))))))); +} +template <typename T1, typename T2> +CONSTEXPR_F bool operator<=(const civil_time<T1>& lhs, + const civil_time<T2>& rhs) noexcept { + return !(rhs < lhs); +} +template <typename T1, typename T2> +CONSTEXPR_F bool operator>=(const civil_time<T1>& lhs, + const civil_time<T2>& rhs) noexcept { + return !(lhs < rhs); +} +template <typename T1, typename T2> +CONSTEXPR_F bool operator>(const civil_time<T1>& lhs, + const civil_time<T2>& rhs) noexcept { + return rhs < lhs; +} +template <typename T1, typename T2> +CONSTEXPR_F bool operator==(const civil_time<T1>& lhs, + const civil_time<T2>& rhs) noexcept { + return lhs.year() == rhs.year() && lhs.month() == rhs.month() && + lhs.day() == rhs.day() && lhs.hour() == rhs.hour() && + lhs.minute() == rhs.minute() && lhs.second() == rhs.second(); +} +template <typename T1, typename T2> +CONSTEXPR_F bool operator!=(const civil_time<T1>& lhs, + const civil_time<T2>& rhs) noexcept { + return !(lhs == rhs); +} + +//////////////////////////////////////////////////////////////////////// + +enum class weekday { + monday, + tuesday, + wednesday, + thursday, + friday, + saturday, + sunday, +}; + +CONSTEXPR_F weekday get_weekday(const civil_second& cs) noexcept { + CONSTEXPR_D weekday k_weekday_by_mon_off[13] = { + weekday::monday, weekday::tuesday, weekday::wednesday, + weekday::thursday, weekday::friday, weekday::saturday, + weekday::sunday, weekday::monday, weekday::tuesday, + weekday::wednesday, weekday::thursday, weekday::friday, + weekday::saturday, + }; + CONSTEXPR_D int k_weekday_offsets[1 + 12] = { + -1, 0, 3, 2, 5, 0, 3, 5, 1, 4, 6, 2, 4, + }; + year_t wd = 2400 + (cs.year() % 400) - (cs.month() < 3); + wd += wd / 4 - wd / 100 + wd / 400; + wd += k_weekday_offsets[cs.month()] + cs.day(); + return k_weekday_by_mon_off[wd % 7 + 6]; +} + +//////////////////////////////////////////////////////////////////////// + +CONSTEXPR_F civil_day next_weekday(civil_day cd, weekday wd) noexcept { + CONSTEXPR_D weekday k_weekdays_forw[14] = { + weekday::monday, weekday::tuesday, weekday::wednesday, + weekday::thursday, weekday::friday, weekday::saturday, + weekday::sunday, weekday::monday, weekday::tuesday, + weekday::wednesday, weekday::thursday, weekday::friday, + weekday::saturday, weekday::sunday, + }; + weekday base = get_weekday(cd); + for (int i = 0;; ++i) { + if (base == k_weekdays_forw[i]) { + for (int j = i + 1;; ++j) { + if (wd == k_weekdays_forw[j]) { + return cd + (j - i); + } + } + } + } +} + +CONSTEXPR_F civil_day prev_weekday(civil_day cd, weekday wd) noexcept { + CONSTEXPR_D weekday k_weekdays_back[14] = { + weekday::sunday, weekday::saturday, weekday::friday, + weekday::thursday, weekday::wednesday, weekday::tuesday, + weekday::monday, weekday::sunday, weekday::saturday, + weekday::friday, weekday::thursday, weekday::wednesday, + weekday::tuesday, weekday::monday, + }; + weekday base = get_weekday(cd); + for (int i = 0;; ++i) { + if (base == k_weekdays_back[i]) { + for (int j = i + 1;; ++j) { + if (wd == k_weekdays_back[j]) { + return cd - (j - i); + } + } + } + } +} + +CONSTEXPR_F int get_yearday(const civil_second& cs) noexcept { + CONSTEXPR_D int k_month_offsets[1 + 12] = { + -1, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, + }; + const int feb29 = (cs.month() > 2 && impl::is_leap_year(cs.year())); + return k_month_offsets[cs.month()] + feb29 + cs.day(); +} + +//////////////////////////////////////////////////////////////////////// + +std::ostream& operator<<(std::ostream& os, const civil_year& y); +std::ostream& operator<<(std::ostream& os, const civil_month& m); +std::ostream& operator<<(std::ostream& os, const civil_day& d); +std::ostream& operator<<(std::ostream& os, const civil_hour& h); +std::ostream& operator<<(std::ostream& os, const civil_minute& m); +std::ostream& operator<<(std::ostream& os, const civil_second& s); +std::ostream& operator<<(std::ostream& os, weekday wd); + +} // namespace detail +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#undef CONSTEXPR_M +#undef CONSTEXPR_F +#undef CONSTEXPR_D + +#endif // ABSL_TIME_INTERNAL_CCTZ_CIVIL_TIME_DETAIL_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/time_zone.h b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/time_zone.h new file mode 100644 index 000000000000..5562a37bc807 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/time_zone.h @@ -0,0 +1,386 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A library for translating between absolute times (represented by +// std::chrono::time_points of the std::chrono::system_clock) and civil +// times (represented by cctz::civil_second) using the rules defined by +// a time zone (cctz::time_zone). + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_H_ + +#include <chrono> +#include <cstdint> +#include <string> +#include <utility> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// Convenience aliases. Not intended as public API points. +template <typename D> +using time_point = std::chrono::time_point<std::chrono::system_clock, D>; +using seconds = std::chrono::duration<std::int_fast64_t>; +using sys_seconds = seconds; // Deprecated. Use cctz::seconds instead. + +namespace detail { +template <typename D> +inline std::pair<time_point<seconds>, D> split_seconds( + const time_point<D>& tp) { + auto sec = std::chrono::time_point_cast<seconds>(tp); + auto sub = tp - sec; + if (sub.count() < 0) { + sec -= seconds(1); + sub += seconds(1); + } + return {sec, std::chrono::duration_cast<D>(sub)}; +} +inline std::pair<time_point<seconds>, seconds> split_seconds( + const time_point<seconds>& tp) { + return {tp, seconds::zero()}; +} +} // namespace detail + +// cctz::time_zone is an opaque, small, value-type class representing a +// geo-political region within which particular rules are used for mapping +// between absolute and civil times. Time zones are named using the TZ +// identifiers from the IANA Time Zone Database, such as "America/Los_Angeles" +// or "Australia/Sydney". Time zones are created from factory functions such +// as load_time_zone(). Note: strings like "PST" and "EDT" are not valid TZ +// identifiers. +// +// Example: +// cctz::time_zone utc = cctz::utc_time_zone(); +// cctz::time_zone pst = cctz::fixed_time_zone(std::chrono::hours(-8)); +// cctz::time_zone loc = cctz::local_time_zone(); +// cctz::time_zone lax; +// if (!cctz::load_time_zone("America/Los_Angeles", &lax)) { ... } +// +// See also: +// - http://www.iana.org/time-zones +// - https://en.wikipedia.org/wiki/Zoneinfo +class time_zone { + public: + time_zone() : time_zone(nullptr) {} // Equivalent to UTC + time_zone(const time_zone&) = default; + time_zone& operator=(const time_zone&) = default; + + std::string name() const; + + // An absolute_lookup represents the civil time (cctz::civil_second) within + // this time_zone at the given absolute time (time_point). There are + // additionally a few other fields that may be useful when working with + // older APIs, such as std::tm. + // + // Example: + // const cctz::time_zone tz = ... + // const auto tp = std::chrono::system_clock::now(); + // const cctz::time_zone::absolute_lookup al = tz.lookup(tp); + struct absolute_lookup { + civil_second cs; + // Note: The following fields exist for backward compatibility with older + // APIs. Accessing these fields directly is a sign of imprudent logic in + // the calling code. Modern time-related code should only access this data + // indirectly by way of cctz::format(). + int offset; // civil seconds east of UTC + bool is_dst; // is offset non-standard? + const char* abbr; // time-zone abbreviation (e.g., "PST") + }; + absolute_lookup lookup(const time_point<seconds>& tp) const; + template <typename D> + absolute_lookup lookup(const time_point<D>& tp) const { + return lookup(detail::split_seconds(tp).first); + } + + // A civil_lookup represents the absolute time(s) (time_point) that + // correspond to the given civil time (cctz::civil_second) within this + // time_zone. Usually the given civil time represents a unique instant + // in time, in which case the conversion is unambiguous. However, + // within this time zone, the given civil time may be skipped (e.g., + // during a positive UTC offset shift), or repeated (e.g., during a + // negative UTC offset shift). To account for these possibilities, + // civil_lookup is richer than just a single time_point. + // + // In all cases the civil_lookup::kind enum will indicate the nature + // of the given civil-time argument, and the pre, trans, and post + // members will give the absolute time answers using the pre-transition + // offset, the transition point itself, and the post-transition offset, + // respectively (all three times are equal if kind == UNIQUE). If any + // of these three absolute times is outside the representable range of a + // time_point<seconds> the field is set to its maximum/minimum value. + // + // Example: + // cctz::time_zone lax; + // if (!cctz::load_time_zone("America/Los_Angeles", &lax)) { ... } + // + // // A unique civil time. + // auto jan01 = lax.lookup(cctz::civil_second(2011, 1, 1, 0, 0, 0)); + // // jan01.kind == cctz::time_zone::civil_lookup::UNIQUE + // // jan01.pre is 2011/01/01 00:00:00 -0800 + // // jan01.trans is 2011/01/01 00:00:00 -0800 + // // jan01.post is 2011/01/01 00:00:00 -0800 + // + // // A Spring DST transition, when there is a gap in civil time. + // auto mar13 = lax.lookup(cctz::civil_second(2011, 3, 13, 2, 15, 0)); + // // mar13.kind == cctz::time_zone::civil_lookup::SKIPPED + // // mar13.pre is 2011/03/13 03:15:00 -0700 + // // mar13.trans is 2011/03/13 03:00:00 -0700 + // // mar13.post is 2011/03/13 01:15:00 -0800 + // + // // A Fall DST transition, when civil times are repeated. + // auto nov06 = lax.lookup(cctz::civil_second(2011, 11, 6, 1, 15, 0)); + // // nov06.kind == cctz::time_zone::civil_lookup::REPEATED + // // nov06.pre is 2011/11/06 01:15:00 -0700 + // // nov06.trans is 2011/11/06 01:00:00 -0800 + // // nov06.post is 2011/11/06 01:15:00 -0800 + struct civil_lookup { + enum civil_kind { + UNIQUE, // the civil time was singular (pre == trans == post) + SKIPPED, // the civil time did not exist (pre >= trans > post) + REPEATED, // the civil time was ambiguous (pre < trans <= post) + } kind; + time_point<seconds> pre; // uses the pre-transition offset + time_point<seconds> trans; // instant of civil-offset change + time_point<seconds> post; // uses the post-transition offset + }; + civil_lookup lookup(const civil_second& cs) const; + + // Finds the time of the next/previous offset change in this time zone. + // + // By definition, next_transition(tp, &trans) returns false when tp has + // its maximum value, and prev_transition(tp, &trans) returns false + // when tp has its minimum value. If the zone has no transitions, the + // result will also be false no matter what the argument. + // + // Otherwise, when tp has its minimum value, next_transition(tp, &trans) + // returns true and sets trans to the first recorded transition. Chains + // of calls to next_transition()/prev_transition() will eventually return + // false, but it is unspecified exactly when next_transition(tp, &trans) + // jumps to false, or what time is set by prev_transition(tp, &trans) for + // a very distant tp. + // + // Note: Enumeration of time-zone transitions is for informational purposes + // only. Modern time-related code should not care about when offset changes + // occur. + // + // Example: + // cctz::time_zone nyc; + // if (!cctz::load_time_zone("America/New_York", &nyc)) { ... } + // const auto now = std::chrono::system_clock::now(); + // auto tp = cctz::time_point<cctz::seconds>::min(); + // cctz::time_zone::civil_transition trans; + // while (tp <= now && nyc.next_transition(tp, &trans)) { + // // transition: trans.from -> trans.to + // tp = nyc.lookup(trans.to).trans; + // } + struct civil_transition { + civil_second from; // the civil time we jump from + civil_second to; // the civil time we jump to + }; + bool next_transition(const time_point<seconds>& tp, + civil_transition* trans) const; + template <typename D> + bool next_transition(const time_point<D>& tp, civil_transition* trans) const { + return next_transition(detail::split_seconds(tp).first, trans); + } + bool prev_transition(const time_point<seconds>& tp, + civil_transition* trans) const; + template <typename D> + bool prev_transition(const time_point<D>& tp, civil_transition* trans) const { + return prev_transition(detail::split_seconds(tp).first, trans); + } + + // version() and description() provide additional information about the + // time zone. The content of each of the returned strings is unspecified, + // however, when the IANA Time Zone Database is the underlying data source + // the version() string will be in the familar form (e.g, "2018e") or + // empty when unavailable. + // + // Note: These functions are for informational or testing purposes only. + std::string version() const; // empty when unknown + std::string description() const; + + // Relational operators. + friend bool operator==(time_zone lhs, time_zone rhs) { + return &lhs.effective_impl() == &rhs.effective_impl(); + } + friend bool operator!=(time_zone lhs, time_zone rhs) { return !(lhs == rhs); } + + template <typename H> + friend H AbslHashValue(H h, time_zone tz) { + return H::combine(std::move(h), &tz.effective_impl()); + } + + class Impl; + + private: + explicit time_zone(const Impl* impl) : impl_(impl) {} + const Impl& effective_impl() const; // handles implicit UTC + const Impl* impl_; +}; + +// Loads the named time zone. May perform I/O on the initial load. +// If the name is invalid, or some other kind of error occurs, returns +// false and "*tz" is set to the UTC time zone. +bool load_time_zone(const std::string& name, time_zone* tz); + +// Returns a time_zone representing UTC. Cannot fail. +time_zone utc_time_zone(); + +// Returns a time zone that is a fixed offset (seconds east) from UTC. +// Note: If the absolute value of the offset is greater than 24 hours +// you'll get UTC (i.e., zero offset) instead. +time_zone fixed_time_zone(const seconds& offset); + +// Returns a time zone representing the local time zone. Falls back to UTC. +// Note: local_time_zone.name() may only be something like "localtime". +time_zone local_time_zone(); + +// Returns the civil time (cctz::civil_second) within the given time zone at +// the given absolute time (time_point). Since the additional fields provided +// by the time_zone::absolute_lookup struct should rarely be needed in modern +// code, this convert() function is simpler and should be preferred. +template <typename D> +inline civil_second convert(const time_point<D>& tp, const time_zone& tz) { + return tz.lookup(tp).cs; +} + +// Returns the absolute time (time_point) that corresponds to the given civil +// time within the given time zone. If the civil time is not unique (i.e., if +// it was either repeated or non-existent), then the returned time_point is +// the best estimate that preserves relative order. That is, this function +// guarantees that if cs1 < cs2, then convert(cs1, tz) <= convert(cs2, tz). +inline time_point<seconds> convert(const civil_second& cs, + const time_zone& tz) { + const time_zone::civil_lookup cl = tz.lookup(cs); + if (cl.kind == time_zone::civil_lookup::SKIPPED) return cl.trans; + return cl.pre; +} + +namespace detail { +using femtoseconds = std::chrono::duration<std::int_fast64_t, std::femto>; +std::string format(const std::string&, const time_point<seconds>&, + const femtoseconds&, const time_zone&); +bool parse(const std::string&, const std::string&, const time_zone&, + time_point<seconds>*, femtoseconds*, std::string* err = nullptr); +} // namespace detail + +// Formats the given time_point in the given cctz::time_zone according to +// the provided format string. Uses strftime()-like formatting options, +// with the following extensions: +// +// - %Ez - RFC3339-compatible numeric UTC offset (+hh:mm or -hh:mm) +// - %E*z - Full-resolution numeric UTC offset (+hh:mm:ss or -hh:mm:ss) +// - %E#S - Seconds with # digits of fractional precision +// - %E*S - Seconds with full fractional precision (a literal '*') +// - %E#f - Fractional seconds with # digits of precision +// - %E*f - Fractional seconds with full precision (a literal '*') +// - %E4Y - Four-character years (-999 ... -001, 0000, 0001 ... 9999) +// - %ET - The RFC3339 "date-time" separator "T" +// +// Note that %E0S behaves like %S, and %E0f produces no characters. In +// contrast %E*f always produces at least one digit, which may be '0'. +// +// Note that %Y produces as many characters as it takes to fully render the +// year. A year outside of [-999:9999] when formatted with %E4Y will produce +// more than four characters, just like %Y. +// +// Tip: Format strings should include the UTC offset (e.g., %z, %Ez, or %E*z) +// so that the resulting string uniquely identifies an absolute time. +// +// Example: +// cctz::time_zone lax; +// if (!cctz::load_time_zone("America/Los_Angeles", &lax)) { ... } +// auto tp = cctz::convert(cctz::civil_second(2013, 1, 2, 3, 4, 5), lax); +// std::string f = cctz::format("%H:%M:%S", tp, lax); // "03:04:05" +// f = cctz::format("%H:%M:%E3S", tp, lax); // "03:04:05.000" +template <typename D> +inline std::string format(const std::string& fmt, const time_point<D>& tp, + const time_zone& tz) { + const auto p = detail::split_seconds(tp); + const auto n = std::chrono::duration_cast<detail::femtoseconds>(p.second); + return detail::format(fmt, p.first, n, tz); +} + +// Parses an input string according to the provided format string and +// returns the corresponding time_point. Uses strftime()-like formatting +// options, with the same extensions as cctz::format(), but with the +// exceptions that %E#S is interpreted as %E*S, and %E#f as %E*f. %Ez +// and %E*z also accept the same inputs, which (along with %z) includes +// 'z' and 'Z' as synonyms for +00:00. %ET accepts either 'T' or 't'. +// +// %Y consumes as many numeric characters as it can, so the matching data +// should always be terminated with a non-numeric. %E4Y always consumes +// exactly four characters, including any sign. +// +// Unspecified fields are taken from the default date and time of ... +// +// "1970-01-01 00:00:00.0 +0000" +// +// For example, parsing a string of "15:45" (%H:%M) will return a time_point +// that represents "1970-01-01 15:45:00.0 +0000". +// +// Note that parse() returns time instants, so it makes most sense to parse +// fully-specified date/time strings that include a UTC offset (%z, %Ez, or +// %E*z). +// +// Note also that parse() only heeds the fields year, month, day, hour, +// minute, (fractional) second, and UTC offset. Other fields, like weekday (%a +// or %A), while parsed for syntactic validity, are ignored in the conversion. +// +// Date and time fields that are out-of-range will be treated as errors rather +// than normalizing them like cctz::civil_second() would do. For example, it +// is an error to parse the date "Oct 32, 2013" because 32 is out of range. +// +// A second of ":60" is normalized to ":00" of the following minute with +// fractional seconds discarded. The following table shows how the given +// seconds and subseconds will be parsed: +// +// "59.x" -> 59.x // exact +// "60.x" -> 00.0 // normalized +// "00.x" -> 00.x // exact +// +// Errors are indicated by returning false. +// +// Example: +// const cctz::time_zone tz = ... +// std::chrono::system_clock::time_point tp; +// if (cctz::parse("%Y-%m-%d", "2015-10-09", tz, &tp)) { +// ... +// } +template <typename D> +inline bool parse(const std::string& fmt, const std::string& input, + const time_zone& tz, time_point<D>* tpp) { + time_point<seconds> sec; + detail::femtoseconds fs; + const bool b = detail::parse(fmt, input, tz, &sec, &fs); + if (b) { + // TODO: Return false if unrepresentable as a time_point<D>. + *tpp = std::chrono::time_point_cast<D>(sec); + *tpp += std::chrono::duration_cast<D>(fs); + } + return b; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/zone_info_source.h b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/zone_info_source.h new file mode 100644 index 000000000000..012eb4ec30e0 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/include/cctz/zone_info_source.h @@ -0,0 +1,102 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_ZONE_INFO_SOURCE_H_ +#define ABSL_TIME_INTERNAL_CCTZ_ZONE_INFO_SOURCE_H_ + +#include <cstddef> +#include <functional> +#include <memory> +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// A stdio-like interface for providing zoneinfo data for a particular zone. +class ZoneInfoSource { + public: + virtual ~ZoneInfoSource(); + + virtual std::size_t Read(void* ptr, std::size_t size) = 0; // like fread() + virtual int Skip(std::size_t offset) = 0; // like fseek() + + // Until the zoneinfo data supports versioning information, we provide + // a way for a ZoneInfoSource to indicate it out-of-band. The default + // implementation returns an empty string. + virtual std::string Version() const; +}; + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz_extension { + +// A function-pointer type for a factory that returns a ZoneInfoSource +// given the name of a time zone and a fallback factory. Returns null +// when the data for the named zone cannot be found. +using ZoneInfoSourceFactory = + std::unique_ptr<absl::time_internal::cctz::ZoneInfoSource> (*)( + const std::string&, + const std::function<std::unique_ptr< + absl::time_internal::cctz::ZoneInfoSource>(const std::string&)>&); + +// The user can control the mapping of zone names to zoneinfo data by +// providing a definition for cctz_extension::zone_info_source_factory. +// For example, given functions my_factory() and my_other_factory() that +// can return a ZoneInfoSource for a named zone, we could inject them into +// cctz::load_time_zone() with: +// +// namespace cctz_extension { +// namespace { +// std::unique_ptr<cctz::ZoneInfoSource> CustomFactory( +// const std::string& name, +// const std::function<std::unique_ptr<cctz::ZoneInfoSource>( +// const std::string& name)>& fallback_factory) { +// if (auto zip = my_factory(name)) return zip; +// if (auto zip = fallback_factory(name)) return zip; +// if (auto zip = my_other_factory(name)) return zip; +// return nullptr; +// } +// } // namespace +// ZoneInfoSourceFactory zone_info_source_factory = CustomFactory; +// } // namespace cctz_extension +// +// This might be used, say, to use zoneinfo data embedded in the program, +// or read from a (possibly compressed) file archive, or both. +// +// cctz_extension::zone_info_source_factory() will be called: +// (1) from the same thread as the cctz::load_time_zone() call, +// (2) only once for any zone name, and +// (3) serially (i.e., no concurrent execution). +// +// The fallback factory obtains zoneinfo data by reading files in ${TZDIR}, +// and it is used automatically when no zone_info_source_factory definition +// is linked into the program. +extern ZoneInfoSourceFactory zone_info_source_factory; + +} // namespace cctz_extension +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_ZONE_INFO_SOURCE_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/cctz_benchmark.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/cctz_benchmark.cc new file mode 100644 index 000000000000..4e39188ff3e6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/cctz_benchmark.cc @@ -0,0 +1,1030 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <algorithm> +#include <cassert> +#include <chrono> +#include <ctime> +#include <random> +#include <string> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" +#include "time_zone_impl.h" + +namespace { + +namespace cctz = absl::time_internal::cctz; + +void BM_Difference_Days(benchmark::State& state) { + const cctz::civil_day c(2014, 8, 22); + const cctz::civil_day epoch(1970, 1, 1); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(c - epoch); + } +} +BENCHMARK(BM_Difference_Days); + +void BM_Step_Days(benchmark::State& state) { + const cctz::civil_day kStart(2014, 8, 22); + cctz::civil_day c = kStart; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(++c); + } +} +BENCHMARK(BM_Step_Days); + +void BM_GetWeekday(benchmark::State& state) { + const cctz::civil_day c(2014, 8, 22); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::get_weekday(c)); + } +} +BENCHMARK(BM_GetWeekday); + +void BM_NextWeekday(benchmark::State& state) { + const cctz::civil_day kStart(2014, 8, 22); + const cctz::civil_day kDays[7] = { + kStart + 0, kStart + 1, kStart + 2, kStart + 3, + kStart + 4, kStart + 5, kStart + 6, + }; + const cctz::weekday kWeekdays[7] = { + cctz::weekday::monday, cctz::weekday::tuesday, cctz::weekday::wednesday, + cctz::weekday::thursday, cctz::weekday::friday, cctz::weekday::saturday, + cctz::weekday::sunday, + }; + while (state.KeepRunningBatch(7 * 7)) { + for (const auto from : kDays) { + for (const auto to : kWeekdays) { + benchmark::DoNotOptimize(cctz::next_weekday(from, to)); + } + } + } +} +BENCHMARK(BM_NextWeekday); + +void BM_PrevWeekday(benchmark::State& state) { + const cctz::civil_day kStart(2014, 8, 22); + const cctz::civil_day kDays[7] = { + kStart + 0, kStart + 1, kStart + 2, kStart + 3, + kStart + 4, kStart + 5, kStart + 6, + }; + const cctz::weekday kWeekdays[7] = { + cctz::weekday::monday, cctz::weekday::tuesday, cctz::weekday::wednesday, + cctz::weekday::thursday, cctz::weekday::friday, cctz::weekday::saturday, + cctz::weekday::sunday, + }; + while (state.KeepRunningBatch(7 * 7)) { + for (const auto from : kDays) { + for (const auto to : kWeekdays) { + benchmark::DoNotOptimize(cctz::prev_weekday(from, to)); + } + } + } +} +BENCHMARK(BM_PrevWeekday); + +const char RFC3339_full[] = "%Y-%m-%d%ET%H:%M:%E*S%Ez"; +const char RFC3339_sec[] = "%Y-%m-%d%ET%H:%M:%S%Ez"; + +const char RFC1123_full[] = "%a, %d %b %Y %H:%M:%S %z"; +const char RFC1123_no_wday[] = "%d %b %Y %H:%M:%S %z"; + +// A list of known time-zone names. +// TODO: Refactor with src/time_zone_lookup_test.cc. +const char* const kTimeZoneNames[] = {"Africa/Abidjan", + "Africa/Accra", + "Africa/Addis_Ababa", + "Africa/Algiers", + "Africa/Asmara", + "Africa/Asmera", + "Africa/Bamako", + "Africa/Bangui", + "Africa/Banjul", + "Africa/Bissau", + "Africa/Blantyre", + "Africa/Brazzaville", + "Africa/Bujumbura", + "Africa/Cairo", + "Africa/Casablanca", + "Africa/Ceuta", + "Africa/Conakry", + "Africa/Dakar", + "Africa/Dar_es_Salaam", + "Africa/Djibouti", + "Africa/Douala", + "Africa/El_Aaiun", + "Africa/Freetown", + "Africa/Gaborone", + "Africa/Harare", + "Africa/Johannesburg", + "Africa/Juba", + "Africa/Kampala", + "Africa/Khartoum", + "Africa/Kigali", + "Africa/Kinshasa", + "Africa/Lagos", + "Africa/Libreville", + "Africa/Lome", + "Africa/Luanda", + "Africa/Lubumbashi", + "Africa/Lusaka", + "Africa/Malabo", + "Africa/Maputo", + "Africa/Maseru", + "Africa/Mbabane", + "Africa/Mogadishu", + "Africa/Monrovia", + "Africa/Nairobi", + "Africa/Ndjamena", + "Africa/Niamey", + "Africa/Nouakchott", + "Africa/Ouagadougou", + "Africa/Porto-Novo", + "Africa/Sao_Tome", + "Africa/Timbuktu", + "Africa/Tripoli", + "Africa/Tunis", + "Africa/Windhoek", + "America/Adak", + "America/Anchorage", + "America/Anguilla", + "America/Antigua", + "America/Araguaina", + "America/Argentina/Buenos_Aires", + "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", + "America/Argentina/Cordoba", + "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", + "America/Argentina/Mendoza", + "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", + "America/Argentina/San_Juan", + "America/Argentina/San_Luis", + "America/Argentina/Tucuman", + "America/Argentina/Ushuaia", + "America/Aruba", + "America/Asuncion", + "America/Atikokan", + "America/Atka", + "America/Bahia", + "America/Bahia_Banderas", + "America/Barbados", + "America/Belem", + "America/Belize", + "America/Blanc-Sablon", + "America/Boa_Vista", + "America/Bogota", + "America/Boise", + "America/Buenos_Aires", + "America/Cambridge_Bay", + "America/Campo_Grande", + "America/Cancun", + "America/Caracas", + "America/Catamarca", + "America/Cayenne", + "America/Cayman", + "America/Chicago", + "America/Chihuahua", + "America/Coral_Harbour", + "America/Cordoba", + "America/Costa_Rica", + "America/Creston", + "America/Cuiaba", + "America/Curacao", + "America/Danmarkshavn", + "America/Dawson", + "America/Dawson_Creek", + "America/Denver", + "America/Detroit", + "America/Dominica", + "America/Edmonton", + "America/Eirunepe", + "America/El_Salvador", + "America/Ensenada", + "America/Fort_Nelson", + "America/Fort_Wayne", + "America/Fortaleza", + "America/Glace_Bay", + "America/Godthab", + "America/Goose_Bay", + "America/Grand_Turk", + "America/Grenada", + "America/Guadeloupe", + "America/Guatemala", + "America/Guayaquil", + "America/Guyana", + "America/Halifax", + "America/Havana", + "America/Hermosillo", + "America/Indiana/Indianapolis", + "America/Indiana/Knox", + "America/Indiana/Marengo", + "America/Indiana/Petersburg", + "America/Indiana/Tell_City", + "America/Indiana/Vevay", + "America/Indiana/Vincennes", + "America/Indiana/Winamac", + "America/Indianapolis", + "America/Inuvik", + "America/Iqaluit", + "America/Jamaica", + "America/Jujuy", + "America/Juneau", + "America/Kentucky/Louisville", + "America/Kentucky/Monticello", + "America/Knox_IN", + "America/Kralendijk", + "America/La_Paz", + "America/Lima", + "America/Los_Angeles", + "America/Louisville", + "America/Lower_Princes", + "America/Maceio", + "America/Managua", + "America/Manaus", + "America/Marigot", + "America/Martinique", + "America/Matamoros", + "America/Mazatlan", + "America/Mendoza", + "America/Menominee", + "America/Merida", + "America/Metlakatla", + "America/Mexico_City", + "America/Miquelon", + "America/Moncton", + "America/Monterrey", + "America/Montevideo", + "America/Montreal", + "America/Montserrat", + "America/Nassau", + "America/New_York", + "America/Nipigon", + "America/Nome", + "America/Noronha", + "America/North_Dakota/Beulah", + "America/North_Dakota/Center", + "America/North_Dakota/New_Salem", + "America/Nuuk", + "America/Ojinaga", + "America/Panama", + "America/Pangnirtung", + "America/Paramaribo", + "America/Phoenix", + "America/Port-au-Prince", + "America/Port_of_Spain", + "America/Porto_Acre", + "America/Porto_Velho", + "America/Puerto_Rico", + "America/Punta_Arenas", + "America/Rainy_River", + "America/Rankin_Inlet", + "America/Recife", + "America/Regina", + "America/Resolute", + "America/Rio_Branco", + "America/Rosario", + "America/Santa_Isabel", + "America/Santarem", + "America/Santiago", + "America/Santo_Domingo", + "America/Sao_Paulo", + "America/Scoresbysund", + "America/Shiprock", + "America/Sitka", + "America/St_Barthelemy", + "America/St_Johns", + "America/St_Kitts", + "America/St_Lucia", + "America/St_Thomas", + "America/St_Vincent", + "America/Swift_Current", + "America/Tegucigalpa", + "America/Thule", + "America/Thunder_Bay", + "America/Tijuana", + "America/Toronto", + "America/Tortola", + "America/Vancouver", + "America/Virgin", + "America/Whitehorse", + "America/Winnipeg", + "America/Yakutat", + "America/Yellowknife", + "Antarctica/Casey", + "Antarctica/Davis", + "Antarctica/DumontDUrville", + "Antarctica/Macquarie", + "Antarctica/Mawson", + "Antarctica/McMurdo", + "Antarctica/Palmer", + "Antarctica/Rothera", + "Antarctica/South_Pole", + "Antarctica/Syowa", + "Antarctica/Troll", + "Antarctica/Vostok", + "Arctic/Longyearbyen", + "Asia/Aden", + "Asia/Almaty", + "Asia/Amman", + "Asia/Anadyr", + "Asia/Aqtau", + "Asia/Aqtobe", + "Asia/Ashgabat", + "Asia/Ashkhabad", + "Asia/Atyrau", + "Asia/Baghdad", + "Asia/Bahrain", + "Asia/Baku", + "Asia/Bangkok", + "Asia/Barnaul", + "Asia/Beirut", + "Asia/Bishkek", + "Asia/Brunei", + "Asia/Calcutta", + "Asia/Chita", + "Asia/Choibalsan", + "Asia/Chongqing", + "Asia/Chungking", + "Asia/Colombo", + "Asia/Dacca", + "Asia/Damascus", + "Asia/Dhaka", + "Asia/Dili", + "Asia/Dubai", + "Asia/Dushanbe", + "Asia/Famagusta", + "Asia/Gaza", + "Asia/Harbin", + "Asia/Hebron", + "Asia/Ho_Chi_Minh", + "Asia/Hong_Kong", + "Asia/Hovd", + "Asia/Irkutsk", + "Asia/Istanbul", + "Asia/Jakarta", + "Asia/Jayapura", + "Asia/Jerusalem", + "Asia/Kabul", + "Asia/Kamchatka", + "Asia/Karachi", + "Asia/Kashgar", + "Asia/Kathmandu", + "Asia/Katmandu", + "Asia/Khandyga", + "Asia/Kolkata", + "Asia/Krasnoyarsk", + "Asia/Kuala_Lumpur", + "Asia/Kuching", + "Asia/Kuwait", + "Asia/Macao", + "Asia/Macau", + "Asia/Magadan", + "Asia/Makassar", + "Asia/Manila", + "Asia/Muscat", + "Asia/Nicosia", + "Asia/Novokuznetsk", + "Asia/Novosibirsk", + "Asia/Omsk", + "Asia/Oral", + "Asia/Phnom_Penh", + "Asia/Pontianak", + "Asia/Pyongyang", + "Asia/Qatar", + "Asia/Qostanay", + "Asia/Qyzylorda", + "Asia/Rangoon", + "Asia/Riyadh", + "Asia/Saigon", + "Asia/Sakhalin", + "Asia/Samarkand", + "Asia/Seoul", + "Asia/Shanghai", + "Asia/Singapore", + "Asia/Srednekolymsk", + "Asia/Taipei", + "Asia/Tashkent", + "Asia/Tbilisi", + "Asia/Tehran", + "Asia/Tel_Aviv", + "Asia/Thimbu", + "Asia/Thimphu", + "Asia/Tokyo", + "Asia/Tomsk", + "Asia/Ujung_Pandang", + "Asia/Ulaanbaatar", + "Asia/Ulan_Bator", + "Asia/Urumqi", + "Asia/Ust-Nera", + "Asia/Vientiane", + "Asia/Vladivostok", + "Asia/Yakutsk", + "Asia/Yangon", + "Asia/Yekaterinburg", + "Asia/Yerevan", + "Atlantic/Azores", + "Atlantic/Bermuda", + "Atlantic/Canary", + "Atlantic/Cape_Verde", + "Atlantic/Faeroe", + "Atlantic/Faroe", + "Atlantic/Jan_Mayen", + "Atlantic/Madeira", + "Atlantic/Reykjavik", + "Atlantic/South_Georgia", + "Atlantic/St_Helena", + "Atlantic/Stanley", + "Australia/ACT", + "Australia/Adelaide", + "Australia/Brisbane", + "Australia/Broken_Hill", + "Australia/Canberra", + "Australia/Currie", + "Australia/Darwin", + "Australia/Eucla", + "Australia/Hobart", + "Australia/LHI", + "Australia/Lindeman", + "Australia/Lord_Howe", + "Australia/Melbourne", + "Australia/NSW", + "Australia/North", + "Australia/Perth", + "Australia/Queensland", + "Australia/South", + "Australia/Sydney", + "Australia/Tasmania", + "Australia/Victoria", + "Australia/West", + "Australia/Yancowinna", + "Brazil/Acre", + "Brazil/DeNoronha", + "Brazil/East", + "Brazil/West", + "CET", + "CST6CDT", + "Canada/Atlantic", + "Canada/Central", + "Canada/Eastern", + "Canada/Mountain", + "Canada/Newfoundland", + "Canada/Pacific", + "Canada/Saskatchewan", + "Canada/Yukon", + "Chile/Continental", + "Chile/EasterIsland", + "Cuba", + "EET", + "EST", + "EST5EDT", + "Egypt", + "Eire", + "Etc/GMT", + "Etc/GMT+0", + "Etc/GMT+1", + "Etc/GMT+10", + "Etc/GMT+11", + "Etc/GMT+12", + "Etc/GMT+2", + "Etc/GMT+3", + "Etc/GMT+4", + "Etc/GMT+5", + "Etc/GMT+6", + "Etc/GMT+7", + "Etc/GMT+8", + "Etc/GMT+9", + "Etc/GMT-0", + "Etc/GMT-1", + "Etc/GMT-10", + "Etc/GMT-11", + "Etc/GMT-12", + "Etc/GMT-13", + "Etc/GMT-14", + "Etc/GMT-2", + "Etc/GMT-3", + "Etc/GMT-4", + "Etc/GMT-5", + "Etc/GMT-6", + "Etc/GMT-7", + "Etc/GMT-8", + "Etc/GMT-9", + "Etc/GMT0", + "Etc/Greenwich", + "Etc/UCT", + "Etc/UTC", + "Etc/Universal", + "Etc/Zulu", + "Europe/Amsterdam", + "Europe/Andorra", + "Europe/Astrakhan", + "Europe/Athens", + "Europe/Belfast", + "Europe/Belgrade", + "Europe/Berlin", + "Europe/Bratislava", + "Europe/Brussels", + "Europe/Bucharest", + "Europe/Budapest", + "Europe/Busingen", + "Europe/Chisinau", + "Europe/Copenhagen", + "Europe/Dublin", + "Europe/Gibraltar", + "Europe/Guernsey", + "Europe/Helsinki", + "Europe/Isle_of_Man", + "Europe/Istanbul", + "Europe/Jersey", + "Europe/Kaliningrad", + "Europe/Kiev", + "Europe/Kirov", + "Europe/Lisbon", + "Europe/Ljubljana", + "Europe/London", + "Europe/Luxembourg", + "Europe/Madrid", + "Europe/Malta", + "Europe/Mariehamn", + "Europe/Minsk", + "Europe/Monaco", + "Europe/Moscow", + "Europe/Nicosia", + "Europe/Oslo", + "Europe/Paris", + "Europe/Podgorica", + "Europe/Prague", + "Europe/Riga", + "Europe/Rome", + "Europe/Samara", + "Europe/San_Marino", + "Europe/Sarajevo", + "Europe/Saratov", + "Europe/Simferopol", + "Europe/Skopje", + "Europe/Sofia", + "Europe/Stockholm", + "Europe/Tallinn", + "Europe/Tirane", + "Europe/Tiraspol", + "Europe/Ulyanovsk", + "Europe/Uzhgorod", + "Europe/Vaduz", + "Europe/Vatican", + "Europe/Vienna", + "Europe/Vilnius", + "Europe/Volgograd", + "Europe/Warsaw", + "Europe/Zagreb", + "Europe/Zaporozhye", + "Europe/Zurich", + "GB", + "GB-Eire", + "GMT", + "GMT+0", + "GMT-0", + "GMT0", + "Greenwich", + "HST", + "Hongkong", + "Iceland", + "Indian/Antananarivo", + "Indian/Chagos", + "Indian/Christmas", + "Indian/Cocos", + "Indian/Comoro", + "Indian/Kerguelen", + "Indian/Mahe", + "Indian/Maldives", + "Indian/Mauritius", + "Indian/Mayotte", + "Indian/Reunion", + "Iran", + "Israel", + "Jamaica", + "Japan", + "Kwajalein", + "Libya", + "MET", + "MST", + "MST7MDT", + "Mexico/BajaNorte", + "Mexico/BajaSur", + "Mexico/General", + "NZ", + "NZ-CHAT", + "Navajo", + "PRC", + "PST8PDT", + "Pacific/Apia", + "Pacific/Auckland", + "Pacific/Bougainville", + "Pacific/Chatham", + "Pacific/Chuuk", + "Pacific/Easter", + "Pacific/Efate", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Fiji", + "Pacific/Funafuti", + "Pacific/Galapagos", + "Pacific/Gambier", + "Pacific/Guadalcanal", + "Pacific/Guam", + "Pacific/Honolulu", + "Pacific/Johnston", + "Pacific/Kiritimati", + "Pacific/Kosrae", + "Pacific/Kwajalein", + "Pacific/Majuro", + "Pacific/Marquesas", + "Pacific/Midway", + "Pacific/Nauru", + "Pacific/Niue", + "Pacific/Norfolk", + "Pacific/Noumea", + "Pacific/Pago_Pago", + "Pacific/Palau", + "Pacific/Pitcairn", + "Pacific/Pohnpei", + "Pacific/Ponape", + "Pacific/Port_Moresby", + "Pacific/Rarotonga", + "Pacific/Saipan", + "Pacific/Samoa", + "Pacific/Tahiti", + "Pacific/Tarawa", + "Pacific/Tongatapu", + "Pacific/Truk", + "Pacific/Wake", + "Pacific/Wallis", + "Pacific/Yap", + "Poland", + "Portugal", + "ROC", + "ROK", + "Singapore", + "Turkey", + "UCT", + "US/Alaska", + "US/Aleutian", + "US/Arizona", + "US/Central", + "US/East-Indiana", + "US/Eastern", + "US/Hawaii", + "US/Indiana-Starke", + "US/Michigan", + "US/Mountain", + "US/Pacific", + "US/Samoa", + "UTC", + "Universal", + "W-SU", + "WET", + "Zulu", + nullptr}; + +std::vector<std::string> AllTimeZoneNames() { + std::vector<std::string> names; + for (const char* const* namep = kTimeZoneNames; *namep != nullptr; ++namep) { + names.push_back(std::string("file:") + *namep); + } + assert(!names.empty()); + + std::mt19937 urbg(42); // a UniformRandomBitGenerator with fixed seed + std::shuffle(names.begin(), names.end(), urbg); + return names; +} + +cctz::time_zone TestTimeZone() { + cctz::time_zone tz; + cctz::load_time_zone("America/Los_Angeles", &tz); + return tz; +} + +void BM_Zone_LoadUTCTimeZoneFirst(benchmark::State& state) { + cctz::time_zone tz; + cctz::load_time_zone("UTC", &tz); // in case we're first + cctz::time_zone::Impl::ClearTimeZoneMapTestOnly(); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::load_time_zone("UTC", &tz)); + } +} +BENCHMARK(BM_Zone_LoadUTCTimeZoneFirst); + +void BM_Zone_LoadUTCTimeZoneLast(benchmark::State& state) { + cctz::time_zone tz; + for (const auto& name : AllTimeZoneNames()) { + cctz::load_time_zone(name, &tz); // prime cache + } + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::load_time_zone("UTC", &tz)); + } +} +BENCHMARK(BM_Zone_LoadUTCTimeZoneLast); + +void BM_Zone_LoadTimeZoneFirst(benchmark::State& state) { + cctz::time_zone tz = cctz::utc_time_zone(); // in case we're first + const std::string name = "file:America/Los_Angeles"; + while (state.KeepRunning()) { + state.PauseTiming(); + cctz::time_zone::Impl::ClearTimeZoneMapTestOnly(); + state.ResumeTiming(); + benchmark::DoNotOptimize(cctz::load_time_zone(name, &tz)); + } +} +BENCHMARK(BM_Zone_LoadTimeZoneFirst); + +void BM_Zone_LoadTimeZoneCached(benchmark::State& state) { + cctz::time_zone tz = cctz::utc_time_zone(); // in case we're first + cctz::time_zone::Impl::ClearTimeZoneMapTestOnly(); + const std::string name = "file:America/Los_Angeles"; + cctz::load_time_zone(name, &tz); // prime cache + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::load_time_zone(name, &tz)); + } +} +BENCHMARK(BM_Zone_LoadTimeZoneCached); + +void BM_Zone_LoadLocalTimeZoneCached(benchmark::State& state) { + cctz::utc_time_zone(); // in case we're first + cctz::time_zone::Impl::ClearTimeZoneMapTestOnly(); + cctz::local_time_zone(); // prime cache + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::local_time_zone()); + } +} +BENCHMARK(BM_Zone_LoadLocalTimeZoneCached); + +void BM_Zone_LoadAllTimeZonesFirst(benchmark::State& state) { + cctz::time_zone tz; + const std::vector<std::string> names = AllTimeZoneNames(); + for (auto index = names.size(); state.KeepRunning(); ++index) { + if (index == names.size()) { + index = 0; + } + if (index == 0) { + state.PauseTiming(); + cctz::time_zone::Impl::ClearTimeZoneMapTestOnly(); + state.ResumeTiming(); + } + benchmark::DoNotOptimize(cctz::load_time_zone(names[index], &tz)); + } +} +BENCHMARK(BM_Zone_LoadAllTimeZonesFirst); + +void BM_Zone_LoadAllTimeZonesCached(benchmark::State& state) { + cctz::time_zone tz; + const std::vector<std::string> names = AllTimeZoneNames(); + for (const auto& name : names) { + cctz::load_time_zone(name, &tz); // prime cache + } + for (auto index = names.size(); state.KeepRunning(); ++index) { + if (index == names.size()) { + index = 0; + } + benchmark::DoNotOptimize(cctz::load_time_zone(names[index], &tz)); + } +} +BENCHMARK(BM_Zone_LoadAllTimeZonesCached); + +void BM_Zone_TimeZoneEqualityImplicit(benchmark::State& state) { + cctz::time_zone tz; // implicit UTC + while (state.KeepRunning()) { + benchmark::DoNotOptimize(tz == tz); + } +} +BENCHMARK(BM_Zone_TimeZoneEqualityImplicit); + +void BM_Zone_TimeZoneEqualityExplicit(benchmark::State& state) { + cctz::time_zone tz = cctz::utc_time_zone(); // explicit UTC + while (state.KeepRunning()) { + benchmark::DoNotOptimize(tz == tz); + } +} +BENCHMARK(BM_Zone_TimeZoneEqualityExplicit); + +void BM_Zone_UTCTimeZone(benchmark::State& state) { + cctz::time_zone tz; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::utc_time_zone()); + } +} +BENCHMARK(BM_Zone_UTCTimeZone); + +// In each "ToCivil" benchmark we switch between two instants separated +// by at least one transition in order to defeat any internal caching of +// previous results (e.g., see local_time_hint_). +// +// The "UTC" variants use UTC instead of the Google/local time zone. + +void BM_Time_ToCivil_CCTZ(benchmark::State& state) { + const cctz::time_zone tz = TestTimeZone(); + std::chrono::system_clock::time_point tp = + std::chrono::system_clock::from_time_t(1384569027); + std::chrono::system_clock::time_point tp2 = + std::chrono::system_clock::from_time_t(1418962578); + while (state.KeepRunning()) { + std::swap(tp, tp2); + tp += std::chrono::seconds(1); + benchmark::DoNotOptimize(cctz::convert(tp, tz)); + } +} +BENCHMARK(BM_Time_ToCivil_CCTZ); + +void BM_Time_ToCivil_Libc(benchmark::State& state) { + // No timezone support, so just use localtime. + time_t t = 1384569027; + time_t t2 = 1418962578; + struct tm tm; + while (state.KeepRunning()) { + std::swap(t, t2); + t += 1; +#if defined(_WIN32) || defined(_WIN64) + benchmark::DoNotOptimize(localtime_s(&tm, &t)); +#else + benchmark::DoNotOptimize(localtime_r(&t, &tm)); +#endif + } +} +BENCHMARK(BM_Time_ToCivil_Libc); + +void BM_Time_ToCivilUTC_CCTZ(benchmark::State& state) { + const cctz::time_zone tz = cctz::utc_time_zone(); + std::chrono::system_clock::time_point tp = + std::chrono::system_clock::from_time_t(1384569027); + while (state.KeepRunning()) { + tp += std::chrono::seconds(1); + benchmark::DoNotOptimize(cctz::convert(tp, tz)); + } +} +BENCHMARK(BM_Time_ToCivilUTC_CCTZ); + +void BM_Time_ToCivilUTC_Libc(benchmark::State& state) { + time_t t = 1384569027; + struct tm tm; + while (state.KeepRunning()) { + t += 1; +#if defined(_WIN32) || defined(_WIN64) + benchmark::DoNotOptimize(gmtime_s(&tm, &t)); +#else + benchmark::DoNotOptimize(gmtime_r(&t, &tm)); +#endif + } +} +BENCHMARK(BM_Time_ToCivilUTC_Libc); + +// In each "FromCivil" benchmark we switch between two YMDhms values +// separated by at least one transition in order to defeat any internal +// caching of previous results (e.g., see time_local_hint_). +// +// The "UTC" variants use UTC instead of the Google/local time zone. +// The "Day0" variants require normalization of the day of month. + +void BM_Time_FromCivil_CCTZ(benchmark::State& state) { + const cctz::time_zone tz = TestTimeZone(); + int i = 0; + while (state.KeepRunning()) { + if ((i++ & 1) == 0) { + benchmark::DoNotOptimize( + cctz::convert(cctz::civil_second(2014, 12, 18, 20, 16, 18), tz)); + } else { + benchmark::DoNotOptimize( + cctz::convert(cctz::civil_second(2013, 11, 15, 18, 30, 27), tz)); + } + } +} +BENCHMARK(BM_Time_FromCivil_CCTZ); + +void BM_Time_FromCivil_Libc(benchmark::State& state) { + // No timezone support, so just use localtime. + int i = 0; + while (state.KeepRunning()) { + struct tm tm; + if ((i++ & 1) == 0) { + tm.tm_year = 2014 - 1900; + tm.tm_mon = 12 - 1; + tm.tm_mday = 18; + tm.tm_hour = 20; + tm.tm_min = 16; + tm.tm_sec = 18; + } else { + tm.tm_year = 2013 - 1900; + tm.tm_mon = 11 - 1; + tm.tm_mday = 15; + tm.tm_hour = 18; + tm.tm_min = 30; + tm.tm_sec = 27; + } + tm.tm_isdst = -1; + benchmark::DoNotOptimize(mktime(&tm)); + } +} +BENCHMARK(BM_Time_FromCivil_Libc); + +void BM_Time_FromCivilUTC_CCTZ(benchmark::State& state) { + const cctz::time_zone tz = cctz::utc_time_zone(); + while (state.KeepRunning()) { + benchmark::DoNotOptimize( + cctz::convert(cctz::civil_second(2014, 12, 18, 20, 16, 18), tz)); + } +} +BENCHMARK(BM_Time_FromCivilUTC_CCTZ); + +// There is no BM_Time_FromCivilUTC_Libc. + +void BM_Time_FromCivilDay0_CCTZ(benchmark::State& state) { + const cctz::time_zone tz = TestTimeZone(); + int i = 0; + while (state.KeepRunning()) { + if ((i++ & 1) == 0) { + benchmark::DoNotOptimize( + cctz::convert(cctz::civil_second(2014, 12, 0, 20, 16, 18), tz)); + } else { + benchmark::DoNotOptimize( + cctz::convert(cctz::civil_second(2013, 11, 0, 18, 30, 27), tz)); + } + } +} +BENCHMARK(BM_Time_FromCivilDay0_CCTZ); + +void BM_Time_FromCivilDay0_Libc(benchmark::State& state) { + // No timezone support, so just use localtime. + int i = 0; + while (state.KeepRunning()) { + struct tm tm; + if ((i++ & 1) == 0) { + tm.tm_year = 2014 - 1900; + tm.tm_mon = 12 - 1; + tm.tm_mday = 0; + tm.tm_hour = 20; + tm.tm_min = 16; + tm.tm_sec = 18; + } else { + tm.tm_year = 2013 - 1900; + tm.tm_mon = 11 - 1; + tm.tm_mday = 0; + tm.tm_hour = 18; + tm.tm_min = 30; + tm.tm_sec = 27; + } + tm.tm_isdst = -1; + benchmark::DoNotOptimize(mktime(&tm)); + } +} +BENCHMARK(BM_Time_FromCivilDay0_Libc); + +const char* const kFormats[] = { + RFC1123_full, // 0 + RFC1123_no_wday, // 1 + RFC3339_full, // 2 + RFC3339_sec, // 3 + "%Y-%m-%d%ET%H:%M:%S", // 4 + "%Y-%m-%d", // 5 +}; +const int kNumFormats = sizeof(kFormats) / sizeof(kFormats[0]); + +void BM_Format_FormatTime(benchmark::State& state) { + const std::string fmt = kFormats[state.range(0)]; + state.SetLabel(fmt); + const cctz::time_zone tz = TestTimeZone(); + const std::chrono::system_clock::time_point tp = + cctz::convert(cctz::civil_second(1977, 6, 28, 9, 8, 7), tz) + + std::chrono::microseconds(1); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::format(fmt, tp, tz)); + } +} +BENCHMARK(BM_Format_FormatTime)->DenseRange(0, kNumFormats - 1); + +void BM_Format_ParseTime(benchmark::State& state) { + const std::string fmt = kFormats[state.range(0)]; + state.SetLabel(fmt); + const cctz::time_zone tz = TestTimeZone(); + std::chrono::system_clock::time_point tp = + cctz::convert(cctz::civil_second(1977, 6, 28, 9, 8, 7), tz) + + std::chrono::microseconds(1); + const std::string when = cctz::format(fmt, tp, tz); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(cctz::parse(fmt, when, tz, &tp)); + } +} +BENCHMARK(BM_Format_ParseTime)->DenseRange(0, kNumFormats - 1); + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/civil_time_detail.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/civil_time_detail.cc new file mode 100644 index 000000000000..0b07e397e560 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/civil_time_detail.cc @@ -0,0 +1,94 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/internal/cctz/include/cctz/civil_time_detail.h" + +#include <iomanip> +#include <ostream> +#include <sstream> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { +namespace detail { + +// Output stream operators output a format matching YYYY-MM-DDThh:mm:ss, +// while omitting fields inferior to the type's alignment. For example, +// civil_day is formatted only as YYYY-MM-DD. +std::ostream& operator<<(std::ostream& os, const civil_year& y) { + std::stringstream ss; + ss << y.year(); // No padding. + return os << ss.str(); +} +std::ostream& operator<<(std::ostream& os, const civil_month& m) { + std::stringstream ss; + ss << civil_year(m) << '-'; + ss << std::setfill('0') << std::setw(2) << m.month(); + return os << ss.str(); +} +std::ostream& operator<<(std::ostream& os, const civil_day& d) { + std::stringstream ss; + ss << civil_month(d) << '-'; + ss << std::setfill('0') << std::setw(2) << d.day(); + return os << ss.str(); +} +std::ostream& operator<<(std::ostream& os, const civil_hour& h) { + std::stringstream ss; + ss << civil_day(h) << 'T'; + ss << std::setfill('0') << std::setw(2) << h.hour(); + return os << ss.str(); +} +std::ostream& operator<<(std::ostream& os, const civil_minute& m) { + std::stringstream ss; + ss << civil_hour(m) << ':'; + ss << std::setfill('0') << std::setw(2) << m.minute(); + return os << ss.str(); +} +std::ostream& operator<<(std::ostream& os, const civil_second& s) { + std::stringstream ss; + ss << civil_minute(s) << ':'; + ss << std::setfill('0') << std::setw(2) << s.second(); + return os << ss.str(); +} + +//////////////////////////////////////////////////////////////////////// + +std::ostream& operator<<(std::ostream& os, weekday wd) { + switch (wd) { + case weekday::monday: + return os << "Monday"; + case weekday::tuesday: + return os << "Tuesday"; + case weekday::wednesday: + return os << "Wednesday"; + case weekday::thursday: + return os << "Thursday"; + case weekday::friday: + return os << "Friday"; + case weekday::saturday: + return os << "Saturday"; + case weekday::sunday: + return os << "Sunday"; + } + return os; // Should never get here, but -Wreturn-type may warn without this. +} + +} // namespace detail +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/civil_time_test.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/civil_time_test.cc new file mode 100644 index 000000000000..a5a71230419f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/civil_time_test.cc @@ -0,0 +1,1066 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/internal/cctz/include/cctz/civil_time.h" + +#include <iomanip> +#include <limits> +#include <sstream> +#include <string> +#include <type_traits> + +#include "gtest/gtest.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +template <typename T> +std::string Format(const T& t) { + std::stringstream ss; + ss << t; + return ss.str(); +} + +} // namespace + +#if __cpp_constexpr >= 201304 || (defined(_MSC_VER) && _MSC_VER >= 1910) +// Construction constexpr tests + +TEST(CivilTime, Normal) { + constexpr civil_second css(2016, 1, 28, 17, 14, 12); + static_assert(css.second() == 12, "Normal.second"); + constexpr civil_minute cmm(2016, 1, 28, 17, 14); + static_assert(cmm.minute() == 14, "Normal.minute"); + constexpr civil_hour chh(2016, 1, 28, 17); + static_assert(chh.hour() == 17, "Normal.hour"); + constexpr civil_day cd(2016, 1, 28); + static_assert(cd.day() == 28, "Normal.day"); + constexpr civil_month cm(2016, 1); + static_assert(cm.month() == 1, "Normal.month"); + constexpr civil_year cy(2016); + static_assert(cy.year() == 2016, "Normal.year"); +} + +TEST(CivilTime, Conversion) { + constexpr civil_year cy(2016); + static_assert(cy.year() == 2016, "Conversion.year"); + constexpr civil_month cm(cy); + static_assert(cm.month() == 1, "Conversion.month"); + constexpr civil_day cd(cm); + static_assert(cd.day() == 1, "Conversion.day"); + constexpr civil_hour chh(cd); + static_assert(chh.hour() == 0, "Conversion.hour"); + constexpr civil_minute cmm(chh); + static_assert(cmm.minute() == 0, "Conversion.minute"); + constexpr civil_second css(cmm); + static_assert(css.second() == 0, "Conversion.second"); +} + +// Normalization constexpr tests + +TEST(CivilTime, Normalized) { + constexpr civil_second cs(2016, 1, 28, 17, 14, 12); + static_assert(cs.year() == 2016, "Normalized.year"); + static_assert(cs.month() == 1, "Normalized.month"); + static_assert(cs.day() == 28, "Normalized.day"); + static_assert(cs.hour() == 17, "Normalized.hour"); + static_assert(cs.minute() == 14, "Normalized.minute"); + static_assert(cs.second() == 12, "Normalized.second"); +} + +TEST(CivilTime, SecondOverflow) { + constexpr civil_second cs(2016, 1, 28, 17, 14, 121); + static_assert(cs.year() == 2016, "SecondOverflow.year"); + static_assert(cs.month() == 1, "SecondOverflow.month"); + static_assert(cs.day() == 28, "SecondOverflow.day"); + static_assert(cs.hour() == 17, "SecondOverflow.hour"); + static_assert(cs.minute() == 16, "SecondOverflow.minute"); + static_assert(cs.second() == 1, "SecondOverflow.second"); +} + +TEST(CivilTime, SecondUnderflow) { + constexpr civil_second cs(2016, 1, 28, 17, 14, -121); + static_assert(cs.year() == 2016, "SecondUnderflow.year"); + static_assert(cs.month() == 1, "SecondUnderflow.month"); + static_assert(cs.day() == 28, "SecondUnderflow.day"); + static_assert(cs.hour() == 17, "SecondUnderflow.hour"); + static_assert(cs.minute() == 11, "SecondUnderflow.minute"); + static_assert(cs.second() == 59, "SecondUnderflow.second"); +} + +TEST(CivilTime, MinuteOverflow) { + constexpr civil_second cs(2016, 1, 28, 17, 121, 12); + static_assert(cs.year() == 2016, "MinuteOverflow.year"); + static_assert(cs.month() == 1, "MinuteOverflow.month"); + static_assert(cs.day() == 28, "MinuteOverflow.day"); + static_assert(cs.hour() == 19, "MinuteOverflow.hour"); + static_assert(cs.minute() == 1, "MinuteOverflow.minute"); + static_assert(cs.second() == 12, "MinuteOverflow.second"); +} + +TEST(CivilTime, MinuteUnderflow) { + constexpr civil_second cs(2016, 1, 28, 17, -121, 12); + static_assert(cs.year() == 2016, "MinuteUnderflow.year"); + static_assert(cs.month() == 1, "MinuteUnderflow.month"); + static_assert(cs.day() == 28, "MinuteUnderflow.day"); + static_assert(cs.hour() == 14, "MinuteUnderflow.hour"); + static_assert(cs.minute() == 59, "MinuteUnderflow.minute"); + static_assert(cs.second() == 12, "MinuteUnderflow.second"); +} + +TEST(CivilTime, HourOverflow) { + constexpr civil_second cs(2016, 1, 28, 49, 14, 12); + static_assert(cs.year() == 2016, "HourOverflow.year"); + static_assert(cs.month() == 1, "HourOverflow.month"); + static_assert(cs.day() == 30, "HourOverflow.day"); + static_assert(cs.hour() == 1, "HourOverflow.hour"); + static_assert(cs.minute() == 14, "HourOverflow.minute"); + static_assert(cs.second() == 12, "HourOverflow.second"); +} + +TEST(CivilTime, HourUnderflow) { + constexpr civil_second cs(2016, 1, 28, -49, 14, 12); + static_assert(cs.year() == 2016, "HourUnderflow.year"); + static_assert(cs.month() == 1, "HourUnderflow.month"); + static_assert(cs.day() == 25, "HourUnderflow.day"); + static_assert(cs.hour() == 23, "HourUnderflow.hour"); + static_assert(cs.minute() == 14, "HourUnderflow.minute"); + static_assert(cs.second() == 12, "HourUnderflow.second"); +} + +TEST(CivilTime, MonthOverflow) { + constexpr civil_second cs(2016, 25, 28, 17, 14, 12); + static_assert(cs.year() == 2018, "MonthOverflow.year"); + static_assert(cs.month() == 1, "MonthOverflow.month"); + static_assert(cs.day() == 28, "MonthOverflow.day"); + static_assert(cs.hour() == 17, "MonthOverflow.hour"); + static_assert(cs.minute() == 14, "MonthOverflow.minute"); + static_assert(cs.second() == 12, "MonthOverflow.second"); +} + +TEST(CivilTime, MonthUnderflow) { + constexpr civil_second cs(2016, -25, 28, 17, 14, 12); + static_assert(cs.year() == 2013, "MonthUnderflow.year"); + static_assert(cs.month() == 11, "MonthUnderflow.month"); + static_assert(cs.day() == 28, "MonthUnderflow.day"); + static_assert(cs.hour() == 17, "MonthUnderflow.hour"); + static_assert(cs.minute() == 14, "MonthUnderflow.minute"); + static_assert(cs.second() == 12, "MonthUnderflow.second"); +} + +TEST(CivilTime, C4Overflow) { + constexpr civil_second cs(2016, 1, 292195, 17, 14, 12); + static_assert(cs.year() == 2816, "C4Overflow.year"); + static_assert(cs.month() == 1, "C4Overflow.month"); + static_assert(cs.day() == 1, "C4Overflow.day"); + static_assert(cs.hour() == 17, "C4Overflow.hour"); + static_assert(cs.minute() == 14, "C4Overflow.minute"); + static_assert(cs.second() == 12, "C4Overflow.second"); +} + +TEST(CivilTime, C4Underflow) { + constexpr civil_second cs(2016, 1, -292195, 17, 14, 12); + static_assert(cs.year() == 1215, "C4Underflow.year"); + static_assert(cs.month() == 12, "C4Underflow.month"); + static_assert(cs.day() == 30, "C4Underflow.day"); + static_assert(cs.hour() == 17, "C4Underflow.hour"); + static_assert(cs.minute() == 14, "C4Underflow.minute"); + static_assert(cs.second() == 12, "C4Underflow.second"); +} + +TEST(CivilTime, MixedNormalization) { + constexpr civil_second cs(2016, -42, 122, 99, -147, 4949); + static_assert(cs.year() == 2012, "MixedNormalization.year"); + static_assert(cs.month() == 10, "MixedNormalization.month"); + static_assert(cs.day() == 4, "MixedNormalization.day"); + static_assert(cs.hour() == 1, "MixedNormalization.hour"); + static_assert(cs.minute() == 55, "MixedNormalization.minute"); + static_assert(cs.second() == 29, "MixedNormalization.second"); +} + +// Relational constexpr tests + +TEST(CivilTime, Less) { + constexpr civil_second cs1(2016, 1, 28, 17, 14, 12); + constexpr civil_second cs2(2016, 1, 28, 17, 14, 13); + constexpr bool less = cs1 < cs2; + static_assert(less, "Less"); +} + +// Arithmetic constexpr tests + +TEST(CivilTime, Addition) { + constexpr civil_second cs1(2016, 1, 28, 17, 14, 12); + constexpr civil_second cs2 = cs1 + 50; + static_assert(cs2.year() == 2016, "Addition.year"); + static_assert(cs2.month() == 1, "Addition.month"); + static_assert(cs2.day() == 28, "Addition.day"); + static_assert(cs2.hour() == 17, "Addition.hour"); + static_assert(cs2.minute() == 15, "Addition.minute"); + static_assert(cs2.second() == 2, "Addition.second"); +} + +TEST(CivilTime, Subtraction) { + constexpr civil_second cs1(2016, 1, 28, 17, 14, 12); + constexpr civil_second cs2 = cs1 - 50; + static_assert(cs2.year() == 2016, "Subtraction.year"); + static_assert(cs2.month() == 1, "Subtraction.month"); + static_assert(cs2.day() == 28, "Subtraction.day"); + static_assert(cs2.hour() == 17, "Subtraction.hour"); + static_assert(cs2.minute() == 13, "Subtraction.minute"); + static_assert(cs2.second() == 22, "Subtraction.second"); +} + +TEST(CivilTime, Difference) { + constexpr civil_day cd1(2016, 1, 28); + constexpr civil_day cd2(2015, 1, 28); + constexpr int diff = cd1 - cd2; + static_assert(diff == 365, "Difference"); +} + +// NOTE: Run this with --copt=-ftrapv to detect overflow problems. +TEST(CivilTime, ConstructionWithHugeYear) { + constexpr civil_hour h(-9223372036854775807, 1, 1, -1); + static_assert(h.year() == -9223372036854775807 - 1, + "ConstructionWithHugeYear"); + static_assert(h.month() == 12, "ConstructionWithHugeYear"); + static_assert(h.day() == 31, "ConstructionWithHugeYear"); + static_assert(h.hour() == 23, "ConstructionWithHugeYear"); +} + +// NOTE: Run this with --copt=-ftrapv to detect overflow problems. +TEST(CivilTime, DifferenceWithHugeYear) { + { + constexpr civil_day d1(9223372036854775807, 1, 1); + constexpr civil_day d2(9223372036854775807, 12, 31); + static_assert(d2 - d1 == 364, "DifferenceWithHugeYear"); + } + { + constexpr civil_day d1(-9223372036854775807 - 1, 1, 1); + constexpr civil_day d2(-9223372036854775807 - 1, 12, 31); + static_assert(d2 - d1 == 365, "DifferenceWithHugeYear"); + } + { + // Check the limits of the return value at the end of the year range. + constexpr civil_day d1(9223372036854775807, 1, 1); + constexpr civil_day d2(9198119301927009252, 6, 6); + static_assert(d1 - d2 == 9223372036854775807, "DifferenceWithHugeYear"); + static_assert((d2 - 1) - d1 == -9223372036854775807 - 1, + "DifferenceWithHugeYear"); + } + { + // Check the limits of the return value at the start of the year range. + constexpr civil_day d1(-9223372036854775807 - 1, 1, 1); + constexpr civil_day d2(-9198119301927009254, 7, 28); + static_assert(d2 - d1 == 9223372036854775807, "DifferenceWithHugeYear"); + static_assert(d1 - (d2 + 1) == -9223372036854775807 - 1, + "DifferenceWithHugeYear"); + } + { + // Check the limits of the return value from either side of year 0. + constexpr civil_day d1(-12626367463883278, 9, 3); + constexpr civil_day d2(12626367463883277, 3, 28); + static_assert(d2 - d1 == 9223372036854775807, "DifferenceWithHugeYear"); + static_assert(d1 - (d2 + 1) == -9223372036854775807 - 1, + "DifferenceWithHugeYear"); + } +} + +// NOTE: Run this with --copt=-ftrapv to detect overflow problems. +TEST(CivilTime, DifferenceNoIntermediateOverflow) { + { + // The difference up to the minute field would be below the minimum + // diff_t, but the 52 extra seconds brings us back to the minimum. + constexpr civil_second s1(-292277022657, 1, 27, 8, 29 - 1, 52); + constexpr civil_second s2(1970, 1, 1, 0, 0 - 1, 0); + static_assert(s1 - s2 == -9223372036854775807 - 1, + "DifferenceNoIntermediateOverflow"); + } + { + // The difference up to the minute field would be above the maximum + // diff_t, but the -53 extra seconds brings us back to the maximum. + constexpr civil_second s1(292277026596, 12, 4, 15, 30, 7 - 7); + constexpr civil_second s2(1970, 1, 1, 0, 0, 0 - 7); + static_assert(s1 - s2 == 9223372036854775807, + "DifferenceNoIntermediateOverflow"); + } +} + +// Helper constexpr tests + +TEST(CivilTime, WeekDay) { + constexpr civil_day cd(2016, 1, 28); + constexpr weekday wd = get_weekday(cd); + static_assert(wd == weekday::thursday, "Weekday"); +} + +TEST(CivilTime, NextWeekDay) { + constexpr civil_day cd(2016, 1, 28); + constexpr civil_day next = next_weekday(cd, weekday::thursday); + static_assert(next.year() == 2016, "NextWeekDay.year"); + static_assert(next.month() == 2, "NextWeekDay.month"); + static_assert(next.day() == 4, "NextWeekDay.day"); +} + +TEST(CivilTime, PrevWeekDay) { + constexpr civil_day cd(2016, 1, 28); + constexpr civil_day prev = prev_weekday(cd, weekday::thursday); + static_assert(prev.year() == 2016, "PrevWeekDay.year"); + static_assert(prev.month() == 1, "PrevWeekDay.month"); + static_assert(prev.day() == 21, "PrevWeekDay.day"); +} + +TEST(CivilTime, YearDay) { + constexpr civil_day cd(2016, 1, 28); + constexpr int yd = get_yearday(cd); + static_assert(yd == 28, "YearDay"); +} +#endif // __cpp_constexpr >= 201304 || (defined(_MSC_VER) && _MSC_VER >= 1910) + +// The remaining tests do not use constexpr. + +TEST(CivilTime, DefaultConstruction) { + civil_second ss; + EXPECT_EQ("1970-01-01T00:00:00", Format(ss)); + + civil_minute mm; + EXPECT_EQ("1970-01-01T00:00", Format(mm)); + + civil_hour hh; + EXPECT_EQ("1970-01-01T00", Format(hh)); + + civil_day d; + EXPECT_EQ("1970-01-01", Format(d)); + + civil_month m; + EXPECT_EQ("1970-01", Format(m)); + + civil_year y; + EXPECT_EQ("1970", Format(y)); +} + +TEST(CivilTime, StructMember) { + struct S { + civil_day day; + }; + S s = {}; + EXPECT_EQ(civil_day{}, s.day); +} + +TEST(CivilTime, FieldsConstruction) { + EXPECT_EQ("2015-01-02T03:04:05", Format(civil_second(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02T03:04:00", Format(civil_second(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02T03:00:00", Format(civil_second(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02T00:00:00", Format(civil_second(2015, 1, 2))); + EXPECT_EQ("2015-01-01T00:00:00", Format(civil_second(2015, 1))); + EXPECT_EQ("2015-01-01T00:00:00", Format(civil_second(2015))); + + EXPECT_EQ("2015-01-02T03:04", Format(civil_minute(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02T03:04", Format(civil_minute(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02T03:00", Format(civil_minute(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02T00:00", Format(civil_minute(2015, 1, 2))); + EXPECT_EQ("2015-01-01T00:00", Format(civil_minute(2015, 1))); + EXPECT_EQ("2015-01-01T00:00", Format(civil_minute(2015))); + + EXPECT_EQ("2015-01-02T03", Format(civil_hour(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02T03", Format(civil_hour(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02T03", Format(civil_hour(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02T00", Format(civil_hour(2015, 1, 2))); + EXPECT_EQ("2015-01-01T00", Format(civil_hour(2015, 1))); + EXPECT_EQ("2015-01-01T00", Format(civil_hour(2015))); + + EXPECT_EQ("2015-01-02", Format(civil_day(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01-02", Format(civil_day(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01-02", Format(civil_day(2015, 1, 2, 3))); + EXPECT_EQ("2015-01-02", Format(civil_day(2015, 1, 2))); + EXPECT_EQ("2015-01-01", Format(civil_day(2015, 1))); + EXPECT_EQ("2015-01-01", Format(civil_day(2015))); + + EXPECT_EQ("2015-01", Format(civil_month(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015-01", Format(civil_month(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015-01", Format(civil_month(2015, 1, 2, 3))); + EXPECT_EQ("2015-01", Format(civil_month(2015, 1, 2))); + EXPECT_EQ("2015-01", Format(civil_month(2015, 1))); + EXPECT_EQ("2015-01", Format(civil_month(2015))); + + EXPECT_EQ("2015", Format(civil_year(2015, 1, 2, 3, 4, 5))); + EXPECT_EQ("2015", Format(civil_year(2015, 1, 2, 3, 4))); + EXPECT_EQ("2015", Format(civil_year(2015, 1, 2, 3))); + EXPECT_EQ("2015", Format(civil_year(2015, 1, 2))); + EXPECT_EQ("2015", Format(civil_year(2015, 1))); + EXPECT_EQ("2015", Format(civil_year(2015))); +} + +TEST(CivilTime, FieldsConstructionLimits) { + const int kIntMax = std::numeric_limits<int>::max(); + EXPECT_EQ("2038-01-19T03:14:07", + Format(civil_second(1970, 1, 1, 0, 0, kIntMax))); + EXPECT_EQ("6121-02-11T05:21:07", + Format(civil_second(1970, 1, 1, 0, kIntMax, kIntMax))); + EXPECT_EQ("251104-11-20T12:21:07", + Format(civil_second(1970, 1, 1, kIntMax, kIntMax, kIntMax))); + EXPECT_EQ("6130715-05-30T12:21:07", + Format(civil_second(1970, 1, kIntMax, kIntMax, kIntMax, kIntMax))); + EXPECT_EQ( + "185087685-11-26T12:21:07", + Format(civil_second(1970, kIntMax, kIntMax, kIntMax, kIntMax, kIntMax))); + + const int kIntMin = std::numeric_limits<int>::min(); + EXPECT_EQ("1901-12-13T20:45:52", + Format(civil_second(1970, 1, 1, 0, 0, kIntMin))); + EXPECT_EQ("-2182-11-20T18:37:52", + Format(civil_second(1970, 1, 1, 0, kIntMin, kIntMin))); + EXPECT_EQ("-247165-02-11T10:37:52", + Format(civil_second(1970, 1, 1, kIntMin, kIntMin, kIntMin))); + EXPECT_EQ("-6126776-08-01T10:37:52", + Format(civil_second(1970, 1, kIntMin, kIntMin, kIntMin, kIntMin))); + EXPECT_EQ( + "-185083747-10-31T10:37:52", + Format(civil_second(1970, kIntMin, kIntMin, kIntMin, kIntMin, kIntMin))); +} + +TEST(CivilTime, ImplicitCrossAlignment) { + civil_year year(2015); + civil_month month = year; + civil_day day = month; + civil_hour hour = day; + civil_minute minute = hour; + civil_second second = minute; + + second = year; + EXPECT_EQ(second, year); + second = month; + EXPECT_EQ(second, month); + second = day; + EXPECT_EQ(second, day); + second = hour; + EXPECT_EQ(second, hour); + second = minute; + EXPECT_EQ(second, minute); + + minute = year; + EXPECT_EQ(minute, year); + minute = month; + EXPECT_EQ(minute, month); + minute = day; + EXPECT_EQ(minute, day); + minute = hour; + EXPECT_EQ(minute, hour); + + hour = year; + EXPECT_EQ(hour, year); + hour = month; + EXPECT_EQ(hour, month); + hour = day; + EXPECT_EQ(hour, day); + + day = year; + EXPECT_EQ(day, year); + day = month; + EXPECT_EQ(day, month); + + month = year; + EXPECT_EQ(month, year); + + // Ensures unsafe conversions are not allowed. + EXPECT_FALSE((std::is_convertible<civil_second, civil_minute>::value)); + EXPECT_FALSE((std::is_convertible<civil_second, civil_hour>::value)); + EXPECT_FALSE((std::is_convertible<civil_second, civil_day>::value)); + EXPECT_FALSE((std::is_convertible<civil_second, civil_month>::value)); + EXPECT_FALSE((std::is_convertible<civil_second, civil_year>::value)); + + EXPECT_FALSE((std::is_convertible<civil_minute, civil_hour>::value)); + EXPECT_FALSE((std::is_convertible<civil_minute, civil_day>::value)); + EXPECT_FALSE((std::is_convertible<civil_minute, civil_month>::value)); + EXPECT_FALSE((std::is_convertible<civil_minute, civil_year>::value)); + + EXPECT_FALSE((std::is_convertible<civil_hour, civil_day>::value)); + EXPECT_FALSE((std::is_convertible<civil_hour, civil_month>::value)); + EXPECT_FALSE((std::is_convertible<civil_hour, civil_year>::value)); + + EXPECT_FALSE((std::is_convertible<civil_day, civil_month>::value)); + EXPECT_FALSE((std::is_convertible<civil_day, civil_year>::value)); + + EXPECT_FALSE((std::is_convertible<civil_month, civil_year>::value)); +} + +TEST(CivilTime, ExplicitCrossAlignment) { + // + // Assign from smaller units -> larger units + // + + civil_second second(2015, 1, 2, 3, 4, 5); + EXPECT_EQ("2015-01-02T03:04:05", Format(second)); + + civil_minute minute(second); + EXPECT_EQ("2015-01-02T03:04", Format(minute)); + + civil_hour hour(minute); + EXPECT_EQ("2015-01-02T03", Format(hour)); + + civil_day day(hour); + EXPECT_EQ("2015-01-02", Format(day)); + + civil_month month(day); + EXPECT_EQ("2015-01", Format(month)); + + civil_year year(month); + EXPECT_EQ("2015", Format(year)); + + // + // Now assign from larger units -> smaller units + // + + month = civil_month(year); + EXPECT_EQ("2015-01", Format(month)); + + day = civil_day(month); + EXPECT_EQ("2015-01-01", Format(day)); + + hour = civil_hour(day); + EXPECT_EQ("2015-01-01T00", Format(hour)); + + minute = civil_minute(hour); + EXPECT_EQ("2015-01-01T00:00", Format(minute)); + + second = civil_second(minute); + EXPECT_EQ("2015-01-01T00:00:00", Format(second)); +} + +// Metafunction to test whether difference is allowed between two types. +template <typename T1, typename T2> +struct HasDifference { + template <typename U1, typename U2> + static std::false_type test(...); + template <typename U1, typename U2> + static std::true_type test(decltype(std::declval<U1>() - std::declval<U2>())); + static constexpr bool value = decltype(test<T1, T2>(0))::value; +}; + +TEST(CivilTime, DisallowCrossAlignedDifference) { + // Difference is allowed between types with the same alignment. + static_assert(HasDifference<civil_second, civil_second>::value, ""); + static_assert(HasDifference<civil_minute, civil_minute>::value, ""); + static_assert(HasDifference<civil_hour, civil_hour>::value, ""); + static_assert(HasDifference<civil_day, civil_day>::value, ""); + static_assert(HasDifference<civil_month, civil_month>::value, ""); + static_assert(HasDifference<civil_year, civil_year>::value, ""); + + // Difference is disallowed between types with different alignments. + static_assert(!HasDifference<civil_second, civil_minute>::value, ""); + static_assert(!HasDifference<civil_second, civil_hour>::value, ""); + static_assert(!HasDifference<civil_second, civil_day>::value, ""); + static_assert(!HasDifference<civil_second, civil_month>::value, ""); + static_assert(!HasDifference<civil_second, civil_year>::value, ""); + + static_assert(!HasDifference<civil_minute, civil_hour>::value, ""); + static_assert(!HasDifference<civil_minute, civil_day>::value, ""); + static_assert(!HasDifference<civil_minute, civil_month>::value, ""); + static_assert(!HasDifference<civil_minute, civil_year>::value, ""); + + static_assert(!HasDifference<civil_hour, civil_day>::value, ""); + static_assert(!HasDifference<civil_hour, civil_month>::value, ""); + static_assert(!HasDifference<civil_hour, civil_year>::value, ""); + + static_assert(!HasDifference<civil_day, civil_month>::value, ""); + static_assert(!HasDifference<civil_day, civil_year>::value, ""); + + static_assert(!HasDifference<civil_month, civil_year>::value, ""); +} + +TEST(CivilTime, ValueSemantics) { + const civil_hour a(2015, 1, 2, 3); + const civil_hour b = a; + const civil_hour c(b); + civil_hour d; + d = c; + EXPECT_EQ("2015-01-02T03", Format(d)); +} + +TEST(CivilTime, Relational) { + // Tests that the alignment unit is ignored in comparison. + const civil_year year(2014); + const civil_month month(year); + EXPECT_EQ(year, month); + +#define TEST_RELATIONAL(OLDER, YOUNGER) \ + do { \ + EXPECT_FALSE(OLDER < OLDER); \ + EXPECT_FALSE(OLDER > OLDER); \ + EXPECT_TRUE(OLDER >= OLDER); \ + EXPECT_TRUE(OLDER <= OLDER); \ + EXPECT_FALSE(YOUNGER < YOUNGER); \ + EXPECT_FALSE(YOUNGER > YOUNGER); \ + EXPECT_TRUE(YOUNGER >= YOUNGER); \ + EXPECT_TRUE(YOUNGER <= YOUNGER); \ + EXPECT_EQ(OLDER, OLDER); \ + EXPECT_NE(OLDER, YOUNGER); \ + EXPECT_LT(OLDER, YOUNGER); \ + EXPECT_LE(OLDER, YOUNGER); \ + EXPECT_GT(YOUNGER, OLDER); \ + EXPECT_GE(YOUNGER, OLDER); \ + } while (0) + + // Alignment is ignored in comparison (verified above), so kSecond is used + // to test comparison in all field positions. + TEST_RELATIONAL(civil_second(2014, 1, 1, 0, 0, 0), + civil_second(2015, 1, 1, 0, 0, 0)); + TEST_RELATIONAL(civil_second(2014, 1, 1, 0, 0, 0), + civil_second(2014, 2, 1, 0, 0, 0)); + TEST_RELATIONAL(civil_second(2014, 1, 1, 0, 0, 0), + civil_second(2014, 1, 2, 0, 0, 0)); + TEST_RELATIONAL(civil_second(2014, 1, 1, 0, 0, 0), + civil_second(2014, 1, 1, 1, 0, 0)); + TEST_RELATIONAL(civil_second(2014, 1, 1, 1, 0, 0), + civil_second(2014, 1, 1, 1, 1, 0)); + TEST_RELATIONAL(civil_second(2014, 1, 1, 1, 1, 0), + civil_second(2014, 1, 1, 1, 1, 1)); + + // Tests the relational operators of two different civil-time types. + TEST_RELATIONAL(civil_day(2014, 1, 1), civil_minute(2014, 1, 1, 1, 1)); + TEST_RELATIONAL(civil_day(2014, 1, 1), civil_month(2014, 2)); + +#undef TEST_RELATIONAL +} + +TEST(CivilTime, Arithmetic) { + civil_second second(2015, 1, 2, 3, 4, 5); + EXPECT_EQ("2015-01-02T03:04:06", Format(second += 1)); + EXPECT_EQ("2015-01-02T03:04:07", Format(second + 1)); + EXPECT_EQ("2015-01-02T03:04:08", Format(2 + second)); + EXPECT_EQ("2015-01-02T03:04:05", Format(second - 1)); + EXPECT_EQ("2015-01-02T03:04:05", Format(second -= 1)); + EXPECT_EQ("2015-01-02T03:04:05", Format(second++)); + EXPECT_EQ("2015-01-02T03:04:07", Format(++second)); + EXPECT_EQ("2015-01-02T03:04:07", Format(second--)); + EXPECT_EQ("2015-01-02T03:04:05", Format(--second)); + + civil_minute minute(2015, 1, 2, 3, 4); + EXPECT_EQ("2015-01-02T03:05", Format(minute += 1)); + EXPECT_EQ("2015-01-02T03:06", Format(minute + 1)); + EXPECT_EQ("2015-01-02T03:07", Format(2 + minute)); + EXPECT_EQ("2015-01-02T03:04", Format(minute - 1)); + EXPECT_EQ("2015-01-02T03:04", Format(minute -= 1)); + EXPECT_EQ("2015-01-02T03:04", Format(minute++)); + EXPECT_EQ("2015-01-02T03:06", Format(++minute)); + EXPECT_EQ("2015-01-02T03:06", Format(minute--)); + EXPECT_EQ("2015-01-02T03:04", Format(--minute)); + + civil_hour hour(2015, 1, 2, 3); + EXPECT_EQ("2015-01-02T04", Format(hour += 1)); + EXPECT_EQ("2015-01-02T05", Format(hour + 1)); + EXPECT_EQ("2015-01-02T06", Format(2 + hour)); + EXPECT_EQ("2015-01-02T03", Format(hour - 1)); + EXPECT_EQ("2015-01-02T03", Format(hour -= 1)); + EXPECT_EQ("2015-01-02T03", Format(hour++)); + EXPECT_EQ("2015-01-02T05", Format(++hour)); + EXPECT_EQ("2015-01-02T05", Format(hour--)); + EXPECT_EQ("2015-01-02T03", Format(--hour)); + + civil_day day(2015, 1, 2); + EXPECT_EQ("2015-01-03", Format(day += 1)); + EXPECT_EQ("2015-01-04", Format(day + 1)); + EXPECT_EQ("2015-01-05", Format(2 + day)); + EXPECT_EQ("2015-01-02", Format(day - 1)); + EXPECT_EQ("2015-01-02", Format(day -= 1)); + EXPECT_EQ("2015-01-02", Format(day++)); + EXPECT_EQ("2015-01-04", Format(++day)); + EXPECT_EQ("2015-01-04", Format(day--)); + EXPECT_EQ("2015-01-02", Format(--day)); + + civil_month month(2015, 1); + EXPECT_EQ("2015-02", Format(month += 1)); + EXPECT_EQ("2015-03", Format(month + 1)); + EXPECT_EQ("2015-04", Format(2 + month)); + EXPECT_EQ("2015-01", Format(month - 1)); + EXPECT_EQ("2015-01", Format(month -= 1)); + EXPECT_EQ("2015-01", Format(month++)); + EXPECT_EQ("2015-03", Format(++month)); + EXPECT_EQ("2015-03", Format(month--)); + EXPECT_EQ("2015-01", Format(--month)); + + civil_year year(2015); + EXPECT_EQ("2016", Format(year += 1)); + EXPECT_EQ("2017", Format(year + 1)); + EXPECT_EQ("2018", Format(2 + year)); + EXPECT_EQ("2015", Format(year - 1)); + EXPECT_EQ("2015", Format(year -= 1)); + EXPECT_EQ("2015", Format(year++)); + EXPECT_EQ("2017", Format(++year)); + EXPECT_EQ("2017", Format(year--)); + EXPECT_EQ("2015", Format(--year)); +} + +TEST(CivilTime, ArithmeticLimits) { + const int kIntMax = std::numeric_limits<int>::max(); + const int kIntMin = std::numeric_limits<int>::min(); + + civil_second second(1970, 1, 1, 0, 0, 0); + second += kIntMax; + EXPECT_EQ("2038-01-19T03:14:07", Format(second)); + second -= kIntMax; + EXPECT_EQ("1970-01-01T00:00:00", Format(second)); + second += kIntMin; + EXPECT_EQ("1901-12-13T20:45:52", Format(second)); + second -= kIntMin; + EXPECT_EQ("1970-01-01T00:00:00", Format(second)); + + civil_minute minute(1970, 1, 1, 0, 0); + minute += kIntMax; + EXPECT_EQ("6053-01-23T02:07", Format(minute)); + minute -= kIntMax; + EXPECT_EQ("1970-01-01T00:00", Format(minute)); + minute += kIntMin; + EXPECT_EQ("-2114-12-08T21:52", Format(minute)); + minute -= kIntMin; + EXPECT_EQ("1970-01-01T00:00", Format(minute)); + + civil_hour hour(1970, 1, 1, 0); + hour += kIntMax; + EXPECT_EQ("246953-10-09T07", Format(hour)); + hour -= kIntMax; + EXPECT_EQ("1970-01-01T00", Format(hour)); + hour += kIntMin; + EXPECT_EQ("-243014-03-24T16", Format(hour)); + hour -= kIntMin; + EXPECT_EQ("1970-01-01T00", Format(hour)); + + civil_day day(1970, 1, 1); + day += kIntMax; + EXPECT_EQ("5881580-07-11", Format(day)); + day -= kIntMax; + EXPECT_EQ("1970-01-01", Format(day)); + day += kIntMin; + EXPECT_EQ("-5877641-06-23", Format(day)); + day -= kIntMin; + EXPECT_EQ("1970-01-01", Format(day)); + + civil_month month(1970, 1); + month += kIntMax; + EXPECT_EQ("178958940-08", Format(month)); + month -= kIntMax; + EXPECT_EQ("1970-01", Format(month)); + month += kIntMin; + EXPECT_EQ("-178955001-05", Format(month)); + month -= kIntMin; + EXPECT_EQ("1970-01", Format(month)); + + civil_year year(0); + year += kIntMax; + EXPECT_EQ("2147483647", Format(year)); + year -= kIntMax; + EXPECT_EQ("0", Format(year)); + year += kIntMin; + EXPECT_EQ("-2147483648", Format(year)); + year -= kIntMin; + EXPECT_EQ("0", Format(year)); +} + +TEST(CivilTime, ArithmeticDifference) { + civil_second second(2015, 1, 2, 3, 4, 5); + EXPECT_EQ(0, second - second); + EXPECT_EQ(10, (second + 10) - second); + EXPECT_EQ(-10, (second - 10) - second); + + civil_minute minute(2015, 1, 2, 3, 4); + EXPECT_EQ(0, minute - minute); + EXPECT_EQ(10, (minute + 10) - minute); + EXPECT_EQ(-10, (minute - 10) - minute); + + civil_hour hour(2015, 1, 2, 3); + EXPECT_EQ(0, hour - hour); + EXPECT_EQ(10, (hour + 10) - hour); + EXPECT_EQ(-10, (hour - 10) - hour); + + civil_day day(2015, 1, 2); + EXPECT_EQ(0, day - day); + EXPECT_EQ(10, (day + 10) - day); + EXPECT_EQ(-10, (day - 10) - day); + + civil_month month(2015, 1); + EXPECT_EQ(0, month - month); + EXPECT_EQ(10, (month + 10) - month); + EXPECT_EQ(-10, (month - 10) - month); + + civil_year year(2015); + EXPECT_EQ(0, year - year); + EXPECT_EQ(10, (year + 10) - year); + EXPECT_EQ(-10, (year - 10) - year); +} + +TEST(CivilTime, DifferenceLimits) { + const int kIntMax = std::numeric_limits<int>::max(); + const int kIntMin = std::numeric_limits<int>::min(); + + // Check day arithmetic at the end of the year range. + const civil_day max_day(kIntMax, 12, 31); + EXPECT_EQ(1, max_day - (max_day - 1)); + EXPECT_EQ(-1, (max_day - 1) - max_day); + + // Check day arithmetic at the end of the year range. + const civil_day min_day(kIntMin, 1, 1); + EXPECT_EQ(1, (min_day + 1) - min_day); + EXPECT_EQ(-1, min_day - (min_day + 1)); + + // Check the limits of the return value. + const civil_day d1(1970, 1, 1); + const civil_day d2(5881580, 7, 11); + EXPECT_EQ(kIntMax, d2 - d1); + EXPECT_EQ(kIntMin, d1 - (d2 + 1)); +} + +TEST(CivilTime, Properties) { + civil_second ss(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, ss.year()); + EXPECT_EQ(2, ss.month()); + EXPECT_EQ(3, ss.day()); + EXPECT_EQ(4, ss.hour()); + EXPECT_EQ(5, ss.minute()); + EXPECT_EQ(6, ss.second()); + EXPECT_EQ(weekday::tuesday, get_weekday(ss)); + EXPECT_EQ(34, get_yearday(ss)); + + civil_minute mm(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, mm.year()); + EXPECT_EQ(2, mm.month()); + EXPECT_EQ(3, mm.day()); + EXPECT_EQ(4, mm.hour()); + EXPECT_EQ(5, mm.minute()); + EXPECT_EQ(0, mm.second()); + EXPECT_EQ(weekday::tuesday, get_weekday(mm)); + EXPECT_EQ(34, get_yearday(mm)); + + civil_hour hh(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, hh.year()); + EXPECT_EQ(2, hh.month()); + EXPECT_EQ(3, hh.day()); + EXPECT_EQ(4, hh.hour()); + EXPECT_EQ(0, hh.minute()); + EXPECT_EQ(0, hh.second()); + EXPECT_EQ(weekday::tuesday, get_weekday(hh)); + EXPECT_EQ(34, get_yearday(hh)); + + civil_day d(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, d.year()); + EXPECT_EQ(2, d.month()); + EXPECT_EQ(3, d.day()); + EXPECT_EQ(0, d.hour()); + EXPECT_EQ(0, d.minute()); + EXPECT_EQ(0, d.second()); + EXPECT_EQ(weekday::tuesday, get_weekday(d)); + EXPECT_EQ(34, get_yearday(d)); + + civil_month m(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, m.year()); + EXPECT_EQ(2, m.month()); + EXPECT_EQ(1, m.day()); + EXPECT_EQ(0, m.hour()); + EXPECT_EQ(0, m.minute()); + EXPECT_EQ(0, m.second()); + EXPECT_EQ(weekday::sunday, get_weekday(m)); + EXPECT_EQ(32, get_yearday(m)); + + civil_year y(2015, 2, 3, 4, 5, 6); + EXPECT_EQ(2015, y.year()); + EXPECT_EQ(1, y.month()); + EXPECT_EQ(1, y.day()); + EXPECT_EQ(0, y.hour()); + EXPECT_EQ(0, y.minute()); + EXPECT_EQ(0, y.second()); + EXPECT_EQ(weekday::thursday, get_weekday(y)); + EXPECT_EQ(1, get_yearday(y)); +} + +TEST(CivilTime, OutputStream) { + // Tests formatting of civil_year, which does not pad. + EXPECT_EQ("2016", Format(civil_year(2016))); + EXPECT_EQ("123", Format(civil_year(123))); + EXPECT_EQ("0", Format(civil_year(0))); + EXPECT_EQ("-1", Format(civil_year(-1))); + + // Tests formatting of sub-year types, which pad to 2 digits + EXPECT_EQ("2016-02", Format(civil_month(2016, 2))); + EXPECT_EQ("2016-02-03", Format(civil_day(2016, 2, 3))); + EXPECT_EQ("2016-02-03T04", Format(civil_hour(2016, 2, 3, 4))); + EXPECT_EQ("2016-02-03T04:05", Format(civil_minute(2016, 2, 3, 4, 5))); + EXPECT_EQ("2016-02-03T04:05:06", Format(civil_second(2016, 2, 3, 4, 5, 6))); + + // Tests formatting of weekday. + EXPECT_EQ("Monday", Format(weekday::monday)); + EXPECT_EQ("Tuesday", Format(weekday::tuesday)); + EXPECT_EQ("Wednesday", Format(weekday::wednesday)); + EXPECT_EQ("Thursday", Format(weekday::thursday)); + EXPECT_EQ("Friday", Format(weekday::friday)); + EXPECT_EQ("Saturday", Format(weekday::saturday)); + EXPECT_EQ("Sunday", Format(weekday::sunday)); +} + +TEST(CivilTime, OutputStreamLeftFillWidth) { + civil_second cs(2016, 2, 3, 4, 5, 6); + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << civil_year(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016.................X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << civil_month(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02..............X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << civil_day(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03...........X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << civil_hour(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03T04........X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << civil_minute(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03T04:05.....X..", ss.str()); + } + { + std::stringstream ss; + ss << std::left << std::setfill('.'); + ss << std::setw(3) << 'X'; + ss << std::setw(21) << civil_second(cs); + ss << std::setw(3) << 'X'; + EXPECT_EQ("X..2016-02-03T04:05:06..X..", ss.str()); + } +} + +TEST(CivilTime, NextPrevWeekday) { + // Jan 1, 1970 was a Thursday. + const civil_day thursday(1970, 1, 1); + EXPECT_EQ(weekday::thursday, get_weekday(thursday)); + + // Thursday -> Thursday + civil_day d = next_weekday(thursday, weekday::thursday); + EXPECT_EQ(7, d - thursday) << Format(d); + EXPECT_EQ(d - 14, prev_weekday(thursday, weekday::thursday)); + + // Thursday -> Friday + d = next_weekday(thursday, weekday::friday); + EXPECT_EQ(1, d - thursday) << Format(d); + EXPECT_EQ(d - 7, prev_weekday(thursday, weekday::friday)); + + // Thursday -> Saturday + d = next_weekday(thursday, weekday::saturday); + EXPECT_EQ(2, d - thursday) << Format(d); + EXPECT_EQ(d - 7, prev_weekday(thursday, weekday::saturday)); + + // Thursday -> Sunday + d = next_weekday(thursday, weekday::sunday); + EXPECT_EQ(3, d - thursday) << Format(d); + EXPECT_EQ(d - 7, prev_weekday(thursday, weekday::sunday)); + + // Thursday -> Monday + d = next_weekday(thursday, weekday::monday); + EXPECT_EQ(4, d - thursday) << Format(d); + EXPECT_EQ(d - 7, prev_weekday(thursday, weekday::monday)); + + // Thursday -> Tuesday + d = next_weekday(thursday, weekday::tuesday); + EXPECT_EQ(5, d - thursday) << Format(d); + EXPECT_EQ(d - 7, prev_weekday(thursday, weekday::tuesday)); + + // Thursday -> Wednesday + d = next_weekday(thursday, weekday::wednesday); + EXPECT_EQ(6, d - thursday) << Format(d); + EXPECT_EQ(d - 7, prev_weekday(thursday, weekday::wednesday)); +} + +TEST(CivilTime, NormalizeWithHugeYear) { + civil_month c(9223372036854775807, 1); + EXPECT_EQ("9223372036854775807-01", Format(c)); + c = c - 1; // Causes normalization + EXPECT_EQ("9223372036854775806-12", Format(c)); + + c = civil_month(-9223372036854775807 - 1, 1); + EXPECT_EQ("-9223372036854775808-01", Format(c)); + c = c + 12; // Causes normalization + EXPECT_EQ("-9223372036854775807-01", Format(c)); +} + +TEST(CivilTime, LeapYears) { + // Test data for leap years. + const struct { + int year; + int days; + struct { + int month; + int day; + } leap_day; // The date of the day after Feb 28. + } kLeapYearTable[]{ + {1900, 365, {3, 1}}, {1999, 365, {3, 1}}, + {2000, 366, {2, 29}}, // leap year + {2001, 365, {3, 1}}, {2002, 365, {3, 1}}, + {2003, 365, {3, 1}}, {2004, 366, {2, 29}}, // leap year + {2005, 365, {3, 1}}, {2006, 365, {3, 1}}, + {2007, 365, {3, 1}}, {2008, 366, {2, 29}}, // leap year + {2009, 365, {3, 1}}, {2100, 365, {3, 1}}, + }; + + for (const auto& e : kLeapYearTable) { + // Tests incrementing through the leap day. + const civil_day feb28(e.year, 2, 28); + const civil_day next_day = feb28 + 1; + EXPECT_EQ(e.leap_day.month, next_day.month()); + EXPECT_EQ(e.leap_day.day, next_day.day()); + + // Tests difference in days of leap years. + const civil_year year(feb28); + const civil_year next_year = year + 1; + EXPECT_EQ(e.days, civil_day(next_year) - civil_day(year)); + } +} + +TEST(CivilTime, FirstThursdayInMonth) { + const civil_day nov1(2014, 11, 1); + const civil_day thursday = next_weekday(nov1 - 1, weekday::thursday); + EXPECT_EQ("2014-11-06", Format(thursday)); + + // Bonus: Date of Thanksgiving in the United States + // Rule: Fourth Thursday of November + const civil_day thanksgiving = thursday + 7 * 3; + EXPECT_EQ("2014-11-27", Format(thanksgiving)); +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_fixed.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_fixed.cc new file mode 100644 index 000000000000..303c0244a824 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_fixed.cc @@ -0,0 +1,140 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "time_zone_fixed.h" + +#include <algorithm> +#include <cassert> +#include <chrono> +#include <cstring> +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +// The prefix used for the internal names of fixed-offset zones. +const char kFixedZonePrefix[] = "Fixed/UTC"; + +const char kDigits[] = "0123456789"; + +char* Format02d(char* p, int v) { + *p++ = kDigits[(v / 10) % 10]; + *p++ = kDigits[v % 10]; + return p; +} + +int Parse02d(const char* p) { + if (const char* ap = std::strchr(kDigits, *p)) { + int v = static_cast<int>(ap - kDigits); + if (const char* bp = std::strchr(kDigits, *++p)) { + return (v * 10) + static_cast<int>(bp - kDigits); + } + } + return -1; +} + +} // namespace + +bool FixedOffsetFromName(const std::string& name, seconds* offset) { + if (name.compare(0, std::string::npos, "UTC", 3) == 0) { + *offset = seconds::zero(); + return true; + } + + const std::size_t prefix_len = sizeof(kFixedZonePrefix) - 1; + const char* const ep = kFixedZonePrefix + prefix_len; + if (name.size() != prefix_len + 9) // <prefix>+99:99:99 + return false; + if (!std::equal(kFixedZonePrefix, ep, name.begin())) return false; + const char* np = name.data() + prefix_len; + if (np[0] != '+' && np[0] != '-') return false; + if (np[3] != ':' || np[6] != ':') // see note below about large offsets + return false; + + int hours = Parse02d(np + 1); + if (hours == -1) return false; + int mins = Parse02d(np + 4); + if (mins == -1) return false; + int secs = Parse02d(np + 7); + if (secs == -1) return false; + + secs += ((hours * 60) + mins) * 60; + if (secs > 24 * 60 * 60) return false; // outside supported offset range + *offset = seconds(secs * (np[0] == '-' ? -1 : 1)); // "-" means west + return true; +} + +std::string FixedOffsetToName(const seconds& offset) { + if (offset == seconds::zero()) return "UTC"; + if (offset < std::chrono::hours(-24) || offset > std::chrono::hours(24)) { + // We don't support fixed-offset zones more than 24 hours + // away from UTC to avoid complications in rendering such + // offsets and to (somewhat) limit the total number of zones. + return "UTC"; + } + int offset_seconds = static_cast<int>(offset.count()); + const char sign = (offset_seconds < 0 ? '-' : '+'); + int offset_minutes = offset_seconds / 60; + offset_seconds %= 60; + if (sign == '-') { + if (offset_seconds > 0) { + offset_seconds -= 60; + offset_minutes += 1; + } + offset_seconds = -offset_seconds; + offset_minutes = -offset_minutes; + } + int offset_hours = offset_minutes / 60; + offset_minutes %= 60; + const std::size_t prefix_len = sizeof(kFixedZonePrefix) - 1; + char buf[prefix_len + sizeof("-24:00:00")]; + char* ep = std::copy(kFixedZonePrefix, kFixedZonePrefix + prefix_len, buf); + *ep++ = sign; + ep = Format02d(ep, offset_hours); + *ep++ = ':'; + ep = Format02d(ep, offset_minutes); + *ep++ = ':'; + ep = Format02d(ep, offset_seconds); + *ep++ = '\0'; + assert(ep == buf + sizeof(buf)); + return buf; +} + +std::string FixedOffsetToAbbr(const seconds& offset) { + std::string abbr = FixedOffsetToName(offset); + const std::size_t prefix_len = sizeof(kFixedZonePrefix) - 1; + if (abbr.size() == prefix_len + 9) { // <prefix>+99:99:99 + abbr.erase(0, prefix_len); // +99:99:99 + abbr.erase(6, 1); // +99:9999 + abbr.erase(3, 1); // +999999 + if (abbr[5] == '0' && abbr[6] == '0') { // +999900 + abbr.erase(5, 2); // +9999 + if (abbr[3] == '0' && abbr[4] == '0') { // +9900 + abbr.erase(3, 2); // +99 + } + } + } + return abbr; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_fixed.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_fixed.h new file mode 100644 index 000000000000..e74a0bbd9dde --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_fixed.h @@ -0,0 +1,52 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_FIXED_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_FIXED_H_ + +#include <string> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// Helper functions for dealing with the names and abbreviations +// of time zones that are a fixed offset (seconds east) from UTC. +// FixedOffsetFromName() extracts the offset from a valid fixed-offset +// name, while FixedOffsetToName() and FixedOffsetToAbbr() generate +// the canonical zone name and abbreviation respectively for the given +// offset. +// +// A fixed-offset name looks like "Fixed/UTC<+-><hours>:<mins>:<secs>". +// Its abbreviation is of the form "UTC(<+->H?H(MM(SS)?)?)?" where the +// optional pieces are omitted when their values are zero. (Note that +// the sign is the opposite of that used in a POSIX TZ specification.) +// +// Note: FixedOffsetFromName() fails on syntax errors or when the parsed +// offset exceeds 24 hours. FixedOffsetToName() and FixedOffsetToAbbr() +// both produce "UTC" when the argument offset exceeds 24 hours. +bool FixedOffsetFromName(const std::string& name, seconds* offset); +std::string FixedOffsetToName(const seconds& offset); +std::string FixedOffsetToAbbr(const seconds& offset); + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_FIXED_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_format.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_format.cc new file mode 100644 index 000000000000..d8cb047425ee --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_format.cc @@ -0,0 +1,1029 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if !defined(HAS_STRPTIME) +#if !defined(_MSC_VER) && !defined(__MINGW32__) +#define HAS_STRPTIME 1 // assume everyone has strptime() except windows +#endif +#endif + +#if defined(HAS_STRPTIME) && HAS_STRPTIME +#if !defined(_XOPEN_SOURCE) +#define _XOPEN_SOURCE // Definedness suffices for strptime. +#endif +#endif + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +// Include time.h directly since, by C++ standards, ctime doesn't have to +// declare strptime. +#include <time.h> + +#include <cctype> +#include <chrono> +#include <cstddef> +#include <cstdint> +#include <cstring> +#include <ctime> +#include <limits> +#include <string> +#include <vector> +#if !HAS_STRPTIME +#include <iomanip> +#include <sstream> +#endif + +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "time_zone_if.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { +namespace detail { + +namespace { + +#if !HAS_STRPTIME +// Build a strptime() using C++11's std::get_time(). +char* strptime(const char* s, const char* fmt, std::tm* tm) { + std::istringstream input(s); + input >> std::get_time(tm, fmt); + if (input.fail()) return nullptr; + return const_cast<char*>(s) + + (input.eof() ? strlen(s) : static_cast<std::size_t>(input.tellg())); +} +#endif + +// Convert a cctz::weekday to a tm_wday value (0-6, Sunday = 0). +int ToTmWday(weekday wd) { + switch (wd) { + case weekday::sunday: + return 0; + case weekday::monday: + return 1; + case weekday::tuesday: + return 2; + case weekday::wednesday: + return 3; + case weekday::thursday: + return 4; + case weekday::friday: + return 5; + case weekday::saturday: + return 6; + } + return 0; /*NOTREACHED*/ +} + +// Convert a tm_wday value (0-6, Sunday = 0) to a cctz::weekday. +weekday FromTmWday(int tm_wday) { + switch (tm_wday) { + case 0: + return weekday::sunday; + case 1: + return weekday::monday; + case 2: + return weekday::tuesday; + case 3: + return weekday::wednesday; + case 4: + return weekday::thursday; + case 5: + return weekday::friday; + case 6: + return weekday::saturday; + } + return weekday::sunday; /*NOTREACHED*/ +} + +std::tm ToTM(const time_zone::absolute_lookup& al) { + std::tm tm{}; + tm.tm_sec = al.cs.second(); + tm.tm_min = al.cs.minute(); + tm.tm_hour = al.cs.hour(); + tm.tm_mday = al.cs.day(); + tm.tm_mon = al.cs.month() - 1; + + // Saturate tm.tm_year is cases of over/underflow. + if (al.cs.year() < std::numeric_limits<int>::min() + 1900) { + tm.tm_year = std::numeric_limits<int>::min(); + } else if (al.cs.year() - 1900 > std::numeric_limits<int>::max()) { + tm.tm_year = std::numeric_limits<int>::max(); + } else { + tm.tm_year = static_cast<int>(al.cs.year() - 1900); + } + + tm.tm_wday = ToTmWday(get_weekday(al.cs)); + tm.tm_yday = get_yearday(al.cs) - 1; + tm.tm_isdst = al.is_dst ? 1 : 0; + return tm; +} + +// Returns the week of the year [0:53] given a civil day and the day on +// which weeks are defined to start. +int ToWeek(const civil_day& cd, weekday week_start) { + const civil_day d(cd.year() % 400, cd.month(), cd.day()); + return static_cast<int>((d - prev_weekday(civil_year(d), week_start)) / 7); +} + +const char kDigits[] = "0123456789"; + +// Formats a 64-bit integer in the given field width. Note that it is up +// to the caller of Format64() [and Format02d()/FormatOffset()] to ensure +// that there is sufficient space before ep to hold the conversion. +char* Format64(char* ep, int width, std::int_fast64_t v) { + bool neg = false; + if (v < 0) { + --width; + neg = true; + if (v == std::numeric_limits<std::int_fast64_t>::min()) { + // Avoid negating minimum value. + std::int_fast64_t last_digit = -(v % 10); + v /= 10; + if (last_digit < 0) { + ++v; + last_digit += 10; + } + --width; + *--ep = kDigits[last_digit]; + } + v = -v; + } + do { + --width; + *--ep = kDigits[v % 10]; + } while (v /= 10); + while (--width >= 0) *--ep = '0'; // zero pad + if (neg) *--ep = '-'; + return ep; +} + +// Formats [0 .. 99] as %02d. +char* Format02d(char* ep, int v) { + *--ep = kDigits[v % 10]; + *--ep = kDigits[(v / 10) % 10]; + return ep; +} + +// Formats a UTC offset, like +00:00. +char* FormatOffset(char* ep, int offset, const char* mode) { + // TODO: Follow the RFC3339 "Unknown Local Offset Convention" and + // generate a "negative zero" when we're formatting a zero offset + // as the result of a failed load_time_zone(). + char sign = '+'; + if (offset < 0) { + offset = -offset; // bounded by 24h so no overflow + sign = '-'; + } + const int seconds = offset % 60; + const int minutes = (offset /= 60) % 60; + const int hours = offset /= 60; + const char sep = mode[0]; + const bool ext = (sep != '\0' && mode[1] == '*'); + const bool ccc = (ext && mode[2] == ':'); + if (ext && (!ccc || seconds != 0)) { + ep = Format02d(ep, seconds); + *--ep = sep; + } else { + // If we're not rendering seconds, sub-minute negative offsets + // should get a positive sign (e.g., offset=-10s => "+00:00"). + if (hours == 0 && minutes == 0) sign = '+'; + } + if (!ccc || minutes != 0 || seconds != 0) { + ep = Format02d(ep, minutes); + if (sep != '\0') *--ep = sep; + } + ep = Format02d(ep, hours); + *--ep = sign; + return ep; +} + +// Formats a std::tm using strftime(3). +void FormatTM(std::string* out, const std::string& fmt, const std::tm& tm) { + // strftime(3) returns the number of characters placed in the output + // array (which may be 0 characters). It also returns 0 to indicate + // an error, like the array wasn't large enough. To accommodate this, + // the following code grows the buffer size from 2x the format string + // length up to 32x. + for (std::size_t i = 2; i != 32; i *= 2) { + std::size_t buf_size = fmt.size() * i; + std::vector<char> buf(buf_size); + if (std::size_t len = strftime(&buf[0], buf_size, fmt.c_str(), &tm)) { + out->append(&buf[0], len); + return; + } + } +} + +// Used for %E#S/%E#f specifiers and for data values in parse(). +template <typename T> +const char* ParseInt(const char* dp, int width, T min, T max, T* vp) { + if (dp != nullptr) { + const T kmin = std::numeric_limits<T>::min(); + bool erange = false; + bool neg = false; + T value = 0; + if (*dp == '-') { + neg = true; + if (width <= 0 || --width != 0) { + ++dp; + } else { + dp = nullptr; // width was 1 + } + } + if (const char* const bp = dp) { + while (const char* cp = strchr(kDigits, *dp)) { + int d = static_cast<int>(cp - kDigits); + if (d >= 10) break; + if (value < kmin / 10) { + erange = true; + break; + } + value *= 10; + if (value < kmin + d) { + erange = true; + break; + } + value -= d; + dp += 1; + if (width > 0 && --width == 0) break; + } + if (dp != bp && !erange && (neg || value != kmin)) { + if (!neg || value != 0) { + if (!neg) value = -value; // make positive + if (min <= value && value <= max) { + *vp = value; + } else { + dp = nullptr; + } + } else { + dp = nullptr; + } + } else { + dp = nullptr; + } + } + } + return dp; +} + +// The number of base-10 digits that can be represented by a signed 64-bit +// integer. That is, 10^kDigits10_64 <= 2^63 - 1 < 10^(kDigits10_64 + 1). +const int kDigits10_64 = 18; + +// 10^n for everything that can be represented by a signed 64-bit integer. +const std::int_fast64_t kExp10[kDigits10_64 + 1] = { + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 10000000000, + 100000000000, + 1000000000000, + 10000000000000, + 100000000000000, + 1000000000000000, + 10000000000000000, + 100000000000000000, + 1000000000000000000, +}; + +} // namespace + +// Uses strftime(3) to format the given Time. The following extended format +// specifiers are also supported: +// +// - %Ez - RFC3339-compatible numeric UTC offset (+hh:mm or -hh:mm) +// - %E*z - Full-resolution numeric UTC offset (+hh:mm:ss or -hh:mm:ss) +// - %E#S - Seconds with # digits of fractional precision +// - %E*S - Seconds with full fractional precision (a literal '*') +// - %E4Y - Four-character years (-999 ... -001, 0000, 0001 ... 9999) +// - %ET - The RFC3339 "date-time" separator "T" +// +// The standard specifiers from RFC3339_* (%Y, %m, %d, %H, %M, and %S) are +// handled internally for performance reasons. strftime(3) is slow due to +// a POSIX requirement to respect changes to ${TZ}. +// +// The TZ/GNU %s extension is handled internally because strftime() has +// to use mktime() to generate it, and that assumes the local time zone. +// +// We also handle the %z and %Z specifiers to accommodate platforms that do +// not support the tm_gmtoff and tm_zone extensions to std::tm. +// +// Requires that zero() <= fs < seconds(1). +std::string format(const std::string& format, const time_point<seconds>& tp, + const detail::femtoseconds& fs, const time_zone& tz) { + std::string result; + result.reserve(format.size()); // A reasonable guess for the result size. + const time_zone::absolute_lookup al = tz.lookup(tp); + const std::tm tm = ToTM(al); + + // Scratch buffer for internal conversions. + char buf[3 + kDigits10_64]; // enough for longest conversion + char* const ep = buf + sizeof(buf); + char* bp; // works back from ep + + // Maintain three, disjoint subsequences that span format. + // [format.begin() ... pending) : already formatted into result + // [pending ... cur) : formatting pending, but no special cases + // [cur ... format.end()) : unexamined + // Initially, everything is in the unexamined part. + const char* pending = format.c_str(); // NUL terminated + const char* cur = pending; + const char* end = pending + format.length(); + + while (cur != end) { // while something is unexamined + // Moves cur to the next percent sign. + const char* start = cur; + while (cur != end && *cur != '%') ++cur; + + // If the new pending text is all ordinary, copy it out. + if (cur != start && pending == start) { + result.append(pending, static_cast<std::size_t>(cur - pending)); + pending = start = cur; + } + + // Span the sequential percent signs. + const char* percent = cur; + while (cur != end && *cur == '%') ++cur; + + // If the new pending text is all percents, copy out one + // percent for every matched pair, then skip those pairs. + if (cur != start && pending == start) { + std::size_t escaped = static_cast<std::size_t>(cur - pending) / 2; + result.append(pending, escaped); + pending += escaped * 2; + // Also copy out a single trailing percent. + if (pending != cur && cur == end) { + result.push_back(*pending++); + } + } + + // Loop unless we have an unescaped percent. + if (cur == end || (cur - percent) % 2 == 0) continue; + + // Simple specifiers that we handle ourselves. + if (strchr("YmdeUuWwHMSzZs%", *cur)) { + if (cur - 1 != pending) { + FormatTM(&result, std::string(pending, cur - 1), tm); + } + switch (*cur) { + case 'Y': + // This avoids the tm.tm_year overflow problem for %Y, however + // tm.tm_year will still be used by other specifiers like %D. + bp = Format64(ep, 0, al.cs.year()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'm': + bp = Format02d(ep, al.cs.month()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'd': + case 'e': + bp = Format02d(ep, al.cs.day()); + if (*cur == 'e' && *bp == '0') *bp = ' '; // for Windows + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'U': + bp = Format02d(ep, ToWeek(civil_day(al.cs), weekday::sunday)); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'u': + bp = Format64(ep, 0, tm.tm_wday ? tm.tm_wday : 7); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'W': + bp = Format02d(ep, ToWeek(civil_day(al.cs), weekday::monday)); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'w': + bp = Format64(ep, 0, tm.tm_wday); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'H': + bp = Format02d(ep, al.cs.hour()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'M': + bp = Format02d(ep, al.cs.minute()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'S': + bp = Format02d(ep, al.cs.second()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'z': + bp = FormatOffset(ep, al.offset, ""); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case 'Z': + result.append(al.abbr); + break; + case 's': + bp = Format64(ep, 0, ToUnixSeconds(tp)); + result.append(bp, static_cast<std::size_t>(ep - bp)); + break; + case '%': + result.push_back('%'); + break; + } + pending = ++cur; + continue; + } + + // More complex specifiers that we handle ourselves. + if (*cur == ':' && cur + 1 != end) { + if (*(cur + 1) == 'z') { + // Formats %:z. + if (cur - 1 != pending) { + FormatTM(&result, std::string(pending, cur - 1), tm); + } + bp = FormatOffset(ep, al.offset, ":"); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = cur += 2; + continue; + } + if (*(cur + 1) == ':' && cur + 2 != end) { + if (*(cur + 2) == 'z') { + // Formats %::z. + if (cur - 1 != pending) { + FormatTM(&result, std::string(pending, cur - 1), tm); + } + bp = FormatOffset(ep, al.offset, ":*"); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = cur += 3; + continue; + } + if (*(cur + 2) == ':' && cur + 3 != end) { + if (*(cur + 3) == 'z') { + // Formats %:::z. + if (cur - 1 != pending) { + FormatTM(&result, std::string(pending, cur - 1), tm); + } + bp = FormatOffset(ep, al.offset, ":*:"); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = cur += 4; + continue; + } + } + } + } + + // Loop if there is no E modifier. + if (*cur != 'E' || ++cur == end) continue; + + // Format our extensions. + if (*cur == 'T') { + // Formats %ET. + if (cur - 2 != pending) { + FormatTM(&result, std::string(pending, cur - 2), tm); + } + result.append("T"); + pending = ++cur; + } else if (*cur == 'z') { + // Formats %Ez. + if (cur - 2 != pending) { + FormatTM(&result, std::string(pending, cur - 2), tm); + } + bp = FormatOffset(ep, al.offset, ":"); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = ++cur; + } else if (*cur == '*' && cur + 1 != end && *(cur + 1) == 'z') { + // Formats %E*z. + if (cur - 2 != pending) { + FormatTM(&result, std::string(pending, cur - 2), tm); + } + bp = FormatOffset(ep, al.offset, ":*"); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = cur += 2; + } else if (*cur == '*' && cur + 1 != end && + (*(cur + 1) == 'S' || *(cur + 1) == 'f')) { + // Formats %E*S or %E*F. + if (cur - 2 != pending) { + FormatTM(&result, std::string(pending, cur - 2), tm); + } + char* cp = ep; + bp = Format64(cp, 15, fs.count()); + while (cp != bp && cp[-1] == '0') --cp; + switch (*(cur + 1)) { + case 'S': + if (cp != bp) *--bp = '.'; + bp = Format02d(bp, al.cs.second()); + break; + case 'f': + if (cp == bp) *--bp = '0'; + break; + } + result.append(bp, static_cast<std::size_t>(cp - bp)); + pending = cur += 2; + } else if (*cur == '4' && cur + 1 != end && *(cur + 1) == 'Y') { + // Formats %E4Y. + if (cur - 2 != pending) { + FormatTM(&result, std::string(pending, cur - 2), tm); + } + bp = Format64(ep, 4, al.cs.year()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = cur += 2; + } else if (std::isdigit(*cur)) { + // Possibly found %E#S or %E#f. + int n = 0; + if (const char* np = ParseInt(cur, 0, 0, 1024, &n)) { + if (*np == 'S' || *np == 'f') { + // Formats %E#S or %E#f. + if (cur - 2 != pending) { + FormatTM(&result, std::string(pending, cur - 2), tm); + } + bp = ep; + if (n > 0) { + if (n > kDigits10_64) n = kDigits10_64; + bp = Format64(bp, n, + (n > 15) ? fs.count() * kExp10[n - 15] + : fs.count() / kExp10[15 - n]); + if (*np == 'S') *--bp = '.'; + } + if (*np == 'S') bp = Format02d(bp, al.cs.second()); + result.append(bp, static_cast<std::size_t>(ep - bp)); + pending = cur = ++np; + } + } + } + } + + // Formats any remaining data. + if (end != pending) { + FormatTM(&result, std::string(pending, end), tm); + } + + return result; +} + +namespace { + +const char* ParseOffset(const char* dp, const char* mode, int* offset) { + if (dp != nullptr) { + const char first = *dp++; + if (first == '+' || first == '-') { + char sep = mode[0]; + int hours = 0; + int minutes = 0; + int seconds = 0; + const char* ap = ParseInt(dp, 2, 0, 23, &hours); + if (ap != nullptr && ap - dp == 2) { + dp = ap; + if (sep != '\0' && *ap == sep) ++ap; + const char* bp = ParseInt(ap, 2, 0, 59, &minutes); + if (bp != nullptr && bp - ap == 2) { + dp = bp; + if (sep != '\0' && *bp == sep) ++bp; + const char* cp = ParseInt(bp, 2, 0, 59, &seconds); + if (cp != nullptr && cp - bp == 2) dp = cp; + } + *offset = ((hours * 60 + minutes) * 60) + seconds; + if (first == '-') *offset = -*offset; + } else { + dp = nullptr; + } + } else if (first == 'Z' || first == 'z') { // Zulu + *offset = 0; + } else { + dp = nullptr; + } + } + return dp; +} + +const char* ParseZone(const char* dp, std::string* zone) { + zone->clear(); + if (dp != nullptr) { + while (*dp != '\0' && !std::isspace(*dp)) zone->push_back(*dp++); + if (zone->empty()) dp = nullptr; + } + return dp; +} + +const char* ParseSubSeconds(const char* dp, detail::femtoseconds* subseconds) { + if (dp != nullptr) { + std::int_fast64_t v = 0; + std::int_fast64_t exp = 0; + const char* const bp = dp; + while (const char* cp = strchr(kDigits, *dp)) { + int d = static_cast<int>(cp - kDigits); + if (d >= 10) break; + if (exp < 15) { + exp += 1; + v *= 10; + v += d; + } + ++dp; + } + if (dp != bp) { + v *= kExp10[15 - exp]; + *subseconds = detail::femtoseconds(v); + } else { + dp = nullptr; + } + } + return dp; +} + +// Parses a string into a std::tm using strptime(3). +const char* ParseTM(const char* dp, const char* fmt, std::tm* tm) { + if (dp != nullptr) { + dp = strptime(dp, fmt, tm); + } + return dp; +} + +// Sets year, tm_mon and tm_mday given the year, week_num, and tm_wday, +// and the day on which weeks are defined to start. Returns false if year +// would need to move outside its bounds. +bool FromWeek(int week_num, weekday week_start, year_t* year, std::tm* tm) { + const civil_year y(*year % 400); + civil_day cd = prev_weekday(y, week_start); // week 0 + cd = next_weekday(cd - 1, FromTmWday(tm->tm_wday)) + (week_num * 7); + if (const year_t shift = cd.year() - y.year()) { + if (shift > 0) { + if (*year > std::numeric_limits<year_t>::max() - shift) return false; + } else { + if (*year < std::numeric_limits<year_t>::min() - shift) return false; + } + *year += shift; + } + tm->tm_mon = cd.month() - 1; + tm->tm_mday = cd.day(); + return true; +} + +} // namespace + +// Uses strptime(3) to parse the given input. Supports the same extended +// format specifiers as format(), although %E#S and %E*S are treated +// identically (and similarly for %E#f and %E*f). %Ez and %E*z also accept +// the same inputs. %ET accepts either 'T' or 't'. +// +// The standard specifiers from RFC3339_* (%Y, %m, %d, %H, %M, and %S) are +// handled internally so that we can normally avoid strptime() altogether +// (which is particularly helpful when the native implementation is broken). +// +// The TZ/GNU %s extension is handled internally because strptime() has to +// use localtime_r() to generate it, and that assumes the local time zone. +// +// We also handle the %z specifier to accommodate platforms that do not +// support the tm_gmtoff extension to std::tm. %Z is parsed but ignored. +bool parse(const std::string& format, const std::string& input, + const time_zone& tz, time_point<seconds>* sec, + detail::femtoseconds* fs, std::string* err) { + // The unparsed input. + const char* data = input.c_str(); // NUL terminated + + // Skips leading whitespace. + while (std::isspace(*data)) ++data; + + const year_t kyearmax = std::numeric_limits<year_t>::max(); + const year_t kyearmin = std::numeric_limits<year_t>::min(); + + // Sets default values for unspecified fields. + bool saw_year = false; + year_t year = 1970; + std::tm tm{}; + tm.tm_year = 1970 - 1900; + tm.tm_mon = 1 - 1; // Jan + tm.tm_mday = 1; + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + tm.tm_wday = 4; // Thu + tm.tm_yday = 0; + tm.tm_isdst = 0; + auto subseconds = detail::femtoseconds::zero(); + bool saw_offset = false; + int offset = 0; // No offset from passed tz. + std::string zone = "UTC"; + + const char* fmt = format.c_str(); // NUL terminated + bool twelve_hour = false; + bool afternoon = false; + int week_num = -1; + weekday week_start = weekday::sunday; + + bool saw_percent_s = false; + std::int_fast64_t percent_s = 0; + + // Steps through format, one specifier at a time. + while (data != nullptr && *fmt != '\0') { + if (std::isspace(*fmt)) { + while (std::isspace(*data)) ++data; + while (std::isspace(*++fmt)) continue; + continue; + } + + if (*fmt != '%') { + if (*data == *fmt) { + ++data; + ++fmt; + } else { + data = nullptr; + } + continue; + } + + const char* percent = fmt; + if (*++fmt == '\0') { + data = nullptr; + continue; + } + switch (*fmt++) { + case 'Y': + // Symmetrically with FormatTime(), directly handing %Y avoids the + // tm.tm_year overflow problem. However, tm.tm_year will still be + // used by other specifiers like %D. + data = ParseInt(data, 0, kyearmin, kyearmax, &year); + if (data != nullptr) saw_year = true; + continue; + case 'm': + data = ParseInt(data, 2, 1, 12, &tm.tm_mon); + if (data != nullptr) tm.tm_mon -= 1; + week_num = -1; + continue; + case 'd': + case 'e': + data = ParseInt(data, 2, 1, 31, &tm.tm_mday); + week_num = -1; + continue; + case 'U': + data = ParseInt(data, 0, 0, 53, &week_num); + week_start = weekday::sunday; + continue; + case 'W': + data = ParseInt(data, 0, 0, 53, &week_num); + week_start = weekday::monday; + continue; + case 'u': + data = ParseInt(data, 0, 1, 7, &tm.tm_wday); + if (data != nullptr) tm.tm_wday %= 7; + continue; + case 'w': + data = ParseInt(data, 0, 0, 6, &tm.tm_wday); + continue; + case 'H': + data = ParseInt(data, 2, 0, 23, &tm.tm_hour); + twelve_hour = false; + continue; + case 'M': + data = ParseInt(data, 2, 0, 59, &tm.tm_min); + continue; + case 'S': + data = ParseInt(data, 2, 0, 60, &tm.tm_sec); + continue; + case 'I': + case 'l': + case 'r': // probably uses %I + twelve_hour = true; + break; + case 'R': // uses %H + case 'T': // uses %H + case 'c': // probably uses %H + case 'X': // probably uses %H + twelve_hour = false; + break; + case 'z': + data = ParseOffset(data, "", &offset); + if (data != nullptr) saw_offset = true; + continue; + case 'Z': // ignored; zone abbreviations are ambiguous + data = ParseZone(data, &zone); + continue; + case 's': + data = + ParseInt(data, 0, std::numeric_limits<std::int_fast64_t>::min(), + std::numeric_limits<std::int_fast64_t>::max(), &percent_s); + if (data != nullptr) saw_percent_s = true; + continue; + case ':': + if (fmt[0] == 'z' || + (fmt[0] == ':' && + (fmt[1] == 'z' || (fmt[1] == ':' && fmt[2] == 'z')))) { + data = ParseOffset(data, ":", &offset); + if (data != nullptr) saw_offset = true; + fmt += (fmt[0] == 'z') ? 1 : (fmt[1] == 'z') ? 2 : 3; + continue; + } + break; + case '%': + data = (*data == '%' ? data + 1 : nullptr); + continue; + case 'E': + if (fmt[0] == 'T') { + if (*data == 'T' || *data == 't') { + ++data; + ++fmt; + } else { + data = nullptr; + } + continue; + } + if (fmt[0] == 'z' || (fmt[0] == '*' && fmt[1] == 'z')) { + data = ParseOffset(data, ":", &offset); + if (data != nullptr) saw_offset = true; + fmt += (fmt[0] == 'z') ? 1 : 2; + continue; + } + if (fmt[0] == '*' && fmt[1] == 'S') { + data = ParseInt(data, 2, 0, 60, &tm.tm_sec); + if (data != nullptr && *data == '.') { + data = ParseSubSeconds(data + 1, &subseconds); + } + fmt += 2; + continue; + } + if (fmt[0] == '*' && fmt[1] == 'f') { + if (data != nullptr && std::isdigit(*data)) { + data = ParseSubSeconds(data, &subseconds); + } + fmt += 2; + continue; + } + if (fmt[0] == '4' && fmt[1] == 'Y') { + const char* bp = data; + data = ParseInt(data, 4, year_t{-999}, year_t{9999}, &year); + if (data != nullptr) { + if (data - bp == 4) { + saw_year = true; + } else { + data = nullptr; // stopped too soon + } + } + fmt += 2; + continue; + } + if (std::isdigit(*fmt)) { + int n = 0; // value ignored + if (const char* np = ParseInt(fmt, 0, 0, 1024, &n)) { + if (*np == 'S') { + data = ParseInt(data, 2, 0, 60, &tm.tm_sec); + if (data != nullptr && *data == '.') { + data = ParseSubSeconds(data + 1, &subseconds); + } + fmt = ++np; + continue; + } + if (*np == 'f') { + if (data != nullptr && std::isdigit(*data)) { + data = ParseSubSeconds(data, &subseconds); + } + fmt = ++np; + continue; + } + } + } + if (*fmt == 'c') twelve_hour = false; // probably uses %H + if (*fmt == 'X') twelve_hour = false; // probably uses %H + if (*fmt != '\0') ++fmt; + break; + case 'O': + if (*fmt == 'H') twelve_hour = false; + if (*fmt == 'I') twelve_hour = true; + if (*fmt != '\0') ++fmt; + break; + } + + // Parses the current specifier. + const char* orig_data = data; + std::string spec(percent, static_cast<std::size_t>(fmt - percent)); + data = ParseTM(data, spec.c_str(), &tm); + + // If we successfully parsed %p we need to remember whether the result + // was AM or PM so that we can adjust tm_hour before time_zone::lookup(). + // So reparse the input with a known AM hour, and check if it is shifted + // to a PM hour. + if (spec == "%p" && data != nullptr) { + std::string test_input = "1"; + test_input.append(orig_data, static_cast<std::size_t>(data - orig_data)); + const char* test_data = test_input.c_str(); + std::tm tmp{}; + ParseTM(test_data, "%I%p", &tmp); + afternoon = (tmp.tm_hour == 13); + } + } + + // Adjust a 12-hour tm_hour value if it should be in the afternoon. + if (twelve_hour && afternoon && tm.tm_hour < 12) { + tm.tm_hour += 12; + } + + if (data == nullptr) { + if (err != nullptr) *err = "Failed to parse input"; + return false; + } + + // Skip any remaining whitespace. + while (std::isspace(*data)) ++data; + + // parse() must consume the entire input string. + if (*data != '\0') { + if (err != nullptr) *err = "Illegal trailing data in input string"; + return false; + } + + // If we saw %s then we ignore anything else and return that time. + if (saw_percent_s) { + *sec = FromUnixSeconds(percent_s); + *fs = detail::femtoseconds::zero(); + return true; + } + + // If we saw %z, %Ez, or %E*z then we want to interpret the parsed fields + // in UTC and then shift by that offset. Otherwise we want to interpret + // the fields directly in the passed time_zone. + time_zone ptz = saw_offset ? utc_time_zone() : tz; + + // Allows a leap second of 60 to normalize forward to the following ":00". + if (tm.tm_sec == 60) { + tm.tm_sec -= 1; + offset -= 1; + subseconds = detail::femtoseconds::zero(); + } + + if (!saw_year) { + year = year_t{tm.tm_year}; + if (year > kyearmax - 1900) { + // Platform-dependent, maybe unreachable. + if (err != nullptr) *err = "Out-of-range year"; + return false; + } + year += 1900; + } + + // Compute year, tm.tm_mon and tm.tm_mday if we parsed a week number. + if (week_num != -1) { + if (!FromWeek(week_num, week_start, &year, &tm)) { + if (err != nullptr) *err = "Out-of-range field"; + return false; + } + } + + const int month = tm.tm_mon + 1; + civil_second cs(year, month, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec); + + // parse() should not allow normalization. Due to the restricted field + // ranges above (see ParseInt()), the only possibility is for days to roll + // into months. That is, parsing "Sep 31" should not produce "Oct 1". + if (cs.month() != month || cs.day() != tm.tm_mday) { + if (err != nullptr) *err = "Out-of-range field"; + return false; + } + + // Accounts for the offset adjustment before converting to absolute time. + if ((offset < 0 && cs > civil_second::max() + offset) || + (offset > 0 && cs < civil_second::min() + offset)) { + if (err != nullptr) *err = "Out-of-range field"; + return false; + } + cs -= offset; + + const auto tp = ptz.lookup(cs).pre; + // Checks for overflow/underflow and returns an error as necessary. + if (tp == time_point<seconds>::max()) { + const auto al = ptz.lookup(time_point<seconds>::max()); + if (cs > al.cs) { + if (err != nullptr) *err = "Out-of-range field"; + return false; + } + } + if (tp == time_point<seconds>::min()) { + const auto al = ptz.lookup(time_point<seconds>::min()); + if (cs < al.cs) { + if (err != nullptr) *err = "Out-of-range field"; + return false; + } + } + + *sec = tp; + *fs = subseconds; + return true; +} + +} // namespace detail +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_format_test.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_format_test.cc new file mode 100644 index 000000000000..a11f93e2a597 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_format_test.cc @@ -0,0 +1,1603 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <chrono> +#include <iomanip> +#include <sstream> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace chrono = std::chrono; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +// This helper is a macro so that failed expectations show up with the +// correct line numbers. +#define ExpectTime(tp, tz, y, m, d, hh, mm, ss, off, isdst, zone) \ + do { \ + time_zone::absolute_lookup al = tz.lookup(tp); \ + EXPECT_EQ(y, al.cs.year()); \ + EXPECT_EQ(m, al.cs.month()); \ + EXPECT_EQ(d, al.cs.day()); \ + EXPECT_EQ(hh, al.cs.hour()); \ + EXPECT_EQ(mm, al.cs.minute()); \ + EXPECT_EQ(ss, al.cs.second()); \ + EXPECT_EQ(off, al.offset); \ + EXPECT_TRUE(isdst == al.is_dst); \ + EXPECT_STREQ(zone, al.abbr); \ + } while (0) + +const char RFC3339_full[] = "%Y-%m-%d%ET%H:%M:%E*S%Ez"; +const char RFC3339_sec[] = "%Y-%m-%d%ET%H:%M:%S%Ez"; + +const char RFC1123_full[] = "%a, %d %b %Y %H:%M:%S %z"; +const char RFC1123_no_wday[] = "%d %b %Y %H:%M:%S %z"; + +// A helper that tests the given format specifier by itself, and with leading +// and trailing characters. For example: TestFormatSpecifier(tp, "%a", "Thu"). +template <typename D> +void TestFormatSpecifier(time_point<D> tp, time_zone tz, const std::string& fmt, + const std::string& ans) { + EXPECT_EQ(ans, format(fmt, tp, tz)) << fmt; + EXPECT_EQ("xxx " + ans, format("xxx " + fmt, tp, tz)); + EXPECT_EQ(ans + " yyy", format(fmt + " yyy", tp, tz)); + EXPECT_EQ("xxx " + ans + " yyy", format("xxx " + fmt + " yyy", tp, tz)); +} + +} // namespace + +// +// Testing format() +// + +TEST(Format, TimePointResolution) { + const char kFmt[] = "%H:%M:%E*S"; + const time_zone utc = utc_time_zone(); + const time_point<chrono::nanoseconds> t0 = + chrono::system_clock::from_time_t(1420167845) + + chrono::milliseconds(123) + chrono::microseconds(456) + + chrono::nanoseconds(789); + EXPECT_EQ( + "03:04:05.123456789", + format(kFmt, chrono::time_point_cast<chrono::nanoseconds>(t0), utc)); + EXPECT_EQ( + "03:04:05.123456", + format(kFmt, chrono::time_point_cast<chrono::microseconds>(t0), utc)); + EXPECT_EQ( + "03:04:05.123", + format(kFmt, chrono::time_point_cast<chrono::milliseconds>(t0), utc)); + EXPECT_EQ("03:04:05", + format(kFmt, chrono::time_point_cast<chrono::seconds>(t0), utc)); + EXPECT_EQ( + "03:04:05", + format(kFmt, + chrono::time_point_cast<absl::time_internal::cctz::seconds>(t0), + utc)); + EXPECT_EQ("03:04:00", + format(kFmt, chrono::time_point_cast<chrono::minutes>(t0), utc)); + EXPECT_EQ("03:00:00", + format(kFmt, chrono::time_point_cast<chrono::hours>(t0), utc)); +} + +TEST(Format, TimePointExtendedResolution) { + const char kFmt[] = "%H:%M:%E*S"; + const time_zone utc = utc_time_zone(); + const time_point<absl::time_internal::cctz::seconds> tp = + chrono::time_point_cast<absl::time_internal::cctz::seconds>( + chrono::system_clock::from_time_t(0)) + + chrono::hours(12) + chrono::minutes(34) + chrono::seconds(56); + + EXPECT_EQ( + "12:34:56.123456789012345", + detail::format(kFmt, tp, detail::femtoseconds(123456789012345), utc)); + EXPECT_EQ( + "12:34:56.012345678901234", + detail::format(kFmt, tp, detail::femtoseconds(12345678901234), utc)); + EXPECT_EQ("12:34:56.001234567890123", + detail::format(kFmt, tp, detail::femtoseconds(1234567890123), utc)); + EXPECT_EQ("12:34:56.000123456789012", + detail::format(kFmt, tp, detail::femtoseconds(123456789012), utc)); + + EXPECT_EQ("12:34:56.000000000000123", + detail::format(kFmt, tp, detail::femtoseconds(123), utc)); + EXPECT_EQ("12:34:56.000000000000012", + detail::format(kFmt, tp, detail::femtoseconds(12), utc)); + EXPECT_EQ("12:34:56.000000000000001", + detail::format(kFmt, tp, detail::femtoseconds(1), utc)); +} + +TEST(Format, Basics) { + time_zone tz = utc_time_zone(); + time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); + + // Starts with a couple basic edge cases. + EXPECT_EQ("", format("", tp, tz)); + EXPECT_EQ(" ", format(" ", tp, tz)); + EXPECT_EQ(" ", format(" ", tp, tz)); + EXPECT_EQ("xxx", format("xxx", tp, tz)); + std::string big(128, 'x'); + EXPECT_EQ(big, format(big, tp, tz)); + // Cause the 1024-byte buffer to grow. + std::string bigger(100000, 'x'); + EXPECT_EQ(bigger, format(bigger, tp, tz)); + + tp += chrono::hours(13) + chrono::minutes(4) + chrono::seconds(5); + tp += chrono::milliseconds(6) + chrono::microseconds(7) + + chrono::nanoseconds(8); + EXPECT_EQ("1970-01-01", format("%Y-%m-%d", tp, tz)); + EXPECT_EQ("13:04:05", format("%H:%M:%S", tp, tz)); + EXPECT_EQ("13:04:05.006", format("%H:%M:%E3S", tp, tz)); + EXPECT_EQ("13:04:05.006007", format("%H:%M:%E6S", tp, tz)); + EXPECT_EQ("13:04:05.006007008", format("%H:%M:%E9S", tp, tz)); +} + +TEST(Format, PosixConversions) { + const time_zone tz = utc_time_zone(); + auto tp = chrono::system_clock::from_time_t(0); + + TestFormatSpecifier(tp, tz, "%d", "01"); + TestFormatSpecifier(tp, tz, "%e", " 1"); // extension but internal support + TestFormatSpecifier(tp, tz, "%H", "00"); + TestFormatSpecifier(tp, tz, "%I", "12"); + TestFormatSpecifier(tp, tz, "%j", "001"); + TestFormatSpecifier(tp, tz, "%m", "01"); + TestFormatSpecifier(tp, tz, "%M", "00"); + TestFormatSpecifier(tp, tz, "%S", "00"); + TestFormatSpecifier(tp, tz, "%U", "00"); +#if !defined(__EMSCRIPTEN__) + TestFormatSpecifier(tp, tz, "%w", "4"); // 4=Thursday +#endif + TestFormatSpecifier(tp, tz, "%W", "00"); + TestFormatSpecifier(tp, tz, "%y", "70"); + TestFormatSpecifier(tp, tz, "%Y", "1970"); + TestFormatSpecifier(tp, tz, "%z", "+0000"); + TestFormatSpecifier(tp, tz, "%Z", "UTC"); + TestFormatSpecifier(tp, tz, "%%", "%"); + +#if defined(__linux__) + // SU/C99/TZ extensions + TestFormatSpecifier(tp, tz, "%C", "19"); + TestFormatSpecifier(tp, tz, "%D", "01/01/70"); + TestFormatSpecifier(tp, tz, "%F", "1970-01-01"); + TestFormatSpecifier(tp, tz, "%g", "70"); + TestFormatSpecifier(tp, tz, "%G", "1970"); + TestFormatSpecifier(tp, tz, "%k", " 0"); + TestFormatSpecifier(tp, tz, "%l", "12"); + TestFormatSpecifier(tp, tz, "%n", "\n"); + TestFormatSpecifier(tp, tz, "%R", "00:00"); + TestFormatSpecifier(tp, tz, "%t", "\t"); + TestFormatSpecifier(tp, tz, "%T", "00:00:00"); + TestFormatSpecifier(tp, tz, "%u", "4"); // 4=Thursday + TestFormatSpecifier(tp, tz, "%V", "01"); + TestFormatSpecifier(tp, tz, "%s", "0"); +#endif +} + +TEST(Format, LocaleSpecific) { + const time_zone tz = utc_time_zone(); + auto tp = chrono::system_clock::from_time_t(0); + + TestFormatSpecifier(tp, tz, "%a", "Thu"); + TestFormatSpecifier(tp, tz, "%A", "Thursday"); + TestFormatSpecifier(tp, tz, "%b", "Jan"); + TestFormatSpecifier(tp, tz, "%B", "January"); + + // %c should at least produce the numeric year and time-of-day. + const std::string s = format("%c", tp, utc_time_zone()); + EXPECT_THAT(s, testing::HasSubstr("1970")); + EXPECT_THAT(s, testing::HasSubstr("00:00:00")); + + TestFormatSpecifier(tp, tz, "%p", "AM"); + TestFormatSpecifier(tp, tz, "%x", "01/01/70"); + TestFormatSpecifier(tp, tz, "%X", "00:00:00"); + +#if defined(__linux__) + // SU/C99/TZ extensions + TestFormatSpecifier(tp, tz, "%h", "Jan"); // Same as %b + TestFormatSpecifier(tp, tz, "%P", "am"); + TestFormatSpecifier(tp, tz, "%r", "12:00:00 AM"); + + // Modified conversion specifiers %E_ + TestFormatSpecifier(tp, tz, "%Ec", "Thu Jan 1 00:00:00 1970"); + TestFormatSpecifier(tp, tz, "%EC", "19"); + TestFormatSpecifier(tp, tz, "%Ex", "01/01/70"); + TestFormatSpecifier(tp, tz, "%EX", "00:00:00"); + TestFormatSpecifier(tp, tz, "%Ey", "70"); + TestFormatSpecifier(tp, tz, "%EY", "1970"); + + // Modified conversion specifiers %O_ + TestFormatSpecifier(tp, tz, "%Od", "01"); + TestFormatSpecifier(tp, tz, "%Oe", " 1"); + TestFormatSpecifier(tp, tz, "%OH", "00"); + TestFormatSpecifier(tp, tz, "%OI", "12"); + TestFormatSpecifier(tp, tz, "%Om", "01"); + TestFormatSpecifier(tp, tz, "%OM", "00"); + TestFormatSpecifier(tp, tz, "%OS", "00"); + TestFormatSpecifier(tp, tz, "%Ou", "4"); // 4=Thursday + TestFormatSpecifier(tp, tz, "%OU", "00"); + TestFormatSpecifier(tp, tz, "%OV", "01"); + TestFormatSpecifier(tp, tz, "%Ow", "4"); // 4=Thursday + TestFormatSpecifier(tp, tz, "%OW", "00"); + TestFormatSpecifier(tp, tz, "%Oy", "70"); +#endif +} + +TEST(Format, Escaping) { + const time_zone tz = utc_time_zone(); + auto tp = chrono::system_clock::from_time_t(0); + + TestFormatSpecifier(tp, tz, "%%", "%"); + TestFormatSpecifier(tp, tz, "%%a", "%a"); + TestFormatSpecifier(tp, tz, "%%b", "%b"); + TestFormatSpecifier(tp, tz, "%%Ea", "%Ea"); + TestFormatSpecifier(tp, tz, "%%Es", "%Es"); + TestFormatSpecifier(tp, tz, "%%E3S", "%E3S"); + TestFormatSpecifier(tp, tz, "%%OS", "%OS"); + TestFormatSpecifier(tp, tz, "%%O3S", "%O3S"); + + // Multiple levels of escaping. + TestFormatSpecifier(tp, tz, "%%%Y", "%1970"); + TestFormatSpecifier(tp, tz, "%%%E3S", "%00.000"); + TestFormatSpecifier(tp, tz, "%%%%E3S", "%%E3S"); +} + +TEST(Format, ExtendedSeconds) { + const time_zone tz = utc_time_zone(); + + // No subseconds. + time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); + tp += chrono::seconds(5); + EXPECT_EQ("05", format("%E*S", tp, tz)); + EXPECT_EQ("05", format("%E0S", tp, tz)); + EXPECT_EQ("05.0", format("%E1S", tp, tz)); + EXPECT_EQ("05.00", format("%E2S", tp, tz)); + EXPECT_EQ("05.000", format("%E3S", tp, tz)); + EXPECT_EQ("05.0000", format("%E4S", tp, tz)); + EXPECT_EQ("05.00000", format("%E5S", tp, tz)); + EXPECT_EQ("05.000000", format("%E6S", tp, tz)); + EXPECT_EQ("05.0000000", format("%E7S", tp, tz)); + EXPECT_EQ("05.00000000", format("%E8S", tp, tz)); + EXPECT_EQ("05.000000000", format("%E9S", tp, tz)); + EXPECT_EQ("05.0000000000", format("%E10S", tp, tz)); + EXPECT_EQ("05.00000000000", format("%E11S", tp, tz)); + EXPECT_EQ("05.000000000000", format("%E12S", tp, tz)); + EXPECT_EQ("05.0000000000000", format("%E13S", tp, tz)); + EXPECT_EQ("05.00000000000000", format("%E14S", tp, tz)); + EXPECT_EQ("05.000000000000000", format("%E15S", tp, tz)); + + // With subseconds. + tp += chrono::milliseconds(6) + chrono::microseconds(7) + + chrono::nanoseconds(8); + EXPECT_EQ("05.006007008", format("%E*S", tp, tz)); + EXPECT_EQ("05", format("%E0S", tp, tz)); + EXPECT_EQ("05.0", format("%E1S", tp, tz)); + EXPECT_EQ("05.00", format("%E2S", tp, tz)); + EXPECT_EQ("05.006", format("%E3S", tp, tz)); + EXPECT_EQ("05.0060", format("%E4S", tp, tz)); + EXPECT_EQ("05.00600", format("%E5S", tp, tz)); + EXPECT_EQ("05.006007", format("%E6S", tp, tz)); + EXPECT_EQ("05.0060070", format("%E7S", tp, tz)); + EXPECT_EQ("05.00600700", format("%E8S", tp, tz)); + EXPECT_EQ("05.006007008", format("%E9S", tp, tz)); + EXPECT_EQ("05.0060070080", format("%E10S", tp, tz)); + EXPECT_EQ("05.00600700800", format("%E11S", tp, tz)); + EXPECT_EQ("05.006007008000", format("%E12S", tp, tz)); + EXPECT_EQ("05.0060070080000", format("%E13S", tp, tz)); + EXPECT_EQ("05.00600700800000", format("%E14S", tp, tz)); + EXPECT_EQ("05.006007008000000", format("%E15S", tp, tz)); + + // Times before the Unix epoch. + tp = chrono::system_clock::from_time_t(0) + chrono::microseconds(-1); + EXPECT_EQ("1969-12-31 23:59:59.999999", + format("%Y-%m-%d %H:%M:%E*S", tp, tz)); + + // Here is a "%E*S" case we got wrong for a while. While the first + // instant below is correctly rendered as "...:07.333304", the second + // one used to appear as "...:07.33330499999999999". + tp = chrono::system_clock::from_time_t(0) + + chrono::microseconds(1395024427333304); + EXPECT_EQ("2014-03-17 02:47:07.333304", + format("%Y-%m-%d %H:%M:%E*S", tp, tz)); + tp += chrono::microseconds(1); + EXPECT_EQ("2014-03-17 02:47:07.333305", + format("%Y-%m-%d %H:%M:%E*S", tp, tz)); +} + +TEST(Format, ExtendedSubeconds) { + const time_zone tz = utc_time_zone(); + + // No subseconds. + time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); + tp += chrono::seconds(5); + EXPECT_EQ("0", format("%E*f", tp, tz)); + EXPECT_EQ("", format("%E0f", tp, tz)); + EXPECT_EQ("0", format("%E1f", tp, tz)); + EXPECT_EQ("00", format("%E2f", tp, tz)); + EXPECT_EQ("000", format("%E3f", tp, tz)); + EXPECT_EQ("0000", format("%E4f", tp, tz)); + EXPECT_EQ("00000", format("%E5f", tp, tz)); + EXPECT_EQ("000000", format("%E6f", tp, tz)); + EXPECT_EQ("0000000", format("%E7f", tp, tz)); + EXPECT_EQ("00000000", format("%E8f", tp, tz)); + EXPECT_EQ("000000000", format("%E9f", tp, tz)); + EXPECT_EQ("0000000000", format("%E10f", tp, tz)); + EXPECT_EQ("00000000000", format("%E11f", tp, tz)); + EXPECT_EQ("000000000000", format("%E12f", tp, tz)); + EXPECT_EQ("0000000000000", format("%E13f", tp, tz)); + EXPECT_EQ("00000000000000", format("%E14f", tp, tz)); + EXPECT_EQ("000000000000000", format("%E15f", tp, tz)); + + // With subseconds. + tp += chrono::milliseconds(6) + chrono::microseconds(7) + + chrono::nanoseconds(8); + EXPECT_EQ("006007008", format("%E*f", tp, tz)); + EXPECT_EQ("", format("%E0f", tp, tz)); + EXPECT_EQ("0", format("%E1f", tp, tz)); + EXPECT_EQ("00", format("%E2f", tp, tz)); + EXPECT_EQ("006", format("%E3f", tp, tz)); + EXPECT_EQ("0060", format("%E4f", tp, tz)); + EXPECT_EQ("00600", format("%E5f", tp, tz)); + EXPECT_EQ("006007", format("%E6f", tp, tz)); + EXPECT_EQ("0060070", format("%E7f", tp, tz)); + EXPECT_EQ("00600700", format("%E8f", tp, tz)); + EXPECT_EQ("006007008", format("%E9f", tp, tz)); + EXPECT_EQ("0060070080", format("%E10f", tp, tz)); + EXPECT_EQ("00600700800", format("%E11f", tp, tz)); + EXPECT_EQ("006007008000", format("%E12f", tp, tz)); + EXPECT_EQ("0060070080000", format("%E13f", tp, tz)); + EXPECT_EQ("00600700800000", format("%E14f", tp, tz)); + EXPECT_EQ("006007008000000", format("%E15f", tp, tz)); + + // Times before the Unix epoch. + tp = chrono::system_clock::from_time_t(0) + chrono::microseconds(-1); + EXPECT_EQ("1969-12-31 23:59:59.999999", + format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); + + // Here is a "%E*S" case we got wrong for a while. While the first + // instant below is correctly rendered as "...:07.333304", the second + // one used to appear as "...:07.33330499999999999". + tp = chrono::system_clock::from_time_t(0) + + chrono::microseconds(1395024427333304); + EXPECT_EQ("2014-03-17 02:47:07.333304", + format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); + tp += chrono::microseconds(1); + EXPECT_EQ("2014-03-17 02:47:07.333305", + format("%Y-%m-%d %H:%M:%S.%E*f", tp, tz)); +} + +TEST(Format, CompareExtendSecondsVsSubseconds) { + const time_zone tz = utc_time_zone(); + + // This test case illustrates the differences/similarities between: + // fmt_A: %E<prec>S + // fmt_B: %S.%E<prec>f + auto fmt_A = [](const std::string& prec) { return "%E" + prec + "S"; }; + auto fmt_B = [](const std::string& prec) { return "%S.%E" + prec + "f"; }; + + // No subseconds: + time_point<chrono::nanoseconds> tp = chrono::system_clock::from_time_t(0); + tp += chrono::seconds(5); + // ... %E*S and %S.%E*f are different. + EXPECT_EQ("05", format(fmt_A("*"), tp, tz)); + EXPECT_EQ("05.0", format(fmt_B("*"), tp, tz)); + // ... %E0S and %S.%E0f are different. + EXPECT_EQ("05", format(fmt_A("0"), tp, tz)); + EXPECT_EQ("05.", format(fmt_B("0"), tp, tz)); + // ... %E<prec>S and %S.%E<prec>f are the same for prec in [1:15]. + for (int prec = 1; prec <= 15; ++prec) { + const std::string a = format(fmt_A(std::to_string(prec)), tp, tz); + const std::string b = format(fmt_B(std::to_string(prec)), tp, tz); + EXPECT_EQ(a, b) << "prec=" << prec; + } + + // With subseconds: + // ... %E*S and %S.%E*f are the same. + tp += chrono::milliseconds(6) + chrono::microseconds(7) + + chrono::nanoseconds(8); + EXPECT_EQ("05.006007008", format(fmt_A("*"), tp, tz)); + EXPECT_EQ("05.006007008", format(fmt_B("*"), tp, tz)); + // ... %E0S and %S.%E0f are different. + EXPECT_EQ("05", format(fmt_A("0"), tp, tz)); + EXPECT_EQ("05.", format(fmt_B("0"), tp, tz)); + // ... %E<prec>S and %S.%E<prec>f are the same for prec in [1:15]. + for (int prec = 1; prec <= 15; ++prec) { + const std::string a = format(fmt_A(std::to_string(prec)), tp, tz); + const std::string b = format(fmt_B(std::to_string(prec)), tp, tz); + EXPECT_EQ(a, b) << "prec=" << prec; + } +} + +TEST(Format, ExtendedOffset) { + const auto tp = chrono::system_clock::from_time_t(0); + + auto tz = fixed_time_zone(absl::time_internal::cctz::seconds::zero()); + TestFormatSpecifier(tp, tz, "%z", "+0000"); + TestFormatSpecifier(tp, tz, "%:z", "+00:00"); + TestFormatSpecifier(tp, tz, "%Ez", "+00:00"); + + tz = fixed_time_zone(chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "+0000"); + TestFormatSpecifier(tp, tz, "%:z", "+00:00"); + TestFormatSpecifier(tp, tz, "%Ez", "+00:00"); + + tz = fixed_time_zone(-chrono::seconds(56)); // NOTE: +00:00 + TestFormatSpecifier(tp, tz, "%z", "+0000"); + TestFormatSpecifier(tp, tz, "%:z", "+00:00"); + TestFormatSpecifier(tp, tz, "%Ez", "+00:00"); + + tz = fixed_time_zone(chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%z", "+0034"); + TestFormatSpecifier(tp, tz, "%:z", "+00:34"); + TestFormatSpecifier(tp, tz, "%Ez", "+00:34"); + + tz = fixed_time_zone(-chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%z", "-0034"); + TestFormatSpecifier(tp, tz, "%:z", "-00:34"); + TestFormatSpecifier(tp, tz, "%Ez", "-00:34"); + + tz = fixed_time_zone(chrono::minutes(34) + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "+0034"); + TestFormatSpecifier(tp, tz, "%:z", "+00:34"); + TestFormatSpecifier(tp, tz, "%Ez", "+00:34"); + + tz = fixed_time_zone(-chrono::minutes(34) - chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "-0034"); + TestFormatSpecifier(tp, tz, "%:z", "-00:34"); + TestFormatSpecifier(tp, tz, "%Ez", "-00:34"); + + tz = fixed_time_zone(chrono::hours(12)); + TestFormatSpecifier(tp, tz, "%z", "+1200"); + TestFormatSpecifier(tp, tz, "%:z", "+12:00"); + TestFormatSpecifier(tp, tz, "%Ez", "+12:00"); + + tz = fixed_time_zone(-chrono::hours(12)); + TestFormatSpecifier(tp, tz, "%z", "-1200"); + TestFormatSpecifier(tp, tz, "%:z", "-12:00"); + TestFormatSpecifier(tp, tz, "%Ez", "-12:00"); + + tz = fixed_time_zone(chrono::hours(12) + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "+1200"); + TestFormatSpecifier(tp, tz, "%:z", "+12:00"); + TestFormatSpecifier(tp, tz, "%Ez", "+12:00"); + + tz = fixed_time_zone(-chrono::hours(12) - chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "-1200"); + TestFormatSpecifier(tp, tz, "%:z", "-12:00"); + TestFormatSpecifier(tp, tz, "%Ez", "-12:00"); + + tz = fixed_time_zone(chrono::hours(12) + chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%z", "+1234"); + TestFormatSpecifier(tp, tz, "%:z", "+12:34"); + TestFormatSpecifier(tp, tz, "%Ez", "+12:34"); + + tz = fixed_time_zone(-chrono::hours(12) - chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%z", "-1234"); + TestFormatSpecifier(tp, tz, "%:z", "-12:34"); + TestFormatSpecifier(tp, tz, "%Ez", "-12:34"); + + tz = fixed_time_zone(chrono::hours(12) + chrono::minutes(34) + + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "+1234"); + TestFormatSpecifier(tp, tz, "%:z", "+12:34"); + TestFormatSpecifier(tp, tz, "%Ez", "+12:34"); + + tz = fixed_time_zone(-chrono::hours(12) - chrono::minutes(34) - + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%z", "-1234"); + TestFormatSpecifier(tp, tz, "%:z", "-12:34"); + TestFormatSpecifier(tp, tz, "%Ez", "-12:34"); +} + +TEST(Format, ExtendedSecondOffset) { + const auto tp = chrono::system_clock::from_time_t(0); + + auto tz = fixed_time_zone(absl::time_internal::cctz::seconds::zero()); + TestFormatSpecifier(tp, tz, "%E*z", "+00:00:00"); + TestFormatSpecifier(tp, tz, "%::z", "+00:00:00"); + TestFormatSpecifier(tp, tz, "%:::z", "+00"); + + tz = fixed_time_zone(chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "+00:00:56"); + TestFormatSpecifier(tp, tz, "%::z", "+00:00:56"); + TestFormatSpecifier(tp, tz, "%:::z", "+00:00:56"); + + tz = fixed_time_zone(-chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "-00:00:56"); + TestFormatSpecifier(tp, tz, "%::z", "-00:00:56"); + TestFormatSpecifier(tp, tz, "%:::z", "-00:00:56"); + + tz = fixed_time_zone(chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%E*z", "+00:34:00"); + TestFormatSpecifier(tp, tz, "%::z", "+00:34:00"); + TestFormatSpecifier(tp, tz, "%:::z", "+00:34"); + + tz = fixed_time_zone(-chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%E*z", "-00:34:00"); + TestFormatSpecifier(tp, tz, "%::z", "-00:34:00"); + TestFormatSpecifier(tp, tz, "%:::z", "-00:34"); + + tz = fixed_time_zone(chrono::minutes(34) + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "+00:34:56"); + TestFormatSpecifier(tp, tz, "%::z", "+00:34:56"); + TestFormatSpecifier(tp, tz, "%:::z", "+00:34:56"); + + tz = fixed_time_zone(-chrono::minutes(34) - chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "-00:34:56"); + TestFormatSpecifier(tp, tz, "%::z", "-00:34:56"); + TestFormatSpecifier(tp, tz, "%:::z", "-00:34:56"); + + tz = fixed_time_zone(chrono::hours(12)); + TestFormatSpecifier(tp, tz, "%E*z", "+12:00:00"); + TestFormatSpecifier(tp, tz, "%::z", "+12:00:00"); + TestFormatSpecifier(tp, tz, "%:::z", "+12"); + + tz = fixed_time_zone(-chrono::hours(12)); + TestFormatSpecifier(tp, tz, "%E*z", "-12:00:00"); + TestFormatSpecifier(tp, tz, "%::z", "-12:00:00"); + TestFormatSpecifier(tp, tz, "%:::z", "-12"); + + tz = fixed_time_zone(chrono::hours(12) + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "+12:00:56"); + TestFormatSpecifier(tp, tz, "%::z", "+12:00:56"); + TestFormatSpecifier(tp, tz, "%:::z", "+12:00:56"); + + tz = fixed_time_zone(-chrono::hours(12) - chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "-12:00:56"); + TestFormatSpecifier(tp, tz, "%::z", "-12:00:56"); + TestFormatSpecifier(tp, tz, "%:::z", "-12:00:56"); + + tz = fixed_time_zone(chrono::hours(12) + chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%E*z", "+12:34:00"); + TestFormatSpecifier(tp, tz, "%::z", "+12:34:00"); + TestFormatSpecifier(tp, tz, "%:::z", "+12:34"); + + tz = fixed_time_zone(-chrono::hours(12) - chrono::minutes(34)); + TestFormatSpecifier(tp, tz, "%E*z", "-12:34:00"); + TestFormatSpecifier(tp, tz, "%::z", "-12:34:00"); + TestFormatSpecifier(tp, tz, "%:::z", "-12:34"); + + tz = fixed_time_zone(chrono::hours(12) + chrono::minutes(34) + + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "+12:34:56"); + TestFormatSpecifier(tp, tz, "%::z", "+12:34:56"); + TestFormatSpecifier(tp, tz, "%:::z", "+12:34:56"); + + tz = fixed_time_zone(-chrono::hours(12) - chrono::minutes(34) - + chrono::seconds(56)); + TestFormatSpecifier(tp, tz, "%E*z", "-12:34:56"); + TestFormatSpecifier(tp, tz, "%::z", "-12:34:56"); + TestFormatSpecifier(tp, tz, "%:::z", "-12:34:56"); +} + +TEST(Format, ExtendedYears) { + const time_zone utc = utc_time_zone(); + const char e4y_fmt[] = "%E4Y%m%d"; // no separators + + // %E4Y zero-pads the year to produce at least 4 chars, including the sign. + auto tp = convert(civil_second(-999, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("-9991127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(-99, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("-0991127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(-9, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("-0091127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(-1, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("-0011127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(0, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("00001127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(1, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("00011127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(9, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("00091127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(99, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("00991127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(999, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("09991127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(9999, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("99991127", format(e4y_fmt, tp, utc)); + + // When the year is outside [-999:9999], more than 4 chars are produced. + tp = convert(civil_second(-1000, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("-10001127", format(e4y_fmt, tp, utc)); + tp = convert(civil_second(10000, 11, 27, 0, 0, 0), utc); + EXPECT_EQ("100001127", format(e4y_fmt, tp, utc)); +} + +TEST(Format, RFC3339Format) { + time_zone tz; + EXPECT_TRUE(load_time_zone("America/Los_Angeles", &tz)); + + time_point<chrono::nanoseconds> tp = + convert(civil_second(1977, 6, 28, 9, 8, 7), tz); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::milliseconds(100); + EXPECT_EQ("1977-06-28T09:08:07.1-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::milliseconds(20); + EXPECT_EQ("1977-06-28T09:08:07.12-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::milliseconds(3); + EXPECT_EQ("1977-06-28T09:08:07.123-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::microseconds(400); + EXPECT_EQ("1977-06-28T09:08:07.1234-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::microseconds(50); + EXPECT_EQ("1977-06-28T09:08:07.12345-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::microseconds(6); + EXPECT_EQ("1977-06-28T09:08:07.123456-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::nanoseconds(700); + EXPECT_EQ("1977-06-28T09:08:07.1234567-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::nanoseconds(80); + EXPECT_EQ("1977-06-28T09:08:07.12345678-07:00", format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); + + tp += chrono::nanoseconds(9); + EXPECT_EQ("1977-06-28T09:08:07.123456789-07:00", + format(RFC3339_full, tp, tz)); + EXPECT_EQ("1977-06-28T09:08:07-07:00", format(RFC3339_sec, tp, tz)); +} + +TEST(Format, RFC1123Format) { // locale specific + time_zone tz; + EXPECT_TRUE(load_time_zone("America/Los_Angeles", &tz)); + + auto tp = convert(civil_second(1977, 6, 28, 9, 8, 7), tz); + EXPECT_EQ("Tue, 28 Jun 1977 09:08:07 -0700", format(RFC1123_full, tp, tz)); + EXPECT_EQ("28 Jun 1977 09:08:07 -0700", format(RFC1123_no_wday, tp, tz)); +} + +TEST(Format, Week) { + const time_zone utc = utc_time_zone(); + + auto tp = convert(civil_second(2017, 1, 1, 0, 0, 0), utc); + EXPECT_EQ("2017-01-7", format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2017-00-0", format("%Y-%W-%w", tp, utc)); + + tp = convert(civil_second(2017, 12, 31, 0, 0, 0), utc); + EXPECT_EQ("2017-53-7", format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2017-52-0", format("%Y-%W-%w", tp, utc)); + + tp = convert(civil_second(2018, 1, 1, 0, 0, 0), utc); + EXPECT_EQ("2018-00-1", format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2018-01-1", format("%Y-%W-%w", tp, utc)); + + tp = convert(civil_second(2018, 12, 31, 0, 0, 0), utc); + EXPECT_EQ("2018-52-1", format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2018-53-1", format("%Y-%W-%w", tp, utc)); + + tp = convert(civil_second(2019, 1, 1, 0, 0, 0), utc); + EXPECT_EQ("2019-00-2", format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2019-00-2", format("%Y-%W-%w", tp, utc)); + + tp = convert(civil_second(2019, 12, 31, 0, 0, 0), utc); + EXPECT_EQ("2019-52-2", format("%Y-%U-%u", tp, utc)); + EXPECT_EQ("2019-52-2", format("%Y-%W-%w", tp, utc)); +} + +// +// Testing parse() +// + +TEST(Parse, TimePointResolution) { + const char kFmt[] = "%H:%M:%E*S"; + const time_zone utc = utc_time_zone(); + + time_point<chrono::nanoseconds> tp_ns; + EXPECT_TRUE(parse(kFmt, "03:04:05.123456789", utc, &tp_ns)); + EXPECT_EQ("03:04:05.123456789", format(kFmt, tp_ns, utc)); + EXPECT_TRUE(parse(kFmt, "03:04:05.123456", utc, &tp_ns)); + EXPECT_EQ("03:04:05.123456", format(kFmt, tp_ns, utc)); + + time_point<chrono::microseconds> tp_us; + EXPECT_TRUE(parse(kFmt, "03:04:05.123456789", utc, &tp_us)); + EXPECT_EQ("03:04:05.123456", format(kFmt, tp_us, utc)); + EXPECT_TRUE(parse(kFmt, "03:04:05.123456", utc, &tp_us)); + EXPECT_EQ("03:04:05.123456", format(kFmt, tp_us, utc)); + EXPECT_TRUE(parse(kFmt, "03:04:05.123", utc, &tp_us)); + EXPECT_EQ("03:04:05.123", format(kFmt, tp_us, utc)); + + time_point<chrono::milliseconds> tp_ms; + EXPECT_TRUE(parse(kFmt, "03:04:05.123456", utc, &tp_ms)); + EXPECT_EQ("03:04:05.123", format(kFmt, tp_ms, utc)); + EXPECT_TRUE(parse(kFmt, "03:04:05.123", utc, &tp_ms)); + EXPECT_EQ("03:04:05.123", format(kFmt, tp_ms, utc)); + EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_ms)); + EXPECT_EQ("03:04:05", format(kFmt, tp_ms, utc)); + + time_point<chrono::seconds> tp_s; + EXPECT_TRUE(parse(kFmt, "03:04:05.123", utc, &tp_s)); + EXPECT_EQ("03:04:05", format(kFmt, tp_s, utc)); + EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_s)); + EXPECT_EQ("03:04:05", format(kFmt, tp_s, utc)); + + time_point<chrono::minutes> tp_m; + EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_m)); + EXPECT_EQ("03:04:00", format(kFmt, tp_m, utc)); + + time_point<chrono::hours> tp_h; + EXPECT_TRUE(parse(kFmt, "03:04:05", utc, &tp_h)); + EXPECT_EQ("03:00:00", format(kFmt, tp_h, utc)); +} + +TEST(Parse, TimePointExtendedResolution) { + const char kFmt[] = "%H:%M:%E*S"; + const time_zone utc = utc_time_zone(); + + time_point<absl::time_internal::cctz::seconds> tp; + detail::femtoseconds fs; + EXPECT_TRUE(detail::parse(kFmt, "12:34:56.123456789012345", utc, &tp, &fs)); + EXPECT_EQ("12:34:56.123456789012345", detail::format(kFmt, tp, fs, utc)); + EXPECT_TRUE(detail::parse(kFmt, "12:34:56.012345678901234", utc, &tp, &fs)); + EXPECT_EQ("12:34:56.012345678901234", detail::format(kFmt, tp, fs, utc)); + EXPECT_TRUE(detail::parse(kFmt, "12:34:56.001234567890123", utc, &tp, &fs)); + EXPECT_EQ("12:34:56.001234567890123", detail::format(kFmt, tp, fs, utc)); + EXPECT_TRUE(detail::parse(kFmt, "12:34:56.000000000000123", utc, &tp, &fs)); + EXPECT_EQ("12:34:56.000000000000123", detail::format(kFmt, tp, fs, utc)); + EXPECT_TRUE(detail::parse(kFmt, "12:34:56.000000000000012", utc, &tp, &fs)); + EXPECT_EQ("12:34:56.000000000000012", detail::format(kFmt, tp, fs, utc)); + EXPECT_TRUE(detail::parse(kFmt, "12:34:56.000000000000001", utc, &tp, &fs)); + EXPECT_EQ("12:34:56.000000000000001", detail::format(kFmt, tp, fs, utc)); +} + +TEST(Parse, Basics) { + time_zone tz = utc_time_zone(); + time_point<chrono::nanoseconds> tp = + chrono::system_clock::from_time_t(1234567890); + + // Simple edge cases. + EXPECT_TRUE(parse("", "", tz, &tp)); + EXPECT_EQ(chrono::system_clock::from_time_t(0), tp); // everything defaulted + EXPECT_TRUE(parse(" ", " ", tz, &tp)); + EXPECT_TRUE(parse(" ", " ", tz, &tp)); + EXPECT_TRUE(parse("x", "x", tz, &tp)); + EXPECT_TRUE(parse("xxx", "xxx", tz, &tp)); + + EXPECT_TRUE( + parse("%Y-%m-%d %H:%M:%S %z", "2013-06-28 19:08:09 -0800", tz, &tp)); + ExpectTime(tp, tz, 2013, 6, 29, 3, 8, 9, 0, false, "UTC"); +} + +TEST(Parse, WithTimeZone) { + time_zone tz; + EXPECT_TRUE(load_time_zone("America/Los_Angeles", &tz)); + time_point<chrono::nanoseconds> tp; + + // We can parse a string without a UTC offset if we supply a timezone. + EXPECT_TRUE(parse("%Y-%m-%d %H:%M:%S", "2013-06-28 19:08:09", tz, &tp)); + ExpectTime(tp, tz, 2013, 6, 28, 19, 8, 9, -7 * 60 * 60, true, "PDT"); + + // But the timezone is ignored when a UTC offset is present. + EXPECT_TRUE(parse("%Y-%m-%d %H:%M:%S %z", "2013-06-28 19:08:09 +0800", + utc_time_zone(), &tp)); + ExpectTime(tp, tz, 2013, 6, 28, 19 - 8 - 7, 8, 9, -7 * 60 * 60, true, "PDT"); + + // Check a skipped time (a Spring DST transition). parse() uses the + // pre-transition offset. + EXPECT_TRUE(parse("%Y-%m-%d %H:%M:%S", "2011-03-13 02:15:00", tz, &tp)); + ExpectTime(tp, tz, 2011, 3, 13, 3, 15, 0, -7 * 60 * 60, true, "PDT"); + + // Check a repeated time (a Fall DST transition). parse() uses the + // pre-transition offset. + EXPECT_TRUE(parse("%Y-%m-%d %H:%M:%S", "2011-11-06 01:15:00", tz, &tp)); + ExpectTime(tp, tz, 2011, 11, 6, 1, 15, 0, -7 * 60 * 60, true, "PDT"); +} + +TEST(Parse, LeapSecond) { + time_zone tz; + EXPECT_TRUE(load_time_zone("America/Los_Angeles", &tz)); + time_point<chrono::nanoseconds> tp; + + // ":59" -> ":59" + EXPECT_TRUE(parse(RFC3339_full, "2013-06-28T07:08:59-08:00", tz, &tp)); + ExpectTime(tp, tz, 2013, 6, 28, 8, 8, 59, -7 * 60 * 60, true, "PDT"); + + // ":59.5" -> ":59.5" + EXPECT_TRUE(parse(RFC3339_full, "2013-06-28T07:08:59.5-08:00", tz, &tp)); + ExpectTime(tp, tz, 2013, 6, 28, 8, 8, 59, -7 * 60 * 60, true, "PDT"); + + // ":60" -> ":00" + EXPECT_TRUE(parse(RFC3339_full, "2013-06-28T07:08:60-08:00", tz, &tp)); + ExpectTime(tp, tz, 2013, 6, 28, 8, 9, 0, -7 * 60 * 60, true, "PDT"); + + // ":60.5" -> ":00.0" + EXPECT_TRUE(parse(RFC3339_full, "2013-06-28T07:08:60.5-08:00", tz, &tp)); + ExpectTime(tp, tz, 2013, 6, 28, 8, 9, 0, -7 * 60 * 60, true, "PDT"); + + // ":61" -> error + EXPECT_FALSE(parse(RFC3339_full, "2013-06-28T07:08:61-08:00", tz, &tp)); +} + +TEST(Parse, ErrorCases) { + const time_zone tz = utc_time_zone(); + auto tp = chrono::system_clock::from_time_t(0); + + // Illegal trailing data. + EXPECT_FALSE(parse("%S", "123", tz, &tp)); + + // Can't parse an illegal format specifier. + EXPECT_FALSE(parse("%Q", "x", tz, &tp)); + + // Fails because of trailing, unparsed data "blah". + EXPECT_FALSE(parse("%m-%d", "2-3 blah", tz, &tp)); + + // Trailing whitespace is allowed. + EXPECT_TRUE(parse("%m-%d", "2-3 ", tz, &tp)); + EXPECT_EQ(2, convert(tp, utc_time_zone()).month()); + EXPECT_EQ(3, convert(tp, utc_time_zone()).day()); + + // Feb 31 requires normalization. + EXPECT_FALSE(parse("%m-%d", "2-31", tz, &tp)); + + // Check that we cannot have spaces in UTC offsets. + EXPECT_TRUE(parse("%z", "-0203", tz, &tp)); + EXPECT_FALSE(parse("%z", "- 2 3", tz, &tp)); + EXPECT_TRUE(parse("%Ez", "-02:03", tz, &tp)); + EXPECT_FALSE(parse("%Ez", "- 2: 3", tz, &tp)); + + // Check that we reject other malformed UTC offsets. + EXPECT_FALSE(parse("%Ez", "+-08:00", tz, &tp)); + EXPECT_FALSE(parse("%Ez", "-+08:00", tz, &tp)); + + // Check that we do not accept "-0" in fields that allow zero. + EXPECT_FALSE(parse("%Y", "-0", tz, &tp)); + EXPECT_FALSE(parse("%E4Y", "-0", tz, &tp)); + EXPECT_FALSE(parse("%H", "-0", tz, &tp)); + EXPECT_FALSE(parse("%M", "-0", tz, &tp)); + EXPECT_FALSE(parse("%S", "-0", tz, &tp)); + EXPECT_FALSE(parse("%z", "+-000", tz, &tp)); + EXPECT_FALSE(parse("%Ez", "+-0:00", tz, &tp)); + EXPECT_FALSE(parse("%z", "-00-0", tz, &tp)); + EXPECT_FALSE(parse("%Ez", "-00:-0", tz, &tp)); +} + +TEST(Parse, PosixConversions) { + time_zone tz = utc_time_zone(); + auto tp = chrono::system_clock::from_time_t(0); + const auto reset = convert(civil_second(1977, 6, 28, 9, 8, 7), tz); + + tp = reset; + EXPECT_TRUE(parse("%d", "15", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).day()); + + // %e is an extension, but is supported internally. + tp = reset; + EXPECT_TRUE(parse("%e", "15", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).day()); // Equivalent to %d + + tp = reset; + EXPECT_TRUE(parse("%H", "17", tz, &tp)); + EXPECT_EQ(17, convert(tp, tz).hour()); + + tp = reset; + EXPECT_TRUE(parse("%I", "5", tz, &tp)); + EXPECT_EQ(5, convert(tp, tz).hour()); + + // %j is parsed but ignored. + EXPECT_TRUE(parse("%j", "32", tz, &tp)); + + tp = reset; + EXPECT_TRUE(parse("%m", "11", tz, &tp)); + EXPECT_EQ(11, convert(tp, tz).month()); + + tp = reset; + EXPECT_TRUE(parse("%M", "33", tz, &tp)); + EXPECT_EQ(33, convert(tp, tz).minute()); + + tp = reset; + EXPECT_TRUE(parse("%S", "55", tz, &tp)); + EXPECT_EQ(55, convert(tp, tz).second()); + + // %U is parsed but ignored. + EXPECT_TRUE(parse("%U", "15", tz, &tp)); + + // %w is parsed but ignored. + EXPECT_TRUE(parse("%w", "2", tz, &tp)); + + // %W is parsed but ignored. + EXPECT_TRUE(parse("%W", "22", tz, &tp)); + + tp = reset; + EXPECT_TRUE(parse("%y", "04", tz, &tp)); + EXPECT_EQ(2004, convert(tp, tz).year()); + + tp = reset; + EXPECT_TRUE(parse("%Y", "2004", tz, &tp)); + EXPECT_EQ(2004, convert(tp, tz).year()); + + EXPECT_TRUE(parse("%%", "%", tz, &tp)); + +#if defined(__linux__) + // SU/C99/TZ extensions + + // Because we handle each (non-internal) specifier in a separate call + // to strptime(), there is no way to group %C and %y together. So we + // just skip the %C/%y case. +#if 0 + tp = reset; + EXPECT_TRUE(parse("%C %y", "20 04", tz, &tp)); + EXPECT_EQ(2004, convert(tp, tz).year()); +#endif + + tp = reset; + EXPECT_TRUE(parse("%D", "02/03/04", tz, &tp)); + EXPECT_EQ(2, convert(tp, tz).month()); + EXPECT_EQ(3, convert(tp, tz).day()); + EXPECT_EQ(2004, convert(tp, tz).year()); + + EXPECT_TRUE(parse("%n", "\n", tz, &tp)); + + tp = reset; + EXPECT_TRUE(parse("%R", "03:44", tz, &tp)); + EXPECT_EQ(3, convert(tp, tz).hour()); + EXPECT_EQ(44, convert(tp, tz).minute()); + + EXPECT_TRUE(parse("%t", "\t\v\f\n\r ", tz, &tp)); + + tp = reset; + EXPECT_TRUE(parse("%T", "03:44:55", tz, &tp)); + EXPECT_EQ(3, convert(tp, tz).hour()); + EXPECT_EQ(44, convert(tp, tz).minute()); + EXPECT_EQ(55, convert(tp, tz).second()); + + tp = reset; + EXPECT_TRUE(parse("%s", "1234567890", tz, &tp)); + EXPECT_EQ(chrono::system_clock::from_time_t(1234567890), tp); + + // %s conversion, like %z/%Ez, pays no heed to the optional zone. + time_zone lax; + EXPECT_TRUE(load_time_zone("America/Los_Angeles", &lax)); + tp = reset; + EXPECT_TRUE(parse("%s", "1234567890", lax, &tp)); + EXPECT_EQ(chrono::system_clock::from_time_t(1234567890), tp); + + // This is most important when the time has the same YMDhms + // breakdown in the zone as some other time. For example, ... + // 1414917000 in US/Pacific -> Sun Nov 2 01:30:00 2014 (PDT) + // 1414920600 in US/Pacific -> Sun Nov 2 01:30:00 2014 (PST) + tp = reset; + EXPECT_TRUE(parse("%s", "1414917000", lax, &tp)); + EXPECT_EQ(chrono::system_clock::from_time_t(1414917000), tp); + tp = reset; + EXPECT_TRUE(parse("%s", "1414920600", lax, &tp)); + EXPECT_EQ(chrono::system_clock::from_time_t(1414920600), tp); +#endif +} + +TEST(Parse, LocaleSpecific) { + time_zone tz = utc_time_zone(); + auto tp = chrono::system_clock::from_time_t(0); + const auto reset = convert(civil_second(1977, 6, 28, 9, 8, 7), tz); + + // %a is parsed but ignored. + EXPECT_TRUE(parse("%a", "Mon", tz, &tp)); + + // %A is parsed but ignored. + EXPECT_TRUE(parse("%A", "Monday", tz, &tp)); + + tp = reset; + EXPECT_TRUE(parse("%b", "Feb", tz, &tp)); + EXPECT_EQ(2, convert(tp, tz).month()); + + tp = reset; + EXPECT_TRUE(parse("%B", "February", tz, &tp)); + EXPECT_EQ(2, convert(tp, tz).month()); + + // %p is parsed but ignored if it's alone. But it's used with %I. + EXPECT_TRUE(parse("%p", "AM", tz, &tp)); + tp = reset; + EXPECT_TRUE(parse("%I %p", "5 PM", tz, &tp)); + EXPECT_EQ(17, convert(tp, tz).hour()); + + tp = reset; + EXPECT_TRUE(parse("%x", "02/03/04", tz, &tp)); + if (convert(tp, tz).month() == 2) { + EXPECT_EQ(3, convert(tp, tz).day()); + } else { + EXPECT_EQ(2, convert(tp, tz).day()); + EXPECT_EQ(3, convert(tp, tz).month()); + } + EXPECT_EQ(2004, convert(tp, tz).year()); + + tp = reset; + EXPECT_TRUE(parse("%X", "15:44:55", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).hour()); + EXPECT_EQ(44, convert(tp, tz).minute()); + EXPECT_EQ(55, convert(tp, tz).second()); + +#if defined(__linux__) + // SU/C99/TZ extensions + + tp = reset; + EXPECT_TRUE(parse("%h", "Feb", tz, &tp)); + EXPECT_EQ(2, convert(tp, tz).month()); // Equivalent to %b + + tp = reset; + EXPECT_TRUE(parse("%l %p", "5 PM", tz, &tp)); + EXPECT_EQ(17, convert(tp, tz).hour()); + + tp = reset; + EXPECT_TRUE(parse("%r", "03:44:55 PM", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).hour()); + EXPECT_EQ(44, convert(tp, tz).minute()); + EXPECT_EQ(55, convert(tp, tz).second()); + + tp = reset; + EXPECT_TRUE(parse("%Ec", "Tue Nov 19 05:06:07 2013", tz, &tp)); + EXPECT_EQ(convert(civil_second(2013, 11, 19, 5, 6, 7), tz), tp); + + // Modified conversion specifiers %E_ + + tp = reset; + EXPECT_TRUE(parse("%Ex", "02/03/04", tz, &tp)); + EXPECT_EQ(2, convert(tp, tz).month()); + EXPECT_EQ(3, convert(tp, tz).day()); + EXPECT_EQ(2004, convert(tp, tz).year()); + + tp = reset; + EXPECT_TRUE(parse("%EX", "15:44:55", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).hour()); + EXPECT_EQ(44, convert(tp, tz).minute()); + EXPECT_EQ(55, convert(tp, tz).second()); + + // %Ey, the year offset from %EC, doesn't really make sense alone as there + // is no way to represent it in tm_year (%EC is not simply the century). + // Yet, because we handle each (non-internal) specifier in a separate call + // to strptime(), there is no way to group %EC and %Ey either. So we just + // skip the %EC and %Ey cases. + + tp = reset; + EXPECT_TRUE(parse("%EY", "2004", tz, &tp)); + EXPECT_EQ(2004, convert(tp, tz).year()); + + // Modified conversion specifiers %O_ + + tp = reset; + EXPECT_TRUE(parse("%Od", "15", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).day()); + + tp = reset; + EXPECT_TRUE(parse("%Oe", "15", tz, &tp)); + EXPECT_EQ(15, convert(tp, tz).day()); // Equivalent to %d + + tp = reset; + EXPECT_TRUE(parse("%OH", "17", tz, &tp)); + EXPECT_EQ(17, convert(tp, tz).hour()); + + tp = reset; + EXPECT_TRUE(parse("%OI", "5", tz, &tp)); + EXPECT_EQ(5, convert(tp, tz).hour()); + + tp = reset; + EXPECT_TRUE(parse("%Om", "11", tz, &tp)); + EXPECT_EQ(11, convert(tp, tz).month()); + + tp = reset; + EXPECT_TRUE(parse("%OM", "33", tz, &tp)); + EXPECT_EQ(33, convert(tp, tz).minute()); + + tp = reset; + EXPECT_TRUE(parse("%OS", "55", tz, &tp)); + EXPECT_EQ(55, convert(tp, tz).second()); + + // %OU is parsed but ignored. + EXPECT_TRUE(parse("%OU", "15", tz, &tp)); + + // %Ow is parsed but ignored. + EXPECT_TRUE(parse("%Ow", "2", tz, &tp)); + + // %OW is parsed but ignored. + EXPECT_TRUE(parse("%OW", "22", tz, &tp)); + + tp = reset; + EXPECT_TRUE(parse("%Oy", "04", tz, &tp)); + EXPECT_EQ(2004, convert(tp, tz).year()); +#endif +} + +TEST(Parse, ExtendedSeconds) { + const time_zone tz = utc_time_zone(); + const time_point<chrono::nanoseconds> unix_epoch = + chrono::system_clock::from_time_t(0); + + // All %E<prec>S cases are treated the same as %E*S on input. + auto precisions = {"*", "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", "10", "11", "12", "13", "14", "15"}; + for (const std::string& prec : precisions) { + const std::string fmt = "%E" + prec + "S"; + SCOPED_TRACE(fmt); + time_point<chrono::nanoseconds> tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "5", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.0", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.00", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.6", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5) + chrono::milliseconds(600), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.60", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5) + chrono::milliseconds(600), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.600", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5) + chrono::milliseconds(600), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.67", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5) + chrono::milliseconds(670), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.670", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5) + chrono::milliseconds(670), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "05.678", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::seconds(5) + chrono::milliseconds(678), tp); + } + + // Here is a "%E*S" case we got wrong for a while. The fractional + // part of the first instant is less than 2^31 and was correctly + // parsed, while the second (and any subsecond field >=2^31) failed. + time_point<chrono::nanoseconds> tp = unix_epoch; + EXPECT_TRUE(parse("%E*S", "0.2147483647", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::nanoseconds(214748364), tp); + tp = unix_epoch; + EXPECT_TRUE(parse("%E*S", "0.2147483648", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::nanoseconds(214748364), tp); + + // We should also be able to specify long strings of digits far + // beyond the current resolution and have them convert the same way. + tp = unix_epoch; + EXPECT_TRUE(parse( + "%E*S", "0.214748364801234567890123456789012345678901234567890123456789", + tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::nanoseconds(214748364), tp); +} + +TEST(Parse, ExtendedSecondsScan) { + const time_zone tz = utc_time_zone(); + time_point<chrono::nanoseconds> tp; + for (int ms = 0; ms < 1000; ms += 111) { + for (int us = 0; us < 1000; us += 27) { + const int micros = ms * 1000 + us; + for (int ns = 0; ns < 1000; ns += 9) { + const auto expected = chrono::system_clock::from_time_t(0) + + chrono::nanoseconds(micros * 1000 + ns); + std::ostringstream oss; + oss << "0." << std::setfill('0') << std::setw(3); + oss << ms << std::setw(3) << us << std::setw(3) << ns; + const std::string input = oss.str(); + EXPECT_TRUE(parse("%E*S", input, tz, &tp)); + EXPECT_EQ(expected, tp) << input; + } + } + } +} + +TEST(Parse, ExtendedSubeconds) { + const time_zone tz = utc_time_zone(); + const time_point<chrono::nanoseconds> unix_epoch = + chrono::system_clock::from_time_t(0); + + // All %E<prec>f cases are treated the same as %E*f on input. + auto precisions = {"*", "0", "1", "2", "3", "4", "5", "6", "7", + "8", "9", "10", "11", "12", "13", "14", "15"}; + for (const std::string& prec : precisions) { + const std::string fmt = "%E" + prec + "f"; + SCOPED_TRACE(fmt); + time_point<chrono::nanoseconds> tp = unix_epoch - chrono::seconds(1); + EXPECT_TRUE(parse(fmt, "", tz, &tp)); + EXPECT_EQ(unix_epoch, tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "6", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::milliseconds(600), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "60", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::milliseconds(600), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "600", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::milliseconds(600), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "67", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::milliseconds(670), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "670", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::milliseconds(670), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "678", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::milliseconds(678), tp); + tp = unix_epoch; + EXPECT_TRUE(parse(fmt, "6789", tz, &tp)); + EXPECT_EQ( + unix_epoch + chrono::milliseconds(678) + chrono::microseconds(900), tp); + } + + // Here is a "%E*f" case we got wrong for a while. The fractional + // part of the first instant is less than 2^31 and was correctly + // parsed, while the second (and any subsecond field >=2^31) failed. + time_point<chrono::nanoseconds> tp = unix_epoch; + EXPECT_TRUE(parse("%E*f", "2147483647", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::nanoseconds(214748364), tp); + tp = unix_epoch; + EXPECT_TRUE(parse("%E*f", "2147483648", tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::nanoseconds(214748364), tp); + + // We should also be able to specify long strings of digits far + // beyond the current resolution and have them convert the same way. + tp = unix_epoch; + EXPECT_TRUE(parse( + "%E*f", "214748364801234567890123456789012345678901234567890123456789", + tz, &tp)); + EXPECT_EQ(unix_epoch + chrono::nanoseconds(214748364), tp); +} + +TEST(Parse, ExtendedSubecondsScan) { + time_point<chrono::nanoseconds> tp; + const time_zone tz = utc_time_zone(); + for (int ms = 0; ms < 1000; ms += 111) { + for (int us = 0; us < 1000; us += 27) { + const int micros = ms * 1000 + us; + for (int ns = 0; ns < 1000; ns += 9) { + std::ostringstream oss; + oss << std::setfill('0') << std::setw(3) << ms; + oss << std::setw(3) << us << std::setw(3) << ns; + const std::string nanos = oss.str(); + const auto expected = chrono::system_clock::from_time_t(0) + + chrono::nanoseconds(micros * 1000 + ns); + for (int ps = 0; ps < 1000; ps += 250) { + std::ostringstream ps_oss; + oss << std::setfill('0') << std::setw(3) << ps; + const std::string input = nanos + ps_oss.str() + "999"; + EXPECT_TRUE(parse("%E*f", input, tz, &tp)); + EXPECT_EQ(expected + chrono::nanoseconds(ps) / 1000, tp) << input; + } + } + } + } +} + +TEST(Parse, ExtendedOffset) { + const time_zone utc = utc_time_zone(); + time_point<absl::time_internal::cctz::seconds> tp; + + EXPECT_TRUE(parse("%Ez", "+00:00", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse("%Ez", "-12:34", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 34, 0), utc), tp); + EXPECT_TRUE(parse("%Ez", "+12:34", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 11, 26, 0), utc), tp); + EXPECT_FALSE(parse("%Ez", "-12:3", utc, &tp)); + + for (auto fmt : {"%Ez", "%z"}) { + EXPECT_TRUE(parse(fmt, "+0000", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-1234", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 34, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "+1234", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 11, 26, 0), utc), tp); + EXPECT_FALSE(parse(fmt, "-123", utc, &tp)); + + EXPECT_TRUE(parse(fmt, "+00", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-12", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "+12", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 12, 0, 0), utc), tp); + EXPECT_FALSE(parse(fmt, "-1", utc, &tp)); + } +} + +TEST(Parse, ExtendedSecondOffset) { + const time_zone utc = utc_time_zone(); + time_point<absl::time_internal::cctz::seconds> tp; + + for (auto fmt : {"%Ez", "%E*z", "%:z", "%::z", "%:::z"}) { + EXPECT_TRUE(parse(fmt, "+00:00:00", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-12:34:56", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 34, 56), utc), tp); + EXPECT_TRUE(parse(fmt, "+12:34:56", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 11, 25, 4), utc), tp); + EXPECT_FALSE(parse(fmt, "-12:34:5", utc, &tp)); + + EXPECT_TRUE(parse(fmt, "+000000", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-123456", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 34, 56), utc), tp); + EXPECT_TRUE(parse(fmt, "+123456", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 11, 25, 4), utc), tp); + EXPECT_FALSE(parse(fmt, "-12345", utc, &tp)); + + EXPECT_TRUE(parse(fmt, "+00:00", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-12:34", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 34, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "+12:34", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 11, 26, 0), utc), tp); + EXPECT_FALSE(parse(fmt, "-12:3", utc, &tp)); + + EXPECT_TRUE(parse(fmt, "+0000", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-1234", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 34, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "+1234", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 11, 26, 0), utc), tp); + EXPECT_FALSE(parse(fmt, "-123", utc, &tp)); + + EXPECT_TRUE(parse(fmt, "+00", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "-12", utc, &tp)); + EXPECT_EQ(convert(civil_second(1970, 1, 1, 12, 0, 0), utc), tp); + EXPECT_TRUE(parse(fmt, "+12", utc, &tp)); + EXPECT_EQ(convert(civil_second(1969, 12, 31, 12, 0, 0), utc), tp); + EXPECT_FALSE(parse(fmt, "-1", utc, &tp)); + } +} + +TEST(Parse, ExtendedYears) { + const time_zone utc = utc_time_zone(); + const char e4y_fmt[] = "%E4Y%m%d"; // no separators + time_point<absl::time_internal::cctz::seconds> tp; + + // %E4Y consumes exactly four chars, including any sign. + EXPECT_TRUE(parse(e4y_fmt, "-9991127", utc, &tp)); + EXPECT_EQ(convert(civil_second(-999, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "-0991127", utc, &tp)); + EXPECT_EQ(convert(civil_second(-99, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "-0091127", utc, &tp)); + EXPECT_EQ(convert(civil_second(-9, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "-0011127", utc, &tp)); + EXPECT_EQ(convert(civil_second(-1, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "00001127", utc, &tp)); + EXPECT_EQ(convert(civil_second(0, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "00011127", utc, &tp)); + EXPECT_EQ(convert(civil_second(1, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "00091127", utc, &tp)); + EXPECT_EQ(convert(civil_second(9, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "00991127", utc, &tp)); + EXPECT_EQ(convert(civil_second(99, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "09991127", utc, &tp)); + EXPECT_EQ(convert(civil_second(999, 11, 27, 0, 0, 0), utc), tp); + EXPECT_TRUE(parse(e4y_fmt, "99991127", utc, &tp)); + EXPECT_EQ(convert(civil_second(9999, 11, 27, 0, 0, 0), utc), tp); + + // When the year is outside [-999:9999], the parse fails. + EXPECT_FALSE(parse(e4y_fmt, "-10001127", utc, &tp)); + EXPECT_FALSE(parse(e4y_fmt, "100001127", utc, &tp)); +} + +TEST(Parse, RFC3339Format) { + const time_zone tz = utc_time_zone(); + time_point<chrono::nanoseconds> tp; + EXPECT_TRUE(parse(RFC3339_sec, "2014-02-12T20:21:00+00:00", tz, &tp)); + ExpectTime(tp, tz, 2014, 2, 12, 20, 21, 0, 0, false, "UTC"); + + // Check that %ET also accepts "t". + time_point<chrono::nanoseconds> tp2; + EXPECT_TRUE(parse(RFC3339_sec, "2014-02-12t20:21:00+00:00", tz, &tp2)); + EXPECT_EQ(tp, tp2); + + // Check that %Ez also accepts "Z" as a synonym for "+00:00". + time_point<chrono::nanoseconds> tp3; + EXPECT_TRUE(parse(RFC3339_sec, "2014-02-12T20:21:00Z", tz, &tp3)); + EXPECT_EQ(tp, tp3); + + // Check that %Ez also accepts "z" as a synonym for "+00:00". + time_point<chrono::nanoseconds> tp4; + EXPECT_TRUE(parse(RFC3339_sec, "2014-02-12T20:21:00z", tz, &tp4)); + EXPECT_EQ(tp, tp4); +} + +TEST(Parse, Week) { + const time_zone utc = utc_time_zone(); + time_point<absl::time_internal::cctz::seconds> tp; + + auto exp = convert(civil_second(2017, 1, 1, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2017-01-7", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2017-00-0", utc, &tp)); + EXPECT_EQ(exp, tp); + + exp = convert(civil_second(2017, 12, 31, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2017-53-7", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2017-52-0", utc, &tp)); + EXPECT_EQ(exp, tp); + + exp = convert(civil_second(2018, 1, 1, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2018-00-1", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2018-01-1", utc, &tp)); + EXPECT_EQ(exp, tp); + + exp = convert(civil_second(2018, 12, 31, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2018-52-1", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2018-53-1", utc, &tp)); + EXPECT_EQ(exp, tp); + + exp = convert(civil_second(2019, 1, 1, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2019-00-2", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2019-00-2", utc, &tp)); + EXPECT_EQ(exp, tp); + + exp = convert(civil_second(2019, 12, 31, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2019-52-2", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2019-52-2", utc, &tp)); + EXPECT_EQ(exp, tp); +} + +TEST(Parse, WeekYearShift) { + // %U/%W conversions with week values in {0, 52, 53} can slip + // into the previous/following calendar years. + const time_zone utc = utc_time_zone(); + time_point<absl::time_internal::cctz::seconds> tp; + + auto exp = convert(civil_second(2019, 12, 31, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2020-00-2", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2020-00-2", utc, &tp)); + EXPECT_EQ(exp, tp); + + exp = convert(civil_second(2021, 1, 1, 0, 0, 0), utc); + EXPECT_TRUE(parse("%Y-%U-%u", "2020-52-5", utc, &tp)); + EXPECT_EQ(exp, tp); + EXPECT_TRUE(parse("%Y-%W-%w", "2020-52-5", utc, &tp)); + EXPECT_EQ(exp, tp); + + // Slipping into the previous/following calendar years should fail when + // we're already at the extremes. + EXPECT_FALSE(parse("%Y-%U-%u", "-9223372036854775808-0-7", utc, &tp)); + EXPECT_FALSE(parse("%Y-%U-%u", "9223372036854775807-53-7", utc, &tp)); +} + +TEST(Parse, MaxRange) { + const time_zone utc = utc_time_zone(); + time_point<absl::time_internal::cctz::seconds> tp; + + // tests the upper limit using +00:00 offset + EXPECT_TRUE( + parse(RFC3339_sec, "292277026596-12-04T15:30:07+00:00", utc, &tp)); + EXPECT_EQ(tp, time_point<absl::time_internal::cctz::seconds>::max()); + EXPECT_FALSE( + parse(RFC3339_sec, "292277026596-12-04T15:30:08+00:00", utc, &tp)); + + // tests the upper limit using -01:00 offset + EXPECT_TRUE( + parse(RFC3339_sec, "292277026596-12-04T14:30:07-01:00", utc, &tp)); + EXPECT_EQ(tp, time_point<absl::time_internal::cctz::seconds>::max()); + EXPECT_FALSE( + parse(RFC3339_sec, "292277026596-12-04T15:30:07-01:00", utc, &tp)); + + // tests the lower limit using +00:00 offset + EXPECT_TRUE( + parse(RFC3339_sec, "-292277022657-01-27T08:29:52+00:00", utc, &tp)); + EXPECT_EQ(tp, time_point<absl::time_internal::cctz::seconds>::min()); + EXPECT_FALSE( + parse(RFC3339_sec, "-292277022657-01-27T08:29:51+00:00", utc, &tp)); + + // tests the lower limit using +01:00 offset + EXPECT_TRUE( + parse(RFC3339_sec, "-292277022657-01-27T09:29:52+01:00", utc, &tp)); + EXPECT_EQ(tp, time_point<absl::time_internal::cctz::seconds>::min()); + EXPECT_FALSE( + parse(RFC3339_sec, "-292277022657-01-27T08:29:51+01:00", utc, &tp)); + + // tests max/min civil-second overflow + EXPECT_FALSE( + parse(RFC3339_sec, "9223372036854775807-12-31T23:59:59-00:01", utc, &tp)); + EXPECT_FALSE(parse(RFC3339_sec, "-9223372036854775808-01-01T00:00:00+00:01", + utc, &tp)); + + // TODO: Add tests that parsing times with fractional seconds overflow + // appropriately. This can't be done until cctz::parse() properly detects + // overflow when combining the chrono seconds and femto. +} + +// +// Roundtrip test for format()/parse(). +// + +TEST(FormatParse, RoundTrip) { + time_zone lax; + EXPECT_TRUE(load_time_zone("America/Los_Angeles", &lax)); + const auto in = convert(civil_second(1977, 6, 28, 9, 8, 7), lax); + const auto subseconds = chrono::nanoseconds(654321); + + // RFC3339, which renders subseconds. + { + time_point<chrono::nanoseconds> out; + const std::string s = format(RFC3339_full, in + subseconds, lax); + EXPECT_TRUE(parse(RFC3339_full, s, lax, &out)) << s; + EXPECT_EQ(in + subseconds, out); // RFC3339_full includes %Ez + } + + // RFC1123, which only does whole seconds. + { + time_point<chrono::nanoseconds> out; + const std::string s = format(RFC1123_full, in, lax); + EXPECT_TRUE(parse(RFC1123_full, s, lax, &out)) << s; + EXPECT_EQ(in, out); // RFC1123_full includes %z + } + +#if defined(_WIN32) || defined(_WIN64) + // Initial investigations indicate the %c does not roundtrip on Windows. + // TODO: Figure out what is going on here (perhaps a locale problem). +#elif defined(__EMSCRIPTEN__) + // strftime() and strptime() use different defintions for "%c" under + // emscripten (see https://github.com/kripken/emscripten/pull/7491), + // causing its round-trip test to fail. +#else + // Even though we don't know what %c will produce, it should roundtrip, + // but only in the 0-offset timezone. + { + time_point<chrono::nanoseconds> out; + time_zone utc = utc_time_zone(); + const std::string s = format("%c", in, utc); + EXPECT_TRUE(parse("%c", s, utc, &out)) << s; + EXPECT_EQ(in, out); + } +#endif +} + +TEST(FormatParse, RoundTripDistantFuture) { + const time_zone utc = utc_time_zone(); + const time_point<absl::time_internal::cctz::seconds> in = + time_point<absl::time_internal::cctz::seconds>::max(); + const std::string s = format(RFC3339_full, in, utc); + time_point<absl::time_internal::cctz::seconds> out; + EXPECT_TRUE(parse(RFC3339_full, s, utc, &out)) << s; + EXPECT_EQ(in, out); +} + +TEST(FormatParse, RoundTripDistantPast) { + const time_zone utc = utc_time_zone(); + const time_point<absl::time_internal::cctz::seconds> in = + time_point<absl::time_internal::cctz::seconds>::min(); + const std::string s = format(RFC3339_full, in, utc); + time_point<absl::time_internal::cctz::seconds> out; + EXPECT_TRUE(parse(RFC3339_full, s, utc, &out)) << s; + EXPECT_EQ(in, out); +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_if.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_if.cc new file mode 100644 index 000000000000..0319b2f98e67 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_if.cc @@ -0,0 +1,45 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "time_zone_if.h" + +#include "absl/base/config.h" +#include "time_zone_info.h" +#include "time_zone_libc.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +std::unique_ptr<TimeZoneIf> TimeZoneIf::Load(const std::string& name) { + // Support "libc:localtime" and "libc:*" to access the legacy + // localtime and UTC support respectively from the C library. + if (name.compare(0, 5, "libc:") == 0) { + return std::unique_ptr<TimeZoneIf>(new TimeZoneLibC(name.substr(5))); + } + + // Otherwise use the "zoneinfo" implementation by default. + std::unique_ptr<TimeZoneInfo> tz(new TimeZoneInfo); + if (!tz->Load(name)) tz.reset(); + return std::unique_ptr<TimeZoneIf>(tz.release()); +} + +// Defined out-of-line to avoid emitting a weak vtable in all TUs. +TimeZoneIf::~TimeZoneIf() {} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_if.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_if.h new file mode 100644 index 000000000000..32c0891c1e77 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_if.h @@ -0,0 +1,76 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_IF_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_IF_H_ + +#include <chrono> +#include <cstdint> +#include <memory> +#include <string> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// A simple interface used to hide time-zone complexities from time_zone::Impl. +// Subclasses implement the functions for civil-time conversions in the zone. +class TimeZoneIf { + public: + // A factory function for TimeZoneIf implementations. + static std::unique_ptr<TimeZoneIf> Load(const std::string& name); + + virtual ~TimeZoneIf(); + + virtual time_zone::absolute_lookup BreakTime( + const time_point<seconds>& tp) const = 0; + virtual time_zone::civil_lookup MakeTime(const civil_second& cs) const = 0; + + virtual bool NextTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const = 0; + virtual bool PrevTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const = 0; + + virtual std::string Version() const = 0; + virtual std::string Description() const = 0; + + protected: + TimeZoneIf() {} +}; + +// Convert between time_point<seconds> and a count of seconds since the +// Unix epoch. We assume that the std::chrono::system_clock and the +// Unix clock are second aligned, but not that they share an epoch. +inline std::int_fast64_t ToUnixSeconds(const time_point<seconds>& tp) { + return (tp - std::chrono::time_point_cast<seconds>( + std::chrono::system_clock::from_time_t(0))) + .count(); +} +inline time_point<seconds> FromUnixSeconds(std::int_fast64_t t) { + return std::chrono::time_point_cast<seconds>( + std::chrono::system_clock::from_time_t(0)) + + seconds(t); +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_IF_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_impl.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_impl.cc new file mode 100644 index 000000000000..f34e3aec84d6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_impl.cc @@ -0,0 +1,113 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "time_zone_impl.h" + +#include <deque> +#include <memory> +#include <mutex> +#include <string> +#include <unordered_map> +#include <utility> + +#include "absl/base/config.h" +#include "time_zone_fixed.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +// time_zone::Impls are linked into a map to support fast lookup by name. +using TimeZoneImplByName = + std::unordered_map<std::string, const time_zone::Impl*>; +TimeZoneImplByName* time_zone_map = nullptr; + +// Mutual exclusion for time_zone_map. +std::mutex& TimeZoneMutex() { + // This mutex is intentionally "leaked" to avoid the static deinitialization + // order fiasco (std::mutex's destructor is not trivial on many platforms). + static std::mutex* time_zone_mutex = new std::mutex; + return *time_zone_mutex; +} + +} // namespace + +time_zone time_zone::Impl::UTC() { return time_zone(UTCImpl()); } + +bool time_zone::Impl::LoadTimeZone(const std::string& name, time_zone* tz) { + const Impl* const utc_impl = UTCImpl(); + + // Check for UTC (which is never a key in time_zone_map). + auto offset = seconds::zero(); + if (FixedOffsetFromName(name, &offset) && offset == seconds::zero()) { + *tz = time_zone(utc_impl); + return true; + } + + // Check whether the time zone has already been loaded. + { + std::lock_guard<std::mutex> lock(TimeZoneMutex()); + if (time_zone_map != nullptr) { + TimeZoneImplByName::const_iterator itr = time_zone_map->find(name); + if (itr != time_zone_map->end()) { + *tz = time_zone(itr->second); + return itr->second != utc_impl; + } + } + } + + // Load the new time zone (outside the lock). + std::unique_ptr<const Impl> new_impl(new Impl(name)); + + // Add the new time zone to the map. + std::lock_guard<std::mutex> lock(TimeZoneMutex()); + if (time_zone_map == nullptr) time_zone_map = new TimeZoneImplByName; + const Impl*& impl = (*time_zone_map)[name]; + if (impl == nullptr) { // this thread won any load race + impl = new_impl->zone_ ? new_impl.release() : utc_impl; + } + *tz = time_zone(impl); + return impl != utc_impl; +} + +void time_zone::Impl::ClearTimeZoneMapTestOnly() { + std::lock_guard<std::mutex> lock(TimeZoneMutex()); + if (time_zone_map != nullptr) { + // Existing time_zone::Impl* entries are in the wild, so we can't delete + // them. Instead, we move them to a private container, where they are + // logically unreachable but not "leaked". Future requests will result + // in reloading the data. + static auto* cleared = new std::deque<const time_zone::Impl*>; + for (const auto& element : *time_zone_map) { + cleared->push_back(element.second); + } + time_zone_map->clear(); + } +} + +time_zone::Impl::Impl(const std::string& name) + : name_(name), zone_(TimeZoneIf::Load(name_)) {} + +const time_zone::Impl* time_zone::Impl::UTCImpl() { + static const Impl* utc_impl = new Impl("UTC"); // never fails + return utc_impl; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_impl.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_impl.h new file mode 100644 index 000000000000..7d747ba96617 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_impl.h @@ -0,0 +1,93 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_IMPL_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_IMPL_H_ + +#include <memory> +#include <string> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" +#include "time_zone_if.h" +#include "time_zone_info.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// time_zone::Impl is the internal object referenced by a cctz::time_zone. +class time_zone::Impl { + public: + // The UTC time zone. Also used for other time zones that fail to load. + static time_zone UTC(); + + // Load a named time zone. Returns false if the name is invalid, or if + // some other kind of error occurs. Note that loading "UTC" never fails. + static bool LoadTimeZone(const std::string& name, time_zone* tz); + + // Clears the map of cached time zones. Primarily for use in benchmarks + // that gauge the performance of loading/parsing the time-zone data. + static void ClearTimeZoneMapTestOnly(); + + // The primary key is the time-zone ID (e.g., "America/New_York"). + const std::string& Name() const { + // TODO: It would nice if the zoneinfo data included the zone name. + return name_; + } + + // Breaks a time_point down to civil-time components in this time zone. + time_zone::absolute_lookup BreakTime(const time_point<seconds>& tp) const { + return zone_->BreakTime(tp); + } + + // Converts the civil-time components in this time zone into a time_point. + // That is, the opposite of BreakTime(). The requested civil time may be + // ambiguous or illegal due to a change of UTC offset. + time_zone::civil_lookup MakeTime(const civil_second& cs) const { + return zone_->MakeTime(cs); + } + + // Finds the time of the next/previous offset change in this time zone. + bool NextTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const { + return zone_->NextTransition(tp, trans); + } + bool PrevTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const { + return zone_->PrevTransition(tp, trans); + } + + // Returns an implementation-defined version string for this time zone. + std::string Version() const { return zone_->Version(); } + + // Returns an implementation-defined description of this time zone. + std::string Description() const { return zone_->Description(); } + + private: + explicit Impl(const std::string& name); + static const Impl* UTCImpl(); + + const std::string name_; + std::unique_ptr<TimeZoneIf> zone_; +}; + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_IMPL_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_info.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_info.cc new file mode 100644 index 000000000000..8039353e585e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_info.cc @@ -0,0 +1,965 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file implements the TimeZoneIf interface using the "zoneinfo" +// data provided by the IANA Time Zone Database (i.e., the only real game +// in town). +// +// TimeZoneInfo represents the history of UTC-offset changes within a time +// zone. Most changes are due to daylight-saving rules, but occasionally +// shifts are made to the time-zone's base offset. The database only attempts +// to be definitive for times since 1970, so be wary of local-time conversions +// before that. Also, rule and zone-boundary changes are made at the whim +// of governments, so the conversion of future times needs to be taken with +// a grain of salt. +// +// For more information see tzfile(5), http://www.iana.org/time-zones, or +// https://en.wikipedia.org/wiki/Zoneinfo. +// +// Note that we assume the proleptic Gregorian calendar and 60-second +// minutes throughout. + +#include "time_zone_info.h" + +#include <algorithm> +#include <cassert> +#include <chrono> +#include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <functional> +#include <memory> +#include <sstream> +#include <string> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "time_zone_fixed.h" +#include "time_zone_posix.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +inline bool IsLeap(year_t year) { + return (year % 4) == 0 && ((year % 100) != 0 || (year % 400) == 0); +} + +// The number of days in non-leap and leap years respectively. +const std::int_least32_t kDaysPerYear[2] = {365, 366}; + +// The day offsets of the beginning of each (1-based) month in non-leap and +// leap years respectively (e.g., 335 days before December in a leap year). +const std::int_least16_t kMonthOffsets[2][1 + 12 + 1] = { + {-1, 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365}, + {-1, 0, 31, 60, 91, 121, 152, 182, 213, 244, 274, 305, 335, 366}, +}; + +// We reject leap-second encoded zoneinfo and so assume 60-second minutes. +const std::int_least32_t kSecsPerDay = 24 * 60 * 60; + +// 400-year chunks always have 146097 days (20871 weeks). +const std::int_least64_t kSecsPer400Years = 146097LL * kSecsPerDay; + +// Like kDaysPerYear[] but scaled up by a factor of kSecsPerDay. +const std::int_least32_t kSecsPerYear[2] = { + 365 * kSecsPerDay, + 366 * kSecsPerDay, +}; + +// Convert a cctz::weekday to a POSIX TZ weekday number (0==Sun, ..., 6=Sat). +inline int ToPosixWeekday(weekday wd) { + switch (wd) { + case weekday::sunday: + return 0; + case weekday::monday: + return 1; + case weekday::tuesday: + return 2; + case weekday::wednesday: + return 3; + case weekday::thursday: + return 4; + case weekday::friday: + return 5; + case weekday::saturday: + return 6; + } + return 0; /*NOTREACHED*/ +} + +// Single-byte, unsigned numeric values are encoded directly. +inline std::uint_fast8_t Decode8(const char* cp) { + return static_cast<std::uint_fast8_t>(*cp) & 0xff; +} + +// Multi-byte, numeric values are encoded using a MSB first, +// twos-complement representation. These helpers decode, from +// the given address, 4-byte and 8-byte values respectively. +// Note: If int_fastXX_t == intXX_t and this machine is not +// twos complement, then there will be at least one input value +// we cannot represent. +std::int_fast32_t Decode32(const char* cp) { + std::uint_fast32_t v = 0; + for (int i = 0; i != (32 / 8); ++i) v = (v << 8) | Decode8(cp++); + const std::int_fast32_t s32max = 0x7fffffff; + const auto s32maxU = static_cast<std::uint_fast32_t>(s32max); + if (v <= s32maxU) return static_cast<std::int_fast32_t>(v); + return static_cast<std::int_fast32_t>(v - s32maxU - 1) - s32max - 1; +} + +std::int_fast64_t Decode64(const char* cp) { + std::uint_fast64_t v = 0; + for (int i = 0; i != (64 / 8); ++i) v = (v << 8) | Decode8(cp++); + const std::int_fast64_t s64max = 0x7fffffffffffffff; + const auto s64maxU = static_cast<std::uint_fast64_t>(s64max); + if (v <= s64maxU) return static_cast<std::int_fast64_t>(v); + return static_cast<std::int_fast64_t>(v - s64maxU - 1) - s64max - 1; +} + +// Generate a year-relative offset for a PosixTransition. +std::int_fast64_t TransOffset(bool leap_year, int jan1_weekday, + const PosixTransition& pt) { + std::int_fast64_t days = 0; + switch (pt.date.fmt) { + case PosixTransition::J: { + days = pt.date.j.day; + if (!leap_year || days < kMonthOffsets[1][3]) days -= 1; + break; + } + case PosixTransition::N: { + days = pt.date.n.day; + break; + } + case PosixTransition::M: { + const bool last_week = (pt.date.m.week == 5); + days = kMonthOffsets[leap_year][pt.date.m.month + last_week]; + const std::int_fast64_t weekday = (jan1_weekday + days) % 7; + if (last_week) { + days -= (weekday + 7 - 1 - pt.date.m.weekday) % 7 + 1; + } else { + days += (pt.date.m.weekday + 7 - weekday) % 7; + days += (pt.date.m.week - 1) * 7; + } + break; + } + } + return (days * kSecsPerDay) + pt.time.offset; +} + +inline time_zone::civil_lookup MakeUnique(const time_point<seconds>& tp) { + time_zone::civil_lookup cl; + cl.kind = time_zone::civil_lookup::UNIQUE; + cl.pre = cl.trans = cl.post = tp; + return cl; +} + +inline time_zone::civil_lookup MakeUnique(std::int_fast64_t unix_time) { + return MakeUnique(FromUnixSeconds(unix_time)); +} + +inline time_zone::civil_lookup MakeSkipped(const Transition& tr, + const civil_second& cs) { + time_zone::civil_lookup cl; + cl.kind = time_zone::civil_lookup::SKIPPED; + cl.pre = FromUnixSeconds(tr.unix_time - 1 + (cs - tr.prev_civil_sec)); + cl.trans = FromUnixSeconds(tr.unix_time); + cl.post = FromUnixSeconds(tr.unix_time - (tr.civil_sec - cs)); + return cl; +} + +inline time_zone::civil_lookup MakeRepeated(const Transition& tr, + const civil_second& cs) { + time_zone::civil_lookup cl; + cl.kind = time_zone::civil_lookup::REPEATED; + cl.pre = FromUnixSeconds(tr.unix_time - 1 - (tr.prev_civil_sec - cs)); + cl.trans = FromUnixSeconds(tr.unix_time); + cl.post = FromUnixSeconds(tr.unix_time + (cs - tr.civil_sec)); + return cl; +} + +inline civil_second YearShift(const civil_second& cs, year_t shift) { + return civil_second(cs.year() + shift, cs.month(), cs.day(), cs.hour(), + cs.minute(), cs.second()); +} + +} // namespace + +// What (no leap-seconds) UTC+seconds zoneinfo would look like. +bool TimeZoneInfo::ResetToBuiltinUTC(const seconds& offset) { + transition_types_.resize(1); + TransitionType& tt(transition_types_.back()); + tt.utc_offset = static_cast<std::int_least32_t>(offset.count()); + tt.is_dst = false; + tt.abbr_index = 0; + + // We temporarily add some redundant, contemporary (2015 through 2025) + // transitions for performance reasons. See TimeZoneInfo::LocalTime(). + // TODO: Fix the performance issue and remove the extra transitions. + transitions_.clear(); + transitions_.reserve(12); + for (const std::int_fast64_t unix_time : { + -(1LL << 59), // a "first half" transition + 1420070400LL, // 2015-01-01T00:00:00+00:00 + 1451606400LL, // 2016-01-01T00:00:00+00:00 + 1483228800LL, // 2017-01-01T00:00:00+00:00 + 1514764800LL, // 2018-01-01T00:00:00+00:00 + 1546300800LL, // 2019-01-01T00:00:00+00:00 + 1577836800LL, // 2020-01-01T00:00:00+00:00 + 1609459200LL, // 2021-01-01T00:00:00+00:00 + 1640995200LL, // 2022-01-01T00:00:00+00:00 + 1672531200LL, // 2023-01-01T00:00:00+00:00 + 1704067200LL, // 2024-01-01T00:00:00+00:00 + 1735689600LL, // 2025-01-01T00:00:00+00:00 + }) { + Transition& tr(*transitions_.emplace(transitions_.end())); + tr.unix_time = unix_time; + tr.type_index = 0; + tr.civil_sec = LocalTime(tr.unix_time, tt).cs; + tr.prev_civil_sec = tr.civil_sec - 1; + } + + default_transition_type_ = 0; + abbreviations_ = FixedOffsetToAbbr(offset); + abbreviations_.append(1, '\0'); + future_spec_.clear(); // never needed for a fixed-offset zone + extended_ = false; + + tt.civil_max = LocalTime(seconds::max().count(), tt).cs; + tt.civil_min = LocalTime(seconds::min().count(), tt).cs; + + transitions_.shrink_to_fit(); + return true; +} + +// Builds the in-memory header using the raw bytes from the file. +bool TimeZoneInfo::Header::Build(const tzhead& tzh) { + std::int_fast32_t v; + if ((v = Decode32(tzh.tzh_timecnt)) < 0) return false; + timecnt = static_cast<std::size_t>(v); + if ((v = Decode32(tzh.tzh_typecnt)) < 0) return false; + typecnt = static_cast<std::size_t>(v); + if ((v = Decode32(tzh.tzh_charcnt)) < 0) return false; + charcnt = static_cast<std::size_t>(v); + if ((v = Decode32(tzh.tzh_leapcnt)) < 0) return false; + leapcnt = static_cast<std::size_t>(v); + if ((v = Decode32(tzh.tzh_ttisstdcnt)) < 0) return false; + ttisstdcnt = static_cast<std::size_t>(v); + if ((v = Decode32(tzh.tzh_ttisutcnt)) < 0) return false; + ttisutcnt = static_cast<std::size_t>(v); + return true; +} + +// How many bytes of data are associated with this header. The result +// depends upon whether this is a section with 4-byte or 8-byte times. +std::size_t TimeZoneInfo::Header::DataLength(std::size_t time_len) const { + std::size_t len = 0; + len += (time_len + 1) * timecnt; // unix_time + type_index + len += (4 + 1 + 1) * typecnt; // utc_offset + is_dst + abbr_index + len += 1 * charcnt; // abbreviations + len += (time_len + 4) * leapcnt; // leap-time + TAI-UTC + len += 1 * ttisstdcnt; // UTC/local indicators + len += 1 * ttisutcnt; // standard/wall indicators + return len; +} + +// zic(8) can generate no-op transitions when a zone changes rules at an +// instant when there is actually no discontinuity. So we check whether +// two transitions have equivalent types (same offset/is_dst/abbr). +bool TimeZoneInfo::EquivTransitions(std::uint_fast8_t tt1_index, + std::uint_fast8_t tt2_index) const { + if (tt1_index == tt2_index) return true; + const TransitionType& tt1(transition_types_[tt1_index]); + const TransitionType& tt2(transition_types_[tt2_index]); + if (tt1.utc_offset != tt2.utc_offset) return false; + if (tt1.is_dst != tt2.is_dst) return false; + if (tt1.abbr_index != tt2.abbr_index) return false; + return true; +} + +// Find/make a transition type with these attributes. +bool TimeZoneInfo::GetTransitionType(std::int_fast32_t utc_offset, bool is_dst, + const std::string& abbr, + std::uint_least8_t* index) { + std::size_t type_index = 0; + std::size_t abbr_index = abbreviations_.size(); + for (; type_index != transition_types_.size(); ++type_index) { + const TransitionType& tt(transition_types_[type_index]); + const char* tt_abbr = &abbreviations_[tt.abbr_index]; + if (tt_abbr == abbr) abbr_index = tt.abbr_index; + if (tt.utc_offset == utc_offset && tt.is_dst == is_dst) { + if (abbr_index == tt.abbr_index) break; // reuse + } + } + if (type_index > 255 || abbr_index > 255) { + // No index space (8 bits) available for a new type or abbreviation. + return false; + } + if (type_index == transition_types_.size()) { + TransitionType& tt(*transition_types_.emplace(transition_types_.end())); + tt.utc_offset = static_cast<std::int_least32_t>(utc_offset); + tt.is_dst = is_dst; + if (abbr_index == abbreviations_.size()) { + abbreviations_.append(abbr); + abbreviations_.append(1, '\0'); + } + tt.abbr_index = static_cast<std::uint_least8_t>(abbr_index); + } + *index = static_cast<std::uint_least8_t>(type_index); + return true; +} + +// Use the POSIX-TZ-environment-variable-style string to handle times +// in years after the last transition stored in the zoneinfo data. +bool TimeZoneInfo::ExtendTransitions() { + extended_ = false; + if (future_spec_.empty()) return true; // last transition prevails + + PosixTimeZone posix; + if (!ParsePosixSpec(future_spec_, &posix)) return false; + + // Find transition type for the future std specification. + std::uint_least8_t std_ti; + if (!GetTransitionType(posix.std_offset, false, posix.std_abbr, &std_ti)) + return false; + + if (posix.dst_abbr.empty()) { // std only + // The future specification should match the last transition, and + // that means that handling the future will fall out naturally. + return EquivTransitions(transitions_.back().type_index, std_ti); + } + + // Find transition type for the future dst specification. + std::uint_least8_t dst_ti; + if (!GetTransitionType(posix.dst_offset, true, posix.dst_abbr, &dst_ti)) + return false; + + // Extend the transitions for an additional 400 years using the + // future specification. Years beyond those can be handled by + // mapping back to a cycle-equivalent year within that range. + // We may need two additional transitions for the current year. + transitions_.reserve(transitions_.size() + 400 * 2 + 2); + extended_ = true; + + const Transition& last(transitions_.back()); + const std::int_fast64_t last_time = last.unix_time; + const TransitionType& last_tt(transition_types_[last.type_index]); + last_year_ = LocalTime(last_time, last_tt).cs.year(); + bool leap_year = IsLeap(last_year_); + const civil_second jan1(last_year_); + std::int_fast64_t jan1_time = jan1 - civil_second(); + int jan1_weekday = ToPosixWeekday(get_weekday(jan1)); + + Transition dst = {0, dst_ti, civil_second(), civil_second()}; + Transition std = {0, std_ti, civil_second(), civil_second()}; + for (const year_t limit = last_year_ + 400;; ++last_year_) { + auto dst_trans_off = TransOffset(leap_year, jan1_weekday, posix.dst_start); + auto std_trans_off = TransOffset(leap_year, jan1_weekday, posix.dst_end); + dst.unix_time = jan1_time + dst_trans_off - posix.std_offset; + std.unix_time = jan1_time + std_trans_off - posix.dst_offset; + const auto* ta = dst.unix_time < std.unix_time ? &dst : &std; + const auto* tb = dst.unix_time < std.unix_time ? &std : &dst; + if (last_time < tb->unix_time) { + if (last_time < ta->unix_time) transitions_.push_back(*ta); + transitions_.push_back(*tb); + } + if (last_year_ == limit) break; + jan1_time += kSecsPerYear[leap_year]; + jan1_weekday = (jan1_weekday + kDaysPerYear[leap_year]) % 7; + leap_year = !leap_year && IsLeap(last_year_ + 1); + } + + return true; +} + +bool TimeZoneInfo::Load(ZoneInfoSource* zip) { + // Read and validate the header. + tzhead tzh; + if (zip->Read(&tzh, sizeof(tzh)) != sizeof(tzh)) return false; + if (strncmp(tzh.tzh_magic, TZ_MAGIC, sizeof(tzh.tzh_magic)) != 0) + return false; + Header hdr; + if (!hdr.Build(tzh)) return false; + std::size_t time_len = 4; + if (tzh.tzh_version[0] != '\0') { + // Skip the 4-byte data. + if (zip->Skip(hdr.DataLength(time_len)) != 0) return false; + // Read and validate the header for the 8-byte data. + if (zip->Read(&tzh, sizeof(tzh)) != sizeof(tzh)) return false; + if (strncmp(tzh.tzh_magic, TZ_MAGIC, sizeof(tzh.tzh_magic)) != 0) + return false; + if (tzh.tzh_version[0] == '\0') return false; + if (!hdr.Build(tzh)) return false; + time_len = 8; + } + if (hdr.typecnt == 0) return false; + if (hdr.leapcnt != 0) { + // This code assumes 60-second minutes so we do not want + // the leap-second encoded zoneinfo. We could reverse the + // compensation, but the "right" encoding is rarely used + // so currently we simply reject such data. + return false; + } + if (hdr.ttisstdcnt != 0 && hdr.ttisstdcnt != hdr.typecnt) return false; + if (hdr.ttisutcnt != 0 && hdr.ttisutcnt != hdr.typecnt) return false; + + // Read the data into a local buffer. + std::size_t len = hdr.DataLength(time_len); + std::vector<char> tbuf(len); + if (zip->Read(tbuf.data(), len) != len) return false; + const char* bp = tbuf.data(); + + // Decode and validate the transitions. + transitions_.reserve(hdr.timecnt + 2); + transitions_.resize(hdr.timecnt); + for (std::size_t i = 0; i != hdr.timecnt; ++i) { + transitions_[i].unix_time = (time_len == 4) ? Decode32(bp) : Decode64(bp); + bp += time_len; + if (i != 0) { + // Check that the transitions are ordered by time (as zic guarantees). + if (!Transition::ByUnixTime()(transitions_[i - 1], transitions_[i])) + return false; // out of order + } + } + bool seen_type_0 = false; + for (std::size_t i = 0; i != hdr.timecnt; ++i) { + transitions_[i].type_index = Decode8(bp++); + if (transitions_[i].type_index >= hdr.typecnt) return false; + if (transitions_[i].type_index == 0) seen_type_0 = true; + } + + // Decode and validate the transition types. + transition_types_.reserve(hdr.typecnt + 2); + transition_types_.resize(hdr.typecnt); + for (std::size_t i = 0; i != hdr.typecnt; ++i) { + transition_types_[i].utc_offset = + static_cast<std::int_least32_t>(Decode32(bp)); + if (transition_types_[i].utc_offset >= kSecsPerDay || + transition_types_[i].utc_offset <= -kSecsPerDay) + return false; + bp += 4; + transition_types_[i].is_dst = (Decode8(bp++) != 0); + transition_types_[i].abbr_index = Decode8(bp++); + if (transition_types_[i].abbr_index >= hdr.charcnt) return false; + } + + // Determine the before-first-transition type. + default_transition_type_ = 0; + if (seen_type_0 && hdr.timecnt != 0) { + std::uint_fast8_t index = 0; + if (transition_types_[0].is_dst) { + index = transitions_[0].type_index; + while (index != 0 && transition_types_[index].is_dst) --index; + } + while (index != hdr.typecnt && transition_types_[index].is_dst) ++index; + if (index != hdr.typecnt) default_transition_type_ = index; + } + + // Copy all the abbreviations. + abbreviations_.reserve(hdr.charcnt + 10); + abbreviations_.assign(bp, hdr.charcnt); + bp += hdr.charcnt; + + // Skip the unused portions. We've already dispensed with leap-second + // encoded zoneinfo. The ttisstd/ttisgmt indicators only apply when + // interpreting a POSIX spec that does not include start/end rules, and + // that isn't the case here (see "zic -p"). + bp += (8 + 4) * hdr.leapcnt; // leap-time + TAI-UTC + bp += 1 * hdr.ttisstdcnt; // UTC/local indicators + bp += 1 * hdr.ttisutcnt; // standard/wall indicators + assert(bp == tbuf.data() + tbuf.size()); + + future_spec_.clear(); + if (tzh.tzh_version[0] != '\0') { + // Snarf up the NL-enclosed future POSIX spec. Note + // that version '3' files utilize an extended format. + auto get_char = [](ZoneInfoSource* azip) -> int { + unsigned char ch; // all non-EOF results are positive + return (azip->Read(&ch, 1) == 1) ? ch : EOF; + }; + if (get_char(zip) != '\n') return false; + for (int c = get_char(zip); c != '\n'; c = get_char(zip)) { + if (c == EOF) return false; + future_spec_.push_back(static_cast<char>(c)); + } + } + + // We don't check for EOF so that we're forwards compatible. + + // If we did not find version information during the standard loading + // process (as of tzh_version '3' that is unsupported), then ask the + // ZoneInfoSource for any out-of-bound version string it may be privy to. + if (version_.empty()) { + version_ = zip->Version(); + } + + // Trim redundant transitions. zic may have added these to work around + // differences between the glibc and reference implementations (see + // zic.c:dontmerge) and the Qt library (see zic.c:WORK_AROUND_QTBUG_53071). + // For us, they just get in the way when we do future_spec_ extension. + while (hdr.timecnt > 1) { + if (!EquivTransitions(transitions_[hdr.timecnt - 1].type_index, + transitions_[hdr.timecnt - 2].type_index)) { + break; + } + hdr.timecnt -= 1; + } + transitions_.resize(hdr.timecnt); + + // Ensure that there is always a transition in the first half of the + // time line (the second half is handled below) so that the signed + // difference between a civil_second and the civil_second of its + // previous transition is always representable, without overflow. + if (transitions_.empty() || transitions_.front().unix_time >= 0) { + Transition& tr(*transitions_.emplace(transitions_.begin())); + tr.unix_time = -(1LL << 59); // -18267312070-10-26T17:01:52+00:00 + tr.type_index = default_transition_type_; + } + + // Extend the transitions using the future specification. + if (!ExtendTransitions()) return false; + + // Ensure that there is always a transition in the second half of the + // time line (the first half is handled above) so that the signed + // difference between a civil_second and the civil_second of its + // previous transition is always representable, without overflow. + const Transition& last(transitions_.back()); + if (last.unix_time < 0) { + const std::uint_fast8_t type_index = last.type_index; + Transition& tr(*transitions_.emplace(transitions_.end())); + tr.unix_time = 2147483647; // 2038-01-19T03:14:07+00:00 + tr.type_index = type_index; + } + + // Compute the local civil time for each transition and the preceding + // second. These will be used for reverse conversions in MakeTime(). + const TransitionType* ttp = &transition_types_[default_transition_type_]; + for (std::size_t i = 0; i != transitions_.size(); ++i) { + Transition& tr(transitions_[i]); + tr.prev_civil_sec = LocalTime(tr.unix_time, *ttp).cs - 1; + ttp = &transition_types_[tr.type_index]; + tr.civil_sec = LocalTime(tr.unix_time, *ttp).cs; + if (i != 0) { + // Check that the transitions are ordered by civil time. Essentially + // this means that an offset change cannot cross another such change. + // No one does this in practice, and we depend on it in MakeTime(). + if (!Transition::ByCivilTime()(transitions_[i - 1], tr)) + return false; // out of order + } + } + + // Compute the maximum/minimum civil times that can be converted to a + // time_point<seconds> for each of the zone's transition types. + for (auto& tt : transition_types_) { + tt.civil_max = LocalTime(seconds::max().count(), tt).cs; + tt.civil_min = LocalTime(seconds::min().count(), tt).cs; + } + + transitions_.shrink_to_fit(); + return true; +} + +namespace { + +// fopen(3) adaptor. +inline FILE* FOpen(const char* path, const char* mode) { +#if defined(_MSC_VER) + FILE* fp; + if (fopen_s(&fp, path, mode) != 0) fp = nullptr; + return fp; +#else + return fopen(path, mode); // TODO: Enable the close-on-exec flag. +#endif +} + +// A stdio(3)-backed implementation of ZoneInfoSource. +class FileZoneInfoSource : public ZoneInfoSource { + public: + static std::unique_ptr<ZoneInfoSource> Open(const std::string& name); + + std::size_t Read(void* ptr, std::size_t size) override { + size = std::min(size, len_); + std::size_t nread = fread(ptr, 1, size, fp_.get()); + len_ -= nread; + return nread; + } + int Skip(std::size_t offset) override { + offset = std::min(offset, len_); + int rc = fseek(fp_.get(), static_cast<long>(offset), SEEK_CUR); + if (rc == 0) len_ -= offset; + return rc; + } + std::string Version() const override { + // TODO: It would nice if the zoneinfo data included the tzdb version. + return std::string(); + } + + protected: + explicit FileZoneInfoSource( + FILE* fp, std::size_t len = std::numeric_limits<std::size_t>::max()) + : fp_(fp, fclose), len_(len) {} + + private: + std::unique_ptr<FILE, int (*)(FILE*)> fp_; + std::size_t len_; +}; + +std::unique_ptr<ZoneInfoSource> FileZoneInfoSource::Open( + const std::string& name) { + // Use of the "file:" prefix is intended for testing purposes only. + const std::size_t pos = (name.compare(0, 5, "file:") == 0) ? 5 : 0; + + // Map the time-zone name to a path name. + std::string path; + if (pos == name.size() || name[pos] != '/') { + const char* tzdir = "/usr/share/zoneinfo"; + char* tzdir_env = nullptr; +#if defined(_MSC_VER) + _dupenv_s(&tzdir_env, nullptr, "TZDIR"); +#else + tzdir_env = std::getenv("TZDIR"); +#endif + if (tzdir_env && *tzdir_env) tzdir = tzdir_env; + path += tzdir; + path += '/'; +#if defined(_MSC_VER) + free(tzdir_env); +#endif + } + path.append(name, pos, std::string::npos); + + // Open the zoneinfo file. + FILE* fp = FOpen(path.c_str(), "rb"); + if (fp == nullptr) return nullptr; + std::size_t length = 0; + if (fseek(fp, 0, SEEK_END) == 0) { + long offset = ftell(fp); + if (offset >= 0) { + length = static_cast<std::size_t>(offset); + } + rewind(fp); + } + return std::unique_ptr<ZoneInfoSource>(new FileZoneInfoSource(fp, length)); +} + +class AndroidZoneInfoSource : public FileZoneInfoSource { + public: + static std::unique_ptr<ZoneInfoSource> Open(const std::string& name); + std::string Version() const override { return version_; } + + private: + explicit AndroidZoneInfoSource(FILE* fp, std::size_t len, const char* vers) + : FileZoneInfoSource(fp, len), version_(vers) {} + std::string version_; +}; + +std::unique_ptr<ZoneInfoSource> AndroidZoneInfoSource::Open( + const std::string& name) { + // Use of the "file:" prefix is intended for testing purposes only. + const std::size_t pos = (name.compare(0, 5, "file:") == 0) ? 5 : 0; + + // See Android's libc/tzcode/bionic.cpp for additional information. + for (const char* tzdata : {"/data/misc/zoneinfo/current/tzdata", + "/system/usr/share/zoneinfo/tzdata"}) { + std::unique_ptr<FILE, int (*)(FILE*)> fp(FOpen(tzdata, "rb"), fclose); + if (fp.get() == nullptr) continue; + + char hbuf[24]; // covers header.zonetab_offset too + if (fread(hbuf, 1, sizeof(hbuf), fp.get()) != sizeof(hbuf)) continue; + if (strncmp(hbuf, "tzdata", 6) != 0) continue; + const char* vers = (hbuf[11] == '\0') ? hbuf + 6 : ""; + const std::int_fast32_t index_offset = Decode32(hbuf + 12); + const std::int_fast32_t data_offset = Decode32(hbuf + 16); + if (index_offset < 0 || data_offset < index_offset) continue; + if (fseek(fp.get(), static_cast<long>(index_offset), SEEK_SET) != 0) + continue; + + char ebuf[52]; // covers entry.unused too + const std::size_t index_size = + static_cast<std::size_t>(data_offset - index_offset); + const std::size_t zonecnt = index_size / sizeof(ebuf); + if (zonecnt * sizeof(ebuf) != index_size) continue; + for (std::size_t i = 0; i != zonecnt; ++i) { + if (fread(ebuf, 1, sizeof(ebuf), fp.get()) != sizeof(ebuf)) break; + const std::int_fast32_t start = data_offset + Decode32(ebuf + 40); + const std::int_fast32_t length = Decode32(ebuf + 44); + if (start < 0 || length < 0) break; + ebuf[40] = '\0'; // ensure zone name is NUL terminated + if (strcmp(name.c_str() + pos, ebuf) == 0) { + if (fseek(fp.get(), static_cast<long>(start), SEEK_SET) != 0) break; + return std::unique_ptr<ZoneInfoSource>(new AndroidZoneInfoSource( + fp.release(), static_cast<std::size_t>(length), vers)); + } + } + } + + return nullptr; +} + +} // namespace + +bool TimeZoneInfo::Load(const std::string& name) { + // We can ensure that the loading of UTC or any other fixed-offset + // zone never fails because the simple, fixed-offset state can be + // internally generated. Note that this depends on our choice to not + // accept leap-second encoded ("right") zoneinfo. + auto offset = seconds::zero(); + if (FixedOffsetFromName(name, &offset)) { + return ResetToBuiltinUTC(offset); + } + + // Find and use a ZoneInfoSource to load the named zone. + auto zip = cctz_extension::zone_info_source_factory( + name, [](const std::string& n) -> std::unique_ptr<ZoneInfoSource> { + if (auto z = FileZoneInfoSource::Open(n)) return z; + if (auto z = AndroidZoneInfoSource::Open(n)) return z; + return nullptr; + }); + return zip != nullptr && Load(zip.get()); +} + +// BreakTime() translation for a particular transition type. +time_zone::absolute_lookup TimeZoneInfo::LocalTime( + std::int_fast64_t unix_time, const TransitionType& tt) const { + // A civil time in "+offset" looks like (time+offset) in UTC. + // Note: We perform two additions in the civil_second domain to + // sidestep the chance of overflow in (unix_time + tt.utc_offset). + return {(civil_second() + unix_time) + tt.utc_offset, tt.utc_offset, + tt.is_dst, &abbreviations_[tt.abbr_index]}; +} + +// BreakTime() translation for a particular transition. +time_zone::absolute_lookup TimeZoneInfo::LocalTime(std::int_fast64_t unix_time, + const Transition& tr) const { + const TransitionType& tt = transition_types_[tr.type_index]; + // Note: (unix_time - tr.unix_time) will never overflow as we + // have ensured that there is always a "nearby" transition. + return {tr.civil_sec + (unix_time - tr.unix_time), // TODO: Optimize. + tt.utc_offset, tt.is_dst, &abbreviations_[tt.abbr_index]}; +} + +// MakeTime() translation with a conversion-preserving +N * 400-year shift. +time_zone::civil_lookup TimeZoneInfo::TimeLocal(const civil_second& cs, + year_t c4_shift) const { + assert(last_year_ - 400 < cs.year() && cs.year() <= last_year_); + time_zone::civil_lookup cl = MakeTime(cs); + if (c4_shift > seconds::max().count() / kSecsPer400Years) { + cl.pre = cl.trans = cl.post = time_point<seconds>::max(); + } else { + const auto offset = seconds(c4_shift * kSecsPer400Years); + const auto limit = time_point<seconds>::max() - offset; + for (auto* tp : {&cl.pre, &cl.trans, &cl.post}) { + if (*tp > limit) { + *tp = time_point<seconds>::max(); + } else { + *tp += offset; + } + } + } + return cl; +} + +time_zone::absolute_lookup TimeZoneInfo::BreakTime( + const time_point<seconds>& tp) const { + std::int_fast64_t unix_time = ToUnixSeconds(tp); + const std::size_t timecnt = transitions_.size(); + assert(timecnt != 0); // We always add a transition. + + if (unix_time < transitions_[0].unix_time) { + return LocalTime(unix_time, transition_types_[default_transition_type_]); + } + if (unix_time >= transitions_[timecnt - 1].unix_time) { + // After the last transition. If we extended the transitions using + // future_spec_, shift back to a supported year using the 400-year + // cycle of calendaric equivalence and then compensate accordingly. + if (extended_) { + const std::int_fast64_t diff = + unix_time - transitions_[timecnt - 1].unix_time; + const year_t shift = diff / kSecsPer400Years + 1; + const auto d = seconds(shift * kSecsPer400Years); + time_zone::absolute_lookup al = BreakTime(tp - d); + al.cs = YearShift(al.cs, shift * 400); + return al; + } + return LocalTime(unix_time, transitions_[timecnt - 1]); + } + + const std::size_t hint = local_time_hint_.load(std::memory_order_relaxed); + if (0 < hint && hint < timecnt) { + if (transitions_[hint - 1].unix_time <= unix_time) { + if (unix_time < transitions_[hint].unix_time) { + return LocalTime(unix_time, transitions_[hint - 1]); + } + } + } + + const Transition target = {unix_time, 0, civil_second(), civil_second()}; + const Transition* begin = &transitions_[0]; + const Transition* tr = std::upper_bound(begin, begin + timecnt, target, + Transition::ByUnixTime()); + local_time_hint_.store(static_cast<std::size_t>(tr - begin), + std::memory_order_relaxed); + return LocalTime(unix_time, *--tr); +} + +time_zone::civil_lookup TimeZoneInfo::MakeTime(const civil_second& cs) const { + const std::size_t timecnt = transitions_.size(); + assert(timecnt != 0); // We always add a transition. + + // Find the first transition after our target civil time. + const Transition* tr = nullptr; + const Transition* begin = &transitions_[0]; + const Transition* end = begin + timecnt; + if (cs < begin->civil_sec) { + tr = begin; + } else if (cs >= transitions_[timecnt - 1].civil_sec) { + tr = end; + } else { + const std::size_t hint = time_local_hint_.load(std::memory_order_relaxed); + if (0 < hint && hint < timecnt) { + if (transitions_[hint - 1].civil_sec <= cs) { + if (cs < transitions_[hint].civil_sec) { + tr = begin + hint; + } + } + } + if (tr == nullptr) { + const Transition target = {0, 0, cs, civil_second()}; + tr = std::upper_bound(begin, end, target, Transition::ByCivilTime()); + time_local_hint_.store(static_cast<std::size_t>(tr - begin), + std::memory_order_relaxed); + } + } + + if (tr == begin) { + if (tr->prev_civil_sec >= cs) { + // Before first transition, so use the default offset. + const TransitionType& tt(transition_types_[default_transition_type_]); + if (cs < tt.civil_min) return MakeUnique(time_point<seconds>::min()); + return MakeUnique(cs - (civil_second() + tt.utc_offset)); + } + // tr->prev_civil_sec < cs < tr->civil_sec + return MakeSkipped(*tr, cs); + } + + if (tr == end) { + if (cs > (--tr)->prev_civil_sec) { + // After the last transition. If we extended the transitions using + // future_spec_, shift back to a supported year using the 400-year + // cycle of calendaric equivalence and then compensate accordingly. + if (extended_ && cs.year() > last_year_) { + const year_t shift = (cs.year() - last_year_ - 1) / 400 + 1; + return TimeLocal(YearShift(cs, shift * -400), shift); + } + const TransitionType& tt(transition_types_[tr->type_index]); + if (cs > tt.civil_max) return MakeUnique(time_point<seconds>::max()); + return MakeUnique(tr->unix_time + (cs - tr->civil_sec)); + } + // tr->civil_sec <= cs <= tr->prev_civil_sec + return MakeRepeated(*tr, cs); + } + + if (tr->prev_civil_sec < cs) { + // tr->prev_civil_sec < cs < tr->civil_sec + return MakeSkipped(*tr, cs); + } + + if (cs <= (--tr)->prev_civil_sec) { + // tr->civil_sec <= cs <= tr->prev_civil_sec + return MakeRepeated(*tr, cs); + } + + // In between transitions. + return MakeUnique(tr->unix_time + (cs - tr->civil_sec)); +} + +std::string TimeZoneInfo::Version() const { return version_; } + +std::string TimeZoneInfo::Description() const { + std::ostringstream oss; + oss << "#trans=" << transitions_.size(); + oss << " #types=" << transition_types_.size(); + oss << " spec='" << future_spec_ << "'"; + return oss.str(); +} + +bool TimeZoneInfo::NextTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const { + if (transitions_.empty()) return false; + const Transition* begin = &transitions_[0]; + const Transition* end = begin + transitions_.size(); + if (begin->unix_time <= -(1LL << 59)) { + // Do not report the BIG_BANG found in some zoneinfo data as it is + // really a sentinel, not a transition. See pre-2018f tz/zic.c. + ++begin; + } + std::int_fast64_t unix_time = ToUnixSeconds(tp); + const Transition target = {unix_time, 0, civil_second(), civil_second()}; + const Transition* tr = + std::upper_bound(begin, end, target, Transition::ByUnixTime()); + for (; tr != end; ++tr) { // skip no-op transitions + std::uint_fast8_t prev_type_index = + (tr == begin) ? default_transition_type_ : tr[-1].type_index; + if (!EquivTransitions(prev_type_index, tr[0].type_index)) break; + } + // When tr == end we return false, ignoring future_spec_. + if (tr == end) return false; + trans->from = tr->prev_civil_sec + 1; + trans->to = tr->civil_sec; + return true; +} + +bool TimeZoneInfo::PrevTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const { + if (transitions_.empty()) return false; + const Transition* begin = &transitions_[0]; + const Transition* end = begin + transitions_.size(); + if (begin->unix_time <= -(1LL << 59)) { + // Do not report the BIG_BANG found in some zoneinfo data as it is + // really a sentinel, not a transition. See pre-2018f tz/zic.c. + ++begin; + } + std::int_fast64_t unix_time = ToUnixSeconds(tp); + if (FromUnixSeconds(unix_time) != tp) { + if (unix_time == std::numeric_limits<std::int_fast64_t>::max()) { + if (end == begin) return false; // Ignore future_spec_. + trans->from = (--end)->prev_civil_sec + 1; + trans->to = end->civil_sec; + return true; + } + unix_time += 1; // ceils + } + const Transition target = {unix_time, 0, civil_second(), civil_second()}; + const Transition* tr = + std::lower_bound(begin, end, target, Transition::ByUnixTime()); + for (; tr != begin; --tr) { // skip no-op transitions + std::uint_fast8_t prev_type_index = + (tr - 1 == begin) ? default_transition_type_ : tr[-2].type_index; + if (!EquivTransitions(prev_type_index, tr[-1].type_index)) break; + } + // When tr == end we return the "last" transition, ignoring future_spec_. + if (tr == begin) return false; + trans->from = (--tr)->prev_civil_sec + 1; + trans->to = tr->civil_sec; + return true; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_info.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_info.h new file mode 100644 index 000000000000..2467ff559d35 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_info.h @@ -0,0 +1,137 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_INFO_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_INFO_H_ + +#include <atomic> +#include <cstddef> +#include <cstdint> +#include <string> +#include <vector> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" +#include "absl/time/internal/cctz/include/cctz/zone_info_source.h" +#include "time_zone_if.h" +#include "tzfile.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// A transition to a new UTC offset. +struct Transition { + std::int_least64_t unix_time; // the instant of this transition + std::uint_least8_t type_index; // index of the transition type + civil_second civil_sec; // local civil time of transition + civil_second prev_civil_sec; // local civil time one second earlier + + struct ByUnixTime { + inline bool operator()(const Transition& lhs, const Transition& rhs) const { + return lhs.unix_time < rhs.unix_time; + } + }; + struct ByCivilTime { + inline bool operator()(const Transition& lhs, const Transition& rhs) const { + return lhs.civil_sec < rhs.civil_sec; + } + }; +}; + +// The characteristics of a particular transition. +struct TransitionType { + std::int_least32_t utc_offset; // the new prevailing UTC offset + civil_second civil_max; // max convertible civil time for offset + civil_second civil_min; // min convertible civil time for offset + bool is_dst; // did we move into daylight-saving time + std::uint_least8_t abbr_index; // index of the new abbreviation +}; + +// A time zone backed by the IANA Time Zone Database (zoneinfo). +class TimeZoneInfo : public TimeZoneIf { + public: + TimeZoneInfo() = default; + TimeZoneInfo(const TimeZoneInfo&) = delete; + TimeZoneInfo& operator=(const TimeZoneInfo&) = delete; + + // Loads the zoneinfo for the given name, returning true if successful. + bool Load(const std::string& name); + + // TimeZoneIf implementations. + time_zone::absolute_lookup BreakTime( + const time_point<seconds>& tp) const override; + time_zone::civil_lookup MakeTime(const civil_second& cs) const override; + bool NextTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const override; + bool PrevTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const override; + std::string Version() const override; + std::string Description() const override; + + private: + struct Header { // counts of: + std::size_t timecnt; // transition times + std::size_t typecnt; // transition types + std::size_t charcnt; // zone abbreviation characters + std::size_t leapcnt; // leap seconds (we expect none) + std::size_t ttisstdcnt; // UTC/local indicators (unused) + std::size_t ttisutcnt; // standard/wall indicators (unused) + + bool Build(const tzhead& tzh); + std::size_t DataLength(std::size_t time_len) const; + }; + + bool GetTransitionType(std::int_fast32_t utc_offset, bool is_dst, + const std::string& abbr, std::uint_least8_t* index); + bool EquivTransitions(std::uint_fast8_t tt1_index, + std::uint_fast8_t tt2_index) const; + bool ExtendTransitions(); + + bool ResetToBuiltinUTC(const seconds& offset); + bool Load(ZoneInfoSource* zip); + + // Helpers for BreakTime() and MakeTime(). + time_zone::absolute_lookup LocalTime(std::int_fast64_t unix_time, + const TransitionType& tt) const; + time_zone::absolute_lookup LocalTime(std::int_fast64_t unix_time, + const Transition& tr) const; + time_zone::civil_lookup TimeLocal(const civil_second& cs, + year_t c4_shift) const; + + std::vector<Transition> transitions_; // ordered by unix_time and civil_sec + std::vector<TransitionType> transition_types_; // distinct transition types + std::uint_fast8_t default_transition_type_; // for before first transition + std::string abbreviations_; // all the NUL-terminated abbreviations + + std::string version_; // the tzdata version if available + std::string future_spec_; // for after the last zic transition + bool extended_; // future_spec_ was used to generate transitions + year_t last_year_; // the final year of the generated transitions + + // We remember the transitions found during the last BreakTime() and + // MakeTime() calls. If the next request is for the same transition we + // will avoid re-searching. + mutable std::atomic<std::size_t> local_time_hint_ = {}; // BreakTime() hint + mutable std::atomic<std::size_t> time_local_hint_ = {}; // MakeTime() hint +}; + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_INFO_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_libc.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_libc.cc new file mode 100644 index 000000000000..887dd097c650 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_libc.cc @@ -0,0 +1,315 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#if defined(_WIN32) || defined(_WIN64) +#define _CRT_SECURE_NO_WARNINGS 1 +#endif + +#include "time_zone_libc.h" + +#include <chrono> +#include <ctime> +#include <limits> +#include <utility> + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +#if defined(_AIX) +extern "C" { +extern long altzone; +} +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +#if defined(_WIN32) || defined(_WIN64) +// Uses the globals: '_timezone', '_dstbias' and '_tzname'. +auto tm_gmtoff(const std::tm& tm) -> decltype(_timezone + _dstbias) { + const bool is_dst = tm.tm_isdst > 0; + return _timezone + (is_dst ? _dstbias : 0); +} +auto tm_zone(const std::tm& tm) -> decltype(_tzname[0]) { + const bool is_dst = tm.tm_isdst > 0; + return _tzname[is_dst]; +} +#elif defined(__sun) || defined(_AIX) +// Uses the globals: 'timezone', 'altzone' and 'tzname'. +auto tm_gmtoff(const std::tm& tm) -> decltype(timezone) { + const bool is_dst = tm.tm_isdst > 0; + return is_dst ? altzone : timezone; +} +auto tm_zone(const std::tm& tm) -> decltype(tzname[0]) { + const bool is_dst = tm.tm_isdst > 0; + return tzname[is_dst]; +} +#elif defined(__native_client__) || defined(__myriad2__) || \ + defined(__EMSCRIPTEN__) +// Uses the globals: 'timezone' and 'tzname'. +auto tm_gmtoff(const std::tm& tm) -> decltype(_timezone + 0) { + const bool is_dst = tm.tm_isdst > 0; + return _timezone + (is_dst ? 60 * 60 : 0); +} +auto tm_zone(const std::tm& tm) -> decltype(tzname[0]) { + const bool is_dst = tm.tm_isdst > 0; + return tzname[is_dst]; +} +#else +// Adapt to different spellings of the struct std::tm extension fields. +#if defined(tm_gmtoff) +auto tm_gmtoff(const std::tm& tm) -> decltype(tm.tm_gmtoff) { + return tm.tm_gmtoff; +} +#elif defined(__tm_gmtoff) +auto tm_gmtoff(const std::tm& tm) -> decltype(tm.__tm_gmtoff) { + return tm.__tm_gmtoff; +} +#else +template <typename T> +auto tm_gmtoff(const T& tm) -> decltype(tm.tm_gmtoff) { + return tm.tm_gmtoff; +} +template <typename T> +auto tm_gmtoff(const T& tm) -> decltype(tm.__tm_gmtoff) { + return tm.__tm_gmtoff; +} +#endif // tm_gmtoff +#if defined(tm_zone) +auto tm_zone(const std::tm& tm) -> decltype(tm.tm_zone) { return tm.tm_zone; } +#elif defined(__tm_zone) +auto tm_zone(const std::tm& tm) -> decltype(tm.__tm_zone) { + return tm.__tm_zone; +} +#else +template <typename T> +auto tm_zone(const T& tm) -> decltype(tm.tm_zone) { + return tm.tm_zone; +} +template <typename T> +auto tm_zone(const T& tm) -> decltype(tm.__tm_zone) { + return tm.__tm_zone; +} +#endif // tm_zone +#endif + +inline std::tm* gm_time(const std::time_t* timep, std::tm* result) { +#if defined(_WIN32) || defined(_WIN64) + return gmtime_s(result, timep) ? nullptr : result; +#else + return gmtime_r(timep, result); +#endif +} + +inline std::tm* local_time(const std::time_t* timep, std::tm* result) { +#if defined(_WIN32) || defined(_WIN64) + return localtime_s(result, timep) ? nullptr : result; +#else + return localtime_r(timep, result); +#endif +} + +// Converts a civil second and "dst" flag into a time_t and UTC offset. +// Returns false if time_t cannot represent the requested civil second. +// Caller must have already checked that cs.year() will fit into a tm_year. +bool make_time(const civil_second& cs, int is_dst, std::time_t* t, int* off) { + std::tm tm; + tm.tm_year = static_cast<int>(cs.year() - year_t{1900}); + tm.tm_mon = cs.month() - 1; + tm.tm_mday = cs.day(); + tm.tm_hour = cs.hour(); + tm.tm_min = cs.minute(); + tm.tm_sec = cs.second(); + tm.tm_isdst = is_dst; + *t = std::mktime(&tm); + if (*t == std::time_t{-1}) { + std::tm tm2; + const std::tm* tmp = local_time(t, &tm2); + if (tmp == nullptr || tmp->tm_year != tm.tm_year || + tmp->tm_mon != tm.tm_mon || tmp->tm_mday != tm.tm_mday || + tmp->tm_hour != tm.tm_hour || tmp->tm_min != tm.tm_min || + tmp->tm_sec != tm.tm_sec) { + // A true error (not just one second before the epoch). + return false; + } + } + *off = static_cast<int>(tm_gmtoff(tm)); + return true; +} + +// Find the least time_t in [lo:hi] where local time matches offset, given: +// (1) lo doesn't match, (2) hi does, and (3) there is only one transition. +std::time_t find_trans(std::time_t lo, std::time_t hi, int offset) { + std::tm tm; + while (lo + 1 != hi) { + const std::time_t mid = lo + (hi - lo) / 2; + std::tm* tmp = local_time(&mid, &tm); + if (tmp != nullptr) { + if (tm_gmtoff(*tmp) == offset) { + hi = mid; + } else { + lo = mid; + } + } else { + // If std::tm cannot hold some result we resort to a linear search, + // ignoring all failed conversions. Slow, but never really happens. + while (++lo != hi) { + tmp = local_time(&lo, &tm); + if (tmp != nullptr) { + if (tm_gmtoff(*tmp) == offset) break; + } + } + return lo; + } + } + return hi; +} + +} // namespace + +TimeZoneLibC::TimeZoneLibC(const std::string& name) + : local_(name == "localtime") {} + +time_zone::absolute_lookup TimeZoneLibC::BreakTime( + const time_point<seconds>& tp) const { + time_zone::absolute_lookup al; + al.offset = 0; + al.is_dst = false; + al.abbr = "-00"; + + const std::int_fast64_t s = ToUnixSeconds(tp); + + // If std::time_t cannot hold the input we saturate the output. + if (s < std::numeric_limits<std::time_t>::min()) { + al.cs = civil_second::min(); + return al; + } + if (s > std::numeric_limits<std::time_t>::max()) { + al.cs = civil_second::max(); + return al; + } + + const std::time_t t = static_cast<std::time_t>(s); + std::tm tm; + std::tm* tmp = local_ ? local_time(&t, &tm) : gm_time(&t, &tm); + + // If std::tm cannot hold the result we saturate the output. + if (tmp == nullptr) { + al.cs = (s < 0) ? civil_second::min() : civil_second::max(); + return al; + } + + const year_t year = tmp->tm_year + year_t{1900}; + al.cs = civil_second(year, tmp->tm_mon + 1, tmp->tm_mday, tmp->tm_hour, + tmp->tm_min, tmp->tm_sec); + al.offset = static_cast<int>(tm_gmtoff(*tmp)); + al.abbr = local_ ? tm_zone(*tmp) : "UTC"; // as expected by cctz + al.is_dst = tmp->tm_isdst > 0; + return al; +} + +time_zone::civil_lookup TimeZoneLibC::MakeTime(const civil_second& cs) const { + if (!local_) { + // If time_point<seconds> cannot hold the result we saturate. + static const civil_second min_tp_cs = + civil_second() + ToUnixSeconds(time_point<seconds>::min()); + static const civil_second max_tp_cs = + civil_second() + ToUnixSeconds(time_point<seconds>::max()); + const time_point<seconds> tp = (cs < min_tp_cs) ? time_point<seconds>::min() + : (cs > max_tp_cs) + ? time_point<seconds>::max() + : FromUnixSeconds(cs - civil_second()); + return {time_zone::civil_lookup::UNIQUE, tp, tp, tp}; + } + + // If tm_year cannot hold the requested year we saturate the result. + if (cs.year() < 0) { + if (cs.year() < std::numeric_limits<int>::min() + year_t{1900}) { + const time_point<seconds> tp = time_point<seconds>::min(); + return {time_zone::civil_lookup::UNIQUE, tp, tp, tp}; + } + } else { + if (cs.year() - year_t{1900} > std::numeric_limits<int>::max()) { + const time_point<seconds> tp = time_point<seconds>::max(); + return {time_zone::civil_lookup::UNIQUE, tp, tp, tp}; + } + } + + // We probe with "is_dst" values of 0 and 1 to try to distinguish unique + // civil seconds from skipped or repeated ones. This is not always possible + // however, as the "dst" flag does not change over some offset transitions. + // We are also subject to the vagaries of mktime() implementations. + std::time_t t0, t1; + int offset0, offset1; + if (make_time(cs, 0, &t0, &offset0) && make_time(cs, 1, &t1, &offset1)) { + if (t0 == t1) { + // The civil time was singular (pre == trans == post). + const time_point<seconds> tp = FromUnixSeconds(t0); + return {time_zone::civil_lookup::UNIQUE, tp, tp, tp}; + } + + if (t0 > t1) { + std::swap(t0, t1); + std::swap(offset0, offset1); + } + const std::time_t tt = find_trans(t0, t1, offset1); + const time_point<seconds> trans = FromUnixSeconds(tt); + + if (offset0 < offset1) { + // The civil time did not exist (pre >= trans > post). + const time_point<seconds> pre = FromUnixSeconds(t1); + const time_point<seconds> post = FromUnixSeconds(t0); + return {time_zone::civil_lookup::SKIPPED, pre, trans, post}; + } + + // The civil time was ambiguous (pre < trans <= post). + const time_point<seconds> pre = FromUnixSeconds(t0); + const time_point<seconds> post = FromUnixSeconds(t1); + return {time_zone::civil_lookup::REPEATED, pre, trans, post}; + } + + // make_time() failed somehow so we saturate the result. + const time_point<seconds> tp = (cs < civil_second()) + ? time_point<seconds>::min() + : time_point<seconds>::max(); + return {time_zone::civil_lookup::UNIQUE, tp, tp, tp}; +} + +bool TimeZoneLibC::NextTransition(const time_point<seconds>&, + time_zone::civil_transition*) const { + return false; +} + +bool TimeZoneLibC::PrevTransition(const time_point<seconds>&, + time_zone::civil_transition*) const { + return false; +} + +std::string TimeZoneLibC::Version() const { + return std::string(); // unknown +} + +std::string TimeZoneLibC::Description() const { + return local_ ? "localtime" : "UTC"; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_libc.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_libc.h new file mode 100644 index 000000000000..1da9039a15ee --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_libc.h @@ -0,0 +1,55 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_LIBC_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_LIBC_H_ + +#include <string> + +#include "absl/base/config.h" +#include "time_zone_if.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// A time zone backed by gmtime_r(3), localtime_r(3), and mktime(3), +// and which therefore only supports UTC and the local time zone. +// TODO: Add support for fixed offsets from UTC. +class TimeZoneLibC : public TimeZoneIf { + public: + explicit TimeZoneLibC(const std::string& name); + + // TimeZoneIf implementations. + time_zone::absolute_lookup BreakTime( + const time_point<seconds>& tp) const override; + time_zone::civil_lookup MakeTime(const civil_second& cs) const override; + bool NextTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const override; + bool PrevTransition(const time_point<seconds>& tp, + time_zone::civil_transition* trans) const override; + std::string Version() const override; + std::string Description() const override; + + private: + const bool local_; // localtime or UTC +}; + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_LIBC_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_lookup.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_lookup.cc new file mode 100644 index 000000000000..efdea64b4eb1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_lookup.cc @@ -0,0 +1,187 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +#if defined(__ANDROID__) +#include <sys/system_properties.h> +#if defined(__ANDROID_API__) && __ANDROID_API__ >= 21 +#include <dlfcn.h> +#endif +#endif + +#if defined(__APPLE__) +#include <CoreFoundation/CFTimeZone.h> + +#include <vector> +#endif + +#include <cstdlib> +#include <cstring> +#include <string> + +#include "time_zone_fixed.h" +#include "time_zone_impl.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +#if defined(__ANDROID__) && defined(__ANDROID_API__) && __ANDROID_API__ >= 21 +namespace { +// Android 'L' removes __system_property_get() from the NDK, however +// it is still a hidden symbol in libc so we use dlsym() to access it. +// See Chromium's base/sys_info_android.cc for a similar example. + +using property_get_func = int (*)(const char*, char*); + +property_get_func LoadSystemPropertyGet() { + int flag = RTLD_LAZY | RTLD_GLOBAL; +#if defined(RTLD_NOLOAD) + flag |= RTLD_NOLOAD; // libc.so should already be resident +#endif + if (void* handle = dlopen("libc.so", flag)) { + void* sym = dlsym(handle, "__system_property_get"); + dlclose(handle); + return reinterpret_cast<property_get_func>(sym); + } + return nullptr; +} + +int __system_property_get(const char* name, char* value) { + static property_get_func system_property_get = LoadSystemPropertyGet(); + return system_property_get ? system_property_get(name, value) : -1; +} + +} // namespace +#endif + +std::string time_zone::name() const { return effective_impl().Name(); } + +time_zone::absolute_lookup time_zone::lookup( + const time_point<seconds>& tp) const { + return effective_impl().BreakTime(tp); +} + +time_zone::civil_lookup time_zone::lookup(const civil_second& cs) const { + return effective_impl().MakeTime(cs); +} + +bool time_zone::next_transition(const time_point<seconds>& tp, + civil_transition* trans) const { + return effective_impl().NextTransition(tp, trans); +} + +bool time_zone::prev_transition(const time_point<seconds>& tp, + civil_transition* trans) const { + return effective_impl().PrevTransition(tp, trans); +} + +std::string time_zone::version() const { return effective_impl().Version(); } + +std::string time_zone::description() const { + return effective_impl().Description(); +} + +const time_zone::Impl& time_zone::effective_impl() const { + if (impl_ == nullptr) { + // Dereferencing an implicit-UTC time_zone is expected to be + // rare, so we don't mind paying a small synchronization cost. + return *time_zone::Impl::UTC().impl_; + } + return *impl_; +} + +bool load_time_zone(const std::string& name, time_zone* tz) { + return time_zone::Impl::LoadTimeZone(name, tz); +} + +time_zone utc_time_zone() { + return time_zone::Impl::UTC(); // avoid name lookup +} + +time_zone fixed_time_zone(const seconds& offset) { + time_zone tz; + load_time_zone(FixedOffsetToName(offset), &tz); + return tz; +} + +time_zone local_time_zone() { + const char* zone = ":localtime"; +#if defined(__ANDROID__) + char sysprop[PROP_VALUE_MAX]; + if (__system_property_get("persist.sys.timezone", sysprop) > 0) { + zone = sysprop; + } +#endif +#if defined(__APPLE__) + std::vector<char> buffer; + CFTimeZoneRef tz_default = CFTimeZoneCopyDefault(); + if (CFStringRef tz_name = CFTimeZoneGetName(tz_default)) { + CFStringEncoding encoding = kCFStringEncodingUTF8; + CFIndex length = CFStringGetLength(tz_name); + buffer.resize(CFStringGetMaximumSizeForEncoding(length, encoding) + 1); + if (CFStringGetCString(tz_name, &buffer[0], buffer.size(), encoding)) { + zone = &buffer[0]; + } + } + CFRelease(tz_default); +#endif + + // Allow ${TZ} to override to default zone. + char* tz_env = nullptr; +#if defined(_MSC_VER) + _dupenv_s(&tz_env, nullptr, "TZ"); +#else + tz_env = std::getenv("TZ"); +#endif + if (tz_env) zone = tz_env; + + // We only support the "[:]<zone-name>" form. + if (*zone == ':') ++zone; + + // Map "localtime" to a system-specific name, but + // allow ${LOCALTIME} to override the default name. + char* localtime_env = nullptr; + if (strcmp(zone, "localtime") == 0) { +#if defined(_MSC_VER) + // System-specific default is just "localtime". + _dupenv_s(&localtime_env, nullptr, "LOCALTIME"); +#else + zone = "/etc/localtime"; // System-specific default. + localtime_env = std::getenv("LOCALTIME"); +#endif + if (localtime_env) zone = localtime_env; + } + + const std::string name = zone; +#if defined(_MSC_VER) + free(localtime_env); + free(tz_env); +#endif + + time_zone tz; + load_time_zone(name, &tz); // Falls back to UTC. + // TODO: Follow the RFC3339 "Unknown Local Offset Convention" and + // arrange for %z to generate "-0000" when we don't know the local + // offset because the load_time_zone() failed and we're using UTC. + return tz; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_lookup_test.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_lookup_test.cc new file mode 100644 index 000000000000..9a1a8d6e400c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_lookup_test.cc @@ -0,0 +1,1442 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <chrono> +#include <cstddef> +#include <cstdlib> +#include <future> +#include <limits> +#include <string> +#include <thread> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace chrono = std::chrono; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +// A list of known time-zone names. +const char* const kTimeZoneNames[] = {"Africa/Abidjan", + "Africa/Accra", + "Africa/Addis_Ababa", + "Africa/Algiers", + "Africa/Asmara", + "Africa/Asmera", + "Africa/Bamako", + "Africa/Bangui", + "Africa/Banjul", + "Africa/Bissau", + "Africa/Blantyre", + "Africa/Brazzaville", + "Africa/Bujumbura", + "Africa/Cairo", + "Africa/Casablanca", + "Africa/Ceuta", + "Africa/Conakry", + "Africa/Dakar", + "Africa/Dar_es_Salaam", + "Africa/Djibouti", + "Africa/Douala", + "Africa/El_Aaiun", + "Africa/Freetown", + "Africa/Gaborone", + "Africa/Harare", + "Africa/Johannesburg", + "Africa/Juba", + "Africa/Kampala", + "Africa/Khartoum", + "Africa/Kigali", + "Africa/Kinshasa", + "Africa/Lagos", + "Africa/Libreville", + "Africa/Lome", + "Africa/Luanda", + "Africa/Lubumbashi", + "Africa/Lusaka", + "Africa/Malabo", + "Africa/Maputo", + "Africa/Maseru", + "Africa/Mbabane", + "Africa/Mogadishu", + "Africa/Monrovia", + "Africa/Nairobi", + "Africa/Ndjamena", + "Africa/Niamey", + "Africa/Nouakchott", + "Africa/Ouagadougou", + "Africa/Porto-Novo", + "Africa/Sao_Tome", + "Africa/Timbuktu", + "Africa/Tripoli", + "Africa/Tunis", + "Africa/Windhoek", + "America/Adak", + "America/Anchorage", + "America/Anguilla", + "America/Antigua", + "America/Araguaina", + "America/Argentina/Buenos_Aires", + "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", + "America/Argentina/Cordoba", + "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", + "America/Argentina/Mendoza", + "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", + "America/Argentina/San_Juan", + "America/Argentina/San_Luis", + "America/Argentina/Tucuman", + "America/Argentina/Ushuaia", + "America/Aruba", + "America/Asuncion", + "America/Atikokan", + "America/Atka", + "America/Bahia", + "America/Bahia_Banderas", + "America/Barbados", + "America/Belem", + "America/Belize", + "America/Blanc-Sablon", + "America/Boa_Vista", + "America/Bogota", + "America/Boise", + "America/Buenos_Aires", + "America/Cambridge_Bay", + "America/Campo_Grande", + "America/Cancun", + "America/Caracas", + "America/Catamarca", + "America/Cayenne", + "America/Cayman", + "America/Chicago", + "America/Chihuahua", + "America/Coral_Harbour", + "America/Cordoba", + "America/Costa_Rica", + "America/Creston", + "America/Cuiaba", + "America/Curacao", + "America/Danmarkshavn", + "America/Dawson", + "America/Dawson_Creek", + "America/Denver", + "America/Detroit", + "America/Dominica", + "America/Edmonton", + "America/Eirunepe", + "America/El_Salvador", + "America/Ensenada", + "America/Fort_Nelson", + "America/Fort_Wayne", + "America/Fortaleza", + "America/Glace_Bay", + "America/Godthab", + "America/Goose_Bay", + "America/Grand_Turk", + "America/Grenada", + "America/Guadeloupe", + "America/Guatemala", + "America/Guayaquil", + "America/Guyana", + "America/Halifax", + "America/Havana", + "America/Hermosillo", + "America/Indiana/Indianapolis", + "America/Indiana/Knox", + "America/Indiana/Marengo", + "America/Indiana/Petersburg", + "America/Indiana/Tell_City", + "America/Indiana/Vevay", + "America/Indiana/Vincennes", + "America/Indiana/Winamac", + "America/Indianapolis", + "America/Inuvik", + "America/Iqaluit", + "America/Jamaica", + "America/Jujuy", + "America/Juneau", + "America/Kentucky/Louisville", + "America/Kentucky/Monticello", + "America/Knox_IN", + "America/Kralendijk", + "America/La_Paz", + "America/Lima", + "America/Los_Angeles", + "America/Louisville", + "America/Lower_Princes", + "America/Maceio", + "America/Managua", + "America/Manaus", + "America/Marigot", + "America/Martinique", + "America/Matamoros", + "America/Mazatlan", + "America/Mendoza", + "America/Menominee", + "America/Merida", + "America/Metlakatla", + "America/Mexico_City", + "America/Miquelon", + "America/Moncton", + "America/Monterrey", + "America/Montevideo", + "America/Montreal", + "America/Montserrat", + "America/Nassau", + "America/New_York", + "America/Nipigon", + "America/Nome", + "America/Noronha", + "America/North_Dakota/Beulah", + "America/North_Dakota/Center", + "America/North_Dakota/New_Salem", + "America/Nuuk", + "America/Ojinaga", + "America/Panama", + "America/Pangnirtung", + "America/Paramaribo", + "America/Phoenix", + "America/Port-au-Prince", + "America/Port_of_Spain", + "America/Porto_Acre", + "America/Porto_Velho", + "America/Puerto_Rico", + "America/Punta_Arenas", + "America/Rainy_River", + "America/Rankin_Inlet", + "America/Recife", + "America/Regina", + "America/Resolute", + "America/Rio_Branco", + "America/Rosario", + "America/Santa_Isabel", + "America/Santarem", + "America/Santiago", + "America/Santo_Domingo", + "America/Sao_Paulo", + "America/Scoresbysund", + "America/Shiprock", + "America/Sitka", + "America/St_Barthelemy", + "America/St_Johns", + "America/St_Kitts", + "America/St_Lucia", + "America/St_Thomas", + "America/St_Vincent", + "America/Swift_Current", + "America/Tegucigalpa", + "America/Thule", + "America/Thunder_Bay", + "America/Tijuana", + "America/Toronto", + "America/Tortola", + "America/Vancouver", + "America/Virgin", + "America/Whitehorse", + "America/Winnipeg", + "America/Yakutat", + "America/Yellowknife", + "Antarctica/Casey", + "Antarctica/Davis", + "Antarctica/DumontDUrville", + "Antarctica/Macquarie", + "Antarctica/Mawson", + "Antarctica/McMurdo", + "Antarctica/Palmer", + "Antarctica/Rothera", + "Antarctica/South_Pole", + "Antarctica/Syowa", + "Antarctica/Troll", + "Antarctica/Vostok", + "Arctic/Longyearbyen", + "Asia/Aden", + "Asia/Almaty", + "Asia/Amman", + "Asia/Anadyr", + "Asia/Aqtau", + "Asia/Aqtobe", + "Asia/Ashgabat", + "Asia/Ashkhabad", + "Asia/Atyrau", + "Asia/Baghdad", + "Asia/Bahrain", + "Asia/Baku", + "Asia/Bangkok", + "Asia/Barnaul", + "Asia/Beirut", + "Asia/Bishkek", + "Asia/Brunei", + "Asia/Calcutta", + "Asia/Chita", + "Asia/Choibalsan", + "Asia/Chongqing", + "Asia/Chungking", + "Asia/Colombo", + "Asia/Dacca", + "Asia/Damascus", + "Asia/Dhaka", + "Asia/Dili", + "Asia/Dubai", + "Asia/Dushanbe", + "Asia/Famagusta", + "Asia/Gaza", + "Asia/Harbin", + "Asia/Hebron", + "Asia/Ho_Chi_Minh", + "Asia/Hong_Kong", + "Asia/Hovd", + "Asia/Irkutsk", + "Asia/Istanbul", + "Asia/Jakarta", + "Asia/Jayapura", + "Asia/Jerusalem", + "Asia/Kabul", + "Asia/Kamchatka", + "Asia/Karachi", + "Asia/Kashgar", + "Asia/Kathmandu", + "Asia/Katmandu", + "Asia/Khandyga", + "Asia/Kolkata", + "Asia/Krasnoyarsk", + "Asia/Kuala_Lumpur", + "Asia/Kuching", + "Asia/Kuwait", + "Asia/Macao", + "Asia/Macau", + "Asia/Magadan", + "Asia/Makassar", + "Asia/Manila", + "Asia/Muscat", + "Asia/Nicosia", + "Asia/Novokuznetsk", + "Asia/Novosibirsk", + "Asia/Omsk", + "Asia/Oral", + "Asia/Phnom_Penh", + "Asia/Pontianak", + "Asia/Pyongyang", + "Asia/Qatar", + "Asia/Qostanay", + "Asia/Qyzylorda", + "Asia/Rangoon", + "Asia/Riyadh", + "Asia/Saigon", + "Asia/Sakhalin", + "Asia/Samarkand", + "Asia/Seoul", + "Asia/Shanghai", + "Asia/Singapore", + "Asia/Srednekolymsk", + "Asia/Taipei", + "Asia/Tashkent", + "Asia/Tbilisi", + "Asia/Tehran", + "Asia/Tel_Aviv", + "Asia/Thimbu", + "Asia/Thimphu", + "Asia/Tokyo", + "Asia/Tomsk", + "Asia/Ujung_Pandang", + "Asia/Ulaanbaatar", + "Asia/Ulan_Bator", + "Asia/Urumqi", + "Asia/Ust-Nera", + "Asia/Vientiane", + "Asia/Vladivostok", + "Asia/Yakutsk", + "Asia/Yangon", + "Asia/Yekaterinburg", + "Asia/Yerevan", + "Atlantic/Azores", + "Atlantic/Bermuda", + "Atlantic/Canary", + "Atlantic/Cape_Verde", + "Atlantic/Faeroe", + "Atlantic/Faroe", + "Atlantic/Jan_Mayen", + "Atlantic/Madeira", + "Atlantic/Reykjavik", + "Atlantic/South_Georgia", + "Atlantic/St_Helena", + "Atlantic/Stanley", + "Australia/ACT", + "Australia/Adelaide", + "Australia/Brisbane", + "Australia/Broken_Hill", + "Australia/Canberra", + "Australia/Currie", + "Australia/Darwin", + "Australia/Eucla", + "Australia/Hobart", + "Australia/LHI", + "Australia/Lindeman", + "Australia/Lord_Howe", + "Australia/Melbourne", + "Australia/NSW", + "Australia/North", + "Australia/Perth", + "Australia/Queensland", + "Australia/South", + "Australia/Sydney", + "Australia/Tasmania", + "Australia/Victoria", + "Australia/West", + "Australia/Yancowinna", + "Brazil/Acre", + "Brazil/DeNoronha", + "Brazil/East", + "Brazil/West", + "CET", + "CST6CDT", + "Canada/Atlantic", + "Canada/Central", + "Canada/Eastern", + "Canada/Mountain", + "Canada/Newfoundland", + "Canada/Pacific", + "Canada/Saskatchewan", + "Canada/Yukon", + "Chile/Continental", + "Chile/EasterIsland", + "Cuba", + "EET", + "EST", + "EST5EDT", + "Egypt", + "Eire", + "Etc/GMT", + "Etc/GMT+0", + "Etc/GMT+1", + "Etc/GMT+10", + "Etc/GMT+11", + "Etc/GMT+12", + "Etc/GMT+2", + "Etc/GMT+3", + "Etc/GMT+4", + "Etc/GMT+5", + "Etc/GMT+6", + "Etc/GMT+7", + "Etc/GMT+8", + "Etc/GMT+9", + "Etc/GMT-0", + "Etc/GMT-1", + "Etc/GMT-10", + "Etc/GMT-11", + "Etc/GMT-12", + "Etc/GMT-13", + "Etc/GMT-14", + "Etc/GMT-2", + "Etc/GMT-3", + "Etc/GMT-4", + "Etc/GMT-5", + "Etc/GMT-6", + "Etc/GMT-7", + "Etc/GMT-8", + "Etc/GMT-9", + "Etc/GMT0", + "Etc/Greenwich", + "Etc/UCT", + "Etc/UTC", + "Etc/Universal", + "Etc/Zulu", + "Europe/Amsterdam", + "Europe/Andorra", + "Europe/Astrakhan", + "Europe/Athens", + "Europe/Belfast", + "Europe/Belgrade", + "Europe/Berlin", + "Europe/Bratislava", + "Europe/Brussels", + "Europe/Bucharest", + "Europe/Budapest", + "Europe/Busingen", + "Europe/Chisinau", + "Europe/Copenhagen", + "Europe/Dublin", + "Europe/Gibraltar", + "Europe/Guernsey", + "Europe/Helsinki", + "Europe/Isle_of_Man", + "Europe/Istanbul", + "Europe/Jersey", + "Europe/Kaliningrad", + "Europe/Kiev", + "Europe/Kirov", + "Europe/Lisbon", + "Europe/Ljubljana", + "Europe/London", + "Europe/Luxembourg", + "Europe/Madrid", + "Europe/Malta", + "Europe/Mariehamn", + "Europe/Minsk", + "Europe/Monaco", + "Europe/Moscow", + "Europe/Nicosia", + "Europe/Oslo", + "Europe/Paris", + "Europe/Podgorica", + "Europe/Prague", + "Europe/Riga", + "Europe/Rome", + "Europe/Samara", + "Europe/San_Marino", + "Europe/Sarajevo", + "Europe/Saratov", + "Europe/Simferopol", + "Europe/Skopje", + "Europe/Sofia", + "Europe/Stockholm", + "Europe/Tallinn", + "Europe/Tirane", + "Europe/Tiraspol", + "Europe/Ulyanovsk", + "Europe/Uzhgorod", + "Europe/Vaduz", + "Europe/Vatican", + "Europe/Vienna", + "Europe/Vilnius", + "Europe/Volgograd", + "Europe/Warsaw", + "Europe/Zagreb", + "Europe/Zaporozhye", + "Europe/Zurich", + "GB", + "GB-Eire", + "GMT", + "GMT+0", + "GMT-0", + "GMT0", + "Greenwich", + "HST", + "Hongkong", + "Iceland", + "Indian/Antananarivo", + "Indian/Chagos", + "Indian/Christmas", + "Indian/Cocos", + "Indian/Comoro", + "Indian/Kerguelen", + "Indian/Mahe", + "Indian/Maldives", + "Indian/Mauritius", + "Indian/Mayotte", + "Indian/Reunion", + "Iran", + "Israel", + "Jamaica", + "Japan", + "Kwajalein", + "Libya", + "MET", + "MST", + "MST7MDT", + "Mexico/BajaNorte", + "Mexico/BajaSur", + "Mexico/General", + "NZ", + "NZ-CHAT", + "Navajo", + "PRC", + "PST8PDT", + "Pacific/Apia", + "Pacific/Auckland", + "Pacific/Bougainville", + "Pacific/Chatham", + "Pacific/Chuuk", + "Pacific/Easter", + "Pacific/Efate", + "Pacific/Enderbury", + "Pacific/Fakaofo", + "Pacific/Fiji", + "Pacific/Funafuti", + "Pacific/Galapagos", + "Pacific/Gambier", + "Pacific/Guadalcanal", + "Pacific/Guam", + "Pacific/Honolulu", + "Pacific/Johnston", + "Pacific/Kiritimati", + "Pacific/Kosrae", + "Pacific/Kwajalein", + "Pacific/Majuro", + "Pacific/Marquesas", + "Pacific/Midway", + "Pacific/Nauru", + "Pacific/Niue", + "Pacific/Norfolk", + "Pacific/Noumea", + "Pacific/Pago_Pago", + "Pacific/Palau", + "Pacific/Pitcairn", + "Pacific/Pohnpei", + "Pacific/Ponape", + "Pacific/Port_Moresby", + "Pacific/Rarotonga", + "Pacific/Saipan", + "Pacific/Samoa", + "Pacific/Tahiti", + "Pacific/Tarawa", + "Pacific/Tongatapu", + "Pacific/Truk", + "Pacific/Wake", + "Pacific/Wallis", + "Pacific/Yap", + "Poland", + "Portugal", + "ROC", + "ROK", + "Singapore", + "Turkey", + "UCT", + "US/Alaska", + "US/Aleutian", + "US/Arizona", + "US/Central", + "US/East-Indiana", + "US/Eastern", + "US/Hawaii", + "US/Indiana-Starke", + "US/Michigan", + "US/Mountain", + "US/Pacific", + "US/Samoa", + "UTC", + "Universal", + "W-SU", + "WET", + "Zulu", + nullptr}; + +// Helper to return a loaded time zone by value (UTC on error). +time_zone LoadZone(const std::string& name) { + time_zone tz; + load_time_zone(name, &tz); + return tz; +} + +// This helper is a macro so that failed expectations show up with the +// correct line numbers. +#define ExpectTime(tp, tz, y, m, d, hh, mm, ss, off, isdst, zone) \ + do { \ + time_zone::absolute_lookup al = tz.lookup(tp); \ + EXPECT_EQ(y, al.cs.year()); \ + EXPECT_EQ(m, al.cs.month()); \ + EXPECT_EQ(d, al.cs.day()); \ + EXPECT_EQ(hh, al.cs.hour()); \ + EXPECT_EQ(mm, al.cs.minute()); \ + EXPECT_EQ(ss, al.cs.second()); \ + EXPECT_EQ(off, al.offset); \ + EXPECT_TRUE(isdst == al.is_dst); \ + /* EXPECT_STREQ(zone, al.abbr); */ \ + } while (0) + +// These tests sometimes run on platforms that have zoneinfo data so old +// that the transition we are attempting to check does not exist, most +// notably Android emulators. Fortunately, AndroidZoneInfoSource supports +// time_zone::version() so, in cases where we've learned that it matters, +// we can make the check conditionally. +int VersionCmp(time_zone tz, const std::string& target) { + std::string version = tz.version(); + if (version.empty() && !target.empty()) return 1; // unknown > known + return version.compare(target); +} + +} // namespace + +#if !defined(__EMSCRIPTEN__) +TEST(TimeZones, LoadZonesConcurrently) { + std::promise<void> ready_promise; + std::shared_future<void> ready_future(ready_promise.get_future()); + auto load_zones = [ready_future](std::promise<void>* started, + std::set<std::string>* failures) { + started->set_value(); + ready_future.wait(); + for (const char* const* np = kTimeZoneNames; *np != nullptr; ++np) { + std::string zone = *np; + time_zone tz; + if (load_time_zone(zone, &tz)) { + EXPECT_EQ(zone, tz.name()); + } else { + failures->insert(zone); + } + } + }; + + const std::size_t n_threads = 128; + std::vector<std::thread> threads; + std::vector<std::set<std::string>> thread_failures(n_threads); + for (std::size_t i = 0; i != n_threads; ++i) { + std::promise<void> started; + threads.emplace_back(load_zones, &started, &thread_failures[i]); + started.get_future().wait(); + } + ready_promise.set_value(); + for (auto& thread : threads) { + thread.join(); + } + + // Allow a small number of failures to account for skew between + // the contents of kTimeZoneNames and the zoneinfo data source. +#if defined(__ANDROID__) + // Cater to the possibility of using an even older zoneinfo data + // source when running on Android, where it is difficult to override + // the bionic tzdata provided by the test environment. + const std::size_t max_failures = 20; +#else + const std::size_t max_failures = 3; +#endif + std::set<std::string> failures; + for (const auto& thread_failure : thread_failures) { + failures.insert(thread_failure.begin(), thread_failure.end()); + } + EXPECT_LE(failures.size(), max_failures) << testing::PrintToString(failures); +} +#endif + +TEST(TimeZone, NamedTimeZones) { + const time_zone utc = utc_time_zone(); + EXPECT_EQ("UTC", utc.name()); + const time_zone nyc = LoadZone("America/New_York"); + EXPECT_EQ("America/New_York", nyc.name()); + const time_zone syd = LoadZone("Australia/Sydney"); + EXPECT_EQ("Australia/Sydney", syd.name()); + const time_zone fixed0 = + fixed_time_zone(absl::time_internal::cctz::seconds::zero()); + EXPECT_EQ("UTC", fixed0.name()); + const time_zone fixed_pos = fixed_time_zone( + chrono::hours(3) + chrono::minutes(25) + chrono::seconds(45)); + EXPECT_EQ("Fixed/UTC+03:25:45", fixed_pos.name()); + const time_zone fixed_neg = fixed_time_zone( + -(chrono::hours(12) + chrono::minutes(34) + chrono::seconds(56))); + EXPECT_EQ("Fixed/UTC-12:34:56", fixed_neg.name()); +} + +TEST(TimeZone, Failures) { + time_zone tz; + EXPECT_FALSE(load_time_zone(":America/Los_Angeles", &tz)); + + tz = LoadZone("America/Los_Angeles"); + EXPECT_FALSE(load_time_zone("Invalid/TimeZone", &tz)); + EXPECT_EQ(chrono::system_clock::from_time_t(0), + convert(civil_second(1970, 1, 1, 0, 0, 0), tz)); // UTC + + // Ensures that the load still fails on a subsequent attempt. + tz = LoadZone("America/Los_Angeles"); + EXPECT_FALSE(load_time_zone("Invalid/TimeZone", &tz)); + EXPECT_EQ(chrono::system_clock::from_time_t(0), + convert(civil_second(1970, 1, 1, 0, 0, 0), tz)); // UTC + + // Loading an empty string timezone should fail. + tz = LoadZone("America/Los_Angeles"); + EXPECT_FALSE(load_time_zone("", &tz)); + EXPECT_EQ(chrono::system_clock::from_time_t(0), + convert(civil_second(1970, 1, 1, 0, 0, 0), tz)); // UTC +} + +TEST(TimeZone, Equality) { + const time_zone a; + const time_zone b; + EXPECT_EQ(a, b); + EXPECT_EQ(a.name(), b.name()); + + const time_zone implicit_utc; + const time_zone explicit_utc = utc_time_zone(); + EXPECT_EQ(implicit_utc, explicit_utc); + EXPECT_EQ(implicit_utc.name(), explicit_utc.name()); + + const time_zone fixed_zero = + fixed_time_zone(absl::time_internal::cctz::seconds::zero()); + EXPECT_EQ(fixed_zero, LoadZone(fixed_zero.name())); + EXPECT_EQ(fixed_zero, explicit_utc); + + const time_zone fixed_utc = LoadZone("Fixed/UTC+00:00:00"); + EXPECT_EQ(fixed_utc, LoadZone(fixed_utc.name())); + EXPECT_EQ(fixed_utc, explicit_utc); + + const time_zone fixed_pos = fixed_time_zone( + chrono::hours(3) + chrono::minutes(25) + chrono::seconds(45)); + EXPECT_EQ(fixed_pos, LoadZone(fixed_pos.name())); + EXPECT_NE(fixed_pos, explicit_utc); + const time_zone fixed_neg = fixed_time_zone( + -(chrono::hours(12) + chrono::minutes(34) + chrono::seconds(56))); + EXPECT_EQ(fixed_neg, LoadZone(fixed_neg.name())); + EXPECT_NE(fixed_neg, explicit_utc); + + const time_zone fixed_lim = fixed_time_zone(chrono::hours(24)); + EXPECT_EQ(fixed_lim, LoadZone(fixed_lim.name())); + EXPECT_NE(fixed_lim, explicit_utc); + const time_zone fixed_ovfl = + fixed_time_zone(chrono::hours(24) + chrono::seconds(1)); + EXPECT_EQ(fixed_ovfl, LoadZone(fixed_ovfl.name())); + EXPECT_EQ(fixed_ovfl, explicit_utc); + + EXPECT_EQ(fixed_time_zone(chrono::seconds(1)), + fixed_time_zone(chrono::seconds(1))); + + const time_zone local = local_time_zone(); + EXPECT_EQ(local, LoadZone(local.name())); + + time_zone la = LoadZone("America/Los_Angeles"); + time_zone nyc = LoadZone("America/New_York"); + EXPECT_NE(la, nyc); +} + +TEST(StdChronoTimePoint, TimeTAlignment) { + // Ensures that the Unix epoch and the system clock epoch are an integral + // number of seconds apart. This simplifies conversions to/from time_t. + auto diff = + chrono::system_clock::time_point() - chrono::system_clock::from_time_t(0); + EXPECT_EQ(chrono::system_clock::time_point::duration::zero(), + diff % chrono::seconds(1)); +} + +TEST(BreakTime, TimePointResolution) { + const time_zone utc = utc_time_zone(); + const auto t0 = chrono::system_clock::from_time_t(0); + + ExpectTime(chrono::time_point_cast<chrono::nanoseconds>(t0), utc, 1970, 1, 1, + 0, 0, 0, 0, false, "UTC"); + ExpectTime(chrono::time_point_cast<chrono::microseconds>(t0), utc, 1970, 1, 1, + 0, 0, 0, 0, false, "UTC"); + ExpectTime(chrono::time_point_cast<chrono::milliseconds>(t0), utc, 1970, 1, 1, + 0, 0, 0, 0, false, "UTC"); + ExpectTime(chrono::time_point_cast<chrono::seconds>(t0), utc, 1970, 1, 1, 0, + 0, 0, 0, false, "UTC"); + ExpectTime(chrono::time_point_cast<absl::time_internal::cctz::seconds>(t0), + utc, 1970, 1, 1, 0, 0, 0, 0, false, "UTC"); + ExpectTime(chrono::time_point_cast<chrono::minutes>(t0), utc, 1970, 1, 1, 0, + 0, 0, 0, false, "UTC"); + ExpectTime(chrono::time_point_cast<chrono::hours>(t0), utc, 1970, 1, 1, 0, 0, + 0, 0, false, "UTC"); +} + +TEST(BreakTime, LocalTimeInUTC) { + const time_zone tz = utc_time_zone(); + const auto tp = chrono::system_clock::from_time_t(0); + ExpectTime(tp, tz, 1970, 1, 1, 0, 0, 0, 0, false, "UTC"); + EXPECT_EQ(weekday::thursday, get_weekday(convert(tp, tz))); +} + +TEST(BreakTime, LocalTimeInUTCUnaligned) { + const time_zone tz = utc_time_zone(); + const auto tp = + chrono::system_clock::from_time_t(0) - chrono::milliseconds(500); + ExpectTime(tp, tz, 1969, 12, 31, 23, 59, 59, 0, false, "UTC"); + EXPECT_EQ(weekday::wednesday, get_weekday(convert(tp, tz))); +} + +TEST(BreakTime, LocalTimePosix) { + // See IEEE Std 1003.1-1988 B.2.3 General Terms, Epoch. + const time_zone tz = utc_time_zone(); + const auto tp = chrono::system_clock::from_time_t(536457599); + ExpectTime(tp, tz, 1986, 12, 31, 23, 59, 59, 0, false, "UTC"); + EXPECT_EQ(weekday::wednesday, get_weekday(convert(tp, tz))); +} + +TEST(TimeZoneImpl, LocalTimeInFixed) { + const absl::time_internal::cctz::seconds offset = + -(chrono::hours(8) + chrono::minutes(33) + chrono::seconds(47)); + const time_zone tz = fixed_time_zone(offset); + const auto tp = chrono::system_clock::from_time_t(0); + ExpectTime(tp, tz, 1969, 12, 31, 15, 26, 13, offset.count(), false, + "-083347"); + EXPECT_EQ(weekday::wednesday, get_weekday(convert(tp, tz))); +} + +TEST(BreakTime, LocalTimeInNewYork) { + const time_zone tz = LoadZone("America/New_York"); + const auto tp = chrono::system_clock::from_time_t(45); + ExpectTime(tp, tz, 1969, 12, 31, 19, 0, 45, -5 * 60 * 60, false, "EST"); + EXPECT_EQ(weekday::wednesday, get_weekday(convert(tp, tz))); +} + +TEST(BreakTime, LocalTimeInMTV) { + const time_zone tz = LoadZone("America/Los_Angeles"); + const auto tp = chrono::system_clock::from_time_t(1380855729); + ExpectTime(tp, tz, 2013, 10, 3, 20, 2, 9, -7 * 60 * 60, true, "PDT"); + EXPECT_EQ(weekday::thursday, get_weekday(convert(tp, tz))); +} + +TEST(BreakTime, LocalTimeInSydney) { + const time_zone tz = LoadZone("Australia/Sydney"); + const auto tp = chrono::system_clock::from_time_t(90); + ExpectTime(tp, tz, 1970, 1, 1, 10, 1, 30, 10 * 60 * 60, false, "AEST"); + EXPECT_EQ(weekday::thursday, get_weekday(convert(tp, tz))); +} + +TEST(MakeTime, TimePointResolution) { + const time_zone utc = utc_time_zone(); + const time_point<chrono::nanoseconds> tp_ns = + convert(civil_second(2015, 1, 2, 3, 4, 5), utc); + EXPECT_EQ("04:05", format("%M:%E*S", tp_ns, utc)); + const time_point<chrono::microseconds> tp_us = + convert(civil_second(2015, 1, 2, 3, 4, 5), utc); + EXPECT_EQ("04:05", format("%M:%E*S", tp_us, utc)); + const time_point<chrono::milliseconds> tp_ms = + convert(civil_second(2015, 1, 2, 3, 4, 5), utc); + EXPECT_EQ("04:05", format("%M:%E*S", tp_ms, utc)); + const time_point<chrono::seconds> tp_s = + convert(civil_second(2015, 1, 2, 3, 4, 5), utc); + EXPECT_EQ("04:05", format("%M:%E*S", tp_s, utc)); + const time_point<absl::time_internal::cctz::seconds> tp_s64 = + convert(civil_second(2015, 1, 2, 3, 4, 5), utc); + EXPECT_EQ("04:05", format("%M:%E*S", tp_s64, utc)); + + // These next two require chrono::time_point_cast because the conversion + // from a resolution of seconds (the return value of convert()) to a + // coarser resolution requires an explicit cast. + const time_point<chrono::minutes> tp_m = + chrono::time_point_cast<chrono::minutes>( + convert(civil_second(2015, 1, 2, 3, 4, 5), utc)); + EXPECT_EQ("04:00", format("%M:%E*S", tp_m, utc)); + const time_point<chrono::hours> tp_h = chrono::time_point_cast<chrono::hours>( + convert(civil_second(2015, 1, 2, 3, 4, 5), utc)); + EXPECT_EQ("00:00", format("%M:%E*S", tp_h, utc)); +} + +TEST(MakeTime, Normalization) { + const time_zone tz = LoadZone("America/New_York"); + const auto tp = convert(civil_second(2009, 2, 13, 18, 31, 30), tz); + EXPECT_EQ(chrono::system_clock::from_time_t(1234567890), tp); + + // Now requests for the same time_point but with out-of-range fields. + EXPECT_EQ(tp, convert(civil_second(2008, 14, 13, 18, 31, 30), tz)); // month + EXPECT_EQ(tp, convert(civil_second(2009, 1, 44, 18, 31, 30), tz)); // day + EXPECT_EQ(tp, convert(civil_second(2009, 2, 12, 42, 31, 30), tz)); // hour + EXPECT_EQ(tp, convert(civil_second(2009, 2, 13, 17, 91, 30), tz)); // minute + EXPECT_EQ(tp, convert(civil_second(2009, 2, 13, 18, 30, 90), tz)); // second +} + +// NOTE: Run this with -ftrapv to detect overflow problems. +TEST(MakeTime, SysSecondsLimits) { + const char RFC3339[] = "%Y-%m-%d%ET%H:%M:%S%Ez"; + const time_zone utc = utc_time_zone(); + const time_zone east = fixed_time_zone(chrono::hours(14)); + const time_zone west = fixed_time_zone(-chrono::hours(14)); + time_point<absl::time_internal::cctz::seconds> tp; + + // Approach the maximal time_point<cctz::seconds> value from below. + tp = convert(civil_second(292277026596, 12, 4, 15, 30, 6), utc); + EXPECT_EQ("292277026596-12-04T15:30:06+00:00", format(RFC3339, tp, utc)); + tp = convert(civil_second(292277026596, 12, 4, 15, 30, 7), utc); + EXPECT_EQ("292277026596-12-04T15:30:07+00:00", format(RFC3339, tp, utc)); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + tp = convert(civil_second(292277026596, 12, 4, 15, 30, 8), utc); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + tp = convert(civil_second::max(), utc); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + + // Checks that we can also get the maximal value for a far-east zone. + tp = convert(civil_second(292277026596, 12, 5, 5, 30, 7), east); + EXPECT_EQ("292277026596-12-05T05:30:07+14:00", format(RFC3339, tp, east)); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + tp = convert(civil_second(292277026596, 12, 5, 5, 30, 8), east); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + tp = convert(civil_second::max(), east); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + + // Checks that we can also get the maximal value for a far-west zone. + tp = convert(civil_second(292277026596, 12, 4, 1, 30, 7), west); + EXPECT_EQ("292277026596-12-04T01:30:07-14:00", format(RFC3339, tp, west)); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + tp = convert(civil_second(292277026596, 12, 4, 7, 30, 8), west); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + tp = convert(civil_second::max(), west); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::max(), tp); + + // Approach the minimal time_point<cctz::seconds> value from above. + tp = convert(civil_second(-292277022657, 1, 27, 8, 29, 53), utc); + EXPECT_EQ("-292277022657-01-27T08:29:53+00:00", format(RFC3339, tp, utc)); + tp = convert(civil_second(-292277022657, 1, 27, 8, 29, 52), utc); + EXPECT_EQ("-292277022657-01-27T08:29:52+00:00", format(RFC3339, tp, utc)); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + tp = convert(civil_second(-292277022657, 1, 27, 8, 29, 51), utc); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + tp = convert(civil_second::min(), utc); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + + // Checks that we can also get the minimal value for a far-east zone. + tp = convert(civil_second(-292277022657, 1, 27, 22, 29, 52), east); + EXPECT_EQ("-292277022657-01-27T22:29:52+14:00", format(RFC3339, tp, east)); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + tp = convert(civil_second(-292277022657, 1, 27, 22, 29, 51), east); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + tp = convert(civil_second::min(), east); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + + // Checks that we can also get the minimal value for a far-west zone. + tp = convert(civil_second(-292277022657, 1, 26, 18, 29, 52), west); + EXPECT_EQ("-292277022657-01-26T18:29:52-14:00", format(RFC3339, tp, west)); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + tp = convert(civil_second(-292277022657, 1, 26, 18, 29, 51), west); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + tp = convert(civil_second::min(), west); + EXPECT_EQ(time_point<absl::time_internal::cctz::seconds>::min(), tp); + + // Some similar checks for the "libc" time-zone implementation. + if (sizeof(std::time_t) >= 8) { + // Checks that "tm_year + 1900", as used by the "libc" implementation, + // can produce year values beyond the range on an int without overflow. +#if defined(_WIN32) || defined(_WIN64) + // localtime_s() and gmtime_s() don't believe in years outside [1970:3000]. +#else + const time_zone cut = LoadZone("libc:UTC"); + const year_t max_tm_year = year_t{std::numeric_limits<int>::max()} + 1900; + tp = convert(civil_second(max_tm_year, 12, 31, 23, 59, 59), cut); +#if defined(__FreeBSD__) || defined(__OpenBSD__) + // The BSD gmtime_r() fails on extreme positive tm_year values. +#else + EXPECT_EQ("2147485547-12-31T23:59:59+00:00", format(RFC3339, tp, cut)); +#endif + const year_t min_tm_year = year_t{std::numeric_limits<int>::min()} + 1900; + tp = convert(civil_second(min_tm_year, 1, 1, 0, 0, 0), cut); + EXPECT_EQ("-2147481748-01-01T00:00:00+00:00", format(RFC3339, tp, cut)); +#endif + } +} + +TEST(MakeTime, LocalTimeLibC) { + // Checks that cctz and libc agree on transition points in [1970:2037]. + // + // We limit this test case to environments where: + // 1) we know how to change the time zone used by localtime()/mktime(), + // 2) cctz and localtime()/mktime() will use similar-enough tzdata, and + // 3) we have some idea about how mktime() behaves during transitions. +#if defined(__linux__) && !defined(__ANDROID__) + const char* const ep = getenv("TZ"); + std::string tz_name = (ep != nullptr) ? ep : ""; + for (const char* const* np = kTimeZoneNames; *np != nullptr; ++np) { + ASSERT_EQ(0, setenv("TZ", *np, 1)); // change what "localtime" means + const auto zi = local_time_zone(); + const auto lc = LoadZone("libc:localtime"); + time_zone::civil_transition transition; + for (auto tp = zi.lookup(civil_second()).trans; + zi.next_transition(tp, &transition); + tp = zi.lookup(transition.to).trans) { + const auto fcl = zi.lookup(transition.from); + const auto tcl = zi.lookup(transition.to); + civil_second cs; // compare cs in zi and lc + if (fcl.kind == time_zone::civil_lookup::UNIQUE) { + if (tcl.kind == time_zone::civil_lookup::UNIQUE) { + // Both unique; must be an is_dst or abbr change. + ASSERT_EQ(transition.from, transition.to); + const auto trans = fcl.trans; + const auto tal = zi.lookup(trans); + const auto tprev = trans - absl::time_internal::cctz::seconds(1); + const auto pal = zi.lookup(tprev); + if (pal.is_dst == tal.is_dst) { + ASSERT_STRNE(pal.abbr, tal.abbr); + } + continue; + } + ASSERT_EQ(time_zone::civil_lookup::REPEATED, tcl.kind); + cs = transition.to; + } else { + ASSERT_EQ(time_zone::civil_lookup::UNIQUE, tcl.kind); + ASSERT_EQ(time_zone::civil_lookup::SKIPPED, fcl.kind); + cs = transition.from; + } + if (cs.year() > 2037) break; // limit test time (and to 32-bit time_t) + const auto cl_zi = zi.lookup(cs); + if (zi.lookup(cl_zi.pre).is_dst == zi.lookup(cl_zi.post).is_dst) { + // The "libc" implementation cannot correctly classify transitions + // that don't change the "tm_isdst" flag. In Europe/Volgograd, for + // example, there is a SKIPPED transition from +03 to +04 with dst=F + // on both sides ... + // 1540681199 = 2018-10-28 01:59:59 +03:00:00 [dst=F off=10800] + // 1540681200 = 2018-10-28 03:00:00 +04:00:00 [dst=F off=14400] + // but std::mktime(2018-10-28 02:00:00, tm_isdst=0) fails, unlike, + // say, the similar Europe/Chisinau transition from +02 to +03 ... + // 1521935999 = 2018-03-25 01:59:59 +02:00:00 [dst=F off=7200] + // 1521936000 = 2018-03-25 03:00:00 +03:00:00 [dst=T off=10800] + // where std::mktime(2018-03-25 02:00:00, tm_isdst=0) succeeds and + // returns 1521936000. + continue; + } + if (cs == civil_second(2037, 10, 4, 2, 0, 0)) { + const std::string tzname = *np; + if (tzname == "Africa/Casablanca" || tzname == "Africa/El_Aaiun") { + // The "libc" implementation gets this transition wrong (at least + // until 2018g when it was removed), returning an offset of 3600 + // instead of 0. TODO: Revert this when 2018g is ubiquitous. + continue; + } + } + const auto cl_lc = lc.lookup(cs); + SCOPED_TRACE(testing::Message() << "For " << cs << " in " << *np); + EXPECT_EQ(cl_zi.kind, cl_lc.kind); + EXPECT_EQ(cl_zi.pre, cl_lc.pre); + EXPECT_EQ(cl_zi.trans, cl_lc.trans); + EXPECT_EQ(cl_zi.post, cl_lc.post); + } + } + if (ep == nullptr) { + ASSERT_EQ(0, unsetenv("TZ")); + } else { + ASSERT_EQ(0, setenv("TZ", tz_name.c_str(), 1)); + } +#endif +} + +TEST(NextTransition, UTC) { + const auto tz = utc_time_zone(); + time_zone::civil_transition trans; + + auto tp = time_point<absl::time_internal::cctz::seconds>::min(); + EXPECT_FALSE(tz.next_transition(tp, &trans)); + + tp = time_point<absl::time_internal::cctz::seconds>::max(); + EXPECT_FALSE(tz.next_transition(tp, &trans)); +} + +TEST(PrevTransition, UTC) { + const auto tz = utc_time_zone(); + time_zone::civil_transition trans; + + auto tp = time_point<absl::time_internal::cctz::seconds>::max(); + EXPECT_FALSE(tz.prev_transition(tp, &trans)); + + tp = time_point<absl::time_internal::cctz::seconds>::min(); + EXPECT_FALSE(tz.prev_transition(tp, &trans)); +} + +TEST(NextTransition, AmericaNewYork) { + const auto tz = LoadZone("America/New_York"); + time_zone::civil_transition trans; + + auto tp = convert(civil_second(2018, 6, 30, 0, 0, 0), tz); + EXPECT_TRUE(tz.next_transition(tp, &trans)); + EXPECT_EQ(civil_second(2018, 11, 4, 2, 0, 0), trans.from); + EXPECT_EQ(civil_second(2018, 11, 4, 1, 0, 0), trans.to); + + tp = time_point<absl::time_internal::cctz::seconds>::max(); + EXPECT_FALSE(tz.next_transition(tp, &trans)); + + tp = time_point<absl::time_internal::cctz::seconds>::min(); + EXPECT_TRUE(tz.next_transition(tp, &trans)); + if (trans.from == civil_second(1918, 3, 31, 2, 0, 0)) { + // It looks like the tzdata is only 32 bit (probably macOS), + // which bottoms out at 1901-12-13T20:45:52+00:00. + EXPECT_EQ(civil_second(1918, 3, 31, 3, 0, 0), trans.to); + } else { + EXPECT_EQ(civil_second(1883, 11, 18, 12, 3, 58), trans.from); + EXPECT_EQ(civil_second(1883, 11, 18, 12, 0, 0), trans.to); + } +} + +TEST(PrevTransition, AmericaNewYork) { + const auto tz = LoadZone("America/New_York"); + time_zone::civil_transition trans; + + auto tp = convert(civil_second(2018, 6, 30, 0, 0, 0), tz); + EXPECT_TRUE(tz.prev_transition(tp, &trans)); + EXPECT_EQ(civil_second(2018, 3, 11, 2, 0, 0), trans.from); + EXPECT_EQ(civil_second(2018, 3, 11, 3, 0, 0), trans.to); + + tp = time_point<absl::time_internal::cctz::seconds>::min(); + EXPECT_FALSE(tz.prev_transition(tp, &trans)); + + tp = time_point<absl::time_internal::cctz::seconds>::max(); + EXPECT_TRUE(tz.prev_transition(tp, &trans)); + // We have a transition but we don't know which one. +} + +TEST(TimeZoneEdgeCase, AmericaNewYork) { + const time_zone tz = LoadZone("America/New_York"); + + // Spring 1:59:59 -> 3:00:00 + auto tp = convert(civil_second(2013, 3, 10, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 3, 10, 1, 59, 59, -5 * 3600, false, "EST"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 3, 10, 3, 0, 0, -4 * 3600, true, "EDT"); + + // Fall 1:59:59 -> 1:00:00 + tp = convert(civil_second(2013, 11, 3, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 11, 3, 1, 59, 59, -4 * 3600, true, "EDT"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 11, 3, 1, 0, 0, -5 * 3600, false, "EST"); +} + +TEST(TimeZoneEdgeCase, AmericaLosAngeles) { + const time_zone tz = LoadZone("America/Los_Angeles"); + + // Spring 1:59:59 -> 3:00:00 + auto tp = convert(civil_second(2013, 3, 10, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 3, 10, 1, 59, 59, -8 * 3600, false, "PST"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 3, 10, 3, 0, 0, -7 * 3600, true, "PDT"); + + // Fall 1:59:59 -> 1:00:00 + tp = convert(civil_second(2013, 11, 3, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 11, 3, 1, 59, 59, -7 * 3600, true, "PDT"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 11, 3, 1, 0, 0, -8 * 3600, false, "PST"); +} + +TEST(TimeZoneEdgeCase, ArizonaNoTransition) { + const time_zone tz = LoadZone("America/Phoenix"); + + // No transition in Spring. + auto tp = convert(civil_second(2013, 3, 10, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 3, 10, 1, 59, 59, -7 * 3600, false, "MST"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 3, 10, 2, 0, 0, -7 * 3600, false, "MST"); + + // No transition in Fall. + tp = convert(civil_second(2013, 11, 3, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 11, 3, 1, 59, 59, -7 * 3600, false, "MST"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 11, 3, 2, 0, 0, -7 * 3600, false, "MST"); +} + +TEST(TimeZoneEdgeCase, AsiaKathmandu) { + const time_zone tz = LoadZone("Asia/Kathmandu"); + + // A non-DST offset change from +0530 to +0545 + // + // 504901799 == Tue, 31 Dec 1985 23:59:59 +0530 (+0530) + // 504901800 == Wed, 1 Jan 1986 00:15:00 +0545 (+0545) + auto tp = convert(civil_second(1985, 12, 31, 23, 59, 59), tz); + ExpectTime(tp, tz, 1985, 12, 31, 23, 59, 59, 5.5 * 3600, false, "+0530"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 1986, 1, 1, 0, 15, 0, 5.75 * 3600, false, "+0545"); +} + +TEST(TimeZoneEdgeCase, PacificChatham) { + const time_zone tz = LoadZone("Pacific/Chatham"); + + // One-hour DST offset changes, but at atypical values + // + // 1365256799 == Sun, 7 Apr 2013 03:44:59 +1345 (+1345) + // 1365256800 == Sun, 7 Apr 2013 02:45:00 +1245 (+1245) + auto tp = convert(civil_second(2013, 4, 7, 3, 44, 59), tz); + ExpectTime(tp, tz, 2013, 4, 7, 3, 44, 59, 13.75 * 3600, true, "+1345"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 4, 7, 2, 45, 0, 12.75 * 3600, false, "+1245"); + + // 1380376799 == Sun, 29 Sep 2013 02:44:59 +1245 (+1245) + // 1380376800 == Sun, 29 Sep 2013 03:45:00 +1345 (+1345) + tp = convert(civil_second(2013, 9, 29, 2, 44, 59), tz); + ExpectTime(tp, tz, 2013, 9, 29, 2, 44, 59, 12.75 * 3600, false, "+1245"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 9, 29, 3, 45, 0, 13.75 * 3600, true, "+1345"); +} + +TEST(TimeZoneEdgeCase, AustraliaLordHowe) { + const time_zone tz = LoadZone("Australia/Lord_Howe"); + + // Half-hour DST offset changes + // + // 1365260399 == Sun, 7 Apr 2013 01:59:59 +1100 (+11) + // 1365260400 == Sun, 7 Apr 2013 01:30:00 +1030 (+1030) + auto tp = convert(civil_second(2013, 4, 7, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 4, 7, 1, 59, 59, 11 * 3600, true, "+11"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 4, 7, 1, 30, 0, 10.5 * 3600, false, "+1030"); + + // 1380986999 == Sun, 6 Oct 2013 01:59:59 +1030 (+1030) + // 1380987000 == Sun, 6 Oct 2013 02:30:00 +1100 (+11) + tp = convert(civil_second(2013, 10, 6, 1, 59, 59), tz); + ExpectTime(tp, tz, 2013, 10, 6, 1, 59, 59, 10.5 * 3600, false, "+1030"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2013, 10, 6, 2, 30, 0, 11 * 3600, true, "+11"); +} + +TEST(TimeZoneEdgeCase, PacificApia) { + const time_zone tz = LoadZone("Pacific/Apia"); + + // At the end of December 2011, Samoa jumped forward by one day, + // skipping 30 December from the local calendar, when the nation + // moved to the west of the International Date Line. + // + // A one-day, non-DST offset change + // + // 1325239199 == Thu, 29 Dec 2011 23:59:59 -1000 (-10) + // 1325239200 == Sat, 31 Dec 2011 00:00:00 +1400 (+14) + auto tp = convert(civil_second(2011, 12, 29, 23, 59, 59), tz); + ExpectTime(tp, tz, 2011, 12, 29, 23, 59, 59, -10 * 3600, true, "-10"); + EXPECT_EQ(363, get_yearday(convert(tp, tz))); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2011, 12, 31, 0, 0, 0, 14 * 3600, true, "+14"); + EXPECT_EQ(365, get_yearday(convert(tp, tz))); +} + +TEST(TimeZoneEdgeCase, AfricaCairo) { + const time_zone tz = LoadZone("Africa/Cairo"); + + if (VersionCmp(tz, "2014c") >= 0) { + // An interesting case of midnight not existing. + // + // 1400191199 == Thu, 15 May 2014 23:59:59 +0200 (EET) + // 1400191200 == Fri, 16 May 2014 01:00:00 +0300 (EEST) + auto tp = convert(civil_second(2014, 5, 15, 23, 59, 59), tz); + ExpectTime(tp, tz, 2014, 5, 15, 23, 59, 59, 2 * 3600, false, "EET"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2014, 5, 16, 1, 0, 0, 3 * 3600, true, "EEST"); + } +} + +TEST(TimeZoneEdgeCase, AfricaMonrovia) { + const time_zone tz = LoadZone("Africa/Monrovia"); + + if (VersionCmp(tz, "2017b") >= 0) { + // Strange offset change -00:44:30 -> +00:00:00 (non-DST) + // + // 63593069 == Thu, 6 Jan 1972 23:59:59 -0044 (MMT) + // 63593070 == Fri, 7 Jan 1972 00:44:30 +0000 (GMT) + auto tp = convert(civil_second(1972, 1, 6, 23, 59, 59), tz); + ExpectTime(tp, tz, 1972, 1, 6, 23, 59, 59, -44.5 * 60, false, "MMT"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 1972, 1, 7, 0, 44, 30, 0 * 60, false, "GMT"); + } +} + +TEST(TimeZoneEdgeCase, AmericaJamaica) { + // Jamaica discontinued DST transitions in 1983, and is now at a + // constant -0500. This makes it an interesting edge-case target. + // Note that the 32-bit times used in a (tzh_version == 0) zoneinfo + // file cannot represent the abbreviation-only transition of 1890, + // so we ignore the abbreviation by expecting what we received. + const time_zone tz = LoadZone("America/Jamaica"); + + // Before the first transition. + if (!tz.version().empty() && VersionCmp(tz, "2018d") >= 0) { + // We avoid the expectations on the -18430 offset below unless we are + // certain we have commit 907241e (Fix off-by-1 error for Jamaica and + // T&C before 1913) from 2018d. TODO: Remove the "version() not empty" + // part when 2018d is generally available from /usr/share/zoneinfo. + auto tp = convert(civil_second(1889, 12, 31, 0, 0, 0), tz); + ExpectTime(tp, tz, 1889, 12, 31, 0, 0, 0, -18430, false, + tz.lookup(tp).abbr); + + // Over the first (abbreviation-change only) transition. + // -2524503170 == Tue, 31 Dec 1889 23:59:59 -0507 (LMT) + // -2524503169 == Wed, 1 Jan 1890 00:00:00 -0507 (KMT) + tp = convert(civil_second(1889, 12, 31, 23, 59, 59), tz); + ExpectTime(tp, tz, 1889, 12, 31, 23, 59, 59, -18430, false, + tz.lookup(tp).abbr); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 1890, 1, 1, 0, 0, 0, -18430, false, "KMT"); + } + + // Over the last (DST) transition. + // 436341599 == Sun, 30 Oct 1983 01:59:59 -0400 (EDT) + // 436341600 == Sun, 30 Oct 1983 01:00:00 -0500 (EST) + auto tp = convert(civil_second(1983, 10, 30, 1, 59, 59), tz); + ExpectTime(tp, tz, 1983, 10, 30, 1, 59, 59, -4 * 3600, true, "EDT"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 1983, 10, 30, 1, 0, 0, -5 * 3600, false, "EST"); + + // After the last transition. + tp = convert(civil_second(1983, 12, 31, 23, 59, 59), tz); + ExpectTime(tp, tz, 1983, 12, 31, 23, 59, 59, -5 * 3600, false, "EST"); +} + +TEST(TimeZoneEdgeCase, WET) { + // Cover some non-existent times within forward transitions. + const time_zone tz = LoadZone("WET"); + + // Before the first transition. + auto tp = convert(civil_second(1977, 1, 1, 0, 0, 0), tz); + ExpectTime(tp, tz, 1977, 1, 1, 0, 0, 0, 0, false, "WET"); + + // Over the first transition. + // 228877199 == Sun, 3 Apr 1977 00:59:59 +0000 (WET) + // 228877200 == Sun, 3 Apr 1977 02:00:00 +0100 (WEST) + tp = convert(civil_second(1977, 4, 3, 0, 59, 59), tz); + ExpectTime(tp, tz, 1977, 4, 3, 0, 59, 59, 0, false, "WET"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 1977, 4, 3, 2, 0, 0, 1 * 3600, true, "WEST"); + + // A non-existent time within the first transition. + time_zone::civil_lookup cl1 = tz.lookup(civil_second(1977, 4, 3, 1, 15, 0)); + EXPECT_EQ(time_zone::civil_lookup::SKIPPED, cl1.kind); + ExpectTime(cl1.pre, tz, 1977, 4, 3, 2, 15, 0, 1 * 3600, true, "WEST"); + ExpectTime(cl1.trans, tz, 1977, 4, 3, 2, 0, 0, 1 * 3600, true, "WEST"); + ExpectTime(cl1.post, tz, 1977, 4, 3, 0, 15, 0, 0 * 3600, false, "WET"); + + // A non-existent time within the second forward transition. + time_zone::civil_lookup cl2 = tz.lookup(civil_second(1978, 4, 2, 1, 15, 0)); + EXPECT_EQ(time_zone::civil_lookup::SKIPPED, cl2.kind); + ExpectTime(cl2.pre, tz, 1978, 4, 2, 2, 15, 0, 1 * 3600, true, "WEST"); + ExpectTime(cl2.trans, tz, 1978, 4, 2, 2, 0, 0, 1 * 3600, true, "WEST"); + ExpectTime(cl2.post, tz, 1978, 4, 2, 0, 15, 0, 0 * 3600, false, "WET"); +} + +TEST(TimeZoneEdgeCase, FixedOffsets) { + const time_zone gmtm5 = LoadZone("Etc/GMT+5"); // -0500 + auto tp = convert(civil_second(1970, 1, 1, 0, 0, 0), gmtm5); + ExpectTime(tp, gmtm5, 1970, 1, 1, 0, 0, 0, -5 * 3600, false, "-05"); + EXPECT_EQ(chrono::system_clock::from_time_t(5 * 3600), tp); + + const time_zone gmtp5 = LoadZone("Etc/GMT-5"); // +0500 + tp = convert(civil_second(1970, 1, 1, 0, 0, 0), gmtp5); + ExpectTime(tp, gmtp5, 1970, 1, 1, 0, 0, 0, 5 * 3600, false, "+05"); + EXPECT_EQ(chrono::system_clock::from_time_t(-5 * 3600), tp); +} + +TEST(TimeZoneEdgeCase, NegativeYear) { + // Tests transition from year 0 (aka 1BCE) to year -1. + const time_zone tz = utc_time_zone(); + auto tp = convert(civil_second(0, 1, 1, 0, 0, 0), tz); + ExpectTime(tp, tz, 0, 1, 1, 0, 0, 0, 0 * 3600, false, "UTC"); + EXPECT_EQ(weekday::saturday, get_weekday(convert(tp, tz))); + tp -= absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, -1, 12, 31, 23, 59, 59, 0 * 3600, false, "UTC"); + EXPECT_EQ(weekday::friday, get_weekday(convert(tp, tz))); +} + +TEST(TimeZoneEdgeCase, UTC32bitLimit) { + const time_zone tz = utc_time_zone(); + + // Limits of signed 32-bit time_t + // + // 2147483647 == Tue, 19 Jan 2038 03:14:07 +0000 (UTC) + // 2147483648 == Tue, 19 Jan 2038 03:14:08 +0000 (UTC) + auto tp = convert(civil_second(2038, 1, 19, 3, 14, 7), tz); + ExpectTime(tp, tz, 2038, 1, 19, 3, 14, 7, 0 * 3600, false, "UTC"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 2038, 1, 19, 3, 14, 8, 0 * 3600, false, "UTC"); +} + +TEST(TimeZoneEdgeCase, UTC5DigitYear) { + const time_zone tz = utc_time_zone(); + + // Rollover to 5-digit year + // + // 253402300799 == Fri, 31 Dec 9999 23:59:59 +0000 (UTC) + // 253402300800 == Sat, 1 Jan 1000 00:00:00 +0000 (UTC) + auto tp = convert(civil_second(9999, 12, 31, 23, 59, 59), tz); + ExpectTime(tp, tz, 9999, 12, 31, 23, 59, 59, 0 * 3600, false, "UTC"); + tp += absl::time_internal::cctz::seconds(1); + ExpectTime(tp, tz, 10000, 1, 1, 0, 0, 0, 0 * 3600, false, "UTC"); +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_posix.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_posix.cc new file mode 100644 index 000000000000..5cdd09e89d0e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_posix.cc @@ -0,0 +1,159 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "time_zone_posix.h" + +#include <cstddef> +#include <cstring> +#include <limits> +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +namespace { + +const char kDigits[] = "0123456789"; + +const char* ParseInt(const char* p, int min, int max, int* vp) { + int value = 0; + const char* op = p; + const int kMaxInt = std::numeric_limits<int>::max(); + for (; const char* dp = strchr(kDigits, *p); ++p) { + int d = static_cast<int>(dp - kDigits); + if (d >= 10) break; // '\0' + if (value > kMaxInt / 10) return nullptr; + value *= 10; + if (value > kMaxInt - d) return nullptr; + value += d; + } + if (p == op || value < min || value > max) return nullptr; + *vp = value; + return p; +} + +// abbr = <.*?> | [^-+,\d]{3,} +const char* ParseAbbr(const char* p, std::string* abbr) { + const char* op = p; + if (*p == '<') { // special zoneinfo <...> form + while (*++p != '>') { + if (*p == '\0') return nullptr; + } + abbr->assign(op + 1, static_cast<std::size_t>(p - op) - 1); + return ++p; + } + while (*p != '\0') { + if (strchr("-+,", *p)) break; + if (strchr(kDigits, *p)) break; + ++p; + } + if (p - op < 3) return nullptr; + abbr->assign(op, static_cast<std::size_t>(p - op)); + return p; +} + +// offset = [+|-]hh[:mm[:ss]] (aggregated into single seconds value) +const char* ParseOffset(const char* p, int min_hour, int max_hour, int sign, + std::int_fast32_t* offset) { + if (p == nullptr) return nullptr; + if (*p == '+' || *p == '-') { + if (*p++ == '-') sign = -sign; + } + int hours = 0; + int minutes = 0; + int seconds = 0; + + p = ParseInt(p, min_hour, max_hour, &hours); + if (p == nullptr) return nullptr; + if (*p == ':') { + p = ParseInt(p + 1, 0, 59, &minutes); + if (p == nullptr) return nullptr; + if (*p == ':') { + p = ParseInt(p + 1, 0, 59, &seconds); + if (p == nullptr) return nullptr; + } + } + *offset = sign * ((((hours * 60) + minutes) * 60) + seconds); + return p; +} + +// datetime = ( Jn | n | Mm.w.d ) [ / offset ] +const char* ParseDateTime(const char* p, PosixTransition* res) { + if (p != nullptr && *p == ',') { + if (*++p == 'M') { + int month = 0; + if ((p = ParseInt(p + 1, 1, 12, &month)) != nullptr && *p == '.') { + int week = 0; + if ((p = ParseInt(p + 1, 1, 5, &week)) != nullptr && *p == '.') { + int weekday = 0; + if ((p = ParseInt(p + 1, 0, 6, &weekday)) != nullptr) { + res->date.fmt = PosixTransition::M; + res->date.m.month = static_cast<std::int_fast8_t>(month); + res->date.m.week = static_cast<std::int_fast8_t>(week); + res->date.m.weekday = static_cast<std::int_fast8_t>(weekday); + } + } + } + } else if (*p == 'J') { + int day = 0; + if ((p = ParseInt(p + 1, 1, 365, &day)) != nullptr) { + res->date.fmt = PosixTransition::J; + res->date.j.day = static_cast<std::int_fast16_t>(day); + } + } else { + int day = 0; + if ((p = ParseInt(p, 0, 365, &day)) != nullptr) { + res->date.fmt = PosixTransition::N; + res->date.n.day = static_cast<std::int_fast16_t>(day); + } + } + } + if (p != nullptr) { + res->time.offset = 2 * 60 * 60; // default offset is 02:00:00 + if (*p == '/') p = ParseOffset(p + 1, -167, 167, 1, &res->time.offset); + } + return p; +} + +} // namespace + +// spec = std offset [ dst [ offset ] , datetime , datetime ] +bool ParsePosixSpec(const std::string& spec, PosixTimeZone* res) { + const char* p = spec.c_str(); + if (*p == ':') return false; + + p = ParseAbbr(p, &res->std_abbr); + p = ParseOffset(p, 0, 24, -1, &res->std_offset); + if (p == nullptr) return false; + if (*p == '\0') return true; + + p = ParseAbbr(p, &res->dst_abbr); + if (p == nullptr) return false; + res->dst_offset = res->std_offset + (60 * 60); // default + if (*p != ',') p = ParseOffset(p, 0, 24, -1, &res->dst_offset); + + p = ParseDateTime(p, &res->dst_start); + p = ParseDateTime(p, &res->dst_end); + + return p != nullptr && *p == '\0'; +} + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_posix.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_posix.h new file mode 100644 index 000000000000..0cf29055f56a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/time_zone_posix.h @@ -0,0 +1,132 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Parsing of a POSIX zone spec as described in the TZ part of section 8.3 in +// http://pubs.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap08.html. +// +// The current POSIX spec for America/Los_Angeles is "PST8PDT,M3.2.0,M11.1.0", +// which would be broken down as ... +// +// PosixTimeZone { +// std_abbr = "PST" +// std_offset = -28800 +// dst_abbr = "PDT" +// dst_offset = -25200 +// dst_start = PosixTransition { +// date { +// m { +// month = 3 +// week = 2 +// weekday = 0 +// } +// } +// time { +// offset = 7200 +// } +// } +// dst_end = PosixTransition { +// date { +// m { +// month = 11 +// week = 1 +// weekday = 0 +// } +// } +// time { +// offset = 7200 +// } +// } +// } + +#ifndef ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_POSIX_H_ +#define ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_POSIX_H_ + +#include <cstdint> +#include <string> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// The date/time of the transition. The date is specified as either: +// (J) the Nth day of the year (1 <= N <= 365), excluding leap days, or +// (N) the Nth day of the year (0 <= N <= 365), including leap days, or +// (M) the Nth weekday of a month (e.g., the 2nd Sunday in March). +// The time, specified as a day offset, identifies the particular moment +// of the transition, and may be negative or >= 24h, and in which case +// it would take us to another day, and perhaps week, or even month. +struct PosixTransition { + enum DateFormat { J, N, M }; + + struct Date { + struct NonLeapDay { + std::int_fast16_t day; // day of non-leap year [1:365] + }; + struct Day { + std::int_fast16_t day; // day of year [0:365] + }; + struct MonthWeekWeekday { + std::int_fast8_t month; // month of year [1:12] + std::int_fast8_t week; // week of month [1:5] (5==last) + std::int_fast8_t weekday; // 0==Sun, ..., 6=Sat + }; + + DateFormat fmt; + + union { + NonLeapDay j; + Day n; + MonthWeekWeekday m; + }; + }; + + struct Time { + std::int_fast32_t offset; // seconds before/after 00:00:00 + }; + + Date date; + Time time; +}; + +// The entirety of a POSIX-string specified time-zone rule. The standard +// abbreviation and offset are always given. If the time zone includes +// daylight saving, then the daylight abbrevation is non-empty and the +// remaining fields are also valid. Note that the start/end transitions +// are not ordered---in the southern hemisphere the transition to end +// daylight time occurs first in any particular year. +struct PosixTimeZone { + std::string std_abbr; + std::int_fast32_t std_offset; + + std::string dst_abbr; + std::int_fast32_t dst_offset; + PosixTransition dst_start; + PosixTransition dst_end; +}; + +// Breaks down a POSIX time-zone specification into its constituent pieces, +// filling in any missing values (DST offset, or start/end transition times) +// with the standard-defined defaults. Returns false if the specification +// could not be parsed (although some fields of *res may have been altered). +bool ParsePosixSpec(const std::string& spec, PosixTimeZone* res); + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_CCTZ_TIME_ZONE_POSIX_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/tzfile.h b/third_party/abseil_cpp/absl/time/internal/cctz/src/tzfile.h new file mode 100644 index 000000000000..269fa36c537a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/tzfile.h @@ -0,0 +1,122 @@ +/* Layout and location of TZif files. */ + +#ifndef TZFILE_H + +#define TZFILE_H + +/* +** This file is in the public domain, so clarified as of +** 1996-06-05 by Arthur David Olson. +*/ + +/* +** This header is for use ONLY with the time conversion code. +** There is no guarantee that it will remain unchanged, +** or that it will remain at all. +** Do NOT copy it to any system include directory. +** Thank you! +*/ + +/* +** Information about time zone files. +*/ + +#ifndef TZDIR +#define TZDIR "/usr/share/zoneinfo" /* Time zone object file directory */ +#endif /* !defined TZDIR */ + +#ifndef TZDEFAULT +#define TZDEFAULT "/etc/localtime" +#endif /* !defined TZDEFAULT */ + +#ifndef TZDEFRULES +#define TZDEFRULES "posixrules" +#endif /* !defined TZDEFRULES */ + +/* See Internet RFC 8536 for more details about the following format. */ + +/* +** Each file begins with. . . +*/ + +#define TZ_MAGIC "TZif" + +struct tzhead { + char tzh_magic[4]; /* TZ_MAGIC */ + char tzh_version[1]; /* '\0' or '2' or '3' as of 2013 */ + char tzh_reserved[15]; /* reserved; must be zero */ + char tzh_ttisutcnt[4]; /* coded number of trans. time flags */ + char tzh_ttisstdcnt[4]; /* coded number of trans. time flags */ + char tzh_leapcnt[4]; /* coded number of leap seconds */ + char tzh_timecnt[4]; /* coded number of transition times */ + char tzh_typecnt[4]; /* coded number of local time types */ + char tzh_charcnt[4]; /* coded number of abbr. chars */ +}; + +/* +** . . .followed by. . . +** +** tzh_timecnt (char [4])s coded transition times a la time(2) +** tzh_timecnt (unsigned char)s types of local time starting at above +** tzh_typecnt repetitions of +** one (char [4]) coded UT offset in seconds +** one (unsigned char) used to set tm_isdst +** one (unsigned char) that's an abbreviation list index +** tzh_charcnt (char)s '\0'-terminated zone abbreviations +** tzh_leapcnt repetitions of +** one (char [4]) coded leap second transition times +** one (char [4]) total correction after above +** tzh_ttisstdcnt (char)s indexed by type; if 1, transition +** time is standard time, if 0, +** transition time is local (wall clock) +** time; if absent, transition times are +** assumed to be local time +** tzh_ttisutcnt (char)s indexed by type; if 1, transition +** time is UT, if 0, transition time is +** local time; if absent, transition +** times are assumed to be local time. +** When this is 1, the corresponding +** std/wall indicator must also be 1. +*/ + +/* +** If tzh_version is '2' or greater, the above is followed by a second instance +** of tzhead and a second instance of the data in which each coded transition +** time uses 8 rather than 4 chars, +** then a POSIX-TZ-environment-variable-style string for use in handling +** instants after the last transition time stored in the file +** (with nothing between the newlines if there is no POSIX representation for +** such instants). +** +** If tz_version is '3' or greater, the above is extended as follows. +** First, the POSIX TZ string's hour offset may range from -167 +** through 167 as compared to the POSIX-required 0 through 24. +** Second, its DST start time may be January 1 at 00:00 and its stop +** time December 31 at 24:00 plus the difference between DST and +** standard time, indicating DST all year. +*/ + +/* +** In the current implementation, "tzset()" refuses to deal with files that +** exceed any of the limits below. +*/ + +#ifndef TZ_MAX_TIMES +#define TZ_MAX_TIMES 2000 +#endif /* !defined TZ_MAX_TIMES */ + +#ifndef TZ_MAX_TYPES +/* This must be at least 17 for Europe/Samara and Europe/Vilnius. */ +#define TZ_MAX_TYPES 256 /* Limited by what (unsigned char)'s can hold */ +#endif /* !defined TZ_MAX_TYPES */ + +#ifndef TZ_MAX_CHARS +#define TZ_MAX_CHARS 50 /* Maximum number of abbreviation characters */ + /* (limited by what unsigned chars can hold) */ +#endif /* !defined TZ_MAX_CHARS */ + +#ifndef TZ_MAX_LEAPS +#define TZ_MAX_LEAPS 50 /* Maximum number of leap second corrections */ +#endif /* !defined TZ_MAX_LEAPS */ + +#endif /* !defined TZFILE_H */ diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/src/zone_info_source.cc b/third_party/abseil_cpp/absl/time/internal/cctz/src/zone_info_source.cc new file mode 100644 index 000000000000..72095339c3c1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/src/zone_info_source.cc @@ -0,0 +1,116 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/internal/cctz/include/cctz/zone_info_source.h" + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz { + +// Defined out-of-line to avoid emitting a weak vtable in all TUs. +ZoneInfoSource::~ZoneInfoSource() {} +std::string ZoneInfoSource::Version() const { return std::string(); } + +} // namespace cctz +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz_extension { + +namespace { + +// A default for cctz_extension::zone_info_source_factory, which simply +// defers to the fallback factory. +std::unique_ptr<absl::time_internal::cctz::ZoneInfoSource> DefaultFactory( + const std::string& name, + const std::function< + std::unique_ptr<absl::time_internal::cctz::ZoneInfoSource>( + const std::string& name)>& fallback_factory) { + return fallback_factory(name); +} + +} // namespace + +// A "weak" definition for cctz_extension::zone_info_source_factory. +// The user may override this with their own "strong" definition (see +// zone_info_source.h). +#if !defined(__has_attribute) +#define __has_attribute(x) 0 +#endif +// MinGW is GCC on Windows, so while it asserts __has_attribute(weak), the +// Windows linker cannot handle that. Nor does the MinGW compiler know how to +// pass "#pragma comment(linker, ...)" to the Windows linker. +#if (__has_attribute(weak) || defined(__GNUC__)) && !defined(__MINGW32__) +ZoneInfoSourceFactory zone_info_source_factory __attribute__((weak)) = + DefaultFactory; +#elif defined(_MSC_VER) && !defined(__MINGW32__) && !defined(_LIBCPP_VERSION) +extern ZoneInfoSourceFactory zone_info_source_factory; +extern ZoneInfoSourceFactory default_factory; +ZoneInfoSourceFactory default_factory = DefaultFactory; +#if defined(_M_IX86) +#pragma comment( \ + linker, \ + "/alternatename:?zone_info_source_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@3P6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@ABV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@ABV?$function@$$A6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@ABV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@2@@Z@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@@ZA=?default_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@3P6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@ABV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@ABV?$function@$$A6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@ABV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@2@@Z@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@@ZA") +#elif defined(_M_IA_64) || defined(_M_AMD64) || defined(_M_ARM) || \ + defined(_M_ARM64) +#pragma comment( \ + linker, \ + "/alternatename:?zone_info_source_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@3P6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@AEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@AEBV?$function@$$A6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@AEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@2@@Z@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@@ZEA=?default_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@3P6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@AEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@AEBV?$function@$$A6A?AV?$unique_ptr@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ + "@@@std@@@std@@AEBV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@2@@Z@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ + "@@ZEA") +#else +#error Unsupported MSVC platform +#endif // _M_<PLATFORM> +#else +// Make it a "strong" definition if we have no other choice. +ZoneInfoSourceFactory zone_info_source_factory = DefaultFactory; +#endif + +} // namespace cctz_extension +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/README.zoneinfo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/README.zoneinfo new file mode 100644 index 000000000000..95fb4a91d17e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/README.zoneinfo @@ -0,0 +1,37 @@ +testdata/zoneinfo contains time-zone data files that may be used with CCTZ. +Install them in a location referenced by the ${TZDIR} environment variable. +Symbolic and hard links have been eliminated for portability. + +On Linux systems the distribution's versions of these files can probably +already be found in the default ${TZDIR} location, /usr/share/zoneinfo. + +New versions can be generated using the following shell script. + + #!/bin/sh - + set -e + DESTDIR=$(mktemp -d) + trap "rm -fr ${DESTDIR}" 0 2 15 + ( + cd ${DESTDIR} + git clone https://github.com/eggert/tz.git + make --directory=tz \ + install DESTDIR=${DESTDIR} \ + DATAFORM=vanguard \ + TZDIR=/zoneinfo \ + REDO=posix_only \ + LOCALTIME=Factory \ + TZDATA_TEXT= \ + ZONETABLES=zone1970.tab + tar --create --dereference --hard-dereference --file tzfile.tar \ + --directory=tz tzfile.h + tar --create --dereference --hard-dereference --file zoneinfo.tar \ + --exclude=zoneinfo/posixrules zoneinfo \ + --directory=tz version + ) + tar --extract --directory src --file ${DESTDIR}/tzfile.tar + tar --extract --directory testdata --file ${DESTDIR}/zoneinfo.tar + exit 0 + +To run the CCTZ tests using the testdata/zoneinfo files, execute: + + bazel test --test_env=TZDIR=${PWD}/testdata/zoneinfo ... diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/version b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/version new file mode 100644 index 000000000000..b4410dce16a3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/version @@ -0,0 +1 @@ +2020d diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Abidjan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Abidjan new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Abidjan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra new file mode 100644 index 000000000000..9ca907bf9bb1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Addis_Ababa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Addis_Ababa new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Addis_Ababa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Algiers b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Algiers new file mode 100644 index 000000000000..56a4dd2a19fa --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Algiers Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Asmara b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Asmara new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Asmara Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Asmera b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Asmera new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Asmera Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bamako b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bamako new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bamako Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bangui b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bangui new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bangui Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Banjul b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Banjul new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Banjul Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bissau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bissau new file mode 100644 index 000000000000..0da1d1e211bc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bissau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Blantyre b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Blantyre new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Blantyre Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Brazzaville b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Brazzaville new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Brazzaville Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bujumbura b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bujumbura new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Bujumbura Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Cairo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Cairo new file mode 100644 index 000000000000..ea38c970086c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Cairo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Casablanca b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Casablanca new file mode 100644 index 000000000000..0263c90bd0d8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Casablanca Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ceuta b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ceuta new file mode 100644 index 000000000000..a461dceaa2ad --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ceuta Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Conakry b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Conakry new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Conakry Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Dakar b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Dakar new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Dakar Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Dar_es_Salaam b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Dar_es_Salaam new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Dar_es_Salaam Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Djibouti b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Djibouti new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Djibouti Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Douala b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Douala new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Douala Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/El_Aaiun b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/El_Aaiun new file mode 100644 index 000000000000..772e23c4cd8b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/El_Aaiun Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Freetown b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Freetown new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Freetown Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Gaborone b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Gaborone new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Gaborone Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Harare b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Harare new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Harare Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Johannesburg b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Johannesburg new file mode 100644 index 000000000000..bada0638f8a2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Johannesburg Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Juba b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Juba new file mode 100644 index 000000000000..36b05220ac00 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Juba Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kampala b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kampala new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kampala Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Khartoum b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Khartoum new file mode 100644 index 000000000000..3f8e44b8a6e1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Khartoum Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kigali b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kigali new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kigali Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kinshasa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kinshasa new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Kinshasa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lagos b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lagos new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lagos Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Libreville b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Libreville new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Libreville Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lome b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lome new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lome Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Luanda b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Luanda new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Luanda Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lubumbashi b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lubumbashi new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lubumbashi Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lusaka b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lusaka new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Lusaka Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Malabo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Malabo new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Malabo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Maputo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Maputo new file mode 100644 index 000000000000..651e5cf67a54 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Maputo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Maseru b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Maseru new file mode 100644 index 000000000000..bada0638f8a2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Maseru Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Mbabane b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Mbabane new file mode 100644 index 000000000000..bada0638f8a2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Mbabane Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Mogadishu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Mogadishu new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Mogadishu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Monrovia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Monrovia new file mode 100644 index 000000000000..837780922f23 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Monrovia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Nairobi b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Nairobi new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Nairobi Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ndjamena b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ndjamena new file mode 100644 index 000000000000..ecbc0966dc2d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ndjamena Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Niamey b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Niamey new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Niamey Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Nouakchott b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Nouakchott new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Nouakchott Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ouagadougou b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ouagadougou new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Ouagadougou Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Porto-Novo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Porto-Novo new file mode 100644 index 000000000000..2f2ce2f7728e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Porto-Novo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Sao_Tome b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Sao_Tome new file mode 100644 index 000000000000..425ad3fda7c5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Sao_Tome Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Timbuktu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Timbuktu new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Timbuktu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Tripoli b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Tripoli new file mode 100644 index 000000000000..e0c89971aabe --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Tripoli Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Tunis b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Tunis new file mode 100644 index 000000000000..ca324cb4cd26 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Tunis Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Windhoek b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Windhoek new file mode 100644 index 000000000000..0edc52b9b783 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Africa/Windhoek Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Adak b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Adak new file mode 100644 index 000000000000..b1497bda631e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Adak Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Anchorage b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Anchorage new file mode 100644 index 000000000000..cdf0572be31d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Anchorage Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Araguaina b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Araguaina new file mode 100644 index 000000000000..f66c9f79d6cd --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Araguaina Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Buenos_Aires b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Buenos_Aires new file mode 100644 index 000000000000..d6f999b8605c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Buenos_Aires Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Catamarca b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Catamarca new file mode 100644 index 000000000000..1dcc8d85434c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Catamarca Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/ComodRivadavia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/ComodRivadavia new file mode 100644 index 000000000000..1dcc8d85434c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/ComodRivadavia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Cordoba b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Cordoba new file mode 100644 index 000000000000..35a52e53d123 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Cordoba Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Jujuy b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Jujuy new file mode 100644 index 000000000000..b275f27c0287 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Jujuy Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/La_Rioja b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/La_Rioja new file mode 100644 index 000000000000..23fca1220522 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/La_Rioja Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Mendoza b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Mendoza new file mode 100644 index 000000000000..691c56978a03 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Mendoza Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Rio_Gallegos b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Rio_Gallegos new file mode 100644 index 000000000000..991d1fae69ee --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Rio_Gallegos Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Salta b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Salta new file mode 100644 index 000000000000..58863e0436d1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Salta Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/San_Juan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/San_Juan new file mode 100644 index 000000000000..7eba33c1c5b1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/San_Juan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/San_Luis b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/San_Luis new file mode 100644 index 000000000000..0a81cbddfa28 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/San_Luis Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Tucuman b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Tucuman new file mode 100644 index 000000000000..10556d5d856a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Tucuman Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Ushuaia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Ushuaia new file mode 100644 index 000000000000..e03175027692 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Argentina/Ushuaia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba new file mode 100644 index 000000000000..d6ddf7d8f6bf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Asuncion b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Asuncion new file mode 100644 index 000000000000..622503674225 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Asuncion Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan new file mode 100644 index 000000000000..c82871528308 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Atka b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Atka new file mode 100644 index 000000000000..b1497bda631e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Atka Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bahia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bahia new file mode 100644 index 000000000000..7969e3076687 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bahia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bahia_Banderas b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bahia_Banderas new file mode 100644 index 000000000000..cbe22a7622d8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bahia_Banderas Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados new file mode 100644 index 000000000000..9d3afa6a53bd --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Belem b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Belem new file mode 100644 index 000000000000..e0d7653c64c1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Belem Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Belize b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Belize new file mode 100644 index 000000000000..de99b845019f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Belize Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon new file mode 100644 index 000000000000..7096b69a564b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Boa_Vista b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Boa_Vista new file mode 100644 index 000000000000..fca97207b283 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Boa_Vista Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bogota b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bogota new file mode 100644 index 000000000000..6cb53d4e6125 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Bogota Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Boise b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Boise new file mode 100644 index 000000000000..72fec9e8c52a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Boise Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Buenos_Aires b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Buenos_Aires new file mode 100644 index 000000000000..d6f999b8605c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Buenos_Aires Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cambridge_Bay b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cambridge_Bay new file mode 100644 index 000000000000..0a2225244a67 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cambridge_Bay Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Campo_Grande b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Campo_Grande new file mode 100644 index 000000000000..6855e4e9fe02 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Campo_Grande Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cancun b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cancun new file mode 100644 index 000000000000..640b259fd0f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cancun Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Caracas b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Caracas new file mode 100644 index 000000000000..8dbe6ff74127 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Caracas Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Catamarca b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Catamarca new file mode 100644 index 000000000000..1dcc8d85434c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Catamarca Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cayenne b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cayenne new file mode 100644 index 000000000000..cd49f0534438 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cayenne Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cayman b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cayman new file mode 100644 index 000000000000..9154643f4c91 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cayman Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Chicago b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Chicago new file mode 100644 index 000000000000..b01688065392 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Chicago Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Chihuahua b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Chihuahua new file mode 100644 index 000000000000..e1780a5750e3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Chihuahua Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour new file mode 100644 index 000000000000..c82871528308 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cordoba b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cordoba new file mode 100644 index 000000000000..35a52e53d123 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cordoba Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Costa_Rica b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Costa_Rica new file mode 100644 index 000000000000..08f0128ee681 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Costa_Rica Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Creston b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Creston new file mode 100644 index 000000000000..9d69a0ab819e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Creston Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cuiaba b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cuiaba new file mode 100644 index 000000000000..c09a87558d53 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Cuiaba Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao new file mode 100644 index 000000000000..d6ddf7d8f6bf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Danmarkshavn b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Danmarkshavn new file mode 100644 index 000000000000..8718efcce25c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Danmarkshavn Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dawson b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dawson new file mode 100644 index 000000000000..07e4c5f4ac38 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dawson Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dawson_Creek b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dawson_Creek new file mode 100644 index 000000000000..761d1d9af536 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dawson_Creek Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Denver b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Denver new file mode 100644 index 000000000000..09e54e5c7c5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Denver Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Detroit b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Detroit new file mode 100644 index 000000000000..6eb3ac46ec56 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Detroit Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Edmonton b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Edmonton new file mode 100644 index 000000000000..645ee9453073 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Edmonton Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Eirunepe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Eirunepe new file mode 100644 index 000000000000..7da4b98fe3c7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Eirunepe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/El_Salvador b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/El_Salvador new file mode 100644 index 000000000000..43484117e285 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/El_Salvador Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Ensenada b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Ensenada new file mode 100644 index 000000000000..19ccd3576d46 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Ensenada Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fort_Nelson b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fort_Nelson new file mode 100644 index 000000000000..2a49c6c50f4c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fort_Nelson Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fort_Wayne b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fort_Wayne new file mode 100644 index 000000000000..6b08d15bdaba --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fort_Wayne Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fortaleza b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fortaleza new file mode 100644 index 000000000000..092e40d70122 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Fortaleza Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Glace_Bay b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Glace_Bay new file mode 100644 index 000000000000..f85eb3415758 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Glace_Bay Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Godthab b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Godthab new file mode 100644 index 000000000000..4ddc99d8b74c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Godthab Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Goose_Bay b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Goose_Bay new file mode 100644 index 000000000000..820e0dd2cd7e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Goose_Bay Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Grand_Turk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Grand_Turk new file mode 100644 index 000000000000..062fcaed8e07 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Grand_Turk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guatemala b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guatemala new file mode 100644 index 000000000000..8aa8e588e3cb --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guatemala Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guayaquil b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guayaquil new file mode 100644 index 000000000000..381ae6c46326 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guayaquil Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana new file mode 100644 index 000000000000..ebd85d0f3e02 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Halifax b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Halifax new file mode 100644 index 000000000000..9fa850a7d4c3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Halifax Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Havana b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Havana new file mode 100644 index 000000000000..e06629d36841 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Havana Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Hermosillo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Hermosillo new file mode 100644 index 000000000000..8283239ecab2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Hermosillo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Indianapolis b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Indianapolis new file mode 100644 index 000000000000..6b08d15bdaba --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Indianapolis Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Knox b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Knox new file mode 100644 index 000000000000..b187d5f8c756 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Knox Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Marengo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Marengo new file mode 100644 index 000000000000..a730fe666b65 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Marengo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Petersburg b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Petersburg new file mode 100644 index 000000000000..341a0235ef48 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Petersburg Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Tell_City b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Tell_City new file mode 100644 index 000000000000..76e1f6285b55 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Tell_City Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Vevay b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Vevay new file mode 100644 index 000000000000..f2acf6cbbd65 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Vevay Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Vincennes b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Vincennes new file mode 100644 index 000000000000..c255f89b6da9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Vincennes Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Winamac b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Winamac new file mode 100644 index 000000000000..8700ed9f065e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indiana/Winamac Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indianapolis b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indianapolis new file mode 100644 index 000000000000..6b08d15bdaba --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Indianapolis Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Inuvik b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Inuvik new file mode 100644 index 000000000000..af3107db51e3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Inuvik Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Iqaluit b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Iqaluit new file mode 100644 index 000000000000..eb2c99cca53a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Iqaluit Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Jamaica b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Jamaica new file mode 100644 index 000000000000..be6b1b6f1e77 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Jamaica Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Jujuy b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Jujuy new file mode 100644 index 000000000000..b275f27c0287 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Jujuy Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Juneau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Juneau new file mode 100644 index 000000000000..e347b369f780 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Juneau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kentucky/Louisville b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kentucky/Louisville new file mode 100644 index 000000000000..f2136d6ed41b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kentucky/Louisville Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kentucky/Monticello b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kentucky/Monticello new file mode 100644 index 000000000000..d9f54a18bbe6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kentucky/Monticello Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Knox_IN b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Knox_IN new file mode 100644 index 000000000000..b187d5f8c756 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Knox_IN Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk new file mode 100644 index 000000000000..d6ddf7d8f6bf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/La_Paz b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/La_Paz new file mode 100644 index 000000000000..68ddaae768e6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/La_Paz Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Lima b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Lima new file mode 100644 index 000000000000..b643c5517f92 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Lima Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Los_Angeles b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Los_Angeles new file mode 100644 index 000000000000..aaf07787ad92 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Los_Angeles Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Louisville b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Louisville new file mode 100644 index 000000000000..f2136d6ed41b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Louisville Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes new file mode 100644 index 000000000000..d6ddf7d8f6bf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Maceio b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Maceio new file mode 100644 index 000000000000..dbb8d57d91d6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Maceio Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Managua b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Managua new file mode 100644 index 000000000000..86ef76bf2241 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Managua Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Manaus b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Manaus new file mode 100644 index 000000000000..59c952ebc651 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Manaus Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Martinique b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Martinique new file mode 100644 index 000000000000..25c0232d9549 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Martinique Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Matamoros b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Matamoros new file mode 100644 index 000000000000..722751b20e52 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Matamoros Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mazatlan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mazatlan new file mode 100644 index 000000000000..4c819fab0247 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mazatlan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mendoza b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mendoza new file mode 100644 index 000000000000..691c56978a03 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mendoza Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Menominee b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Menominee new file mode 100644 index 000000000000..28d2c56e1a99 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Menominee Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Merida b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Merida new file mode 100644 index 000000000000..d3b0ca12c95f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Merida Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Metlakatla b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Metlakatla new file mode 100644 index 000000000000..9fefee388e6f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Metlakatla Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mexico_City b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mexico_City new file mode 100644 index 000000000000..ffcf8bee10e5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Mexico_City Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Miquelon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Miquelon new file mode 100644 index 000000000000..3b62585d0a13 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Miquelon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Moncton b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Moncton new file mode 100644 index 000000000000..ecb69ef2c96a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Moncton Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Monterrey b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Monterrey new file mode 100644 index 000000000000..dea9e3f5864f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Monterrey Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montevideo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montevideo new file mode 100644 index 000000000000..4b2fb3e560f6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montevideo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montreal b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montreal new file mode 100644 index 000000000000..fe6be8ea8c97 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montreal Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau new file mode 100644 index 000000000000..cf1e92dc0f53 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/New_York b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/New_York new file mode 100644 index 000000000000..2b6c2eea14df --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/New_York Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nipigon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nipigon new file mode 100644 index 000000000000..b9f67a9f9495 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nipigon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nome b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nome new file mode 100644 index 000000000000..23ead1c004ff --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nome Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Noronha b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Noronha new file mode 100644 index 000000000000..9e74745ca791 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Noronha Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/Beulah b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/Beulah new file mode 100644 index 000000000000..becf4383301e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/Beulah Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/Center b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/Center new file mode 100644 index 000000000000..d03bda045d31 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/Center Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/New_Salem b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/New_Salem new file mode 100644 index 000000000000..ecefc15d8cdf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/North_Dakota/New_Salem Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nuuk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nuuk new file mode 100644 index 000000000000..4ddc99d8b74c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Nuuk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Ojinaga b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Ojinaga new file mode 100644 index 000000000000..da0909cb212f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Ojinaga Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Panama b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Panama new file mode 100644 index 000000000000..9154643f4c91 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Panama Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Pangnirtung b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Pangnirtung new file mode 100644 index 000000000000..5be6f9b0160d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Pangnirtung Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Paramaribo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Paramaribo new file mode 100644 index 000000000000..24f925a2dd33 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Paramaribo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Phoenix b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Phoenix new file mode 100644 index 000000000000..c2bd2f949b24 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Phoenix Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Port-au-Prince b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Port-au-Prince new file mode 100644 index 000000000000..3e75731baa7c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Port-au-Prince Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Porto_Acre b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Porto_Acre new file mode 100644 index 000000000000..fb5185ca6028 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Porto_Acre Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Porto_Velho b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Porto_Velho new file mode 100644 index 000000000000..7f8047d9396f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Porto_Velho Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Puerto_Rico b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Puerto_Rico new file mode 100644 index 000000000000..47b4dc34160d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Puerto_Rico Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas new file mode 100644 index 000000000000..5c9a20b947f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rainy_River b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rainy_River new file mode 100644 index 000000000000..d6ddda4822d3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rainy_River Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rankin_Inlet b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rankin_Inlet new file mode 100644 index 000000000000..92e2ed2dbe0f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rankin_Inlet Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Recife b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Recife new file mode 100644 index 000000000000..305abcb8a221 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Recife Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Regina b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Regina new file mode 100644 index 000000000000..a3f8217a544e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Regina Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Resolute b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Resolute new file mode 100644 index 000000000000..a84d1dfdb3a8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Resolute Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rio_Branco b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rio_Branco new file mode 100644 index 000000000000..fb5185ca6028 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rio_Branco Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rosario b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rosario new file mode 100644 index 000000000000..35a52e53d123 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Rosario Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santa_Isabel b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santa_Isabel new file mode 100644 index 000000000000..19ccd3576d46 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santa_Isabel Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santarem b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santarem new file mode 100644 index 000000000000..f81d144206ac --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santarem Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago new file mode 100644 index 000000000000..8d6032264b65 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santo_Domingo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santo_Domingo new file mode 100644 index 000000000000..3e0785086639 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Santo_Domingo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Sao_Paulo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Sao_Paulo new file mode 100644 index 000000000000..a16da2c4d5a9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Sao_Paulo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Scoresbysund b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Scoresbysund new file mode 100644 index 000000000000..6db49124e2bb --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Scoresbysund Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Shiprock b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Shiprock new file mode 100644 index 000000000000..09e54e5c7c5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Shiprock Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Sitka b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Sitka new file mode 100644 index 000000000000..36681ed78eaf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Sitka Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Johns b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Johns new file mode 100644 index 000000000000..e5f2aec2bb2b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Johns Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Swift_Current b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Swift_Current new file mode 100644 index 000000000000..bdbb494487de --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Swift_Current Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tegucigalpa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tegucigalpa new file mode 100644 index 000000000000..38036a32831d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tegucigalpa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Thule b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Thule new file mode 100644 index 000000000000..f38dc56bf20d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Thule Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Thunder_Bay b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Thunder_Bay new file mode 100644 index 000000000000..fcb032804311 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Thunder_Bay Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tijuana b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tijuana new file mode 100644 index 000000000000..19ccd3576d46 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tijuana Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Toronto b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Toronto new file mode 100644 index 000000000000..fe6be8ea8c97 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Toronto Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Vancouver b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Vancouver new file mode 100644 index 000000000000..c998491112ea --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Vancouver Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin new file mode 100644 index 000000000000..f4fe59034242 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Whitehorse b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Whitehorse new file mode 100644 index 000000000000..878b6a92f740 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Whitehorse Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Winnipeg b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Winnipeg new file mode 100644 index 000000000000..7e646d18e188 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Winnipeg Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Yakutat b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Yakutat new file mode 100644 index 000000000000..773feba89d36 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Yakutat Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Yellowknife b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Yellowknife new file mode 100644 index 000000000000..c779cef92c8a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/America/Yellowknife Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Casey b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Casey new file mode 100644 index 000000000000..30315cc07893 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Casey Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Davis b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Davis new file mode 100644 index 000000000000..3ec32224f298 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Davis Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville new file mode 100644 index 000000000000..c0cfc85a29e8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Macquarie b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Macquarie new file mode 100644 index 000000000000..232717b613f1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Macquarie Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Mawson b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Mawson new file mode 100644 index 000000000000..05e4c6c58673 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Mawson Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/McMurdo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/McMurdo new file mode 100644 index 000000000000..afb392931847 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/McMurdo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Palmer b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Palmer new file mode 100644 index 000000000000..32c1941634ae --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Palmer Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Rothera b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Rothera new file mode 100644 index 000000000000..ea49c00b2240 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Rothera Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/South_Pole b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/South_Pole new file mode 100644 index 000000000000..afb392931847 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/South_Pole Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa new file mode 100644 index 000000000000..97d80d752c8e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Troll b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Troll new file mode 100644 index 000000000000..4e31affb50e3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Troll Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Vostok b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Vostok new file mode 100644 index 000000000000..6e3290717e5a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Vostok Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Arctic/Longyearbyen b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Arctic/Longyearbyen new file mode 100644 index 000000000000..dfc509570a5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Arctic/Longyearbyen Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aden b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aden new file mode 100644 index 000000000000..01c47ccb86cc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aden Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Almaty b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Almaty new file mode 100644 index 000000000000..3ec4fc89fe99 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Almaty Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman new file mode 100644 index 000000000000..1bd09fef27c9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Anadyr b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Anadyr new file mode 100644 index 000000000000..551884d322bc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Anadyr Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aqtau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aqtau new file mode 100644 index 000000000000..3a40d1175a7d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aqtau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aqtobe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aqtobe new file mode 100644 index 000000000000..62c5840a83e2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Aqtobe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ashgabat b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ashgabat new file mode 100644 index 000000000000..848216726908 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ashgabat Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ashkhabad b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ashkhabad new file mode 100644 index 000000000000..848216726908 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ashkhabad Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Atyrau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Atyrau new file mode 100644 index 000000000000..cb2c82f657c7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Atyrau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Baghdad b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Baghdad new file mode 100644 index 000000000000..a3ce97599100 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Baghdad Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bahrain b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bahrain new file mode 100644 index 000000000000..7409d74983c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bahrain Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Baku b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Baku new file mode 100644 index 000000000000..96203d7a4266 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Baku Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bangkok b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bangkok new file mode 100644 index 000000000000..ed687d2985c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bangkok Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Barnaul b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Barnaul new file mode 100644 index 000000000000..ff976dd3b27a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Barnaul Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Beirut b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Beirut new file mode 100644 index 000000000000..55dce5722cc9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Beirut Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bishkek b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bishkek new file mode 100644 index 000000000000..fe7832cdf99e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Bishkek Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Brunei b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Brunei new file mode 100644 index 000000000000..e67b411b9f5a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Brunei Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Calcutta b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Calcutta new file mode 100644 index 000000000000..00bc80a65e9a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Calcutta Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chita b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chita new file mode 100644 index 000000000000..9d49cd35cd5e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chita Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Choibalsan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Choibalsan new file mode 100644 index 000000000000..0a948c2eaca3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Choibalsan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chongqing b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chongqing new file mode 100644 index 000000000000..d6b66984a2f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chongqing Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chungking b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chungking new file mode 100644 index 000000000000..d6b66984a2f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Chungking Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Colombo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Colombo new file mode 100644 index 000000000000..3eeb1b72b689 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Colombo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dacca b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dacca new file mode 100644 index 000000000000..28136808b6d1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dacca Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Damascus b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Damascus new file mode 100644 index 000000000000..168ef9baa47e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Damascus Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dhaka b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dhaka new file mode 100644 index 000000000000..28136808b6d1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dhaka Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dili b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dili new file mode 100644 index 000000000000..bb7be9f3a471 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dili Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dubai b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dubai new file mode 100644 index 000000000000..58d75bc26eec --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dubai Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dushanbe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dushanbe new file mode 100644 index 000000000000..d83fb076a256 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Dushanbe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Famagusta b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Famagusta new file mode 100644 index 000000000000..cc44179564af --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Famagusta Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza new file mode 100644 index 000000000000..4278ffa512b3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Harbin b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Harbin new file mode 100644 index 000000000000..d6b66984a2f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Harbin Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron new file mode 100644 index 000000000000..e55318aaca8c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ho_Chi_Minh b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ho_Chi_Minh new file mode 100644 index 000000000000..7ca997250288 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ho_Chi_Minh Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hong_Kong b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hong_Kong new file mode 100644 index 000000000000..c80e364801be --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hong_Kong Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hovd b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hovd new file mode 100644 index 000000000000..6e08a261274e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hovd Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Irkutsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Irkutsk new file mode 100644 index 000000000000..550e2a08773b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Irkutsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Istanbul b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Istanbul new file mode 100644 index 000000000000..c89186687300 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Istanbul Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jakarta b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jakarta new file mode 100644 index 000000000000..c9752d2f23eb --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jakarta Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jayapura b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jayapura new file mode 100644 index 000000000000..7c22f539d948 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jayapura Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jerusalem b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jerusalem new file mode 100644 index 000000000000..e6e6cc6ca584 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Jerusalem Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kabul b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kabul new file mode 100644 index 000000000000..660ce4cf6957 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kabul Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kamchatka b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kamchatka new file mode 100644 index 000000000000..c65155402db6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kamchatka Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Karachi b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Karachi new file mode 100644 index 000000000000..e56d5afdafb2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Karachi Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kashgar b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kashgar new file mode 100644 index 000000000000..69ff7f6fb497 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kashgar Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kathmandu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kathmandu new file mode 100644 index 000000000000..3a0d330ffd2f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kathmandu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Katmandu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Katmandu new file mode 100644 index 000000000000..3a0d330ffd2f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Katmandu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Khandyga b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Khandyga new file mode 100644 index 000000000000..aeb733202acd --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Khandyga Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kolkata b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kolkata new file mode 100644 index 000000000000..00bc80a65e9a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kolkata Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Krasnoyarsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Krasnoyarsk new file mode 100644 index 000000000000..e0d4fcb5c3d7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Krasnoyarsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuala_Lumpur b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuala_Lumpur new file mode 100644 index 000000000000..e93dd5141bca --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuala_Lumpur Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuching b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuching new file mode 100644 index 000000000000..59bc6e40b7bb --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuching Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuwait b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuwait new file mode 100644 index 000000000000..01c47ccb86cc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Kuwait Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Macao b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Macao new file mode 100644 index 000000000000..c22f75e42db6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Macao Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Macau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Macau new file mode 100644 index 000000000000..c22f75e42db6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Macau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Magadan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Magadan new file mode 100644 index 000000000000..16bac8444656 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Magadan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Makassar b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Makassar new file mode 100644 index 000000000000..5990010b6497 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Makassar Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Manila b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Manila new file mode 100644 index 000000000000..3c3584e09ae4 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Manila Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Muscat b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Muscat new file mode 100644 index 000000000000..58d75bc26eec --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Muscat Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Nicosia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Nicosia new file mode 100644 index 000000000000..c210d0a5989c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Nicosia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Novokuznetsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Novokuznetsk new file mode 100644 index 000000000000..9378d50539df --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Novokuznetsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Novosibirsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Novosibirsk new file mode 100644 index 000000000000..65a9fa2cd2e8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Novosibirsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Omsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Omsk new file mode 100644 index 000000000000..dc0ed422f619 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Omsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Oral b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Oral new file mode 100644 index 000000000000..25a63ec8b995 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Oral Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Phnom_Penh b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Phnom_Penh new file mode 100644 index 000000000000..ed687d2985c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Phnom_Penh Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Pontianak b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Pontianak new file mode 100644 index 000000000000..285bed2c63a5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Pontianak Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Pyongyang b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Pyongyang new file mode 100644 index 000000000000..57240cf89fb3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Pyongyang Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qatar b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qatar new file mode 100644 index 000000000000..7409d74983c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qatar Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qostanay b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qostanay new file mode 100644 index 000000000000..ff6fe61d5075 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qostanay Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qyzylorda b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qyzylorda new file mode 100644 index 000000000000..fe4d6c6d6d44 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Qyzylorda Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Rangoon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Rangoon new file mode 100644 index 000000000000..14b2ad09ead5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Rangoon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Riyadh b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Riyadh new file mode 100644 index 000000000000..01c47ccb86cc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Riyadh Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Saigon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Saigon new file mode 100644 index 000000000000..7ca997250288 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Saigon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Sakhalin b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Sakhalin new file mode 100644 index 000000000000..69f0faad1e72 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Sakhalin Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Samarkand b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Samarkand new file mode 100644 index 000000000000..c43e27c5d4bd --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Samarkand Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Seoul b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Seoul new file mode 100644 index 000000000000..1755147fab44 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Seoul Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Shanghai b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Shanghai new file mode 100644 index 000000000000..d6b66984a2f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Shanghai Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Singapore b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Singapore new file mode 100644 index 000000000000..350d77e28ee7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Singapore Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Srednekolymsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Srednekolymsk new file mode 100644 index 000000000000..7fdee5cbee2b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Srednekolymsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Taipei b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Taipei new file mode 100644 index 000000000000..35d89d036d07 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Taipei Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tashkent b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tashkent new file mode 100644 index 000000000000..65ee428ce1c5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tashkent Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tbilisi b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tbilisi new file mode 100644 index 000000000000..166e4341d6ce --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tbilisi Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tehran b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tehran new file mode 100644 index 000000000000..f1555f0032f5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tehran Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tel_Aviv b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tel_Aviv new file mode 100644 index 000000000000..e6e6cc6ca584 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tel_Aviv Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Thimbu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Thimbu new file mode 100644 index 000000000000..0edc72cfe46b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Thimbu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Thimphu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Thimphu new file mode 100644 index 000000000000..0edc72cfe46b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Thimphu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tokyo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tokyo new file mode 100644 index 000000000000..1aa066ce38fc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tokyo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tomsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tomsk new file mode 100644 index 000000000000..c3c307d7b99f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Tomsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ujung_Pandang b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ujung_Pandang new file mode 100644 index 000000000000..5990010b6497 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ujung_Pandang Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ulaanbaatar b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ulaanbaatar new file mode 100644 index 000000000000..6f5d3a15abbe --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ulaanbaatar Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ulan_Bator b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ulan_Bator new file mode 100644 index 000000000000..6f5d3a15abbe --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ulan_Bator Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Urumqi b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Urumqi new file mode 100644 index 000000000000..69ff7f6fb497 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Urumqi Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ust-Nera b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ust-Nera new file mode 100644 index 000000000000..c39331e3aa7d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Ust-Nera Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Vientiane b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Vientiane new file mode 100644 index 000000000000..ed687d2985c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Vientiane Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Vladivostok b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Vladivostok new file mode 100644 index 000000000000..72a3d4e87a0d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Vladivostok Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yakutsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yakutsk new file mode 100644 index 000000000000..336f932e8d45 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yakutsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yangon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yangon new file mode 100644 index 000000000000..14b2ad09ead5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yangon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yekaterinburg b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yekaterinburg new file mode 100644 index 000000000000..a3bf7f29b6f1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yekaterinburg Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yerevan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yerevan new file mode 100644 index 000000000000..6dd927cb9410 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Asia/Yerevan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores new file mode 100644 index 000000000000..b7f75a9cf650 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Bermuda b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Bermuda new file mode 100644 index 000000000000..aa3301478f72 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Bermuda Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Canary b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Canary new file mode 100644 index 000000000000..5ab3243a5f01 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Canary Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Cape_Verde b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Cape_Verde new file mode 100644 index 000000000000..8f7de1c0a19a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Cape_Verde Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Faeroe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Faeroe new file mode 100644 index 000000000000..9558bf7180ac --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Faeroe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Faroe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Faroe new file mode 100644 index 000000000000..9558bf7180ac --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Faroe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Jan_Mayen b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Jan_Mayen new file mode 100644 index 000000000000..dfc509570a5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Jan_Mayen Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira new file mode 100644 index 000000000000..7c3a49c0e8cd --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Reykjavik b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Reykjavik new file mode 100644 index 000000000000..2451aca76d7c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Reykjavik Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/South_Georgia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/South_Georgia new file mode 100644 index 000000000000..7fa5f4683538 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/South_Georgia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/St_Helena b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/St_Helena new file mode 100644 index 000000000000..8906e88c819d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/St_Helena Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Stanley b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Stanley new file mode 100644 index 000000000000..1a4c8ea86361 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Stanley Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/ACT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/ACT new file mode 100644 index 000000000000..f235d0dc742a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/ACT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Adelaide b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Adelaide new file mode 100644 index 000000000000..f397b3b9eda6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Adelaide Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Brisbane b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Brisbane new file mode 100644 index 000000000000..c7915db30bae --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Brisbane Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Broken_Hill b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Broken_Hill new file mode 100644 index 000000000000..ed0d17a610e0 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Broken_Hill Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Canberra b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Canberra new file mode 100644 index 000000000000..f235d0dc742a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Canberra Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Currie b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Currie new file mode 100644 index 000000000000..55ceaefc6075 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Currie Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Darwin b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Darwin new file mode 100644 index 000000000000..7114153b65c3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Darwin Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Eucla b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Eucla new file mode 100644 index 000000000000..9fbc01fe6779 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Eucla Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Hobart b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Hobart new file mode 100644 index 000000000000..21ef2d396f6b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Hobart Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/LHI b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/LHI new file mode 100644 index 000000000000..4d4ec8ceea9a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/LHI Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Lindeman b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Lindeman new file mode 100644 index 000000000000..e271d5b3e50e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Lindeman Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Lord_Howe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Lord_Howe new file mode 100644 index 000000000000..4d4ec8ceea9a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Lord_Howe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Melbourne b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Melbourne new file mode 100644 index 000000000000..c7160da33f3e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Melbourne Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/NSW b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/NSW new file mode 100644 index 000000000000..f235d0dc742a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/NSW Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/North b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/North new file mode 100644 index 000000000000..7114153b65c3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/North Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Perth b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Perth new file mode 100644 index 000000000000..e449b03fcf25 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Perth Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Queensland b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Queensland new file mode 100644 index 000000000000..c7915db30bae --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Queensland Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/South b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/South new file mode 100644 index 000000000000..f397b3b9eda6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/South Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Sydney b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Sydney new file mode 100644 index 000000000000..f235d0dc742a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Sydney Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Tasmania b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Tasmania new file mode 100644 index 000000000000..21ef2d396f6b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Tasmania Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Victoria b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Victoria new file mode 100644 index 000000000000..c7160da33f3e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Victoria Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/West b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/West new file mode 100644 index 000000000000..e449b03fcf25 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/West Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Yancowinna b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Yancowinna new file mode 100644 index 000000000000..ed0d17a610e0 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Australia/Yancowinna Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/Acre b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/Acre new file mode 100644 index 000000000000..fb5185ca6028 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/Acre Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/DeNoronha b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/DeNoronha new file mode 100644 index 000000000000..9e74745ca791 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/DeNoronha Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/East b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/East new file mode 100644 index 000000000000..a16da2c4d5a9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/East Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/West b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/West new file mode 100644 index 000000000000..59c952ebc651 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Brazil/West Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/CET b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/CET new file mode 100644 index 000000000000..546748d6eace --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/CET Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/CST6CDT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/CST6CDT new file mode 100644 index 000000000000..d9315580584f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/CST6CDT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Atlantic b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Atlantic new file mode 100644 index 000000000000..9fa850a7d4c3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Atlantic Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Central b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Central new file mode 100644 index 000000000000..7e646d18e188 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Central Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Eastern b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Eastern new file mode 100644 index 000000000000..fe6be8ea8c97 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Eastern Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Mountain b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Mountain new file mode 100644 index 000000000000..645ee9453073 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Mountain Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Newfoundland b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Newfoundland new file mode 100644 index 000000000000..e5f2aec2bb2b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Newfoundland Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Pacific b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Pacific new file mode 100644 index 000000000000..c998491112ea --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Pacific Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Saskatchewan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Saskatchewan new file mode 100644 index 000000000000..a3f8217a544e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Saskatchewan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Yukon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Yukon new file mode 100644 index 000000000000..878b6a92f740 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Canada/Yukon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental new file mode 100644 index 000000000000..8d6032264b65 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Chile/EasterIsland b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Chile/EasterIsland new file mode 100644 index 000000000000..d29bcd68b020 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Chile/EasterIsland Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Cuba b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Cuba new file mode 100644 index 000000000000..e06629d36841 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Cuba Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EET b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EET new file mode 100644 index 000000000000..378919ea1131 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EET Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EST b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EST new file mode 100644 index 000000000000..3ae969114563 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EST Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EST5EDT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EST5EDT new file mode 100644 index 000000000000..50c95e0cb076 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/EST5EDT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Egypt b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Egypt new file mode 100644 index 000000000000..ea38c970086c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Egypt Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Eire b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Eire new file mode 100644 index 000000000000..4a45ea8f73f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Eire Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+0 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+0 new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+0 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+1 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+1 new file mode 100644 index 000000000000..98d5dcf917c6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+1 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+10 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+10 new file mode 100644 index 000000000000..ecb287e66786 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+10 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+11 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+11 new file mode 100644 index 000000000000..e941412971a4 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+11 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+12 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+12 new file mode 100644 index 000000000000..9c95bd0736da --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+12 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+2 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+2 new file mode 100644 index 000000000000..6d5ce3db7323 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+2 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+3 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+3 new file mode 100644 index 000000000000..5ef7be71fd96 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+3 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+4 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+4 new file mode 100644 index 000000000000..75f16216f0d3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+4 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+5 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+5 new file mode 100644 index 000000000000..589990ae8966 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+5 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+6 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+6 new file mode 100644 index 000000000000..fcb60ca2465a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+6 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+7 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+7 new file mode 100644 index 000000000000..c0427a40eef9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+7 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+8 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+8 new file mode 100644 index 000000000000..9bdc2283c07d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+8 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+9 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+9 new file mode 100644 index 000000000000..ca7a81f656f2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT+9 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-0 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-0 new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-0 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-1 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-1 new file mode 100644 index 000000000000..cb45601c958d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-1 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-10 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-10 new file mode 100644 index 000000000000..11d988e10a3e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-10 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-11 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-11 new file mode 100644 index 000000000000..f4c5d5cc29b5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-11 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-12 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-12 new file mode 100644 index 000000000000..cd397b02cdde --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-12 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-13 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-13 new file mode 100644 index 000000000000..8fad7c6b0bef --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-13 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-14 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-14 new file mode 100644 index 000000000000..a595e60eeae1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-14 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-2 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-2 new file mode 100644 index 000000000000..97b44a9baecf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-2 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-3 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-3 new file mode 100644 index 000000000000..4eb17ff0057b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-3 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-4 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-4 new file mode 100644 index 000000000000..13aef80cbbcf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-4 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-5 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-5 new file mode 100644 index 000000000000..83a28169552f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-5 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-6 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-6 new file mode 100644 index 000000000000..79a983e5454a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-6 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-7 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-7 new file mode 100644 index 000000000000..e136690e165a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-7 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-8 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-8 new file mode 100644 index 000000000000..bc70fe416fdb --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-8 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-9 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-9 new file mode 100644 index 000000000000..d18cedd524f4 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT-9 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT0 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT0 new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/GMT0 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Greenwich b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Greenwich new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Greenwich Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/UCT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/UCT new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/UCT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/UTC b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/UTC new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/UTC Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Universal b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Universal new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Universal Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Zulu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Zulu new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Etc/Zulu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Amsterdam b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Amsterdam new file mode 100644 index 000000000000..4a6fa1d4945d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Amsterdam Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Andorra b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Andorra new file mode 100644 index 000000000000..38685d4219d2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Andorra Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Astrakhan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Astrakhan new file mode 100644 index 000000000000..aff8d82d2a2d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Astrakhan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Athens b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Athens new file mode 100644 index 000000000000..231bf9c3b713 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Athens Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Belfast b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Belfast new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Belfast Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Belgrade b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Belgrade new file mode 100644 index 000000000000..a1bf9281ed1b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Belgrade Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Berlin b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Berlin new file mode 100644 index 000000000000..465546bd396a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Berlin Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Bratislava b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Bratislava new file mode 100644 index 000000000000..fb7c145ac4c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Bratislava Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Brussels b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Brussels new file mode 100644 index 000000000000..31973271d2f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Brussels Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Bucharest b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Bucharest new file mode 100644 index 000000000000..efa689ba9e0a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Bucharest Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Budapest b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Budapest new file mode 100644 index 000000000000..940be4670a64 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Budapest Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Busingen b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Busingen new file mode 100644 index 000000000000..388df2969f2d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Busingen Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Chisinau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Chisinau new file mode 100644 index 000000000000..6970b14c506c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Chisinau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Copenhagen b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Copenhagen new file mode 100644 index 000000000000..45984a7599d2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Copenhagen Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Dublin b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Dublin new file mode 100644 index 000000000000..4a45ea8f73f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Dublin Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Gibraltar b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Gibraltar new file mode 100644 index 000000000000..017bb2e34746 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Gibraltar Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Guernsey b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Guernsey new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Guernsey Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Helsinki b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Helsinki new file mode 100644 index 000000000000..ff5e56530570 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Helsinki Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Isle_of_Man b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Isle_of_Man new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Isle_of_Man Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Istanbul b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Istanbul new file mode 100644 index 000000000000..c89186687300 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Istanbul Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Jersey b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Jersey new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Jersey Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kaliningrad b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kaliningrad new file mode 100644 index 000000000000..0ec475647055 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kaliningrad Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev new file mode 100644 index 000000000000..8f83cefbcc11 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kirov b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kirov new file mode 100644 index 000000000000..d1c93c540c26 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kirov Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon new file mode 100644 index 000000000000..64841661a01a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Ljubljana b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Ljubljana new file mode 100644 index 000000000000..a1bf9281ed1b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Ljubljana Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/London b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/London new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/London Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Luxembourg b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Luxembourg new file mode 100644 index 000000000000..682bcbf61ac7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Luxembourg Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Madrid b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Madrid new file mode 100644 index 000000000000..60bdf4d07e6e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Madrid Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Malta b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Malta new file mode 100644 index 000000000000..27539c2243ff --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Malta Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Mariehamn b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Mariehamn new file mode 100644 index 000000000000..ff5e56530570 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Mariehamn Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Minsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Minsk new file mode 100644 index 000000000000..30d3a672bf64 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Minsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Monaco b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Monaco new file mode 100644 index 000000000000..f30dfc7014f4 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Monaco Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Moscow b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Moscow new file mode 100644 index 000000000000..5e6b6de6451b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Moscow Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Nicosia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Nicosia new file mode 100644 index 000000000000..c210d0a5989c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Nicosia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Oslo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Oslo new file mode 100644 index 000000000000..dfc509570a5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Oslo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Paris b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Paris new file mode 100644 index 000000000000..00a27264c2cb --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Paris Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Podgorica b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Podgorica new file mode 100644 index 000000000000..a1bf9281ed1b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Podgorica Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Prague b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Prague new file mode 100644 index 000000000000..fb7c145ac4c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Prague Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Riga b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Riga new file mode 100644 index 000000000000..26af4c90b3b7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Riga Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Rome b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Rome new file mode 100644 index 000000000000..639ca3be4062 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Rome Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Samara b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Samara new file mode 100644 index 000000000000..8d0c26e5c852 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Samara Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/San_Marino b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/San_Marino new file mode 100644 index 000000000000..639ca3be4062 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/San_Marino Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Sarajevo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Sarajevo new file mode 100644 index 000000000000..a1bf9281ed1b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Sarajevo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Saratov b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Saratov new file mode 100644 index 000000000000..2684d8f8b89f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Saratov Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol new file mode 100644 index 000000000000..88a6f3bdb469 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Skopje b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Skopje new file mode 100644 index 000000000000..a1bf9281ed1b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Skopje Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Sofia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Sofia new file mode 100644 index 000000000000..eabc972a22df --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Sofia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Stockholm b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Stockholm new file mode 100644 index 000000000000..dd3eb3227852 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Stockholm Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tallinn b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tallinn new file mode 100644 index 000000000000..5321bbd46e7a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tallinn Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tirane b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tirane new file mode 100644 index 000000000000..743a7337ffd8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tirane Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tiraspol b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tiraspol new file mode 100644 index 000000000000..6970b14c506c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Tiraspol Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Ulyanovsk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Ulyanovsk new file mode 100644 index 000000000000..bb842cb1f508 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Ulyanovsk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod new file mode 100644 index 000000000000..a5755685e390 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vaduz b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vaduz new file mode 100644 index 000000000000..388df2969f2d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vaduz Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vatican b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vatican new file mode 100644 index 000000000000..639ca3be4062 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vatican Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vienna b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vienna new file mode 100644 index 000000000000..75339e98d0a7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vienna Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vilnius b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vilnius new file mode 100644 index 000000000000..75b2eebb57c5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Vilnius Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Volgograd b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Volgograd new file mode 100644 index 000000000000..a486ad42eac2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Volgograd Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Warsaw b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Warsaw new file mode 100644 index 000000000000..efe1a40f2a8f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Warsaw Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zagreb b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zagreb new file mode 100644 index 000000000000..a1bf9281ed1b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zagreb Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye new file mode 100644 index 000000000000..4ea8dae45a6f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zurich b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zurich new file mode 100644 index 000000000000..388df2969f2d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zurich Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Factory b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Factory new file mode 100644 index 000000000000..b4dd7735ed5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Factory Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GB b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GB new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GB Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GB-Eire b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GB-Eire new file mode 100644 index 000000000000..323cd3818ac6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GB-Eire Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT+0 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT+0 new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT+0 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT-0 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT-0 new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT-0 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT0 b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT0 new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/GMT0 Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Greenwich b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Greenwich new file mode 100644 index 000000000000..157573b1d340 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Greenwich Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/HST b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/HST new file mode 100644 index 000000000000..160a53e045c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/HST Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Hongkong b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Hongkong new file mode 100644 index 000000000000..c80e364801be --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Hongkong Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Iceland b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Iceland new file mode 100644 index 000000000000..2451aca76d7c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Iceland Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Antananarivo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Antananarivo new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Antananarivo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Chagos b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Chagos new file mode 100644 index 000000000000..8b8ce226b6b7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Chagos Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Christmas b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Christmas new file mode 100644 index 000000000000..766024b36c50 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Christmas Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Cocos b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Cocos new file mode 100644 index 000000000000..117503410ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Cocos Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Comoro b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Comoro new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Comoro Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Kerguelen b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Kerguelen new file mode 100644 index 000000000000..8ce93e01247e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Kerguelen Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mahe b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mahe new file mode 100644 index 000000000000..75362bbf033d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mahe Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Maldives b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Maldives new file mode 100644 index 000000000000..58a82e4eb701 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Maldives Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mauritius b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mauritius new file mode 100644 index 000000000000..7c1113488200 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mauritius Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mayotte b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mayotte new file mode 100644 index 000000000000..d3c0bb328ce9 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Mayotte Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Reunion b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Reunion new file mode 100644 index 000000000000..248a7c93a52b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Indian/Reunion Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Iran b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Iran new file mode 100644 index 000000000000..f1555f0032f5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Iran Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Israel b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Israel new file mode 100644 index 000000000000..e6e6cc6ca584 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Israel Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Jamaica b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Jamaica new file mode 100644 index 000000000000..be6b1b6f1e77 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Jamaica Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Japan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Japan new file mode 100644 index 000000000000..1aa066ce38fc --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Japan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Kwajalein b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Kwajalein new file mode 100644 index 000000000000..9416d522d0a3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Kwajalein Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Libya b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Libya new file mode 100644 index 000000000000..e0c89971aabe --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Libya Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MET b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MET new file mode 100644 index 000000000000..6f0558c3b6f4 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MET Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MST b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MST new file mode 100644 index 000000000000..a0953d1e791e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MST Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MST7MDT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MST7MDT new file mode 100644 index 000000000000..137867c8bf5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/MST7MDT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/BajaNorte b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/BajaNorte new file mode 100644 index 000000000000..19ccd3576d46 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/BajaNorte Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/BajaSur b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/BajaSur new file mode 100644 index 000000000000..4c819fab0247 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/BajaSur Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/General b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/General new file mode 100644 index 000000000000..ffcf8bee10e5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Mexico/General Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/NZ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/NZ new file mode 100644 index 000000000000..afb392931847 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/NZ Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/NZ-CHAT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/NZ-CHAT new file mode 100644 index 000000000000..f06065ebd183 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/NZ-CHAT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Navajo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Navajo new file mode 100644 index 000000000000..09e54e5c7c5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Navajo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/PRC b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/PRC new file mode 100644 index 000000000000..d6b66984a2f3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/PRC Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/PST8PDT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/PST8PDT new file mode 100644 index 000000000000..fde4833f6be3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/PST8PDT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia new file mode 100644 index 000000000000..244af26f8a2f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Auckland b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Auckland new file mode 100644 index 000000000000..afb392931847 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Auckland Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Bougainville b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Bougainville new file mode 100644 index 000000000000..7c667093c50d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Bougainville Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Chatham b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Chatham new file mode 100644 index 000000000000..f06065ebd183 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Chatham Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Chuuk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Chuuk new file mode 100644 index 000000000000..ea3fb5cd3c9e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Chuuk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Easter b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Easter new file mode 100644 index 000000000000..d29bcd68b020 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Easter Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Efate b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Efate new file mode 100644 index 000000000000..906971e2cecf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Efate Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury new file mode 100644 index 000000000000..b22ab147c3b6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fakaofo b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fakaofo new file mode 100644 index 000000000000..b7b30213e154 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fakaofo Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji new file mode 100644 index 000000000000..e3934e423c9c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Funafuti b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Funafuti new file mode 100644 index 000000000000..78ab35b6b02f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Funafuti Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Galapagos b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Galapagos new file mode 100644 index 000000000000..a9403eca467d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Galapagos Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Gambier b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Gambier new file mode 100644 index 000000000000..ddfc34ffc097 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Gambier Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Guadalcanal b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Guadalcanal new file mode 100644 index 000000000000..720c67901740 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Guadalcanal Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Guam b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Guam new file mode 100644 index 000000000000..bf9a2d955fc2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Guam Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Honolulu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Honolulu new file mode 100644 index 000000000000..40e3d492e6c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Honolulu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Johnston b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Johnston new file mode 100644 index 000000000000..40e3d492e6c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Johnston Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kiritimati b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kiritimati new file mode 100644 index 000000000000..2f676d3bf5c8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kiritimati Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kosrae b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kosrae new file mode 100644 index 000000000000..f5d58242c819 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kosrae Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kwajalein b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kwajalein new file mode 100644 index 000000000000..9416d522d0a3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kwajalein Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Majuro b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Majuro new file mode 100644 index 000000000000..9228ee02ede2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Majuro Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Marquesas b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Marquesas new file mode 100644 index 000000000000..6ea24b72cd95 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Marquesas Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Midway b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Midway new file mode 100644 index 000000000000..001289ceecff --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Midway Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Nauru b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Nauru new file mode 100644 index 000000000000..ae13aac7792a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Nauru Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue new file mode 100644 index 000000000000..7b3579351395 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Norfolk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Norfolk new file mode 100644 index 000000000000..79e2a9419adf --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Norfolk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Noumea b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Noumea new file mode 100644 index 000000000000..824f814160ee --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Noumea Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pago_Pago b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pago_Pago new file mode 100644 index 000000000000..001289ceecff --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pago_Pago Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Palau b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Palau new file mode 100644 index 000000000000..bc8eb7a55b8a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Palau Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pitcairn b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pitcairn new file mode 100644 index 000000000000..8a4ba4d30a6b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pitcairn Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pohnpei b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pohnpei new file mode 100644 index 000000000000..b92b254a9af1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Pohnpei Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Ponape b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Ponape new file mode 100644 index 000000000000..b92b254a9af1 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Ponape Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Port_Moresby b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Port_Moresby new file mode 100644 index 000000000000..5d8fc3a1b253 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Port_Moresby Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga new file mode 100644 index 000000000000..143a1883b0f8 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Saipan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Saipan new file mode 100644 index 000000000000..bf9a2d955fc2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Saipan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Samoa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Samoa new file mode 100644 index 000000000000..001289ceecff --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Samoa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tahiti b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tahiti new file mode 100644 index 000000000000..50a064fa0166 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tahiti Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tarawa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tarawa new file mode 100644 index 000000000000..6bc216823e00 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tarawa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu new file mode 100644 index 000000000000..54aeb0ffa227 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Truk b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Truk new file mode 100644 index 000000000000..ea3fb5cd3c9e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Truk Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Wake b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Wake new file mode 100644 index 000000000000..71cca8877d33 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Wake Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Wallis b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Wallis new file mode 100644 index 000000000000..4bce89300306 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Wallis Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Yap b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Yap new file mode 100644 index 000000000000..ea3fb5cd3c9e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Yap Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Poland b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Poland new file mode 100644 index 000000000000..efe1a40f2a8f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Poland Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Portugal b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Portugal new file mode 100644 index 000000000000..64841661a01a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Portugal Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/ROC b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/ROC new file mode 100644 index 000000000000..35d89d036d07 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/ROC Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/ROK b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/ROK new file mode 100644 index 000000000000..1755147fab44 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/ROK Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Singapore b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Singapore new file mode 100644 index 000000000000..350d77e28ee7 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Singapore Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Turkey b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Turkey new file mode 100644 index 000000000000..c89186687300 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Turkey Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/UCT b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/UCT new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/UCT Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Alaska b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Alaska new file mode 100644 index 000000000000..cdf0572be31d --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Alaska Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Aleutian b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Aleutian new file mode 100644 index 000000000000..b1497bda631e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Aleutian Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Arizona b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Arizona new file mode 100644 index 000000000000..c2bd2f949b24 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Arizona Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Central b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Central new file mode 100644 index 000000000000..b01688065392 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Central Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/East-Indiana b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/East-Indiana new file mode 100644 index 000000000000..6b08d15bdaba --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/East-Indiana Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Eastern b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Eastern new file mode 100644 index 000000000000..2b6c2eea14df --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Eastern Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Hawaii b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Hawaii new file mode 100644 index 000000000000..40e3d492e6c2 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Hawaii Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Indiana-Starke b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Indiana-Starke new file mode 100644 index 000000000000..b187d5f8c756 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Indiana-Starke Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Michigan b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Michigan new file mode 100644 index 000000000000..6eb3ac46ec56 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Michigan Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Mountain b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Mountain new file mode 100644 index 000000000000..09e54e5c7c5b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Mountain Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Pacific b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Pacific new file mode 100644 index 000000000000..aaf07787ad92 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Pacific Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Samoa b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Samoa new file mode 100644 index 000000000000..001289ceecff --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/US/Samoa Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/UTC b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/UTC new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/UTC Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Universal b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Universal new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Universal Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/W-SU b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/W-SU new file mode 100644 index 000000000000..5e6b6de6451b --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/W-SU Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/WET b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/WET new file mode 100644 index 000000000000..423c6c203a50 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/WET Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Zulu b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Zulu new file mode 100644 index 000000000000..00841a62213e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/Zulu Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/iso3166.tab b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/iso3166.tab new file mode 100644 index 000000000000..a4ff61a4d321 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/iso3166.tab @@ -0,0 +1,274 @@ +# ISO 3166 alpha-2 country codes +# +# This file is in the public domain, so clarified as of +# 2009-05-17 by Arthur David Olson. +# +# From Paul Eggert (2015-05-02): +# This file contains a table of two-letter country codes. Columns are +# separated by a single tab. Lines beginning with '#' are comments. +# All text uses UTF-8 encoding. The columns of the table are as follows: +# +# 1. ISO 3166-1 alpha-2 country code, current as of +# ISO 3166-1 N976 (2018-11-06). See: Updates on ISO 3166-1 +# https://isotc.iso.org/livelink/livelink/Open/16944257 +# 2. The usual English name for the coded region, +# chosen so that alphabetic sorting of subsets produces helpful lists. +# This is not the same as the English name in the ISO 3166 tables. +# +# The table is sorted by country code. +# +# This table is intended as an aid for users, to help them select time +# zone data appropriate for their practical needs. It is not intended +# to take or endorse any position on legal or territorial claims. +# +#country- +#code name of country, territory, area, or subdivision +AD Andorra +AE United Arab Emirates +AF Afghanistan +AG Antigua & Barbuda +AI Anguilla +AL Albania +AM Armenia +AO Angola +AQ Antarctica +AR Argentina +AS Samoa (American) +AT Austria +AU Australia +AW Aruba +AX Åland Islands +AZ Azerbaijan +BA Bosnia & Herzegovina +BB Barbados +BD Bangladesh +BE Belgium +BF Burkina Faso +BG Bulgaria +BH Bahrain +BI Burundi +BJ Benin +BL St Barthelemy +BM Bermuda +BN Brunei +BO Bolivia +BQ Caribbean NL +BR Brazil +BS Bahamas +BT Bhutan +BV Bouvet Island +BW Botswana +BY Belarus +BZ Belize +CA Canada +CC Cocos (Keeling) Islands +CD Congo (Dem. Rep.) +CF Central African Rep. +CG Congo (Rep.) +CH Switzerland +CI Côte d'Ivoire +CK Cook Islands +CL Chile +CM Cameroon +CN China +CO Colombia +CR Costa Rica +CU Cuba +CV Cape Verde +CW Curaçao +CX Christmas Island +CY Cyprus +CZ Czech Republic +DE Germany +DJ Djibouti +DK Denmark +DM Dominica +DO Dominican Republic +DZ Algeria +EC Ecuador +EE Estonia +EG Egypt +EH Western Sahara +ER Eritrea +ES Spain +ET Ethiopia +FI Finland +FJ Fiji +FK Falkland Islands +FM Micronesia +FO Faroe Islands +FR France +GA Gabon +GB Britain (UK) +GD Grenada +GE Georgia +GF French Guiana +GG Guernsey +GH Ghana +GI Gibraltar +GL Greenland +GM Gambia +GN Guinea +GP Guadeloupe +GQ Equatorial Guinea +GR Greece +GS South Georgia & the South Sandwich Islands +GT Guatemala +GU Guam +GW Guinea-Bissau +GY Guyana +HK Hong Kong +HM Heard Island & McDonald Islands +HN Honduras +HR Croatia +HT Haiti +HU Hungary +ID Indonesia +IE Ireland +IL Israel +IM Isle of Man +IN India +IO British Indian Ocean Territory +IQ Iraq +IR Iran +IS Iceland +IT Italy +JE Jersey +JM Jamaica +JO Jordan +JP Japan +KE Kenya +KG Kyrgyzstan +KH Cambodia +KI Kiribati +KM Comoros +KN St Kitts & Nevis +KP Korea (North) +KR Korea (South) +KW Kuwait +KY Cayman Islands +KZ Kazakhstan +LA Laos +LB Lebanon +LC St Lucia +LI Liechtenstein +LK Sri Lanka +LR Liberia +LS Lesotho +LT Lithuania +LU Luxembourg +LV Latvia +LY Libya +MA Morocco +MC Monaco +MD Moldova +ME Montenegro +MF St Martin (French) +MG Madagascar +MH Marshall Islands +MK North Macedonia +ML Mali +MM Myanmar (Burma) +MN Mongolia +MO Macau +MP Northern Mariana Islands +MQ Martinique +MR Mauritania +MS Montserrat +MT Malta +MU Mauritius +MV Maldives +MW Malawi +MX Mexico +MY Malaysia +MZ Mozambique +NA Namibia +NC New Caledonia +NE Niger +NF Norfolk Island +NG Nigeria +NI Nicaragua +NL Netherlands +NO Norway +NP Nepal +NR Nauru +NU Niue +NZ New Zealand +OM Oman +PA Panama +PE Peru +PF French Polynesia +PG Papua New Guinea +PH Philippines +PK Pakistan +PL Poland +PM St Pierre & Miquelon +PN Pitcairn +PR Puerto Rico +PS Palestine +PT Portugal +PW Palau +PY Paraguay +QA Qatar +RE Réunion +RO Romania +RS Serbia +RU Russia +RW Rwanda +SA Saudi Arabia +SB Solomon Islands +SC Seychelles +SD Sudan +SE Sweden +SG Singapore +SH St Helena +SI Slovenia +SJ Svalbard & Jan Mayen +SK Slovakia +SL Sierra Leone +SM San Marino +SN Senegal +SO Somalia +SR Suriname +SS South Sudan +ST Sao Tome & Principe +SV El Salvador +SX St Maarten (Dutch) +SY Syria +SZ Eswatini (Swaziland) +TC Turks & Caicos Is +TD Chad +TF French Southern & Antarctic Lands +TG Togo +TH Thailand +TJ Tajikistan +TK Tokelau +TL East Timor +TM Turkmenistan +TN Tunisia +TO Tonga +TR Turkey +TT Trinidad & Tobago +TV Tuvalu +TW Taiwan +TZ Tanzania +UA Ukraine +UG Uganda +UM US minor outlying islands +US United States +UY Uruguay +UZ Uzbekistan +VA Vatican City +VC St Vincent +VE Venezuela +VG Virgin Islands (UK) +VI Virgin Islands (US) +VN Vietnam +VU Vanuatu +WF Wallis & Futuna +WS Samoa (western) +YE Yemen +YT Mayotte +ZA South Africa +ZM Zambia +ZW Zimbabwe diff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/localtime b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/localtime new file mode 100644 index 000000000000..afeeb88d0628 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/localtime Binary files differdiff --git a/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab new file mode 100644 index 000000000000..53ee77e88e5e --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab @@ -0,0 +1,384 @@ +# tzdb timezone descriptions +# +# This file is in the public domain. +# +# From Paul Eggert (2018-06-27): +# This file contains a table where each row stands for a timezone where +# civil timestamps have agreed since 1970. Columns are separated by +# a single tab. Lines beginning with '#' are comments. All text uses +# UTF-8 encoding. The columns of the table are as follows: +# +# 1. The countries that overlap the timezone, as a comma-separated list +# of ISO 3166 2-character country codes. See the file 'iso3166.tab'. +# 2. Latitude and longitude of the timezone's principal location +# in ISO 6709 sign-degrees-minutes-seconds format, +# either ±DDMM±DDDMM or ±DDMMSS±DDDMMSS, +# first latitude (+ is north), then longitude (+ is east). +# 3. Timezone name used in value of TZ environment variable. +# Please see the theory.html file for how these names are chosen. +# If multiple timezones overlap a country, each has a row in the +# table, with each column 1 containing the country code. +# 4. Comments; present if and only if a country has multiple timezones. +# +# If a timezone covers multiple countries, the most-populous city is used, +# and that country is listed first in column 1; any other countries +# are listed alphabetically by country code. The table is sorted +# first by country code, then (if possible) by an order within the +# country that (1) makes some geographical sense, and (2) puts the +# most populous timezones first, where that does not contradict (1). +# +# This table is intended as an aid for users, to help them select timezones +# appropriate for their practical needs. It is not intended to take or +# endorse any position on legal or territorial claims. +# +#country- +#codes coordinates TZ comments +AD +4230+00131 Europe/Andorra +AE,OM +2518+05518 Asia/Dubai +AF +3431+06912 Asia/Kabul +AL +4120+01950 Europe/Tirane +AM +4011+04430 Asia/Yerevan +AQ -6617+11031 Antarctica/Casey Casey +AQ -6835+07758 Antarctica/Davis Davis +AQ -6640+14001 Antarctica/DumontDUrville Dumont-d'Urville +AQ -6736+06253 Antarctica/Mawson Mawson +AQ -6448-06406 Antarctica/Palmer Palmer +AQ -6734-06808 Antarctica/Rothera Rothera +AQ -690022+0393524 Antarctica/Syowa Syowa +AQ -720041+0023206 Antarctica/Troll Troll +AQ -7824+10654 Antarctica/Vostok Vostok +AR -3436-05827 America/Argentina/Buenos_Aires Buenos Aires (BA, CF) +AR -3124-06411 America/Argentina/Cordoba Argentina (most areas: CB, CC, CN, ER, FM, MN, SE, SF) +AR -2447-06525 America/Argentina/Salta Salta (SA, LP, NQ, RN) +AR -2411-06518 America/Argentina/Jujuy Jujuy (JY) +AR -2649-06513 America/Argentina/Tucuman Tucumán (TM) +AR -2828-06547 America/Argentina/Catamarca Catamarca (CT); Chubut (CH) +AR -2926-06651 America/Argentina/La_Rioja La Rioja (LR) +AR -3132-06831 America/Argentina/San_Juan San Juan (SJ) +AR -3253-06849 America/Argentina/Mendoza Mendoza (MZ) +AR -3319-06621 America/Argentina/San_Luis San Luis (SL) +AR -5138-06913 America/Argentina/Rio_Gallegos Santa Cruz (SC) +AR -5448-06818 America/Argentina/Ushuaia Tierra del Fuego (TF) +AS,UM -1416-17042 Pacific/Pago_Pago Samoa, Midway +AT +4813+01620 Europe/Vienna +AU -3133+15905 Australia/Lord_Howe Lord Howe Island +AU -5430+15857 Antarctica/Macquarie Macquarie Island +AU -4253+14719 Australia/Hobart Tasmania (most areas) +AU -3956+14352 Australia/Currie Tasmania (King Island) +AU -3749+14458 Australia/Melbourne Victoria +AU -3352+15113 Australia/Sydney New South Wales (most areas) +AU -3157+14127 Australia/Broken_Hill New South Wales (Yancowinna) +AU -2728+15302 Australia/Brisbane Queensland (most areas) +AU -2016+14900 Australia/Lindeman Queensland (Whitsunday Islands) +AU -3455+13835 Australia/Adelaide South Australia +AU -1228+13050 Australia/Darwin Northern Territory +AU -3157+11551 Australia/Perth Western Australia (most areas) +AU -3143+12852 Australia/Eucla Western Australia (Eucla) +AZ +4023+04951 Asia/Baku +BB +1306-05937 America/Barbados +BD +2343+09025 Asia/Dhaka +BE +5050+00420 Europe/Brussels +BG +4241+02319 Europe/Sofia +BM +3217-06446 Atlantic/Bermuda +BN +0456+11455 Asia/Brunei +BO -1630-06809 America/La_Paz +BR -0351-03225 America/Noronha Atlantic islands +BR -0127-04829 America/Belem Pará (east); Amapá +BR -0343-03830 America/Fortaleza Brazil (northeast: MA, PI, CE, RN, PB) +BR -0803-03454 America/Recife Pernambuco +BR -0712-04812 America/Araguaina Tocantins +BR -0940-03543 America/Maceio Alagoas, Sergipe +BR -1259-03831 America/Bahia Bahia +BR -2332-04637 America/Sao_Paulo Brazil (southeast: GO, DF, MG, ES, RJ, SP, PR, SC, RS) +BR -2027-05437 America/Campo_Grande Mato Grosso do Sul +BR -1535-05605 America/Cuiaba Mato Grosso +BR -0226-05452 America/Santarem Pará (west) +BR -0846-06354 America/Porto_Velho Rondônia +BR +0249-06040 America/Boa_Vista Roraima +BR -0308-06001 America/Manaus Amazonas (east) +BR -0640-06952 America/Eirunepe Amazonas (west) +BR -0958-06748 America/Rio_Branco Acre +BS +2505-07721 America/Nassau +BT +2728+08939 Asia/Thimphu +BY +5354+02734 Europe/Minsk +BZ +1730-08812 America/Belize +CA +4734-05243 America/St_Johns Newfoundland; Labrador (southeast) +CA +4439-06336 America/Halifax Atlantic - NS (most areas); PE +CA +4612-05957 America/Glace_Bay Atlantic - NS (Cape Breton) +CA +4606-06447 America/Moncton Atlantic - New Brunswick +CA +5320-06025 America/Goose_Bay Atlantic - Labrador (most areas) +CA +5125-05707 America/Blanc-Sablon AST - QC (Lower North Shore) +CA +4339-07923 America/Toronto Eastern - ON, QC (most areas) +CA +4901-08816 America/Nipigon Eastern - ON, QC (no DST 1967-73) +CA +4823-08915 America/Thunder_Bay Eastern - ON (Thunder Bay) +CA +6344-06828 America/Iqaluit Eastern - NU (most east areas) +CA +6608-06544 America/Pangnirtung Eastern - NU (Pangnirtung) +CA +484531-0913718 America/Atikokan EST - ON (Atikokan); NU (Coral H) +CA +4953-09709 America/Winnipeg Central - ON (west); Manitoba +CA +4843-09434 America/Rainy_River Central - ON (Rainy R, Ft Frances) +CA +744144-0944945 America/Resolute Central - NU (Resolute) +CA +624900-0920459 America/Rankin_Inlet Central - NU (central) +CA +5024-10439 America/Regina CST - SK (most areas) +CA +5017-10750 America/Swift_Current CST - SK (midwest) +CA +5333-11328 America/Edmonton Mountain - AB; BC (E); SK (W) +CA +690650-1050310 America/Cambridge_Bay Mountain - NU (west) +CA +6227-11421 America/Yellowknife Mountain - NT (central) +CA +682059-1334300 America/Inuvik Mountain - NT (west) +CA +4906-11631 America/Creston MST - BC (Creston) +CA +5946-12014 America/Dawson_Creek MST - BC (Dawson Cr, Ft St John) +CA +5848-12242 America/Fort_Nelson MST - BC (Ft Nelson) +CA +4916-12307 America/Vancouver Pacific - BC (most areas) +CA +6043-13503 America/Whitehorse Pacific - Yukon (east) +CA +6404-13925 America/Dawson Pacific - Yukon (west) +CC -1210+09655 Indian/Cocos +CH,DE,LI +4723+00832 Europe/Zurich Swiss time +CI,BF,GM,GN,ML,MR,SH,SL,SN,TG +0519-00402 Africa/Abidjan +CK -2114-15946 Pacific/Rarotonga +CL -3327-07040 America/Santiago Chile (most areas) +CL -5309-07055 America/Punta_Arenas Region of Magallanes +CL -2709-10926 Pacific/Easter Easter Island +CN +3114+12128 Asia/Shanghai Beijing Time +CN +4348+08735 Asia/Urumqi Xinjiang Time +CO +0436-07405 America/Bogota +CR +0956-08405 America/Costa_Rica +CU +2308-08222 America/Havana +CV +1455-02331 Atlantic/Cape_Verde +CW,AW,BQ,SX +1211-06900 America/Curacao +CX -1025+10543 Indian/Christmas +CY +3510+03322 Asia/Nicosia Cyprus (most areas) +CY +3507+03357 Asia/Famagusta Northern Cyprus +CZ,SK +5005+01426 Europe/Prague +DE +5230+01322 Europe/Berlin Germany (most areas) +DK +5540+01235 Europe/Copenhagen +DO +1828-06954 America/Santo_Domingo +DZ +3647+00303 Africa/Algiers +EC -0210-07950 America/Guayaquil Ecuador (mainland) +EC -0054-08936 Pacific/Galapagos Galápagos Islands +EE +5925+02445 Europe/Tallinn +EG +3003+03115 Africa/Cairo +EH +2709-01312 Africa/El_Aaiun +ES +4024-00341 Europe/Madrid Spain (mainland) +ES +3553-00519 Africa/Ceuta Ceuta, Melilla +ES +2806-01524 Atlantic/Canary Canary Islands +FI,AX +6010+02458 Europe/Helsinki +FJ -1808+17825 Pacific/Fiji +FK -5142-05751 Atlantic/Stanley +FM +0725+15147 Pacific/Chuuk Chuuk/Truk, Yap +FM +0658+15813 Pacific/Pohnpei Pohnpei/Ponape +FM +0519+16259 Pacific/Kosrae Kosrae +FO +6201-00646 Atlantic/Faroe +FR +4852+00220 Europe/Paris +GB,GG,IM,JE +513030-0000731 Europe/London +GE +4143+04449 Asia/Tbilisi +GF +0456-05220 America/Cayenne +GH +0533-00013 Africa/Accra +GI +3608-00521 Europe/Gibraltar +GL +6411-05144 America/Nuuk Greenland (most areas) +GL +7646-01840 America/Danmarkshavn National Park (east coast) +GL +7029-02158 America/Scoresbysund Scoresbysund/Ittoqqortoormiit +GL +7634-06847 America/Thule Thule/Pituffik +GR +3758+02343 Europe/Athens +GS -5416-03632 Atlantic/South_Georgia +GT +1438-09031 America/Guatemala +GU,MP +1328+14445 Pacific/Guam +GW +1151-01535 Africa/Bissau +GY +0648-05810 America/Guyana +HK +2217+11409 Asia/Hong_Kong +HN +1406-08713 America/Tegucigalpa +HT +1832-07220 America/Port-au-Prince +HU +4730+01905 Europe/Budapest +ID -0610+10648 Asia/Jakarta Java, Sumatra +ID -0002+10920 Asia/Pontianak Borneo (west, central) +ID -0507+11924 Asia/Makassar Borneo (east, south); Sulawesi/Celebes, Bali, Nusa Tengarra; Timor (west) +ID -0232+14042 Asia/Jayapura New Guinea (West Papua / Irian Jaya); Malukus/Moluccas +IE +5320-00615 Europe/Dublin +IL +314650+0351326 Asia/Jerusalem +IN +2232+08822 Asia/Kolkata +IO -0720+07225 Indian/Chagos +IQ +3321+04425 Asia/Baghdad +IR +3540+05126 Asia/Tehran +IS +6409-02151 Atlantic/Reykjavik +IT,SM,VA +4154+01229 Europe/Rome +JM +175805-0764736 America/Jamaica +JO +3157+03556 Asia/Amman +JP +353916+1394441 Asia/Tokyo +KE,DJ,ER,ET,KM,MG,SO,TZ,UG,YT -0117+03649 Africa/Nairobi +KG +4254+07436 Asia/Bishkek +KI +0125+17300 Pacific/Tarawa Gilbert Islands +KI -0308-17105 Pacific/Enderbury Phoenix Islands +KI +0152-15720 Pacific/Kiritimati Line Islands +KP +3901+12545 Asia/Pyongyang +KR +3733+12658 Asia/Seoul +KZ +4315+07657 Asia/Almaty Kazakhstan (most areas) +KZ +4448+06528 Asia/Qyzylorda Qyzylorda/Kyzylorda/Kzyl-Orda +KZ +5312+06337 Asia/Qostanay Qostanay/Kostanay/Kustanay +KZ +5017+05710 Asia/Aqtobe Aqtöbe/Aktobe +KZ +4431+05016 Asia/Aqtau Mangghystaū/Mankistau +KZ +4707+05156 Asia/Atyrau Atyraū/Atirau/Gur'yev +KZ +5113+05121 Asia/Oral West Kazakhstan +LB +3353+03530 Asia/Beirut +LK +0656+07951 Asia/Colombo +LR +0618-01047 Africa/Monrovia +LT +5441+02519 Europe/Vilnius +LU +4936+00609 Europe/Luxembourg +LV +5657+02406 Europe/Riga +LY +3254+01311 Africa/Tripoli +MA +3339-00735 Africa/Casablanca +MC +4342+00723 Europe/Monaco +MD +4700+02850 Europe/Chisinau +MH +0709+17112 Pacific/Majuro Marshall Islands (most areas) +MH +0905+16720 Pacific/Kwajalein Kwajalein +MM +1647+09610 Asia/Yangon +MN +4755+10653 Asia/Ulaanbaatar Mongolia (most areas) +MN +4801+09139 Asia/Hovd Bayan-Ölgii, Govi-Altai, Hovd, Uvs, Zavkhan +MN +4804+11430 Asia/Choibalsan Dornod, Sükhbaatar +MO +221150+1133230 Asia/Macau +MQ +1436-06105 America/Martinique +MT +3554+01431 Europe/Malta +MU -2010+05730 Indian/Mauritius +MV +0410+07330 Indian/Maldives +MX +1924-09909 America/Mexico_City Central Time +MX +2105-08646 America/Cancun Eastern Standard Time - Quintana Roo +MX +2058-08937 America/Merida Central Time - Campeche, Yucatán +MX +2540-10019 America/Monterrey Central Time - Durango; Coahuila, Nuevo León, Tamaulipas (most areas) +MX +2550-09730 America/Matamoros Central Time US - Coahuila, Nuevo León, Tamaulipas (US border) +MX +2313-10625 America/Mazatlan Mountain Time - Baja California Sur, Nayarit, Sinaloa +MX +2838-10605 America/Chihuahua Mountain Time - Chihuahua (most areas) +MX +2934-10425 America/Ojinaga Mountain Time US - Chihuahua (US border) +MX +2904-11058 America/Hermosillo Mountain Standard Time - Sonora +MX +3232-11701 America/Tijuana Pacific Time US - Baja California +MX +2048-10515 America/Bahia_Banderas Central Time - Bahía de Banderas +MY +0310+10142 Asia/Kuala_Lumpur Malaysia (peninsula) +MY +0133+11020 Asia/Kuching Sabah, Sarawak +MZ,BI,BW,CD,MW,RW,ZM,ZW -2558+03235 Africa/Maputo Central Africa Time +NA -2234+01706 Africa/Windhoek +NC -2216+16627 Pacific/Noumea +NF -2903+16758 Pacific/Norfolk +NG,AO,BJ,CD,CF,CG,CM,GA,GQ,NE +0627+00324 Africa/Lagos West Africa Time +NI +1209-08617 America/Managua +NL +5222+00454 Europe/Amsterdam +NO,SJ +5955+01045 Europe/Oslo +NP +2743+08519 Asia/Kathmandu +NR -0031+16655 Pacific/Nauru +NU -1901-16955 Pacific/Niue +NZ,AQ -3652+17446 Pacific/Auckland New Zealand time +NZ -4357-17633 Pacific/Chatham Chatham Islands +PA,KY +0858-07932 America/Panama +PE -1203-07703 America/Lima +PF -1732-14934 Pacific/Tahiti Society Islands +PF -0900-13930 Pacific/Marquesas Marquesas Islands +PF -2308-13457 Pacific/Gambier Gambier Islands +PG -0930+14710 Pacific/Port_Moresby Papua New Guinea (most areas) +PG -0613+15534 Pacific/Bougainville Bougainville +PH +1435+12100 Asia/Manila +PK +2452+06703 Asia/Karachi +PL +5215+02100 Europe/Warsaw +PM +4703-05620 America/Miquelon +PN -2504-13005 Pacific/Pitcairn +PR +182806-0660622 America/Puerto_Rico +PS +3130+03428 Asia/Gaza Gaza Strip +PS +313200+0350542 Asia/Hebron West Bank +PT +3843-00908 Europe/Lisbon Portugal (mainland) +PT +3238-01654 Atlantic/Madeira Madeira Islands +PT +3744-02540 Atlantic/Azores Azores +PW +0720+13429 Pacific/Palau +PY -2516-05740 America/Asuncion +QA,BH +2517+05132 Asia/Qatar +RE,TF -2052+05528 Indian/Reunion Réunion, Crozet, Scattered Islands +RO +4426+02606 Europe/Bucharest +RS,BA,HR,ME,MK,SI +4450+02030 Europe/Belgrade +RU +5443+02030 Europe/Kaliningrad MSK-01 - Kaliningrad +RU +554521+0373704 Europe/Moscow MSK+00 - Moscow area +# Mention RU and UA alphabetically. See "territorial claims" above. +RU,UA +4457+03406 Europe/Simferopol Crimea +RU +5836+04939 Europe/Kirov MSK+00 - Kirov +RU +4621+04803 Europe/Astrakhan MSK+01 - Astrakhan +RU +4844+04425 Europe/Volgograd MSK+01 - Volgograd +RU +5134+04602 Europe/Saratov MSK+01 - Saratov +RU +5420+04824 Europe/Ulyanovsk MSK+01 - Ulyanovsk +RU +5312+05009 Europe/Samara MSK+01 - Samara, Udmurtia +RU +5651+06036 Asia/Yekaterinburg MSK+02 - Urals +RU +5500+07324 Asia/Omsk MSK+03 - Omsk +RU +5502+08255 Asia/Novosibirsk MSK+04 - Novosibirsk +RU +5322+08345 Asia/Barnaul MSK+04 - Altai +RU +5630+08458 Asia/Tomsk MSK+04 - Tomsk +RU +5345+08707 Asia/Novokuznetsk MSK+04 - Kemerovo +RU +5601+09250 Asia/Krasnoyarsk MSK+04 - Krasnoyarsk area +RU +5216+10420 Asia/Irkutsk MSK+05 - Irkutsk, Buryatia +RU +5203+11328 Asia/Chita MSK+06 - Zabaykalsky +RU +6200+12940 Asia/Yakutsk MSK+06 - Lena River +RU +623923+1353314 Asia/Khandyga MSK+06 - Tomponsky, Ust-Maysky +RU +4310+13156 Asia/Vladivostok MSK+07 - Amur River +RU +643337+1431336 Asia/Ust-Nera MSK+07 - Oymyakonsky +RU +5934+15048 Asia/Magadan MSK+08 - Magadan +RU +4658+14242 Asia/Sakhalin MSK+08 - Sakhalin Island +RU +6728+15343 Asia/Srednekolymsk MSK+08 - Sakha (E); North Kuril Is +RU +5301+15839 Asia/Kamchatka MSK+09 - Kamchatka +RU +6445+17729 Asia/Anadyr MSK+09 - Bering Sea +SA,KW,YE +2438+04643 Asia/Riyadh +SB -0932+16012 Pacific/Guadalcanal +SC -0440+05528 Indian/Mahe +SD +1536+03232 Africa/Khartoum +SE +5920+01803 Europe/Stockholm +SG +0117+10351 Asia/Singapore +SR +0550-05510 America/Paramaribo +SS +0451+03137 Africa/Juba +ST +0020+00644 Africa/Sao_Tome +SV +1342-08912 America/El_Salvador +SY +3330+03618 Asia/Damascus +TC +2128-07108 America/Grand_Turk +TD +1207+01503 Africa/Ndjamena +TF -492110+0701303 Indian/Kerguelen Kerguelen, St Paul Island, Amsterdam Island +TH,KH,LA,VN +1345+10031 Asia/Bangkok Indochina (most areas) +TJ +3835+06848 Asia/Dushanbe +TK -0922-17114 Pacific/Fakaofo +TL -0833+12535 Asia/Dili +TM +3757+05823 Asia/Ashgabat +TN +3648+01011 Africa/Tunis +TO -2110-17510 Pacific/Tongatapu +TR +4101+02858 Europe/Istanbul +TT,AG,AI,BL,DM,GD,GP,KN,LC,MF,MS,VC,VG,VI +1039-06131 America/Port_of_Spain +TV -0831+17913 Pacific/Funafuti +TW +2503+12130 Asia/Taipei +UA +5026+03031 Europe/Kiev Ukraine (most areas) +UA +4837+02218 Europe/Uzhgorod Transcarpathia +UA +4750+03510 Europe/Zaporozhye Zaporozhye and east Lugansk +UM +1917+16637 Pacific/Wake Wake Island +US +404251-0740023 America/New_York Eastern (most areas) +US +421953-0830245 America/Detroit Eastern - MI (most areas) +US +381515-0854534 America/Kentucky/Louisville Eastern - KY (Louisville area) +US +364947-0845057 America/Kentucky/Monticello Eastern - KY (Wayne) +US +394606-0860929 America/Indiana/Indianapolis Eastern - IN (most areas) +US +384038-0873143 America/Indiana/Vincennes Eastern - IN (Da, Du, K, Mn) +US +410305-0863611 America/Indiana/Winamac Eastern - IN (Pulaski) +US +382232-0862041 America/Indiana/Marengo Eastern - IN (Crawford) +US +382931-0871643 America/Indiana/Petersburg Eastern - IN (Pike) +US +384452-0850402 America/Indiana/Vevay Eastern - IN (Switzerland) +US +415100-0873900 America/Chicago Central (most areas) +US +375711-0864541 America/Indiana/Tell_City Central - IN (Perry) +US +411745-0863730 America/Indiana/Knox Central - IN (Starke) +US +450628-0873651 America/Menominee Central - MI (Wisconsin border) +US +470659-1011757 America/North_Dakota/Center Central - ND (Oliver) +US +465042-1012439 America/North_Dakota/New_Salem Central - ND (Morton rural) +US +471551-1014640 America/North_Dakota/Beulah Central - ND (Mercer) +US +394421-1045903 America/Denver Mountain (most areas) +US +433649-1161209 America/Boise Mountain - ID (south); OR (east) +US +332654-1120424 America/Phoenix MST - Arizona (except Navajo) +US +340308-1181434 America/Los_Angeles Pacific +US +611305-1495401 America/Anchorage Alaska (most areas) +US +581807-1342511 America/Juneau Alaska - Juneau area +US +571035-1351807 America/Sitka Alaska - Sitka area +US +550737-1313435 America/Metlakatla Alaska - Annette Island +US +593249-1394338 America/Yakutat Alaska - Yakutat +US +643004-1652423 America/Nome Alaska (west) +US +515248-1763929 America/Adak Aleutian Islands +US,UM +211825-1575130 Pacific/Honolulu Hawaii +UY -345433-0561245 America/Montevideo +UZ +3940+06648 Asia/Samarkand Uzbekistan (west) +UZ +4120+06918 Asia/Tashkent Uzbekistan (east) +VE +1030-06656 America/Caracas +VN +1045+10640 Asia/Ho_Chi_Minh Vietnam (south) +VU -1740+16825 Pacific/Efate +WF -1318-17610 Pacific/Wallis +WS -1350-17144 Pacific/Apia +ZA,LS,SZ -2615+02800 Africa/Johannesburg diff --git a/third_party/abseil_cpp/absl/time/internal/get_current_time_chrono.inc b/third_party/abseil_cpp/absl/time/internal/get_current_time_chrono.inc new file mode 100644 index 000000000000..5eeb6406aaf3 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/get_current_time_chrono.inc @@ -0,0 +1,31 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <chrono> +#include <cstdint> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { + +static int64_t GetCurrentTimeNanosFromSystem() { + return std::chrono::duration_cast<std::chrono::nanoseconds>( + std::chrono::system_clock::now() - + std::chrono::system_clock::from_time_t(0)) + .count(); +} + +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/get_current_time_posix.inc b/third_party/abseil_cpp/absl/time/internal/get_current_time_posix.inc new file mode 100644 index 000000000000..42072000ae3c --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/get_current_time_posix.inc @@ -0,0 +1,24 @@ +#include "absl/time/clock.h" + +#include <sys/time.h> +#include <ctime> +#include <cstdint> + +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { + +static int64_t GetCurrentTimeNanosFromSystem() { + const int64_t kNanosPerSecond = 1000 * 1000 * 1000; + struct timespec ts; + ABSL_RAW_CHECK(clock_gettime(CLOCK_REALTIME, &ts) == 0, + "Failed to read real-time clock."); + return (int64_t{ts.tv_sec} * kNanosPerSecond + + int64_t{ts.tv_nsec}); +} + +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/test_util.cc b/third_party/abseil_cpp/absl/time/internal/test_util.cc new file mode 100644 index 000000000000..9a485a0750e4 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/test_util.cc @@ -0,0 +1,131 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/internal/test_util.h" + +#include <algorithm> +#include <cstddef> +#include <cstring> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/time/internal/cctz/include/cctz/zone_info_source.h" + +namespace cctz = absl::time_internal::cctz; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { + +TimeZone LoadTimeZone(const std::string& name) { + TimeZone tz; + ABSL_RAW_CHECK(LoadTimeZone(name, &tz), name.c_str()); + return tz; +} + +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { +namespace cctz_extension { +namespace { + +// Embed the zoneinfo data for time zones used during tests and benchmarks. +// The data was generated using "xxd -i zoneinfo-file". There is no need +// to update the data as long as the tests do not depend on recent changes +// (and the past rules remain the same). +#include "absl/time/internal/zoneinfo.inc" + +const struct ZoneInfo { + const char* name; + const char* data; + std::size_t length; +} kZoneInfo[] = { + // The three real time zones used by :time_test and :time_benchmark. + {"America/Los_Angeles", // + reinterpret_cast<char*>(America_Los_Angeles), America_Los_Angeles_len}, + {"America/New_York", // + reinterpret_cast<char*>(America_New_York), America_New_York_len}, + {"Australia/Sydney", // + reinterpret_cast<char*>(Australia_Sydney), Australia_Sydney_len}, + + // Other zones named in tests but which should fail to load. + {"Invalid/TimeZone", nullptr, 0}, + {"", nullptr, 0}, + + // Also allow for loading the local time zone under TZ=US/Pacific. + {"US/Pacific", // + reinterpret_cast<char*>(America_Los_Angeles), America_Los_Angeles_len}, + + // Allows use of the local time zone from a system-specific location. +#ifdef _MSC_VER + {"localtime", // + reinterpret_cast<char*>(America_Los_Angeles), America_Los_Angeles_len}, +#else + {"/etc/localtime", // + reinterpret_cast<char*>(America_Los_Angeles), America_Los_Angeles_len}, +#endif +}; + +class TestZoneInfoSource : public cctz::ZoneInfoSource { + public: + TestZoneInfoSource(const char* data, std::size_t size) + : data_(data), end_(data + size) {} + + std::size_t Read(void* ptr, std::size_t size) override { + const std::size_t len = std::min<std::size_t>(size, end_ - data_); + memcpy(ptr, data_, len); + data_ += len; + return len; + } + + int Skip(std::size_t offset) override { + data_ += std::min<std::size_t>(offset, end_ - data_); + return 0; + } + + private: + const char* data_; + const char* const end_; +}; + +std::unique_ptr<cctz::ZoneInfoSource> TestFactory( + const std::string& name, + const std::function<std::unique_ptr<cctz::ZoneInfoSource>( + const std::string& name)>& /*fallback_factory*/) { + for (const ZoneInfo& zoneinfo : kZoneInfo) { + if (name == zoneinfo.name) { + if (zoneinfo.data == nullptr) return nullptr; + return std::unique_ptr<cctz::ZoneInfoSource>( + new TestZoneInfoSource(zoneinfo.data, zoneinfo.length)); + } + } + ABSL_RAW_LOG(FATAL, "Unexpected time zone \"%s\" in test", name.c_str()); + return nullptr; +} + +} // namespace + +#if !defined(__MINGW32__) +// MinGW does not support the weak symbol extension mechanism. +ZoneInfoSourceFactory zone_info_source_factory = TestFactory; +#endif + +} // namespace cctz_extension +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/internal/test_util.h b/third_party/abseil_cpp/absl/time/internal/test_util.h new file mode 100644 index 000000000000..5c4bf1f680ee --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/test_util.h @@ -0,0 +1,33 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TIME_INTERNAL_TEST_UTIL_H_ +#define ABSL_TIME_INTERNAL_TEST_UTIL_H_ + +#include <string> + +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace time_internal { + +// Loads the named timezone, but dies on any failure. +absl::TimeZone LoadTimeZone(const std::string& name); + +} // namespace time_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_INTERNAL_TEST_UTIL_H_ diff --git a/third_party/abseil_cpp/absl/time/internal/zoneinfo.inc b/third_party/abseil_cpp/absl/time/internal/zoneinfo.inc new file mode 100644 index 000000000000..bfed82990dd5 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/internal/zoneinfo.inc @@ -0,0 +1,729 @@ +unsigned char America_Los_Angeles[] = { + 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xba, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0x80, 0x00, 0x00, 0x00, + 0x9e, 0xa6, 0x48, 0xa0, 0x9f, 0xbb, 0x15, 0x90, 0xa0, 0x86, 0x2a, 0xa0, + 0xa1, 0x9a, 0xf7, 0x90, 0xcb, 0x89, 0x1a, 0xa0, 0xd2, 0x23, 0xf4, 0x70, + 0xd2, 0x61, 0x26, 0x10, 0xd6, 0xfe, 0x74, 0x5c, 0xd8, 0x80, 0xad, 0x90, + 0xda, 0xfe, 0xc3, 0x90, 0xdb, 0xc0, 0x90, 0x10, 0xdc, 0xde, 0xa5, 0x90, + 0xdd, 0xa9, 0xac, 0x90, 0xde, 0xbe, 0x87, 0x90, 0xdf, 0x89, 0x8e, 0x90, + 0xe0, 0x9e, 0x69, 0x90, 0xe1, 0x69, 0x70, 0x90, 0xe2, 0x7e, 0x4b, 0x90, + 0xe3, 0x49, 0x52, 0x90, 0xe4, 0x5e, 0x2d, 0x90, 0xe5, 0x29, 0x34, 0x90, + 0xe6, 0x47, 0x4a, 0x10, 0xe7, 0x12, 0x51, 0x10, 0xe8, 0x27, 0x2c, 0x10, + 0xe8, 0xf2, 0x33, 0x10, 0xea, 0x07, 0x0e, 0x10, 0xea, 0xd2, 0x15, 0x10, + 0xeb, 0xe6, 0xf0, 0x10, 0xec, 0xb1, 0xf7, 0x10, 0xed, 0xc6, 0xd2, 0x10, + 0xee, 0x91, 0xd9, 0x10, 0xef, 0xaf, 0xee, 0x90, 0xf0, 0x71, 0xbb, 0x10, + 0xf1, 0x8f, 0xd0, 0x90, 0xf2, 0x7f, 0xc1, 0x90, 0xf3, 0x6f, 0xb2, 0x90, + 0xf4, 0x5f, 0xa3, 0x90, 0xf5, 0x4f, 0x94, 0x90, 0xf6, 0x3f, 0x85, 0x90, + 0xf7, 0x2f, 0x76, 0x90, 0xf8, 0x28, 0xa2, 0x10, 0xf9, 0x0f, 0x58, 0x90, + 0xfa, 0x08, 0x84, 0x10, 0xfa, 0xf8, 0x83, 0x20, 0xfb, 0xe8, 0x66, 0x10, + 0xfc, 0xd8, 0x65, 0x20, 0xfd, 0xc8, 0x48, 0x10, 0xfe, 0xb8, 0x47, 0x20, + 0xff, 0xa8, 0x2a, 0x10, 0x00, 0x98, 0x29, 0x20, 0x01, 0x88, 0x0c, 0x10, + 0x02, 0x78, 0x0b, 0x20, 0x03, 0x71, 0x28, 0x90, 0x04, 0x61, 0x27, 0xa0, + 0x05, 0x51, 0x0a, 0x90, 0x06, 0x41, 0x09, 0xa0, 0x07, 0x30, 0xec, 0x90, + 0x07, 0x8d, 0x43, 0xa0, 0x09, 0x10, 0xce, 0x90, 0x09, 0xad, 0xbf, 0x20, + 0x0a, 0xf0, 0xb0, 0x90, 0x0b, 0xe0, 0xaf, 0xa0, 0x0c, 0xd9, 0xcd, 0x10, + 0x0d, 0xc0, 0x91, 0xa0, 0x0e, 0xb9, 0xaf, 0x10, 0x0f, 0xa9, 0xae, 0x20, + 0x10, 0x99, 0x91, 0x10, 0x11, 0x89, 0x90, 0x20, 0x12, 0x79, 0x73, 0x10, + 0x13, 0x69, 0x72, 0x20, 0x14, 0x59, 0x55, 0x10, 0x15, 0x49, 0x54, 0x20, + 0x16, 0x39, 0x37, 0x10, 0x17, 0x29, 0x36, 0x20, 0x18, 0x22, 0x53, 0x90, + 0x19, 0x09, 0x18, 0x20, 0x1a, 0x02, 0x35, 0x90, 0x1a, 0xf2, 0x34, 0xa0, + 0x1b, 0xe2, 0x17, 0x90, 0x1c, 0xd2, 0x16, 0xa0, 0x1d, 0xc1, 0xf9, 0x90, + 0x1e, 0xb1, 0xf8, 0xa0, 0x1f, 0xa1, 0xdb, 0x90, 0x20, 0x76, 0x2b, 0x20, + 0x21, 0x81, 0xbd, 0x90, 0x22, 0x56, 0x0d, 0x20, 0x23, 0x6a, 0xda, 0x10, + 0x24, 0x35, 0xef, 0x20, 0x25, 0x4a, 0xbc, 0x10, 0x26, 0x15, 0xd1, 0x20, + 0x27, 0x2a, 0x9e, 0x10, 0x27, 0xfe, 0xed, 0xa0, 0x29, 0x0a, 0x80, 0x10, + 0x29, 0xde, 0xcf, 0xa0, 0x2a, 0xea, 0x62, 0x10, 0x2b, 0xbe, 0xb1, 0xa0, + 0x2c, 0xd3, 0x7e, 0x90, 0x2d, 0x9e, 0x93, 0xa0, 0x2e, 0xb3, 0x60, 0x90, + 0x2f, 0x7e, 0x75, 0xa0, 0x30, 0x93, 0x42, 0x90, 0x31, 0x67, 0x92, 0x20, + 0x32, 0x73, 0x24, 0x90, 0x33, 0x47, 0x74, 0x20, 0x34, 0x53, 0x06, 0x90, + 0x35, 0x27, 0x56, 0x20, 0x36, 0x32, 0xe8, 0x90, 0x37, 0x07, 0x38, 0x20, + 0x38, 0x1c, 0x05, 0x10, 0x38, 0xe7, 0x1a, 0x20, 0x39, 0xfb, 0xe7, 0x10, + 0x3a, 0xc6, 0xfc, 0x20, 0x3b, 0xdb, 0xc9, 0x10, 0x3c, 0xb0, 0x18, 0xa0, + 0x3d, 0xbb, 0xab, 0x10, 0x3e, 0x8f, 0xfa, 0xa0, 0x3f, 0x9b, 0x8d, 0x10, + 0x40, 0x6f, 0xdc, 0xa0, 0x41, 0x84, 0xa9, 0x90, 0x42, 0x4f, 0xbe, 0xa0, + 0x43, 0x64, 0x8b, 0x90, 0x44, 0x2f, 0xa0, 0xa0, 0x45, 0x44, 0x6d, 0x90, + 0x45, 0xf3, 0xd3, 0x20, 0x47, 0x2d, 0x8a, 0x10, 0x47, 0xd3, 0xb5, 0x20, + 0x49, 0x0d, 0x6c, 0x10, 0x49, 0xb3, 0x97, 0x20, 0x4a, 0xed, 0x4e, 0x10, + 0x4b, 0x9c, 0xb3, 0xa0, 0x4c, 0xd6, 0x6a, 0x90, 0x4d, 0x7c, 0x95, 0xa0, + 0x4e, 0xb6, 0x4c, 0x90, 0x4f, 0x5c, 0x77, 0xa0, 0x50, 0x96, 0x2e, 0x90, + 0x51, 0x3c, 0x59, 0xa0, 0x52, 0x76, 0x10, 0x90, 0x53, 0x1c, 0x3b, 0xa0, + 0x54, 0x55, 0xf2, 0x90, 0x54, 0xfc, 0x1d, 0xa0, 0x56, 0x35, 0xd4, 0x90, + 0x56, 0xe5, 0x3a, 0x20, 0x58, 0x1e, 0xf1, 0x10, 0x58, 0xc5, 0x1c, 0x20, + 0x59, 0xfe, 0xd3, 0x10, 0x5a, 0xa4, 0xfe, 0x20, 0x5b, 0xde, 0xb5, 0x10, + 0x5c, 0x84, 0xe0, 0x20, 0x5d, 0xbe, 0x97, 0x10, 0x5e, 0x64, 0xc2, 0x20, + 0x5f, 0x9e, 0x79, 0x10, 0x60, 0x4d, 0xde, 0xa0, 0x61, 0x87, 0x95, 0x90, + 0x62, 0x2d, 0xc0, 0xa0, 0x63, 0x67, 0x77, 0x90, 0x64, 0x0d, 0xa2, 0xa0, + 0x65, 0x47, 0x59, 0x90, 0x65, 0xed, 0x84, 0xa0, 0x67, 0x27, 0x3b, 0x90, + 0x67, 0xcd, 0x66, 0xa0, 0x69, 0x07, 0x1d, 0x90, 0x69, 0xad, 0x48, 0xa0, + 0x6a, 0xe6, 0xff, 0x90, 0x6b, 0x96, 0x65, 0x20, 0x6c, 0xd0, 0x1c, 0x10, + 0x6d, 0x76, 0x47, 0x20, 0x6e, 0xaf, 0xfe, 0x10, 0x6f, 0x56, 0x29, 0x20, + 0x70, 0x8f, 0xe0, 0x10, 0x71, 0x36, 0x0b, 0x20, 0x72, 0x6f, 0xc2, 0x10, + 0x73, 0x15, 0xed, 0x20, 0x74, 0x4f, 0xa4, 0x10, 0x74, 0xff, 0x09, 0xa0, + 0x76, 0x38, 0xc0, 0x90, 0x76, 0xde, 0xeb, 0xa0, 0x78, 0x18, 0xa2, 0x90, + 0x78, 0xbe, 0xcd, 0xa0, 0x79, 0xf8, 0x84, 0x90, 0x7a, 0x9e, 0xaf, 0xa0, + 0x7b, 0xd8, 0x66, 0x90, 0x7c, 0x7e, 0x91, 0xa0, 0x7d, 0xb8, 0x48, 0x90, + 0x7e, 0x5e, 0x73, 0xa0, 0x7f, 0x98, 0x2a, 0x90, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x03, 0x04, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0xff, 0xff, 0x91, 0x26, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x90, + 0x01, 0x04, 0xff, 0xff, 0x8f, 0x80, 0x00, 0x08, 0xff, 0xff, 0x9d, 0x90, + 0x01, 0x0c, 0xff, 0xff, 0x9d, 0x90, 0x01, 0x10, 0x4c, 0x4d, 0x54, 0x00, + 0x50, 0x44, 0x54, 0x00, 0x50, 0x53, 0x54, 0x00, 0x50, 0x57, 0x54, 0x00, + 0x50, 0x50, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xbb, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0xf8, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x5e, 0x04, + 0x1a, 0xc0, 0xff, 0xff, 0xff, 0xff, 0x9e, 0xa6, 0x48, 0xa0, 0xff, 0xff, + 0xff, 0xff, 0x9f, 0xbb, 0x15, 0x90, 0xff, 0xff, 0xff, 0xff, 0xa0, 0x86, + 0x2a, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xa1, 0x9a, 0xf7, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xcb, 0x89, 0x1a, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x23, + 0xf4, 0x70, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x61, 0x26, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xd6, 0xfe, 0x74, 0x5c, 0xff, 0xff, 0xff, 0xff, 0xd8, 0x80, + 0xad, 0x90, 0xff, 0xff, 0xff, 0xff, 0xda, 0xfe, 0xc3, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xdb, 0xc0, 0x90, 0x10, 0xff, 0xff, 0xff, 0xff, 0xdc, 0xde, + 0xa5, 0x90, 0xff, 0xff, 0xff, 0xff, 0xdd, 0xa9, 0xac, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xde, 0xbe, 0x87, 0x90, 0xff, 0xff, 0xff, 0xff, 0xdf, 0x89, + 0x8e, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x9e, 0x69, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xe1, 0x69, 0x70, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe2, 0x7e, + 0x4b, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x49, 0x52, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xe4, 0x5e, 0x2d, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x29, + 0x34, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe6, 0x47, 0x4a, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xe7, 0x12, 0x51, 0x10, 0xff, 0xff, 0xff, 0xff, 0xe8, 0x27, + 0x2c, 0x10, 0xff, 0xff, 0xff, 0xff, 0xe8, 0xf2, 0x33, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xea, 0x07, 0x0e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xea, 0xd2, + 0x15, 0x10, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xe6, 0xf0, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xec, 0xb1, 0xf7, 0x10, 0xff, 0xff, 0xff, 0xff, 0xed, 0xc6, + 0xd2, 0x10, 0xff, 0xff, 0xff, 0xff, 0xee, 0x91, 0xd9, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xef, 0xaf, 0xee, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x71, + 0xbb, 0x10, 0xff, 0xff, 0xff, 0xff, 0xf1, 0x8f, 0xd0, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xf2, 0x7f, 0xc1, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf3, 0x6f, + 0xb2, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf4, 0x5f, 0xa3, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xf5, 0x4f, 0x94, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf6, 0x3f, + 0x85, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf7, 0x2f, 0x76, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xf8, 0x28, 0xa2, 0x10, 0xff, 0xff, 0xff, 0xff, 0xf9, 0x0f, + 0x58, 0x90, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x08, 0x84, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xfa, 0xf8, 0x83, 0x20, 0xff, 0xff, 0xff, 0xff, 0xfb, 0xe8, + 0x66, 0x10, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xd8, 0x65, 0x20, 0xff, 0xff, + 0xff, 0xff, 0xfd, 0xc8, 0x48, 0x10, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xb8, + 0x47, 0x20, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa8, 0x2a, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x98, 0x29, 0x20, 0x00, 0x00, 0x00, 0x00, 0x01, 0x88, + 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x02, 0x78, 0x0b, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x03, 0x71, 0x28, 0x90, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, + 0x27, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x05, 0x51, 0x0a, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x06, 0x41, 0x09, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x07, 0x30, + 0xec, 0x90, 0x00, 0x00, 0x00, 0x00, 0x07, 0x8d, 0x43, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x09, 0x10, 0xce, 0x90, 0x00, 0x00, 0x00, 0x00, 0x09, 0xad, + 0xbf, 0x20, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xf0, 0xb0, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x0b, 0xe0, 0xaf, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xd9, + 0xcd, 0x10, 0x00, 0x00, 0x00, 0x00, 0x0d, 0xc0, 0x91, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x0e, 0xb9, 0xaf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xa9, + 0xae, 0x20, 0x00, 0x00, 0x00, 0x00, 0x10, 0x99, 0x91, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x11, 0x89, 0x90, 0x20, 0x00, 0x00, 0x00, 0x00, 0x12, 0x79, + 0x73, 0x10, 0x00, 0x00, 0x00, 0x00, 0x13, 0x69, 0x72, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x14, 0x59, 0x55, 0x10, 0x00, 0x00, 0x00, 0x00, 0x15, 0x49, + 0x54, 0x20, 0x00, 0x00, 0x00, 0x00, 0x16, 0x39, 0x37, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x17, 0x29, 0x36, 0x20, 0x00, 0x00, 0x00, 0x00, 0x18, 0x22, + 0x53, 0x90, 0x00, 0x00, 0x00, 0x00, 0x19, 0x09, 0x18, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x1a, 0x02, 0x35, 0x90, 0x00, 0x00, 0x00, 0x00, 0x1a, 0xf2, + 0x34, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe2, 0x17, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x1c, 0xd2, 0x16, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xc1, + 0xf9, 0x90, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xb1, 0xf8, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x1f, 0xa1, 0xdb, 0x90, 0x00, 0x00, 0x00, 0x00, 0x20, 0x76, + 0x2b, 0x20, 0x00, 0x00, 0x00, 0x00, 0x21, 0x81, 0xbd, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x22, 0x56, 0x0d, 0x20, 0x00, 0x00, 0x00, 0x00, 0x23, 0x6a, + 0xda, 0x10, 0x00, 0x00, 0x00, 0x00, 0x24, 0x35, 0xef, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x25, 0x4a, 0xbc, 0x10, 0x00, 0x00, 0x00, 0x00, 0x26, 0x15, + 0xd1, 0x20, 0x00, 0x00, 0x00, 0x00, 0x27, 0x2a, 0x9e, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x27, 0xfe, 0xed, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x29, 0x0a, + 0x80, 0x10, 0x00, 0x00, 0x00, 0x00, 0x29, 0xde, 0xcf, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x2a, 0xea, 0x62, 0x10, 0x00, 0x00, 0x00, 0x00, 0x2b, 0xbe, + 0xb1, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd3, 0x7e, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x2d, 0x9e, 0x93, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x2e, 0xb3, + 0x60, 0x90, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x7e, 0x75, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x30, 0x93, 0x42, 0x90, 0x00, 0x00, 0x00, 0x00, 0x31, 0x67, + 0x92, 0x20, 0x00, 0x00, 0x00, 0x00, 0x32, 0x73, 0x24, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x33, 0x47, 0x74, 0x20, 0x00, 0x00, 0x00, 0x00, 0x34, 0x53, + 0x06, 0x90, 0x00, 0x00, 0x00, 0x00, 0x35, 0x27, 0x56, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x36, 0x32, 0xe8, 0x90, 0x00, 0x00, 0x00, 0x00, 0x37, 0x07, + 0x38, 0x20, 0x00, 0x00, 0x00, 0x00, 0x38, 0x1c, 0x05, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x38, 0xe7, 0x1a, 0x20, 0x00, 0x00, 0x00, 0x00, 0x39, 0xfb, + 0xe7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x3a, 0xc6, 0xfc, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x3b, 0xdb, 0xc9, 0x10, 0x00, 0x00, 0x00, 0x00, 0x3c, 0xb0, + 0x18, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xbb, 0xab, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x3e, 0x8f, 0xfa, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x9b, + 0x8d, 0x10, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0xdc, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x41, 0x84, 0xa9, 0x90, 0x00, 0x00, 0x00, 0x00, 0x42, 0x4f, + 0xbe, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x43, 0x64, 0x8b, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x44, 0x2f, 0xa0, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x45, 0x44, + 0x6d, 0x90, 0x00, 0x00, 0x00, 0x00, 0x45, 0xf3, 0xd3, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x47, 0x2d, 0x8a, 0x10, 0x00, 0x00, 0x00, 0x00, 0x47, 0xd3, + 0xb5, 0x20, 0x00, 0x00, 0x00, 0x00, 0x49, 0x0d, 0x6c, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x49, 0xb3, 0x97, 0x20, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xed, + 0x4e, 0x10, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x9c, 0xb3, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x4c, 0xd6, 0x6a, 0x90, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x7c, + 0x95, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x4e, 0xb6, 0x4c, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x4f, 0x5c, 0x77, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x50, 0x96, + 0x2e, 0x90, 0x00, 0x00, 0x00, 0x00, 0x51, 0x3c, 0x59, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x52, 0x76, 0x10, 0x90, 0x00, 0x00, 0x00, 0x00, 0x53, 0x1c, + 0x3b, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0xf2, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x54, 0xfc, 0x1d, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x56, 0x35, + 0xd4, 0x90, 0x00, 0x00, 0x00, 0x00, 0x56, 0xe5, 0x3a, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x58, 0x1e, 0xf1, 0x10, 0x00, 0x00, 0x00, 0x00, 0x58, 0xc5, + 0x1c, 0x20, 0x00, 0x00, 0x00, 0x00, 0x59, 0xfe, 0xd3, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x5a, 0xa4, 0xfe, 0x20, 0x00, 0x00, 0x00, 0x00, 0x5b, 0xde, + 0xb5, 0x10, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x84, 0xe0, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x5d, 0xbe, 0x97, 0x10, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x64, + 0xc2, 0x20, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x9e, 0x79, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x60, 0x4d, 0xde, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x61, 0x87, + 0x95, 0x90, 0x00, 0x00, 0x00, 0x00, 0x62, 0x2d, 0xc0, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x63, 0x67, 0x77, 0x90, 0x00, 0x00, 0x00, 0x00, 0x64, 0x0d, + 0xa2, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x65, 0x47, 0x59, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x65, 0xed, 0x84, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x67, 0x27, + 0x3b, 0x90, 0x00, 0x00, 0x00, 0x00, 0x67, 0xcd, 0x66, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x69, 0x07, 0x1d, 0x90, 0x00, 0x00, 0x00, 0x00, 0x69, 0xad, + 0x48, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x6a, 0xe6, 0xff, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x6b, 0x96, 0x65, 0x20, 0x00, 0x00, 0x00, 0x00, 0x6c, 0xd0, + 0x1c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x76, 0x47, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x6e, 0xaf, 0xfe, 0x10, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x56, + 0x29, 0x20, 0x00, 0x00, 0x00, 0x00, 0x70, 0x8f, 0xe0, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x71, 0x36, 0x0b, 0x20, 0x00, 0x00, 0x00, 0x00, 0x72, 0x6f, + 0xc2, 0x10, 0x00, 0x00, 0x00, 0x00, 0x73, 0x15, 0xed, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x74, 0x4f, 0xa4, 0x10, 0x00, 0x00, 0x00, 0x00, 0x74, 0xff, + 0x09, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x76, 0x38, 0xc0, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x76, 0xde, 0xeb, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x78, 0x18, + 0xa2, 0x90, 0x00, 0x00, 0x00, 0x00, 0x78, 0xbe, 0xcd, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x79, 0xf8, 0x84, 0x90, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x9e, + 0xaf, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x7b, 0xd8, 0x66, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x7c, 0x7e, 0x91, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x7d, 0xb8, + 0x48, 0x90, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x5e, 0x73, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x7f, 0x98, 0x2a, 0x90, 0x00, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x03, 0x04, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0xff, 0xff, 0x91, 0x26, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x90, 0x01, + 0x04, 0xff, 0xff, 0x8f, 0x80, 0x00, 0x08, 0xff, 0xff, 0x9d, 0x90, 0x01, + 0x0c, 0xff, 0xff, 0x9d, 0x90, 0x01, 0x10, 0x4c, 0x4d, 0x54, 0x00, 0x50, + 0x44, 0x54, 0x00, 0x50, 0x53, 0x54, 0x00, 0x50, 0x57, 0x54, 0x00, 0x50, + 0x50, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x0a, 0x50, 0x53, 0x54, 0x38, 0x50, 0x44, 0x54, 0x2c, 0x4d, 0x33, + 0x2e, 0x32, 0x2e, 0x30, 0x2c, 0x4d, 0x31, 0x31, 0x2e, 0x31, 0x2e, 0x30, + 0x0a +}; +unsigned int America_Los_Angeles_len = 2845; +unsigned char America_New_York[] = { + 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0x80, 0x00, 0x00, 0x00, + 0x9e, 0xa6, 0x1e, 0x70, 0x9f, 0xba, 0xeb, 0x60, 0xa0, 0x86, 0x00, 0x70, + 0xa1, 0x9a, 0xcd, 0x60, 0xa2, 0x65, 0xe2, 0x70, 0xa3, 0x83, 0xe9, 0xe0, + 0xa4, 0x6a, 0xae, 0x70, 0xa5, 0x35, 0xa7, 0x60, 0xa6, 0x53, 0xca, 0xf0, + 0xa7, 0x15, 0x89, 0x60, 0xa8, 0x33, 0xac, 0xf0, 0xa8, 0xfe, 0xa5, 0xe0, + 0xaa, 0x13, 0x8e, 0xf0, 0xaa, 0xde, 0x87, 0xe0, 0xab, 0xf3, 0x70, 0xf0, + 0xac, 0xbe, 0x69, 0xe0, 0xad, 0xd3, 0x52, 0xf0, 0xae, 0x9e, 0x4b, 0xe0, + 0xaf, 0xb3, 0x34, 0xf0, 0xb0, 0x7e, 0x2d, 0xe0, 0xb1, 0x9c, 0x51, 0x70, + 0xb2, 0x67, 0x4a, 0x60, 0xb3, 0x7c, 0x33, 0x70, 0xb4, 0x47, 0x2c, 0x60, + 0xb5, 0x5c, 0x15, 0x70, 0xb6, 0x27, 0x0e, 0x60, 0xb7, 0x3b, 0xf7, 0x70, + 0xb8, 0x06, 0xf0, 0x60, 0xb9, 0x1b, 0xd9, 0x70, 0xb9, 0xe6, 0xd2, 0x60, + 0xbb, 0x04, 0xf5, 0xf0, 0xbb, 0xc6, 0xb4, 0x60, 0xbc, 0xe4, 0xd7, 0xf0, + 0xbd, 0xaf, 0xd0, 0xe0, 0xbe, 0xc4, 0xb9, 0xf0, 0xbf, 0x8f, 0xb2, 0xe0, + 0xc0, 0xa4, 0x9b, 0xf0, 0xc1, 0x6f, 0x94, 0xe0, 0xc2, 0x84, 0x7d, 0xf0, + 0xc3, 0x4f, 0x76, 0xe0, 0xc4, 0x64, 0x5f, 0xf0, 0xc5, 0x2f, 0x58, 0xe0, + 0xc6, 0x4d, 0x7c, 0x70, 0xc7, 0x0f, 0x3a, 0xe0, 0xc8, 0x2d, 0x5e, 0x70, + 0xc8, 0xf8, 0x57, 0x60, 0xca, 0x0d, 0x40, 0x70, 0xca, 0xd8, 0x39, 0x60, + 0xcb, 0x88, 0xf0, 0x70, 0xd2, 0x23, 0xf4, 0x70, 0xd2, 0x60, 0xfb, 0xe0, + 0xd3, 0x75, 0xe4, 0xf0, 0xd4, 0x40, 0xdd, 0xe0, 0xd5, 0x55, 0xc6, 0xf0, + 0xd6, 0x20, 0xbf, 0xe0, 0xd7, 0x35, 0xa8, 0xf0, 0xd8, 0x00, 0xa1, 0xe0, + 0xd9, 0x15, 0x8a, 0xf0, 0xd9, 0xe0, 0x83, 0xe0, 0xda, 0xfe, 0xa7, 0x70, + 0xdb, 0xc0, 0x65, 0xe0, 0xdc, 0xde, 0x89, 0x70, 0xdd, 0xa9, 0x82, 0x60, + 0xde, 0xbe, 0x6b, 0x70, 0xdf, 0x89, 0x64, 0x60, 0xe0, 0x9e, 0x4d, 0x70, + 0xe1, 0x69, 0x46, 0x60, 0xe2, 0x7e, 0x2f, 0x70, 0xe3, 0x49, 0x28, 0x60, + 0xe4, 0x5e, 0x11, 0x70, 0xe5, 0x57, 0x2e, 0xe0, 0xe6, 0x47, 0x2d, 0xf0, + 0xe7, 0x37, 0x10, 0xe0, 0xe8, 0x27, 0x0f, 0xf0, 0xe9, 0x16, 0xf2, 0xe0, + 0xea, 0x06, 0xf1, 0xf0, 0xea, 0xf6, 0xd4, 0xe0, 0xeb, 0xe6, 0xd3, 0xf0, + 0xec, 0xd6, 0xb6, 0xe0, 0xed, 0xc6, 0xb5, 0xf0, 0xee, 0xbf, 0xd3, 0x60, + 0xef, 0xaf, 0xd2, 0x70, 0xf0, 0x9f, 0xb5, 0x60, 0xf1, 0x8f, 0xb4, 0x70, + 0xf2, 0x7f, 0x97, 0x60, 0xf3, 0x6f, 0x96, 0x70, 0xf4, 0x5f, 0x79, 0x60, + 0xf5, 0x4f, 0x78, 0x70, 0xf6, 0x3f, 0x5b, 0x60, 0xf7, 0x2f, 0x5a, 0x70, + 0xf8, 0x28, 0x77, 0xe0, 0xf9, 0x0f, 0x3c, 0x70, 0xfa, 0x08, 0x59, 0xe0, + 0xfa, 0xf8, 0x58, 0xf0, 0xfb, 0xe8, 0x3b, 0xe0, 0xfc, 0xd8, 0x3a, 0xf0, + 0xfd, 0xc8, 0x1d, 0xe0, 0xfe, 0xb8, 0x1c, 0xf0, 0xff, 0xa7, 0xff, 0xe0, + 0x00, 0x97, 0xfe, 0xf0, 0x01, 0x87, 0xe1, 0xe0, 0x02, 0x77, 0xe0, 0xf0, + 0x03, 0x70, 0xfe, 0x60, 0x04, 0x60, 0xfd, 0x70, 0x05, 0x50, 0xe0, 0x60, + 0x06, 0x40, 0xdf, 0x70, 0x07, 0x30, 0xc2, 0x60, 0x07, 0x8d, 0x19, 0x70, + 0x09, 0x10, 0xa4, 0x60, 0x09, 0xad, 0x94, 0xf0, 0x0a, 0xf0, 0x86, 0x60, + 0x0b, 0xe0, 0x85, 0x70, 0x0c, 0xd9, 0xa2, 0xe0, 0x0d, 0xc0, 0x67, 0x70, + 0x0e, 0xb9, 0x84, 0xe0, 0x0f, 0xa9, 0x83, 0xf0, 0x10, 0x99, 0x66, 0xe0, + 0x11, 0x89, 0x65, 0xf0, 0x12, 0x79, 0x48, 0xe0, 0x13, 0x69, 0x47, 0xf0, + 0x14, 0x59, 0x2a, 0xe0, 0x15, 0x49, 0x29, 0xf0, 0x16, 0x39, 0x0c, 0xe0, + 0x17, 0x29, 0x0b, 0xf0, 0x18, 0x22, 0x29, 0x60, 0x19, 0x08, 0xed, 0xf0, + 0x1a, 0x02, 0x0b, 0x60, 0x1a, 0xf2, 0x0a, 0x70, 0x1b, 0xe1, 0xed, 0x60, + 0x1c, 0xd1, 0xec, 0x70, 0x1d, 0xc1, 0xcf, 0x60, 0x1e, 0xb1, 0xce, 0x70, + 0x1f, 0xa1, 0xb1, 0x60, 0x20, 0x76, 0x00, 0xf0, 0x21, 0x81, 0x93, 0x60, + 0x22, 0x55, 0xe2, 0xf0, 0x23, 0x6a, 0xaf, 0xe0, 0x24, 0x35, 0xc4, 0xf0, + 0x25, 0x4a, 0x91, 0xe0, 0x26, 0x15, 0xa6, 0xf0, 0x27, 0x2a, 0x73, 0xe0, + 0x27, 0xfe, 0xc3, 0x70, 0x29, 0x0a, 0x55, 0xe0, 0x29, 0xde, 0xa5, 0x70, + 0x2a, 0xea, 0x37, 0xe0, 0x2b, 0xbe, 0x87, 0x70, 0x2c, 0xd3, 0x54, 0x60, + 0x2d, 0x9e, 0x69, 0x70, 0x2e, 0xb3, 0x36, 0x60, 0x2f, 0x7e, 0x4b, 0x70, + 0x30, 0x93, 0x18, 0x60, 0x31, 0x67, 0x67, 0xf0, 0x32, 0x72, 0xfa, 0x60, + 0x33, 0x47, 0x49, 0xf0, 0x34, 0x52, 0xdc, 0x60, 0x35, 0x27, 0x2b, 0xf0, + 0x36, 0x32, 0xbe, 0x60, 0x37, 0x07, 0x0d, 0xf0, 0x38, 0x1b, 0xda, 0xe0, + 0x38, 0xe6, 0xef, 0xf0, 0x39, 0xfb, 0xbc, 0xe0, 0x3a, 0xc6, 0xd1, 0xf0, + 0x3b, 0xdb, 0x9e, 0xe0, 0x3c, 0xaf, 0xee, 0x70, 0x3d, 0xbb, 0x80, 0xe0, + 0x3e, 0x8f, 0xd0, 0x70, 0x3f, 0x9b, 0x62, 0xe0, 0x40, 0x6f, 0xb2, 0x70, + 0x41, 0x84, 0x7f, 0x60, 0x42, 0x4f, 0x94, 0x70, 0x43, 0x64, 0x61, 0x60, + 0x44, 0x2f, 0x76, 0x70, 0x45, 0x44, 0x43, 0x60, 0x45, 0xf3, 0xa8, 0xf0, + 0x47, 0x2d, 0x5f, 0xe0, 0x47, 0xd3, 0x8a, 0xf0, 0x49, 0x0d, 0x41, 0xe0, + 0x49, 0xb3, 0x6c, 0xf0, 0x4a, 0xed, 0x23, 0xe0, 0x4b, 0x9c, 0x89, 0x70, + 0x4c, 0xd6, 0x40, 0x60, 0x4d, 0x7c, 0x6b, 0x70, 0x4e, 0xb6, 0x22, 0x60, + 0x4f, 0x5c, 0x4d, 0x70, 0x50, 0x96, 0x04, 0x60, 0x51, 0x3c, 0x2f, 0x70, + 0x52, 0x75, 0xe6, 0x60, 0x53, 0x1c, 0x11, 0x70, 0x54, 0x55, 0xc8, 0x60, + 0x54, 0xfb, 0xf3, 0x70, 0x56, 0x35, 0xaa, 0x60, 0x56, 0xe5, 0x0f, 0xf0, + 0x58, 0x1e, 0xc6, 0xe0, 0x58, 0xc4, 0xf1, 0xf0, 0x59, 0xfe, 0xa8, 0xe0, + 0x5a, 0xa4, 0xd3, 0xf0, 0x5b, 0xde, 0x8a, 0xe0, 0x5c, 0x84, 0xb5, 0xf0, + 0x5d, 0xbe, 0x6c, 0xe0, 0x5e, 0x64, 0x97, 0xf0, 0x5f, 0x9e, 0x4e, 0xe0, + 0x60, 0x4d, 0xb4, 0x70, 0x61, 0x87, 0x6b, 0x60, 0x62, 0x2d, 0x96, 0x70, + 0x63, 0x67, 0x4d, 0x60, 0x64, 0x0d, 0x78, 0x70, 0x65, 0x47, 0x2f, 0x60, + 0x65, 0xed, 0x5a, 0x70, 0x67, 0x27, 0x11, 0x60, 0x67, 0xcd, 0x3c, 0x70, + 0x69, 0x06, 0xf3, 0x60, 0x69, 0xad, 0x1e, 0x70, 0x6a, 0xe6, 0xd5, 0x60, + 0x6b, 0x96, 0x3a, 0xf0, 0x6c, 0xcf, 0xf1, 0xe0, 0x6d, 0x76, 0x1c, 0xf0, + 0x6e, 0xaf, 0xd3, 0xe0, 0x6f, 0x55, 0xfe, 0xf0, 0x70, 0x8f, 0xb5, 0xe0, + 0x71, 0x35, 0xe0, 0xf0, 0x72, 0x6f, 0x97, 0xe0, 0x73, 0x15, 0xc2, 0xf0, + 0x74, 0x4f, 0x79, 0xe0, 0x74, 0xfe, 0xdf, 0x70, 0x76, 0x38, 0x96, 0x60, + 0x76, 0xde, 0xc1, 0x70, 0x78, 0x18, 0x78, 0x60, 0x78, 0xbe, 0xa3, 0x70, + 0x79, 0xf8, 0x5a, 0x60, 0x7a, 0x9e, 0x85, 0x70, 0x7b, 0xd8, 0x3c, 0x60, + 0x7c, 0x7e, 0x67, 0x70, 0x7d, 0xb8, 0x1e, 0x60, 0x7e, 0x5e, 0x49, 0x70, + 0x7f, 0x98, 0x00, 0x60, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x03, 0x04, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0xff, 0xff, 0xba, 0x9e, 0x00, 0x00, 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x04, + 0xff, 0xff, 0xb9, 0xb0, 0x00, 0x08, 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x0c, + 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x10, 0x4c, 0x4d, 0x54, 0x00, 0x45, 0x44, + 0x54, 0x00, 0x45, 0x53, 0x54, 0x00, 0x45, 0x57, 0x54, 0x00, 0x45, 0x50, + 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0xf8, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x5e, 0x03, 0xf0, 0x90, + 0xff, 0xff, 0xff, 0xff, 0x9e, 0xa6, 0x1e, 0x70, 0xff, 0xff, 0xff, 0xff, + 0x9f, 0xba, 0xeb, 0x60, 0xff, 0xff, 0xff, 0xff, 0xa0, 0x86, 0x00, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xa1, 0x9a, 0xcd, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xa2, 0x65, 0xe2, 0x70, 0xff, 0xff, 0xff, 0xff, 0xa3, 0x83, 0xe9, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xa4, 0x6a, 0xae, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xa5, 0x35, 0xa7, 0x60, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x53, 0xca, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xa7, 0x15, 0x89, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xa8, 0x33, 0xac, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xa8, 0xfe, 0xa5, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xaa, 0x13, 0x8e, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xaa, 0xde, 0x87, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xab, 0xf3, 0x70, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xac, 0xbe, 0x69, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xad, 0xd3, 0x52, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xae, 0x9e, 0x4b, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xaf, 0xb3, 0x34, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xb0, 0x7e, 0x2d, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xb1, 0x9c, 0x51, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xb2, 0x67, 0x4a, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xb3, 0x7c, 0x33, 0x70, 0xff, 0xff, 0xff, 0xff, 0xb4, 0x47, 0x2c, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xb5, 0x5c, 0x15, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xb6, 0x27, 0x0e, 0x60, 0xff, 0xff, 0xff, 0xff, 0xb7, 0x3b, 0xf7, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xb8, 0x06, 0xf0, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xb9, 0x1b, 0xd9, 0x70, 0xff, 0xff, 0xff, 0xff, 0xb9, 0xe6, 0xd2, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xbb, 0x04, 0xf5, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xbb, 0xc6, 0xb4, 0x60, 0xff, 0xff, 0xff, 0xff, 0xbc, 0xe4, 0xd7, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xbd, 0xaf, 0xd0, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xbe, 0xc4, 0xb9, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x8f, 0xb2, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xc0, 0xa4, 0x9b, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xc1, 0x6f, 0x94, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x84, 0x7d, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xc3, 0x4f, 0x76, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xc4, 0x64, 0x5f, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xc5, 0x2f, 0x58, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xc6, 0x4d, 0x7c, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xc7, 0x0f, 0x3a, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xc8, 0x2d, 0x5e, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xc8, 0xf8, 0x57, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xca, 0x0d, 0x40, 0x70, 0xff, 0xff, 0xff, 0xff, 0xca, 0xd8, 0x39, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xcb, 0x88, 0xf0, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xd2, 0x23, 0xf4, 0x70, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x60, 0xfb, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xd3, 0x75, 0xe4, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xd4, 0x40, 0xdd, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xd5, 0x55, 0xc6, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xd6, 0x20, 0xbf, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xd7, 0x35, 0xa8, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xd8, 0x00, 0xa1, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xd9, 0x15, 0x8a, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xd9, 0xe0, 0x83, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xda, 0xfe, 0xa7, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xdb, 0xc0, 0x65, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xdc, 0xde, 0x89, 0x70, 0xff, 0xff, 0xff, 0xff, 0xdd, 0xa9, 0x82, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xde, 0xbe, 0x6b, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xdf, 0x89, 0x64, 0x60, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x9e, 0x4d, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xe1, 0x69, 0x46, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xe2, 0x7e, 0x2f, 0x70, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x49, 0x28, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xe4, 0x5e, 0x11, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xe5, 0x57, 0x2e, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xe6, 0x47, 0x2d, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xe7, 0x37, 0x10, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xe8, 0x27, 0x0f, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xe9, 0x16, 0xf2, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xea, 0x06, 0xf1, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xea, 0xf6, 0xd4, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xe6, 0xd3, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xec, 0xd6, 0xb6, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xed, 0xc6, 0xb5, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xee, 0xbf, 0xd3, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xef, 0xaf, 0xd2, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xf0, 0x9f, 0xb5, 0x60, 0xff, 0xff, 0xff, 0xff, 0xf1, 0x8f, 0xb4, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xf2, 0x7f, 0x97, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xf3, 0x6f, 0x96, 0x70, 0xff, 0xff, 0xff, 0xff, 0xf4, 0x5f, 0x79, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xf5, 0x4f, 0x78, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xf6, 0x3f, 0x5b, 0x60, 0xff, 0xff, 0xff, 0xff, 0xf7, 0x2f, 0x5a, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xf8, 0x28, 0x77, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xf9, 0x0f, 0x3c, 0x70, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x08, 0x59, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xfa, 0xf8, 0x58, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xfb, 0xe8, 0x3b, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xd8, 0x3a, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xfd, 0xc8, 0x1d, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xfe, 0xb8, 0x1c, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, 0xff, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xfe, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x01, 0x87, 0xe1, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x02, 0x77, 0xe0, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x70, 0xfe, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x60, 0xfd, 0x70, 0x00, 0x00, 0x00, 0x00, 0x05, 0x50, 0xe0, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x06, 0x40, 0xdf, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x30, 0xc2, 0x60, 0x00, 0x00, 0x00, 0x00, 0x07, 0x8d, 0x19, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x09, 0x10, 0xa4, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x09, 0xad, 0x94, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xf0, 0x86, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x0b, 0xe0, 0x85, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x0c, 0xd9, 0xa2, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x0d, 0xc0, 0x67, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0xb9, 0x84, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0xa9, 0x83, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x10, 0x99, 0x66, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x11, 0x89, 0x65, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x12, 0x79, 0x48, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x13, 0x69, 0x47, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x59, 0x2a, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x15, 0x49, 0x29, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x16, 0x39, 0x0c, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x17, 0x29, 0x0b, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x18, 0x22, 0x29, 0x60, 0x00, 0x00, 0x00, 0x00, 0x19, 0x08, 0xed, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x1a, 0x02, 0x0b, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x1a, 0xf2, 0x0a, 0x70, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe1, 0xed, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0xd1, 0xec, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0xc1, 0xcf, 0x60, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xb1, 0xce, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0xa1, 0xb1, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x76, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x21, 0x81, 0x93, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x22, 0x55, 0xe2, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x23, 0x6a, 0xaf, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x24, 0x35, 0xc4, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x25, 0x4a, 0x91, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x26, 0x15, 0xa6, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x27, 0x2a, 0x73, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x27, 0xfe, 0xc3, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x29, 0x0a, 0x55, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x29, 0xde, 0xa5, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x2a, 0xea, 0x37, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x2b, 0xbe, 0x87, 0x70, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd3, 0x54, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x2d, 0x9e, 0x69, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x2e, 0xb3, 0x36, 0x60, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x7e, 0x4b, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x93, 0x18, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x31, 0x67, 0x67, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x32, 0x72, 0xfa, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x33, 0x47, 0x49, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x34, 0x52, 0xdc, 0x60, 0x00, 0x00, 0x00, 0x00, 0x35, 0x27, 0x2b, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x36, 0x32, 0xbe, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x37, 0x07, 0x0d, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x38, 0x1b, 0xda, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x38, 0xe6, 0xef, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x39, 0xfb, 0xbc, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x3a, 0xc6, 0xd1, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x3b, 0xdb, 0x9e, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x3c, 0xaf, 0xee, 0x70, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xbb, 0x80, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x3e, 0x8f, 0xd0, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0x9b, 0x62, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0xb2, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x41, 0x84, 0x7f, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x42, 0x4f, 0x94, 0x70, 0x00, 0x00, 0x00, 0x00, 0x43, 0x64, 0x61, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x44, 0x2f, 0x76, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x45, 0x44, 0x43, 0x60, 0x00, 0x00, 0x00, 0x00, 0x45, 0xf3, 0xa8, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x47, 0x2d, 0x5f, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x47, 0xd3, 0x8a, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x49, 0x0d, 0x41, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x49, 0xb3, 0x6c, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x4a, 0xed, 0x23, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x9c, 0x89, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x4c, 0xd6, 0x40, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x4d, 0x7c, 0x6b, 0x70, 0x00, 0x00, 0x00, 0x00, 0x4e, 0xb6, 0x22, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x4f, 0x5c, 0x4d, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x96, 0x04, 0x60, 0x00, 0x00, 0x00, 0x00, 0x51, 0x3c, 0x2f, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x52, 0x75, 0xe6, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x1c, 0x11, 0x70, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0xc8, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x54, 0xfb, 0xf3, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x56, 0x35, 0xaa, 0x60, 0x00, 0x00, 0x00, 0x00, 0x56, 0xe5, 0x0f, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x58, 0x1e, 0xc6, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x58, 0xc4, 0xf1, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x59, 0xfe, 0xa8, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x5a, 0xa4, 0xd3, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x5b, 0xde, 0x8a, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x84, 0xb5, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x5d, 0xbe, 0x6c, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x5e, 0x64, 0x97, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x9e, 0x4e, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x4d, 0xb4, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x87, 0x6b, 0x60, 0x00, 0x00, 0x00, 0x00, 0x62, 0x2d, 0x96, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x63, 0x67, 0x4d, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x64, 0x0d, 0x78, 0x70, 0x00, 0x00, 0x00, 0x00, 0x65, 0x47, 0x2f, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x65, 0xed, 0x5a, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x27, 0x11, 0x60, 0x00, 0x00, 0x00, 0x00, 0x67, 0xcd, 0x3c, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x69, 0x06, 0xf3, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xad, 0x1e, 0x70, 0x00, 0x00, 0x00, 0x00, 0x6a, 0xe6, 0xd5, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x6b, 0x96, 0x3a, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x6c, 0xcf, 0xf1, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x76, 0x1c, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x6e, 0xaf, 0xd3, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0x55, 0xfe, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x70, 0x8f, 0xb5, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x35, 0xe0, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x72, 0x6f, 0x97, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x73, 0x15, 0xc2, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x74, 0x4f, 0x79, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x74, 0xfe, 0xdf, 0x70, 0x00, 0x00, 0x00, 0x00, 0x76, 0x38, 0x96, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x76, 0xde, 0xc1, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x78, 0x18, 0x78, 0x60, 0x00, 0x00, 0x00, 0x00, 0x78, 0xbe, 0xa3, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xf8, 0x5a, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x7a, 0x9e, 0x85, 0x70, 0x00, 0x00, 0x00, 0x00, 0x7b, 0xd8, 0x3c, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x7c, 0x7e, 0x67, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x7d, 0xb8, 0x1e, 0x60, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x5e, 0x49, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0x98, 0x00, 0x60, 0x00, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x03, 0x04, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0xff, 0xff, 0xba, 0x9e, 0x00, 0x00, 0xff, + 0xff, 0xc7, 0xc0, 0x01, 0x04, 0xff, 0xff, 0xb9, 0xb0, 0x00, 0x08, 0xff, + 0xff, 0xc7, 0xc0, 0x01, 0x0c, 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x10, 0x4c, + 0x4d, 0x54, 0x00, 0x45, 0x44, 0x54, 0x00, 0x45, 0x53, 0x54, 0x00, 0x45, + 0x57, 0x54, 0x00, 0x45, 0x50, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x0a, 0x45, 0x53, 0x54, 0x35, 0x45, 0x44, + 0x54, 0x2c, 0x4d, 0x33, 0x2e, 0x32, 0x2e, 0x30, 0x2c, 0x4d, 0x31, 0x31, + 0x2e, 0x31, 0x2e, 0x30, 0x0a +}; +unsigned int America_New_York_len = 3545; +unsigned char Australia_Sydney[] = { + 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x0e, 0x80, 0x00, 0x00, 0x00, + 0x9c, 0x4e, 0xa6, 0x9c, 0x9c, 0xbc, 0x20, 0xf0, 0xcb, 0x54, 0xb3, 0x00, + 0xcb, 0xc7, 0x57, 0x70, 0xcc, 0xb7, 0x56, 0x80, 0xcd, 0xa7, 0x39, 0x70, + 0xce, 0xa0, 0x73, 0x00, 0xcf, 0x87, 0x1b, 0x70, 0x03, 0x70, 0x39, 0x80, + 0x04, 0x0d, 0x1c, 0x00, 0x05, 0x50, 0x1b, 0x80, 0x05, 0xf6, 0x38, 0x80, + 0x07, 0x2f, 0xfd, 0x80, 0x07, 0xd6, 0x1a, 0x80, 0x09, 0x0f, 0xdf, 0x80, + 0x09, 0xb5, 0xfc, 0x80, 0x0a, 0xef, 0xc1, 0x80, 0x0b, 0x9f, 0x19, 0x00, + 0x0c, 0xd8, 0xde, 0x00, 0x0d, 0x7e, 0xfb, 0x00, 0x0e, 0xb8, 0xc0, 0x00, + 0x0f, 0x5e, 0xdd, 0x00, 0x10, 0x98, 0xa2, 0x00, 0x11, 0x3e, 0xbf, 0x00, + 0x12, 0x78, 0x84, 0x00, 0x13, 0x1e, 0xa1, 0x00, 0x14, 0x58, 0x66, 0x00, + 0x14, 0xfe, 0x83, 0x00, 0x16, 0x38, 0x48, 0x00, 0x17, 0x0c, 0x89, 0x80, + 0x18, 0x21, 0x64, 0x80, 0x18, 0xc7, 0x81, 0x80, 0x1a, 0x01, 0x46, 0x80, + 0x1a, 0xa7, 0x63, 0x80, 0x1b, 0xe1, 0x28, 0x80, 0x1c, 0x87, 0x45, 0x80, + 0x1d, 0xc1, 0x0a, 0x80, 0x1e, 0x79, 0x9c, 0x80, 0x1f, 0x97, 0xb2, 0x00, + 0x20, 0x59, 0x7e, 0x80, 0x21, 0x80, 0xce, 0x80, 0x22, 0x42, 0x9b, 0x00, + 0x23, 0x69, 0xeb, 0x00, 0x24, 0x22, 0x7d, 0x00, 0x25, 0x49, 0xcd, 0x00, + 0x25, 0xef, 0xea, 0x00, 0x27, 0x29, 0xaf, 0x00, 0x27, 0xcf, 0xcc, 0x00, + 0x29, 0x09, 0x91, 0x00, 0x29, 0xaf, 0xae, 0x00, 0x2a, 0xe9, 0x73, 0x00, + 0x2b, 0x98, 0xca, 0x80, 0x2c, 0xd2, 0x8f, 0x80, 0x2d, 0x78, 0xac, 0x80, + 0x2e, 0xb2, 0x71, 0x80, 0x2f, 0x58, 0x8e, 0x80, 0x30, 0x92, 0x53, 0x80, + 0x31, 0x5d, 0x5a, 0x80, 0x32, 0x72, 0x35, 0x80, 0x33, 0x3d, 0x3c, 0x80, + 0x34, 0x52, 0x17, 0x80, 0x35, 0x1d, 0x1e, 0x80, 0x36, 0x31, 0xf9, 0x80, + 0x36, 0xfd, 0x00, 0x80, 0x38, 0x1b, 0x16, 0x00, 0x38, 0xdc, 0xe2, 0x80, + 0x39, 0xa7, 0xe9, 0x80, 0x3a, 0xbc, 0xc4, 0x80, 0x3b, 0xda, 0xda, 0x00, + 0x3c, 0xa5, 0xe1, 0x00, 0x3d, 0xba, 0xbc, 0x00, 0x3e, 0x85, 0xc3, 0x00, + 0x3f, 0x9a, 0x9e, 0x00, 0x40, 0x65, 0xa5, 0x00, 0x41, 0x83, 0xba, 0x80, + 0x42, 0x45, 0x87, 0x00, 0x43, 0x63, 0x9c, 0x80, 0x44, 0x2e, 0xa3, 0x80, + 0x45, 0x43, 0x7e, 0x80, 0x46, 0x05, 0x4b, 0x00, 0x47, 0x23, 0x60, 0x80, + 0x47, 0xf7, 0xa2, 0x00, 0x48, 0xe7, 0x93, 0x00, 0x49, 0xd7, 0x84, 0x00, + 0x4a, 0xc7, 0x75, 0x00, 0x4b, 0xb7, 0x66, 0x00, 0x4c, 0xa7, 0x57, 0x00, + 0x4d, 0x97, 0x48, 0x00, 0x4e, 0x87, 0x39, 0x00, 0x4f, 0x77, 0x2a, 0x00, + 0x50, 0x70, 0x55, 0x80, 0x51, 0x60, 0x46, 0x80, 0x52, 0x50, 0x37, 0x80, + 0x53, 0x40, 0x28, 0x80, 0x54, 0x30, 0x19, 0x80, 0x55, 0x20, 0x0a, 0x80, + 0x56, 0x0f, 0xfb, 0x80, 0x56, 0xff, 0xec, 0x80, 0x57, 0xef, 0xdd, 0x80, + 0x58, 0xdf, 0xce, 0x80, 0x59, 0xcf, 0xbf, 0x80, 0x5a, 0xbf, 0xb0, 0x80, + 0x5b, 0xb8, 0xdc, 0x00, 0x5c, 0xa8, 0xcd, 0x00, 0x5d, 0x98, 0xbe, 0x00, + 0x5e, 0x88, 0xaf, 0x00, 0x5f, 0x78, 0xa0, 0x00, 0x60, 0x68, 0x91, 0x00, + 0x61, 0x58, 0x82, 0x00, 0x62, 0x48, 0x73, 0x00, 0x63, 0x38, 0x64, 0x00, + 0x64, 0x28, 0x55, 0x00, 0x65, 0x18, 0x46, 0x00, 0x66, 0x11, 0x71, 0x80, + 0x67, 0x01, 0x62, 0x80, 0x67, 0xf1, 0x53, 0x80, 0x68, 0xe1, 0x44, 0x80, + 0x69, 0xd1, 0x35, 0x80, 0x6a, 0xc1, 0x26, 0x80, 0x6b, 0xb1, 0x17, 0x80, + 0x6c, 0xa1, 0x08, 0x80, 0x6d, 0x90, 0xf9, 0x80, 0x6e, 0x80, 0xea, 0x80, + 0x6f, 0x70, 0xdb, 0x80, 0x70, 0x6a, 0x07, 0x00, 0x71, 0x59, 0xf8, 0x00, + 0x72, 0x49, 0xe9, 0x00, 0x73, 0x39, 0xda, 0x00, 0x74, 0x29, 0xcb, 0x00, + 0x75, 0x19, 0xbc, 0x00, 0x76, 0x09, 0xad, 0x00, 0x76, 0xf9, 0x9e, 0x00, + 0x77, 0xe9, 0x8f, 0x00, 0x78, 0xd9, 0x80, 0x00, 0x79, 0xc9, 0x71, 0x00, + 0x7a, 0xb9, 0x62, 0x00, 0x7b, 0xb2, 0x8d, 0x80, 0x7c, 0xa2, 0x7e, 0x80, + 0x7d, 0x92, 0x6f, 0x80, 0x7e, 0x82, 0x60, 0x80, 0x7f, 0x72, 0x51, 0x80, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, + 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x00, 0x00, + 0x8d, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x9a, 0xb0, 0x01, 0x04, 0x00, 0x00, + 0x8c, 0xa0, 0x00, 0x09, 0x00, 0x00, 0x9a, 0xb0, 0x01, 0x04, 0x00, 0x00, + 0x8c, 0xa0, 0x00, 0x09, 0x4c, 0x4d, 0x54, 0x00, 0x41, 0x45, 0x44, 0x54, + 0x00, 0x41, 0x45, 0x53, 0x54, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x0e, + 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x73, 0x16, 0x7f, 0x3c, 0xff, 0xff, 0xff, 0xff, 0x9c, 0x4e, 0xa6, 0x9c, + 0xff, 0xff, 0xff, 0xff, 0x9c, 0xbc, 0x20, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xcb, 0x54, 0xb3, 0x00, 0xff, 0xff, 0xff, 0xff, 0xcb, 0xc7, 0x57, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xcc, 0xb7, 0x56, 0x80, 0xff, 0xff, 0xff, 0xff, + 0xcd, 0xa7, 0x39, 0x70, 0xff, 0xff, 0xff, 0xff, 0xce, 0xa0, 0x73, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xcf, 0x87, 0x1b, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x70, 0x39, 0x80, 0x00, 0x00, 0x00, 0x00, 0x04, 0x0d, 0x1c, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x05, 0x50, 0x1b, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x05, 0xf6, 0x38, 0x80, 0x00, 0x00, 0x00, 0x00, 0x07, 0x2f, 0xfd, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x07, 0xd6, 0x1a, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x09, 0x0f, 0xdf, 0x80, 0x00, 0x00, 0x00, 0x00, 0x09, 0xb5, 0xfc, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x0a, 0xef, 0xc1, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x0b, 0x9f, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xd8, 0xde, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0d, 0x7e, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0xb8, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x5e, 0xdd, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x10, 0x98, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x11, 0x3e, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x78, 0x84, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x13, 0x1e, 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x14, 0x58, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0xfe, 0x83, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x16, 0x38, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x17, 0x0c, 0x89, 0x80, 0x00, 0x00, 0x00, 0x00, 0x18, 0x21, 0x64, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x18, 0xc7, 0x81, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1a, 0x01, 0x46, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1a, 0xa7, 0x63, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe1, 0x28, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1c, 0x87, 0x45, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xc1, 0x0a, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x1e, 0x79, 0x9c, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1f, 0x97, 0xb2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x59, 0x7e, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x21, 0x80, 0xce, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x22, 0x42, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x69, 0xeb, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x24, 0x22, 0x7d, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x25, 0x49, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0xef, 0xea, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x27, 0x29, 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x27, 0xcf, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x09, 0x91, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x29, 0xaf, 0xae, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2a, 0xe9, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x98, 0xca, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd2, 0x8f, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x2d, 0x78, 0xac, 0x80, 0x00, 0x00, 0x00, 0x00, 0x2e, 0xb2, 0x71, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x2f, 0x58, 0x8e, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x30, 0x92, 0x53, 0x80, 0x00, 0x00, 0x00, 0x00, 0x31, 0x5d, 0x5a, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x32, 0x72, 0x35, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x33, 0x3d, 0x3c, 0x80, 0x00, 0x00, 0x00, 0x00, 0x34, 0x52, 0x17, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x35, 0x1d, 0x1e, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x36, 0x31, 0xf9, 0x80, 0x00, 0x00, 0x00, 0x00, 0x36, 0xfd, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x38, 0x1b, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x38, 0xdc, 0xe2, 0x80, 0x00, 0x00, 0x00, 0x00, 0x39, 0xa7, 0xe9, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x3a, 0xbc, 0xc4, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x3b, 0xda, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0xa5, 0xe1, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3d, 0xba, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3e, 0x85, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x9a, 0x9e, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x40, 0x65, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x41, 0x83, 0xba, 0x80, 0x00, 0x00, 0x00, 0x00, 0x42, 0x45, 0x87, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x43, 0x63, 0x9c, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x44, 0x2e, 0xa3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x45, 0x43, 0x7e, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x46, 0x05, 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x47, 0x23, 0x60, 0x80, 0x00, 0x00, 0x00, 0x00, 0x47, 0xf7, 0xa2, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x48, 0xe7, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x49, 0xd7, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xc7, 0x75, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4b, 0xb7, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4c, 0xa7, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x97, 0x48, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4e, 0x87, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4f, 0x77, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x70, 0x55, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x51, 0x60, 0x46, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x52, 0x50, 0x37, 0x80, 0x00, 0x00, 0x00, 0x00, 0x53, 0x40, 0x28, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x54, 0x30, 0x19, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x55, 0x20, 0x0a, 0x80, 0x00, 0x00, 0x00, 0x00, 0x56, 0x0f, 0xfb, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x56, 0xff, 0xec, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x57, 0xef, 0xdd, 0x80, 0x00, 0x00, 0x00, 0x00, 0x58, 0xdf, 0xce, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x59, 0xcf, 0xbf, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x5a, 0xbf, 0xb0, 0x80, 0x00, 0x00, 0x00, 0x00, 0x5b, 0xb8, 0xdc, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5c, 0xa8, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x5d, 0x98, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x88, 0xaf, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5f, 0x78, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x60, 0x68, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x58, 0x82, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x62, 0x48, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x63, 0x38, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x28, 0x55, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x65, 0x18, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x66, 0x11, 0x71, 0x80, 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, 0x62, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x67, 0xf1, 0x53, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x68, 0xe1, 0x44, 0x80, 0x00, 0x00, 0x00, 0x00, 0x69, 0xd1, 0x35, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x6a, 0xc1, 0x26, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x6b, 0xb1, 0x17, 0x80, 0x00, 0x00, 0x00, 0x00, 0x6c, 0xa1, 0x08, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x6d, 0x90, 0xf9, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x6e, 0x80, 0xea, 0x80, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x70, 0xdb, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x70, 0x6a, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x71, 0x59, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x49, 0xe9, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x73, 0x39, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x74, 0x29, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x19, 0xbc, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x76, 0x09, 0xad, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x76, 0xf9, 0x9e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0xe9, 0x8f, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x78, 0xd9, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x79, 0xc9, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0xb9, 0x62, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7b, 0xb2, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x7c, 0xa2, 0x7e, 0x80, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x92, 0x6f, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x7e, 0x82, 0x60, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x7f, 0x72, 0x51, 0x80, 0x00, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, + 0x03, 0x04, 0x03, 0x00, 0x00, 0x8d, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x9a, + 0xb0, 0x01, 0x04, 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, 0x00, 0x00, 0x9a, + 0xb0, 0x01, 0x04, 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, 0x4c, 0x4d, 0x54, + 0x00, 0x41, 0x45, 0x44, 0x54, 0x00, 0x41, 0x45, 0x53, 0x54, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x41, 0x45, + 0x53, 0x54, 0x2d, 0x31, 0x30, 0x41, 0x45, 0x44, 0x54, 0x2c, 0x4d, 0x31, + 0x30, 0x2e, 0x31, 0x2e, 0x30, 0x2c, 0x4d, 0x34, 0x2e, 0x31, 0x2e, 0x30, + 0x2f, 0x33, 0x0a +}; +unsigned int Australia_Sydney_len = 2223; diff --git a/third_party/abseil_cpp/absl/time/time.cc b/third_party/abseil_cpp/absl/time/time.cc new file mode 100644 index 000000000000..1ec2026e254f --- /dev/null +++ b/third_party/abseil_cpp/absl/time/time.cc @@ -0,0 +1,500 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The implementation of the absl::Time class, which is declared in +// //absl/time.h. +// +// The representation for an absl::Time is an absl::Duration offset from the +// epoch. We use the traditional Unix epoch (1970-01-01 00:00:00 +0000) +// for convenience, but this is not exposed in the API and could be changed. +// +// NOTE: To keep type verbosity to a minimum, the following variable naming +// conventions are used throughout this file. +// +// tz: An absl::TimeZone +// ci: An absl::TimeZone::CivilInfo +// ti: An absl::TimeZone::TimeInfo +// cd: An absl::CivilDay or a cctz::civil_day +// cs: An absl::CivilSecond or a cctz::civil_second +// bd: An absl::Time::Breakdown +// cl: A cctz::time_zone::civil_lookup +// al: A cctz::time_zone::absolute_lookup + +#include "absl/time/time.h" + +#if defined(_MSC_VER) +#include <winsock2.h> // for timeval +#endif + +#include <cstring> +#include <ctime> +#include <limits> + +#include "absl/time/internal/cctz/include/cctz/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace cctz = absl::time_internal::cctz; + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +inline cctz::time_point<cctz::seconds> unix_epoch() { + return std::chrono::time_point_cast<cctz::seconds>( + std::chrono::system_clock::from_time_t(0)); +} + +// Floors d to the next unit boundary closer to negative infinity. +inline int64_t FloorToUnit(absl::Duration d, absl::Duration unit) { + absl::Duration rem; + int64_t q = absl::IDivDuration(d, unit, &rem); + return (q > 0 || rem >= ZeroDuration() || + q == std::numeric_limits<int64_t>::min()) + ? q + : q - 1; +} + +inline absl::Time::Breakdown InfiniteFutureBreakdown() { + absl::Time::Breakdown bd; + bd.year = std::numeric_limits<int64_t>::max(); + bd.month = 12; + bd.day = 31; + bd.hour = 23; + bd.minute = 59; + bd.second = 59; + bd.subsecond = absl::InfiniteDuration(); + bd.weekday = 4; + bd.yearday = 365; + bd.offset = 0; + bd.is_dst = false; + bd.zone_abbr = "-00"; + return bd; +} + +inline absl::Time::Breakdown InfinitePastBreakdown() { + Time::Breakdown bd; + bd.year = std::numeric_limits<int64_t>::min(); + bd.month = 1; + bd.day = 1; + bd.hour = 0; + bd.minute = 0; + bd.second = 0; + bd.subsecond = -absl::InfiniteDuration(); + bd.weekday = 7; + bd.yearday = 1; + bd.offset = 0; + bd.is_dst = false; + bd.zone_abbr = "-00"; + return bd; +} + +inline absl::TimeZone::CivilInfo InfiniteFutureCivilInfo() { + TimeZone::CivilInfo ci; + ci.cs = CivilSecond::max(); + ci.subsecond = InfiniteDuration(); + ci.offset = 0; + ci.is_dst = false; + ci.zone_abbr = "-00"; + return ci; +} + +inline absl::TimeZone::CivilInfo InfinitePastCivilInfo() { + TimeZone::CivilInfo ci; + ci.cs = CivilSecond::min(); + ci.subsecond = -InfiniteDuration(); + ci.offset = 0; + ci.is_dst = false; + ci.zone_abbr = "-00"; + return ci; +} + +inline absl::TimeConversion InfiniteFutureTimeConversion() { + absl::TimeConversion tc; + tc.pre = tc.trans = tc.post = absl::InfiniteFuture(); + tc.kind = absl::TimeConversion::UNIQUE; + tc.normalized = true; + return tc; +} + +inline TimeConversion InfinitePastTimeConversion() { + absl::TimeConversion tc; + tc.pre = tc.trans = tc.post = absl::InfinitePast(); + tc.kind = absl::TimeConversion::UNIQUE; + tc.normalized = true; + return tc; +} + +// Makes a Time from sec, overflowing to InfiniteFuture/InfinitePast as +// necessary. If sec is min/max, then consult cs+tz to check for overlow. +Time MakeTimeWithOverflow(const cctz::time_point<cctz::seconds>& sec, + const cctz::civil_second& cs, + const cctz::time_zone& tz, + bool* normalized = nullptr) { + const auto max = cctz::time_point<cctz::seconds>::max(); + const auto min = cctz::time_point<cctz::seconds>::min(); + if (sec == max) { + const auto al = tz.lookup(max); + if (cs > al.cs) { + if (normalized) *normalized = true; + return absl::InfiniteFuture(); + } + } + if (sec == min) { + const auto al = tz.lookup(min); + if (cs < al.cs) { + if (normalized) *normalized = true; + return absl::InfinitePast(); + } + } + const auto hi = (sec - unix_epoch()).count(); + return time_internal::FromUnixDuration(time_internal::MakeDuration(hi)); +} + +// Returns Mon=1..Sun=7. +inline int MapWeekday(const cctz::weekday& wd) { + switch (wd) { + case cctz::weekday::monday: + return 1; + case cctz::weekday::tuesday: + return 2; + case cctz::weekday::wednesday: + return 3; + case cctz::weekday::thursday: + return 4; + case cctz::weekday::friday: + return 5; + case cctz::weekday::saturday: + return 6; + case cctz::weekday::sunday: + return 7; + } + return 1; +} + +bool FindTransition(const cctz::time_zone& tz, + bool (cctz::time_zone::*find_transition)( + const cctz::time_point<cctz::seconds>& tp, + cctz::time_zone::civil_transition* trans) const, + Time t, TimeZone::CivilTransition* trans) { + // Transitions are second-aligned, so we can discard any fractional part. + const auto tp = unix_epoch() + cctz::seconds(ToUnixSeconds(t)); + cctz::time_zone::civil_transition tr; + if (!(tz.*find_transition)(tp, &tr)) return false; + trans->from = CivilSecond(tr.from); + trans->to = CivilSecond(tr.to); + return true; +} + +} // namespace + +// +// Time +// + +absl::Time::Breakdown Time::In(absl::TimeZone tz) const { + if (*this == absl::InfiniteFuture()) return InfiniteFutureBreakdown(); + if (*this == absl::InfinitePast()) return InfinitePastBreakdown(); + + const auto tp = unix_epoch() + cctz::seconds(time_internal::GetRepHi(rep_)); + const auto al = cctz::time_zone(tz).lookup(tp); + const auto cs = al.cs; + const auto cd = cctz::civil_day(cs); + + absl::Time::Breakdown bd; + bd.year = cs.year(); + bd.month = cs.month(); + bd.day = cs.day(); + bd.hour = cs.hour(); + bd.minute = cs.minute(); + bd.second = cs.second(); + bd.subsecond = time_internal::MakeDuration(0, time_internal::GetRepLo(rep_)); + bd.weekday = MapWeekday(cctz::get_weekday(cd)); + bd.yearday = cctz::get_yearday(cd); + bd.offset = al.offset; + bd.is_dst = al.is_dst; + bd.zone_abbr = al.abbr; + return bd; +} + +// +// Conversions from/to other time types. +// + +absl::Time FromUDate(double udate) { + return time_internal::FromUnixDuration(absl::Milliseconds(udate)); +} + +absl::Time FromUniversal(int64_t universal) { + return absl::UniversalEpoch() + 100 * absl::Nanoseconds(universal); +} + +int64_t ToUnixNanos(Time t) { + if (time_internal::GetRepHi(time_internal::ToUnixDuration(t)) >= 0 && + time_internal::GetRepHi(time_internal::ToUnixDuration(t)) >> 33 == 0) { + return (time_internal::GetRepHi(time_internal::ToUnixDuration(t)) * + 1000 * 1000 * 1000) + + (time_internal::GetRepLo(time_internal::ToUnixDuration(t)) / 4); + } + return FloorToUnit(time_internal::ToUnixDuration(t), absl::Nanoseconds(1)); +} + +int64_t ToUnixMicros(Time t) { + if (time_internal::GetRepHi(time_internal::ToUnixDuration(t)) >= 0 && + time_internal::GetRepHi(time_internal::ToUnixDuration(t)) >> 43 == 0) { + return (time_internal::GetRepHi(time_internal::ToUnixDuration(t)) * + 1000 * 1000) + + (time_internal::GetRepLo(time_internal::ToUnixDuration(t)) / 4000); + } + return FloorToUnit(time_internal::ToUnixDuration(t), absl::Microseconds(1)); +} + +int64_t ToUnixMillis(Time t) { + if (time_internal::GetRepHi(time_internal::ToUnixDuration(t)) >= 0 && + time_internal::GetRepHi(time_internal::ToUnixDuration(t)) >> 53 == 0) { + return (time_internal::GetRepHi(time_internal::ToUnixDuration(t)) * 1000) + + (time_internal::GetRepLo(time_internal::ToUnixDuration(t)) / + (4000 * 1000)); + } + return FloorToUnit(time_internal::ToUnixDuration(t), absl::Milliseconds(1)); +} + +int64_t ToUnixSeconds(Time t) { + return time_internal::GetRepHi(time_internal::ToUnixDuration(t)); +} + +time_t ToTimeT(Time t) { return absl::ToTimespec(t).tv_sec; } + +double ToUDate(Time t) { + return absl::FDivDuration(time_internal::ToUnixDuration(t), + absl::Milliseconds(1)); +} + +int64_t ToUniversal(absl::Time t) { + return absl::FloorToUnit(t - absl::UniversalEpoch(), absl::Nanoseconds(100)); +} + +absl::Time TimeFromTimespec(timespec ts) { + return time_internal::FromUnixDuration(absl::DurationFromTimespec(ts)); +} + +absl::Time TimeFromTimeval(timeval tv) { + return time_internal::FromUnixDuration(absl::DurationFromTimeval(tv)); +} + +timespec ToTimespec(Time t) { + timespec ts; + absl::Duration d = time_internal::ToUnixDuration(t); + if (!time_internal::IsInfiniteDuration(d)) { + ts.tv_sec = time_internal::GetRepHi(d); + if (ts.tv_sec == time_internal::GetRepHi(d)) { // no time_t narrowing + ts.tv_nsec = time_internal::GetRepLo(d) / 4; // floor + return ts; + } + } + if (d >= absl::ZeroDuration()) { + ts.tv_sec = std::numeric_limits<time_t>::max(); + ts.tv_nsec = 1000 * 1000 * 1000 - 1; + } else { + ts.tv_sec = std::numeric_limits<time_t>::min(); + ts.tv_nsec = 0; + } + return ts; +} + +timeval ToTimeval(Time t) { + timeval tv; + timespec ts = absl::ToTimespec(t); + tv.tv_sec = ts.tv_sec; + if (tv.tv_sec != ts.tv_sec) { // narrowing + if (ts.tv_sec < 0) { + tv.tv_sec = std::numeric_limits<decltype(tv.tv_sec)>::min(); + tv.tv_usec = 0; + } else { + tv.tv_sec = std::numeric_limits<decltype(tv.tv_sec)>::max(); + tv.tv_usec = 1000 * 1000 - 1; + } + return tv; + } + tv.tv_usec = static_cast<int>(ts.tv_nsec / 1000); // suseconds_t + return tv; +} + +Time FromChrono(const std::chrono::system_clock::time_point& tp) { + return time_internal::FromUnixDuration(time_internal::FromChrono( + tp - std::chrono::system_clock::from_time_t(0))); +} + +std::chrono::system_clock::time_point ToChronoTime(absl::Time t) { + using D = std::chrono::system_clock::duration; + auto d = time_internal::ToUnixDuration(t); + if (d < ZeroDuration()) d = Floor(d, FromChrono(D{1})); + return std::chrono::system_clock::from_time_t(0) + + time_internal::ToChronoDuration<D>(d); +} + +// +// TimeZone +// + +absl::TimeZone::CivilInfo TimeZone::At(Time t) const { + if (t == absl::InfiniteFuture()) return InfiniteFutureCivilInfo(); + if (t == absl::InfinitePast()) return InfinitePastCivilInfo(); + + const auto ud = time_internal::ToUnixDuration(t); + const auto tp = unix_epoch() + cctz::seconds(time_internal::GetRepHi(ud)); + const auto al = cz_.lookup(tp); + + TimeZone::CivilInfo ci; + ci.cs = CivilSecond(al.cs); + ci.subsecond = time_internal::MakeDuration(0, time_internal::GetRepLo(ud)); + ci.offset = al.offset; + ci.is_dst = al.is_dst; + ci.zone_abbr = al.abbr; + return ci; +} + +absl::TimeZone::TimeInfo TimeZone::At(CivilSecond ct) const { + const cctz::civil_second cs(ct); + const auto cl = cz_.lookup(cs); + + TimeZone::TimeInfo ti; + switch (cl.kind) { + case cctz::time_zone::civil_lookup::UNIQUE: + ti.kind = TimeZone::TimeInfo::UNIQUE; + break; + case cctz::time_zone::civil_lookup::SKIPPED: + ti.kind = TimeZone::TimeInfo::SKIPPED; + break; + case cctz::time_zone::civil_lookup::REPEATED: + ti.kind = TimeZone::TimeInfo::REPEATED; + break; + } + ti.pre = MakeTimeWithOverflow(cl.pre, cs, cz_); + ti.trans = MakeTimeWithOverflow(cl.trans, cs, cz_); + ti.post = MakeTimeWithOverflow(cl.post, cs, cz_); + return ti; +} + +bool TimeZone::NextTransition(Time t, CivilTransition* trans) const { + return FindTransition(cz_, &cctz::time_zone::next_transition, t, trans); +} + +bool TimeZone::PrevTransition(Time t, CivilTransition* trans) const { + return FindTransition(cz_, &cctz::time_zone::prev_transition, t, trans); +} + +// +// Conversions involving time zones. +// + +absl::TimeConversion ConvertDateTime(int64_t year, int mon, int day, int hour, + int min, int sec, TimeZone tz) { + // Avoids years that are too extreme for CivilSecond to normalize. + if (year > 300000000000) return InfiniteFutureTimeConversion(); + if (year < -300000000000) return InfinitePastTimeConversion(); + + const CivilSecond cs(year, mon, day, hour, min, sec); + const auto ti = tz.At(cs); + + TimeConversion tc; + tc.pre = ti.pre; + tc.trans = ti.trans; + tc.post = ti.post; + switch (ti.kind) { + case TimeZone::TimeInfo::UNIQUE: + tc.kind = TimeConversion::UNIQUE; + break; + case TimeZone::TimeInfo::SKIPPED: + tc.kind = TimeConversion::SKIPPED; + break; + case TimeZone::TimeInfo::REPEATED: + tc.kind = TimeConversion::REPEATED; + break; + } + tc.normalized = false; + if (year != cs.year() || mon != cs.month() || day != cs.day() || + hour != cs.hour() || min != cs.minute() || sec != cs.second()) { + tc.normalized = true; + } + return tc; +} + +absl::Time FromTM(const struct tm& tm, absl::TimeZone tz) { + civil_year_t tm_year = tm.tm_year; + // Avoids years that are too extreme for CivilSecond to normalize. + if (tm_year > 300000000000ll) return InfiniteFuture(); + if (tm_year < -300000000000ll) return InfinitePast(); + int tm_mon = tm.tm_mon; + if (tm_mon == std::numeric_limits<int>::max()) { + tm_mon -= 12; + tm_year += 1; + } + const auto ti = tz.At(CivilSecond(tm_year + 1900, tm_mon + 1, tm.tm_mday, + tm.tm_hour, tm.tm_min, tm.tm_sec)); + return tm.tm_isdst == 0 ? ti.post : ti.pre; +} + +struct tm ToTM(absl::Time t, absl::TimeZone tz) { + struct tm tm = {}; + + const auto ci = tz.At(t); + const auto& cs = ci.cs; + tm.tm_sec = cs.second(); + tm.tm_min = cs.minute(); + tm.tm_hour = cs.hour(); + tm.tm_mday = cs.day(); + tm.tm_mon = cs.month() - 1; + + // Saturates tm.tm_year in cases of over/underflow, accounting for the fact + // that tm.tm_year is years since 1900. + if (cs.year() < std::numeric_limits<int>::min() + 1900) { + tm.tm_year = std::numeric_limits<int>::min(); + } else if (cs.year() > std::numeric_limits<int>::max()) { + tm.tm_year = std::numeric_limits<int>::max() - 1900; + } else { + tm.tm_year = static_cast<int>(cs.year() - 1900); + } + + switch (GetWeekday(cs)) { + case Weekday::sunday: + tm.tm_wday = 0; + break; + case Weekday::monday: + tm.tm_wday = 1; + break; + case Weekday::tuesday: + tm.tm_wday = 2; + break; + case Weekday::wednesday: + tm.tm_wday = 3; + break; + case Weekday::thursday: + tm.tm_wday = 4; + break; + case Weekday::friday: + tm.tm_wday = 5; + break; + case Weekday::saturday: + tm.tm_wday = 6; + break; + } + tm.tm_yday = GetYearDay(cs) - 1; + tm.tm_isdst = ci.is_dst ? 1 : 0; + + return tm; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/time/time.h b/third_party/abseil_cpp/absl/time/time.h new file mode 100644 index 000000000000..72508031acd6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/time.h @@ -0,0 +1,1581 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: time.h +// ----------------------------------------------------------------------------- +// +// This header file defines abstractions for computing with absolute points +// in time, durations of time, and formatting and parsing time within a given +// time zone. The following abstractions are defined: +// +// * `absl::Time` defines an absolute, specific instance in time +// * `absl::Duration` defines a signed, fixed-length span of time +// * `absl::TimeZone` defines geopolitical time zone regions (as collected +// within the IANA Time Zone database (https://www.iana.org/time-zones)). +// +// Note: Absolute times are distinct from civil times, which refer to the +// human-scale time commonly represented by `YYYY-MM-DD hh:mm:ss`. The mapping +// between absolute and civil times can be specified by use of time zones +// (`absl::TimeZone` within this API). That is: +// +// Civil Time = F(Absolute Time, Time Zone) +// Absolute Time = G(Civil Time, Time Zone) +// +// See civil_time.h for abstractions related to constructing and manipulating +// civil time. +// +// Example: +// +// absl::TimeZone nyc; +// // LoadTimeZone() may fail so it's always better to check for success. +// if (!absl::LoadTimeZone("America/New_York", &nyc)) { +// // handle error case +// } +// +// // My flight leaves NYC on Jan 2, 2017 at 03:04:05 +// absl::CivilSecond cs(2017, 1, 2, 3, 4, 5); +// absl::Time takeoff = absl::FromCivil(cs, nyc); +// +// absl::Duration flight_duration = absl::Hours(21) + absl::Minutes(35); +// absl::Time landing = takeoff + flight_duration; +// +// absl::TimeZone syd; +// if (!absl::LoadTimeZone("Australia/Sydney", &syd)) { +// // handle error case +// } +// std::string s = absl::FormatTime( +// "My flight will land in Sydney on %Y-%m-%d at %H:%M:%S", +// landing, syd); + +#ifndef ABSL_TIME_TIME_H_ +#define ABSL_TIME_TIME_H_ + +#if !defined(_MSC_VER) +#include <sys/time.h> +#else +// We don't include `winsock2.h` because it drags in `windows.h` and friends, +// and they define conflicting macros like OPAQUE, ERROR, and more. This has the +// potential to break Abseil users. +// +// Instead we only forward declare `timeval` and require Windows users include +// `winsock2.h` themselves. This is both inconsistent and troublesome, but so is +// including 'windows.h' so we are picking the lesser of two evils here. +struct timeval; +#endif +#include <chrono> // NOLINT(build/c++11) +#include <cmath> +#include <cstdint> +#include <ctime> +#include <ostream> +#include <string> +#include <type_traits> +#include <utility> + +#include "absl/base/macros.h" +#include "absl/strings/string_view.h" +#include "absl/time/civil_time.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class Duration; // Defined below +class Time; // Defined below +class TimeZone; // Defined below + +namespace time_internal { +int64_t IDivDuration(bool satq, Duration num, Duration den, Duration* rem); +constexpr Time FromUnixDuration(Duration d); +constexpr Duration ToUnixDuration(Time t); +constexpr int64_t GetRepHi(Duration d); +constexpr uint32_t GetRepLo(Duration d); +constexpr Duration MakeDuration(int64_t hi, uint32_t lo); +constexpr Duration MakeDuration(int64_t hi, int64_t lo); +inline Duration MakePosDoubleDuration(double n); +constexpr int64_t kTicksPerNanosecond = 4; +constexpr int64_t kTicksPerSecond = 1000 * 1000 * 1000 * kTicksPerNanosecond; +template <std::intmax_t N> +constexpr Duration FromInt64(int64_t v, std::ratio<1, N>); +constexpr Duration FromInt64(int64_t v, std::ratio<60>); +constexpr Duration FromInt64(int64_t v, std::ratio<3600>); +template <typename T> +using EnableIfIntegral = typename std::enable_if< + std::is_integral<T>::value || std::is_enum<T>::value, int>::type; +template <typename T> +using EnableIfFloat = + typename std::enable_if<std::is_floating_point<T>::value, int>::type; +} // namespace time_internal + +// Duration +// +// The `absl::Duration` class represents a signed, fixed-length span of time. +// A `Duration` is generated using a unit-specific factory function, or is +// the result of subtracting one `absl::Time` from another. Durations behave +// like unit-safe integers and they support all the natural integer-like +// arithmetic operations. Arithmetic overflows and saturates at +/- infinity. +// `Duration` should be passed by value rather than const reference. +// +// Factory functions `Nanoseconds()`, `Microseconds()`, `Milliseconds()`, +// `Seconds()`, `Minutes()`, `Hours()` and `InfiniteDuration()` allow for +// creation of constexpr `Duration` values +// +// Examples: +// +// constexpr absl::Duration ten_ns = absl::Nanoseconds(10); +// constexpr absl::Duration min = absl::Minutes(1); +// constexpr absl::Duration hour = absl::Hours(1); +// absl::Duration dur = 60 * min; // dur == hour +// absl::Duration half_sec = absl::Milliseconds(500); +// absl::Duration quarter_sec = 0.25 * absl::Seconds(1); +// +// `Duration` values can be easily converted to an integral number of units +// using the division operator. +// +// Example: +// +// constexpr absl::Duration dur = absl::Milliseconds(1500); +// int64_t ns = dur / absl::Nanoseconds(1); // ns == 1500000000 +// int64_t ms = dur / absl::Milliseconds(1); // ms == 1500 +// int64_t sec = dur / absl::Seconds(1); // sec == 1 (subseconds truncated) +// int64_t min = dur / absl::Minutes(1); // min == 0 +// +// See the `IDivDuration()` and `FDivDuration()` functions below for details on +// how to access the fractional parts of the quotient. +// +// Alternatively, conversions can be performed using helpers such as +// `ToInt64Microseconds()` and `ToDoubleSeconds()`. +class Duration { + public: + // Value semantics. + constexpr Duration() : rep_hi_(0), rep_lo_(0) {} // zero-length duration + + // Copyable. +#if !defined(__clang__) && defined(_MSC_VER) && _MSC_VER < 1910 + // Explicitly defining the constexpr copy constructor avoids an MSVC bug. + constexpr Duration(const Duration& d) + : rep_hi_(d.rep_hi_), rep_lo_(d.rep_lo_) {} +#else + constexpr Duration(const Duration& d) = default; +#endif + Duration& operator=(const Duration& d) = default; + + // Compound assignment operators. + Duration& operator+=(Duration d); + Duration& operator-=(Duration d); + Duration& operator*=(int64_t r); + Duration& operator*=(double r); + Duration& operator/=(int64_t r); + Duration& operator/=(double r); + Duration& operator%=(Duration rhs); + + // Overloads that forward to either the int64_t or double overloads above. + // Integer operands must be representable as int64_t. + template <typename T> + Duration& operator*=(T r) { + int64_t x = r; + return *this *= x; + } + template <typename T> + Duration& operator/=(T r) { + int64_t x = r; + return *this /= x; + } + Duration& operator*=(float r) { return *this *= static_cast<double>(r); } + Duration& operator/=(float r) { return *this /= static_cast<double>(r); } + + template <typename H> + friend H AbslHashValue(H h, Duration d) { + return H::combine(std::move(h), d.rep_hi_, d.rep_lo_); + } + + private: + friend constexpr int64_t time_internal::GetRepHi(Duration d); + friend constexpr uint32_t time_internal::GetRepLo(Duration d); + friend constexpr Duration time_internal::MakeDuration(int64_t hi, + uint32_t lo); + constexpr Duration(int64_t hi, uint32_t lo) : rep_hi_(hi), rep_lo_(lo) {} + int64_t rep_hi_; + uint32_t rep_lo_; +}; + +// Relational Operators +constexpr bool operator<(Duration lhs, Duration rhs); +constexpr bool operator>(Duration lhs, Duration rhs) { return rhs < lhs; } +constexpr bool operator>=(Duration lhs, Duration rhs) { return !(lhs < rhs); } +constexpr bool operator<=(Duration lhs, Duration rhs) { return !(rhs < lhs); } +constexpr bool operator==(Duration lhs, Duration rhs); +constexpr bool operator!=(Duration lhs, Duration rhs) { return !(lhs == rhs); } + +// Additive Operators +constexpr Duration operator-(Duration d); +inline Duration operator+(Duration lhs, Duration rhs) { return lhs += rhs; } +inline Duration operator-(Duration lhs, Duration rhs) { return lhs -= rhs; } + +// Multiplicative Operators +// Integer operands must be representable as int64_t. +template <typename T> +Duration operator*(Duration lhs, T rhs) { + return lhs *= rhs; +} +template <typename T> +Duration operator*(T lhs, Duration rhs) { + return rhs *= lhs; +} +template <typename T> +Duration operator/(Duration lhs, T rhs) { + return lhs /= rhs; +} +inline int64_t operator/(Duration lhs, Duration rhs) { + return time_internal::IDivDuration(true, lhs, rhs, + &lhs); // trunc towards zero +} +inline Duration operator%(Duration lhs, Duration rhs) { return lhs %= rhs; } + +// IDivDuration() +// +// Divides a numerator `Duration` by a denominator `Duration`, returning the +// quotient and remainder. The remainder always has the same sign as the +// numerator. The returned quotient and remainder respect the identity: +// +// numerator = denominator * quotient + remainder +// +// Returned quotients are capped to the range of `int64_t`, with the difference +// spilling into the remainder to uphold the above identity. This means that the +// remainder returned could differ from the remainder returned by +// `Duration::operator%` for huge quotients. +// +// See also the notes on `InfiniteDuration()` below regarding the behavior of +// division involving zero and infinite durations. +// +// Example: +// +// constexpr absl::Duration a = +// absl::Seconds(std::numeric_limits<int64_t>::max()); // big +// constexpr absl::Duration b = absl::Nanoseconds(1); // small +// +// absl::Duration rem = a % b; +// // rem == absl::ZeroDuration() +// +// // Here, q would overflow int64_t, so rem accounts for the difference. +// int64_t q = absl::IDivDuration(a, b, &rem); +// // q == std::numeric_limits<int64_t>::max(), rem == a - b * q +inline int64_t IDivDuration(Duration num, Duration den, Duration* rem) { + return time_internal::IDivDuration(true, num, den, + rem); // trunc towards zero +} + +// FDivDuration() +// +// Divides a `Duration` numerator into a fractional number of units of a +// `Duration` denominator. +// +// See also the notes on `InfiniteDuration()` below regarding the behavior of +// division involving zero and infinite durations. +// +// Example: +// +// double d = absl::FDivDuration(absl::Milliseconds(1500), absl::Seconds(1)); +// // d == 1.5 +double FDivDuration(Duration num, Duration den); + +// ZeroDuration() +// +// Returns a zero-length duration. This function behaves just like the default +// constructor, but the name helps make the semantics clear at call sites. +constexpr Duration ZeroDuration() { return Duration(); } + +// AbsDuration() +// +// Returns the absolute value of a duration. +inline Duration AbsDuration(Duration d) { + return (d < ZeroDuration()) ? -d : d; +} + +// Trunc() +// +// Truncates a duration (toward zero) to a multiple of a non-zero unit. +// +// Example: +// +// absl::Duration d = absl::Nanoseconds(123456789); +// absl::Duration a = absl::Trunc(d, absl::Microseconds(1)); // 123456us +Duration Trunc(Duration d, Duration unit); + +// Floor() +// +// Floors a duration using the passed duration unit to its largest value not +// greater than the duration. +// +// Example: +// +// absl::Duration d = absl::Nanoseconds(123456789); +// absl::Duration b = absl::Floor(d, absl::Microseconds(1)); // 123456us +Duration Floor(Duration d, Duration unit); + +// Ceil() +// +// Returns the ceiling of a duration using the passed duration unit to its +// smallest value not less than the duration. +// +// Example: +// +// absl::Duration d = absl::Nanoseconds(123456789); +// absl::Duration c = absl::Ceil(d, absl::Microseconds(1)); // 123457us +Duration Ceil(Duration d, Duration unit); + +// InfiniteDuration() +// +// Returns an infinite `Duration`. To get a `Duration` representing negative +// infinity, use `-InfiniteDuration()`. +// +// Duration arithmetic overflows to +/- infinity and saturates. In general, +// arithmetic with `Duration` infinities is similar to IEEE 754 infinities +// except where IEEE 754 NaN would be involved, in which case +/- +// `InfiniteDuration()` is used in place of a "nan" Duration. +// +// Examples: +// +// constexpr absl::Duration inf = absl::InfiniteDuration(); +// const absl::Duration d = ... any finite duration ... +// +// inf == inf + inf +// inf == inf + d +// inf == inf - inf +// -inf == d - inf +// +// inf == d * 1e100 +// inf == inf / 2 +// 0 == d / inf +// INT64_MAX == inf / d +// +// d < inf +// -inf < d +// +// // Division by zero returns infinity, or INT64_MIN/MAX where appropriate. +// inf == d / 0 +// INT64_MAX == d / absl::ZeroDuration() +// +// The examples involving the `/` operator above also apply to `IDivDuration()` +// and `FDivDuration()`. +constexpr Duration InfiniteDuration(); + +// Nanoseconds() +// Microseconds() +// Milliseconds() +// Seconds() +// Minutes() +// Hours() +// +// Factory functions for constructing `Duration` values from an integral number +// of the unit indicated by the factory function's name. The number must be +// representable as int64_t. +// +// NOTE: no "Days()" factory function exists because "a day" is ambiguous. +// Civil days are not always 24 hours long, and a 24-hour duration often does +// not correspond with a civil day. If a 24-hour duration is needed, use +// `absl::Hours(24)`. If you actually want a civil day, use absl::CivilDay +// from civil_time.h. +// +// Example: +// +// absl::Duration a = absl::Seconds(60); +// absl::Duration b = absl::Minutes(1); // b == a +constexpr Duration Nanoseconds(int64_t n); +constexpr Duration Microseconds(int64_t n); +constexpr Duration Milliseconds(int64_t n); +constexpr Duration Seconds(int64_t n); +constexpr Duration Minutes(int64_t n); +constexpr Duration Hours(int64_t n); + +// Factory overloads for constructing `Duration` values from a floating-point +// number of the unit indicated by the factory function's name. These functions +// exist for convenience, but they are not as efficient as the integral +// factories, which should be preferred. +// +// Example: +// +// auto a = absl::Seconds(1.5); // OK +// auto b = absl::Milliseconds(1500); // BETTER +template <typename T, time_internal::EnableIfFloat<T> = 0> +Duration Nanoseconds(T n) { + return n * Nanoseconds(1); +} +template <typename T, time_internal::EnableIfFloat<T> = 0> +Duration Microseconds(T n) { + return n * Microseconds(1); +} +template <typename T, time_internal::EnableIfFloat<T> = 0> +Duration Milliseconds(T n) { + return n * Milliseconds(1); +} +template <typename T, time_internal::EnableIfFloat<T> = 0> +Duration Seconds(T n) { + if (n >= 0) { // Note: `NaN >= 0` is false. + if (n >= static_cast<T>((std::numeric_limits<int64_t>::max)())) { + return InfiniteDuration(); + } + return time_internal::MakePosDoubleDuration(n); + } else { + if (std::isnan(n)) + return std::signbit(n) ? -InfiniteDuration() : InfiniteDuration(); + if (n <= (std::numeric_limits<int64_t>::min)()) return -InfiniteDuration(); + return -time_internal::MakePosDoubleDuration(-n); + } +} +template <typename T, time_internal::EnableIfFloat<T> = 0> +Duration Minutes(T n) { + return n * Minutes(1); +} +template <typename T, time_internal::EnableIfFloat<T> = 0> +Duration Hours(T n) { + return n * Hours(1); +} + +// ToInt64Nanoseconds() +// ToInt64Microseconds() +// ToInt64Milliseconds() +// ToInt64Seconds() +// ToInt64Minutes() +// ToInt64Hours() +// +// Helper functions that convert a Duration to an integral count of the +// indicated unit. These functions are shorthand for the `IDivDuration()` +// function above; see its documentation for details about overflow, etc. +// +// Example: +// +// absl::Duration d = absl::Milliseconds(1500); +// int64_t isec = absl::ToInt64Seconds(d); // isec == 1 +int64_t ToInt64Nanoseconds(Duration d); +int64_t ToInt64Microseconds(Duration d); +int64_t ToInt64Milliseconds(Duration d); +int64_t ToInt64Seconds(Duration d); +int64_t ToInt64Minutes(Duration d); +int64_t ToInt64Hours(Duration d); + +// ToDoubleNanoSeconds() +// ToDoubleMicroseconds() +// ToDoubleMilliseconds() +// ToDoubleSeconds() +// ToDoubleMinutes() +// ToDoubleHours() +// +// Helper functions that convert a Duration to a floating point count of the +// indicated unit. These functions are shorthand for the `FDivDuration()` +// function above; see its documentation for details about overflow, etc. +// +// Example: +// +// absl::Duration d = absl::Milliseconds(1500); +// double dsec = absl::ToDoubleSeconds(d); // dsec == 1.5 +double ToDoubleNanoseconds(Duration d); +double ToDoubleMicroseconds(Duration d); +double ToDoubleMilliseconds(Duration d); +double ToDoubleSeconds(Duration d); +double ToDoubleMinutes(Duration d); +double ToDoubleHours(Duration d); + +// FromChrono() +// +// Converts any of the pre-defined std::chrono durations to an absl::Duration. +// +// Example: +// +// std::chrono::milliseconds ms(123); +// absl::Duration d = absl::FromChrono(ms); +constexpr Duration FromChrono(const std::chrono::nanoseconds& d); +constexpr Duration FromChrono(const std::chrono::microseconds& d); +constexpr Duration FromChrono(const std::chrono::milliseconds& d); +constexpr Duration FromChrono(const std::chrono::seconds& d); +constexpr Duration FromChrono(const std::chrono::minutes& d); +constexpr Duration FromChrono(const std::chrono::hours& d); + +// ToChronoNanoseconds() +// ToChronoMicroseconds() +// ToChronoMilliseconds() +// ToChronoSeconds() +// ToChronoMinutes() +// ToChronoHours() +// +// Converts an absl::Duration to any of the pre-defined std::chrono durations. +// If overflow would occur, the returned value will saturate at the min/max +// chrono duration value instead. +// +// Example: +// +// absl::Duration d = absl::Microseconds(123); +// auto x = absl::ToChronoMicroseconds(d); +// auto y = absl::ToChronoNanoseconds(d); // x == y +// auto z = absl::ToChronoSeconds(absl::InfiniteDuration()); +// // z == std::chrono::seconds::max() +std::chrono::nanoseconds ToChronoNanoseconds(Duration d); +std::chrono::microseconds ToChronoMicroseconds(Duration d); +std::chrono::milliseconds ToChronoMilliseconds(Duration d); +std::chrono::seconds ToChronoSeconds(Duration d); +std::chrono::minutes ToChronoMinutes(Duration d); +std::chrono::hours ToChronoHours(Duration d); + +// FormatDuration() +// +// Returns a string representing the duration in the form "72h3m0.5s". +// Returns "inf" or "-inf" for +/- `InfiniteDuration()`. +std::string FormatDuration(Duration d); + +// Output stream operator. +inline std::ostream& operator<<(std::ostream& os, Duration d) { + return os << FormatDuration(d); +} + +// ParseDuration() +// +// Parses a duration string consisting of a possibly signed sequence of +// decimal numbers, each with an optional fractional part and a unit +// suffix. The valid suffixes are "ns", "us" "ms", "s", "m", and "h". +// Simple examples include "300ms", "-1.5h", and "2h45m". Parses "0" as +// `ZeroDuration()`. Parses "inf" and "-inf" as +/- `InfiniteDuration()`. +bool ParseDuration(absl::string_view dur_string, Duration* d); + +// Support for flag values of type Duration. Duration flags must be specified +// in a format that is valid input for absl::ParseDuration(). +bool AbslParseFlag(absl::string_view text, Duration* dst, std::string* error); +std::string AbslUnparseFlag(Duration d); +ABSL_DEPRECATED("Use AbslParseFlag() instead.") +bool ParseFlag(const std::string& text, Duration* dst, std::string* error); +ABSL_DEPRECATED("Use AbslUnparseFlag() instead.") +std::string UnparseFlag(Duration d); + +// Time +// +// An `absl::Time` represents a specific instant in time. Arithmetic operators +// are provided for naturally expressing time calculations. Instances are +// created using `absl::Now()` and the `absl::From*()` factory functions that +// accept the gamut of other time representations. Formatting and parsing +// functions are provided for conversion to and from strings. `absl::Time` +// should be passed by value rather than const reference. +// +// `absl::Time` assumes there are 60 seconds in a minute, which means the +// underlying time scales must be "smeared" to eliminate leap seconds. +// See https://developers.google.com/time/smear. +// +// Even though `absl::Time` supports a wide range of timestamps, exercise +// caution when using values in the distant past. `absl::Time` uses the +// Proleptic Gregorian calendar, which extends the Gregorian calendar backward +// to dates before its introduction in 1582. +// See https://en.wikipedia.org/wiki/Proleptic_Gregorian_calendar +// for more information. Use the ICU calendar classes to convert a date in +// some other calendar (http://userguide.icu-project.org/datetime/calendar). +// +// Similarly, standardized time zones are a reasonably recent innovation, with +// the Greenwich prime meridian being established in 1884. The TZ database +// itself does not profess accurate offsets for timestamps prior to 1970. The +// breakdown of future timestamps is subject to the whim of regional +// governments. +// +// The `absl::Time` class represents an instant in time as a count of clock +// ticks of some granularity (resolution) from some starting point (epoch). +// +// `absl::Time` uses a resolution that is high enough to avoid loss in +// precision, and a range that is wide enough to avoid overflow, when +// converting between tick counts in most Google time scales (i.e., resolution +// of at least one nanosecond, and range +/-100 billion years). Conversions +// between the time scales are performed by truncating (towards negative +// infinity) to the nearest representable point. +// +// Examples: +// +// absl::Time t1 = ...; +// absl::Time t2 = t1 + absl::Minutes(2); +// absl::Duration d = t2 - t1; // == absl::Minutes(2) +// +class Time { + public: + // Value semantics. + + // Returns the Unix epoch. However, those reading your code may not know + // or expect the Unix epoch as the default value, so make your code more + // readable by explicitly initializing all instances before use. + // + // Example: + // absl::Time t = absl::UnixEpoch(); + // absl::Time t = absl::Now(); + // absl::Time t = absl::TimeFromTimeval(tv); + // absl::Time t = absl::InfinitePast(); + constexpr Time() = default; + + // Copyable. + constexpr Time(const Time& t) = default; + Time& operator=(const Time& t) = default; + + // Assignment operators. + Time& operator+=(Duration d) { + rep_ += d; + return *this; + } + Time& operator-=(Duration d) { + rep_ -= d; + return *this; + } + + // Time::Breakdown + // + // The calendar and wall-clock (aka "civil time") components of an + // `absl::Time` in a certain `absl::TimeZone`. This struct is not + // intended to represent an instant in time. So, rather than passing + // a `Time::Breakdown` to a function, pass an `absl::Time` and an + // `absl::TimeZone`. + // + // Deprecated. Use `absl::TimeZone::CivilInfo`. + struct + Breakdown { + int64_t year; // year (e.g., 2013) + int month; // month of year [1:12] + int day; // day of month [1:31] + int hour; // hour of day [0:23] + int minute; // minute of hour [0:59] + int second; // second of minute [0:59] + Duration subsecond; // [Seconds(0):Seconds(1)) if finite + int weekday; // 1==Mon, ..., 7=Sun + int yearday; // day of year [1:366] + + // Note: The following fields exist for backward compatibility + // with older APIs. Accessing these fields directly is a sign of + // imprudent logic in the calling code. Modern time-related code + // should only access this data indirectly by way of FormatTime(). + // These fields are undefined for InfiniteFuture() and InfinitePast(). + int offset; // seconds east of UTC + bool is_dst; // is offset non-standard? + const char* zone_abbr; // time-zone abbreviation (e.g., "PST") + }; + + // Time::In() + // + // Returns the breakdown of this instant in the given TimeZone. + // + // Deprecated. Use `absl::TimeZone::At(Time)`. + Breakdown In(TimeZone tz) const; + + template <typename H> + friend H AbslHashValue(H h, Time t) { + return H::combine(std::move(h), t.rep_); + } + + private: + friend constexpr Time time_internal::FromUnixDuration(Duration d); + friend constexpr Duration time_internal::ToUnixDuration(Time t); + friend constexpr bool operator<(Time lhs, Time rhs); + friend constexpr bool operator==(Time lhs, Time rhs); + friend Duration operator-(Time lhs, Time rhs); + friend constexpr Time UniversalEpoch(); + friend constexpr Time InfiniteFuture(); + friend constexpr Time InfinitePast(); + constexpr explicit Time(Duration rep) : rep_(rep) {} + Duration rep_; +}; + +// Relational Operators +constexpr bool operator<(Time lhs, Time rhs) { return lhs.rep_ < rhs.rep_; } +constexpr bool operator>(Time lhs, Time rhs) { return rhs < lhs; } +constexpr bool operator>=(Time lhs, Time rhs) { return !(lhs < rhs); } +constexpr bool operator<=(Time lhs, Time rhs) { return !(rhs < lhs); } +constexpr bool operator==(Time lhs, Time rhs) { return lhs.rep_ == rhs.rep_; } +constexpr bool operator!=(Time lhs, Time rhs) { return !(lhs == rhs); } + +// Additive Operators +inline Time operator+(Time lhs, Duration rhs) { return lhs += rhs; } +inline Time operator+(Duration lhs, Time rhs) { return rhs += lhs; } +inline Time operator-(Time lhs, Duration rhs) { return lhs -= rhs; } +inline Duration operator-(Time lhs, Time rhs) { return lhs.rep_ - rhs.rep_; } + +// UnixEpoch() +// +// Returns the `absl::Time` representing "1970-01-01 00:00:00.0 +0000". +constexpr Time UnixEpoch() { return Time(); } + +// UniversalEpoch() +// +// Returns the `absl::Time` representing "0001-01-01 00:00:00.0 +0000", the +// epoch of the ICU Universal Time Scale. +constexpr Time UniversalEpoch() { + // 719162 is the number of days from 0001-01-01 to 1970-01-01, + // assuming the Gregorian calendar. + return Time(time_internal::MakeDuration(-24 * 719162 * int64_t{3600}, 0U)); +} + +// InfiniteFuture() +// +// Returns an `absl::Time` that is infinitely far in the future. +constexpr Time InfiniteFuture() { + return Time( + time_internal::MakeDuration((std::numeric_limits<int64_t>::max)(), ~0U)); +} + +// InfinitePast() +// +// Returns an `absl::Time` that is infinitely far in the past. +constexpr Time InfinitePast() { + return Time( + time_internal::MakeDuration((std::numeric_limits<int64_t>::min)(), ~0U)); +} + +// FromUnixNanos() +// FromUnixMicros() +// FromUnixMillis() +// FromUnixSeconds() +// FromTimeT() +// FromUDate() +// FromUniversal() +// +// Creates an `absl::Time` from a variety of other representations. +constexpr Time FromUnixNanos(int64_t ns); +constexpr Time FromUnixMicros(int64_t us); +constexpr Time FromUnixMillis(int64_t ms); +constexpr Time FromUnixSeconds(int64_t s); +constexpr Time FromTimeT(time_t t); +Time FromUDate(double udate); +Time FromUniversal(int64_t universal); + +// ToUnixNanos() +// ToUnixMicros() +// ToUnixMillis() +// ToUnixSeconds() +// ToTimeT() +// ToUDate() +// ToUniversal() +// +// Converts an `absl::Time` to a variety of other representations. Note that +// these operations round down toward negative infinity where necessary to +// adjust to the resolution of the result type. Beware of possible time_t +// over/underflow in ToTime{T,val,spec}() on 32-bit platforms. +int64_t ToUnixNanos(Time t); +int64_t ToUnixMicros(Time t); +int64_t ToUnixMillis(Time t); +int64_t ToUnixSeconds(Time t); +time_t ToTimeT(Time t); +double ToUDate(Time t); +int64_t ToUniversal(Time t); + +// DurationFromTimespec() +// DurationFromTimeval() +// ToTimespec() +// ToTimeval() +// TimeFromTimespec() +// TimeFromTimeval() +// ToTimespec() +// ToTimeval() +// +// Some APIs use a timespec or a timeval as a Duration (e.g., nanosleep(2) +// and select(2)), while others use them as a Time (e.g. clock_gettime(2) +// and gettimeofday(2)), so conversion functions are provided for both cases. +// The "to timespec/val" direction is easily handled via overloading, but +// for "from timespec/val" the desired type is part of the function name. +Duration DurationFromTimespec(timespec ts); +Duration DurationFromTimeval(timeval tv); +timespec ToTimespec(Duration d); +timeval ToTimeval(Duration d); +Time TimeFromTimespec(timespec ts); +Time TimeFromTimeval(timeval tv); +timespec ToTimespec(Time t); +timeval ToTimeval(Time t); + +// FromChrono() +// +// Converts a std::chrono::system_clock::time_point to an absl::Time. +// +// Example: +// +// auto tp = std::chrono::system_clock::from_time_t(123); +// absl::Time t = absl::FromChrono(tp); +// // t == absl::FromTimeT(123) +Time FromChrono(const std::chrono::system_clock::time_point& tp); + +// ToChronoTime() +// +// Converts an absl::Time to a std::chrono::system_clock::time_point. If +// overflow would occur, the returned value will saturate at the min/max time +// point value instead. +// +// Example: +// +// absl::Time t = absl::FromTimeT(123); +// auto tp = absl::ToChronoTime(t); +// // tp == std::chrono::system_clock::from_time_t(123); +std::chrono::system_clock::time_point ToChronoTime(Time); + +// Support for flag values of type Time. Time flags must be specified in a +// format that matches absl::RFC3339_full. For example: +// +// --start_time=2016-01-02T03:04:05.678+08:00 +// +// Note: A UTC offset (or 'Z' indicating a zero-offset from UTC) is required. +// +// Additionally, if you'd like to specify a time as a count of +// seconds/milliseconds/etc from the Unix epoch, use an absl::Duration flag +// and add that duration to absl::UnixEpoch() to get an absl::Time. +bool AbslParseFlag(absl::string_view text, Time* t, std::string* error); +std::string AbslUnparseFlag(Time t); +ABSL_DEPRECATED("Use AbslParseFlag() instead.") +bool ParseFlag(const std::string& text, Time* t, std::string* error); +ABSL_DEPRECATED("Use AbslUnparseFlag() instead.") +std::string UnparseFlag(Time t); + +// TimeZone +// +// The `absl::TimeZone` is an opaque, small, value-type class representing a +// geo-political region within which particular rules are used for converting +// between absolute and civil times (see https://git.io/v59Ly). `absl::TimeZone` +// values are named using the TZ identifiers from the IANA Time Zone Database, +// such as "America/Los_Angeles" or "Australia/Sydney". `absl::TimeZone` values +// are created from factory functions such as `absl::LoadTimeZone()`. Note: +// strings like "PST" and "EDT" are not valid TZ identifiers. Prefer to pass by +// value rather than const reference. +// +// For more on the fundamental concepts of time zones, absolute times, and civil +// times, see https://github.com/google/cctz#fundamental-concepts +// +// Examples: +// +// absl::TimeZone utc = absl::UTCTimeZone(); +// absl::TimeZone pst = absl::FixedTimeZone(-8 * 60 * 60); +// absl::TimeZone loc = absl::LocalTimeZone(); +// absl::TimeZone lax; +// if (!absl::LoadTimeZone("America/Los_Angeles", &lax)) { +// // handle error case +// } +// +// See also: +// - https://github.com/google/cctz +// - https://www.iana.org/time-zones +// - https://en.wikipedia.org/wiki/Zoneinfo +class TimeZone { + public: + explicit TimeZone(time_internal::cctz::time_zone tz) : cz_(tz) {} + TimeZone() = default; // UTC, but prefer UTCTimeZone() to be explicit. + + // Copyable. + TimeZone(const TimeZone&) = default; + TimeZone& operator=(const TimeZone&) = default; + + explicit operator time_internal::cctz::time_zone() const { return cz_; } + + std::string name() const { return cz_.name(); } + + // TimeZone::CivilInfo + // + // Information about the civil time corresponding to an absolute time. + // This struct is not intended to represent an instant in time. So, rather + // than passing a `TimeZone::CivilInfo` to a function, pass an `absl::Time` + // and an `absl::TimeZone`. + struct CivilInfo { + CivilSecond cs; + Duration subsecond; + + // Note: The following fields exist for backward compatibility + // with older APIs. Accessing these fields directly is a sign of + // imprudent logic in the calling code. Modern time-related code + // should only access this data indirectly by way of FormatTime(). + // These fields are undefined for InfiniteFuture() and InfinitePast(). + int offset; // seconds east of UTC + bool is_dst; // is offset non-standard? + const char* zone_abbr; // time-zone abbreviation (e.g., "PST") + }; + + // TimeZone::At(Time) + // + // Returns the civil time for this TimeZone at a certain `absl::Time`. + // If the input time is infinite, the output civil second will be set to + // CivilSecond::max() or min(), and the subsecond will be infinite. + // + // Example: + // + // const auto epoch = lax.At(absl::UnixEpoch()); + // // epoch.cs == 1969-12-31 16:00:00 + // // epoch.subsecond == absl::ZeroDuration() + // // epoch.offset == -28800 + // // epoch.is_dst == false + // // epoch.abbr == "PST" + CivilInfo At(Time t) const; + + // TimeZone::TimeInfo + // + // Information about the absolute times corresponding to a civil time. + // (Subseconds must be handled separately.) + // + // It is possible for a caller to pass a civil-time value that does + // not represent an actual or unique instant in time (due to a shift + // in UTC offset in the TimeZone, which results in a discontinuity in + // the civil-time components). For example, a daylight-saving-time + // transition skips or repeats civil times---in the United States, + // March 13, 2011 02:15 never occurred, while November 6, 2011 01:15 + // occurred twice---so requests for such times are not well-defined. + // To account for these possibilities, `absl::TimeZone::TimeInfo` is + // richer than just a single `absl::Time`. + struct TimeInfo { + enum CivilKind { + UNIQUE, // the civil time was singular (pre == trans == post) + SKIPPED, // the civil time did not exist (pre >= trans > post) + REPEATED, // the civil time was ambiguous (pre < trans <= post) + } kind; + Time pre; // time calculated using the pre-transition offset + Time trans; // when the civil-time discontinuity occurred + Time post; // time calculated using the post-transition offset + }; + + // TimeZone::At(CivilSecond) + // + // Returns an `absl::TimeInfo` containing the absolute time(s) for this + // TimeZone at an `absl::CivilSecond`. When the civil time is skipped or + // repeated, returns times calculated using the pre-transition and post- + // transition UTC offsets, plus the transition time itself. + // + // Examples: + // + // // A unique civil time + // const auto jan01 = lax.At(absl::CivilSecond(2011, 1, 1, 0, 0, 0)); + // // jan01.kind == TimeZone::TimeInfo::UNIQUE + // // jan01.pre is 2011-01-01 00:00:00 -0800 + // // jan01.trans is 2011-01-01 00:00:00 -0800 + // // jan01.post is 2011-01-01 00:00:00 -0800 + // + // // A Spring DST transition, when there is a gap in civil time + // const auto mar13 = lax.At(absl::CivilSecond(2011, 3, 13, 2, 15, 0)); + // // mar13.kind == TimeZone::TimeInfo::SKIPPED + // // mar13.pre is 2011-03-13 03:15:00 -0700 + // // mar13.trans is 2011-03-13 03:00:00 -0700 + // // mar13.post is 2011-03-13 01:15:00 -0800 + // + // // A Fall DST transition, when civil times are repeated + // const auto nov06 = lax.At(absl::CivilSecond(2011, 11, 6, 1, 15, 0)); + // // nov06.kind == TimeZone::TimeInfo::REPEATED + // // nov06.pre is 2011-11-06 01:15:00 -0700 + // // nov06.trans is 2011-11-06 01:00:00 -0800 + // // nov06.post is 2011-11-06 01:15:00 -0800 + TimeInfo At(CivilSecond ct) const; + + // TimeZone::NextTransition() + // TimeZone::PrevTransition() + // + // Finds the time of the next/previous offset change in this time zone. + // + // By definition, `NextTransition(t, &trans)` returns false when `t` is + // `InfiniteFuture()`, and `PrevTransition(t, &trans)` returns false + // when `t` is `InfinitePast()`. If the zone has no transitions, the + // result will also be false no matter what the argument. + // + // Otherwise, when `t` is `InfinitePast()`, `NextTransition(t, &trans)` + // returns true and sets `trans` to the first recorded transition. Chains + // of calls to `NextTransition()/PrevTransition()` will eventually return + // false, but it is unspecified exactly when `NextTransition(t, &trans)` + // jumps to false, or what time is set by `PrevTransition(t, &trans)` for + // a very distant `t`. + // + // Note: Enumeration of time-zone transitions is for informational purposes + // only. Modern time-related code should not care about when offset changes + // occur. + // + // Example: + // absl::TimeZone nyc; + // if (!absl::LoadTimeZone("America/New_York", &nyc)) { ... } + // const auto now = absl::Now(); + // auto t = absl::InfinitePast(); + // absl::TimeZone::CivilTransition trans; + // while (t <= now && nyc.NextTransition(t, &trans)) { + // // transition: trans.from -> trans.to + // t = nyc.At(trans.to).trans; + // } + struct CivilTransition { + CivilSecond from; // the civil time we jump from + CivilSecond to; // the civil time we jump to + }; + bool NextTransition(Time t, CivilTransition* trans) const; + bool PrevTransition(Time t, CivilTransition* trans) const; + + template <typename H> + friend H AbslHashValue(H h, TimeZone tz) { + return H::combine(std::move(h), tz.cz_); + } + + private: + friend bool operator==(TimeZone a, TimeZone b) { return a.cz_ == b.cz_; } + friend bool operator!=(TimeZone a, TimeZone b) { return a.cz_ != b.cz_; } + friend std::ostream& operator<<(std::ostream& os, TimeZone tz) { + return os << tz.name(); + } + + time_internal::cctz::time_zone cz_; +}; + +// LoadTimeZone() +// +// Loads the named zone. May perform I/O on the initial load of the named +// zone. If the name is invalid, or some other kind of error occurs, returns +// `false` and `*tz` is set to the UTC time zone. +inline bool LoadTimeZone(absl::string_view name, TimeZone* tz) { + if (name == "localtime") { + *tz = TimeZone(time_internal::cctz::local_time_zone()); + return true; + } + time_internal::cctz::time_zone cz; + const bool b = time_internal::cctz::load_time_zone(std::string(name), &cz); + *tz = TimeZone(cz); + return b; +} + +// FixedTimeZone() +// +// Returns a TimeZone that is a fixed offset (seconds east) from UTC. +// Note: If the absolute value of the offset is greater than 24 hours +// you'll get UTC (i.e., no offset) instead. +inline TimeZone FixedTimeZone(int seconds) { + return TimeZone( + time_internal::cctz::fixed_time_zone(std::chrono::seconds(seconds))); +} + +// UTCTimeZone() +// +// Convenience method returning the UTC time zone. +inline TimeZone UTCTimeZone() { + return TimeZone(time_internal::cctz::utc_time_zone()); +} + +// LocalTimeZone() +// +// Convenience method returning the local time zone, or UTC if there is +// no configured local zone. Warning: Be wary of using LocalTimeZone(), +// and particularly so in a server process, as the zone configured for the +// local machine should be irrelevant. Prefer an explicit zone name. +inline TimeZone LocalTimeZone() { + return TimeZone(time_internal::cctz::local_time_zone()); +} + +// ToCivilSecond() +// ToCivilMinute() +// ToCivilHour() +// ToCivilDay() +// ToCivilMonth() +// ToCivilYear() +// +// Helpers for TimeZone::At(Time) to return particularly aligned civil times. +// +// Example: +// +// absl::Time t = ...; +// absl::TimeZone tz = ...; +// const auto cd = absl::ToCivilDay(t, tz); +inline CivilSecond ToCivilSecond(Time t, TimeZone tz) { + return tz.At(t).cs; // already a CivilSecond +} +inline CivilMinute ToCivilMinute(Time t, TimeZone tz) { + return CivilMinute(tz.At(t).cs); +} +inline CivilHour ToCivilHour(Time t, TimeZone tz) { + return CivilHour(tz.At(t).cs); +} +inline CivilDay ToCivilDay(Time t, TimeZone tz) { + return CivilDay(tz.At(t).cs); +} +inline CivilMonth ToCivilMonth(Time t, TimeZone tz) { + return CivilMonth(tz.At(t).cs); +} +inline CivilYear ToCivilYear(Time t, TimeZone tz) { + return CivilYear(tz.At(t).cs); +} + +// FromCivil() +// +// Helper for TimeZone::At(CivilSecond) that provides "order-preserving +// semantics." If the civil time maps to a unique time, that time is +// returned. If the civil time is repeated in the given time zone, the +// time using the pre-transition offset is returned. Otherwise, the +// civil time is skipped in the given time zone, and the transition time +// is returned. This means that for any two civil times, ct1 and ct2, +// (ct1 < ct2) => (FromCivil(ct1) <= FromCivil(ct2)), the equal case +// being when two non-existent civil times map to the same transition time. +// +// Note: Accepts civil times of any alignment. +inline Time FromCivil(CivilSecond ct, TimeZone tz) { + const auto ti = tz.At(ct); + if (ti.kind == TimeZone::TimeInfo::SKIPPED) return ti.trans; + return ti.pre; +} + +// TimeConversion +// +// An `absl::TimeConversion` represents the conversion of year, month, day, +// hour, minute, and second values (i.e., a civil time), in a particular +// `absl::TimeZone`, to a time instant (an absolute time), as returned by +// `absl::ConvertDateTime()`. Legacy version of `absl::TimeZone::TimeInfo`. +// +// Deprecated. Use `absl::TimeZone::TimeInfo`. +struct + TimeConversion { + Time pre; // time calculated using the pre-transition offset + Time trans; // when the civil-time discontinuity occurred + Time post; // time calculated using the post-transition offset + + enum Kind { + UNIQUE, // the civil time was singular (pre == trans == post) + SKIPPED, // the civil time did not exist + REPEATED, // the civil time was ambiguous + }; + Kind kind; + + bool normalized; // input values were outside their valid ranges +}; + +// ConvertDateTime() +// +// Legacy version of `absl::TimeZone::At(absl::CivilSecond)` that takes +// the civil time as six, separate values (YMDHMS). +// +// The input month, day, hour, minute, and second values can be outside +// of their valid ranges, in which case they will be "normalized" during +// the conversion. +// +// Example: +// +// // "October 32" normalizes to "November 1". +// absl::TimeConversion tc = +// absl::ConvertDateTime(2013, 10, 32, 8, 30, 0, lax); +// // tc.kind == TimeConversion::UNIQUE && tc.normalized == true +// // absl::ToCivilDay(tc.pre, tz).month() == 11 +// // absl::ToCivilDay(tc.pre, tz).day() == 1 +// +// Deprecated. Use `absl::TimeZone::At(CivilSecond)`. +TimeConversion ConvertDateTime(int64_t year, int mon, int day, int hour, + int min, int sec, TimeZone tz); + +// FromDateTime() +// +// A convenience wrapper for `absl::ConvertDateTime()` that simply returns +// the "pre" `absl::Time`. That is, the unique result, or the instant that +// is correct using the pre-transition offset (as if the transition never +// happened). +// +// Example: +// +// absl::Time t = absl::FromDateTime(2017, 9, 26, 9, 30, 0, lax); +// // t = 2017-09-26 09:30:00 -0700 +// +// Deprecated. Use `absl::FromCivil(CivilSecond, TimeZone)`. Note that the +// behavior of `FromCivil()` differs from `FromDateTime()` for skipped civil +// times. If you care about that see `absl::TimeZone::At(absl::CivilSecond)`. +inline Time FromDateTime(int64_t year, int mon, int day, int hour, + int min, int sec, TimeZone tz) { + return ConvertDateTime(year, mon, day, hour, min, sec, tz).pre; +} + +// FromTM() +// +// Converts the `tm_year`, `tm_mon`, `tm_mday`, `tm_hour`, `tm_min`, and +// `tm_sec` fields to an `absl::Time` using the given time zone. See ctime(3) +// for a description of the expected values of the tm fields. If the indicated +// time instant is not unique (see `absl::TimeZone::At(absl::CivilSecond)` +// above), the `tm_isdst` field is consulted to select the desired instant +// (`tm_isdst` > 0 means DST, `tm_isdst` == 0 means no DST, `tm_isdst` < 0 +// means use the post-transition offset). +Time FromTM(const struct tm& tm, TimeZone tz); + +// ToTM() +// +// Converts the given `absl::Time` to a struct tm using the given time zone. +// See ctime(3) for a description of the values of the tm fields. +struct tm ToTM(Time t, TimeZone tz); + +// RFC3339_full +// RFC3339_sec +// +// FormatTime()/ParseTime() format specifiers for RFC3339 date/time strings, +// with trailing zeros trimmed or with fractional seconds omitted altogether. +// +// Note that RFC3339_sec[] matches an ISO 8601 extended format for date and +// time with UTC offset. Also note the use of "%Y": RFC3339 mandates that +// years have exactly four digits, but we allow them to take their natural +// width. +ABSL_DLL extern const char RFC3339_full[]; // %Y-%m-%d%ET%H:%M:%E*S%Ez +ABSL_DLL extern const char RFC3339_sec[]; // %Y-%m-%d%ET%H:%M:%S%Ez + +// RFC1123_full +// RFC1123_no_wday +// +// FormatTime()/ParseTime() format specifiers for RFC1123 date/time strings. +ABSL_DLL extern const char RFC1123_full[]; // %a, %d %b %E4Y %H:%M:%S %z +ABSL_DLL extern const char RFC1123_no_wday[]; // %d %b %E4Y %H:%M:%S %z + +// FormatTime() +// +// Formats the given `absl::Time` in the `absl::TimeZone` according to the +// provided format string. Uses strftime()-like formatting options, with +// the following extensions: +// +// - %Ez - RFC3339-compatible numeric UTC offset (+hh:mm or -hh:mm) +// - %E*z - Full-resolution numeric UTC offset (+hh:mm:ss or -hh:mm:ss) +// - %E#S - Seconds with # digits of fractional precision +// - %E*S - Seconds with full fractional precision (a literal '*') +// - %E#f - Fractional seconds with # digits of precision +// - %E*f - Fractional seconds with full precision (a literal '*') +// - %E4Y - Four-character years (-999 ... -001, 0000, 0001 ... 9999) +// - %ET - The RFC3339 "date-time" separator "T" +// +// Note that %E0S behaves like %S, and %E0f produces no characters. In +// contrast %E*f always produces at least one digit, which may be '0'. +// +// Note that %Y produces as many characters as it takes to fully render the +// year. A year outside of [-999:9999] when formatted with %E4Y will produce +// more than four characters, just like %Y. +// +// We recommend that format strings include the UTC offset (%z, %Ez, or %E*z) +// so that the result uniquely identifies a time instant. +// +// Example: +// +// absl::CivilSecond cs(2013, 1, 2, 3, 4, 5); +// absl::Time t = absl::FromCivil(cs, lax); +// std::string f = absl::FormatTime("%H:%M:%S", t, lax); // "03:04:05" +// f = absl::FormatTime("%H:%M:%E3S", t, lax); // "03:04:05.000" +// +// Note: If the given `absl::Time` is `absl::InfiniteFuture()`, the returned +// string will be exactly "infinite-future". If the given `absl::Time` is +// `absl::InfinitePast()`, the returned string will be exactly "infinite-past". +// In both cases the given format string and `absl::TimeZone` are ignored. +// +std::string FormatTime(absl::string_view format, Time t, TimeZone tz); + +// Convenience functions that format the given time using the RFC3339_full +// format. The first overload uses the provided TimeZone, while the second +// uses LocalTimeZone(). +std::string FormatTime(Time t, TimeZone tz); +std::string FormatTime(Time t); + +// Output stream operator. +inline std::ostream& operator<<(std::ostream& os, Time t) { + return os << FormatTime(t); +} + +// ParseTime() +// +// Parses an input string according to the provided format string and +// returns the corresponding `absl::Time`. Uses strftime()-like formatting +// options, with the same extensions as FormatTime(), but with the +// exceptions that %E#S is interpreted as %E*S, and %E#f as %E*f. %Ez +// and %E*z also accept the same inputs, which (along with %z) includes +// 'z' and 'Z' as synonyms for +00:00. %ET accepts either 'T' or 't'. +// +// %Y consumes as many numeric characters as it can, so the matching data +// should always be terminated with a non-numeric. %E4Y always consumes +// exactly four characters, including any sign. +// +// Unspecified fields are taken from the default date and time of ... +// +// "1970-01-01 00:00:00.0 +0000" +// +// For example, parsing a string of "15:45" (%H:%M) will return an absl::Time +// that represents "1970-01-01 15:45:00.0 +0000". +// +// Note that since ParseTime() returns time instants, it makes the most sense +// to parse fully-specified date/time strings that include a UTC offset (%z, +// %Ez, or %E*z). +// +// Note also that `absl::ParseTime()` only heeds the fields year, month, day, +// hour, minute, (fractional) second, and UTC offset. Other fields, like +// weekday (%a or %A), while parsed for syntactic validity, are ignored +// in the conversion. +// +// Date and time fields that are out-of-range will be treated as errors +// rather than normalizing them like `absl::CivilSecond` does. For example, +// it is an error to parse the date "Oct 32, 2013" because 32 is out of range. +// +// A leap second of ":60" is normalized to ":00" of the following minute +// with fractional seconds discarded. The following table shows how the +// given seconds and subseconds will be parsed: +// +// "59.x" -> 59.x // exact +// "60.x" -> 00.0 // normalized +// "00.x" -> 00.x // exact +// +// Errors are indicated by returning false and assigning an error message +// to the "err" out param if it is non-null. +// +// Note: If the input string is exactly "infinite-future", the returned +// `absl::Time` will be `absl::InfiniteFuture()` and `true` will be returned. +// If the input string is "infinite-past", the returned `absl::Time` will be +// `absl::InfinitePast()` and `true` will be returned. +// +bool ParseTime(absl::string_view format, absl::string_view input, Time* time, + std::string* err); + +// Like ParseTime() above, but if the format string does not contain a UTC +// offset specification (%z/%Ez/%E*z) then the input is interpreted in the +// given TimeZone. This means that the input, by itself, does not identify a +// unique instant. Being time-zone dependent, it also admits the possibility +// of ambiguity or non-existence, in which case the "pre" time (as defined +// by TimeZone::TimeInfo) is returned. For these reasons we recommend that +// all date/time strings include a UTC offset so they're context independent. +bool ParseTime(absl::string_view format, absl::string_view input, TimeZone tz, + Time* time, std::string* err); + +// ============================================================================ +// Implementation Details Follow +// ============================================================================ + +namespace time_internal { + +// Creates a Duration with a given representation. +// REQUIRES: hi,lo is a valid representation of a Duration as specified +// in time/duration.cc. +constexpr Duration MakeDuration(int64_t hi, uint32_t lo = 0) { + return Duration(hi, lo); +} + +constexpr Duration MakeDuration(int64_t hi, int64_t lo) { + return MakeDuration(hi, static_cast<uint32_t>(lo)); +} + +// Make a Duration value from a floating-point number, as long as that number +// is in the range [ 0 .. numeric_limits<int64_t>::max ), that is, as long as +// it's positive and can be converted to int64_t without risk of UB. +inline Duration MakePosDoubleDuration(double n) { + const int64_t int_secs = static_cast<int64_t>(n); + const uint32_t ticks = static_cast<uint32_t>( + (n - static_cast<double>(int_secs)) * kTicksPerSecond + 0.5); + return ticks < kTicksPerSecond + ? MakeDuration(int_secs, ticks) + : MakeDuration(int_secs + 1, ticks - kTicksPerSecond); +} + +// Creates a normalized Duration from an almost-normalized (sec,ticks) +// pair. sec may be positive or negative. ticks must be in the range +// -kTicksPerSecond < *ticks < kTicksPerSecond. If ticks is negative it +// will be normalized to a positive value in the resulting Duration. +constexpr Duration MakeNormalizedDuration(int64_t sec, int64_t ticks) { + return (ticks < 0) ? MakeDuration(sec - 1, ticks + kTicksPerSecond) + : MakeDuration(sec, ticks); +} + +// Provide access to the Duration representation. +constexpr int64_t GetRepHi(Duration d) { return d.rep_hi_; } +constexpr uint32_t GetRepLo(Duration d) { return d.rep_lo_; } + +// Returns true iff d is positive or negative infinity. +constexpr bool IsInfiniteDuration(Duration d) { return GetRepLo(d) == ~0U; } + +// Returns an infinite Duration with the opposite sign. +// REQUIRES: IsInfiniteDuration(d) +constexpr Duration OppositeInfinity(Duration d) { + return GetRepHi(d) < 0 + ? MakeDuration((std::numeric_limits<int64_t>::max)(), ~0U) + : MakeDuration((std::numeric_limits<int64_t>::min)(), ~0U); +} + +// Returns (-n)-1 (equivalently -(n+1)) without avoidable overflow. +constexpr int64_t NegateAndSubtractOne(int64_t n) { + // Note: Good compilers will optimize this expression to ~n when using + // a two's-complement representation (which is required for int64_t). + return (n < 0) ? -(n + 1) : (-n) - 1; +} + +// Map between a Time and a Duration since the Unix epoch. Note that these +// functions depend on the above mentioned choice of the Unix epoch for the +// Time representation (and both need to be Time friends). Without this +// knowledge, we would need to add-in/subtract-out UnixEpoch() respectively. +constexpr Time FromUnixDuration(Duration d) { return Time(d); } +constexpr Duration ToUnixDuration(Time t) { return t.rep_; } + +template <std::intmax_t N> +constexpr Duration FromInt64(int64_t v, std::ratio<1, N>) { + static_assert(0 < N && N <= 1000 * 1000 * 1000, "Unsupported ratio"); + // Subsecond ratios cannot overflow. + return MakeNormalizedDuration( + v / N, v % N * kTicksPerNanosecond * 1000 * 1000 * 1000 / N); +} +constexpr Duration FromInt64(int64_t v, std::ratio<60>) { + return (v <= (std::numeric_limits<int64_t>::max)() / 60 && + v >= (std::numeric_limits<int64_t>::min)() / 60) + ? MakeDuration(v * 60) + : v > 0 ? InfiniteDuration() : -InfiniteDuration(); +} +constexpr Duration FromInt64(int64_t v, std::ratio<3600>) { + return (v <= (std::numeric_limits<int64_t>::max)() / 3600 && + v >= (std::numeric_limits<int64_t>::min)() / 3600) + ? MakeDuration(v * 3600) + : v > 0 ? InfiniteDuration() : -InfiniteDuration(); +} + +// IsValidRep64<T>(0) is true if the expression `int64_t{std::declval<T>()}` is +// valid. That is, if a T can be assigned to an int64_t without narrowing. +template <typename T> +constexpr auto IsValidRep64(int) -> decltype(int64_t{std::declval<T>()} == 0) { + return true; +} +template <typename T> +constexpr auto IsValidRep64(char) -> bool { + return false; +} + +// Converts a std::chrono::duration to an absl::Duration. +template <typename Rep, typename Period> +constexpr Duration FromChrono(const std::chrono::duration<Rep, Period>& d) { + static_assert(IsValidRep64<Rep>(0), "duration::rep is invalid"); + return FromInt64(int64_t{d.count()}, Period{}); +} + +template <typename Ratio> +int64_t ToInt64(Duration d, Ratio) { + // Note: This may be used on MSVC, which may have a system_clock period of + // std::ratio<1, 10 * 1000 * 1000> + return ToInt64Seconds(d * Ratio::den / Ratio::num); +} +// Fastpath implementations for the 6 common duration units. +inline int64_t ToInt64(Duration d, std::nano) { + return ToInt64Nanoseconds(d); +} +inline int64_t ToInt64(Duration d, std::micro) { + return ToInt64Microseconds(d); +} +inline int64_t ToInt64(Duration d, std::milli) { + return ToInt64Milliseconds(d); +} +inline int64_t ToInt64(Duration d, std::ratio<1>) { + return ToInt64Seconds(d); +} +inline int64_t ToInt64(Duration d, std::ratio<60>) { + return ToInt64Minutes(d); +} +inline int64_t ToInt64(Duration d, std::ratio<3600>) { + return ToInt64Hours(d); +} + +// Converts an absl::Duration to a chrono duration of type T. +template <typename T> +T ToChronoDuration(Duration d) { + using Rep = typename T::rep; + using Period = typename T::period; + static_assert(IsValidRep64<Rep>(0), "duration::rep is invalid"); + if (time_internal::IsInfiniteDuration(d)) + return d < ZeroDuration() ? (T::min)() : (T::max)(); + const auto v = ToInt64(d, Period{}); + if (v > (std::numeric_limits<Rep>::max)()) return (T::max)(); + if (v < (std::numeric_limits<Rep>::min)()) return (T::min)(); + return T{v}; +} + +} // namespace time_internal + +constexpr Duration Nanoseconds(int64_t n) { + return time_internal::FromInt64(n, std::nano{}); +} +constexpr Duration Microseconds(int64_t n) { + return time_internal::FromInt64(n, std::micro{}); +} +constexpr Duration Milliseconds(int64_t n) { + return time_internal::FromInt64(n, std::milli{}); +} +constexpr Duration Seconds(int64_t n) { + return time_internal::FromInt64(n, std::ratio<1>{}); +} +constexpr Duration Minutes(int64_t n) { + return time_internal::FromInt64(n, std::ratio<60>{}); +} +constexpr Duration Hours(int64_t n) { + return time_internal::FromInt64(n, std::ratio<3600>{}); +} + +constexpr bool operator<(Duration lhs, Duration rhs) { + return time_internal::GetRepHi(lhs) != time_internal::GetRepHi(rhs) + ? time_internal::GetRepHi(lhs) < time_internal::GetRepHi(rhs) + : time_internal::GetRepHi(lhs) == (std::numeric_limits<int64_t>::min)() + ? time_internal::GetRepLo(lhs) + 1 < + time_internal::GetRepLo(rhs) + 1 + : time_internal::GetRepLo(lhs) < time_internal::GetRepLo(rhs); +} + +constexpr bool operator==(Duration lhs, Duration rhs) { + return time_internal::GetRepHi(lhs) == time_internal::GetRepHi(rhs) && + time_internal::GetRepLo(lhs) == time_internal::GetRepLo(rhs); +} + +constexpr Duration operator-(Duration d) { + // This is a little interesting because of the special cases. + // + // If rep_lo_ is zero, we have it easy; it's safe to negate rep_hi_, we're + // dealing with an integral number of seconds, and the only special case is + // the maximum negative finite duration, which can't be negated. + // + // Infinities stay infinite, and just change direction. + // + // Finally we're in the case where rep_lo_ is non-zero, and we can borrow + // a second's worth of ticks and avoid overflow (as negating int64_t-min + 1 + // is safe). + return time_internal::GetRepLo(d) == 0 + ? time_internal::GetRepHi(d) == + (std::numeric_limits<int64_t>::min)() + ? InfiniteDuration() + : time_internal::MakeDuration(-time_internal::GetRepHi(d)) + : time_internal::IsInfiniteDuration(d) + ? time_internal::OppositeInfinity(d) + : time_internal::MakeDuration( + time_internal::NegateAndSubtractOne( + time_internal::GetRepHi(d)), + time_internal::kTicksPerSecond - + time_internal::GetRepLo(d)); +} + +constexpr Duration InfiniteDuration() { + return time_internal::MakeDuration((std::numeric_limits<int64_t>::max)(), + ~0U); +} + +constexpr Duration FromChrono(const std::chrono::nanoseconds& d) { + return time_internal::FromChrono(d); +} +constexpr Duration FromChrono(const std::chrono::microseconds& d) { + return time_internal::FromChrono(d); +} +constexpr Duration FromChrono(const std::chrono::milliseconds& d) { + return time_internal::FromChrono(d); +} +constexpr Duration FromChrono(const std::chrono::seconds& d) { + return time_internal::FromChrono(d); +} +constexpr Duration FromChrono(const std::chrono::minutes& d) { + return time_internal::FromChrono(d); +} +constexpr Duration FromChrono(const std::chrono::hours& d) { + return time_internal::FromChrono(d); +} + +constexpr Time FromUnixNanos(int64_t ns) { + return time_internal::FromUnixDuration(Nanoseconds(ns)); +} + +constexpr Time FromUnixMicros(int64_t us) { + return time_internal::FromUnixDuration(Microseconds(us)); +} + +constexpr Time FromUnixMillis(int64_t ms) { + return time_internal::FromUnixDuration(Milliseconds(ms)); +} + +constexpr Time FromUnixSeconds(int64_t s) { + return time_internal::FromUnixDuration(Seconds(s)); +} + +constexpr Time FromTimeT(time_t t) { + return time_internal::FromUnixDuration(Seconds(t)); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TIME_TIME_H_ diff --git a/third_party/abseil_cpp/absl/time/time_benchmark.cc b/third_party/abseil_cpp/absl/time/time_benchmark.cc new file mode 100644 index 000000000000..99e6279984e6 --- /dev/null +++ b/third_party/abseil_cpp/absl/time/time_benchmark.cc @@ -0,0 +1,316 @@ +// Copyright 2018 The Abseil Authors. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/time.h" + +#if !defined(_WIN32) +#include <sys/time.h> +#endif // _WIN32 +#include <algorithm> +#include <cmath> +#include <cstddef> +#include <cstring> +#include <ctime> +#include <memory> +#include <string> + +#include "absl/time/clock.h" +#include "absl/time/internal/test_util.h" +#include "benchmark/benchmark.h" + +namespace { + +// +// Addition/Subtraction of a duration +// + +void BM_Time_Arithmetic(benchmark::State& state) { + const absl::Duration nano = absl::Nanoseconds(1); + const absl::Duration sec = absl::Seconds(1); + absl::Time t = absl::UnixEpoch(); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(t += nano); + benchmark::DoNotOptimize(t -= sec); + } +} +BENCHMARK(BM_Time_Arithmetic); + +// +// Time difference +// + +void BM_Time_Difference(benchmark::State& state) { + absl::Time start = absl::Now(); + absl::Time end = start + absl::Nanoseconds(1); + absl::Duration diff; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(diff += end - start); + } +} +BENCHMARK(BM_Time_Difference); + +// +// ToDateTime +// +// In each "ToDateTime" benchmark we switch between two instants +// separated by at least one transition in order to defeat any +// internal caching of previous results (e.g., see local_time_hint_). +// +// The "UTC" variants use UTC instead of the Google/local time zone. +// + +void BM_Time_ToDateTime_Absl(benchmark::State& state) { + const absl::TimeZone tz = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + absl::Time t = absl::FromUnixSeconds(1384569027); + absl::Time t2 = absl::FromUnixSeconds(1418962578); + while (state.KeepRunning()) { + std::swap(t, t2); + t += absl::Seconds(1); + benchmark::DoNotOptimize(t.In(tz)); + } +} +BENCHMARK(BM_Time_ToDateTime_Absl); + +void BM_Time_ToDateTime_Libc(benchmark::State& state) { + // No timezone support, so just use localtime. + time_t t = 1384569027; + time_t t2 = 1418962578; + while (state.KeepRunning()) { + std::swap(t, t2); + t += 1; + struct tm tm; +#if !defined(_WIN32) + benchmark::DoNotOptimize(localtime_r(&t, &tm)); +#else // _WIN32 + benchmark::DoNotOptimize(localtime_s(&tm, &t)); +#endif // _WIN32 + } +} +BENCHMARK(BM_Time_ToDateTime_Libc); + +void BM_Time_ToDateTimeUTC_Absl(benchmark::State& state) { + const absl::TimeZone tz = absl::UTCTimeZone(); + absl::Time t = absl::FromUnixSeconds(1384569027); + while (state.KeepRunning()) { + t += absl::Seconds(1); + benchmark::DoNotOptimize(t.In(tz)); + } +} +BENCHMARK(BM_Time_ToDateTimeUTC_Absl); + +void BM_Time_ToDateTimeUTC_Libc(benchmark::State& state) { + time_t t = 1384569027; + while (state.KeepRunning()) { + t += 1; + struct tm tm; +#if !defined(_WIN32) + benchmark::DoNotOptimize(gmtime_r(&t, &tm)); +#else // _WIN32 + benchmark::DoNotOptimize(gmtime_s(&tm, &t)); +#endif // _WIN32 + } +} +BENCHMARK(BM_Time_ToDateTimeUTC_Libc); + +// +// FromUnixMicros +// + +void BM_Time_FromUnixMicros(benchmark::State& state) { + int i = 0; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::FromUnixMicros(i)); + ++i; + } +} +BENCHMARK(BM_Time_FromUnixMicros); + +void BM_Time_ToUnixNanos(benchmark::State& state) { + const absl::Time t = absl::UnixEpoch() + absl::Seconds(123); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(ToUnixNanos(t)); + } +} +BENCHMARK(BM_Time_ToUnixNanos); + +void BM_Time_ToUnixMicros(benchmark::State& state) { + const absl::Time t = absl::UnixEpoch() + absl::Seconds(123); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(ToUnixMicros(t)); + } +} +BENCHMARK(BM_Time_ToUnixMicros); + +void BM_Time_ToUnixMillis(benchmark::State& state) { + const absl::Time t = absl::UnixEpoch() + absl::Seconds(123); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(ToUnixMillis(t)); + } +} +BENCHMARK(BM_Time_ToUnixMillis); + +void BM_Time_ToUnixSeconds(benchmark::State& state) { + const absl::Time t = absl::UnixEpoch() + absl::Seconds(123); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToUnixSeconds(t)); + } +} +BENCHMARK(BM_Time_ToUnixSeconds); + +// +// FromCivil +// +// In each "FromCivil" benchmark we switch between two YMDhms values +// separated by at least one transition in order to defeat any internal +// caching of previous results (e.g., see time_local_hint_). +// +// The "UTC" variants use UTC instead of the Google/local time zone. +// The "Day0" variants require normalization of the day of month. +// + +void BM_Time_FromCivil_Absl(benchmark::State& state) { + const absl::TimeZone tz = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + int i = 0; + while (state.KeepRunning()) { + if ((i & 1) == 0) { + absl::FromCivil(absl::CivilSecond(2014, 12, 18, 20, 16, 18), tz); + } else { + absl::FromCivil(absl::CivilSecond(2013, 11, 15, 18, 30, 27), tz); + } + ++i; + } +} +BENCHMARK(BM_Time_FromCivil_Absl); + +void BM_Time_FromCivil_Libc(benchmark::State& state) { + // No timezone support, so just use localtime. + int i = 0; + while (state.KeepRunning()) { + struct tm tm; + if ((i & 1) == 0) { + tm.tm_year = 2014 - 1900; + tm.tm_mon = 12 - 1; + tm.tm_mday = 18; + tm.tm_hour = 20; + tm.tm_min = 16; + tm.tm_sec = 18; + } else { + tm.tm_year = 2013 - 1900; + tm.tm_mon = 11 - 1; + tm.tm_mday = 15; + tm.tm_hour = 18; + tm.tm_min = 30; + tm.tm_sec = 27; + } + tm.tm_isdst = -1; + mktime(&tm); + ++i; + } +} +BENCHMARK(BM_Time_FromCivil_Libc); + +void BM_Time_FromCivilUTC_Absl(benchmark::State& state) { + const absl::TimeZone tz = absl::UTCTimeZone(); + while (state.KeepRunning()) { + absl::FromCivil(absl::CivilSecond(2014, 12, 18, 20, 16, 18), tz); + } +} +BENCHMARK(BM_Time_FromCivilUTC_Absl); + +void BM_Time_FromCivilDay0_Absl(benchmark::State& state) { + const absl::TimeZone tz = + absl::time_internal::LoadTimeZone("America/Los_Angeles"); + int i = 0; + while (state.KeepRunning()) { + if ((i & 1) == 0) { + absl::FromCivil(absl::CivilSecond(2014, 12, 0, 20, 16, 18), tz); + } else { + absl::FromCivil(absl::CivilSecond(2013, 11, 0, 18, 30, 27), tz); + } + ++i; + } +} +BENCHMARK(BM_Time_FromCivilDay0_Absl); + +void BM_Time_FromCivilDay0_Libc(benchmark::State& state) { + // No timezone support, so just use localtime. + int i = 0; + while (state.KeepRunning()) { + struct tm tm; + if ((i & 1) == 0) { + tm.tm_year = 2014 - 1900; + tm.tm_mon = 12 - 1; + tm.tm_mday = 0; + tm.tm_hour = 20; + tm.tm_min = 16; + tm.tm_sec = 18; + } else { + tm.tm_year = 2013 - 1900; + tm.tm_mon = 11 - 1; + tm.tm_mday = 0; + tm.tm_hour = 18; + tm.tm_min = 30; + tm.tm_sec = 27; + } + tm.tm_isdst = -1; + mktime(&tm); + ++i; + } +} +BENCHMARK(BM_Time_FromCivilDay0_Libc); + +// +// To/FromTimespec +// + +void BM_Time_ToTimespec(benchmark::State& state) { + absl::Time now = absl::Now(); + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::ToTimespec(now)); + } +} +BENCHMARK(BM_Time_ToTimespec); + +void BM_Time_FromTimespec(benchmark::State& state) { + timespec ts = absl::ToTimespec(absl::Now()); + while (state.KeepRunning()) { + if (++ts.tv_nsec == 1000 * 1000 * 1000) { + ++ts.tv_sec; + ts.tv_nsec = 0; + } + benchmark::DoNotOptimize(absl::TimeFromTimespec(ts)); + } +} +BENCHMARK(BM_Time_FromTimespec); + +// +// Comparison with InfiniteFuture/Past +// + +void BM_Time_InfiniteFuture(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::InfiniteFuture()); + } +} +BENCHMARK(BM_Time_InfiniteFuture); + +void BM_Time_InfinitePast(benchmark::State& state) { + while (state.KeepRunning()) { + benchmark::DoNotOptimize(absl::InfinitePast()); + } +} +BENCHMARK(BM_Time_InfinitePast); + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/time_test.cc b/third_party/abseil_cpp/absl/time/time_test.cc new file mode 100644 index 000000000000..cde9423feb1a --- /dev/null +++ b/third_party/abseil_cpp/absl/time/time_test.cc @@ -0,0 +1,1280 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/time.h" + +#if defined(_MSC_VER) +#include <winsock2.h> // for timeval +#endif + +#include <chrono> // NOLINT(build/c++11) +#include <cstring> +#include <ctime> +#include <iomanip> +#include <limits> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/numeric/int128.h" +#include "absl/time/clock.h" +#include "absl/time/internal/test_util.h" + +namespace { + +#if defined(GTEST_USES_SIMPLE_RE) && GTEST_USES_SIMPLE_RE +const char kZoneAbbrRE[] = ".*"; // just punt +#else +const char kZoneAbbrRE[] = "[A-Za-z]{3,4}|[-+][0-9]{2}([0-9]{2})?"; +#endif + +// This helper is a macro so that failed expectations show up with the +// correct line numbers. +#define EXPECT_CIVIL_INFO(ci, y, m, d, h, min, s, off, isdst) \ + do { \ + EXPECT_EQ(y, ci.cs.year()); \ + EXPECT_EQ(m, ci.cs.month()); \ + EXPECT_EQ(d, ci.cs.day()); \ + EXPECT_EQ(h, ci.cs.hour()); \ + EXPECT_EQ(min, ci.cs.minute()); \ + EXPECT_EQ(s, ci.cs.second()); \ + EXPECT_EQ(off, ci.offset); \ + EXPECT_EQ(isdst, ci.is_dst); \ + EXPECT_THAT(ci.zone_abbr, testing::MatchesRegex(kZoneAbbrRE)); \ + } while (0) + +// A gMock matcher to match timespec values. Use this matcher like: +// timespec ts1, ts2; +// EXPECT_THAT(ts1, TimespecMatcher(ts2)); +MATCHER_P(TimespecMatcher, ts, "") { + if (ts.tv_sec == arg.tv_sec && ts.tv_nsec == arg.tv_nsec) return true; + *result_listener << "expected: {" << ts.tv_sec << ", " << ts.tv_nsec << "} "; + *result_listener << "actual: {" << arg.tv_sec << ", " << arg.tv_nsec << "}"; + return false; +} + +// A gMock matcher to match timeval values. Use this matcher like: +// timeval tv1, tv2; +// EXPECT_THAT(tv1, TimevalMatcher(tv2)); +MATCHER_P(TimevalMatcher, tv, "") { + if (tv.tv_sec == arg.tv_sec && tv.tv_usec == arg.tv_usec) return true; + *result_listener << "expected: {" << tv.tv_sec << ", " << tv.tv_usec << "} "; + *result_listener << "actual: {" << arg.tv_sec << ", " << arg.tv_usec << "}"; + return false; +} + +TEST(Time, ConstExpr) { + constexpr absl::Time t0 = absl::UnixEpoch(); + static_assert(t0 == absl::Time(), "UnixEpoch"); + constexpr absl::Time t1 = absl::InfiniteFuture(); + static_assert(t1 != absl::Time(), "InfiniteFuture"); + constexpr absl::Time t2 = absl::InfinitePast(); + static_assert(t2 != absl::Time(), "InfinitePast"); + constexpr absl::Time t3 = absl::FromUnixNanos(0); + static_assert(t3 == absl::Time(), "FromUnixNanos"); + constexpr absl::Time t4 = absl::FromUnixMicros(0); + static_assert(t4 == absl::Time(), "FromUnixMicros"); + constexpr absl::Time t5 = absl::FromUnixMillis(0); + static_assert(t5 == absl::Time(), "FromUnixMillis"); + constexpr absl::Time t6 = absl::FromUnixSeconds(0); + static_assert(t6 == absl::Time(), "FromUnixSeconds"); + constexpr absl::Time t7 = absl::FromTimeT(0); + static_assert(t7 == absl::Time(), "FromTimeT"); +} + +TEST(Time, ValueSemantics) { + absl::Time a; // Default construction + absl::Time b = a; // Copy construction + EXPECT_EQ(a, b); + absl::Time c(a); // Copy construction (again) + EXPECT_EQ(a, b); + EXPECT_EQ(a, c); + EXPECT_EQ(b, c); + b = c; // Assignment + EXPECT_EQ(a, b); + EXPECT_EQ(a, c); + EXPECT_EQ(b, c); +} + +TEST(Time, UnixEpoch) { + const auto ci = absl::UTCTimeZone().At(absl::UnixEpoch()); + EXPECT_EQ(absl::CivilSecond(1970, 1, 1, 0, 0, 0), ci.cs); + EXPECT_EQ(absl::ZeroDuration(), ci.subsecond); + EXPECT_EQ(absl::Weekday::thursday, absl::GetWeekday(ci.cs)); +} + +TEST(Time, Breakdown) { + absl::TimeZone tz = absl::time_internal::LoadTimeZone("America/New_York"); + absl::Time t = absl::UnixEpoch(); + + // The Unix epoch as seen in NYC. + auto ci = tz.At(t); + EXPECT_CIVIL_INFO(ci, 1969, 12, 31, 19, 0, 0, -18000, false); + EXPECT_EQ(absl::ZeroDuration(), ci.subsecond); + EXPECT_EQ(absl::Weekday::wednesday, absl::GetWeekday(ci.cs)); + + // Just before the epoch. + t -= absl::Nanoseconds(1); + ci = tz.At(t); + EXPECT_CIVIL_INFO(ci, 1969, 12, 31, 18, 59, 59, -18000, false); + EXPECT_EQ(absl::Nanoseconds(999999999), ci.subsecond); + EXPECT_EQ(absl::Weekday::wednesday, absl::GetWeekday(ci.cs)); + + // Some time later. + t += absl::Hours(24) * 2735; + t += absl::Hours(18) + absl::Minutes(30) + absl::Seconds(15) + + absl::Nanoseconds(9); + ci = tz.At(t); + EXPECT_CIVIL_INFO(ci, 1977, 6, 28, 14, 30, 15, -14400, true); + EXPECT_EQ(8, ci.subsecond / absl::Nanoseconds(1)); + EXPECT_EQ(absl::Weekday::tuesday, absl::GetWeekday(ci.cs)); +} + +TEST(Time, AdditiveOperators) { + const absl::Duration d = absl::Nanoseconds(1); + const absl::Time t0; + const absl::Time t1 = t0 + d; + + EXPECT_EQ(d, t1 - t0); + EXPECT_EQ(-d, t0 - t1); + EXPECT_EQ(t0, t1 - d); + + absl::Time t(t0); + EXPECT_EQ(t0, t); + t += d; + EXPECT_EQ(t0 + d, t); + EXPECT_EQ(d, t - t0); + t -= d; + EXPECT_EQ(t0, t); + + // Tests overflow between subseconds and seconds. + t = absl::UnixEpoch(); + t += absl::Milliseconds(500); + EXPECT_EQ(absl::UnixEpoch() + absl::Milliseconds(500), t); + t += absl::Milliseconds(600); + EXPECT_EQ(absl::UnixEpoch() + absl::Milliseconds(1100), t); + t -= absl::Milliseconds(600); + EXPECT_EQ(absl::UnixEpoch() + absl::Milliseconds(500), t); + t -= absl::Milliseconds(500); + EXPECT_EQ(absl::UnixEpoch(), t); +} + +TEST(Time, RelationalOperators) { + constexpr absl::Time t1 = absl::FromUnixNanos(0); + constexpr absl::Time t2 = absl::FromUnixNanos(1); + constexpr absl::Time t3 = absl::FromUnixNanos(2); + + static_assert(absl::Time() == t1, ""); + static_assert(t1 == t1, ""); + static_assert(t2 == t2, ""); + static_assert(t3 == t3, ""); + + static_assert(t1 < t2, ""); + static_assert(t2 < t3, ""); + static_assert(t1 < t3, ""); + + static_assert(t1 <= t1, ""); + static_assert(t1 <= t2, ""); + static_assert(t2 <= t2, ""); + static_assert(t2 <= t3, ""); + static_assert(t3 <= t3, ""); + static_assert(t1 <= t3, ""); + + static_assert(t2 > t1, ""); + static_assert(t3 > t2, ""); + static_assert(t3 > t1, ""); + + static_assert(t2 >= t2, ""); + static_assert(t2 >= t1, ""); + static_assert(t3 >= t3, ""); + static_assert(t3 >= t2, ""); + static_assert(t1 >= t1, ""); + static_assert(t3 >= t1, ""); +} + +TEST(Time, Infinity) { + constexpr absl::Time ifuture = absl::InfiniteFuture(); + constexpr absl::Time ipast = absl::InfinitePast(); + + static_assert(ifuture == ifuture, ""); + static_assert(ipast == ipast, ""); + static_assert(ipast < ifuture, ""); + static_assert(ifuture > ipast, ""); + + // Arithmetic saturates + EXPECT_EQ(ifuture, ifuture + absl::Seconds(1)); + EXPECT_EQ(ifuture, ifuture - absl::Seconds(1)); + EXPECT_EQ(ipast, ipast + absl::Seconds(1)); + EXPECT_EQ(ipast, ipast - absl::Seconds(1)); + + EXPECT_EQ(absl::InfiniteDuration(), ifuture - ifuture); + EXPECT_EQ(absl::InfiniteDuration(), ifuture - ipast); + EXPECT_EQ(-absl::InfiniteDuration(), ipast - ifuture); + EXPECT_EQ(-absl::InfiniteDuration(), ipast - ipast); + + constexpr absl::Time t = absl::UnixEpoch(); // Any finite time. + static_assert(t < ifuture, ""); + static_assert(t > ipast, ""); + + EXPECT_EQ(ifuture, t + absl::InfiniteDuration()); + EXPECT_EQ(ipast, t - absl::InfiniteDuration()); +} + +TEST(Time, FloorConversion) { +#define TEST_FLOOR_CONVERSION(TO, FROM) \ + EXPECT_EQ(1, TO(FROM(1001))); \ + EXPECT_EQ(1, TO(FROM(1000))); \ + EXPECT_EQ(0, TO(FROM(999))); \ + EXPECT_EQ(0, TO(FROM(1))); \ + EXPECT_EQ(0, TO(FROM(0))); \ + EXPECT_EQ(-1, TO(FROM(-1))); \ + EXPECT_EQ(-1, TO(FROM(-999))); \ + EXPECT_EQ(-1, TO(FROM(-1000))); \ + EXPECT_EQ(-2, TO(FROM(-1001))); + + TEST_FLOOR_CONVERSION(absl::ToUnixMicros, absl::FromUnixNanos); + TEST_FLOOR_CONVERSION(absl::ToUnixMillis, absl::FromUnixMicros); + TEST_FLOOR_CONVERSION(absl::ToUnixSeconds, absl::FromUnixMillis); + TEST_FLOOR_CONVERSION(absl::ToTimeT, absl::FromUnixMillis); + +#undef TEST_FLOOR_CONVERSION + + // Tests ToUnixNanos. + EXPECT_EQ(1, absl::ToUnixNanos(absl::UnixEpoch() + absl::Nanoseconds(3) / 2)); + EXPECT_EQ(1, absl::ToUnixNanos(absl::UnixEpoch() + absl::Nanoseconds(1))); + EXPECT_EQ(0, absl::ToUnixNanos(absl::UnixEpoch() + absl::Nanoseconds(1) / 2)); + EXPECT_EQ(0, absl::ToUnixNanos(absl::UnixEpoch() + absl::Nanoseconds(0))); + EXPECT_EQ(-1, + absl::ToUnixNanos(absl::UnixEpoch() - absl::Nanoseconds(1) / 2)); + EXPECT_EQ(-1, absl::ToUnixNanos(absl::UnixEpoch() - absl::Nanoseconds(1))); + EXPECT_EQ(-2, + absl::ToUnixNanos(absl::UnixEpoch() - absl::Nanoseconds(3) / 2)); + + // Tests ToUniversal, which uses a different epoch than the tests above. + EXPECT_EQ(1, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(101))); + EXPECT_EQ(1, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(100))); + EXPECT_EQ(0, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(99))); + EXPECT_EQ(0, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(1))); + EXPECT_EQ(0, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(0))); + EXPECT_EQ(-1, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(-1))); + EXPECT_EQ(-1, + absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(-99))); + EXPECT_EQ( + -1, absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(-100))); + EXPECT_EQ( + -2, absl::ToUniversal(absl::UniversalEpoch() + absl::Nanoseconds(-101))); + + // Tests ToTimespec()/TimeFromTimespec() + const struct { + absl::Time t; + timespec ts; + } to_ts[] = { + {absl::FromUnixSeconds(1) + absl::Nanoseconds(1), {1, 1}}, + {absl::FromUnixSeconds(1) + absl::Nanoseconds(1) / 2, {1, 0}}, + {absl::FromUnixSeconds(1) + absl::Nanoseconds(0), {1, 0}}, + {absl::FromUnixSeconds(0) + absl::Nanoseconds(0), {0, 0}}, + {absl::FromUnixSeconds(0) - absl::Nanoseconds(1) / 2, {-1, 999999999}}, + {absl::FromUnixSeconds(0) - absl::Nanoseconds(1), {-1, 999999999}}, + {absl::FromUnixSeconds(-1) + absl::Nanoseconds(1), {-1, 1}}, + {absl::FromUnixSeconds(-1) + absl::Nanoseconds(1) / 2, {-1, 0}}, + {absl::FromUnixSeconds(-1) + absl::Nanoseconds(0), {-1, 0}}, + {absl::FromUnixSeconds(-1) - absl::Nanoseconds(1) / 2, {-2, 999999999}}, + }; + for (const auto& test : to_ts) { + EXPECT_THAT(absl::ToTimespec(test.t), TimespecMatcher(test.ts)); + } + const struct { + timespec ts; + absl::Time t; + } from_ts[] = { + {{1, 1}, absl::FromUnixSeconds(1) + absl::Nanoseconds(1)}, + {{1, 0}, absl::FromUnixSeconds(1) + absl::Nanoseconds(0)}, + {{0, 0}, absl::FromUnixSeconds(0) + absl::Nanoseconds(0)}, + {{0, -1}, absl::FromUnixSeconds(0) - absl::Nanoseconds(1)}, + {{-1, 999999999}, absl::FromUnixSeconds(0) - absl::Nanoseconds(1)}, + {{-1, 1}, absl::FromUnixSeconds(-1) + absl::Nanoseconds(1)}, + {{-1, 0}, absl::FromUnixSeconds(-1) + absl::Nanoseconds(0)}, + {{-1, -1}, absl::FromUnixSeconds(-1) - absl::Nanoseconds(1)}, + {{-2, 999999999}, absl::FromUnixSeconds(-1) - absl::Nanoseconds(1)}, + }; + for (const auto& test : from_ts) { + EXPECT_EQ(test.t, absl::TimeFromTimespec(test.ts)); + } + + // Tests ToTimeval()/TimeFromTimeval() (same as timespec above) + const struct { + absl::Time t; + timeval tv; + } to_tv[] = { + {absl::FromUnixSeconds(1) + absl::Microseconds(1), {1, 1}}, + {absl::FromUnixSeconds(1) + absl::Microseconds(1) / 2, {1, 0}}, + {absl::FromUnixSeconds(1) + absl::Microseconds(0), {1, 0}}, + {absl::FromUnixSeconds(0) + absl::Microseconds(0), {0, 0}}, + {absl::FromUnixSeconds(0) - absl::Microseconds(1) / 2, {-1, 999999}}, + {absl::FromUnixSeconds(0) - absl::Microseconds(1), {-1, 999999}}, + {absl::FromUnixSeconds(-1) + absl::Microseconds(1), {-1, 1}}, + {absl::FromUnixSeconds(-1) + absl::Microseconds(1) / 2, {-1, 0}}, + {absl::FromUnixSeconds(-1) + absl::Microseconds(0), {-1, 0}}, + {absl::FromUnixSeconds(-1) - absl::Microseconds(1) / 2, {-2, 999999}}, + }; + for (const auto& test : to_tv) { + EXPECT_THAT(ToTimeval(test.t), TimevalMatcher(test.tv)); + } + const struct { + timeval tv; + absl::Time t; + } from_tv[] = { + {{1, 1}, absl::FromUnixSeconds(1) + absl::Microseconds(1)}, + {{1, 0}, absl::FromUnixSeconds(1) + absl::Microseconds(0)}, + {{0, 0}, absl::FromUnixSeconds(0) + absl::Microseconds(0)}, + {{0, -1}, absl::FromUnixSeconds(0) - absl::Microseconds(1)}, + {{-1, 999999}, absl::FromUnixSeconds(0) - absl::Microseconds(1)}, + {{-1, 1}, absl::FromUnixSeconds(-1) + absl::Microseconds(1)}, + {{-1, 0}, absl::FromUnixSeconds(-1) + absl::Microseconds(0)}, + {{-1, -1}, absl::FromUnixSeconds(-1) - absl::Microseconds(1)}, + {{-2, 999999}, absl::FromUnixSeconds(-1) - absl::Microseconds(1)}, + }; + for (const auto& test : from_tv) { + EXPECT_EQ(test.t, absl::TimeFromTimeval(test.tv)); + } + + // Tests flooring near negative infinity. + const int64_t min_plus_1 = std::numeric_limits<int64_t>::min() + 1; + EXPECT_EQ(min_plus_1, absl::ToUnixSeconds(absl::FromUnixSeconds(min_plus_1))); + EXPECT_EQ(std::numeric_limits<int64_t>::min(), + absl::ToUnixSeconds(absl::FromUnixSeconds(min_plus_1) - + absl::Nanoseconds(1) / 2)); + + // Tests flooring near positive infinity. + EXPECT_EQ(std::numeric_limits<int64_t>::max(), + absl::ToUnixSeconds( + absl::FromUnixSeconds(std::numeric_limits<int64_t>::max()) + + absl::Nanoseconds(1) / 2)); + EXPECT_EQ(std::numeric_limits<int64_t>::max(), + absl::ToUnixSeconds( + absl::FromUnixSeconds(std::numeric_limits<int64_t>::max()))); + EXPECT_EQ(std::numeric_limits<int64_t>::max() - 1, + absl::ToUnixSeconds( + absl::FromUnixSeconds(std::numeric_limits<int64_t>::max()) - + absl::Nanoseconds(1) / 2)); +} + +TEST(Time, RoundtripConversion) { +#define TEST_CONVERSION_ROUND_TRIP(SOURCE, FROM, TO, MATCHER) \ + EXPECT_THAT(TO(FROM(SOURCE)), MATCHER(SOURCE)) + + // FromUnixNanos() and ToUnixNanos() + int64_t now_ns = absl::GetCurrentTimeNanos(); + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromUnixNanos, absl::ToUnixNanos, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromUnixNanos, absl::ToUnixNanos, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromUnixNanos, absl::ToUnixNanos, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(now_ns, absl::FromUnixNanos, absl::ToUnixNanos, + testing::Eq) + << now_ns; + + // FromUnixMicros() and ToUnixMicros() + int64_t now_us = absl::GetCurrentTimeNanos() / 1000; + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromUnixMicros, absl::ToUnixMicros, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromUnixMicros, absl::ToUnixMicros, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromUnixMicros, absl::ToUnixMicros, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(now_us, absl::FromUnixMicros, absl::ToUnixMicros, + testing::Eq) + << now_us; + + // FromUnixMillis() and ToUnixMillis() + int64_t now_ms = absl::GetCurrentTimeNanos() / 1000000; + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromUnixMillis, absl::ToUnixMillis, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromUnixMillis, absl::ToUnixMillis, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromUnixMillis, absl::ToUnixMillis, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(now_ms, absl::FromUnixMillis, absl::ToUnixMillis, + testing::Eq) + << now_ms; + + // FromUnixSeconds() and ToUnixSeconds() + int64_t now_s = std::time(nullptr); + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromUnixSeconds, absl::ToUnixSeconds, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromUnixSeconds, absl::ToUnixSeconds, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromUnixSeconds, absl::ToUnixSeconds, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(now_s, absl::FromUnixSeconds, absl::ToUnixSeconds, + testing::Eq) + << now_s; + + // FromTimeT() and ToTimeT() + time_t now_time_t = std::time(nullptr); + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromTimeT, absl::ToTimeT, testing::Eq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromTimeT, absl::ToTimeT, testing::Eq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromTimeT, absl::ToTimeT, testing::Eq); + TEST_CONVERSION_ROUND_TRIP(now_time_t, absl::FromTimeT, absl::ToTimeT, + testing::Eq) + << now_time_t; + + // TimeFromTimeval() and ToTimeval() + timeval tv; + tv.tv_sec = -1; + tv.tv_usec = 0; + TEST_CONVERSION_ROUND_TRIP(tv, absl::TimeFromTimeval, absl::ToTimeval, + TimevalMatcher); + tv.tv_sec = -1; + tv.tv_usec = 999999; + TEST_CONVERSION_ROUND_TRIP(tv, absl::TimeFromTimeval, absl::ToTimeval, + TimevalMatcher); + tv.tv_sec = 0; + tv.tv_usec = 0; + TEST_CONVERSION_ROUND_TRIP(tv, absl::TimeFromTimeval, absl::ToTimeval, + TimevalMatcher); + tv.tv_sec = 0; + tv.tv_usec = 1; + TEST_CONVERSION_ROUND_TRIP(tv, absl::TimeFromTimeval, absl::ToTimeval, + TimevalMatcher); + tv.tv_sec = 1; + tv.tv_usec = 0; + TEST_CONVERSION_ROUND_TRIP(tv, absl::TimeFromTimeval, absl::ToTimeval, + TimevalMatcher); + + // TimeFromTimespec() and ToTimespec() + timespec ts; + ts.tv_sec = -1; + ts.tv_nsec = 0; + TEST_CONVERSION_ROUND_TRIP(ts, absl::TimeFromTimespec, absl::ToTimespec, + TimespecMatcher); + ts.tv_sec = -1; + ts.tv_nsec = 999999999; + TEST_CONVERSION_ROUND_TRIP(ts, absl::TimeFromTimespec, absl::ToTimespec, + TimespecMatcher); + ts.tv_sec = 0; + ts.tv_nsec = 0; + TEST_CONVERSION_ROUND_TRIP(ts, absl::TimeFromTimespec, absl::ToTimespec, + TimespecMatcher); + ts.tv_sec = 0; + ts.tv_nsec = 1; + TEST_CONVERSION_ROUND_TRIP(ts, absl::TimeFromTimespec, absl::ToTimespec, + TimespecMatcher); + ts.tv_sec = 1; + ts.tv_nsec = 0; + TEST_CONVERSION_ROUND_TRIP(ts, absl::TimeFromTimespec, absl::ToTimespec, + TimespecMatcher); + + // FromUDate() and ToUDate() + double now_ud = absl::GetCurrentTimeNanos() / 1000000; + TEST_CONVERSION_ROUND_TRIP(-1.5, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(-0.5, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(0.5, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(1.5, absl::FromUDate, absl::ToUDate, + testing::DoubleEq); + TEST_CONVERSION_ROUND_TRIP(now_ud, absl::FromUDate, absl::ToUDate, + testing::DoubleEq) + << std::fixed << std::setprecision(17) << now_ud; + + // FromUniversal() and ToUniversal() + int64_t now_uni = ((719162LL * (24 * 60 * 60)) * (1000 * 1000 * 10)) + + (absl::GetCurrentTimeNanos() / 100); + TEST_CONVERSION_ROUND_TRIP(-1, absl::FromUniversal, absl::ToUniversal, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(0, absl::FromUniversal, absl::ToUniversal, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(1, absl::FromUniversal, absl::ToUniversal, + testing::Eq); + TEST_CONVERSION_ROUND_TRIP(now_uni, absl::FromUniversal, absl::ToUniversal, + testing::Eq) + << now_uni; + +#undef TEST_CONVERSION_ROUND_TRIP +} + +template <typename Duration> +std::chrono::system_clock::time_point MakeChronoUnixTime(const Duration& d) { + return std::chrono::system_clock::from_time_t(0) + d; +} + +TEST(Time, FromChrono) { + EXPECT_EQ(absl::FromTimeT(-1), + absl::FromChrono(std::chrono::system_clock::from_time_t(-1))); + EXPECT_EQ(absl::FromTimeT(0), + absl::FromChrono(std::chrono::system_clock::from_time_t(0))); + EXPECT_EQ(absl::FromTimeT(1), + absl::FromChrono(std::chrono::system_clock::from_time_t(1))); + + EXPECT_EQ( + absl::FromUnixMillis(-1), + absl::FromChrono(MakeChronoUnixTime(std::chrono::milliseconds(-1)))); + EXPECT_EQ(absl::FromUnixMillis(0), + absl::FromChrono(MakeChronoUnixTime(std::chrono::milliseconds(0)))); + EXPECT_EQ(absl::FromUnixMillis(1), + absl::FromChrono(MakeChronoUnixTime(std::chrono::milliseconds(1)))); + + // Chrono doesn't define exactly its range and precision (neither does + // absl::Time), so let's simply test +/- ~100 years to make sure things work. + const auto century_sec = 60 * 60 * 24 * 365 * int64_t{100}; + const auto century = std::chrono::seconds(century_sec); + const auto chrono_future = MakeChronoUnixTime(century); + const auto chrono_past = MakeChronoUnixTime(-century); + EXPECT_EQ(absl::FromUnixSeconds(century_sec), + absl::FromChrono(chrono_future)); + EXPECT_EQ(absl::FromUnixSeconds(-century_sec), absl::FromChrono(chrono_past)); + + // Roundtrip them both back to chrono. + EXPECT_EQ(chrono_future, + absl::ToChronoTime(absl::FromUnixSeconds(century_sec))); + EXPECT_EQ(chrono_past, + absl::ToChronoTime(absl::FromUnixSeconds(-century_sec))); +} + +TEST(Time, ToChronoTime) { + EXPECT_EQ(std::chrono::system_clock::from_time_t(-1), + absl::ToChronoTime(absl::FromTimeT(-1))); + EXPECT_EQ(std::chrono::system_clock::from_time_t(0), + absl::ToChronoTime(absl::FromTimeT(0))); + EXPECT_EQ(std::chrono::system_clock::from_time_t(1), + absl::ToChronoTime(absl::FromTimeT(1))); + + EXPECT_EQ(MakeChronoUnixTime(std::chrono::milliseconds(-1)), + absl::ToChronoTime(absl::FromUnixMillis(-1))); + EXPECT_EQ(MakeChronoUnixTime(std::chrono::milliseconds(0)), + absl::ToChronoTime(absl::FromUnixMillis(0))); + EXPECT_EQ(MakeChronoUnixTime(std::chrono::milliseconds(1)), + absl::ToChronoTime(absl::FromUnixMillis(1))); + + // Time before the Unix epoch should floor, not trunc. + const auto tick = absl::Nanoseconds(1) / 4; + EXPECT_EQ(std::chrono::system_clock::from_time_t(0) - + std::chrono::system_clock::duration(1), + absl::ToChronoTime(absl::UnixEpoch() - tick)); +} + +// Check that absl::int128 works as a std::chrono::duration representation. +TEST(Time, Chrono128) { + // Define a std::chrono::time_point type whose time[sic]_since_epoch() is + // a signed 128-bit count of attoseconds. This has a range and resolution + // (currently) beyond those of absl::Time, and undoubtedly also beyond those + // of std::chrono::system_clock::time_point. + // + // Note: The to/from-chrono support should probably be updated to handle + // such wide representations. + using Timestamp = + std::chrono::time_point<std::chrono::system_clock, + std::chrono::duration<absl::int128, std::atto>>; + + // Expect that we can round-trip the std::chrono::system_clock::time_point + // extremes through both absl::Time and Timestamp, and that Timestamp can + // handle the (current) absl::Time extremes. + // + // Note: We should use std::chrono::floor() instead of time_point_cast(), + // but floor() is only available since c++17. + for (const auto tp : {std::chrono::system_clock::time_point::min(), + std::chrono::system_clock::time_point::max()}) { + EXPECT_EQ(tp, absl::ToChronoTime(absl::FromChrono(tp))); + EXPECT_EQ(tp, std::chrono::time_point_cast< + std::chrono::system_clock::time_point::duration>( + std::chrono::time_point_cast<Timestamp::duration>(tp))); + } + Timestamp::duration::rep v = std::numeric_limits<int64_t>::min(); + v *= Timestamp::duration::period::den; + auto ts = Timestamp(Timestamp::duration(v)); + ts += std::chrono::duration<int64_t, std::atto>(0); + EXPECT_EQ(std::numeric_limits<int64_t>::min(), + ts.time_since_epoch().count() / Timestamp::duration::period::den); + EXPECT_EQ(0, + ts.time_since_epoch().count() % Timestamp::duration::period::den); + v = std::numeric_limits<int64_t>::max(); + v *= Timestamp::duration::period::den; + ts = Timestamp(Timestamp::duration(v)); + ts += std::chrono::duration<int64_t, std::atto>(999999999750000000); + EXPECT_EQ(std::numeric_limits<int64_t>::max(), + ts.time_since_epoch().count() / Timestamp::duration::period::den); + EXPECT_EQ(999999999750000000, + ts.time_since_epoch().count() % Timestamp::duration::period::den); +} + +TEST(Time, TimeZoneAt) { + const absl::TimeZone nyc = + absl::time_internal::LoadTimeZone("America/New_York"); + const std::string fmt = "%a, %e %b %Y %H:%M:%S %z (%Z)"; + + // A non-transition where the civil time is unique. + absl::CivilSecond nov01(2013, 11, 1, 8, 30, 0); + const auto nov01_ci = nyc.At(nov01); + EXPECT_EQ(absl::TimeZone::TimeInfo::UNIQUE, nov01_ci.kind); + EXPECT_EQ("Fri, 1 Nov 2013 08:30:00 -0400 (EDT)", + absl::FormatTime(fmt, nov01_ci.pre, nyc)); + EXPECT_EQ(nov01_ci.pre, nov01_ci.trans); + EXPECT_EQ(nov01_ci.pre, nov01_ci.post); + EXPECT_EQ(nov01_ci.pre, absl::FromCivil(nov01, nyc)); + + // A Spring DST transition, when there is a gap in civil time + // and we prefer the later of the possible interpretations of a + // non-existent time. + absl::CivilSecond mar13(2011, 3, 13, 2, 15, 0); + const auto mar_ci = nyc.At(mar13); + EXPECT_EQ(absl::TimeZone::TimeInfo::SKIPPED, mar_ci.kind); + EXPECT_EQ("Sun, 13 Mar 2011 03:15:00 -0400 (EDT)", + absl::FormatTime(fmt, mar_ci.pre, nyc)); + EXPECT_EQ("Sun, 13 Mar 2011 03:00:00 -0400 (EDT)", + absl::FormatTime(fmt, mar_ci.trans, nyc)); + EXPECT_EQ("Sun, 13 Mar 2011 01:15:00 -0500 (EST)", + absl::FormatTime(fmt, mar_ci.post, nyc)); + EXPECT_EQ(mar_ci.trans, absl::FromCivil(mar13, nyc)); + + // A Fall DST transition, when civil times are repeated and + // we prefer the earlier of the possible interpretations of an + // ambiguous time. + absl::CivilSecond nov06(2011, 11, 6, 1, 15, 0); + const auto nov06_ci = nyc.At(nov06); + EXPECT_EQ(absl::TimeZone::TimeInfo::REPEATED, nov06_ci.kind); + EXPECT_EQ("Sun, 6 Nov 2011 01:15:00 -0400 (EDT)", + absl::FormatTime(fmt, nov06_ci.pre, nyc)); + EXPECT_EQ("Sun, 6 Nov 2011 01:00:00 -0500 (EST)", + absl::FormatTime(fmt, nov06_ci.trans, nyc)); + EXPECT_EQ("Sun, 6 Nov 2011 01:15:00 -0500 (EST)", + absl::FormatTime(fmt, nov06_ci.post, nyc)); + EXPECT_EQ(nov06_ci.pre, absl::FromCivil(nov06, nyc)); + + // Check that (time_t) -1 is handled correctly. + absl::CivilSecond minus1(1969, 12, 31, 18, 59, 59); + const auto minus1_cl = nyc.At(minus1); + EXPECT_EQ(absl::TimeZone::TimeInfo::UNIQUE, minus1_cl.kind); + EXPECT_EQ(-1, absl::ToTimeT(minus1_cl.pre)); + EXPECT_EQ("Wed, 31 Dec 1969 18:59:59 -0500 (EST)", + absl::FormatTime(fmt, minus1_cl.pre, nyc)); + EXPECT_EQ("Wed, 31 Dec 1969 23:59:59 +0000 (UTC)", + absl::FormatTime(fmt, minus1_cl.pre, absl::UTCTimeZone())); +} + +// FromCivil(CivilSecond(year, mon, day, hour, min, sec), UTCTimeZone()) +// has a specialized fastpath implementation, which we exercise here. +TEST(Time, FromCivilUTC) { + const absl::TimeZone utc = absl::UTCTimeZone(); + const std::string fmt = "%a, %e %b %Y %H:%M:%S %z (%Z)"; + const int kMax = std::numeric_limits<int>::max(); + const int kMin = std::numeric_limits<int>::min(); + absl::Time t; + + // 292091940881 is the last positive year to use the fastpath. + t = absl::FromCivil( + absl::CivilSecond(292091940881, kMax, kMax, kMax, kMax, kMax), utc); + EXPECT_EQ("Fri, 25 Nov 292277026596 12:21:07 +0000 (UTC)", + absl::FormatTime(fmt, t, utc)); + t = absl::FromCivil( + absl::CivilSecond(292091940882, kMax, kMax, kMax, kMax, kMax), utc); + EXPECT_EQ("infinite-future", absl::FormatTime(fmt, t, utc)); // no overflow + + // -292091936940 is the last negative year to use the fastpath. + t = absl::FromCivil( + absl::CivilSecond(-292091936940, kMin, kMin, kMin, kMin, kMin), utc); + EXPECT_EQ("Fri, 1 Nov -292277022657 10:37:52 +0000 (UTC)", + absl::FormatTime(fmt, t, utc)); + t = absl::FromCivil( + absl::CivilSecond(-292091936941, kMin, kMin, kMin, kMin, kMin), utc); + EXPECT_EQ("infinite-past", absl::FormatTime(fmt, t, utc)); // no underflow + + // Check that we're counting leap years correctly. + t = absl::FromCivil(absl::CivilSecond(1900, 2, 28, 23, 59, 59), utc); + EXPECT_EQ("Wed, 28 Feb 1900 23:59:59 +0000 (UTC)", + absl::FormatTime(fmt, t, utc)); + t = absl::FromCivil(absl::CivilSecond(1900, 3, 1, 0, 0, 0), utc); + EXPECT_EQ("Thu, 1 Mar 1900 00:00:00 +0000 (UTC)", + absl::FormatTime(fmt, t, utc)); + t = absl::FromCivil(absl::CivilSecond(2000, 2, 29, 23, 59, 59), utc); + EXPECT_EQ("Tue, 29 Feb 2000 23:59:59 +0000 (UTC)", + absl::FormatTime(fmt, t, utc)); + t = absl::FromCivil(absl::CivilSecond(2000, 3, 1, 0, 0, 0), utc); + EXPECT_EQ("Wed, 1 Mar 2000 00:00:00 +0000 (UTC)", + absl::FormatTime(fmt, t, utc)); +} + +TEST(Time, ToTM) { + const absl::TimeZone utc = absl::UTCTimeZone(); + + // Compares the results of ToTM() to gmtime_r() for lots of times over the + // course of a few days. + const absl::Time start = + absl::FromCivil(absl::CivilSecond(2014, 1, 2, 3, 4, 5), utc); + const absl::Time end = + absl::FromCivil(absl::CivilSecond(2014, 1, 5, 3, 4, 5), utc); + for (absl::Time t = start; t < end; t += absl::Seconds(30)) { + const struct tm tm_bt = ToTM(t, utc); + const time_t tt = absl::ToTimeT(t); + struct tm tm_lc; +#ifdef _WIN32 + gmtime_s(&tm_lc, &tt); +#else + gmtime_r(&tt, &tm_lc); +#endif + EXPECT_EQ(tm_lc.tm_year, tm_bt.tm_year); + EXPECT_EQ(tm_lc.tm_mon, tm_bt.tm_mon); + EXPECT_EQ(tm_lc.tm_mday, tm_bt.tm_mday); + EXPECT_EQ(tm_lc.tm_hour, tm_bt.tm_hour); + EXPECT_EQ(tm_lc.tm_min, tm_bt.tm_min); + EXPECT_EQ(tm_lc.tm_sec, tm_bt.tm_sec); + EXPECT_EQ(tm_lc.tm_wday, tm_bt.tm_wday); + EXPECT_EQ(tm_lc.tm_yday, tm_bt.tm_yday); + EXPECT_EQ(tm_lc.tm_isdst, tm_bt.tm_isdst); + + ASSERT_FALSE(HasFailure()); + } + + // Checks that the tm_isdst field is correct when in standard time. + const absl::TimeZone nyc = + absl::time_internal::LoadTimeZone("America/New_York"); + absl::Time t = absl::FromCivil(absl::CivilSecond(2014, 3, 1, 0, 0, 0), nyc); + struct tm tm = ToTM(t, nyc); + EXPECT_FALSE(tm.tm_isdst); + + // Checks that the tm_isdst field is correct when in daylight time. + t = absl::FromCivil(absl::CivilSecond(2014, 4, 1, 0, 0, 0), nyc); + tm = ToTM(t, nyc); + EXPECT_TRUE(tm.tm_isdst); + + // Checks overflow. + tm = ToTM(absl::InfiniteFuture(), nyc); + EXPECT_EQ(std::numeric_limits<int>::max() - 1900, tm.tm_year); + EXPECT_EQ(11, tm.tm_mon); + EXPECT_EQ(31, tm.tm_mday); + EXPECT_EQ(23, tm.tm_hour); + EXPECT_EQ(59, tm.tm_min); + EXPECT_EQ(59, tm.tm_sec); + EXPECT_EQ(4, tm.tm_wday); + EXPECT_EQ(364, tm.tm_yday); + EXPECT_FALSE(tm.tm_isdst); + + // Checks underflow. + tm = ToTM(absl::InfinitePast(), nyc); + EXPECT_EQ(std::numeric_limits<int>::min(), tm.tm_year); + EXPECT_EQ(0, tm.tm_mon); + EXPECT_EQ(1, tm.tm_mday); + EXPECT_EQ(0, tm.tm_hour); + EXPECT_EQ(0, tm.tm_min); + EXPECT_EQ(0, tm.tm_sec); + EXPECT_EQ(0, tm.tm_wday); + EXPECT_EQ(0, tm.tm_yday); + EXPECT_FALSE(tm.tm_isdst); +} + +TEST(Time, FromTM) { + const absl::TimeZone nyc = + absl::time_internal::LoadTimeZone("America/New_York"); + + // Verifies that tm_isdst doesn't affect anything when the time is unique. + struct tm tm; + std::memset(&tm, 0, sizeof(tm)); + tm.tm_year = 2014 - 1900; + tm.tm_mon = 6 - 1; + tm.tm_mday = 28; + tm.tm_hour = 1; + tm.tm_min = 2; + tm.tm_sec = 3; + tm.tm_isdst = -1; + absl::Time t = FromTM(tm, nyc); + EXPECT_EQ("2014-06-28T01:02:03-04:00", absl::FormatTime(t, nyc)); // DST + tm.tm_isdst = 0; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-06-28T01:02:03-04:00", absl::FormatTime(t, nyc)); // DST + tm.tm_isdst = 1; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-06-28T01:02:03-04:00", absl::FormatTime(t, nyc)); // DST + + // Adjusts tm to refer to an ambiguous time. + tm.tm_year = 2014 - 1900; + tm.tm_mon = 11 - 1; + tm.tm_mday = 2; + tm.tm_hour = 1; + tm.tm_min = 30; + tm.tm_sec = 42; + tm.tm_isdst = -1; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-11-02T01:30:42-04:00", absl::FormatTime(t, nyc)); // DST + tm.tm_isdst = 0; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-11-02T01:30:42-05:00", absl::FormatTime(t, nyc)); // STD + tm.tm_isdst = 1; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-11-02T01:30:42-04:00", absl::FormatTime(t, nyc)); // DST + + // Adjusts tm to refer to a skipped time. + tm.tm_year = 2014 - 1900; + tm.tm_mon = 3 - 1; + tm.tm_mday = 9; + tm.tm_hour = 2; + tm.tm_min = 30; + tm.tm_sec = 42; + tm.tm_isdst = -1; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-03-09T03:30:42-04:00", absl::FormatTime(t, nyc)); // DST + tm.tm_isdst = 0; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-03-09T01:30:42-05:00", absl::FormatTime(t, nyc)); // STD + tm.tm_isdst = 1; + t = FromTM(tm, nyc); + EXPECT_EQ("2014-03-09T03:30:42-04:00", absl::FormatTime(t, nyc)); // DST + + // Adjusts tm to refer to a time with a year larger than 2147483647. + tm.tm_year = 2147483647 - 1900 + 1; + tm.tm_mon = 6 - 1; + tm.tm_mday = 28; + tm.tm_hour = 1; + tm.tm_min = 2; + tm.tm_sec = 3; + tm.tm_isdst = -1; + t = FromTM(tm, absl::UTCTimeZone()); + EXPECT_EQ("2147483648-06-28T01:02:03+00:00", + absl::FormatTime(t, absl::UTCTimeZone())); + + // Adjusts tm to refer to a time with a very large month. + tm.tm_year = 2019 - 1900; + tm.tm_mon = 2147483647; + tm.tm_mday = 28; + tm.tm_hour = 1; + tm.tm_min = 2; + tm.tm_sec = 3; + tm.tm_isdst = -1; + t = FromTM(tm, absl::UTCTimeZone()); + EXPECT_EQ("178958989-08-28T01:02:03+00:00", + absl::FormatTime(t, absl::UTCTimeZone())); +} + +TEST(Time, TMRoundTrip) { + const absl::TimeZone nyc = + absl::time_internal::LoadTimeZone("America/New_York"); + + // Test round-tripping across a skipped transition + absl::Time start = absl::FromCivil(absl::CivilHour(2014, 3, 9, 0), nyc); + absl::Time end = absl::FromCivil(absl::CivilHour(2014, 3, 9, 4), nyc); + for (absl::Time t = start; t < end; t += absl::Minutes(1)) { + struct tm tm = ToTM(t, nyc); + absl::Time rt = FromTM(tm, nyc); + EXPECT_EQ(rt, t); + } + + // Test round-tripping across an ambiguous transition + start = absl::FromCivil(absl::CivilHour(2014, 11, 2, 0), nyc); + end = absl::FromCivil(absl::CivilHour(2014, 11, 2, 4), nyc); + for (absl::Time t = start; t < end; t += absl::Minutes(1)) { + struct tm tm = ToTM(t, nyc); + absl::Time rt = FromTM(tm, nyc); + EXPECT_EQ(rt, t); + } + + // Test round-tripping of unique instants crossing a day boundary + start = absl::FromCivil(absl::CivilHour(2014, 6, 27, 22), nyc); + end = absl::FromCivil(absl::CivilHour(2014, 6, 28, 4), nyc); + for (absl::Time t = start; t < end; t += absl::Minutes(1)) { + struct tm tm = ToTM(t, nyc); + absl::Time rt = FromTM(tm, nyc); + EXPECT_EQ(rt, t); + } +} + +TEST(Time, Range) { + // The API's documented range is +/- 100 billion years. + const absl::Duration range = absl::Hours(24) * 365.2425 * 100000000000; + + // Arithmetic and comparison still works at +/-range around base values. + absl::Time bases[2] = {absl::UnixEpoch(), absl::Now()}; + for (const auto base : bases) { + absl::Time bottom = base - range; + EXPECT_GT(bottom, bottom - absl::Nanoseconds(1)); + EXPECT_LT(bottom, bottom + absl::Nanoseconds(1)); + absl::Time top = base + range; + EXPECT_GT(top, top - absl::Nanoseconds(1)); + EXPECT_LT(top, top + absl::Nanoseconds(1)); + absl::Duration full_range = 2 * range; + EXPECT_EQ(full_range, top - bottom); + EXPECT_EQ(-full_range, bottom - top); + } +} + +TEST(Time, Limits) { + // It is an implementation detail that Time().rep_ == ZeroDuration(), + // and that the resolution of a Duration is 1/4 of a nanosecond. + const absl::Time zero; + const absl::Time max = + zero + absl::Seconds(std::numeric_limits<int64_t>::max()) + + absl::Nanoseconds(999999999) + absl::Nanoseconds(3) / 4; + const absl::Time min = + zero + absl::Seconds(std::numeric_limits<int64_t>::min()); + + // Some simple max/min bounds checks. + EXPECT_LT(max, absl::InfiniteFuture()); + EXPECT_GT(min, absl::InfinitePast()); + EXPECT_LT(zero, max); + EXPECT_GT(zero, min); + EXPECT_GE(absl::UnixEpoch(), min); + EXPECT_LT(absl::UnixEpoch(), max); + + // Check sign of Time differences. + EXPECT_LT(absl::ZeroDuration(), max - zero); + EXPECT_LT(absl::ZeroDuration(), + zero - absl::Nanoseconds(1) / 4 - min); // avoid zero - min + + // Arithmetic works at max - 0.25ns and min + 0.25ns. + EXPECT_GT(max, max - absl::Nanoseconds(1) / 4); + EXPECT_LT(min, min + absl::Nanoseconds(1) / 4); +} + +TEST(Time, ConversionSaturation) { + const absl::TimeZone utc = absl::UTCTimeZone(); + absl::Time t; + + const auto max_time_t = std::numeric_limits<time_t>::max(); + const auto min_time_t = std::numeric_limits<time_t>::min(); + time_t tt = max_time_t - 1; + t = absl::FromTimeT(tt); + tt = absl::ToTimeT(t); + EXPECT_EQ(max_time_t - 1, tt); + t += absl::Seconds(1); + tt = absl::ToTimeT(t); + EXPECT_EQ(max_time_t, tt); + t += absl::Seconds(1); // no effect + tt = absl::ToTimeT(t); + EXPECT_EQ(max_time_t, tt); + + tt = min_time_t + 1; + t = absl::FromTimeT(tt); + tt = absl::ToTimeT(t); + EXPECT_EQ(min_time_t + 1, tt); + t -= absl::Seconds(1); + tt = absl::ToTimeT(t); + EXPECT_EQ(min_time_t, tt); + t -= absl::Seconds(1); // no effect + tt = absl::ToTimeT(t); + EXPECT_EQ(min_time_t, tt); + + const auto max_timeval_sec = + std::numeric_limits<decltype(timeval::tv_sec)>::max(); + const auto min_timeval_sec = + std::numeric_limits<decltype(timeval::tv_sec)>::min(); + timeval tv; + tv.tv_sec = max_timeval_sec; + tv.tv_usec = 999998; + t = absl::TimeFromTimeval(tv); + tv = ToTimeval(t); + EXPECT_EQ(max_timeval_sec, tv.tv_sec); + EXPECT_EQ(999998, tv.tv_usec); + t += absl::Microseconds(1); + tv = ToTimeval(t); + EXPECT_EQ(max_timeval_sec, tv.tv_sec); + EXPECT_EQ(999999, tv.tv_usec); + t += absl::Microseconds(1); // no effect + tv = ToTimeval(t); + EXPECT_EQ(max_timeval_sec, tv.tv_sec); + EXPECT_EQ(999999, tv.tv_usec); + + tv.tv_sec = min_timeval_sec; + tv.tv_usec = 1; + t = absl::TimeFromTimeval(tv); + tv = ToTimeval(t); + EXPECT_EQ(min_timeval_sec, tv.tv_sec); + EXPECT_EQ(1, tv.tv_usec); + t -= absl::Microseconds(1); + tv = ToTimeval(t); + EXPECT_EQ(min_timeval_sec, tv.tv_sec); + EXPECT_EQ(0, tv.tv_usec); + t -= absl::Microseconds(1); // no effect + tv = ToTimeval(t); + EXPECT_EQ(min_timeval_sec, tv.tv_sec); + EXPECT_EQ(0, tv.tv_usec); + + const auto max_timespec_sec = + std::numeric_limits<decltype(timespec::tv_sec)>::max(); + const auto min_timespec_sec = + std::numeric_limits<decltype(timespec::tv_sec)>::min(); + timespec ts; + ts.tv_sec = max_timespec_sec; + ts.tv_nsec = 999999998; + t = absl::TimeFromTimespec(ts); + ts = absl::ToTimespec(t); + EXPECT_EQ(max_timespec_sec, ts.tv_sec); + EXPECT_EQ(999999998, ts.tv_nsec); + t += absl::Nanoseconds(1); + ts = absl::ToTimespec(t); + EXPECT_EQ(max_timespec_sec, ts.tv_sec); + EXPECT_EQ(999999999, ts.tv_nsec); + t += absl::Nanoseconds(1); // no effect + ts = absl::ToTimespec(t); + EXPECT_EQ(max_timespec_sec, ts.tv_sec); + EXPECT_EQ(999999999, ts.tv_nsec); + + ts.tv_sec = min_timespec_sec; + ts.tv_nsec = 1; + t = absl::TimeFromTimespec(ts); + ts = absl::ToTimespec(t); + EXPECT_EQ(min_timespec_sec, ts.tv_sec); + EXPECT_EQ(1, ts.tv_nsec); + t -= absl::Nanoseconds(1); + ts = absl::ToTimespec(t); + EXPECT_EQ(min_timespec_sec, ts.tv_sec); + EXPECT_EQ(0, ts.tv_nsec); + t -= absl::Nanoseconds(1); // no effect + ts = absl::ToTimespec(t); + EXPECT_EQ(min_timespec_sec, ts.tv_sec); + EXPECT_EQ(0, ts.tv_nsec); + + // Checks how TimeZone::At() saturates on infinities. + auto ci = utc.At(absl::InfiniteFuture()); + EXPECT_CIVIL_INFO(ci, std::numeric_limits<int64_t>::max(), 12, 31, 23, 59, 59, + 0, false); + EXPECT_EQ(absl::InfiniteDuration(), ci.subsecond); + EXPECT_EQ(absl::Weekday::thursday, absl::GetWeekday(ci.cs)); + EXPECT_EQ(365, absl::GetYearDay(ci.cs)); + EXPECT_STREQ("-00", ci.zone_abbr); // artifact of TimeZone::At() + ci = utc.At(absl::InfinitePast()); + EXPECT_CIVIL_INFO(ci, std::numeric_limits<int64_t>::min(), 1, 1, 0, 0, 0, 0, + false); + EXPECT_EQ(-absl::InfiniteDuration(), ci.subsecond); + EXPECT_EQ(absl::Weekday::sunday, absl::GetWeekday(ci.cs)); + EXPECT_EQ(1, absl::GetYearDay(ci.cs)); + EXPECT_STREQ("-00", ci.zone_abbr); // artifact of TimeZone::At() + + // Approach the maximal Time value from below. + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 4, 15, 30, 6), utc); + EXPECT_EQ("292277026596-12-04T15:30:06+00:00", + absl::FormatTime(absl::RFC3339_full, t, utc)); + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 4, 15, 30, 7), utc); + EXPECT_EQ("292277026596-12-04T15:30:07+00:00", + absl::FormatTime(absl::RFC3339_full, t, utc)); + EXPECT_EQ( + absl::UnixEpoch() + absl::Seconds(std::numeric_limits<int64_t>::max()), + t); + + // Checks that we can also get the maximal Time value for a far-east zone. + const absl::TimeZone plus14 = absl::FixedTimeZone(14 * 60 * 60); + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 5, 5, 30, 7), plus14); + EXPECT_EQ("292277026596-12-05T05:30:07+14:00", + absl::FormatTime(absl::RFC3339_full, t, plus14)); + EXPECT_EQ( + absl::UnixEpoch() + absl::Seconds(std::numeric_limits<int64_t>::max()), + t); + + // One second later should push us to infinity. + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 4, 15, 30, 8), utc); + EXPECT_EQ("infinite-future", absl::FormatTime(absl::RFC3339_full, t, utc)); + + // Approach the minimal Time value from above. + t = absl::FromCivil(absl::CivilSecond(-292277022657, 1, 27, 8, 29, 53), utc); + EXPECT_EQ("-292277022657-01-27T08:29:53+00:00", + absl::FormatTime(absl::RFC3339_full, t, utc)); + t = absl::FromCivil(absl::CivilSecond(-292277022657, 1, 27, 8, 29, 52), utc); + EXPECT_EQ("-292277022657-01-27T08:29:52+00:00", + absl::FormatTime(absl::RFC3339_full, t, utc)); + EXPECT_EQ( + absl::UnixEpoch() + absl::Seconds(std::numeric_limits<int64_t>::min()), + t); + + // Checks that we can also get the minimal Time value for a far-west zone. + const absl::TimeZone minus12 = absl::FixedTimeZone(-12 * 60 * 60); + t = absl::FromCivil(absl::CivilSecond(-292277022657, 1, 26, 20, 29, 52), + minus12); + EXPECT_EQ("-292277022657-01-26T20:29:52-12:00", + absl::FormatTime(absl::RFC3339_full, t, minus12)); + EXPECT_EQ( + absl::UnixEpoch() + absl::Seconds(std::numeric_limits<int64_t>::min()), + t); + + // One second before should push us to -infinity. + t = absl::FromCivil(absl::CivilSecond(-292277022657, 1, 27, 8, 29, 51), utc); + EXPECT_EQ("infinite-past", absl::FormatTime(absl::RFC3339_full, t, utc)); +} + +// In zones with POSIX-style recurring rules we use special logic to +// handle conversions in the distant future. Here we check the limits +// of those conversions, particularly with respect to integer overflow. +TEST(Time, ExtendedConversionSaturation) { + const absl::TimeZone syd = + absl::time_internal::LoadTimeZone("Australia/Sydney"); + const absl::TimeZone nyc = + absl::time_internal::LoadTimeZone("America/New_York"); + const absl::Time max = + absl::FromUnixSeconds(std::numeric_limits<int64_t>::max()); + absl::TimeZone::CivilInfo ci; + absl::Time t; + + // The maximal time converted in each zone. + ci = syd.At(max); + EXPECT_CIVIL_INFO(ci, 292277026596, 12, 5, 2, 30, 7, 39600, true); + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 5, 2, 30, 7), syd); + EXPECT_EQ(max, t); + ci = nyc.At(max); + EXPECT_CIVIL_INFO(ci, 292277026596, 12, 4, 10, 30, 7, -18000, false); + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 4, 10, 30, 7), nyc); + EXPECT_EQ(max, t); + + // One second later should push us to infinity. + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 5, 2, 30, 8), syd); + EXPECT_EQ(absl::InfiniteFuture(), t); + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 4, 10, 30, 8), nyc); + EXPECT_EQ(absl::InfiniteFuture(), t); + + // And we should stick there. + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 5, 2, 30, 9), syd); + EXPECT_EQ(absl::InfiniteFuture(), t); + t = absl::FromCivil(absl::CivilSecond(292277026596, 12, 4, 10, 30, 9), nyc); + EXPECT_EQ(absl::InfiniteFuture(), t); + + // All the way up to a saturated date/time, without overflow. + t = absl::FromCivil(absl::CivilSecond::max(), syd); + EXPECT_EQ(absl::InfiniteFuture(), t); + t = absl::FromCivil(absl::CivilSecond::max(), nyc); + EXPECT_EQ(absl::InfiniteFuture(), t); +} + +TEST(Time, FromCivilAlignment) { + const absl::TimeZone utc = absl::UTCTimeZone(); + const absl::CivilSecond cs(2015, 2, 3, 4, 5, 6); + absl::Time t = absl::FromCivil(cs, utc); + EXPECT_EQ("2015-02-03T04:05:06+00:00", absl::FormatTime(t, utc)); + t = absl::FromCivil(absl::CivilMinute(cs), utc); + EXPECT_EQ("2015-02-03T04:05:00+00:00", absl::FormatTime(t, utc)); + t = absl::FromCivil(absl::CivilHour(cs), utc); + EXPECT_EQ("2015-02-03T04:00:00+00:00", absl::FormatTime(t, utc)); + t = absl::FromCivil(absl::CivilDay(cs), utc); + EXPECT_EQ("2015-02-03T00:00:00+00:00", absl::FormatTime(t, utc)); + t = absl::FromCivil(absl::CivilMonth(cs), utc); + EXPECT_EQ("2015-02-01T00:00:00+00:00", absl::FormatTime(t, utc)); + t = absl::FromCivil(absl::CivilYear(cs), utc); + EXPECT_EQ("2015-01-01T00:00:00+00:00", absl::FormatTime(t, utc)); +} + +TEST(Time, LegacyDateTime) { + const absl::TimeZone utc = absl::UTCTimeZone(); + const std::string ymdhms = "%Y-%m-%d %H:%M:%S"; + const int kMax = std::numeric_limits<int>::max(); + const int kMin = std::numeric_limits<int>::min(); + absl::Time t; + + t = absl::FromDateTime(std::numeric_limits<absl::civil_year_t>::max(), kMax, + kMax, kMax, kMax, kMax, utc); + EXPECT_EQ("infinite-future", + absl::FormatTime(ymdhms, t, utc)); // no overflow + t = absl::FromDateTime(std::numeric_limits<absl::civil_year_t>::min(), kMin, + kMin, kMin, kMin, kMin, utc); + EXPECT_EQ("infinite-past", absl::FormatTime(ymdhms, t, utc)); // no overflow + + // Check normalization. + EXPECT_TRUE(absl::ConvertDateTime(2013, 10, 32, 8, 30, 0, utc).normalized); + t = absl::FromDateTime(2015, 1, 1, 0, 0, 60, utc); + EXPECT_EQ("2015-01-01 00:01:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, 1, 0, 60, 0, utc); + EXPECT_EQ("2015-01-01 01:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, 1, 24, 0, 0, utc); + EXPECT_EQ("2015-01-02 00:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, 32, 0, 0, 0, utc); + EXPECT_EQ("2015-02-01 00:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 13, 1, 0, 0, 0, utc); + EXPECT_EQ("2016-01-01 00:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 13, 32, 60, 60, 60, utc); + EXPECT_EQ("2016-02-03 13:01:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, 1, 0, 0, -1, utc); + EXPECT_EQ("2014-12-31 23:59:59", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, 1, 0, -1, 0, utc); + EXPECT_EQ("2014-12-31 23:59:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, 1, -1, 0, 0, utc); + EXPECT_EQ("2014-12-31 23:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, 1, -1, 0, 0, 0, utc); + EXPECT_EQ("2014-12-30 00:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, -1, 1, 0, 0, 0, utc); + EXPECT_EQ("2014-11-01 00:00:00", absl::FormatTime(ymdhms, t, utc)); + t = absl::FromDateTime(2015, -1, -1, -1, -1, -1, utc); + EXPECT_EQ("2014-10-29 22:58:59", absl::FormatTime(ymdhms, t, utc)); +} + +TEST(Time, NextTransitionUTC) { + const auto tz = absl::UTCTimeZone(); + absl::TimeZone::CivilTransition trans; + + auto t = absl::InfinitePast(); + EXPECT_FALSE(tz.NextTransition(t, &trans)); + + t = absl::InfiniteFuture(); + EXPECT_FALSE(tz.NextTransition(t, &trans)); +} + +TEST(Time, PrevTransitionUTC) { + const auto tz = absl::UTCTimeZone(); + absl::TimeZone::CivilTransition trans; + + auto t = absl::InfiniteFuture(); + EXPECT_FALSE(tz.PrevTransition(t, &trans)); + + t = absl::InfinitePast(); + EXPECT_FALSE(tz.PrevTransition(t, &trans)); +} + +TEST(Time, NextTransitionNYC) { + const auto tz = absl::time_internal::LoadTimeZone("America/New_York"); + absl::TimeZone::CivilTransition trans; + + auto t = absl::FromCivil(absl::CivilSecond(2018, 6, 30, 0, 0, 0), tz); + EXPECT_TRUE(tz.NextTransition(t, &trans)); + EXPECT_EQ(absl::CivilSecond(2018, 11, 4, 2, 0, 0), trans.from); + EXPECT_EQ(absl::CivilSecond(2018, 11, 4, 1, 0, 0), trans.to); + + t = absl::InfiniteFuture(); + EXPECT_FALSE(tz.NextTransition(t, &trans)); + + t = absl::InfinitePast(); + EXPECT_TRUE(tz.NextTransition(t, &trans)); + if (trans.from == absl::CivilSecond(1918, 03, 31, 2, 0, 0)) { + // It looks like the tzdata is only 32 bit (probably macOS), + // which bottoms out at 1901-12-13T20:45:52+00:00. + EXPECT_EQ(absl::CivilSecond(1918, 3, 31, 3, 0, 0), trans.to); + } else { + EXPECT_EQ(absl::CivilSecond(1883, 11, 18, 12, 3, 58), trans.from); + EXPECT_EQ(absl::CivilSecond(1883, 11, 18, 12, 0, 0), trans.to); + } +} + +TEST(Time, PrevTransitionNYC) { + const auto tz = absl::time_internal::LoadTimeZone("America/New_York"); + absl::TimeZone::CivilTransition trans; + + auto t = absl::FromCivil(absl::CivilSecond(2018, 6, 30, 0, 0, 0), tz); + EXPECT_TRUE(tz.PrevTransition(t, &trans)); + EXPECT_EQ(absl::CivilSecond(2018, 3, 11, 2, 0, 0), trans.from); + EXPECT_EQ(absl::CivilSecond(2018, 3, 11, 3, 0, 0), trans.to); + + t = absl::InfinitePast(); + EXPECT_FALSE(tz.PrevTransition(t, &trans)); + + t = absl::InfiniteFuture(); + EXPECT_TRUE(tz.PrevTransition(t, &trans)); + // We have a transition but we don't know which one. +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/time/time_zone_test.cc b/third_party/abseil_cpp/absl/time/time_zone_test.cc new file mode 100644 index 000000000000..229fcfccb0ad --- /dev/null +++ b/third_party/abseil_cpp/absl/time/time_zone_test.cc @@ -0,0 +1,97 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/time/internal/cctz/include/cctz/time_zone.h" + +#include "gtest/gtest.h" +#include "absl/time/internal/test_util.h" +#include "absl/time/time.h" + +namespace cctz = absl::time_internal::cctz; + +namespace { + +TEST(TimeZone, ValueSemantics) { + absl::TimeZone tz; + absl::TimeZone tz2 = tz; // Copy-construct + EXPECT_EQ(tz, tz2); + tz2 = tz; // Copy-assign + EXPECT_EQ(tz, tz2); +} + +TEST(TimeZone, Equality) { + absl::TimeZone a, b; + EXPECT_EQ(a, b); + EXPECT_EQ(a.name(), b.name()); + + absl::TimeZone implicit_utc; + absl::TimeZone explicit_utc = absl::UTCTimeZone(); + EXPECT_EQ(implicit_utc, explicit_utc); + EXPECT_EQ(implicit_utc.name(), explicit_utc.name()); + + absl::TimeZone la = absl::time_internal::LoadTimeZone("America/Los_Angeles"); + absl::TimeZone nyc = absl::time_internal::LoadTimeZone("America/New_York"); + EXPECT_NE(la, nyc); +} + +TEST(TimeZone, CCTZConversion) { + const cctz::time_zone cz = cctz::utc_time_zone(); + const absl::TimeZone tz(cz); + EXPECT_EQ(cz, cctz::time_zone(tz)); +} + +TEST(TimeZone, DefaultTimeZones) { + absl::TimeZone tz; + EXPECT_EQ("UTC", absl::TimeZone().name()); + EXPECT_EQ("UTC", absl::UTCTimeZone().name()); +} + +TEST(TimeZone, FixedTimeZone) { + const absl::TimeZone tz = absl::FixedTimeZone(123); + const cctz::time_zone cz = cctz::fixed_time_zone(cctz::seconds(123)); + EXPECT_EQ(tz, absl::TimeZone(cz)); +} + +TEST(TimeZone, LocalTimeZone) { + const absl::TimeZone local_tz = absl::LocalTimeZone(); + absl::TimeZone tz = absl::time_internal::LoadTimeZone("localtime"); + EXPECT_EQ(tz, local_tz); +} + +TEST(TimeZone, NamedTimeZones) { + absl::TimeZone nyc = absl::time_internal::LoadTimeZone("America/New_York"); + EXPECT_EQ("America/New_York", nyc.name()); + absl::TimeZone syd = absl::time_internal::LoadTimeZone("Australia/Sydney"); + EXPECT_EQ("Australia/Sydney", syd.name()); + absl::TimeZone fixed = absl::FixedTimeZone((((3 * 60) + 25) * 60) + 45); + EXPECT_EQ("Fixed/UTC+03:25:45", fixed.name()); +} + +TEST(TimeZone, Failures) { + absl::TimeZone tz = absl::time_internal::LoadTimeZone("America/Los_Angeles"); + EXPECT_FALSE(LoadTimeZone("Invalid/TimeZone", &tz)); + EXPECT_EQ(absl::UTCTimeZone(), tz); // guaranteed fallback to UTC + + // Ensures that the load still fails on a subsequent attempt. + tz = absl::time_internal::LoadTimeZone("America/Los_Angeles"); + EXPECT_FALSE(LoadTimeZone("Invalid/TimeZone", &tz)); + EXPECT_EQ(absl::UTCTimeZone(), tz); // guaranteed fallback to UTC + + // Loading an empty string timezone should fail. + tz = absl::time_internal::LoadTimeZone("America/Los_Angeles"); + EXPECT_FALSE(LoadTimeZone("", &tz)); + EXPECT_EQ(absl::UTCTimeZone(), tz); // guaranteed fallback to UTC +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/types/BUILD.bazel b/third_party/abseil_cpp/absl/types/BUILD.bazel new file mode 100644 index 000000000000..83be9360ff99 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/BUILD.bazel @@ -0,0 +1,336 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "any", + hdrs = ["any.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bad_any_cast", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:fast_type_id", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_library( + name = "bad_any_cast", + hdrs = ["bad_any_cast.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bad_any_cast_impl", + "//absl/base:config", + ], +) + +cc_library( + name = "bad_any_cast_impl", + srcs = [ + "bad_any_cast.cc", + "bad_any_cast.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + "//absl/base:config", + "//absl/base:raw_logging_internal", + ], +) + +cc_test( + name = "any_test", + size = "small", + srcs = [ + "any_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":any", + "//absl/base:config", + "//absl/base:exception_testing", + "//absl/base:raw_logging_internal", + "//absl/container:test_instance_tracker", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "any_exception_safety_test", + srcs = ["any_exception_safety_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":any", + "//absl/base:config", + "//absl/base:exception_safety_testing", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "span", + srcs = [ + "internal/span.h", + ], + hdrs = [ + "span.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/algorithm", + "//absl/base:core_headers", + "//absl/base:throw_delegate", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "span_test", + size = "small", + srcs = ["span_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":span", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:exception_testing", + "//absl/container:fixed_array", + "//absl/container:inlined_vector", + "//absl/hash:hash_testing", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "optional", + srcs = ["internal/optional.h"], + hdrs = ["optional.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bad_optional_access", + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_library( + name = "bad_optional_access", + srcs = ["bad_optional_access.cc"], + hdrs = ["bad_optional_access.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "bad_variant_access", + srcs = ["bad_variant_access.cc"], + hdrs = ["bad_variant_access.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:config", + "//absl/base:raw_logging_internal", + ], +) + +cc_test( + name = "optional_test", + size = "small", + srcs = [ + "optional_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":optional", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/meta:type_traits", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "optional_exception_safety_test", + srcs = [ + "optional_exception_safety_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":optional", + "//absl/base:config", + "//absl/base:exception_safety_testing", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "conformance_testing", + testonly = 1, + hdrs = [ + "internal/conformance_aliases.h", + "internal/conformance_archetype.h", + "internal/conformance_profile.h", + "internal/conformance_testing.h", + "internal/conformance_testing_helpers.h", + "internal/parentheses.h", + "internal/transform_args.h", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/algorithm:container", + "//absl/meta:type_traits", + "//absl/strings", + "//absl/utility", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "conformance_testing_test", + size = "small", + srcs = [ + "internal/conformance_testing_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":conformance_testing", + "//absl/meta:type_traits", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "variant", + srcs = ["internal/variant.h"], + hdrs = ["variant.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bad_variant_access", + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_test( + name = "variant_test", + size = "small", + srcs = ["variant_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":variant", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/memory", + "//absl/meta:type_traits", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "variant_benchmark", + srcs = [ + "variant_benchmark.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + deps = [ + ":variant", + "//absl/utility", + "@com_github_google_benchmark//:benchmark_main", + ], +) + +cc_test( + name = "variant_exception_safety_test", + size = "small", + srcs = [ + "variant_exception_safety_test.cc", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":variant", + "//absl/base:config", + "//absl/base:exception_safety_testing", + "//absl/memory", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "compare", + hdrs = ["compare.h"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + "//absl/base:core_headers", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "compare_test", + size = "small", + srcs = [ + "compare_test.cc", + ], + copts = ABSL_TEST_COPTS, + deps = [ + ":compare", + "//absl/base", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/types/CMakeLists.txt b/third_party/abseil_cpp/absl/types/CMakeLists.txt new file mode 100644 index 000000000000..3f99ad8a4b4d --- /dev/null +++ b/third_party/abseil_cpp/absl/types/CMakeLists.txt @@ -0,0 +1,373 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +absl_cc_library( + NAME + any + HDRS + "any.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_any_cast + absl::config + absl::core_headers + absl::fast_type_id + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bad_any_cast + HDRS + "bad_any_cast.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_any_cast_impl + absl::config + PUBLIC +) + +absl_cc_library( + NAME + bad_any_cast_impl + SRCS + "bad_any_cast.h" + "bad_any_cast.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal +) + +absl_cc_test( + NAME + any_test + SRCS + "any_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::any + absl::config + absl::exception_testing + absl::raw_logging_internal + absl::test_instance_tracker + gmock_main +) + +absl_cc_test( + NAME + any_test_noexceptions + SRCS + "any_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::any + absl::config + absl::exception_testing + absl::raw_logging_internal + absl::test_instance_tracker + gmock_main +) + +absl_cc_test( + NAME + any_exception_safety_test + SRCS + "any_exception_safety_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::any + absl::config + absl::exception_safety_testing + gmock_main +) + +absl_cc_library( + NAME + span + HDRS + "span.h" + SRCS + "internal/span.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::core_headers + absl::throw_delegate + absl::type_traits + PUBLIC +) + +absl_cc_test( + NAME + span_test + SRCS + "span_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::span + absl::base + absl::config + absl::core_headers + absl::exception_testing + absl::fixed_array + absl::inlined_vector + absl::hash_testing + absl::strings + gmock_main +) + +absl_cc_test( + NAME + span_test_noexceptions + SRCS + "span_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::span + absl::base + absl::config + absl::core_headers + absl::exception_testing + absl::fixed_array + absl::inlined_vector + absl::hash_testing + absl::strings + gmock_main +) + +absl_cc_library( + NAME + optional + HDRS + "optional.h" + SRCS + "internal/optional.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_optional_access + absl::base_internal + absl::config + absl::core_headers + absl::memory + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_library( + NAME + bad_optional_access + HDRS + "bad_optional_access.h" + SRCS + "bad_optional_access.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal + PUBLIC +) + +absl_cc_library( + NAME + bad_variant_access + HDRS + "bad_variant_access.h" + SRCS + "bad_variant_access.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::raw_logging_internal + PUBLIC +) + +absl_cc_test( + NAME + optional_test + SRCS + "optional_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::optional + absl::config + absl::raw_logging_internal + absl::strings + absl::type_traits + gmock_main +) + +absl_cc_test( + NAME + optional_exception_safety_test + SRCS + "optional_exception_safety_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::optional + absl::config + absl::exception_safety_testing + gmock_main +) + +absl_cc_library( + NAME + conformance_testing + HDRS + "internal/conformance_aliases.h" + "internal/conformance_archetype.h" + "internal/conformance_profile.h" + "internal/conformance_testing.h" + "internal/conformance_testing_helpers.h" + "internal/parentheses.h" + "internal/transform_args.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::algorithm + absl::debugging + absl::type_traits + absl::strings + absl::utility + gmock_main + TESTONLY +) + +absl_cc_test( + NAME + conformance_testing_test + SRCS + "internal/conformance_testing_test.cc" + COPTS + ${ABSL_TEST_COPTS} + ${ABSL_EXCEPTIONS_FLAG} + LINKOPTS + ${ABSL_EXCEPTIONS_FLAG_LINKOPTS} + DEPS + absl::conformance_testing + absl::type_traits + gmock_main +) + +absl_cc_test( + NAME + conformance_testing_test_no_exceptions + SRCS + "internal/conformance_testing_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::conformance_testing + absl::type_traits + gmock_main +) + +absl_cc_library( + NAME + variant + HDRS + "variant.h" + SRCS + "internal/variant.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::bad_variant_access + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_test( + NAME + variant_test + SRCS + "variant_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::variant + absl::config + absl::core_headers + absl::memory + absl::type_traits + absl::strings + gmock_main +) + +absl_cc_library( + NAME + compare + HDRS + "compare.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::type_traits + PUBLIC +) + +absl_cc_test( + NAME + compare_test + SRCS + "compare_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::compare + gmock_main +) + +# TODO(cohenjon,zhangxy) Figure out why this test is failing on gcc 4.8 +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) +else() +absl_cc_test( + NAME + variant_exception_safety_test + SRCS + "variant_exception_safety_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::variant + absl::config + absl::exception_safety_testing + absl::memory + gmock_main +) +endif() diff --git a/third_party/abseil_cpp/absl/types/any.h b/third_party/abseil_cpp/absl/types/any.h new file mode 100644 index 000000000000..fc5a07469f80 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/any.h @@ -0,0 +1,528 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// any.h +// ----------------------------------------------------------------------------- +// +// This header file define the `absl::any` type for holding a type-safe value +// of any type. The 'absl::any` type is useful for providing a way to hold +// something that is, as yet, unspecified. Such unspecified types +// traditionally are passed between API boundaries until they are later cast to +// their "destination" types. To cast to such a destination type, use +// `absl::any_cast()`. Note that when casting an `absl::any`, you must cast it +// to an explicit type; implicit conversions will throw. +// +// Example: +// +// auto a = absl::any(65); +// absl::any_cast<int>(a); // 65 +// absl::any_cast<char>(a); // throws absl::bad_any_cast +// absl::any_cast<std::string>(a); // throws absl::bad_any_cast +// +// `absl::any` is a C++11 compatible version of the C++17 `std::any` abstraction +// and is designed to be a drop-in replacement for code compliant with C++17. +// +// Traditionally, the behavior of casting to a temporary unspecified type has +// been accomplished with the `void *` paradigm, where the pointer was to some +// other unspecified type. `absl::any` provides an "owning" version of `void *` +// that avoids issues of pointer management. +// +// Note: just as in the case of `void *`, use of `absl::any` (and its C++17 +// version `std::any`) is a code smell indicating that your API might not be +// constructed correctly. We have seen that most uses of `any` are unwarranted, +// and `absl::any`, like `std::any`, is difficult to use properly. Before using +// this abstraction, make sure that you should not instead be rewriting your +// code to be more specific. +// +// Abseil has also released an `absl::variant` type (a C++11 compatible version +// of the C++17 `std::variant`), which is generally preferred for use over +// `absl::any`. +#ifndef ABSL_TYPES_ANY_H_ +#define ABSL_TYPES_ANY_H_ + +#include "absl/base/config.h" +#include "absl/utility/utility.h" + +#ifdef ABSL_USES_STD_ANY + +#include <any> // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN +using std::any; +using std::any_cast; +using std::bad_any_cast; +using std::make_any; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_ANY + +#include <algorithm> +#include <cstddef> +#include <initializer_list> +#include <memory> +#include <stdexcept> +#include <type_traits> +#include <typeinfo> +#include <utility> + +#include "absl/base/internal/fast_type_id.h" +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" +#include "absl/types/bad_any_cast.h" + +// NOTE: This macro is an implementation detail that is undefined at the bottom +// of the file. It is not intended for expansion directly from user code. +#ifdef ABSL_ANY_DETAIL_HAS_RTTI +#error ABSL_ANY_DETAIL_HAS_RTTI cannot be directly set +#elif !defined(__GNUC__) || defined(__GXX_RTTI) +#define ABSL_ANY_DETAIL_HAS_RTTI 1 +#endif // !defined(__GNUC__) || defined(__GXX_RTTI) + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class any; + +// swap() +// +// Swaps two `absl::any` values. Equivalent to `x.swap(y) where `x` and `y` are +// `absl::any` types. +void swap(any& x, any& y) noexcept; + +// make_any() +// +// Constructs an `absl::any` of type `T` with the given arguments. +template <typename T, typename... Args> +any make_any(Args&&... args); + +// Overload of `absl::make_any()` for constructing an `absl::any` type from an +// initializer list. +template <typename T, typename U, typename... Args> +any make_any(std::initializer_list<U> il, Args&&... args); + +// any_cast() +// +// Statically casts the value of a `const absl::any` type to the given type. +// This function will throw `absl::bad_any_cast` if the stored value type of the +// `absl::any` does not match the cast. +// +// `any_cast()` can also be used to get a reference to the internal storage iff +// a reference type is passed as its `ValueType`: +// +// Example: +// +// absl::any my_any = std::vector<int>(); +// absl::any_cast<std::vector<int>&>(my_any).push_back(42); +template <typename ValueType> +ValueType any_cast(const any& operand); + +// Overload of `any_cast()` to statically cast the value of a non-const +// `absl::any` type to the given type. This function will throw +// `absl::bad_any_cast` if the stored value type of the `absl::any` does not +// match the cast. +template <typename ValueType> +ValueType any_cast(any& operand); // NOLINT(runtime/references) + +// Overload of `any_cast()` to statically cast the rvalue of an `absl::any` +// type. This function will throw `absl::bad_any_cast` if the stored value type +// of the `absl::any` does not match the cast. +template <typename ValueType> +ValueType any_cast(any&& operand); + +// Overload of `any_cast()` to statically cast the value of a const pointer +// `absl::any` type to the given pointer type, or `nullptr` if the stored value +// type of the `absl::any` does not match the cast. +template <typename ValueType> +const ValueType* any_cast(const any* operand) noexcept; + +// Overload of `any_cast()` to statically cast the value of a pointer +// `absl::any` type to the given pointer type, or `nullptr` if the stored value +// type of the `absl::any` does not match the cast. +template <typename ValueType> +ValueType* any_cast(any* operand) noexcept; + +// ----------------------------------------------------------------------------- +// absl::any +// ----------------------------------------------------------------------------- +// +// An `absl::any` object provides the facility to either store an instance of a +// type, known as the "contained object", or no value. An `absl::any` is used to +// store values of types that are unknown at compile time. The `absl::any` +// object, when containing a value, must contain a value type; storing a +// reference type is neither desired nor supported. +// +// An `absl::any` can only store a type that is copy-constructible; move-only +// types are not allowed within an `any` object. +// +// Example: +// +// auto a = absl::any(65); // Literal, copyable +// auto b = absl::any(std::vector<int>()); // Default-initialized, copyable +// std::unique_ptr<Foo> my_foo; +// auto c = absl::any(std::move(my_foo)); // Error, not copy-constructible +// +// Note that `absl::any` makes use of decayed types (`absl::decay_t` in this +// context) to remove const-volatile qualifiers (known as "cv qualifiers"), +// decay functions to function pointers, etc. We essentially "decay" a given +// type into its essential type. +// +// `absl::any` makes use of decayed types when determining the basic type `T` of +// the value to store in the any's contained object. In the documentation below, +// we explicitly denote this by using the phrase "a decayed type of `T`". +// +// Example: +// +// const int a = 4; +// absl::any foo(a); // Decay ensures we store an "int", not a "const int&". +// +// void my_function() {} +// absl::any bar(my_function); // Decay ensures we store a function pointer. +// +// `absl::any` is a C++11 compatible version of the C++17 `std::any` abstraction +// and is designed to be a drop-in replacement for code compliant with C++17. +class any { + private: + template <typename T> + struct IsInPlaceType; + + public: + // Constructors + + // Constructs an empty `absl::any` object (`any::has_value()` will return + // `false`). + constexpr any() noexcept; + + // Copy constructs an `absl::any` object with a "contained object" of the + // passed type of `other` (or an empty `absl::any` if `other.has_value()` is + // `false`. + any(const any& other) + : obj_(other.has_value() ? other.obj_->Clone() + : std::unique_ptr<ObjInterface>()) {} + + // Move constructs an `absl::any` object with a "contained object" of the + // passed type of `other` (or an empty `absl::any` if `other.has_value()` is + // `false`). + any(any&& other) noexcept = default; + + // Constructs an `absl::any` object with a "contained object" of the decayed + // type of `T`, which is initialized via `std::forward<T>(value)`. + // + // This constructor will not participate in overload resolution if the + // decayed type of `T` is not copy-constructible. + template < + typename T, typename VT = absl::decay_t<T>, + absl::enable_if_t<!absl::disjunction< + std::is_same<any, VT>, IsInPlaceType<VT>, + absl::negation<std::is_copy_constructible<VT> > >::value>* = nullptr> + any(T&& value) : obj_(new Obj<VT>(in_place, std::forward<T>(value))) {} + + // Constructs an `absl::any` object with a "contained object" of the decayed + // type of `T`, which is initialized via `std::forward<T>(value)`. + template <typename T, typename... Args, typename VT = absl::decay_t<T>, + absl::enable_if_t<absl::conjunction< + std::is_copy_constructible<VT>, + std::is_constructible<VT, Args...>>::value>* = nullptr> + explicit any(in_place_type_t<T> /*tag*/, Args&&... args) + : obj_(new Obj<VT>(in_place, std::forward<Args>(args)...)) {} + + // Constructs an `absl::any` object with a "contained object" of the passed + // type `VT` as a decayed type of `T`. `VT` is initialized as if + // direct-non-list-initializing an object of type `VT` with the arguments + // `initializer_list, std::forward<Args>(args)...`. + template < + typename T, typename U, typename... Args, typename VT = absl::decay_t<T>, + absl::enable_if_t< + absl::conjunction<std::is_copy_constructible<VT>, + std::is_constructible<VT, std::initializer_list<U>&, + Args...>>::value>* = nullptr> + explicit any(in_place_type_t<T> /*tag*/, std::initializer_list<U> ilist, + Args&&... args) + : obj_(new Obj<VT>(in_place, ilist, std::forward<Args>(args)...)) {} + + // Assignment operators + + // Copy assigns an `absl::any` object with a "contained object" of the + // passed type. + any& operator=(const any& rhs) { + any(rhs).swap(*this); + return *this; + } + + // Move assigns an `absl::any` object with a "contained object" of the + // passed type. `rhs` is left in a valid but otherwise unspecified state. + any& operator=(any&& rhs) noexcept { + any(std::move(rhs)).swap(*this); + return *this; + } + + // Assigns an `absl::any` object with a "contained object" of the passed type. + template <typename T, typename VT = absl::decay_t<T>, + absl::enable_if_t<absl::conjunction< + absl::negation<std::is_same<VT, any>>, + std::is_copy_constructible<VT>>::value>* = nullptr> + any& operator=(T&& rhs) { + any tmp(in_place_type_t<VT>(), std::forward<T>(rhs)); + tmp.swap(*this); + return *this; + } + + // Modifiers + + // any::emplace() + // + // Emplaces a value within an `absl::any` object by calling `any::reset()`, + // initializing the contained value as if direct-non-list-initializing an + // object of type `VT` with the arguments `std::forward<Args>(args)...`, and + // returning a reference to the new contained value. + // + // Note: If an exception is thrown during the call to `VT`'s constructor, + // `*this` does not contain a value, and any previously contained value has + // been destroyed. + template < + typename T, typename... Args, typename VT = absl::decay_t<T>, + absl::enable_if_t<std::is_copy_constructible<VT>::value && + std::is_constructible<VT, Args...>::value>* = nullptr> + VT& emplace(Args&&... args) { + reset(); // NOTE: reset() is required here even in the world of exceptions. + Obj<VT>* const object_ptr = + new Obj<VT>(in_place, std::forward<Args>(args)...); + obj_ = std::unique_ptr<ObjInterface>(object_ptr); + return object_ptr->value; + } + + // Overload of `any::emplace()` to emplace a value within an `absl::any` + // object by calling `any::reset()`, initializing the contained value as if + // direct-non-list-initializing an object of type `VT` with the arguments + // `initializer_list, std::forward<Args>(args)...`, and returning a reference + // to the new contained value. + // + // Note: If an exception is thrown during the call to `VT`'s constructor, + // `*this` does not contain a value, and any previously contained value has + // been destroyed. The function shall not participate in overload resolution + // unless `is_copy_constructible_v<VT>` is `true` and + // `is_constructible_v<VT, initializer_list<U>&, Args...>` is `true`. + template < + typename T, typename U, typename... Args, typename VT = absl::decay_t<T>, + absl::enable_if_t<std::is_copy_constructible<VT>::value && + std::is_constructible<VT, std::initializer_list<U>&, + Args...>::value>* = nullptr> + VT& emplace(std::initializer_list<U> ilist, Args&&... args) { + reset(); // NOTE: reset() is required here even in the world of exceptions. + Obj<VT>* const object_ptr = + new Obj<VT>(in_place, ilist, std::forward<Args>(args)...); + obj_ = std::unique_ptr<ObjInterface>(object_ptr); + return object_ptr->value; + } + + // any::reset() + // + // Resets the state of the `absl::any` object, destroying the contained object + // if present. + void reset() noexcept { obj_ = nullptr; } + + // any::swap() + // + // Swaps the passed value and the value of this `absl::any` object. + void swap(any& other) noexcept { obj_.swap(other.obj_); } + + // Observers + + // any::has_value() + // + // Returns `true` if the `any` object has a contained value, otherwise + // returns `false`. + bool has_value() const noexcept { return obj_ != nullptr; } + +#if ABSL_ANY_DETAIL_HAS_RTTI + // Returns: typeid(T) if *this has a contained object of type T, otherwise + // typeid(void). + const std::type_info& type() const noexcept { + if (has_value()) { + return obj_->Type(); + } + + return typeid(void); + } +#endif // ABSL_ANY_DETAIL_HAS_RTTI + + private: + // Tagged type-erased abstraction for holding a cloneable object. + class ObjInterface { + public: + virtual ~ObjInterface() = default; + virtual std::unique_ptr<ObjInterface> Clone() const = 0; + virtual const void* ObjTypeId() const noexcept = 0; +#if ABSL_ANY_DETAIL_HAS_RTTI + virtual const std::type_info& Type() const noexcept = 0; +#endif // ABSL_ANY_DETAIL_HAS_RTTI + }; + + // Hold a value of some queryable type, with an ability to Clone it. + template <typename T> + class Obj : public ObjInterface { + public: + template <typename... Args> + explicit Obj(in_place_t /*tag*/, Args&&... args) + : value(std::forward<Args>(args)...) {} + + std::unique_ptr<ObjInterface> Clone() const final { + return std::unique_ptr<ObjInterface>(new Obj(in_place, value)); + } + + const void* ObjTypeId() const noexcept final { return IdForType<T>(); } + +#if ABSL_ANY_DETAIL_HAS_RTTI + const std::type_info& Type() const noexcept final { return typeid(T); } +#endif // ABSL_ANY_DETAIL_HAS_RTTI + + T value; + }; + + std::unique_ptr<ObjInterface> CloneObj() const { + if (!obj_) return nullptr; + return obj_->Clone(); + } + + template <typename T> + constexpr static const void* IdForType() { + // Note: This type dance is to make the behavior consistent with typeid. + using NormalizedType = + typename std::remove_cv<typename std::remove_reference<T>::type>::type; + + return base_internal::FastTypeId<NormalizedType>(); + } + + const void* GetObjTypeId() const { + return obj_ ? obj_->ObjTypeId() : base_internal::FastTypeId<void>(); + } + + // `absl::any` nonmember functions // + + // Description at the declaration site (top of file). + template <typename ValueType> + friend ValueType any_cast(const any& operand); + + // Description at the declaration site (top of file). + template <typename ValueType> + friend ValueType any_cast(any& operand); // NOLINT(runtime/references) + + // Description at the declaration site (top of file). + template <typename T> + friend const T* any_cast(const any* operand) noexcept; + + // Description at the declaration site (top of file). + template <typename T> + friend T* any_cast(any* operand) noexcept; + + std::unique_ptr<ObjInterface> obj_; +}; + +// ----------------------------------------------------------------------------- +// Implementation Details +// ----------------------------------------------------------------------------- + +constexpr any::any() noexcept = default; + +template <typename T> +struct any::IsInPlaceType : std::false_type {}; + +template <typename T> +struct any::IsInPlaceType<in_place_type_t<T>> : std::true_type {}; + +inline void swap(any& x, any& y) noexcept { x.swap(y); } + +// Description at the declaration site (top of file). +template <typename T, typename... Args> +any make_any(Args&&... args) { + return any(in_place_type_t<T>(), std::forward<Args>(args)...); +} + +// Description at the declaration site (top of file). +template <typename T, typename U, typename... Args> +any make_any(std::initializer_list<U> il, Args&&... args) { + return any(in_place_type_t<T>(), il, std::forward<Args>(args)...); +} + +// Description at the declaration site (top of file). +template <typename ValueType> +ValueType any_cast(const any& operand) { + using U = typename std::remove_cv< + typename std::remove_reference<ValueType>::type>::type; + static_assert(std::is_constructible<ValueType, const U&>::value, + "Invalid ValueType"); + auto* const result = (any_cast<U>)(&operand); + if (result == nullptr) { + any_internal::ThrowBadAnyCast(); + } + return static_cast<ValueType>(*result); +} + +// Description at the declaration site (top of file). +template <typename ValueType> +ValueType any_cast(any& operand) { // NOLINT(runtime/references) + using U = typename std::remove_cv< + typename std::remove_reference<ValueType>::type>::type; + static_assert(std::is_constructible<ValueType, U&>::value, + "Invalid ValueType"); + auto* result = (any_cast<U>)(&operand); + if (result == nullptr) { + any_internal::ThrowBadAnyCast(); + } + return static_cast<ValueType>(*result); +} + +// Description at the declaration site (top of file). +template <typename ValueType> +ValueType any_cast(any&& operand) { + using U = typename std::remove_cv< + typename std::remove_reference<ValueType>::type>::type; + static_assert(std::is_constructible<ValueType, U>::value, + "Invalid ValueType"); + return static_cast<ValueType>(std::move((any_cast<U&>)(operand))); +} + +// Description at the declaration site (top of file). +template <typename T> +const T* any_cast(const any* operand) noexcept { + using U = + typename std::remove_cv<typename std::remove_reference<T>::type>::type; + return operand && operand->GetObjTypeId() == any::IdForType<U>() + ? std::addressof( + static_cast<const any::Obj<U>*>(operand->obj_.get())->value) + : nullptr; +} + +// Description at the declaration site (top of file). +template <typename T> +T* any_cast(any* operand) noexcept { + using U = + typename std::remove_cv<typename std::remove_reference<T>::type>::type; + return operand && operand->GetObjTypeId() == any::IdForType<U>() + ? std::addressof( + static_cast<any::Obj<U>*>(operand->obj_.get())->value) + : nullptr; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_ANY_DETAIL_HAS_RTTI + +#endif // ABSL_USES_STD_ANY + +#endif // ABSL_TYPES_ANY_H_ diff --git a/third_party/abseil_cpp/absl/types/any_exception_safety_test.cc b/third_party/abseil_cpp/absl/types/any_exception_safety_test.cc new file mode 100644 index 000000000000..31c1140135f0 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/any_exception_safety_test.cc @@ -0,0 +1,173 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/any.h" + +#include "absl/base/config.h" + +// This test is a no-op when absl::any is an alias for std::any and when +// exceptions are not enabled. +#if !defined(ABSL_USES_STD_ANY) && defined(ABSL_HAVE_EXCEPTIONS) + +#include <typeinfo> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/internal/exception_safety_testing.h" + +using Thrower = testing::ThrowingValue<>; +using NoThrowMoveThrower = + testing::ThrowingValue<testing::TypeSpec::kNoThrowMove>; +using ThrowerList = std::initializer_list<Thrower>; +using ThrowerVec = std::vector<Thrower>; +using ThrowingAlloc = testing::ThrowingAllocator<Thrower>; +using ThrowingThrowerVec = std::vector<Thrower, ThrowingAlloc>; + +namespace { + +testing::AssertionResult AnyInvariants(absl::any* a) { + using testing::AssertionFailure; + using testing::AssertionSuccess; + + if (a->has_value()) { + if (a->type() == typeid(void)) { + return AssertionFailure() + << "A non-empty any should not have type `void`"; + } + } else { + if (a->type() != typeid(void)) { + return AssertionFailure() + << "An empty any should have type void, but has type " + << a->type().name(); + } + } + + // Make sure that reset() changes any to a valid state. + a->reset(); + if (a->has_value()) { + return AssertionFailure() << "A reset `any` should be valueless"; + } + if (a->type() != typeid(void)) { + return AssertionFailure() << "A reset `any` should have type() of `void`, " + "but instead has type " + << a->type().name(); + } + try { + auto unused = absl::any_cast<Thrower>(*a); + static_cast<void>(unused); + return AssertionFailure() + << "A reset `any` should not be able to be any_cast"; + } catch (const absl::bad_any_cast&) { + } catch (...) { + return AssertionFailure() + << "Unexpected exception thrown from absl::any_cast"; + } + return AssertionSuccess(); +} + +testing::AssertionResult AnyIsEmpty(absl::any* a) { + if (!a->has_value()) { + return testing::AssertionSuccess(); + } + return testing::AssertionFailure() + << "a should be empty, but instead has value " + << absl::any_cast<Thrower>(*a).Get(); +} + +TEST(AnyExceptionSafety, Ctors) { + Thrower val(1); + testing::TestThrowingCtor<absl::any>(val); + + Thrower copy(val); + testing::TestThrowingCtor<absl::any>(copy); + + testing::TestThrowingCtor<absl::any>(absl::in_place_type_t<Thrower>(), 1); + + testing::TestThrowingCtor<absl::any>(absl::in_place_type_t<ThrowerVec>(), + ThrowerList{val}); + + testing::TestThrowingCtor<absl::any, + absl::in_place_type_t<ThrowingThrowerVec>, + ThrowerList, ThrowingAlloc>( + absl::in_place_type_t<ThrowingThrowerVec>(), {val}, ThrowingAlloc()); +} + +TEST(AnyExceptionSafety, Assignment) { + auto original = + absl::any(absl::in_place_type_t<Thrower>(), 1, testing::nothrow_ctor); + auto any_is_strong = [original](absl::any* ap) { + return testing::AssertionResult(ap->has_value() && + absl::any_cast<Thrower>(original) == + absl::any_cast<Thrower>(*ap)); + }; + auto any_strong_tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(original) + .WithContracts(AnyInvariants, any_is_strong); + + Thrower val(2); + absl::any any_val(val); + NoThrowMoveThrower mv_val(2); + + auto assign_any = [&any_val](absl::any* ap) { *ap = any_val; }; + auto assign_val = [&val](absl::any* ap) { *ap = val; }; + auto move = [&val](absl::any* ap) { *ap = std::move(val); }; + auto move_movable = [&mv_val](absl::any* ap) { *ap = std::move(mv_val); }; + + EXPECT_TRUE(any_strong_tester.Test(assign_any)); + EXPECT_TRUE(any_strong_tester.Test(assign_val)); + EXPECT_TRUE(any_strong_tester.Test(move)); + EXPECT_TRUE(any_strong_tester.Test(move_movable)); + + auto empty_any_is_strong = [](absl::any* ap) { + return testing::AssertionResult{!ap->has_value()}; + }; + auto strong_empty_any_tester = + testing::MakeExceptionSafetyTester() + .WithInitialValue(absl::any{}) + .WithContracts(AnyInvariants, empty_any_is_strong); + + EXPECT_TRUE(strong_empty_any_tester.Test(assign_any)); + EXPECT_TRUE(strong_empty_any_tester.Test(assign_val)); + EXPECT_TRUE(strong_empty_any_tester.Test(move)); +} + +TEST(AnyExceptionSafety, Emplace) { + auto initial_val = + absl::any{absl::in_place_type_t<Thrower>(), 1, testing::nothrow_ctor}; + auto one_tester = testing::MakeExceptionSafetyTester() + .WithInitialValue(initial_val) + .WithContracts(AnyInvariants, AnyIsEmpty); + + auto emp_thrower = [](absl::any* ap) { ap->emplace<Thrower>(2); }; + auto emp_throwervec = [](absl::any* ap) { + std::initializer_list<Thrower> il{Thrower(2, testing::nothrow_ctor)}; + ap->emplace<ThrowerVec>(il); + }; + auto emp_movethrower = [](absl::any* ap) { + ap->emplace<NoThrowMoveThrower>(2); + }; + + EXPECT_TRUE(one_tester.Test(emp_thrower)); + EXPECT_TRUE(one_tester.Test(emp_throwervec)); + EXPECT_TRUE(one_tester.Test(emp_movethrower)); + + auto empty_tester = one_tester.WithInitialValue(absl::any{}); + + EXPECT_TRUE(empty_tester.Test(emp_thrower)); + EXPECT_TRUE(empty_tester.Test(emp_throwervec)); +} + +} // namespace + +#endif // #if !defined(ABSL_USES_STD_ANY) && defined(ABSL_HAVE_EXCEPTIONS) diff --git a/third_party/abseil_cpp/absl/types/any_test.cc b/third_party/abseil_cpp/absl/types/any_test.cc new file mode 100644 index 000000000000..70e4ba22b12b --- /dev/null +++ b/third_party/abseil_cpp/absl/types/any_test.cc @@ -0,0 +1,781 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/any.h" + +// This test is a no-op when absl::any is an alias for std::any. +#if !defined(ABSL_USES_STD_ANY) + +#include <initializer_list> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/exception_testing.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/container/internal/test_instance_tracker.h" + +namespace { +using absl::test_internal::CopyableOnlyInstance; +using absl::test_internal::InstanceTracker; + +template <typename T> +const T& AsConst(const T& t) { + return t; +} + +struct MoveOnly { + MoveOnly() = default; + explicit MoveOnly(int value) : value(value) {} + MoveOnly(MoveOnly&&) = default; + MoveOnly& operator=(MoveOnly&&) = default; + + int value = 0; +}; + +struct CopyOnly { + CopyOnly() = default; + explicit CopyOnly(int value) : value(value) {} + CopyOnly(CopyOnly&&) = delete; + CopyOnly& operator=(CopyOnly&&) = delete; + CopyOnly(const CopyOnly&) = default; + CopyOnly& operator=(const CopyOnly&) = default; + + int value = 0; +}; + +struct MoveOnlyWithListConstructor { + MoveOnlyWithListConstructor() = default; + explicit MoveOnlyWithListConstructor(std::initializer_list<int> /*ilist*/, + int value) + : value(value) {} + MoveOnlyWithListConstructor(MoveOnlyWithListConstructor&&) = default; + MoveOnlyWithListConstructor& operator=(MoveOnlyWithListConstructor&&) = + default; + + int value = 0; +}; + +struct IntMoveOnlyCopyOnly { + IntMoveOnlyCopyOnly(int value, MoveOnly /*move_only*/, CopyOnly /*copy_only*/) + : value(value) {} + + int value; +}; + +struct ListMoveOnlyCopyOnly { + ListMoveOnlyCopyOnly(std::initializer_list<int> ilist, MoveOnly /*move_only*/, + CopyOnly /*copy_only*/) + : values(ilist) {} + + std::vector<int> values; +}; + +using FunctionType = void(); +void FunctionToEmplace() {} + +using ArrayType = int[2]; +using DecayedArray = absl::decay_t<ArrayType>; + +TEST(AnyTest, Noexcept) { + static_assert(std::is_nothrow_default_constructible<absl::any>(), ""); + static_assert(std::is_nothrow_move_constructible<absl::any>(), ""); + static_assert(std::is_nothrow_move_assignable<absl::any>(), ""); + static_assert(noexcept(std::declval<absl::any&>().has_value()), ""); + static_assert(noexcept(std::declval<absl::any&>().type()), ""); + static_assert(noexcept(absl::any_cast<int>(std::declval<absl::any*>())), ""); + static_assert( + noexcept(std::declval<absl::any&>().swap(std::declval<absl::any&>())), + ""); + + using std::swap; + static_assert( + noexcept(swap(std::declval<absl::any&>(), std::declval<absl::any&>())), + ""); +} + +TEST(AnyTest, HasValue) { + absl::any o; + EXPECT_FALSE(o.has_value()); + o.emplace<int>(); + EXPECT_TRUE(o.has_value()); + o.reset(); + EXPECT_FALSE(o.has_value()); +} + +TEST(AnyTest, Type) { + absl::any o; + EXPECT_EQ(typeid(void), o.type()); + o.emplace<int>(5); + EXPECT_EQ(typeid(int), o.type()); + o.emplace<float>(5.f); + EXPECT_EQ(typeid(float), o.type()); + o.reset(); + EXPECT_EQ(typeid(void), o.type()); +} + +TEST(AnyTest, EmptyPointerCast) { + // pointer-to-unqualified overload + { + absl::any o; + EXPECT_EQ(nullptr, absl::any_cast<int>(&o)); + o.emplace<int>(); + EXPECT_NE(nullptr, absl::any_cast<int>(&o)); + o.reset(); + EXPECT_EQ(nullptr, absl::any_cast<int>(&o)); + } + + // pointer-to-const overload + { + absl::any o; + EXPECT_EQ(nullptr, absl::any_cast<int>(&AsConst(o))); + o.emplace<int>(); + EXPECT_NE(nullptr, absl::any_cast<int>(&AsConst(o))); + o.reset(); + EXPECT_EQ(nullptr, absl::any_cast<int>(&AsConst(o))); + } +} + +TEST(AnyTest, InPlaceConstruction) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type_t<IntMoveOnlyCopyOnly>(), 5, MoveOnly(), + copy_only); + IntMoveOnlyCopyOnly& v = absl::any_cast<IntMoveOnlyCopyOnly&>(o); + EXPECT_EQ(5, v.value); +} + +TEST(AnyTest, InPlaceConstructionVariableTemplate) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type<IntMoveOnlyCopyOnly>, 5, MoveOnly(), + copy_only); + auto& v = absl::any_cast<IntMoveOnlyCopyOnly&>(o); + EXPECT_EQ(5, v.value); +} + +TEST(AnyTest, InPlaceConstructionWithCV) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type_t<const volatile IntMoveOnlyCopyOnly>(), 5, + MoveOnly(), copy_only); + IntMoveOnlyCopyOnly& v = absl::any_cast<IntMoveOnlyCopyOnly&>(o); + EXPECT_EQ(5, v.value); +} + +TEST(AnyTest, InPlaceConstructionWithCVVariableTemplate) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type<const volatile IntMoveOnlyCopyOnly>, 5, + MoveOnly(), copy_only); + auto& v = absl::any_cast<IntMoveOnlyCopyOnly&>(o); + EXPECT_EQ(5, v.value); +} + +TEST(AnyTest, InPlaceConstructionWithFunction) { + absl::any o(absl::in_place_type_t<FunctionType>(), FunctionToEmplace); + FunctionType*& construction_result = absl::any_cast<FunctionType*&>(o); + EXPECT_EQ(&FunctionToEmplace, construction_result); +} + +TEST(AnyTest, InPlaceConstructionWithFunctionVariableTemplate) { + absl::any o(absl::in_place_type<FunctionType>, FunctionToEmplace); + auto& construction_result = absl::any_cast<FunctionType*&>(o); + EXPECT_EQ(&FunctionToEmplace, construction_result); +} + +TEST(AnyTest, InPlaceConstructionWithArray) { + ArrayType ar = {5, 42}; + absl::any o(absl::in_place_type_t<ArrayType>(), ar); + DecayedArray& construction_result = absl::any_cast<DecayedArray&>(o); + EXPECT_EQ(&ar[0], construction_result); +} + +TEST(AnyTest, InPlaceConstructionWithArrayVariableTemplate) { + ArrayType ar = {5, 42}; + absl::any o(absl::in_place_type<ArrayType>, ar); + auto& construction_result = absl::any_cast<DecayedArray&>(o); + EXPECT_EQ(&ar[0], construction_result); +} + +TEST(AnyTest, InPlaceConstructionIlist) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type_t<ListMoveOnlyCopyOnly>(), {1, 2, 3, 4}, + MoveOnly(), copy_only); + ListMoveOnlyCopyOnly& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + std::vector<int> expected_values = {1, 2, 3, 4}; + EXPECT_EQ(expected_values, v.values); +} + +TEST(AnyTest, InPlaceConstructionIlistVariableTemplate) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type<ListMoveOnlyCopyOnly>, {1, 2, 3, 4}, + MoveOnly(), copy_only); + auto& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + std::vector<int> expected_values = {1, 2, 3, 4}; + EXPECT_EQ(expected_values, v.values); +} + +TEST(AnyTest, InPlaceConstructionIlistWithCV) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type_t<const volatile ListMoveOnlyCopyOnly>(), + {1, 2, 3, 4}, MoveOnly(), copy_only); + ListMoveOnlyCopyOnly& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + std::vector<int> expected_values = {1, 2, 3, 4}; + EXPECT_EQ(expected_values, v.values); +} + +TEST(AnyTest, InPlaceConstructionIlistWithCVVariableTemplate) { + const CopyOnly copy_only{}; + absl::any o(absl::in_place_type<const volatile ListMoveOnlyCopyOnly>, + {1, 2, 3, 4}, MoveOnly(), copy_only); + auto& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + std::vector<int> expected_values = {1, 2, 3, 4}; + EXPECT_EQ(expected_values, v.values); +} + +TEST(AnyTest, InPlaceNoArgs) { + absl::any o(absl::in_place_type_t<int>{}); + EXPECT_EQ(0, absl::any_cast<int&>(o)); +} + +TEST(AnyTest, InPlaceNoArgsVariableTemplate) { + absl::any o(absl::in_place_type<int>); + EXPECT_EQ(0, absl::any_cast<int&>(o)); +} + +template <typename Enabler, typename T, typename... Args> +struct CanEmplaceAnyImpl : std::false_type {}; + +template <typename T, typename... Args> +struct CanEmplaceAnyImpl< + absl::void_t<decltype( + std::declval<absl::any&>().emplace<T>(std::declval<Args>()...))>, + T, Args...> : std::true_type {}; + +template <typename T, typename... Args> +using CanEmplaceAny = CanEmplaceAnyImpl<void, T, Args...>; + +TEST(AnyTest, Emplace) { + const CopyOnly copy_only{}; + absl::any o; + EXPECT_TRUE((std::is_same<decltype(o.emplace<IntMoveOnlyCopyOnly>( + 5, MoveOnly(), copy_only)), + IntMoveOnlyCopyOnly&>::value)); + IntMoveOnlyCopyOnly& emplace_result = + o.emplace<IntMoveOnlyCopyOnly>(5, MoveOnly(), copy_only); + EXPECT_EQ(5, emplace_result.value); + IntMoveOnlyCopyOnly& v = absl::any_cast<IntMoveOnlyCopyOnly&>(o); + EXPECT_EQ(5, v.value); + EXPECT_EQ(&emplace_result, &v); + + static_assert(!CanEmplaceAny<int, int, int>::value, ""); + static_assert(!CanEmplaceAny<MoveOnly, MoveOnly>::value, ""); +} + +TEST(AnyTest, EmplaceWithCV) { + const CopyOnly copy_only{}; + absl::any o; + EXPECT_TRUE( + (std::is_same<decltype(o.emplace<const volatile IntMoveOnlyCopyOnly>( + 5, MoveOnly(), copy_only)), + IntMoveOnlyCopyOnly&>::value)); + IntMoveOnlyCopyOnly& emplace_result = + o.emplace<const volatile IntMoveOnlyCopyOnly>(5, MoveOnly(), copy_only); + EXPECT_EQ(5, emplace_result.value); + IntMoveOnlyCopyOnly& v = absl::any_cast<IntMoveOnlyCopyOnly&>(o); + EXPECT_EQ(5, v.value); + EXPECT_EQ(&emplace_result, &v); +} + +TEST(AnyTest, EmplaceWithFunction) { + absl::any o; + EXPECT_TRUE( + (std::is_same<decltype(o.emplace<FunctionType>(FunctionToEmplace)), + FunctionType*&>::value)); + FunctionType*& emplace_result = o.emplace<FunctionType>(FunctionToEmplace); + EXPECT_EQ(&FunctionToEmplace, emplace_result); +} + +TEST(AnyTest, EmplaceWithArray) { + absl::any o; + ArrayType ar = {5, 42}; + EXPECT_TRUE( + (std::is_same<decltype(o.emplace<ArrayType>(ar)), DecayedArray&>::value)); + DecayedArray& emplace_result = o.emplace<ArrayType>(ar); + EXPECT_EQ(&ar[0], emplace_result); +} + +TEST(AnyTest, EmplaceIlist) { + const CopyOnly copy_only{}; + absl::any o; + EXPECT_TRUE((std::is_same<decltype(o.emplace<ListMoveOnlyCopyOnly>( + {1, 2, 3, 4}, MoveOnly(), copy_only)), + ListMoveOnlyCopyOnly&>::value)); + ListMoveOnlyCopyOnly& emplace_result = + o.emplace<ListMoveOnlyCopyOnly>({1, 2, 3, 4}, MoveOnly(), copy_only); + ListMoveOnlyCopyOnly& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + EXPECT_EQ(&v, &emplace_result); + std::vector<int> expected_values = {1, 2, 3, 4}; + EXPECT_EQ(expected_values, v.values); + + static_assert(!CanEmplaceAny<int, std::initializer_list<int>>::value, ""); + static_assert(!CanEmplaceAny<MoveOnlyWithListConstructor, + std::initializer_list<int>, int>::value, + ""); +} + +TEST(AnyTest, EmplaceIlistWithCV) { + const CopyOnly copy_only{}; + absl::any o; + EXPECT_TRUE( + (std::is_same<decltype(o.emplace<const volatile ListMoveOnlyCopyOnly>( + {1, 2, 3, 4}, MoveOnly(), copy_only)), + ListMoveOnlyCopyOnly&>::value)); + ListMoveOnlyCopyOnly& emplace_result = + o.emplace<const volatile ListMoveOnlyCopyOnly>({1, 2, 3, 4}, MoveOnly(), + copy_only); + ListMoveOnlyCopyOnly& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + EXPECT_EQ(&v, &emplace_result); + std::vector<int> expected_values = {1, 2, 3, 4}; + EXPECT_EQ(expected_values, v.values); +} + +TEST(AnyTest, EmplaceNoArgs) { + absl::any o; + o.emplace<int>(); + EXPECT_EQ(0, absl::any_cast<int>(o)); +} + +TEST(AnyTest, ConversionConstruction) { + { + absl::any o = 5; + EXPECT_EQ(5, absl::any_cast<int>(o)); + } + + { + const CopyOnly copy_only(5); + absl::any o = copy_only; + EXPECT_EQ(5, absl::any_cast<CopyOnly&>(o).value); + } + + static_assert(!std::is_convertible<MoveOnly, absl::any>::value, ""); +} + +TEST(AnyTest, ConversionAssignment) { + { + absl::any o; + o = 5; + EXPECT_EQ(5, absl::any_cast<int>(o)); + } + + { + const CopyOnly copy_only(5); + absl::any o; + o = copy_only; + EXPECT_EQ(5, absl::any_cast<CopyOnly&>(o).value); + } + + static_assert(!std::is_assignable<MoveOnly, absl::any>::value, ""); +} + +// Suppress MSVC warnings. +// 4521: multiple copy constructors specified +// We wrote multiple of them to test that the correct overloads are selected. +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable : 4521) +#endif + +// Weird type for testing, only used to make sure we "properly" perfect-forward +// when being placed into an absl::any (use the l-value constructor if given an +// l-value rather than use the copy constructor). +struct WeirdConstructor42 { + explicit WeirdConstructor42(int value) : value(value) {} + + // Copy-constructor + WeirdConstructor42(const WeirdConstructor42& other) : value(other.value) {} + + // L-value "weird" constructor (used when given an l-value) + WeirdConstructor42( + WeirdConstructor42& /*other*/) // NOLINT(runtime/references) + : value(42) {} + + int value; +}; +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + +TEST(AnyTest, WeirdConversionConstruction) { + { + const WeirdConstructor42 source(5); + absl::any o = source; // Actual copy + EXPECT_EQ(5, absl::any_cast<WeirdConstructor42&>(o).value); + } + + { + WeirdConstructor42 source(5); + absl::any o = source; // Weird "conversion" + EXPECT_EQ(42, absl::any_cast<WeirdConstructor42&>(o).value); + } +} + +TEST(AnyTest, WeirdConversionAssignment) { + { + const WeirdConstructor42 source(5); + absl::any o; + o = source; // Actual copy + EXPECT_EQ(5, absl::any_cast<WeirdConstructor42&>(o).value); + } + + { + WeirdConstructor42 source(5); + absl::any o; + o = source; // Weird "conversion" + EXPECT_EQ(42, absl::any_cast<WeirdConstructor42&>(o).value); + } +} + +struct Value {}; + +TEST(AnyTest, AnyCastValue) { + { + absl::any o; + o.emplace<int>(5); + EXPECT_EQ(5, absl::any_cast<int>(o)); + EXPECT_EQ(5, absl::any_cast<int>(AsConst(o))); + static_assert( + std::is_same<decltype(absl::any_cast<Value>(o)), Value>::value, ""); + } + + { + absl::any o; + o.emplace<int>(5); + EXPECT_EQ(5, absl::any_cast<const int>(o)); + EXPECT_EQ(5, absl::any_cast<const int>(AsConst(o))); + static_assert(std::is_same<decltype(absl::any_cast<const Value>(o)), + const Value>::value, + ""); + } +} + +TEST(AnyTest, AnyCastReference) { + { + absl::any o; + o.emplace<int>(5); + EXPECT_EQ(5, absl::any_cast<int&>(o)); + EXPECT_EQ(5, absl::any_cast<const int&>(AsConst(o))); + static_assert( + std::is_same<decltype(absl::any_cast<Value&>(o)), Value&>::value, ""); + } + + { + absl::any o; + o.emplace<int>(5); + EXPECT_EQ(5, absl::any_cast<const int>(o)); + EXPECT_EQ(5, absl::any_cast<const int>(AsConst(o))); + static_assert(std::is_same<decltype(absl::any_cast<const Value&>(o)), + const Value&>::value, + ""); + } + + { + absl::any o; + o.emplace<int>(5); + EXPECT_EQ(5, absl::any_cast<int&&>(std::move(o))); + static_assert(std::is_same<decltype(absl::any_cast<Value&&>(std::move(o))), + Value&&>::value, + ""); + } + + { + absl::any o; + o.emplace<int>(5); + EXPECT_EQ(5, absl::any_cast<const int>(std::move(o))); + static_assert( + std::is_same<decltype(absl::any_cast<const Value&&>(std::move(o))), + const Value&&>::value, + ""); + } +} + +TEST(AnyTest, AnyCastPointer) { + { + absl::any o; + EXPECT_EQ(nullptr, absl::any_cast<char>(&o)); + o.emplace<int>(5); + EXPECT_EQ(nullptr, absl::any_cast<char>(&o)); + o.emplace<char>('a'); + EXPECT_EQ('a', *absl::any_cast<char>(&o)); + static_assert( + std::is_same<decltype(absl::any_cast<Value>(&o)), Value*>::value, ""); + } + + { + absl::any o; + EXPECT_EQ(nullptr, absl::any_cast<const char>(&o)); + o.emplace<int>(5); + EXPECT_EQ(nullptr, absl::any_cast<const char>(&o)); + o.emplace<char>('a'); + EXPECT_EQ('a', *absl::any_cast<const char>(&o)); + static_assert(std::is_same<decltype(absl::any_cast<const Value>(&o)), + const Value*>::value, + ""); + } +} + +TEST(AnyTest, MakeAny) { + const CopyOnly copy_only{}; + auto o = absl::make_any<IntMoveOnlyCopyOnly>(5, MoveOnly(), copy_only); + static_assert(std::is_same<decltype(o), absl::any>::value, ""); + EXPECT_EQ(5, absl::any_cast<IntMoveOnlyCopyOnly&>(o).value); +} + +TEST(AnyTest, MakeAnyIList) { + const CopyOnly copy_only{}; + auto o = + absl::make_any<ListMoveOnlyCopyOnly>({1, 2, 3}, MoveOnly(), copy_only); + static_assert(std::is_same<decltype(o), absl::any>::value, ""); + ListMoveOnlyCopyOnly& v = absl::any_cast<ListMoveOnlyCopyOnly&>(o); + std::vector<int> expected_values = {1, 2, 3}; + EXPECT_EQ(expected_values, v.values); +} + +// Test the use of copy constructor and operator= +TEST(AnyTest, Copy) { + InstanceTracker tracker_raii; + + { + absl::any o(absl::in_place_type<CopyableOnlyInstance>, 123); + CopyableOnlyInstance* f1 = absl::any_cast<CopyableOnlyInstance>(&o); + + absl::any o2(o); + const CopyableOnlyInstance* f2 = absl::any_cast<CopyableOnlyInstance>(&o2); + EXPECT_EQ(123, f2->value()); + EXPECT_NE(f1, f2); + + absl::any o3; + o3 = o2; + const CopyableOnlyInstance* f3 = absl::any_cast<CopyableOnlyInstance>(&o3); + EXPECT_EQ(123, f3->value()); + EXPECT_NE(f2, f3); + + const absl::any o4(4); + // copy construct from const lvalue ref. + absl::any o5 = o4; + EXPECT_EQ(4, absl::any_cast<int>(o4)); + EXPECT_EQ(4, absl::any_cast<int>(o5)); + + // Copy construct from const rvalue ref. + absl::any o6 = std::move(o4); // NOLINT + EXPECT_EQ(4, absl::any_cast<int>(o4)); + EXPECT_EQ(4, absl::any_cast<int>(o6)); + } +} + +TEST(AnyTest, Move) { + InstanceTracker tracker_raii; + + absl::any any1; + any1.emplace<CopyableOnlyInstance>(5); + + // This is a copy, so copy count increases to 1. + absl::any any2 = any1; + EXPECT_EQ(5, absl::any_cast<CopyableOnlyInstance&>(any1).value()); + EXPECT_EQ(5, absl::any_cast<CopyableOnlyInstance&>(any2).value()); + EXPECT_EQ(1, tracker_raii.copies()); + + // This isn't a copy, so copy count doesn't increase. + absl::any any3 = std::move(any2); + EXPECT_EQ(5, absl::any_cast<CopyableOnlyInstance&>(any3).value()); + EXPECT_EQ(1, tracker_raii.copies()); + + absl::any any4; + any4 = std::move(any3); + EXPECT_EQ(5, absl::any_cast<CopyableOnlyInstance&>(any4).value()); + EXPECT_EQ(1, tracker_raii.copies()); + + absl::any tmp4(4); + absl::any o4(std::move(tmp4)); // move construct + EXPECT_EQ(4, absl::any_cast<int>(o4)); + o4 = *&o4; // self assign + EXPECT_EQ(4, absl::any_cast<int>(o4)); + EXPECT_TRUE(o4.has_value()); + + absl::any o5; + absl::any tmp5(5); + o5 = std::move(tmp5); // move assign + EXPECT_EQ(5, absl::any_cast<int>(o5)); +} + +// Reset the ObjectOwner with an object of a different type +TEST(AnyTest, Reset) { + absl::any o; + o.emplace<int>(); + + o.reset(); + EXPECT_FALSE(o.has_value()); + + o.emplace<char>(); + EXPECT_TRUE(o.has_value()); +} + +TEST(AnyTest, ConversionConstructionCausesOneCopy) { + InstanceTracker tracker_raii; + CopyableOnlyInstance counter(5); + absl::any o(counter); + EXPECT_EQ(5, absl::any_cast<CopyableOnlyInstance&>(o).value()); + EXPECT_EQ(1, tracker_raii.copies()); +} + +////////////////////////////////// +// Tests for Exception Behavior // +////////////////////////////////// + +#if defined(ABSL_USES_STD_ANY) + +// If using a std `any` implementation, we can't check for a specific message. +#define ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(...) \ + ABSL_BASE_INTERNAL_EXPECT_FAIL((__VA_ARGS__), absl::bad_any_cast, \ + "") + +#else + +// If using the absl `any` implementation, we can rely on a specific message. +#define ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(...) \ + ABSL_BASE_INTERNAL_EXPECT_FAIL((__VA_ARGS__), absl::bad_any_cast, \ + "Bad any cast") + +#endif // defined(ABSL_USES_STD_ANY) + +TEST(AnyTest, ThrowBadAlloc) { + { + absl::any a; + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<int&>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const int&>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<int&&>(absl::any{})); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const int&&>(absl::any{})); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<int>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const int>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<int>(absl::any{})); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const int>(absl::any{})); + + // const absl::any operand + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const int&>(AsConst(a))); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<int>(AsConst(a))); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const int>(AsConst(a))); + } + + { + absl::any a(absl::in_place_type<int>); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<float&>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const float&>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<float&&>(absl::any{})); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST( + absl::any_cast<const float&&>(absl::any{})); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<float>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const float>(a)); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<float>(absl::any{})); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const float>(absl::any{})); + + // const absl::any operand + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const float&>(AsConst(a))); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<float>(AsConst(a))); + ABSL_ANY_TEST_EXPECT_BAD_ANY_CAST(absl::any_cast<const float>(AsConst(a))); + } +} + +class BadCopy {}; + +struct BadCopyable { + BadCopyable() = default; + BadCopyable(BadCopyable&&) = default; + BadCopyable(const BadCopyable&) { +#ifdef ABSL_HAVE_EXCEPTIONS + throw BadCopy(); +#else + ABSL_RAW_LOG(FATAL, "Bad copy"); +#endif + } +}; + +#define ABSL_ANY_TEST_EXPECT_BAD_COPY(...) \ + ABSL_BASE_INTERNAL_EXPECT_FAIL((__VA_ARGS__), BadCopy, "Bad copy") + +// Test the guarantees regarding exceptions in copy/assign. +TEST(AnyTest, FailedCopy) { + { + const BadCopyable bad{}; + ABSL_ANY_TEST_EXPECT_BAD_COPY(absl::any{bad}); + } + + { + absl::any src(absl::in_place_type<BadCopyable>); + ABSL_ANY_TEST_EXPECT_BAD_COPY(absl::any{src}); + } + + { + BadCopyable bad; + absl::any target; + ABSL_ANY_TEST_EXPECT_BAD_COPY(target = bad); + } + + { + BadCopyable bad; + absl::any target(absl::in_place_type<BadCopyable>); + ABSL_ANY_TEST_EXPECT_BAD_COPY(target = bad); + EXPECT_TRUE(target.has_value()); + } + + { + absl::any src(absl::in_place_type<BadCopyable>); + absl::any target; + ABSL_ANY_TEST_EXPECT_BAD_COPY(target = src); + EXPECT_FALSE(target.has_value()); + } + + { + absl::any src(absl::in_place_type<BadCopyable>); + absl::any target(absl::in_place_type<BadCopyable>); + ABSL_ANY_TEST_EXPECT_BAD_COPY(target = src); + EXPECT_TRUE(target.has_value()); + } +} + +// Test the guarantees regarding exceptions in emplace. +TEST(AnyTest, FailedEmplace) { + { + BadCopyable bad; + absl::any target; + ABSL_ANY_TEST_EXPECT_BAD_COPY(target.emplace<BadCopyable>(bad)); + } + + { + BadCopyable bad; + absl::any target(absl::in_place_type<int>); + ABSL_ANY_TEST_EXPECT_BAD_COPY(target.emplace<BadCopyable>(bad)); +#if defined(ABSL_USES_STD_ANY) && defined(__GLIBCXX__) + // libstdc++ std::any::emplace() implementation (as of 7.2) has a bug: if an + // exception is thrown, *this contains a value. +#define ABSL_GLIBCXX_ANY_EMPLACE_EXCEPTION_BUG 1 +#endif +#if defined(ABSL_HAVE_EXCEPTIONS) && \ + !defined(ABSL_GLIBCXX_ANY_EMPLACE_EXCEPTION_BUG) + EXPECT_FALSE(target.has_value()); +#endif + } +} + +} // namespace + +#endif // #if !defined(ABSL_USES_STD_ANY) diff --git a/third_party/abseil_cpp/absl/types/bad_any_cast.cc b/third_party/abseil_cpp/absl/types/bad_any_cast.cc new file mode 100644 index 000000000000..b0592cc9bcc5 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/bad_any_cast.cc @@ -0,0 +1,46 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/bad_any_cast.h" + +#ifndef ABSL_USES_STD_ANY + +#include <cstdlib> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +bad_any_cast::~bad_any_cast() = default; + +const char* bad_any_cast::what() const noexcept { return "Bad any cast"; } + +namespace any_internal { + +void ThrowBadAnyCast() { +#ifdef ABSL_HAVE_EXCEPTIONS + throw bad_any_cast(); +#else + ABSL_RAW_LOG(FATAL, "Bad any cast"); + std::abort(); +#endif +} + +} // namespace any_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_ANY diff --git a/third_party/abseil_cpp/absl/types/bad_any_cast.h b/third_party/abseil_cpp/absl/types/bad_any_cast.h new file mode 100644 index 000000000000..114cef80cdde --- /dev/null +++ b/third_party/abseil_cpp/absl/types/bad_any_cast.h @@ -0,0 +1,75 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// bad_any_cast.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::bad_any_cast` type. + +#ifndef ABSL_TYPES_BAD_ANY_CAST_H_ +#define ABSL_TYPES_BAD_ANY_CAST_H_ + +#include <typeinfo> + +#include "absl/base/config.h" + +#ifdef ABSL_USES_STD_ANY + +#include <any> + +namespace absl { +ABSL_NAMESPACE_BEGIN +using std::bad_any_cast; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_ANY + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// bad_any_cast +// ----------------------------------------------------------------------------- +// +// An `absl::bad_any_cast` type is an exception type that is thrown when +// failing to successfully cast the return value of an `absl::any` object. +// +// Example: +// +// auto a = absl::any(65); +// absl::any_cast<int>(a); // 65 +// try { +// absl::any_cast<char>(a); +// } catch(const absl::bad_any_cast& e) { +// std::cout << "Bad any cast: " << e.what() << '\n'; +// } +class bad_any_cast : public std::bad_cast { + public: + ~bad_any_cast() override; + const char* what() const noexcept override; +}; + +namespace any_internal { + +[[noreturn]] void ThrowBadAnyCast(); + +} // namespace any_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_ANY + +#endif // ABSL_TYPES_BAD_ANY_CAST_H_ diff --git a/third_party/abseil_cpp/absl/types/bad_optional_access.cc b/third_party/abseil_cpp/absl/types/bad_optional_access.cc new file mode 100644 index 000000000000..26aca70d9cff --- /dev/null +++ b/third_party/abseil_cpp/absl/types/bad_optional_access.cc @@ -0,0 +1,48 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/bad_optional_access.h" + +#ifndef ABSL_USES_STD_OPTIONAL + +#include <cstdlib> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +bad_optional_access::~bad_optional_access() = default; + +const char* bad_optional_access::what() const noexcept { + return "optional has no value"; +} + +namespace optional_internal { + +void throw_bad_optional_access() { +#ifdef ABSL_HAVE_EXCEPTIONS + throw bad_optional_access(); +#else + ABSL_RAW_LOG(FATAL, "Bad optional access"); + abort(); +#endif +} + +} // namespace optional_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_OPTIONAL diff --git a/third_party/abseil_cpp/absl/types/bad_optional_access.h b/third_party/abseil_cpp/absl/types/bad_optional_access.h new file mode 100644 index 000000000000..a500286adc42 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/bad_optional_access.h @@ -0,0 +1,78 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// bad_optional_access.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::bad_optional_access` type. + +#ifndef ABSL_TYPES_BAD_OPTIONAL_ACCESS_H_ +#define ABSL_TYPES_BAD_OPTIONAL_ACCESS_H_ + +#include <stdexcept> + +#include "absl/base/config.h" + +#ifdef ABSL_USES_STD_OPTIONAL + +#include <optional> + +namespace absl { +ABSL_NAMESPACE_BEGIN +using std::bad_optional_access; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_OPTIONAL + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// bad_optional_access +// ----------------------------------------------------------------------------- +// +// An `absl::bad_optional_access` type is an exception type that is thrown when +// attempting to access an `absl::optional` object that does not contain a +// value. +// +// Example: +// +// absl::optional<int> o; +// +// try { +// int n = o.value(); +// } catch(const absl::bad_optional_access& e) { +// std::cout << "Bad optional access: " << e.what() << '\n'; +// } +class bad_optional_access : public std::exception { + public: + bad_optional_access() = default; + ~bad_optional_access() override; + const char* what() const noexcept override; +}; + +namespace optional_internal { + +// throw delegator +[[noreturn]] void throw_bad_optional_access(); + +} // namespace optional_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_OPTIONAL + +#endif // ABSL_TYPES_BAD_OPTIONAL_ACCESS_H_ diff --git a/third_party/abseil_cpp/absl/types/bad_variant_access.cc b/third_party/abseil_cpp/absl/types/bad_variant_access.cc new file mode 100644 index 000000000000..3dc88cc09f23 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/bad_variant_access.cc @@ -0,0 +1,64 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/bad_variant_access.h" + +#ifndef ABSL_USES_STD_VARIANT + +#include <cstdlib> +#include <stdexcept> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +////////////////////////// +// [variant.bad.access] // +////////////////////////// + +bad_variant_access::~bad_variant_access() = default; + +const char* bad_variant_access::what() const noexcept { + return "Bad variant access"; +} + +namespace variant_internal { + +void ThrowBadVariantAccess() { +#ifdef ABSL_HAVE_EXCEPTIONS + throw bad_variant_access(); +#else + ABSL_RAW_LOG(FATAL, "Bad variant access"); + abort(); // TODO(calabrese) Remove once RAW_LOG FATAL is noreturn. +#endif +} + +void Rethrow() { +#ifdef ABSL_HAVE_EXCEPTIONS + throw; +#else + ABSL_RAW_LOG(FATAL, + "Internal error in absl::variant implementation. Attempted to " + "rethrow an exception when building with exceptions disabled."); + abort(); // TODO(calabrese) Remove once RAW_LOG FATAL is noreturn. +#endif +} + +} // namespace variant_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_VARIANT diff --git a/third_party/abseil_cpp/absl/types/bad_variant_access.h b/third_party/abseil_cpp/absl/types/bad_variant_access.h new file mode 100644 index 000000000000..095969f91eee --- /dev/null +++ b/third_party/abseil_cpp/absl/types/bad_variant_access.h @@ -0,0 +1,82 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// bad_variant_access.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::bad_variant_access` type. + +#ifndef ABSL_TYPES_BAD_VARIANT_ACCESS_H_ +#define ABSL_TYPES_BAD_VARIANT_ACCESS_H_ + +#include <stdexcept> + +#include "absl/base/config.h" + +#ifdef ABSL_USES_STD_VARIANT + +#include <variant> + +namespace absl { +ABSL_NAMESPACE_BEGIN +using std::bad_variant_access; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_VARIANT + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// bad_variant_access +// ----------------------------------------------------------------------------- +// +// An `absl::bad_variant_access` type is an exception type that is thrown in +// the following cases: +// +// * Calling `absl::get(absl::variant) with an index or type that does not +// match the currently selected alternative type +// * Calling `absl::visit on an `absl::variant` that is in the +// `variant::valueless_by_exception` state. +// +// Example: +// +// absl::variant<int, std::string> v; +// v = 1; +// try { +// absl::get<std::string>(v); +// } catch(const absl::bad_variant_access& e) { +// std::cout << "Bad variant access: " << e.what() << '\n'; +// } +class bad_variant_access : public std::exception { + public: + bad_variant_access() noexcept = default; + ~bad_variant_access() override; + const char* what() const noexcept override; +}; + +namespace variant_internal { + +[[noreturn]] void ThrowBadVariantAccess(); +[[noreturn]] void Rethrow(); + +} // namespace variant_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_USES_STD_VARIANT + +#endif // ABSL_TYPES_BAD_VARIANT_ACCESS_H_ diff --git a/third_party/abseil_cpp/absl/types/compare.h b/third_party/abseil_cpp/absl/types/compare.h new file mode 100644 index 000000000000..19b076e7f19c --- /dev/null +++ b/third_party/abseil_cpp/absl/types/compare.h @@ -0,0 +1,600 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// compare.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::weak_equality`, `absl::strong_equality`, +// `absl::partial_ordering`, `absl::weak_ordering`, and `absl::strong_ordering` +// types for storing the results of three way comparisons. +// +// Example: +// absl::weak_ordering compare(const std::string& a, const std::string& b); +// +// These are C++11 compatible versions of the C++20 corresponding types +// (`std::weak_equality`, etc.) and are designed to be drop-in replacements +// for code compliant with C++20. + +#ifndef ABSL_TYPES_COMPARE_H_ +#define ABSL_TYPES_COMPARE_H_ + +#include <cstddef> +#include <cstdint> +#include <cstdlib> +#include <type_traits> + +#include "absl/base/attributes.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace compare_internal { + +using value_type = int8_t; + +template <typename T> +struct Fail { + static_assert(sizeof(T) < 0, "Only literal `0` is allowed."); +}; + +// We need the NullPtrT template to avoid triggering the modernize-use-nullptr +// ClangTidy warning in user code. +template <typename NullPtrT = std::nullptr_t> +struct OnlyLiteralZero { + constexpr OnlyLiteralZero(NullPtrT) noexcept {} // NOLINT + + // Fails compilation when `nullptr` or integral type arguments other than + // `int` are passed. This constructor doesn't accept `int` because literal `0` + // has type `int`. Literal `0` arguments will be implicitly converted to + // `std::nullptr_t` and accepted by the above constructor, while other `int` + // arguments will fail to be converted and cause compilation failure. + template < + typename T, + typename = typename std::enable_if< + std::is_same<T, std::nullptr_t>::value || + (std::is_integral<T>::value && !std::is_same<T, int>::value)>::type, + typename = typename Fail<T>::type> + OnlyLiteralZero(T); // NOLINT +}; + +enum class eq : value_type { + equal = 0, + equivalent = equal, + nonequal = 1, + nonequivalent = nonequal, +}; + +enum class ord : value_type { less = -1, greater = 1 }; + +enum class ncmp : value_type { unordered = -127 }; + +// Define macros to allow for creation or emulation of C++17 inline variables +// based on whether the feature is supported. Note: we can't use +// ABSL_INTERNAL_INLINE_CONSTEXPR here because the variables here are of +// incomplete types so they need to be defined after the types are complete. +#ifdef __cpp_inline_variables + +// A no-op expansion that can be followed by a semicolon at class level. +#define ABSL_COMPARE_INLINE_BASECLASS_DECL(name) static_assert(true, "") + +#define ABSL_COMPARE_INLINE_SUBCLASS_DECL(type, name) \ + static const type name + +#define ABSL_COMPARE_INLINE_INIT(type, name, init) \ + inline constexpr type type::name(init) + +#else // __cpp_inline_variables + +#define ABSL_COMPARE_INLINE_BASECLASS_DECL(name) \ + ABSL_CONST_INIT static const T name + +// A no-op expansion that can be followed by a semicolon at class level. +#define ABSL_COMPARE_INLINE_SUBCLASS_DECL(type, name) static_assert(true, "") + +#define ABSL_COMPARE_INLINE_INIT(type, name, init) \ + template <typename T> \ + const T compare_internal::type##_base<T>::name(init) + +#endif // __cpp_inline_variables + +// These template base classes allow for defining the values of the constants +// in the header file (for performance) without using inline variables (which +// aren't available in C++11). +template <typename T> +struct weak_equality_base { + ABSL_COMPARE_INLINE_BASECLASS_DECL(equivalent); + ABSL_COMPARE_INLINE_BASECLASS_DECL(nonequivalent); +}; + +template <typename T> +struct strong_equality_base { + ABSL_COMPARE_INLINE_BASECLASS_DECL(equal); + ABSL_COMPARE_INLINE_BASECLASS_DECL(nonequal); + ABSL_COMPARE_INLINE_BASECLASS_DECL(equivalent); + ABSL_COMPARE_INLINE_BASECLASS_DECL(nonequivalent); +}; + +template <typename T> +struct partial_ordering_base { + ABSL_COMPARE_INLINE_BASECLASS_DECL(less); + ABSL_COMPARE_INLINE_BASECLASS_DECL(equivalent); + ABSL_COMPARE_INLINE_BASECLASS_DECL(greater); + ABSL_COMPARE_INLINE_BASECLASS_DECL(unordered); +}; + +template <typename T> +struct weak_ordering_base { + ABSL_COMPARE_INLINE_BASECLASS_DECL(less); + ABSL_COMPARE_INLINE_BASECLASS_DECL(equivalent); + ABSL_COMPARE_INLINE_BASECLASS_DECL(greater); +}; + +template <typename T> +struct strong_ordering_base { + ABSL_COMPARE_INLINE_BASECLASS_DECL(less); + ABSL_COMPARE_INLINE_BASECLASS_DECL(equal); + ABSL_COMPARE_INLINE_BASECLASS_DECL(equivalent); + ABSL_COMPARE_INLINE_BASECLASS_DECL(greater); +}; + +} // namespace compare_internal + +class weak_equality + : public compare_internal::weak_equality_base<weak_equality> { + explicit constexpr weak_equality(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::weak_equality_base<weak_equality>; + + public: + ABSL_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, equivalent); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(weak_equality, nonequivalent); + + // Comparisons + friend constexpr bool operator==( + weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + weak_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_equality v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator==(weak_equality v1, + weak_equality v2) noexcept { + return v1.value_ == v2.value_; + } + friend constexpr bool operator!=(weak_equality v1, + weak_equality v2) noexcept { + return v1.value_ != v2.value_; + } + + private: + compare_internal::value_type value_; +}; +ABSL_COMPARE_INLINE_INIT(weak_equality, equivalent, + compare_internal::eq::equivalent); +ABSL_COMPARE_INLINE_INIT(weak_equality, nonequivalent, + compare_internal::eq::nonequivalent); + +class strong_equality + : public compare_internal::strong_equality_base<strong_equality> { + explicit constexpr strong_equality(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::strong_equality_base<strong_equality>; + + public: + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equal); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequal); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, equivalent); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_equality, nonequivalent); + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==( + strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + strong_equality v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_equality v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator==(strong_equality v1, + strong_equality v2) noexcept { + return v1.value_ == v2.value_; + } + friend constexpr bool operator!=(strong_equality v1, + strong_equality v2) noexcept { + return v1.value_ != v2.value_; + } + + private: + compare_internal::value_type value_; +}; +ABSL_COMPARE_INLINE_INIT(strong_equality, equal, compare_internal::eq::equal); +ABSL_COMPARE_INLINE_INIT(strong_equality, nonequal, + compare_internal::eq::nonequal); +ABSL_COMPARE_INLINE_INIT(strong_equality, equivalent, + compare_internal::eq::equivalent); +ABSL_COMPARE_INLINE_INIT(strong_equality, nonequivalent, + compare_internal::eq::nonequivalent); + +class partial_ordering + : public compare_internal::partial_ordering_base<partial_ordering> { + explicit constexpr partial_ordering(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr partial_ordering(compare_internal::ord v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr partial_ordering(compare_internal::ncmp v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::partial_ordering_base<partial_ordering>; + + constexpr bool is_ordered() const noexcept { + return value_ != + compare_internal::value_type(compare_internal::ncmp::unordered); + } + + public: + ABSL_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, less); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, equivalent); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, greater); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(partial_ordering, unordered); + + // Conversion + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + // Comparisons + friend constexpr bool operator==( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ == 0; + } + friend constexpr bool operator!=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return !v.is_ordered() || v.value_ != 0; + } + friend constexpr bool operator<( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ < 0; + } + friend constexpr bool operator<=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ <= 0; + } + friend constexpr bool operator>( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ > 0; + } + friend constexpr bool operator>=( + partial_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.is_ordered() && v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return !v.is_ordered() || 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + partial_ordering v) noexcept { + return v.is_ordered() && 0 >= v.value_; + } + friend constexpr bool operator==(partial_ordering v1, + partial_ordering v2) noexcept { + return v1.value_ == v2.value_; + } + friend constexpr bool operator!=(partial_ordering v1, + partial_ordering v2) noexcept { + return v1.value_ != v2.value_; + } + + private: + compare_internal::value_type value_; +}; +ABSL_COMPARE_INLINE_INIT(partial_ordering, less, compare_internal::ord::less); +ABSL_COMPARE_INLINE_INIT(partial_ordering, equivalent, + compare_internal::eq::equivalent); +ABSL_COMPARE_INLINE_INIT(partial_ordering, greater, + compare_internal::ord::greater); +ABSL_COMPARE_INLINE_INIT(partial_ordering, unordered, + compare_internal::ncmp::unordered); + +class weak_ordering + : public compare_internal::weak_ordering_base<weak_ordering> { + explicit constexpr weak_ordering(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr weak_ordering(compare_internal::ord v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::weak_ordering_base<weak_ordering>; + + public: + ABSL_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, less); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, equivalent); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(weak_ordering, greater); + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less + : partial_ordering::greater); + } + // Comparisons + friend constexpr bool operator==( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=( + weak_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + weak_ordering v) noexcept { + return 0 >= v.value_; + } + friend constexpr bool operator==(weak_ordering v1, + weak_ordering v2) noexcept { + return v1.value_ == v2.value_; + } + friend constexpr bool operator!=(weak_ordering v1, + weak_ordering v2) noexcept { + return v1.value_ != v2.value_; + } + + private: + compare_internal::value_type value_; +}; +ABSL_COMPARE_INLINE_INIT(weak_ordering, less, compare_internal::ord::less); +ABSL_COMPARE_INLINE_INIT(weak_ordering, equivalent, + compare_internal::eq::equivalent); +ABSL_COMPARE_INLINE_INIT(weak_ordering, greater, + compare_internal::ord::greater); + +class strong_ordering + : public compare_internal::strong_ordering_base<strong_ordering> { + explicit constexpr strong_ordering(compare_internal::eq v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + explicit constexpr strong_ordering(compare_internal::ord v) noexcept + : value_(static_cast<compare_internal::value_type>(v)) {} + friend struct compare_internal::strong_ordering_base<strong_ordering>; + + public: + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, less); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equal); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, equivalent); + ABSL_COMPARE_INLINE_SUBCLASS_DECL(strong_ordering, greater); + + // Conversions + constexpr operator weak_equality() const noexcept { // NOLINT + return value_ == 0 ? weak_equality::equivalent + : weak_equality::nonequivalent; + } + constexpr operator strong_equality() const noexcept { // NOLINT + return value_ == 0 ? strong_equality::equal : strong_equality::nonequal; + } + constexpr operator partial_ordering() const noexcept { // NOLINT + return value_ == 0 ? partial_ordering::equivalent + : (value_ < 0 ? partial_ordering::less + : partial_ordering::greater); + } + constexpr operator weak_ordering() const noexcept { // NOLINT + return value_ == 0 + ? weak_ordering::equivalent + : (value_ < 0 ? weak_ordering::less : weak_ordering::greater); + } + // Comparisons + friend constexpr bool operator==( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ == 0; + } + friend constexpr bool operator!=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ != 0; + } + friend constexpr bool operator<( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ < 0; + } + friend constexpr bool operator<=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ <= 0; + } + friend constexpr bool operator>( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ > 0; + } + friend constexpr bool operator>=( + strong_ordering v, compare_internal::OnlyLiteralZero<>) noexcept { + return v.value_ >= 0; + } + friend constexpr bool operator==(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 == v.value_; + } + friend constexpr bool operator!=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 != v.value_; + } + friend constexpr bool operator<(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 < v.value_; + } + friend constexpr bool operator<=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 <= v.value_; + } + friend constexpr bool operator>(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 > v.value_; + } + friend constexpr bool operator>=(compare_internal::OnlyLiteralZero<>, + strong_ordering v) noexcept { + return 0 >= v.value_; + } + friend constexpr bool operator==(strong_ordering v1, + strong_ordering v2) noexcept { + return v1.value_ == v2.value_; + } + friend constexpr bool operator!=(strong_ordering v1, + strong_ordering v2) noexcept { + return v1.value_ != v2.value_; + } + + private: + compare_internal::value_type value_; +}; +ABSL_COMPARE_INLINE_INIT(strong_ordering, less, compare_internal::ord::less); +ABSL_COMPARE_INLINE_INIT(strong_ordering, equal, compare_internal::eq::equal); +ABSL_COMPARE_INLINE_INIT(strong_ordering, equivalent, + compare_internal::eq::equivalent); +ABSL_COMPARE_INLINE_INIT(strong_ordering, greater, + compare_internal::ord::greater); + +#undef ABSL_COMPARE_INLINE_BASECLASS_DECL +#undef ABSL_COMPARE_INLINE_SUBCLASS_DECL +#undef ABSL_COMPARE_INLINE_INIT + +namespace compare_internal { +// We also provide these comparator adapter functions for internal absl use. + +// Helper functions to do a boolean comparison of two keys given a boolean +// or three-way comparator. +// SFINAE prevents implicit conversions to bool (such as from int). +template <typename Bool, + absl::enable_if_t<std::is_same<bool, Bool>::value, int> = 0> +constexpr bool compare_result_as_less_than(const Bool r) { return r; } +constexpr bool compare_result_as_less_than(const absl::weak_ordering r) { + return r < 0; +} + +template <typename Compare, typename K, typename LK> +constexpr bool do_less_than_comparison(const Compare &compare, const K &x, + const LK &y) { + return compare_result_as_less_than(compare(x, y)); +} + +// Helper functions to do a three-way comparison of two keys given a boolean or +// three-way comparator. +// SFINAE prevents implicit conversions to int (such as from bool). +template <typename Int, + absl::enable_if_t<std::is_same<int, Int>::value, int> = 0> +constexpr absl::weak_ordering compare_result_as_ordering(const Int c) { + return c < 0 ? absl::weak_ordering::less + : c == 0 ? absl::weak_ordering::equivalent + : absl::weak_ordering::greater; +} +constexpr absl::weak_ordering compare_result_as_ordering( + const absl::weak_ordering c) { + return c; +} + +template < + typename Compare, typename K, typename LK, + absl::enable_if_t<!std::is_same<bool, absl::result_of_t<Compare( + const K &, const LK &)>>::value, + int> = 0> +constexpr absl::weak_ordering do_three_way_comparison(const Compare &compare, + const K &x, const LK &y) { + return compare_result_as_ordering(compare(x, y)); +} +template < + typename Compare, typename K, typename LK, + absl::enable_if_t<std::is_same<bool, absl::result_of_t<Compare( + const K &, const LK &)>>::value, + int> = 0> +constexpr absl::weak_ordering do_three_way_comparison(const Compare &compare, + const K &x, const LK &y) { + return compare(x, y) ? absl::weak_ordering::less + : compare(y, x) ? absl::weak_ordering::greater + : absl::weak_ordering::equivalent; +} + +} // namespace compare_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TYPES_COMPARE_H_ diff --git a/third_party/abseil_cpp/absl/types/compare_test.cc b/third_party/abseil_cpp/absl/types/compare_test.cc new file mode 100644 index 000000000000..8095baf9569f --- /dev/null +++ b/third_party/abseil_cpp/absl/types/compare_test.cc @@ -0,0 +1,389 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/compare.h" + +#include "gtest/gtest.h" +#include "absl/base/casts.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +// This is necessary to avoid a bunch of lint warnings suggesting that we use +// EXPECT_EQ/etc., which doesn't work in this case because they convert the `0` +// to an int, which can't be converted to the unspecified zero type. +bool Identity(bool b) { return b; } + +TEST(Compare, WeakEquality) { + EXPECT_TRUE(Identity(weak_equality::equivalent == 0)); + EXPECT_TRUE(Identity(0 == weak_equality::equivalent)); + EXPECT_TRUE(Identity(weak_equality::nonequivalent != 0)); + EXPECT_TRUE(Identity(0 != weak_equality::nonequivalent)); + const weak_equality values[] = {weak_equality::equivalent, + weak_equality::nonequivalent}; + for (const auto& lhs : values) { + for (const auto& rhs : values) { + const bool are_equal = &lhs == &rhs; + EXPECT_EQ(lhs == rhs, are_equal); + EXPECT_EQ(lhs != rhs, !are_equal); + } + } +} + +TEST(Compare, StrongEquality) { + EXPECT_TRUE(Identity(strong_equality::equal == 0)); + EXPECT_TRUE(Identity(0 == strong_equality::equal)); + EXPECT_TRUE(Identity(strong_equality::nonequal != 0)); + EXPECT_TRUE(Identity(0 != strong_equality::nonequal)); + EXPECT_TRUE(Identity(strong_equality::equivalent == 0)); + EXPECT_TRUE(Identity(0 == strong_equality::equivalent)); + EXPECT_TRUE(Identity(strong_equality::nonequivalent != 0)); + EXPECT_TRUE(Identity(0 != strong_equality::nonequivalent)); + const strong_equality values[] = {strong_equality::equal, + strong_equality::nonequal}; + for (const auto& lhs : values) { + for (const auto& rhs : values) { + const bool are_equal = &lhs == &rhs; + EXPECT_EQ(lhs == rhs, are_equal); + EXPECT_EQ(lhs != rhs, !are_equal); + } + } + EXPECT_TRUE(Identity(strong_equality::equivalent == strong_equality::equal)); + EXPECT_TRUE( + Identity(strong_equality::nonequivalent == strong_equality::nonequal)); +} + +TEST(Compare, PartialOrdering) { + EXPECT_TRUE(Identity(partial_ordering::less < 0)); + EXPECT_TRUE(Identity(0 > partial_ordering::less)); + EXPECT_TRUE(Identity(partial_ordering::less <= 0)); + EXPECT_TRUE(Identity(0 >= partial_ordering::less)); + EXPECT_TRUE(Identity(partial_ordering::equivalent == 0)); + EXPECT_TRUE(Identity(0 == partial_ordering::equivalent)); + EXPECT_TRUE(Identity(partial_ordering::greater > 0)); + EXPECT_TRUE(Identity(0 < partial_ordering::greater)); + EXPECT_TRUE(Identity(partial_ordering::greater >= 0)); + EXPECT_TRUE(Identity(0 <= partial_ordering::greater)); + EXPECT_TRUE(Identity(partial_ordering::unordered != 0)); + EXPECT_TRUE(Identity(0 != partial_ordering::unordered)); + EXPECT_FALSE(Identity(partial_ordering::unordered < 0)); + EXPECT_FALSE(Identity(0 < partial_ordering::unordered)); + EXPECT_FALSE(Identity(partial_ordering::unordered <= 0)); + EXPECT_FALSE(Identity(0 <= partial_ordering::unordered)); + EXPECT_FALSE(Identity(partial_ordering::unordered > 0)); + EXPECT_FALSE(Identity(0 > partial_ordering::unordered)); + EXPECT_FALSE(Identity(partial_ordering::unordered >= 0)); + EXPECT_FALSE(Identity(0 >= partial_ordering::unordered)); + const partial_ordering values[] = { + partial_ordering::less, partial_ordering::equivalent, + partial_ordering::greater, partial_ordering::unordered}; + for (const auto& lhs : values) { + for (const auto& rhs : values) { + const bool are_equal = &lhs == &rhs; + EXPECT_EQ(lhs == rhs, are_equal); + EXPECT_EQ(lhs != rhs, !are_equal); + } + } +} + +TEST(Compare, WeakOrdering) { + EXPECT_TRUE(Identity(weak_ordering::less < 0)); + EXPECT_TRUE(Identity(0 > weak_ordering::less)); + EXPECT_TRUE(Identity(weak_ordering::less <= 0)); + EXPECT_TRUE(Identity(0 >= weak_ordering::less)); + EXPECT_TRUE(Identity(weak_ordering::equivalent == 0)); + EXPECT_TRUE(Identity(0 == weak_ordering::equivalent)); + EXPECT_TRUE(Identity(weak_ordering::greater > 0)); + EXPECT_TRUE(Identity(0 < weak_ordering::greater)); + EXPECT_TRUE(Identity(weak_ordering::greater >= 0)); + EXPECT_TRUE(Identity(0 <= weak_ordering::greater)); + const weak_ordering values[] = { + weak_ordering::less, weak_ordering::equivalent, weak_ordering::greater}; + for (const auto& lhs : values) { + for (const auto& rhs : values) { + const bool are_equal = &lhs == &rhs; + EXPECT_EQ(lhs == rhs, are_equal); + EXPECT_EQ(lhs != rhs, !are_equal); + } + } +} + +TEST(Compare, StrongOrdering) { + EXPECT_TRUE(Identity(strong_ordering::less < 0)); + EXPECT_TRUE(Identity(0 > strong_ordering::less)); + EXPECT_TRUE(Identity(strong_ordering::less <= 0)); + EXPECT_TRUE(Identity(0 >= strong_ordering::less)); + EXPECT_TRUE(Identity(strong_ordering::equal == 0)); + EXPECT_TRUE(Identity(0 == strong_ordering::equal)); + EXPECT_TRUE(Identity(strong_ordering::equivalent == 0)); + EXPECT_TRUE(Identity(0 == strong_ordering::equivalent)); + EXPECT_TRUE(Identity(strong_ordering::greater > 0)); + EXPECT_TRUE(Identity(0 < strong_ordering::greater)); + EXPECT_TRUE(Identity(strong_ordering::greater >= 0)); + EXPECT_TRUE(Identity(0 <= strong_ordering::greater)); + const strong_ordering values[] = { + strong_ordering::less, strong_ordering::equal, strong_ordering::greater}; + for (const auto& lhs : values) { + for (const auto& rhs : values) { + const bool are_equal = &lhs == &rhs; + EXPECT_EQ(lhs == rhs, are_equal); + EXPECT_EQ(lhs != rhs, !are_equal); + } + } + EXPECT_TRUE(Identity(strong_ordering::equivalent == strong_ordering::equal)); +} + +TEST(Compare, Conversions) { + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_equality::equal) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_equality::nonequal) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_equality::equivalent) == 0)); + EXPECT_TRUE(Identity( + implicit_cast<weak_equality>(strong_equality::nonequivalent) != 0)); + + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(partial_ordering::less) != 0)); + EXPECT_TRUE(Identity( + implicit_cast<weak_equality>(partial_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(partial_ordering::greater) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(partial_ordering::unordered) != 0)); + + EXPECT_TRUE(implicit_cast<weak_equality>(weak_ordering::less) != 0); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(weak_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(weak_ordering::greater) != 0)); + + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(weak_ordering::less) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(weak_ordering::less) < 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(weak_ordering::less) <= 0)); + EXPECT_TRUE(Identity( + implicit_cast<partial_ordering>(weak_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(weak_ordering::greater) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(weak_ordering::greater) > 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(weak_ordering::greater) >= 0)); + + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_ordering::less) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_ordering::equal) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_equality>(strong_ordering::greater) != 0)); + + EXPECT_TRUE( + Identity(implicit_cast<strong_equality>(strong_ordering::less) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<strong_equality>(strong_ordering::equal) == 0)); + EXPECT_TRUE(Identity( + implicit_cast<strong_equality>(strong_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<strong_equality>(strong_ordering::greater) != 0)); + + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::less) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::less) < 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::less) <= 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::equal) == 0)); + EXPECT_TRUE(Identity( + implicit_cast<partial_ordering>(strong_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::greater) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::greater) > 0)); + EXPECT_TRUE( + Identity(implicit_cast<partial_ordering>(strong_ordering::greater) >= 0)); + + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::less) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::less) < 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::less) <= 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::equal) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::equivalent) == 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::greater) != 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::greater) > 0)); + EXPECT_TRUE( + Identity(implicit_cast<weak_ordering>(strong_ordering::greater) >= 0)); +} + +struct WeakOrderingLess { + template <typename T> + absl::weak_ordering operator()(const T& a, const T& b) const { + return a < b ? absl::weak_ordering::less + : a == b ? absl::weak_ordering::equivalent + : absl::weak_ordering::greater; + } +}; + +TEST(CompareResultAsLessThan, SanityTest) { + EXPECT_FALSE(absl::compare_internal::compare_result_as_less_than(false)); + EXPECT_TRUE(absl::compare_internal::compare_result_as_less_than(true)); + + EXPECT_TRUE( + absl::compare_internal::compare_result_as_less_than(weak_ordering::less)); + EXPECT_FALSE(absl::compare_internal::compare_result_as_less_than( + weak_ordering::equivalent)); + EXPECT_FALSE(absl::compare_internal::compare_result_as_less_than( + weak_ordering::greater)); +} + +TEST(DoLessThanComparison, SanityTest) { + std::less<int> less; + WeakOrderingLess weak; + + EXPECT_TRUE(absl::compare_internal::do_less_than_comparison(less, -1, 0)); + EXPECT_TRUE(absl::compare_internal::do_less_than_comparison(weak, -1, 0)); + + EXPECT_FALSE(absl::compare_internal::do_less_than_comparison(less, 10, 10)); + EXPECT_FALSE(absl::compare_internal::do_less_than_comparison(weak, 10, 10)); + + EXPECT_FALSE(absl::compare_internal::do_less_than_comparison(less, 10, 5)); + EXPECT_FALSE(absl::compare_internal::do_less_than_comparison(weak, 10, 5)); +} + +TEST(CompareResultAsOrdering, SanityTest) { + EXPECT_TRUE( + Identity(absl::compare_internal::compare_result_as_ordering(-1) < 0)); + EXPECT_FALSE( + Identity(absl::compare_internal::compare_result_as_ordering(-1) == 0)); + EXPECT_FALSE( + Identity(absl::compare_internal::compare_result_as_ordering(-1) > 0)); + EXPECT_TRUE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::less) < 0)); + EXPECT_FALSE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::less) == 0)); + EXPECT_FALSE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::less) > 0)); + + EXPECT_FALSE( + Identity(absl::compare_internal::compare_result_as_ordering(0) < 0)); + EXPECT_TRUE( + Identity(absl::compare_internal::compare_result_as_ordering(0) == 0)); + EXPECT_FALSE( + Identity(absl::compare_internal::compare_result_as_ordering(0) > 0)); + EXPECT_FALSE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::equivalent) < 0)); + EXPECT_TRUE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::equivalent) == 0)); + EXPECT_FALSE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::equivalent) > 0)); + + EXPECT_FALSE( + Identity(absl::compare_internal::compare_result_as_ordering(1) < 0)); + EXPECT_FALSE( + Identity(absl::compare_internal::compare_result_as_ordering(1) == 0)); + EXPECT_TRUE( + Identity(absl::compare_internal::compare_result_as_ordering(1) > 0)); + EXPECT_FALSE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::greater) < 0)); + EXPECT_FALSE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::greater) == 0)); + EXPECT_TRUE(Identity(absl::compare_internal::compare_result_as_ordering( + weak_ordering::greater) > 0)); +} + +TEST(DoThreeWayComparison, SanityTest) { + std::less<int> less; + WeakOrderingLess weak; + + EXPECT_TRUE(Identity( + absl::compare_internal::do_three_way_comparison(less, -1, 0) < 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(less, -1, 0) == 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(less, -1, 0) > 0)); + EXPECT_TRUE(Identity( + absl::compare_internal::do_three_way_comparison(weak, -1, 0) < 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(weak, -1, 0) == 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(weak, -1, 0) > 0)); + + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(less, 10, 10) < 0)); + EXPECT_TRUE(Identity( + absl::compare_internal::do_three_way_comparison(less, 10, 10) == 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(less, 10, 10) > 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(weak, 10, 10) < 0)); + EXPECT_TRUE(Identity( + absl::compare_internal::do_three_way_comparison(weak, 10, 10) == 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(weak, 10, 10) > 0)); + + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(less, 10, 5) < 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(less, 10, 5) == 0)); + EXPECT_TRUE(Identity( + absl::compare_internal::do_three_way_comparison(less, 10, 5) > 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(weak, 10, 5) < 0)); + EXPECT_FALSE(Identity( + absl::compare_internal::do_three_way_comparison(weak, 10, 5) == 0)); + EXPECT_TRUE(Identity( + absl::compare_internal::do_three_way_comparison(weak, 10, 5) > 0)); +} + +#ifdef __cpp_inline_variables +TEST(Compare, StaticAsserts) { + static_assert(weak_equality::equivalent == 0, ""); + static_assert(weak_equality::nonequivalent != 0, ""); + + static_assert(strong_equality::equal == 0, ""); + static_assert(strong_equality::nonequal != 0, ""); + static_assert(strong_equality::equivalent == 0, ""); + static_assert(strong_equality::nonequivalent != 0, ""); + + static_assert(partial_ordering::less < 0, ""); + static_assert(partial_ordering::equivalent == 0, ""); + static_assert(partial_ordering::greater > 0, ""); + static_assert(partial_ordering::unordered != 0, ""); + + static_assert(weak_ordering::less < 0, ""); + static_assert(weak_ordering::equivalent == 0, ""); + static_assert(weak_ordering::greater > 0, ""); + + static_assert(strong_ordering::less < 0, ""); + static_assert(strong_ordering::equal == 0, ""); + static_assert(strong_ordering::equivalent == 0, ""); + static_assert(strong_ordering::greater > 0, ""); +} +#endif // __cpp_inline_variables + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/types/internal/conformance_aliases.h b/third_party/abseil_cpp/absl/types/internal/conformance_aliases.h new file mode 100644 index 000000000000..0cc6884e309e --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/conformance_aliases.h @@ -0,0 +1,447 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// regularity_aliases.h +// ----------------------------------------------------------------------------- +// +// This file contains type aliases of common ConformanceProfiles and Archetypes +// so that they can be directly used by name without creating them from scratch. + +#ifndef ABSL_TYPES_INTERNAL_CONFORMANCE_ALIASES_H_ +#define ABSL_TYPES_INTERNAL_CONFORMANCE_ALIASES_H_ + +#include "absl/types/internal/conformance_archetype.h" +#include "absl/types/internal/conformance_profile.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { + +// Creates both a Profile and a corresponding Archetype with root name "name". +#define ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS(name, ...) \ + struct name##Profile : __VA_ARGS__ {}; \ + \ + using name##Archetype = ::absl::types_internal::Archetype<name##Profile>; \ + \ + template <class AbslInternalProfileTag> \ + using name##Archetype##_ = ::absl::types_internal::Archetype< \ + ::absl::types_internal::StrongProfileTypedef<name##Profile, \ + AbslInternalProfileTag>> + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasTrivialDefaultConstructor, + ConformanceProfile<default_constructible::trivial>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowDefaultConstructor, + ConformanceProfile<default_constructible::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasDefaultConstructor, ConformanceProfile<default_constructible::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasTrivialMoveConstructor, ConformanceProfile<default_constructible::maybe, + move_constructible::trivial>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowMoveConstructor, ConformanceProfile<default_constructible::maybe, + move_constructible::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasMoveConstructor, + ConformanceProfile<default_constructible::maybe, move_constructible::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasTrivialCopyConstructor, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::trivial>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowCopyConstructor, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasCopyConstructor, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasTrivialMoveAssign, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::trivial>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowMoveAssign, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasMoveAssign, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasTrivialCopyAssign, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::trivial>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowCopyAssign, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasCopyAssign, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasTrivialDestructor, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::trivial>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowDestructor, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasDestructor, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowEquality, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasEquality, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowInequality, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::maybe, + inequality_comparable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasInequality, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::maybe, inequality_comparable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowLessThan, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::maybe, inequality_comparable::maybe, + less_than_comparable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasLessThan, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::maybe, inequality_comparable::maybe, + less_than_comparable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowLessEqual, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::maybe, inequality_comparable::maybe, + less_than_comparable::maybe, + less_equal_comparable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasLessEqual, + ConformanceProfile<default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, + equality_comparable::maybe, inequality_comparable::maybe, + less_than_comparable::maybe, + less_equal_comparable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowGreaterEqual, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasGreaterEqual, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowGreaterThan, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::maybe, + greater_than_comparable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasGreaterThan, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::maybe, + greater_than_comparable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasNothrowSwap, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::maybe, + greater_than_comparable::maybe, swappable::nothrow>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasSwap, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::maybe, + greater_than_comparable::maybe, swappable::yes>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HasStdHashSpecialization, + ConformanceProfile< + default_constructible::maybe, move_constructible::maybe, + copy_constructible::maybe, move_assignable::maybe, + copy_assignable::maybe, destructible::maybe, equality_comparable::maybe, + inequality_comparable::maybe, less_than_comparable::maybe, + less_equal_comparable::maybe, greater_equal_comparable::maybe, + greater_than_comparable::maybe, swappable::maybe, hashable::yes>); + +//////////////////////////////////////////////////////////////////////////////// +//// The remaining aliases are combinations of the previous aliases. //// +//////////////////////////////////////////////////////////////////////////////// + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + Equatable, CombineProfiles<HasEqualityProfile, HasInequalityProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + Comparable, + CombineProfiles<EquatableProfile, HasLessThanProfile, HasLessEqualProfile, + HasGreaterEqualProfile, HasGreaterThanProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + NothrowEquatable, + CombineProfiles<HasNothrowEqualityProfile, HasNothrowInequalityProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + NothrowComparable, + CombineProfiles<NothrowEquatableProfile, HasNothrowLessThanProfile, + HasNothrowLessEqualProfile, HasNothrowGreaterEqualProfile, + HasNothrowGreaterThanProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + Value, + CombineProfiles<HasNothrowMoveConstructorProfile, HasCopyConstructorProfile, + HasNothrowMoveAssignProfile, HasCopyAssignProfile, + HasNothrowDestructorProfile, HasNothrowSwapProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + EquatableValue, CombineProfiles<EquatableProfile, ValueProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + ComparableValue, CombineProfiles<ComparableProfile, ValueProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + DefaultConstructibleValue, + CombineProfiles<HasDefaultConstructorProfile, ValueProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + NothrowMoveConstructible, CombineProfiles<HasNothrowMoveConstructorProfile, + HasNothrowDestructorProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + EquatableNothrowMoveConstructible, + CombineProfiles<EquatableProfile, NothrowMoveConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + ComparableNothrowMoveConstructible, + CombineProfiles<ComparableProfile, NothrowMoveConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + DefaultConstructibleNothrowMoveConstructible, + CombineProfiles<HasDefaultConstructorProfile, + NothrowMoveConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + CopyConstructible, + CombineProfiles<HasNothrowMoveConstructorProfile, HasCopyConstructorProfile, + HasNothrowDestructorProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + EquatableCopyConstructible, + CombineProfiles<EquatableProfile, CopyConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + ComparableCopyConstructible, + CombineProfiles<ComparableProfile, CopyConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + DefaultConstructibleCopyConstructible, + CombineProfiles<HasDefaultConstructorProfile, CopyConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + NothrowMovable, + CombineProfiles<HasNothrowMoveConstructorProfile, + HasNothrowMoveAssignProfile, HasNothrowDestructorProfile, + HasNothrowSwapProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + EquatableNothrowMovable, + CombineProfiles<EquatableProfile, NothrowMovableProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + ComparableNothrowMovable, + CombineProfiles<ComparableProfile, NothrowMovableProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + DefaultConstructibleNothrowMovable, + CombineProfiles<HasDefaultConstructorProfile, NothrowMovableProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + TrivialSpecialMemberFunctions, + CombineProfiles<HasTrivialDefaultConstructorProfile, + HasTrivialMoveConstructorProfile, + HasTrivialCopyConstructorProfile, + HasTrivialMoveAssignProfile, HasTrivialCopyAssignProfile, + HasTrivialDestructorProfile, HasNothrowSwapProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + TriviallyComplete, + CombineProfiles<TrivialSpecialMemberFunctionsProfile, ComparableProfile, + HasStdHashSpecializationProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HashableNothrowMoveConstructible, + CombineProfiles<HasStdHashSpecializationProfile, + NothrowMoveConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HashableCopyConstructible, + CombineProfiles<HasStdHashSpecializationProfile, CopyConstructibleProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HashableNothrowMovable, + CombineProfiles<HasStdHashSpecializationProfile, NothrowMovableProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + HashableValue, + CombineProfiles<HasStdHashSpecializationProfile, ValueProfile>); + +ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS( + ComparableHashableValue, + CombineProfiles<HashableValueProfile, ComparableProfile>); + +// The "preferred" profiles that we support in Abseil. +template <template <class...> class Receiver> +using ExpandBasicProfiles = + Receiver<NothrowMoveConstructibleProfile, CopyConstructibleProfile, + NothrowMovableProfile, ValueProfile>; + +// The basic profiles except that they are also all Equatable. +template <template <class...> class Receiver> +using ExpandBasicEquatableProfiles = + Receiver<EquatableNothrowMoveConstructibleProfile, + EquatableCopyConstructibleProfile, EquatableNothrowMovableProfile, + EquatableValueProfile>; + +// The basic profiles except that they are also all Comparable. +template <template <class...> class Receiver> +using ExpandBasicComparableProfiles = + Receiver<ComparableNothrowMoveConstructibleProfile, + ComparableCopyConstructibleProfile, + ComparableNothrowMovableProfile, ComparableValueProfile>; + +// The basic profiles except that they are also all Hashable. +template <template <class...> class Receiver> +using ExpandBasicHashableProfiles = + Receiver<HashableNothrowMoveConstructibleProfile, + HashableCopyConstructibleProfile, HashableNothrowMovableProfile, + HashableValueProfile>; + +// The basic profiles except that they are also all DefaultConstructible. +template <template <class...> class Receiver> +using ExpandBasicDefaultConstructibleProfiles = + Receiver<DefaultConstructibleNothrowMoveConstructibleProfile, + DefaultConstructibleCopyConstructibleProfile, + DefaultConstructibleNothrowMovableProfile, + DefaultConstructibleValueProfile>; + +// The type profiles that we support in Abseil (all of the previous lists). +template <template <class...> class Receiver> +using ExpandSupportedProfiles = Receiver< + NothrowMoveConstructibleProfile, CopyConstructibleProfile, + NothrowMovableProfile, ValueProfile, + EquatableNothrowMoveConstructibleProfile, EquatableCopyConstructibleProfile, + EquatableNothrowMovableProfile, EquatableValueProfile, + ComparableNothrowMoveConstructibleProfile, + ComparableCopyConstructibleProfile, ComparableNothrowMovableProfile, + ComparableValueProfile, DefaultConstructibleNothrowMoveConstructibleProfile, + DefaultConstructibleCopyConstructibleProfile, + DefaultConstructibleNothrowMovableProfile, DefaultConstructibleValueProfile, + HashableNothrowMoveConstructibleProfile, HashableCopyConstructibleProfile, + HashableNothrowMovableProfile, HashableValueProfile>; + +// TODO(calabrese) Include types that have throwing move constructors, since in +// practice we still need to support them because of standard library types with +// (potentially) non-noexcept moves. + +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_INTERNAL_PROFILE_AND_ARCHETYPE_ALIAS + +#endif // ABSL_TYPES_INTERNAL_CONFORMANCE_ALIASES_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/conformance_archetype.h b/third_party/abseil_cpp/absl/types/internal/conformance_archetype.h new file mode 100644 index 000000000000..2349e0f72635 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/conformance_archetype.h @@ -0,0 +1,978 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// conformance_archetype.h +// ----------------------------------------------------------------------------- +// +// This file contains a facility for generating "archetypes" of out of +// "Conformance Profiles" (see "conformance_profiles.h" for more information +// about Conformance Profiles). An archetype is a type that aims to support the +// bare minimum requirements of a given Conformance Profile. For instance, an +// archetype that corresponds to an ImmutableProfile has exactly a nothrow +// move-constructor, a potentially-throwing copy constructor, a nothrow +// destructor, with all other special-member-functions deleted. These archetypes +// are useful for testing to make sure that templates are able to work with the +// kinds of types that they claim to support (i.e. that they do not accidentally +// under-constrain), +// +// The main type template in this file is the Archetype template, which takes +// a Conformance Profile as a template argument and its instantiations are a +// minimum-conforming model of that profile. + +#ifndef ABSL_TYPES_INTERNAL_CONFORMANCE_ARCHETYPE_H_ +#define ABSL_TYPES_INTERNAL_CONFORMANCE_ARCHETYPE_H_ + +#include <cstddef> +#include <functional> +#include <type_traits> +#include <utility> + +#include "absl/meta/type_traits.h" +#include "absl/types/internal/conformance_profile.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { + +// A minimum-conforming implementation of a type with properties specified in +// `Prof`, where `Prof` is a valid Conformance Profile. +template <class Prof, class /*Enabler*/ = void> +class Archetype; + +// Given an Archetype, obtain the properties of the profile associated with that +// archetype. +template <class Archetype> +struct PropertiesOfArchetype; + +template <class Prof> +struct PropertiesOfArchetype<Archetype<Prof>> { + using type = PropertiesOfT<Prof>; +}; + +template <class Archetype> +using PropertiesOfArchetypeT = typename PropertiesOfArchetype<Archetype>::type; + +// A metafunction to determine if a type is an `Archetype`. +template <class T> +struct IsArchetype : std::false_type {}; + +template <class Prof> +struct IsArchetype<Archetype<Prof>> : std::true_type {}; + +// A constructor tag type used when creating an Archetype with internal state. +struct MakeArchetypeState {}; + +// Data stored within an archetype that is copied/compared/hashed when the +// corresponding operations are used. +using ArchetypeState = std::size_t; + +//////////////////////////////////////////////////////////////////////////////// +// This section of the file defines a chain of base classes for Archetype, // +// where each base defines a specific special member function with the // +// appropriate properties (deleted, noexcept(false), noexcept, or trivial). // +//////////////////////////////////////////////////////////////////////////////// + +// The bottom-most base, which contains the state and the default constructor. +template <default_constructible DefaultConstructibleValue> +struct ArchetypeStateBase { + static_assert(DefaultConstructibleValue == default_constructible::yes || + DefaultConstructibleValue == default_constructible::nothrow, + ""); + + ArchetypeStateBase() noexcept( + DefaultConstructibleValue == + default_constructible:: + nothrow) /*Vacuous archetype_state initialization*/ {} + explicit ArchetypeStateBase(MakeArchetypeState, ArchetypeState state) noexcept + : archetype_state(state) {} + + ArchetypeState archetype_state; +}; + +template <> +struct ArchetypeStateBase<default_constructible::maybe> { + explicit ArchetypeStateBase() = delete; + explicit ArchetypeStateBase(MakeArchetypeState, ArchetypeState state) noexcept + : archetype_state(state) {} + + ArchetypeState archetype_state; +}; + +template <> +struct ArchetypeStateBase<default_constructible::trivial> { + ArchetypeStateBase() = default; + explicit ArchetypeStateBase(MakeArchetypeState, ArchetypeState state) noexcept + : archetype_state(state) {} + + ArchetypeState archetype_state; +}; + +// The move-constructor base +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue> +struct ArchetypeMoveConstructor + : ArchetypeStateBase<DefaultConstructibleValue> { + static_assert(MoveConstructibleValue == move_constructible::yes || + MoveConstructibleValue == move_constructible::nothrow, + ""); + + explicit ArchetypeMoveConstructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeStateBase<DefaultConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeMoveConstructor() = default; + ArchetypeMoveConstructor(ArchetypeMoveConstructor&& other) noexcept( + MoveConstructibleValue == move_constructible::nothrow) + : ArchetypeStateBase<DefaultConstructibleValue>(MakeArchetypeState(), + other.archetype_state) {} + ArchetypeMoveConstructor(const ArchetypeMoveConstructor&) = default; + ArchetypeMoveConstructor& operator=(ArchetypeMoveConstructor&&) = default; + ArchetypeMoveConstructor& operator=(const ArchetypeMoveConstructor&) = + default; +}; + +template <default_constructible DefaultConstructibleValue> +struct ArchetypeMoveConstructor<DefaultConstructibleValue, + move_constructible::trivial> + : ArchetypeStateBase<DefaultConstructibleValue> { + explicit ArchetypeMoveConstructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeStateBase<DefaultConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeMoveConstructor() = default; +}; + +// The copy-constructor base +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue> +struct ArchetypeCopyConstructor + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue> { + static_assert(CopyConstructibleValue == copy_constructible::yes || + CopyConstructibleValue == copy_constructible::nothrow, + ""); + explicit ArchetypeCopyConstructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeCopyConstructor() = default; + ArchetypeCopyConstructor(ArchetypeCopyConstructor&&) = default; + ArchetypeCopyConstructor(const ArchetypeCopyConstructor& other) noexcept( + CopyConstructibleValue == copy_constructible::nothrow) + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue>( + MakeArchetypeState(), other.archetype_state) {} + ArchetypeCopyConstructor& operator=(ArchetypeCopyConstructor&&) = default; + ArchetypeCopyConstructor& operator=(const ArchetypeCopyConstructor&) = + default; +}; + +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue> +struct ArchetypeCopyConstructor<DefaultConstructibleValue, + MoveConstructibleValue, + copy_constructible::maybe> + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue> { + explicit ArchetypeCopyConstructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeCopyConstructor() = default; + ArchetypeCopyConstructor(ArchetypeCopyConstructor&&) = default; + ArchetypeCopyConstructor(const ArchetypeCopyConstructor&) = delete; + ArchetypeCopyConstructor& operator=(ArchetypeCopyConstructor&&) = default; + ArchetypeCopyConstructor& operator=(const ArchetypeCopyConstructor&) = + default; +}; + +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue> +struct ArchetypeCopyConstructor<DefaultConstructibleValue, + MoveConstructibleValue, + copy_constructible::trivial> + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue> { + explicit ArchetypeCopyConstructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeMoveConstructor<DefaultConstructibleValue, + MoveConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeCopyConstructor() = default; +}; + +// The move-assign base +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue> +struct ArchetypeMoveAssign + : ArchetypeCopyConstructor<DefaultConstructibleValue, + MoveConstructibleValue, CopyConstructibleValue> { + static_assert(MoveAssignableValue == move_assignable::yes || + MoveAssignableValue == move_assignable::nothrow, + ""); + explicit ArchetypeMoveAssign(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeCopyConstructor<DefaultConstructibleValue, + MoveConstructibleValue, + CopyConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeMoveAssign() = default; + ArchetypeMoveAssign(ArchetypeMoveAssign&&) = default; + ArchetypeMoveAssign(const ArchetypeMoveAssign&) = default; + ArchetypeMoveAssign& operator=(ArchetypeMoveAssign&& other) noexcept( + MoveAssignableValue == move_assignable::nothrow) { + this->archetype_state = other.archetype_state; + return *this; + } + + ArchetypeMoveAssign& operator=(const ArchetypeMoveAssign&) = default; +}; + +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue> +struct ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, move_assignable::trivial> + : ArchetypeCopyConstructor<DefaultConstructibleValue, + MoveConstructibleValue, CopyConstructibleValue> { + explicit ArchetypeMoveAssign(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeCopyConstructor<DefaultConstructibleValue, + MoveConstructibleValue, + CopyConstructibleValue>(MakeArchetypeState(), + state) {} + + ArchetypeMoveAssign() = default; +}; + +// The copy-assign base +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue, + copy_assignable CopyAssignableValue> +struct ArchetypeCopyAssign + : ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue> { + static_assert(CopyAssignableValue == copy_assignable::yes || + CopyAssignableValue == copy_assignable::nothrow, + ""); + explicit ArchetypeCopyAssign(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue>( + MakeArchetypeState(), state) {} + + ArchetypeCopyAssign() = default; + ArchetypeCopyAssign(ArchetypeCopyAssign&&) = default; + ArchetypeCopyAssign(const ArchetypeCopyAssign&) = default; + ArchetypeCopyAssign& operator=(ArchetypeCopyAssign&&) = default; + + ArchetypeCopyAssign& operator=(const ArchetypeCopyAssign& other) noexcept( + CopyAssignableValue == copy_assignable::nothrow) { + this->archetype_state = other.archetype_state; + return *this; + } +}; + +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue> +struct ArchetypeCopyAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + copy_assignable::maybe> + : ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue> { + explicit ArchetypeCopyAssign(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue>( + MakeArchetypeState(), state) {} + + ArchetypeCopyAssign() = default; + ArchetypeCopyAssign(ArchetypeCopyAssign&&) = default; + ArchetypeCopyAssign(const ArchetypeCopyAssign&) = default; + ArchetypeCopyAssign& operator=(ArchetypeCopyAssign&&) = default; + ArchetypeCopyAssign& operator=(const ArchetypeCopyAssign&) = delete; +}; + +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue> +struct ArchetypeCopyAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + copy_assignable::trivial> + : ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue> { + explicit ArchetypeCopyAssign(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeMoveAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue>( + MakeArchetypeState(), state) {} + + ArchetypeCopyAssign() = default; +}; + +// The destructor base +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue, + copy_assignable CopyAssignableValue, destructible DestructibleValue> +struct ArchetypeDestructor + : ArchetypeCopyAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + CopyAssignableValue> { + static_assert(DestructibleValue == destructible::yes || + DestructibleValue == destructible::nothrow, + ""); + + explicit ArchetypeDestructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeCopyAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + CopyAssignableValue>(MakeArchetypeState(), state) {} + + ArchetypeDestructor() = default; + ArchetypeDestructor(ArchetypeDestructor&&) = default; + ArchetypeDestructor(const ArchetypeDestructor&) = default; + ArchetypeDestructor& operator=(ArchetypeDestructor&&) = default; + ArchetypeDestructor& operator=(const ArchetypeDestructor&) = default; + ~ArchetypeDestructor() noexcept(DestructibleValue == destructible::nothrow) {} +}; + +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue, + copy_assignable CopyAssignableValue> +struct ArchetypeDestructor<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + CopyAssignableValue, destructible::trivial> + : ArchetypeCopyAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + CopyAssignableValue> { + explicit ArchetypeDestructor(MakeArchetypeState, + ArchetypeState state) noexcept + : ArchetypeCopyAssign<DefaultConstructibleValue, MoveConstructibleValue, + CopyConstructibleValue, MoveAssignableValue, + CopyAssignableValue>(MakeArchetypeState(), state) {} + + ArchetypeDestructor() = default; +}; + +// An alias to the top of the chain of bases for special-member functions. +// NOTE: move_constructible::maybe, move_assignable::maybe, and +// destructible::maybe are handled in the top-level type by way of SFINAE. +// Because of this, we never instantiate the base classes with +// move_constructible::maybe, move_assignable::maybe, or destructible::maybe so +// that we minimize the number of different possible type-template +// instantiations. +template <default_constructible DefaultConstructibleValue, + move_constructible MoveConstructibleValue, + copy_constructible CopyConstructibleValue, + move_assignable MoveAssignableValue, + copy_assignable CopyAssignableValue, destructible DestructibleValue> +using ArchetypeSpecialMembersBase = ArchetypeDestructor< + DefaultConstructibleValue, + MoveConstructibleValue != move_constructible::maybe + ? MoveConstructibleValue + : move_constructible::nothrow, + CopyConstructibleValue, + MoveAssignableValue != move_assignable::maybe ? MoveAssignableValue + : move_assignable::nothrow, + CopyAssignableValue, + DestructibleValue != destructible::maybe ? DestructibleValue + : destructible::nothrow>; + +// A function that is used to create an archetype with some associated state. +template <class Arch> +Arch MakeArchetype(ArchetypeState state) noexcept { + static_assert(IsArchetype<Arch>::value, + "The explicit template argument to MakeArchetype is required " + "to be an Archetype."); + return Arch(MakeArchetypeState(), state); +} + +// This is used to conditionally delete "copy" and "move" constructors in a way +// that is consistent with what the ConformanceProfile requires and that also +// strictly enforces the arguments to the copy/move to not come from implicit +// conversions when dealing with the Archetype. +template <class Prof, class T> +constexpr bool ShouldDeleteConstructor() { + return !((PropertiesOfT<Prof>::move_constructible_support != + move_constructible::maybe && + std::is_same<T, Archetype<Prof>>::value) || + (PropertiesOfT<Prof>::copy_constructible_support != + copy_constructible::maybe && + (std::is_same<T, const Archetype<Prof>&>::value || + std::is_same<T, Archetype<Prof>&>::value || + std::is_same<T, const Archetype<Prof>>::value))); +} + +// This is used to conditionally delete "copy" and "move" assigns in a way +// that is consistent with what the ConformanceProfile requires and that also +// strictly enforces the arguments to the copy/move to not come from implicit +// conversions when dealing with the Archetype. +template <class Prof, class T> +constexpr bool ShouldDeleteAssign() { + return !( + (PropertiesOfT<Prof>::move_assignable_support != move_assignable::maybe && + std::is_same<T, Archetype<Prof>>::value) || + (PropertiesOfT<Prof>::copy_assignable_support != copy_assignable::maybe && + (std::is_same<T, const Archetype<Prof>&>::value || + std::is_same<T, Archetype<Prof>&>::value || + std::is_same<T, const Archetype<Prof>>::value))); +} + +// TODO(calabrese) Inherit from a chain of secondary bases to pull in the +// associated functions of other concepts. +template <class Prof, class Enabler> +class Archetype : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + static_assert(std::is_same<Enabler, void>::value, + "An explicit type must not be passed as the second template " + "argument to 'Archetype`."); + + // The cases mentioned in these static_asserts are expected to be handled in + // the partial template specializations of Archetype that follow this + // definition. + static_assert(PropertiesOfT<Prof>::destructible_support != + destructible::maybe, + ""); + static_assert(PropertiesOfT<Prof>::move_constructible_support != + move_constructible::maybe || + PropertiesOfT<Prof>::copy_constructible_support == + copy_constructible::maybe, + ""); + static_assert(PropertiesOfT<Prof>::move_assignable_support != + move_assignable::maybe || + PropertiesOfT<Prof>::copy_assignable_support == + copy_assignable::maybe, + ""); + + public: + Archetype() = default; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +template <class Prof> +class Archetype<Prof, typename std::enable_if< + PropertiesOfT<Prof>::move_constructible_support != + move_constructible::maybe && + PropertiesOfT<Prof>::move_assignable_support == + move_assignable::maybe && + PropertiesOfT<Prof>::destructible_support != + destructible::maybe>::type> + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + public: + Archetype() = default; + Archetype(Archetype&&) = default; + Archetype(const Archetype&) = default; + Archetype& operator=(Archetype&&) = delete; + Archetype& operator=(const Archetype&) = default; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +template <class Prof> +class Archetype<Prof, typename std::enable_if< + PropertiesOfT<Prof>::move_constructible_support == + move_constructible::maybe && + PropertiesOfT<Prof>::move_assignable_support == + move_assignable::maybe && + PropertiesOfT<Prof>::destructible_support != + destructible::maybe>::type> + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + public: + Archetype() = default; + Archetype(Archetype&&) = delete; + Archetype(const Archetype&) = default; + Archetype& operator=(Archetype&&) = delete; + Archetype& operator=(const Archetype&) = default; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +template <class Prof> +class Archetype<Prof, typename std::enable_if< + PropertiesOfT<Prof>::move_constructible_support == + move_constructible::maybe && + PropertiesOfT<Prof>::move_assignable_support != + move_assignable::maybe && + PropertiesOfT<Prof>::destructible_support != + destructible::maybe>::type> + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + public: + Archetype() = default; + Archetype(Archetype&&) = delete; + Archetype(const Archetype&) = default; + Archetype& operator=(Archetype&&) = default; + Archetype& operator=(const Archetype&) = default; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +template <class Prof> +class Archetype<Prof, typename std::enable_if< + PropertiesOfT<Prof>::move_constructible_support != + move_constructible::maybe && + PropertiesOfT<Prof>::move_assignable_support == + move_assignable::maybe && + PropertiesOfT<Prof>::destructible_support == + destructible::maybe>::type> + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + public: + Archetype() = default; + Archetype(Archetype&&) = default; + Archetype(const Archetype&) = default; + Archetype& operator=(Archetype&&) = delete; + Archetype& operator=(const Archetype&) = default; + ~Archetype() = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +template <class Prof> +class Archetype<Prof, typename std::enable_if< + PropertiesOfT<Prof>::move_constructible_support == + move_constructible::maybe && + PropertiesOfT<Prof>::move_assignable_support == + move_assignable::maybe && + PropertiesOfT<Prof>::destructible_support == + destructible::maybe>::type> + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + public: + Archetype() = default; + Archetype(Archetype&&) = delete; + Archetype(const Archetype&) = default; + Archetype& operator=(Archetype&&) = delete; + Archetype& operator=(const Archetype&) = default; + ~Archetype() = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +template <class Prof> +class Archetype<Prof, typename std::enable_if< + PropertiesOfT<Prof>::move_constructible_support == + move_constructible::maybe && + PropertiesOfT<Prof>::move_assignable_support != + move_assignable::maybe && + PropertiesOfT<Prof>::destructible_support == + destructible::maybe>::type> + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support> { + public: + Archetype() = default; + Archetype(Archetype&&) = delete; + Archetype(const Archetype&) = default; + Archetype& operator=(Archetype&&) = default; + Archetype& operator=(const Archetype&) = default; + ~Archetype() = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteConstructor<Prof, T>()>::type* = nullptr> + Archetype(T&&) = delete; + + // Disallow moves when requested, and disallow implicit conversions. + template <class T, typename std::enable_if< + ShouldDeleteAssign<Prof, T>()>::type* = nullptr> + Archetype& operator=(T&&) = delete; + + using ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>::archetype_state; + + private: + explicit Archetype(MakeArchetypeState, ArchetypeState state) noexcept + : ArchetypeSpecialMembersBase< + PropertiesOfT<Prof>::default_constructible_support, + PropertiesOfT<Prof>::move_constructible_support, + PropertiesOfT<Prof>::copy_constructible_support, + PropertiesOfT<Prof>::move_assignable_support, + PropertiesOfT<Prof>::copy_assignable_support, + PropertiesOfT<Prof>::destructible_support>(MakeArchetypeState(), + state) {} + + friend Archetype MakeArchetype<Archetype>(ArchetypeState) noexcept; +}; + +// Explicitly deleted swap for Archetype if the profile does not require swap. +// It is important to delete it rather than simply leave it out so that the +// "using std::swap;" idiom will result in this deleted overload being picked. +template <class Prof, + absl::enable_if_t<!PropertiesOfT<Prof>::is_swappable, int> = 0> +void swap(Archetype<Prof>&, Archetype<Prof>&) = delete; // NOLINT + +// A conditionally-noexcept swap implementation for Archetype when the profile +// supports swap. +template <class Prof, + absl::enable_if_t<PropertiesOfT<Prof>::is_swappable, int> = 0> +void swap(Archetype<Prof>& lhs, Archetype<Prof>& rhs) // NOLINT + noexcept(PropertiesOfT<Prof>::swappable_support != swappable::yes) { + std::swap(lhs.archetype_state, rhs.archetype_state); +} + +// A convertible-to-bool type that is used as the return type of comparison +// operators since the standard doesn't always require exactly bool. +struct NothrowBool { + explicit NothrowBool() = delete; + ~NothrowBool() = default; + + // TODO(calabrese) Delete the copy constructor in C++17 mode since guaranteed + // elision makes it not required when returning from a function. + // NothrowBool(NothrowBool const&) = delete; + + NothrowBool& operator=(NothrowBool const&) = delete; + + explicit operator bool() const noexcept { return value; } + + static NothrowBool make(bool const value) noexcept { + return NothrowBool(value); + } + + private: + explicit NothrowBool(bool const value) noexcept : value(value) {} + + bool value; +}; + +// A convertible-to-bool type that is used as the return type of comparison +// operators since the standard doesn't always require exactly bool. +// Note: ExceptionalBool has a conversion operator that is not noexcept, so +// that even when a comparison operator is noexcept, that operation may still +// potentially throw when converted to bool. +struct ExceptionalBool { + explicit ExceptionalBool() = delete; + ~ExceptionalBool() = default; + + // TODO(calabrese) Delete the copy constructor in C++17 mode since guaranteed + // elision makes it not required when returning from a function. + // ExceptionalBool(ExceptionalBool const&) = delete; + + ExceptionalBool& operator=(ExceptionalBool const&) = delete; + + explicit operator bool() const { return value; } // NOLINT + + static ExceptionalBool make(bool const value) noexcept { + return ExceptionalBool(value); + } + + private: + explicit ExceptionalBool(bool const value) noexcept : value(value) {} + + bool value; +}; + +// The following macro is only used as a helper in this file to stamp out +// comparison operator definitions. It is undefined after usage. +// +// NOTE: Non-nothrow operators throw via their result's conversion to bool even +// though the operation itself is noexcept. +#define ABSL_TYPES_INTERNAL_OP(enum_name, op) \ + template <class Prof> \ + absl::enable_if_t<!PropertiesOfT<Prof>::is_##enum_name, bool> operator op( \ + const Archetype<Prof>&, const Archetype<Prof>&) = delete; \ + \ + template <class Prof> \ + typename absl::enable_if_t< \ + PropertiesOfT<Prof>::is_##enum_name, \ + std::conditional<PropertiesOfT<Prof>::enum_name##_support == \ + enum_name::nothrow, \ + NothrowBool, ExceptionalBool>>::type \ + operator op(const Archetype<Prof>& lhs, \ + const Archetype<Prof>& rhs) noexcept { \ + return absl::conditional_t< \ + PropertiesOfT<Prof>::enum_name##_support == enum_name::nothrow, \ + NothrowBool, ExceptionalBool>::make(lhs.archetype_state op \ + rhs.archetype_state); \ + } + +ABSL_TYPES_INTERNAL_OP(equality_comparable, ==); +ABSL_TYPES_INTERNAL_OP(inequality_comparable, !=); +ABSL_TYPES_INTERNAL_OP(less_than_comparable, <); +ABSL_TYPES_INTERNAL_OP(less_equal_comparable, <=); +ABSL_TYPES_INTERNAL_OP(greater_equal_comparable, >=); +ABSL_TYPES_INTERNAL_OP(greater_than_comparable, >); + +#undef ABSL_TYPES_INTERNAL_OP + +// Base class for std::hash specializations when an Archetype doesn't support +// hashing. +struct PoisonedHash { + PoisonedHash() = delete; + PoisonedHash(const PoisonedHash&) = delete; + PoisonedHash& operator=(const PoisonedHash&) = delete; +}; + +// Base class for std::hash specializations when an Archetype supports hashing. +template <class Prof> +struct EnabledHash { + using argument_type = Archetype<Prof>; + using result_type = std::size_t; + result_type operator()(const argument_type& arg) const { + return std::hash<ArchetypeState>()(arg.archetype_state); + } +}; + +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl + +namespace std { + +template <class Prof> // NOLINT +struct hash<::absl::types_internal::Archetype<Prof>> + : conditional<::absl::types_internal::PropertiesOfT<Prof>::is_hashable, + ::absl::types_internal::EnabledHash<Prof>, + ::absl::types_internal::PoisonedHash>::type {}; + +} // namespace std + +#endif // ABSL_TYPES_INTERNAL_CONFORMANCE_ARCHETYPE_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/conformance_profile.h b/third_party/abseil_cpp/absl/types/internal/conformance_profile.h new file mode 100644 index 000000000000..cf64ff4fcd1e --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/conformance_profile.h @@ -0,0 +1,931 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// conformance_profiles.h +// ----------------------------------------------------------------------------- +// +// This file contains templates for representing "Regularity Profiles" and +// concisely-named versions of commonly used Regularity Profiles. +// +// A Regularity Profile is a compile-time description of the types of operations +// that a given type supports, along with properties of those operations when +// they do exist. For instance, a Regularity Profile may describe a type that +// has a move-constructor that is noexcept and a copy constructor that is not +// noexcept. This description can then be examined and passed around to other +// templates for the purposes of asserting expectations on user-defined types +// via a series trait checks, or for determining what kinds of run-time tests +// are able to be performed. +// +// Regularity Profiles are also used when creating "archetypes," which are +// minimum-conforming types that meet all of the requirements of a given +// Regularity Profile. For more information regarding archetypes, see +// "conformance_archetypes.h". + +#ifndef ABSL_TYPES_INTERNAL_CONFORMANCE_PROFILE_H_ +#define ABSL_TYPES_INTERNAL_CONFORMANCE_PROFILE_H_ + +#include <set> +#include <type_traits> +#include <utility> +#include <vector> + +#include "gtest/gtest.h" +#include "absl/algorithm/container.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/internal/conformance_testing_helpers.h" +#include "absl/utility/utility.h" + +// TODO(calabrese) Add support for extending profiles. + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { + +// Converts an enum to its underlying integral value. +template <typename Enum> +constexpr absl::underlying_type_t<Enum> UnderlyingValue(Enum value) { + return static_cast<absl::underlying_type_t<Enum>>(value); +} + +// A tag type used in place of a matcher when checking that an assertion result +// does not actually contain any errors. +struct NoError {}; + +// ----------------------------------------------------------------------------- +// ConformanceErrors +// ----------------------------------------------------------------------------- +class ConformanceErrors { + public: + // Setup the error reporting mechanism by seeding it with the name of the type + // that is being tested. + explicit ConformanceErrors(std::string type_name) + : assertion_result_(false), type_name_(std::move(type_name)) { + assertion_result_ << "\n\n" + "Assuming the following type alias:\n" + "\n" + " using _T = " + << type_name_ << ";\n\n"; + outputDivider(); + } + + // Adds the test name to the list of successfully run tests iff it was not + // previously reported as failing. This behavior is useful for tests that + // have multiple parts, where failures and successes are reported individually + // with the same test name. + void addTestSuccess(absl::string_view test_name) { + auto normalized_test_name = absl::AsciiStrToLower(test_name); + + // If the test is already reported as failing, do not add it to the list of + // successes. + if (test_failures_.find(normalized_test_name) == test_failures_.end()) { + test_successes_.insert(std::move(normalized_test_name)); + } + } + + // Streams a single error description into the internal buffer (a visual + // divider is automatically inserted after the error so that multiple errors + // are visibly distinct). + // + // This function increases the error count by 1. + // + // TODO(calabrese) Determine desired behavior when if this function throws. + template <class... P> + void addTestFailure(absl::string_view test_name, const P&... args) { + // Output a message related to the test failure. + assertion_result_ << "\n\n" + "Failed test: " + << test_name << "\n\n"; + addTestFailureImpl(args...); + assertion_result_ << "\n\n"; + outputDivider(); + + auto normalized_test_name = absl::AsciiStrToLower(test_name); + + // If previous parts of this test succeeded, remove it from that set. + test_successes_.erase(normalized_test_name); + + // Add the test name to the list of failed tests. + test_failures_.insert(std::move(normalized_test_name)); + + has_error_ = true; + } + + // Convert this object into a testing::AssertionResult instance such that it + // can be used with gtest. + ::testing::AssertionResult assertionResult() const { + return has_error_ ? assertion_result_ : ::testing::AssertionSuccess(); + } + + // Convert this object into a testing::AssertionResult instance such that it + // can be used with gtest. This overload expects errors, using the specified + // matcher. + ::testing::AssertionResult expectFailedTests( + const std::set<std::string>& test_names) const { + // Since we are expecting nonconformance, output an error message when the + // type actually conformed to the specified profile. + if (!has_error_) { + return ::testing::AssertionFailure() + << "Unexpected conformance of type:\n" + " " + << type_name_ << "\n\n"; + } + + // Get a list of all expected failures that did not actually fail + // (or that were not run). + std::vector<std::string> nonfailing_tests; + absl::c_set_difference(test_names, test_failures_, + std::back_inserter(nonfailing_tests)); + + // Get a list of all "expected failures" that were never actually run. + std::vector<std::string> unrun_tests; + absl::c_set_difference(nonfailing_tests, test_successes_, + std::back_inserter(unrun_tests)); + + // Report when the user specified tests that were not run. + if (!unrun_tests.empty()) { + const bool tests_were_run = + !(test_failures_.empty() && test_successes_.empty()); + + // Prepare an assertion result used in the case that tests pass that were + // expected to fail. + ::testing::AssertionResult result = ::testing::AssertionFailure(); + result << "When testing type:\n " << type_name_ + << "\n\nThe following tests were expected to fail but were not " + "run"; + + if (tests_were_run) result << " (was the test name spelled correctly?)"; + + result << ":\n\n"; + + // List all of the tests that unexpectedly passed. + for (const auto& test_name : unrun_tests) { + result << " " << test_name << "\n"; + } + + if (!tests_were_run) result << "\nNo tests were run."; + + if (!test_failures_.empty()) { + // List test failures + result << "\nThe tests that were run and failed are:\n\n"; + for (const auto& test_name : test_failures_) { + result << " " << test_name << "\n"; + } + } + + if (!test_successes_.empty()) { + // List test successes + result << "\nThe tests that were run and succeeded are:\n\n"; + for (const auto& test_name : test_successes_) { + result << " " << test_name << "\n"; + } + } + + return result; + } + + // If some tests passed when they were expected to fail, alert the caller. + if (nonfailing_tests.empty()) return ::testing::AssertionSuccess(); + + // Prepare an assertion result used in the case that tests pass that were + // expected to fail. + ::testing::AssertionResult unexpected_successes = + ::testing::AssertionFailure(); + unexpected_successes << "When testing type:\n " << type_name_ + << "\n\nThe following tests passed when they were " + "expected to fail:\n\n"; + + // List all of the tests that unexpectedly passed. + for (const auto& test_name : nonfailing_tests) { + unexpected_successes << " " << test_name << "\n"; + } + + return unexpected_successes; + } + + private: + void outputDivider() { + assertion_result_ << "========================================"; + } + + void addTestFailureImpl() {} + + template <class H, class... T> + void addTestFailureImpl(const H& head, const T&... tail) { + assertion_result_ << head; + addTestFailureImpl(tail...); + } + + ::testing::AssertionResult assertion_result_; + std::set<std::string> test_failures_; + std::set<std::string> test_successes_; + std::string type_name_; + bool has_error_ = false; +}; + +template <class T, class /*Enabler*/ = void> +struct PropertiesOfImpl {}; + +template <class T> +struct PropertiesOfImpl<T, absl::void_t<typename T::properties>> { + using type = typename T::properties; +}; + +template <class T> +struct PropertiesOfImpl<T, absl::void_t<typename T::profile_alias_of>> { + using type = typename PropertiesOfImpl<typename T::profile_alias_of>::type; +}; + +template <class T> +struct PropertiesOf : PropertiesOfImpl<T> {}; + +template <class T> +using PropertiesOfT = typename PropertiesOf<T>::type; + +// NOTE: These enums use this naming convention to be consistent with the +// standard trait names, which is useful since it allows us to match up each +// enum name with a corresponding trait name in macro definitions. + +// An enum that describes the various expectations on an operations existence. +enum class function_support { maybe, yes, nothrow, trivial }; + +constexpr const char* PessimisticPropertyDescription(function_support v) { + return v == function_support::maybe + ? "no" + : v == function_support::yes + ? "yes, potentially throwing" + : v == function_support::nothrow ? "yes, nothrow" + : "yes, trivial"; +} + +// Return a string that describes the kind of property support that was +// expected. +inline std::string ExpectedFunctionKindList(function_support min, + function_support max) { + if (min == max) { + std::string result = + absl::StrCat("Expected:\n ", + PessimisticPropertyDescription( + static_cast<function_support>(UnderlyingValue(min))), + "\n"); + return result; + } + + std::string result = "Expected one of:\n"; + for (auto curr_support = UnderlyingValue(min); + curr_support <= UnderlyingValue(max); ++curr_support) { + absl::StrAppend(&result, " ", + PessimisticPropertyDescription( + static_cast<function_support>(curr_support)), + "\n"); + } + + return result; +} + +template <class Enum> +void ExpectModelOfImpl(ConformanceErrors* errors, Enum min_support, + Enum max_support, Enum kind) { + const auto kind_value = UnderlyingValue(kind); + const auto min_support_value = UnderlyingValue(min_support); + const auto max_support_value = UnderlyingValue(max_support); + + if (!(kind_value >= min_support_value && kind_value <= max_support_value)) { + errors->addTestFailure( + PropertyName(kind), "**Failed property expectation**\n\n", + ExpectedFunctionKindList( + static_cast<function_support>(min_support_value), + static_cast<function_support>(max_support_value)), + '\n', "Actual:\n ", + PessimisticPropertyDescription( + static_cast<function_support>(kind_value))); + } else { + errors->addTestSuccess(PropertyName(kind)); + } +} + +#define ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM(description, name) \ + enum class name { maybe, yes, nothrow, trivial }; \ + \ + constexpr const char* PropertyName(name v) { return description; } \ + static_assert(true, "") // Force a semicolon when using this macro. + +ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM("support for default construction", + default_constructible); +ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM("support for move construction", + move_constructible); +ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM("support for copy construction", + copy_constructible); +ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM("support for move assignment", + move_assignable); +ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM("support for copy assignment", + copy_assignable); +ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM("support for destruction", + destructible); + +#undef ABSL_INTERNAL_SPECIAL_MEMBER_FUNCTION_ENUM + +#define ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM(description, name) \ + enum class name { maybe, yes, nothrow }; \ + \ + constexpr const char* PropertyName(name v) { return description; } \ + static_assert(true, "") // Force a semicolon when using this macro. + +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for ==", equality_comparable); +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for !=", inequality_comparable); +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for <", less_than_comparable); +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for <=", less_equal_comparable); +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for >=", + greater_equal_comparable); +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for >", greater_than_comparable); + +ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM("support for swap", swappable); + +#undef ABSL_INTERNAL_INTRINSIC_FUNCTION_ENUM + +enum class hashable { maybe, yes }; + +constexpr const char* PropertyName(hashable v) { + return "support for std::hash"; +} + +template <class T> +using AlwaysFalse = std::false_type; + +#define ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(name, property) \ + template <class T> \ + constexpr property property##_support_of() { \ + return std::is_##property<T>::value \ + ? std::is_nothrow_##property<T>::value \ + ? absl::is_trivially_##property<T>::value \ + ? property::trivial \ + : property::nothrow \ + : property::yes \ + : property::maybe; \ + } \ + \ + template <class T, class MinProf, class MaxProf> \ + void ExpectModelOf##name(ConformanceErrors* errors) { \ + (ExpectModelOfImpl)(errors, PropertiesOfT<MinProf>::property##_support, \ + PropertiesOfT<MaxProf>::property##_support, \ + property##_support_of<T>()); \ + } + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(DefaultConstructible, + default_constructible); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(MoveConstructible, + move_constructible); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(CopyConstructible, + copy_constructible); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(MoveAssignable, + move_assignable); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(CopyAssignable, + copy_assignable); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER(Destructible, destructible); + +#undef ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_SPECIAL_MEMBER + +void BoolFunction(bool) noexcept; + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction for checking if an operation exists through SFINAE. +// +// `T` is the type to test and Op is an alias containing the expression to test. +template <class T, template <class...> class Op, class = void> +struct IsOpableImpl : std::false_type {}; + +template <class T, template <class...> class Op> +struct IsOpableImpl<T, Op, absl::void_t<Op<T>>> : std::true_type {}; + +template <template <class...> class Op> +struct IsOpable { + template <class T> + using apply = typename IsOpableImpl<T, Op>::type; +}; +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction for checking if an operation exists and is also noexcept +// through SFINAE and the noexcept operator. +/// +// `T` is the type to test and Op is an alias containing the expression to test. +template <class T, template <class...> class Op, class = void> +struct IsNothrowOpableImpl : std::false_type {}; + +template <class T, template <class...> class Op> +struct IsNothrowOpableImpl<T, Op, absl::enable_if_t<Op<T>::value>> + : std::true_type {}; + +template <template <class...> class Op> +struct IsNothrowOpable { + template <class T> + using apply = typename IsNothrowOpableImpl<T, Op>::type; +}; +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// A macro that produces the necessary function for reporting what kind of +// support a specific comparison operation has and a function for reporting an +// error if a given type's support for that operation does not meet the expected +// requirements. +#define ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(name, property, op) \ + template <class T, \ + class Result = std::integral_constant< \ + bool, noexcept((BoolFunction)(std::declval<const T&>() op \ + std::declval<const T&>()))>> \ + using name = Result; \ + \ + template <class T> \ + constexpr property property##_support_of() { \ + return IsOpable<name>::apply<T>::value \ + ? IsNothrowOpable<name>::apply<T>::value ? property::nothrow \ + : property::yes \ + : property::maybe; \ + } \ + \ + template <class T, class MinProf, class MaxProf> \ + void ExpectModelOf##name(ConformanceErrors* errors) { \ + (ExpectModelOfImpl)(errors, PropertiesOfT<MinProf>::property##_support, \ + PropertiesOfT<MaxProf>::property##_support, \ + property##_support_of<T>()); \ + } +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// Generate the necessary support-checking and error reporting functions for +// each of the comparison operators. +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(EqualityComparable, + equality_comparable, ==); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(InequalityComparable, + inequality_comparable, !=); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(LessThanComparable, + less_than_comparable, <); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(LessEqualComparable, + less_equal_comparable, <=); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(GreaterEqualComparable, + greater_equal_comparable, >=); + +ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON(GreaterThanComparable, + greater_than_comparable, >); + +#undef ABSL_INTERNAL_PESSIMISTIC_MODEL_OF_COMPARISON +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// The necessary support-checking and error-reporting functions for swap. +template <class T> +constexpr swappable swappable_support_of() { + return type_traits_internal::IsSwappable<T>::value + ? type_traits_internal::IsNothrowSwappable<T>::value + ? swappable::nothrow + : swappable::yes + : swappable::maybe; +} + +template <class T, class MinProf, class MaxProf> +void ExpectModelOfSwappable(ConformanceErrors* errors) { + (ExpectModelOfImpl)(errors, PropertiesOfT<MinProf>::swappable_support, + PropertiesOfT<MaxProf>::swappable_support, + swappable_support_of<T>()); +} +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// The necessary support-checking and error-reporting functions for std::hash. +template <class T> +constexpr hashable hashable_support_of() { + return type_traits_internal::IsHashable<T>::value ? hashable::yes + : hashable::maybe; +} + +template <class T, class MinProf, class MaxProf> +void ExpectModelOfHashable(ConformanceErrors* errors) { + (ExpectModelOfImpl)(errors, PropertiesOfT<MinProf>::hashable_support, + PropertiesOfT<MaxProf>::hashable_support, + hashable_support_of<T>()); +} +// +//////////////////////////////////////////////////////////////////////////////// + +template < + default_constructible DefaultConstructibleValue = + default_constructible::maybe, + move_constructible MoveConstructibleValue = move_constructible::maybe, + copy_constructible CopyConstructibleValue = copy_constructible::maybe, + move_assignable MoveAssignableValue = move_assignable::maybe, + copy_assignable CopyAssignableValue = copy_assignable::maybe, + destructible DestructibleValue = destructible::maybe, + equality_comparable EqualityComparableValue = equality_comparable::maybe, + inequality_comparable InequalityComparableValue = + inequality_comparable::maybe, + less_than_comparable LessThanComparableValue = less_than_comparable::maybe, + less_equal_comparable LessEqualComparableValue = + less_equal_comparable::maybe, + greater_equal_comparable GreaterEqualComparableValue = + greater_equal_comparable::maybe, + greater_than_comparable GreaterThanComparableValue = + greater_than_comparable::maybe, + swappable SwappableValue = swappable::maybe, + hashable HashableValue = hashable::maybe> +struct ConformanceProfile { + using properties = ConformanceProfile; + + static constexpr default_constructible + default_constructible_support = // NOLINT + DefaultConstructibleValue; + + static constexpr move_constructible move_constructible_support = // NOLINT + MoveConstructibleValue; + + static constexpr copy_constructible copy_constructible_support = // NOLINT + CopyConstructibleValue; + + static constexpr move_assignable move_assignable_support = // NOLINT + MoveAssignableValue; + + static constexpr copy_assignable copy_assignable_support = // NOLINT + CopyAssignableValue; + + static constexpr destructible destructible_support = // NOLINT + DestructibleValue; + + static constexpr equality_comparable equality_comparable_support = // NOLINT + EqualityComparableValue; + + static constexpr inequality_comparable + inequality_comparable_support = // NOLINT + InequalityComparableValue; + + static constexpr less_than_comparable + less_than_comparable_support = // NOLINT + LessThanComparableValue; + + static constexpr less_equal_comparable + less_equal_comparable_support = // NOLINT + LessEqualComparableValue; + + static constexpr greater_equal_comparable + greater_equal_comparable_support = // NOLINT + GreaterEqualComparableValue; + + static constexpr greater_than_comparable + greater_than_comparable_support = // NOLINT + GreaterThanComparableValue; + + static constexpr swappable swappable_support = SwappableValue; // NOLINT + + static constexpr hashable hashable_support = HashableValue; // NOLINT + + static constexpr bool is_default_constructible = // NOLINT + DefaultConstructibleValue != default_constructible::maybe; + + static constexpr bool is_move_constructible = // NOLINT + MoveConstructibleValue != move_constructible::maybe; + + static constexpr bool is_copy_constructible = // NOLINT + CopyConstructibleValue != copy_constructible::maybe; + + static constexpr bool is_move_assignable = // NOLINT + MoveAssignableValue != move_assignable::maybe; + + static constexpr bool is_copy_assignable = // NOLINT + CopyAssignableValue != copy_assignable::maybe; + + static constexpr bool is_destructible = // NOLINT + DestructibleValue != destructible::maybe; + + static constexpr bool is_equality_comparable = // NOLINT + EqualityComparableValue != equality_comparable::maybe; + + static constexpr bool is_inequality_comparable = // NOLINT + InequalityComparableValue != inequality_comparable::maybe; + + static constexpr bool is_less_than_comparable = // NOLINT + LessThanComparableValue != less_than_comparable::maybe; + + static constexpr bool is_less_equal_comparable = // NOLINT + LessEqualComparableValue != less_equal_comparable::maybe; + + static constexpr bool is_greater_equal_comparable = // NOLINT + GreaterEqualComparableValue != greater_equal_comparable::maybe; + + static constexpr bool is_greater_than_comparable = // NOLINT + GreaterThanComparableValue != greater_than_comparable::maybe; + + static constexpr bool is_swappable = // NOLINT + SwappableValue != swappable::maybe; + + static constexpr bool is_hashable = // NOLINT + HashableValue != hashable::maybe; +}; + +//////////////////////////////////////////////////////////////////////////////// +// +// Compliant SFINAE-friendliness is not always present on the standard library +// implementations that we support. This helper-struct (and associated enum) is +// used as a means to conditionally check the hashability support of a type. +enum class CheckHashability { no, yes }; + +template <class T, CheckHashability ShouldCheckHashability> +struct conservative_hashable_support_of; + +template <class T> +struct conservative_hashable_support_of<T, CheckHashability::no> { + static constexpr hashable Invoke() { return hashable::maybe; } +}; + +template <class T> +struct conservative_hashable_support_of<T, CheckHashability::yes> { + static constexpr hashable Invoke() { return hashable_support_of<T>(); } +}; +// +//////////////////////////////////////////////////////////////////////////////// + +// The ConformanceProfile that is expected based on introspection into the type +// by way of trait checks. +template <class T, CheckHashability ShouldCheckHashability> +struct SyntacticConformanceProfileOf { + using properties = ConformanceProfile< + default_constructible_support_of<T>(), move_constructible_support_of<T>(), + copy_constructible_support_of<T>(), move_assignable_support_of<T>(), + copy_assignable_support_of<T>(), destructible_support_of<T>(), + equality_comparable_support_of<T>(), + inequality_comparable_support_of<T>(), + less_than_comparable_support_of<T>(), + less_equal_comparable_support_of<T>(), + greater_equal_comparable_support_of<T>(), + greater_than_comparable_support_of<T>(), swappable_support_of<T>(), + conservative_hashable_support_of<T, ShouldCheckHashability>::Invoke()>; +}; + +#define ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF_IMPL(type, name) \ + template <default_constructible DefaultConstructibleValue, \ + move_constructible MoveConstructibleValue, \ + copy_constructible CopyConstructibleValue, \ + move_assignable MoveAssignableValue, \ + copy_assignable CopyAssignableValue, \ + destructible DestructibleValue, \ + equality_comparable EqualityComparableValue, \ + inequality_comparable InequalityComparableValue, \ + less_than_comparable LessThanComparableValue, \ + less_equal_comparable LessEqualComparableValue, \ + greater_equal_comparable GreaterEqualComparableValue, \ + greater_than_comparable GreaterThanComparableValue, \ + swappable SwappableValue, hashable HashableValue> \ + constexpr type ConformanceProfile< \ + DefaultConstructibleValue, MoveConstructibleValue, \ + CopyConstructibleValue, MoveAssignableValue, CopyAssignableValue, \ + DestructibleValue, EqualityComparableValue, InequalityComparableValue, \ + LessThanComparableValue, LessEqualComparableValue, \ + GreaterEqualComparableValue, GreaterThanComparableValue, SwappableValue, \ + HashableValue>::name + +#define ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(type) \ + ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF_IMPL(type, \ + type##_support); \ + ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF_IMPL(bool, is_##type) + +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(default_constructible); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(move_constructible); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(copy_constructible); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(move_assignable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(copy_assignable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(destructible); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(equality_comparable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(inequality_comparable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(less_than_comparable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(less_equal_comparable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(greater_equal_comparable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(greater_than_comparable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(swappable); +ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(hashable); + +#undef ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF +#undef ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF_IMPL + +// Retrieve the enum with the minimum underlying value. +// Note: std::min is not constexpr in C++11, which is why this is necessary. +template <class H> +constexpr H MinEnum(H head) { + return head; +} + +template <class H, class N, class... T> +constexpr H MinEnum(H head, N next, T... tail) { + return (UnderlyingValue)(head) < (UnderlyingValue)(next) + ? (MinEnum)(head, tail...) + : (MinEnum)(next, tail...); +} + +template <class... Profs> +struct MinimalProfiles { + static constexpr default_constructible + default_constructible_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::default_constructible_support...); + + static constexpr move_constructible move_constructible_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::move_constructible_support...); + + static constexpr copy_constructible copy_constructible_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::copy_constructible_support...); + + static constexpr move_assignable move_assignable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::move_assignable_support...); + + static constexpr copy_assignable copy_assignable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::copy_assignable_support...); + + static constexpr destructible destructible_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::destructible_support...); + + static constexpr equality_comparable equality_comparable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::equality_comparable_support...); + + static constexpr inequality_comparable + inequality_comparable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::inequality_comparable_support...); + + static constexpr less_than_comparable + less_than_comparable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::less_than_comparable_support...); + + static constexpr less_equal_comparable + less_equal_comparable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::less_equal_comparable_support...); + + static constexpr greater_equal_comparable + greater_equal_comparable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::greater_equal_comparable_support...); + + static constexpr greater_than_comparable + greater_than_comparable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::greater_than_comparable_support...); + + static constexpr swappable swappable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::swappable_support...); + + static constexpr hashable hashable_support = // NOLINT + (MinEnum)(PropertiesOfT<Profs>::hashable_support...); + + using properties = ConformanceProfile< + default_constructible_support, move_constructible_support, + copy_constructible_support, move_assignable_support, + copy_assignable_support, destructible_support, + equality_comparable_support, inequality_comparable_support, + less_than_comparable_support, less_equal_comparable_support, + greater_equal_comparable_support, greater_than_comparable_support, + swappable_support, hashable_support>; +}; + +// Retrieve the enum with the greatest underlying value. +// Note: std::max is not constexpr in C++11, which is why this is necessary. +template <class H> +constexpr H MaxEnum(H head) { + return head; +} + +template <class H, class N, class... T> +constexpr H MaxEnum(H head, N next, T... tail) { + return (UnderlyingValue)(next) < (UnderlyingValue)(head) + ? (MaxEnum)(head, tail...) + : (MaxEnum)(next, tail...); +} + +template <class... Profs> +struct CombineProfilesImpl { + static constexpr default_constructible + default_constructible_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::default_constructible_support...); + + static constexpr move_constructible move_constructible_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::move_constructible_support...); + + static constexpr copy_constructible copy_constructible_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::copy_constructible_support...); + + static constexpr move_assignable move_assignable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::move_assignable_support...); + + static constexpr copy_assignable copy_assignable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::copy_assignable_support...); + + static constexpr destructible destructible_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::destructible_support...); + + static constexpr equality_comparable equality_comparable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::equality_comparable_support...); + + static constexpr inequality_comparable + inequality_comparable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::inequality_comparable_support...); + + static constexpr less_than_comparable + less_than_comparable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::less_than_comparable_support...); + + static constexpr less_equal_comparable + less_equal_comparable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::less_equal_comparable_support...); + + static constexpr greater_equal_comparable + greater_equal_comparable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::greater_equal_comparable_support...); + + static constexpr greater_than_comparable + greater_than_comparable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::greater_than_comparable_support...); + + static constexpr swappable swappable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::swappable_support...); + + static constexpr hashable hashable_support = // NOLINT + (MaxEnum)(PropertiesOfT<Profs>::hashable_support...); + + using properties = ConformanceProfile< + default_constructible_support, move_constructible_support, + copy_constructible_support, move_assignable_support, + copy_assignable_support, destructible_support, + equality_comparable_support, inequality_comparable_support, + less_than_comparable_support, less_equal_comparable_support, + greater_equal_comparable_support, greater_than_comparable_support, + swappable_support, hashable_support>; +}; + +// NOTE: We use this as opposed to a direct alias of CombineProfilesImpl so that +// when named aliases of CombineProfiles are created (such as in +// conformance_aliases.h), we only pay for the combination algorithm on the +// profiles that are actually used. +template <class... Profs> +struct CombineProfiles { + using profile_alias_of = CombineProfilesImpl<Profs...>; +}; + +template <> +struct CombineProfiles<> { + using properties = ConformanceProfile<>; +}; + +template <class Profile, class Tag> +struct StrongProfileTypedef { + using properties = PropertiesOfT<Profile>; +}; + +template <class T, class /*Enabler*/ = void> +struct IsProfileImpl : std::false_type {}; + +template <class T> +struct IsProfileImpl<T, absl::void_t<PropertiesOfT<T>>> : std::true_type {}; + +template <class T> +struct IsProfile : IsProfileImpl<T>::type {}; + +// A tag that describes which set of properties we will check when the user +// requires a strict match in conformance (as opposed to a loose match which +// allows more-refined support of any given operation). +// +// Currently only the RegularityDomain exists and it includes all operations +// that the conformance testing suite knows about. The intent is that if the +// suite is expanded to support extension, such as for checking conformance of +// concepts like Iterators or Containers, additional corresponding domains can +// be created. +struct RegularityDomain {}; + +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TYPES_INTERNAL_CONFORMANCE_PROFILE_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/conformance_testing.h b/third_party/abseil_cpp/absl/types/internal/conformance_testing.h new file mode 100644 index 000000000000..487b0f786b48 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/conformance_testing.h @@ -0,0 +1,1386 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// conformance_testing.h +// ----------------------------------------------------------------------------- +// + +#ifndef ABSL_TYPES_INTERNAL_CONFORMANCE_TESTING_H_ +#define ABSL_TYPES_INTERNAL_CONFORMANCE_TESTING_H_ + +//////////////////////////////////////////////////////////////////////////////// +// // +// Many templates in this file take a `T` and a `Prof` type as explicit // +// template arguments. These are a type to be checked and a // +// "Regularity Profile" that describes what operations that type `T` is // +// expected to support. See "regularity_profiles.h" for more details // +// regarding Regularity Profiles. // +// // +//////////////////////////////////////////////////////////////////////////////// + +#include <cstddef> +#include <set> +#include <tuple> +#include <type_traits> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/ascii.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/internal/conformance_aliases.h" +#include "absl/types/internal/conformance_archetype.h" +#include "absl/types/internal/conformance_profile.h" +#include "absl/types/internal/conformance_testing_helpers.h" +#include "absl/types/internal/parentheses.h" +#include "absl/types/internal/transform_args.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { + +// Returns true if the compiler incorrectly greedily instantiates constexpr +// templates in any unevaluated context. +constexpr bool constexpr_instantiation_when_unevaluated() { +#if defined(__apple_build_version__) // TODO(calabrese) Make more specific + return true; +#elif defined(__clang__) + return __clang_major__ < 4; +#elif defined(__GNUC__) + // TODO(calabrese) Figure out why gcc 7 fails (seems like a different bug) + return __GNUC__ < 5 || (__GNUC__ == 5 && __GNUC_MINOR__ < 2) || __GNUC__ >= 7; +#else + return false; +#endif +} + +// Returns true if the standard library being used incorrectly produces an error +// when instantiating the definition of a poisoned std::hash specialization. +constexpr bool poisoned_hash_fails_instantiation() { +#if defined(_MSC_VER) && !defined(_LIBCPP_VERSION) + return _MSC_VER < 1914; +#else + return false; +#endif +} + +template <class Fun> +struct GeneratorType { + decltype(std::declval<const Fun&>()()) operator()() const + noexcept(noexcept(std::declval<const Fun&>()())) { + return fun(); + } + + Fun fun; + const char* description; +}; + +// A "make" function for the GeneratorType template that deduces the function +// object type. +template <class Fun, + absl::enable_if_t<IsNullaryCallable<Fun>::value>** = nullptr> +GeneratorType<Fun> Generator(Fun fun, const char* description) { + return GeneratorType<Fun>{absl::move(fun), description}; +} + +// A type that contains a set of nullary function objects that each return an +// instance of the same type and value (though possibly different +// representations, such as +0 and -0 or two vectors with the same elements but +// with different capacities). +template <class... Funs> +struct EquivalenceClassType { + std::tuple<GeneratorType<Funs>...> generators; +}; + +// A "make" function for the EquivalenceClassType template that deduces the +// function object types and is constrained such that a user can only pass in +// function objects that all have the same return type. +template <class... Funs, absl::enable_if_t<AreGeneratorsWithTheSameReturnType< + Funs...>::value>** = nullptr> +EquivalenceClassType<Funs...> EquivalenceClass(GeneratorType<Funs>... funs) { + return {std::make_tuple(absl::move(funs)...)}; +} + +// A type that contains an ordered series of EquivalenceClassTypes, from +// smallest value to largest value. +template <class... EqClasses> +struct OrderedEquivalenceClasses { + std::tuple<EqClasses...> eq_classes; +}; + +// An object containing the parts of a given (name, initialization expression), +// and is capable of generating a string that describes the given. +struct GivenDeclaration { + std::string outputDeclaration(std::size_t width) const { + const std::size_t indent_size = 2; + std::string result = absl::StrCat(" ", name); + + if (!expression.empty()) { + // Indent + result.resize(indent_size + width, ' '); + absl::StrAppend(&result, " = ", expression, ";\n"); + } else { + absl::StrAppend(&result, ";\n"); + } + + return result; + } + + std::string name; + std::string expression; +}; + +// Produce a string that contains all of the givens of an error report. +template <class... Decls> +std::string PrepareGivenContext(const Decls&... decls) { + const std::size_t width = (std::max)({decls.name.size()...}); + return absl::StrCat("Given:\n", decls.outputDeclaration(width)..., "\n"); +} + +//////////////////////////////////////////////////////////////////////////////// +// Function objects that perform a check for each comparison operator // +//////////////////////////////////////////////////////////////////////////////// + +#define ABSL_INTERNAL_EXPECT_OP(name, op) \ + struct Expect##name { \ + template <class T> \ + void operator()(absl::string_view test_name, absl::string_view context, \ + const T& lhs, const T& rhs, absl::string_view lhs_name, \ + absl::string_view rhs_name) const { \ + if (!static_cast<bool>(lhs op rhs)) { \ + errors->addTestFailure( \ + test_name, absl::StrCat(context, \ + "**Unexpected comparison result**\n" \ + "\n" \ + "Expression:\n" \ + " ", \ + lhs_name, " " #op " ", rhs_name, \ + "\n" \ + "\n" \ + "Expected: true\n" \ + " Actual: false")); \ + } else { \ + errors->addTestSuccess(test_name); \ + } \ + } \ + \ + ConformanceErrors* errors; \ + }; \ + \ + struct ExpectNot##name { \ + template <class T> \ + void operator()(absl::string_view test_name, absl::string_view context, \ + const T& lhs, const T& rhs, absl::string_view lhs_name, \ + absl::string_view rhs_name) const { \ + if (lhs op rhs) { \ + errors->addTestFailure( \ + test_name, absl::StrCat(context, \ + "**Unexpected comparison result**\n" \ + "\n" \ + "Expression:\n" \ + " ", \ + lhs_name, " " #op " ", rhs_name, \ + "\n" \ + "\n" \ + "Expected: false\n" \ + " Actual: true")); \ + } else { \ + errors->addTestSuccess(test_name); \ + } \ + } \ + \ + ConformanceErrors* errors; \ + } + +ABSL_INTERNAL_EXPECT_OP(Eq, ==); +ABSL_INTERNAL_EXPECT_OP(Ne, !=); +ABSL_INTERNAL_EXPECT_OP(Lt, <); +ABSL_INTERNAL_EXPECT_OP(Le, <=); +ABSL_INTERNAL_EXPECT_OP(Ge, >=); +ABSL_INTERNAL_EXPECT_OP(Gt, >); + +#undef ABSL_INTERNAL_EXPECT_OP + +// A function object that verifies that two objects hash to the same value by +// way of the std::hash specialization. +struct ExpectSameHash { + template <class T> + void operator()(absl::string_view test_name, absl::string_view context, + const T& lhs, const T& rhs, absl::string_view lhs_name, + absl::string_view rhs_name) const { + if (std::hash<T>()(lhs) != std::hash<T>()(rhs)) { + errors->addTestFailure( + test_name, absl::StrCat(context, + "**Unexpected hash result**\n" + "\n" + "Expression:\n" + " std::hash<T>()(", + lhs_name, ") == std::hash<T>()(", rhs_name, + ")\n" + "\n" + "Expected: true\n" + " Actual: false")); + } else { + errors->addTestSuccess(test_name); + } + } + + ConformanceErrors* errors; +}; + +// A function template that takes two objects and verifies that each comparison +// operator behaves in a way that is consistent with equality. It has "OneWay" +// in the name because the first argument will always be the left-hand operand +// of the corresponding comparison operator and the second argument will +// always be the right-hand operand. It will never switch that order. +// At a higher level in the test suite, the one-way form is called once for each +// of the two possible orders whenever lhs and rhs are not the same initializer. +template <class T, class Prof> +void ExpectOneWayEquality(ConformanceErrors* errors, + absl::string_view test_name, + absl::string_view context, const T& lhs, const T& rhs, + absl::string_view lhs_name, + absl::string_view rhs_name) { + If<PropertiesOfT<Prof>::is_equality_comparable>::Invoke( + ExpectEq{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); + + If<PropertiesOfT<Prof>::is_inequality_comparable>::Invoke( + ExpectNotNe{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); + + If<PropertiesOfT<Prof>::is_less_than_comparable>::Invoke( + ExpectNotLt{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); + + If<PropertiesOfT<Prof>::is_less_equal_comparable>::Invoke( + ExpectLe{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); + + If<PropertiesOfT<Prof>::is_greater_equal_comparable>::Invoke( + ExpectGe{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); + + If<PropertiesOfT<Prof>::is_greater_than_comparable>::Invoke( + ExpectNotGt{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); + + If<PropertiesOfT<Prof>::is_hashable>::Invoke( + ExpectSameHash{errors}, test_name, context, lhs, rhs, lhs_name, rhs_name); +} + +// A function template that takes two objects and verifies that each comparison +// operator behaves in a way that is consistent with equality. This function +// differs from ExpectOneWayEquality in that this will do checks with argument +// order reversed in addition to in-order. +template <class T, class Prof> +void ExpectEquality(ConformanceErrors* errors, absl::string_view test_name, + absl::string_view context, const T& lhs, const T& rhs, + absl::string_view lhs_name, absl::string_view rhs_name) { + (ExpectOneWayEquality<T, Prof>)(errors, test_name, context, lhs, rhs, + lhs_name, rhs_name); + (ExpectOneWayEquality<T, Prof>)(errors, test_name, context, rhs, lhs, + rhs_name, lhs_name); +} + +// Given a generator, makes sure that a generated value and a moved-from +// generated value are equal. +template <class T, class Prof> +struct ExpectMoveConstructOneGenerator { + template <class Fun> + void operator()(const Fun& generator) const { + const T object = generator(); + const T moved_object = absl::move(generator()); // Force no elision. + + (ExpectEquality<T, Prof>)(errors, "Move construction", + PrepareGivenContext( + GivenDeclaration{"const _T object", + generator.description}, + GivenDeclaration{"const _T moved_object", + std::string("std::move(") + + generator.description + + ")"}), + object, moved_object, "object", "moved_object"); + } + + ConformanceErrors* errors; +}; + +// Given a generator, makes sure that a generated value and a copied-from +// generated value are equal. +template <class T, class Prof> +struct ExpectCopyConstructOneGenerator { + template <class Fun> + void operator()(const Fun& generator) const { + const T object = generator(); + const T copied_object = static_cast<const T&>(generator()); + + (ExpectEquality<T, Prof>)(errors, "Copy construction", + PrepareGivenContext( + GivenDeclaration{"const _T object", + generator.description}, + GivenDeclaration{ + "const _T copied_object", + std::string("static_cast<const _T&>(") + + generator.description + ")"}), + object, copied_object, "object", "copied_object"); + } + + ConformanceErrors* errors; +}; + +// Default-construct and do nothing before destruction. +// +// This is useful in exercising the codepath of default construction followed by +// destruction, but does not explicitly test anything. An example of where this +// might fail is a default destructor that default-initializes a scalar and a +// destructor reads the value of that member. Sanitizers can catch this as long +// as our test attempts to execute such a case. +template <class T> +struct ExpectDefaultConstructWithDestruct { + void operator()() const { + // Scoped so that destructor gets called before reporting success. + { + T object; + static_cast<void>(object); + } + + errors->addTestSuccess("Default construction"); + } + + ConformanceErrors* errors; +}; + +// Check move-assign into a default-constructed object. +template <class T, class Prof> +struct ExpectDefaultConstructWithMoveAssign { + template <class Fun> + void operator()(const Fun& generator) const { + const T source_of_truth = generator(); + T object; + object = generator(); + + (ExpectEquality<T, Prof>)(errors, "Move assignment", + PrepareGivenContext( + GivenDeclaration{"const _T object", + generator.description}, + GivenDeclaration{"_T object", ""}, + GivenDeclaration{"object", + generator.description}), + object, source_of_truth, "std::as_const(object)", + "source_of_truth"); + } + + ConformanceErrors* errors; +}; + +// Check copy-assign into a default-constructed object. +template <class T, class Prof> +struct ExpectDefaultConstructWithCopyAssign { + template <class Fun> + void operator()(const Fun& generator) const { + const T source_of_truth = generator(); + T object; + object = static_cast<const T&>(generator()); + + (ExpectEquality<T, Prof>)(errors, "Copy assignment", + PrepareGivenContext( + GivenDeclaration{"const _T source_of_truth", + generator.description}, + GivenDeclaration{"_T object", ""}, + GivenDeclaration{ + "object", + std::string("static_cast<const _T&>(") + + generator.description + ")"}), + object, source_of_truth, "std::as_const(object)", + "source_of_truth"); + } + + ConformanceErrors* errors; +}; + +// Perform a self move-assign. +template <class T, class Prof> +struct ExpectSelfMoveAssign { + template <class Fun> + void operator()(const Fun& generator) const { + T object = generator(); + object = absl::move(object); + + // NOTE: Self move-assign results in a valid-but-unspecified state. + + (ExpectEquality<T, Prof>)(errors, "Move assignment", + PrepareGivenContext( + GivenDeclaration{"_T object", + generator.description}, + GivenDeclaration{"object", + "std::move(object)"}), + object, object, "object", "object"); + } + + ConformanceErrors* errors; +}; + +// Perform a self copy-assign. +template <class T, class Prof> +struct ExpectSelfCopyAssign { + template <class Fun> + void operator()(const Fun& generator) const { + const T source_of_truth = generator(); + T object = generator(); + const T& const_object = object; + object = const_object; + + (ExpectEquality<T, Prof>)(errors, "Copy assignment", + PrepareGivenContext( + GivenDeclaration{"const _T source_of_truth", + generator.description}, + GivenDeclaration{"_T object", + generator.description}, + GivenDeclaration{"object", + "std::as_const(object)"}), + const_object, source_of_truth, + "std::as_const(object)", "source_of_truth"); + } + + ConformanceErrors* errors; +}; + +// Perform a self-swap. +template <class T, class Prof> +struct ExpectSelfSwap { + template <class Fun> + void operator()(const Fun& generator) const { + const T source_of_truth = generator(); + T object = generator(); + + type_traits_internal::Swap(object, object); + + std::string preliminary_info = absl::StrCat( + PrepareGivenContext( + GivenDeclaration{"const _T source_of_truth", generator.description}, + GivenDeclaration{"_T object", generator.description}), + "After performing a self-swap:\n" + " using std::swap;\n" + " swap(object, object);\n" + "\n"); + + (ExpectEquality<T, Prof>)(errors, "Swap", std::move(preliminary_info), + object, source_of_truth, "std::as_const(object)", + "source_of_truth"); + } + + ConformanceErrors* errors; +}; + +// Perform each of the single-generator checks when necessary operations are +// supported. +template <class T, class Prof> +struct ExpectSelfComparison { + template <class Fun> + void operator()(const Fun& generator) const { + const T object = generator(); + (ExpectOneWayEquality<T, Prof>)(errors, "Comparison", + PrepareGivenContext(GivenDeclaration{ + "const _T object", + generator.description}), + object, object, "object", "object"); + } + + ConformanceErrors* errors; +}; + +// Perform each of the single-generator checks when necessary operations are +// supported. +template <class T, class Prof> +struct ExpectConsistency { + template <class Fun> + void operator()(const Fun& generator) const { + If<PropertiesOfT<Prof>::is_move_constructible>::Invoke( + ExpectMoveConstructOneGenerator<T, Prof>{errors}, generator); + + If<PropertiesOfT<Prof>::is_copy_constructible>::Invoke( + ExpectCopyConstructOneGenerator<T, Prof>{errors}, generator); + + If<PropertiesOfT<Prof>::is_default_constructible && + PropertiesOfT<Prof>::is_move_assignable>:: + Invoke(ExpectDefaultConstructWithMoveAssign<T, Prof>{errors}, + generator); + + If<PropertiesOfT<Prof>::is_default_constructible && + PropertiesOfT<Prof>::is_copy_assignable>:: + Invoke(ExpectDefaultConstructWithCopyAssign<T, Prof>{errors}, + generator); + + If<PropertiesOfT<Prof>::is_move_assignable>::Invoke( + ExpectSelfMoveAssign<T, Prof>{errors}, generator); + + If<PropertiesOfT<Prof>::is_copy_assignable>::Invoke( + ExpectSelfCopyAssign<T, Prof>{errors}, generator); + + If<PropertiesOfT<Prof>::is_swappable>::Invoke( + ExpectSelfSwap<T, Prof>{errors}, generator); + } + + ConformanceErrors* errors; +}; + +// Check move-assign with two different values. +template <class T, class Prof> +struct ExpectMoveAssign { + template <class Fun0, class Fun1> + void operator()(const Fun0& generator0, const Fun1& generator1) const { + const T source_of_truth1 = generator1(); + T object = generator0(); + object = generator1(); + + (ExpectEquality<T, Prof>)(errors, "Move assignment", + PrepareGivenContext( + GivenDeclaration{"const _T source_of_truth1", + generator1.description}, + GivenDeclaration{"_T object", + generator0.description}, + GivenDeclaration{"object", + generator1.description}), + object, source_of_truth1, "std::as_const(object)", + "source_of_truth1"); + } + + ConformanceErrors* errors; +}; + +// Check copy-assign with two different values. +template <class T, class Prof> +struct ExpectCopyAssign { + template <class Fun0, class Fun1> + void operator()(const Fun0& generator0, const Fun1& generator1) const { + const T source_of_truth1 = generator1(); + T object = generator0(); + object = static_cast<const T&>(generator1()); + + (ExpectEquality<T, Prof>)(errors, "Copy assignment", + PrepareGivenContext( + GivenDeclaration{"const _T source_of_truth1", + generator1.description}, + GivenDeclaration{"_T object", + generator0.description}, + GivenDeclaration{ + "object", + std::string("static_cast<const _T&>(") + + generator1.description + ")"}), + object, source_of_truth1, "std::as_const(object)", + "source_of_truth1"); + } + + ConformanceErrors* errors; +}; + +// Check swap with two different values. +template <class T, class Prof> +struct ExpectSwap { + template <class Fun0, class Fun1> + void operator()(const Fun0& generator0, const Fun1& generator1) const { + const T source_of_truth0 = generator0(); + const T source_of_truth1 = generator1(); + T object0 = generator0(); + T object1 = generator1(); + + type_traits_internal::Swap(object0, object1); + + const std::string context = + PrepareGivenContext( + GivenDeclaration{"const _T source_of_truth0", + generator0.description}, + GivenDeclaration{"const _T source_of_truth1", + generator1.description}, + GivenDeclaration{"_T object0", generator0.description}, + GivenDeclaration{"_T object1", generator1.description}) + + "After performing a swap:\n" + " using std::swap;\n" + " swap(object0, object1);\n" + "\n"; + + (ExpectEquality<T, Prof>)(errors, "Swap", context, object0, + source_of_truth1, "std::as_const(object0)", + "source_of_truth1"); + (ExpectEquality<T, Prof>)(errors, "Swap", context, object1, + source_of_truth0, "std::as_const(object1)", + "source_of_truth0"); + } + + ConformanceErrors* errors; +}; + +// Validate that `generator0` and `generator1` produce values that are equal. +template <class T, class Prof> +struct ExpectEquivalenceClassComparison { + template <class Fun0, class Fun1> + void operator()(const Fun0& generator0, const Fun1& generator1) const { + const T object0 = generator0(); + const T object1 = generator1(); + + (ExpectEquality<T, Prof>)(errors, "Comparison", + PrepareGivenContext( + GivenDeclaration{"const _T object0", + generator0.description}, + GivenDeclaration{"const _T object1", + generator1.description}), + object0, object1, "object0", "object1"); + } + + ConformanceErrors* errors; +}; + +// Validate that all objects in the same equivalence-class have the same value. +template <class T, class Prof> +struct ExpectEquivalenceClassConsistency { + template <class Fun0, class Fun1> + void operator()(const Fun0& generator0, const Fun1& generator1) const { + If<PropertiesOfT<Prof>::is_move_assignable>::Invoke( + ExpectMoveAssign<T, Prof>{errors}, generator0, generator1); + + If<PropertiesOfT<Prof>::is_copy_assignable>::Invoke( + ExpectCopyAssign<T, Prof>{errors}, generator0, generator1); + + If<PropertiesOfT<Prof>::is_swappable>::Invoke(ExpectSwap<T, Prof>{errors}, + generator0, generator1); + } + + ConformanceErrors* errors; +}; + +// Given a "lesser" object and a "greater" object, perform every combination of +// comparison operators supported for the type, expecting consistent results. +template <class T, class Prof> +void ExpectOrdered(ConformanceErrors* errors, absl::string_view context, + const T& small, const T& big, absl::string_view small_name, + absl::string_view big_name) { + const absl::string_view test_name = "Comparison"; + + If<PropertiesOfT<Prof>::is_equality_comparable>::Invoke( + ExpectNotEq{errors}, test_name, context, small, big, small_name, + big_name); + If<PropertiesOfT<Prof>::is_equality_comparable>::Invoke( + ExpectNotEq{errors}, test_name, context, big, small, big_name, + small_name); + + If<PropertiesOfT<Prof>::is_inequality_comparable>::Invoke( + ExpectNe{errors}, test_name, context, small, big, small_name, big_name); + If<PropertiesOfT<Prof>::is_inequality_comparable>::Invoke( + ExpectNe{errors}, test_name, context, big, small, big_name, small_name); + + If<PropertiesOfT<Prof>::is_less_than_comparable>::Invoke( + ExpectLt{errors}, test_name, context, small, big, small_name, big_name); + If<PropertiesOfT<Prof>::is_less_than_comparable>::Invoke( + ExpectNotLt{errors}, test_name, context, big, small, big_name, + small_name); + + If<PropertiesOfT<Prof>::is_less_equal_comparable>::Invoke( + ExpectLe{errors}, test_name, context, small, big, small_name, big_name); + If<PropertiesOfT<Prof>::is_less_equal_comparable>::Invoke( + ExpectNotLe{errors}, test_name, context, big, small, big_name, + small_name); + + If<PropertiesOfT<Prof>::is_greater_equal_comparable>::Invoke( + ExpectNotGe{errors}, test_name, context, small, big, small_name, + big_name); + If<PropertiesOfT<Prof>::is_greater_equal_comparable>::Invoke( + ExpectGe{errors}, test_name, context, big, small, big_name, small_name); + + If<PropertiesOfT<Prof>::is_greater_than_comparable>::Invoke( + ExpectNotGt{errors}, test_name, context, small, big, small_name, + big_name); + If<PropertiesOfT<Prof>::is_greater_than_comparable>::Invoke( + ExpectGt{errors}, test_name, context, big, small, big_name, small_name); +} + +// For every two elements of an equivalence class, makes sure that those two +// elements compare equal, including checks with the same argument passed as +// both operands. +template <class T, class Prof> +struct ExpectEquivalenceClassComparisons { + template <class... Funs> + void operator()(EquivalenceClassType<Funs...> eq_class) const { + (ForEachTupleElement)(ExpectSelfComparison<T, Prof>{errors}, + eq_class.generators); + + (ForEveryTwo)(ExpectEquivalenceClassComparison<T, Prof>{errors}, + eq_class.generators); + } + + ConformanceErrors* errors; +}; + +// For every element of an equivalence class, makes sure that the element is +// self-consistent (in other words, if any of move/copy/swap are defined, +// perform those operations and make such that results and operands still +// compare equal to known values whenever it is required for that operation. +template <class T, class Prof> +struct ExpectEquivalenceClass { + template <class... Funs> + void operator()(EquivalenceClassType<Funs...> eq_class) const { + (ForEachTupleElement)(ExpectConsistency<T, Prof>{errors}, + eq_class.generators); + + (ForEveryTwo)(ExpectEquivalenceClassConsistency<T, Prof>{errors}, + eq_class.generators); + } + + ConformanceErrors* errors; +}; + +// Validate that the passed-in argument is a generator of a greater value than +// the one produced by the "small_gen" datamember with respect to all of the +// comparison operators that Prof requires, with both argument orders to test. +template <class T, class Prof, class SmallGenerator> +struct ExpectBiggerGeneratorThanComparisons { + template <class BigGenerator> + void operator()(BigGenerator big_gen) const { + const T small = small_gen(); + const T big = big_gen(); + + (ExpectOrdered<T, Prof>)(errors, + PrepareGivenContext( + GivenDeclaration{"const _T small", + small_gen.description}, + GivenDeclaration{"const _T big", + big_gen.description}), + small, big, "small", "big"); + } + + SmallGenerator small_gen; + ConformanceErrors* errors; +}; + +// Perform all of the move, copy, and swap checks on the value generated by +// `small_gen` and the value generated by `big_gen`. +template <class T, class Prof, class SmallGenerator> +struct ExpectBiggerGeneratorThan { + template <class BigGenerator> + void operator()(BigGenerator big_gen) const { + If<PropertiesOfT<Prof>::is_move_assignable>::Invoke( + ExpectMoveAssign<T, Prof>{errors}, small_gen, big_gen); + If<PropertiesOfT<Prof>::is_move_assignable>::Invoke( + ExpectMoveAssign<T, Prof>{errors}, big_gen, small_gen); + + If<PropertiesOfT<Prof>::is_copy_assignable>::Invoke( + ExpectCopyAssign<T, Prof>{errors}, small_gen, big_gen); + If<PropertiesOfT<Prof>::is_copy_assignable>::Invoke( + ExpectCopyAssign<T, Prof>{errors}, big_gen, small_gen); + + If<PropertiesOfT<Prof>::is_swappable>::Invoke(ExpectSwap<T, Prof>{errors}, + small_gen, big_gen); + } + + SmallGenerator small_gen; + ConformanceErrors* errors; +}; + +// Validate that the result of a generator is greater than the results of all +// generators in an equivalence class with respect to comparisons. +template <class T, class Prof, class SmallGenerator> +struct ExpectBiggerGeneratorThanEqClassesComparisons { + template <class BigEqClass> + void operator()(BigEqClass big_eq_class) const { + (ForEachTupleElement)( + ExpectBiggerGeneratorThanComparisons<T, Prof, SmallGenerator>{small_gen, + errors}, + big_eq_class.generators); + } + + SmallGenerator small_gen; + ConformanceErrors* errors; +}; + +// Validate that the non-comparison binary operations required by Prof are +// correct for the result of each generator of big_eq_class and a generator of +// the logically smaller value returned by small_gen. +template <class T, class Prof, class SmallGenerator> +struct ExpectBiggerGeneratorThanEqClasses { + template <class BigEqClass> + void operator()(BigEqClass big_eq_class) const { + (ForEachTupleElement)( + ExpectBiggerGeneratorThan<T, Prof, SmallGenerator>{small_gen, errors}, + big_eq_class.generators); + } + + SmallGenerator small_gen; + ConformanceErrors* errors; +}; + +// Validate that each equivalence class that is passed is logically less than +// the equivalence classes that comes later on in the argument list. +template <class T, class Prof> +struct ExpectOrderedEquivalenceClassesComparisons { + template <class... BigEqClasses> + struct Impl { + // Validate that the value produced by `small_gen` is less than all of the + // values generated by those of the logically larger equivalence classes. + template <class SmallGenerator> + void operator()(SmallGenerator small_gen) const { + (ForEachTupleElement)(ExpectBiggerGeneratorThanEqClassesComparisons< + T, Prof, SmallGenerator>{small_gen, errors}, + big_eq_classes); + } + + std::tuple<BigEqClasses...> big_eq_classes; + ConformanceErrors* errors; + }; + + // When given no equivalence classes, no validation is necessary. + void operator()() const {} + + template <class SmallEqClass, class... BigEqClasses> + void operator()(SmallEqClass small_eq_class, + BigEqClasses... big_eq_classes) const { + // For each generator in the first equivalence class, make sure that it is + // less than each of those in the logically greater equivalence classes. + (ForEachTupleElement)( + Impl<BigEqClasses...>{std::make_tuple(absl::move(big_eq_classes)...), + errors}, + small_eq_class.generators); + + // Recurse so that all equivalence class combinations are checked. + (*this)(absl::move(big_eq_classes)...); + } + + ConformanceErrors* errors; +}; + +// Validate that the non-comparison binary operations required by Prof are +// correct for the result of each generator of big_eq_classes and a generator of +// the logically smaller value returned by small_gen. +template <class T, class Prof> +struct ExpectOrderedEquivalenceClasses { + template <class... BigEqClasses> + struct Impl { + template <class SmallGenerator> + void operator()(SmallGenerator small_gen) const { + (ForEachTupleElement)( + ExpectBiggerGeneratorThanEqClasses<T, Prof, SmallGenerator>{small_gen, + errors}, + big_eq_classes); + } + + std::tuple<BigEqClasses...> big_eq_classes; + ConformanceErrors* errors; + }; + + // Check that small_eq_class is logically consistent and also is logically + // less than all values in big_eq_classes. + template <class SmallEqClass, class... BigEqClasses> + void operator()(SmallEqClass small_eq_class, + BigEqClasses... big_eq_classes) const { + (ForEachTupleElement)( + Impl<BigEqClasses...>{std::make_tuple(absl::move(big_eq_classes)...), + errors}, + small_eq_class.generators); + + (*this)(absl::move(big_eq_classes)...); + } + + // Terminating case of operator(). + void operator()() const {} + + ConformanceErrors* errors; +}; + +// Validate that a type meets the syntactic requirements of std::hash if the +// range of profiles requires it. +template <class T, class MinProf, class MaxProf> +struct ExpectHashable { + void operator()() const { + ExpectModelOfHashable<T, MinProf, MaxProf>(errors); + } + + ConformanceErrors* errors; +}; + +// Validate that the type `T` meets all of the requirements associated with +// `MinProf` and without going beyond the syntactic properties of `MaxProf`. +template <class T, class MinProf, class MaxProf> +struct ExpectModels { + void operator()(ConformanceErrors* errors) const { + ExpectModelOfDefaultConstructible<T, MinProf, MaxProf>(errors); + ExpectModelOfMoveConstructible<T, MinProf, MaxProf>(errors); + ExpectModelOfCopyConstructible<T, MinProf, MaxProf>(errors); + ExpectModelOfMoveAssignable<T, MinProf, MaxProf>(errors); + ExpectModelOfCopyAssignable<T, MinProf, MaxProf>(errors); + ExpectModelOfDestructible<T, MinProf, MaxProf>(errors); + ExpectModelOfEqualityComparable<T, MinProf, MaxProf>(errors); + ExpectModelOfInequalityComparable<T, MinProf, MaxProf>(errors); + ExpectModelOfLessThanComparable<T, MinProf, MaxProf>(errors); + ExpectModelOfLessEqualComparable<T, MinProf, MaxProf>(errors); + ExpectModelOfGreaterEqualComparable<T, MinProf, MaxProf>(errors); + ExpectModelOfGreaterThanComparable<T, MinProf, MaxProf>(errors); + ExpectModelOfSwappable<T, MinProf, MaxProf>(errors); + + // Only check hashability on compilers that have a compliant default-hash. + If<!poisoned_hash_fails_instantiation()>::Invoke( + ExpectHashable<T, MinProf, MaxProf>{errors}); + } +}; + +// A metafunction that yields a Profile matching the set of properties that are +// safe to be checked (lack-of-hashability is only checked on standard library +// implementations that are standards compliant in that they provide a std::hash +// primary template that is SFINAE-friendly) +template <class LogicalProf, class T> +struct MinimalCheckableProfile { + using type = + MinimalProfiles<PropertiesOfT<LogicalProf>, + PropertiesOfT<SyntacticConformanceProfileOf< + T, !PropertiesOfT<LogicalProf>::is_hashable && + poisoned_hash_fails_instantiation() + ? CheckHashability::no + : CheckHashability::yes>>>; +}; + +// An identity metafunction +template <class T> +struct Always { + using type = T; +}; + +// Validate the T meets all of the necessary requirements of LogicalProf, with +// syntactic requirements defined by the profile range [MinProf, MaxProf]. +template <class T, class LogicalProf, class MinProf, class MaxProf, + class... EqClasses> +ConformanceErrors ExpectRegularityImpl( + OrderedEquivalenceClasses<EqClasses...> vals) { + ConformanceErrors errors((NameOf<T>())); + + If<!constexpr_instantiation_when_unevaluated()>::Invoke( + ExpectModels<T, MinProf, MaxProf>(), &errors); + + using minimal_profile = typename absl::conditional_t< + constexpr_instantiation_when_unevaluated(), Always<LogicalProf>, + MinimalCheckableProfile<LogicalProf, T>>::type; + + If<PropertiesOfT<minimal_profile>::is_default_constructible>::Invoke( + ExpectDefaultConstructWithDestruct<T>{&errors}); + + ////////////////////////////////////////////////////////////////////////////// + // Perform all comparison checks first, since later checks depend on their + // correctness. + // + // Check all of the comparisons for all values in the same equivalence + // class (equal with respect to comparison operators and hash the same). + (ForEachTupleElement)( + ExpectEquivalenceClassComparisons<T, minimal_profile>{&errors}, + vals.eq_classes); + + // Check all of the comparisons for each combination of values that are in + // different equivalence classes (not equal with respect to comparison + // operators). + absl::apply( + ExpectOrderedEquivalenceClassesComparisons<T, minimal_profile>{&errors}, + vals.eq_classes); + // + ////////////////////////////////////////////////////////////////////////////// + + // Perform remaining checks, relying on comparisons. + // TODO(calabrese) short circuit if any comparisons above failed. + (ForEachTupleElement)(ExpectEquivalenceClass<T, minimal_profile>{&errors}, + vals.eq_classes); + + absl::apply(ExpectOrderedEquivalenceClasses<T, minimal_profile>{&errors}, + vals.eq_classes); + + return errors; +} + +// A type that represents a range of profiles that are acceptable to be matched. +// +// `MinProf` is the minimum set of syntactic requirements that must be met. +// +// `MaxProf` is the maximum set of syntactic requirements that must be met. +// This maximum is particularly useful for certain "strictness" checking. Some +// examples for when this is useful: +// +// * Making sure that a type is move-only (rather than simply movable) +// +// * Making sure that a member function is *not* noexcept in cases where it +// cannot be noexcept, such as if a dependent datamember has certain +// operations that are not noexcept. +// +// * Making sure that a type tightly matches a spec, such as the standard. +// +// `LogicalProf` is the Profile for which run-time testing is to take place. +// +// Note: The reason for `LogicalProf` is because it is often the case, when +// dealing with templates, that a declaration of a given operation is specified, +// but whose body would fail to instantiate. Examples include the +// copy-constructor of a standard container when the element-type is move-only, +// or the comparison operators of a standard container when the element-type +// does not have the necessary comparison operations defined. The `LogicalProf` +// parameter allows us to capture the intent of what should be tested at +// run-time, even in the cases where syntactically it might otherwise appear as +// though the type undergoing testing supports more than it actually does. +template <class LogicalProf, class MinProf = LogicalProf, + class MaxProf = MinProf> +struct ProfileRange { + using logical_profile = LogicalProf; + using min_profile = MinProf; + using max_profile = MaxProf; +}; + +// Similar to ProfileRange except that it creates a profile range that is +// coupled with a Domain and is used when testing that a type matches exactly +// the "minimum" requirements of LogicalProf. +template <class StrictnessDomain, class LogicalProf, + class MinProf = LogicalProf, class MaxProf = MinProf> +struct StrictProfileRange { + // We do not yet support extension. + static_assert( + std::is_same<StrictnessDomain, RegularityDomain>::value, + "Currently, the only valid StrictnessDomain is RegularityDomain."); + using strictness_domain = StrictnessDomain; + using logical_profile = LogicalProf; + using min_profile = MinProf; + using max_profile = MaxProf; +}; + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction that creates a StrictProfileRange from a Domain and either a +// Profile or ProfileRange. +template <class StrictnessDomain, class ProfOrRange> +struct MakeStrictProfileRange; + +template <class StrictnessDomain, class LogicalProf> +struct MakeStrictProfileRange { + using type = StrictProfileRange<StrictnessDomain, LogicalProf>; +}; + +template <class StrictnessDomain, class LogicalProf, class MinProf, + class MaxProf> +struct MakeStrictProfileRange<StrictnessDomain, + ProfileRange<LogicalProf, MinProf, MaxProf>> { + using type = + StrictProfileRange<StrictnessDomain, LogicalProf, MinProf, MaxProf>; +}; + +template <class StrictnessDomain, class ProfOrRange> +using MakeStrictProfileRangeT = + typename MakeStrictProfileRange<StrictnessDomain, ProfOrRange>::type; +// +//////////////////////////////////////////////////////////////////////////////// + +// A profile in the RegularityDomain with the strongest possible requirements. +using MostStrictProfile = + CombineProfiles<TriviallyCompleteProfile, NothrowComparableProfile>; + +// Forms a ProfileRange that treats the Profile as the bare minimum requirements +// of a type. +template <class LogicalProf, class MinProf = LogicalProf> +using LooseProfileRange = StrictProfileRange<RegularityDomain, LogicalProf, + MinProf, MostStrictProfile>; + +template <class Prof> +using MakeLooseProfileRangeT = Prof; + +//////////////////////////////////////////////////////////////////////////////// +// +// The following classes implement the metafunction ProfileRangeOfT<T> that +// takes either a Profile or ProfileRange and yields the ProfileRange to be +// used during testing. +// +template <class T, class /*Enabler*/ = void> +struct ProfileRangeOfImpl; + +template <class T> +struct ProfileRangeOfImpl<T, absl::void_t<PropertiesOfT<T>>> { + using type = LooseProfileRange<T>; +}; + +template <class T> +struct ProfileRangeOf : ProfileRangeOfImpl<T> {}; + +template <class StrictnessDomain, class LogicalProf, class MinProf, + class MaxProf> +struct ProfileRangeOf< + StrictProfileRange<StrictnessDomain, LogicalProf, MinProf, MaxProf>> { + using type = + StrictProfileRange<StrictnessDomain, LogicalProf, MinProf, MaxProf>; +}; + +template <class T> +using ProfileRangeOfT = typename ProfileRangeOf<T>::type; +// +//////////////////////////////////////////////////////////////////////////////// + +// Extract the logical profile of a range (what will be runtime tested). +template <class T> +using LogicalProfileOfT = typename ProfileRangeOfT<T>::logical_profile; + +// Extract the minimal syntactic profile of a range (error if not at least). +template <class T> +using MinProfileOfT = typename ProfileRangeOfT<T>::min_profile; + +// Extract the maximum syntactic profile of a range (error if more than). +template <class T> +using MaxProfileOfT = typename ProfileRangeOfT<T>::max_profile; + +//////////////////////////////////////////////////////////////////////////////// +// +template <class T> +struct IsProfileOrProfileRange : IsProfile<T>::type {}; + +template <class StrictnessDomain, class LogicalProf, class MinProf, + class MaxProf> +struct IsProfileOrProfileRange< + StrictProfileRange<StrictnessDomain, LogicalProf, MinProf, MaxProf>> + : std::true_type {}; +// +//////////////////////////////////////////////////////////////////////////////// + +// TODO(calabrese): Consider naming the functions in this class the same as +// the macros (defined later on) so that auto-complete leads to the correct name +// and so that a user cannot accidentally call a function rather than the macro +// form. +template <bool ExpectSuccess, class T, class... EqClasses> +struct ExpectConformanceOf { + // Add a value to be tested. Subsequent calls to this function on the same + // object must specify logically "larger" values with respect to the + // comparison operators of the type, if any. + // + // NOTE: This function should not be called directly. A stateless lambda is + // implicitly formed and passed when using the INITIALIZER macro at the bottom + // of this file. + template <class Fun, + absl::enable_if_t<std::is_same< + ResultOfGeneratorT<GeneratorType<Fun>>, T>::value>** = nullptr> + ABSL_MUST_USE_RESULT ExpectConformanceOf<ExpectSuccess, T, EqClasses..., + EquivalenceClassType<Fun>> + initializer(GeneratorType<Fun> fun) && { + return { + {std::tuple_cat(absl::move(ordered_vals.eq_classes), + std::make_tuple((EquivalenceClass)(absl::move(fun))))}, + std::move(expected_failed_tests)}; + } + + template <class... TestNames, + absl::enable_if_t<!ExpectSuccess && sizeof...(EqClasses) == 0 && + absl::conjunction<std::is_convertible< + TestNames, absl::string_view>...>::value>** = + nullptr> + ABSL_MUST_USE_RESULT ExpectConformanceOf<ExpectSuccess, T, EqClasses...> + due_to(TestNames&&... test_names) && { + (InsertEach)(&expected_failed_tests, + absl::AsciiStrToLower(absl::string_view(test_names))...); + + return {absl::move(ordered_vals), std::move(expected_failed_tests)}; + } + + template <class... TestNames, int = 0, // MSVC disambiguator + absl::enable_if_t<ExpectSuccess && sizeof...(EqClasses) == 0 && + absl::conjunction<std::is_convertible< + TestNames, absl::string_view>...>::value>** = + nullptr> + ABSL_MUST_USE_RESULT ExpectConformanceOf<ExpectSuccess, T, EqClasses...> + due_to(TestNames&&... test_names) && { + // TODO(calabrese) Instead have DUE_TO only exist via a CRTP base. + // This would produce better errors messages than the static_assert. + static_assert(!ExpectSuccess, + "DUE_TO cannot be called when conformance is expected -- did " + "you mean to use ASSERT_NONCONFORMANCE_OF?"); + } + + // Add a value to be tested. Subsequent calls to this function on the same + // object must specify logically "larger" values with respect to the + // comparison operators of the type, if any. + // + // NOTE: This function should not be called directly. A stateful lambda is + // implicitly formed and passed when using the INITIALIZER macro at the bottom + // of this file. + template <class Fun, + absl::enable_if_t<std::is_same< + ResultOfGeneratorT<GeneratorType<Fun>>, T>::value>** = nullptr> + ABSL_MUST_USE_RESULT ExpectConformanceOf<ExpectSuccess, T, EqClasses..., + EquivalenceClassType<Fun>> + dont_class_directly_stateful_initializer(GeneratorType<Fun> fun) && { + return { + {std::tuple_cat(absl::move(ordered_vals.eq_classes), + std::make_tuple((EquivalenceClass)(absl::move(fun))))}, + std::move(expected_failed_tests)}; + } + + // Add a set of value to be tested, where each value is equal with respect to + // the comparison operators and std::hash specialization, if defined. + template < + class... Funs, + absl::void_t<absl::enable_if_t<std::is_same< + ResultOfGeneratorT<GeneratorType<Funs>>, T>::value>...>** = nullptr> + ABSL_MUST_USE_RESULT ExpectConformanceOf<ExpectSuccess, T, EqClasses..., + EquivalenceClassType<Funs...>> + equivalence_class(GeneratorType<Funs>... funs) && { + return {{std::tuple_cat( + absl::move(ordered_vals.eq_classes), + std::make_tuple((EquivalenceClass)(absl::move(funs)...)))}, + std::move(expected_failed_tests)}; + } + + // Execute the tests for the captured set of values, strictly matching a range + // of expected profiles in a given domain. + template < + class ProfRange, + absl::enable_if_t<IsProfileOrProfileRange<ProfRange>::value>** = nullptr> + ABSL_MUST_USE_RESULT ::testing::AssertionResult with_strict_profile( + ProfRange /*profile*/) { + ConformanceErrors test_result = + (ExpectRegularityImpl< + T, LogicalProfileOfT<ProfRange>, MinProfileOfT<ProfRange>, + MaxProfileOfT<ProfRange>>)(absl::move(ordered_vals)); + + return ExpectSuccess ? test_result.assertionResult() + : test_result.expectFailedTests(expected_failed_tests); + } + + // Execute the tests for the captured set of values, loosely matching a range + // of expected profiles (loose in that an interface is allowed to be more + // refined that a profile suggests, such as a type having a noexcept copy + // constructor when all that is required is that the copy constructor exists). + template <class Prof, absl::enable_if_t<IsProfile<Prof>::value>** = nullptr> + ABSL_MUST_USE_RESULT ::testing::AssertionResult with_loose_profile( + Prof /*profile*/) { + ConformanceErrors test_result = + (ExpectRegularityImpl< + T, Prof, Prof, + CombineProfiles<TriviallyCompleteProfile, + NothrowComparableProfile>>)(absl:: + move(ordered_vals)); + + return ExpectSuccess ? test_result.assertionResult() + : test_result.expectFailedTests(expected_failed_tests); + } + + OrderedEquivalenceClasses<EqClasses...> ordered_vals; + std::set<std::string> expected_failed_tests; +}; + +template <class T> +using ExpectConformanceOfType = ExpectConformanceOf</*ExpectSuccess=*/true, T>; + +template <class T> +using ExpectNonconformanceOfType = + ExpectConformanceOf</*ExpectSuccess=*/false, T>; + +struct EquivalenceClassMaker { + // TODO(calabrese) Constrain to callable + template <class Fun> + static GeneratorType<Fun> initializer(GeneratorType<Fun> fun) { + return fun; + } +}; + +// A top-level macro that begins the builder pattern. +// +// The argument here takes the datatype to be tested. +#define ABSL_INTERNAL_ASSERT_CONFORMANCE_OF(...) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if ABSL_INTERNAL_LPAREN \ + const ::testing::AssertionResult gtest_ar = \ + ABSL_INTERNAL_LPAREN ::absl::types_internal::ExpectConformanceOfType< \ + __VA_ARGS__>() + +// Akin to ASSERT_CONFORMANCE_OF except that it expects failure and tries to +// match text. +#define ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(...) \ + GTEST_AMBIGUOUS_ELSE_BLOCKER_ \ + if ABSL_INTERNAL_LPAREN \ + const ::testing::AssertionResult gtest_ar = \ + ABSL_INTERNAL_LPAREN ::absl::types_internal::ExpectNonconformanceOfType< \ + __VA_ARGS__>() + +//////////////////////////////////////////////////////////////////////////////// +// NOTE: The following macros look like they are recursive, but are not (macros +// cannot recurse). These actually refer to member functions of the same name. +// This is done intentionally so that a user cannot accidentally invoke a +// member function of the conformance-testing suite without going through the +// macro. +//////////////////////////////////////////////////////////////////////////////// + +// Specify expected test failures as comma-separated strings. +#define DUE_TO(...) due_to(__VA_ARGS__) + +// Specify a value to be tested. +// +// Note: Internally, this takes an expression and turns it into the return value +// of lambda that captures no data. The expression is stringized during +// preprocessing so that it can be used in error reports. +#define INITIALIZER(...) \ + initializer(::absl::types_internal::Generator( \ + [] { return __VA_ARGS__; }, ABSL_INTERNAL_STRINGIZE(__VA_ARGS__))) + +// Specify a value to be tested. +// +// Note: Internally, this takes an expression and turns it into the return value +// of lambda that captures data by reference. The expression is stringized +// during preprocessing so that it can be used in error reports. +#define STATEFUL_INITIALIZER(...) \ + stateful_initializer(::absl::types_internal::Generator( \ + [&] { return __VA_ARGS__; }, ABSL_INTERNAL_STRINGIZE(__VA_ARGS__))) + +// Used in the builder-pattern. +// +// Takes a series of INITIALIZER and/or STATEFUL_INITIALIZER invocations and +// forwards them along to be tested, grouping them such that the testing suite +// knows that they are supposed to represent the same logical value (the values +// compare the same, hash the same, etc.). +#define EQUIVALENCE_CLASS(...) \ + equivalence_class(ABSL_INTERNAL_TRANSFORM_ARGS( \ + ABSL_INTERNAL_PREPEND_EQ_MAKER, __VA_ARGS__)) + +// An invocation of this or WITH_STRICT_PROFILE must end the builder-pattern. +// It takes a Profile as its argument. +// +// This executes the tests and allows types that are "more referined" than the +// profile specifies, but not less. For instance, if the Profile specifies +// noexcept copy-constructiblity, the test will fail if the copy-constructor is +// not noexcept, however, it will succeed if the copy constructor is trivial. +// +// This is useful for testing that a type meets some minimum set of +// requirements. +#define WITH_LOOSE_PROFILE(...) \ + with_loose_profile( \ + ::absl::types_internal::MakeLooseProfileRangeT<__VA_ARGS__>()) \ + ABSL_INTERNAL_RPAREN ABSL_INTERNAL_RPAREN; \ + else GTEST_FATAL_FAILURE_(gtest_ar.failure_message()) // NOLINT + +// An invocation of this or WITH_STRICT_PROFILE must end the builder-pattern. +// It takes a Domain and a Profile as its arguments. +// +// This executes the tests and disallows types that differ at all from the +// properties of the Profile. For instance, if the Profile specifies noexcept +// copy-constructiblity, the test will fail if the copy constructor is trivial. +// +// This is useful for testing that a type does not do anything more than a +// specification requires, such as to minimize things like Hyrum's Law, or more +// commonly, to prevent a type from being "accidentally" copy-constructible in +// a way that may produce incorrect results, simply because the user forget to +// delete that operation. +#define WITH_STRICT_PROFILE(...) \ + with_strict_profile( \ + ::absl::types_internal::MakeStrictProfileRangeT<__VA_ARGS__>()) \ + ABSL_INTERNAL_RPAREN ABSL_INTERNAL_RPAREN; \ + else GTEST_FATAL_FAILURE_(gtest_ar.failure_message()) // NOLINT + +// Internal macro that is used in the internals of the EDSL when forming +// equivalence classes. +#define ABSL_INTERNAL_PREPEND_EQ_MAKER(arg) \ + ::absl::types_internal::EquivalenceClassMaker().arg + +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TYPES_INTERNAL_CONFORMANCE_TESTING_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/conformance_testing_helpers.h b/third_party/abseil_cpp/absl/types/internal/conformance_testing_helpers.h new file mode 100644 index 000000000000..00775f960c36 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/conformance_testing_helpers.h @@ -0,0 +1,391 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_TYPES_INTERNAL_CONFORMANCE_TESTING_HELPERS_H_ +#define ABSL_TYPES_INTERNAL_CONFORMANCE_TESTING_HELPERS_H_ + +// Checks to determine whether or not we can use abi::__cxa_demangle +#if (defined(__ANDROID__) || defined(ANDROID)) && !defined(OS_ANDROID) +#define ABSL_INTERNAL_OS_ANDROID +#endif + +// We support certain compilers only. See demangle.h for details. +#if defined(OS_ANDROID) && (defined(__i386__) || defined(__x86_64__)) +#define ABSL_TYPES_INTERNAL_HAS_CXA_DEMANGLE 0 +#elif (__GNUC__ >= 4 || (__GNUC__ >= 3 && __GNUC_MINOR__ >= 4)) && \ + !defined(__mips__) +#define ABSL_TYPES_INTERNAL_HAS_CXA_DEMANGLE 1 +#elif defined(__clang__) && !defined(_MSC_VER) +#define ABSL_TYPES_INTERNAL_HAS_CXA_DEMANGLE 1 +#else +#define ABSL_TYPES_INTERNAL_HAS_CXA_DEMANGLE 0 +#endif + +#include <tuple> +#include <type_traits> +#include <utility> + +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" +#include "absl/utility/utility.h" + +#if ABSL_TYPES_INTERNAL_HAS_CXA_DEMANGLE +#include <cxxabi.h> + +#include <cstdlib> +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace types_internal { + +// Return a readable name for type T. +template <class T> +absl::string_view NameOfImpl() { +// TODO(calabrese) Investigate using debugging:internal_demangle as a fallback. +#if ABSL_TYPES_INTERNAL_HAS_CXA_DEMANGLE + int status = 0; + char* demangled_name = nullptr; + + demangled_name = + abi::__cxa_demangle(typeid(T).name(), nullptr, nullptr, &status); + + if (status == 0 && demangled_name != nullptr) { + return demangled_name; + } else { + return typeid(T).name(); + } +#else + return typeid(T).name(); +#endif + // NOTE: We intentionally leak demangled_name so that it remains valid + // throughout the remainder of the program. +} + +// Given a type, returns as nice of a type name as we can produce (demangled). +// +// Note: This currently strips cv-qualifiers and references, but that is okay +// because we only use this internally with unqualified object types. +template <class T> +std::string NameOf() { + static const absl::string_view result = NameOfImpl<T>(); + return std::string(result); +} + +//////////////////////////////////////////////////////////////////////////////// +// +// Metafunction to check if a type is callable with no explicit arguments +template <class Fun, class /*Enabler*/ = void> +struct IsNullaryCallableImpl : std::false_type {}; + +template <class Fun> +struct IsNullaryCallableImpl< + Fun, absl::void_t<decltype(std::declval<const Fun&>()())>> + : std::true_type { + using result_type = decltype(std::declval<const Fun&>()()); + + template <class ValueType> + using for_type = std::is_same<ValueType, result_type>; + + using void_if_true = void; +}; + +template <class Fun> +struct IsNullaryCallable : IsNullaryCallableImpl<Fun> {}; +// +//////////////////////////////////////////////////////////////////////////////// + +// A type that contains a function object that returns an instance of a type +// that is undergoing conformance testing. This function is required to always +// return the same value upon invocation. +template <class Fun> +struct GeneratorType; + +// A type that contains a tuple of GeneratorType<Fun> where each Fun has the +// same return type. The result of each of the different generators should all +// be equal values, though the underlying object representation may differ (such +// as if one returns 0.0 and another return -0.0, or if one returns an empty +// vector and another returns an empty vector with a different capacity. +template <class... Funs> +struct EquivalenceClassType; + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction to check if a type is a specialization of EquivalenceClassType +template <class T> +struct IsEquivalenceClass : std::false_type {}; + +template <> +struct IsEquivalenceClass<EquivalenceClassType<>> : std::true_type { + using self = IsEquivalenceClass; + + // A metafunction to check if this EquivalenceClassType is a valid + // EquivalenceClassType for a type `ValueType` that is undergoing testing + template <class ValueType> + using for_type = std::true_type; +}; + +template <class Head, class... Tail> +struct IsEquivalenceClass<EquivalenceClassType<Head, Tail...>> + : std::true_type { + using self = IsEquivalenceClass; + + // The type undergoing conformance testing that this EquivalenceClass + // corresponds to + using result_type = typename IsNullaryCallable<Head>::result_type; + + // A metafunction to check if this EquivalenceClassType is a valid + // EquivalenceClassType for a type `ValueType` that is undergoing testing + template <class ValueType> + using for_type = std::is_same<ValueType, result_type>; +}; +// +//////////////////////////////////////////////////////////////////////////////// + +// A type that contains an ordered series of EquivalenceClassTypes, where the +// the function object of each underlying GeneratorType has the same return type +// +// These equivalence classes are required to be in a logical ascending order +// that is consistent with comparison operators that are defined for the return +// type of each GeneratorType, if any. +template <class... EqClasses> +struct OrderedEquivalenceClasses; + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction to determine the return type of the function object contained +// in a GeneratorType specialization. +template <class T> +struct ResultOfGenerator {}; + +template <class Fun> +struct ResultOfGenerator<GeneratorType<Fun>> { + using type = decltype(std::declval<const Fun&>()()); +}; + +template <class Fun> +using ResultOfGeneratorT = typename ResultOfGenerator<GeneratorType<Fun>>::type; +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction that yields true iff each of Funs is a GeneratorType +// specialization and they all contain functions with the same return type +template <class /*Enabler*/, class... Funs> +struct AreGeneratorsWithTheSameReturnTypeImpl : std::false_type {}; + +template <> +struct AreGeneratorsWithTheSameReturnTypeImpl<void> : std::true_type {}; + +template <class Head, class... Tail> +struct AreGeneratorsWithTheSameReturnTypeImpl< + typename std::enable_if<absl::conjunction<std::is_same< + ResultOfGeneratorT<Head>, ResultOfGeneratorT<Tail>>...>::value>::type, + Head, Tail...> : std::true_type {}; + +template <class... Funs> +struct AreGeneratorsWithTheSameReturnType + : AreGeneratorsWithTheSameReturnTypeImpl<void, Funs...>::type {}; +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction that yields true iff each of Funs is an EquivalenceClassType +// specialization and they all contain GeneratorType specializations that have +// the same return type +template <class... EqClasses> +struct AreEquivalenceClassesOfTheSameType { + static_assert(sizeof...(EqClasses) != sizeof...(EqClasses), ""); +}; + +template <> +struct AreEquivalenceClassesOfTheSameType<> : std::true_type { + using self = AreEquivalenceClassesOfTheSameType; + + // Metafunction to check that a type is the same as all of the equivalence + // classes, if any. + // Note: In this specialization there are no equivalence classes, so the + // value type is always compatible. + template <class /*ValueType*/> + using for_type = std::true_type; +}; + +template <class... Funs> +struct AreEquivalenceClassesOfTheSameType<EquivalenceClassType<Funs...>> + : std::true_type { + using self = AreEquivalenceClassesOfTheSameType; + + // Metafunction to check that a type is the same as all of the equivalence + // classes, if any. + template <class ValueType> + using for_type = typename IsEquivalenceClass< + EquivalenceClassType<Funs...>>::template for_type<ValueType>; +}; + +template <class... TailEqClasses> +struct AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<>, EquivalenceClassType<>, TailEqClasses...> + : AreEquivalenceClassesOfTheSameType<TailEqClasses...>::self {}; + +template <class HeadNextFun, class... TailNextFuns, class... TailEqClasses> +struct AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<>, EquivalenceClassType<HeadNextFun, TailNextFuns...>, + TailEqClasses...> + : AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<HeadNextFun, TailNextFuns...>, + TailEqClasses...>::self {}; + +template <class HeadHeadFun, class... TailHeadFuns, class... TailEqClasses> +struct AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<HeadHeadFun, TailHeadFuns...>, EquivalenceClassType<>, + TailEqClasses...> + : AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<HeadHeadFun, TailHeadFuns...>, + TailEqClasses...>::self {}; + +template <class HeadHeadFun, class... TailHeadFuns, class HeadNextFun, + class... TailNextFuns, class... TailEqClasses> +struct AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<HeadHeadFun, TailHeadFuns...>, + EquivalenceClassType<HeadNextFun, TailNextFuns...>, TailEqClasses...> + : absl::conditional_t< + IsNullaryCallable<HeadNextFun>::template for_type< + typename IsNullaryCallable<HeadHeadFun>::result_type>::value, + AreEquivalenceClassesOfTheSameType< + EquivalenceClassType<HeadHeadFun, TailHeadFuns...>, + TailEqClasses...>, + std::false_type> {}; +// +//////////////////////////////////////////////////////////////////////////////// + +// Execute a function for each passed-in parameter. +template <class Fun, class... Cases> +void ForEachParameter(const Fun& fun, const Cases&... cases) { + const std::initializer_list<bool> results = { + (static_cast<void>(fun(cases)), true)...}; + + (void)results; +} + +// Execute a function on each passed-in parameter (using a bound function). +template <class Fun> +struct ForEachParameterFun { + template <class... T> + void operator()(const T&... cases) const { + (ForEachParameter)(fun, cases...); + } + + Fun fun; +}; + +// Execute a function on each element of a tuple. +template <class Fun, class Tup> +void ForEachTupleElement(const Fun& fun, const Tup& tup) { + absl::apply(ForEachParameterFun<Fun>{fun}, tup); +} + +//////////////////////////////////////////////////////////////////////////////// +// +// Execute a function for each combination of two elements of a tuple, including +// combinations of an element with itself. +template <class Fun, class... T> +struct ForEveryTwoImpl { + template <class Lhs> + struct WithBoundLhs { + template <class Rhs> + void operator()(const Rhs& rhs) const { + fun(lhs, rhs); + } + + Fun fun; + Lhs lhs; + }; + + template <class Lhs> + void operator()(const Lhs& lhs) const { + (ForEachTupleElement)(WithBoundLhs<Lhs>{fun, lhs}, args); + } + + Fun fun; + std::tuple<T...> args; +}; + +template <class Fun, class... T> +void ForEveryTwo(const Fun& fun, std::tuple<T...> args) { + (ForEachTupleElement)(ForEveryTwoImpl<Fun, T...>{fun, args}, args); +} +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +// +// Insert all values into an associative container +template<class Container> +void InsertEach(Container* cont) { +} + +template<class Container, class H, class... T> +void InsertEach(Container* cont, H&& head, T&&... tail) { + cont->insert(head); + (InsertEach)(cont, tail...); +} +// +//////////////////////////////////////////////////////////////////////////////// +// A template with a nested "Invoke" static-member-function that executes a +// passed-in Callable when `Condition` is true, otherwise it ignores the +// Callable. This is useful for executing a function object with a condition +// that corresponds to whether or not the Callable can be safely instantiated. +// It has some overlapping uses with C++17 `if constexpr`. +template <bool Condition> +struct If; + +template <> +struct If</*Condition =*/false> { + template <class Fun, class... P> + static void Invoke(const Fun& /*fun*/, P&&... /*args*/) {} +}; + +template <> +struct If</*Condition =*/true> { + template <class Fun, class... P> + static void Invoke(const Fun& fun, P&&... args) { + // TODO(calabrese) Use std::invoke equivalent instead of function-call. + fun(absl::forward<P>(args)...); + } +}; + +// +// ABSL_INTERNAL_STRINGIZE(...) +// +// This variadic macro transforms its arguments into a c-string literal after +// expansion. +// +// Example: +// +// ABSL_INTERNAL_STRINGIZE(std::array<int, 10>) +// +// Results in: +// +// "std::array<int, 10>" +#define ABSL_INTERNAL_STRINGIZE(...) ABSL_INTERNAL_STRINGIZE_IMPL((__VA_ARGS__)) +#define ABSL_INTERNAL_STRINGIZE_IMPL(arg) ABSL_INTERNAL_STRINGIZE_IMPL2 arg +#define ABSL_INTERNAL_STRINGIZE_IMPL2(...) #__VA_ARGS__ + +} // namespace types_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TYPES_INTERNAL_CONFORMANCE_TESTING_HELPERS_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/conformance_testing_test.cc b/third_party/abseil_cpp/absl/types/internal/conformance_testing_test.cc new file mode 100644 index 000000000000..cf262fa6c222 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/conformance_testing_test.cc @@ -0,0 +1,1556 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/internal/conformance_testing.h" + +#include <new> +#include <type_traits> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/meta/type_traits.h" +#include "absl/types/internal/conformance_aliases.h" +#include "absl/types/internal/conformance_profile.h" + +namespace { + +namespace ti = absl::types_internal; + +template <class T> +using DefaultConstructibleWithNewImpl = decltype(::new (std::nothrow) T); + +template <class T> +using DefaultConstructibleWithNew = + absl::type_traits_internal::is_detected<DefaultConstructibleWithNewImpl, T>; + +template <class T> +using MoveConstructibleWithNewImpl = + decltype(::new (std::nothrow) T(std::declval<T>())); + +template <class T> +using MoveConstructibleWithNew = + absl::type_traits_internal::is_detected<MoveConstructibleWithNewImpl, T>; + +template <class T> +using CopyConstructibleWithNewImpl = + decltype(::new (std::nothrow) T(std::declval<const T&>())); + +template <class T> +using CopyConstructibleWithNew = + absl::type_traits_internal::is_detected<CopyConstructibleWithNewImpl, T>; + +template <class T, + class Result = + std::integral_constant<bool, noexcept(::new (std::nothrow) T)>> +using NothrowDefaultConstructibleWithNewImpl = + typename std::enable_if<Result::value>::type; + +template <class T> +using NothrowDefaultConstructibleWithNew = + absl::type_traits_internal::is_detected< + NothrowDefaultConstructibleWithNewImpl, T>; + +template <class T, + class Result = std::integral_constant< + bool, noexcept(::new (std::nothrow) T(std::declval<T>()))>> +using NothrowMoveConstructibleWithNewImpl = + typename std::enable_if<Result::value>::type; + +template <class T> +using NothrowMoveConstructibleWithNew = + absl::type_traits_internal::is_detected<NothrowMoveConstructibleWithNewImpl, + T>; + +template <class T, + class Result = std::integral_constant< + bool, noexcept(::new (std::nothrow) T(std::declval<const T&>()))>> +using NothrowCopyConstructibleWithNewImpl = + typename std::enable_if<Result::value>::type; + +template <class T> +using NothrowCopyConstructibleWithNew = + absl::type_traits_internal::is_detected<NothrowCopyConstructibleWithNewImpl, + T>; + +// NOTE: ?: is used to verify contextually-convertible to bool and not simply +// implicit or explicit convertibility. +#define ABSL_INTERNAL_COMPARISON_OP_EXPR(op) \ + ((std::declval<const T&>() op std::declval<const T&>()) ? true : true) + +#define ABSL_INTERNAL_COMPARISON_OP_TRAIT(name, op) \ + template <class T> \ + using name##Impl = decltype(ABSL_INTERNAL_COMPARISON_OP_EXPR(op)); \ + \ + template <class T> \ + using name = absl::type_traits_internal::is_detected<name##Impl, T>; \ + \ + template <class T, \ + class Result = std::integral_constant< \ + bool, noexcept(ABSL_INTERNAL_COMPARISON_OP_EXPR(op))>> \ + using Nothrow##name##Impl = typename std::enable_if<Result::value>::type; \ + \ + template <class T> \ + using Nothrow##name = \ + absl::type_traits_internal::is_detected<Nothrow##name##Impl, T> + +ABSL_INTERNAL_COMPARISON_OP_TRAIT(EqualityComparable, ==); +ABSL_INTERNAL_COMPARISON_OP_TRAIT(InequalityComparable, !=); +ABSL_INTERNAL_COMPARISON_OP_TRAIT(LessThanComparable, <); +ABSL_INTERNAL_COMPARISON_OP_TRAIT(LessEqualComparable, <=); +ABSL_INTERNAL_COMPARISON_OP_TRAIT(GreaterEqualComparable, >=); +ABSL_INTERNAL_COMPARISON_OP_TRAIT(GreaterThanComparable, >); + +#undef ABSL_INTERNAL_COMPARISON_OP_TRAIT + +template <class T> +class ProfileTest : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(ProfileTest); + +TYPED_TEST_P(ProfileTest, HasAppropriateConstructionProperties) { + using profile = typename TypeParam::profile; + using arch = typename TypeParam::arch; + using expected_profile = typename TypeParam::expected_profile; + + using props = ti::PropertiesOfT<profile>; + using arch_props = ti::PropertiesOfArchetypeT<arch>; + using expected_props = ti::PropertiesOfT<expected_profile>; + + // Make sure all of the properties are as expected. + // There are seemingly redundant tests here to make it easier to diagnose + // the specifics of the failure if something were to go wrong. + EXPECT_TRUE((std::is_same<props, arch_props>::value)); + EXPECT_TRUE((std::is_same<props, expected_props>::value)); + EXPECT_TRUE((std::is_same<arch_props, expected_props>::value)); + + EXPECT_EQ(props::default_constructible_support, + expected_props::default_constructible_support); + + EXPECT_EQ(props::move_constructible_support, + expected_props::move_constructible_support); + + EXPECT_EQ(props::copy_constructible_support, + expected_props::copy_constructible_support); + + EXPECT_EQ(props::destructible_support, expected_props::destructible_support); + + // Avoid additional error message noise when profile and archetype match with + // each other but were not what was expected. + if (!std::is_same<props, arch_props>::value) { + EXPECT_EQ(arch_props::default_constructible_support, + expected_props::default_constructible_support); + + EXPECT_EQ(arch_props::move_constructible_support, + expected_props::move_constructible_support); + + EXPECT_EQ(arch_props::copy_constructible_support, + expected_props::copy_constructible_support); + + EXPECT_EQ(arch_props::destructible_support, + expected_props::destructible_support); + } + + ////////////////////////////////////////////////////////////////////////////// + // Default constructor checks // + ////////////////////////////////////////////////////////////////////////////// + EXPECT_EQ(props::default_constructible_support, + expected_props::default_constructible_support); + + switch (expected_props::default_constructible_support) { + case ti::default_constructible::maybe: + EXPECT_FALSE(DefaultConstructibleWithNew<arch>::value); + EXPECT_FALSE(NothrowDefaultConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_FALSE(std::is_default_constructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_default_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_default_constructible<arch>::value); + } + break; + case ti::default_constructible::yes: + EXPECT_TRUE(DefaultConstructibleWithNew<arch>::value); + EXPECT_FALSE(NothrowDefaultConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_default_constructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_default_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_default_constructible<arch>::value); + } + break; + case ti::default_constructible::nothrow: + EXPECT_TRUE(DefaultConstructibleWithNew<arch>::value); + EXPECT_TRUE(NothrowDefaultConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_default_constructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_default_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_default_constructible<arch>::value); + + // Constructor traits also check the destructor. + if (std::is_nothrow_destructible<arch>::value) { + EXPECT_TRUE(std::is_nothrow_default_constructible<arch>::value); + } + } + break; + case ti::default_constructible::trivial: + EXPECT_TRUE(DefaultConstructibleWithNew<arch>::value); + EXPECT_TRUE(NothrowDefaultConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_default_constructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_default_constructible<arch>::value); + + // Constructor triviality traits require trivially destructible types. + if (absl::is_trivially_destructible<arch>::value) { + EXPECT_TRUE(absl::is_trivially_default_constructible<arch>::value); + } + } + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Move constructor checks // + ////////////////////////////////////////////////////////////////////////////// + EXPECT_EQ(props::move_constructible_support, + expected_props::move_constructible_support); + + switch (expected_props::move_constructible_support) { + case ti::move_constructible::maybe: + EXPECT_FALSE(MoveConstructibleWithNew<arch>::value); + EXPECT_FALSE(NothrowMoveConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_FALSE(std::is_move_constructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_move_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_move_constructible<arch>::value); + } + break; + case ti::move_constructible::yes: + EXPECT_TRUE(MoveConstructibleWithNew<arch>::value); + EXPECT_FALSE(NothrowMoveConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_move_constructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_move_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_move_constructible<arch>::value); + } + break; + case ti::move_constructible::nothrow: + EXPECT_TRUE(MoveConstructibleWithNew<arch>::value); + EXPECT_TRUE(NothrowMoveConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_move_constructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_move_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_move_constructible<arch>::value); + + // Constructor traits also check the destructor. + if (std::is_nothrow_destructible<arch>::value) { + EXPECT_TRUE(std::is_nothrow_move_constructible<arch>::value); + } + } + break; + case ti::move_constructible::trivial: + EXPECT_TRUE(MoveConstructibleWithNew<arch>::value); + EXPECT_TRUE(NothrowMoveConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_move_constructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_move_constructible<arch>::value); + + // Constructor triviality traits require trivially destructible types. + if (absl::is_trivially_destructible<arch>::value) { + EXPECT_TRUE(absl::is_trivially_move_constructible<arch>::value); + } + } + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Copy constructor checks // + ////////////////////////////////////////////////////////////////////////////// + EXPECT_EQ(props::copy_constructible_support, + expected_props::copy_constructible_support); + + switch (expected_props::copy_constructible_support) { + case ti::copy_constructible::maybe: + EXPECT_FALSE(CopyConstructibleWithNew<arch>::value); + EXPECT_FALSE(NothrowCopyConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_FALSE(std::is_copy_constructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_copy_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<arch>::value); + } + break; + case ti::copy_constructible::yes: + EXPECT_TRUE(CopyConstructibleWithNew<arch>::value); + EXPECT_FALSE(NothrowCopyConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_copy_constructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_copy_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<arch>::value); + } + break; + case ti::copy_constructible::nothrow: + EXPECT_TRUE(CopyConstructibleWithNew<arch>::value); + EXPECT_TRUE(NothrowCopyConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_copy_constructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_copy_constructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<arch>::value); + + // Constructor traits also check the destructor. + if (std::is_nothrow_destructible<arch>::value) { + EXPECT_TRUE(std::is_nothrow_copy_constructible<arch>::value); + } + } + break; + case ti::copy_constructible::trivial: + EXPECT_TRUE(CopyConstructibleWithNew<arch>::value); + EXPECT_TRUE(NothrowCopyConstructibleWithNew<arch>::value); + + // Standard constructible traits depend on the destructor. + if (std::is_destructible<arch>::value) { + EXPECT_TRUE(std::is_copy_constructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_copy_constructible<arch>::value); + + // Constructor triviality traits require trivially destructible types. + if (absl::is_trivially_destructible<arch>::value) { + EXPECT_TRUE(absl::is_trivially_copy_constructible<arch>::value); + } + } + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Destructible checks // + ////////////////////////////////////////////////////////////////////////////// + EXPECT_EQ(props::destructible_support, expected_props::destructible_support); + + switch (expected_props::destructible_support) { + case ti::destructible::maybe: + EXPECT_FALSE(std::is_destructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_destructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_destructible<arch>::value); + break; + case ti::destructible::yes: + EXPECT_TRUE(std::is_destructible<arch>::value); + EXPECT_FALSE(std::is_nothrow_destructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_destructible<arch>::value); + break; + case ti::destructible::nothrow: + EXPECT_TRUE(std::is_destructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_destructible<arch>::value); + EXPECT_FALSE(absl::is_trivially_destructible<arch>::value); + break; + case ti::destructible::trivial: + EXPECT_TRUE(std::is_destructible<arch>::value); + EXPECT_TRUE(std::is_nothrow_destructible<arch>::value); + EXPECT_TRUE(absl::is_trivially_destructible<arch>::value); + break; + } +} + +TYPED_TEST_P(ProfileTest, HasAppropriateAssignmentProperties) { + using profile = typename TypeParam::profile; + using arch = typename TypeParam::arch; + using expected_profile = typename TypeParam::expected_profile; + + using props = ti::PropertiesOfT<profile>; + using arch_props = ti::PropertiesOfArchetypeT<arch>; + using expected_props = ti::PropertiesOfT<expected_profile>; + + // Make sure all of the properties are as expected. + // There are seemingly redundant tests here to make it easier to diagnose + // the specifics of the failure if something were to go wrong. + EXPECT_TRUE((std::is_same<props, arch_props>::value)); + EXPECT_TRUE((std::is_same<props, expected_props>::value)); + EXPECT_TRUE((std::is_same<arch_props, expected_props>::value)); + + EXPECT_EQ(props::move_assignable_support, + expected_props::move_assignable_support); + + EXPECT_EQ(props::copy_assignable_support, + expected_props::copy_assignable_support); + + // Avoid additional error message noise when profile and archetype match with + // each other but were not what was expected. + if (!std::is_same<props, arch_props>::value) { + EXPECT_EQ(arch_props::move_assignable_support, + expected_props::move_assignable_support); + + EXPECT_EQ(arch_props::copy_assignable_support, + expected_props::copy_assignable_support); + } + + ////////////////////////////////////////////////////////////////////////////// + // Move assignment checks // + ////////////////////////////////////////////////////////////////////////////// + EXPECT_EQ(props::move_assignable_support, + expected_props::move_assignable_support); + + switch (expected_props::move_assignable_support) { + case ti::move_assignable::maybe: + EXPECT_FALSE(std::is_move_assignable<arch>::value); + EXPECT_FALSE(std::is_nothrow_move_assignable<arch>::value); + EXPECT_FALSE(absl::is_trivially_move_assignable<arch>::value); + break; + case ti::move_assignable::yes: + EXPECT_TRUE(std::is_move_assignable<arch>::value); + EXPECT_FALSE(std::is_nothrow_move_assignable<arch>::value); + EXPECT_FALSE(absl::is_trivially_move_assignable<arch>::value); + break; + case ti::move_assignable::nothrow: + EXPECT_TRUE(std::is_move_assignable<arch>::value); + EXPECT_TRUE(std::is_nothrow_move_assignable<arch>::value); + EXPECT_FALSE(absl::is_trivially_move_assignable<arch>::value); + break; + case ti::move_assignable::trivial: + EXPECT_TRUE(std::is_move_assignable<arch>::value); + EXPECT_TRUE(std::is_nothrow_move_assignable<arch>::value); + EXPECT_TRUE(absl::is_trivially_move_assignable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Copy assignment checks // + ////////////////////////////////////////////////////////////////////////////// + EXPECT_EQ(props::copy_assignable_support, + expected_props::copy_assignable_support); + + switch (expected_props::copy_assignable_support) { + case ti::copy_assignable::maybe: + EXPECT_FALSE(std::is_copy_assignable<arch>::value); + EXPECT_FALSE(std::is_nothrow_copy_assignable<arch>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<arch>::value); + break; + case ti::copy_assignable::yes: + EXPECT_TRUE(std::is_copy_assignable<arch>::value); + EXPECT_FALSE(std::is_nothrow_copy_assignable<arch>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<arch>::value); + break; + case ti::copy_assignable::nothrow: + EXPECT_TRUE(std::is_copy_assignable<arch>::value); + EXPECT_TRUE(std::is_nothrow_copy_assignable<arch>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<arch>::value); + break; + case ti::copy_assignable::trivial: + EXPECT_TRUE(std::is_copy_assignable<arch>::value); + EXPECT_TRUE(std::is_nothrow_copy_assignable<arch>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<arch>::value); + break; + } +} + +TYPED_TEST_P(ProfileTest, HasAppropriateComparisonProperties) { + using profile = typename TypeParam::profile; + using arch = typename TypeParam::arch; + using expected_profile = typename TypeParam::expected_profile; + + using props = ti::PropertiesOfT<profile>; + using arch_props = ti::PropertiesOfArchetypeT<arch>; + using expected_props = ti::PropertiesOfT<expected_profile>; + + // Make sure all of the properties are as expected. + // There are seemingly redundant tests here to make it easier to diagnose + // the specifics of the failure if something were to go wrong. + EXPECT_TRUE((std::is_same<props, arch_props>::value)); + EXPECT_TRUE((std::is_same<props, expected_props>::value)); + EXPECT_TRUE((std::is_same<arch_props, expected_props>::value)); + + EXPECT_EQ(props::equality_comparable_support, + expected_props::equality_comparable_support); + + EXPECT_EQ(props::inequality_comparable_support, + expected_props::inequality_comparable_support); + + EXPECT_EQ(props::less_than_comparable_support, + expected_props::less_than_comparable_support); + + EXPECT_EQ(props::less_equal_comparable_support, + expected_props::less_equal_comparable_support); + + EXPECT_EQ(props::greater_equal_comparable_support, + expected_props::greater_equal_comparable_support); + + EXPECT_EQ(props::greater_than_comparable_support, + expected_props::greater_than_comparable_support); + + // Avoid additional error message noise when profile and archetype match with + // each other but were not what was expected. + if (!std::is_same<props, arch_props>::value) { + EXPECT_EQ(arch_props::equality_comparable_support, + expected_props::equality_comparable_support); + + EXPECT_EQ(arch_props::inequality_comparable_support, + expected_props::inequality_comparable_support); + + EXPECT_EQ(arch_props::less_than_comparable_support, + expected_props::less_than_comparable_support); + + EXPECT_EQ(arch_props::less_equal_comparable_support, + expected_props::less_equal_comparable_support); + + EXPECT_EQ(arch_props::greater_equal_comparable_support, + expected_props::greater_equal_comparable_support); + + EXPECT_EQ(arch_props::greater_than_comparable_support, + expected_props::greater_than_comparable_support); + } + + ////////////////////////////////////////////////////////////////////////////// + // Equality comparable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::equality_comparable_support) { + case ti::equality_comparable::maybe: + EXPECT_FALSE(EqualityComparable<arch>::value); + EXPECT_FALSE(NothrowEqualityComparable<arch>::value); + break; + case ti::equality_comparable::yes: + EXPECT_TRUE(EqualityComparable<arch>::value); + EXPECT_FALSE(NothrowEqualityComparable<arch>::value); + break; + case ti::equality_comparable::nothrow: + EXPECT_TRUE(EqualityComparable<arch>::value); + EXPECT_TRUE(NothrowEqualityComparable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Inequality comparable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::inequality_comparable_support) { + case ti::inequality_comparable::maybe: + EXPECT_FALSE(InequalityComparable<arch>::value); + EXPECT_FALSE(NothrowInequalityComparable<arch>::value); + break; + case ti::inequality_comparable::yes: + EXPECT_TRUE(InequalityComparable<arch>::value); + EXPECT_FALSE(NothrowInequalityComparable<arch>::value); + break; + case ti::inequality_comparable::nothrow: + EXPECT_TRUE(InequalityComparable<arch>::value); + EXPECT_TRUE(NothrowInequalityComparable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Less than comparable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::less_than_comparable_support) { + case ti::less_than_comparable::maybe: + EXPECT_FALSE(LessThanComparable<arch>::value); + EXPECT_FALSE(NothrowLessThanComparable<arch>::value); + break; + case ti::less_than_comparable::yes: + EXPECT_TRUE(LessThanComparable<arch>::value); + EXPECT_FALSE(NothrowLessThanComparable<arch>::value); + break; + case ti::less_than_comparable::nothrow: + EXPECT_TRUE(LessThanComparable<arch>::value); + EXPECT_TRUE(NothrowLessThanComparable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Less equal comparable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::less_equal_comparable_support) { + case ti::less_equal_comparable::maybe: + EXPECT_FALSE(LessEqualComparable<arch>::value); + EXPECT_FALSE(NothrowLessEqualComparable<arch>::value); + break; + case ti::less_equal_comparable::yes: + EXPECT_TRUE(LessEqualComparable<arch>::value); + EXPECT_FALSE(NothrowLessEqualComparable<arch>::value); + break; + case ti::less_equal_comparable::nothrow: + EXPECT_TRUE(LessEqualComparable<arch>::value); + EXPECT_TRUE(NothrowLessEqualComparable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Greater equal comparable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::greater_equal_comparable_support) { + case ti::greater_equal_comparable::maybe: + EXPECT_FALSE(GreaterEqualComparable<arch>::value); + EXPECT_FALSE(NothrowGreaterEqualComparable<arch>::value); + break; + case ti::greater_equal_comparable::yes: + EXPECT_TRUE(GreaterEqualComparable<arch>::value); + EXPECT_FALSE(NothrowGreaterEqualComparable<arch>::value); + break; + case ti::greater_equal_comparable::nothrow: + EXPECT_TRUE(GreaterEqualComparable<arch>::value); + EXPECT_TRUE(NothrowGreaterEqualComparable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Greater than comparable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::greater_than_comparable_support) { + case ti::greater_than_comparable::maybe: + EXPECT_FALSE(GreaterThanComparable<arch>::value); + EXPECT_FALSE(NothrowGreaterThanComparable<arch>::value); + break; + case ti::greater_than_comparable::yes: + EXPECT_TRUE(GreaterThanComparable<arch>::value); + EXPECT_FALSE(NothrowGreaterThanComparable<arch>::value); + break; + case ti::greater_than_comparable::nothrow: + EXPECT_TRUE(GreaterThanComparable<arch>::value); + EXPECT_TRUE(NothrowGreaterThanComparable<arch>::value); + break; + } +} + +TYPED_TEST_P(ProfileTest, HasAppropriateAuxilliaryProperties) { + using profile = typename TypeParam::profile; + using arch = typename TypeParam::arch; + using expected_profile = typename TypeParam::expected_profile; + + using props = ti::PropertiesOfT<profile>; + using arch_props = ti::PropertiesOfArchetypeT<arch>; + using expected_props = ti::PropertiesOfT<expected_profile>; + + // Make sure all of the properties are as expected. + // There are seemingly redundant tests here to make it easier to diagnose + // the specifics of the failure if something were to go wrong. + EXPECT_TRUE((std::is_same<props, arch_props>::value)); + EXPECT_TRUE((std::is_same<props, expected_props>::value)); + EXPECT_TRUE((std::is_same<arch_props, expected_props>::value)); + + EXPECT_EQ(props::swappable_support, expected_props::swappable_support); + + EXPECT_EQ(props::hashable_support, expected_props::hashable_support); + + // Avoid additional error message noise when profile and archetype match with + // each other but were not what was expected. + if (!std::is_same<props, arch_props>::value) { + EXPECT_EQ(arch_props::swappable_support, expected_props::swappable_support); + + EXPECT_EQ(arch_props::hashable_support, expected_props::hashable_support); + } + + ////////////////////////////////////////////////////////////////////////////// + // Swappable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::swappable_support) { + case ti::swappable::maybe: + EXPECT_FALSE(absl::type_traits_internal::IsSwappable<arch>::value); + EXPECT_FALSE(absl::type_traits_internal::IsNothrowSwappable<arch>::value); + break; + case ti::swappable::yes: + EXPECT_TRUE(absl::type_traits_internal::IsSwappable<arch>::value); + EXPECT_FALSE(absl::type_traits_internal::IsNothrowSwappable<arch>::value); + break; + case ti::swappable::nothrow: + EXPECT_TRUE(absl::type_traits_internal::IsSwappable<arch>::value); + EXPECT_TRUE(absl::type_traits_internal::IsNothrowSwappable<arch>::value); + break; + } + + ////////////////////////////////////////////////////////////////////////////// + // Hashable checks // + ////////////////////////////////////////////////////////////////////////////// + switch (expected_props::hashable_support) { + case ti::hashable::maybe: +#if ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + EXPECT_FALSE(absl::type_traits_internal::IsHashable<arch>::value); +#endif // ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + break; + case ti::hashable::yes: + EXPECT_TRUE(absl::type_traits_internal::IsHashable<arch>::value); + break; + } +} + +REGISTER_TYPED_TEST_SUITE_P(ProfileTest, HasAppropriateConstructionProperties, + HasAppropriateAssignmentProperties, + HasAppropriateComparisonProperties, + HasAppropriateAuxilliaryProperties); + +template <class Profile, class Arch, class ExpectedProfile> +struct ProfileAndExpectation { + using profile = Profile; + using arch = Arch; + using expected_profile = ExpectedProfile; +}; + +using CoreProfilesToTest = ::testing::Types< + // The terminating case of combine (all properties are "maybe"). + ProfileAndExpectation<ti::CombineProfiles<>, + ti::Archetype<ti::CombineProfiles<>>, + ti::ConformanceProfile<>>, + + // Core default constructor profiles + ProfileAndExpectation< + ti::HasDefaultConstructorProfile, ti::HasDefaultConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::yes>>, + ProfileAndExpectation< + ti::HasNothrowDefaultConstructorProfile, + ti::HasNothrowDefaultConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::nothrow>>, + ProfileAndExpectation< + ti::HasTrivialDefaultConstructorProfile, + ti::HasTrivialDefaultConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::trivial>>, + + // Core move constructor profiles + ProfileAndExpectation< + ti::HasMoveConstructorProfile, ti::HasMoveConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::maybe, + ti::move_constructible::yes>>, + ProfileAndExpectation< + ti::HasNothrowMoveConstructorProfile, + ti::HasNothrowMoveConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::maybe, + ti::move_constructible::nothrow>>, + ProfileAndExpectation< + ti::HasTrivialMoveConstructorProfile, + ti::HasTrivialMoveConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::maybe, + ti::move_constructible::trivial>>, + + // Core copy constructor profiles + ProfileAndExpectation< + ti::HasCopyConstructorProfile, ti::HasCopyConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::maybe, + ti::move_constructible::maybe, + ti::copy_constructible::yes>>, + ProfileAndExpectation< + ti::HasNothrowCopyConstructorProfile, + ti::HasNothrowCopyConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::maybe, + ti::move_constructible::maybe, + ti::copy_constructible::nothrow>>, + ProfileAndExpectation< + ti::HasTrivialCopyConstructorProfile, + ti::HasTrivialCopyConstructorArchetype, + ti::ConformanceProfile<ti::default_constructible::maybe, + ti::move_constructible::maybe, + ti::copy_constructible::trivial>>, + + // Core move assignment profiles + ProfileAndExpectation< + ti::HasMoveAssignProfile, ti::HasMoveAssignArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::yes>>, + ProfileAndExpectation< + ti::HasNothrowMoveAssignProfile, ti::HasNothrowMoveAssignArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::nothrow>>, + ProfileAndExpectation< + ti::HasTrivialMoveAssignProfile, ti::HasTrivialMoveAssignArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::trivial>>, + + // Core copy assignment profiles + ProfileAndExpectation< + ti::HasCopyAssignProfile, ti::HasCopyAssignArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::yes>>, + ProfileAndExpectation< + ti::HasNothrowCopyAssignProfile, ti::HasNothrowCopyAssignArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::nothrow>>, + ProfileAndExpectation< + ti::HasTrivialCopyAssignProfile, ti::HasTrivialCopyAssignArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::trivial>>, + + // Core destructor profiles + ProfileAndExpectation< + ti::HasDestructorProfile, ti::HasDestructorArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::yes>>, + ProfileAndExpectation< + ti::HasNothrowDestructorProfile, ti::HasNothrowDestructorArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow>>, + ProfileAndExpectation< + ti::HasTrivialDestructorProfile, ti::HasTrivialDestructorArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::trivial>>, + + // Core equality comparable profiles + ProfileAndExpectation< + ti::HasEqualityProfile, ti::HasEqualityArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::yes>>, + ProfileAndExpectation< + ti::HasNothrowEqualityProfile, ti::HasNothrowEqualityArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::nothrow>>, + + // Core inequality comparable profiles + ProfileAndExpectation< + ti::HasInequalityProfile, ti::HasInequalityArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::yes>>, + ProfileAndExpectation< + ti::HasNothrowInequalityProfile, ti::HasNothrowInequalityArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, + ti::inequality_comparable::nothrow>>, + + // Core less than comparable profiles + ProfileAndExpectation< + ti::HasLessThanProfile, ti::HasLessThanArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::yes>>, + ProfileAndExpectation< + ti::HasNothrowLessThanProfile, ti::HasNothrowLessThanArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::nothrow>>, + + // Core less equal comparable profiles + ProfileAndExpectation< + ti::HasLessEqualProfile, ti::HasLessEqualArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::yes>>, + ProfileAndExpectation< + ti::HasNothrowLessEqualProfile, ti::HasNothrowLessEqualArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, + ti::less_equal_comparable::nothrow>>, + + // Core greater equal comparable profiles + ProfileAndExpectation< + ti::HasGreaterEqualProfile, ti::HasGreaterEqualArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::yes>>, + ProfileAndExpectation< + ti::HasNothrowGreaterEqualProfile, ti::HasNothrowGreaterEqualArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::nothrow>>, + + // Core greater than comparable profiles + ProfileAndExpectation< + ti::HasGreaterThanProfile, ti::HasGreaterThanArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::yes>>, + ProfileAndExpectation< + ti::HasNothrowGreaterThanProfile, ti::HasNothrowGreaterThanArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::nothrow>>, + + // Core swappable profiles + ProfileAndExpectation< + ti::HasSwapProfile, ti::HasSwapArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::yes>>, + ProfileAndExpectation< + ti::HasNothrowSwapProfile, ti::HasNothrowSwapArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::nothrow>>, + + // Core hashable profiles + ProfileAndExpectation< + ti::HasStdHashSpecializationProfile, + ti::HasStdHashSpecializationArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::maybe, + ti::hashable::yes>>>; + +using CommonProfilesToTest = ::testing::Types< + // NothrowMoveConstructible + ProfileAndExpectation< + ti::NothrowMoveConstructibleProfile, + ti::NothrowMoveConstructibleArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow>>, + + // CopyConstructible + ProfileAndExpectation< + ti::CopyConstructibleProfile, ti::CopyConstructibleArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::yes, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow>>, + + // NothrowMovable + ProfileAndExpectation< + ti::NothrowMovableProfile, ti::NothrowMovableArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::maybe, ti::move_assignable::nothrow, + ti::copy_assignable::maybe, ti::destructible::nothrow, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::nothrow>>, + + // Value + ProfileAndExpectation< + ti::ValueProfile, ti::ValueArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::yes, ti::move_assignable::nothrow, + ti::copy_assignable::yes, ti::destructible::nothrow, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::nothrow>>, + + //////////////////////////////////////////////////////////////////////////// + // Common but also DefaultConstructible // + //////////////////////////////////////////////////////////////////////////// + + // DefaultConstructibleNothrowMoveConstructible + ProfileAndExpectation< + ti::DefaultConstructibleNothrowMoveConstructibleProfile, + ti::DefaultConstructibleNothrowMoveConstructibleArchetype, + ti::ConformanceProfile< + ti::default_constructible::yes, ti::move_constructible::nothrow, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow>>, + + // DefaultConstructibleCopyConstructible + ProfileAndExpectation< + ti::DefaultConstructibleCopyConstructibleProfile, + ti::DefaultConstructibleCopyConstructibleArchetype, + ti::ConformanceProfile< + ti::default_constructible::yes, ti::move_constructible::nothrow, + ti::copy_constructible::yes, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow>>, + + // DefaultConstructibleNothrowMovable + ProfileAndExpectation< + ti::DefaultConstructibleNothrowMovableProfile, + ti::DefaultConstructibleNothrowMovableArchetype, + ti::ConformanceProfile< + ti::default_constructible::yes, ti::move_constructible::nothrow, + ti::copy_constructible::maybe, ti::move_assignable::nothrow, + ti::copy_assignable::maybe, ti::destructible::nothrow, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::nothrow>>, + + // DefaultConstructibleValue + ProfileAndExpectation< + ti::DefaultConstructibleValueProfile, + ti::DefaultConstructibleValueArchetype, + ti::ConformanceProfile< + ti::default_constructible::yes, ti::move_constructible::nothrow, + ti::copy_constructible::yes, ti::move_assignable::nothrow, + ti::copy_assignable::yes, ti::destructible::nothrow, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::nothrow>>>; + +using ComparableHelpersProfilesToTest = ::testing::Types< + // Equatable + ProfileAndExpectation< + ti::EquatableProfile, ti::EquatableArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::yes, ti::inequality_comparable::yes>>, + + // Comparable + ProfileAndExpectation< + ti::ComparableProfile, ti::ComparableArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::yes, ti::inequality_comparable::yes, + ti::less_than_comparable::yes, ti::less_equal_comparable::yes, + ti::greater_equal_comparable::yes, + ti::greater_than_comparable::yes>>, + + // NothrowEquatable + ProfileAndExpectation< + ti::NothrowEquatableProfile, ti::NothrowEquatableArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::nothrow, + ti::inequality_comparable::nothrow>>, + + // NothrowComparable + ProfileAndExpectation< + ti::NothrowComparableProfile, ti::NothrowComparableArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::maybe, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::maybe, + ti::equality_comparable::nothrow, + ti::inequality_comparable::nothrow, + ti::less_than_comparable::nothrow, + ti::less_equal_comparable::nothrow, + ti::greater_equal_comparable::nothrow, + ti::greater_than_comparable::nothrow>>>; + +using CommonComparableProfilesToTest = ::testing::Types< + // ComparableNothrowMoveConstructible + ProfileAndExpectation< + ti::ComparableNothrowMoveConstructibleProfile, + ti::ComparableNothrowMoveConstructibleArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::maybe, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow, + ti::equality_comparable::yes, ti::inequality_comparable::yes, + ti::less_than_comparable::yes, ti::less_equal_comparable::yes, + ti::greater_equal_comparable::yes, + ti::greater_than_comparable::yes>>, + + // ComparableCopyConstructible + ProfileAndExpectation< + ti::ComparableCopyConstructibleProfile, + ti::ComparableCopyConstructibleArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::yes, ti::move_assignable::maybe, + ti::copy_assignable::maybe, ti::destructible::nothrow, + ti::equality_comparable::yes, ti::inequality_comparable::yes, + ti::less_than_comparable::yes, ti::less_equal_comparable::yes, + ti::greater_equal_comparable::yes, + ti::greater_than_comparable::yes>>, + + // ComparableNothrowMovable + ProfileAndExpectation< + ti::ComparableNothrowMovableProfile, + ti::ComparableNothrowMovableArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::maybe, ti::move_assignable::nothrow, + ti::copy_assignable::maybe, ti::destructible::nothrow, + ti::equality_comparable::yes, ti::inequality_comparable::yes, + ti::less_than_comparable::yes, ti::less_equal_comparable::yes, + ti::greater_equal_comparable::yes, ti::greater_than_comparable::yes, + ti::swappable::nothrow>>, + + // ComparableValue + ProfileAndExpectation< + ti::ComparableValueProfile, ti::ComparableValueArchetype, + ti::ConformanceProfile< + ti::default_constructible::maybe, ti::move_constructible::nothrow, + ti::copy_constructible::yes, ti::move_assignable::nothrow, + ti::copy_assignable::yes, ti::destructible::nothrow, + ti::equality_comparable::yes, ti::inequality_comparable::yes, + ti::less_than_comparable::yes, ti::less_equal_comparable::yes, + ti::greater_equal_comparable::yes, ti::greater_than_comparable::yes, + ti::swappable::nothrow>>>; + +using TrivialProfilesToTest = ::testing::Types< + ProfileAndExpectation< + ti::TrivialSpecialMemberFunctionsProfile, + ti::TrivialSpecialMemberFunctionsArchetype, + ti::ConformanceProfile< + ti::default_constructible::trivial, ti::move_constructible::trivial, + ti::copy_constructible::trivial, ti::move_assignable::trivial, + ti::copy_assignable::trivial, ti::destructible::trivial, + ti::equality_comparable::maybe, ti::inequality_comparable::maybe, + ti::less_than_comparable::maybe, ti::less_equal_comparable::maybe, + ti::greater_equal_comparable::maybe, + ti::greater_than_comparable::maybe, ti::swappable::nothrow>>, + + ProfileAndExpectation< + ti::TriviallyCompleteProfile, ti::TriviallyCompleteArchetype, + ti::ConformanceProfile< + ti::default_constructible::trivial, ti::move_constructible::trivial, + ti::copy_constructible::trivial, ti::move_assignable::trivial, + ti::copy_assignable::trivial, ti::destructible::trivial, + ti::equality_comparable::yes, ti::inequality_comparable::yes, + ti::less_than_comparable::yes, ti::less_equal_comparable::yes, + ti::greater_equal_comparable::yes, ti::greater_than_comparable::yes, + ti::swappable::nothrow, ti::hashable::yes>>>; + +INSTANTIATE_TYPED_TEST_SUITE_P(Core, ProfileTest, CoreProfilesToTest); +INSTANTIATE_TYPED_TEST_SUITE_P(Common, ProfileTest, CommonProfilesToTest); +INSTANTIATE_TYPED_TEST_SUITE_P(ComparableHelpers, ProfileTest, + ComparableHelpersProfilesToTest); +INSTANTIATE_TYPED_TEST_SUITE_P(CommonComparable, ProfileTest, + CommonComparableProfilesToTest); +INSTANTIATE_TYPED_TEST_SUITE_P(Trivial, ProfileTest, TrivialProfilesToTest); + +TEST(ConformanceTestingTest, Basic) { + using profile = ti::CombineProfiles<ti::TriviallyCompleteProfile, + ti::NothrowComparableProfile>; + + using lim = std::numeric_limits<float>; + + ABSL_INTERNAL_ASSERT_CONFORMANCE_OF(float) + .INITIALIZER(-lim::infinity()) + .INITIALIZER(lim::lowest()) + .INITIALIZER(-1.f) + .INITIALIZER(-lim::min()) + .EQUIVALENCE_CLASS(INITIALIZER(-0.f), INITIALIZER(0.f)) + .INITIALIZER(lim::min()) + .INITIALIZER(1.f) + .INITIALIZER(lim::max()) + .INITIALIZER(lim::infinity()) + .WITH_STRICT_PROFILE(absl::types_internal::RegularityDomain, profile); +} + +struct BadMoveConstruct { + BadMoveConstruct() = default; + BadMoveConstruct(BadMoveConstruct&& other) noexcept + : value(other.value + 1) {} + BadMoveConstruct& operator=(BadMoveConstruct&& other) noexcept = default; + int value = 0; + + friend bool operator==(BadMoveConstruct const& lhs, + BadMoveConstruct const& rhs) { + return lhs.value == rhs.value; + } + friend bool operator!=(BadMoveConstruct const& lhs, + BadMoveConstruct const& rhs) { + return lhs.value != rhs.value; + } +}; + +struct BadMoveAssign { + BadMoveAssign() = default; + BadMoveAssign(BadMoveAssign&& other) noexcept = default; + BadMoveAssign& operator=(BadMoveAssign&& other) noexcept { + int new_value = other.value + 1; + value = new_value; + return *this; + } + int value = 0; + + friend bool operator==(BadMoveAssign const& lhs, BadMoveAssign const& rhs) { + return lhs.value == rhs.value; + } + friend bool operator!=(BadMoveAssign const& lhs, BadMoveAssign const& rhs) { + return lhs.value != rhs.value; + } +}; + +enum class WhichCompIsBad { eq, ne, lt, le, ge, gt }; + +template <WhichCompIsBad Which> +struct BadCompare { + int value; + + friend bool operator==(BadCompare const& lhs, BadCompare const& rhs) { + return Which == WhichCompIsBad::eq ? lhs.value != rhs.value + : lhs.value == rhs.value; + } + + friend bool operator!=(BadCompare const& lhs, BadCompare const& rhs) { + return Which == WhichCompIsBad::ne ? lhs.value == rhs.value + : lhs.value != rhs.value; + } + + friend bool operator<(BadCompare const& lhs, BadCompare const& rhs) { + return Which == WhichCompIsBad::lt ? lhs.value >= rhs.value + : lhs.value < rhs.value; + } + + friend bool operator<=(BadCompare const& lhs, BadCompare const& rhs) { + return Which == WhichCompIsBad::le ? lhs.value > rhs.value + : lhs.value <= rhs.value; + } + + friend bool operator>=(BadCompare const& lhs, BadCompare const& rhs) { + return Which == WhichCompIsBad::ge ? lhs.value < rhs.value + : lhs.value >= rhs.value; + } + + friend bool operator>(BadCompare const& lhs, BadCompare const& rhs) { + return Which == WhichCompIsBad::gt ? lhs.value <= rhs.value + : lhs.value > rhs.value; + } +}; + +TEST(ConformanceTestingDeathTest, Failures) { + { + using profile = ti::CombineProfiles<ti::TriviallyCompleteProfile, + ti::NothrowComparableProfile>; + + // Note: The initializers are intentionally in the wrong order. + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(float) + .INITIALIZER(1.f) + .INITIALIZER(0.f) + .WITH_LOOSE_PROFILE(profile); + } + + { + using profile = + ti::CombineProfiles<ti::NothrowMovableProfile, ti::EquatableProfile>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadMoveConstruct) + .DUE_TO("Move construction") + .INITIALIZER(BadMoveConstruct()) + .WITH_LOOSE_PROFILE(profile); + } + + { + using profile = + ti::CombineProfiles<ti::NothrowMovableProfile, ti::EquatableProfile>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadMoveAssign) + .DUE_TO("Move assignment") + .INITIALIZER(BadMoveAssign()) + .WITH_LOOSE_PROFILE(profile); + } +} + +TEST(ConformanceTestingDeathTest, CompFailures) { + using profile = ti::ComparableProfile; + + { + using BadComp = BadCompare<WhichCompIsBad::eq>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadComp) + .DUE_TO("Comparison") + .INITIALIZER(BadComp{0}) + .INITIALIZER(BadComp{1}) + .WITH_LOOSE_PROFILE(profile); + } + + { + using BadComp = BadCompare<WhichCompIsBad::ne>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadComp) + .DUE_TO("Comparison") + .INITIALIZER(BadComp{0}) + .INITIALIZER(BadComp{1}) + .WITH_LOOSE_PROFILE(profile); + } + + { + using BadComp = BadCompare<WhichCompIsBad::lt>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadComp) + .DUE_TO("Comparison") + .INITIALIZER(BadComp{0}) + .INITIALIZER(BadComp{1}) + .WITH_LOOSE_PROFILE(profile); + } + + { + using BadComp = BadCompare<WhichCompIsBad::le>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadComp) + .DUE_TO("Comparison") + .INITIALIZER(BadComp{0}) + .INITIALIZER(BadComp{1}) + .WITH_LOOSE_PROFILE(profile); + } + + { + using BadComp = BadCompare<WhichCompIsBad::ge>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadComp) + .DUE_TO("Comparison") + .INITIALIZER(BadComp{0}) + .INITIALIZER(BadComp{1}) + .WITH_LOOSE_PROFILE(profile); + } + + { + using BadComp = BadCompare<WhichCompIsBad::gt>; + + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadComp) + .DUE_TO("Comparison") + .INITIALIZER(BadComp{0}) + .INITIALIZER(BadComp{1}) + .WITH_LOOSE_PROFILE(profile); + } +} + +struct BadSelfMove { + BadSelfMove() = default; + BadSelfMove(BadSelfMove&&) = default; + BadSelfMove& operator=(BadSelfMove&& other) noexcept { + if (this == &other) { + broken_state = true; + } + return *this; + } + + friend bool operator==(const BadSelfMove& lhs, const BadSelfMove& rhs) { + return !(lhs.broken_state || rhs.broken_state); + } + + friend bool operator!=(const BadSelfMove& lhs, const BadSelfMove& rhs) { + return lhs.broken_state || rhs.broken_state; + } + + bool broken_state = false; +}; + +TEST(ConformanceTestingDeathTest, SelfMoveFailure) { + using profile = ti::EquatableNothrowMovableProfile; + + { + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadSelfMove) + .DUE_TO("Move assignment") + .INITIALIZER(BadSelfMove()) + .WITH_LOOSE_PROFILE(profile); + } +} + +struct BadSelfCopy { + BadSelfCopy() = default; + BadSelfCopy(BadSelfCopy&&) = default; + BadSelfCopy(const BadSelfCopy&) = default; + BadSelfCopy& operator=(BadSelfCopy&&) = default; + BadSelfCopy& operator=(BadSelfCopy const& other) { + if (this == &other) { + broken_state = true; + } + return *this; + } + + friend bool operator==(const BadSelfCopy& lhs, const BadSelfCopy& rhs) { + return !(lhs.broken_state || rhs.broken_state); + } + + friend bool operator!=(const BadSelfCopy& lhs, const BadSelfCopy& rhs) { + return lhs.broken_state || rhs.broken_state; + } + + bool broken_state = false; +}; + +TEST(ConformanceTestingDeathTest, SelfCopyFailure) { + using profile = ti::EquatableValueProfile; + + { + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadSelfCopy) + .DUE_TO("Copy assignment") + .INITIALIZER(BadSelfCopy()) + .WITH_LOOSE_PROFILE(profile); + } +} + +struct BadSelfSwap { + friend void swap(BadSelfSwap& lhs, BadSelfSwap& rhs) noexcept { + if (&lhs == &rhs) lhs.broken_state = true; + } + + friend bool operator==(const BadSelfSwap& lhs, const BadSelfSwap& rhs) { + return !(lhs.broken_state || rhs.broken_state); + } + + friend bool operator!=(const BadSelfSwap& lhs, const BadSelfSwap& rhs) { + return lhs.broken_state || rhs.broken_state; + } + + bool broken_state = false; +}; + +TEST(ConformanceTestingDeathTest, SelfSwapFailure) { + using profile = ti::EquatableNothrowMovableProfile; + + { + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadSelfSwap) + .DUE_TO("Swap") + .INITIALIZER(BadSelfSwap()) + .WITH_LOOSE_PROFILE(profile); + } +} + +struct BadDefaultInitializedMoveAssign { + BadDefaultInitializedMoveAssign() : default_initialized(true) {} + explicit BadDefaultInitializedMoveAssign(int v) : value(v) {} + BadDefaultInitializedMoveAssign( + BadDefaultInitializedMoveAssign&& other) noexcept + : value(other.value) {} + BadDefaultInitializedMoveAssign& operator=( + BadDefaultInitializedMoveAssign&& other) noexcept { + value = other.value; + if (default_initialized) ++value; // Bad move if lhs is default initialized + return *this; + } + + friend bool operator==(const BadDefaultInitializedMoveAssign& lhs, + const BadDefaultInitializedMoveAssign& rhs) { + return lhs.value == rhs.value; + } + + friend bool operator!=(const BadDefaultInitializedMoveAssign& lhs, + const BadDefaultInitializedMoveAssign& rhs) { + return lhs.value != rhs.value; + } + + bool default_initialized = false; + int value = 0; +}; + +TEST(ConformanceTestingDeathTest, DefaultInitializedMoveAssignFailure) { + using profile = + ti::CombineProfiles<ti::DefaultConstructibleNothrowMovableProfile, + ti::EquatableProfile>; + + { + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadDefaultInitializedMoveAssign) + .DUE_TO("move assignment") + .INITIALIZER(BadDefaultInitializedMoveAssign(0)) + .WITH_LOOSE_PROFILE(profile); + } +} + +struct BadDefaultInitializedCopyAssign { + BadDefaultInitializedCopyAssign() : default_initialized(true) {} + explicit BadDefaultInitializedCopyAssign(int v) : value(v) {} + BadDefaultInitializedCopyAssign( + BadDefaultInitializedCopyAssign&& other) noexcept + : value(other.value) {} + BadDefaultInitializedCopyAssign(const BadDefaultInitializedCopyAssign& other) + : value(other.value) {} + + BadDefaultInitializedCopyAssign& operator=( + BadDefaultInitializedCopyAssign&& other) noexcept { + value = other.value; + return *this; + } + + BadDefaultInitializedCopyAssign& operator=( + const BadDefaultInitializedCopyAssign& other) { + value = other.value; + if (default_initialized) ++value; // Bad move if lhs is default initialized + return *this; + } + + friend bool operator==(const BadDefaultInitializedCopyAssign& lhs, + const BadDefaultInitializedCopyAssign& rhs) { + return lhs.value == rhs.value; + } + + friend bool operator!=(const BadDefaultInitializedCopyAssign& lhs, + const BadDefaultInitializedCopyAssign& rhs) { + return lhs.value != rhs.value; + } + + bool default_initialized = false; + int value = 0; +}; + +TEST(ConformanceTestingDeathTest, DefaultInitializedAssignFailure) { + using profile = ti::CombineProfiles<ti::DefaultConstructibleValueProfile, + ti::EquatableProfile>; + + { + ABSL_INTERNAL_ASSERT_NONCONFORMANCE_OF(BadDefaultInitializedCopyAssign) + .DUE_TO("copy assignment") + .INITIALIZER(BadDefaultInitializedCopyAssign(0)) + .WITH_LOOSE_PROFILE(profile); + } +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/types/internal/optional.h b/third_party/abseil_cpp/absl/types/internal/optional.h new file mode 100644 index 000000000000..92932b6001b3 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/optional.h @@ -0,0 +1,396 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef ABSL_TYPES_INTERNAL_OPTIONAL_H_ +#define ABSL_TYPES_INTERNAL_OPTIONAL_H_ + +#include <functional> +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/inline_variable.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +// ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS +// +// Inheriting constructors is supported in GCC 4.8+, Clang 3.3+ and MSVC 2015. +// __cpp_inheriting_constructors is a predefined macro and a recommended way to +// check for this language feature, but GCC doesn't support it until 5.0 and +// Clang doesn't support it until 3.6. +// Also, MSVC 2015 has a bug: it doesn't inherit the constexpr template +// constructor. For example, the following code won't work on MSVC 2015 Update3: +// struct Base { +// int t; +// template <typename T> +// constexpr Base(T t_) : t(t_) {} +// }; +// struct Foo : Base { +// using Base::Base; +// } +// constexpr Foo foo(0); // doesn't work on MSVC 2015 +#if defined(__clang__) +#if __has_feature(cxx_inheriting_constructors) +#define ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 +#endif +#elif (defined(__GNUC__) && \ + (__GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ >= 8)) || \ + (__cpp_inheriting_constructors >= 200802) || \ + (defined(_MSC_VER) && _MSC_VER >= 1910) +#define ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS 1 +#endif + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Forward declaration +template <typename T> +class optional; + +namespace optional_internal { + +// This tag type is used as a constructor parameter type for `nullopt_t`. +struct init_t { + explicit init_t() = default; +}; + +struct empty_struct {}; + +// This class stores the data in optional<T>. +// It is specialized based on whether T is trivially destructible. +// This is the specialization for non trivially destructible type. +template <typename T, bool unused = std::is_trivially_destructible<T>::value> +class optional_data_dtor_base { + struct dummy_type { + static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); + // Use an array to avoid GCC 6 placement-new warning. + empty_struct data[sizeof(T) / sizeof(empty_struct)]; + }; + + protected: + // Whether there is data or not. + bool engaged_; + // Data storage + union { + T data_; + dummy_type dummy_; + }; + + void destruct() noexcept { + if (engaged_) { + data_.~T(); + engaged_ = false; + } + } + + // dummy_ must be initialized for constexpr constructor. + constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} + + template <typename... Args> + constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) + : engaged_(true), data_(absl::forward<Args>(args)...) {} + + ~optional_data_dtor_base() { destruct(); } +}; + +// Specialization for trivially destructible type. +template <typename T> +class optional_data_dtor_base<T, true> { + struct dummy_type { + static_assert(sizeof(T) % sizeof(empty_struct) == 0, ""); + // Use array to avoid GCC 6 placement-new warning. + empty_struct data[sizeof(T) / sizeof(empty_struct)]; + }; + + protected: + // Whether there is data or not. + bool engaged_; + // Data storage + union { + T data_; + dummy_type dummy_; + }; + void destruct() noexcept { engaged_ = false; } + + // dummy_ must be initialized for constexpr constructor. + constexpr optional_data_dtor_base() noexcept : engaged_(false), dummy_{{}} {} + + template <typename... Args> + constexpr explicit optional_data_dtor_base(in_place_t, Args&&... args) + : engaged_(true), data_(absl::forward<Args>(args)...) {} +}; + +template <typename T> +class optional_data_base : public optional_data_dtor_base<T> { + protected: + using base = optional_data_dtor_base<T>; +#ifdef ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS + using base::base; +#else + optional_data_base() = default; + + template <typename... Args> + constexpr explicit optional_data_base(in_place_t t, Args&&... args) + : base(t, absl::forward<Args>(args)...) {} +#endif + + template <typename... Args> + void construct(Args&&... args) { + // Use dummy_'s address to work around casting cv-qualified T* to void*. + ::new (static_cast<void*>(&this->dummy_)) T(std::forward<Args>(args)...); + this->engaged_ = true; + } + + template <typename U> + void assign(U&& u) { + if (this->engaged_) { + this->data_ = std::forward<U>(u); + } else { + construct(std::forward<U>(u)); + } + } +}; + +// TODO(absl-team): Add another class using +// std::is_trivially_move_constructible trait when available to match +// http://cplusplus.github.io/LWG/lwg-defects.html#2900, for types that +// have trivial move but nontrivial copy. +// Also, we should be checking is_trivially_copyable here, which is not +// supported now, so we use is_trivially_* traits instead. +template <typename T, + bool unused = absl::is_trivially_copy_constructible<T>::value&& + absl::is_trivially_copy_assignable<typename std::remove_cv< + T>::type>::value&& std::is_trivially_destructible<T>::value> +class optional_data; + +// Trivially copyable types +template <typename T> +class optional_data<T, true> : public optional_data_base<T> { + protected: +#ifdef ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS + using optional_data_base<T>::optional_data_base; +#else + optional_data() = default; + + template <typename... Args> + constexpr explicit optional_data(in_place_t t, Args&&... args) + : optional_data_base<T>(t, absl::forward<Args>(args)...) {} +#endif +}; + +template <typename T> +class optional_data<T, false> : public optional_data_base<T> { + protected: +#ifdef ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS + using optional_data_base<T>::optional_data_base; +#else + template <typename... Args> + constexpr explicit optional_data(in_place_t t, Args&&... args) + : optional_data_base<T>(t, absl::forward<Args>(args)...) {} +#endif + + optional_data() = default; + + optional_data(const optional_data& rhs) : optional_data_base<T>() { + if (rhs.engaged_) { + this->construct(rhs.data_); + } + } + + optional_data(optional_data&& rhs) noexcept( + absl::default_allocator_is_nothrow::value || + std::is_nothrow_move_constructible<T>::value) + : optional_data_base<T>() { + if (rhs.engaged_) { + this->construct(std::move(rhs.data_)); + } + } + + optional_data& operator=(const optional_data& rhs) { + if (rhs.engaged_) { + this->assign(rhs.data_); + } else { + this->destruct(); + } + return *this; + } + + optional_data& operator=(optional_data&& rhs) noexcept( + std::is_nothrow_move_assignable<T>::value&& + std::is_nothrow_move_constructible<T>::value) { + if (rhs.engaged_) { + this->assign(std::move(rhs.data_)); + } else { + this->destruct(); + } + return *this; + } +}; + +// Ordered by level of restriction, from low to high. +// Copyable implies movable. +enum class copy_traits { copyable = 0, movable = 1, non_movable = 2 }; + +// Base class for enabling/disabling copy/move constructor. +template <copy_traits> +class optional_ctor_base; + +template <> +class optional_ctor_base<copy_traits::copyable> { + public: + constexpr optional_ctor_base() = default; + optional_ctor_base(const optional_ctor_base&) = default; + optional_ctor_base(optional_ctor_base&&) = default; + optional_ctor_base& operator=(const optional_ctor_base&) = default; + optional_ctor_base& operator=(optional_ctor_base&&) = default; +}; + +template <> +class optional_ctor_base<copy_traits::movable> { + public: + constexpr optional_ctor_base() = default; + optional_ctor_base(const optional_ctor_base&) = delete; + optional_ctor_base(optional_ctor_base&&) = default; + optional_ctor_base& operator=(const optional_ctor_base&) = default; + optional_ctor_base& operator=(optional_ctor_base&&) = default; +}; + +template <> +class optional_ctor_base<copy_traits::non_movable> { + public: + constexpr optional_ctor_base() = default; + optional_ctor_base(const optional_ctor_base&) = delete; + optional_ctor_base(optional_ctor_base&&) = delete; + optional_ctor_base& operator=(const optional_ctor_base&) = default; + optional_ctor_base& operator=(optional_ctor_base&&) = default; +}; + +// Base class for enabling/disabling copy/move assignment. +template <copy_traits> +class optional_assign_base; + +template <> +class optional_assign_base<copy_traits::copyable> { + public: + constexpr optional_assign_base() = default; + optional_assign_base(const optional_assign_base&) = default; + optional_assign_base(optional_assign_base&&) = default; + optional_assign_base& operator=(const optional_assign_base&) = default; + optional_assign_base& operator=(optional_assign_base&&) = default; +}; + +template <> +class optional_assign_base<copy_traits::movable> { + public: + constexpr optional_assign_base() = default; + optional_assign_base(const optional_assign_base&) = default; + optional_assign_base(optional_assign_base&&) = default; + optional_assign_base& operator=(const optional_assign_base&) = delete; + optional_assign_base& operator=(optional_assign_base&&) = default; +}; + +template <> +class optional_assign_base<copy_traits::non_movable> { + public: + constexpr optional_assign_base() = default; + optional_assign_base(const optional_assign_base&) = default; + optional_assign_base(optional_assign_base&&) = default; + optional_assign_base& operator=(const optional_assign_base&) = delete; + optional_assign_base& operator=(optional_assign_base&&) = delete; +}; + +template <typename T> +struct ctor_copy_traits { + static constexpr copy_traits traits = + std::is_copy_constructible<T>::value + ? copy_traits::copyable + : std::is_move_constructible<T>::value ? copy_traits::movable + : copy_traits::non_movable; +}; + +template <typename T> +struct assign_copy_traits { + static constexpr copy_traits traits = + absl::is_copy_assignable<T>::value && std::is_copy_constructible<T>::value + ? copy_traits::copyable + : absl::is_move_assignable<T>::value && + std::is_move_constructible<T>::value + ? copy_traits::movable + : copy_traits::non_movable; +}; + +// Whether T is constructible or convertible from optional<U>. +template <typename T, typename U> +struct is_constructible_convertible_from_optional + : std::integral_constant< + bool, std::is_constructible<T, optional<U>&>::value || + std::is_constructible<T, optional<U>&&>::value || + std::is_constructible<T, const optional<U>&>::value || + std::is_constructible<T, const optional<U>&&>::value || + std::is_convertible<optional<U>&, T>::value || + std::is_convertible<optional<U>&&, T>::value || + std::is_convertible<const optional<U>&, T>::value || + std::is_convertible<const optional<U>&&, T>::value> {}; + +// Whether T is constructible or convertible or assignable from optional<U>. +template <typename T, typename U> +struct is_constructible_convertible_assignable_from_optional + : std::integral_constant< + bool, is_constructible_convertible_from_optional<T, U>::value || + std::is_assignable<T&, optional<U>&>::value || + std::is_assignable<T&, optional<U>&&>::value || + std::is_assignable<T&, const optional<U>&>::value || + std::is_assignable<T&, const optional<U>&&>::value> {}; + +// Helper function used by [optional.relops], [optional.comp_with_t], +// for checking whether an expression is convertible to bool. +bool convertible_to_bool(bool); + +// Base class for std::hash<absl::optional<T>>: +// If std::hash<std::remove_const_t<T>> is enabled, it provides operator() to +// compute the hash; Otherwise, it is disabled. +// Reference N4659 23.14.15 [unord.hash]. +template <typename T, typename = size_t> +struct optional_hash_base { + optional_hash_base() = delete; + optional_hash_base(const optional_hash_base&) = delete; + optional_hash_base(optional_hash_base&&) = delete; + optional_hash_base& operator=(const optional_hash_base&) = delete; + optional_hash_base& operator=(optional_hash_base&&) = delete; +}; + +template <typename T> +struct optional_hash_base<T, decltype(std::hash<absl::remove_const_t<T> >()( + std::declval<absl::remove_const_t<T> >()))> { + using argument_type = absl::optional<T>; + using result_type = size_t; + size_t operator()(const absl::optional<T>& opt) const { + absl::type_traits_internal::AssertHashEnabled<absl::remove_const_t<T>>(); + if (opt) { + return std::hash<absl::remove_const_t<T> >()(*opt); + } else { + return static_cast<size_t>(0x297814aaad196e6dULL); + } + } +}; + +} // namespace optional_internal +ABSL_NAMESPACE_END +} // namespace absl + +#undef ABSL_OPTIONAL_USE_INHERITING_CONSTRUCTORS + +#endif // ABSL_TYPES_INTERNAL_OPTIONAL_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/parentheses.h b/third_party/abseil_cpp/absl/types/internal/parentheses.h new file mode 100644 index 000000000000..5aebee8fdeb9 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/parentheses.h @@ -0,0 +1,34 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// parentheses.h +// ----------------------------------------------------------------------------- +// +// This file contains macros that expand to a left parenthesis and a right +// parenthesis. These are in their own file and are generated from macros +// because otherwise clang-format gets confused and clang-format off directives +// do not help. +// +// The parentheses macros are used when wanting to require a rescan before +// expansion of parenthesized text appearing after a function-style macro name. + +#ifndef ABSL_TYPES_INTERNAL_PARENTHESES_H_ +#define ABSL_TYPES_INTERNAL_PARENTHESES_H_ + +#define ABSL_INTERNAL_LPAREN ( + +#define ABSL_INTERNAL_RPAREN ) + +#endif // ABSL_TYPES_INTERNAL_PARENTHESES_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/span.h b/third_party/abseil_cpp/absl/types/internal/span.h new file mode 100644 index 000000000000..112612f4bdae --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/span.h @@ -0,0 +1,128 @@ +// +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#ifndef ABSL_TYPES_INTERNAL_SPAN_H_ +#define ABSL_TYPES_INTERNAL_SPAN_H_ + +#include <algorithm> +#include <cstddef> +#include <string> +#include <type_traits> + +#include "absl/algorithm/algorithm.h" +#include "absl/base/internal/throw_delegate.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace span_internal { +// A constexpr min function +constexpr size_t Min(size_t a, size_t b) noexcept { return a < b ? a : b; } + +// Wrappers for access to container data pointers. +template <typename C> +constexpr auto GetDataImpl(C& c, char) noexcept // NOLINT(runtime/references) + -> decltype(c.data()) { + return c.data(); +} + +// Before C++17, std::string::data returns a const char* in all cases. +inline char* GetDataImpl(std::string& s, // NOLINT(runtime/references) + int) noexcept { + return &s[0]; +} + +template <typename C> +constexpr auto GetData(C& c) noexcept // NOLINT(runtime/references) + -> decltype(GetDataImpl(c, 0)) { + return GetDataImpl(c, 0); +} + +// Detection idioms for size() and data(). +template <typename C> +using HasSize = + std::is_integral<absl::decay_t<decltype(std::declval<C&>().size())>>; + +// We want to enable conversion from vector<T*> to Span<const T* const> but +// disable conversion from vector<Derived> to Span<Base>. Here we use +// the fact that U** is convertible to Q* const* if and only if Q is the same +// type or a more cv-qualified version of U. We also decay the result type of +// data() to avoid problems with classes which have a member function data() +// which returns a reference. +template <typename T, typename C> +using HasData = + std::is_convertible<absl::decay_t<decltype(GetData(std::declval<C&>()))>*, + T* const*>; + +// Extracts value type from a Container +template <typename C> +struct ElementType { + using type = typename absl::remove_reference_t<C>::value_type; +}; + +template <typename T, size_t N> +struct ElementType<T (&)[N]> { + using type = T; +}; + +template <typename C> +using ElementT = typename ElementType<C>::type; + +template <typename T> +using EnableIfMutable = + typename std::enable_if<!std::is_const<T>::value, int>::type; + +template <template <typename> class SpanT, typename T> +bool EqualImpl(SpanT<T> a, SpanT<T> b) { + static_assert(std::is_const<T>::value, ""); + return absl::equal(a.begin(), a.end(), b.begin(), b.end()); +} + +template <template <typename> class SpanT, typename T> +bool LessThanImpl(SpanT<T> a, SpanT<T> b) { + // We can't use value_type since that is remove_cv_t<T>, so we go the long way + // around. + static_assert(std::is_const<T>::value, ""); + return std::lexicographical_compare(a.begin(), a.end(), b.begin(), b.end()); +} + +// The `IsConvertible` classes here are needed because of the +// `std::is_convertible` bug in libcxx when compiled with GCC. This build +// configuration is used by Android NDK toolchain. Reference link: +// https://bugs.llvm.org/show_bug.cgi?id=27538. +template <typename From, typename To> +struct IsConvertibleHelper { + private: + static std::true_type testval(To); + static std::false_type testval(...); + + public: + using type = decltype(testval(std::declval<From>())); +}; + +template <typename From, typename To> +struct IsConvertible : IsConvertibleHelper<From, To>::type {}; + +// TODO(zhangxy): replace `IsConvertible` with `std::is_convertible` once the +// older version of libcxx is not supported. +template <typename From, typename To> +using EnableIfConvertibleTo = + typename std::enable_if<IsConvertible<From, To>::value>::type; +} // namespace span_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TYPES_INTERNAL_SPAN_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/transform_args.h b/third_party/abseil_cpp/absl/types/internal/transform_args.h new file mode 100644 index 000000000000..4a0ab42ac49f --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/transform_args.h @@ -0,0 +1,246 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// transform_args.h +// ----------------------------------------------------------------------------- +// +// This file contains a higher-order macro that "transforms" each element of a +// a variadic argument by a provided secondary macro. + +#ifndef ABSL_TYPES_INTERNAL_TRANSFORM_ARGS_H_ +#define ABSL_TYPES_INTERNAL_TRANSFORM_ARGS_H_ + +// +// ABSL_INTERNAL_CAT(a, b) +// +// This macro takes two arguments and concatenates them together via ## after +// expansion. +// +// Example: +// +// ABSL_INTERNAL_CAT(foo_, bar) +// +// Results in: +// +// foo_bar +#define ABSL_INTERNAL_CAT(a, b) ABSL_INTERNAL_CAT_IMPL(a, b) +#define ABSL_INTERNAL_CAT_IMPL(a, b) a##b + +// +// ABSL_INTERNAL_TRANSFORM_ARGS(m, ...) +// +// This macro takes another macro as an argument followed by a trailing series +// of additional parameters (up to 32 additional arguments). It invokes the +// passed-in macro once for each of the additional arguments, with the +// expansions separated by commas. +// +// Example: +// +// ABSL_INTERNAL_TRANSFORM_ARGS(MY_MACRO, a, b, c) +// +// Results in: +// +// MY_MACRO(a), MY_MACRO(b), MY_MACRO(c) +// +// TODO(calabrese) Handle no arguments as a special case. +#define ABSL_INTERNAL_TRANSFORM_ARGS(m, ...) \ + ABSL_INTERNAL_CAT(ABSL_INTERNAL_TRANSFORM_ARGS, \ + ABSL_INTERNAL_NUM_ARGS(__VA_ARGS__)) \ + (m, __VA_ARGS__) + +#define ABSL_INTERNAL_TRANSFORM_ARGS1(m, a0) m(a0) + +#define ABSL_INTERNAL_TRANSFORM_ARGS2(m, a0, a1) m(a0), m(a1) + +#define ABSL_INTERNAL_TRANSFORM_ARGS3(m, a0, a1, a2) m(a0), m(a1), m(a2) + +#define ABSL_INTERNAL_TRANSFORM_ARGS4(m, a0, a1, a2, a3) \ + m(a0), m(a1), m(a2), m(a3) + +#define ABSL_INTERNAL_TRANSFORM_ARGS5(m, a0, a1, a2, a3, a4) \ + m(a0), m(a1), m(a2), m(a3), m(a4) + +#define ABSL_INTERNAL_TRANSFORM_ARGS6(m, a0, a1, a2, a3, a4, a5) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5) + +#define ABSL_INTERNAL_TRANSFORM_ARGS7(m, a0, a1, a2, a3, a4, a5, a6) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6) + +#define ABSL_INTERNAL_TRANSFORM_ARGS8(m, a0, a1, a2, a3, a4, a5, a6, a7) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7) + +#define ABSL_INTERNAL_TRANSFORM_ARGS9(m, a0, a1, a2, a3, a4, a5, a6, a7, a8) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8) + +#define ABSL_INTERNAL_TRANSFORM_ARGS10(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9) + +#define ABSL_INTERNAL_TRANSFORM_ARGS11(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), m(a10) + +#define ABSL_INTERNAL_TRANSFORM_ARGS12(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11) + +#define ABSL_INTERNAL_TRANSFORM_ARGS13(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12) + +#define ABSL_INTERNAL_TRANSFORM_ARGS14(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13) + +#define ABSL_INTERNAL_TRANSFORM_ARGS15(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14) + +#define ABSL_INTERNAL_TRANSFORM_ARGS16(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15) + +#define ABSL_INTERNAL_TRANSFORM_ARGS17(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16) + +#define ABSL_INTERNAL_TRANSFORM_ARGS18(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17) + +#define ABSL_INTERNAL_TRANSFORM_ARGS19(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18) + +#define ABSL_INTERNAL_TRANSFORM_ARGS20(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19) + +#define ABSL_INTERNAL_TRANSFORM_ARGS21(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19, a20) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20) + +#define ABSL_INTERNAL_TRANSFORM_ARGS22(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19, a20, a21) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21) + +#define ABSL_INTERNAL_TRANSFORM_ARGS23(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19, a20, a21, a22) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22) + +#define ABSL_INTERNAL_TRANSFORM_ARGS24(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19, a20, a21, a22, a23) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23) + +#define ABSL_INTERNAL_TRANSFORM_ARGS25(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19, a20, a21, a22, a23, a24) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24) + +#define ABSL_INTERNAL_TRANSFORM_ARGS26( \ + m, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, \ + a16, a17, a18, a19, a20, a21, a22, a23, a24, a25) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25) + +#define ABSL_INTERNAL_TRANSFORM_ARGS27( \ + m, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, \ + a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25), m(a26) + +#define ABSL_INTERNAL_TRANSFORM_ARGS28( \ + m, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, \ + a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25), m(a26), m(a27) + +#define ABSL_INTERNAL_TRANSFORM_ARGS29( \ + m, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, \ + a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25), m(a26), m(a27), \ + m(a28) + +#define ABSL_INTERNAL_TRANSFORM_ARGS30( \ + m, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, \ + a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25), m(a26), m(a27), \ + m(a28), m(a29) + +#define ABSL_INTERNAL_TRANSFORM_ARGS31( \ + m, a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, \ + a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29, a30) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25), m(a26), m(a27), \ + m(a28), m(a29), m(a30) + +#define ABSL_INTERNAL_TRANSFORM_ARGS32(m, a0, a1, a2, a3, a4, a5, a6, a7, a8, \ + a9, a10, a11, a12, a13, a14, a15, a16, \ + a17, a18, a19, a20, a21, a22, a23, a24, \ + a25, a26, a27, a28, a29, a30, a31) \ + m(a0), m(a1), m(a2), m(a3), m(a4), m(a5), m(a6), m(a7), m(a8), m(a9), \ + m(a10), m(a11), m(a12), m(a13), m(a14), m(a15), m(a16), m(a17), m(a18), \ + m(a19), m(a20), m(a21), m(a22), m(a23), m(a24), m(a25), m(a26), m(a27), \ + m(a28), m(a29), m(a30), m(a31) + +#define ABSL_INTERNAL_NUM_ARGS_IMPL(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, \ + a10, a11, a12, a13, a14, a15, a16, a17, \ + a18, a19, a20, a21, a22, a23, a24, a25, \ + a26, a27, a28, a29, a30, a31, result, ...) \ + result + +#define ABSL_INTERNAL_FORCE_EXPANSION(...) __VA_ARGS__ + +#define ABSL_INTERNAL_NUM_ARGS(...) \ + ABSL_INTERNAL_FORCE_EXPANSION(ABSL_INTERNAL_NUM_ARGS_IMPL( \ + __VA_ARGS__, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, \ + 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, )) + +#endif // ABSL_TYPES_INTERNAL_TRANSFORM_ARGS_H_ diff --git a/third_party/abseil_cpp/absl/types/internal/variant.h b/third_party/abseil_cpp/absl/types/internal/variant.h new file mode 100644 index 000000000000..772008c74e6d --- /dev/null +++ b/third_party/abseil_cpp/absl/types/internal/variant.h @@ -0,0 +1,1646 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Implementation details of absl/types/variant.h, pulled into a +// separate file to avoid cluttering the top of the API header with +// implementation details. + +#ifndef ABSL_TYPES_variant_internal_H_ +#define ABSL_TYPES_variant_internal_H_ + +#include <cassert> +#include <cstddef> +#include <cstdlib> +#include <memory> +#include <stdexcept> +#include <tuple> +#include <type_traits> + +#include "absl/base/config.h" +#include "absl/base/internal/identity.h" +#include "absl/base/internal/inline_variable.h" +#include "absl/base/internal/invoke.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/meta/type_traits.h" +#include "absl/types/bad_variant_access.h" +#include "absl/utility/utility.h" + +#if !defined(ABSL_USES_STD_VARIANT) + +namespace absl { +ABSL_NAMESPACE_BEGIN + +template <class... Types> +class variant; + +ABSL_INTERNAL_INLINE_CONSTEXPR(size_t, variant_npos, static_cast<size_t>(-1)); + +template <class T> +struct variant_size; + +template <std::size_t I, class T> +struct variant_alternative; + +namespace variant_internal { + +// NOTE: See specializations below for details. +template <std::size_t I, class T> +struct VariantAlternativeSfinae {}; + +// Requires: I < variant_size_v<T>. +// +// Value: The Ith type of Types... +template <std::size_t I, class T0, class... Tn> +struct VariantAlternativeSfinae<I, variant<T0, Tn...>> + : VariantAlternativeSfinae<I - 1, variant<Tn...>> {}; + +// Value: T0 +template <class T0, class... Ts> +struct VariantAlternativeSfinae<0, variant<T0, Ts...>> { + using type = T0; +}; + +template <std::size_t I, class T> +using VariantAlternativeSfinaeT = typename VariantAlternativeSfinae<I, T>::type; + +// NOTE: Requires T to be a reference type. +template <class T, class U> +struct GiveQualsTo; + +template <class T, class U> +struct GiveQualsTo<T&, U> { + using type = U&; +}; + +template <class T, class U> +struct GiveQualsTo<T&&, U> { + using type = U&&; +}; + +template <class T, class U> +struct GiveQualsTo<const T&, U> { + using type = const U&; +}; + +template <class T, class U> +struct GiveQualsTo<const T&&, U> { + using type = const U&&; +}; + +template <class T, class U> +struct GiveQualsTo<volatile T&, U> { + using type = volatile U&; +}; + +template <class T, class U> +struct GiveQualsTo<volatile T&&, U> { + using type = volatile U&&; +}; + +template <class T, class U> +struct GiveQualsTo<volatile const T&, U> { + using type = volatile const U&; +}; + +template <class T, class U> +struct GiveQualsTo<volatile const T&&, U> { + using type = volatile const U&&; +}; + +template <class T, class U> +using GiveQualsToT = typename GiveQualsTo<T, U>::type; + +// Convenience alias, since size_t integral_constant is used a lot in this file. +template <std::size_t I> +using SizeT = std::integral_constant<std::size_t, I>; + +using NPos = SizeT<variant_npos>; + +template <class Variant, class T, class = void> +struct IndexOfConstructedType {}; + +template <std::size_t I, class Variant> +struct VariantAccessResultImpl; + +template <std::size_t I, template <class...> class Variantemplate, class... T> +struct VariantAccessResultImpl<I, Variantemplate<T...>&> { + using type = typename absl::variant_alternative<I, variant<T...>>::type&; +}; + +template <std::size_t I, template <class...> class Variantemplate, class... T> +struct VariantAccessResultImpl<I, const Variantemplate<T...>&> { + using type = + const typename absl::variant_alternative<I, variant<T...>>::type&; +}; + +template <std::size_t I, template <class...> class Variantemplate, class... T> +struct VariantAccessResultImpl<I, Variantemplate<T...>&&> { + using type = typename absl::variant_alternative<I, variant<T...>>::type&&; +}; + +template <std::size_t I, template <class...> class Variantemplate, class... T> +struct VariantAccessResultImpl<I, const Variantemplate<T...>&&> { + using type = + const typename absl::variant_alternative<I, variant<T...>>::type&&; +}; + +template <std::size_t I, class Variant> +using VariantAccessResult = + typename VariantAccessResultImpl<I, Variant&&>::type; + +// NOTE: This is used instead of std::array to reduce instantiation overhead. +template <class T, std::size_t Size> +struct SimpleArray { + static_assert(Size != 0, ""); + T value[Size]; +}; + +template <class T> +struct AccessedType { + using type = T; +}; + +template <class T> +using AccessedTypeT = typename AccessedType<T>::type; + +template <class T, std::size_t Size> +struct AccessedType<SimpleArray<T, Size>> { + using type = AccessedTypeT<T>; +}; + +template <class T> +constexpr T AccessSimpleArray(const T& value) { + return value; +} + +template <class T, std::size_t Size, class... SizeT> +constexpr AccessedTypeT<T> AccessSimpleArray(const SimpleArray<T, Size>& table, + std::size_t head_index, + SizeT... tail_indices) { + return AccessSimpleArray(table.value[head_index], tail_indices...); +} + +// Note: Intentionally is an alias. +template <class T> +using AlwaysZero = SizeT<0>; + +template <class Op, class... Vs> +struct VisitIndicesResultImpl { + using type = absl::result_of_t<Op(AlwaysZero<Vs>...)>; +}; + +template <class Op, class... Vs> +using VisitIndicesResultT = typename VisitIndicesResultImpl<Op, Vs...>::type; + +template <class ReturnType, class FunctionObject, class EndIndices, + class BoundIndices> +struct MakeVisitationMatrix; + +template <class ReturnType, class FunctionObject, std::size_t... Indices> +constexpr ReturnType call_with_indices(FunctionObject&& function) { + static_assert( + std::is_same<ReturnType, decltype(std::declval<FunctionObject>()( + SizeT<Indices>()...))>::value, + "Not all visitation overloads have the same return type."); + return absl::forward<FunctionObject>(function)(SizeT<Indices>()...); +} + +template <class ReturnType, class FunctionObject, std::size_t... BoundIndices> +struct MakeVisitationMatrix<ReturnType, FunctionObject, index_sequence<>, + index_sequence<BoundIndices...>> { + using ResultType = ReturnType (*)(FunctionObject&&); + static constexpr ResultType Run() { + return &call_with_indices<ReturnType, FunctionObject, + (BoundIndices - 1)...>; + } +}; + +template <typename Is, std::size_t J> +struct AppendToIndexSequence; + +template <typename Is, std::size_t J> +using AppendToIndexSequenceT = typename AppendToIndexSequence<Is, J>::type; + +template <std::size_t... Is, std::size_t J> +struct AppendToIndexSequence<index_sequence<Is...>, J> { + using type = index_sequence<Is..., J>; +}; + +template <class ReturnType, class FunctionObject, class EndIndices, + class CurrIndices, class BoundIndices> +struct MakeVisitationMatrixImpl; + +template <class ReturnType, class FunctionObject, class EndIndices, + std::size_t... CurrIndices, class BoundIndices> +struct MakeVisitationMatrixImpl<ReturnType, FunctionObject, EndIndices, + index_sequence<CurrIndices...>, BoundIndices> { + using ResultType = SimpleArray< + typename MakeVisitationMatrix<ReturnType, FunctionObject, EndIndices, + index_sequence<>>::ResultType, + sizeof...(CurrIndices)>; + + static constexpr ResultType Run() { + return {{MakeVisitationMatrix< + ReturnType, FunctionObject, EndIndices, + AppendToIndexSequenceT<BoundIndices, CurrIndices>>::Run()...}}; + } +}; + +template <class ReturnType, class FunctionObject, std::size_t HeadEndIndex, + std::size_t... TailEndIndices, std::size_t... BoundIndices> +struct MakeVisitationMatrix<ReturnType, FunctionObject, + index_sequence<HeadEndIndex, TailEndIndices...>, + index_sequence<BoundIndices...>> + : MakeVisitationMatrixImpl<ReturnType, FunctionObject, + index_sequence<TailEndIndices...>, + absl::make_index_sequence<HeadEndIndex>, + index_sequence<BoundIndices...>> {}; + +struct UnreachableSwitchCase { + template <class Op> + [[noreturn]] static VisitIndicesResultT<Op, std::size_t> Run( + Op&& /*ignored*/) { +#if ABSL_HAVE_BUILTIN(__builtin_unreachable) || \ + (defined(__GNUC__) && !defined(__clang__)) + __builtin_unreachable(); +#elif defined(_MSC_VER) + __assume(false); +#else + // Try to use assert of false being identified as an unreachable intrinsic. + // NOTE: We use assert directly to increase chances of exploiting an assume + // intrinsic. + assert(false); // NOLINT + + // Hack to silence potential no return warning -- cause an infinite loop. + return Run(absl::forward<Op>(op)); +#endif // Checks for __builtin_unreachable + } +}; + +template <class Op, std::size_t I> +struct ReachableSwitchCase { + static VisitIndicesResultT<Op, std::size_t> Run(Op&& op) { + return absl::base_internal::invoke(absl::forward<Op>(op), SizeT<I>()); + } +}; + +// The number 33 is just a guess at a reasonable maximum to our switch. It is +// not based on any analysis. The reason it is a power of 2 plus 1 instead of a +// power of 2 is because the number was picked to correspond to a power of 2 +// amount of "normal" alternatives, plus one for the possibility of the user +// providing "monostate" in addition to the more natural alternatives. +ABSL_INTERNAL_INLINE_CONSTEXPR(std::size_t, MaxUnrolledVisitCases, 33); + +// Note: The default-definition is for unreachable cases. +template <bool IsReachable> +struct PickCaseImpl { + template <class Op, std::size_t I> + using Apply = UnreachableSwitchCase; +}; + +template <> +struct PickCaseImpl</*IsReachable =*/true> { + template <class Op, std::size_t I> + using Apply = ReachableSwitchCase<Op, I>; +}; + +// Note: This form of dance with template aliases is to make sure that we +// instantiate a number of templates proportional to the number of variant +// alternatives rather than a number of templates proportional to our +// maximum unrolled amount of visitation cases (aliases are effectively +// "free" whereas other template instantiations are costly). +template <class Op, std::size_t I, std::size_t EndIndex> +using PickCase = typename PickCaseImpl<(I < EndIndex)>::template Apply<Op, I>; + +template <class ReturnType> +[[noreturn]] ReturnType TypedThrowBadVariantAccess() { + absl::variant_internal::ThrowBadVariantAccess(); +} + +// Given N variant sizes, determine the number of cases there would need to be +// in a single switch-statement that would cover every possibility in the +// corresponding N-ary visit operation. +template <std::size_t... NumAlternatives> +struct NumCasesOfSwitch; + +template <std::size_t HeadNumAlternatives, std::size_t... TailNumAlternatives> +struct NumCasesOfSwitch<HeadNumAlternatives, TailNumAlternatives...> { + static constexpr std::size_t value = + (HeadNumAlternatives + 1) * + NumCasesOfSwitch<TailNumAlternatives...>::value; +}; + +template <> +struct NumCasesOfSwitch<> { + static constexpr std::size_t value = 1; +}; + +// A switch statement optimizes better than the table of function pointers. +template <std::size_t EndIndex> +struct VisitIndicesSwitch { + static_assert(EndIndex <= MaxUnrolledVisitCases, + "Maximum unrolled switch size exceeded."); + + template <class Op> + static VisitIndicesResultT<Op, std::size_t> Run(Op&& op, std::size_t i) { + switch (i) { + case 0: + return PickCase<Op, 0, EndIndex>::Run(absl::forward<Op>(op)); + case 1: + return PickCase<Op, 1, EndIndex>::Run(absl::forward<Op>(op)); + case 2: + return PickCase<Op, 2, EndIndex>::Run(absl::forward<Op>(op)); + case 3: + return PickCase<Op, 3, EndIndex>::Run(absl::forward<Op>(op)); + case 4: + return PickCase<Op, 4, EndIndex>::Run(absl::forward<Op>(op)); + case 5: + return PickCase<Op, 5, EndIndex>::Run(absl::forward<Op>(op)); + case 6: + return PickCase<Op, 6, EndIndex>::Run(absl::forward<Op>(op)); + case 7: + return PickCase<Op, 7, EndIndex>::Run(absl::forward<Op>(op)); + case 8: + return PickCase<Op, 8, EndIndex>::Run(absl::forward<Op>(op)); + case 9: + return PickCase<Op, 9, EndIndex>::Run(absl::forward<Op>(op)); + case 10: + return PickCase<Op, 10, EndIndex>::Run(absl::forward<Op>(op)); + case 11: + return PickCase<Op, 11, EndIndex>::Run(absl::forward<Op>(op)); + case 12: + return PickCase<Op, 12, EndIndex>::Run(absl::forward<Op>(op)); + case 13: + return PickCase<Op, 13, EndIndex>::Run(absl::forward<Op>(op)); + case 14: + return PickCase<Op, 14, EndIndex>::Run(absl::forward<Op>(op)); + case 15: + return PickCase<Op, 15, EndIndex>::Run(absl::forward<Op>(op)); + case 16: + return PickCase<Op, 16, EndIndex>::Run(absl::forward<Op>(op)); + case 17: + return PickCase<Op, 17, EndIndex>::Run(absl::forward<Op>(op)); + case 18: + return PickCase<Op, 18, EndIndex>::Run(absl::forward<Op>(op)); + case 19: + return PickCase<Op, 19, EndIndex>::Run(absl::forward<Op>(op)); + case 20: + return PickCase<Op, 20, EndIndex>::Run(absl::forward<Op>(op)); + case 21: + return PickCase<Op, 21, EndIndex>::Run(absl::forward<Op>(op)); + case 22: + return PickCase<Op, 22, EndIndex>::Run(absl::forward<Op>(op)); + case 23: + return PickCase<Op, 23, EndIndex>::Run(absl::forward<Op>(op)); + case 24: + return PickCase<Op, 24, EndIndex>::Run(absl::forward<Op>(op)); + case 25: + return PickCase<Op, 25, EndIndex>::Run(absl::forward<Op>(op)); + case 26: + return PickCase<Op, 26, EndIndex>::Run(absl::forward<Op>(op)); + case 27: + return PickCase<Op, 27, EndIndex>::Run(absl::forward<Op>(op)); + case 28: + return PickCase<Op, 28, EndIndex>::Run(absl::forward<Op>(op)); + case 29: + return PickCase<Op, 29, EndIndex>::Run(absl::forward<Op>(op)); + case 30: + return PickCase<Op, 30, EndIndex>::Run(absl::forward<Op>(op)); + case 31: + return PickCase<Op, 31, EndIndex>::Run(absl::forward<Op>(op)); + case 32: + return PickCase<Op, 32, EndIndex>::Run(absl::forward<Op>(op)); + default: + ABSL_ASSERT(i == variant_npos); + return absl::base_internal::invoke(absl::forward<Op>(op), NPos()); + } + } +}; + +template <std::size_t... EndIndices> +struct VisitIndicesFallback { + template <class Op, class... SizeT> + static VisitIndicesResultT<Op, SizeT...> Run(Op&& op, SizeT... indices) { + return AccessSimpleArray( + MakeVisitationMatrix<VisitIndicesResultT<Op, SizeT...>, Op, + index_sequence<(EndIndices + 1)...>, + index_sequence<>>::Run(), + (indices + 1)...)(absl::forward<Op>(op)); + } +}; + +// Take an N-dimensional series of indices and convert them into a single index +// without loss of information. The purpose of this is to be able to convert an +// N-ary visit operation into a single switch statement. +template <std::size_t...> +struct FlattenIndices; + +template <std::size_t HeadSize, std::size_t... TailSize> +struct FlattenIndices<HeadSize, TailSize...> { + template<class... SizeType> + static constexpr std::size_t Run(std::size_t head, SizeType... tail) { + return head + HeadSize * FlattenIndices<TailSize...>::Run(tail...); + } +}; + +template <> +struct FlattenIndices<> { + static constexpr std::size_t Run() { return 0; } +}; + +// Take a single "flattened" index (flattened by FlattenIndices) and determine +// the value of the index of one of the logically represented dimensions. +template <std::size_t I, std::size_t IndexToGet, std::size_t HeadSize, + std::size_t... TailSize> +struct UnflattenIndex { + static constexpr std::size_t value = + UnflattenIndex<I / HeadSize, IndexToGet - 1, TailSize...>::value; +}; + +template <std::size_t I, std::size_t HeadSize, std::size_t... TailSize> +struct UnflattenIndex<I, 0, HeadSize, TailSize...> { + static constexpr std::size_t value = (I % HeadSize); +}; + +// The backend for converting an N-ary visit operation into a unary visit. +template <class IndexSequence, std::size_t... EndIndices> +struct VisitIndicesVariadicImpl; + +template <std::size_t... N, std::size_t... EndIndices> +struct VisitIndicesVariadicImpl<absl::index_sequence<N...>, EndIndices...> { + // A type that can take an N-ary function object and converts it to a unary + // function object that takes a single, flattened index, and "unflattens" it + // into its individual dimensions when forwarding to the wrapped object. + template <class Op> + struct FlattenedOp { + template <std::size_t I> + VisitIndicesResultT<Op, decltype(EndIndices)...> operator()( + SizeT<I> /*index*/) && { + return base_internal::invoke( + absl::forward<Op>(op), + SizeT<UnflattenIndex<I, N, (EndIndices + 1)...>::value - + std::size_t{1}>()...); + } + + Op&& op; + }; + + template <class Op, class... SizeType> + static VisitIndicesResultT<Op, decltype(EndIndices)...> Run( + Op&& op, SizeType... i) { + return VisitIndicesSwitch<NumCasesOfSwitch<EndIndices...>::value>::Run( + FlattenedOp<Op>{absl::forward<Op>(op)}, + FlattenIndices<(EndIndices + std::size_t{1})...>::Run( + (i + std::size_t{1})...)); + } +}; + +template <std::size_t... EndIndices> +struct VisitIndicesVariadic + : VisitIndicesVariadicImpl<absl::make_index_sequence<sizeof...(EndIndices)>, + EndIndices...> {}; + +// This implementation will flatten N-ary visit operations into a single switch +// statement when the number of cases would be less than our maximum specified +// switch-statement size. +// TODO(calabrese) +// Based on benchmarks, determine whether the function table approach actually +// does optimize better than a chain of switch statements and possibly update +// the implementation accordingly. Also consider increasing the maximum switch +// size. +template <std::size_t... EndIndices> +struct VisitIndices + : absl::conditional_t<(NumCasesOfSwitch<EndIndices...>::value <= + MaxUnrolledVisitCases), + VisitIndicesVariadic<EndIndices...>, + VisitIndicesFallback<EndIndices...>> {}; + +template <std::size_t EndIndex> +struct VisitIndices<EndIndex> + : absl::conditional_t<(EndIndex <= MaxUnrolledVisitCases), + VisitIndicesSwitch<EndIndex>, + VisitIndicesFallback<EndIndex>> {}; + +// Suppress bogus warning on MSVC: MSVC complains that the `reinterpret_cast` +// below is returning the address of a temporary or local object. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4172) +#endif // _MSC_VER + +// TODO(calabrese) std::launder +// TODO(calabrese) constexpr +// NOTE: DO NOT REMOVE the `inline` keyword as it is necessary to work around a +// MSVC bug. See https://github.com/abseil/abseil-cpp/issues/129 for details. +template <class Self, std::size_t I> +inline VariantAccessResult<I, Self> AccessUnion(Self&& self, SizeT<I> /*i*/) { + return reinterpret_cast<VariantAccessResult<I, Self>>(self); +} + +#ifdef _MSC_VER +#pragma warning(pop) +#endif // _MSC_VER + +template <class T> +void DeducedDestroy(T& self) { // NOLINT + self.~T(); +} + +// NOTE: This type exists as a single entity for variant and its bases to +// befriend. It contains helper functionality that manipulates the state of the +// variant, such as the implementation of things like assignment and emplace +// operations. +struct VariantCoreAccess { + template <class VariantType> + static typename VariantType::Variant& Derived(VariantType& self) { // NOLINT + return static_cast<typename VariantType::Variant&>(self); + } + + template <class VariantType> + static const typename VariantType::Variant& Derived( + const VariantType& self) { // NOLINT + return static_cast<const typename VariantType::Variant&>(self); + } + + template <class VariantType> + static void Destroy(VariantType& self) { // NOLINT + Derived(self).destroy(); + self.index_ = absl::variant_npos; + } + + template <class Variant> + static void SetIndex(Variant& self, std::size_t i) { // NOLINT + self.index_ = i; + } + + template <class Variant> + static void InitFrom(Variant& self, Variant&& other) { // NOLINT + VisitIndices<absl::variant_size<Variant>::value>::Run( + InitFromVisitor<Variant, Variant&&>{&self, + std::forward<Variant>(other)}, + other.index()); + self.index_ = other.index(); + } + + // Access a variant alternative, assuming the index is correct. + template <std::size_t I, class Variant> + static VariantAccessResult<I, Variant> Access(Variant&& self) { + // This cast instead of invocation of AccessUnion with an rvalue is a + // workaround for msvc. Without this there is a runtime failure when dealing + // with rvalues. + // TODO(calabrese) Reduce test case and find a simpler workaround. + return static_cast<VariantAccessResult<I, Variant>>( + variant_internal::AccessUnion(self.state_, SizeT<I>())); + } + + // Access a variant alternative, throwing if the index is incorrect. + template <std::size_t I, class Variant> + static VariantAccessResult<I, Variant> CheckedAccess(Variant&& self) { + if (ABSL_PREDICT_FALSE(self.index_ != I)) { + TypedThrowBadVariantAccess<VariantAccessResult<I, Variant>>(); + } + + return Access<I>(absl::forward<Variant>(self)); + } + + // The implementation of the move-assignment operation for a variant. + template <class VType> + struct MoveAssignVisitor { + using DerivedType = typename VType::Variant; + template <std::size_t NewIndex> + void operator()(SizeT<NewIndex> /*new_i*/) const { + if (left->index_ == NewIndex) { + Access<NewIndex>(*left) = std::move(Access<NewIndex>(*right)); + } else { + Derived(*left).template emplace<NewIndex>( + std::move(Access<NewIndex>(*right))); + } + } + + void operator()(SizeT<absl::variant_npos> /*new_i*/) const { + Destroy(*left); + } + + VType* left; + VType* right; + }; + + template <class VType> + static MoveAssignVisitor<VType> MakeMoveAssignVisitor(VType* left, + VType* other) { + return {left, other}; + } + + // The implementation of the assignment operation for a variant. + template <class VType> + struct CopyAssignVisitor { + using DerivedType = typename VType::Variant; + template <std::size_t NewIndex> + void operator()(SizeT<NewIndex> /*new_i*/) const { + using New = + typename absl::variant_alternative<NewIndex, DerivedType>::type; + + if (left->index_ == NewIndex) { + Access<NewIndex>(*left) = Access<NewIndex>(*right); + } else if (std::is_nothrow_copy_constructible<New>::value || + !std::is_nothrow_move_constructible<New>::value) { + Derived(*left).template emplace<NewIndex>(Access<NewIndex>(*right)); + } else { + Derived(*left) = DerivedType(Derived(*right)); + } + } + + void operator()(SizeT<absl::variant_npos> /*new_i*/) const { + Destroy(*left); + } + + VType* left; + const VType* right; + }; + + template <class VType> + static CopyAssignVisitor<VType> MakeCopyAssignVisitor(VType* left, + const VType& other) { + return {left, &other}; + } + + // The implementation of conversion-assignment operations for variant. + template <class Left, class QualifiedNew> + struct ConversionAssignVisitor { + using NewIndex = + variant_internal::IndexOfConstructedType<Left, QualifiedNew>; + + void operator()(SizeT<NewIndex::value> /*old_i*/ + ) const { + Access<NewIndex::value>(*left) = absl::forward<QualifiedNew>(other); + } + + template <std::size_t OldIndex> + void operator()(SizeT<OldIndex> /*old_i*/ + ) const { + using New = + typename absl::variant_alternative<NewIndex::value, Left>::type; + if (std::is_nothrow_constructible<New, QualifiedNew>::value || + !std::is_nothrow_move_constructible<New>::value) { + left->template emplace<NewIndex::value>( + absl::forward<QualifiedNew>(other)); + } else { + // the standard says "equivalent to + // operator=(variant(std::forward<T>(t)))", but we use `emplace` here + // because the variant's move assignment operator could be deleted. + left->template emplace<NewIndex::value>( + New(absl::forward<QualifiedNew>(other))); + } + } + + Left* left; + QualifiedNew&& other; + }; + + template <class Left, class QualifiedNew> + static ConversionAssignVisitor<Left, QualifiedNew> + MakeConversionAssignVisitor(Left* left, QualifiedNew&& qual) { + return {left, absl::forward<QualifiedNew>(qual)}; + } + + // Backend for operations for `emplace()` which destructs `*self` then + // construct a new alternative with `Args...`. + template <std::size_t NewIndex, class Self, class... Args> + static typename absl::variant_alternative<NewIndex, Self>::type& Replace( + Self* self, Args&&... args) { + Destroy(*self); + using New = typename absl::variant_alternative<NewIndex, Self>::type; + New* const result = ::new (static_cast<void*>(&self->state_)) + New(absl::forward<Args>(args)...); + self->index_ = NewIndex; + return *result; + } + + template <class LeftVariant, class QualifiedRightVariant> + struct InitFromVisitor { + template <std::size_t NewIndex> + void operator()(SizeT<NewIndex> /*new_i*/) const { + using Alternative = + typename variant_alternative<NewIndex, LeftVariant>::type; + ::new (static_cast<void*>(&left->state_)) Alternative( + Access<NewIndex>(std::forward<QualifiedRightVariant>(right))); + } + + void operator()(SizeT<absl::variant_npos> /*new_i*/) const { + // This space intentionally left blank. + } + LeftVariant* left; + QualifiedRightVariant&& right; + }; +}; + +template <class Expected, class... T> +struct IndexOfImpl; + +template <class Expected> +struct IndexOfImpl<Expected> { + using IndexFromEnd = SizeT<0>; + using MatchedIndexFromEnd = IndexFromEnd; + using MultipleMatches = std::false_type; +}; + +template <class Expected, class Head, class... Tail> +struct IndexOfImpl<Expected, Head, Tail...> : IndexOfImpl<Expected, Tail...> { + using IndexFromEnd = + SizeT<IndexOfImpl<Expected, Tail...>::IndexFromEnd::value + 1>; +}; + +template <class Expected, class... Tail> +struct IndexOfImpl<Expected, Expected, Tail...> + : IndexOfImpl<Expected, Tail...> { + using IndexFromEnd = + SizeT<IndexOfImpl<Expected, Tail...>::IndexFromEnd::value + 1>; + using MatchedIndexFromEnd = IndexFromEnd; + using MultipleMatches = std::integral_constant< + bool, IndexOfImpl<Expected, Tail...>::MatchedIndexFromEnd::value != 0>; +}; + +template <class Expected, class... Types> +struct IndexOfMeta { + using Results = IndexOfImpl<Expected, Types...>; + static_assert(!Results::MultipleMatches::value, + "Attempted to access a variant by specifying a type that " + "matches more than one alternative."); + static_assert(Results::MatchedIndexFromEnd::value != 0, + "Attempted to access a variant by specifying a type that does " + "not match any alternative."); + using type = SizeT<sizeof...(Types) - Results::MatchedIndexFromEnd::value>; +}; + +template <class Expected, class... Types> +using IndexOf = typename IndexOfMeta<Expected, Types...>::type; + +template <class Variant, class T, std::size_t CurrIndex> +struct UnambiguousIndexOfImpl; + +// Terminating case encountered once we've checked all of the alternatives +template <class T, std::size_t CurrIndex> +struct UnambiguousIndexOfImpl<variant<>, T, CurrIndex> : SizeT<CurrIndex> {}; + +// Case where T is not Head +template <class Head, class... Tail, class T, std::size_t CurrIndex> +struct UnambiguousIndexOfImpl<variant<Head, Tail...>, T, CurrIndex> + : UnambiguousIndexOfImpl<variant<Tail...>, T, CurrIndex + 1>::type {}; + +// Case where T is Head +template <class Head, class... Tail, std::size_t CurrIndex> +struct UnambiguousIndexOfImpl<variant<Head, Tail...>, Head, CurrIndex> + : SizeT<UnambiguousIndexOfImpl<variant<Tail...>, Head, 0>::value == + sizeof...(Tail) + ? CurrIndex + : CurrIndex + sizeof...(Tail) + 1> {}; + +template <class Variant, class T> +struct UnambiguousIndexOf; + +struct NoMatch { + struct type {}; +}; + +template <class... Alts, class T> +struct UnambiguousIndexOf<variant<Alts...>, T> + : std::conditional<UnambiguousIndexOfImpl<variant<Alts...>, T, 0>::value != + sizeof...(Alts), + UnambiguousIndexOfImpl<variant<Alts...>, T, 0>, + NoMatch>::type::type {}; + +template <class T, std::size_t /*Dummy*/> +using UnambiguousTypeOfImpl = T; + +template <class Variant, class T> +using UnambiguousTypeOfT = + UnambiguousTypeOfImpl<T, UnambiguousIndexOf<Variant, T>::value>; + +template <class H, class... T> +class VariantStateBase; + +// This is an implementation of the "imaginary function" that is described in +// [variant.ctor] +// It is used in order to determine which alternative to construct during +// initialization from some type T. +template <class Variant, std::size_t I = 0> +struct ImaginaryFun; + +template <std::size_t I> +struct ImaginaryFun<variant<>, I> { + static void Run() = delete; +}; + +template <class H, class... T, std::size_t I> +struct ImaginaryFun<variant<H, T...>, I> : ImaginaryFun<variant<T...>, I + 1> { + using ImaginaryFun<variant<T...>, I + 1>::Run; + + // NOTE: const& and && are used instead of by-value due to lack of guaranteed + // move elision of C++17. This may have other minor differences, but tests + // pass. + static SizeT<I> Run(const H&, SizeT<I>); + static SizeT<I> Run(H&&, SizeT<I>); +}; + +// The following metafunctions are used in constructor and assignment +// constraints. +template <class Self, class T> +struct IsNeitherSelfNorInPlace : std::true_type {}; + +template <class Self> +struct IsNeitherSelfNorInPlace<Self, Self> : std::false_type {}; + +template <class Self, class T> +struct IsNeitherSelfNorInPlace<Self, in_place_type_t<T>> : std::false_type {}; + +template <class Self, std::size_t I> +struct IsNeitherSelfNorInPlace<Self, in_place_index_t<I>> : std::false_type {}; + +template <class Variant, class T, class = void> +struct ConversionIsPossibleImpl : std::false_type {}; + +template <class Variant, class T> +struct ConversionIsPossibleImpl< + Variant, T, + void_t<decltype(ImaginaryFun<Variant>::Run(std::declval<T>(), {}))>> + : std::true_type {}; + +template <class Variant, class T> +struct ConversionIsPossible : ConversionIsPossibleImpl<Variant, T>::type {}; + +template <class Variant, class T> +struct IndexOfConstructedType< + Variant, T, + void_t<decltype(ImaginaryFun<Variant>::Run(std::declval<T>(), {}))>> + : decltype(ImaginaryFun<Variant>::Run(std::declval<T>(), {})) {}; + +template <std::size_t... Is> +struct ContainsVariantNPos + : absl::negation<std::is_same< // NOLINT + absl::integer_sequence<bool, 0 <= Is...>, + absl::integer_sequence<bool, Is != absl::variant_npos...>>> {}; + +template <class Op, class... QualifiedVariants> +using RawVisitResult = + absl::result_of_t<Op(VariantAccessResult<0, QualifiedVariants>...)>; + +// NOTE: The spec requires that all return-paths yield the same type and is not +// SFINAE-friendly, so we can deduce the return type by examining the first +// result. If it's not callable, then we get an error, but are compliant and +// fast to compile. +// TODO(calabrese) Possibly rewrite in a way that yields better compile errors +// at the cost of longer compile-times. +template <class Op, class... QualifiedVariants> +struct VisitResultImpl { + using type = + absl::result_of_t<Op(VariantAccessResult<0, QualifiedVariants>...)>; +}; + +// Done in two steps intentionally so that we don't cause substitution to fail. +template <class Op, class... QualifiedVariants> +using VisitResult = typename VisitResultImpl<Op, QualifiedVariants...>::type; + +template <class Op, class... QualifiedVariants> +struct PerformVisitation { + using ReturnType = VisitResult<Op, QualifiedVariants...>; + + template <std::size_t... Is> + constexpr ReturnType operator()(SizeT<Is>... indices) const { + return Run(typename ContainsVariantNPos<Is...>::type{}, + absl::index_sequence_for<QualifiedVariants...>(), indices...); + } + + template <std::size_t... TupIs, std::size_t... Is> + constexpr ReturnType Run(std::false_type /*has_valueless*/, + index_sequence<TupIs...>, SizeT<Is>...) const { + static_assert( + std::is_same<ReturnType, + absl::result_of_t<Op(VariantAccessResult< + Is, QualifiedVariants>...)>>::value, + "All visitation overloads must have the same return type."); + return absl::base_internal::invoke( + absl::forward<Op>(op), + VariantCoreAccess::Access<Is>( + absl::forward<QualifiedVariants>(std::get<TupIs>(variant_tup)))...); + } + + template <std::size_t... TupIs, std::size_t... Is> + [[noreturn]] ReturnType Run(std::true_type /*has_valueless*/, + index_sequence<TupIs...>, SizeT<Is>...) const { + absl::variant_internal::ThrowBadVariantAccess(); + } + + // TODO(calabrese) Avoid using a tuple, which causes lots of instantiations + // Attempts using lambda variadic captures fail on current GCC. + std::tuple<QualifiedVariants&&...> variant_tup; + Op&& op; +}; + +template <class... T> +union Union; + +// We want to allow for variant<> to be trivial. For that, we need the default +// constructor to be trivial, which means we can't define it ourselves. +// Instead, we use a non-default constructor that takes NoopConstructorTag +// that doesn't affect the triviality of the types. +struct NoopConstructorTag {}; + +template <std::size_t I> +struct EmplaceTag {}; + +template <> +union Union<> { + constexpr explicit Union(NoopConstructorTag) noexcept {} +}; + +// Suppress bogus warning on MSVC: MSVC complains that Union<T...> has a defined +// deleted destructor from the `std::is_destructible` check below. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4624) +#endif // _MSC_VER + +template <class Head, class... Tail> +union Union<Head, Tail...> { + using TailUnion = Union<Tail...>; + + explicit constexpr Union(NoopConstructorTag /*tag*/) noexcept + : tail(NoopConstructorTag()) {} + + template <class... P> + explicit constexpr Union(EmplaceTag<0>, P&&... args) + : head(absl::forward<P>(args)...) {} + + template <std::size_t I, class... P> + explicit constexpr Union(EmplaceTag<I>, P&&... args) + : tail(EmplaceTag<I - 1>{}, absl::forward<P>(args)...) {} + + Head head; + TailUnion tail; +}; + +#ifdef _MSC_VER +#pragma warning(pop) +#endif // _MSC_VER + +// TODO(calabrese) Just contain a Union in this union (certain configs fail). +template <class... T> +union DestructibleUnionImpl; + +template <> +union DestructibleUnionImpl<> { + constexpr explicit DestructibleUnionImpl(NoopConstructorTag) noexcept {} +}; + +template <class Head, class... Tail> +union DestructibleUnionImpl<Head, Tail...> { + using TailUnion = DestructibleUnionImpl<Tail...>; + + explicit constexpr DestructibleUnionImpl(NoopConstructorTag /*tag*/) noexcept + : tail(NoopConstructorTag()) {} + + template <class... P> + explicit constexpr DestructibleUnionImpl(EmplaceTag<0>, P&&... args) + : head(absl::forward<P>(args)...) {} + + template <std::size_t I, class... P> + explicit constexpr DestructibleUnionImpl(EmplaceTag<I>, P&&... args) + : tail(EmplaceTag<I - 1>{}, absl::forward<P>(args)...) {} + + ~DestructibleUnionImpl() {} + + Head head; + TailUnion tail; +}; + +// This union type is destructible even if one or more T are not trivially +// destructible. In the case that all T are trivially destructible, then so is +// this resultant type. +template <class... T> +using DestructibleUnion = + absl::conditional_t<std::is_destructible<Union<T...>>::value, Union<T...>, + DestructibleUnionImpl<T...>>; + +// Deepest base, containing the actual union and the discriminator +template <class H, class... T> +class VariantStateBase { + protected: + using Variant = variant<H, T...>; + + template <class LazyH = H, + class ConstructibleH = absl::enable_if_t< + std::is_default_constructible<LazyH>::value, LazyH>> + constexpr VariantStateBase() noexcept( + std::is_nothrow_default_constructible<ConstructibleH>::value) + : state_(EmplaceTag<0>()), index_(0) {} + + template <std::size_t I, class... P> + explicit constexpr VariantStateBase(EmplaceTag<I> tag, P&&... args) + : state_(tag, absl::forward<P>(args)...), index_(I) {} + + explicit constexpr VariantStateBase(NoopConstructorTag) + : state_(NoopConstructorTag()), index_(variant_npos) {} + + void destroy() {} // Does nothing (shadowed in child if non-trivial) + + DestructibleUnion<H, T...> state_; + std::size_t index_; +}; + +using absl::internal::identity; + +// OverloadSet::Overload() is a unary function which is overloaded to +// take any of the element types of the variant, by reference-to-const. +// The return type of the overload on T is identity<T>, so that you +// can statically determine which overload was called. +// +// Overload() is not defined, so it can only be called in unevaluated +// contexts. +template <typename... Ts> +struct OverloadSet; + +template <typename T, typename... Ts> +struct OverloadSet<T, Ts...> : OverloadSet<Ts...> { + using Base = OverloadSet<Ts...>; + static identity<T> Overload(const T&); + using Base::Overload; +}; + +template <> +struct OverloadSet<> { + // For any case not handled above. + static void Overload(...); +}; + +template <class T> +using LessThanResult = decltype(std::declval<T>() < std::declval<T>()); + +template <class T> +using GreaterThanResult = decltype(std::declval<T>() > std::declval<T>()); + +template <class T> +using LessThanOrEqualResult = decltype(std::declval<T>() <= std::declval<T>()); + +template <class T> +using GreaterThanOrEqualResult = + decltype(std::declval<T>() >= std::declval<T>()); + +template <class T> +using EqualResult = decltype(std::declval<T>() == std::declval<T>()); + +template <class T> +using NotEqualResult = decltype(std::declval<T>() != std::declval<T>()); + +using type_traits_internal::is_detected_convertible; + +template <class... T> +using RequireAllHaveEqualT = absl::enable_if_t< + absl::conjunction<is_detected_convertible<bool, EqualResult, T>...>::value, + bool>; + +template <class... T> +using RequireAllHaveNotEqualT = + absl::enable_if_t<absl::conjunction<is_detected_convertible< + bool, NotEqualResult, T>...>::value, + bool>; + +template <class... T> +using RequireAllHaveLessThanT = + absl::enable_if_t<absl::conjunction<is_detected_convertible< + bool, LessThanResult, T>...>::value, + bool>; + +template <class... T> +using RequireAllHaveLessThanOrEqualT = + absl::enable_if_t<absl::conjunction<is_detected_convertible< + bool, LessThanOrEqualResult, T>...>::value, + bool>; + +template <class... T> +using RequireAllHaveGreaterThanOrEqualT = + absl::enable_if_t<absl::conjunction<is_detected_convertible< + bool, GreaterThanOrEqualResult, T>...>::value, + bool>; + +template <class... T> +using RequireAllHaveGreaterThanT = + absl::enable_if_t<absl::conjunction<is_detected_convertible< + bool, GreaterThanResult, T>...>::value, + bool>; + +// Helper template containing implementations details of variant that can't go +// in the private section. For convenience, this takes the variant type as a +// single template parameter. +template <typename T> +struct VariantHelper; + +template <typename... Ts> +struct VariantHelper<variant<Ts...>> { + // Type metafunction which returns the element type selected if + // OverloadSet::Overload() is well-formed when called with argument type U. + template <typename U> + using BestMatch = decltype( + variant_internal::OverloadSet<Ts...>::Overload(std::declval<U>())); + + // Type metafunction which returns true if OverloadSet::Overload() is + // well-formed when called with argument type U. + // CanAccept can't be just an alias because there is a MSVC bug on parameter + // pack expansion involving decltype. + template <typename U> + struct CanAccept : + std::integral_constant<bool, !std::is_void<BestMatch<U>>::value> {}; + + // Type metafunction which returns true if Other is an instantiation of + // variant, and variants's converting constructor from Other will be + // well-formed. We will use this to remove constructors that would be + // ill-formed from the overload set. + template <typename Other> + struct CanConvertFrom; + + template <typename... Us> + struct CanConvertFrom<variant<Us...>> + : public absl::conjunction<CanAccept<Us>...> {}; +}; + +// A type with nontrivial copy ctor and trivial move ctor. +struct TrivialMoveOnly { + TrivialMoveOnly(TrivialMoveOnly&&) = default; +}; + +// Trait class to detect whether a type is trivially move constructible. +// A union's defaulted copy/move constructor is deleted if any variant member's +// copy/move constructor is nontrivial. +template <typename T> +struct IsTriviallyMoveConstructible: + std::is_move_constructible<Union<T, TrivialMoveOnly>> {}; + +// To guarantee triviality of all special-member functions that can be trivial, +// we use a chain of conditional bases for each one. +// The order of inheritance of bases from child to base are logically: +// +// variant +// VariantCopyAssignBase +// VariantMoveAssignBase +// VariantCopyBase +// VariantMoveBase +// VariantStateBaseDestructor +// VariantStateBase +// +// Note that there is a separate branch at each base that is dependent on +// whether or not that corresponding special-member-function can be trivial in +// the resultant variant type. + +template <class... T> +class VariantStateBaseDestructorNontrivial; + +template <class... T> +class VariantMoveBaseNontrivial; + +template <class... T> +class VariantCopyBaseNontrivial; + +template <class... T> +class VariantMoveAssignBaseNontrivial; + +template <class... T> +class VariantCopyAssignBaseNontrivial; + +// Base that is dependent on whether or not the destructor can be trivial. +template <class... T> +using VariantStateBaseDestructor = + absl::conditional_t<std::is_destructible<Union<T...>>::value, + VariantStateBase<T...>, + VariantStateBaseDestructorNontrivial<T...>>; + +// Base that is dependent on whether or not the move-constructor can be +// implicitly generated by the compiler (trivial or deleted). +// Previously we were using `std::is_move_constructible<Union<T...>>` to check +// whether all Ts have trivial move constructor, but it ran into a GCC bug: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84866 +// So we have to use a different approach (i.e. `HasTrivialMoveConstructor`) to +// work around the bug. +template <class... T> +using VariantMoveBase = absl::conditional_t< + absl::disjunction< + absl::negation<absl::conjunction<std::is_move_constructible<T>...>>, + absl::conjunction<IsTriviallyMoveConstructible<T>...>>::value, + VariantStateBaseDestructor<T...>, VariantMoveBaseNontrivial<T...>>; + +// Base that is dependent on whether or not the copy-constructor can be trivial. +template <class... T> +using VariantCopyBase = absl::conditional_t< + absl::disjunction< + absl::negation<absl::conjunction<std::is_copy_constructible<T>...>>, + std::is_copy_constructible<Union<T...>>>::value, + VariantMoveBase<T...>, VariantCopyBaseNontrivial<T...>>; + +// Base that is dependent on whether or not the move-assign can be trivial. +template <class... T> +using VariantMoveAssignBase = absl::conditional_t< + absl::disjunction< + absl::conjunction<absl::is_move_assignable<Union<T...>>, + std::is_move_constructible<Union<T...>>, + std::is_destructible<Union<T...>>>, + absl::negation<absl::conjunction<std::is_move_constructible<T>..., + // Note: We're not qualifying this with + // absl:: because it doesn't compile + // under MSVC. + is_move_assignable<T>...>>>::value, + VariantCopyBase<T...>, VariantMoveAssignBaseNontrivial<T...>>; + +// Base that is dependent on whether or not the copy-assign can be trivial. +template <class... T> +using VariantCopyAssignBase = absl::conditional_t< + absl::disjunction< + absl::conjunction<absl::is_copy_assignable<Union<T...>>, + std::is_copy_constructible<Union<T...>>, + std::is_destructible<Union<T...>>>, + absl::negation<absl::conjunction<std::is_copy_constructible<T>..., + // Note: We're not qualifying this with + // absl:: because it doesn't compile + // under MSVC. + is_copy_assignable<T>...>>>::value, + VariantMoveAssignBase<T...>, VariantCopyAssignBaseNontrivial<T...>>; + +template <class... T> +using VariantBase = VariantCopyAssignBase<T...>; + +template <class... T> +class VariantStateBaseDestructorNontrivial : protected VariantStateBase<T...> { + private: + using Base = VariantStateBase<T...>; + + protected: + using Base::Base; + + VariantStateBaseDestructorNontrivial() = default; + VariantStateBaseDestructorNontrivial(VariantStateBaseDestructorNontrivial&&) = + default; + VariantStateBaseDestructorNontrivial( + const VariantStateBaseDestructorNontrivial&) = default; + VariantStateBaseDestructorNontrivial& operator=( + VariantStateBaseDestructorNontrivial&&) = default; + VariantStateBaseDestructorNontrivial& operator=( + const VariantStateBaseDestructorNontrivial&) = default; + + struct Destroyer { + template <std::size_t I> + void operator()(SizeT<I> i) const { + using Alternative = + typename absl::variant_alternative<I, variant<T...>>::type; + variant_internal::AccessUnion(self->state_, i).~Alternative(); + } + + void operator()(SizeT<absl::variant_npos> /*i*/) const { + // This space intentionally left blank + } + + VariantStateBaseDestructorNontrivial* self; + }; + + void destroy() { VisitIndices<sizeof...(T)>::Run(Destroyer{this}, index_); } + + ~VariantStateBaseDestructorNontrivial() { destroy(); } + + protected: + using Base::index_; + using Base::state_; +}; + +template <class... T> +class VariantMoveBaseNontrivial : protected VariantStateBaseDestructor<T...> { + private: + using Base = VariantStateBaseDestructor<T...>; + + protected: + using Base::Base; + + struct Construct { + template <std::size_t I> + void operator()(SizeT<I> i) const { + using Alternative = + typename absl::variant_alternative<I, variant<T...>>::type; + ::new (static_cast<void*>(&self->state_)) Alternative( + variant_internal::AccessUnion(absl::move(other->state_), i)); + } + + void operator()(SizeT<absl::variant_npos> /*i*/) const {} + + VariantMoveBaseNontrivial* self; + VariantMoveBaseNontrivial* other; + }; + + VariantMoveBaseNontrivial() = default; + VariantMoveBaseNontrivial(VariantMoveBaseNontrivial&& other) noexcept( + absl::conjunction<std::is_nothrow_move_constructible<T>...>::value) + : Base(NoopConstructorTag()) { + VisitIndices<sizeof...(T)>::Run(Construct{this, &other}, other.index_); + index_ = other.index_; + } + + VariantMoveBaseNontrivial(VariantMoveBaseNontrivial const&) = default; + + VariantMoveBaseNontrivial& operator=(VariantMoveBaseNontrivial&&) = default; + VariantMoveBaseNontrivial& operator=(VariantMoveBaseNontrivial const&) = + default; + + protected: + using Base::index_; + using Base::state_; +}; + +template <class... T> +class VariantCopyBaseNontrivial : protected VariantMoveBase<T...> { + private: + using Base = VariantMoveBase<T...>; + + protected: + using Base::Base; + + VariantCopyBaseNontrivial() = default; + VariantCopyBaseNontrivial(VariantCopyBaseNontrivial&&) = default; + + struct Construct { + template <std::size_t I> + void operator()(SizeT<I> i) const { + using Alternative = + typename absl::variant_alternative<I, variant<T...>>::type; + ::new (static_cast<void*>(&self->state_)) + Alternative(variant_internal::AccessUnion(other->state_, i)); + } + + void operator()(SizeT<absl::variant_npos> /*i*/) const {} + + VariantCopyBaseNontrivial* self; + const VariantCopyBaseNontrivial* other; + }; + + VariantCopyBaseNontrivial(VariantCopyBaseNontrivial const& other) + : Base(NoopConstructorTag()) { + VisitIndices<sizeof...(T)>::Run(Construct{this, &other}, other.index_); + index_ = other.index_; + } + + VariantCopyBaseNontrivial& operator=(VariantCopyBaseNontrivial&&) = default; + VariantCopyBaseNontrivial& operator=(VariantCopyBaseNontrivial const&) = + default; + + protected: + using Base::index_; + using Base::state_; +}; + +template <class... T> +class VariantMoveAssignBaseNontrivial : protected VariantCopyBase<T...> { + friend struct VariantCoreAccess; + + private: + using Base = VariantCopyBase<T...>; + + protected: + using Base::Base; + + VariantMoveAssignBaseNontrivial() = default; + VariantMoveAssignBaseNontrivial(VariantMoveAssignBaseNontrivial&&) = default; + VariantMoveAssignBaseNontrivial(const VariantMoveAssignBaseNontrivial&) = + default; + VariantMoveAssignBaseNontrivial& operator=( + VariantMoveAssignBaseNontrivial const&) = default; + + VariantMoveAssignBaseNontrivial& + operator=(VariantMoveAssignBaseNontrivial&& other) noexcept( + absl::conjunction<std::is_nothrow_move_constructible<T>..., + std::is_nothrow_move_assignable<T>...>::value) { + VisitIndices<sizeof...(T)>::Run( + VariantCoreAccess::MakeMoveAssignVisitor(this, &other), other.index_); + return *this; + } + + protected: + using Base::index_; + using Base::state_; +}; + +template <class... T> +class VariantCopyAssignBaseNontrivial : protected VariantMoveAssignBase<T...> { + friend struct VariantCoreAccess; + + private: + using Base = VariantMoveAssignBase<T...>; + + protected: + using Base::Base; + + VariantCopyAssignBaseNontrivial() = default; + VariantCopyAssignBaseNontrivial(VariantCopyAssignBaseNontrivial&&) = default; + VariantCopyAssignBaseNontrivial(const VariantCopyAssignBaseNontrivial&) = + default; + VariantCopyAssignBaseNontrivial& operator=( + VariantCopyAssignBaseNontrivial&&) = default; + + VariantCopyAssignBaseNontrivial& operator=( + const VariantCopyAssignBaseNontrivial& other) { + VisitIndices<sizeof...(T)>::Run( + VariantCoreAccess::MakeCopyAssignVisitor(this, other), other.index_); + return *this; + } + + protected: + using Base::index_; + using Base::state_; +}; + +//////////////////////////////////////// +// Visitors for Comparison Operations // +//////////////////////////////////////// + +template <class... Types> +struct EqualsOp { + const variant<Types...>* v; + const variant<Types...>* w; + + constexpr bool operator()(SizeT<absl::variant_npos> /*v_i*/) const { + return true; + } + + template <std::size_t I> + constexpr bool operator()(SizeT<I> /*v_i*/) const { + return VariantCoreAccess::Access<I>(*v) == VariantCoreAccess::Access<I>(*w); + } +}; + +template <class... Types> +struct NotEqualsOp { + const variant<Types...>* v; + const variant<Types...>* w; + + constexpr bool operator()(SizeT<absl::variant_npos> /*v_i*/) const { + return false; + } + + template <std::size_t I> + constexpr bool operator()(SizeT<I> /*v_i*/) const { + return VariantCoreAccess::Access<I>(*v) != VariantCoreAccess::Access<I>(*w); + } +}; + +template <class... Types> +struct LessThanOp { + const variant<Types...>* v; + const variant<Types...>* w; + + constexpr bool operator()(SizeT<absl::variant_npos> /*v_i*/) const { + return false; + } + + template <std::size_t I> + constexpr bool operator()(SizeT<I> /*v_i*/) const { + return VariantCoreAccess::Access<I>(*v) < VariantCoreAccess::Access<I>(*w); + } +}; + +template <class... Types> +struct GreaterThanOp { + const variant<Types...>* v; + const variant<Types...>* w; + + constexpr bool operator()(SizeT<absl::variant_npos> /*v_i*/) const { + return false; + } + + template <std::size_t I> + constexpr bool operator()(SizeT<I> /*v_i*/) const { + return VariantCoreAccess::Access<I>(*v) > VariantCoreAccess::Access<I>(*w); + } +}; + +template <class... Types> +struct LessThanOrEqualsOp { + const variant<Types...>* v; + const variant<Types...>* w; + + constexpr bool operator()(SizeT<absl::variant_npos> /*v_i*/) const { + return true; + } + + template <std::size_t I> + constexpr bool operator()(SizeT<I> /*v_i*/) const { + return VariantCoreAccess::Access<I>(*v) <= VariantCoreAccess::Access<I>(*w); + } +}; + +template <class... Types> +struct GreaterThanOrEqualsOp { + const variant<Types...>* v; + const variant<Types...>* w; + + constexpr bool operator()(SizeT<absl::variant_npos> /*v_i*/) const { + return true; + } + + template <std::size_t I> + constexpr bool operator()(SizeT<I> /*v_i*/) const { + return VariantCoreAccess::Access<I>(*v) >= VariantCoreAccess::Access<I>(*w); + } +}; + +// Precondition: v.index() == w.index(); +template <class... Types> +struct SwapSameIndex { + variant<Types...>* v; + variant<Types...>* w; + template <std::size_t I> + void operator()(SizeT<I>) const { + type_traits_internal::Swap(VariantCoreAccess::Access<I>(*v), + VariantCoreAccess::Access<I>(*w)); + } + + void operator()(SizeT<variant_npos>) const {} +}; + +// TODO(calabrese) do this from a different namespace for proper adl usage +template <class... Types> +struct Swap { + variant<Types...>* v; + variant<Types...>* w; + + void generic_swap() const { + variant<Types...> tmp(std::move(*w)); + VariantCoreAccess::Destroy(*w); + VariantCoreAccess::InitFrom(*w, std::move(*v)); + VariantCoreAccess::Destroy(*v); + VariantCoreAccess::InitFrom(*v, std::move(tmp)); + } + + void operator()(SizeT<absl::variant_npos> /*w_i*/) const { + if (!v->valueless_by_exception()) { + generic_swap(); + } + } + + template <std::size_t Wi> + void operator()(SizeT<Wi> /*w_i*/) { + if (v->index() == Wi) { + VisitIndices<sizeof...(Types)>::Run(SwapSameIndex<Types...>{v, w}, Wi); + } else { + generic_swap(); + } + } +}; + +template <typename Variant, typename = void, typename... Ts> +struct VariantHashBase { + VariantHashBase() = delete; + VariantHashBase(const VariantHashBase&) = delete; + VariantHashBase(VariantHashBase&&) = delete; + VariantHashBase& operator=(const VariantHashBase&) = delete; + VariantHashBase& operator=(VariantHashBase&&) = delete; +}; + +struct VariantHashVisitor { + template <typename T> + size_t operator()(const T& t) { + return std::hash<T>{}(t); + } +}; + +template <typename Variant, typename... Ts> +struct VariantHashBase<Variant, + absl::enable_if_t<absl::conjunction< + type_traits_internal::IsHashable<Ts>...>::value>, + Ts...> { + using argument_type = Variant; + using result_type = size_t; + size_t operator()(const Variant& var) const { + type_traits_internal::AssertHashEnabled<Ts...>(); + if (var.valueless_by_exception()) { + return 239799884; + } + size_t result = VisitIndices<variant_size<Variant>::value>::Run( + PerformVisitation<VariantHashVisitor, const Variant&>{ + std::forward_as_tuple(var), VariantHashVisitor{}}, + var.index()); + // Combine the index and the hash result in order to distinguish + // std::variant<int, int> holding the same value as different alternative. + return result ^ var.index(); + } +}; + +} // namespace variant_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // !defined(ABSL_USES_STD_VARIANT) +#endif // ABSL_TYPES_variant_internal_H_ diff --git a/third_party/abseil_cpp/absl/types/optional.h b/third_party/abseil_cpp/absl/types/optional.h new file mode 100644 index 000000000000..61540cfdb26a --- /dev/null +++ b/third_party/abseil_cpp/absl/types/optional.h @@ -0,0 +1,776 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// optional.h +// ----------------------------------------------------------------------------- +// +// This header file defines the `absl::optional` type for holding a value which +// may or may not be present. This type is useful for providing value semantics +// for operations that may either wish to return or hold "something-or-nothing". +// +// Example: +// +// // A common way to signal operation failure is to provide an output +// // parameter and a bool return type: +// bool AcquireResource(const Input&, Resource * out); +// +// // Providing an absl::optional return type provides a cleaner API: +// absl::optional<Resource> AcquireResource(const Input&); +// +// `absl::optional` is a C++11 compatible version of the C++17 `std::optional` +// abstraction and is designed to be a drop-in replacement for code compliant +// with C++17. +#ifndef ABSL_TYPES_OPTIONAL_H_ +#define ABSL_TYPES_OPTIONAL_H_ + +#include "absl/base/config.h" // TODO(calabrese) IWYU removal? +#include "absl/utility/utility.h" + +#ifdef ABSL_USES_STD_OPTIONAL + +#include <optional> // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN +using std::bad_optional_access; +using std::optional; +using std::make_optional; +using std::nullopt_t; +using std::nullopt; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_OPTIONAL + +#include <cassert> +#include <functional> +#include <initializer_list> +#include <type_traits> +#include <utility> + +#include "absl/base/attributes.h" +#include "absl/base/internal/inline_variable.h" +#include "absl/meta/type_traits.h" +#include "absl/types/bad_optional_access.h" +#include "absl/types/internal/optional.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// nullopt_t +// +// Class type for `absl::nullopt` used to indicate an `absl::optional<T>` type +// that does not contain a value. +struct nullopt_t { + // It must not be default-constructible to avoid ambiguity for opt = {}. + explicit constexpr nullopt_t(optional_internal::init_t) noexcept {} +}; + +// nullopt +// +// A tag constant of type `absl::nullopt_t` used to indicate an empty +// `absl::optional` in certain functions, such as construction or assignment. +ABSL_INTERNAL_INLINE_CONSTEXPR(nullopt_t, nullopt, + nullopt_t(optional_internal::init_t())); + +// ----------------------------------------------------------------------------- +// absl::optional +// ----------------------------------------------------------------------------- +// +// A value of type `absl::optional<T>` holds either a value of `T` or an +// "empty" value. When it holds a value of `T`, it stores it as a direct +// sub-object, so `sizeof(optional<T>)` is approximately +// `sizeof(T) + sizeof(bool)`. +// +// This implementation is based on the specification in the latest draft of the +// C++17 `std::optional` specification as of May 2017, section 20.6. +// +// Differences between `absl::optional<T>` and `std::optional<T>` include: +// +// * `constexpr` is not used for non-const member functions. +// (dependency on some differences between C++11 and C++14.) +// * `absl::nullopt` and `absl::in_place` are not declared `constexpr`. We +// need the inline variable support in C++17 for external linkage. +// * Throws `absl::bad_optional_access` instead of +// `std::bad_optional_access`. +// * `make_optional()` cannot be declared `constexpr` due to the absence of +// guaranteed copy elision. +// * The move constructor's `noexcept` specification is stronger, i.e. if the +// default allocator is non-throwing (via setting +// `ABSL_ALLOCATOR_NOTHROW`), it evaluates to `noexcept(true)`, because +// we assume +// a) move constructors should only throw due to allocation failure and +// b) if T's move constructor allocates, it uses the same allocation +// function as the default allocator. +// +template <typename T> +class optional : private optional_internal::optional_data<T>, + private optional_internal::optional_ctor_base< + optional_internal::ctor_copy_traits<T>::traits>, + private optional_internal::optional_assign_base< + optional_internal::assign_copy_traits<T>::traits> { + using data_base = optional_internal::optional_data<T>; + + public: + typedef T value_type; + + // Constructors + + // Constructs an `optional` holding an empty value, NOT a default constructed + // `T`. + constexpr optional() noexcept {} + + // Constructs an `optional` initialized with `nullopt` to hold an empty value. + constexpr optional(nullopt_t) noexcept {} // NOLINT(runtime/explicit) + + // Copy constructor, standard semantics + optional(const optional&) = default; + + // Move constructor, standard semantics + optional(optional&&) = default; + + // Constructs a non-empty `optional` direct-initialized value of type `T` from + // the arguments `std::forward<Args>(args)...` within the `optional`. + // (The `in_place_t` is a tag used to indicate that the contained object + // should be constructed in-place.) + template <typename InPlaceT, typename... Args, + absl::enable_if_t<absl::conjunction< + std::is_same<InPlaceT, in_place_t>, + std::is_constructible<T, Args&&...> >::value>* = nullptr> + constexpr explicit optional(InPlaceT, Args&&... args) + : data_base(in_place_t(), absl::forward<Args>(args)...) {} + + // Constructs a non-empty `optional` direct-initialized value of type `T` from + // the arguments of an initializer_list and `std::forward<Args>(args)...`. + // (The `in_place_t` is a tag used to indicate that the contained object + // should be constructed in-place.) + template <typename U, typename... Args, + typename = typename std::enable_if<std::is_constructible< + T, std::initializer_list<U>&, Args&&...>::value>::type> + constexpr explicit optional(in_place_t, std::initializer_list<U> il, + Args&&... args) + : data_base(in_place_t(), il, absl::forward<Args>(args)...) { + } + + // Value constructor (implicit) + template < + typename U = T, + typename std::enable_if< + absl::conjunction<absl::negation<std::is_same< + in_place_t, typename std::decay<U>::type> >, + absl::negation<std::is_same< + optional<T>, typename std::decay<U>::type> >, + std::is_convertible<U&&, T>, + std::is_constructible<T, U&&> >::value, + bool>::type = false> + constexpr optional(U&& v) : data_base(in_place_t(), absl::forward<U>(v)) {} + + // Value constructor (explicit) + template < + typename U = T, + typename std::enable_if< + absl::conjunction<absl::negation<std::is_same< + in_place_t, typename std::decay<U>::type>>, + absl::negation<std::is_same< + optional<T>, typename std::decay<U>::type>>, + absl::negation<std::is_convertible<U&&, T>>, + std::is_constructible<T, U&&>>::value, + bool>::type = false> + explicit constexpr optional(U&& v) + : data_base(in_place_t(), absl::forward<U>(v)) {} + + // Converting copy constructor (implicit) + template <typename U, + typename std::enable_if< + absl::conjunction< + absl::negation<std::is_same<T, U> >, + std::is_constructible<T, const U&>, + absl::negation< + optional_internal:: + is_constructible_convertible_from_optional<T, U> >, + std::is_convertible<const U&, T> >::value, + bool>::type = false> + optional(const optional<U>& rhs) { + if (rhs) { + this->construct(*rhs); + } + } + + // Converting copy constructor (explicit) + template <typename U, + typename std::enable_if< + absl::conjunction< + absl::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, + absl::negation< + optional_internal:: + is_constructible_convertible_from_optional<T, U>>, + absl::negation<std::is_convertible<const U&, T>>>::value, + bool>::type = false> + explicit optional(const optional<U>& rhs) { + if (rhs) { + this->construct(*rhs); + } + } + + // Converting move constructor (implicit) + template <typename U, + typename std::enable_if< + absl::conjunction< + absl::negation<std::is_same<T, U> >, + std::is_constructible<T, U&&>, + absl::negation< + optional_internal:: + is_constructible_convertible_from_optional<T, U> >, + std::is_convertible<U&&, T> >::value, + bool>::type = false> + optional(optional<U>&& rhs) { + if (rhs) { + this->construct(std::move(*rhs)); + } + } + + // Converting move constructor (explicit) + template < + typename U, + typename std::enable_if< + absl::conjunction< + absl::negation<std::is_same<T, U>>, std::is_constructible<T, U&&>, + absl::negation< + optional_internal::is_constructible_convertible_from_optional< + T, U>>, + absl::negation<std::is_convertible<U&&, T>>>::value, + bool>::type = false> + explicit optional(optional<U>&& rhs) { + if (rhs) { + this->construct(std::move(*rhs)); + } + } + + // Destructor. Trivial if `T` is trivially destructible. + ~optional() = default; + + // Assignment Operators + + // Assignment from `nullopt` + // + // Example: + // + // struct S { int value; }; + // optional<S> opt = absl::nullopt; // Could also use opt = { }; + optional& operator=(nullopt_t) noexcept { + this->destruct(); + return *this; + } + + // Copy assignment operator, standard semantics + optional& operator=(const optional& src) = default; + + // Move assignment operator, standard semantics + optional& operator=(optional&& src) = default; + + // Value assignment operators + template < + typename U = T, + typename = typename std::enable_if<absl::conjunction< + absl::negation< + std::is_same<optional<T>, typename std::decay<U>::type>>, + absl::negation< + absl::conjunction<std::is_scalar<T>, + std::is_same<T, typename std::decay<U>::type>>>, + std::is_constructible<T, U>, std::is_assignable<T&, U>>::value>::type> + optional& operator=(U&& v) { + this->assign(std::forward<U>(v)); + return *this; + } + + template < + typename U, + typename = typename std::enable_if<absl::conjunction< + absl::negation<std::is_same<T, U>>, + std::is_constructible<T, const U&>, std::is_assignable<T&, const U&>, + absl::negation< + optional_internal:: + is_constructible_convertible_assignable_from_optional< + T, U>>>::value>::type> + optional& operator=(const optional<U>& rhs) { + if (rhs) { + this->assign(*rhs); + } else { + this->destruct(); + } + return *this; + } + + template <typename U, + typename = typename std::enable_if<absl::conjunction< + absl::negation<std::is_same<T, U>>, std::is_constructible<T, U>, + std::is_assignable<T&, U>, + absl::negation< + optional_internal:: + is_constructible_convertible_assignable_from_optional< + T, U>>>::value>::type> + optional& operator=(optional<U>&& rhs) { + if (rhs) { + this->assign(std::move(*rhs)); + } else { + this->destruct(); + } + return *this; + } + + // Modifiers + + // optional::reset() + // + // Destroys the inner `T` value of an `absl::optional` if one is present. + ABSL_ATTRIBUTE_REINITIALIZES void reset() noexcept { this->destruct(); } + + // optional::emplace() + // + // (Re)constructs the underlying `T` in-place with the given forwarded + // arguments. + // + // Example: + // + // optional<Foo> opt; + // opt.emplace(arg1,arg2,arg3); // Constructs Foo(arg1,arg2,arg3) + // + // If the optional is non-empty, and the `args` refer to subobjects of the + // current object, then behaviour is undefined, because the current object + // will be destructed before the new object is constructed with `args`. + template <typename... Args, + typename = typename std::enable_if< + std::is_constructible<T, Args&&...>::value>::type> + T& emplace(Args&&... args) { + this->destruct(); + this->construct(std::forward<Args>(args)...); + return reference(); + } + + // Emplace reconstruction overload for an initializer list and the given + // forwarded arguments. + // + // Example: + // + // struct Foo { + // Foo(std::initializer_list<int>); + // }; + // + // optional<Foo> opt; + // opt.emplace({1,2,3}); // Constructs Foo({1,2,3}) + template <typename U, typename... Args, + typename = typename std::enable_if<std::is_constructible< + T, std::initializer_list<U>&, Args&&...>::value>::type> + T& emplace(std::initializer_list<U> il, Args&&... args) { + this->destruct(); + this->construct(il, std::forward<Args>(args)...); + return reference(); + } + + // Swaps + + // Swap, standard semantics + void swap(optional& rhs) noexcept( + std::is_nothrow_move_constructible<T>::value&& + type_traits_internal::IsNothrowSwappable<T>::value) { + if (*this) { + if (rhs) { + type_traits_internal::Swap(**this, *rhs); + } else { + rhs.construct(std::move(**this)); + this->destruct(); + } + } else { + if (rhs) { + this->construct(std::move(*rhs)); + rhs.destruct(); + } else { + // No effect (swap(disengaged, disengaged)). + } + } + } + + // Observers + + // optional::operator->() + // + // Accesses the underlying `T` value's member `m` of an `optional`. If the + // `optional` is empty, behavior is undefined. + // + // If you need myOpt->foo in constexpr, use (*myOpt).foo instead. + const T* operator->() const { + ABSL_HARDENING_ASSERT(this->engaged_); + return std::addressof(this->data_); + } + T* operator->() { + ABSL_HARDENING_ASSERT(this->engaged_); + return std::addressof(this->data_); + } + + // optional::operator*() + // + // Accesses the underlying `T` value of an `optional`. If the `optional` is + // empty, behavior is undefined. + constexpr const T& operator*() const& { + return ABSL_HARDENING_ASSERT(this->engaged_), reference(); + } + T& operator*() & { + ABSL_HARDENING_ASSERT(this->engaged_); + return reference(); + } + constexpr const T&& operator*() const && { + return ABSL_HARDENING_ASSERT(this->engaged_), absl::move(reference()); + } + T&& operator*() && { + ABSL_HARDENING_ASSERT(this->engaged_); + return std::move(reference()); + } + + // optional::operator bool() + // + // Returns false if and only if the `optional` is empty. + // + // if (opt) { + // // do something with *opt or opt->; + // } else { + // // opt is empty. + // } + // + constexpr explicit operator bool() const noexcept { return this->engaged_; } + + // optional::has_value() + // + // Determines whether the `optional` contains a value. Returns `false` if and + // only if `*this` is empty. + constexpr bool has_value() const noexcept { return this->engaged_; } + +// Suppress bogus warning on MSVC: MSVC complains call to reference() after +// throw_bad_optional_access() is unreachable. +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4702) +#endif // _MSC_VER + // optional::value() + // + // Returns a reference to an `optional`s underlying value. The constness + // and lvalue/rvalue-ness of the `optional` is preserved to the view of + // the `T` sub-object. Throws `absl::bad_optional_access` when the `optional` + // is empty. + constexpr const T& value() const & { + return static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference()); + } + T& value() & { + return static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference()); + } + T&& value() && { // NOLINT(build/c++11) + return std::move( + static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference())); + } + constexpr const T&& value() const && { // NOLINT(build/c++11) + return absl::move( + static_cast<bool>(*this) + ? reference() + : (optional_internal::throw_bad_optional_access(), reference())); + } +#ifdef _MSC_VER +#pragma warning(pop) +#endif // _MSC_VER + + // optional::value_or() + // + // Returns either the value of `T` or a passed default `v` if the `optional` + // is empty. + template <typename U> + constexpr T value_or(U&& v) const& { + static_assert(std::is_copy_constructible<value_type>::value, + "optional<T>::value_or: T must be copy constructible"); + static_assert(std::is_convertible<U&&, value_type>::value, + "optional<T>::value_or: U must be convertible to T"); + return static_cast<bool>(*this) + ? **this + : static_cast<T>(absl::forward<U>(v)); + } + template <typename U> + T value_or(U&& v) && { // NOLINT(build/c++11) + static_assert(std::is_move_constructible<value_type>::value, + "optional<T>::value_or: T must be move constructible"); + static_assert(std::is_convertible<U&&, value_type>::value, + "optional<T>::value_or: U must be convertible to T"); + return static_cast<bool>(*this) ? std::move(**this) + : static_cast<T>(std::forward<U>(v)); + } + + private: + // Private accessors for internal storage viewed as reference to T. + constexpr const T& reference() const { return this->data_; } + T& reference() { return this->data_; } + + // T constraint checks. You can't have an optional of nullopt_t, in_place_t + // or a reference. + static_assert( + !std::is_same<nullopt_t, typename std::remove_cv<T>::type>::value, + "optional<nullopt_t> is not allowed."); + static_assert( + !std::is_same<in_place_t, typename std::remove_cv<T>::type>::value, + "optional<in_place_t> is not allowed."); + static_assert(!std::is_reference<T>::value, + "optional<reference> is not allowed."); +}; + +// Non-member functions + +// swap() +// +// Performs a swap between two `absl::optional` objects, using standard +// semantics. +template <typename T, typename std::enable_if< + std::is_move_constructible<T>::value && + type_traits_internal::IsSwappable<T>::value, + bool>::type = false> +void swap(optional<T>& a, optional<T>& b) noexcept(noexcept(a.swap(b))) { + a.swap(b); +} + +// make_optional() +// +// Creates a non-empty `optional<T>` where the type of `T` is deduced. An +// `absl::optional` can also be explicitly instantiated with +// `make_optional<T>(v)`. +// +// Note: `make_optional()` constructions may be declared `constexpr` for +// trivially copyable types `T`. Non-trivial types require copy elision +// support in C++17 for `make_optional` to support `constexpr` on such +// non-trivial types. +// +// Example: +// +// constexpr absl::optional<int> opt = absl::make_optional(1); +// static_assert(opt.value() == 1, ""); +template <typename T> +constexpr optional<typename std::decay<T>::type> make_optional(T&& v) { + return optional<typename std::decay<T>::type>(absl::forward<T>(v)); +} + +template <typename T, typename... Args> +constexpr optional<T> make_optional(Args&&... args) { + return optional<T>(in_place_t(), absl::forward<Args>(args)...); +} + +template <typename T, typename U, typename... Args> +constexpr optional<T> make_optional(std::initializer_list<U> il, + Args&&... args) { + return optional<T>(in_place_t(), il, + absl::forward<Args>(args)...); +} + +// Relational operators [optional.relops] + +// Empty optionals are considered equal to each other and less than non-empty +// optionals. Supports relations between optional<T> and optional<U>, between +// optional<T> and U, and between optional<T> and nullopt. +// +// Note: We're careful to support T having non-bool relationals. + +// Requires: The expression, e.g. "*x == *y" shall be well-formed and its result +// shall be convertible to bool. +// The C++17 (N4606) "Returns:" statements are translated into +// code in an obvious way here, and the original text retained as function docs. +// Returns: If bool(x) != bool(y), false; otherwise if bool(x) == false, true; +// otherwise *x == *y. +template <typename T, typename U> +constexpr auto operator==(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x == *y)) { + return static_cast<bool>(x) != static_cast<bool>(y) + ? false + : static_cast<bool>(x) == false ? true + : static_cast<bool>(*x == *y); +} + +// Returns: If bool(x) != bool(y), true; otherwise, if bool(x) == false, false; +// otherwise *x != *y. +template <typename T, typename U> +constexpr auto operator!=(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x != *y)) { + return static_cast<bool>(x) != static_cast<bool>(y) + ? true + : static_cast<bool>(x) == false ? false + : static_cast<bool>(*x != *y); +} +// Returns: If !y, false; otherwise, if !x, true; otherwise *x < *y. +template <typename T, typename U> +constexpr auto operator<(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x < *y)) { + return !y ? false : !x ? true : static_cast<bool>(*x < *y); +} +// Returns: If !x, false; otherwise, if !y, true; otherwise *x > *y. +template <typename T, typename U> +constexpr auto operator>(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x > *y)) { + return !x ? false : !y ? true : static_cast<bool>(*x > *y); +} +// Returns: If !x, true; otherwise, if !y, false; otherwise *x <= *y. +template <typename T, typename U> +constexpr auto operator<=(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x <= *y)) { + return !x ? true : !y ? false : static_cast<bool>(*x <= *y); +} +// Returns: If !y, true; otherwise, if !x, false; otherwise *x >= *y. +template <typename T, typename U> +constexpr auto operator>=(const optional<T>& x, const optional<U>& y) + -> decltype(optional_internal::convertible_to_bool(*x >= *y)) { + return !y ? true : !x ? false : static_cast<bool>(*x >= *y); +} + +// Comparison with nullopt [optional.nullops] +// The C++17 (N4606) "Returns:" statements are used directly here. +template <typename T> +constexpr bool operator==(const optional<T>& x, nullopt_t) noexcept { + return !x; +} +template <typename T> +constexpr bool operator==(nullopt_t, const optional<T>& x) noexcept { + return !x; +} +template <typename T> +constexpr bool operator!=(const optional<T>& x, nullopt_t) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator!=(nullopt_t, const optional<T>& x) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator<(const optional<T>&, nullopt_t) noexcept { + return false; +} +template <typename T> +constexpr bool operator<(nullopt_t, const optional<T>& x) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator<=(const optional<T>& x, nullopt_t) noexcept { + return !x; +} +template <typename T> +constexpr bool operator<=(nullopt_t, const optional<T>&) noexcept { + return true; +} +template <typename T> +constexpr bool operator>(const optional<T>& x, nullopt_t) noexcept { + return static_cast<bool>(x); +} +template <typename T> +constexpr bool operator>(nullopt_t, const optional<T>&) noexcept { + return false; +} +template <typename T> +constexpr bool operator>=(const optional<T>&, nullopt_t) noexcept { + return true; +} +template <typename T> +constexpr bool operator>=(nullopt_t, const optional<T>& x) noexcept { + return !x; +} + +// Comparison with T [optional.comp_with_t] + +// Requires: The expression, e.g. "*x == v" shall be well-formed and its result +// shall be convertible to bool. +// The C++17 (N4606) "Equivalent to:" statements are used directly here. +template <typename T, typename U> +constexpr auto operator==(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x == v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x == v) : false; +} +template <typename T, typename U> +constexpr auto operator==(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v == *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v == *x) : false; +} +template <typename T, typename U> +constexpr auto operator!=(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x != v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x != v) : true; +} +template <typename T, typename U> +constexpr auto operator!=(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v != *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v != *x) : true; +} +template <typename T, typename U> +constexpr auto operator<(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x < v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x < v) : true; +} +template <typename T, typename U> +constexpr auto operator<(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v < *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v < *x) : false; +} +template <typename T, typename U> +constexpr auto operator<=(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x <= v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x <= v) : true; +} +template <typename T, typename U> +constexpr auto operator<=(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v <= *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v <= *x) : false; +} +template <typename T, typename U> +constexpr auto operator>(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x > v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x > v) : false; +} +template <typename T, typename U> +constexpr auto operator>(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v > *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v > *x) : true; +} +template <typename T, typename U> +constexpr auto operator>=(const optional<T>& x, const U& v) + -> decltype(optional_internal::convertible_to_bool(*x >= v)) { + return static_cast<bool>(x) ? static_cast<bool>(*x >= v) : false; +} +template <typename T, typename U> +constexpr auto operator>=(const U& v, const optional<T>& x) + -> decltype(optional_internal::convertible_to_bool(v >= *x)) { + return static_cast<bool>(x) ? static_cast<bool>(v >= *x) : true; +} + +ABSL_NAMESPACE_END +} // namespace absl + +namespace std { + +// std::hash specialization for absl::optional. +template <typename T> +struct hash<absl::optional<T> > + : absl::optional_internal::optional_hash_base<T> {}; + +} // namespace std + +#undef ABSL_MSVC_CONSTEXPR_BUG_IN_UNION_LIKE_CLASS + +#endif // ABSL_USES_STD_OPTIONAL + +#endif // ABSL_TYPES_OPTIONAL_H_ diff --git a/third_party/abseil_cpp/absl/types/optional_exception_safety_test.cc b/third_party/abseil_cpp/absl/types/optional_exception_safety_test.cc new file mode 100644 index 000000000000..8e5fe851dbb7 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/optional_exception_safety_test.cc @@ -0,0 +1,292 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/optional.h" + +#include "absl/base/config.h" + +// This test is a no-op when absl::optional is an alias for std::optional and +// when exceptions are not enabled. +#if !defined(ABSL_USES_STD_OPTIONAL) && defined(ABSL_HAVE_EXCEPTIONS) + +#include "gtest/gtest.h" +#include "absl/base/internal/exception_safety_testing.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +namespace { + +using ::testing::AssertionFailure; +using ::testing::AssertionResult; +using ::testing::AssertionSuccess; +using ::testing::MakeExceptionSafetyTester; + +using Thrower = testing::ThrowingValue<testing::TypeSpec::kEverythingThrows>; +using Optional = absl::optional<Thrower>; + +using MoveThrower = testing::ThrowingValue<testing::TypeSpec::kNoThrowMove>; +using MoveOptional = absl::optional<MoveThrower>; + +constexpr int kInitialInteger = 5; +constexpr int kUpdatedInteger = 10; + +template <typename OptionalT> +bool ValueThrowsBadOptionalAccess(const OptionalT& optional) try { + return (static_cast<void>(optional.value()), false); +} catch (const absl::bad_optional_access&) { + return true; +} + +template <typename OptionalT> +AssertionResult OptionalInvariants(OptionalT* optional_ptr) { + // Check the current state post-throw for validity + auto& optional = *optional_ptr; + + if (optional.has_value() && ValueThrowsBadOptionalAccess(optional)) { + return AssertionFailure() + << "Optional with value should not throw bad_optional_access when " + "accessing the value."; + } + if (!optional.has_value() && !ValueThrowsBadOptionalAccess(optional)) { + return AssertionFailure() + << "Optional without a value should throw bad_optional_access when " + "accessing the value."; + } + + // Reset to a known state + optional.reset(); + + // Confirm that the known post-reset state is valid + if (optional.has_value()) { + return AssertionFailure() + << "Optional should not contain a value after being reset."; + } + if (!ValueThrowsBadOptionalAccess(optional)) { + return AssertionFailure() << "Optional should throw bad_optional_access " + "when accessing the value after being reset."; + } + + return AssertionSuccess(); +} + +template <typename OptionalT> +AssertionResult CheckDisengaged(OptionalT* optional_ptr) { + auto& optional = *optional_ptr; + + if (optional.has_value()) { + return AssertionFailure() + << "Expected optional to not contain a value but a value was found."; + } + + return AssertionSuccess(); +} + +template <typename OptionalT> +AssertionResult CheckEngaged(OptionalT* optional_ptr) { + auto& optional = *optional_ptr; + + if (!optional.has_value()) { + return AssertionFailure() + << "Expected optional to contain a value but no value was found."; + } + + return AssertionSuccess(); +} + +TEST(OptionalExceptionSafety, ThrowingConstructors) { + auto thrower_nonempty = Optional(Thrower(kInitialInteger)); + testing::TestThrowingCtor<Optional>(thrower_nonempty); + + auto integer_nonempty = absl::optional<int>(kInitialInteger); + testing::TestThrowingCtor<Optional>(integer_nonempty); + testing::TestThrowingCtor<Optional>(std::move(integer_nonempty)); // NOLINT + + testing::TestThrowingCtor<Optional>(kInitialInteger); + using ThrowerVec = std::vector<Thrower, testing::ThrowingAllocator<Thrower>>; + testing::TestThrowingCtor<absl::optional<ThrowerVec>>( + absl::in_place, + std::initializer_list<Thrower>{Thrower(), Thrower(), Thrower()}, + testing::ThrowingAllocator<Thrower>()); +} + +TEST(OptionalExceptionSafety, NothrowConstructors) { + // This constructor is marked noexcept. If it throws, the program will + // terminate. + testing::TestThrowingCtor<MoveOptional>(MoveOptional(kUpdatedInteger)); +} + +TEST(OptionalExceptionSafety, Emplace) { + // Test the basic guarantee plus test the result of optional::has_value() + // is false in all cases + auto disengaged_test = MakeExceptionSafetyTester().WithContracts( + OptionalInvariants<Optional>, CheckDisengaged<Optional>); + auto disengaged_test_empty = disengaged_test.WithInitialValue(Optional()); + auto disengaged_test_nonempty = + disengaged_test.WithInitialValue(Optional(kInitialInteger)); + + auto emplace_thrower_directly = [](Optional* optional_ptr) { + optional_ptr->emplace(kUpdatedInteger); + }; + EXPECT_TRUE(disengaged_test_empty.Test(emplace_thrower_directly)); + EXPECT_TRUE(disengaged_test_nonempty.Test(emplace_thrower_directly)); + + auto emplace_thrower_copy = [](Optional* optional_ptr) { + auto thrower = Thrower(kUpdatedInteger, testing::nothrow_ctor); + optional_ptr->emplace(thrower); + }; + EXPECT_TRUE(disengaged_test_empty.Test(emplace_thrower_copy)); + EXPECT_TRUE(disengaged_test_nonempty.Test(emplace_thrower_copy)); +} + +TEST(OptionalExceptionSafety, EverythingThrowsSwap) { + // Test the basic guarantee plus test the result of optional::has_value() + // remains the same + auto test = + MakeExceptionSafetyTester().WithContracts(OptionalInvariants<Optional>); + auto disengaged_test_empty = test.WithInitialValue(Optional()) + .WithContracts(CheckDisengaged<Optional>); + auto engaged_test_nonempty = test.WithInitialValue(Optional(kInitialInteger)) + .WithContracts(CheckEngaged<Optional>); + + auto swap_empty = [](Optional* optional_ptr) { + auto empty = Optional(); + optional_ptr->swap(empty); + }; + EXPECT_TRUE(engaged_test_nonempty.Test(swap_empty)); + + auto swap_nonempty = [](Optional* optional_ptr) { + auto nonempty = + Optional(absl::in_place, kUpdatedInteger, testing::nothrow_ctor); + optional_ptr->swap(nonempty); + }; + EXPECT_TRUE(disengaged_test_empty.Test(swap_nonempty)); + EXPECT_TRUE(engaged_test_nonempty.Test(swap_nonempty)); +} + +TEST(OptionalExceptionSafety, NoThrowMoveSwap) { + // Tests the nothrow guarantee for optional of T with non-throwing move + { + auto empty = MoveOptional(); + auto nonempty = MoveOptional(kInitialInteger); + EXPECT_TRUE(testing::TestNothrowOp([&]() { nonempty.swap(empty); })); + } + { + auto nonempty = MoveOptional(kUpdatedInteger); + auto empty = MoveOptional(); + EXPECT_TRUE(testing::TestNothrowOp([&]() { empty.swap(nonempty); })); + } + { + auto nonempty_from = MoveOptional(kUpdatedInteger); + auto nonempty_to = MoveOptional(kInitialInteger); + EXPECT_TRUE( + testing::TestNothrowOp([&]() { nonempty_to.swap(nonempty_from); })); + } +} + +TEST(OptionalExceptionSafety, CopyAssign) { + // Test the basic guarantee plus test the result of optional::has_value() + // remains the same + auto test = + MakeExceptionSafetyTester().WithContracts(OptionalInvariants<Optional>); + auto disengaged_test_empty = test.WithInitialValue(Optional()) + .WithContracts(CheckDisengaged<Optional>); + auto engaged_test_nonempty = test.WithInitialValue(Optional(kInitialInteger)) + .WithContracts(CheckEngaged<Optional>); + + auto copyassign_nonempty = [](Optional* optional_ptr) { + auto nonempty = + Optional(absl::in_place, kUpdatedInteger, testing::nothrow_ctor); + *optional_ptr = nonempty; + }; + EXPECT_TRUE(disengaged_test_empty.Test(copyassign_nonempty)); + EXPECT_TRUE(engaged_test_nonempty.Test(copyassign_nonempty)); + + auto copyassign_thrower = [](Optional* optional_ptr) { + auto thrower = Thrower(kUpdatedInteger, testing::nothrow_ctor); + *optional_ptr = thrower; + }; + EXPECT_TRUE(disengaged_test_empty.Test(copyassign_thrower)); + EXPECT_TRUE(engaged_test_nonempty.Test(copyassign_thrower)); +} + +TEST(OptionalExceptionSafety, MoveAssign) { + // Test the basic guarantee plus test the result of optional::has_value() + // remains the same + auto test = + MakeExceptionSafetyTester().WithContracts(OptionalInvariants<Optional>); + auto disengaged_test_empty = test.WithInitialValue(Optional()) + .WithContracts(CheckDisengaged<Optional>); + auto engaged_test_nonempty = test.WithInitialValue(Optional(kInitialInteger)) + .WithContracts(CheckEngaged<Optional>); + + auto moveassign_empty = [](Optional* optional_ptr) { + auto empty = Optional(); + *optional_ptr = std::move(empty); + }; + EXPECT_TRUE(engaged_test_nonempty.Test(moveassign_empty)); + + auto moveassign_nonempty = [](Optional* optional_ptr) { + auto nonempty = + Optional(absl::in_place, kUpdatedInteger, testing::nothrow_ctor); + *optional_ptr = std::move(nonempty); + }; + EXPECT_TRUE(disengaged_test_empty.Test(moveassign_nonempty)); + EXPECT_TRUE(engaged_test_nonempty.Test(moveassign_nonempty)); + + auto moveassign_thrower = [](Optional* optional_ptr) { + auto thrower = Thrower(kUpdatedInteger, testing::nothrow_ctor); + *optional_ptr = std::move(thrower); + }; + EXPECT_TRUE(disengaged_test_empty.Test(moveassign_thrower)); + EXPECT_TRUE(engaged_test_nonempty.Test(moveassign_thrower)); +} + +TEST(OptionalExceptionSafety, NothrowMoveAssign) { + // Tests the nothrow guarantee for optional of T with non-throwing move + { + auto empty = MoveOptional(); + auto nonempty = MoveOptional(kInitialInteger); + EXPECT_TRUE(testing::TestNothrowOp([&]() { nonempty = std::move(empty); })); + } + { + auto nonempty = MoveOptional(kInitialInteger); + auto empty = MoveOptional(); + EXPECT_TRUE(testing::TestNothrowOp([&]() { empty = std::move(nonempty); })); + } + { + auto nonempty_from = MoveOptional(kUpdatedInteger); + auto nonempty_to = MoveOptional(kInitialInteger); + EXPECT_TRUE(testing::TestNothrowOp( + [&]() { nonempty_to = std::move(nonempty_from); })); + } + { + auto thrower = MoveThrower(kUpdatedInteger); + auto empty = MoveOptional(); + EXPECT_TRUE(testing::TestNothrowOp([&]() { empty = std::move(thrower); })); + } + { + auto thrower = MoveThrower(kUpdatedInteger); + auto nonempty = MoveOptional(kInitialInteger); + EXPECT_TRUE( + testing::TestNothrowOp([&]() { nonempty = std::move(thrower); })); + } +} + +} // namespace + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // #if !defined(ABSL_USES_STD_OPTIONAL) && defined(ABSL_HAVE_EXCEPTIONS) diff --git a/third_party/abseil_cpp/absl/types/optional_test.cc b/third_party/abseil_cpp/absl/types/optional_test.cc new file mode 100644 index 000000000000..7ef142cb9937 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/optional_test.cc @@ -0,0 +1,1659 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/optional.h" + +// This test is a no-op when absl::optional is an alias for std::optional. +#if !defined(ABSL_USES_STD_OPTIONAL) + +#include <string> +#include <type_traits> +#include <utility> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" + +struct Hashable {}; + +namespace std { +template <> +struct hash<Hashable> { + size_t operator()(const Hashable&) { return 0; } +}; +} // namespace std + +struct NonHashable {}; + +namespace { + +std::string TypeQuals(std::string&) { return "&"; } +std::string TypeQuals(std::string&&) { return "&&"; } +std::string TypeQuals(const std::string&) { return "c&"; } +std::string TypeQuals(const std::string&&) { return "c&&"; } + +struct StructorListener { + int construct0 = 0; + int construct1 = 0; + int construct2 = 0; + int listinit = 0; + int copy = 0; + int move = 0; + int copy_assign = 0; + int move_assign = 0; + int destruct = 0; + int volatile_copy = 0; + int volatile_move = 0; + int volatile_copy_assign = 0; + int volatile_move_assign = 0; +}; + +// Suppress MSVC warnings. +// 4521: multiple copy constructors specified +// 4522: multiple assignment operators specified +// We wrote multiple of them to test that the correct overloads are selected. +#ifdef _MSC_VER +#pragma warning( push ) +#pragma warning( disable : 4521) +#pragma warning( disable : 4522) +#endif +struct Listenable { + static StructorListener* listener; + + Listenable() { ++listener->construct0; } + explicit Listenable(int /*unused*/) { ++listener->construct1; } + Listenable(int /*unused*/, int /*unused*/) { ++listener->construct2; } + Listenable(std::initializer_list<int> /*unused*/) { ++listener->listinit; } + Listenable(const Listenable& /*unused*/) { ++listener->copy; } + Listenable(const volatile Listenable& /*unused*/) { + ++listener->volatile_copy; + } + Listenable(volatile Listenable&& /*unused*/) { ++listener->volatile_move; } + Listenable(Listenable&& /*unused*/) { ++listener->move; } + Listenable& operator=(const Listenable& /*unused*/) { + ++listener->copy_assign; + return *this; + } + Listenable& operator=(Listenable&& /*unused*/) { + ++listener->move_assign; + return *this; + } + // use void return type instead of volatile T& to work around GCC warning + // when the assignment's returned reference is ignored. + void operator=(const volatile Listenable& /*unused*/) volatile { + ++listener->volatile_copy_assign; + } + void operator=(volatile Listenable&& /*unused*/) volatile { + ++listener->volatile_move_assign; + } + ~Listenable() { ++listener->destruct; } +}; +#ifdef _MSC_VER +#pragma warning( pop ) +#endif + +StructorListener* Listenable::listener = nullptr; + +// ABSL_HAVE_NO_CONSTEXPR_INITIALIZER_LIST is defined to 1 when the standard +// library implementation doesn't marked initializer_list's default constructor +// constexpr. The C++11 standard doesn't specify constexpr on it, but C++14 +// added it. However, libstdc++ 4.7 marked it constexpr. +#if defined(_LIBCPP_VERSION) && \ + (_LIBCPP_STD_VER <= 11 || defined(_LIBCPP_HAS_NO_CXX14_CONSTEXPR)) +#define ABSL_HAVE_NO_CONSTEXPR_INITIALIZER_LIST 1 +#endif + +struct ConstexprType { + enum CtorTypes { + kCtorDefault, + kCtorInt, + kCtorInitializerList, + kCtorConstChar + }; + constexpr ConstexprType() : x(kCtorDefault) {} + constexpr explicit ConstexprType(int i) : x(kCtorInt) {} +#ifndef ABSL_HAVE_NO_CONSTEXPR_INITIALIZER_LIST + constexpr ConstexprType(std::initializer_list<int> il) + : x(kCtorInitializerList) {} +#endif + constexpr ConstexprType(const char*) // NOLINT(runtime/explicit) + : x(kCtorConstChar) {} + int x; +}; + +struct Copyable { + Copyable() {} + Copyable(const Copyable&) {} + Copyable& operator=(const Copyable&) { return *this; } +}; + +struct MoveableThrow { + MoveableThrow() {} + MoveableThrow(MoveableThrow&&) {} + MoveableThrow& operator=(MoveableThrow&&) { return *this; } +}; + +struct MoveableNoThrow { + MoveableNoThrow() {} + MoveableNoThrow(MoveableNoThrow&&) noexcept {} + MoveableNoThrow& operator=(MoveableNoThrow&&) noexcept { return *this; } +}; + +struct NonMovable { + NonMovable() {} + NonMovable(const NonMovable&) = delete; + NonMovable& operator=(const NonMovable&) = delete; + NonMovable(NonMovable&&) = delete; + NonMovable& operator=(NonMovable&&) = delete; +}; + +struct NoDefault { + NoDefault() = delete; + NoDefault(const NoDefault&) {} + NoDefault& operator=(const NoDefault&) { return *this; } +}; + +struct ConvertsFromInPlaceT { + ConvertsFromInPlaceT(absl::in_place_t) {} // NOLINT +}; + +TEST(optionalTest, DefaultConstructor) { + absl::optional<int> empty; + EXPECT_FALSE(empty); + constexpr absl::optional<int> cempty; + static_assert(!cempty.has_value(), ""); + EXPECT_TRUE( + std::is_nothrow_default_constructible<absl::optional<int>>::value); +} + +TEST(optionalTest, nulloptConstructor) { + absl::optional<int> empty(absl::nullopt); + EXPECT_FALSE(empty); + constexpr absl::optional<int> cempty{absl::nullopt}; + static_assert(!cempty.has_value(), ""); + EXPECT_TRUE((std::is_nothrow_constructible<absl::optional<int>, + absl::nullopt_t>::value)); +} + +TEST(optionalTest, CopyConstructor) { + { + absl::optional<int> empty, opt42 = 42; + absl::optional<int> empty_copy(empty); + EXPECT_FALSE(empty_copy); + absl::optional<int> opt42_copy(opt42); + EXPECT_TRUE(opt42_copy); + EXPECT_EQ(42, *opt42_copy); + } + { + absl::optional<const int> empty, opt42 = 42; + absl::optional<const int> empty_copy(empty); + EXPECT_FALSE(empty_copy); + absl::optional<const int> opt42_copy(opt42); + EXPECT_TRUE(opt42_copy); + EXPECT_EQ(42, *opt42_copy); + } + { + absl::optional<volatile int> empty, opt42 = 42; + absl::optional<volatile int> empty_copy(empty); + EXPECT_FALSE(empty_copy); + absl::optional<volatile int> opt42_copy(opt42); + EXPECT_TRUE(opt42_copy); + EXPECT_EQ(42, *opt42_copy); + } + // test copyablility + EXPECT_TRUE(std::is_copy_constructible<absl::optional<int>>::value); + EXPECT_TRUE(std::is_copy_constructible<absl::optional<Copyable>>::value); + EXPECT_FALSE( + std::is_copy_constructible<absl::optional<MoveableThrow>>::value); + EXPECT_FALSE( + std::is_copy_constructible<absl::optional<MoveableNoThrow>>::value); + EXPECT_FALSE(std::is_copy_constructible<absl::optional<NonMovable>>::value); + + EXPECT_FALSE( + absl::is_trivially_copy_constructible<absl::optional<Copyable>>::value); +#if defined(ABSL_USES_STD_OPTIONAL) && defined(__GLIBCXX__) + // libstdc++ std::optional implementation (as of 7.2) has a bug: when T is + // trivially copyable, optional<T> is not trivially copyable (due to one of + // its base class is unconditionally nontrivial). +#define ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG 1 +#endif +#ifndef ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG + EXPECT_TRUE( + absl::is_trivially_copy_constructible<absl::optional<int>>::value); + EXPECT_TRUE( + absl::is_trivially_copy_constructible<absl::optional<const int>>::value); +#ifndef _MSC_VER + // See defect report "Trivial copy/move constructor for class with volatile + // member" at + // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#2094 + // A class with non-static data member of volatile-qualified type should still + // have a trivial copy constructor if the data member is trivial. + // Also a cv-qualified scalar type should be trivially copyable. + EXPECT_TRUE(absl::is_trivially_copy_constructible< + absl::optional<volatile int>>::value); +#endif // _MSC_VER +#endif // ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG + + // constexpr copy constructor for trivially copyable types + { + constexpr absl::optional<int> o1; + constexpr absl::optional<int> o2 = o1; + static_assert(!o2, ""); + } + { + constexpr absl::optional<int> o1 = 42; + constexpr absl::optional<int> o2 = o1; + static_assert(o2, ""); + static_assert(*o2 == 42, ""); + } + { + struct TrivialCopyable { + constexpr TrivialCopyable() : x(0) {} + constexpr explicit TrivialCopyable(int i) : x(i) {} + int x; + }; + constexpr absl::optional<TrivialCopyable> o1(42); + constexpr absl::optional<TrivialCopyable> o2 = o1; + static_assert(o2, ""); + static_assert((*o2).x == 42, ""); +#ifndef ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG + EXPECT_TRUE(absl::is_trivially_copy_constructible< + absl::optional<TrivialCopyable>>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible< + absl::optional<const TrivialCopyable>>::value); +#endif + // When testing with VS 2017 15.3, there seems to be a bug in MSVC + // std::optional when T is volatile-qualified. So skipping this test. + // Bug report: + // https://connect.microsoft.com/VisualStudio/feedback/details/3142534 +#if defined(ABSL_USES_STD_OPTIONAL) && defined(_MSC_VER) && _MSC_VER >= 1911 +#define ABSL_MSVC_OPTIONAL_VOLATILE_COPY_BUG 1 +#endif +#ifndef ABSL_MSVC_OPTIONAL_VOLATILE_COPY_BUG + EXPECT_FALSE(std::is_copy_constructible< + absl::optional<volatile TrivialCopyable>>::value); +#endif + } +} + +TEST(optionalTest, MoveConstructor) { + absl::optional<int> empty, opt42 = 42; + absl::optional<int> empty_move(std::move(empty)); + EXPECT_FALSE(empty_move); + absl::optional<int> opt42_move(std::move(opt42)); + EXPECT_TRUE(opt42_move); + EXPECT_EQ(42, opt42_move); + // test movability + EXPECT_TRUE(std::is_move_constructible<absl::optional<int>>::value); + EXPECT_TRUE(std::is_move_constructible<absl::optional<Copyable>>::value); + EXPECT_TRUE(std::is_move_constructible<absl::optional<MoveableThrow>>::value); + EXPECT_TRUE( + std::is_move_constructible<absl::optional<MoveableNoThrow>>::value); + EXPECT_FALSE(std::is_move_constructible<absl::optional<NonMovable>>::value); + // test noexcept + EXPECT_TRUE(std::is_nothrow_move_constructible<absl::optional<int>>::value); +#ifndef ABSL_USES_STD_OPTIONAL + EXPECT_EQ( + absl::default_allocator_is_nothrow::value, + std::is_nothrow_move_constructible<absl::optional<MoveableThrow>>::value); +#endif + EXPECT_TRUE(std::is_nothrow_move_constructible< + absl::optional<MoveableNoThrow>>::value); +} + +TEST(optionalTest, Destructor) { + struct Trivial {}; + + struct NonTrivial { + NonTrivial(const NonTrivial&) {} + NonTrivial& operator=(const NonTrivial&) { return *this; } + ~NonTrivial() {} + }; + + EXPECT_TRUE(std::is_trivially_destructible<absl::optional<int>>::value); + EXPECT_TRUE(std::is_trivially_destructible<absl::optional<Trivial>>::value); + EXPECT_FALSE( + std::is_trivially_destructible<absl::optional<NonTrivial>>::value); +} + +TEST(optionalTest, InPlaceConstructor) { + constexpr absl::optional<ConstexprType> opt0{absl::in_place_t()}; + static_assert(opt0, ""); + static_assert((*opt0).x == ConstexprType::kCtorDefault, ""); + constexpr absl::optional<ConstexprType> opt1{absl::in_place_t(), 1}; + static_assert(opt1, ""); + static_assert((*opt1).x == ConstexprType::kCtorInt, ""); +#ifndef ABSL_HAVE_NO_CONSTEXPR_INITIALIZER_LIST + constexpr absl::optional<ConstexprType> opt2{absl::in_place_t(), {1, 2}}; + static_assert(opt2, ""); + static_assert((*opt2).x == ConstexprType::kCtorInitializerList, ""); +#endif + + EXPECT_FALSE((std::is_constructible<absl::optional<ConvertsFromInPlaceT>, + absl::in_place_t>::value)); + EXPECT_FALSE((std::is_constructible<absl::optional<ConvertsFromInPlaceT>, + const absl::in_place_t&>::value)); + EXPECT_TRUE( + (std::is_constructible<absl::optional<ConvertsFromInPlaceT>, + absl::in_place_t, absl::in_place_t>::value)); + + EXPECT_FALSE((std::is_constructible<absl::optional<NoDefault>, + absl::in_place_t>::value)); + EXPECT_FALSE((std::is_constructible<absl::optional<NoDefault>, + absl::in_place_t&&>::value)); +} + +// template<U=T> optional(U&&); +TEST(optionalTest, ValueConstructor) { + constexpr absl::optional<int> opt0(0); + static_assert(opt0, ""); + static_assert(*opt0 == 0, ""); + EXPECT_TRUE((std::is_convertible<int, absl::optional<int>>::value)); + // Copy initialization ( = "abc") won't work due to optional(optional&&) + // is not constexpr. Use list initialization instead. This invokes + // absl::optional<ConstexprType>::absl::optional<U>(U&&), with U = const char + // (&) [4], which direct-initializes the ConstexprType value held by the + // optional via ConstexprType::ConstexprType(const char*). + constexpr absl::optional<ConstexprType> opt1 = {"abc"}; + static_assert(opt1, ""); + static_assert(ConstexprType::kCtorConstChar == (*opt1).x, ""); + EXPECT_TRUE( + (std::is_convertible<const char*, absl::optional<ConstexprType>>::value)); + // direct initialization + constexpr absl::optional<ConstexprType> opt2{2}; + static_assert(opt2, ""); + static_assert(ConstexprType::kCtorInt == (*opt2).x, ""); + EXPECT_FALSE( + (std::is_convertible<int, absl::optional<ConstexprType>>::value)); + + // this invokes absl::optional<int>::optional(int&&) + // NOTE: this has different behavior than assignment, e.g. + // "opt3 = {};" clears the optional rather than setting the value to 0 + // According to C++17 standard N4659 [over.ics.list] 16.3.3.1.5, (9.2)- "if + // the initializer list has no elements, the implicit conversion is the + // identity conversion", so `optional(int&&)` should be a better match than + // `optional(optional&&)` which is a user-defined conversion. + // Note: GCC 7 has a bug with this overload selection when compiled with + // `-std=c++17`. +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ == 7 && \ + __cplusplus == 201703L +#define ABSL_GCC7_OVER_ICS_LIST_BUG 1 +#endif +#ifndef ABSL_GCC7_OVER_ICS_LIST_BUG + constexpr absl::optional<int> opt3({}); + static_assert(opt3, ""); + static_assert(*opt3 == 0, ""); +#endif + + // this invokes the move constructor with a default constructed optional + // because non-template function is a better match than template function. + absl::optional<ConstexprType> opt4({}); + EXPECT_FALSE(opt4); +} + +struct Implicit {}; + +struct Explicit {}; + +struct Convert { + Convert(const Implicit&) // NOLINT(runtime/explicit) + : implicit(true), move(false) {} + Convert(Implicit&&) // NOLINT(runtime/explicit) + : implicit(true), move(true) {} + explicit Convert(const Explicit&) : implicit(false), move(false) {} + explicit Convert(Explicit&&) : implicit(false), move(true) {} + + bool implicit; + bool move; +}; + +struct ConvertFromOptional { + ConvertFromOptional(const Implicit&) // NOLINT(runtime/explicit) + : implicit(true), move(false), from_optional(false) {} + ConvertFromOptional(Implicit&&) // NOLINT(runtime/explicit) + : implicit(true), move(true), from_optional(false) {} + ConvertFromOptional( + const absl::optional<Implicit>&) // NOLINT(runtime/explicit) + : implicit(true), move(false), from_optional(true) {} + ConvertFromOptional(absl::optional<Implicit>&&) // NOLINT(runtime/explicit) + : implicit(true), move(true), from_optional(true) {} + explicit ConvertFromOptional(const Explicit&) + : implicit(false), move(false), from_optional(false) {} + explicit ConvertFromOptional(Explicit&&) + : implicit(false), move(true), from_optional(false) {} + explicit ConvertFromOptional(const absl::optional<Explicit>&) + : implicit(false), move(false), from_optional(true) {} + explicit ConvertFromOptional(absl::optional<Explicit>&&) + : implicit(false), move(true), from_optional(true) {} + + bool implicit; + bool move; + bool from_optional; +}; + +TEST(optionalTest, ConvertingConstructor) { + absl::optional<Implicit> i_empty; + absl::optional<Implicit> i(absl::in_place); + absl::optional<Explicit> e_empty; + absl::optional<Explicit> e(absl::in_place); + { + // implicitly constructing absl::optional<Convert> from + // absl::optional<Implicit> + absl::optional<Convert> empty = i_empty; + EXPECT_FALSE(empty); + absl::optional<Convert> opt_copy = i; + EXPECT_TRUE(opt_copy); + EXPECT_TRUE(opt_copy->implicit); + EXPECT_FALSE(opt_copy->move); + absl::optional<Convert> opt_move = absl::optional<Implicit>(absl::in_place); + EXPECT_TRUE(opt_move); + EXPECT_TRUE(opt_move->implicit); + EXPECT_TRUE(opt_move->move); + } + { + // explicitly constructing absl::optional<Convert> from + // absl::optional<Explicit> + absl::optional<Convert> empty(e_empty); + EXPECT_FALSE(empty); + absl::optional<Convert> opt_copy(e); + EXPECT_TRUE(opt_copy); + EXPECT_FALSE(opt_copy->implicit); + EXPECT_FALSE(opt_copy->move); + EXPECT_FALSE((std::is_convertible<const absl::optional<Explicit>&, + absl::optional<Convert>>::value)); + absl::optional<Convert> opt_move{absl::optional<Explicit>(absl::in_place)}; + EXPECT_TRUE(opt_move); + EXPECT_FALSE(opt_move->implicit); + EXPECT_TRUE(opt_move->move); + EXPECT_FALSE((std::is_convertible<absl::optional<Explicit>&&, + absl::optional<Convert>>::value)); + } + { + // implicitly constructing absl::optional<ConvertFromOptional> from + // absl::optional<Implicit> via + // ConvertFromOptional(absl::optional<Implicit>&&) check that + // ConvertFromOptional(Implicit&&) is NOT called + static_assert( + std::is_convertible<absl::optional<Implicit>, + absl::optional<ConvertFromOptional>>::value, + ""); + absl::optional<ConvertFromOptional> opt0 = i_empty; + EXPECT_TRUE(opt0); + EXPECT_TRUE(opt0->implicit); + EXPECT_FALSE(opt0->move); + EXPECT_TRUE(opt0->from_optional); + absl::optional<ConvertFromOptional> opt1 = absl::optional<Implicit>(); + EXPECT_TRUE(opt1); + EXPECT_TRUE(opt1->implicit); + EXPECT_TRUE(opt1->move); + EXPECT_TRUE(opt1->from_optional); + } + { + // implicitly constructing absl::optional<ConvertFromOptional> from + // absl::optional<Explicit> via + // ConvertFromOptional(absl::optional<Explicit>&&) check that + // ConvertFromOptional(Explicit&&) is NOT called + absl::optional<ConvertFromOptional> opt0(e_empty); + EXPECT_TRUE(opt0); + EXPECT_FALSE(opt0->implicit); + EXPECT_FALSE(opt0->move); + EXPECT_TRUE(opt0->from_optional); + EXPECT_FALSE( + (std::is_convertible<const absl::optional<Explicit>&, + absl::optional<ConvertFromOptional>>::value)); + absl::optional<ConvertFromOptional> opt1{absl::optional<Explicit>()}; + EXPECT_TRUE(opt1); + EXPECT_FALSE(opt1->implicit); + EXPECT_TRUE(opt1->move); + EXPECT_TRUE(opt1->from_optional); + EXPECT_FALSE( + (std::is_convertible<absl::optional<Explicit>&&, + absl::optional<ConvertFromOptional>>::value)); + } +} + +TEST(optionalTest, StructorBasic) { + StructorListener listener; + Listenable::listener = &listener; + { + absl::optional<Listenable> empty; + EXPECT_FALSE(empty); + absl::optional<Listenable> opt0(absl::in_place); + EXPECT_TRUE(opt0); + absl::optional<Listenable> opt1(absl::in_place, 1); + EXPECT_TRUE(opt1); + absl::optional<Listenable> opt2(absl::in_place, 1, 2); + EXPECT_TRUE(opt2); + } + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(1, listener.construct1); + EXPECT_EQ(1, listener.construct2); + EXPECT_EQ(3, listener.destruct); +} + +TEST(optionalTest, CopyMoveStructor) { + StructorListener listener; + Listenable::listener = &listener; + absl::optional<Listenable> original(absl::in_place); + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(0, listener.copy); + EXPECT_EQ(0, listener.move); + absl::optional<Listenable> copy(original); + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(1, listener.copy); + EXPECT_EQ(0, listener.move); + absl::optional<Listenable> move(std::move(original)); + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(1, listener.copy); + EXPECT_EQ(1, listener.move); +} + +TEST(optionalTest, ListInit) { + StructorListener listener; + Listenable::listener = &listener; + absl::optional<Listenable> listinit1(absl::in_place, {1}); + absl::optional<Listenable> listinit2(absl::in_place, {1, 2}); + EXPECT_EQ(2, listener.listinit); +} + +TEST(optionalTest, AssignFromNullopt) { + absl::optional<int> opt(1); + opt = absl::nullopt; + EXPECT_FALSE(opt); + + StructorListener listener; + Listenable::listener = &listener; + absl::optional<Listenable> opt1(absl::in_place); + opt1 = absl::nullopt; + EXPECT_FALSE(opt1); + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(1, listener.destruct); + + EXPECT_TRUE(( + std::is_nothrow_assignable<absl::optional<int>, absl::nullopt_t>::value)); + EXPECT_TRUE((std::is_nothrow_assignable<absl::optional<Listenable>, + absl::nullopt_t>::value)); +} + +TEST(optionalTest, CopyAssignment) { + const absl::optional<int> empty, opt1 = 1, opt2 = 2; + absl::optional<int> empty_to_opt1, opt1_to_opt2, opt2_to_empty; + + EXPECT_FALSE(empty_to_opt1); + empty_to_opt1 = empty; + EXPECT_FALSE(empty_to_opt1); + empty_to_opt1 = opt1; + EXPECT_TRUE(empty_to_opt1); + EXPECT_EQ(1, empty_to_opt1.value()); + + EXPECT_FALSE(opt1_to_opt2); + opt1_to_opt2 = opt1; + EXPECT_TRUE(opt1_to_opt2); + EXPECT_EQ(1, opt1_to_opt2.value()); + opt1_to_opt2 = opt2; + EXPECT_TRUE(opt1_to_opt2); + EXPECT_EQ(2, opt1_to_opt2.value()); + + EXPECT_FALSE(opt2_to_empty); + opt2_to_empty = opt2; + EXPECT_TRUE(opt2_to_empty); + EXPECT_EQ(2, opt2_to_empty.value()); + opt2_to_empty = empty; + EXPECT_FALSE(opt2_to_empty); + + EXPECT_FALSE(absl::is_copy_assignable<absl::optional<const int>>::value); + EXPECT_TRUE(absl::is_copy_assignable<absl::optional<Copyable>>::value); + EXPECT_FALSE(absl::is_copy_assignable<absl::optional<MoveableThrow>>::value); + EXPECT_FALSE( + absl::is_copy_assignable<absl::optional<MoveableNoThrow>>::value); + EXPECT_FALSE(absl::is_copy_assignable<absl::optional<NonMovable>>::value); + + EXPECT_TRUE(absl::is_trivially_copy_assignable<int>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<volatile int>::value); + + struct Trivial { + int i; + }; + struct NonTrivial { + NonTrivial& operator=(const NonTrivial&) { return *this; } + int i; + }; + + EXPECT_TRUE(absl::is_trivially_copy_assignable<Trivial>::value); + EXPECT_FALSE(absl::is_copy_assignable<const Trivial>::value); + EXPECT_FALSE(absl::is_copy_assignable<volatile Trivial>::value); + EXPECT_TRUE(absl::is_copy_assignable<NonTrivial>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<NonTrivial>::value); + + // std::optional doesn't support volatile nontrivial types. +#ifndef ABSL_USES_STD_OPTIONAL + { + StructorListener listener; + Listenable::listener = &listener; + + absl::optional<volatile Listenable> empty, set(absl::in_place); + EXPECT_EQ(1, listener.construct0); + absl::optional<volatile Listenable> empty_to_empty, empty_to_set, + set_to_empty(absl::in_place), set_to_set(absl::in_place); + EXPECT_EQ(3, listener.construct0); + empty_to_empty = empty; // no effect + empty_to_set = set; // copy construct + set_to_empty = empty; // destruct + set_to_set = set; // copy assign + EXPECT_EQ(1, listener.volatile_copy); + EXPECT_EQ(0, listener.volatile_move); + EXPECT_EQ(1, listener.destruct); + EXPECT_EQ(1, listener.volatile_copy_assign); + } +#endif // ABSL_USES_STD_OPTIONAL +} + +TEST(optionalTest, MoveAssignment) { + { + StructorListener listener; + Listenable::listener = &listener; + + absl::optional<Listenable> empty1, empty2, set1(absl::in_place), + set2(absl::in_place); + EXPECT_EQ(2, listener.construct0); + absl::optional<Listenable> empty_to_empty, empty_to_set, + set_to_empty(absl::in_place), set_to_set(absl::in_place); + EXPECT_EQ(4, listener.construct0); + empty_to_empty = std::move(empty1); + empty_to_set = std::move(set1); + set_to_empty = std::move(empty2); + set_to_set = std::move(set2); + EXPECT_EQ(0, listener.copy); + EXPECT_EQ(1, listener.move); + EXPECT_EQ(1, listener.destruct); + EXPECT_EQ(1, listener.move_assign); + } + // std::optional doesn't support volatile nontrivial types. +#ifndef ABSL_USES_STD_OPTIONAL + { + StructorListener listener; + Listenable::listener = &listener; + + absl::optional<volatile Listenable> empty1, empty2, set1(absl::in_place), + set2(absl::in_place); + EXPECT_EQ(2, listener.construct0); + absl::optional<volatile Listenable> empty_to_empty, empty_to_set, + set_to_empty(absl::in_place), set_to_set(absl::in_place); + EXPECT_EQ(4, listener.construct0); + empty_to_empty = std::move(empty1); // no effect + empty_to_set = std::move(set1); // move construct + set_to_empty = std::move(empty2); // destruct + set_to_set = std::move(set2); // move assign + EXPECT_EQ(0, listener.volatile_copy); + EXPECT_EQ(1, listener.volatile_move); + EXPECT_EQ(1, listener.destruct); + EXPECT_EQ(1, listener.volatile_move_assign); + } +#endif // ABSL_USES_STD_OPTIONAL + EXPECT_FALSE(absl::is_move_assignable<absl::optional<const int>>::value); + EXPECT_TRUE(absl::is_move_assignable<absl::optional<Copyable>>::value); + EXPECT_TRUE(absl::is_move_assignable<absl::optional<MoveableThrow>>::value); + EXPECT_TRUE(absl::is_move_assignable<absl::optional<MoveableNoThrow>>::value); + EXPECT_FALSE(absl::is_move_assignable<absl::optional<NonMovable>>::value); + + EXPECT_FALSE( + std::is_nothrow_move_assignable<absl::optional<MoveableThrow>>::value); + EXPECT_TRUE( + std::is_nothrow_move_assignable<absl::optional<MoveableNoThrow>>::value); +} + +struct NoConvertToOptional { + // disable implicit conversion from const NoConvertToOptional& + // to absl::optional<NoConvertToOptional>. + NoConvertToOptional(const NoConvertToOptional&) = delete; +}; + +struct CopyConvert { + CopyConvert(const NoConvertToOptional&); + CopyConvert& operator=(const CopyConvert&) = delete; + CopyConvert& operator=(const NoConvertToOptional&); +}; + +struct CopyConvertFromOptional { + CopyConvertFromOptional(const NoConvertToOptional&); + CopyConvertFromOptional(const absl::optional<NoConvertToOptional>&); + CopyConvertFromOptional& operator=(const CopyConvertFromOptional&) = delete; + CopyConvertFromOptional& operator=(const NoConvertToOptional&); + CopyConvertFromOptional& operator=( + const absl::optional<NoConvertToOptional>&); +}; + +struct MoveConvert { + MoveConvert(NoConvertToOptional&&); + MoveConvert& operator=(const MoveConvert&) = delete; + MoveConvert& operator=(NoConvertToOptional&&); +}; + +struct MoveConvertFromOptional { + MoveConvertFromOptional(NoConvertToOptional&&); + MoveConvertFromOptional(absl::optional<NoConvertToOptional>&&); + MoveConvertFromOptional& operator=(const MoveConvertFromOptional&) = delete; + MoveConvertFromOptional& operator=(NoConvertToOptional&&); + MoveConvertFromOptional& operator=(absl::optional<NoConvertToOptional>&&); +}; + +// template <typename U = T> absl::optional<T>& operator=(U&& v); +TEST(optionalTest, ValueAssignment) { + absl::optional<int> opt; + EXPECT_FALSE(opt); + opt = 42; + EXPECT_TRUE(opt); + EXPECT_EQ(42, opt.value()); + opt = absl::nullopt; + EXPECT_FALSE(opt); + opt = 42; + EXPECT_TRUE(opt); + EXPECT_EQ(42, opt.value()); + opt = 43; + EXPECT_TRUE(opt); + EXPECT_EQ(43, opt.value()); + opt = {}; // this should clear optional + EXPECT_FALSE(opt); + + opt = {44}; + EXPECT_TRUE(opt); + EXPECT_EQ(44, opt.value()); + + // U = const NoConvertToOptional& + EXPECT_TRUE((std::is_assignable<absl::optional<CopyConvert>&, + const NoConvertToOptional&>::value)); + // U = const absl::optional<NoConvertToOptional>& + EXPECT_TRUE((std::is_assignable<absl::optional<CopyConvertFromOptional>&, + const NoConvertToOptional&>::value)); + // U = const NoConvertToOptional& triggers SFINAE because + // std::is_constructible_v<MoveConvert, const NoConvertToOptional&> is false + EXPECT_FALSE((std::is_assignable<absl::optional<MoveConvert>&, + const NoConvertToOptional&>::value)); + // U = NoConvertToOptional + EXPECT_TRUE((std::is_assignable<absl::optional<MoveConvert>&, + NoConvertToOptional&&>::value)); + // U = const NoConvertToOptional& triggers SFINAE because + // std::is_constructible_v<MoveConvertFromOptional, const + // NoConvertToOptional&> is false + EXPECT_FALSE((std::is_assignable<absl::optional<MoveConvertFromOptional>&, + const NoConvertToOptional&>::value)); + // U = NoConvertToOptional + EXPECT_TRUE((std::is_assignable<absl::optional<MoveConvertFromOptional>&, + NoConvertToOptional&&>::value)); + // U = const absl::optional<NoConvertToOptional>& + EXPECT_TRUE( + (std::is_assignable<absl::optional<CopyConvertFromOptional>&, + const absl::optional<NoConvertToOptional>&>::value)); + // U = absl::optional<NoConvertToOptional> + EXPECT_TRUE( + (std::is_assignable<absl::optional<MoveConvertFromOptional>&, + absl::optional<NoConvertToOptional>&&>::value)); +} + +// template <typename U> absl::optional<T>& operator=(const absl::optional<U>& +// rhs); template <typename U> absl::optional<T>& operator=(absl::optional<U>&& +// rhs); +TEST(optionalTest, ConvertingAssignment) { + absl::optional<int> opt_i; + absl::optional<char> opt_c('c'); + opt_i = opt_c; + EXPECT_TRUE(opt_i); + EXPECT_EQ(*opt_c, *opt_i); + opt_i = absl::optional<char>(); + EXPECT_FALSE(opt_i); + opt_i = absl::optional<char>('d'); + EXPECT_TRUE(opt_i); + EXPECT_EQ('d', *opt_i); + + absl::optional<std::string> opt_str; + absl::optional<const char*> opt_cstr("abc"); + opt_str = opt_cstr; + EXPECT_TRUE(opt_str); + EXPECT_EQ(std::string("abc"), *opt_str); + opt_str = absl::optional<const char*>(); + EXPECT_FALSE(opt_str); + opt_str = absl::optional<const char*>("def"); + EXPECT_TRUE(opt_str); + EXPECT_EQ(std::string("def"), *opt_str); + + // operator=(const absl::optional<U>&) with U = NoConvertToOptional + EXPECT_TRUE( + (std::is_assignable<absl::optional<CopyConvert>, + const absl::optional<NoConvertToOptional>&>::value)); + // operator=(const absl::optional<U>&) with U = NoConvertToOptional + // triggers SFINAE because + // std::is_constructible_v<MoveConvert, const NoConvertToOptional&> is false + EXPECT_FALSE( + (std::is_assignable<absl::optional<MoveConvert>&, + const absl::optional<NoConvertToOptional>&>::value)); + // operator=(absl::optional<U>&&) with U = NoConvertToOptional + EXPECT_TRUE( + (std::is_assignable<absl::optional<MoveConvert>&, + absl::optional<NoConvertToOptional>&&>::value)); + // operator=(const absl::optional<U>&) with U = NoConvertToOptional triggers + // SFINAE because std::is_constructible_v<MoveConvertFromOptional, const + // NoConvertToOptional&> is false. operator=(U&&) with U = const + // absl::optional<NoConverToOptional>& triggers SFINAE because + // std::is_constructible<MoveConvertFromOptional, + // absl::optional<NoConvertToOptional>&&> is true. + EXPECT_FALSE( + (std::is_assignable<absl::optional<MoveConvertFromOptional>&, + const absl::optional<NoConvertToOptional>&>::value)); +} + +TEST(optionalTest, ResetAndHasValue) { + StructorListener listener; + Listenable::listener = &listener; + absl::optional<Listenable> opt; + EXPECT_FALSE(opt); + EXPECT_FALSE(opt.has_value()); + opt.emplace(); + EXPECT_TRUE(opt); + EXPECT_TRUE(opt.has_value()); + opt.reset(); + EXPECT_FALSE(opt); + EXPECT_FALSE(opt.has_value()); + EXPECT_EQ(1, listener.destruct); + opt.reset(); + EXPECT_FALSE(opt); + EXPECT_FALSE(opt.has_value()); + + constexpr absl::optional<int> empty; + static_assert(!empty.has_value(), ""); + constexpr absl::optional<int> nonempty(1); + static_assert(nonempty.has_value(), ""); +} + +TEST(optionalTest, Emplace) { + StructorListener listener; + Listenable::listener = &listener; + absl::optional<Listenable> opt; + EXPECT_FALSE(opt); + opt.emplace(1); + EXPECT_TRUE(opt); + opt.emplace(1, 2); + EXPECT_EQ(1, listener.construct1); + EXPECT_EQ(1, listener.construct2); + EXPECT_EQ(1, listener.destruct); + + absl::optional<std::string> o; + EXPECT_TRUE((std::is_same<std::string&, decltype(o.emplace("abc"))>::value)); + std::string& ref = o.emplace("abc"); + EXPECT_EQ(&ref, &o.value()); +} + +TEST(optionalTest, ListEmplace) { + StructorListener listener; + Listenable::listener = &listener; + absl::optional<Listenable> opt; + EXPECT_FALSE(opt); + opt.emplace({1}); + EXPECT_TRUE(opt); + opt.emplace({1, 2}); + EXPECT_EQ(2, listener.listinit); + EXPECT_EQ(1, listener.destruct); + + absl::optional<Listenable> o; + EXPECT_TRUE((std::is_same<Listenable&, decltype(o.emplace({1}))>::value)); + Listenable& ref = o.emplace({1}); + EXPECT_EQ(&ref, &o.value()); +} + +TEST(optionalTest, Swap) { + absl::optional<int> opt_empty, opt1 = 1, opt2 = 2; + EXPECT_FALSE(opt_empty); + EXPECT_TRUE(opt1); + EXPECT_EQ(1, opt1.value()); + EXPECT_TRUE(opt2); + EXPECT_EQ(2, opt2.value()); + swap(opt_empty, opt1); + EXPECT_FALSE(opt1); + EXPECT_TRUE(opt_empty); + EXPECT_EQ(1, opt_empty.value()); + EXPECT_TRUE(opt2); + EXPECT_EQ(2, opt2.value()); + swap(opt_empty, opt1); + EXPECT_FALSE(opt_empty); + EXPECT_TRUE(opt1); + EXPECT_EQ(1, opt1.value()); + EXPECT_TRUE(opt2); + EXPECT_EQ(2, opt2.value()); + swap(opt1, opt2); + EXPECT_FALSE(opt_empty); + EXPECT_TRUE(opt1); + EXPECT_EQ(2, opt1.value()); + EXPECT_TRUE(opt2); + EXPECT_EQ(1, opt2.value()); + + EXPECT_TRUE(noexcept(opt1.swap(opt2))); + EXPECT_TRUE(noexcept(swap(opt1, opt2))); +} + +template <int v> +struct DeletedOpAddr { + int value = v; + constexpr DeletedOpAddr() = default; + constexpr const DeletedOpAddr<v>* operator&() const = delete; // NOLINT + DeletedOpAddr<v>* operator&() = delete; // NOLINT +}; + +// The static_assert featuring a constexpr call to operator->() is commented out +// to document the fact that the current implementation of absl::optional<T> +// expects such usecases to be malformed and not compile. +TEST(optionalTest, OperatorAddr) { + constexpr int v = -1; + { // constexpr + constexpr absl::optional<DeletedOpAddr<v>> opt(absl::in_place_t{}); + static_assert(opt.has_value(), ""); + // static_assert(opt->value == v, ""); + static_assert((*opt).value == v, ""); + } + { // non-constexpr + const absl::optional<DeletedOpAddr<v>> opt(absl::in_place_t{}); + EXPECT_TRUE(opt.has_value()); + EXPECT_TRUE(opt->value == v); + EXPECT_TRUE((*opt).value == v); + } +} + +TEST(optionalTest, PointerStuff) { + absl::optional<std::string> opt(absl::in_place, "foo"); + EXPECT_EQ("foo", *opt); + const auto& opt_const = opt; + EXPECT_EQ("foo", *opt_const); + EXPECT_EQ(opt->size(), 3); + EXPECT_EQ(opt_const->size(), 3); + + constexpr absl::optional<ConstexprType> opt1(1); + static_assert((*opt1).x == ConstexprType::kCtorInt, ""); +} + +// gcc has a bug pre 4.9.1 where it doesn't do correct overload resolution +// when overloads are const-qualified and *this is an raluve. +// Skip that test to make the build green again when using the old compiler. +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=59296 is fixed in 4.9.1. +#if defined(__GNUC__) && !defined(__clang__) +#define GCC_VERSION (__GNUC__ * 10000 \ + + __GNUC_MINOR__ * 100 \ + + __GNUC_PATCHLEVEL__) +#if GCC_VERSION < 40901 +#define ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG +#endif +#endif + +// MSVC has a bug with "cv-qualifiers in class construction", fixed in 2017. See +// https://docs.microsoft.com/en-us/cpp/cpp-conformance-improvements-2017#bug-fixes +// The compiler some incorrectly ingores the cv-qualifier when generating a +// class object via a constructor call. For example: +// +// class optional { +// constexpr T&& value() &&; +// constexpr const T&& value() const &&; +// } +// +// using COI = const absl::optional<int>; +// static_assert(2 == COI(2).value(), ""); // const && +// +// This should invoke the "const &&" overload but since it ignores the const +// qualifier it finds the "&&" overload the best candidate. +#if defined(_MSC_VER) && _MSC_VER < 1910 +#define ABSL_SKIP_OVERLOAD_TEST_DUE_TO_MSVC_BUG +#endif + +TEST(optionalTest, Value) { + using O = absl::optional<std::string>; + using CO = const absl::optional<std::string>; + using OC = absl::optional<const std::string>; + O lvalue(absl::in_place, "lvalue"); + CO clvalue(absl::in_place, "clvalue"); + OC lvalue_c(absl::in_place, "lvalue_c"); + EXPECT_EQ("lvalue", lvalue.value()); + EXPECT_EQ("clvalue", clvalue.value()); + EXPECT_EQ("lvalue_c", lvalue_c.value()); + EXPECT_EQ("xvalue", O(absl::in_place, "xvalue").value()); + EXPECT_EQ("xvalue_c", OC(absl::in_place, "xvalue_c").value()); +#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG + EXPECT_EQ("cxvalue", CO(absl::in_place, "cxvalue").value()); +#endif + EXPECT_EQ("&", TypeQuals(lvalue.value())); + EXPECT_EQ("c&", TypeQuals(clvalue.value())); + EXPECT_EQ("c&", TypeQuals(lvalue_c.value())); + EXPECT_EQ("&&", TypeQuals(O(absl::in_place, "xvalue").value())); +#if !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_MSVC_BUG) && \ + !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG) + EXPECT_EQ("c&&", TypeQuals(CO(absl::in_place, "cxvalue").value())); +#endif + EXPECT_EQ("c&&", TypeQuals(OC(absl::in_place, "xvalue_c").value())); + + // test on volatile type + using OV = absl::optional<volatile int>; + OV lvalue_v(absl::in_place, 42); + EXPECT_EQ(42, lvalue_v.value()); + EXPECT_EQ(42, OV(42).value()); + EXPECT_TRUE((std::is_same<volatile int&, decltype(lvalue_v.value())>::value)); + EXPECT_TRUE((std::is_same<volatile int&&, decltype(OV(42).value())>::value)); + + // test exception throw on value() + absl::optional<int> empty; +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW((void)empty.value(), absl::bad_optional_access); +#else + EXPECT_DEATH_IF_SUPPORTED((void)empty.value(), "Bad optional access"); +#endif + + // test constexpr value() + constexpr absl::optional<int> o1(1); + static_assert(1 == o1.value(), ""); // const & +#if !defined(_MSC_VER) && !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG) + using COI = const absl::optional<int>; + static_assert(2 == COI(2).value(), ""); // const && +#endif +} + +TEST(optionalTest, DerefOperator) { + using O = absl::optional<std::string>; + using CO = const absl::optional<std::string>; + using OC = absl::optional<const std::string>; + O lvalue(absl::in_place, "lvalue"); + CO clvalue(absl::in_place, "clvalue"); + OC lvalue_c(absl::in_place, "lvalue_c"); + EXPECT_EQ("lvalue", *lvalue); + EXPECT_EQ("clvalue", *clvalue); + EXPECT_EQ("lvalue_c", *lvalue_c); + EXPECT_EQ("xvalue", *O(absl::in_place, "xvalue")); + EXPECT_EQ("xvalue_c", *OC(absl::in_place, "xvalue_c")); +#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG + EXPECT_EQ("cxvalue", *CO(absl::in_place, "cxvalue")); +#endif + EXPECT_EQ("&", TypeQuals(*lvalue)); + EXPECT_EQ("c&", TypeQuals(*clvalue)); + EXPECT_EQ("&&", TypeQuals(*O(absl::in_place, "xvalue"))); +#if !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_MSVC_BUG) && \ + !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG) + EXPECT_EQ("c&&", TypeQuals(*CO(absl::in_place, "cxvalue"))); +#endif + EXPECT_EQ("c&&", TypeQuals(*OC(absl::in_place, "xvalue_c"))); + + // test on volatile type + using OV = absl::optional<volatile int>; + OV lvalue_v(absl::in_place, 42); + EXPECT_EQ(42, *lvalue_v); + EXPECT_EQ(42, *OV(42)); + EXPECT_TRUE((std::is_same<volatile int&, decltype(*lvalue_v)>::value)); + EXPECT_TRUE((std::is_same<volatile int&&, decltype(*OV(42))>::value)); + + constexpr absl::optional<int> opt1(1); + static_assert(*opt1 == 1, ""); +#if !defined(_MSC_VER) && !defined(ABSL_SKIP_OVERLOAD_TEST_DUE_TO_GCC_BUG) + using COI = const absl::optional<int>; + static_assert(*COI(2) == 2, ""); +#endif +} + +TEST(optionalTest, ValueOr) { + absl::optional<double> opt_empty, opt_set = 1.2; + EXPECT_EQ(42.0, opt_empty.value_or(42)); + EXPECT_EQ(1.2, opt_set.value_or(42)); + EXPECT_EQ(42.0, absl::optional<double>().value_or(42)); + EXPECT_EQ(1.2, absl::optional<double>(1.2).value_or(42)); + + constexpr absl::optional<double> copt_empty, copt_set = {1.2}; + static_assert(42.0 == copt_empty.value_or(42), ""); + static_assert(1.2 == copt_set.value_or(42), ""); +#ifndef ABSL_SKIP_OVERLOAD_TEST_DUE_TO_MSVC_BUG + using COD = const absl::optional<double>; + static_assert(42.0 == COD().value_or(42), ""); + static_assert(1.2 == COD(1.2).value_or(42), ""); +#endif +} + +// make_optional cannot be constexpr until C++17 +TEST(optionalTest, make_optional) { + auto opt_int = absl::make_optional(42); + EXPECT_TRUE((std::is_same<decltype(opt_int), absl::optional<int>>::value)); + EXPECT_EQ(42, opt_int); + + StructorListener listener; + Listenable::listener = &listener; + + absl::optional<Listenable> opt0 = absl::make_optional<Listenable>(); + EXPECT_EQ(1, listener.construct0); + absl::optional<Listenable> opt1 = absl::make_optional<Listenable>(1); + EXPECT_EQ(1, listener.construct1); + absl::optional<Listenable> opt2 = absl::make_optional<Listenable>(1, 2); + EXPECT_EQ(1, listener.construct2); + absl::optional<Listenable> opt3 = absl::make_optional<Listenable>({1}); + absl::optional<Listenable> opt4 = absl::make_optional<Listenable>({1, 2}); + EXPECT_EQ(2, listener.listinit); + + // Constexpr tests on trivially copyable types + // optional<T> has trivial copy/move ctors when T is trivially copyable. + // For nontrivial types with constexpr constructors, we need copy elision in + // C++17 for make_optional to be constexpr. + { + constexpr absl::optional<int> c_opt = absl::make_optional(42); + static_assert(c_opt.value() == 42, ""); + } + { + struct TrivialCopyable { + constexpr TrivialCopyable() : x(0) {} + constexpr explicit TrivialCopyable(int i) : x(i) {} + int x; + }; + + constexpr TrivialCopyable v; + constexpr absl::optional<TrivialCopyable> c_opt0 = absl::make_optional(v); + static_assert((*c_opt0).x == 0, ""); + constexpr absl::optional<TrivialCopyable> c_opt1 = + absl::make_optional<TrivialCopyable>(); + static_assert((*c_opt1).x == 0, ""); + constexpr absl::optional<TrivialCopyable> c_opt2 = + absl::make_optional<TrivialCopyable>(42); + static_assert((*c_opt2).x == 42, ""); + } +} + +template <typename T, typename U> +void optionalTest_Comparisons_EXPECT_LESS(T x, U y) { + EXPECT_FALSE(x == y); + EXPECT_TRUE(x != y); + EXPECT_TRUE(x < y); + EXPECT_FALSE(x > y); + EXPECT_TRUE(x <= y); + EXPECT_FALSE(x >= y); +} + +template <typename T, typename U> +void optionalTest_Comparisons_EXPECT_SAME(T x, U y) { + EXPECT_TRUE(x == y); + EXPECT_FALSE(x != y); + EXPECT_FALSE(x < y); + EXPECT_FALSE(x > y); + EXPECT_TRUE(x <= y); + EXPECT_TRUE(x >= y); +} + +template <typename T, typename U> +void optionalTest_Comparisons_EXPECT_GREATER(T x, U y) { + EXPECT_FALSE(x == y); + EXPECT_TRUE(x != y); + EXPECT_FALSE(x < y); + EXPECT_TRUE(x > y); + EXPECT_FALSE(x <= y); + EXPECT_TRUE(x >= y); +} + + +template <typename T, typename U, typename V> +void TestComparisons() { + absl::optional<T> ae, a2{2}, a4{4}; + absl::optional<U> be, b2{2}, b4{4}; + V v3 = 3; + + // LHS: absl::nullopt, ae, a2, v3, a4 + // RHS: absl::nullopt, be, b2, v3, b4 + + // optionalTest_Comparisons_EXPECT_NOT_TO_WORK(absl::nullopt,absl::nullopt); + optionalTest_Comparisons_EXPECT_SAME(absl::nullopt, be); + optionalTest_Comparisons_EXPECT_LESS(absl::nullopt, b2); + // optionalTest_Comparisons_EXPECT_NOT_TO_WORK(absl::nullopt,v3); + optionalTest_Comparisons_EXPECT_LESS(absl::nullopt, b4); + + optionalTest_Comparisons_EXPECT_SAME(ae, absl::nullopt); + optionalTest_Comparisons_EXPECT_SAME(ae, be); + optionalTest_Comparisons_EXPECT_LESS(ae, b2); + optionalTest_Comparisons_EXPECT_LESS(ae, v3); + optionalTest_Comparisons_EXPECT_LESS(ae, b4); + + optionalTest_Comparisons_EXPECT_GREATER(a2, absl::nullopt); + optionalTest_Comparisons_EXPECT_GREATER(a2, be); + optionalTest_Comparisons_EXPECT_SAME(a2, b2); + optionalTest_Comparisons_EXPECT_LESS(a2, v3); + optionalTest_Comparisons_EXPECT_LESS(a2, b4); + + // optionalTest_Comparisons_EXPECT_NOT_TO_WORK(v3,absl::nullopt); + optionalTest_Comparisons_EXPECT_GREATER(v3, be); + optionalTest_Comparisons_EXPECT_GREATER(v3, b2); + optionalTest_Comparisons_EXPECT_SAME(v3, v3); + optionalTest_Comparisons_EXPECT_LESS(v3, b4); + + optionalTest_Comparisons_EXPECT_GREATER(a4, absl::nullopt); + optionalTest_Comparisons_EXPECT_GREATER(a4, be); + optionalTest_Comparisons_EXPECT_GREATER(a4, b2); + optionalTest_Comparisons_EXPECT_GREATER(a4, v3); + optionalTest_Comparisons_EXPECT_SAME(a4, b4); +} + +struct Int1 { + Int1() = default; + Int1(int i) : i(i) {} // NOLINT(runtime/explicit) + int i; +}; + +struct Int2 { + Int2() = default; + Int2(int i) : i(i) {} // NOLINT(runtime/explicit) + int i; +}; + +// comparison between Int1 and Int2 +constexpr bool operator==(const Int1& lhs, const Int2& rhs) { + return lhs.i == rhs.i; +} +constexpr bool operator!=(const Int1& lhs, const Int2& rhs) { + return !(lhs == rhs); +} +constexpr bool operator<(const Int1& lhs, const Int2& rhs) { + return lhs.i < rhs.i; +} +constexpr bool operator<=(const Int1& lhs, const Int2& rhs) { + return lhs < rhs || lhs == rhs; +} +constexpr bool operator>(const Int1& lhs, const Int2& rhs) { + return !(lhs <= rhs); +} +constexpr bool operator>=(const Int1& lhs, const Int2& rhs) { + return !(lhs < rhs); +} + +TEST(optionalTest, Comparisons) { + TestComparisons<int, int, int>(); + TestComparisons<const int, int, int>(); + TestComparisons<Int1, int, int>(); + TestComparisons<int, Int2, int>(); + TestComparisons<Int1, Int2, int>(); + + // compare absl::optional<std::string> with const char* + absl::optional<std::string> opt_str = "abc"; + const char* cstr = "abc"; + EXPECT_TRUE(opt_str == cstr); + // compare absl::optional<std::string> with absl::optional<const char*> + absl::optional<const char*> opt_cstr = cstr; + EXPECT_TRUE(opt_str == opt_cstr); + // compare absl::optional<std::string> with absl::optional<absl::string_view> + absl::optional<absl::string_view> e1; + absl::optional<std::string> e2; + EXPECT_TRUE(e1 == e2); +} + + +TEST(optionalTest, SwapRegression) { + StructorListener listener; + Listenable::listener = &listener; + + { + absl::optional<Listenable> a; + absl::optional<Listenable> b(absl::in_place); + a.swap(b); + } + + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(1, listener.move); + EXPECT_EQ(2, listener.destruct); + + { + absl::optional<Listenable> a(absl::in_place); + absl::optional<Listenable> b; + a.swap(b); + } + + EXPECT_EQ(2, listener.construct0); + EXPECT_EQ(2, listener.move); + EXPECT_EQ(4, listener.destruct); +} + +TEST(optionalTest, BigStringLeakCheck) { + constexpr size_t n = 1 << 16; + + using OS = absl::optional<std::string>; + + OS a; + OS b = absl::nullopt; + OS c = std::string(n, 'c'); + std::string sd(n, 'd'); + OS d = sd; + OS e(absl::in_place, n, 'e'); + OS f; + f.emplace(n, 'f'); + + OS ca(a); + OS cb(b); + OS cc(c); + OS cd(d); + OS ce(e); + + OS oa; + OS ob = absl::nullopt; + OS oc = std::string(n, 'c'); + std::string sod(n, 'd'); + OS od = sod; + OS oe(absl::in_place, n, 'e'); + OS of; + of.emplace(n, 'f'); + + OS ma(std::move(oa)); + OS mb(std::move(ob)); + OS mc(std::move(oc)); + OS md(std::move(od)); + OS me(std::move(oe)); + OS mf(std::move(of)); + + OS aa1; + OS ab1 = absl::nullopt; + OS ac1 = std::string(n, 'c'); + std::string sad1(n, 'd'); + OS ad1 = sad1; + OS ae1(absl::in_place, n, 'e'); + OS af1; + af1.emplace(n, 'f'); + + OS aa2; + OS ab2 = absl::nullopt; + OS ac2 = std::string(n, 'c'); + std::string sad2(n, 'd'); + OS ad2 = sad2; + OS ae2(absl::in_place, n, 'e'); + OS af2; + af2.emplace(n, 'f'); + + aa1 = af2; + ab1 = ae2; + ac1 = ad2; + ad1 = ac2; + ae1 = ab2; + af1 = aa2; + + OS aa3; + OS ab3 = absl::nullopt; + OS ac3 = std::string(n, 'c'); + std::string sad3(n, 'd'); + OS ad3 = sad3; + OS ae3(absl::in_place, n, 'e'); + OS af3; + af3.emplace(n, 'f'); + + aa3 = absl::nullopt; + ab3 = absl::nullopt; + ac3 = absl::nullopt; + ad3 = absl::nullopt; + ae3 = absl::nullopt; + af3 = absl::nullopt; + + OS aa4; + OS ab4 = absl::nullopt; + OS ac4 = std::string(n, 'c'); + std::string sad4(n, 'd'); + OS ad4 = sad4; + OS ae4(absl::in_place, n, 'e'); + OS af4; + af4.emplace(n, 'f'); + + aa4 = OS(absl::in_place, n, 'a'); + ab4 = OS(absl::in_place, n, 'b'); + ac4 = OS(absl::in_place, n, 'c'); + ad4 = OS(absl::in_place, n, 'd'); + ae4 = OS(absl::in_place, n, 'e'); + af4 = OS(absl::in_place, n, 'f'); + + OS aa5; + OS ab5 = absl::nullopt; + OS ac5 = std::string(n, 'c'); + std::string sad5(n, 'd'); + OS ad5 = sad5; + OS ae5(absl::in_place, n, 'e'); + OS af5; + af5.emplace(n, 'f'); + + std::string saa5(n, 'a'); + std::string sab5(n, 'a'); + std::string sac5(n, 'a'); + std::string sad52(n, 'a'); + std::string sae5(n, 'a'); + std::string saf5(n, 'a'); + + aa5 = saa5; + ab5 = sab5; + ac5 = sac5; + ad5 = sad52; + ae5 = sae5; + af5 = saf5; + + OS aa6; + OS ab6 = absl::nullopt; + OS ac6 = std::string(n, 'c'); + std::string sad6(n, 'd'); + OS ad6 = sad6; + OS ae6(absl::in_place, n, 'e'); + OS af6; + af6.emplace(n, 'f'); + + aa6 = std::string(n, 'a'); + ab6 = std::string(n, 'b'); + ac6 = std::string(n, 'c'); + ad6 = std::string(n, 'd'); + ae6 = std::string(n, 'e'); + af6 = std::string(n, 'f'); + + OS aa7; + OS ab7 = absl::nullopt; + OS ac7 = std::string(n, 'c'); + std::string sad7(n, 'd'); + OS ad7 = sad7; + OS ae7(absl::in_place, n, 'e'); + OS af7; + af7.emplace(n, 'f'); + + aa7.emplace(n, 'A'); + ab7.emplace(n, 'B'); + ac7.emplace(n, 'C'); + ad7.emplace(n, 'D'); + ae7.emplace(n, 'E'); + af7.emplace(n, 'F'); +} + +TEST(optionalTest, MoveAssignRegression) { + StructorListener listener; + Listenable::listener = &listener; + + { + absl::optional<Listenable> a; + Listenable b; + a = std::move(b); + } + + EXPECT_EQ(1, listener.construct0); + EXPECT_EQ(1, listener.move); + EXPECT_EQ(2, listener.destruct); +} + +TEST(optionalTest, ValueType) { + EXPECT_TRUE((std::is_same<absl::optional<int>::value_type, int>::value)); + EXPECT_TRUE((std::is_same<absl::optional<std::string>::value_type, + std::string>::value)); + EXPECT_FALSE( + (std::is_same<absl::optional<int>::value_type, absl::nullopt_t>::value)); +} + +template <typename T> +struct is_hash_enabled_for { + template <typename U, typename = decltype(std::hash<U>()(std::declval<U>()))> + static std::true_type test(int); + + template <typename U> + static std::false_type test(...); + + static constexpr bool value = decltype(test<T>(0))::value; +}; + +TEST(optionalTest, Hash) { + std::hash<absl::optional<int>> hash; + std::set<size_t> hashcodes; + hashcodes.insert(hash(absl::nullopt)); + for (int i = 0; i < 100; ++i) { + hashcodes.insert(hash(i)); + } + EXPECT_GT(hashcodes.size(), 90); + + static_assert(is_hash_enabled_for<absl::optional<int>>::value, ""); + static_assert(is_hash_enabled_for<absl::optional<Hashable>>::value, ""); + static_assert( + absl::type_traits_internal::IsHashable<absl::optional<int>>::value, ""); + static_assert( + absl::type_traits_internal::IsHashable<absl::optional<Hashable>>::value, + ""); + absl::type_traits_internal::AssertHashEnabled<absl::optional<int>>(); + absl::type_traits_internal::AssertHashEnabled<absl::optional<Hashable>>(); + +#if ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + static_assert(!is_hash_enabled_for<absl::optional<NonHashable>>::value, ""); + static_assert(!absl::type_traits_internal::IsHashable< + absl::optional<NonHashable>>::value, + ""); +#endif + + // libstdc++ std::optional is missing remove_const_t, i.e. it's using + // std::hash<T> rather than std::hash<std::remove_const_t<T>>. + // Reference: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82262 +#ifndef __GLIBCXX__ + static_assert(is_hash_enabled_for<absl::optional<const int>>::value, ""); + static_assert(is_hash_enabled_for<absl::optional<const Hashable>>::value, ""); + std::hash<absl::optional<const int>> c_hash; + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(hash(i), c_hash(i)); + } +#endif +} + +struct MoveMeNoThrow { + MoveMeNoThrow() : x(0) {} + [[noreturn]] MoveMeNoThrow(const MoveMeNoThrow& other) : x(other.x) { + ABSL_RAW_LOG(FATAL, "Should not be called."); + abort(); + } + MoveMeNoThrow(MoveMeNoThrow&& other) noexcept : x(other.x) {} + int x; +}; + +struct MoveMeThrow { + MoveMeThrow() : x(0) {} + MoveMeThrow(const MoveMeThrow& other) : x(other.x) {} + MoveMeThrow(MoveMeThrow&& other) : x(other.x) {} + int x; +}; + +TEST(optionalTest, NoExcept) { + static_assert( + std::is_nothrow_move_constructible<absl::optional<MoveMeNoThrow>>::value, + ""); +#ifndef ABSL_USES_STD_OPTIONAL + static_assert(absl::default_allocator_is_nothrow::value == + std::is_nothrow_move_constructible< + absl::optional<MoveMeThrow>>::value, + ""); +#endif + std::vector<absl::optional<MoveMeNoThrow>> v; + for (int i = 0; i < 10; ++i) v.emplace_back(); +} + +struct AnyLike { + AnyLike(AnyLike&&) = default; + AnyLike(const AnyLike&) = default; + + template <typename ValueType, + typename T = typename std::decay<ValueType>::type, + typename std::enable_if< + !absl::disjunction< + std::is_same<AnyLike, T>, + absl::negation<std::is_copy_constructible<T>>>::value, + int>::type = 0> + AnyLike(ValueType&&) {} // NOLINT(runtime/explicit) + + AnyLike& operator=(AnyLike&&) = default; + AnyLike& operator=(const AnyLike&) = default; + + template <typename ValueType, + typename T = typename std::decay<ValueType>::type> + typename std::enable_if< + absl::conjunction<absl::negation<std::is_same<AnyLike, T>>, + std::is_copy_constructible<T>>::value, + AnyLike&>::type + operator=(ValueType&& /* rhs */) { + return *this; + } +}; + +TEST(optionalTest, ConstructionConstraints) { + EXPECT_TRUE((std::is_constructible<AnyLike, absl::optional<AnyLike>>::value)); + + EXPECT_TRUE( + (std::is_constructible<AnyLike, const absl::optional<AnyLike>&>::value)); + + EXPECT_TRUE((std::is_constructible<absl::optional<AnyLike>, AnyLike>::value)); + EXPECT_TRUE( + (std::is_constructible<absl::optional<AnyLike>, const AnyLike&>::value)); + + EXPECT_TRUE((std::is_convertible<absl::optional<AnyLike>, AnyLike>::value)); + + EXPECT_TRUE( + (std::is_convertible<const absl::optional<AnyLike>&, AnyLike>::value)); + + EXPECT_TRUE((std::is_convertible<AnyLike, absl::optional<AnyLike>>::value)); + EXPECT_TRUE( + (std::is_convertible<const AnyLike&, absl::optional<AnyLike>>::value)); + + EXPECT_TRUE(std::is_move_constructible<absl::optional<AnyLike>>::value); + EXPECT_TRUE(std::is_copy_constructible<absl::optional<AnyLike>>::value); +} + +TEST(optionalTest, AssignmentConstraints) { + EXPECT_TRUE((std::is_assignable<AnyLike&, absl::optional<AnyLike>>::value)); + EXPECT_TRUE( + (std::is_assignable<AnyLike&, const absl::optional<AnyLike>&>::value)); + EXPECT_TRUE((std::is_assignable<absl::optional<AnyLike>&, AnyLike>::value)); + EXPECT_TRUE( + (std::is_assignable<absl::optional<AnyLike>&, const AnyLike&>::value)); + EXPECT_TRUE(std::is_move_assignable<absl::optional<AnyLike>>::value); + EXPECT_TRUE(absl::is_copy_assignable<absl::optional<AnyLike>>::value); +} + +#if !defined(__EMSCRIPTEN__) +struct NestedClassBug { + struct Inner { + bool dummy = false; + }; + absl::optional<Inner> value; +}; + +TEST(optionalTest, InPlaceTSFINAEBug) { + NestedClassBug b; + ((void)b); + using Inner = NestedClassBug::Inner; + + EXPECT_TRUE((std::is_default_constructible<Inner>::value)); + EXPECT_TRUE((std::is_constructible<Inner>::value)); + EXPECT_TRUE( + (std::is_constructible<absl::optional<Inner>, absl::in_place_t>::value)); + + absl::optional<Inner> o(absl::in_place); + EXPECT_TRUE(o.has_value()); + o.emplace(); + EXPECT_TRUE(o.has_value()); +} +#endif // !defined(__EMSCRIPTEN__) + +} // namespace + +#endif // #if !defined(ABSL_USES_STD_OPTIONAL) diff --git a/third_party/abseil_cpp/absl/types/span.h b/third_party/abseil_cpp/absl/types/span.h new file mode 100644 index 000000000000..95fe79262d67 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/span.h @@ -0,0 +1,726 @@ +// +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// span.h +// ----------------------------------------------------------------------------- +// +// This header file defines a `Span<T>` type for holding a reference to existing +// array data. The `Span` object, much like the `absl::string_view` object, +// does not own such data itself, and the data being referenced by the span must +// outlive the span itself. Unlike `view` type references, a span can hold a +// reference to mutable data (and can mutate it for underlying types of +// non-const T.) A span provides a lightweight way to pass a reference to such +// data. +// +// Additionally, this header file defines `MakeSpan()` and `MakeConstSpan()` +// factory functions, for clearly creating spans of type `Span<T>` or read-only +// `Span<const T>` when such types may be difficult to identify due to issues +// with implicit conversion. +// +// The C++20 draft standard includes a `std::span` type. As of June 2020, the +// differences between `absl::Span` and `std::span` are: +// * `absl::Span` has `operator==` (which is likely a design bug, +// per https://abseil.io/blog/20180531-regular-types) +// * `absl::Span` has the factory functions `MakeSpan()` and +// `MakeConstSpan()` +// * bounds-checked access to `absl::Span` is accomplished with `at()` +// * `absl::Span` has compiler-provided move and copy constructors and +// assignment. This is due to them being specified as `constexpr`, but that +// implies const in C++11. +// * `absl::Span` has no `element_type` typedef +// * A read-only `absl::Span<const T>` can be implicitly constructed from an +// initializer list. +// * `absl::Span` has no `bytes()`, `size_bytes()`, `as_bytes()`, or +// `as_mutable_bytes()` methods +// * `absl::Span` has no static extent template parameter, nor constructors +// which exist only because of the static extent parameter. +// * `absl::Span` has an explicit mutable-reference constructor +// +// For more information, see the class comments below. +#ifndef ABSL_TYPES_SPAN_H_ +#define ABSL_TYPES_SPAN_H_ + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <initializer_list> +#include <iterator> +#include <type_traits> +#include <utility> + +#include "absl/base/internal/throw_delegate.h" +#include "absl/base/macros.h" +#include "absl/base/optimization.h" +#include "absl/base/port.h" // TODO(strel): remove this include +#include "absl/meta/type_traits.h" +#include "absl/types/internal/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +//------------------------------------------------------------------------------ +// Span +//------------------------------------------------------------------------------ +// +// A `Span` is an "array reference" type for holding a reference of contiguous +// array data; the `Span` object does not and cannot own such data itself. A +// span provides an easy way to provide overloads for anything operating on +// contiguous sequences without needing to manage pointers and array lengths +// manually. + +// A span is conceptually a pointer (ptr) and a length (size) into an already +// existing array of contiguous memory; the array it represents references the +// elements "ptr[0] .. ptr[size-1]". Passing a properly-constructed `Span` +// instead of raw pointers avoids many issues related to index out of bounds +// errors. +// +// Spans may also be constructed from containers holding contiguous sequences. +// Such containers must supply `data()` and `size() const` methods (e.g +// `std::vector<T>`, `absl::InlinedVector<T, N>`). All implicit conversions to +// `absl::Span` from such containers will create spans of type `const T`; +// spans which can mutate their values (of type `T`) must use explicit +// constructors. +// +// A `Span<T>` is somewhat analogous to an `absl::string_view`, but for an array +// of elements of type `T`, and unlike an `absl::string_view`, a span can hold a +// reference to mutable data. A user of `Span` must ensure that the data being +// pointed to outlives the `Span` itself. +// +// You can construct a `Span<T>` in several ways: +// +// * Explicitly from a reference to a container type +// * Explicitly from a pointer and size +// * Implicitly from a container type (but only for spans of type `const T`) +// * Using the `MakeSpan()` or `MakeConstSpan()` factory functions. +// +// Examples: +// +// // Construct a Span explicitly from a container: +// std::vector<int> v = {1, 2, 3, 4, 5}; +// auto span = absl::Span<const int>(v); +// +// // Construct a Span explicitly from a C-style array: +// int a[5] = {1, 2, 3, 4, 5}; +// auto span = absl::Span<const int>(a); +// +// // Construct a Span implicitly from a container +// void MyRoutine(absl::Span<const int> a) { +// ... +// } +// std::vector v = {1,2,3,4,5}; +// MyRoutine(v) // convert to Span<const T> +// +// Note that `Span` objects, in addition to requiring that the memory they +// point to remains alive, must also ensure that such memory does not get +// reallocated. Therefore, to avoid undefined behavior, containers with +// associated spans should not invoke operations that may reallocate memory +// (such as resizing) or invalidate iterators into the container. +// +// One common use for a `Span` is when passing arguments to a routine that can +// accept a variety of array types (e.g. a `std::vector`, `absl::InlinedVector`, +// a C-style array, etc.). Instead of creating overloads for each case, you +// can simply specify a `Span` as the argument to such a routine. +// +// Example: +// +// void MyRoutine(absl::Span<const int> a) { +// ... +// } +// +// std::vector v = {1,2,3,4,5}; +// MyRoutine(v); +// +// absl::InlinedVector<int, 4> my_inline_vector; +// MyRoutine(my_inline_vector); +// +// // Explicit constructor from pointer,size +// int* my_array = new int[10]; +// MyRoutine(absl::Span<const int>(my_array, 10)); +template <typename T> +class Span { + private: + // Used to determine whether a Span can be constructed from a container of + // type C. + template <typename C> + using EnableIfConvertibleFrom = + typename std::enable_if<span_internal::HasData<T, C>::value && + span_internal::HasSize<C>::value>::type; + + // Used to SFINAE-enable a function when the slice elements are const. + template <typename U> + using EnableIfConstView = + typename std::enable_if<std::is_const<T>::value, U>::type; + + // Used to SFINAE-enable a function when the slice elements are mutable. + template <typename U> + using EnableIfMutableView = + typename std::enable_if<!std::is_const<T>::value, U>::type; + + public: + using value_type = absl::remove_cv_t<T>; + using pointer = T*; + using const_pointer = const T*; + using reference = T&; + using const_reference = const T&; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator<iterator>; + using const_reverse_iterator = std::reverse_iterator<const_iterator>; + using size_type = size_t; + using difference_type = ptrdiff_t; + + static const size_type npos = ~(size_type(0)); + + constexpr Span() noexcept : Span(nullptr, 0) {} + constexpr Span(pointer array, size_type length) noexcept + : ptr_(array), len_(length) {} + + // Implicit conversion constructors + template <size_t N> + constexpr Span(T (&a)[N]) noexcept // NOLINT(runtime/explicit) + : Span(a, N) {} + + // Explicit reference constructor for a mutable `Span<T>` type. Can be + // replaced with MakeSpan() to infer the type parameter. + template <typename V, typename = EnableIfConvertibleFrom<V>, + typename = EnableIfMutableView<V>> + explicit Span(V& v) noexcept // NOLINT(runtime/references) + : Span(span_internal::GetData(v), v.size()) {} + + // Implicit reference constructor for a read-only `Span<const T>` type + template <typename V, typename = EnableIfConvertibleFrom<V>, + typename = EnableIfConstView<V>> + constexpr Span(const V& v) noexcept // NOLINT(runtime/explicit) + : Span(span_internal::GetData(v), v.size()) {} + + // Implicit constructor from an initializer list, making it possible to pass a + // brace-enclosed initializer list to a function expecting a `Span`. Such + // spans constructed from an initializer list must be of type `Span<const T>`. + // + // void Process(absl::Span<const int> x); + // Process({1, 2, 3}); + // + // Note that as always the array referenced by the span must outlive the span. + // Since an initializer list constructor acts as if it is fed a temporary + // array (cf. C++ standard [dcl.init.list]/5), it's safe to use this + // constructor only when the `std::initializer_list` itself outlives the span. + // In order to meet this requirement it's sufficient to ensure that neither + // the span nor a copy of it is used outside of the expression in which it's + // created: + // + // // Assume that this function uses the array directly, not retaining any + // // copy of the span or pointer to any of its elements. + // void Process(absl::Span<const int> ints); + // + // // Okay: the std::initializer_list<int> will reference a temporary array + // // that isn't destroyed until after the call to Process returns. + // Process({ 17, 19 }); + // + // // Not okay: the storage used by the std::initializer_list<int> is not + // // allowed to be referenced after the first line. + // absl::Span<const int> ints = { 17, 19 }; + // Process(ints); + // + // // Not okay for the same reason as above: even when the elements of the + // // initializer list expression are not temporaries the underlying array + // // is, so the initializer list must still outlive the span. + // const int foo = 17; + // absl::Span<const int> ints = { foo }; + // Process(ints); + // + template <typename LazyT = T, + typename = EnableIfConstView<LazyT>> + Span( + std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit) + : Span(v.begin(), v.size()) {} + + // Accessors + + // Span::data() + // + // Returns a pointer to the span's underlying array of data (which is held + // outside the span). + constexpr pointer data() const noexcept { return ptr_; } + + // Span::size() + // + // Returns the size of this span. + constexpr size_type size() const noexcept { return len_; } + + // Span::length() + // + // Returns the length (size) of this span. + constexpr size_type length() const noexcept { return size(); } + + // Span::empty() + // + // Returns a boolean indicating whether or not this span is considered empty. + constexpr bool empty() const noexcept { return size() == 0; } + + // Span::operator[] + // + // Returns a reference to the i'th element of this span. + constexpr reference operator[](size_type i) const noexcept { + // MSVC 2015 accepts this as constexpr, but not ptr_[i] + return ABSL_HARDENING_ASSERT(i < size()), *(data() + i); + } + + // Span::at() + // + // Returns a reference to the i'th element of this span. + constexpr reference at(size_type i) const { + return ABSL_PREDICT_TRUE(i < size()) // + ? *(data() + i) + : (base_internal::ThrowStdOutOfRange( + "Span::at failed bounds check"), + *(data() + i)); + } + + // Span::front() + // + // Returns a reference to the first element of this span. The span must not + // be empty. + constexpr reference front() const noexcept { + return ABSL_HARDENING_ASSERT(size() > 0), *data(); + } + + // Span::back() + // + // Returns a reference to the last element of this span. The span must not + // be empty. + constexpr reference back() const noexcept { + return ABSL_HARDENING_ASSERT(size() > 0), *(data() + size() - 1); + } + + // Span::begin() + // + // Returns an iterator pointing to the first element of this span, or `end()` + // if the span is empty. + constexpr iterator begin() const noexcept { return data(); } + + // Span::cbegin() + // + // Returns a const iterator pointing to the first element of this span, or + // `end()` if the span is empty. + constexpr const_iterator cbegin() const noexcept { return begin(); } + + // Span::end() + // + // Returns an iterator pointing just beyond the last element at the + // end of this span. This iterator acts as a placeholder; attempting to + // access it results in undefined behavior. + constexpr iterator end() const noexcept { return data() + size(); } + + // Span::cend() + // + // Returns a const iterator pointing just beyond the last element at the + // end of this span. This iterator acts as a placeholder; attempting to + // access it results in undefined behavior. + constexpr const_iterator cend() const noexcept { return end(); } + + // Span::rbegin() + // + // Returns a reverse iterator pointing to the last element at the end of this + // span, or `rend()` if the span is empty. + constexpr reverse_iterator rbegin() const noexcept { + return reverse_iterator(end()); + } + + // Span::crbegin() + // + // Returns a const reverse iterator pointing to the last element at the end of + // this span, or `crend()` if the span is empty. + constexpr const_reverse_iterator crbegin() const noexcept { return rbegin(); } + + // Span::rend() + // + // Returns a reverse iterator pointing just before the first element + // at the beginning of this span. This pointer acts as a placeholder; + // attempting to access its element results in undefined behavior. + constexpr reverse_iterator rend() const noexcept { + return reverse_iterator(begin()); + } + + // Span::crend() + // + // Returns a reverse const iterator pointing just before the first element + // at the beginning of this span. This pointer acts as a placeholder; + // attempting to access its element results in undefined behavior. + constexpr const_reverse_iterator crend() const noexcept { return rend(); } + + // Span mutations + + // Span::remove_prefix() + // + // Removes the first `n` elements from the span. + void remove_prefix(size_type n) noexcept { + ABSL_HARDENING_ASSERT(size() >= n); + ptr_ += n; + len_ -= n; + } + + // Span::remove_suffix() + // + // Removes the last `n` elements from the span. + void remove_suffix(size_type n) noexcept { + ABSL_HARDENING_ASSERT(size() >= n); + len_ -= n; + } + + // Span::subspan() + // + // Returns a `Span` starting at element `pos` and of length `len`. Both `pos` + // and `len` are of type `size_type` and thus non-negative. Parameter `pos` + // must be <= size(). Any `len` value that points past the end of the span + // will be trimmed to at most size() - `pos`. A default `len` value of `npos` + // ensures the returned subspan continues until the end of the span. + // + // Examples: + // + // std::vector<int> vec = {10, 11, 12, 13}; + // absl::MakeSpan(vec).subspan(1, 2); // {11, 12} + // absl::MakeSpan(vec).subspan(2, 8); // {12, 13} + // absl::MakeSpan(vec).subspan(1); // {11, 12, 13} + // absl::MakeSpan(vec).subspan(4); // {} + // absl::MakeSpan(vec).subspan(5); // throws std::out_of_range + constexpr Span subspan(size_type pos = 0, size_type len = npos) const { + return (pos <= size()) + ? Span(data() + pos, span_internal::Min(size() - pos, len)) + : (base_internal::ThrowStdOutOfRange("pos > size()"), Span()); + } + + // Span::first() + // + // Returns a `Span` containing first `len` elements. Parameter `len` is of + // type `size_type` and thus non-negative. `len` value must be <= size(). + // + // Examples: + // + // std::vector<int> vec = {10, 11, 12, 13}; + // absl::MakeSpan(vec).first(1); // {10} + // absl::MakeSpan(vec).first(3); // {10, 11, 12} + // absl::MakeSpan(vec).first(5); // throws std::out_of_range + constexpr Span first(size_type len) const { + return (len <= size()) + ? Span(data(), len) + : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); + } + + // Span::last() + // + // Returns a `Span` containing last `len` elements. Parameter `len` is of + // type `size_type` and thus non-negative. `len` value must be <= size(). + // + // Examples: + // + // std::vector<int> vec = {10, 11, 12, 13}; + // absl::MakeSpan(vec).last(1); // {13} + // absl::MakeSpan(vec).last(3); // {11, 12, 13} + // absl::MakeSpan(vec).last(5); // throws std::out_of_range + constexpr Span last(size_type len) const { + return (len <= size()) + ? Span(size() - len + data(), len) + : (base_internal::ThrowStdOutOfRange("len > size()"), Span()); + } + + // Support for absl::Hash. + template <typename H> + friend H AbslHashValue(H h, Span v) { + return H::combine(H::combine_contiguous(std::move(h), v.data(), v.size()), + v.size()); + } + + private: + pointer ptr_; + size_type len_; +}; + +template <typename T> +const typename Span<T>::size_type Span<T>::npos; + +// Span relationals + +// Equality is compared element-by-element, while ordering is lexicographical. +// We provide three overloads for each operator to cover any combination on the +// left or right hand side of mutable Span<T>, read-only Span<const T>, and +// convertible-to-read-only Span<T>. +// TODO(zhangxy): Due to MSVC overload resolution bug with partial ordering +// template functions, 5 overloads per operator is needed as a workaround. We +// should update them to 3 overloads per operator using non-deduced context like +// string_view, i.e. +// - (Span<T>, Span<T>) +// - (Span<T>, non_deduced<Span<const T>>) +// - (non_deduced<Span<const T>>, Span<T>) + +// operator== +template <typename T> +bool operator==(Span<T> a, Span<T> b) { + return span_internal::EqualImpl<Span, const T>(a, b); +} +template <typename T> +bool operator==(Span<const T> a, Span<T> b) { + return span_internal::EqualImpl<Span, const T>(a, b); +} +template <typename T> +bool operator==(Span<T> a, Span<const T> b) { + return span_internal::EqualImpl<Span, const T>(a, b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator==(const U& a, Span<T> b) { + return span_internal::EqualImpl<Span, const T>(a, b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator==(Span<T> a, const U& b) { + return span_internal::EqualImpl<Span, const T>(a, b); +} + +// operator!= +template <typename T> +bool operator!=(Span<T> a, Span<T> b) { + return !(a == b); +} +template <typename T> +bool operator!=(Span<const T> a, Span<T> b) { + return !(a == b); +} +template <typename T> +bool operator!=(Span<T> a, Span<const T> b) { + return !(a == b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator!=(const U& a, Span<T> b) { + return !(a == b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator!=(Span<T> a, const U& b) { + return !(a == b); +} + +// operator< +template <typename T> +bool operator<(Span<T> a, Span<T> b) { + return span_internal::LessThanImpl<Span, const T>(a, b); +} +template <typename T> +bool operator<(Span<const T> a, Span<T> b) { + return span_internal::LessThanImpl<Span, const T>(a, b); +} +template <typename T> +bool operator<(Span<T> a, Span<const T> b) { + return span_internal::LessThanImpl<Span, const T>(a, b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator<(const U& a, Span<T> b) { + return span_internal::LessThanImpl<Span, const T>(a, b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator<(Span<T> a, const U& b) { + return span_internal::LessThanImpl<Span, const T>(a, b); +} + +// operator> +template <typename T> +bool operator>(Span<T> a, Span<T> b) { + return b < a; +} +template <typename T> +bool operator>(Span<const T> a, Span<T> b) { + return b < a; +} +template <typename T> +bool operator>(Span<T> a, Span<const T> b) { + return b < a; +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator>(const U& a, Span<T> b) { + return b < a; +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator>(Span<T> a, const U& b) { + return b < a; +} + +// operator<= +template <typename T> +bool operator<=(Span<T> a, Span<T> b) { + return !(b < a); +} +template <typename T> +bool operator<=(Span<const T> a, Span<T> b) { + return !(b < a); +} +template <typename T> +bool operator<=(Span<T> a, Span<const T> b) { + return !(b < a); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator<=(const U& a, Span<T> b) { + return !(b < a); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator<=(Span<T> a, const U& b) { + return !(b < a); +} + +// operator>= +template <typename T> +bool operator>=(Span<T> a, Span<T> b) { + return !(a < b); +} +template <typename T> +bool operator>=(Span<const T> a, Span<T> b) { + return !(a < b); +} +template <typename T> +bool operator>=(Span<T> a, Span<const T> b) { + return !(a < b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator>=(const U& a, Span<T> b) { + return !(a < b); +} +template < + typename T, typename U, + typename = span_internal::EnableIfConvertibleTo<U, absl::Span<const T>>> +bool operator>=(Span<T> a, const U& b) { + return !(a < b); +} + +// MakeSpan() +// +// Constructs a mutable `Span<T>`, deducing `T` automatically from either a +// container or pointer+size. +// +// Because a read-only `Span<const T>` is implicitly constructed from container +// types regardless of whether the container itself is a const container, +// constructing mutable spans of type `Span<T>` from containers requires +// explicit constructors. The container-accepting version of `MakeSpan()` +// deduces the type of `T` by the constness of the pointer received from the +// container's `data()` member. Similarly, the pointer-accepting version returns +// a `Span<const T>` if `T` is `const`, and a `Span<T>` otherwise. +// +// Examples: +// +// void MyRoutine(absl::Span<MyComplicatedType> a) { +// ... +// }; +// // my_vector is a container of non-const types +// std::vector<MyComplicatedType> my_vector; +// +// // Constructing a Span implicitly attempts to create a Span of type +// // `Span<const T>` +// MyRoutine(my_vector); // error, type mismatch +// +// // Explicitly constructing the Span is verbose +// MyRoutine(absl::Span<MyComplicatedType>(my_vector)); +// +// // Use MakeSpan() to make an absl::Span<T> +// MyRoutine(absl::MakeSpan(my_vector)); +// +// // Construct a span from an array ptr+size +// absl::Span<T> my_span() { +// return absl::MakeSpan(&array[0], num_elements_); +// } +// +template <int&... ExplicitArgumentBarrier, typename T> +constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept { + return Span<T>(ptr, size); +} + +template <int&... ExplicitArgumentBarrier, typename T> +Span<T> MakeSpan(T* begin, T* end) noexcept { + return ABSL_HARDENING_ASSERT(begin <= end), Span<T>(begin, end - begin); +} + +template <int&... ExplicitArgumentBarrier, typename C> +constexpr auto MakeSpan(C& c) noexcept // NOLINT(runtime/references) + -> decltype(absl::MakeSpan(span_internal::GetData(c), c.size())) { + return MakeSpan(span_internal::GetData(c), c.size()); +} + +template <int&... ExplicitArgumentBarrier, typename T, size_t N> +constexpr Span<T> MakeSpan(T (&array)[N]) noexcept { + return Span<T>(array, N); +} + +// MakeConstSpan() +// +// Constructs a `Span<const T>` as with `MakeSpan`, deducing `T` automatically, +// but always returning a `Span<const T>`. +// +// Examples: +// +// void ProcessInts(absl::Span<const int> some_ints); +// +// // Call with a pointer and size. +// int array[3] = { 0, 0, 0 }; +// ProcessInts(absl::MakeConstSpan(&array[0], 3)); +// +// // Call with a [begin, end) pair. +// ProcessInts(absl::MakeConstSpan(&array[0], &array[3])); +// +// // Call directly with an array. +// ProcessInts(absl::MakeConstSpan(array)); +// +// // Call with a contiguous container. +// std::vector<int> some_ints = ...; +// ProcessInts(absl::MakeConstSpan(some_ints)); +// ProcessInts(absl::MakeConstSpan(std::vector<int>{ 0, 0, 0 })); +// +template <int&... ExplicitArgumentBarrier, typename T> +constexpr Span<const T> MakeConstSpan(T* ptr, size_t size) noexcept { + return Span<const T>(ptr, size); +} + +template <int&... ExplicitArgumentBarrier, typename T> +Span<const T> MakeConstSpan(T* begin, T* end) noexcept { + return ABSL_HARDENING_ASSERT(begin <= end), Span<const T>(begin, end - begin); +} + +template <int&... ExplicitArgumentBarrier, typename C> +constexpr auto MakeConstSpan(const C& c) noexcept -> decltype(MakeSpan(c)) { + return MakeSpan(c); +} + +template <int&... ExplicitArgumentBarrier, typename T, size_t N> +constexpr Span<const T> MakeConstSpan(const T (&array)[N]) noexcept { + return Span<const T>(array, N); +} +ABSL_NAMESPACE_END +} // namespace absl +#endif // ABSL_TYPES_SPAN_H_ diff --git a/third_party/abseil_cpp/absl/types/span_test.cc b/third_party/abseil_cpp/absl/types/span_test.cc new file mode 100644 index 000000000000..2584339bd309 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/span_test.cc @@ -0,0 +1,846 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/span.h" + +#include <array> +#include <initializer_list> +#include <numeric> +#include <stdexcept> +#include <string> +#include <type_traits> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/exception_testing.h" +#include "absl/base/options.h" +#include "absl/container/fixed_array.h" +#include "absl/container/inlined_vector.h" +#include "absl/hash/hash_testing.h" +#include "absl/strings/str_cat.h" + +namespace { + +MATCHER_P(DataIs, data, + absl::StrCat("data() ", negation ? "isn't " : "is ", + testing::PrintToString(data))) { + return arg.data() == data; +} + +template <typename T> +auto SpanIs(T data, size_t size) + -> decltype(testing::AllOf(DataIs(data), testing::SizeIs(size))) { + return testing::AllOf(DataIs(data), testing::SizeIs(size)); +} + +template <typename Container> +auto SpanIs(const Container& c) -> decltype(SpanIs(c.data(), c.size())) { + return SpanIs(c.data(), c.size()); +} + +std::vector<int> MakeRamp(int len, int offset = 0) { + std::vector<int> v(len); + std::iota(v.begin(), v.end(), offset); + return v; +} + +TEST(IntSpan, EmptyCtors) { + absl::Span<int> s; + EXPECT_THAT(s, SpanIs(nullptr, 0)); +} + +TEST(IntSpan, PtrLenCtor) { + int a[] = {1, 2, 3}; + absl::Span<int> s(&a[0], 2); + EXPECT_THAT(s, SpanIs(a, 2)); +} + +TEST(IntSpan, ArrayCtor) { + int a[] = {1, 2, 3}; + absl::Span<int> s(a); + EXPECT_THAT(s, SpanIs(a, 3)); + + EXPECT_TRUE((std::is_constructible<absl::Span<const int>, int[3]>::value)); + EXPECT_TRUE( + (std::is_constructible<absl::Span<const int>, const int[3]>::value)); + EXPECT_FALSE((std::is_constructible<absl::Span<int>, const int[3]>::value)); + EXPECT_TRUE((std::is_convertible<int[3], absl::Span<const int>>::value)); + EXPECT_TRUE( + (std::is_convertible<const int[3], absl::Span<const int>>::value)); +} + +template <typename T> +void TakesGenericSpan(absl::Span<T>) {} + +TEST(IntSpan, ContainerCtor) { + std::vector<int> empty; + absl::Span<int> s_empty(empty); + EXPECT_THAT(s_empty, SpanIs(empty)); + + std::vector<int> filled{1, 2, 3}; + absl::Span<int> s_filled(filled); + EXPECT_THAT(s_filled, SpanIs(filled)); + + absl::Span<int> s_from_span(filled); + EXPECT_THAT(s_from_span, SpanIs(s_filled)); + + absl::Span<const int> const_filled = filled; + EXPECT_THAT(const_filled, SpanIs(filled)); + + absl::Span<const int> const_from_span = s_filled; + EXPECT_THAT(const_from_span, SpanIs(s_filled)); + + EXPECT_TRUE( + (std::is_convertible<std::vector<int>&, absl::Span<const int>>::value)); + EXPECT_TRUE( + (std::is_convertible<absl::Span<int>&, absl::Span<const int>>::value)); + + TakesGenericSpan(absl::Span<int>(filled)); +} + +// A struct supplying shallow data() const. +struct ContainerWithShallowConstData { + std::vector<int> storage; + int* data() const { return const_cast<int*>(storage.data()); } + int size() const { return storage.size(); } +}; + +TEST(IntSpan, ShallowConstness) { + const ContainerWithShallowConstData c{MakeRamp(20)}; + absl::Span<int> s( + c); // We should be able to do this even though data() is const. + s[0] = -1; + EXPECT_EQ(c.storage[0], -1); +} + +TEST(CharSpan, StringCtor) { + std::string empty = ""; + absl::Span<char> s_empty(empty); + EXPECT_THAT(s_empty, SpanIs(empty)); + + std::string abc = "abc"; + absl::Span<char> s_abc(abc); + EXPECT_THAT(s_abc, SpanIs(abc)); + + absl::Span<const char> s_const_abc = abc; + EXPECT_THAT(s_const_abc, SpanIs(abc)); + + EXPECT_FALSE((std::is_constructible<absl::Span<int>, std::string>::value)); + EXPECT_FALSE( + (std::is_constructible<absl::Span<const int>, std::string>::value)); + EXPECT_TRUE( + (std::is_convertible<std::string, absl::Span<const char>>::value)); +} + +TEST(IntSpan, FromConstPointer) { + EXPECT_TRUE((std::is_constructible<absl::Span<const int* const>, + std::vector<int*>>::value)); + EXPECT_TRUE((std::is_constructible<absl::Span<const int* const>, + std::vector<const int*>>::value)); + EXPECT_FALSE(( + std::is_constructible<absl::Span<const int*>, std::vector<int*>>::value)); + EXPECT_FALSE(( + std::is_constructible<absl::Span<int*>, std::vector<const int*>>::value)); +} + +struct TypeWithMisleadingData { + int& data() { return i; } + int size() { return 1; } + int i; +}; + +struct TypeWithMisleadingSize { + int* data() { return &i; } + const char* size() { return "1"; } + int i; +}; + +TEST(IntSpan, EvilTypes) { + EXPECT_FALSE( + (std::is_constructible<absl::Span<int>, TypeWithMisleadingData&>::value)); + EXPECT_FALSE( + (std::is_constructible<absl::Span<int>, TypeWithMisleadingSize&>::value)); +} + +struct Base { + int* data() { return &i; } + int size() { return 1; } + int i; +}; +struct Derived : Base {}; + +TEST(IntSpan, SpanOfDerived) { + EXPECT_TRUE((std::is_constructible<absl::Span<int>, Base&>::value)); + EXPECT_TRUE((std::is_constructible<absl::Span<int>, Derived&>::value)); + EXPECT_FALSE( + (std::is_constructible<absl::Span<Base>, std::vector<Derived>>::value)); +} + +void TestInitializerList(absl::Span<const int> s, const std::vector<int>& v) { + EXPECT_TRUE(absl::equal(s.begin(), s.end(), v.begin(), v.end())); +} + +TEST(ConstIntSpan, InitializerListConversion) { + TestInitializerList({}, {}); + TestInitializerList({1}, {1}); + TestInitializerList({1, 2, 3}, {1, 2, 3}); + + EXPECT_FALSE((std::is_constructible<absl::Span<int>, + std::initializer_list<int>>::value)); + EXPECT_FALSE(( + std::is_convertible<absl::Span<int>, std::initializer_list<int>>::value)); +} + +TEST(IntSpan, Data) { + int i; + absl::Span<int> s(&i, 1); + EXPECT_EQ(&i, s.data()); +} + +TEST(IntSpan, SizeLengthEmpty) { + absl::Span<int> empty; + EXPECT_EQ(empty.size(), 0); + EXPECT_TRUE(empty.empty()); + EXPECT_EQ(empty.size(), empty.length()); + + auto v = MakeRamp(10); + absl::Span<int> s(v); + EXPECT_EQ(s.size(), 10); + EXPECT_FALSE(s.empty()); + EXPECT_EQ(s.size(), s.length()); +} + +TEST(IntSpan, ElementAccess) { + auto v = MakeRamp(10); + absl::Span<int> s(v); + for (int i = 0; i < s.size(); ++i) { + EXPECT_EQ(s[i], s.at(i)); + } + + EXPECT_EQ(s.front(), s[0]); + EXPECT_EQ(s.back(), s[9]); + +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + EXPECT_DEATH_IF_SUPPORTED(s[-1], ""); + EXPECT_DEATH_IF_SUPPORTED(s[10], ""); +#endif +} + +TEST(IntSpan, AtThrows) { + auto v = MakeRamp(10); + absl::Span<int> s(v); + + EXPECT_EQ(s.at(9), 9); + ABSL_BASE_INTERNAL_EXPECT_FAIL(s.at(10), std::out_of_range, + "failed bounds check"); +} + +TEST(IntSpan, RemovePrefixAndSuffix) { + auto v = MakeRamp(20, 1); + absl::Span<int> s(v); + EXPECT_EQ(s.size(), 20); + + s.remove_suffix(0); + s.remove_prefix(0); + EXPECT_EQ(s.size(), 20); + + s.remove_prefix(1); + EXPECT_EQ(s.size(), 19); + EXPECT_EQ(s[0], 2); + + s.remove_suffix(1); + EXPECT_EQ(s.size(), 18); + EXPECT_EQ(s.back(), 19); + + s.remove_prefix(7); + EXPECT_EQ(s.size(), 11); + EXPECT_EQ(s[0], 9); + + s.remove_suffix(11); + EXPECT_EQ(s.size(), 0); + + EXPECT_EQ(v, MakeRamp(20, 1)); + +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + absl::Span<int> prefix_death(v); + EXPECT_DEATH_IF_SUPPORTED(prefix_death.remove_prefix(21), ""); + absl::Span<int> suffix_death(v); + EXPECT_DEATH_IF_SUPPORTED(suffix_death.remove_suffix(21), ""); +#endif +} + +TEST(IntSpan, Subspan) { + std::vector<int> empty; + EXPECT_EQ(absl::MakeSpan(empty).subspan(), empty); + EXPECT_THAT(absl::MakeSpan(empty).subspan(0, 0), SpanIs(empty)); + EXPECT_THAT(absl::MakeSpan(empty).subspan(0, absl::Span<const int>::npos), + SpanIs(empty)); + + auto ramp = MakeRamp(10); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(), SpanIs(ramp)); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(0, 10), SpanIs(ramp)); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(0, absl::Span<const int>::npos), + SpanIs(ramp)); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(0, 3), SpanIs(ramp.data(), 3)); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(5, absl::Span<const int>::npos), + SpanIs(ramp.data() + 5, 5)); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(3, 3), SpanIs(ramp.data() + 3, 3)); + EXPECT_THAT(absl::MakeSpan(ramp).subspan(10, 5), SpanIs(ramp.data() + 10, 0)); + +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(absl::MakeSpan(ramp).subspan(11, 5), std::out_of_range); +#else + EXPECT_DEATH_IF_SUPPORTED(absl::MakeSpan(ramp).subspan(11, 5), ""); +#endif +} + +TEST(IntSpan, First) { + std::vector<int> empty; + EXPECT_THAT(absl::MakeSpan(empty).first(0), SpanIs(empty)); + + auto ramp = MakeRamp(10); + EXPECT_THAT(absl::MakeSpan(ramp).first(0), SpanIs(ramp.data(), 0)); + EXPECT_THAT(absl::MakeSpan(ramp).first(10), SpanIs(ramp)); + EXPECT_THAT(absl::MakeSpan(ramp).first(3), SpanIs(ramp.data(), 3)); + +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(absl::MakeSpan(ramp).first(11), std::out_of_range); +#else + EXPECT_DEATH_IF_SUPPORTED(absl::MakeSpan(ramp).first(11), ""); +#endif +} + +TEST(IntSpan, Last) { + std::vector<int> empty; + EXPECT_THAT(absl::MakeSpan(empty).last(0), SpanIs(empty)); + + auto ramp = MakeRamp(10); + EXPECT_THAT(absl::MakeSpan(ramp).last(0), SpanIs(ramp.data() + 10, 0)); + EXPECT_THAT(absl::MakeSpan(ramp).last(10), SpanIs(ramp)); + EXPECT_THAT(absl::MakeSpan(ramp).last(3), SpanIs(ramp.data() + 7, 3)); + +#ifdef ABSL_HAVE_EXCEPTIONS + EXPECT_THROW(absl::MakeSpan(ramp).last(11), std::out_of_range); +#else + EXPECT_DEATH_IF_SUPPORTED(absl::MakeSpan(ramp).last(11), ""); +#endif +} + +TEST(IntSpan, MakeSpanPtrLength) { + std::vector<int> empty; + auto s_empty = absl::MakeSpan(empty.data(), empty.size()); + EXPECT_THAT(s_empty, SpanIs(empty)); + + std::array<int, 3> a{{1, 2, 3}}; + auto s = absl::MakeSpan(a.data(), a.size()); + EXPECT_THAT(s, SpanIs(a)); + + EXPECT_THAT(absl::MakeConstSpan(empty.data(), empty.size()), SpanIs(s_empty)); + EXPECT_THAT(absl::MakeConstSpan(a.data(), a.size()), SpanIs(s)); +} + +TEST(IntSpan, MakeSpanTwoPtrs) { + std::vector<int> empty; + auto s_empty = absl::MakeSpan(empty.data(), empty.data()); + EXPECT_THAT(s_empty, SpanIs(empty)); + + std::vector<int> v{1, 2, 3}; + auto s = absl::MakeSpan(v.data(), v.data() + 1); + EXPECT_THAT(s, SpanIs(v.data(), 1)); + + EXPECT_THAT(absl::MakeConstSpan(empty.data(), empty.data()), SpanIs(s_empty)); + EXPECT_THAT(absl::MakeConstSpan(v.data(), v.data() + 1), SpanIs(s)); +} + +TEST(IntSpan, MakeSpanContainer) { + std::vector<int> empty; + auto s_empty = absl::MakeSpan(empty); + EXPECT_THAT(s_empty, SpanIs(empty)); + + std::vector<int> v{1, 2, 3}; + auto s = absl::MakeSpan(v); + EXPECT_THAT(s, SpanIs(v)); + + EXPECT_THAT(absl::MakeConstSpan(empty), SpanIs(s_empty)); + EXPECT_THAT(absl::MakeConstSpan(v), SpanIs(s)); + + EXPECT_THAT(absl::MakeSpan(s), SpanIs(s)); + EXPECT_THAT(absl::MakeConstSpan(s), SpanIs(s)); +} + +TEST(CharSpan, MakeSpanString) { + std::string empty = ""; + auto s_empty = absl::MakeSpan(empty); + EXPECT_THAT(s_empty, SpanIs(empty)); + + std::string str = "abc"; + auto s_str = absl::MakeSpan(str); + EXPECT_THAT(s_str, SpanIs(str)); + + EXPECT_THAT(absl::MakeConstSpan(empty), SpanIs(s_empty)); + EXPECT_THAT(absl::MakeConstSpan(str), SpanIs(s_str)); +} + +TEST(IntSpan, MakeSpanArray) { + int a[] = {1, 2, 3}; + auto s = absl::MakeSpan(a); + EXPECT_THAT(s, SpanIs(a, 3)); + + const int ca[] = {1, 2, 3}; + auto s_ca = absl::MakeSpan(ca); + EXPECT_THAT(s_ca, SpanIs(ca, 3)); + + EXPECT_THAT(absl::MakeConstSpan(a), SpanIs(s)); + EXPECT_THAT(absl::MakeConstSpan(ca), SpanIs(s_ca)); +} + +// Compile-asserts that the argument has the expected decayed type. +template <typename Expected, typename T> +void CheckType(const T& /* value */) { + testing::StaticAssertTypeEq<Expected, T>(); +} + +TEST(IntSpan, MakeSpanTypes) { + std::vector<int> vec; + const std::vector<int> cvec; + int a[1]; + const int ca[] = {1}; + int* ip = a; + const int* cip = ca; + std::string s = ""; + const std::string cs = ""; + CheckType<absl::Span<int>>(absl::MakeSpan(vec)); + CheckType<absl::Span<const int>>(absl::MakeSpan(cvec)); + CheckType<absl::Span<int>>(absl::MakeSpan(ip, ip + 1)); + CheckType<absl::Span<int>>(absl::MakeSpan(ip, 1)); + CheckType<absl::Span<const int>>(absl::MakeSpan(cip, cip + 1)); + CheckType<absl::Span<const int>>(absl::MakeSpan(cip, 1)); + CheckType<absl::Span<int>>(absl::MakeSpan(a)); + CheckType<absl::Span<int>>(absl::MakeSpan(a, a + 1)); + CheckType<absl::Span<int>>(absl::MakeSpan(a, 1)); + CheckType<absl::Span<const int>>(absl::MakeSpan(ca)); + CheckType<absl::Span<const int>>(absl::MakeSpan(ca, ca + 1)); + CheckType<absl::Span<const int>>(absl::MakeSpan(ca, 1)); + CheckType<absl::Span<char>>(absl::MakeSpan(s)); + CheckType<absl::Span<const char>>(absl::MakeSpan(cs)); +} + +TEST(ConstIntSpan, MakeConstSpanTypes) { + std::vector<int> vec; + const std::vector<int> cvec; + int array[1]; + const int carray[] = {0}; + int* ptr = array; + const int* cptr = carray; + std::string s = ""; + std::string cs = ""; + CheckType<absl::Span<const int>>(absl::MakeConstSpan(vec)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(cvec)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(ptr, ptr + 1)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(ptr, 1)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(cptr, cptr + 1)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(cptr, 1)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(array)); + CheckType<absl::Span<const int>>(absl::MakeConstSpan(carray)); + CheckType<absl::Span<const char>>(absl::MakeConstSpan(s)); + CheckType<absl::Span<const char>>(absl::MakeConstSpan(cs)); +} + +TEST(IntSpan, Equality) { + const int arr1[] = {1, 2, 3, 4, 5}; + int arr2[] = {1, 2, 3, 4, 5}; + std::vector<int> vec1(std::begin(arr1), std::end(arr1)); + std::vector<int> vec2 = vec1; + std::vector<int> other_vec = {2, 4, 6, 8, 10}; + // These two slices are from different vectors, but have the same size and + // have the same elements (right now). They should compare equal. Test both + // == and !=. + const absl::Span<const int> from1 = vec1; + const absl::Span<const int> from2 = vec2; + EXPECT_EQ(from1, from1); + EXPECT_FALSE(from1 != from1); + EXPECT_EQ(from1, from2); + EXPECT_FALSE(from1 != from2); + + // These two slices have different underlying vector values. They should be + // considered not equal. Test both == and !=. + const absl::Span<const int> from_other = other_vec; + EXPECT_NE(from1, from_other); + EXPECT_FALSE(from1 == from_other); + + // Comparison between a vector and its slice should be equal. And vice-versa. + // This ensures implicit conversion to Span works on both sides of ==. + EXPECT_EQ(vec1, from1); + EXPECT_FALSE(vec1 != from1); + EXPECT_EQ(from1, vec1); + EXPECT_FALSE(from1 != vec1); + + // This verifies that absl::Span<T> can be compared freely with + // absl::Span<const T>. + const absl::Span<int> mutable_from1(vec1); + const absl::Span<int> mutable_from2(vec2); + EXPECT_EQ(from1, mutable_from1); + EXPECT_EQ(mutable_from1, from1); + EXPECT_EQ(mutable_from1, mutable_from2); + EXPECT_EQ(mutable_from2, mutable_from1); + + // Comparison between a vector and its slice should be equal for mutable + // Spans as well. + EXPECT_EQ(vec1, mutable_from1); + EXPECT_FALSE(vec1 != mutable_from1); + EXPECT_EQ(mutable_from1, vec1); + EXPECT_FALSE(mutable_from1 != vec1); + + // Comparison between convertible-to-Span-of-const and Span-of-mutable. Arrays + // are used because they're the only value type which converts to a + // Span-of-mutable. EXPECT_TRUE is used instead of EXPECT_EQ to avoid + // array-to-pointer decay. + EXPECT_TRUE(arr1 == mutable_from1); + EXPECT_FALSE(arr1 != mutable_from1); + EXPECT_TRUE(mutable_from1 == arr1); + EXPECT_FALSE(mutable_from1 != arr1); + + // Comparison between convertible-to-Span-of-mutable and Span-of-const + EXPECT_TRUE(arr2 == from1); + EXPECT_FALSE(arr2 != from1); + EXPECT_TRUE(from1 == arr2); + EXPECT_FALSE(from1 != arr2); + + // With a different size, the array slices should not be equal. + EXPECT_NE(from1, absl::Span<const int>(from1).subspan(0, from1.size() - 1)); + + // With different contents, the array slices should not be equal. + ++vec2.back(); + EXPECT_NE(from1, from2); +} + +class IntSpanOrderComparisonTest : public testing::Test { + public: + IntSpanOrderComparisonTest() + : arr_before_{1, 2, 3}, + arr_after_{1, 2, 4}, + carr_after_{1, 2, 4}, + vec_before_(std::begin(arr_before_), std::end(arr_before_)), + vec_after_(std::begin(arr_after_), std::end(arr_after_)), + before_(vec_before_), + after_(vec_after_), + cbefore_(vec_before_), + cafter_(vec_after_) {} + + protected: + int arr_before_[3], arr_after_[3]; + const int carr_after_[3]; + std::vector<int> vec_before_, vec_after_; + absl::Span<int> before_, after_; + absl::Span<const int> cbefore_, cafter_; +}; + +TEST_F(IntSpanOrderComparisonTest, CompareSpans) { + EXPECT_TRUE(cbefore_ < cafter_); + EXPECT_TRUE(cbefore_ <= cafter_); + EXPECT_TRUE(cafter_ > cbefore_); + EXPECT_TRUE(cafter_ >= cbefore_); + + EXPECT_FALSE(cbefore_ > cafter_); + EXPECT_FALSE(cafter_ < cbefore_); + + EXPECT_TRUE(before_ < after_); + EXPECT_TRUE(before_ <= after_); + EXPECT_TRUE(after_ > before_); + EXPECT_TRUE(after_ >= before_); + + EXPECT_FALSE(before_ > after_); + EXPECT_FALSE(after_ < before_); + + EXPECT_TRUE(cbefore_ < after_); + EXPECT_TRUE(cbefore_ <= after_); + EXPECT_TRUE(after_ > cbefore_); + EXPECT_TRUE(after_ >= cbefore_); + + EXPECT_FALSE(cbefore_ > after_); + EXPECT_FALSE(after_ < cbefore_); +} + +TEST_F(IntSpanOrderComparisonTest, SpanOfConstAndContainer) { + EXPECT_TRUE(cbefore_ < vec_after_); + EXPECT_TRUE(cbefore_ <= vec_after_); + EXPECT_TRUE(vec_after_ > cbefore_); + EXPECT_TRUE(vec_after_ >= cbefore_); + + EXPECT_FALSE(cbefore_ > vec_after_); + EXPECT_FALSE(vec_after_ < cbefore_); + + EXPECT_TRUE(arr_before_ < cafter_); + EXPECT_TRUE(arr_before_ <= cafter_); + EXPECT_TRUE(cafter_ > arr_before_); + EXPECT_TRUE(cafter_ >= arr_before_); + + EXPECT_FALSE(arr_before_ > cafter_); + EXPECT_FALSE(cafter_ < arr_before_); +} + +TEST_F(IntSpanOrderComparisonTest, SpanOfMutableAndContainer) { + EXPECT_TRUE(vec_before_ < after_); + EXPECT_TRUE(vec_before_ <= after_); + EXPECT_TRUE(after_ > vec_before_); + EXPECT_TRUE(after_ >= vec_before_); + + EXPECT_FALSE(vec_before_ > after_); + EXPECT_FALSE(after_ < vec_before_); + + EXPECT_TRUE(before_ < carr_after_); + EXPECT_TRUE(before_ <= carr_after_); + EXPECT_TRUE(carr_after_ > before_); + EXPECT_TRUE(carr_after_ >= before_); + + EXPECT_FALSE(before_ > carr_after_); + EXPECT_FALSE(carr_after_ < before_); +} + +TEST_F(IntSpanOrderComparisonTest, EqualSpans) { + EXPECT_FALSE(before_ < before_); + EXPECT_TRUE(before_ <= before_); + EXPECT_FALSE(before_ > before_); + EXPECT_TRUE(before_ >= before_); +} + +TEST_F(IntSpanOrderComparisonTest, Subspans) { + auto subspan = before_.subspan(0, 1); + EXPECT_TRUE(subspan < before_); + EXPECT_TRUE(subspan <= before_); + EXPECT_TRUE(before_ > subspan); + EXPECT_TRUE(before_ >= subspan); + + EXPECT_FALSE(subspan > before_); + EXPECT_FALSE(before_ < subspan); +} + +TEST_F(IntSpanOrderComparisonTest, EmptySpans) { + absl::Span<int> empty; + EXPECT_FALSE(empty < empty); + EXPECT_TRUE(empty <= empty); + EXPECT_FALSE(empty > empty); + EXPECT_TRUE(empty >= empty); + + EXPECT_TRUE(empty < before_); + EXPECT_TRUE(empty <= before_); + EXPECT_TRUE(before_ > empty); + EXPECT_TRUE(before_ >= empty); + + EXPECT_FALSE(empty > before_); + EXPECT_FALSE(before_ < empty); +} + +TEST(IntSpan, ExposesContainerTypesAndConsts) { + absl::Span<int> slice; + CheckType<absl::Span<int>::iterator>(slice.begin()); + EXPECT_TRUE((std::is_convertible<decltype(slice.begin()), + absl::Span<int>::const_iterator>::value)); + CheckType<absl::Span<int>::const_iterator>(slice.cbegin()); + EXPECT_TRUE((std::is_convertible<decltype(slice.end()), + absl::Span<int>::const_iterator>::value)); + CheckType<absl::Span<int>::const_iterator>(slice.cend()); + CheckType<absl::Span<int>::reverse_iterator>(slice.rend()); + EXPECT_TRUE( + (std::is_convertible<decltype(slice.rend()), + absl::Span<int>::const_reverse_iterator>::value)); + CheckType<absl::Span<int>::const_reverse_iterator>(slice.crend()); + testing::StaticAssertTypeEq<int, absl::Span<int>::value_type>(); + testing::StaticAssertTypeEq<int, absl::Span<const int>::value_type>(); + testing::StaticAssertTypeEq<int*, absl::Span<int>::pointer>(); + testing::StaticAssertTypeEq<const int*, absl::Span<const int>::pointer>(); + testing::StaticAssertTypeEq<int&, absl::Span<int>::reference>(); + testing::StaticAssertTypeEq<const int&, absl::Span<const int>::reference>(); + testing::StaticAssertTypeEq<const int&, absl::Span<int>::const_reference>(); + testing::StaticAssertTypeEq<const int&, + absl::Span<const int>::const_reference>(); + EXPECT_EQ(static_cast<absl::Span<int>::size_type>(-1), absl::Span<int>::npos); +} + +TEST(IntSpan, IteratorsAndReferences) { + auto accept_pointer = [](int*) {}; + auto accept_reference = [](int&) {}; + auto accept_iterator = [](absl::Span<int>::iterator) {}; + auto accept_const_iterator = [](absl::Span<int>::const_iterator) {}; + auto accept_reverse_iterator = [](absl::Span<int>::reverse_iterator) {}; + auto accept_const_reverse_iterator = + [](absl::Span<int>::const_reverse_iterator) {}; + + int a[1]; + absl::Span<int> s = a; + + accept_pointer(s.data()); + accept_iterator(s.begin()); + accept_const_iterator(s.begin()); + accept_const_iterator(s.cbegin()); + accept_iterator(s.end()); + accept_const_iterator(s.end()); + accept_const_iterator(s.cend()); + accept_reverse_iterator(s.rbegin()); + accept_const_reverse_iterator(s.rbegin()); + accept_const_reverse_iterator(s.crbegin()); + accept_reverse_iterator(s.rend()); + accept_const_reverse_iterator(s.rend()); + accept_const_reverse_iterator(s.crend()); + + accept_reference(s[0]); + accept_reference(s.at(0)); + accept_reference(s.front()); + accept_reference(s.back()); +} + +TEST(IntSpan, IteratorsAndReferences_Const) { + auto accept_pointer = [](int*) {}; + auto accept_reference = [](int&) {}; + auto accept_iterator = [](absl::Span<int>::iterator) {}; + auto accept_const_iterator = [](absl::Span<int>::const_iterator) {}; + auto accept_reverse_iterator = [](absl::Span<int>::reverse_iterator) {}; + auto accept_const_reverse_iterator = + [](absl::Span<int>::const_reverse_iterator) {}; + + int a[1]; + const absl::Span<int> s = a; + + accept_pointer(s.data()); + accept_iterator(s.begin()); + accept_const_iterator(s.begin()); + accept_const_iterator(s.cbegin()); + accept_iterator(s.end()); + accept_const_iterator(s.end()); + accept_const_iterator(s.cend()); + accept_reverse_iterator(s.rbegin()); + accept_const_reverse_iterator(s.rbegin()); + accept_const_reverse_iterator(s.crbegin()); + accept_reverse_iterator(s.rend()); + accept_const_reverse_iterator(s.rend()); + accept_const_reverse_iterator(s.crend()); + + accept_reference(s[0]); + accept_reference(s.at(0)); + accept_reference(s.front()); + accept_reference(s.back()); +} + +TEST(IntSpan, NoexceptTest) { + int a[] = {1, 2, 3}; + std::vector<int> v; + EXPECT_TRUE(noexcept(absl::Span<const int>())); + EXPECT_TRUE(noexcept(absl::Span<const int>(a, 2))); + EXPECT_TRUE(noexcept(absl::Span<const int>(a))); + EXPECT_TRUE(noexcept(absl::Span<const int>(v))); + EXPECT_TRUE(noexcept(absl::Span<int>(v))); + EXPECT_TRUE(noexcept(absl::Span<const int>({1, 2, 3}))); + EXPECT_TRUE(noexcept(absl::MakeSpan(v))); + EXPECT_TRUE(noexcept(absl::MakeSpan(a))); + EXPECT_TRUE(noexcept(absl::MakeSpan(a, 2))); + EXPECT_TRUE(noexcept(absl::MakeSpan(a, a + 1))); + EXPECT_TRUE(noexcept(absl::MakeConstSpan(v))); + EXPECT_TRUE(noexcept(absl::MakeConstSpan(a))); + EXPECT_TRUE(noexcept(absl::MakeConstSpan(a, 2))); + EXPECT_TRUE(noexcept(absl::MakeConstSpan(a, a + 1))); + + absl::Span<int> s(v); + EXPECT_TRUE(noexcept(s.data())); + EXPECT_TRUE(noexcept(s.size())); + EXPECT_TRUE(noexcept(s.length())); + EXPECT_TRUE(noexcept(s.empty())); + EXPECT_TRUE(noexcept(s[0])); + EXPECT_TRUE(noexcept(s.front())); + EXPECT_TRUE(noexcept(s.back())); + EXPECT_TRUE(noexcept(s.begin())); + EXPECT_TRUE(noexcept(s.cbegin())); + EXPECT_TRUE(noexcept(s.end())); + EXPECT_TRUE(noexcept(s.cend())); + EXPECT_TRUE(noexcept(s.rbegin())); + EXPECT_TRUE(noexcept(s.crbegin())); + EXPECT_TRUE(noexcept(s.rend())); + EXPECT_TRUE(noexcept(s.crend())); + EXPECT_TRUE(noexcept(s.remove_prefix(0))); + EXPECT_TRUE(noexcept(s.remove_suffix(0))); +} + +// ConstexprTester exercises expressions in a constexpr context. Simply placing +// the expression in a constexpr function is not enough, as some compilers will +// simply compile the constexpr function as runtime code. Using template +// parameters forces compile-time execution. +template <int i> +struct ConstexprTester {}; + +#define ABSL_TEST_CONSTEXPR(expr) \ + do { \ + ABSL_ATTRIBUTE_UNUSED ConstexprTester<(expr, 1)> t; \ + } while (0) + +struct ContainerWithConstexprMethods { + constexpr int size() const { return 1; } + constexpr const int* data() const { return &i; } + const int i; +}; + +TEST(ConstIntSpan, ConstexprTest) { + static constexpr int a[] = {1, 2, 3}; + static constexpr int sized_arr[2] = {1, 2}; + static constexpr ContainerWithConstexprMethods c{1}; + ABSL_TEST_CONSTEXPR(absl::Span<const int>()); + ABSL_TEST_CONSTEXPR(absl::Span<const int>(a, 2)); + ABSL_TEST_CONSTEXPR(absl::Span<const int>(sized_arr)); + ABSL_TEST_CONSTEXPR(absl::Span<const int>(c)); + ABSL_TEST_CONSTEXPR(absl::MakeSpan(&a[0], 1)); + ABSL_TEST_CONSTEXPR(absl::MakeSpan(c)); + ABSL_TEST_CONSTEXPR(absl::MakeSpan(a)); + ABSL_TEST_CONSTEXPR(absl::MakeConstSpan(&a[0], 1)); + ABSL_TEST_CONSTEXPR(absl::MakeConstSpan(c)); + ABSL_TEST_CONSTEXPR(absl::MakeConstSpan(a)); + + constexpr absl::Span<const int> span = c; + ABSL_TEST_CONSTEXPR(span.data()); + ABSL_TEST_CONSTEXPR(span.size()); + ABSL_TEST_CONSTEXPR(span.length()); + ABSL_TEST_CONSTEXPR(span.empty()); + ABSL_TEST_CONSTEXPR(span.begin()); + ABSL_TEST_CONSTEXPR(span.cbegin()); + ABSL_TEST_CONSTEXPR(span.subspan(0, 0)); + ABSL_TEST_CONSTEXPR(span.first(1)); + ABSL_TEST_CONSTEXPR(span.last(1)); + ABSL_TEST_CONSTEXPR(span[0]); +} + +struct BigStruct { + char bytes[10000]; +}; + +TEST(Span, SpanSize) { + EXPECT_LE(sizeof(absl::Span<int>), 2 * sizeof(void*)); + EXPECT_LE(sizeof(absl::Span<BigStruct>), 2 * sizeof(void*)); +} + +TEST(Span, Hash) { + int array[] = {1, 2, 3, 4}; + int array2[] = {1, 2, 3}; + using T = absl::Span<const int>; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + {// Empties + T(), T(nullptr, 0), T(array, 0), T(array2, 0), + // Different array with same value + T(array, 3), T(array2), T({1, 2, 3}), + // Same array, but different length + T(array, 1), T(array, 2), + // Same length, but different array + T(array + 1, 2), T(array + 2, 2)})); +} + +} // namespace diff --git a/third_party/abseil_cpp/absl/types/variant.h b/third_party/abseil_cpp/absl/types/variant.h new file mode 100644 index 000000000000..ac93464bf8d0 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/variant.h @@ -0,0 +1,866 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// variant.h +// ----------------------------------------------------------------------------- +// +// This header file defines an `absl::variant` type for holding a type-safe +// value of some prescribed set of types (noted as alternative types), and +// associated functions for managing variants. +// +// The `absl::variant` type is a form of type-safe union. An `absl::variant` +// should always hold a value of one of its alternative types (except in the +// "valueless by exception state" -- see below). A default-constructed +// `absl::variant` will hold the value of its first alternative type, provided +// it is default-constructible. +// +// In exceptional cases due to error, an `absl::variant` can hold no +// value (known as a "valueless by exception" state), though this is not the +// norm. +// +// As with `absl::optional`, an `absl::variant` -- when it holds a value -- +// allocates a value of that type directly within the `variant` itself; it +// cannot hold a reference, array, or the type `void`; it can, however, hold a +// pointer to externally managed memory. +// +// `absl::variant` is a C++11 compatible version of the C++17 `std::variant` +// abstraction and is designed to be a drop-in replacement for code compliant +// with C++17. + +#ifndef ABSL_TYPES_VARIANT_H_ +#define ABSL_TYPES_VARIANT_H_ + +#include "absl/base/config.h" +#include "absl/utility/utility.h" + +#ifdef ABSL_USES_STD_VARIANT + +#include <variant> // IWYU pragma: export + +namespace absl { +ABSL_NAMESPACE_BEGIN +using std::bad_variant_access; +using std::get; +using std::get_if; +using std::holds_alternative; +using std::monostate; +using std::variant; +using std::variant_alternative; +using std::variant_alternative_t; +using std::variant_npos; +using std::variant_size; +using std::variant_size_v; +using std::visit; +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_USES_STD_VARIANT + +#include <functional> +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/meta/type_traits.h" +#include "absl/types/internal/variant.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// ----------------------------------------------------------------------------- +// absl::variant +// ----------------------------------------------------------------------------- +// +// An `absl::variant` type is a form of type-safe union. An `absl::variant` -- +// except in exceptional cases -- always holds a value of one of its alternative +// types. +// +// Example: +// +// // Construct a variant that holds either an integer or a std::string and +// // assign it to a std::string. +// absl::variant<int, std::string> v = std::string("abc"); +// +// // A default-constructed variant will hold a value-initialized value of +// // the first alternative type. +// auto a = absl::variant<int, std::string>(); // Holds an int of value '0'. +// +// // variants are assignable. +// +// // copy assignment +// auto v1 = absl::variant<int, std::string>("abc"); +// auto v2 = absl::variant<int, std::string>(10); +// v2 = v1; // copy assign +// +// // move assignment +// auto v1 = absl::variant<int, std::string>("abc"); +// v1 = absl::variant<int, std::string>(10); +// +// // assignment through type conversion +// a = 128; // variant contains int +// a = "128"; // variant contains std::string +// +// An `absl::variant` holding a value of one of its alternative types `T` holds +// an allocation of `T` directly within the variant itself. An `absl::variant` +// is not allowed to allocate additional storage, such as dynamic memory, to +// allocate the contained value. The contained value shall be allocated in a +// region of the variant storage suitably aligned for all alternative types. +template <typename... Ts> +class variant; + +// swap() +// +// Swaps two `absl::variant` values. This function is equivalent to `v.swap(w)` +// where `v` and `w` are `absl::variant` types. +// +// Note that this function requires all alternative types to be both swappable +// and move-constructible, because any two variants may refer to either the same +// type (in which case, they will be swapped) or to two different types (in +// which case the values will need to be moved). +// +template < + typename... Ts, + absl::enable_if_t< + absl::conjunction<std::is_move_constructible<Ts>..., + type_traits_internal::IsSwappable<Ts>...>::value, + int> = 0> +void swap(variant<Ts...>& v, variant<Ts...>& w) noexcept(noexcept(v.swap(w))) { + v.swap(w); +} + +// variant_size +// +// Returns the number of alternative types available for a given `absl::variant` +// type as a compile-time constant expression. As this is a class template, it +// is not generally useful for accessing the number of alternative types of +// any given `absl::variant` instance. +// +// Example: +// +// auto a = absl::variant<int, std::string>; +// constexpr int num_types = +// absl::variant_size<absl::variant<int, std::string>>(); +// +// // You can also use the member constant `value`. +// constexpr int num_types = +// absl::variant_size<absl::variant<int, std::string>>::value; +// +// // `absl::variant_size` is more valuable for use in generic code: +// template <typename Variant> +// constexpr bool IsVariantMultivalue() { +// return absl::variant_size<Variant>() > 1; +// } +// +// Note that the set of cv-qualified specializations of `variant_size` are +// provided to ensure that those specializations compile (especially when passed +// within template logic). +template <class T> +struct variant_size; + +template <class... Ts> +struct variant_size<variant<Ts...>> + : std::integral_constant<std::size_t, sizeof...(Ts)> {}; + +// Specialization of `variant_size` for const qualified variants. +template <class T> +struct variant_size<const T> : variant_size<T>::type {}; + +// Specialization of `variant_size` for volatile qualified variants. +template <class T> +struct variant_size<volatile T> : variant_size<T>::type {}; + +// Specialization of `variant_size` for const volatile qualified variants. +template <class T> +struct variant_size<const volatile T> : variant_size<T>::type {}; + +// variant_alternative +// +// Returns the alternative type for a given `absl::variant` at the passed +// index value as a compile-time constant expression. As this is a class +// template resulting in a type, it is not useful for access of the run-time +// value of any given `absl::variant` variable. +// +// Example: +// +// // The type of the 0th alternative is "int". +// using alternative_type_0 +// = absl::variant_alternative<0, absl::variant<int, std::string>>::type; +// +// static_assert(std::is_same<alternative_type_0, int>::value, ""); +// +// // `absl::variant_alternative` is more valuable for use in generic code: +// template <typename Variant> +// constexpr bool IsFirstElementTrivial() { +// return std::is_trivial_v<variant_alternative<0, Variant>::type>; +// } +// +// Note that the set of cv-qualified specializations of `variant_alternative` +// are provided to ensure that those specializations compile (especially when +// passed within template logic). +template <std::size_t I, class T> +struct variant_alternative; + +template <std::size_t I, class... Types> +struct variant_alternative<I, variant<Types...>> { + using type = + variant_internal::VariantAlternativeSfinaeT<I, variant<Types...>>; +}; + +// Specialization of `variant_alternative` for const qualified variants. +template <std::size_t I, class T> +struct variant_alternative<I, const T> { + using type = const typename variant_alternative<I, T>::type; +}; + +// Specialization of `variant_alternative` for volatile qualified variants. +template <std::size_t I, class T> +struct variant_alternative<I, volatile T> { + using type = volatile typename variant_alternative<I, T>::type; +}; + +// Specialization of `variant_alternative` for const volatile qualified +// variants. +template <std::size_t I, class T> +struct variant_alternative<I, const volatile T> { + using type = const volatile typename variant_alternative<I, T>::type; +}; + +// Template type alias for variant_alternative<I, T>::type. +// +// Example: +// +// using alternative_type_0 +// = absl::variant_alternative_t<0, absl::variant<int, std::string>>; +// static_assert(std::is_same<alternative_type_0, int>::value, ""); +template <std::size_t I, class T> +using variant_alternative_t = typename variant_alternative<I, T>::type; + +// holds_alternative() +// +// Checks whether the given variant currently holds a given alternative type, +// returning `true` if so. +// +// Example: +// +// absl::variant<int, std::string> foo = 42; +// if (absl::holds_alternative<int>(foo)) { +// std::cout << "The variant holds an integer"; +// } +template <class T, class... Types> +constexpr bool holds_alternative(const variant<Types...>& v) noexcept { + static_assert( + variant_internal::UnambiguousIndexOfImpl<variant<Types...>, T, + 0>::value != sizeof...(Types), + "The type T must occur exactly once in Types..."); + return v.index() == + variant_internal::UnambiguousIndexOf<variant<Types...>, T>::value; +} + +// get() +// +// Returns a reference to the value currently within a given variant, using +// either a unique alternative type amongst the variant's set of alternative +// types, or the variant's index value. Attempting to get a variant's value +// using a type that is not unique within the variant's set of alternative types +// is a compile-time error. If the index of the alternative being specified is +// different from the index of the alternative that is currently stored, throws +// `absl::bad_variant_access`. +// +// Example: +// +// auto a = absl::variant<int, std::string>; +// +// // Get the value by type (if unique). +// int i = absl::get<int>(a); +// +// auto b = absl::variant<int, int>; +// +// // Getting the value by a type that is not unique is ill-formed. +// int j = absl::get<int>(b); // Compile Error! +// +// // Getting value by index not ambiguous and allowed. +// int k = absl::get<1>(b); + +// Overload for getting a variant's lvalue by type. +template <class T, class... Types> +constexpr T& get(variant<Types...>& v) { // NOLINT + return variant_internal::VariantCoreAccess::CheckedAccess< + variant_internal::IndexOf<T, Types...>::value>(v); +} + +// Overload for getting a variant's rvalue by type. +// Note: `absl::move()` is required to allow use of constexpr in C++11. +template <class T, class... Types> +constexpr T&& get(variant<Types...>&& v) { + return variant_internal::VariantCoreAccess::CheckedAccess< + variant_internal::IndexOf<T, Types...>::value>(absl::move(v)); +} + +// Overload for getting a variant's const lvalue by type. +template <class T, class... Types> +constexpr const T& get(const variant<Types...>& v) { + return variant_internal::VariantCoreAccess::CheckedAccess< + variant_internal::IndexOf<T, Types...>::value>(v); +} + +// Overload for getting a variant's const rvalue by type. +// Note: `absl::move()` is required to allow use of constexpr in C++11. +template <class T, class... Types> +constexpr const T&& get(const variant<Types...>&& v) { + return variant_internal::VariantCoreAccess::CheckedAccess< + variant_internal::IndexOf<T, Types...>::value>(absl::move(v)); +} + +// Overload for getting a variant's lvalue by index. +template <std::size_t I, class... Types> +constexpr variant_alternative_t<I, variant<Types...>>& get( + variant<Types...>& v) { // NOLINT + return variant_internal::VariantCoreAccess::CheckedAccess<I>(v); +} + +// Overload for getting a variant's rvalue by index. +// Note: `absl::move()` is required to allow use of constexpr in C++11. +template <std::size_t I, class... Types> +constexpr variant_alternative_t<I, variant<Types...>>&& get( + variant<Types...>&& v) { + return variant_internal::VariantCoreAccess::CheckedAccess<I>(absl::move(v)); +} + +// Overload for getting a variant's const lvalue by index. +template <std::size_t I, class... Types> +constexpr const variant_alternative_t<I, variant<Types...>>& get( + const variant<Types...>& v) { + return variant_internal::VariantCoreAccess::CheckedAccess<I>(v); +} + +// Overload for getting a variant's const rvalue by index. +// Note: `absl::move()` is required to allow use of constexpr in C++11. +template <std::size_t I, class... Types> +constexpr const variant_alternative_t<I, variant<Types...>>&& get( + const variant<Types...>&& v) { + return variant_internal::VariantCoreAccess::CheckedAccess<I>(absl::move(v)); +} + +// get_if() +// +// Returns a pointer to the value currently stored within a given variant, if +// present, using either a unique alternative type amongst the variant's set of +// alternative types, or the variant's index value. If such a value does not +// exist, returns `nullptr`. +// +// As with `get`, attempting to get a variant's value using a type that is not +// unique within the variant's set of alternative types is a compile-time error. + +// Overload for getting a pointer to the value stored in the given variant by +// index. +template <std::size_t I, class... Types> +constexpr absl::add_pointer_t<variant_alternative_t<I, variant<Types...>>> +get_if(variant<Types...>* v) noexcept { + return (v != nullptr && v->index() == I) + ? std::addressof( + variant_internal::VariantCoreAccess::Access<I>(*v)) + : nullptr; +} + +// Overload for getting a pointer to the const value stored in the given +// variant by index. +template <std::size_t I, class... Types> +constexpr absl::add_pointer_t<const variant_alternative_t<I, variant<Types...>>> +get_if(const variant<Types...>* v) noexcept { + return (v != nullptr && v->index() == I) + ? std::addressof( + variant_internal::VariantCoreAccess::Access<I>(*v)) + : nullptr; +} + +// Overload for getting a pointer to the value stored in the given variant by +// type. +template <class T, class... Types> +constexpr absl::add_pointer_t<T> get_if(variant<Types...>* v) noexcept { + return absl::get_if<variant_internal::IndexOf<T, Types...>::value>(v); +} + +// Overload for getting a pointer to the const value stored in the given variant +// by type. +template <class T, class... Types> +constexpr absl::add_pointer_t<const T> get_if( + const variant<Types...>* v) noexcept { + return absl::get_if<variant_internal::IndexOf<T, Types...>::value>(v); +} + +// visit() +// +// Calls a provided functor on a given set of variants. `absl::visit()` is +// commonly used to conditionally inspect the state of a given variant (or set +// of variants). +// +// The functor must return the same type when called with any of the variants' +// alternatives. +// +// Example: +// +// // Define a visitor functor +// struct GetVariant { +// template<typename T> +// void operator()(const T& i) const { +// std::cout << "The variant's value is: " << i; +// } +// }; +// +// // Declare our variant, and call `absl::visit()` on it. +// // Note that `GetVariant()` returns void in either case. +// absl::variant<int, std::string> foo = std::string("foo"); +// GetVariant visitor; +// absl::visit(visitor, foo); // Prints `The variant's value is: foo' +template <typename Visitor, typename... Variants> +variant_internal::VisitResult<Visitor, Variants...> visit(Visitor&& vis, + Variants&&... vars) { + return variant_internal:: + VisitIndices<variant_size<absl::decay_t<Variants> >::value...>::Run( + variant_internal::PerformVisitation<Visitor, Variants...>{ + std::forward_as_tuple(absl::forward<Variants>(vars)...), + absl::forward<Visitor>(vis)}, + vars.index()...); +} + +// monostate +// +// The monostate class serves as a first alternative type for a variant for +// which the first variant type is otherwise not default-constructible. +struct monostate {}; + +// `absl::monostate` Relational Operators + +constexpr bool operator<(monostate, monostate) noexcept { return false; } +constexpr bool operator>(monostate, monostate) noexcept { return false; } +constexpr bool operator<=(monostate, monostate) noexcept { return true; } +constexpr bool operator>=(monostate, monostate) noexcept { return true; } +constexpr bool operator==(monostate, monostate) noexcept { return true; } +constexpr bool operator!=(monostate, monostate) noexcept { return false; } + + +//------------------------------------------------------------------------------ +// `absl::variant` Template Definition +//------------------------------------------------------------------------------ +template <typename T0, typename... Tn> +class variant<T0, Tn...> : private variant_internal::VariantBase<T0, Tn...> { + static_assert(absl::conjunction<std::is_object<T0>, + std::is_object<Tn>...>::value, + "Attempted to instantiate a variant containing a non-object " + "type."); + // Intentionally not qualifying `negation` with `absl::` to work around a bug + // in MSVC 2015 with inline namespace and variadic template. + static_assert(absl::conjunction<negation<std::is_array<T0> >, + negation<std::is_array<Tn> >...>::value, + "Attempted to instantiate a variant containing an array type."); + static_assert(absl::conjunction<std::is_nothrow_destructible<T0>, + std::is_nothrow_destructible<Tn>...>::value, + "Attempted to instantiate a variant containing a non-nothrow " + "destructible type."); + + friend struct variant_internal::VariantCoreAccess; + + private: + using Base = variant_internal::VariantBase<T0, Tn...>; + + public: + // Constructors + + // Constructs a variant holding a default-initialized value of the first + // alternative type. + constexpr variant() /*noexcept(see 111above)*/ = default; + + // Copy constructor, standard semantics + variant(const variant& other) = default; + + // Move constructor, standard semantics + variant(variant&& other) /*noexcept(see above)*/ = default; + + // Constructs a variant of an alternative type specified by overload + // resolution of the provided forwarding arguments through + // direct-initialization. + // + // Note: If the selected constructor is a constexpr constructor, this + // constructor shall be a constexpr constructor. + // + // NOTE: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0608r1.html + // has been voted passed the design phase in the C++ standard meeting in Mar + // 2018. It will be implemented and integrated into `absl::variant`. + template < + class T, + std::size_t I = std::enable_if< + variant_internal::IsNeitherSelfNorInPlace<variant, + absl::decay_t<T>>::value, + variant_internal::IndexOfConstructedType<variant, T>>::type::value, + class Tj = absl::variant_alternative_t<I, variant>, + absl::enable_if_t<std::is_constructible<Tj, T>::value>* = + nullptr> + constexpr variant(T&& t) noexcept(std::is_nothrow_constructible<Tj, T>::value) + : Base(variant_internal::EmplaceTag<I>(), absl::forward<T>(t)) {} + + // Constructs a variant of an alternative type from the arguments through + // direct-initialization. + // + // Note: If the selected constructor is a constexpr constructor, this + // constructor shall be a constexpr constructor. + template <class T, class... Args, + typename std::enable_if<std::is_constructible< + variant_internal::UnambiguousTypeOfT<variant, T>, + Args...>::value>::type* = nullptr> + constexpr explicit variant(in_place_type_t<T>, Args&&... args) + : Base(variant_internal::EmplaceTag< + variant_internal::UnambiguousIndexOf<variant, T>::value>(), + absl::forward<Args>(args)...) {} + + // Constructs a variant of an alternative type from an initializer list + // and other arguments through direct-initialization. + // + // Note: If the selected constructor is a constexpr constructor, this + // constructor shall be a constexpr constructor. + template <class T, class U, class... Args, + typename std::enable_if<std::is_constructible< + variant_internal::UnambiguousTypeOfT<variant, T>, + std::initializer_list<U>&, Args...>::value>::type* = nullptr> + constexpr explicit variant(in_place_type_t<T>, std::initializer_list<U> il, + Args&&... args) + : Base(variant_internal::EmplaceTag< + variant_internal::UnambiguousIndexOf<variant, T>::value>(), + il, absl::forward<Args>(args)...) {} + + // Constructs a variant of an alternative type from a provided index, + // through value-initialization using the provided forwarded arguments. + template <std::size_t I, class... Args, + typename std::enable_if<std::is_constructible< + variant_internal::VariantAlternativeSfinaeT<I, variant>, + Args...>::value>::type* = nullptr> + constexpr explicit variant(in_place_index_t<I>, Args&&... args) + : Base(variant_internal::EmplaceTag<I>(), absl::forward<Args>(args)...) {} + + // Constructs a variant of an alternative type from a provided index, + // through value-initialization of an initializer list and the provided + // forwarded arguments. + template <std::size_t I, class U, class... Args, + typename std::enable_if<std::is_constructible< + variant_internal::VariantAlternativeSfinaeT<I, variant>, + std::initializer_list<U>&, Args...>::value>::type* = nullptr> + constexpr explicit variant(in_place_index_t<I>, std::initializer_list<U> il, + Args&&... args) + : Base(variant_internal::EmplaceTag<I>(), il, + absl::forward<Args>(args)...) {} + + // Destructors + + // Destroys the variant's currently contained value, provided that + // `absl::valueless_by_exception()` is false. + ~variant() = default; + + // Assignment Operators + + // Copy assignment operator + variant& operator=(const variant& other) = default; + + // Move assignment operator + variant& operator=(variant&& other) /*noexcept(see above)*/ = default; + + // Converting assignment operator + // + // NOTE: http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p0608r1.html + // has been voted passed the design phase in the C++ standard meeting in Mar + // 2018. It will be implemented and integrated into `absl::variant`. + template < + class T, + std::size_t I = std::enable_if< + !std::is_same<absl::decay_t<T>, variant>::value, + variant_internal::IndexOfConstructedType<variant, T>>::type::value, + class Tj = absl::variant_alternative_t<I, variant>, + typename std::enable_if<std::is_assignable<Tj&, T>::value && + std::is_constructible<Tj, T>::value>::type* = + nullptr> + variant& operator=(T&& t) noexcept( + std::is_nothrow_assignable<Tj&, T>::value&& + std::is_nothrow_constructible<Tj, T>::value) { + variant_internal::VisitIndices<sizeof...(Tn) + 1>::Run( + variant_internal::VariantCoreAccess::MakeConversionAssignVisitor( + this, absl::forward<T>(t)), + index()); + + return *this; + } + + + // emplace() Functions + + // Constructs a value of the given alternative type T within the variant. The + // existing value of the variant is destroyed first (provided that + // `absl::valueless_by_exception()` is false). Requires that T is unambiguous + // in the variant. + // + // Example: + // + // absl::variant<std::vector<int>, int, std::string> v; + // v.emplace<int>(99); + // v.emplace<std::string>("abc"); + template < + class T, class... Args, + typename std::enable_if<std::is_constructible< + absl::variant_alternative_t< + variant_internal::UnambiguousIndexOf<variant, T>::value, variant>, + Args...>::value>::type* = nullptr> + T& emplace(Args&&... args) { + return variant_internal::VariantCoreAccess::Replace< + variant_internal::UnambiguousIndexOf<variant, T>::value>( + this, absl::forward<Args>(args)...); + } + + // Constructs a value of the given alternative type T within the variant using + // an initializer list. The existing value of the variant is destroyed first + // (provided that `absl::valueless_by_exception()` is false). Requires that T + // is unambiguous in the variant. + // + // Example: + // + // absl::variant<std::vector<int>, int, std::string> v; + // v.emplace<std::vector<int>>({0, 1, 2}); + template < + class T, class U, class... Args, + typename std::enable_if<std::is_constructible< + absl::variant_alternative_t< + variant_internal::UnambiguousIndexOf<variant, T>::value, variant>, + std::initializer_list<U>&, Args...>::value>::type* = nullptr> + T& emplace(std::initializer_list<U> il, Args&&... args) { + return variant_internal::VariantCoreAccess::Replace< + variant_internal::UnambiguousIndexOf<variant, T>::value>( + this, il, absl::forward<Args>(args)...); + } + + // Destroys the current value of the variant (provided that + // `absl::valueless_by_exception()` is false) and constructs a new value at + // the given index. + // + // Example: + // + // absl::variant<std::vector<int>, int, int> v; + // v.emplace<1>(99); + // v.emplace<2>(98); + // v.emplace<int>(99); // Won't compile. 'int' isn't a unique type. + template <std::size_t I, class... Args, + typename std::enable_if< + std::is_constructible<absl::variant_alternative_t<I, variant>, + Args...>::value>::type* = nullptr> + absl::variant_alternative_t<I, variant>& emplace(Args&&... args) { + return variant_internal::VariantCoreAccess::Replace<I>( + this, absl::forward<Args>(args)...); + } + + // Destroys the current value of the variant (provided that + // `absl::valueless_by_exception()` is false) and constructs a new value at + // the given index using an initializer list and the provided arguments. + // + // Example: + // + // absl::variant<std::vector<int>, int, int> v; + // v.emplace<0>({0, 1, 2}); + template <std::size_t I, class U, class... Args, + typename std::enable_if<std::is_constructible< + absl::variant_alternative_t<I, variant>, + std::initializer_list<U>&, Args...>::value>::type* = nullptr> + absl::variant_alternative_t<I, variant>& emplace(std::initializer_list<U> il, + Args&&... args) { + return variant_internal::VariantCoreAccess::Replace<I>( + this, il, absl::forward<Args>(args)...); + } + + // variant::valueless_by_exception() + // + // Returns false if and only if the variant currently holds a valid value. + constexpr bool valueless_by_exception() const noexcept { + return this->index_ == absl::variant_npos; + } + + // variant::index() + // + // Returns the index value of the variant's currently selected alternative + // type. + constexpr std::size_t index() const noexcept { return this->index_; } + + // variant::swap() + // + // Swaps the values of two variant objects. + // + void swap(variant& rhs) noexcept( + absl::conjunction< + std::is_nothrow_move_constructible<T0>, + std::is_nothrow_move_constructible<Tn>..., + type_traits_internal::IsNothrowSwappable<T0>, + type_traits_internal::IsNothrowSwappable<Tn>...>::value) { + return variant_internal::VisitIndices<sizeof...(Tn) + 1>::Run( + variant_internal::Swap<T0, Tn...>{this, &rhs}, rhs.index()); + } +}; + +// We need a valid declaration of variant<> for SFINAE and overload resolution +// to work properly above, but we don't need a full declaration since this type +// will never be constructed. This declaration, though incomplete, suffices. +template <> +class variant<>; + +//------------------------------------------------------------------------------ +// Relational Operators +//------------------------------------------------------------------------------ +// +// If neither operand is in the `variant::valueless_by_exception` state: +// +// * If the index of both variants is the same, the relational operator +// returns the result of the corresponding relational operator for the +// corresponding alternative type. +// * If the index of both variants is not the same, the relational operator +// returns the result of that operation applied to the value of the left +// operand's index and the value of the right operand's index. +// * If at least one operand is in the valueless_by_exception state: +// - A variant in the valueless_by_exception state is only considered equal +// to another variant in the valueless_by_exception state. +// - If exactly one operand is in the valueless_by_exception state, the +// variant in the valueless_by_exception state is less than the variant +// that is not in the valueless_by_exception state. +// +// Note: The value 1 is added to each index in the relational comparisons such +// that the index corresponding to the valueless_by_exception state wraps around +// to 0 (the lowest value for the index type), and the remaining indices stay in +// the same relative order. + +// Equal-to operator +template <typename... Types> +constexpr variant_internal::RequireAllHaveEqualT<Types...> operator==( + const variant<Types...>& a, const variant<Types...>& b) { + return (a.index() == b.index()) && + variant_internal::VisitIndices<sizeof...(Types)>::Run( + variant_internal::EqualsOp<Types...>{&a, &b}, a.index()); +} + +// Not equal operator +template <typename... Types> +constexpr variant_internal::RequireAllHaveNotEqualT<Types...> operator!=( + const variant<Types...>& a, const variant<Types...>& b) { + return (a.index() != b.index()) || + variant_internal::VisitIndices<sizeof...(Types)>::Run( + variant_internal::NotEqualsOp<Types...>{&a, &b}, a.index()); +} + +// Less-than operator +template <typename... Types> +constexpr variant_internal::RequireAllHaveLessThanT<Types...> operator<( + const variant<Types...>& a, const variant<Types...>& b) { + return (a.index() != b.index()) + ? (a.index() + 1) < (b.index() + 1) + : variant_internal::VisitIndices<sizeof...(Types)>::Run( + variant_internal::LessThanOp<Types...>{&a, &b}, a.index()); +} + +// Greater-than operator +template <typename... Types> +constexpr variant_internal::RequireAllHaveGreaterThanT<Types...> operator>( + const variant<Types...>& a, const variant<Types...>& b) { + return (a.index() != b.index()) + ? (a.index() + 1) > (b.index() + 1) + : variant_internal::VisitIndices<sizeof...(Types)>::Run( + variant_internal::GreaterThanOp<Types...>{&a, &b}, + a.index()); +} + +// Less-than or equal-to operator +template <typename... Types> +constexpr variant_internal::RequireAllHaveLessThanOrEqualT<Types...> operator<=( + const variant<Types...>& a, const variant<Types...>& b) { + return (a.index() != b.index()) + ? (a.index() + 1) < (b.index() + 1) + : variant_internal::VisitIndices<sizeof...(Types)>::Run( + variant_internal::LessThanOrEqualsOp<Types...>{&a, &b}, + a.index()); +} + +// Greater-than or equal-to operator +template <typename... Types> +constexpr variant_internal::RequireAllHaveGreaterThanOrEqualT<Types...> +operator>=(const variant<Types...>& a, const variant<Types...>& b) { + return (a.index() != b.index()) + ? (a.index() + 1) > (b.index() + 1) + : variant_internal::VisitIndices<sizeof...(Types)>::Run( + variant_internal::GreaterThanOrEqualsOp<Types...>{&a, &b}, + a.index()); +} + +ABSL_NAMESPACE_END +} // namespace absl + +namespace std { + +// hash() +template <> // NOLINT +struct hash<absl::monostate> { + std::size_t operator()(absl::monostate) const { return 0; } +}; + +template <class... T> // NOLINT +struct hash<absl::variant<T...>> + : absl::variant_internal::VariantHashBase<absl::variant<T...>, void, + absl::remove_const_t<T>...> {}; + +} // namespace std + +#endif // ABSL_USES_STD_VARIANT + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace variant_internal { + +// Helper visitor for converting a variant<Ts...>` into another type (mostly +// variant) that can be constructed from any type. +template <typename To> +struct ConversionVisitor { + template <typename T> + To operator()(T&& v) const { + return To(std::forward<T>(v)); + } +}; + +} // namespace variant_internal + +// ConvertVariantTo() +// +// Helper functions to convert an `absl::variant` to a variant of another set of +// types, provided that the alternative type of the new variant type can be +// converted from any type in the source variant. +// +// Example: +// +// absl::variant<name1, name2, float> InternalReq(const Req&); +// +// // name1 and name2 are convertible to name +// absl::variant<name, float> ExternalReq(const Req& req) { +// return absl::ConvertVariantTo<absl::variant<name, float>>( +// InternalReq(req)); +// } +template <typename To, typename Variant> +To ConvertVariantTo(Variant&& variant) { + return absl::visit(variant_internal::ConversionVisitor<To>{}, + std::forward<Variant>(variant)); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_TYPES_VARIANT_H_ diff --git a/third_party/abseil_cpp/absl/types/variant_benchmark.cc b/third_party/abseil_cpp/absl/types/variant_benchmark.cc new file mode 100644 index 000000000000..350b17536413 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/variant_benchmark.cc @@ -0,0 +1,222 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit tests for the variant template. The 'is' and 'IsEmpty' methods +// of variant are not explicitly tested because they are used repeatedly +// in building other tests. All other public variant methods should have +// explicit tests. + +#include "absl/types/variant.h" + +#include <cstddef> +#include <cstdlib> +#include <string> +#include <tuple> + +#include "benchmark/benchmark.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +template <std::size_t I> +struct VariantAlternative { + char member; +}; + +template <class Indices> +struct VariantOfAlternativesImpl; + +template <std::size_t... Indices> +struct VariantOfAlternativesImpl<absl::index_sequence<Indices...>> { + using type = absl::variant<VariantAlternative<Indices>...>; +}; + +template <std::size_t NumAlternatives> +using VariantOfAlternatives = typename VariantOfAlternativesImpl< + absl::make_index_sequence<NumAlternatives>>::type; + +struct Empty {}; + +template <class... T> +void Ignore(T...) noexcept {} + +template <class T> +Empty DoNotOptimizeAndReturnEmpty(T&& arg) noexcept { + benchmark::DoNotOptimize(arg); + return {}; +} + +struct VisitorApplier { + struct Visitor { + template <class... T> + void operator()(T&&... args) const noexcept { + Ignore(DoNotOptimizeAndReturnEmpty(args)...); + } + }; + + template <class... Vars> + void operator()(const Vars&... vars) const noexcept { + absl::visit(Visitor(), vars...); + } +}; + +template <std::size_t NumIndices, std::size_t CurrIndex = NumIndices - 1> +struct MakeWithIndex { + using Variant = VariantOfAlternatives<NumIndices>; + + static Variant Run(std::size_t index) { + return index == CurrIndex + ? Variant(absl::in_place_index_t<CurrIndex>()) + : MakeWithIndex<NumIndices, CurrIndex - 1>::Run(index); + } +}; + +template <std::size_t NumIndices> +struct MakeWithIndex<NumIndices, 0> { + using Variant = VariantOfAlternatives<NumIndices>; + + static Variant Run(std::size_t /*index*/) { return Variant(); } +}; + +template <std::size_t NumIndices, class Dimensions> +struct MakeVariantTuple; + +template <class T, std::size_t /*I*/> +using always_t = T; + +template <std::size_t NumIndices> +VariantOfAlternatives<NumIndices> MakeVariant(std::size_t dimension, + std::size_t index) { + return dimension == 0 + ? MakeWithIndex<NumIndices>::Run(index % NumIndices) + : MakeVariant<NumIndices>(dimension - 1, index / NumIndices); +} + +template <std::size_t NumIndices, std::size_t... Dimensions> +struct MakeVariantTuple<NumIndices, absl::index_sequence<Dimensions...>> { + using VariantTuple = + std::tuple<always_t<VariantOfAlternatives<NumIndices>, Dimensions>...>; + + static VariantTuple Run(int index) { + return std::make_tuple(MakeVariant<NumIndices>(Dimensions, index)...); + } +}; + +constexpr std::size_t integral_pow(std::size_t base, std::size_t power) { + return power == 0 ? 1 : base * integral_pow(base, power - 1); +} + +template <std::size_t End, std::size_t I = 0> +struct VisitTestBody { + template <class Vars, class State> + static bool Run(Vars& vars, State& state) { + if (state.KeepRunning()) { + absl::apply(VisitorApplier(), vars[I]); + return VisitTestBody<End, I + 1>::Run(vars, state); + } + return false; + } +}; + +template <std::size_t End> +struct VisitTestBody<End, End> { + template <class Vars, class State> + static bool Run(Vars& /*vars*/, State& /*state*/) { + return true; + } +}; + +// Visit operations where branch prediction is likely to give a boost. +template <std::size_t NumIndices, std::size_t NumDimensions = 1> +void BM_RedundantVisit(benchmark::State& state) { + auto vars = + MakeVariantTuple<NumIndices, absl::make_index_sequence<NumDimensions>>:: + Run(static_cast<std::size_t>(state.range(0))); + + for (auto _ : state) { // NOLINT + benchmark::DoNotOptimize(vars); + absl::apply(VisitorApplier(), vars); + } +} + +// Visit operations where branch prediction is unlikely to give a boost. +template <std::size_t NumIndices, std::size_t NumDimensions = 1> +void BM_Visit(benchmark::State& state) { + constexpr std::size_t num_possibilities = + integral_pow(NumIndices, NumDimensions); + + using VariantTupleMaker = + MakeVariantTuple<NumIndices, absl::make_index_sequence<NumDimensions>>; + using Tuple = typename VariantTupleMaker::VariantTuple; + + Tuple vars[num_possibilities]; + for (std::size_t i = 0; i < num_possibilities; ++i) + vars[i] = VariantTupleMaker::Run(i); + + while (VisitTestBody<num_possibilities>::Run(vars, state)) { + } +} + +// Visitation +// Each visit is on a different variant with a different active alternative) + +// Unary visit +BENCHMARK_TEMPLATE(BM_Visit, 1); +BENCHMARK_TEMPLATE(BM_Visit, 2); +BENCHMARK_TEMPLATE(BM_Visit, 3); +BENCHMARK_TEMPLATE(BM_Visit, 4); +BENCHMARK_TEMPLATE(BM_Visit, 5); +BENCHMARK_TEMPLATE(BM_Visit, 6); +BENCHMARK_TEMPLATE(BM_Visit, 7); +BENCHMARK_TEMPLATE(BM_Visit, 8); +BENCHMARK_TEMPLATE(BM_Visit, 16); +BENCHMARK_TEMPLATE(BM_Visit, 32); +BENCHMARK_TEMPLATE(BM_Visit, 64); + +// Binary visit +BENCHMARK_TEMPLATE(BM_Visit, 1, 2); +BENCHMARK_TEMPLATE(BM_Visit, 2, 2); +BENCHMARK_TEMPLATE(BM_Visit, 3, 2); +BENCHMARK_TEMPLATE(BM_Visit, 4, 2); +BENCHMARK_TEMPLATE(BM_Visit, 5, 2); + +// Ternary visit +BENCHMARK_TEMPLATE(BM_Visit, 1, 3); +BENCHMARK_TEMPLATE(BM_Visit, 2, 3); +BENCHMARK_TEMPLATE(BM_Visit, 3, 3); + +// Quaternary visit +BENCHMARK_TEMPLATE(BM_Visit, 1, 4); +BENCHMARK_TEMPLATE(BM_Visit, 2, 4); + +// Redundant Visitation +// Each visit consistently has the same alternative active + +// Unary visit +BENCHMARK_TEMPLATE(BM_RedundantVisit, 1)->Arg(0); +BENCHMARK_TEMPLATE(BM_RedundantVisit, 2)->DenseRange(0, 1); +BENCHMARK_TEMPLATE(BM_RedundantVisit, 8)->DenseRange(0, 7); + +// Binary visit +BENCHMARK_TEMPLATE(BM_RedundantVisit, 1, 2)->Arg(0); +BENCHMARK_TEMPLATE(BM_RedundantVisit, 2, 2) + ->DenseRange(0, integral_pow(2, 2) - 1); +BENCHMARK_TEMPLATE(BM_RedundantVisit, 4, 2) + ->DenseRange(0, integral_pow(4, 2) - 1); + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/third_party/abseil_cpp/absl/types/variant_exception_safety_test.cc b/third_party/abseil_cpp/absl/types/variant_exception_safety_test.cc new file mode 100644 index 000000000000..439c6e1df367 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/variant_exception_safety_test.cc @@ -0,0 +1,532 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/types/variant.h" + +#include "absl/base/config.h" + +// This test is a no-op when absl::variant is an alias for std::variant and when +// exceptions are not enabled. +#if !defined(ABSL_USES_STD_VARIANT) && defined(ABSL_HAVE_EXCEPTIONS) + +#include <iostream> +#include <memory> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/internal/exception_safety_testing.h" +#include "absl/memory/memory.h" + +// See comment in absl/base/config.h +#if !defined(ABSL_INTERNAL_MSVC_2017_DBG_MODE) + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +using ::testing::MakeExceptionSafetyTester; +using ::testing::strong_guarantee; +using ::testing::TestNothrowOp; +using ::testing::TestThrowingCtor; + +using Thrower = testing::ThrowingValue<>; +using CopyNothrow = testing::ThrowingValue<testing::TypeSpec::kNoThrowCopy>; +using MoveNothrow = testing::ThrowingValue<testing::TypeSpec::kNoThrowMove>; +using ThrowingAlloc = testing::ThrowingAllocator<Thrower>; +using ThrowerVec = std::vector<Thrower, ThrowingAlloc>; +using ThrowingVariant = + absl::variant<Thrower, CopyNothrow, MoveNothrow, ThrowerVec>; + +struct ConversionException {}; + +template <class T> +struct ExceptionOnConversion { + operator T() const { // NOLINT + throw ConversionException(); + } +}; + +// Forces a variant into the valueless by exception state. +void ToValuelessByException(ThrowingVariant& v) { // NOLINT + try { + v.emplace<Thrower>(); + v.emplace<Thrower>(ExceptionOnConversion<Thrower>()); + } catch (const ConversionException&) { + // This space intentionally left blank. + } +} + +// Check that variant is still in a usable state after an exception is thrown. +testing::AssertionResult VariantInvariants(ThrowingVariant* v) { + using testing::AssertionFailure; + using testing::AssertionSuccess; + + // Try using the active alternative + if (absl::holds_alternative<Thrower>(*v)) { + auto& t = absl::get<Thrower>(*v); + t = Thrower{-100}; + if (t.Get() != -100) { + return AssertionFailure() << "Thrower should be assigned -100"; + } + } else if (absl::holds_alternative<ThrowerVec>(*v)) { + auto& tv = absl::get<ThrowerVec>(*v); + tv.clear(); + tv.emplace_back(-100); + if (tv.size() != 1 || tv[0].Get() != -100) { + return AssertionFailure() << "ThrowerVec should be {Thrower{-100}}"; + } + } else if (absl::holds_alternative<CopyNothrow>(*v)) { + auto& t = absl::get<CopyNothrow>(*v); + t = CopyNothrow{-100}; + if (t.Get() != -100) { + return AssertionFailure() << "CopyNothrow should be assigned -100"; + } + } else if (absl::holds_alternative<MoveNothrow>(*v)) { + auto& t = absl::get<MoveNothrow>(*v); + t = MoveNothrow{-100}; + if (t.Get() != -100) { + return AssertionFailure() << "MoveNothrow should be assigned -100"; + } + } + + // Try making variant valueless_by_exception + if (!v->valueless_by_exception()) ToValuelessByException(*v); + if (!v->valueless_by_exception()) { + return AssertionFailure() << "Variant should be valueless_by_exception"; + } + try { + auto unused = absl::get<Thrower>(*v); + static_cast<void>(unused); + return AssertionFailure() << "Variant should not contain Thrower"; + } catch (const absl::bad_variant_access&) { + } catch (...) { + return AssertionFailure() << "Unexpected exception throw from absl::get"; + } + + // Try using the variant + v->emplace<Thrower>(100); + if (!absl::holds_alternative<Thrower>(*v) || + absl::get<Thrower>(*v) != Thrower(100)) { + return AssertionFailure() << "Variant should contain Thrower(100)"; + } + v->emplace<ThrowerVec>({Thrower(100)}); + if (!absl::holds_alternative<ThrowerVec>(*v) || + absl::get<ThrowerVec>(*v)[0] != Thrower(100)) { + return AssertionFailure() + << "Variant should contain ThrowerVec{Thrower(100)}"; + } + return AssertionSuccess(); +} + +template <typename... Args> +Thrower ExpectedThrower(Args&&... args) { + return Thrower(42, args...); +} + +ThrowerVec ExpectedThrowerVec() { return {Thrower(100), Thrower(200)}; } +ThrowingVariant ValuelessByException() { + ThrowingVariant v; + ToValuelessByException(v); + return v; +} +ThrowingVariant WithThrower() { return Thrower(39); } +ThrowingVariant WithThrowerVec() { + return ThrowerVec{Thrower(1), Thrower(2), Thrower(3)}; +} +ThrowingVariant WithCopyNoThrow() { return CopyNothrow(39); } +ThrowingVariant WithMoveNoThrow() { return MoveNothrow(39); } + +TEST(VariantExceptionSafetyTest, DefaultConstructor) { + TestThrowingCtor<ThrowingVariant>(); +} + +TEST(VariantExceptionSafetyTest, CopyConstructor) { + { + ThrowingVariant v(ExpectedThrower()); + TestThrowingCtor<ThrowingVariant>(v); + } + { + ThrowingVariant v(ExpectedThrowerVec()); + TestThrowingCtor<ThrowingVariant>(v); + } + { + ThrowingVariant v(ValuelessByException()); + TestThrowingCtor<ThrowingVariant>(v); + } +} + +TEST(VariantExceptionSafetyTest, MoveConstructor) { + { + ThrowingVariant v(ExpectedThrower()); + TestThrowingCtor<ThrowingVariant>(std::move(v)); + } + { + ThrowingVariant v(ExpectedThrowerVec()); + TestThrowingCtor<ThrowingVariant>(std::move(v)); + } + { + ThrowingVariant v(ValuelessByException()); + TestThrowingCtor<ThrowingVariant>(std::move(v)); + } +} + +TEST(VariantExceptionSafetyTest, ValueConstructor) { + TestThrowingCtor<ThrowingVariant>(ExpectedThrower()); + TestThrowingCtor<ThrowingVariant>(ExpectedThrowerVec()); +} + +TEST(VariantExceptionSafetyTest, InPlaceTypeConstructor) { + TestThrowingCtor<ThrowingVariant>(absl::in_place_type_t<Thrower>{}, + ExpectedThrower()); + TestThrowingCtor<ThrowingVariant>(absl::in_place_type_t<ThrowerVec>{}, + ExpectedThrowerVec()); +} + +TEST(VariantExceptionSafetyTest, InPlaceIndexConstructor) { + TestThrowingCtor<ThrowingVariant>(absl::in_place_index_t<0>{}, + ExpectedThrower()); + TestThrowingCtor<ThrowingVariant>(absl::in_place_index_t<3>{}, + ExpectedThrowerVec()); +} + +TEST(VariantExceptionSafetyTest, CopyAssign) { + // variant& operator=(const variant& rhs); + // Let j be rhs.index() + { + // - neither *this nor rhs holds a value + const ThrowingVariant rhs = ValuelessByException(); + ThrowingVariant lhs = ValuelessByException(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = rhs; })); + } + { + // - *this holds a value but rhs does not + const ThrowingVariant rhs = ValuelessByException(); + ThrowingVariant lhs = WithThrower(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = rhs; })); + } + // - index() == j + { + const ThrowingVariant rhs(ExpectedThrower()); + auto tester = + MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithOperation([&rhs](ThrowingVariant* lhs) { *lhs = rhs; }); + EXPECT_TRUE(tester.WithContracts(VariantInvariants).Test()); + EXPECT_FALSE(tester.WithContracts(strong_guarantee).Test()); + } + { + const ThrowingVariant rhs(ExpectedThrowerVec()); + auto tester = + MakeExceptionSafetyTester() + .WithInitialValue(WithThrowerVec()) + .WithOperation([&rhs](ThrowingVariant* lhs) { *lhs = rhs; }); + EXPECT_TRUE(tester.WithContracts(VariantInvariants).Test()); + EXPECT_FALSE(tester.WithContracts(strong_guarantee).Test()); + } + // libstdc++ std::variant has bugs on copy assignment regarding exception + // safety. +#if !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) + // index() != j + // if is_nothrow_copy_constructible_v<Tj> or + // !is_nothrow_move_constructible<Tj> is true, equivalent to + // emplace<j>(get<j>(rhs)) + { + // is_nothrow_copy_constructible_v<Tj> == true + // should not throw because emplace() invokes Tj's copy ctor + // which should not throw. + const ThrowingVariant rhs(CopyNothrow{}); + ThrowingVariant lhs = WithThrower(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = rhs; })); + } + { + // is_nothrow_copy_constructible<Tj> == false && + // is_nothrow_move_constructible<Tj> == false + // should provide basic guarantee because emplace() invokes Tj's copy ctor + // which may throw. + const ThrowingVariant rhs(ExpectedThrower()); + auto tester = + MakeExceptionSafetyTester() + .WithInitialValue(WithCopyNoThrow()) + .WithOperation([&rhs](ThrowingVariant* lhs) { *lhs = rhs; }); + EXPECT_TRUE(tester + .WithContracts(VariantInvariants, + [](ThrowingVariant* lhs) { + return lhs->valueless_by_exception(); + }) + .Test()); + EXPECT_FALSE(tester.WithContracts(strong_guarantee).Test()); + } +#endif // !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) + { + // is_nothrow_copy_constructible_v<Tj> == false && + // is_nothrow_move_constructible_v<Tj> == true + // should provide strong guarantee because it is equivalent to + // operator=(variant(rhs)) which creates a temporary then invoke the move + // ctor which shouldn't throw. + const ThrowingVariant rhs(MoveNothrow{}); + EXPECT_TRUE(MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithContracts(VariantInvariants, strong_guarantee) + .Test([&rhs](ThrowingVariant* lhs) { *lhs = rhs; })); + } +} + +TEST(VariantExceptionSafetyTest, MoveAssign) { + // variant& operator=(variant&& rhs); + // Let j be rhs.index() + { + // - neither *this nor rhs holds a value + ThrowingVariant rhs = ValuelessByException(); + ThrowingVariant lhs = ValuelessByException(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = std::move(rhs); })); + } + { + // - *this holds a value but rhs does not + ThrowingVariant rhs = ValuelessByException(); + ThrowingVariant lhs = WithThrower(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = std::move(rhs); })); + } + { + // - index() == j + // assign get<j>(std::move(rhs)) to the value contained in *this. + // If an exception is thrown during call to Tj's move assignment, the state + // of the contained value is as defined by the exception safety guarantee of + // Tj's move assignment; index() will be j. + ThrowingVariant rhs(ExpectedThrower()); + size_t j = rhs.index(); + // Since Thrower's move assignment has basic guarantee, so should variant's. + auto tester = MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithOperation([&](ThrowingVariant* lhs) { + auto copy = rhs; + *lhs = std::move(copy); + }); + EXPECT_TRUE(tester + .WithContracts( + VariantInvariants, + [&](ThrowingVariant* lhs) { return lhs->index() == j; }) + .Test()); + EXPECT_FALSE(tester.WithContracts(strong_guarantee).Test()); + } + { + // libstdc++ introduced a regression between 2018-09-25 and 2019-01-06. + // The fix is targeted for gcc-9. + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87431#c7 + // https://gcc.gnu.org/viewcvs/gcc?view=revision&revision=267614 +#if !(defined(ABSL_USES_STD_VARIANT) && \ + defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE == 8) + // - otherwise (index() != j), equivalent to + // emplace<j>(get<j>(std::move(rhs))) + // - If an exception is thrown during the call to Tj's move construction + // (with j being rhs.index()), the variant will hold no value. + ThrowingVariant rhs(CopyNothrow{}); + EXPECT_TRUE(MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithContracts(VariantInvariants, + [](ThrowingVariant* lhs) { + return lhs->valueless_by_exception(); + }) + .Test([&](ThrowingVariant* lhs) { + auto copy = rhs; + *lhs = std::move(copy); + })); +#endif // !(defined(ABSL_USES_STD_VARIANT) && + // defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE == 8) + } +} + +TEST(VariantExceptionSafetyTest, ValueAssign) { + // template<class T> variant& operator=(T&& t); + // Let Tj be the type that is selected by overload resolution to be assigned. + { + // If *this holds a Tj, assigns std::forward<T>(t) to the value contained in + // *this. If an exception is thrown during the assignment of + // std::forward<T>(t) to the value contained in *this, the state of the + // contained value and t are as defined by the exception safety guarantee of + // the assignment expression; valueless_by_exception() will be false. + // Since Thrower's copy/move assignment has basic guarantee, so should + // variant's. + Thrower rhs = ExpectedThrower(); + // copy assign + auto copy_tester = + MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithOperation([rhs](ThrowingVariant* lhs) { *lhs = rhs; }); + EXPECT_TRUE(copy_tester + .WithContracts(VariantInvariants, + [](ThrowingVariant* lhs) { + return !lhs->valueless_by_exception(); + }) + .Test()); + EXPECT_FALSE(copy_tester.WithContracts(strong_guarantee).Test()); + // move assign + auto move_tester = MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithOperation([&](ThrowingVariant* lhs) { + auto copy = rhs; + *lhs = std::move(copy); + }); + EXPECT_TRUE(move_tester + .WithContracts(VariantInvariants, + [](ThrowingVariant* lhs) { + return !lhs->valueless_by_exception(); + }) + .Test()); + + EXPECT_FALSE(move_tester.WithContracts(strong_guarantee).Test()); + } + // Otherwise (*this holds something else), if is_nothrow_constructible_v<Tj, + // T> || !is_nothrow_move_constructible_v<Tj> is true, equivalent to + // emplace<j>(std::forward<T>(t)). + // We simplify the test by letting T = `const Tj&` or `Tj&&`, so we can reuse + // the CopyNothrow and MoveNothrow types. + + // if is_nothrow_constructible_v<Tj, T> + // (i.e. is_nothrow_copy/move_constructible_v<Tj>) is true, emplace() just + // invokes the copy/move constructor and it should not throw. + { + const CopyNothrow rhs; + ThrowingVariant lhs = WithThrower(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = rhs; })); + } + { + MoveNothrow rhs; + ThrowingVariant lhs = WithThrower(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs = std::move(rhs); })); + } + // if is_nothrow_constructible_v<Tj, T> == false && + // is_nothrow_move_constructible<Tj> == false + // emplace() invokes the copy/move constructor which may throw so it should + // provide basic guarantee and variant object might not hold a value. + { + Thrower rhs = ExpectedThrower(); + // copy + auto copy_tester = + MakeExceptionSafetyTester() + .WithInitialValue(WithCopyNoThrow()) + .WithOperation([&rhs](ThrowingVariant* lhs) { *lhs = rhs; }); + EXPECT_TRUE(copy_tester + .WithContracts(VariantInvariants, + [](ThrowingVariant* lhs) { + return lhs->valueless_by_exception(); + }) + .Test()); + EXPECT_FALSE(copy_tester.WithContracts(strong_guarantee).Test()); + // move + auto move_tester = MakeExceptionSafetyTester() + .WithInitialValue(WithCopyNoThrow()) + .WithOperation([](ThrowingVariant* lhs) { + *lhs = ExpectedThrower(testing::nothrow_ctor); + }); + EXPECT_TRUE(move_tester + .WithContracts(VariantInvariants, + [](ThrowingVariant* lhs) { + return lhs->valueless_by_exception(); + }) + .Test()); + EXPECT_FALSE(move_tester.WithContracts(strong_guarantee).Test()); + } + // Otherwise (if is_nothrow_constructible_v<Tj, T> == false && + // is_nothrow_move_constructible<Tj> == true), + // equivalent to operator=(variant(std::forward<T>(t))) + // This should have strong guarantee because it creates a temporary variant + // and operator=(variant&&) invokes Tj's move ctor which doesn't throw. + // libstdc++ std::variant has bugs on conversion assignment regarding + // exception safety. +#if !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) + { + MoveNothrow rhs; + EXPECT_TRUE(MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithContracts(VariantInvariants, strong_guarantee) + .Test([&rhs](ThrowingVariant* lhs) { *lhs = rhs; })); + } +#endif // !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) +} + +TEST(VariantExceptionSafetyTest, Emplace) { + // If an exception during the initialization of the contained value, the + // variant might not hold a value. The standard requires emplace() to provide + // only basic guarantee. + { + Thrower args = ExpectedThrower(); + auto tester = MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithOperation([&args](ThrowingVariant* v) { + v->emplace<Thrower>(args); + }); + EXPECT_TRUE(tester + .WithContracts(VariantInvariants, + [](ThrowingVariant* v) { + return v->valueless_by_exception(); + }) + .Test()); + EXPECT_FALSE(tester.WithContracts(strong_guarantee).Test()); + } +} + +TEST(VariantExceptionSafetyTest, Swap) { + // if both are valueless_by_exception(), no effect + { + ThrowingVariant rhs = ValuelessByException(); + ThrowingVariant lhs = ValuelessByException(); + EXPECT_TRUE(TestNothrowOp([&]() { lhs.swap(rhs); })); + } + // if index() == rhs.index(), calls swap(get<i>(*this), get<i>(rhs)) + // where i is index(). + { + ThrowingVariant rhs = ExpectedThrower(); + EXPECT_TRUE(MakeExceptionSafetyTester() + .WithInitialValue(WithThrower()) + .WithContracts(VariantInvariants) + .Test([&](ThrowingVariant* lhs) { + auto copy = rhs; + lhs->swap(copy); + })); + } + // Otherwise, exchanges the value of rhs and *this. The exception safety + // involves variant in moved-from state which is not specified in the + // standard, and since swap is 3-step it's impossible for it to provide a + // overall strong guarantee. So, we are only checking basic guarantee here. + { + ThrowingVariant rhs = ExpectedThrower(); + EXPECT_TRUE(MakeExceptionSafetyTester() + .WithInitialValue(WithCopyNoThrow()) + .WithContracts(VariantInvariants) + .Test([&](ThrowingVariant* lhs) { + auto copy = rhs; + lhs->swap(copy); + })); + } + { + ThrowingVariant rhs = ExpectedThrower(); + EXPECT_TRUE(MakeExceptionSafetyTester() + .WithInitialValue(WithCopyNoThrow()) + .WithContracts(VariantInvariants) + .Test([&](ThrowingVariant* lhs) { + auto copy = rhs; + copy.swap(*lhs); + })); + } +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl + +#endif // !defined(ABSL_INTERNAL_MSVC_2017_DBG_MODE) + +#endif // #if !defined(ABSL_USES_STD_VARIANT) && defined(ABSL_HAVE_EXCEPTIONS) diff --git a/third_party/abseil_cpp/absl/types/variant_test.cc b/third_party/abseil_cpp/absl/types/variant_test.cc new file mode 100644 index 000000000000..cf237334da02 --- /dev/null +++ b/third_party/abseil_cpp/absl/types/variant_test.cc @@ -0,0 +1,2718 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Unit tests for the variant template. The 'is' and 'IsEmpty' methods +// of variant are not explicitly tested because they are used repeatedly +// in building other tests. All other public variant methods should have +// explicit tests. + +#include "absl/types/variant.h" + +// This test is a no-op when absl::variant is an alias for std::variant. +#if !defined(ABSL_USES_STD_VARIANT) + +#include <algorithm> +#include <cstddef> +#include <functional> +#include <initializer_list> +#include <memory> +#include <ostream> +#include <queue> +#include <type_traits> +#include <unordered_set> +#include <utility> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/port.h" +#include "absl/memory/memory.h" +#include "absl/meta/type_traits.h" +#include "absl/strings/string_view.h" + +#ifdef ABSL_HAVE_EXCEPTIONS + +#define ABSL_VARIANT_TEST_EXPECT_FAIL(expr, exception_t, text) \ + EXPECT_THROW(expr, exception_t) + +#else + +#define ABSL_VARIANT_TEST_EXPECT_FAIL(expr, exception_t, text) \ + EXPECT_DEATH_IF_SUPPORTED(expr, text) + +#endif // ABSL_HAVE_EXCEPTIONS + +#define ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(...) \ + ABSL_VARIANT_TEST_EXPECT_FAIL((void)(__VA_ARGS__), absl::bad_variant_access, \ + "Bad variant access") + +struct Hashable {}; + +namespace std { +template <> +struct hash<Hashable> { + size_t operator()(const Hashable&); +}; +} // namespace std + +struct NonHashable {}; + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +using ::testing::DoubleEq; +using ::testing::Pointee; +using ::testing::VariantWith; + +struct MoveCanThrow { + MoveCanThrow() : v(0) {} + MoveCanThrow(int v) : v(v) {} // NOLINT(runtime/explicit) + MoveCanThrow(const MoveCanThrow& other) : v(other.v) {} + MoveCanThrow& operator=(const MoveCanThrow& /*other*/) { return *this; } + int v; +}; + +bool operator==(MoveCanThrow lhs, MoveCanThrow rhs) { return lhs.v == rhs.v; } +bool operator!=(MoveCanThrow lhs, MoveCanThrow rhs) { return lhs.v != rhs.v; } +bool operator<(MoveCanThrow lhs, MoveCanThrow rhs) { return lhs.v < rhs.v; } +bool operator<=(MoveCanThrow lhs, MoveCanThrow rhs) { return lhs.v <= rhs.v; } +bool operator>=(MoveCanThrow lhs, MoveCanThrow rhs) { return lhs.v >= rhs.v; } +bool operator>(MoveCanThrow lhs, MoveCanThrow rhs) { return lhs.v > rhs.v; } + +// This helper class allows us to determine if it was swapped with std::swap() +// or with its friend swap() function. +struct SpecialSwap { + explicit SpecialSwap(int i) : i(i) {} + friend void swap(SpecialSwap& a, SpecialSwap& b) { + a.special_swap = b.special_swap = true; + std::swap(a.i, b.i); + } + bool operator==(SpecialSwap other) const { return i == other.i; } + int i; + bool special_swap = false; +}; + +struct MoveOnlyWithListConstructor { + MoveOnlyWithListConstructor() = default; + explicit MoveOnlyWithListConstructor(std::initializer_list<int> /*ilist*/, + int value) + : value(value) {} + MoveOnlyWithListConstructor(MoveOnlyWithListConstructor&&) = default; + MoveOnlyWithListConstructor& operator=(MoveOnlyWithListConstructor&&) = + default; + + int value = 0; +}; + +#ifdef ABSL_HAVE_EXCEPTIONS + +struct ConversionException {}; + +template <class T> +struct ExceptionOnConversion { + operator T() const { // NOLINT(runtime/explicit) + throw ConversionException(); + } +}; + +// Forces a variant into the valueless by exception state. +template <class H, class... T> +void ToValuelessByException(absl::variant<H, T...>& v) { // NOLINT + try { + v.template emplace<0>(ExceptionOnConversion<H>()); + } catch (ConversionException& /*e*/) { + // This space intentionally left blank. + } +} + +#endif // ABSL_HAVE_EXCEPTIONS + +// An indexed sequence of distinct structures holding a single +// value of type T +template<typename T, size_t N> +struct ValueHolder { + explicit ValueHolder(const T& x) : value(x) {} + typedef T value_type; + value_type value; + static const size_t kIndex = N; +}; +template<typename T, size_t N> +const size_t ValueHolder<T, N>::kIndex; + +// The following three functions make ValueHolder compatible with +// EXPECT_EQ and EXPECT_NE +template<typename T, size_t N> +inline bool operator==(const ValueHolder<T, N>& left, + const ValueHolder<T, N>& right) { + return left.value == right.value; +} + +template<typename T, size_t N> +inline bool operator!=(const ValueHolder<T, N>& left, + const ValueHolder<T, N>& right) { + return left.value != right.value; +} + +template<typename T, size_t N> +inline std::ostream& operator<<( + std::ostream& stream, const ValueHolder<T, N>& object) { + return stream << object.value; +} + +// Makes a variant holding twelve uniquely typed T wrappers. +template<typename T> +struct VariantFactory { + typedef variant<ValueHolder<T, 1>, ValueHolder<T, 2>, ValueHolder<T, 3>, + ValueHolder<T, 4>> + Type; +}; + +// A typelist in 1:1 with VariantFactory, to use type driven unit tests. +typedef ::testing::Types<ValueHolder<size_t, 1>, ValueHolder<size_t, 2>, + ValueHolder<size_t, 3>, + ValueHolder<size_t, 4>> VariantTypes; + +// Increments the provided counter pointer in the destructor +struct IncrementInDtor { + explicit IncrementInDtor(int* counter) : counter(counter) {} + ~IncrementInDtor() { *counter += 1; } + int* counter; +}; + +struct IncrementInDtorCopyCanThrow { + explicit IncrementInDtorCopyCanThrow(int* counter) : counter(counter) {} + IncrementInDtorCopyCanThrow(IncrementInDtorCopyCanThrow&& other) noexcept = + default; + IncrementInDtorCopyCanThrow(const IncrementInDtorCopyCanThrow& other) + : counter(other.counter) {} + IncrementInDtorCopyCanThrow& operator=( + IncrementInDtorCopyCanThrow&&) noexcept = default; + IncrementInDtorCopyCanThrow& operator=( + IncrementInDtorCopyCanThrow const& other) { + counter = other.counter; + return *this; + } + ~IncrementInDtorCopyCanThrow() { *counter += 1; } + int* counter; +}; + +// This is defined so operator== for ValueHolder<IncrementInDtor> will +// return true if two IncrementInDtor objects increment the same +// counter +inline bool operator==(const IncrementInDtor& left, + const IncrementInDtor& right) { + return left.counter == right.counter; +} + +// This is defined so EXPECT_EQ can work with IncrementInDtor +inline std::ostream& operator<<( + std::ostream& stream, const IncrementInDtor& object) { + return stream << object.counter; +} + +// A class that can be copied, but not assigned. +class CopyNoAssign { + public: + explicit CopyNoAssign(int value) : foo(value) {} + CopyNoAssign(const CopyNoAssign& other) : foo(other.foo) {} + int foo; + private: + const CopyNoAssign& operator=(const CopyNoAssign&); +}; + +// A class that can neither be copied nor assigned. We provide +// overloads for the constructor with up to four parameters so we can +// test the overloads of variant::emplace. +class NonCopyable { + public: + NonCopyable() + : value(0) {} + explicit NonCopyable(int value1) + : value(value1) {} + + NonCopyable(int value1, int value2) + : value(value1 + value2) {} + + NonCopyable(int value1, int value2, int value3) + : value(value1 + value2 + value3) {} + + NonCopyable(int value1, int value2, int value3, int value4) + : value(value1 + value2 + value3 + value4) {} + NonCopyable(const NonCopyable&) = delete; + NonCopyable& operator=(const NonCopyable&) = delete; + int value; +}; + +// A typed test and typed test case over the VariantTypes typelist, +// from which we derive a number of tests that will execute for one of +// each type. +template <typename T> +class VariantTypesTest : public ::testing::Test {}; +TYPED_TEST_SUITE(VariantTypesTest, VariantTypes); + +//////////////////// +// [variant.ctor] // +//////////////////// + +struct NonNoexceptDefaultConstructible { + NonNoexceptDefaultConstructible() {} + int value = 5; +}; + +struct NonDefaultConstructible { + NonDefaultConstructible() = delete; +}; + +TEST(VariantTest, TestDefaultConstructor) { + { + using X = variant<int>; + constexpr variant<int> x{}; + ASSERT_FALSE(x.valueless_by_exception()); + ASSERT_EQ(0, x.index()); + EXPECT_EQ(0, absl::get<0>(x)); + EXPECT_TRUE(std::is_nothrow_default_constructible<X>::value); + } + + { + using X = variant<NonNoexceptDefaultConstructible>; + X x{}; + ASSERT_FALSE(x.valueless_by_exception()); + ASSERT_EQ(0, x.index()); + EXPECT_EQ(5, absl::get<0>(x).value); + EXPECT_FALSE(std::is_nothrow_default_constructible<X>::value); + } + + { + using X = variant<int, NonNoexceptDefaultConstructible>; + X x{}; + ASSERT_FALSE(x.valueless_by_exception()); + ASSERT_EQ(0, x.index()); + EXPECT_EQ(0, absl::get<0>(x)); + EXPECT_TRUE(std::is_nothrow_default_constructible<X>::value); + } + + { + using X = variant<NonNoexceptDefaultConstructible, int>; + X x{}; + ASSERT_FALSE(x.valueless_by_exception()); + ASSERT_EQ(0, x.index()); + EXPECT_EQ(5, absl::get<0>(x).value); + EXPECT_FALSE(std::is_nothrow_default_constructible<X>::value); + } + EXPECT_FALSE( + std::is_default_constructible<variant<NonDefaultConstructible>>::value); + EXPECT_FALSE((std::is_default_constructible< + variant<NonDefaultConstructible, int>>::value)); + EXPECT_TRUE((std::is_default_constructible< + variant<int, NonDefaultConstructible>>::value)); +} + +// Test that for each slot, copy constructing a variant with that type +// produces a sensible object that correctly reports its type, and +// that copies the provided value. +TYPED_TEST(VariantTypesTest, TestCopyCtor) { + typedef typename VariantFactory<typename TypeParam::value_type>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + using value_type2 = absl::variant_alternative_t<1, Variant>; + using value_type3 = absl::variant_alternative_t<2, Variant>; + using value_type4 = absl::variant_alternative_t<3, Variant>; + const TypeParam value(TypeParam::kIndex); + Variant original(value); + Variant copied(original); + EXPECT_TRUE(absl::holds_alternative<value_type1>(copied) || + TypeParam::kIndex != 1); + EXPECT_TRUE(absl::holds_alternative<value_type2>(copied) || + TypeParam::kIndex != 2); + EXPECT_TRUE(absl::holds_alternative<value_type3>(copied) || + TypeParam::kIndex != 3); + EXPECT_TRUE(absl::holds_alternative<value_type4>(copied) || + TypeParam::kIndex != 4); + EXPECT_TRUE((absl::get_if<value_type1>(&original) == + absl::get_if<value_type1>(&copied)) || + TypeParam::kIndex == 1); + EXPECT_TRUE((absl::get_if<value_type2>(&original) == + absl::get_if<value_type2>(&copied)) || + TypeParam::kIndex == 2); + EXPECT_TRUE((absl::get_if<value_type3>(&original) == + absl::get_if<value_type3>(&copied)) || + TypeParam::kIndex == 3); + EXPECT_TRUE((absl::get_if<value_type4>(&original) == + absl::get_if<value_type4>(&copied)) || + TypeParam::kIndex == 4); + EXPECT_TRUE((absl::get_if<value_type1>(&original) == + absl::get_if<value_type1>(&copied)) || + TypeParam::kIndex == 1); + EXPECT_TRUE((absl::get_if<value_type2>(&original) == + absl::get_if<value_type2>(&copied)) || + TypeParam::kIndex == 2); + EXPECT_TRUE((absl::get_if<value_type3>(&original) == + absl::get_if<value_type3>(&copied)) || + TypeParam::kIndex == 3); + EXPECT_TRUE((absl::get_if<value_type4>(&original) == + absl::get_if<value_type4>(&copied)) || + TypeParam::kIndex == 4); + const TypeParam* ovalptr = absl::get_if<TypeParam>(&original); + const TypeParam* cvalptr = absl::get_if<TypeParam>(&copied); + ASSERT_TRUE(ovalptr != nullptr); + ASSERT_TRUE(cvalptr != nullptr); + EXPECT_EQ(*ovalptr, *cvalptr); + TypeParam* mutable_ovalptr = absl::get_if<TypeParam>(&original); + TypeParam* mutable_cvalptr = absl::get_if<TypeParam>(&copied); + ASSERT_TRUE(mutable_ovalptr != nullptr); + ASSERT_TRUE(mutable_cvalptr != nullptr); + EXPECT_EQ(*mutable_ovalptr, *mutable_cvalptr); +} + +template <class> +struct MoveOnly { + MoveOnly() = default; + explicit MoveOnly(int value) : value(value) {} + MoveOnly(MoveOnly&&) = default; + MoveOnly& operator=(MoveOnly&&) = default; + int value = 5; +}; + +TEST(VariantTest, TestMoveConstruct) { + using V = variant<MoveOnly<class A>, MoveOnly<class B>, MoveOnly<class C>>; + + V v(in_place_index<1>, 10); + V v2 = absl::move(v); + EXPECT_EQ(10, absl::get<1>(v2).value); +} + +// Used internally to emulate missing triviality traits for tests. +template <class T> +union SingleUnion { + T member; +}; + +// NOTE: These don't work with types that can't be union members. +// They are just for testing. +template <class T> +struct is_trivially_move_constructible + : std::is_move_constructible<SingleUnion<T>>::type {}; + +template <class T> +struct is_trivially_move_assignable + : absl::is_move_assignable<SingleUnion<T>>::type {}; + +TEST(VariantTest, NothrowMoveConstructible) { + // Verify that variant is nothrow move constructible iff its template + // arguments are. + using U = std::unique_ptr<int>; + struct E { + E(E&&) {} + }; + static_assert(std::is_nothrow_move_constructible<variant<U>>::value, ""); + static_assert(std::is_nothrow_move_constructible<variant<U, int>>::value, ""); + static_assert(!std::is_nothrow_move_constructible<variant<U, E>>::value, ""); +} + +// Test that for each slot, constructing a variant with that type +// produces a sensible object that correctly reports its type, and +// that copies the provided value. +TYPED_TEST(VariantTypesTest, TestValueCtor) { + typedef typename VariantFactory<typename TypeParam::value_type>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + using value_type2 = absl::variant_alternative_t<1, Variant>; + using value_type3 = absl::variant_alternative_t<2, Variant>; + using value_type4 = absl::variant_alternative_t<3, Variant>; + const TypeParam value(TypeParam::kIndex); + Variant v(value); + EXPECT_TRUE(absl::holds_alternative<value_type1>(v) || + TypeParam::kIndex != 1); + EXPECT_TRUE(absl::holds_alternative<value_type2>(v) || + TypeParam::kIndex != 2); + EXPECT_TRUE(absl::holds_alternative<value_type3>(v) || + TypeParam::kIndex != 3); + EXPECT_TRUE(absl::holds_alternative<value_type4>(v) || + TypeParam::kIndex != 4); + EXPECT_TRUE(nullptr != absl::get_if<value_type1>(&v) || + TypeParam::kIndex != 1); + EXPECT_TRUE(nullptr != absl::get_if<value_type2>(&v) || + TypeParam::kIndex != 2); + EXPECT_TRUE(nullptr != absl::get_if<value_type3>(&v) || + TypeParam::kIndex != 3); + EXPECT_TRUE(nullptr != absl::get_if<value_type4>(&v) || + TypeParam::kIndex != 4); + EXPECT_TRUE(nullptr != absl::get_if<value_type1>(&v) || + TypeParam::kIndex != 1); + EXPECT_TRUE(nullptr != absl::get_if<value_type2>(&v) || + TypeParam::kIndex != 2); + EXPECT_TRUE(nullptr != absl::get_if<value_type3>(&v) || + TypeParam::kIndex != 3); + EXPECT_TRUE(nullptr != absl::get_if<value_type4>(&v) || + TypeParam::kIndex != 4); + const TypeParam* valptr = absl::get_if<TypeParam>(&v); + ASSERT_TRUE(nullptr != valptr); + EXPECT_EQ(value.value, valptr->value); + const TypeParam* mutable_valptr = absl::get_if<TypeParam>(&v); + ASSERT_TRUE(nullptr != mutable_valptr); + EXPECT_EQ(value.value, mutable_valptr->value); +} + +TEST(VariantTest, AmbiguousValueConstructor) { + EXPECT_FALSE((std::is_convertible<int, absl::variant<int, int>>::value)); + EXPECT_FALSE((std::is_constructible<absl::variant<int, int>, int>::value)); +} + +TEST(VariantTest, InPlaceType) { + using Var = variant<int, std::string, NonCopyable, std::vector<int>>; + + Var v1(in_place_type_t<int>(), 7); + ASSERT_TRUE(absl::holds_alternative<int>(v1)); + EXPECT_EQ(7, absl::get<int>(v1)); + + Var v2(in_place_type_t<std::string>(), "ABC"); + ASSERT_TRUE(absl::holds_alternative<std::string>(v2)); + EXPECT_EQ("ABC", absl::get<std::string>(v2)); + + Var v3(in_place_type_t<std::string>(), "ABC", 2); + ASSERT_TRUE(absl::holds_alternative<std::string>(v3)); + EXPECT_EQ("AB", absl::get<std::string>(v3)); + + Var v4(in_place_type_t<NonCopyable>{}); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v4)); + + Var v5(in_place_type_t<std::vector<int>>(), {1, 2, 3}); + ASSERT_TRUE(absl::holds_alternative<std::vector<int>>(v5)); + EXPECT_THAT(absl::get<std::vector<int>>(v5), ::testing::ElementsAre(1, 2, 3)); +} + +TEST(VariantTest, InPlaceTypeVariableTemplate) { + using Var = variant<int, std::string, NonCopyable, std::vector<int>>; + + Var v1(in_place_type<int>, 7); + ASSERT_TRUE(absl::holds_alternative<int>(v1)); + EXPECT_EQ(7, absl::get<int>(v1)); + + Var v2(in_place_type<std::string>, "ABC"); + ASSERT_TRUE(absl::holds_alternative<std::string>(v2)); + EXPECT_EQ("ABC", absl::get<std::string>(v2)); + + Var v3(in_place_type<std::string>, "ABC", 2); + ASSERT_TRUE(absl::holds_alternative<std::string>(v3)); + EXPECT_EQ("AB", absl::get<std::string>(v3)); + + Var v4(in_place_type<NonCopyable>); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v4)); + + Var v5(in_place_type<std::vector<int>>, {1, 2, 3}); + ASSERT_TRUE(absl::holds_alternative<std::vector<int>>(v5)); + EXPECT_THAT(absl::get<std::vector<int>>(v5), ::testing::ElementsAre(1, 2, 3)); +} + +TEST(VariantTest, InPlaceTypeInitializerList) { + using Var = + variant<int, std::string, NonCopyable, MoveOnlyWithListConstructor>; + + Var v1(in_place_type_t<MoveOnlyWithListConstructor>(), {1, 2, 3, 4, 5}, 6); + ASSERT_TRUE(absl::holds_alternative<MoveOnlyWithListConstructor>(v1)); + EXPECT_EQ(6, absl::get<MoveOnlyWithListConstructor>(v1).value); +} + +TEST(VariantTest, InPlaceTypeInitializerListVariabletemplate) { + using Var = + variant<int, std::string, NonCopyable, MoveOnlyWithListConstructor>; + + Var v1(in_place_type<MoveOnlyWithListConstructor>, {1, 2, 3, 4, 5}, 6); + ASSERT_TRUE(absl::holds_alternative<MoveOnlyWithListConstructor>(v1)); + EXPECT_EQ(6, absl::get<MoveOnlyWithListConstructor>(v1).value); +} + +TEST(VariantTest, InPlaceIndex) { + using Var = variant<int, std::string, NonCopyable, std::vector<int>>; + + Var v1(in_place_index_t<0>(), 7); + ASSERT_TRUE(absl::holds_alternative<int>(v1)); + EXPECT_EQ(7, absl::get<int>(v1)); + + Var v2(in_place_index_t<1>(), "ABC"); + ASSERT_TRUE(absl::holds_alternative<std::string>(v2)); + EXPECT_EQ("ABC", absl::get<std::string>(v2)); + + Var v3(in_place_index_t<1>(), "ABC", 2); + ASSERT_TRUE(absl::holds_alternative<std::string>(v3)); + EXPECT_EQ("AB", absl::get<std::string>(v3)); + + Var v4(in_place_index_t<2>{}); + EXPECT_TRUE(absl::holds_alternative<NonCopyable>(v4)); + + // Verify that a variant with only non-copyables can still be constructed. + EXPECT_TRUE(absl::holds_alternative<NonCopyable>( + variant<NonCopyable>(in_place_index_t<0>{}))); + + Var v5(in_place_index_t<3>(), {1, 2, 3}); + ASSERT_TRUE(absl::holds_alternative<std::vector<int>>(v5)); + EXPECT_THAT(absl::get<std::vector<int>>(v5), ::testing::ElementsAre(1, 2, 3)); +} + +TEST(VariantTest, InPlaceIndexVariableTemplate) { + using Var = variant<int, std::string, NonCopyable, std::vector<int>>; + + Var v1(in_place_index<0>, 7); + ASSERT_TRUE(absl::holds_alternative<int>(v1)); + EXPECT_EQ(7, absl::get<int>(v1)); + + Var v2(in_place_index<1>, "ABC"); + ASSERT_TRUE(absl::holds_alternative<std::string>(v2)); + EXPECT_EQ("ABC", absl::get<std::string>(v2)); + + Var v3(in_place_index<1>, "ABC", 2); + ASSERT_TRUE(absl::holds_alternative<std::string>(v3)); + EXPECT_EQ("AB", absl::get<std::string>(v3)); + + Var v4(in_place_index<2>); + EXPECT_TRUE(absl::holds_alternative<NonCopyable>(v4)); + + // Verify that a variant with only non-copyables can still be constructed. + EXPECT_TRUE(absl::holds_alternative<NonCopyable>( + variant<NonCopyable>(in_place_index<0>))); + + Var v5(in_place_index<3>, {1, 2, 3}); + ASSERT_TRUE(absl::holds_alternative<std::vector<int>>(v5)); + EXPECT_THAT(absl::get<std::vector<int>>(v5), ::testing::ElementsAre(1, 2, 3)); +} + +TEST(VariantTest, InPlaceIndexInitializerList) { + using Var = + variant<int, std::string, NonCopyable, MoveOnlyWithListConstructor>; + + Var v1(in_place_index_t<3>(), {1, 2, 3, 4, 5}, 6); + ASSERT_TRUE(absl::holds_alternative<MoveOnlyWithListConstructor>(v1)); + EXPECT_EQ(6, absl::get<MoveOnlyWithListConstructor>(v1).value); +} + +TEST(VariantTest, InPlaceIndexInitializerListVariableTemplate) { + using Var = + variant<int, std::string, NonCopyable, MoveOnlyWithListConstructor>; + + Var v1(in_place_index<3>, {1, 2, 3, 4, 5}, 6); + ASSERT_TRUE(absl::holds_alternative<MoveOnlyWithListConstructor>(v1)); + EXPECT_EQ(6, absl::get<MoveOnlyWithListConstructor>(v1).value); +} + +//////////////////// +// [variant.dtor] // +//////////////////// + +// Make sure that the destructor destroys the contained value +TEST(VariantTest, TestDtor) { + typedef VariantFactory<IncrementInDtor>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + using value_type2 = absl::variant_alternative_t<1, Variant>; + using value_type3 = absl::variant_alternative_t<2, Variant>; + using value_type4 = absl::variant_alternative_t<3, Variant>; + int counter = 0; + IncrementInDtor counter_adjuster(&counter); + EXPECT_EQ(0, counter); + + value_type1 value1(counter_adjuster); + { Variant object(value1); } + EXPECT_EQ(1, counter); + + value_type2 value2(counter_adjuster); + { Variant object(value2); } + EXPECT_EQ(2, counter); + + value_type3 value3(counter_adjuster); + { Variant object(value3); } + EXPECT_EQ(3, counter); + + value_type4 value4(counter_adjuster); + { Variant object(value4); } + EXPECT_EQ(4, counter); +} + +#ifdef ABSL_HAVE_EXCEPTIONS + +// See comment in absl/base/config.h +#if defined(ABSL_INTERNAL_MSVC_2017_DBG_MODE) +TEST(VariantTest, DISABLED_TestDtorValuelessByException) +#else +// Test destruction when in the valueless_by_exception state. +TEST(VariantTest, TestDtorValuelessByException) +#endif +{ + int counter = 0; + IncrementInDtor counter_adjuster(&counter); + + { + using Variant = VariantFactory<IncrementInDtor>::Type; + + Variant v(in_place_index<0>, counter_adjuster); + EXPECT_EQ(0, counter); + + ToValuelessByException(v); + ASSERT_TRUE(v.valueless_by_exception()); + EXPECT_EQ(1, counter); + } + EXPECT_EQ(1, counter); +} + +#endif // ABSL_HAVE_EXCEPTIONS + +////////////////////// +// [variant.assign] // +////////////////////// + +// Test that self-assignment doesn't destroy the current value +TEST(VariantTest, TestSelfAssignment) { + typedef VariantFactory<IncrementInDtor>::Type Variant; + int counter = 0; + IncrementInDtor counter_adjuster(&counter); + absl::variant_alternative_t<0, Variant> value(counter_adjuster); + Variant object(value); + object.operator=(object); + EXPECT_EQ(0, counter); + + // A string long enough that it's likely to defeat any inline representation + // optimization. + const std::string long_str(128, 'a'); + + std::string foo = long_str; + foo = *&foo; + EXPECT_EQ(long_str, foo); + + variant<int, std::string> so = long_str; + ASSERT_EQ(1, so.index()); + EXPECT_EQ(long_str, absl::get<1>(so)); + so = *&so; + + ASSERT_EQ(1, so.index()); + EXPECT_EQ(long_str, absl::get<1>(so)); +} + +// Test that assigning a variant<..., T, ...> to a variant<..., T, ...> produces +// a variant<..., T, ...> with the correct value. +TYPED_TEST(VariantTypesTest, TestAssignmentCopiesValueSameTypes) { + typedef typename VariantFactory<typename TypeParam::value_type>::Type Variant; + const TypeParam value(TypeParam::kIndex); + const Variant source(value); + Variant target(TypeParam(value.value + 1)); + ASSERT_TRUE(absl::holds_alternative<TypeParam>(source)); + ASSERT_TRUE(absl::holds_alternative<TypeParam>(target)); + ASSERT_NE(absl::get<TypeParam>(source), absl::get<TypeParam>(target)); + target = source; + ASSERT_TRUE(absl::holds_alternative<TypeParam>(source)); + ASSERT_TRUE(absl::holds_alternative<TypeParam>(target)); + EXPECT_EQ(absl::get<TypeParam>(source), absl::get<TypeParam>(target)); +} + +// Test that assisnging a variant<..., T, ...> to a variant<1, ...> +// produces a variant<..., T, ...> with the correct value. +TYPED_TEST(VariantTypesTest, TestAssignmentCopiesValuesVaryingSourceType) { + typedef typename VariantFactory<typename TypeParam::value_type>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + const TypeParam value(TypeParam::kIndex); + const Variant source(value); + ASSERT_TRUE(absl::holds_alternative<TypeParam>(source)); + Variant target(value_type1(1)); + ASSERT_TRUE(absl::holds_alternative<value_type1>(target)); + target = source; + EXPECT_TRUE(absl::holds_alternative<TypeParam>(source)); + EXPECT_TRUE(absl::holds_alternative<TypeParam>(target)); + EXPECT_EQ(absl::get<TypeParam>(source), absl::get<TypeParam>(target)); +} + +// Test that assigning a variant<1, ...> to a variant<..., T, ...> +// produces a variant<1, ...> with the correct value. +TYPED_TEST(VariantTypesTest, TestAssignmentCopiesValuesVaryingTargetType) { + typedef typename VariantFactory<typename TypeParam::value_type>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + const Variant source(value_type1(1)); + ASSERT_TRUE(absl::holds_alternative<value_type1>(source)); + const TypeParam value(TypeParam::kIndex); + Variant target(value); + ASSERT_TRUE(absl::holds_alternative<TypeParam>(target)); + target = source; + EXPECT_TRUE(absl::holds_alternative<value_type1>(target)); + EXPECT_TRUE(absl::holds_alternative<value_type1>(source)); + EXPECT_EQ(absl::get<value_type1>(source), absl::get<value_type1>(target)); +} + +// Test that operator=<T> works, that assigning a new value destroys +// the old and that assigning the new value again does not redestroy +// the old +TEST(VariantTest, TestAssign) { + typedef VariantFactory<IncrementInDtor>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + using value_type2 = absl::variant_alternative_t<1, Variant>; + using value_type3 = absl::variant_alternative_t<2, Variant>; + using value_type4 = absl::variant_alternative_t<3, Variant>; + + const int kSize = 4; + int counter[kSize]; + std::unique_ptr<IncrementInDtor> counter_adjustor[kSize]; + for (int i = 0; i != kSize; i++) { + counter[i] = 0; + counter_adjustor[i] = absl::make_unique<IncrementInDtor>(&counter[i]); + } + + value_type1 v1(*counter_adjustor[0]); + value_type2 v2(*counter_adjustor[1]); + value_type3 v3(*counter_adjustor[2]); + value_type4 v4(*counter_adjustor[3]); + + // Test that reassignment causes destruction of old value + { + Variant object(v1); + object = v2; + object = v3; + object = v4; + object = v1; + } + + EXPECT_EQ(2, counter[0]); + EXPECT_EQ(1, counter[1]); + EXPECT_EQ(1, counter[2]); + EXPECT_EQ(1, counter[3]); + + std::fill(std::begin(counter), std::end(counter), 0); + + // Test that self-assignment does not cause destruction of old value + { + Variant object(v1); + object.operator=(object); + EXPECT_EQ(0, counter[0]); + } + { + Variant object(v2); + object.operator=(object); + EXPECT_EQ(0, counter[1]); + } + { + Variant object(v3); + object.operator=(object); + EXPECT_EQ(0, counter[2]); + } + { + Variant object(v4); + object.operator=(object); + EXPECT_EQ(0, counter[3]); + } + + EXPECT_EQ(1, counter[0]); + EXPECT_EQ(1, counter[1]); + EXPECT_EQ(1, counter[2]); + EXPECT_EQ(1, counter[3]); +} + +// This tests that we perform a backup if the copy-assign can throw but the move +// cannot throw. +TEST(VariantTest, TestBackupAssign) { + typedef VariantFactory<IncrementInDtorCopyCanThrow>::Type Variant; + using value_type1 = absl::variant_alternative_t<0, Variant>; + using value_type2 = absl::variant_alternative_t<1, Variant>; + using value_type3 = absl::variant_alternative_t<2, Variant>; + using value_type4 = absl::variant_alternative_t<3, Variant>; + + const int kSize = 4; + int counter[kSize]; + std::unique_ptr<IncrementInDtorCopyCanThrow> counter_adjustor[kSize]; + for (int i = 0; i != kSize; i++) { + counter[i] = 0; + counter_adjustor[i].reset(new IncrementInDtorCopyCanThrow(&counter[i])); + } + + value_type1 v1(*counter_adjustor[0]); + value_type2 v2(*counter_adjustor[1]); + value_type3 v3(*counter_adjustor[2]); + value_type4 v4(*counter_adjustor[3]); + + // Test that reassignment causes destruction of old value + { + Variant object(v1); + object = v2; + object = v3; + object = v4; + object = v1; + } + + // libstdc++ doesn't pass this test +#if !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) + EXPECT_EQ(3, counter[0]); + EXPECT_EQ(2, counter[1]); + EXPECT_EQ(2, counter[2]); + EXPECT_EQ(2, counter[3]); +#endif + + std::fill(std::begin(counter), std::end(counter), 0); + + // Test that self-assignment does not cause destruction of old value + { + Variant object(v1); + object.operator=(object); + EXPECT_EQ(0, counter[0]); + } + { + Variant object(v2); + object.operator=(object); + EXPECT_EQ(0, counter[1]); + } + { + Variant object(v3); + object.operator=(object); + EXPECT_EQ(0, counter[2]); + } + { + Variant object(v4); + object.operator=(object); + EXPECT_EQ(0, counter[3]); + } + + EXPECT_EQ(1, counter[0]); + EXPECT_EQ(1, counter[1]); + EXPECT_EQ(1, counter[2]); + EXPECT_EQ(1, counter[3]); +} + +/////////////////// +// [variant.mod] // +/////////////////// + +TEST(VariantTest, TestEmplaceBasic) { + using Variant = variant<int, char>; + + Variant v(absl::in_place_index<0>, 0); + + { + char& emplace_result = v.emplace<char>(); + ASSERT_TRUE(absl::holds_alternative<char>(v)); + EXPECT_EQ(absl::get<char>(v), 0); + EXPECT_EQ(&emplace_result, &absl::get<char>(v)); + } + + // Make sure that another emplace does zero-initialization + absl::get<char>(v) = 'a'; + v.emplace<char>('b'); + ASSERT_TRUE(absl::holds_alternative<char>(v)); + EXPECT_EQ(absl::get<char>(v), 'b'); + + { + int& emplace_result = v.emplace<int>(); + EXPECT_TRUE(absl::holds_alternative<int>(v)); + EXPECT_EQ(absl::get<int>(v), 0); + EXPECT_EQ(&emplace_result, &absl::get<int>(v)); + } +} + +TEST(VariantTest, TestEmplaceInitializerList) { + using Var = + variant<int, std::string, NonCopyable, MoveOnlyWithListConstructor>; + + Var v1(absl::in_place_index<0>, 555); + MoveOnlyWithListConstructor& emplace_result = + v1.emplace<MoveOnlyWithListConstructor>({1, 2, 3, 4, 5}, 6); + ASSERT_TRUE(absl::holds_alternative<MoveOnlyWithListConstructor>(v1)); + EXPECT_EQ(6, absl::get<MoveOnlyWithListConstructor>(v1).value); + EXPECT_EQ(&emplace_result, &absl::get<MoveOnlyWithListConstructor>(v1)); +} + +TEST(VariantTest, TestEmplaceIndex) { + using Variant = variant<int, char>; + + Variant v(absl::in_place_index<0>, 555); + + { + char& emplace_result = v.emplace<1>(); + ASSERT_TRUE(absl::holds_alternative<char>(v)); + EXPECT_EQ(absl::get<char>(v), 0); + EXPECT_EQ(&emplace_result, &absl::get<char>(v)); + } + + // Make sure that another emplace does zero-initialization + absl::get<char>(v) = 'a'; + v.emplace<1>('b'); + ASSERT_TRUE(absl::holds_alternative<char>(v)); + EXPECT_EQ(absl::get<char>(v), 'b'); + + { + int& emplace_result = v.emplace<0>(); + EXPECT_TRUE(absl::holds_alternative<int>(v)); + EXPECT_EQ(absl::get<int>(v), 0); + EXPECT_EQ(&emplace_result, &absl::get<int>(v)); + } +} + +TEST(VariantTest, TestEmplaceIndexInitializerList) { + using Var = + variant<int, std::string, NonCopyable, MoveOnlyWithListConstructor>; + + Var v1(absl::in_place_index<0>, 555); + MoveOnlyWithListConstructor& emplace_result = + v1.emplace<3>({1, 2, 3, 4, 5}, 6); + ASSERT_TRUE(absl::holds_alternative<MoveOnlyWithListConstructor>(v1)); + EXPECT_EQ(6, absl::get<MoveOnlyWithListConstructor>(v1).value); + EXPECT_EQ(&emplace_result, &absl::get<MoveOnlyWithListConstructor>(v1)); +} + +////////////////////// +// [variant.status] // +////////////////////// + +TEST(VariantTest, Index) { + using Var = variant<int, std::string, double>; + + Var v = 1; + EXPECT_EQ(0, v.index()); + v = "str"; + EXPECT_EQ(1, v.index()); + v = 0.; + EXPECT_EQ(2, v.index()); + + Var v2 = v; + EXPECT_EQ(2, v2.index()); + v2.emplace<int>(3); + EXPECT_EQ(0, v2.index()); +} + +TEST(VariantTest, NotValuelessByException) { + using Var = variant<int, std::string, double>; + + Var v = 1; + EXPECT_FALSE(v.valueless_by_exception()); + v = "str"; + EXPECT_FALSE(v.valueless_by_exception()); + v = 0.; + EXPECT_FALSE(v.valueless_by_exception()); + + Var v2 = v; + EXPECT_FALSE(v.valueless_by_exception()); + v2.emplace<int>(3); + EXPECT_FALSE(v.valueless_by_exception()); +} + +#ifdef ABSL_HAVE_EXCEPTIONS + +TEST(VariantTest, IndexValuelessByException) { + using Var = variant<MoveCanThrow, std::string, double>; + + Var v(absl::in_place_index<0>); + EXPECT_EQ(0, v.index()); + ToValuelessByException(v); + EXPECT_EQ(absl::variant_npos, v.index()); + v = "str"; + EXPECT_EQ(1, v.index()); +} + +TEST(VariantTest, ValuelessByException) { + using Var = variant<MoveCanThrow, std::string, double>; + + Var v(absl::in_place_index<0>); + EXPECT_FALSE(v.valueless_by_exception()); + ToValuelessByException(v); + EXPECT_TRUE(v.valueless_by_exception()); + v = "str"; + EXPECT_FALSE(v.valueless_by_exception()); +} + +#endif // ABSL_HAVE_EXCEPTIONS + +//////////////////// +// [variant.swap] // +//////////////////// + +TEST(VariantTest, MemberSwap) { + SpecialSwap v1(3); + SpecialSwap v2(7); + + variant<SpecialSwap> a = v1, b = v2; + + EXPECT_THAT(a, VariantWith<SpecialSwap>(v1)); + EXPECT_THAT(b, VariantWith<SpecialSwap>(v2)); + + a.swap(b); + EXPECT_THAT(a, VariantWith<SpecialSwap>(v2)); + EXPECT_THAT(b, VariantWith<SpecialSwap>(v1)); + EXPECT_TRUE(absl::get<SpecialSwap>(a).special_swap); + + using V = variant<MoveCanThrow, std::string, int>; + int i = 33; + std::string s = "abc"; + { + // lhs and rhs holds different alternative + V lhs(i), rhs(s); + lhs.swap(rhs); + EXPECT_THAT(lhs, VariantWith<std::string>(s)); + EXPECT_THAT(rhs, VariantWith<int>(i)); + } +#ifdef ABSL_HAVE_EXCEPTIONS + V valueless(in_place_index<0>); + ToValuelessByException(valueless); + { + // lhs is valueless + V lhs(valueless), rhs(i); + lhs.swap(rhs); + EXPECT_THAT(lhs, VariantWith<int>(i)); + EXPECT_TRUE(rhs.valueless_by_exception()); + } + { + // rhs is valueless + V lhs(s), rhs(valueless); + lhs.swap(rhs); + EXPECT_THAT(rhs, VariantWith<std::string>(s)); + EXPECT_TRUE(lhs.valueless_by_exception()); + } + { + // both are valueless + V lhs(valueless), rhs(valueless); + lhs.swap(rhs); + EXPECT_TRUE(lhs.valueless_by_exception()); + EXPECT_TRUE(rhs.valueless_by_exception()); + } +#endif // ABSL_HAVE_EXCEPTIONS +} + +////////////////////// +// [variant.helper] // +////////////////////// + +TEST(VariantTest, VariantSize) { + { + using Size1Variant = absl::variant<int>; + EXPECT_EQ(1, absl::variant_size<Size1Variant>::value); + EXPECT_EQ(1, absl::variant_size<const Size1Variant>::value); + EXPECT_EQ(1, absl::variant_size<volatile Size1Variant>::value); + EXPECT_EQ(1, absl::variant_size<const volatile Size1Variant>::value); + } + + { + using Size3Variant = absl::variant<int, float, int>; + EXPECT_EQ(3, absl::variant_size<Size3Variant>::value); + EXPECT_EQ(3, absl::variant_size<const Size3Variant>::value); + EXPECT_EQ(3, absl::variant_size<volatile Size3Variant>::value); + EXPECT_EQ(3, absl::variant_size<const volatile Size3Variant>::value); + } +} + +TEST(VariantTest, VariantAlternative) { + { + using V = absl::variant<float, int, const char*>; + EXPECT_TRUE( + (std::is_same<float, absl::variant_alternative_t<0, V>>::value)); + EXPECT_TRUE((std::is_same<const float, + absl::variant_alternative_t<0, const V>>::value)); + EXPECT_TRUE( + (std::is_same<volatile float, + absl::variant_alternative_t<0, volatile V>>::value)); + EXPECT_TRUE(( + std::is_same<const volatile float, + absl::variant_alternative_t<0, const volatile V>>::value)); + + EXPECT_TRUE((std::is_same<int, absl::variant_alternative_t<1, V>>::value)); + EXPECT_TRUE((std::is_same<const int, + absl::variant_alternative_t<1, const V>>::value)); + EXPECT_TRUE( + (std::is_same<volatile int, + absl::variant_alternative_t<1, volatile V>>::value)); + EXPECT_TRUE(( + std::is_same<const volatile int, + absl::variant_alternative_t<1, const volatile V>>::value)); + + EXPECT_TRUE( + (std::is_same<const char*, absl::variant_alternative_t<2, V>>::value)); + EXPECT_TRUE((std::is_same<const char* const, + absl::variant_alternative_t<2, const V>>::value)); + EXPECT_TRUE( + (std::is_same<const char* volatile, + absl::variant_alternative_t<2, volatile V>>::value)); + EXPECT_TRUE(( + std::is_same<const char* const volatile, + absl::variant_alternative_t<2, const volatile V>>::value)); + } + + { + using V = absl::variant<float, volatile int, const char*>; + EXPECT_TRUE( + (std::is_same<float, absl::variant_alternative_t<0, V>>::value)); + EXPECT_TRUE((std::is_same<const float, + absl::variant_alternative_t<0, const V>>::value)); + EXPECT_TRUE( + (std::is_same<volatile float, + absl::variant_alternative_t<0, volatile V>>::value)); + EXPECT_TRUE(( + std::is_same<const volatile float, + absl::variant_alternative_t<0, const volatile V>>::value)); + + EXPECT_TRUE( + (std::is_same<volatile int, absl::variant_alternative_t<1, V>>::value)); + EXPECT_TRUE((std::is_same<const volatile int, + absl::variant_alternative_t<1, const V>>::value)); + EXPECT_TRUE( + (std::is_same<volatile int, + absl::variant_alternative_t<1, volatile V>>::value)); + EXPECT_TRUE(( + std::is_same<const volatile int, + absl::variant_alternative_t<1, const volatile V>>::value)); + + EXPECT_TRUE( + (std::is_same<const char*, absl::variant_alternative_t<2, V>>::value)); + EXPECT_TRUE((std::is_same<const char* const, + absl::variant_alternative_t<2, const V>>::value)); + EXPECT_TRUE( + (std::is_same<const char* volatile, + absl::variant_alternative_t<2, volatile V>>::value)); + EXPECT_TRUE(( + std::is_same<const char* const volatile, + absl::variant_alternative_t<2, const volatile V>>::value)); + } +} + +/////////////////// +// [variant.get] // +/////////////////// + +TEST(VariantTest, HoldsAlternative) { + using Var = variant<int, std::string, double>; + + Var v = 1; + EXPECT_TRUE(absl::holds_alternative<int>(v)); + EXPECT_FALSE(absl::holds_alternative<std::string>(v)); + EXPECT_FALSE(absl::holds_alternative<double>(v)); + v = "str"; + EXPECT_FALSE(absl::holds_alternative<int>(v)); + EXPECT_TRUE(absl::holds_alternative<std::string>(v)); + EXPECT_FALSE(absl::holds_alternative<double>(v)); + v = 0.; + EXPECT_FALSE(absl::holds_alternative<int>(v)); + EXPECT_FALSE(absl::holds_alternative<std::string>(v)); + EXPECT_TRUE(absl::holds_alternative<double>(v)); + + Var v2 = v; + EXPECT_FALSE(absl::holds_alternative<int>(v2)); + EXPECT_FALSE(absl::holds_alternative<std::string>(v2)); + EXPECT_TRUE(absl::holds_alternative<double>(v2)); + v2.emplace<int>(3); + EXPECT_TRUE(absl::holds_alternative<int>(v2)); + EXPECT_FALSE(absl::holds_alternative<std::string>(v2)); + EXPECT_FALSE(absl::holds_alternative<double>(v2)); +} + +TEST(VariantTest, GetIndex) { + using Var = variant<int, std::string, double, int>; + + { + Var v(absl::in_place_index<0>, 0); + + using LValueGetType = decltype(absl::get<0>(v)); + using RValueGetType = decltype(absl::get<0>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, int&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, int&&>::value)); + EXPECT_EQ(absl::get<0>(v), 0); + EXPECT_EQ(absl::get<0>(absl::move(v)), 0); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<0>(const_v)); + using ConstRValueGetType = decltype(absl::get<0>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const int&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const int&&>::value)); + EXPECT_EQ(absl::get<0>(const_v), 0); + EXPECT_EQ(absl::get<0>(absl::move(const_v)), 0); + } + + { + Var v = std::string("Hello"); + + using LValueGetType = decltype(absl::get<1>(v)); + using RValueGetType = decltype(absl::get<1>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, std::string&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, std::string&&>::value)); + EXPECT_EQ(absl::get<1>(v), "Hello"); + EXPECT_EQ(absl::get<1>(absl::move(v)), "Hello"); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<1>(const_v)); + using ConstRValueGetType = decltype(absl::get<1>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const std::string&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const std::string&&>::value)); + EXPECT_EQ(absl::get<1>(const_v), "Hello"); + EXPECT_EQ(absl::get<1>(absl::move(const_v)), "Hello"); + } + + { + Var v = 2.0; + + using LValueGetType = decltype(absl::get<2>(v)); + using RValueGetType = decltype(absl::get<2>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, double&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, double&&>::value)); + EXPECT_EQ(absl::get<2>(v), 2.); + EXPECT_EQ(absl::get<2>(absl::move(v)), 2.); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<2>(const_v)); + using ConstRValueGetType = decltype(absl::get<2>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const double&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const double&&>::value)); + EXPECT_EQ(absl::get<2>(const_v), 2.); + EXPECT_EQ(absl::get<2>(absl::move(const_v)), 2.); + } + + { + Var v(absl::in_place_index<0>, 0); + v.emplace<3>(1); + + using LValueGetType = decltype(absl::get<3>(v)); + using RValueGetType = decltype(absl::get<3>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, int&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, int&&>::value)); + EXPECT_EQ(absl::get<3>(v), 1); + EXPECT_EQ(absl::get<3>(absl::move(v)), 1); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<3>(const_v)); + using ConstRValueGetType = decltype(absl::get<3>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const int&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const int&&>::value)); + EXPECT_EQ(absl::get<3>(const_v), 1); + EXPECT_EQ(absl::get<3>(absl::move(const_v)), 1); // NOLINT + } +} + +TEST(VariantTest, BadGetIndex) { + using Var = variant<int, std::string, double>; + + { + Var v = 1; + + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<1>(v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<1>(std::move(v))); + + const Var& const_v = v; + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<1>(const_v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS( + absl::get<1>(std::move(const_v))); // NOLINT + } + + { + Var v = std::string("Hello"); + + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<0>(v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<0>(std::move(v))); + + const Var& const_v = v; + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<0>(const_v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS( + absl::get<0>(std::move(const_v))); // NOLINT + } +} + +TEST(VariantTest, GetType) { + using Var = variant<int, std::string, double>; + + { + Var v = 1; + + using LValueGetType = decltype(absl::get<int>(v)); + using RValueGetType = decltype(absl::get<int>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, int&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, int&&>::value)); + EXPECT_EQ(absl::get<int>(v), 1); + EXPECT_EQ(absl::get<int>(absl::move(v)), 1); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<int>(const_v)); + using ConstRValueGetType = decltype(absl::get<int>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const int&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const int&&>::value)); + EXPECT_EQ(absl::get<int>(const_v), 1); + EXPECT_EQ(absl::get<int>(absl::move(const_v)), 1); + } + + { + Var v = std::string("Hello"); + + using LValueGetType = decltype(absl::get<1>(v)); + using RValueGetType = decltype(absl::get<1>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, std::string&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, std::string&&>::value)); + EXPECT_EQ(absl::get<std::string>(v), "Hello"); + EXPECT_EQ(absl::get<std::string>(absl::move(v)), "Hello"); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<1>(const_v)); + using ConstRValueGetType = decltype(absl::get<1>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const std::string&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const std::string&&>::value)); + EXPECT_EQ(absl::get<std::string>(const_v), "Hello"); + EXPECT_EQ(absl::get<std::string>(absl::move(const_v)), "Hello"); + } + + { + Var v = 2.0; + + using LValueGetType = decltype(absl::get<2>(v)); + using RValueGetType = decltype(absl::get<2>(absl::move(v))); + + EXPECT_TRUE((std::is_same<LValueGetType, double&>::value)); + EXPECT_TRUE((std::is_same<RValueGetType, double&&>::value)); + EXPECT_EQ(absl::get<double>(v), 2.); + EXPECT_EQ(absl::get<double>(absl::move(v)), 2.); + + const Var& const_v = v; + using ConstLValueGetType = decltype(absl::get<2>(const_v)); + using ConstRValueGetType = decltype(absl::get<2>(absl::move(const_v))); + EXPECT_TRUE((std::is_same<ConstLValueGetType, const double&>::value)); + EXPECT_TRUE((std::is_same<ConstRValueGetType, const double&&>::value)); + EXPECT_EQ(absl::get<double>(const_v), 2.); + EXPECT_EQ(absl::get<double>(absl::move(const_v)), 2.); + } +} + +TEST(VariantTest, BadGetType) { + using Var = variant<int, std::string, double>; + + { + Var v = 1; + + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<std::string>(v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS( + absl::get<std::string>(std::move(v))); + + const Var& const_v = v; + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS( + absl::get<std::string>(const_v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS( + absl::get<std::string>(std::move(const_v))); // NOLINT + } + + { + Var v = std::string("Hello"); + + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<int>(v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<int>(std::move(v))); + + const Var& const_v = v; + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS(absl::get<int>(const_v)); + ABSL_VARIANT_TEST_EXPECT_BAD_VARIANT_ACCESS( + absl::get<int>(std::move(const_v))); // NOLINT + } +} + +TEST(VariantTest, GetIfIndex) { + using Var = variant<int, std::string, double, int>; + + { + Var v(absl::in_place_index<0>, 0); + EXPECT_TRUE(noexcept(absl::get_if<0>(&v))); + + { + auto* elem = absl::get_if<0>(&v); + EXPECT_TRUE((std::is_same<decltype(elem), int*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, 0); + { + auto* bad_elem = absl::get_if<1>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), std::string*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<2>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), double*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<3>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + } + + const Var& const_v = v; + EXPECT_TRUE(noexcept(absl::get_if<0>(&const_v))); + + { + auto* elem = absl::get_if<0>(&const_v); + EXPECT_TRUE((std::is_same<decltype(elem), const int*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, 0); + { + auto* bad_elem = absl::get_if<1>(&const_v); + EXPECT_TRUE( + (std::is_same<decltype(bad_elem), const std::string*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<2>(&const_v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const double*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<3>(&const_v); + EXPECT_EQ(bad_elem, nullptr); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const int*>::value)); + } + } + } + + { + Var v = std::string("Hello"); + EXPECT_TRUE(noexcept(absl::get_if<1>(&v))); + + { + auto* elem = absl::get_if<1>(&v); + EXPECT_TRUE((std::is_same<decltype(elem), std::string*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, "Hello"); + { + auto* bad_elem = absl::get_if<0>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<2>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), double*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<3>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + } + + const Var& const_v = v; + EXPECT_TRUE(noexcept(absl::get_if<1>(&const_v))); + + { + auto* elem = absl::get_if<1>(&const_v); + EXPECT_TRUE((std::is_same<decltype(elem), const std::string*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, "Hello"); + { + auto* bad_elem = absl::get_if<0>(&const_v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<2>(&const_v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const double*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<3>(&const_v); + EXPECT_EQ(bad_elem, nullptr); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const int*>::value)); + } + } + } + + { + Var v = 2.0; + EXPECT_TRUE(noexcept(absl::get_if<2>(&v))); + + { + auto* elem = absl::get_if<2>(&v); + EXPECT_TRUE((std::is_same<decltype(elem), double*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, 2.0); + { + auto* bad_elem = absl::get_if<0>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<1>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), std::string*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<3>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + } + + const Var& const_v = v; + EXPECT_TRUE(noexcept(absl::get_if<2>(&const_v))); + + { + auto* elem = absl::get_if<2>(&const_v); + EXPECT_TRUE((std::is_same<decltype(elem), const double*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, 2.0); + { + auto* bad_elem = absl::get_if<0>(&const_v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<1>(&const_v); + EXPECT_TRUE( + (std::is_same<decltype(bad_elem), const std::string*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<3>(&const_v); + EXPECT_EQ(bad_elem, nullptr); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const int*>::value)); + } + } + } + + { + Var v(absl::in_place_index<0>, 0); + v.emplace<3>(1); + EXPECT_TRUE(noexcept(absl::get_if<3>(&v))); + + { + auto* elem = absl::get_if<3>(&v); + EXPECT_TRUE((std::is_same<decltype(elem), int*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, 1); + { + auto* bad_elem = absl::get_if<0>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<1>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), std::string*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<2>(&v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), double*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + } + + const Var& const_v = v; + EXPECT_TRUE(noexcept(absl::get_if<3>(&const_v))); + + { + auto* elem = absl::get_if<3>(&const_v); + EXPECT_TRUE((std::is_same<decltype(elem), const int*>::value)); + ASSERT_NE(elem, nullptr); + EXPECT_EQ(*elem, 1); + { + auto* bad_elem = absl::get_if<0>(&const_v); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const int*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<1>(&const_v); + EXPECT_TRUE( + (std::is_same<decltype(bad_elem), const std::string*>::value)); + EXPECT_EQ(bad_elem, nullptr); + } + { + auto* bad_elem = absl::get_if<2>(&const_v); + EXPECT_EQ(bad_elem, nullptr); + EXPECT_TRUE((std::is_same<decltype(bad_elem), const double*>::value)); + } + } + } +} + +////////////////////// +// [variant.relops] // +////////////////////// + +TEST(VariantTest, OperatorEquals) { + variant<int, std::string> a(1), b(1); + EXPECT_TRUE(a == b); + EXPECT_TRUE(b == a); + EXPECT_FALSE(a != b); + EXPECT_FALSE(b != a); + + b = "str"; + EXPECT_FALSE(a == b); + EXPECT_FALSE(b == a); + EXPECT_TRUE(a != b); + EXPECT_TRUE(b != a); + + b = 0; + EXPECT_FALSE(a == b); + EXPECT_FALSE(b == a); + EXPECT_TRUE(a != b); + EXPECT_TRUE(b != a); + + a = b = "foo"; + EXPECT_TRUE(a == b); + EXPECT_TRUE(b == a); + EXPECT_FALSE(a != b); + EXPECT_FALSE(b != a); + + a = "bar"; + EXPECT_FALSE(a == b); + EXPECT_FALSE(b == a); + EXPECT_TRUE(a != b); + EXPECT_TRUE(b != a); +} + +TEST(VariantTest, OperatorRelational) { + variant<int, std::string> a(1), b(1); + EXPECT_FALSE(a < b); + EXPECT_FALSE(b < a); + EXPECT_FALSE(a > b); + EXPECT_FALSE(b > a); + EXPECT_TRUE(a <= b); + EXPECT_TRUE(b <= a); + EXPECT_TRUE(a >= b); + EXPECT_TRUE(b >= a); + + b = "str"; + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); + EXPECT_FALSE(a > b); + EXPECT_TRUE(b > a); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(b <= a); + EXPECT_FALSE(a >= b); + EXPECT_TRUE(b >= a); + + b = 0; + EXPECT_FALSE(a < b); + EXPECT_TRUE(b < a); + EXPECT_TRUE(a > b); + EXPECT_FALSE(b > a); + EXPECT_FALSE(a <= b); + EXPECT_TRUE(b <= a); + EXPECT_TRUE(a >= b); + EXPECT_FALSE(b >= a); + + a = b = "foo"; + EXPECT_FALSE(a < b); + EXPECT_FALSE(b < a); + EXPECT_FALSE(a > b); + EXPECT_FALSE(b > a); + EXPECT_TRUE(a <= b); + EXPECT_TRUE(b <= a); + EXPECT_TRUE(a >= b); + EXPECT_TRUE(b >= a); + + a = "bar"; + EXPECT_TRUE(a < b); + EXPECT_FALSE(b < a); + EXPECT_FALSE(a > b); + EXPECT_TRUE(b > a); + EXPECT_TRUE(a <= b); + EXPECT_FALSE(b <= a); + EXPECT_FALSE(a >= b); + EXPECT_TRUE(b >= a); +} + +#ifdef ABSL_HAVE_EXCEPTIONS + +TEST(VariantTest, ValuelessOperatorEquals) { + variant<MoveCanThrow, std::string> int_v(1), string_v("Hello"), + valueless(absl::in_place_index<0>), + other_valueless(absl::in_place_index<0>); + ToValuelessByException(valueless); + ToValuelessByException(other_valueless); + + EXPECT_TRUE(valueless == other_valueless); + EXPECT_TRUE(other_valueless == valueless); + EXPECT_FALSE(valueless == int_v); + EXPECT_FALSE(valueless == string_v); + EXPECT_FALSE(int_v == valueless); + EXPECT_FALSE(string_v == valueless); + + EXPECT_FALSE(valueless != other_valueless); + EXPECT_FALSE(other_valueless != valueless); + EXPECT_TRUE(valueless != int_v); + EXPECT_TRUE(valueless != string_v); + EXPECT_TRUE(int_v != valueless); + EXPECT_TRUE(string_v != valueless); +} + +TEST(VariantTest, ValuelessOperatorRelational) { + variant<MoveCanThrow, std::string> int_v(1), string_v("Hello"), + valueless(absl::in_place_index<0>), + other_valueless(absl::in_place_index<0>); + ToValuelessByException(valueless); + ToValuelessByException(other_valueless); + + EXPECT_FALSE(valueless < other_valueless); + EXPECT_FALSE(other_valueless < valueless); + EXPECT_TRUE(valueless < int_v); + EXPECT_TRUE(valueless < string_v); + EXPECT_FALSE(int_v < valueless); + EXPECT_FALSE(string_v < valueless); + + EXPECT_TRUE(valueless <= other_valueless); + EXPECT_TRUE(other_valueless <= valueless); + EXPECT_TRUE(valueless <= int_v); + EXPECT_TRUE(valueless <= string_v); + EXPECT_FALSE(int_v <= valueless); + EXPECT_FALSE(string_v <= valueless); + + EXPECT_TRUE(valueless >= other_valueless); + EXPECT_TRUE(other_valueless >= valueless); + EXPECT_FALSE(valueless >= int_v); + EXPECT_FALSE(valueless >= string_v); + EXPECT_TRUE(int_v >= valueless); + EXPECT_TRUE(string_v >= valueless); + + EXPECT_FALSE(valueless > other_valueless); + EXPECT_FALSE(other_valueless > valueless); + EXPECT_FALSE(valueless > int_v); + EXPECT_FALSE(valueless > string_v); + EXPECT_TRUE(int_v > valueless); + EXPECT_TRUE(string_v > valueless); +} + +#endif + +///////////////////// +// [variant.visit] // +///////////////////// + +template <typename T> +struct ConvertTo { + template <typename U> + T operator()(const U& u) const { + return u; + } +}; + +TEST(VariantTest, VisitSimple) { + variant<std::string, const char*> v = "A"; + + std::string str = absl::visit(ConvertTo<std::string>{}, v); + EXPECT_EQ("A", str); + + v = std::string("B"); + + absl::string_view piece = absl::visit(ConvertTo<absl::string_view>{}, v); + EXPECT_EQ("B", piece); + + struct StrLen { + int operator()(const char* s) const { return strlen(s); } + int operator()(const std::string& s) const { return s.size(); } + }; + + v = "SomeStr"; + EXPECT_EQ(7, absl::visit(StrLen{}, v)); + v = std::string("VeryLargeThisTime"); + EXPECT_EQ(17, absl::visit(StrLen{}, v)); +} + +TEST(VariantTest, VisitRValue) { + variant<std::string> v = std::string("X"); + struct Visitor { + bool operator()(const std::string&) const { return false; } + bool operator()(std::string&&) const { return true; } // NOLINT + + int operator()(const std::string&, const std::string&) const { return 0; } + int operator()(const std::string&, std::string&&) const { + return 1; + } // NOLINT + int operator()(std::string&&, const std::string&) const { + return 2; + } // NOLINT + int operator()(std::string&&, std::string&&) const { return 3; } // NOLINT + }; + EXPECT_FALSE(absl::visit(Visitor{}, v)); + EXPECT_TRUE(absl::visit(Visitor{}, absl::move(v))); + + // Also test the variadic overload. + EXPECT_EQ(0, absl::visit(Visitor{}, v, v)); + EXPECT_EQ(1, absl::visit(Visitor{}, v, absl::move(v))); + EXPECT_EQ(2, absl::visit(Visitor{}, absl::move(v), v)); + EXPECT_EQ(3, absl::visit(Visitor{}, absl::move(v), absl::move(v))); +} + +TEST(VariantTest, VisitRValueVisitor) { + variant<std::string> v = std::string("X"); + struct Visitor { + bool operator()(const std::string&) const& { return false; } + bool operator()(const std::string&) && { return true; } + }; + Visitor visitor; + EXPECT_FALSE(absl::visit(visitor, v)); + EXPECT_TRUE(absl::visit(Visitor{}, v)); +} + +TEST(VariantTest, VisitResultTypeDifferent) { + variant<std::string> v = std::string("X"); + struct LValue_LValue {}; + struct RValue_LValue {}; + struct LValue_RValue {}; + struct RValue_RValue {}; + struct Visitor { + LValue_LValue operator()(const std::string&) const& { return {}; } + RValue_LValue operator()(std::string&&) const& { return {}; } // NOLINT + LValue_RValue operator()(const std::string&) && { return {}; } + RValue_RValue operator()(std::string&&) && { return {}; } // NOLINT + } visitor; + + EXPECT_TRUE( + (std::is_same<LValue_LValue, decltype(absl::visit(visitor, v))>::value)); + EXPECT_TRUE( + (std::is_same<RValue_LValue, + decltype(absl::visit(visitor, absl::move(v)))>::value)); + EXPECT_TRUE(( + std::is_same<LValue_RValue, decltype(absl::visit(Visitor{}, v))>::value)); + EXPECT_TRUE( + (std::is_same<RValue_RValue, + decltype(absl::visit(Visitor{}, absl::move(v)))>::value)); +} + +TEST(VariantTest, VisitVariadic) { + using A = variant<int, std::string>; + using B = variant<std::unique_ptr<int>, absl::string_view>; + + struct Visitor { + std::pair<int, int> operator()(int a, std::unique_ptr<int> b) const { + return {a, *b}; + } + std::pair<int, int> operator()(absl::string_view a, + std::unique_ptr<int> b) const { + return {static_cast<int>(a.size()), static_cast<int>(*b)}; + } + std::pair<int, int> operator()(int a, absl::string_view b) const { + return {a, static_cast<int>(b.size())}; + } + std::pair<int, int> operator()(absl::string_view a, + absl::string_view b) const { + return {static_cast<int>(a.size()), static_cast<int>(b.size())}; + } + }; + + EXPECT_THAT(absl::visit(Visitor(), A(1), B(std::unique_ptr<int>(new int(7)))), + ::testing::Pair(1, 7)); + EXPECT_THAT(absl::visit(Visitor(), A(1), B(absl::string_view("ABC"))), + ::testing::Pair(1, 3)); + EXPECT_THAT(absl::visit(Visitor(), A(std::string("BBBBB")), + B(std::unique_ptr<int>(new int(7)))), + ::testing::Pair(5, 7)); + EXPECT_THAT(absl::visit(Visitor(), A(std::string("BBBBB")), + B(absl::string_view("ABC"))), + ::testing::Pair(5, 3)); +} + +TEST(VariantTest, VisitNoArgs) { + EXPECT_EQ(5, absl::visit([] { return 5; })); +} + +struct ConstFunctor { + int operator()(int a, int b) const { return a - b; } +}; + +struct MutableFunctor { + int operator()(int a, int b) { return a - b; } +}; + +struct Class { + int Method(int a, int b) { return a - b; } + int ConstMethod(int a, int b) const { return a - b; } + + int member; +}; + +TEST(VariantTest, VisitReferenceWrapper) { + ConstFunctor cf; + MutableFunctor mf; + absl::variant<int> three = 3; + absl::variant<int> two = 2; + + EXPECT_EQ(1, absl::visit(std::cref(cf), three, two)); + EXPECT_EQ(1, absl::visit(std::ref(cf), three, two)); + EXPECT_EQ(1, absl::visit(std::ref(mf), three, two)); +} + +// libstdc++ std::variant doesn't support the INVOKE semantics. +#if !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) +TEST(VariantTest, VisitMemberFunction) { + absl::variant<std::unique_ptr<Class>> p(absl::make_unique<Class>()); + absl::variant<std::unique_ptr<const Class>> cp( + absl::make_unique<const Class>()); + absl::variant<int> three = 3; + absl::variant<int> two = 2; + + EXPECT_EQ(1, absl::visit(&Class::Method, p, three, two)); + EXPECT_EQ(1, absl::visit(&Class::ConstMethod, p, three, two)); + EXPECT_EQ(1, absl::visit(&Class::ConstMethod, cp, three, two)); +} + +TEST(VariantTest, VisitDataMember) { + absl::variant<std::unique_ptr<Class>> p(absl::make_unique<Class>(Class{42})); + absl::variant<std::unique_ptr<const Class>> cp( + absl::make_unique<const Class>(Class{42})); + EXPECT_EQ(42, absl::visit(&Class::member, p)); + + absl::visit(&Class::member, p) = 5; + EXPECT_EQ(5, absl::visit(&Class::member, p)); + + EXPECT_EQ(42, absl::visit(&Class::member, cp)); +} +#endif // !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) + +///////////////////////// +// [variant.monostate] // +///////////////////////// + +TEST(VariantTest, MonostateBasic) { + absl::monostate mono; + (void)mono; + + // TODO(mattcalabrese) Expose move triviality metafunctions in absl. + EXPECT_TRUE(absl::is_trivially_default_constructible<absl::monostate>::value); + EXPECT_TRUE(is_trivially_move_constructible<absl::monostate>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<absl::monostate>::value); + EXPECT_TRUE(is_trivially_move_assignable<absl::monostate>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<absl::monostate>::value); + EXPECT_TRUE(absl::is_trivially_destructible<absl::monostate>::value); +} + +TEST(VariantTest, VariantMonostateDefaultConstruction) { + absl::variant<absl::monostate, NonDefaultConstructible> var; + EXPECT_EQ(var.index(), 0); +} + +//////////////////////////////// +// [variant.monostate.relops] // +//////////////////////////////// + +TEST(VariantTest, MonostateComparisons) { + absl::monostate lhs, rhs; + + EXPECT_EQ(lhs, lhs); + EXPECT_EQ(lhs, rhs); + + EXPECT_FALSE(lhs != lhs); + EXPECT_FALSE(lhs != rhs); + EXPECT_FALSE(lhs < lhs); + EXPECT_FALSE(lhs < rhs); + EXPECT_FALSE(lhs > lhs); + EXPECT_FALSE(lhs > rhs); + + EXPECT_LE(lhs, lhs); + EXPECT_LE(lhs, rhs); + EXPECT_GE(lhs, lhs); + EXPECT_GE(lhs, rhs); + + EXPECT_TRUE(noexcept(std::declval<absl::monostate>() == + std::declval<absl::monostate>())); + EXPECT_TRUE(noexcept(std::declval<absl::monostate>() != + std::declval<absl::monostate>())); + EXPECT_TRUE(noexcept(std::declval<absl::monostate>() < + std::declval<absl::monostate>())); + EXPECT_TRUE(noexcept(std::declval<absl::monostate>() > + std::declval<absl::monostate>())); + EXPECT_TRUE(noexcept(std::declval<absl::monostate>() <= + std::declval<absl::monostate>())); + EXPECT_TRUE(noexcept(std::declval<absl::monostate>() >= + std::declval<absl::monostate>())); +} + +/////////////////////// +// [variant.specalg] // +/////////////////////// + +TEST(VariantTest, NonmemberSwap) { + using std::swap; + + SpecialSwap v1(3); + SpecialSwap v2(7); + + variant<SpecialSwap> a = v1, b = v2; + + EXPECT_THAT(a, VariantWith<SpecialSwap>(v1)); + EXPECT_THAT(b, VariantWith<SpecialSwap>(v2)); + + std::swap(a, b); + EXPECT_THAT(a, VariantWith<SpecialSwap>(v2)); + EXPECT_THAT(b, VariantWith<SpecialSwap>(v1)); +#ifndef ABSL_USES_STD_VARIANT + EXPECT_FALSE(absl::get<SpecialSwap>(a).special_swap); +#endif + + swap(a, b); + EXPECT_THAT(a, VariantWith<SpecialSwap>(v1)); + EXPECT_THAT(b, VariantWith<SpecialSwap>(v2)); + EXPECT_TRUE(absl::get<SpecialSwap>(b).special_swap); +} + +////////////////////////// +// [variant.bad.access] // +////////////////////////// + +TEST(VariantTest, BadAccess) { + EXPECT_TRUE(noexcept(absl::bad_variant_access())); + absl::bad_variant_access exception_obj; + std::exception* base = &exception_obj; + (void)base; +} + +//////////////////// +// [variant.hash] // +//////////////////// + +TEST(VariantTest, MonostateHash) { + absl::monostate mono, other_mono; + std::hash<absl::monostate> const hasher{}; + static_assert(std::is_same<decltype(hasher(mono)), std::size_t>::value, ""); + EXPECT_EQ(hasher(mono), hasher(other_mono)); +} + +TEST(VariantTest, Hash) { + static_assert(type_traits_internal::IsHashable<variant<int>>::value, ""); + static_assert(type_traits_internal::IsHashable<variant<Hashable>>::value, ""); + static_assert(type_traits_internal::IsHashable<variant<int, Hashable>>::value, + ""); + +#if ABSL_META_INTERNAL_STD_HASH_SFINAE_FRIENDLY_ + static_assert(!type_traits_internal::IsHashable<variant<NonHashable>>::value, + ""); + static_assert( + !type_traits_internal::IsHashable<variant<Hashable, NonHashable>>::value, + ""); +#endif + +// MSVC std::hash<std::variant> does not use the index, thus produce the same +// result on the same value as different alternative. +#if !(defined(_MSC_VER) && defined(ABSL_USES_STD_VARIANT)) + { + // same value as different alternative + variant<int, int> v0(in_place_index<0>, 42); + variant<int, int> v1(in_place_index<1>, 42); + std::hash<variant<int, int>> hash; + EXPECT_NE(hash(v0), hash(v1)); + } +#endif // !(defined(_MSC_VER) && defined(ABSL_USES_STD_VARIANT)) + + { + std::hash<variant<int>> hash; + std::set<size_t> hashcodes; + for (int i = 0; i < 100; ++i) { + hashcodes.insert(hash(i)); + } + EXPECT_GT(hashcodes.size(), 90); + + // test const-qualified + static_assert(type_traits_internal::IsHashable<variant<const int>>::value, + ""); + static_assert( + type_traits_internal::IsHashable<variant<const Hashable>>::value, ""); + std::hash<absl::variant<const int>> c_hash; + for (int i = 0; i < 100; ++i) { + EXPECT_EQ(hash(i), c_hash(i)); + } + } +} + +//////////////////////////////////////// +// Miscellaneous and deprecated tests // +//////////////////////////////////////// + +// Test that a set requiring a basic type conversion works correctly +#if !defined(ABSL_USES_STD_VARIANT) +TEST(VariantTest, TestConvertingSet) { + typedef variant<double> Variant; + Variant v(1.0); + const int two = 2; + v = two; + EXPECT_TRUE(absl::holds_alternative<double>(v)); + ASSERT_TRUE(nullptr != absl::get_if<double>(&v)); + EXPECT_DOUBLE_EQ(2, absl::get<double>(v)); +} +#endif // ABSL_USES_STD_VARIANT + +// Test that a vector of variants behaves reasonably. +TEST(VariantTest, Container) { + typedef variant<int, float> Variant; + + // Creation of vector should work + std::vector<Variant> vec; + vec.push_back(Variant(10)); + vec.push_back(Variant(20.0f)); + + // Vector resizing should work if we supply a value for new slots + vec.resize(10, Variant(0)); +} + +// Test that a variant with a non-copyable type can be constructed and +// manipulated to some degree. +TEST(VariantTest, TestVariantWithNonCopyableType) { + typedef variant<int, NonCopyable> Variant; + const int kValue = 1; + Variant v(kValue); + ASSERT_TRUE(absl::holds_alternative<int>(v)); + EXPECT_EQ(kValue, absl::get<int>(v)); +} + +// Test that a variant with a non-copyable type can be transformed to +// the non-copyable type with a call to `emplace` for different numbers +// of arguments. We do not need to test this for each of T1 ... T8 +// because `emplace` does not overload on T1 ... to T8, so if this +// works for any one of T1 ... T8, then it works for all of them. We +// do need to test that it works with varying numbers of parameters +// though. +TEST(VariantTest, TestEmplace) { + typedef variant<int, NonCopyable> Variant; + const int kValue = 1; + Variant v(kValue); + ASSERT_TRUE(absl::holds_alternative<int>(v)); + EXPECT_EQ(kValue, absl::get<int>(v)); + + // emplace with zero arguments, then back to 'int' + v.emplace<NonCopyable>(); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v)); + EXPECT_EQ(0, absl::get<NonCopyable>(v).value); + v = kValue; + ASSERT_TRUE(absl::holds_alternative<int>(v)); + + // emplace with one argument: + v.emplace<NonCopyable>(1); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v)); + EXPECT_EQ(1, absl::get<NonCopyable>(v).value); + v = kValue; + ASSERT_TRUE(absl::holds_alternative<int>(v)); + + // emplace with two arguments: + v.emplace<NonCopyable>(1, 2); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v)); + EXPECT_EQ(3, absl::get<NonCopyable>(v).value); + v = kValue; + ASSERT_TRUE(absl::holds_alternative<int>(v)); + + // emplace with three arguments + v.emplace<NonCopyable>(1, 2, 3); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v)); + EXPECT_EQ(6, absl::get<NonCopyable>(v).value); + v = kValue; + ASSERT_TRUE(absl::holds_alternative<int>(v)); + + // emplace with four arguments + v.emplace<NonCopyable>(1, 2, 3, 4); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v)); + EXPECT_EQ(10, absl::get<NonCopyable>(v).value); + v = kValue; + ASSERT_TRUE(absl::holds_alternative<int>(v)); +} + +TEST(VariantTest, TestEmplaceDestroysCurrentValue) { + typedef variant<int, IncrementInDtor, NonCopyable> Variant; + int counter = 0; + Variant v(0); + ASSERT_TRUE(absl::holds_alternative<int>(v)); + v.emplace<IncrementInDtor>(&counter); + ASSERT_TRUE(absl::holds_alternative<IncrementInDtor>(v)); + ASSERT_EQ(0, counter); + v.emplace<NonCopyable>(); + ASSERT_TRUE(absl::holds_alternative<NonCopyable>(v)); + EXPECT_EQ(1, counter); +} + +TEST(VariantTest, TestMoveSemantics) { + typedef variant<std::unique_ptr<int>, std::unique_ptr<std::string>> Variant; + + // Construct a variant by moving from an element value. + Variant v(absl::WrapUnique(new int(10))); + EXPECT_TRUE(absl::holds_alternative<std::unique_ptr<int>>(v)); + + // Construct a variant by moving from another variant. + Variant v2(absl::move(v)); + ASSERT_TRUE(absl::holds_alternative<std::unique_ptr<int>>(v2)); + ASSERT_NE(nullptr, absl::get<std::unique_ptr<int>>(v2)); + EXPECT_EQ(10, *absl::get<std::unique_ptr<int>>(v2)); + + // Moving from a variant object leaves it holding moved-from value of the + // same element type. + EXPECT_TRUE(absl::holds_alternative<std::unique_ptr<int>>(v)); + ASSERT_NE(nullptr, absl::get_if<std::unique_ptr<int>>(&v)); + EXPECT_EQ(nullptr, absl::get<std::unique_ptr<int>>(v)); + + // Assign a variant from an element value by move. + v = absl::make_unique<std::string>("foo"); + ASSERT_TRUE(absl::holds_alternative<std::unique_ptr<std::string>>(v)); + EXPECT_EQ("foo", *absl::get<std::unique_ptr<std::string>>(v)); + + // Move-assign a variant. + v2 = absl::move(v); + ASSERT_TRUE(absl::holds_alternative<std::unique_ptr<std::string>>(v2)); + EXPECT_EQ("foo", *absl::get<std::unique_ptr<std::string>>(v2)); + EXPECT_TRUE(absl::holds_alternative<std::unique_ptr<std::string>>(v)); +} + +variant<int, std::string> PassThrough(const variant<int, std::string>& arg) { + return arg; +} + +TEST(VariantTest, TestImplicitConversion) { + EXPECT_TRUE(absl::holds_alternative<int>(PassThrough(0))); + + // We still need the explicit cast for std::string, because C++ won't apply + // two user-defined implicit conversions in a row. + EXPECT_TRUE( + absl::holds_alternative<std::string>(PassThrough(std::string("foo")))); +} + +struct Convertible2; +struct Convertible1 { + Convertible1() {} + Convertible1(const Convertible1&) {} + Convertible1& operator=(const Convertible1&) { return *this; } + + // implicit conversion from Convertible2 + Convertible1(const Convertible2&) {} // NOLINT(runtime/explicit) +}; + +struct Convertible2 { + Convertible2() {} + Convertible2(const Convertible2&) {} + Convertible2& operator=(const Convertible2&) { return *this; } + + // implicit conversion from Convertible1 + Convertible2(const Convertible1&) {} // NOLINT(runtime/explicit) +}; + +TEST(VariantTest, TestRvalueConversion) { +#if !defined(ABSL_USES_STD_VARIANT) + variant<double, std::string> var( + ConvertVariantTo<variant<double, std::string>>( + variant<std::string, int>(0))); + ASSERT_TRUE(absl::holds_alternative<double>(var)); + EXPECT_EQ(0.0, absl::get<double>(var)); + + var = ConvertVariantTo<variant<double, std::string>>( + variant<const char*, float>("foo")); + ASSERT_TRUE(absl::holds_alternative<std::string>(var)); + EXPECT_EQ("foo", absl::get<std::string>(var)); + + variant<double> singleton( + ConvertVariantTo<variant<double>>(variant<int, float>(42))); + ASSERT_TRUE(absl::holds_alternative<double>(singleton)); + EXPECT_EQ(42.0, absl::get<double>(singleton)); + + singleton = ConvertVariantTo<variant<double>>(variant<int, float>(3.14f)); + ASSERT_TRUE(absl::holds_alternative<double>(singleton)); + EXPECT_FLOAT_EQ(3.14f, static_cast<float>(absl::get<double>(singleton))); + + singleton = ConvertVariantTo<variant<double>>(variant<int>(0)); + ASSERT_TRUE(absl::holds_alternative<double>(singleton)); + EXPECT_EQ(0.0, absl::get<double>(singleton)); + + variant<int32_t, uint32_t> variant2( + ConvertVariantTo<variant<int32_t, uint32_t>>(variant<int32_t>(42))); + ASSERT_TRUE(absl::holds_alternative<int32_t>(variant2)); + EXPECT_EQ(42, absl::get<int32_t>(variant2)); + + variant2 = + ConvertVariantTo<variant<int32_t, uint32_t>>(variant<uint32_t>(42)); + ASSERT_TRUE(absl::holds_alternative<uint32_t>(variant2)); + EXPECT_EQ(42, absl::get<uint32_t>(variant2)); +#endif // !ABSL_USES_STD_VARIANT + + variant<Convertible1, Convertible2> variant3( + ConvertVariantTo<variant<Convertible1, Convertible2>>( + (variant<Convertible2, Convertible1>(Convertible1())))); + ASSERT_TRUE(absl::holds_alternative<Convertible1>(variant3)); + + variant3 = ConvertVariantTo<variant<Convertible1, Convertible2>>( + variant<Convertible2, Convertible1>(Convertible2())); + ASSERT_TRUE(absl::holds_alternative<Convertible2>(variant3)); +} + +TEST(VariantTest, TestLvalueConversion) { +#if !defined(ABSL_USES_STD_VARIANT) + variant<std::string, int> source1 = 0; + variant<double, std::string> destination( + ConvertVariantTo<variant<double, std::string>>(source1)); + ASSERT_TRUE(absl::holds_alternative<double>(destination)); + EXPECT_EQ(0.0, absl::get<double>(destination)); + + variant<const char*, float> source2 = "foo"; + destination = ConvertVariantTo<variant<double, std::string>>(source2); + ASSERT_TRUE(absl::holds_alternative<std::string>(destination)); + EXPECT_EQ("foo", absl::get<std::string>(destination)); + + variant<int, float> source3(42); + variant<double> singleton(ConvertVariantTo<variant<double>>(source3)); + ASSERT_TRUE(absl::holds_alternative<double>(singleton)); + EXPECT_EQ(42.0, absl::get<double>(singleton)); + + source3 = 3.14f; + singleton = ConvertVariantTo<variant<double>>(source3); + ASSERT_TRUE(absl::holds_alternative<double>(singleton)); + EXPECT_FLOAT_EQ(3.14f, static_cast<float>(absl::get<double>(singleton))); + + variant<int> source4(0); + singleton = ConvertVariantTo<variant<double>>(source4); + ASSERT_TRUE(absl::holds_alternative<double>(singleton)); + EXPECT_EQ(0.0, absl::get<double>(singleton)); + + variant<int32_t> source5(42); + variant<int32_t, uint32_t> variant2( + ConvertVariantTo<variant<int32_t, uint32_t>>(source5)); + ASSERT_TRUE(absl::holds_alternative<int32_t>(variant2)); + EXPECT_EQ(42, absl::get<int32_t>(variant2)); + + variant<uint32_t> source6(42); + variant2 = ConvertVariantTo<variant<int32_t, uint32_t>>(source6); + ASSERT_TRUE(absl::holds_alternative<uint32_t>(variant2)); + EXPECT_EQ(42, absl::get<uint32_t>(variant2)); +#endif + + variant<Convertible2, Convertible1> source7((Convertible1())); + variant<Convertible1, Convertible2> variant3( + ConvertVariantTo<variant<Convertible1, Convertible2>>(source7)); + ASSERT_TRUE(absl::holds_alternative<Convertible1>(variant3)); + + source7 = Convertible2(); + variant3 = ConvertVariantTo<variant<Convertible1, Convertible2>>(source7); + ASSERT_TRUE(absl::holds_alternative<Convertible2>(variant3)); +} + +TEST(VariantTest, TestMoveConversion) { + using Variant = + variant<std::unique_ptr<const int>, std::unique_ptr<const std::string>>; + using OtherVariant = + variant<std::unique_ptr<int>, std::unique_ptr<std::string>>; + + Variant var( + ConvertVariantTo<Variant>(OtherVariant{absl::make_unique<int>(0)})); + ASSERT_TRUE(absl::holds_alternative<std::unique_ptr<const int>>(var)); + ASSERT_NE(absl::get<std::unique_ptr<const int>>(var), nullptr); + EXPECT_EQ(0, *absl::get<std::unique_ptr<const int>>(var)); + + var = ConvertVariantTo<Variant>( + OtherVariant(absl::make_unique<std::string>("foo"))); + ASSERT_TRUE(absl::holds_alternative<std::unique_ptr<const std::string>>(var)); + EXPECT_EQ("foo", *absl::get<std::unique_ptr<const std::string>>(var)); +} + +TEST(VariantTest, DoesNotMoveFromLvalues) { + // We use shared_ptr here because it's both copyable and movable, and + // a moved-from shared_ptr is guaranteed to be null, so we can detect + // whether moving or copying has occurred. + using Variant = + variant<std::shared_ptr<const int>, std::shared_ptr<const std::string>>; + using OtherVariant = + variant<std::shared_ptr<int>, std::shared_ptr<std::string>>; + + Variant v1(std::make_shared<const int>(0)); + + // Test copy constructor + Variant v2(v1); + EXPECT_EQ(absl::get<std::shared_ptr<const int>>(v1), + absl::get<std::shared_ptr<const int>>(v2)); + + // Test copy-assignment operator + v1 = std::make_shared<const std::string>("foo"); + v2 = v1; + EXPECT_EQ(absl::get<std::shared_ptr<const std::string>>(v1), + absl::get<std::shared_ptr<const std::string>>(v2)); + + // Test converting copy constructor + OtherVariant other(std::make_shared<int>(0)); + Variant v3(ConvertVariantTo<Variant>(other)); + EXPECT_EQ(absl::get<std::shared_ptr<int>>(other), + absl::get<std::shared_ptr<const int>>(v3)); + + other = std::make_shared<std::string>("foo"); + v3 = ConvertVariantTo<Variant>(other); + EXPECT_EQ(absl::get<std::shared_ptr<std::string>>(other), + absl::get<std::shared_ptr<const std::string>>(v3)); +} + +TEST(VariantTest, TestRvalueConversionViaConvertVariantTo) { +#if !defined(ABSL_USES_STD_VARIANT) + variant<double, std::string> var( + ConvertVariantTo<variant<double, std::string>>( + variant<std::string, int>(3))); + EXPECT_THAT(absl::get_if<double>(&var), Pointee(3.0)); + + var = ConvertVariantTo<variant<double, std::string>>( + variant<const char*, float>("foo")); + EXPECT_THAT(absl::get_if<std::string>(&var), Pointee(std::string("foo"))); + + variant<double> singleton( + ConvertVariantTo<variant<double>>(variant<int, float>(42))); + EXPECT_THAT(absl::get_if<double>(&singleton), Pointee(42.0)); + + singleton = ConvertVariantTo<variant<double>>(variant<int, float>(3.14f)); + EXPECT_THAT(absl::get_if<double>(&singleton), Pointee(DoubleEq(3.14f))); + + singleton = ConvertVariantTo<variant<double>>(variant<int>(3)); + EXPECT_THAT(absl::get_if<double>(&singleton), Pointee(3.0)); + + variant<int32_t, uint32_t> variant2( + ConvertVariantTo<variant<int32_t, uint32_t>>(variant<int32_t>(42))); + EXPECT_THAT(absl::get_if<int32_t>(&variant2), Pointee(42)); + + variant2 = + ConvertVariantTo<variant<int32_t, uint32_t>>(variant<uint32_t>(42)); + EXPECT_THAT(absl::get_if<uint32_t>(&variant2), Pointee(42)); +#endif + + variant<Convertible1, Convertible2> variant3( + ConvertVariantTo<variant<Convertible1, Convertible2>>( + (variant<Convertible2, Convertible1>(Convertible1())))); + ASSERT_TRUE(absl::holds_alternative<Convertible1>(variant3)); + + variant3 = ConvertVariantTo<variant<Convertible1, Convertible2>>( + variant<Convertible2, Convertible1>(Convertible2())); + ASSERT_TRUE(absl::holds_alternative<Convertible2>(variant3)); +} + +TEST(VariantTest, TestLvalueConversionViaConvertVariantTo) { +#if !defined(ABSL_USES_STD_VARIANT) + variant<std::string, int> source1 = 3; + variant<double, std::string> destination( + ConvertVariantTo<variant<double, std::string>>(source1)); + EXPECT_THAT(absl::get_if<double>(&destination), Pointee(3.0)); + + variant<const char*, float> source2 = "foo"; + destination = ConvertVariantTo<variant<double, std::string>>(source2); + EXPECT_THAT(absl::get_if<std::string>(&destination), + Pointee(std::string("foo"))); + + variant<int, float> source3(42); + variant<double> singleton(ConvertVariantTo<variant<double>>(source3)); + EXPECT_THAT(absl::get_if<double>(&singleton), Pointee(42.0)); + + source3 = 3.14f; + singleton = ConvertVariantTo<variant<double>>(source3); + EXPECT_FLOAT_EQ(3.14f, static_cast<float>(absl::get<double>(singleton))); + EXPECT_THAT(absl::get_if<double>(&singleton), Pointee(DoubleEq(3.14f))); + + variant<int> source4(3); + singleton = ConvertVariantTo<variant<double>>(source4); + EXPECT_THAT(absl::get_if<double>(&singleton), Pointee(3.0)); + + variant<int32_t> source5(42); + variant<int32_t, uint32_t> variant2( + ConvertVariantTo<variant<int32_t, uint32_t>>(source5)); + EXPECT_THAT(absl::get_if<int32_t>(&variant2), Pointee(42)); + + variant<uint32_t> source6(42); + variant2 = ConvertVariantTo<variant<int32_t, uint32_t>>(source6); + EXPECT_THAT(absl::get_if<uint32_t>(&variant2), Pointee(42)); +#endif // !ABSL_USES_STD_VARIANT + + variant<Convertible2, Convertible1> source7((Convertible1())); + variant<Convertible1, Convertible2> variant3( + ConvertVariantTo<variant<Convertible1, Convertible2>>(source7)); + ASSERT_TRUE(absl::holds_alternative<Convertible1>(variant3)); + + source7 = Convertible2(); + variant3 = ConvertVariantTo<variant<Convertible1, Convertible2>>(source7); + ASSERT_TRUE(absl::holds_alternative<Convertible2>(variant3)); +} + +TEST(VariantTest, TestMoveConversionViaConvertVariantTo) { + using Variant = + variant<std::unique_ptr<const int>, std::unique_ptr<const std::string>>; + using OtherVariant = + variant<std::unique_ptr<int>, std::unique_ptr<std::string>>; + + Variant var( + ConvertVariantTo<Variant>(OtherVariant{absl::make_unique<int>(3)})); + EXPECT_THAT(absl::get_if<std::unique_ptr<const int>>(&var), + Pointee(Pointee(3))); + + var = ConvertVariantTo<Variant>( + OtherVariant(absl::make_unique<std::string>("foo"))); + EXPECT_THAT(absl::get_if<std::unique_ptr<const std::string>>(&var), + Pointee(Pointee(std::string("foo")))); +} + +// If all alternatives are trivially copy/move constructible, variant should +// also be trivially copy/move constructible. This is not required by the +// standard and we know that libstdc++ variant doesn't have this feature. +// For more details see the paper: +// http://open-std.org/JTC1/SC22/WG21/docs/papers/2017/p0602r0.html +#if !(defined(ABSL_USES_STD_VARIANT) && defined(__GLIBCXX__)) +#define ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY 1 +#endif + +TEST(VariantTest, TestCopyAndMoveTypeTraits) { + EXPECT_TRUE(std::is_copy_constructible<variant<std::string>>::value); + EXPECT_TRUE(absl::is_copy_assignable<variant<std::string>>::value); + EXPECT_TRUE(std::is_move_constructible<variant<std::string>>::value); + EXPECT_TRUE(absl::is_move_assignable<variant<std::string>>::value); + EXPECT_TRUE(std::is_move_constructible<variant<std::unique_ptr<int>>>::value); + EXPECT_TRUE(absl::is_move_assignable<variant<std::unique_ptr<int>>>::value); + EXPECT_FALSE( + std::is_copy_constructible<variant<std::unique_ptr<int>>>::value); + EXPECT_FALSE(absl::is_copy_assignable<variant<std::unique_ptr<int>>>::value); + + EXPECT_FALSE( + absl::is_trivially_copy_constructible<variant<std::string>>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<variant<std::string>>::value); +#if ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY + EXPECT_TRUE(absl::is_trivially_copy_constructible<variant<int>>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<variant<int>>::value); + EXPECT_TRUE(is_trivially_move_constructible<variant<int>>::value); + EXPECT_TRUE(is_trivially_move_assignable<variant<int>>::value); +#endif // ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY +} + +TEST(VariantTest, TestVectorOfMoveonlyVariant) { + // Verify that variant<MoveonlyType> works correctly as a std::vector element. + std::vector<variant<std::unique_ptr<int>, std::string>> vec; + vec.push_back(absl::make_unique<int>(42)); + vec.emplace_back("Hello"); + vec.reserve(3); + auto another_vec = absl::move(vec); + // As a sanity check, verify vector contents. + ASSERT_EQ(2, another_vec.size()); + EXPECT_EQ(42, *absl::get<std::unique_ptr<int>>(another_vec[0])); + EXPECT_EQ("Hello", absl::get<std::string>(another_vec[1])); +} + +TEST(VariantTest, NestedVariant) { +#if ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY + static_assert(absl::is_trivially_copy_constructible<variant<int>>(), ""); + static_assert(absl::is_trivially_copy_assignable<variant<int>>(), ""); + static_assert(is_trivially_move_constructible<variant<int>>(), ""); + static_assert(is_trivially_move_assignable<variant<int>>(), ""); + + static_assert(absl::is_trivially_copy_constructible<variant<variant<int>>>(), + ""); + static_assert(absl::is_trivially_copy_assignable<variant<variant<int>>>(), + ""); + static_assert(is_trivially_move_constructible<variant<variant<int>>>(), ""); + static_assert(is_trivially_move_assignable<variant<variant<int>>>(), ""); +#endif // ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY + + variant<int> x(42); + variant<variant<int>> y(x); + variant<variant<int>> z(y); + EXPECT_TRUE(absl::holds_alternative<variant<int>>(z)); + EXPECT_EQ(x, absl::get<variant<int>>(z)); +} + +struct TriviallyDestructible { + TriviallyDestructible(TriviallyDestructible&&) {} + TriviallyDestructible(const TriviallyDestructible&) {} + TriviallyDestructible& operator=(TriviallyDestructible&&) { return *this; } + TriviallyDestructible& operator=(const TriviallyDestructible&) { + return *this; + } +}; + +struct TriviallyMovable { + TriviallyMovable(TriviallyMovable&&) = default; + TriviallyMovable(TriviallyMovable const&) {} + TriviallyMovable& operator=(const TriviallyMovable&) { return *this; } +}; + +struct TriviallyCopyable { + TriviallyCopyable(const TriviallyCopyable&) = default; + TriviallyCopyable& operator=(const TriviallyCopyable&) { return *this; } +}; + +struct TriviallyMoveAssignable { + TriviallyMoveAssignable(TriviallyMoveAssignable&&) = default; + TriviallyMoveAssignable(const TriviallyMoveAssignable&) {} + TriviallyMoveAssignable& operator=(TriviallyMoveAssignable&&) = default; + TriviallyMoveAssignable& operator=(const TriviallyMoveAssignable&) { + return *this; + } +}; + +struct TriviallyCopyAssignable {}; + +#if ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY +TEST(VariantTest, TestTriviality) { + { + using TrivDestVar = absl::variant<TriviallyDestructible>; + + EXPECT_FALSE(is_trivially_move_constructible<TrivDestVar>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<TrivDestVar>::value); + EXPECT_FALSE(is_trivially_move_assignable<TrivDestVar>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<TrivDestVar>::value); + EXPECT_TRUE(absl::is_trivially_destructible<TrivDestVar>::value); + } + + { + using TrivMoveVar = absl::variant<TriviallyMovable>; + + EXPECT_TRUE(is_trivially_move_constructible<TrivMoveVar>::value); + EXPECT_FALSE(absl::is_trivially_copy_constructible<TrivMoveVar>::value); + EXPECT_FALSE(is_trivially_move_assignable<TrivMoveVar>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<TrivMoveVar>::value); + EXPECT_TRUE(absl::is_trivially_destructible<TrivMoveVar>::value); + } + + { + using TrivCopyVar = absl::variant<TriviallyCopyable>; + + EXPECT_TRUE(is_trivially_move_constructible<TrivCopyVar>::value); + EXPECT_TRUE(absl::is_trivially_copy_constructible<TrivCopyVar>::value); + EXPECT_FALSE(is_trivially_move_assignable<TrivCopyVar>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<TrivCopyVar>::value); + EXPECT_TRUE(absl::is_trivially_destructible<TrivCopyVar>::value); + } + + { + using TrivMoveAssignVar = absl::variant<TriviallyMoveAssignable>; + + EXPECT_TRUE(is_trivially_move_constructible<TrivMoveAssignVar>::value); + EXPECT_FALSE( + absl::is_trivially_copy_constructible<TrivMoveAssignVar>::value); + EXPECT_TRUE(is_trivially_move_assignable<TrivMoveAssignVar>::value); + EXPECT_FALSE(absl::is_trivially_copy_assignable<TrivMoveAssignVar>::value); + EXPECT_TRUE(absl::is_trivially_destructible<TrivMoveAssignVar>::value); + } + + { + using TrivCopyAssignVar = absl::variant<TriviallyCopyAssignable>; + + EXPECT_TRUE(is_trivially_move_constructible<TrivCopyAssignVar>::value); + EXPECT_TRUE( + absl::is_trivially_copy_constructible<TrivCopyAssignVar>::value); + EXPECT_TRUE(is_trivially_move_assignable<TrivCopyAssignVar>::value); + EXPECT_TRUE(absl::is_trivially_copy_assignable<TrivCopyAssignVar>::value); + EXPECT_TRUE(absl::is_trivially_destructible<TrivCopyAssignVar>::value); + } +} +#endif // ABSL_VARIANT_PROPAGATE_COPY_MOVE_TRIVIALITY + +// To verify that absl::variant correctly use the nontrivial move ctor of its +// member rather than use the trivial copy constructor. +TEST(VariantTest, MoveCtorBug) { + // To simulate std::tuple in libstdc++. + struct TrivialCopyNontrivialMove { + TrivialCopyNontrivialMove() = default; + TrivialCopyNontrivialMove(const TrivialCopyNontrivialMove&) = default; + TrivialCopyNontrivialMove(TrivialCopyNontrivialMove&&) { called = true; } + bool called = false; + }; + { + using V = absl::variant<TrivialCopyNontrivialMove, int>; + V v1(absl::in_place_index<0>); + // this should invoke the move ctor, rather than the trivial copy ctor. + V v2(std::move(v1)); + EXPECT_TRUE(absl::get<0>(v2).called); + } + { + // this case failed to compile before our fix due to a GCC bug. + using V = absl::variant<int, TrivialCopyNontrivialMove>; + V v1(absl::in_place_index<1>); + // this should invoke the move ctor, rather than the trivial copy ctor. + V v2(std::move(v1)); + EXPECT_TRUE(absl::get<1>(v2).called); + } +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl + +#endif // #if !defined(ABSL_USES_STD_VARIANT) diff --git a/third_party/abseil_cpp/absl/utility/BUILD.bazel b/third_party/abseil_cpp/absl/utility/BUILD.bazel new file mode 100644 index 000000000000..02b2c407f847 --- /dev/null +++ b/third_party/abseil_cpp/absl/utility/BUILD.bazel @@ -0,0 +1,55 @@ +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +cc_library( + name = "utility", + hdrs = [ + "utility.h", + ], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:base_internal", + "//absl/base:config", + "//absl/meta:type_traits", + ], +) + +cc_test( + name = "utility_test", + srcs = ["utility_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":utility", + "//absl/base:core_headers", + "//absl/memory", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/third_party/abseil_cpp/absl/utility/CMakeLists.txt b/third_party/abseil_cpp/absl/utility/CMakeLists.txt new file mode 100644 index 000000000000..e1edd19aa013 --- /dev/null +++ b/third_party/abseil_cpp/absl/utility/CMakeLists.txt @@ -0,0 +1,44 @@ +# +# Copyright 2017 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +absl_cc_library( + NAME + utility + HDRS + "utility.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::config + absl::type_traits + PUBLIC +) + +absl_cc_test( + NAME + utility_test + SRCS + "utility_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::utility + absl::core_headers + absl::memory + absl::strings + gmock_main +) diff --git a/third_party/abseil_cpp/absl/utility/utility.h b/third_party/abseil_cpp/absl/utility/utility.h new file mode 100644 index 000000000000..bf9232209a9b --- /dev/null +++ b/third_party/abseil_cpp/absl/utility/utility.h @@ -0,0 +1,350 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// This header file contains C++11 versions of standard <utility> header +// abstractions available within C++14 and C++17, and are designed to be drop-in +// replacement for code compliant with C++14 and C++17. +// +// The following abstractions are defined: +// +// * integer_sequence<T, Ints...> == std::integer_sequence<T, Ints...> +// * index_sequence<Ints...> == std::index_sequence<Ints...> +// * make_integer_sequence<T, N> == std::make_integer_sequence<T, N> +// * make_index_sequence<N> == std::make_index_sequence<N> +// * index_sequence_for<Ts...> == std::index_sequence_for<Ts...> +// * apply<Functor, Tuple> == std::apply<Functor, Tuple> +// * exchange<T> == std::exchange<T> +// * make_from_tuple<T> == std::make_from_tuple<T> +// +// This header file also provides the tag types `in_place_t`, `in_place_type_t`, +// and `in_place_index_t`, as well as the constant `in_place`, and +// `constexpr` `std::move()` and `std::forward()` implementations in C++11. +// +// References: +// +// https://en.cppreference.com/w/cpp/utility/integer_sequence +// https://en.cppreference.com/w/cpp/utility/apply +// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2013/n3658.html + +#ifndef ABSL_UTILITY_UTILITY_H_ +#define ABSL_UTILITY_UTILITY_H_ + +#include <cstddef> +#include <cstdlib> +#include <tuple> +#include <utility> + +#include "absl/base/config.h" +#include "absl/base/internal/inline_variable.h" +#include "absl/base/internal/invoke.h" +#include "absl/meta/type_traits.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// integer_sequence +// +// Class template representing a compile-time integer sequence. An instantiation +// of `integer_sequence<T, Ints...>` has a sequence of integers encoded in its +// type through its template arguments (which is a common need when +// working with C++11 variadic templates). `absl::integer_sequence` is designed +// to be a drop-in replacement for C++14's `std::integer_sequence`. +// +// Example: +// +// template< class T, T... Ints > +// void user_function(integer_sequence<T, Ints...>); +// +// int main() +// { +// // user_function's `T` will be deduced to `int` and `Ints...` +// // will be deduced to `0, 1, 2, 3, 4`. +// user_function(make_integer_sequence<int, 5>()); +// } +template <typename T, T... Ints> +struct integer_sequence { + using value_type = T; + static constexpr size_t size() noexcept { return sizeof...(Ints); } +}; + +// index_sequence +// +// A helper template for an `integer_sequence` of `size_t`, +// `absl::index_sequence` is designed to be a drop-in replacement for C++14's +// `std::index_sequence`. +template <size_t... Ints> +using index_sequence = integer_sequence<size_t, Ints...>; + +namespace utility_internal { + +template <typename Seq, size_t SeqSize, size_t Rem> +struct Extend; + +// Note that SeqSize == sizeof...(Ints). It's passed explicitly for efficiency. +template <typename T, T... Ints, size_t SeqSize> +struct Extend<integer_sequence<T, Ints...>, SeqSize, 0> { + using type = integer_sequence<T, Ints..., (Ints + SeqSize)...>; +}; + +template <typename T, T... Ints, size_t SeqSize> +struct Extend<integer_sequence<T, Ints...>, SeqSize, 1> { + using type = integer_sequence<T, Ints..., (Ints + SeqSize)..., 2 * SeqSize>; +}; + +// Recursion helper for 'make_integer_sequence<T, N>'. +// 'Gen<T, N>::type' is an alias for 'integer_sequence<T, 0, 1, ... N-1>'. +template <typename T, size_t N> +struct Gen { + using type = + typename Extend<typename Gen<T, N / 2>::type, N / 2, N % 2>::type; +}; + +template <typename T> +struct Gen<T, 0> { + using type = integer_sequence<T>; +}; + +template <typename T> +struct InPlaceTypeTag { + explicit InPlaceTypeTag() = delete; + InPlaceTypeTag(const InPlaceTypeTag&) = delete; + InPlaceTypeTag& operator=(const InPlaceTypeTag&) = delete; +}; + +template <size_t I> +struct InPlaceIndexTag { + explicit InPlaceIndexTag() = delete; + InPlaceIndexTag(const InPlaceIndexTag&) = delete; + InPlaceIndexTag& operator=(const InPlaceIndexTag&) = delete; +}; + +} // namespace utility_internal + +// Compile-time sequences of integers + +// make_integer_sequence +// +// This template alias is equivalent to +// `integer_sequence<int, 0, 1, ..., N-1>`, and is designed to be a drop-in +// replacement for C++14's `std::make_integer_sequence`. +template <typename T, T N> +using make_integer_sequence = typename utility_internal::Gen<T, N>::type; + +// make_index_sequence +// +// This template alias is equivalent to `index_sequence<0, 1, ..., N-1>`, +// and is designed to be a drop-in replacement for C++14's +// `std::make_index_sequence`. +template <size_t N> +using make_index_sequence = make_integer_sequence<size_t, N>; + +// index_sequence_for +// +// Converts a typename pack into an index sequence of the same length, and +// is designed to be a drop-in replacement for C++14's +// `std::index_sequence_for()` +template <typename... Ts> +using index_sequence_for = make_index_sequence<sizeof...(Ts)>; + +// Tag types + +#ifdef ABSL_USES_STD_OPTIONAL + +using std::in_place_t; +using std::in_place; + +#else // ABSL_USES_STD_OPTIONAL + +// in_place_t +// +// Tag type used to specify in-place construction, such as with +// `absl::optional`, designed to be a drop-in replacement for C++17's +// `std::in_place_t`. +struct in_place_t {}; + +ABSL_INTERNAL_INLINE_CONSTEXPR(in_place_t, in_place, {}); + +#endif // ABSL_USES_STD_OPTIONAL + +#if defined(ABSL_USES_STD_ANY) || defined(ABSL_USES_STD_VARIANT) +using std::in_place_type; +using std::in_place_type_t; +#else + +// in_place_type_t +// +// Tag type used for in-place construction when the type to construct needs to +// be specified, such as with `absl::any`, designed to be a drop-in replacement +// for C++17's `std::in_place_type_t`. +template <typename T> +using in_place_type_t = void (*)(utility_internal::InPlaceTypeTag<T>); + +template <typename T> +void in_place_type(utility_internal::InPlaceTypeTag<T>) {} +#endif // ABSL_USES_STD_ANY || ABSL_USES_STD_VARIANT + +#ifdef ABSL_USES_STD_VARIANT +using std::in_place_index; +using std::in_place_index_t; +#else + +// in_place_index_t +// +// Tag type used for in-place construction when the type to construct needs to +// be specified, such as with `absl::any`, designed to be a drop-in replacement +// for C++17's `std::in_place_index_t`. +template <size_t I> +using in_place_index_t = void (*)(utility_internal::InPlaceIndexTag<I>); + +template <size_t I> +void in_place_index(utility_internal::InPlaceIndexTag<I>) {} +#endif // ABSL_USES_STD_VARIANT + +// Constexpr move and forward + +// move() +// +// A constexpr version of `std::move()`, designed to be a drop-in replacement +// for C++14's `std::move()`. +template <typename T> +constexpr absl::remove_reference_t<T>&& move(T&& t) noexcept { + return static_cast<absl::remove_reference_t<T>&&>(t); +} + +// forward() +// +// A constexpr version of `std::forward()`, designed to be a drop-in replacement +// for C++14's `std::forward()`. +template <typename T> +constexpr T&& forward( + absl::remove_reference_t<T>& t) noexcept { // NOLINT(runtime/references) + return static_cast<T&&>(t); +} + +namespace utility_internal { +// Helper method for expanding tuple into a called method. +template <typename Functor, typename Tuple, std::size_t... Indexes> +auto apply_helper(Functor&& functor, Tuple&& t, index_sequence<Indexes...>) + -> decltype(absl::base_internal::invoke( + absl::forward<Functor>(functor), + std::get<Indexes>(absl::forward<Tuple>(t))...)) { + return absl::base_internal::invoke( + absl::forward<Functor>(functor), + std::get<Indexes>(absl::forward<Tuple>(t))...); +} + +} // namespace utility_internal + +// apply +// +// Invokes a Callable using elements of a tuple as its arguments. +// Each element of the tuple corresponds to an argument of the call (in order). +// Both the Callable argument and the tuple argument are perfect-forwarded. +// For member-function Callables, the first tuple element acts as the `this` +// pointer. `absl::apply` is designed to be a drop-in replacement for C++17's +// `std::apply`. Unlike C++17's `std::apply`, this is not currently `constexpr`. +// +// Example: +// +// class Foo { +// public: +// void Bar(int); +// }; +// void user_function1(int, std::string); +// void user_function2(std::unique_ptr<Foo>); +// auto user_lambda = [](int, int) {}; +// +// int main() +// { +// std::tuple<int, std::string> tuple1(42, "bar"); +// // Invokes the first user function on int, std::string. +// absl::apply(&user_function1, tuple1); +// +// std::tuple<std::unique_ptr<Foo>> tuple2(absl::make_unique<Foo>()); +// // Invokes the user function that takes ownership of the unique +// // pointer. +// absl::apply(&user_function2, std::move(tuple2)); +// +// auto foo = absl::make_unique<Foo>(); +// std::tuple<Foo*, int> tuple3(foo.get(), 42); +// // Invokes the method Bar on foo with one argument, 42. +// absl::apply(&Foo::Bar, tuple3); +// +// std::tuple<int, int> tuple4(8, 9); +// // Invokes a lambda. +// absl::apply(user_lambda, tuple4); +// } +template <typename Functor, typename Tuple> +auto apply(Functor&& functor, Tuple&& t) + -> decltype(utility_internal::apply_helper( + absl::forward<Functor>(functor), absl::forward<Tuple>(t), + absl::make_index_sequence<std::tuple_size< + typename std::remove_reference<Tuple>::type>::value>{})) { + return utility_internal::apply_helper( + absl::forward<Functor>(functor), absl::forward<Tuple>(t), + absl::make_index_sequence<std::tuple_size< + typename std::remove_reference<Tuple>::type>::value>{}); +} + +// exchange +// +// Replaces the value of `obj` with `new_value` and returns the old value of +// `obj`. `absl::exchange` is designed to be a drop-in replacement for C++14's +// `std::exchange`. +// +// Example: +// +// Foo& operator=(Foo&& other) { +// ptr1_ = absl::exchange(other.ptr1_, nullptr); +// int1_ = absl::exchange(other.int1_, -1); +// return *this; +// } +template <typename T, typename U = T> +T exchange(T& obj, U&& new_value) { + T old_value = absl::move(obj); + obj = absl::forward<U>(new_value); + return old_value; +} + +namespace utility_internal { +template <typename T, typename Tuple, size_t... I> +T make_from_tuple_impl(Tuple&& tup, absl::index_sequence<I...>) { + return T(std::get<I>(std::forward<Tuple>(tup))...); +} +} // namespace utility_internal + +// make_from_tuple +// +// Given the template parameter type `T` and a tuple of arguments +// `std::tuple(arg0, arg1, ..., argN)` constructs an object of type `T` as if by +// calling `T(arg0, arg1, ..., argN)`. +// +// Example: +// +// std::tuple<const char*, size_t> args("hello world", 5); +// auto s = absl::make_from_tuple<std::string>(args); +// assert(s == "hello"); +// +template <typename T, typename Tuple> +constexpr T make_from_tuple(Tuple&& tup) { + return utility_internal::make_from_tuple_impl<T>( + std::forward<Tuple>(tup), + absl::make_index_sequence< + std::tuple_size<absl::decay_t<Tuple>>::value>{}); +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_UTILITY_UTILITY_H_ diff --git a/third_party/abseil_cpp/absl/utility/utility_test.cc b/third_party/abseil_cpp/absl/utility/utility_test.cc new file mode 100644 index 000000000000..f044ad644a4d --- /dev/null +++ b/third_party/abseil_cpp/absl/utility/utility_test.cc @@ -0,0 +1,376 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/utility/utility.h" + +#include <sstream> +#include <string> +#include <tuple> +#include <type_traits> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" + +namespace { + +#ifdef _MSC_VER +// Warnings for unused variables in this test are false positives. On other +// platforms, they are suppressed by ABSL_ATTRIBUTE_UNUSED, but that doesn't +// work on MSVC. +// Both the unused variables and the name length warnings are due to calls +// to absl::make_index_sequence with very large values, creating very long type +// names. The resulting warnings are so long they make build output unreadable. +#pragma warning( push ) +#pragma warning( disable : 4503 ) // decorated name length exceeded +#pragma warning( disable : 4101 ) // unreferenced local variable +#endif // _MSC_VER + +using ::testing::ElementsAre; +using ::testing::Pointee; +using ::testing::StaticAssertTypeEq; + +TEST(IntegerSequenceTest, ValueType) { + StaticAssertTypeEq<int, absl::integer_sequence<int>::value_type>(); + StaticAssertTypeEq<char, absl::integer_sequence<char>::value_type>(); +} + +TEST(IntegerSequenceTest, Size) { + EXPECT_EQ(0, (absl::integer_sequence<int>::size())); + EXPECT_EQ(1, (absl::integer_sequence<int, 0>::size())); + EXPECT_EQ(1, (absl::integer_sequence<int, 1>::size())); + EXPECT_EQ(2, (absl::integer_sequence<int, 1, 2>::size())); + EXPECT_EQ(3, (absl::integer_sequence<int, 0, 1, 2>::size())); + EXPECT_EQ(3, (absl::integer_sequence<int, -123, 123, 456>::size())); + constexpr size_t sz = absl::integer_sequence<int, 0, 1>::size(); + EXPECT_EQ(2, sz); +} + +TEST(IntegerSequenceTest, MakeIndexSequence) { + StaticAssertTypeEq<absl::index_sequence<>, absl::make_index_sequence<0>>(); + StaticAssertTypeEq<absl::index_sequence<0>, absl::make_index_sequence<1>>(); + StaticAssertTypeEq<absl::index_sequence<0, 1>, + absl::make_index_sequence<2>>(); + StaticAssertTypeEq<absl::index_sequence<0, 1, 2>, + absl::make_index_sequence<3>>(); +} + +TEST(IntegerSequenceTest, MakeIntegerSequence) { + StaticAssertTypeEq<absl::integer_sequence<int>, + absl::make_integer_sequence<int, 0>>(); + StaticAssertTypeEq<absl::integer_sequence<int, 0>, + absl::make_integer_sequence<int, 1>>(); + StaticAssertTypeEq<absl::integer_sequence<int, 0, 1>, + absl::make_integer_sequence<int, 2>>(); + StaticAssertTypeEq<absl::integer_sequence<int, 0, 1, 2>, + absl::make_integer_sequence<int, 3>>(); +} + +template <typename... Ts> +class Counter {}; + +template <size_t... Is> +void CountAll(absl::index_sequence<Is...>) { + // We only need an alias here, but instantiate a variable to silence warnings + // for unused typedefs in some compilers. + ABSL_ATTRIBUTE_UNUSED Counter<absl::make_index_sequence<Is>...> seq; +} + +// This test verifies that absl::make_index_sequence can handle large arguments +// without blowing up template instantiation stack, going OOM or taking forever +// to compile (there is hard 15 minutes limit imposed by forge). +TEST(IntegerSequenceTest, MakeIndexSequencePerformance) { + // O(log N) template instantiations. + // We only need an alias here, but instantiate a variable to silence warnings + // for unused typedefs in some compilers. + ABSL_ATTRIBUTE_UNUSED absl::make_index_sequence<(1 << 16) - 1> seq; + // O(N) template instantiations. + CountAll(absl::make_index_sequence<(1 << 8) - 1>()); +} + +template <typename F, typename Tup, size_t... Is> +auto ApplyFromTupleImpl(F f, const Tup& tup, absl::index_sequence<Is...>) + -> decltype(f(std::get<Is>(tup)...)) { + return f(std::get<Is>(tup)...); +} + +template <typename Tup> +using TupIdxSeq = absl::make_index_sequence<std::tuple_size<Tup>::value>; + +template <typename F, typename Tup> +auto ApplyFromTuple(F f, const Tup& tup) + -> decltype(ApplyFromTupleImpl(f, tup, TupIdxSeq<Tup>{})) { + return ApplyFromTupleImpl(f, tup, TupIdxSeq<Tup>{}); +} + +template <typename T> +std::string Fmt(const T& x) { + std::ostringstream os; + os << x; + return os.str(); +} + +struct PoorStrCat { + template <typename... Args> + std::string operator()(const Args&... args) const { + std::string r; + for (const auto& e : {Fmt(args)...}) r += e; + return r; + } +}; + +template <typename Tup, size_t... Is> +std::vector<std::string> TupStringVecImpl(const Tup& tup, + absl::index_sequence<Is...>) { + return {Fmt(std::get<Is>(tup))...}; +} + +template <typename... Ts> +std::vector<std::string> TupStringVec(const std::tuple<Ts...>& tup) { + return TupStringVecImpl(tup, absl::index_sequence_for<Ts...>()); +} + +TEST(MakeIndexSequenceTest, ApplyFromTupleExample) { + PoorStrCat f{}; + EXPECT_EQ("12abc3.14", f(12, "abc", 3.14)); + EXPECT_EQ("12abc3.14", ApplyFromTuple(f, std::make_tuple(12, "abc", 3.14))); +} + +TEST(IndexSequenceForTest, Basic) { + StaticAssertTypeEq<absl::index_sequence<>, absl::index_sequence_for<>>(); + StaticAssertTypeEq<absl::index_sequence<0>, absl::index_sequence_for<int>>(); + StaticAssertTypeEq<absl::index_sequence<0, 1, 2, 3>, + absl::index_sequence_for<int, void, char, int>>(); +} + +TEST(IndexSequenceForTest, Example) { + EXPECT_THAT(TupStringVec(std::make_tuple(12, "abc", 3.14)), + ElementsAre("12", "abc", "3.14")); +} + +int Function(int a, int b) { return a - b; } + +int Sink(std::unique_ptr<int> p) { return *p; } + +std::unique_ptr<int> Factory(int n) { return absl::make_unique<int>(n); } + +void NoOp() {} + +struct ConstFunctor { + int operator()(int a, int b) const { return a - b; } +}; + +struct MutableFunctor { + int operator()(int a, int b) { return a - b; } +}; + +struct EphemeralFunctor { + EphemeralFunctor() {} + EphemeralFunctor(const EphemeralFunctor&) {} + EphemeralFunctor(EphemeralFunctor&&) {} + int operator()(int a, int b) && { return a - b; } +}; + +struct OverloadedFunctor { + OverloadedFunctor() {} + OverloadedFunctor(const OverloadedFunctor&) {} + OverloadedFunctor(OverloadedFunctor&&) {} + template <typename... Args> + std::string operator()(const Args&... args) & { + return absl::StrCat("&", args...); + } + template <typename... Args> + std::string operator()(const Args&... args) const& { + return absl::StrCat("const&", args...); + } + template <typename... Args> + std::string operator()(const Args&... args) && { + return absl::StrCat("&&", args...); + } +}; + +struct Class { + int Method(int a, int b) { return a - b; } + int ConstMethod(int a, int b) const { return a - b; } + + int member; +}; + +struct FlipFlop { + int ConstMethod() const { return member; } + FlipFlop operator*() const { return {-member}; } + + int member; +}; + +TEST(ApplyTest, Function) { + EXPECT_EQ(1, absl::apply(Function, std::make_tuple(3, 2))); + EXPECT_EQ(1, absl::apply(&Function, std::make_tuple(3, 2))); +} + +TEST(ApplyTest, NonCopyableArgument) { + EXPECT_EQ(42, absl::apply(Sink, std::make_tuple(absl::make_unique<int>(42)))); +} + +TEST(ApplyTest, NonCopyableResult) { + EXPECT_THAT(absl::apply(Factory, std::make_tuple(42)), + ::testing::Pointee(42)); +} + +TEST(ApplyTest, VoidResult) { absl::apply(NoOp, std::tuple<>()); } + +TEST(ApplyTest, ConstFunctor) { + EXPECT_EQ(1, absl::apply(ConstFunctor(), std::make_tuple(3, 2))); +} + +TEST(ApplyTest, MutableFunctor) { + MutableFunctor f; + EXPECT_EQ(1, absl::apply(f, std::make_tuple(3, 2))); + EXPECT_EQ(1, absl::apply(MutableFunctor(), std::make_tuple(3, 2))); +} +TEST(ApplyTest, EphemeralFunctor) { + EphemeralFunctor f; + EXPECT_EQ(1, absl::apply(std::move(f), std::make_tuple(3, 2))); + EXPECT_EQ(1, absl::apply(EphemeralFunctor(), std::make_tuple(3, 2))); +} +TEST(ApplyTest, OverloadedFunctor) { + OverloadedFunctor f; + const OverloadedFunctor& cf = f; + + EXPECT_EQ("&", absl::apply(f, std::tuple<>{})); + EXPECT_EQ("& 42", absl::apply(f, std::make_tuple(" 42"))); + + EXPECT_EQ("const&", absl::apply(cf, std::tuple<>{})); + EXPECT_EQ("const& 42", absl::apply(cf, std::make_tuple(" 42"))); + + EXPECT_EQ("&&", absl::apply(std::move(f), std::tuple<>{})); + OverloadedFunctor f2; + EXPECT_EQ("&& 42", absl::apply(std::move(f2), std::make_tuple(" 42"))); +} + +TEST(ApplyTest, ReferenceWrapper) { + ConstFunctor cf; + MutableFunctor mf; + EXPECT_EQ(1, absl::apply(std::cref(cf), std::make_tuple(3, 2))); + EXPECT_EQ(1, absl::apply(std::ref(cf), std::make_tuple(3, 2))); + EXPECT_EQ(1, absl::apply(std::ref(mf), std::make_tuple(3, 2))); +} + +TEST(ApplyTest, MemberFunction) { + std::unique_ptr<Class> p(new Class); + std::unique_ptr<const Class> cp(new Class); + EXPECT_EQ( + 1, absl::apply(&Class::Method, + std::tuple<std::unique_ptr<Class>&, int, int>(p, 3, 2))); + EXPECT_EQ(1, absl::apply(&Class::Method, + std::tuple<Class*, int, int>(p.get(), 3, 2))); + EXPECT_EQ( + 1, absl::apply(&Class::Method, std::tuple<Class&, int, int>(*p, 3, 2))); + + EXPECT_EQ( + 1, absl::apply(&Class::ConstMethod, + std::tuple<std::unique_ptr<Class>&, int, int>(p, 3, 2))); + EXPECT_EQ(1, absl::apply(&Class::ConstMethod, + std::tuple<Class*, int, int>(p.get(), 3, 2))); + EXPECT_EQ(1, absl::apply(&Class::ConstMethod, + std::tuple<Class&, int, int>(*p, 3, 2))); + + EXPECT_EQ(1, absl::apply(&Class::ConstMethod, + std::tuple<std::unique_ptr<const Class>&, int, int>( + cp, 3, 2))); + EXPECT_EQ(1, absl::apply(&Class::ConstMethod, + std::tuple<const Class*, int, int>(cp.get(), 3, 2))); + EXPECT_EQ(1, absl::apply(&Class::ConstMethod, + std::tuple<const Class&, int, int>(*cp, 3, 2))); + + EXPECT_EQ(1, absl::apply(&Class::Method, + std::make_tuple(absl::make_unique<Class>(), 3, 2))); + EXPECT_EQ(1, absl::apply(&Class::ConstMethod, + std::make_tuple(absl::make_unique<Class>(), 3, 2))); + EXPECT_EQ( + 1, absl::apply(&Class::ConstMethod, + std::make_tuple(absl::make_unique<const Class>(), 3, 2))); +} + +TEST(ApplyTest, DataMember) { + std::unique_ptr<Class> p(new Class{42}); + std::unique_ptr<const Class> cp(new Class{42}); + EXPECT_EQ( + 42, absl::apply(&Class::member, std::tuple<std::unique_ptr<Class>&>(p))); + EXPECT_EQ(42, absl::apply(&Class::member, std::tuple<Class&>(*p))); + EXPECT_EQ(42, absl::apply(&Class::member, std::tuple<Class*>(p.get()))); + + absl::apply(&Class::member, std::tuple<std::unique_ptr<Class>&>(p)) = 42; + absl::apply(&Class::member, std::tuple<Class*>(p.get())) = 42; + absl::apply(&Class::member, std::tuple<Class&>(*p)) = 42; + + EXPECT_EQ(42, absl::apply(&Class::member, + std::tuple<std::unique_ptr<const Class>&>(cp))); + EXPECT_EQ(42, absl::apply(&Class::member, std::tuple<const Class&>(*cp))); + EXPECT_EQ(42, + absl::apply(&Class::member, std::tuple<const Class*>(cp.get()))); +} + +TEST(ApplyTest, FlipFlop) { + FlipFlop obj = {42}; + // This call could resolve to (obj.*&FlipFlop::ConstMethod)() or + // ((*obj).*&FlipFlop::ConstMethod)(). We verify that it's the former. + EXPECT_EQ(42, absl::apply(&FlipFlop::ConstMethod, std::make_tuple(obj))); + EXPECT_EQ(42, absl::apply(&FlipFlop::member, std::make_tuple(obj))); +} + +TEST(ExchangeTest, MoveOnly) { + auto a = Factory(1); + EXPECT_EQ(1, *a); + auto b = absl::exchange(a, Factory(2)); + EXPECT_EQ(2, *a); + EXPECT_EQ(1, *b); +} + +TEST(MakeFromTupleTest, String) { + EXPECT_EQ( + absl::make_from_tuple<std::string>(std::make_tuple("hello world", 5)), + "hello"); +} + +TEST(MakeFromTupleTest, MoveOnlyParameter) { + struct S { + S(std::unique_ptr<int> n, std::unique_ptr<int> m) : value(*n + *m) {} + int value = 0; + }; + auto tup = + std::make_tuple(absl::make_unique<int>(3), absl::make_unique<int>(4)); + auto s = absl::make_from_tuple<S>(std::move(tup)); + EXPECT_EQ(s.value, 7); +} + +TEST(MakeFromTupleTest, NoParameters) { + struct S { + S() : value(1) {} + int value = 2; + }; + EXPECT_EQ(absl::make_from_tuple<S>(std::make_tuple()).value, 1); +} + +TEST(MakeFromTupleTest, Pair) { + EXPECT_EQ( + (absl::make_from_tuple<std::pair<bool, int>>(std::make_tuple(true, 17))), + std::make_pair(true, 17)); +} + +} // namespace + diff --git a/third_party/abseil_cpp/ci/absl_alternate_options.h b/third_party/abseil_cpp/ci/absl_alternate_options.h new file mode 100644 index 000000000000..29b020d9fa65 --- /dev/null +++ b/third_party/abseil_cpp/ci/absl_alternate_options.h @@ -0,0 +1,29 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Alternate options.h file, used in continuous integration testing to exercise +// option settings not used by default. + +#ifndef ABSL_BASE_OPTIONS_H_ +#define ABSL_BASE_OPTIONS_H_ + +#define ABSL_OPTION_USE_STD_ANY 0 +#define ABSL_OPTION_USE_STD_OPTIONAL 0 +#define ABSL_OPTION_USE_STD_STRING_VIEW 0 +#define ABSL_OPTION_USE_STD_VARIANT 0 +#define ABSL_OPTION_USE_INLINE_NAMESPACE 1 +#define ABSL_OPTION_INLINE_NAMESPACE_NAME ns +#define ABSL_OPTION_HARDENED 1 + +#endif // ABSL_BASE_OPTIONS_H_ diff --git a/third_party/abseil_cpp/ci/cmake_common.sh b/third_party/abseil_cpp/ci/cmake_common.sh new file mode 100644 index 000000000000..aec8a117174e --- /dev/null +++ b/third_party/abseil_cpp/ci/cmake_common.sh @@ -0,0 +1,25 @@ +# Copyright 2020 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The commit of GoogleTest to be used in the CMake tests in this directory. +# Keep this in sync with the commit in the WORKSPACE file. +readonly ABSL_GOOGLETEST_COMMIT="8567b09290fe402cf01923e2131c5635b8ed851b" + +# Avoid depending on GitHub by looking for a cached copy of the commit first. +if [[ -r "${KOKORO_GFILE_DIR:-}/distdir/${ABSL_GOOGLETEST_COMMIT}.zip" ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_GFILE_DIR}/distdir,target=/distdir,readonly ${DOCKER_EXTRA_ARGS:-}" + ABSL_GOOGLETEST_DOWNLOAD_URL="file:///distdir/${ABSL_GOOGLETEST_COMMIT}.zip" +else + ABSL_GOOGLETEST_DOWNLOAD_URL="https://github.com/google/googletest/archive/${ABSL_GOOGLETEST_COMMIT}.zip" +fi diff --git a/third_party/abseil_cpp/ci/cmake_install_test.sh b/third_party/abseil_cpp/ci/cmake_install_test.sh new file mode 100755 index 000000000000..5bf540c53828 --- /dev/null +++ b/third_party/abseil_cpp/ci/cmake_install_test.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +source "${ABSEIL_ROOT}/ci/cmake_common.sh" + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_GCC_LATEST_CONTAINER} + +time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp,readonly \ + --workdir=/abseil-cpp \ + --tmpfs=/buildfs:exec \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CFLAGS="-Werror" \ + -e CXXFLAGS="-Werror" \ + ${DOCKER_CONTAINER} \ + /bin/bash CMake/install_test_project/test.sh $@ diff --git a/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_asan_bazel.sh b/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_asan_bazel.sh new file mode 100755 index 000000000000..ffbb8327e731 --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_asan_bazel.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script that can be invoked to test abseil-cpp in a hermetic environment +# using a Docker image on Linux. You must have Docker installed to use this +# script. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +if [[ -z ${STD:-} ]]; then + STD="c++11 c++14 c++17 c++20" +fi + +if [[ -z ${COMPILATION_MODE:-} ]]; then + COMPILATION_MODE="fastbuild opt" +fi + +if [[ -z ${EXCEPTIONS_MODE:-} ]]; then + EXCEPTIONS_MODE="-fno-exceptions -fexceptions" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_CLANG_LATEST_CONTAINER} + +# USE_BAZEL_CACHE=1 only works on Kokoro. +# Without access to the credentials this won't work. +if [[ ${USE_BAZEL_CACHE:-0} -ne 0 ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_KEYSTORE_DIR},target=/keystore,readonly ${DOCKER_EXTRA_ARGS:-}" + # Bazel doesn't track changes to tools outside of the workspace + # (e.g. /usr/bin/gcc), so by appending the docker container to the + # remote_http_cache url, we make changes to the container part of + # the cache key. Hashing the key is to make it shorter and url-safe. + container_key=$(echo ${DOCKER_CONTAINER} | sha256sum | head -c 16) + BAZEL_EXTRA_ARGS="--remote_http_cache=https://storage.googleapis.com/absl-bazel-remote-cache/${container_key} --google_credentials=/keystore/73103_absl-bazel-remote-cache ${BAZEL_EXTRA_ARGS:-}" +fi + +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_GFILE_DIR}/distdir,target=/distdir,readonly ${DOCKER_EXTRA_ARGS:-}" + BAZEL_EXTRA_ARGS="--distdir=/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + +for std in ${STD}; do + for compilation_mode in ${COMPILATION_MODE}; do + for exceptions_mode in ${EXCEPTIONS_MODE}; do + echo "--------------------------------------------------------------------" + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp,readonly \ + --workdir=/abseil-cpp \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CC="/opt/llvm/clang/bin/clang" \ + -e BAZEL_CXXOPTS="-std=${std}:-nostdinc++" \ + -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx/lib:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx/lib" \ + -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx/include/c++/v1" \ + ${DOCKER_EXTRA_ARGS:-} \ + ${DOCKER_CONTAINER} \ + /usr/local/bin/bazel test ... \ + --compilation_mode="${compilation_mode}" \ + --copt="${exceptions_mode}" \ + --copt="-fsanitize=address" \ + --copt="-fsanitize=float-divide-by-zero" \ + --copt="-fsanitize=nullability" \ + --copt="-fsanitize=undefined" \ + --copt="-fno-sanitize-blacklist" \ + --copt=-Werror \ + --keep_going \ + --linkopt="-fsanitize=address" \ + --linkopt="-fsanitize-link-c++-runtime" \ + --show_timestamps \ + --test_env="ASAN_SYMBOLIZER_PATH=/opt/llvm/clang/bin/llvm-symbolizer" \ + --test_env="TZDIR=/abseil-cpp/absl/time/internal/cctz/testdata/zoneinfo" \ + --test_env="UBSAN_OPTIONS=print_stacktrace=1" \ + --test_env="UBSAN_SYMBOLIZER_PATH=/opt/llvm/clang/bin/llvm-symbolizer" \ + --test_output=errors \ + --test_tag_filters="-benchmark,-noasan" \ + ${BAZEL_EXTRA_ARGS:-} + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_bazel.sh b/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_bazel.sh new file mode 100755 index 000000000000..f6a2221e8a00 --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_bazel.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script that can be invoked to test abseil-cpp in a hermetic environment +# using a Docker image on Linux. You must have Docker installed to use this +# script. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +if [[ -z ${STD:-} ]]; then + STD="c++11 c++14 c++17 c++20" +fi + +if [[ -z ${COMPILATION_MODE:-} ]]; then + COMPILATION_MODE="fastbuild opt" +fi + +if [[ -z ${EXCEPTIONS_MODE:-} ]]; then + EXCEPTIONS_MODE="-fno-exceptions -fexceptions" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_CLANG_LATEST_CONTAINER} + +# USE_BAZEL_CACHE=1 only works on Kokoro. +# Without access to the credentials this won't work. +if [[ ${USE_BAZEL_CACHE:-0} -ne 0 ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_KEYSTORE_DIR},target=/keystore,readonly ${DOCKER_EXTRA_ARGS:-}" + # Bazel doesn't track changes to tools outside of the workspace + # (e.g. /usr/bin/gcc), so by appending the docker container to the + # remote_http_cache url, we make changes to the container part of + # the cache key. Hashing the key is to make it shorter and url-safe. + container_key=$(echo ${DOCKER_CONTAINER} | sha256sum | head -c 16) + BAZEL_EXTRA_ARGS="--remote_http_cache=https://storage.googleapis.com/absl-bazel-remote-cache/${container_key} --google_credentials=/keystore/73103_absl-bazel-remote-cache ${BAZEL_EXTRA_ARGS:-}" +fi + +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_GFILE_DIR}/distdir,target=/distdir,readonly ${DOCKER_EXTRA_ARGS:-}" + BAZEL_EXTRA_ARGS="--distdir=/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + +for std in ${STD}; do + for compilation_mode in ${COMPILATION_MODE}; do + for exceptions_mode in ${EXCEPTIONS_MODE}; do + echo "--------------------------------------------------------------------" + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp-ro,readonly \ + --tmpfs=/abseil-cpp \ + --workdir=/abseil-cpp \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CC="/opt/llvm/clang/bin/clang" \ + -e BAZEL_CXXOPTS="-std=${std}:-nostdinc++" \ + -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx/lib:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx/lib" \ + -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx/include/c++/v1" \ + ${DOCKER_EXTRA_ARGS:-} \ + ${DOCKER_CONTAINER} \ + /bin/sh -c " + cp -r /abseil-cpp-ro/* /abseil-cpp/ + if [ -n \"${ALTERNATE_OPTIONS:-}\" ]; then + cp ${ALTERNATE_OPTIONS:-} absl/base/options.h || exit 1 + fi + /usr/local/bin/bazel test ... \ + --compilation_mode=\"${compilation_mode}\" \ + --copt=\"${exceptions_mode}\" \ + --copt=-Werror \ + --define=\"absl=1\" \ + --keep_going \ + --show_timestamps \ + --test_env=\"GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1\" \ + --test_env=\"TZDIR=/abseil-cpp/absl/time/internal/cctz/testdata/zoneinfo\" \ + --test_output=errors \ + --test_tag_filters=-benchmark \ + ${BAZEL_EXTRA_ARGS:-}" + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_tsan_bazel.sh b/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_tsan_bazel.sh new file mode 100755 index 000000000000..e70e8214cd10 --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_clang-latest_libcxx_tsan_bazel.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script that can be invoked to test abseil-cpp in a hermetic environment +# using a Docker image on Linux. You must have Docker installed to use this +# script. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +if [[ -z ${STD:-} ]]; then + STD="c++11 c++14 c++17 c++20" +fi + +if [[ -z ${COMPILATION_MODE:-} ]]; then + COMPILATION_MODE="fastbuild opt" +fi + +if [[ -z ${EXCEPTIONS_MODE:-} ]]; then + EXCEPTIONS_MODE="-fno-exceptions -fexceptions" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_CLANG_LATEST_CONTAINER} + +# USE_BAZEL_CACHE=1 only works on Kokoro. +# Without access to the credentials this won't work. +if [[ ${USE_BAZEL_CACHE:-0} -ne 0 ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_KEYSTORE_DIR},target=/keystore,readonly ${DOCKER_EXTRA_ARGS:-}" + # Bazel doesn't track changes to tools outside of the workspace + # (e.g. /usr/bin/gcc), so by appending the docker container to the + # remote_http_cache url, we make changes to the container part of + # the cache key. Hashing the key is to make it shorter and url-safe. + container_key=$(echo ${DOCKER_CONTAINER} | sha256sum | head -c 16) + BAZEL_EXTRA_ARGS="--remote_http_cache=https://storage.googleapis.com/absl-bazel-remote-cache/${container_key} --google_credentials=/keystore/73103_absl-bazel-remote-cache ${BAZEL_EXTRA_ARGS:-}" +fi + +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_GFILE_DIR}/distdir,target=/distdir,readonly ${DOCKER_EXTRA_ARGS:-}" + BAZEL_EXTRA_ARGS="--distdir=/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + +for std in ${STD}; do + for compilation_mode in ${COMPILATION_MODE}; do + for exceptions_mode in ${EXCEPTIONS_MODE}; do + echo "--------------------------------------------------------------------" + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp,readonly \ + --workdir=/abseil-cpp \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CC="/opt/llvm/clang/bin/clang" \ + -e BAZEL_CXXOPTS="-std=${std}:-nostdinc++" \ + -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx-tsan/lib:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx-tsan/lib" \ + -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx-tsan/include/c++/v1" \ + ${DOCKER_EXTRA_ARGS:-} \ + ${DOCKER_CONTAINER} \ + /usr/local/bin/bazel test ... \ + --build_tag_filters="-notsan" \ + --compilation_mode="${compilation_mode}" \ + --copt="${exceptions_mode}" \ + --copt="-fsanitize=thread" \ + --copt="-fno-sanitize-blacklist" \ + --copt=-Werror \ + --keep_going \ + --linkopt="-fsanitize=thread" \ + --show_timestamps \ + --test_env="TSAN_OPTIONS=report_atomic_races=0" \ + --test_env="TSAN_SYMBOLIZER_PATH=/opt/llvm/clang/bin/llvm-symbolizer" \ + --test_env="TZDIR=/abseil-cpp/absl/time/internal/cctz/testdata/zoneinfo" \ + --test_output=errors \ + --test_tag_filters="-benchmark,-notsan" \ + ${BAZEL_EXTRA_ARGS:-} + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_clang-latest_libstdcxx_bazel.sh b/third_party/abseil_cpp/ci/linux_clang-latest_libstdcxx_bazel.sh new file mode 100755 index 000000000000..0986ff40ccf9 --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_clang-latest_libstdcxx_bazel.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script that can be invoked to test abseil-cpp in a hermetic environment +# using a Docker image on Linux. You must have Docker installed to use this +# script. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +if [[ -z ${STD:-} ]]; then + STD="c++11 c++14 c++17 c++20" +fi + +if [[ -z ${COMPILATION_MODE:-} ]]; then + COMPILATION_MODE="fastbuild opt" +fi + +if [[ -z ${EXCEPTIONS_MODE:-} ]]; then + EXCEPTIONS_MODE="-fno-exceptions -fexceptions" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_CLANG_LATEST_CONTAINER} + +# USE_BAZEL_CACHE=1 only works on Kokoro. +# Without access to the credentials this won't work. +if [[ ${USE_BAZEL_CACHE:-0} -ne 0 ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_KEYSTORE_DIR},target=/keystore,readonly ${DOCKER_EXTRA_ARGS:-}" + # Bazel doesn't track changes to tools outside of the workspace + # (e.g. /usr/bin/gcc), so by appending the docker container to the + # remote_http_cache url, we make changes to the container part of + # the cache key. Hashing the key is to make it shorter and url-safe. + container_key=$(echo ${DOCKER_CONTAINER} | sha256sum | head -c 16) + BAZEL_EXTRA_ARGS="--remote_http_cache=https://storage.googleapis.com/absl-bazel-remote-cache/${container_key} --google_credentials=/keystore/73103_absl-bazel-remote-cache ${BAZEL_EXTRA_ARGS:-}" +fi + +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_GFILE_DIR}/distdir,target=/distdir,readonly ${DOCKER_EXTRA_ARGS:-}" + BAZEL_EXTRA_ARGS="--distdir=/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + +for std in ${STD}; do + for compilation_mode in ${COMPILATION_MODE}; do + for exceptions_mode in ${EXCEPTIONS_MODE}; do + echo "--------------------------------------------------------------------" + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp,readonly \ + --workdir=/abseil-cpp \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CC="/opt/llvm/clang/bin/clang" \ + -e BAZEL_CXXOPTS="-std=${std}" \ + ${DOCKER_EXTRA_ARGS:-} \ + ${DOCKER_CONTAINER} \ + /usr/local/bin/bazel test ... \ + --compilation_mode="${compilation_mode}" \ + --copt="--gcc-toolchain=/usr/local" \ + --copt="${exceptions_mode}" \ + --copt=-Werror \ + --define="absl=1" \ + --keep_going \ + --linkopt="--gcc-toolchain=/usr/local" \ + --show_timestamps \ + --test_env="GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1" \ + --test_env="TZDIR=/abseil-cpp/absl/time/internal/cctz/testdata/zoneinfo" \ + --test_output=errors \ + --test_tag_filters=-benchmark \ + ${BAZEL_EXTRA_ARGS:-} + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_docker_containers.sh b/third_party/abseil_cpp/ci/linux_docker_containers.sh new file mode 100644 index 000000000000..1c29d9a1f17f --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_docker_containers.sh @@ -0,0 +1,21 @@ +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The file contains Docker container identifiers currently used by test scripts. +# Test scripts should source this file to get the identifiers. + +readonly LINUX_ALPINE_CONTAINER="gcr.io/google.com/absl-177019/alpine:20201026" +readonly LINUX_CLANG_LATEST_CONTAINER="gcr.io/google.com/absl-177019/linux_hybrid-latest:20201008" +readonly LINUX_GCC_LATEST_CONTAINER="gcr.io/google.com/absl-177019/linux_hybrid-latest:20201008" +readonly LINUX_GCC_FLOOR_CONTAINER="gcr.io/google.com/absl-177019/linux_gcc-floor:20201015" diff --git a/third_party/abseil_cpp/ci/linux_gcc-floor_libstdcxx_bazel.sh b/third_party/abseil_cpp/ci/linux_gcc-floor_libstdcxx_bazel.sh new file mode 100755 index 000000000000..224aef81cbe7 --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_gcc-floor_libstdcxx_bazel.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# +# Copyright 2020 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script that can be invoked to test abseil-cpp in a hermetic environment +# using a Docker image on Linux. You must have Docker installed to use this +# script. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +if [[ -z ${STD:-} ]]; then + STD="c++11 c++14" +fi + +if [[ -z ${COMPILATION_MODE:-} ]]; then + COMPILATION_MODE="fastbuild opt" +fi + +if [[ -z ${EXCEPTIONS_MODE:-} ]]; then + EXCEPTIONS_MODE="-fno-exceptions -fexceptions" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_GCC_FLOOR_CONTAINER} + +# USE_BAZEL_CACHE=1 only works on Kokoro. +# Without access to the credentials this won't work. +if [[ ${USE_BAZEL_CACHE:-0} -ne 0 ]]; then + DOCKER_EXTRA_ARGS="--volume=${KOKORO_KEYSTORE_DIR}:/keystore:ro ${DOCKER_EXTRA_ARGS:-}" + # Bazel doesn't track changes to tools outside of the workspace + # (e.g. /usr/bin/gcc), so by appending the docker container to the + # remote_http_cache url, we make changes to the container part of + # the cache key. Hashing the key is to make it shorter and url-safe. + container_key=$(echo ${DOCKER_CONTAINER} | sha256sum | head -c 16) + BAZEL_EXTRA_ARGS="--remote_http_cache=https://storage.googleapis.com/absl-bazel-remote-cache/${container_key} --google_credentials=/keystore/73103_absl-bazel-remote-cache ${BAZEL_EXTRA_ARGS:-}" +fi + +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + DOCKER_EXTRA_ARGS="--volume=${KOKORO_GFILE_DIR}/distdir:/distdir:ro ${DOCKER_EXTRA_ARGS:-}" + BAZEL_EXTRA_ARGS="--distdir=/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + +for std in ${STD}; do + for compilation_mode in ${COMPILATION_MODE}; do + for exceptions_mode in ${EXCEPTIONS_MODE}; do + echo "--------------------------------------------------------------------" + time docker run \ + --volume="${ABSEIL_ROOT}:/abseil-cpp:ro" \ + --workdir=/abseil-cpp \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CC="/usr/local/bin/gcc" \ + -e BAZEL_CXXOPTS="-std=${std}" \ + ${DOCKER_EXTRA_ARGS:-} \ + ${DOCKER_CONTAINER} \ + /usr/local/bin/bazel test ... \ + --compilation_mode="${compilation_mode}" \ + --copt="${exceptions_mode}" \ + --copt=-Werror \ + --define="absl=1" \ + --keep_going \ + --show_timestamps \ + --test_env="GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1" \ + --test_env="TZDIR=/abseil-cpp/absl/time/internal/cctz/testdata/zoneinfo" \ + --test_output=errors \ + --test_tag_filters=-benchmark \ + ${BAZEL_EXTRA_ARGS:-} + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_gcc-latest_libstdcxx_bazel.sh b/third_party/abseil_cpp/ci/linux_gcc-latest_libstdcxx_bazel.sh new file mode 100755 index 000000000000..37d89d9f829c --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_gcc-latest_libstdcxx_bazel.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script that can be invoked to test abseil-cpp in a hermetic environment +# using a Docker image on Linux. You must have Docker installed to use this +# script. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +if [[ -z ${STD:-} ]]; then + STD="c++11 c++14 c++17 c++20" +fi + +if [[ -z ${COMPILATION_MODE:-} ]]; then + COMPILATION_MODE="fastbuild opt" +fi + +if [[ -z ${EXCEPTIONS_MODE:-} ]]; then + EXCEPTIONS_MODE="-fno-exceptions -fexceptions" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_GCC_LATEST_CONTAINER} + +# USE_BAZEL_CACHE=1 only works on Kokoro. +# Without access to the credentials this won't work. +if [[ ${USE_BAZEL_CACHE:-0} -ne 0 ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_KEYSTORE_DIR},target=/keystore,readonly ${DOCKER_EXTRA_ARGS:-}" + # Bazel doesn't track changes to tools outside of the workspace + # (e.g. /usr/bin/gcc), so by appending the docker container to the + # remote_http_cache url, we make changes to the container part of + # the cache key. Hashing the key is to make it shorter and url-safe. + container_key=$(echo ${DOCKER_CONTAINER} | sha256sum | head -c 16) + BAZEL_EXTRA_ARGS="--remote_http_cache=https://storage.googleapis.com/absl-bazel-remote-cache/${container_key} --google_credentials=/keystore/73103_absl-bazel-remote-cache ${BAZEL_EXTRA_ARGS:-}" +fi + +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + DOCKER_EXTRA_ARGS="--mount type=bind,source=${KOKORO_GFILE_DIR}/distdir,target=/distdir,readonly ${DOCKER_EXTRA_ARGS:-}" + BAZEL_EXTRA_ARGS="--distdir=/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + +for std in ${STD}; do + for compilation_mode in ${COMPILATION_MODE}; do + for exceptions_mode in ${EXCEPTIONS_MODE}; do + echo "--------------------------------------------------------------------" + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp-ro,readonly \ + --tmpfs=/abseil-cpp \ + --workdir=/abseil-cpp \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CC="/usr/local/bin/gcc" \ + -e BAZEL_CXXOPTS="-std=${std}" \ + ${DOCKER_EXTRA_ARGS:-} \ + ${DOCKER_CONTAINER} \ + /bin/sh -c " + cp -r /abseil-cpp-ro/* /abseil-cpp/ + if [ -n \"${ALTERNATE_OPTIONS:-}\" ]; then + cp ${ALTERNATE_OPTIONS:-} absl/base/options.h || exit 1 + fi + /usr/local/bin/bazel test ... \ + --compilation_mode=\"${compilation_mode}\" \ + --copt=\"${exceptions_mode}\" \ + --copt=-Werror \ + --define=\"absl=1\" \ + --keep_going \ + --show_timestamps \ + --test_env=\"GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1\" \ + --test_env=\"TZDIR=/abseil-cpp/absl/time/internal/cctz/testdata/zoneinfo\" \ + --test_output=errors \ + --test_tag_filters=-benchmark \ + ${BAZEL_EXTRA_ARGS:-}" + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_gcc-latest_libstdcxx_cmake.sh b/third_party/abseil_cpp/ci/linux_gcc-latest_libstdcxx_cmake.sh new file mode 100755 index 000000000000..ed9cfa385251 --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_gcc-latest_libstdcxx_cmake.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +source "${ABSEIL_ROOT}/ci/cmake_common.sh" + +if [[ -z ${ABSL_CMAKE_CXX_STANDARDS:-} ]]; then + ABSL_CMAKE_CXX_STANDARDS="11 14 17 20" +fi + +if [[ -z ${ABSL_CMAKE_BUILD_TYPES:-} ]]; then + ABSL_CMAKE_BUILD_TYPES="Debug Release" +fi + +if [[ -z ${ABSL_CMAKE_BUILD_SHARED:-} ]]; then + ABSL_CMAKE_BUILD_SHARED="OFF ON" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_GCC_LATEST_CONTAINER} + +for std in ${ABSL_CMAKE_CXX_STANDARDS}; do + for compilation_mode in ${ABSL_CMAKE_BUILD_TYPES}; do + for build_shared in ${ABSL_CMAKE_BUILD_SHARED}; do + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp,readonly \ + --tmpfs=/buildfs:exec \ + --workdir=/buildfs \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CFLAGS="-Werror" \ + -e CXXFLAGS="-Werror" \ + ${DOCKER_EXTRA_ARGS:-} \ + "${DOCKER_CONTAINER}" \ + /bin/bash -c " + cmake /abseil-cpp \ + -DABSL_GOOGLETEST_DOWNLOAD_URL=${ABSL_GOOGLETEST_DOWNLOAD_URL} \ + -DABSL_RUN_TESTS=ON \ + -DBUILD_SHARED_LIBS=${build_shared} \ + -DCMAKE_BUILD_TYPE=${compilation_mode} \ + -DCMAKE_CXX_STANDARD=${std} \ + -DCMAKE_MODULE_LINKER_FLAGS=\"-Wl,--no-undefined\" && \ + make -j$(nproc) && \ + ctest -j$(nproc) --output-on-failure" + done + done +done diff --git a/third_party/abseil_cpp/ci/linux_gcc_alpine_cmake.sh b/third_party/abseil_cpp/ci/linux_gcc_alpine_cmake.sh new file mode 100755 index 000000000000..31310ac791ec --- /dev/null +++ b/third_party/abseil_cpp/ci/linux_gcc_alpine_cmake.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +source "${ABSEIL_ROOT}/ci/cmake_common.sh" + +if [[ -z ${ABSL_CMAKE_CXX_STANDARDS:-} ]]; then + ABSL_CMAKE_CXX_STANDARDS="11 14 17" +fi + +if [[ -z ${ABSL_CMAKE_BUILD_TYPES:-} ]]; then + ABSL_CMAKE_BUILD_TYPES="Debug Release" +fi + +if [[ -z ${ABSL_CMAKE_BUILD_SHARED:-} ]]; then + ABSL_CMAKE_BUILD_SHARED="OFF ON" +fi + +source "${ABSEIL_ROOT}/ci/linux_docker_containers.sh" +readonly DOCKER_CONTAINER=${LINUX_ALPINE_CONTAINER} + +for std in ${ABSL_CMAKE_CXX_STANDARDS}; do + for compilation_mode in ${ABSL_CMAKE_BUILD_TYPES}; do + for build_shared in ${ABSL_CMAKE_BUILD_SHARED}; do + time docker run \ + --mount type=bind,source="${ABSEIL_ROOT}",target=/abseil-cpp,readonly \ + --tmpfs=/buildfs:exec \ + --workdir=/buildfs \ + --cap-add=SYS_PTRACE \ + --rm \ + -e CFLAGS="-Werror" \ + -e CXXFLAGS="-Werror" \ + ${DOCKER_EXTRA_ARGS:-} \ + "${DOCKER_CONTAINER}" \ + /bin/sh -c " + cmake /abseil-cpp \ + -DABSL_GOOGLETEST_DOWNLOAD_URL=${ABSL_GOOGLETEST_DOWNLOAD_URL} \ + -DABSL_RUN_TESTS=ON \ + -DCMAKE_BUILD_TYPE=${compilation_mode} \ + -DCMAKE_CXX_STANDARD=${std} \ + -DCMAKE_MODULE_LINKER_FLAGS=\"-Wl,--no-undefined\" && \ + make -j$(nproc) && \ + ctest -j$(nproc) --output-on-failure" + done + done +done diff --git a/third_party/abseil_cpp/ci/macos_xcode_bazel.sh b/third_party/abseil_cpp/ci/macos_xcode_bazel.sh new file mode 100755 index 000000000000..738adf947655 --- /dev/null +++ b/third_party/abseil_cpp/ci/macos_xcode_bazel.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is invoked on Kokoro to test Abseil on macOS. +# It is not hermetic and may break when Kokoro is updated. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(realpath $(dirname ${0})/..)" +fi + +# If we are running on Kokoro, check for a versioned Bazel binary. +KOKORO_GFILE_BAZEL_BIN="bazel-2.0.0-darwin-x86_64" +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -f ${KOKORO_GFILE_DIR}/${KOKORO_GFILE_BAZEL_BIN} ]]; then + BAZEL_BIN="${KOKORO_GFILE_DIR}/${KOKORO_GFILE_BAZEL_BIN}" + chmod +x ${BAZEL_BIN} +else + BAZEL_BIN="bazel" +fi + +# Print the compiler and Bazel versions. +echo "---------------" +gcc -v +echo "---------------" +${BAZEL_BIN} version +echo "---------------" + +cd ${ABSEIL_ROOT} + +if [[ -n "${ALTERNATE_OPTIONS:-}" ]]; then + cp ${ALTERNATE_OPTIONS:-} absl/base/options.h || exit 1 +fi + +${BAZEL_BIN} test ... \ + --copt=-Werror \ + --keep_going \ + --show_timestamps \ + --test_env="TZDIR=${ABSEIL_ROOT}/absl/time/internal/cctz/testdata/zoneinfo" \ + --test_output=errors \ + --test_tag_filters=-benchmark diff --git a/third_party/abseil_cpp/ci/macos_xcode_cmake.sh b/third_party/abseil_cpp/ci/macos_xcode_cmake.sh new file mode 100755 index 000000000000..0847b3ea78a9 --- /dev/null +++ b/third_party/abseil_cpp/ci/macos_xcode_cmake.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Copyright 2019 The Abseil Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euox pipefail + +if [[ -z ${ABSEIL_ROOT:-} ]]; then + ABSEIL_ROOT="$(dirname ${0})/.." +fi +ABSEIL_ROOT=$(realpath ${ABSEIL_ROOT}) + +source "${ABSEIL_ROOT}/ci/cmake_common.sh" + +# The MacOS build doesn't run in a docker container, so we have to override ABSL_GOOGLETEST_DOWNLOAD_URL. +if [[ -r "${KOKORO_GFILE_DIR}/distdir/${ABSL_GOOGLETEST_COMMIT}.zip" ]]; then + ABSL_GOOGLETEST_DOWNLOAD_URL="file://${KOKORO_GFILE_DIR}/distdir/${ABSL_GOOGLETEST_COMMIT}.zip" +fi + +if [[ -z ${ABSL_CMAKE_BUILD_TYPES:-} ]]; then + ABSL_CMAKE_BUILD_TYPES="Debug" +fi + +if [[ -z ${ABSL_CMAKE_BUILD_SHARED:-} ]]; then + ABSL_CMAKE_BUILD_SHARED="OFF ON" +fi + +for compilation_mode in ${ABSL_CMAKE_BUILD_TYPES}; do + for build_shared in ${ABSL_CMAKE_BUILD_SHARED}; do + BUILD_DIR=$(mktemp -d ${compilation_mode}.XXXXXXXX) + cd ${BUILD_DIR} + + # TODO(absl-team): Enable -Werror once all warnings are fixed. + time cmake ${ABSEIL_ROOT} \ + -GXcode \ + -DBUILD_SHARED_LIBS=${build_shared} \ + -DCMAKE_BUILD_TYPE=${compilation_mode} \ + -DCMAKE_CXX_STANDARD=11 \ + -DCMAKE_MODULE_LINKER_FLAGS="-Wl,--no-undefined" \ + -DABSL_GOOGLETEST_DOWNLOAD_URL="${ABSL_GOOGLETEST_DOWNLOAD_URL}" \ + -DABSL_RUN_TESTS=ON + time cmake --build . + time ctest -C ${compilation_mode} --output-on-failure + done +done diff --git a/third_party/abseil_cpp/conanfile.py b/third_party/abseil_cpp/conanfile.py new file mode 100755 index 000000000000..926ec5ccd64b --- /dev/null +++ b/third_party/abseil_cpp/conanfile.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Note: Conan is supported on a best-effort basis. Abseil doesn't use Conan +# internally, so we won't know if it stops working. We may ask community +# members to help us debug any problems that arise. + +from conans import ConanFile, CMake, tools +from conans.errors import ConanInvalidConfiguration +from conans.model.version import Version + + +class AbseilConan(ConanFile): + name = "abseil" + url = "https://github.com/abseil/abseil-cpp" + homepage = url + author = "Abseil <abseil-io@googlegroups.com>" + description = "Abseil Common Libraries (C++) from Google" + license = "Apache-2.0" + topics = ("conan", "abseil", "abseil-cpp", "google", "common-libraries") + exports = ["LICENSE"] + exports_sources = ["CMakeLists.txt", "CMake/*", "absl/*"] + generators = "cmake" + settings = "os", "arch", "compiler", "build_type" + + def configure(self): + if self.settings.os == "Windows" and \ + self.settings.compiler == "Visual Studio" and \ + Version(self.settings.compiler.version.value) < "14": + raise ConanInvalidConfiguration("Abseil does not support MSVC < 14") + + def build(self): + tools.replace_in_file("CMakeLists.txt", "project(absl CXX)", "project(absl CXX)\ninclude(conanbuildinfo.cmake)\nconan_basic_setup()") + cmake = CMake(self) + cmake.definitions["BUILD_TESTING"] = False + cmake.configure() + cmake.build() + + def package(self): + self.copy("LICENSE", dst="licenses") + self.copy("*.h", dst="include", src=".") + self.copy("*.inc", dst="include", src=".") + self.copy("*.a", dst="lib", src=".", keep_path=False) + self.copy("*.lib", dst="lib", src=".", keep_path=False) + + def package_info(self): + if self.settings.os == "Linux": + self.cpp_info.libs = ["-Wl,--start-group"] + self.cpp_info.libs.extend(tools.collect_libs(self)) + if self.settings.os == "Linux": + self.cpp_info.libs.extend(["-Wl,--end-group", "pthread"]) diff --git a/third_party/abseil_cpp/default.nix b/third_party/abseil_cpp/default.nix new file mode 100644 index 000000000000..66ab4f6ba143 --- /dev/null +++ b/third_party/abseil_cpp/default.nix @@ -0,0 +1,31 @@ +{ pkgs, lib, ... }: + +let inherit (pkgs) cmake fullLlvm11Stdenv; +in pkgs.abseil-cpp.override { + stdenv = fullLlvm11Stdenv; + cxxStandard = "17"; +} + +/* TODO(tazjin): update abseil subtree + +fullLlvm11Stdenv.mkDerivation rec { + pname = "abseil-cpp"; + version = "20200519-768eb2ca+tvl-1"; + src = ./.; + nativeBuildInputs = [ cmake ]; + # TODO: run tests + # doCheck = true; + + cmakeFlags = [ + "-DCMAKE_CXX_STANDARD=17" + #"-DABSL_RUN_TESTS=1" + ]; + + meta = with lib; { + description = "An open-source collection of C++ code designed to augment the C++ standard library"; + homepage = https://abseil.io/; + license = licenses.asl20; + maintainers = [ maintainers.andersk ]; + }; +} +*/ diff --git a/third_party/agenix/default.nix b/third_party/agenix/default.nix new file mode 100644 index 000000000000..91fd9576b091 --- /dev/null +++ b/third_party/agenix/default.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: + +let + commit = "52ea2f8c3231cc2b5302fa28c63588aacb77ea29"; + src = builtins.fetchTarball { + url = "https://github.com/ryantm/agenix/archive/${commit}.tar.gz"; + sha256 = "1sqgbriwmvxcmqp0zbk7873psk9g60a53fgrr9p0jafki5zzgvdx"; + }; + agenix = import src { + inherit pkgs; + }; +in { + inherit src; + cli = agenix.agenix; +} diff --git a/third_party/alsi/OWNERS b/third_party/alsi/OWNERS new file mode 100644 index 000000000000..79a422fcde1c --- /dev/null +++ b/third_party/alsi/OWNERS @@ -0,0 +1,3 @@ +inherit: true +owners: + - grfn diff --git a/third_party/alsi/default.nix b/third_party/alsi/default.nix new file mode 100644 index 000000000000..8969374176d9 --- /dev/null +++ b/third_party/alsi/default.nix @@ -0,0 +1,25 @@ +{ pkgs, ... }: + +with pkgs; + +stdenv.mkDerivation { + name = "alsi"; + pname = "alsi"; + version = "0.4.8"; + + src = fetchFromGitHub { + owner = "trizen"; + repo = "alsi"; + rev = "fe2a925caad38d4cc7afe10d74ba60c5db09ee66"; + sha256 = "060xlalfclrda5f1h3svj4v2gr19mdrsc62vrg7hgii0f3lib7j5"; + }; + + buildInputs = [ + (perl.withPackages (ps: with ps; [ DataDump ])) + ]; + + installPhase = '' + mkdir -p $out/bin + cp alsi $out/bin/alsi + ''; +} diff --git a/third_party/apereo-cas/default.nix b/third_party/apereo-cas/default.nix new file mode 100644 index 000000000000..bda607c07287 --- /dev/null +++ b/third_party/apereo-cas/default.nix @@ -0,0 +1,63 @@ +{ pkgs, lib, ... }: + +let + name = "apereo-cas-${version}"; + version = "6.2.0"; + overlay = ./overlay; + + jdk = pkgs.jdk11; + gradle = pkgs.gradle_6; + + meta = with lib; { + homepage = "https://www.apereo.org/projects/cas"; + description = "CAS provides enterprise single sign-on for the Web"; + platforms = platforms.linux; + licenses = licenses.asl20; + }; + + deps = pkgs.stdenvNoCC.mkDerivation { + inherit meta; + + name = "${name}-deps"; + src = overlay; + nativeBuildInputs = with pkgs; [ gradle perl ]; + + buildPhase = '' + export GRADLE_USER_HOME="$(mktemp -d)" + gradle --no-daemon build -x test downloadShell + ''; + + # perl code mavenizes paths (see pkgs/servers/ma1sd from nixpkgs) + installPhase = '' + find $GRADLE_USER_HOME/caches/modules-2 -type f -regex '.*\.\([wj]ar\|pom\)' \ + | perl -pe 's#(.*/([^/]+)/([^/]+)/([^/]+)/[0-9a-f]{30,40}/([^/\s]+))$# $src = $1; ($x = $2) =~ tr|\.|/|; $dir1 = $3; $dir2 = $4; ($y = $5) =~ s|-jvm||n; "install -Dm444 $src \$out/$x/$dir1/$dir2/$y" #e' \ + | sh + install -m444 build/libs/cas-server-support-shell-* $out/cas-server-shell.jar + ''; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "07wxmgljs8v0pmnryqjz6dr6jl93x3023y8zx3al5314mmqkpaan"; + }; +in +pkgs.stdenvNoCC.mkDerivation { + inherit name version meta; + + src = overlay; + nativeBuildInputs = with pkgs; [ gradle makeWrapper ]; + buildInputs = with pkgs; [ jdk ]; + + buildPhase = '' + export GRADLE_USER_HOME="$(mktemp -d)" + + gradle -PcacheLocation="${deps}" --offline --no-daemon build -x test + ''; + + installPhase = '' + install -D build/libs/cas.war $out/lib/cas.war + cp ${deps}/cas-server-shell.jar $out/lib/cas-server-shell.jar + cp -R etc $out/etc + makeWrapper ${jdk}/bin/java $out/bin/cas --add-flags "-Dcas.standalone.configurationDirectory=$out/etc/cas/config -jar $out/lib/cas.war" + makeWrapper ${jdk}/bin/java $out/bin/cas-shell --add-flags "-Dcas.standalone.configurationDirectory=$out/etc/cas/config -jar $out/lib/cas-server-shell.jar" + ''; +} diff --git a/third_party/apereo-cas/overlay/.dockerignore b/third_party/apereo-cas/overlay/.dockerignore new file mode 100644 index 000000000000..5e04587dd08d --- /dev/null +++ b/third_party/apereo-cas/overlay/.dockerignore @@ -0,0 +1,7 @@ +target/** +build/** +bin/** +.idea/** +.history/** +.github/** +.git/** diff --git a/third_party/apereo-cas/overlay/.gitattributes b/third_party/apereo-cas/overlay/.gitattributes new file mode 100644 index 000000000000..8fc5677e1dd7 --- /dev/null +++ b/third_party/apereo-cas/overlay/.gitattributes @@ -0,0 +1,6 @@ +# Set line endings to LF, even on Windows. Otherwise, execution within Docker fails. +# See https://help.github.com/articles/dealing-with-line-endings/ +*.sh text eol=lf +gradlew text eol=lf +*.cmd text eol=crlf +*.bat text eol=crlf diff --git a/third_party/apereo-cas/overlay/.github/FUNDING.yml b/third_party/apereo-cas/overlay/.github/FUNDING.yml new file mode 100644 index 000000000000..30b54fb7341b --- /dev/null +++ b/third_party/apereo-cas/overlay/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: ['https://www.apereo.org/content/apereo-membership'] diff --git a/third_party/apereo-cas/overlay/.github/renovate.json b/third_party/apereo-cas/overlay/.github/renovate.json new file mode 100644 index 000000000000..58fa77f03cf0 --- /dev/null +++ b/third_party/apereo-cas/overlay/.github/renovate.json @@ -0,0 +1,11 @@ +{ + "extends": [ + "config:base", + ":preserveSemverRanges", + ":rebaseStalePrs", + ":disableRateLimiting", + ":semanticCommits", + ":semanticCommitTypeAll(renovatebot)" + ], + "labels": ["dependencies", "bot"] +} diff --git a/third_party/apereo-cas/overlay/.gitignore b/third_party/apereo-cas/overlay/.gitignore new file mode 100755 index 000000000000..45840be342c7 --- /dev/null +++ b/third_party/apereo-cas/overlay/.gitignore @@ -0,0 +1,22 @@ +.classpath +!/.project +.project +.settings +.history +.vscode +target/ +.idea/ +.DS_Store +.idea +overlays/ +.gradle/ +build/ +log/ +bin/ +*.war +*.iml +*.log +tmp/ +./apache-tomcat +apache-tomcat.zip +config-metadata.properties diff --git a/third_party/apereo-cas/overlay/.mergify.yml b/third_party/apereo-cas/overlay/.mergify.yml new file mode 100644 index 000000000000..4fcbdbe4acab --- /dev/null +++ b/third_party/apereo-cas/overlay/.mergify.yml @@ -0,0 +1,32 @@ +# +# Licensed to Apereo under one or more contributor license +# agreements. See the NOTICE file distributed with this work +# for additional information regarding copyright ownership. +# Apereo licenses this file to you under the Apache License, +# Version 2.0 (the "License"); you may not use this file +# except in compliance with the License. You may obtain a +# copy of the License at the following location: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +pull_request_rules: +- name: automatic merge by dependabot + conditions: + - status-success=continuous-integration/travis-ci/pr + - status-success=WIP + - "#changes-requested-reviews-by=0" + - base=master + - label=dependencies + actions: + merge: + method: merge + strict: true + delete_head_branch: \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/.travis.yml b/third_party/apereo-cas/overlay/.travis.yml new file mode 100644 index 000000000000..8347dd1719ea --- /dev/null +++ b/third_party/apereo-cas/overlay/.travis.yml @@ -0,0 +1,62 @@ +language: java +sudo: required +dist: trusty +services: + - docker +branches: + only: + - master +before_cache: + - rm -rf $HOME/.gradle/caches/5.*/ + - rm -rf $HOME/.gradle/caches/4.*/ + - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ + - find ~/.gradle/caches/ -name "*.lock" -type f -delete +cache: + bundler: false + cargo: false + directories: + - $HOME/.m2 + - $HOME/.npm/ + - $HOME/.gradle/caches/ + - $HOME/.gradle/wrapper/ +env: + global: + - JAVA_OPTS="-Xms512m -Xmx4048m -Xss128m -XX:ReservedCodeCacheSize=512m -XX:+UseG1GC -Xverify:none -server" + - GRADLE_OPTS="-Xms512m -Xmx1024m -Xss128m -XX:ReservedCodeCacheSize=512m -XX:+UseG1GC -Xverify:none -server" +jdk: +- openjdk11 +before_install: +- echo -e "Configuring Gradle wrapper...\n" +- mkdir -p ~/.gradle && echo "org.gradle.daemon=false" >> ~/.gradle/gradle.properties +- chmod -R 777 ./gradlew +- chmod -R 777 *.sh +install: true +stages: + - build + - validate + - docker +jobs: + include: + - stage: build + script: ./gradlew clean build --stacktrace --no-daemon --refresh-dependencies -Dorg.gradle.internal.http.socketTimeout=600000 -Dorg.gradle.internal.http.connectionTimeout=600000 + name: "Build CAS" + ############################################ + - stage: validate + script: ./gradlew downloadShell + name: "Download CAS Shell" + - stage: validate + script: ./gradlew listTemplateViews + name: "List CAS Template Views" + - stage: validate + script: ./gradlew explodeWar + name: "Unzip CAS Web Application" + ############################################ + - stage: docker + script: ./gradlew build jibDockerBuild --stacktrace --no-daemon --refresh-dependencies + name: "Build Docker Image via Jib" + - stage: docker + script: docker-compose build + name: "Build Docker Image via Docker Compose" + - stage: docker + script: ./docker-build.sh + name: "Build Docker Image" \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/Dockerfile b/third_party/apereo-cas/overlay/Dockerfile new file mode 100644 index 000000000000..17ab5648beb9 --- /dev/null +++ b/third_party/apereo-cas/overlay/Dockerfile @@ -0,0 +1,40 @@ +FROM adoptopenjdk/openjdk11:alpine-slim AS overlay + +RUN mkdir -p cas-overlay +COPY ./src cas-overlay/src/ +COPY ./gradle/ cas-overlay/gradle/ +COPY ./gradlew ./settings.gradle ./build.gradle ./gradle.properties /cas-overlay/ + +RUN mkdir -p ~/.gradle \ + && echo "org.gradle.daemon=false" >> ~/.gradle/gradle.properties \ + && echo "org.gradle.configureondemand=true" >> ~/.gradle/gradle.properties \ + && cd cas-overlay \ + && chmod 750 ./gradlew \ + && ./gradlew --version; + +RUN cd cas-overlay \ + && ./gradlew clean build --parallel --no-daemon; + +FROM adoptopenjdk/openjdk11:alpine-jre AS cas + +LABEL "Organization"="Apereo" +LABEL "Description"="Apereo CAS" + +RUN cd / \ + && mkdir -p /etc/cas/config \ + && mkdir -p /etc/cas/services \ + && mkdir -p /etc/cas/saml \ + && mkdir -p cas-overlay; + +COPY etc/cas/ /etc/cas/ +COPY etc/cas/config/ /etc/cas/config/ +COPY etc/cas/services/ /etc/cas/services/ +COPY etc/cas/saml/ /etc/cas/saml/ +COPY --from=overlay cas-overlay/build/libs/cas.war cas-overlay/ + +EXPOSE 8080 8443 + +ENV PATH $PATH:$JAVA_HOME/bin:. + +WORKDIR cas-overlay +ENTRYPOINT ["java", "-server", "-noverify", "-Xmx2048M", "-jar", "cas.war"] diff --git a/third_party/apereo-cas/overlay/LICENSE.txt b/third_party/apereo-cas/overlay/LICENSE.txt new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/third_party/apereo-cas/overlay/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/apereo-cas/overlay/README.md b/third_party/apereo-cas/overlay/README.md new file mode 100644 index 000000000000..ec8d96483616 --- /dev/null +++ b/third_party/apereo-cas/overlay/README.md @@ -0,0 +1,146 @@ +CAS Overlay Template [](https://travis-ci.org/apereo/cas-overlay-template) +======================= + +Generic CAS WAR overlay to exercise the latest versions of CAS. This overlay could be freely used as a starting template for local CAS war overlays. + +# Versions + +- CAS `6.2.x` +- JDK `11` + +# Overview + +To build the project, use: + +```bash +# Use --refresh-dependencies to force-update SNAPSHOT versions +./gradlew[.bat] clean build +``` + +To see what commands are available to the build script, run: + +```bash +./gradlew[.bat] tasks +``` + +To launch into the CAS command-line shell: + +```bash +./gradlew[.bat] downloadShell runShell +``` + +To fetch and overlay a CAS resource or view, use: + +```bash +./gradlew[.bat] getResource -PresourceName=[resource-name] +``` + +To list all available CAS views and templates: + +```bash +./gradlew[.bat] listTemplateViews +``` + +To unzip and explode the CAS web application file and the internal resources jar: + +```bash +./gradlew[.bat] explodeWar +``` + +# Configuration + +- The `etc` directory contains the configuration files and directories that need to be copied to `/etc/cas/config`. + +```bash +./gradlew[.bat] copyCasConfiguration +``` + +- The specifics of the build are controlled using the `gradle.properties` file. + +## Adding Modules + +CAS modules may be specified under the `dependencies` block of the [Gradle build script](build.gradle): + +```gradle +dependencies { + compile "org.apereo.cas:cas-server-some-module:${project.casVersion}" + ... +} +``` + +To collect the list of all project modules and dependencies: + +```bash +./gradlew[.bat] allDependencies +``` + +### Clear Gradle Cache + +If you need to, on Linux/Unix systems, you can delete all the existing artifacts (artifacts and metadata) Gradle has downloaded using: + +```bash +# Only do this when absolutely necessary +rm -rf $HOME/.gradle/caches/ +``` + +Same strategy applies to Windows too, provided you switch `$HOME` to its equivalent in the above command. + +# Deployment + +- Create a keystore file `thekeystore` under `/etc/cas`. Use the password `changeit` for both the keystore and the key/certificate entries. This can either be done using the JDK's `keytool` utility or via the following command: + +```bash +./gradlew[.bat] createKeystore +``` + +- Ensure the keystore is loaded up with keys and certificates of the server. + +On a successful deployment via the following methods, CAS will be available at: + +* `https://cas.server.name:8443/cas` + +## Executable WAR + +Run the CAS web application as an executable WAR: + +```bash +./gradlew[.bat] run +``` + +Debug the CAS web application as an executable WAR: + +```bash +./gradlew[.bat] debug +``` + +Run the CAS web application as a *standalone* executable WAR: + +```bash +./gradlew[.bat] clean executable +``` + +## External + +Deploy the binary web application file `cas.war` after a successful build to a servlet container of choice. + +## Docker + +The following strategies outline how to build and deploy CAS Docker images. + +### Jib + +The overlay embraces the [Jib Gradle Plugin](https://github.com/GoogleContainerTools/jib) to provide easy-to-use out-of-the-box tooling for building CAS docker images. Jib is an open-source Java containerizer from Google that lets Java developers build containers using the tools they know. It is a container image builder that handles all the steps of packaging your application into a container image. It does not require you to write a Dockerfile or have Docker installed, and it is directly integrated into the overlay. + +```bash +./gradlew build jibDockerBuild +``` + +### Dockerfile + +You can also use the native Docker tooling and the provided `Dockerfile` to build and run CAS. + +```bash +chmod +x *.sh +./docker-build.sh +./docker-run.sh +``` diff --git a/third_party/apereo-cas/overlay/build.gradle b/third_party/apereo-cas/overlay/build.gradle new file mode 100644 index 000000000000..464fce48d189 --- /dev/null +++ b/third_party/apereo-cas/overlay/build.gradle @@ -0,0 +1,124 @@ +buildscript { + repositories { + if (rootProject.hasProperty("cacheLocation")) { + mavenLocal() + maven { url cacheLocation } + } else { + mavenLocal() + gradlePluginPortal() + mavenCentral() + jcenter() + maven { + url "https://repo.spring.io/libs-milestone" + mavenContent { releasesOnly() } + } + maven { + url "https://repo.spring.io/libs-snapshot" + mavenContent { snapshotsOnly() } + } + maven { + url "https://plugins.gradle.org/m2/" + mavenContent { releasesOnly() } + } + } + } + dependencies { + classpath "de.undercouch:gradle-download-task:${project.gradleDownloadTaskVersion}" + classpath "org.springframework.boot:spring-boot-gradle-plugin:${project.springBootVersion}" + classpath "gradle.plugin.com.google.cloud.tools:jib-gradle-plugin:${project.jibVersion}" + classpath "io.freefair.gradle:maven-plugin:${project.gradleMavenPluginVersion}" + classpath "io.freefair.gradle:lombok-plugin:${project.gradleLombokPluginVersion}" + } +} + +repositories { + if (rootProject.hasProperty("cacheLocation")) { + mavenLocal() + maven { url cacheLocation } + } else { + mavenLocal() + mavenCentral() + jcenter() + maven { + url "https://oss.sonatype.org/content/repositories/snapshots" + mavenContent { snapshotsOnly() } + } + maven { + mavenContent { releasesOnly() } + url "https://build.shibboleth.net/nexus/content/repositories/releases/" + } + maven { + mavenContent { releasesOnly() } + url "https://repo.spring.io/milestone/" + } + maven { + url "https://repo.spring.io/snapshot/" + mavenContent { snapshotsOnly() } + } + maven { + mavenContent { snapshotsOnly() } + url "https://oss.jfrog.org/artifactory/oss-snapshot-local" + } + } +} + +def casServerVersion = project.'cas.version' +def casWebApplicationBinaryName = "cas.war" + +project.ext."casServerVersion" = casServerVersion +project.ext."casWebApplicationBinaryName" = casWebApplicationBinaryName + +apply plugin: "io.freefair.war-overlay" +apply plugin: "io.freefair.lombok" +apply from: rootProject.file("gradle/tasks.gradle") + +apply plugin: "war" +apply plugin: "eclipse" +apply plugin: "idea" + +apply from: rootProject.file("gradle/springboot.gradle") +apply from: rootProject.file("gradle/dockerjib.gradle") + +dependencies { + // Other CAS dependencies/modules may be listed here... + implementation "org.apereo.cas:cas-server-support-ldap:${project.'cas.version'}" + implementation "org.apereo.cas:cas-server-support-json-service-registry:${project.'cas.version'}" + implementation "org.apereo.cas:cas-server-support-saml-idp:${project.'cas.version'}" + implementation "org.apereo.cas:cas-server-support-oidc:${project.'cas.version'}" +} + +tasks.findByName("jibDockerBuild") + .dependsOn(copyWebAppIntoJib, copyConfigIntoJib) + .finalizedBy(deleteWebAppFromJib) + +tasks.findByName("jib") + .dependsOn(copyWebAppIntoJib, copyConfigIntoJib) + .finalizedBy(deleteWebAppFromJib) + +configurations.all { + resolutionStrategy { + cacheChangingModulesFor 0, "seconds" + cacheDynamicVersionsFor 0, "seconds" + + preferProjectModules() + + def failIfConflict = project.hasProperty("failOnVersionConflict") && Boolean.valueOf(project.getProperty("failOnVersionConflict")) + if (failIfConflict) { + failOnVersionConflict() + } + } +} + +eclipse { + classpath { + downloadSources = true + downloadJavadoc = true + } +} + +idea { + module { + downloadJavadoc = true + downloadSources = true + } +} diff --git a/third_party/apereo-cas/overlay/docker-build.sh b/third_party/apereo-cas/overlay/docker-build.sh new file mode 100755 index 000000000000..dc098ade96d6 --- /dev/null +++ b/third_party/apereo-cas/overlay/docker-build.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +image_tag=(`cat gradle.properties | grep "cas.version" | cut -d= -f2`) + +echo "Building CAS docker image tagged as [v$image_tag]" +# read -p "Press [Enter] to continue..." any_key; + +docker build --tag="apereo/cas:v$image_tag" . \ + && echo "Built CAS image successfully tagged as apereo/cas:v$image_tag" \ + && docker images "apereo/cas:v$image_tag" \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/docker-compose.yml b/third_party/apereo-cas/overlay/docker-compose.yml new file mode 100644 index 000000000000..8f2e6ca7c9fb --- /dev/null +++ b/third_party/apereo-cas/overlay/docker-compose.yml @@ -0,0 +1,7 @@ +version: '3' +services: + cas: + build: . + ports: + - "8443:8443" + - "8080:8080" \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/docker-push.sh b/third_party/apereo-cas/overlay/docker-push.sh new file mode 100755 index 000000000000..ca7b784e6dc8 --- /dev/null +++ b/third_party/apereo-cas/overlay/docker-push.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +read -p "Docker username: " docker_user +read -s -p "Docker password: " docker_psw + +echo "$docker_psw" | docker login --username "$docker_user" --password-stdin + +image_tag=(`cat gradle.properties | grep "cas.version" | cut -d= -f2`) + +echo "Pushing CAS docker image tagged as v$image_tag to apereo/cas..." +docker push apereo/cas:"v$image_tag" \ + && echo "Pushed apereo/cas:v$image_tag successfully."; \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/docker-run.sh b/third_party/apereo-cas/overlay/docker-run.sh new file mode 100755 index 000000000000..6b4ce9755919 --- /dev/null +++ b/third_party/apereo-cas/overlay/docker-run.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +docker stop cas > /dev/null 2>&1 +docker rm cas > /dev/null 2>&1 +image_tag=(`cat gradle.properties | grep "cas.version" | cut -d= -f2`) +docker run -d -p 8080:8080 -p 8443:8443 --name="cas" apereo/cas:"v${image_tag}" +docker logs -f cas \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/etc/cas/config/cas.properties b/third_party/apereo-cas/overlay/etc/cas/config/cas.properties new file mode 100644 index 000000000000..01515613d7ac --- /dev/null +++ b/third_party/apereo-cas/overlay/etc/cas/config/cas.properties @@ -0,0 +1,39 @@ +cas.server.name=https://login.tvl.fyi +cas.server.prefix=${cas.server.name} +cas.server.scope=tvl.fyi +server.tomcat.basedir=/etc/cas/tomcat +server.servlet.context.path=/ + +cas.service-registry.json.location=file:/etc/cas/services + +server.port=8444 +server.address=127.0.0.1 +server.ssl.enabled=false + +# Enable X-Forwarded-For using Tomcat. +server.forward-headers-strategy=NATIVE +server.tomcat.remoteip.remote-ip-header=x-forwarded-for +server.tomcat.remoteip.protocol-header=x-forwarded-proto + +server.tomcat.basedir=/etc/cas/tomcat +server.servlet.context-path=/ + +cas.authn.saml-idp.entity-id=https://login.tvl.fyi + +cas.authn.accept.users= + +cas.authn.attribute-repository.default-attributes-to-release=uid,mail,displayName,username,principalLdapDn + +cas.authn.ldap[0].pool-passivator=NONE +cas.authn.ldap[0].principal-attribute-list=cn:uid,mail,displayName,cn:username +cas.authn.ldap[0].collect-dn-attribute=true +cas.authn.ldap[0].principal-dn-attribute-name=principalLdapDn +cas.authn.ldap[0].ldap-url=ldap://localhost +cas.authn.ldap[0].use-start-tls=false +cas.authn.ldap[0].validator.base-dn=dc=tvl,dc=fyi +cas.authn.ldap[0].base-dn=dc=tvl,dc=fyi +cas.authn.ldap[0].search-filter=cn={user} +cas.authn.ldap[0].dn-format=cn=%s,ou=users,dc=tvl,dc=fyi +cas.authn.ldap[0].type=DIRECT + +cas.authn.oidc.issuer=${cas.server.name}/oidc diff --git a/third_party/apereo-cas/overlay/etc/cas/config/log4j2.xml b/third_party/apereo-cas/overlay/etc/cas/config/log4j2.xml new file mode 100644 index 000000000000..3130a09f402d --- /dev/null +++ b/third_party/apereo-cas/overlay/etc/cas/config/log4j2.xml @@ -0,0 +1,84 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<Configuration monitorInterval="5" packages="org.apereo.cas.logging"> + <Properties> + <Property name="baseDir">/var/log</Property> + + <Property name="cas.log.level">info</Property> + <Property name="spring.webflow.log.level">warn</Property> + <Property name="spring.security.log.level">info</Property> + <Property name="spring.cloud.log.level">warn</Property> + <Property name="spring.boot.admin.log.level">debug</Property> + <Property name="spring.web.log.level">warn</Property> + <Property name="spring.boot.log.level">warn</Property> + <Property name="ldap.log.level">warn</Property> + <Property name="pac4j.log.level">warn</Property> + <Property name="opensaml.log.level">warn</Property> + <Property name="hazelcast.log.level">warn</Property> + </Properties> + <Appenders> + <Console name="console" target="SYSTEM_OUT"> + <PatternLayout pattern="%highlight{%d %p [%c] - <%m>}%n"/> + </Console> + + <CasAppender name="casConsole"> + <AppenderRef ref="console" /> + </CasAppender> + </Appenders> + <Loggers> + <!-- If adding a Logger with level set higher than warn, make category as selective as possible --> + <!-- Loggers inherit appenders from Root Logger unless additivity is false --> + <AsyncLogger name="org.apereo" level="${sys:cas.log.level}" includeLocation="true"/> + <AsyncLogger name="org.apereo.services.persondir" level="${sys:cas.log.level}" includeLocation="true"/> + <AsyncLogger name="org.apereo.cas.web.flow" level="${sys:cas.log.level}" includeLocation="true"/> + <AsyncLogger name="org.apereo.spring" level="${sys:cas.log.level}" includeLocation="true"/> + + <AsyncLogger name="org.apache" level="warn" /> + <AsyncLogger name="org.apache.http" level="error" /> + + <AsyncLogger name="org.springframework.boot" level="${sys:spring.boot.log.level" includeLocation="true"/> + <AsyncLogger name="org.springframework.boot.context.embedded" level="info" includeLocation="true" /> + <AsyncLogger name="org.springframework.boot.autoconfigure.security.servlet.UserDetailsServiceAutoConfiguration" + level="${sys:spring.security.log.level}" includeLocation="true" /> + <AsyncLogger name="org.springframework.boot.autoconfigure.security" level="${sys:spring.security.log.level}" includeLocation="true"/> + <AsyncLogger name="org.springframework.boot.devtools" level="off" includeLocation="true"/> + + <AsyncLogger name="org.springframework" level="warn" includeLocation="true" /> + <AsyncLogger name="org.springframework.webflow" level="${sys:spring.webflow.log.level}" includeLocation="true"/> + <AsyncLogger name="org.springframework.aop" level="warn" includeLocation="true" /> + <AsyncLogger name="org.springframework.web" level="warn" includeLocation="true"/> + <AsyncLogger name="org.springframework.session" level="warn" includeLocation="true"/> + <AsyncLogger name="org.springframework.scheduling" level="info" includeLocation="true"/> + <AsyncLogger name="org.springframework.cloud.vault" level="warn" includeLocation="true" /> + <AsyncLogger name="org.springframework.web.client" level="warn" includeLocation="true" /> + <AsyncLogger name="org.springframework.security" level="${sys:spring.security.log.level}" includeLocation="true"/> + <AsyncLogger name="org.springframework.cloud" level="${sys:spring.cloud.log.level}" includeLocation="true"/> + <AsyncLogger name="org.springframework.amqp" level="error" /> + <AsyncLogger name="org.springframework.integration" level="warn" includeLocation="true"/> + <AsyncLogger name="org.springframework.messaging" level="warn" includeLocation="true"/> + <AsyncLogger name="org.springframework.web" level="${sys:spring.web.log.level}" includeLocation="true"/> + <AsyncLogger name="org.springframework.orm.jpa" level="warn" includeLocation="true"/> + <AsyncLogger name="org.springframework.scheduling" level="warn" includeLocation="true"/> + <AsyncLogger name="org.springframework.context.annotation" level="off" includeLocation="true"/> + <AsyncLogger name="org.springframework.web.socket" level="warn" includeLocation="true"/> + + <AsyncLogger name="org.thymeleaf" level="warn" includeLocation="true"/> + + <AsyncLogger name="org.pac4j" level="${sys:pac4j.log.level}" includeLocation="true"/> + + <AsyncLogger name="org.opensaml" level="${sys:opensaml.log.level}" includeLocation="true"/> + <AsyncLogger name="PROTOCOL_MESSAGE" level="${sys:opensaml.log.level}" includeLocation="true" /> + + <AsyncLogger name="net.sf.ehcache" level="warn" includeLocation="true"/> + <AsyncLogger name="com.couchbase" level="warn" includeLocation="true"/> + <AsyncLogger name="de.codecentric" level="${sys:spring.boot.admin.log.level}" includeLocation="true"/> + <AsyncLogger name="net.jradius" level="warn" includeLocation="true" /> + <AsyncLogger name="org.openid4java" level="warn" includeLocation="true" /> + <AsyncLogger name="org.ldaptive" level="${sys:ldap.log.level}" includeLocation="true"/> + <AsyncLogger name="com.hazelcast" level="${sys:hazelcast.log.level}" includeLocation="true"/> + + <!-- All Loggers inherit appenders specified here, unless additivity="false" on the Logger --> + <AsyncRoot level="warn"> + <AppenderRef ref="casConsole"/> + </AsyncRoot> + </Loggers> +</Configuration> diff --git a/third_party/apereo-cas/overlay/etc/cas/saml/.gitkeep b/third_party/apereo-cas/overlay/etc/cas/saml/.gitkeep new file mode 100644 index 000000000000..882c99944dc5 --- /dev/null +++ b/third_party/apereo-cas/overlay/etc/cas/saml/.gitkeep @@ -0,0 +1 @@ +This directory is references in the Dockerfile so it needs to be here. \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/etc/cas/services/.donotdel b/third_party/apereo-cas/overlay/etc/cas/services/.donotdel new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/third_party/apereo-cas/overlay/etc/cas/services/.donotdel diff --git a/third_party/apereo-cas/overlay/etc/cas/services/samltest-1.json b/third_party/apereo-cas/overlay/etc/cas/services/samltest-1.json new file mode 100644 index 000000000000..37ea1be98135 --- /dev/null +++ b/third_party/apereo-cas/overlay/etc/cas/services/samltest-1.json @@ -0,0 +1,8 @@ +{ + "@class" : "org.apereo.cas.support.saml.services.SamlRegisteredService", + "serviceId" : "https://samltest.id/saml/sp", + "name" : "SAMLTest SP", + "id" : 1, + "evaluationOrder" : 10, + "metadataLocation" : "https://samltest.id/saml/sp" +} diff --git a/third_party/apereo-cas/overlay/etc/cas/thekeystore b/third_party/apereo-cas/overlay/etc/cas/thekeystore new file mode 100644 index 000000000000..78f49baf743b --- /dev/null +++ b/third_party/apereo-cas/overlay/etc/cas/thekeystore Binary files differdiff --git a/third_party/apereo-cas/overlay/gradle.properties b/third_party/apereo-cas/overlay/gradle.properties new file mode 100644 index 000000000000..f49d921e1eac --- /dev/null +++ b/third_party/apereo-cas/overlay/gradle.properties @@ -0,0 +1,28 @@ +# Versions +cas.version=6.2.0 +springBootVersion=2.2.8.RELEASE + +# Use -jetty, -undertow to other containers +# Or blank if you want to deploy to an external container +appServer=-tomcat +executable=false + +tomcatVersion=9.0.36 + +group=org.apereo.cas +sourceCompatibility=11 +targetCompatibility=11 + +jibVersion=2.4.0 + +# Location of the downloaded CAS shell JAR +shellDir=build/libs +ivyVersion=2.4.0 +gradleDownloadTaskVersion=3.4.3 +gradleMavenPluginVersion=4.1.5 +gradleLombokPluginVersion=4.1.5 + +# use without "-slim" in tag name if you want tools like jstack, adds about 100MB to image size +# (https://hub.docker.com/r/adoptopenjdk/openjdk11/tags/) +baseDockerImage=adoptopenjdk/openjdk11:alpine-jre +allowInsecureRegistries=false diff --git a/third_party/apereo-cas/overlay/gradle/dockerjib.gradle b/third_party/apereo-cas/overlay/gradle/dockerjib.gradle new file mode 100644 index 000000000000..dbadd5aa19bd --- /dev/null +++ b/third_party/apereo-cas/overlay/gradle/dockerjib.gradle @@ -0,0 +1,52 @@ +apply plugin: "com.google.cloud.tools.jib" + +jib { + from { + image = project.baseDockerImage + } + to { + image = "${project.group}/${project.name}" + /** + ecr-login: Amazon Elastic Container Registry (ECR) + gcr: Google Container Registry (GCR) + osxkeychain: Docker Hub + */ + credHelper = "osxkeychain" + /** + auth { + username = "*******" + password = "*******" + } + */ + tags = ["v" + casServerVersion] + } + container { + creationTime = "USE_CURRENT_TIMESTAMP" + entrypoint = ['docker/entrypoint.sh'] + ports = ['80', '443', '8080', '8443'] + labels = [version:casServerVersion, name:project.name, group:project.group] + } + extraDirectories { + paths = 'src/main/jib' + permissions = [ + '/docker/entrypoint.sh': '755' + ] + } + allowInsecureRegistries = project.allowInsecureRegistries +} + +task copyWebAppIntoJib(type: Copy, group: "Docker", description: "Copy the web application into Docker image") { + dependsOn build + from "build/libs/${casWebApplicationBinaryName}" + into "src/main/jib/docker/cas/war" +} + +task copyConfigIntoJib(type: Copy, group: "Docker", description: "Copy the CAS configuration into Docker image") { + dependsOn build + from "etc/cas" + into "src/main/jib/docker/cas" +} + +task deleteWebAppFromJib(type: Delete, group: "Docker", description: "Explodes the CAS web application archive") { + delete "src/main/jib/docker/cas" +} \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/gradle/springboot.gradle b/third_party/apereo-cas/overlay/gradle/springboot.gradle new file mode 100644 index 000000000000..79804b18e4a6 --- /dev/null +++ b/third_party/apereo-cas/overlay/gradle/springboot.gradle @@ -0,0 +1,101 @@ +apply plugin: "org.springframework.boot" + +configurations { + bootRunConfig.extendsFrom compileClasspath +} + +dependencies { + bootRunConfig "org.apereo.cas:cas-server-webapp-init:${casServerVersion}" + bootRunConfig "org.apereo.cas:cas-server-webapp-tomcat:${casServerVersion}" + bootRunConfig "org.springframework.boot:spring-boot-devtools:${project.springBootVersion}" +} + +sourceSets { + bootRunSources { + resources { + srcDirs new File("//etc/cas/templates/"), + new File("${project.getProjectDir()}/src/main/resources/") + } + } +} + +bootRun { + classpath = configurations.bootRunConfig + sourceSets.main.compileClasspath + sourceSets.main.runtimeClasspath + doFirst { + sourceResources sourceSets.bootRunSources + systemProperties = System.properties + } + + def list = [] + list.add("-XX:TieredStopAtLevel=1") + list.add("-Xverify:none") + list.add("--add-modules") + list.add("java.se") + list.add("--add-exports") + list.add("java.base/jdk.internal.ref=ALL-UNNAMED") + list.add("--add-opens") + list.add("java.base/java.lang=ALL-UNNAMED") + list.add("--add-opens") + list.add("java.base/java.nio=ALL-UNNAMED") + list.add("--add-opens") + list.add("java.base/sun.nio.ch=ALL-UNNAMED") + list.add("--add-opens") + list.add("java.management/sun.management=ALL-UNNAMED") + list.add("--add-opens") + list.add("jdk.management/com.sun.management.internal=ALL-UNNAMED") + + list.add("-XX:+UnlockExperimentalVMOptions") + list.add("-XX:+EnableJVMCI") + list.add("-XX:+UseJVMCICompiler") + + list.add("-Xrunjdwp:transport=dt_socket,address=5000,server=y,suspend=n") + + jvmArgs = list + + def appArgList = [] + args = appArgList +} + +springBoot { + buildInfo() + mainClassName = "org.apereo.cas.web.CasWebApplication" +} + +bootWar { + def executable = project.hasProperty("executable") && Boolean.valueOf(project.getProperty("executable")) + if (executable) { + logger.info "Including launch script for executable WAR artifact" + launchScript() + } else { + logger.info "WAR artifact is not marked as an executable" + } + archiveName "${casWebApplicationBinaryName}" + baseName "cas" + excludeDevtools = false + + entryCompression = ZipEntryCompression.STORED + /* + attachClasses = true + classesClassifier = 'classes' + archiveClasses = true + */ + overlays { + /* + https://docs.freefair.io/gradle-plugins/current/reference/#_io_freefair_war_overlay + Note: The "excludes" property is only for files in the war dependency. + If a jar is excluded from the war, it could be brought back into the final war as a dependency + of non-war dependencies. Those should be excluded via normal gradle dependency exclusions. + */ + cas { + from "org.apereo.cas:cas-server-webapp${project.appServer}:${casServerVersion}@war" + provided = false + /* + excludes = ["WEB-INF/lib/somejar-1.0*"] + enableCompilation = true + includes = ["*.xyz"] + targetPath = "sub-path/bar" + skip = false + */ + } + } +} diff --git a/third_party/apereo-cas/overlay/gradle/tasks.gradle b/third_party/apereo-cas/overlay/gradle/tasks.gradle new file mode 100644 index 000000000000..297555b1e44d --- /dev/null +++ b/third_party/apereo-cas/overlay/gradle/tasks.gradle @@ -0,0 +1,354 @@ +import org.apache.ivy.util.url.ApacheURLLister +import org.apache.tools.ant.taskdefs.condition.Os +import org.gradle.internal.logging.text.StyledTextOutputFactory + +import java.nio.file.Files +import java.nio.file.Paths +import java.nio.file.StandardCopyOption + +import static org.gradle.internal.logging.text.StyledTextOutput.Style + +buildscript { + repositories { + if (rootProject.hasProperty("cacheLocation")) { + mavenLocal() + maven { url cacheLocation } + } else { + mavenLocal() + mavenCentral() + jcenter() + maven { + url "https://oss.sonatype.org/content/repositories/snapshots" + mavenContent { snapshotsOnly() } + } + } + } + + dependencies { + classpath "org.apache.ivy:ivy:${project.ivyVersion}" + classpath "org.apereo.cas:cas-server-core-api-configuration-model:${project.'cas.version'}" + classpath "org.apereo.cas:cas-server-core-configuration-metadata-repository:${project.'cas.version'}" + } +} + +apply plugin: "de.undercouch.download" + +def tomcatDirectory = "${buildDir}/apache-tomcat-${tomcatVersion}" +project.ext."tomcatDirectory" = tomcatDirectory + +def explodedDir = "${buildDir}/cas" +def explodedResourcesDir = "${buildDir}/cas-resources" + +def resourcesJarName = "cas-server-webapp-resources" +def templateViewsJarName = "cas-server-support-thymeleaf" + +task copyCasConfiguration(type: Copy, group: "CAS", description: "Copy the CAS configuration from this project to /etc/cas/config") { + from "etc/cas/config" + into new File('/etc/cas/config').absolutePath + doFirst { + new File('/etc/cas/config').mkdirs() + } +} + +task explodeWarOnly(type: Copy, group: "CAS", description: "Explodes the CAS web application archive") { + dependsOn 'build' + from zipTree("build/libs/${casWebApplicationBinaryName}") + into explodedDir + doLast { + println "Exploded WAR into ${explodedDir}" + } +} + +task explodeWar(type: Copy, group: "CAS", description: "Explodes the CAS archive and resources jar from the CAS web application archive") { + dependsOn explodeWarOnly + from zipTree("${explodedDir}/WEB-INF/lib/${templateViewsJarName}-${casServerVersion}.jar") + into explodedResourcesDir + + from zipTree("${explodedDir}/WEB-INF/lib/${resourcesJarName}-${casServerVersion}.jar") + into explodedResourcesDir + + doLast { + println "Exploded WAR resources into ${explodedResourcesDir}" + } +} + +task run(group: "build", description: "Run the CAS web application in embedded container mode") { + dependsOn 'build' + doLast { + def casRunArgs = Arrays.asList("-server -noverify -Xmx2048M -XX:+TieredCompilation -XX:TieredStopAtLevel=1".split(" ")) + javaexec { + main = "-jar" + jvmArgs = casRunArgs + args = ["build/libs/${casWebApplicationBinaryName}"] + systemProperties = System.properties + logger.info "Started ${commandLine}" + } + } +} + +task setExecutable(group: "CAS", description: "Configure the project to run in executable mode") { + doFirst { + project.setProperty("executable", "true") + logger.info "Configuring the project as executable" + } +} + +task executable(type: Exec, group: "CAS", description: "Run the CAS web application in standalone executable mode") { + dependsOn setExecutable, 'build' + doFirst { + workingDir "." + if (!Os.isFamily(Os.FAMILY_WINDOWS)) { + commandLine "chmod", "+x", bootWar.archivePath + } + logger.info "Running ${bootWar.archivePath}" + commandLine bootWar.archivePath + } +} + +task debug(group: "CAS", description: "Debug the CAS web application in embedded mode on port 5005") { + dependsOn 'build' + doLast { + logger.info "Debugging process is started in a suspended state, listening on port 5005." + def casArgs = Arrays.asList("-Xmx2048M".split(" ")) + javaexec { + main = "-jar" + jvmArgs = casArgs + debug = true + args = ["build/libs/${casWebApplicationBinaryName}"] + systemProperties = System.properties + logger.info "Started ${commandLine}" + } + } +} + +task downloadShell(group: "Shell", description: "Download CAS shell jar from snapshot or release maven repo") { + doFirst { + mkdir "${project.shellDir}" + } + doLast { + def downloadFile + if (isRunningCasServerSnapshot(casServerVersion)) { + def snapshotDir = "https://oss.sonatype.org/content/repositories/snapshots/org/apereo/cas/cas-server-support-shell/${casServerVersion}/" + def files = new ApacheURLLister().listFiles(new URL(snapshotDir)) + files = files.sort { it.path } + files.each { + if (it.path.endsWith(".jar")) { + downloadFile = it + } + } + } else { + downloadFile = "https://repo1.maven.org/maven2/org/apereo/cas/cas-server-support-shell/${casServerVersion}/cas-server-support-shell-${casServerVersion}.jar" + } + logger.info "Downloading file: ${downloadFile}" + download { + src downloadFile + dest new File("${project.shellDir}", "cas-server-support-shell-${casServerVersion}.jar") + overwrite false + } + } +} + +task runShell(group: "Shell", description: "Run the CAS shell") { + dependsOn downloadShell + doLast { + println "Run the following command to launch the shell:\n\tjava -jar ${project.shellDir}/cas-server-support-shell-${casServerVersion}.jar" + } +} + +task debugShell(group: "Shell", description: "Run the CAS shell with debug options, wait for debugger on port 5005") { + dependsOn downloadShell + doLast { + println """ + Run the following command to launch the shell:\n\t + java -Xrunjdwp:transport=dt_socket,address=5000,server=y,suspend=y -jar ${project.shellDir}/cas-server-support-shell-${casServerVersion}.jar + """ + } +} + +task showConfiguration(group: "CAS", description: "Show configurations for each dependency, etc") { + doLast() { + def cfg = project.hasProperty("configuration") ? project.property("configuration") : "compile" + configurations.getByName(cfg).each { println it } + } +} + +task allDependenciesInsight(group: "build", type: DependencyInsightReportTask, description: "Produce insight information for all dependencies") {} + +task allDependencies(group: "build", type: DependencyReportTask, description: "Display a graph of all project dependencies") {} + +task casVersion(group: "CAS", description: "Display the current CAS version") { + doFirst { + def verbose = project.hasProperty("verbose") && Boolean.valueOf(project.getProperty("verbose")) + if (verbose) { + def out = services.get(StyledTextOutputFactory).create("CAS") + println "******************************************************************" + out.withStyle(Style.Info).println "Apereo CAS $casServerVersion" + out.withStyle(Style.Description).println "Enterprise Single SignOn for all earthlings and beyond" + out.withStyle(Style.SuccessHeader).println "- GitHub: " + out.withStyle(Style.Success).println "https://github.com/apereo/cas" + out.withStyle(Style.SuccessHeader).println "- Docs: " + out.withStyle(Style.Success).println "https://apereo.github.io/cas" + out.withStyle(Style.SuccessHeader).println "- Blog: " + out.withStyle(Style.Success).println "https://apereo.github.io" + println "******************************************************************" + } else { + println casServerVersion + } + } +} + +task createKeystore(group: "CAS", description: "Create CAS keystore") { + doFirst { + mkdir "/etc/cas" + + def keystorePath = "/etc/cas/thekeystore" + + def dn = "CN=cas.example.org,OU=Example,OU=Org,C=US" + if (project.hasProperty("certificateDn")) { + dn = project.getProperty("certificateDn") + } + def subjectAltName = "dns:example.org,dns:localhost,ip:127.0.0.1" + if (project.hasProperty("certificateSubAltName")) { + subjectAltName = project.getProperty("certificateSubAltName") + } + // this will fail if thekeystore exists and has cert with cas alias already (so delete if you want to recreate) + logger.info "Generating keystore for CAS with DN ${dn}" + exec { + workingDir "." + commandLine "keytool", "-genkeypair", "-alias", "cas", + "-keyalg", "RSA", + "-keypass", "changeit", "-storepass", "changeit", + "-keystore", keystorePath, + "-dname", dn, "-ext", "SAN=${subjectAltName}" + } + logger.info "Exporting cert from keystore..." + exec { + workingDir "." + commandLine "keytool", "-exportcert", "-alias", "cas", + "-storepass", "changeit", "-keystore", keystorePath, + "-file", "/etc/cas/cas.cer" + } + logger.info "Import /etc/cas/cas.cer into your Java truststore (JAVA_HOME/lib/security/cacerts)" + } +} + +task listTemplateViews(group: "CAS", description: "List all CAS views") { + dependsOn explodeWar + + doFirst { + fileTree(explodedResourcesDir).matching { + include "**/*.html" + } + .collect { + return it.path.replace(explodedResourcesDir, "") + } + .toSorted() + .each { println it } + } +} + +task exportConfigMetadata(group: "CAS", description: "Export collection of CAS properties") { + doLast { + def file = new File(project.rootDir, 'config-metadata.properties') + file.withWriter('utf-8') { writer -> + def metadataRepository = new org.apereo.cas.metadata.CasConfigurationMetadataRepository() + def repository = metadataRepository.repository; + repository.allGroups + .values() + .sort { o1, o2 -> o1.id <=> o2.id } + .each({ group -> + def groupProperties = group.properties + if (!groupProperties.isEmpty()) { + def groupId = group.id.equalsIgnoreCase("_ROOT_GROUP_") ? "" : group.id + "." + + writer.writeLine("# Group ${group.id}"); + writer.writeLine("# ====================") + groupProperties + .values() + .sort { o1, o2 -> o1.id <=> o2.id } + .each({ property -> + def description = property.shortDescription + if (!property.shortDescription?.equalsIgnoreCase(property.description) && property.description != null) { + description = property.description.replace('\n', '#') + } + writer.writeLine("# ${description}"); + writer.writeLine("# Type: ${property.type}"); + if (property.deprecated) { + def deprecation = property.deprecation + writer.writeLine("# This setting is deprecated with a severity level of ${deprecation.level}.") + if (deprecation.shortReason != null) { + writer.writeLine("# because ${deprecation.shortReason}") + } + if (deprecation.replacement != null) { + writer.writeLine("# Replace with: ${deprecation.replacement}") + } + } + property.hints.valueHints.each { + if (it.value instanceof Object[]) { + if (it.value[0].toString().contains("RequiresModule")) { + writer.writeLine("# Required module: org.apereo.cas:${it.description}") + writer.writeLine("# Automatically included/available: ${it.value[1]}") + } + } + if (it.value.toString().contains("RequiredProperty")) { + writer.writeLine("# Note: This setting is required!") + } + } + writer.writeLine("${groupId}${property.name}=${property.defaultValue}") + writer.writeLine("") + }); + } + }); + } + println "Configuration metadata is available at ${file.absolutePath}" + } +} + +task getResource(group: "CAS", description: "Fetch a CAS resource and move it into the overlay") { + dependsOn explodeWar + + doFirst { + def resourceName = project.getProperty("resourceName") + + def results = fileTree(explodedResourcesDir).matching { + include "**/${resourceName}.*" + include "**/${resourceName}" + } + if (results.isEmpty()) { + println "No resources could be found matching ${resourceName}" + return + } + if (results.size() > 1) { + println "Multiple resources found matching ${resourceName}:\n" + results.each { + println "\t-" + it.path.replace(explodedResourcesDir, "") + } + println "\nNarrow down your search criteria and try again." + return + } + + def fromFile = explodedResourcesDir + def resourcesDir = "src/main/resources" + mkdir resourcesDir + + def resourceFile = results[0].canonicalPath + def toResourceFile = resourceFile.replace(fromFile, resourcesDir) + + def parent = file(toResourceFile).getParent() + mkdir parent + + Files.copy(Paths.get(resourceFile), Paths.get(toResourceFile), StandardCopyOption.REPLACE_EXISTING) + println "Copied file ${resourceFile} to ${toResourceFile}" + } +} + +def isRunningCasServerSnapshot(casServerVersion) { + return "${casServerVersion}".contains("-SNAPSHOT") +} + +task verifyRequiredJavaVersion { + logger.info "Checking current Java version ${JavaVersion.current()} for required Java version ${project.targetCompatibility}" + if (!JavaVersion.current().name.equalsIgnoreCase("${project.targetCompatibility}")) { + throw new GradleException("Current Java version ${JavaVersion.current()} does not match required Java version ${project.targetCompatibility}") + } +} diff --git a/third_party/apereo-cas/overlay/gradle/wrapper/gradle-wrapper.jar b/third_party/apereo-cas/overlay/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000000..62d4c053550b --- /dev/null +++ b/third_party/apereo-cas/overlay/gradle/wrapper/gradle-wrapper.jar Binary files differdiff --git a/third_party/apereo-cas/overlay/gradle/wrapper/gradle-wrapper.properties b/third_party/apereo-cas/overlay/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000000..4c5803d13ca5 --- /dev/null +++ b/third_party/apereo-cas/overlay/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/third_party/apereo-cas/overlay/gradlew b/third_party/apereo-cas/overlay/gradlew new file mode 100755 index 000000000000..fbd7c515832d --- /dev/null +++ b/third_party/apereo-cas/overlay/gradlew @@ -0,0 +1,185 @@ +#!/usr/bin/env sh + +# +# Copyright 2015 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin or MSYS, switch paths to Windows format before running java +if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=`expr $i + 1` + done + case $i in + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=`save "$@"` + +# Collect all arguments for the java command, following the shell quoting and substitution rules +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +exec "$JAVACMD" "$@" diff --git a/third_party/apereo-cas/overlay/gradlew.bat b/third_party/apereo-cas/overlay/gradlew.bat new file mode 100644 index 000000000000..5093609d512a --- /dev/null +++ b/third_party/apereo-cas/overlay/gradlew.bat @@ -0,0 +1,104 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/third_party/apereo-cas/overlay/lombok.config b/third_party/apereo-cas/overlay/lombok.config new file mode 100644 index 000000000000..6aa51d71ec39 --- /dev/null +++ b/third_party/apereo-cas/overlay/lombok.config @@ -0,0 +1,2 @@ +# This file is generated by the 'io.freefair.lombok' Gradle plugin +config.stopBubbling = true diff --git a/third_party/apereo-cas/overlay/settings.gradle b/third_party/apereo-cas/overlay/settings.gradle new file mode 100644 index 000000000000..3ad50900ea59 --- /dev/null +++ b/third_party/apereo-cas/overlay/settings.gradle @@ -0,0 +1 @@ +rootProject.name='cas' \ No newline at end of file diff --git a/third_party/apereo-cas/overlay/src/main/jib/docker/entrypoint.sh b/third_party/apereo-cas/overlay/src/main/jib/docker/entrypoint.sh new file mode 100755 index 000000000000..a3a0895b04df --- /dev/null +++ b/third_party/apereo-cas/overlay/src/main/jib/docker/entrypoint.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +#echo -e "\nChecking java..." +#java -version + +#echo -e "\nCreating CAS configuration directories..." +mkdir -p /etc/cas/config +mkdir -p /etc/cas/services + +#echo "Listing provided CAS docker artifacts..." +#ls -R docker/cas + +#echo -e "\nMoving CAS configuration artifacts..." +mv docker/cas/thekeystore /etc/cas 2>/dev/null +mv docker/cas/config/*.* /etc/cas/config 2>/dev/null +mv docker/cas/services/*.* /etc/cas/services 2>/dev/null + +#echo -e "\nListing CAS configuration under /etc/cas..." +#ls -R /etc/cas + +echo -e "\nRunning CAS..." +exec java -Xms512m -Xmx2048M -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -jar docker/cas/war/cas.war diff --git a/third_party/apereo-cas/overlay/src/main/webapp/WEB-INF/web.xml b/third_party/apereo-cas/overlay/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 000000000000..072a6a0fdb42 --- /dev/null +++ b/third_party/apereo-cas/overlay/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?> +<web-app version="3.1" + metadata-complete="true" + xmlns="http://xmlns.jcp.org/xml/ns/javaee" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd"> +</web-app> \ No newline at end of file diff --git a/third_party/bat_syntaxes/Prolog.sublime-syntax b/third_party/bat_syntaxes/Prolog.sublime-syntax new file mode 100644 index 000000000000..b03066ac060f --- /dev/null +++ b/third_party/bat_syntaxes/Prolog.sublime-syntax @@ -0,0 +1,1319 @@ +# SPDX-License-Identifier: MIT +# Generated from code at https://github.com/BenjaminSchaaf/swi-prolog-sublime-syntax +--- +# http://www.sublimetext.com/docs/3/syntax.html +name: Prolog +file_extensions: + - pl + - pro +first_line_match: '^#!.*\bswipl\b' +scope: source.prolog +contexts: + atom-entity|meta: + - meta_content_scope: entity.name.predicate.prolog + - match: '' + pop: true + atom-functor|meta: + - meta_content_scope: meta.path.prolog variable.function.functor.prolog + - match: '' + pop: true + atom-string|0: + - meta_include_prototype: false + - match: '\\([abcefnrstv''\"`\n\\]|x\h\h+\\?|u\h{4}|U\h{8})' + scope: constant.character.escape.prolog + - match: '''' + pop: true + compound-term|0: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [compound-term|1, value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [compound-term|1, value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [compound-term|1, value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [compound-term|1, value-without-comma|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [compound-term|1, value-without-comma|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [compound-term|1, value-without-comma|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [compound-term|1, value-without-comma|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [compound-term|1, value-without-comma|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [compound-term|1, value-without-comma|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [compound-term|1, value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [compound-term|1, value-without-comma|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [compound-term|1, value-without-comma|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [compound-term|1, value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [compound-term|1, value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [compound-term|1, value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [compound-term|1, value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [compound-term|1, value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [compound-term|1, value-without-comma|0, single-value|2] + - match: '\)' + scope: punctuation.section.parens.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + compound-term|1: + - match: ',' + scope: punctuation.separator.sequence.prolog + push: compound-term|2 + - match: '\)' + scope: punctuation.section.parens.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + compound-term|2: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: value-without-comma|0 + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: value-without-comma|0 + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: value-without-comma|0 + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: value-without-comma|0 + - match: '_' + scope: language.constant.underscore.prolog + set: value-without-comma|0 + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [value-without-comma|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + fact|0: + - match: ':-' + scope: keyword.operator.definition.begin.prolog + set: fact|1 + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [fact|2, value|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [fact|2, value|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [fact|2, value|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [fact|2, value|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [fact|2, value|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [fact|2, value|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [fact|2, value|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [fact|2, value|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [fact|2, value|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [fact|2, value|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [fact|2, value|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [fact|2, value|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [fact|2, value|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [fact|2, value|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [fact|2, value|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [fact|2, value|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [fact|2, value|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [fact|2, value|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + fact|1: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [fact|2, value|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [fact|2, value|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [fact|2, value|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [fact|2, value|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [fact|2, value|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [fact|2, value|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [fact|2, value|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [fact|2, value|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [fact|2, value|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [fact|2, value|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [fact|2, value|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [fact|2, value|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [fact|2, value|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [fact|2, value|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [fact|2, value|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [fact|2, value|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [fact|2, value|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [fact|2, value|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + fact|2: + - match: '\.' + scope: keyword.operator.definition.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + list|0: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [list|1, value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [list|1, value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [list|1, value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [list|1, value-without-comma|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [list|1, value-without-comma|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [list|1, value-without-comma|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [list|1, value-without-comma|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [list|1, value-without-comma|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [list|1, value-without-comma|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [list|1, value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [list|1, value-without-comma|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [list|1, value-without-comma|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [list|1, value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [list|1, value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [list|1, value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [list|1, value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [list|1, value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [list|1, value-without-comma|0, single-value|2] + - match: '\]' + scope: punctuation.section.brackets.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + list|1: + - match: ',' + scope: punctuation.separator.sequence.prolog + push: list|2 + - match: '\|' + scope: punctuation.separator.sequence.prolog + set: list|3 + - match: '\]' + scope: punctuation.section.brackets.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + list|2: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: value-without-comma|0 + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: value-without-comma|0 + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: value-without-comma|0 + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: value-without-comma|0 + - match: '_' + scope: language.constant.underscore.prolog + set: value-without-comma|0 + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [value-without-comma|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + list|3: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [list|4, value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [list|4, value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [list|4, value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [list|4, value-without-comma|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [list|4, value-without-comma|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [list|4, value-without-comma|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [list|4, value-without-comma|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [list|4, value-without-comma|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [list|4, value-without-comma|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [list|4, value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [list|4, value-without-comma|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [list|4, value-without-comma|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [list|4, value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [list|4, value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [list|4, value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [list|4, value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [list|4, value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [list|4, value-without-comma|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + list|4: + - match: '\]' + scope: punctuation.section.brackets.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + main: + - match: '^#!' + scope: comment.line.number-sign.prolog punctuation.definition.comment.number-sign.prolog + push: shebang|0 + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: entity.name.predicate.prolog + push: rule|0 + - match: '''' + scope: entity.name.predicate.prolog + push: [rule|0, atom-entity|meta, atom-string|0] + - match: '(?=\S)' + push: fact|0 + - match: '\S' + scope: invalid.illegal.prolog + nested-comment|0: + - meta_content_scope: comment.block.nested.prolog + - match: '/\*(\*(?!/))?' + scope: comment.block.nested.prolog punctuation.definition.comment.prolog + push: nested-comment|0 + - match: '\*/' + scope: comment.block.nested.prolog punctuation.definition.comment.prolog + pop: true + number|0: + - match: '[0-9_]+' + scope: constant.numeric.integer.prolog + - match: '(?=\S)' + pop: true + operator-compound-term|0: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [operator-compound-term|1, value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [operator-compound-term|1, value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [operator-compound-term|1, value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [operator-compound-term|1, value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [operator-compound-term|1, value-without-comma|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [operator-compound-term|1, value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [operator-compound-term|1, value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [operator-compound-term|1, value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [operator-compound-term|1, value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [operator-compound-term|1, value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [operator-compound-term|1, value-without-comma|0, single-value|2] + - match: '\)' + scope: punctuation.section.parans.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + operator-compound-term|1: + - match: ',' + scope: punctuation.separator.sequence.prolog + push: operator-compound-term|2 + - match: '\)' + scope: punctuation.section.parans.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + operator-compound-term|2: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: value-without-comma|0 + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: value-without-comma|0 + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: value-without-comma|0 + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: value-without-comma|0 + - match: '_' + scope: language.constant.underscore.prolog + set: value-without-comma|0 + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [value-without-comma|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + prototype: + - match: '(%+).*\n?' + scope: comment.line.percentage.prolog + captures: + 1: punctuation.definition.comment.prolog + - match: '/\*(\*(?!/))?' + scope: comment.block.nested.prolog punctuation.definition.comment.prolog + push: nested-comment|0 + rule|0: + - match: '\(' + scope: punctuation.section.parens.begin.prolog + set: [rule|1, compound-term|0] + - match: ':-' + scope: keyword.operator.definition.begin.prolog + set: rule|2 + - match: '\.' + scope: keyword.operator.definition.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + rule|1: + - match: ':-' + scope: keyword.operator.definition.begin.prolog + set: rule|2 + - match: '\.' + scope: keyword.operator.definition.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + rule|2: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [rule|3, value|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [rule|3, value|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [rule|3, value|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [rule|3, value|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [rule|3, value|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [rule|3, value|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [rule|3, value|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [rule|3, value|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [rule|3, value|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [rule|3, value|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [rule|3, value|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [rule|3, value|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [rule|3, value|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [rule|3, value|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [rule|3, value|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [rule|3, value|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [rule|3, value|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [rule|3, value|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + rule|3: + - match: '\.' + scope: keyword.operator.definition.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + set|0: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [set|1, value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [set|1, value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [set|1, value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [set|1, value-without-comma|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [set|1, value-without-comma|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [set|1, value-without-comma|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [set|1, value-without-comma|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [set|1, value-without-comma|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [set|1, value-without-comma|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [set|1, value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [set|1, value-without-comma|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [set|1, value-without-comma|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [set|1, value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [set|1, value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [set|1, value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [set|1, value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [set|1, value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [set|1, value-without-comma|0, single-value|2] + - match: '\}' + scope: punctuation.section.braces.begin.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + set|1: + - match: ',' + scope: punctuation.separator.sequence.prolog + push: set|2 + - match: '\}' + scope: punctuation.section.braces.begin.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + set|2: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [value-without-comma|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [value-without-comma|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: value-without-comma|0 + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: value-without-comma|0 + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: value-without-comma|0 + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: value-without-comma|0 + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [value-without-comma|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: value-without-comma|0 + - match: '_' + scope: language.constant.underscore.prolog + set: value-without-comma|0 + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [value-without-comma|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [value-without-comma|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [value-without-comma|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [value-without-comma|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [value-without-comma|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [value-without-comma|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + shebang|0: + - meta_content_scope: comment.line.number-sign.prolog + - match: '$\n?' + scope: comment.line.number-sign.prolog + pop: true + single-value|0: + - match: '\(' + scope: punctuation.section.parens.begin.prolog + set: compound-term|0 + - match: '(?=\S)' + pop: true + single-value|1: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: single-value|0 + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: operator-compound-term|0 + - match: '!' + scope: keyword.control.cut.prolog + pop: true + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + pop: true + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + pop: true + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: number|0 + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + pop: true + - match: '_' + scope: language.constant.underscore.prolog + pop: true + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: string|0 + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: list|0 + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: set|0 + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: single-value|1 + - match: '\\\+' + scope: keyword.control.negation.prolog + set: single-value|1 + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: single-value|2 + - match: '\S' + scope: invalid.illegal.prolog + pop: true + single-value|2: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: [single-value|3, value|0, single-value|0] + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [single-value|3, value|0, single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: [single-value|3, value|0, operator-compound-term|0] + - match: '!' + scope: keyword.control.cut.prolog + set: [single-value|3, value|0] + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [single-value|3, value|0] + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [single-value|3, value|0] + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + set: [single-value|3, value|0] + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + set: [single-value|3, value|0] + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + set: [single-value|3, value|0] + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: [single-value|3, value|0, number|0] + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + set: [single-value|3, value|0] + - match: '_' + scope: language.constant.underscore.prolog + set: [single-value|3, value|0] + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: [single-value|3, value|0, string|0] + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: [single-value|3, value|0, list|0] + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: [single-value|3, value|0, set|0] + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: [single-value|3, value|0, single-value|1] + - match: '\\\+' + scope: keyword.control.negation.prolog + set: [single-value|3, value|0, single-value|1] + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: [single-value|3, value|0, single-value|2] + - match: '\S' + scope: invalid.illegal.prolog + pop: true + single-value|3: + - match: '\)' + scope: punctuation.section.group.end.prolog + pop: true + - match: '\S' + scope: invalid.illegal.prolog + pop: true + string|0: + - meta_content_scope: meta.string.prolog string.quoted.double.prolog + - meta_include_prototype: false + - match: '\\([abcefnrstv''\"`\n\\]|x\h\h+\\?|u\h{4}|U\h{8})' + scope: constant.character.escape.prolog + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.end.prolog + pop: true + value-without-comma|0: + - match: '\bis\b|>>|\^|=\.\.|=?<|>=?|==?|\*\*?|\+|->?|/|#=|\\=' + scope: keyword.operator.prolog + push: value-without-comma|1 + - match: ';' + scope: keyword.operator.logical.or.prolog + push: value-without-comma|1 + - match: '->' + scope: keyword.operator.logical.if.prolog + push: value-without-comma|1 + - match: '(?=\S)' + pop: true + value-without-comma|1: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: single-value|0 + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: operator-compound-term|0 + - match: '!' + scope: keyword.control.cut.prolog + pop: true + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + pop: true + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + pop: true + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: number|0 + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + pop: true + - match: '_' + scope: language.constant.underscore.prolog + pop: true + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: string|0 + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: list|0 + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: set|0 + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: single-value|1 + - match: '\\\+' + scope: keyword.control.negation.prolog + set: single-value|1 + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: single-value|2 + - match: '\S' + scope: invalid.illegal.prolog + pop: true + value|0: + - match: '\bis\b|>>|\^|=\.\.|=?<|>=?|==?|\*\*?|\+|->?|/|#=|\\=' + scope: keyword.operator.prolog + push: value|1 + - match: ';' + scope: keyword.operator.logical.or.prolog + push: value|1 + - match: '->' + scope: keyword.operator.logical.if.prolog + push: value|1 + - match: ',' + scope: keyword.operator.logical.and.prolog + push: value|1 + - match: '(?=\S)' + pop: true + value|1: + - match: '\b[a-z][[:alpha:]0-9_]*\b' + scope: meta.path.prolog variable.function.functor.prolog + set: single-value|0 + - match: '''' + scope: meta.path.prolog variable.function.functor.prolog + set: [single-value|0, atom-functor|meta, atom-string|0] + - match: '([~^&*\-+=|\\/<>][~^&*\-+=|\\/<>.,]*)(\()' + captures: + 1: constant.character.swi-prolog.prolog + 2: punctuation.section.parens.begin.prolog + set: operator-compound-term|0 + - match: '!' + scope: keyword.control.cut.prolog + pop: true + - match: '(0b)[01_]+' + scope: constant.numeric.integer.binary.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '(0x)[\h_]+' + scope: constant.numeric.integer.hexadecimal.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '(0o)[0-7_]+' + scope: constant.numeric.integer.octal.prolog + captures: + 1: punctuation.definition.number.base.prolog + pop: true + - match: '([0-9]{1,2})('')[0-9a-z]+' + scope: constant.numeric.integer.prolog + captures: + 1: punctuation.definition.number.base.prolog + 2: punctuation.separator.base.prolog + pop: true + - match: '[0-9]+\.[0-9]+' + scope: constant.numeric.float.prolog + pop: true + - match: '[+-]?[0-9_]+' + scope: constant.numeric.integer.prolog + set: number|0 + - match: '\b[A-Z][[:alpha:]0-9_]*\b|\b_[[:alpha:]0-9_]+\b' + scope: variable.parameter.prolog + pop: true + - match: '_' + scope: language.constant.underscore.prolog + pop: true + - match: '"' + scope: meta.string.prolog string.quoted.double.prolog punctuation.definition.string.begin.prolog + set: string|0 + - match: '\[' + scope: punctuation.section.brackets.begin.prolog + set: list|0 + - match: '\{' + scope: punctuation.section.braces.begin.prolog + set: set|0 + - match: '\+|-' + scope: keyword.operator.arithmetic.prolog + set: single-value|1 + - match: '\\\+' + scope: keyword.control.negation.prolog + set: single-value|1 + - match: '\(' + scope: punctuation.section.group.begin.prolog + set: single-value|2 + - match: '\S' + scope: invalid.illegal.prolog + pop: true diff --git a/third_party/bat_syntaxes/default.nix b/third_party/bat_syntaxes/default.nix new file mode 100644 index 000000000000..2b7d025fae57 --- /dev/null +++ b/third_party/bat_syntaxes/default.nix @@ -0,0 +1,17 @@ +# For depot projects that make use of syntect (primarily +# //tools/cheddar) the included syntax set is taken from bat. +# +# However, bat lacks some of the syntaxes we are interested in. This +# package creates a new binary syntax set which bundles our additional +# syntaxes on top of bat's existing ones. +{ pkgs, ... }: + +let + inherit (pkgs) bat runCommandNoCC; +in runCommandNoCC "bat-syntaxes.bin" {} '' + export HOME=$PWD + mkdir -p .config/bat/syntaxes + cp ${./Prolog.sublime-syntax} .config/bat/syntaxes + ${bat}/bin/bat cache --build + mv .cache/bat/syntaxes.bin $out +'' diff --git a/third_party/bufbuild/default.nix b/third_party/bufbuild/default.nix new file mode 100644 index 000000000000..12683b1062a8 --- /dev/null +++ b/third_party/bufbuild/default.nix @@ -0,0 +1,29 @@ +# buf.build is a Protobuf linter and breaking change detector. +# Several binaries are produced. +{ pkgs, lib, ... }: + +pkgs.buildGoModule { + pname = "buf"; + version = "v0.20.1"; + vendorSha256 = "1gg5c7aiqb4w1zxwsraxxpln33xkmkzlp1h69xgi9i08zvrfipqs"; + + src = pkgs.fetchFromGitHub { + owner = "bufbuild"; + repo = "buf"; + rev = "5e8bf4c800de911764ffdf8d2188b7f6f54476e4"; + sha256 = "1rni5swfnb4sbrd9rls4mc3902xhqrlsja96lfcdfjzx08g6kg20"; + }; + + doCheck = false; + + # TODO(riking): postinstall produce shell completions for bash, fish, zsh + # bin/buf bash-completion + # bin/buf zsh-completion + # # bin/buf manpages # not yet functional + + meta = with lib; { + description = "Protobuf linter and breaking change detector"; + homepage = "https://buf.build/docs/introduction"; + license = licenses.asl20; + }; +} diff --git a/third_party/buzz/default.nix b/third_party/buzz/default.nix new file mode 100644 index 000000000000..2af9df912e94 --- /dev/null +++ b/third_party/buzz/default.nix @@ -0,0 +1,30 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + src = pkgs.fetchFromGitHub { + owner = "jonhoo"; + repo = "buzz"; + rev = "02479643ed1b0325050245dbb3b70411b8cffb7a"; + sha256 = "1spklfv02qlinlail5rmhh1c4926gyrkr2ydd9g6z919rxkl0ywk"; + }; + + buildInputs = with pkgs; [ + pkgconfig + dbus_libs + glib + openssl + cairo + pango + atk + gdk-pixbuf + gtk3 + dbus-glib + libappindicator-gtk3 + llvmPackages.llvm + llvmPackages.bintools + llvmPackages.clang + llvmPackages.libclang + ]; + + LIBCLANG_PATH = "${pkgs.llvmPackages.libclang.lib}/lib/libclang.so"; +} diff --git a/third_party/cgit/.gitignore b/third_party/cgit/.gitignore new file mode 100644 index 000000000000..661df346c24f --- /dev/null +++ b/third_party/cgit/.gitignore @@ -0,0 +1,12 @@ +# Files I don't care to see in git-status/commit +/cgit +cgit.conf +CGIT-CFLAGS +VERSION +cgitrc.5 +cgitrc.5.fo +cgitrc.5.html +cgitrc.5.pdf +cgitrc.5.xml +*.o +*.d diff --git a/third_party/cgit/.mailmap b/third_party/cgit/.mailmap new file mode 100644 index 000000000000..03b54796cf83 --- /dev/null +++ b/third_party/cgit/.mailmap @@ -0,0 +1,10 @@ +Florian Pritz <bluewind@xinu.at> <bluewind@xssn.at> +Harley Laue <losinggeneration@gmail.com> <losinggeneration@aim.com> +John Keeping <john@keeping.me.uk> <john@metanate.com> +Lars Hjemli <hjemli@gmail.com> <larsh@hal-2004.(none)> +Lars Hjemli <hjemli@gmail.com> <larsh@hatman.(none)> +Lars Hjemli <hjemli@gmail.com> <larsh@slackbox.hjemli.net> +Lars Hjemli <hjemli@gmail.com> <larsh@slaptop.hjemli.net> +Lukas Fleischer <lfleischer@lfos.de> <cgit@cryptocrack.de> +Lukas Fleischer <lfleischer@lfos.de> <info@cryptocrack.de> +Stefan Bühler <source@stbuehler.de> <lighttpd@stbuehler.de> diff --git a/third_party/cgit/.skip-subtree b/third_party/cgit/.skip-subtree new file mode 100644 index 000000000000..c108a7d34f2b --- /dev/null +++ b/third_party/cgit/.skip-subtree @@ -0,0 +1 @@ +Subtrees of this directory belong to cgit (third-party). diff --git a/third_party/cgit/AUTHORS b/third_party/cgit/AUTHORS new file mode 100644 index 000000000000..031de338f904 --- /dev/null +++ b/third_party/cgit/AUTHORS @@ -0,0 +1,13 @@ +Maintainer: + Jason A. Donenfeld <Jason@zx2c4.com> + +Contributors: + Jason A. Donenfeld <Jason@zx2c4.com> + Lukas Fleischer <cgit@cryptocrack.de> + Johan Herland <johan@herland.net> + Lars Hjemli <hjemli@gmail.com> + Ferry Huberts <ferry.huberts@pelagic.nl> + John Keeping <john@keeping.me.uk> + +Previous Maintainer: + Lars Hjemli <hjemli@gmail.com> diff --git a/third_party/cgit/COPYING b/third_party/cgit/COPYING new file mode 100644 index 000000000000..d159169d1050 --- /dev/null +++ b/third_party/cgit/COPYING @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + <signature of Ty Coon>, 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/third_party/cgit/Makefile b/third_party/cgit/Makefile new file mode 100644 index 000000000000..9153a393f770 --- /dev/null +++ b/third_party/cgit/Makefile @@ -0,0 +1,170 @@ +all:: + +CGIT_VERSION = v1.2.3 +CGIT_SCRIPT_NAME = cgit.cgi +CGIT_SCRIPT_PATH = /var/www/htdocs/cgit +CGIT_DATA_PATH = $(CGIT_SCRIPT_PATH) +CGIT_CONFIG = /etc/cgitrc +CACHE_ROOT = /var/cache/cgit +prefix = /usr/local +libdir = $(prefix)/lib +filterdir = $(libdir)/cgit/filters +docdir = $(prefix)/share/doc/cgit +htmldir = $(docdir) +pdfdir = $(docdir) +mandir = $(prefix)/share/man +SHA1_HEADER = <openssl/sha.h> +GIT_VER = 2.34.0 +GIT_URL = https://www.kernel.org/pub/software/scm/git/git-$(GIT_VER).tar.xz +INSTALL = install +COPYTREE = cp -r +MAN5_TXT = $(wildcard *.5.txt) +MAN_TXT = $(MAN5_TXT) +DOC_MAN5 = $(patsubst %.txt,%,$(MAN5_TXT)) +DOC_HTML = $(patsubst %.txt,%.html,$(MAN_TXT)) +DOC_PDF = $(patsubst %.txt,%.pdf,$(MAN_TXT)) + +ASCIIDOC = asciidoc +ASCIIDOC_EXTRA = +ASCIIDOC_HTML = xhtml11 +ASCIIDOC_COMMON = $(ASCIIDOC) $(ASCIIDOC_EXTRA) +TXT_TO_HTML = $(ASCIIDOC_COMMON) -b $(ASCIIDOC_HTML) + +# Define NO_C99_FORMAT if your formatted IO functions (printf/scanf et.al.) +# do not support the 'size specifiers' introduced by C99, namely ll, hh, +# j, z, t. (representing long long int, char, intmax_t, size_t, ptrdiff_t). +# some C compilers supported these specifiers prior to C99 as an extension. +# +# Define HAVE_LINUX_SENDFILE to use sendfile() + +#-include config.mak + +-include git/config.mak.uname +# +# Let the user override the above settings. +# +-include cgit.conf + +export CGIT_VERSION CGIT_SCRIPT_NAME CGIT_SCRIPT_PATH CGIT_DATA_PATH CGIT_CONFIG CACHE_ROOT + +# +# Define a way to invoke make in subdirs quietly, shamelessly ripped +# from git.git +# +QUIET_SUBDIR0 = +$(MAKE) -C # space to separate -C and subdir +QUIET_SUBDIR1 = + +ifneq ($(findstring w,$(MAKEFLAGS)),w) +PRINT_DIR = --no-print-directory +else # "make -w" +NO_SUBDIR = : +endif + +ifndef V + QUIET_SUBDIR0 = +@subdir= + QUIET_SUBDIR1 = ;$(NO_SUBDIR) echo ' ' SUBDIR $$subdir; \ + $(MAKE) $(PRINT_DIR) -C $$subdir + QUIET_TAGS = @echo ' ' TAGS $@; + export V +endif + +.SUFFIXES: + +all:: cgit + +cgit: + $(QUIET_SUBDIR0)git $(QUIET_SUBDIR1) -f ../cgit.mk ../cgit $(EXTRA_GIT_TARGETS) NO_CURL=1 + +sparse: + $(QUIET_SUBDIR0)git $(QUIET_SUBDIR1) -f ../cgit.mk NO_CURL=1 cgit-sparse + +test: + @$(MAKE) --no-print-directory cgit EXTRA_GIT_TARGETS=all + $(QUIET_SUBDIR0)tests $(QUIET_SUBDIR1) all + +install: all + $(INSTALL) -m 0755 -d $(DESTDIR)$(CGIT_SCRIPT_PATH) + $(INSTALL) -m 0755 cgit $(DESTDIR)$(CGIT_SCRIPT_PATH)/$(CGIT_SCRIPT_NAME) + $(INSTALL) -m 0755 -d $(DESTDIR)$(CGIT_DATA_PATH) + $(INSTALL) -m 0644 cgit.css $(DESTDIR)$(CGIT_DATA_PATH)/cgit.css + $(INSTALL) -m 0644 cgit.png $(DESTDIR)$(CGIT_DATA_PATH)/cgit.png + $(INSTALL) -m 0644 favicon.ico $(DESTDIR)$(CGIT_DATA_PATH)/favicon.ico + $(INSTALL) -m 0644 robots.txt $(DESTDIR)$(CGIT_DATA_PATH)/robots.txt + $(INSTALL) -m 0755 -d $(DESTDIR)$(filterdir) + $(COPYTREE) filters/* $(DESTDIR)$(filterdir) + +install-doc: install-man install-html install-pdf + +install-man: doc-man + $(INSTALL) -m 0755 -d $(DESTDIR)$(mandir)/man5 + $(INSTALL) -m 0644 $(DOC_MAN5) $(DESTDIR)$(mandir)/man5 + +install-html: doc-html + $(INSTALL) -m 0755 -d $(DESTDIR)$(htmldir) + $(INSTALL) -m 0644 $(DOC_HTML) $(DESTDIR)$(htmldir) + +install-pdf: doc-pdf + $(INSTALL) -m 0755 -d $(DESTDIR)$(pdfdir) + $(INSTALL) -m 0644 $(DOC_PDF) $(DESTDIR)$(pdfdir) + +uninstall: + rm -f $(DESTDIR)$(CGIT_SCRIPT_PATH)/$(CGIT_SCRIPT_NAME) + rm -f $(DESTDIR)$(CGIT_DATA_PATH)/cgit.css + rm -f $(DESTDIR)$(CGIT_DATA_PATH)/cgit.png + rm -f $(DESTDIR)$(CGIT_DATA_PATH)/favicon.ico + +uninstall-doc: uninstall-man uninstall-html uninstall-pdf + +uninstall-man: + @for i in $(DOC_MAN5); do \ + rm -fv $(DESTDIR)$(mandir)/man5/$$i; \ + done + +uninstall-html: + @for i in $(DOC_HTML); do \ + rm -fv $(DESTDIR)$(htmldir)/$$i; \ + done + +uninstall-pdf: + @for i in $(DOC_PDF); do \ + rm -fv $(DESTDIR)$(pdfdir)/$$i; \ + done + +doc: doc-man doc-html doc-pdf +doc-man: doc-man5 +doc-man5: $(DOC_MAN5) +doc-html: $(DOC_HTML) +doc-pdf: $(DOC_PDF) + +%.5 : %.5.txt + a2x -f manpage $< + +$(DOC_HTML): %.html : %.txt + $(TXT_TO_HTML) -o $@+ $< && \ + mv $@+ $@ + +$(DOC_PDF): %.pdf : %.txt + a2x -f pdf cgitrc.5.txt + +clean: clean-doc + $(RM) cgit VERSION CGIT-CFLAGS *.o tags + $(RM) -r .deps + +cleanall: clean + $(MAKE) -C git clean + +clean-doc: + $(RM) cgitrc.5 cgitrc.5.html cgitrc.5.pdf cgitrc.5.xml cgitrc.5.fo + +get-git: + curl -L $(GIT_URL) | tar -xJf - && rm -rf git && mv git-$(GIT_VER) git + +tags: + $(QUIET_TAGS)find . -name '*.[ch]' | xargs ctags + +.PHONY: all cgit git get-git +.PHONY: clean clean-doc cleanall +.PHONY: doc doc-html doc-man doc-pdf +.PHONY: install install-doc install-html install-man install-pdf +.PHONY: tags test +.PHONY: uninstall uninstall-doc uninstall-html uninstall-man uninstall-pdf diff --git a/third_party/cgit/README b/third_party/cgit/README new file mode 100644 index 000000000000..7a6b4a40ca57 --- /dev/null +++ b/third_party/cgit/README @@ -0,0 +1,99 @@ +cgit - CGI for Git +================== + +This is an attempt to create a fast web interface for the Git SCM, using a +built-in cache to decrease server I/O pressure. + +Installation +------------ + +Building cgit involves building a proper version of Git. How to do this +depends on how you obtained the cgit sources: + +a) If you're working in a cloned cgit repository, you first need to +initialize and update the Git submodule: + + $ git submodule init # register the Git submodule in .git/config + $ $EDITOR .git/config # if you want to specify a different url for git + $ git submodule update # clone/fetch and checkout correct git version + +b) If you're building from a cgit tarball, you can download a proper git +version like this: + + $ make get-git + +When either a) or b) has been performed, you can build and install cgit like +this: + + $ make + $ sudo make install + +This will install `cgit.cgi` and `cgit.css` into `/var/www/htdocs/cgit`. You +can configure this location (and a few other things) by providing a `cgit.conf` +file (see the Makefile for details). + +If you'd like to compile without Lua support, you may use: + + $ make NO_LUA=1 + +And if you'd like to specify a Lua implementation, you may use: + + $ make LUA_PKGCONFIG=lua5.1 + +If this is not specified, the Lua implementation will be auto-detected, +preferring LuaJIT if many are present. Acceptable values are generally "lua", +"luajit", "lua5.1", and "lua5.2". + + +Dependencies +------------ + +* libzip +* libcrypto (OpenSSL) +* libssl (OpenSSL) +* optional: luajit or lua, most reliably used when pkg-config is available + +Apache configuration +-------------------- + +A new `Directory` section must probably be added for cgit, possibly something +like this: + + <Directory "/var/www/htdocs/cgit/"> + AllowOverride None + Options +ExecCGI + Order allow,deny + Allow from all + </Directory> + + +Runtime configuration +--------------------- + +The file `/etc/cgitrc` is read by cgit before handling a request. In addition +to runtime parameters, this file may also contain a list of repositories +displayed by cgit (see `cgitrc.5.txt` for further details). + +The cache +--------- + +When cgit is invoked it looks for a cache file matching the request and +returns it to the client. If no such cache file exists (or if it has expired), +the content for the request is written into the proper cache file before the +file is returned. + +If the cache file has expired but cgit is unable to obtain a lock for it, the +stale cache file is returned to the client. This is done to favour page +throughput over page freshness. + +The generated content contains the complete response to the client, including +the HTTP headers `Modified` and `Expires`. + +Online presence +--------------- + +* The cgit homepage is hosted by cgit at <https://git.zx2c4.com/cgit/about/> + +* Patches, bug reports, discussions and support should go to the cgit + mailing list: <cgit@lists.zx2c4.com>. To sign up, visit + <https://lists.zx2c4.com/mailman/listinfo/cgit> diff --git a/third_party/cgit/cache.c b/third_party/cgit/cache.c new file mode 100644 index 000000000000..55199e8fe555 --- /dev/null +++ b/third_party/cgit/cache.c @@ -0,0 +1,468 @@ +/* cache.c: cache management + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + * + * + * The cache is just a directory structure where each file is a cache slot, + * and each filename is based on the hash of some key (e.g. the cgit url). + * Each file contains the full key followed by the cached content for that + * key. + * + */ + +#include "cgit.h" +#include "cache.h" +#include "html.h" +#ifdef HAVE_LINUX_SENDFILE +#include <sys/sendfile.h> +#endif + +#define CACHE_BUFSIZE (1024 * 4) + +struct cache_slot { + const char *key; + size_t keylen; + int ttl; + cache_fill_fn fn; + int cache_fd; + int lock_fd; + int stdout_fd; + const char *cache_name; + const char *lock_name; + int match; + struct stat cache_st; + int bufsize; + char buf[CACHE_BUFSIZE]; +}; + +/* Open an existing cache slot and fill the cache buffer with + * (part of) the content of the cache file. Return 0 on success + * and errno otherwise. + */ +static int open_slot(struct cache_slot *slot) +{ + char *bufz; + ssize_t bufkeylen = -1; + + slot->cache_fd = open(slot->cache_name, O_RDONLY); + if (slot->cache_fd == -1) + return errno; + + if (fstat(slot->cache_fd, &slot->cache_st)) + return errno; + + slot->bufsize = xread(slot->cache_fd, slot->buf, sizeof(slot->buf)); + if (slot->bufsize < 0) + return errno; + + bufz = memchr(slot->buf, 0, slot->bufsize); + if (bufz) + bufkeylen = bufz - slot->buf; + + if (slot->key) + slot->match = bufkeylen == slot->keylen && + !memcmp(slot->key, slot->buf, bufkeylen + 1); + + return 0; +} + +/* Close the active cache slot */ +static int close_slot(struct cache_slot *slot) +{ + int err = 0; + if (slot->cache_fd > 0) { + if (close(slot->cache_fd)) + err = errno; + else + slot->cache_fd = -1; + } + return err; +} + +/* Print the content of the active cache slot (but skip the key). */ +static int print_slot(struct cache_slot *slot) +{ +#ifdef HAVE_LINUX_SENDFILE + off_t start_off; + int ret; + + start_off = slot->keylen + 1; + + do { + ret = sendfile(STDOUT_FILENO, slot->cache_fd, &start_off, + slot->cache_st.st_size - start_off); + if (ret < 0) { + if (errno == EAGAIN || errno == EINTR) + continue; + return errno; + } + return 0; + } while (1); +#else + ssize_t i, j; + + i = lseek(slot->cache_fd, slot->keylen + 1, SEEK_SET); + if (i != slot->keylen + 1) + return errno; + + do { + i = j = xread(slot->cache_fd, slot->buf, sizeof(slot->buf)); + if (i > 0) + j = xwrite(STDOUT_FILENO, slot->buf, i); + } while (i > 0 && j == i); + + if (i < 0 || j != i) + return errno; + else + return 0; +#endif +} + +/* Check if the slot has expired */ +static int is_expired(struct cache_slot *slot) +{ + if (slot->ttl < 0) + return 0; + else + return slot->cache_st.st_mtime + slot->ttl * 60 < time(NULL); +} + +/* Check if the slot has been modified since we opened it. + * NB: If stat() fails, we pretend the file is modified. + */ +static int is_modified(struct cache_slot *slot) +{ + struct stat st; + + if (stat(slot->cache_name, &st)) + return 1; + return (st.st_ino != slot->cache_st.st_ino || + st.st_mtime != slot->cache_st.st_mtime || + st.st_size != slot->cache_st.st_size); +} + +/* Close an open lockfile */ +static int close_lock(struct cache_slot *slot) +{ + int err = 0; + if (slot->lock_fd > 0) { + if (close(slot->lock_fd)) + err = errno; + else + slot->lock_fd = -1; + } + return err; +} + +/* Create a lockfile used to store the generated content for a cache + * slot, and write the slot key + \0 into it. + * Returns 0 on success and errno otherwise. + */ +static int lock_slot(struct cache_slot *slot) +{ + struct flock lock = { + .l_type = F_WRLCK, + .l_whence = SEEK_SET, + .l_start = 0, + .l_len = 0, + }; + + slot->lock_fd = open(slot->lock_name, O_RDWR | O_CREAT, + S_IRUSR | S_IWUSR); + if (slot->lock_fd == -1) + return errno; + if (fcntl(slot->lock_fd, F_SETLK, &lock) < 0) { + int saved_errno = errno; + close(slot->lock_fd); + slot->lock_fd = -1; + return saved_errno; + } + if (xwrite(slot->lock_fd, slot->key, slot->keylen + 1) < 0) + return errno; + return 0; +} + +/* Release the current lockfile. If `replace_old_slot` is set the + * lockfile replaces the old cache slot, otherwise the lockfile is + * just deleted. + */ +static int unlock_slot(struct cache_slot *slot, int replace_old_slot) +{ + int err; + + if (replace_old_slot) + err = rename(slot->lock_name, slot->cache_name); + else + err = unlink(slot->lock_name); + + /* Restore stdout and close the temporary FD. */ + if (slot->stdout_fd >= 0) { + dup2(slot->stdout_fd, STDOUT_FILENO); + close(slot->stdout_fd); + slot->stdout_fd = -1; + } + + if (err) + return errno; + + return 0; +} + +/* Generate the content for the current cache slot by redirecting + * stdout to the lock-fd and invoking the callback function + */ +static int fill_slot(struct cache_slot *slot) +{ + /* Preserve stdout */ + slot->stdout_fd = dup(STDOUT_FILENO); + if (slot->stdout_fd == -1) + return errno; + + /* Redirect stdout to lockfile */ + if (dup2(slot->lock_fd, STDOUT_FILENO) == -1) + return errno; + + /* Generate cache content */ + slot->fn(); + + /* Make sure any buffered data is flushed to the file */ + if (fflush(stdout)) + return errno; + + /* update stat info */ + if (fstat(slot->lock_fd, &slot->cache_st)) + return errno; + + return 0; +} + +/* Crude implementation of 32-bit FNV-1 hash algorithm, + * see http://www.isthe.com/chongo/tech/comp/fnv/ for details + * about the magic numbers. + */ +#define FNV_OFFSET 0x811c9dc5 +#define FNV_PRIME 0x01000193 + +unsigned long hash_str(const char *str) +{ + unsigned long h = FNV_OFFSET; + unsigned char *s = (unsigned char *)str; + + if (!s) + return h; + + while (*s) { + h *= FNV_PRIME; + h ^= *s++; + } + return h; +} + +static int process_slot(struct cache_slot *slot) +{ + int err; + + err = open_slot(slot); + if (!err && slot->match) { + if (is_expired(slot)) { + if (!lock_slot(slot)) { + /* If the cachefile has been replaced between + * `open_slot` and `lock_slot`, we'll just + * serve the stale content from the original + * cachefile. This way we avoid pruning the + * newly generated slot. The same code-path + * is chosen if fill_slot() fails for some + * reason. + * + * TODO? check if the new slot contains the + * same key as the old one, since we would + * prefer to serve the newest content. + * This will require us to open yet another + * file-descriptor and read and compare the + * key from the new file, so for now we're + * lazy and just ignore the new file. + */ + if (is_modified(slot) || fill_slot(slot)) { + unlock_slot(slot, 0); + close_lock(slot); + } else { + close_slot(slot); + unlock_slot(slot, 1); + slot->cache_fd = slot->lock_fd; + } + } + } + if ((err = print_slot(slot)) != 0) { + cache_log("[cgit] error printing cache %s: %s (%d)\n", + slot->cache_name, + strerror(err), + err); + } + close_slot(slot); + return err; + } + + /* If the cache slot does not exist (or its key doesn't match the + * current key), lets try to create a new cache slot for this + * request. If this fails (for whatever reason), lets just generate + * the content without caching it and fool the caller to believe + * everything worked out (but print a warning on stdout). + */ + + close_slot(slot); + if ((err = lock_slot(slot)) != 0) { + cache_log("[cgit] Unable to lock slot %s: %s (%d)\n", + slot->lock_name, strerror(err), err); + slot->fn(); + return 0; + } + + if ((err = fill_slot(slot)) != 0) { + cache_log("[cgit] Unable to fill slot %s: %s (%d)\n", + slot->lock_name, strerror(err), err); + unlock_slot(slot, 0); + close_lock(slot); + slot->fn(); + return 0; + } + // We've got a valid cache slot in the lock file, which + // is about to replace the old cache slot. But if we + // release the lockfile and then try to open the new cache + // slot, we might get a race condition with a concurrent + // writer for the same cache slot (with a different key). + // Lets avoid such a race by just printing the content of + // the lock file. + slot->cache_fd = slot->lock_fd; + unlock_slot(slot, 1); + if ((err = print_slot(slot)) != 0) { + cache_log("[cgit] error printing cache %s: %s (%d)\n", + slot->cache_name, + strerror(err), + err); + } + close_slot(slot); + return err; +} + +/* Print cached content to stdout, generate the content if necessary. */ +int cache_process(int size, const char *path, const char *key, int ttl, + cache_fill_fn fn) +{ + unsigned long hash; + int i; + struct strbuf filename = STRBUF_INIT; + struct strbuf lockname = STRBUF_INIT; + struct cache_slot slot; + int result; + + /* If the cache is disabled, just generate the content */ + if (size <= 0 || ttl == 0) { + fn(); + return 0; + } + + /* Verify input, calculate filenames */ + if (!path) { + cache_log("[cgit] Cache path not specified, caching is disabled\n"); + fn(); + return 0; + } + if (!key) + key = ""; + hash = hash_str(key) % size; + strbuf_addstr(&filename, path); + strbuf_ensure_end(&filename, '/'); + for (i = 0; i < 8; i++) { + strbuf_addf(&filename, "%x", (unsigned char)(hash & 0xf)); + hash >>= 4; + } + strbuf_addbuf(&lockname, &filename); + strbuf_addstr(&lockname, ".lock"); + slot.fn = fn; + slot.ttl = ttl; + slot.stdout_fd = -1; + slot.cache_name = filename.buf; + slot.lock_name = lockname.buf; + slot.key = key; + slot.keylen = strlen(key); + result = process_slot(&slot); + + strbuf_release(&filename); + strbuf_release(&lockname); + return result; +} + +/* Return a strftime formatted date/time + * NB: the result from this function is to shared memory + */ +static char *sprintftime(const char *format, time_t time) +{ + static char buf[64]; + struct tm tm; + + if (!time) + return NULL; + gmtime_r(&time, &tm); + strftime(buf, sizeof(buf)-1, format, &tm); + return buf; +} + +int cache_ls(const char *path) +{ + DIR *dir; + struct dirent *ent; + int err = 0; + struct cache_slot slot = { NULL }; + struct strbuf fullname = STRBUF_INIT; + size_t prefixlen; + + if (!path) { + cache_log("[cgit] cache path not specified\n"); + return -1; + } + dir = opendir(path); + if (!dir) { + err = errno; + cache_log("[cgit] unable to open path %s: %s (%d)\n", + path, strerror(err), err); + return err; + } + strbuf_addstr(&fullname, path); + strbuf_ensure_end(&fullname, '/'); + prefixlen = fullname.len; + while ((ent = readdir(dir)) != NULL) { + if (strlen(ent->d_name) != 8) + continue; + strbuf_setlen(&fullname, prefixlen); + strbuf_addstr(&fullname, ent->d_name); + slot.cache_name = fullname.buf; + if ((err = open_slot(&slot)) != 0) { + cache_log("[cgit] unable to open path %s: %s (%d)\n", + fullname.buf, strerror(err), err); + continue; + } + htmlf("%s %s %10"PRIuMAX" %s\n", + fullname.buf, + sprintftime("%Y-%m-%d %H:%M:%S", + slot.cache_st.st_mtime), + (uintmax_t)slot.cache_st.st_size, + slot.buf); + close_slot(&slot); + } + closedir(dir); + strbuf_release(&fullname); + return 0; +} + +/* Print a message to stdout */ +void cache_log(const char *format, ...) +{ + va_list args; + va_start(args, format); + vfprintf(stderr, format, args); + va_end(args); +} + diff --git a/third_party/cgit/cache.h b/third_party/cgit/cache.h new file mode 100644 index 000000000000..470da4fc1564 --- /dev/null +++ b/third_party/cgit/cache.h @@ -0,0 +1,37 @@ +/* + * Since git has it's own cache.h which we include, + * lets test on CGIT_CACHE_H to avoid confusion + */ + +#ifndef CGIT_CACHE_H +#define CGIT_CACHE_H + +typedef void (*cache_fill_fn)(void); + + +/* Print cached content to stdout, generate the content if necessary. + * + * Parameters + * size max number of cache files + * path directory used to store cache files + * key the key used to lookup cache files + * ttl max cache time in seconds for this key + * fn content generator function for this key + * + * Return value + * 0 indicates success, everything else is an error + */ +extern int cache_process(int size, const char *path, const char *key, int ttl, + cache_fill_fn fn); + + +/* List info about all cache entries on stdout */ +extern int cache_ls(const char *path); + +/* Print a message to stdout */ +__attribute__((format (printf,1,2))) +extern void cache_log(const char *format, ...); + +extern unsigned long hash_str(const char *str); + +#endif /* CGIT_CACHE_H */ diff --git a/third_party/cgit/cgit.c b/third_party/cgit/cgit.c new file mode 100644 index 000000000000..4b2d86c88129 --- /dev/null +++ b/third_party/cgit/cgit.c @@ -0,0 +1,1112 @@ +/* cgit.c: cgi for the git scm + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "cache.h" +#include "cmd.h" +#include "configfile.h" +#include "html.h" +#include "ui-shared.h" +#include "ui-stats.h" +#include "ui-blob.h" +#include "ui-summary.h" +#include "scan-tree.h" + +const char *cgit_version = CGIT_VERSION; + +__attribute__((constructor)) +static void constructor_environment() +{ + /* Do not look in /etc/ for gitconfig and gitattributes. */ + setenv("GIT_CONFIG_NOSYSTEM", "1", 1); + setenv("GIT_ATTR_NOSYSTEM", "1", 1); + unsetenv("HOME"); + unsetenv("XDG_CONFIG_HOME"); +} + +static void add_mimetype(const char *name, const char *value) +{ + struct string_list_item *item; + + item = string_list_insert(&ctx.cfg.mimetypes, name); + item->util = xstrdup(value); +} + +static void process_cached_repolist(const char *path); + +static void repo_config(struct cgit_repo *repo, const char *name, const char *value) +{ + const char *path; + struct string_list_item *item; + + if (!strcmp(name, "name")) + repo->name = xstrdup(value); + else if (!strcmp(name, "clone-url")) + repo->clone_url = xstrdup(value); + else if (!strcmp(name, "desc")) + repo->desc = xstrdup(value); + else if (!strcmp(name, "owner")) + repo->owner = xstrdup(value); + else if (!strcmp(name, "homepage")) + repo->homepage = xstrdup(value); + else if (!strcmp(name, "defbranch")) + repo->defbranch = xstrdup(value); + else if (!strcmp(name, "extra-head-content")) + repo->extra_head_content = xstrdup(value); + else if (!strcmp(name, "snapshots")) + repo->snapshots = ctx.cfg.snapshots & cgit_parse_snapshots_mask(value); + else if (!strcmp(name, "enable-blame")) + repo->enable_blame = atoi(value); + else if (!strcmp(name, "enable-commit-graph")) + repo->enable_commit_graph = atoi(value); + else if (!strcmp(name, "enable-log-filecount")) + repo->enable_log_filecount = atoi(value); + else if (!strcmp(name, "enable-log-linecount")) + repo->enable_log_linecount = atoi(value); + else if (!strcmp(name, "enable-remote-branches")) + repo->enable_remote_branches = atoi(value); + else if (!strcmp(name, "enable-subject-links")) + repo->enable_subject_links = atoi(value); + else if (!strcmp(name, "enable-html-serving")) + repo->enable_html_serving = atoi(value); + else if (!strcmp(name, "branch-sort")) { + if (!strcmp(value, "age")) + repo->branch_sort = 1; + if (!strcmp(value, "name")) + repo->branch_sort = 0; + } else if (!strcmp(name, "commit-sort")) { + if (!strcmp(value, "date")) + repo->commit_sort = 1; + if (!strcmp(value, "topo")) + repo->commit_sort = 2; + } else if (!strcmp(name, "max-stats")) + repo->max_stats = cgit_find_stats_period(value, NULL); + else if (!strcmp(name, "module-link")) + repo->module_link= xstrdup(value); + else if (skip_prefix(name, "module-link.", &path)) { + item = string_list_append(&repo->submodules, xstrdup(path)); + item->util = xstrdup(value); + } else if (!strcmp(name, "section")) + repo->section = xstrdup(value); + else if (!strcmp(name, "snapshot-prefix")) + repo->snapshot_prefix = xstrdup(value); + else if (!strcmp(name, "readme") && value != NULL) { + if (repo->readme.items == ctx.cfg.readme.items) + memset(&repo->readme, 0, sizeof(repo->readme)); + string_list_append(&repo->readme, xstrdup(value)); + } else if (!strcmp(name, "logo") && value != NULL) + repo->logo = xstrdup(value); + else if (!strcmp(name, "logo-link") && value != NULL) + repo->logo_link = xstrdup(value); + else if (!strcmp(name, "hide")) + repo->hide = atoi(value); + else if (!strcmp(name, "ignore")) + repo->ignore = atoi(value); + else if (ctx.cfg.enable_filter_overrides) { + if (!strcmp(name, "about-filter")) + repo->about_filter = cgit_new_filter(value, ABOUT); + else if (!strcmp(name, "commit-filter")) + repo->commit_filter = cgit_new_filter(value, COMMIT); + else if (!strcmp(name, "source-filter")) + repo->source_filter = cgit_new_filter(value, SOURCE); + else if (!strcmp(name, "email-filter")) + repo->email_filter = cgit_new_filter(value, EMAIL); + else if (!strcmp(name, "owner-filter")) + repo->owner_filter = cgit_new_filter(value, OWNER); + } +} + +static void config_cb(const char *name, const char *value) +{ + const char *arg; + + if (!strcmp(name, "section")) + ctx.cfg.section = xstrdup(value); + else if (!strcmp(name, "repo.url")) + ctx.repo = cgit_add_repo(value); + else if (ctx.repo && !strcmp(name, "repo.path")) + ctx.repo->path = trim_end(value, '/'); + else if (ctx.repo && skip_prefix(name, "repo.", &arg)) + repo_config(ctx.repo, arg, value); + else if (!strcmp(name, "readme")) + string_list_append(&ctx.cfg.readme, xstrdup(value)); + else if (!strcmp(name, "root-title")) + ctx.cfg.root_title = xstrdup(value); + else if (!strcmp(name, "root-desc")) + ctx.cfg.root_desc = xstrdup(value); + else if (!strcmp(name, "root-readme")) + ctx.cfg.root_readme = xstrdup(value); + else if (!strcmp(name, "css")) + ctx.cfg.css = xstrdup(value); + else if (!strcmp(name, "favicon")) + ctx.cfg.favicon = xstrdup(value); + else if (!strcmp(name, "footer")) + ctx.cfg.footer = xstrdup(value); + else if (!strcmp(name, "head-include")) + ctx.cfg.head_include = xstrdup(value); + else if (!strcmp(name, "header")) + ctx.cfg.header = xstrdup(value); + else if (!strcmp(name, "logo")) + ctx.cfg.logo = xstrdup(value); + else if (!strcmp(name, "logo-link")) + ctx.cfg.logo_link = xstrdup(value); + else if (!strcmp(name, "module-link")) + ctx.cfg.module_link = xstrdup(value); + else if (!strcmp(name, "strict-export")) + ctx.cfg.strict_export = xstrdup(value); + else if (!strcmp(name, "virtual-root")) + ctx.cfg.virtual_root = ensure_end(value, '/'); + else if (!strcmp(name, "noplainemail")) + ctx.cfg.noplainemail = atoi(value); + else if (!strcmp(name, "noheader")) + ctx.cfg.noheader = atoi(value); + else if (!strcmp(name, "snapshots")) + ctx.cfg.snapshots = cgit_parse_snapshots_mask(value); + else if (!strcmp(name, "enable-filter-overrides")) + ctx.cfg.enable_filter_overrides = atoi(value); + else if (!strcmp(name, "enable-follow-links")) + ctx.cfg.enable_follow_links = atoi(value); + else if (!strcmp(name, "enable-http-clone")) + ctx.cfg.enable_http_clone = atoi(value); + else if (!strcmp(name, "enable-index-links")) + ctx.cfg.enable_index_links = atoi(value); + else if (!strcmp(name, "enable-index-owner")) + ctx.cfg.enable_index_owner = atoi(value); + else if (!strcmp(name, "enable-blame")) + ctx.cfg.enable_blame = atoi(value); + else if (!strcmp(name, "enable-commit-graph")) + ctx.cfg.enable_commit_graph = atoi(value); + else if (!strcmp(name, "enable-log-filecount")) + ctx.cfg.enable_log_filecount = atoi(value); + else if (!strcmp(name, "enable-log-linecount")) + ctx.cfg.enable_log_linecount = atoi(value); + else if (!strcmp(name, "enable-remote-branches")) + ctx.cfg.enable_remote_branches = atoi(value); + else if (!strcmp(name, "enable-subject-links")) + ctx.cfg.enable_subject_links = atoi(value); + else if (!strcmp(name, "enable-html-serving")) + ctx.cfg.enable_html_serving = atoi(value); + else if (!strcmp(name, "enable-tree-linenumbers")) + ctx.cfg.enable_tree_linenumbers = atoi(value); + else if (!strcmp(name, "enable-git-config")) + ctx.cfg.enable_git_config = atoi(value); + else if (!strcmp(name, "max-stats")) + ctx.cfg.max_stats = cgit_find_stats_period(value, NULL); + else if (!strcmp(name, "cache-size")) + ctx.cfg.cache_size = atoi(value); + else if (!strcmp(name, "cache-root")) + ctx.cfg.cache_root = xstrdup(expand_macros(value)); + else if (!strcmp(name, "cache-root-ttl")) + ctx.cfg.cache_root_ttl = atoi(value); + else if (!strcmp(name, "cache-repo-ttl")) + ctx.cfg.cache_repo_ttl = atoi(value); + else if (!strcmp(name, "cache-scanrc-ttl")) + ctx.cfg.cache_scanrc_ttl = atoi(value); + else if (!strcmp(name, "cache-static-ttl")) + ctx.cfg.cache_static_ttl = atoi(value); + else if (!strcmp(name, "cache-dynamic-ttl")) + ctx.cfg.cache_dynamic_ttl = atoi(value); + else if (!strcmp(name, "cache-about-ttl")) + ctx.cfg.cache_about_ttl = atoi(value); + else if (!strcmp(name, "cache-snapshot-ttl")) + ctx.cfg.cache_snapshot_ttl = atoi(value); + else if (!strcmp(name, "case-sensitive-sort")) + ctx.cfg.case_sensitive_sort = atoi(value); + else if (!strcmp(name, "about-filter")) + ctx.cfg.about_filter = cgit_new_filter(value, ABOUT); + else if (!strcmp(name, "commit-filter")) + ctx.cfg.commit_filter = cgit_new_filter(value, COMMIT); + else if (!strcmp(name, "email-filter")) + ctx.cfg.email_filter = cgit_new_filter(value, EMAIL); + else if (!strcmp(name, "owner-filter")) + ctx.cfg.owner_filter = cgit_new_filter(value, OWNER); + else if (!strcmp(name, "auth-filter")) + ctx.cfg.auth_filter = cgit_new_filter(value, AUTH); + else if (!strcmp(name, "embedded")) + ctx.cfg.embedded = atoi(value); + else if (!strcmp(name, "max-atom-items")) + ctx.cfg.max_atom_items = atoi(value); + else if (!strcmp(name, "max-message-length")) + ctx.cfg.max_msg_len = atoi(value); + else if (!strcmp(name, "max-repodesc-length")) + ctx.cfg.max_repodesc_len = atoi(value); + else if (!strcmp(name, "max-blob-size")) + ctx.cfg.max_blob_size = atoi(value); + else if (!strcmp(name, "max-repo-count")) + ctx.cfg.max_repo_count = atoi(value); + else if (!strcmp(name, "max-commit-count")) + ctx.cfg.max_commit_count = atoi(value); + else if (!strcmp(name, "project-list")) + ctx.cfg.project_list = xstrdup(expand_macros(value)); + else if (!strcmp(name, "scan-path")) + if (ctx.cfg.cache_size) + process_cached_repolist(expand_macros(value)); + else if (ctx.cfg.project_list) + scan_projects(expand_macros(value), + ctx.cfg.project_list, repo_config); + else + scan_tree(expand_macros(value), repo_config); + else if (!strcmp(name, "scan-hidden-path")) + ctx.cfg.scan_hidden_path = atoi(value); + else if (!strcmp(name, "section-from-path")) + ctx.cfg.section_from_path = atoi(value); + else if (!strcmp(name, "repository-sort")) + ctx.cfg.repository_sort = xstrdup(value); + else if (!strcmp(name, "section-sort")) + ctx.cfg.section_sort = atoi(value); + else if (!strcmp(name, "source-filter")) + ctx.cfg.source_filter = cgit_new_filter(value, SOURCE); + else if (!strcmp(name, "summary-log")) + ctx.cfg.summary_log = atoi(value); + else if (!strcmp(name, "summary-branches")) + ctx.cfg.summary_branches = atoi(value); + else if (!strcmp(name, "summary-tags")) + ctx.cfg.summary_tags = atoi(value); + else if (!strcmp(name, "side-by-side-diffs")) + ctx.cfg.difftype = atoi(value) ? DIFF_SSDIFF : DIFF_UNIFIED; + else if (!strcmp(name, "agefile")) + ctx.cfg.agefile = xstrdup(value); + else if (!strcmp(name, "mimetype-file")) + ctx.cfg.mimetype_file = xstrdup(value); + else if (!strcmp(name, "renamelimit")) + ctx.cfg.renamelimit = atoi(value); + else if (!strcmp(name, "remove-suffix")) + ctx.cfg.remove_suffix = atoi(value); + else if (!strcmp(name, "robots")) + ctx.cfg.robots = xstrdup(value); + else if (!strcmp(name, "clone-prefix")) + ctx.cfg.clone_prefix = xstrdup(value); + else if (!strcmp(name, "clone-url")) + ctx.cfg.clone_url = xstrdup(value); + else if (!strcmp(name, "local-time")) + ctx.cfg.local_time = atoi(value); + else if (!strcmp(name, "commit-sort")) { + if (!strcmp(value, "date")) + ctx.cfg.commit_sort = 1; + if (!strcmp(value, "topo")) + ctx.cfg.commit_sort = 2; + } else if (!strcmp(name, "branch-sort")) { + if (!strcmp(value, "age")) + ctx.cfg.branch_sort = 1; + if (!strcmp(value, "name")) + ctx.cfg.branch_sort = 0; + } else if (skip_prefix(name, "mimetype.", &arg)) + add_mimetype(arg, value); + else if (!strcmp(name, "include")) + parse_configfile(expand_macros(value), config_cb); +} + +static void querystring_cb(const char *name, const char *value) +{ + if (!value) + value = ""; + + if (!strcmp(name,"r")) { + ctx.qry.repo = xstrdup(value); + ctx.repo = cgit_get_repoinfo(value); + } else if (!strcmp(name, "p")) { + ctx.qry.page = xstrdup(value); + } else if (!strcmp(name, "url")) { + if (*value == '/') + value++; + ctx.qry.url = xstrdup(value); + cgit_parse_url(value); + } else if (!strcmp(name, "qt")) { + ctx.qry.grep = xstrdup(value); + } else if (!strcmp(name, "q")) { + ctx.qry.search = xstrdup(value); + } else if (!strcmp(name, "h")) { + ctx.qry.head = xstrdup(value); + ctx.qry.has_symref = 1; + } else if (!strcmp(name, "id")) { + ctx.qry.oid = xstrdup(value); + ctx.qry.has_oid = 1; + } else if (!strcmp(name, "id2")) { + ctx.qry.oid2 = xstrdup(value); + ctx.qry.has_oid = 1; + } else if (!strcmp(name, "ofs")) { + ctx.qry.ofs = atoi(value); + } else if (!strcmp(name, "path")) { + ctx.qry.path = trim_end(value, '/'); + } else if (!strcmp(name, "name")) { + ctx.qry.name = xstrdup(value); + } else if (!strcmp(name, "s")) { + ctx.qry.sort = xstrdup(value); + } else if (!strcmp(name, "showmsg")) { + ctx.qry.showmsg = atoi(value); + } else if (!strcmp(name, "period")) { + ctx.qry.period = xstrdup(value); + } else if (!strcmp(name, "dt")) { + ctx.qry.difftype = atoi(value); + ctx.qry.has_difftype = 1; + } else if (!strcmp(name, "ss")) { + /* No longer generated, but there may be links out there. */ + ctx.qry.difftype = atoi(value) ? DIFF_SSDIFF : DIFF_UNIFIED; + ctx.qry.has_difftype = 1; + } else if (!strcmp(name, "all")) { + ctx.qry.show_all = atoi(value); + } else if (!strcmp(name, "context")) { + ctx.qry.context = atoi(value); + } else if (!strcmp(name, "ignorews")) { + ctx.qry.ignorews = atoi(value); + } else if (!strcmp(name, "follow")) { + ctx.qry.follow = atoi(value); + } +} + +static void prepare_context(void) +{ + memset(&ctx, 0, sizeof(ctx)); + ctx.cfg.agefile = "info/web/last-modified"; + ctx.cfg.cache_size = 0; + ctx.cfg.cache_max_create_time = 5; + ctx.cfg.cache_root = CGIT_CACHE_ROOT; + ctx.cfg.cache_about_ttl = 15; + ctx.cfg.cache_snapshot_ttl = 5; + ctx.cfg.cache_repo_ttl = 5; + ctx.cfg.cache_root_ttl = 5; + ctx.cfg.cache_scanrc_ttl = 15; + ctx.cfg.cache_dynamic_ttl = 5; + ctx.cfg.cache_static_ttl = -1; + ctx.cfg.case_sensitive_sort = 1; + ctx.cfg.branch_sort = 0; + ctx.cfg.commit_sort = 0; + ctx.cfg.css = "/cgit.css"; + ctx.cfg.logo = "/cgit.png"; + ctx.cfg.favicon = "/favicon.ico"; + ctx.cfg.local_time = 0; + ctx.cfg.enable_http_clone = 1; + ctx.cfg.enable_index_owner = 1; + ctx.cfg.enable_tree_linenumbers = 1; + ctx.cfg.enable_git_config = 0; + ctx.cfg.max_repo_count = 50; + ctx.cfg.max_commit_count = 50; + ctx.cfg.max_lock_attempts = 5; + ctx.cfg.max_msg_len = 80; + ctx.cfg.max_repodesc_len = 80; + ctx.cfg.max_blob_size = 0; + ctx.cfg.max_stats = 0; + ctx.cfg.project_list = NULL; + ctx.cfg.renamelimit = -1; + ctx.cfg.remove_suffix = 0; + ctx.cfg.robots = "index, nofollow"; + ctx.cfg.root_title = "Git repository browser"; + ctx.cfg.root_desc = "a fast webinterface for the git dscm"; + ctx.cfg.scan_hidden_path = 0; + ctx.cfg.script_name = CGIT_SCRIPT_NAME; + ctx.cfg.section = ""; + ctx.cfg.repository_sort = "name"; + ctx.cfg.section_sort = 1; + ctx.cfg.summary_branches = 10; + ctx.cfg.summary_log = 10; + ctx.cfg.summary_tags = 10; + ctx.cfg.max_atom_items = 10; + ctx.cfg.difftype = DIFF_UNIFIED; + ctx.env.cgit_config = getenv("CGIT_CONFIG"); + ctx.env.http_host = getenv("HTTP_HOST"); + ctx.env.https = getenv("HTTPS"); + ctx.env.no_http = getenv("NO_HTTP"); + ctx.env.path_info = getenv("PATH_INFO"); + ctx.env.query_string = getenv("QUERY_STRING"); + ctx.env.request_method = getenv("REQUEST_METHOD"); + ctx.env.script_name = getenv("SCRIPT_NAME"); + ctx.env.server_name = getenv("SERVER_NAME"); + ctx.env.server_port = getenv("SERVER_PORT"); + ctx.env.http_cookie = getenv("HTTP_COOKIE"); + ctx.env.http_referer = getenv("HTTP_REFERER"); + ctx.env.content_length = getenv("CONTENT_LENGTH") ? strtoul(getenv("CONTENT_LENGTH"), NULL, 10) : 0; + ctx.env.authenticated = 0; + ctx.page.mimetype = "text/html"; + ctx.page.charset = PAGE_ENCODING; + ctx.page.filename = NULL; + ctx.page.size = 0; + ctx.page.modified = time(NULL); + ctx.page.expires = ctx.page.modified; + ctx.page.etag = NULL; + string_list_init_dup(&ctx.cfg.mimetypes); + if (ctx.env.script_name) + ctx.cfg.script_name = xstrdup(ctx.env.script_name); + if (ctx.env.query_string) + ctx.qry.raw = xstrdup(ctx.env.query_string); + if (!ctx.env.cgit_config) + ctx.env.cgit_config = CGIT_CONFIG; +} + +struct refmatch { + char *req_ref; + char *first_ref; + int match; +}; + +static int find_current_ref(const char *refname, const struct object_id *oid, + int flags, void *cb_data) +{ + struct refmatch *info; + + info = (struct refmatch *)cb_data; + if (!strcmp(refname, info->req_ref)) + info->match = 1; + if (!info->first_ref) + info->first_ref = xstrdup(refname); + return info->match; +} + +static void free_refmatch_inner(struct refmatch *info) +{ + if (info->first_ref) + free(info->first_ref); +} + +static char *find_default_branch(struct cgit_repo *repo) +{ + struct refmatch info; + char *ref; + + info.req_ref = repo->defbranch; + info.first_ref = NULL; + info.match = 0; + for_each_branch_ref(find_current_ref, &info); + if (info.match) + ref = info.req_ref; + else + ref = info.first_ref; + if (ref) + ref = xstrdup(ref); + free_refmatch_inner(&info); + + return ref; +} + +static char *guess_defbranch(void) +{ + const char *ref, *refname; + struct object_id oid; + + ref = resolve_ref_unsafe("HEAD", 0, &oid, NULL); + if (!ref || !skip_prefix(ref, "refs/heads/", &refname)) + return "master"; + return xstrdup(refname); +} + +/* The caller must free filename and ref after calling this. */ +static inline void parse_readme(const char *readme, char **filename, char **ref, struct cgit_repo *repo) +{ + const char *colon; + + *filename = NULL; + *ref = NULL; + + if (!readme || !readme[0]) + return; + + /* Check if the readme is tracked in the git repo. */ + colon = strchr(readme, ':'); + if (colon && strlen(colon) > 1) { + /* If it starts with a colon, we want to use + * the default branch */ + if (colon == readme && repo->defbranch) + *ref = xstrdup(repo->defbranch); + else + *ref = xstrndup(readme, colon - readme); + readme = colon + 1; + } + + /* Prepend repo path to relative readme path unless tracked. */ + if (!(*ref) && readme[0] != '/') + *filename = fmtalloc("%s/%s", repo->path, readme); + else + *filename = xstrdup(readme); +} +static void choose_readme(struct cgit_repo *repo) +{ + int found; + char *filename, *ref; + struct string_list_item *entry; + + if (!repo->readme.nr) + return; + + found = 0; + for_each_string_list_item(entry, &repo->readme) { + parse_readme(entry->string, &filename, &ref, repo); + if (!filename) { + free(filename); + free(ref); + continue; + } + if (ref) { + if (cgit_ref_path_exists(filename, ref, 1)) { + found = 1; + break; + } + } + else if (!access(filename, R_OK)) { + found = 1; + break; + } + free(filename); + free(ref); + } + repo->readme.strdup_strings = 1; + string_list_clear(&repo->readme, 0); + repo->readme.strdup_strings = 0; + if (found) + string_list_append(&repo->readme, filename)->util = ref; +} + +static void print_no_repo_clone_urls(const char *url) +{ + html("<tr><td><a rel='vcs-git' href='"); + html_url_path(url); + html("' title='"); + html_attr(ctx.repo->name); + html(" Git repository'>"); + html_txt(url); + html("</a></td></tr>\n"); +} + +static void prepare_repo_env(int *nongit) +{ + /* The path to the git repository. */ + setenv("GIT_DIR", ctx.repo->path, 1); + + /* Setup the git directory and initialize the notes system. Both of these + * load local configuration from the git repository, so we do them both while + * the HOME variables are unset. */ + setup_git_directory_gently(nongit); + load_display_notes(NULL); +} + +static int prepare_repo_cmd(int nongit) +{ + struct object_id oid; + int rc; + + if (nongit) { + const char *name = ctx.repo->name; + rc = errno; + ctx.page.title = fmtalloc("%s - %s", ctx.cfg.root_title, + "config error"); + ctx.repo = NULL; + cgit_print_http_headers(); + cgit_print_docstart(); + cgit_print_pageheader(); + cgit_print_error("Failed to open %s: %s", name, + rc ? strerror(rc) : "Not a valid git repository"); + cgit_print_docend(); + return 1; + } + ctx.page.title = fmtalloc("%s - %s", ctx.repo->name, ctx.repo->desc); + + if (!ctx.repo->defbranch) + ctx.repo->defbranch = guess_defbranch(); + + if (!ctx.qry.head) { + ctx.qry.nohead = 1; + ctx.qry.head = find_default_branch(ctx.repo); + } + + if (!ctx.qry.head) { + cgit_print_http_headers(); + cgit_print_docstart(); + cgit_print_pageheader(); + cgit_print_error("Repository seems to be empty"); + if (!strcmp(ctx.qry.page, "summary")) { + html("<table class='list'><tr class='nohover'><td> </td></tr><tr class='nohover'><th class='left'>Clone</th></tr>\n"); + cgit_prepare_repo_env(ctx.repo); + cgit_add_clone_urls(print_no_repo_clone_urls); + html("</table>\n"); + } + cgit_print_docend(); + return 1; + } + + if (get_oid(ctx.qry.head, &oid)) { + char *old_head = ctx.qry.head; + ctx.qry.head = xstrdup(ctx.repo->defbranch); + cgit_print_error_page(404, "Not found", + "Invalid branch: %s", old_head); + free(old_head); + return 1; + } + string_list_sort(&ctx.repo->submodules); + cgit_prepare_repo_env(ctx.repo); + choose_readme(ctx.repo); + return 0; +} + +static inline void open_auth_filter(const char *function) +{ + cgit_open_filter(ctx.cfg.auth_filter, function, + ctx.env.http_cookie ? ctx.env.http_cookie : "", + ctx.env.request_method ? ctx.env.request_method : "", + ctx.env.query_string ? ctx.env.query_string : "", + ctx.env.http_referer ? ctx.env.http_referer : "", + ctx.env.path_info ? ctx.env.path_info : "", + ctx.env.http_host ? ctx.env.http_host : "", + ctx.env.https ? ctx.env.https : "", + ctx.qry.repo ? ctx.qry.repo : "", + ctx.qry.page ? ctx.qry.page : "", + cgit_currentfullurl(), + cgit_loginurl()); +} + +/* We intentionally keep this rather small, instead of looping and + * feeding it to the filter a couple bytes at a time. This way, the + * filter itself does not need to handle any denial of service or + * buffer bloat issues. If this winds up being too small, people + * will complain on the mailing list, and we'll increase it as needed. */ +#define MAX_AUTHENTICATION_POST_BYTES 4096 +/* The filter is expected to spit out "Status: " and all headers. */ +static inline void authenticate_post(void) +{ + char buffer[MAX_AUTHENTICATION_POST_BYTES]; + ssize_t len; + + open_auth_filter("authenticate-post"); + len = ctx.env.content_length; + if (len > MAX_AUTHENTICATION_POST_BYTES) + len = MAX_AUTHENTICATION_POST_BYTES; + if ((len = read(STDIN_FILENO, buffer, len)) < 0) + die_errno("Could not read POST from stdin"); + if (write(STDOUT_FILENO, buffer, len) < 0) + die_errno("Could not write POST to stdout"); + cgit_close_filter(ctx.cfg.auth_filter); + exit(0); +} + +static inline void authenticate_cookie(void) +{ + /* If we don't have an auth_filter, consider all cookies valid, and thus return early. */ + if (!ctx.cfg.auth_filter) { + ctx.env.authenticated = 1; + return; + } + + /* If we're having something POST'd to /login, we're authenticating POST, + * instead of the cookie, so call authenticate_post and bail out early. + * This pattern here should match /?p=login with POST. */ + if (ctx.env.request_method && ctx.qry.page && !ctx.repo && \ + !strcmp(ctx.env.request_method, "POST") && !strcmp(ctx.qry.page, "login")) { + authenticate_post(); + return; + } + + /* If we've made it this far, we're authenticating the cookie for real, so do that. */ + open_auth_filter("authenticate-cookie"); + ctx.env.authenticated = cgit_close_filter(ctx.cfg.auth_filter); +} + +static void process_request(void) +{ + struct cgit_cmd *cmd; + int nongit = 0; + + /* If we're not yet authenticated, no matter what page we're on, + * display the authentication body from the auth_filter. This should + * never be cached. */ + if (!ctx.env.authenticated) { + ctx.page.title = "Authentication Required"; + cgit_print_http_headers(); + cgit_print_docstart(); + cgit_print_pageheader(); + open_auth_filter("body"); + cgit_close_filter(ctx.cfg.auth_filter); + cgit_print_docend(); + return; + } + + if (ctx.repo) + prepare_repo_env(&nongit); + + cmd = cgit_get_cmd(); + if (!cmd) { + ctx.page.title = "cgit error"; + cgit_print_error_page(404, "Not found", "Invalid request"); + return; + } + + if (!ctx.cfg.enable_http_clone && cmd->is_clone) { + ctx.page.title = "cgit error"; + cgit_print_error_page(404, "Not found", "Invalid request"); + return; + } + + if (cmd->want_repo && !ctx.repo) { + cgit_print_error_page(400, "Bad request", + "No repository selected"); + return; + } + + /* If cmd->want_vpath is set, assume ctx.qry.path contains a "virtual" + * in-project path limit to be made available at ctx.qry.vpath. + * Otherwise, no path limit is in effect (ctx.qry.vpath = NULL). + */ + ctx.qry.vpath = cmd->want_vpath ? ctx.qry.path : NULL; + + if (ctx.repo && prepare_repo_cmd(nongit)) + return; + + cmd->fn(); +} + +static int cmp_repos(const void *a, const void *b) +{ + const struct cgit_repo *ra = a, *rb = b; + return strcmp(ra->url, rb->url); +} + +static char *build_snapshot_setting(int bitmap) +{ + const struct cgit_snapshot_format *f; + struct strbuf result = STRBUF_INIT; + + for (f = cgit_snapshot_formats; f->suffix; f++) { + if (cgit_snapshot_format_bit(f) & bitmap) { + if (result.len) + strbuf_addch(&result, ' '); + strbuf_addstr(&result, f->suffix); + } + } + return strbuf_detach(&result, NULL); +} + +static char *get_first_line(char *txt) +{ + char *t = xstrdup(txt); + char *p = strchr(t, '\n'); + if (p) + *p = '\0'; + return t; +} + +static void print_repo(FILE *f, struct cgit_repo *repo) +{ + struct string_list_item *item; + fprintf(f, "repo.url=%s\n", repo->url); + fprintf(f, "repo.name=%s\n", repo->name); + fprintf(f, "repo.path=%s\n", repo->path); + if (repo->owner) + fprintf(f, "repo.owner=%s\n", repo->owner); + if (repo->desc) { + char *tmp = get_first_line(repo->desc); + fprintf(f, "repo.desc=%s\n", tmp); + free(tmp); + } + for_each_string_list_item(item, &repo->readme) { + if (item->util) + fprintf(f, "repo.readme=%s:%s\n", (char *)item->util, item->string); + else + fprintf(f, "repo.readme=%s\n", item->string); + } + if (repo->defbranch) + fprintf(f, "repo.defbranch=%s\n", repo->defbranch); + if (repo->extra_head_content) + fprintf(f, "repo.extra-head-content=%s\n", repo->extra_head_content); + if (repo->module_link) + fprintf(f, "repo.module-link=%s\n", repo->module_link); + if (repo->section) + fprintf(f, "repo.section=%s\n", repo->section); + if (repo->homepage) + fprintf(f, "repo.homepage=%s\n", repo->homepage); + if (repo->clone_url) + fprintf(f, "repo.clone-url=%s\n", repo->clone_url); + fprintf(f, "repo.enable-blame=%d\n", + repo->enable_blame); + fprintf(f, "repo.enable-commit-graph=%d\n", + repo->enable_commit_graph); + fprintf(f, "repo.enable-log-filecount=%d\n", + repo->enable_log_filecount); + fprintf(f, "repo.enable-log-linecount=%d\n", + repo->enable_log_linecount); + if (repo->about_filter && repo->about_filter != ctx.cfg.about_filter) + cgit_fprintf_filter(repo->about_filter, f, "repo.about-filter="); + if (repo->commit_filter && repo->commit_filter != ctx.cfg.commit_filter) + cgit_fprintf_filter(repo->commit_filter, f, "repo.commit-filter="); + if (repo->source_filter && repo->source_filter != ctx.cfg.source_filter) + cgit_fprintf_filter(repo->source_filter, f, "repo.source-filter="); + if (repo->email_filter && repo->email_filter != ctx.cfg.email_filter) + cgit_fprintf_filter(repo->email_filter, f, "repo.email-filter="); + if (repo->owner_filter && repo->owner_filter != ctx.cfg.owner_filter) + cgit_fprintf_filter(repo->owner_filter, f, "repo.owner-filter="); + if (repo->snapshots != ctx.cfg.snapshots) { + char *tmp = build_snapshot_setting(repo->snapshots); + fprintf(f, "repo.snapshots=%s\n", tmp ? tmp : ""); + free(tmp); + } + if (repo->snapshot_prefix) + fprintf(f, "repo.snapshot-prefix=%s\n", repo->snapshot_prefix); + if (repo->max_stats != ctx.cfg.max_stats) + fprintf(f, "repo.max-stats=%s\n", + cgit_find_stats_periodname(repo->max_stats)); + if (repo->logo) + fprintf(f, "repo.logo=%s\n", repo->logo); + if (repo->logo_link) + fprintf(f, "repo.logo-link=%s\n", repo->logo_link); + fprintf(f, "repo.enable-remote-branches=%d\n", repo->enable_remote_branches); + fprintf(f, "repo.enable-subject-links=%d\n", repo->enable_subject_links); + fprintf(f, "repo.enable-html-serving=%d\n", repo->enable_html_serving); + if (repo->branch_sort == 1) + fprintf(f, "repo.branch-sort=age\n"); + if (repo->commit_sort) { + if (repo->commit_sort == 1) + fprintf(f, "repo.commit-sort=date\n"); + else if (repo->commit_sort == 2) + fprintf(f, "repo.commit-sort=topo\n"); + } + fprintf(f, "repo.hide=%d\n", repo->hide); + fprintf(f, "repo.ignore=%d\n", repo->ignore); + fprintf(f, "\n"); +} + +static void print_repolist(FILE *f, struct cgit_repolist *list, int start) +{ + int i; + + for (i = start; i < list->count; i++) + print_repo(f, &list->repos[i]); +} + +/* Scan 'path' for git repositories, save the resulting repolist in 'cached_rc' + * and return 0 on success. + */ +static int generate_cached_repolist(const char *path, const char *cached_rc) +{ + struct strbuf locked_rc = STRBUF_INIT; + int result = 0; + int idx; + FILE *f; + + strbuf_addf(&locked_rc, "%s.lock", cached_rc); + f = fopen(locked_rc.buf, "wx"); + if (!f) { + /* Inform about the error unless the lockfile already existed, + * since that only means we've got concurrent requests. + */ + result = errno; + if (result != EEXIST) + fprintf(stderr, "[cgit] Error opening %s: %s (%d)\n", + locked_rc.buf, strerror(result), result); + goto out; + } + idx = cgit_repolist.count; + if (ctx.cfg.project_list) + scan_projects(path, ctx.cfg.project_list, repo_config); + else + scan_tree(path, repo_config); + print_repolist(f, &cgit_repolist, idx); + if (rename(locked_rc.buf, cached_rc)) + fprintf(stderr, "[cgit] Error renaming %s to %s: %s (%d)\n", + locked_rc.buf, cached_rc, strerror(errno), errno); + fclose(f); +out: + strbuf_release(&locked_rc); + return result; +} + +static void process_cached_repolist(const char *path) +{ + struct stat st; + struct strbuf cached_rc = STRBUF_INIT; + time_t age; + unsigned long hash; + + hash = hash_str(path); + if (ctx.cfg.project_list) + hash += hash_str(ctx.cfg.project_list); + strbuf_addf(&cached_rc, "%s/rc-%8lx", ctx.cfg.cache_root, hash); + + if (stat(cached_rc.buf, &st)) { + /* Nothing is cached, we need to scan without forking. And + * if we fail to generate a cached repolist, we need to + * invoke scan_tree manually. + */ + if (generate_cached_repolist(path, cached_rc.buf)) { + if (ctx.cfg.project_list) + scan_projects(path, ctx.cfg.project_list, + repo_config); + else + scan_tree(path, repo_config); + } + goto out; + } + + parse_configfile(cached_rc.buf, config_cb); + + /* If the cached configfile hasn't expired, lets exit now */ + age = time(NULL) - st.st_mtime; + if (age <= (ctx.cfg.cache_scanrc_ttl * 60)) + goto out; + + /* The cached repolist has been parsed, but it was old. So lets + * rescan the specified path and generate a new cached repolist + * in a child-process to avoid latency for the current request. + */ + if (fork()) + goto out; + + exit(generate_cached_repolist(path, cached_rc.buf)); +out: + strbuf_release(&cached_rc); +} + +static void cgit_parse_args(int argc, const char **argv) +{ + int i; + const char *arg; + int scan = 0; + + for (i = 1; i < argc; i++) { + if (!strcmp(argv[i], "--version")) { + printf("CGit %s | https://git.zx2c4.com/cgit/\n\nCompiled in features:\n", CGIT_VERSION); +#ifdef NO_LUA + printf("[-] "); +#else + printf("[+] "); +#endif + printf("Lua scripting\n"); +#ifndef HAVE_LINUX_SENDFILE + printf("[-] "); +#else + printf("[+] "); +#endif + printf("Linux sendfile() usage\n"); + + exit(0); + } + if (skip_prefix(argv[i], "--cache=", &arg)) { + ctx.cfg.cache_root = xstrdup(arg); + } else if (!strcmp(argv[i], "--nohttp")) { + ctx.env.no_http = "1"; + } else if (skip_prefix(argv[i], "--query=", &arg)) { + ctx.qry.raw = xstrdup(arg); + } else if (skip_prefix(argv[i], "--repo=", &arg)) { + ctx.qry.repo = xstrdup(arg); + } else if (skip_prefix(argv[i], "--page=", &arg)) { + ctx.qry.page = xstrdup(arg); + } else if (skip_prefix(argv[i], "--head=", &arg)) { + ctx.qry.head = xstrdup(arg); + ctx.qry.has_symref = 1; + } else if (skip_prefix(argv[i], "--oid=", &arg)) { + ctx.qry.oid = xstrdup(arg); + ctx.qry.has_oid = 1; + } else if (skip_prefix(argv[i], "--ofs=", &arg)) { + ctx.qry.ofs = atoi(arg); + } else if (skip_prefix(argv[i], "--scan-tree=", &arg) || + skip_prefix(argv[i], "--scan-path=", &arg)) { + /* + * HACK: The global snapshot bit mask defines the set + * of allowed snapshot formats, but the config file + * hasn't been parsed yet so the mask is currently 0. + * By setting all bits high before scanning we make + * sure that any in-repo cgitrc snapshot setting is + * respected by scan_tree(). + * + * NOTE: We assume that there aren't more than 8 + * different snapshot formats supported by cgit... + */ + ctx.cfg.snapshots = 0xFF; + scan++; + scan_tree(arg, repo_config); + } + } + if (scan) { + qsort(cgit_repolist.repos, cgit_repolist.count, + sizeof(struct cgit_repo), cmp_repos); + print_repolist(stdout, &cgit_repolist, 0); + exit(0); + } +} + +static int calc_ttl(void) +{ + if (!ctx.repo) + return ctx.cfg.cache_root_ttl; + + if (!ctx.qry.page) + return ctx.cfg.cache_repo_ttl; + + if (!strcmp(ctx.qry.page, "about")) + return ctx.cfg.cache_about_ttl; + + if (!strcmp(ctx.qry.page, "snapshot")) + return ctx.cfg.cache_snapshot_ttl; + + if (ctx.qry.has_oid) + return ctx.cfg.cache_static_ttl; + + if (ctx.qry.has_symref) + return ctx.cfg.cache_dynamic_ttl; + + return ctx.cfg.cache_repo_ttl; +} + +int cmd_main(int argc, const char **argv) +{ + const char *path; + int err, ttl; + + cgit_init_filters(); + atexit(cgit_cleanup_filters); + + prepare_context(); + cgit_repolist.length = 0; + cgit_repolist.count = 0; + cgit_repolist.repos = NULL; + + cgit_parse_args(argc, argv); + parse_configfile(expand_macros(ctx.env.cgit_config), config_cb); + ctx.repo = NULL; + http_parse_querystring(ctx.qry.raw, querystring_cb); + + /* If virtual-root isn't specified in cgitrc, lets pretend + * that virtual-root equals SCRIPT_NAME, minus any possibly + * trailing slashes. + */ + if (!ctx.cfg.virtual_root && ctx.cfg.script_name) + ctx.cfg.virtual_root = ensure_end(ctx.cfg.script_name, '/'); + + /* If no url parameter is specified on the querystring, lets + * use PATH_INFO as url. This allows cgit to work with virtual + * urls without the need for rewriterules in the webserver (as + * long as PATH_INFO is included in the cache lookup key). + */ + path = ctx.env.path_info; + if (!ctx.qry.url && path) { + if (path[0] == '/') + path++; + ctx.qry.url = xstrdup(path); + if (ctx.qry.raw) { + char *newqry = fmtalloc("%s?%s", path, ctx.qry.raw); + free(ctx.qry.raw); + ctx.qry.raw = newqry; + } else + ctx.qry.raw = xstrdup(ctx.qry.url); + cgit_parse_url(ctx.qry.url); + } + + /* Before we go any further, we set ctx.env.authenticated by checking to see + * if the supplied cookie is valid. All cookies are valid if there is no + * auth_filter. If there is an auth_filter, the filter decides. */ + authenticate_cookie(); + + ttl = calc_ttl(); + if (ttl < 0) + ctx.page.expires += 10 * 365 * 24 * 60 * 60; /* 10 years */ + else + ctx.page.expires += ttl * 60; + if (!ctx.env.authenticated || (ctx.env.request_method && !strcmp(ctx.env.request_method, "HEAD"))) + ctx.cfg.cache_size = 0; + err = cache_process(ctx.cfg.cache_size, ctx.cfg.cache_root, + ctx.qry.raw, ttl, process_request); + cgit_cleanup_filters(); + if (err) + cgit_print_error("Error processing page: %s (%d)", + strerror(err), err); + return err; +} diff --git a/third_party/cgit/cgit.css b/third_party/cgit/cgit.css new file mode 100644 index 000000000000..dfa144d05dd5 --- /dev/null +++ b/third_party/cgit/cgit.css @@ -0,0 +1,895 @@ +div#cgit { + padding: 0em; + margin: 0em; + font-family: sans-serif; + font-size: 10pt; + color: #333; + background: white; + padding: 4px; +} + +div#cgit a { + color: blue; + text-decoration: none; +} + +div#cgit a:hover { + text-decoration: underline; +} + +div#cgit table { + border-collapse: collapse; +} + +div#cgit table#header { + width: 100%; + margin-bottom: 1em; +} + +div#cgit table#header td.logo { + width: 96px; + vertical-align: top; +} + +div#cgit table#header td.main { + font-size: 250%; + padding-left: 10px; + white-space: nowrap; +} + +div#cgit table#header td.main a { + color: #000; +} + +div#cgit table#header td.form { + text-align: right; + vertical-align: bottom; + padding-right: 1em; + padding-bottom: 2px; + white-space: nowrap; +} + +div#cgit table#header td.form form, +div#cgit table#header td.form input, +div#cgit table#header td.form select { + font-size: 90%; +} + +div#cgit table#header td.sub { + color: #777; + border-top: solid 1px #ccc; + padding-left: 10px; +} + +div#cgit table.tabs { + border-bottom: solid 3px #ccc; + border-collapse: collapse; + margin-top: 2em; + margin-bottom: 0px; + width: 100%; +} + +div#cgit table.tabs td { + padding: 0px 1em; + vertical-align: bottom; +} + +div#cgit table.tabs td a { + padding: 2px 0.75em; + color: #777; + font-size: 110%; +} + +div#cgit table.tabs td a.active { + color: #000; + background-color: #ccc; +} + +div#cgit table.tabs a[href^="http://"]:after, div#cgit table.tabs a[href^="https://"]:after { + content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAQAAAAnOwc2AAAAAmJLR0QA/4ePzL8AAAAJcEhZcwAACxMAAAsTAQCanBgAAAAHdElNRQfgAhcJDQY+gm2TAAAAHWlUWHRDb21tZW50AAAAAABDcmVhdGVkIHdpdGggR0lNUGQuZQcAAABbSURBVAhbY2BABs4MU4CwhYHBh2Erww4wrGFQZHjI8B8IgUIscJWyDHcggltQhI4zGDCcRwhChPggHIggP1QoAVmQkSETrGoHsiAEsACtBYN0oDAMbgU6EBcAAL2eHUt4XUU4AAAAAElFTkSuQmCC); + opacity: 0.5; + margin: 0 0 0 5px; +} + +div#cgit table.tabs td.form { + text-align: right; +} + +div#cgit table.tabs td.form form { + padding-bottom: 2px; + font-size: 90%; + white-space: nowrap; +} + +div#cgit table.tabs td.form input, +div#cgit table.tabs td.form select { + font-size: 90%; +} + +div#cgit div.path { + margin: 0px; + padding: 5px 2em 2px 2em; + color: #000; + background-color: #eee; +} + +div#cgit div.content { + margin: 0px; + padding: 2em; + border-bottom: solid 3px #ccc; +} + + +div#cgit table.list { + width: 100%; + border: none; + border-collapse: collapse; +} + +div#cgit table.list tr { + background: white; +} + +div#cgit table.list tr.logheader { + background: #eee; +} + +div#cgit table.list tr:nth-child(even) { + background: #f7f7f7; +} + +div#cgit table.list tr:nth-child(odd) { + background: white; +} + +div#cgit table.list tr:hover { + background: #eee; +} + +div#cgit table.list tr.nohover { + background: white; +} + +div#cgit table.list tr.nohover:hover { + background: white; +} + +div#cgit table.list tr.nohover-highlight:hover:nth-child(even) { + background: #f7f7f7; +} + +div#cgit table.list tr.nohover-highlight:hover:nth-child(odd) { + background: white; +} + +div#cgit table.list th { + font-weight: bold; + /* color: #888; + border-top: dashed 1px #888; + border-bottom: dashed 1px #888; + */ + padding: 0.1em 0.5em 0.05em 0.5em; + vertical-align: baseline; +} + +div#cgit table.list td { + border: none; + padding: 0.1em 0.5em 0.1em 0.5em; +} + +div#cgit table.list td.commitgraph { + font-family: monospace; + white-space: pre; +} + +div#cgit table.list td.commitgraph .column1 { + color: #a00; +} + +div#cgit table.list td.commitgraph .column2 { + color: #0a0; +} + +div#cgit table.list td.commitgraph .column3 { + color: #aa0; +} + +div#cgit table.list td.commitgraph .column4 { + color: #00a; +} + +div#cgit table.list td.commitgraph .column5 { + color: #a0a; +} + +div#cgit table.list td.commitgraph .column6 { + color: #0aa; +} + +div#cgit table.list td.logsubject { + font-family: monospace; + font-weight: bold; +} + +div#cgit table.list td.logmsg { + font-family: monospace; + white-space: pre; + padding: 0 0.5em; +} + +div#cgit table.list td a { + color: black; +} + +div#cgit table.list td a.ls-dir { + font-weight: bold; + color: #00f; +} + +div#cgit table.list td a:hover { + color: #00f; +} + +div#cgit img { + border: none; +} + +div#cgit input#switch-btn { + margin: 2px 0px 0px 0px; +} + +div#cgit td#sidebar input.txt { + width: 100%; + margin: 2px 0px 0px 0px; +} + +div#cgit table#grid { + margin: 0px; +} + +div#cgit td#content { + vertical-align: top; + padding: 1em 2em 1em 1em; + border: none; +} + +div#cgit div#summary { + vertical-align: top; + margin-bottom: 1em; +} + +div#cgit table#downloads { + float: right; + border-collapse: collapse; + border: solid 1px #777; + margin-left: 0.5em; + margin-bottom: 0.5em; +} + +div#cgit table#downloads th { + background-color: #ccc; +} + +div#cgit div#blob { + border: solid 1px black; +} + +div#cgit div.error { + color: red; + font-weight: bold; + margin: 1em 2em; +} + +div#cgit a.ls-blob, div#cgit a.ls-dir, div#cgit .ls-mod { + font-family: monospace; +} + +div#cgit td.ls-size { + text-align: right; + font-family: monospace; + width: 10em; +} + +div#cgit td.ls-mode { + font-family: monospace; + width: 10em; +} + +div#cgit table.blob { + margin-top: 0.5em; + border-top: solid 1px black; +} + +div#cgit table.blob td.hashes, +div#cgit table.blob td.lines { + margin: 0; padding: 0 0 0 0.5em; + vertical-align: top; + color: black; +} + +div#cgit table.blob td.linenumbers { + margin: 0; padding: 0 0.5em 0 0.5em; + vertical-align: top; + text-align: right; + border-right: 1px solid gray; +} + +div#cgit table.blob pre { + padding: 0; margin: 0; +} + +div#cgit table.blob td.linenumbers a, +div#cgit table.ssdiff td.lineno a { + color: gray; + text-align: right; + text-decoration: none; +} + +div#cgit table.blob td.linenumbers a:hover, +div#cgit table.ssdiff td.lineno a:hover { + color: black; +} + +div#cgit table.blame td.hashes, +div#cgit table.blame td.lines, +div#cgit table.blame td.linenumbers { + padding: 0; +} + +div#cgit table.blame td.hashes div.alt, +div#cgit table.blame td.lines div.alt { + padding: 0 0.5em 0 0.5em; +} + +div#cgit table.blame td.linenumbers div.alt { + padding: 0 0.5em 0 0; +} + +div#cgit table.blame div.alt:nth-child(even) { + background: #eee; +} + +div#cgit table.blame div.alt:nth-child(odd) { + background: white; +} + +div#cgit table.blame td.lines > div { + position: relative; +} + +div#cgit table.blame td.lines > div > pre { + padding: 0 0 0 0.5em; + position: absolute; + top: 0; +} + +div#cgit table.bin-blob { + margin-top: 0.5em; + border: solid 1px black; +} + +div#cgit table.bin-blob th { + font-family: monospace; + white-space: pre; + border: solid 1px #777; + padding: 0.5em 1em; +} + +div#cgit table.bin-blob td { + font-family: monospace; + white-space: pre; + border-left: solid 1px #777; + padding: 0em 1em; +} + +div#cgit table.nowrap td { + white-space: nowrap; +} + +div#cgit table.commit-info { + border-collapse: collapse; + margin-top: 1.5em; +} + +div#cgit div.cgit-panel { + float: right; + margin-top: 1.5em; +} + +div#cgit div.cgit-panel table { + border-collapse: collapse; + border: solid 1px #aaa; + background-color: #eee; +} + +div#cgit div.cgit-panel th { + text-align: center; +} + +div#cgit div.cgit-panel td { + padding: 0.25em 0.5em; +} + +div#cgit div.cgit-panel td.label { + padding-right: 0.5em; +} + +div#cgit div.cgit-panel td.ctrl { + padding-left: 0.5em; +} + +div#cgit table.commit-info th { + text-align: left; + font-weight: normal; + padding: 0.1em 1em 0.1em 0.1em; + vertical-align: top; +} + +div#cgit table.commit-info td { + font-weight: normal; + padding: 0.1em 1em 0.1em 0.1em; +} + +div#cgit div.commit-subject { + font-weight: bold; + font-size: 125%; + margin: 1.5em 0em 0.5em 0em; + padding: 0em; +} + +div#cgit div.commit-msg { + white-space: pre; + font-family: monospace; +} + +div#cgit div.notes-header { + font-weight: bold; + padding-top: 1.5em; +} + +div#cgit div.notes { + white-space: pre; + font-family: monospace; + border: solid 1px #ee9; + background-color: #ffd; + padding: 0.3em 2em 0.3em 1em; + float: left; +} + +div#cgit div.notes-footer { + clear: left; +} + +div#cgit div.diffstat-header { + font-weight: bold; + padding-top: 1.5em; +} + +div#cgit table.diffstat { + border-collapse: collapse; + border: solid 1px #aaa; + background-color: #eee; +} + +div#cgit table.diffstat th { + font-weight: normal; + text-align: left; + text-decoration: underline; + padding: 0.1em 1em 0.1em 0.1em; + font-size: 100%; +} + +div#cgit table.diffstat td { + padding: 0.2em 0.2em 0.1em 0.1em; + font-size: 100%; + border: none; +} + +div#cgit table.diffstat td.mode { + white-space: nowrap; +} + +div#cgit table.diffstat td span.modechange { + padding-left: 1em; + color: red; +} + +div#cgit table.diffstat td.add a { + color: green; +} + +div#cgit table.diffstat td.del a { + color: red; +} + +div#cgit table.diffstat td.upd a { + color: blue; +} + +div#cgit table.diffstat td.graph { + width: 500px; + vertical-align: middle; +} + +div#cgit table.diffstat td.graph table { + border: none; +} + +div#cgit table.diffstat td.graph td { + padding: 0px; + border: 0px; + height: 7pt; +} + +div#cgit table.diffstat td.graph td.add { + background-color: #5c5; +} + +div#cgit table.diffstat td.graph td.rem { + background-color: #c55; +} + +div#cgit div.diffstat-summary { + color: #888; + padding-top: 0.5em; +} + +div#cgit table.diff { + width: 100%; +} + +div#cgit table.diff td { + font-family: monospace; + white-space: pre; +} + +div#cgit table.diff td div.head { + font-weight: bold; + margin-top: 1em; + color: black; +} + +div#cgit table.diff td div.hunk { + color: #009; +} + +div#cgit table.diff td div.add { + color: green; +} + +div#cgit table.diff td div.del { + color: red; +} + +div#cgit .oid { + font-family: monospace; + font-size: 90%; +} + +div#cgit .left { + text-align: left; +} + +div#cgit .right { + text-align: right; +} + +div#cgit table.list td.reposection { + font-style: italic; + color: #888; +} + +div#cgit a.button { + font-size: 80%; + padding: 0em 0.5em; +} + +div#cgit a.primary { + font-size: 100%; +} + +div#cgit a.secondary { + font-size: 90%; +} + +div#cgit td.toplevel-repo { + +} + +div#cgit table.list td.sublevel-repo { + padding-left: 1.5em; +} + +div#cgit ul.pager { + list-style-type: none; + text-align: center; + margin: 1em 0em 0em 0em; + padding: 0; +} + +div#cgit ul.pager li { + display: inline-block; + margin: 0.25em 0.5em; +} + +div#cgit ul.pager a { + color: #777; +} + +div#cgit ul.pager .current { + font-weight: bold; +} + +div#cgit span.age-mins { + font-weight: bold; + color: #080; +} + +div#cgit span.age-hours { + color: #080; +} + +div#cgit span.age-days { + color: #040; +} + +div#cgit span.age-weeks { + color: #444; +} + +div#cgit span.age-months { + color: #888; +} + +div#cgit span.age-years { + color: #bbb; +} + +div#cgit span.insertions { + color: #080; +} + +div#cgit span.deletions { + color: #800; +} + +div#cgit div.footer { + margin-top: 0.5em; + text-align: center; + font-size: 80%; + color: #ccc; +} + +div#cgit div.footer a { + color: #ccc; + text-decoration: none; +} + +div#cgit div.footer a:hover { + text-decoration: underline; +} + +div#cgit a.branch-deco { + color: #000; + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #88ff88; + border: solid 1px #007700; +} + +div#cgit a.tag-deco { + color: #000; + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ffff88; + border: solid 1px #777700; +} + +div#cgit a.tag-annotated-deco { + color: #000; + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ffcc88; + border: solid 1px #777700; +} + +div#cgit a.remote-deco { + color: #000; + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ccccff; + border: solid 1px #000077; +} + +div#cgit a.deco { + color: #000; + margin: 0px 0.5em; + padding: 0px 0.25em; + background-color: #ff8888; + border: solid 1px #770000; +} + +div#cgit div.commit-subject a.branch-deco, +div#cgit div.commit-subject a.tag-deco, +div#cgit div.commit-subject a.tag-annotated-deco, +div#cgit div.commit-subject a.remote-deco, +div#cgit div.commit-subject a.deco { + margin-left: 1em; + font-size: 75%; +} + +div#cgit table.stats { + border: solid 1px black; + border-collapse: collapse; +} + +div#cgit table.stats th { + text-align: left; + padding: 1px 0.5em; + background-color: #eee; + border: solid 1px black; +} + +div#cgit table.stats td { + text-align: right; + padding: 1px 0.5em; + border: solid 1px black; +} + +div#cgit table.stats td.total { + font-weight: bold; + text-align: left; +} + +div#cgit table.stats td.sum { + color: #c00; + font-weight: bold; +/* background-color: #eee; */ +} + +div#cgit table.stats td.left { + text-align: left; +} + +div#cgit table.vgraph { + border-collapse: separate; + border: solid 1px black; + height: 200px; +} + +div#cgit table.vgraph th { + background-color: #eee; + font-weight: bold; + border: solid 1px white; + padding: 1px 0.5em; +} + +div#cgit table.vgraph td { + vertical-align: bottom; + padding: 0px 10px; +} + +div#cgit table.vgraph div.bar { + background-color: #eee; +} + +div#cgit table.hgraph { + border: solid 1px black; + width: 800px; +} + +div#cgit table.hgraph th { + background-color: #eee; + font-weight: bold; + border: solid 1px black; + padding: 1px 0.5em; +} + +div#cgit table.hgraph td { + vertical-align: middle; + padding: 2px 2px; +} + +div#cgit table.hgraph div.bar { + background-color: #eee; + height: 1em; +} + +div#cgit table.ssdiff { + width: 100%; +} + +div#cgit table.ssdiff td { + font-size: 75%; + font-family: monospace; + white-space: pre; + padding: 1px 4px 1px 4px; + border-left: solid 1px #aaa; + border-right: solid 1px #aaa; +} + +div#cgit table.ssdiff td.add { + color: black; + background: #cfc; + min-width: 50%; +} + +div#cgit table.ssdiff td.add_dark { + color: black; + background: #aca; + min-width: 50%; +} + +div#cgit table.ssdiff span.add { + background: #cfc; + font-weight: bold; +} + +div#cgit table.ssdiff td.del { + color: black; + background: #fcc; + min-width: 50%; +} + +div#cgit table.ssdiff td.del_dark { + color: black; + background: #caa; + min-width: 50%; +} + +div#cgit table.ssdiff span.del { + background: #fcc; + font-weight: bold; +} + +div#cgit table.ssdiff td.changed { + color: black; + background: #ffc; + min-width: 50%; +} + +div#cgit table.ssdiff td.changed_dark { + color: black; + background: #cca; + min-width: 50%; +} + +div#cgit table.ssdiff td.lineno { + color: black; + background: #eee; + text-align: right; + width: 3em; + min-width: 3em; +} + +div#cgit table.ssdiff td.hunk { + color: black; + background: #ccf; + border-top: solid 1px #aaa; + border-bottom: solid 1px #aaa; +} + +div#cgit table.ssdiff td.head { + border-top: solid 1px #aaa; + border-bottom: solid 1px #aaa; +} + +div#cgit table.ssdiff td.head div.head { + font-weight: bold; + color: black; +} + +div#cgit table.ssdiff td.foot { + border-top: solid 1px #aaa; + border-left: none; + border-right: none; + border-bottom: none; +} + +div#cgit table.ssdiff td.space { + border: none; +} + +div#cgit table.ssdiff td.space div { + min-height: 3em; +} diff --git a/third_party/cgit/cgit.h b/third_party/cgit/cgit.h new file mode 100644 index 000000000000..69b5c13232e7 --- /dev/null +++ b/third_party/cgit/cgit.h @@ -0,0 +1,398 @@ +#ifndef CGIT_H +#define CGIT_H + + +#include <git-compat-util.h> +#include <stdbool.h> + +#include <cache.h> +#include <grep.h> +#include <object.h> +#include <object-store.h> +#include <tree.h> +#include <commit.h> +#include <tag.h> +#include <diff.h> +#include <diffcore.h> +#include <strvec.h> +#include <refs.h> +#include <revision.h> +#include <log-tree.h> +#include <archive.h> +#include <string-list.h> +#include <xdiff-interface.h> +#include <xdiff/xdiff.h> +#include <utf8.h> +#include <notes.h> +#include <graph.h> + +/* Add isgraph(x) to Git's sane ctype support (see git-compat-util.h) */ +#undef isgraph +#define isgraph(x) (isprint((x)) && !isspace((x))) + + +/* + * Limits used for relative dates + */ +#define TM_MIN 60 +#define TM_HOUR (TM_MIN * 60) +#define TM_DAY (TM_HOUR * 24) +#define TM_WEEK (TM_DAY * 7) +#define TM_YEAR (TM_DAY * 365) +#define TM_MONTH (TM_YEAR / 12.0) + + +/* + * Default encoding + */ +#define PAGE_ENCODING "UTF-8" + +#define BIT(x) (1U << (x)) + +typedef void (*configfn)(const char *name, const char *value); +typedef void (*filepair_fn)(struct diff_filepair *pair); +typedef void (*linediff_fn)(char *line, int len); + +typedef enum { + DIFF_UNIFIED, DIFF_SSDIFF, DIFF_STATONLY +} diff_type; + +typedef enum { + ABOUT, COMMIT, SOURCE, EMAIL, AUTH, OWNER +} filter_type; + +struct cgit_filter { + int (*open)(struct cgit_filter *, va_list ap); + int (*close)(struct cgit_filter *); + void (*fprintf)(struct cgit_filter *, FILE *, const char *prefix); + void (*cleanup)(struct cgit_filter *); + int argument_count; +}; + +struct cgit_exec_filter { + struct cgit_filter base; + char *cmd; + char **argv; + int old_stdout; + int pid; +}; + +struct cgit_repo { + char *url; + char *name; + char *path; + char *desc; + char *extra_head_content; + char *owner; + char *homepage; + char *defbranch; + char *module_link; + struct string_list readme; + char *section; + char *clone_url; + char *logo; + char *logo_link; + char *snapshot_prefix; + int snapshots; + int enable_blame; + int enable_commit_graph; + int enable_log_filecount; + int enable_log_linecount; + int enable_remote_branches; + int enable_subject_links; + int enable_html_serving; + int max_stats; + int branch_sort; + int commit_sort; + time_t mtime; + struct cgit_filter *about_filter; + struct cgit_filter *commit_filter; + struct cgit_filter *source_filter; + struct cgit_filter *email_filter; + struct cgit_filter *owner_filter; + struct string_list submodules; + int hide; + int ignore; +}; + +typedef void (*repo_config_fn)(struct cgit_repo *repo, const char *name, + const char *value); + +struct cgit_repolist { + int length; + int count; + struct cgit_repo *repos; +}; + +struct commitinfo { + struct commit *commit; + char *author; + char *author_email; + unsigned long author_date; + int author_tz; + char *committer; + char *committer_email; + unsigned long committer_date; + int committer_tz; + char *subject; + char *msg; + char *msg_encoding; +}; + +struct taginfo { + char *tagger; + char *tagger_email; + unsigned long tagger_date; + int tagger_tz; + char *msg; +}; + +struct refinfo { + const char *refname; + struct object *object; + union { + struct taginfo *tag; + struct commitinfo *commit; + }; +}; + +struct reflist { + struct refinfo **refs; + int alloc; + int count; +}; + +struct cgit_query { + int has_symref; + int has_oid; + int has_difftype; + char *raw; + char *repo; + char *page; + char *search; + char *grep; + char *head; + char *oid; + char *oid2; + char *path; + char *name; + char *url; + char *period; + int ofs; + int nohead; + char *sort; + int showmsg; + diff_type difftype; + int show_all; + int context; + int ignorews; + int follow; + char *vpath; +}; + +struct cgit_config { + char *agefile; + char *cache_root; + char *clone_prefix; + char *clone_url; + char *css; + char *favicon; + char *footer; + char *head_include; + char *header; + char *logo; + char *logo_link; + char *mimetype_file; + char *module_link; + char *project_list; + struct string_list readme; + char *robots; + char *root_title; + char *root_desc; + char *root_readme; + char *script_name; + char *section; + char *repository_sort; + char *virtual_root; /* Always ends with '/'. */ + char *strict_export; + int cache_size; + int cache_dynamic_ttl; + int cache_max_create_time; + int cache_repo_ttl; + int cache_root_ttl; + int cache_scanrc_ttl; + int cache_static_ttl; + int cache_about_ttl; + int cache_snapshot_ttl; + int case_sensitive_sort; + int embedded; + int enable_filter_overrides; + int enable_follow_links; + int enable_http_clone; + int enable_index_links; + int enable_index_owner; + int enable_blame; + int enable_commit_graph; + int enable_log_filecount; + int enable_log_linecount; + int enable_remote_branches; + int enable_subject_links; + int enable_html_serving; + int enable_tree_linenumbers; + int enable_git_config; + int local_time; + int max_atom_items; + int max_repo_count; + int max_commit_count; + int max_lock_attempts; + int max_msg_len; + int max_repodesc_len; + int max_blob_size; + int max_stats; + int noplainemail; + int noheader; + int renamelimit; + int remove_suffix; + int scan_hidden_path; + int section_from_path; + int snapshots; + int section_sort; + int summary_branches; + int summary_log; + int summary_tags; + diff_type difftype; + int branch_sort; + int commit_sort; + struct string_list mimetypes; + struct cgit_filter *about_filter; + struct cgit_filter *commit_filter; + struct cgit_filter *source_filter; + struct cgit_filter *email_filter; + struct cgit_filter *owner_filter; + struct cgit_filter *auth_filter; +}; + +struct cgit_page { + time_t modified; + time_t expires; + size_t size; + const char *mimetype; + const char *charset; + const char *filename; + const char *etag; + const char *title; + int status; + const char *statusmsg; +}; + +struct cgit_environment { + const char *cgit_config; + const char *http_host; + const char *https; + const char *no_http; + const char *path_info; + const char *query_string; + const char *request_method; + const char *script_name; + const char *server_name; + const char *server_port; + const char *http_cookie; + const char *http_referer; + unsigned int content_length; + int authenticated; +}; + +struct cgit_context { + struct cgit_environment env; + struct cgit_query qry; + struct cgit_config cfg; + struct cgit_repo *repo; + struct cgit_page page; +}; + +typedef int (*write_archive_fn_t)(const char *, const char *); + +struct cgit_snapshot_format { + const char *suffix; + const char *mimetype; + write_archive_fn_t write_func; +}; + +extern const char *cgit_version; + +extern struct cgit_repolist cgit_repolist; +extern struct cgit_context ctx; +extern const struct cgit_snapshot_format cgit_snapshot_formats[]; + +extern char *cgit_default_repo_desc; +extern struct cgit_repo *cgit_add_repo(const char *url); +extern struct cgit_repo *cgit_get_repoinfo(const char *url); +extern void cgit_repo_config_cb(const char *name, const char *value); + +extern int chk_zero(int result, char *msg); +extern int chk_positive(int result, char *msg); +extern int chk_non_negative(int result, char *msg); + +extern char *trim_end(const char *str, char c); +extern char *ensure_end(const char *str, char c); + +extern void strbuf_ensure_end(struct strbuf *sb, char c); + +extern void cgit_add_ref(struct reflist *list, struct refinfo *ref); +extern void cgit_free_reflist_inner(struct reflist *list); +extern int cgit_refs_cb(const char *refname, const struct object_id *oid, + int flags, void *cb_data); + +extern void cgit_free_commitinfo(struct commitinfo *info); +extern void cgit_free_taginfo(struct taginfo *info); + +void cgit_diff_tree_cb(struct diff_queue_struct *q, + struct diff_options *options, void *data); + +extern int cgit_diff_files(const struct object_id *old_oid, + const struct object_id *new_oid, + unsigned long *old_size, unsigned long *new_size, + int *binary, int context, int ignorews, + linediff_fn fn); + +extern void cgit_diff_tree(const struct object_id *old_oid, + const struct object_id *new_oid, + filepair_fn fn, const char *prefix, int ignorews); + +extern void cgit_diff_commit(struct commit *commit, filepair_fn fn, + const char *prefix); + +__attribute__((format (printf,1,2))) +extern char *fmt(const char *format,...); + +__attribute__((format (printf,1,2))) +extern char *fmtalloc(const char *format,...); + +extern struct commitinfo *cgit_parse_commit(struct commit *commit); +extern struct taginfo *cgit_parse_tag(struct tag *tag); +extern void cgit_parse_url(const char *url); + +extern const char *cgit_repobasename(const char *reponame); + +extern int cgit_parse_snapshots_mask(const char *str); +extern const struct object_id *cgit_snapshot_get_sig(const char *ref, + const struct cgit_snapshot_format *f); +extern const unsigned cgit_snapshot_format_bit(const struct cgit_snapshot_format *f); + +extern int cgit_open_filter(struct cgit_filter *filter, ...); +extern int cgit_close_filter(struct cgit_filter *filter); +extern void cgit_fprintf_filter(struct cgit_filter *filter, FILE *f, const char *prefix); +extern void cgit_exec_filter_init(struct cgit_exec_filter *filter, char *cmd, char **argv); +extern struct cgit_filter *cgit_new_filter(const char *cmd, filter_type filtertype); +extern void cgit_cleanup_filters(void); +extern void cgit_init_filters(void); + +extern void cgit_prepare_repo_env(struct cgit_repo * repo); + +extern int readfile(const char *path, char **buf, size_t *size); + +extern char *expand_macros(const char *txt); + +extern char *get_mimetype_for_filename(const char *filename); + +#endif /* CGIT_H */ diff --git a/third_party/cgit/cgit.mk b/third_party/cgit/cgit.mk new file mode 100644 index 000000000000..3fcc1ca31440 --- /dev/null +++ b/third_party/cgit/cgit.mk @@ -0,0 +1,141 @@ +# This Makefile is run in the "git" directory in order to re-use Git's +# build variables and operating system detection. Hence all files in +# CGit's directory must be prefixed with "../". +include Makefile + +CGIT_PREFIX = ../ + +-include $(CGIT_PREFIX)cgit.conf + +# The CGIT_* variables are inherited when this file is called from the +# main Makefile - they are defined there. + +$(CGIT_PREFIX)VERSION: force-version + @cd $(CGIT_PREFIX) && '$(SHELL_PATH_SQ)' ./gen-version.sh "$(CGIT_VERSION)" +-include $(CGIT_PREFIX)VERSION +.PHONY: force-version + +# CGIT_CFLAGS is a separate variable so that we can track it separately +# and avoid rebuilding all of Git when these variables change. +CGIT_CFLAGS += -DCGIT_CONFIG='"$(CGIT_CONFIG)"' +CGIT_CFLAGS += -DCGIT_SCRIPT_NAME='"$(CGIT_SCRIPT_NAME)"' +CGIT_CFLAGS += -DCGIT_CACHE_ROOT='"$(CACHE_ROOT)"' + +PKG_CONFIG ?= pkg-config + +ifdef NO_C99_FORMAT + CFLAGS += -DNO_C99_FORMAT +endif + +ifdef NO_LUA + LUA_MESSAGE := linking without specified Lua support + CGIT_CFLAGS += -DNO_LUA +else +ifeq ($(LUA_PKGCONFIG),) + LUA_PKGCONFIG := $(shell for pc in luajit lua lua5.2 lua5.1; do \ + $(PKG_CONFIG) --exists $$pc 2>/dev/null && echo $$pc && break; \ + done) + LUA_MODE := autodetected +else + LUA_MODE := specified +endif +ifneq ($(LUA_PKGCONFIG),) + LUA_MESSAGE := linking with $(LUA_MODE) $(LUA_PKGCONFIG) + LUA_LIBS := $(shell $(PKG_CONFIG) --libs $(LUA_PKGCONFIG) 2>/dev/null) + LUA_CFLAGS := $(shell $(PKG_CONFIG) --cflags $(LUA_PKGCONFIG) 2>/dev/null) + CGIT_LIBS += $(LUA_LIBS) + CGIT_CFLAGS += $(LUA_CFLAGS) +else + LUA_MESSAGE := linking without autodetected Lua support + NO_LUA := YesPlease + CGIT_CFLAGS += -DNO_LUA +endif + +endif + +# Add -ldl to linker flags on systems that commonly use GNU libc. +ifneq (,$(filter $(uname_S),Linux GNU GNU/kFreeBSD)) + CGIT_LIBS += -ldl +endif + +# glibc 2.1+ offers sendfile which the most common C library on Linux +ifeq ($(uname_S),Linux) + HAVE_LINUX_SENDFILE = YesPlease +endif + +ifdef HAVE_LINUX_SENDFILE + CGIT_CFLAGS += -DHAVE_LINUX_SENDFILE +endif + +CGIT_OBJ_NAMES += cgit.o +CGIT_OBJ_NAMES += cache.o +CGIT_OBJ_NAMES += cmd.o +CGIT_OBJ_NAMES += configfile.o +CGIT_OBJ_NAMES += filter.o +CGIT_OBJ_NAMES += html.o +CGIT_OBJ_NAMES += parsing.o +CGIT_OBJ_NAMES += scan-tree.o +CGIT_OBJ_NAMES += shared.o +CGIT_OBJ_NAMES += ui-atom.o +CGIT_OBJ_NAMES += ui-blame.o +CGIT_OBJ_NAMES += ui-blob.o +CGIT_OBJ_NAMES += ui-clone.o +CGIT_OBJ_NAMES += ui-commit.o +CGIT_OBJ_NAMES += ui-diff.o +CGIT_OBJ_NAMES += ui-log.o +CGIT_OBJ_NAMES += ui-patch.o +CGIT_OBJ_NAMES += ui-plain.o +CGIT_OBJ_NAMES += ui-refs.o +CGIT_OBJ_NAMES += ui-repolist.o +CGIT_OBJ_NAMES += ui-shared.o +CGIT_OBJ_NAMES += ui-snapshot.o +CGIT_OBJ_NAMES += ui-ssdiff.o +CGIT_OBJ_NAMES += ui-stats.o +CGIT_OBJ_NAMES += ui-summary.o +CGIT_OBJ_NAMES += ui-tag.o +CGIT_OBJ_NAMES += ui-tree.o + +CGIT_OBJS := $(addprefix $(CGIT_PREFIX),$(CGIT_OBJ_NAMES)) + +# Only cgit.c reference CGIT_VERSION so we only rebuild its objects when the +# version changes. +CGIT_VERSION_OBJS := $(addprefix $(CGIT_PREFIX),cgit.o cgit.sp) +$(CGIT_VERSION_OBJS): $(CGIT_PREFIX)VERSION +$(CGIT_VERSION_OBJS): EXTRA_CPPFLAGS = \ + -DCGIT_VERSION='"$(CGIT_VERSION)"' + +# Git handles dependencies using ":=" so dependencies in CGIT_OBJ are not +# handled by that and we must handle them ourselves. +cgit_dep_files := $(foreach f,$(CGIT_OBJS),$(dir $f).depend/$(notdir $f).d) +cgit_dep_files_present := $(wildcard $(cgit_dep_files)) +ifneq ($(cgit_dep_files_present),) +include $(cgit_dep_files_present) +endif + +ifeq ($(wildcard $(CGIT_PREFIX).depend),) +missing_dep_dirs += $(CGIT_PREFIX).depend +endif + +$(CGIT_PREFIX).depend: + @mkdir -p $@ + +$(CGIT_PREFIX)CGIT-CFLAGS: FORCE + @FLAGS='$(subst ','\'',$(CGIT_CFLAGS))'; \ + if test x"$$FLAGS" != x"`cat ../CGIT-CFLAGS 2>/dev/null`" ; then \ + echo 1>&2 " * new CGit build flags"; \ + echo "$$FLAGS" >$(CGIT_PREFIX)CGIT-CFLAGS; \ + fi + +$(CGIT_OBJS): %.o: %.c GIT-CFLAGS $(CGIT_PREFIX)CGIT-CFLAGS $(missing_dep_dirs) + $(QUIET_CC)$(CC) -o $*.o -c $(dep_args) $(ALL_CFLAGS) $(EXTRA_CPPFLAGS) $(CGIT_CFLAGS) $< + +$(CGIT_PREFIX)cgit: $(CGIT_OBJS) GIT-LDFLAGS $(GITLIBS) + @echo 1>&1 " * $(LUA_MESSAGE)" + $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS) $(CGIT_LIBS) + +CGIT_SP_OBJS := $(patsubst %.o,%.sp,$(CGIT_OBJS)) + +$(CGIT_SP_OBJS): %.sp: %.c GIT-CFLAGS $(CGIT_PREFIX)CGIT-CFLAGS FORCE + $(QUIET_SP)cgcc -no-compile $(ALL_CFLAGS) $(EXTRA_CPPFLAGS) $(CGIT_CFLAGS) $(SPARSE_FLAGS) $< + +cgit-sparse: $(CGIT_SP_OBJS) diff --git a/third_party/cgit/cgit.png b/third_party/cgit/cgit.png new file mode 100644 index 000000000000..425528ee3951 --- /dev/null +++ b/third_party/cgit/cgit.png Binary files differdiff --git a/third_party/cgit/cgitrc.5.txt b/third_party/cgit/cgitrc.5.txt new file mode 100644 index 000000000000..33a6a8c0c758 --- /dev/null +++ b/third_party/cgit/cgitrc.5.txt @@ -0,0 +1,1011 @@ +:man source: cgit +:man manual: cgit + +CGITRC(5) +======== + + +NAME +---- +cgitrc - runtime configuration for cgit + + +SYNOPSIS +-------- +Cgitrc contains all runtime settings for cgit, including the list of git +repositories, formatted as a line-separated list of NAME=VALUE pairs. Blank +lines, and lines starting with '#', are ignored. + + +LOCATION +-------- +The default location of cgitrc, defined at compile time, is /etc/cgitrc. At +runtime, cgit will consult the environment variable CGIT_CONFIG and, if +defined, use its value instead. + + +GLOBAL SETTINGS +--------------- +about-filter:: + Specifies a command which will be invoked to format the content of + about pages (both top-level and for each repository). The command will + get the content of the about-file on its STDIN, the name of the file + as the first argument, and the STDOUT from the command will be + included verbatim on the about page. Default value: none. See + also: "FILTER API". + +agefile:: + Specifies a path, relative to each repository path, which can be used + to specify the date and time of the youngest commit in the repository. + The first line in the file is used as input to the "parse_date" + function in libgit. Recommended timestamp-format is "yyyy-mm-dd + hh:mm:ss". You may want to generate this file from a post-receive + hook. Default value: "info/web/last-modified". + +auth-filter:: + Specifies a command that will be invoked for authenticating repository + access. Receives quite a few arguments, and data on both stdin and + stdout for authentication processing. Details follow later in this + document. If no auth-filter is specified, no authentication is + performed. Default value: none. See also: "FILTER API". + +branch-sort:: + Flag which, when set to "age", enables date ordering in the branch ref + list, and when set to "name" enables ordering by branch name. Default + value: "name". + +cache-about-ttl:: + Number which specifies the time-to-live, in minutes, for the cached + version of the repository about page. See also: "CACHE". Default + value: "15". + +cache-dynamic-ttl:: + Number which specifies the time-to-live, in minutes, for the cached + version of repository pages accessed without a fixed SHA1. See also: + "CACHE". Default value: "5". + +cache-repo-ttl:: + Number which specifies the time-to-live, in minutes, for the cached + version of the repository summary page. See also: "CACHE". Default + value: "5". + +cache-root:: + Path used to store the cgit cache entries. Default value: + "/var/cache/cgit". See also: "MACRO EXPANSION". + +cache-root-ttl:: + Number which specifies the time-to-live, in minutes, for the cached + version of the repository index page. See also: "CACHE". Default + value: "5". + +cache-scanrc-ttl:: + Number which specifies the time-to-live, in minutes, for the result + of scanning a path for git repositories. See also: "CACHE". Default + value: "15". + +case-sensitive-sort:: + Sort items in the repo list case sensitively. Default value: "1". + See also: repository-sort, section-sort. + +cache-size:: + The maximum number of entries in the cgit cache. When set to "0", + caching is disabled. See also: "CACHE". Default value: "0" + +cache-snapshot-ttl:: + Number which specifies the time-to-live, in minutes, for the cached + version of snapshots. See also: "CACHE". Default value: "5". + +cache-static-ttl:: + Number which specifies the time-to-live, in minutes, for the cached + version of repository pages accessed with a fixed SHA1. See also: + "CACHE". Default value: -1". + +clone-prefix:: + Space-separated list of common prefixes which, when combined with a + repository url, generates valid clone urls for the repository. This + setting is only used if `repo.clone-url` is unspecified. Default value: + none. + +clone-url:: + Space-separated list of clone-url templates. This setting is only + used if `repo.clone-url` is unspecified. Default value: none. See + also: "MACRO EXPANSION", "FILTER API". + +commit-filter:: + Specifies a command which will be invoked to format commit messages. + The command will get the message on its STDIN, and the STDOUT from the + command will be included verbatim as the commit message, i.e. this can + be used to implement bugtracker integration. Default value: none. + See also: "FILTER API". + +commit-sort:: + Flag which, when set to "date", enables strict date ordering in the + commit log, and when set to "topo" enables strict topological + ordering. If unset, the default ordering of "git log" is used. Default + value: unset. + +css:: + Url which specifies the css document to include in all cgit pages. + Default value: "/cgit.css". + +email-filter:: + Specifies a command which will be invoked to format names and email + address of committers, authors, and taggers, as represented in various + places throughout the cgit interface. This command will receive an + email address and an origin page string as its command line arguments, + and the text to format on STDIN. It is to write the formatted text back + out onto STDOUT. Default value: none. See also: "FILTER API". + +embedded:: + Flag which, when set to "1", will make cgit generate a html fragment + suitable for embedding in other html pages. Default value: none. See + also: "noheader". + +enable-blame:: + Flag which, when set to "1", will allow cgit to provide a "blame" page + for files, and will make it generate links to that page in appropriate + places. Default value: "0". + +enable-commit-graph:: + Flag which, when set to "1", will make cgit print an ASCII-art commit + history graph to the left of the commit messages in the repository + log page. Default value: "0". + +enable-filter-overrides:: + Flag which, when set to "1", allows all filter settings to be + overridden in repository-specific cgitrc files. Default value: none. + +enable-follow-links:: + Flag which, when set to "1", allows users to follow a file in the log + view. Default value: "0". + +enable-git-config:: + Flag which, when set to "1", will allow cgit to use git config to set + any repo specific settings. This option is used in conjunction with + "scan-path", and must be defined prior, to augment repo-specific + settings. The keys gitweb.owner, gitweb.category, gitweb.description, + and gitweb.homepage will map to the cgit keys repo.owner, repo.section, + repo.desc, and repo.homepage respectively. All git config keys that begin + with "cgit." will be mapped to the corresponding "repo." key in cgit. + Default value: "0". See also: scan-path, section-from-path. + +enable-http-clone:: + If set to "1", cgit will act as a dumb HTTP endpoint for git clones. + You can add "http://$HTTP_HOST$SCRIPT_NAME/$CGIT_REPO_URL" to clone-url + to expose this feature. If you use an alternate way of serving git + repositories, you may wish to disable this. Default value: "1". + +enable-html-serving:: + Flag which, when set to "1", will allow the /plain handler to serve + mimetype headers that result in the file being treated as HTML by the + browser. When set to "0", such file types are returned instead as + text/plain or application/octet-stream. Default value: "0". See also: + "repo.enable-html-serving". + +enable-index-links:: + Flag which, when set to "1", will make cgit generate extra links for + each repo in the repository index (specifically, to the "summary", + "commit" and "tree" pages). Default value: "0". + +enable-index-owner:: + Flag which, when set to "1", will make cgit display the owner of + each repo in the repository index. Default value: "1". + +enable-log-filecount:: + Flag which, when set to "1", will make cgit print the number of + modified files for each commit on the repository log page. Default + value: "0". + +enable-log-linecount:: + Flag which, when set to "1", will make cgit print the number of added + and removed lines for each commit on the repository log page. Default + value: "0". + +enable-remote-branches:: + Flag which, when set to "1", will make cgit display remote branches + in the summary and refs views. Default value: "0". See also: + "repo.enable-remote-branches". + +enable-subject-links:: + Flag which, when set to "1", will make cgit use the subject of the + parent commit as link text when generating links to parent commits + in commit view. Default value: "0". See also: + "repo.enable-subject-links". + +enable-tree-linenumbers:: + Flag which, when set to "1", will make cgit generate linenumber links + for plaintext blobs printed in the tree view. Default value: "1". + +favicon:: + Url used as link to a shortcut icon for cgit. It is suggested to use + the value "/favicon.ico" since certain browsers will ignore other + values. Default value: "/favicon.ico". + +footer:: + The content of the file specified with this option will be included + verbatim at the bottom of all pages (i.e. it replaces the standard + "generated by..." message. Default value: none. + +head-include:: + The content of the file specified with this option will be included + verbatim in the html HEAD section on all pages. Default value: none. + +header:: + The content of the file specified with this option will be included + verbatim at the top of all pages. Default value: none. + +include:: + Name of a configfile to include before the rest of the current config- + file is parsed. Default value: none. See also: "MACRO EXPANSION". + +local-time:: + Flag which, if set to "1", makes cgit print commit and tag times in the + servers timezone. Default value: "0". + +logo:: + Url which specifies the source of an image which will be used as a logo + on all cgit pages. Default value: "/cgit.png". + +logo-link:: + Url loaded when clicking on the cgit logo image. If unspecified the + calculated url of the repository index page will be used. Default + value: none. + +max-atom-items:: + Specifies the number of items to display in atom feeds view. Default + value: "10". + +max-blob-size:: + Specifies the maximum size of a blob to display HTML for in KBytes. + Default value: "0" (limit disabled). + +max-commit-count:: + Specifies the number of entries to list per page in "log" view. Default + value: "50". + +max-message-length:: + Specifies the maximum number of commit message characters to display in + "log" view. Default value: "80". + +max-repo-count:: + Specifies the number of entries to list per page on the repository + index page. Default value: "50". + +max-repodesc-length:: + Specifies the maximum number of repo description characters to display + on the repository index page. Default value: "80". + +max-stats:: + Set the default maximum statistics period. Valid values are "week", + "month", "quarter" and "year". If unspecified, statistics are + disabled. Default value: none. See also: "repo.max-stats". + +mimetype.<ext>:: + Set the mimetype for the specified filename extension. This is used + by the `plain` command when returning blob content. + +mimetype-file:: + Specifies the file to use for automatic mimetype lookup. If specified + then this field is used as a fallback when no "mimetype.<ext>" match is + found. If unspecified then no such lookup is performed. The typical file + to use on a Linux system is /etc/mime.types. The format of the file must + comply to: + - a comment line is an empty line or a line starting with a hash (#), + optionally preceded by whitespace + - a non-comment line starts with the mimetype (like image/png), followed + by one or more file extensions (like jpg), all separated by whitespace + Default value: none. See also: "mimetype.<ext>". + +module-link:: + Text which will be used as the formatstring for a hyperlink when a + submodule is printed in a directory listing. The arguments for the + formatstring are the path and SHA1 of the submodule commit. Default + value: none. + +noplainemail:: + If set to "1" showing full author email addresses will be disabled. + Default value: "0". + +noheader:: + Flag which, when set to "1", will make cgit omit the standard header + on all pages. Default value: none. See also: "embedded". + +owner-filter:: + Specifies a command which will be invoked to format the Owner + column of the main page. The command will get the owner on STDIN, + and the STDOUT from the command will be included verbatim in the + table. This can be used to link to additional context such as an + owners home page. When active this filter is used instead of the + default owner query url. Default value: none. + See also: "FILTER API". + +project-list:: + A list of subdirectories inside of scan-path, relative to it, that + should loaded as git repositories. This must be defined prior to + scan-path. Default value: none. See also: scan-path, "MACRO + EXPANSION". + +readme:: + Text which will be used as default value for "repo.readme". Multiple + config keys may be specified, and cgit will use the first found file + in this list. This is useful in conjunction with scan-path. Default + value: none. See also: scan-path, repo.readme. + +remove-suffix:: + If set to "1" and scan-path is enabled, if any repositories are found + with a suffix of ".git", this suffix will be removed for the url and + name. This must be defined prior to scan-path. Default value: "0". + See also: scan-path. + +renamelimit:: + Maximum number of files to consider when detecting renames. The value + "-1" uses the compiletime value in git (for further info, look at + `man git-diff`). Default value: "-1". + +repository-sort:: + The way in which repositories in each section are sorted. Valid values + are "name" for sorting by the repo name or "age" for sorting by the + most recently updated repository. Default value: "name". See also: + section, case-sensitive-sort, section-sort. + +robots:: + Text used as content for the "robots" meta-tag. Default value: + "index, nofollow". + +root-desc:: + Text printed below the heading on the repository index page. Default + value: "a fast webinterface for the git dscm". + +root-readme:: + The content of the file specified with this option will be included + verbatim below the "about" link on the repository index page. Default + value: none. + +root-title:: + Text printed as heading on the repository index page. Default value: + "Git Repository Browser". + +scan-hidden-path:: + If set to "1" and scan-path is enabled, scan-path will recurse into + directories whose name starts with a period ('.'). Otherwise, + scan-path will stay away from such directories (considered as + "hidden"). Note that this does not apply to the ".git" directory in + non-bare repos. This must be defined prior to scan-path. + Default value: 0. See also: scan-path. + +scan-path:: + A path which will be scanned for repositories. If caching is enabled, + the result will be cached as a cgitrc include-file in the cache + directory. If project-list has been defined prior to scan-path, + scan-path loads only the directories listed in the file pointed to by + project-list. Be advised that only the global settings taken + before the scan-path directive will be applied to each repository. + Default value: none. See also: cache-scanrc-ttl, project-list, + "MACRO EXPANSION". + +section:: + The name of the current repository section - all repositories defined + after this option will inherit the current section name. Default value: + none. + +section-sort:: + Flag which, when set to "1", will sort the sections on the repository + listing by name. Set this flag to "0" if the order in the cgitrc file should + be preserved. Default value: "1". See also: section, + case-sensitive-sort, repository-sort. + +section-from-path:: + A number which, if defined prior to scan-path, specifies how many + path elements from each repo path to use as a default section name. + If negative, cgit will discard the specified number of path elements + above the repo directory. Default value: "0". + +side-by-side-diffs:: + If set to "1" shows side-by-side diffs instead of unidiffs per + default. Default value: "0". + +snapshots:: + Text which specifies the default set of snapshot formats that cgit + generates links for. The value is a space-separated list of zero or + more of the values "tar", "tar.gz", "tar.bz2", "tar.lz", "tar.xz", + "tar.zst" and "zip". The special value "all" enables all snapshot + formats. Default value: none. + All compressors use default settings. Some settings can be influenced + with environment variables, for example set ZSTD_CLEVEL=10 in web + server environment for higher (but slower) zstd compression. + +source-filter:: + Specifies a command which will be invoked to format plaintext blobs + in the tree view. The command will get the blob content on its STDIN + and the name of the blob as its only command line argument. The STDOUT + from the command will be included verbatim as the blob contents, i.e. + this can be used to implement e.g. syntax highlighting. Default value: + none. See also: "FILTER API". + +summary-branches:: + Specifies the number of branches to display in the repository "summary" + view. Default value: "10". + +summary-log:: + Specifies the number of log entries to display in the repository + "summary" view. Default value: "10". + +summary-tags:: + Specifies the number of tags to display in the repository "summary" + view. Default value: "10". + +strict-export:: + Filename which, if specified, needs to be present within the repository + for cgit to allow access to that repository. This can be used to emulate + gitweb's EXPORT_OK and STRICT_EXPORT functionality and limit cgit's + repositories to match those exported by git-daemon. This option must + be defined prior to scan-path. + +virtual-root:: + Url which, if specified, will be used as root for all cgit links. It + will also cause cgit to generate 'virtual urls', i.e. urls like + '/cgit/tree/README' as opposed to '?r=cgit&p=tree&path=README'. Default + value: none. + NOTE: cgit has recently learned how to use PATH_INFO to achieve the + same kind of virtual urls, so this option will probably be deprecated. + + +REPOSITORY SETTINGS +------------------- +repo.about-filter:: + Override the default about-filter. Default value: none. See also: + "enable-filter-overrides". See also: "FILTER API". + +repo.branch-sort:: + Flag which, when set to "age", enables date ordering in the branch ref + list, and when set to "name" enables ordering by branch name. Default + value: "name". + +repo.clone-url:: + A list of space-separated urls which can be used to clone this repo. + Default value: none. See also: "MACRO EXPANSION". + +repo.commit-filter:: + Override the default commit-filter. Default value: none. See also: + "enable-filter-overrides". See also: "FILTER API". + +repo.commit-sort:: + Flag which, when set to "date", enables strict date ordering in the + commit log, and when set to "topo" enables strict topological + ordering. If unset, the default ordering of "git log" is used. Default + value: unset. + +repo.defbranch:: + The name of the default branch for this repository. If no such branch + exists in the repository, the first branch name (when sorted) is used + as default instead. Default value: branch pointed to by HEAD, or + "master" if there is no suitable HEAD. + +repo.desc:: + The value to show as repository description. Default value: none. + +repo.email-filter:: + Override the default email-filter. Default value: none. See also: + "enable-filter-overrides". See also: "FILTER API". + +repo.enable-blame:: + A flag which can be used to disable the global setting + `enable-blame'. Default value: none. + +repo.enable-commit-graph:: + A flag which can be used to disable the global setting + `enable-commit-graph'. Default value: none. + +repo.enable-html-serving:: + A flag which can be used to override the global setting + `enable-html-serving`. Default value: none. + +repo.enable-log-filecount:: + A flag which can be used to disable the global setting + `enable-log-filecount'. Default value: none. + +repo.enable-log-linecount:: + A flag which can be used to disable the global setting + `enable-log-linecount'. Default value: none. + +repo.enable-remote-branches:: + Flag which, when set to "1", will make cgit display remote branches + in the summary and refs views. Default value: <enable-remote-branches>. + +repo.enable-subject-links:: + A flag which can be used to override the global setting + `enable-subject-links'. Default value: none. + +repo.extra-head-content:: + This value will be added verbatim to the head section of each page + displayed for this repo. Default value: none. + +repo.hide:: + Flag which, when set to "1", hides the repository from the repository + index. The repository can still be accessed by providing a direct path. + Default value: "0". See also: "repo.ignore". + +repo.homepage:: + The value to show as repository homepage. Default value: none. + +repo.ignore:: + Flag which, when set to "1", ignores the repository. The repository + is not shown in the index and cannot be accessed by providing a direct + path. Default value: "0". See also: "repo.hide". + +repo.logo:: + Url which specifies the source of an image which will be used as a logo + on this repo's pages. Default value: global logo. + +repo.logo-link:: + Url loaded when clicking on the cgit logo image. If unspecified the + calculated url of the repository index page will be used. Default + value: global logo-link. + +repo.module-link:: + Text which will be used as the formatstring for a hyperlink when a + submodule is printed in a directory listing. The arguments for the + formatstring are the path and SHA1 of the submodule commit. Default + value: <module-link> + +repo.module-link.<path>:: + Text which will be used as the formatstring for a hyperlink when a + submodule with the specified subdirectory path is printed in a + directory listing. The only argument for the formatstring is the SHA1 + of the submodule commit. Default value: none. + +repo.max-stats:: + Override the default maximum statistics period. Valid values are equal + to the values specified for the global "max-stats" setting. Default + value: none. + +repo.name:: + The value to show as repository name. Default value: <repo.url>. + +repo.owner:: + A value used to identify the owner of the repository. Default value: + none. + +repo.owner-filter:: + Override the default owner-filter. Default value: none. See also: + "enable-filter-overrides". See also: "FILTER API". + +repo.path:: + An absolute path to the repository directory. For non-bare repositories + this is the .git-directory. Default value: none. + +repo.readme:: + A path (relative to <repo.path>) which specifies a file to include + verbatim as the "About" page for this repo. You may also specify a + git refspec by head or by hash by prepending the refspec followed by + a colon. For example, "master:docs/readme.mkd". If the value begins + with a colon, i.e. ":docs/readme.rst", the default branch of the + repository will be used. Sharing any file will expose that entire + directory tree to the "/about/PATH" endpoints, so be sure that there + are no non-public files located in the same directory as the readme + file. Default value: <readme>. + +repo.section:: + Override the current section name for this repository. Default value: + none. + +repo.snapshots:: + A mask of snapshot formats for this repo that cgit generates links for, + restricted by the global "snapshots" setting. Default value: + <snapshots>. + +repo.snapshot-prefix:: + Prefix to use for snapshot links instead of the repository basename. + For example, the "linux-stable" repository may wish to set this to + "linux" so that snapshots are in the format "linux-3.15.4" instead + of "linux-stable-3.15.4". Default value: <empty> meaning to use + the repository basename. + +repo.source-filter:: + Override the default source-filter. Default value: none. See also: + "enable-filter-overrides". See also: "FILTER API". + +repo.url:: + The relative url used to access the repository. This must be the first + setting specified for each repo. Default value: none. + + +REPOSITORY-SPECIFIC CGITRC FILE +------------------------------- +When the option "scan-path" is used to auto-discover git repositories, cgit +will try to parse the file "cgitrc" within any found repository. Such a +repo-specific config file may contain any of the repo-specific options +described above, except "repo.url" and "repo.path". Additionally, the "filter" +options are only acknowledged in repo-specific config files when +"enable-filter-overrides" is set to "1". + +Note: the "repo." prefix is dropped from the option names in repo-specific +config files, e.g. "repo.desc" becomes "desc". + + +FILTER API +---------- +By default, filters are separate processes that are executed each time they +are needed. Alternative technologies may be used by prefixing the filter +specification with the relevant string; available values are: + +'exec:':: + The default "one process per filter" mode. + +'lua:':: + Executes the script using a built-in Lua interpreter. The script is + loaded once per execution of cgit, and may be called multiple times + during cgit's lifetime, making it a good choice for repeated filters + such as the 'email filter'. It responds to three functions: + + 'filter_open(argument1, argument2, argument3, ...)':: + This is called upon activation of the filter for a particular + set of data. + 'filter_write(buffer)':: + This is called whenever cgit writes data to the webpage. + 'filter_close()':: + This is called when the current filtering operation is + completed. It must return an integer value. Usually 0 + indicates success. + + Additionally, cgit exposes to the Lua the following built-in functions: + + 'html(str)':: + Writes 'str' to the webpage. + 'html_txt(str)':: + HTML escapes and writes 'str' to the webpage. + 'html_attr(str)':: + HTML escapes for an attribute and writes "str' to the webpage. + 'html_url_path(str)':: + URL escapes for a path and writes 'str' to the webpage. + 'html_url_arg(str)':: + URL escapes for an argument and writes 'str' to the webpage. + 'html_include(file)':: + Includes 'file' in webpage. + + +Parameters are provided to filters as follows. + +about filter:: + This filter is given a single parameter: the filename of the source + file to filter. The filter can use the filename to determine (for + example) the type of syntax to follow when formatting the readme file. + The about text that is to be filtered is available on standard input + and the filtered text is expected on standard output. + +auth filter:: + The authentication filter receives 12 parameters: + - filter action, explained below, which specifies which action the + filter is called for + - http cookie + - http method + - http referer + - http path + - http https flag + - cgit repo + - cgit page + - cgit url + - cgit login url + When the filter action is "body", this filter must write to output the + HTML for displaying the login form, which POSTs to the login url. When + the filter action is "authenticate-cookie", this filter must validate + the http cookie and return a 0 if it is invalid or 1 if it is invalid, + in the exit code / close function. If the filter action is + "authenticate-post", this filter receives POST'd parameters on + standard input, and should write a complete CGI response, preferably + with a 302 redirect, and write to output one or more "Set-Cookie" + HTTP headers, each followed by a newline. + + Please see `filters/simple-authentication.lua` for a clear example + script that may be modified. + +commit filter:: + This filter is given no arguments. The commit message text that is to + be filtered is available on standard input and the filtered text is + expected on standard output. + +email filter:: + This filter is given two parameters: the email address of the relevant + author and a string indicating the originating page. The filter will + then receive the text string to format on standard input and is + expected to write to standard output the formatted text to be included + in the page. + +owner filter:: + This filter is given no arguments. The owner text is available on + standard input and the filter is expected to write to standard + output. The output is included in the Owner column. + +source filter:: + This filter is given a single parameter: the filename of the source + file to filter. The filter can use the filename to determine (for + example) the syntax highlighting mode. The contents of the source + file that is to be filtered is available on standard input and the + filtered contents is expected on standard output. + + +All filters are handed the following environment variables: + +- CGIT_REPO_URL (from repo.url) +- CGIT_REPO_NAME (from repo.name) +- CGIT_REPO_PATH (from repo.path) +- CGIT_REPO_OWNER (from repo.owner) +- CGIT_REPO_DEFBRANCH (from repo.defbranch) +- CGIT_REPO_SECTION (from repo.section) +- CGIT_REPO_CLONE_URL (from repo.clone-url) + +If a setting is not defined for a repository and the corresponding global +setting is also not defined (if applicable), then the corresponding +environment variable will be unset. + + +MACRO EXPANSION +--------------- +The following cgitrc options support a simple macro expansion feature, +where tokens prefixed with "$" are replaced with the value of a similarly +named environment variable: + +- cache-root +- include +- project-list +- scan-path + +Macro expansion will also happen on the content of $CGIT_CONFIG, if +defined. + +One usage of this feature is virtual hosting, which in its simplest form +can be accomplished by adding the following line to /etc/cgitrc: + + include=/etc/cgitrc.d/$HTTP_HOST + +The following options are expanded during request processing, and support +the environment variables defined in "FILTER API": + +- clone-url +- repo.clone-url + + +CACHE +----- + +All cache ttl values are in minutes. Negative ttl values indicate that a page +type will never expire, and thus the first time a URL is accessed, the result +will be cached indefinitely, even if the underlying git repository changes. +Conversely, when a ttl value is zero, the cache is disabled for that +particular page type, and the page type is never cached. + +SIGNATURES +---------- + +Cgit can host .asc signatures corresponding to various snapshot formats, +through use of git notes. For example, the following command may be used to +add a signature to a .tar.xz archive: + + git notes --ref=refs/notes/signatures/tar.xz add -C "$( + gpg --output - --armor --detach-sign cgit-1.1.tar.xz | + git hash-object -w --stdin + )" v1.1 + +If it is instead desirable to attach a signature of the underlying .tar, this +will be linked, as a special case, beside a .tar.* link that does not have its +own signature. For example, a signature of a tarball of the latest tag might +be added with a similar command: + + tag="$(git describe --abbrev=0)" + git notes --ref=refs/notes/signatures/tar add -C "$( + git archive --format tar --prefix "cgit-${tag#v}/" "$tag" | + gpg --output - --armor --detach-sign | + git hash-object -w --stdin + )" "$tag" + +Since git-archive(1) is expected to produce stable output between versions, +this allows one to generate a long-term signature of the contents of a given +tag. + +EXAMPLE CGITRC FILE +------------------- + +.... +# Enable caching of up to 1000 output entries +cache-size=1000 + + +# Specify some default clone urls using macro expansion +clone-url=git://foo.org/$CGIT_REPO_URL git@foo.org:$CGIT_REPO_URL + +# Specify the css url +css=/css/cgit.css + + +# Show owner on index page +enable-index-owner=1 + + +# Allow http transport git clone +enable-http-clone=1 + + +# Show extra links for each repository on the index page +enable-index-links=1 + + +# Enable blame page and create links to it from tree page +enable-blame=1 + + +# Enable ASCII art commit history graph on the log pages +enable-commit-graph=1 + + +# Show number of affected files per commit on the log pages +enable-log-filecount=1 + + +# Show number of added/removed lines per commit on the log pages +enable-log-linecount=1 + + +# Sort branches by date +branch-sort=age + + +# Add a cgit favicon +favicon=/favicon.ico + + +# Use a custom logo +logo=/img/mylogo.png + + +# Enable statistics per week, month and quarter +max-stats=quarter + + +# Set the title and heading of the repository index page +root-title=example.com git repositories + + +# Set a subheading for the repository index page +root-desc=tracking the foobar development + + +# Include some more info about example.com on the index page +root-readme=/var/www/htdocs/about.html + + +# Allow download of tar.gz, tar.bz2 and zip-files +snapshots=tar.gz tar.bz2 zip + + +## +## List of common mimetypes +## + +mimetype.gif=image/gif +mimetype.html=text/html +mimetype.jpg=image/jpeg +mimetype.jpeg=image/jpeg +mimetype.pdf=application/pdf +mimetype.png=image/png +mimetype.svg=image/svg+xml + + +# Highlight source code with python pygments-based highlighter +source-filter=/var/www/cgit/filters/syntax-highlighting.py + +# Format markdown, restructuredtext, manpages, text files, and html files +# through the right converters +about-filter=/var/www/cgit/filters/about-formatting.sh + +## +## Search for these files in the root of the default branch of repositories +## for coming up with the about page: +## +readme=:README.md +readme=:readme.md +readme=:README.mkd +readme=:readme.mkd +readme=:README.rst +readme=:readme.rst +readme=:README.html +readme=:readme.html +readme=:README.htm +readme=:readme.htm +readme=:README.txt +readme=:readme.txt +readme=:README +readme=:readme +readme=:INSTALL.md +readme=:install.md +readme=:INSTALL.mkd +readme=:install.mkd +readme=:INSTALL.rst +readme=:install.rst +readme=:INSTALL.html +readme=:install.html +readme=:INSTALL.htm +readme=:install.htm +readme=:INSTALL.txt +readme=:install.txt +readme=:INSTALL +readme=:install + + +## +## List of repositories. +## PS: Any repositories listed when section is unset will not be +## displayed under a section heading +## PPS: This list could be kept in a different file (e.g. '/etc/cgitrepos') +## and included like this: +## include=/etc/cgitrepos +## + + +repo.url=foo +repo.path=/pub/git/foo.git +repo.desc=the master foo repository +repo.owner=fooman@example.com +repo.readme=info/web/about.html + + +repo.url=bar +repo.path=/pub/git/bar.git +repo.desc=the bars for your foo +repo.owner=barman@example.com +repo.readme=info/web/about.html + + +# The next repositories will be displayed under the 'extras' heading +section=extras + + +repo.url=baz +repo.path=/pub/git/baz.git +repo.desc=a set of extensions for bar users + +repo.url=wiz +repo.path=/pub/git/wiz.git +repo.desc=the wizard of foo + + +# Add some mirrored repositories +section=mirrors + + +repo.url=git +repo.path=/pub/git/git.git +repo.desc=the dscm + + +repo.url=linux +repo.path=/pub/git/linux.git +repo.desc=the kernel + +# Disable adhoc downloads of this repo +repo.snapshots=0 + +# Disable line-counts for this repo +repo.enable-log-linecount=0 + +# Restrict the max statistics period for this repo +repo.max-stats=month +.... + + +BUGS +---- +Comments currently cannot appear on the same line as a setting; the comment +will be included as part of the value. E.g. this line: + + robots=index # allow indexing + +will generate the following html element: + + <meta name='robots' content='index # allow indexing'/> + + + +AUTHOR +------ +Lars Hjemli <hjemli@gmail.com> +Jason A. Donenfeld <Jason@zx2c4.com> diff --git a/third_party/cgit/cmd.c b/third_party/cgit/cmd.c new file mode 100644 index 000000000000..c664e894f788 --- /dev/null +++ b/third_party/cgit/cmd.c @@ -0,0 +1,186 @@ +/* cmd.c: the cgit command dispatcher + * + * Copyright (C) 2006-2017 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "cmd.h" +#include "cache.h" +#include "ui-shared.h" +#include "ui-atom.h" +#include "ui-blame.h" +#include "ui-blob.h" +#include "ui-clone.h" +#include "ui-commit.h" +#include "ui-diff.h" +#include "ui-log.h" +#include "ui-patch.h" +#include "ui-plain.h" +#include "ui-refs.h" +#include "ui-repolist.h" +#include "ui-snapshot.h" +#include "ui-stats.h" +#include "ui-summary.h" +#include "ui-tag.h" +#include "ui-tree.h" + +static void HEAD_fn(void) +{ + cgit_clone_head(); +} + +static void atom_fn(void) +{ + cgit_print_atom(ctx.qry.head, ctx.qry.path, ctx.cfg.max_atom_items); +} + +static void about_fn(void) +{ + cgit_print_repo_readme(ctx.qry.path); +} + +static void blame_fn(void) +{ + if (ctx.repo->enable_blame) + cgit_print_blame(); + else + cgit_print_error_page(403, "Forbidden", "Blame is disabled"); +} + +static void blob_fn(void) +{ + cgit_print_blob(ctx.qry.oid, ctx.qry.path, ctx.qry.head, 0); +} + +static void commit_fn(void) +{ + cgit_print_commit(ctx.qry.oid, ctx.qry.path); +} + +static void diff_fn(void) +{ + cgit_print_diff(ctx.qry.oid, ctx.qry.oid2, ctx.qry.path, 1, 0); +} + +static void rawdiff_fn(void) +{ + cgit_print_diff(ctx.qry.oid, ctx.qry.oid2, ctx.qry.path, 1, 1); +} + +static void info_fn(void) +{ + cgit_clone_info(); +} + +static void log_fn(void) +{ + cgit_print_log(ctx.qry.oid, ctx.qry.ofs, ctx.cfg.max_commit_count, + ctx.qry.grep, ctx.qry.search, ctx.qry.path, 1, + ctx.repo->enable_commit_graph, + ctx.repo->commit_sort); +} + +static void ls_cache_fn(void) +{ + ctx.page.mimetype = "text/plain"; + ctx.page.filename = "ls-cache.txt"; + cgit_print_http_headers(); + cache_ls(ctx.cfg.cache_root); +} + +static void objects_fn(void) +{ + cgit_clone_objects(); +} + +static void repolist_fn(void) +{ + cgit_print_repolist(); +} + +static void patch_fn(void) +{ + cgit_print_patch(ctx.qry.oid, ctx.qry.oid2, ctx.qry.path); +} + +static void plain_fn(void) +{ + cgit_print_plain(); +} + +static void refs_fn(void) +{ + cgit_print_refs(); +} + +static void snapshot_fn(void) +{ + cgit_print_snapshot(ctx.qry.head, ctx.qry.oid, ctx.qry.path, + ctx.qry.nohead); +} + +static void stats_fn(void) +{ + cgit_show_stats(); +} + +static void summary_fn(void) +{ + cgit_print_summary(); +} + +static void tag_fn(void) +{ + cgit_print_tag(ctx.qry.oid); +} + +static void tree_fn(void) +{ + cgit_print_tree(ctx.qry.oid, ctx.qry.path); +} + +#define def_cmd(name, want_repo, want_vpath, is_clone) \ + {#name, name##_fn, want_repo, want_vpath, is_clone} + +struct cgit_cmd *cgit_get_cmd(void) +{ + static struct cgit_cmd cmds[] = { + def_cmd(HEAD, 1, 0, 1), + def_cmd(atom, 1, 0, 0), + def_cmd(about, 1, 1, 0), + def_cmd(blame, 1, 1, 0), + def_cmd(blob, 1, 0, 0), + def_cmd(commit, 1, 1, 0), + def_cmd(diff, 1, 1, 0), + def_cmd(info, 1, 0, 1), + def_cmd(log, 1, 1, 0), + def_cmd(ls_cache, 0, 0, 0), + def_cmd(objects, 1, 0, 1), + def_cmd(patch, 1, 1, 0), + def_cmd(plain, 1, 0, 0), + def_cmd(rawdiff, 1, 1, 0), + def_cmd(refs, 1, 0, 0), + def_cmd(repolist, 0, 0, 0), + def_cmd(snapshot, 1, 0, 0), + def_cmd(stats, 1, 1, 0), + def_cmd(summary, 1, 0, 0), + def_cmd(tag, 1, 0, 0), + def_cmd(tree, 1, 1, 0), + }; + int i; + + if (ctx.qry.page == NULL) { + if (ctx.repo) + ctx.qry.page = "summary"; + else + ctx.qry.page = "repolist"; + } + + for (i = 0; i < sizeof(cmds)/sizeof(*cmds); i++) + if (!strcmp(ctx.qry.page, cmds[i].name)) + return &cmds[i]; + return NULL; +} diff --git a/third_party/cgit/cmd.h b/third_party/cgit/cmd.h new file mode 100644 index 000000000000..6249b1d89243 --- /dev/null +++ b/third_party/cgit/cmd.h @@ -0,0 +1,16 @@ +#ifndef CMD_H +#define CMD_H + +typedef void (*cgit_cmd_fn)(void); + +struct cgit_cmd { + const char *name; + cgit_cmd_fn fn; + unsigned int want_repo:1, + want_vpath:1, + is_clone:1; +}; + +extern struct cgit_cmd *cgit_get_cmd(void); + +#endif /* CMD_H */ diff --git a/third_party/cgit/configfile.c b/third_party/cgit/configfile.c new file mode 100644 index 000000000000..e0391091e147 --- /dev/null +++ b/third_party/cgit/configfile.c @@ -0,0 +1,90 @@ +/* configfile.c: parsing of config files + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include <git-compat-util.h> +#include "configfile.h" + +static int next_char(FILE *f) +{ + int c = fgetc(f); + if (c == '\r') { + c = fgetc(f); + if (c != '\n') { + ungetc(c, f); + c = '\r'; + } + } + return c; +} + +static void skip_line(FILE *f) +{ + int c; + + while ((c = next_char(f)) && c != '\n' && c != EOF) + ; +} + +static int read_config_line(FILE *f, struct strbuf *name, struct strbuf *value) +{ + int c = next_char(f); + + strbuf_reset(name); + strbuf_reset(value); + + /* Skip comments and preceding spaces. */ + for(;;) { + if (c == EOF) + return 0; + else if (c == '#' || c == ';') + skip_line(f); + else if (!isspace(c)) + break; + c = next_char(f); + } + + /* Read variable name. */ + while (c != '=') { + if (c == '\n' || c == EOF) + return 0; + strbuf_addch(name, c); + c = next_char(f); + } + + /* Read variable value. */ + c = next_char(f); + while (c != '\n' && c != EOF) { + strbuf_addch(value, c); + c = next_char(f); + } + + return 1; +} + +int parse_configfile(const char *filename, configfile_value_fn fn) +{ + static int nesting; + struct strbuf name = STRBUF_INIT; + struct strbuf value = STRBUF_INIT; + FILE *f; + + /* cancel deeply nested include-commands */ + if (nesting > 8) + return -1; + if (!(f = fopen(filename, "r"))) + return -1; + nesting++; + while (read_config_line(f, &name, &value)) + fn(name.buf, value.buf); + nesting--; + fclose(f); + strbuf_release(&name); + strbuf_release(&value); + return 0; +} + diff --git a/third_party/cgit/configfile.h b/third_party/cgit/configfile.h new file mode 100644 index 000000000000..af7ca1973590 --- /dev/null +++ b/third_party/cgit/configfile.h @@ -0,0 +1,10 @@ +#ifndef CONFIGFILE_H +#define CONFIGFILE_H + +#include "cgit.h" + +typedef void (*configfile_value_fn)(const char *name, const char *value); + +extern int parse_configfile(const char *filename, configfile_value_fn fn); + +#endif /* CONFIGFILE_H */ diff --git a/third_party/cgit/contrib/hooks/post-receive.agefile b/third_party/cgit/contrib/hooks/post-receive.agefile new file mode 100755 index 000000000000..2f72ae9c0dd5 --- /dev/null +++ b/third_party/cgit/contrib/hooks/post-receive.agefile @@ -0,0 +1,19 @@ +#!/bin/sh +# +# An example hook to update the "agefile" for CGit's idle time calculation. +# +# This hook assumes that you are using the default agefile location of +# "info/web/last-modified". If you change the value in your cgitrc then you +# must also change it here. +# +# To install the hook, copy (or link) it to the file "hooks/post-receive" in +# each of your repositories. +# + +agefile="$(git rev-parse --git-dir)"/info/web/last-modified + +mkdir -p "$(dirname "$agefile")" && +git for-each-ref \ + --sort=-authordate --count=1 \ + --format='%(authordate:iso8601)' \ + >"$agefile" diff --git a/third_party/cgit/default.nix b/third_party/cgit/default.nix new file mode 100644 index 000000000000..025877ee4b72 --- /dev/null +++ b/third_party/cgit/default.nix @@ -0,0 +1,43 @@ +{ depot, lib, pkgs, ... }: + +let + inherit (pkgs) stdenv gzip bzip2 xz luajit zlib autoconf openssl pkgconfig; +in stdenv.mkDerivation rec { + pname = "cgit"; + version = "master"; + src = ./.; + + nativeBuildInputs = [ autoconf pkgconfig ]; + buildInputs = [ openssl zlib luajit ]; + + enableParallelBuilding = true; + + postPatch = '' + sed -e 's|"gzip"|"${gzip}/bin/gzip"|' \ + -e 's|"bzip2"|"${bzip2.bin}/bin/bzip2"|' \ + -e 's|"xz"|"${xz.bin}/bin/xz"|' \ + -i ui-snapshot.c + ''; + + # Give cgit the git source tree including depot patches. Note that + # the version expected by cgit should be kept in sync with the + # version available in nixpkgs. + # + # TODO(tazjin): Add an assert for this somewhere so we notice it on + # channel bumps. + preBuild = '' + rm -rf git # remove submodule dir ... + cp -r --no-preserve=ownership,mode ${pkgs.srcOnly depot.third_party.git} git + makeFlagsArray+=(prefix="$out" CGIT_SCRIPT_PATH="$out/cgit/") + cat tvl-extra.css >> cgit.css + ''; + + meta = { + homepage = https://git.zx2c4.com/cgit/about/; + repositories.git = git://git.zx2c4.com/cgit; + description = "Web frontend for git repositories"; + license = lib.licenses.gpl2; + platforms = lib.platforms.linux; + maintainers = with lib.maintainers; [ bjornfor ]; + }; +} diff --git a/third_party/cgit/favicon.ico b/third_party/cgit/favicon.ico new file mode 100644 index 000000000000..56ff59384f2c --- /dev/null +++ b/third_party/cgit/favicon.ico Binary files differdiff --git a/third_party/cgit/filter.c b/third_party/cgit/filter.c new file mode 100644 index 000000000000..181d9a892f0b --- /dev/null +++ b/third_party/cgit/filter.c @@ -0,0 +1,457 @@ +/* filter.c: filter framework functions + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "html.h" +#ifndef NO_LUA +#include <dlfcn.h> +#include <lua.h> +#include <lualib.h> +#include <lauxlib.h> +#endif + +static inline void reap_filter(struct cgit_filter *filter) +{ + if (filter && filter->cleanup) + filter->cleanup(filter); +} + +void cgit_cleanup_filters(void) +{ + int i; + reap_filter(ctx.cfg.about_filter); + reap_filter(ctx.cfg.commit_filter); + reap_filter(ctx.cfg.source_filter); + reap_filter(ctx.cfg.email_filter); + reap_filter(ctx.cfg.owner_filter); + reap_filter(ctx.cfg.auth_filter); + for (i = 0; i < cgit_repolist.count; ++i) { + reap_filter(cgit_repolist.repos[i].about_filter); + reap_filter(cgit_repolist.repos[i].commit_filter); + reap_filter(cgit_repolist.repos[i].source_filter); + reap_filter(cgit_repolist.repos[i].email_filter); + reap_filter(cgit_repolist.repos[i].owner_filter); + } +} + +static int open_exec_filter(struct cgit_filter *base, va_list ap) +{ + struct cgit_exec_filter *filter = (struct cgit_exec_filter *)base; + int pipe_fh[2]; + int i; + + for (i = 0; i < filter->base.argument_count; i++) + filter->argv[i + 1] = va_arg(ap, char *); + + filter->old_stdout = chk_positive(dup(STDOUT_FILENO), + "Unable to duplicate STDOUT"); + chk_zero(pipe(pipe_fh), "Unable to create pipe to subprocess"); + filter->pid = chk_non_negative(fork(), "Unable to create subprocess"); + if (filter->pid == 0) { + close(pipe_fh[1]); + chk_non_negative(dup2(pipe_fh[0], STDIN_FILENO), + "Unable to use pipe as STDIN"); + execvp(filter->cmd, filter->argv); + die_errno("Unable to exec subprocess %s", filter->cmd); + } + close(pipe_fh[0]); + chk_non_negative(dup2(pipe_fh[1], STDOUT_FILENO), + "Unable to use pipe as STDOUT"); + close(pipe_fh[1]); + return 0; +} + +static int close_exec_filter(struct cgit_filter *base) +{ + struct cgit_exec_filter *filter = (struct cgit_exec_filter *)base; + int i, exit_status = 0; + + chk_non_negative(dup2(filter->old_stdout, STDOUT_FILENO), + "Unable to restore STDOUT"); + close(filter->old_stdout); + if (filter->pid < 0) + goto done; + waitpid(filter->pid, &exit_status, 0); + if (WIFEXITED(exit_status)) + goto done; + die("Subprocess %s exited abnormally", filter->cmd); + +done: + for (i = 0; i < filter->base.argument_count; i++) + filter->argv[i + 1] = NULL; + return WEXITSTATUS(exit_status); + +} + +static void fprintf_exec_filter(struct cgit_filter *base, FILE *f, const char *prefix) +{ + struct cgit_exec_filter *filter = (struct cgit_exec_filter *)base; + fprintf(f, "%sexec:%s\n", prefix, filter->cmd); +} + +static void cleanup_exec_filter(struct cgit_filter *base) +{ + struct cgit_exec_filter *filter = (struct cgit_exec_filter *)base; + if (filter->argv) { + free(filter->argv); + filter->argv = NULL; + } + if (filter->cmd) { + free(filter->cmd); + filter->cmd = NULL; + } +} + +static struct cgit_filter *new_exec_filter(const char *cmd, int argument_count) +{ + struct cgit_exec_filter *f; + int args_size = 0; + + f = xmalloc(sizeof(*f)); + /* We leave argv for now and assign it below. */ + cgit_exec_filter_init(f, xstrdup(cmd), NULL); + f->base.argument_count = argument_count; + args_size = (2 + argument_count) * sizeof(char *); + f->argv = xmalloc(args_size); + memset(f->argv, 0, args_size); + f->argv[0] = f->cmd; + return &f->base; +} + +void cgit_exec_filter_init(struct cgit_exec_filter *filter, char *cmd, char **argv) +{ + memset(filter, 0, sizeof(*filter)); + filter->base.open = open_exec_filter; + filter->base.close = close_exec_filter; + filter->base.fprintf = fprintf_exec_filter; + filter->base.cleanup = cleanup_exec_filter; + filter->cmd = cmd; + filter->argv = argv; + /* The argument count for open_filter is zero by default, unless called from new_filter, above. */ + filter->base.argument_count = 0; +} + +#ifdef NO_LUA +void cgit_init_filters(void) +{ +} +#endif + +#ifndef NO_LUA +static ssize_t (*libc_write)(int fd, const void *buf, size_t count); +static ssize_t (*filter_write)(struct cgit_filter *base, const void *buf, size_t count) = NULL; +static struct cgit_filter *current_write_filter = NULL; + +void cgit_init_filters(void) +{ + libc_write = dlsym(RTLD_NEXT, "write"); + if (!libc_write) + die("Could not locate libc's write function"); +} + +ssize_t write(int fd, const void *buf, size_t count) +{ + if (fd != STDOUT_FILENO || !filter_write) + return libc_write(fd, buf, count); + return filter_write(current_write_filter, buf, count); +} + +static inline void hook_write(struct cgit_filter *filter, ssize_t (*new_write)(struct cgit_filter *base, const void *buf, size_t count)) +{ + /* We want to avoid buggy nested patterns. */ + assert(filter_write == NULL); + assert(current_write_filter == NULL); + current_write_filter = filter; + filter_write = new_write; +} + +static inline void unhook_write(void) +{ + assert(filter_write != NULL); + assert(current_write_filter != NULL); + filter_write = NULL; + current_write_filter = NULL; +} + +struct lua_filter { + struct cgit_filter base; + char *script_file; + lua_State *lua_state; +}; + +static void error_lua_filter(struct lua_filter *filter) +{ + die("Lua error in %s: %s", filter->script_file, lua_tostring(filter->lua_state, -1)); + lua_pop(filter->lua_state, 1); +} + +static ssize_t write_lua_filter(struct cgit_filter *base, const void *buf, size_t count) +{ + struct lua_filter *filter = (struct lua_filter *)base; + + lua_getglobal(filter->lua_state, "filter_write"); + lua_pushlstring(filter->lua_state, buf, count); + if (lua_pcall(filter->lua_state, 1, 0, 0)) { + error_lua_filter(filter); + errno = EIO; + return -1; + } + return count; +} + +static inline int hook_lua_filter(lua_State *lua_state, void (*fn)(const char *txt)) +{ + const char *str; + ssize_t (*save_filter_write)(struct cgit_filter *base, const void *buf, size_t count); + struct cgit_filter *save_filter; + + str = lua_tostring(lua_state, 1); + if (!str) + return 0; + + save_filter_write = filter_write; + save_filter = current_write_filter; + unhook_write(); + fn(str); + hook_write(save_filter, save_filter_write); + + return 0; +} + +static int html_lua_filter(lua_State *lua_state) +{ + return hook_lua_filter(lua_state, html); +} + +static int html_txt_lua_filter(lua_State *lua_state) +{ + return hook_lua_filter(lua_state, html_txt); +} + +static int html_attr_lua_filter(lua_State *lua_state) +{ + return hook_lua_filter(lua_state, html_attr); +} + +static int html_url_path_lua_filter(lua_State *lua_state) +{ + return hook_lua_filter(lua_state, html_url_path); +} + +static int html_url_arg_lua_filter(lua_State *lua_state) +{ + return hook_lua_filter(lua_state, html_url_arg); +} + +static int html_include_lua_filter(lua_State *lua_state) +{ + return hook_lua_filter(lua_state, (void (*)(const char *))html_include); +} + +static void cleanup_lua_filter(struct cgit_filter *base) +{ + struct lua_filter *filter = (struct lua_filter *)base; + + if (!filter->lua_state) + return; + + lua_close(filter->lua_state); + filter->lua_state = NULL; + if (filter->script_file) { + free(filter->script_file); + filter->script_file = NULL; + } +} + +static int init_lua_filter(struct lua_filter *filter) +{ + if (filter->lua_state) + return 0; + + if (!(filter->lua_state = luaL_newstate())) + return 1; + + luaL_openlibs(filter->lua_state); + + lua_pushcfunction(filter->lua_state, html_lua_filter); + lua_setglobal(filter->lua_state, "html"); + lua_pushcfunction(filter->lua_state, html_txt_lua_filter); + lua_setglobal(filter->lua_state, "html_txt"); + lua_pushcfunction(filter->lua_state, html_attr_lua_filter); + lua_setglobal(filter->lua_state, "html_attr"); + lua_pushcfunction(filter->lua_state, html_url_path_lua_filter); + lua_setglobal(filter->lua_state, "html_url_path"); + lua_pushcfunction(filter->lua_state, html_url_arg_lua_filter); + lua_setglobal(filter->lua_state, "html_url_arg"); + lua_pushcfunction(filter->lua_state, html_include_lua_filter); + lua_setglobal(filter->lua_state, "html_include"); + + if (luaL_dofile(filter->lua_state, filter->script_file)) { + error_lua_filter(filter); + lua_close(filter->lua_state); + filter->lua_state = NULL; + return 1; + } + return 0; +} + +static int open_lua_filter(struct cgit_filter *base, va_list ap) +{ + struct lua_filter *filter = (struct lua_filter *)base; + int i; + + if (init_lua_filter(filter)) + return 1; + + hook_write(base, write_lua_filter); + + lua_getglobal(filter->lua_state, "filter_open"); + for (i = 0; i < filter->base.argument_count; ++i) + lua_pushstring(filter->lua_state, va_arg(ap, char *)); + if (lua_pcall(filter->lua_state, filter->base.argument_count, 0, 0)) { + error_lua_filter(filter); + return 1; + } + return 0; +} + +static int close_lua_filter(struct cgit_filter *base) +{ + struct lua_filter *filter = (struct lua_filter *)base; + int ret = 0; + + lua_getglobal(filter->lua_state, "filter_close"); + if (lua_pcall(filter->lua_state, 0, 1, 0)) { + error_lua_filter(filter); + ret = -1; + } else { + ret = lua_tonumber(filter->lua_state, -1); + lua_pop(filter->lua_state, 1); + } + + unhook_write(); + return ret; +} + +static void fprintf_lua_filter(struct cgit_filter *base, FILE *f, const char *prefix) +{ + struct lua_filter *filter = (struct lua_filter *)base; + fprintf(f, "%slua:%s\n", prefix, filter->script_file); +} + + +static struct cgit_filter *new_lua_filter(const char *cmd, int argument_count) +{ + struct lua_filter *filter; + + filter = xmalloc(sizeof(*filter)); + memset(filter, 0, sizeof(*filter)); + filter->base.open = open_lua_filter; + filter->base.close = close_lua_filter; + filter->base.fprintf = fprintf_lua_filter; + filter->base.cleanup = cleanup_lua_filter; + filter->base.argument_count = argument_count; + filter->script_file = xstrdup(cmd); + + return &filter->base; +} + +#endif + + +int cgit_open_filter(struct cgit_filter *filter, ...) +{ + int result; + va_list ap; + if (!filter) + return 0; + va_start(ap, filter); + result = filter->open(filter, ap); + va_end(ap); + return result; +} + +int cgit_close_filter(struct cgit_filter *filter) +{ + if (!filter) + return 0; + return filter->close(filter); +} + +void cgit_fprintf_filter(struct cgit_filter *filter, FILE *f, const char *prefix) +{ + (filter->fprintf)(filter, f, prefix); +} + + + +static const struct { + const char *prefix; + struct cgit_filter *(*ctor)(const char *cmd, int argument_count); +} filter_specs[] = { + { "exec", new_exec_filter }, +#ifndef NO_LUA + { "lua", new_lua_filter }, +#endif +}; + +struct cgit_filter *cgit_new_filter(const char *cmd, filter_type filtertype) +{ + char *colon; + int i; + size_t len; + int argument_count; + + if (!cmd || !cmd[0]) + return NULL; + + colon = strchr(cmd, ':'); + len = colon - cmd; + /* + * In case we're running on Windows, don't allow a single letter before + * the colon. + */ + if (len == 1) + colon = NULL; + + switch (filtertype) { + case AUTH: + argument_count = 12; + break; + + case EMAIL: + argument_count = 2; + break; + + case OWNER: + argument_count = 0; + break; + + case SOURCE: + case ABOUT: + argument_count = 1; + break; + + case COMMIT: + default: + argument_count = 0; + break; + } + + /* If no prefix is given, exec filter is the default. */ + if (!colon) + return new_exec_filter(cmd, argument_count); + + for (i = 0; i < ARRAY_SIZE(filter_specs); i++) { + if (len == strlen(filter_specs[i].prefix) && + !strncmp(filter_specs[i].prefix, cmd, len)) + return filter_specs[i].ctor(colon + 1, argument_count); + } + + die("Invalid filter type: %.*s", (int) len, cmd); +} diff --git a/third_party/cgit/filters/about-formatting.sh b/third_party/cgit/filters/about-formatting.sh new file mode 100755 index 000000000000..85daf9c26b07 --- /dev/null +++ b/third_party/cgit/filters/about-formatting.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# This may be used with the about-filter or repo.about-filter setting in cgitrc. +# It passes formatting of about pages to differing programs, depending on the usage. + +# Markdown support requires python and markdown-python. +# RestructuredText support requires python and docutils. +# Man page support requires groff. + +# The following environment variables can be used to retrieve the configuration +# of the repository for which this script is called: +# CGIT_REPO_URL ( = repo.url setting ) +# CGIT_REPO_NAME ( = repo.name setting ) +# CGIT_REPO_PATH ( = repo.path setting ) +# CGIT_REPO_OWNER ( = repo.owner setting ) +# CGIT_REPO_DEFBRANCH ( = repo.defbranch setting ) +# CGIT_REPO_SECTION ( = section setting ) +# CGIT_REPO_CLONE_URL ( = repo.clone-url setting ) + +cd "$(dirname $0)/html-converters/" +case "$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]')" in + *.markdown|*.mdown|*.md|*.mkd) exec ./md2html; ;; + *.rst) exec ./rst2html; ;; + *.[1-9]) exec ./man2html; ;; + *.htm|*.html) exec cat; ;; + *.txt|*) exec ./txt2html; ;; +esac diff --git a/third_party/cgit/filters/commit-links.sh b/third_party/cgit/filters/commit-links.sh new file mode 100755 index 000000000000..796ac308d237 --- /dev/null +++ b/third_party/cgit/filters/commit-links.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# This script can be used to generate links in commit messages. +# +# To use this script, refer to this file with either the commit-filter or the +# repo.commit-filter options in cgitrc. +# +# The following environment variables can be used to retrieve the configuration +# of the repository for which this script is called: +# CGIT_REPO_URL ( = repo.url setting ) +# CGIT_REPO_NAME ( = repo.name setting ) +# CGIT_REPO_PATH ( = repo.path setting ) +# CGIT_REPO_OWNER ( = repo.owner setting ) +# CGIT_REPO_DEFBRANCH ( = repo.defbranch setting ) +# CGIT_REPO_SECTION ( = section setting ) +# CGIT_REPO_CLONE_URL ( = repo.clone-url setting ) +# + +regex='' + +# This expression generates links to commits referenced by their SHA1. +regex=$regex' +s|\b([0-9a-fA-F]{7,64})\b|<a href="./?id=\1">\1</a>|g' + +# This expression generates links to a fictional bugtracker. +regex=$regex' +s|#([0-9]+)\b|<a href="http://bugs.example.com/?bug=\1">#\1</a>|g' + +sed -re "$regex" diff --git a/third_party/cgit/filters/email-gravatar.lua b/third_party/cgit/filters/email-gravatar.lua new file mode 100644 index 000000000000..c39b490d6bb6 --- /dev/null +++ b/third_party/cgit/filters/email-gravatar.lua @@ -0,0 +1,35 @@ +-- This script may be used with the email-filter or repo.email-filter settings in cgitrc. +-- It adds gravatar icons to author names. It is designed to be used with the lua: +-- prefix in filters. It is much faster than the corresponding python script. +-- +-- Requirements: +-- luaossl +-- <http://25thandclement.com/~william/projects/luaossl.html> +-- + +local digest = require("openssl.digest") + +function md5_hex(input) + local b = digest.new("md5"):final(input) + local x = "" + for i = 1, #b do + x = x .. string.format("%.2x", string.byte(b, i)) + end + return x +end + +function filter_open(email, page) + buffer = "" + md5 = md5_hex(email:sub(2, -2):lower()) +end + +function filter_close() + html("<img src='//www.gravatar.com/avatar/" .. md5 .. "?s=13&d=retro' width='13' height='13' alt='Gravatar' /> " .. buffer) + return 0 +end + +function filter_write(str) + buffer = buffer .. str +end + + diff --git a/third_party/cgit/filters/email-gravatar.py b/third_party/cgit/filters/email-gravatar.py new file mode 100755 index 000000000000..d70440ea543c --- /dev/null +++ b/third_party/cgit/filters/email-gravatar.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Please prefer the email-gravatar.lua using lua: as a prefix over this script. This +# script is very slow, in comparison. +# +# This script may be used with the email-filter or repo.email-filter settings in cgitrc. +# +# The following environment variables can be used to retrieve the configuration +# of the repository for which this script is called: +# CGIT_REPO_URL ( = repo.url setting ) +# CGIT_REPO_NAME ( = repo.name setting ) +# CGIT_REPO_PATH ( = repo.path setting ) +# CGIT_REPO_OWNER ( = repo.owner setting ) +# CGIT_REPO_DEFBRANCH ( = repo.defbranch setting ) +# CGIT_REPO_SECTION ( = section setting ) +# CGIT_REPO_CLONE_URL ( = repo.clone-url setting ) +# +# It receives an email address on argv[1] and text on stdin. It prints +# to stdout that text prepended by a gravatar at 10pt. + +import sys +import hashlib +import codecs + +email = sys.argv[1].lower().strip() +if email[0] == '<': + email = email[1:] +if email[-1] == '>': + email = email[0:-1] + +page = sys.argv[2] + +sys.stdin = codecs.getreader("utf-8")(sys.stdin.detach()) +sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach()) + +md5 = hashlib.md5(email.encode()).hexdigest() +text = sys.stdin.read().strip() + +print("<img src='//www.gravatar.com/avatar/" + md5 + "?s=13&d=retro' width='13' height='13' alt='Gravatar' /> " + text) diff --git a/third_party/cgit/filters/email-libravatar.lua b/third_party/cgit/filters/email-libravatar.lua new file mode 100644 index 000000000000..7336baf830a1 --- /dev/null +++ b/third_party/cgit/filters/email-libravatar.lua @@ -0,0 +1,36 @@ +-- This script may be used with the email-filter or repo.email-filter settings in cgitrc. +-- It adds libravatar icons to author names. It is designed to be used with the lua: +-- prefix in filters. +-- +-- Requirements: +-- luaossl +-- <http://25thandclement.com/~william/projects/luaossl.html> +-- + +local digest = require("openssl.digest") + +function md5_hex(input) + local b = digest.new("md5"):final(input) + local x = "" + for i = 1, #b do + x = x .. string.format("%.2x", string.byte(b, i)) + end + return x +end + +function filter_open(email, page) + buffer = "" + md5 = md5_hex(email:sub(2, -2):lower()) +end + +function filter_close() + baseurl = os.getenv("HTTPS") and "https://seccdn.libravatar.org/" or "http://cdn.libravatar.org/" + html("<img src='" .. baseurl .. "avatar/" .. md5 .. "?s=13&d=retro' width='13' height='13' alt='Libravatar' /> " .. buffer) + return 0 +end + +function filter_write(str) + buffer = buffer .. str +end + + diff --git a/third_party/cgit/filters/file-authentication.lua b/third_party/cgit/filters/file-authentication.lua new file mode 100644 index 000000000000..024880463c2a --- /dev/null +++ b/third_party/cgit/filters/file-authentication.lua @@ -0,0 +1,359 @@ +-- This script may be used with the auth-filter. +-- +-- Requirements: +-- luaossl +-- <http://25thandclement.com/~william/projects/luaossl.html> +-- luaposix +-- <https://github.com/luaposix/luaposix> +-- +local sysstat = require("posix.sys.stat") +local unistd = require("posix.unistd") +local rand = require("openssl.rand") +local hmac = require("openssl.hmac") + +-- This file should contain a series of lines in the form of: +-- username1:hash1 +-- username2:hash2 +-- username3:hash3 +-- ... +-- Hashes can be generated using something like `mkpasswd -m sha-512 -R 300000`. +-- This file should not be world-readable. +local users_filename = "/etc/cgit-auth/users" + +-- This file should contain a series of lines in the form of: +-- groupname1:username1,username2,username3,... +-- ... +local groups_filename = "/etc/cgit-auth/groups" + +-- This file should contain a series of lines in the form of: +-- reponame1:groupname1,groupname2,groupname3,... +-- ... +local repos_filename = "/etc/cgit-auth/repos" + +-- Set this to a path this script can write to for storing a persistent +-- cookie secret, which should not be world-readable. +local secret_filename = "/var/cache/cgit/auth-secret" + +-- +-- +-- Authentication functions follow below. Swap these out if you want different authentication semantics. +-- +-- + +-- Looks up a hash for a given user. +function lookup_hash(user) + local line + for line in io.lines(users_filename) do + local u, h = string.match(line, "(.-):(.+)") + if u:lower() == user:lower() then + return h + end + end + return nil +end + +-- Looks up users for a given repo. +function lookup_users(repo) + local users = nil + local groups = nil + local line, group, user + for line in io.lines(repos_filename) do + local r, g = string.match(line, "(.-):(.+)") + if r == repo then + groups = { } + for group in string.gmatch(g, "([^,]+)") do + groups[group:lower()] = true + end + break + end + end + if groups == nil then + return nil + end + for line in io.lines(groups_filename) do + local g, u = string.match(line, "(.-):(.+)") + if groups[g:lower()] then + if users == nil then + users = { } + end + for user in string.gmatch(u, "([^,]+)") do + users[user:lower()] = true + end + end + end + return users +end + + +-- Sets HTTP cookie headers based on post and sets up redirection. +function authenticate_post() + local hash = lookup_hash(post["username"]) + local redirect = validate_value("redirect", post["redirect"]) + + if redirect == nil then + not_found() + return 0 + end + + redirect_to(redirect) + + if hash == nil or hash ~= unistd.crypt(post["password"], hash) then + set_cookie("cgitauth", "") + else + -- One week expiration time + local username = secure_value("username", post["username"], os.time() + 604800) + set_cookie("cgitauth", username) + end + + html("\n") + return 0 +end + + +-- Returns 1 if the cookie is valid and 0 if it is not. +function authenticate_cookie() + accepted_users = lookup_users(cgit["repo"]) + if accepted_users == nil then + -- We return as valid if the repo is not protected. + return 1 + end + + local username = validate_value("username", get_cookie(http["cookie"], "cgitauth")) + if username == nil or not accepted_users[username:lower()] then + return 0 + else + return 1 + end +end + +-- Prints the html for the login form. +function body() + html("<h2>Authentication Required</h2>") + html("<form method='post' action='") + html_attr(cgit["login"]) + html("'>") + html("<input type='hidden' name='redirect' value='") + html_attr(secure_value("redirect", cgit["url"], 0)) + html("' />") + html("<table>") + html("<tr><td><label for='username'>Username:</label></td><td><input id='username' name='username' autofocus /></td></tr>") + html("<tr><td><label for='password'>Password:</label></td><td><input id='password' name='password' type='password' /></td></tr>") + html("<tr><td colspan='2'><input value='Login' type='submit' /></td></tr>") + html("</table></form>") + + return 0 +end + + + +-- +-- +-- Wrapper around filter API, exposing the http table, the cgit table, and the post table to the above functions. +-- +-- + +local actions = {} +actions["authenticate-post"] = authenticate_post +actions["authenticate-cookie"] = authenticate_cookie +actions["body"] = body + +function filter_open(...) + action = actions[select(1, ...)] + + http = {} + http["cookie"] = select(2, ...) + http["method"] = select(3, ...) + http["query"] = select(4, ...) + http["referer"] = select(5, ...) + http["path"] = select(6, ...) + http["host"] = select(7, ...) + http["https"] = select(8, ...) + + cgit = {} + cgit["repo"] = select(9, ...) + cgit["page"] = select(10, ...) + cgit["url"] = select(11, ...) + cgit["login"] = select(12, ...) + +end + +function filter_close() + return action() +end + +function filter_write(str) + post = parse_qs(str) +end + + +-- +-- +-- Utility functions based on keplerproject/wsapi. +-- +-- + +function url_decode(str) + if not str then + return "" + end + str = string.gsub(str, "+", " ") + str = string.gsub(str, "%%(%x%x)", function(h) return string.char(tonumber(h, 16)) end) + str = string.gsub(str, "\r\n", "\n") + return str +end + +function url_encode(str) + if not str then + return "" + end + str = string.gsub(str, "\n", "\r\n") + str = string.gsub(str, "([^%w ])", function(c) return string.format("%%%02X", string.byte(c)) end) + str = string.gsub(str, " ", "+") + return str +end + +function parse_qs(qs) + local tab = {} + for key, val in string.gmatch(qs, "([^&=]+)=([^&=]*)&?") do + tab[url_decode(key)] = url_decode(val) + end + return tab +end + +function get_cookie(cookies, name) + cookies = string.gsub(";" .. cookies .. ";", "%s*;%s*", ";") + return url_decode(string.match(cookies, ";" .. name .. "=(.-);")) +end + +function tohex(b) + local x = "" + for i = 1, #b do + x = x .. string.format("%.2x", string.byte(b, i)) + end + return x +end + +-- +-- +-- Cookie construction and validation helpers. +-- +-- + +local secret = nil + +-- Loads a secret from a file, creates a secret, or returns one from memory. +function get_secret() + if secret ~= nil then + return secret + end + local secret_file = io.open(secret_filename, "r") + if secret_file == nil then + local old_umask = sysstat.umask(63) + local temporary_filename = secret_filename .. ".tmp." .. tohex(rand.bytes(16)) + local temporary_file = io.open(temporary_filename, "w") + if temporary_file == nil then + os.exit(177) + end + temporary_file:write(tohex(rand.bytes(32))) + temporary_file:close() + unistd.link(temporary_filename, secret_filename) -- Intentionally fails in the case that another process is doing the same. + unistd.unlink(temporary_filename) + sysstat.umask(old_umask) + secret_file = io.open(secret_filename, "r") + end + if secret_file == nil then + os.exit(177) + end + secret = secret_file:read() + secret_file:close() + if secret:len() ~= 64 then + os.exit(177) + end + return secret +end + +-- Returns value of cookie if cookie is valid. Otherwise returns nil. +function validate_value(expected_field, cookie) + local i = 0 + local value = "" + local field = "" + local expiration = 0 + local salt = "" + local chmac = "" + + if cookie == nil or cookie:len() < 3 or cookie:sub(1, 1) == "|" then + return nil + end + + for component in string.gmatch(cookie, "[^|]+") do + if i == 0 then + field = component + elseif i == 1 then + value = component + elseif i == 2 then + expiration = tonumber(component) + if expiration == nil then + expiration = -1 + end + elseif i == 3 then + salt = component + elseif i == 4 then + chmac = component + else + break + end + i = i + 1 + end + + if chmac == nil or chmac:len() == 0 then + return nil + end + + -- Lua hashes strings, so these comparisons are time invariant. + if chmac ~= tohex(hmac.new(get_secret(), "sha256"):final(field .. "|" .. value .. "|" .. tostring(expiration) .. "|" .. salt)) then + return nil + end + + if expiration == -1 or (expiration ~= 0 and expiration <= os.time()) then + return nil + end + + if url_decode(field) ~= expected_field then + return nil + end + + return url_decode(value) +end + +function secure_value(field, value, expiration) + if value == nil or value:len() <= 0 then + return "" + end + + local authstr = "" + local salt = tohex(rand.bytes(16)) + value = url_encode(value) + field = url_encode(field) + authstr = field .. "|" .. value .. "|" .. tostring(expiration) .. "|" .. salt + authstr = authstr .. "|" .. tohex(hmac.new(get_secret(), "sha256"):final(authstr)) + return authstr +end + +function set_cookie(cookie, value) + html("Set-Cookie: " .. cookie .. "=" .. value .. "; HttpOnly") + if http["https"] == "yes" or http["https"] == "on" or http["https"] == "1" then + html("; secure") + end + html("\n") +end + +function redirect_to(url) + html("Status: 302 Redirect\n") + html("Cache-Control: no-cache, no-store\n") + html("Location: " .. url .. "\n") +end + +function not_found() + html("Status: 404 Not Found\n") + html("Cache-Control: no-cache, no-store\n\n") +end diff --git a/third_party/cgit/filters/gentoo-ldap-authentication.lua b/third_party/cgit/filters/gentoo-ldap-authentication.lua new file mode 100644 index 000000000000..673c88d1021c --- /dev/null +++ b/third_party/cgit/filters/gentoo-ldap-authentication.lua @@ -0,0 +1,360 @@ +-- This script may be used with the auth-filter. Be sure to configure it as you wish. +-- +-- Requirements: +-- luaossl +-- <http://25thandclement.com/~william/projects/luaossl.html> +-- lualdap >= 1.2 +-- <https://git.zx2c4.com/lualdap/about/> +-- luaposix +-- <https://github.com/luaposix/luaposix> +-- +local sysstat = require("posix.sys.stat") +local unistd = require("posix.unistd") +local lualdap = require("lualdap") +local rand = require("openssl.rand") +local hmac = require("openssl.hmac") + +-- +-- +-- Configure these variables for your settings. +-- +-- + +-- A list of password protected repositories, with which gentooAccess +-- group is allowed to access each one. +local protected_repos = { + glouglou = "infra", + portage = "dev" +} + +-- Set this to a path this script can write to for storing a persistent +-- cookie secret, which should be guarded. +local secret_filename = "/var/cache/cgit/auth-secret" + + +-- +-- +-- Authentication functions follow below. Swap these out if you want different authentication semantics. +-- +-- + +-- Sets HTTP cookie headers based on post and sets up redirection. +function authenticate_post() + local redirect = validate_value("redirect", post["redirect"]) + + if redirect == nil then + not_found() + return 0 + end + + redirect_to(redirect) + + local groups = gentoo_ldap_user_groups(post["username"], post["password"]) + if groups == nil then + set_cookie("cgitauth", "") + else + -- One week expiration time + set_cookie("cgitauth", secure_value("gentoogroups", table.concat(groups, ","), os.time() + 604800)) + end + + html("\n") + return 0 +end + + +-- Returns 1 if the cookie is valid and 0 if it is not. +function authenticate_cookie() + local required_group = protected_repos[cgit["repo"]] + if required_group == nil then + -- We return as valid if the repo is not protected. + return 1 + end + + local user_groups = validate_value("gentoogroups", get_cookie(http["cookie"], "cgitauth")) + if user_groups == nil or user_groups == "" then + return 0 + end + for group in string.gmatch(user_groups, "[^,]+") do + if group == required_group then + return 1 + end + end + return 0 +end + +-- Prints the html for the login form. +function body() + html("<h2>Gentoo LDAP Authentication Required</h2>") + html("<form method='post' action='") + html_attr(cgit["login"]) + html("'>") + html("<input type='hidden' name='redirect' value='") + html_attr(secure_value("redirect", cgit["url"], 0)) + html("' />") + html("<table>") + html("<tr><td><label for='username'>Username:</label></td><td><input id='username' name='username' autofocus /></td></tr>") + html("<tr><td><label for='password'>Password:</label></td><td><input id='password' name='password' type='password' /></td></tr>") + html("<tr><td colspan='2'><input value='Login' type='submit' /></td></tr>") + html("</table></form>") + + return 0 +end + +-- +-- +-- Gentoo LDAP support. +-- +-- + +function gentoo_ldap_user_groups(username, password) + -- Ensure the user is alphanumeric + if username == nil or username:match("%W") then + return nil + end + + local who = "uid=" .. username .. ",ou=devs,dc=gentoo,dc=org" + + local ldap, err = lualdap.open_simple { + uri = "ldap://ldap1.gentoo.org", + who = who, + password = password, + starttls = true, + certfile = "/var/www/uwsgi/cgit/gentoo-ldap/star.gentoo.org.crt", + keyfile = "/var/www/uwsgi/cgit/gentoo-ldap/star.gentoo.org.key", + cacertfile = "/var/www/uwsgi/cgit/gentoo-ldap/ca.pem" + } + if ldap == nil then + return nil + end + + local group_suffix = ".group" + local group_suffix_len = group_suffix:len() + local groups = {} + for dn, attribs in ldap:search { base = who, scope = "subtree" } do + local access = attribs["gentooAccess"] + if dn == who and access ~= nil then + for i, v in ipairs(access) do + local vlen = v:len() + if vlen > group_suffix_len and v:sub(-group_suffix_len) == group_suffix then + table.insert(groups, v:sub(1, vlen - group_suffix_len)) + end + end + end + end + + ldap:close() + + return groups +end + +-- +-- +-- Wrapper around filter API, exposing the http table, the cgit table, and the post table to the above functions. +-- +-- + +local actions = {} +actions["authenticate-post"] = authenticate_post +actions["authenticate-cookie"] = authenticate_cookie +actions["body"] = body + +function filter_open(...) + action = actions[select(1, ...)] + + http = {} + http["cookie"] = select(2, ...) + http["method"] = select(3, ...) + http["query"] = select(4, ...) + http["referer"] = select(5, ...) + http["path"] = select(6, ...) + http["host"] = select(7, ...) + http["https"] = select(8, ...) + + cgit = {} + cgit["repo"] = select(9, ...) + cgit["page"] = select(10, ...) + cgit["url"] = select(11, ...) + cgit["login"] = select(12, ...) + +end + +function filter_close() + return action() +end + +function filter_write(str) + post = parse_qs(str) +end + + +-- +-- +-- Utility functions based on keplerproject/wsapi. +-- +-- + +function url_decode(str) + if not str then + return "" + end + str = string.gsub(str, "+", " ") + str = string.gsub(str, "%%(%x%x)", function(h) return string.char(tonumber(h, 16)) end) + str = string.gsub(str, "\r\n", "\n") + return str +end + +function url_encode(str) + if not str then + return "" + end + str = string.gsub(str, "\n", "\r\n") + str = string.gsub(str, "([^%w ])", function(c) return string.format("%%%02X", string.byte(c)) end) + str = string.gsub(str, " ", "+") + return str +end + +function parse_qs(qs) + local tab = {} + for key, val in string.gmatch(qs, "([^&=]+)=([^&=]*)&?") do + tab[url_decode(key)] = url_decode(val) + end + return tab +end + +function get_cookie(cookies, name) + cookies = string.gsub(";" .. cookies .. ";", "%s*;%s*", ";") + return string.match(cookies, ";" .. name .. "=(.-);") +end + +function tohex(b) + local x = "" + for i = 1, #b do + x = x .. string.format("%.2x", string.byte(b, i)) + end + return x +end + +-- +-- +-- Cookie construction and validation helpers. +-- +-- + +local secret = nil + +-- Loads a secret from a file, creates a secret, or returns one from memory. +function get_secret() + if secret ~= nil then + return secret + end + local secret_file = io.open(secret_filename, "r") + if secret_file == nil then + local old_umask = sysstat.umask(63) + local temporary_filename = secret_filename .. ".tmp." .. tohex(rand.bytes(16)) + local temporary_file = io.open(temporary_filename, "w") + if temporary_file == nil then + os.exit(177) + end + temporary_file:write(tohex(rand.bytes(32))) + temporary_file:close() + unistd.link(temporary_filename, secret_filename) -- Intentionally fails in the case that another process is doing the same. + unistd.unlink(temporary_filename) + sysstat.umask(old_umask) + secret_file = io.open(secret_filename, "r") + end + if secret_file == nil then + os.exit(177) + end + secret = secret_file:read() + secret_file:close() + if secret:len() ~= 64 then + os.exit(177) + end + return secret +end + +-- Returns value of cookie if cookie is valid. Otherwise returns nil. +function validate_value(expected_field, cookie) + local i = 0 + local value = "" + local field = "" + local expiration = 0 + local salt = "" + local chmac = "" + + if cookie == nil or cookie:len() < 3 or cookie:sub(1, 1) == "|" then + return nil + end + + for component in string.gmatch(cookie, "[^|]+") do + if i == 0 then + field = component + elseif i == 1 then + value = component + elseif i == 2 then + expiration = tonumber(component) + if expiration == nil then + expiration = -1 + end + elseif i == 3 then + salt = component + elseif i == 4 then + chmac = component + else + break + end + i = i + 1 + end + + if chmac == nil or chmac:len() == 0 then + return nil + end + + -- Lua hashes strings, so these comparisons are time invariant. + if chmac ~= tohex(hmac.new(get_secret(), "sha256"):final(field .. "|" .. value .. "|" .. tostring(expiration) .. "|" .. salt)) then + return nil + end + + if expiration == -1 or (expiration ~= 0 and expiration <= os.time()) then + return nil + end + + if url_decode(field) ~= expected_field then + return nil + end + + return url_decode(value) +end + +function secure_value(field, value, expiration) + if value == nil or value:len() <= 0 then + return "" + end + + local authstr = "" + local salt = tohex(rand.bytes(16)) + value = url_encode(value) + field = url_encode(field) + authstr = field .. "|" .. value .. "|" .. tostring(expiration) .. "|" .. salt + authstr = authstr .. "|" .. tohex(hmac.new(get_secret(), "sha256"):final(authstr)) + return authstr +end + +function set_cookie(cookie, value) + html("Set-Cookie: " .. cookie .. "=" .. value .. "; HttpOnly") + if http["https"] == "yes" or http["https"] == "on" or http["https"] == "1" then + html("; secure") + end + html("\n") +end + +function redirect_to(url) + html("Status: 302 Redirect\n") + html("Cache-Control: no-cache, no-store\n") + html("Location: " .. url .. "\n") +end + +function not_found() + html("Status: 404 Not Found\n") + html("Cache-Control: no-cache, no-store\n\n") +end diff --git a/third_party/cgit/filters/html-converters/man2html b/third_party/cgit/filters/html-converters/man2html new file mode 100755 index 000000000000..0ef78841810e --- /dev/null +++ b/third_party/cgit/filters/html-converters/man2html @@ -0,0 +1,4 @@ +#!/bin/sh +echo "<div style=\"font-family: monospace\">" +groff -mandoc -T html -P -r -P -l | egrep -v '(<html>|<head>|<meta|<title>|</title>|</head>|<body>|</body>|</html>|<!DOCTYPE|"http://www.w3.org)' +echo "</div>" diff --git a/third_party/cgit/filters/html-converters/md2html b/third_party/cgit/filters/html-converters/md2html new file mode 100755 index 000000000000..59f43a84167f --- /dev/null +++ b/third_party/cgit/filters/html-converters/md2html @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +import markdown +import sys +import io +from pygments.formatters import HtmlFormatter +from markdown.extensions.toc import TocExtension +sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8') +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8') +sys.stdout.write(''' +<style> +.markdown-body { + font-size: 14px; + line-height: 1.6; + overflow: hidden; +} +.markdown-body>*:first-child { + margin-top: 0 !important; +} +.markdown-body>*:last-child { + margin-bottom: 0 !important; +} +.markdown-body a.absent { + color: #c00; +} +.markdown-body a.anchor { + display: block; + padding-left: 30px; + margin-left: -30px; + cursor: pointer; + position: absolute; + top: 0; + left: 0; + bottom: 0; +} +.markdown-body h1, .markdown-body h2, .markdown-body h3, .markdown-body h4, .markdown-body h5, .markdown-body h6 { + margin: 20px 0 10px; + padding: 0; + font-weight: bold; + -webkit-font-smoothing: antialiased; + cursor: text; + position: relative; +} +.markdown-body h1 .mini-icon-link, .markdown-body h2 .mini-icon-link, .markdown-body h3 .mini-icon-link, .markdown-body h4 .mini-icon-link, .markdown-body h5 .mini-icon-link, .markdown-body h6 .mini-icon-link { + display: none; + color: #000; +} +.markdown-body h1:hover a.anchor, .markdown-body h2:hover a.anchor, .markdown-body h3:hover a.anchor, .markdown-body h4:hover a.anchor, .markdown-body h5:hover a.anchor, .markdown-body h6:hover a.anchor { + text-decoration: none; + line-height: 1; + padding-left: 0; + margin-left: -22px; + top: 15%; +} +.markdown-body h1:hover a.anchor .mini-icon-link, .markdown-body h2:hover a.anchor .mini-icon-link, .markdown-body h3:hover a.anchor .mini-icon-link, .markdown-body h4:hover a.anchor .mini-icon-link, .markdown-body h5:hover a.anchor .mini-icon-link, .markdown-body h6:hover a.anchor .mini-icon-link { + display: inline-block; +} +div#cgit .markdown-body h1 a.toclink, div#cgit .markdown-body h2 a.toclink, div#cgit .markdown-body h3 a.toclink, div#cgit .markdown-body h4 a.toclink, div#cgit .markdown-body h5 a.toclink, div#cgit .markdown-body h6 a.toclink { + color: black; +} +.markdown-body h1 tt, .markdown-body h1 code, .markdown-body h2 tt, .markdown-body h2 code, .markdown-body h3 tt, .markdown-body h3 code, .markdown-body h4 tt, .markdown-body h4 code, .markdown-body h5 tt, .markdown-body h5 code, .markdown-body h6 tt, .markdown-body h6 code { + font-size: inherit; +} +.markdown-body h1 { + font-size: 28px; + color: #000; +} +.markdown-body h2 { + font-size: 24px; + border-bottom: 1px solid #ccc; + color: #000; +} +.markdown-body h3 { + font-size: 18px; +} +.markdown-body h4 { + font-size: 16px; +} +.markdown-body h5 { + font-size: 14px; +} +.markdown-body h6 { + color: #777; + font-size: 14px; +} +.markdown-body p, .markdown-body blockquote, .markdown-body ul, .markdown-body ol, .markdown-body dl, .markdown-body table, .markdown-body pre { + margin: 15px 0; +} +.markdown-body hr { + border: 2px solid #ccc; +} +.markdown-body>h2:first-child, .markdown-body>h1:first-child, .markdown-body>h1:first-child+h2, .markdown-body>h3:first-child, .markdown-body>h4:first-child, .markdown-body>h5:first-child, .markdown-body>h6:first-child { + margin-top: 0; + padding-top: 0; +} +.markdown-body a:first-child h1, .markdown-body a:first-child h2, .markdown-body a:first-child h3, .markdown-body a:first-child h4, .markdown-body a:first-child h5, .markdown-body a:first-child h6 { + margin-top: 0; + padding-top: 0; +} +.markdown-body h1+p, .markdown-body h2+p, .markdown-body h3+p, .markdown-body h4+p, .markdown-body h5+p, .markdown-body h6+p { + margin-top: 0; +} +.markdown-body li p.first { + display: inline-block; +} +.markdown-body ul, .markdown-body ol { + padding-left: 30px; +} +.markdown-body ul.no-list, .markdown-body ol.no-list { + list-style-type: none; + padding: 0; +} +.markdown-body ul li>:first-child, .markdown-body ul li ul:first-of-type, .markdown-body ul li ol:first-of-type, .markdown-body ol li>:first-child, .markdown-body ol li ul:first-of-type, .markdown-body ol li ol:first-of-type { + margin-top: 0px; +} +.markdown-body ul li p:last-of-type, .markdown-body ol li p:last-of-type { + margin-bottom: 0; +} +.markdown-body ul ul, .markdown-body ul ol, .markdown-body ol ol, .markdown-body ol ul { + margin-bottom: 0; +} +.markdown-body dl { + padding: 0; +} +.markdown-body dl dt { + font-size: 14px; + font-weight: bold; + font-style: italic; + padding: 0; + margin: 15px 0 5px; +} +.markdown-body dl dt:first-child { + padding: 0; +} +.markdown-body dl dt>:first-child { + margin-top: 0px; +} +.markdown-body dl dt>:last-child { + margin-bottom: 0px; +} +.markdown-body dl dd { + margin: 0 0 15px; + padding: 0 15px; +} +.markdown-body dl dd>:first-child { + margin-top: 0px; +} +.markdown-body dl dd>:last-child { + margin-bottom: 0px; +} +.markdown-body blockquote { + border-left: 4px solid #DDD; + padding: 0 15px; + color: #777; +} +.markdown-body blockquote>:first-child { + margin-top: 0px; +} +.markdown-body blockquote>:last-child { + margin-bottom: 0px; +} +.markdown-body table th { + font-weight: bold; +} +.markdown-body table th, .markdown-body table td { + border: 1px solid #ccc; + padding: 6px 13px; +} +.markdown-body table tr { + border-top: 1px solid #ccc; + background-color: #fff; +} +.markdown-body table tr:nth-child(2n) { + background-color: #f8f8f8; +} +.markdown-body img { + max-width: 100%; + -moz-box-sizing: border-box; + box-sizing: border-box; +} +.markdown-body span.frame { + display: block; + overflow: hidden; +} +.markdown-body span.frame>span { + border: 1px solid #ddd; + display: block; + float: left; + overflow: hidden; + margin: 13px 0 0; + padding: 7px; + width: auto; +} +.markdown-body span.frame span img { + display: block; + float: left; +} +.markdown-body span.frame span span { + clear: both; + color: #333; + display: block; + padding: 5px 0 0; +} +.markdown-body span.align-center { + display: block; + overflow: hidden; + clear: both; +} +.markdown-body span.align-center>span { + display: block; + overflow: hidden; + margin: 13px auto 0; + text-align: center; +} +.markdown-body span.align-center span img { + margin: 0 auto; + text-align: center; +} +.markdown-body span.align-right { + display: block; + overflow: hidden; + clear: both; +} +.markdown-body span.align-right>span { + display: block; + overflow: hidden; + margin: 13px 0 0; + text-align: right; +} +.markdown-body span.align-right span img { + margin: 0; + text-align: right; +} +.markdown-body span.float-left { + display: block; + margin-right: 13px; + overflow: hidden; + float: left; +} +.markdown-body span.float-left span { + margin: 13px 0 0; +} +.markdown-body span.float-right { + display: block; + margin-left: 13px; + overflow: hidden; + float: right; +} +.markdown-body span.float-right>span { + display: block; + overflow: hidden; + margin: 13px auto 0; + text-align: right; +} +.markdown-body code, .markdown-body tt { + margin: 0 2px; + padding: 0px 5px; + border: 1px solid #eaeaea; + background-color: #f8f8f8; + border-radius: 3px; +} +.markdown-body code { + white-space: nowrap; +} +.markdown-body pre>code { + margin: 0; + padding: 0; + white-space: pre; + border: none; + background: transparent; +} +.markdown-body .highlight pre, .markdown-body pre { + background-color: #f8f8f8; + border: 1px solid #ccc; + font-size: 13px; + line-height: 19px; + overflow: auto; + padding: 6px 10px; + border-radius: 3px; +} +.markdown-body pre code, .markdown-body pre tt { + margin: 0; + padding: 0; + background-color: transparent; + border: none; +} +''') +sys.stdout.write(HtmlFormatter(style='pastie').get_style_defs('.highlight')) +sys.stdout.write(''' +</style> +''') +sys.stdout.write("<div class='markdown-body'>") +sys.stdout.flush() +# Note: you may want to run this through bleach for sanitization +markdown.markdownFromFile( + output_format="html5", + extensions=[ + "markdown.extensions.fenced_code", + "markdown.extensions.codehilite", + "markdown.extensions.tables", + "markdown.extensions.sane_lists", + TocExtension(anchorlink=True)], + extension_configs={ + "markdown.extensions.codehilite":{"css_class":"highlight"}}) +sys.stdout.write("</div>") diff --git a/third_party/cgit/filters/html-converters/rst2html b/third_party/cgit/filters/html-converters/rst2html new file mode 100755 index 000000000000..02d90f81c9af --- /dev/null +++ b/third_party/cgit/filters/html-converters/rst2html @@ -0,0 +1,2 @@ +#!/bin/bash +exec rst2html.py --template <(echo -e "%(stylesheet)s\n%(body_pre_docinfo)s\n%(docinfo)s\n%(body)s") diff --git a/third_party/cgit/filters/html-converters/txt2html b/third_party/cgit/filters/html-converters/txt2html new file mode 100755 index 000000000000..495eeceb279e --- /dev/null +++ b/third_party/cgit/filters/html-converters/txt2html @@ -0,0 +1,4 @@ +#!/bin/sh +echo "<pre>" +sed "s|&|\\&|g;s|'|\\'|g;s|\"|\\"|g;s|<|\\<|g;s|>|\\>|g" +echo "</pre>" diff --git a/third_party/cgit/filters/owner-example.lua b/third_party/cgit/filters/owner-example.lua new file mode 100644 index 000000000000..50fc25a8e53c --- /dev/null +++ b/third_party/cgit/filters/owner-example.lua @@ -0,0 +1,17 @@ +-- This script is an example of an owner-filter. It replaces the +-- usual query link with one to a fictional homepage. This script may +-- be used with the owner-filter or repo.owner-filter settings in +-- cgitrc with the `lua:` prefix. + +function filter_open() + buffer = "" +end + +function filter_close() + html(string.format("<a href=\"%s\">%s</a>", "http://wiki.example.com/about/" .. buffer, buffer)) + return 0 +end + +function filter_write(str) + buffer = buffer .. str +end diff --git a/third_party/cgit/filters/simple-authentication.lua b/third_party/cgit/filters/simple-authentication.lua new file mode 100644 index 000000000000..23d345763bf6 --- /dev/null +++ b/third_party/cgit/filters/simple-authentication.lua @@ -0,0 +1,314 @@ +-- This script may be used with the auth-filter. Be sure to configure it as you wish. +-- +-- Requirements: +-- luaossl +-- <http://25thandclement.com/~william/projects/luaossl.html> +-- luaposix +-- <https://github.com/luaposix/luaposix> +-- +local sysstat = require("posix.sys.stat") +local unistd = require("posix.unistd") +local rand = require("openssl.rand") +local hmac = require("openssl.hmac") + +-- +-- +-- Configure these variables for your settings. +-- +-- + +-- A list of password protected repositories along with the users who can access them. +local protected_repos = { + glouglou = { laurent = true, jason = true }, + qt = { jason = true, bob = true } +} + +-- A list of users and hashes, generated with `mkpasswd -m sha-512 -R 300000`. +local users = { + jason = "$6$rounds=300000$YYJct3n/o.ruYK$HhpSeuCuW1fJkpvMZOZzVizeLsBKcGA/aF2UPuV5v60JyH2MVSG6P511UMTj2F3H75.IT2HIlnvXzNb60FcZH1", + laurent = "$6$rounds=300000$dP0KNHwYb3JKigT$pN/LG7rWxQ4HniFtx5wKyJXBJUKP7R01zTNZ0qSK/aivw8ywGAOdfYiIQFqFhZFtVGvr11/7an.nesvm8iJUi.", + bob = "$6$rounds=300000$jCLCCt6LUpTz$PI1vvd1yaVYcCzqH8QAJFcJ60b6W/6sjcOsU7mAkNo7IE8FRGW1vkjF8I/T5jt/auv5ODLb1L4S2s.CAyZyUC" +} + +-- Set this to a path this script can write to for storing a persistent +-- cookie secret, which should be guarded. +local secret_filename = "/var/cache/cgit/auth-secret" + +-- +-- +-- Authentication functions follow below. Swap these out if you want different authentication semantics. +-- +-- + +-- Sets HTTP cookie headers based on post and sets up redirection. +function authenticate_post() + local hash = users[post["username"]] + local redirect = validate_value("redirect", post["redirect"]) + + if redirect == nil then + not_found() + return 0 + end + + redirect_to(redirect) + + if hash == nil or hash ~= unistd.crypt(post["password"], hash) then + set_cookie("cgitauth", "") + else + -- One week expiration time + local username = secure_value("username", post["username"], os.time() + 604800) + set_cookie("cgitauth", username) + end + + html("\n") + return 0 +end + + +-- Returns 1 if the cookie is valid and 0 if it is not. +function authenticate_cookie() + accepted_users = protected_repos[cgit["repo"]] + if accepted_users == nil then + -- We return as valid if the repo is not protected. + return 1 + end + + local username = validate_value("username", get_cookie(http["cookie"], "cgitauth")) + if username == nil or not accepted_users[username:lower()] then + return 0 + else + return 1 + end +end + +-- Prints the html for the login form. +function body() + html("<h2>Authentication Required</h2>") + html("<form method='post' action='") + html_attr(cgit["login"]) + html("'>") + html("<input type='hidden' name='redirect' value='") + html_attr(secure_value("redirect", cgit["url"], 0)) + html("' />") + html("<table>") + html("<tr><td><label for='username'>Username:</label></td><td><input id='username' name='username' autofocus /></td></tr>") + html("<tr><td><label for='password'>Password:</label></td><td><input id='password' name='password' type='password' /></td></tr>") + html("<tr><td colspan='2'><input value='Login' type='submit' /></td></tr>") + html("</table></form>") + + return 0 +end + + + +-- +-- +-- Wrapper around filter API, exposing the http table, the cgit table, and the post table to the above functions. +-- +-- + +local actions = {} +actions["authenticate-post"] = authenticate_post +actions["authenticate-cookie"] = authenticate_cookie +actions["body"] = body + +function filter_open(...) + action = actions[select(1, ...)] + + http = {} + http["cookie"] = select(2, ...) + http["method"] = select(3, ...) + http["query"] = select(4, ...) + http["referer"] = select(5, ...) + http["path"] = select(6, ...) + http["host"] = select(7, ...) + http["https"] = select(8, ...) + + cgit = {} + cgit["repo"] = select(9, ...) + cgit["page"] = select(10, ...) + cgit["url"] = select(11, ...) + cgit["login"] = select(12, ...) + +end + +function filter_close() + return action() +end + +function filter_write(str) + post = parse_qs(str) +end + + +-- +-- +-- Utility functions based on keplerproject/wsapi. +-- +-- + +function url_decode(str) + if not str then + return "" + end + str = string.gsub(str, "+", " ") + str = string.gsub(str, "%%(%x%x)", function(h) return string.char(tonumber(h, 16)) end) + str = string.gsub(str, "\r\n", "\n") + return str +end + +function url_encode(str) + if not str then + return "" + end + str = string.gsub(str, "\n", "\r\n") + str = string.gsub(str, "([^%w ])", function(c) return string.format("%%%02X", string.byte(c)) end) + str = string.gsub(str, " ", "+") + return str +end + +function parse_qs(qs) + local tab = {} + for key, val in string.gmatch(qs, "([^&=]+)=([^&=]*)&?") do + tab[url_decode(key)] = url_decode(val) + end + return tab +end + +function get_cookie(cookies, name) + cookies = string.gsub(";" .. cookies .. ";", "%s*;%s*", ";") + return url_decode(string.match(cookies, ";" .. name .. "=(.-);")) +end + +function tohex(b) + local x = "" + for i = 1, #b do + x = x .. string.format("%.2x", string.byte(b, i)) + end + return x +end + +-- +-- +-- Cookie construction and validation helpers. +-- +-- + +local secret = nil + +-- Loads a secret from a file, creates a secret, or returns one from memory. +function get_secret() + if secret ~= nil then + return secret + end + local secret_file = io.open(secret_filename, "r") + if secret_file == nil then + local old_umask = sysstat.umask(63) + local temporary_filename = secret_filename .. ".tmp." .. tohex(rand.bytes(16)) + local temporary_file = io.open(temporary_filename, "w") + if temporary_file == nil then + os.exit(177) + end + temporary_file:write(tohex(rand.bytes(32))) + temporary_file:close() + unistd.link(temporary_filename, secret_filename) -- Intentionally fails in the case that another process is doing the same. + unistd.unlink(temporary_filename) + sysstat.umask(old_umask) + secret_file = io.open(secret_filename, "r") + end + if secret_file == nil then + os.exit(177) + end + secret = secret_file:read() + secret_file:close() + if secret:len() ~= 64 then + os.exit(177) + end + return secret +end + +-- Returns value of cookie if cookie is valid. Otherwise returns nil. +function validate_value(expected_field, cookie) + local i = 0 + local value = "" + local field = "" + local expiration = 0 + local salt = "" + local chmac = "" + + if cookie == nil or cookie:len() < 3 or cookie:sub(1, 1) == "|" then + return nil + end + + for component in string.gmatch(cookie, "[^|]+") do + if i == 0 then + field = component + elseif i == 1 then + value = component + elseif i == 2 then + expiration = tonumber(component) + if expiration == nil then + expiration = -1 + end + elseif i == 3 then + salt = component + elseif i == 4 then + chmac = component + else + break + end + i = i + 1 + end + + if chmac == nil or chmac:len() == 0 then + return nil + end + + -- Lua hashes strings, so these comparisons are time invariant. + if chmac ~= tohex(hmac.new(get_secret(), "sha256"):final(field .. "|" .. value .. "|" .. tostring(expiration) .. "|" .. salt)) then + return nil + end + + if expiration == -1 or (expiration ~= 0 and expiration <= os.time()) then + return nil + end + + if url_decode(field) ~= expected_field then + return nil + end + + return url_decode(value) +end + +function secure_value(field, value, expiration) + if value == nil or value:len() <= 0 then + return "" + end + + local authstr = "" + local salt = tohex(rand.bytes(16)) + value = url_encode(value) + field = url_encode(field) + authstr = field .. "|" .. value .. "|" .. tostring(expiration) .. "|" .. salt + authstr = authstr .. "|" .. tohex(hmac.new(get_secret(), "sha256"):final(authstr)) + return authstr +end + +function set_cookie(cookie, value) + html("Set-Cookie: " .. cookie .. "=" .. value .. "; HttpOnly") + if http["https"] == "yes" or http["https"] == "on" or http["https"] == "1" then + html("; secure") + end + html("\n") +end + +function redirect_to(url) + html("Status: 302 Redirect\n") + html("Cache-Control: no-cache, no-store\n") + html("Location: " .. url .. "\n") +end + +function not_found() + html("Status: 404 Not Found\n") + html("Cache-Control: no-cache, no-store\n\n") +end diff --git a/third_party/cgit/filters/syntax-highlighting.py b/third_party/cgit/filters/syntax-highlighting.py new file mode 100755 index 000000000000..e912594c48a7 --- /dev/null +++ b/third_party/cgit/filters/syntax-highlighting.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +# This script uses Pygments and Python3. You must have both installed +# for this to work. +# +# http://pygments.org/ +# http://python.org/ +# +# It may be used with the source-filter or repo.source-filter settings +# in cgitrc. +# +# The following environment variables can be used to retrieve the +# configuration of the repository for which this script is called: +# CGIT_REPO_URL ( = repo.url setting ) +# CGIT_REPO_NAME ( = repo.name setting ) +# CGIT_REPO_PATH ( = repo.path setting ) +# CGIT_REPO_OWNER ( = repo.owner setting ) +# CGIT_REPO_DEFBRANCH ( = repo.defbranch setting ) +# CGIT_REPO_SECTION ( = section setting ) +# CGIT_REPO_CLONE_URL ( = repo.clone-url setting ) + + +import sys +import io +from pygments import highlight +from pygments.util import ClassNotFound +from pygments.lexers import TextLexer +from pygments.lexers import guess_lexer +from pygments.lexers import guess_lexer_for_filename +from pygments.formatters import HtmlFormatter + + +sys.stdin = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors='replace') +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') +data = sys.stdin.read() +filename = sys.argv[1] +formatter = HtmlFormatter(style='pastie', nobackground=True) + +try: + lexer = guess_lexer_for_filename(filename, data) +except ClassNotFound: + # check if there is any shebang + if data[0:2] == '#!': + lexer = guess_lexer(data) + else: + lexer = TextLexer() +except TypeError: + lexer = TextLexer() + +# highlight! :-) +# printout pygments' css definitions as well +sys.stdout.write('<style>') +sys.stdout.write(formatter.get_style_defs('.highlight')) +sys.stdout.write('</style>') +sys.stdout.write(highlight(data, lexer, formatter, outfile=None)) diff --git a/third_party/cgit/filters/syntax-highlighting.sh b/third_party/cgit/filters/syntax-highlighting.sh new file mode 100755 index 000000000000..840bc34fffd4 --- /dev/null +++ b/third_party/cgit/filters/syntax-highlighting.sh @@ -0,0 +1,121 @@ +#!/bin/sh +# This script can be used to implement syntax highlighting in the cgit +# tree-view by referring to this file with the source-filter or repo.source- +# filter options in cgitrc. +# +# This script requires a shell supporting the ${var##pattern} syntax. +# It is supported by at least dash and bash, however busybox environments +# might have to use an external call to sed instead. +# +# Note: the highlight command (http://www.andre-simon.de/) uses css for syntax +# highlighting, so you'll probably want something like the following included +# in your css file: +# +# Style definition file generated by highlight 2.4.8, http://www.andre-simon.de/ +# +# table.blob .num { color:#2928ff; } +# table.blob .esc { color:#ff00ff; } +# table.blob .str { color:#ff0000; } +# table.blob .dstr { color:#818100; } +# table.blob .slc { color:#838183; font-style:italic; } +# table.blob .com { color:#838183; font-style:italic; } +# table.blob .dir { color:#008200; } +# table.blob .sym { color:#000000; } +# table.blob .kwa { color:#000000; font-weight:bold; } +# table.blob .kwb { color:#830000; } +# table.blob .kwc { color:#000000; font-weight:bold; } +# table.blob .kwd { color:#010181; } +# +# +# Style definition file generated by highlight 2.6.14, http://www.andre-simon.de/ +# +# body.hl { background-color:#ffffff; } +# pre.hl { color:#000000; background-color:#ffffff; font-size:10pt; font-family:'Courier New';} +# .hl.num { color:#2928ff; } +# .hl.esc { color:#ff00ff; } +# .hl.str { color:#ff0000; } +# .hl.dstr { color:#818100; } +# .hl.slc { color:#838183; font-style:italic; } +# .hl.com { color:#838183; font-style:italic; } +# .hl.dir { color:#008200; } +# .hl.sym { color:#000000; } +# .hl.line { color:#555555; } +# .hl.mark { background-color:#ffffbb;} +# .hl.kwa { color:#000000; font-weight:bold; } +# .hl.kwb { color:#830000; } +# .hl.kwc { color:#000000; font-weight:bold; } +# .hl.kwd { color:#010181; } +# +# +# Style definition file generated by highlight 3.8, http://www.andre-simon.de/ +# +# body.hl { background-color:#e0eaee; } +# pre.hl { color:#000000; background-color:#e0eaee; font-size:10pt; font-family:'Courier New';} +# .hl.num { color:#b07e00; } +# .hl.esc { color:#ff00ff; } +# .hl.str { color:#bf0303; } +# .hl.pps { color:#818100; } +# .hl.slc { color:#838183; font-style:italic; } +# .hl.com { color:#838183; font-style:italic; } +# .hl.ppc { color:#008200; } +# .hl.opt { color:#000000; } +# .hl.lin { color:#555555; } +# .hl.kwa { color:#000000; font-weight:bold; } +# .hl.kwb { color:#0057ae; } +# .hl.kwc { color:#000000; font-weight:bold; } +# .hl.kwd { color:#010181; } +# +# +# Style definition file generated by highlight 3.13, http://www.andre-simon.de/ +# +# body.hl { background-color:#e0eaee; } +# pre.hl { color:#000000; background-color:#e0eaee; font-size:10pt; font-family:'Courier New',monospace;} +# .hl.num { color:#b07e00; } +# .hl.esc { color:#ff00ff; } +# .hl.str { color:#bf0303; } +# .hl.pps { color:#818100; } +# .hl.slc { color:#838183; font-style:italic; } +# .hl.com { color:#838183; font-style:italic; } +# .hl.ppc { color:#008200; } +# .hl.opt { color:#000000; } +# .hl.ipl { color:#0057ae; } +# .hl.lin { color:#555555; } +# .hl.kwa { color:#000000; font-weight:bold; } +# .hl.kwb { color:#0057ae; } +# .hl.kwc { color:#000000; font-weight:bold; } +# .hl.kwd { color:#010181; } +# +# +# The following environment variables can be used to retrieve the configuration +# of the repository for which this script is called: +# CGIT_REPO_URL ( = repo.url setting ) +# CGIT_REPO_NAME ( = repo.name setting ) +# CGIT_REPO_PATH ( = repo.path setting ) +# CGIT_REPO_OWNER ( = repo.owner setting ) +# CGIT_REPO_DEFBRANCH ( = repo.defbranch setting ) +# CGIT_REPO_SECTION ( = section setting ) +# CGIT_REPO_CLONE_URL ( = repo.clone-url setting ) +# + +# store filename and extension in local vars +BASENAME="$1" +EXTENSION="${BASENAME##*.}" + +[ "${BASENAME}" = "${EXTENSION}" ] && EXTENSION=txt +[ -z "${EXTENSION}" ] && EXTENSION=txt + +# map Makefile and Makefile.* to .mk +[ "${BASENAME%%.*}" = "Makefile" ] && EXTENSION=mk + +# highlight versions 2 and 3 have different commandline options. Specifically, +# the -X option that is used for version 2 is replaced by the -O xhtml option +# for version 3. +# +# Version 2 can be found (for example) on EPEL 5, while version 3 can be +# found (for example) on EPEL 6. +# +# This is for version 2 +exec highlight --force -f -I -X -S "$EXTENSION" 2>/dev/null + +# This is for version 3 +#exec highlight --force -f -I -O xhtml -S "$EXTENSION" 2>/dev/null diff --git a/third_party/cgit/gen-version.sh b/third_party/cgit/gen-version.sh new file mode 100755 index 000000000000..80cf49af486b --- /dev/null +++ b/third_party/cgit/gen-version.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# Get version-info specified in Makefile +V=$1 + +# Use `git describe` to get current version if we're inside a git repo +if test "$(git rev-parse --git-dir 2>/dev/null)" = '.git' +then + V=$(git describe --abbrev=4 HEAD 2>/dev/null) +fi + +new="CGIT_VERSION = $V" +old=$(cat VERSION 2>/dev/null) + +# Exit if VERSION is uptodate +test "$old" = "$new" && exit 0 + +# Update VERSION with new version-info +echo "$new" > VERSION +cat VERSION diff --git a/third_party/cgit/html.c b/third_party/cgit/html.c new file mode 100644 index 000000000000..7f81965fddcd --- /dev/null +++ b/third_party/cgit/html.c @@ -0,0 +1,344 @@ +/* html.c: helper functions for html output + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "html.h" +#include "url.h" + +/* Percent-encoding of each character, except: a-zA-Z0-9!$()*,./:;@- */ +static const char* url_escape_table[256] = { + "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07", + "%08", "%09", "%0a", "%0b", "%0c", "%0d", "%0e", "%0f", + "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17", + "%18", "%19", "%1a", "%1b", "%1c", "%1d", "%1e", "%1f", + "%20", NULL, "%22", "%23", NULL, "%25", "%26", "%27", + NULL, NULL, NULL, "%2b", NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, "%3c", "%3d", "%3e", "%3f", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, "%5c", NULL, "%5e", NULL, + "%60", NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, "%7b", "%7c", "%7d", NULL, "%7f", + "%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87", + "%88", "%89", "%8a", "%8b", "%8c", "%8d", "%8e", "%8f", + "%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97", + "%98", "%99", "%9a", "%9b", "%9c", "%9d", "%9e", "%9f", + "%a0", "%a1", "%a2", "%a3", "%a4", "%a5", "%a6", "%a7", + "%a8", "%a9", "%aa", "%ab", "%ac", "%ad", "%ae", "%af", + "%b0", "%b1", "%b2", "%b3", "%b4", "%b5", "%b6", "%b7", + "%b8", "%b9", "%ba", "%bb", "%bc", "%bd", "%be", "%bf", + "%c0", "%c1", "%c2", "%c3", "%c4", "%c5", "%c6", "%c7", + "%c8", "%c9", "%ca", "%cb", "%cc", "%cd", "%ce", "%cf", + "%d0", "%d1", "%d2", "%d3", "%d4", "%d5", "%d6", "%d7", + "%d8", "%d9", "%da", "%db", "%dc", "%dd", "%de", "%df", + "%e0", "%e1", "%e2", "%e3", "%e4", "%e5", "%e6", "%e7", + "%e8", "%e9", "%ea", "%eb", "%ec", "%ed", "%ee", "%ef", + "%f0", "%f1", "%f2", "%f3", "%f4", "%f5", "%f6", "%f7", + "%f8", "%f9", "%fa", "%fb", "%fc", "%fd", "%fe", "%ff" +}; + +char *fmt(const char *format, ...) +{ + static char buf[8][1024]; + static int bufidx; + int len; + va_list args; + + bufidx++; + bufidx &= 7; + + va_start(args, format); + len = vsnprintf(buf[bufidx], sizeof(buf[bufidx]), format, args); + va_end(args); + if (len > sizeof(buf[bufidx])) { + fprintf(stderr, "[html.c] string truncated: %s\n", format); + exit(1); + } + return buf[bufidx]; +} + +char *fmtalloc(const char *format, ...) +{ + struct strbuf sb = STRBUF_INIT; + va_list args; + + va_start(args, format); + strbuf_vaddf(&sb, format, args); + va_end(args); + + return strbuf_detach(&sb, NULL); +} + +void html_raw(const char *data, size_t size) +{ + if (write(STDOUT_FILENO, data, size) != size) + die_errno("write error on html output"); +} + +void html(const char *txt) +{ + html_raw(txt, strlen(txt)); +} + +void htmlf(const char *format, ...) +{ + va_list args; + struct strbuf buf = STRBUF_INIT; + + va_start(args, format); + strbuf_vaddf(&buf, format, args); + va_end(args); + html(buf.buf); + strbuf_release(&buf); +} + +void html_txtf(const char *format, ...) +{ + va_list args; + + va_start(args, format); + html_vtxtf(format, args); + va_end(args); +} + +void html_vtxtf(const char *format, va_list ap) +{ + va_list cp; + struct strbuf buf = STRBUF_INIT; + + va_copy(cp, ap); + strbuf_vaddf(&buf, format, cp); + va_end(cp); + html_txt(buf.buf); + strbuf_release(&buf); +} + +void html_txt(const char *txt) +{ + if (txt) + html_ntxt(txt, strlen(txt)); +} + +ssize_t html_ntxt(const char *txt, size_t len) +{ + const char *t = txt; + ssize_t slen; + + if (len > SSIZE_MAX) + return -1; + + slen = (ssize_t) len; + while (t && *t && slen--) { + int c = *t; + if (c == '<' || c == '>' || c == '&') { + html_raw(txt, t - txt); + if (c == '>') + html(">"); + else if (c == '<') + html("<"); + else if (c == '&') + html("&"); + txt = t + 1; + } + t++; + } + if (t != txt) + html_raw(txt, t - txt); + return slen; +} + +void html_attrf(const char *fmt, ...) +{ + va_list ap; + struct strbuf sb = STRBUF_INIT; + + va_start(ap, fmt); + strbuf_vaddf(&sb, fmt, ap); + va_end(ap); + + html_attr(sb.buf); + strbuf_release(&sb); +} + +void html_attr(const char *txt) +{ + const char *t = txt; + while (t && *t) { + int c = *t; + if (c == '<' || c == '>' || c == '\'' || c == '\"' || c == '&') { + html_raw(txt, t - txt); + if (c == '>') + html(">"); + else if (c == '<') + html("<"); + else if (c == '\'') + html("'"); + else if (c == '"') + html("""); + else if (c == '&') + html("&"); + txt = t + 1; + } + t++; + } + if (t != txt) + html(txt); +} + +void html_url_path(const char *txt) +{ + const char *t = txt; + while (t && *t) { + unsigned char c = *t; + const char *e = url_escape_table[c]; + if (e && c != '+' && c != '&') { + html_raw(txt, t - txt); + html(e); + txt = t + 1; + } + t++; + } + if (t != txt) + html(txt); +} + +void html_url_arg(const char *txt) +{ + const char *t = txt; + while (t && *t) { + unsigned char c = *t; + const char *e = url_escape_table[c]; + if (c == ' ') + e = "+"; + if (e) { + html_raw(txt, t - txt); + html(e); + txt = t + 1; + } + t++; + } + if (t != txt) + html(txt); +} + +void html_header_arg_in_quotes(const char *txt) +{ + const char *t = txt; + while (t && *t) { + unsigned char c = *t; + const char *e = NULL; + if (c == '\\') + e = "\\\\"; + else if (c == '\r') + e = "\\r"; + else if (c == '\n') + e = "\\n"; + else if (c == '"') + e = "\\\""; + if (e) { + html_raw(txt, t - txt); + html(e); + txt = t + 1; + } + t++; + } + if (t != txt) + html(txt); + +} + +void html_hidden(const char *name, const char *value) +{ + html("<input type='hidden' name='"); + html_attr(name); + html("' value='"); + html_attr(value); + html("'/>"); +} + +void html_option(const char *value, const char *text, const char *selected_value) +{ + html("<option value='"); + html_attr(value); + html("'"); + if (selected_value && !strcmp(selected_value, value)) + html(" selected='selected'"); + html(">"); + html_txt(text); + html("</option>\n"); +} + +void html_intoption(int value, const char *text, int selected_value) +{ + htmlf("<option value='%d'%s>", value, + value == selected_value ? " selected='selected'" : ""); + html_txt(text); + html("</option>"); +} + +void html_link_open(const char *url, const char *title, const char *class) +{ + html("<a href='"); + html_attr(url); + if (title) { + html("' title='"); + html_attr(title); + } + if (class) { + html("' class='"); + html_attr(class); + } + html("'>"); +} + +void html_link_close(void) +{ + html("</a>"); +} + +void html_fileperm(unsigned short mode) +{ + htmlf("%c%c%c", (mode & 4 ? 'r' : '-'), + (mode & 2 ? 'w' : '-'), (mode & 1 ? 'x' : '-')); +} + +int html_include(const char *filename) +{ + FILE *f; + char buf[4096]; + size_t len; + + if (!(f = fopen(filename, "r"))) { + fprintf(stderr, "[cgit] Failed to include file %s: %s (%d).\n", + filename, strerror(errno), errno); + return -1; + } + while ((len = fread(buf, 1, 4096, f)) > 0) + html_raw(buf, len); + fclose(f); + return 0; +} + +void http_parse_querystring(const char *txt, void (*fn)(const char *name, const char *value)) +{ + const char *t = txt; + + while (t && *t) { + char *name = url_decode_parameter_name(&t); + if (*name) { + char *value = url_decode_parameter_value(&t); + fn(name, value); + free(value); + } + free(name); + } +} diff --git a/third_party/cgit/html.h b/third_party/cgit/html.h new file mode 100644 index 000000000000..fa4de77587f2 --- /dev/null +++ b/third_party/cgit/html.h @@ -0,0 +1,37 @@ +#ifndef HTML_H +#define HTML_H + +#include "cgit.h" + +extern void html_raw(const char *txt, size_t size); +extern void html(const char *txt); + +__attribute__((format (printf,1,2))) +extern void htmlf(const char *format,...); + +__attribute__((format (printf,1,2))) +extern void html_txtf(const char *format,...); + +__attribute__((format (printf,1,0))) +extern void html_vtxtf(const char *format, va_list ap); + +__attribute__((format (printf,1,2))) +extern void html_attrf(const char *format,...); + +extern void html_txt(const char *txt); +extern ssize_t html_ntxt(const char *txt, size_t len); +extern void html_attr(const char *txt); +extern void html_url_path(const char *txt); +extern void html_url_arg(const char *txt); +extern void html_header_arg_in_quotes(const char *txt); +extern void html_hidden(const char *name, const char *value); +extern void html_option(const char *value, const char *text, const char *selected_value); +extern void html_intoption(int value, const char *text, int selected_value); +extern void html_link_open(const char *url, const char *title, const char *class); +extern void html_link_close(void); +extern void html_fileperm(unsigned short mode); +extern int html_include(const char *filename); + +extern void http_parse_querystring(const char *txt, void (*fn)(const char *name, const char *value)); + +#endif /* HTML_H */ diff --git a/third_party/cgit/parsing.c b/third_party/cgit/parsing.c new file mode 100644 index 000000000000..e093aaf7019c --- /dev/null +++ b/third_party/cgit/parsing.c @@ -0,0 +1,223 @@ +/* parsing.c: parsing of config files + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" + +/* + * url syntax: [repo ['/' cmd [ '/' path]]] + * repo: any valid repo url, may contain '/' + * cmd: log | commit | diff | tree | view | blob | snapshot + * path: any valid path, may contain '/' + * + */ +void cgit_parse_url(const char *url) +{ + char *c, *cmd, *p; + struct cgit_repo *repo; + + if (!url || url[0] == '\0') + return; + + ctx.qry.page = NULL; + ctx.repo = cgit_get_repoinfo(url); + if (ctx.repo) { + ctx.qry.repo = ctx.repo->url; + return; + } + + cmd = NULL; + c = strchr(url, '/'); + while (c) { + c[0] = '\0'; + repo = cgit_get_repoinfo(url); + if (repo) { + ctx.repo = repo; + cmd = c; + } + c[0] = '/'; + c = strchr(c + 1, '/'); + } + + if (ctx.repo) { + ctx.qry.repo = ctx.repo->url; + p = strchr(cmd + 1, '/'); + if (p) { + p[0] = '\0'; + if (p[1]) + ctx.qry.path = trim_end(p + 1, '/'); + } + if (cmd[1]) + ctx.qry.page = xstrdup(cmd + 1); + } +} + +static char *substr(const char *head, const char *tail) +{ + char *buf; + + if (tail < head) + return xstrdup(""); + buf = xmalloc(tail - head + 1); + strlcpy(buf, head, tail - head + 1); + return buf; +} + +static void parse_user(const char *t, char **name, char **email, unsigned long *date, int *tz) +{ + struct ident_split ident; + unsigned email_len; + + if (!split_ident_line(&ident, t, (uintptr_t)strchrnul(t, '\n') - (uintptr_t)t)) { + *name = substr(ident.name_begin, ident.name_end); + + email_len = ident.mail_end - ident.mail_begin; + *email = xmalloc(strlen("<") + email_len + strlen(">") + 1); + xsnprintf(*email, email_len + 3, "<%.*s>", email_len, ident.mail_begin); + + if (ident.date_begin) + *date = strtoul(ident.date_begin, NULL, 10); + if (ident.tz_begin) + *tz = atoi(ident.tz_begin); + } +} + +#ifdef NO_ICONV +#define reencode(a, b, c) +#else +static const char *reencode(char **txt, const char *src_enc, const char *dst_enc) +{ + char *tmp; + + if (!txt) + return NULL; + + if (!*txt || !src_enc || !dst_enc) + return *txt; + + /* no encoding needed if src_enc equals dst_enc */ + if (!strcasecmp(src_enc, dst_enc)) + return *txt; + + tmp = reencode_string(*txt, dst_enc, src_enc); + if (tmp) { + free(*txt); + *txt = tmp; + } + return *txt; +} +#endif + +static const char *next_header_line(const char *p) +{ + p = strchr(p, '\n'); + if (!p) + return NULL; + return p + 1; +} + +static int end_of_header(const char *p) +{ + return !p || (*p == '\n'); +} + +struct commitinfo *cgit_parse_commit(struct commit *commit) +{ + struct commitinfo *ret; + const char *p = repo_get_commit_buffer(the_repository, commit, NULL); + const char *t; + + ret = xcalloc(1, sizeof(struct commitinfo)); + ret->commit = commit; + + if (!p) + return ret; + + if (!skip_prefix(p, "tree ", &p)) + die("Bad commit: %s", oid_to_hex(&commit->object.oid)); + p += the_hash_algo->hexsz + 1; + + while (skip_prefix(p, "parent ", &p)) + p += the_hash_algo->hexsz + 1; + + if (p && skip_prefix(p, "author ", &p)) { + parse_user(p, &ret->author, &ret->author_email, + &ret->author_date, &ret->author_tz); + p = next_header_line(p); + } + + if (p && skip_prefix(p, "committer ", &p)) { + parse_user(p, &ret->committer, &ret->committer_email, + &ret->committer_date, &ret->committer_tz); + p = next_header_line(p); + } + + if (p && skip_prefix(p, "encoding ", &p)) { + t = strchr(p, '\n'); + if (t) { + ret->msg_encoding = substr(p, t + 1); + p = t + 1; + } + } + + if (!ret->msg_encoding) + ret->msg_encoding = xstrdup("UTF-8"); + + while (!end_of_header(p)) + p = next_header_line(p); + while (p && *p == '\n') + p++; + if (!p) + return ret; + + t = strchrnul(p, '\n'); + ret->subject = substr(p, t); + while (*t == '\n') + t++; + ret->msg = xstrdup(t); + + reencode(&ret->author, ret->msg_encoding, PAGE_ENCODING); + reencode(&ret->author_email, ret->msg_encoding, PAGE_ENCODING); + reencode(&ret->committer, ret->msg_encoding, PAGE_ENCODING); + reencode(&ret->committer_email, ret->msg_encoding, PAGE_ENCODING); + reencode(&ret->subject, ret->msg_encoding, PAGE_ENCODING); + reencode(&ret->msg, ret->msg_encoding, PAGE_ENCODING); + + return ret; +} + +struct taginfo *cgit_parse_tag(struct tag *tag) +{ + void *data; + enum object_type type; + unsigned long size; + const char *p; + struct taginfo *ret = NULL; + + data = read_object_file(&tag->object.oid, &type, &size); + if (!data || type != OBJ_TAG) + goto cleanup; + + ret = xcalloc(1, sizeof(struct taginfo)); + + for (p = data; !end_of_header(p); p = next_header_line(p)) { + if (skip_prefix(p, "tagger ", &p)) { + parse_user(p, &ret->tagger, &ret->tagger_email, + &ret->tagger_date, &ret->tagger_tz); + } + } + + while (p && *p == '\n') + p++; + + if (p && *p) + ret->msg = xstrdup(p); + +cleanup: + free(data); + return ret; +} diff --git a/third_party/cgit/robots.txt b/third_party/cgit/robots.txt new file mode 100644 index 000000000000..4ce948fec2c9 --- /dev/null +++ b/third_party/cgit/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Disallow: /*/snapshot/* +Allow: / diff --git a/third_party/cgit/scan-tree.c b/third_party/cgit/scan-tree.c new file mode 100644 index 000000000000..6a2f65a86b2c --- /dev/null +++ b/third_party/cgit/scan-tree.c @@ -0,0 +1,270 @@ +/* scan-tree.c + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "scan-tree.h" +#include "configfile.h" +#include "html.h" +#include <config.h> + +/* return 1 if path contains a objects/ directory and a HEAD file */ +static int is_git_dir(const char *path) +{ + struct stat st; + struct strbuf pathbuf = STRBUF_INIT; + int result = 0; + + strbuf_addf(&pathbuf, "%s/objects", path); + if (stat(pathbuf.buf, &st)) { + if (errno != ENOENT) + fprintf(stderr, "Error checking path %s: %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + if (!S_ISDIR(st.st_mode)) + goto out; + + strbuf_reset(&pathbuf); + strbuf_addf(&pathbuf, "%s/HEAD", path); + if (stat(pathbuf.buf, &st)) { + if (errno != ENOENT) + fprintf(stderr, "Error checking path %s: %s (%d)\n", + path, strerror(errno), errno); + goto out; + } + if (!S_ISREG(st.st_mode)) + goto out; + + result = 1; +out: + strbuf_release(&pathbuf); + return result; +} + +static struct cgit_repo *repo; +static repo_config_fn config_fn; + +static void scan_tree_repo_config(const char *name, const char *value) +{ + config_fn(repo, name, value); +} + +static int gitconfig_config(const char *key, const char *value, void *cb) +{ + const char *name; + + if (!strcmp(key, "gitweb.owner")) + config_fn(repo, "owner", value); + else if (!strcmp(key, "gitweb.description")) + config_fn(repo, "desc", value); + else if (!strcmp(key, "gitweb.category")) + config_fn(repo, "section", value); + else if (!strcmp(key, "gitweb.homepage")) + config_fn(repo, "homepage", value); + else if (skip_prefix(key, "cgit.", &name)) + config_fn(repo, name, value); + + return 0; +} + +static char *xstrrchr(char *s, char *from, int c) +{ + while (from >= s && *from != c) + from--; + return from < s ? NULL : from; +} + +static void add_repo(const char *base, struct strbuf *path, repo_config_fn fn) +{ + struct stat st; + struct passwd *pwd; + size_t pathlen; + struct strbuf rel = STRBUF_INIT; + char *p, *slash; + int n; + size_t size; + + if (stat(path->buf, &st)) { + fprintf(stderr, "Error accessing %s: %s (%d)\n", + path->buf, strerror(errno), errno); + return; + } + + strbuf_addch(path, '/'); + pathlen = path->len; + + if (ctx.cfg.strict_export) { + strbuf_addstr(path, ctx.cfg.strict_export); + if(stat(path->buf, &st)) + return; + strbuf_setlen(path, pathlen); + } + + strbuf_addstr(path, "noweb"); + if (!stat(path->buf, &st)) + return; + strbuf_setlen(path, pathlen); + + if (!starts_with(path->buf, base)) + strbuf_addbuf(&rel, path); + else + strbuf_addstr(&rel, path->buf + strlen(base) + 1); + + if (!strcmp(rel.buf + rel.len - 5, "/.git")) + strbuf_setlen(&rel, rel.len - 5); + else if (rel.len && rel.buf[rel.len - 1] == '/') + strbuf_setlen(&rel, rel.len - 1); + + repo = cgit_add_repo(rel.buf); + config_fn = fn; + if (ctx.cfg.enable_git_config) { + strbuf_addstr(path, "config"); + git_config_from_file(gitconfig_config, path->buf, NULL); + strbuf_setlen(path, pathlen); + } + + if (ctx.cfg.remove_suffix) { + size_t urllen; + strip_suffix(repo->url, ".git", &urllen); + strip_suffix_mem(repo->url, &urllen, "/"); + repo->url[urllen] = '\0'; + } + repo->path = xstrdup(path->buf); + while (!repo->owner) { + if ((pwd = getpwuid(st.st_uid)) == NULL) { + fprintf(stderr, "Error reading owner-info for %s: %s (%d)\n", + path->buf, strerror(errno), errno); + break; + } + if (pwd->pw_gecos) + if ((p = strchr(pwd->pw_gecos, ','))) + *p = '\0'; + repo->owner = xstrdup(pwd->pw_gecos ? pwd->pw_gecos : pwd->pw_name); + } + + if (repo->desc == cgit_default_repo_desc || !repo->desc) { + strbuf_addstr(path, "description"); + if (!stat(path->buf, &st)) + readfile(path->buf, &repo->desc, &size); + strbuf_setlen(path, pathlen); + } + + if (ctx.cfg.section_from_path) { + n = ctx.cfg.section_from_path; + if (n > 0) { + slash = rel.buf - 1; + while (slash && n && (slash = strchr(slash + 1, '/'))) + n--; + } else { + slash = rel.buf + rel.len; + while (slash && n && (slash = xstrrchr(rel.buf, slash - 1, '/'))) + n++; + } + if (slash && !n) { + *slash = '\0'; + repo->section = xstrdup(rel.buf); + *slash = '/'; + if (starts_with(repo->name, repo->section)) { + repo->name += strlen(repo->section); + if (*repo->name == '/') + repo->name++; + } + } + } + + strbuf_addstr(path, "cgitrc"); + if (!stat(path->buf, &st)) + parse_configfile(path->buf, &scan_tree_repo_config); + + strbuf_release(&rel); +} + +static void scan_path(const char *base, const char *path, repo_config_fn fn) +{ + DIR *dir = opendir(path); + struct dirent *ent; + struct strbuf pathbuf = STRBUF_INIT; + size_t pathlen = strlen(path); + struct stat st; + + if (!dir) { + fprintf(stderr, "Error opening directory %s: %s (%d)\n", + path, strerror(errno), errno); + return; + } + + strbuf_add(&pathbuf, path, strlen(path)); + if (is_git_dir(pathbuf.buf)) { + add_repo(base, &pathbuf, fn); + goto end; + } + strbuf_addstr(&pathbuf, "/.git"); + if (is_git_dir(pathbuf.buf)) { + add_repo(base, &pathbuf, fn); + goto end; + } + /* + * Add one because we don't want to lose the trailing '/' when we + * reset the length of pathbuf in the loop below. + */ + pathlen++; + while ((ent = readdir(dir)) != NULL) { + if (ent->d_name[0] == '.') { + if (ent->d_name[1] == '\0') + continue; + if (ent->d_name[1] == '.' && ent->d_name[2] == '\0') + continue; + if (!ctx.cfg.scan_hidden_path) + continue; + } + strbuf_setlen(&pathbuf, pathlen); + strbuf_addstr(&pathbuf, ent->d_name); + if (stat(pathbuf.buf, &st)) { + fprintf(stderr, "Error checking path %s: %s (%d)\n", + pathbuf.buf, strerror(errno), errno); + continue; + } + if (S_ISDIR(st.st_mode)) + scan_path(base, pathbuf.buf, fn); + } +end: + strbuf_release(&pathbuf); + closedir(dir); +} + +void scan_projects(const char *path, const char *projectsfile, repo_config_fn fn) +{ + struct strbuf line = STRBUF_INIT; + FILE *projects; + int err; + + projects = fopen(projectsfile, "r"); + if (!projects) { + fprintf(stderr, "Error opening projectsfile %s: %s (%d)\n", + projectsfile, strerror(errno), errno); + return; + } + while (strbuf_getline(&line, projects) != EOF) { + if (!line.len) + continue; + strbuf_insert(&line, 0, "/", 1); + strbuf_insert(&line, 0, path, strlen(path)); + scan_path(path, line.buf, fn); + } + if ((err = ferror(projects))) { + fprintf(stderr, "Error reading from projectsfile %s: %s (%d)\n", + projectsfile, strerror(err), err); + } + fclose(projects); + strbuf_release(&line); +} + +void scan_tree(const char *path, repo_config_fn fn) +{ + scan_path(path, path, fn); +} diff --git a/third_party/cgit/scan-tree.h b/third_party/cgit/scan-tree.h new file mode 100644 index 000000000000..1afbd4bbcdbf --- /dev/null +++ b/third_party/cgit/scan-tree.h @@ -0,0 +1,2 @@ +extern void scan_projects(const char *path, const char *projectsfile, repo_config_fn fn); +extern void scan_tree(const char *path, repo_config_fn fn); diff --git a/third_party/cgit/shared.c b/third_party/cgit/shared.c new file mode 100644 index 000000000000..8115469a7cec --- /dev/null +++ b/third_party/cgit/shared.c @@ -0,0 +1,579 @@ +/* shared.c: global vars + some callback functions + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" + +struct cgit_repolist cgit_repolist; +struct cgit_context ctx; + +int chk_zero(int result, char *msg) +{ + if (result != 0) + die_errno("%s", msg); + return result; +} + +int chk_positive(int result, char *msg) +{ + if (result <= 0) + die_errno("%s", msg); + return result; +} + +int chk_non_negative(int result, char *msg) +{ + if (result < 0) + die_errno("%s", msg); + return result; +} + +char *cgit_default_repo_desc = "[no description]"; +struct cgit_repo *cgit_add_repo(const char *url) +{ + struct cgit_repo *ret; + + if (++cgit_repolist.count > cgit_repolist.length) { + if (cgit_repolist.length == 0) + cgit_repolist.length = 8; + else + cgit_repolist.length *= 2; + cgit_repolist.repos = xrealloc(cgit_repolist.repos, + cgit_repolist.length * + sizeof(struct cgit_repo)); + } + + ret = &cgit_repolist.repos[cgit_repolist.count-1]; + memset(ret, 0, sizeof(struct cgit_repo)); + ret->url = trim_end(url, '/'); + ret->name = ret->url; + ret->path = NULL; + ret->desc = cgit_default_repo_desc; + ret->extra_head_content = NULL; + ret->owner = NULL; + ret->homepage = NULL; + ret->section = ctx.cfg.section; + ret->snapshots = ctx.cfg.snapshots; + ret->enable_blame = ctx.cfg.enable_blame; + ret->enable_commit_graph = ctx.cfg.enable_commit_graph; + ret->enable_log_filecount = ctx.cfg.enable_log_filecount; + ret->enable_log_linecount = ctx.cfg.enable_log_linecount; + ret->enable_remote_branches = ctx.cfg.enable_remote_branches; + ret->enable_subject_links = ctx.cfg.enable_subject_links; + ret->enable_html_serving = ctx.cfg.enable_html_serving; + ret->max_stats = ctx.cfg.max_stats; + ret->branch_sort = ctx.cfg.branch_sort; + ret->commit_sort = ctx.cfg.commit_sort; + ret->module_link = ctx.cfg.module_link; + ret->readme = ctx.cfg.readme; + ret->mtime = -1; + ret->about_filter = ctx.cfg.about_filter; + ret->commit_filter = ctx.cfg.commit_filter; + ret->source_filter = ctx.cfg.source_filter; + ret->email_filter = ctx.cfg.email_filter; + ret->owner_filter = ctx.cfg.owner_filter; + ret->clone_url = ctx.cfg.clone_url; + ret->submodules.strdup_strings = 1; + ret->hide = ret->ignore = 0; + return ret; +} + +struct cgit_repo *cgit_get_repoinfo(const char *url) +{ + int i; + struct cgit_repo *repo; + + for (i = 0; i < cgit_repolist.count; i++) { + repo = &cgit_repolist.repos[i]; + if (repo->ignore) + continue; + if (!strcmp(repo->url, url)) + return repo; + } + return NULL; +} + +void cgit_free_commitinfo(struct commitinfo *info) +{ + free(info->author); + free(info->author_email); + free(info->committer); + free(info->committer_email); + free(info->subject); + free(info->msg); + free(info->msg_encoding); + free(info); +} + +char *trim_end(const char *str, char c) +{ + int len; + + if (str == NULL) + return NULL; + len = strlen(str); + while (len > 0 && str[len - 1] == c) + len--; + if (len == 0) + return NULL; + return xstrndup(str, len); +} + +char *ensure_end(const char *str, char c) +{ + size_t len = strlen(str); + char *result; + + if (len && str[len - 1] == c) + return xstrndup(str, len); + + result = xmalloc(len + 2); + memcpy(result, str, len); + result[len] = '/'; + result[len + 1] = '\0'; + return result; +} + +void strbuf_ensure_end(struct strbuf *sb, char c) +{ + if (!sb->len || sb->buf[sb->len - 1] != c) + strbuf_addch(sb, c); +} + +void cgit_add_ref(struct reflist *list, struct refinfo *ref) +{ + size_t size; + + if (list->count >= list->alloc) { + list->alloc += (list->alloc ? list->alloc : 4); + size = list->alloc * sizeof(struct refinfo *); + list->refs = xrealloc(list->refs, size); + } + list->refs[list->count++] = ref; +} + +static struct refinfo *cgit_mk_refinfo(const char *refname, const struct object_id *oid) +{ + struct refinfo *ref; + + ref = xmalloc(sizeof (struct refinfo)); + ref->refname = xstrdup(refname); + ref->object = parse_object(the_repository, oid); + switch (ref->object->type) { + case OBJ_TAG: + ref->tag = cgit_parse_tag((struct tag *)ref->object); + break; + case OBJ_COMMIT: + ref->commit = cgit_parse_commit((struct commit *)ref->object); + break; + } + return ref; +} + +void cgit_free_taginfo(struct taginfo *tag) +{ + if (tag->tagger) + free(tag->tagger); + if (tag->tagger_email) + free(tag->tagger_email); + if (tag->msg) + free(tag->msg); + free(tag); +} + +static void cgit_free_refinfo(struct refinfo *ref) +{ + if (ref->refname) + free((char *)ref->refname); + switch (ref->object->type) { + case OBJ_TAG: + cgit_free_taginfo(ref->tag); + break; + case OBJ_COMMIT: + cgit_free_commitinfo(ref->commit); + break; + } + free(ref); +} + +void cgit_free_reflist_inner(struct reflist *list) +{ + int i; + + for (i = 0; i < list->count; i++) { + cgit_free_refinfo(list->refs[i]); + } + free(list->refs); +} + +int cgit_refs_cb(const char *refname, const struct object_id *oid, int flags, + void *cb_data) +{ + struct reflist *list = (struct reflist *)cb_data; + struct refinfo *info = cgit_mk_refinfo(refname, oid); + + if (info) + cgit_add_ref(list, info); + return 0; +} + +void cgit_diff_tree_cb(struct diff_queue_struct *q, + struct diff_options *options, void *data) +{ + int i; + + for (i = 0; i < q->nr; i++) { + if (q->queue[i]->status == 'U') + continue; + ((filepair_fn)data)(q->queue[i]); + } +} + +static int load_mmfile(mmfile_t *file, const struct object_id *oid) +{ + enum object_type type; + + if (is_null_oid(oid)) { + file->ptr = (char *)""; + file->size = 0; + } else { + file->ptr = read_object_file(oid, &type, + (unsigned long *)&file->size); + } + return 1; +} + +/* + * Receive diff-buffers from xdiff and concatenate them as + * needed across multiple callbacks. + * + * This is basically a copy of xdiff-interface.c/xdiff_outf(), + * ripped from git and modified to use globals instead of + * a special callback-struct. + */ +static char *diffbuf = NULL; +static int buflen = 0; + +static int filediff_cb(void *priv, mmbuffer_t *mb, int nbuf) +{ + int i; + + for (i = 0; i < nbuf; i++) { + if (mb[i].ptr[mb[i].size-1] != '\n') { + /* Incomplete line */ + diffbuf = xrealloc(diffbuf, buflen + mb[i].size); + memcpy(diffbuf + buflen, mb[i].ptr, mb[i].size); + buflen += mb[i].size; + continue; + } + + /* we have a complete line */ + if (!diffbuf) { + ((linediff_fn)priv)(mb[i].ptr, mb[i].size); + continue; + } + diffbuf = xrealloc(diffbuf, buflen + mb[i].size); + memcpy(diffbuf + buflen, mb[i].ptr, mb[i].size); + ((linediff_fn)priv)(diffbuf, buflen + mb[i].size); + free(diffbuf); + diffbuf = NULL; + buflen = 0; + } + if (diffbuf) { + ((linediff_fn)priv)(diffbuf, buflen); + free(diffbuf); + diffbuf = NULL; + buflen = 0; + } + return 0; +} + +int cgit_diff_files(const struct object_id *old_oid, + const struct object_id *new_oid, unsigned long *old_size, + unsigned long *new_size, int *binary, int context, + int ignorews, linediff_fn fn) +{ + mmfile_t file1, file2; + xpparam_t diff_params; + xdemitconf_t emit_params; + xdemitcb_t emit_cb; + + if (!load_mmfile(&file1, old_oid) || !load_mmfile(&file2, new_oid)) + return 1; + + *old_size = file1.size; + *new_size = file2.size; + + if ((file1.ptr && buffer_is_binary(file1.ptr, file1.size)) || + (file2.ptr && buffer_is_binary(file2.ptr, file2.size))) { + *binary = 1; + if (file1.size) + free(file1.ptr); + if (file2.size) + free(file2.ptr); + return 0; + } + + memset(&diff_params, 0, sizeof(diff_params)); + memset(&emit_params, 0, sizeof(emit_params)); + memset(&emit_cb, 0, sizeof(emit_cb)); + diff_params.flags = XDF_NEED_MINIMAL; + if (ignorews) + diff_params.flags |= XDF_IGNORE_WHITESPACE; + emit_params.ctxlen = context > 0 ? context : 3; + emit_params.flags = XDL_EMIT_FUNCNAMES; + emit_cb.out_line = filediff_cb; + emit_cb.priv = fn; + xdl_diff(&file1, &file2, &diff_params, &emit_params, &emit_cb); + if (file1.size) + free(file1.ptr); + if (file2.size) + free(file2.ptr); + return 0; +} + +void cgit_diff_tree(const struct object_id *old_oid, + const struct object_id *new_oid, + filepair_fn fn, const char *prefix, int ignorews) +{ + struct diff_options opt; + struct pathspec_item item; + + memset(&item, 0, sizeof(item)); + diff_setup(&opt); + opt.output_format = DIFF_FORMAT_CALLBACK; + opt.detect_rename = 1; + opt.rename_limit = ctx.cfg.renamelimit; + opt.flags.recursive = 1; + if (ignorews) + DIFF_XDL_SET(&opt, IGNORE_WHITESPACE); + opt.format_callback = cgit_diff_tree_cb; + opt.format_callback_data = fn; + if (prefix) { + item.match = xstrdup(prefix); + item.len = strlen(prefix); + opt.pathspec.nr = 1; + opt.pathspec.items = &item; + } + diff_setup_done(&opt); + + if (old_oid && !is_null_oid(old_oid)) + diff_tree_oid(old_oid, new_oid, "", &opt); + else + diff_root_tree_oid(new_oid, "", &opt); + diffcore_std(&opt); + diff_flush(&opt); + + free(item.match); +} + +void cgit_diff_commit(struct commit *commit, filepair_fn fn, const char *prefix) +{ + const struct object_id *old_oid = NULL; + + if (commit->parents) + old_oid = &commit->parents->item->object.oid; + cgit_diff_tree(old_oid, &commit->object.oid, fn, prefix, + ctx.qry.ignorews); +} + +int cgit_parse_snapshots_mask(const char *str) +{ + struct string_list tokens = STRING_LIST_INIT_DUP; + struct string_list_item *item; + const struct cgit_snapshot_format *f; + int rv = 0; + + /* favor legacy setting */ + if (atoi(str)) + return 1; + + if (strcmp(str, "all") == 0) + return INT_MAX; + + string_list_split(&tokens, str, ' ', -1); + string_list_remove_empty_items(&tokens, 0); + + for_each_string_list_item(item, &tokens) { + for (f = cgit_snapshot_formats; f->suffix; f++) { + if (!strcmp(item->string, f->suffix) || + !strcmp(item->string, f->suffix + 1)) { + rv |= cgit_snapshot_format_bit(f); + break; + } + } + } + + string_list_clear(&tokens, 0); + return rv; +} + +typedef struct { + char * name; + char * value; +} cgit_env_var; + +void cgit_prepare_repo_env(struct cgit_repo * repo) +{ + cgit_env_var env_vars[] = { + { .name = "CGIT_REPO_URL", .value = repo->url }, + { .name = "CGIT_REPO_NAME", .value = repo->name }, + { .name = "CGIT_REPO_PATH", .value = repo->path }, + { .name = "CGIT_REPO_OWNER", .value = repo->owner }, + { .name = "CGIT_REPO_DEFBRANCH", .value = repo->defbranch }, + { .name = "CGIT_REPO_SECTION", .value = repo->section }, + { .name = "CGIT_REPO_CLONE_URL", .value = repo->clone_url } + }; + int env_var_count = ARRAY_SIZE(env_vars); + cgit_env_var *p, *q; + static char *warn = "cgit warning: failed to set env: %s=%s\n"; + + p = env_vars; + q = p + env_var_count; + for (; p < q; p++) + if (p->value && setenv(p->name, p->value, 1)) + fprintf(stderr, warn, p->name, p->value); +} + +/* Read the content of the specified file into a newly allocated buffer, + * zeroterminate the buffer and return 0 on success, errno otherwise. + */ +int readfile(const char *path, char **buf, size_t *size) +{ + int fd, e; + struct stat st; + + fd = open(path, O_RDONLY); + if (fd == -1) + return errno; + if (fstat(fd, &st)) { + e = errno; + close(fd); + return e; + } + if (!S_ISREG(st.st_mode)) { + close(fd); + return EISDIR; + } + *buf = xmalloc(st.st_size + 1); + *size = read_in_full(fd, *buf, st.st_size); + e = errno; + (*buf)[*size] = '\0'; + close(fd); + return (*size == st.st_size ? 0 : e); +} + +static int is_token_char(char c) +{ + return isalnum(c) || c == '_'; +} + +/* Replace name with getenv(name), return pointer to zero-terminating char + */ +static char *expand_macro(char *name, int maxlength) +{ + char *value; + size_t len; + + len = 0; + value = getenv(name); + if (value) { + len = strlen(value) + 1; + if (len > maxlength) + len = maxlength; + strlcpy(name, value, len); + --len; + } + return name + len; +} + +#define EXPBUFSIZE (1024 * 8) + +/* Replace all tokens prefixed by '$' in the specified text with the + * value of the named environment variable. + * NB: the return value is a static buffer, i.e. it must be strdup'd + * by the caller. + */ +char *expand_macros(const char *txt) +{ + static char result[EXPBUFSIZE]; + char *p, *start; + int len; + + p = result; + start = NULL; + while (p < result + EXPBUFSIZE - 1 && txt && *txt) { + *p = *txt; + if (start) { + if (!is_token_char(*txt)) { + if (p - start > 0) { + *p = '\0'; + len = result + EXPBUFSIZE - start - 1; + p = expand_macro(start, len) - 1; + } + start = NULL; + txt--; + } + p++; + txt++; + continue; + } + if (*txt == '$') { + start = p; + txt++; + continue; + } + p++; + txt++; + } + *p = '\0'; + if (start && p - start > 0) { + len = result + EXPBUFSIZE - start - 1; + p = expand_macro(start, len); + *p = '\0'; + } + return result; +} + +char *get_mimetype_for_filename(const char *filename) +{ + char *ext, *mimetype, *token, line[1024], *saveptr; + FILE *file; + struct string_list_item *mime; + + if (!filename) + return NULL; + + ext = strrchr(filename, '.'); + if (!ext) + return NULL; + ++ext; + if (!ext[0]) + return NULL; + mime = string_list_lookup(&ctx.cfg.mimetypes, ext); + if (mime) + return xstrdup(mime->util); + + if (!ctx.cfg.mimetype_file) + return NULL; + file = fopen(ctx.cfg.mimetype_file, "r"); + if (!file) + return NULL; + while (fgets(line, sizeof(line), file)) { + if (!line[0] || line[0] == '#') + continue; + mimetype = strtok_r(line, " \t\r\n", &saveptr); + while ((token = strtok_r(NULL, " \t\r\n", &saveptr))) { + if (!strcasecmp(ext, token)) { + fclose(file); + return xstrdup(mimetype); + } + } + } + fclose(file); + return NULL; +} diff --git a/third_party/cgit/tests/.gitignore b/third_party/cgit/tests/.gitignore new file mode 100644 index 000000000000..3fd2e965c8d2 --- /dev/null +++ b/third_party/cgit/tests/.gitignore @@ -0,0 +1,2 @@ +trash\ directory.t* +test-results diff --git a/third_party/cgit/tests/Makefile b/third_party/cgit/tests/Makefile new file mode 100644 index 000000000000..65e111733803 --- /dev/null +++ b/third_party/cgit/tests/Makefile @@ -0,0 +1,17 @@ +include ../git/config.mak.uname +-include ../cgit.conf + +SHELL_PATH ?= $(SHELL) +SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH)) + +T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh) + +all: $(T) + +$(T): + @'$(SHELL_PATH_SQ)' $@ $(CGIT_TEST_OPTS) + +clean: + $(RM) -rf trash + +.PHONY: $(T) clean diff --git a/third_party/cgit/tests/filters/dump.lua b/third_party/cgit/tests/filters/dump.lua new file mode 100644 index 000000000000..1f15c931051c --- /dev/null +++ b/third_party/cgit/tests/filters/dump.lua @@ -0,0 +1,17 @@ +function filter_open(...) + buffer = "" + for i = 1, select("#", ...) do + buffer = buffer .. select(i, ...) .. " " + end +end + +function filter_close() + html(buffer) + return 0 +end + +function filter_write(str) + buffer = buffer .. string.upper(str) +end + + diff --git a/third_party/cgit/tests/filters/dump.sh b/third_party/cgit/tests/filters/dump.sh new file mode 100755 index 000000000000..da6f7a1b181b --- /dev/null +++ b/third_party/cgit/tests/filters/dump.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +[ "$#" -gt 0 ] && printf "%s " "$*" +tr '[:lower:]' '[:upper:]' diff --git a/third_party/cgit/tests/setup.sh b/third_party/cgit/tests/setup.sh new file mode 100755 index 000000000000..8db810ff114f --- /dev/null +++ b/third_party/cgit/tests/setup.sh @@ -0,0 +1,180 @@ +# This file should be sourced by all test-scripts +# +# Main functions: +# prepare_tests(description) - setup for testing, i.e. create repos+config +# run_test(description, script) - run one test, i.e. eval script +# +# Helper functions +# cgit_query(querystring) - call cgit with the specified querystring +# cgit_url(url) - call cgit with the specified virtual url +# +# Example script: +# +# . setup.sh +# prepare_tests "html validation" +# run_test 'repo index' 'cgit_url "/" | tidy -e' +# run_test 'repo summary' 'cgit_url "/foo" | tidy -e' + +# We don't want to run Git commands through Valgrind, so we filter out the +# --valgrind option here and handle it ourselves. We copy the arguments +# assuming that none contain a newline, although other whitespace is +# preserved. +LF=' +' +test_argv= + +while test $# != 0 +do + case "$1" in + --va|--val|--valg|--valgr|--valgri|--valgrin|--valgrind) + cgit_valgrind=t + test_argv="$test_argv${LF}--verbose" + ;; + *) + test_argv="$test_argv$LF$1" + ;; + esac + shift +done + +OLDIFS=$IFS +IFS=$LF +set -- $test_argv +IFS=$OLDIFS + +: ${TEST_DIRECTORY=$(pwd)/../git/t} +: ${TEST_OUTPUT_DIRECTORY=$(pwd)} +TEST_NO_CREATE_REPO=YesPlease +. "$TEST_DIRECTORY"/test-lib.sh + +# Prepend the directory containing cgit to PATH. +if test -n "$cgit_valgrind" +then + GIT_VALGRIND="$TEST_DIRECTORY/valgrind" + CGIT_VALGRIND=$(cd ../valgrind && pwd) + PATH="$CGIT_VALGRIND/bin:$PATH" + export GIT_VALGRIND CGIT_VALGRIND +else + PATH="$(pwd)/../..:$PATH" +fi + +FILTER_DIRECTORY=$(cd ../filters && pwd) + +if cgit --version | grep -F -q "[+] Lua scripting"; then + export CGIT_HAS_LUA=1 +else + export CGIT_HAS_LUA=0 +fi + +mkrepo() { + name=$1 + count=$2 + test_create_repo "$name" + ( + cd "$name" + n=1 + while test $n -le $count + do + echo $n >file-$n + git add file-$n + git commit -m "commit $n" + n=$(expr $n + 1) + done + case "$3" in + testplus) + echo "hello" >a+b + git add a+b + git commit -m "add a+b" + git branch "1+2" + ;; + commit-graph) + git commit-graph write + ;; + esac + ) +} + +setup_repos() +{ + rm -rf cache + mkdir -p cache + mkrepo repos/foo 5 >/dev/null + mkrepo repos/bar 50 commit-graph >/dev/null + mkrepo repos/foo+bar 10 testplus >/dev/null + mkrepo "repos/with space" 2 >/dev/null + mkrepo repos/filter 5 testplus >/dev/null + cat >cgitrc <<EOF +virtual-root=/ +cache-root=$PWD/cache + +cache-size=1021 +snapshots=tar.gz tar.bz tar.lz tar.xz tar.zst zip +enable-log-filecount=1 +enable-log-linecount=1 +summary-log=5 +summary-branches=5 +summary-tags=5 +clone-url=git://example.org/\$CGIT_REPO_URL.git +enable-filter-overrides=1 + +repo.url=foo +repo.path=$PWD/repos/foo/.git +# Do not specify a description for this repo, as it then will be assigned +# the constant value "[no description]" (which actually used to cause a +# segfault). + +repo.url=bar +repo.path=$PWD/repos/bar/.git +repo.desc=the bar repo + +repo.url=foo+bar +repo.path=$PWD/repos/foo+bar/.git +repo.desc=the foo+bar repo + +repo.url=with space +repo.path=$PWD/repos/with space/.git +repo.desc=spaced repo + +repo.url=filter-exec +repo.path=$PWD/repos/filter/.git +repo.desc=filtered repo +repo.about-filter=exec:$FILTER_DIRECTORY/dump.sh +repo.commit-filter=exec:$FILTER_DIRECTORY/dump.sh +repo.email-filter=exec:$FILTER_DIRECTORY/dump.sh +repo.source-filter=exec:$FILTER_DIRECTORY/dump.sh +repo.readme=master:a+b +EOF + + if [ $CGIT_HAS_LUA -eq 1 ]; then + cat >>cgitrc <<EOF +repo.url=filter-lua +repo.path=$PWD/repos/filter/.git +repo.desc=filtered repo +repo.about-filter=lua:$FILTER_DIRECTORY/dump.lua +repo.commit-filter=lua:$FILTER_DIRECTORY/dump.lua +repo.email-filter=lua:$FILTER_DIRECTORY/dump.lua +repo.source-filter=lua:$FILTER_DIRECTORY/dump.lua +repo.readme=master:a+b +EOF + fi +} + +cgit_query() +{ + CGIT_CONFIG="$PWD/cgitrc" QUERY_STRING="$1" cgit +} + +cgit_url() +{ + CGIT_CONFIG="$PWD/cgitrc" QUERY_STRING="url=$1" cgit +} + +strip_headers() { + while read -r line + do + test -z "$line" && break + done + cat +} + +test -z "$CGIT_TEST_NO_CREATE_REPOS" && setup_repos diff --git a/third_party/cgit/tests/t0001-validate-git-versions.sh b/third_party/cgit/tests/t0001-validate-git-versions.sh new file mode 100755 index 000000000000..dd84fe3fcb94 --- /dev/null +++ b/third_party/cgit/tests/t0001-validate-git-versions.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +if [ "${CGIT_TEST_NO_GIT_VERSION}" = "YesPlease" ]; then + exit 0 +fi + +test_description='Check Git version is correct' +CGIT_TEST_NO_CREATE_REPOS=YesPlease +. ./setup.sh + +test_expect_success 'extract Git version from Makefile' ' + sed -n -e "/^GIT_VER[ ]*=/ { + s/^GIT_VER[ ]*=[ ]*// + p + }" ../../Makefile >makefile_version +' + +# Note that Git's GIT-VERSION-GEN script applies "s/-/./g" to the version +# string to produce the internal version in the GIT-VERSION-FILE, so we +# must apply the same transformation to the version in the Makefile before +# comparing them. +test_expect_success 'test Git version matches Makefile' ' + ( cat ../../git/GIT-VERSION-FILE || echo "No GIT-VERSION-FILE" ) | + sed -e "s/GIT_VERSION[ ]*=[ ]*//" -e "s/\\.dirty$//" >git_version && + sed -e "s/-/./g" makefile_version >makefile_git_version && + test_cmp git_version makefile_git_version +' + +test_expect_success 'test submodule version matches Makefile' ' + if ! test -e ../../git/.git + then + echo "git/ is not a Git repository" >&2 + else + ( + cd ../.. && + sm_oid=$(git ls-files --stage -- git | + sed -e "s/^[0-9]* \\([0-9a-f]*\\) [0-9] .*$/\\1/") && + cd git && + git describe --match "v[0-9]*" $sm_oid + ) | sed -e "s/^v//" -e "s/-/./" >sm_version && + test_cmp sm_version makefile_version + fi +' + +test_done diff --git a/third_party/cgit/tests/t0010-validate-html.sh b/third_party/cgit/tests/t0010-validate-html.sh new file mode 100755 index 000000000000..ca08d69d142b --- /dev/null +++ b/third_party/cgit/tests/t0010-validate-html.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +test_description='Validate html with tidy' +. ./setup.sh + + +test_url() +{ + tidy_opt="-eq" + test -z "$NO_TIDY_WARNINGS" || tidy_opt+=" --show-warnings no" + cgit_url "$1" >tidy-$test_count.tmp || return + sed -e "1,4d" tidy-$test_count.tmp >tidy-$test_count || return + "$tidy" $tidy_opt tidy-$test_count + rc=$? + + # tidy returns with exitcode 1 on warnings, 2 on error + if test $rc = 2 + then + false + else + : + fi +} + +tidy=`which tidy 2>/dev/null` +test -n "$tidy" || { + skip_all='Skipping html validation tests: tidy not found' + test_done + exit +} + +test_expect_success 'index page' 'test_url ""' +test_expect_success 'foo' 'test_url "foo"' +test_expect_success 'foo/log' 'test_url "foo/log"' +test_expect_success 'foo/tree' 'test_url "foo/tree"' +test_expect_success 'foo/tree/file-1' 'test_url "foo/tree/file-1"' +test_expect_success 'foo/commit' 'test_url "foo/commit"' +test_expect_success 'foo/diff' 'test_url "foo/diff"' + +test_done diff --git a/third_party/cgit/tests/t0020-validate-cache.sh b/third_party/cgit/tests/t0020-validate-cache.sh new file mode 100755 index 000000000000..657765d897b5 --- /dev/null +++ b/third_party/cgit/tests/t0020-validate-cache.sh @@ -0,0 +1,78 @@ +#!/bin/sh + +test_description='Validate cache' +. ./setup.sh + +test_expect_success 'verify cache-size=0' ' + + rm -f cache/* && + sed -e "s/cache-size=1021$/cache-size=0/" cgitrc >cgitrc.tmp && + mv -f cgitrc.tmp cgitrc && + cgit_url "" && + cgit_url "foo" && + cgit_url "foo/refs" && + cgit_url "foo/tree" && + cgit_url "foo/log" && + cgit_url "foo/diff" && + cgit_url "foo/patch" && + cgit_url "bar" && + cgit_url "bar/refs" && + cgit_url "bar/tree" && + cgit_url "bar/log" && + cgit_url "bar/diff" && + cgit_url "bar/patch" && + ls cache >output && + test_line_count = 0 output +' + +test_expect_success 'verify cache-size=1' ' + + rm -f cache/* && + sed -e "s/cache-size=0$/cache-size=1/" cgitrc >cgitrc.tmp && + mv -f cgitrc.tmp cgitrc && + cgit_url "" && + cgit_url "foo" && + cgit_url "foo/refs" && + cgit_url "foo/tree" && + cgit_url "foo/log" && + cgit_url "foo/diff" && + cgit_url "foo/patch" && + cgit_url "bar" && + cgit_url "bar/refs" && + cgit_url "bar/tree" && + cgit_url "bar/log" && + cgit_url "bar/diff" && + cgit_url "bar/patch" && + ls cache >output && + test_line_count = 1 output +' + +test_expect_success 'verify cache-size=1021' ' + + rm -f cache/* && + sed -e "s/cache-size=1$/cache-size=1021/" cgitrc >cgitrc.tmp && + mv -f cgitrc.tmp cgitrc && + cgit_url "" && + cgit_url "foo" && + cgit_url "foo/refs" && + cgit_url "foo/tree" && + cgit_url "foo/log" && + cgit_url "foo/diff" && + cgit_url "foo/patch" && + cgit_url "bar" && + cgit_url "bar/refs" && + cgit_url "bar/tree" && + cgit_url "bar/log" && + cgit_url "bar/diff" && + cgit_url "bar/patch" && + ls cache >output && + test_line_count = 13 output && + cgit_url "foo/ls_cache" >output.full && + strip_headers <output.full >output && + test_line_count = 13 output && + # Check that ls_cache output is cached correctly + cgit_url "foo/ls_cache" >output.second && + test_cmp output.full output.second +' + +test_done diff --git a/third_party/cgit/tests/t0101-index.sh b/third_party/cgit/tests/t0101-index.sh new file mode 100755 index 000000000000..82ef9b04e55e --- /dev/null +++ b/third_party/cgit/tests/t0101-index.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +test_description='Check content on index page' +. ./setup.sh + +test_expect_success 'generate index page' 'cgit_url "" >tmp' +test_expect_success 'find foo repo' 'grep "foo" tmp' +test_expect_success 'find foo description' 'grep "\[no description\]" tmp' +test_expect_success 'find bar repo' 'grep "bar" tmp' +test_expect_success 'find bar description' 'grep "the bar repo" tmp' +test_expect_success 'find foo+bar repo' 'grep ">foo+bar<" tmp' +test_expect_success 'verify foo+bar link' 'grep "/foo+bar/" tmp' +test_expect_success 'verify "with%20space" link' 'grep "/with%20space/" tmp' +test_expect_success 'no tree-link' '! grep "foo/tree" tmp' +test_expect_success 'no log-link' '! grep "foo/log" tmp' + +test_done diff --git a/third_party/cgit/tests/t0102-summary.sh b/third_party/cgit/tests/t0102-summary.sh new file mode 100755 index 000000000000..b8864cb18791 --- /dev/null +++ b/third_party/cgit/tests/t0102-summary.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +test_description='Check content on summary page' +. ./setup.sh + +test_expect_success 'generate foo summary' 'cgit_url "foo" >tmp' +test_expect_success 'find commit 1' 'grep "commit 1" tmp' +test_expect_success 'find commit 5' 'grep "commit 5" tmp' +test_expect_success 'find branch master' 'grep "master" tmp' +test_expect_success 'no tags' '! grep "tags" tmp' +test_expect_success 'clone-url expanded correctly' ' + grep "git://example.org/foo.git" tmp +' + +test_expect_success 'generate bar summary' 'cgit_url "bar" >tmp' +test_expect_success 'no commit 45' '! grep "commit 45" tmp' +test_expect_success 'find commit 46' 'grep "commit 46" tmp' +test_expect_success 'find commit 50' 'grep "commit 50" tmp' +test_expect_success 'find branch master' 'grep "master" tmp' +test_expect_success 'no tags' '! grep "tags" tmp' +test_expect_success 'clone-url expanded correctly' ' + grep "git://example.org/bar.git" tmp +' + +test_done diff --git a/third_party/cgit/tests/t0103-log.sh b/third_party/cgit/tests/t0103-log.sh new file mode 100755 index 000000000000..bdf1435a16fe --- /dev/null +++ b/third_party/cgit/tests/t0103-log.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +test_description='Check content on log page' +. ./setup.sh + +test_expect_success 'generate foo/log' 'cgit_url "foo/log" >tmp' +test_expect_success 'find commit 1' 'grep "commit 1" tmp' +test_expect_success 'find commit 5' 'grep "commit 5" tmp' + +test_expect_success 'generate bar/log' 'cgit_url "bar/log" >tmp' +test_expect_success 'find commit 1' 'grep "commit 1" tmp' +test_expect_success 'find commit 50' 'grep "commit 50" tmp' + +test_expect_success 'generate "with%20space/log?qt=grep&q=commit+1"' ' + cgit_url "with+space/log&qt=grep&q=commit+1" >tmp +' +test_expect_success 'find commit 1' 'grep "commit 1" tmp' +test_expect_success 'find link with %20 in path' 'grep "/with%20space/log/?qt=grep" tmp' +test_expect_success 'find link with + in arg' 'grep "/log/?qt=grep&q=commit+1" tmp' +test_expect_success 'no links with space in path' '! grep "href=./with space/" tmp' +test_expect_success 'no links with space in arg' '! grep "q=commit 1" tmp' +test_expect_success 'commit 2 is not visible' '! grep "commit 2" tmp' + +test_done diff --git a/third_party/cgit/tests/t0104-tree.sh b/third_party/cgit/tests/t0104-tree.sh new file mode 100755 index 000000000000..2e140f5939c1 --- /dev/null +++ b/third_party/cgit/tests/t0104-tree.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='Check content on tree page' +. ./setup.sh + +test_expect_success 'generate bar/tree' 'cgit_url "bar/tree" >tmp' +test_expect_success 'find file-1' 'grep "file-1" tmp' +test_expect_success 'find file-50' 'grep "file-50" tmp' + +test_expect_success 'generate bar/tree/file-50' 'cgit_url "bar/tree/file-50" >tmp' + +test_expect_success 'find line 1' ' + grep "<a id=.n1. href=.#n1.>1</a>" tmp +' + +test_expect_success 'no line 2' ' + ! grep "<a id=.n2. href=.#n2.>2</a>" tmp +' + +test_expect_success 'generate foo+bar/tree' 'cgit_url "foo%2bbar/tree" >tmp' + +test_expect_success 'verify a+b link' ' + grep "/foo+bar/tree/a+b" tmp +' + +test_expect_success 'generate foo+bar/tree?h=1+2' 'cgit_url "foo%2bbar/tree&h=1%2b2" >tmp' + +test_expect_success 'verify a+b?h=1+2 link' ' + grep "/foo+bar/tree/a+b?h=1%2b2" tmp +' + +test_done diff --git a/third_party/cgit/tests/t0105-commit.sh b/third_party/cgit/tests/t0105-commit.sh new file mode 100755 index 000000000000..1a12ee39a9e7 --- /dev/null +++ b/third_party/cgit/tests/t0105-commit.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +test_description='Check content on commit page' +. ./setup.sh + +test_expect_success 'generate foo/commit' 'cgit_url "foo/commit" >tmp' +test_expect_success 'find tree link' 'grep "<a href=./foo/tree/.>" tmp' +test_expect_success 'find parent link' 'grep -E "<a href=./foo/commit/\?id=.+>" tmp' + +test_expect_success 'find commit subject' ' + grep "<div class=.commit-subject.>commit 5<" tmp +' + +test_expect_success 'find commit msg' 'grep "<div class=.commit-msg.></div>" tmp' +test_expect_success 'find diffstat' 'grep "<table summary=.diffstat. class=.diffstat.>" tmp' + +test_expect_success 'find diff summary' ' + grep "1 files changed, 1 insertions, 0 deletions" tmp +' + +test_expect_success 'get root commit' ' + root=$(cd repos/foo && git rev-list --reverse HEAD | head -1) && + cgit_url "foo/commit&id=$root" >tmp && + grep "</html>" tmp +' + +test_expect_success 'root commit contains diffstat' ' + grep "<a href=./foo/diff/file-1.id=[0-9a-f]\{40,64\}.>file-1</a>" tmp +' + +test_expect_success 'root commit contains diff' ' + grep ">diff --git a/file-1 b/file-1<" tmp && + grep "<div class=.add.>+1</div>" tmp +' + +test_done diff --git a/third_party/cgit/tests/t0106-diff.sh b/third_party/cgit/tests/t0106-diff.sh new file mode 100755 index 000000000000..82b645ec72ab --- /dev/null +++ b/third_party/cgit/tests/t0106-diff.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +test_description='Check content on diff page' +. ./setup.sh + +test_expect_success 'generate foo/diff' 'cgit_url "foo/diff" >tmp' +test_expect_success 'find diff header' 'grep "a/file-5 b/file-5" tmp' +test_expect_success 'find blob link' 'grep "<a href=./foo/tree/file-5?id=" tmp' +test_expect_success 'find added file' 'grep "new file mode 100644" tmp' + +test_expect_success 'find hunk header' ' + grep "<div class=.hunk.>@@ -0,0 +1 @@</div>" tmp +' + +test_expect_success 'find added line' ' + grep "<div class=.add.>+5</div>" tmp +' + +test_done diff --git a/third_party/cgit/tests/t0107-snapshot.sh b/third_party/cgit/tests/t0107-snapshot.sh new file mode 100755 index 000000000000..0811ec407442 --- /dev/null +++ b/third_party/cgit/tests/t0107-snapshot.sh @@ -0,0 +1,205 @@ +#!/bin/sh + +test_description='Verify snapshot' +. ./setup.sh + +test_expect_success 'get foo/snapshot/master.tar.gz' ' + cgit_url "foo/snapshot/master.tar.gz" >tmp +' + +test_expect_success 'check html headers' ' + head -n 1 tmp | + grep "Content-Type: application/x-gzip" && + + head -n 2 tmp | + grep "Content-Disposition: inline; filename=.master.tar.gz." +' + +test_expect_success 'strip off the header lines' ' + strip_headers <tmp >master.tar.gz +' + +test_expect_success 'verify gzip format' ' + gunzip --test master.tar.gz +' + +test_expect_success 'untar' ' + rm -rf master && + gzip -dc master.tar.gz | tar -xf - +' + +test_expect_success 'count files' ' + ls master/ >output && + test_line_count = 5 output +' + +test_expect_success 'verify untarred file-5' ' + grep "^5$" master/file-5 && + test_line_count = 1 master/file-5 +' + +if test -n "$(which lzip 2>/dev/null)"; then + test_set_prereq LZIP +else + say 'Skipping LZIP validation tests: lzip not found' +fi + +test_expect_success LZIP 'get foo/snapshot/master.tar.lz' ' + cgit_url "foo/snapshot/master.tar.lz" >tmp +' + +test_expect_success LZIP 'check html headers' ' + head -n 1 tmp | + grep "Content-Type: application/x-lzip" && + + head -n 2 tmp | + grep "Content-Disposition: inline; filename=.master.tar.lz." +' + +test_expect_success LZIP 'strip off the header lines' ' + strip_headers <tmp >master.tar.lz +' + +test_expect_success LZIP 'verify lzip format' ' + lzip --test master.tar.lz +' + +test_expect_success LZIP 'untar' ' + rm -rf master && + lzip -dc master.tar.lz | tar -xf - +' + +test_expect_success LZIP 'count files' ' + ls master/ >output && + test_line_count = 5 output +' + +test_expect_success LZIP 'verify untarred file-5' ' + grep "^5$" master/file-5 && + test_line_count = 1 master/file-5 +' + +if test -n "$(which xz 2>/dev/null)"; then + test_set_prereq XZ +else + say 'Skipping XZ validation tests: xz not found' +fi + +test_expect_success XZ 'get foo/snapshot/master.tar.xz' ' + cgit_url "foo/snapshot/master.tar.xz" >tmp +' + +test_expect_success XZ 'check html headers' ' + head -n 1 tmp | + grep "Content-Type: application/x-xz" && + + head -n 2 tmp | + grep "Content-Disposition: inline; filename=.master.tar.xz." +' + +test_expect_success XZ 'strip off the header lines' ' + strip_headers <tmp >master.tar.xz +' + +test_expect_success XZ 'verify xz format' ' + xz --test master.tar.xz +' + +test_expect_success XZ 'untar' ' + rm -rf master && + xz -dc master.tar.xz | tar -xf - +' + +test_expect_success XZ 'count files' ' + ls master/ >output && + test_line_count = 5 output +' + +test_expect_success XZ 'verify untarred file-5' ' + grep "^5$" master/file-5 && + test_line_count = 1 master/file-5 +' + +if test -n "$(which zstd 2>/dev/null)"; then + test_set_prereq ZSTD +else + say 'Skipping ZSTD validation tests: zstd not found' +fi + +test_expect_success ZSTD 'get foo/snapshot/master.tar.zst' ' + cgit_url "foo/snapshot/master.tar.zst" >tmp +' + +test_expect_success ZSTD 'check html headers' ' + head -n 1 tmp | + grep "Content-Type: application/x-zstd" && + + head -n 2 tmp | + grep "Content-Disposition: inline; filename=.master.tar.zst." +' + +test_expect_success ZSTD 'strip off the header lines' ' + strip_headers <tmp >master.tar.zst +' + +test_expect_success ZSTD 'verify zstd format' ' + zstd --test master.tar.zst +' + +test_expect_success ZSTD 'untar' ' + rm -rf master && + zstd -dc master.tar.zst | tar -xf - +' + +test_expect_success ZSTD 'count files' ' + ls master/ >output && + test_line_count = 5 output +' + +test_expect_success ZSTD 'verify untarred file-5' ' + grep "^5$" master/file-5 && + test_line_count = 1 master/file-5 +' + +test_expect_success 'get foo/snapshot/master.zip' ' + cgit_url "foo/snapshot/master.zip" >tmp +' + +test_expect_success 'check HTML headers (zip)' ' + head -n 1 tmp | + grep "Content-Type: application/x-zip" && + + head -n 2 tmp | + grep "Content-Disposition: inline; filename=.master.zip." +' + +test_expect_success 'strip off the header lines (zip)' ' + strip_headers <tmp >master.zip +' + +if test -n "$(which unzip 2>/dev/null)"; then + test_set_prereq UNZIP +else + say 'Skipping ZIP validation tests: unzip not found' +fi + +test_expect_success UNZIP 'verify zip format' ' + unzip -t master.zip +' + +test_expect_success UNZIP 'unzip' ' + rm -rf master && + unzip master.zip +' + +test_expect_success UNZIP 'count files (zip)' ' + ls master/ >output && + test_line_count = 5 output +' + +test_expect_success UNZIP 'verify unzipped file-5' ' + grep "^5$" master/file-5 && + test_line_count = 1 master/file-5 +' + +test_done diff --git a/third_party/cgit/tests/t0108-patch.sh b/third_party/cgit/tests/t0108-patch.sh new file mode 100755 index 000000000000..013d68024d5d --- /dev/null +++ b/third_party/cgit/tests/t0108-patch.sh @@ -0,0 +1,62 @@ +#!/bin/sh + +test_description='Check content on patch page' +. ./setup.sh + +test_expect_success 'generate foo/patch' ' + cgit_query "url=foo/patch" >tmp +' + +test_expect_success 'find `From:` line' ' + grep "^From: " tmp +' + +test_expect_success 'find `Date:` line' ' + grep "^Date: " tmp +' + +test_expect_success 'find `Subject:` line' ' + grep "^Subject: commit 5" tmp +' + +test_expect_success 'find `cgit` signature' ' + tail -2 tmp | head -1 | grep "^cgit" +' + +test_expect_success 'compare with output of git-format-patch(1)' ' + CGIT_VERSION=$(sed -n "s/CGIT_VERSION = //p" ../../VERSION) && + git --git-dir="$PWD/repos/foo/.git" format-patch --subject-prefix="" --signature="cgit $CGIT_VERSION" --stdout HEAD^ >tmp2 && + strip_headers <tmp >tmp_ && + test_cmp tmp_ tmp2 +' + +test_expect_success 'find initial commit' ' + root=$(git --git-dir="$PWD/repos/foo/.git" rev-list --max-parents=0 HEAD) +' + +test_expect_success 'generate patch for initial commit' ' + cgit_query "url=foo/patch&id=$root" >tmp +' + +test_expect_success 'find `cgit` signature' ' + tail -2 tmp | head -1 | grep "^cgit" +' + +test_expect_success 'generate patches for multiple commits' ' + id=$(git --git-dir="$PWD/repos/foo/.git" rev-parse HEAD) && + id2=$(git --git-dir="$PWD/repos/foo/.git" rev-parse HEAD~3) && + cgit_query "url=foo/patch&id=$id&id2=$id2" >tmp +' + +test_expect_success 'find `cgit` signature' ' + tail -2 tmp | head -1 | grep "^cgit" +' + +test_expect_success 'compare with output of git-format-patch(1)' ' + CGIT_VERSION=$(sed -n "s/CGIT_VERSION = //p" ../../VERSION) && + git --git-dir="$PWD/repos/foo/.git" format-patch -N --subject-prefix="" --signature="cgit $CGIT_VERSION" --stdout HEAD~3..HEAD >tmp2 && + strip_headers <tmp >tmp_ && + test_cmp tmp_ tmp2 +' + +test_done diff --git a/third_party/cgit/tests/t0109-gitconfig.sh b/third_party/cgit/tests/t0109-gitconfig.sh new file mode 100755 index 000000000000..189ef2816620 --- /dev/null +++ b/third_party/cgit/tests/t0109-gitconfig.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +test_description='Ensure that git does not access $HOME' +. ./setup.sh + +test -n "$(which strace 2>/dev/null)" || { + skip_all='Skipping access validation tests: strace not found' + test_done + exit +} + +strace true 2>/dev/null || { + skip_all='Skipping access validation tests: strace not functional' + test_done + exit +} + +test_no_home_access () { + non_existent_path="/path/to/some/place/that/does/not/possibly/exist" + while test -d "$non_existent_path"; do + non_existent_path="$non_existent_path/$(date +%N)" + done && + strace \ + -E HOME="$non_existent_path" \ + -E CGIT_CONFIG="$PWD/cgitrc" \ + -E QUERY_STRING="url=$1" \ + -e access -f -o strace.out cgit && + ! grep "$non_existent_path" strace.out +} + +test_no_home_access_success() { + test_expect_success "do not access \$HOME: $1" " + test_no_home_access '$1' + " +} + +test_no_home_access_success +test_no_home_access_success foo +test_no_home_access_success foo/refs +test_no_home_access_success foo/log +test_no_home_access_success foo/tree +test_no_home_access_success foo/tree/file-1 +test_no_home_access_success foo/commit +test_no_home_access_success foo/diff +test_no_home_access_success foo/patch +test_no_home_access_success foo/snapshot/master.tar.gz + +test_done diff --git a/third_party/cgit/tests/t0110-rawdiff.sh b/third_party/cgit/tests/t0110-rawdiff.sh new file mode 100755 index 000000000000..66fa7d5d37e8 --- /dev/null +++ b/third_party/cgit/tests/t0110-rawdiff.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +test_description='Check content on rawdiff page' +. ./setup.sh + +test_expect_success 'generate foo/rawdiff' ' + cgit_query "url=foo/rawdiff" >tmp +' + +test_expect_success 'compare with output of git-diff(1)' ' + git --git-dir="$PWD/repos/foo/.git" diff HEAD^.. >tmp2 && + sed "1,4d" tmp >tmp_ && + cmp tmp_ tmp2 +' + +test_expect_success 'find initial commit' ' + root=$(git --git-dir="$PWD/repos/foo/.git" rev-list --max-parents=0 HEAD) +' + +test_expect_success 'generate diff for initial commit' ' + cgit_query "url=foo/rawdiff&id=$root" >tmp +' + +test_expect_success 'compare with output of git-diff-tree(1)' ' + git --git-dir="$PWD/repos/foo/.git" diff-tree -p --no-commit-id --root "$root" >tmp2 && + sed "1,4d" tmp >tmp_ && + cmp tmp_ tmp2 +' + +test_expect_success 'generate diff for multiple commits' ' + id=$(git --git-dir="$PWD/repos/foo/.git" rev-parse HEAD) && + id2=$(git --git-dir="$PWD/repos/foo/.git" rev-parse HEAD~3) && + cgit_query "url=foo/rawdiff&id=$id&id2=$id2" >tmp +' + +test_expect_success 'compare with output of git-diff(1)' ' + git --git-dir="$PWD/repos/foo/.git" diff HEAD~3..HEAD >tmp2 && + sed "1,4d" tmp >tmp_ && + cmp tmp_ tmp2 +' + +test_done diff --git a/third_party/cgit/tests/t0111-filter.sh b/third_party/cgit/tests/t0111-filter.sh new file mode 100755 index 000000000000..2fdc3669f492 --- /dev/null +++ b/third_party/cgit/tests/t0111-filter.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +test_description='Check filtered content' +. ./setup.sh + +prefixes="exec" +if [ $CGIT_HAS_LUA -eq 1 ]; then + prefixes="$prefixes lua" +fi + +for prefix in $prefixes +do + test_expect_success "generate filter-$prefix/tree/a%2bb" " + cgit_url 'filter-$prefix/tree/a%2bb' >tmp + " + + test_expect_success "check whether the $prefix source filter works" ' + grep "<code>a+b HELLO$" tmp + ' + + test_expect_success "generate filter-$prefix/about/" " + cgit_url 'filter-$prefix/about/' >tmp + " + + test_expect_success "check whether the $prefix about filter works" ' + grep "<div id='"'"'summary'"'"'>a+b HELLO$" tmp + ' + + test_expect_success "generate filter-$prefix/commit/" " + cgit_url 'filter-$prefix/commit/' >tmp + " + + test_expect_success "check whether the $prefix commit filter works" ' + grep "<div class='"'"'commit-subject'"'"'>ADD A+B" tmp + ' + + test_expect_success "check whether the $prefix email filter works for authors" ' + grep "<author@example.com> commit A U THOR <AUTHOR@EXAMPLE.COM>" tmp + ' + + test_expect_success "check whether the $prefix email filter works for committers" ' + grep "<committer@example.com> commit C O MITTER <COMMITTER@EXAMPLE.COM>" tmp + ' +done + +test_done diff --git a/third_party/cgit/tests/valgrind/bin/cgit b/third_party/cgit/tests/valgrind/bin/cgit new file mode 100755 index 000000000000..dcdfbe532085 --- /dev/null +++ b/third_party/cgit/tests/valgrind/bin/cgit @@ -0,0 +1,12 @@ +#!/bin/sh + +# Note that we currently use Git's suppression file and there are variables +# $GIT_VALGRIND and $CGIT_VALGRIND which point to different places. +exec valgrind -q --error-exitcode=126 \ + --suppressions="$GIT_VALGRIND/default.supp" \ + --gen-suppressions=all \ + --leak-check=no \ + --track-origins=yes \ + --log-fd=4 \ + --input-fd=4 \ + "$CGIT_VALGRIND/../../cgit" "$@" diff --git a/third_party/cgit/tvl-extra.css b/third_party/cgit/tvl-extra.css new file mode 100644 index 000000000000..41f5041d62aa --- /dev/null +++ b/third_party/cgit/tvl-extra.css @@ -0,0 +1,35 @@ +/* limit the width of /about/** to help readability */ +.content #summary { + max-width: 800px; +} + +/* highlight cheddar callouts in cgit about views */ +.cheddar-callout { + display: block; + padding: 10px; +} + +.cheddar-question { + color: #3367d6; + background-color: #e8f0fe; +} + +.cheddar-todo { + color: #616161; + background-color: #eeeeee; +} + +.cheddar-tip { + color: #00796b; + background-color: #e0f2f1; +} + +.cheddar-warning { + color: #a52714; + background-color: #fbe9e7; +} + +/* add some padding next to the logo */ +td.logo { + padding-right: 10px; +} diff --git a/third_party/cgit/ui-atom.c b/third_party/cgit/ui-atom.c new file mode 100644 index 000000000000..1056f3639752 --- /dev/null +++ b/third_party/cgit/ui-atom.c @@ -0,0 +1,149 @@ +/* ui-atom.c: functions for atom feeds + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-atom.h" +#include "html.h" +#include "ui-shared.h" + +static void add_entry(struct commit *commit, const char *host) +{ + char delim = '&'; + char *hex; + char *mail, *t, *t2; + struct commitinfo *info; + + info = cgit_parse_commit(commit); + hex = oid_to_hex(&commit->object.oid); + html("<entry>\n"); + html("<title>"); + html_txt(info->subject); + html("</title>\n"); + html("<updated>"); + html_txt(show_date(info->committer_date, 0, + date_mode_from_type(DATE_ISO8601_STRICT))); + html("</updated>\n"); + html("<author>\n"); + if (info->author) { + html("<name>"); + html_txt(info->author); + html("</name>\n"); + } + if (info->author_email && !ctx.cfg.noplainemail) { + mail = xstrdup(info->author_email); + t = strchr(mail, '<'); + if (t) + t++; + else + t = mail; + t2 = strchr(t, '>'); + if (t2) + *t2 = '\0'; + html("<email>"); + html_txt(t); + html("</email>\n"); + free(mail); + } + html("</author>\n"); + html("<published>"); + html_txt(show_date(info->author_date, 0, + date_mode_from_type(DATE_ISO8601_STRICT))); + html("</published>\n"); + if (host) { + char *pageurl; + html("<link rel='alternate' type='text/html' href='"); + html(cgit_httpscheme()); + html_attr(host); + pageurl = cgit_pageurl(ctx.repo->url, "commit", NULL); + html_attr(pageurl); + if (ctx.cfg.virtual_root) + delim = '?'; + html_attrf("%cid=%s", delim, hex); + html("'/>\n"); + free(pageurl); + } + htmlf("<id>%s</id>\n", hex); + html("<content type='text'>\n"); + html_txt(info->msg); + html("</content>\n"); + html("<content type='xhtml'>\n"); + html("<div xmlns='http://www.w3.org/1999/xhtml'>\n"); + html("<pre>\n"); + html_txt(info->msg); + html("</pre>\n"); + html("</div>\n"); + html("</content>\n"); + html("</entry>\n"); + cgit_free_commitinfo(info); +} + + +void cgit_print_atom(char *tip, const char *path, int max_count) +{ + char *host; + const char *argv[] = {NULL, tip, NULL, NULL, NULL}; + struct commit *commit; + struct rev_info rev; + int argc = 2; + + if (ctx.qry.show_all) + argv[1] = "--all"; + else if (!tip) + argv[1] = ctx.qry.head; + + if (path) { + argv[argc++] = "--"; + argv[argc++] = path; + } + + init_revisions(&rev, NULL); + rev.abbrev = DEFAULT_ABBREV; + rev.commit_format = CMIT_FMT_DEFAULT; + rev.verbose_header = 1; + rev.show_root_diff = 0; + rev.max_count = max_count; + setup_revisions(argc, argv, &rev, NULL); + prepare_revision_walk(&rev); + + host = cgit_hosturl(); + ctx.page.mimetype = "text/xml"; + ctx.page.charset = "utf-8"; + cgit_print_http_headers(); + html("<feed xmlns='http://www.w3.org/2005/Atom'>\n"); + html("<title>"); + html_txt(ctx.repo->name); + if (path) { + html("/"); + html_txt(path); + } + if (tip && !ctx.qry.show_all) { + html(", branch "); + html_txt(tip); + } + html("</title>\n"); + html("<subtitle>"); + html_txt(ctx.repo->desc); + html("</subtitle>\n"); + if (host) { + char *repourl = cgit_repourl(ctx.repo->url); + html("<link rel='alternate' type='text/html' href='"); + html(cgit_httpscheme()); + html_attr(host); + html_attr(repourl); + html("'/>\n"); + free(repourl); + } + while ((commit = get_revision(&rev)) != NULL) { + add_entry(commit, host); + free_commit_buffer(the_repository->parsed_objects, commit); + free_commit_list(commit->parents); + commit->parents = NULL; + } + html("</feed>\n"); + free(host); +} diff --git a/third_party/cgit/ui-atom.h b/third_party/cgit/ui-atom.h new file mode 100644 index 000000000000..dda953bbf4ff --- /dev/null +++ b/third_party/cgit/ui-atom.h @@ -0,0 +1,6 @@ +#ifndef UI_ATOM_H +#define UI_ATOM_H + +extern void cgit_print_atom(char *tip, const char *path, int max_count); + +#endif diff --git a/third_party/cgit/ui-blame.c b/third_party/cgit/ui-blame.c new file mode 100644 index 000000000000..786a7105d5ac --- /dev/null +++ b/third_party/cgit/ui-blame.c @@ -0,0 +1,306 @@ +/* ui-blame.c: functions for blame output + * + * Copyright (C) 2006-2017 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-blame.h" +#include "html.h" +#include "ui-shared.h" +#include "strvec.h" +#include "blame.h" + + +static char *emit_suspect_detail(struct blame_origin *suspect) +{ + struct commitinfo *info; + struct strbuf detail = STRBUF_INIT; + + info = cgit_parse_commit(suspect->commit); + + strbuf_addf(&detail, "author %s", info->author); + if (!ctx.cfg.noplainemail) + strbuf_addf(&detail, " %s", info->author_email); + strbuf_addf(&detail, " %s\n", + show_date(info->author_date, info->author_tz, + cgit_date_mode(DATE_DOTTIME))); + + strbuf_addf(&detail, "committer %s", info->committer); + if (!ctx.cfg.noplainemail) + strbuf_addf(&detail, " %s", info->committer_email); + strbuf_addf(&detail, " %s\n\n", + show_date(info->committer_date, info->committer_tz, + cgit_date_mode(DATE_DOTTIME))); + + strbuf_addstr(&detail, info->subject); + + cgit_free_commitinfo(info); + return strbuf_detach(&detail, NULL); +} + +static void emit_blame_entry_hash(struct blame_entry *ent) +{ + struct blame_origin *suspect = ent->suspect; + struct object_id *oid = &suspect->commit->object.oid; + unsigned long line = 0; + + char *detail = emit_suspect_detail(suspect); + html("<span class='oid'>"); + cgit_commit_link(find_unique_abbrev(oid, DEFAULT_ABBREV), detail, + NULL, ctx.qry.head, oid_to_hex(oid), suspect->path); + html("</span>"); + free(detail); + + while (line++ < ent->num_lines) + html("\n"); +} + +static void emit_blame_entry_linenumber(struct blame_entry *ent) +{ + const char *numberfmt = "<a id='n%1$d' href='#n%1$d'>%1$d</a>\n"; + + unsigned long lineno = ent->lno; + while (lineno < ent->lno + ent->num_lines) + htmlf(numberfmt, ++lineno); +} + +static void emit_blame_entry_line_background(struct blame_scoreboard *sb, + struct blame_entry *ent) +{ + unsigned long line; + size_t len, maxlen = 2; + const char* pos, *endpos; + + for (line = ent->lno; line < ent->lno + ent->num_lines; line++) { + html("\n"); + pos = blame_nth_line(sb, line); + endpos = blame_nth_line(sb, line + 1); + len = 0; + while (pos < endpos) { + len++; + if (*pos++ == '\t') + len = (len + 7) & ~7; + } + if (len > maxlen) + maxlen = len; + } + + for (len = 0; len < maxlen - 1; len++) + html(" "); +} + +struct walk_tree_context { + char *curr_rev; + int match_baselen; + int state; +}; + +static void print_object(const struct object_id *oid, const char *path, + const char *basename, const char *rev) +{ + enum object_type type; + char *buf; + unsigned long size; + struct strvec rev_argv = STRVEC_INIT; + struct rev_info revs; + struct blame_scoreboard sb; + struct blame_origin *o; + struct blame_entry *ent = NULL; + + type = oid_object_info(the_repository, oid, &size); + if (type == OBJ_BAD) { + cgit_print_error_page(404, "Not found", "Bad object name: %s", + oid_to_hex(oid)); + return; + } + + buf = read_object_file(oid, &type, &size); + if (!buf) { + cgit_print_error_page(500, "Internal server error", + "Error reading object %s", oid_to_hex(oid)); + return; + } + + strvec_push(&rev_argv, "blame"); + strvec_push(&rev_argv, rev); + init_revisions(&revs, NULL); + revs.diffopt.flags.allow_textconv = 1; + setup_revisions(rev_argv.nr, rev_argv.v, &revs, NULL); + init_scoreboard(&sb); + sb.revs = &revs; + sb.repo = the_repository; + sb.path = path; + setup_scoreboard(&sb, &o); + o->suspects = blame_entry_prepend(NULL, 0, sb.num_lines, o); + prio_queue_put(&sb.commits, o->commit); + blame_origin_decref(o); + sb.ent = NULL; + sb.path = path; + assign_blame(&sb, 0); + blame_sort_final(&sb); + blame_coalesce(&sb); + + cgit_set_title_from_path(path); + + cgit_print_layout_start(); + htmlf("blob: %s (", oid_to_hex(oid)); + cgit_plain_link("plain", NULL, NULL, ctx.qry.head, rev, path); + html(") ("); + cgit_tree_link("tree", NULL, NULL, ctx.qry.head, rev, path); + html(")\n"); + + if (buffer_is_binary(buf, size)) { + html("<div class='error'>blob is binary.</div>"); + goto cleanup; + } + if (ctx.cfg.max_blob_size && size / 1024 > ctx.cfg.max_blob_size) { + htmlf("<div class='error'>blob size (%ldKB)" + " exceeds display size limit (%dKB).</div>", + size / 1024, ctx.cfg.max_blob_size); + goto cleanup; + } + + html("<table class='blame blob'>\n<tr>\n"); + + /* Commit hashes */ + html("<td class='hashes'>"); + for (ent = sb.ent; ent; ent = ent->next) { + html("<div class='alt'><pre>"); + emit_blame_entry_hash(ent); + html("</pre></div>"); + } + html("</td>\n"); + + /* Line numbers */ + if (ctx.cfg.enable_tree_linenumbers) { + html("<td class='linenumbers'>"); + for (ent = sb.ent; ent; ent = ent->next) { + html("<div class='alt'><pre>"); + emit_blame_entry_linenumber(ent); + html("</pre></div>"); + } + html("</td>\n"); + } + + html("<td class='lines'><div>"); + + /* Colored bars behind lines */ + html("<div>"); + for (ent = sb.ent; ent; ) { + struct blame_entry *e = ent->next; + html("<div class='alt'><pre>"); + emit_blame_entry_line_background(&sb, ent); + html("</pre></div>"); + free(ent); + ent = e; + } + html("</div>"); + + free((void *)sb.final_buf); + + /* Lines */ + html("<pre><code>"); + if (ctx.repo->source_filter) { + char *filter_arg = xstrdup(basename); + cgit_open_filter(ctx.repo->source_filter, filter_arg); + html_raw(buf, size); + cgit_close_filter(ctx.repo->source_filter); + free(filter_arg); + } else { + html_txt(buf); + } + html("</code></pre>"); + + html("</div></td>\n"); + + html("</tr>\n</table>\n"); + + cgit_print_layout_end(); + +cleanup: + free(buf); +} + +static int walk_tree(const struct object_id *oid, struct strbuf *base, + const char *pathname, unsigned mode, void *cbdata) +{ + struct walk_tree_context *walk_tree_ctx = cbdata; + + if (base->len == walk_tree_ctx->match_baselen) { + if (S_ISREG(mode)) { + struct strbuf buffer = STRBUF_INIT; + strbuf_addbuf(&buffer, base); + strbuf_addstr(&buffer, pathname); + print_object(oid, buffer.buf, pathname, + walk_tree_ctx->curr_rev); + strbuf_release(&buffer); + walk_tree_ctx->state = 1; + } else if (S_ISDIR(mode)) { + walk_tree_ctx->state = 2; + } + } else if (base->len < INT_MAX + && (int)base->len > walk_tree_ctx->match_baselen) { + walk_tree_ctx->state = 2; + } else if (S_ISDIR(mode)) { + return READ_TREE_RECURSIVE; + } + return 0; +} + +static int basedir_len(const char *path) +{ + char *p = strrchr(path, '/'); + if (p) + return p - path + 1; + return 0; +} + +void cgit_print_blame(void) +{ + const char *rev = ctx.qry.oid; + struct object_id oid; + struct commit *commit; + struct pathspec_item path_items = { + .match = ctx.qry.path, + .len = ctx.qry.path ? strlen(ctx.qry.path) : 0 + }; + struct pathspec paths = { + .nr = 1, + .items = &path_items + }; + struct walk_tree_context walk_tree_ctx = { + .state = 0 + }; + + if (!rev) + rev = ctx.qry.head; + + if (get_oid(rev, &oid)) { + cgit_print_error_page(404, "Not found", + "Invalid revision name: %s", rev); + return; + } + commit = lookup_commit_reference(the_repository, &oid); + if (!commit || parse_commit(commit)) { + cgit_print_error_page(404, "Not found", + "Invalid commit reference: %s", rev); + return; + } + + walk_tree_ctx.curr_rev = xstrdup(rev); + walk_tree_ctx.match_baselen = (path_items.match) ? + basedir_len(path_items.match) : -1; + + read_tree(the_repository, repo_get_commit_tree(the_repository, commit), + &paths, walk_tree, &walk_tree_ctx); + if (!walk_tree_ctx.state) + cgit_print_error_page(404, "Not found", "Not found"); + else if (walk_tree_ctx.state == 2) + cgit_print_error_page(404, "No blame for folders", + "Blame is not available for folders."); + + free(walk_tree_ctx.curr_rev); +} diff --git a/third_party/cgit/ui-blame.h b/third_party/cgit/ui-blame.h new file mode 100644 index 000000000000..5b97e035914e --- /dev/null +++ b/third_party/cgit/ui-blame.h @@ -0,0 +1,6 @@ +#ifndef UI_BLAME_H +#define UI_BLAME_H + +extern void cgit_print_blame(void); + +#endif /* UI_BLAME_H */ diff --git a/third_party/cgit/ui-blob.c b/third_party/cgit/ui-blob.c new file mode 100644 index 000000000000..c10ae42ebd05 --- /dev/null +++ b/third_party/cgit/ui-blob.c @@ -0,0 +1,182 @@ +/* ui-blob.c: show blob content + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-blob.h" +#include "html.h" +#include "ui-shared.h" + +struct walk_tree_context { + const char *match_path; + struct object_id *matched_oid; + unsigned int found_path:1; + unsigned int file_only:1; +}; + +static int walk_tree(const struct object_id *oid, struct strbuf *base, + const char *pathname, unsigned mode, void *cbdata) +{ + struct walk_tree_context *walk_tree_ctx = cbdata; + + if (walk_tree_ctx->file_only && !S_ISREG(mode)) + return READ_TREE_RECURSIVE; + if (strncmp(base->buf, walk_tree_ctx->match_path, base->len) + || strcmp(walk_tree_ctx->match_path + base->len, pathname)) + return READ_TREE_RECURSIVE; + oidcpy(walk_tree_ctx->matched_oid, oid); + walk_tree_ctx->found_path = 1; + return 0; +} + +int cgit_ref_path_exists(const char *path, const char *ref, int file_only) +{ + struct object_id oid; + unsigned long size; + struct pathspec_item path_items = { + .match = xstrdup(path), + .len = strlen(path) + }; + struct pathspec paths = { + .nr = 1, + .items = &path_items + }; + struct walk_tree_context walk_tree_ctx = { + .match_path = path, + .matched_oid = &oid, + .found_path = 0, + .file_only = file_only + }; + + if (get_oid(ref, &oid)) + goto done; + if (oid_object_info(the_repository, &oid, &size) != OBJ_COMMIT) + goto done; + read_tree(the_repository, + repo_get_commit_tree(the_repository, lookup_commit_reference(the_repository, &oid)), + &paths, walk_tree, &walk_tree_ctx); + +done: + free(path_items.match); + return walk_tree_ctx.found_path; +} + +int cgit_print_file(char *path, const char *head, int file_only) +{ + struct object_id oid; + enum object_type type; + char *buf; + unsigned long size; + struct commit *commit; + struct pathspec_item path_items = { + .match = path, + .len = strlen(path) + }; + struct pathspec paths = { + .nr = 1, + .items = &path_items + }; + struct walk_tree_context walk_tree_ctx = { + .match_path = path, + .matched_oid = &oid, + .found_path = 0, + .file_only = file_only + }; + + if (get_oid(head, &oid)) + return -1; + type = oid_object_info(the_repository, &oid, &size); + if (type == OBJ_COMMIT) { + commit = lookup_commit_reference(the_repository, &oid); + read_tree(the_repository, repo_get_commit_tree(the_repository, commit), + &paths, walk_tree, &walk_tree_ctx); + if (!walk_tree_ctx.found_path) + return -1; + type = oid_object_info(the_repository, &oid, &size); + } + if (type == OBJ_BAD) + return -1; + buf = read_object_file(&oid, &type, &size); + if (!buf) + return -1; + buf[size] = '\0'; + html_raw(buf, size); + free(buf); + return 0; +} + +void cgit_print_blob(const char *hex, char *path, const char *head, int file_only) +{ + struct object_id oid; + enum object_type type; + char *buf; + unsigned long size; + struct commit *commit; + struct pathspec_item path_items = { + .match = path, + .len = path ? strlen(path) : 0 + }; + struct pathspec paths = { + .nr = 1, + .items = &path_items + }; + struct walk_tree_context walk_tree_ctx = { + .match_path = path, + .matched_oid = &oid, + .found_path = 0, + .file_only = file_only + }; + + if (hex) { + if (get_oid_hex(hex, &oid)) { + cgit_print_error_page(400, "Bad request", + "Bad hex value: %s", hex); + return; + } + } else { + if (get_oid(head, &oid)) { + cgit_print_error_page(404, "Not found", + "Bad ref: %s", head); + return; + } + } + + type = oid_object_info(the_repository, &oid, &size); + + if ((!hex) && type == OBJ_COMMIT && path) { + commit = lookup_commit_reference(the_repository, &oid); + read_tree(the_repository, repo_get_commit_tree(the_repository, commit), + &paths, walk_tree, &walk_tree_ctx); + type = oid_object_info(the_repository, &oid, &size); + } + + if (type == OBJ_BAD) { + cgit_print_error_page(404, "Not found", + "Bad object name: %s", hex); + return; + } + + buf = read_object_file(&oid, &type, &size); + if (!buf) { + cgit_print_error_page(500, "Internal server error", + "Error reading object %s", hex); + return; + } + + buf[size] = '\0'; + if (buffer_is_binary(buf, size)) + ctx.page.mimetype = "application/octet-stream"; + else + ctx.page.mimetype = "text/plain"; + ctx.page.filename = path; + + html("X-Content-Type-Options: nosniff\n"); + html("Content-Security-Policy: default-src 'none'\n"); + cgit_print_http_headers(); + html_raw(buf, size); + free(buf); +} diff --git a/third_party/cgit/ui-blob.h b/third_party/cgit/ui-blob.h new file mode 100644 index 000000000000..16847b20b17a --- /dev/null +++ b/third_party/cgit/ui-blob.h @@ -0,0 +1,8 @@ +#ifndef UI_BLOB_H +#define UI_BLOB_H + +extern int cgit_ref_path_exists(const char *path, const char *ref, int file_only); +extern int cgit_print_file(char *path, const char *head, int file_only); +extern void cgit_print_blob(const char *hex, char *path, const char *head, int file_only); + +#endif /* UI_BLOB_H */ diff --git a/third_party/cgit/ui-clone.c b/third_party/cgit/ui-clone.c new file mode 100644 index 000000000000..5dccb639761a --- /dev/null +++ b/third_party/cgit/ui-clone.c @@ -0,0 +1,126 @@ +/* ui-clone.c: functions for http cloning, based on + * git's http-backend.c by Shawn O. Pearce + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-clone.h" +#include "html.h" +#include "ui-shared.h" +#include "packfile.h" +#include "object-store.h" + +static int print_ref_info(const char *refname, const struct object_id *oid, + int flags, void *cb_data) +{ + struct object *obj; + + if (!(obj = parse_object(the_repository, oid))) + return 0; + + htmlf("%s\t%s\n", oid_to_hex(oid), refname); + if (obj->type == OBJ_TAG) { + if (!(obj = deref_tag(the_repository, obj, refname, 0))) + return 0; + htmlf("%s\t%s^{}\n", oid_to_hex(&obj->oid), refname); + } + return 0; +} + +static void print_pack_info(void) +{ + struct packed_git *pack; + char *offset; + + ctx.page.mimetype = "text/plain"; + ctx.page.filename = "objects/info/packs"; + cgit_print_http_headers(); + reprepare_packed_git(the_repository); + for (pack = get_packed_git(the_repository); pack; pack = pack->next) { + if (pack->pack_local) { + offset = strrchr(pack->pack_name, '/'); + if (offset && offset[1] != '\0') + ++offset; + else + offset = pack->pack_name; + htmlf("P %s\n", offset); + } + } +} + +static void send_file(const char *path) +{ + struct stat st; + + if (stat(path, &st)) { + switch (errno) { + case ENOENT: + cgit_print_error_page(404, "Not found", "Not found"); + break; + case EACCES: + cgit_print_error_page(403, "Forbidden", "Forbidden"); + break; + default: + cgit_print_error_page(400, "Bad request", "Bad request"); + } + return; + } + ctx.page.mimetype = "application/octet-stream"; + ctx.page.filename = path; + skip_prefix(path, ctx.repo->path, &ctx.page.filename); + skip_prefix(ctx.page.filename, "/", &ctx.page.filename); + cgit_print_http_headers(); + html_include(path); +} + +void cgit_clone_info(void) +{ + if (!ctx.qry.path || strcmp(ctx.qry.path, "refs")) { + cgit_print_error_page(400, "Bad request", "Bad request"); + return; + } + + ctx.page.mimetype = "text/plain"; + ctx.page.filename = "info/refs"; + cgit_print_http_headers(); + for_each_ref(print_ref_info, NULL); +} + +void cgit_clone_objects(void) +{ + char *p; + + if (!ctx.qry.path) + goto err; + + if (!strcmp(ctx.qry.path, "info/packs")) { + print_pack_info(); + return; + } + + /* Avoid directory traversal by forbidding "..", but also work around + * other funny business by just specifying a fairly strict format. For + * example, now we don't have to stress out about the Cygwin port. + */ + for (p = ctx.qry.path; *p; ++p) { + if (*p == '.' && *(p + 1) == '.') + goto err; + if (!isalnum(*p) && *p != '/' && *p != '.' && *p != '-') + goto err; + } + + send_file(git_path("objects/%s", ctx.qry.path)); + return; + +err: + cgit_print_error_page(400, "Bad request", "Bad request"); +} + +void cgit_clone_head(void) +{ + send_file(git_path("%s", "HEAD")); +} diff --git a/third_party/cgit/ui-clone.h b/third_party/cgit/ui-clone.h new file mode 100644 index 000000000000..3e460a3dbcf2 --- /dev/null +++ b/third_party/cgit/ui-clone.h @@ -0,0 +1,8 @@ +#ifndef UI_CLONE_H +#define UI_CLONE_H + +void cgit_clone_info(void); +void cgit_clone_objects(void); +void cgit_clone_head(void); + +#endif /* UI_CLONE_H */ diff --git a/third_party/cgit/ui-commit.c b/third_party/cgit/ui-commit.c new file mode 100644 index 000000000000..96eecebe1931 --- /dev/null +++ b/third_party/cgit/ui-commit.c @@ -0,0 +1,148 @@ +/* ui-commit.c: generate commit view + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-commit.h" +#include "html.h" +#include "ui-shared.h" +#include "ui-diff.h" +#include "ui-log.h" + +void cgit_print_commit(char *hex, const char *prefix) +{ + struct commit *commit, *parent; + struct commitinfo *info, *parent_info; + struct commit_list *p; + struct strbuf notes = STRBUF_INIT; + struct object_id oid; + char *tmp, *tmp2; + int parents = 0; + + if (!hex) + hex = ctx.qry.head; + + if (get_oid(hex, &oid)) { + cgit_print_error_page(400, "Bad request", + "Bad object id: %s", hex); + return; + } + commit = lookup_commit_reference(the_repository, &oid); + if (!commit) { + cgit_print_error_page(404, "Not found", + "Bad commit reference: %s", hex); + return; + } + info = cgit_parse_commit(commit); + + format_display_notes(&oid, ¬es, PAGE_ENCODING, 0); + + load_ref_decorations(NULL, DECORATE_FULL_REFS); + + ctx.page.title = fmtalloc("%s - %s", info->subject, ctx.page.title); + cgit_print_layout_start(); + cgit_print_diff_ctrls(); + html("<table summary='commit info' class='commit-info'>\n"); + html("<tr><th>author</th><td>"); + cgit_open_filter(ctx.repo->email_filter, info->author_email, "commit"); + html_txt(info->author); + if (!ctx.cfg.noplainemail) { + html(" "); + html_txt(info->author_email); + } + cgit_close_filter(ctx.repo->email_filter); + html("</td><td class='right'>"); + html_txt(show_date(info->author_date, info->author_tz, + cgit_date_mode(DATE_DOTTIME))); + html("</td></tr>\n"); + html("<tr><th>committer</th><td>"); + cgit_open_filter(ctx.repo->email_filter, info->committer_email, "commit"); + html_txt(info->committer); + if (!ctx.cfg.noplainemail) { + html(" "); + html_txt(info->committer_email); + } + cgit_close_filter(ctx.repo->email_filter); + html("</td><td class='right'>"); + html_txt(show_date(info->committer_date, info->committer_tz, + cgit_date_mode(DATE_DOTTIME))); + html("</td></tr>\n"); + html("<tr><th>commit</th><td colspan='2' class='oid'>"); + tmp = oid_to_hex(&commit->object.oid); + cgit_commit_link(tmp, NULL, NULL, ctx.qry.head, tmp, prefix); + html(" ("); + cgit_patch_link("patch", NULL, NULL, NULL, tmp, prefix); + html(")</td></tr>\n"); + html("<tr><th>tree</th><td colspan='2' class='oid'>"); + tmp = xstrdup(hex); + cgit_tree_link(oid_to_hex(get_commit_tree_oid(commit)), NULL, NULL, + ctx.qry.head, tmp, NULL); + if (prefix) { + html(" /"); + cgit_tree_link(prefix, NULL, NULL, ctx.qry.head, tmp, prefix); + } + free(tmp); + html("</td></tr>\n"); + for (p = commit->parents; p; p = p->next) { + parent = lookup_commit_reference(the_repository, &p->item->object.oid); + if (!parent) { + html("<tr><td colspan='3'>"); + cgit_print_error("Error reading parent commit"); + html("</td></tr>"); + continue; + } + html("<tr><th>parent</th>" + "<td colspan='2' class='oid'>"); + tmp = tmp2 = oid_to_hex(&p->item->object.oid); + if (ctx.repo->enable_subject_links) { + parent_info = cgit_parse_commit(parent); + tmp2 = parent_info->subject; + } + cgit_commit_link(tmp2, NULL, NULL, ctx.qry.head, tmp, prefix); + html(" ("); + cgit_diff_link("diff", NULL, NULL, ctx.qry.head, hex, + oid_to_hex(&p->item->object.oid), prefix); + html(")</td></tr>"); + parents++; + } + if (ctx.repo->snapshots) { + html("<tr><th>download</th><td colspan='2' class='oid'>"); + cgit_print_snapshot_links(ctx.repo, hex, "<br/>"); + html("</td></tr>"); + } + html("</table>\n"); + html("<div class='commit-subject'>"); + cgit_open_filter(ctx.repo->commit_filter); + html_txt(info->subject); + cgit_close_filter(ctx.repo->commit_filter); + show_commit_decorations(commit); + html("</div>"); + html("<div class='commit-msg'>"); + cgit_open_filter(ctx.repo->commit_filter); + html_txt(info->msg); + cgit_close_filter(ctx.repo->commit_filter); + html("</div>"); + if (notes.len != 0) { + html("<div class='notes-header'>Notes</div>"); + html("<div class='notes'>"); + cgit_open_filter(ctx.repo->commit_filter); + html_txt(notes.buf); + cgit_close_filter(ctx.repo->commit_filter); + html("</div>"); + html("<div class='notes-footer'></div>"); + } + if (parents < 3) { + if (parents) + tmp = oid_to_hex(&commit->parents->item->object.oid); + else + tmp = NULL; + cgit_print_diff(ctx.qry.oid, tmp, prefix, 0, 0); + } + strbuf_release(¬es); + cgit_free_commitinfo(info); + cgit_print_layout_end(); +} diff --git a/third_party/cgit/ui-commit.h b/third_party/cgit/ui-commit.h new file mode 100644 index 000000000000..8198b4bacc36 --- /dev/null +++ b/third_party/cgit/ui-commit.h @@ -0,0 +1,6 @@ +#ifndef UI_COMMIT_H +#define UI_COMMIT_H + +extern void cgit_print_commit(char *hex, const char *prefix); + +#endif /* UI_COMMIT_H */ diff --git a/third_party/cgit/ui-diff.c b/third_party/cgit/ui-diff.c new file mode 100644 index 000000000000..5ed5990c2915 --- /dev/null +++ b/third_party/cgit/ui-diff.c @@ -0,0 +1,501 @@ +/* ui-diff.c: show diff between two blobs + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-diff.h" +#include "html.h" +#include "ui-shared.h" +#include "ui-ssdiff.h" + +struct object_id old_rev_oid[1]; +struct object_id new_rev_oid[1]; + +static int files, slots; +static int total_adds, total_rems, max_changes; +static int lines_added, lines_removed; + +static struct fileinfo { + char status; + struct object_id old_oid[1]; + struct object_id new_oid[1]; + unsigned short old_mode; + unsigned short new_mode; + char *old_path; + char *new_path; + unsigned int added; + unsigned int removed; + unsigned long old_size; + unsigned long new_size; + unsigned int binary:1; +} *items; + +static int use_ssdiff = 0; +static struct diff_filepair *current_filepair; +static const char *current_prefix; + +struct diff_filespec *cgit_get_current_old_file(void) +{ + return current_filepair->one; +} + +struct diff_filespec *cgit_get_current_new_file(void) +{ + return current_filepair->two; +} + +static void print_fileinfo(struct fileinfo *info) +{ + char *class; + + switch (info->status) { + case DIFF_STATUS_ADDED: + class = "add"; + break; + case DIFF_STATUS_COPIED: + class = "cpy"; + break; + case DIFF_STATUS_DELETED: + class = "del"; + break; + case DIFF_STATUS_MODIFIED: + class = "upd"; + break; + case DIFF_STATUS_RENAMED: + class = "mov"; + break; + case DIFF_STATUS_TYPE_CHANGED: + class = "typ"; + break; + case DIFF_STATUS_UNKNOWN: + class = "unk"; + break; + case DIFF_STATUS_UNMERGED: + class = "stg"; + break; + default: + die("bug: unhandled diff status %c", info->status); + } + + html("<tr>"); + html("<td class='mode'>"); + if (is_null_oid(info->new_oid)) { + cgit_print_filemode(info->old_mode); + } else { + cgit_print_filemode(info->new_mode); + } + + if (info->old_mode != info->new_mode && + !is_null_oid(info->old_oid) && + !is_null_oid(info->new_oid)) { + html("<span class='modechange'>["); + cgit_print_filemode(info->old_mode); + html("]</span>"); + } + htmlf("</td><td class='%s'>", class); + cgit_diff_link(info->new_path, NULL, NULL, ctx.qry.head, ctx.qry.oid, + ctx.qry.oid2, info->new_path); + if (info->status == DIFF_STATUS_COPIED || info->status == DIFF_STATUS_RENAMED) { + htmlf(" (%s from ", + info->status == DIFF_STATUS_COPIED ? "copied" : "renamed"); + html_txt(info->old_path); + html(")"); + } + html("</td><td class='right'>"); + if (info->binary) { + htmlf("bin</td><td class='graph'>%ld -> %ld bytes", + info->old_size, info->new_size); + return; + } + htmlf("%d", info->added + info->removed); + html("</td><td class='graph'>"); + htmlf("<table summary='file diffstat' width='%d%%'><tr>", (max_changes > 100 ? 100 : max_changes)); + htmlf("<td class='add' style='width: %.1f%%;'/>", + info->added * 100.0 / max_changes); + htmlf("<td class='rem' style='width: %.1f%%;'/>", + info->removed * 100.0 / max_changes); + htmlf("<td class='none' style='width: %.1f%%;'/>", + (max_changes - info->removed - info->added) * 100.0 / max_changes); + html("</tr></table></td></tr>\n"); +} + +static void count_diff_lines(char *line, int len) +{ + if (line && (len > 0)) { + if (line[0] == '+') + lines_added++; + else if (line[0] == '-') + lines_removed++; + } +} + +static int show_filepair(struct diff_filepair *pair) +{ + /* Always show if we have no limiting prefix. */ + if (!current_prefix) + return 1; + + /* Show if either path in the pair begins with the prefix. */ + if (starts_with(pair->one->path, current_prefix) || + starts_with(pair->two->path, current_prefix)) + return 1; + + /* Otherwise we don't want to show this filepair. */ + return 0; +} + +static void inspect_filepair(struct diff_filepair *pair) +{ + int binary = 0; + unsigned long old_size = 0; + unsigned long new_size = 0; + + if (!show_filepair(pair)) + return; + + files++; + lines_added = 0; + lines_removed = 0; + cgit_diff_files(&pair->one->oid, &pair->two->oid, &old_size, &new_size, + &binary, 0, ctx.qry.ignorews, count_diff_lines); + if (files >= slots) { + if (slots == 0) + slots = 4; + else + slots = slots * 2; + items = xrealloc(items, slots * sizeof(struct fileinfo)); + } + items[files-1].status = pair->status; + oidcpy(items[files-1].old_oid, &pair->one->oid); + oidcpy(items[files-1].new_oid, &pair->two->oid); + items[files-1].old_mode = pair->one->mode; + items[files-1].new_mode = pair->two->mode; + items[files-1].old_path = xstrdup(pair->one->path); + items[files-1].new_path = xstrdup(pair->two->path); + items[files-1].added = lines_added; + items[files-1].removed = lines_removed; + items[files-1].old_size = old_size; + items[files-1].new_size = new_size; + items[files-1].binary = binary; + if (lines_added + lines_removed > max_changes) + max_changes = lines_added + lines_removed; + total_adds += lines_added; + total_rems += lines_removed; +} + +static void cgit_print_diffstat(const struct object_id *old_oid, + const struct object_id *new_oid, + const char *prefix) +{ + int i; + + html("<div class='diffstat-header'>"); + cgit_diff_link("Diffstat", NULL, NULL, ctx.qry.head, ctx.qry.oid, + ctx.qry.oid2, NULL); + if (prefix) { + html(" (limited to '"); + html_txt(prefix); + html("')"); + } + html("</div>"); + html("<table summary='diffstat' class='diffstat'>"); + max_changes = 0; + cgit_diff_tree(old_oid, new_oid, inspect_filepair, prefix, + ctx.qry.ignorews); + for (i = 0; i<files; i++) + print_fileinfo(&items[i]); + html("</table>"); + html("<div class='diffstat-summary'>"); + htmlf("%d files changed, %d insertions, %d deletions", + files, total_adds, total_rems); + html("</div>"); +} + + +/* + * print a single line returned from xdiff + */ +static void print_line(char *line, int len) +{ + char *class = "ctx"; + char c = line[len-1]; + + if (line[0] == '+') + class = "add"; + else if (line[0] == '-') + class = "del"; + else if (line[0] == '@') + class = "hunk"; + + htmlf("<div class='%s'>", class); + line[len-1] = '\0'; + html_txt(line); + html("</div>"); + line[len-1] = c; +} + +static void header(const struct object_id *oid1, char *path1, int mode1, + const struct object_id *oid2, char *path2, int mode2) +{ + char *abbrev1, *abbrev2; + int subproject; + + subproject = (S_ISGITLINK(mode1) || S_ISGITLINK(mode2)); + html("<div class='head'>"); + html("diff --git a/"); + html_txt(path1); + html(" b/"); + html_txt(path2); + + if (mode1 == 0) + htmlf("<br/>new file mode %.6o", mode2); + + if (mode2 == 0) + htmlf("<br/>deleted file mode %.6o", mode1); + + if (!subproject) { + abbrev1 = xstrdup(find_unique_abbrev(oid1, DEFAULT_ABBREV)); + abbrev2 = xstrdup(find_unique_abbrev(oid2, DEFAULT_ABBREV)); + htmlf("<br/>index %s..%s", abbrev1, abbrev2); + free(abbrev1); + free(abbrev2); + if (mode1 != 0 && mode2 != 0) { + htmlf(" %.6o", mode1); + if (mode2 != mode1) + htmlf("..%.6o", mode2); + } + if (is_null_oid(oid1)) { + path1 = "dev/null"; + html("<br/>--- /"); + } else + html("<br/>--- a/"); + if (mode1 != 0) + cgit_tree_link(path1, NULL, NULL, ctx.qry.head, + oid_to_hex(old_rev_oid), path1); + else + html_txt(path1); + if (is_null_oid(oid2)) { + path2 = "dev/null"; + html("<br/>+++ /"); + } else + html("<br/>+++ b/"); + if (mode2 != 0) + cgit_tree_link(path2, NULL, NULL, ctx.qry.head, + oid_to_hex(new_rev_oid), path2); + else + html_txt(path2); + } + html("</div>"); +} + +static void filepair_cb(struct diff_filepair *pair) +{ + unsigned long old_size = 0; + unsigned long new_size = 0; + int binary = 0; + linediff_fn print_line_fn = print_line; + + if (!show_filepair(pair)) + return; + + current_filepair = pair; + if (use_ssdiff) { + cgit_ssdiff_header_begin(); + print_line_fn = cgit_ssdiff_line_cb; + } + header(&pair->one->oid, pair->one->path, pair->one->mode, + &pair->two->oid, pair->two->path, pair->two->mode); + if (use_ssdiff) + cgit_ssdiff_header_end(); + if (S_ISGITLINK(pair->one->mode) || S_ISGITLINK(pair->two->mode)) { + if (S_ISGITLINK(pair->one->mode)) + print_line_fn(fmt("-Subproject %s", oid_to_hex(&pair->one->oid)), 52); + if (S_ISGITLINK(pair->two->mode)) + print_line_fn(fmt("+Subproject %s", oid_to_hex(&pair->two->oid)), 52); + if (use_ssdiff) + cgit_ssdiff_footer(); + return; + } + if (cgit_diff_files(&pair->one->oid, &pair->two->oid, &old_size, + &new_size, &binary, ctx.qry.context, + ctx.qry.ignorews, print_line_fn)) + cgit_print_error("Error running diff"); + if (binary) { + if (use_ssdiff) + html("<tr><td colspan='4'>Binary files differ</td></tr>"); + else + html("Binary files differ"); + } + if (use_ssdiff) + cgit_ssdiff_footer(); +} + +void cgit_print_diff_ctrls(void) +{ + int i, curr; + + html("<div class='cgit-panel'>"); + html("<b>diff options</b>"); + html("<form method='get'>"); + cgit_add_hidden_formfields(1, 0, ctx.qry.page); + html("<table>"); + html("<tr><td colspan='2'/></tr>"); + html("<tr>"); + html("<td class='label'>context:</td>"); + html("<td class='ctrl'>"); + html("<select name='context' onchange='this.form.submit();'>"); + curr = ctx.qry.context; + if (!curr) + curr = 3; + for (i = 1; i <= 10; i++) + html_intoption(i, fmt("%d", i), curr); + for (i = 15; i <= 40; i += 5) + html_intoption(i, fmt("%d", i), curr); + html("</select>"); + html("</td>"); + html("</tr><tr>"); + html("<td class='label'>space:</td>"); + html("<td class='ctrl'>"); + html("<select name='ignorews' onchange='this.form.submit();'>"); + html_intoption(0, "include", ctx.qry.ignorews); + html_intoption(1, "ignore", ctx.qry.ignorews); + html("</select>"); + html("</td>"); + html("</tr><tr>"); + html("<td class='label'>mode:</td>"); + html("<td class='ctrl'>"); + html("<select name='dt' onchange='this.form.submit();'>"); + curr = ctx.qry.has_difftype ? ctx.qry.difftype : ctx.cfg.difftype; + html_intoption(0, "unified", curr); + html_intoption(1, "ssdiff", curr); + html_intoption(2, "stat only", curr); + html("</select></td></tr>"); + html("<tr><td/><td class='ctrl'>"); + html("<noscript><input type='submit' value='reload'/></noscript>"); + html("</td></tr></table>"); + html("</form>"); + html("</div>"); +} + +void cgit_print_diff(const char *new_rev, const char *old_rev, + const char *prefix, int show_ctrls, int raw) +{ + struct commit *commit, *commit2; + const struct object_id *old_tree_oid, *new_tree_oid; + diff_type difftype; + + /* + * If "follow" is set then the diff machinery needs to examine the + * entire commit to detect renames so we must limit the paths in our + * own callbacks and not pass the prefix to the diff machinery. + */ + if (ctx.qry.follow && ctx.cfg.enable_follow_links) { + current_prefix = prefix; + prefix = ""; + } else { + current_prefix = NULL; + } + + if (!new_rev) + new_rev = ctx.qry.head; + if (get_oid(new_rev, new_rev_oid)) { + cgit_print_error_page(404, "Not found", + "Bad object name: %s", new_rev); + return; + } + commit = lookup_commit_reference(the_repository, new_rev_oid); + if (!commit || parse_commit(commit)) { + cgit_print_error_page(404, "Not found", + "Bad commit: %s", oid_to_hex(new_rev_oid)); + return; + } + new_tree_oid = get_commit_tree_oid(commit); + + if (old_rev) { + if (get_oid(old_rev, old_rev_oid)) { + cgit_print_error_page(404, "Not found", + "Bad object name: %s", old_rev); + return; + } + } else if (commit->parents && commit->parents->item) { + oidcpy(old_rev_oid, &commit->parents->item->object.oid); + } else { + oidclr(old_rev_oid); + } + + if (!is_null_oid(old_rev_oid)) { + commit2 = lookup_commit_reference(the_repository, old_rev_oid); + if (!commit2 || parse_commit(commit2)) { + cgit_print_error_page(404, "Not found", + "Bad commit: %s", oid_to_hex(old_rev_oid)); + return; + } + old_tree_oid = get_commit_tree_oid(commit2); + } else { + old_tree_oid = NULL; + } + + if (raw) { + struct diff_options diffopt; + + diff_setup(&diffopt); + diffopt.output_format = DIFF_FORMAT_PATCH; + diffopt.flags.recursive = 1; + diff_setup_done(&diffopt); + + ctx.page.mimetype = "text/plain"; + cgit_print_http_headers(); + if (old_tree_oid) { + diff_tree_oid(old_tree_oid, new_tree_oid, "", + &diffopt); + } else { + diff_root_tree_oid(new_tree_oid, "", &diffopt); + } + diffcore_std(&diffopt); + diff_flush(&diffopt); + + return; + } + + difftype = ctx.qry.has_difftype ? ctx.qry.difftype : ctx.cfg.difftype; + use_ssdiff = difftype == DIFF_SSDIFF; + + if (show_ctrls) { + cgit_print_layout_start(); + cgit_print_diff_ctrls(); + } + + /* + * Clicking on a link to a file in the diff stat should show a diff + * of the file, showing the diff stat limited to a single file is + * pretty useless. All links from this point on will be to + * individual files, so we simply reset the difftype in the query + * here to avoid propagating DIFF_STATONLY to the individual files. + */ + if (difftype == DIFF_STATONLY) + ctx.qry.difftype = ctx.cfg.difftype; + + cgit_print_diffstat(old_rev_oid, new_rev_oid, prefix); + + if (difftype == DIFF_STATONLY) + return; + + if (use_ssdiff) { + html("<table summary='ssdiff' class='ssdiff'>"); + } else { + html("<table summary='diff' class='diff'>"); + html("<tr><td>"); + } + cgit_diff_tree(old_rev_oid, new_rev_oid, filepair_cb, prefix, + ctx.qry.ignorews); + if (!use_ssdiff) + html("</td></tr>"); + html("</table>"); + + if (show_ctrls) + cgit_print_layout_end(); +} diff --git a/third_party/cgit/ui-diff.h b/third_party/cgit/ui-diff.h new file mode 100644 index 000000000000..39264a164fbc --- /dev/null +++ b/third_party/cgit/ui-diff.h @@ -0,0 +1,15 @@ +#ifndef UI_DIFF_H +#define UI_DIFF_H + +extern void cgit_print_diff_ctrls(void); + +extern void cgit_print_diff(const char *new_hex, const char *old_hex, + const char *prefix, int show_ctrls, int raw); + +extern struct diff_filespec *cgit_get_current_old_file(void); +extern struct diff_filespec *cgit_get_current_new_file(void); + +extern struct object_id old_rev_oid[1]; +extern struct object_id new_rev_oid[1]; + +#endif /* UI_DIFF_H */ diff --git a/third_party/cgit/ui-log.c b/third_party/cgit/ui-log.c new file mode 100644 index 000000000000..20774bf82fa5 --- /dev/null +++ b/third_party/cgit/ui-log.c @@ -0,0 +1,551 @@ +/* ui-log.c: functions for log output + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-log.h" +#include "html.h" +#include "ui-shared.h" +#include "strvec.h" + +static int files, add_lines, rem_lines, lines_counted; + +/* + * The list of available column colors in the commit graph. + */ +static const char *column_colors_html[] = { + "<span class='column1'>", + "<span class='column2'>", + "<span class='column3'>", + "<span class='column4'>", + "<span class='column5'>", + "<span class='column6'>", + "</span>", +}; + +#define COLUMN_COLORS_HTML_MAX (ARRAY_SIZE(column_colors_html) - 1) + +static void count_lines(char *line, int size) +{ + if (size <= 0) + return; + + if (line[0] == '+') + add_lines++; + + else if (line[0] == '-') + rem_lines++; +} + +static void inspect_files(struct diff_filepair *pair) +{ + unsigned long old_size = 0; + unsigned long new_size = 0; + int binary = 0; + + files++; + if (ctx.repo->enable_log_linecount) + cgit_diff_files(&pair->one->oid, &pair->two->oid, &old_size, + &new_size, &binary, 0, ctx.qry.ignorews, + count_lines); +} + +void show_commit_decorations(struct commit *commit) +{ + const struct name_decoration *deco; + static char buf[1024]; + + buf[sizeof(buf) - 1] = 0; + deco = get_name_decoration(&commit->object); + if (!deco) + return; + html("<span class='decoration'>"); + while (deco) { + struct object_id oid_tag, peeled; + int is_annotated = 0; + + strlcpy(buf, prettify_refname(deco->name), sizeof(buf)); + switch(deco->type) { + case DECORATION_NONE: + /* If the git-core doesn't recognize it, + * don't display anything. */ + break; + case DECORATION_REF_LOCAL: + cgit_log_link(buf, NULL, "branch-deco", buf, NULL, + ctx.qry.vpath, 0, NULL, NULL, + ctx.qry.showmsg, 0); + break; + case DECORATION_REF_TAG: + if (!read_ref(deco->name, &oid_tag) && !peel_iterated_oid(&oid_tag, &peeled)) + is_annotated = !oideq(&oid_tag, &peeled); + cgit_tag_link(buf, NULL, is_annotated ? "tag-annotated-deco" : "tag-deco", buf); + break; + case DECORATION_REF_REMOTE: + if (!ctx.repo->enable_remote_branches) + break; + cgit_log_link(buf, NULL, "remote-deco", NULL, + oid_to_hex(&commit->object.oid), + ctx.qry.vpath, 0, NULL, NULL, + ctx.qry.showmsg, 0); + break; + default: + cgit_commit_link(buf, NULL, "deco", ctx.qry.head, + oid_to_hex(&commit->object.oid), + ctx.qry.vpath); + break; + } + deco = deco->next; + } + html("</span>"); +} + +static void handle_rename(struct diff_filepair *pair) +{ + /* + * After we have seen a rename, we generate links to the previous + * name of the file so that commit & diff views get fed the path + * that is correct for the commit they are showing, avoiding the + * need to walk the entire history leading back to every commit we + * show in order detect renames. + */ + if (0 != strcmp(ctx.qry.vpath, pair->two->path)) { + free(ctx.qry.vpath); + ctx.qry.vpath = xstrdup(pair->two->path); + } + inspect_files(pair); +} + +static int show_commit(struct commit *commit, struct rev_info *revs) +{ + struct commit_list *parents = commit->parents; + struct commit *parent; + int found = 0, saved_fmt; + struct diff_flags saved_flags = revs->diffopt.flags; + + /* Always show if we're not in "follow" mode with a single file. */ + if (!ctx.qry.follow) + return 1; + + /* + * In "follow" mode, we don't show merges. This is consistent with + * "git log --follow -- <file>". + */ + if (parents && parents->next) + return 0; + + /* + * If this is the root commit, do what rev_info tells us. + */ + if (!parents) + return revs->show_root_diff; + + /* When we get here we have precisely one parent. */ + parent = parents->item; + /* If we can't parse the commit, let print_commit() report an error. */ + if (parse_commit(parent)) + return 1; + + files = 0; + add_lines = 0; + rem_lines = 0; + + revs->diffopt.flags.recursive = 1; + diff_tree_oid(get_commit_tree_oid(parent), + get_commit_tree_oid(commit), + "", &revs->diffopt); + diffcore_std(&revs->diffopt); + + found = !diff_queue_is_empty(); + saved_fmt = revs->diffopt.output_format; + revs->diffopt.output_format = DIFF_FORMAT_CALLBACK; + revs->diffopt.format_callback = cgit_diff_tree_cb; + revs->diffopt.format_callback_data = handle_rename; + diff_flush(&revs->diffopt); + revs->diffopt.output_format = saved_fmt; + revs->diffopt.flags = saved_flags; + + lines_counted = 1; + return found; +} + +static void print_commit(struct commit *commit, struct rev_info *revs) +{ + struct commitinfo *info; + int columns = revs->graph ? 4 : 3; + struct strbuf graphbuf = STRBUF_INIT; + struct strbuf msgbuf = STRBUF_INIT; + + if (ctx.repo->enable_log_filecount) + columns++; + if (ctx.repo->enable_log_linecount) + columns++; + + if (revs->graph) { + /* Advance graph until current commit */ + while (!graph_next_line(revs->graph, &graphbuf)) { + /* Print graph segment in otherwise empty table row */ + html("<tr class='nohover'><td class='commitgraph'>"); + html(graphbuf.buf); + htmlf("</td><td colspan='%d' /></tr>\n", columns); + strbuf_setlen(&graphbuf, 0); + } + /* Current commit's graph segment is now ready in graphbuf */ + } + + info = cgit_parse_commit(commit); + htmlf("<tr%s>", ctx.qry.showmsg ? " class='logheader'" : ""); + + if (revs->graph) { + /* Print graph segment for current commit */ + html("<td class='commitgraph'>"); + html(graphbuf.buf); + html("</td>"); + strbuf_setlen(&graphbuf, 0); + } + else { + html("<td>"); + cgit_print_age(info->committer_date, info->committer_tz, TM_WEEK * 2); + html("</td>"); + } + + htmlf("<td%s>", ctx.qry.showmsg ? " class='logsubject'" : ""); + if (ctx.qry.showmsg) { + /* line-wrap long commit subjects instead of truncating them */ + size_t subject_len = strlen(info->subject); + + if (subject_len > ctx.cfg.max_msg_len && + ctx.cfg.max_msg_len >= 15) { + /* symbol for signaling line-wrap (in PAGE_ENCODING) */ + const char wrap_symbol[] = { ' ', 0xE2, 0x86, 0xB5, 0 }; + int i = ctx.cfg.max_msg_len - strlen(wrap_symbol); + + /* Rewind i to preceding space character */ + while (i > 0 && !isspace(info->subject[i])) + --i; + if (!i) /* Oops, zero spaces. Reset i */ + i = ctx.cfg.max_msg_len - strlen(wrap_symbol); + + /* add remainder starting at i to msgbuf */ + strbuf_add(&msgbuf, info->subject + i, subject_len - i); + strbuf_trim(&msgbuf); + strbuf_add(&msgbuf, "\n\n", 2); + + /* Place wrap_symbol at position i in info->subject */ + strlcpy(info->subject + i, wrap_symbol, subject_len - i + 1); + } + } + cgit_commit_link(info->subject, NULL, NULL, ctx.qry.head, + oid_to_hex(&commit->object.oid), ctx.qry.vpath); + show_commit_decorations(commit); + html("</td><td>"); + cgit_open_filter(ctx.repo->email_filter, info->author_email, "log"); + html_txt(info->author); + cgit_close_filter(ctx.repo->email_filter); + + if (revs->graph) { + html("</td><td>"); + cgit_print_age(info->committer_date, info->committer_tz, TM_WEEK * 2); + } + + if (!lines_counted && (ctx.repo->enable_log_filecount || + ctx.repo->enable_log_linecount)) { + files = 0; + add_lines = 0; + rem_lines = 0; + cgit_diff_commit(commit, inspect_files, ctx.qry.vpath); + } + + if (ctx.repo->enable_log_filecount) + htmlf("</td><td>%d", files); + if (ctx.repo->enable_log_linecount) + htmlf("</td><td><span class='deletions'>-%d</span>/" + "<span class='insertions'>+%d</span>", rem_lines, add_lines); + + html("</td></tr>\n"); + + if ((revs->graph && !graph_is_commit_finished(revs->graph)) + || ctx.qry.showmsg) { /* Print a second table row */ + html("<tr class='nohover-highlight'>"); + + if (ctx.qry.showmsg) { + /* Concatenate commit message + notes in msgbuf */ + if (info->msg && *(info->msg)) { + strbuf_addstr(&msgbuf, info->msg); + strbuf_addch(&msgbuf, '\n'); + } + format_display_notes(&commit->object.oid, + &msgbuf, PAGE_ENCODING, 0); + strbuf_addch(&msgbuf, '\n'); + strbuf_ltrim(&msgbuf); + } + + if (revs->graph) { + int lines = 0; + + /* Calculate graph padding */ + if (ctx.qry.showmsg) { + /* Count #lines in commit message + notes */ + const char *p = msgbuf.buf; + lines = 1; + while ((p = strchr(p, '\n'))) { + p++; + lines++; + } + } + + /* Print graph padding */ + html("<td class='commitgraph'>"); + while (lines > 0 || !graph_is_commit_finished(revs->graph)) { + if (graphbuf.len) + html("\n"); + strbuf_setlen(&graphbuf, 0); + graph_next_line(revs->graph, &graphbuf); + html(graphbuf.buf); + lines--; + } + html("</td>\n"); + } + else + html("<td/>"); /* Empty 'Age' column */ + + /* Print msgbuf into remainder of table row */ + htmlf("<td colspan='%d'%s>\n", columns - (revs->graph ? 1 : 0), + ctx.qry.showmsg ? " class='logmsg'" : ""); + html_txt(msgbuf.buf); + html("</td></tr>\n"); + } + + strbuf_release(&msgbuf); + strbuf_release(&graphbuf); + cgit_free_commitinfo(info); +} + +static const char *disambiguate_ref(const char *ref, int *must_free_result) +{ + struct object_id oid; + struct strbuf longref = STRBUF_INIT; + + strbuf_addf(&longref, "refs/heads/%s", ref); + if (get_oid(longref.buf, &oid) == 0) { + *must_free_result = 1; + return strbuf_detach(&longref, NULL); + } + + *must_free_result = 0; + strbuf_release(&longref); + return ref; +} + +static char *next_token(char **src) +{ + char *result; + + if (!src || !*src) + return NULL; + while (isspace(**src)) + (*src)++; + if (!**src) + return NULL; + result = *src; + while (**src) { + if (isspace(**src)) { + **src = '\0'; + (*src)++; + break; + } + (*src)++; + } + return result; +} + +void cgit_print_log(const char *tip, int ofs, int cnt, char *grep, char *pattern, + const char *path, int pager, int commit_graph, int commit_sort) +{ + struct rev_info rev; + struct commit *commit; + struct strvec rev_argv = STRVEC_INIT; + int i, columns = commit_graph ? 4 : 3; + int must_free_tip = 0; + + /* rev_argv.argv[0] will be ignored by setup_revisions */ + strvec_push(&rev_argv, "log_rev_setup"); + + if (!tip) + tip = ctx.qry.head; + tip = disambiguate_ref(tip, &must_free_tip); + strvec_push(&rev_argv, tip); + + if (grep && pattern && *pattern) { + pattern = xstrdup(pattern); + if (!strcmp(grep, "grep") || !strcmp(grep, "author") || + !strcmp(grep, "committer")) { + strvec_pushf(&rev_argv, "--%s=%s", grep, pattern); + } else if (!strcmp(grep, "range")) { + char *arg; + /* Split the pattern at whitespace and add each token + * as a revision expression. Do not accept other + * rev-list options. Also, replace the previously + * pushed tip (it's no longer relevant). + */ + strvec_pop(&rev_argv); + while ((arg = next_token(&pattern))) { + if (*arg == '-') { + fprintf(stderr, "Bad range expr: %s\n", + arg); + break; + } + strvec_push(&rev_argv, arg); + } + } + } + + if (!path || !ctx.cfg.enable_follow_links) { + /* + * If we don't have a path, "follow" is a no-op so make sure + * the variable is set to false to avoid needing to check + * both this and whether we have a path everywhere. + */ + ctx.qry.follow = 0; + } + + if (commit_graph && !ctx.qry.follow) { + strvec_push(&rev_argv, "--graph"); + strvec_push(&rev_argv, "--color"); + graph_set_column_colors(column_colors_html, + COLUMN_COLORS_HTML_MAX); + } + + if (commit_sort == 1) + strvec_push(&rev_argv, "--date-order"); + else if (commit_sort == 2) + strvec_push(&rev_argv, "--topo-order"); + + if (path && ctx.qry.follow) + strvec_push(&rev_argv, "--follow"); + strvec_push(&rev_argv, "--"); + if (path) + strvec_push(&rev_argv, path); + + init_revisions(&rev, NULL); + rev.abbrev = DEFAULT_ABBREV; + rev.commit_format = CMIT_FMT_DEFAULT; + rev.verbose_header = 1; + rev.show_root_diff = 0; + rev.ignore_missing = 1; + rev.simplify_history = 1; + setup_revisions(rev_argv.nr, rev_argv.v, &rev, NULL); + load_ref_decorations(NULL, DECORATE_FULL_REFS); + rev.show_decorations = 1; + rev.grep_filter.ignore_case = 1; + + rev.diffopt.detect_rename = 1; + rev.diffopt.rename_limit = ctx.cfg.renamelimit; + if (ctx.qry.ignorews) + DIFF_XDL_SET(&rev.diffopt, IGNORE_WHITESPACE); + + compile_grep_patterns(&rev.grep_filter); + prepare_revision_walk(&rev); + + if (pager) { + cgit_print_layout_start(); + html("<table class='list nowrap'>"); + } + + html("<tr class='nohover'>"); + if (commit_graph) + html("<th></th>"); + else + html("<th class='left'>Age</th>"); + html("<th class='left'>Commit message"); + if (pager) { + html(" ("); + cgit_log_link(ctx.qry.showmsg ? "Collapse" : "Expand", NULL, + NULL, ctx.qry.head, ctx.qry.oid, + ctx.qry.vpath, ctx.qry.ofs, ctx.qry.grep, + ctx.qry.search, ctx.qry.showmsg ? 0 : 1, + ctx.qry.follow); + html(")"); + } + html("</th><th class='left'>Author</th>"); + if (rev.graph) + html("<th class='left'>Age</th>"); + if (ctx.repo->enable_log_filecount) { + html("<th class='left'>Files</th>"); + columns++; + } + if (ctx.repo->enable_log_linecount) { + html("<th class='left'>Lines</th>"); + columns++; + } + html("</tr>\n"); + + if (ofs<0) + ofs = 0; + + for (i = 0; i < ofs && (commit = get_revision(&rev)) != NULL; /* nop */) { + if (show_commit(commit, &rev)) + i++; + free_commit_buffer(the_repository->parsed_objects, commit); + free_commit_list(commit->parents); + commit->parents = NULL; + } + + for (i = 0; i < cnt && (commit = get_revision(&rev)) != NULL; /* nop */) { + /* + * In "follow" mode, we must count the files and lines the + * first time we invoke diff on a given commit, and we need + * to do that to see if the commit touches the path we care + * about, so we do it in show_commit. Hence we must clear + * lines_counted here. + * + * This has the side effect of avoiding running diff twice + * when we are both following renames and showing file + * and/or line counts. + */ + lines_counted = 0; + if (show_commit(commit, &rev)) { + i++; + print_commit(commit, &rev); + } + free_commit_buffer(the_repository->parsed_objects, commit); + free_commit_list(commit->parents); + commit->parents = NULL; + } + if (pager) { + html("</table><ul class='pager'>"); + if (ofs > 0) { + html("<li>"); + cgit_log_link("[prev]", NULL, NULL, ctx.qry.head, + ctx.qry.oid, ctx.qry.vpath, + ofs - cnt, ctx.qry.grep, + ctx.qry.search, ctx.qry.showmsg, + ctx.qry.follow); + html("</li>"); + } + if ((commit = get_revision(&rev)) != NULL) { + html("<li>"); + cgit_log_link("[next]", NULL, NULL, ctx.qry.head, + ctx.qry.oid, ctx.qry.vpath, + ofs + cnt, ctx.qry.grep, + ctx.qry.search, ctx.qry.showmsg, + ctx.qry.follow); + html("</li>"); + } + html("</ul>"); + cgit_print_layout_end(); + } else if ((commit = get_revision(&rev)) != NULL) { + htmlf("<tr class='nohover'><td colspan='%d'>", columns); + cgit_log_link("[...]", NULL, NULL, ctx.qry.head, NULL, + ctx.qry.vpath, 0, NULL, NULL, ctx.qry.showmsg, + ctx.qry.follow); + html("</td></tr>\n"); + } + + /* If we allocated tip then it is safe to cast away const. */ + if (must_free_tip) + free((char*) tip); +} diff --git a/third_party/cgit/ui-log.h b/third_party/cgit/ui-log.h new file mode 100644 index 000000000000..325607cdab62 --- /dev/null +++ b/third_party/cgit/ui-log.h @@ -0,0 +1,9 @@ +#ifndef UI_LOG_H +#define UI_LOG_H + +extern void cgit_print_log(const char *tip, int ofs, int cnt, char *grep, + char *pattern, const char *path, int pager, + int commit_graph, int commit_sort); +extern void show_commit_decorations(struct commit *commit); + +#endif /* UI_LOG_H */ diff --git a/third_party/cgit/ui-patch.c b/third_party/cgit/ui-patch.c new file mode 100644 index 000000000000..4ac03cbef1d8 --- /dev/null +++ b/third_party/cgit/ui-patch.c @@ -0,0 +1,98 @@ +/* ui-patch.c: generate patch view + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-patch.h" +#include "html.h" +#include "ui-shared.h" + +/* two commit hashes with two dots in between and termination */ +#define REV_RANGE_LEN 2 * GIT_MAX_HEXSZ + 3 + +void cgit_print_patch(const char *new_rev, const char *old_rev, + const char *prefix) +{ + struct rev_info rev; + struct commit *commit; + struct object_id new_rev_oid, old_rev_oid; + char rev_range[REV_RANGE_LEN]; + const char *rev_argv[] = { NULL, "--reverse", "--format=email", rev_range, "--", prefix, NULL }; + int rev_argc = ARRAY_SIZE(rev_argv) - 1; + char *patchname; + + if (!prefix) + rev_argc--; + + if (!new_rev) + new_rev = ctx.qry.head; + + if (get_oid(new_rev, &new_rev_oid)) { + cgit_print_error_page(404, "Not found", + "Bad object id: %s", new_rev); + return; + } + commit = lookup_commit_reference(the_repository, &new_rev_oid); + if (!commit) { + cgit_print_error_page(404, "Not found", + "Bad commit reference: %s", new_rev); + return; + } + + if (old_rev) { + if (get_oid(old_rev, &old_rev_oid)) { + cgit_print_error_page(404, "Not found", + "Bad object id: %s", old_rev); + return; + } + if (!lookup_commit_reference(the_repository, &old_rev_oid)) { + cgit_print_error_page(404, "Not found", + "Bad commit reference: %s", old_rev); + return; + } + } else if (commit->parents && commit->parents->item) { + oidcpy(&old_rev_oid, &commit->parents->item->object.oid); + } else { + oidclr(&old_rev_oid); + } + + if (is_null_oid(&old_rev_oid)) { + memcpy(rev_range, oid_to_hex(&new_rev_oid), the_hash_algo->hexsz + 1); + } else { + xsnprintf(rev_range, REV_RANGE_LEN, "%s..%s", oid_to_hex(&old_rev_oid), + oid_to_hex(&new_rev_oid)); + } + + patchname = fmt("%s.patch", rev_range); + ctx.page.mimetype = "text/plain"; + ctx.page.filename = patchname; + cgit_print_http_headers(); + + if (ctx.cfg.noplainemail) { + rev_argv[2] = "--format=format:From %H Mon Sep 17 00:00:00 " + "2001%nFrom: %an%nDate: %aD%n%w(78,0,1)Subject: " + "%s%n%n%w(0)%b"; + } + + init_revisions(&rev, NULL); + rev.abbrev = DEFAULT_ABBREV; + rev.verbose_header = 1; + rev.diff = 1; + rev.show_root_diff = 1; + rev.max_parents = 1; + rev.diffopt.output_format |= DIFF_FORMAT_DIFFSTAT | + DIFF_FORMAT_PATCH | DIFF_FORMAT_SUMMARY; + if (prefix) + rev.diffopt.stat_sep = fmt("(limited to '%s')\n\n", prefix); + setup_revisions(rev_argc, rev_argv, &rev, NULL); + prepare_revision_walk(&rev); + + while ((commit = get_revision(&rev)) != NULL) { + log_tree_commit(&rev, commit); + printf("-- \ncgit %s\n\n", cgit_version); + } +} diff --git a/third_party/cgit/ui-patch.h b/third_party/cgit/ui-patch.h new file mode 100644 index 000000000000..7a6cacd5acfe --- /dev/null +++ b/third_party/cgit/ui-patch.h @@ -0,0 +1,7 @@ +#ifndef UI_PATCH_H +#define UI_PATCH_H + +extern void cgit_print_patch(const char *new_rev, const char *old_rev, + const char *prefix); + +#endif /* UI_PATCH_H */ diff --git a/third_party/cgit/ui-plain.c b/third_party/cgit/ui-plain.c new file mode 100644 index 000000000000..65a205fad701 --- /dev/null +++ b/third_party/cgit/ui-plain.c @@ -0,0 +1,207 @@ +/* ui-plain.c: functions for output of plain blobs by path + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-plain.h" +#include "html.h" +#include "ui-shared.h" + +struct walk_tree_context { + int match_baselen; + int match; +}; + +static int print_object(const struct object_id *oid, const char *path) +{ + enum object_type type; + char *buf, *mimetype; + unsigned long size; + + type = oid_object_info(the_repository, oid, &size); + if (type == OBJ_BAD) { + cgit_print_error_page(404, "Not found", "Not found"); + return 0; + } + + buf = read_object_file(oid, &type, &size); + if (!buf) { + cgit_print_error_page(404, "Not found", "Not found"); + return 0; + } + + mimetype = get_mimetype_for_filename(path); + ctx.page.mimetype = mimetype; + + if (!ctx.repo->enable_html_serving) { + html("X-Content-Type-Options: nosniff\n"); + html("Content-Security-Policy: default-src 'none'\n"); + if (mimetype) { + /* Built-in white list allows PDF and everything that isn't text/ and application/ */ + if ((!strncmp(mimetype, "text/", 5) || !strncmp(mimetype, "application/", 12)) && strcmp(mimetype, "application/pdf")) + ctx.page.mimetype = NULL; + } + } + + if (!ctx.page.mimetype) { + if (buffer_is_binary(buf, size)) { + ctx.page.mimetype = "application/octet-stream"; + ctx.page.charset = NULL; + } else { + ctx.page.mimetype = "text/plain"; + } + } + ctx.page.filename = path; + ctx.page.size = size; + ctx.page.etag = oid_to_hex(oid); + cgit_print_http_headers(); + html_raw(buf, size); + free(mimetype); + free(buf); + return 1; +} + +static char *buildpath(const char *base, int baselen, const char *path) +{ + if (path[0]) + return fmtalloc("%.*s%s/", baselen, base, path); + else + return fmtalloc("%.*s/", baselen, base); +} + +static void print_dir(const struct object_id *oid, const char *base, + int baselen, const char *path) +{ + char *fullpath, *slash; + size_t len; + + fullpath = buildpath(base, baselen, path); + slash = (fullpath[0] == '/' ? "" : "/"); + ctx.page.etag = oid_to_hex(oid); + cgit_print_http_headers(); + htmlf("<html><head><title>%s", slash); + html_txt(fullpath); + htmlf("</title></head>\n<body>\n<h2>%s", slash); + html_txt(fullpath); + html("</h2>\n<ul>\n"); + len = strlen(fullpath); + if (len > 1) { + fullpath[len - 1] = 0; + slash = strrchr(fullpath, '/'); + if (slash) + *(slash + 1) = 0; + else { + free(fullpath); + fullpath = NULL; + } + html("<li>"); + cgit_plain_link("../", NULL, NULL, ctx.qry.head, ctx.qry.oid, + fullpath); + html("</li>\n"); + } + free(fullpath); +} + +static void print_dir_entry(const struct object_id *oid, const char *base, + int baselen, const char *path, unsigned mode) +{ + char *fullpath; + + fullpath = buildpath(base, baselen, path); + if (!S_ISDIR(mode) && !S_ISGITLINK(mode)) + fullpath[strlen(fullpath) - 1] = 0; + html(" <li>"); + if (S_ISGITLINK(mode)) { + cgit_submodule_link(NULL, fullpath, oid_to_hex(oid)); + } else + cgit_plain_link(path, NULL, NULL, ctx.qry.head, ctx.qry.oid, + fullpath); + html("</li>\n"); + free(fullpath); +} + +static void print_dir_tail(void) +{ + html(" </ul>\n</body></html>\n"); +} + +static int walk_tree(const struct object_id *oid, struct strbuf *base, + const char *pathname, unsigned mode, void *cbdata) +{ + struct walk_tree_context *walk_tree_ctx = cbdata; + + if (base->len == walk_tree_ctx->match_baselen) { + if (S_ISREG(mode) || S_ISLNK(mode)) { + if (print_object(oid, pathname)) + walk_tree_ctx->match = 1; + } else if (S_ISDIR(mode)) { + print_dir(oid, base->buf, base->len, pathname); + walk_tree_ctx->match = 2; + return READ_TREE_RECURSIVE; + } + } else if (base->len < INT_MAX && (int)base->len > walk_tree_ctx->match_baselen) { + print_dir_entry(oid, base->buf, base->len, pathname, mode); + walk_tree_ctx->match = 2; + } else if (S_ISDIR(mode)) { + return READ_TREE_RECURSIVE; + } + + return 0; +} + +static int basedir_len(const char *path) +{ + char *p = strrchr(path, '/'); + if (p) + return p - path + 1; + return 0; +} + +void cgit_print_plain(void) +{ + const char *rev = ctx.qry.oid; + struct object_id oid; + struct commit *commit; + struct pathspec_item path_items = { + .match = ctx.qry.path, + .len = ctx.qry.path ? strlen(ctx.qry.path) : 0 + }; + struct pathspec paths = { + .nr = 1, + .items = &path_items + }; + struct walk_tree_context walk_tree_ctx = { + .match = 0 + }; + + if (!rev) + rev = ctx.qry.head; + + if (get_oid(rev, &oid)) { + cgit_print_error_page(404, "Not found", "Not found"); + return; + } + commit = lookup_commit_reference(the_repository, &oid); + if (!commit || parse_commit(commit)) { + cgit_print_error_page(404, "Not found", "Not found"); + return; + } + if (!path_items.match) { + path_items.match = ""; + walk_tree_ctx.match_baselen = -1; + print_dir(get_commit_tree_oid(commit), "", 0, ""); + walk_tree_ctx.match = 2; + } + else + walk_tree_ctx.match_baselen = basedir_len(path_items.match); + read_tree(the_repository, repo_get_commit_tree(the_repository, commit), + &paths, walk_tree, &walk_tree_ctx); + if (!walk_tree_ctx.match) + cgit_print_error_page(404, "Not found", "Not found"); + else if (walk_tree_ctx.match == 2) + print_dir_tail(); +} diff --git a/third_party/cgit/ui-plain.h b/third_party/cgit/ui-plain.h new file mode 100644 index 000000000000..5bff07b83b99 --- /dev/null +++ b/third_party/cgit/ui-plain.h @@ -0,0 +1,6 @@ +#ifndef UI_PLAIN_H +#define UI_PLAIN_H + +extern void cgit_print_plain(void); + +#endif /* UI_PLAIN_H */ diff --git a/third_party/cgit/ui-refs.c b/third_party/cgit/ui-refs.c new file mode 100644 index 000000000000..456f610df41c --- /dev/null +++ b/third_party/cgit/ui-refs.c @@ -0,0 +1,219 @@ +/* ui-refs.c: browse symbolic refs + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-refs.h" +#include "html.h" +#include "ui-shared.h" + +static inline int cmp_age(int age1, int age2) +{ + /* age1 and age2 are assumed to be non-negative */ + return age2 - age1; +} + +static int cmp_ref_name(const void *a, const void *b) +{ + struct refinfo *r1 = *(struct refinfo **)a; + struct refinfo *r2 = *(struct refinfo **)b; + + return strcmp(r1->refname, r2->refname); +} + +static int cmp_branch_age(const void *a, const void *b) +{ + struct refinfo *r1 = *(struct refinfo **)a; + struct refinfo *r2 = *(struct refinfo **)b; + + return cmp_age(r1->commit->committer_date, r2->commit->committer_date); +} + +static int get_ref_age(struct refinfo *ref) +{ + if (!ref->object) + return 0; + switch (ref->object->type) { + case OBJ_TAG: + return ref->tag ? ref->tag->tagger_date : 0; + case OBJ_COMMIT: + return ref->commit ? ref->commit->committer_date : 0; + } + return 0; +} + +static int cmp_tag_age(const void *a, const void *b) +{ + struct refinfo *r1 = *(struct refinfo **)a; + struct refinfo *r2 = *(struct refinfo **)b; + + return cmp_age(get_ref_age(r1), get_ref_age(r2)); +} + +static int print_branch(struct refinfo *ref) +{ + struct commitinfo *info = ref->commit; + char *name = (char *)ref->refname; + + if (!info) + return 1; + html("<tr><td>"); + cgit_log_link(name, NULL, NULL, name, NULL, NULL, 0, NULL, NULL, + ctx.qry.showmsg, 0); + html("</td><td>"); + + if (ref->object->type == OBJ_COMMIT) { + cgit_commit_link(info->subject, NULL, NULL, name, NULL, NULL); + html("</td><td>"); + cgit_open_filter(ctx.repo->email_filter, info->author_email, "refs"); + html_txt(info->author); + cgit_close_filter(ctx.repo->email_filter); + html("</td><td colspan='2'>"); + cgit_print_age(info->committer_date, info->committer_tz, -1); + } else { + html("</td><td></td><td>"); + cgit_object_link(ref->object); + } + html("</td></tr>\n"); + return 0; +} + +static void print_tag_header(void) +{ + html("<tr class='nohover'><th class='left'>Tag</th>" + "<th class='left'>Download</th>" + "<th class='left'>Author</th>" + "<th class='left' colspan='2'>Age</th></tr>\n"); +} + +static int print_tag(struct refinfo *ref) +{ + struct tag *tag = NULL; + struct taginfo *info = NULL; + char *name = (char *)ref->refname; + struct object *obj = ref->object; + + if (obj->type == OBJ_TAG) { + tag = (struct tag *)obj; + obj = tag->tagged; + info = ref->tag; + if (!info) + return 1; + } + + html("<tr><td>"); + cgit_tag_link(name, NULL, NULL, name); + html("</td><td>"); + if (ctx.repo->snapshots && (obj->type == OBJ_COMMIT)) + cgit_print_snapshot_links(ctx.repo, name, " "); + else + cgit_object_link(obj); + html("</td><td>"); + if (info) { + if (info->tagger) { + cgit_open_filter(ctx.repo->email_filter, info->tagger_email, "refs"); + html_txt(info->tagger); + cgit_close_filter(ctx.repo->email_filter); + } + } else if (ref->object->type == OBJ_COMMIT) { + cgit_open_filter(ctx.repo->email_filter, ref->commit->author_email, "refs"); + html_txt(ref->commit->author); + cgit_close_filter(ctx.repo->email_filter); + } + html("</td><td colspan='2'>"); + if (info) { + if (info->tagger_date > 0) + cgit_print_age(info->tagger_date, info->tagger_tz, -1); + } else if (ref->object->type == OBJ_COMMIT) { + cgit_print_age(ref->commit->commit->date, 0, -1); + } + html("</td></tr>\n"); + + return 0; +} + +static void print_refs_link(const char *path) +{ + html("<tr class='nohover'><td colspan='5'>"); + cgit_refs_link("[...]", NULL, NULL, ctx.qry.head, NULL, path); + html("</td></tr>"); +} + +void cgit_print_branches(int maxcount) +{ + struct reflist list; + int i; + + html("<tr class='nohover'><th class='left'>Branch</th>" + "<th class='left'>Commit message</th>" + "<th class='left'>Author</th>" + "<th class='left' colspan='2'>Age</th></tr>\n"); + + list.refs = NULL; + list.alloc = list.count = 0; + for_each_branch_ref(cgit_refs_cb, &list); + if (ctx.repo->enable_remote_branches) + for_each_remote_ref(cgit_refs_cb, &list); + + if (maxcount == 0 || maxcount > list.count) + maxcount = list.count; + + qsort(list.refs, list.count, sizeof(*list.refs), cmp_branch_age); + if (ctx.repo->branch_sort == 0) + qsort(list.refs, maxcount, sizeof(*list.refs), cmp_ref_name); + + for (i = 0; i < maxcount; i++) + print_branch(list.refs[i]); + + if (maxcount < list.count) + print_refs_link("heads"); + + cgit_free_reflist_inner(&list); +} + +void cgit_print_tags(int maxcount) +{ + struct reflist list; + int i; + + list.refs = NULL; + list.alloc = list.count = 0; + for_each_tag_ref(cgit_refs_cb, &list); + if (list.count == 0) + return; + qsort(list.refs, list.count, sizeof(*list.refs), cmp_tag_age); + if (!maxcount) + maxcount = list.count; + else if (maxcount > list.count) + maxcount = list.count; + print_tag_header(); + for (i = 0; i < maxcount; i++) + print_tag(list.refs[i]); + + if (maxcount < list.count) + print_refs_link("tags"); + + cgit_free_reflist_inner(&list); +} + +void cgit_print_refs(void) +{ + cgit_print_layout_start(); + html("<table class='list nowrap'>"); + + if (ctx.qry.path && starts_with(ctx.qry.path, "heads")) + cgit_print_branches(0); + else if (ctx.qry.path && starts_with(ctx.qry.path, "tags")) + cgit_print_tags(0); + else { + cgit_print_branches(0); + html("<tr class='nohover'><td colspan='5'> </td></tr>"); + cgit_print_tags(0); + } + html("</table>"); + cgit_print_layout_end(); +} diff --git a/third_party/cgit/ui-refs.h b/third_party/cgit/ui-refs.h new file mode 100644 index 000000000000..1d4a54a2c869 --- /dev/null +++ b/third_party/cgit/ui-refs.h @@ -0,0 +1,8 @@ +#ifndef UI_REFS_H +#define UI_REFS_H + +extern void cgit_print_branches(int maxcount); +extern void cgit_print_tags(int maxcount); +extern void cgit_print_refs(void); + +#endif /* UI_REFS_H */ diff --git a/third_party/cgit/ui-repolist.c b/third_party/cgit/ui-repolist.c new file mode 100644 index 000000000000..529a2038baa2 --- /dev/null +++ b/third_party/cgit/ui-repolist.c @@ -0,0 +1,379 @@ +/* ui-repolist.c: functions for generating the repolist page + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-repolist.h" +#include "html.h" +#include "ui-shared.h" + +static time_t read_agefile(const char *path) +{ + time_t result; + size_t size; + char *buf = NULL; + struct strbuf date_buf = STRBUF_INIT; + + if (readfile(path, &buf, &size)) { + free(buf); + return 0; + } + + if (parse_date(buf, &date_buf) == 0) + result = strtoul(date_buf.buf, NULL, 10); + else + result = 0; + free(buf); + strbuf_release(&date_buf); + return result; +} + +static int get_repo_modtime(const struct cgit_repo *repo, time_t *mtime) +{ + struct strbuf path = STRBUF_INIT; + struct stat s; + struct cgit_repo *r = (struct cgit_repo *)repo; + + if (repo->mtime != -1) { + *mtime = repo->mtime; + return 1; + } + strbuf_addf(&path, "%s/%s", repo->path, ctx.cfg.agefile); + if (stat(path.buf, &s) == 0) { + *mtime = read_agefile(path.buf); + if (*mtime) { + r->mtime = *mtime; + goto end; + } + } + + strbuf_reset(&path); + strbuf_addf(&path, "%s/refs/heads/%s", repo->path, + repo->defbranch ? repo->defbranch : "master"); + if (stat(path.buf, &s) == 0) { + *mtime = s.st_mtime; + r->mtime = *mtime; + goto end; + } + + strbuf_reset(&path); + strbuf_addf(&path, "%s/%s", repo->path, "packed-refs"); + if (stat(path.buf, &s) == 0) { + *mtime = s.st_mtime; + r->mtime = *mtime; + goto end; + } + + *mtime = 0; + r->mtime = *mtime; +end: + strbuf_release(&path); + return (r->mtime != 0); +} + +static void print_modtime(struct cgit_repo *repo) +{ + time_t t; + if (get_repo_modtime(repo, &t)) + cgit_print_age(t, 0, -1); +} + +static int is_match(struct cgit_repo *repo) +{ + if (!ctx.qry.search) + return 1; + if (repo->url && strcasestr(repo->url, ctx.qry.search)) + return 1; + if (repo->name && strcasestr(repo->name, ctx.qry.search)) + return 1; + if (repo->desc && strcasestr(repo->desc, ctx.qry.search)) + return 1; + if (repo->owner && strcasestr(repo->owner, ctx.qry.search)) + return 1; + return 0; +} + +static int is_in_url(struct cgit_repo *repo) +{ + if (!ctx.qry.url) + return 1; + if (repo->url && starts_with(repo->url, ctx.qry.url)) + return 1; + return 0; +} + +static int is_visible(struct cgit_repo *repo) +{ + if (repo->hide || repo->ignore) + return 0; + if (!(is_match(repo) && is_in_url(repo))) + return 0; + return 1; +} + +static int any_repos_visible(void) +{ + int i; + + for (i = 0; i < cgit_repolist.count; i++) { + if (is_visible(&cgit_repolist.repos[i])) + return 1; + } + return 0; +} + +static void print_sort_header(const char *title, const char *sort) +{ + char *currenturl = cgit_currenturl(); + html("<th class='left'><a href='"); + html_attr(currenturl); + htmlf("?s=%s", sort); + if (ctx.qry.search) { + html("&q="); + html_url_arg(ctx.qry.search); + } + htmlf("'>%s</a></th>", title); + free(currenturl); +} + +static void print_header(void) +{ + html("<tr class='nohover'>"); + print_sort_header("Name", "name"); + print_sort_header("Description", "desc"); + if (ctx.cfg.enable_index_owner) + print_sort_header("Owner", "owner"); + print_sort_header("Idle", "idle"); + if (ctx.cfg.enable_index_links) + html("<th class='left'>Links</th>"); + html("</tr>\n"); +} + + +static void print_pager(int items, int pagelen, char *search, char *sort) +{ + int i, ofs; + char *class = NULL; + html("<ul class='pager'>"); + for (i = 0, ofs = 0; ofs < items; i++, ofs = i * pagelen) { + class = (ctx.qry.ofs == ofs) ? "current" : NULL; + html("<li>"); + cgit_index_link(fmt("[%d]", i + 1), fmt("Page %d", i + 1), + class, search, sort, ofs, 0); + html("</li>"); + } + html("</ul>"); +} + +static int cmp(const char *s1, const char *s2) +{ + if (s1 && s2) { + if (ctx.cfg.case_sensitive_sort) + return strcmp(s1, s2); + else + return strcasecmp(s1, s2); + } + if (s1 && !s2) + return -1; + if (s2 && !s1) + return 1; + return 0; +} + +static int sort_name(const void *a, const void *b) +{ + const struct cgit_repo *r1 = a; + const struct cgit_repo *r2 = b; + + return cmp(r1->name, r2->name); +} + +static int sort_desc(const void *a, const void *b) +{ + const struct cgit_repo *r1 = a; + const struct cgit_repo *r2 = b; + + return cmp(r1->desc, r2->desc); +} + +static int sort_owner(const void *a, const void *b) +{ + const struct cgit_repo *r1 = a; + const struct cgit_repo *r2 = b; + + return cmp(r1->owner, r2->owner); +} + +static int sort_idle(const void *a, const void *b) +{ + const struct cgit_repo *r1 = a; + const struct cgit_repo *r2 = b; + time_t t1, t2; + + t1 = t2 = 0; + get_repo_modtime(r1, &t1); + get_repo_modtime(r2, &t2); + return t2 - t1; +} + +static int sort_section(const void *a, const void *b) +{ + const struct cgit_repo *r1 = a; + const struct cgit_repo *r2 = b; + int result; + + result = cmp(r1->section, r2->section); + if (!result) { + if (!strcmp(ctx.cfg.repository_sort, "age")) + result = sort_idle(r1, r2); + if (!result) + result = cmp(r1->name, r2->name); + } + return result; +} + +struct sortcolumn { + const char *name; + int (*fn)(const void *a, const void *b); +}; + +static const struct sortcolumn sortcolumn[] = { + {"section", sort_section}, + {"name", sort_name}, + {"desc", sort_desc}, + {"owner", sort_owner}, + {"idle", sort_idle}, + {NULL, NULL} +}; + +static int sort_repolist(char *field) +{ + const struct sortcolumn *column; + + for (column = &sortcolumn[0]; column->name; column++) { + if (strcmp(field, column->name)) + continue; + qsort(cgit_repolist.repos, cgit_repolist.count, + sizeof(struct cgit_repo), column->fn); + return 1; + } + return 0; +} + + +void cgit_print_repolist(void) +{ + int i, columns = 3, hits = 0, header = 0; + char *last_section = NULL; + char *section; + char *repourl; + int sorted = 0; + + if (!any_repos_visible()) { + cgit_print_error_page(404, "Not found", "No repositories found"); + return; + } + + if (ctx.cfg.enable_index_links) + ++columns; + if (ctx.cfg.enable_index_owner) + ++columns; + + ctx.page.title = ctx.cfg.root_title; + cgit_print_http_headers(); + cgit_print_docstart(); + cgit_print_pageheader(); + + if (ctx.qry.sort) + sorted = sort_repolist(ctx.qry.sort); + else if (ctx.cfg.section_sort) + sort_repolist("section"); + + html("<table summary='repository list' class='list nowrap'>"); + for (i = 0; i < cgit_repolist.count; i++) { + ctx.repo = &cgit_repolist.repos[i]; + if (!is_visible(ctx.repo)) + continue; + hits++; + if (hits <= ctx.qry.ofs) + continue; + if (hits > ctx.qry.ofs + ctx.cfg.max_repo_count) + continue; + if (!header++) + print_header(); + section = ctx.repo->section; + if (section && !strcmp(section, "")) + section = NULL; + if (!sorted && + ((last_section == NULL && section != NULL) || + (last_section != NULL && section == NULL) || + (last_section != NULL && section != NULL && + strcmp(section, last_section)))) { + htmlf("<tr class='nohover-highlight'><td colspan='%d' class='reposection'>", + columns); + html_txt(section); + html("</td></tr>"); + last_section = section; + } + htmlf("<tr><td class='%s'>", + !sorted && section ? "sublevel-repo" : "toplevel-repo"); + cgit_summary_link(ctx.repo->name, ctx.repo->name, NULL, NULL); + html("</td><td>"); + repourl = cgit_repourl(ctx.repo->url); + html_link_open(repourl, NULL, NULL); + free(repourl); + if (html_ntxt(ctx.repo->desc, ctx.cfg.max_repodesc_len) < 0) + html("..."); + html_link_close(); + html("</td><td>"); + if (ctx.cfg.enable_index_owner) { + if (ctx.repo->owner_filter) { + cgit_open_filter(ctx.repo->owner_filter); + html_txt(ctx.repo->owner); + cgit_close_filter(ctx.repo->owner_filter); + } else { + char *currenturl = cgit_currenturl(); + html("<a href='"); + html_attr(currenturl); + html("?q="); + html_url_arg(ctx.repo->owner); + html("'>"); + html_txt(ctx.repo->owner); + html("</a>"); + free(currenturl); + } + html("</td><td>"); + } + print_modtime(ctx.repo); + html("</td>"); + if (ctx.cfg.enable_index_links) { + html("<td>"); + cgit_summary_link("summary", NULL, "button", NULL); + cgit_log_link("log", NULL, "button", NULL, NULL, NULL, + 0, NULL, NULL, ctx.qry.showmsg, 0); + cgit_tree_link("tree", NULL, "button", NULL, NULL, NULL); + html("</td>"); + } + html("</tr>\n"); + } + html("</table>"); + if (hits > ctx.cfg.max_repo_count) + print_pager(hits, ctx.cfg.max_repo_count, ctx.qry.search, ctx.qry.sort); + cgit_print_docend(); +} + +void cgit_print_site_readme(void) +{ + cgit_print_layout_start(); + if (!ctx.cfg.root_readme) + goto done; + cgit_open_filter(ctx.cfg.about_filter, ctx.cfg.root_readme); + html_include(ctx.cfg.root_readme); + cgit_close_filter(ctx.cfg.about_filter); +done: + cgit_print_layout_end(); +} diff --git a/third_party/cgit/ui-repolist.h b/third_party/cgit/ui-repolist.h new file mode 100644 index 000000000000..1b6b3227dbc0 --- /dev/null +++ b/third_party/cgit/ui-repolist.h @@ -0,0 +1,7 @@ +#ifndef UI_REPOLIST_H +#define UI_REPOLIST_H + +extern void cgit_print_repolist(void); +extern void cgit_print_site_readme(void); + +#endif /* UI_REPOLIST_H */ diff --git a/third_party/cgit/ui-shared.c b/third_party/cgit/ui-shared.c new file mode 100644 index 000000000000..cf7907ffe393 --- /dev/null +++ b/third_party/cgit/ui-shared.c @@ -0,0 +1,1222 @@ +/* ui-shared.c: common web output functions + * + * Copyright (C) 2006-2017 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-shared.h" +#include "cmd.h" +#include "html.h" +#include "version.h" + +static const char cgit_doctype[] = +"<!DOCTYPE html>\n"; + +static char *http_date(time_t t) +{ + static char day[][4] = + {"Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"}; + static char month[][4] = + {"Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"}; + struct tm tm; + gmtime_r(&t, &tm); + return fmt("%s, %02d %s %04d %02d:%02d:%02d GMT", day[tm.tm_wday], + tm.tm_mday, month[tm.tm_mon], 1900 + tm.tm_year, + tm.tm_hour, tm.tm_min, tm.tm_sec); +} + +void cgit_print_error(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + cgit_vprint_error(fmt, ap); + va_end(ap); +} + +void cgit_vprint_error(const char *fmt, va_list ap) +{ + va_list cp; + html("<div class='error'>"); + va_copy(cp, ap); + html_vtxtf(fmt, cp); + va_end(cp); + html("</div>\n"); +} + +const char *cgit_httpscheme(void) +{ + if (ctx.env.https && !strcmp(ctx.env.https, "on")) + return "https://"; + else + return "http://"; +} + +char *cgit_hosturl(void) +{ + if (ctx.env.http_host) + return xstrdup(ctx.env.http_host); + if (!ctx.env.server_name) + return NULL; + if (!ctx.env.server_port || atoi(ctx.env.server_port) == 80) + return xstrdup(ctx.env.server_name); + return fmtalloc("%s:%s", ctx.env.server_name, ctx.env.server_port); +} + +char *cgit_currenturl(void) +{ + const char *root = cgit_rooturl(); + + if (!ctx.qry.url) + return xstrdup(root); + if (root[0] && root[strlen(root) - 1] == '/') + return fmtalloc("%s%s", root, ctx.qry.url); + return fmtalloc("%s/%s", root, ctx.qry.url); +} + +char *cgit_currentfullurl(void) +{ + const char *root = cgit_rooturl(); + const char *orig_query = ctx.env.query_string ? ctx.env.query_string : ""; + size_t len = strlen(orig_query); + char *query = xmalloc(len + 2), *start_url, *ret; + + /* Remove all url=... parts from query string */ + memcpy(query + 1, orig_query, len + 1); + query[0] = '?'; + start_url = query; + while ((start_url = strstr(start_url, "url=")) != NULL) { + if (start_url[-1] == '?' || start_url[-1] == '&') { + const char *end_url = strchr(start_url, '&'); + if (end_url) + memmove(start_url, end_url + 1, strlen(end_url)); + else + start_url[0] = '\0'; + } else + ++start_url; + } + if (!query[1]) + query[0] = '\0'; + + if (!ctx.qry.url) + ret = fmtalloc("%s%s", root, query); + else if (root[0] && root[strlen(root) - 1] == '/') + ret = fmtalloc("%s%s%s", root, ctx.qry.url, query); + else + ret = fmtalloc("%s/%s%s", root, ctx.qry.url, query); + free(query); + return ret; +} + +const char *cgit_rooturl(void) +{ + if (ctx.cfg.virtual_root) + return ctx.cfg.virtual_root; + else + return ctx.cfg.script_name; +} + +const char *cgit_loginurl(void) +{ + static const char *login_url; + if (!login_url) + login_url = fmtalloc("%s?p=login", cgit_rooturl()); + return login_url; +} + +char *cgit_repourl(const char *reponame) +{ + // my cgit instance *only* serves the depot, hence that's the only value ever + // needed. + return fmtalloc("/"); +} + +char *cgit_fileurl(const char *reponame, const char *pagename, + const char *filename, const char *query) +{ + struct strbuf sb = STRBUF_INIT; + + strbuf_addf(&sb, "%s%s/%s", ctx.cfg.virtual_root, + pagename, (filename ? filename:"")); + + if (query) { + strbuf_addf(&sb, "%s%s", "?", query); + } + + return strbuf_detach(&sb, NULL); +} + +char *cgit_pageurl(const char *reponame, const char *pagename, + const char *query) +{ + return cgit_fileurl(reponame, pagename, NULL, query); +} + +const char *cgit_repobasename(const char *reponame) +{ + /* I assume we don't need to store more than one repo basename */ + static char rvbuf[1024]; + int p; + const char *rv; + size_t len; + + len = strlcpy(rvbuf, reponame, sizeof(rvbuf)); + if (len >= sizeof(rvbuf)) + die("cgit_repobasename: truncated repository name '%s'", reponame); + p = len - 1; + /* strip trailing slashes */ + while (p && rvbuf[p] == '/') + rvbuf[p--] = '\0'; + /* strip trailing .git */ + if (p >= 3 && starts_with(&rvbuf[p-3], ".git")) { + p -= 3; + rvbuf[p--] = '\0'; + } + /* strip more trailing slashes if any */ + while (p && rvbuf[p] == '/') + rvbuf[p--] = '\0'; + /* find last slash in the remaining string */ + rv = strrchr(rvbuf, '/'); + if (rv) + return ++rv; + return rvbuf; +} + +const char *cgit_snapshot_prefix(const struct cgit_repo *repo) +{ + if (repo->snapshot_prefix) + return repo->snapshot_prefix; + + return cgit_repobasename(repo->url); +} + +static void site_url(const char *page, const char *search, const char *sort, int ofs, int always_root) +{ + char *delim = "?"; + + if (always_root || page) + html_attr(cgit_rooturl()); + else { + char *currenturl = cgit_currenturl(); + html_attr(currenturl); + free(currenturl); + } + + if (page) { + htmlf("?p=%s", page); + delim = "&"; + } + if (search) { + html(delim); + html("q="); + html_attr(search); + delim = "&"; + } + if (sort) { + html(delim); + html("s="); + html_attr(sort); + delim = "&"; + } + if (ofs) { + html(delim); + htmlf("ofs=%d", ofs); + } +} + +static void site_link(const char *page, const char *name, const char *title, + const char *class, const char *search, const char *sort, int ofs, int always_root) +{ + html("<a"); + if (title) { + html(" title='"); + html_attr(title); + html("'"); + } + if (class) { + html(" class='"); + html_attr(class); + html("'"); + } + html(" href='"); + site_url(page, search, sort, ofs, always_root); + html("'>"); + html_txt(name); + html("</a>"); +} + +void cgit_index_link(const char *name, const char *title, const char *class, + const char *pattern, const char *sort, int ofs, int always_root) +{ + site_link(NULL, name, title, class, pattern, sort, ofs, always_root); +} + +static char *repolink(const char *title, const char *class, const char *page, + const char *head, const char *path) +{ + char *delim = "?"; + + html("<a"); + if (title) { + html(" title='"); + html_attr(title); + html("'"); + } + if (class) { + html(" class='"); + html_attr(class); + html("'"); + } + html(" href='"); + if (ctx.cfg.virtual_root) { + html_url_path(ctx.cfg.virtual_root); + if (page) { + html_url_path(page); + html("/"); + if (path) + html_url_path(path); + } + } else { + html_url_path(ctx.cfg.script_name); + html("?url="); + html_url_arg(ctx.repo->url); + if (ctx.repo->url[strlen(ctx.repo->url) - 1] != '/') + html("/"); + if (page) { + html_url_arg(page); + html("/"); + if (path) + html_url_arg(path); + } + delim = "&"; + } + if (head && ctx.repo->defbranch && strcmp(head, ctx.repo->defbranch)) { + html(delim); + html("h="); + html_url_arg(head); + delim = "&"; + } + return fmt("%s", delim); +} + +static void reporevlink(const char *page, const char *name, const char *title, + const char *class, const char *head, const char *rev, + const char *path) +{ + char *delim; + + delim = repolink(title, class, page, head, path); + if (rev && ctx.qry.head != NULL && strcmp(rev, ctx.qry.head)) { + html(delim); + html("id="); + html_url_arg(rev); + } + html("'>"); + html_txt(name); + html("</a>"); +} + +void cgit_summary_link(const char *name, const char *title, const char *class, + const char *head) +{ + reporevlink(NULL, name, title, class, head, NULL, NULL); +} + +void cgit_tag_link(const char *name, const char *title, const char *class, + const char *tag) +{ + reporevlink("tag", name, title, class, tag, NULL, NULL); +} + +void cgit_about_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + reporevlink("about", name, title, class, head, rev, path); +} + +void cgit_tree_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + reporevlink("tree", name, title, class, head, rev, path); +} + +void cgit_plain_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + reporevlink("plain", name, title, class, head, rev, path); +} + +void cgit_blame_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + reporevlink("blame", name, title, class, head, rev, path); +} + +void cgit_log_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path, + int ofs, const char *grep, const char *pattern, int showmsg, + int follow) +{ + char *delim; + + delim = repolink(title, class, "log", head, path); + if (rev && ctx.qry.head && strcmp(rev, ctx.qry.head)) { + html(delim); + html("id="); + html_url_arg(rev); + delim = "&"; + } + if (grep && pattern) { + html(delim); + html("qt="); + html_url_arg(grep); + delim = "&"; + html(delim); + html("q="); + html_url_arg(pattern); + } + if (ofs > 0) { + html(delim); + html("ofs="); + htmlf("%d", ofs); + delim = "&"; + } + if (showmsg) { + html(delim); + html("showmsg=1"); + delim = "&"; + } + if (follow) { + html(delim); + html("follow=1"); + } + html("'>"); + html_txt(name); + html("</a>"); +} + +void cgit_commit_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + char *delim; + + delim = repolink(title, class, "commit", head, path); + if (rev && ctx.qry.head && strcmp(rev, ctx.qry.head)) { + html(delim); + html("id="); + html_url_arg(rev); + delim = "&"; + } + if (ctx.qry.difftype) { + html(delim); + htmlf("dt=%d", ctx.qry.difftype); + delim = "&"; + } + if (ctx.qry.context > 0 && ctx.qry.context != 3) { + html(delim); + html("context="); + htmlf("%d", ctx.qry.context); + delim = "&"; + } + if (ctx.qry.ignorews) { + html(delim); + html("ignorews=1"); + delim = "&"; + } + if (ctx.qry.follow) { + html(delim); + html("follow=1"); + } + html("'>"); + if (name[0] != '\0') { + if (strlen(name) > ctx.cfg.max_msg_len && ctx.cfg.max_msg_len >= 15) { + html_ntxt(name, ctx.cfg.max_msg_len - 3); + html("..."); + } else + html_txt(name); + } else + html_txt("(no commit message)"); + html("</a>"); +} + +void cgit_refs_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + reporevlink("refs", name, title, class, head, rev, path); +} + +void cgit_snapshot_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, + const char *archivename) +{ + reporevlink("snapshot", name, title, class, head, rev, archivename); +} + +void cgit_diff_link(const char *name, const char *title, const char *class, + const char *head, const char *new_rev, const char *old_rev, + const char *path) +{ + char *delim; + + delim = repolink(title, class, "diff", head, path); + if (new_rev && ctx.qry.head != NULL && strcmp(new_rev, ctx.qry.head)) { + html(delim); + html("id="); + html_url_arg(new_rev); + delim = "&"; + } + if (old_rev) { + html(delim); + html("id2="); + html_url_arg(old_rev); + delim = "&"; + } + if (ctx.qry.difftype) { + html(delim); + htmlf("dt=%d", ctx.qry.difftype); + delim = "&"; + } + if (ctx.qry.context > 0 && ctx.qry.context != 3) { + html(delim); + html("context="); + htmlf("%d", ctx.qry.context); + delim = "&"; + } + if (ctx.qry.ignorews) { + html(delim); + html("ignorews=1"); + delim = "&"; + } + if (ctx.qry.follow) { + html(delim); + html("follow=1"); + } + html("'>"); + html_txt(name); + html("</a>"); +} + +void cgit_patch_link(const char *name, const char *title, const char *class, + const char *head, const char *rev, const char *path) +{ + reporevlink("patch", name, title, class, head, rev, path); +} + +void cgit_stats_link(const char *name, const char *title, const char *class, + const char *head, const char *path) +{ + reporevlink("stats", name, title, class, head, NULL, path); +} + +static void cgit_self_link(char *name, const char *title, const char *class) +{ + if (!strcmp(ctx.qry.page, "repolist")) + cgit_index_link(name, title, class, ctx.qry.search, ctx.qry.sort, + ctx.qry.ofs, 1); + else if (!strcmp(ctx.qry.page, "about")) + cgit_about_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "summary")) + cgit_summary_link(name, title, class, ctx.qry.head); + else if (!strcmp(ctx.qry.page, "tag")) + cgit_tag_link(name, title, class, ctx.qry.has_oid ? + ctx.qry.oid : ctx.qry.head); + else if (!strcmp(ctx.qry.page, "tree")) + cgit_tree_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "plain")) + cgit_plain_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "blame")) + cgit_blame_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "log")) + cgit_log_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path, ctx.qry.ofs, + ctx.qry.grep, ctx.qry.search, + ctx.qry.showmsg, ctx.qry.follow); + else if (!strcmp(ctx.qry.page, "commit")) + cgit_commit_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "patch")) + cgit_patch_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "refs")) + cgit_refs_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "snapshot")) + cgit_snapshot_link(name, title, class, ctx.qry.head, + ctx.qry.has_oid ? ctx.qry.oid : NULL, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "diff")) + cgit_diff_link(name, title, class, ctx.qry.head, + ctx.qry.oid, ctx.qry.oid2, + ctx.qry.path); + else if (!strcmp(ctx.qry.page, "stats")) + cgit_stats_link(name, title, class, ctx.qry.head, + ctx.qry.path); + else { + /* Don't known how to make link for this page */ + repolink(title, class, ctx.qry.page, ctx.qry.head, ctx.qry.path); + html("><!-- cgit_self_link() doesn't know how to make link for page '"); + html_txt(ctx.qry.page); + html("' -->"); + html_txt(name); + html("</a>"); + } +} + +void cgit_object_link(struct object *obj) +{ + char *page, *shortrev, *fullrev, *name; + + fullrev = oid_to_hex(&obj->oid); + shortrev = xstrdup(fullrev); + shortrev[10] = '\0'; + if (obj->type == OBJ_COMMIT) { + cgit_commit_link(fmt("commit %s...", shortrev), NULL, NULL, + ctx.qry.head, fullrev, NULL); + return; + } else if (obj->type == OBJ_TREE) + page = "tree"; + else if (obj->type == OBJ_TAG) + page = "tag"; + else + page = "blob"; + name = fmt("%s %s...", type_name(obj->type), shortrev); + reporevlink(page, name, NULL, NULL, ctx.qry.head, fullrev, NULL); +} + +static struct string_list_item *lookup_path(struct string_list *list, + const char *path) +{ + struct string_list_item *item; + + while (path && path[0]) { + if ((item = string_list_lookup(list, path))) + return item; + if (!(path = strchr(path, '/'))) + break; + path++; + } + return NULL; +} + +void cgit_submodule_link(const char *class, char *path, const char *rev) +{ + struct string_list *list; + struct string_list_item *item; + char tail, *dir; + size_t len; + + len = 0; + tail = 0; + list = &ctx.repo->submodules; + item = lookup_path(list, path); + if (!item) { + len = strlen(path); + tail = path[len - 1]; + if (tail == '/') { + path[len - 1] = 0; + item = lookup_path(list, path); + } + } + if (item || ctx.repo->module_link) { + html("<a "); + if (class) + htmlf("class='%s' ", class); + html("href='"); + if (item) { + html_attrf(item->util, rev); + } else { + dir = strrchr(path, '/'); + if (dir) + dir++; + else + dir = path; + html_attrf(ctx.repo->module_link, dir, rev); + } + html("'>"); + html_txt(path); + html("</a>"); + } else { + html("<span"); + if (class) + htmlf(" class='%s'", class); + html(">"); + html_txt(path); + html("</span>"); + } + html_txtf(" @ %.7s", rev); + if (item && tail) + path[len - 1] = tail; +} + +const struct date_mode *cgit_date_mode(enum date_mode_type type) +{ + static struct date_mode mode; + mode.type = type; + mode.local = ctx.cfg.local_time; + return &mode; +} + +static void print_rel_date(time_t t, int tz, double value, + const char *class, const char *suffix) +{ + htmlf("<span class='%s' title='", class); + html_attr(show_date(t, tz, cgit_date_mode(DATE_DOTTIME))); + htmlf("'>%.0f %s</span>", value, suffix); +} + +void cgit_print_age(time_t t, int tz, time_t max_relative) +{ + time_t now, secs; + + if (!t) + return; + time(&now); + secs = now - t; + if (secs < 0) + secs = 0; + + if (secs > max_relative && max_relative >= 0) { + html("<span title='"); + html_attr(show_date(t, tz, cgit_date_mode(DATE_DOTTIME))); + html("'>"); + html_txt(show_date(t, tz, cgit_date_mode(DATE_SHORT))); + html("</span>"); + return; + } + + if (secs < TM_HOUR * 2) { + print_rel_date(t, tz, secs * 1.0 / TM_MIN, "age-mins", "min."); + return; + } + if (secs < TM_DAY * 2) { + print_rel_date(t, tz, secs * 1.0 / TM_HOUR, "age-hours", "hours"); + return; + } + if (secs < TM_WEEK * 2) { + print_rel_date(t, tz, secs * 1.0 / TM_DAY, "age-days", "days"); + return; + } + if (secs < TM_MONTH * 2) { + print_rel_date(t, tz, secs * 1.0 / TM_WEEK, "age-weeks", "weeks"); + return; + } + if (secs < TM_YEAR * 2) { + print_rel_date(t, tz, secs * 1.0 / TM_MONTH, "age-months", "months"); + return; + } + print_rel_date(t, tz, secs * 1.0 / TM_YEAR, "age-years", "years"); +} + +void cgit_print_http_headers(void) +{ + if (ctx.env.no_http && !strcmp(ctx.env.no_http, "1")) + return; + + if (ctx.page.status) + htmlf("Status: %d %s\n", ctx.page.status, ctx.page.statusmsg); + if (ctx.page.mimetype && ctx.page.charset) + htmlf("Content-Type: %s; charset=%s\n", ctx.page.mimetype, + ctx.page.charset); + else if (ctx.page.mimetype) + htmlf("Content-Type: %s\n", ctx.page.mimetype); + if (ctx.page.size) + htmlf("Content-Length: %zd\n", ctx.page.size); + if (ctx.page.filename) { + html("Content-Disposition: inline; filename=\""); + html_header_arg_in_quotes(ctx.page.filename); + html("\"\n"); + } + if (!ctx.env.authenticated) + html("Cache-Control: no-cache, no-store\n"); + htmlf("Last-Modified: %s\n", http_date(ctx.page.modified)); + htmlf("Expires: %s\n", http_date(ctx.page.expires)); + if (ctx.page.etag) + htmlf("ETag: \"%s\"\n", ctx.page.etag); + html("\n"); + if (ctx.env.request_method && !strcmp(ctx.env.request_method, "HEAD")) + exit(0); +} + +void cgit_redirect(const char *url, bool permanent) +{ + htmlf("Status: %d %s\n", permanent ? 301 : 302, permanent ? "Moved" : "Found"); + html("Location: "); + html_url_path(url); + html("\n\n"); +} + +static void print_rel_vcs_link(const char *url) +{ + html("<link rel='vcs-git' href='"); + html_attr(url); + html("' title='"); + html_attr(ctx.repo->name); + html(" Git repository'/>\n"); +} + +void cgit_print_docstart(void) +{ + char *host = cgit_hosturl(); + + if (ctx.cfg.embedded) { + if (ctx.cfg.header) + html_include(ctx.cfg.header); + return; + } + + html(cgit_doctype); + html("<html lang='en'>\n"); + html("<head>\n"); + html("<title>"); + html_txt(ctx.page.title); + html("</title>\n"); + htmlf("<meta name='generator' content='cgit %s'/>\n", cgit_version); + if (ctx.cfg.robots && *ctx.cfg.robots) + htmlf("<meta name='robots' content='%s'/>\n", ctx.cfg.robots); + html("<link rel='stylesheet' type='text/css' href='"); + html_attr(ctx.cfg.css); + html("'/>\n"); + if (ctx.cfg.favicon) { + html("<link rel='shortcut icon' href='"); + html_attr(ctx.cfg.favicon); + html("'/>\n"); + } + if (host && ctx.repo && ctx.qry.head) { + char *fileurl; + struct strbuf sb = STRBUF_INIT; + strbuf_addf(&sb, "h=%s", ctx.qry.head); + + html("<link rel='alternate' title='Atom feed' href='"); + html(cgit_httpscheme()); + html_attr(host); + fileurl = cgit_fileurl(ctx.repo->url, "atom", ctx.qry.vpath, + sb.buf); + html_attr(fileurl); + html("' type='application/atom+xml'/>\n"); + strbuf_release(&sb); + free(fileurl); + } + if (ctx.repo) + cgit_add_clone_urls(print_rel_vcs_link); + if (ctx.cfg.head_include) + html_include(ctx.cfg.head_include); + if (ctx.repo && ctx.repo->extra_head_content) + html(ctx.repo->extra_head_content); + html("</head>\n"); + html("<body>\n"); + if (ctx.cfg.header) + html_include(ctx.cfg.header); + free(host); +} + +void cgit_print_docend(void) +{ + html("</div> <!-- class=content -->\n"); + if (ctx.cfg.embedded) { + html("</div> <!-- id=cgit -->\n"); + if (ctx.cfg.footer) + html_include(ctx.cfg.footer); + return; + } + if (ctx.cfg.footer) + html_include(ctx.cfg.footer); + else { + htmlf("<div class='footer'>generated by <a href='https://git.zx2c4.com/cgit/about/'>cgit %s</a> " + "(<a href='https://git-scm.com/'>git %s</a>) at ", cgit_version, git_version_string); + html_txt(show_date(time(NULL), 0, cgit_date_mode(DATE_DOTTIME))); + html("</div>\n"); + } + html("</div> <!-- id=cgit -->\n"); + html("</body>\n</html>\n"); +} + +void cgit_print_error_page(int code, const char *msg, const char *fmt, ...) +{ + va_list ap; + ctx.page.expires = ctx.cfg.cache_dynamic_ttl; + ctx.page.status = code; + ctx.page.statusmsg = msg; + cgit_print_layout_start(); + va_start(ap, fmt); + cgit_vprint_error(fmt, ap); + va_end(ap); + cgit_print_layout_end(); +} + +void cgit_print_layout_start(void) +{ + cgit_print_http_headers(); + cgit_print_docstart(); + cgit_print_pageheader(); +} + +void cgit_print_layout_end(void) +{ + cgit_print_docend(); +} + +static void add_clone_urls(void (*fn)(const char *), char *txt, char *suffix) +{ + struct strbuf **url_list = strbuf_split_str(txt, ' ', 0); + int i; + + for (i = 0; url_list[i]; i++) { + strbuf_rtrim(url_list[i]); + if (url_list[i]->len == 0) + continue; + if (suffix && *suffix) + strbuf_addf(url_list[i], "/%s", suffix); + fn(url_list[i]->buf); + } + + strbuf_list_free(url_list); +} + +void cgit_add_clone_urls(void (*fn)(const char *)) +{ + if (ctx.repo->clone_url) + add_clone_urls(fn, expand_macros(ctx.repo->clone_url), NULL); + else if (ctx.cfg.clone_prefix) + add_clone_urls(fn, ctx.cfg.clone_prefix, ctx.repo->url); +} + +static int print_this_commit_option(void) +{ + struct object_id oid; + if (get_oid(ctx.qry.head, &oid)) + return 1; + html_option(oid_to_hex(&oid), "this commit", ctx.qry.head); + return 0; +} + +static int print_branch_option(const char *refname, const struct object_id *oid, + int flags, void *cb_data) +{ + char *name = (char *)refname; + html_option(name, name, ctx.qry.head); + return 0; +} + +void cgit_add_hidden_formfields(int incl_head, int incl_search, + const char *page) +{ + if (!ctx.cfg.virtual_root) { + struct strbuf url = STRBUF_INIT; + + strbuf_addf(&url, "%s/%s", ctx.qry.repo, page); + if (ctx.qry.vpath) + strbuf_addf(&url, "/%s", ctx.qry.vpath); + html_hidden("url", url.buf); + strbuf_release(&url); + } + + if (incl_head && ctx.qry.head && ctx.repo->defbranch && + strcmp(ctx.qry.head, ctx.repo->defbranch)) + html_hidden("h", ctx.qry.head); + + if (ctx.qry.oid) + html_hidden("id", ctx.qry.oid); + if (ctx.qry.oid2) + html_hidden("id2", ctx.qry.oid2); + if (ctx.qry.showmsg) + html_hidden("showmsg", "1"); + + if (incl_search) { + if (ctx.qry.grep) + html_hidden("qt", ctx.qry.grep); + if (ctx.qry.search) + html_hidden("q", ctx.qry.search); + } +} + +static const char *hc(const char *page) +{ + if (!ctx.qry.page) + return NULL; + + return strcmp(ctx.qry.page, page) ? NULL : "active"; +} + +static void cgit_print_path_crumbs(char *path) +{ + char *old_path = ctx.qry.path; + char *p = path, *q, *end = path + strlen(path); + int levels = 0; + + ctx.qry.path = NULL; + cgit_self_link("root", NULL, NULL); + ctx.qry.path = p = path; + while (p < end) { + if (!(q = strchr(p, '/')) || levels > 15) + q = end; + *q = '\0'; + html_txt("/"); + cgit_self_link(p, NULL, NULL); + if (q < end) + *q = '/'; + p = q + 1; + ++levels; + } + ctx.qry.path = old_path; +} + +static void print_header(void) +{ + char *logo = NULL, *logo_link = NULL; + + html("<table id='header'>\n"); + html("<tr>\n"); + + if (ctx.repo && ctx.repo->logo && *ctx.repo->logo) + logo = ctx.repo->logo; + else + logo = ctx.cfg.logo; + if (ctx.repo && ctx.repo->logo_link && *ctx.repo->logo_link) + logo_link = ctx.repo->logo_link; + else + logo_link = ctx.cfg.logo_link; + if (logo && *logo) { + html("<td class='logo' rowspan='2'><a href='"); + if (logo_link && *logo_link) + html_attr(logo_link); + else + html_attr(cgit_rooturl()); + html("'><img src='"); + html_attr(logo); + html("' alt='cgit logo'/></a></td>\n"); + } + + html("<td class='main'>"); + if (ctx.repo) { + cgit_summary_link(ctx.repo->name, ctx.repo->name, NULL, NULL); + if (ctx.env.authenticated) { + html("</td><td class='form'>"); + html("<form method='get'>\n"); + cgit_add_hidden_formfields(0, 1, ctx.qry.page); + html("<select name='h' onchange='this.form.submit();'>\n"); + print_this_commit_option(); + html("<optgroup label='branches'>"); + for_each_branch_ref(print_branch_option, ctx.qry.head); + if (ctx.repo->enable_remote_branches) + for_each_remote_ref(print_branch_option, ctx.qry.head); + html("</optgroup>"); + html("</select> "); + html("<input type='submit' value='switch'/>"); + html("</form>"); + } + } else + html_txt(ctx.cfg.root_title); + html("</td></tr>\n"); + + html("<tr><td class='sub'>"); + if (ctx.repo) { + html_txt(ctx.repo->desc); + html("</td><td class='sub right'>"); + html_txt(ctx.repo->owner); + } else { + if (ctx.cfg.root_desc) + html_txt(ctx.cfg.root_desc); + } + html("</td></tr></table>\n"); +} + +void cgit_print_pageheader(void) +{ + html("<div id='cgit'>"); + if (!ctx.env.authenticated || !ctx.cfg.noheader) + print_header(); + + html("<table class='tabs'><tr><td>\n"); + if (ctx.env.authenticated && ctx.repo) { + if (ctx.repo->readme.nr) { + cgit_about_link("about", NULL, hc("about"), ctx.qry.head, + ctx.qry.oid, ctx.qry.vpath); + } + cgit_summary_link("summary", NULL, hc("summary"), + ctx.qry.head); + cgit_refs_link("refs", NULL, hc("refs"), ctx.qry.head, + ctx.qry.oid, NULL); + cgit_log_link("log", NULL, hc("log"), ctx.qry.head, + NULL, ctx.qry.vpath, 0, NULL, NULL, + ctx.qry.showmsg, ctx.qry.follow); + if (ctx.qry.page && !strcmp(ctx.qry.page, "blame")) + cgit_blame_link("blame", NULL, hc("blame"), ctx.qry.head, + ctx.qry.oid, ctx.qry.vpath); + else + cgit_tree_link("tree", NULL, hc("tree"), ctx.qry.head, + ctx.qry.oid, ctx.qry.vpath); + cgit_commit_link("commit", NULL, hc("commit"), + ctx.qry.head, ctx.qry.oid, ctx.qry.vpath); + cgit_diff_link("diff", NULL, hc("diff"), ctx.qry.head, + ctx.qry.oid, ctx.qry.oid2, ctx.qry.vpath); + if (ctx.repo->max_stats) + cgit_stats_link("stats", NULL, hc("stats"), + ctx.qry.head, ctx.qry.vpath); + if (ctx.repo->homepage) { + html("<a href='"); + html_attr(ctx.repo->homepage); + html("'>homepage</a>"); + } + html("</td><td class='form'>"); + html("<form class='right' method='get' action='"); + if (ctx.cfg.virtual_root) { + char *fileurl = cgit_fileurl(ctx.qry.repo, "log", + ctx.qry.vpath, NULL); + html_url_path(fileurl); + free(fileurl); + } + html("'>\n"); + cgit_add_hidden_formfields(1, 0, "log"); + html("<select name='qt'>\n"); + html_option("grep", "log msg", ctx.qry.grep); + html_option("author", "author", ctx.qry.grep); + html_option("committer", "committer", ctx.qry.grep); + html_option("range", "range", ctx.qry.grep); + html("</select>\n"); + html("<input class='txt' type='search' size='10' name='q' value='"); + html_attr(ctx.qry.search); + html("'/>\n"); + html("<input type='submit' value='search'/>\n"); + html("</form>\n"); + } else if (ctx.env.authenticated) { + char *currenturl = cgit_currenturl(); + site_link(NULL, "index", NULL, hc("repolist"), NULL, NULL, 0, 1); + if (ctx.cfg.root_readme) + site_link("about", "about", NULL, hc("about"), + NULL, NULL, 0, 1); + html("</td><td class='form'>"); + html("<form method='get' action='"); + html_attr(currenturl); + html("'>\n"); + html("<input type='search' name='q' size='10' value='"); + html_attr(ctx.qry.search); + html("'/>\n"); + html("<input type='submit' value='search'/>\n"); + html("</form>"); + free(currenturl); + } + html("</td></tr></table>\n"); + if (ctx.env.authenticated && ctx.repo && ctx.qry.vpath) { + html("<div class='path'>"); + html("path: "); + cgit_print_path_crumbs(ctx.qry.vpath); + if (ctx.cfg.enable_follow_links && !strcmp(ctx.qry.page, "log")) { + html(" ("); + ctx.qry.follow = !ctx.qry.follow; + cgit_self_link(ctx.qry.follow ? "follow" : "unfollow", + NULL, NULL); + ctx.qry.follow = !ctx.qry.follow; + html(")"); + } + html("</div>"); + } + html("<div class='content'>"); +} + +void cgit_print_filemode(unsigned short mode) +{ + if (S_ISDIR(mode)) + html("d"); + else if (S_ISLNK(mode)) + html("l"); + else if (S_ISGITLINK(mode)) + html("m"); + else + html("-"); + html_fileperm(mode >> 6); + html_fileperm(mode >> 3); + html_fileperm(mode); +} + +void cgit_compose_snapshot_prefix(struct strbuf *filename, const char *base, + const char *ref) +{ + struct object_id oid; + + /* + * Prettify snapshot names by stripping leading "v" or "V" if the tag + * name starts with {v,V}[0-9] and the prettify mapping is injective, + * i.e. each stripped tag can be inverted without ambiguities. + */ + if (get_oid(fmt("refs/tags/%s", ref), &oid) == 0 && + (ref[0] == 'v' || ref[0] == 'V') && isdigit(ref[1]) && + ((get_oid(fmt("refs/tags/%s", ref + 1), &oid) == 0) + + (get_oid(fmt("refs/tags/v%s", ref + 1), &oid) == 0) + + (get_oid(fmt("refs/tags/V%s", ref + 1), &oid) == 0) == 1)) + ref++; + + strbuf_addf(filename, "%s-%s", base, ref); +} + +void cgit_print_snapshot_links(const struct cgit_repo *repo, const char *ref, + const char *separator) +{ + const struct cgit_snapshot_format *f; + struct strbuf filename = STRBUF_INIT; + const char *basename; + size_t prefixlen; + + basename = cgit_snapshot_prefix(repo); + if (starts_with(ref, basename)) + strbuf_addstr(&filename, ref); + else + cgit_compose_snapshot_prefix(&filename, basename, ref); + + prefixlen = filename.len; + for (f = cgit_snapshot_formats; f->suffix; f++) { + if (!(repo->snapshots & cgit_snapshot_format_bit(f))) + continue; + strbuf_setlen(&filename, prefixlen); + strbuf_addstr(&filename, f->suffix); + cgit_snapshot_link(filename.buf, NULL, NULL, NULL, NULL, + filename.buf); + if (cgit_snapshot_get_sig(ref, f)) { + strbuf_addstr(&filename, ".asc"); + html(" ("); + cgit_snapshot_link("sig", NULL, NULL, NULL, NULL, + filename.buf); + html(")"); + } else if (starts_with(f->suffix, ".tar") && cgit_snapshot_get_sig(ref, &cgit_snapshot_formats[0])) { + strbuf_setlen(&filename, strlen(filename.buf) - strlen(f->suffix)); + strbuf_addstr(&filename, ".tar.asc"); + html(" ("); + cgit_snapshot_link("sig", NULL, NULL, NULL, NULL, + filename.buf); + html(")"); + } + html(separator); + } + strbuf_release(&filename); +} + +void cgit_set_title_from_path(const char *path) +{ + struct strbuf sb = STRBUF_INIT; + const char *slash, *last_slash; + + if (!path) + return; + + for (last_slash = path + strlen(path); (slash = memrchr(path, '/', last_slash - path)) != NULL; last_slash = slash) { + strbuf_add(&sb, slash + 1, last_slash - slash - 1); + strbuf_addstr(&sb, " \xc2\xab "); + } + strbuf_add(&sb, path, last_slash - path); + strbuf_addf(&sb, " - %s", ctx.page.title); + ctx.page.title = strbuf_detach(&sb, NULL); +} diff --git a/third_party/cgit/ui-shared.h b/third_party/cgit/ui-shared.h new file mode 100644 index 000000000000..6964873a6394 --- /dev/null +++ b/third_party/cgit/ui-shared.h @@ -0,0 +1,87 @@ +#ifndef UI_SHARED_H +#define UI_SHARED_H + +extern const char *cgit_httpscheme(void); +extern char *cgit_hosturl(void); +extern const char *cgit_rooturl(void); +extern char *cgit_currenturl(void); +extern char *cgit_currentfullurl(void); +extern const char *cgit_loginurl(void); +extern char *cgit_repourl(const char *reponame); +extern char *cgit_fileurl(const char *reponame, const char *pagename, + const char *filename, const char *query); +extern char *cgit_pageurl(const char *reponame, const char *pagename, + const char *query); + +extern void cgit_add_clone_urls(void (*fn)(const char *)); + +extern void cgit_index_link(const char *name, const char *title, + const char *class, const char *pattern, const char *sort, int ofs, int always_root); +extern void cgit_summary_link(const char *name, const char *title, + const char *class, const char *head); +extern void cgit_tag_link(const char *name, const char *title, + const char *class, const char *tag); +extern void cgit_tree_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *path); +extern void cgit_plain_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *path); +extern void cgit_blame_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *path); +extern void cgit_log_link(const char *name, const char *title, + const char *class, const char *head, const char *rev, + const char *path, int ofs, const char *grep, + const char *pattern, int showmsg, int follow); +extern void cgit_commit_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *path); +extern void cgit_patch_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *path); +extern void cgit_refs_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *path); +extern void cgit_snapshot_link(const char *name, const char *title, + const char *class, const char *head, + const char *rev, const char *archivename); +extern void cgit_diff_link(const char *name, const char *title, + const char *class, const char *head, + const char *new_rev, const char *old_rev, + const char *path); +extern void cgit_stats_link(const char *name, const char *title, + const char *class, const char *head, + const char *path); +extern void cgit_object_link(struct object *obj); + +extern void cgit_submodule_link(const char *class, char *path, + const char *rev); + +extern void cgit_print_layout_start(void); +extern void cgit_print_layout_end(void); + +__attribute__((format (printf,1,2))) +extern void cgit_print_error(const char *fmt, ...); +__attribute__((format (printf,1,0))) +extern void cgit_vprint_error(const char *fmt, va_list ap); +extern const struct date_mode *cgit_date_mode(enum date_mode_type type); +extern void cgit_print_age(time_t t, int tz, time_t max_relative); +extern void cgit_print_http_headers(void); +extern void cgit_redirect(const char *url, bool permanent); +extern void cgit_print_docstart(void); +extern void cgit_print_docend(void); +__attribute__((format (printf,3,4))) +extern void cgit_print_error_page(int code, const char *msg, const char *fmt, ...); +extern void cgit_print_pageheader(void); +extern void cgit_print_filemode(unsigned short mode); +extern void cgit_compose_snapshot_prefix(struct strbuf *filename, + const char *base, const char *ref); +extern void cgit_print_snapshot_links(const struct cgit_repo *repo, + const char *ref, const char *separator); +extern const char *cgit_snapshot_prefix(const struct cgit_repo *repo); +extern void cgit_add_hidden_formfields(int incl_head, int incl_search, + const char *page); + +extern void cgit_set_title_from_path(const char *path); +#endif /* UI_SHARED_H */ diff --git a/third_party/cgit/ui-snapshot.c b/third_party/cgit/ui-snapshot.c new file mode 100644 index 000000000000..18361a655384 --- /dev/null +++ b/third_party/cgit/ui-snapshot.c @@ -0,0 +1,316 @@ +/* ui-snapshot.c: generate snapshot of a commit + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-snapshot.h" +#include "html.h" +#include "ui-shared.h" + +static int write_archive_type(const char *format, const char *hex, const char *prefix) +{ + struct strvec argv = STRVEC_INIT; + const char **nargv; + int result; + strvec_push(&argv, "snapshot"); + strvec_push(&argv, format); + if (prefix) { + struct strbuf buf = STRBUF_INIT; + strbuf_addstr(&buf, prefix); + strbuf_addch(&buf, '/'); + strvec_push(&argv, "--prefix"); + strvec_push(&argv, buf.buf); + strbuf_release(&buf); + } + strvec_push(&argv, hex); + /* + * Now we need to copy the pointers to arguments into a new + * structure because write_archive will rearrange its arguments + * which may result in duplicated/missing entries causing leaks + * or double-frees in strvec_clear. + */ + nargv = xmalloc(sizeof(char *) * (argv.nr + 1)); + /* strvec guarantees a trailing NULL entry. */ + memcpy(nargv, argv.v, sizeof(char *) * (argv.nr + 1)); + + result = write_archive(argv.nr, nargv, NULL, the_repository, NULL, 0); + strvec_clear(&argv); + free(nargv); + return result; +} + +static int write_tar_archive(const char *hex, const char *prefix) +{ + return write_archive_type("--format=tar", hex, prefix); +} + +static int write_zip_archive(const char *hex, const char *prefix) +{ + return write_archive_type("--format=zip", hex, prefix); +} + +static int write_compressed_tar_archive(const char *hex, + const char *prefix, + char *filter_argv[]) +{ + int rv; + struct cgit_exec_filter f; + cgit_exec_filter_init(&f, filter_argv[0], filter_argv); + + cgit_open_filter(&f.base); + rv = write_tar_archive(hex, prefix); + cgit_close_filter(&f.base); + return rv; +} + +static int write_tar_gzip_archive(const char *hex, const char *prefix) +{ + char *argv[] = { "gzip", "-n", NULL }; + return write_compressed_tar_archive(hex, prefix, argv); +} + +static int write_tar_bzip2_archive(const char *hex, const char *prefix) +{ + char *argv[] = { "bzip2", NULL }; + return write_compressed_tar_archive(hex, prefix, argv); +} + +static int write_tar_lzip_archive(const char *hex, const char *prefix) +{ + char *argv[] = { "lzip", NULL }; + return write_compressed_tar_archive(hex, prefix, argv); +} + +static int write_tar_xz_archive(const char *hex, const char *prefix) +{ + char *argv[] = { "xz", NULL }; + return write_compressed_tar_archive(hex, prefix, argv); +} + +static int write_tar_zstd_archive(const char *hex, const char *prefix) +{ + char *argv[] = { "zstd", "-T0", NULL }; + return write_compressed_tar_archive(hex, prefix, argv); +} + +const struct cgit_snapshot_format cgit_snapshot_formats[] = { + /* .tar must remain the 0 index */ + { ".tar", "application/x-tar", write_tar_archive }, + { ".tar.gz", "application/x-gzip", write_tar_gzip_archive }, + { ".tar.bz2", "application/x-bzip2", write_tar_bzip2_archive }, + { ".tar.lz", "application/x-lzip", write_tar_lzip_archive }, + { ".tar.xz", "application/x-xz", write_tar_xz_archive }, + { ".tar.zst", "application/x-zstd", write_tar_zstd_archive }, + { ".zip", "application/x-zip", write_zip_archive }, + { NULL } +}; + +static struct notes_tree snapshot_sig_notes[ARRAY_SIZE(cgit_snapshot_formats)]; + +const struct object_id *cgit_snapshot_get_sig(const char *ref, + const struct cgit_snapshot_format *f) +{ + struct notes_tree *tree; + struct object_id oid; + + if (get_oid(ref, &oid)) + return NULL; + + tree = &snapshot_sig_notes[f - &cgit_snapshot_formats[0]]; + if (!tree->initialized) { + struct strbuf notes_ref = STRBUF_INIT; + + strbuf_addf(¬es_ref, "refs/notes/signatures/%s", + f->suffix + 1); + + init_notes(tree, notes_ref.buf, combine_notes_ignore, 0); + strbuf_release(¬es_ref); + } + + return get_note(tree, &oid); +} + +static const struct cgit_snapshot_format *get_format(const char *filename) +{ + const struct cgit_snapshot_format *fmt; + + for (fmt = cgit_snapshot_formats; fmt->suffix; fmt++) { + if (ends_with(filename, fmt->suffix)) + return fmt; + } + return NULL; +} + +const unsigned cgit_snapshot_format_bit(const struct cgit_snapshot_format *f) +{ + return BIT(f - &cgit_snapshot_formats[0]); +} + +static int make_snapshot(const struct cgit_snapshot_format *format, + const char *hex, const char *prefix, + const char *filename) +{ + struct object_id oid; + + if (get_oid(hex, &oid)) { + cgit_print_error_page(404, "Not found", + "Bad object id: %s", hex); + return 1; + } + if (!lookup_commit_reference(the_repository, &oid)) { + cgit_print_error_page(400, "Bad request", + "Not a commit reference: %s", hex); + return 1; + } + ctx.page.etag = oid_to_hex(&oid); + ctx.page.mimetype = xstrdup(format->mimetype); + ctx.page.filename = xstrdup(filename); + cgit_print_http_headers(); + init_archivers(); + format->write_func(hex, prefix); + return 0; +} + +static int write_sig(const struct cgit_snapshot_format *format, + const char *hex, const char *archive, + const char *filename) +{ + const struct object_id *note = cgit_snapshot_get_sig(hex, format); + enum object_type type; + unsigned long size; + char *buf; + + if (!note) { + cgit_print_error_page(404, "Not found", + "No signature for %s", archive); + return 0; + } + + buf = read_object_file(note, &type, &size); + if (!buf) { + cgit_print_error_page(404, "Not found", "Not found"); + return 0; + } + + html("X-Content-Type-Options: nosniff\n"); + html("Content-Security-Policy: default-src 'none'\n"); + ctx.page.etag = oid_to_hex(note); + ctx.page.mimetype = xstrdup("application/pgp-signature"); + ctx.page.filename = xstrdup(filename); + cgit_print_http_headers(); + + html_raw(buf, size); + free(buf); + return 0; +} + +/* Try to guess the requested revision from the requested snapshot name. + * First the format extension is stripped, e.g. "cgit-0.7.2.tar.gz" become + * "cgit-0.7.2". If this is a valid commit object name we've got a winner. + * Otherwise, if the snapshot name has a prefix matching the result from + * repo_basename(), we strip the basename and any following '-' and '_' + * characters ("cgit-0.7.2" -> "0.7.2") and check the resulting name once + * more. If this still isn't a valid commit object name, we check if pre- + * pending a 'v' or a 'V' to the remaining snapshot name ("0.7.2" -> + * "v0.7.2") gives us something valid. + */ +static const char *get_ref_from_filename(const struct cgit_repo *repo, + const char *filename, + const struct cgit_snapshot_format *format) +{ + const char *reponame; + struct object_id oid; + struct strbuf snapshot = STRBUF_INIT; + int result = 1; + + strbuf_addstr(&snapshot, filename); + strbuf_setlen(&snapshot, snapshot.len - strlen(format->suffix)); + + if (get_oid(snapshot.buf, &oid) == 0) + goto out; + + reponame = cgit_snapshot_prefix(repo); + if (starts_with(snapshot.buf, reponame)) { + const char *new_start = snapshot.buf; + new_start += strlen(reponame); + while (new_start && (*new_start == '-' || *new_start == '_')) + new_start++; + strbuf_splice(&snapshot, 0, new_start - snapshot.buf, "", 0); + } + + if (get_oid(snapshot.buf, &oid) == 0) + goto out; + + strbuf_insert(&snapshot, 0, "v", 1); + if (get_oid(snapshot.buf, &oid) == 0) + goto out; + + strbuf_splice(&snapshot, 0, 1, "V", 1); + if (get_oid(snapshot.buf, &oid) == 0) + goto out; + + result = 0; + strbuf_release(&snapshot); + +out: + return result ? strbuf_detach(&snapshot, NULL) : NULL; +} + +void cgit_print_snapshot(const char *head, const char *hex, + const char *filename, int dwim) +{ + const struct cgit_snapshot_format* f; + const char *sig_filename = NULL; + char *adj_filename = NULL; + char *prefix = NULL; + + if (!filename) { + cgit_print_error_page(400, "Bad request", + "No snapshot name specified"); + return; + } + + if (ends_with(filename, ".asc")) { + sig_filename = filename; + + /* Strip ".asc" from filename for common format processing */ + adj_filename = xstrdup(filename); + adj_filename[strlen(adj_filename) - 4] = '\0'; + filename = adj_filename; + } + + f = get_format(filename); + if (!f || (!sig_filename && !(ctx.repo->snapshots & cgit_snapshot_format_bit(f)))) { + cgit_print_error_page(400, "Bad request", + "Unsupported snapshot format: %s", filename); + return; + } + + if (!hex && dwim) { + hex = get_ref_from_filename(ctx.repo, filename, f); + if (hex == NULL) { + cgit_print_error_page(404, "Not found", "Not found"); + return; + } + prefix = xstrdup(filename); + prefix[strlen(filename) - strlen(f->suffix)] = '\0'; + } + + if (!hex) + hex = head; + + if (!prefix) + prefix = xstrdup(cgit_snapshot_prefix(ctx.repo)); + + if (sig_filename) + write_sig(f, hex, filename, sig_filename); + else + make_snapshot(f, hex, prefix, filename); + + free(prefix); + free(adj_filename); +} diff --git a/third_party/cgit/ui-snapshot.h b/third_party/cgit/ui-snapshot.h new file mode 100644 index 000000000000..a8deec3697b4 --- /dev/null +++ b/third_party/cgit/ui-snapshot.h @@ -0,0 +1,7 @@ +#ifndef UI_SNAPSHOT_H +#define UI_SNAPSHOT_H + +extern void cgit_print_snapshot(const char *head, const char *hex, + const char *filename, int dwim); + +#endif /* UI_SNAPSHOT_H */ diff --git a/third_party/cgit/ui-ssdiff.c b/third_party/cgit/ui-ssdiff.c new file mode 100644 index 000000000000..af8bc9e0659b --- /dev/null +++ b/third_party/cgit/ui-ssdiff.c @@ -0,0 +1,420 @@ +#include "cgit.h" +#include "ui-ssdiff.h" +#include "html.h" +#include "ui-shared.h" +#include "ui-diff.h" + +extern int use_ssdiff; + +static int current_old_line, current_new_line; +static int **L = NULL; + +struct deferred_lines { + int line_no; + char *line; + struct deferred_lines *next; +}; + +static struct deferred_lines *deferred_old, *deferred_old_last; +static struct deferred_lines *deferred_new, *deferred_new_last; + +static void create_or_reset_lcs_table(void) +{ + int i; + + if (L != NULL) { + memset(*L, 0, sizeof(int) * MAX_SSDIFF_SIZE); + return; + } + + // xcalloc will die if we ran out of memory; + // not very helpful for debugging + L = (int**)xcalloc(MAX_SSDIFF_M, sizeof(int *)); + *L = (int*)xcalloc(MAX_SSDIFF_SIZE, sizeof(int)); + + for (i = 1; i < MAX_SSDIFF_M; i++) { + L[i] = *L + i * MAX_SSDIFF_N; + } +} + +static char *longest_common_subsequence(char *A, char *B) +{ + int i, j, ri; + int m = strlen(A); + int n = strlen(B); + int tmp1, tmp2; + int lcs_length; + char *result; + + // We bail if the lines are too long + if (m >= MAX_SSDIFF_M || n >= MAX_SSDIFF_N) + return NULL; + + create_or_reset_lcs_table(); + + for (i = m; i >= 0; i--) { + for (j = n; j >= 0; j--) { + if (A[i] == '\0' || B[j] == '\0') { + L[i][j] = 0; + } else if (A[i] == B[j]) { + L[i][j] = 1 + L[i + 1][j + 1]; + } else { + tmp1 = L[i + 1][j]; + tmp2 = L[i][j + 1]; + L[i][j] = (tmp1 > tmp2 ? tmp1 : tmp2); + } + } + } + + lcs_length = L[0][0]; + result = xmalloc(lcs_length + 2); + memset(result, 0, sizeof(*result) * (lcs_length + 2)); + + ri = 0; + i = 0; + j = 0; + while (i < m && j < n) { + if (A[i] == B[j]) { + result[ri] = A[i]; + ri += 1; + i += 1; + j += 1; + } else if (L[i + 1][j] >= L[i][j + 1]) { + i += 1; + } else { + j += 1; + } + } + + return result; +} + +static int line_from_hunk(char *line, char type) +{ + char *buf1, *buf2; + int len, res; + + buf1 = strchr(line, type); + if (buf1 == NULL) + return 0; + buf1 += 1; + buf2 = strchr(buf1, ','); + if (buf2 == NULL) + return 0; + len = buf2 - buf1; + buf2 = xmalloc(len + 1); + strlcpy(buf2, buf1, len + 1); + res = atoi(buf2); + free(buf2); + return res; +} + +static char *replace_tabs(char *line) +{ + char *prev_buf = line; + char *cur_buf; + size_t linelen = strlen(line); + int n_tabs = 0; + int i; + char *result; + size_t result_len; + + if (linelen == 0) { + result = xmalloc(1); + result[0] = '\0'; + return result; + } + + for (i = 0; i < linelen; i++) { + if (line[i] == '\t') + n_tabs += 1; + } + result_len = linelen + n_tabs * 8; + result = xmalloc(result_len + 1); + result[0] = '\0'; + + for (;;) { + cur_buf = strchr(prev_buf, '\t'); + if (!cur_buf) { + linelen = strlen(result); + strlcpy(&result[linelen], prev_buf, result_len - linelen + 1); + break; + } else { + linelen = strlen(result); + strlcpy(&result[linelen], prev_buf, cur_buf - prev_buf + 1); + linelen = strlen(result); + memset(&result[linelen], ' ', 8 - (linelen % 8)); + result[linelen + 8 - (linelen % 8)] = '\0'; + } + prev_buf = cur_buf + 1; + } + return result; +} + +static int calc_deferred_lines(struct deferred_lines *start) +{ + struct deferred_lines *item = start; + int result = 0; + while (item) { + result += 1; + item = item->next; + } + return result; +} + +static void deferred_old_add(char *line, int line_no) +{ + struct deferred_lines *item = xmalloc(sizeof(struct deferred_lines)); + item->line = xstrdup(line); + item->line_no = line_no; + item->next = NULL; + if (deferred_old) { + deferred_old_last->next = item; + deferred_old_last = item; + } else { + deferred_old = deferred_old_last = item; + } +} + +static void deferred_new_add(char *line, int line_no) +{ + struct deferred_lines *item = xmalloc(sizeof(struct deferred_lines)); + item->line = xstrdup(line); + item->line_no = line_no; + item->next = NULL; + if (deferred_new) { + deferred_new_last->next = item; + deferred_new_last = item; + } else { + deferred_new = deferred_new_last = item; + } +} + +static void print_part_with_lcs(char *class, char *line, char *lcs) +{ + int line_len = strlen(line); + int i, j; + char c[2] = " "; + int same = 1; + + j = 0; + for (i = 0; i < line_len; i++) { + c[0] = line[i]; + if (same) { + if (line[i] == lcs[j]) + j += 1; + else { + same = 0; + htmlf("<span class='%s'>", class); + } + } else if (line[i] == lcs[j]) { + same = 1; + html("</span>"); + j += 1; + } + html_txt(c); + } + if (!same) + html("</span>"); +} + +static void print_ssdiff_line(char *class, + int old_line_no, + char *old_line, + int new_line_no, + char *new_line, int individual_chars) +{ + char *lcs = NULL; + + if (old_line) + old_line = replace_tabs(old_line + 1); + if (new_line) + new_line = replace_tabs(new_line + 1); + if (individual_chars && old_line && new_line) + lcs = longest_common_subsequence(old_line, new_line); + html("<tr>\n"); + if (old_line_no > 0) { + struct diff_filespec *old_file = cgit_get_current_old_file(); + char *lineno_str = fmt("n%d", old_line_no); + char *id_str = fmt("id=%s#%s", is_null_oid(&old_file->oid)?"HEAD":oid_to_hex(old_rev_oid), lineno_str); + char *fileurl = cgit_fileurl(ctx.repo->url, "tree", old_file->path, id_str); + html("<td class='lineno'><a href='"); + html(fileurl); + htmlf("'>%s</a>", lineno_str + 1); + html("</td>"); + htmlf("<td class='%s'>", class); + free(fileurl); + } else if (old_line) + htmlf("<td class='lineno'></td><td class='%s'>", class); + else + htmlf("<td class='lineno'></td><td class='%s_dark'>", class); + if (old_line) { + if (lcs) + print_part_with_lcs("del", old_line, lcs); + else + html_txt(old_line); + } + + html("</td>\n"); + if (new_line_no > 0) { + struct diff_filespec *new_file = cgit_get_current_new_file(); + char *lineno_str = fmt("n%d", new_line_no); + char *id_str = fmt("id=%s#%s", is_null_oid(&new_file->oid)?"HEAD":oid_to_hex(new_rev_oid), lineno_str); + char *fileurl = cgit_fileurl(ctx.repo->url, "tree", new_file->path, id_str); + html("<td class='lineno'><a href='"); + html(fileurl); + htmlf("'>%s</a>", lineno_str + 1); + html("</td>"); + htmlf("<td class='%s'>", class); + free(fileurl); + } else if (new_line) + htmlf("<td class='lineno'></td><td class='%s'>", class); + else + htmlf("<td class='lineno'></td><td class='%s_dark'>", class); + if (new_line) { + if (lcs) + print_part_with_lcs("add", new_line, lcs); + else + html_txt(new_line); + } + + html("</td></tr>"); + if (lcs) + free(lcs); + if (new_line) + free(new_line); + if (old_line) + free(old_line); +} + +static void print_deferred_old_lines(void) +{ + struct deferred_lines *iter_old, *tmp; + iter_old = deferred_old; + while (iter_old) { + print_ssdiff_line("del", iter_old->line_no, + iter_old->line, -1, NULL, 0); + tmp = iter_old->next; + free(iter_old); + iter_old = tmp; + } +} + +static void print_deferred_new_lines(void) +{ + struct deferred_lines *iter_new, *tmp; + iter_new = deferred_new; + while (iter_new) { + print_ssdiff_line("add", -1, NULL, + iter_new->line_no, iter_new->line, 0); + tmp = iter_new->next; + free(iter_new); + iter_new = tmp; + } +} + +static void print_deferred_changed_lines(void) +{ + struct deferred_lines *iter_old, *iter_new, *tmp; + int n_old_lines = calc_deferred_lines(deferred_old); + int n_new_lines = calc_deferred_lines(deferred_new); + int individual_chars = (n_old_lines == n_new_lines ? 1 : 0); + + iter_old = deferred_old; + iter_new = deferred_new; + while (iter_old || iter_new) { + if (iter_old && iter_new) + print_ssdiff_line("changed", iter_old->line_no, + iter_old->line, + iter_new->line_no, iter_new->line, + individual_chars); + else if (iter_old) + print_ssdiff_line("changed", iter_old->line_no, + iter_old->line, -1, NULL, 0); + else if (iter_new) + print_ssdiff_line("changed", -1, NULL, + iter_new->line_no, iter_new->line, 0); + if (iter_old) { + tmp = iter_old->next; + free(iter_old); + iter_old = tmp; + } + + if (iter_new) { + tmp = iter_new->next; + free(iter_new); + iter_new = tmp; + } + } +} + +void cgit_ssdiff_print_deferred_lines(void) +{ + if (!deferred_old && !deferred_new) + return; + if (deferred_old && !deferred_new) + print_deferred_old_lines(); + else if (!deferred_old && deferred_new) + print_deferred_new_lines(); + else + print_deferred_changed_lines(); + deferred_old = deferred_old_last = NULL; + deferred_new = deferred_new_last = NULL; +} + +/* + * print a single line returned from xdiff + */ +void cgit_ssdiff_line_cb(char *line, int len) +{ + char c = line[len - 1]; + line[len - 1] = '\0'; + if (line[0] == '@') { + current_old_line = line_from_hunk(line, '-'); + current_new_line = line_from_hunk(line, '+'); + } + + if (line[0] == ' ') { + if (deferred_old || deferred_new) + cgit_ssdiff_print_deferred_lines(); + print_ssdiff_line("ctx", current_old_line, line, + current_new_line, line, 0); + current_old_line += 1; + current_new_line += 1; + } else if (line[0] == '+') { + deferred_new_add(line, current_new_line); + current_new_line += 1; + } else if (line[0] == '-') { + deferred_old_add(line, current_old_line); + current_old_line += 1; + } else if (line[0] == '@') { + html("<tr><td colspan='4' class='hunk'>"); + html_txt(line); + html("</td></tr>"); + } else { + html("<tr><td colspan='4' class='ctx'>"); + html_txt(line); + html("</td></tr>"); + } + line[len - 1] = c; +} + +void cgit_ssdiff_header_begin(void) +{ + current_old_line = -1; + current_new_line = -1; + html("<tr><td class='space' colspan='4'><div></div></td></tr>"); + html("<tr><td class='head' colspan='4'>"); +} + +void cgit_ssdiff_header_end(void) +{ + html("</td></tr>"); +} + +void cgit_ssdiff_footer(void) +{ + if (deferred_old || deferred_new) + cgit_ssdiff_print_deferred_lines(); + html("<tr><td class='foot' colspan='4'></td></tr>"); +} diff --git a/third_party/cgit/ui-ssdiff.h b/third_party/cgit/ui-ssdiff.h new file mode 100644 index 000000000000..11f2714407c7 --- /dev/null +++ b/third_party/cgit/ui-ssdiff.h @@ -0,0 +1,25 @@ +#ifndef UI_SSDIFF_H +#define UI_SSDIFF_H + +/* + * ssdiff line limits + */ +#ifndef MAX_SSDIFF_M +#define MAX_SSDIFF_M 128 +#endif + +#ifndef MAX_SSDIFF_N +#define MAX_SSDIFF_N 128 +#endif +#define MAX_SSDIFF_SIZE ((MAX_SSDIFF_M) * (MAX_SSDIFF_N)) + +extern void cgit_ssdiff_print_deferred_lines(void); + +extern void cgit_ssdiff_line_cb(char *line, int len); + +extern void cgit_ssdiff_header_begin(void); +extern void cgit_ssdiff_header_end(void); + +extern void cgit_ssdiff_footer(void); + +#endif /* UI_SSDIFF_H */ diff --git a/third_party/cgit/ui-stats.c b/third_party/cgit/ui-stats.c new file mode 100644 index 000000000000..09b3625e7e0c --- /dev/null +++ b/third_party/cgit/ui-stats.c @@ -0,0 +1,426 @@ +#include "cgit.h" +#include "ui-stats.h" +#include "html.h" +#include "ui-shared.h" + +struct authorstat { + long total; + struct string_list list; +}; + +#define DAY_SECS (60 * 60 * 24) +#define WEEK_SECS (DAY_SECS * 7) + +static void trunc_week(struct tm *tm) +{ + time_t t = timegm(tm); + t -= ((tm->tm_wday + 6) % 7) * DAY_SECS; + gmtime_r(&t, tm); +} + +static void dec_week(struct tm *tm) +{ + time_t t = timegm(tm); + t -= WEEK_SECS; + gmtime_r(&t, tm); +} + +static void inc_week(struct tm *tm) +{ + time_t t = timegm(tm); + t += WEEK_SECS; + gmtime_r(&t, tm); +} + +static char *pretty_week(struct tm *tm) +{ + static char buf[10]; + + strftime(buf, sizeof(buf), "W%V %G", tm); + return buf; +} + +static void trunc_month(struct tm *tm) +{ + tm->tm_mday = 1; +} + +static void dec_month(struct tm *tm) +{ + tm->tm_mon--; + if (tm->tm_mon < 0) { + tm->tm_year--; + tm->tm_mon = 11; + } +} + +static void inc_month(struct tm *tm) +{ + tm->tm_mon++; + if (tm->tm_mon > 11) { + tm->tm_year++; + tm->tm_mon = 0; + } +} + +static char *pretty_month(struct tm *tm) +{ + static const char *months[] = { + "Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" + }; + return fmt("%s %d", months[tm->tm_mon], tm->tm_year + 1900); +} + +static void trunc_quarter(struct tm *tm) +{ + trunc_month(tm); + while (tm->tm_mon % 3 != 0) + dec_month(tm); +} + +static void dec_quarter(struct tm *tm) +{ + dec_month(tm); + dec_month(tm); + dec_month(tm); +} + +static void inc_quarter(struct tm *tm) +{ + inc_month(tm); + inc_month(tm); + inc_month(tm); +} + +static char *pretty_quarter(struct tm *tm) +{ + return fmt("Q%d %d", tm->tm_mon / 3 + 1, tm->tm_year + 1900); +} + +static void trunc_year(struct tm *tm) +{ + trunc_month(tm); + tm->tm_mon = 0; +} + +static void dec_year(struct tm *tm) +{ + tm->tm_year--; +} + +static void inc_year(struct tm *tm) +{ + tm->tm_year++; +} + +static char *pretty_year(struct tm *tm) +{ + return fmt("%d", tm->tm_year + 1900); +} + +static const struct cgit_period periods[] = { + {'w', "week", 12, 4, trunc_week, dec_week, inc_week, pretty_week}, + {'m', "month", 12, 4, trunc_month, dec_month, inc_month, pretty_month}, + {'q', "quarter", 12, 4, trunc_quarter, dec_quarter, inc_quarter, pretty_quarter}, + {'y', "year", 12, 4, trunc_year, dec_year, inc_year, pretty_year}, +}; + +/* Given a period code or name, return a period index (1, 2, 3 or 4) + * and update the period pointer to the correcsponding struct. + * If no matching code is found, return 0. + */ +int cgit_find_stats_period(const char *expr, const struct cgit_period **period) +{ + int i; + char code = '\0'; + + if (!expr) + return 0; + + if (strlen(expr) == 1) + code = expr[0]; + + for (i = 0; i < sizeof(periods) / sizeof(periods[0]); i++) + if (periods[i].code == code || !strcmp(periods[i].name, expr)) { + if (period) + *period = &periods[i]; + return i + 1; + } + return 0; +} + +const char *cgit_find_stats_periodname(int idx) +{ + if (idx > 0 && idx < 4) + return periods[idx - 1].name; + else + return ""; +} + +static void add_commit(struct string_list *authors, struct commit *commit, + const struct cgit_period *period) +{ + struct commitinfo *info; + struct string_list_item *author, *item; + struct authorstat *authorstat; + struct string_list *items; + char *tmp; + struct tm date; + time_t t; + uintptr_t *counter; + + info = cgit_parse_commit(commit); + tmp = xstrdup(info->author); + author = string_list_insert(authors, tmp); + if (!author->util) + author->util = xcalloc(1, sizeof(struct authorstat)); + else + free(tmp); + authorstat = author->util; + items = &authorstat->list; + t = info->committer_date; + gmtime_r(&t, &date); + period->trunc(&date); + tmp = xstrdup(period->pretty(&date)); + item = string_list_insert(items, tmp); + counter = (uintptr_t *)&item->util; + if (*counter) + free(tmp); + (*counter)++; + + authorstat->total++; + cgit_free_commitinfo(info); +} + +static int cmp_total_commits(const void *a1, const void *a2) +{ + const struct string_list_item *i1 = a1; + const struct string_list_item *i2 = a2; + const struct authorstat *auth1 = i1->util; + const struct authorstat *auth2 = i2->util; + + return auth2->total - auth1->total; +} + +/* Walk the commit DAG and collect number of commits per author per + * timeperiod into a nested string_list collection. + */ +static struct string_list collect_stats(const struct cgit_period *period) +{ + struct string_list authors; + struct rev_info rev; + struct commit *commit; + const char *argv[] = {NULL, ctx.qry.head, NULL, NULL, NULL, NULL}; + int argc = 3; + time_t now; + long i; + struct tm tm; + char tmp[11]; + + time(&now); + gmtime_r(&now, &tm); + period->trunc(&tm); + for (i = 1; i < period->count; i++) + period->dec(&tm); + strftime(tmp, sizeof(tmp), "%Y-%m-%d", &tm); + argv[2] = xstrdup(fmt("--since=%s", tmp)); + if (ctx.qry.path) { + argv[3] = "--"; + argv[4] = ctx.qry.path; + argc += 2; + } + init_revisions(&rev, NULL); + rev.abbrev = DEFAULT_ABBREV; + rev.commit_format = CMIT_FMT_DEFAULT; + rev.max_parents = 1; + rev.verbose_header = 1; + rev.show_root_diff = 0; + setup_revisions(argc, argv, &rev, NULL); + prepare_revision_walk(&rev); + memset(&authors, 0, sizeof(authors)); + while ((commit = get_revision(&rev)) != NULL) { + add_commit(&authors, commit, period); + free_commit_buffer(the_repository->parsed_objects, commit); + free_commit_list(commit->parents); + commit->parents = NULL; + } + return authors; +} + +static void print_combined_authorrow(struct string_list *authors, int from, + int to, const char *name, + const char *leftclass, + const char *centerclass, + const char *rightclass, + const struct cgit_period *period) +{ + struct string_list_item *author; + struct authorstat *authorstat; + struct string_list *items; + struct string_list_item *date; + time_t now; + long i, j, total, subtotal; + struct tm tm; + char *tmp; + + time(&now); + gmtime_r(&now, &tm); + period->trunc(&tm); + for (i = 1; i < period->count; i++) + period->dec(&tm); + + total = 0; + htmlf("<tr><td class='%s'>%s</td>", leftclass, + fmt(name, to - from + 1)); + for (j = 0; j < period->count; j++) { + tmp = period->pretty(&tm); + period->inc(&tm); + subtotal = 0; + for (i = from; i <= to; i++) { + author = &authors->items[i]; + authorstat = author->util; + items = &authorstat->list; + date = string_list_lookup(items, tmp); + if (date) + subtotal += (uintptr_t)date->util; + } + htmlf("<td class='%s'>%ld</td>", centerclass, subtotal); + total += subtotal; + } + htmlf("<td class='%s'>%ld</td></tr>", rightclass, total); +} + +static void print_authors(struct string_list *authors, int top, + const struct cgit_period *period) +{ + struct string_list_item *author; + struct authorstat *authorstat; + struct string_list *items; + struct string_list_item *date; + time_t now; + long i, j, total; + struct tm tm; + char *tmp; + + time(&now); + gmtime_r(&now, &tm); + period->trunc(&tm); + for (i = 1; i < period->count; i++) + period->dec(&tm); + + html("<table class='stats'><tr><th>Author</th>"); + for (j = 0; j < period->count; j++) { + tmp = period->pretty(&tm); + htmlf("<th>%s</th>", tmp); + period->inc(&tm); + } + html("<th>Total</th></tr>\n"); + + if (top <= 0 || top > authors->nr) + top = authors->nr; + + for (i = 0; i < top; i++) { + author = &authors->items[i]; + html("<tr><td class='left'>"); + html_txt(author->string); + html("</td>"); + authorstat = author->util; + items = &authorstat->list; + total = 0; + for (j = 0; j < period->count; j++) + period->dec(&tm); + for (j = 0; j < period->count; j++) { + tmp = period->pretty(&tm); + period->inc(&tm); + date = string_list_lookup(items, tmp); + if (!date) + html("<td>0</td>"); + else { + htmlf("<td>%lu</td>", (uintptr_t)date->util); + total += (uintptr_t)date->util; + } + } + htmlf("<td class='sum'>%ld</td></tr>", total); + } + + if (top < authors->nr) + print_combined_authorrow(authors, top, authors->nr - 1, + "Others (%ld)", "left", "", "sum", period); + + print_combined_authorrow(authors, 0, authors->nr - 1, "Total", + "total", "sum", "sum", period); + html("</table>"); +} + +/* Create a sorted string_list with one entry per author. The util-field + * for each author is another string_list which is used to calculate the + * number of commits per time-interval. + */ +void cgit_show_stats(void) +{ + struct string_list authors; + const struct cgit_period *period; + int top, i; + const char *code = "w"; + + if (ctx.qry.period) + code = ctx.qry.period; + + i = cgit_find_stats_period(code, &period); + if (!i) { + cgit_print_error_page(404, "Not found", + "Unknown statistics type: %c", code[0]); + return; + } + if (i > ctx.repo->max_stats) { + cgit_print_error_page(400, "Bad request", + "Statistics type disabled: %s", period->name); + return; + } + authors = collect_stats(period); + qsort(authors.items, authors.nr, sizeof(struct string_list_item), + cmp_total_commits); + + top = ctx.qry.ofs; + if (!top) + top = 10; + + cgit_print_layout_start(); + html("<div class='cgit-panel'>"); + html("<b>stat options</b>"); + html("<form method='get'>"); + cgit_add_hidden_formfields(1, 0, "stats"); + html("<table><tr><td colspan='2'/></tr>"); + if (ctx.repo->max_stats > 1) { + html("<tr><td class='label'>Period:</td>"); + html("<td class='ctrl'><select name='period' onchange='this.form.submit();'>"); + for (i = 0; i < ctx.repo->max_stats; i++) + html_option(fmt("%c", periods[i].code), + periods[i].name, fmt("%c", period->code)); + html("</select></td></tr>"); + } + html("<tr><td class='label'>Authors:</td>"); + html("<td class='ctrl'><select name='ofs' onchange='this.form.submit();'>"); + html_intoption(10, "10", top); + html_intoption(25, "25", top); + html_intoption(50, "50", top); + html_intoption(100, "100", top); + html_intoption(-1, "all", top); + html("</select></td></tr>"); + html("<tr><td/><td class='ctrl'>"); + html("<noscript><input type='submit' value='Reload'/></noscript>"); + html("</td></tr></table>"); + html("</form>"); + html("</div>"); + htmlf("<h2>Commits per author per %s", period->name); + if (ctx.qry.path) { + html(" (path '"); + html_txt(ctx.qry.path); + html("')"); + } + html("</h2>"); + print_authors(&authors, top, period); + cgit_print_layout_end(); +} + diff --git a/third_party/cgit/ui-stats.h b/third_party/cgit/ui-stats.h new file mode 100644 index 000000000000..0e61b03da3b3 --- /dev/null +++ b/third_party/cgit/ui-stats.h @@ -0,0 +1,28 @@ +#ifndef UI_STATS_H +#define UI_STATS_H + +#include "cgit.h" + +struct cgit_period { + const char code; + const char *name; + int max_periods; + int count; + + /* Convert a tm value to the first day in the period */ + void (*trunc)(struct tm *tm); + + /* Update tm value to start of next/previous period */ + void (*dec)(struct tm *tm); + void (*inc)(struct tm *tm); + + /* Pretty-print a tm value */ + char *(*pretty)(struct tm *tm); +}; + +extern int cgit_find_stats_period(const char *expr, const struct cgit_period **period); +extern const char *cgit_find_stats_periodname(int idx); + +extern void cgit_show_stats(void); + +#endif /* UI_STATS_H */ diff --git a/third_party/cgit/ui-summary.c b/third_party/cgit/ui-summary.c new file mode 100644 index 000000000000..d1bb6a59faea --- /dev/null +++ b/third_party/cgit/ui-summary.c @@ -0,0 +1,160 @@ +/* ui-summary.c: functions for generating repo summary page + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-summary.h" +#include "html.h" +#include "ui-blob.h" +#include "ui-log.h" +#include "ui-plain.h" +#include "ui-refs.h" +#include "ui-shared.h" + +static int urls; + +static void print_url(const char *url) +{ + int columns = 3; + + if (ctx.repo->enable_log_filecount) + columns++; + if (ctx.repo->enable_log_linecount) + columns++; + + if (urls++ == 0) { + htmlf("<tr class='nohover'><td colspan='%d'> </td></tr>", columns); + htmlf("<tr class='nohover'><th class='left' colspan='%d'>Clone</th></tr>\n", columns); + } + + htmlf("<tr><td colspan='%d'><a rel='vcs-git' href='", columns); + html_url_path(url); + html("' title='"); + html_attr(ctx.repo->name); + html(" Git repository'>"); + html_txt(url); + html("</a></td></tr>\n"); +} + +void cgit_print_summary(void) +{ + int columns = 3; + + if (ctx.repo->enable_log_filecount) + columns++; + if (ctx.repo->enable_log_linecount) + columns++; + + cgit_print_layout_start(); + html("<table summary='repository info' class='list nowrap'>"); + cgit_print_branches(ctx.cfg.summary_branches); + htmlf("<tr class='nohover'><td colspan='%d'> </td></tr>", columns); + cgit_print_tags(ctx.cfg.summary_tags); + if (ctx.cfg.summary_log > 0) { + htmlf("<tr class='nohover'><td colspan='%d'> </td></tr>", columns); + cgit_print_log(ctx.qry.head, 0, ctx.cfg.summary_log, NULL, + NULL, NULL, 0, 0, 0); + } + urls = 0; + cgit_add_clone_urls(print_url); + html("</table>"); + cgit_print_layout_end(); +} + +/* The caller must free the return value. */ +static char* append_readme_path(const char *filename, const char *ref, const char *path) +{ + char *file, *base_dir, *full_path, *resolved_base = NULL, *resolved_full = NULL; + /* If a subpath is specified for the about page, make it relative + * to the directory containing the configured readme. */ + + file = xstrdup(filename); + base_dir = dirname(file); + if (!strcmp(base_dir, ".") || !strcmp(base_dir, "..")) { + if (!ref) { + free(file); + return NULL; + } + full_path = xstrdup(path); + } else + full_path = fmtalloc("%s/%s", base_dir, path); + + if (!ref) { + resolved_base = realpath(base_dir, NULL); + resolved_full = realpath(full_path, NULL); + if (!resolved_base || !resolved_full || !starts_with(resolved_full, resolved_base)) { + free(full_path); + full_path = NULL; + } + } + + free(file); + free(resolved_base); + free(resolved_full); + + return full_path; +} + +void cgit_print_repo_readme(const char *path) +{ + char *filename, *ref, *mimetype; + int free_filename = 0; + + mimetype = get_mimetype_for_filename(path); + if (mimetype && (!strncmp(mimetype, "image/", 6) || !strncmp(mimetype, "video/", 6))) { + ctx.page.mimetype = mimetype; + ctx.page.charset = NULL; + cgit_print_plain(); + free(mimetype); + return; + } + free(mimetype); + + cgit_print_layout_start(); + if (ctx.repo->readme.nr == 0) + goto done; + + filename = ctx.repo->readme.items[0].string; + ref = ctx.repo->readme.items[0].util; + + if (path) { + free_filename = 1; + filename = append_readme_path(filename, ref, path); + if (!filename) + goto done; + } + + /* Determine which file to serve by checking whether the given filename is + * already a valid file and otherwise appending the expected file name of + * the readme. + * + * If neither yield a valid file, the user gets a blank page. Could probably + * do with an error message in between there, but whatever. + */ + if (path && ref && !cgit_ref_path_exists(filename, ref, 1)) { + filename = fmtalloc("%s/%s", path, ctx.repo->readme.items[0].string); + free_filename = 1; + } + + /* Print the calculated readme, either from the git repo or from the + * filesystem, while applying the about-filter. + */ + html("<div id='summary'>"); + cgit_open_filter(ctx.repo->about_filter, filename); + if (ref) + cgit_print_file(filename, ref, 1); + else + html_include(filename); + cgit_close_filter(ctx.repo->about_filter); + + html("</div>"); + if (free_filename) + free(filename); + +done: + cgit_print_layout_end(); +} diff --git a/third_party/cgit/ui-summary.h b/third_party/cgit/ui-summary.h new file mode 100644 index 000000000000..cba696af53af --- /dev/null +++ b/third_party/cgit/ui-summary.h @@ -0,0 +1,7 @@ +#ifndef UI_SUMMARY_H +#define UI_SUMMARY_H + +extern void cgit_print_summary(void); +extern void cgit_print_repo_readme(const char *path); + +#endif /* UI_SUMMARY_H */ diff --git a/third_party/cgit/ui-tag.c b/third_party/cgit/ui-tag.c new file mode 100644 index 000000000000..08789102f678 --- /dev/null +++ b/third_party/cgit/ui-tag.c @@ -0,0 +1,120 @@ +/* ui-tag.c: display a tag + * + * Copyright (C) 2006-2014 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-tag.h" +#include "html.h" +#include "ui-shared.h" + +static void print_tag_content(char *buf) +{ + char *p; + + if (!buf) + return; + + html("<div class='commit-subject'>"); + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + html_txt(buf); + html("</div>"); + if (p) { + html("<div class='commit-msg'>"); + html_txt(++p); + html("</div>"); + } +} + +static void print_download_links(char *revname) +{ + html("<tr><th>download</th><td class='oid'>"); + cgit_print_snapshot_links(ctx.repo, revname, "<br/>"); + html("</td></tr>"); +} + +void cgit_print_tag(char *revname) +{ + struct strbuf fullref = STRBUF_INIT; + struct object_id oid; + struct object *obj; + + if (!revname) + revname = ctx.qry.head; + + strbuf_addf(&fullref, "refs/tags/%s", revname); + if (get_oid(fullref.buf, &oid)) { + cgit_print_error_page(404, "Not found", + "Bad tag reference: %s", revname); + goto cleanup; + } + obj = parse_object(the_repository, &oid); + if (!obj) { + cgit_print_error_page(500, "Internal server error", + "Bad object id: %s", oid_to_hex(&oid)); + goto cleanup; + } + if (obj->type == OBJ_TAG) { + struct tag *tag; + struct taginfo *info; + + tag = lookup_tag(the_repository, &oid); + if (!tag || parse_tag(tag) || !(info = cgit_parse_tag(tag))) { + cgit_print_error_page(500, "Internal server error", + "Bad tag object: %s", revname); + goto cleanup; + } + cgit_print_layout_start(); + html("<table class='commit-info'>\n"); + html("<tr><td>tag name</td><td>"); + html_txt(revname); + htmlf(" (%s)</td></tr>\n", oid_to_hex(&oid)); + if (info->tagger_date > 0) { + html("<tr><td>tag date</td><td>"); + html_txt(show_date(info->tagger_date, info->tagger_tz, + cgit_date_mode(DATE_DOTTIME))); + html("</td></tr>\n"); + } + if (info->tagger) { + html("<tr><td>tagged by</td><td>"); + cgit_open_filter(ctx.repo->email_filter, info->tagger_email, "tag"); + html_txt(info->tagger); + if (info->tagger_email && !ctx.cfg.noplainemail) { + html(" "); + html_txt(info->tagger_email); + } + cgit_close_filter(ctx.repo->email_filter); + html("</td></tr>\n"); + } + html("<tr><td>tagged object</td><td class='oid'>"); + cgit_object_link(tag->tagged); + html("</td></tr>\n"); + if (ctx.repo->snapshots) + print_download_links(revname); + html("</table>\n"); + print_tag_content(info->msg); + cgit_print_layout_end(); + cgit_free_taginfo(info); + } else { + cgit_print_layout_start(); + html("<table class='commit-info'>\n"); + html("<tr><td>tag name</td><td>"); + html_txt(revname); + html("</td></tr>\n"); + html("<tr><td>tagged object</td><td class='oid'>"); + cgit_object_link(obj); + html("</td></tr>\n"); + if (ctx.repo->snapshots) + print_download_links(revname); + html("</table>\n"); + cgit_print_layout_end(); + } + +cleanup: + strbuf_release(&fullref); +} diff --git a/third_party/cgit/ui-tag.h b/third_party/cgit/ui-tag.h new file mode 100644 index 000000000000..d295cdcdbd59 --- /dev/null +++ b/third_party/cgit/ui-tag.h @@ -0,0 +1,6 @@ +#ifndef UI_TAG_H +#define UI_TAG_H + +extern void cgit_print_tag(char *revname); + +#endif /* UI_TAG_H */ diff --git a/third_party/cgit/ui-tree.c b/third_party/cgit/ui-tree.c new file mode 100644 index 000000000000..19074507635a --- /dev/null +++ b/third_party/cgit/ui-tree.c @@ -0,0 +1,405 @@ +/* ui-tree.c: functions for tree output + * + * Copyright (C) 2006-2017 cgit Development Team <cgit@lists.zx2c4.com> + * + * Licensed under GNU General Public License v2 + * (see COPYING for full license text) + */ + +#include "cgit.h" +#include "ui-tree.h" +#include "html.h" +#include "ui-shared.h" + +struct walk_tree_context { + char *curr_rev; + char *match_path; + int state; +}; + +static void print_text_buffer(const char *name, char *buf, unsigned long size) +{ + unsigned long lineno, idx; + const char *numberfmt = "<a id='n%1$d' href='#n%1$d'>%1$d</a>\n"; + + html("<table summary='blob content' class='blob'>\n"); + + if (ctx.cfg.enable_tree_linenumbers) { + html("<tr><td class='linenumbers'><pre>"); + idx = 0; + lineno = 0; + + if (size) { + htmlf(numberfmt, ++lineno); + while (idx < size - 1) { // skip absolute last newline + if (buf[idx] == '\n') + htmlf(numberfmt, ++lineno); + idx++; + } + } + html("</pre></td>\n"); + } + else { + html("<tr>\n"); + } + + if (ctx.repo->source_filter) { + char *filter_arg = xstrdup(name); + html("<td class='lines'><pre><code>"); + cgit_open_filter(ctx.repo->source_filter, filter_arg); + html_raw(buf, size); + cgit_close_filter(ctx.repo->source_filter); + free(filter_arg); + html("</code></pre></td></tr></table>\n"); + return; + } + + html("<td class='lines'><pre><code>"); + html_txt(buf); + html("</code></pre></td></tr></table>\n"); +} + +#define ROWLEN 32 + +static void print_binary_buffer(char *buf, unsigned long size) +{ + unsigned long ofs, idx; + static char ascii[ROWLEN + 1]; + + html("<table summary='blob content' class='bin-blob'>\n"); + html("<tr><th>ofs</th><th>hex dump</th><th>ascii</th></tr>"); + for (ofs = 0; ofs < size; ofs += ROWLEN, buf += ROWLEN) { + htmlf("<tr><td class='right'>%04lx</td><td class='hex'>", ofs); + for (idx = 0; idx < ROWLEN && ofs + idx < size; idx++) + htmlf("%*s%02x", + idx == 16 ? 4 : 1, "", + buf[idx] & 0xff); + html(" </td><td class='hex'>"); + for (idx = 0; idx < ROWLEN && ofs + idx < size; idx++) + ascii[idx] = isgraph(buf[idx]) ? buf[idx] : '.'; + ascii[idx] = '\0'; + html_txt(ascii); + html("</td></tr>\n"); + } + html("</table>\n"); +} + +static void print_object(const struct object_id *oid, const char *path, const char *basename, const char *rev) +{ + enum object_type type; + char *buf; + unsigned long size; + int is_binary; + + type = oid_object_info(the_repository, oid, &size); + if (type == OBJ_BAD) { + cgit_print_error_page(404, "Not found", + "Bad object name: %s", oid_to_hex(oid)); + return; + } + + buf = read_object_file(oid, &type, &size); + if (!buf) { + cgit_print_error_page(500, "Internal server error", + "Error reading object %s", oid_to_hex(oid)); + return; + } + is_binary = buffer_is_binary(buf, size); + + cgit_set_title_from_path(path); + + cgit_print_layout_start(); + htmlf("blob: %s (", oid_to_hex(oid)); + cgit_plain_link("plain", NULL, NULL, ctx.qry.head, + rev, path); + if (ctx.repo->enable_blame && !is_binary) { + html(") ("); + cgit_blame_link("blame", NULL, NULL, ctx.qry.head, + rev, path); + } + html(")\n"); + + if (ctx.cfg.max_blob_size && size / 1024 > ctx.cfg.max_blob_size) { + htmlf("<div class='error'>blob size (%ldKB) exceeds display size limit (%dKB).</div>", + size / 1024, ctx.cfg.max_blob_size); + return; + } + + if (is_binary) + print_binary_buffer(buf, size); + else + print_text_buffer(basename, buf, size); + + free(buf); +} + +struct single_tree_ctx { + struct strbuf *path; + struct object_id oid; + char *name; + size_t count; +}; + +static int single_tree_cb(const struct object_id *oid, struct strbuf *base, + const char *pathname, unsigned mode, void *cbdata) +{ + struct single_tree_ctx *ctx = cbdata; + + if (++ctx->count > 1) + return -1; + + if (!S_ISDIR(mode)) { + ctx->count = 2; + return -1; + } + + ctx->name = xstrdup(pathname); + oidcpy(&ctx->oid, oid); + strbuf_addf(ctx->path, "/%s", pathname); + return 0; +} + +static void write_tree_link(const struct object_id *oid, char *name, + char *rev, struct strbuf *fullpath) +{ + size_t initial_length = fullpath->len; + struct tree *tree; + struct single_tree_ctx tree_ctx = { + .path = fullpath, + .count = 1, + }; + struct pathspec paths = { + .nr = 0 + }; + + oidcpy(&tree_ctx.oid, oid); + + while (tree_ctx.count == 1) { + cgit_tree_link(name, NULL, "ls-dir", ctx.qry.head, rev, + fullpath->buf); + + tree = lookup_tree(the_repository, &tree_ctx.oid); + if (!tree) + return; + + free(tree_ctx.name); + tree_ctx.name = NULL; + tree_ctx.count = 0; + + read_tree(the_repository, tree, &paths, single_tree_cb, &tree_ctx); + + if (tree_ctx.count != 1) + break; + + html(" / "); + name = tree_ctx.name; + } + + strbuf_setlen(fullpath, initial_length); +} + +static int ls_item(const struct object_id *oid, struct strbuf *base, + const char *pathname, unsigned mode, void *cbdata) +{ + struct walk_tree_context *walk_tree_ctx = cbdata; + char *name; + struct strbuf fullpath = STRBUF_INIT; + struct strbuf linkpath = STRBUF_INIT; + struct strbuf class = STRBUF_INIT; + enum object_type type; + unsigned long size = 0; + char *buf; + + name = xstrdup(pathname); + strbuf_addf(&fullpath, "%s%s%s", ctx.qry.path ? ctx.qry.path : "", + ctx.qry.path ? "/" : "", name); + + if (!S_ISGITLINK(mode)) { + type = oid_object_info(the_repository, oid, &size); + if (type == OBJ_BAD) { + htmlf("<tr><td colspan='3'>Bad object: %s %s</td></tr>", + name, + oid_to_hex(oid)); + goto cleanup; + } + } + + html("<tr><td class='ls-mode'>"); + cgit_print_filemode(mode); + html("</td><td>"); + if (S_ISGITLINK(mode)) { + cgit_submodule_link("ls-mod", fullpath.buf, oid_to_hex(oid)); + } else if (S_ISDIR(mode)) { + write_tree_link(oid, name, walk_tree_ctx->curr_rev, + &fullpath); + } else { + char *ext = strrchr(name, '.'); + strbuf_addstr(&class, "ls-blob"); + if (ext) + strbuf_addf(&class, " %s", ext + 1); + cgit_tree_link(name, NULL, class.buf, ctx.qry.head, + walk_tree_ctx->curr_rev, fullpath.buf); + } + if (S_ISLNK(mode)) { + html(" -> "); + buf = read_object_file(oid, &type, &size); + if (!buf) { + htmlf("Error reading object: %s", oid_to_hex(oid)); + goto cleanup; + } + strbuf_addbuf(&linkpath, &fullpath); + strbuf_addf(&linkpath, "/../%s", buf); + strbuf_normalize_path(&linkpath); + cgit_tree_link(buf, NULL, class.buf, ctx.qry.head, + walk_tree_ctx->curr_rev, linkpath.buf); + free(buf); + strbuf_release(&linkpath); + } + htmlf("</td><td class='ls-size'>%li</td>", size); + + html("<td>"); + cgit_log_link("log", NULL, "button", ctx.qry.head, + walk_tree_ctx->curr_rev, fullpath.buf, 0, NULL, NULL, + ctx.qry.showmsg, 0); + if (ctx.repo->max_stats) + cgit_stats_link("stats", NULL, "button", ctx.qry.head, + fullpath.buf); + if (!S_ISGITLINK(mode)) + cgit_plain_link("plain", NULL, "button", ctx.qry.head, + walk_tree_ctx->curr_rev, fullpath.buf); + if (!S_ISDIR(mode) && ctx.repo->enable_blame) + cgit_blame_link("blame", NULL, "button", ctx.qry.head, + walk_tree_ctx->curr_rev, fullpath.buf); + html("</td></tr>\n"); + +cleanup: + free(name); + strbuf_release(&fullpath); + strbuf_release(&class); + return 0; +} + +static void ls_head(void) +{ + cgit_print_layout_start(); + html("<table summary='tree listing' class='list'>\n"); + html("<tr class='nohover'>"); + html("<th class='left'>Mode</th>"); + html("<th class='left'>Name</th>"); + html("<th class='right'>Size</th>"); + html("<th/>"); + html("</tr>\n"); +} + +static void ls_tail(void) +{ + html("</table>\n"); + cgit_print_layout_end(); +} + +static void ls_tree(const struct object_id *oid, const char *path, struct walk_tree_context *walk_tree_ctx) +{ + struct tree *tree; + struct pathspec paths = { + .nr = 0 + }; + + tree = parse_tree_indirect(oid); + if (!tree) { + cgit_print_error_page(404, "Not found", + "Not a tree object: %s", oid_to_hex(oid)); + return; + } + + ls_head(); + read_tree(the_repository, tree, &paths, ls_item, walk_tree_ctx); + ls_tail(); +} + + +static int walk_tree(const struct object_id *oid, struct strbuf *base, + const char *pathname, unsigned mode, void *cbdata) +{ + struct walk_tree_context *walk_tree_ctx = cbdata; + + if (walk_tree_ctx->state == 0) { + struct strbuf buffer = STRBUF_INIT; + + strbuf_addbuf(&buffer, base); + strbuf_addstr(&buffer, pathname); + if (strcmp(walk_tree_ctx->match_path, buffer.buf)) + return READ_TREE_RECURSIVE; + + if (S_ISDIR(mode)) { + walk_tree_ctx->state = 1; + cgit_set_title_from_path(buffer.buf); + strbuf_release(&buffer); + ls_head(); + return READ_TREE_RECURSIVE; + } else { + walk_tree_ctx->state = 2; + print_object(oid, buffer.buf, pathname, walk_tree_ctx->curr_rev); + strbuf_release(&buffer); + return 0; + } + } + ls_item(oid, base, pathname, mode, walk_tree_ctx); + return 0; +} + +/* + * Show a tree or a blob + * rev: the commit pointing at the root tree object + * path: path to tree or blob + */ +void cgit_print_tree(const char *rev, char *path) +{ + struct object_id oid; + struct commit *commit; + struct pathspec_item path_items = { + .match = path, + .len = path ? strlen(path) : 0 + }; + struct pathspec paths = { + .nr = path ? 1 : 0, + .items = &path_items + }; + struct walk_tree_context walk_tree_ctx = { + .match_path = path, + .state = 0 + }; + + if (!rev) + rev = ctx.qry.head; + + if (get_oid(rev, &oid)) { + cgit_print_error_page(404, "Not found", + "Invalid revision name: %s", rev); + return; + } + commit = lookup_commit_reference(the_repository, &oid); + if (!commit || parse_commit(commit)) { + cgit_print_error_page(404, "Not found", + "Invalid commit reference: %s", rev); + return; + } + + walk_tree_ctx.curr_rev = xstrdup(rev); + + if (path == NULL) { + ls_tree(get_commit_tree_oid(commit), NULL, &walk_tree_ctx); + goto cleanup; + } + + read_tree(the_repository, repo_get_commit_tree(the_repository, commit), + &paths, walk_tree, &walk_tree_ctx); + if (walk_tree_ctx.state == 1) + ls_tail(); + else if (walk_tree_ctx.state == 2) + cgit_print_layout_end(); + else + cgit_print_error_page(404, "Not found", "Path not found"); + +cleanup: + free(walk_tree_ctx.curr_rev); +} diff --git a/third_party/cgit/ui-tree.h b/third_party/cgit/ui-tree.h new file mode 100644 index 000000000000..bbd34e3566a5 --- /dev/null +++ b/third_party/cgit/ui-tree.h @@ -0,0 +1,6 @@ +#ifndef UI_TREE_H +#define UI_TREE_H + +extern void cgit_print_tree(const char *rev, char *path); + +#endif /* UI_TREE_H */ diff --git a/third_party/default.nix b/third_party/default.nix new file mode 100644 index 000000000000..5a244ed22039 --- /dev/null +++ b/third_party/default.nix @@ -0,0 +1,52 @@ +# This file defines the root of all external dependency imports (i.e. +# third-party code) in the TVL package tree. +# +# There are two categories of third-party programs: +# +# 1) Programs in nixpkgs, the NixOS package set. For these, you might +# want to look at //third_party/nixpkgs (for the package set +# imports) and //third_party/overlays (for modifications in these +# imported package sets). +# +# 2) Third-party software packaged in this repository. This is all +# other folders below //third_party, other than the ones mentioned +# above. + +{ pkgs, ... }: + +{ + # Expose a partially applied NixOS, expecting an attribute set with + # a `configuration` key. Exposing it like this makes it possible to + # modify some of the base configuration used by NixOS. + # + # This partially reimplements the code in + # <nixpkgs/nixos/default.nix> as we need to modify its internals to + # be able to pass `specialArgs`. We depend on this because `depot` + # needs to be partially evaluated in NixOS configuration before + # module imports are resolved. + nixos = { + configuration, + specialArgs ? {}, + system ? builtins.currentSystem, + ... + }: + let + eval = import "${pkgs.path}/nixos/lib/eval-config.nix" { + inherit specialArgs system; + modules = [ configuration ]; + }; + + # This is for `nixos-rebuild build-vm'. + vmConfig = (import "${pkgs.path}/nixos/lib/eval-config.nix" { + inherit specialArgs system; + modules = [ + configuration + "${pkgs.path}/nixos/modules/virtualisation/qemu-vm.nix" + ]; + }).config; + in { + inherit (eval) pkgs config options; + system = eval.config.system.build.toplevel; + vm = vmConfig.system.build.vm; + }; +} diff --git a/third_party/dfmt/default.nix b/third_party/dfmt/default.nix new file mode 100644 index 000000000000..5ce0d7541c25 --- /dev/null +++ b/third_party/dfmt/default.nix @@ -0,0 +1,36 @@ +# dfmt is a code formatter for D +{ pkgs, lib, ... }: + +pkgs.stdenv.mkDerivation rec { + pname = "dfmt"; + version = "0.13.4"; + + src = pkgs.fetchFromGitHub { + owner = "dlang-community"; + repo = "dfmt"; + rev = "v${version}"; + sha256 = "02a8qvrmnl1c15y6irxlhqpr0hjj5s8qk0jc20ivj0fl6p4v9shj"; + fetchSubmodules = true; + }; + + nativeBuildInputs = with pkgs; [ + dmd + + # fake git that will be used to fetch the version string + (pkgs.writeShellScriptBin "git" "echo 'v${version}'") + ]; + + DFLAGS = "-release"; + + installPhase = '' + mkdir -p $out/bin + cp bin/dfmt $out/bin/ + strip -s $out/bin/dfmt + ''; + + meta = { + description = "D code formatter"; + homepage = "https://github.com/dlang-community/dfmt"; + license = lib.licenses.boost; + }; +} diff --git a/third_party/dhall/OWNERS b/third_party/dhall/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/third_party/dhall/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/third_party/dhall/default.nix b/third_party/dhall/default.nix new file mode 100644 index 000000000000..6f3782233fa3 --- /dev/null +++ b/third_party/dhall/default.nix @@ -0,0 +1,22 @@ +{ pkgs, ... }: + +let + # binary releases of dhall tools, since the build in nixpkgs is + # broken most of the time. The binaries are also fully static + # builds, instead of the half-static crap that nixpkgs produces. + easy-dhall-nix = + import (builtins.fetchTarball { + url = "https://github.com/justinwoo/easy-dhall-nix/archive/eae7f64c4d6c70681e5a56c84198236930ba425e.tar.gz"; + sha256 = "1y2x15v8a679vlpxazjpibfwajp6zph60f8wjcm4xflbvazk0dx7"; + }) { inherit pkgs; }; +in { + dhall = easy-dhall-nix.dhall-simple; + dhall-bash = easy-dhall-nix.dhall-bash-simple; + dhall-docs = easy-dhall-nix.dhall-docs-simple; + dhall-json = easy-dhall-nix.dhall-json-simple; + dhall-lsp-server = easy-dhall-nix.dhall-lsp-simple; + dhall-nix = easy-dhall-nix.dhall-nix-simple; + # not yet in dhall-simple + # dhall-nixpkgs = easy-dhall-nix.dhall-nixpkgs-simple; + dhall-yaml = easy-dhall-nix.dhall-yaml-simple; +} diff --git a/third_party/elmPackages_0_18/default.nix b/third_party/elmPackages_0_18/default.nix new file mode 100644 index 000000000000..55d66f062926 --- /dev/null +++ b/third_party/elmPackages_0_18/default.nix @@ -0,0 +1,15 @@ +# Backports Elm packages for Elm 0.18 from an older channel of +# nixpkgs. +# +# Elm 0.19 changed the language & package ecosystem completely, +# essentially requiring a partial rewrite of all Elm apps. However, +# //fun/gemma uses Elm 0.18 and I don't have time to rewrite it. + +{ pkgs, ... }: + +(import (pkgs.fetchFromGitHub { + owner = "NixOS"; + repo = "nixpkgs"; + rev = "14f9ee66e63077539252f8b4550049381a082518"; + sha256 = "1wn7nmb1cqfk2j91l3rwc6yhimfkzxprb8wknw5wi57yhq9m6lv1"; +}) {}).elmPackages diff --git a/third_party/emacs/rcirc/default.nix b/third_party/emacs/rcirc/default.nix new file mode 100644 index 000000000000..f34cbed7896a --- /dev/null +++ b/third_party/emacs/rcirc/default.nix @@ -0,0 +1,7 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage rec { + pname = "rcirc"; + version = "1"; + src = ./rcirc.el; +} diff --git a/third_party/emacs/rcirc/rcirc.el b/third_party/emacs/rcirc/rcirc.el new file mode 100644 index 000000000000..b59cb4e9afe4 --- /dev/null +++ b/third_party/emacs/rcirc/rcirc.el @@ -0,0 +1,3133 @@ +;;; rcirc.el --- default, simple IRC client -*- lexical-binding: t; -*- + +;; Copyright (C) 2005-2019 Free Software Foundation, Inc. + +;; Author: Ryan Yeske <rcyeske@gmail.com> +;; Maintainers: Ryan Yeske <rcyeske@gmail.com>, +;; Leo Liu <sdl.web@gmail.com> +;; Keywords: comm + +;; This file is part of GNU Emacs. + +;; GNU Emacs is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; GNU Emacs is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Internet Relay Chat (IRC) is a form of instant communication over +;; the Internet. It is mainly designed for group (many-to-many) +;; communication in discussion forums called channels, but also allows +;; one-to-one communication. + +;; Rcirc has simple defaults and clear and consistent behavior. +;; Message arrival timestamps, activity notification on the mode line, +;; message filling, nick completion, and keepalive pings are all +;; enabled by default, but can easily be adjusted or turned off. Each +;; discussion takes place in its own buffer and there is a single +;; server buffer per connection. + +;; Open a new irc connection with: +;; M-x irc RET + +;;; Todo: + +;;; Code: + +(require 'cl-lib) +(require 'ring) +(require 'time-date) +(require 'subr-x) + +(defgroup rcirc nil + "Simple IRC client." + :version "22.1" + :prefix "rcirc-" + :link '(custom-manual "(rcirc)") + :group 'applications) + +(defcustom rcirc-server-alist + '(("irc.freenode.net" :channels ("#rcirc") + ;; Don't use the TLS port by default, in case gnutls is not available. + ;; :port 7000 :encryption tls + )) + "An alist of IRC connections to establish when running `rcirc'. +Each element looks like (SERVER-NAME PARAMETERS). + +SERVER-NAME is a string describing the server to connect +to. + +The optional PARAMETERS come in pairs PARAMETER VALUE. + +The following parameters are recognized: + +`:nick' + +VALUE must be a string. If absent, `rcirc-default-nick' is used +for this connection. + +`:port' + +VALUE must be a number or string. If absent, +`rcirc-default-port' is used. + +`:user-name' + +VALUE must be a string. If absent, `rcirc-default-user-name' is +used. + +`:password' + +VALUE must be a string. If absent, no PASS command will be sent +to the server. + +`:full-name' + +VALUE must be a string. If absent, `rcirc-default-full-name' is +used. + +`:channels' + +VALUE must be a list of strings describing which channels to join +when connecting to this server. If absent, no channels will be +connected to automatically. + +`:encryption' + +VALUE must be `plain' (the default) for unencrypted connections, or `tls' +for connections using SSL/TLS. + +`:server-alias' + +VALUE must be a string that will be used instead of the server name for +display purposes. If absent, the real server name will be displayed instead." + :type '(alist :key-type string + :value-type (plist :options + ((:nick string) + (:port integer) + (:user-name string) + (:password string) + (:full-name string) + (:channels (repeat string)) + (:encryption (choice (const tls) + (const plain))) + (:server-alias string)))) + :group 'rcirc) + +(defcustom rcirc-default-port 6667 + "The default port to connect to." + :type 'integer + :group 'rcirc) + +(defcustom rcirc-default-nick (user-login-name) + "Your nick." + :type 'string + :group 'rcirc) + +(defcustom rcirc-default-user-name "user" + "Your user name sent to the server when connecting." + :version "24.1" ; changed default + :type 'string + :group 'rcirc) + +(defcustom rcirc-default-full-name "unknown" + "The full name sent to the server when connecting." + :version "24.1" ; changed default + :type 'string + :group 'rcirc) + +(defcustom rcirc-fill-flag t + "Non-nil means line-wrap messages printed in channel buffers." + :type 'boolean + :group 'rcirc) + +(defcustom rcirc-fill-column nil + "Column beyond which automatic line-wrapping should happen. +If nil, use value of `fill-column'. +If a function (e.g., `frame-text-width' or `window-text-width'), +call it to compute the number of columns." + :risky t ; can get funcalled + :type '(choice (const :tag "Value of `fill-column'" nil) + (integer :tag "Number of columns") + (function :tag "Function returning the number of columns")) + :group 'rcirc) + +(defcustom rcirc-fill-prefix nil + "Text to insert before filled lines. +If nil, calculate the prefix dynamically to line up text +underneath each nick." + :type '(choice (const :tag "Dynamic" nil) + (string :tag "Prefix text")) + :group 'rcirc) + +(defvar rcirc-ignore-buffer-activity-flag nil + "If non-nil, ignore activity in this buffer.") +(make-variable-buffer-local 'rcirc-ignore-buffer-activity-flag) + +(defvar rcirc-low-priority-flag nil + "If non-nil, activity in this buffer is considered low priority.") +(make-variable-buffer-local 'rcirc-low-priority-flag) + +(defcustom rcirc-omit-responses + '("JOIN" "PART" "QUIT" "NICK") + "Responses which will be hidden when `rcirc-omit-mode' is enabled." + :type '(repeat string) + :group 'rcirc) + +(defvar rcirc-prompt-start-marker nil) + +(define-minor-mode rcirc-omit-mode + "Toggle the hiding of \"uninteresting\" lines. +With a prefix argument ARG, enable Rcirc-Omit mode if ARG is +positive, and disable it otherwise. If called from Lisp, enable +the mode if ARG is omitted or nil. + +Uninteresting lines are those whose responses are listed in +`rcirc-omit-responses'." + nil " Omit" nil + (if rcirc-omit-mode + (progn + (add-to-invisibility-spec '(rcirc-omit . nil)) + (message "Rcirc-Omit mode enabled")) + (remove-from-invisibility-spec '(rcirc-omit . nil)) + (message "Rcirc-Omit mode disabled")) + (dolist (window (get-buffer-window-list (current-buffer))) + (with-selected-window window + (recenter (when (> (point) rcirc-prompt-start-marker) -1))))) + +(defcustom rcirc-time-format "%H:%M " + "Describes how timestamps are printed. +Used as the first arg to `format-time-string'." + :type 'string + :group 'rcirc) + +(defcustom rcirc-input-ring-size 1024 + "Size of input history ring." + :type 'integer + :group 'rcirc) + +(defcustom rcirc-read-only-flag t + "Non-nil means make text in IRC buffers read-only." + :type 'boolean + :group 'rcirc) + +(defcustom rcirc-buffer-maximum-lines nil + "The maximum size in lines for rcirc buffers. +Channel buffers are truncated from the top to be no greater than this +number. If zero or nil, no truncating is done." + :type '(choice (const :tag "No truncation" nil) + (integer :tag "Number of lines")) + :group 'rcirc) + +(defcustom rcirc-scroll-show-maximum-output t + "If non-nil, scroll buffer to keep the point at the bottom of +the window." + :type 'boolean + :group 'rcirc) + +(defcustom rcirc-authinfo nil + "List of authentication passwords. +Each element of the list is a list with a SERVER-REGEXP string +and a method symbol followed by method specific arguments. + +The valid METHOD symbols are `nickserv', `chanserv' and +`bitlbee'. + +The ARGUMENTS for each METHOD symbol are: + `nickserv': NICK PASSWORD [NICKSERV-NICK] + `chanserv': NICK CHANNEL PASSWORD + `bitlbee': NICK PASSWORD + `quakenet': ACCOUNT PASSWORD + +Examples: + ((\"freenode\" nickserv \"bob\" \"p455w0rd\") + (\"freenode\" chanserv \"bob\" \"#bobland\" \"passwd99\") + (\"bitlbee\" bitlbee \"robert\" \"sekrit\") + (\"dal.net\" nickserv \"bob\" \"sekrit\" \"NickServ@services.dal.net\") + (\"quakenet.org\" quakenet \"bobby\" \"sekrit\"))" + :type '(alist :key-type (string :tag "Server") + :value-type (choice (list :tag "NickServ" + (const nickserv) + (string :tag "Nick") + (string :tag "Password")) + (list :tag "ChanServ" + (const chanserv) + (string :tag "Nick") + (string :tag "Channel") + (string :tag "Password")) + (list :tag "BitlBee" + (const bitlbee) + (string :tag "Nick") + (string :tag "Password")) + (list :tag "QuakeNet" + (const quakenet) + (string :tag "Account") + (string :tag "Password")))) + :group 'rcirc) + +(defcustom rcirc-auto-authenticate-flag t + "Non-nil means automatically send authentication string to server. +See also `rcirc-authinfo'." + :type 'boolean + :group 'rcirc) + +(defcustom rcirc-authenticate-before-join t + "Non-nil means authenticate to services before joining channels. +Currently only works with NickServ on some networks." + :version "24.1" + :type 'boolean + :group 'rcirc) + +(defcustom rcirc-prompt "> " + "Prompt string to use in IRC buffers. + +The following replacements are made: +%n is your nick. +%s is the server. +%t is the buffer target, a channel or a user. + +Setting this alone will not affect the prompt; +use either M-x customize or also call `rcirc-update-prompt'." + :type 'string + :set 'rcirc-set-changed + :initialize 'custom-initialize-default + :group 'rcirc) + +(defcustom rcirc-keywords nil + "List of keywords to highlight in message text." + :type '(repeat string) + :group 'rcirc) + +(defcustom rcirc-ignore-list () + "List of ignored nicks. +Use /ignore to list them, use /ignore NICK to add or remove a nick." + :type '(repeat string) + :group 'rcirc) + +(defvar rcirc-ignore-list-automatic () + "List of ignored nicks added to `rcirc-ignore-list' because of renaming. +When an ignored person renames, their nick is added to both lists. +Nicks will be removed from the automatic list on follow-up renamings or +parts.") + +(defcustom rcirc-bright-nicks nil + "List of nicks to be emphasized. +See `rcirc-bright-nick' face." + :type '(repeat string) + :group 'rcirc) + +(defcustom rcirc-dim-nicks nil + "List of nicks to be deemphasized. +See `rcirc-dim-nick' face." + :type '(repeat string) + :group 'rcirc) + +(define-obsolete-variable-alias 'rcirc-print-hooks + 'rcirc-print-functions "24.3") +(defcustom rcirc-print-functions nil + "Hook run after text is printed. +Called with 5 arguments, PROCESS, SENDER, RESPONSE, TARGET and TEXT." + :type 'hook + :group 'rcirc) + +(defvar rcirc-authenticated-hook nil + "Hook run after successfully authenticated.") + +(defcustom rcirc-always-use-server-buffer-flag nil + "Non-nil means messages without a channel target will go to the server buffer." + :type 'boolean + :group 'rcirc) + +(defcustom rcirc-decode-coding-system 'utf-8 + "Coding system used to decode incoming irc messages. +Set to `undecided' if you want the encoding of the incoming +messages autodetected." + :type 'coding-system + :group 'rcirc) + +(defcustom rcirc-encode-coding-system 'utf-8 + "Coding system used to encode outgoing irc messages." + :type 'coding-system + :group 'rcirc) + +(defcustom rcirc-coding-system-alist nil + "Alist to decide a coding system to use for a channel I/O operation. +The format is ((PATTERN . VAL) ...). +PATTERN is either a string or a cons of strings. +If PATTERN is a string, it is used to match a target. +If PATTERN is a cons of strings, the car part is used to match a +target, and the cdr part is used to match a server. +VAL is either a coding system or a cons of coding systems. +If VAL is a coding system, it is used for both decoding and encoding +messages. +If VAL is a cons of coding systems, the car part is used for decoding, +and the cdr part is used for encoding." + :type '(alist :key-type (choice (string :tag "Channel Regexp") + (cons (string :tag "Channel Regexp") + (string :tag "Server Regexp"))) + :value-type (choice coding-system + (cons (coding-system :tag "Decode") + (coding-system :tag "Encode")))) + :group 'rcirc) + +(defcustom rcirc-multiline-major-mode 'fundamental-mode + "Major-mode function to use in multiline edit buffers." + :type 'function + :group 'rcirc) + +(defcustom rcirc-nick-completion-format "%s: " + "Format string to use in nick completions. + +The format string is only used when completing at the beginning +of a line. The string is passed as the first argument to +`format' with the nickname as the second argument." + :version "24.1" + :type 'string + :group 'rcirc) + +(defcustom rcirc-kill-channel-buffers nil + "When non-nil, kill channel buffers when the server buffer is killed. +Only the channel buffers associated with the server in question +will be killed." + :version "24.3" + :type 'boolean + :group 'rcirc) + +(defvar rcirc-nick nil) + +(defvar rcirc-prompt-end-marker nil) + +(defvar rcirc-nick-table nil) + +(defvar rcirc-recent-quit-alist nil + "Alist of nicks that have recently quit or parted the channel.") + +(defvar rcirc-nick-syntax-table + (let ((table (make-syntax-table text-mode-syntax-table))) + (mapc (lambda (c) (modify-syntax-entry c "w" table)) + "[]\\`_^{|}-") + (modify-syntax-entry ?' "_" table) + table) + "Syntax table which includes all nick characters as word constituents.") + +;; each process has an alist of (target . buffer) pairs +(defvar rcirc-buffer-alist nil) + +(defvar rcirc-activity nil + "List of buffers with unviewed activity.") + +(defvar rcirc-activity-string "" + "String displayed in mode line representing `rcirc-activity'.") +(put 'rcirc-activity-string 'risky-local-variable t) + +(defvar rcirc-server-buffer nil + "The server buffer associated with this channel buffer.") + +(defvar rcirc-target nil + "The channel or user associated with this buffer.") + +(defvar rcirc-urls nil + "List of URLs seen in the current buffer and their start positions.") +(put 'rcirc-urls 'permanent-local t) + +(defvar rcirc-timeout-seconds 600 + "Kill connection after this many seconds if there is no activity.") + +(defconst rcirc-id-string (concat "rcirc on GNU Emacs " emacs-version)) + +(defvar rcirc-startup-channels nil) + +(defvar rcirc-server-name-history nil + "History variable for \\[rcirc] call.") + +(defvar rcirc-server-port-history nil + "History variable for \\[rcirc] call.") + +(defvar rcirc-nick-name-history nil + "History variable for \\[rcirc] call.") + +(defvar rcirc-user-name-history nil + "History variable for \\[rcirc] call.") + +(defvar rcirc-last-message-time nil) + +;;;###autoload +(defun rcirc (arg) + "Connect to all servers in `rcirc-server-alist'. + +Do not connect to a server if it is already connected. + +If ARG is non-nil, instead prompt for connection parameters." + (interactive "P") + (if arg + (let* ((server (completing-read "IRC Server: " + rcirc-server-alist + nil nil + (caar rcirc-server-alist) + 'rcirc-server-name-history)) + (server-plist (cdr (assoc-string server rcirc-server-alist))) + (port (read-string "IRC Port: " + (number-to-string + (or (plist-get server-plist :port) + rcirc-default-port)) + 'rcirc-server-port-history)) + (nick (read-string "IRC Nick: " + (or (plist-get server-plist :nick) + rcirc-default-nick) + 'rcirc-nick-name-history)) + (user-name (read-string "IRC Username: " + (or (plist-get server-plist :user-name) + rcirc-default-user-name) + 'rcirc-user-name-history)) + (password (read-passwd "IRC Password: " nil + (plist-get server-plist :password))) + (channels (split-string + (read-string "IRC Channels: " + (mapconcat 'identity + (plist-get server-plist + :channels) + " ")) + "[, ]+" t)) + (encryption (rcirc-prompt-for-encryption server-plist))) + (rcirc-connect server port nick user-name + rcirc-default-full-name + channels password encryption)) + ;; connect to servers in `rcirc-server-alist' + (let (connected-servers) + (dolist (c rcirc-server-alist) + (let ((server (car c)) + (nick (or (plist-get (cdr c) :nick) rcirc-default-nick)) + (port (or (plist-get (cdr c) :port) rcirc-default-port)) + (user-name (or (plist-get (cdr c) :user-name) + rcirc-default-user-name)) + (full-name (or (plist-get (cdr c) :full-name) + rcirc-default-full-name)) + (channels (plist-get (cdr c) :channels)) + (password (plist-get (cdr c) :password)) + (encryption (plist-get (cdr c) :encryption)) + (server-alias (plist-get (cdr c) :server-alias)) + contact) + (when server + (let (connected) + (dolist (p (rcirc-process-list)) + (when (string= (or server-alias server) (process-name p)) + (setq connected p))) + (if (not connected) + (condition-case nil + (rcirc-connect server port nick user-name + full-name channels password encryption + server-alias) + (quit (message "Quit connecting to %s" + (or server-alias server)))) + (with-current-buffer (process-buffer connected) + (setq contact (process-contact + (get-buffer-process (current-buffer)) :name)) + (setq connected-servers + (cons (if (stringp contact) + contact (or server-alias server)) + connected-servers)))))))) + (when connected-servers + (message "Already connected to %s" + (if (cdr connected-servers) + (concat (mapconcat 'identity (butlast connected-servers) ", ") + ", and " + (car (last connected-servers))) + (car connected-servers))))))) + +;;;###autoload +(defalias 'irc 'rcirc) + + +(defvar rcirc-process-output nil) +(defvar rcirc-topic nil) +(defvar rcirc-keepalive-timer nil) +(defvar rcirc-last-server-message-time nil) +(defvar rcirc-server nil) ; server provided by server +(defvar rcirc-server-name nil) ; server name given by 001 response +(defvar rcirc-timeout-timer nil) +(defvar rcirc-user-authenticated nil) +(defvar rcirc-user-disconnect nil) +(defvar rcirc-connecting nil) +(defvar rcirc-connection-info nil) +(defvar rcirc-process nil) + +;;;###autoload +(defun rcirc-connect (server &optional port nick user-name + full-name startup-channels password encryption + server-alias) + (save-excursion + (message "Connecting to %s..." (or server-alias server)) + (let* ((inhibit-eol-conversion) + (port-number (if port + (if (stringp port) + (string-to-number port) + port) + rcirc-default-port)) + (nick (or nick rcirc-default-nick)) + (user-name (or user-name rcirc-default-user-name)) + (full-name (or full-name rcirc-default-full-name)) + (startup-channels startup-channels) + (process (open-network-stream + (or server-alias server) nil server port-number + :type (or encryption 'plain)))) + ;; set up process + (set-process-coding-system process 'raw-text 'raw-text) + (switch-to-buffer (rcirc-generate-new-buffer-name process nil)) + (set-process-buffer process (current-buffer)) + (rcirc-mode process nil) + (set-process-sentinel process 'rcirc-sentinel) + (set-process-filter process 'rcirc-filter) + + (setq-local rcirc-connection-info + (list server port nick user-name full-name startup-channels + password encryption server-alias)) + (setq-local rcirc-process process) + (setq-local rcirc-server server) + (setq-local rcirc-server-name + (or server-alias server)) ; Update when we get 001 response. + (setq-local rcirc-buffer-alist nil) + (setq-local rcirc-nick-table (make-hash-table :test 'equal)) + (setq-local rcirc-nick nick) + (setq-local rcirc-process-output nil) + (setq-local rcirc-startup-channels startup-channels) + (setq-local rcirc-last-server-message-time (current-time)) + + (setq-local rcirc-timeout-timer nil) + (setq-local rcirc-user-disconnect nil) + (setq-local rcirc-user-authenticated nil) + (setq-local rcirc-connecting t) + + (add-hook 'auto-save-hook 'rcirc-log-write) + + ;; identify + (unless (zerop (length password)) + (rcirc-send-string process (concat "PASS " password))) + (rcirc-send-string process (concat "NICK " nick)) + (rcirc-send-string process "CAP LS 302") + (rcirc-send-string process (concat "USER " user-name + " 0 * :" full-name)) + + ;; setup ping timer if necessary + (unless rcirc-keepalive-timer + (setq rcirc-keepalive-timer + (run-at-time 0 (/ rcirc-timeout-seconds 2) 'rcirc-keepalive))) + + (message "Connecting to %s...done" (or server-alias server)) + + ;; return process object + process))) + +(defmacro with-rcirc-process-buffer (process &rest body) + (declare (indent 1) (debug t)) + `(with-current-buffer (process-buffer ,process) + ,@body)) + +(defmacro with-rcirc-server-buffer (&rest body) + (declare (indent 0) (debug t)) + `(with-current-buffer rcirc-server-buffer + ,@body)) + +(define-obsolete-function-alias 'rcirc-float-time 'float-time "26.1") + +(defun rcirc-prompt-for-encryption (server-plist) + "Prompt the user for the encryption method to use. +SERVER-PLIST is the property list for the server." + (let ((msg "Encryption (default %s): ") + (choices '("plain" "tls")) + (default (or (plist-get server-plist :encryption) + 'plain))) + (intern + (completing-read (format msg default) + choices nil t nil nil (symbol-name default))))) + +(defun rcirc-keepalive () + "Send keep alive pings to active rcirc processes. +Kill processes that have not received a server message since the +last ping." + (if (rcirc-process-list) + (mapc (lambda (process) + (with-rcirc-process-buffer process + (when (not rcirc-connecting) + (rcirc-send-ctcp process + rcirc-nick + (format "KEEPALIVE %f" + (float-time)))))) + (rcirc-process-list)) + ;; no processes, clean up timer + (when (timerp rcirc-keepalive-timer) + (cancel-timer rcirc-keepalive-timer)) + (setq rcirc-keepalive-timer nil))) + +(defun rcirc-handler-ctcp-KEEPALIVE (process _target _sender message) + (with-rcirc-process-buffer process + (setq header-line-format (format "%f" (- (float-time) + (string-to-number message)))))) + +(defvar rcirc-debug-buffer "*rcirc debug*") +(defvar rcirc-debug-flag nil + "If non-nil, write information to `rcirc-debug-buffer'.") +(defun rcirc-debug (process text) + "Add an entry to the debug log including PROCESS and TEXT. +Debug text is written to `rcirc-debug-buffer' if `rcirc-debug-flag' +is non-nil." + (when rcirc-debug-flag + (with-current-buffer (get-buffer-create rcirc-debug-buffer) + (goto-char (point-max)) + (insert (concat + "[" + (format-time-string "%Y-%m-%dT%T ") (process-name process) + "] " + text))))) + +(define-obsolete-variable-alias 'rcirc-sentinel-hooks + 'rcirc-sentinel-functions "24.3") +(defvar rcirc-sentinel-functions nil + "Hook functions called when the process sentinel is called. +Functions are called with PROCESS and SENTINEL arguments.") + +(defcustom rcirc-reconnect-delay 0 + "The minimum interval in seconds between reconnect attempts. +When 0, do not auto-reconnect." + :version "25.1" + :type 'integer + :group 'rcirc) + +(defvar rcirc-last-connect-time nil + "The last time the buffer was connected.") + +(defun rcirc-sentinel (process sentinel) + "Called when PROCESS receives SENTINEL." + (let ((sentinel (replace-regexp-in-string "\n" "" sentinel))) + (rcirc-debug process (format "SENTINEL: %S %S\n" process sentinel)) + (with-rcirc-process-buffer process + (dolist (buffer (cons nil (mapcar 'cdr rcirc-buffer-alist))) + (with-current-buffer (or buffer (current-buffer)) + (rcirc-print process "rcirc.el" "ERROR" rcirc-target + (format "%s: %s (%S)" + (process-name process) + sentinel + (process-status process)) + (not rcirc-target)) + (rcirc-disconnect-buffer))) + (when (and (string= sentinel "deleted") + (< 0 rcirc-reconnect-delay)) + (let ((now (current-time))) + (when (or (null rcirc-last-connect-time) + (< rcirc-reconnect-delay + (float-time (time-subtract now rcirc-last-connect-time)))) + (setq rcirc-last-connect-time now) + (rcirc-cmd-reconnect nil)))) + (run-hook-with-args 'rcirc-sentinel-functions process sentinel)))) + +(defun rcirc-disconnect-buffer (&optional buffer) + (with-current-buffer (or buffer (current-buffer)) + ;; set rcirc-target to nil for each channel so cleanup + ;; doesn't happen when we reconnect + (setq rcirc-target nil) + (setq mode-line-process ":disconnected"))) + +(defun rcirc-process-list () + "Return a list of rcirc processes." + (let (ps) + (mapc (lambda (p) + (when (buffer-live-p (process-buffer p)) + (with-rcirc-process-buffer p + (when (eq major-mode 'rcirc-mode) + (setq ps (cons p ps)))))) + (process-list)) + ps)) + +(define-obsolete-variable-alias 'rcirc-receive-message-hooks + 'rcirc-receive-message-functions "24.3") +(defvar rcirc-receive-message-functions nil + "Hook functions run when a message is received from server. +Function is called with PROCESS, COMMAND, SENDER, ARGS and LINE.") +(defun rcirc-filter (process output) + "Called when PROCESS receives OUTPUT." + (rcirc-debug process output) + (rcirc-reschedule-timeout process) + (with-rcirc-process-buffer process + (setq rcirc-last-server-message-time (current-time)) + (setq rcirc-process-output (concat rcirc-process-output output)) + (when (= ?\n (aref rcirc-process-output + (1- (length rcirc-process-output)))) + (let ((lines (split-string rcirc-process-output "[\n\r]" t))) + (setq rcirc-process-output nil) + (dolist (line lines) + (rcirc-process-server-response process line)))))) + +(defun rcirc-reschedule-timeout (process) + (with-rcirc-process-buffer process + (when (not rcirc-connecting) + (with-rcirc-process-buffer process + (when rcirc-timeout-timer (cancel-timer rcirc-timeout-timer)) + (setq rcirc-timeout-timer (run-at-time rcirc-timeout-seconds nil + 'rcirc-delete-process + process)))))) + +(defun rcirc-delete-process (process) + (delete-process process)) + +(defvar rcirc-trap-errors-flag t) +(defun rcirc-process-server-response (process text) + (if rcirc-trap-errors-flag + (condition-case err + (rcirc-process-server-response-1 process text) + (error + (rcirc-print process "RCIRC" "ERROR" nil + (format "\"%s\" %s" text err) t))) + (rcirc-process-server-response-1 process text))) + +(defun rcirc-handle-message-tags (tags) + (if-let* ((time (cdr (assoc "time" tags))) + (timestamp (floor (float-time (date-to-time time))))) + (setq rcirc-last-message-time timestamp))) + +(defun rcirc-parse-tags (tags) + "Parse TAGS message prefix." + (mapcar (lambda (tag) + (let ((p (split-string tag "="))) + `(,(car p) . ,(cadr p)))) + (split-string tags ";"))) + +(defun rcirc-process-server-response-1 (process text) + + ;; attempt to extract and handle IRCv3 message tags (which contain server-time) + (if (string-match "^\\(@\\([^ ]+\\) \\)?\\(\\(:[^ ]+ \\)?[^ ]+ .+\\)$" text) + (let ((tags (match-string 2 text)) + (rest (match-string 3 text))) + (when tags + (rcirc-handle-message-tags (rcirc-parse-tags tags))) + (setq text rest))) + + (if (string-match "^\\(:\\([^ ]+\\) \\)?\\([^ ]+\\) \\(.+\\)$" text) + (let* ((user (match-string 2 text)) + (sender (rcirc-user-nick user)) + (cmd (match-string 3 text)) + (args (match-string 4 text)) + (handler (intern-soft (concat "rcirc-handler-" cmd)))) + (string-match "^\\([^:]*\\):?\\(.+\\)?$" args) + (let* ((args1 (match-string 1 args)) + (args2 (match-string 2 args)) + (args (delq nil (append (split-string args1 " " t) + (list args2))))) + (if (not (fboundp handler)) + (rcirc-handler-generic process cmd sender args text) + (funcall handler process sender args text)) + (run-hook-with-args 'rcirc-receive-message-functions + process cmd sender args text))) + (message "UNHANDLED: %s" text))) + +(defvar rcirc-responses-no-activity '("305" "306") + "Responses that don't trigger activity in the mode-line indicator.") + +(defun rcirc-handler-generic (process response sender args _text) + "Generic server response handler." + (rcirc-print process sender response nil + (mapconcat 'identity (cdr args) " ") + (not (member response rcirc-responses-no-activity)))) + +(defun rcirc--connection-open-p (process) + (memq (process-status process) '(run open))) + +(defun rcirc-send-string (process string) + "Send PROCESS a STRING plus a newline." + (let ((string (concat (encode-coding-string string rcirc-encode-coding-system) + "\n"))) + (unless (rcirc--connection-open-p process) + (error "Network connection to %s is not open" + (process-name process))) + (rcirc-debug process string) + (process-send-string process string))) + +(defun rcirc-send-privmsg (process target string) + (rcirc-send-string process (format "PRIVMSG %s :%s" target string))) + +(defun rcirc-send-ctcp (process target request &optional args) + (let ((args (if args (concat " " args) ""))) + (rcirc-send-privmsg process target + (format "\C-a%s%s\C-a" request args)))) + +(defun rcirc-buffer-process (&optional buffer) + "Return the process associated with channel BUFFER. +With no argument or nil as argument, use the current buffer." + (let ((buffer (or buffer (and (buffer-live-p rcirc-server-buffer) + rcirc-server-buffer)))) + (if buffer + (with-current-buffer buffer rcirc-process) + rcirc-process))) + +(defun rcirc-server-name (process) + "Return PROCESS server name, given by the 001 response." + (with-rcirc-process-buffer process + (or rcirc-server-name + (warn "server name for process %S unknown" process)))) + +(defun rcirc-nick (process) + "Return PROCESS nick." + (with-rcirc-process-buffer process + (or rcirc-nick rcirc-default-nick))) + +(defun rcirc-buffer-nick (&optional buffer) + "Return the nick associated with BUFFER. +With no argument or nil as argument, use the current buffer." + (with-current-buffer (or buffer (current-buffer)) + (with-current-buffer rcirc-server-buffer + (or rcirc-nick rcirc-default-nick)))) + +(defvar rcirc-max-message-length 420 + "Messages longer than this value will be split.") + +(defun rcirc-split-message (message) + "Split MESSAGE into chunks within `rcirc-max-message-length'." + ;; `rcirc-encode-coding-system' can have buffer-local value. + (let ((encoding rcirc-encode-coding-system)) + (with-temp-buffer + (insert message) + (goto-char (point-min)) + (let (result) + (while (not (eobp)) + (goto-char (or (byte-to-position rcirc-max-message-length) + (point-max))) + ;; max message length is 512 including CRLF + (while (and (not (bobp)) + (> (length (encode-coding-region + (point-min) (point) encoding t)) + rcirc-max-message-length)) + (forward-char -1)) + (push (delete-and-extract-region (point-min) (point)) result)) + (nreverse result))))) + +(defun rcirc-send-message (process target message &optional noticep silent) + "Send TARGET associated with PROCESS a privmsg with text MESSAGE. +If NOTICEP is non-nil, send a notice instead of privmsg. +If SILENT is non-nil, do not print the message in any irc buffer." + (let ((response (if noticep "NOTICE" "PRIVMSG"))) + (rcirc-get-buffer-create process target) + (dolist (msg (rcirc-split-message message)) + (rcirc-send-string process (concat response " " target " :" msg)) + (unless silent + (rcirc-print process (rcirc-nick process) response target msg))))) + +(defvar rcirc-input-ring nil) +(defvar rcirc-input-ring-index 0) + +(defun rcirc-prev-input-string (arg) + (ring-ref rcirc-input-ring (+ rcirc-input-ring-index arg))) + +(defun rcirc-insert-prev-input () + (interactive) + (when (<= rcirc-prompt-end-marker (point)) + (delete-region rcirc-prompt-end-marker (point-max)) + (insert (rcirc-prev-input-string 0)) + (setq rcirc-input-ring-index (1+ rcirc-input-ring-index)))) + +(defun rcirc-insert-next-input () + (interactive) + (when (<= rcirc-prompt-end-marker (point)) + (delete-region rcirc-prompt-end-marker (point-max)) + (setq rcirc-input-ring-index (1- rcirc-input-ring-index)) + (insert (rcirc-prev-input-string -1)))) + +(defvar rcirc-server-commands + '("/admin" "/away" "/connect" "/die" "/error" "/info" + "/invite" "/ison" "/join" "/kick" "/kill" "/links" + "/list" "/lusers" "/mode" "/motd" "/names" "/nick" + "/notice" "/oper" "/part" "/pass" "/ping" "/pong" + "/privmsg" "/quit" "/rehash" "/restart" "/service" "/servlist" + "/server" "/squery" "/squit" "/stats" "/summon" "/time" + "/topic" "/trace" "/user" "/userhost" "/users" "/version" + "/wallops" "/who" "/whois" "/whowas") + "A list of user commands by IRC server. +The value defaults to RFCs 1459 and 2812.") + +;; /me and /ctcp are not defined by `defun-rcirc-command'. +(defvar rcirc-client-commands '("/me" "/ctcp") + "A list of user commands defined by IRC client rcirc. +The list is updated automatically by `defun-rcirc-command'.") + +(defun rcirc-completion-at-point () + "Function used for `completion-at-point-functions' in `rcirc-mode'." + (and (rcirc-looking-at-input) + (let* ((beg (save-excursion + ;; On some networks it is common to message or + ;; mention someone using @nick instead of just + ;; nick. + (if (re-search-backward "[[:space:]@]" rcirc-prompt-end-marker t) + (1+ (point)) + rcirc-prompt-end-marker))) + (table (if (and (= beg rcirc-prompt-end-marker) + (eq (char-after beg) ?/)) + (delete-dups + (nconc (sort (copy-sequence rcirc-client-commands) + 'string-lessp) + (sort (copy-sequence rcirc-server-commands) + 'string-lessp))) + (rcirc-channel-nicks (rcirc-buffer-process) + rcirc-target)))) + (list beg (point) table)))) + +(defvar rcirc-completions nil) +(defvar rcirc-completion-start nil) + +(defun rcirc-complete () + "Cycle through completions from list of nicks in channel or IRC commands. +IRC command completion is performed only if `/' is the first input char." + (interactive) + (unless (rcirc-looking-at-input) + (error "Point not located after rcirc prompt")) + (if (eq last-command this-command) + (setq rcirc-completions + (append (cdr rcirc-completions) (list (car rcirc-completions)))) + (let ((completion-ignore-case t) + (table (rcirc-completion-at-point))) + (setq rcirc-completion-start (car table)) + (setq rcirc-completions + (and rcirc-completion-start + (all-completions (buffer-substring rcirc-completion-start + (cadr table)) + (nth 2 table)))))) + (let ((completion (car rcirc-completions))) + (when completion + (delete-region rcirc-completion-start (point)) + (insert + (cond + ((= (aref completion 0) ?/) (concat completion " ")) + ((= rcirc-completion-start rcirc-prompt-end-marker) + (format rcirc-nick-completion-format completion)) + (t completion)))))) + +(defun set-rcirc-decode-coding-system (coding-system) + "Set the decode coding system used in this channel." + (interactive "zCoding system for incoming messages: ") + (setq-local rcirc-decode-coding-system coding-system)) + +(defun set-rcirc-encode-coding-system (coding-system) + "Set the encode coding system used in this channel." + (interactive "zCoding system for outgoing messages: ") + (setq-local rcirc-encode-coding-system coding-system)) + +(defvar rcirc-mode-map + (let ((map (make-sparse-keymap))) + (define-key map (kbd "RET") 'rcirc-send-input) + (define-key map (kbd "M-p") 'rcirc-insert-prev-input) + (define-key map (kbd "M-n") 'rcirc-insert-next-input) + (define-key map (kbd "TAB") 'rcirc-complete) + (define-key map (kbd "C-c C-b") 'rcirc-browse-url) + (define-key map (kbd "C-c C-c") 'rcirc-edit-multiline) + (define-key map (kbd "C-c C-j") 'rcirc-cmd-join) + (define-key map (kbd "C-c C-k") 'rcirc-cmd-kick) + (define-key map (kbd "C-c C-l") 'rcirc-toggle-low-priority) + (define-key map (kbd "C-c C-d") 'rcirc-cmd-mode) + (define-key map (kbd "C-c C-m") 'rcirc-cmd-msg) + (define-key map (kbd "C-c C-r") 'rcirc-cmd-nick) ; rename + (define-key map (kbd "C-c C-o") 'rcirc-omit-mode) + (define-key map (kbd "C-c C-p") 'rcirc-cmd-part) + (define-key map (kbd "C-c C-q") 'rcirc-cmd-query) + (define-key map (kbd "C-c C-t") 'rcirc-cmd-topic) + (define-key map (kbd "C-c C-n") 'rcirc-cmd-names) + (define-key map (kbd "C-c C-w") 'rcirc-cmd-whois) + (define-key map (kbd "C-c C-x") 'rcirc-cmd-quit) + (define-key map (kbd "C-c TAB") ; C-i + 'rcirc-toggle-ignore-buffer-activity) + (define-key map (kbd "C-c C-s") 'rcirc-switch-to-server-buffer) + (define-key map (kbd "C-c C-a") 'rcirc-jump-to-first-unread-line) + map) + "Keymap for rcirc mode.") + +(defvar rcirc-short-buffer-name nil + "Generated abbreviation to use to indicate buffer activity.") + +(defvar rcirc-mode-hook nil + "Hook run when setting up rcirc buffer.") + +(defvar rcirc-last-post-time nil) + +(defvar rcirc-log-alist nil + "Alist of lines to log to disk when `rcirc-log-flag' is non-nil. +Each element looks like (FILENAME . TEXT).") + +(defvar rcirc-current-line 0 + "The current number of responses printed in this channel. +This number is independent of the number of lines in the buffer.") + +(defun rcirc-mode (process target) + ;; FIXME: Use define-derived-mode. + "Major mode for IRC channel buffers. + +\\{rcirc-mode-map}" + (kill-all-local-variables) + (use-local-map rcirc-mode-map) + (setq mode-name "rcirc") + (setq major-mode 'rcirc-mode) + (setq mode-line-process nil) + + (setq-local rcirc-input-ring + ;; If rcirc-input-ring is already a ring with desired + ;; size do not re-initialize. + (if (and (ring-p rcirc-input-ring) + (= (ring-size rcirc-input-ring) + rcirc-input-ring-size)) + rcirc-input-ring + (make-ring rcirc-input-ring-size))) + (setq-local rcirc-server-buffer (process-buffer process)) + (setq-local rcirc-target target) + (setq-local rcirc-topic nil) + (setq-local rcirc-last-post-time (current-time)) + (setq-local fill-paragraph-function 'rcirc-fill-paragraph) + (setq-local rcirc-recent-quit-alist nil) + (setq-local rcirc-current-line 0) + (setq-local rcirc-last-connect-time (current-time)) + + (use-hard-newlines t) + (setq-local rcirc-short-buffer-name nil) + (setq-local rcirc-urls nil) + + ;; setup for omitting responses + (setq buffer-invisibility-spec '()) + (setq buffer-display-table (make-display-table)) + (set-display-table-slot buffer-display-table 4 + (let ((glyph (make-glyph-code + ?. 'font-lock-keyword-face))) + (make-vector 3 glyph))) + + (dolist (i rcirc-coding-system-alist) + (let ((chan (if (consp (car i)) (caar i) (car i))) + (serv (if (consp (car i)) (cdar i) ""))) + (when (and (string-match chan (or target "")) + (string-match serv (rcirc-server-name process))) + (setq-local rcirc-decode-coding-system + (if (consp (cdr i)) (cadr i) (cdr i))) + (setq-local rcirc-encode-coding-system + (if (consp (cdr i)) (cddr i) (cdr i)))))) + + ;; setup the prompt and markers + (setq-local rcirc-prompt-start-marker (point-max-marker)) + (setq-local rcirc-prompt-end-marker (point-max-marker)) + (rcirc-update-prompt) + (goto-char rcirc-prompt-end-marker) + + (setq-local overlay-arrow-position (make-marker)) + + ;; if the user changes the major mode or kills the buffer, there is + ;; cleanup work to do + (add-hook 'change-major-mode-hook 'rcirc-change-major-mode-hook nil t) + (add-hook 'kill-buffer-hook 'rcirc-kill-buffer-hook nil t) + + ;; add to buffer list, and update buffer abbrevs + (when target ; skip server buffer + (let ((buffer (current-buffer))) + (with-rcirc-process-buffer process + (setq rcirc-buffer-alist (cons (cons target buffer) + rcirc-buffer-alist)))) + (rcirc-update-short-buffer-names)) + + (add-hook 'completion-at-point-functions + 'rcirc-completion-at-point nil 'local) + + (run-mode-hooks 'rcirc-mode-hook)) + +(defun rcirc-update-prompt (&optional all) + "Reset the prompt string in the current buffer. + +If ALL is non-nil, update prompts in all IRC buffers." + (if all + (mapc (lambda (process) + (mapc (lambda (buffer) + (with-current-buffer buffer + (rcirc-update-prompt))) + (with-rcirc-process-buffer process + (mapcar 'cdr rcirc-buffer-alist)))) + (rcirc-process-list)) + (let ((inhibit-read-only t) + (prompt (or rcirc-prompt ""))) + (mapc (lambda (rep) + (setq prompt + (replace-regexp-in-string (car rep) (cdr rep) prompt))) + (list (cons "%n" (rcirc-buffer-nick)) + (cons "%s" (with-rcirc-server-buffer rcirc-server-name)) + (cons "%t" (or rcirc-target "")))) + (save-excursion + (delete-region rcirc-prompt-start-marker rcirc-prompt-end-marker) + (goto-char rcirc-prompt-start-marker) + (let ((start (point))) + (insert-before-markers prompt) + (set-marker rcirc-prompt-start-marker start) + (when (not (zerop (- rcirc-prompt-end-marker + rcirc-prompt-start-marker))) + (add-text-properties rcirc-prompt-start-marker + rcirc-prompt-end-marker + (list 'face 'rcirc-prompt + 'read-only t 'field t + 'front-sticky t 'rear-nonsticky t)))))))) + +(defun rcirc-set-changed (option value) + "Set OPTION to VALUE and do updates after a customization change." + (set-default option value) + (cond ((eq option 'rcirc-prompt) + (rcirc-update-prompt 'all)) + (t + (error "Bad option %s" option)))) + +(defun rcirc-channel-p (target) + "Return t if TARGET is a channel name." + (and target + (not (zerop (length target))) + (or (eq (aref target 0) ?#) + (eq (aref target 0) ?&)))) + +(defcustom rcirc-log-directory "~/.emacs.d/rcirc-log" + "Directory to keep IRC logfiles." + :type 'directory + :group 'rcirc) + +(defcustom rcirc-log-flag nil + "Non-nil means log IRC activity to disk. +Logfiles are kept in `rcirc-log-directory'." + :type 'boolean + :group 'rcirc) + +(defun rcirc-kill-buffer-hook () + "Part the channel when killing an rcirc buffer. + +If `rcirc-kill-channel-buffers' is non-nil and the killed buffer +is a server buffer, kills all of the channel buffers associated +with it." + (when (eq major-mode 'rcirc-mode) + (when (and rcirc-log-flag + rcirc-log-directory) + (rcirc-log-write)) + (rcirc-clean-up-buffer "Killed buffer") + (when (and rcirc-buffer-alist ;; it's a server buffer + rcirc-kill-channel-buffers) + (dolist (channel rcirc-buffer-alist) + (kill-buffer (cdr channel)))))) + +(defun rcirc-change-major-mode-hook () + "Part the channel when changing the major-mode." + (rcirc-clean-up-buffer "Changed major mode")) + +(defun rcirc-clean-up-buffer (reason) + (let ((buffer (current-buffer))) + (rcirc-clear-activity buffer) + (when (and (rcirc-buffer-process) + (rcirc--connection-open-p (rcirc-buffer-process))) + (with-rcirc-server-buffer + (setq rcirc-buffer-alist + (rassq-delete-all buffer rcirc-buffer-alist))) + (rcirc-update-short-buffer-names) + (if (rcirc-channel-p rcirc-target) + (rcirc-send-string (rcirc-buffer-process) + (concat "PART " rcirc-target " :" reason)) + (when rcirc-target + (rcirc-remove-nick-channel (rcirc-buffer-process) + (rcirc-buffer-nick) + rcirc-target)))) + (setq rcirc-target nil))) + +(defun rcirc-generate-new-buffer-name (process target) + "Return a buffer name based on PROCESS and TARGET. +This is used for the initial name given to IRC buffers." + (substring-no-properties + (if target + (concat target "@" (process-name process)) + (concat "*" (process-name process) "*")))) + +(defun rcirc-get-buffer (process target &optional server) + "Return the buffer associated with the PROCESS and TARGET. + +If optional argument SERVER is non-nil, return the server buffer +if there is no existing buffer for TARGET, otherwise return nil." + (with-rcirc-process-buffer process + (if (null target) + (current-buffer) + (let ((buffer (cdr (assoc-string target rcirc-buffer-alist t)))) + (or buffer (when server (current-buffer))))))) + +(defun rcirc-get-buffer-create (process target) + "Return the buffer associated with the PROCESS and TARGET. +Create the buffer if it doesn't exist." + (let ((buffer (rcirc-get-buffer process target))) + (if (and buffer (buffer-live-p buffer)) + (with-current-buffer buffer + (when (not rcirc-target) + (setq rcirc-target target)) + buffer) + ;; create the buffer + (with-rcirc-process-buffer process + (let ((new-buffer (get-buffer-create + (rcirc-generate-new-buffer-name process target)))) + (with-current-buffer new-buffer + (rcirc-mode process target) + (rcirc-put-nick-channel process (rcirc-nick process) target + rcirc-current-line)) + new-buffer))))) + +(defun rcirc-send-input () + "Send input to target associated with the current buffer." + (interactive) + (if (< (point) rcirc-prompt-end-marker) + ;; copy the line down to the input area + (progn + (forward-line 0) + (let ((start (if (eq (point) (point-min)) + (point) + (if (get-text-property (1- (point)) 'hard) + (point) + (previous-single-property-change (point) 'hard)))) + (end (next-single-property-change (1+ (point)) 'hard))) + (goto-char (point-max)) + (insert (replace-regexp-in-string + "\n\\s-+" " " + (buffer-substring-no-properties start end))))) + ;; process input + (goto-char (point-max)) + (when (not (equal 0 (- (point) rcirc-prompt-end-marker))) + ;; delete a trailing newline + (when (eq (point) (point-at-bol)) + (delete-char -1)) + (let ((input (buffer-substring-no-properties + rcirc-prompt-end-marker (point)))) + (dolist (line (split-string input "\n")) + (rcirc-process-input-line line)) + ;; add to input-ring + (save-excursion + (ring-insert rcirc-input-ring input) + (setq rcirc-input-ring-index 0)))))) + +(defun rcirc-fill-paragraph (&optional justify) + (interactive "P") + (when (> (point) rcirc-prompt-end-marker) + (save-restriction + (narrow-to-region rcirc-prompt-end-marker (point-max)) + (let ((fill-column rcirc-max-message-length)) + (fill-region (point-min) (point-max) justify))))) + +(defun rcirc-process-input-line (line) + (if (string-match "^/\\([^ ]+\\) ?\\(.*\\)$" line) + (rcirc-process-command (match-string 1 line) + (match-string 2 line) + line) + (rcirc-process-message line))) + +(defun rcirc-process-message (line) + (if (not rcirc-target) + (message "Not joined (no target)") + (delete-region rcirc-prompt-end-marker (point)) + (rcirc-send-message (rcirc-buffer-process) rcirc-target line) + (setq rcirc-last-post-time (current-time)))) + +(defun rcirc-process-command (command args line) + (if (eq (aref command 0) ?/) + ;; "//text" will send "/text" as a message + (rcirc-process-message (substring line 1)) + (let ((fun (intern-soft (concat "rcirc-cmd-" command))) + (process (rcirc-buffer-process))) + (newline) + (with-current-buffer (current-buffer) + (delete-region rcirc-prompt-end-marker (point)) + (if (string= command "me") + (rcirc-print process (rcirc-buffer-nick) + "ACTION" rcirc-target args) + (rcirc-print process (rcirc-buffer-nick) + "COMMAND" rcirc-target line)) + (set-marker rcirc-prompt-end-marker (point)) + (if (fboundp fun) + (funcall fun args process rcirc-target) + (rcirc-send-string process + (concat command " :" args))))))) + +(defvar rcirc-parent-buffer nil) +(make-variable-buffer-local 'rcirc-parent-buffer) +(put 'rcirc-parent-buffer 'permanent-local t) +(defvar rcirc-window-configuration nil) +(defun rcirc-edit-multiline () + "Move current edit to a dedicated buffer." + (interactive) + (let ((pos (1+ (- (point) rcirc-prompt-end-marker)))) + (goto-char (point-max)) + (let ((text (buffer-substring-no-properties rcirc-prompt-end-marker + (point))) + (parent (buffer-name))) + (delete-region rcirc-prompt-end-marker (point)) + (setq rcirc-window-configuration (current-window-configuration)) + (pop-to-buffer (concat "*multiline " parent "*")) + (funcall rcirc-multiline-major-mode) + (rcirc-multiline-minor-mode 1) + (setq rcirc-parent-buffer parent) + (insert text) + (and (> pos 0) (goto-char pos)) + (message "Type C-c C-c to return text to %s, or C-c C-k to cancel" parent)))) + +(defvar rcirc-multiline-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map (kbd "C-c C-c") 'rcirc-multiline-minor-submit) + (define-key map (kbd "C-x C-s") 'rcirc-multiline-minor-submit) + (define-key map (kbd "C-c C-k") 'rcirc-multiline-minor-cancel) + (define-key map (kbd "ESC ESC ESC") 'rcirc-multiline-minor-cancel) + map) + "Keymap for multiline mode in rcirc.") + +(define-minor-mode rcirc-multiline-minor-mode + "Minor mode for editing multiple lines in rcirc. +With a prefix argument ARG, enable the mode if ARG is positive, +and disable it otherwise. If called from Lisp, enable the mode +if ARG is omitted or nil." + :init-value nil + :lighter " rcirc-mline" + :keymap rcirc-multiline-minor-mode-map + :global nil + :group 'rcirc + (setq fill-column rcirc-max-message-length)) + +(defun rcirc-multiline-minor-submit () + "Send the text in buffer back to parent buffer." + (interactive) + (untabify (point-min) (point-max)) + (let ((text (buffer-substring (point-min) (point-max))) + (buffer (current-buffer)) + (pos (point))) + (set-buffer rcirc-parent-buffer) + (goto-char (point-max)) + (insert text) + (kill-buffer buffer) + (set-window-configuration rcirc-window-configuration) + (goto-char (+ rcirc-prompt-end-marker (1- pos))))) + +(defun rcirc-multiline-minor-cancel () + "Cancel the multiline edit." + (interactive) + (kill-buffer (current-buffer)) + (set-window-configuration rcirc-window-configuration)) + +(defun rcirc-any-buffer (process) + "Return a buffer for PROCESS, either the one selected or the process buffer." + (if rcirc-always-use-server-buffer-flag + (process-buffer process) + (let ((buffer (window-buffer))) + (if (and buffer + (with-current-buffer buffer + (and (eq major-mode 'rcirc-mode) + (eq (rcirc-buffer-process) process)))) + buffer + (process-buffer process))))) + +(defcustom rcirc-response-formats + '(("PRIVMSG" . "<%N> %m") + ("NOTICE" . "-%N- %m") + ("ACTION" . "[%N %m]") + ("COMMAND" . "%m") + ("ERROR" . "%fw!!! %m") + (t . "%fp*** %fs%n %r %m")) + "An alist of formats used for printing responses. +The format is looked up using the response-type as a key; +if no match is found, the default entry (with a key of t) is used. + +The entry's value part should be a string, which is inserted with +the of the following escape sequences replaced by the described values: + + %m The message text + %n The sender's nick + %N The sender's nick (with face `rcirc-my-nick' or `rcirc-other-nick') + %r The response-type + %t The target + %fw Following text uses the face `font-lock-warning-face' + %fp Following text uses the face `rcirc-server-prefix' + %fs Following text uses the face `rcirc-server' + %f[FACE] Following text uses the face FACE + %f- Following text uses the default face + %% A literal `%' character" + :type '(alist :key-type (choice (string :tag "Type") + (const :tag "Default" t)) + :value-type string) + :group 'rcirc) + +(defun rcirc-format-response-string (process sender response target text) + "Return a nicely-formatted response string, incorporating TEXT +\(and perhaps other arguments). The specific formatting used +is found by looking up RESPONSE in `rcirc-response-formats'." + (with-temp-buffer + (insert (or (cdr (assoc response rcirc-response-formats)) + (cdr (assq t rcirc-response-formats)))) + (goto-char (point-min)) + (let ((start (point-min)) + (sender (if (or (not sender) + (string= (rcirc-server-name process) sender)) + "" + sender)) + face) + (while (re-search-forward "%\\(\\(f\\(.\\)\\)\\|\\(.\\)\\)" nil t) + (rcirc-add-face start (match-beginning 0) face) + (setq start (match-beginning 0)) + (replace-match + (cl-case (aref (match-string 1) 0) + (?f (setq face + (cl-case (string-to-char (match-string 3)) + (?w 'font-lock-warning-face) + (?p 'rcirc-server-prefix) + (?s 'rcirc-server) + (t nil))) + "") + (?n sender) + (?N (let ((my-nick (rcirc-nick process))) + (save-match-data + (with-syntax-table rcirc-nick-syntax-table + (rcirc-facify sender + (cond ((string= sender my-nick) + 'rcirc-my-nick) + ((and rcirc-bright-nicks + (string-match + (regexp-opt rcirc-bright-nicks + 'words) + sender)) + 'rcirc-bright-nick) + ((and rcirc-dim-nicks + (string-match + (regexp-opt rcirc-dim-nicks + 'words) + sender)) + 'rcirc-dim-nick) + (t + 'rcirc-other-nick))))))) + (?m (propertize text 'rcirc-text text)) + (?r response) + (?t (or target "")) + (t (concat "UNKNOWN CODE:" (match-string 0)))) + t t nil 0) + (rcirc-add-face (match-beginning 0) (match-end 0) face)) + (rcirc-add-face start (match-beginning 0) face)) + (buffer-substring (point-min) (point-max)))) + +(defun rcirc-target-buffer (process sender response target _text) + "Return a buffer to print the server response." + (cl-assert (not (bufferp target))) + (with-rcirc-process-buffer process + (cond ((not target) + (rcirc-any-buffer process)) + ((not (rcirc-channel-p target)) + ;; message from another user + (if (or (string= response "PRIVMSG") + (string= response "ACTION")) + (rcirc-get-buffer-create process (if (string= sender rcirc-nick) + target + sender)) + (rcirc-get-buffer process target t))) + ((or (rcirc-get-buffer process target) + (rcirc-any-buffer process)))))) + +(defvar rcirc-activity-types nil) +(make-variable-buffer-local 'rcirc-activity-types) +(defvar rcirc-last-sender nil) +(make-variable-buffer-local 'rcirc-last-sender) + +(defcustom rcirc-omit-threshold 100 + "Number of lines since last activity from a nick before `rcirc-omit-responses' are omitted." + :type 'integer + :group 'rcirc) + +(defcustom rcirc-log-process-buffers nil + "Non-nil if rcirc process buffers should be logged to disk." + :group 'rcirc + :type 'boolean + :version "24.1") + +(defun rcirc-last-quit-line (process nick target) + "Return the line number where NICK left TARGET. +Returns nil if the information is not recorded." + (let ((chanbuf (rcirc-get-buffer process target))) + (when chanbuf + (cdr (assoc-string nick (with-current-buffer chanbuf + rcirc-recent-quit-alist)))))) + +(defun rcirc-last-line (process nick target) + "Return the line from the last activity from NICK in TARGET." + (let ((line (or (cdr (assoc-string target + (gethash nick (with-rcirc-server-buffer + rcirc-nick-table)) t)) + (rcirc-last-quit-line process nick target)))) + (if line + line + ;;(message "line is nil for %s in %s" nick target) + nil))) + +(defun rcirc-elapsed-lines (process nick target) + "Return the number of lines since activity from NICK in TARGET." + (let ((last-activity-line (rcirc-last-line process nick target))) + (when (and last-activity-line + (> last-activity-line 0)) + (- rcirc-current-line last-activity-line)))) + +(defvar rcirc-markup-text-functions + '(rcirc-markup-attributes + rcirc-markup-my-nick + rcirc-markup-urls + rcirc-markup-keywords + rcirc-markup-bright-nicks) + + "List of functions used to manipulate text before it is printed. + +Each function takes two arguments, SENDER, and RESPONSE. The +buffer is narrowed with the text to be printed and the point is +at the beginning of the `rcirc-text' propertized text.") + +(defun rcirc-print (process sender response target text &optional activity) + "Print TEXT in the buffer associated with TARGET. +Format based on SENDER and RESPONSE. If ACTIVITY is non-nil, +record activity." + (or text (setq text "")) + (unless (and (or (member sender rcirc-ignore-list) + (member (with-syntax-table rcirc-nick-syntax-table + (when (string-match "^\\([^/]\\w*\\)[:,]" text) + (match-string 1 text))) + rcirc-ignore-list)) + ;; do not ignore if we sent the message + (not (string= sender (rcirc-nick process)))) + (let* ((buffer (rcirc-target-buffer process sender response target text)) + (inhibit-read-only t)) + (with-current-buffer buffer + (let ((moving (= (point) rcirc-prompt-end-marker)) + (old-point (point-marker)) + (fill-start (marker-position rcirc-prompt-start-marker))) + + (setq text (decode-coding-string text rcirc-decode-coding-system)) + (unless (string= sender (rcirc-nick process)) + ;; mark the line with overlay arrow + (unless (or (marker-position overlay-arrow-position) + (get-buffer-window (current-buffer)) + (member response rcirc-omit-responses)) + (set-marker overlay-arrow-position + (marker-position rcirc-prompt-start-marker)))) + + ;; temporarily set the marker insertion-type because + ;; insert-before-markers results in hidden text in new buffers + (goto-char rcirc-prompt-start-marker) + (set-marker-insertion-type rcirc-prompt-start-marker t) + (set-marker-insertion-type rcirc-prompt-end-marker t) + + (let ((start (point))) + (insert (rcirc-format-response-string process sender response nil + text) + (propertize "\n" 'hard t)) + + ;; squeeze spaces out of text before rcirc-text + (fill-region fill-start + (1- (or (next-single-property-change fill-start + 'rcirc-text) + rcirc-prompt-end-marker))) + + ;; run markup functions + (save-excursion + (save-restriction + (narrow-to-region start rcirc-prompt-start-marker) + (goto-char (or (next-single-property-change start 'rcirc-text) + (point))) + (when (rcirc-buffer-process) + (save-excursion (rcirc-markup-timestamp sender response)) + (dolist (fn rcirc-markup-text-functions) + (save-excursion (funcall fn sender response))) + (when rcirc-fill-flag + (save-excursion (rcirc-markup-fill sender response)))) + + (when rcirc-read-only-flag + (add-text-properties (point-min) (point-max) + '(read-only t front-sticky t)))) + ;; make text omittable + (let ((last-activity-lines (rcirc-elapsed-lines process sender target))) + (if (and (not (string= (rcirc-nick process) sender)) + (member response rcirc-omit-responses) + (or (not last-activity-lines) + (< rcirc-omit-threshold last-activity-lines))) + (put-text-property (1- start) (1- rcirc-prompt-start-marker) + 'invisible 'rcirc-omit) + ;; otherwise increment the line count + (setq rcirc-current-line (1+ rcirc-current-line)))))) + + (set-marker-insertion-type rcirc-prompt-start-marker nil) + (set-marker-insertion-type rcirc-prompt-end-marker nil) + + ;; truncate buffer if it is very long + (save-excursion + (when (and rcirc-buffer-maximum-lines + (> rcirc-buffer-maximum-lines 0) + (= (forward-line (- rcirc-buffer-maximum-lines)) 0)) + (delete-region (point-min) (point)))) + + ;; set the window point for buffers show in windows + (walk-windows (lambda (w) + (when (and (not (eq (selected-window) w)) + (eq (current-buffer) + (window-buffer w)) + (>= (window-point w) + rcirc-prompt-end-marker)) + (set-window-point w (point-max)))) + nil t) + + ;; restore the point + (goto-char (if moving rcirc-prompt-end-marker old-point)) + + ;; keep window on bottom line if it was already there + (when rcirc-scroll-show-maximum-output + (let ((window (get-buffer-window))) + (when window + (with-selected-window window + (when (eq major-mode 'rcirc-mode) + (when (<= (- (window-height) + (count-screen-lines (window-point) + (window-start)) + 1) + 0) + (recenter -1))))))) + + ;; flush undo (can we do something smarter here?) + (buffer-disable-undo) + (buffer-enable-undo)) + + ;; record mode line activity + (when (and activity + (not rcirc-ignore-buffer-activity-flag) + (not (and rcirc-dim-nicks sender + (string-match (regexp-opt rcirc-dim-nicks) sender) + (rcirc-channel-p target)))) + (rcirc-record-activity (current-buffer) + (when (not (rcirc-channel-p rcirc-target)) + 'nick))) + + (when (and rcirc-log-flag + (or target + rcirc-log-process-buffers)) + (rcirc-log process sender response target text)) + + (sit-for 0) ; displayed text before hook + (run-hook-with-args 'rcirc-print-functions + process sender response target text))))) + +(defun rcirc-generate-log-filename (process target) + (if target + (rcirc-generate-new-buffer-name process target) + (process-name process))) + +(defcustom rcirc-log-filename-function 'rcirc-generate-log-filename + "A function to generate the filename used by rcirc's logging facility. + +It is called with two arguments, PROCESS and TARGET (see +`rcirc-generate-new-buffer-name' for their meaning), and should +return the filename, or nil if no logging is desired for this +session. + +If the returned filename is absolute (`file-name-absolute-p' +returns t), then it is used as-is, otherwise the resulting file +is put into `rcirc-log-directory'. + +The filename is then cleaned using `convert-standard-filename' to +guarantee valid filenames for the current OS." + :group 'rcirc + :type 'function) + +(defun rcirc-log (process sender response target text) + "Record line in `rcirc-log', to be later written to disk." + (let ((filename (funcall rcirc-log-filename-function process target))) + (unless (null filename) + (let ((cell (assoc-string filename rcirc-log-alist)) + (line (concat (format-time-string rcirc-time-format) + (substring-no-properties + (rcirc-format-response-string process sender + response target text)) + "\n"))) + (if cell + (setcdr cell (concat (cdr cell) line)) + (setq rcirc-log-alist + (cons (cons filename line) rcirc-log-alist))))))) + +(defun rcirc-log-write () + "Flush `rcirc-log-alist' data to disk. + +Log data is written to `rcirc-log-directory', except for +log-files with absolute names (see `rcirc-log-filename-function')." + (dolist (cell rcirc-log-alist) + (let ((filename (convert-standard-filename + (expand-file-name (car cell) + rcirc-log-directory))) + (coding-system-for-write 'utf-8)) + (make-directory (file-name-directory filename) t) + (with-temp-buffer + (insert (cdr cell)) + (write-region (point-min) (point-max) filename t 'quiet)))) + (setq rcirc-log-alist nil)) + +(defun rcirc-view-log-file () + "View logfile corresponding to the current buffer." + (interactive) + (find-file-other-window + (expand-file-name (funcall rcirc-log-filename-function + (rcirc-buffer-process) rcirc-target) + rcirc-log-directory))) + +(defun rcirc-join-channels (process channels) + "Join CHANNELS." + (save-window-excursion + (dolist (channel channels) + (with-rcirc-process-buffer process + (rcirc-cmd-join channel process))))) + +;;; nick management +(defvar rcirc-nick-prefix-chars "~&@%+") +(defun rcirc-user-nick (user) + "Return the nick from USER. Remove any non-nick junk." + (save-match-data + (if (string-match (concat "^[" rcirc-nick-prefix-chars + "]?\\([^! ]+\\)!?") (or user "")) + (match-string 1 user) + user))) + +(defun rcirc-nick-channels (process nick) + "Return list of channels for NICK." + (with-rcirc-process-buffer process + (mapcar (lambda (x) (car x)) + (gethash nick rcirc-nick-table)))) + +(defun rcirc-put-nick-channel (process nick channel &optional line) + "Add CHANNEL to list associated with NICK. +Update the associated linestamp if LINE is non-nil. + +If the record doesn't exist, and LINE is nil, set the linestamp +to zero." + (let ((nick (rcirc-user-nick nick))) + (with-rcirc-process-buffer process + (let* ((chans (gethash nick rcirc-nick-table)) + (record (assoc-string channel chans t))) + (if record + (when line (setcdr record line)) + (puthash nick (cons (cons channel (or line 0)) + chans) + rcirc-nick-table)))))) + +(defun rcirc-nick-remove (process nick) + "Remove NICK from table." + (with-rcirc-process-buffer process + (remhash nick rcirc-nick-table))) + +(defun rcirc-remove-nick-channel (process nick channel) + "Remove the CHANNEL from list associated with NICK." + (with-rcirc-process-buffer process + (let* ((chans (gethash nick rcirc-nick-table)) + (newchans + ;; instead of assoc-string-delete-all: + (let ((record (assoc-string channel chans t))) + (when record + (setcar record 'delete) + (assq-delete-all 'delete chans))))) + (if newchans + (puthash nick newchans rcirc-nick-table) + (remhash nick rcirc-nick-table))))) + +(defun rcirc-channel-nicks (process target) + "Return the list of nicks associated with TARGET sorted by last activity." + (when target + (if (rcirc-channel-p target) + (with-rcirc-process-buffer process + (let (nicks) + (maphash + (lambda (k v) + (let ((record (assoc-string target v t))) + (if record + (setq nicks (cons (cons k (cdr record)) nicks))))) + rcirc-nick-table) + (mapcar (lambda (x) (car x)) + (sort nicks (lambda (x y) + (let ((lx (or (cdr x) 0)) + (ly (or (cdr y) 0))) + (< ly lx))))))) + (list target)))) + +(defun rcirc-ignore-update-automatic (nick) + "Remove NICK from `rcirc-ignore-list' +if NICK is also on `rcirc-ignore-list-automatic'." + (when (member nick rcirc-ignore-list-automatic) + (setq rcirc-ignore-list-automatic + (delete nick rcirc-ignore-list-automatic) + rcirc-ignore-list + (delete nick rcirc-ignore-list)))) + +(defun rcirc-nickname< (s1 s2) + "Return t if IRC nickname S1 is less than S2, and nil otherwise. +Operator nicknames (@) are considered less than voiced +nicknames (+). Any other nicknames are greater than voiced +nicknames. The comparison is case-insensitive." + (setq s1 (downcase s1) + s2 (downcase s2)) + (let* ((s1-op (eq ?@ (string-to-char s1))) + (s2-op (eq ?@ (string-to-char s2)))) + (if s1-op + (if s2-op + (string< (substring s1 1) (substring s2 1)) + t) + (if s2-op + nil + (string< s1 s2))))) + +(defun rcirc-sort-nicknames-join (input sep) + "Return a string of sorted nicknames. +INPUT is a string containing nicknames separated by SEP. +This function does not alter the INPUT string." + (let* ((parts (split-string input sep t)) + (sorted (sort parts 'rcirc-nickname<))) + (mapconcat 'identity sorted sep))) + +;;; activity tracking +(defvar rcirc-track-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map (kbd "C-c C-@") 'rcirc-next-active-buffer) + (define-key map (kbd "C-c C-SPC") 'rcirc-next-active-buffer) + map) + "Keymap for rcirc track minor mode.") + +;;;###autoload +(define-minor-mode rcirc-track-minor-mode + "Global minor mode for tracking activity in rcirc buffers. +With a prefix argument ARG, enable the mode if ARG is positive, +and disable it otherwise. If called from Lisp, enable the mode +if ARG is omitted or nil." + :init-value nil + :lighter "" + :keymap rcirc-track-minor-mode-map + :global t + :group 'rcirc + (or global-mode-string (setq global-mode-string '(""))) + ;; toggle the mode-line channel indicator + (if rcirc-track-minor-mode + (progn + (and (not (memq 'rcirc-activity-string global-mode-string)) + (setq global-mode-string + (append global-mode-string '(rcirc-activity-string)))) + (add-hook 'window-configuration-change-hook + 'rcirc-window-configuration-change)) + (setq global-mode-string + (delete 'rcirc-activity-string global-mode-string)) + (remove-hook 'window-configuration-change-hook + 'rcirc-window-configuration-change))) + +(or (assq 'rcirc-ignore-buffer-activity-flag minor-mode-alist) + (setq minor-mode-alist + (cons '(rcirc-ignore-buffer-activity-flag " Ignore") minor-mode-alist))) +(or (assq 'rcirc-low-priority-flag minor-mode-alist) + (setq minor-mode-alist + (cons '(rcirc-low-priority-flag " LowPri") minor-mode-alist))) + +(defun rcirc-toggle-ignore-buffer-activity () + "Toggle the value of `rcirc-ignore-buffer-activity-flag'." + (interactive) + (setq rcirc-ignore-buffer-activity-flag + (not rcirc-ignore-buffer-activity-flag)) + (message (if rcirc-ignore-buffer-activity-flag + "Ignore activity in this buffer" + "Notice activity in this buffer")) + (force-mode-line-update)) + +(defun rcirc-toggle-low-priority () + "Toggle the value of `rcirc-low-priority-flag'." + (interactive) + (setq rcirc-low-priority-flag + (not rcirc-low-priority-flag)) + (message (if rcirc-low-priority-flag + "Activity in this buffer is low priority" + "Activity in this buffer is normal priority")) + (force-mode-line-update)) + +(defun rcirc-switch-to-server-buffer () + "Switch to the server buffer associated with current channel buffer." + (interactive) + (unless (buffer-live-p rcirc-server-buffer) + (error "No such buffer")) + (switch-to-buffer rcirc-server-buffer)) + +(defun rcirc-jump-to-first-unread-line () + "Move the point to the first unread line in this buffer." + (interactive) + (if (marker-position overlay-arrow-position) + (goto-char overlay-arrow-position) + (message "No unread messages"))) + +(defun rcirc-bury-buffers () + "Bury all RCIRC buffers." + (interactive) + (dolist (buf (buffer-list)) + (when (eq 'rcirc-mode (with-current-buffer buf major-mode)) + (bury-buffer buf) ; buffers not shown + (quit-windows-on buf)))) ; buffers shown in a window + +(defun rcirc-next-active-buffer (arg) + "Switch to the next rcirc buffer with activity. +With prefix ARG, go to the next low priority buffer with activity." + (interactive "P") + (let* ((pair (rcirc-split-activity rcirc-activity)) + (lopri (car pair)) + (hipri (cdr pair))) + (if (or (and (not arg) hipri) + (and arg lopri)) + (progn + (switch-to-buffer (car (if arg lopri hipri))) + (when (> (point) rcirc-prompt-start-marker) + (recenter -1))) + (rcirc-bury-buffers) + (message "No IRC activity.%s" + (if lopri + (concat + " Type C-u " (key-description (this-command-keys)) + " for low priority activity.") + ""))))) + +(define-obsolete-variable-alias 'rcirc-activity-hooks + 'rcirc-activity-functions "24.3") +(defvar rcirc-activity-functions nil + "Hook to be run when there is channel activity. + +Functions are called with a single argument, the buffer with the +activity. Only run if the buffer is not visible and +`rcirc-ignore-buffer-activity-flag' is non-nil.") + +(defun rcirc-record-activity (buffer &optional type) + "Record BUFFER activity with TYPE." + (with-current-buffer buffer + (let ((old-activity rcirc-activity) + (old-types rcirc-activity-types)) + (when (not (get-buffer-window (current-buffer) t)) + (setq rcirc-activity + (sort (if (memq (current-buffer) rcirc-activity) rcirc-activity + (cons (current-buffer) rcirc-activity)) + (lambda (b1 b2) + (let ((t1 (with-current-buffer b1 rcirc-last-post-time)) + (t2 (with-current-buffer b2 rcirc-last-post-time))) + (time-less-p t2 t1))))) + (cl-pushnew type rcirc-activity-types) + (unless (and (equal rcirc-activity old-activity) + (member type old-types)) + (rcirc-update-activity-string))))) + (run-hook-with-args 'rcirc-activity-functions buffer)) + +(defun rcirc-clear-activity (buffer) + "Clear the BUFFER activity." + (setq rcirc-activity (remove buffer rcirc-activity)) + (with-current-buffer buffer + (setq rcirc-activity-types nil))) + +(defun rcirc-clear-unread (buffer) + "Erase the last read message arrow from BUFFER." + (when (buffer-live-p buffer) + (with-current-buffer buffer + (set-marker overlay-arrow-position nil)))) + +(defun rcirc-split-activity (activity) + "Return a cons cell with ACTIVITY split into (lopri . hipri)." + (let (lopri hipri) + (dolist (buf activity) + (with-current-buffer buf + (if (and rcirc-low-priority-flag + (not (member 'nick rcirc-activity-types))) + (push buf lopri) + (push buf hipri)))) + (cons (nreverse lopri) (nreverse hipri)))) + +(defvar rcirc-update-activity-string-hook nil + "Hook run whenever the activity string is updated.") + +;; TODO: add mouse properties +(defun rcirc-update-activity-string () + "Update mode-line string." + (let* ((pair (rcirc-split-activity rcirc-activity)) + (lopri (car pair)) + (hipri (cdr pair))) + (setq rcirc-activity-string + (cond ((or hipri lopri) + (concat (and hipri "[") + (rcirc-activity-string hipri) + (and hipri lopri ",") + (and lopri + (concat "(" + (rcirc-activity-string lopri) + ")")) + (and hipri "]"))) + ((not (null (rcirc-process-list))) + "[]") + (t "[]"))) + (run-hooks 'rcirc-update-activity-string-hook))) + +(defun rcirc-activity-string (buffers) + (mapconcat (lambda (b) + (let ((s (substring-no-properties (rcirc-short-buffer-name b)))) + (with-current-buffer b + (dolist (type rcirc-activity-types) + (rcirc-add-face 0 (length s) + (cl-case type + (nick 'rcirc-track-nick) + (keyword 'rcirc-track-keyword)) + s))) + s)) + buffers ",")) + +(defun rcirc-short-buffer-name (buffer) + "Return a short name for BUFFER to use in the mode line indicator." + (with-current-buffer buffer + (or rcirc-short-buffer-name (buffer-name)))) + +(defun rcirc-visible-buffers () + "Return a list of the visible buffers that are in rcirc-mode." + (let (acc) + (walk-windows (lambda (w) + (with-current-buffer (window-buffer w) + (when (eq major-mode 'rcirc-mode) + (push (current-buffer) acc))))) + acc)) + +(defvar rcirc-visible-buffers nil) +(defun rcirc-window-configuration-change () + (unless (minibuffer-window-active-p (minibuffer-window)) + ;; delay this until command has finished to make sure window is + ;; actually visible before clearing activity + (add-hook 'post-command-hook 'rcirc-window-configuration-change-1))) + +(defun rcirc-window-configuration-change-1 () + ;; clear activity and overlay arrows + (let* ((old-activity rcirc-activity) + (hidden-buffers rcirc-visible-buffers)) + + (setq rcirc-visible-buffers (rcirc-visible-buffers)) + + (dolist (vbuf rcirc-visible-buffers) + (setq hidden-buffers (delq vbuf hidden-buffers)) + ;; clear activity for all visible buffers + (rcirc-clear-activity vbuf)) + + ;; clear unread arrow from recently hidden buffers + (dolist (hbuf hidden-buffers) + (rcirc-clear-unread hbuf)) + + ;; remove any killed buffers from list + (setq rcirc-activity + (delq nil (mapcar (lambda (buf) (when (buffer-live-p buf) buf)) + rcirc-activity))) + ;; update the mode-line string + (unless (equal old-activity rcirc-activity) + (rcirc-update-activity-string))) + + (remove-hook 'post-command-hook 'rcirc-window-configuration-change-1)) + + +;;; buffer name abbreviation +(defun rcirc-update-short-buffer-names () + (let ((bufalist + (apply 'append (mapcar (lambda (process) + (with-rcirc-process-buffer process + rcirc-buffer-alist)) + (rcirc-process-list))))) + (dolist (i (rcirc-abbreviate bufalist)) + (when (buffer-live-p (cdr i)) + (with-current-buffer (cdr i) + (setq rcirc-short-buffer-name (car i))))))) + +(defun rcirc-abbreviate (pairs) + (apply 'append (mapcar 'rcirc-rebuild-tree (rcirc-make-trees pairs)))) + +(defun rcirc-rebuild-tree (tree &optional acc) + (let ((ch (char-to-string (car tree)))) + (dolist (x (cdr tree)) + (if (listp x) + (setq acc (append acc + (mapcar (lambda (y) + (cons (concat ch (car y)) + (cdr y))) + (rcirc-rebuild-tree x)))) + (setq acc (cons (cons ch x) acc)))) + acc)) + +(defun rcirc-make-trees (pairs) + (let (alist) + (mapc (lambda (pair) + (if (consp pair) + (let* ((str (car pair)) + (data (cdr pair)) + (char (unless (zerop (length str)) + (aref str 0))) + (rest (unless (zerop (length str)) + (substring str 1))) + (part (if char (assq char alist)))) + (if part + ;; existing partition + (setcdr part (cons (cons rest data) (cdr part))) + ;; new partition + (setq alist (cons (if char + (list char (cons rest data)) + data) + alist)))) + (setq alist (cons pair alist)))) + pairs) + ;; recurse into cdrs of alist + (mapc (lambda (x) + (when (and (listp x) (listp (cadr x))) + (setcdr x (if (> (length (cdr x)) 1) + (rcirc-make-trees (cdr x)) + (setcdr x (list (cl-cdadr x))))))) + alist))) + +;;; /commands these are called with 3 args: PROCESS, TARGET, which is +;; the current buffer/channel/user, and ARGS, which is a string +;; containing the text following the /cmd. + +(defmacro defun-rcirc-command (command argument docstring interactive-form + &rest body) + "Define a command." + `(progn + (add-to-list 'rcirc-client-commands ,(concat "/" (symbol-name command))) + (defun ,(intern (concat "rcirc-cmd-" (symbol-name command))) + (,@argument &optional process target) + ,(concat docstring "\n\nNote: If PROCESS or TARGET are nil, the values given" + "\nby `rcirc-buffer-process' and `rcirc-target' will be used.") + ,interactive-form + (let ((process (or process (rcirc-buffer-process))) + (target (or target rcirc-target))) + (ignore target) ; mark `target' variable as ignorable + ,@body)))) + +(defun-rcirc-command msg (message) + "Send private MESSAGE to TARGET." + (interactive "i") + (if (null message) + (progn + (setq target (completing-read "Message nick: " + (with-rcirc-server-buffer + rcirc-nick-table))) + (when (> (length target) 0) + (setq message (read-string (format "Message %s: " target))) + (when (> (length message) 0) + (rcirc-send-message process target message)))) + (if (not (string-match "\\([^ ]+\\) \\(.+\\)" message)) + (message "Not enough args, or something.") + (setq target (match-string 1 message) + message (match-string 2 message)) + (rcirc-send-message process target message)))) + +(defun-rcirc-command query (nick) + "Open a private chat buffer to NICK." + (interactive (list (completing-read "Query nick: " + (with-rcirc-server-buffer rcirc-nick-table)))) + (let ((existing-buffer (rcirc-get-buffer process nick))) + (switch-to-buffer (or existing-buffer + (rcirc-get-buffer-create process nick))) + (when (not existing-buffer) + (rcirc-cmd-whois nick)))) + +(defun-rcirc-command join (channels) + "Join CHANNELS. +CHANNELS is a comma- or space-separated string of channel names." + (interactive "sJoin channels: ") + (let* ((split-channels (split-string channels "[ ,]" t)) + (buffers (mapcar (lambda (ch) + (rcirc-get-buffer-create process ch)) + split-channels)) + (channels (mapconcat 'identity split-channels ","))) + (rcirc-send-string process (concat "JOIN " channels)) + (when (not (eq (selected-window) (minibuffer-window))) + (dolist (b buffers) ;; order the new channel buffers in the buffer list + (switch-to-buffer b))))) + +(defun-rcirc-command invite (nick-channel) + "Invite NICK to CHANNEL." + (interactive (list + (concat + (completing-read "Invite nick: " + (with-rcirc-server-buffer rcirc-nick-table)) + " " + (read-string "Channel: ")))) + (rcirc-send-string process (concat "INVITE " nick-channel))) + +;; TODO: /part #channel reason, or consider removing #channel altogether +(defun-rcirc-command part (channel) + "Part CHANNEL." + (interactive "sPart channel: ") + (let ((channel (if (> (length channel) 0) channel target))) + (rcirc-send-string process (concat "PART " channel " :" rcirc-id-string)))) + +(defun-rcirc-command quit (reason) + "Send a quit message to server with REASON." + (interactive "sQuit reason: ") + (rcirc-send-string process (concat "QUIT :" + (if (not (zerop (length reason))) + reason + rcirc-id-string)))) + +(defun-rcirc-command reconnect (_) + "Reconnect to current server." + (interactive "i") + (with-rcirc-server-buffer + (cond + (rcirc-connecting (message "Already connecting")) + ((process-live-p process) (message "Server process is alive")) + (t (let ((conn-info rcirc-connection-info)) + (setf (nth 5 conn-info) + (cl-remove-if-not #'rcirc-channel-p + (mapcar #'car rcirc-buffer-alist))) + (apply #'rcirc-connect conn-info)))))) + +(defun-rcirc-command nick (nick) + "Change nick to NICK." + (interactive "i") + (when (null nick) + (setq nick (read-string "New nick: " (rcirc-nick process)))) + (rcirc-send-string process (concat "NICK " nick))) + +(defun-rcirc-command names (channel) + "Display list of names in CHANNEL or in current channel if CHANNEL is nil. +If called interactively, prompt for a channel when prefix arg is supplied." + (interactive "P") + (if (called-interactively-p 'interactive) + (if channel + (setq channel (read-string "List names in channel: " target)))) + (let ((channel (if (> (length channel) 0) + channel + target))) + (rcirc-send-string process (concat "NAMES " channel)))) + +(defun-rcirc-command topic (topic) + "List TOPIC for the TARGET channel. +With a prefix arg, prompt for new topic." + (interactive "P") + (if (and (called-interactively-p 'interactive) topic) + (setq topic (read-string "New Topic: " rcirc-topic))) + (rcirc-send-string process (concat "TOPIC " target + (when (> (length topic) 0) + (concat " :" topic))))) + +(defun-rcirc-command whois (nick) + "Request information from server about NICK." + (interactive (list + (completing-read "Whois: " + (with-rcirc-server-buffer rcirc-nick-table)))) + (rcirc-send-string process (concat "WHOIS " nick))) + +(defun-rcirc-command mode (args) + "Set mode with ARGS." + (interactive (list (concat (read-string "Mode nick or channel: ") + " " (read-string "Mode: ")))) + (rcirc-send-string process (concat "MODE " args))) + +(defun-rcirc-command list (channels) + "Request information on CHANNELS from server." + (interactive "sList Channels: ") + (rcirc-send-string process (concat "LIST " channels))) + +(defun-rcirc-command oper (args) + "Send operator command to server." + (interactive "sOper args: ") + (rcirc-send-string process (concat "OPER " args))) + +(defun-rcirc-command quote (message) + "Send MESSAGE literally to server." + (interactive "sServer message: ") + (rcirc-send-string process message)) + +(defun-rcirc-command kick (arg) + "Kick NICK from current channel." + (interactive (list + (concat (completing-read "Kick nick: " + (rcirc-channel-nicks + (rcirc-buffer-process) + rcirc-target)) + (read-from-minibuffer "Kick reason: ")))) + (let* ((arglist (split-string arg)) + (argstring (concat (car arglist) " :" + (mapconcat 'identity (cdr arglist) " ")))) + (rcirc-send-string process (concat "KICK " target " " argstring)))) + +(defun rcirc-cmd-ctcp (args &optional process _target) + (if (string-match "^\\([^ ]+\\)\\s-+\\(.+\\)$" args) + (let* ((target (match-string 1 args)) + (request (upcase (match-string 2 args))) + (function (intern-soft (concat "rcirc-ctcp-sender-" request)))) + (if (fboundp function) ;; use special function if available + (funcall function process target request) + (rcirc-send-ctcp process target request))) + (rcirc-print process (rcirc-nick process) "ERROR" nil + "usage: /ctcp NICK REQUEST"))) + +(defun rcirc-ctcp-sender-PING (process target _request) + "Send a CTCP PING message to TARGET." + (let ((timestamp (format-time-string "%s"))) + (rcirc-send-ctcp process target "PING" timestamp))) + +(defun rcirc-cmd-me (args &optional process target) + (rcirc-send-ctcp process target "ACTION" args)) + +(defun rcirc-add-or-remove (set &rest elements) + (dolist (elt elements) + (if (and elt (not (string= "" elt))) + (setq set (if (member-ignore-case elt set) + (delete elt set) + (cons elt set))))) + set) + +(defun-rcirc-command ignore (nick) + "Manage the ignore list. +Ignore NICK, unignore NICK if already ignored, or list ignored +nicks when no NICK is given. When listing ignored nicks, the +ones added to the list automatically are marked with an asterisk." + (interactive "sToggle ignoring of nick: ") + (setq rcirc-ignore-list + (apply #'rcirc-add-or-remove rcirc-ignore-list + (split-string nick nil t))) + (rcirc-print process nil "IGNORE" target + (mapconcat + (lambda (nick) + (concat nick + (if (member nick rcirc-ignore-list-automatic) + "*" ""))) + rcirc-ignore-list " "))) + +(defun-rcirc-command bright (nick) + "Manage the bright nick list." + (interactive "sToggle emphasis of nick: ") + (setq rcirc-bright-nicks + (apply #'rcirc-add-or-remove rcirc-bright-nicks + (split-string nick nil t))) + (rcirc-print process nil "BRIGHT" target + (mapconcat 'identity rcirc-bright-nicks " "))) + +(defun-rcirc-command dim (nick) + "Manage the dim nick list." + (interactive "sToggle deemphasis of nick: ") + (setq rcirc-dim-nicks + (apply #'rcirc-add-or-remove rcirc-dim-nicks + (split-string nick nil t))) + (rcirc-print process nil "DIM" target + (mapconcat 'identity rcirc-dim-nicks " "))) + +(defun-rcirc-command keyword (keyword) + "Manage the keyword list. +Mark KEYWORD, unmark KEYWORD if already marked, or list marked +keywords when no KEYWORD is given." + (interactive "sToggle highlighting of keyword: ") + (setq rcirc-keywords + (apply #'rcirc-add-or-remove rcirc-keywords + (split-string keyword nil t))) + (rcirc-print process nil "KEYWORD" target + (mapconcat 'identity rcirc-keywords " "))) + + +(defun rcirc-add-face (start end name &optional object) + "Add face NAME to the face text property of the text from START to END." + (when name + (let ((pos start) + next prop) + (while (< pos end) + (setq prop (get-text-property pos 'font-lock-face object) + next (next-single-property-change pos 'font-lock-face object end)) + (unless (member name (get-text-property pos 'font-lock-face object)) + (add-text-properties pos next + (list 'font-lock-face (cons name prop)) object)) + (setq pos next))))) + +(defun rcirc-facify (string face) + "Return a copy of STRING with FACE property added." + (let ((string (or string ""))) + (rcirc-add-face 0 (length string) face string) + string)) + +(defvar rcirc-url-regexp + (concat + "\\b\\(\\(www\\.\\|\\(s?https?\\|ftp\\|file\\|gopher\\|" + "nntp\\|news\\|telnet\\|wais\\|mailto\\|info\\):\\)" + "\\(//[-a-z0-9_.]+:[0-9]*\\)?" + (if (string-match "[[:digit:]]" "1") ;; Support POSIX? + (let ((chars "-a-z0-9_=#$@~%&*+\\/[:word:]") + (punct "!?:;.,")) + (concat + "\\(?:" + ;; Match paired parentheses, e.g. in Wikipedia URLs: + "[" chars punct "]+" "(" "[" chars punct "]+" "[" chars "]*)" "[" chars "]" + "\\|" + "[" chars punct "]+" "[" chars "]" + "\\)")) + (concat ;; XEmacs 21.4 doesn't support POSIX. + "\\([-a-z0-9_=!?#$@~%&*+\\/:;.,]\\|\\w\\)+" + "\\([-a-z0-9_=#$@~%&*+\\/]\\|\\w\\)")) + "\\)") + "Regexp matching URLs. Set to nil to disable URL features in rcirc.") + +;; cf cl-remove-if-not +(defun rcirc-condition-filter (condp lst) + "Remove all items not satisfying condition CONDP in list LST. +CONDP is a function that takes a list element as argument and returns +non-nil if that element should be included. Returns a new list." + (delq nil (mapcar (lambda (x) (and (funcall condp x) x)) lst))) + +(defun rcirc-browse-url (&optional arg) + "Prompt for URL to browse based on URLs in buffer before point. + +If ARG is given, opens the URL in a new browser window." + (interactive "P") + (let* ((point (point)) + (filtered (rcirc-condition-filter + (lambda (x) (>= point (cdr x))) + rcirc-urls)) + (completions (mapcar (lambda (x) (car x)) filtered)) + (defaults (mapcar (lambda (x) (car x)) filtered))) + (browse-url (completing-read "Rcirc browse-url: " + completions nil nil (car defaults) nil defaults) + arg))) + +(defun rcirc-markup-timestamp (_sender _response) + (goto-char (point-min)) + (insert (rcirc-facify (format-time-string rcirc-time-format + (let ((time rcirc-last-message-time)) + (when time (setq rcirc-last-message-time nil)) + time)) + 'rcirc-timestamp))) + +(defun rcirc-markup-attributes (_sender _response) + (while (re-search-forward "\\([\C-b\C-_\C-v]\\).*?\\(\\1\\|\C-o\\)" nil t) + (rcirc-add-face (match-beginning 0) (match-end 0) + (cl-case (char-after (match-beginning 1)) + (?\C-b 'bold) + (?\C-v 'italic) + (?\C-_ 'underline))) + ;; keep the ^O since it could terminate other attributes + (when (not (eq ?\C-o (char-before (match-end 2)))) + (delete-region (match-beginning 2) (match-end 2))) + (delete-region (match-beginning 1) (match-end 1)) + (goto-char (match-beginning 1))) + ;; remove the ^O characters now + (goto-char (point-min)) + (while (re-search-forward "\C-o+" nil t) + (delete-region (match-beginning 0) (match-end 0)))) + +(defun rcirc-markup-my-nick (_sender response) + (with-syntax-table rcirc-nick-syntax-table + (while (re-search-forward (concat "\\b" + (regexp-quote (rcirc-nick + (rcirc-buffer-process))) + "\\b") + nil t) + (rcirc-add-face (match-beginning 0) (match-end 0) + 'rcirc-nick-in-message) + (when (string= response "PRIVMSG") + (rcirc-add-face (point-min) (point-max) + 'rcirc-nick-in-message-full-line) + (rcirc-record-activity (current-buffer) 'nick))))) + +(defun rcirc-markup-urls (_sender _response) + (while (and rcirc-url-regexp ;; nil means disable URL catching + (re-search-forward rcirc-url-regexp nil t)) + (let* ((start (match-beginning 0)) + (end (match-end 0)) + (url (match-string-no-properties 0)) + (link-text (buffer-substring-no-properties start end))) + ;; Add a button for the URL. Note that we use `make-text-button', + ;; rather than `make-button', as text-buttons are much faster in + ;; large buffers. + (make-text-button start end + 'face 'rcirc-url + 'follow-link t + 'rcirc-url url + 'action (lambda (button) + (browse-url (button-get button 'rcirc-url)))) + ;; record the url if it is not already the latest stored url + (when (not (string= link-text (caar rcirc-urls))) + (push (cons link-text start) rcirc-urls))))) + +(defun rcirc-markup-keywords (sender response) + (when (and (string= response "PRIVMSG") + (not (string= sender (rcirc-nick (rcirc-buffer-process))))) + (let* ((target (or rcirc-target "")) + (keywords (delq nil (mapcar (lambda (keyword) + (when (not (string-match keyword + target)) + keyword)) + rcirc-keywords)))) + (when keywords + (while (re-search-forward (regexp-opt keywords 'words) nil t) + (rcirc-add-face (match-beginning 0) (match-end 0) 'rcirc-keyword) + (rcirc-record-activity (current-buffer) 'keyword)))))) + +(defun rcirc-markup-bright-nicks (_sender response) + (when (and rcirc-bright-nicks + (string= response "NAMES")) + (with-syntax-table rcirc-nick-syntax-table + (while (re-search-forward (regexp-opt rcirc-bright-nicks 'words) nil t) + (rcirc-add-face (match-beginning 0) (match-end 0) + 'rcirc-bright-nick))))) + +(defun rcirc-markup-fill (_sender response) + (when (not (string= response "372")) ; /motd + (let ((fill-prefix + (or rcirc-fill-prefix + (make-string (- (point) (line-beginning-position)) ?\s))) + (fill-column (- (cond ((null rcirc-fill-column) fill-column) + ((functionp rcirc-fill-column) + (funcall rcirc-fill-column)) + (t rcirc-fill-column)) + ;; make sure ... doesn't cause line wrapping + 3))) + (fill-region (point) (point-max) nil t)))) + +;;; handlers +;; these are called with the server PROCESS, the SENDER, which is a +;; server or a user, depending on the command, the ARGS, which is a +;; list of strings, and the TEXT, which is the original server text, +;; verbatim +(defun rcirc-handler-001 (process sender args text) + (rcirc-handler-generic process "001" sender args text) + (with-rcirc-process-buffer process + (setq rcirc-connecting nil) + (rcirc-reschedule-timeout process) + (setq rcirc-server-name sender) + (setq rcirc-nick (car args)) + (rcirc-update-prompt) + (if rcirc-auto-authenticate-flag + (if (and rcirc-authenticate-before-join + ;; We have to ensure that there's an authentication + ;; entry for that server. Else, + ;; rcirc-authenticated-hook won't be triggered, and + ;; autojoin won't happen at all. + (let (auth-required) + (dolist (s rcirc-authinfo auth-required) + (when (string-match (car s) rcirc-server-name) + (setq auth-required t))))) + (progn + (add-hook 'rcirc-authenticated-hook 'rcirc-join-channels-post-auth t t) + (rcirc-authenticate)) + (rcirc-authenticate) + (rcirc-join-channels process rcirc-startup-channels)) + (rcirc-join-channels process rcirc-startup-channels)))) + +(defun rcirc-join-channels-post-auth (process) + "Join `rcirc-startup-channels' after authenticating." + (with-rcirc-process-buffer process + (rcirc-join-channels process rcirc-startup-channels))) + +(defun rcirc-handler-PRIVMSG (process sender args text) + (rcirc-check-auth-status process sender args text) + (let ((target (if (rcirc-channel-p (car args)) + (car args) + sender)) + (message (or (cadr args) ""))) + (if (string-match "^\C-a\\(.*\\)\C-a$" message) + (rcirc-handler-CTCP process target sender (match-string 1 message)) + (rcirc-print process sender "PRIVMSG" target message t)) + ;; update nick linestamp + (with-current-buffer (rcirc-get-buffer process target t) + (rcirc-put-nick-channel process sender target rcirc-current-line)))) + +(defun rcirc-handler-NOTICE (process sender args text) + (rcirc-check-auth-status process sender args text) + (let ((target (car args)) + (message (cadr args))) + (if (string-match "^\C-a\\(.*\\)\C-a$" message) + (rcirc-handler-CTCP-response process target sender + (match-string 1 message)) + (rcirc-print process sender "NOTICE" + (cond ((rcirc-channel-p target) + target) + ;;; -ChanServ- [#gnu] Welcome... + ((string-match "\\[\\(#[^] ]+\\)\\]" message) + (match-string 1 message)) + (sender + (if (string= sender (rcirc-server-name process)) + nil ; server notice + sender))) + message t)))) + +(defun rcirc-check-auth-status (process sender args _text) + "Check if the user just authenticated. +If authenticated, runs `rcirc-authenticated-hook' with PROCESS as +the only argument." + (with-rcirc-process-buffer process + (when (and (not rcirc-user-authenticated) + rcirc-authenticate-before-join + rcirc-auto-authenticate-flag) + (let ((target (car args)) + (message (cadr args))) + (when (or + (and ;; nickserv + (string= sender "NickServ") + (string= target rcirc-nick) + (member message + (list + (format "You are now identified for \C-b%s\C-b." rcirc-nick) + (format "You are successfully identified as \C-b%s\C-b." rcirc-nick) + "Password accepted - you are now recognized." + ))) + (and ;; quakenet + (string= sender "Q") + (string= target rcirc-nick) + (string-match "\\`You are now logged in as .+\\.\\'" message))) + (setq rcirc-user-authenticated t) + (run-hook-with-args 'rcirc-authenticated-hook process) + (remove-hook 'rcirc-authenticated-hook 'rcirc-join-channels-post-auth t)))))) + +(defun rcirc-handler-WALLOPS (process sender args _text) + (rcirc-print process sender "WALLOPS" sender (car args) t)) + +(defun rcirc-handler-JOIN (process sender args _text) + (let ((channel (car args))) + (with-current-buffer (rcirc-get-buffer-create process channel) + ;; when recently rejoining, restore the linestamp + (rcirc-put-nick-channel process sender channel + (let ((last-activity-lines + (rcirc-elapsed-lines process sender channel))) + (when (and last-activity-lines + (< last-activity-lines rcirc-omit-threshold)) + (rcirc-last-line process sender channel)))) + ;; reset mode-line-process in case joining a channel with an + ;; already open buffer (after getting kicked e.g.) + (setq mode-line-process nil)) + + (rcirc-print process sender "JOIN" channel "") + + ;; print in private chat buffer if it exists + (when (rcirc-get-buffer (rcirc-buffer-process) sender) + (rcirc-print process sender "JOIN" sender channel)))) + +;; PART and KICK are handled the same way +(defun rcirc-handler-PART-or-KICK (process _response channel _sender nick _args) + (rcirc-ignore-update-automatic nick) + (if (not (string= nick (rcirc-nick process))) + ;; this is someone else leaving + (progn + (rcirc-maybe-remember-nick-quit process nick channel) + (rcirc-remove-nick-channel process nick channel)) + ;; this is us leaving + (mapc (lambda (n) + (rcirc-remove-nick-channel process n channel)) + (rcirc-channel-nicks process channel)) + + ;; if the buffer is still around, make it inactive + (let ((buffer (rcirc-get-buffer process channel))) + (when buffer + (rcirc-disconnect-buffer buffer))))) + +(defun rcirc-handler-PART (process sender args _text) + (let* ((channel (car args)) + (reason (cadr args)) + (message (concat channel " " reason))) + (rcirc-print process sender "PART" channel message) + ;; print in private chat buffer if it exists + (when (rcirc-get-buffer (rcirc-buffer-process) sender) + (rcirc-print process sender "PART" sender message)) + + (rcirc-handler-PART-or-KICK process "PART" channel sender sender reason))) + +(defun rcirc-handler-KICK (process sender args _text) + (let* ((channel (car args)) + (nick (cadr args)) + (reason (nth 2 args)) + (message (concat nick " " channel " " reason))) + (rcirc-print process sender "KICK" channel message t) + ;; print in private chat buffer if it exists + (when (rcirc-get-buffer (rcirc-buffer-process) nick) + (rcirc-print process sender "KICK" nick message)) + + (rcirc-handler-PART-or-KICK process "KICK" channel sender nick reason))) + +(defun rcirc-maybe-remember-nick-quit (process nick channel) + "Remember NICK as leaving CHANNEL if they recently spoke." + (let ((elapsed-lines (rcirc-elapsed-lines process nick channel))) + (when (and elapsed-lines + (< elapsed-lines rcirc-omit-threshold)) + (let ((buffer (rcirc-get-buffer process channel))) + (when buffer + (with-current-buffer buffer + (let ((record (assoc-string nick rcirc-recent-quit-alist t)) + (line (rcirc-last-line process nick channel))) + (if record + (setcdr record line) + (setq rcirc-recent-quit-alist + (cons (cons nick line) + rcirc-recent-quit-alist)))))))))) + +(defun rcirc-handler-QUIT (process sender args _text) + (rcirc-ignore-update-automatic sender) + (mapc (lambda (channel) + ;; broadcast quit message each channel + (rcirc-print process sender "QUIT" channel (apply 'concat args)) + ;; record nick in quit table if they recently spoke + (rcirc-maybe-remember-nick-quit process sender channel)) + (rcirc-nick-channels process sender)) + (rcirc-nick-remove process sender)) + +(defun rcirc-handler-NICK (process sender args _text) + (let* ((old-nick sender) + (new-nick (car args)) + (channels (rcirc-nick-channels process old-nick))) + ;; update list of ignored nicks + (rcirc-ignore-update-automatic old-nick) + (when (member old-nick rcirc-ignore-list) + (add-to-list 'rcirc-ignore-list new-nick) + (add-to-list 'rcirc-ignore-list-automatic new-nick)) + ;; print message to nick's channels + (dolist (target channels) + (rcirc-print process sender "NICK" target new-nick)) + ;; update private chat buffer, if it exists + (let ((chat-buffer (rcirc-get-buffer process old-nick))) + (when chat-buffer + (with-current-buffer chat-buffer + (rcirc-print process sender "NICK" old-nick new-nick) + (setq rcirc-target new-nick) + (rename-buffer (rcirc-generate-new-buffer-name process new-nick))))) + ;; remove old nick and add new one + (with-rcirc-process-buffer process + (let ((v (gethash old-nick rcirc-nick-table))) + (remhash old-nick rcirc-nick-table) + (puthash new-nick v rcirc-nick-table)) + ;; if this is our nick... + (when (string= old-nick rcirc-nick) + (setq rcirc-nick new-nick) + (rcirc-update-prompt t) + ;; reauthenticate + (when rcirc-auto-authenticate-flag (rcirc-authenticate)))))) + +(defun rcirc-handler-PING (process _sender args _text) + (rcirc-send-string process (concat "PONG :" (car args)))) + +(defun rcirc-handler-PONG (_process _sender _args _text) + ;; do nothing + ) + +(defun rcirc-handler-TOPIC (process sender args _text) + (let ((topic (cadr args))) + (rcirc-print process sender "TOPIC" (car args) topic) + (with-current-buffer (rcirc-get-buffer process (car args)) + (setq rcirc-topic topic)))) + +(defvar rcirc-nick-away-alist nil) +(defun rcirc-handler-301 (process _sender args text) + "RPL_AWAY" + (let* ((nick (cadr args)) + (rec (assoc-string nick rcirc-nick-away-alist)) + (away-message (nth 2 args))) + (when (or (not rec) + (not (string= (cdr rec) away-message))) + ;; away message has changed + (rcirc-handler-generic process "AWAY" nick (cdr args) text) + (if rec + (setcdr rec away-message) + (setq rcirc-nick-away-alist (cons (cons nick away-message) + rcirc-nick-away-alist)))))) + +(defun rcirc-handler-317 (process sender args _text) + "RPL_WHOISIDLE" + (let* ((nick (nth 1 args)) + (idle-secs (string-to-number (nth 2 args))) + (idle-string + (if (< idle-secs most-positive-fixnum) + (format-seconds "%yy %dd %hh %mm %z%ss" idle-secs) + "a very long time")) + (signon-time (seconds-to-time (string-to-number (nth 3 args)))) + (signon-string (format-time-string "%c" signon-time)) + (message (format "%s idle for %s, signed on %s" + nick idle-string signon-string))) + (rcirc-print process sender "317" nil message t))) + +(defun rcirc-handler-332 (process _sender args _text) + "RPL_TOPIC" + (let ((buffer (or (rcirc-get-buffer process (cadr args)) + (rcirc-get-temp-buffer-create process (cadr args))))) + (with-current-buffer buffer + (setq rcirc-topic (nth 2 args))))) + +(defun rcirc-handler-333 (process sender args _text) + "333 says who set the topic and when. +Not in rfc1459.txt" + (let ((buffer (or (rcirc-get-buffer process (cadr args)) + (rcirc-get-temp-buffer-create process (cadr args))))) + (with-current-buffer buffer + (let ((setter (nth 2 args)) + (time (current-time-string + (seconds-to-time + (string-to-number (cl-cadddr args)))))) + (rcirc-print process sender "TOPIC" (cadr args) + (format "%s (%s on %s)" rcirc-topic setter time)))))) + +(defun rcirc-handler-477 (process sender args _text) + "ERR_NOCHANMODES" + (rcirc-print process sender "477" (cadr args) (nth 2 args))) + +(defun rcirc-handler-MODE (process sender args _text) + (let ((target (car args)) + (msg (mapconcat 'identity (cdr args) " "))) + (rcirc-print process sender "MODE" + (if (string= target (rcirc-nick process)) + nil + target) + msg) + + ;; print in private chat buffers if they exist + (mapc (lambda (nick) + (when (rcirc-get-buffer process nick) + (rcirc-print process sender "MODE" nick msg))) + (cddr args)))) + +(defun rcirc-get-temp-buffer-create (process channel) + "Return a buffer based on PROCESS and CHANNEL." + (let ((tmpnam (concat " " (downcase channel) "TMP" (process-name process)))) + (get-buffer-create tmpnam))) + +(defun rcirc-handler-353 (process _sender args _text) + "RPL_NAMREPLY" + (let ((channel (nth 2 args)) + (names (or (nth 3 args) ""))) + (mapc (lambda (nick) + (rcirc-put-nick-channel process nick channel)) + (split-string names " " t)) + ;; create a temporary buffer to insert the names into + ;; rcirc-handler-366 (RPL_ENDOFNAMES) will handle it + (with-current-buffer (rcirc-get-temp-buffer-create process channel) + (goto-char (point-max)) + (insert (car (last args)) " ")))) + +(defun rcirc-handler-366 (process sender args _text) + "RPL_ENDOFNAMES" + (let* ((channel (cadr args)) + (buffer (rcirc-get-temp-buffer-create process channel))) + (with-current-buffer buffer + (rcirc-print process sender "NAMES" channel + (let ((content (buffer-substring (point-min) (point-max)))) + (rcirc-sort-nicknames-join content " ")))) + (kill-buffer buffer))) + +(defun rcirc-handler-433 (process sender args text) + "ERR_NICKNAMEINUSE" + (rcirc-handler-generic process "433" sender args text) + (let* ((new-nick (concat (cadr args) "`"))) + (with-rcirc-process-buffer process + (rcirc-cmd-nick new-nick nil process)))) + +(defun rcirc-authenticate () + "Send authentication to process associated with current buffer. +Passwords are stored in `rcirc-authinfo' (which see)." + (interactive) + (with-rcirc-server-buffer + (dolist (i rcirc-authinfo) + (let ((process (rcirc-buffer-process)) + (server (car i)) + (nick (nth 2 i)) + (method (cadr i)) + (args (cl-cdddr i))) + (when (and (string-match server rcirc-server)) + (if (and (memq method '(nickserv chanserv bitlbee)) + (string-match nick rcirc-nick)) + ;; the following methods rely on the user's nickname. + (cl-case method + (nickserv + (rcirc-send-privmsg + process + (or (cadr args) "NickServ") + (concat "IDENTIFY " (car args)))) + (chanserv + (rcirc-send-privmsg + process + "ChanServ" + (format "IDENTIFY %s %s" (car args) (cadr args)))) + (bitlbee + (rcirc-send-privmsg + process + "&bitlbee" + (concat "IDENTIFY " (car args))))) + ;; quakenet authentication doesn't rely on the user's nickname. + ;; the variable `nick' here represents the Q account name. + (when (eq method 'quakenet) + (rcirc-send-privmsg + process + "Q@CServe.quakenet.org" + (format "AUTH %s %s" nick (car args)))))))))) + +(defun rcirc-handler-INVITE (process sender args _text) + (rcirc-print process sender "INVITE" nil (mapconcat 'identity args " ") t)) + +(defun rcirc-handler-ERROR (process sender args _text) + (rcirc-print process sender "ERROR" nil (mapconcat 'identity args " "))) + +(defun rcirc-handler-CTCP (process target sender text) + (if (string-match "^\\([^ ]+\\) *\\(.*\\)$" text) + (let* ((request (upcase (match-string 1 text))) + (args (match-string 2 text)) + (handler (intern-soft (concat "rcirc-handler-ctcp-" request)))) + (if (not (fboundp handler)) + (rcirc-print process sender "ERROR" target + (format "%s sent unsupported ctcp: %s" sender text) + t) + (funcall handler process target sender args) + (unless (or (string= request "ACTION") + (string= request "KEEPALIVE")) + (rcirc-print process sender "CTCP" target + (format "%s" text) t)))))) + +(defun rcirc-handler-ctcp-VERSION (process _target sender _args) + (rcirc-send-string process + (concat "NOTICE " sender + " :\C-aVERSION " rcirc-id-string + "\C-a"))) + +(defun rcirc-handler-ctcp-ACTION (process target sender args) + (rcirc-print process sender "ACTION" target args t)) + +(defun rcirc-handler-ctcp-TIME (process _target sender _args) + (rcirc-send-string process + (concat "NOTICE " sender + " :\C-aTIME " (current-time-string) "\C-a"))) + +(defun rcirc-handler-CTCP-response (process _target sender message) + (rcirc-print process sender "CTCP" nil message t)) + +(defun rcirc-handler-CAP (process _sender args _text) + (when (equal (cadr args) "LS") + (rcirc-send-string process "CAP REQ :server-time")) + + (when (or (equal (cadr args) "ACK") + (equal (cadr args) "NAK")) + ;; Capability negotiation is best-effort here, I know that my + ;; servers support server-time and thus we end negotiation + ;; immediately. + (rcirc-send-string process "CAP END"))) + +(defgroup rcirc-faces nil + "Faces for rcirc." + :group 'rcirc + :group 'faces) + +(defface rcirc-my-nick ; font-lock-function-name-face + '((((class color) (min-colors 88) (background light)) :foreground "Blue1") + (((class color) (min-colors 88) (background dark)) :foreground "LightSkyBlue") + (((class color) (min-colors 16) (background light)) :foreground "Blue") + (((class color) (min-colors 16) (background dark)) :foreground "LightSkyBlue") + (((class color) (min-colors 8)) :foreground "blue" :weight bold) + (t :inverse-video t :weight bold)) + "Rcirc face for my messages." + :group 'rcirc-faces) + +(defface rcirc-other-nick ; font-lock-variable-name-face + '((((class grayscale) (background light)) + :foreground "Gray90" :weight bold :slant italic) + (((class grayscale) (background dark)) + :foreground "DimGray" :weight bold :slant italic) + (((class color) (min-colors 88) (background light)) :foreground "DarkGoldenrod") + (((class color) (min-colors 88) (background dark)) :foreground "LightGoldenrod") + (((class color) (min-colors 16) (background light)) :foreground "DarkGoldenrod") + (((class color) (min-colors 16) (background dark)) :foreground "LightGoldenrod") + (((class color) (min-colors 8)) :foreground "yellow" :weight light) + (t :weight bold :slant italic)) + "Rcirc face for other users' messages." + :group 'rcirc-faces) + +(defface rcirc-bright-nick + '((((class grayscale) (background light)) + :foreground "LightGray" :weight bold :underline t) + (((class grayscale) (background dark)) + :foreground "Gray50" :weight bold :underline t) + (((class color) (min-colors 88) (background light)) :foreground "CadetBlue") + (((class color) (min-colors 88) (background dark)) :foreground "Aquamarine") + (((class color) (min-colors 16) (background light)) :foreground "CadetBlue") + (((class color) (min-colors 16) (background dark)) :foreground "Aquamarine") + (((class color) (min-colors 8)) :foreground "magenta") + (t :weight bold :underline t)) + "Rcirc face for nicks matched by `rcirc-bright-nicks'." + :group 'rcirc-faces) + +(defface rcirc-dim-nick + '((t :inherit default)) + "Rcirc face for nicks in `rcirc-dim-nicks'." + :group 'rcirc-faces) + +(defface rcirc-server ; font-lock-comment-face + '((((class grayscale) (background light)) + :foreground "DimGray" :weight bold :slant italic) + (((class grayscale) (background dark)) + :foreground "LightGray" :weight bold :slant italic) + (((class color) (min-colors 88) (background light)) + :foreground "Firebrick") + (((class color) (min-colors 88) (background dark)) + :foreground "chocolate1") + (((class color) (min-colors 16) (background light)) + :foreground "red") + (((class color) (min-colors 16) (background dark)) + :foreground "red1") + (((class color) (min-colors 8) (background light))) + (((class color) (min-colors 8) (background dark))) + (t :weight bold :slant italic)) + "Rcirc face for server messages." + :group 'rcirc-faces) + +(defface rcirc-server-prefix ; font-lock-comment-delimiter-face + '((default :inherit rcirc-server) + (((class grayscale))) + (((class color) (min-colors 16))) + (((class color) (min-colors 8) (background light)) + :foreground "red") + (((class color) (min-colors 8) (background dark)) + :foreground "red1")) + "Rcirc face for server prefixes." + :group 'rcirc-faces) + +(defface rcirc-timestamp + '((t :inherit default)) + "Rcirc face for timestamps." + :group 'rcirc-faces) + +(defface rcirc-nick-in-message ; font-lock-keyword-face + '((((class grayscale) (background light)) :foreground "LightGray" :weight bold) + (((class grayscale) (background dark)) :foreground "DimGray" :weight bold) + (((class color) (min-colors 88) (background light)) :foreground "Purple") + (((class color) (min-colors 88) (background dark)) :foreground "Cyan1") + (((class color) (min-colors 16) (background light)) :foreground "Purple") + (((class color) (min-colors 16) (background dark)) :foreground "Cyan") + (((class color) (min-colors 8)) :foreground "cyan" :weight bold) + (t :weight bold)) + "Rcirc face for instances of your nick within messages." + :group 'rcirc-faces) + +(defface rcirc-nick-in-message-full-line '((t :weight bold)) + "Rcirc face for emphasizing the entire message when your nick is mentioned." + :group 'rcirc-faces) + +(defface rcirc-prompt ; comint-highlight-prompt + '((((min-colors 88) (background dark)) :foreground "cyan1") + (((background dark)) :foreground "cyan") + (t :foreground "dark blue")) + "Rcirc face for prompts." + :group 'rcirc-faces) + +(defface rcirc-track-nick + '((((type tty)) :inherit default) + (t :inverse-video t)) + "Rcirc face used in the mode-line when your nick is mentioned." + :group 'rcirc-faces) + +(defface rcirc-track-keyword '((t :weight bold)) + "Rcirc face used in the mode-line when keywords are mentioned." + :group 'rcirc-faces) + +(defface rcirc-url '((t :weight bold)) + "Rcirc face used to highlight urls." + :group 'rcirc-faces) + +(defface rcirc-keyword '((t :inherit highlight)) + "Rcirc face used to highlight keywords." + :group 'rcirc-faces) + + +;; When using M-x flyspell-mode, only check words after the prompt +(put 'rcirc-mode 'flyspell-mode-predicate 'rcirc-looking-at-input) +(defun rcirc-looking-at-input () + "Returns true if point is past the input marker." + (>= (point) rcirc-prompt-end-marker)) + + +(provide 'rcirc) + +;;; rcirc.el ends here diff --git a/.elpaignore b/third_party/exwm/.elpaignore index b43bf86b50fd..b43bf86b50fd 100644 --- a/.elpaignore +++ b/third_party/exwm/.elpaignore diff --git a/third_party/exwm/.gitignore b/third_party/exwm/.gitignore new file mode 100644 index 000000000000..9e4b0ee5b48e --- /dev/null +++ b/third_party/exwm/.gitignore @@ -0,0 +1,3 @@ +*.elc +*-pkg.el +*-autoloads.el diff --git a/third_party/exwm/README.md b/third_party/exwm/README.md new file mode 100644 index 000000000000..6d7e0dd1ff17 --- /dev/null +++ b/third_party/exwm/README.md @@ -0,0 +1,21 @@ +# Emacs X Window Manager + +EXWM (Emacs X Window Manager) is a full-featured tiling X window manager +for Emacs built on top of [XELB](https://github.com/ch11ng/xelb). +It features: ++ Fully keyboard-driven operations ++ Hybrid layout modes (tiling & stacking) ++ Dynamic workspace support ++ ICCCM/EWMH compliance ++ (Optional) RandR (multi-monitor) support ++ (Optional) Builtin system tray ++ (Optional) Builtin input method + +Please check out the +[screenshots](https://github.com/ch11ng/exwm/wiki/Screenshots) +to get an overview of what EXWM is capable of, +and the [user guide](https://github.com/ch11ng/exwm/wiki) +for a detailed explanation of its usage. + +**Note**: If you install EXWM from source, it's recommended to install +XELB also from source (otherwise install both from GNU ELPA). diff --git a/exwm-cm.el b/third_party/exwm/exwm-cm.el index 8a4501003042..8a4501003042 100644 --- a/exwm-cm.el +++ b/third_party/exwm/exwm-cm.el diff --git a/exwm-config.el b/third_party/exwm/exwm-config.el index 9609f4cf3ce0..9609f4cf3ce0 100644 --- a/exwm-config.el +++ b/third_party/exwm/exwm-config.el diff --git a/exwm-core.el b/third_party/exwm/exwm-core.el index 5356ef9b97e1..5356ef9b97e1 100644 --- a/exwm-core.el +++ b/third_party/exwm/exwm-core.el diff --git a/exwm-floating.el b/third_party/exwm/exwm-floating.el index a9f9315b710f..a9f9315b710f 100644 --- a/exwm-floating.el +++ b/third_party/exwm/exwm-floating.el diff --git a/exwm-input.el b/third_party/exwm/exwm-input.el index c27ee1b04f9f..50676217f1ad 100644 --- a/exwm-input.el +++ b/third_party/exwm/exwm-input.el @@ -117,9 +117,6 @@ defined in `exwm-mode-map' here." (defvar exwm-input--simulation-keys nil "Simulation keys in line-mode.") -(defvar exwm-input--skip-buffer-list-update nil - "Skip the upcoming 'buffer-list-update'.") - (defvar exwm-input--temp-line-mode nil "Non-nil indicates it's in temporary line-mode for char-mode.") @@ -138,8 +135,16 @@ defined in `exwm-mode-map' here." "Timer for deferring the update of input focus.") (defvar exwm-input--update-focus-window nil "The (Emacs) window to be focused. +It also helps us discern whether a `buffer-list-update-hook' was caused by a +different window having been selected. + This value should always be overwritten.") +(defvar exwm-input--update-focus-window-buffer nil + "Buffer displayed in `exwm-input--update-focus-window'. +Helps us discern whether a `buffer-list-update-hook' was caused by the selected +window switching to a different buffer.") + (defvar exwm-input--echo-area-timer nil "Timer for detecting echo area dirty.") (defvar exwm-input--event-hook nil @@ -296,13 +301,24 @@ ARGS are additional arguments to CALLBACK." (defun exwm-input--on-buffer-list-update () "Run in `buffer-list-update-hook' to track input focus." - (when (and (not (exwm-workspace--client-p)) - (not exwm-input--skip-buffer-list-update)) - (exwm--log "current-buffer=%S selected-window=%S" - (current-buffer) (selected-window)) - (redirect-frame-focus (selected-frame) nil) - (setq exwm-input--update-focus-window (selected-window)) - (exwm-input--update-focus-defer))) + ;; `buffer-list-update-hook' is invoked by several functions + ;; (`get-buffer-create', `select-window', `with-temp-buffer', etc.), but we + ;; just want to notice when a different window has been selected, or when the + ;; selected window displays a different buffer, so that we can set the focus + ;; to the associated X window (in case of an `exwm-mode' buffer). In order to + ;; differentiate, we keep track of the last selected window and buffer in the + ;; `exwm-input--update-focus-window' and + ;; `exwm-input--update-focus-window-buffer' variables. + (let* ((win (selected-window)) + (buf (window-buffer win))) + (when (and (not (exwm-workspace--client-p)) + (not (and (eq exwm-input--update-focus-window win) + (eq exwm-input--update-focus-window-buffer buf)))) + (exwm--log "selected-window=%S current-buffer=%S" win buf) + (setq exwm-input--update-focus-window win) + (setq exwm-input--update-focus-window-buffer buf) + (redirect-frame-focus (selected-frame) nil) + (exwm-input--update-focus-defer)))) (defun exwm-input--update-focus-defer () "Defer updating input focus." diff --git a/exwm-layout.el b/third_party/exwm/exwm-layout.el index 9173a1c049df..9173a1c049df 100644 --- a/exwm-layout.el +++ b/third_party/exwm/exwm-layout.el diff --git a/exwm-manage.el b/third_party/exwm/exwm-manage.el index c3d47f72259f..e940257fc9f8 100644 --- a/exwm-manage.el +++ b/third_party/exwm/exwm-manage.el @@ -151,7 +151,6 @@ want to match against EXWM internal variables such as `exwm-title', (defvar exwm-manage--ping-lock nil "Non-nil indicates EXWM is pinging a window.") -(defvar exwm-input--skip-buffer-list-update) (defvar exwm-input-prefix-keys) (defvar exwm-workspace--current) (defvar exwm-workspace--id-struts-alist) @@ -263,8 +262,7 @@ want to match against EXWM internal variables such as `exwm-title', (make-instance 'xcb:ChangeSaveSet :mode xcb:SetMode:Insert :window id)) - (with-current-buffer (let ((exwm-input--skip-buffer-list-update t)) - (generate-new-buffer "*EXWM*")) + (with-current-buffer (generate-new-buffer "*EXWM*") ;; Keep the oldest X window first. (setq exwm--id-buffer-alist (nconc exwm--id-buffer-alist `((,id . ,(current-buffer))))) @@ -349,8 +347,7 @@ want to match against EXWM internal variables such as `exwm-title', :stack-mode xcb:StackMode:Below))) (xcb:flush exwm--connection) (setq exwm--id-buffer-alist (assq-delete-all id exwm--id-buffer-alist)) - (let ((kill-buffer-query-functions nil) - (exwm-input--skip-buffer-list-update t)) + (let ((kill-buffer-query-functions nil)) (kill-buffer (current-buffer))) (throw 'return 'ignored)) (let ((index (plist-get exwm--configurations 'workspace))) diff --git a/exwm-randr.el b/third_party/exwm/exwm-randr.el index 68bfdd70e949..68bfdd70e949 100644 --- a/exwm-randr.el +++ b/third_party/exwm/exwm-randr.el diff --git a/exwm-systemtray.el b/third_party/exwm/exwm-systemtray.el index 43b3e1eaef41..43b3e1eaef41 100644 --- a/exwm-systemtray.el +++ b/third_party/exwm/exwm-systemtray.el diff --git a/exwm-workspace.el b/third_party/exwm/exwm-workspace.el index c513347119a2..c513347119a2 100644 --- a/exwm-workspace.el +++ b/third_party/exwm/exwm-workspace.el diff --git a/exwm-xim.el b/third_party/exwm/exwm-xim.el index 9589648d22ca..9589648d22ca 100644 --- a/exwm-xim.el +++ b/third_party/exwm/exwm-xim.el diff --git a/exwm.el b/third_party/exwm/exwm.el index b025f6b49aa5..b025f6b49aa5 100644 --- a/exwm.el +++ b/third_party/exwm/exwm.el diff --git a/xinitrc b/third_party/exwm/xinitrc index 591e4199144f..591e4199144f 100644 --- a/xinitrc +++ b/third_party/exwm/xinitrc diff --git a/third_party/farmhash/default.nix b/third_party/farmhash/default.nix new file mode 100644 index 000000000000..d7431dbd0108 --- /dev/null +++ b/third_party/farmhash/default.nix @@ -0,0 +1,13 @@ +# Google's farmhash family of hash functions +{ pkgs, ... }: + +pkgs.stdenv.mkDerivation { + name = "farmhash"; + + src = pkgs.fetchFromGitHub { + owner = "google"; + repo = "farmhash"; + rev = "0d859a811870d10f53a594927d0d0b97573ad06d"; + sha256 = "1w2583m5289hby0r91gds5yia6l8qpmzkl5b9bv58g5gacfj2h17"; + }; +} diff --git a/third_party/gerrit/0001-Use-detzip-in-download_bower.py.patch b/third_party/gerrit/0001-Use-detzip-in-download_bower.py.patch new file mode 100644 index 000000000000..7d197795b725 --- /dev/null +++ b/third_party/gerrit/0001-Use-detzip-in-download_bower.py.patch @@ -0,0 +1,25 @@ +From 621cadcc1dd71e9397c21cf8cf0f1aae4f6f7057 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown <git@lukegb.com> +Date: Thu, 2 Jul 2020 23:02:09 +0100 +Subject: [PATCH 1/7] Use detzip in download_bower.py + +--- + tools/js/download_bower.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/tools/js/download_bower.py b/tools/js/download_bower.py +index d541b565a9..ffdae60f95 100755 +--- a/tools/js/download_bower.py ++++ b/tools/js/download_bower.py +@@ -110,7 +110,7 @@ def main(): + args.b, '--quiet', 'install', '%s#%s' % (args.p, args.v))) + bc = os.path.join(cwd, 'bower_components') + subprocess.check_call( +- ['zip', '-q', '--exclude', '.bower.json', '-r', cached, args.n], ++ ['detzip', '--exclude', '.bower.json', cached, args.n], + cwd=bc) + + if args.s: +-- +2.32.0 + diff --git a/third_party/gerrit/0002-Syntax-highlight-nix.patch b/third_party/gerrit/0002-Syntax-highlight-nix.patch new file mode 100644 index 000000000000..256da0a3c930 --- /dev/null +++ b/third_party/gerrit/0002-Syntax-highlight-nix.patch @@ -0,0 +1,24 @@ +From 924647c354576ade0dc46fdf30596967f58bb4c6 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown <git@lukegb.com> +Date: Thu, 2 Jul 2020 23:02:32 +0100 +Subject: [PATCH 2/7] Syntax highlight nix + +--- + .../app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts b/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts +index 081d28d749..2762ccc625 100644 +--- a/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts ++++ b/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts +@@ -99,6 +99,7 @@ const LANGUAGE_MAP = new Map<string, string>([ + ['text/x-vhdl', 'vhdl'], + ['text/x-yaml', 'yaml'], + ['text/vbscript', 'vbscript'], ++ ['application/x-mix-transfer', 'nix'], + ]); + const ASYNC_DELAY = 10; + +-- +2.32.0 + diff --git a/third_party/gerrit/0003-Syntax-highlight-rules.pl.patch b/third_party/gerrit/0003-Syntax-highlight-rules.pl.patch new file mode 100644 index 000000000000..02bb3397eabb --- /dev/null +++ b/third_party/gerrit/0003-Syntax-highlight-rules.pl.patch @@ -0,0 +1,46 @@ +From be348f64eda257ae0af1f89552548d3e8eca3688 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown <git@lukegb.com> +Date: Thu, 2 Jul 2020 23:02:43 +0100 +Subject: [PATCH 3/7] Syntax highlight rules.pl + +--- + .../diff/gr-syntax-layer/gr-syntax-layer.ts | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts b/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts +index 2762ccc625..598e14589f 100644 +--- a/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts ++++ b/polygerrit-ui/app/elements/diff/gr-syntax-layer/gr-syntax-layer.ts +@@ -103,6 +103,10 @@ const LANGUAGE_MAP = new Map<string, string>([ + ]); + const ASYNC_DELAY = 10; + ++const FILENAME_OVERRIDES = new Map<string, string>([ ++ ['rules.pl', 'prolog'], ++]); ++ + const CLASS_SAFELIST = new Set<string>([ + 'gr-diff gr-syntax gr-syntax-attr', + 'gr-diff gr-syntax gr-syntax-attribute', +@@ -241,10 +245,17 @@ export class GrSyntaxLayer implements DiffLayer { + } + } + ++ _basename(filename: string): string { ++ const pieces = filename.split(/\//); ++ return pieces[pieces.length-1]; ++ } ++ + _getLanguage(metaInfo: DiffFileMetaInfo) { + // The Gerrit API provides only content-type, but for other users of + // gr-diff it may be more convenient to specify the language directly. +- return metaInfo.language ?? LANGUAGE_MAP.get(metaInfo.content_type); ++ return metaInfo.language ?? ++ FILENAME_OVERRIDES.get(this._basename(metaInfo.name)) ?? ++ LANGUAGE_MAP.get(metaInfo.content_type); + } + + /** +-- +2.32.0 + diff --git a/third_party/gerrit/0004-Add-titles-to-CLs-over-HTTP.patch b/third_party/gerrit/0004-Add-titles-to-CLs-over-HTTP.patch new file mode 100644 index 000000000000..8e78e5f535e3 --- /dev/null +++ b/third_party/gerrit/0004-Add-titles-to-CLs-over-HTTP.patch @@ -0,0 +1,217 @@ +From 32bf13d8316f93828d2ff47ccfca38d4e7a634b1 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown <git@lukegb.com> +Date: Thu, 2 Jul 2020 23:03:02 +0100 +Subject: [PATCH 4/7] Add titles to CLs over HTTP + +--- + .../gerrit/httpd/raw/IndexHtmlUtil.java | 13 +++- + .../google/gerrit/httpd/raw/IndexServlet.java | 8 ++- + .../google/gerrit/httpd/raw/StaticModule.java | 6 +- + .../gerrit/httpd/raw/TitleComputer.java | 67 +++++++++++++++++++ + .../gerrit/httpd/raw/PolyGerritIndexHtml.soy | 4 +- + 5 files changed, 90 insertions(+), 8 deletions(-) + create mode 100644 java/com/google/gerrit/httpd/raw/TitleComputer.java + +diff --git a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java +index 8d52f5ad50..a9cfceb3b6 100644 +--- a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java ++++ b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java +@@ -39,6 +39,7 @@ import java.util.Arrays; + import java.util.Collections; + import java.util.HashMap; + import java.util.Map; ++import java.util.Optional; + import java.util.Set; + import java.util.function.Function; + +@@ -60,13 +61,14 @@ public class IndexHtmlUtil { + String faviconPath, + Map<String, String[]> urlParameterMap, + Function<String, SanitizedContent> urlInScriptTagOrdainer, +- String requestedURL) ++ String requestedURL, ++ TitleComputer titleComputer) + throws URISyntaxException, RestApiException { + ImmutableMap.Builder<String, Object> data = ImmutableMap.builder(); + data.putAll( + staticTemplateData( + canonicalURL, cdnPath, faviconPath, urlParameterMap, urlInScriptTagOrdainer)) +- .putAll(dynamicTemplateData(gerritApi, requestedURL)); ++ .putAll(dynamicTemplateData(gerritApi, requestedURL, titleComputer)); + Set<String> enabledExperiments = experimentFeatures.getEnabledExperimentFeatures(); + + if (!enabledExperiments.isEmpty()) { +@@ -77,7 +79,9 @@ public class IndexHtmlUtil { + + /** Returns dynamic parameters of {@code index.html}. */ + public static ImmutableMap<String, Object> dynamicTemplateData( +- GerritApi gerritApi, String requestedURL) throws RestApiException, URISyntaxException { ++ GerritApi gerritApi, ++ String requestedURL, ++ TitleComputer titleComputer) throws RestApiException, URISyntaxException { + ImmutableMap.Builder<String, Object> data = ImmutableMap.builder(); + Map<String, SanitizedContent> initialData = new HashMap<>(); + Server serverApi = gerritApi.config().server(); +@@ -128,6 +132,9 @@ public class IndexHtmlUtil { + // Don't render data + } + ++ Optional<String> title = titleComputer.computeTitle(requestedURL); ++ title.ifPresent(s -> data.put("title", s)); ++ + data.put("gerritInitialData", initialData); + return data.build(); + } +diff --git a/java/com/google/gerrit/httpd/raw/IndexServlet.java b/java/com/google/gerrit/httpd/raw/IndexServlet.java +index 3f2c2028ae..7861c007df 100644 +--- a/java/com/google/gerrit/httpd/raw/IndexServlet.java ++++ b/java/com/google/gerrit/httpd/raw/IndexServlet.java +@@ -46,13 +46,15 @@ public class IndexServlet extends HttpServlet { + private final ExperimentFeatures experimentFeatures; + private final SoySauce soySauce; + private final Function<String, SanitizedContent> urlOrdainer; ++ private TitleComputer titleComputer; + + IndexServlet( + @Nullable String canonicalUrl, + @Nullable String cdnPath, + @Nullable String faviconPath, + GerritApi gerritApi, +- ExperimentFeatures experimentFeatures) { ++ ExperimentFeatures experimentFeatures, ++ TitleComputer titleComputer) { + this.canonicalUrl = canonicalUrl; + this.cdnPath = cdnPath; + this.faviconPath = faviconPath; +@@ -67,6 +69,7 @@ public class IndexServlet extends HttpServlet { + (s) -> + UnsafeSanitizedContentOrdainer.ordainAsSafe( + s, SanitizedContent.ContentKind.TRUSTED_RESOURCE_URI); ++ this.titleComputer = titleComputer; + } + + @Override +@@ -85,7 +88,8 @@ public class IndexServlet extends HttpServlet { + faviconPath, + parameterMap, + urlOrdainer, +- requestUrl); ++ requestUrl, ++ titleComputer); + renderer = soySauce.renderTemplate("com.google.gerrit.httpd.raw.Index").setData(templateData); + } catch (URISyntaxException | RestApiException e) { + throw new IOException(e); +diff --git a/java/com/google/gerrit/httpd/raw/StaticModule.java b/java/com/google/gerrit/httpd/raw/StaticModule.java +index bb1eb92525..6b20c504d2 100644 +--- a/java/com/google/gerrit/httpd/raw/StaticModule.java ++++ b/java/com/google/gerrit/httpd/raw/StaticModule.java +@@ -224,11 +224,13 @@ public class StaticModule extends ServletModule { + @CanonicalWebUrl @Nullable String canonicalUrl, + @GerritServerConfig Config cfg, + GerritApi gerritApi, +- ExperimentFeatures experimentFeatures) { ++ ExperimentFeatures experimentFeatures, ++ TitleComputer titleComputer) { + String cdnPath = + options.useDevCdn() ? options.devCdn() : cfg.getString("gerrit", null, "cdnPath"); + String faviconPath = cfg.getString("gerrit", null, "faviconPath"); +- return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, experimentFeatures); ++ return new IndexServlet(canonicalUrl, cdnPath, faviconPath, gerritApi, ++ experimentFeatures, titleComputer); + } + + @Provides +diff --git a/java/com/google/gerrit/httpd/raw/TitleComputer.java b/java/com/google/gerrit/httpd/raw/TitleComputer.java +new file mode 100644 +index 0000000000..8fd2053ad0 +--- /dev/null ++++ b/java/com/google/gerrit/httpd/raw/TitleComputer.java +@@ -0,0 +1,67 @@ ++package com.google.gerrit.httpd.raw; ++ ++import com.google.common.flogger.FluentLogger; ++import com.google.gerrit.entities.Change; ++import com.google.gerrit.extensions.restapi.ResourceConflictException; ++import com.google.gerrit.extensions.restapi.ResourceNotFoundException; ++import com.google.gerrit.server.change.ChangeResource; ++import com.google.gerrit.server.permissions.PermissionBackendException; ++import com.google.gerrit.server.restapi.change.ChangesCollection; ++import com.google.inject.Inject; ++import com.google.inject.Provider; ++import com.google.inject.Singleton; ++ ++import java.net.MalformedURLException; ++import java.net.URL; ++import java.util.Optional; ++import java.util.regex.Matcher; ++import java.util.regex.Pattern; ++ ++@Singleton ++public class TitleComputer { ++ private static final FluentLogger logger = FluentLogger.forEnclosingClass(); ++ ++ @Inject ++ public TitleComputer(Provider<ChangesCollection> changes) { ++ this.changes = changes; ++ } ++ ++ public Optional<String> computeTitle(String requestedURI) { ++ URL url = null; ++ try { ++ url = new URL(requestedURI); ++ } catch (MalformedURLException e) { ++ logger.atWarning().log("Failed to turn %s into a URL.", requestedURI); ++ return Optional.empty(); ++ } ++ ++ // Try to turn this into a change. ++ Optional<Change.Id> changeId = tryExtractChange(url.getPath()); ++ if (changeId.isPresent()) { ++ return titleFromChangeId(changeId.get()); ++ } ++ ++ return Optional.empty(); ++ } ++ ++ private static final Pattern extractChangeIdRegex = Pattern.compile("^/(?:c/.*/\\+/)?(?<changeId>[0-9]+)(?:/[0-9]+)?(?:/.*)?$"); ++ private final Provider<ChangesCollection> changes; ++ ++ private Optional<Change.Id> tryExtractChange(String path) { ++ Matcher m = extractChangeIdRegex.matcher(path); ++ if (!m.matches()) { ++ return Optional.empty(); ++ } ++ return Change.Id.tryParse(m.group("changeId")); ++ } ++ ++ private Optional<String> titleFromChangeId(Change.Id changeId) { ++ ChangesCollection changesCollection = changes.get(); ++ try { ++ ChangeResource changeResource = changesCollection.parse(changeId); ++ return Optional.of(changeResource.getChange().getSubject()); ++ } catch (ResourceConflictException | ResourceNotFoundException | PermissionBackendException e) { ++ return Optional.empty(); ++ } ++ } ++} +diff --git a/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy b/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy +index 11717fb8a4..1ae9046360 100644 +--- a/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy ++++ b/resources/com/google/gerrit/httpd/raw/PolyGerritIndexHtml.soy +@@ -33,10 +33,12 @@ + {@param? defaultDashboardHex: ?} + {@param? dashboardQuery: ?} + {@param? userIsAuthenticated: ?} ++ {@param? title: ?} + <!DOCTYPE html>{\n} + <html lang="en">{\n} + <meta charset="utf-8">{\n} +- <meta name="description" content="Gerrit Code Review">{\n} ++ {if $title}<title>{$title} · Gerrit Code Review</title>{\n}{/if} ++ <meta name="description" content="{if $title}{$title} · {/if}Gerrit Code Review">{\n} + <meta name="referrer" content="never">{\n} + <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=0">{\n} + +-- +2.32.0 + diff --git a/third_party/gerrit/0005-When-using-local-fonts-always-assume-Gerrit-is-mount.patch b/third_party/gerrit/0005-When-using-local-fonts-always-assume-Gerrit-is-mount.patch new file mode 100644 index 000000000000..b664ea0ea6b2 --- /dev/null +++ b/third_party/gerrit/0005-When-using-local-fonts-always-assume-Gerrit-is-mount.patch @@ -0,0 +1,26 @@ +From bd7db44cabb6de64f03adbaf5e24c73e022a8932 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown <git@lukegb.com> +Date: Sat, 11 Jul 2020 00:45:57 +0000 +Subject: [PATCH 5/7] When using local fonts, always assume Gerrit is mounted + at the root. + +--- + polygerrit-ui/app/rollup.config.js | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/polygerrit-ui/app/rollup.config.js b/polygerrit-ui/app/rollup.config.js +index d93b5eab39..c862c9bbae 100644 +--- a/polygerrit-ui/app/rollup.config.js ++++ b/polygerrit-ui/app/rollup.config.js +@@ -50,7 +50,7 @@ const importLocalFontMetaUrlResolver = function() { + name: 'import-meta-url-resolver', + resolveImportMeta: function (property, data) { + if(property === 'url' && data.moduleId.endsWith('/@polymer/font-roboto-local/roboto.js')) { +- return 'new URL("..", document.baseURI).href'; ++ return 'new URL("/", document.baseURI).href'; + } + return null; + } +-- +2.32.0 + diff --git a/third_party/gerrit/0006-Always-use-Google-Fonts.patch b/third_party/gerrit/0006-Always-use-Google-Fonts.patch new file mode 100644 index 000000000000..5b817d0b55a3 --- /dev/null +++ b/third_party/gerrit/0006-Always-use-Google-Fonts.patch @@ -0,0 +1,28 @@ +From d71f51afe12a280b92831070a583b15c8b6bc2f4 Mon Sep 17 00:00:00 2001 +From: Luke Granger-Brown <git@lukegb.com> +Date: Sat, 11 Jul 2020 00:46:13 +0000 +Subject: [PATCH 6/7] Always use Google Fonts. + +We're not a corporate, and we're not behind the GFW. Always use Google Fonts, +because even though we no longer get the caching benefits (boo, browsers), +it is still a better geographically-distributed CDN. +--- + java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java +index a9cfceb3b6..9c287c6e45 100644 +--- a/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java ++++ b/java/com/google/gerrit/httpd/raw/IndexHtmlUtil.java +@@ -184,7 +184,7 @@ public class IndexHtmlUtil { + if (urlParameterMap.containsKey("ce")) { + data.put("polyfillCE", "true"); + } +- if (urlParameterMap.containsKey("gf")) { ++ if (/* urlParameterMap.containsKey("gf") || */ true) { + data.put("useGoogleFonts", "true"); + } + +-- +2.32.0 + diff --git a/third_party/gerrit/default.nix b/third_party/gerrit/default.nix new file mode 100644 index 000000000000..f886a0426d52 --- /dev/null +++ b/third_party/gerrit/default.nix @@ -0,0 +1,150 @@ +{ depot, pkgs, ... }: + +let + detzip = depot.nix.buildGo.program { + name = "detzip"; + srcs = [ ./detzip.go ]; + }; + bazelRunScript = pkgs.writeShellScriptBin "bazel-run" '' + yarn config set cache-folder "$bazelOut/external/yarn_cache" + export HOME="$bazelOut/external/home" + mkdir -p "$bazelOut/external/home" + exec /bin/bazel "$@" + ''; + bazelTop = pkgs.buildFHSUserEnv { + name = "bazel"; + targetPkgs = pkgs: [ + (pkgs.bazel.override { enableNixHacks = true; }) + detzip + pkgs.jdk11_headless + pkgs.zlib + pkgs.python + pkgs.curl + pkgs.nodejs + pkgs.yarn + pkgs.git + bazelRunScript + ]; + runScript = "/bin/bazel-run"; + }; + bazel = bazelTop // { override = x: bazelTop; }; + version = "3.4.0"; +in +pkgs.lib.makeOverridable pkgs.buildBazelPackage { + pname = "gerrit"; + inherit version; + + src = pkgs.fetchgit { + url = "https://gerrit.googlesource.com/gerrit"; + rev = "471c1c15a7bc294d10e246df43812942b5ac8a13"; + branchName = "v${version}"; + sha256 = "sha256:0ayj0bcsxjln8qydkj9j7yiqibmjgd3bcpqvgsdzdx072wzx01c0"; + fetchSubmodules = true; + }; + + patches = [ + ./0001-Use-detzip-in-download_bower.py.patch + ./0002-Syntax-highlight-nix.patch + ./0003-Syntax-highlight-rules.pl.patch + ./0004-Add-titles-to-CLs-over-HTTP.patch + ./0005-When-using-local-fonts-always-assume-Gerrit-is-mount.patch + ./0006-Always-use-Google-Fonts.patch + ]; + + bazelTarget = "release api-skip-javadoc"; + inherit bazel; + + bazelFlags = [ + "--repository_cache=" + "--disk_cache=" + ]; + removeRulesCC = false; + fetchConfigured = true; + + fetchAttrs = { + sha256 = "sha256:16a1lbz40bc7217ldwf7x8yzlzxc714h965ldjnmfbvi3jq3hk6y"; + preBuild = '' + rm .bazelversion + ''; + + installPhase = '' + runHook preInstall + + # Remove all built in external workspaces, Bazel will recreate them when building + rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker} + rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker} + rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker} + rm -rf $bazelOut/external/{local_*,\@local_*.marker} + + # Clear markers + find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \; + + # Remove all vcs files + rm -rf $(find $bazelOut/external -type d -name .git) + rm -rf $(find $bazelOut/external -type d -name .svn) + rm -rf $(find $bazelOut/external -type d -name .hg) + + # Removing top-level symlinks along with their markers. + # This is needed because they sometimes point to temporary paths (?). + # For example, in Tensorflow-gpu build: + # platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms + find $bazelOut/external -maxdepth 1 -type l | while read symlink; do + name="$(basename "$symlink")" + rm -rf "$symlink" "$bazelOut/external/@$name.marker" + done + + # Patching symlinks to remove build directory reference + find $bazelOut/external -type l | while read symlink; do + new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")" + rm "$symlink" + ln -sf "$new_target" "$symlink" + done + + echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version + + # Gerrit fixups: + # Remove polymer-bridges and ba-linkify, they're in-repo + rm -rf $bazelOut/external/yarn_cache/v6/npm-polymer-bridges-* + rm -rf $bazelOut/external/yarn_cache/v6/npm-ba-linkify-* + # Normalize permissions on .yarn-{tarball,metadata} files + find $bazelOut/external/yarn_cache \( -name .yarn-tarball.tgz -or -name .yarn-metadata.json \) -exec chmod 644 {} + + + (cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/) + + runHook postInstall + ''; + }; + + buildAttrs = { + preConfigure = '' + rm .bazelversion + ''; + installPhase = '' + mkdir -p "$out"/webapps/ "$out"/share/api/ + cp bazel-bin/release.war "$out"/webapps/gerrit-${version}.war + unzip bazel-bin/api-skip-javadoc.zip -d "$out"/share/api + ''; + + nativeBuildInputs = with pkgs; [ + unzip + ]; + }; + + passthru = { + # A list of plugins that are part of the gerrit.war file. + # Use `java -jar gerrit.war ls | grep -Po '(?<=plugins/)[^.]+' | sed -e 's,^,",' -e 's,$,",' | sort` to generate that list. + plugins = [ + "codemirror-editor" + "commit-message-length-validator" + "delete-project" + "download-commands" + "gitiles" + "hooks" + "plugin-manager" + "replication" + "reviewnotes" + "singleusergroup" + "webhooks" + ]; + }; +} diff --git a/third_party/gerrit/detzip.go b/third_party/gerrit/detzip.go new file mode 100644 index 000000000000..511c18ecfe17 --- /dev/null +++ b/third_party/gerrit/detzip.go @@ -0,0 +1,97 @@ +package main + +import ( + "archive/zip" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "sort" + "strings" +) + +var ( + exclude = flag.String("exclude", "", "comma-separated list of filenames to exclude (in any directory)") +) + +func init() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s [zip file] [directory]:\n", os.Args[0]) + flag.PrintDefaults() + } +} + +func listToMap(ss []string) map[string]bool { + m := make(map[string]bool) + for _, s := range ss { + m[s] = true + } + return m +} + +func main() { + flag.Parse() + if flag.NArg() != 2 { + flag.Usage() + os.Exit(1) + } + + outPath := flag.Arg(0) + dirPath := flag.Arg(1) + + excludeFiles := listToMap(strings.Split(*exclude, ",")) + + // Aggregate all files first. + var files []string + filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + if excludeFiles[info.Name()] { + return nil + } + files = append(files, path) + return nil + }) + + // Create zip + outW, err := os.Create(outPath) + if err != nil { + log.Fatalf("Create(%q): %v", outPath, err) + } + + zipW := zip.NewWriter(outW) + + // Output files in alphabetical order + sort.Strings(files) + for _, f := range files { + fw, err := zipW.CreateHeader(&zip.FileHeader{ + Name: f, + Method: zip.Store, + }) + if err != nil { + log.Fatalf("creating %q in zip: %v", f, err) + } + + ff, err := os.Open(f) + if err != nil { + log.Fatalf("opening %q: %v", f, err) + } + if _, err := io.Copy(fw, ff); err != nil { + log.Fatalf("copying %q to zip: %v", f, err) + } + ff.Close() + } + + if err := zipW.Close(); err != nil { + log.Fatalf("writing ZIP central directory: %v", err) + } + if err := outW.Close(); err != nil { + log.Fatalf("closing ZIP file: %v", err) + } +} diff --git a/third_party/gerrit_plugins/builder.nix b/third_party/gerrit_plugins/builder.nix new file mode 100644 index 000000000000..ff1754e088f3 --- /dev/null +++ b/third_party/gerrit_plugins/builder.nix @@ -0,0 +1,33 @@ +{ depot, pkgs, ... }: +{ + buildGerritBazelPlugin = { + name, + src, + depsOutputHash, + overlayPluginCmd ? '' + cp -R "${src}" "$out/plugins/${name}" + '', + postPatch ? "", + }: ((depot.third_party.gerrit.override { + name = "${name}.jar"; + + src = pkgs.runCommandLocal "${name}-src" {} '' + cp -R "${depot.third_party.gerrit.src}" "$out" + chmod +w "$out/plugins" + ${overlayPluginCmd} + ''; + + bazelTarget = "//plugins/${name}"; + }).overrideAttrs (super: { + deps = super.deps.overrideAttrs (superDeps: { + outputHash = depsOutputHash; + }); + installPhase = '' + cp "bazel-bin/plugins/${name}/${name}.jar" "$out" + ''; + postPatch = if super ? postPatch then '' + ${super.postPatch} + ${postPatch} + '' else postPatch; + })); +} diff --git a/third_party/gerrit_plugins/default.nix b/third_party/gerrit_plugins/default.nix new file mode 100644 index 000000000000..8131ca2eb014 --- /dev/null +++ b/third_party/gerrit_plugins/default.nix @@ -0,0 +1,22 @@ +{ depot, pkgs, ... }@args: + +let + inherit (import ./builder.nix args) buildGerritBazelPlugin; +in depot.nix.readTree.drvTargets { + # https://gerrit.googlesource.com/plugins/owners + owners = buildGerritBazelPlugin rec { + name = "owners"; + depsOutputHash = "sha256:0qx3675lkj241c1sqs6xia5jpcwha2ib3mv32cilmh0k3cwdyyh2"; + src = pkgs.fetchgit { + url = "https://gerrit.googlesource.com/plugins/owners"; + rev = "99a9ab585532d172d141b4641dfc70081513dfc2"; + sha256 = "sha256:1xn9qb7q94jxfx7yq0zjqjm16gfyzzif13sak9x6j4f9r68frcd4"; + }; + overlayPluginCmd = '' + chmod +w "$out" "$out/plugins/external_plugin_deps.bzl" + cp -R "${src}/owners" "$out/plugins/owners" + cp "${src}/external_plugin_deps.bzl" "$out/plugins/external_plugin_deps.bzl" + cp -R "${src}/owners-common" "$out/owners-common" + ''; + }; +} diff --git a/third_party/gerrit_plugins/oauth/cas-6x.patch b/third_party/gerrit_plugins/oauth/cas-6x.patch new file mode 100644 index 000000000000..7494298b3f26 --- /dev/null +++ b/third_party/gerrit_plugins/oauth/cas-6x.patch @@ -0,0 +1,69 @@ +diff --git a/src/main/java/com/googlesource/gerrit/plugins/oauth/CasApi.java b/src/main/java/com/googlesource/gerrit/plugins/oauth/CasApi.java +index 450549f..27310cd 100644 +--- a/src/main/java/com/googlesource/gerrit/plugins/oauth/CasApi.java ++++ b/src/main/java/com/googlesource/gerrit/plugins/oauth/CasApi.java +@@ -15,7 +15,7 @@ + package com.googlesource.gerrit.plugins.oauth; + + import com.github.scribejava.core.builder.api.DefaultApi20; +-import com.github.scribejava.core.extractors.OAuth2AccessTokenExtractor; ++import com.github.scribejava.core.extractors.OAuth2AccessTokenJsonExtractor; + import com.github.scribejava.core.extractors.TokenExtractor; + import com.github.scribejava.core.model.OAuth2AccessToken; + import com.github.scribejava.core.oauth2.bearersignature.BearerSignature; +@@ -47,6 +47,6 @@ public class CasApi extends DefaultApi20 { + + @Override + public TokenExtractor<OAuth2AccessToken> getAccessTokenExtractor() { +- return OAuth2AccessTokenExtractor.instance(); ++ return OAuth2AccessTokenJsonExtractor.instance(); + } + } +diff --git a/src/main/java/com/googlesource/gerrit/plugins/oauth/CasOAuthService.java b/src/main/java/com/googlesource/gerrit/plugins/oauth/CasOAuthService.java +index 5f3e4a1..fc5bc50 100644 +--- a/src/main/java/com/googlesource/gerrit/plugins/oauth/CasOAuthService.java ++++ b/src/main/java/com/googlesource/gerrit/plugins/oauth/CasOAuthService.java +@@ -106,36 +106,14 @@ class CasOAuthService implements OAuthServiceProvider { + throw new IOException(String.format("CAS response missing id: %s", response.getBody())); + } + +- JsonElement attrListJson = jsonObject.get("attributes"); +- if (attrListJson == null) { +- throw new IOException( +- String.format("CAS response missing attributes: %s", response.getBody())); +- } +- + String email = null, name = null, login = null; +- if (attrListJson.isJsonArray()) { +- // It is possible for CAS to be configured to not return any attributes (email, name, +- // login), +- // in which case, +- // CAS returns an empty JSON object "attributes":{}, rather than "null" or an empty JSON +- // array +- // "attributes": [] +- +- JsonArray attrJson = attrListJson.getAsJsonArray(); +- for (JsonElement elem : attrJson) { +- if (elem == null || !elem.isJsonObject()) { +- throw new IOException(String.format("Invalid JSON '%s': not a JSON Object", elem)); +- } +- JsonObject obj = elem.getAsJsonObject(); +- +- String property = getStringElement(obj, "email"); +- if (property != null) email = property; +- property = getStringElement(obj, "name"); +- if (property != null) name = property; +- property = getStringElement(obj, "login"); +- if (property != null) login = property; +- } +- } ++ ++ String property = getStringElement(jsonObject, "mail"); ++ if (property != null) email = property; ++ property = getStringElement(jsonObject, "displayName"); ++ if (property != null) name = property; ++ property = getStringElement(jsonObject, "uid"); ++ if (property != null) login = property; + + return new OAuthUserInfo( + CAS_PROVIDER_PREFIX + id.getAsString(), diff --git a/third_party/gerrit_plugins/oauth/default.nix b/third_party/gerrit_plugins/oauth/default.nix new file mode 100644 index 000000000000..2298c0b39d93 --- /dev/null +++ b/third_party/gerrit_plugins/oauth/default.nix @@ -0,0 +1,26 @@ +{ depot, pkgs, ... }@args: + +let + inherit (import ../builder.nix args) buildGerritBazelPlugin; +in buildGerritBazelPlugin rec { + name = "oauth"; + depsOutputHash = "sha256:0ww88msym6zr5z86k5az1kmw3hv8d9giniwkii4lwnzf3kc5qnrx"; + src = pkgs.fetchgit { + url = "https://gerrit.googlesource.com/plugins/oauth"; + rev = "4aa7322db5ec221b2419e12a9ec7af5b8c66659c"; + sha256 = "1szra3pjl0axf4a7k96flpk7rhfvp37rdxay4gbglh939gzbba88"; + }; + overlayPluginCmd = '' + chmod +w "$out" "$out/plugins/external_plugin_deps.bzl" + cp -R "${src}" "$out/plugins/${name}" + cp "${src}/external_plugin_deps.bzl" "$out/plugins/external_plugin_deps.bzl" + ''; + + # The code in the OAuth repo expects CAS to return oauth2 access tokens as urlencoded. + # Our version of CAS returns them as JSON instead. + postPatch = '' + pushd plugins/oauth + patch -p1 <${./cas-6x.patch} + popd + ''; +} diff --git a/third_party/git/0001-feat-third_party-git-date-add-dottime-format.patch b/third_party/git/0001-feat-third_party-git-date-add-dottime-format.patch new file mode 100644 index 000000000000..0e11b2ca63d0 --- /dev/null +++ b/third_party/git/0001-feat-third_party-git-date-add-dottime-format.patch @@ -0,0 +1,119 @@ +From 46c52e6b462f9b1c9aa08a1b1bba79dd16302a9c Mon Sep 17 00:00:00 2001 +From: Vincent Ambo <tazjin@google.com> +Date: Mon, 6 Jan 2020 16:00:52 +0000 +Subject: [PATCH] feat(third_party/git/date): add "dottime" format + +Adds dottime (as defined on https://dotti.me) as a timestamp format. + +This format is designed to simplify working with timestamps across +many different timezones by keeping the timestamp format itself in +UTC (and indicating this with a dot character), but appending the +local offset. + +This is implemented as a new format because the timestamp needs to be +rendered both as UTC and including the offset, an implementation using +a strftime formatting string is not sufficient. +--- + Documentation/rev-list-options.txt | 3 +++ + builtin/blame.c | 3 +++ + cache.h | 3 ++- + date.c | 17 +++++++++++++++++ + t/t0006-date.sh | 2 ++ + 5 files changed, 27 insertions(+), 1 deletion(-) + +diff --git a/Documentation/rev-list-options.txt b/Documentation/rev-list-options.txt +index b7bd27e171..273971bdd0 100644 +--- a/Documentation/rev-list-options.txt ++++ b/Documentation/rev-list-options.txt +@@ -1052,6 +1052,9 @@ omitted. + 1970). As with `--raw`, this is always in UTC and therefore `-local` + has no effect. + ++`--date=dottime` shows the date in dottime format (rendered as UTC, ++but suffixed with the local timezone offset if given) ++ + `--date=format:...` feeds the format `...` to your system `strftime`, + except for %z and %Z, which are handled internally. + Use `--date=format:%c` to show the date in your system locale's +diff --git a/builtin/blame.c b/builtin/blame.c +index 641523ff9a..fa0240237c 100644 +--- a/builtin/blame.c ++++ b/builtin/blame.c +@@ -1005,6 +1005,9 @@ int cmd_blame(int argc, const char **argv, const char *prefix) + case DATE_STRFTIME: + blame_date_width = strlen(show_date(0, 0, &blame_date_mode)) + 1; /* add the null */ + break; ++ case DATE_DOTTIME: ++ blame_date_width = sizeof("2006-10-19T15·00-0700"); ++ break; + } + blame_date_width -= 1; /* strip the null */ + +diff --git a/cache.h b/cache.h +index 0c245d4f10..fa79e37b49 100644 +--- a/cache.h ++++ b/cache.h +@@ -1570,7 +1570,8 @@ enum date_mode_type { + DATE_RFC2822, + DATE_STRFTIME, + DATE_RAW, +- DATE_UNIX ++ DATE_UNIX, ++ DATE_DOTTIME + }; + + struct date_mode { +diff --git a/date.c b/date.c +index c55ea47e96..7fe4fb982b 100644 +--- a/date.c ++++ b/date.c +@@ -347,6 +347,21 @@ const char *show_date(timestamp_t time, int tz, const struct date_mode *mode) + tm->tm_mday, + tm->tm_hour, tm->tm_min, tm->tm_sec, + sign, tz / 100, tz % 100); ++ } else if (mode->type == DATE_DOTTIME) { ++ char sign = (tz >= 0) ? '+' : '-'; ++ tz = abs(tz); ++ ++ // Time is converted again without the timezone as the ++ // dottime format includes the zone only in offset ++ // position. ++ time_t t = gm_time_t(time, 0); ++ gmtime_r(&t, tm); ++ strbuf_addf(&timebuf, "%04d-%02d-%02dT%02d·%02d%c%02d%02d", ++ tm->tm_year + 1900, ++ tm->tm_mon + 1, ++ tm->tm_mday, ++ tm->tm_hour, tm->tm_min, ++ sign, tz / 100, tz % 100); + } else if (mode->type == DATE_RFC2822) + strbuf_addf(&timebuf, "%.3s, %d %.3s %d %02d:%02d:%02d %+05d", + weekday_names[tm->tm_wday], tm->tm_mday, +@@ -955,6 +970,8 @@ static enum date_mode_type parse_date_type(const char *format, const char **end) + return DATE_UNIX; + if (skip_prefix(format, "format", end)) + return DATE_STRFTIME; ++ if (skip_prefix(format, "dottime", end)) ++ return DATE_DOTTIME; + /* + * Please update $__git_log_date_formats in + * git-completion.bash when you add new formats. +diff --git a/t/t0006-date.sh b/t/t0006-date.sh +index 6b757d7169..5deb558497 100755 +--- a/t/t0006-date.sh ++++ b/t/t0006-date.sh +@@ -49,9 +49,11 @@ check_show short "$TIME" '2016-06-15' + check_show default "$TIME" 'Wed Jun 15 16:13:20 2016 +0200' + check_show raw "$TIME" '1466000000 +0200' + check_show unix "$TIME" '1466000000' ++check_show dottime "$TIME" '2016-06-15T14·13+0200' + check_show iso-local "$TIME" '2016-06-15 14:13:20 +0000' + check_show raw-local "$TIME" '1466000000 +0000' + check_show unix-local "$TIME" '1466000000' ++check_show dottime-local "$TIME" '2016-06-15T14·13+0000' + + check_show 'format:%z' "$TIME" '+0200' + check_show 'format-local:%z' "$TIME" '+0000' +-- +2.32.0 + diff --git a/third_party/git/default.nix b/third_party/git/default.nix new file mode 100644 index 000000000000..75131c03c2ff --- /dev/null +++ b/third_party/git/default.nix @@ -0,0 +1,9 @@ +# git with custom patches. This is also used by cgit via +# `pkgs.srcOnly`. +{ pkgs, ... }: + +pkgs.git.overrideAttrs(old: { + patches = (old.patches or []) ++ [ + ./0001-feat-third_party-git-date-add-dottime-format.patch + ]; +}) diff --git a/third_party/gitignoreSource/default.nix b/third_party/gitignoreSource/default.nix new file mode 100644 index 000000000000..8bdd974e8dbe --- /dev/null +++ b/third_party/gitignoreSource/default.nix @@ -0,0 +1,18 @@ +{ pkgs, ... }: + +let + gitignoreNix = import (pkgs.fetchFromGitHub { + owner = "hercules-ci"; + repo = "gitignore"; + rev = "f9e996052b5af4032fe6150bba4a6fe4f7b9d698"; + sha256 = "0jrh5ghisaqdd0vldbywags20m2cxpkbbk5jjjmwaw0gr8nhsafv"; + }) { inherit (pkgs) lib; }; + +in { + __functor = _: gitignoreNix.gitignoreSource; + + # expose extra functions here + inherit (gitignoreNix) + gitignoreFilter + ; +} diff --git a/third_party/glog/.bazelci/presubmit.yml b/third_party/glog/.bazelci/presubmit.yml new file mode 100644 index 000000000000..38e8515e9a8e --- /dev/null +++ b/third_party/glog/.bazelci/presubmit.yml @@ -0,0 +1,25 @@ +--- +platforms: + ubuntu1604: + build_targets: + - "..." + test_targets: + - "..." + ubuntu1804: + build_targets: + - "..." + test_targets: + - "..." + macos: + build_targets: + - "..." + test_targets: + - "..." + windows: + # Optional: use VS 2017 instead of 2015. + environment: + BAZEL_VC: "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\BuildTools\\VC" + build_targets: + - "..." + test_targets: + - "..." diff --git a/third_party/glog/.gitignore b/third_party/glog/.gitignore new file mode 100644 index 000000000000..b3158cf319eb --- /dev/null +++ b/third_party/glog/.gitignore @@ -0,0 +1,12 @@ +/*.cmake +/*.filters +/*.sln +/*.vcxproj +autom4te.cache +bazel-* +CMakeCache.txt +CMakeFiles/ +config.h +glog-*.tar.gz +packages/debian-* +packages/rpm-unknown diff --git a/third_party/glog/.travis.ubuntu.sh b/third_party/glog/.travis.ubuntu.sh new file mode 100755 index 000000000000..1043f31ef911 --- /dev/null +++ b/third_party/glog/.travis.ubuntu.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e +set -x + +uname -a + +cmake -H. -B_build_${TOOLCHAIN} -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/${TOOLCHAIN}.cmake" +cmake --build _build_${TOOLCHAIN} -- -j4 + +if [ "$RUN_TESTS" = true ]; then + case "$TOOLCHAIN" in linux-mingw*) + echo "copy runtime libraries needed for tests into build directory" + cp /usr/x86_64-w64-mingw32/lib/libwinpthread-1.dll /usr/lib/gcc/x86_64-w64-mingw32/7.3-win32/{libstdc++-6.dll,libgcc_s_seh-1.dll} _build_${TOOLCHAIN} + esac + CTEST_OUTPUT_ON_FAILURE=1 cmake --build _build_${TOOLCHAIN} --target test +fi + + diff --git a/third_party/glog/.travis.yml b/third_party/glog/.travis.yml new file mode 100644 index 000000000000..911fa5e0e181 --- /dev/null +++ b/third_party/glog/.travis.yml @@ -0,0 +1,119 @@ +sudo: required +language: bash +services: + - docker + +# define the build matrix +env: + global: + - PROJECT_DIR: . + - TOOLCHAIN: gcc-cxx11 + - BUILD_PACKAGES: cmake + - RUN_TESTS: true + +matrix: + include: + # Linux { + + # Ubuntu 14.04 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=trusty + BUILD_ARCH=amd64 + BUILD_PACKAGES=cmake3 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=trusty + BUILD_ARCH=i386 + BUILD_PACKAGES=cmake3 + # Ubuntu 16.04 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=xenial + BUILD_ARCH=amd64 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=xenial + BUILD_ARCH=i386 + # Ubuntu 18.04 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=gcc-cxx98 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=gcc-gnuxx11 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=gcc-cxx17 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=clang-cxx17 + BUILD_PACKAGES="cmake clang" + # } // end Linux + + # Windows build with mingw-w64 on Ubuntu 18.04 + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=linux-mingw-w64-cxx11 + BUILD_PACKAGES="cmake mingw-w64 wine-stable" + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=linux-mingw-w64-gnuxx11 + BUILD_PACKAGES="cmake mingw-w64 wine-stable" + - os: linux + env: > + BUILD_FLAVOR=ubuntu + BUILD_RELEASE=bionic + BUILD_ARCH=amd64 + TOOLCHAIN=linux-mingw-w64-cxx17 + BUILD_PACKAGES="cmake mingw-w64 wine-stable" + +before_install: + # use the Dockerfile.<distro>.template file for building the image with sed magic + - | + sed \ + -e "s/@BUILD_FLAVOR@/${BUILD_FLAVOR}/g" \ + -e "s/@BUILD_RELEASE@/${BUILD_RELEASE}/g" \ + -e "s/@BUILD_ARCH@/${BUILD_ARCH}/g" \ + -e "s/@BUILD_PACKAGES@/${BUILD_PACKAGES}/g" \ + Dockerfile.$BUILD_FLAVOR.template | tee Dockerfile.$BUILD_FLAVOR.$BUILD_RELEASE.$BUILD_ARCH + - docker build -f Dockerfile.$BUILD_FLAVOR.$BUILD_RELEASE.$BUILD_ARCH -t glog-devel . + +script: | + # run the respective .travis.<distro>.sh script + docker run \ + -e BUILD_FLAVOR="$BUILD_FLAVOR" \ + -e BUILD_RELEASE="$BUILD_RELEASE" \ + -e BUILD_ARCH="$BUILD_ARCH" \ + -e PROJECT_DIR="$PROJECT_DIR" \ + -e TOOLCHAIN="$TOOLCHAIN" \ + -e RUN_TESTS="$RUN_TESTS" \ + -it glog-devel ./.travis.$BUILD_FLAVOR.sh + diff --git a/third_party/glog/AUTHORS b/third_party/glog/AUTHORS new file mode 100644 index 000000000000..1780c12baca2 --- /dev/null +++ b/third_party/glog/AUTHORS @@ -0,0 +1,26 @@ +# This is the official list of glog authors for copyright purposes. +# This file is distinct from the CONTRIBUTORS files. +# See the latter for an explanation. +# +# Names should be added to this file as: +# Name or Organization <email address> +# The email address is not required for organizations. +# +# Please keep the list sorted. + +Abhishek Dasgupta <abhi2743@gmail.com> +Abhishek Parmar <abhishek@orng.net> +Andrew Schwartzmeyer <andrew@schwartzmeyer.com> +Andy Ying <andy@trailofbits.com> +Brian Silverman <bsilver16384@gmail.com> +Google Inc. +Guillaume Dumont <dumont.guillaume@gmail.com> +Marco Wang <m.aesophor@gmail.com> +Michael Tanner <michael@tannertaxpro.com> +MiniLight <MiniLightAR@Gmail.com> +romange <romange@users.noreply.github.com> +Roman Perepelitsa <roman.perepelitsa@gmail.com> +Sergiu Deitsch <sergiu.deitsch@gmail.com> +tbennun <tbennun@gmail.com> +Teddy Reed <teddy@prosauce.org> +Zhongming Qu <qzmfranklin@gmail.com> diff --git a/third_party/glog/BUILD b/third_party/glog/BUILD new file mode 100644 index 000000000000..37fb27bb4315 --- /dev/null +++ b/third_party/glog/BUILD @@ -0,0 +1,5 @@ +licenses(['notice']) + +load(':bazel/glog.bzl', 'glog_library') + +glog_library() diff --git a/third_party/glog/CMakeLists.txt b/third_party/glog/CMakeLists.txt new file mode 100644 index 000000000000..79b4d9ed61c9 --- /dev/null +++ b/third_party/glog/CMakeLists.txt @@ -0,0 +1,848 @@ +cmake_minimum_required (VERSION 3.0) + +if (POLICY CMP0042) + cmake_policy (SET CMP0042 NEW) +endif (POLICY CMP0042) + +if (POLICY CMP0063) + cmake_policy (SET CMP0063 NEW) +endif (POLICY CMP0063) + +project(glog VERSION 0.4.0 LANGUAGES C CXX) + +set (CPACK_PACKAGE_NAME glog) +set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "") +set (CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) +set (CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) +set (CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) +set (CPACK_PACKAGE_VERSION ${PROJECT_VERSION}) + +option (WITH_GFLAGS "Use gflags" ON) +option (WITH_THREADS "Enable multithreading support" ON) +option (WITH_TLS "Enable Thread Local Storage (TLS) support" ON) +option (BUILD_SHARED_LIBS "Build shared libraries" OFF) +option (PRINT_UNSYMBOLIZED_STACK_TRACES + "Print file offsets in traces instead of symbolizing" OFF) +option (WITH_PKGCONFIG "Enable pkg-config support" ON) +option (WITH_UNWIND "Enable libunwind support" ON) +option (WITH_SYMBOLIZE "Enable symbolize module" ON) + +if (NOT WITH_UNWIND) + set (CMAKE_DISABLE_FIND_PACKAGE_Unwind ON) +endif (NOT WITH_UNWIND) + +if (NOT WITH_THREADS) + set (CMAKE_DISABLE_FIND_PACKAGE_Threads ON) +endif (NOT WITH_THREADS) + +list (APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake) + +include (CheckCSourceCompiles) +include (CheckCXXCompilerFlag) +include (CheckCXXSourceCompiles) +include (CheckCXXSourceRuns) +include (CheckFunctionExists) +include (CheckIncludeFile) +include (CheckIncludeFileCXX) +include (CheckLibraryExists) +include (CheckStructHasMember) +include (CheckSymbolExists) +include (CheckTypeSize) +include (CMakePackageConfigHelpers) +include (CMakePushCheckState) +include (CPack) +include (CTest) +include (DetermineGflagsNamespace) +include (GetCacheVariables) +include (GNUInstallDirs) + +set (CMAKE_DEBUG_POSTFIX d) +set (CMAKE_THREAD_PREFER_PTHREAD 1) + +if (WITH_GFLAGS) + find_package (gflags 2.2.0) + + if (gflags_FOUND) + set (HAVE_LIB_GFLAGS 1) + determine_gflags_namespace (gflags_NAMESPACE) + endif (gflags_FOUND) +endif (WITH_GFLAGS) + +find_package (Threads) +find_package (Unwind) + +if (Unwind_FOUND) + set (HAVE_LIB_UNWIND 1) + set (HAVE_UNWIND_H 1) +endif (Unwind_FOUND) + +check_include_file (dlfcn.h HAVE_DLFCN_H) +check_include_file (execinfo.h HAVE_EXECINFO_H) +check_include_file (glob.h HAVE_GLOB_H) +check_include_file (inttypes.h HAVE_INTTYPES_H) +check_include_file (memory.h HAVE_MEMORY_H) +check_include_file (pwd.h HAVE_PWD_H) +check_include_file (stdint.h HAVE_STDINT_H) +check_include_file (stdlib.h HAVE_STDLIB_H) +check_include_file (string.h HAVE_STRING_H) +check_include_file (strings.h HAVE_STRINGS_H) +check_include_file (sys/stat.h HAVE_SYS_STAT_H) +check_include_file (sys/syscall.h HAVE_SYS_SYSCALL_H) +check_include_file (sys/time.h HAVE_SYS_TIME_H) +check_include_file (sys/types.h HAVE_SYS_TYPES_H) +check_include_file (sys/utsname.h HAVE_SYS_UTSNAME_H) +check_include_file (sys/wait.h HAVE_SYS_WAIT_H) +check_include_file (syscall.h HAVE_SYSCALL_H) +check_include_file (syslog.h HAVE_SYSLOG_H) +check_include_file (ucontext.h HAVE_UCONTEXT_H) +check_include_file (unistd.h HAVE_UNISTD_H) +check_include_file (pwd.h HAVE_PWD_H) + +check_include_file_cxx ("ext/hash_map" HAVE_EXT_HASH_MAP) +check_include_file_cxx ("ext/hash_set" HAVE_EXT_HASH_SET) +check_include_file_cxx ("ext/slist" HAVE_EXT_SLIST) +check_include_file_cxx ("tr1/unordered_map" HAVE_TR1_UNORDERED_MAP) +check_include_file_cxx ("tr1/unordered_set" HAVE_TR1_UNORDERED_SET) +check_include_file_cxx ("unordered_map" HAVE_UNORDERED_MAP) +check_include_file_cxx ("unordered_set" HAVE_UNORDERED_SET) + +check_type_size ("unsigned __int16" HAVE___UINT16) +check_type_size (u_int16_t HAVE_U_INT16_T) +check_type_size (uint16_t HAVE_UINT16_T) + +check_function_exists (dladdr HAVE_DLADDR) +check_function_exists (fcntl HAVE_FCNTL) +check_function_exists (pread HAVE_PREAD) +check_function_exists (pwrite HAVE_PWRITE) +check_function_exists (sigaction HAVE_SIGACTION) +check_function_exists (sigaltstack HAVE_SIGALSTACK) + +# NOTE gcc does not fail if you pass a non-existent -Wno-* option as an +# argument. However, it will happily fail if you pass the corresponding -W* +# option. So, we check whether options that disable warnings exist by testing +# the availability of the corresponding option that enables the warning. This +# eliminates the need to check for compiler for several (mainly Clang) options. + +check_cxx_compiler_flag (-Wdeprecated HAVE_NO_DEPRECATED) +check_cxx_compiler_flag (-Wunnamed-type-template-args + HAVE_NO_UNNAMED_TYPE_TEMPLATE_ARGS) + +# NOTE: Cannot use check_function_exists here since >=vc-14.0 can define +# snprintf as an inline function +check_symbol_exists (snprintf stdio.h HAVE_SNPRINTF) + +check_library_exists (dbghelp UnDecorateSymbolName "" HAVE_DBGHELP) + +check_c_source_compiles (" +#include <stdlib.h> +static void foo(void) __attribute__ ((unused)); +int main(void) { return 0; } +" HAVE___ATTRIBUTE__) + +check_c_source_compiles (" +#include <stdlib.h> +static void foo(void) __attribute__ ((visibility(\"default\"))); +int main(void) { return 0; } +" HAVE___ATTRIBUTE__VISIBILITY_DEFAULT) + +check_c_source_compiles (" +#include <stdlib.h> +static void foo(void) __attribute__ ((visibility(\"hidden\"))); +int main(void) { return 0; } +" HAVE___ATTRIBUTE__VISIBILITY_HIDDEN) + +check_c_source_compiles (" +int main(void) { if (__builtin_expect(0, 0)) return 1; return 0; } +" HAVE___BUILTIN_EXPECT) + +check_c_source_compiles (" +int main(void) +{ + int a; if (__sync_val_compare_and_swap(&a, 0, 1)) return 1; return 0; +} +" HAVE___SYNC_VAL_COMPARE_AND_SWAP) + +check_c_source_compiles (" +#define _XOPEN_SOURCE 500 +#include <pthread.h> +int main(void) +{ + pthread_rwlock_t l; + pthread_rwlock_init(&l, NULL); + pthread_rwlock_rdlock(&l); + return 0; +} +" HAVE_RWLOCK) + +check_c_source_compiles (" +__declspec(selectany) int a; +int main(void) { return 0; } +" HAVE___DECLSPEC) + +check_cxx_source_compiles (" +#include <vector> +vector<int> t; int main() { } +" STL_NO_NAMESPACE) + +check_cxx_source_compiles (" +#include <vector> +std::vector<int> t; int main() { } +" STL_STD_NAMESPACE) + +check_cxx_source_compiles (" +#include <iostream> +std::ostream& operator<<(std::ostream&, struct s); +using ::operator<<; +int main() { } +" HAVE_USING_OPERATOR) + +check_cxx_source_compiles (" +namespace Outer { namespace Inner { int i = 0; }} +using namespace Outer::Inner;; +int main() { return i; } +" HAVE_NAMESPACES) + +check_cxx_source_compiles (" +__thread int tls; +int main() { } +" HAVE_GCC_TLS) + +check_cxx_source_compiles (" +__declspec(thread) int tls; +int main() { } +" HAVE_MSVC_TLS) + +check_cxx_source_compiles (" +thread_local int tls; +int main() { } +" HAVE_CXX11_TLS) + +check_cxx_source_compiles (" +#include <type_traits> +std::aligned_storage<sizeof(char), alignof(char)>::type data; +int main() { } +" HAVE_ALIGNED_STORAGE) + +if (WITH_TLS) + # Cygwin does not support the thread attribute. Don't bother. + if (HAVE_GCC_TLS) + set (GLOG_THREAD_LOCAL_STORAGE "__thread") + elseif (HAVE_MSVC_TLS) + set (GLOG_THREAD_LOCAL_STORAGE "__declspec(thread)") + elseif (HAVE_CXX11_TLS) + set (GLOG_THREAD_LOCAL_STORAGE thread_local) + endif (HAVE_GCC_TLS) +endif (WITH_TLS) + +set (_PC_FIELDS + "gregs[REG_PC]" + "gregs[REG_EIP]" + "gregs[REG_RIP]" + "sc_ip" + "uc_regs->gregs[PT_NIP]" + "gregs[R15]" + "arm_pc" + "mc_eip" + "mc_rip" + "__gregs[REG_EIP]" + "__gregs[REG_RIP]" + "ss.eip" + "__ss.__eip" + "ss.rip" + "__ss.__rip" + "ss.srr0" + "__ss.__srr0" +) + +set (_PC_HEADERS ucontext.h signal.h) + +if (HAVE_UCONTEXT_H AND NOT PC_FROM_UCONTEXT) + foreach (_PC_FIELD ${_PC_FIELDS}) + foreach (_PC_HEADER ${_PC_HEADERS}) + set (_TMP + ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/uctfield.c) + file (WRITE ${_TMP} " +#define _GNU_SOURCE 1 +#include <${_PC_HEADER}> +int main(void) +{ + ucontext_t u; + return u.${_PC_FIELD} == 0; +} +") + try_compile (HAVE_PC_FROM_UCONTEXT ${CMAKE_CURRENT_BINARY_DIR} ${_TMP} + COMPILE_DEFINITIONS _GNU_SOURCE=1) + + if (HAVE_PC_FROM_UCONTEXT) + set (PC_FROM_UCONTEXT ${_PC_FIELD} CACHE) + endif (HAVE_PC_FROM_UCONTEXT) + endforeach (_PC_HEADER) + endforeach (_PC_FIELD) +endif (HAVE_UCONTEXT_H AND NOT PC_FROM_UCONTEXT) + +if (STL_STD_NAMESPACE) + set (STL_NAMESPACE std) +else (STL_STD_NAMESPACE) + set (STL_NAMESPACE "") +endif (STL_STD_NAMESPACE) + +set (GOOGLE_NAMESPACE google) +set (_START_GOOGLE_NAMESPACE_ "namespace ${GOOGLE_NAMESPACE} {") +set (_END_GOOGLE_NAMESPACE_ "}") + +if (HAVE___UINT16) + set (ac_cv_have___uint16 1) +else (HAVE___UINT16) + set (ac_cv_have___uint16 0) +endif (HAVE___UINT16) + +if (HAVE_INTTYPES_H) + set (ac_cv_have_inttypes_h 1) +else (HAVE_INTTYPES_H) + set (ac_cv_have_inttypes_h 0) +endif (HAVE_INTTYPES_H) + +if (HAVE_LIB_GFLAGS) + set (ac_cv_have_libgflags 1) +else (HAVE_LIB_GFLAGS) + set (ac_cv_have_libgflags 0) +endif (HAVE_LIB_GFLAGS) + +if (HAVE_STDINT_H) + set (ac_cv_have_stdint_h 1) +else (HAVE_STDINT_H) + set (ac_cv_have_stdint_h 0) +endif (HAVE_STDINT_H) + +if (HAVE_SYS_TYPES_H) + set (ac_cv_have_systypes_h 1) +else (HAVE_SYS_TYPES_H) + set (ac_cv_have_systypes_h 0) +endif (HAVE_SYS_TYPES_H) + +if (HAVE_U_INT16_T) + set (ac_cv_have_u_int16_t 1) +else (HAVE_U_INT16_T) + set (ac_cv_have_u_int16_t 0) +endif (HAVE_U_INT16_T) + +if (HAVE_UINT16_T) + set (ac_cv_have_uint16_t 1) +else (HAVE_UINT16_T) + set (ac_cv_have_uint16_t 0) +endif (HAVE_UINT16_T) + +if (HAVE_UNISTD_H) + set (ac_cv_have_unistd_h 1) +else (HAVE_UNISTD_H) + set (ac_cv_have_unistd_h 0) +endif (HAVE_UNISTD_H) + +set (ac_google_namespace ${GOOGLE_NAMESPACE}) +set (ac_google_end_namespace ${_END_GOOGLE_NAMESPACE_}) +set (ac_google_start_namespace ${_START_GOOGLE_NAMESPACE_}) + +if (HAVE___ATTRIBUTE__) + set (ac_cv___attribute___noreturn "__attribute__((noreturn))") + set (ac_cv___attribute___noinline "__attribute__((noinline))") +elseif (HAVE___DECLSPEC) + set (ac_cv___attribute___noreturn "__declspec(noreturn)") + #set (ac_cv___attribute___noinline "__declspec(noinline)") +endif (HAVE___ATTRIBUTE__) + +if (HAVE___BUILTIN_EXPECT) + set (ac_cv_have___builtin_expect 1) +else (HAVE___BUILTIN_EXPECT) + set (ac_cv_have___builtin_expect 0) +endif (HAVE___BUILTIN_EXPECT) + +if (HAVE_USING_OPERATOR) + set (ac_cv_cxx_using_operator 1) +else (HAVE_USING_OPERATOR) + set (ac_cv_cxx_using_operator 0) +endif (HAVE_USING_OPERATOR) + +if (HAVE_EXECINFO_H) + set (HAVE_STACKTRACE 1) +endif (HAVE_EXECINFO_H) + +if (WITH_SYMBOLIZE) + if (WIN32 OR CYGWIN) + cmake_push_check_state (RESET) + set (CMAKE_REQUIRED_LIBRARIES DbgHelp) + + check_cxx_source_runs ([=[ + #include <windows.h> + #include <dbghelp.h> + #include <cstdlib> + + void foobar() { } + + int main() + { + HANDLE process = GetCurrentProcess(); + + if (!SymInitialize(process, NULL, TRUE)) + return EXIT_FAILURE; + + char buf[sizeof(SYMBOL_INFO) + MAX_SYM_NAME]; + SYMBOL_INFO *symbol = reinterpret_cast<SYMBOL_INFO *>(buf); + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + symbol->MaxNameLen = MAX_SYM_NAME; + + void* const pc = reinterpret_cast<void*>(&foobar); + BOOL ret = SymFromAddr(process, reinterpret_cast<DWORD64>(pc), 0, symbol); + + return ret ? EXIT_SUCCESS : EXIT_FAILURE; + } + ]=] HAVE_SYMBOLIZE) + + cmake_pop_check_state () + + if (HAVE_SYMBOLIZE) + set (HAVE_STACKTRACE 1) + endif (HAVE_SYMBOLIZE) + elseif (UNIX OR (APPLE AND HAVE_DLADDR)) + set (HAVE_SYMBOLIZE 1) + endif (WIN32 OR CYGWIN) +endif (WITH_SYMBOLIZE) + +check_cxx_source_compiles (" +#include <cstdlib> +#include <time.h> +int main() +{ + time_t timep; + struct tm result; + localtime_r(&timep, &result); + return EXIT_SUCCESS; +} +" HAVE_LOCALTIME_R) + +set (SIZEOF_VOID_P ${CMAKE_SIZEOF_VOID_P}) + +if (WITH_THREADS AND Threads_FOUND) + if (CMAKE_USE_PTHREADS_INIT) + set (HAVE_PTHREAD 1) + endif (CMAKE_USE_PTHREADS_INIT) +else (WITH_THREADS AND Threads_FOUND) + set (NO_THREADS 1) +endif (WITH_THREADS AND Threads_FOUND) + +set (TEST_SRC_DIR \"${CMAKE_CURRENT_SOURCE_DIR}\") + +configure_file (src/config.h.cmake.in config.h) +configure_file (src/glog/logging.h.in glog/logging.h @ONLY) +configure_file (src/glog/raw_logging.h.in glog/raw_logging.h @ONLY) +configure_file (src/glog/stl_logging.h.in glog/stl_logging.h @ONLY) +configure_file (src/glog/vlog_is_on.h.in glog/vlog_is_on.h @ONLY) +if (WITH_PKGCONFIG) + set (VERSION ${PROJECT_VERSION}) + set (prefix ${CMAKE_INSTALL_PREFIX}) + set (exec_prefix ${CMAKE_INSTALL_FULL_BINDIR}) + set (libdir ${CMAKE_INSTALL_FULL_LIBDIR}) + set (includedir ${CMAKE_INSTALL_FULL_INCLUDEDIR}) + + configure_file ( + "${PROJECT_SOURCE_DIR}/libglog.pc.in" + "${PROJECT_BINARY_DIR}/libglog.pc" + @ONLY + ) + + unset (VERSION) + unset (prefix) + unset (exec_prefix) + unset (libdir) + unset (includedir) +endif (WITH_PKGCONFIG) + +set (CMAKE_CXX_VISIBILITY_PRESET default) +set (CMAKE_VISIBILITY_INLINES_HIDDEN 1) + +set (GLOG_PUBLIC_H + ${CMAKE_CURRENT_BINARY_DIR}/glog/logging.h + ${CMAKE_CURRENT_BINARY_DIR}/glog/raw_logging.h + ${CMAKE_CURRENT_BINARY_DIR}/glog/stl_logging.h + ${CMAKE_CURRENT_BINARY_DIR}/glog/vlog_is_on.h + src/glog/log_severity.h +) + +set (GLOG_SRCS + ${GLOG_PUBLIC_H} + src/base/commandlineflags.h + src/base/googleinit.h + src/base/mutex.h + src/demangle.cc + src/demangle.h + src/logging.cc + src/raw_logging.cc + src/symbolize.cc + src/symbolize.h + src/utilities.cc + src/utilities.h + src/vlog_is_on.cc +) + +if (HAVE_PTHREAD OR WIN32) + list (APPEND GLOG_SRCS src/signalhandler.cc) +endif (HAVE_PTHREAD OR WIN32) + +if (WIN32) + list (APPEND GLOG_SRCS + src/windows/port.cc + src/windows/port.h + ) +endif (WIN32) + +add_compile_options ($<$<AND:$<BOOL:${HAVE_NO_UNNAMED_TYPE_TEMPLATE_ARGS}>,$<NOT:$<CXX_COMPILER_ID:GNU>>>:-Wno-unnamed-type-template-args>) + +set (_glog_CMake_BINDIR ${CMAKE_INSTALL_BINDIR}) +set (_glog_CMake_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}) +set (_glog_CMake_LIBDIR ${CMAKE_INSTALL_LIBDIR}) +set (_glog_CMake_INSTALLDIR ${_glog_CMake_LIBDIR}/cmake/glog) + +set (_glog_CMake_DIR glog/cmake) +set (_glog_CMake_DATADIR ${CMAKE_INSTALL_DATAROOTDIR}/${_glog_CMake_DIR}) +set (_glog_BINARY_CMake_DATADIR + ${CMAKE_CURRENT_BINARY_DIR}/${_glog_CMake_DATADIR}) + +# Add additional CMake find modules here. +set (_glog_CMake_MODULES) + +if (Unwind_FOUND) + # Copy the module only if libunwind is actually used. + list (APPEND _glog_CMake_MODULES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/FindUnwind.cmake) +endif (Unwind_FOUND) + +# Generate file name for each module in the binary directory +foreach (_file ${_glog_CMake_MODULES}) + get_filename_component (_module "${_file}" NAME) + + list (APPEND _glog_BINARY_CMake_MODULES + ${_glog_BINARY_CMake_DATADIR}/${_module}) +endforeach (_file) + +if (_glog_CMake_MODULES) + # Copy modules to binary directory during the build + add_custom_command (OUTPUT ${_glog_BINARY_CMake_MODULES} + COMMAND ${CMAKE_COMMAND} -E make_directory + ${_glog_BINARY_CMake_DATADIR} + COMMAND ${CMAKE_COMMAND} -E copy ${_glog_CMake_MODULES} + ${_glog_BINARY_CMake_DATADIR} + DEPENDS ${_glog_CMake_MODULES} + COMMENT "Copying find modules..." + ) +endif (_glog_CMake_MODULES) + +add_library (glog + ${GLOG_SRCS} + ${_glog_BINARY_CMake_MODULES} +) + +add_library(glog::glog ALIAS glog) + +set_target_properties (glog PROPERTIES POSITION_INDEPENDENT_CODE ON) + +if (Unwind_FOUND) + target_link_libraries (glog PUBLIC unwind::unwind) + set (Unwind_DEPENDENCY "find_dependency (Unwind ${Unwind_VERSION})") +endif (Unwind_FOUND) + +if (HAVE_DBGHELP) + target_link_libraries (glog PUBLIC dbghelp) +endif (HAVE_DBGHELP) + +if (HAVE_PTHREAD) + target_link_libraries (glog PUBLIC ${CMAKE_THREAD_LIBS_INIT}) +endif (HAVE_PTHREAD) + +if (WIN32 AND HAVE_SNPRINTF) + set_property (SOURCE src/windows/port.cc APPEND PROPERTY COMPILE_DEFINITIONS + HAVE_SNPRINTF) +endif (WIN32 AND HAVE_SNPRINTF) + +if (gflags_FOUND) + target_link_libraries (glog PUBLIC gflags) + + if (NOT BUILD_SHARED_LIBS) + # Don't use __declspec(dllexport|dllimport) if this is a static build + targeT_compile_definitions (glog PUBLIC GFLAGS_DLL_DECLARE_FLAG= GFLAGS_DLL_DEFINE_FLAG=) + endif (NOT BUILD_SHARED_LIBS) +endif (gflags_FOUND) + +set_target_properties (glog PROPERTIES VERSION ${PROJECT_VERSION}) +set_target_properties (glog PROPERTIES SOVERSION ${PROJECT_VERSION_MAJOR}) + +if (WIN32) + target_compile_definitions (glog PUBLIC GLOG_NO_ABBREVIATED_SEVERITIES) +endif (WIN32) + +set_target_properties (glog PROPERTIES PUBLIC_HEADER "${GLOG_PUBLIC_H}") + +target_include_directories (glog BEFORE PUBLIC + "$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>" + "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src>" + "$<INSTALL_INTERFACE:${_glog_CMake_INCLUDE_DIR}>" + PRIVATE ${CMAKE_CURRENT_BINARY_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src) + +if (WIN32) + target_include_directories (glog PUBLIC + "$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/src/windows>" + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/windows) +endif (WIN32) + +set_target_properties (glog PROPERTIES DEFINE_SYMBOL LIBGLOG_EXPORTS) + +if (NOT BUILD_SHARED_LIBS) + target_compile_definitions (glog PUBLIC GOOGLE_GLOG_DLL_DECL=) +else (NOT BUILD_SHARED_LIBS) + target_compile_definitions (glog PRIVATE GOOGLE_GLOG_IS_A_DLL=1) + + if (HAVE___DECLSPEC) + set (_EXPORT "__declspec(dllexport)") + set (_IMPORT "__declspec(dllimport)") + elseif (HAVE___ATTRIBUTE__VISIBILITY_DEFAULT) + set (_EXPORT "__attribute__((visibility(\"default\")))") + set (_IMPORT "") + endif (HAVE___DECLSPEC) + + target_compile_definitions (glog PRIVATE + "GOOGLE_GLOG_DLL_DECL=${_EXPORT}") + target_compile_definitions (glog INTERFACE + "GOOGLE_GLOG_DLL_DECL=${_IMPORT}") + target_compile_definitions (glog INTERFACE + "GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS=${_IMPORT}") +endif (NOT BUILD_SHARED_LIBS) + +# Unit testing + +if (BUILD_TESTING) + add_executable (logging_unittest + src/logging_unittest.cc + ) + + target_link_libraries (logging_unittest PRIVATE glog) + + add_executable (stl_logging_unittest + src/stl_logging_unittest.cc + ) + + target_link_libraries (stl_logging_unittest PRIVATE glog) + + if (HAVE_NO_DEPRECATED) + set_property (TARGET stl_logging_unittest APPEND PROPERTY COMPILE_OPTIONS + -Wno-deprecated) + endif (HAVE_NO_DEPRECATED) + + if (HAVE_UNORDERED_MAP AND HAVE_UNORDERED_SET) + target_compile_definitions (stl_logging_unittest PRIVATE + GLOG_STL_LOGGING_FOR_UNORDERED) + endif (HAVE_UNORDERED_MAP AND HAVE_UNORDERED_SET) + + if (HAVE_TR1_UNORDERED_MAP AND HAVE_TR1_UNORDERED_SET) + target_compile_definitions (stl_logging_unittest PRIVATE + GLOG_STL_LOGGING_FOR_TR1_UNORDERED) + endif (HAVE_TR1_UNORDERED_MAP AND HAVE_TR1_UNORDERED_SET) + + if (HAVE_EXT_HASH_MAP AND HAVE_EXT_HASH_SET) + target_compile_definitions (stl_logging_unittest PRIVATE + GLOG_STL_LOGGING_FOR_EXT_HASH) + endif (HAVE_EXT_HASH_MAP AND HAVE_EXT_HASH_SET) + + if (HAVE_EXT_SLIST) + target_compile_definitions (stl_logging_unittest PRIVATE + GLOG_STL_LOGGING_FOR_EXT_SLIST) + endif (HAVE_EXT_SLIST) + + if (HAVE_SYMBOLIZE) + add_executable (symbolize_unittest + src/symbolize_unittest.cc + ) + + target_link_libraries (symbolize_unittest PRIVATE glog) + endif (HAVE_SYMBOLIZE) + + add_executable (demangle_unittest + src/demangle_unittest.cc + ) + + target_link_libraries (demangle_unittest PRIVATE glog) + + if (HAVE_STACKTRACE) + add_executable (stacktrace_unittest + src/stacktrace_unittest.cc + ) + + target_link_libraries (stacktrace_unittest PRIVATE glog) + endif (HAVE_STACKTRACE) + + add_executable (utilities_unittest + src/utilities_unittest.cc + ) + + target_link_libraries (utilities_unittest PRIVATE glog) + + if (HAVE_STACKTRACE AND HAVE_SYMBOLIZE) + add_executable (signalhandler_unittest + src/signalhandler_unittest.cc + ) + + target_link_libraries (signalhandler_unittest PRIVATE glog) + endif (HAVE_STACKTRACE AND HAVE_SYMBOLIZE) + + add_test (NAME demangle COMMAND demangle_unittest) + add_test (NAME logging COMMAND logging_unittest) + + if (TARGET signalhandler_unittest) + add_test (NAME signalhandler COMMAND signalhandler_unittest) + endif (TARGET signalhandler_unittest) + + if (TARGET stacktrace_unittest) + add_test (NAME stacktrace COMMAND stacktrace_unittest) + set_tests_properties(stacktrace PROPERTIES TIMEOUT 30) + endif (TARGET stacktrace_unittest) + + add_test (NAME stl_logging COMMAND stl_logging_unittest) + + if (TARGET symbolize_unittest) + add_test (NAME symbolize COMMAND symbolize_unittest) + endif (TARGET symbolize_unittest) + + # Generate an initial cache + + get_cache_variables (_CACHEVARS EXCLUDE CMAKE_MAKE_PROGRAM) + + set (_INITIAL_CACHE + ${CMAKE_CURRENT_BINARY_DIR}/test_package_config/glog_package_config_initial_cache.cmake) + + # Package config test + + add_test (NAME cmake_package_config_init COMMAND ${CMAKE_COMMAND} + -DTEST_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/test_package_config + -DINITIAL_CACHE=${_INITIAL_CACHE} + -DCACHEVARS=${_CACHEVARS} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestInitPackageConfig.cmake + ) + + add_test (NAME cmake_package_config_generate COMMAND ${CMAKE_COMMAND} + -DPATH=$ENV{PATH} + -DINITIAL_CACHE=${_INITIAL_CACHE} + -DTEST_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}/test_package_config/working_config + -DSOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}/src/package_config_unittest/working_config + -DPACKAGE_DIR=${CMAKE_CURRENT_BINARY_DIR} + -DGENERATOR=${CMAKE_GENERATOR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/TestPackageConfig.cmake + ) + + add_test (NAME cmake_package_config_build COMMAND + ${CMAKE_COMMAND} --build ${CMAKE_CURRENT_BINARY_DIR}/test_package_config/working_config + ) + + add_test (NAME cmake_package_config_cleanup COMMAND ${CMAKE_COMMAND} -E + remove_directory + ${CMAKE_CURRENT_BINARY_DIR}/test_package_config + ) + + # Fixtures setup + set_tests_properties (cmake_package_config_init PROPERTIES FIXTURES_SETUP + cmake_package_config) + set_tests_properties (cmake_package_config_generate PROPERTIES FIXTURES_SETUP + cmake_package_config_working) + + # Fixtures cleanup + set_tests_properties (cmake_package_config_cleanup PROPERTIES FIXTURES_CLEANUP + cmake_package_config) + + # Fixture requirements + set_tests_properties (cmake_package_config_generate PROPERTIES + FIXTURES_REQUIRED cmake_package_config) + set_tests_properties (cmake_package_config_build PROPERTIES + FIXTURES_REQUIRED "cmake_package_config;cmake_package_config_working") +endif (BUILD_TESTING) + +install (TARGETS glog + EXPORT glog-targets + RUNTIME DESTINATION ${_glog_CMake_BINDIR} + PUBLIC_HEADER DESTINATION ${_glog_CMake_INCLUDE_DIR}/glog + LIBRARY DESTINATION ${_glog_CMake_LIBDIR} + ARCHIVE DESTINATION ${_glog_CMake_LIBDIR}) + +if (WITH_PKGCONFIG) + install ( + FILES "${PROJECT_BINARY_DIR}/libglog.pc" + DESTINATION "${_glog_CMake_LIBDIR}/pkgconfig" + ) +endif (WITH_PKGCONFIG) + +set (glog_CMake_VERSION 3.0) + +if (gflags_FOUND) + # Ensure clients locate only the package config and not third party find + # modules having the same name. This avoid cmake_policy PUSH/POP errors. + if (CMAKE_VERSION VERSION_LESS 3.9) + set (gflags_DEPENDENCY "find_dependency (gflags ${gflags_VERSION})") + else (CMAKE_VERSION VERSION_LESS 3.9) + # Passing additional find_package arguments to find_dependency is possible + # starting with CMake 3.9. + set (glog_CMake_VERSION 3.9) + set (gflags_DEPENDENCY "find_dependency (gflags ${gflags_VERSION} NO_MODULE)") + endif (CMAKE_VERSION VERSION_LESS 3.9) +endif (gflags_FOUND) + +configure_package_config_file (glog-config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/glog-config.cmake + INSTALL_DESTINATION ${_glog_CMake_INSTALLDIR} + NO_CHECK_REQUIRED_COMPONENTS_MACRO) + +write_basic_package_version_file (glog-config-version.cmake VERSION + ${PROJECT_VERSION} COMPATIBILITY SameMajorVersion) + +export (TARGETS glog NAMESPACE glog:: FILE glog-targets.cmake) +export (PACKAGE glog) + +get_filename_component (_PREFIX "${CMAKE_INSTALL_PREFIX}" ABSOLUTE) + +# Directory containing the find modules relative to the config install +# directory. +file (RELATIVE_PATH glog_REL_CMake_MODULES + ${_PREFIX}/${_glog_CMake_INSTALLDIR} + ${_PREFIX}/${_glog_CMake_DATADIR}/glog-modules.cmake) + +get_filename_component (glog_REL_CMake_DATADIR ${glog_REL_CMake_MODULES} + DIRECTORY) + +set (glog_FULL_CMake_DATADIR + ${CMAKE_CURRENT_BINARY_DIR}/${_glog_CMake_DATADIR}) + +configure_file (glog-modules.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/glog-modules.cmake @ONLY) + +install (CODE +" +set (glog_FULL_CMake_DATADIR \"\\\${CMAKE_CURRENT_LIST_DIR}/${glog_REL_CMake_DATADIR}\") +configure_file (${CMAKE_CURRENT_SOURCE_DIR}/glog-modules.cmake.in + \"${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/glog-modules.cmake\" @ONLY) +file (INSTALL + \"${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/glog-modules.cmake\" + DESTINATION + \"\${CMAKE_INSTALL_PREFIX}/${_glog_CMake_INSTALLDIR}\") +" + COMPONENT Development +) + +install (FILES + ${CMAKE_CURRENT_BINARY_DIR}/glog-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/glog-config-version.cmake + ${CMAKE_CURRENT_BINARY_DIR}/glog-modules.cmake + DESTINATION ${_glog_CMake_INSTALLDIR}) + +# Find modules in share/glog/cmake +install (DIRECTORY ${_glog_BINARY_CMake_DATADIR} + DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/glog + COMPONENT Development + FILES_MATCHING PATTERN "*.cmake" +) + +install (EXPORT glog-targets NAMESPACE glog:: DESTINATION + ${_glog_CMake_INSTALLDIR}) diff --git a/third_party/glog/CONTRIBUTING.md b/third_party/glog/CONTRIBUTING.md new file mode 100644 index 000000000000..43de4c9d4709 --- /dev/null +++ b/third_party/glog/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# How to contribute # + +We'd love to accept your patches and contributions to this project. There are +a just a few small guidelines you need to follow. + + +## Contributor License Agreement ## + +Contributions to any Google project must be accompanied by a Contributor +License Agreement. This is not a copyright **assignment**, it simply gives +Google permission to use and redistribute your contributions as part of the +project. + + * If you are an individual writing original source code and you're sure you + own the intellectual property, then you'll need to sign an [individual + CLA][]. + + * If you work for a company that wants to allow you to contribute your work, + then you'll need to sign a [corporate CLA][]. + +You generally only need to submit a CLA once, so if you've already submitted +one (even if it was for a different project), you probably don't need to do it +again. + +[individual CLA]: https://developers.google.com/open-source/cla/individual +[corporate CLA]: https://developers.google.com/open-source/cla/corporate + +Once your CLA is submitted (or if you already submitted one for +another Google project), make a commit adding yourself to the +[AUTHORS][] and [CONTRIBUTORS][] files. This commit can be part +of your first [pull request][]. + +[AUTHORS]: AUTHORS +[CONTRIBUTORS]: CONTRIBUTORS + + +## Submitting a patch ## + + 1. It's generally best to start by opening a new issue describing the bug or + feature you're intending to fix. Even if you think it's relatively minor, + it's helpful to know what people are working on. Mention in the initial + issue that you are planning to work on that bug or feature so that it can + be assigned to you. + + 1. Follow the normal process of [forking][] the project, and setup a new + branch to work in. It's important that each group of changes be done in + separate branches in order to ensure that a pull request only includes the + commits related to that bug or feature. + + 1. Do your best to have [well-formed commit messages][] for each change. + This provides consistency throughout the project, and ensures that commit + messages are able to be formatted properly by various git tools. + + 1. Finally, push the commits to your fork and submit a [pull request][]. + +[forking]: https://help.github.com/articles/fork-a-repo +[well-formed commit messages]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html +[pull request]: https://help.github.com/articles/creating-a-pull-request diff --git a/third_party/glog/CONTRIBUTORS b/third_party/glog/CONTRIBUTORS new file mode 100644 index 000000000000..d90f85988543 --- /dev/null +++ b/third_party/glog/CONTRIBUTORS @@ -0,0 +1,48 @@ +# People who have agreed to one of the CLAs and can contribute patches. +# The AUTHORS file lists the copyright holders; this file +# lists people. For example, Google employees are listed here +# but not in AUTHORS, because Google holds the copyright. +# +# Names should be added to this file only after verifying that +# the individual or the individual's organization has agreed to +# the appropriate Contributor License Agreement, found here: +# +# https://developers.google.com/open-source/cla/individual +# https://developers.google.com/open-source/cla/corporate +# +# The agreement for individuals can be filled out on the web. +# +# When adding J Random Contributor's name to this file, +# either J's name or J's organization's name should be +# added to the AUTHORS file, depending on whether the +# individual or corporate CLA was used. +# +# Names should be added to this file as: +# Name <email address> +# +# Please keep the list sorted. + +Abhishek Dasgupta <abhi2743@gmail.com> +Abhishek Parmar <abhishek@orng.net> +Andrew Schwartzmeyer <andrew@schwartzmeyer.com> +Andy Ying <andy@trailofbits.com> +Brian Silverman <bsilver16384@gmail.com> +Fumitoshi Ukai <ukai@google.com> +Guillaume Dumont <dumont.guillaume@gmail.com> +Håkan L. S. Younes <hyounes@google.com> +Ivan Penkov <ivanpe@google.com> +Jacob Trimble <modmaker@google.com> +Jim Ray <jimray@google.com> +Marco Wang <m.aesophor@gmail.com> +Michael Darr <mdarr@matician.com> +Michael Tanner <michael@tannertaxpro.com> +MiniLight <MiniLightAR@Gmail.com> +Peter Collingbourne <pcc@google.com> +Rodrigo Queiro <rodrigoq@google.com> +romange <romange@users.noreply.github.com> +Roman Perepelitsa <roman.perepelitsa@gmail.com> +Sergiu Deitsch <sergiu.deitsch@gmail.com> +Shinichiro Hamaji <hamaji@google.com> +tbennun <tbennun@gmail.com> +Teddy Reed <teddy@prosauce.org> +Zhongming Qu <qzmfranklin@gmail.com> diff --git a/third_party/glog/COPYING b/third_party/glog/COPYING new file mode 100644 index 000000000000..38396b580b37 --- /dev/null +++ b/third_party/glog/COPYING @@ -0,0 +1,65 @@ +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +A function gettimeofday in utilities.cc is based on + +http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/COPYING&q=GetSystemTimeAsFileTime%20license:bsd + +The license of this code is: + +Copyright (c) 2003-2008, Jouni Malinen <j@w1.fi> and contributors +All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name(s) of the above-listed copyright holder(s) nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/glog/ChangeLog b/third_party/glog/ChangeLog new file mode 100644 index 000000000000..f8e43f8b39d3 --- /dev/null +++ b/third_party/glog/ChangeLog @@ -0,0 +1,99 @@ +2019-01-22 Google Inc. <opensource@google.com> + + * google-glog: version 0.4.0. + * See git log for the details. + +2017-05-09 Google Inc. <opensource@google.com> + + * google-glog: version 0.3.5 + * See git log for the details. + +2015-03-09 Google Inc. <opensource@google.com> + + * google-glog: version 0.3.4 + * See git log for the details. + +2013-02-01 Google Inc. <opensource@google.com> + + * google-glog: version 0.3.3 + * Add --disable-rtti option for configure. + * Visual Studio build and test fix. + * QNX build fix (thanks vanuan). + * Reduce warnings. + * Fixed LOG_SYSRESULT (thanks ukai). + * FreeBSD build fix (thanks yyanagisawa). + * Clang build fix. + * Now users can re-initialize glog after ShutdownGoogleLogging. + * Color output support by GLOG_colorlogtostderr (thanks alexs). + * Now glog's ABI around flags are compatible with gflags. + * Document mentions how to modify flags from user programs. + +2012-01-12 Google Inc. <opensource@google.com> + + * google-glog: version 0.3.2 + * Clang support. + * Demangler and stacktrace improvement for newer GCCs. + * Now fork(2) doesn't mess up log files. + * Make valgrind happier. + * Reduce warnings for more -W options. + * Provide a workaround for ERROR defined by windows.h. + +2010-06-15 Google Inc. <opensource@google.com> + + * google-glog: version 0.3.1 + * GLOG_* environment variables now work even when gflags is installed. + * Snow leopard support. + * Now we can build and test from out side tree. + * Add DCHECK_NOTNULL. + * Add ShutdownGoogleLogging to close syslog (thanks DGunchev) + * Fix --enable-frame-pointers option (thanks kazuki.ohta) + * Fix libunwind detection (thanks giantchen) + +2009-07-30 Google Inc. <opensource@google.com> + + * google-glog: version 0.3.0 + * Fix a deadlock happened when user uses glog with recent gflags. + * Suppress several unnecessary warnings (thanks keir). + * NetBSD and OpenBSD support. + * Use Win32API GetComputeNameA properly (thanks magila). + * Fix user name detection for Windows (thanks ademin). + * Fix several minor bugs. + +2009-04-10 Google Inc. <opensource@google.com> + * google-glog: version 0.2.1 + * Fix timestamps of VC++ version. + * Add pkg-config support (thanks Tomasz) + * Fix build problem when building with gtest (thanks Michael) + * Add --with-gflags option for configure (thanks Michael) + * Fixes for GCC 4.4 (thanks John) + +2009-01-23 Google Inc. <opensource@google.com> + * google-glog: version 0.2 + * Add initial Windows VC++ support. + * Google testing/mocking frameworks integration. + * Link pthread library automatically. + * Flush logs in signal handlers. + * Add macros LOG_TO_STRING, LOG_AT_LEVEL, DVLOG, and LOG_TO_SINK_ONLY. + * Log microseconds. + * Add --log_backtrace_at option. + * Fix some minor bugs. + +2008-11-18 Google Inc. <opensource@google.com> + * google-glog: version 0.1.2 + * Add InstallFailureSignalHandler(). (satorux) + * Re-organize the way to produce stacktraces. + * Don't define unnecessary macro DISALLOW_EVIL_CONSTRUCTORS. + +2008-10-15 Google Inc. <opensource@google.com> + * google-glog: version 0.1.1 + * Support symbolize for MacOSX 10.5. + * BUG FIX: --vmodule didn't work with gflags. + * BUG FIX: symbolize_unittest failed with GCC 4.3. + * Several fixes on the document. + +2008-10-07 Google Inc. <opensource@google.com> + + * google-glog: initial release: + The glog package contains a library that implements application-level + logging. This library provides logging APIs based on C++-style + streams and various helper macros. diff --git a/third_party/glog/Dockerfile.ubuntu.template b/third_party/glog/Dockerfile.ubuntu.template new file mode 100644 index 000000000000..14f9b9589170 --- /dev/null +++ b/third_party/glog/Dockerfile.ubuntu.template @@ -0,0 +1,14 @@ +# Build Ubuntu image +FROM @BUILD_ARCH@/@BUILD_FLAVOR@:@BUILD_RELEASE@ + +# see https://docs.docker.com/engine/userguide/eng-image/dockerfile_best-practices/#/run +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + @BUILD_PACKAGES@ \ + build-essential \ + g++ + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY . /usr/src/app diff --git a/third_party/glog/INSTALL b/third_party/glog/INSTALL new file mode 100644 index 000000000000..e9530ac08c63 --- /dev/null +++ b/third_party/glog/INSTALL @@ -0,0 +1,313 @@ +Installation Instructions +************************* + +Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005, +2006, 2007 Free Software Foundation, Inc. + +This file is free documentation; the Free Software Foundation gives +unlimited permission to copy, distribute and modify it. + +Glog-Specific Install Notes +================================ + +*** NOTE FOR 64-BIT LINUX SYSTEMS + +The glibc built-in stack-unwinder on 64-bit systems has some problems +with the glog libraries. (In particular, if you are using +InstallFailureSignalHandler(), the signal may be raised in the middle +of malloc, holding some malloc-related locks when they invoke the +stack unwinder. The built-in stack unwinder may call malloc +recursively, which may require the thread to acquire a lock it already +holds: deadlock.) + +For that reason, if you use a 64-bit system and you need +InstallFailureSignalHandler(), we strongly recommend you install +libunwind before trying to configure or install google glog. +libunwind can be found at + + http://download.savannah.nongnu.org/releases/libunwind/libunwind-snap-070410.tar.gz + +Even if you already have libunwind installed, you will probably still +need to install from the snapshot to get the latest version. + +CAUTION: if you install libunwind from the URL above, be aware that +you may have trouble if you try to statically link your binary with +glog: that is, if you link with 'gcc -static -lgcc_eh ...'. This +is because both libunwind and libgcc implement the same C++ exception +handling APIs, but they implement them differently on some platforms. +This is not likely to be a problem on ia64, but may be on x86-64. + +Also, if you link binaries statically, make sure that you add +-Wl,--eh-frame-hdr to your linker options. This is required so that +libunwind can find the information generated by the compiler required +for stack unwinding. + +Using -static is rare, though, so unless you know this will affect you +it probably won't. + +If you cannot or do not wish to install libunwind, you can still try +to use two kinds of stack-unwinder: 1. glibc built-in stack-unwinder +and 2. frame pointer based stack-unwinder. + +1. As we already mentioned, glibc's unwinder has a deadlock issue. +However, if you don't use InstallFailureSignalHandler() or you don't +worry about the rare possibilities of deadlocks, you can use this +stack-unwinder. If you specify no options and libunwind isn't +detected on your system, the configure script chooses this unwinder by +default. + +2. The frame pointer based stack unwinder requires that your +application, the glog library, and system libraries like libc, all be +compiled with a frame pointer. This is *not* the default for x86-64. + +If you are on x86-64 system, know that you have a set of system +libraries with frame-pointers enabled, and compile all your +applications with -fno-omit-frame-pointer, then you can enable the +frame pointer based stack unwinder by passing the +--enable-frame-pointers flag to configure. + + +Basic Installation +================== + +Briefly, the shell commands `./configure; make; make install' should +configure, build, and install this package. The following +more-detailed instructions are generic; see the `README.md' file for +instructions specific to this package. + + The `configure' shell script attempts to guess correct values for +various system-dependent variables used during compilation. It uses +those values to create a `Makefile' in each directory of the package. +It may also create one or more `.h' files containing system-dependent +definitions. Finally, it creates a shell script `config.status' that +you can run in the future to recreate the current configuration, and a +file `config.log' containing compiler output (useful mainly for +debugging `configure'). + + It can also use an optional file (typically called `config.cache' +and enabled with `--cache-file=config.cache' or simply `-C') that saves +the results of its tests to speed up reconfiguring. Caching is +disabled by default to prevent problems with accidental use of stale +cache files. + + If you need to do unusual things to compile the package, please try +to figure out how `configure' could check whether to do them, and mail +diffs or instructions to the address given in the `README.md' so they can +be considered for the next release. If you are using the cache, and at +some point `config.cache' contains results you don't want to keep, you +may remove or edit it. + + The file `configure.ac' (or `configure.in') is used to create +`configure' by a program called `autoconf'. You need `configure.ac' if +you want to change it or regenerate `configure' using a newer version +of `autoconf'. + +The simplest way to compile this package is: + + 1. `cd' to the directory containing the package's source code and type + `./configure' to configure the package for your system. + + Running `configure' might take a while. While running, it prints + some messages telling which features it is checking for. + + 2. Type `make' to compile the package. + + 3. Optionally, type `make check' to run any self-tests that come with + the package. + + 4. Type `make install' to install the programs and any data files and + documentation. + + 5. You can remove the program binaries and object files from the + source code directory by typing `make clean'. To also remove the + files that `configure' created (so you can compile the package for + a different kind of computer), type `make distclean'. There is + also a `make maintainer-clean' target, but that is intended mainly + for the package's developers. If you use it, you may have to get + all sorts of other programs in order to regenerate files that came + with the distribution. + + 6. Often, you can also type `make uninstall' to remove the installed + files again. + + +Building glog - Using vcpkg +=========================== + +The url of vcpkg is: https://github.com/Microsoft/vcpkg +You can download and install glog using the vcpkg dependency manager: + + git clone https://github.com/Microsoft/vcpkg.git + cd vcpkg + ./bootstrap-vcpkg.sh + ./vcpkg integrate install + ./vcpkg install glog + +The glog port in vcpkg is kept up to date by Microsoft team members and community contributors. If the version is out of date, please create an issue or pull request on the vcpkg repository. + + +Compilers and Options +===================== + +Some systems require unusual options for compilation or linking that the +`configure' script does not know about. Run `./configure --help' for +details on some of the pertinent environment variables. + + You can give `configure' initial values for configuration parameters +by setting variables in the command line or in the environment. Here +is an example: + + ./configure CC=c99 CFLAGS=-g LIBS=-lposix + + *Note Defining Variables::, for more details. + +Compiling For Multiple Architectures +==================================== + +You can compile the package for more than one kind of computer at the +same time, by placing the object files for each architecture in their +own directory. To do this, you can use GNU `make'. `cd' to the +directory where you want the object files and executables to go and run +the `configure' script. `configure' automatically checks for the +source code in the directory that `configure' is in and in `..'. + + With a non-GNU `make', it is safer to compile the package for one +architecture at a time in the source code directory. After you have +installed the package for one architecture, use `make distclean' before +reconfiguring for another architecture. + +Installation Names +================== + +By default, `make install' installs the package's commands under +`/usr/local/bin', include files under `/usr/local/include', etc. You +can specify an installation prefix other than `/usr/local' by giving +`configure' the option `--prefix=PREFIX'. + + You can specify separate installation prefixes for +architecture-specific files and architecture-independent files. If you +pass the option `--exec-prefix=PREFIX' to `configure', the package uses +PREFIX as the prefix for installing programs and libraries. +Documentation and other data files still use the regular prefix. + + In addition, if you use an unusual directory layout you can give +options like `--bindir=DIR' to specify different values for particular +kinds of files. Run `configure --help' for a list of the directories +you can set and what kinds of files go in them. + + If the package supports it, you can cause programs to be installed +with an extra prefix or suffix on their names by giving `configure' the +option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'. + +Optional Features +================= + +Some packages pay attention to `--enable-FEATURE' options to +`configure', where FEATURE indicates an optional part of the package. +They may also pay attention to `--with-PACKAGE' options, where PACKAGE +is something like `gnu-as' or `x' (for the X Window System). The +`README.md' should mention any `--enable-' and `--with-' options that the +package recognizes. + + For packages that use the X Window System, `configure' can usually +find the X include and library files automatically, but if it doesn't, +you can use the `configure' options `--x-includes=DIR' and +`--x-libraries=DIR' to specify their locations. + +Specifying the System Type +========================== + +There may be some features `configure' cannot figure out automatically, +but needs to determine by the type of machine the package will run on. +Usually, assuming the package is built to be run on the _same_ +architectures, `configure' can figure that out, but if it prints a +message saying it cannot guess the machine type, give it the +`--build=TYPE' option. TYPE can either be a short name for the system +type, such as `sun4', or a canonical name which has the form: + + CPU-COMPANY-SYSTEM + +where SYSTEM can have one of these forms: + + OS KERNEL-OS + + See the file `config.sub' for the possible values of each field. If +`config.sub' isn't included in this package, then this package doesn't +need to know the machine type. + + If you are _building_ compiler tools for cross-compiling, you should +use the option `--target=TYPE' to select the type of system they will +produce code for. + + If you want to _use_ a cross compiler, that generates code for a +platform different from the build platform, you should specify the +"host" platform (i.e., that on which the generated programs will +eventually be run) with `--host=TYPE'. + +Sharing Defaults +================ + +If you want to set default values for `configure' scripts to share, you +can create a site shell script called `config.site' that gives default +values for variables like `CC', `cache_file', and `prefix'. +`configure' looks for `PREFIX/share/config.site' if it exists, then +`PREFIX/etc/config.site' if it exists. Or, you can set the +`CONFIG_SITE' environment variable to the location of the site script. +A warning: not all `configure' scripts look for a site script. + +Defining Variables +================== + +Variables not defined in a site shell script can be set in the +environment passed to `configure'. However, some packages may run +configure again during the build, and the customized values of these +variables may be lost. In order to avoid this problem, you should set +them in the `configure' command line, using `VAR=value'. For example: + + ./configure CC=/usr/local2/bin/gcc + +causes the specified `gcc' to be used as the C compiler (unless it is +overridden in the site shell script). + +Unfortunately, this technique does not work for `CONFIG_SHELL' due to +an Autoconf bug. Until the bug is fixed you can use this workaround: + + CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash + +`configure' Invocation +====================== + +`configure' recognizes the following options to control how it operates. + +`--help' +`-h' + Print a summary of the options to `configure', and exit. + +`--version' +`-V' + Print the version of Autoconf used to generate the `configure' + script, and exit. + +`--cache-file=FILE' + Enable the cache: use and save the results of the tests in FILE, + traditionally `config.cache'. FILE defaults to `/dev/null' to + disable caching. + +`--config-cache' +`-C' + Alias for `--cache-file=config.cache'. + +`--quiet' +`--silent' +`-q' + Do not print messages saying which checks are being made. To + suppress all normal output, redirect it to `/dev/null' (any error + messages will still be shown). + +`--srcdir=DIR' + Look for the package's source code in directory DIR. Usually + `configure' can determine that directory automatically. + +`configure' also accepts some other, not widely useful, options. Run +`configure --help' for more details. + diff --git a/third_party/glog/Makefile.am b/third_party/glog/Makefile.am new file mode 100644 index 000000000000..e792332a02ca --- /dev/null +++ b/third_party/glog/Makefile.am @@ -0,0 +1,259 @@ +## Process this file with automake to produce Makefile.in + +AUTOMAKE_OPTIONS = subdir-objects foreign + +# Make sure that when we re-make ./configure, we get the macros we need +ACLOCAL_AMFLAGS = -I m4 + +# This is so we can #include <glog/foo> +AM_CPPFLAGS = -I$(top_srcdir)/src + +# This is mostly based on configure options +AM_CXXFLAGS = + +# These are good warnings to turn on by default +if GCC + AM_CXXFLAGS += -Wall -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare +endif + +# These are x86-specific, having to do with frame-pointers +if X86_64 +if ENABLE_FRAME_POINTERS + AM_CXXFLAGS += -fno-omit-frame-pointer +else + # TODO(csilvers): check if -fomit-frame-pointer might be in $(CXXFLAGS), + # before setting this. + AM_CXXFLAGS += -DNO_FRAME_POINTER +endif +endif + +if DISABLE_RTTI + AM_CXXFLAGS += -fno-rtti +endif + +glogincludedir = $(includedir)/glog +## The .h files you want to install (that is, .h files that people +## who install this package can include in their own applications.) +## We have to include both the .h and .h.in forms. The latter we +## put in noinst_HEADERS. +gloginclude_HEADERS = src/glog/log_severity.h +nodist_gloginclude_HEADERS = src/glog/logging.h src/glog/raw_logging.h src/glog/vlog_is_on.h src/glog/stl_logging.h +noinst_HEADERS = src/glog/logging.h.in src/glog/raw_logging.h.in src/glog/vlog_is_on.h.in src/glog/stl_logging.h.in + +## This is for HTML and other documentation you want to install. +## Add your documentation files (in doc/) in addition to these +## top-level boilerplate files. Also add a TODO file if you have one. +dist_doc_DATA = AUTHORS COPYING ChangeLog INSTALL README.md README.windows \ + doc/designstyle.css doc/glog.html + +## The libraries (.so's) you want to install +lib_LTLIBRARIES = + +# The libraries libglog depends on. +COMMON_LIBS = $(PTHREAD_LIBS) $(GFLAGS_LIBS) $(UNWIND_LIBS) +# Compile switches for our unittest. +TEST_CFLAGS = $(GTEST_CFLAGS) $(GMOCK_CFLAGS) $(GFLAGS_CFLAGS) \ + $(MINGW_CFLAGS) $(AM_CXXFLAGS) +# Libraries for our unittest. +TEST_LIBS = $(GTEST_LIBS) $(GMOCK_LIBS) $(GFLAGS_LIBS) + +## unittests you want to run when people type 'make check'. +## TESTS is for binary unittests, check_SCRIPTS for script-based unittests. +## TESTS_ENVIRONMENT sets environment variables for when you run unittest, +## but it only seems to take effect for *binary* unittests (argh!) +TESTS = +# Set a small stack size so that (at least on Linux) PIEs are mapped at a lower +# address than DSOs. This is used by symbolize_pie_unittest to check that we can +# successfully symbolize PIEs loaded at low addresses. +TESTS_ENVIRONMENT = ulimit -s 8192; +check_SCRIPTS = +# Every time you add a unittest to check_SCRIPTS, add it here too +noinst_SCRIPTS = +# Binaries used for script-based unittests. +TEST_BINARIES = + +TESTS += logging_unittest +logging_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/logging_unittest.cc \ + src/config_for_unittests.h \ + src/mock-log.h +nodist_logging_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +logging_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +logging_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +logging_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +check_SCRIPTS += logging_striplog_test_sh +noinst_SCRIPTS += src/logging_striplog_test.sh +logging_striplog_test_sh: logging_striptest0 logging_striptest2 logging_striptest10 + $(top_srcdir)/src/logging_striplog_test.sh + +check_SCRIPTS += demangle_unittest_sh +noinst_SCRIPTS += src/demangle_unittest.sh +demangle_unittest_sh: demangle_unittest + $(builddir)/demangle_unittest # force to create lt-demangle_unittest + $(top_srcdir)/src/demangle_unittest.sh + +check_SCRIPTS += signalhandler_unittest_sh +noinst_SCRIPTS += src/signalhandler_unittest.sh +signalhandler_unittest_sh: signalhandler_unittest + $(builddir)/signalhandler_unittest # force to create lt-signalhandler_unittest + $(top_srcdir)/src/signalhandler_unittest.sh + +TEST_BINARIES += logging_striptest0 +logging_striptest0_SOURCES = $(gloginclude_HEADERS) \ + src/logging_striptest_main.cc +nodist_logging_striptest0_SOURCES = $(nodist_gloginclude_HEADERS) +logging_striptest0_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +logging_striptest0_LDFLAGS = $(PTHREAD_CFLAGS) +logging_striptest0_LDADD = libglog.la $(COMMON_LIBS) + +TEST_BINARIES += logging_striptest2 +logging_striptest2_SOURCES = $(gloginclude_HEADERS) \ + src/logging_striptest2.cc +nodist_logging_striptest2_SOURCES = $(nodist_gloginclude_HEADERS) +logging_striptest2_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +logging_striptest2_LDFLAGS = $(PTHREAD_CFLAGS) +logging_striptest2_LDADD = libglog.la $(COMMON_LIBS) + +TEST_BINARIES += logging_striptest10 +logging_striptest10_SOURCES = $(gloginclude_HEADERS) \ + src/logging_striptest10.cc +nodist_logging_striptest10_SOURCES = $(nodist_gloginclude_HEADERS) +logging_striptest10_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +logging_striptest10_LDFLAGS = $(PTHREAD_CFLAGS) +logging_striptest10_LDADD = libglog.la $(COMMON_LIBS) + +TESTS += demangle_unittest +demangle_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/demangle_unittest.cc +nodist_demangle_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +demangle_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +demangle_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +demangle_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +TESTS += stacktrace_unittest +stacktrace_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/stacktrace_unittest.cc +nodist_stacktrace_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +stacktrace_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +stacktrace_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +stacktrace_unittest_LDADD = libglog.la $(COMMON_LIBS) + +TESTS += symbolize_unittest +symbolize_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/symbolize_unittest.cc +nodist_symbolize_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +symbolize_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +symbolize_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +symbolize_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +TESTS += symbolize_pie_unittest +symbolize_pie_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/symbolize_unittest.cc +nodist_symbolize_pie_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +symbolize_pie_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) -fPIE +symbolize_pie_unittest_LDFLAGS = $(PTHREAD_CFLAGS) -pie +symbolize_pie_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +TESTS += stl_logging_unittest +stl_logging_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/stl_logging_unittest.cc +nodist_stl_logging_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +stl_logging_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +stl_logging_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +stl_logging_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +TEST_BINARIES += signalhandler_unittest +signalhandler_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/signalhandler_unittest.cc +nodist_signalhandler_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +signalhandler_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +signalhandler_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +signalhandler_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +TESTS += utilities_unittest +utilities_unittest_SOURCES = $(gloginclude_HEADERS) \ + src/utilities_unittest.cc +nodist_utilities_unittest_SOURCES = $(nodist_gloginclude_HEADERS) +utilities_unittest_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +utilities_unittest_LDFLAGS = $(PTHREAD_CFLAGS) +utilities_unittest_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) + +if HAVE_GMOCK +TESTS += mock_log_test +mock_log_test_SOURCES = $(gloginclude_HEADERS) \ + src/mock-log_test.cc +nodist_mock_log_test_SOURCES = $(nodist_gloginclude_HEADERS) +mock_log_test_CXXFLAGS = $(PTHREAD_CFLAGS) $(TEST_CFLAGS) +mock_log_test_LDFLAGS = $(PTHREAD_CFLAGS) +mock_log_test_LDADD = libglog.la $(COMMON_LIBS) $(TEST_LIBS) +endif + +## vvvv RULES TO MAKE THE LIBRARIES, BINARIES, AND UNITTESTS + +lib_LTLIBRARIES += libglog.la +libglog_la_SOURCES = $(gloginclude_HEADERS) \ + src/logging.cc src/raw_logging.cc src/vlog_is_on.cc \ + src/utilities.cc src/utilities.h \ + src/demangle.cc src/demangle.h \ + src/stacktrace.h \ + src/stacktrace_generic-inl.h \ + src/stacktrace_libunwind-inl.h \ + src/stacktrace_powerpc-inl.h \ + src/stacktrace_x86-inl.h \ + src/stacktrace_x86_64-inl.h \ + src/symbolize.cc src/symbolize.h \ + src/signalhandler.cc \ + src/base/mutex.h src/base/googleinit.h \ + src/base/commandlineflags.h src/googletest.h +nodist_libglog_la_SOURCES = $(nodist_gloginclude_HEADERS) + +libglog_la_CXXFLAGS = $(PTHREAD_CFLAGS) $(GFLAGS_CFLAGS) $(MINGW_CFLAGS) \ + $(AM_CXXFLAGS) -DNDEBUG +libglog_la_LDFLAGS = $(PTHREAD_CFLAGS) $(GFLAGS_LDFLAGS) +libglog_la_LIBADD = $(COMMON_LIBS) + +## The location of the windows project file for each binary we make +WINDOWS_PROJECTS = google-glog.sln +WINDOWS_PROJECTS += vsprojects/libglog/libglog.vcproj +WINDOWS_PROJECTS += vsprojects/logging_unittest/logging_unittest.vcproj +WINDOWS_PROJECTS += vsprojects/libglog_static/libglog_static.vcproj +WINDOWS_PROJECTS += vsprojects/logging_unittest_static/logging_unittest_static.vcproj + +## ^^^^ END OF RULES TO MAKE THE LIBRARIES, BINARIES, AND UNITTESTS + + +## This should always include $(TESTS), but may also include other +## binaries that you compile but don't want automatically installed. +noinst_PROGRAMS = $(TESTS) $(TEST_BINARIES) + +rpm: dist-gzip packages/rpm.sh packages/rpm/rpm.spec + @cd packages && ./rpm.sh ${PACKAGE} ${VERSION} + +deb: dist-gzip packages/deb.sh packages/deb/* + @cd packages && ./deb.sh ${PACKAGE} ${VERSION} + +# Windows wants write permission to .vcproj files and maybe even sln files. +dist-hook: + test -e "$(distdir)/vsprojects" \ + && chmod -R u+w $(distdir)/*.sln $(distdir)/vsprojects/ + +libtool: $(LIBTOOL_DEPS) + $(SHELL) ./config.status --recheck + +EXTRA_DIST = packages/rpm.sh packages/rpm/rpm.spec \ + packages/deb.sh packages/deb/* \ + $(SCRIPTS) src/logging_unittest.err src/demangle_unittest.txt \ + src/windows/config.h src/windows/port.h src/windows/port.cc \ + src/windows/preprocess.sh \ + src/windows/glog/log_severity.h src/windows/glog/logging.h \ + src/windows/glog/raw_logging.h src/windows/glog/stl_logging.h \ + src/windows/glog/vlog_is_on.h \ + $(WINDOWS_PROJECTS) + +CLEANFILES = core demangle.dm demangle.nm signalhandler.out* \ + signalhandler_unittest.*.log.INFO.* + +# Add pkgconfig file +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = libglog.pc diff --git a/third_party/glog/README.md b/third_party/glog/README.md new file mode 100644 index 000000000000..965766c7cafc --- /dev/null +++ b/third_party/glog/README.md @@ -0,0 +1,10 @@ +[](https://travis-ci.org/google/glog/builds) +[](https://ci.appveyor.com/project/google-admin/glog/history) + +This repository contains a C++ implementation of the Google logging +module. Documentation for the implementation is in doc/. + +See INSTALL for (generic) installation instructions for C++: basically +```sh +./autogen.sh && ./configure && make && make install +``` diff --git a/third_party/glog/README.windows b/third_party/glog/README.windows new file mode 100644 index 000000000000..ea6ccc20bde6 --- /dev/null +++ b/third_party/glog/README.windows @@ -0,0 +1,17 @@ +This project has been ported to Windows, including stack tracing, signal +handling, and unit tests. + +A Visual Studio solution file is explicitly not provided because it is not +maintainable. Instead, a CMake build system exists to generate the correct +solution for your version of Visual Studio. + +In short, + (1) Install CMake from: https://cmake.org/download/ + (2) With CMake on your PATH, run `cmake .` to generate the build files + (3) Either use `cmake --build`, or open the generated solution + +CMake provides different generators, and by default will pick the most relevant +one to your environment. If you need a specific version of Visual Studio, use +`cmake . -G <generator-name>`, and see `cmake --help` for the available +generators. Also see `-T <toolset-name>`, which can used to request the native +x64 toolchain with `-T host=x64`. \ No newline at end of file diff --git a/third_party/glog/WORKSPACE b/third_party/glog/WORKSPACE new file mode 100644 index 000000000000..b5760d04caaa --- /dev/null +++ b/third_party/glog/WORKSPACE @@ -0,0 +1,10 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "com_github_gflags_gflags", + strip_prefix = "gflags-2.2.2", + urls = [ + "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz", + "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz", + ], +) diff --git a/third_party/glog/appveyor.yml b/third_party/glog/appveyor.yml new file mode 100644 index 000000000000..ecc42c4f291b --- /dev/null +++ b/third_party/glog/appveyor.yml @@ -0,0 +1,71 @@ +# global environment variables +environment: + global: + # path to source directory of project to be built + PROJECT_DIR: . + # output test results for failing tests + CTEST_OUTPUT_ON_FAILURE: 1 + + # test matrix + matrix: + + - TOOLCHAIN: "vs-14-2015-sdk-8-1" + GENERATOR: "Visual Studio 14 2015 Win64" + TEST_TARGET: RUN_TESTS + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + + - TOOLCHAIN: "vs-14-2015-win64" + GENERATOR: "Visual Studio 14 2015 Win64" + TEST_TARGET: RUN_TESTS + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + + - TOOLCHAIN: "vs-15-2017-win64" + GENERATOR: "Visual Studio 15 2017 Win64" + TEST_TARGET: RUN_TESTS + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + + - TOOLCHAIN: "vs-15-2017-win64-cxx17" + GENERATOR: "Visual Studio 15 2017 Win64" + TEST_TARGET: RUN_TESTS + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + + - TOOLCHAIN: "mingw-cxx11" + GENERATOR: "MinGW Makefiles" + MINGW_PATH: "C:\\mingw-w64\\x86_64-7.2.0-posix-seh-rt_v5-rev1\\mingw64\\bin" + TEST_TARGET: test + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + + - TOOLCHAIN: "mingw-gnuxx11" + GENERATOR: "MinGW Makefiles" + MINGW_PATH: "C:\\mingw-w64\\x86_64-7.2.0-posix-seh-rt_v5-rev1\\mingw64\\bin" + TEST_TARGET: test + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + + - TOOLCHAIN: "mingw-cxx17" + GENERATOR: "MinGW Makefiles" + MINGW_PATH: "C:\\mingw-w64\\x86_64-7.2.0-posix-seh-rt_v5-rev1\\mingw64\\bin" + TEST_TARGET: test + APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017 + +install: + # Remove entry with sh.exe from PATH to fix error with MinGW toolchain + # (For MinGW make to work correctly sh.exe must NOT be in your path) + # * http://stackoverflow.com/a/3870338/2288008 + - cmd: set PATH=%PATH:C:\Program Files\Git\usr\bin;=% + + # set MINGW path + - cmd: IF DEFINED MINGW_PATH set PATH=%MINGW_PATH%;%PATH% + + # Visual Studio 15 2017: Mimic behavior of older versions + - cmd: set VS150COMNTOOLS=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\Tools + +build_script: + - cmd: cmake -H. -B_build_%TOOLCHAIN%_Debug -G "%GENERATOR%" -DCMAKE_TOOLCHAIN_FILE="%cd%\toolchains\%TOOLCHAIN%.cmake" + - cmd: cmake --build _build_%TOOLCHAIN%_Debug --config Debug + #- cmd: cmake -H. -B_build_%TOOLCHAIN%_Release -G "%GENERATOR%" -DCMAKE_TOOLCHAIN_FILE="%cd%\toolchains\%TOOLCHAIN%.cmake" + #- cmd: cmake --build _build_%TOOLCHAIN%_Release --config Release + # add git back to PATH for `diff` command in case of error + - cmd: set PATH=C:\Program Files\Git\usr\bin;%PATH% + - cmd: IF DEFINED TEST_TARGET cmake --build _build_%TOOLCHAIN%_Debug --target %TEST_TARGET% + #- cmd: IF DEFINED TEST_TARGET cmake --build _build_%TOOLCHAIN%_Release --target %TEST_TARGET% + diff --git a/third_party/glog/autogen.sh b/third_party/glog/autogen.sh new file mode 100755 index 000000000000..08b659040959 --- /dev/null +++ b/third_party/glog/autogen.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +set -eu + +autoreconf -i diff --git a/third_party/glog/bazel/example/BUILD b/third_party/glog/bazel/example/BUILD new file mode 100644 index 000000000000..6b10e2b2fb3f --- /dev/null +++ b/third_party/glog/bazel/example/BUILD @@ -0,0 +1,8 @@ +cc_test( + name = "main", + size = "small", + srcs = ["main.cc"], + deps = [ + "//:glog", + ], +) diff --git a/third_party/glog/bazel/example/main.cc b/third_party/glog/bazel/example/main.cc new file mode 100644 index 000000000000..fef01dc15752 --- /dev/null +++ b/third_party/glog/bazel/example/main.cc @@ -0,0 +1,22 @@ +#include <gflags/gflags.h> +#include <glog/logging.h> +#include <glog/stl_logging.h> + +int main(int argc, char* argv[]) { + // Initialize Google's logging library. + google::InitGoogleLogging(argv[0]); + + // Optional: parse command line flags + gflags::ParseCommandLineFlags(&argc, &argv, true); + + LOG(INFO) << "Hello, world!"; + + // glog/stl_logging.h allows logging STL containers. + std::vector<int> x; + x.push_back(1); + x.push_back(2); + x.push_back(3); + LOG(INFO) << "ABC, it's easy as " << x; + + return 0; +} diff --git a/third_party/glog/bazel/glog.bzl b/third_party/glog/bazel/glog.bzl new file mode 100644 index 000000000000..a46e2d148ba8 --- /dev/null +++ b/third_party/glog/bazel/glog.bzl @@ -0,0 +1,219 @@ +# Implement a macro glog_library() that the BUILD file can load. + +# By default, glog is built with gflags support. You can change this behavior +# by using glog_library(with_gflags=0) +# +# This file is inspired by the following sample BUILD files: +# https://github.com/google/glog/issues/61 +# https://github.com/google/glog/files/393474/BUILD.txt +# +# Known issue: the namespace parameter is not supported on Win32. + +def glog_library(namespace = "google", with_gflags = 1, **kwargs): + if native.repository_name() != "@": + repo_name = native.repository_name().lstrip("@") + gendir = "$(GENDIR)/external/" + repo_name + src_windows = "external/%s/src/windows" % repo_name + else: + gendir = "$(GENDIR)" + src_windows = "src/windows" + + # Config setting for WebAssembly target. + native.config_setting( + name = "wasm", + values = {"cpu": "wasm"}, + ) + + common_copts = [ + "-DGLOG_BAZEL_BUILD", + "-DHAVE_STDINT_H", + "-DHAVE_STRING_H", + "-DHAVE_UNWIND_H", + ] + (["-DHAVE_LIB_GFLAGS"] if with_gflags else []) + + wasm_copts = [ + # Disable warnings that exists in glog. + "-Wno-sign-compare", + "-Wno-unused-function", + "-Wno-unused-local-typedefs", + "-Wno-unused-variable", + # Inject a C++ namespace. + "-DGOOGLE_NAMESPACE='%s'" % namespace, + # Allows src/base/mutex.h to include pthread.h. + "-DHAVE_PTHREAD", + # Allows src/logging.cc to determine the host name. + "-DHAVE_SYS_UTSNAME_H", + # For src/utilities.cc. + "-DHAVE_SYS_TIME_H", + # Enable dumping stacktrace upon sigaction. + "-DHAVE_SIGACTION", + # For logging.cc. + "-DHAVE_PREAD", + "-DHAVE___ATTRIBUTE__", + "-I%s/glog_internal" % gendir, + ] + + linux_or_darwin_copts = wasm_copts + [ + # For src/utilities.cc. + "-DHAVE_SYS_SYSCALL_H", + ] + + freebsd_only_copts = [ + # Enable declaration of _Unwind_Backtrace + "-D_GNU_SOURCE", + ] + + darwin_only_copts = [ + # For stacktrace. + "-DHAVE_DLADDR", + ] + + windows_only_copts = [ + "-DHAVE_SNPRINTF", + "-I" + src_windows, + ] + + windows_only_srcs = [ + "src/glog/log_severity.h", + "src/windows/config.h", + "src/windows/port.cc", + "src/windows/port.h", + ] + + gflags_deps = ["@com_github_gflags_gflags//:gflags"] if with_gflags else [] + + native.cc_library( + name = "glog", + visibility = ["//visibility:public"], + srcs = [ + "src/base/commandlineflags.h", + "src/base/googleinit.h", + "src/base/mutex.h", + "src/demangle.cc", + "src/demangle.h", + "src/logging.cc", + "src/raw_logging.cc", + "src/signalhandler.cc", + "src/stacktrace.h", + "src/stacktrace_generic-inl.h", + "src/stacktrace_libunwind-inl.h", + "src/stacktrace_powerpc-inl.h", + "src/stacktrace_windows-inl.h", + "src/stacktrace_x86-inl.h", + "src/stacktrace_x86_64-inl.h", + "src/symbolize.cc", + "src/symbolize.h", + "src/utilities.cc", + "src/utilities.h", + "src/vlog_is_on.cc", + ] + select({ + "@bazel_tools//src/conditions:windows": windows_only_srcs, + "//conditions:default": [":config_h"], + }), + copts = + select({ + "@bazel_tools//src/conditions:windows": common_copts + windows_only_copts, + "@bazel_tools//src/conditions:darwin": common_copts + linux_or_darwin_copts + darwin_only_copts, + "@bazel_tools//src/conditions:freebsd": common_copts + linux_or_darwin_copts + freebsd_only_copts, + ":wasm": common_copts + wasm_copts, + "//conditions:default": common_copts + linux_or_darwin_copts, + }), + deps = [ + ":glog_headers", + ] + gflags_deps, + **kwargs + ) + + # glog headers vary depending on the os. + native.cc_library( + name = "glog_headers", + deps = select({ + "@bazel_tools//src/conditions:windows": [":windows_glog_headers"], + "//conditions:default": [":default_glog_headers"], + }), + ) + + native.cc_library( + name = "windows_glog_headers", + hdrs = native.glob(["src/windows/glog/*.h"]), + strip_include_prefix = "src/windows", + # We need to override the default GOOGLE_GLOG_DLL_DECL from + # src/windows/glog/*.h to match src/windows/config.h. + defines = ["GOOGLE_GLOG_DLL_DECL=__declspec(dllexport)"], + deps = [":strip_include_prefix_hack"], + ) + + # Workaround https://github.com/bazelbuild/bazel/issues/6337 by declaring + # the dependencies without strip_include_prefix. + native.cc_library( + name = "strip_include_prefix_hack", + hdrs = native.glob(["src/windows/*.h"]), + ) + + native.cc_library( + name = "default_glog_headers", + strip_include_prefix = "src", + hdrs = [ + "src/glog/log_severity.h", + ":logging_h", + ":raw_logging_h", + ":stl_logging_h", + ":vlog_is_on_h", + ], + ) + native.genrule( + name = "config_h", + srcs = [ + "src/config.h.cmake.in", + ], + outs = [ + "glog_internal/config.h", + ], + cmd = "awk '{ gsub(/^#cmakedefine/, \"//cmakedefine\"); print; }' $< > $@", + ) + + native.genrule( + name = "gen_sh", + outs = [ + "gen.sh", + ], + cmd = r'''\ +#!/bin/sh +cat > $@ <<"EOF" +sed -e 's/@ac_cv_cxx_using_operator@/1/g' \ + -e 's/@ac_cv_have_unistd_h@/1/g' \ + -e 's/@ac_cv_have_stdint_h@/1/g' \ + -e 's/@ac_cv_have_systypes_h@/1/g' \ + -e 's/@ac_cv_have_libgflags@/{}/g' \ + -e 's/@ac_cv_have_uint16_t@/1/g' \ + -e 's/@ac_cv_have___builtin_expect@/1/g' \ + -e 's/@ac_cv_have_.*@/0/g' \ + -e 's/@ac_google_start_namespace@/namespace google {{/g' \ + -e 's/@ac_google_end_namespace@/}}/g' \ + -e 's/@ac_google_namespace@/google/g' \ + -e 's/@ac_cv___attribute___noinline@/__attribute__((noinline))/g' \ + -e 's/@ac_cv___attribute___noreturn@/__attribute__((noreturn))/g' \ + -e 's/@ac_cv___attribute___printf_4_5@/__attribute__((__format__ (__printf__, 4, 5)))/g' +EOF +'''.format(int(with_gflags)), + ) + + [ + native.genrule( + name = "%s_h" % f, + srcs = [ + "src/glog/%s.h.in" % f, + ], + outs = [ + "src/glog/%s.h" % f, + ], + cmd = "$(location :gen_sh) < $< > $@", + tools = [":gen_sh"], + ) + for f in [ + "vlog_is_on", + "stl_logging", + "raw_logging", + "logging", + ] + ] diff --git a/third_party/glog/cmake/DetermineGflagsNamespace.cmake b/third_party/glog/cmake/DetermineGflagsNamespace.cmake new file mode 100644 index 000000000000..3dde42b4b332 --- /dev/null +++ b/third_party/glog/cmake/DetermineGflagsNamespace.cmake @@ -0,0 +1,69 @@ +macro(determine_gflags_namespace VARIABLE) + if (NOT DEFINED "${VARIABLE}") + if (CMAKE_REQUIRED_INCLUDES) + set (CHECK_INCLUDE_FILE_CXX_INCLUDE_DIRS "-DINCLUDE_DIRECTORIES=${CMAKE_REQUIRED_INCLUDES}") + else () + set (CHECK_INCLUDE_FILE_CXX_INCLUDE_DIRS) + endif () + + set(MACRO_CHECK_INCLUDE_FILE_FLAGS ${CMAKE_REQUIRED_FLAGS}) + + set(_NAMESPACES gflags google) + set(_check_code +" +#include <gflags/gflags.h> + +int main(int argc, char**argv) +{ + GLOG_GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); +} +") + if (NOT CMAKE_REQUIRED_QUIET) + message (STATUS "Looking for gflags namespace") + endif () + if (${ARGC} EQUAL 3) + set (CMAKE_CXX_FLAGS_SAVE ${CMAKE_CXX_FLAGS}) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${ARGV2}") + endif () + + set (_check_file + ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/DetermineGflagsNamespace.cxx) + + foreach (_namespace ${_NAMESPACES}) + file (WRITE "${_check_file}" "${_check_code}") + try_compile (${VARIABLE} + "${CMAKE_BINARY_DIR}" "${_check_file}" + COMPILE_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}" -DGLOG_GFLAGS_NAMESPACE=${_namespace} + LINK_LIBRARIES gflags + CMAKE_FLAGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + OUTPUT_VARIABLE OUTPUT) + + if (${VARIABLE}) + set (${VARIABLE} ${_namespace} CACHE INTERNAL "gflags namespace" FORCE) + break () + else () + file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Determining the gflags namespace ${_namespace} failed with the following output:\n" + "${OUTPUT}\n\n") + endif () + endforeach (_namespace) + + if (${ARGC} EQUAL 3) + set (CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS_SAVE}) + endif () + + if (${VARIABLE}) + if (NOT CMAKE_REQUIRED_QUIET) + message (STATUS "Looking for gflags namespace - ${${VARIABLE}}") + endif () + file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log + "Determining the gflags namespace passed with the following output:\n" + "${OUTPUT}\n\n") + else () + if (NOT CMAKE_REQUIRED_QUIET) + message (STATUS "Looking for gflags namespace - failed") + endif () + set (${VARIABLE} ${_namespace} CACHE INTERNAL "gflags namespace") + endif () + endif () +endmacro () diff --git a/third_party/glog/cmake/FindUnwind.cmake b/third_party/glog/cmake/FindUnwind.cmake new file mode 100644 index 000000000000..8941bb0dacca --- /dev/null +++ b/third_party/glog/cmake/FindUnwind.cmake @@ -0,0 +1,78 @@ +# - Try to find libunwind +# Once done this will define +# +# Unwind_FOUND - system has libunwind +# unwind::unwind - cmake target for libunwind + +include (FindPackageHandleStandardArgs) + +find_path (Unwind_INCLUDE_DIR NAMES unwind.h libunwind.h DOC "unwind include directory") +find_library (Unwind_LIBRARY NAMES unwind DOC "unwind library") + +if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + set (Unwind_ARCH "arm") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^aarch64") + set (Unwind_ARCH "aarch64") +elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR + CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64" OR + CMAKE_SYSTEM_PROCESSOR STREQUAL "corei7-64") + set (Unwind_ARCH "x86_64") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$") + set (Unwind_ARCH "x86") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^ppc64") + set (Unwind_ARCH "ppc64") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^ppc") + set (Unwind_ARCH "ppc32") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^mips") + set (Unwind_ARCH "mips") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^hppa") + set (Unwind_ARCH "hppa") +elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^ia64") + set (Unwind_ARCH "ia64") +endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm") + +find_library (Unwind_PLATFORM_LIBRARY NAMES "unwind-${Unwind_ARCH}" + DOC "unwind library platform") + +mark_as_advanced (Unwind_INCLUDE_DIR Unwind_LIBRARY Unwind_PLATFORM_LIBRARY) + +# Extract version information +if (Unwind_LIBRARY) + set (_Unwind_VERSION_HEADER ${Unwind_INCLUDE_DIR}/libunwind-common.h) + + if (EXISTS ${_Unwind_VERSION_HEADER}) + FILE (READ ${_Unwind_VERSION_HEADER} _Unwind_VERSION_CONTENTS) + + string (REGEX REPLACE ".*#define UNW_VERSION_MAJOR[ \t]+([0-9]+).*" "\\1" + Unwind_VERSION_MAJOR "${_Unwind_VERSION_CONTENTS}") + string (REGEX REPLACE ".*#define UNW_VERSION_MINOR[ \t]+([0-9]+).*" "\\1" + Unwind_VERSION_MINOR "${_Unwind_VERSION_CONTENTS}") + string (REGEX REPLACE ".*#define UNW_VERSION_EXTRA[ \t]+([0-9]+).*" "\\1" + Unwind_VERSION_PATCH "${_Unwind_VERSION_CONTENTS}") + + set (Unwind_VERSION + ${Unwind_VERSION_MAJOR}.${Unwind_VERSION_MINOR}.${Unwind_VERSION_PATCH}) + set (Unwind_VERSION_COMPONENTS 3) + endif (EXISTS ${_Unwind_VERSION_HEADER}) +endif (Unwind_LIBRARY) + +# handle the QUIETLY and REQUIRED arguments and set Unwind_FOUND to TRUE +# if all listed variables are TRUE +find_package_handle_standard_args (Unwind REQUIRED_VARS Unwind_INCLUDE_DIR + Unwind_LIBRARY Unwind_PLATFORM_LIBRARY VERSION_VAR Unwind_VERSION) + +if (Unwind_FOUND) + if (NOT TARGET unwind::unwind) + add_library (unwind::unwind INTERFACE IMPORTED) + + set_property (TARGET unwind::unwind PROPERTY + INTERFACE_INCLUDE_DIRECTORIES ${Unwind_INCLUDE_DIR} + ) + set_property (TARGET unwind::unwind PROPERTY + INTERFACE_LINK_LIBRARIES ${Unwind_LIBRARY} ${Unwind_PLATFORM_LIBRARY} + ) + set_property (TARGET unwind::unwind PROPERTY + IMPORTED_CONFIGURATIONS RELEASE + ) + endif (NOT TARGET unwind::unwind) +endif (Unwind_FOUND) diff --git a/third_party/glog/cmake/GetCacheVariables.cmake b/third_party/glog/cmake/GetCacheVariables.cmake new file mode 100644 index 000000000000..20485a8c8f8f --- /dev/null +++ b/third_party/glog/cmake/GetCacheVariables.cmake @@ -0,0 +1,63 @@ +cmake_policy (PUSH) +cmake_policy (VERSION 3.3) + +include (CMakeParseArguments) + +function (get_cache_variables _CACHEVARS) + set (_SINGLE) + set (_MULTI EXCLUDE) + set (_OPTIONS) + + cmake_parse_arguments (_ARGS "${_OPTIONS}" "${_SINGLE}" "${_MULTI}" ${ARGS} ${ARGN}) + + get_cmake_property (_VARIABLES VARIABLES) + + set (CACHEVARS) + + foreach (_VAR ${_VARIABLES}) + if (DEFINED _ARGS_EXCLUDE) + if ("${_VAR}" IN_LIST _ARGS_EXCLUDE) + continue () + endif ("${_VAR}" IN_LIST _ARGS_EXCLUDE) + endif (DEFINED _ARGS_EXCLUDE) + + get_property (_CACHEVARTYPE CACHE ${_VAR} PROPERTY TYPE) + + if ("${_CACHEVARTYPE}" STREQUAL INTERNAL OR + "${_CACHEVARTYPE}" STREQUAL STATIC OR + "${_CACHEVARTYPE}" STREQUAL UNINITIALIZED) + continue () + endif ("${_CACHEVARTYPE}" STREQUAL INTERNAL OR + "${_CACHEVARTYPE}" STREQUAL STATIC OR + "${_CACHEVARTYPE}" STREQUAL UNINITIALIZED) + + get_property (_CACHEVARVAL CACHE ${_VAR} PROPERTY VALUE) + + if ("${_CACHEVARVAL}" STREQUAL "") + continue () + endif ("${_CACHEVARVAL}" STREQUAL "") + + get_property (_CACHEVARDOC CACHE ${_VAR} PROPERTY HELPSTRING) + + # Escape " in values + string (REPLACE "\"" "\\\"" _CACHEVARVAL "${_CACHEVARVAL}") + # Escape " in help strings + string (REPLACE "\"" "\\\"" _CACHEVARDOC "${_CACHEVARDOC}") + # Escape ; in values + string (REPLACE ";" "\\\;" _CACHEVARVAL "${_CACHEVARVAL}") + # Escape backslash in values + string (REGEX REPLACE "\\\\([^\"])" "\\\\\\1" _CACHEVARVAL "${_CACHEVARVAL}") + + if (NOT "${_CACHEVARTYPE}" STREQUAL BOOL) + set (_CACHEVARVAL "\"${_CACHEVARVAL}\"") + endif (NOT "${_CACHEVARTYPE}" STREQUAL BOOL) + + if (NOT "${_CACHEVARTYPE}" STREQUAL "" AND NOT "${_CACHEVARVAL}" STREQUAL "") + set (CACHEVARS "${CACHEVARS}set (${_VAR} ${_CACHEVARVAL} CACHE ${_CACHEVARTYPE} \"${_CACHEVARDOC}\")\n") + endif (NOT "${_CACHEVARTYPE}" STREQUAL "" AND NOT "${_CACHEVARVAL}" STREQUAL "") + endforeach (_VAR) + + set (${_CACHEVARS} ${CACHEVARS} PARENT_SCOPE) +endfunction (get_cache_variables) + +cmake_policy (POP) diff --git a/third_party/glog/cmake/INSTALL.md b/third_party/glog/cmake/INSTALL.md new file mode 100644 index 000000000000..1377d652ed09 --- /dev/null +++ b/third_party/glog/cmake/INSTALL.md @@ -0,0 +1,81 @@ +# Glog - CMake Support + +Glog comes with a CMake build script ([CMakeLists.txt](../CMakeLists.txt)) that can be used on a wide range of platforms. +If you don't have CMake installed already, you can download it for free from <http://www.cmake.org/>. + +CMake works by generating native makefiles or build projects that can be used in the compiler environment of your choice. +You can either build Glog with CMake as a standalone project or it can be incorporated into an existing CMake build for another project. + +## Table of Contents + +- [Building Glog with CMake](#building-glog-with-cmake) +- [Consuming Glog in a CMake Project](#consuming-glog-in-a-cmake-project) +- [Incorporating Glog into a CMake Project](#incorporating-glog-into-a-cmake-project) + +## Building Glog with CMake + +When building Glog as a standalone project, on Unix-like systems with GNU Make as build tool, the typical workflow is: + +1. Get the source code and change to it. +e.g. cloning with git: +```bash +git clone git@github.com:google/glog.git +cd glog +``` + +2. Run CMake to configure the build tree. +```bash +cmake -H. -Bbuild -G "Unix Makefiles" +``` +note: To get the list of available generators (e.g. Visual Studio), use `-G ""` + +3. Afterwards, generated files can be used to compile the project. +```bash +cmake --build build +``` + +4. Test the build software (optional). +```bash +cmake --build build --target test +``` + +5. Install the built files (optional). +```bash +cmake --build build --target install +``` + +## Consuming Glog in a CMake Project + +If you have Glog installed in your system, you can use the CMake command +`find_package()` to include it in your CMake Project. + +```cmake +cmake_minimum_required(VERSION 3.0.2) +project(myproj VERSION 1.0) + +find_package(glog 0.4.0 REQUIRED) + +add_executable(myapp main.cpp) +target_link_libraries(myapp glog::glog) +``` + +Compile definitions and options will be added automatically to your target as +needed. + +## Incorporating Glog into a CMake Project + +You can also use the CMake command `add_subdirectory()` to include Glog directly from a subdirectory of your project. +The **glog::glog** target is in this case an ALIAS library target for the **glog** library target. + +```cmake +cmake_minimum_required(VERSION 3.0.2) +project(myproj VERSION 1.0) + +add_subdirectory(glog) + +add_executable(myapp main.cpp) +target_link_libraries(myapp glog::glog) +``` + +Again, compile definitions and options will be added automatically to your target as +needed. diff --git a/third_party/glog/cmake/TestInitPackageConfig.cmake b/third_party/glog/cmake/TestInitPackageConfig.cmake new file mode 100644 index 000000000000..6cb11ccf2977 --- /dev/null +++ b/third_party/glog/cmake/TestInitPackageConfig.cmake @@ -0,0 +1,12 @@ +# Create the build directory +execute_process ( + COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_BINARY_DIR} + RESULT_VARIABLE _DIRECTORY_CREATED_SUCCEEDED +) + +if (NOT _DIRECTORY_CREATED_SUCCEEDED EQUAL 0) + message (FATAL_ERROR "Failed to create build directory") +endif (NOT _DIRECTORY_CREATED_SUCCEEDED EQUAL 0) + +file (WRITE ${INITIAL_CACHE} "${CACHEVARS}") + diff --git a/third_party/glog/cmake/TestPackageConfig.cmake b/third_party/glog/cmake/TestPackageConfig.cmake new file mode 100644 index 000000000000..191c3cac2351 --- /dev/null +++ b/third_party/glog/cmake/TestPackageConfig.cmake @@ -0,0 +1,33 @@ +# Create the build directory +execute_process ( + COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_BINARY_DIR} + RESULT_VARIABLE _DIRECTORY_CREATED_SUCCEEDED +) + +if (NOT _DIRECTORY_CREATED_SUCCEEDED EQUAL 0) + message (FATAL_ERROR "Failed to create build directory") +endif (NOT _DIRECTORY_CREATED_SUCCEEDED EQUAL 0) + +# Capture the PATH environment variable content set during project +# generation stage. This is required because later during the build stage +# the PATH is modified again (e.g., for MinGW AppVeyor CI builds) by adding +# back the directory containing git.exe. Incidently, the Git installation +# directory also contains sh.exe which causes MinGW Makefile generation to +# fail. +set (ENV{PATH} ${PATH}) + +# Run CMake +execute_process ( + COMMAND ${CMAKE_COMMAND} -C ${INITIAL_CACHE} + -G ${GENERATOR} + -DCMAKE_PREFIX_PATH=${PACKAGE_DIR} + -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON + -DCMAKE_FIND_PACKAGE_NO_SYSTEM_PACKAGE_REGISTRY=ON + ${SOURCE_DIR} + WORKING_DIRECTORY ${TEST_BINARY_DIR} + RESULT_VARIABLE _GENERATE_SUCCEEDED +) + +if (NOT _GENERATE_SUCCEEDED EQUAL 0) + message (FATAL_ERROR "Failed to generate project files using CMake") +endif (NOT _GENERATE_SUCCEEDED EQUAL 0) diff --git a/third_party/glog/configure.ac b/third_party/glog/configure.ac new file mode 100644 index 000000000000..80005c267e53 --- /dev/null +++ b/third_party/glog/configure.ac @@ -0,0 +1,246 @@ +## Process this file with autoconf to produce configure. +## In general, the safest way to proceed is to run the following: +## % aclocal -I . -I `pwd`/../autoconf && autoheader && autoconf && automake + +# make sure we're interpreted by some minimal autoconf +AC_PREREQ(2.57) + +AC_INIT(glog, 0.4.0, opensource@google.com) +# The argument here is just something that should be in the current directory +# (for sanity checking) +AC_CONFIG_SRCDIR(README.md) +AC_CONFIG_MACRO_DIR([m4]) +AM_INIT_AUTOMAKE +AM_CONFIG_HEADER(src/config.h) + +AC_LANG(C++) + +# Checks for programs. +AC_PROG_CC +AC_PROG_CPP +AC_PROG_CXX +AM_CONDITIONAL(GCC, test "$GCC" = yes) # let the Makefile know if we're gcc + +AC_PROG_LIBTOOL +AC_SUBST(LIBTOOL_DEPS) + +# Check whether some low-level functions/files are available +AC_HEADER_STDC + +# These are tested for by AC_HEADER_STDC, but I check again to set the var +AC_CHECK_HEADER(stdint.h, ac_cv_have_stdint_h=1, ac_cv_have_stdint_h=0) +AC_CHECK_HEADER(sys/types.h, ac_cv_have_systypes_h=1, ac_cv_have_systypes_h=0) +AC_CHECK_HEADER(inttypes.h, ac_cv_have_inttypes_h=1, ac_cv_have_inttypes_h=0) +AC_CHECK_HEADER(pwd.h, ac_cv_have_pwd_h=1, ac_cv_have_pwd_h=0) +AC_CHECK_HEADERS(unistd.h, ac_cv_have_unistd_h=1, ac_cv_have_unistd_h=0) +AC_CHECK_HEADERS(syscall.h) +AC_CHECK_HEADERS(sys/syscall.h) +# For backtrace with glibc. +AC_CHECK_HEADERS(execinfo.h) +# For backtrace with libunwind. +AC_CHECK_HEADERS(libunwind.h, ac_cv_have_libunwind_h=1, ac_cv_have_libunwind_h=0) +AC_CHECK_HEADERS(ucontext.h) +AC_CHECK_HEADERS(sys/utsname.h) +AC_CHECK_HEADERS(pwd.h) +AC_CHECK_HEADERS(syslog.h) +AC_CHECK_HEADERS(sys/time.h) +AC_CHECK_HEADERS(glob.h) +# For backtrace with gcc. +AC_CHECK_HEADERS(unwind.h) + +AC_CHECK_HEADER(windows.h, ac_cv_have_windows_h=1, ac_cv_have_windows_h=0) +if test x"$ac_cv_have_windows_h" = x"1"; then + MINGW_CFLAGS=-Isrc/windows +fi + +AC_CHECK_SIZEOF(void *) + +# These are the types I need. We look for them in either stdint.h, +# sys/types.h, or inttypes.h, all of which are part of the default-includes. +AC_CHECK_TYPE(uint16_t, ac_cv_have_uint16_t=1, ac_cv_have_uint16_t=0) +AC_CHECK_TYPE(u_int16_t, ac_cv_have_u_int16_t=1, ac_cv_have_u_int16_t=0) +AC_CHECK_TYPE(__uint16, ac_cv_have___uint16=1, ac_cv_have___uint16=0) + +AC_CHECK_FUNC(sigaltstack, + AC_DEFINE(HAVE_SIGALTSTACK, 1, + [Define if you have the `sigaltstack' function])) +AC_CHECK_FUNC(sigaction, + AC_DEFINE(HAVE_SIGACTION, 1, + [Define if you have the 'sigaction' function])) +AC_CHECK_FUNC(dladdr, + AC_DEFINE(HAVE_DLADDR, 1, + [Define if you have the `dladdr' function])) +AC_CHECK_FUNC(fcntl, + AC_DEFINE(HAVE_FCNTL, 1, + [Define if you have the `fcntl' function])) +AC_CHECK_FUNC(pread, + AC_DEFINE(HAVE_PREAD, 1, + [Define if you have the 'pread' function])) +AC_CHECK_FUNC(pwrite, + AC_DEFINE(HAVE_PWRITE, 1, + [Define if you have the 'pwrite' function])) + +AX_C___ATTRIBUTE__ +# We only care about these two attributes. +if test x"$ac_cv___attribute__" = x"yes"; then + ac_cv___attribute___noreturn="__attribute__ ((noreturn))" + ac_cv___attribute___noinline="__attribute__ ((noinline))" + ac_cv___attribute___printf_4_5="__attribute__((__format__ (__printf__, 4, 5)))" +else + ac_cv___attribute___noreturn= + ac_cv___attribute___noinline= + ac_cv___attribute___printf_4_5= +fi + +AX_C___BUILTIN_EXPECT +if test x"$ac_cv___builtin_expect" = x"yes"; then + ac_cv_have___builtin_expect=1 +else + ac_cv_have___builtin_expect=0 +fi + +AX_C___SYNC_VAL_COMPARE_AND_SWAP + +# On x86_64, instead of libunwind, we can choose to compile with frame-pointers +# (This isn't needed on i386, where -fno-omit-frame-pointer is the default). +AC_ARG_ENABLE(frame_pointers, + AS_HELP_STRING([--enable-frame-pointers], + [On x86_64 systems, compile with -fno-omit-frame-pointer (see INSTALL)]),, + enable_frame_pointers=no) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM(, [return __x86_64__ == 1 ? 0 : 1])], + [is_x86_64=yes], [is_x86_64=no]) +AM_CONDITIONAL(ENABLE_FRAME_POINTERS, test "$enable_frame_pointers" = yes) +AM_CONDITIONAL(X86_64, test "$is_x86_64" = yes) + +AC_ARG_ENABLE(rtti, + AS_HELP_STRING([--disable-rtti], + [Disable RTTI in glog])) +AM_CONDITIONAL(DISABLE_RTTI, test x"$enable_rtti" = x"no") +if test x"$enable_rtti" = x"no"; then + AC_DEFINE(DISABLE_RTTI, 1, [define if glog doesn't use RTTI]) +fi + +# Some of the code in this directory depends on pthreads +ACX_PTHREAD +if test x"$acx_pthread_ok" = x"yes"; then + # To make libglog depend on libpthread on Linux, we need to add + # -lpthread in addition to -pthread. + AC_CHECK_LIB(pthread, pthread_self) +fi + +# Check if there is google-gflags library installed. +SAVE_CFLAGS="$CFLAGS" +SAVE_LIBS="$LIBS" +AC_ARG_WITH(gflags, AS_HELP_STRING[--with-gflags=GFLAGS_DIR], + GFLAGS_CFLAGS="-I${with_gflags}/include" + GFLAGS_LIBS="-L${with_gflags}/lib -lgflags" + CFLAGS="$CFLAGS $GFLAGS_CFLAGS" + LIBS="$LIBS $GFLAGS_LIBS" +) +AC_CHECK_LIB(gflags, main, ac_cv_have_libgflags=1, ac_cv_have_libgflags=0) +if test x"$ac_cv_have_libgflags" = x"1"; then + AC_DEFINE(HAVE_LIB_GFLAGS, 1, [define if you have google gflags library]) + if test x"$GFLAGS_LIBS" = x""; then + GFLAGS_LIBS="-lgflags" + fi +else + GFLAGS_CFLAGS= + GFLAGS_LIBS= +fi +CFLAGS="$SAVE_CFLAGS" +LIBS="$SAVE_LIBS" + +# TODO(hamaji): Use official m4 macros provided by testing libraries +# once the m4 macro of Google Mocking becomes ready. +# Check if there is Google Test library installed. +AC_CHECK_PROG(GTEST_CONFIG, gtest-config, "yes") +AC_CHECK_LIB(gtest, main, have_gtest_lib="yes") +if test x"$GTEST_CONFIG" = "xyes" -a x"$have_gtest_lib" = "xyes"; then + GTEST_CFLAGS=`gtest-config --cppflags --cxxflags` + GTEST_LIBS=`gtest-config --ldflags --libs` + AC_DEFINE(HAVE_LIB_GTEST, 1, [define if you have google gtest library]) + + # Check if there is Google Mocking library installed. + AC_CHECK_PROG(GMOCK_CONFIG, gmock-config, "yes") + if test x"$GMOCK_CONFIG" = "xyes"; then + GMOCK_CFLAGS=`gmock-config --cppflags --cxxflags` + GMOCK_LIBS=`gmock-config --ldflags --libs` + AC_DEFINE(HAVE_LIB_GMOCK, 1, [define if you have google gmock library]) + else + # We don't run test cases which use Google Mocking framework. + GMOCK_CFLAGS= + GMOCK_LIBS= + fi +else + # We'll use src/googletest.h for our unittests. + GTEST_CFLAGS= + GTEST_LIBS= +fi +AM_CONDITIONAL(HAVE_GMOCK, test x"$GMOCK_CONFIG" = "xyes") + +# We want to link in libunwind if it exists +UNWIND_LIBS= +# Unfortunately, we need to check the header file in addition to the +# lib file to check if libunwind is available since libunwind-0.98 +# doesn't install all necessary header files. +if test x"$ac_cv_have_libunwind_h" = x"1"; then + AC_CHECK_LIB(unwind, backtrace, UNWIND_LIBS=-lunwind) +fi +AC_SUBST(UNWIND_LIBS) +if test x"$UNWIND_LIBS" != x""; then + AC_DEFINE(HAVE_LIB_UNWIND, 1, [define if you have libunwind]) +fi + +# We'd like to use read/write locks in several places in the code. +# See if our pthreads support extends to that. Note: for linux, it +# does as long as you define _XOPEN_SOURCE appropriately. +AC_RWLOCK + +# Find out what namespace 'normal' STL code lives in, and also what namespace +# the user wants our classes to be defined in +AC_CXX_STL_NAMESPACE +AC_DEFINE_GOOGLE_NAMESPACE(google) + +AC_CXX_USING_OPERATOR + +AC_PC_FROM_UCONTEXT(AC_MSG_WARN(Could not find the PC. Will not output failed addresses...)) + +AC_DEFINE_UNQUOTED(TEST_SRC_DIR, "$srcdir", [location of source code]) + +AC_ARG_ENABLE(unsymbolized-traces, + AS_HELP_STRING([--enable-unsymbolized-traces], + [Print file offsets in traces instead of symbolizing.]), + enable_unsymbolized_traces=yes) +if test x"$enable_unsymbolized_traces" = x"yes"; then + AC_DEFINE(PRINT_UNSYMBOLIZED_STACK_TRACES, 1, + [define if we should print file offsets in traces instead of symbolizing.]) +fi + +# These are what's needed by logging.h.in and raw_logging.h.in +AC_SUBST(ac_google_start_namespace) +AC_SUBST(ac_google_end_namespace) +AC_SUBST(ac_google_namespace) +AC_SUBST(ac_cv_cxx_using_operator) +AC_SUBST(ac_cv___attribute___noreturn) +AC_SUBST(ac_cv___attribute___noinline) +AC_SUBST(ac_cv___attribute___printf_4_5) +AC_SUBST(ac_cv_have___builtin_expect) +AC_SUBST(ac_cv_have_stdint_h) +AC_SUBST(ac_cv_have_systypes_h) +AC_SUBST(ac_cv_have_inttypes_h) +AC_SUBST(ac_cv_have_unistd_h) +AC_SUBST(ac_cv_have_uint16_t) +AC_SUBST(ac_cv_have_u_int16_t) +AC_SUBST(ac_cv_have___uint16) +AC_SUBST(ac_cv_have_libgflags) +AC_SUBST(GFLAGS_CFLAGS) +AC_SUBST(GTEST_CFLAGS) +AC_SUBST(GMOCK_CFLAGS) +AC_SUBST(MINGW_CFLAGS) +AC_SUBST(GFLAGS_LIBS) +AC_SUBST(GTEST_LIBS) +AC_SUBST(GMOCK_LIBS) + +# Write generated configuration file +AC_CONFIG_FILES([Makefile src/glog/logging.h src/glog/raw_logging.h src/glog/vlog_is_on.h src/glog/stl_logging.h]) +AC_OUTPUT(libglog.pc) diff --git a/third_party/glog/default.nix b/third_party/glog/default.nix new file mode 100644 index 000000000000..a28bcbaea9e5 --- /dev/null +++ b/third_party/glog/default.nix @@ -0,0 +1,21 @@ +{ pkgs, lib, ... }: + +let inherit (pkgs) fullLlvm11Stdenv cmake; +in fullLlvm11Stdenv.mkDerivation { + name = "glog"; + version = "20200527-unstable"; + src = ./.; + + nativeBuildInputs = [ cmake ]; + cmakeFlags = [ + "-DCMAKE_CXX_STANDARD=17" + "-WITH_GFLAGS=OFF" + ]; + + meta = with lib; { + homepage = "https://github.com/google/glog"; + license = licenses.bsd3; + description = "Library for application-level logging"; + platforms = platforms.unix; + }; +} diff --git a/third_party/glog/doc/designstyle.css b/third_party/glog/doc/designstyle.css new file mode 100644 index 000000000000..f5d1ec2f767a --- /dev/null +++ b/third_party/glog/doc/designstyle.css @@ -0,0 +1,115 @@ +body { + background-color: #ffffff; + color: black; + margin-right: 1in; + margin-left: 1in; +} + + +h1, h2, h3, h4, h5, h6 { + color: #3366ff; + font-family: sans-serif; +} +@media print { + /* Darker version for printing */ + h1, h2, h3, h4, h5, h6 { + color: #000080; + font-family: helvetica, sans-serif; + } +} + +h1 { + text-align: center; + font-size: 18pt; +} +h2 { + margin-left: -0.5in; +} +h3 { + margin-left: -0.25in; +} +h4 { + margin-left: -0.125in; +} +hr { + margin-left: -1in; +} + +/* Definition lists: definition term bold */ +dt { + font-weight: bold; +} + +address { + text-align: right; +} +/* Use the <code> tag for bits of code and <var> for variables and objects. */ +code,pre,samp,var { + color: #006000; +} +/* Use the <file> tag for file and directory paths and names. */ +file { + color: #905050; + font-family: monospace; +} +/* Use the <kbd> tag for stuff the user should type. */ +kbd { + color: #600000; +} +div.note p { + float: right; + width: 3in; + margin-right: 0%; + padding: 1px; + border: 2px solid #6060a0; + background-color: #fffff0; +} + +UL.nobullets { + list-style-type: none; + list-style-image: none; + margin-left: -1em; +} + +/* +body:after { + content: "Google Confidential"; +} +*/ + +/* pretty printing styles. See prettify.js */ +.str { color: #080; } +.kwd { color: #008; } +.com { color: #800; } +.typ { color: #606; } +.lit { color: #066; } +.pun { color: #660; } +.pln { color: #000; } +.tag { color: #008; } +.atn { color: #606; } +.atv { color: #080; } +pre.prettyprint { padding: 2px; border: 1px solid #888; } + +.embsrc { background: #eee; } + +@media print { + .str { color: #060; } + .kwd { color: #006; font-weight: bold; } + .com { color: #600; font-style: italic; } + .typ { color: #404; font-weight: bold; } + .lit { color: #044; } + .pun { color: #440; } + .pln { color: #000; } + .tag { color: #006; font-weight: bold; } + .atn { color: #404; } + .atv { color: #060; } +} + +/* Table Column Headers */ +.hdr { + color: #006; + font-weight: bold; + background-color: #dddddd; } +.hdr2 { + color: #006; + background-color: #eeeeee; } \ No newline at end of file diff --git a/third_party/glog/doc/glog.html b/third_party/glog/doc/glog.html new file mode 100644 index 000000000000..699d2ceb80b1 --- /dev/null +++ b/third_party/glog/doc/glog.html @@ -0,0 +1,631 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN"> + +<html> +<head> +<title>How To Use Google Logging Library (glog)</title> + +<meta http-equiv="Content-Type" content="text/html; charset=utf-8"> +<link href="http://www.google.com/favicon.ico" type="image/x-icon" + rel="shortcut icon"> +<link href="designstyle.css" type="text/css" rel="stylesheet"> +<style type="text/css"> +<!-- + ol.bluelist li { + color: #3366ff; + font-family: sans-serif; + } + ol.bluelist li p { + color: #000; + font-family: "Times Roman", times, serif; + } + ul.blacklist li { + color: #000; + font-family: "Times Roman", times, serif; + } +//--> +</style> +</head> + +<body> + +<h1>How To Use Google Logging Library (glog)</h1> +<small>(as of +<script type=text/javascript> + var lm = new Date(document.lastModified); + document.write(lm.toDateString()); +</script>) +</small> +<br> + +<h2> <A NAME=intro>Introduction</A> </h2> + +<p><b>Google glog</b> is a library that implements application-level +logging. This library provides logging APIs based on C++-style +streams and various helper macros. +You can log a message by simply streaming things to LOG(<a +particular <a href="#severity">severity level</a>>), e.g. + +<pre> + #include <glog/logging.h> + + int main(int argc, char* argv[]) { + // Initialize Google's logging library. + google::InitGoogleLogging(argv[0]); + + // ... + LOG(INFO) << "Found " << num_cookies << " cookies"; + } +</pre> + +<p>Google glog defines a series of macros that simplify many common logging +tasks. You can log messages by severity level, control logging +behavior from the command line, log based on conditionals, abort the +program when expected conditions are not met, introduce your own +verbose logging levels, and more. This document describes the +functionality supported by glog. Please note that this document +doesn't describe all features in this library, but the most useful +ones. If you want to find less common features, please check +header files under <code>src/glog</code> directory. + +<h2> <A NAME=severity>Severity Level</A> </h2> + +<p> +You can specify one of the following severity levels (in +increasing order of severity): <code>INFO</code>, <code>WARNING</code>, +<code>ERROR</code>, and <code>FATAL</code>. +Logging a <code>FATAL</code> message terminates the program (after the +message is logged). +Note that messages of a given severity are logged not only in the +logfile for that severity, but also in all logfiles of lower severity. +E.g., a message of severity <code>FATAL</code> will be logged to the +logfiles of severity <code>FATAL</code>, <code>ERROR</code>, +<code>WARNING</code>, and <code>INFO</code>. + +<p> +The <code>DFATAL</code> severity logs a <code>FATAL</code> error in +debug mode (i.e., there is no <code>NDEBUG</code> macro defined), but +avoids halting the program in production by automatically reducing the +severity to <code>ERROR</code>. + +<p>Unless otherwise specified, glog writes to the filename +"/tmp/<program name>.<hostname>.<user name>.log.<severity level>.<date>.<time>.<pid>" +(e.g., "/tmp/hello_world.example.com.hamaji.log.INFO.20080709-222411.10474"). +By default, glog copies the log messages of severity level +<code>ERROR</code> or <code>FATAL</code> to standard error (stderr) +in addition to log files. + +<h2><A NAME=flags>Setting Flags</A></h2> + +<p>Several flags influence glog's output behavior. +If the <a href="https://github.com/gflags/gflags">Google +gflags library</a> is installed on your machine, the +<code>configure</code> script (see the INSTALL file in the package for +detail of this script) will automatically detect and use it, +allowing you to pass flags on the command line. For example, if you +want to turn the flag <code>--logtostderr</code> on, you can start +your application with the following command line: + +<pre> + ./your_application --logtostderr=1 +</pre> + +If the Google gflags library isn't installed, you set flags via +environment variables, prefixing the flag name with "GLOG_", e.g. + +<pre> + GLOG_logtostderr=1 ./your_application +</pre> + +<!-- TODO(hamaji): Fill the version number +<p>By glog version 0.x.x, you can use GLOG_* environment variables +even if you have gflags. If both an environment variable and a flag +are specified, the value specified by a flag wins. E.g., if GLOG_v=0 +and --v=1, the verbosity will be 1, not 0. +--> + +<p>The following flags are most commonly used: + +<dl> +<dt><code>logtostderr</code> (<code>bool</code>, default=<code>false</code>) +<dd>Log messages to stderr instead of logfiles.<br> +Note: you can set binary flags to <code>true</code> by specifying +<code>1</code>, <code>true</code>, or <code>yes</code> (case +insensitive). +Also, you can set binary flags to <code>false</code> by specifying +<code>0</code>, <code>false</code>, or <code>no</code> (again, case +insensitive). +<dt><code>stderrthreshold</code> (<code>int</code>, default=2, which +is <code>ERROR</code>) +<dd>Copy log messages at or above this level to stderr in +addition to logfiles. The numbers of severity levels +<code>INFO</code>, <code>WARNING</code>, <code>ERROR</code>, and +<code>FATAL</code> are 0, 1, 2, and 3, respectively. +<dt><code>minloglevel</code> (<code>int</code>, default=0, which +is <code>INFO</code>) +<dd>Log messages at or above this level. Again, the numbers of +severity levels <code>INFO</code>, <code>WARNING</code>, +<code>ERROR</code>, and <code>FATAL</code> are 0, 1, 2, and 3, +respectively. +<dt><code>log_dir</code> (<code>string</code>, default="") +<dd>If specified, logfiles are written into this directory instead +of the default logging directory. +<dt><code>v</code> (<code>int</code>, default=0) +<dd>Show all <code>VLOG(m)</code> messages for <code>m</code> less or +equal the value of this flag. Overridable by --vmodule. +See <a href="#verbose">the section about verbose logging</a> for more +detail. +<dt><code>vmodule</code> (<code>string</code>, default="") +<dd>Per-module verbose level. The argument has to contain a +comma-separated list of <module name>=<log level>. +<module name> +is a glob pattern (e.g., <code>gfs*</code> for all modules whose name +starts with "gfs"), matched against the filename base +(that is, name ignoring .cc/.h./-inl.h). +<log level> overrides any value given by --v. +See also <a href="#verbose">the section about verbose logging</a>. +</dl> + +<p>There are some other flags defined in logging.cc. Please grep the +source code for "DEFINE_" to see a complete list of all flags. + +<p>You can also modify flag values in your program by modifying global +variables <code>FLAGS_*</code> . Most settings start working +immediately after you update <code>FLAGS_*</code> . The exceptions are +the flags related to destination files. For example, you might want to +set <code>FLAGS_log_dir</code> before +calling <code>google::InitGoogleLogging</code> . Here is an example: + +<pre> + LOG(INFO) << "file"; + // Most flags work immediately after updating values. + FLAGS_logtostderr = 1; + LOG(INFO) << "stderr"; + FLAGS_logtostderr = 0; + // This won't change the log destination. If you want to set this + // value, you should do this before google::InitGoogleLogging . + FLAGS_log_dir = "/some/log/directory"; + LOG(INFO) << "the same file"; +</pre> + +<h2><A NAME=conditional>Conditional / Occasional Logging</A></h2> + +<p>Sometimes, you may only want to log a message under certain +conditions. You can use the following macros to perform conditional +logging: + +<pre> + LOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; +</pre> + +The "Got lots of cookies" message is logged only when the variable +<code>num_cookies</code> exceeds 10. + +If a line of code is executed many times, it may be useful to only log +a message at certain intervals. This kind of logging is most useful +for informational messages. + +<pre> + LOG_EVERY_N(INFO, 10) << "Got the " << google::COUNTER << "th cookie"; +</pre> + +<p>The above line outputs a log messages on the 1st, 11th, +21st, ... times it is executed. Note that the special +<code>google::COUNTER</code> value is used to identify which repetition is +happening. + +<p>You can combine conditional and occasional logging with the +following macro. + +<pre> + LOG_IF_EVERY_N(INFO, (size > 1024), 10) << "Got the " << google::COUNTER + << "th big cookie"; +</pre> + +<p>Instead of outputting a message every nth time, you can also limit +the output to the first n occurrences: + +<pre> + LOG_FIRST_N(INFO, 20) << "Got the " << google::COUNTER << "th cookie"; +</pre> + +<p>Outputs log messages for the first 20 times it is executed. Again, +the <code>google::COUNTER</code> identifier indicates which repetition is +happening. + +<h2><A NAME=debug>Debug Mode Support</A></h2> + +<p>Special "debug mode" logging macros only have an effect in debug +mode and are compiled away to nothing for non-debug mode +compiles. Use these macros to avoid slowing down your production +application due to excessive logging. + +<pre> + DLOG(INFO) << "Found cookies"; + + DLOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; + + DLOG_EVERY_N(INFO, 10) << "Got the " << google::COUNTER << "th cookie"; +</pre> + +<h2><A NAME=check>CHECK Macros</A></h2> + +<p>It is a good practice to check expected conditions in your program +frequently to detect errors as early as possible. The +<code>CHECK</code> macro provides the ability to abort the application +when a condition is not met, similar to the <code>assert</code> macro +defined in the standard C library. + +<p><code>CHECK</code> aborts the application if a condition is not +true. Unlike <code>assert</code>, it is *not* controlled by +<code>NDEBUG</code>, so the check will be executed regardless of +compilation mode. Therefore, <code>fp->Write(x)</code> in the +following example is always executed: + +<pre> + CHECK(fp->Write(x) == 4) << "Write failed!"; +</pre> + +<p>There are various helper macros for +equality/inequality checks - <code>CHECK_EQ</code>, +<code>CHECK_NE</code>, <code>CHECK_LE</code>, <code>CHECK_LT</code>, +<code>CHECK_GE</code>, and <code>CHECK_GT</code>. +They compare two values, and log a +<code>FATAL</code> message including the two values when the result is +not as expected. The values must have <code>operator<<(ostream, +...)</code> defined. + +<p>You may append to the error message like so: + +<pre> + CHECK_NE(1, 2) << ": The world must be ending!"; +</pre> + +<p>We are very careful to ensure that each argument is evaluated exactly +once, and that anything which is legal to pass as a function argument is +legal here. In particular, the arguments may be temporary expressions +which will end up being destroyed at the end of the apparent statement, +for example: + +<pre> + CHECK_EQ(string("abc")[1], 'b'); +</pre> + +<p>The compiler reports an error if one of the arguments is a +pointer and the other is NULL. To work around this, simply static_cast +NULL to the type of the desired pointer. + +<pre> + CHECK_EQ(some_ptr, static_cast<SomeType*>(NULL)); +</pre> + +<p>Better yet, use the CHECK_NOTNULL macro: + +<pre> + CHECK_NOTNULL(some_ptr); + some_ptr->DoSomething(); +</pre> + +<p>Since this macro returns the given pointer, this is very useful in +constructor initializer lists. + +<pre> + struct S { + S(Something* ptr) : ptr_(CHECK_NOTNULL(ptr)) {} + Something* ptr_; + }; +</pre> + +<p>Note that you cannot use this macro as a C++ stream due to this +feature. Please use <code>CHECK_EQ</code> described above to log a +custom message before aborting the application. + +<p>If you are comparing C strings (char *), a handy set of macros +performs case sensitive as well as case insensitive comparisons - +<code>CHECK_STREQ</code>, <code>CHECK_STRNE</code>, +<code>CHECK_STRCASEEQ</code>, and <code>CHECK_STRCASENE</code>. The +CASE versions are case-insensitive. You can safely pass <code>NULL</code> +pointers for this macro. They treat <code>NULL</code> and any +non-<code>NULL</code> string as not equal. Two <code>NULL</code>s are +equal. + +<p>Note that both arguments may be temporary strings which are +destructed at the end of the current "full expression" +(e.g., <code>CHECK_STREQ(Foo().c_str(), Bar().c_str())</code> where +<code>Foo</code> and <code>Bar</code> return C++'s +<code>std::string</code>). + +<p>The <code>CHECK_DOUBLE_EQ</code> macro checks the equality of two +floating point values, accepting a small error margin. +<code>CHECK_NEAR</code> accepts a third floating point argument, which +specifies the acceptable error margin. + +<h2><A NAME=verbose>Verbose Logging</A></h2> + +<p>When you are chasing difficult bugs, thorough log messages are very +useful. However, you may want to ignore too verbose messages in usual +development. For such verbose logging, glog provides the +<code>VLOG</code> macro, which allows you to define your own numeric +logging levels. The <code>--v</code> command line option controls +which verbose messages are logged: + +<pre> + VLOG(1) << "I'm printed when you run the program with --v=1 or higher"; + VLOG(2) << "I'm printed when you run the program with --v=2 or higher"; +</pre> + +<p>With <code>VLOG</code>, the lower the verbose level, the more +likely messages are to be logged. For example, if +<code>--v==1</code>, <code>VLOG(1)</code> will log, but +<code>VLOG(2)</code> will not log. This is opposite of the severity +level, where <code>INFO</code> is 0, and <code>ERROR</code> is 2. +<code>--minloglevel</code> of 1 will log <code>WARNING</code> and +above. Though you can specify any integers for both <code>VLOG</code> +macro and <code>--v</code> flag, the common values for them are small +positive integers. For example, if you write <code>VLOG(0)</code>, +you should specify <code>--v=-1</code> or lower to silence it. This +is less useful since we may not want verbose logs by default in most +cases. The <code>VLOG</code> macros always log at the +<code>INFO</code> log level (when they log at all). + +<p>Verbose logging can be controlled from the command line on a +per-module basis: + +<pre> + --vmodule=mapreduce=2,file=1,gfs*=3 --v=0 +</pre> + +<p>will: + +<ul> + <li>a. Print VLOG(2) and lower messages from mapreduce.{h,cc} + <li>b. Print VLOG(1) and lower messages from file.{h,cc} + <li>c. Print VLOG(3) and lower messages from files prefixed with "gfs" + <li>d. Print VLOG(0) and lower messages from elsewhere +</ul> + +<p>The wildcarding functionality shown by (c) supports both '*' +(matches 0 or more characters) and '?' (matches any single character) +wildcards. Please also check the section about <a +href="#flags">command line flags</a>. + +<p>There's also <code>VLOG_IS_ON(n)</code> "verbose level" condition +macro. This macro returns true when the <code>--v</code> is equal or +greater than <code>n</code>. To be used as + +<pre> + if (VLOG_IS_ON(2)) { + // do some logging preparation and logging + // that can't be accomplished with just VLOG(2) << ...; + } +</pre> + +<p>Verbose level condition macros <code>VLOG_IF</code>, +<code>VLOG_EVERY_N</code> and <code>VLOG_IF_EVERY_N</code> behave +analogous to <code>LOG_IF</code>, <code>LOG_EVERY_N</code>, +<code>LOF_IF_EVERY</code>, but accept a numeric verbosity level as +opposed to a severity level. + +<pre> + VLOG_IF(1, (size > 1024)) + << "I'm printed when size is more than 1024 and when you run the " + "program with --v=1 or more"; + VLOG_EVERY_N(1, 10) + << "I'm printed every 10th occurrence, and when you run the program " + "with --v=1 or more. Present occurence is " << google::COUNTER; + VLOG_IF_EVERY_N(1, (size > 1024), 10) + << "I'm printed on every 10th occurence of case when size is more " + " than 1024, when you run the program with --v=1 or more. "; + "Present occurence is " << google::COUNTER; +</pre> + +<h2> <A name="signal">Failure Signal Handler</A> </h2> + +<p> +The library provides a convenient signal handler that will dump useful +information when the program crashes on certain signals such as SIGSEGV. +The signal handler can be installed by +google::InstallFailureSignalHandler(). The following is an example of output +from the signal handler. + +<pre> +*** Aborted at 1225095260 (unix time) try "date -d @1225095260" if you are using GNU date *** +*** SIGSEGV (@0x0) received by PID 17711 (TID 0x7f893090a6f0) from PID 0; stack trace: *** +PC: @ 0x412eb1 TestWaitingLogSink::send() + @ 0x7f892fb417d0 (unknown) + @ 0x412eb1 TestWaitingLogSink::send() + @ 0x7f89304f7f06 google::LogMessage::SendToLog() + @ 0x7f89304f35af google::LogMessage::Flush() + @ 0x7f89304f3739 google::LogMessage::~LogMessage() + @ 0x408cf4 TestLogSinkWaitTillSent() + @ 0x4115de main + @ 0x7f892f7ef1c4 (unknown) + @ 0x4046f9 (unknown) +</pre> + +<p> +By default, the signal handler writes the failure dump to the standard +error. You can customize the destination by InstallFailureWriter(). + +<h2> <A name="misc">Miscellaneous Notes</A> </h2> + +<h3><A NAME=message>Performance of Messages</A></h3> + +<p>The conditional logging macros provided by glog (e.g., +<code>CHECK</code>, <code>LOG_IF</code>, <code>VLOG</code>, ...) are +carefully implemented and don't execute the right hand side +expressions when the conditions are false. So, the following check +may not sacrifice the performance of your application. + +<pre> + CHECK(obj.ok) << obj.CreatePrettyFormattedStringButVerySlow(); +</pre> + +<h3><A NAME=failure>User-defined Failure Function</A></h3> + +<p><code>FATAL</code> severity level messages or unsatisfied +<code>CHECK</code> condition terminate your program. You can change +the behavior of the termination by +<code>InstallFailureFunction</code>. + +<pre> + void YourFailureFunction() { + // Reports something... + exit(1); + } + + int main(int argc, char* argv[]) { + google::InstallFailureFunction(&YourFailureFunction); + } +</pre> + +<p>By default, glog tries to dump stacktrace and makes the program +exit with status 1. The stacktrace is produced only when you run the +program on an architecture for which glog supports stack tracing (as +of September 2008, glog supports stack tracing for x86 and x86_64). + +<h3><A NAME=raw>Raw Logging</A></h3> + +<p>The header file <code><glog/raw_logging.h></code> can be +used for thread-safe logging, which does not allocate any memory or +acquire any locks. Therefore, the macros defined in this +header file can be used by low-level memory allocation and +synchronization code. +Please check <code>src/glog/raw_logging.h.in</code> for detail. +</p> + +<h3><A NAME=plog>Google Style perror()</A></h3> + +<p><code>PLOG()</code> and <code>PLOG_IF()</code> and +<code>PCHECK()</code> behave exactly like their <code>LOG*</code> and +<code>CHECK</code> equivalents with the addition that they append a +description of the current state of errno to their output lines. +E.g. + +<pre> + PCHECK(write(1, NULL, 2) >= 0) << "Write NULL failed"; +</pre> + +<p>This check fails with the following error message. + +<pre> + F0825 185142 test.cc:22] Check failed: write(1, NULL, 2) >= 0 Write NULL failed: Bad address [14] +</pre> + +<h3><A NAME=syslog>Syslog</A></h3> + +<p><code>SYSLOG</code>, <code>SYSLOG_IF</code>, and +<code>SYSLOG_EVERY_N</code> macros are available. +These log to syslog in addition to the normal logs. Be aware that +logging to syslog can drastically impact performance, especially if +syslog is configured for remote logging! Make sure you understand the +implications of outputting to syslog before you use these macros. In +general, it's wise to use these macros sparingly. + +<h3><A NAME=strip>Strip Logging Messages</A></h3> + +<p>Strings used in log messages can increase the size of your binary +and present a privacy concern. You can therefore instruct glog to +remove all strings which fall below a certain severity level by using +the GOOGLE_STRIP_LOG macro: + +<p>If your application has code like this: + +<pre> + #define GOOGLE_STRIP_LOG 1 // this must go before the #include! + #include <glog/logging.h> +</pre> + +<p>The compiler will remove the log messages whose severities are less +than the specified integer value. Since +<code>VLOG</code> logs at the severity level <code>INFO</code> +(numeric value <code>0</code>), +setting <code>GOOGLE_STRIP_LOG</code> to 1 or greater removes +all log messages associated with <code>VLOG</code>s as well as +<code>INFO</code> log statements. + +<h3><A NAME=strip>Automatically Remove Old Logs</A></h3> + +<p>To enable the log cleaner: + +<pre> +google::EnableLogCleaner(3); // keep your logs for 3 days +</pre> + +And then Google glog will check if there are overdue logs whenever +a flush is performed. In this example, any log file from your project whose +last modified time is greater than 3 days will be unlink()ed. + +<p>This feature can be disabled at any time (if it has been enabled) + +<pre> +google::DisableLogCleaner(); +</pre> + +<h3><A NAME=windows>Notes for Windows users</A></h3> + +<p>Google glog defines a severity level <code>ERROR</code>, which is +also defined in <code>windows.h</code> . You can make glog not define +<code>INFO</code>, <code>WARNING</code>, <code>ERROR</code>, +and <code>FATAL</code> by defining +<code>GLOG_NO_ABBREVIATED_SEVERITIES</code> before +including <code>glog/logging.h</code> . Even with this macro, you can +still use the iostream like logging facilities: + +<pre> + #define GLOG_NO_ABBREVIATED_SEVERITIES + #include <windows.h> + #include <glog/logging.h> + + // ... + + LOG(ERROR) << "This should work"; + LOG_IF(ERROR, x > y) << "This should be also OK"; +</pre> + +<p> +However, you cannot +use <code>INFO</code>, <code>WARNING</code>, <code>ERROR</code>, +and <code>FATAL</code> anymore for functions defined +in <code>glog/logging.h</code> . + +<pre> + #define GLOG_NO_ABBREVIATED_SEVERITIES + #include <windows.h> + #include <glog/logging.h> + + // ... + + // This won't work. + // google::FlushLogFiles(google::ERROR); + + // Use this instead. + google::FlushLogFiles(google::GLOG_ERROR); +</pre> + +<p> +If you don't need <code>ERROR</code> defined +by <code>windows.h</code>, there are a couple of more workarounds +which sometimes don't work: + +<ul> + <li>#define <code>WIN32_LEAN_AND_MEAN</code> or <code>NOGDI</code> + <strong>before</strong> you #include <code>windows.h</code> . + <li>#undef <code>ERROR</code> <strong>after</strong> you #include + <code>windows.h</code> . +</ul> + +<p>See <a href="http://code.google.com/p/google-glog/issues/detail?id=33"> +this issue</a> for more detail. + +<hr> +<address> +Shinichiro Hamaji<br> +Gregor Hohpe<br> +<script type=text/javascript> + var lm = new Date(document.lastModified); + document.write(lm.toDateString()); +</script> +</address> + +</body> +</html> diff --git a/third_party/glog/glog-config.cmake.in b/third_party/glog/glog-config.cmake.in new file mode 100644 index 000000000000..5c5c9c0da262 --- /dev/null +++ b/third_party/glog/glog-config.cmake.in @@ -0,0 +1,13 @@ +if (CMAKE_VERSION VERSION_LESS @glog_CMake_VERSION@) + message (FATAL_ERROR "CMake >= @glog_CMake_VERSION@ required") +endif (CMAKE_VERSION VERSION_LESS @glog_CMake_VERSION@) + +@PACKAGE_INIT@ + +include (CMakeFindDependencyMacro) +include (${CMAKE_CURRENT_LIST_DIR}/glog-modules.cmake) + +@gflags_DEPENDENCY@ +@Unwind_DEPENDENCY@ + +include (${CMAKE_CURRENT_LIST_DIR}/glog-targets.cmake) diff --git a/third_party/glog/glog-modules.cmake.in b/third_party/glog/glog-modules.cmake.in new file mode 100644 index 000000000000..71c516078c97 --- /dev/null +++ b/third_party/glog/glog-modules.cmake.in @@ -0,0 +1,18 @@ +cmake_policy (PUSH) +cmake_policy (SET CMP0057 NEW) + +if (CMAKE_VERSION VERSION_LESS 3.3) + message (FATAL_ERROR "glog-modules.cmake requires the consumer " + "to use CMake 3.3 (or newer)") +endif (CMAKE_VERSION VERSION_LESS 3.3) + +set (glog_MODULE_PATH "@glog_FULL_CMake_DATADIR@") +list (APPEND CMAKE_MODULE_PATH ${glog_MODULE_PATH}) + +if (NOT glog_MODULE_PATH IN_LIST CMAKE_MODULE_PATH) + message (FATAL_ERROR "Cannot add '${glog_MODULE_PATH}' to " + "CMAKE_MODULE_PATH. This will cause glog-config.cmake to fail at " + "locating required find modules. Make sure CMAKE_MODULE_PATH is not a cache variable.") +endif (NOT glog_MODULE_PATH IN_LIST CMAKE_MODULE_PATH) + +cmake_policy (POP) diff --git a/third_party/glog/libglog.pc.in b/third_party/glog/libglog.pc.in new file mode 100644 index 000000000000..ad2b0774d89e --- /dev/null +++ b/third_party/glog/libglog.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libglog +Description: Google Log (glog) C++ logging framework +Version: @VERSION@ +Libs: -L${libdir} -lglog +Cflags: -I${includedir} diff --git a/third_party/glog/m4/ac_have_attribute.m4 b/third_party/glog/m4/ac_have_attribute.m4 new file mode 100644 index 000000000000..19f4021e9901 --- /dev/null +++ b/third_party/glog/m4/ac_have_attribute.m4 @@ -0,0 +1,16 @@ +AC_DEFUN([AX_C___ATTRIBUTE__], [ + AC_MSG_CHECKING(for __attribute__) + AC_CACHE_VAL(ac_cv___attribute__, [ + AC_TRY_COMPILE( + [#include <stdlib.h> + static void foo(void) __attribute__ ((unused)); + void foo(void) { exit(1); }], + [], + ac_cv___attribute__=yes, + ac_cv___attribute__=no + )]) + if test "$ac_cv___attribute__" = "yes"; then + AC_DEFINE(HAVE___ATTRIBUTE__, 1, [define if your compiler has __attribute__]) + fi + AC_MSG_RESULT($ac_cv___attribute__) +]) diff --git a/third_party/glog/m4/ac_have_builtin_expect.m4 b/third_party/glog/m4/ac_have_builtin_expect.m4 new file mode 100644 index 000000000000..e91b6fd987ed --- /dev/null +++ b/third_party/glog/m4/ac_have_builtin_expect.m4 @@ -0,0 +1,14 @@ +AC_DEFUN([AX_C___BUILTIN_EXPECT], [ + AC_MSG_CHECKING(for __builtin_expect) + AC_CACHE_VAL(ac_cv___builtin_expect, [ + AC_TRY_COMPILE( + [int foo(void) { if (__builtin_expect(0, 0)) return 1; return 0; }], + [], + ac_cv___builtin_expect=yes, + ac_cv___builtin_expect=no + )]) + if test "$ac_cv___builtin_expect" = "yes"; then + AC_DEFINE(HAVE___BUILTIN_EXPECT, 1, [define if your compiler has __builtin_expect]) + fi + AC_MSG_RESULT($ac_cv___builtin_expect) +]) diff --git a/third_party/glog/m4/ac_have_sync_val_compare_and_swap.m4 b/third_party/glog/m4/ac_have_sync_val_compare_and_swap.m4 new file mode 100644 index 000000000000..88b027e23a29 --- /dev/null +++ b/third_party/glog/m4/ac_have_sync_val_compare_and_swap.m4 @@ -0,0 +1,14 @@ +AC_DEFUN([AX_C___SYNC_VAL_COMPARE_AND_SWAP], [ + AC_MSG_CHECKING(for __sync_val_compare_and_swap) + AC_CACHE_VAL(ac_cv___sync_val_compare_and_swap, [ + AC_TRY_LINK( + [], + [int a; if (__sync_val_compare_and_swap(&a, 0, 1)) return 1; return 0;], + ac_cv___sync_val_compare_and_swap=yes, + ac_cv___sync_val_compare_and_swap=no + )]) + if test "$ac_cv___sync_val_compare_and_swap" = "yes"; then + AC_DEFINE(HAVE___SYNC_VAL_COMPARE_AND_SWAP, 1, [define if your compiler has __sync_val_compare_and_swap]) + fi + AC_MSG_RESULT($ac_cv___sync_val_compare_and_swap) +]) diff --git a/third_party/glog/m4/ac_rwlock.m4 b/third_party/glog/m4/ac_rwlock.m4 new file mode 100644 index 000000000000..5065bcc1c5c2 --- /dev/null +++ b/third_party/glog/m4/ac_rwlock.m4 @@ -0,0 +1,31 @@ +# TODO(csilvers): it would be better to actually try to link against +# -pthreads, to make sure it defines these methods, but that may be +# too hard, since pthread support is really tricky. + +# Check for support for pthread_rwlock_init() etc. +# These aren't posix, but are widely supported. To get them on linux, +# you need to define _XOPEN_SOURCE first, so this check assumes your +# application does that. +# +# Note: OS X (as of 6/1/06) seems to support pthread_rwlock, but +# doesn't define PTHREAD_RWLOCK_INITIALIZER. Therefore, we don't test +# that particularly macro. It's probably best if you don't use that +# macro in your code either. + +AC_DEFUN([AC_RWLOCK], +[AC_CACHE_CHECK(support for pthread_rwlock_* functions, +ac_cv_rwlock, +[AC_LANG_SAVE + AC_LANG_C + AC_TRY_COMPILE([#define _XOPEN_SOURCE 500 + #include <pthread.h>], + [pthread_rwlock_t l; pthread_rwlock_init(&l, NULL); + pthread_rwlock_rdlock(&l); + return 0;], + ac_cv_rwlock=yes, ac_cv_rwlock=no) + AC_LANG_RESTORE +]) +if test "$ac_cv_rwlock" = yes; then + AC_DEFINE(HAVE_RWLOCK,1,[define if the compiler implements pthread_rwlock_*]) +fi +]) diff --git a/third_party/glog/m4/acx_pthread.m4 b/third_party/glog/m4/acx_pthread.m4 new file mode 100644 index 000000000000..2cf20de144a1 --- /dev/null +++ b/third_party/glog/m4/acx_pthread.m4 @@ -0,0 +1,363 @@ +# This was retrieved from +# http://svn.0pointer.de/viewvc/trunk/common/acx_pthread.m4?revision=1277&root=avahi +# See also (perhaps for new versions?) +# http://svn.0pointer.de/viewvc/trunk/common/acx_pthread.m4?root=avahi +# +# We've rewritten the inconsistency check code (from avahi), to work +# more broadly. In particular, it no longer assumes ld accepts -zdefs. +# This caused a restructing of the code, but the functionality has only +# changed a little. + +dnl @synopsis ACX_PTHREAD([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) +dnl +dnl @summary figure out how to build C programs using POSIX threads +dnl +dnl This macro figures out how to build C programs using POSIX threads. +dnl It sets the PTHREAD_LIBS output variable to the threads library and +dnl linker flags, and the PTHREAD_CFLAGS output variable to any special +dnl C compiler flags that are needed. (The user can also force certain +dnl compiler flags/libs to be tested by setting these environment +dnl variables.) +dnl +dnl Also sets PTHREAD_CC to any special C compiler that is needed for +dnl multi-threaded programs (defaults to the value of CC otherwise). +dnl (This is necessary on AIX to use the special cc_r compiler alias.) +dnl +dnl NOTE: You are assumed to not only compile your program with these +dnl flags, but also link it with them as well. e.g. you should link +dnl with $PTHREAD_CC $CFLAGS $PTHREAD_CFLAGS $LDFLAGS ... $PTHREAD_LIBS +dnl $LIBS +dnl +dnl If you are only building threads programs, you may wish to use +dnl these variables in your default LIBS, CFLAGS, and CC: +dnl +dnl LIBS="$PTHREAD_LIBS $LIBS" +dnl CFLAGS="$CFLAGS $PTHREAD_CFLAGS" +dnl CC="$PTHREAD_CC" +dnl +dnl In addition, if the PTHREAD_CREATE_JOINABLE thread-attribute +dnl constant has a nonstandard name, defines PTHREAD_CREATE_JOINABLE to +dnl that name (e.g. PTHREAD_CREATE_UNDETACHED on AIX). +dnl +dnl ACTION-IF-FOUND is a list of shell commands to run if a threads +dnl library is found, and ACTION-IF-NOT-FOUND is a list of commands to +dnl run it if it is not found. If ACTION-IF-FOUND is not specified, the +dnl default action will define HAVE_PTHREAD. +dnl +dnl Please let the authors know if this macro fails on any platform, or +dnl if you have any other suggestions or comments. This macro was based +dnl on work by SGJ on autoconf scripts for FFTW (www.fftw.org) (with +dnl help from M. Frigo), as well as ac_pthread and hb_pthread macros +dnl posted by Alejandro Forero Cuervo to the autoconf macro repository. +dnl We are also grateful for the helpful feedback of numerous users. +dnl +dnl @category InstalledPackages +dnl @author Steven G. Johnson <stevenj@alum.mit.edu> +dnl @version 2006-05-29 +dnl @license GPLWithACException +dnl +dnl Checks for GCC shared/pthread inconsistency based on work by +dnl Marcin Owsiany <marcin@owsiany.pl> + + +AC_DEFUN([ACX_PTHREAD], [ +AC_REQUIRE([AC_CANONICAL_HOST]) +AC_LANG_SAVE +AC_LANG_C +acx_pthread_ok=no + +# We used to check for pthread.h first, but this fails if pthread.h +# requires special compiler flags (e.g. on True64 or Sequent). +# It gets checked for in the link test anyway. + +# First of all, check if the user has set any of the PTHREAD_LIBS, +# etcetera environment variables, and if threads linking works using +# them: +if test x"$PTHREAD_LIBS$PTHREAD_CFLAGS" != x; then + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + AC_MSG_CHECKING([for pthread_join in LIBS=$PTHREAD_LIBS with CFLAGS=$PTHREAD_CFLAGS]) + AC_TRY_LINK_FUNC(pthread_join, acx_pthread_ok=yes) + AC_MSG_RESULT($acx_pthread_ok) + if test x"$acx_pthread_ok" = xno; then + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" + fi + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" +fi + +# We must check for the threads library under a number of different +# names; the ordering is very important because some systems +# (e.g. DEC) have both -lpthread and -lpthreads, where one of the +# libraries is broken (non-POSIX). + +# Create a list of thread flags to try. Items starting with a "-" are +# C compiler flags, and other items are library names, except for "none" +# which indicates that we try without any flags at all, and "pthread-config" +# which is a program returning the flags for the Pth emulation library. + +acx_pthread_flags="pthreads none -Kthread -kthread lthread -pthread -pthreads -mthreads pthread --thread-safe -mt pthread-config" + +# The ordering *is* (sometimes) important. Some notes on the +# individual items follow: + +# pthreads: AIX (must check this before -lpthread) +# none: in case threads are in libc; should be tried before -Kthread and +# other compiler flags to prevent continual compiler warnings +# -Kthread: Sequent (threads in libc, but -Kthread needed for pthread.h) +# -kthread: FreeBSD kernel threads (preferred to -pthread since SMP-able) +# lthread: LinuxThreads port on FreeBSD (also preferred to -pthread) +# -pthread: Linux/gcc (kernel threads), BSD/gcc (userland threads) +# -pthreads: Solaris/gcc +# -mthreads: Mingw32/gcc, Lynx/gcc +# -mt: Sun Workshop C (may only link SunOS threads [-lthread], but it +# doesn't hurt to check since this sometimes defines pthreads too; +# also defines -D_REENTRANT) +# ... -mt is also the pthreads flag for HP/aCC +# pthread: Linux, etcetera +# --thread-safe: KAI C++ +# pthread-config: use pthread-config program (for GNU Pth library) + +case "${host_cpu}-${host_os}" in + *solaris*) + + # On Solaris (at least, for some versions), libc contains stubbed + # (non-functional) versions of the pthreads routines, so link-based + # tests will erroneously succeed. (We need to link with -pthreads/-mt/ + # -lpthread.) (The stubs are missing pthread_cleanup_push, or rather + # a function called by this macro, so we could check for that, but + # who knows whether they'll stub that too in a future libc.) So, + # we'll just look for -pthreads and -lpthread first: + + acx_pthread_flags="-pthreads pthread -mt -pthread $acx_pthread_flags" + ;; +esac + +if test x"$acx_pthread_ok" = xno; then +for flag in $acx_pthread_flags; do + + case $flag in + none) + AC_MSG_CHECKING([whether pthreads work without any flags]) + ;; + + -*) + AC_MSG_CHECKING([whether pthreads work with $flag]) + PTHREAD_CFLAGS="$flag" + ;; + + pthread-config) + AC_CHECK_PROG(acx_pthread_config, pthread-config, yes, no) + if test x"$acx_pthread_config" = xno; then continue; fi + PTHREAD_CFLAGS="`pthread-config --cflags`" + PTHREAD_LIBS="`pthread-config --ldflags` `pthread-config --libs`" + ;; + + *) + AC_MSG_CHECKING([for the pthreads library -l$flag]) + PTHREAD_LIBS="-l$flag" + ;; + esac + + save_LIBS="$LIBS" + save_CFLAGS="$CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Check for various functions. We must include pthread.h, + # since some functions may be macros. (On the Sequent, we + # need a special flag -Kthread to make this header compile.) + # We check for pthread_join because it is in -lpthread on IRIX + # while pthread_create is in libc. We check for pthread_attr_init + # due to DEC craziness with -lpthreads. We check for + # pthread_cleanup_push because it is one of the few pthread + # functions on Solaris that doesn't have a non-functional libc stub. + # We try pthread_create on general principles. + AC_TRY_LINK([#include <pthread.h>], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [acx_pthread_ok=yes]) + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + + AC_MSG_RESULT($acx_pthread_ok) + if test "x$acx_pthread_ok" = xyes; then + break; + fi + + PTHREAD_LIBS="" + PTHREAD_CFLAGS="" +done +fi + +# Various other checks: +if test "x$acx_pthread_ok" = xyes; then + save_LIBS="$LIBS" + LIBS="$PTHREAD_LIBS $LIBS" + save_CFLAGS="$CFLAGS" + CFLAGS="$CFLAGS $PTHREAD_CFLAGS" + + # Detect AIX lossage: JOINABLE attribute is called UNDETACHED. + AC_MSG_CHECKING([for joinable pthread attribute]) + attr_name=unknown + for attr in PTHREAD_CREATE_JOINABLE PTHREAD_CREATE_UNDETACHED; do + AC_TRY_LINK([#include <pthread.h>], [int attr=$attr; return attr;], + [attr_name=$attr; break]) + done + AC_MSG_RESULT($attr_name) + if test "$attr_name" != PTHREAD_CREATE_JOINABLE; then + AC_DEFINE_UNQUOTED(PTHREAD_CREATE_JOINABLE, $attr_name, + [Define to necessary symbol if this constant + uses a non-standard name on your system.]) + fi + + AC_MSG_CHECKING([if more special flags are required for pthreads]) + flag=no + case "${host_cpu}-${host_os}" in + *-aix* | *-freebsd* | *-darwin*) flag="-D_THREAD_SAFE";; + *solaris* | *-osf* | *-hpux*) flag="-D_REENTRANT";; + esac + AC_MSG_RESULT(${flag}) + if test "x$flag" != xno; then + PTHREAD_CFLAGS="$flag $PTHREAD_CFLAGS" + fi + + LIBS="$save_LIBS" + CFLAGS="$save_CFLAGS" + # More AIX lossage: must compile with xlc_r or cc_r + if test x"$GCC" != xyes; then + AC_CHECK_PROGS(PTHREAD_CC, xlc_r cc_r, ${CC}) + else + PTHREAD_CC=$CC + fi + + # The next part tries to detect GCC inconsistency with -shared on some + # architectures and systems. The problem is that in certain + # configurations, when -shared is specified, GCC "forgets" to + # internally use various flags which are still necessary. + + # + # Prepare the flags + # + save_CFLAGS="$CFLAGS" + save_LIBS="$LIBS" + save_CC="$CC" + + # Try with the flags determined by the earlier checks. + # + # -Wl,-z,defs forces link-time symbol resolution, so that the + # linking checks with -shared actually have any value + # + # FIXME: -fPIC is required for -shared on many architectures, + # so we specify it here, but the right way would probably be to + # properly detect whether it is actually required. + CFLAGS="-shared -fPIC -Wl,-z,defs $CFLAGS $PTHREAD_CFLAGS" + LIBS="$PTHREAD_LIBS $LIBS" + CC="$PTHREAD_CC" + + # In order not to create several levels of indentation, we test + # the value of "$done" until we find the cure or run out of ideas. + done="no" + + # First, make sure the CFLAGS we added are actually accepted by our + # compiler. If not (and OS X's ld, for instance, does not accept -z), + # then we can't do this test. + if test x"$done" = xno; then + AC_MSG_CHECKING([whether to check for GCC pthread/shared inconsistencies]) + AC_TRY_LINK(,, , [done=yes]) + + if test "x$done" = xyes ; then + AC_MSG_RESULT([no]) + else + AC_MSG_RESULT([yes]) + fi + fi + + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -pthread is sufficient with -shared]) + AC_TRY_LINK([#include <pthread.h>], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + fi + fi + + # + # Linux gcc on some architectures such as mips/mipsel forgets + # about -lpthread + # + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -lpthread fixes that]) + LIBS="-lpthread $PTHREAD_LIBS $save_LIBS" + AC_TRY_LINK([#include <pthread.h>], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + PTHREAD_LIBS="-lpthread $PTHREAD_LIBS" + else + AC_MSG_RESULT([no]) + fi + fi + # + # FreeBSD 4.10 gcc forgets to use -lc_r instead of -lc + # + if test x"$done" = xno; then + AC_MSG_CHECKING([whether -lc_r fixes that]) + LIBS="-lc_r $PTHREAD_LIBS $save_LIBS" + AC_TRY_LINK([#include <pthread.h>], + [pthread_t th; pthread_join(th, 0); + pthread_attr_init(0); pthread_cleanup_push(0, 0); + pthread_create(0,0,0,0); pthread_cleanup_pop(0); ], + [done=yes]) + + if test "x$done" = xyes; then + AC_MSG_RESULT([yes]) + PTHREAD_LIBS="-lc_r $PTHREAD_LIBS" + else + AC_MSG_RESULT([no]) + fi + fi + if test x"$done" = xno; then + # OK, we have run out of ideas + AC_MSG_WARN([Impossible to determine how to use pthreads with shared libraries]) + + # so it's not safe to assume that we may use pthreads + acx_pthread_ok=no + fi + + CFLAGS="$save_CFLAGS" + LIBS="$save_LIBS" + CC="$save_CC" +else + PTHREAD_CC="$CC" +fi + +AC_SUBST(PTHREAD_LIBS) +AC_SUBST(PTHREAD_CFLAGS) +AC_SUBST(PTHREAD_CC) + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x"$acx_pthread_ok" = xyes; then + ifelse([$1],,AC_DEFINE(HAVE_PTHREAD,1,[Define if you have POSIX threads libraries and header files.]),[$1]) + : +else + acx_pthread_ok=no + $2 +fi +AC_LANG_RESTORE +])dnl ACX_PTHREAD diff --git a/third_party/glog/m4/google_namespace.m4 b/third_party/glog/m4/google_namespace.m4 new file mode 100644 index 000000000000..79e0a6d42a6b --- /dev/null +++ b/third_party/glog/m4/google_namespace.m4 @@ -0,0 +1,36 @@ +# Allow users to override the namespace we define our application's classes in +# Arg $1 is the default namespace to use if --enable-namespace isn't present. + +# In general, $1 should be 'google', so we put all our exported symbols in a +# unique namespace that is not likely to conflict with anyone else. However, +# when it makes sense -- for instance, when publishing stl-like code -- you +# may want to go with a different default, like 'std'. + +AC_DEFUN([AC_DEFINE_GOOGLE_NAMESPACE], + [google_namespace_default=[$1] + AC_ARG_ENABLE(namespace, [ --enable-namespace=FOO to define these Google + classes in the FOO namespace. --disable-namespace + to define them in the global namespace. Default + is to define them in namespace $1.], + [case "$enableval" in + yes) google_namespace="$google_namespace_default" ;; + no) google_namespace="" ;; + *) google_namespace="$enableval" ;; + esac], + [google_namespace="$google_namespace_default"]) + if test -n "$google_namespace"; then + ac_google_namespace="$google_namespace" + ac_google_start_namespace="namespace $google_namespace {" + ac_google_end_namespace="}" + else + ac_google_namespace="" + ac_google_start_namespace="" + ac_google_end_namespace="" + fi + AC_DEFINE_UNQUOTED(GOOGLE_NAMESPACE, $ac_google_namespace, + Namespace for Google classes) + AC_DEFINE_UNQUOTED(_START_GOOGLE_NAMESPACE_, $ac_google_start_namespace, + Puts following code inside the Google namespace) + AC_DEFINE_UNQUOTED(_END_GOOGLE_NAMESPACE_, $ac_google_end_namespace, + Stops putting the code inside the Google namespace) +]) diff --git a/third_party/glog/m4/ltsugar.m4 b/third_party/glog/m4/ltsugar.m4 new file mode 100644 index 000000000000..9000a057d31d --- /dev/null +++ b/third_party/glog/m4/ltsugar.m4 @@ -0,0 +1,123 @@ +# ltsugar.m4 -- libtool m4 base layer. -*-Autoconf-*- +# +# Copyright (C) 2004, 2005, 2007, 2008 Free Software Foundation, Inc. +# Written by Gary V. Vaughan, 2004 +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +# serial 6 ltsugar.m4 + +# This is to help aclocal find these macros, as it can't see m4_define. +AC_DEFUN([LTSUGAR_VERSION], [m4_if([0.1])]) + + +# lt_join(SEP, ARG1, [ARG2...]) +# ----------------------------- +# Produce ARG1SEPARG2...SEPARGn, omitting [] arguments and their +# associated separator. +# Needed until we can rely on m4_join from Autoconf 2.62, since all earlier +# versions in m4sugar had bugs. +m4_define([lt_join], +[m4_if([$#], [1], [], + [$#], [2], [[$2]], + [m4_if([$2], [], [], [[$2]_])$0([$1], m4_shift(m4_shift($@)))])]) +m4_define([_lt_join], +[m4_if([$#$2], [2], [], + [m4_if([$2], [], [], [[$1$2]])$0([$1], m4_shift(m4_shift($@)))])]) + + +# lt_car(LIST) +# lt_cdr(LIST) +# ------------ +# Manipulate m4 lists. +# These macros are necessary as long as will still need to support +# Autoconf-2.59 which quotes differently. +m4_define([lt_car], [[$1]]) +m4_define([lt_cdr], +[m4_if([$#], 0, [m4_fatal([$0: cannot be called without arguments])], + [$#], 1, [], + [m4_dquote(m4_shift($@))])]) +m4_define([lt_unquote], $1) + + +# lt_append(MACRO-NAME, STRING, [SEPARATOR]) +# ------------------------------------------ +# Redefine MACRO-NAME to hold its former content plus `SEPARATOR'`STRING'. +# Note that neither SEPARATOR nor STRING are expanded; they are appended +# to MACRO-NAME as is (leaving the expansion for when MACRO-NAME is invoked). +# No SEPARATOR is output if MACRO-NAME was previously undefined (different +# than defined and empty). +# +# This macro is needed until we can rely on Autoconf 2.62, since earlier +# versions of m4sugar mistakenly expanded SEPARATOR but not STRING. +m4_define([lt_append], +[m4_define([$1], + m4_ifdef([$1], [m4_defn([$1])[$3]])[$2])]) + + + +# lt_combine(SEP, PREFIX-LIST, INFIX, SUFFIX1, [SUFFIX2...]) +# ---------------------------------------------------------- +# Produce a SEP delimited list of all paired combinations of elements of +# PREFIX-LIST with SUFFIX1 through SUFFIXn. Each element of the list +# has the form PREFIXmINFIXSUFFIXn. +# Needed until we can rely on m4_combine added in Autoconf 2.62. +m4_define([lt_combine], +[m4_if(m4_eval([$# > 3]), [1], + [m4_pushdef([_Lt_sep], [m4_define([_Lt_sep], m4_defn([lt_car]))])]]dnl +[[m4_foreach([_Lt_prefix], [$2], + [m4_foreach([_Lt_suffix], + ]m4_dquote(m4_dquote(m4_shift(m4_shift(m4_shift($@)))))[, + [_Lt_sep([$1])[]m4_defn([_Lt_prefix])[$3]m4_defn([_Lt_suffix])])])])]) + + +# lt_if_append_uniq(MACRO-NAME, VARNAME, [SEPARATOR], [UNIQ], [NOT-UNIQ]) +# ----------------------------------------------------------------------- +# Iff MACRO-NAME does not yet contain VARNAME, then append it (delimited +# by SEPARATOR if supplied) and expand UNIQ, else NOT-UNIQ. +m4_define([lt_if_append_uniq], +[m4_ifdef([$1], + [m4_if(m4_index([$3]m4_defn([$1])[$3], [$3$2$3]), [-1], + [lt_append([$1], [$2], [$3])$4], + [$5])], + [lt_append([$1], [$2], [$3])$4])]) + + +# lt_dict_add(DICT, KEY, VALUE) +# ----------------------------- +m4_define([lt_dict_add], +[m4_define([$1($2)], [$3])]) + + +# lt_dict_add_subkey(DICT, KEY, SUBKEY, VALUE) +# -------------------------------------------- +m4_define([lt_dict_add_subkey], +[m4_define([$1($2:$3)], [$4])]) + + +# lt_dict_fetch(DICT, KEY, [SUBKEY]) +# ---------------------------------- +m4_define([lt_dict_fetch], +[m4_ifval([$3], + m4_ifdef([$1($2:$3)], [m4_defn([$1($2:$3)])]), + m4_ifdef([$1($2)], [m4_defn([$1($2)])]))]) + + +# lt_if_dict_fetch(DICT, KEY, [SUBKEY], VALUE, IF-TRUE, [IF-FALSE]) +# ----------------------------------------------------------------- +m4_define([lt_if_dict_fetch], +[m4_if(lt_dict_fetch([$1], [$2], [$3]), [$4], + [$5], + [$6])]) + + +# lt_dict_filter(DICT, [SUBKEY], VALUE, [SEPARATOR], KEY, [...]) +# -------------------------------------------------------------- +m4_define([lt_dict_filter], +[m4_if([$5], [], [], + [lt_join(m4_quote(m4_default([$4], [[, ]])), + lt_unquote(m4_split(m4_normalize(m4_foreach(_Lt_key, lt_car([m4_shiftn(4, $@)]), + [lt_if_dict_fetch([$1], _Lt_key, [$2], [$3], [_Lt_key ])])))))])[]dnl +]) diff --git a/third_party/glog/m4/lt~obsolete.m4 b/third_party/glog/m4/lt~obsolete.m4 new file mode 100644 index 000000000000..c573da90c5cc --- /dev/null +++ b/third_party/glog/m4/lt~obsolete.m4 @@ -0,0 +1,98 @@ +# lt~obsolete.m4 -- aclocal satisfying obsolete definitions. -*-Autoconf-*- +# +# Copyright (C) 2004, 2005, 2007, 2009 Free Software Foundation, Inc. +# Written by Scott James Remnant, 2004. +# +# This file is free software; the Free Software Foundation gives +# unlimited permission to copy and/or distribute it, with or without +# modifications, as long as this notice is preserved. + +# serial 5 lt~obsolete.m4 + +# These exist entirely to fool aclocal when bootstrapping libtool. +# +# In the past libtool.m4 has provided macros via AC_DEFUN (or AU_DEFUN) +# which have later been changed to m4_define as they aren't part of the +# exported API, or moved to Autoconf or Automake where they belong. +# +# The trouble is, aclocal is a bit thick. It'll see the old AC_DEFUN +# in /usr/share/aclocal/libtool.m4 and remember it, then when it sees us +# using a macro with the same name in our local m4/libtool.m4 it'll +# pull the old libtool.m4 in (it doesn't see our shiny new m4_define +# and doesn't know about Autoconf macros at all.) +# +# So we provide this file, which has a silly filename so it's always +# included after everything else. This provides aclocal with the +# AC_DEFUNs it wants, but when m4 processes it, it doesn't do anything +# because those macros already exist, or will be overwritten later. +# We use AC_DEFUN over AU_DEFUN for compatibility with aclocal-1.6. +# +# Anytime we withdraw an AC_DEFUN or AU_DEFUN, remember to add it here. +# Yes, that means every name once taken will need to remain here until +# we give up compatibility with versions before 1.7, at which point +# we need to keep only those names which we still refer to. + +# This is to help aclocal find these macros, as it can't see m4_define. +AC_DEFUN([LTOBSOLETE_VERSION], [m4_if([1])]) + +m4_ifndef([AC_LIBTOOL_LINKER_OPTION], [AC_DEFUN([AC_LIBTOOL_LINKER_OPTION])]) +m4_ifndef([AC_PROG_EGREP], [AC_DEFUN([AC_PROG_EGREP])]) +m4_ifndef([_LT_AC_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_AC_PROG_ECHO_BACKSLASH])]) +m4_ifndef([_LT_AC_SHELL_INIT], [AC_DEFUN([_LT_AC_SHELL_INIT])]) +m4_ifndef([_LT_AC_SYS_LIBPATH_AIX], [AC_DEFUN([_LT_AC_SYS_LIBPATH_AIX])]) +m4_ifndef([_LT_PROG_LTMAIN], [AC_DEFUN([_LT_PROG_LTMAIN])]) +m4_ifndef([_LT_AC_TAGVAR], [AC_DEFUN([_LT_AC_TAGVAR])]) +m4_ifndef([AC_LTDL_ENABLE_INSTALL], [AC_DEFUN([AC_LTDL_ENABLE_INSTALL])]) +m4_ifndef([AC_LTDL_PREOPEN], [AC_DEFUN([AC_LTDL_PREOPEN])]) +m4_ifndef([_LT_AC_SYS_COMPILER], [AC_DEFUN([_LT_AC_SYS_COMPILER])]) +m4_ifndef([_LT_AC_LOCK], [AC_DEFUN([_LT_AC_LOCK])]) +m4_ifndef([AC_LIBTOOL_SYS_OLD_ARCHIVE], [AC_DEFUN([AC_LIBTOOL_SYS_OLD_ARCHIVE])]) +m4_ifndef([_LT_AC_TRY_DLOPEN_SELF], [AC_DEFUN([_LT_AC_TRY_DLOPEN_SELF])]) +m4_ifndef([AC_LIBTOOL_PROG_CC_C_O], [AC_DEFUN([AC_LIBTOOL_PROG_CC_C_O])]) +m4_ifndef([AC_LIBTOOL_SYS_HARD_LINK_LOCKS], [AC_DEFUN([AC_LIBTOOL_SYS_HARD_LINK_LOCKS])]) +m4_ifndef([AC_LIBTOOL_OBJDIR], [AC_DEFUN([AC_LIBTOOL_OBJDIR])]) +m4_ifndef([AC_LTDL_OBJDIR], [AC_DEFUN([AC_LTDL_OBJDIR])]) +m4_ifndef([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH], [AC_DEFUN([AC_LIBTOOL_PROG_LD_HARDCODE_LIBPATH])]) +m4_ifndef([AC_LIBTOOL_SYS_LIB_STRIP], [AC_DEFUN([AC_LIBTOOL_SYS_LIB_STRIP])]) +m4_ifndef([AC_PATH_MAGIC], [AC_DEFUN([AC_PATH_MAGIC])]) +m4_ifndef([AC_PROG_LD_GNU], [AC_DEFUN([AC_PROG_LD_GNU])]) +m4_ifndef([AC_PROG_LD_RELOAD_FLAG], [AC_DEFUN([AC_PROG_LD_RELOAD_FLAG])]) +m4_ifndef([AC_DEPLIBS_CHECK_METHOD], [AC_DEFUN([AC_DEPLIBS_CHECK_METHOD])]) +m4_ifndef([AC_LIBTOOL_PROG_COMPILER_NO_RTTI], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_NO_RTTI])]) +m4_ifndef([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE], [AC_DEFUN([AC_LIBTOOL_SYS_GLOBAL_SYMBOL_PIPE])]) +m4_ifndef([AC_LIBTOOL_PROG_COMPILER_PIC], [AC_DEFUN([AC_LIBTOOL_PROG_COMPILER_PIC])]) +m4_ifndef([AC_LIBTOOL_PROG_LD_SHLIBS], [AC_DEFUN([AC_LIBTOOL_PROG_LD_SHLIBS])]) +m4_ifndef([AC_LIBTOOL_POSTDEP_PREDEP], [AC_DEFUN([AC_LIBTOOL_POSTDEP_PREDEP])]) +m4_ifndef([LT_AC_PROG_EGREP], [AC_DEFUN([LT_AC_PROG_EGREP])]) +m4_ifndef([LT_AC_PROG_SED], [AC_DEFUN([LT_AC_PROG_SED])]) +m4_ifndef([_LT_CC_BASENAME], [AC_DEFUN([_LT_CC_BASENAME])]) +m4_ifndef([_LT_COMPILER_BOILERPLATE], [AC_DEFUN([_LT_COMPILER_BOILERPLATE])]) +m4_ifndef([_LT_LINKER_BOILERPLATE], [AC_DEFUN([_LT_LINKER_BOILERPLATE])]) +m4_ifndef([_AC_PROG_LIBTOOL], [AC_DEFUN([_AC_PROG_LIBTOOL])]) +m4_ifndef([AC_LIBTOOL_SETUP], [AC_DEFUN([AC_LIBTOOL_SETUP])]) +m4_ifndef([_LT_AC_CHECK_DLFCN], [AC_DEFUN([_LT_AC_CHECK_DLFCN])]) +m4_ifndef([AC_LIBTOOL_SYS_DYNAMIC_LINKER], [AC_DEFUN([AC_LIBTOOL_SYS_DYNAMIC_LINKER])]) +m4_ifndef([_LT_AC_TAGCONFIG], [AC_DEFUN([_LT_AC_TAGCONFIG])]) +m4_ifndef([AC_DISABLE_FAST_INSTALL], [AC_DEFUN([AC_DISABLE_FAST_INSTALL])]) +m4_ifndef([_LT_AC_LANG_CXX], [AC_DEFUN([_LT_AC_LANG_CXX])]) +m4_ifndef([_LT_AC_LANG_F77], [AC_DEFUN([_LT_AC_LANG_F77])]) +m4_ifndef([_LT_AC_LANG_GCJ], [AC_DEFUN([_LT_AC_LANG_GCJ])]) +m4_ifndef([AC_LIBTOOL_LANG_C_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_C_CONFIG])]) +m4_ifndef([_LT_AC_LANG_C_CONFIG], [AC_DEFUN([_LT_AC_LANG_C_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_CXX_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_CXX_CONFIG])]) +m4_ifndef([_LT_AC_LANG_CXX_CONFIG], [AC_DEFUN([_LT_AC_LANG_CXX_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_F77_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_F77_CONFIG])]) +m4_ifndef([_LT_AC_LANG_F77_CONFIG], [AC_DEFUN([_LT_AC_LANG_F77_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_GCJ_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_GCJ_CONFIG])]) +m4_ifndef([_LT_AC_LANG_GCJ_CONFIG], [AC_DEFUN([_LT_AC_LANG_GCJ_CONFIG])]) +m4_ifndef([AC_LIBTOOL_LANG_RC_CONFIG], [AC_DEFUN([AC_LIBTOOL_LANG_RC_CONFIG])]) +m4_ifndef([_LT_AC_LANG_RC_CONFIG], [AC_DEFUN([_LT_AC_LANG_RC_CONFIG])]) +m4_ifndef([AC_LIBTOOL_CONFIG], [AC_DEFUN([AC_LIBTOOL_CONFIG])]) +m4_ifndef([_LT_AC_FILE_LTDLL_C], [AC_DEFUN([_LT_AC_FILE_LTDLL_C])]) +m4_ifndef([_LT_REQUIRED_DARWIN_CHECKS], [AC_DEFUN([_LT_REQUIRED_DARWIN_CHECKS])]) +m4_ifndef([_LT_AC_PROG_CXXCPP], [AC_DEFUN([_LT_AC_PROG_CXXCPP])]) +m4_ifndef([_LT_PREPARE_SED_QUOTE_VARS], [AC_DEFUN([_LT_PREPARE_SED_QUOTE_VARS])]) +m4_ifndef([_LT_PROG_ECHO_BACKSLASH], [AC_DEFUN([_LT_PROG_ECHO_BACKSLASH])]) +m4_ifndef([_LT_PROG_F77], [AC_DEFUN([_LT_PROG_F77])]) +m4_ifndef([_LT_PROG_FC], [AC_DEFUN([_LT_PROG_FC])]) +m4_ifndef([_LT_PROG_CXX], [AC_DEFUN([_LT_PROG_CXX])]) diff --git a/third_party/glog/m4/namespaces.m4 b/third_party/glog/m4/namespaces.m4 new file mode 100644 index 000000000000..d78dbe4cc2e4 --- /dev/null +++ b/third_party/glog/m4/namespaces.m4 @@ -0,0 +1,15 @@ +# Checks whether the compiler implements namespaces +AC_DEFUN([AC_CXX_NAMESPACES], + [AC_CACHE_CHECK(whether the compiler implements namespaces, + ac_cv_cxx_namespaces, + [AC_LANG_SAVE + AC_LANG_CPLUSPLUS + AC_TRY_COMPILE([namespace Outer { + namespace Inner { int i = 0; }}], + [using namespace Outer::Inner; return i;], + ac_cv_cxx_namespaces=yes, + ac_cv_cxx_namespaces=no) + AC_LANG_RESTORE]) + if test "$ac_cv_cxx_namespaces" = yes; then + AC_DEFINE(HAVE_NAMESPACES, 1, [define if the compiler implements namespaces]) + fi]) diff --git a/third_party/glog/m4/pc_from_ucontext.m4 b/third_party/glog/m4/pc_from_ucontext.m4 new file mode 100644 index 000000000000..daffddb536fd --- /dev/null +++ b/third_party/glog/m4/pc_from_ucontext.m4 @@ -0,0 +1,71 @@ +# We want to access the "PC" (Program Counter) register from a struct +# ucontext. Every system has its own way of doing that. We try all the +# possibilities we know about. Note REG_PC should come first (REG_RIP +# is also defined on solaris, but does the wrong thing). + +# OpenBSD doesn't have ucontext.h, but we can get PC from ucontext_t +# by using signal.h. + +# The first argument of AC_PC_FROM_UCONTEXT will be invoked when we +# cannot find a way to obtain PC from ucontext. + +AC_DEFUN([AC_PC_FROM_UCONTEXT], + [AC_CHECK_HEADERS(ucontext.h) + AC_CHECK_HEADERS(sys/ucontext.h) # ucontext on OS X 10.6 (at least) + AC_MSG_CHECKING([how to access the program counter from a struct ucontext]) + pc_fields=" uc_mcontext.gregs[[REG_PC]]" # Solaris x86 (32 + 64 bit) + pc_fields="$pc_fields uc_mcontext.gregs[[REG_EIP]]" # Linux (i386) + pc_fields="$pc_fields uc_mcontext.gregs[[REG_RIP]]" # Linux (x86_64) + pc_fields="$pc_fields uc_mcontext.sc_ip" # Linux (ia64) + pc_fields="$pc_fields uc_mcontext.uc_regs->gregs[[PT_NIP]]" # Linux (ppc) + pc_fields="$pc_fields uc_mcontext.gregs[[R15]]" # Linux (arm old [untested]) + pc_fields="$pc_fields uc_mcontext.arm_pc" # Linux (arm new [untested]) + pc_fields="$pc_fields uc_mcontext.mc_eip" # FreeBSD (i386) + pc_fields="$pc_fields uc_mcontext.mc_rip" # FreeBSD (x86_64 [untested]) + pc_fields="$pc_fields uc_mcontext.__gregs[[_REG_EIP]]" # NetBSD (i386) + pc_fields="$pc_fields uc_mcontext.__gregs[[_REG_RIP]]" # NetBSD (x86_64) + pc_fields="$pc_fields uc_mcontext->ss.eip" # OS X (i386, <=10.4) + pc_fields="$pc_fields uc_mcontext->__ss.__eip" # OS X (i386, >=10.5) + pc_fields="$pc_fields uc_mcontext->ss.rip" # OS X (x86_64) + pc_fields="$pc_fields uc_mcontext->__ss.__rip" # OS X (>=10.5 [untested]) + pc_fields="$pc_fields uc_mcontext->ss.srr0" # OS X (ppc, ppc64 [untested]) + pc_fields="$pc_fields uc_mcontext->__ss.__srr0" # OS X (>=10.5 [untested]) + pc_field_found=false + for pc_field in $pc_fields; do + if ! $pc_field_found; then + if test "x$ac_cv_header_sys_ucontext_h" = xyes; then + AC_TRY_COMPILE([#define _GNU_SOURCE 1 + #include <sys/ucontext.h>], + [ucontext_t u; return u.$pc_field == 0;], + AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field, + How to access the PC from a struct ucontext) + AC_MSG_RESULT([$pc_field]) + pc_field_found=true) + else + AC_TRY_COMPILE([#define _GNU_SOURCE 1 + #include <ucontext.h>], + [ucontext_t u; return u.$pc_field == 0;], + AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field, + How to access the PC from a struct ucontext) + AC_MSG_RESULT([$pc_field]) + pc_field_found=true) + fi + fi + done + if ! $pc_field_found; then + pc_fields=" sc_eip" # OpenBSD (i386) + pc_fields="$pc_fields sc_rip" # OpenBSD (x86_64) + for pc_field in $pc_fields; do + if ! $pc_field_found; then + AC_TRY_COMPILE([#include <signal.h>], + [ucontext_t u; return u.$pc_field == 0;], + AC_DEFINE_UNQUOTED(PC_FROM_UCONTEXT, $pc_field, + How to access the PC from a struct ucontext) + AC_MSG_RESULT([$pc_field]) + pc_field_found=true) + fi + done + fi + if ! $pc_field_found; then + [$1] + fi]) diff --git a/third_party/glog/m4/stl_namespace.m4 b/third_party/glog/m4/stl_namespace.m4 new file mode 100644 index 000000000000..989ad80696a3 --- /dev/null +++ b/third_party/glog/m4/stl_namespace.m4 @@ -0,0 +1,25 @@ +# We check what namespace stl code like vector expects to be executed in + +AC_DEFUN([AC_CXX_STL_NAMESPACE], + [AC_CACHE_CHECK( + what namespace STL code is in, + ac_cv_cxx_stl_namespace, + [AC_REQUIRE([AC_CXX_NAMESPACES]) + AC_LANG_SAVE + AC_LANG_CPLUSPLUS + AC_TRY_COMPILE([#include <vector>], + [vector<int> t; return 0;], + ac_cv_cxx_stl_namespace=none) + AC_TRY_COMPILE([#include <vector>], + [std::vector<int> t; return 0;], + ac_cv_cxx_stl_namespace=std) + AC_LANG_RESTORE]) + if test "$ac_cv_cxx_stl_namespace" = none; then + AC_DEFINE(STL_NAMESPACE,, + [the namespace where STL code like vector<> is defined]) + fi + if test "$ac_cv_cxx_stl_namespace" = std; then + AC_DEFINE(STL_NAMESPACE,std, + [the namespace where STL code like vector<> is defined]) + fi +]) diff --git a/third_party/glog/m4/using_operator.m4 b/third_party/glog/m4/using_operator.m4 new file mode 100644 index 000000000000..95a9951eb346 --- /dev/null +++ b/third_party/glog/m4/using_operator.m4 @@ -0,0 +1,15 @@ +AC_DEFUN([AC_CXX_USING_OPERATOR], + [AC_CACHE_CHECK( + whether compiler supports using ::operator<<, + ac_cv_cxx_using_operator, + [AC_LANG_SAVE + AC_LANG_CPLUSPLUS + AC_TRY_COMPILE([#include <iostream> + std::ostream& operator<<(std::ostream&, struct s);], + [using ::operator<<; return 0;], + ac_cv_cxx_using_operator=1, + ac_cv_cxx_using_operator=0) + AC_LANG_RESTORE]) + if test "$ac_cv_cxx_using_operator" = 1; then + AC_DEFINE(HAVE_USING_OPERATOR, 1, [define if the compiler supports using expression for operator]) + fi]) diff --git a/third_party/glog/packages/deb.sh b/third_party/glog/packages/deb.sh new file mode 100755 index 000000000000..a1cdf321ef58 --- /dev/null +++ b/third_party/glog/packages/deb.sh @@ -0,0 +1,73 @@ +#!/bin/bash -e + +# This takes one commandline argument, the name of the package. If no +# name is given, then we'll end up just using the name associated with +# an arbitrary .tar.gz file in the rootdir. That's fine: there's probably +# only one. +# +# Run this from the 'packages' directory, just under rootdir + +## Set LIB to lib if exporting a library, empty-string else +LIB= +#LIB=lib + +PACKAGE="$1" +VERSION="$2" + +# We can only build Debian packages, if the Debian build tools are installed +if [ \! -x /usr/bin/debuild ]; then + echo "Cannot find /usr/bin/debuild. Not building Debian packages." 1>&2 + exit 0 +fi + +# Double-check we're in the packages directory, just under rootdir +if [ \! -r ../Makefile -a \! -r ../INSTALL ]; then + echo "Must run $0 in the 'packages' directory, under the root directory." 1>&2 + echo "Also, you must run \"make dist\" before running this script." 1>&2 + exit 0 +fi + +# Find the top directory for this package +topdir="${PWD%/*}" + +# Find the tar archive built by "make dist" +archive="$PACKAGE-$VERSION" +if [ -z "${archive}" ]; then + echo "Cannot find ../$PACKAGE*.tar.gz. Run \"make dist\" first." 1>&2 + exit 0 +fi + +# Create a pristine directory for building the Debian package files +trap 'rm -rf '`pwd`/tmp'; exit $?' EXIT SIGHUP SIGINT SIGTERM + +rm -rf tmp +mkdir -p tmp +cd tmp + +package="google-glog_$VERSION" + +# Debian has very specific requirements about the naming of build +# directories, and tar archives. It also wants to write all generated +# packages to the parent of the source directory. We accommodate these +# requirements by building directly from the tar file. +ln -s "${topdir}/${archive}.tar.gz" "${LIB}${package}.orig.tar.gz" +tar zfx "${LIB}${package}.orig.tar.gz" +mv "${archive}" "${LIB}${package}" +cd "${LIB}${package}" +# This is one of those 'specific requirements': where the deb control files live +cp -a "packages/deb" "debian" + +# Now, we can call Debian's standard build tool +debuild -uc -us +cd ../.. # get back to the original top-level dir + +# We'll put the result in a subdirectory that's named after the OS version +# we've made this .deb file for. +destdir="debian-$(cat /etc/debian_version 2>/dev/null || echo UNKNOWN)" + +rm -rf "$destdir" +mkdir -p "$destdir" +mv $(find tmp -mindepth 1 -maxdepth 1 -type f) "$destdir" + +echo +echo "The Debian package files are located in $PWD/$destdir" diff --git a/third_party/glog/packages/deb/README b/third_party/glog/packages/deb/README new file mode 100644 index 000000000000..57becfda757e --- /dev/null +++ b/third_party/glog/packages/deb/README @@ -0,0 +1,7 @@ +The list of files here isn't complete. For a step-by-step guide on +how to set this package up correctly, check out + http://www.debian.org/doc/maint-guide/ + +Most of the files that are in this directory are boilerplate. +However, you may need to change the list of binary-arch dependencies +in 'rules'. diff --git a/third_party/glog/packages/deb/changelog b/third_party/glog/packages/deb/changelog new file mode 100644 index 000000000000..4cbed1bd0cde --- /dev/null +++ b/third_party/glog/packages/deb/changelog @@ -0,0 +1,71 @@ +google-glog (0.4.0-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Tue, 22 Jan 2019 21:00:26 +0900 + +google-glog (0.3.5-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Tue, 09 May 2017 16:22:12 +0900 + +google-glog (0.3.4-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Tue, 10 Mar 2015 12:02:20 +0900 + +google-glog (0.3.3-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Fri, 01 Feb 2012 14:54:14 +0900 + +google-glog (0.3.2-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Thu, 12 Jan 2012 17:36:14 +0900 + +google-glog (0.3.1-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Tue, 15 Jun 2010 13:50:47 +0900 + +google-glog (0.3-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Thu, 30 Jul 2009 21:31:35 +0900 + +google-glog (0.2.1-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Fri, 10 Apr 2009 15:24:17 +0900 + +google-glog (0.2-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Fri, 23 Jan 2009 03:14:29 +0900 + +google-glog (0.1.2-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Tue, 18 Nov 2008 20:37:00 +0900 + +google-glog (0.1.1-1) unstable; urgency=low + + * New upstream release. + + -- Google Inc. <opensource@google.com> Wed, 15 Oct 2008 20:38:19 +0900 + +google-glog (0.1-1) unstable; urgency=low + + * Initial release. + + -- Google Inc. <opensource@google.com> Sat, 10 May 2008 12:31:10 +0900 diff --git a/third_party/glog/packages/deb/compat b/third_party/glog/packages/deb/compat new file mode 100644 index 000000000000..b8626c4cff28 --- /dev/null +++ b/third_party/glog/packages/deb/compat @@ -0,0 +1 @@ +4 diff --git a/third_party/glog/packages/deb/control b/third_party/glog/packages/deb/control new file mode 100644 index 000000000000..110a72cb3560 --- /dev/null +++ b/third_party/glog/packages/deb/control @@ -0,0 +1,23 @@ +Source: google-glog +Priority: optional +Maintainer: Google Inc. <opensource@google.com> +Build-Depends: debhelper (>= 4.0.0), binutils +Standards-Version: 3.6.1 + +Package: libgoogle-glog-dev +Section: libdevel +Architecture: any +Depends: libgoogle-glog0 (= ${Source-Version}) +Description: a library that implements application-level logging. + This library provides logging APIs based on C++-style streams and + various helper macros. The devel package contains static and debug + libraries and header files for developing applications that use the + google-glog package. + +Package: libgoogle-glog0 +Section: libs +Architecture: any +Depends: ${shlibs:Depends} +Description: a library that implements application-level logging. + This library provides logging APIs based on C++-style streams and + various helper macros. diff --git a/third_party/glog/packages/deb/copyright b/third_party/glog/packages/deb/copyright new file mode 100644 index 000000000000..b2ce8e8f023c --- /dev/null +++ b/third_party/glog/packages/deb/copyright @@ -0,0 +1,35 @@ +This package was debianized by Google Inc. <opensource@google.com> on +13 June 2008. + +It was downloaded from https://github.com/google/glog + +Upstream Author: opensource@google.com + +Copyright (c) 2008, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/glog/packages/deb/docs b/third_party/glog/packages/deb/docs new file mode 100644 index 000000000000..6da25d4d3113 --- /dev/null +++ b/third_party/glog/packages/deb/docs @@ -0,0 +1,7 @@ +AUTHORS +COPYING +ChangeLog +INSTALL +README.md +doc/designstyle.css +doc/glog.html diff --git a/third_party/glog/packages/deb/libgoogle-glog-dev.dirs b/third_party/glog/packages/deb/libgoogle-glog-dev.dirs new file mode 100644 index 000000000000..bddfbf5c112b --- /dev/null +++ b/third_party/glog/packages/deb/libgoogle-glog-dev.dirs @@ -0,0 +1,4 @@ +usr/lib +usr/lib/pkgconfig +usr/include +usr/include/glog diff --git a/third_party/glog/packages/deb/libgoogle-glog-dev.install b/third_party/glog/packages/deb/libgoogle-glog-dev.install new file mode 100644 index 000000000000..9c61e86eaf6f --- /dev/null +++ b/third_party/glog/packages/deb/libgoogle-glog-dev.install @@ -0,0 +1,10 @@ +usr/include/glog/* +usr/lib/lib*.so +usr/lib/lib*.a +usr/lib/*.la +usr/lib/pkgconfig/* +debian/tmp/usr/include/glog/* +debian/tmp/usr/lib/lib*.so +debian/tmp/usr/lib/lib*.a +debian/tmp/usr/lib/*.la +debian/tmp/usr/lib/pkgconfig/* diff --git a/third_party/glog/packages/deb/libgoogle-glog0.dirs b/third_party/glog/packages/deb/libgoogle-glog0.dirs new file mode 100644 index 000000000000..68457717bd8e --- /dev/null +++ b/third_party/glog/packages/deb/libgoogle-glog0.dirs @@ -0,0 +1 @@ +usr/lib diff --git a/third_party/glog/packages/deb/libgoogle-glog0.install b/third_party/glog/packages/deb/libgoogle-glog0.install new file mode 100644 index 000000000000..704ea87ab754 --- /dev/null +++ b/third_party/glog/packages/deb/libgoogle-glog0.install @@ -0,0 +1,2 @@ +usr/lib/lib*.so.* +debian/tmp/usr/lib/lib*.so.* diff --git a/third_party/glog/packages/deb/rules b/third_party/glog/packages/deb/rules new file mode 100755 index 000000000000..f520befd259d --- /dev/null +++ b/third_party/glog/packages/deb/rules @@ -0,0 +1,117 @@ +#!/usr/bin/make -f +# -*- makefile -*- +# Sample debian/rules that uses debhelper. +# This file was originally written by Joey Hess and Craig Small. +# As a special exception, when this file is copied by dh-make into a +# dh-make output file, you may use that output file without restriction. +# This special exception was added by Craig Small in version 0.37 of dh-make. + +# Uncomment this to turn on verbose mode. +#export DH_VERBOSE=1 + + +# These are used for cross-compiling and for saving the configure script +# from having to guess our platform (since we know it already) +DEB_HOST_GNU_TYPE ?= $(shell dpkg-architecture -qDEB_HOST_GNU_TYPE) +DEB_BUILD_GNU_TYPE ?= $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE) + + +CFLAGS = -Wall -g + +ifneq (,$(findstring noopt,$(DEB_BUILD_OPTIONS))) + CFLAGS += -O0 +else + CFLAGS += -O2 +endif +ifeq (,$(findstring nostrip,$(DEB_BUILD_OPTIONS))) + INSTALL_PROGRAM += -s +endif + +# shared library versions, option 1 +#version=2.0.5 +#major=2 +# option 2, assuming the library is created as src/.libs/libfoo.so.2.0.5 or so +version=`ls src/.libs/lib*.so.* | \ + awk '{if (match($$0,/[0-9]+\.[0-9]+\.[0-9]+$$/)) print substr($$0,RSTART)}'` +major=`ls src/.libs/lib*.so.* | \ + awk '{if (match($$0,/\.so\.[0-9]+$$/)) print substr($$0,RSTART+4)}'` + +config.status: configure + dh_testdir + # Add here commands to configure the package. + CFLAGS="$(CFLAGS)" ./configure --host=$(DEB_HOST_GNU_TYPE) --build=$(DEB_BUILD_GNU_TYPE) --prefix=/usr --mandir=\$${prefix}/share/man --infodir=\$${prefix}/share/info + + +build: build-stamp +build-stamp: config.status + dh_testdir + + # Add here commands to compile the package. + $(MAKE) + + touch build-stamp + +clean: + dh_testdir + dh_testroot + rm -f build-stamp + + # Add here commands to clean up after the build process. + -$(MAKE) distclean +ifneq "$(wildcard /usr/share/misc/config.sub)" "" + cp -f /usr/share/misc/config.sub config.sub +endif +ifneq "$(wildcard /usr/share/misc/config.guess)" "" + cp -f /usr/share/misc/config.guess config.guess +endif + + + dh_clean + +install: build + dh_testdir + dh_testroot + dh_clean -k + dh_installdirs + + # Add here commands to install the package into debian/tmp + $(MAKE) install DESTDIR=$(CURDIR)/debian/tmp + + +# Build architecture-independent files here. +binary-indep: build install +# We have nothing to do by default. + +# Build architecture-dependent files here. +binary-arch: build install + dh_testdir + dh_testroot + dh_installchangelogs ChangeLog + dh_installdocs + dh_installexamples + dh_install --sourcedir=debian/tmp +# dh_installmenu +# dh_installdebconf +# dh_installlogrotate +# dh_installemacsen +# dh_installpam +# dh_installmime +# dh_installinit +# dh_installcron +# dh_installinfo + dh_installman + dh_link + dh_strip + dh_compress + dh_fixperms +# dh_perl +# dh_python + dh_makeshlibs + dh_installdeb + dh_shlibdeps + dh_gencontrol + dh_md5sums + dh_builddeb + +binary: binary-indep binary-arch +.PHONY: build clean binary-indep binary-arch binary install diff --git a/third_party/glog/packages/rpm.sh b/third_party/glog/packages/rpm.sh new file mode 100755 index 000000000000..e5649a22581b --- /dev/null +++ b/third_party/glog/packages/rpm.sh @@ -0,0 +1,75 @@ +#!/bin/sh -e + +# Run this from the 'packages' directory, just under rootdir + +# We can only build rpm packages, if the rpm build tools are installed +if [ \! -x /usr/bin/rpmbuild ] +then + echo "Cannot find /usr/bin/rpmbuild. Not building an rpm." 1>&2 + exit 0 +fi + +# Check the commandline flags +PACKAGE="$1" +VERSION="$2" +fullname="${PACKAGE}-${VERSION}" +archive=../$fullname.tar.gz + +if [ -z "$1" -o -z "$2" ] +then + echo "Usage: $0 <package name> <package version>" 1>&2 + exit 0 +fi + +# Double-check we're in the packages directory, just under rootdir +if [ \! -r ../Makefile -a \! -r ../INSTALL ] +then + echo "Must run $0 in the 'packages' directory, under the root directory." 1>&2 + echo "Also, you must run \"make dist\" before running this script." 1>&2 + exit 0 +fi + +if [ \! -r "$archive" ] +then + echo "Cannot find $archive. Run \"make dist\" first." 1>&2 + exit 0 +fi + +# Create the directory where the input lives, and where the output should live +RPM_SOURCE_DIR="/tmp/rpmsource-$fullname" +RPM_BUILD_DIR="/tmp/rpmbuild-$fullname" + +trap 'rm -rf $RPM_SOURCE_DIR $RPM_BUILD_DIR; exit $?' EXIT SIGHUP SIGINT SIGTERM + +rm -rf "$RPM_SOURCE_DIR" "$RPM_BUILD_DIR" +mkdir "$RPM_SOURCE_DIR" +mkdir "$RPM_BUILD_DIR" + +cp "$archive" "$RPM_SOURCE_DIR"/v"$VERSION".tar.gz + +rpmbuild -bb rpm/rpm.spec \ + --define "NAME $PACKAGE" \ + --define "VERSION $VERSION" \ + --define "_sourcedir $RPM_SOURCE_DIR" \ + --define "_builddir $RPM_BUILD_DIR" \ + --define "_rpmdir $RPM_SOURCE_DIR" + +# We put the output in a directory based on what system we've built for +destdir=rpm-unknown +if [ -r /etc/issue ] +then + grep "Red Hat.*release 7" /etc/issue >/dev/null 2>&1 && destdir=rh7 + grep "Red Hat.*release 8" /etc/issue >/dev/null 2>&1 && destdir=rh8 + grep "Red Hat.*release 9" /etc/issue >/dev/null 2>&1 && destdir=rh9 + if grep Fedora /etc/issue >/dev/null; then + destdir=fc`grep Fedora /etc/issue | cut -d' ' -f 4`; + fi +fi + +rm -rf "$destdir" +mkdir -p "$destdir" +# We want to get not only the main package but devel etc, hence the middle * +mv "$RPM_SOURCE_DIR"/*/"${PACKAGE}"-*"${VERSION}"*.rpm "$destdir" + +echo +echo "The rpm package file(s) are located in $PWD/$destdir" diff --git a/third_party/glog/packages/rpm/rpm.spec b/third_party/glog/packages/rpm/rpm.spec new file mode 100644 index 000000000000..24ef134df520 --- /dev/null +++ b/third_party/glog/packages/rpm/rpm.spec @@ -0,0 +1,72 @@ +%define RELEASE 1 +%define rel %{?CUSTOM_RELEASE} %{!?CUSTOM_RELEASE:%RELEASE} +%define prefix /usr + +Name: %NAME +Summary: A C++ application logging library +Version: %VERSION +Release: %rel +Group: Development/Libraries +URL: http://github.com/google/glog +License: BSD +Vendor: Google +Packager: Google Inc. <opensource@google.com> +Source: https://github.com/google/glog/archive/v%{VERSION}.tar.gz +Distribution: Redhat 7 and above. +Buildroot: %{_tmppath}/%{name}-root +Prefix: %prefix + +%description +The %name package contains a library that implements application-level +logging. This library provides logging APIs based on C++-style +streams and various helper macros. + +%package devel +Summary: A C++ application logging library +Group: Development/Libraries +Requires: %{NAME} = %{VERSION} + +%description devel +The %name-devel package contains static and debug libraries and header +files for developing applications that use the %name package. + +%changelog + * Wed Mar 26 2008 <opensource@google.com> + - First draft + +%prep +%setup + +%build +./configure +make prefix=%prefix + +%install +rm -rf $RPM_BUILD_ROOT +make prefix=$RPM_BUILD_ROOT%{prefix} install + +%clean +rm -rf $RPM_BUILD_ROOT + +%files +%defattr(-,root,root) + +## Mark all installed files within /usr/share/doc/{package name} as +## documentation. This depends on the following two lines appearing in +## Makefile.am: +## docdir = $(prefix)/share/doc/$(PACKAGE)-$(VERSION) +## dist_doc_DATA = AUTHORS COPYING ChangeLog INSTALL README.md +%docdir %{prefix}/share/doc/%{NAME}-%{VERSION} +%{prefix}/share/doc/%{NAME}-%{VERSION}/* + +%{prefix}/lib/libglog.so.0 +%{prefix}/lib/libglog.so.0.0.0 + +%files devel +%defattr(-,root,root) + +%{prefix}/include/glog +%{prefix}/lib/libglog.a +%{prefix}/lib/libglog.la +%{prefix}/lib/libglog.so +%{prefix}/lib/pkgconfig/libglog.pc diff --git a/third_party/glog/src/base/commandlineflags.h b/third_party/glog/src/base/commandlineflags.h new file mode 100644 index 000000000000..c8d50890269a --- /dev/null +++ b/third_party/glog/src/base/commandlineflags.h @@ -0,0 +1,133 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// This file is a compatibility layer that defines Google's version of +// command line flags that are used for configuration. +// +// We put flags into their own namespace. It is purposefully +// named in an opaque way that people should have trouble typing +// directly. The idea is that DEFINE puts the flag in the weird +// namespace, and DECLARE imports the flag from there into the +// current namespace. The net result is to force people to use +// DECLARE to get access to a flag, rather than saying +// extern bool FLAGS_logtostderr; +// or some such instead. We want this so we can put extra +// functionality (like sanity-checking) in DECLARE if we want, +// and make sure it is picked up everywhere. +// +// We also put the type of the variable in the namespace, so that +// people can't DECLARE_int32 something that they DEFINE_bool'd +// elsewhere. +#ifndef BASE_COMMANDLINEFLAGS_H__ +#define BASE_COMMANDLINEFLAGS_H__ + +#include "config.h" +#include <string> +#include <string.h> // for memchr +#include <stdlib.h> // for getenv + +#ifdef HAVE_LIB_GFLAGS + +#include <gflags/gflags.h> + +#else + +#include "glog/logging.h" + +#define DECLARE_VARIABLE(type, shorttype, name, tn) \ + namespace fL##shorttype { \ + extern GOOGLE_GLOG_DLL_DECL type FLAGS_##name; \ + } \ + using fL##shorttype::FLAGS_##name +#define DEFINE_VARIABLE(type, shorttype, name, value, meaning, tn) \ + namespace fL##shorttype { \ + GOOGLE_GLOG_DLL_DECL type FLAGS_##name(value); \ + char FLAGS_no##name; \ + } \ + using fL##shorttype::FLAGS_##name + +// bool specialization +#define DECLARE_bool(name) \ + DECLARE_VARIABLE(bool, B, name, bool) +#define DEFINE_bool(name, value, meaning) \ + DEFINE_VARIABLE(bool, B, name, value, meaning, bool) + +// int32 specialization +#define DECLARE_int32(name) \ + DECLARE_VARIABLE(GOOGLE_NAMESPACE::int32, I, name, int32) +#define DEFINE_int32(name, value, meaning) \ + DEFINE_VARIABLE(GOOGLE_NAMESPACE::int32, I, name, value, meaning, int32) + +// Special case for string, because we have to specify the namespace +// std::string, which doesn't play nicely with our FLAG__namespace hackery. +#define DECLARE_string(name) \ + namespace fLS { \ + extern GOOGLE_GLOG_DLL_DECL std::string& FLAGS_##name; \ + } \ + using fLS::FLAGS_##name +#define DEFINE_string(name, value, meaning) \ + namespace fLS { \ + std::string FLAGS_##name##_buf(value); \ + GOOGLE_GLOG_DLL_DECL std::string& FLAGS_##name = FLAGS_##name##_buf; \ + char FLAGS_no##name; \ + } \ + using fLS::FLAGS_##name + +#endif // HAVE_LIB_GFLAGS + +// Define GLOG_DEFINE_* using DEFINE_* . By using these macros, we +// have GLOG_* environ variables even if we have gflags installed. +// +// If both an environment variable and a flag are specified, the value +// specified by a flag wins. E.g., if GLOG_v=0 and --v=1, the +// verbosity will be 1, not 0. + +#define GLOG_DEFINE_bool(name, value, meaning) \ + DEFINE_bool(name, EnvToBool("GLOG_" #name, value), meaning) + +#define GLOG_DEFINE_int32(name, value, meaning) \ + DEFINE_int32(name, EnvToInt("GLOG_" #name, value), meaning) + +#define GLOG_DEFINE_string(name, value, meaning) \ + DEFINE_string(name, EnvToString("GLOG_" #name, value), meaning) + +// These macros (could be functions, but I don't want to bother with a .cc +// file), make it easier to initialize flags from the environment. + +#define EnvToString(envname, dflt) \ + (!getenv(envname) ? (dflt) : getenv(envname)) + +#define EnvToBool(envname, dflt) \ + (!getenv(envname) ? (dflt) : memchr("tTyY1\0", getenv(envname)[0], 6) != NULL) + +#define EnvToInt(envname, dflt) \ + (!getenv(envname) ? (dflt) : strtol(getenv(envname), NULL, 10)) + +#endif // BASE_COMMANDLINEFLAGS_H__ diff --git a/third_party/glog/src/base/googleinit.h b/third_party/glog/src/base/googleinit.h new file mode 100644 index 000000000000..5a8b515cd604 --- /dev/null +++ b/third_party/glog/src/base/googleinit.h @@ -0,0 +1,51 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// Author: Jacob Hoffman-Andrews + +#ifndef _GOOGLEINIT_H +#define _GOOGLEINIT_H + +class GoogleInitializer { + public: + typedef void (*void_function)(void); + GoogleInitializer(const char*, void_function f) { + f(); + } +}; + +#define REGISTER_MODULE_INITIALIZER(name, body) \ + namespace { \ + static void google_init_module_##name () { body; } \ + GoogleInitializer google_initializer_module_##name(#name, \ + google_init_module_##name); \ + } + +#endif /* _GOOGLEINIT_H */ diff --git a/third_party/glog/src/base/mutex.h b/third_party/glog/src/base/mutex.h new file mode 100644 index 000000000000..ced2b9950edc --- /dev/null +++ b/third_party/glog/src/base/mutex.h @@ -0,0 +1,333 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// --- +// Author: Craig Silverstein. +// +// A simple mutex wrapper, supporting locks and read-write locks. +// You should assume the locks are *not* re-entrant. +// +// To use: you should define the following macros in your configure.ac: +// ACX_PTHREAD +// AC_RWLOCK +// The latter is defined in ../autoconf. +// +// This class is meant to be internal-only and should be wrapped by an +// internal namespace. Before you use this module, please give the +// name of your internal namespace for this module. Or, if you want +// to expose it, you'll want to move it to the Google namespace. We +// cannot put this class in global namespace because there can be some +// problems when we have multiple versions of Mutex in each shared object. +// +// NOTE: by default, we have #ifdef'ed out the TryLock() method. +// This is for two reasons: +// 1) TryLock() under Windows is a bit annoying (it requires a +// #define to be defined very early). +// 2) TryLock() is broken for NO_THREADS mode, at least in NDEBUG +// mode. +// If you need TryLock(), and either these two caveats are not a +// problem for you, or you're willing to work around them, then +// feel free to #define GMUTEX_TRYLOCK, or to remove the #ifdefs +// in the code below. +// +// CYGWIN NOTE: Cygwin support for rwlock seems to be buggy: +// http://www.cygwin.com/ml/cygwin/2008-12/msg00017.html +// Because of that, we might as well use windows locks for +// cygwin. They seem to be more reliable than the cygwin pthreads layer. +// +// TRICKY IMPLEMENTATION NOTE: +// This class is designed to be safe to use during +// dynamic-initialization -- that is, by global constructors that are +// run before main() starts. The issue in this case is that +// dynamic-initialization happens in an unpredictable order, and it +// could be that someone else's dynamic initializer could call a +// function that tries to acquire this mutex -- but that all happens +// before this mutex's constructor has run. (This can happen even if +// the mutex and the function that uses the mutex are in the same .cc +// file.) Basically, because Mutex does non-trivial work in its +// constructor, it's not, in the naive implementation, safe to use +// before dynamic initialization has run on it. +// +// The solution used here is to pair the actual mutex primitive with a +// bool that is set to true when the mutex is dynamically initialized. +// (Before that it's false.) Then we modify all mutex routines to +// look at the bool, and not try to lock/unlock until the bool makes +// it to true (which happens after the Mutex constructor has run.) +// +// This works because before main() starts -- particularly, during +// dynamic initialization -- there are no threads, so a) it's ok that +// the mutex operations are a no-op, since we don't need locking then +// anyway; and b) we can be quite confident our bool won't change +// state between a call to Lock() and a call to Unlock() (that would +// require a global constructor in one translation unit to call Lock() +// and another global constructor in another translation unit to call +// Unlock() later, which is pretty perverse). +// +// That said, it's tricky, and can conceivably fail; it's safest to +// avoid trying to acquire a mutex in a global constructor, if you +// can. One way it can fail is that a really smart compiler might +// initialize the bool to true at static-initialization time (too +// early) rather than at dynamic-initialization time. To discourage +// that, we set is_safe_ to true in code (not the constructor +// colon-initializer) and set it to true via a function that always +// evaluates to true, but that the compiler can't know always +// evaluates to true. This should be good enough. + +#ifndef GOOGLE_MUTEX_H_ +#define GOOGLE_MUTEX_H_ + +#include "config.h" // to figure out pthreads support + +#if defined(NO_THREADS) + typedef int MutexType; // to keep a lock-count +#elif defined(_WIN32) || defined(__CYGWIN32__) || defined(__CYGWIN64__) +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN // We only need minimal includes +# endif +# ifdef GMUTEX_TRYLOCK + // We need Windows NT or later for TryEnterCriticalSection(). If you + // don't need that functionality, you can remove these _WIN32_WINNT + // lines, and change TryLock() to assert(0) or something. +# ifndef _WIN32_WINNT +# define _WIN32_WINNT 0x0400 +# endif +# endif +// To avoid macro definition of ERROR. +# ifndef NOGDI +# define NOGDI +# endif +// To avoid macro definition of min/max. +# ifndef NOMINMAX +# define NOMINMAX +# endif +# include <windows.h> + typedef CRITICAL_SECTION MutexType; +#elif defined(HAVE_PTHREAD) && defined(HAVE_RWLOCK) + // Needed for pthread_rwlock_*. If it causes problems, you could take it + // out, but then you'd have to unset HAVE_RWLOCK (at least on linux -- it + // *does* cause problems for FreeBSD, or MacOSX, but isn't needed + // for locking there.) +# ifdef __linux__ +# ifndef _XOPEN_SOURCE // Some other header might have already set it for us. +# define _XOPEN_SOURCE 500 // may be needed to get the rwlock calls +# endif +# endif +# include <pthread.h> + typedef pthread_rwlock_t MutexType; +#elif defined(HAVE_PTHREAD) +# include <pthread.h> + typedef pthread_mutex_t MutexType; +#else +# error Need to implement mutex.h for your architecture, or #define NO_THREADS +#endif + +// We need to include these header files after defining _XOPEN_SOURCE +// as they may define the _XOPEN_SOURCE macro. +#include <assert.h> +#include <stdlib.h> // for abort() + +#define MUTEX_NAMESPACE glog_internal_namespace_ + +namespace MUTEX_NAMESPACE { + +class Mutex { + public: + // Create a Mutex that is not held by anybody. This constructor is + // typically used for Mutexes allocated on the heap or the stack. + // See below for a recommendation for constructing global Mutex + // objects. + inline Mutex(); + + // Destructor + inline ~Mutex(); + + inline void Lock(); // Block if needed until free then acquire exclusively + inline void Unlock(); // Release a lock acquired via Lock() +#ifdef GMUTEX_TRYLOCK + inline bool TryLock(); // If free, Lock() and return true, else return false +#endif + // Note that on systems that don't support read-write locks, these may + // be implemented as synonyms to Lock() and Unlock(). So you can use + // these for efficiency, but don't use them anyplace where being able + // to do shared reads is necessary to avoid deadlock. + inline void ReaderLock(); // Block until free or shared then acquire a share + inline void ReaderUnlock(); // Release a read share of this Mutex + inline void WriterLock() { Lock(); } // Acquire an exclusive lock + inline void WriterUnlock() { Unlock(); } // Release a lock from WriterLock() + + // TODO(hamaji): Do nothing, implement correctly. + inline void AssertHeld() {} + + private: + MutexType mutex_; + // We want to make sure that the compiler sets is_safe_ to true only + // when we tell it to, and never makes assumptions is_safe_ is + // always true. volatile is the most reliable way to do that. + volatile bool is_safe_; + + inline void SetIsSafe() { is_safe_ = true; } + + // Catch the error of writing Mutex when intending MutexLock. + Mutex(Mutex* /*ignored*/) {} + // Disallow "evil" constructors + Mutex(const Mutex&); + void operator=(const Mutex&); +}; + +// Now the implementation of Mutex for various systems +#if defined(NO_THREADS) + +// When we don't have threads, we can be either reading or writing, +// but not both. We can have lots of readers at once (in no-threads +// mode, that's most likely to happen in recursive function calls), +// but only one writer. We represent this by having mutex_ be -1 when +// writing and a number > 0 when reading (and 0 when no lock is held). +// +// In debug mode, we assert these invariants, while in non-debug mode +// we do nothing, for efficiency. That's why everything is in an +// assert. + +Mutex::Mutex() : mutex_(0) { } +Mutex::~Mutex() { assert(mutex_ == 0); } +void Mutex::Lock() { assert(--mutex_ == -1); } +void Mutex::Unlock() { assert(mutex_++ == -1); } +#ifdef GMUTEX_TRYLOCK +bool Mutex::TryLock() { if (mutex_) return false; Lock(); return true; } +#endif +void Mutex::ReaderLock() { assert(++mutex_ > 0); } +void Mutex::ReaderUnlock() { assert(mutex_-- > 0); } + +#elif defined(_WIN32) || defined(__CYGWIN32__) || defined(__CYGWIN64__) + +Mutex::Mutex() { InitializeCriticalSection(&mutex_); SetIsSafe(); } +Mutex::~Mutex() { DeleteCriticalSection(&mutex_); } +void Mutex::Lock() { if (is_safe_) EnterCriticalSection(&mutex_); } +void Mutex::Unlock() { if (is_safe_) LeaveCriticalSection(&mutex_); } +#ifdef GMUTEX_TRYLOCK +bool Mutex::TryLock() { return is_safe_ ? + TryEnterCriticalSection(&mutex_) != 0 : true; } +#endif +void Mutex::ReaderLock() { Lock(); } // we don't have read-write locks +void Mutex::ReaderUnlock() { Unlock(); } + +#elif defined(HAVE_PTHREAD) && defined(HAVE_RWLOCK) + +#define SAFE_PTHREAD(fncall) do { /* run fncall if is_safe_ is true */ \ + if (is_safe_ && fncall(&mutex_) != 0) abort(); \ +} while (0) + +Mutex::Mutex() { + SetIsSafe(); + if (is_safe_ && pthread_rwlock_init(&mutex_, NULL) != 0) abort(); +} +Mutex::~Mutex() { SAFE_PTHREAD(pthread_rwlock_destroy); } +void Mutex::Lock() { SAFE_PTHREAD(pthread_rwlock_wrlock); } +void Mutex::Unlock() { SAFE_PTHREAD(pthread_rwlock_unlock); } +#ifdef GMUTEX_TRYLOCK +bool Mutex::TryLock() { return is_safe_ ? + pthread_rwlock_trywrlock(&mutex_) == 0 : + true; } +#endif +void Mutex::ReaderLock() { SAFE_PTHREAD(pthread_rwlock_rdlock); } +void Mutex::ReaderUnlock() { SAFE_PTHREAD(pthread_rwlock_unlock); } +#undef SAFE_PTHREAD + +#elif defined(HAVE_PTHREAD) + +#define SAFE_PTHREAD(fncall) do { /* run fncall if is_safe_ is true */ \ + if (is_safe_ && fncall(&mutex_) != 0) abort(); \ +} while (0) + +Mutex::Mutex() { + SetIsSafe(); + if (is_safe_ && pthread_mutex_init(&mutex_, NULL) != 0) abort(); +} +Mutex::~Mutex() { SAFE_PTHREAD(pthread_mutex_destroy); } +void Mutex::Lock() { SAFE_PTHREAD(pthread_mutex_lock); } +void Mutex::Unlock() { SAFE_PTHREAD(pthread_mutex_unlock); } +#ifdef GMUTEX_TRYLOCK +bool Mutex::TryLock() { return is_safe_ ? + pthread_mutex_trylock(&mutex_) == 0 : true; } +#endif +void Mutex::ReaderLock() { Lock(); } +void Mutex::ReaderUnlock() { Unlock(); } +#undef SAFE_PTHREAD + +#endif + +// -------------------------------------------------------------------------- +// Some helper classes + +// MutexLock(mu) acquires mu when constructed and releases it when destroyed. +class MutexLock { + public: + explicit MutexLock(Mutex *mu) : mu_(mu) { mu_->Lock(); } + ~MutexLock() { mu_->Unlock(); } + private: + Mutex * const mu_; + // Disallow "evil" constructors + MutexLock(const MutexLock&); + void operator=(const MutexLock&); +}; + +// ReaderMutexLock and WriterMutexLock do the same, for rwlocks +class ReaderMutexLock { + public: + explicit ReaderMutexLock(Mutex *mu) : mu_(mu) { mu_->ReaderLock(); } + ~ReaderMutexLock() { mu_->ReaderUnlock(); } + private: + Mutex * const mu_; + // Disallow "evil" constructors + ReaderMutexLock(const ReaderMutexLock&); + void operator=(const ReaderMutexLock&); +}; + +class WriterMutexLock { + public: + explicit WriterMutexLock(Mutex *mu) : mu_(mu) { mu_->WriterLock(); } + ~WriterMutexLock() { mu_->WriterUnlock(); } + private: + Mutex * const mu_; + // Disallow "evil" constructors + WriterMutexLock(const WriterMutexLock&); + void operator=(const WriterMutexLock&); +}; + +// Catch bug where variable name is omitted, e.g. MutexLock (&mu); +#define MutexLock(x) COMPILE_ASSERT(0, mutex_lock_decl_missing_var_name) +#define ReaderMutexLock(x) COMPILE_ASSERT(0, rmutex_lock_decl_missing_var_name) +#define WriterMutexLock(x) COMPILE_ASSERT(0, wmutex_lock_decl_missing_var_name) + +} // namespace MUTEX_NAMESPACE + +using namespace MUTEX_NAMESPACE; + +#undef MUTEX_NAMESPACE + +#endif /* #define GOOGLE_MUTEX_H__ */ diff --git a/third_party/glog/src/config.h.cmake.in b/third_party/glog/src/config.h.cmake.in new file mode 100644 index 000000000000..20f099050ee5 --- /dev/null +++ b/third_party/glog/src/config.h.cmake.in @@ -0,0 +1,219 @@ +#ifndef GLOG_CONFIG_H +#define GLOG_CONFIG_H + +/* define if glog doesn't use RTTI */ +#cmakedefine DISABLE_RTTI + +/* Namespace for Google classes */ +#cmakedefine GOOGLE_NAMESPACE ${GOOGLE_NAMESPACE} + +/* Define if you have the `dladdr' function */ +#cmakedefine HAVE_DLADDR + +/* Define if you have the `snprintf' function */ +#cmakedefine HAVE_SNPRINTF + +/* Define to 1 if you have the <dlfcn.h> header file. */ +#cmakedefine HAVE_DLFCN_H + +/* Define to 1 if you have the <execinfo.h> header file. */ +#cmakedefine HAVE_EXECINFO_H + +/* Define if you have the `fcntl' function */ +#cmakedefine HAVE_FCNTL + +/* Define to 1 if you have the <glob.h> header file. */ +#cmakedefine HAVE_GLOB_H + +/* Define to 1 if you have the <inttypes.h> header file. */ +#cmakedefine HAVE_INTTYPES_H ${HAVE_INTTYPES_H} + +/* Define to 1 if you have the `pthread' library (-lpthread). */ +#cmakedefine HAVE_LIBPTHREAD + +/* Define to 1 if you have the <libunwind.h> header file. */ +#cmakedefine HAVE_LIBUNWIND_H + +/* define if you have google gflags library */ +#cmakedefine HAVE_LIB_GFLAGS + +/* define if you have google gmock library */ +#cmakedefine HAVE_LIB_GMOCK + +/* define if you have google gtest library */ +#cmakedefine HAVE_LIB_GTEST + +/* define if you have libunwind */ +#cmakedefine HAVE_LIB_UNWIND + +/* Define to 1 if you have the <memory.h> header file. */ +#cmakedefine HAVE_MEMORY_H + +/* define to disable multithreading support. */ +#cmakedefine NO_THREADS + +/* define if the compiler implements namespaces */ +#cmakedefine HAVE_NAMESPACES + +/* Define if you have the 'pread' function */ +#cmakedefine HAVE_PREAD + +/* Define if you have POSIX threads libraries and header files. */ +#cmakedefine HAVE_PTHREAD + +/* Define to 1 if you have the <pwd.h> header file. */ +#cmakedefine HAVE_PWD_H + +/* Define if you have the 'pwrite' function */ +#cmakedefine HAVE_PWRITE + +/* define if the compiler implements pthread_rwlock_* */ +#cmakedefine HAVE_RWLOCK + +/* Define if you have the 'sigaction' function */ +#cmakedefine HAVE_SIGACTION + +/* Define if you have the `sigaltstack' function */ +#cmakedefine HAVE_SIGALTSTACK + +/* Define to 1 if you have the <stdint.h> header file. */ +#cmakedefine HAVE_STDINT_H ${HAVE_STDINT_H} + +/* Define to 1 if you have the <stdlib.h> header file. */ +#cmakedefine HAVE_STDLIB_H + +/* Define to 1 if you have the <strings.h> header file. */ +#cmakedefine HAVE_STRINGS_H + +/* Define to 1 if you have the <string.h> header file. */ +#cmakedefine HAVE_STRING_H + +/* Define to 1 if you have the <syscall.h> header file. */ +#cmakedefine HAVE_SYSCALL_H + +/* Define to 1 if you have the <syslog.h> header file. */ +#cmakedefine HAVE_SYSLOG_H + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#cmakedefine HAVE_SYS_STAT_H + +/* Define to 1 if you have the <sys/syscall.h> header file. */ +#cmakedefine HAVE_SYS_SYSCALL_H + +/* Define to 1 if you have the <sys/time.h> header file. */ +#cmakedefine HAVE_SYS_TIME_H + +/* Define to 1 if you have the <sys/types.h> header file. */ +#cmakedefine HAVE_SYS_TYPES_H ${HAVE_SYS_TYPES_H} + +/* Define to 1 if you have the <sys/ucontext.h> header file. */ +#cmakedefine HAVE_SYS_UCONTEXT_H + +/* Define to 1 if you have the <sys/utsname.h> header file. */ +#cmakedefine HAVE_SYS_UTSNAME_H + +/* Define to 1 if you have the <sys/wait.h> header file. */ +#cmakedefine HAVE_SYS_WAIT_H + +/* Define to 1 if you have the <ucontext.h> header file. */ +#cmakedefine HAVE_UCONTEXT_H + +/* Define to 1 if you have the <unistd.h> header file. */ +#cmakedefine HAVE_UNISTD_H ${HAVE_UNISTD_H} + +/* Define to 1 if you have the <unwind.h> header file. */ +#cmakedefine HAVE_UNWIND_H ${HAVE_UNWIND_H} + +/* define if the compiler supports using expression for operator */ +#cmakedefine HAVE_USING_OPERATOR + +/* define if your compiler has __attribute__ */ +#cmakedefine HAVE___ATTRIBUTE__ + +/* define if your compiler has __builtin_expect */ +#cmakedefine HAVE___BUILTIN_EXPECT ${HAVE___BUILTIN_EXPECT} + +/* define if your compiler has __sync_val_compare_and_swap */ +#cmakedefine HAVE___SYNC_VAL_COMPARE_AND_SWAP + +/* define if symbolize support is available */ +#cmakedefine HAVE_SYMBOLIZE + +/* define if localtime_r is available in time.h */ +#cmakedefine HAVE_LOCALTIME_R + +/* Define to the sub-directory in which libtool stores uninstalled libraries. + */ +#cmakedefine LT_OBJDIR + +/* Name of package */ +#cmakedefine PACKAGE + +/* Define to the address where bug reports for this package should be sent. */ +#cmakedefine PACKAGE_BUGREPORT + +/* Define to the full name of this package. */ +#cmakedefine PACKAGE_NAME + +/* Define to the full name and version of this package. */ +#cmakedefine PACKAGE_STRING + +/* Define to the one symbol short name of this package. */ +#cmakedefine PACKAGE_TARNAME + +/* Define to the home page for this package. */ +#cmakedefine PACKAGE_URL + +/* Define to the version of this package. */ +#cmakedefine PACKAGE_VERSION + +/* How to access the PC from a struct ucontext */ +#cmakedefine PC_FROM_UCONTEXT + +/* define if we should print file offsets in traces instead of symbolizing. */ +#cmakedefine PRINT_UNSYMBOLIZED_STACK_TRACES + +/* Define to necessary symbol if this constant uses a non-standard name on + your system. */ +#cmakedefine PTHREAD_CREATE_JOINABLE + +/* The size of `void *', as computed by sizeof. */ +#cmakedefine SIZEOF_VOID_P ${SIZEOF_VOID_P} + +/* Define to 1 if you have the ANSI C header files. */ +#cmakedefine STDC_HEADERS + +/* the namespace where STL code like vector<> is defined */ +#cmakedefine STL_NAMESPACE ${STL_NAMESPACE} + +/* location of source code */ +#cmakedefine TEST_SRC_DIR ${TEST_SRC_DIR} + +/* Define to necessary thread-local storage attribute. */ +#cmakedefine GLOG_THREAD_LOCAL_STORAGE ${GLOG_THREAD_LOCAL_STORAGE} + +/* Check whether aligned_storage and alignof present */ +#cmakedefine HAVE_ALIGNED_STORAGE ${HAVE_ALIGNED_STORAGE} + +/* Version number of package */ +#cmakedefine VERSION + +#ifdef GLOG_BAZEL_BUILD + +/* TODO(rodrigoq): remove this workaround once bazel#3979 is resolved: + * https://github.com/bazelbuild/bazel/issues/3979 */ +#define _START_GOOGLE_NAMESPACE_ namespace GOOGLE_NAMESPACE { + +#define _END_GOOGLE_NAMESPACE_ } + +#else + +/* Stops putting the code inside the Google namespace */ +#cmakedefine _END_GOOGLE_NAMESPACE_ ${_END_GOOGLE_NAMESPACE_} + +/* Puts following code inside the Google namespace */ +#cmakedefine _START_GOOGLE_NAMESPACE_ ${_START_GOOGLE_NAMESPACE_} + +#endif + +#endif // GLOG_CONFIG_H diff --git a/third_party/glog/src/config_for_unittests.h b/third_party/glog/src/config_for_unittests.h new file mode 100644 index 000000000000..13ea8eab7a39 --- /dev/null +++ b/third_party/glog/src/config_for_unittests.h @@ -0,0 +1,66 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// --- +// All Rights Reserved. +// +// Author: Craig Silverstein +// Copied from google-perftools and modified by Shinichiro Hamaji +// +// This file is needed for windows -- unittests are not part of the +// glog dll, but still want to include config.h just like the +// dll does, so they can use internal tools and APIs for testing. +// +// The problem is that config.h declares GOOGLE_GLOG_DLL_DECL to be +// for exporting symbols, but the unittest needs to *import* symbols +// (since it's not the dll). +// +// The solution is to have this file, which is just like config.h but +// sets GOOGLE_GLOG_DLL_DECL to do a dllimport instead of a dllexport. +// +// The reason we need this extra GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS +// variable is in case people want to set GOOGLE_GLOG_DLL_DECL explicitly +// to something other than __declspec(dllexport). In that case, they +// may want to use something other than __declspec(dllimport) for the +// unittest case. For that, we allow folks to define both +// GOOGLE_GLOG_DLL_DECL and GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS explicitly. +// +// NOTE: This file is equivalent to config.h on non-windows systems, +// which never defined GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS and always +// define GOOGLE_GLOG_DLL_DECL to the empty string. + +#include "config.h" + +#undef GOOGLE_GLOG_DLL_DECL +#ifdef GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS +# define GOOGLE_GLOG_DLL_DECL GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS +#else +// if DLL_DECL_FOR_UNITTESTS isn't defined, use "" +# define GOOGLE_GLOG_DLL_DECL +#endif diff --git a/third_party/glog/src/demangle.cc b/third_party/glog/src/demangle.cc new file mode 100644 index 000000000000..9369f9a65c2d --- /dev/null +++ b/third_party/glog/src/demangle.cc @@ -0,0 +1,1356 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// For reference check out: +// http://itanium-cxx-abi.github.io/cxx-abi/abi.html#mangling +// +// Note that we only have partial C++0x support yet. + +#include <stdio.h> // for NULL +#include "utilities.h" +#include "demangle.h" + +#if defined(OS_WINDOWS) +#include <dbghelp.h> +#pragma comment(lib, "dbghelp") +#endif + +_START_GOOGLE_NAMESPACE_ + +#if !defined(OS_WINDOWS) +typedef struct { + const char *abbrev; + const char *real_name; +} AbbrevPair; + +// List of operators from Itanium C++ ABI. +static const AbbrevPair kOperatorList[] = { + { "nw", "new" }, + { "na", "new[]" }, + { "dl", "delete" }, + { "da", "delete[]" }, + { "ps", "+" }, + { "ng", "-" }, + { "ad", "&" }, + { "de", "*" }, + { "co", "~" }, + { "pl", "+" }, + { "mi", "-" }, + { "ml", "*" }, + { "dv", "/" }, + { "rm", "%" }, + { "an", "&" }, + { "or", "|" }, + { "eo", "^" }, + { "aS", "=" }, + { "pL", "+=" }, + { "mI", "-=" }, + { "mL", "*=" }, + { "dV", "/=" }, + { "rM", "%=" }, + { "aN", "&=" }, + { "oR", "|=" }, + { "eO", "^=" }, + { "ls", "<<" }, + { "rs", ">>" }, + { "lS", "<<=" }, + { "rS", ">>=" }, + { "eq", "==" }, + { "ne", "!=" }, + { "lt", "<" }, + { "gt", ">" }, + { "le", "<=" }, + { "ge", ">=" }, + { "nt", "!" }, + { "aa", "&&" }, + { "oo", "||" }, + { "pp", "++" }, + { "mm", "--" }, + { "cm", "," }, + { "pm", "->*" }, + { "pt", "->" }, + { "cl", "()" }, + { "ix", "[]" }, + { "qu", "?" }, + { "st", "sizeof" }, + { "sz", "sizeof" }, + { NULL, NULL }, +}; + +// List of builtin types from Itanium C++ ABI. +static const AbbrevPair kBuiltinTypeList[] = { + { "v", "void" }, + { "w", "wchar_t" }, + { "b", "bool" }, + { "c", "char" }, + { "a", "signed char" }, + { "h", "unsigned char" }, + { "s", "short" }, + { "t", "unsigned short" }, + { "i", "int" }, + { "j", "unsigned int" }, + { "l", "long" }, + { "m", "unsigned long" }, + { "x", "long long" }, + { "y", "unsigned long long" }, + { "n", "__int128" }, + { "o", "unsigned __int128" }, + { "f", "float" }, + { "d", "double" }, + { "e", "long double" }, + { "g", "__float128" }, + { "z", "ellipsis" }, + { NULL, NULL } +}; + +// List of substitutions Itanium C++ ABI. +static const AbbrevPair kSubstitutionList[] = { + { "St", "" }, + { "Sa", "allocator" }, + { "Sb", "basic_string" }, + // std::basic_string<char, std::char_traits<char>,std::allocator<char> > + { "Ss", "string"}, + // std::basic_istream<char, std::char_traits<char> > + { "Si", "istream" }, + // std::basic_ostream<char, std::char_traits<char> > + { "So", "ostream" }, + // std::basic_iostream<char, std::char_traits<char> > + { "Sd", "iostream" }, + { NULL, NULL } +}; + +// State needed for demangling. +typedef struct { + const char *mangled_cur; // Cursor of mangled name. + char *out_cur; // Cursor of output string. + const char *out_begin; // Beginning of output string. + const char *out_end; // End of output string. + const char *prev_name; // For constructors/destructors. + int prev_name_length; // For constructors/destructors. + short nest_level; // For nested names. + bool append; // Append flag. + bool overflowed; // True if output gets overflowed. +} State; + +// We don't use strlen() in libc since it's not guaranteed to be async +// signal safe. +static size_t StrLen(const char *str) { + size_t len = 0; + while (*str != '\0') { + ++str; + ++len; + } + return len; +} + +// Returns true if "str" has at least "n" characters remaining. +static bool AtLeastNumCharsRemaining(const char *str, int n) { + for (int i = 0; i < n; ++i) { + if (str[i] == '\0') { + return false; + } + } + return true; +} + +// Returns true if "str" has "prefix" as a prefix. +static bool StrPrefix(const char *str, const char *prefix) { + size_t i = 0; + while (str[i] != '\0' && prefix[i] != '\0' && + str[i] == prefix[i]) { + ++i; + } + return prefix[i] == '\0'; // Consumed everything in "prefix". +} + +static void InitState(State *state, const char *mangled, + char *out, int out_size) { + state->mangled_cur = mangled; + state->out_cur = out; + state->out_begin = out; + state->out_end = out + out_size; + state->prev_name = NULL; + state->prev_name_length = -1; + state->nest_level = -1; + state->append = true; + state->overflowed = false; +} + +// Returns true and advances "mangled_cur" if we find "one_char_token" +// at "mangled_cur" position. It is assumed that "one_char_token" does +// not contain '\0'. +static bool ParseOneCharToken(State *state, const char one_char_token) { + if (state->mangled_cur[0] == one_char_token) { + ++state->mangled_cur; + return true; + } + return false; +} + +// Returns true and advances "mangled_cur" if we find "two_char_token" +// at "mangled_cur" position. It is assumed that "two_char_token" does +// not contain '\0'. +static bool ParseTwoCharToken(State *state, const char *two_char_token) { + if (state->mangled_cur[0] == two_char_token[0] && + state->mangled_cur[1] == two_char_token[1]) { + state->mangled_cur += 2; + return true; + } + return false; +} + +// Returns true and advances "mangled_cur" if we find any character in +// "char_class" at "mangled_cur" position. +static bool ParseCharClass(State *state, const char *char_class) { + const char *p = char_class; + for (; *p != '\0'; ++p) { + if (state->mangled_cur[0] == *p) { + ++state->mangled_cur; + return true; + } + } + return false; +} + +// This function is used for handling an optional non-terminal. +static bool Optional(bool) { + return true; +} + +// This function is used for handling <non-terminal>+ syntax. +typedef bool (*ParseFunc)(State *); +static bool OneOrMore(ParseFunc parse_func, State *state) { + if (parse_func(state)) { + while (parse_func(state)) { + } + return true; + } + return false; +} + +// This function is used for handling <non-terminal>* syntax. The function +// always returns true and must be followed by a termination token or a +// terminating sequence not handled by parse_func (e.g. +// ParseOneCharToken(state, 'E')). +static bool ZeroOrMore(ParseFunc parse_func, State *state) { + while (parse_func(state)) { + } + return true; +} + +// Append "str" at "out_cur". If there is an overflow, "overflowed" +// is set to true for later use. The output string is ensured to +// always terminate with '\0' as long as there is no overflow. +static void Append(State *state, const char * const str, const int length) { + int i; + for (i = 0; i < length; ++i) { + if (state->out_cur + 1 < state->out_end) { // +1 for '\0' + *state->out_cur = str[i]; + ++state->out_cur; + } else { + state->overflowed = true; + break; + } + } + if (!state->overflowed) { + *state->out_cur = '\0'; // Terminate it with '\0' + } +} + +// We don't use equivalents in libc to avoid locale issues. +static bool IsLower(char c) { + return c >= 'a' && c <= 'z'; +} + +static bool IsAlpha(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static bool IsDigit(char c) { + return c >= '0' && c <= '9'; +} + +// Returns true if "str" is a function clone suffix. These suffixes are used +// by GCC 4.5.x and later versions to indicate functions which have been +// cloned during optimization. We treat any sequence (.<alpha>+.<digit>+)+ as +// a function clone suffix. +static bool IsFunctionCloneSuffix(const char *str) { + size_t i = 0; + while (str[i] != '\0') { + // Consume a single .<alpha>+.<digit>+ sequence. + if (str[i] != '.' || !IsAlpha(str[i + 1])) { + return false; + } + i += 2; + while (IsAlpha(str[i])) { + ++i; + } + if (str[i] != '.' || !IsDigit(str[i + 1])) { + return false; + } + i += 2; + while (IsDigit(str[i])) { + ++i; + } + } + return true; // Consumed everything in "str". +} + +// Append "str" with some tweaks, iff "append" state is true. +// Returns true so that it can be placed in "if" conditions. +static void MaybeAppendWithLength(State *state, const char * const str, + const int length) { + if (state->append && length > 0) { + // Append a space if the output buffer ends with '<' and "str" + // starts with '<' to avoid <<<. + if (str[0] == '<' && state->out_begin < state->out_cur && + state->out_cur[-1] == '<') { + Append(state, " ", 1); + } + // Remember the last identifier name for ctors/dtors. + if (IsAlpha(str[0]) || str[0] == '_') { + state->prev_name = state->out_cur; + state->prev_name_length = length; + } + Append(state, str, length); + } +} + +// A convenient wrapper arount MaybeAppendWithLength(). +static bool MaybeAppend(State *state, const char * const str) { + if (state->append) { + int length = StrLen(str); + MaybeAppendWithLength(state, str, length); + } + return true; +} + +// This function is used for handling nested names. +static bool EnterNestedName(State *state) { + state->nest_level = 0; + return true; +} + +// This function is used for handling nested names. +static bool LeaveNestedName(State *state, short prev_value) { + state->nest_level = prev_value; + return true; +} + +// Disable the append mode not to print function parameters, etc. +static bool DisableAppend(State *state) { + state->append = false; + return true; +} + +// Restore the append mode to the previous state. +static bool RestoreAppend(State *state, bool prev_value) { + state->append = prev_value; + return true; +} + +// Increase the nest level for nested names. +static void MaybeIncreaseNestLevel(State *state) { + if (state->nest_level > -1) { + ++state->nest_level; + } +} + +// Appends :: for nested names if necessary. +static void MaybeAppendSeparator(State *state) { + if (state->nest_level >= 1) { + MaybeAppend(state, "::"); + } +} + +// Cancel the last separator if necessary. +static void MaybeCancelLastSeparator(State *state) { + if (state->nest_level >= 1 && state->append && + state->out_begin <= state->out_cur - 2) { + state->out_cur -= 2; + *state->out_cur = '\0'; + } +} + +// Returns true if the identifier of the given length pointed to by +// "mangled_cur" is anonymous namespace. +static bool IdentifierIsAnonymousNamespace(State *state, int length) { + static const char anon_prefix[] = "_GLOBAL__N_"; + return (length > (int)sizeof(anon_prefix) - 1 && // Should be longer. + StrPrefix(state->mangled_cur, anon_prefix)); +} + +// Forward declarations of our parsing functions. +static bool ParseMangledName(State *state); +static bool ParseEncoding(State *state); +static bool ParseName(State *state); +static bool ParseUnscopedName(State *state); +static bool ParseUnscopedTemplateName(State *state); +static bool ParseNestedName(State *state); +static bool ParsePrefix(State *state); +static bool ParseUnqualifiedName(State *state); +static bool ParseSourceName(State *state); +static bool ParseLocalSourceName(State *state); +static bool ParseNumber(State *state, int *number_out); +static bool ParseFloatNumber(State *state); +static bool ParseSeqId(State *state); +static bool ParseIdentifier(State *state, int length); +static bool ParseAbiTags(State *state); +static bool ParseAbiTag(State *state); +static bool ParseOperatorName(State *state); +static bool ParseSpecialName(State *state); +static bool ParseCallOffset(State *state); +static bool ParseNVOffset(State *state); +static bool ParseVOffset(State *state); +static bool ParseCtorDtorName(State *state); +static bool ParseType(State *state); +static bool ParseCVQualifiers(State *state); +static bool ParseBuiltinType(State *state); +static bool ParseFunctionType(State *state); +static bool ParseBareFunctionType(State *state); +static bool ParseClassEnumType(State *state); +static bool ParseArrayType(State *state); +static bool ParsePointerToMemberType(State *state); +static bool ParseTemplateParam(State *state); +static bool ParseTemplateTemplateParam(State *state); +static bool ParseTemplateArgs(State *state); +static bool ParseTemplateArg(State *state); +static bool ParseExpression(State *state); +static bool ParseExprPrimary(State *state); +static bool ParseLocalName(State *state); +static bool ParseDiscriminator(State *state); +static bool ParseSubstitution(State *state); + +// Implementation note: the following code is a straightforward +// translation of the Itanium C++ ABI defined in BNF with a couple of +// exceptions. +// +// - Support GNU extensions not defined in the Itanium C++ ABI +// - <prefix> and <template-prefix> are combined to avoid infinite loop +// - Reorder patterns to shorten the code +// - Reorder patterns to give greedier functions precedence +// We'll mark "Less greedy than" for these cases in the code +// +// Each parsing function changes the state and returns true on +// success. Otherwise, don't change the state and returns false. To +// ensure that the state isn't changed in the latter case, we save the +// original state before we call more than one parsing functions +// consecutively with &&, and restore the state if unsuccessful. See +// ParseEncoding() as an example of this convention. We follow the +// convention throughout the code. +// +// Originally we tried to do demangling without following the full ABI +// syntax but it turned out we needed to follow the full syntax to +// parse complicated cases like nested template arguments. Note that +// implementing a full-fledged demangler isn't trivial (libiberty's +// cp-demangle.c has +4300 lines). +// +// Note that (foo) in <(foo) ...> is a modifier to be ignored. +// +// Reference: +// - Itanium C++ ABI +// <http://www.codesourcery.com/cxx-abi/abi.html#mangling> + +// <mangled-name> ::= _Z <encoding> +static bool ParseMangledName(State *state) { + return ParseTwoCharToken(state, "_Z") && ParseEncoding(state); +} + +// <encoding> ::= <(function) name> <bare-function-type> +// ::= <(data) name> +// ::= <special-name> +static bool ParseEncoding(State *state) { + State copy = *state; + if (ParseName(state) && ParseBareFunctionType(state)) { + return true; + } + *state = copy; + + if (ParseName(state) || ParseSpecialName(state)) { + return true; + } + return false; +} + +// <name> ::= <nested-name> +// ::= <unscoped-template-name> <template-args> +// ::= <unscoped-name> +// ::= <local-name> +static bool ParseName(State *state) { + if (ParseNestedName(state) || ParseLocalName(state)) { + return true; + } + + State copy = *state; + if (ParseUnscopedTemplateName(state) && + ParseTemplateArgs(state)) { + return true; + } + *state = copy; + + // Less greedy than <unscoped-template-name> <template-args>. + if (ParseUnscopedName(state)) { + return true; + } + return false; +} + +// <unscoped-name> ::= <unqualified-name> +// ::= St <unqualified-name> +static bool ParseUnscopedName(State *state) { + if (ParseUnqualifiedName(state)) { + return true; + } + + State copy = *state; + if (ParseTwoCharToken(state, "St") && + MaybeAppend(state, "std::") && + ParseUnqualifiedName(state)) { + return true; + } + *state = copy; + return false; +} + +// <unscoped-template-name> ::= <unscoped-name> +// ::= <substitution> +static bool ParseUnscopedTemplateName(State *state) { + return ParseUnscopedName(state) || ParseSubstitution(state); +} + +// <nested-name> ::= N [<CV-qualifiers>] <prefix> <unqualified-name> E +// ::= N [<CV-qualifiers>] <template-prefix> <template-args> E +static bool ParseNestedName(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'N') && + EnterNestedName(state) && + Optional(ParseCVQualifiers(state)) && + ParsePrefix(state) && + LeaveNestedName(state, copy.nest_level) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + return false; +} + +// This part is tricky. If we literally translate them to code, we'll +// end up infinite loop. Hence we merge them to avoid the case. +// +// <prefix> ::= <prefix> <unqualified-name> +// ::= <template-prefix> <template-args> +// ::= <template-param> +// ::= <substitution> +// ::= # empty +// <template-prefix> ::= <prefix> <(template) unqualified-name> +// ::= <template-param> +// ::= <substitution> +static bool ParsePrefix(State *state) { + bool has_something = false; + while (true) { + MaybeAppendSeparator(state); + if (ParseTemplateParam(state) || + ParseSubstitution(state) || + ParseUnscopedName(state)) { + has_something = true; + MaybeIncreaseNestLevel(state); + continue; + } + MaybeCancelLastSeparator(state); + if (has_something && ParseTemplateArgs(state)) { + return ParsePrefix(state); + } else { + break; + } + } + return true; +} + +// <unqualified-name> ::= <operator-name> +// ::= <ctor-dtor-name> +// ::= <source-name> [<abi-tags>] +// ::= <local-source-name> [<abi-tags>] +static bool ParseUnqualifiedName(State *state) { + return (ParseOperatorName(state) || + ParseCtorDtorName(state) || + (ParseSourceName(state) && Optional(ParseAbiTags(state))) || + (ParseLocalSourceName(state) && Optional(ParseAbiTags(state)))); +} + +// <source-name> ::= <positive length number> <identifier> +static bool ParseSourceName(State *state) { + State copy = *state; + int length = -1; + if (ParseNumber(state, &length) && ParseIdentifier(state, length)) { + return true; + } + *state = copy; + return false; +} + +// <local-source-name> ::= L <source-name> [<discriminator>] +// +// References: +// http://gcc.gnu.org/bugzilla/show_bug.cgi?id=31775 +// http://gcc.gnu.org/viewcvs?view=rev&revision=124467 +static bool ParseLocalSourceName(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'L') && ParseSourceName(state) && + Optional(ParseDiscriminator(state))) { + return true; + } + *state = copy; + return false; +} + +// <number> ::= [n] <non-negative decimal integer> +// If "number_out" is non-null, then *number_out is set to the value of the +// parsed number on success. +static bool ParseNumber(State *state, int *number_out) { + int sign = 1; + if (ParseOneCharToken(state, 'n')) { + sign = -1; + } + const char *p = state->mangled_cur; + int number = 0; + for (;*p != '\0'; ++p) { + if (IsDigit(*p)) { + number = number * 10 + (*p - '0'); + } else { + break; + } + } + if (p != state->mangled_cur) { // Conversion succeeded. + state->mangled_cur = p; + if (number_out != NULL) { + *number_out = number * sign; + } + return true; + } + return false; +} + +// Floating-point literals are encoded using a fixed-length lowercase +// hexadecimal string. +static bool ParseFloatNumber(State *state) { + const char *p = state->mangled_cur; + for (;*p != '\0'; ++p) { + if (!IsDigit(*p) && !(*p >= 'a' && *p <= 'f')) { + break; + } + } + if (p != state->mangled_cur) { // Conversion succeeded. + state->mangled_cur = p; + return true; + } + return false; +} + +// The <seq-id> is a sequence number in base 36, +// using digits and upper case letters +static bool ParseSeqId(State *state) { + const char *p = state->mangled_cur; + for (;*p != '\0'; ++p) { + if (!IsDigit(*p) && !(*p >= 'A' && *p <= 'Z')) { + break; + } + } + if (p != state->mangled_cur) { // Conversion succeeded. + state->mangled_cur = p; + return true; + } + return false; +} + +// <identifier> ::= <unqualified source code identifier> (of given length) +static bool ParseIdentifier(State *state, int length) { + if (length == -1 || + !AtLeastNumCharsRemaining(state->mangled_cur, length)) { + return false; + } + if (IdentifierIsAnonymousNamespace(state, length)) { + MaybeAppend(state, "(anonymous namespace)"); + } else { + MaybeAppendWithLength(state, state->mangled_cur, length); + } + state->mangled_cur += length; + return true; +} + +// <abi-tags> ::= <abi-tag> [<abi-tags>] +static bool ParseAbiTags(State *state) { + State copy = *state; + DisableAppend(state); + if (OneOrMore(ParseAbiTag, state)) { + RestoreAppend(state, copy.append); + return true; + } + *state = copy; + return false; +} + +// <abi-tag> ::= B <source-name> +static bool ParseAbiTag(State *state) { + return ParseOneCharToken(state, 'B') && ParseSourceName(state); +} + +// <operator-name> ::= nw, and other two letters cases +// ::= cv <type> # (cast) +// ::= v <digit> <source-name> # vendor extended operator +static bool ParseOperatorName(State *state) { + if (!AtLeastNumCharsRemaining(state->mangled_cur, 2)) { + return false; + } + // First check with "cv" (cast) case. + State copy = *state; + if (ParseTwoCharToken(state, "cv") && + MaybeAppend(state, "operator ") && + EnterNestedName(state) && + ParseType(state) && + LeaveNestedName(state, copy.nest_level)) { + return true; + } + *state = copy; + + // Then vendor extended operators. + if (ParseOneCharToken(state, 'v') && ParseCharClass(state, "0123456789") && + ParseSourceName(state)) { + return true; + } + *state = copy; + + // Other operator names should start with a lower alphabet followed + // by a lower/upper alphabet. + if (!(IsLower(state->mangled_cur[0]) && + IsAlpha(state->mangled_cur[1]))) { + return false; + } + // We may want to perform a binary search if we really need speed. + const AbbrevPair *p; + for (p = kOperatorList; p->abbrev != NULL; ++p) { + if (state->mangled_cur[0] == p->abbrev[0] && + state->mangled_cur[1] == p->abbrev[1]) { + MaybeAppend(state, "operator"); + if (IsLower(*p->real_name)) { // new, delete, etc. + MaybeAppend(state, " "); + } + MaybeAppend(state, p->real_name); + state->mangled_cur += 2; + return true; + } + } + return false; +} + +// <special-name> ::= TV <type> +// ::= TT <type> +// ::= TI <type> +// ::= TS <type> +// ::= Tc <call-offset> <call-offset> <(base) encoding> +// ::= GV <(object) name> +// ::= T <call-offset> <(base) encoding> +// G++ extensions: +// ::= TC <type> <(offset) number> _ <(base) type> +// ::= TF <type> +// ::= TJ <type> +// ::= GR <name> +// ::= GA <encoding> +// ::= Th <call-offset> <(base) encoding> +// ::= Tv <call-offset> <(base) encoding> +// +// Note: we don't care much about them since they don't appear in +// stack traces. The are special data. +static bool ParseSpecialName(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'T') && + ParseCharClass(state, "VTIS") && + ParseType(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "Tc") && ParseCallOffset(state) && + ParseCallOffset(state) && ParseEncoding(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "GV") && + ParseName(state)) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'T') && ParseCallOffset(state) && + ParseEncoding(state)) { + return true; + } + *state = copy; + + // G++ extensions + if (ParseTwoCharToken(state, "TC") && ParseType(state) && + ParseNumber(state, NULL) && ParseOneCharToken(state, '_') && + DisableAppend(state) && + ParseType(state)) { + RestoreAppend(state, copy.append); + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'T') && ParseCharClass(state, "FJ") && + ParseType(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "GR") && ParseName(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "GA") && ParseEncoding(state)) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'T') && ParseCharClass(state, "hv") && + ParseCallOffset(state) && ParseEncoding(state)) { + return true; + } + *state = copy; + return false; +} + +// <call-offset> ::= h <nv-offset> _ +// ::= v <v-offset> _ +static bool ParseCallOffset(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'h') && + ParseNVOffset(state) && ParseOneCharToken(state, '_')) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'v') && + ParseVOffset(state) && ParseOneCharToken(state, '_')) { + return true; + } + *state = copy; + + return false; +} + +// <nv-offset> ::= <(offset) number> +static bool ParseNVOffset(State *state) { + return ParseNumber(state, NULL); +} + +// <v-offset> ::= <(offset) number> _ <(virtual offset) number> +static bool ParseVOffset(State *state) { + State copy = *state; + if (ParseNumber(state, NULL) && ParseOneCharToken(state, '_') && + ParseNumber(state, NULL)) { + return true; + } + *state = copy; + return false; +} + +// <ctor-dtor-name> ::= C1 | C2 | C3 +// ::= D0 | D1 | D2 +static bool ParseCtorDtorName(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'C') && + ParseCharClass(state, "123")) { + const char * const prev_name = state->prev_name; + const int prev_name_length = state->prev_name_length; + MaybeAppendWithLength(state, prev_name, prev_name_length); + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'D') && + ParseCharClass(state, "012")) { + const char * const prev_name = state->prev_name; + const int prev_name_length = state->prev_name_length; + MaybeAppend(state, "~"); + MaybeAppendWithLength(state, prev_name, prev_name_length); + return true; + } + *state = copy; + return false; +} + +// <type> ::= <CV-qualifiers> <type> +// ::= P <type> # pointer-to +// ::= R <type> # reference-to +// ::= O <type> # rvalue reference-to (C++0x) +// ::= C <type> # complex pair (C 2000) +// ::= G <type> # imaginary (C 2000) +// ::= U <source-name> <type> # vendor extended type qualifier +// ::= <builtin-type> +// ::= <function-type> +// ::= <class-enum-type> +// ::= <array-type> +// ::= <pointer-to-member-type> +// ::= <template-template-param> <template-args> +// ::= <template-param> +// ::= <substitution> +// ::= Dp <type> # pack expansion of (C++0x) +// ::= Dt <expression> E # decltype of an id-expression or class +// # member access (C++0x) +// ::= DT <expression> E # decltype of an expression (C++0x) +// +static bool ParseType(State *state) { + // We should check CV-qualifers, and PRGC things first. + State copy = *state; + if (ParseCVQualifiers(state) && ParseType(state)) { + return true; + } + *state = copy; + + if (ParseCharClass(state, "OPRCG") && ParseType(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "Dp") && ParseType(state)) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'D') && ParseCharClass(state, "tT") && + ParseExpression(state) && ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'U') && ParseSourceName(state) && + ParseType(state)) { + return true; + } + *state = copy; + + if (ParseBuiltinType(state) || + ParseFunctionType(state) || + ParseClassEnumType(state) || + ParseArrayType(state) || + ParsePointerToMemberType(state) || + ParseSubstitution(state)) { + return true; + } + + if (ParseTemplateTemplateParam(state) && + ParseTemplateArgs(state)) { + return true; + } + *state = copy; + + // Less greedy than <template-template-param> <template-args>. + if (ParseTemplateParam(state)) { + return true; + } + + return false; +} + +// <CV-qualifiers> ::= [r] [V] [K] +// We don't allow empty <CV-qualifiers> to avoid infinite loop in +// ParseType(). +static bool ParseCVQualifiers(State *state) { + int num_cv_qualifiers = 0; + num_cv_qualifiers += ParseOneCharToken(state, 'r'); + num_cv_qualifiers += ParseOneCharToken(state, 'V'); + num_cv_qualifiers += ParseOneCharToken(state, 'K'); + return num_cv_qualifiers > 0; +} + +// <builtin-type> ::= v, etc. +// ::= u <source-name> +static bool ParseBuiltinType(State *state) { + const AbbrevPair *p; + for (p = kBuiltinTypeList; p->abbrev != NULL; ++p) { + if (state->mangled_cur[0] == p->abbrev[0]) { + MaybeAppend(state, p->real_name); + ++state->mangled_cur; + return true; + } + } + + State copy = *state; + if (ParseOneCharToken(state, 'u') && ParseSourceName(state)) { + return true; + } + *state = copy; + return false; +} + +// <function-type> ::= F [Y] <bare-function-type> E +static bool ParseFunctionType(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'F') && + Optional(ParseOneCharToken(state, 'Y')) && + ParseBareFunctionType(state) && ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + return false; +} + +// <bare-function-type> ::= <(signature) type>+ +static bool ParseBareFunctionType(State *state) { + State copy = *state; + DisableAppend(state); + if (OneOrMore(ParseType, state)) { + RestoreAppend(state, copy.append); + MaybeAppend(state, "()"); + return true; + } + *state = copy; + return false; +} + +// <class-enum-type> ::= <name> +static bool ParseClassEnumType(State *state) { + return ParseName(state); +} + +// <array-type> ::= A <(positive dimension) number> _ <(element) type> +// ::= A [<(dimension) expression>] _ <(element) type> +static bool ParseArrayType(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'A') && ParseNumber(state, NULL) && + ParseOneCharToken(state, '_') && ParseType(state)) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'A') && Optional(ParseExpression(state)) && + ParseOneCharToken(state, '_') && ParseType(state)) { + return true; + } + *state = copy; + return false; +} + +// <pointer-to-member-type> ::= M <(class) type> <(member) type> +static bool ParsePointerToMemberType(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'M') && ParseType(state) && + ParseType(state)) { + return true; + } + *state = copy; + return false; +} + +// <template-param> ::= T_ +// ::= T <parameter-2 non-negative number> _ +static bool ParseTemplateParam(State *state) { + if (ParseTwoCharToken(state, "T_")) { + MaybeAppend(state, "?"); // We don't support template substitutions. + return true; + } + + State copy = *state; + if (ParseOneCharToken(state, 'T') && ParseNumber(state, NULL) && + ParseOneCharToken(state, '_')) { + MaybeAppend(state, "?"); // We don't support template substitutions. + return true; + } + *state = copy; + return false; +} + + +// <template-template-param> ::= <template-param> +// ::= <substitution> +static bool ParseTemplateTemplateParam(State *state) { + return (ParseTemplateParam(state) || + ParseSubstitution(state)); +} + +// <template-args> ::= I <template-arg>+ E +static bool ParseTemplateArgs(State *state) { + State copy = *state; + DisableAppend(state); + if (ParseOneCharToken(state, 'I') && + OneOrMore(ParseTemplateArg, state) && + ParseOneCharToken(state, 'E')) { + RestoreAppend(state, copy.append); + MaybeAppend(state, "<>"); + return true; + } + *state = copy; + return false; +} + +// <template-arg> ::= <type> +// ::= <expr-primary> +// ::= I <template-arg>* E # argument pack +// ::= J <template-arg>* E # argument pack +// ::= X <expression> E +static bool ParseTemplateArg(State *state) { + State copy = *state; + if ((ParseOneCharToken(state, 'I') || ParseOneCharToken(state, 'J')) && + ZeroOrMore(ParseTemplateArg, state) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + + if (ParseType(state) || + ParseExprPrimary(state)) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'X') && ParseExpression(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + return false; +} + +// <expression> ::= <template-param> +// ::= <expr-primary> +// ::= <unary operator-name> <expression> +// ::= <binary operator-name> <expression> <expression> +// ::= <trinary operator-name> <expression> <expression> +// <expression> +// ::= st <type> +// ::= sr <type> <unqualified-name> <template-args> +// ::= sr <type> <unqualified-name> +static bool ParseExpression(State *state) { + if (ParseTemplateParam(state) || ParseExprPrimary(state)) { + return true; + } + + State copy = *state; + if (ParseOperatorName(state) && + ParseExpression(state) && + ParseExpression(state) && + ParseExpression(state)) { + return true; + } + *state = copy; + + if (ParseOperatorName(state) && + ParseExpression(state) && + ParseExpression(state)) { + return true; + } + *state = copy; + + if (ParseOperatorName(state) && + ParseExpression(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "st") && ParseType(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "sr") && ParseType(state) && + ParseUnqualifiedName(state) && + ParseTemplateArgs(state)) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "sr") && ParseType(state) && + ParseUnqualifiedName(state)) { + return true; + } + *state = copy; + return false; +} + +// <expr-primary> ::= L <type> <(value) number> E +// ::= L <type> <(value) float> E +// ::= L <mangled-name> E +// // A bug in g++'s C++ ABI version 2 (-fabi-version=2). +// ::= LZ <encoding> E +static bool ParseExprPrimary(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'L') && ParseType(state) && + ParseNumber(state, NULL) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'L') && ParseType(state) && + ParseFloatNumber(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'L') && ParseMangledName(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + + if (ParseTwoCharToken(state, "LZ") && ParseEncoding(state) && + ParseOneCharToken(state, 'E')) { + return true; + } + *state = copy; + + return false; +} + +// <local-name> := Z <(function) encoding> E <(entity) name> +// [<discriminator>] +// := Z <(function) encoding> E s [<discriminator>] +static bool ParseLocalName(State *state) { + State copy = *state; + if (ParseOneCharToken(state, 'Z') && ParseEncoding(state) && + ParseOneCharToken(state, 'E') && MaybeAppend(state, "::") && + ParseName(state) && Optional(ParseDiscriminator(state))) { + return true; + } + *state = copy; + + if (ParseOneCharToken(state, 'Z') && ParseEncoding(state) && + ParseTwoCharToken(state, "Es") && Optional(ParseDiscriminator(state))) { + return true; + } + *state = copy; + return false; +} + +// <discriminator> := _ <(non-negative) number> +static bool ParseDiscriminator(State *state) { + State copy = *state; + if (ParseOneCharToken(state, '_') && ParseNumber(state, NULL)) { + return true; + } + *state = copy; + return false; +} + +// <substitution> ::= S_ +// ::= S <seq-id> _ +// ::= St, etc. +static bool ParseSubstitution(State *state) { + if (ParseTwoCharToken(state, "S_")) { + MaybeAppend(state, "?"); // We don't support substitutions. + return true; + } + + State copy = *state; + if (ParseOneCharToken(state, 'S') && ParseSeqId(state) && + ParseOneCharToken(state, '_')) { + MaybeAppend(state, "?"); // We don't support substitutions. + return true; + } + *state = copy; + + // Expand abbreviations like "St" => "std". + if (ParseOneCharToken(state, 'S')) { + const AbbrevPair *p; + for (p = kSubstitutionList; p->abbrev != NULL; ++p) { + if (state->mangled_cur[0] == p->abbrev[1]) { + MaybeAppend(state, "std"); + if (p->real_name[0] != '\0') { + MaybeAppend(state, "::"); + MaybeAppend(state, p->real_name); + } + ++state->mangled_cur; + return true; + } + } + } + *state = copy; + return false; +} + +// Parse <mangled-name>, optionally followed by either a function-clone suffix +// or version suffix. Returns true only if all of "mangled_cur" was consumed. +static bool ParseTopLevelMangledName(State *state) { + if (ParseMangledName(state)) { + if (state->mangled_cur[0] != '\0') { + // Drop trailing function clone suffix, if any. + if (IsFunctionCloneSuffix(state->mangled_cur)) { + return true; + } + // Append trailing version suffix if any. + // ex. _Z3foo@@GLIBCXX_3.4 + if (state->mangled_cur[0] == '@') { + MaybeAppend(state, state->mangled_cur); + return true; + } + return false; // Unconsumed suffix. + } + return true; + } + return false; +} +#endif + +// The demangler entry point. +bool Demangle(const char *mangled, char *out, int out_size) { +#if defined(OS_WINDOWS) + // When built with incremental linking, the Windows debugger + // library provides a more complicated `Symbol->Name` with the + // Incremental Linking Table offset, which looks like + // `@ILT+1105(?func@Foo@@SAXH@Z)`. However, the demangler expects + // only the mangled symbol, `?func@Foo@@SAXH@Z`. Fortunately, the + // mangled symbol is guaranteed not to have parentheses, + // so we search for `(` and extract up to `)`. + // + // Since we may be in a signal handler here, we cannot use `std::string`. + char buffer[1024]; // Big enough for a sane symbol. + const char *lparen = strchr(mangled, '('); + if (lparen) { + // Extract the string `(?...)` + const char *rparen = strchr(lparen, ')'); + size_t length = rparen - lparen - 1; + strncpy(buffer, lparen + 1, length); + buffer[length] = '\0'; + mangled = buffer; + } // Else the symbol wasn't inside a set of parentheses + // We use the ANSI version to ensure the string type is always `char *`. + return UnDecorateSymbolName(mangled, out, out_size, UNDNAME_COMPLETE); +#else + State state; + InitState(&state, mangled, out, out_size); + return ParseTopLevelMangledName(&state) && !state.overflowed; +#endif +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/demangle.h b/third_party/glog/src/demangle.h new file mode 100644 index 000000000000..991b6ffcf2e6 --- /dev/null +++ b/third_party/glog/src/demangle.h @@ -0,0 +1,85 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// An async-signal-safe and thread-safe demangler for Itanium C++ ABI +// (aka G++ V3 ABI). + +// The demangler is implemented to be used in async signal handlers to +// symbolize stack traces. We cannot use libstdc++'s +// abi::__cxa_demangle() in such signal handlers since it's not async +// signal safe (it uses malloc() internally). +// +// Note that this demangler doesn't support full demangling. More +// specifically, it doesn't print types of function parameters and +// types of template arguments. It just skips them. However, it's +// still very useful to extract basic information such as class, +// function, constructor, destructor, and operator names. +// +// See the implementation note in demangle.cc if you are interested. +// +// Example: +// +// | Mangled Name | The Demangler | abi::__cxa_demangle() +// |---------------|---------------|----------------------- +// | _Z1fv | f() | f() +// | _Z1fi | f() | f(int) +// | _Z3foo3bar | foo() | foo(bar) +// | _Z1fIiEvi | f<>() | void f<int>(int) +// | _ZN1N1fE | N::f | N::f +// | _ZN3Foo3BarEv | Foo::Bar() | Foo::Bar() +// | _Zrm1XS_" | operator%() | operator%(X, X) +// | _ZN3FooC1Ev | Foo::Foo() | Foo::Foo() +// | _Z1fSs | f() | f(std::basic_string<char, +// | | | std::char_traits<char>, +// | | | std::allocator<char> >) +// +// See the unit test for more examples. +// +// Note: we might want to write demanglers for ABIs other than Itanium +// C++ ABI in the future. +// + +#ifndef BASE_DEMANGLE_H_ +#define BASE_DEMANGLE_H_ + +#include "config.h" +#include "glog/logging.h" + +_START_GOOGLE_NAMESPACE_ + +// Demangle "mangled". On success, return true and write the +// demangled symbol name to "out". Otherwise, return false. +// "out" is modified even if demangling is unsuccessful. +bool GOOGLE_GLOG_DLL_DECL Demangle(const char *mangled, char *out, int out_size); + +_END_GOOGLE_NAMESPACE_ + +#endif // BASE_DEMANGLE_H_ diff --git a/third_party/glog/src/demangle_unittest.cc b/third_party/glog/src/demangle_unittest.cc new file mode 100644 index 000000000000..be483411f1a0 --- /dev/null +++ b/third_party/glog/src/demangle_unittest.cc @@ -0,0 +1,168 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// Unit tests for functions in demangle.c. + +#include "utilities.h" + +#include <iostream> +#include <fstream> +#include <string> +#include "glog/logging.h" +#include "demangle.h" +#include "googletest.h" +#include "config.h" + +#ifdef HAVE_LIB_GFLAGS +#include <gflags/gflags.h> +using namespace GFLAGS_NAMESPACE; +#endif + +GLOG_DEFINE_bool(demangle_filter, false, + "Run demangle_unittest in filter mode"); + +using namespace std; +using namespace GOOGLE_NAMESPACE; + +// A wrapper function for Demangle() to make the unit test simple. +static const char *DemangleIt(const char * const mangled) { + static char demangled[4096]; + if (Demangle(mangled, demangled, sizeof(demangled))) { + return demangled; + } else { + return mangled; + } +} + +#if defined(OS_WINDOWS) + +TEST(Demangle, Windows) { + EXPECT_STREQ( + "public: static void __cdecl Foo::func(int)", + DemangleIt("?func@Foo@@SAXH@Z")); + EXPECT_STREQ( + "public: static void __cdecl Foo::func(int)", + DemangleIt("@ILT+1105(?func@Foo@@SAXH@Z)")); + EXPECT_STREQ( + "int __cdecl foobarArray(int * const)", + DemangleIt("?foobarArray@@YAHQAH@Z")); +} + +#else + +// Test corner cases of bounary conditions. +TEST(Demangle, CornerCases) { + const size_t size = 10; + char tmp[size] = { 0 }; + const char *demangled = "foobar()"; + const char *mangled = "_Z6foobarv"; + EXPECT_TRUE(Demangle(mangled, tmp, sizeof(tmp))); + // sizeof("foobar()") == size - 1 + EXPECT_STREQ(demangled, tmp); + EXPECT_TRUE(Demangle(mangled, tmp, size - 1)); + EXPECT_STREQ(demangled, tmp); + EXPECT_FALSE(Demangle(mangled, tmp, size - 2)); // Not enough. + EXPECT_FALSE(Demangle(mangled, tmp, 1)); + EXPECT_FALSE(Demangle(mangled, tmp, 0)); + EXPECT_FALSE(Demangle(mangled, NULL, 0)); // Should not cause SEGV. +} + +// Test handling of functions suffixed with .clone.N, which is used by GCC +// 4.5.x, and .constprop.N and .isra.N, which are used by GCC 4.6.x. These +// suffixes are used to indicate functions which have been cloned during +// optimization. We ignore these suffixes. +TEST(Demangle, Clones) { + char tmp[20]; + EXPECT_TRUE(Demangle("_ZL3Foov", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.clone.3", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.constprop.80", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.isra.18", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + EXPECT_TRUE(Demangle("_ZL3Foov.isra.2.constprop.18", tmp, sizeof(tmp))); + EXPECT_STREQ("Foo()", tmp); + // Invalid (truncated), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.clo", tmp, sizeof(tmp))); + // Invalid (.clone. not followed by number), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.clone.", tmp, sizeof(tmp))); + // Invalid (.clone. followed by non-number), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.clone.foo", tmp, sizeof(tmp))); + // Invalid (.constprop. not followed by number), should not demangle. + EXPECT_FALSE(Demangle("_ZL3Foov.isra.2.constprop.", tmp, sizeof(tmp))); +} + +TEST(Demangle, FromFile) { + string test_file = FLAGS_test_srcdir + "/src/demangle_unittest.txt"; + ifstream f(test_file.c_str()); // The file should exist. + EXPECT_FALSE(f.fail()); + + string line; + while (getline(f, line)) { + // Lines start with '#' are considered as comments. + if (line.empty() || line[0] == '#') { + continue; + } + // Each line should contain a mangled name and a demangled name + // separated by '\t'. Example: "_Z3foo\tfoo" + string::size_type tab_pos = line.find('\t'); + EXPECT_NE(string::npos, tab_pos); + string mangled = line.substr(0, tab_pos); + string demangled = line.substr(tab_pos + 1); + EXPECT_EQ(demangled, DemangleIt(mangled.c_str())); + } +} + +#endif + +int main(int argc, char **argv) { +#ifdef HAVE_LIB_GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif + InitGoogleTest(&argc, argv); + + FLAGS_logtostderr = true; + InitGoogleLogging(argv[0]); + if (FLAGS_demangle_filter) { + // Read from cin and write to cout. + string line; + while (getline(cin, line, '\n')) { + cout << DemangleIt(line.c_str()) << endl; + } + return 0; + } else if (argc > 1) { + cout << DemangleIt(argv[1]) << endl; + return 0; + } else { + return RUN_ALL_TESTS(); + } +} diff --git a/third_party/glog/src/demangle_unittest.sh b/third_party/glog/src/demangle_unittest.sh new file mode 100755 index 000000000000..91deee2198da --- /dev/null +++ b/third_party/glog/src/demangle_unittest.sh @@ -0,0 +1,95 @@ +#! /bin/sh +# +# Copyright (c) 2006, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Author: Satoru Takabayashi +# +# Unit tests for demangle.c with a real binary. + +set -e + +die () { + echo $1 + exit 1 +} + +BINDIR=".libs" +LIBGLOG="$BINDIR/libglog.so" + +DEMANGLER="$BINDIR/demangle_unittest" + +if test -e "$DEMANGLER"; then + # We need shared object. + export LD_LIBRARY_PATH=$BINDIR + export DYLD_LIBRARY_PATH=$BINDIR +else + # For windows + DEMANGLER="./demangle_unittest.exe" + if ! test -e "$DEMANGLER"; then + echo "We coundn't find demangle_unittest binary." + exit 1 + fi +fi + +# Extract C++ mangled symbols from libbase.so. +NM_OUTPUT="demangle.nm" +nm "$LIBGLOG" | perl -nle 'print $1 if /\s(_Z\S+$)/' > "$NM_OUTPUT" + +# Check if mangled symbols exist. If there are none, we quit. +# The binary is more likely compiled with GCC 2.95 or something old. +if ! grep --quiet '^_Z' "$NM_OUTPUT"; then + echo "PASS" + exit 0 +fi + +# Demangle the symbols using our demangler. +DM_OUTPUT="demangle.dm" +GLOG_demangle_filter=1 "$DEMANGLER" --demangle_filter < "$NM_OUTPUT" > "$DM_OUTPUT" + +# Calculate the numbers of lines. +NM_LINES=`wc -l "$NM_OUTPUT" | awk '{ print $1 }'` +DM_LINES=`wc -l "$DM_OUTPUT" | awk '{ print $1 }'` + +# Compare the numbers of lines. They must be the same. +if test "$NM_LINES" != "$DM_LINES"; then + die "$NM_OUTPUT and $DM_OUTPUT don't have the same numbers of lines" +fi + +# Check if mangled symbols exist. They must not exist. +if grep --quiet '^_Z' "$DM_OUTPUT"; then + MANGLED=`grep '^_Z' "$DM_OUTPUT" | wc -l | awk '{ print \$1 }'` + echo "Mangled symbols ($MANGLED out of $NM_LINES) found in $DM_OUTPUT:" + grep '^_Z' "$DM_OUTPUT" + die "Mangled symbols ($MANGLED out of $NM_LINES) found in $DM_OUTPUT" +fi + +# All C++ symbols are demangled successfully. +echo "PASS" +exit 0 diff --git a/third_party/glog/src/demangle_unittest.txt b/third_party/glog/src/demangle_unittest.txt new file mode 100644 index 000000000000..07e28bce2f8f --- /dev/null +++ b/third_party/glog/src/demangle_unittest.txt @@ -0,0 +1,145 @@ +# Test caces for demangle_unittest. Each line consists of a +# tab-separated pair of mangled and demangled symbol names. + +# Constructors and destructors. +_ZN3FooC1Ev Foo::Foo() +_ZN3FooD1Ev Foo::~Foo() +_ZNSoD0Ev std::ostream::~ostream() + +# G++ extensions. +_ZTCN10LogMessage9LogStreamE0_So LogMessage::LogStream +_ZTv0_n12_N10LogMessage9LogStreamD0Ev LogMessage::LogStream::~LogStream() +_ZThn4_N7icu_3_410UnicodeSetD0Ev icu_3_4::UnicodeSet::~UnicodeSet() + +# A bug in g++'s C++ ABI version 2 (-fabi-version=2). +_ZN7NSSInfoI5groupjjXadL_Z10getgrgid_rEELZ19nss_getgrgid_r_nameEEC1Ei NSSInfo<>::NSSInfo() + +# C linkage symbol names. Should keep them untouched. +main main +Demangle Demangle +_ZERO _ZERO + +# Cast operator. +_Zcviv operator int() +_ZN3foocviEv foo::operator int() + +# Versioned symbols. +_Z3Foo@GLIBCXX_3.4 Foo@GLIBCXX_3.4 +_Z3Foo@@GLIBCXX_3.4 Foo@@GLIBCXX_3.4 + +# Abbreviations. +_ZNSaE std::allocator +_ZNSbE std::basic_string +_ZNSdE std::iostream +_ZNSiE std::istream +_ZNSoE std::ostream +_ZNSsE std::string + +# Substitutions. We just replace them with ?. +_ZN3fooS_E foo::? +_ZN3foo3barS0_E foo::bar::? +_ZNcvT_IiEEv operator ?<>() + +# "<< <" case. +_ZlsI3fooE operator<< <> + +# ABI tags. +_Z1AB3barB3foo A +_ZN3fooL3barB5cxx11E foo::bar + +# Random things we found interesting. +_ZN3FooISt6vectorISsSaISsEEEclEv Foo<>::operator()() +_ZTI9Callback1IiE Callback1<> +_ZN7icu_3_47UMemorynwEj icu_3_4::UMemory::operator new() +_ZNSt6vectorIbE9push_backE std::vector<>::push_back +_ZNSt6vectorIbSaIbEE9push_backEb std::vector<>::push_back() +_ZlsRSoRK15PRIVATE_Counter operator<<() +_ZSt6fill_nIPPN9__gnu_cxx15_Hashtable_nodeISt4pairIKPKcjEEEjS8_ET_SA_T0_RKT1_ std::fill_n<>() +_ZZ3FoovE3Bar Foo()::Bar +_ZGVZ7UpTimervE8up_timer UpTimer()::up_timer + +# Test cases from gcc-4.1.0/libstdc++-v3/testsuite/demangle. +# Collected by: +# % grep verify_demangle **/*.cc | perl -nle 'print $1 if /"(_Z.*?)"/' | +# sort | uniq +# +# Note that the following symbols are invalid. +# That's why they are not demangled. +# - _ZNZN1N1fEiE1X1gE +# - _ZNZN1N1fEiE1X1gEv +# - _Z1xINiEE +_Z1fA37_iPS_ f() +_Z1fAszL_ZZNK1N1A1fEvE3foo_0E_i f() +_Z1fI1APS0_PKS0_EvT_T0_T1_PA4_S3_M1CS8_ f<>() +_Z1fI1XENT_1tES2_ f<>() +_Z1fI1XEvPVN1AIT_E1TE f<>() +_Z1fILi1ELc120EEv1AIXplT_cviLd4028ae147ae147aeEEE f<>() +_Z1fILi1ELc120EEv1AIXplT_cviLf3f800000EEE f<>() +_Z1fILi5E1AEvN1CIXqugtT_Li0ELi1ELi2EEE1qE f<>() +_Z1fILi5E1AEvN1CIXstN1T1tEEXszsrS2_1tEE1qE f<>() +_Z1fILi5EEvN1AIXcvimlT_Li22EEE1qE f<>() +_Z1fIiEvi f<>() +_Z1fKPFiiE f() +_Z1fM1AFivEPS0_ f() +_Z1fM1AKFivE f() +_Z1fM1AKFvvE f() +_Z1fPFPA1_ivE f() +_Z1fPFYPFiiEiE f() +_Z1fPFvvEM1SFvvE f() +_Z1fPKM1AFivE f() +_Z1fi f() +_Z1fv f() +_Z1jM1AFivEPS1_ j() +_Z1rM1GFivEMS_KFivES_M1HFivES1_4whatIKS_E5what2IS8_ES3_ r() +_Z1sPA37_iPS0_ s() +_Z1xINiEE _Z1xINiEE +_Z3absILi11EEvv abs<>() +_Z3foo3bar foo() +_Z3foo5Hello5WorldS0_S_ foo() +_Z3fooA30_A_i foo() +_Z3fooIA6_KiEvA9_KT_rVPrS4_ foo<>() +_Z3fooILi2EEvRAplT_Li1E_i foo<>() +_Z3fooIiFvdEiEvv foo<>() +_Z3fooPM2ABi foo() +_Z3fooc foo() +_Z3fooiPiPS_PS0_PS1_PS2_PS3_PS4_PS5_PS6_PS7_PS8_PS9_PSA_PSB_PSC_ foo() +_Z3kooPA28_A30_i koo() +_Z4makeI7FactoryiET_IT0_Ev make<>() +_Z5firstI3DuoEvS0_ first<>() +_Z5firstI3DuoEvT_ first<>() +_Z9hairyfuncM1YKFPVPFrPA2_PM1XKFKPA3_ilEPcEiE hairyfunc() +_ZGVN5libcw24_GLOBAL__N_cbll.cc0ZhUKa23compiler_bug_workaroundISt6vectorINS_13omanip_id_tctINS_5debug32memblk_types_manipulator_data_ctEEESaIS6_EEE3idsE libcw::(anonymous namespace)::compiler_bug_workaround<>::ids +_ZN12libcw_app_ct10add_optionIS_EEvMT_FvPKcES3_cS3_S3_ libcw_app_ct::add_option<>() +_ZN1AIfEcvT_IiEEv A<>::operator ?<>() +_ZN1N1TIiiE2mfES0_IddE N::T<>::mf() +_ZN1N1fE N::f +_ZN1f1fE f::f +_ZN3FooIA4_iE3barE Foo<>::bar +_ZN5Arena5levelE Arena::level +_ZN5StackIiiE5levelE Stack<>::level +_ZN5libcw5debug13cwprint_usingINS_9_private_12GlobalObjectEEENS0_17cwprint_using_tctIT_EERKS5_MS5_KFvRSt7ostreamE libcw::debug::cwprint_using<>() +_ZN6System5Sound4beepEv System::Sound::beep() +_ZNKSt14priority_queueIP27timer_event_request_base_ctSt5dequeIS1_SaIS1_EE13timer_greaterE3topEv std::priority_queue<>::top() +_ZNKSt15_Deque_iteratorIP15memory_block_stRKS1_PS2_EeqERKS5_ std::_Deque_iterator<>::operator==() +_ZNKSt17__normal_iteratorIPK6optionSt6vectorIS0_SaIS0_EEEmiERKS6_ std::__normal_iterator<>::operator-() +_ZNSbIcSt11char_traitsIcEN5libcw5debug27no_alloc_checking_allocatorEE12_S_constructIPcEES6_T_S7_RKS3_ std::basic_string<>::_S_construct<>() +_ZNSt13_Alloc_traitsISbIcSt18string_char_traitsIcEN5libcw5debug9_private_17allocator_adaptorIcSt24__default_alloc_templateILb0ELi327664EELb1EEEENS5_IS9_S7_Lb1EEEE15_S_instancelessE std::_Alloc_traits<>::_S_instanceless +_ZNSt3_In4wardE std::_In::ward +_ZNZN1N1fEiE1X1gE _ZNZN1N1fEiE1X1gE +_ZNZN1N1fEiE1X1gEv _ZNZN1N1fEiE1X1gEv +_ZSt1BISt1DIP1ARKS2_PS3_ES0_IS2_RS2_PS2_ES2_ET0_T_SB_SA_PT1_ std::B<>() +_ZSt5state std::state +_ZTI7a_class a_class +_ZZN1N1fEiE1p N::f()::p +_ZZN1N1fEiEs N::f() +_ZlsRK1XS1_ operator<<() +_ZlsRKU3fooU4bart1XS0_ operator<<() +_ZlsRKU3fooU4bart1XS2_ operator<<() +_ZlsRSoRKSs operator<<() +_ZngILi42EEvN1AIXplT_Li2EEE1TE operator-<>() +_ZplR1XS0_ operator+() +_Zrm1XS_ operator%() + +# Template argument packs can start with I or J. +_Z3addIIiEEvDpT_ add<>() +_Z3addIJiEEvDpT_ add<>() diff --git a/third_party/glog/src/glog/log_severity.h b/third_party/glog/src/glog/log_severity.h new file mode 100644 index 000000000000..987cc57dae91 --- /dev/null +++ b/third_party/glog/src/glog/log_severity.h @@ -0,0 +1,107 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef BASE_LOG_SEVERITY_H__ +#define BASE_LOG_SEVERITY_H__ + +// The recommended semantics of the log levels are as follows: +// +// INFO: +// Use for state changes or other major events, or to aid debugging. +// WARNING: +// Use for undesired but relatively expected events, which may indicate a +// problem +// ERROR: +// Use for undesired and unexpected events that the program can recover from. +// All ERRORs should be actionable - it should be appropriate to file a bug +// whenever an ERROR occurs in production. +// FATAL: +// Use for undesired and unexpected events that the program cannot recover +// from. + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +// Variables of type LogSeverity are widely taken to lie in the range +// [0, NUM_SEVERITIES-1]. Be careful to preserve this assumption if +// you ever need to change their values or add a new severity. +typedef int LogSeverity; + +const int GLOG_INFO = 0, GLOG_WARNING = 1, GLOG_ERROR = 2, GLOG_FATAL = 3, + NUM_SEVERITIES = 4; +#ifndef GLOG_NO_ABBREVIATED_SEVERITIES +# ifdef ERROR +# error ERROR macro is defined. Define GLOG_NO_ABBREVIATED_SEVERITIES before including logging.h. See the document for detail. +# endif +const int INFO = GLOG_INFO, WARNING = GLOG_WARNING, + ERROR = GLOG_ERROR, FATAL = GLOG_FATAL; +#endif + +// DFATAL is FATAL in debug mode, ERROR in normal mode +#ifdef NDEBUG +#define DFATAL_LEVEL ERROR +#else +#define DFATAL_LEVEL FATAL +#endif + +extern GOOGLE_GLOG_DLL_DECL const char* const LogSeverityNames[NUM_SEVERITIES]; + +// NDEBUG usage helpers related to (RAW_)DCHECK: +// +// DEBUG_MODE is for small !NDEBUG uses like +// if (DEBUG_MODE) foo.CheckThatFoo(); +// instead of substantially more verbose +// #ifndef NDEBUG +// foo.CheckThatFoo(); +// #endif +// +// IF_DEBUG_MODE is for small !NDEBUG uses like +// IF_DEBUG_MODE( string error; ) +// DCHECK(Foo(&error)) << error; +// instead of substantially more verbose +// #ifndef NDEBUG +// string error; +// DCHECK(Foo(&error)) << error; +// #endif +// +#ifdef NDEBUG +enum { DEBUG_MODE = 0 }; +#define IF_DEBUG_MODE(x) +#else +enum { DEBUG_MODE = 1 }; +#define IF_DEBUG_MODE(x) x +#endif + +#endif // BASE_LOG_SEVERITY_H__ diff --git a/third_party/glog/src/glog/logging.h.in b/third_party/glog/src/glog/logging.h.in new file mode 100644 index 000000000000..1e552fc1a177 --- /dev/null +++ b/third_party/glog/src/glog/logging.h.in @@ -0,0 +1,1689 @@ +// Copyright (c) 1999, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Ray Sidney +// +// This file contains #include information about logging-related stuff. +// Pretty much everybody needs to #include this file so that they can +// log various happenings. +// +#ifndef _LOGGING_H_ +#define _LOGGING_H_ + +#include <errno.h> +#include <string.h> +#include <time.h> +#include <iosfwd> +#include <ostream> +#include <sstream> +#include <string> +#if @ac_cv_have_unistd_h@ +# include <unistd.h> +#endif +#include <vector> + +#if defined(_MSC_VER) +#define GLOG_MSVC_PUSH_DISABLE_WARNING(n) __pragma(warning(push)) \ + __pragma(warning(disable:n)) +#define GLOG_MSVC_POP_WARNING() __pragma(warning(pop)) +#else +#define GLOG_MSVC_PUSH_DISABLE_WARNING(n) +#define GLOG_MSVC_POP_WARNING() +#endif + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +// We care a lot about number of bits things take up. Unfortunately, +// systems define their bit-specific ints in a lot of different ways. +// We use our own way, and have a typedef to get there. +// Note: these commands below may look like "#if 1" or "#if 0", but +// that's because they were constructed that way at ./configure time. +// Look at logging.h.in to see how they're calculated (based on your config). +#if @ac_cv_have_stdint_h@ +#include <stdint.h> // the normal place uint16_t is defined +#endif +#if @ac_cv_have_systypes_h@ +#include <sys/types.h> // the normal place u_int16_t is defined +#endif +#if @ac_cv_have_inttypes_h@ +#include <inttypes.h> // a third place for uint16_t or u_int16_t +#endif + +#if @ac_cv_have_libgflags@ +#include <gflags/gflags.h> +#endif + +@ac_google_start_namespace@ + +#if @ac_cv_have_uint16_t@ // the C99 format +typedef int32_t int32; +typedef uint32_t uint32; +typedef int64_t int64; +typedef uint64_t uint64; +#elif @ac_cv_have_u_int16_t@ // the BSD format +typedef int32_t int32; +typedef u_int32_t uint32; +typedef int64_t int64; +typedef u_int64_t uint64; +#elif @ac_cv_have___uint16@ // the windows (vc7) format +typedef __int32 int32; +typedef unsigned __int32 uint32; +typedef __int64 int64; +typedef unsigned __int64 uint64; +#else +#error Do not know how to define a 32-bit integer quantity on your system +#endif + +@ac_google_end_namespace@ + +// The global value of GOOGLE_STRIP_LOG. All the messages logged to +// LOG(XXX) with severity less than GOOGLE_STRIP_LOG will not be displayed. +// If it can be determined at compile time that the message will not be +// printed, the statement will be compiled out. +// +// Example: to strip out all INFO and WARNING messages, use the value +// of 2 below. To make an exception for WARNING messages from a single +// file, add "#define GOOGLE_STRIP_LOG 1" to that file _before_ including +// base/logging.h +#ifndef GOOGLE_STRIP_LOG +#define GOOGLE_STRIP_LOG 0 +#endif + +// GCC can be told that a certain branch is not likely to be taken (for +// instance, a CHECK failure), and use that information in static analysis. +// Giving it this information can help it optimize for the common case in +// the absence of better information (ie. -fprofile-arcs). +// +#ifndef GOOGLE_PREDICT_BRANCH_NOT_TAKEN +#if @ac_cv_have___builtin_expect@ +#define GOOGLE_PREDICT_BRANCH_NOT_TAKEN(x) (__builtin_expect(x, 0)) +#else +#define GOOGLE_PREDICT_BRANCH_NOT_TAKEN(x) x +#endif +#endif + +#ifndef GOOGLE_PREDICT_FALSE +#if @ac_cv_have___builtin_expect@ +#define GOOGLE_PREDICT_FALSE(x) (__builtin_expect(x, 0)) +#else +#define GOOGLE_PREDICT_FALSE(x) x +#endif +#endif + +#ifndef GOOGLE_PREDICT_TRUE +#if @ac_cv_have___builtin_expect@ +#define GOOGLE_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) +#else +#define GOOGLE_PREDICT_TRUE(x) x +#endif +#endif + + +// Make a bunch of macros for logging. The way to log things is to stream +// things to LOG(<a particular severity level>). E.g., +// +// LOG(INFO) << "Found " << num_cookies << " cookies"; +// +// You can capture log messages in a string, rather than reporting them +// immediately: +// +// vector<string> errors; +// LOG_STRING(ERROR, &errors) << "Couldn't parse cookie #" << cookie_num; +// +// This pushes back the new error onto 'errors'; if given a NULL pointer, +// it reports the error via LOG(ERROR). +// +// You can also do conditional logging: +// +// LOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; +// +// You can also do occasional logging (log every n'th occurrence of an +// event): +// +// LOG_EVERY_N(INFO, 10) << "Got the " << google::COUNTER << "th cookie"; +// +// The above will cause log messages to be output on the 1st, 11th, 21st, ... +// times it is executed. Note that the special google::COUNTER value is used +// to identify which repetition is happening. +// +// You can also do occasional conditional logging (log every n'th +// occurrence of an event, when condition is satisfied): +// +// LOG_IF_EVERY_N(INFO, (size > 1024), 10) << "Got the " << google::COUNTER +// << "th big cookie"; +// +// You can log messages the first N times your code executes a line. E.g. +// +// LOG_FIRST_N(INFO, 20) << "Got the " << google::COUNTER << "th cookie"; +// +// Outputs log messages for the first 20 times it is executed. +// +// Analogous SYSLOG, SYSLOG_IF, and SYSLOG_EVERY_N macros are available. +// These log to syslog as well as to the normal logs. If you use these at +// all, you need to be aware that syslog can drastically reduce performance, +// especially if it is configured for remote logging! Don't use these +// unless you fully understand this and have a concrete need to use them. +// Even then, try to minimize your use of them. +// +// There are also "debug mode" logging macros like the ones above: +// +// DLOG(INFO) << "Found cookies"; +// +// DLOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; +// +// DLOG_EVERY_N(INFO, 10) << "Got the " << google::COUNTER << "th cookie"; +// +// All "debug mode" logging is compiled away to nothing for non-debug mode +// compiles. +// +// We also have +// +// LOG_ASSERT(assertion); +// DLOG_ASSERT(assertion); +// +// which is syntactic sugar for {,D}LOG_IF(FATAL, assert fails) << assertion; +// +// There are "verbose level" logging macros. They look like +// +// VLOG(1) << "I'm printed when you run the program with --v=1 or more"; +// VLOG(2) << "I'm printed when you run the program with --v=2 or more"; +// +// These always log at the INFO log level (when they log at all). +// The verbose logging can also be turned on module-by-module. For instance, +// --vmodule=mapreduce=2,file=1,gfs*=3 --v=0 +// will cause: +// a. VLOG(2) and lower messages to be printed from mapreduce.{h,cc} +// b. VLOG(1) and lower messages to be printed from file.{h,cc} +// c. VLOG(3) and lower messages to be printed from files prefixed with "gfs" +// d. VLOG(0) and lower messages to be printed from elsewhere +// +// The wildcarding functionality shown by (c) supports both '*' (match +// 0 or more characters) and '?' (match any single character) wildcards. +// +// There's also VLOG_IS_ON(n) "verbose level" condition macro. To be used as +// +// if (VLOG_IS_ON(2)) { +// // do some logging preparation and logging +// // that can't be accomplished with just VLOG(2) << ...; +// } +// +// There are also VLOG_IF, VLOG_EVERY_N and VLOG_IF_EVERY_N "verbose level" +// condition macros for sample cases, when some extra computation and +// preparation for logs is not needed. +// VLOG_IF(1, (size > 1024)) +// << "I'm printed when size is more than 1024 and when you run the " +// "program with --v=1 or more"; +// VLOG_EVERY_N(1, 10) +// << "I'm printed every 10th occurrence, and when you run the program " +// "with --v=1 or more. Present occurence is " << google::COUNTER; +// VLOG_IF_EVERY_N(1, (size > 1024), 10) +// << "I'm printed on every 10th occurence of case when size is more " +// " than 1024, when you run the program with --v=1 or more. "; +// "Present occurence is " << google::COUNTER; +// +// The supported severity levels for macros that allow you to specify one +// are (in increasing order of severity) INFO, WARNING, ERROR, and FATAL. +// Note that messages of a given severity are logged not only in the +// logfile for that severity, but also in all logfiles of lower severity. +// E.g., a message of severity FATAL will be logged to the logfiles of +// severity FATAL, ERROR, WARNING, and INFO. +// +// There is also the special severity of DFATAL, which logs FATAL in +// debug mode, ERROR in normal mode. +// +// Very important: logging a message at the FATAL severity level causes +// the program to terminate (after the message is logged). +// +// Unless otherwise specified, logs will be written to the filename +// "<program name>.<hostname>.<user name>.log.<severity level>.", followed +// by the date, time, and pid (you can't prevent the date, time, and pid +// from being in the filename). +// +// The logging code takes two flags: +// --v=# set the verbose level +// --logtostderr log all the messages to stderr instead of to logfiles + +// LOG LINE PREFIX FORMAT +// +// Log lines have this form: +// +// Lyyyymmdd hh:mm:ss.uuuuuu threadid file:line] msg... +// +// where the fields are defined as follows: +// +// L A single character, representing the log level +// (eg 'I' for INFO) +// yyyy The year +// mm The month (zero padded; ie May is '05') +// dd The day (zero padded) +// hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds +// threadid The space-padded thread ID as returned by GetTID() +// (this matches the PID on Linux) +// file The file name +// line The line number +// msg The user-supplied message +// +// Example: +// +// I1103 11:57:31.739339 24395 google.cc:2341] Command line: ./some_prog +// I1103 11:57:31.739403 24395 google.cc:2342] Process id 24395 +// +// NOTE: although the microseconds are useful for comparing events on +// a single machine, clocks on different machines may not be well +// synchronized. Hence, use caution when comparing the low bits of +// timestamps from different machines. + +#ifndef DECLARE_VARIABLE +#define MUST_UNDEF_GFLAGS_DECLARE_MACROS +#define DECLARE_VARIABLE(type, shorttype, name, tn) \ + namespace fL##shorttype { \ + extern GOOGLE_GLOG_DLL_DECL type FLAGS_##name; \ + } \ + using fL##shorttype::FLAGS_##name + +// bool specialization +#define DECLARE_bool(name) \ + DECLARE_VARIABLE(bool, B, name, bool) + +// int32 specialization +#define DECLARE_int32(name) \ + DECLARE_VARIABLE(@ac_google_namespace@::int32, I, name, int32) + +// Special case for string, because we have to specify the namespace +// std::string, which doesn't play nicely with our FLAG__namespace hackery. +#define DECLARE_string(name) \ + namespace fLS { \ + extern GOOGLE_GLOG_DLL_DECL std::string& FLAGS_##name; \ + } \ + using fLS::FLAGS_##name +#endif + +// Set whether appending a timestamp to the log file name +DECLARE_bool(timestamp_in_logfile_name); + +// Set whether log messages go to stderr instead of logfiles +DECLARE_bool(logtostderr); + +// Set whether log messages go to stderr in addition to logfiles. +DECLARE_bool(alsologtostderr); + +// Set color messages logged to stderr (if supported by terminal). +DECLARE_bool(colorlogtostderr); + +// Log messages at a level >= this flag are automatically sent to +// stderr in addition to log files. +DECLARE_int32(stderrthreshold); + +// Set whether the log prefix should be prepended to each line of output. +DECLARE_bool(log_prefix); + +// Log messages at a level <= this flag are buffered. +// Log messages at a higher level are flushed immediately. +DECLARE_int32(logbuflevel); + +// Sets the maximum number of seconds which logs may be buffered for. +DECLARE_int32(logbufsecs); + +// Log suppression level: messages logged at a lower level than this +// are suppressed. +DECLARE_int32(minloglevel); + +// If specified, logfiles are written into this directory instead of the +// default logging directory. +DECLARE_string(log_dir); + +// Set the log file mode. +DECLARE_int32(logfile_mode); + +// Sets the path of the directory into which to put additional links +// to the log files. +DECLARE_string(log_link); + +DECLARE_int32(v); // in vlog_is_on.cc + +// Sets the maximum log file size (in MB). +DECLARE_int32(max_log_size); + +// Sets whether to avoid logging to the disk if the disk is full. +DECLARE_bool(stop_logging_if_full_disk); + +#ifdef MUST_UNDEF_GFLAGS_DECLARE_MACROS +#undef MUST_UNDEF_GFLAGS_DECLARE_MACROS +#undef DECLARE_VARIABLE +#undef DECLARE_bool +#undef DECLARE_int32 +#undef DECLARE_string +#endif + +// Log messages below the GOOGLE_STRIP_LOG level will be compiled away for +// security reasons. See LOG(severtiy) below. + +// A few definitions of macros that don't generate much code. Since +// LOG(INFO) and its ilk are used all over our code, it's +// better to have compact code for these operations. + +#if GOOGLE_STRIP_LOG == 0 +#define COMPACT_GOOGLE_LOG_INFO @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__) +#define LOG_TO_STRING_INFO(message) @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_INFO, message) +#else +#define COMPACT_GOOGLE_LOG_INFO @ac_google_namespace@::NullStream() +#define LOG_TO_STRING_INFO(message) @ac_google_namespace@::NullStream() +#endif + +#if GOOGLE_STRIP_LOG <= 1 +#define COMPACT_GOOGLE_LOG_WARNING @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_WARNING) +#define LOG_TO_STRING_WARNING(message) @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_WARNING, message) +#else +#define COMPACT_GOOGLE_LOG_WARNING @ac_google_namespace@::NullStream() +#define LOG_TO_STRING_WARNING(message) @ac_google_namespace@::NullStream() +#endif + +#if GOOGLE_STRIP_LOG <= 2 +#define COMPACT_GOOGLE_LOG_ERROR @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ERROR) +#define LOG_TO_STRING_ERROR(message) @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ERROR, message) +#else +#define COMPACT_GOOGLE_LOG_ERROR @ac_google_namespace@::NullStream() +#define LOG_TO_STRING_ERROR(message) @ac_google_namespace@::NullStream() +#endif + +#if GOOGLE_STRIP_LOG <= 3 +#define COMPACT_GOOGLE_LOG_FATAL @ac_google_namespace@::LogMessageFatal( \ + __FILE__, __LINE__) +#define LOG_TO_STRING_FATAL(message) @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_FATAL, message) +#else +#define COMPACT_GOOGLE_LOG_FATAL @ac_google_namespace@::NullStreamFatal() +#define LOG_TO_STRING_FATAL(message) @ac_google_namespace@::NullStreamFatal() +#endif + +#if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON) +#define DCHECK_IS_ON() 0 +#else +#define DCHECK_IS_ON() 1 +#endif + +// For DFATAL, we want to use LogMessage (as opposed to +// LogMessageFatal), to be consistent with the original behavior. +#if !DCHECK_IS_ON() +#define COMPACT_GOOGLE_LOG_DFATAL COMPACT_GOOGLE_LOG_ERROR +#elif GOOGLE_STRIP_LOG <= 3 +#define COMPACT_GOOGLE_LOG_DFATAL @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_FATAL) +#else +#define COMPACT_GOOGLE_LOG_DFATAL @ac_google_namespace@::NullStreamFatal() +#endif + +#define GOOGLE_LOG_INFO(counter) @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_INFO, counter, &@ac_google_namespace@::LogMessage::SendToLog) +#define SYSLOG_INFO(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_INFO, counter, \ + &@ac_google_namespace@::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_WARNING(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_WARNING, counter, \ + &@ac_google_namespace@::LogMessage::SendToLog) +#define SYSLOG_WARNING(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_WARNING, counter, \ + &@ac_google_namespace@::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_ERROR(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_ERROR, counter, \ + &@ac_google_namespace@::LogMessage::SendToLog) +#define SYSLOG_ERROR(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_ERROR, counter, \ + &@ac_google_namespace@::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_FATAL(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_FATAL, counter, \ + &@ac_google_namespace@::LogMessage::SendToLog) +#define SYSLOG_FATAL(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_FATAL, counter, \ + &@ac_google_namespace@::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_DFATAL(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::DFATAL_LEVEL, counter, \ + &@ac_google_namespace@::LogMessage::SendToLog) +#define SYSLOG_DFATAL(counter) \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::DFATAL_LEVEL, counter, \ + &@ac_google_namespace@::LogMessage::SendToSyslogAndLog) + +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) || defined(__CYGWIN32__) +// A very useful logging macro to log windows errors: +#define LOG_SYSRESULT(result) \ + if (FAILED(HRESULT_FROM_WIN32(result))) { \ + LPSTR message = NULL; \ + LPSTR msg = reinterpret_cast<LPSTR>(&message); \ + DWORD message_length = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | \ + FORMAT_MESSAGE_FROM_SYSTEM, \ + 0, result, 0, msg, 100, NULL); \ + if (message_length > 0) { \ + @ac_google_namespace@::LogMessage(__FILE__, __LINE__, @ac_google_namespace@::GLOG_ERROR, 0, \ + &@ac_google_namespace@::LogMessage::SendToLog).stream() \ + << reinterpret_cast<const char*>(message); \ + LocalFree(message); \ + } \ + } +#endif + +// We use the preprocessor's merging operator, "##", so that, e.g., +// LOG(INFO) becomes the token GOOGLE_LOG_INFO. There's some funny +// subtle difference between ostream member streaming functions (e.g., +// ostream::operator<<(int) and ostream non-member streaming functions +// (e.g., ::operator<<(ostream&, string&): it turns out that it's +// impossible to stream something like a string directly to an unnamed +// ostream. We employ a neat hack by calling the stream() member +// function of LogMessage which seems to avoid the problem. +#define LOG(severity) COMPACT_GOOGLE_LOG_ ## severity.stream() +#define SYSLOG(severity) SYSLOG_ ## severity(0).stream() + +@ac_google_start_namespace@ + +// They need the definitions of integer types. +#include "glog/log_severity.h" +#include "glog/vlog_is_on.h" + +// Initialize google's logging library. You will see the program name +// specified by argv0 in log outputs. +GOOGLE_GLOG_DLL_DECL void InitGoogleLogging(const char* argv0); + +// Shutdown google's logging library. +GOOGLE_GLOG_DLL_DECL void ShutdownGoogleLogging(); + +// Install a function which will be called after LOG(FATAL). +GOOGLE_GLOG_DLL_DECL void InstallFailureFunction(void (*fail_func)()); + +// Enable/Disable old log cleaner. +GOOGLE_GLOG_DLL_DECL void EnableLogCleaner(int overdue_days); +GOOGLE_GLOG_DLL_DECL void DisableLogCleaner(); + + +class LogSink; // defined below + +// If a non-NULL sink pointer is given, we push this message to that sink. +// For LOG_TO_SINK we then do normal LOG(severity) logging as well. +// This is useful for capturing messages and passing/storing them +// somewhere more specific than the global log of the process. +// Argument types: +// LogSink* sink; +// LogSeverity severity; +// The cast is to disambiguate NULL arguments. +#define LOG_TO_SINK(sink, severity) \ + @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, \ + @ac_google_namespace@::GLOG_ ## severity, \ + static_cast<@ac_google_namespace@::LogSink*>(sink), true).stream() +#define LOG_TO_SINK_BUT_NOT_TO_LOGFILE(sink, severity) \ + @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, \ + @ac_google_namespace@::GLOG_ ## severity, \ + static_cast<@ac_google_namespace@::LogSink*>(sink), false).stream() + +// If a non-NULL string pointer is given, we write this message to that string. +// We then do normal LOG(severity) logging as well. +// This is useful for capturing messages and storing them somewhere more +// specific than the global log of the process. +// Argument types: +// string* message; +// LogSeverity severity; +// The cast is to disambiguate NULL arguments. +// NOTE: LOG(severity) expands to LogMessage().stream() for the specified +// severity. +#define LOG_TO_STRING(severity, message) \ + LOG_TO_STRING_##severity(static_cast<string*>(message)).stream() + +// If a non-NULL pointer is given, we push the message onto the end +// of a vector of strings; otherwise, we report it with LOG(severity). +// This is handy for capturing messages and perhaps passing them back +// to the caller, rather than reporting them immediately. +// Argument types: +// LogSeverity severity; +// vector<string> *outvec; +// The cast is to disambiguate NULL arguments. +#define LOG_STRING(severity, outvec) \ + LOG_TO_STRING_##severity(static_cast<std::vector<std::string>*>(outvec)).stream() + +#define LOG_IF(severity, condition) \ + static_cast<void>(0), \ + !(condition) ? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & LOG(severity) +#define SYSLOG_IF(severity, condition) \ + static_cast<void>(0), \ + !(condition) ? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & SYSLOG(severity) + +#define LOG_ASSERT(condition) \ + LOG_IF(FATAL, !(condition)) << "Assert failed: " #condition +#define SYSLOG_ASSERT(condition) \ + SYSLOG_IF(FATAL, !(condition)) << "Assert failed: " #condition + +// CHECK dies with a fatal error if condition is not true. It is *not* +// controlled by DCHECK_IS_ON(), so the check will be executed regardless of +// compilation mode. Therefore, it is safe to do things like: +// CHECK(fp->Write(x) == 4) +#define CHECK(condition) \ + LOG_IF(FATAL, GOOGLE_PREDICT_BRANCH_NOT_TAKEN(!(condition))) \ + << "Check failed: " #condition " " + +// A container for a string pointer which can be evaluated to a bool - +// true iff the pointer is NULL. +struct CheckOpString { + CheckOpString(std::string* str) : str_(str) { } + // No destructor: if str_ is non-NULL, we're about to LOG(FATAL), + // so there's no point in cleaning up str_. + operator bool() const { + return GOOGLE_PREDICT_BRANCH_NOT_TAKEN(str_ != NULL); + } + std::string* str_; +}; + +// Function is overloaded for integral types to allow static const +// integrals declared in classes and not defined to be used as arguments to +// CHECK* macros. It's not encouraged though. +template <class T> +inline const T& GetReferenceableValue(const T& t) { return t; } +inline char GetReferenceableValue(char t) { return t; } +inline unsigned char GetReferenceableValue(unsigned char t) { return t; } +inline signed char GetReferenceableValue(signed char t) { return t; } +inline short GetReferenceableValue(short t) { return t; } +inline unsigned short GetReferenceableValue(unsigned short t) { return t; } +inline int GetReferenceableValue(int t) { return t; } +inline unsigned int GetReferenceableValue(unsigned int t) { return t; } +inline long GetReferenceableValue(long t) { return t; } +inline unsigned long GetReferenceableValue(unsigned long t) { return t; } +inline long long GetReferenceableValue(long long t) { return t; } +inline unsigned long long GetReferenceableValue(unsigned long long t) { + return t; +} + +// This is a dummy class to define the following operator. +struct DummyClassToDefineOperator {}; + +@ac_google_end_namespace@ + +// Define global operator<< to declare using ::operator<<. +// This declaration will allow use to use CHECK macros for user +// defined classes which have operator<< (e.g., stl_logging.h). +inline std::ostream& operator<<( + std::ostream& out, const google::DummyClassToDefineOperator&) { + return out; +} + +@ac_google_start_namespace@ + +// This formats a value for a failing CHECK_XX statement. Ordinarily, +// it uses the definition for operator<<, with a few special cases below. +template <typename T> +inline void MakeCheckOpValueString(std::ostream* os, const T& v) { + (*os) << v; +} + +// Overrides for char types provide readable values for unprintable +// characters. +template <> GOOGLE_GLOG_DLL_DECL +void MakeCheckOpValueString(std::ostream* os, const char& v); +template <> GOOGLE_GLOG_DLL_DECL +void MakeCheckOpValueString(std::ostream* os, const signed char& v); +template <> GOOGLE_GLOG_DLL_DECL +void MakeCheckOpValueString(std::ostream* os, const unsigned char& v); + +// Build the error message string. Specify no inlining for code size. +template <typename T1, typename T2> +std::string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) + @ac_cv___attribute___noinline@; + +namespace base { +namespace internal { + +// If "s" is less than base_logging::INFO, returns base_logging::INFO. +// If "s" is greater than base_logging::FATAL, returns +// base_logging::ERROR. Otherwise, returns "s". +LogSeverity NormalizeSeverity(LogSeverity s); + +} // namespace internal + +// A helper class for formatting "expr (V1 vs. V2)" in a CHECK_XX +// statement. See MakeCheckOpString for sample usage. Other +// approaches were considered: use of a template method (e.g., +// base::BuildCheckOpString(exprtext, base::Print<T1>, &v1, +// base::Print<T2>, &v2), however this approach has complications +// related to volatile arguments and function-pointer arguments). +class GOOGLE_GLOG_DLL_DECL CheckOpMessageBuilder { + public: + // Inserts "exprtext" and " (" to the stream. + explicit CheckOpMessageBuilder(const char *exprtext); + // Deletes "stream_". + ~CheckOpMessageBuilder(); + // For inserting the first variable. + std::ostream* ForVar1() { return stream_; } + // For inserting the second variable (adds an intermediate " vs. "). + std::ostream* ForVar2(); + // Get the result (inserts the closing ")"). + std::string* NewString(); + + private: + std::ostringstream *stream_; +}; + +} // namespace base + +template <typename T1, typename T2> +std::string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) { + base::CheckOpMessageBuilder comb(exprtext); + MakeCheckOpValueString(comb.ForVar1(), v1); + MakeCheckOpValueString(comb.ForVar2(), v2); + return comb.NewString(); +} + +// Helper functions for CHECK_OP macro. +// The (int, int) specialization works around the issue that the compiler +// will not instantiate the template version of the function on values of +// unnamed enum type - see comment below. +#define DEFINE_CHECK_OP_IMPL(name, op) \ + template <typename T1, typename T2> \ + inline std::string* name##Impl(const T1& v1, const T2& v2, \ + const char* exprtext) { \ + if (GOOGLE_PREDICT_TRUE(v1 op v2)) return NULL; \ + else return MakeCheckOpString(v1, v2, exprtext); \ + } \ + inline std::string* name##Impl(int v1, int v2, const char* exprtext) { \ + return name##Impl<int, int>(v1, v2, exprtext); \ + } + +// We use the full name Check_EQ, Check_NE, etc. in case the file including +// base/logging.h provides its own #defines for the simpler names EQ, NE, etc. +// This happens if, for example, those are used as token names in a +// yacc grammar. +DEFINE_CHECK_OP_IMPL(Check_EQ, ==) // Compilation error with CHECK_EQ(NULL, x)? +DEFINE_CHECK_OP_IMPL(Check_NE, !=) // Use CHECK(x == NULL) instead. +DEFINE_CHECK_OP_IMPL(Check_LE, <=) +DEFINE_CHECK_OP_IMPL(Check_LT, < ) +DEFINE_CHECK_OP_IMPL(Check_GE, >=) +DEFINE_CHECK_OP_IMPL(Check_GT, > ) +#undef DEFINE_CHECK_OP_IMPL + +// Helper macro for binary operators. +// Don't use this macro directly in your code, use CHECK_EQ et al below. + +#if defined(STATIC_ANALYSIS) +// Only for static analysis tool to know that it is equivalent to assert +#define CHECK_OP_LOG(name, op, val1, val2, log) CHECK((val1) op (val2)) +#elif DCHECK_IS_ON() +// In debug mode, avoid constructing CheckOpStrings if possible, +// to reduce the overhead of CHECK statments by 2x. +// Real DCHECK-heavy tests have seen 1.5x speedups. + +// The meaning of "string" might be different between now and +// when this macro gets invoked (e.g., if someone is experimenting +// with other string implementations that get defined after this +// file is included). Save the current meaning now and use it +// in the macro. +typedef std::string _Check_string; +#define CHECK_OP_LOG(name, op, val1, val2, log) \ + while (@ac_google_namespace@::_Check_string* _result = \ + @ac_google_namespace@::Check##name##Impl( \ + @ac_google_namespace@::GetReferenceableValue(val1), \ + @ac_google_namespace@::GetReferenceableValue(val2), \ + #val1 " " #op " " #val2)) \ + log(__FILE__, __LINE__, \ + @ac_google_namespace@::CheckOpString(_result)).stream() +#else +// In optimized mode, use CheckOpString to hint to compiler that +// the while condition is unlikely. +#define CHECK_OP_LOG(name, op, val1, val2, log) \ + while (@ac_google_namespace@::CheckOpString _result = \ + @ac_google_namespace@::Check##name##Impl( \ + @ac_google_namespace@::GetReferenceableValue(val1), \ + @ac_google_namespace@::GetReferenceableValue(val2), \ + #val1 " " #op " " #val2)) \ + log(__FILE__, __LINE__, _result).stream() +#endif // STATIC_ANALYSIS, DCHECK_IS_ON() + +#if GOOGLE_STRIP_LOG <= 3 +#define CHECK_OP(name, op, val1, val2) \ + CHECK_OP_LOG(name, op, val1, val2, @ac_google_namespace@::LogMessageFatal) +#else +#define CHECK_OP(name, op, val1, val2) \ + CHECK_OP_LOG(name, op, val1, val2, @ac_google_namespace@::NullStreamFatal) +#endif // STRIP_LOG <= 3 + +// Equality/Inequality checks - compare two values, and log a FATAL message +// including the two values when the result is not as expected. The values +// must have operator<<(ostream, ...) defined. +// +// You may append to the error message like so: +// CHECK_NE(1, 2) << ": The world must be ending!"; +// +// We are very careful to ensure that each argument is evaluated exactly +// once, and that anything which is legal to pass as a function argument is +// legal here. In particular, the arguments may be temporary expressions +// which will end up being destroyed at the end of the apparent statement, +// for example: +// CHECK_EQ(string("abc")[1], 'b'); +// +// WARNING: These don't compile correctly if one of the arguments is a pointer +// and the other is NULL. To work around this, simply static_cast NULL to the +// type of the desired pointer. + +#define CHECK_EQ(val1, val2) CHECK_OP(_EQ, ==, val1, val2) +#define CHECK_NE(val1, val2) CHECK_OP(_NE, !=, val1, val2) +#define CHECK_LE(val1, val2) CHECK_OP(_LE, <=, val1, val2) +#define CHECK_LT(val1, val2) CHECK_OP(_LT, < , val1, val2) +#define CHECK_GE(val1, val2) CHECK_OP(_GE, >=, val1, val2) +#define CHECK_GT(val1, val2) CHECK_OP(_GT, > , val1, val2) + +// Check that the input is non NULL. This very useful in constructor +// initializer lists. + +#define CHECK_NOTNULL(val) \ + @ac_google_namespace@::CheckNotNull(__FILE__, __LINE__, "'" #val "' Must be non NULL", (val)) + +// Helper functions for string comparisons. +// To avoid bloat, the definitions are in logging.cc. +#define DECLARE_CHECK_STROP_IMPL(func, expected) \ + GOOGLE_GLOG_DLL_DECL std::string* Check##func##expected##Impl( \ + const char* s1, const char* s2, const char* names); +DECLARE_CHECK_STROP_IMPL(strcmp, true) +DECLARE_CHECK_STROP_IMPL(strcmp, false) +DECLARE_CHECK_STROP_IMPL(strcasecmp, true) +DECLARE_CHECK_STROP_IMPL(strcasecmp, false) +#undef DECLARE_CHECK_STROP_IMPL + +// Helper macro for string comparisons. +// Don't use this macro directly in your code, use CHECK_STREQ et al below. +#define CHECK_STROP(func, op, expected, s1, s2) \ + while (@ac_google_namespace@::CheckOpString _result = \ + @ac_google_namespace@::Check##func##expected##Impl((s1), (s2), \ + #s1 " " #op " " #s2)) \ + LOG(FATAL) << *_result.str_ + + +// String (char*) equality/inequality checks. +// CASE versions are case-insensitive. +// +// Note that "s1" and "s2" may be temporary strings which are destroyed +// by the compiler at the end of the current "full expression" +// (e.g. CHECK_STREQ(Foo().c_str(), Bar().c_str())). + +#define CHECK_STREQ(s1, s2) CHECK_STROP(strcmp, ==, true, s1, s2) +#define CHECK_STRNE(s1, s2) CHECK_STROP(strcmp, !=, false, s1, s2) +#define CHECK_STRCASEEQ(s1, s2) CHECK_STROP(strcasecmp, ==, true, s1, s2) +#define CHECK_STRCASENE(s1, s2) CHECK_STROP(strcasecmp, !=, false, s1, s2) + +#define CHECK_INDEX(I,A) CHECK(I < (sizeof(A)/sizeof(A[0]))) +#define CHECK_BOUND(B,A) CHECK(B <= (sizeof(A)/sizeof(A[0]))) + +#define CHECK_DOUBLE_EQ(val1, val2) \ + do { \ + CHECK_LE((val1), (val2)+0.000000000000001L); \ + CHECK_GE((val1), (val2)-0.000000000000001L); \ + } while (0) + +#define CHECK_NEAR(val1, val2, margin) \ + do { \ + CHECK_LE((val1), (val2)+(margin)); \ + CHECK_GE((val1), (val2)-(margin)); \ + } while (0) + +// perror()..googly style! +// +// PLOG() and PLOG_IF() and PCHECK() behave exactly like their LOG* and +// CHECK equivalents with the addition that they postpend a description +// of the current state of errno to their output lines. + +#define PLOG(severity) GOOGLE_PLOG(severity, 0).stream() + +#define GOOGLE_PLOG(severity, counter) \ + @ac_google_namespace@::ErrnoLogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ ## severity, counter, \ + &@ac_google_namespace@::LogMessage::SendToLog) + +#define PLOG_IF(severity, condition) \ + static_cast<void>(0), \ + !(condition) ? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & PLOG(severity) + +// A CHECK() macro that postpends errno if the condition is false. E.g. +// +// if (poll(fds, nfds, timeout) == -1) { PCHECK(errno == EINTR); ... } +#define PCHECK(condition) \ + PLOG_IF(FATAL, GOOGLE_PREDICT_BRANCH_NOT_TAKEN(!(condition))) \ + << "Check failed: " #condition " " + +// A CHECK() macro that lets you assert the success of a function that +// returns -1 and sets errno in case of an error. E.g. +// +// CHECK_ERR(mkdir(path, 0700)); +// +// or +// +// int fd = open(filename, flags); CHECK_ERR(fd) << ": open " << filename; +#define CHECK_ERR(invocation) \ +PLOG_IF(FATAL, GOOGLE_PREDICT_BRANCH_NOT_TAKEN((invocation) == -1)) \ + << #invocation + +// Use macro expansion to create, for each use of LOG_EVERY_N(), static +// variables with the __LINE__ expansion as part of the variable name. +#define LOG_EVERY_N_VARNAME(base, line) LOG_EVERY_N_VARNAME_CONCAT(base, line) +#define LOG_EVERY_N_VARNAME_CONCAT(base, line) base ## line + +#define LOG_OCCURRENCES LOG_EVERY_N_VARNAME(occurrences_, __LINE__) +#define LOG_OCCURRENCES_MOD_N LOG_EVERY_N_VARNAME(occurrences_mod_n_, __LINE__) + +#define SOME_KIND_OF_LOG_EVERY_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (LOG_OCCURRENCES_MOD_N == 1) \ + @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +#define SOME_KIND_OF_LOG_IF_EVERY_N(severity, condition, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (condition && \ + ((LOG_OCCURRENCES_MOD_N=(LOG_OCCURRENCES_MOD_N + 1) % n) == (1 % n))) \ + @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +#define SOME_KIND_OF_PLOG_EVERY_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (LOG_OCCURRENCES_MOD_N == 1) \ + @ac_google_namespace@::ErrnoLogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +#define SOME_KIND_OF_LOG_FIRST_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0; \ + if (LOG_OCCURRENCES <= n) \ + ++LOG_OCCURRENCES; \ + if (LOG_OCCURRENCES <= n) \ + @ac_google_namespace@::LogMessage( \ + __FILE__, __LINE__, @ac_google_namespace@::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +namespace glog_internal_namespace_ { +template <bool> +struct CompileAssert { +}; +struct CrashReason; + +// Returns true if FailureSignalHandler is installed. +// Needs to be exported since it's used by the signalhandler_unittest. +GOOGLE_GLOG_DLL_DECL bool IsFailureSignalHandlerInstalled(); +} // namespace glog_internal_namespace_ + +#define LOG_EVERY_N(severity, n) \ + SOME_KIND_OF_LOG_EVERY_N(severity, (n), @ac_google_namespace@::LogMessage::SendToLog) + +#define SYSLOG_EVERY_N(severity, n) \ + SOME_KIND_OF_LOG_EVERY_N(severity, (n), @ac_google_namespace@::LogMessage::SendToSyslogAndLog) + +#define PLOG_EVERY_N(severity, n) \ + SOME_KIND_OF_PLOG_EVERY_N(severity, (n), @ac_google_namespace@::LogMessage::SendToLog) + +#define LOG_FIRST_N(severity, n) \ + SOME_KIND_OF_LOG_FIRST_N(severity, (n), @ac_google_namespace@::LogMessage::SendToLog) + +#define LOG_IF_EVERY_N(severity, condition, n) \ + SOME_KIND_OF_LOG_IF_EVERY_N(severity, (condition), (n), @ac_google_namespace@::LogMessage::SendToLog) + +// We want the special COUNTER value available for LOG_EVERY_X()'ed messages +enum PRIVATE_Counter {COUNTER}; + +#ifdef GLOG_NO_ABBREVIATED_SEVERITIES +// wingdi.h defines ERROR to be 0. When we call LOG(ERROR), it gets +// substituted with 0, and it expands to COMPACT_GOOGLE_LOG_0. To allow us +// to keep using this syntax, we define this macro to do the same thing +// as COMPACT_GOOGLE_LOG_ERROR. +#define COMPACT_GOOGLE_LOG_0 COMPACT_GOOGLE_LOG_ERROR +#define SYSLOG_0 SYSLOG_ERROR +#define LOG_TO_STRING_0 LOG_TO_STRING_ERROR +// Needed for LOG_IS_ON(ERROR). +const LogSeverity GLOG_0 = GLOG_ERROR; +#else +// Users may include windows.h after logging.h without +// GLOG_NO_ABBREVIATED_SEVERITIES nor WIN32_LEAN_AND_MEAN. +// For this case, we cannot detect if ERROR is defined before users +// actually use ERROR. Let's make an undefined symbol to warn users. +# define GLOG_ERROR_MSG ERROR_macro_is_defined_Define_GLOG_NO_ABBREVIATED_SEVERITIES_before_including_logging_h_See_the_document_for_detail +# define COMPACT_GOOGLE_LOG_0 GLOG_ERROR_MSG +# define SYSLOG_0 GLOG_ERROR_MSG +# define LOG_TO_STRING_0 GLOG_ERROR_MSG +# define GLOG_0 GLOG_ERROR_MSG +#endif + +// Plus some debug-logging macros that get compiled to nothing for production + +#if DCHECK_IS_ON() + +#define DLOG(severity) LOG(severity) +#define DVLOG(verboselevel) VLOG(verboselevel) +#define DLOG_IF(severity, condition) LOG_IF(severity, condition) +#define DLOG_EVERY_N(severity, n) LOG_EVERY_N(severity, n) +#define DLOG_IF_EVERY_N(severity, condition, n) \ + LOG_IF_EVERY_N(severity, condition, n) +#define DLOG_ASSERT(condition) LOG_ASSERT(condition) + +// debug-only checking. executed if DCHECK_IS_ON(). +#define DCHECK(condition) CHECK(condition) +#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) +#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) +#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) +#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) +#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) +#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) +#define DCHECK_NOTNULL(val) CHECK_NOTNULL(val) +#define DCHECK_STREQ(str1, str2) CHECK_STREQ(str1, str2) +#define DCHECK_STRCASEEQ(str1, str2) CHECK_STRCASEEQ(str1, str2) +#define DCHECK_STRNE(str1, str2) CHECK_STRNE(str1, str2) +#define DCHECK_STRCASENE(str1, str2) CHECK_STRCASENE(str1, str2) + +#else // !DCHECK_IS_ON() + +#define DLOG(severity) \ + static_cast<void>(0), \ + true ? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & LOG(severity) + +#define DVLOG(verboselevel) \ + static_cast<void>(0), \ + (true || !VLOG_IS_ON(verboselevel)) ? \ + (void) 0 : @ac_google_namespace@::LogMessageVoidify() & LOG(INFO) + +#define DLOG_IF(severity, condition) \ + static_cast<void>(0), \ + (true || !(condition)) ? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & LOG(severity) + +#define DLOG_EVERY_N(severity, n) \ + static_cast<void>(0), \ + true ? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & LOG(severity) + +#define DLOG_IF_EVERY_N(severity, condition, n) \ + static_cast<void>(0), \ + (true || !(condition))? (void) 0 : @ac_google_namespace@::LogMessageVoidify() & LOG(severity) + +#define DLOG_ASSERT(condition) \ + static_cast<void>(0), \ + true ? (void) 0 : LOG_ASSERT(condition) + +// MSVC warning C4127: conditional expression is constant +#define DCHECK(condition) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK(condition) + +#define DCHECK_EQ(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_EQ(val1, val2) + +#define DCHECK_NE(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_NE(val1, val2) + +#define DCHECK_LE(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_LE(val1, val2) + +#define DCHECK_LT(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_LT(val1, val2) + +#define DCHECK_GE(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_GE(val1, val2) + +#define DCHECK_GT(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_GT(val1, val2) + +// You may see warnings in release mode if you don't use the return +// value of DCHECK_NOTNULL. Please just use DCHECK for such cases. +#define DCHECK_NOTNULL(val) (val) + +#define DCHECK_STREQ(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STREQ(str1, str2) + +#define DCHECK_STRCASEEQ(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STRCASEEQ(str1, str2) + +#define DCHECK_STRNE(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STRNE(str1, str2) + +#define DCHECK_STRCASENE(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STRCASENE(str1, str2) + +#endif // DCHECK_IS_ON() + +// Log only in verbose mode. + +#define VLOG(verboselevel) LOG_IF(INFO, VLOG_IS_ON(verboselevel)) + +#define VLOG_IF(verboselevel, condition) \ + LOG_IF(INFO, (condition) && VLOG_IS_ON(verboselevel)) + +#define VLOG_EVERY_N(verboselevel, n) \ + LOG_IF_EVERY_N(INFO, VLOG_IS_ON(verboselevel), n) + +#define VLOG_IF_EVERY_N(verboselevel, condition, n) \ + LOG_IF_EVERY_N(INFO, (condition) && VLOG_IS_ON(verboselevel), n) + +namespace base_logging { + +// LogMessage::LogStream is a std::ostream backed by this streambuf. +// This class ignores overflow and leaves two bytes at the end of the +// buffer to allow for a '\n' and '\0'. +class GOOGLE_GLOG_DLL_DECL LogStreamBuf : public std::streambuf { + public: + // REQUIREMENTS: "len" must be >= 2 to account for the '\n' and '\0'. + LogStreamBuf(char *buf, int len) { + setp(buf, buf + len - 2); + } + + // This effectively ignores overflow. + virtual int_type overflow(int_type ch) { + return ch; + } + + // Legacy public ostrstream method. + size_t pcount() const { return pptr() - pbase(); } + char* pbase() const { return std::streambuf::pbase(); } +}; + +} // namespace base_logging + +// +// This class more or less represents a particular log message. You +// create an instance of LogMessage and then stream stuff to it. +// When you finish streaming to it, ~LogMessage is called and the +// full message gets streamed to the appropriate destination. +// +// You shouldn't actually use LogMessage's constructor to log things, +// though. You should use the LOG() macro (and variants thereof) +// above. +class GOOGLE_GLOG_DLL_DECL LogMessage { +public: + enum { + // Passing kNoLogPrefix for the line number disables the + // log-message prefix. Useful for using the LogMessage + // infrastructure as a printing utility. See also the --log_prefix + // flag for controlling the log-message prefix on an + // application-wide basis. + kNoLogPrefix = -1 + }; + + // LogStream inherit from non-DLL-exported class (std::ostrstream) + // and VC++ produces a warning for this situation. + // However, MSDN says "C4275 can be ignored in Microsoft Visual C++ + // 2005 if you are deriving from a type in the Standard C++ Library" + // http://msdn.microsoft.com/en-us/library/3tdb471s(VS.80).aspx + // Let's just ignore the warning. +GLOG_MSVC_PUSH_DISABLE_WARNING(4275) + class GOOGLE_GLOG_DLL_DECL LogStream : public std::ostream { +GLOG_MSVC_POP_WARNING() + public: + LogStream(char *buf, int len, int ctr) + : std::ostream(NULL), + streambuf_(buf, len), + ctr_(ctr), + self_(this) { + rdbuf(&streambuf_); + } + + int ctr() const { return ctr_; } + void set_ctr(int ctr) { ctr_ = ctr; } + LogStream* self() const { return self_; } + + // Legacy std::streambuf methods. + size_t pcount() const { return streambuf_.pcount(); } + char* pbase() const { return streambuf_.pbase(); } + char* str() const { return pbase(); } + + private: + LogStream(const LogStream&); + LogStream& operator=(const LogStream&); + base_logging::LogStreamBuf streambuf_; + int ctr_; // Counter hack (for the LOG_EVERY_X() macro) + LogStream *self_; // Consistency check hack + }; + +public: + // icc 8 requires this typedef to avoid an internal compiler error. + typedef void (LogMessage::*SendMethod)(); + + LogMessage(const char* file, int line, LogSeverity severity, int ctr, + SendMethod send_method); + + // Two special constructors that generate reduced amounts of code at + // LOG call sites for common cases. + + // Used for LOG(INFO): Implied are: + // severity = INFO, ctr = 0, send_method = &LogMessage::SendToLog. + // + // Using this constructor instead of the more complex constructor above + // saves 19 bytes per call site. + LogMessage(const char* file, int line); + + // Used for LOG(severity) where severity != INFO. Implied + // are: ctr = 0, send_method = &LogMessage::SendToLog + // + // Using this constructor instead of the more complex constructor above + // saves 17 bytes per call site. + LogMessage(const char* file, int line, LogSeverity severity); + + // Constructor to log this message to a specified sink (if not NULL). + // Implied are: ctr = 0, send_method = &LogMessage::SendToSinkAndLog if + // also_send_to_log is true, send_method = &LogMessage::SendToSink otherwise. + LogMessage(const char* file, int line, LogSeverity severity, LogSink* sink, + bool also_send_to_log); + + // Constructor where we also give a vector<string> pointer + // for storing the messages (if the pointer is not NULL). + // Implied are: ctr = 0, send_method = &LogMessage::SaveOrSendToLog. + LogMessage(const char* file, int line, LogSeverity severity, + std::vector<std::string>* outvec); + + // Constructor where we also give a string pointer for storing the + // message (if the pointer is not NULL). Implied are: ctr = 0, + // send_method = &LogMessage::WriteToStringAndLog. + LogMessage(const char* file, int line, LogSeverity severity, + std::string* message); + + // A special constructor used for check failures + LogMessage(const char* file, int line, const CheckOpString& result); + + ~LogMessage(); + + // Flush a buffered message to the sink set in the constructor. Always + // called by the destructor, it may also be called from elsewhere if + // needed. Only the first call is actioned; any later ones are ignored. + void Flush(); + + // An arbitrary limit on the length of a single log message. This + // is so that streaming can be done more efficiently. + static const size_t kMaxLogMessageLen; + + // Theses should not be called directly outside of logging.*, + // only passed as SendMethod arguments to other LogMessage methods: + void SendToLog(); // Actually dispatch to the logs + void SendToSyslogAndLog(); // Actually dispatch to syslog and the logs + + // Call abort() or similar to perform LOG(FATAL) crash. + static void @ac_cv___attribute___noreturn@ Fail(); + + std::ostream& stream(); + + int preserved_errno() const; + + // Must be called without the log_mutex held. (L < log_mutex) + static int64 num_messages(int severity); + + struct LogMessageData; + +private: + // Fully internal SendMethod cases: + void SendToSinkAndLog(); // Send to sink if provided and dispatch to the logs + void SendToSink(); // Send to sink if provided, do nothing otherwise. + + // Write to string if provided and dispatch to the logs. + void WriteToStringAndLog(); + + void SaveOrSendToLog(); // Save to stringvec if provided, else to logs + + void Init(const char* file, int line, LogSeverity severity, + void (LogMessage::*send_method)()); + + // Used to fill in crash information during LOG(FATAL) failures. + void RecordCrashReason(glog_internal_namespace_::CrashReason* reason); + + // Counts of messages sent at each priority: + static int64 num_messages_[NUM_SEVERITIES]; // under log_mutex + + // We keep the data in a separate struct so that each instance of + // LogMessage uses less stack space. + LogMessageData* allocated_; + LogMessageData* data_; + + friend class LogDestination; + + LogMessage(const LogMessage&); + void operator=(const LogMessage&); +}; + +// This class happens to be thread-hostile because all instances share +// a single data buffer, but since it can only be created just before +// the process dies, we don't worry so much. +class GOOGLE_GLOG_DLL_DECL LogMessageFatal : public LogMessage { + public: + LogMessageFatal(const char* file, int line); + LogMessageFatal(const char* file, int line, const CheckOpString& result); + @ac_cv___attribute___noreturn@ ~LogMessageFatal(); +}; + +// A non-macro interface to the log facility; (useful +// when the logging level is not a compile-time constant). +inline void LogAtLevel(int const severity, std::string const &msg) { + LogMessage(__FILE__, __LINE__, severity).stream() << msg; +} + +// A macro alternative of LogAtLevel. New code may want to use this +// version since there are two advantages: 1. this version outputs the +// file name and the line number where this macro is put like other +// LOG macros, 2. this macro can be used as C++ stream. +#define LOG_AT_LEVEL(severity) @ac_google_namespace@::LogMessage(__FILE__, __LINE__, severity).stream() + +// Check if it's compiled in C++11 mode. +// +// GXX_EXPERIMENTAL_CXX0X is defined by gcc and clang up to at least +// gcc-4.7 and clang-3.1 (2011-12-13). __cplusplus was defined to 1 +// in gcc before 4.7 (Crosstool 16) and clang before 3.1, but is +// defined according to the language version in effect thereafter. +// Microsoft Visual Studio 14 (2015) sets __cplusplus==199711 despite +// reasonably good C++11 support, so we set LANG_CXX for it and +// newer versions (_MSC_VER >= 1900). +#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ + (defined(_MSC_VER) && _MSC_VER >= 1900)) && !defined(__UCLIBCXX_MAJOR__) +// Helper for CHECK_NOTNULL(). +// +// In C++11, all cases can be handled by a single function. Since the value +// category of the argument is preserved (also for rvalue references), +// member initializer lists like the one below will compile correctly: +// +// Foo() +// : x_(CHECK_NOTNULL(MethodReturningUniquePtr())) {} +template <typename T> +T CheckNotNull(const char* file, int line, const char* names, T&& t) { + if (t == nullptr) { + LogMessageFatal(file, line, new std::string(names)); + } + return std::forward<T>(t); +} + +#else + +// A small helper for CHECK_NOTNULL(). +template <typename T> +T* CheckNotNull(const char *file, int line, const char *names, T* t) { + if (t == NULL) { + LogMessageFatal(file, line, new std::string(names)); + } + return t; +} +#endif + +// Allow folks to put a counter in the LOG_EVERY_X()'ed messages. This +// only works if ostream is a LogStream. If the ostream is not a +// LogStream you'll get an assert saying as much at runtime. +GOOGLE_GLOG_DLL_DECL std::ostream& operator<<(std::ostream &os, + const PRIVATE_Counter&); + + +// Derived class for PLOG*() above. +class GOOGLE_GLOG_DLL_DECL ErrnoLogMessage : public LogMessage { + public: + + ErrnoLogMessage(const char* file, int line, LogSeverity severity, int ctr, + void (LogMessage::*send_method)()); + + // Postpends ": strerror(errno) [errno]". + ~ErrnoLogMessage(); + + private: + ErrnoLogMessage(const ErrnoLogMessage&); + void operator=(const ErrnoLogMessage&); +}; + + +// This class is used to explicitly ignore values in the conditional +// logging macros. This avoids compiler warnings like "value computed +// is not used" and "statement has no effect". + +class GOOGLE_GLOG_DLL_DECL LogMessageVoidify { + public: + LogMessageVoidify() { } + // This has to be an operator with a precedence lower than << but + // higher than ?: + void operator&(std::ostream&) { } +}; + + +// Flushes all log files that contains messages that are at least of +// the specified severity level. Thread-safe. +GOOGLE_GLOG_DLL_DECL void FlushLogFiles(LogSeverity min_severity); + +// Flushes all log files that contains messages that are at least of +// the specified severity level. Thread-hostile because it ignores +// locking -- used for catastrophic failures. +GOOGLE_GLOG_DLL_DECL void FlushLogFilesUnsafe(LogSeverity min_severity); + +// +// Set the destination to which a particular severity level of log +// messages is sent. If base_filename is "", it means "don't log this +// severity". Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetLogDestination(LogSeverity severity, + const char* base_filename); + +// +// Set the basename of the symlink to the latest log file at a given +// severity. If symlink_basename is empty, do not make a symlink. If +// you don't call this function, the symlink basename is the +// invocation name of the program. Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetLogSymlink(LogSeverity severity, + const char* symlink_basename); + +// +// Used to send logs to some other kind of destination +// Users should subclass LogSink and override send to do whatever they want. +// Implementations must be thread-safe because a shared instance will +// be called from whichever thread ran the LOG(XXX) line. +class GOOGLE_GLOG_DLL_DECL LogSink { + public: + virtual ~LogSink(); + + // Sink's logging logic (message_len is such as to exclude '\n' at the end). + // This method can't use LOG() or CHECK() as logging system mutex(s) are held + // during this call. + virtual void send(LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len, int32 usecs) { + send(severity, full_filename, base_filename, line, + tm_time, message, message_len); + } + // This send() signature is obsolete. + // New implementations should define this in terms of + // the above send() method. + virtual void send(LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len) = 0; + + // Redefine this to implement waiting for + // the sink's logging logic to complete. + // It will be called after each send() returns, + // but before that LogMessage exits or crashes. + // By default this function does nothing. + // Using this function one can implement complex logic for send() + // that itself involves logging; and do all this w/o causing deadlocks and + // inconsistent rearrangement of log messages. + // E.g. if a LogSink has thread-specific actions, the send() method + // can simply add the message to a queue and wake up another thread that + // handles real logging while itself making some LOG() calls; + // WaitTillSent() can be implemented to wait for that logic to complete. + // See our unittest for an example. + virtual void WaitTillSent(); + + // Returns the normal text output of the log message. + // Can be useful to implement send(). + static std::string ToString(LogSeverity severity, const char* file, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len, + int32 usecs); + + // Obsolete + static std::string ToString(LogSeverity severity, const char* file, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len) { + return ToString(severity, file, line, tm_time, message, message_len, 0); + } +}; + +// Add or remove a LogSink as a consumer of logging data. Thread-safe. +GOOGLE_GLOG_DLL_DECL void AddLogSink(LogSink *destination); +GOOGLE_GLOG_DLL_DECL void RemoveLogSink(LogSink *destination); + +// +// Specify an "extension" added to the filename specified via +// SetLogDestination. This applies to all severity levels. It's +// often used to append the port we're listening on to the logfile +// name. Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetLogFilenameExtension( + const char* filename_extension); + +// +// Make it so that all log messages of at least a particular severity +// are logged to stderr (in addition to logging to the usual log +// file(s)). Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetStderrLogging(LogSeverity min_severity); + +// +// Make it so that all log messages go only to stderr. Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void LogToStderr(); + +// +// Make it so that all log messages of at least a particular severity are +// logged via email to a list of addresses (in addition to logging to the +// usual log file(s)). The list of addresses is just a string containing +// the email addresses to send to (separated by spaces, say). Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetEmailLogging(LogSeverity min_severity, + const char* addresses); + +// A simple function that sends email. dest is a commma-separated +// list of addressess. Thread-safe. +GOOGLE_GLOG_DLL_DECL bool SendEmail(const char *dest, + const char *subject, const char *body); + +GOOGLE_GLOG_DLL_DECL const std::vector<std::string>& GetLoggingDirectories(); + +// For tests only: Clear the internal [cached] list of logging directories to +// force a refresh the next time GetLoggingDirectories is called. +// Thread-hostile. +void TestOnly_ClearLoggingDirectoriesList(); + +// Returns a set of existing temporary directories, which will be a +// subset of the directories returned by GetLogginDirectories(). +// Thread-safe. +GOOGLE_GLOG_DLL_DECL void GetExistingTempDirectories( + std::vector<std::string>* list); + +// Print any fatal message again -- useful to call from signal handler +// so that the last thing in the output is the fatal message. +// Thread-hostile, but a race is unlikely. +GOOGLE_GLOG_DLL_DECL void ReprintFatalMessage(); + +// Truncate a log file that may be the append-only output of multiple +// processes and hence can't simply be renamed/reopened (typically a +// stdout/stderr). If the file "path" is > "limit" bytes, copy the +// last "keep" bytes to offset 0 and truncate the rest. Since we could +// be racing with other writers, this approach has the potential to +// lose very small amounts of data. For security, only follow symlinks +// if the path is /proc/self/fd/* +GOOGLE_GLOG_DLL_DECL void TruncateLogFile(const char *path, + int64 limit, int64 keep); + +// Truncate stdout and stderr if they are over the value specified by +// --max_log_size; keep the final 1MB. This function has the same +// race condition as TruncateLogFile. +GOOGLE_GLOG_DLL_DECL void TruncateStdoutStderr(); + +// Return the string representation of the provided LogSeverity level. +// Thread-safe. +GOOGLE_GLOG_DLL_DECL const char* GetLogSeverityName(LogSeverity severity); + +// --------------------------------------------------------------------- +// Implementation details that are not useful to most clients +// --------------------------------------------------------------------- + +// A Logger is the interface used by logging modules to emit entries +// to a log. A typical implementation will dump formatted data to a +// sequence of files. We also provide interfaces that will forward +// the data to another thread so that the invoker never blocks. +// Implementations should be thread-safe since the logging system +// will write to them from multiple threads. + +namespace base { + +class GOOGLE_GLOG_DLL_DECL Logger { + public: + virtual ~Logger(); + + // Writes "message[0,message_len-1]" corresponding to an event that + // occurred at "timestamp". If "force_flush" is true, the log file + // is flushed immediately. + // + // The input message has already been formatted as deemed + // appropriate by the higher level logging facility. For example, + // textual log messages already contain timestamps, and the + // file:linenumber header. + virtual void Write(bool force_flush, + time_t timestamp, + const char* message, + int message_len) = 0; + + // Flush any buffered messages + virtual void Flush() = 0; + + // Get the current LOG file size. + // The returned value is approximate since some + // logged data may not have been flushed to disk yet. + virtual uint32 LogSize() = 0; +}; + +// Get the logger for the specified severity level. The logger +// remains the property of the logging module and should not be +// deleted by the caller. Thread-safe. +extern GOOGLE_GLOG_DLL_DECL Logger* GetLogger(LogSeverity level); + +// Set the logger for the specified severity level. The logger +// becomes the property of the logging module and should not +// be deleted by the caller. Thread-safe. +extern GOOGLE_GLOG_DLL_DECL void SetLogger(LogSeverity level, Logger* logger); + +} + +// glibc has traditionally implemented two incompatible versions of +// strerror_r(). There is a poorly defined convention for picking the +// version that we want, but it is not clear whether it even works with +// all versions of glibc. +// So, instead, we provide this wrapper that automatically detects the +// version that is in use, and then implements POSIX semantics. +// N.B. In addition to what POSIX says, we also guarantee that "buf" will +// be set to an empty string, if this function failed. This means, in most +// cases, you do not need to check the error code and you can directly +// use the value of "buf". It will never have an undefined value. +// DEPRECATED: Use StrError(int) instead. +GOOGLE_GLOG_DLL_DECL int posix_strerror_r(int err, char *buf, size_t len); + +// A thread-safe replacement for strerror(). Returns a string describing the +// given POSIX error code. +GOOGLE_GLOG_DLL_DECL std::string StrError(int err); + +// A class for which we define operator<<, which does nothing. +class GOOGLE_GLOG_DLL_DECL NullStream : public LogMessage::LogStream { + public: + // Initialize the LogStream so the messages can be written somewhere + // (they'll never be actually displayed). This will be needed if a + // NullStream& is implicitly converted to LogStream&, in which case + // the overloaded NullStream::operator<< will not be invoked. + NullStream() : LogMessage::LogStream(message_buffer_, 1, 0) { } + NullStream(const char* /*file*/, int /*line*/, + const CheckOpString& /*result*/) : + LogMessage::LogStream(message_buffer_, 1, 0) { } + NullStream &stream() { return *this; } + private: + // A very short buffer for messages (which we discard anyway). This + // will be needed if NullStream& converted to LogStream& (e.g. as a + // result of a conditional expression). + char message_buffer_[2]; +}; + +// Do nothing. This operator is inline, allowing the message to be +// compiled away. The message will not be compiled away if we do +// something like (flag ? LOG(INFO) : LOG(ERROR)) << message; when +// SKIP_LOG=WARNING. In those cases, NullStream will be implicitly +// converted to LogStream and the message will be computed and then +// quietly discarded. +template<class T> +inline NullStream& operator<<(NullStream &str, const T &) { return str; } + +// Similar to NullStream, but aborts the program (without stack +// trace), like LogMessageFatal. +class GOOGLE_GLOG_DLL_DECL NullStreamFatal : public NullStream { + public: + NullStreamFatal() { } + NullStreamFatal(const char* file, int line, const CheckOpString& result) : + NullStream(file, line, result) { } + @ac_cv___attribute___noreturn@ ~NullStreamFatal() throw () { _exit(1); } +}; + +// Install a signal handler that will dump signal information and a stack +// trace when the program crashes on certain signals. We'll install the +// signal handler for the following signals. +// +// SIGSEGV, SIGILL, SIGFPE, SIGABRT, SIGBUS, and SIGTERM. +// +// By default, the signal handler will write the failure dump to the +// standard error. You can customize the destination by installing your +// own writer function by InstallFailureWriter() below. +// +// Note on threading: +// +// The function should be called before threads are created, if you want +// to use the failure signal handler for all threads. The stack trace +// will be shown only for the thread that receives the signal. In other +// words, stack traces of other threads won't be shown. +GOOGLE_GLOG_DLL_DECL void InstallFailureSignalHandler(); + +// Installs a function that is used for writing the failure dump. "data" +// is the pointer to the beginning of a message to be written, and "size" +// is the size of the message. You should not expect the data is +// terminated with '\0'. +GOOGLE_GLOG_DLL_DECL void InstallFailureWriter( + void (*writer)(const char* data, int size)); + +@ac_google_end_namespace@ + +#endif // _LOGGING_H_ diff --git a/third_party/glog/src/glog/raw_logging.h.in b/third_party/glog/src/glog/raw_logging.h.in new file mode 100644 index 000000000000..31b7ed32d9a7 --- /dev/null +++ b/third_party/glog/src/glog/raw_logging.h.in @@ -0,0 +1,180 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Maxim Lifantsev +// +// Thread-safe logging routines that do not allocate any memory or +// acquire any locks, and can therefore be used by low-level memory +// allocation and synchronization code. + +#ifndef BASE_RAW_LOGGING_H_ +#define BASE_RAW_LOGGING_H_ + +#include <time.h> + +@ac_google_start_namespace@ + +#include "glog/log_severity.h" +#include "glog/vlog_is_on.h" + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +// This is similar to LOG(severity) << format... and VLOG(level) << format.., +// but +// * it is to be used ONLY by low-level modules that can't use normal LOG() +// * it is desiged to be a low-level logger that does not allocate any +// memory and does not need any locks, hence: +// * it logs straight and ONLY to STDERR w/o buffering +// * it uses an explicit format and arguments list +// * it will silently chop off really long message strings +// Usage example: +// RAW_LOG(ERROR, "Failed foo with %i: %s", status, error); +// RAW_VLOG(3, "status is %i", status); +// These will print an almost standard log lines like this to stderr only: +// E20200821 211317 file.cc:123] RAW: Failed foo with 22: bad_file +// I20200821 211317 file.cc:142] RAW: status is 20 +#define RAW_LOG(severity, ...) \ + do { \ + switch (@ac_google_namespace@::GLOG_ ## severity) { \ + case 0: \ + RAW_LOG_INFO(__VA_ARGS__); \ + break; \ + case 1: \ + RAW_LOG_WARNING(__VA_ARGS__); \ + break; \ + case 2: \ + RAW_LOG_ERROR(__VA_ARGS__); \ + break; \ + case 3: \ + RAW_LOG_FATAL(__VA_ARGS__); \ + break; \ + default: \ + break; \ + } \ + } while (0) + +// The following STRIP_LOG testing is performed in the header file so that it's +// possible to completely compile out the logging code and the log messages. +#if STRIP_LOG == 0 +#define RAW_VLOG(verboselevel, ...) \ + do { \ + if (VLOG_IS_ON(verboselevel)) { \ + RAW_LOG_INFO(__VA_ARGS__); \ + } \ + } while (0) +#else +#define RAW_VLOG(verboselevel, ...) RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG == 0 + +#if STRIP_LOG == 0 +#define RAW_LOG_INFO(...) @ac_google_namespace@::RawLog__(@ac_google_namespace@::GLOG_INFO, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_INFO(...) @ac_google_namespace@::RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG == 0 + +#if STRIP_LOG <= 1 +#define RAW_LOG_WARNING(...) @ac_google_namespace@::RawLog__(@ac_google_namespace@::GLOG_WARNING, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_WARNING(...) @ac_google_namespace@::RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG <= 1 + +#if STRIP_LOG <= 2 +#define RAW_LOG_ERROR(...) @ac_google_namespace@::RawLog__(@ac_google_namespace@::GLOG_ERROR, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_ERROR(...) @ac_google_namespace@::RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG <= 2 + +#if STRIP_LOG <= 3 +#define RAW_LOG_FATAL(...) @ac_google_namespace@::RawLog__(@ac_google_namespace@::GLOG_FATAL, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_FATAL(...) \ + do { \ + @ac_google_namespace@::RawLogStub__(0, __VA_ARGS__); \ + exit(1); \ + } while (0) +#endif // STRIP_LOG <= 3 + +// Similar to CHECK(condition) << message, +// but for low-level modules: we use only RAW_LOG that does not allocate memory. +// We do not want to provide args list here to encourage this usage: +// if (!cond) RAW_LOG(FATAL, "foo ...", hard_to_compute_args); +// so that the args are not computed when not needed. +#define RAW_CHECK(condition, message) \ + do { \ + if (!(condition)) { \ + RAW_LOG(FATAL, "Check %s failed: %s", #condition, message); \ + } \ + } while (0) + +// Debug versions of RAW_LOG and RAW_CHECK +#ifndef NDEBUG + +#define RAW_DLOG(severity, ...) RAW_LOG(severity, __VA_ARGS__) +#define RAW_DCHECK(condition, message) RAW_CHECK(condition, message) + +#else // NDEBUG + +#define RAW_DLOG(severity, ...) \ + while (false) \ + RAW_LOG(severity, __VA_ARGS__) +#define RAW_DCHECK(condition, message) \ + while (false) \ + RAW_CHECK(condition, message) + +#endif // NDEBUG + +// Stub log function used to work around for unused variable warnings when +// building with STRIP_LOG > 0. +static inline void RawLogStub__(int /* ignored */, ...) { +} + +// Helper function to implement RAW_LOG and RAW_VLOG +// Logs format... at "severity" level, reporting it +// as called from file:line. +// This does not allocate memory or acquire locks. +GOOGLE_GLOG_DLL_DECL void RawLog__(LogSeverity severity, + const char* file, + int line, + const char* format, ...) + @ac_cv___attribute___printf_4_5@; + +@ac_google_end_namespace@ + +#endif // BASE_RAW_LOGGING_H_ diff --git a/third_party/glog/src/glog/stl_logging.h.in b/third_party/glog/src/glog/stl_logging.h.in new file mode 100644 index 000000000000..600945d2c22f --- /dev/null +++ b/third_party/glog/src/glog/stl_logging.h.in @@ -0,0 +1,220 @@ +// Copyright (c) 2003, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Stream output operators for STL containers; to be used for logging *only*. +// Inclusion of this file lets you do: +// +// list<string> x; +// LOG(INFO) << "data: " << x; +// vector<int> v1, v2; +// CHECK_EQ(v1, v2); +// +// If you want to use this header file with hash maps or slist, you +// need to define macros before including this file: +// +// - GLOG_STL_LOGGING_FOR_UNORDERED - <unordered_map> and <unordered_set> +// - GLOG_STL_LOGGING_FOR_TR1_UNORDERED - <tr1/unordered_(map|set)> +// - GLOG_STL_LOGGING_FOR_EXT_HASH - <ext/hash_(map|set)> +// - GLOG_STL_LOGGING_FOR_EXT_SLIST - <ext/slist> +// + +#ifndef UTIL_GTL_STL_LOGGING_INL_H_ +#define UTIL_GTL_STL_LOGGING_INL_H_ + +#if !@ac_cv_cxx_using_operator@ +# error We do not support stl_logging for this compiler +#endif + +#include <deque> +#include <list> +#include <map> +#include <ostream> +#include <set> +#include <utility> +#include <vector> + +#ifdef GLOG_STL_LOGGING_FOR_UNORDERED +# include <unordered_map> +# include <unordered_set> +#endif + +#ifdef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +# include <tr1/unordered_map> +# include <tr1/unordered_set> +#endif + +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +# include <ext/hash_set> +# include <ext/hash_map> +#endif +#ifdef GLOG_STL_LOGGING_FOR_EXT_SLIST +# include <ext/slist> +#endif + +// Forward declare these two, and define them after all the container streams +// operators so that we can recurse from pair -> container -> container -> pair +// properly. +template<class First, class Second> +std::ostream& operator<<(std::ostream& out, const std::pair<First, Second>& p); + +@ac_google_start_namespace@ + +template<class Iter> +void PrintSequence(std::ostream& out, Iter begin, Iter end); + +@ac_google_end_namespace@ + +#define OUTPUT_TWO_ARG_CONTAINER(Sequence) \ +template<class T1, class T2> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2>& seq) { \ + @ac_google_namespace@::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +OUTPUT_TWO_ARG_CONTAINER(std::vector) +OUTPUT_TWO_ARG_CONTAINER(std::deque) +OUTPUT_TWO_ARG_CONTAINER(std::list) +#ifdef GLOG_STL_LOGGING_FOR_EXT_SLIST +OUTPUT_TWO_ARG_CONTAINER(__gnu_cxx::slist) +#endif + +#undef OUTPUT_TWO_ARG_CONTAINER + +#define OUTPUT_THREE_ARG_CONTAINER(Sequence) \ +template<class T1, class T2, class T3> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2, T3>& seq) { \ + @ac_google_namespace@::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +OUTPUT_THREE_ARG_CONTAINER(std::set) +OUTPUT_THREE_ARG_CONTAINER(std::multiset) + +#undef OUTPUT_THREE_ARG_CONTAINER + +#define OUTPUT_FOUR_ARG_CONTAINER(Sequence) \ +template<class T1, class T2, class T3, class T4> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2, T3, T4>& seq) { \ + @ac_google_namespace@::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +OUTPUT_FOUR_ARG_CONTAINER(std::map) +OUTPUT_FOUR_ARG_CONTAINER(std::multimap) +#ifdef GLOG_STL_LOGGING_FOR_UNORDERED +OUTPUT_FOUR_ARG_CONTAINER(std::unordered_set) +OUTPUT_FOUR_ARG_CONTAINER(std::unordered_multiset) +#endif +#ifdef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +OUTPUT_FOUR_ARG_CONTAINER(std::tr1::unordered_set) +OUTPUT_FOUR_ARG_CONTAINER(std::tr1::unordered_multiset) +#endif +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +OUTPUT_FOUR_ARG_CONTAINER(__gnu_cxx::hash_set) +OUTPUT_FOUR_ARG_CONTAINER(__gnu_cxx::hash_multiset) +#endif + +#undef OUTPUT_FOUR_ARG_CONTAINER + +#define OUTPUT_FIVE_ARG_CONTAINER(Sequence) \ +template<class T1, class T2, class T3, class T4, class T5> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2, T3, T4, T5>& seq) { \ + @ac_google_namespace@::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +#ifdef GLOG_STL_LOGGING_FOR_UNORDERED +OUTPUT_FIVE_ARG_CONTAINER(std::unordered_map) +OUTPUT_FIVE_ARG_CONTAINER(std::unordered_multimap) +#endif +#ifdef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +OUTPUT_FIVE_ARG_CONTAINER(std::tr1::unordered_map) +OUTPUT_FIVE_ARG_CONTAINER(std::tr1::unordered_multimap) +#endif +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +OUTPUT_FIVE_ARG_CONTAINER(__gnu_cxx::hash_map) +OUTPUT_FIVE_ARG_CONTAINER(__gnu_cxx::hash_multimap) +#endif + +#undef OUTPUT_FIVE_ARG_CONTAINER + +template<class First, class Second> +inline std::ostream& operator<<(std::ostream& out, + const std::pair<First, Second>& p) { + out << '(' << p.first << ", " << p.second << ')'; + return out; +} + +@ac_google_start_namespace@ + +template<class Iter> +inline void PrintSequence(std::ostream& out, Iter begin, Iter end) { + // Output at most 100 elements -- appropriate if used for logging. + for (int i = 0; begin != end && i < 100; ++i, ++begin) { + if (i > 0) out << ' '; + out << *begin; + } + if (begin != end) { + out << " ..."; + } +} + +@ac_google_end_namespace@ + +// Note that this is technically undefined behavior! We are adding things into +// the std namespace for a reason though -- we are providing new operations on +// types which are themselves defined with this namespace. Without this, these +// operator overloads cannot be found via ADL. If these definitions are not +// found via ADL, they must be #included before they're used, which requires +// this header to be included before apparently independent other headers. +// +// For example, base/logging.h defines various template functions to implement +// CHECK_EQ(x, y) and stream x and y into the log in the event the check fails. +// It does so via the function template MakeCheckOpValueString: +// template<class T> +// void MakeCheckOpValueString(strstream* ss, const T& v) { +// (*ss) << v; +// } +// Because 'glog/logging.h' is included before 'glog/stl_logging.h', +// subsequent CHECK_EQ(v1, v2) for vector<...> typed variable v1 and v2 can only +// find these operator definitions via ADL. +// +// Even this solution has problems -- it may pull unintended operators into the +// namespace as well, allowing them to also be found via ADL, and creating code +// that only works with a particular order of includes. Long term, we need to +// move all of the *definitions* into namespace std, bet we need to ensure no +// one references them first. This lets us take that step. We cannot define them +// in both because that would create ambiguous overloads when both are found. +namespace std { using ::operator<<; } + +#endif // UTIL_GTL_STL_LOGGING_INL_H_ diff --git a/third_party/glog/src/glog/vlog_is_on.h.in b/third_party/glog/src/glog/vlog_is_on.h.in new file mode 100644 index 000000000000..3f4c4a32a8b7 --- /dev/null +++ b/third_party/glog/src/glog/vlog_is_on.h.in @@ -0,0 +1,129 @@ +// Copyright (c) 1999, 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Ray Sidney and many others +// +// Defines the VLOG_IS_ON macro that controls the variable-verbosity +// conditional logging. +// +// It's used by VLOG and VLOG_IF in logging.h +// and by RAW_VLOG in raw_logging.h to trigger the logging. +// +// It can also be used directly e.g. like this: +// if (VLOG_IS_ON(2)) { +// // do some logging preparation and logging +// // that can't be accomplished e.g. via just VLOG(2) << ...; +// } +// +// The truth value that VLOG_IS_ON(level) returns is determined by +// the three verbosity level flags: +// --v=<n> Gives the default maximal active V-logging level; +// 0 is the default. +// Normally positive values are used for V-logging levels. +// --vmodule=<str> Gives the per-module maximal V-logging levels to override +// the value given by --v. +// E.g. "my_module=2,foo*=3" would change the logging level +// for all code in source files "my_module.*" and "foo*.*" +// ("-inl" suffixes are also disregarded for this matching). +// +// SetVLOGLevel helper function is provided to do limited dynamic control over +// V-logging by overriding the per-module settings given via --vmodule flag. +// +// CAVEAT: --vmodule functionality is not available in non gcc compilers. +// + +#ifndef BASE_VLOG_IS_ON_H_ +#define BASE_VLOG_IS_ON_H_ + +#include "glog/log_severity.h" + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +#if defined(__GNUC__) +// We emit an anonymous static int* variable at every VLOG_IS_ON(n) site. +// (Normally) the first time every VLOG_IS_ON(n) site is hit, +// we determine what variable will dynamically control logging at this site: +// it's either FLAGS_v or an appropriate internal variable +// matching the current source file that represents results of +// parsing of --vmodule flag and/or SetVLOGLevel calls. +#define VLOG_IS_ON(verboselevel) \ + __extension__ \ + ({ static @ac_google_namespace@::int32* vlocal__ = &@ac_google_namespace@::kLogSiteUninitialized; \ + @ac_google_namespace@::int32 verbose_level__ = (verboselevel); \ + (*vlocal__ >= verbose_level__) && \ + ((vlocal__ != &@ac_google_namespace@::kLogSiteUninitialized) || \ + (@ac_google_namespace@::InitVLOG3__(&vlocal__, &FLAGS_v, \ + __FILE__, verbose_level__))); }) +#else +// GNU extensions not available, so we do not support --vmodule. +// Dynamic value of FLAGS_v always controls the logging level. +#define VLOG_IS_ON(verboselevel) (FLAGS_v >= (verboselevel)) +#endif + +// Set VLOG(_IS_ON) level for module_pattern to log_level. +// This lets us dynamically control what is normally set by the --vmodule flag. +// Returns the level that previously applied to module_pattern. +// NOTE: To change the log level for VLOG(_IS_ON) sites +// that have already executed after/during InitGoogleLogging, +// one needs to supply the exact --vmodule pattern that applied to them. +// (If no --vmodule pattern applied to them +// the value of FLAGS_v will continue to control them.) +extern GOOGLE_GLOG_DLL_DECL int SetVLOGLevel(const char* module_pattern, + int log_level); + +// Various declarations needed for VLOG_IS_ON above: ========================= + +// Special value used to indicate that a VLOG_IS_ON site has not been +// initialized. We make this a large value, so the common-case check +// of "*vlocal__ >= verbose_level__" in VLOG_IS_ON definition +// passes in such cases and InitVLOG3__ is then triggered. +extern @ac_google_namespace@::int32 kLogSiteUninitialized; + +// Helper routine which determines the logging info for a particalur VLOG site. +// site_flag is the address of the site-local pointer to the controlling +// verbosity level +// site_default is the default to use for *site_flag +// fname is the current source file name +// verbose_level is the argument to VLOG_IS_ON +// We will return the return value for VLOG_IS_ON +// and if possible set *site_flag appropriately. +extern GOOGLE_GLOG_DLL_DECL bool InitVLOG3__( + @ac_google_namespace@::int32** site_flag, + @ac_google_namespace@::int32* site_default, + const char* fname, + @ac_google_namespace@::int32 verbose_level); + +#endif // BASE_VLOG_IS_ON_H_ diff --git a/third_party/glog/src/googletest.h b/third_party/glog/src/googletest.h new file mode 100644 index 000000000000..a9e7795dcae1 --- /dev/null +++ b/third_party/glog/src/googletest.h @@ -0,0 +1,611 @@ +// Copyright (c) 2009, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Shinichiro Hamaji +// (based on googletest: http://code.google.com/p/googletest/) + +#ifdef GOOGLETEST_H__ +#error You must not include this file twice. +#endif +#define GOOGLETEST_H__ + +#include "utilities.h" + +#include <ctype.h> +#include <setjmp.h> +#include <time.h> + +#include <map> +#include <sstream> +#include <string> +#include <vector> + +#include <stdio.h> +#include <stdlib.h> + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + +#include "base/commandlineflags.h" + +using std::map; +using std::string; +using std::vector; + +_START_GOOGLE_NAMESPACE_ + +extern GOOGLE_GLOG_DLL_DECL void (*g_logging_fail_func)(); + +_END_GOOGLE_NAMESPACE_ + +#undef GOOGLE_GLOG_DLL_DECL +#define GOOGLE_GLOG_DLL_DECL + +static inline string GetTempDir() { +#ifndef OS_WINDOWS + return "/tmp"; +#else + char tmp[MAX_PATH]; + GetTempPathA(MAX_PATH, tmp); + return tmp; +#endif +} + +#if defined(OS_WINDOWS) && defined(_MSC_VER) && !defined(TEST_SRC_DIR) +// The test will run in glog/vsproject/<project name> +// (e.g., glog/vsproject/logging_unittest). +static const char TEST_SRC_DIR[] = "../.."; +#elif !defined(TEST_SRC_DIR) +# warning TEST_SRC_DIR should be defined in config.h +static const char TEST_SRC_DIR[] = "."; +#endif + +static const uint32_t PTR_TEST_VALUE = 0x12345678; + +DEFINE_string(test_tmpdir, GetTempDir(), "Dir we use for temp files"); +DEFINE_string(test_srcdir, TEST_SRC_DIR, + "Source-dir root, needed to find glog_unittest_flagfile"); +DEFINE_bool(run_benchmark, false, "If true, run benchmarks"); +#ifdef NDEBUG +DEFINE_int32(benchmark_iters, 100000000, "Number of iterations per benchmark"); +#else +DEFINE_int32(benchmark_iters, 100000, "Number of iterations per benchmark"); +#endif + +#ifdef HAVE_LIB_GTEST +# include <gtest/gtest.h> +// Use our ASSERT_DEATH implementation. +# undef ASSERT_DEATH +# undef ASSERT_DEBUG_DEATH +using testing::InitGoogleTest; +#else + +_START_GOOGLE_NAMESPACE_ + +void InitGoogleTest(int*, char**); + +void InitGoogleTest(int*, char**) {} + +// The following is some bare-bones testing infrastructure + +#define EXPECT_TRUE(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "Check failed: %s\n", #cond); \ + exit(1); \ + } \ + } while (0) + +#define EXPECT_FALSE(cond) EXPECT_TRUE(!(cond)) + +#define EXPECT_OP(op, val1, val2) \ + do { \ + if (!((val1) op (val2))) { \ + fprintf(stderr, "Check failed: %s %s %s\n", #val1, #op, #val2); \ + exit(1); \ + } \ + } while (0) + +#define EXPECT_EQ(val1, val2) EXPECT_OP(==, val1, val2) +#define EXPECT_NE(val1, val2) EXPECT_OP(!=, val1, val2) +#define EXPECT_GT(val1, val2) EXPECT_OP(>, val1, val2) +#define EXPECT_LT(val1, val2) EXPECT_OP(<, val1, val2) + +#define EXPECT_NAN(arg) \ + do { \ + if (!isnan(arg)) { \ + fprintf(stderr, "Check failed: isnan(%s)\n", #arg); \ + exit(1); \ + } \ + } while (0) + +#define EXPECT_INF(arg) \ + do { \ + if (!isinf(arg)) { \ + fprintf(stderr, "Check failed: isinf(%s)\n", #arg); \ + exit(1); \ + } \ + } while (0) + +#define EXPECT_DOUBLE_EQ(val1, val2) \ + do { \ + if (((val1) < (val2) - 0.001 || (val1) > (val2) + 0.001)) { \ + fprintf(stderr, "Check failed: %s == %s\n", #val1, #val2); \ + exit(1); \ + } \ + } while (0) + +#define EXPECT_STREQ(val1, val2) \ + do { \ + if (strcmp((val1), (val2)) != 0) { \ + fprintf(stderr, "Check failed: streq(%s, %s)\n", #val1, #val2); \ + exit(1); \ + } \ + } while (0) + +vector<void (*)()> g_testlist; // the tests to run + +#define TEST(a, b) \ + struct Test_##a##_##b { \ + Test_##a##_##b() { g_testlist.push_back(&Run); } \ + static void Run() { FlagSaver fs; RunTest(); } \ + static void RunTest(); \ + }; \ + static Test_##a##_##b g_test_##a##_##b; \ + void Test_##a##_##b::RunTest() + + +static inline int RUN_ALL_TESTS() { + vector<void (*)()>::const_iterator it; + for (it = g_testlist.begin(); it != g_testlist.end(); ++it) { + (*it)(); + } + fprintf(stderr, "Passed %d tests\n\nPASS\n", (int)g_testlist.size()); + return 0; +} + +_END_GOOGLE_NAMESPACE_ + +#endif // ! HAVE_LIB_GTEST + +_START_GOOGLE_NAMESPACE_ + +static bool g_called_abort; +static jmp_buf g_jmp_buf; +static inline void CalledAbort() { + g_called_abort = true; + longjmp(g_jmp_buf, 1); +} + +#ifdef OS_WINDOWS +// TODO(hamaji): Death test somehow doesn't work in Windows. +#define ASSERT_DEATH(fn, msg) +#else +#define ASSERT_DEATH(fn, msg) \ + do { \ + g_called_abort = false; \ + /* in logging.cc */ \ + void (*original_logging_fail_func)() = g_logging_fail_func; \ + g_logging_fail_func = &CalledAbort; \ + if (!setjmp(g_jmp_buf)) fn; \ + /* set back to their default */ \ + g_logging_fail_func = original_logging_fail_func; \ + if (!g_called_abort) { \ + fprintf(stderr, "Function didn't die (%s): %s\n", msg, #fn); \ + exit(1); \ + } \ + } while (0) +#endif + +#ifdef NDEBUG +#define ASSERT_DEBUG_DEATH(fn, msg) +#else +#define ASSERT_DEBUG_DEATH(fn, msg) ASSERT_DEATH(fn, msg) +#endif // NDEBUG + +// Benchmark tools. + +#define BENCHMARK(n) static BenchmarkRegisterer __benchmark_ ## n (#n, &n); + +map<string, void (*)(int)> g_benchlist; // the benchmarks to run + +class BenchmarkRegisterer { + public: + BenchmarkRegisterer(const char* name, void (*function)(int iters)) { + EXPECT_TRUE(g_benchlist.insert(std::make_pair(name, function)).second); + } +}; + +static inline void RunSpecifiedBenchmarks() { + if (!FLAGS_run_benchmark) { + return; + } + + int iter_cnt = FLAGS_benchmark_iters; + puts("Benchmark\tTime(ns)\tIterations"); + for (map<string, void (*)(int)>::const_iterator iter = g_benchlist.begin(); + iter != g_benchlist.end(); + ++iter) { + clock_t start = clock(); + iter->second(iter_cnt); + double elapsed_ns = + ((double)clock() - start) / CLOCKS_PER_SEC * 1000*1000*1000; + printf("%s\t%8.2lf\t%10d\n", + iter->first.c_str(), elapsed_ns / iter_cnt, iter_cnt); + } + puts(""); +} + +// ---------------------------------------------------------------------- +// Golden file functions +// ---------------------------------------------------------------------- + +class CapturedStream { + public: + CapturedStream(int fd, const string & filename) : + fd_(fd), + uncaptured_fd_(-1), + filename_(filename) { + Capture(); + } + + ~CapturedStream() { + if (uncaptured_fd_ != -1) { + CHECK(close(uncaptured_fd_) != -1); + } + } + + // Start redirecting output to a file + void Capture() { + // Keep original stream for later + CHECK(uncaptured_fd_ == -1) << ", Stream " << fd_ << " already captured!"; + uncaptured_fd_ = dup(fd_); + CHECK(uncaptured_fd_ != -1); + + // Open file to save stream to + int cap_fd = open(filename_.c_str(), + O_CREAT | O_TRUNC | O_WRONLY, + S_IRUSR | S_IWUSR); + CHECK(cap_fd != -1); + + // Send stdout/stderr to this file + fflush(NULL); + CHECK(dup2(cap_fd, fd_) != -1); + CHECK(close(cap_fd) != -1); + } + + // Remove output redirection + void StopCapture() { + // Restore original stream + if (uncaptured_fd_ != -1) { + fflush(NULL); + CHECK(dup2(uncaptured_fd_, fd_) != -1); + } + } + + const string & filename() const { return filename_; } + + private: + int fd_; // file descriptor being captured + int uncaptured_fd_; // where the stream was originally being sent to + string filename_; // file where stream is being saved +}; +static CapturedStream * s_captured_streams[STDERR_FILENO+1]; +// Redirect a file descriptor to a file. +// fd - Should be STDOUT_FILENO or STDERR_FILENO +// filename - File where output should be stored +static inline void CaptureTestOutput(int fd, const string & filename) { + CHECK((fd == STDOUT_FILENO) || (fd == STDERR_FILENO)); + CHECK(s_captured_streams[fd] == NULL); + s_captured_streams[fd] = new CapturedStream(fd, filename); +} +static inline void CaptureTestStderr() { + CaptureTestOutput(STDERR_FILENO, FLAGS_test_tmpdir + "/captured.err"); +} +// Return the size (in bytes) of a file +static inline size_t GetFileSize(FILE * file) { + fseek(file, 0, SEEK_END); + return static_cast<size_t>(ftell(file)); +} +// Read the entire content of a file as a string +static inline string ReadEntireFile(FILE * file) { + const size_t file_size = GetFileSize(file); + char * const buffer = new char[file_size]; + + size_t bytes_last_read = 0; // # of bytes read in the last fread() + size_t bytes_read = 0; // # of bytes read so far + + fseek(file, 0, SEEK_SET); + + // Keep reading the file until we cannot read further or the + // pre-determined file size is reached. + do { + bytes_last_read = fread(buffer+bytes_read, 1, file_size-bytes_read, file); + bytes_read += bytes_last_read; + } while (bytes_last_read > 0 && bytes_read < file_size); + + const string content = string(buffer, buffer+bytes_read); + delete[] buffer; + + return content; +} +// Get the captured stdout (when fd is STDOUT_FILENO) or stderr (when +// fd is STDERR_FILENO) as a string +static inline string GetCapturedTestOutput(int fd) { + CHECK(fd == STDOUT_FILENO || fd == STDERR_FILENO); + CapturedStream * const cap = s_captured_streams[fd]; + CHECK(cap) + << ": did you forget CaptureTestStdout() or CaptureTestStderr()?"; + + // Make sure everything is flushed. + cap->StopCapture(); + + // Read the captured file. + FILE * const file = fopen(cap->filename().c_str(), "r"); + const string content = ReadEntireFile(file); + fclose(file); + + delete cap; + s_captured_streams[fd] = NULL; + + return content; +} +// Get the captured stderr of a test as a string. +static inline string GetCapturedTestStderr() { + return GetCapturedTestOutput(STDERR_FILENO); +} + +// Check if the string is [IWEF](\d{8}|YEARDATE) +static inline bool IsLoggingPrefix(const string& s) { + if (s.size() != 9) return false; + if (!strchr("IWEF", s[0])) return false; + for (int i = 1; i <= 8; ++i) { + if (!isdigit(s[i]) && s[i] != "YEARDATE"[i-1]) return false; + } + return true; +} + +// Convert log output into normalized form. +// +// Example: +// I20200102 030405 logging_unittest.cc:345] RAW: vlog -1 +// => IYEARDATE TIME__ logging_unittest.cc:LINE] RAW: vlog -1 +static inline string MungeLine(const string& line) { + std::istringstream iss(line); + string before, logcode_date, time, thread_lineinfo; + iss >> logcode_date; + while (!IsLoggingPrefix(logcode_date)) { + before += " " + logcode_date; + if (!(iss >> logcode_date)) { + // We cannot find the header of log output. + return before; + } + } + if (!before.empty()) before += " "; + iss >> time; + iss >> thread_lineinfo; + CHECK(!thread_lineinfo.empty()); + if (thread_lineinfo[thread_lineinfo.size() - 1] != ']') { + // We found thread ID. + string tmp; + iss >> tmp; + CHECK(!tmp.empty()); + CHECK_EQ(']', tmp[tmp.size() - 1]); + thread_lineinfo = "THREADID " + tmp; + } + size_t index = thread_lineinfo.find(':'); + CHECK_NE(string::npos, index); + thread_lineinfo = thread_lineinfo.substr(0, index+1) + "LINE]"; + string rest; + std::getline(iss, rest); + return (before + logcode_date[0] + "YEARDATE TIME__ " + thread_lineinfo + + MungeLine(rest)); +} + +static inline void StringReplace(string* str, + const string& oldsub, + const string& newsub) { + size_t pos = str->find(oldsub); + if (pos != string::npos) { + str->replace(pos, oldsub.size(), newsub.c_str()); + } +} + +static inline string Munge(const string& filename) { + FILE* fp = fopen(filename.c_str(), "rb"); + CHECK(fp != NULL) << filename << ": couldn't open"; + char buf[4096]; + string result; + while (fgets(buf, 4095, fp)) { + string line = MungeLine(buf); + char null_str[256]; + char ptr_str[256]; + sprintf(null_str, "%p", static_cast<void*>(NULL)); + sprintf(ptr_str, "%p", reinterpret_cast<void*>(PTR_TEST_VALUE)); + + StringReplace(&line, "__NULLP__", null_str); + StringReplace(&line, "__PTRTEST__", ptr_str); + + StringReplace(&line, "__SUCCESS__", StrError(0)); + StringReplace(&line, "__ENOENT__", StrError(ENOENT)); + StringReplace(&line, "__EINTR__", StrError(EINTR)); + StringReplace(&line, "__ENXIO__", StrError(ENXIO)); + StringReplace(&line, "__ENOEXEC__", StrError(ENOEXEC)); + result += line + "\n"; + } + fclose(fp); + return result; +} + +static inline void WriteToFile(const string& body, const string& file) { + FILE* fp = fopen(file.c_str(), "wb"); + fwrite(body.data(), 1, body.size(), fp); + fclose(fp); +} + +static inline bool MungeAndDiffTestStderr(const string& golden_filename) { + CapturedStream* cap = s_captured_streams[STDERR_FILENO]; + CHECK(cap) << ": did you forget CaptureTestStderr()?"; + + cap->StopCapture(); + + // Run munge + const string captured = Munge(cap->filename()); + const string golden = Munge(golden_filename); + if (captured != golden) { + fprintf(stderr, + "Test with golden file failed. We'll try to show the diff:\n"); + string munged_golden = golden_filename + ".munged"; + WriteToFile(golden, munged_golden); + string munged_captured = cap->filename() + ".munged"; + WriteToFile(captured, munged_captured); +#ifdef OS_WINDOWS + string diffcmd("fc " + munged_golden + " " + munged_captured); +#else + string diffcmd("diff -u " + munged_golden + " " + munged_captured); +#endif + if (system(diffcmd.c_str()) != 0) { + fprintf(stderr, "diff command was failed.\n"); + } + unlink(munged_golden.c_str()); + unlink(munged_captured.c_str()); + return false; + } + LOG(INFO) << "Diff was successful"; + return true; +} + +// Save flags used from logging_unittest.cc. +#ifndef HAVE_LIB_GFLAGS +struct FlagSaver { + FlagSaver() + : v_(FLAGS_v), + stderrthreshold_(FLAGS_stderrthreshold), + logtostderr_(FLAGS_logtostderr), + alsologtostderr_(FLAGS_alsologtostderr) {} + ~FlagSaver() { + FLAGS_v = v_; + FLAGS_stderrthreshold = stderrthreshold_; + FLAGS_logtostderr = logtostderr_; + FLAGS_alsologtostderr = alsologtostderr_; + } + int v_; + int stderrthreshold_; + bool logtostderr_; + bool alsologtostderr_; +}; +#endif + +class Thread { + public: + virtual ~Thread() {} + + void SetJoinable(bool) {} +#if defined(OS_WINDOWS) && !defined(OS_CYGWIN) + void Start() { + handle_ = CreateThread(NULL, + 0, + (LPTHREAD_START_ROUTINE)&Thread::InvokeThread, + (LPVOID)this, + 0, + &th_); + CHECK(handle_) << "CreateThread"; + } + void Join() { + WaitForSingleObject(handle_, INFINITE); + } +#elif defined(HAVE_PTHREAD) + void Start() { + pthread_create(&th_, NULL, &Thread::InvokeThread, this); + } + void Join() { + pthread_join(th_, NULL); + } +#else +# error No thread implementation. +#endif + + protected: + virtual void Run() = 0; + + private: + static void* InvokeThread(void* self) { + ((Thread*)self)->Run(); + return NULL; + } + +#if defined(OS_WINDOWS) && !defined(OS_CYGWIN) + HANDLE handle_; + DWORD th_; +#else + pthread_t th_; +#endif +}; + +static inline void SleepForMilliseconds(int t) { +#ifndef OS_WINDOWS +# if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 199309L + const struct timespec req = {0, t * 1000 * 1000}; + nanosleep(&req, NULL); +# else + usleep(t * 1000); +# endif +#else + Sleep(t); +#endif +} + +// Add hook for operator new to ensure there are no memory allocation. + +void (*g_new_hook)() = NULL; + +_END_GOOGLE_NAMESPACE_ + +void* operator new(size_t size) { + if (GOOGLE_NAMESPACE::g_new_hook) { + GOOGLE_NAMESPACE::g_new_hook(); + } + return malloc(size); +} + +void* operator new[](size_t size) { + return ::operator new(size); +} + +void operator delete(void* p) { + free(p); +} + +void operator delete[](void* p) { + ::operator delete(p); +} diff --git a/third_party/glog/src/logging.cc b/third_party/glog/src/logging.cc new file mode 100644 index 000000000000..ddcf91007358 --- /dev/null +++ b/third_party/glog/src/logging.cc @@ -0,0 +1,2344 @@ +// Copyright (c) 1999, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#define _GNU_SOURCE 1 // needed for O_NOFOLLOW and pread()/pwrite() + +#include "utilities.h" + +#include <algorithm> +#include <assert.h> +#include <iomanip> +#include <string> +#ifdef HAVE_UNISTD_H +# include <unistd.h> // For _exit. +#endif +#include <climits> +#include <sys/types.h> +#include <sys/stat.h> +#ifdef HAVE_SYS_UTSNAME_H +# include <sys/utsname.h> // For uname. +#endif +#include <time.h> +#include <fcntl.h> +#include <cstdio> +#include <iostream> +#include <stdarg.h> +#include <stdlib.h> +#ifdef HAVE_PWD_H +# include <pwd.h> +#endif +#ifdef HAVE_SYSLOG_H +# include <syslog.h> +#endif +#include <vector> +#include <errno.h> // for errno +#include <sstream> +#ifdef OS_WINDOWS +#include "windows/dirent.h" +#else +#include <dirent.h> // for automatic removal of old logs +#endif +#include "base/commandlineflags.h" // to get the program name +#include "glog/logging.h" +#include "glog/raw_logging.h" +#include "base/googleinit.h" + +#ifdef HAVE_STACKTRACE +# include "stacktrace.h" +#endif + +using std::string; +using std::vector; +using std::setw; +using std::setfill; +using std::hex; +using std::dec; +using std::min; +using std::ostream; +using std::ostringstream; + +using std::FILE; +using std::fwrite; +using std::fclose; +using std::fflush; +using std::fprintf; +using std::perror; + +#ifdef __QNX__ +using std::fdopen; +#endif + +#ifdef _WIN32 +#define fdopen _fdopen +#endif + +// There is no thread annotation support. +#define EXCLUSIVE_LOCKS_REQUIRED(mu) + +static bool BoolFromEnv(const char *varname, bool defval) { + const char* const valstr = getenv(varname); + if (!valstr) { + return defval; + } + return memchr("tTyY1\0", valstr[0], 6) != NULL; +} + +GLOG_DEFINE_bool(timestamp_in_logfile_name, + BoolFromEnv("GOOGLE_TIMESTAMP_IN_LOGFILE_NAME", true), + "put a timestamp at the end of the log file name"); +GLOG_DEFINE_bool(logtostderr, BoolFromEnv("GOOGLE_LOGTOSTDERR", false), + "log messages go to stderr instead of logfiles"); +GLOG_DEFINE_bool(alsologtostderr, BoolFromEnv("GOOGLE_ALSOLOGTOSTDERR", false), + "log messages go to stderr in addition to logfiles"); +GLOG_DEFINE_bool(colorlogtostderr, false, + "color messages logged to stderr (if supported by terminal)"); +#ifdef OS_LINUX +GLOG_DEFINE_bool(drop_log_memory, true, "Drop in-memory buffers of log contents. " + "Logs can grow very quickly and they are rarely read before they " + "need to be evicted from memory. Instead, drop them from memory " + "as soon as they are flushed to disk."); +#endif + +// By default, errors (including fatal errors) get logged to stderr as +// well as the file. +// +// The default is ERROR instead of FATAL so that users can see problems +// when they run a program without having to look in another file. +DEFINE_int32(stderrthreshold, + GOOGLE_NAMESPACE::GLOG_ERROR, + "log messages at or above this level are copied to stderr in " + "addition to logfiles. This flag obsoletes --alsologtostderr."); + +GLOG_DEFINE_string(alsologtoemail, "", + "log messages go to these email addresses " + "in addition to logfiles"); +GLOG_DEFINE_bool(log_prefix, true, + "Prepend the log prefix to the start of each log line"); +GLOG_DEFINE_int32(minloglevel, 0, "Messages logged at a lower level than this don't " + "actually get logged anywhere"); +GLOG_DEFINE_int32(logbuflevel, 0, + "Buffer log messages logged at this level or lower" + " (-1 means don't buffer; 0 means buffer INFO only;" + " ...)"); +GLOG_DEFINE_int32(logbufsecs, 30, + "Buffer log messages for at most this many seconds"); +GLOG_DEFINE_int32(logemaillevel, 999, + "Email log messages logged at this level or higher" + " (0 means email all; 3 means email FATAL only;" + " ...)"); +GLOG_DEFINE_string(logmailer, "/bin/mail", + "Mailer used to send logging email"); + +// Compute the default value for --log_dir +static const char* DefaultLogDir() { + const char* env; + env = getenv("GOOGLE_LOG_DIR"); + if (env != NULL && env[0] != '\0') { + return env; + } + env = getenv("TEST_TMPDIR"); + if (env != NULL && env[0] != '\0') { + return env; + } + return ""; +} + +GLOG_DEFINE_int32(logfile_mode, 0664, "Log file mode/permissions."); + +GLOG_DEFINE_string(log_dir, DefaultLogDir(), + "If specified, logfiles are written into this directory instead " + "of the default logging directory."); +GLOG_DEFINE_string(log_link, "", "Put additional links to the log " + "files in this directory"); + +GLOG_DEFINE_int32(max_log_size, 1800, + "approx. maximum log file size (in MB). A value of 0 will " + "be silently overridden to 1."); + +GLOG_DEFINE_bool(stop_logging_if_full_disk, false, + "Stop attempting to log to disk if the disk is full."); + +GLOG_DEFINE_string(log_backtrace_at, "", + "Emit a backtrace when logging at file:linenum."); + +// TODO(hamaji): consider windows +#define PATH_SEPARATOR '/' + +#ifndef HAVE_PREAD +#if defined(OS_WINDOWS) +#include <basetsd.h> +#define ssize_t SSIZE_T +#endif +static ssize_t pread(int fd, void* buf, size_t count, off_t offset) { + off_t orig_offset = lseek(fd, 0, SEEK_CUR); + if (orig_offset == (off_t)-1) + return -1; + if (lseek(fd, offset, SEEK_CUR) == (off_t)-1) + return -1; + ssize_t len = read(fd, buf, count); + if (len < 0) + return len; + if (lseek(fd, orig_offset, SEEK_SET) == (off_t)-1) + return -1; + return len; +} +#endif // !HAVE_PREAD + +#ifndef HAVE_PWRITE +static ssize_t pwrite(int fd, void* buf, size_t count, off_t offset) { + off_t orig_offset = lseek(fd, 0, SEEK_CUR); + if (orig_offset == (off_t)-1) + return -1; + if (lseek(fd, offset, SEEK_CUR) == (off_t)-1) + return -1; + ssize_t len = write(fd, buf, count); + if (len < 0) + return len; + if (lseek(fd, orig_offset, SEEK_SET) == (off_t)-1) + return -1; + return len; +} +#endif // !HAVE_PWRITE + +static void GetHostName(string* hostname) { +#if defined(HAVE_SYS_UTSNAME_H) + struct utsname buf; + if (0 != uname(&buf)) { + // ensure null termination on failure + *buf.nodename = '\0'; + } + *hostname = buf.nodename; +#elif defined(OS_WINDOWS) + char buf[MAX_COMPUTERNAME_LENGTH + 1]; + DWORD len = MAX_COMPUTERNAME_LENGTH + 1; + if (GetComputerNameA(buf, &len)) { + *hostname = buf; + } else { + hostname->clear(); + } +#else +# warning There is no way to retrieve the host name. + *hostname = "(unknown)"; +#endif +} + +// Returns true iff terminal supports using colors in output. +static bool TerminalSupportsColor() { + bool term_supports_color = false; +#ifdef OS_WINDOWS + // on Windows TERM variable is usually not set, but the console does + // support colors. + term_supports_color = true; +#else + // On non-Windows platforms, we rely on the TERM variable. + const char* const term = getenv("TERM"); + if (term != NULL && term[0] != '\0') { + term_supports_color = + !strcmp(term, "xterm") || + !strcmp(term, "xterm-color") || + !strcmp(term, "xterm-256color") || + !strcmp(term, "screen-256color") || + !strcmp(term, "konsole") || + !strcmp(term, "konsole-16color") || + !strcmp(term, "konsole-256color") || + !strcmp(term, "screen") || + !strcmp(term, "linux") || + !strcmp(term, "cygwin"); + } +#endif + return term_supports_color; +} + +_START_GOOGLE_NAMESPACE_ + +enum GLogColor { + COLOR_DEFAULT, + COLOR_RED, + COLOR_GREEN, + COLOR_YELLOW +}; + +static GLogColor SeverityToColor(LogSeverity severity) { + assert(severity >= 0 && severity < NUM_SEVERITIES); + GLogColor color = COLOR_DEFAULT; + switch (severity) { + case GLOG_INFO: + color = COLOR_DEFAULT; + break; + case GLOG_WARNING: + color = COLOR_YELLOW; + break; + case GLOG_ERROR: + case GLOG_FATAL: + color = COLOR_RED; + break; + default: + // should never get here. + assert(false); + } + return color; +} + +#ifdef OS_WINDOWS + +// Returns the character attribute for the given color. +static WORD GetColorAttribute(GLogColor color) { + switch (color) { + case COLOR_RED: return FOREGROUND_RED; + case COLOR_GREEN: return FOREGROUND_GREEN; + case COLOR_YELLOW: return FOREGROUND_RED | FOREGROUND_GREEN; + default: return 0; + } +} + +#else + +// Returns the ANSI color code for the given color. +static const char* GetAnsiColorCode(GLogColor color) { + switch (color) { + case COLOR_RED: return "1"; + case COLOR_GREEN: return "2"; + case COLOR_YELLOW: return "3"; + case COLOR_DEFAULT: return ""; + }; + return NULL; // stop warning about return type. +} + +#endif // OS_WINDOWS + +// Safely get max_log_size, overriding to 1 if it somehow gets defined as 0 +static int32 MaxLogSize() { + return (FLAGS_max_log_size > 0 ? FLAGS_max_log_size : 1); +} + +// An arbitrary limit on the length of a single log message. This +// is so that streaming can be done more efficiently. +const size_t LogMessage::kMaxLogMessageLen = 30000; + +struct LogMessage::LogMessageData { + LogMessageData(); + + int preserved_errno_; // preserved errno + // Buffer space; contains complete message text. + char message_text_[LogMessage::kMaxLogMessageLen+1]; + LogStream stream_; + char severity_; // What level is this LogMessage logged at? + int line_; // line number where logging call is. + void (LogMessage::*send_method_)(); // Call this in destructor to send + union { // At most one of these is used: union to keep the size low. + LogSink* sink_; // NULL or sink to send message to + std::vector<std::string>* outvec_; // NULL or vector to push message onto + std::string* message_; // NULL or string to write message into + }; + time_t timestamp_; // Time of creation of LogMessage + struct ::tm tm_time_; // Time of creation of LogMessage + int32 usecs_; // Time of creation of LogMessage - microseconds part + size_t num_prefix_chars_; // # of chars of prefix in this message + size_t num_chars_to_log_; // # of chars of msg to send to log + size_t num_chars_to_syslog_; // # of chars of msg to send to syslog + const char* basename_; // basename of file that called LOG + const char* fullname_; // fullname of file that called LOG + bool has_been_flushed_; // false => data has not been flushed + bool first_fatal_; // true => this was first fatal msg + + private: + LogMessageData(const LogMessageData&); + void operator=(const LogMessageData&); +}; + +// A mutex that allows only one thread to log at a time, to keep things from +// getting jumbled. Some other very uncommon logging operations (like +// changing the destination file for log messages of a given severity) also +// lock this mutex. Please be sure that anybody who might possibly need to +// lock it does so. +static Mutex log_mutex; + +// Number of messages sent at each severity. Under log_mutex. +int64 LogMessage::num_messages_[NUM_SEVERITIES] = {0, 0, 0, 0}; + +// Globally disable log writing (if disk is full) +static bool stop_writing = false; + +const char*const LogSeverityNames[NUM_SEVERITIES] = { + "INFO", "WARNING", "ERROR", "FATAL" +}; + +// Has the user called SetExitOnDFatal(true)? +static bool exit_on_dfatal = true; + +const char* GetLogSeverityName(LogSeverity severity) { + return LogSeverityNames[severity]; +} + +static bool SendEmailInternal(const char*dest, const char *subject, + const char*body, bool use_logging); + +base::Logger::~Logger() { +} + +namespace { + +// Encapsulates all file-system related state +class LogFileObject : public base::Logger { + public: + LogFileObject(LogSeverity severity, const char* base_filename); + ~LogFileObject(); + + virtual void Write(bool force_flush, // Should we force a flush here? + time_t timestamp, // Timestamp for this entry + const char* message, + int message_len); + + // Configuration options + void SetBasename(const char* basename); + void SetExtension(const char* ext); + void SetSymlinkBasename(const char* symlink_basename); + + // Normal flushing routine + virtual void Flush(); + + // It is the actual file length for the system loggers, + // i.e., INFO, ERROR, etc. + virtual uint32 LogSize() { + MutexLock l(&lock_); + return file_length_; + } + + // Internal flush routine. Exposed so that FlushLogFilesUnsafe() + // can avoid grabbing a lock. Usually Flush() calls it after + // acquiring lock_. + void FlushUnlocked(); + + private: + static const uint32 kRolloverAttemptFrequency = 0x20; + + Mutex lock_; + bool base_filename_selected_; + string base_filename_; + string symlink_basename_; + string filename_extension_; // option users can specify (eg to add port#) + FILE* file_; + LogSeverity severity_; + uint32 bytes_since_flush_; + uint32 dropped_mem_length_; + uint32 file_length_; + unsigned int rollover_attempt_; + int64 next_flush_time_; // cycle count at which to flush log + + // Actually create a logfile using the value of base_filename_ and the + // optional argument time_pid_string + // REQUIRES: lock_ is held + bool CreateLogfile(const string& time_pid_string); +}; + +} // namespace + +class LogDestination { + public: + friend class LogMessage; + friend void ReprintFatalMessage(); + friend base::Logger* base::GetLogger(LogSeverity); + friend void base::SetLogger(LogSeverity, base::Logger*); + + // These methods are just forwarded to by their global versions. + static void SetLogDestination(LogSeverity severity, + const char* base_filename); + static void SetLogSymlink(LogSeverity severity, + const char* symlink_basename); + static void AddLogSink(LogSink *destination); + static void RemoveLogSink(LogSink *destination); + static void SetLogFilenameExtension(const char* filename_extension); + static void SetStderrLogging(LogSeverity min_severity); + static void SetEmailLogging(LogSeverity min_severity, const char* addresses); + static void LogToStderr(); + // Flush all log files that are at least at the given severity level + static void FlushLogFiles(int min_severity); + static void FlushLogFilesUnsafe(int min_severity); + + // we set the maximum size of our packet to be 1400, the logic being + // to prevent fragmentation. + // Really this number is arbitrary. + static const int kNetworkBytes = 1400; + + static const string& hostname(); + static const bool& terminal_supports_color() { + return terminal_supports_color_; + } + + static void DeleteLogDestinations(); + + private: + LogDestination(LogSeverity severity, const char* base_filename); + ~LogDestination(); + + // Take a log message of a particular severity and log it to stderr + // iff it's of a high enough severity to deserve it. + static void MaybeLogToStderr(LogSeverity severity, const char* message, + size_t len); + + // Take a log message of a particular severity and log it to email + // iff it's of a high enough severity to deserve it. + static void MaybeLogToEmail(LogSeverity severity, const char* message, + size_t len); + // Take a log message of a particular severity and log it to a file + // iff the base filename is not "" (which means "don't log to me") + static void MaybeLogToLogfile(LogSeverity severity, + time_t timestamp, + const char* message, size_t len); + // Take a log message of a particular severity and log it to the file + // for that severity and also for all files with severity less than + // this severity. + static void LogToAllLogfiles(LogSeverity severity, + time_t timestamp, + const char* message, size_t len); + + // Send logging info to all registered sinks. + static void LogToSinks(LogSeverity severity, + const char *full_filename, + const char *base_filename, + int line, + const struct ::tm* tm_time, + const char* message, + size_t message_len, + int32 usecs); + + // Wait for all registered sinks via WaitTillSent + // including the optional one in "data". + static void WaitForSinks(LogMessage::LogMessageData* data); + + static LogDestination* log_destination(LogSeverity severity); + + LogFileObject fileobject_; + base::Logger* logger_; // Either &fileobject_, or wrapper around it + + static LogDestination* log_destinations_[NUM_SEVERITIES]; + static LogSeverity email_logging_severity_; + static string addresses_; + static string hostname_; + static bool terminal_supports_color_; + + // arbitrary global logging destinations. + static vector<LogSink*>* sinks_; + + // Protects the vector sinks_, + // but not the LogSink objects its elements reference. + static Mutex sink_mutex_; + + // Disallow + LogDestination(const LogDestination&); + LogDestination& operator=(const LogDestination&); +}; + +// Errors do not get logged to email by default. +LogSeverity LogDestination::email_logging_severity_ = 99999; + +string LogDestination::addresses_; +string LogDestination::hostname_; + +vector<LogSink*>* LogDestination::sinks_ = NULL; +Mutex LogDestination::sink_mutex_; +bool LogDestination::terminal_supports_color_ = TerminalSupportsColor(); + +/* static */ +const string& LogDestination::hostname() { + if (hostname_.empty()) { + GetHostName(&hostname_); + if (hostname_.empty()) { + hostname_ = "(unknown)"; + } + } + return hostname_; +} + +LogDestination::LogDestination(LogSeverity severity, + const char* base_filename) + : fileobject_(severity, base_filename), + logger_(&fileobject_) { +} + +LogDestination::~LogDestination() { + if (logger_ && logger_ != &fileobject_) { + // Delete user-specified logger set via SetLogger(). + delete logger_; + } +} + +inline void LogDestination::FlushLogFilesUnsafe(int min_severity) { + // assume we have the log_mutex or we simply don't care + // about it + for (int i = min_severity; i < NUM_SEVERITIES; i++) { + LogDestination* log = log_destinations_[i]; + if (log != NULL) { + // Flush the base fileobject_ logger directly instead of going + // through any wrappers to reduce chance of deadlock. + log->fileobject_.FlushUnlocked(); + } + } +} + +inline void LogDestination::FlushLogFiles(int min_severity) { + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&log_mutex); + for (int i = min_severity; i < NUM_SEVERITIES; i++) { + LogDestination* log = log_destination(i); + if (log != NULL) { + log->logger_->Flush(); + } + } +} + +inline void LogDestination::SetLogDestination(LogSeverity severity, + const char* base_filename) { + assert(severity >= 0 && severity < NUM_SEVERITIES); + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&log_mutex); + log_destination(severity)->fileobject_.SetBasename(base_filename); +} + +inline void LogDestination::SetLogSymlink(LogSeverity severity, + const char* symlink_basename) { + CHECK_GE(severity, 0); + CHECK_LT(severity, NUM_SEVERITIES); + MutexLock l(&log_mutex); + log_destination(severity)->fileobject_.SetSymlinkBasename(symlink_basename); +} + +inline void LogDestination::AddLogSink(LogSink *destination) { + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&sink_mutex_); + if (!sinks_) sinks_ = new vector<LogSink*>; + sinks_->push_back(destination); +} + +inline void LogDestination::RemoveLogSink(LogSink *destination) { + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&sink_mutex_); + // This doesn't keep the sinks in order, but who cares? + if (sinks_) { + for (int i = sinks_->size() - 1; i >= 0; i--) { + if ((*sinks_)[i] == destination) { + (*sinks_)[i] = (*sinks_)[sinks_->size() - 1]; + sinks_->pop_back(); + break; + } + } + } +} + +inline void LogDestination::SetLogFilenameExtension(const char* ext) { + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&log_mutex); + for ( int severity = 0; severity < NUM_SEVERITIES; ++severity ) { + log_destination(severity)->fileobject_.SetExtension(ext); + } +} + +inline void LogDestination::SetStderrLogging(LogSeverity min_severity) { + assert(min_severity >= 0 && min_severity < NUM_SEVERITIES); + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&log_mutex); + FLAGS_stderrthreshold = min_severity; +} + +inline void LogDestination::LogToStderr() { + // *Don't* put this stuff in a mutex lock, since SetStderrLogging & + // SetLogDestination already do the locking! + SetStderrLogging(0); // thus everything is "also" logged to stderr + for ( int i = 0; i < NUM_SEVERITIES; ++i ) { + SetLogDestination(i, ""); // "" turns off logging to a logfile + } +} + +inline void LogDestination::SetEmailLogging(LogSeverity min_severity, + const char* addresses) { + assert(min_severity >= 0 && min_severity < NUM_SEVERITIES); + // Prevent any subtle race conditions by wrapping a mutex lock around + // all this stuff. + MutexLock l(&log_mutex); + LogDestination::email_logging_severity_ = min_severity; + LogDestination::addresses_ = addresses; +} + +static void ColoredWriteToStderr(LogSeverity severity, + const char* message, size_t len) { + const GLogColor color = + (LogDestination::terminal_supports_color() && FLAGS_colorlogtostderr) ? + SeverityToColor(severity) : COLOR_DEFAULT; + + // Avoid using cerr from this module since we may get called during + // exit code, and cerr may be partially or fully destroyed by then. + if (COLOR_DEFAULT == color) { + fwrite(message, len, 1, stderr); + return; + } +#ifdef OS_WINDOWS + const HANDLE stderr_handle = GetStdHandle(STD_ERROR_HANDLE); + + // Gets the current text color. + CONSOLE_SCREEN_BUFFER_INFO buffer_info; + GetConsoleScreenBufferInfo(stderr_handle, &buffer_info); + const WORD old_color_attrs = buffer_info.wAttributes; + + // We need to flush the stream buffers into the console before each + // SetConsoleTextAttribute call lest it affect the text that is already + // printed but has not yet reached the console. + fflush(stderr); + SetConsoleTextAttribute(stderr_handle, + GetColorAttribute(color) | FOREGROUND_INTENSITY); + fwrite(message, len, 1, stderr); + fflush(stderr); + // Restores the text color. + SetConsoleTextAttribute(stderr_handle, old_color_attrs); +#else + fprintf(stderr, "\033[0;3%sm", GetAnsiColorCode(color)); + fwrite(message, len, 1, stderr); + fprintf(stderr, "\033[m"); // Resets the terminal to default. +#endif // OS_WINDOWS +} + +static void WriteToStderr(const char* message, size_t len) { + // Avoid using cerr from this module since we may get called during + // exit code, and cerr may be partially or fully destroyed by then. + fwrite(message, len, 1, stderr); +} + +inline void LogDestination::MaybeLogToStderr(LogSeverity severity, + const char* message, size_t len) { + if ((severity >= FLAGS_stderrthreshold) || FLAGS_alsologtostderr) { + ColoredWriteToStderr(severity, message, len); +#ifdef OS_WINDOWS + // On Windows, also output to the debugger + ::OutputDebugStringA(string(message,len).c_str()); +#endif + } +} + + +inline void LogDestination::MaybeLogToEmail(LogSeverity severity, + const char* message, size_t len) { + if (severity >= email_logging_severity_ || + severity >= FLAGS_logemaillevel) { + string to(FLAGS_alsologtoemail); + if (!addresses_.empty()) { + if (!to.empty()) { + to += ","; + } + to += addresses_; + } + const string subject(string("[LOG] ") + LogSeverityNames[severity] + ": " + + glog_internal_namespace_::ProgramInvocationShortName()); + string body(hostname()); + body += "\n\n"; + body.append(message, len); + + // should NOT use SendEmail(). The caller of this function holds the + // log_mutex and SendEmail() calls LOG/VLOG which will block trying to + // acquire the log_mutex object. Use SendEmailInternal() and set + // use_logging to false. + SendEmailInternal(to.c_str(), subject.c_str(), body.c_str(), false); + } +} + + +inline void LogDestination::MaybeLogToLogfile(LogSeverity severity, + time_t timestamp, + const char* message, + size_t len) { + const bool should_flush = severity > FLAGS_logbuflevel; + LogDestination* destination = log_destination(severity); + destination->logger_->Write(should_flush, timestamp, message, len); +} + +inline void LogDestination::LogToAllLogfiles(LogSeverity severity, + time_t timestamp, + const char* message, + size_t len) { + + if ( FLAGS_logtostderr ) { // global flag: never log to file + ColoredWriteToStderr(severity, message, len); + } else { + for (int i = severity; i >= 0; --i) + LogDestination::MaybeLogToLogfile(i, timestamp, message, len); + } +} + +inline void LogDestination::LogToSinks(LogSeverity severity, + const char *full_filename, + const char *base_filename, + int line, + const struct ::tm* tm_time, + const char* message, + size_t message_len, + int32 usecs) { + ReaderMutexLock l(&sink_mutex_); + if (sinks_) { + for (int i = sinks_->size() - 1; i >= 0; i--) { + (*sinks_)[i]->send(severity, full_filename, base_filename, + line, tm_time, message, message_len, usecs); + } + } +} + +inline void LogDestination::WaitForSinks(LogMessage::LogMessageData* data) { + ReaderMutexLock l(&sink_mutex_); + if (sinks_) { + for (int i = sinks_->size() - 1; i >= 0; i--) { + (*sinks_)[i]->WaitTillSent(); + } + } + const bool send_to_sink = + (data->send_method_ == &LogMessage::SendToSink) || + (data->send_method_ == &LogMessage::SendToSinkAndLog); + if (send_to_sink && data->sink_ != NULL) { + data->sink_->WaitTillSent(); + } +} + +LogDestination* LogDestination::log_destinations_[NUM_SEVERITIES]; + +inline LogDestination* LogDestination::log_destination(LogSeverity severity) { + assert(severity >=0 && severity < NUM_SEVERITIES); + if (!log_destinations_[severity]) { + log_destinations_[severity] = new LogDestination(severity, NULL); + } + return log_destinations_[severity]; +} + +void LogDestination::DeleteLogDestinations() { + for (int severity = 0; severity < NUM_SEVERITIES; ++severity) { + delete log_destinations_[severity]; + log_destinations_[severity] = NULL; + } + MutexLock l(&sink_mutex_); + delete sinks_; + sinks_ = NULL; +} + +namespace { + +bool IsGlogLog(const string& filename) { + // Check if filename matches the pattern of a glog file: + // "<program name>.<hostname>.<user name>.log...". + const int kKeywordCount = 4; + std::string keywords[kKeywordCount] = { + glog_internal_namespace_::ProgramInvocationShortName(), + LogDestination::hostname(), + MyUserName(), + "log" + }; + + int start_pos = 0; + for (int i = 0; i < kKeywordCount; i++) { + if (filename.find(keywords[i], start_pos) == filename.npos) { + return false; + } + start_pos += keywords[i].size() + 1; + } + return true; +} + +bool LastModifiedOver(const string& filepath, int days) { + // Try to get the last modified time of this file. + struct stat file_stat; + + if (stat(filepath.c_str(), &file_stat) == 0) { + // A day is 86400 seconds, so 7 days is 86400 * 7 = 604800 seconds. + time_t last_modified_time = file_stat.st_mtime; + time_t current_time = time(NULL); + return difftime(current_time, last_modified_time) > days * 86400; + } + + // If failed to get file stat, don't return true! + return false; +} + +vector<string> GetOverdueLogNames(string log_directory, int days) { + // The names of overdue logs. + vector<string> overdue_log_names; + + // Try to get all files within log_directory. + DIR *dir; + struct dirent *ent; + + char dir_delim = '/'; +#ifdef OS_WINDOWS + dir_delim = '\\'; +#endif + + // If log_directory doesn't end with a slash, append a slash to it. + if (log_directory.at(log_directory.size() - 1) != dir_delim) { + log_directory += dir_delim; + } + + if ((dir=opendir(log_directory.c_str()))) { + while ((ent=readdir(dir))) { + if (!strcmp(ent->d_name, ".") || !strcmp(ent->d_name, "..")) { + continue; + } + string filepath = log_directory + ent->d_name; + if (IsGlogLog(ent->d_name) && LastModifiedOver(filepath, days)) { + overdue_log_names.push_back(filepath); + } + } + closedir(dir); + } + + return overdue_log_names; +} + +// Is log_cleaner enabled? +// This option can be enabled by calling google::EnableLogCleaner(days) +bool log_cleaner_enabled_; +int log_cleaner_overdue_days_ = 7; + +} // namespace + + +namespace { + +LogFileObject::LogFileObject(LogSeverity severity, + const char* base_filename) + : base_filename_selected_(base_filename != NULL), + base_filename_((base_filename != NULL) ? base_filename : ""), + symlink_basename_(glog_internal_namespace_::ProgramInvocationShortName()), + filename_extension_(), + file_(NULL), + severity_(severity), + bytes_since_flush_(0), + dropped_mem_length_(0), + file_length_(0), + rollover_attempt_(kRolloverAttemptFrequency-1), + next_flush_time_(0) { + assert(severity >= 0); + assert(severity < NUM_SEVERITIES); +} + +LogFileObject::~LogFileObject() { + MutexLock l(&lock_); + if (file_ != NULL) { + fclose(file_); + file_ = NULL; + } +} + +void LogFileObject::SetBasename(const char* basename) { + MutexLock l(&lock_); + base_filename_selected_ = true; + if (base_filename_ != basename) { + // Get rid of old log file since we are changing names + if (file_ != NULL) { + fclose(file_); + file_ = NULL; + rollover_attempt_ = kRolloverAttemptFrequency-1; + } + base_filename_ = basename; + } +} + +void LogFileObject::SetExtension(const char* ext) { + MutexLock l(&lock_); + if (filename_extension_ != ext) { + // Get rid of old log file since we are changing names + if (file_ != NULL) { + fclose(file_); + file_ = NULL; + rollover_attempt_ = kRolloverAttemptFrequency-1; + } + filename_extension_ = ext; + } +} + +void LogFileObject::SetSymlinkBasename(const char* symlink_basename) { + MutexLock l(&lock_); + symlink_basename_ = symlink_basename; +} + +void LogFileObject::Flush() { + MutexLock l(&lock_); + FlushUnlocked(); +} + +void LogFileObject::FlushUnlocked(){ + if (file_ != NULL) { + fflush(file_); + bytes_since_flush_ = 0; + } + // Figure out when we are due for another flush. + const int64 next = (FLAGS_logbufsecs + * static_cast<int64>(1000000)); // in usec + next_flush_time_ = CycleClock_Now() + UsecToCycles(next); +} + +bool LogFileObject::CreateLogfile(const string& time_pid_string) { + string string_filename = base_filename_+filename_extension_; + if (FLAGS_timestamp_in_logfile_name) { + string_filename += time_pid_string; + } + const char* filename = string_filename.c_str(); + //only write to files, create if non-existant. + int flags = O_WRONLY | O_CREAT; + if (FLAGS_timestamp_in_logfile_name) { + //demand that the file is unique for our timestamp (fail if it exists). + flags = flags | O_EXCL; + } + int fd = open(filename, flags, FLAGS_logfile_mode); + if (fd == -1) return false; +#ifdef HAVE_FCNTL + // Mark the file close-on-exec. We don't really care if this fails + fcntl(fd, F_SETFD, FD_CLOEXEC); + + // Mark the file as exclusive write access to avoid two clients logging to the + // same file. This applies particularly when !FLAGS_timestamp_in_logfile_name + // (otherwise open would fail because the O_EXCL flag on similar filename). + // locks are released on unlock or close() automatically, only after log is + // released. + // This will work after a fork as it is not inherited (not stored in the fd). + // Lock will not be lost because the file is opened with exclusive lock (write) + // and we will never read from it inside the process. + // TODO windows implementation of this (as flock is not available on mingw). + static struct flock w_lock; + + w_lock.l_type = F_WRLCK; + w_lock.l_start = 0; + w_lock.l_whence = SEEK_SET; + w_lock.l_len = 0; + + int wlock_ret = fcntl(fd, F_SETLK, &w_lock); + if (wlock_ret == -1) { + close(fd); //as we are failing already, do not check errors here + return false; + } +#endif + + //fdopen in append mode so if the file exists it will fseek to the end + file_ = fdopen(fd, "a"); // Make a FILE*. + if (file_ == NULL) { // Man, we're screwed! + close(fd); + if (FLAGS_timestamp_in_logfile_name) { + unlink(filename); // Erase the half-baked evidence: an unusable log file, only if we just created it. + } + return false; + } +#ifdef OS_WINDOWS + // https://github.com/golang/go/issues/27638 - make sure we seek to the end to append + // empirically replicated with wine over mingw build + if (!FLAGS_timestamp_in_logfile_name) { + if (fseek(file_, 0, SEEK_END) != 0) { + return false; + } + } +#endif + // We try to create a symlink called <program_name>.<severity>, + // which is easier to use. (Every time we create a new logfile, + // we destroy the old symlink and create a new one, so it always + // points to the latest logfile.) If it fails, we're sad but it's + // no error. + if (!symlink_basename_.empty()) { + // take directory from filename + const char* slash = strrchr(filename, PATH_SEPARATOR); + const string linkname = + symlink_basename_ + '.' + LogSeverityNames[severity_]; + string linkpath; + if ( slash ) linkpath = string(filename, slash-filename+1); // get dirname + linkpath += linkname; + unlink(linkpath.c_str()); // delete old one if it exists + +#if defined(OS_WINDOWS) + // TODO(hamaji): Create lnk file on Windows? +#elif defined(HAVE_UNISTD_H) + // We must have unistd.h. + // Make the symlink be relative (in the same dir) so that if the + // entire log directory gets relocated the link is still valid. + const char *linkdest = slash ? (slash + 1) : filename; + if (symlink(linkdest, linkpath.c_str()) != 0) { + // silently ignore failures + } + + // Make an additional link to the log file in a place specified by + // FLAGS_log_link, if indicated + if (!FLAGS_log_link.empty()) { + linkpath = FLAGS_log_link + "/" + linkname; + unlink(linkpath.c_str()); // delete old one if it exists + if (symlink(filename, linkpath.c_str()) != 0) { + // silently ignore failures + } + } +#endif + } + + return true; // Everything worked +} + +void LogFileObject::Write(bool force_flush, + time_t timestamp, + const char* message, + int message_len) { + MutexLock l(&lock_); + + // We don't log if the base_name_ is "" (which means "don't write") + if (base_filename_selected_ && base_filename_.empty()) { + return; + } + + if (static_cast<int>(file_length_ >> 20) >= MaxLogSize() || + PidHasChanged()) { + if (file_ != NULL) fclose(file_); + file_ = NULL; + file_length_ = bytes_since_flush_ = dropped_mem_length_ = 0; + rollover_attempt_ = kRolloverAttemptFrequency-1; + } + + // If there's no destination file, make one before outputting + if (file_ == NULL) { + // Try to rollover the log file every 32 log messages. The only time + // this could matter would be when we have trouble creating the log + // file. If that happens, we'll lose lots of log messages, of course! + if (++rollover_attempt_ != kRolloverAttemptFrequency) return; + rollover_attempt_ = 0; + + struct ::tm tm_time; + localtime_r(×tamp, &tm_time); + + // The logfile's filename will have the date/time & pid in it + ostringstream time_pid_stream; + time_pid_stream.fill('0'); + time_pid_stream << 1900+tm_time.tm_year + << setw(2) << 1+tm_time.tm_mon + << setw(2) << tm_time.tm_mday + << '-' + << setw(2) << tm_time.tm_hour + << setw(2) << tm_time.tm_min + << setw(2) << tm_time.tm_sec + << '.' + << GetMainThreadPid(); + const string& time_pid_string = time_pid_stream.str(); + + if (base_filename_selected_) { + if (!CreateLogfile(time_pid_string)) { + perror("Could not create log file"); + fprintf(stderr, "COULD NOT CREATE LOGFILE '%s'!\n", + time_pid_string.c_str()); + return; + } + } else { + // If no base filename for logs of this severity has been set, use a + // default base filename of + // "<program name>.<hostname>.<user name>.log.<severity level>.". So + // logfiles will have names like + // webserver.examplehost.root.log.INFO.19990817-150000.4354, where + // 19990817 is a date (1999 August 17), 150000 is a time (15:00:00), + // and 4354 is the pid of the logging process. The date & time reflect + // when the file was created for output. + // + // Where does the file get put? Successively try the directories + // "/tmp", and "." + string stripped_filename( + glog_internal_namespace_::ProgramInvocationShortName()); + string hostname; + GetHostName(&hostname); + + string uidname = MyUserName(); + // We should not call CHECK() here because this function can be + // called after holding on to log_mutex. We don't want to + // attempt to hold on to the same mutex, and get into a + // deadlock. Simply use a name like invalid-user. + if (uidname.empty()) uidname = "invalid-user"; + + stripped_filename = stripped_filename+'.'+hostname+'.' + +uidname+".log." + +LogSeverityNames[severity_]+'.'; + // We're going to (potentially) try to put logs in several different dirs + const vector<string> & log_dirs = GetLoggingDirectories(); + + // Go through the list of dirs, and try to create the log file in each + // until we succeed or run out of options + bool success = false; + for (vector<string>::const_iterator dir = log_dirs.begin(); + dir != log_dirs.end(); + ++dir) { + base_filename_ = *dir + "/" + stripped_filename; + if ( CreateLogfile(time_pid_string) ) { + success = true; + break; + } + } + // If we never succeeded, we have to give up + if ( success == false ) { + perror("Could not create logging file"); + fprintf(stderr, "COULD NOT CREATE A LOGGINGFILE %s!", + time_pid_string.c_str()); + return; + } + } + + // Write a header message into the log file + ostringstream file_header_stream; + file_header_stream.fill('0'); + file_header_stream << "Log file created at: " + << 1900+tm_time.tm_year << '/' + << setw(2) << 1+tm_time.tm_mon << '/' + << setw(2) << tm_time.tm_mday + << ' ' + << setw(2) << tm_time.tm_hour << ':' + << setw(2) << tm_time.tm_min << ':' + << setw(2) << tm_time.tm_sec << '\n' + << "Running on machine: " + << LogDestination::hostname() << '\n' + << "Log line format: [IWEF]yyyymmdd hh:mm:ss.uuuuuu " + << "threadid file:line] msg" << '\n'; + const string& file_header_string = file_header_stream.str(); + + const int header_len = file_header_string.size(); + fwrite(file_header_string.data(), 1, header_len, file_); + file_length_ += header_len; + bytes_since_flush_ += header_len; + } + + // Write to LOG file + if ( !stop_writing ) { + // fwrite() doesn't return an error when the disk is full, for + // messages that are less than 4096 bytes. When the disk is full, + // it returns the message length for messages that are less than + // 4096 bytes. fwrite() returns 4096 for message lengths that are + // greater than 4096, thereby indicating an error. + errno = 0; + fwrite(message, 1, message_len, file_); + if ( FLAGS_stop_logging_if_full_disk && + errno == ENOSPC ) { // disk full, stop writing to disk + stop_writing = true; // until the disk is + return; + } else { + file_length_ += message_len; + bytes_since_flush_ += message_len; + } + } else { + if ( CycleClock_Now() >= next_flush_time_ ) + stop_writing = false; // check to see if disk has free space. + return; // no need to flush + } + + // See important msgs *now*. Also, flush logs at least every 10^6 chars, + // or every "FLAGS_logbufsecs" seconds. + if ( force_flush || + (bytes_since_flush_ >= 1000000) || + (CycleClock_Now() >= next_flush_time_) ) { + FlushUnlocked(); +#ifdef OS_LINUX + // Only consider files >= 3MiB + if (FLAGS_drop_log_memory && file_length_ >= (3 << 20)) { + // Don't evict the most recent 1-2MiB so as not to impact a tailer + // of the log file and to avoid page rounding issue on linux < 4.7 + uint32 total_drop_length = (file_length_ & ~((1 << 20) - 1)) - (1 << 20); + uint32 this_drop_length = total_drop_length - dropped_mem_length_; + if (this_drop_length >= (2 << 20)) { + // Only advise when >= 2MiB to drop +# if defined(__ANDROID__) && defined(__ANDROID_API__) && (__ANDROID_API__ < 21) + // 'posix_fadvise' introduced in API 21: + // * https://android.googlesource.com/platform/bionic/+/6880f936173081297be0dc12f687d341b86a4cfa/libc/libc.map.txt#732 +# else + posix_fadvise(fileno(file_), dropped_mem_length_, this_drop_length, + POSIX_FADV_DONTNEED); +# endif + dropped_mem_length_ = total_drop_length; + } + } +#endif + // Perform clean up for old logs + if (log_cleaner_enabled_) { + const vector<string>& dirs = GetLoggingDirectories(); + for (size_t i = 0; i < dirs.size(); i++) { + vector<string> logs = GetOverdueLogNames(dirs[i], log_cleaner_overdue_days_); + for (size_t j = 0; j < logs.size(); j++) { + static_cast<void>(unlink(logs[j].c_str())); + } + } + } + } +} + +} // namespace + +// Static log data space to avoid alloc failures in a LOG(FATAL) +// +// Since multiple threads may call LOG(FATAL), and we want to preserve +// the data from the first call, we allocate two sets of space. One +// for exclusive use by the first thread, and one for shared use by +// all other threads. +static Mutex fatal_msg_lock; +static CrashReason crash_reason; +static bool fatal_msg_exclusive = true; +static LogMessage::LogMessageData fatal_msg_data_exclusive; +static LogMessage::LogMessageData fatal_msg_data_shared; + +#ifdef GLOG_THREAD_LOCAL_STORAGE +// Static thread-local log data space to use, because typically at most one +// LogMessageData object exists (in this case glog makes zero heap memory +// allocations). +static GLOG_THREAD_LOCAL_STORAGE bool thread_data_available = true; + +#ifdef HAVE_ALIGNED_STORAGE +static GLOG_THREAD_LOCAL_STORAGE + std::aligned_storage<sizeof(LogMessage::LogMessageData), + alignof(LogMessage::LogMessageData)>::type thread_msg_data; +#else +static GLOG_THREAD_LOCAL_STORAGE + char thread_msg_data[sizeof(void*) + sizeof(LogMessage::LogMessageData)]; +#endif // HAVE_ALIGNED_STORAGE +#endif // defined(GLOG_THREAD_LOCAL_STORAGE) + +LogMessage::LogMessageData::LogMessageData() + : stream_(message_text_, LogMessage::kMaxLogMessageLen, 0) { +} + +LogMessage::LogMessage(const char* file, int line, LogSeverity severity, + int ctr, void (LogMessage::*send_method)()) + : allocated_(NULL) { + Init(file, line, severity, send_method); + data_->stream_.set_ctr(ctr); +} + +LogMessage::LogMessage(const char* file, int line, + const CheckOpString& result) + : allocated_(NULL) { + Init(file, line, GLOG_FATAL, &LogMessage::SendToLog); + stream() << "Check failed: " << (*result.str_) << " "; +} + +LogMessage::LogMessage(const char* file, int line) + : allocated_(NULL) { + Init(file, line, GLOG_INFO, &LogMessage::SendToLog); +} + +LogMessage::LogMessage(const char* file, int line, LogSeverity severity) + : allocated_(NULL) { + Init(file, line, severity, &LogMessage::SendToLog); +} + +LogMessage::LogMessage(const char* file, int line, LogSeverity severity, + LogSink* sink, bool also_send_to_log) + : allocated_(NULL) { + Init(file, line, severity, also_send_to_log ? &LogMessage::SendToSinkAndLog : + &LogMessage::SendToSink); + data_->sink_ = sink; // override Init()'s setting to NULL +} + +LogMessage::LogMessage(const char* file, int line, LogSeverity severity, + vector<string> *outvec) + : allocated_(NULL) { + Init(file, line, severity, &LogMessage::SaveOrSendToLog); + data_->outvec_ = outvec; // override Init()'s setting to NULL +} + +LogMessage::LogMessage(const char* file, int line, LogSeverity severity, + string *message) + : allocated_(NULL) { + Init(file, line, severity, &LogMessage::WriteToStringAndLog); + data_->message_ = message; // override Init()'s setting to NULL +} + +void LogMessage::Init(const char* file, + int line, + LogSeverity severity, + void (LogMessage::*send_method)()) { + allocated_ = NULL; + if (severity != GLOG_FATAL || !exit_on_dfatal) { +#ifdef GLOG_THREAD_LOCAL_STORAGE + // No need for locking, because this is thread local. + if (thread_data_available) { + thread_data_available = false; +#ifdef HAVE_ALIGNED_STORAGE + data_ = new (&thread_msg_data) LogMessageData; +#else + const uintptr_t kAlign = sizeof(void*) - 1; + + char* align_ptr = + reinterpret_cast<char*>(reinterpret_cast<uintptr_t>(thread_msg_data + kAlign) & ~kAlign); + data_ = new (align_ptr) LogMessageData; + assert(reinterpret_cast<uintptr_t>(align_ptr) % sizeof(void*) == 0); +#endif + } else { + allocated_ = new LogMessageData(); + data_ = allocated_; + } +#else // !defined(GLOG_THREAD_LOCAL_STORAGE) + allocated_ = new LogMessageData(); + data_ = allocated_; +#endif // defined(GLOG_THREAD_LOCAL_STORAGE) + data_->first_fatal_ = false; + } else { + MutexLock l(&fatal_msg_lock); + if (fatal_msg_exclusive) { + fatal_msg_exclusive = false; + data_ = &fatal_msg_data_exclusive; + data_->first_fatal_ = true; + } else { + data_ = &fatal_msg_data_shared; + data_->first_fatal_ = false; + } + } + + stream().fill('0'); + data_->preserved_errno_ = errno; + data_->severity_ = severity; + data_->line_ = line; + data_->send_method_ = send_method; + data_->sink_ = NULL; + data_->outvec_ = NULL; + WallTime now = WallTime_Now(); + data_->timestamp_ = static_cast<time_t>(now); + localtime_r(&data_->timestamp_, &data_->tm_time_); + data_->usecs_ = static_cast<int32>((now - data_->timestamp_) * 1000000); + + data_->num_chars_to_log_ = 0; + data_->num_chars_to_syslog_ = 0; + data_->basename_ = const_basename(file); + data_->fullname_ = file; + data_->has_been_flushed_ = false; + + // If specified, prepend a prefix to each line. For example: + // I20201018 160715 f5d4fbb0 logging.cc:1153] + // (log level, GMT year, month, date, time, thread_id, file basename, line) + // We exclude the thread_id for the default thread. + if (FLAGS_log_prefix && (line != kNoLogPrefix)) { + stream() << LogSeverityNames[severity][0] + << setw(4) << 1900+data_->tm_time_.tm_year + << setw(2) << 1+data_->tm_time_.tm_mon + << setw(2) << data_->tm_time_.tm_mday + << ' ' + << setw(2) << data_->tm_time_.tm_hour << ':' + << setw(2) << data_->tm_time_.tm_min << ':' + << setw(2) << data_->tm_time_.tm_sec << "." + << setw(6) << data_->usecs_ + << ' ' + << setfill(' ') << setw(5) + << static_cast<unsigned int>(GetTID()) << setfill('0') + << ' ' + << data_->basename_ << ':' << data_->line_ << "] "; + } + data_->num_prefix_chars_ = data_->stream_.pcount(); + + if (!FLAGS_log_backtrace_at.empty()) { + char fileline[128]; + snprintf(fileline, sizeof(fileline), "%s:%d", data_->basename_, line); +#ifdef HAVE_STACKTRACE + if (!strcmp(FLAGS_log_backtrace_at.c_str(), fileline)) { + string stacktrace; + DumpStackTraceToString(&stacktrace); + stream() << " (stacktrace:\n" << stacktrace << ") "; + } +#endif + } +} + +LogMessage::~LogMessage() { + Flush(); +#ifdef GLOG_THREAD_LOCAL_STORAGE + if (data_ == static_cast<void*>(&thread_msg_data)) { + data_->~LogMessageData(); + thread_data_available = true; + } + else { + delete allocated_; + } +#else // !defined(GLOG_THREAD_LOCAL_STORAGE) + delete allocated_; +#endif // defined(GLOG_THREAD_LOCAL_STORAGE) +} + +int LogMessage::preserved_errno() const { + return data_->preserved_errno_; +} + +ostream& LogMessage::stream() { + return data_->stream_; +} + +// Flush buffered message, called by the destructor, or any other function +// that needs to synchronize the log. +void LogMessage::Flush() { + if (data_->has_been_flushed_ || data_->severity_ < FLAGS_minloglevel) + return; + + data_->num_chars_to_log_ = data_->stream_.pcount(); + data_->num_chars_to_syslog_ = + data_->num_chars_to_log_ - data_->num_prefix_chars_; + + // Do we need to add a \n to the end of this message? + bool append_newline = + (data_->message_text_[data_->num_chars_to_log_-1] != '\n'); + char original_final_char = '\0'; + + // If we do need to add a \n, we'll do it by violating the memory of the + // ostrstream buffer. This is quick, and we'll make sure to undo our + // modification before anything else is done with the ostrstream. It + // would be preferable not to do things this way, but it seems to be + // the best way to deal with this. + if (append_newline) { + original_final_char = data_->message_text_[data_->num_chars_to_log_]; + data_->message_text_[data_->num_chars_to_log_++] = '\n'; + } + + // Prevent any subtle race conditions by wrapping a mutex lock around + // the actual logging action per se. + { + MutexLock l(&log_mutex); + (this->*(data_->send_method_))(); + ++num_messages_[static_cast<int>(data_->severity_)]; + } + LogDestination::WaitForSinks(data_); + + if (append_newline) { + // Fix the ostrstream back how it was before we screwed with it. + // It's 99.44% certain that we don't need to worry about doing this. + data_->message_text_[data_->num_chars_to_log_-1] = original_final_char; + } + + // If errno was already set before we enter the logging call, we'll + // set it back to that value when we return from the logging call. + // It happens often that we log an error message after a syscall + // failure, which can potentially set the errno to some other + // values. We would like to preserve the original errno. + if (data_->preserved_errno_ != 0) { + errno = data_->preserved_errno_; + } + + // Note that this message is now safely logged. If we're asked to flush + // again, as a result of destruction, say, we'll do nothing on future calls. + data_->has_been_flushed_ = true; +} + +// Copy of first FATAL log message so that we can print it out again +// after all the stack traces. To preserve legacy behavior, we don't +// use fatal_msg_data_exclusive. +static time_t fatal_time; +static char fatal_message[256]; + +void ReprintFatalMessage() { + if (fatal_message[0]) { + const int n = strlen(fatal_message); + if (!FLAGS_logtostderr) { + // Also write to stderr (don't color to avoid terminal checks) + WriteToStderr(fatal_message, n); + } + LogDestination::LogToAllLogfiles(GLOG_ERROR, fatal_time, fatal_message, n); + } +} + +// L >= log_mutex (callers must hold the log_mutex). +void LogMessage::SendToLog() EXCLUSIVE_LOCKS_REQUIRED(log_mutex) { + static bool already_warned_before_initgoogle = false; + + log_mutex.AssertHeld(); + + RAW_DCHECK(data_->num_chars_to_log_ > 0 && + data_->message_text_[data_->num_chars_to_log_-1] == '\n', ""); + + // Messages of a given severity get logged to lower severity logs, too + + if (!already_warned_before_initgoogle && !IsGoogleLoggingInitialized()) { + const char w[] = "WARNING: Logging before InitGoogleLogging() is " + "written to STDERR\n"; + WriteToStderr(w, strlen(w)); + already_warned_before_initgoogle = true; + } + + // global flag: never log to file if set. Also -- don't log to a + // file if we haven't parsed the command line flags to get the + // program name. + if (FLAGS_logtostderr || !IsGoogleLoggingInitialized()) { + ColoredWriteToStderr(data_->severity_, + data_->message_text_, data_->num_chars_to_log_); + + // this could be protected by a flag if necessary. + LogDestination::LogToSinks(data_->severity_, + data_->fullname_, data_->basename_, + data_->line_, &data_->tm_time_, + data_->message_text_ + data_->num_prefix_chars_, + (data_->num_chars_to_log_ - + data_->num_prefix_chars_ - 1), + data_->usecs_); + } else { + + // log this message to all log files of severity <= severity_ + LogDestination::LogToAllLogfiles(data_->severity_, data_->timestamp_, + data_->message_text_, + data_->num_chars_to_log_); + + LogDestination::MaybeLogToStderr(data_->severity_, data_->message_text_, + data_->num_chars_to_log_); + LogDestination::MaybeLogToEmail(data_->severity_, data_->message_text_, + data_->num_chars_to_log_); + LogDestination::LogToSinks(data_->severity_, + data_->fullname_, data_->basename_, + data_->line_, &data_->tm_time_, + data_->message_text_ + data_->num_prefix_chars_, + (data_->num_chars_to_log_ + - data_->num_prefix_chars_ - 1), + data_->usecs_); + // NOTE: -1 removes trailing \n + } + + // If we log a FATAL message, flush all the log destinations, then toss + // a signal for others to catch. We leave the logs in a state that + // someone else can use them (as long as they flush afterwards) + if (data_->severity_ == GLOG_FATAL && exit_on_dfatal) { + if (data_->first_fatal_) { + // Store crash information so that it is accessible from within signal + // handlers that may be invoked later. + RecordCrashReason(&crash_reason); + SetCrashReason(&crash_reason); + + // Store shortened fatal message for other logs and GWQ status + const int copy = min<int>(data_->num_chars_to_log_, + sizeof(fatal_message)-1); + memcpy(fatal_message, data_->message_text_, copy); + fatal_message[copy] = '\0'; + fatal_time = data_->timestamp_; + } + + if (!FLAGS_logtostderr) { + for (int i = 0; i < NUM_SEVERITIES; ++i) { + if ( LogDestination::log_destinations_[i] ) + LogDestination::log_destinations_[i]->logger_->Write(true, 0, "", 0); + } + } + + // release the lock that our caller (directly or indirectly) + // LogMessage::~LogMessage() grabbed so that signal handlers + // can use the logging facility. Alternately, we could add + // an entire unsafe logging interface to bypass locking + // for signal handlers but this seems simpler. + log_mutex.Unlock(); + LogDestination::WaitForSinks(data_); + + const char* message = "*** Check failure stack trace: ***\n"; + if (write(STDERR_FILENO, message, strlen(message)) < 0) { + // Ignore errors. + } + Fail(); + } +} + +void LogMessage::RecordCrashReason( + glog_internal_namespace_::CrashReason* reason) { + reason->filename = fatal_msg_data_exclusive.fullname_; + reason->line_number = fatal_msg_data_exclusive.line_; + reason->message = fatal_msg_data_exclusive.message_text_ + + fatal_msg_data_exclusive.num_prefix_chars_; +#ifdef HAVE_STACKTRACE + // Retrieve the stack trace, omitting the logging frames that got us here. + reason->depth = GetStackTrace(reason->stack, ARRAYSIZE(reason->stack), 4); +#else + reason->depth = 0; +#endif +} + +#ifdef HAVE___ATTRIBUTE__ +# define ATTRIBUTE_NORETURN __attribute__((noreturn)) +#else +# define ATTRIBUTE_NORETURN +#endif + +#if defined(OS_WINDOWS) +__declspec(noreturn) +#endif +static void logging_fail() ATTRIBUTE_NORETURN; + +static void logging_fail() { + abort(); +} + +typedef void (*logging_fail_func_t)() ATTRIBUTE_NORETURN; + +GOOGLE_GLOG_DLL_DECL +logging_fail_func_t g_logging_fail_func = &logging_fail; + +void InstallFailureFunction(void (*fail_func)()) { + g_logging_fail_func = (logging_fail_func_t)fail_func; +} + +void LogMessage::Fail() { + g_logging_fail_func(); +} + +// L >= log_mutex (callers must hold the log_mutex). +void LogMessage::SendToSink() EXCLUSIVE_LOCKS_REQUIRED(log_mutex) { + if (data_->sink_ != NULL) { + RAW_DCHECK(data_->num_chars_to_log_ > 0 && + data_->message_text_[data_->num_chars_to_log_-1] == '\n', ""); + data_->sink_->send(data_->severity_, data_->fullname_, data_->basename_, + data_->line_, &data_->tm_time_, + data_->message_text_ + data_->num_prefix_chars_, + (data_->num_chars_to_log_ - + data_->num_prefix_chars_ - 1), + data_->usecs_); + } +} + +// L >= log_mutex (callers must hold the log_mutex). +void LogMessage::SendToSinkAndLog() EXCLUSIVE_LOCKS_REQUIRED(log_mutex) { + SendToSink(); + SendToLog(); +} + +// L >= log_mutex (callers must hold the log_mutex). +void LogMessage::SaveOrSendToLog() EXCLUSIVE_LOCKS_REQUIRED(log_mutex) { + if (data_->outvec_ != NULL) { + RAW_DCHECK(data_->num_chars_to_log_ > 0 && + data_->message_text_[data_->num_chars_to_log_-1] == '\n', ""); + // Omit prefix of message and trailing newline when recording in outvec_. + const char *start = data_->message_text_ + data_->num_prefix_chars_; + int len = data_->num_chars_to_log_ - data_->num_prefix_chars_ - 1; + data_->outvec_->push_back(string(start, len)); + } else { + SendToLog(); + } +} + +void LogMessage::WriteToStringAndLog() EXCLUSIVE_LOCKS_REQUIRED(log_mutex) { + if (data_->message_ != NULL) { + RAW_DCHECK(data_->num_chars_to_log_ > 0 && + data_->message_text_[data_->num_chars_to_log_-1] == '\n', ""); + // Omit prefix of message and trailing newline when writing to message_. + const char *start = data_->message_text_ + data_->num_prefix_chars_; + int len = data_->num_chars_to_log_ - data_->num_prefix_chars_ - 1; + data_->message_->assign(start, len); + } + SendToLog(); +} + +// L >= log_mutex (callers must hold the log_mutex). +void LogMessage::SendToSyslogAndLog() { +#ifdef HAVE_SYSLOG_H + // Before any calls to syslog(), make a single call to openlog() + static bool openlog_already_called = false; + if (!openlog_already_called) { + openlog(glog_internal_namespace_::ProgramInvocationShortName(), + LOG_CONS | LOG_NDELAY | LOG_PID, + LOG_USER); + openlog_already_called = true; + } + + // This array maps Google severity levels to syslog levels + const int SEVERITY_TO_LEVEL[] = { LOG_INFO, LOG_WARNING, LOG_ERR, LOG_EMERG }; + syslog(LOG_USER | SEVERITY_TO_LEVEL[static_cast<int>(data_->severity_)], "%.*s", + int(data_->num_chars_to_syslog_), + data_->message_text_ + data_->num_prefix_chars_); + SendToLog(); +#else + LOG(ERROR) << "No syslog support: message=" << data_->message_text_; +#endif +} + +base::Logger* base::GetLogger(LogSeverity severity) { + MutexLock l(&log_mutex); + return LogDestination::log_destination(severity)->logger_; +} + +void base::SetLogger(LogSeverity severity, base::Logger* logger) { + MutexLock l(&log_mutex); + LogDestination::log_destination(severity)->logger_ = logger; +} + +// L < log_mutex. Acquires and releases mutex_. +int64 LogMessage::num_messages(int severity) { + MutexLock l(&log_mutex); + return num_messages_[severity]; +} + +// Output the COUNTER value. This is only valid if ostream is a +// LogStream. +ostream& operator<<(ostream &os, const PRIVATE_Counter&) { +#ifdef DISABLE_RTTI + LogMessage::LogStream *log = static_cast<LogMessage::LogStream*>(&os); +#else + LogMessage::LogStream *log = dynamic_cast<LogMessage::LogStream*>(&os); +#endif + CHECK(log && log == log->self()) + << "You must not use COUNTER with non-glog ostream"; + os << log->ctr(); + return os; +} + +ErrnoLogMessage::ErrnoLogMessage(const char* file, int line, + LogSeverity severity, int ctr, + void (LogMessage::*send_method)()) + : LogMessage(file, line, severity, ctr, send_method) { +} + +ErrnoLogMessage::~ErrnoLogMessage() { + // Don't access errno directly because it may have been altered + // while streaming the message. + stream() << ": " << StrError(preserved_errno()) << " [" + << preserved_errno() << "]"; +} + +void FlushLogFiles(LogSeverity min_severity) { + LogDestination::FlushLogFiles(min_severity); +} + +void FlushLogFilesUnsafe(LogSeverity min_severity) { + LogDestination::FlushLogFilesUnsafe(min_severity); +} + +void SetLogDestination(LogSeverity severity, const char* base_filename) { + LogDestination::SetLogDestination(severity, base_filename); +} + +void SetLogSymlink(LogSeverity severity, const char* symlink_basename) { + LogDestination::SetLogSymlink(severity, symlink_basename); +} + +LogSink::~LogSink() { +} + +void LogSink::WaitTillSent() { + // noop default +} + +string LogSink::ToString(LogSeverity severity, const char* file, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len, int32 usecs) { + ostringstream stream(string(message, message_len)); + stream.fill('0'); + + stream << LogSeverityNames[severity][0] + << setw(4) << 1900+tm_time->tm_year + << setw(2) << 1+tm_time->tm_mon + << setw(2) << tm_time->tm_mday + << ' ' + << setw(2) << tm_time->tm_hour << ':' + << setw(2) << tm_time->tm_min << ':' + << setw(2) << tm_time->tm_sec << '.' + << setw(6) << usecs + << ' ' + << setfill(' ') << setw(5) << GetTID() << setfill('0') + << ' ' + << file << ':' << line << "] "; + + stream << string(message, message_len); + return stream.str(); +} + +void AddLogSink(LogSink *destination) { + LogDestination::AddLogSink(destination); +} + +void RemoveLogSink(LogSink *destination) { + LogDestination::RemoveLogSink(destination); +} + +void SetLogFilenameExtension(const char* ext) { + LogDestination::SetLogFilenameExtension(ext); +} + +void SetStderrLogging(LogSeverity min_severity) { + LogDestination::SetStderrLogging(min_severity); +} + +void SetEmailLogging(LogSeverity min_severity, const char* addresses) { + LogDestination::SetEmailLogging(min_severity, addresses); +} + +void LogToStderr() { + LogDestination::LogToStderr(); +} + +namespace base { +namespace internal { + +bool GetExitOnDFatal(); +bool GetExitOnDFatal() { + MutexLock l(&log_mutex); + return exit_on_dfatal; +} + +// Determines whether we exit the program for a LOG(DFATAL) message in +// debug mode. It does this by skipping the call to Fail/FailQuietly. +// This is intended for testing only. +// +// This can have some effects on LOG(FATAL) as well. Failure messages +// are always allocated (rather than sharing a buffer), the crash +// reason is not recorded, the "gwq" status message is not updated, +// and the stack trace is not recorded. The LOG(FATAL) *will* still +// exit the program. Since this function is used only in testing, +// these differences are acceptable. +void SetExitOnDFatal(bool value); +void SetExitOnDFatal(bool value) { + MutexLock l(&log_mutex); + exit_on_dfatal = value; +} + +} // namespace internal +} // namespace base + +// Shell-escaping as we need to shell out ot /bin/mail. +static const char kDontNeedShellEscapeChars[] = + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz" + "0123456789+-_.=/:,@"; + +static string ShellEscape(const string& src) { + string result; + if (!src.empty() && // empty string needs quotes + src.find_first_not_of(kDontNeedShellEscapeChars) == string::npos) { + // only contains chars that don't need quotes; it's fine + result.assign(src); + } else if (src.find_first_of('\'') == string::npos) { + // no single quotes; just wrap it in single quotes + result.assign("'"); + result.append(src); + result.append("'"); + } else { + // needs double quote escaping + result.assign("\""); + for (size_t i = 0; i < src.size(); ++i) { + switch (src[i]) { + case '\\': + case '$': + case '"': + case '`': + result.append("\\"); + } + result.append(src, i, 1); + } + result.append("\""); + } + return result; +} + + +// use_logging controls whether the logging functions LOG/VLOG are used +// to log errors. It should be set to false when the caller holds the +// log_mutex. +static bool SendEmailInternal(const char*dest, const char *subject, + const char*body, bool use_logging) { + if (dest && *dest) { + if ( use_logging ) { + VLOG(1) << "Trying to send TITLE:" << subject + << " BODY:" << body << " to " << dest; + } else { + fprintf(stderr, "Trying to send TITLE: %s BODY: %s to %s\n", + subject, body, dest); + } + + string cmd = + FLAGS_logmailer + " -s" + + ShellEscape(subject) + " " + ShellEscape(dest); + VLOG(4) << "Mailing command: " << cmd; + + FILE* pipe = popen(cmd.c_str(), "w"); + if (pipe != NULL) { + // Add the body if we have one + if (body) + fwrite(body, sizeof(char), strlen(body), pipe); + bool ok = pclose(pipe) != -1; + if ( !ok ) { + if ( use_logging ) { + LOG(ERROR) << "Problems sending mail to " << dest << ": " + << StrError(errno); + } else { + fprintf(stderr, "Problems sending mail to %s: %s\n", + dest, StrError(errno).c_str()); + } + } + return ok; + } else { + if ( use_logging ) { + LOG(ERROR) << "Unable to send mail to " << dest; + } else { + fprintf(stderr, "Unable to send mail to %s\n", dest); + } + } + } + return false; +} + +bool SendEmail(const char*dest, const char *subject, const char*body){ + return SendEmailInternal(dest, subject, body, true); +} + +static void GetTempDirectories(vector<string>* list) { + list->clear(); +#ifdef OS_WINDOWS + // On windows we'll try to find a directory in this order: + // C:/Documents & Settings/whomever/TEMP (or whatever GetTempPath() is) + // C:/TMP/ + // C:/TEMP/ + // C:/WINDOWS/ or C:/WINNT/ + // . + char tmp[MAX_PATH]; + if (GetTempPathA(MAX_PATH, tmp)) + list->push_back(tmp); + list->push_back("C:\\tmp\\"); + list->push_back("C:\\temp\\"); +#else + // Directories, in order of preference. If we find a dir that + // exists, we stop adding other less-preferred dirs + const char * candidates[] = { + // Non-null only during unittest/regtest + getenv("TEST_TMPDIR"), + + // Explicitly-supplied temp dirs + getenv("TMPDIR"), getenv("TMP"), + + // If all else fails + "/tmp", + }; + + for (size_t i = 0; i < ARRAYSIZE(candidates); i++) { + const char *d = candidates[i]; + if (!d) continue; // Empty env var + + // Make sure we don't surprise anyone who's expecting a '/' + string dstr = d; + if (dstr[dstr.size() - 1] != '/') { + dstr += "/"; + } + list->push_back(dstr); + + struct stat statbuf; + if (!stat(d, &statbuf) && S_ISDIR(statbuf.st_mode)) { + // We found a dir that exists - we're done. + return; + } + } + +#endif +} + +static vector<string>* logging_directories_list; + +const vector<string>& GetLoggingDirectories() { + // Not strictly thread-safe but we're called early in InitGoogle(). + if (logging_directories_list == NULL) { + logging_directories_list = new vector<string>; + + if ( !FLAGS_log_dir.empty() ) { + // A dir was specified, we should use it + logging_directories_list->push_back(FLAGS_log_dir.c_str()); + } else { + GetTempDirectories(logging_directories_list); +#ifdef OS_WINDOWS + char tmp[MAX_PATH]; + if (GetWindowsDirectoryA(tmp, MAX_PATH)) + logging_directories_list->push_back(tmp); + logging_directories_list->push_back(".\\"); +#else + logging_directories_list->push_back("./"); +#endif + } + } + return *logging_directories_list; +} + +void TestOnly_ClearLoggingDirectoriesList() { + fprintf(stderr, "TestOnly_ClearLoggingDirectoriesList should only be " + "called from test code.\n"); + delete logging_directories_list; + logging_directories_list = NULL; +} + +void GetExistingTempDirectories(vector<string>* list) { + GetTempDirectories(list); + vector<string>::iterator i_dir = list->begin(); + while( i_dir != list->end() ) { + // zero arg to access means test for existence; no constant + // defined on windows + if ( access(i_dir->c_str(), 0) ) { + i_dir = list->erase(i_dir); + } else { + ++i_dir; + } + } +} + +void TruncateLogFile(const char *path, int64 limit, int64 keep) { +#ifdef HAVE_UNISTD_H + struct stat statbuf; + const int kCopyBlockSize = 8 << 10; + char copybuf[kCopyBlockSize]; + int64 read_offset, write_offset; + // Don't follow symlinks unless they're our own fd symlinks in /proc + int flags = O_RDWR; + // TODO(hamaji): Support other environments. +#ifdef OS_LINUX + const char *procfd_prefix = "/proc/self/fd/"; + if (strncmp(procfd_prefix, path, strlen(procfd_prefix))) flags |= O_NOFOLLOW; +#endif + + int fd = open(path, flags); + if (fd == -1) { + if (errno == EFBIG) { + // The log file in question has got too big for us to open. The + // real fix for this would be to compile logging.cc (or probably + // all of base/...) with -D_FILE_OFFSET_BITS=64 but that's + // rather scary. + // Instead just truncate the file to something we can manage + if (truncate(path, 0) == -1) { + PLOG(ERROR) << "Unable to truncate " << path; + } else { + LOG(ERROR) << "Truncated " << path << " due to EFBIG error"; + } + } else { + PLOG(ERROR) << "Unable to open " << path; + } + return; + } + + if (fstat(fd, &statbuf) == -1) { + PLOG(ERROR) << "Unable to fstat()"; + goto out_close_fd; + } + + // See if the path refers to a regular file bigger than the + // specified limit + if (!S_ISREG(statbuf.st_mode)) goto out_close_fd; + if (statbuf.st_size <= limit) goto out_close_fd; + if (statbuf.st_size <= keep) goto out_close_fd; + + // This log file is too large - we need to truncate it + LOG(INFO) << "Truncating " << path << " to " << keep << " bytes"; + + // Copy the last "keep" bytes of the file to the beginning of the file + read_offset = statbuf.st_size - keep; + write_offset = 0; + int bytesin, bytesout; + while ((bytesin = pread(fd, copybuf, sizeof(copybuf), read_offset)) > 0) { + bytesout = pwrite(fd, copybuf, bytesin, write_offset); + if (bytesout == -1) { + PLOG(ERROR) << "Unable to write to " << path; + break; + } else if (bytesout != bytesin) { + LOG(ERROR) << "Expected to write " << bytesin << ", wrote " << bytesout; + } + read_offset += bytesin; + write_offset += bytesout; + } + if (bytesin == -1) PLOG(ERROR) << "Unable to read from " << path; + + // Truncate the remainder of the file. If someone else writes to the + // end of the file after our last read() above, we lose their latest + // data. Too bad ... + if (ftruncate(fd, write_offset) == -1) { + PLOG(ERROR) << "Unable to truncate " << path; + } + + out_close_fd: + close(fd); +#else + LOG(ERROR) << "No log truncation support."; +#endif +} + +void TruncateStdoutStderr() { +#ifdef HAVE_UNISTD_H + int64 limit = MaxLogSize() << 20; + int64 keep = 1 << 20; + TruncateLogFile("/proc/self/fd/1", limit, keep); + TruncateLogFile("/proc/self/fd/2", limit, keep); +#else + LOG(ERROR) << "No log truncation support."; +#endif +} + + +// Helper functions for string comparisons. +#define DEFINE_CHECK_STROP_IMPL(name, func, expected) \ + string* Check##func##expected##Impl(const char* s1, const char* s2, \ + const char* names) { \ + bool equal = s1 == s2 || (s1 && s2 && !func(s1, s2)); \ + if (equal == expected) return NULL; \ + else { \ + ostringstream ss; \ + if (!s1) s1 = ""; \ + if (!s2) s2 = ""; \ + ss << #name " failed: " << names << " (" << s1 << " vs. " << s2 << ")"; \ + return new string(ss.str()); \ + } \ + } +DEFINE_CHECK_STROP_IMPL(CHECK_STREQ, strcmp, true) +DEFINE_CHECK_STROP_IMPL(CHECK_STRNE, strcmp, false) +DEFINE_CHECK_STROP_IMPL(CHECK_STRCASEEQ, strcasecmp, true) +DEFINE_CHECK_STROP_IMPL(CHECK_STRCASENE, strcasecmp, false) +#undef DEFINE_CHECK_STROP_IMPL + +int posix_strerror_r(int err, char *buf, size_t len) { + // Sanity check input parameters + if (buf == NULL || len <= 0) { + errno = EINVAL; + return -1; + } + + // Reset buf and errno, and try calling whatever version of strerror_r() + // is implemented by glibc + buf[0] = '\000'; + int old_errno = errno; + errno = 0; + char *rc = reinterpret_cast<char *>(strerror_r(err, buf, len)); + + // Both versions set errno on failure + if (errno) { + // Should already be there, but better safe than sorry + buf[0] = '\000'; + return -1; + } + errno = old_errno; + + // POSIX is vague about whether the string will be terminated, although + // is indirectly implies that typically ERANGE will be returned, instead + // of truncating the string. This is different from the GNU implementation. + // We play it safe by always terminating the string explicitly. + buf[len-1] = '\000'; + + // If the function succeeded, we can use its exit code to determine the + // semantics implemented by glibc + if (!rc) { + return 0; + } else { + // GNU semantics detected + if (rc == buf) { + return 0; + } else { + buf[0] = '\000'; +#if defined(OS_MACOSX) || defined(OS_FREEBSD) || defined(OS_OPENBSD) + if (reinterpret_cast<intptr_t>(rc) < sys_nerr) { + // This means an error on MacOSX or FreeBSD. + return -1; + } +#endif + strncat(buf, rc, len-1); + return 0; + } + } +} + +string StrError(int err) { + char buf[100]; + int rc = posix_strerror_r(err, buf, sizeof(buf)); + if ((rc < 0) || (buf[0] == '\000')) { + snprintf(buf, sizeof(buf), "Error number %d", err); + } + return buf; +} + +LogMessageFatal::LogMessageFatal(const char* file, int line) : + LogMessage(file, line, GLOG_FATAL) {} + +LogMessageFatal::LogMessageFatal(const char* file, int line, + const CheckOpString& result) : + LogMessage(file, line, result) {} + +LogMessageFatal::~LogMessageFatal() { + Flush(); + LogMessage::Fail(); +} + +namespace base { + +CheckOpMessageBuilder::CheckOpMessageBuilder(const char *exprtext) + : stream_(new ostringstream) { + *stream_ << exprtext << " ("; +} + +CheckOpMessageBuilder::~CheckOpMessageBuilder() { + delete stream_; +} + +ostream* CheckOpMessageBuilder::ForVar2() { + *stream_ << " vs. "; + return stream_; +} + +string* CheckOpMessageBuilder::NewString() { + *stream_ << ")"; + return new string(stream_->str()); +} + +} // namespace base + +template <> +void MakeCheckOpValueString(std::ostream* os, const char& v) { + if (v >= 32 && v <= 126) { + (*os) << "'" << v << "'"; + } else { + (*os) << "char value " << (short)v; + } +} + +template <> +void MakeCheckOpValueString(std::ostream* os, const signed char& v) { + if (v >= 32 && v <= 126) { + (*os) << "'" << v << "'"; + } else { + (*os) << "signed char value " << (short)v; + } +} + +template <> +void MakeCheckOpValueString(std::ostream* os, const unsigned char& v) { + if (v >= 32 && v <= 126) { + (*os) << "'" << v << "'"; + } else { + (*os) << "unsigned char value " << (unsigned short)v; + } +} + +void InitGoogleLogging(const char* argv0) { + glog_internal_namespace_::InitGoogleLoggingUtilities(argv0); +} + +void ShutdownGoogleLogging() { + glog_internal_namespace_::ShutdownGoogleLoggingUtilities(); + LogDestination::DeleteLogDestinations(); + delete logging_directories_list; + logging_directories_list = NULL; +} + +void EnableLogCleaner(int overdue_days) { + log_cleaner_enabled_ = true; + + // Setting overdue_days to 0 day should not be allowed! + // Since all logs will be deleted immediately, which will cause troubles. + if (overdue_days > 0) { + log_cleaner_overdue_days_ = overdue_days; + } +} + +void DisableLogCleaner() { + log_cleaner_enabled_ = false; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/logging_striplog_test.sh b/third_party/glog/src/logging_striplog_test.sh new file mode 100755 index 000000000000..73492bda7a77 --- /dev/null +++ b/third_party/glog/src/logging_striplog_test.sh @@ -0,0 +1,79 @@ +#! /bin/sh +# +# Copyright (c) 2007, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Author: Sergey Ioffe + +get_strings () { + if test -e ".libs/$1"; then + binary=".libs/$1" + elif test -e "$1.exe"; then + binary="$1.exe" + else + echo "We coundn't find $1 binary." + exit 1 + fi + + strings -n 10 $binary | sort | awk '/TESTMESSAGE/ {printf "%s ", $2}' +} + +# Die if "$1" != "$2", print $3 as death reason +check_eq () { + if [ "$1" != "$2" ]; then + echo "Check failed: '$1' == '$2' ${3:+ ($3)}" + exit 1 + fi +} + +die () { + echo $1 + exit 1 +} + +# Check that the string literals are appropriately stripped. This will +# not be the case in debug mode. + +mode=`GLOG_check_mode=1 ./logging_striptest0 2> /dev/null` +if [ "$mode" = "opt" ]; +then + echo "In OPT mode" + check_eq "`get_strings logging_striptest0`" "COND ERROR FATAL INFO USAGE WARNING " + check_eq "`get_strings logging_striptest2`" "COND ERROR FATAL USAGE " + check_eq "`get_strings logging_striptest10`" "" +else + echo "In DBG mode; not checking strings" +fi + +# Check that LOG(FATAL) aborts even for large STRIP_LOG + +./logging_striptest2 2>/dev/null && die "Did not abort for STRIP_LOG=2" +./logging_striptest10 2>/dev/null && die "Did not abort for STRIP_LOG=10" + +echo "PASS" diff --git a/third_party/glog/src/logging_striptest10.cc b/third_party/glog/src/logging_striptest10.cc new file mode 100644 index 000000000000..f6e1078f39a2 --- /dev/null +++ b/third_party/glog/src/logging_striptest10.cc @@ -0,0 +1,35 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sergey Ioffe + +#define GOOGLE_STRIP_LOG 10 + +// Include the actual test. +#include "logging_striptest_main.cc" diff --git a/third_party/glog/src/logging_striptest2.cc b/third_party/glog/src/logging_striptest2.cc new file mode 100644 index 000000000000..a64685c9e5cd --- /dev/null +++ b/third_party/glog/src/logging_striptest2.cc @@ -0,0 +1,35 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sergey Ioffe + +#define GOOGLE_STRIP_LOG 2 + +// Include the actual test. +#include "logging_striptest_main.cc" diff --git a/third_party/glog/src/logging_striptest_main.cc b/third_party/glog/src/logging_striptest_main.cc new file mode 100644 index 000000000000..2fb91272a7b1 --- /dev/null +++ b/third_party/glog/src/logging_striptest_main.cc @@ -0,0 +1,73 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Sergey Ioffe + +// The common part of the striplog tests. + +#include <stdio.h> +#include <string> +#include <iosfwd> +#include "glog/logging.h" +#include "base/commandlineflags.h" +#include "config.h" + +DECLARE_bool(logtostderr); +GLOG_DEFINE_bool(check_mode, false, "Prints 'opt' or 'dbg'"); + +using std::string; +using namespace GOOGLE_NAMESPACE; + +int CheckNoReturn(bool b) { + string s; + if (b) { + LOG(FATAL) << "Fatal"; + } else { + return 0; + } +} + +struct A { }; +std::ostream &operator<<(std::ostream &str, const A&) {return str;} + +int main(int, char* argv[]) { + FLAGS_logtostderr = true; + InitGoogleLogging(argv[0]); + if (FLAGS_check_mode) { + printf("%s\n", DEBUG_MODE ? "dbg" : "opt"); + return 0; + } + LOG(INFO) << "TESTMESSAGE INFO"; + LOG(WARNING) << 2 << "something" << "TESTMESSAGE WARNING" + << 1 << 'c' << A() << std::endl; + LOG(ERROR) << "TESTMESSAGE ERROR"; + bool flag = true; + (flag ? LOG(INFO) : LOG(ERROR)) << "TESTMESSAGE COND"; + LOG(FATAL) << "TESTMESSAGE FATAL"; +} diff --git a/third_party/glog/src/logging_unittest.cc b/third_party/glog/src/logging_unittest.cc new file mode 100644 index 000000000000..73d426b878d7 --- /dev/null +++ b/third_party/glog/src/logging_unittest.cc @@ -0,0 +1,1348 @@ +// Copyright (c) 2002, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Ray Sidney + +#include "config_for_unittests.h" +#include "utilities.h" + +#include <fcntl.h> +#ifdef HAVE_GLOB_H +# include <glob.h> +#endif +#include <sys/stat.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif +#ifdef HAVE_SYS_WAIT_H +# include <sys/wait.h> +#endif + +#include <iomanip> +#include <iostream> +#include <fstream> +#include <memory> +#include <queue> +#include <sstream> +#include <string> +#include <vector> + +#include <stdio.h> +#include <stdlib.h> + +#include "base/commandlineflags.h" +#include "glog/logging.h" +#include "glog/raw_logging.h" +#include "googletest.h" + +DECLARE_string(log_backtrace_at); // logging.cc + +#ifdef HAVE_LIB_GFLAGS +#include <gflags/gflags.h> +using namespace GFLAGS_NAMESPACE; +#endif + +#ifdef HAVE_LIB_GMOCK +#include <gmock/gmock.h> +#include "mock-log.h" +// Introduce several symbols from gmock. +using testing::_; +using testing::AnyNumber; +using testing::HasSubstr; +using testing::AllOf; +using testing::StrNe; +using testing::StrictMock; +using testing::InitGoogleMock; +using GOOGLE_NAMESPACE::glog_testing::ScopedMockLog; +#endif + +using namespace std; +using namespace GOOGLE_NAMESPACE; + +// Some non-advertised functions that we want to test or use. +_START_GOOGLE_NAMESPACE_ +namespace base { +namespace internal { +bool GetExitOnDFatal(); +void SetExitOnDFatal(bool value); +} // namespace internal +} // namespace base +_END_GOOGLE_NAMESPACE_ + +static void TestLogging(bool check_counts); +static void TestRawLogging(); +static void LogWithLevels(int v, int severity, bool err, bool alsoerr); +static void TestLoggingLevels(); +static void TestLogString(); +static void TestLogSink(); +static void TestLogToString(); +static void TestLogSinkWaitTillSent(); +static void TestCHECK(); +static void TestDCHECK(); +static void TestSTREQ(); +static void TestBasename(); +static void TestBasenameAppendWhenNoTimestamp(); +static void TestTwoProcessesWrite(); +static void TestSymlink(); +static void TestExtension(); +static void TestWrapper(); +static void TestErrno(); +static void TestTruncate(); +static void TestCustomLoggerDeletionOnShutdown(); + +static int x = -1; +static void BM_Check1(int n) { + while (n-- > 0) { + CHECK_GE(n, x); + CHECK_GE(n, x); + CHECK_GE(n, x); + CHECK_GE(n, x); + CHECK_GE(n, x); + CHECK_GE(n, x); + CHECK_GE(n, x); + CHECK_GE(n, x); + } +} +BENCHMARK(BM_Check1); + +static void CheckFailure(int a, int b, const char* file, int line, const char* msg); +static void BM_Check3(int n) { + while (n-- > 0) { + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + if (n < x) CheckFailure(n, x, __FILE__, __LINE__, "n < x"); + } +} +BENCHMARK(BM_Check3); + +static void BM_Check2(int n) { + if (n == 17) { + x = 5; + } + while (n-- > 0) { + CHECK(n >= x); + CHECK(n >= x); + CHECK(n >= x); + CHECK(n >= x); + CHECK(n >= x); + CHECK(n >= x); + CHECK(n >= x); + CHECK(n >= x); + } +} +BENCHMARK(BM_Check2); + +static void CheckFailure(int, int, const char* /* file */, int /* line */, + const char* /* msg */) { +} + +static void BM_logspeed(int n) { + while (n-- > 0) { + LOG(INFO) << "test message"; + } +} +BENCHMARK(BM_logspeed); + +static void BM_vlog(int n) { + while (n-- > 0) { + VLOG(1) << "test message"; + } +} +BENCHMARK(BM_vlog); + +int main(int argc, char **argv) { + FLAGS_colorlogtostderr = false; + FLAGS_timestamp_in_logfile_name = true; +#ifdef HAVE_LIB_GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif + // Make sure stderr is not buffered as stderr seems to be buffered + // on recent windows. + setbuf(stderr, NULL); + + // Test some basics before InitGoogleLogging: + CaptureTestStderr(); + LogWithLevels(FLAGS_v, FLAGS_stderrthreshold, + FLAGS_logtostderr, FLAGS_alsologtostderr); + LogWithLevels(0, 0, 0, 0); // simulate "before global c-tors" + const string early_stderr = GetCapturedTestStderr(); + + InitGoogleLogging(argv[0]); + + RunSpecifiedBenchmarks(); + + FLAGS_logtostderr = true; + + InitGoogleTest(&argc, argv); +#ifdef HAVE_LIB_GMOCK + InitGoogleMock(&argc, argv); +#endif + + // so that death tests run before we use threads + CHECK_EQ(RUN_ALL_TESTS(), 0); + + CaptureTestStderr(); + + // re-emit early_stderr + LogMessage("dummy", LogMessage::kNoLogPrefix, GLOG_INFO).stream() << early_stderr; + + TestLogging(true); + TestRawLogging(); + TestLoggingLevels(); + TestLogString(); + TestLogSink(); + TestLogToString(); + TestLogSinkWaitTillSent(); + TestCHECK(); + TestDCHECK(); + TestSTREQ(); + + // TODO: The golden test portion of this test is very flakey. + EXPECT_TRUE( + MungeAndDiffTestStderr(FLAGS_test_srcdir + "/src/logging_unittest.err")); + + FLAGS_logtostderr = false; + + TestBasename(); + TestBasenameAppendWhenNoTimestamp(); + TestTwoProcessesWrite(); + TestSymlink(); + TestExtension(); + TestWrapper(); + TestErrno(); + TestTruncate(); + TestCustomLoggerDeletionOnShutdown(); + + fprintf(stdout, "PASS\n"); + return 0; +} + +void TestLogging(bool check_counts) { + int64 base_num_infos = LogMessage::num_messages(GLOG_INFO); + int64 base_num_warning = LogMessage::num_messages(GLOG_WARNING); + int64 base_num_errors = LogMessage::num_messages(GLOG_ERROR); + + LOG(INFO) << string("foo ") << "bar " << 10 << ' ' << 3.4; + for ( int i = 0; i < 10; ++i ) { + int old_errno = errno; + errno = i; + PLOG_EVERY_N(ERROR, 2) << "Plog every 2, iteration " << COUNTER; + errno = old_errno; + + LOG_EVERY_N(ERROR, 3) << "Log every 3, iteration " << COUNTER << endl; + LOG_EVERY_N(ERROR, 4) << "Log every 4, iteration " << COUNTER << endl; + + LOG_IF_EVERY_N(WARNING, true, 5) << "Log if every 5, iteration " << COUNTER; + LOG_IF_EVERY_N(WARNING, false, 3) + << "Log if every 3, iteration " << COUNTER; + LOG_IF_EVERY_N(INFO, true, 1) << "Log if every 1, iteration " << COUNTER; + LOG_IF_EVERY_N(ERROR, (i < 3), 2) + << "Log if less than 3 every 2, iteration " << COUNTER; + } + LOG_IF(WARNING, true) << "log_if this"; + LOG_IF(WARNING, false) << "don't log_if this"; + + char s[] = "array"; + LOG(INFO) << s; + const char const_s[] = "const array"; + LOG(INFO) << const_s; + int j = 1000; + LOG(ERROR) << string("foo") << ' '<< j << ' ' << setw(10) << j << " " + << setw(1) << hex << j; + + { + google::LogMessage outer(__FILE__, __LINE__, GLOG_ERROR); + outer.stream() << "outer"; + + LOG(ERROR) << "inner"; + } + + LogMessage("foo", LogMessage::kNoLogPrefix, GLOG_INFO).stream() << "no prefix"; + + if (check_counts) { + CHECK_EQ(base_num_infos + 14, LogMessage::num_messages(GLOG_INFO)); + CHECK_EQ(base_num_warning + 3, LogMessage::num_messages(GLOG_WARNING)); + CHECK_EQ(base_num_errors + 17, LogMessage::num_messages(GLOG_ERROR)); + } +} + +static void NoAllocNewHook() { + LOG(FATAL) << "unexpected new"; +} + +struct NewHook { + NewHook() { + g_new_hook = &NoAllocNewHook; + } + ~NewHook() { + g_new_hook = NULL; + } +}; + +TEST(DeathNoAllocNewHook, logging) { + // tests that NewHook used below works + NewHook new_hook; + ASSERT_DEATH({ + new int; + }, "unexpected new"); +} + +void TestRawLogging() { + string* foo = new string("foo "); + string huge_str(50000, 'a'); + + FlagSaver saver; + + // Check that RAW loggging does not use mallocs. + NewHook new_hook; + + RAW_LOG(INFO, "%s%s%d%c%f", foo->c_str(), "bar ", 10, ' ', 3.4); + char s[] = "array"; + RAW_LOG(WARNING, "%s", s); + const char const_s[] = "const array"; + RAW_LOG(INFO, "%s", const_s); + void* p = reinterpret_cast<void*>(PTR_TEST_VALUE); + RAW_LOG(INFO, "ptr %p", p); + p = NULL; + RAW_LOG(INFO, "ptr %p", p); + int j = 1000; + RAW_LOG(ERROR, "%s%d%c%010d%s%1x", foo->c_str(), j, ' ', j, " ", j); + RAW_VLOG(0, "foo %d", j); + +#ifdef NDEBUG + RAW_LOG(INFO, "foo %d", j); // so that have same stderr to compare +#else + RAW_DLOG(INFO, "foo %d", j); // test RAW_DLOG in debug mode +#endif + + // test how long messages are chopped: + RAW_LOG(WARNING, "Huge string: %s", huge_str.c_str()); + RAW_VLOG(0, "Huge string: %s", huge_str.c_str()); + + FLAGS_v = 0; + RAW_LOG(INFO, "log"); + RAW_VLOG(0, "vlog 0 on"); + RAW_VLOG(1, "vlog 1 off"); + RAW_VLOG(2, "vlog 2 off"); + RAW_VLOG(3, "vlog 3 off"); + FLAGS_v = 2; + RAW_LOG(INFO, "log"); + RAW_VLOG(1, "vlog 1 on"); + RAW_VLOG(2, "vlog 2 on"); + RAW_VLOG(3, "vlog 3 off"); + +#ifdef NDEBUG + RAW_DCHECK(1 == 2, " RAW_DCHECK's shouldn't be compiled in normal mode"); +#endif + + RAW_CHECK(1 == 1, "should be ok"); + RAW_DCHECK(true, "should be ok"); + + delete foo; +} + +void LogWithLevels(int v, int severity, bool err, bool alsoerr) { + RAW_LOG(INFO, + "Test: v=%d stderrthreshold=%d logtostderr=%d alsologtostderr=%d", + v, severity, err, alsoerr); + + FlagSaver saver; + + FLAGS_v = v; + FLAGS_stderrthreshold = severity; + FLAGS_logtostderr = err; + FLAGS_alsologtostderr = alsoerr; + + RAW_VLOG(-1, "vlog -1"); + RAW_VLOG(0, "vlog 0"); + RAW_VLOG(1, "vlog 1"); + RAW_LOG(INFO, "log info"); + RAW_LOG(WARNING, "log warning"); + RAW_LOG(ERROR, "log error"); + + VLOG(-1) << "vlog -1"; + VLOG(0) << "vlog 0"; + VLOG(1) << "vlog 1"; + LOG(INFO) << "log info"; + LOG(WARNING) << "log warning"; + LOG(ERROR) << "log error"; + + VLOG_IF(-1, true) << "vlog_if -1"; + VLOG_IF(-1, false) << "don't vlog_if -1"; + VLOG_IF(0, true) << "vlog_if 0"; + VLOG_IF(0, false) << "don't vlog_if 0"; + VLOG_IF(1, true) << "vlog_if 1"; + VLOG_IF(1, false) << "don't vlog_if 1"; + LOG_IF(INFO, true) << "log_if info"; + LOG_IF(INFO, false) << "don't log_if info"; + LOG_IF(WARNING, true) << "log_if warning"; + LOG_IF(WARNING, false) << "don't log_if warning"; + LOG_IF(ERROR, true) << "log_if error"; + LOG_IF(ERROR, false) << "don't log_if error"; + + int c; + c = 1; VLOG_IF(100, c -= 2) << "vlog_if 100 expr"; EXPECT_EQ(c, -1); + c = 1; VLOG_IF(0, c -= 2) << "vlog_if 0 expr"; EXPECT_EQ(c, -1); + c = 1; LOG_IF(INFO, c -= 2) << "log_if info expr"; EXPECT_EQ(c, -1); + c = 1; LOG_IF(ERROR, c -= 2) << "log_if error expr"; EXPECT_EQ(c, -1); + c = 2; VLOG_IF(0, c -= 2) << "don't vlog_if 0 expr"; EXPECT_EQ(c, 0); + c = 2; LOG_IF(ERROR, c -= 2) << "don't log_if error expr"; EXPECT_EQ(c, 0); + + c = 3; LOG_IF_EVERY_N(INFO, c -= 4, 1) << "log_if info every 1 expr"; + EXPECT_EQ(c, -1); + c = 3; LOG_IF_EVERY_N(ERROR, c -= 4, 1) << "log_if error every 1 expr"; + EXPECT_EQ(c, -1); + c = 4; LOG_IF_EVERY_N(ERROR, c -= 4, 3) << "don't log_if info every 3 expr"; + EXPECT_EQ(c, 0); + c = 4; LOG_IF_EVERY_N(ERROR, c -= 4, 3) << "don't log_if error every 3 expr"; + EXPECT_EQ(c, 0); + c = 5; VLOG_IF_EVERY_N(0, c -= 4, 1) << "vlog_if 0 every 1 expr"; + EXPECT_EQ(c, 1); + c = 5; VLOG_IF_EVERY_N(100, c -= 4, 3) << "vlog_if 100 every 3 expr"; + EXPECT_EQ(c, 1); + c = 6; VLOG_IF_EVERY_N(0, c -= 6, 1) << "don't vlog_if 0 every 1 expr"; + EXPECT_EQ(c, 0); + c = 6; VLOG_IF_EVERY_N(100, c -= 6, 3) << "don't vlog_if 100 every 1 expr"; + EXPECT_EQ(c, 0); +} + +void TestLoggingLevels() { + LogWithLevels(0, GLOG_INFO, false, false); + LogWithLevels(1, GLOG_INFO, false, false); + LogWithLevels(-1, GLOG_INFO, false, false); + LogWithLevels(0, GLOG_WARNING, false, false); + LogWithLevels(0, GLOG_ERROR, false, false); + LogWithLevels(0, GLOG_FATAL, false, false); + LogWithLevels(0, GLOG_FATAL, true, false); + LogWithLevels(0, GLOG_FATAL, false, true); + LogWithLevels(1, GLOG_WARNING, false, false); + LogWithLevels(1, GLOG_FATAL, false, true); +} + +TEST(DeathRawCHECK, logging) { + ASSERT_DEATH(RAW_CHECK(false, "failure 1"), + "RAW: Check false failed: failure 1"); + ASSERT_DEBUG_DEATH(RAW_DCHECK(1 == 2, "failure 2"), + "RAW: Check 1 == 2 failed: failure 2"); +} + +void TestLogString() { + vector<string> errors; + vector<string> *no_errors = NULL; + + LOG_STRING(INFO, &errors) << "LOG_STRING: " << "collected info"; + LOG_STRING(WARNING, &errors) << "LOG_STRING: " << "collected warning"; + LOG_STRING(ERROR, &errors) << "LOG_STRING: " << "collected error"; + + LOG_STRING(INFO, no_errors) << "LOG_STRING: " << "reported info"; + LOG_STRING(WARNING, no_errors) << "LOG_STRING: " << "reported warning"; + LOG_STRING(ERROR, NULL) << "LOG_STRING: " << "reported error"; + + for (size_t i = 0; i < errors.size(); ++i) { + LOG(INFO) << "Captured by LOG_STRING: " << errors[i]; + } +} + +void TestLogToString() { + string error; + string* no_error = NULL; + + LOG_TO_STRING(INFO, &error) << "LOG_TO_STRING: " << "collected info"; + LOG(INFO) << "Captured by LOG_TO_STRING: " << error; + LOG_TO_STRING(WARNING, &error) << "LOG_TO_STRING: " << "collected warning"; + LOG(INFO) << "Captured by LOG_TO_STRING: " << error; + LOG_TO_STRING(ERROR, &error) << "LOG_TO_STRING: " << "collected error"; + LOG(INFO) << "Captured by LOG_TO_STRING: " << error; + + LOG_TO_STRING(INFO, no_error) << "LOG_TO_STRING: " << "reported info"; + LOG_TO_STRING(WARNING, no_error) << "LOG_TO_STRING: " << "reported warning"; + LOG_TO_STRING(ERROR, NULL) << "LOG_TO_STRING: " << "reported error"; +} + +class TestLogSinkImpl : public LogSink { + public: + vector<string> errors; + virtual void send(LogSeverity severity, const char* /* full_filename */, + const char* base_filename, int line, + const struct tm* tm_time, + const char* message, size_t message_len, int usecs) { + errors.push_back( + ToString(severity, base_filename, line, tm_time, message, message_len, usecs)); + } + virtual void send(LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct tm* tm_time, + const char* message, size_t message_len) { + send(severity, full_filename, base_filename, line, + tm_time, message, message_len, 0); + } +}; + +void TestLogSink() { + TestLogSinkImpl sink; + LogSink *no_sink = NULL; + + LOG_TO_SINK(&sink, INFO) << "LOG_TO_SINK: " << "collected info"; + LOG_TO_SINK(&sink, WARNING) << "LOG_TO_SINK: " << "collected warning"; + LOG_TO_SINK(&sink, ERROR) << "LOG_TO_SINK: " << "collected error"; + + LOG_TO_SINK(no_sink, INFO) << "LOG_TO_SINK: " << "reported info"; + LOG_TO_SINK(no_sink, WARNING) << "LOG_TO_SINK: " << "reported warning"; + LOG_TO_SINK(NULL, ERROR) << "LOG_TO_SINK: " << "reported error"; + + LOG_TO_SINK_BUT_NOT_TO_LOGFILE(&sink, INFO) + << "LOG_TO_SINK_BUT_NOT_TO_LOGFILE: " << "collected info"; + LOG_TO_SINK_BUT_NOT_TO_LOGFILE(&sink, WARNING) + << "LOG_TO_SINK_BUT_NOT_TO_LOGFILE: " << "collected warning"; + LOG_TO_SINK_BUT_NOT_TO_LOGFILE(&sink, ERROR) + << "LOG_TO_SINK_BUT_NOT_TO_LOGFILE: " << "collected error"; + + LOG_TO_SINK_BUT_NOT_TO_LOGFILE(no_sink, INFO) + << "LOG_TO_SINK_BUT_NOT_TO_LOGFILE: " << "thrashed info"; + LOG_TO_SINK_BUT_NOT_TO_LOGFILE(no_sink, WARNING) + << "LOG_TO_SINK_BUT_NOT_TO_LOGFILE: " << "thrashed warning"; + LOG_TO_SINK_BUT_NOT_TO_LOGFILE(NULL, ERROR) + << "LOG_TO_SINK_BUT_NOT_TO_LOGFILE: " << "thrashed error"; + + LOG(INFO) << "Captured by LOG_TO_SINK:"; + for (size_t i = 0; i < sink.errors.size(); ++i) { + LogMessage("foo", LogMessage::kNoLogPrefix, GLOG_INFO).stream() + << sink.errors[i]; + } +} + +// For testing using CHECK*() on anonymous enums. +enum { + CASE_A, + CASE_B +}; + +void TestCHECK() { + // Tests using CHECK*() on int values. + CHECK(1 == 1); + CHECK_EQ(1, 1); + CHECK_NE(1, 2); + CHECK_GE(1, 1); + CHECK_GE(2, 1); + CHECK_LE(1, 1); + CHECK_LE(1, 2); + CHECK_GT(2, 1); + CHECK_LT(1, 2); + + // Tests using CHECK*() on anonymous enums. + // Apple's GCC doesn't like this. +#if !defined(OS_MACOSX) + CHECK_EQ(CASE_A, CASE_A); + CHECK_NE(CASE_A, CASE_B); + CHECK_GE(CASE_A, CASE_A); + CHECK_GE(CASE_B, CASE_A); + CHECK_LE(CASE_A, CASE_A); + CHECK_LE(CASE_A, CASE_B); + CHECK_GT(CASE_B, CASE_A); + CHECK_LT(CASE_A, CASE_B); +#endif +} + +void TestDCHECK() { +#ifdef NDEBUG + DCHECK( 1 == 2 ) << " DCHECK's shouldn't be compiled in normal mode"; +#endif + DCHECK( 1 == 1 ); + DCHECK_EQ(1, 1); + DCHECK_NE(1, 2); + DCHECK_GE(1, 1); + DCHECK_GE(2, 1); + DCHECK_LE(1, 1); + DCHECK_LE(1, 2); + DCHECK_GT(2, 1); + DCHECK_LT(1, 2); + + int64* orig_ptr = new int64; + int64* ptr = DCHECK_NOTNULL(orig_ptr); + CHECK_EQ(ptr, orig_ptr); + delete orig_ptr; +} + +void TestSTREQ() { + CHECK_STREQ("this", "this"); + CHECK_STREQ(NULL, NULL); + CHECK_STRCASEEQ("this", "tHiS"); + CHECK_STRCASEEQ(NULL, NULL); + CHECK_STRNE("this", "tHiS"); + CHECK_STRNE("this", NULL); + CHECK_STRCASENE("this", "that"); + CHECK_STRCASENE(NULL, "that"); + CHECK_STREQ((string("a")+"b").c_str(), "ab"); + CHECK_STREQ(string("test").c_str(), + (string("te") + string("st")).c_str()); +} + +TEST(DeathSTREQ, logging) { + ASSERT_DEATH(CHECK_STREQ(NULL, "this"), ""); + ASSERT_DEATH(CHECK_STREQ("this", "siht"), ""); + ASSERT_DEATH(CHECK_STRCASEEQ(NULL, "siht"), ""); + ASSERT_DEATH(CHECK_STRCASEEQ("this", "siht"), ""); + ASSERT_DEATH(CHECK_STRNE(NULL, NULL), ""); + ASSERT_DEATH(CHECK_STRNE("this", "this"), ""); + ASSERT_DEATH(CHECK_STREQ((string("a")+"b").c_str(), "abc"), ""); +} + +TEST(CheckNOTNULL, Simple) { + int64 t; + void *ptr = static_cast<void *>(&t); + void *ref = CHECK_NOTNULL(ptr); + EXPECT_EQ(ptr, ref); + CHECK_NOTNULL(reinterpret_cast<char *>(ptr)); + CHECK_NOTNULL(reinterpret_cast<unsigned char *>(ptr)); + CHECK_NOTNULL(reinterpret_cast<int *>(ptr)); + CHECK_NOTNULL(reinterpret_cast<int64 *>(ptr)); +} + +TEST(DeathCheckNN, Simple) { + ASSERT_DEATH(CHECK_NOTNULL(static_cast<void *>(NULL)), ""); +} + +// Get list of file names that match pattern +static void GetFiles(const string& pattern, vector<string>* files) { + files->clear(); +#if defined(HAVE_GLOB_H) + glob_t g; + const int r = glob(pattern.c_str(), 0, NULL, &g); + CHECK((r == 0) || (r == GLOB_NOMATCH)) << ": error matching " << pattern; + for (size_t i = 0; i < g.gl_pathc; i++) { + files->push_back(string(g.gl_pathv[i])); + } + globfree(&g); +#elif defined(OS_WINDOWS) + WIN32_FIND_DATAA data; + HANDLE handle = FindFirstFileA(pattern.c_str(), &data); + size_t index = pattern.rfind('\\'); + if (index == string::npos) { + LOG(FATAL) << "No directory separator."; + } + const string dirname = pattern.substr(0, index + 1); + if (handle == INVALID_HANDLE_VALUE) { + // Finding no files is OK. + return; + } + do { + files->push_back(dirname + data.cFileName); + } while (FindNextFileA(handle, &data)); + BOOL result = FindClose(handle); + LOG_SYSRESULT(result); +#else +# error There is no way to do glob. +#endif +} + +// Delete files patching pattern +static void DeleteFiles(const string& pattern) { + vector<string> files; + GetFiles(pattern, &files); + for (size_t i = 0; i < files.size(); i++) { + CHECK(unlink(files[i].c_str()) == 0) << ": " << strerror(errno); + } +} + +//check string is in file (or is *NOT*, depending on optional checkInFileOrNot) +static void CheckFile(const string& name, const string& expected_string, const bool checkInFileOrNot = true) { + vector<string> files; + GetFiles(name + "*", &files); + CHECK_EQ(files.size(), 1UL); + + FILE* file = fopen(files[0].c_str(), "r"); + CHECK(file != NULL) << ": could not open " << files[0]; + char buf[1000]; + while (fgets(buf, sizeof(buf), file) != NULL) { + char* first = strstr(buf, expected_string.c_str()); + //if first == NULL, not found. + //Terser than if (checkInFileOrNot && first != NULL || !check... + if (checkInFileOrNot != (first == NULL)) { + fclose(file); + return; + } + } + fclose(file); + LOG(FATAL) << "Did " << (checkInFileOrNot? "not " : "") << "find " << expected_string << " in " << files[0]; +} + +static void TestBasename() { + fprintf(stderr, "==== Test setting log file basename\n"); + const string dest = FLAGS_test_tmpdir + "/logging_test_basename"; + DeleteFiles(dest + "*"); + + SetLogDestination(GLOG_INFO, dest.c_str()); + LOG(INFO) << "message to new base"; + FlushLogFiles(GLOG_INFO); + + CheckFile(dest, "message to new base"); + + // Release file handle for the destination file to unlock the file in Windows. + LogToStderr(); + DeleteFiles(dest + "*"); +} + +static void TestBasenameAppendWhenNoTimestamp() { + fprintf(stderr, "==== Test setting log file basename without timestamp and appending properly\n"); + const string dest = FLAGS_test_tmpdir + "/logging_test_basename_append_when_no_timestamp"; + DeleteFiles(dest + "*"); + + ofstream out(dest.c_str()); + out << "test preexisting content" << endl; + out.close(); + + CheckFile(dest, "test preexisting content"); + + FLAGS_timestamp_in_logfile_name=false; + SetLogDestination(GLOG_INFO, dest.c_str()); + LOG(INFO) << "message to new base, appending to preexisting file"; + FlushLogFiles(GLOG_INFO); + FLAGS_timestamp_in_logfile_name=true; + + //if the logging overwrites the file instead of appending it will fail. + CheckFile(dest, "test preexisting content"); + CheckFile(dest, "message to new base, appending to preexisting file"); + + // Release file handle for the destination file to unlock the file in Windows. + LogToStderr(); + DeleteFiles(dest + "*"); +} + +static void TestTwoProcessesWrite() { +// test only implemented for platforms with fork & wait; the actual implementation relies on flock +#if defined(HAVE_SYS_WAIT_H) && defined(HAVE_UNISTD_H) && defined(HAVE_FCNTL) + fprintf(stderr, "==== Test setting log file basename and two processes writing - second should fail\n"); + const string dest = FLAGS_test_tmpdir + "/logging_test_basename_two_processes_writing"; + DeleteFiles(dest + "*"); + + //make both processes write into the same file (easier test) + FLAGS_timestamp_in_logfile_name=false; + SetLogDestination(GLOG_INFO, dest.c_str()); + LOG(INFO) << "message to new base, parent"; + FlushLogFiles(GLOG_INFO); + + pid_t pid = fork(); + CHECK_ERR(pid); + if (pid == 0) { + LOG(INFO) << "message to new base, child - should only appear on STDERR not on the file"; + ShutdownGoogleLogging(); //for children proc + exit(0); + } else if (pid > 0) { + wait(NULL); + } + FLAGS_timestamp_in_logfile_name=true; + + CheckFile(dest, "message to new base, parent"); + CheckFile(dest, "message to new base, child - should only appear on STDERR not on the file", false); + + // Release + LogToStderr(); + DeleteFiles(dest + "*"); +#endif +} + +static void TestSymlink() { +#ifndef OS_WINDOWS + fprintf(stderr, "==== Test setting log file symlink\n"); + string dest = FLAGS_test_tmpdir + "/logging_test_symlink"; + string sym = FLAGS_test_tmpdir + "/symlinkbase"; + DeleteFiles(dest + "*"); + DeleteFiles(sym + "*"); + + SetLogSymlink(GLOG_INFO, "symlinkbase"); + SetLogDestination(GLOG_INFO, dest.c_str()); + LOG(INFO) << "message to new symlink"; + FlushLogFiles(GLOG_INFO); + CheckFile(sym, "message to new symlink"); + + DeleteFiles(dest + "*"); + DeleteFiles(sym + "*"); +#endif +} + +static void TestExtension() { + fprintf(stderr, "==== Test setting log file extension\n"); + string dest = FLAGS_test_tmpdir + "/logging_test_extension"; + DeleteFiles(dest + "*"); + + SetLogDestination(GLOG_INFO, dest.c_str()); + SetLogFilenameExtension("specialextension"); + LOG(INFO) << "message to new extension"; + FlushLogFiles(GLOG_INFO); + CheckFile(dest, "message to new extension"); + + // Check that file name ends with extension + vector<string> filenames; + GetFiles(dest + "*", &filenames); + CHECK_EQ(filenames.size(), 1UL); + CHECK(strstr(filenames[0].c_str(), "specialextension") != NULL); + + // Release file handle for the destination file to unlock the file in Windows. + LogToStderr(); + DeleteFiles(dest + "*"); +} + +struct MyLogger : public base::Logger { + string data; + + virtual void Write(bool /* should_flush */, + time_t /* timestamp */, + const char* message, + int length) { + data.append(message, length); + } + + virtual void Flush() { } + + virtual uint32 LogSize() { return data.length(); } +}; + +static void TestWrapper() { + fprintf(stderr, "==== Test log wrapper\n"); + + MyLogger my_logger; + base::Logger* old_logger = base::GetLogger(GLOG_INFO); + base::SetLogger(GLOG_INFO, &my_logger); + LOG(INFO) << "Send to wrapped logger"; + FlushLogFiles(GLOG_INFO); + base::SetLogger(GLOG_INFO, old_logger); + + CHECK(strstr(my_logger.data.c_str(), "Send to wrapped logger") != NULL); +} + +static void TestErrno() { + fprintf(stderr, "==== Test errno preservation\n"); + + errno = ENOENT; + TestLogging(false); + CHECK_EQ(errno, ENOENT); +} + +static void TestOneTruncate(const char *path, int64 limit, int64 keep, + int64 dsize, int64 ksize, int64 expect) { + int fd; + CHECK_ERR(fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0600)); + + const char *discardstr = "DISCARDME!", *keepstr = "KEEPME!"; + const size_t discard_size = strlen(discardstr), keep_size = strlen(keepstr); + + // Fill the file with the requested data; first discard data, then kept data + int64 written = 0; + while (written < dsize) { + int bytes = min<int64>(dsize - written, discard_size); + CHECK_ERR(write(fd, discardstr, bytes)); + written += bytes; + } + written = 0; + while (written < ksize) { + int bytes = min<int64>(ksize - written, keep_size); + CHECK_ERR(write(fd, keepstr, bytes)); + written += bytes; + } + + TruncateLogFile(path, limit, keep); + + // File should now be shorter + struct stat statbuf; + CHECK_ERR(fstat(fd, &statbuf)); + CHECK_EQ(statbuf.st_size, expect); + CHECK_ERR(lseek(fd, 0, SEEK_SET)); + + // File should contain the suffix of the original file + const size_t buf_size = statbuf.st_size + 1; + char* buf = new char[buf_size]; + memset(buf, 0, buf_size); + CHECK_ERR(read(fd, buf, buf_size)); + + const char *p = buf; + int64 checked = 0; + while (checked < expect) { + int bytes = min<int64>(expect - checked, keep_size); + CHECK(!memcmp(p, keepstr, bytes)); + checked += bytes; + } + close(fd); + delete[] buf; +} + +static void TestTruncate() { +#ifdef HAVE_UNISTD_H + fprintf(stderr, "==== Test log truncation\n"); + string path = FLAGS_test_tmpdir + "/truncatefile"; + + // Test on a small file + TestOneTruncate(path.c_str(), 10, 10, 10, 10, 10); + + // And a big file (multiple blocks to copy) + TestOneTruncate(path.c_str(), 2<<20, 4<<10, 3<<20, 4<<10, 4<<10); + + // Check edge-case limits + TestOneTruncate(path.c_str(), 10, 20, 0, 20, 20); + TestOneTruncate(path.c_str(), 10, 0, 0, 0, 0); + TestOneTruncate(path.c_str(), 10, 50, 0, 10, 10); + TestOneTruncate(path.c_str(), 50, 100, 0, 30, 30); + + // MacOSX 10.4 doesn't fail in this case. + // Windows doesn't have symlink. + // Let's just ignore this test for these cases. +#if !defined(OS_MACOSX) && !defined(OS_WINDOWS) + // Through a symlink should fail to truncate + string linkname = path + ".link"; + unlink(linkname.c_str()); + CHECK_ERR(symlink(path.c_str(), linkname.c_str())); + TestOneTruncate(linkname.c_str(), 10, 10, 0, 30, 30); +#endif + + // The /proc/self path makes sense only for linux. +#if defined(OS_LINUX) + // Through an open fd symlink should work + int fd; + CHECK_ERR(fd = open(path.c_str(), O_APPEND | O_WRONLY)); + char fdpath[64]; + snprintf(fdpath, sizeof(fdpath), "/proc/self/fd/%d", fd); + TestOneTruncate(fdpath, 10, 10, 10, 10, 10); +#endif + +#endif +} + +struct RecordDeletionLogger : public base::Logger { + RecordDeletionLogger(bool* set_on_destruction, + base::Logger* wrapped_logger) : + set_on_destruction_(set_on_destruction), + wrapped_logger_(wrapped_logger) + { + *set_on_destruction_ = false; + } + virtual ~RecordDeletionLogger() { + *set_on_destruction_ = true; + } + virtual void Write(bool force_flush, + time_t timestamp, + const char* message, + int length) { + wrapped_logger_->Write(force_flush, timestamp, message, length); + } + virtual void Flush() { wrapped_logger_->Flush(); } + virtual uint32 LogSize() { return wrapped_logger_->LogSize(); } + private: + bool* set_on_destruction_; + base::Logger* wrapped_logger_; +}; + +static void TestCustomLoggerDeletionOnShutdown() { + bool custom_logger_deleted = false; + base::SetLogger(GLOG_INFO, + new RecordDeletionLogger(&custom_logger_deleted, + base::GetLogger(GLOG_INFO))); + ShutdownGoogleLogging(); + EXPECT_TRUE(custom_logger_deleted); +} + +_START_GOOGLE_NAMESPACE_ +namespace glog_internal_namespace_ { +extern // in logging.cc +bool SafeFNMatch_(const char* pattern, size_t patt_len, + const char* str, size_t str_len); +} // namespace glog_internal_namespace_ +using glog_internal_namespace_::SafeFNMatch_; +_END_GOOGLE_NAMESPACE_ + +static bool WrapSafeFNMatch(string pattern, string str) { + pattern += "abc"; + str += "defgh"; + return SafeFNMatch_(pattern.data(), pattern.size() - 3, + str.data(), str.size() - 5); +} + +TEST(SafeFNMatch, logging) { + CHECK(WrapSafeFNMatch("foo", "foo")); + CHECK(!WrapSafeFNMatch("foo", "bar")); + CHECK(!WrapSafeFNMatch("foo", "fo")); + CHECK(!WrapSafeFNMatch("foo", "foo2")); + CHECK(WrapSafeFNMatch("bar/foo.ext", "bar/foo.ext")); + CHECK(WrapSafeFNMatch("*ba*r/fo*o.ext*", "bar/foo.ext")); + CHECK(!WrapSafeFNMatch("bar/foo.ext", "bar/baz.ext")); + CHECK(!WrapSafeFNMatch("bar/foo.ext", "bar/foo")); + CHECK(!WrapSafeFNMatch("bar/foo.ext", "bar/foo.ext.zip")); + CHECK(WrapSafeFNMatch("ba?/*.ext", "bar/foo.ext")); + CHECK(WrapSafeFNMatch("ba?/*.ext", "baZ/FOO.ext")); + CHECK(!WrapSafeFNMatch("ba?/*.ext", "barr/foo.ext")); + CHECK(!WrapSafeFNMatch("ba?/*.ext", "bar/foo.ext2")); + CHECK(WrapSafeFNMatch("ba?/*", "bar/foo.ext2")); + CHECK(WrapSafeFNMatch("ba?/*", "bar/")); + CHECK(!WrapSafeFNMatch("ba?/?", "bar/")); + CHECK(!WrapSafeFNMatch("ba?/*", "bar")); +} + +// TestWaitingLogSink will save messages here +// No lock: Accessed only by TestLogSinkWriter thread +// and after its demise by its creator. +static vector<string> global_messages; + +// helper for TestWaitingLogSink below. +// Thread that does the logic of TestWaitingLogSink +// It's free to use LOG() itself. +class TestLogSinkWriter : public Thread { + public: + + TestLogSinkWriter() : should_exit_(false) { + SetJoinable(true); + Start(); + } + + // Just buffer it (can't use LOG() here). + void Buffer(const string& message) { + mutex_.Lock(); + RAW_LOG(INFO, "Buffering"); + messages_.push(message); + mutex_.Unlock(); + RAW_LOG(INFO, "Buffered"); + } + + // Wait for the buffer to clear (can't use LOG() here). + void Wait() { + RAW_LOG(INFO, "Waiting"); + mutex_.Lock(); + while (!NoWork()) { + mutex_.Unlock(); + SleepForMilliseconds(1); + mutex_.Lock(); + } + RAW_LOG(INFO, "Waited"); + mutex_.Unlock(); + } + + // Trigger thread exit. + void Stop() { + MutexLock l(&mutex_); + should_exit_ = true; + } + + private: + + // helpers --------------- + + // For creating a "Condition". + bool NoWork() { return messages_.empty(); } + bool HaveWork() { return !messages_.empty() || should_exit_; } + + // Thread body; CAN use LOG() here! + virtual void Run() { + while (1) { + mutex_.Lock(); + while (!HaveWork()) { + mutex_.Unlock(); + SleepForMilliseconds(1); + mutex_.Lock(); + } + if (should_exit_ && messages_.empty()) { + mutex_.Unlock(); + break; + } + // Give the main thread time to log its message, + // so that we get a reliable log capture to compare to golden file. + // Same for the other sleep below. + SleepForMilliseconds(20); + RAW_LOG(INFO, "Sink got a messages"); // only RAW_LOG under mutex_ here + string message = messages_.front(); + messages_.pop(); + // Normally this would be some more real/involved logging logic + // where LOG() usage can't be eliminated, + // e.g. pushing the message over with an RPC: + int messages_left = messages_.size(); + mutex_.Unlock(); + SleepForMilliseconds(20); + // May not use LOG while holding mutex_, because Buffer() + // acquires mutex_, and Buffer is called from LOG(), + // which has its own internal mutex: + // LOG()->LogToSinks()->TestWaitingLogSink::send()->Buffer() + LOG(INFO) << "Sink is sending out a message: " << message; + LOG(INFO) << "Have " << messages_left << " left"; + global_messages.push_back(message); + } + } + + // data --------------- + + Mutex mutex_; + bool should_exit_; + queue<string> messages_; // messages to be logged +}; + +// A log sink that exercises WaitTillSent: +// it pushes data to a buffer and wakes up another thread to do the logging +// (that other thread can than use LOG() itself), +class TestWaitingLogSink : public LogSink { + public: + + TestWaitingLogSink() { + tid_ = pthread_self(); // for thread-specific behavior + AddLogSink(this); + } + ~TestWaitingLogSink() { + RemoveLogSink(this); + writer_.Stop(); + writer_.Join(); + } + + // (re)define LogSink interface + + virtual void send(LogSeverity severity, const char* /* full_filename */, + const char* base_filename, int line, + const struct tm* tm_time, + const char* message, size_t message_len, int usecs) { + // Push it to Writer thread if we are the original logging thread. + // Note: Something like ThreadLocalLogSink is a better choice + // to do thread-specific LogSink logic for real. + if (pthread_equal(tid_, pthread_self())) { + writer_.Buffer(ToString(severity, base_filename, line, + tm_time, message, message_len, usecs)); + } + } + + virtual void send(LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct tm* tm_time, + const char* message, size_t message_len) { + send(severity, full_filename, base_filename, line, tm_time, message, message_len); + } + + virtual void WaitTillSent() { + // Wait for Writer thread if we are the original logging thread. + if (pthread_equal(tid_, pthread_self())) writer_.Wait(); + } + + private: + + pthread_t tid_; + TestLogSinkWriter writer_; +}; + +// Check that LogSink::WaitTillSent can be used in the advertised way. +// We also do golden-stderr comparison. +static void TestLogSinkWaitTillSent() { + { TestWaitingLogSink sink; + // Sleeps give the sink threads time to do all their work, + // so that we get a reliable log capture to compare to the golden file. + LOG(INFO) << "Message 1"; + SleepForMilliseconds(60); + LOG(ERROR) << "Message 2"; + SleepForMilliseconds(60); + LOG(WARNING) << "Message 3"; + SleepForMilliseconds(60); + } + for (size_t i = 0; i < global_messages.size(); ++i) { + LOG(INFO) << "Sink capture: " << global_messages[i]; + } + CHECK_EQ(global_messages.size(), 3UL); +} + +TEST(Strerror, logging) { + int errcode = EINTR; + char *msg = strdup(strerror(errcode)); + const size_t buf_size = strlen(msg) + 1; + char *buf = new char[buf_size]; + CHECK_EQ(posix_strerror_r(errcode, NULL, 0), -1); + buf[0] = 'A'; + CHECK_EQ(posix_strerror_r(errcode, buf, 0), -1); + CHECK_EQ(buf[0], 'A'); + CHECK_EQ(posix_strerror_r(errcode, NULL, buf_size), -1); +#if defined(OS_MACOSX) || defined(OS_FREEBSD) || defined(OS_OPENBSD) + // MacOSX or FreeBSD considers this case is an error since there is + // no enough space. + CHECK_EQ(posix_strerror_r(errcode, buf, 1), -1); +#else + CHECK_EQ(posix_strerror_r(errcode, buf, 1), 0); +#endif + CHECK_STREQ(buf, ""); + CHECK_EQ(posix_strerror_r(errcode, buf, buf_size), 0); + CHECK_STREQ(buf, msg); + delete[] buf; + CHECK_EQ(msg, StrError(errcode)); + free(msg); +} + +// Simple routines to look at the sizes of generated code for LOG(FATAL) and +// CHECK(..) via objdump +/* +static void MyFatal() { + LOG(FATAL) << "Failed"; +} +static void MyCheck(bool a, bool b) { + CHECK_EQ(a, b); +} +*/ +#ifdef HAVE_LIB_GMOCK + +TEST(DVLog, Basic) { + ScopedMockLog log; + +#if NDEBUG + // We are expecting that nothing is logged. + EXPECT_CALL(log, Log(_, _, _)).Times(0); +#else + EXPECT_CALL(log, Log(INFO, __FILE__, "debug log")); +#endif + + FLAGS_v = 1; + DVLOG(1) << "debug log"; +} + +TEST(DVLog, V0) { + ScopedMockLog log; + + // We are expecting that nothing is logged. + EXPECT_CALL(log, Log(_, _, _)).Times(0); + + FLAGS_v = 0; + DVLOG(1) << "debug log"; +} + +TEST(LogAtLevel, Basic) { + ScopedMockLog log; + + // The function version outputs "logging.h" as a file name. + EXPECT_CALL(log, Log(WARNING, StrNe(__FILE__), "function version")); + EXPECT_CALL(log, Log(INFO, __FILE__, "macro version")); + + int severity = WARNING; + LogAtLevel(severity, "function version"); + + severity = INFO; + // We can use the macro version as a C++ stream. + LOG_AT_LEVEL(severity) << "macro" << ' ' << "version"; +} + +TEST(TestExitOnDFatal, ToBeOrNotToBe) { + // Check the default setting... + EXPECT_TRUE(base::internal::GetExitOnDFatal()); + + // Turn off... + base::internal::SetExitOnDFatal(false); + EXPECT_FALSE(base::internal::GetExitOnDFatal()); + + // We don't die. + { + ScopedMockLog log; + //EXPECT_CALL(log, Log(_, _, _)).Times(AnyNumber()); + // LOG(DFATAL) has severity FATAL if debugging, but is + // downgraded to ERROR if not debugging. + const LogSeverity severity = +#ifdef NDEBUG + ERROR; +#else + FATAL; +#endif + EXPECT_CALL(log, Log(severity, __FILE__, "This should not be fatal")); + LOG(DFATAL) << "This should not be fatal"; + } + + // Turn back on... + base::internal::SetExitOnDFatal(true); + EXPECT_TRUE(base::internal::GetExitOnDFatal()); + +#ifdef GTEST_HAS_DEATH_TEST + // Death comes on little cats' feet. + EXPECT_DEBUG_DEATH({ + LOG(DFATAL) << "This should be fatal in debug mode"; + }, "This should be fatal in debug mode"); +#endif +} + +#ifdef HAVE_STACKTRACE + +static void BacktraceAtHelper() { + LOG(INFO) << "Not me"; + +// The vertical spacing of the next 3 lines is significant. + LOG(INFO) << "Backtrace me"; +} +static int kBacktraceAtLine = __LINE__ - 2; // The line of the LOG(INFO) above + +TEST(LogBacktraceAt, DoesNotBacktraceWhenDisabled) { + StrictMock<ScopedMockLog> log; + + FLAGS_log_backtrace_at = ""; + + EXPECT_CALL(log, Log(_, _, "Backtrace me")); + EXPECT_CALL(log, Log(_, _, "Not me")); + + BacktraceAtHelper(); +} + +TEST(LogBacktraceAt, DoesBacktraceAtRightLineWhenEnabled) { + StrictMock<ScopedMockLog> log; + + char where[100]; + snprintf(where, 100, "%s:%d", const_basename(__FILE__), kBacktraceAtLine); + FLAGS_log_backtrace_at = where; + + // The LOG at the specified line should include a stacktrace which includes + // the name of the containing function, followed by the log message. + // We use HasSubstr()s instead of ContainsRegex() for environments + // which don't have regexp. + EXPECT_CALL(log, Log(_, _, AllOf(HasSubstr("stacktrace:"), + HasSubstr("BacktraceAtHelper"), + HasSubstr("main"), + HasSubstr("Backtrace me")))); + // Other LOGs should not include a backtrace. + EXPECT_CALL(log, Log(_, _, "Not me")); + + BacktraceAtHelper(); +} + +#endif // HAVE_STACKTRACE + +#endif // HAVE_LIB_GMOCK + +struct UserDefinedClass { + bool operator==(const UserDefinedClass&) const { return true; } +}; + +inline ostream& operator<<(ostream& out, const UserDefinedClass&) { + out << "OK"; + return out; +} + +TEST(UserDefinedClass, logging) { + UserDefinedClass u; + vector<string> buf; + LOG_STRING(INFO, &buf) << u; + CHECK_EQ(1UL, buf.size()); + CHECK(buf[0].find("OK") != string::npos); + + // We must be able to compile this. + CHECK_EQ(u, u); +} diff --git a/third_party/glog/src/logging_unittest.err b/third_party/glog/src/logging_unittest.err new file mode 100644 index 000000000000..def0e93f1e80 --- /dev/null +++ b/third_party/glog/src/logging_unittest.err @@ -0,0 +1,307 @@ +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=2 logtostderr=0 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +WARNING: Logging before InitGoogleLogging() is written to STDERR +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=0 logtostderr=0 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] foo bar 10 3.4 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Plog every 2, iteration 1: __SUCCESS__ [0] +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 3, iteration 1 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 4, iteration 1 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 5, iteration 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 1 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if less than 3 every 2, iteration 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 2 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Plog every 2, iteration 3: __ENOENT__ [2] +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 3 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if less than 3 every 2, iteration 3 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 3, iteration 4 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 4 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Plog every 2, iteration 5: __EINTR__ [4] +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 4, iteration 5 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 5 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 5, iteration 6 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 6 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Plog every 2, iteration 7: __ENXIO__ [6] +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 3, iteration 7 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 7 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 8 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Plog every 2, iteration 9: __ENOEXEC__ [8] +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 4, iteration 9 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 9 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log every 3, iteration 10 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Log if every 1, iteration 10 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if this +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] array +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] const array +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] foo 1000 0000001000 3e8 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] inner +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] outer +no prefix +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: foo bar 10 3.400000 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: array +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: const array +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: ptr __PTRTEST__ +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: ptr __NULLP__ +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: foo 1000 0000001000 3e8 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: foo 1000 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: foo 1000 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: RAW_LOG ERROR: The Message was too long! +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: RAW_LOG ERROR: The Message was too long! +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 on +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 1 on +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 2 on +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=0 logtostderr=0 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=1 stderrthreshold=0 logtostderr=0 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=-1 stderrthreshold=0 logtostderr=0 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=1 logtostderr=0 alsologtostderr=0 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=2 logtostderr=0 alsologtostderr=0 +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=3 logtostderr=0 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=3 logtostderr=1 alsologtostderr=0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=0 stderrthreshold=3 logtostderr=0 alsologtostderr=1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=1 stderrthreshold=1 logtostderr=0 alsologtostderr=0 +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Test: v=1 stderrthreshold=3 logtostderr=0 alsologtostderr=1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: vlog 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if -1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if info every 1 expr +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] log_if error every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] vlog_if 0 every 1 expr +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_STRING: reported info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_STRING: reported warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_STRING: reported error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_STRING: LOG_STRING: collected info +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_STRING: LOG_STRING: collected warning +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_STRING: LOG_STRING: collected error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: collected info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: collected warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: collected error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: reported info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: reported warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: reported error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_TO_SINK: +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: collected info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: collected warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK: collected error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK_BUT_NOT_TO_LOGFILE: collected info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK_BUT_NOT_TO_LOGFILE: collected warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_SINK_BUT_NOT_TO_LOGFILE: collected error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_STRING: collected info +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_TO_STRING: LOG_TO_STRING: collected info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_STRING: collected warning +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_TO_STRING: LOG_TO_STRING: collected warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_STRING: collected error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Captured by LOG_TO_STRING: LOG_TO_STRING: collected error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_STRING: reported info +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_STRING: reported warning +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] LOG_TO_STRING: reported error +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Buffering +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Buffered +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Waiting +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Sink got a messages +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Waited +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Sink is sending out a message: IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Have 0 left +EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 2 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Buffering +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Buffered +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Waiting +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Sink got a messages +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Waited +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Sink is sending out a message: EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 2 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Have 0 left +WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 3 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Buffering +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Buffered +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Waiting +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Sink got a messages +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] RAW: Waited +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Sink is sending out a message: WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 3 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Have 0 left +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Sink capture: IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 1 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Sink capture: EYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 2 +IYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Sink capture: WYEARDATE TIME__ THREADID logging_unittest.cc:LINE] Message 3 diff --git a/third_party/glog/src/mock-log.h b/third_party/glog/src/mock-log.h new file mode 100644 index 000000000000..30a0f74efeab --- /dev/null +++ b/third_party/glog/src/mock-log.h @@ -0,0 +1,156 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Zhanyong Wan +// +// Defines the ScopedMockLog class (using Google C++ Mocking +// Framework), which is convenient for testing code that uses LOG(). + +#ifndef GLOG_SRC_MOCK_LOG_H_ +#define GLOG_SRC_MOCK_LOG_H_ + +// For GOOGLE_NAMESPACE. This must go first so we get _XOPEN_SOURCE. +#include "utilities.h" + +#include <string> + +#include <gmock/gmock.h> + +#include "glog/logging.h" + +_START_GOOGLE_NAMESPACE_ +namespace glog_testing { + +// A ScopedMockLog object intercepts LOG() messages issued during its +// lifespan. Using this together with Google C++ Mocking Framework, +// it's very easy to test how a piece of code calls LOG(). The +// typical usage: +// +// TEST(FooTest, LogsCorrectly) { +// ScopedMockLog log; +// +// // We expect the WARNING "Something bad!" exactly twice. +// EXPECT_CALL(log, Log(WARNING, _, "Something bad!")) +// .Times(2); +// +// // We allow foo.cc to call LOG(INFO) any number of times. +// EXPECT_CALL(log, Log(INFO, HasSubstr("/foo.cc"), _)) +// .Times(AnyNumber()); +// +// Foo(); // Exercises the code under test. +// } +class ScopedMockLog : public GOOGLE_NAMESPACE::LogSink { + public: + // When a ScopedMockLog object is constructed, it starts to + // intercept logs. + ScopedMockLog() { AddLogSink(this); } + + // When the object is destructed, it stops intercepting logs. + virtual ~ScopedMockLog() { RemoveLogSink(this); } + + // Implements the mock method: + // + // void Log(LogSeverity severity, const string& file_path, + // const string& message); + // + // The second argument to Send() is the full path of the source file + // in which the LOG() was issued. + // + // Note, that in a multi-threaded environment, all LOG() messages from a + // single thread will be handled in sequence, but that cannot be guaranteed + // for messages from different threads. In fact, if the same or multiple + // expectations are matched on two threads concurrently, their actions will + // be executed concurrently as well and may interleave. + MOCK_METHOD3(Log, void(GOOGLE_NAMESPACE::LogSeverity severity, + const std::string& file_path, + const std::string& message)); + + private: + // Implements the send() virtual function in class LogSink. + // Whenever a LOG() statement is executed, this function will be + // invoked with information presented in the LOG(). + // + // The method argument list is long and carries much information a + // test usually doesn't care about, so we trim the list before + // forwarding the call to Log(), which is much easier to use in + // tests. + // + // We still cannot call Log() directly, as it may invoke other LOG() + // messages, either due to Invoke, or due to an error logged in + // Google C++ Mocking Framework code, which would trigger a deadlock + // since a lock is held during send(). + // + // Hence, we save the message for WaitTillSent() which will be called after + // the lock on send() is released, and we'll call Log() inside + // WaitTillSent(). Since while a single send() call may be running at a + // time, multiple WaitTillSent() calls (along with the one send() call) may + // be running simultaneously, we ensure thread-safety of the exchange between + // send() and WaitTillSent(), and that for each message, LOG(), send(), + // WaitTillSent() and Log() are executed in the same thread. + virtual void send(GOOGLE_NAMESPACE::LogSeverity severity, + const char* full_filename, + const char* /*base_filename*/, int /*line*/, + const tm* /*tm_time*/, + const char* message, size_t message_len) { + // We are only interested in the log severity, full file name, and + // log message. + message_info_.severity = severity; + message_info_.file_path = full_filename; + message_info_.message = std::string(message, message_len); + } + + // Implements the WaitTillSent() virtual function in class LogSink. + // It will be executed after send() and after the global logging lock is + // released, so calls within it (or rather within the Log() method called + // within) may also issue LOG() statements. + // + // LOG(), send(), WaitTillSent() and Log() will occur in the same thread for + // a given log message. + virtual void WaitTillSent() { + // First, and very importantly, we save a copy of the message being + // processed before calling Log(), since Log() may indirectly call send() + // and WaitTillSent() in the same thread again. + MessageInfo message_info = message_info_; + Log(message_info.severity, message_info.file_path, message_info.message); + } + + // All relevant information about a logged message that needs to be passed + // from send() to WaitTillSent(). + struct MessageInfo { + GOOGLE_NAMESPACE::LogSeverity severity; + std::string file_path; + std::string message; + }; + MessageInfo message_info_; +}; + +} // namespace glog_testing +_END_GOOGLE_NAMESPACE_ + +#endif // GLOG_SRC_MOCK_LOG_H_ diff --git a/third_party/glog/src/mock-log_test.cc b/third_party/glog/src/mock-log_test.cc new file mode 100644 index 000000000000..7d58a307c222 --- /dev/null +++ b/third_party/glog/src/mock-log_test.cc @@ -0,0 +1,106 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Zhanyong Wan + +// Tests the ScopedMockLog class. + +#include "mock-log.h" + +#include <string> + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +namespace { + +using GOOGLE_NAMESPACE::INFO; +using GOOGLE_NAMESPACE::WARNING; +using GOOGLE_NAMESPACE::ERROR; +using GOOGLE_NAMESPACE::glog_testing::ScopedMockLog; +using std::string; +using testing::_; +using testing::HasSubstr; +using testing::InSequence; +using testing::InvokeWithoutArgs; + +// Tests that ScopedMockLog intercepts LOG()s when it's alive. +TEST(ScopedMockLogTest, InterceptsLog) { + ScopedMockLog log; + + InSequence s; + EXPECT_CALL(log, Log(WARNING, HasSubstr("/mock-log_test.cc"), "Fishy.")); + EXPECT_CALL(log, Log(INFO, _, "Working...")) + .Times(2); + EXPECT_CALL(log, Log(ERROR, _, "Bad!!")); + + LOG(WARNING) << "Fishy."; + LOG(INFO) << "Working..."; + LOG(INFO) << "Working..."; + LOG(ERROR) << "Bad!!"; +} + +void LogBranch() { + LOG(INFO) << "Logging a branch..."; +} + +void LogTree() { + LOG(INFO) << "Logging the whole tree..."; +} + +void LogForest() { + LOG(INFO) << "Logging the entire forest."; + LOG(INFO) << "Logging the entire forest.."; + LOG(INFO) << "Logging the entire forest..."; +} + +// The purpose of the following test is to verify that intercepting logging +// continues to work properly if a LOG statement is executed within the scope +// of a mocked call. +TEST(ScopedMockLogTest, LogDuringIntercept) { + ScopedMockLog log; + InSequence s; + EXPECT_CALL(log, Log(INFO, __FILE__, "Logging a branch...")) + .WillOnce(InvokeWithoutArgs(LogTree)); + EXPECT_CALL(log, Log(INFO, __FILE__, "Logging the whole tree...")) + .WillOnce(InvokeWithoutArgs(LogForest)); + EXPECT_CALL(log, Log(INFO, __FILE__, "Logging the entire forest.")); + EXPECT_CALL(log, Log(INFO, __FILE__, "Logging the entire forest..")); + EXPECT_CALL(log, Log(INFO, __FILE__, "Logging the entire forest...")); + LogBranch(); +} + +} // namespace + +int main(int argc, char **argv) { + GOOGLE_NAMESPACE::InitGoogleLogging(argv[0]); + testing::InitGoogleMock(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/third_party/glog/src/package_config_unittest/working_config/CMakeLists.txt b/third_party/glog/src/package_config_unittest/working_config/CMakeLists.txt new file mode 100644 index 000000000000..d701aa7b1b7a --- /dev/null +++ b/third_party/glog/src/package_config_unittest/working_config/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required (VERSION 3.1) +project (glog_package_config) + +find_package (glog REQUIRED NO_MODULE) + +add_executable (glog_package_config glog_package_config.cc) + +target_link_libraries (glog_package_config PRIVATE glog::glog) diff --git a/third_party/glog/src/package_config_unittest/working_config/glog_package_config.cc b/third_party/glog/src/package_config_unittest/working_config/glog_package_config.cc new file mode 100644 index 000000000000..b7b5cf6c6650 --- /dev/null +++ b/third_party/glog/src/package_config_unittest/working_config/glog_package_config.cc @@ -0,0 +1,6 @@ +#include <glog/logging.h> + +int main(int /*argc*/, char** argv) +{ + google::InitGoogleLogging(argv[0]); +} diff --git a/third_party/glog/src/raw_logging.cc b/third_party/glog/src/raw_logging.cc new file mode 100644 index 000000000000..ba65961e9053 --- /dev/null +++ b/third_party/glog/src/raw_logging.cc @@ -0,0 +1,158 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Maxim Lifantsev +// +// logging_unittest.cc covers the functionality herein + +#include "utilities.h" + +#include <stdarg.h> +#include <stdio.h> +#include <errno.h> +#ifdef HAVE_UNISTD_H +# include <unistd.h> // for close() and write() +#endif +#include <fcntl.h> // for open() +#include <time.h> +#include "config.h" +#include "glog/logging.h" // To pick up flag settings etc. +#include "glog/raw_logging.h" +#include "base/commandlineflags.h" + +#ifdef HAVE_STACKTRACE +# include "stacktrace.h" +#endif + +#if defined(HAVE_SYSCALL_H) +#include <syscall.h> // for syscall() +#elif defined(HAVE_SYS_SYSCALL_H) +#include <sys/syscall.h> // for syscall() +#endif +#ifdef HAVE_UNISTD_H +# include <unistd.h> +#endif + +#if defined(HAVE_SYSCALL_H) || defined(HAVE_SYS_SYSCALL_H) +# define safe_write(fd, s, len) syscall(SYS_write, fd, s, len) +#else + // Not so safe, but what can you do? +# define safe_write(fd, s, len) write(fd, s, len) +#endif + +_START_GOOGLE_NAMESPACE_ + +// CAVEAT: vsnprintf called from *DoRawLog below has some (exotic) code paths +// that invoke malloc() and getenv() that might acquire some locks. +// If this becomes a problem we should reimplement a subset of vsnprintf +// that does not need locks and malloc. + +// Helper for RawLog__ below. +// *DoRawLog writes to *buf of *size and move them past the written portion. +// It returns true iff there was no overflow or error. +static bool DoRawLog(char** buf, int* size, const char* format, ...) { + va_list ap; + va_start(ap, format); + int n = vsnprintf(*buf, *size, format, ap); + va_end(ap); + if (n < 0 || n > *size) return false; + *size -= n; + *buf += n; + return true; +} + +// Helper for RawLog__ below. +inline static bool VADoRawLog(char** buf, int* size, + const char* format, va_list ap) { + int n = vsnprintf(*buf, *size, format, ap); + if (n < 0 || n > *size) return false; + *size -= n; + *buf += n; + return true; +} + +static const int kLogBufSize = 3000; +static bool crashed = false; +static CrashReason crash_reason; +static char crash_buf[kLogBufSize + 1] = { 0 }; // Will end in '\0' + +void RawLog__(LogSeverity severity, const char* file, int line, + const char* format, ...) { + if (!(FLAGS_logtostderr || severity >= FLAGS_stderrthreshold || + FLAGS_alsologtostderr || !IsGoogleLoggingInitialized())) { + return; // this stderr log message is suppressed + } + // can't call localtime_r here: it can allocate + char buffer[kLogBufSize]; + char* buf = buffer; + int size = sizeof(buffer); + + // NOTE: this format should match the specification in base/logging.h + DoRawLog(&buf, &size, "%c00000000 00:00:00.000000 %5u %s:%d] RAW: ", + LogSeverityNames[severity][0], + static_cast<unsigned int>(GetTID()), + const_basename(const_cast<char *>(file)), line); + + // Record the position and size of the buffer after the prefix + const char* msg_start = buf; + const int msg_size = size; + + va_list ap; + va_start(ap, format); + bool no_chop = VADoRawLog(&buf, &size, format, ap); + va_end(ap); + if (no_chop) { + DoRawLog(&buf, &size, "\n"); + } else { + DoRawLog(&buf, &size, "RAW_LOG ERROR: The Message was too long!\n"); + } + // We make a raw syscall to write directly to the stderr file descriptor, + // avoiding FILE buffering (to avoid invoking malloc()), and bypassing + // libc (to side-step any libc interception). + // We write just once to avoid races with other invocations of RawLog__. + safe_write(STDERR_FILENO, buffer, strlen(buffer)); + if (severity == GLOG_FATAL) { + if (!sync_val_compare_and_swap(&crashed, false, true)) { + crash_reason.filename = file; + crash_reason.line_number = line; + memcpy(crash_buf, msg_start, msg_size); // Don't include prefix + crash_reason.message = crash_buf; +#ifdef HAVE_STACKTRACE + crash_reason.depth = + GetStackTrace(crash_reason.stack, ARRAYSIZE(crash_reason.stack), 1); +#else + crash_reason.depth = 0; +#endif + SetCrashReason(&crash_reason); + } + LogMessage::Fail(); // abort() + } +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/signalhandler.cc b/third_party/glog/src/signalhandler.cc new file mode 100644 index 000000000000..955471899afe --- /dev/null +++ b/third_party/glog/src/signalhandler.cc @@ -0,0 +1,403 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// Implementation of InstallFailureSignalHandler(). + +#include "utilities.h" +#include "stacktrace.h" +#include "symbolize.h" +#include "glog/logging.h" + +#include <signal.h> +#include <time.h> +#ifdef HAVE_UCONTEXT_H +# include <ucontext.h> +#endif +#ifdef HAVE_SYS_UCONTEXT_H +# include <sys/ucontext.h> +#endif +#include <algorithm> + +_START_GOOGLE_NAMESPACE_ + +namespace { + +// We'll install the failure signal handler for these signals. We could +// use strsignal() to get signal names, but we don't use it to avoid +// introducing yet another #ifdef complication. +// +// The list should be synced with the comment in signalhandler.h. +const struct { + int number; + const char *name; +} kFailureSignals[] = { + { SIGSEGV, "SIGSEGV" }, + { SIGILL, "SIGILL" }, + { SIGFPE, "SIGFPE" }, + { SIGABRT, "SIGABRT" }, +#if !defined(OS_WINDOWS) + { SIGBUS, "SIGBUS" }, +#endif + { SIGTERM, "SIGTERM" }, +}; + +static bool kFailureSignalHandlerInstalled = false; + +// Returns the program counter from signal context, NULL if unknown. +void* GetPC(void* ucontext_in_void) { +#if (defined(HAVE_UCONTEXT_H) || defined(HAVE_SYS_UCONTEXT_H)) && defined(PC_FROM_UCONTEXT) + if (ucontext_in_void != NULL) { + ucontext_t *context = reinterpret_cast<ucontext_t *>(ucontext_in_void); + return (void*)context->PC_FROM_UCONTEXT; + } +#endif + return NULL; +} + +// The class is used for formatting error messages. We don't use printf() +// as it's not async signal safe. +class MinimalFormatter { + public: + MinimalFormatter(char *buffer, int size) + : buffer_(buffer), + cursor_(buffer), + end_(buffer + size) { + } + + // Returns the number of bytes written in the buffer. + int num_bytes_written() const { return (int) (cursor_ - buffer_); } + + // Appends string from "str" and updates the internal cursor. + void AppendString(const char* str) { + int i = 0; + while (str[i] != '\0' && cursor_ + i < end_) { + cursor_[i] = str[i]; + ++i; + } + cursor_ += i; + } + + // Formats "number" in "radix" and updates the internal cursor. + // Lowercase letters are used for 'a' - 'z'. + void AppendUint64(uint64 number, int radix) { + int i = 0; + while (cursor_ + i < end_) { + const int tmp = number % radix; + number /= radix; + cursor_[i] = (tmp < 10 ? '0' + tmp : 'a' + tmp - 10); + ++i; + if (number == 0) { + break; + } + } + // Reverse the bytes written. + std::reverse(cursor_, cursor_ + i); + cursor_ += i; + } + + // Formats "number" as hexadecimal number, and updates the internal + // cursor. Padding will be added in front if needed. + void AppendHexWithPadding(uint64 number, int width) { + char* start = cursor_; + AppendString("0x"); + AppendUint64(number, 16); + // Move to right and add padding in front if needed. + if (cursor_ < start + width) { + const int64 delta = start + width - cursor_; + std::copy(start, cursor_, start + delta); + std::fill(start, start + delta, ' '); + cursor_ = start + width; + } + } + + private: + char *buffer_; + char *cursor_; + const char * const end_; +}; + +// Writes the given data with the size to the standard error. +void WriteToStderr(const char* data, int size) { + if (write(STDERR_FILENO, data, size) < 0) { + // Ignore errors. + } +} + +// The writer function can be changed by InstallFailureWriter(). +void (*g_failure_writer)(const char* data, int size) = WriteToStderr; + +// Dumps time information. We don't dump human-readable time information +// as localtime() is not guaranteed to be async signal safe. +void DumpTimeInfo() { + time_t time_in_sec = time(NULL); + char buf[256]; // Big enough for time info. + MinimalFormatter formatter(buf, sizeof(buf)); + formatter.AppendString("*** Aborted at "); + formatter.AppendUint64(time_in_sec, 10); + formatter.AppendString(" (unix time)"); + formatter.AppendString(" try \"date -d @"); + formatter.AppendUint64(time_in_sec, 10); + formatter.AppendString("\" if you are using GNU date ***\n"); + g_failure_writer(buf, formatter.num_bytes_written()); +} + +// TODO(hamaji): Use signal instead of sigaction? +#ifdef HAVE_SIGACTION + +// Dumps information about the signal to STDERR. +void DumpSignalInfo(int signal_number, siginfo_t *siginfo) { + // Get the signal name. + const char* signal_name = NULL; + for (size_t i = 0; i < ARRAYSIZE(kFailureSignals); ++i) { + if (signal_number == kFailureSignals[i].number) { + signal_name = kFailureSignals[i].name; + } + } + + char buf[256]; // Big enough for signal info. + MinimalFormatter formatter(buf, sizeof(buf)); + + formatter.AppendString("*** "); + if (signal_name) { + formatter.AppendString(signal_name); + } else { + // Use the signal number if the name is unknown. The signal name + // should be known, but just in case. + formatter.AppendString("Signal "); + formatter.AppendUint64(signal_number, 10); + } + formatter.AppendString(" (@0x"); + formatter.AppendUint64(reinterpret_cast<uintptr_t>(siginfo->si_addr), 16); + formatter.AppendString(")"); + formatter.AppendString(" received by PID "); + formatter.AppendUint64(getpid(), 10); + formatter.AppendString(" (TID 0x"); + // We assume pthread_t is an integral number or a pointer, rather + // than a complex struct. In some environments, pthread_self() + // returns an uint64 but in some other environments pthread_self() + // returns a pointer. Hence we use C-style cast here, rather than + // reinterpret/static_cast, to support both types of environments. + formatter.AppendUint64((uintptr_t)pthread_self(), 16); + formatter.AppendString(") "); + // Only linux has the PID of the signal sender in si_pid. +#ifdef OS_LINUX + formatter.AppendString("from PID "); + formatter.AppendUint64(siginfo->si_pid, 10); + formatter.AppendString("; "); +#endif + formatter.AppendString("stack trace: ***\n"); + g_failure_writer(buf, formatter.num_bytes_written()); +} + +#endif // HAVE_SIGACTION + +// Dumps information about the stack frame to STDERR. +void DumpStackFrameInfo(const char* prefix, void* pc) { + // Get the symbol name. + const char *symbol = "(unknown)"; + char symbolized[1024]; // Big enough for a sane symbol. + // Symbolizes the previous address of pc because pc may be in the + // next function. + if (Symbolize(reinterpret_cast<char *>(pc) - 1, + symbolized, sizeof(symbolized))) { + symbol = symbolized; + } + + char buf[1024]; // Big enough for stack frame info. + MinimalFormatter formatter(buf, sizeof(buf)); + + formatter.AppendString(prefix); + formatter.AppendString("@ "); + const int width = 2 * sizeof(void*) + 2; // + 2 for "0x". + formatter.AppendHexWithPadding(reinterpret_cast<uintptr_t>(pc), width); + formatter.AppendString(" "); + formatter.AppendString(symbol); + formatter.AppendString("\n"); + g_failure_writer(buf, formatter.num_bytes_written()); +} + +// Invoke the default signal handler. +void InvokeDefaultSignalHandler(int signal_number) { +#ifdef HAVE_SIGACTION + struct sigaction sig_action; + memset(&sig_action, 0, sizeof(sig_action)); + sigemptyset(&sig_action.sa_mask); + sig_action.sa_handler = SIG_DFL; + sigaction(signal_number, &sig_action, NULL); + kill(getpid(), signal_number); +#elif defined(OS_WINDOWS) + signal(signal_number, SIG_DFL); + raise(signal_number); +#endif +} + +// This variable is used for protecting FailureSignalHandler() from +// dumping stuff while another thread is doing it. Our policy is to let +// the first thread dump stuff and let other threads wait. +// See also comments in FailureSignalHandler(). +static pthread_t* g_entered_thread_id_pointer = NULL; + +// Dumps signal and stack frame information, and invokes the default +// signal handler once our job is done. +#if defined(OS_WINDOWS) +void FailureSignalHandler(int signal_number) +#else +void FailureSignalHandler(int signal_number, + siginfo_t *signal_info, + void *ucontext) +#endif +{ + // First check if we've already entered the function. We use an atomic + // compare and swap operation for platforms that support it. For other + // platforms, we use a naive method that could lead to a subtle race. + + // We assume pthread_self() is async signal safe, though it's not + // officially guaranteed. + pthread_t my_thread_id = pthread_self(); + // NOTE: We could simply use pthread_t rather than pthread_t* for this, + // if pthread_self() is guaranteed to return non-zero value for thread + // ids, but there is no such guarantee. We need to distinguish if the + // old value (value returned from __sync_val_compare_and_swap) is + // different from the original value (in this case NULL). + pthread_t* old_thread_id_pointer = + glog_internal_namespace_::sync_val_compare_and_swap( + &g_entered_thread_id_pointer, + static_cast<pthread_t*>(NULL), + &my_thread_id); + if (old_thread_id_pointer != NULL) { + // We've already entered the signal handler. What should we do? + if (pthread_equal(my_thread_id, *g_entered_thread_id_pointer)) { + // It looks the current thread is reentering the signal handler. + // Something must be going wrong (maybe we are reentering by another + // type of signal?). Kill ourself by the default signal handler. + InvokeDefaultSignalHandler(signal_number); + } + // Another thread is dumping stuff. Let's wait until that thread + // finishes the job and kills the process. + while (true) { + sleep(1); + } + } + // This is the first time we enter the signal handler. We are going to + // do some interesting stuff from here. + // TODO(satorux): We might want to set timeout here using alarm(), but + // mixing alarm() and sleep() can be a bad idea. + + // First dump time info. + DumpTimeInfo(); + +#if !defined(OS_WINDOWS) + // Get the program counter from ucontext. + void *pc = GetPC(ucontext); + DumpStackFrameInfo("PC: ", pc); +#endif + +#ifdef HAVE_STACKTRACE + // Get the stack traces. + void *stack[32]; + // +1 to exclude this function. + const int depth = GetStackTrace(stack, ARRAYSIZE(stack), 1); +# ifdef HAVE_SIGACTION + DumpSignalInfo(signal_number, signal_info); +# endif + // Dump the stack traces. + for (int i = 0; i < depth; ++i) { + DumpStackFrameInfo(" ", stack[i]); + } +#endif + + // *** TRANSITION *** + // + // BEFORE this point, all code must be async-termination-safe! + // (See WARNING above.) + // + // AFTER this point, we do unsafe things, like using LOG()! + // The process could be terminated or hung at any time. We try to + // do more useful things first and riskier things later. + + // Flush the logs before we do anything in case 'anything' + // causes problems. + FlushLogFilesUnsafe(0); + + // Kill ourself by the default signal handler. + InvokeDefaultSignalHandler(signal_number); +} + +} // namespace + +namespace glog_internal_namespace_ { + +bool IsFailureSignalHandlerInstalled() { +#ifdef HAVE_SIGACTION + // TODO(andschwa): Return kFailureSignalHandlerInstalled? + struct sigaction sig_action; + memset(&sig_action, 0, sizeof(sig_action)); + sigemptyset(&sig_action.sa_mask); + sigaction(SIGABRT, NULL, &sig_action); + if (sig_action.sa_sigaction == &FailureSignalHandler) + return true; +#elif defined(OS_WINDOWS) + return kFailureSignalHandlerInstalled; +#endif // HAVE_SIGACTION + return false; +} + +} // namespace glog_internal_namespace_ + +void InstallFailureSignalHandler() { +#ifdef HAVE_SIGACTION + // Build the sigaction struct. + struct sigaction sig_action; + memset(&sig_action, 0, sizeof(sig_action)); + sigemptyset(&sig_action.sa_mask); + sig_action.sa_flags |= SA_SIGINFO; + sig_action.sa_sigaction = &FailureSignalHandler; + + for (size_t i = 0; i < ARRAYSIZE(kFailureSignals); ++i) { + CHECK_ERR(sigaction(kFailureSignals[i].number, &sig_action, NULL)); + } + kFailureSignalHandlerInstalled = true; +#elif defined(OS_WINDOWS) + for (size_t i = 0; i < ARRAYSIZE(kFailureSignals); ++i) { + CHECK_NE(signal(kFailureSignals[i].number, &FailureSignalHandler), + SIG_ERR); + } + kFailureSignalHandlerInstalled = true; +#endif // HAVE_SIGACTION +} + +void InstallFailureWriter(void (*writer)(const char* data, int size)) { +#if defined(HAVE_SIGACTION) || defined(OS_WINDOWS) + g_failure_writer = writer; +#endif // HAVE_SIGACTION +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/signalhandler_unittest.cc b/third_party/glog/src/signalhandler_unittest.cc new file mode 100644 index 000000000000..36d957b9c353 --- /dev/null +++ b/third_party/glog/src/signalhandler_unittest.cc @@ -0,0 +1,112 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// This is a helper binary for testing signalhandler.cc. The actual test +// is done in signalhandler_unittest.sh. + +#include "utilities.h" + +#if defined(HAVE_PTHREAD) +# include <pthread.h> +#endif +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string> +#include "glog/logging.h" + +#ifdef HAVE_LIB_GFLAGS +#include <gflags/gflags.h> +using namespace GFLAGS_NAMESPACE; +#endif + +using namespace GOOGLE_NAMESPACE; + +static void* DieInThread(void*) { + // We assume pthread_t is an integral number or a pointer, rather + // than a complex struct. In some environments, pthread_self() + // returns an uint64 but in some other environments pthread_self() + // returns a pointer. Hence we use C-style cast here, rather than + // reinterpret/static_cast, to support both types of environments. + fprintf(stderr, "0x%lx is dying\n", (long)pthread_self()); + // Use volatile to prevent from these to be optimized away. + volatile int a = 0; + volatile int b = 1 / a; + fprintf(stderr, "We should have died: b=%d\n", b); + return NULL; +} + +static void WriteToStdout(const char* data, int size) { + if (write(STDOUT_FILENO, data, size) < 0) { + // Ignore errors. + } +} + +int main(int argc, char **argv) { +#if defined(HAVE_STACKTRACE) && defined(HAVE_SYMBOLIZE) + InitGoogleLogging(argv[0]); +#ifdef HAVE_LIB_GFLAGS + ParseCommandLineFlags(&argc, &argv, true); +#endif + InstallFailureSignalHandler(); + const std::string command = argc > 1 ? argv[1] : "none"; + if (command == "segv") { + // We'll check if this is outputted. + LOG(INFO) << "create the log file"; + LOG(INFO) << "a message before segv"; + // We assume 0xDEAD is not writable. + int *a = (int*)0xDEAD; + *a = 0; + } else if (command == "loop") { + fprintf(stderr, "looping\n"); + while (true); + } else if (command == "die_in_thread") { +#if defined(HAVE_PTHREAD) + pthread_t thread; + pthread_create(&thread, NULL, &DieInThread, NULL); + pthread_join(thread, NULL); +#else + fprintf(stderr, "no pthread\n"); + return 1; +#endif + } else if (command == "dump_to_stdout") { + InstallFailureWriter(WriteToStdout); + abort(); + } else if (command == "installed") { + fprintf(stderr, "signal handler installed: %s\n", + IsFailureSignalHandlerInstalled() ? "true" : "false"); + } else { + // Tell the shell script + puts("OK"); + } +#endif + return 0; +} diff --git a/third_party/glog/src/signalhandler_unittest.sh b/third_party/glog/src/signalhandler_unittest.sh new file mode 100755 index 000000000000..265cd458c01e --- /dev/null +++ b/third_party/glog/src/signalhandler_unittest.sh @@ -0,0 +1,131 @@ +#! /bin/sh +# +# Copyright (c) 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# Author: Satoru Takabayashi +# +# Unit tests for signalhandler.cc. + +die () { + echo $1 + exit 1 +} + +BINDIR=".libs" +LIBGLOG="$BINDIR/libglog.so" + +BINARY="$BINDIR/signalhandler_unittest" +LOG_INFO="./signalhandler_unittest.INFO" + +# Remove temporary files. +rm -f signalhandler.out* + +if test -e "$BINARY"; then + # We need shared object. + export LD_LIBRARY_PATH=$BINDIR + export DYLD_LIBRARY_PATH=$BINDIR +else + # For windows + BINARY="./signalhandler_unittest.exe" + if ! test -e "$BINARY"; then + echo "We coundn't find demangle_unittest binary." + exit 1 + fi +fi + +if [ x`$BINARY` != 'xOK' ]; then + echo "PASS (No stacktrace support. We don't run this test.)" + exit 0 +fi + +# The PC cannot be obtained in signal handlers on PowerPC correctly. +# We just skip the test for PowerPC. +if [ x`uname -p` = x"powerpc" ]; then + echo "PASS (We don't test the signal handler on PowerPC.)" + exit 0 +fi + +# Test for a case the program kills itself by SIGSEGV. +GOOGLE_LOG_DIR=. $BINARY segv 2> signalhandler.out1 +for pattern in SIGSEGV 0xdead main "Aborted at [0-9]"; do + if ! grep --quiet "$pattern" signalhandler.out1; then + die "'$pattern' should appear in the output" + fi +done +if ! grep --quiet "a message before segv" $LOG_INFO; then + die "'a message before segv' should appear in the INFO log" +fi +rm -f $LOG_INFO + +# Test for a case the program is killed by this shell script. +# $! = the process id of the last command run in the background. +# $$ = the process id of this shell. +$BINARY loop 2> signalhandler.out2 & +# Wait until "looping" is written in the file. This indicates the program +# is ready to accept signals. +while true; do + if grep --quiet looping signalhandler.out2; then + break + fi +done +kill -TERM $! +wait $! + +from_pid='' +# Only linux has the process ID of the signal sender. +if [ x`uname` = "xLinux" ]; then + from_pid="from PID $$" +fi +for pattern in SIGTERM "by PID $!" "$from_pid" main "Aborted at [0-9]"; do + if ! grep --quiet "$pattern" signalhandler.out2; then + die "'$pattern' should appear in the output" + fi +done + +# Test for a case the program dies in a non-main thread. +$BINARY die_in_thread 2> signalhandler.out3 +EXPECTED_TID="`sed 's/ .*//; q' signalhandler.out3`" + +for pattern in SIGFPE DieInThread "TID $EXPECTED_TID" "Aborted at [0-9]"; do + if ! grep --quiet "$pattern" signalhandler.out3; then + die "'$pattern' should appear in the output" + fi +done + +# Test for a case the program installs a custom failure writer that writes +# stuff to stdout instead of stderr. +$BINARY dump_to_stdout 1> signalhandler.out4 +for pattern in SIGABRT main "Aborted at [0-9]"; do + if ! grep --quiet "$pattern" signalhandler.out4; then + die "'$pattern' should appear in the output" + fi +done + +echo PASS diff --git a/third_party/glog/src/stacktrace.h b/third_party/glog/src/stacktrace.h new file mode 100644 index 000000000000..cb64b33a688e --- /dev/null +++ b/third_party/glog/src/stacktrace.h @@ -0,0 +1,61 @@ +// Copyright (c) 2000 - 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Routines to extract the current stack trace. These functions are +// thread-safe. + +#ifndef BASE_STACKTRACE_H_ +#define BASE_STACKTRACE_H_ + +#include "config.h" +#include "glog/logging.h" + +_START_GOOGLE_NAMESPACE_ + +// This is similar to the GetStackFrames routine, except that it returns +// the stack trace only, and not the stack frame sizes as well. +// Example: +// main() { foo(); } +// foo() { bar(); } +// bar() { +// void* result[10]; +// int depth = GetStackFrames(result, 10, 1); +// } +// +// This produces: +// result[0] foo +// result[1] main +// .... ... +// +// "result" must not be NULL. +GOOGLE_GLOG_DLL_DECL int GetStackTrace(void** result, int max_depth, int skip_count); + +_END_GOOGLE_NAMESPACE_ + +#endif // BASE_STACKTRACE_H_ diff --git a/third_party/glog/src/stacktrace_generic-inl.h b/third_party/glog/src/stacktrace_generic-inl.h new file mode 100644 index 000000000000..fad81d3e3f4f --- /dev/null +++ b/third_party/glog/src/stacktrace_generic-inl.h @@ -0,0 +1,59 @@ +// Copyright (c) 2000 - 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Portable implementation - just use glibc +// +// Note: The glibc implementation may cause a call to malloc. +// This can cause a deadlock in HeapProfiler. +#include <execinfo.h> +#include <string.h> +#include "stacktrace.h" + +_START_GOOGLE_NAMESPACE_ + +// If you change this function, also change GetStackFrames below. +int GetStackTrace(void** result, int max_depth, int skip_count) { + static const int kStackLength = 64; + void * stack[kStackLength]; + int size; + + size = backtrace(stack, kStackLength); + skip_count++; // we want to skip the current frame as well + int result_count = size - skip_count; + if (result_count < 0) + result_count = 0; + if (result_count > max_depth) + result_count = max_depth; + for (int i = 0; i < result_count; i++) + result[i] = stack[i + skip_count]; + + return result_count; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/stacktrace_libunwind-inl.h b/third_party/glog/src/stacktrace_libunwind-inl.h new file mode 100644 index 000000000000..e29a50c004f2 --- /dev/null +++ b/third_party/glog/src/stacktrace_libunwind-inl.h @@ -0,0 +1,90 @@ +// Copyright (c) 2005 - 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Arun Sharma +// +// Produce stack trace using libunwind + +#include "utilities.h" + +extern "C" { +#define UNW_LOCAL_ONLY +#include <libunwind.h> +} +#include "glog/raw_logging.h" +#include "stacktrace.h" + +_START_GOOGLE_NAMESPACE_ + +// Sometimes, we can try to get a stack trace from within a stack +// trace, because libunwind can call mmap (maybe indirectly via an +// internal mmap based memory allocator), and that mmap gets trapped +// and causes a stack-trace request. If were to try to honor that +// recursive request, we'd end up with infinite recursion or deadlock. +// Luckily, it's safe to ignore those subsequent traces. In such +// cases, we return 0 to indicate the situation. +// We can use the GCC __thread syntax here since libunwind is not supported on +// Windows. +static __thread bool g_tl_entered; // Initialized to false. + +// If you change this function, also change GetStackFrames below. +int GetStackTrace(void** result, int max_depth, int skip_count) { + void *ip; + int n = 0; + unw_cursor_t cursor; + unw_context_t uc; + + if (g_tl_entered) { + return 0; + } + g_tl_entered = true; + + unw_getcontext(&uc); + RAW_CHECK(unw_init_local(&cursor, &uc) >= 0, "unw_init_local failed"); + skip_count++; // Do not include the "GetStackTrace" frame + + while (n < max_depth) { + int ret = unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip); + if (ret < 0) + break; + if (skip_count > 0) { + skip_count--; + } else { + result[n++] = ip; + } + ret = unw_step(&cursor); + if (ret <= 0) + break; + } + + g_tl_entered = false; + return n; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/stacktrace_powerpc-inl.h b/third_party/glog/src/stacktrace_powerpc-inl.h new file mode 100644 index 000000000000..03b91089aada --- /dev/null +++ b/third_party/glog/src/stacktrace_powerpc-inl.h @@ -0,0 +1,130 @@ +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Craig Silverstein +// +// Produce stack trace. I'm guessing (hoping!) the code is much like +// for x86. For apple machines, at least, it seems to be; see +// http://developer.apple.com/documentation/mac/runtimehtml/RTArch-59.html +// http://www.linux-foundation.org/spec/ELF/ppc64/PPC-elf64abi-1.9.html#STACK +// Linux has similar code: http://patchwork.ozlabs.org/linuxppc/patch?id=8882 + +#include <stdio.h> +#include <stdint.h> // for uintptr_t +#include "stacktrace.h" + +_START_GOOGLE_NAMESPACE_ + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return NULL if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template<bool STRICT_UNWINDING> +static void **NextStackFrame(void **old_sp) { + void **new_sp = (void **) *old_sp; + + // Check that the transition from frame pointer old_sp to frame + // pointer new_sp isn't clearly bogus + if (STRICT_UNWINDING) { + // With the stack growing downwards, older stack frame must be + // at a greater address that the current one. + if (new_sp <= old_sp) return NULL; + // Assume stack frames larger than 100,000 bytes are bogus. + if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return NULL; + } else { + // In the non-strict mode, allow discontiguous stack frames. + // (alternate-signal-stacks for example). + if (new_sp == old_sp) return NULL; + // And allow frames upto about 1MB. + if ((new_sp > old_sp) + && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return NULL; + } + if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return NULL; + return new_sp; +} + +// This ensures that GetStackTrace stes up the Link Register properly. +void StacktracePowerPCDummyFunction() __attribute__((noinline)); +void StacktracePowerPCDummyFunction() { __asm__ volatile(""); } + +// If you change this function, also change GetStackFrames below. +int GetStackTrace(void** result, int max_depth, int skip_count) { + void **sp; + // Apple OS X uses an old version of gnu as -- both Darwin 7.9.0 (Panther) + // and Darwin 8.8.1 (Tiger) use as 1.38. This means we have to use a + // different asm syntax. I don't know quite the best way to discriminate + // systems using the old as from the new one; I've gone with __APPLE__. +#ifdef __APPLE__ + __asm__ volatile ("mr %0,r1" : "=r" (sp)); +#else + __asm__ volatile ("mr %0,1" : "=r" (sp)); +#endif + + // On PowerPC, the "Link Register" or "Link Record" (LR), is a stack + // entry that holds the return address of the subroutine call (what + // instruction we run after our function finishes). This is the + // same as the stack-pointer of our parent routine, which is what we + // want here. While the compiler will always(?) set up LR for + // subroutine calls, it may not for leaf functions (such as this one). + // This routine forces the compiler (at least gcc) to push it anyway. + StacktracePowerPCDummyFunction(); + + // The LR save area is used by the callee, so the top entry is bogus. + skip_count++; + + int n = 0; + while (sp && n < max_depth) { + if (skip_count > 0) { + skip_count--; + } else { + // PowerPC has 3 main ABIs, which say where in the stack the + // Link Register is. For DARWIN and AIX (used by apple and + // linux ppc64), it's in sp[2]. For SYSV (used by linux ppc), + // it's in sp[1]. +#if defined(_CALL_AIX) || defined(_CALL_DARWIN) + result[n++] = *(sp+2); +#elif defined(_CALL_SYSV) + result[n++] = *(sp+1); +#elif defined(__APPLE__) || ((defined(__linux) || defined(__linux__)) && defined(__PPC64__)) + // This check is in case the compiler doesn't define _CALL_AIX/etc. + result[n++] = *(sp+2); +#elif defined(__linux) + // This check is in case the compiler doesn't define _CALL_SYSV. + result[n++] = *(sp+1); +#else +#error Need to specify the PPC ABI for your archiecture. +#endif + } + // Use strict unwinding rules. + sp = NextStackFrame<true>(sp); + } + return n; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/stacktrace_unittest.cc b/third_party/glog/src/stacktrace_unittest.cc new file mode 100644 index 000000000000..77d34291b381 --- /dev/null +++ b/third_party/glog/src/stacktrace_unittest.cc @@ -0,0 +1,217 @@ +// Copyright (c) 2004, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "utilities.h" + +#include <stdio.h> +#include <stdlib.h> +#include "config.h" +#include "base/commandlineflags.h" +#include "glog/logging.h" +#include "stacktrace.h" + +#ifdef HAVE_EXECINFO_H +# include <execinfo.h> +#endif + +using namespace GOOGLE_NAMESPACE; + +#ifdef HAVE_STACKTRACE + +// Obtain a backtrace, verify that the expected callers are present in the +// backtrace, and maybe print the backtrace to stdout. + +// The sequence of functions whose return addresses we expect to see in the +// backtrace. +const int BACKTRACE_STEPS = 6; + +struct AddressRange { + const void *start, *end; +}; + +// Expected function [start,end] range. +AddressRange expected_range[BACKTRACE_STEPS]; + +#if __GNUC__ +// Using GCC extension: address of a label can be taken with '&&label'. +// Start should be a label somewhere before recursive call, end somewhere +// after it. +#define INIT_ADDRESS_RANGE(fn, start_label, end_label, prange) \ + do { \ + (prange)->start = &&start_label; \ + (prange)->end = &&end_label; \ + CHECK_LT((prange)->start, (prange)->end); \ + } while (0) +// This macro expands into "unmovable" code (opaque to GCC), and that +// prevents GCC from moving a_label up or down in the code. +// Without it, there is no code following the 'end' label, and GCC +// (4.3.1, 4.4.0) thinks it safe to assign &&end an address that is before +// the recursive call. +#define DECLARE_ADDRESS_LABEL(a_label) \ + a_label: do { __asm__ __volatile__(""); } while (0) +// Gcc 4.4.0 may split function into multiple chunks, and the chunk +// performing recursive call may end up later in the code then the return +// instruction (this actually happens with FDO). +// Adjust function range from __builtin_return_address. +#define ADJUST_ADDRESS_RANGE_FROM_RA(prange) \ + do { \ + void *ra = __builtin_return_address(0); \ + CHECK_LT((prange)->start, ra); \ + if (ra > (prange)->end) { \ + printf("Adjusting range from %p..%p to %p..%p\n", \ + (prange)->start, (prange)->end, \ + (prange)->start, ra); \ + (prange)->end = ra; \ + } \ + } while (0) +#else +// Assume the Check* functions below are not longer than 256 bytes. +#define INIT_ADDRESS_RANGE(fn, start_label, end_label, prange) \ + do { \ + (prange)->start = reinterpret_cast<const void *>(&fn); \ + (prange)->end = reinterpret_cast<const char *>(&fn) + 256; \ + } while (0) +#define DECLARE_ADDRESS_LABEL(a_label) do { } while (0) +#define ADJUST_ADDRESS_RANGE_FROM_RA(prange) do { } while (0) +#endif // __GNUC__ + +//-----------------------------------------------------------------------// + +static void CheckRetAddrIsInFunction(void *ret_addr, const AddressRange &range) +{ + CHECK_GE(ret_addr, range.start); + CHECK_LE(ret_addr, range.end); +} + +//-----------------------------------------------------------------------// + +void ATTRIBUTE_NOINLINE CheckStackTrace(int); +static void ATTRIBUTE_NOINLINE CheckStackTraceLeaf(void) { + const int STACK_LEN = 10; + void *stack[STACK_LEN]; + int size; + + ADJUST_ADDRESS_RANGE_FROM_RA(&expected_range[1]); + INIT_ADDRESS_RANGE(CheckStackTraceLeaf, start, end, &expected_range[0]); + DECLARE_ADDRESS_LABEL(start); + size = GetStackTrace(stack, STACK_LEN, 0); + printf("Obtained %d stack frames.\n", size); + CHECK_GE(size, 1); + CHECK_LE(size, STACK_LEN); + + if (1) { +#ifdef HAVE_EXECINFO_H + char **strings = backtrace_symbols(stack, size); + printf("Obtained %d stack frames.\n", size); + for (int i = 0; i < size; i++) + printf("%s %p\n", strings[i], stack[i]); + printf("CheckStackTrace() addr: %p\n", &CheckStackTrace); + free(strings); +#endif + } + for (int i = 0; i < BACKTRACE_STEPS; i++) { + printf("Backtrace %d: expected: %p..%p actual: %p ... ", + i, expected_range[i].start, expected_range[i].end, stack[i]); + fflush(stdout); + CheckRetAddrIsInFunction(stack[i], expected_range[i]); + printf("OK\n"); + } + DECLARE_ADDRESS_LABEL(end); +} + +//-----------------------------------------------------------------------// + +/* Dummy functions to make the backtrace more interesting. */ +static void ATTRIBUTE_NOINLINE CheckStackTrace4(int i) { + ADJUST_ADDRESS_RANGE_FROM_RA(&expected_range[2]); + INIT_ADDRESS_RANGE(CheckStackTrace4, start, end, &expected_range[1]); + DECLARE_ADDRESS_LABEL(start); + for (int j = i; j >= 0; j--) + CheckStackTraceLeaf(); + DECLARE_ADDRESS_LABEL(end); +} +static void ATTRIBUTE_NOINLINE CheckStackTrace3(int i) { + ADJUST_ADDRESS_RANGE_FROM_RA(&expected_range[3]); + INIT_ADDRESS_RANGE(CheckStackTrace3, start, end, &expected_range[2]); + DECLARE_ADDRESS_LABEL(start); + for (int j = i; j >= 0; j--) + CheckStackTrace4(j); + DECLARE_ADDRESS_LABEL(end); +} +static void ATTRIBUTE_NOINLINE CheckStackTrace2(int i) { + ADJUST_ADDRESS_RANGE_FROM_RA(&expected_range[4]); + INIT_ADDRESS_RANGE(CheckStackTrace2, start, end, &expected_range[3]); + DECLARE_ADDRESS_LABEL(start); + for (int j = i; j >= 0; j--) + CheckStackTrace3(j); + DECLARE_ADDRESS_LABEL(end); +} +static void ATTRIBUTE_NOINLINE CheckStackTrace1(int i) { + ADJUST_ADDRESS_RANGE_FROM_RA(&expected_range[5]); + INIT_ADDRESS_RANGE(CheckStackTrace1, start, end, &expected_range[4]); + DECLARE_ADDRESS_LABEL(start); + for (int j = i; j >= 0; j--) + CheckStackTrace2(j); + DECLARE_ADDRESS_LABEL(end); +} +#ifndef __GNUC__ +// On non-GNU environment, we use the address of `CheckStackTrace` to +// guess the address range of this function. This guess is wrong for +// non-static function on Windows. This is probably because +// `&CheckStackTrace` returns the address of a trampoline like PLT, +// not the actual address of `CheckStackTrace`. +// See https://github.com/google/glog/issues/421 for the detail. +static +#endif +void ATTRIBUTE_NOINLINE CheckStackTrace(int i) { + INIT_ADDRESS_RANGE(CheckStackTrace, start, end, &expected_range[5]); + DECLARE_ADDRESS_LABEL(start); + for (int j = i; j >= 0; j--) + CheckStackTrace1(j); + DECLARE_ADDRESS_LABEL(end); +} + +//-----------------------------------------------------------------------// + +int main(int, char ** argv) { + FLAGS_logtostderr = true; + InitGoogleLogging(argv[0]); + + CheckStackTrace(0); + + printf("PASS\n"); + return 0; +} + +#else +int main() { + printf("PASS (no stacktrace support)\n"); + return 0; +} +#endif // HAVE_STACKTRACE diff --git a/third_party/glog/src/stacktrace_windows-inl.h b/third_party/glog/src/stacktrace_windows-inl.h new file mode 100644 index 000000000000..f7553a6d9736 --- /dev/null +++ b/third_party/glog/src/stacktrace_windows-inl.h @@ -0,0 +1,50 @@ +// Copyright (c) 2000 - 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Andrew Schwartzmeyer +// +// Windows implementation - just use CaptureStackBackTrace + +#include "config.h" +#include "port.h" +#include "stacktrace.h" +#include <dbghelp.h> + +_START_GOOGLE_NAMESPACE_ + +int GetStackTrace(void** result, int max_depth, int skip_count) { + if (max_depth > 64) { + max_depth = 64; + } + skip_count++; // we want to skip the current frame as well + // This API is thread-safe (moreover it walks only the current thread). + return CaptureStackBackTrace(skip_count, max_depth, result, NULL); +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/stacktrace_x86-inl.h b/third_party/glog/src/stacktrace_x86-inl.h new file mode 100644 index 000000000000..3b8d5a8282d5 --- /dev/null +++ b/third_party/glog/src/stacktrace_x86-inl.h @@ -0,0 +1,146 @@ +// Copyright (c) 2000 - 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Produce stack trace + +#include <stdint.h> // for uintptr_t + +#include "utilities.h" // for OS_* macros + +#if !defined(OS_WINDOWS) +#include <unistd.h> +#include <sys/mman.h> +#endif + +#include <stdio.h> // for NULL +#include "stacktrace.h" + +_START_GOOGLE_NAMESPACE_ + +// Given a pointer to a stack frame, locate and return the calling +// stackframe, or return NULL if no stackframe can be found. Perform sanity +// checks (the strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template<bool STRICT_UNWINDING> +static void **NextStackFrame(void **old_sp) { + void **new_sp = (void **) *old_sp; + + // Check that the transition from frame pointer old_sp to frame + // pointer new_sp isn't clearly bogus + if (STRICT_UNWINDING) { + // With the stack growing downwards, older stack frame must be + // at a greater address that the current one. + if (new_sp <= old_sp) return NULL; + // Assume stack frames larger than 100,000 bytes are bogus. + if ((uintptr_t)new_sp - (uintptr_t)old_sp > 100000) return NULL; + } else { + // In the non-strict mode, allow discontiguous stack frames. + // (alternate-signal-stacks for example). + if (new_sp == old_sp) return NULL; + // And allow frames upto about 1MB. + if ((new_sp > old_sp) + && ((uintptr_t)new_sp - (uintptr_t)old_sp > 1000000)) return NULL; + } + if ((uintptr_t)new_sp & (sizeof(void *) - 1)) return NULL; +#ifdef __i386__ + // On 64-bit machines, the stack pointer can be very close to + // 0xffffffff, so we explicitly check for a pointer into the + // last two pages in the address space + if ((uintptr_t)new_sp >= 0xffffe000) return NULL; +#endif +#if !defined(OS_WINDOWS) + if (!STRICT_UNWINDING) { + // Lax sanity checks cause a crash in 32-bit tcmalloc/crash_reason_test + // on AMD-based machines with VDSO-enabled kernels. + // Make an extra sanity check to insure new_sp is readable. + // Note: NextStackFrame<false>() is only called while the program + // is already on its last leg, so it's ok to be slow here. + static int page_size = getpagesize(); + void *new_sp_aligned = (void *)((uintptr_t)new_sp & ~(page_size - 1)); + if (msync(new_sp_aligned, page_size, MS_ASYNC) == -1) + return NULL; + } +#endif + return new_sp; +} + +// If you change this function, also change GetStackFrames below. +int GetStackTrace(void** result, int max_depth, int skip_count) { + void **sp; + +#ifdef __GNUC__ +#if __GNUC__ * 100 + __GNUC_MINOR__ >= 402 +#define USE_BUILTIN_FRAME_ADDRESS +#endif +#endif + +#ifdef USE_BUILTIN_FRAME_ADDRESS + sp = reinterpret_cast<void**>(__builtin_frame_address(0)); +#elif defined(__i386__) + // Stack frame format: + // sp[0] pointer to previous frame + // sp[1] caller address + // sp[2] first argument + // ... + sp = (void **)&result - 2; +#elif defined(__x86_64__) + // __builtin_frame_address(0) can return the wrong address on gcc-4.1.0-k8 + unsigned long rbp; + // Move the value of the register %rbp into the local variable rbp. + // We need 'volatile' to prevent this instruction from getting moved + // around during optimization to before function prologue is done. + // An alternative way to achieve this + // would be (before this __asm__ instruction) to call Noop() defined as + // static void Noop() __attribute__ ((noinline)); // prevent inlining + // static void Noop() { asm(""); } // prevent optimizing-away + __asm__ volatile ("mov %%rbp, %0" : "=r" (rbp)); + // Arguments are passed in registers on x86-64, so we can't just + // offset from &result + sp = (void **) rbp; +#endif + + int n = 0; + while (sp && n < max_depth) { + if (*(sp+1) == (void *)0) { + // In 64-bit code, we often see a frame that + // points to itself and has a return address of 0. + break; + } + if (skip_count > 0) { + skip_count--; + } else { + result[n++] = *(sp+1); + } + // Use strict unwinding rules. + sp = NextStackFrame<true>(sp); + } + return n; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/stacktrace_x86_64-inl.h b/third_party/glog/src/stacktrace_x86_64-inl.h new file mode 100644 index 000000000000..f7d1dca85bc1 --- /dev/null +++ b/third_party/glog/src/stacktrace_x86_64-inl.h @@ -0,0 +1,105 @@ +// Copyright (c) 2005 - 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Arun Sharma +// +// Produce stack trace using libgcc + +extern "C" { +#include <stdlib.h> // for NULL +#include <unwind.h> // ABI defined unwinder +} +#include "stacktrace.h" + +_START_GOOGLE_NAMESPACE_ + +typedef struct { + void **result; + int max_depth; + int skip_count; + int count; +} trace_arg_t; + + +// Workaround for the malloc() in _Unwind_Backtrace() issue. +static _Unwind_Reason_Code nop_backtrace(struct _Unwind_Context *uc, void *opq) { + return _URC_NO_REASON; +} + + +// This code is not considered ready to run until +// static initializers run so that we are guaranteed +// that any malloc-related initialization is done. +static bool ready_to_run = false; +class StackTraceInit { + public: + StackTraceInit() { + // Extra call to force initialization + _Unwind_Backtrace(nop_backtrace, NULL); + ready_to_run = true; + } +}; + +static StackTraceInit module_initializer; // Force initialization + +static _Unwind_Reason_Code GetOneFrame(struct _Unwind_Context *uc, void *opq) { + trace_arg_t *targ = (trace_arg_t *) opq; + + if (targ->skip_count > 0) { + targ->skip_count--; + } else { + targ->result[targ->count++] = (void *) _Unwind_GetIP(uc); + } + + if (targ->count == targ->max_depth) + return _URC_END_OF_STACK; + + return _URC_NO_REASON; +} + +// If you change this function, also change GetStackFrames below. +int GetStackTrace(void** result, int max_depth, int skip_count) { + if (!ready_to_run) + return 0; + + trace_arg_t targ; + + skip_count += 1; // Do not include the "GetStackTrace" frame + + targ.result = result; + targ.max_depth = max_depth; + targ.skip_count = skip_count; + targ.count = 0; + + _Unwind_Backtrace(GetOneFrame, &targ); + + return targ.count; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/stl_logging_unittest.cc b/third_party/glog/src/stl_logging_unittest.cc new file mode 100644 index 000000000000..269094c07bfa --- /dev/null +++ b/third_party/glog/src/stl_logging_unittest.cc @@ -0,0 +1,197 @@ +// Copyright (c) 2003, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "config.h" + +#ifdef HAVE_USING_OPERATOR + +#include <functional> +#include <iostream> +#include <map> +#include <ostream> +#include <string> +#include <vector> + +#ifdef __GNUC__ +// C++0x isn't enabled by default in GCC and libc++ does not have +// non-standard ext/* and tr1/unordered_*. +# if defined(_LIBCPP_VERSION) +# ifndef GLOG_STL_LOGGING_FOR_UNORDERED +# define GLOG_STL_LOGGING_FOR_UNORDERED +# endif +# else +# ifndef GLOG_STL_LOGGING_FOR_EXT_HASH +# define GLOG_STL_LOGGING_FOR_EXT_HASH +# endif +# ifndef GLOG_STL_LOGGING_FOR_EXT_SLIST +# define GLOG_STL_LOGGING_FOR_EXT_SLIST +# endif +# ifndef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +# define GLOG_STL_LOGGING_FOR_TR1_UNORDERED +# endif +# endif +#endif + +#include "glog/logging.h" +#include "glog/stl_logging.h" +#include "googletest.h" + +using namespace std; +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +using namespace __gnu_cxx; +#endif + +struct user_hash { + size_t operator()(int x) const { return x; } +}; + +static void TestSTLLogging() { + { + // Test a sequence. + vector<int> v; + v.push_back(10); + v.push_back(20); + v.push_back(30); + ostringstream ss; + ss << v; + EXPECT_EQ(ss.str(), "10 20 30"); + vector<int> copied_v(v); + CHECK_EQ(v, copied_v); // This must compile. + } + + { + // Test a sorted pair associative container. + map< int, string > m; + m[20] = "twenty"; + m[10] = "ten"; + m[30] = "thirty"; + ostringstream ss; + ss << m; + EXPECT_EQ(ss.str(), "(10, ten) (20, twenty) (30, thirty)"); + map< int, string > copied_m(m); + CHECK_EQ(m, copied_m); // This must compile. + } + +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH + { + // Test a hashed simple associative container. + hash_set<int> hs; + hs.insert(10); + hs.insert(20); + hs.insert(30); + ostringstream ss; + ss << hs; + EXPECT_EQ(ss.str(), "10 20 30"); + hash_set<int> copied_hs(hs); + CHECK_EQ(hs, copied_hs); // This must compile. + } +#endif + +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH + { + // Test a hashed pair associative container. + hash_map<int, string> hm; + hm[10] = "ten"; + hm[20] = "twenty"; + hm[30] = "thirty"; + ostringstream ss; + ss << hm; + EXPECT_EQ(ss.str(), "(10, ten) (20, twenty) (30, thirty)"); + hash_map<int, string> copied_hm(hm); + CHECK_EQ(hm, copied_hm); // this must compile + } +#endif + + { + // Test a long sequence. + vector<int> v; + string expected; + for (int i = 0; i < 100; i++) { + v.push_back(i); + if (i > 0) expected += ' '; + char buf[256]; + sprintf(buf, "%d", i); + expected += buf; + } + v.push_back(100); + expected += " ..."; + ostringstream ss; + ss << v; + CHECK_EQ(ss.str(), expected.c_str()); + } + + { + // Test a sorted pair associative container. + // Use a non-default comparison functor. + map< int, string, greater<int> > m; + m[20] = "twenty"; + m[10] = "ten"; + m[30] = "thirty"; + ostringstream ss; + ss << m; + EXPECT_EQ(ss.str(), "(30, thirty) (20, twenty) (10, ten)"); + map< int, string, greater<int> > copied_m(m); + CHECK_EQ(m, copied_m); // This must compile. + } + +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH + { + // Test a hashed simple associative container. + // Use a user defined hash function. + hash_set<int, user_hash> hs; + hs.insert(10); + hs.insert(20); + hs.insert(30); + ostringstream ss; + ss << hs; + EXPECT_EQ(ss.str(), "10 20 30"); + hash_set<int, user_hash> copied_hs(hs); + CHECK_EQ(hs, copied_hs); // This must compile. + } +#endif +} + +int main(int, char**) { + TestSTLLogging(); + std::cout << "PASS\n"; + return 0; +} + +#else + +#include <iostream> + +int main(int, char**) { + std::cout << "We don't support stl_logging for this compiler.\n" + << "(we need compiler support of 'using ::operator<<' " + << "for this feature.)\n"; + return 0; +} + +#endif // HAVE_USING_OPERATOR diff --git a/third_party/glog/src/symbolize.cc b/third_party/glog/src/symbolize.cc new file mode 100644 index 000000000000..76a1b0febeec --- /dev/null +++ b/third_party/glog/src/symbolize.cc @@ -0,0 +1,966 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// Stack-footprint reduction work done by Raksit Ashok +// +// Implementation note: +// +// We don't use heaps but only use stacks. We want to reduce the +// stack consumption so that the symbolizer can run on small stacks. +// +// Here are some numbers collected with GCC 4.1.0 on x86: +// - sizeof(Elf32_Sym) = 16 +// - sizeof(Elf32_Shdr) = 40 +// - sizeof(Elf64_Sym) = 24 +// - sizeof(Elf64_Shdr) = 64 +// +// This implementation is intended to be async-signal-safe but uses +// some functions which are not guaranteed to be so, such as memchr() +// and memmove(). We assume they are async-signal-safe. +// +// Additional header can be specified by the GLOG_BUILD_CONFIG_INCLUDE +// macro to add platform specific defines (e.g. OS_OPENBSD). + +#ifdef GLOG_BUILD_CONFIG_INCLUDE +#include GLOG_BUILD_CONFIG_INCLUDE +#endif // GLOG_BUILD_CONFIG_INCLUDE + +#include "utilities.h" + +#if defined(HAVE_SYMBOLIZE) + +#include <string.h> + +#include <algorithm> +#include <limits> + +#include "symbolize.h" +#include "demangle.h" + +_START_GOOGLE_NAMESPACE_ + +// We don't use assert() since it's not guaranteed to be +// async-signal-safe. Instead we define a minimal assertion +// macro. So far, we don't need pretty printing for __FILE__, etc. + +// A wrapper for abort() to make it callable in ? :. +static int AssertFail() { + abort(); + return 0; // Should not reach. +} + +#define SAFE_ASSERT(expr) ((expr) ? 0 : AssertFail()) + +static SymbolizeCallback g_symbolize_callback = NULL; +void InstallSymbolizeCallback(SymbolizeCallback callback) { + g_symbolize_callback = callback; +} + +static SymbolizeOpenObjectFileCallback g_symbolize_open_object_file_callback = + NULL; +void InstallSymbolizeOpenObjectFileCallback( + SymbolizeOpenObjectFileCallback callback) { + g_symbolize_open_object_file_callback = callback; +} + +// This function wraps the Demangle function to provide an interface +// where the input symbol is demangled in-place. +// To keep stack consumption low, we would like this function to not +// get inlined. +static ATTRIBUTE_NOINLINE void DemangleInplace(char *out, int out_size) { + char demangled[256]; // Big enough for sane demangled symbols. + if (Demangle(out, demangled, sizeof(demangled))) { + // Demangling succeeded. Copy to out if the space allows. + size_t len = strlen(demangled); + if (len + 1 <= (size_t)out_size) { // +1 for '\0'. + SAFE_ASSERT(len < sizeof(demangled)); + memmove(out, demangled, len + 1); + } + } +} + +_END_GOOGLE_NAMESPACE_ + +#if defined(__ELF__) + +#if defined(HAVE_DLFCN_H) +#include <dlfcn.h> +#endif +#if defined(OS_OPENBSD) +#include <sys/exec_elf.h> +#else +#include <elf.h> +#endif +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <stddef.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "symbolize.h" +#include "config.h" +#include "glog/raw_logging.h" + +// Re-runs fn until it doesn't cause EINTR. +#define NO_INTR(fn) do {} while ((fn) < 0 && errno == EINTR) + +_START_GOOGLE_NAMESPACE_ + +// Read up to "count" bytes from "offset" in the file pointed by file +// descriptor "fd" into the buffer starting at "buf" while handling short reads +// and EINTR. On success, return the number of bytes read. Otherwise, return +// -1. +static ssize_t ReadFromOffset(const int fd, void *buf, const size_t count, + const off_t offset) { + SAFE_ASSERT(fd >= 0); + SAFE_ASSERT(count <= std::numeric_limits<ssize_t>::max()); + char *buf0 = reinterpret_cast<char *>(buf); + ssize_t num_bytes = 0; + while (num_bytes < count) { + ssize_t len; + NO_INTR(len = pread(fd, buf0 + num_bytes, count - num_bytes, + offset + num_bytes)); + if (len < 0) { // There was an error other than EINTR. + return -1; + } + if (len == 0) { // Reached EOF. + break; + } + num_bytes += len; + } + SAFE_ASSERT(num_bytes <= count); + return num_bytes; +} + +// Try reading exactly "count" bytes from "offset" bytes in a file +// pointed by "fd" into the buffer starting at "buf" while handling +// short reads and EINTR. On success, return true. Otherwise, return +// false. +static bool ReadFromOffsetExact(const int fd, void *buf, + const size_t count, const off_t offset) { + ssize_t len = ReadFromOffset(fd, buf, count, offset); + return len == count; +} + +// Returns elf_header.e_type if the file pointed by fd is an ELF binary. +static int FileGetElfType(const int fd) { + ElfW(Ehdr) elf_header; + if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + return -1; + } + if (memcmp(elf_header.e_ident, ELFMAG, SELFMAG) != 0) { + return -1; + } + return elf_header.e_type; +} + +// Read the section headers in the given ELF binary, and if a section +// of the specified type is found, set the output to this section header +// and return true. Otherwise, return false. +// To keep stack consumption low, we would like this function to not get +// inlined. +static ATTRIBUTE_NOINLINE bool +GetSectionHeaderByType(const int fd, ElfW(Half) sh_num, const off_t sh_offset, + ElfW(Word) type, ElfW(Shdr) *out) { + // Read at most 16 section headers at a time to save read calls. + ElfW(Shdr) buf[16]; + for (int i = 0; i < sh_num;) { + const ssize_t num_bytes_left = (sh_num - i) * sizeof(buf[0]); + const ssize_t num_bytes_to_read = + (sizeof(buf) > num_bytes_left) ? num_bytes_left : sizeof(buf); + const ssize_t len = ReadFromOffset(fd, buf, num_bytes_to_read, + sh_offset + i * sizeof(buf[0])); + if (len == -1) { + return false; + } + SAFE_ASSERT(len % sizeof(buf[0]) == 0); + const ssize_t num_headers_in_buf = len / sizeof(buf[0]); + SAFE_ASSERT(num_headers_in_buf <= sizeof(buf) / sizeof(buf[0])); + for (int j = 0; j < num_headers_in_buf; ++j) { + if (buf[j].sh_type == type) { + *out = buf[j]; + return true; + } + } + i += num_headers_in_buf; + } + return false; +} + +// There is no particular reason to limit section name to 63 characters, +// but there has (as yet) been no need for anything longer either. +const int kMaxSectionNameLen = 64; + +// name_len should include terminating '\0'. +bool GetSectionHeaderByName(int fd, const char *name, size_t name_len, + ElfW(Shdr) *out) { + ElfW(Ehdr) elf_header; + if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + return false; + } + + ElfW(Shdr) shstrtab; + off_t shstrtab_offset = (elf_header.e_shoff + + elf_header.e_shentsize * elf_header.e_shstrndx); + if (!ReadFromOffsetExact(fd, &shstrtab, sizeof(shstrtab), shstrtab_offset)) { + return false; + } + + for (int i = 0; i < elf_header.e_shnum; ++i) { + off_t section_header_offset = (elf_header.e_shoff + + elf_header.e_shentsize * i); + if (!ReadFromOffsetExact(fd, out, sizeof(*out), section_header_offset)) { + return false; + } + char header_name[kMaxSectionNameLen]; + if (sizeof(header_name) < name_len) { + RAW_LOG(WARNING, "Section name '%s' is too long (%" PRIuS "); " + "section will not be found (even if present).", name, name_len); + // No point in even trying. + return false; + } + off_t name_offset = shstrtab.sh_offset + out->sh_name; + ssize_t n_read = ReadFromOffset(fd, &header_name, name_len, name_offset); + if (n_read == -1) { + return false; + } else if (n_read != name_len) { + // Short read -- name could be at end of file. + continue; + } + if (memcmp(header_name, name, name_len) == 0) { + return true; + } + } + return false; +} + +// Read a symbol table and look for the symbol containing the +// pc. Iterate over symbols in a symbol table and look for the symbol +// containing "pc". On success, return true and write the symbol name +// to out. Otherwise, return false. +// To keep stack consumption low, we would like this function to not get +// inlined. +static ATTRIBUTE_NOINLINE bool +FindSymbol(uint64_t pc, const int fd, char *out, int out_size, + uint64_t symbol_offset, const ElfW(Shdr) *strtab, + const ElfW(Shdr) *symtab) { + if (symtab == NULL) { + return false; + } + const int num_symbols = symtab->sh_size / symtab->sh_entsize; + for (int i = 0; i < num_symbols;) { + off_t offset = symtab->sh_offset + i * symtab->sh_entsize; + + // If we are reading Elf64_Sym's, we want to limit this array to + // 32 elements (to keep stack consumption low), otherwise we can + // have a 64 element Elf32_Sym array. +#if __WORDSIZE == 64 +#define NUM_SYMBOLS 32 +#else +#define NUM_SYMBOLS 64 +#endif + + // Read at most NUM_SYMBOLS symbols at once to save read() calls. + ElfW(Sym) buf[NUM_SYMBOLS]; + int num_symbols_to_read = std::min(NUM_SYMBOLS, num_symbols - i); + const ssize_t len = + ReadFromOffset(fd, &buf, sizeof(buf[0]) * num_symbols_to_read, offset); + SAFE_ASSERT(len % sizeof(buf[0]) == 0); + const ssize_t num_symbols_in_buf = len / sizeof(buf[0]); + SAFE_ASSERT(num_symbols_in_buf <= num_symbols_to_read); + for (int j = 0; j < num_symbols_in_buf; ++j) { + const ElfW(Sym)& symbol = buf[j]; + uint64_t start_address = symbol.st_value; + start_address += symbol_offset; + uint64_t end_address = start_address + symbol.st_size; + if (symbol.st_value != 0 && // Skip null value symbols. + symbol.st_shndx != 0 && // Skip undefined symbols. + start_address <= pc && pc < end_address) { + ssize_t len1 = ReadFromOffset(fd, out, out_size, + strtab->sh_offset + symbol.st_name); + if (len1 <= 0 || memchr(out, '\0', out_size) == NULL) { + memset(out, 0, out_size); + return false; + } + return true; // Obtained the symbol name. + } + } + i += num_symbols_in_buf; + } + return false; +} + +// Get the symbol name of "pc" from the file pointed by "fd". Process +// both regular and dynamic symbol tables if necessary. On success, +// write the symbol name to "out" and return true. Otherwise, return +// false. +static bool GetSymbolFromObjectFile(const int fd, + uint64_t pc, + char* out, + int out_size, + uint64_t base_address) { + // Read the ELF header. + ElfW(Ehdr) elf_header; + if (!ReadFromOffsetExact(fd, &elf_header, sizeof(elf_header), 0)) { + return false; + } + + ElfW(Shdr) symtab, strtab; + + // Consult a regular symbol table first. + if (GetSectionHeaderByType(fd, elf_header.e_shnum, elf_header.e_shoff, + SHT_SYMTAB, &symtab)) { + if (!ReadFromOffsetExact(fd, &strtab, sizeof(strtab), elf_header.e_shoff + + symtab.sh_link * sizeof(symtab))) { + return false; + } + if (FindSymbol(pc, fd, out, out_size, base_address, &strtab, &symtab)) { + return true; // Found the symbol in a regular symbol table. + } + } + + // If the symbol is not found, then consult a dynamic symbol table. + if (GetSectionHeaderByType(fd, elf_header.e_shnum, elf_header.e_shoff, + SHT_DYNSYM, &symtab)) { + if (!ReadFromOffsetExact(fd, &strtab, sizeof(strtab), elf_header.e_shoff + + symtab.sh_link * sizeof(symtab))) { + return false; + } + if (FindSymbol(pc, fd, out, out_size, base_address, &strtab, &symtab)) { + return true; // Found the symbol in a dynamic symbol table. + } + } + + return false; +} + +namespace { +// Thin wrapper around a file descriptor so that the file descriptor +// gets closed for sure. +struct FileDescriptor { + const int fd_; + explicit FileDescriptor(int fd) : fd_(fd) {} + ~FileDescriptor() { + if (fd_ >= 0) { + close(fd_); + } + } + int get() { return fd_; } + + private: + explicit FileDescriptor(const FileDescriptor&); + void operator=(const FileDescriptor&); +}; + +// Helper class for reading lines from file. +// +// Note: we don't use ProcMapsIterator since the object is big (it has +// a 5k array member) and uses async-unsafe functions such as sscanf() +// and snprintf(). +class LineReader { + public: + explicit LineReader(int fd, char *buf, int buf_len, off_t offset) + : fd_(fd), + buf_(buf), + buf_len_(buf_len), + offset_(offset), + bol_(buf), + eol_(buf), + eod_(buf) {} + + // Read '\n'-terminated line from file. On success, modify "bol" + // and "eol", then return true. Otherwise, return false. + // + // Note: if the last line doesn't end with '\n', the line will be + // dropped. It's an intentional behavior to make the code simple. + bool ReadLine(const char **bol, const char **eol) { + if (BufferIsEmpty()) { // First time. + const ssize_t num_bytes = ReadFromOffset(fd_, buf_, buf_len_, offset_); + if (num_bytes <= 0) { // EOF or error. + return false; + } + offset_ += num_bytes; + eod_ = buf_ + num_bytes; + bol_ = buf_; + } else { + bol_ = eol_ + 1; // Advance to the next line in the buffer. + SAFE_ASSERT(bol_ <= eod_); // "bol_" can point to "eod_". + if (!HasCompleteLine()) { + const int incomplete_line_length = eod_ - bol_; + // Move the trailing incomplete line to the beginning. + memmove(buf_, bol_, incomplete_line_length); + // Read text from file and append it. + char * const append_pos = buf_ + incomplete_line_length; + const int capacity_left = buf_len_ - incomplete_line_length; + const ssize_t num_bytes = + ReadFromOffset(fd_, append_pos, capacity_left, offset_); + if (num_bytes <= 0) { // EOF or error. + return false; + } + offset_ += num_bytes; + eod_ = append_pos + num_bytes; + bol_ = buf_; + } + } + eol_ = FindLineFeed(); + if (eol_ == NULL) { // '\n' not found. Malformed line. + return false; + } + *eol_ = '\0'; // Replace '\n' with '\0'. + + *bol = bol_; + *eol = eol_; + return true; + } + + // Beginning of line. + const char *bol() { + return bol_; + } + + // End of line. + const char *eol() { + return eol_; + } + + private: + explicit LineReader(const LineReader&); + void operator=(const LineReader&); + + char *FindLineFeed() { + return reinterpret_cast<char *>(memchr(bol_, '\n', eod_ - bol_)); + } + + bool BufferIsEmpty() { + return buf_ == eod_; + } + + bool HasCompleteLine() { + return !BufferIsEmpty() && FindLineFeed() != NULL; + } + + const int fd_; + char * const buf_; + const int buf_len_; + off_t offset_; + char *bol_; + char *eol_; + const char *eod_; // End of data in "buf_". +}; +} // namespace + +// Place the hex number read from "start" into "*hex". The pointer to +// the first non-hex character or "end" is returned. +static char *GetHex(const char *start, const char *end, uint64_t *hex) { + *hex = 0; + const char *p; + for (p = start; p < end; ++p) { + int ch = *p; + if ((ch >= '0' && ch <= '9') || + (ch >= 'A' && ch <= 'F') || (ch >= 'a' && ch <= 'f')) { + *hex = (*hex << 4) | (ch < 'A' ? ch - '0' : (ch & 0xF) + 9); + } else { // Encountered the first non-hex character. + break; + } + } + SAFE_ASSERT(p <= end); + return const_cast<char *>(p); +} + +// Searches for the object file (from /proc/self/maps) that contains +// the specified pc. If found, sets |start_address| to the start address +// of where this object file is mapped in memory, sets the module base +// address into |base_address|, copies the object file name into +// |out_file_name|, and attempts to open the object file. If the object +// file is opened successfully, returns the file descriptor. Otherwise, +// returns -1. |out_file_name_size| is the size of the file name buffer +// (including the null-terminator). +static ATTRIBUTE_NOINLINE int +OpenObjectFileContainingPcAndGetStartAddress(uint64_t pc, + uint64_t &start_address, + uint64_t &base_address, + char *out_file_name, + int out_file_name_size) { + int object_fd; + + int maps_fd; + NO_INTR(maps_fd = open("/proc/self/maps", O_RDONLY)); + FileDescriptor wrapped_maps_fd(maps_fd); + if (wrapped_maps_fd.get() < 0) { + return -1; + } + + int mem_fd; + NO_INTR(mem_fd = open("/proc/self/mem", O_RDONLY)); + FileDescriptor wrapped_mem_fd(mem_fd); + if (wrapped_mem_fd.get() < 0) { + return -1; + } + + // Iterate over maps and look for the map containing the pc. Then + // look into the symbol tables inside. + char buf[1024]; // Big enough for line of sane /proc/self/maps + int num_maps = 0; + LineReader reader(wrapped_maps_fd.get(), buf, sizeof(buf), 0); + while (true) { + num_maps++; + const char *cursor; + const char *eol; + if (!reader.ReadLine(&cursor, &eol)) { // EOF or malformed line. + return -1; + } + + // Start parsing line in /proc/self/maps. Here is an example: + // + // 08048000-0804c000 r-xp 00000000 08:01 2142121 /bin/cat + // + // We want start address (08048000), end address (0804c000), flags + // (r-xp) and file name (/bin/cat). + + // Read start address. + cursor = GetHex(cursor, eol, &start_address); + if (cursor == eol || *cursor != '-') { + return -1; // Malformed line. + } + ++cursor; // Skip '-'. + + // Read end address. + uint64_t end_address; + cursor = GetHex(cursor, eol, &end_address); + if (cursor == eol || *cursor != ' ') { + return -1; // Malformed line. + } + ++cursor; // Skip ' '. + + // Read flags. Skip flags until we encounter a space or eol. + const char * const flags_start = cursor; + while (cursor < eol && *cursor != ' ') { + ++cursor; + } + // We expect at least four letters for flags (ex. "r-xp"). + if (cursor == eol || cursor < flags_start + 4) { + return -1; // Malformed line. + } + + // Determine the base address by reading ELF headers in process memory. + ElfW(Ehdr) ehdr; + // Skip non-readable maps. + if (flags_start[0] == 'r' && + ReadFromOffsetExact(mem_fd, &ehdr, sizeof(ElfW(Ehdr)), start_address) && + memcmp(ehdr.e_ident, ELFMAG, SELFMAG) == 0) { + switch (ehdr.e_type) { + case ET_EXEC: + base_address = 0; + break; + case ET_DYN: + // Find the segment containing file offset 0. This will correspond + // to the ELF header that we just read. Normally this will have + // virtual address 0, but this is not guaranteed. We must subtract + // the virtual address from the address where the ELF header was + // mapped to get the base address. + // + // If we fail to find a segment for file offset 0, use the address + // of the ELF header as the base address. + base_address = start_address; + for (unsigned i = 0; i != ehdr.e_phnum; ++i) { + ElfW(Phdr) phdr; + if (ReadFromOffsetExact( + mem_fd, &phdr, sizeof(phdr), + start_address + ehdr.e_phoff + i * sizeof(phdr)) && + phdr.p_type == PT_LOAD && phdr.p_offset == 0) { + base_address = start_address - phdr.p_vaddr; + break; + } + } + break; + default: + // ET_REL or ET_CORE. These aren't directly executable, so they don't + // affect the base address. + break; + } + } + + // Check start and end addresses. + if (!(start_address <= pc && pc < end_address)) { + continue; // We skip this map. PC isn't in this map. + } + + // Check flags. We are only interested in "r*x" maps. + if (flags_start[0] != 'r' || flags_start[2] != 'x') { + continue; // We skip this map. + } + ++cursor; // Skip ' '. + + // Read file offset. + uint64_t file_offset; + cursor = GetHex(cursor, eol, &file_offset); + if (cursor == eol || *cursor != ' ') { + return -1; // Malformed line. + } + ++cursor; // Skip ' '. + + // Skip to file name. "cursor" now points to dev. We need to + // skip at least two spaces for dev and inode. + int num_spaces = 0; + while (cursor < eol) { + if (*cursor == ' ') { + ++num_spaces; + } else if (num_spaces >= 2) { + // The first non-space character after skipping two spaces + // is the beginning of the file name. + break; + } + ++cursor; + } + if (cursor == eol) { + return -1; // Malformed line. + } + + // Finally, "cursor" now points to file name of our interest. + NO_INTR(object_fd = open(cursor, O_RDONLY)); + if (object_fd < 0) { + // Failed to open object file. Copy the object file name to + // |out_file_name|. + strncpy(out_file_name, cursor, out_file_name_size); + // Making sure |out_file_name| is always null-terminated. + out_file_name[out_file_name_size - 1] = '\0'; + return -1; + } + return object_fd; + } +} + +// POSIX doesn't define any async-signal safe function for converting +// an integer to ASCII. We'll have to define our own version. +// itoa_r() converts a (signed) integer to ASCII. It returns "buf", if the +// conversion was successful or NULL otherwise. It never writes more than "sz" +// bytes. Output will be truncated as needed, and a NUL character is always +// appended. +// NOTE: code from sandbox/linux/seccomp-bpf/demo.cc. +static char *itoa_r(intptr_t i, char *buf, size_t sz, int base, size_t padding) { + // Make sure we can write at least one NUL byte. + size_t n = 1; + if (n > sz) + return NULL; + + if (base < 2 || base > 16) { + buf[0] = '\000'; + return NULL; + } + + char *start = buf; + + uintptr_t j = i; + + // Handle negative numbers (only for base 10). + if (i < 0 && base == 10) { + // This does "j = -i" while avoiding integer overflow. + j = static_cast<uintptr_t>(-(i + 1)) + 1; + + // Make sure we can write the '-' character. + if (++n > sz) { + buf[0] = '\000'; + return NULL; + } + *start++ = '-'; + } + + // Loop until we have converted the entire number. Output at least one + // character (i.e. '0'). + char *ptr = start; + do { + // Make sure there is still enough space left in our output buffer. + if (++n > sz) { + buf[0] = '\000'; + return NULL; + } + + // Output the next digit. + *ptr++ = "0123456789abcdef"[j % base]; + j /= base; + + if (padding > 0) + padding--; + } while (j > 0 || padding > 0); + + // Terminate the output with a NUL character. + *ptr = '\000'; + + // Conversion to ASCII actually resulted in the digits being in reverse + // order. We can't easily generate them in forward order, as we can't tell + // the number of characters needed until we are done converting. + // So, now, we reverse the string (except for the possible "-" sign). + while (--ptr > start) { + char ch = *ptr; + *ptr = *start; + *start++ = ch; + } + return buf; +} + +// Safely appends string |source| to string |dest|. Never writes past the +// buffer size |dest_size| and guarantees that |dest| is null-terminated. +static void SafeAppendString(const char* source, char* dest, int dest_size) { + int dest_string_length = strlen(dest); + SAFE_ASSERT(dest_string_length < dest_size); + dest += dest_string_length; + dest_size -= dest_string_length; + strncpy(dest, source, dest_size); + // Making sure |dest| is always null-terminated. + dest[dest_size - 1] = '\0'; +} + +// Converts a 64-bit value into a hex string, and safely appends it to |dest|. +// Never writes past the buffer size |dest_size| and guarantees that |dest| is +// null-terminated. +static void SafeAppendHexNumber(uint64_t value, char* dest, int dest_size) { + // 64-bit numbers in hex can have up to 16 digits. + char buf[17] = {'\0'}; + SafeAppendString(itoa_r(value, buf, sizeof(buf), 16, 0), dest, dest_size); +} + +// The implementation of our symbolization routine. If it +// successfully finds the symbol containing "pc" and obtains the +// symbol name, returns true and write the symbol name to "out". +// Otherwise, returns false. If Callback function is installed via +// InstallSymbolizeCallback(), the function is also called in this function, +// and "out" is used as its output. +// To keep stack consumption low, we would like this function to not +// get inlined. +static ATTRIBUTE_NOINLINE bool SymbolizeAndDemangle(void *pc, char *out, + int out_size) { + uint64_t pc0 = reinterpret_cast<uintptr_t>(pc); + uint64_t start_address = 0; + uint64_t base_address = 0; + int object_fd = -1; + + if (out_size < 1) { + return false; + } + out[0] = '\0'; + SafeAppendString("(", out, out_size); + + if (g_symbolize_open_object_file_callback) { + object_fd = g_symbolize_open_object_file_callback(pc0, start_address, + base_address, out + 1, + out_size - 1); + } else { + object_fd = OpenObjectFileContainingPcAndGetStartAddress(pc0, start_address, + base_address, + out + 1, + out_size - 1); + } + + FileDescriptor wrapped_object_fd(object_fd); + +#if defined(PRINT_UNSYMBOLIZED_STACK_TRACES) + { +#else + // Check whether a file name was returned. + if (object_fd < 0) { +#endif + if (out[1]) { + // The object file containing PC was determined successfully however the + // object file was not opened successfully. This is still considered + // success because the object file name and offset are known and tools + // like asan_symbolize.py can be used for the symbolization. + out[out_size - 1] = '\0'; // Making sure |out| is always null-terminated. + SafeAppendString("+0x", out, out_size); + SafeAppendHexNumber(pc0 - base_address, out, out_size); + SafeAppendString(")", out, out_size); + return true; + } + // Failed to determine the object file containing PC. Bail out. + return false; + } + int elf_type = FileGetElfType(wrapped_object_fd.get()); + if (elf_type == -1) { + return false; + } + if (g_symbolize_callback) { + // Run the call back if it's installed. + // Note: relocation (and much of the rest of this code) will be + // wrong for prelinked shared libraries and PIE executables. + uint64_t relocation = (elf_type == ET_DYN) ? start_address : 0; + int num_bytes_written = g_symbolize_callback(wrapped_object_fd.get(), + pc, out, out_size, + relocation); + if (num_bytes_written > 0) { + out += num_bytes_written; + out_size -= num_bytes_written; + } + } + if (!GetSymbolFromObjectFile(wrapped_object_fd.get(), pc0, + out, out_size, base_address)) { + if (out[1] && !g_symbolize_callback) { + // The object file containing PC was opened successfully however the + // symbol was not found. The object may have been stripped. This is still + // considered success because the object file name and offset are known + // and tools like asan_symbolize.py can be used for the symbolization. + out[out_size - 1] = '\0'; // Making sure |out| is always null-terminated. + SafeAppendString("+0x", out, out_size); + SafeAppendHexNumber(pc0 - base_address, out, out_size); + SafeAppendString(")", out, out_size); + return true; + } + return false; + } + + // Symbolization succeeded. Now we try to demangle the symbol. + DemangleInplace(out, out_size); + return true; +} + +_END_GOOGLE_NAMESPACE_ + +#elif defined(OS_MACOSX) && defined(HAVE_DLADDR) + +#include <dlfcn.h> +#include <string.h> + +_START_GOOGLE_NAMESPACE_ + +static ATTRIBUTE_NOINLINE bool SymbolizeAndDemangle(void *pc, char *out, + int out_size) { + Dl_info info; + if (dladdr(pc, &info)) { + if ((int)strlen(info.dli_sname) < out_size) { + strcpy(out, info.dli_sname); + // Symbolization succeeded. Now we try to demangle the symbol. + DemangleInplace(out, out_size); + return true; + } + } + return false; +} + +_END_GOOGLE_NAMESPACE_ + +#elif defined(OS_WINDOWS) || defined(OS_CYGWIN) + +#include <windows.h> +#include <dbghelp.h> + +#ifdef _MSC_VER +#pragma comment(lib, "dbghelp") +#endif + +_START_GOOGLE_NAMESPACE_ + +class SymInitializer { +public: + HANDLE process; + bool ready; + SymInitializer() : process(NULL), ready(false) { + // Initialize the symbol handler. + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms680344(v=vs.85).aspx + process = GetCurrentProcess(); + // Defer symbol loading. + // We do not request undecorated symbols with SYMOPT_UNDNAME + // because the mangling library calls UnDecorateSymbolName. + SymSetOptions(SYMOPT_DEFERRED_LOADS); + if (SymInitialize(process, NULL, true)) { + ready = true; + } + } + ~SymInitializer() { + SymCleanup(process); + // We do not need to close `HANDLE process` because it's a "pseudo handle." + } +private: + SymInitializer(const SymInitializer&); + SymInitializer& operator=(const SymInitializer&); +}; + +static ATTRIBUTE_NOINLINE bool SymbolizeAndDemangle(void *pc, char *out, + int out_size) { + const static SymInitializer symInitializer; + if (!symInitializer.ready) { + return false; + } + // Resolve symbol information from address. + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms680578(v=vs.85).aspx + char buf[sizeof(SYMBOL_INFO) + MAX_SYM_NAME]; + SYMBOL_INFO *symbol = reinterpret_cast<SYMBOL_INFO *>(buf); + symbol->SizeOfStruct = sizeof(SYMBOL_INFO); + symbol->MaxNameLen = MAX_SYM_NAME; + // We use the ANSI version to ensure the string type is always `char *`. + // This could break if a symbol has Unicode in it. + BOOL ret = SymFromAddr(symInitializer.process, + reinterpret_cast<DWORD64>(pc), 0, symbol); + if (ret == 1 && static_cast<int>(symbol->NameLen) < out_size) { + // `NameLen` does not include the null terminating character. + strncpy(out, symbol->Name, static_cast<size_t>(symbol->NameLen) + 1); + out[static_cast<size_t>(symbol->NameLen)] = '\0'; + // Symbolization succeeded. Now we try to demangle the symbol. + DemangleInplace(out, out_size); + return true; + } + return false; +} + +_END_GOOGLE_NAMESPACE_ + +#else +# error BUG: HAVE_SYMBOLIZE was wrongly set +#endif + +_START_GOOGLE_NAMESPACE_ + +bool Symbolize(void *pc, char *out, int out_size) { + SAFE_ASSERT(out_size >= 0); + return SymbolizeAndDemangle(pc, out, out_size); +} + +_END_GOOGLE_NAMESPACE_ + +#else /* HAVE_SYMBOLIZE */ + +#include <assert.h> + +#include "config.h" + +_START_GOOGLE_NAMESPACE_ + +// TODO: Support other environments. +bool Symbolize(void *pc, char *out, int out_size) { + assert(0); + return false; +} + +_END_GOOGLE_NAMESPACE_ + +#endif diff --git a/third_party/glog/src/symbolize.h b/third_party/glog/src/symbolize.h new file mode 100644 index 000000000000..c6f9ec4360ed --- /dev/null +++ b/third_party/glog/src/symbolize.h @@ -0,0 +1,158 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// This library provides Symbolize() function that symbolizes program +// counters to their corresponding symbol names on linux platforms. +// This library has a minimal implementation of an ELF symbol table +// reader (i.e. it doesn't depend on libelf, etc.). +// +// The algorithm used in Symbolize() is as follows. +// +// 1. Go through a list of maps in /proc/self/maps and find the map +// containing the program counter. +// +// 2. Open the mapped file and find a regular symbol table inside. +// Iterate over symbols in the symbol table and look for the symbol +// containing the program counter. If such a symbol is found, +// obtain the symbol name, and demangle the symbol if possible. +// If the symbol isn't found in the regular symbol table (binary is +// stripped), try the same thing with a dynamic symbol table. +// +// Note that Symbolize() is originally implemented to be used in +// FailureSignalHandler() in base/google.cc. Hence it doesn't use +// malloc() and other unsafe operations. It should be both +// thread-safe and async-signal-safe. + +#ifndef BASE_SYMBOLIZE_H_ +#define BASE_SYMBOLIZE_H_ + +#include "utilities.h" +#include "config.h" +#include "glog/logging.h" + +#ifdef HAVE_SYMBOLIZE + +#if defined(__ELF__) // defined by gcc +#if defined(__OpenBSD__) +#include <sys/exec_elf.h> +#else +#include <elf.h> +#endif + +#if !defined(ANDROID) +#include <link.h> // For ElfW() macro. +#endif + +// For systems where SIZEOF_VOID_P is not defined, determine it +// based on __LP64__ (defined by gcc on 64-bit systems) +#if !defined(SIZEOF_VOID_P) +# if defined(__LP64__) +# define SIZEOF_VOID_P 8 +# else +# define SIZEOF_VOID_P 4 +# endif +#endif + +// If there is no ElfW macro, let's define it by ourself. +#ifndef ElfW +# if SIZEOF_VOID_P == 4 +# define ElfW(type) Elf32_##type +# elif SIZEOF_VOID_P == 8 +# define ElfW(type) Elf64_##type +# else +# error "Unknown sizeof(void *)" +# endif +#endif + +_START_GOOGLE_NAMESPACE_ + +// Gets the section header for the given name, if it exists. Returns true on +// success. Otherwise, returns false. +bool GetSectionHeaderByName(int fd, const char *name, size_t name_len, + ElfW(Shdr) *out); + +_END_GOOGLE_NAMESPACE_ + +#endif /* __ELF__ */ + +_START_GOOGLE_NAMESPACE_ + +// Restrictions on the callbacks that follow: +// - The callbacks must not use heaps but only use stacks. +// - The callbacks must be async-signal-safe. + +// Installs a callback function, which will be called right before a symbol name +// is printed. The callback is intended to be used for showing a file name and a +// line number preceding a symbol name. +// "fd" is a file descriptor of the object file containing the program +// counter "pc". The callback function should write output to "out" +// and return the size of the output written. On error, the callback +// function should return -1. +typedef int (*SymbolizeCallback)(int fd, + void* pc, + char* out, + size_t out_size, + uint64_t relocation); +void InstallSymbolizeCallback(SymbolizeCallback callback); + +// Installs a callback function, which will be called instead of +// OpenObjectFileContainingPcAndGetStartAddress. The callback is expected +// to searches for the object file (from /proc/self/maps) that contains +// the specified pc. If found, sets |start_address| to the start address +// of where this object file is mapped in memory, sets the module base +// address into |base_address|, copies the object file name into +// |out_file_name|, and attempts to open the object file. If the object +// file is opened successfully, returns the file descriptor. Otherwise, +// returns -1. |out_file_name_size| is the size of the file name buffer +// (including the null-terminator). +typedef int (*SymbolizeOpenObjectFileCallback)(uint64_t pc, + uint64_t& start_address, + uint64_t& base_address, + char* out_file_name, + int out_file_name_size); +void InstallSymbolizeOpenObjectFileCallback( + SymbolizeOpenObjectFileCallback callback); + +_END_GOOGLE_NAMESPACE_ + +#endif + +_START_GOOGLE_NAMESPACE_ + +// Symbolizes a program counter. On success, returns true and write the +// symbol name to "out". The symbol name is demangled if possible +// (supports symbols generated by GCC 3.x or newer). Otherwise, +// returns false. +GOOGLE_GLOG_DLL_DECL bool Symbolize(void *pc, char *out, int out_size); + +_END_GOOGLE_NAMESPACE_ + +#endif // BASE_SYMBOLIZE_H_ diff --git a/third_party/glog/src/symbolize_unittest.cc b/third_party/glog/src/symbolize_unittest.cc new file mode 100644 index 000000000000..35cc2d31ff9b --- /dev/null +++ b/third_party/glog/src/symbolize_unittest.cc @@ -0,0 +1,425 @@ +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Satoru Takabayashi +// +// Unit tests for functions in symbolize.cc. + +#include "utilities.h" + +#include <signal.h> +#include <iostream> + +#include "glog/logging.h" +#include "symbolize.h" +#include "googletest.h" +#include "config.h" + +#ifdef HAVE_LIB_GFLAGS +#include <gflags/gflags.h> +using namespace GFLAGS_NAMESPACE; +#endif + +using namespace std; +using namespace GOOGLE_NAMESPACE; + +#if defined(HAVE_STACKTRACE) + +#define always_inline + +// A wrapper function for Symbolize() to make the unit test simple. +static const char *TrySymbolize(void *pc) { + static char symbol[4096]; + if (Symbolize(pc, symbol, sizeof(symbol))) { + return symbol; + } else { + return NULL; + } +} + +# if defined(__ELF__) + +// This unit tests make sense only with GCC. +// Uses lots of GCC specific features. +#if defined(__GNUC__) && !defined(__OPENCC__) +# if __GNUC__ >= 4 +# define TEST_WITH_MODERN_GCC +# if __i386__ // always_inline isn't supported for x86_64 with GCC 4.1.0. +# undef always_inline +# define always_inline __attribute__((always_inline)) +# define HAVE_ALWAYS_INLINE +# endif // __i386__ +# else +# endif // __GNUC__ >= 4 +# if defined(__i386__) || defined(__x86_64__) +# define TEST_X86_32_AND_64 1 +# endif // defined(__i386__) || defined(__x86_64__) +#endif + +// Make them C linkage to avoid mangled names. +extern "C" { +void nonstatic_func(); +void nonstatic_func() { + volatile int a = 0; + ++a; +} + +static void static_func() { + volatile int a = 0; + ++a; +} +} + +TEST(Symbolize, Symbolize) { + // We do C-style cast since GCC 2.95.3 doesn't allow + // reinterpret_cast<void *>(&func). + + // Compilers should give us pointers to them. + EXPECT_STREQ("nonstatic_func", TrySymbolize((void *)(&nonstatic_func))); + + // The name of an internal linkage symbol is not specified; allow either a + // mangled or an unmangled name here. + const char *static_func_symbol = TrySymbolize((void *)(&static_func)); + CHECK(NULL != static_func_symbol); + EXPECT_TRUE(strcmp("static_func", static_func_symbol) == 0 || + strcmp("static_func()", static_func_symbol) == 0); + + EXPECT_TRUE(NULL == TrySymbolize(NULL)); +} + +struct Foo { + static void func(int x); +}; + +void ATTRIBUTE_NOINLINE Foo::func(int x) { + volatile int a = x; + ++a; +} + +// With a modern GCC, Symbolize() should return demangled symbol +// names. Function parameters should be omitted. +#ifdef TEST_WITH_MODERN_GCC +TEST(Symbolize, SymbolizeWithDemangling) { + Foo::func(100); + EXPECT_STREQ("Foo::func()", TrySymbolize((void *)(&Foo::func))); +} +#endif + +// Tests that verify that Symbolize footprint is within some limit. + +// To measure the stack footprint of the Symbolize function, we create +// a signal handler (for SIGUSR1 say) that calls the Symbolize function +// on an alternate stack. This alternate stack is initialized to some +// known pattern (0x55, 0x55, 0x55, ...). We then self-send this signal, +// and after the signal handler returns, look at the alternate stack +// buffer to see what portion has been touched. +// +// This trick gives us the the stack footprint of the signal handler. +// But the signal handler, even before the call to Symbolize, consumes +// some stack already. We however only want the stack usage of the +// Symbolize function. To measure this accurately, we install two signal +// handlers: one that does nothing and just returns, and another that +// calls Symbolize. The difference between the stack consumption of these +// two signals handlers should give us the Symbolize stack foorprint. + +static void *g_pc_to_symbolize; +static char g_symbolize_buffer[4096]; +static char *g_symbolize_result; + +static void EmptySignalHandler(int signo) {} + +static void SymbolizeSignalHandler(int signo) { + if (Symbolize(g_pc_to_symbolize, g_symbolize_buffer, + sizeof(g_symbolize_buffer))) { + g_symbolize_result = g_symbolize_buffer; + } else { + g_symbolize_result = NULL; + } +} + +const int kAlternateStackSize = 8096; +const char kAlternateStackFillValue = 0x55; + +// These helper functions look at the alternate stack buffer, and figure +// out what portion of this buffer has been touched - this is the stack +// consumption of the signal handler running on this alternate stack. +static ATTRIBUTE_NOINLINE bool StackGrowsDown(int *x) { + int y; + return &y < x; +} +static int GetStackConsumption(const char* alt_stack) { + int x; + if (StackGrowsDown(&x)) { + for (int i = 0; i < kAlternateStackSize; i++) { + if (alt_stack[i] != kAlternateStackFillValue) { + return (kAlternateStackSize - i); + } + } + } else { + for (int i = (kAlternateStackSize - 1); i >= 0; i--) { + if (alt_stack[i] != kAlternateStackFillValue) { + return i; + } + } + } + return -1; +} + +#ifdef HAVE_SIGALTSTACK + +// Call Symbolize and figure out the stack footprint of this call. +static const char *SymbolizeStackConsumption(void *pc, int *stack_consumed) { + + g_pc_to_symbolize = pc; + + // The alt-signal-stack cannot be heap allocated because there is a + // bug in glibc-2.2 where some signal handler setup code looks at the + // current stack pointer to figure out what thread is currently running. + // Therefore, the alternate stack must be allocated from the main stack + // itself. + char altstack[kAlternateStackSize]; + memset(altstack, kAlternateStackFillValue, kAlternateStackSize); + + // Set up the alt-signal-stack (and save the older one). + stack_t sigstk; + memset(&sigstk, 0, sizeof(stack_t)); + stack_t old_sigstk; + sigstk.ss_sp = altstack; + sigstk.ss_size = kAlternateStackSize; + sigstk.ss_flags = 0; + CHECK_ERR(sigaltstack(&sigstk, &old_sigstk)); + + // Set up SIGUSR1 and SIGUSR2 signal handlers (and save the older ones). + struct sigaction sa; + memset(&sa, 0, sizeof(struct sigaction)); + struct sigaction old_sa1, old_sa2; + sigemptyset(&sa.sa_mask); + sa.sa_flags = SA_ONSTACK; + + // SIGUSR1 maps to EmptySignalHandler. + sa.sa_handler = EmptySignalHandler; + CHECK_ERR(sigaction(SIGUSR1, &sa, &old_sa1)); + + // SIGUSR2 maps to SymbolizeSignalHanlder. + sa.sa_handler = SymbolizeSignalHandler; + CHECK_ERR(sigaction(SIGUSR2, &sa, &old_sa2)); + + // Send SIGUSR1 signal and measure the stack consumption of the empty + // signal handler. + CHECK_ERR(kill(getpid(), SIGUSR1)); + int stack_consumption1 = GetStackConsumption(altstack); + + // Send SIGUSR2 signal and measure the stack consumption of the symbolize + // signal handler. + CHECK_ERR(kill(getpid(), SIGUSR2)); + int stack_consumption2 = GetStackConsumption(altstack); + + // The difference between the two stack consumption values is the + // stack footprint of the Symbolize function. + if (stack_consumption1 != -1 && stack_consumption2 != -1) { + *stack_consumed = stack_consumption2 - stack_consumption1; + } else { + *stack_consumed = -1; + } + + // Log the stack consumption values. + LOG(INFO) << "Stack consumption of empty signal handler: " + << stack_consumption1; + LOG(INFO) << "Stack consumption of symbolize signal handler: " + << stack_consumption2; + LOG(INFO) << "Stack consumption of Symbolize: " << *stack_consumed; + + // Now restore the old alt-signal-stack and signal handlers. + CHECK_ERR(sigaltstack(&old_sigstk, NULL)); + CHECK_ERR(sigaction(SIGUSR1, &old_sa1, NULL)); + CHECK_ERR(sigaction(SIGUSR2, &old_sa2, NULL)); + + return g_symbolize_result; +} + +#ifdef __ppc64__ +// Symbolize stack consumption should be within 4kB. +const int kStackConsumptionUpperLimit = 4096; +#else +// Symbolize stack consumption should be within 2kB. +const int kStackConsumptionUpperLimit = 2048; +#endif + +TEST(Symbolize, SymbolizeStackConsumption) { + int stack_consumed; + const char* symbol; + + symbol = SymbolizeStackConsumption((void *)(&nonstatic_func), + &stack_consumed); + EXPECT_STREQ("nonstatic_func", symbol); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); + + // The name of an internal linkage symbol is not specified; allow either a + // mangled or an unmangled name here. + symbol = SymbolizeStackConsumption((void *)(&static_func), + &stack_consumed); + CHECK(NULL != symbol); + EXPECT_TRUE(strcmp("static_func", symbol) == 0 || + strcmp("static_func()", symbol) == 0); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); +} + +#ifdef TEST_WITH_MODERN_GCC +TEST(Symbolize, SymbolizeWithDemanglingStackConsumption) { + Foo::func(100); + int stack_consumed; + const char* symbol; + + symbol = SymbolizeStackConsumption((void *)(&Foo::func), &stack_consumed); + + EXPECT_STREQ("Foo::func()", symbol); + EXPECT_GT(stack_consumed, 0); + EXPECT_LT(stack_consumed, kStackConsumptionUpperLimit); +} +#endif + +#endif // HAVE_SIGALTSTACK + +// x86 specific tests. Uses some inline assembler. +extern "C" { +inline void* always_inline inline_func() { + void *pc = NULL; +#ifdef TEST_X86_32_AND_64 + __asm__ __volatile__("call 1f; 1: pop %0" : "=r"(pc)); +#endif + return pc; +} + +void* ATTRIBUTE_NOINLINE non_inline_func(); +void* ATTRIBUTE_NOINLINE non_inline_func() { + void *pc = NULL; +#ifdef TEST_X86_32_AND_64 + __asm__ __volatile__("call 1f; 1: pop %0" : "=r"(pc)); +#endif + return pc; +} + +static void ATTRIBUTE_NOINLINE TestWithPCInsideNonInlineFunction() { +#if defined(TEST_X86_32_AND_64) && defined(HAVE_ATTRIBUTE_NOINLINE) + void *pc = non_inline_func(); + const char *symbol = TrySymbolize(pc); + CHECK(symbol != NULL); + CHECK_STREQ(symbol, "non_inline_func"); + cout << "Test case TestWithPCInsideNonInlineFunction passed." << endl; +#endif +} + +static void ATTRIBUTE_NOINLINE TestWithPCInsideInlineFunction() { +#if defined(TEST_X86_32_AND_64) && defined(HAVE_ALWAYS_INLINE) + void *pc = inline_func(); // Must be inlined. + const char *symbol = TrySymbolize(pc); + CHECK(symbol != NULL); + CHECK_STREQ(symbol, __FUNCTION__); + cout << "Test case TestWithPCInsideInlineFunction passed." << endl; +#endif +} +} + +// Test with a return address. +static void ATTRIBUTE_NOINLINE TestWithReturnAddress() { +#if defined(HAVE_ATTRIBUTE_NOINLINE) + void *return_address = __builtin_return_address(0); + const char *symbol = TrySymbolize(return_address); + CHECK(symbol != NULL); + CHECK_STREQ(symbol, "main"); + cout << "Test case TestWithReturnAddress passed." << endl; +#endif +} + +# elif defined(OS_WINDOWS) || defined(OS_CYGWIN) + +#ifdef _MSC_VER +#include <intrin.h> +#pragma intrinsic(_ReturnAddress) +#endif + +struct Foo { + static void func(int x); +}; + +__declspec(noinline) void Foo::func(int x) { + volatile int a = x; + ++a; +} + +TEST(Symbolize, SymbolizeWithDemangling) { + Foo::func(100); + const char* ret = TrySymbolize((void *)(&Foo::func)); + EXPECT_STREQ("public: static void __cdecl Foo::func(int)", ret); +} + +__declspec(noinline) void TestWithReturnAddress() { + void *return_address = +#ifdef __GNUC__ // Cygwin and MinGW support + __builtin_return_address(0) +#else + _ReturnAddress() +#endif + ; + const char *symbol = TrySymbolize(return_address); + CHECK(symbol != NULL); + CHECK_STREQ(symbol, "main"); + cout << "Test case TestWithReturnAddress passed." << endl; +} +# endif // __ELF__ +#endif // HAVE_STACKTRACE + +int main(int argc, char **argv) { + FLAGS_logtostderr = true; + InitGoogleLogging(argv[0]); + InitGoogleTest(&argc, argv); +#if defined(HAVE_SYMBOLIZE) && defined(HAVE_STACKTRACE) +# if defined(__ELF__) + // We don't want to get affected by the callback interface, that may be + // used to install some callback function at InitGoogle() time. + InstallSymbolizeCallback(NULL); + + TestWithPCInsideInlineFunction(); + TestWithPCInsideNonInlineFunction(); + TestWithReturnAddress(); + return RUN_ALL_TESTS(); +# elif defined(OS_WINDOWS) || defined(OS_CYGWIN) + TestWithReturnAddress(); + return RUN_ALL_TESTS(); +# else // OS_WINDOWS + printf("PASS (no symbolize_unittest support)\n"); + return 0; +# endif // __ELF__ +#else + printf("PASS (no symbolize support)\n"); + return 0; +#endif // HAVE_SYMBOLIZE +} diff --git a/third_party/glog/src/utilities.cc b/third_party/glog/src/utilities.cc new file mode 100644 index 000000000000..9a1e35d0229d --- /dev/null +++ b/third_party/glog/src/utilities.cc @@ -0,0 +1,374 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Shinichiro Hamaji + +#include "utilities.h" + +#include <stdio.h> +#include <stdlib.h> + +#include <signal.h> +#ifdef HAVE_SYS_TIME_H +# include <sys/time.h> +#endif +#include <time.h> +#if defined(HAVE_SYSCALL_H) +#include <syscall.h> // for syscall() +#elif defined(HAVE_SYS_SYSCALL_H) +#include <sys/syscall.h> // for syscall() +#endif +#ifdef HAVE_SYSLOG_H +# include <syslog.h> +#endif +#ifdef HAVE_UNISTD_H +# include <unistd.h> // For geteuid. +#endif +#ifdef HAVE_PWD_H +# include <pwd.h> +#endif + +#include "base/googleinit.h" + +using std::string; + +_START_GOOGLE_NAMESPACE_ + +static const char* g_program_invocation_short_name = NULL; + +_END_GOOGLE_NAMESPACE_ + +// The following APIs are all internal. +#ifdef HAVE_STACKTRACE + +#include "stacktrace.h" +#include "symbolize.h" +#include "base/commandlineflags.h" + +GLOG_DEFINE_bool(symbolize_stacktrace, true, + "Symbolize the stack trace in the tombstone"); + +_START_GOOGLE_NAMESPACE_ + +typedef void DebugWriter(const char*, void*); + +// The %p field width for printf() functions is two characters per byte. +// For some environments, add two extra bytes for the leading "0x". +static const int kPrintfPointerFieldWidth = 2 + 2 * sizeof(void*); + +static void DebugWriteToStderr(const char* data, void *) { + // This one is signal-safe. + if (write(STDERR_FILENO, data, strlen(data)) < 0) { + // Ignore errors. + } +} + +static void DebugWriteToString(const char* data, void *arg) { + reinterpret_cast<string*>(arg)->append(data); +} + +#ifdef HAVE_SYMBOLIZE +// Print a program counter and its symbol name. +static void DumpPCAndSymbol(DebugWriter *writerfn, void *arg, void *pc, + const char * const prefix) { + char tmp[1024]; + const char *symbol = "(unknown)"; + // Symbolizes the previous address of pc because pc may be in the + // next function. The overrun happens when the function ends with + // a call to a function annotated noreturn (e.g. CHECK). + if (Symbolize(reinterpret_cast<char *>(pc) - 1, tmp, sizeof(tmp))) { + symbol = tmp; + } + char buf[1024]; + snprintf(buf, sizeof(buf), "%s@ %*p %s\n", + prefix, kPrintfPointerFieldWidth, pc, symbol); + writerfn(buf, arg); +} +#endif + +static void DumpPC(DebugWriter *writerfn, void *arg, void *pc, + const char * const prefix) { + char buf[100]; + snprintf(buf, sizeof(buf), "%s@ %*p\n", + prefix, kPrintfPointerFieldWidth, pc); + writerfn(buf, arg); +} + +// Dump current stack trace as directed by writerfn +static void DumpStackTrace(int skip_count, DebugWriter *writerfn, void *arg) { + // Print stack trace + void* stack[32]; + int depth = GetStackTrace(stack, ARRAYSIZE(stack), skip_count+1); + for (int i = 0; i < depth; i++) { +#if defined(HAVE_SYMBOLIZE) + if (FLAGS_symbolize_stacktrace) { + DumpPCAndSymbol(writerfn, arg, stack[i], " "); + } else { + DumpPC(writerfn, arg, stack[i], " "); + } +#else + DumpPC(writerfn, arg, stack[i], " "); +#endif + } +} + +static void DumpStackTraceAndExit() { + DumpStackTrace(1, DebugWriteToStderr, NULL); + + // TODO(hamaji): Use signal instead of sigaction? + if (IsFailureSignalHandlerInstalled()) { + // Set the default signal handler for SIGABRT, to avoid invoking our + // own signal handler installed by InstallFailureSignalHandler(). +#ifdef HAVE_SIGACTION + struct sigaction sig_action; + memset(&sig_action, 0, sizeof(sig_action)); + sigemptyset(&sig_action.sa_mask); + sig_action.sa_handler = SIG_DFL; + sigaction(SIGABRT, &sig_action, NULL); +#elif defined(OS_WINDOWS) + signal(SIGABRT, SIG_DFL); +#endif // HAVE_SIGACTION + } + + abort(); +} + +_END_GOOGLE_NAMESPACE_ + +#endif // HAVE_STACKTRACE + +_START_GOOGLE_NAMESPACE_ + +namespace glog_internal_namespace_ { + +const char* ProgramInvocationShortName() { + if (g_program_invocation_short_name != NULL) { + return g_program_invocation_short_name; + } else { + // TODO(hamaji): Use /proc/self/cmdline and so? + return "UNKNOWN"; + } +} + +bool IsGoogleLoggingInitialized() { + return g_program_invocation_short_name != NULL; +} + +#ifdef OS_WINDOWS +struct timeval { + long tv_sec, tv_usec; +}; + +// Based on: http://www.google.com/codesearch/p?hl=en#dR3YEbitojA/os_win32.c&q=GetSystemTimeAsFileTime%20license:bsd +// See COPYING for copyright information. +static int gettimeofday(struct timeval *tv, void* tz) { +#define EPOCHFILETIME (116444736000000000ULL) + FILETIME ft; + LARGE_INTEGER li; + uint64 tt; + + GetSystemTimeAsFileTime(&ft); + li.LowPart = ft.dwLowDateTime; + li.HighPart = ft.dwHighDateTime; + tt = (li.QuadPart - EPOCHFILETIME) / 10; + tv->tv_sec = tt / 1000000; + tv->tv_usec = tt % 1000000; + + return 0; +} +#endif + +int64 CycleClock_Now() { + // TODO(hamaji): temporary impementation - it might be too slow. + struct timeval tv; + gettimeofday(&tv, NULL); + return static_cast<int64>(tv.tv_sec) * 1000000 + tv.tv_usec; +} + +int64 UsecToCycles(int64 usec) { + return usec; +} + +WallTime WallTime_Now() { + // Now, cycle clock is retuning microseconds since the epoch. + return CycleClock_Now() * 0.000001; +} + +static int32 g_main_thread_pid = getpid(); +int32 GetMainThreadPid() { + return g_main_thread_pid; +} + +bool PidHasChanged() { + int32 pid = getpid(); + if (g_main_thread_pid == pid) { + return false; + } + g_main_thread_pid = pid; + return true; +} + +pid_t GetTID() { + // On Linux and MacOSX, we try to use gettid(). +#if defined OS_LINUX || defined OS_MACOSX +#ifndef __NR_gettid +#ifdef OS_MACOSX +#define __NR_gettid SYS_gettid +#elif ! defined __i386__ +#error "Must define __NR_gettid for non-x86 platforms" +#else +#define __NR_gettid 224 +#endif +#endif + static bool lacks_gettid = false; + if (!lacks_gettid) { + pid_t tid = syscall(__NR_gettid); + if (tid != -1) { + return tid; + } + // Technically, this variable has to be volatile, but there is a small + // performance penalty in accessing volatile variables and there should + // not be any serious adverse effect if a thread does not immediately see + // the value change to "true". + lacks_gettid = true; + } +#endif // OS_LINUX || OS_MACOSX + + // If gettid() could not be used, we use one of the following. +#if defined OS_LINUX + return getpid(); // Linux: getpid returns thread ID when gettid is absent +#elif defined OS_WINDOWS && !defined OS_CYGWIN + return GetCurrentThreadId(); +#elif defined(HAVE_PTHREAD) + // If none of the techniques above worked, we use pthread_self(). + return (pid_t)(uintptr_t)pthread_self(); +#else + return -1; +#endif +} + +const char* const_basename(const char* filepath) { + const char* base = strrchr(filepath, '/'); +#ifdef OS_WINDOWS // Look for either path separator in Windows + if (!base) + base = strrchr(filepath, '\\'); +#endif + return base ? (base+1) : filepath; +} + +static string g_my_user_name; +const string& MyUserName() { + return g_my_user_name; +} +static void MyUserNameInitializer() { + // TODO(hamaji): Probably this is not portable. +#if defined(OS_WINDOWS) + const char* user = getenv("USERNAME"); +#else + const char* user = getenv("USER"); +#endif + if (user != NULL) { + g_my_user_name = user; + } else { +#if defined(HAVE_PWD_H) && defined(HAVE_UNISTD_H) + struct passwd pwd; + struct passwd* result = NULL; + char buffer[1024] = {'\0'}; + uid_t uid = geteuid(); + int pwuid_res = getpwuid_r(uid, &pwd, buffer, sizeof(buffer), &result); + if (pwuid_res == 0 && result) { + g_my_user_name = pwd.pw_name; + } else { + snprintf(buffer, sizeof(buffer), "uid%d", uid); + g_my_user_name = buffer; + } +#endif + if (g_my_user_name.empty()) { + g_my_user_name = "invalid-user"; + } + } + +} +REGISTER_MODULE_INITIALIZER(utilities, MyUserNameInitializer()); + +#ifdef HAVE_STACKTRACE +void DumpStackTraceToString(string* stacktrace) { + DumpStackTrace(1, DebugWriteToString, stacktrace); +} +#endif + +// We use an atomic operation to prevent problems with calling CrashReason +// from inside the Mutex implementation (potentially through RAW_CHECK). +static const CrashReason* g_reason = 0; + +void SetCrashReason(const CrashReason* r) { + sync_val_compare_and_swap(&g_reason, + reinterpret_cast<const CrashReason*>(0), + r); +} + +void InitGoogleLoggingUtilities(const char* argv0) { + CHECK(!IsGoogleLoggingInitialized()) + << "You called InitGoogleLogging() twice!"; + const char* slash = strrchr(argv0, '/'); +#ifdef OS_WINDOWS + if (!slash) slash = strrchr(argv0, '\\'); +#endif + g_program_invocation_short_name = slash ? slash + 1 : argv0; + +#ifdef HAVE_STACKTRACE + InstallFailureFunction(&DumpStackTraceAndExit); +#endif +} + +void ShutdownGoogleLoggingUtilities() { + CHECK(IsGoogleLoggingInitialized()) + << "You called ShutdownGoogleLogging() without calling InitGoogleLogging() first!"; + g_program_invocation_short_name = NULL; +#ifdef HAVE_SYSLOG_H + closelog(); +#endif +} + +} // namespace glog_internal_namespace_ + +_END_GOOGLE_NAMESPACE_ + +// Make an implementation of stacktrace compiled. +#ifdef STACKTRACE_H +# include STACKTRACE_H +# if 0 +// For include scanners which can't handle macro expansions. +# include "stacktrace_libunwind-inl.h" +# include "stacktrace_x86-inl.h" +# include "stacktrace_x86_64-inl.h" +# include "stacktrace_powerpc-inl.h" +# include "stacktrace_generic-inl.h" +# endif +#endif diff --git a/third_party/glog/src/utilities.h b/third_party/glog/src/utilities.h new file mode 100644 index 000000000000..c66f91467d06 --- /dev/null +++ b/third_party/glog/src/utilities.h @@ -0,0 +1,238 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Shinichiro Hamaji +// +// Define utilties for glog internal usage. + +#ifndef UTILITIES_H__ +#define UTILITIES_H__ + +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) +# define OS_WINDOWS +#elif defined(__CYGWIN__) || defined(__CYGWIN32__) +# define OS_CYGWIN +#elif defined(linux) || defined(__linux) || defined(__linux__) +# ifndef OS_LINUX +# define OS_LINUX +# endif +#elif defined(macintosh) || defined(__APPLE__) || defined(__APPLE_CC__) +# define OS_MACOSX +#elif defined(__FreeBSD__) +# define OS_FREEBSD +#elif defined(__NetBSD__) +# define OS_NETBSD +#elif defined(__OpenBSD__) +# define OS_OPENBSD +#else +// TODO(hamaji): Add other platforms. +#endif + +// printf macros for size_t, in the style of inttypes.h +#ifdef _LP64 +#define __PRIS_PREFIX "z" +#else +#define __PRIS_PREFIX +#endif + +// Use these macros after a % in a printf format string +// to get correct 32/64 bit behavior, like this: +// size_t size = records.size(); +// printf("%"PRIuS"\n", size); + +#define PRIdS __PRIS_PREFIX "d" +#define PRIxS __PRIS_PREFIX "x" +#define PRIuS __PRIS_PREFIX "u" +#define PRIXS __PRIS_PREFIX "X" +#define PRIoS __PRIS_PREFIX "o" + +#include "base/mutex.h" // This must go first so we get _XOPEN_SOURCE + +#include <string> + +#if defined(OS_WINDOWS) +# include "port.h" +#endif + +#include "config.h" +#include "glog/logging.h" + +// There are three different ways we can try to get the stack trace: +// +// 1) The libunwind library. This is still in development, and as a +// separate library adds a new dependency, but doesn't need a frame +// pointer. It also doesn't call malloc. +// +// 2) Our hand-coded stack-unwinder. This depends on a certain stack +// layout, which is used by gcc (and those systems using a +// gcc-compatible ABI) on x86 systems, at least since gcc 2.95. +// It uses the frame pointer to do its work. +// +// 3) The gdb unwinder -- also the one used by the c++ exception code. +// It's obviously well-tested, but has a fatal flaw: it can call +// malloc() from the unwinder. This is a problem because we're +// trying to use the unwinder to instrument malloc(). +// +// 4) The Windows API CaptureStackTrace. +// +// Note: if you add a new implementation here, make sure it works +// correctly when GetStackTrace() is called with max_depth == 0. +// Some code may do that. + +#if defined(HAVE_LIB_UNWIND) +# define STACKTRACE_H "stacktrace_libunwind-inl.h" +#elif !defined(NO_FRAME_POINTER) +# if defined(__i386__) && __GNUC__ >= 2 +# define STACKTRACE_H "stacktrace_x86-inl.h" +# elif defined(__x86_64__) && __GNUC__ >= 2 && HAVE_UNWIND_H +# define STACKTRACE_H "stacktrace_x86_64-inl.h" +# elif (defined(__ppc__) || defined(__PPC__)) && __GNUC__ >= 2 +# define STACKTRACE_H "stacktrace_powerpc-inl.h" +# elif defined(OS_WINDOWS) +# define STACKTRACE_H "stacktrace_windows-inl.h" +# endif +#endif + +#if !defined(STACKTRACE_H) && defined(HAVE_EXECINFO_H) +# define STACKTRACE_H "stacktrace_generic-inl.h" +#endif + +#if defined(STACKTRACE_H) +# define HAVE_STACKTRACE +#endif + +#ifndef HAVE_SYMBOLIZE +// defined by gcc +#if defined(__ELF__) && defined(OS_LINUX) +# define HAVE_SYMBOLIZE +#elif defined(OS_MACOSX) && defined(HAVE_DLADDR) +// Use dladdr to symbolize. +# define HAVE_SYMBOLIZE +#elif defined(OS_WINDOWS) +// Use DbgHelp to symbolize +# define HAVE_SYMBOLIZE +#endif +#endif // !defined(HAVE_SYMBOLIZE) + +#ifndef ARRAYSIZE +// There is a better way, but this is good enough for our purpose. +# define ARRAYSIZE(a) (sizeof(a) / sizeof(*(a))) +#endif + +_START_GOOGLE_NAMESPACE_ + +namespace glog_internal_namespace_ { + +#ifdef HAVE___ATTRIBUTE__ +# define ATTRIBUTE_NOINLINE __attribute__ ((noinline)) +# define HAVE_ATTRIBUTE_NOINLINE +#elif defined(OS_WINDOWS) +# define ATTRIBUTE_NOINLINE __declspec(noinline) +# define HAVE_ATTRIBUTE_NOINLINE +#else +# define ATTRIBUTE_NOINLINE +#endif + +const char* ProgramInvocationShortName(); + +bool IsGoogleLoggingInitialized(); + +int64 CycleClock_Now(); + +int64 UsecToCycles(int64 usec); + +typedef double WallTime; +WallTime WallTime_Now(); + +int32 GetMainThreadPid(); +bool PidHasChanged(); + +pid_t GetTID(); + +const std::string& MyUserName(); + +// Get the part of filepath after the last path separator. +// (Doesn't modify filepath, contrary to basename() in libgen.h.) +const char* const_basename(const char* filepath); + +// Wrapper of __sync_val_compare_and_swap. If the GCC extension isn't +// defined, we try the CPU specific logics (we only support x86 and +// x86_64 for now) first, then use a naive implementation, which has a +// race condition. +template<typename T> +inline T sync_val_compare_and_swap(T* ptr, T oldval, T newval) { +#if defined(HAVE___SYNC_VAL_COMPARE_AND_SWAP) + return __sync_val_compare_and_swap(ptr, oldval, newval); +#elif defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) + T ret; + __asm__ __volatile__("lock; cmpxchg %1, (%2);" + :"=a"(ret) + // GCC may produces %sil or %dil for + // constraint "r", but some of apple's gas + // dosn't know the 8 bit registers. + // We use "q" to avoid these registers. + :"q"(newval), "q"(ptr), "a"(oldval) + :"memory", "cc"); + return ret; +#else + T ret = *ptr; + if (ret == oldval) { + *ptr = newval; + } + return ret; +#endif +} + +void DumpStackTraceToString(std::string* stacktrace); + +struct CrashReason { + CrashReason() : filename(0), line_number(0), message(0), depth(0) {} + + const char* filename; + int line_number; + const char* message; + + // We'll also store a bit of stack trace context at the time of crash as + // it may not be available later on. + void* stack[32]; + int depth; +}; + +void SetCrashReason(const CrashReason* r); + +void InitGoogleLoggingUtilities(const char* argv0); +void ShutdownGoogleLoggingUtilities(); + +} // namespace glog_internal_namespace_ + +_END_GOOGLE_NAMESPACE_ + +using namespace GOOGLE_NAMESPACE::glog_internal_namespace_; + +#endif // UTILITIES_H__ diff --git a/third_party/glog/src/utilities_unittest.cc b/third_party/glog/src/utilities_unittest.cc new file mode 100644 index 000000000000..38e847dfbf75 --- /dev/null +++ b/third_party/glog/src/utilities_unittest.cc @@ -0,0 +1,58 @@ +// Copyright (c) 2008, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Shinichiro Hamaji +#include "utilities.h" +#include "googletest.h" +#include "glog/logging.h" + +#ifdef HAVE_LIB_GFLAGS +#include <gflags/gflags.h> +using namespace GFLAGS_NAMESPACE; +#endif + +using namespace GOOGLE_NAMESPACE; + +TEST(utilities, sync_val_compare_and_swap) { + bool now_entering = false; + EXPECT_FALSE(sync_val_compare_and_swap(&now_entering, false, true)); + EXPECT_TRUE(sync_val_compare_and_swap(&now_entering, false, true)); + EXPECT_TRUE(sync_val_compare_and_swap(&now_entering, false, true)); +} + +TEST(utilities, InitGoogleLoggingDeathTest) { + ASSERT_DEATH(InitGoogleLogging("foobar"), ""); +} + +int main(int argc, char **argv) { + InitGoogleLogging(argv[0]); + InitGoogleTest(&argc, argv); + + CHECK_EQ(RUN_ALL_TESTS(), 0); +} diff --git a/third_party/glog/src/vlog_is_on.cc b/third_party/glog/src/vlog_is_on.cc new file mode 100644 index 000000000000..e8fdbae7dcba --- /dev/null +++ b/third_party/glog/src/vlog_is_on.cc @@ -0,0 +1,257 @@ +// Copyright (c) 1999, 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Ray Sidney and many others +// +// Broken out from logging.cc by Soren Lassen +// logging_unittest.cc covers the functionality herein + +#include "utilities.h" + +#include <string.h> +#include <stdlib.h> +#include <errno.h> +#include <cstdio> +#include <string> +#include "base/commandlineflags.h" +#include "glog/logging.h" +#include "glog/raw_logging.h" +#include "base/googleinit.h" + +// glog doesn't have annotation +#define ANNOTATE_BENIGN_RACE(address, description) + +using std::string; + +GLOG_DEFINE_int32(v, 0, "Show all VLOG(m) messages for m <= this." +" Overridable by --vmodule."); + +GLOG_DEFINE_string(vmodule, "", "per-module verbose level." +" Argument is a comma-separated list of <module name>=<log level>." +" <module name> is a glob pattern, matched against the filename base" +" (that is, name ignoring .cc/.h./-inl.h)." +" <log level> overrides any value given by --v."); + +_START_GOOGLE_NAMESPACE_ + +namespace glog_internal_namespace_ { + +// Used by logging_unittests.cc so can't make it static here. +GOOGLE_GLOG_DLL_DECL bool SafeFNMatch_(const char* pattern, + size_t patt_len, + const char* str, + size_t str_len); + +// Implementation of fnmatch that does not need 0-termination +// of arguments and does not allocate any memory, +// but we only support "*" and "?" wildcards, not the "[...]" patterns. +// It's not a static function for the unittest. +GOOGLE_GLOG_DLL_DECL bool SafeFNMatch_(const char* pattern, + size_t patt_len, + const char* str, + size_t str_len) { + size_t p = 0; + size_t s = 0; + while (1) { + if (p == patt_len && s == str_len) return true; + if (p == patt_len) return false; + if (s == str_len) return p+1 == patt_len && pattern[p] == '*'; + if (pattern[p] == str[s] || pattern[p] == '?') { + p += 1; + s += 1; + continue; + } + if (pattern[p] == '*') { + if (p+1 == patt_len) return true; + do { + if (SafeFNMatch_(pattern+(p+1), patt_len-(p+1), str+s, str_len-s)) { + return true; + } + s += 1; + } while (s != str_len); + return false; + } + return false; + } +} + +} // namespace glog_internal_namespace_ + +using glog_internal_namespace_::SafeFNMatch_; + +int32 kLogSiteUninitialized = 1000; + +// List of per-module log levels from FLAGS_vmodule. +// Once created each element is never deleted/modified +// except for the vlog_level: other threads will read VModuleInfo blobs +// w/o locks and we'll store pointers to vlog_level at VLOG locations +// that will never go away. +// We can't use an STL struct here as we wouldn't know +// when it's safe to delete/update it: other threads need to use it w/o locks. +struct VModuleInfo { + string module_pattern; + mutable int32 vlog_level; // Conceptually this is an AtomicWord, but it's + // too much work to use AtomicWord type here + // w/o much actual benefit. + const VModuleInfo* next; +}; + +// This protects the following global variables. +static Mutex vmodule_lock; +// Pointer to head of the VModuleInfo list. +// It's a map from module pattern to logging level for those module(s). +static VModuleInfo* vmodule_list = 0; +// Boolean initialization flag. +static bool inited_vmodule = false; + +// L >= vmodule_lock. +static void VLOG2Initializer() { + vmodule_lock.AssertHeld(); + // Can now parse --vmodule flag and initialize mapping of module-specific + // logging levels. + inited_vmodule = false; + const char* vmodule = FLAGS_vmodule.c_str(); + const char* sep; + VModuleInfo* head = NULL; + VModuleInfo* tail = NULL; + while ((sep = strchr(vmodule, '=')) != NULL) { + string pattern(vmodule, sep - vmodule); + int module_level; + if (sscanf(sep, "=%d", &module_level) == 1) { + VModuleInfo* info = new VModuleInfo; + info->module_pattern = pattern; + info->vlog_level = module_level; + if (head) tail->next = info; + else head = info; + tail = info; + } + // Skip past this entry + vmodule = strchr(sep, ','); + if (vmodule == NULL) break; + vmodule++; // Skip past "," + } + if (head) { // Put them into the list at the head: + tail->next = vmodule_list; + vmodule_list = head; + } + inited_vmodule = true; +} + +// This can be called very early, so we use SpinLock and RAW_VLOG here. +int SetVLOGLevel(const char* module_pattern, int log_level) { + int result = FLAGS_v; + int const pattern_len = strlen(module_pattern); + bool found = false; + { + MutexLock l(&vmodule_lock); // protect whole read-modify-write + for (const VModuleInfo* info = vmodule_list; + info != NULL; info = info->next) { + if (info->module_pattern == module_pattern) { + if (!found) { + result = info->vlog_level; + found = true; + } + info->vlog_level = log_level; + } else if (!found && + SafeFNMatch_(info->module_pattern.c_str(), + info->module_pattern.size(), + module_pattern, pattern_len)) { + result = info->vlog_level; + found = true; + } + } + if (!found) { + VModuleInfo* info = new VModuleInfo; + info->module_pattern = module_pattern; + info->vlog_level = log_level; + info->next = vmodule_list; + vmodule_list = info; + } + } + RAW_VLOG(1, "Set VLOG level for \"%s\" to %d", module_pattern, log_level); + return result; +} + +// NOTE: Individual VLOG statements cache the integer log level pointers. +// NOTE: This function must not allocate memory or require any locks. +bool InitVLOG3__(int32** site_flag, int32* site_default, + const char* fname, int32 verbose_level) { + MutexLock l(&vmodule_lock); + bool read_vmodule_flag = inited_vmodule; + if (!read_vmodule_flag) { + VLOG2Initializer(); + } + + // protect the errno global in case someone writes: + // VLOG(..) << "The last error was " << strerror(errno) + int old_errno = errno; + + // site_default normally points to FLAGS_v + int32* site_flag_value = site_default; + + // Get basename for file + const char* base = strrchr(fname, '/'); + base = base ? (base+1) : fname; + const char* base_end = strchr(base, '.'); + size_t base_length = base_end ? size_t(base_end - base) : strlen(base); + + // Trim out trailing "-inl" if any + if (base_length >= 4 && (memcmp(base+base_length-4, "-inl", 4) == 0)) { + base_length -= 4; + } + + // TODO: Trim out _unittest suffix? Perhaps it is better to have + // the extra control and just leave it there. + + // find target in vector of modules, replace site_flag_value with + // a module-specific verbose level, if any. + for (const VModuleInfo* info = vmodule_list; + info != NULL; info = info->next) { + if (SafeFNMatch_(info->module_pattern.c_str(), info->module_pattern.size(), + base, base_length)) { + site_flag_value = &info->vlog_level; + // value at info->vlog_level is now what controls + // the VLOG at the caller site forever + break; + } + } + + // Cache the vlog value pointer if --vmodule flag has been parsed. + ANNOTATE_BENIGN_RACE(site_flag, + "*site_flag may be written by several threads," + " but the value will be the same"); + if (read_vmodule_flag) *site_flag = site_flag_value; + + // restore the errno in case something recoverable went wrong during + // the initialization of the VLOG mechanism (see above note "protect the..") + errno = old_errno; + return *site_flag_value >= verbose_level; +} + +_END_GOOGLE_NAMESPACE_ diff --git a/third_party/glog/src/windows/config.h b/third_party/glog/src/windows/config.h new file mode 100755 index 000000000000..2d23fb08159d --- /dev/null +++ b/third_party/glog/src/windows/config.h @@ -0,0 +1,21 @@ +/* src/config.h.in. Generated from configure.ac by autoheader. */ + +/* Namespace for Google classes */ +#define GOOGLE_NAMESPACE google + +/* Stops putting the code inside the Google namespace */ +#define _END_GOOGLE_NAMESPACE_ } + +/* Puts following code inside the Google namespace */ +#define _START_GOOGLE_NAMESPACE_ namespace google { + +/* Always the empty-string on non-windows systems. On windows, should be + "__declspec(dllexport)". This way, when we compile the dll, we export our + functions/classes. It's safe to define this here because config.h is only + used internally, to compile the DLL, and every DLL source file #includes + "config.h" before anything else. */ +#ifndef GOOGLE_GLOG_DLL_DECL +# define GOOGLE_GLOG_IS_A_DLL 1 /* not set if you're statically linking */ +# define GOOGLE_GLOG_DLL_DECL __declspec(dllexport) +# define GOOGLE_GLOG_DLL_DECL_FOR_UNITTESTS __declspec(dllimport) +#endif diff --git a/third_party/glog/src/windows/dirent.h b/third_party/glog/src/windows/dirent.h new file mode 100644 index 000000000000..f7a46dafcbf1 --- /dev/null +++ b/third_party/glog/src/windows/dirent.h @@ -0,0 +1,1160 @@ +/* + * Dirent interface for Microsoft Visual Studio + * + * Copyright (C) 1998-2019 Toni Ronkko + * This file is part of dirent. Dirent may be freely distributed + * under the MIT license. For all details and documentation, see + * https://github.com/tronkko/dirent + */ +#ifndef DIRENT_H +#define DIRENT_H + +/* Hide warnings about unreferenced local functions */ +#if defined(__clang__) +# pragma clang diagnostic ignored "-Wunused-function" +#elif defined(_MSC_VER) +# pragma warning(disable:4505) +#elif defined(__GNUC__) +# pragma GCC diagnostic ignored "-Wunused-function" +#endif + +/* + * Include windows.h without Windows Sockets 1.1 to prevent conflicts with + * Windows Sockets 2.0. + */ +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif +#include <windows.h> + +#include <stdio.h> +#include <stdarg.h> +#include <wchar.h> +#include <string.h> +#include <stdlib.h> +#include <malloc.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> + +/* Indicates that d_type field is available in dirent structure */ +#define _DIRENT_HAVE_D_TYPE + +/* Indicates that d_namlen field is available in dirent structure */ +#define _DIRENT_HAVE_D_NAMLEN + +/* Entries missing from MSVC 6.0 */ +#if !defined(FILE_ATTRIBUTE_DEVICE) +# define FILE_ATTRIBUTE_DEVICE 0x40 +#endif + +/* File type and permission flags for stat(), general mask */ +#if !defined(S_IFMT) +# define S_IFMT _S_IFMT +#endif + +/* Directory bit */ +#if !defined(S_IFDIR) +# define S_IFDIR _S_IFDIR +#endif + +/* Character device bit */ +#if !defined(S_IFCHR) +# define S_IFCHR _S_IFCHR +#endif + +/* Pipe bit */ +#if !defined(S_IFFIFO) +# define S_IFFIFO _S_IFFIFO +#endif + +/* Regular file bit */ +#if !defined(S_IFREG) +# define S_IFREG _S_IFREG +#endif + +/* Read permission */ +#if !defined(S_IREAD) +# define S_IREAD _S_IREAD +#endif + +/* Write permission */ +#if !defined(S_IWRITE) +# define S_IWRITE _S_IWRITE +#endif + +/* Execute permission */ +#if !defined(S_IEXEC) +# define S_IEXEC _S_IEXEC +#endif + +/* Pipe */ +#if !defined(S_IFIFO) +# define S_IFIFO _S_IFIFO +#endif + +/* Block device */ +#if !defined(S_IFBLK) +# define S_IFBLK 0 +#endif + +/* Link */ +#if !defined(S_IFLNK) +# define S_IFLNK 0 +#endif + +/* Socket */ +#if !defined(S_IFSOCK) +# define S_IFSOCK 0 +#endif + +/* Read user permission */ +#if !defined(S_IRUSR) +# define S_IRUSR S_IREAD +#endif + +/* Write user permission */ +#if !defined(S_IWUSR) +# define S_IWUSR S_IWRITE +#endif + +/* Execute user permission */ +#if !defined(S_IXUSR) +# define S_IXUSR 0 +#endif + +/* Read group permission */ +#if !defined(S_IRGRP) +# define S_IRGRP 0 +#endif + +/* Write group permission */ +#if !defined(S_IWGRP) +# define S_IWGRP 0 +#endif + +/* Execute group permission */ +#if !defined(S_IXGRP) +# define S_IXGRP 0 +#endif + +/* Read others permission */ +#if !defined(S_IROTH) +# define S_IROTH 0 +#endif + +/* Write others permission */ +#if !defined(S_IWOTH) +# define S_IWOTH 0 +#endif + +/* Execute others permission */ +#if !defined(S_IXOTH) +# define S_IXOTH 0 +#endif + +/* Maximum length of file name */ +#if !defined(PATH_MAX) +# define PATH_MAX MAX_PATH +#endif +#if !defined(FILENAME_MAX) +# define FILENAME_MAX MAX_PATH +#endif +#if !defined(NAME_MAX) +# define NAME_MAX FILENAME_MAX +#endif + +/* File type flags for d_type */ +#define DT_UNKNOWN 0 +#define DT_REG S_IFREG +#define DT_DIR S_IFDIR +#define DT_FIFO S_IFIFO +#define DT_SOCK S_IFSOCK +#define DT_CHR S_IFCHR +#define DT_BLK S_IFBLK +#define DT_LNK S_IFLNK + +/* Macros for converting between st_mode and d_type */ +#define IFTODT(mode) ((mode) & S_IFMT) +#define DTTOIF(type) (type) + +/* + * File type macros. Note that block devices, sockets and links cannot be + * distinguished on Windows and the macros S_ISBLK, S_ISSOCK and S_ISLNK are + * only defined for compatibility. These macros should always return false + * on Windows. + */ +#if !defined(S_ISFIFO) +# define S_ISFIFO(mode) (((mode) & S_IFMT) == S_IFIFO) +#endif +#if !defined(S_ISDIR) +# define S_ISDIR(mode) (((mode) & S_IFMT) == S_IFDIR) +#endif +#if !defined(S_ISREG) +# define S_ISREG(mode) (((mode) & S_IFMT) == S_IFREG) +#endif +#if !defined(S_ISLNK) +# define S_ISLNK(mode) (((mode) & S_IFMT) == S_IFLNK) +#endif +#if !defined(S_ISSOCK) +# define S_ISSOCK(mode) (((mode) & S_IFMT) == S_IFSOCK) +#endif +#if !defined(S_ISCHR) +# define S_ISCHR(mode) (((mode) & S_IFMT) == S_IFCHR) +#endif +#if !defined(S_ISBLK) +# define S_ISBLK(mode) (((mode) & S_IFMT) == S_IFBLK) +#endif + +/* Return the exact length of the file name without zero terminator */ +#define _D_EXACT_NAMLEN(p) ((p)->d_namlen) + +/* Return the maximum size of a file name */ +#define _D_ALLOC_NAMLEN(p) ((PATH_MAX)+1) + + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Wide-character version */ +struct _wdirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + wchar_t d_name[PATH_MAX+1]; +}; +typedef struct _wdirent _wdirent; + +struct _WDIR { + /* Current directory entry */ + struct _wdirent ent; + + /* Private file data */ + WIN32_FIND_DATAW data; + + /* True if data is valid */ + int cached; + + /* Win32 search handle */ + HANDLE handle; + + /* Initial directory name */ + wchar_t *patt; +}; +typedef struct _WDIR _WDIR; + +/* Multi-byte character version */ +struct dirent { + /* Always zero */ + long d_ino; + + /* File position within stream */ + long d_off; + + /* Structure size */ + unsigned short d_reclen; + + /* Length of name without \0 */ + size_t d_namlen; + + /* File type */ + int d_type; + + /* File name */ + char d_name[PATH_MAX+1]; +}; +typedef struct dirent dirent; + +struct DIR { + struct dirent ent; + struct _WDIR *wdirp; +}; +typedef struct DIR DIR; + + +/* Dirent functions */ +static DIR *opendir (const char *dirname); +static _WDIR *_wopendir (const wchar_t *dirname); + +static struct dirent *readdir (DIR *dirp); +static struct _wdirent *_wreaddir (_WDIR *dirp); + +static int readdir_r( + DIR *dirp, struct dirent *entry, struct dirent **result); +static int _wreaddir_r( + _WDIR *dirp, struct _wdirent *entry, struct _wdirent **result); + +static int closedir (DIR *dirp); +static int _wclosedir (_WDIR *dirp); + +static void rewinddir (DIR* dirp); +static void _wrewinddir (_WDIR* dirp); + +static int scandir (const char *dirname, struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)); + +static int alphasort (const struct dirent **a, const struct dirent **b); + +static int versionsort (const struct dirent **a, const struct dirent **b); + + +/* For compatibility with Symbian */ +#define wdirent _wdirent +#define WDIR _WDIR +#define wopendir _wopendir +#define wreaddir _wreaddir +#define wclosedir _wclosedir +#define wrewinddir _wrewinddir + + +/* Internal utility functions */ +static WIN32_FIND_DATAW *dirent_first (_WDIR *dirp); +static WIN32_FIND_DATAW *dirent_next (_WDIR *dirp); + +static int dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count); + +static int dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, + const wchar_t *wcstr, + size_t count); + +static void dirent_set_errno (int error); + + +/* + * Open directory stream DIRNAME for read and return a pointer to the + * internal working area that is used to retrieve individual directory + * entries. + */ +static _WDIR* +_wopendir( + const wchar_t *dirname) +{ + _WDIR *dirp; + DWORD n; + wchar_t *p; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate new _WDIR structure */ + dirp = (_WDIR*) malloc (sizeof (struct _WDIR)); + if (!dirp) { + return NULL; + } + + /* Reset _WDIR structure */ + dirp->handle = INVALID_HANDLE_VALUE; + dirp->patt = NULL; + dirp->cached = 0; + + /* + * Compute the length of full path plus zero terminator + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW (dirname, 0, NULL, NULL); +#else + /* WinRT */ + n = wcslen (dirname); +#endif + + /* Allocate room for absolute directory name and search pattern */ + dirp->patt = (wchar_t*) malloc (sizeof (wchar_t) * n + 16); + if (dirp->patt == NULL) { + goto exit_closedir; + } + + /* + * Convert relative directory name to an absolute one. This + * allows rewinddir() to function correctly even when current + * working directory is changed between opendir() and rewinddir(). + * + * Note that on WinRT there's no way to convert relative paths + * into absolute paths, so just assume it is an absolute path. + */ +#if WINAPI_FAMILY_PARTITION(WINAPI_PARTITION_DESKTOP) + /* Desktop */ + n = GetFullPathNameW (dirname, n, dirp->patt, NULL); + if (n <= 0) { + goto exit_closedir; + } +#else + /* WinRT */ + wcsncpy_s (dirp->patt, n+1, dirname, n); +#endif + + /* Append search pattern \* to the directory name */ + p = dirp->patt + n; + switch (p[-1]) { + case '\\': + case '/': + case ':': + /* Directory ends in path separator, e.g. c:\temp\ */ + /*NOP*/; + break; + + default: + /* Directory name doesn't end in path separator */ + *p++ = '\\'; + } + *p++ = '*'; + *p = '\0'; + + /* Open directory stream and retrieve the first entry */ + if (!dirent_first (dirp)) { + goto exit_closedir; + } + + /* Success */ + return dirp; + + /* Failure */ +exit_closedir: + _wclosedir (dirp); + return NULL; +} + +/* + * Read next directory entry. + * + * Returns pointer to static directory entry which may be overwritten by + * subsequent calls to _wreaddir(). + */ +static struct _wdirent* +_wreaddir( + _WDIR *dirp) +{ + struct _wdirent *entry; + + /* + * Read directory entry to buffer. We can safely ignore the return value + * as entry will be set to NULL in case of error. + */ + (void) _wreaddir_r (dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry. + * + * Returns zero on success. If end of directory stream is reached, then sets + * result to NULL and returns zero. + */ +static int +_wreaddir_r( + _WDIR *dirp, + struct _wdirent *entry, + struct _wdirent **result) +{ + WIN32_FIND_DATAW *datap; + + /* Read next directory entry */ + datap = dirent_next (dirp); + if (datap) { + size_t n; + DWORD attr; + + /* + * Copy file name as wide-character string. If the file name is too + * long to fit in to the destination buffer, then truncate file name + * to PATH_MAX characters and zero-terminate the buffer. + */ + n = 0; + while (n < PATH_MAX && datap->cFileName[n] != 0) { + entry->d_name[n] = datap->cFileName[n]; + n++; + } + entry->d_name[n] = 0; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n; + + /* File type */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entry->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof (struct _wdirent); + + /* Set result address */ + *result = entry; + + } else { + + /* Return NULL to indicate end of directory */ + *result = NULL; + + } + + return /*OK*/0; +} + +/* + * Close directory stream opened by opendir() function. This invalidates the + * DIR structure as well as any directory entry read previously by + * _wreaddir(). + */ +static int +_wclosedir( + _WDIR *dirp) +{ + int ok; + if (dirp) { + + /* Release search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Release search pattern */ + free (dirp->patt); + + /* Release directory structure */ + free (dirp); + ok = /*success*/0; + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream such that _wreaddir() returns the very first + * file name again. + */ +static void +_wrewinddir( + _WDIR* dirp) +{ + if (dirp) { + /* Release existing search handle */ + if (dirp->handle != INVALID_HANDLE_VALUE) { + FindClose (dirp->handle); + } + + /* Open new search handle */ + dirent_first (dirp); + } +} + +/* Get first directory entry (internal) */ +static WIN32_FIND_DATAW* +dirent_first( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *datap; + DWORD error; + + /* Open directory and retrieve the first entry */ + dirp->handle = FindFirstFileExW( + dirp->patt, FindExInfoStandard, &dirp->data, + FindExSearchNameMatch, NULL, 0); + if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* a directory entry is now waiting in memory */ + datap = &dirp->data; + dirp->cached = 1; + + } else { + + /* Failed to open directory: no directory entry in memory */ + dirp->cached = 0; + datap = NULL; + + /* Set error code */ + error = GetLastError (); + switch (error) { + case ERROR_ACCESS_DENIED: + /* No read access to directory */ + dirent_set_errno (EACCES); + break; + + case ERROR_DIRECTORY: + /* Directory name is invalid */ + dirent_set_errno (ENOTDIR); + break; + + case ERROR_PATH_NOT_FOUND: + default: + /* Cannot find the file */ + dirent_set_errno (ENOENT); + } + + } + return datap; +} + +/* + * Get next directory entry (internal). + * + * Returns + */ +static WIN32_FIND_DATAW* +dirent_next( + _WDIR *dirp) +{ + WIN32_FIND_DATAW *p; + + /* Get next directory entry */ + if (dirp->cached != 0) { + + /* A valid directory entry already in memory */ + p = &dirp->data; + dirp->cached = 0; + + } else if (dirp->handle != INVALID_HANDLE_VALUE) { + + /* Get the next directory entry from stream */ + if (FindNextFileW (dirp->handle, &dirp->data) != FALSE) { + /* Got a file */ + p = &dirp->data; + } else { + /* The very last entry has been processed or an error occurred */ + FindClose (dirp->handle); + dirp->handle = INVALID_HANDLE_VALUE; + p = NULL; + } + + } else { + + /* End of directory stream reached */ + p = NULL; + + } + + return p; +} + +/* + * Open directory stream using plain old C-string. + */ +static DIR* +opendir( + const char *dirname) +{ + struct DIR *dirp; + + /* Must have directory name */ + if (dirname == NULL || dirname[0] == '\0') { + dirent_set_errno (ENOENT); + return NULL; + } + + /* Allocate memory for DIR structure */ + dirp = (DIR*) malloc (sizeof (struct DIR)); + if (!dirp) { + return NULL; + } + { + int error; + wchar_t wname[PATH_MAX + 1]; + size_t n; + + /* Convert directory name to wide-character string */ + error = dirent_mbstowcs_s( + &n, wname, PATH_MAX + 1, dirname, PATH_MAX + 1); + if (error) { + /* + * Cannot convert file name to wide-character string. This + * occurs if the string contains invalid multi-byte sequences or + * the output buffer is too small to contain the resulting + * string. + */ + goto exit_free; + } + + + /* Open directory stream using wide-character name */ + dirp->wdirp = _wopendir (wname); + if (!dirp->wdirp) { + goto exit_free; + } + + } + + /* Success */ + return dirp; + + /* Failure */ +exit_free: + free (dirp); + return NULL; +} + +/* + * Read next directory entry. + */ +static struct dirent* +readdir( + DIR *dirp) +{ + struct dirent *entry; + + /* + * Read directory entry to buffer. We can safely ignore the return value + * as entry will be set to NULL in case of error. + */ + (void) readdir_r (dirp, &dirp->ent, &entry); + + /* Return pointer to statically allocated directory entry */ + return entry; +} + +/* + * Read next directory entry into called-allocated buffer. + * + * Returns zero on success. If the end of directory stream is reached, then + * sets result to NULL and returns zero. + */ +static int +readdir_r( + DIR *dirp, + struct dirent *entry, + struct dirent **result) +{ + WIN32_FIND_DATAW *datap; + + /* Read next directory entry */ + datap = dirent_next (dirp->wdirp); + if (datap) { + size_t n; + int error; + + /* Attempt to convert file name to multi-byte string */ + error = dirent_wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, datap->cFileName, PATH_MAX + 1); + + /* + * If the file name cannot be represented by a multi-byte string, + * then attempt to use old 8+3 file name. This allows traditional + * Unix-code to access some file names despite of unicode + * characters, although file names may seem unfamiliar to the user. + * + * Be ware that the code below cannot come up with a short file + * name unless the file system provides one. At least + * VirtualBox shared folders fail to do this. + */ + if (error && datap->cAlternateFileName[0] != '\0') { + error = dirent_wcstombs_s( + &n, entry->d_name, PATH_MAX + 1, + datap->cAlternateFileName, PATH_MAX + 1); + } + + if (!error) { + DWORD attr; + + /* Length of file name excluding zero terminator */ + entry->d_namlen = n - 1; + + /* File attributes */ + attr = datap->dwFileAttributes; + if ((attr & FILE_ATTRIBUTE_DEVICE) != 0) { + entry->d_type = DT_CHR; + } else if ((attr & FILE_ATTRIBUTE_DIRECTORY) != 0) { + entry->d_type = DT_DIR; + } else { + entry->d_type = DT_REG; + } + + /* Reset dummy fields */ + entry->d_ino = 0; + entry->d_off = 0; + entry->d_reclen = sizeof (struct dirent); + + } else { + + /* + * Cannot convert file name to multi-byte string so construct + * an erroneous directory entry and return that. Note that + * we cannot return NULL as that would stop the processing + * of directory entries completely. + */ + entry->d_name[0] = '?'; + entry->d_name[1] = '\0'; + entry->d_namlen = 1; + entry->d_type = DT_UNKNOWN; + entry->d_ino = 0; + entry->d_off = -1; + entry->d_reclen = 0; + + } + + /* Return pointer to directory entry */ + *result = entry; + + } else { + + /* No more directory entries */ + *result = NULL; + + } + + return /*OK*/0; +} + +/* + * Close directory stream. + */ +static int +closedir( + DIR *dirp) +{ + int ok; + if (dirp) { + + /* Close wide-character directory stream */ + ok = _wclosedir (dirp->wdirp); + dirp->wdirp = NULL; + + /* Release multi-byte character version */ + free (dirp); + + } else { + + /* Invalid directory stream */ + dirent_set_errno (EBADF); + ok = /*failure*/-1; + + } + return ok; +} + +/* + * Rewind directory stream to beginning. + */ +static void +rewinddir( + DIR* dirp) +{ + /* Rewind wide-character string directory stream */ + _wrewinddir (dirp->wdirp); +} + +/* + * Scan directory for entries. + */ +static int +scandir( + const char *dirname, + struct dirent ***namelist, + int (*filter)(const struct dirent*), + int (*compare)(const struct dirent**, const struct dirent**)) +{ + struct dirent **files = NULL; + size_t size = 0; + size_t allocated = 0; + const size_t init_size = 1; + DIR *dir = NULL; + struct dirent *entry; + struct dirent *tmp = NULL; + size_t i; + int result = 0; + + /* Open directory stream */ + dir = opendir (dirname); + if (dir) { + + /* Read directory entries to memory */ + while (1) { + + /* Enlarge pointer table to make room for another pointer */ + if (size >= allocated) { + void *p; + size_t num_entries; + + /* Compute number of entries in the enlarged pointer table */ + if (size < init_size) { + /* Allocate initial pointer table */ + num_entries = init_size; + } else { + /* Double the size */ + num_entries = size * 2; + } + + /* Allocate first pointer table or enlarge existing table */ + p = realloc (files, sizeof (void*) * num_entries); + if (p != NULL) { + /* Got the memory */ + files = (dirent**) p; + allocated = num_entries; + } else { + /* Out of memory */ + result = -1; + break; + } + + } + + /* Allocate room for temporary directory entry */ + if (tmp == NULL) { + tmp = (struct dirent*) malloc (sizeof (struct dirent)); + if (tmp == NULL) { + /* Cannot allocate temporary directory entry */ + result = -1; + break; + } + } + + /* Read directory entry to temporary area */ + if (readdir_r (dir, tmp, &entry) == /*OK*/0) { + + /* Did we get an entry? */ + if (entry != NULL) { + int pass; + + /* Determine whether to include the entry in result */ + if (filter) { + /* Let the filter function decide */ + pass = filter (tmp); + } else { + /* No filter function, include everything */ + pass = 1; + } + + if (pass) { + /* Store the temporary entry to pointer table */ + files[size++] = tmp; + tmp = NULL; + + /* Keep up with the number of files */ + result++; + } + + } else { + + /* + * End of directory stream reached => sort entries and + * exit. + */ + qsort (files, size, sizeof (void*), + (int (*) (const void*, const void*)) compare); + break; + + } + + } else { + /* Error reading directory entry */ + result = /*Error*/ -1; + break; + } + + } + + } else { + /* Cannot open directory */ + result = /*Error*/ -1; + } + + /* Release temporary directory entry */ + free (tmp); + + /* Release allocated memory on error */ + if (result < 0) { + for (i = 0; i < size; i++) { + free (files[i]); + } + free (files); + files = NULL; + } + + /* Close directory stream */ + if (dir) { + closedir (dir); + } + + /* Pass pointer table to caller */ + if (namelist) { + *namelist = files; + } + return result; +} + +/* Alphabetical sorting */ +static int +alphasort( + const struct dirent **a, const struct dirent **b) +{ + return strcoll ((*a)->d_name, (*b)->d_name); +} + +/* Sort versions */ +static int +versionsort( + const struct dirent **a, const struct dirent **b) +{ + /* FIXME: implement strverscmp and use that */ + return alphasort (a, b); +} + +/* Convert multi-byte string to wide character string */ +static int +dirent_mbstowcs_s( + size_t *pReturnValue, + wchar_t *wcstr, + size_t sizeInWords, + const char *mbstr, + size_t count) +{ + int error; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = mbstowcs_s (pReturnValue, wcstr, sizeInWords, mbstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to wide-character string (or count characters) */ + n = mbstowcs (wcstr, mbstr, sizeInWords); + if (!wcstr || n < count) { + + /* Zero-terminate output buffer */ + if (wcstr && sizeInWords) { + if (n >= sizeInWords) { + n = sizeInWords - 1; + } + wcstr[n] = 0; + } + + /* Length of resulting multi-byte string WITH zero terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Could not convert string */ + error = 1; + + } + +#endif + return error; +} + +/* Convert wide-character string to multi-byte string */ +static int +dirent_wcstombs_s( + size_t *pReturnValue, + char *mbstr, + size_t sizeInBytes, /* max size of mbstr */ + const wchar_t *wcstr, + size_t count) +{ + int error; + +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 or later */ + error = wcstombs_s (pReturnValue, mbstr, sizeInBytes, wcstr, count); + +#else + + /* Older Visual Studio or non-Microsoft compiler */ + size_t n; + + /* Convert to multi-byte string (or count the number of bytes needed) */ + n = wcstombs (mbstr, wcstr, sizeInBytes); + if (!mbstr || n < count) { + + /* Zero-terminate output buffer */ + if (mbstr && sizeInBytes) { + if (n >= sizeInBytes) { + n = sizeInBytes - 1; + } + mbstr[n] = '\0'; + } + + /* Length of resulting multi-bytes string WITH zero-terminator */ + if (pReturnValue) { + *pReturnValue = n + 1; + } + + /* Success */ + error = 0; + + } else { + + /* Cannot convert string */ + error = 1; + + } + +#endif + return error; +} + +/* Set errno variable */ +static void +dirent_set_errno( + int error) +{ +#if defined(_MSC_VER) && _MSC_VER >= 1400 + + /* Microsoft Visual Studio 2005 and later */ + _set_errno (error); + +#else + + /* Non-Microsoft compiler or older Microsoft compiler */ + errno = error; + +#endif +} + + +#ifdef __cplusplus +} +#endif +#endif /*DIRENT_H*/ diff --git a/third_party/glog/src/windows/glog/log_severity.h b/third_party/glog/src/windows/glog/log_severity.h new file mode 100644 index 000000000000..22a4191ab8bb --- /dev/null +++ b/third_party/glog/src/windows/glog/log_severity.h @@ -0,0 +1,96 @@ +// This file is automatically generated from src/glog/log_severity.h +// using src/windows/preprocess.sh. +// DO NOT EDIT! + +// Copyright (c) 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef BASE_LOG_SEVERITY_H__ +#define BASE_LOG_SEVERITY_H__ + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +// Variables of type LogSeverity are widely taken to lie in the range +// [0, NUM_SEVERITIES-1]. Be careful to preserve this assumption if +// you ever need to change their values or add a new severity. +typedef int LogSeverity; + +const int GLOG_INFO = 0, GLOG_WARNING = 1, GLOG_ERROR = 2, GLOG_FATAL = 3, + NUM_SEVERITIES = 4; +#ifndef GLOG_NO_ABBREVIATED_SEVERITIES +# ifdef ERROR +# error ERROR macro is defined. Define GLOG_NO_ABBREVIATED_SEVERITIES before including logging.h. See the document for detail. +# endif +const int INFO = GLOG_INFO, WARNING = GLOG_WARNING, + ERROR = GLOG_ERROR, FATAL = GLOG_FATAL; +#endif + +// DFATAL is FATAL in debug mode, ERROR in normal mode +#ifdef NDEBUG +#define DFATAL_LEVEL ERROR +#else +#define DFATAL_LEVEL FATAL +#endif + +extern GOOGLE_GLOG_DLL_DECL const char* const LogSeverityNames[NUM_SEVERITIES]; + +// NDEBUG usage helpers related to (RAW_)DCHECK: +// +// DEBUG_MODE is for small !NDEBUG uses like +// if (DEBUG_MODE) foo.CheckThatFoo(); +// instead of substantially more verbose +// #ifndef NDEBUG +// foo.CheckThatFoo(); +// #endif +// +// IF_DEBUG_MODE is for small !NDEBUG uses like +// IF_DEBUG_MODE( string error; ) +// DCHECK(Foo(&error)) << error; +// instead of substantially more verbose +// #ifndef NDEBUG +// string error; +// DCHECK(Foo(&error)) << error; +// #endif +// +#ifdef NDEBUG +enum { DEBUG_MODE = 0 }; +#define IF_DEBUG_MODE(x) +#else +enum { DEBUG_MODE = 1 }; +#define IF_DEBUG_MODE(x) x +#endif + +#endif // BASE_LOG_SEVERITY_H__ diff --git a/third_party/glog/src/windows/glog/logging.h b/third_party/glog/src/windows/glog/logging.h new file mode 100755 index 000000000000..e111bf07c46c --- /dev/null +++ b/third_party/glog/src/windows/glog/logging.h @@ -0,0 +1,1690 @@ +// This file is automatically generated from src/glog/logging.h.in +// using src/windows/preprocess.sh. +// DO NOT EDIT! + +// Copyright (c) 1999, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Ray Sidney +// +// This file contains #include information about logging-related stuff. +// Pretty much everybody needs to #include this file so that they can +// log various happenings. +// +#ifndef _LOGGING_H_ +#define _LOGGING_H_ + +#include <errno.h> +#include <string.h> +#include <time.h> +#include <iosfwd> +#include <ostream> +#include <sstream> +#include <string> +#if 0 +# include <unistd.h> +#endif +#include <vector> + +#if defined(_MSC_VER) +#define GLOG_MSVC_PUSH_DISABLE_WARNING(n) __pragma(warning(push)) \ + __pragma(warning(disable:n)) +#define GLOG_MSVC_POP_WARNING() __pragma(warning(pop)) +#else +#define GLOG_MSVC_PUSH_DISABLE_WARNING(n) +#define GLOG_MSVC_POP_WARNING() +#endif + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +// We care a lot about number of bits things take up. Unfortunately, +// systems define their bit-specific ints in a lot of different ways. +// We use our own way, and have a typedef to get there. +// Note: these commands below may look like "#if 1" or "#if 0", but +// that's because they were constructed that way at ./configure time. +// Look at logging.h.in to see how they're calculated (based on your config). +#if 0 +#include <stdint.h> // the normal place uint16_t is defined +#endif +#if 0 +#include <sys/types.h> // the normal place u_int16_t is defined +#endif +#if 0 +#include <inttypes.h> // a third place for uint16_t or u_int16_t +#endif + +#if 0 +#include <gflags/gflags.h> +#endif + +namespace google { + +#if 0 // the C99 format +typedef int32_t int32; +typedef uint32_t uint32; +typedef int64_t int64; +typedef uint64_t uint64; +#elif 0 // the BSD format +typedef int32_t int32; +typedef u_int32_t uint32; +typedef int64_t int64; +typedef u_int64_t uint64; +#elif 1 // the windows (vc7) format +typedef __int32 int32; +typedef unsigned __int32 uint32; +typedef __int64 int64; +typedef unsigned __int64 uint64; +#else +#error Do not know how to define a 32-bit integer quantity on your system +#endif + +} + +// The global value of GOOGLE_STRIP_LOG. All the messages logged to +// LOG(XXX) with severity less than GOOGLE_STRIP_LOG will not be displayed. +// If it can be determined at compile time that the message will not be +// printed, the statement will be compiled out. +// +// Example: to strip out all INFO and WARNING messages, use the value +// of 2 below. To make an exception for WARNING messages from a single +// file, add "#define GOOGLE_STRIP_LOG 1" to that file _before_ including +// base/logging.h +#ifndef GOOGLE_STRIP_LOG +#define GOOGLE_STRIP_LOG 0 +#endif + +// GCC can be told that a certain branch is not likely to be taken (for +// instance, a CHECK failure), and use that information in static analysis. +// Giving it this information can help it optimize for the common case in +// the absence of better information (ie. -fprofile-arcs). +// +#ifndef GOOGLE_PREDICT_BRANCH_NOT_TAKEN +#if 0 +#define GOOGLE_PREDICT_BRANCH_NOT_TAKEN(x) (__builtin_expect(x, 0)) +#else +#define GOOGLE_PREDICT_BRANCH_NOT_TAKEN(x) x +#endif +#endif + +#ifndef GOOGLE_PREDICT_FALSE +#if 0 +#define GOOGLE_PREDICT_FALSE(x) (__builtin_expect(x, 0)) +#else +#define GOOGLE_PREDICT_FALSE(x) x +#endif +#endif + +#ifndef GOOGLE_PREDICT_TRUE +#if 0 +#define GOOGLE_PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) +#else +#define GOOGLE_PREDICT_TRUE(x) x +#endif +#endif + + +// Make a bunch of macros for logging. The way to log things is to stream +// things to LOG(<a particular severity level>). E.g., +// +// LOG(INFO) << "Found " << num_cookies << " cookies"; +// +// You can capture log messages in a string, rather than reporting them +// immediately: +// +// vector<string> errors; +// LOG_STRING(ERROR, &errors) << "Couldn't parse cookie #" << cookie_num; +// +// This pushes back the new error onto 'errors'; if given a NULL pointer, +// it reports the error via LOG(ERROR). +// +// You can also do conditional logging: +// +// LOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; +// +// You can also do occasional logging (log every n'th occurrence of an +// event): +// +// LOG_EVERY_N(INFO, 10) << "Got the " << google::COUNTER << "th cookie"; +// +// The above will cause log messages to be output on the 1st, 11th, 21st, ... +// times it is executed. Note that the special google::COUNTER value is used +// to identify which repetition is happening. +// +// You can also do occasional conditional logging (log every n'th +// occurrence of an event, when condition is satisfied): +// +// LOG_IF_EVERY_N(INFO, (size > 1024), 10) << "Got the " << google::COUNTER +// << "th big cookie"; +// +// You can log messages the first N times your code executes a line. E.g. +// +// LOG_FIRST_N(INFO, 20) << "Got the " << google::COUNTER << "th cookie"; +// +// Outputs log messages for the first 20 times it is executed. +// +// Analogous SYSLOG, SYSLOG_IF, and SYSLOG_EVERY_N macros are available. +// These log to syslog as well as to the normal logs. If you use these at +// all, you need to be aware that syslog can drastically reduce performance, +// especially if it is configured for remote logging! Don't use these +// unless you fully understand this and have a concrete need to use them. +// Even then, try to minimize your use of them. +// +// There are also "debug mode" logging macros like the ones above: +// +// DLOG(INFO) << "Found cookies"; +// +// DLOG_IF(INFO, num_cookies > 10) << "Got lots of cookies"; +// +// DLOG_EVERY_N(INFO, 10) << "Got the " << google::COUNTER << "th cookie"; +// +// All "debug mode" logging is compiled away to nothing for non-debug mode +// compiles. +// +// We also have +// +// LOG_ASSERT(assertion); +// DLOG_ASSERT(assertion); +// +// which is syntactic sugar for {,D}LOG_IF(FATAL, assert fails) << assertion; +// +// There are "verbose level" logging macros. They look like +// +// VLOG(1) << "I'm printed when you run the program with --v=1 or more"; +// VLOG(2) << "I'm printed when you run the program with --v=2 or more"; +// +// These always log at the INFO log level (when they log at all). +// The verbose logging can also be turned on module-by-module. For instance, +// --vmodule=mapreduce=2,file=1,gfs*=3 --v=0 +// will cause: +// a. VLOG(2) and lower messages to be printed from mapreduce.{h,cc} +// b. VLOG(1) and lower messages to be printed from file.{h,cc} +// c. VLOG(3) and lower messages to be printed from files prefixed with "gfs" +// d. VLOG(0) and lower messages to be printed from elsewhere +// +// The wildcarding functionality shown by (c) supports both '*' (match +// 0 or more characters) and '?' (match any single character) wildcards. +// +// There's also VLOG_IS_ON(n) "verbose level" condition macro. To be used as +// +// if (VLOG_IS_ON(2)) { +// // do some logging preparation and logging +// // that can't be accomplished with just VLOG(2) << ...; +// } +// +// There are also VLOG_IF, VLOG_EVERY_N and VLOG_IF_EVERY_N "verbose level" +// condition macros for sample cases, when some extra computation and +// preparation for logs is not needed. +// VLOG_IF(1, (size > 1024)) +// << "I'm printed when size is more than 1024 and when you run the " +// "program with --v=1 or more"; +// VLOG_EVERY_N(1, 10) +// << "I'm printed every 10th occurrence, and when you run the program " +// "with --v=1 or more. Present occurence is " << google::COUNTER; +// VLOG_IF_EVERY_N(1, (size > 1024), 10) +// << "I'm printed on every 10th occurence of case when size is more " +// " than 1024, when you run the program with --v=1 or more. "; +// "Present occurence is " << google::COUNTER; +// +// The supported severity levels for macros that allow you to specify one +// are (in increasing order of severity) INFO, WARNING, ERROR, and FATAL. +// Note that messages of a given severity are logged not only in the +// logfile for that severity, but also in all logfiles of lower severity. +// E.g., a message of severity FATAL will be logged to the logfiles of +// severity FATAL, ERROR, WARNING, and INFO. +// +// There is also the special severity of DFATAL, which logs FATAL in +// debug mode, ERROR in normal mode. +// +// Very important: logging a message at the FATAL severity level causes +// the program to terminate (after the message is logged). +// +// Unless otherwise specified, logs will be written to the filename +// "<program name>.<hostname>.<user name>.log.<severity level>.", followed +// by the date, time, and pid (you can't prevent the date, time, and pid +// from being in the filename). +// +// The logging code takes two flags: +// --v=# set the verbose level +// --logtostderr log all the messages to stderr instead of to logfiles + +// LOG LINE PREFIX FORMAT +// +// Log lines have this form: +// +// Lyyyymmdd hh:mm:ss.uuuuuu threadid file:line] msg... +// +// where the fields are defined as follows: +// +// L A single character, representing the log level +// (eg 'I' for INFO) +// yyyy The year +// mm The month (zero padded; ie May is '05') +// dd The day (zero padded) +// hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds +// threadid The space-padded thread ID as returned by GetTID() +// (this matches the PID on Linux) +// file The file name +// line The line number +// msg The user-supplied message +// +// Example: +// +// I1103 11:57:31.739339 24395 google.cc:2341] Command line: ./some_prog +// I1103 11:57:31.739403 24395 google.cc:2342] Process id 24395 +// +// NOTE: although the microseconds are useful for comparing events on +// a single machine, clocks on different machines may not be well +// synchronized. Hence, use caution when comparing the low bits of +// timestamps from different machines. + +#ifndef DECLARE_VARIABLE +#define MUST_UNDEF_GFLAGS_DECLARE_MACROS +#define DECLARE_VARIABLE(type, shorttype, name, tn) \ + namespace fL##shorttype { \ + extern GOOGLE_GLOG_DLL_DECL type FLAGS_##name; \ + } \ + using fL##shorttype::FLAGS_##name + +// bool specialization +#define DECLARE_bool(name) \ + DECLARE_VARIABLE(bool, B, name, bool) + +// int32 specialization +#define DECLARE_int32(name) \ + DECLARE_VARIABLE(google::int32, I, name, int32) + +// Special case for string, because we have to specify the namespace +// std::string, which doesn't play nicely with our FLAG__namespace hackery. +#define DECLARE_string(name) \ + namespace fLS { \ + extern GOOGLE_GLOG_DLL_DECL std::string& FLAGS_##name; \ + } \ + using fLS::FLAGS_##name +#endif + +// Set whether log messages go to stderr instead of logfiles +DECLARE_bool(logtostderr); + +// Set whether log messages go to stderr in addition to logfiles. +DECLARE_bool(alsologtostderr); + +// Set color messages logged to stderr (if supported by terminal). +DECLARE_bool(colorlogtostderr); + +// Log messages at a level >= this flag are automatically sent to +// stderr in addition to log files. +DECLARE_int32(stderrthreshold); + +// Set whether the log prefix should be prepended to each line of output. +DECLARE_bool(log_prefix); + +// Log messages at a level <= this flag are buffered. +// Log messages at a higher level are flushed immediately. +DECLARE_int32(logbuflevel); + +// Sets the maximum number of seconds which logs may be buffered for. +DECLARE_int32(logbufsecs); + +// Log suppression level: messages logged at a lower level than this +// are suppressed. +DECLARE_int32(minloglevel); + +// If specified, logfiles are written into this directory instead of the +// default logging directory. +DECLARE_string(log_dir); + +// Set the log file mode. +DECLARE_int32(logfile_mode); + +// Sets the path of the directory into which to put additional links +// to the log files. +DECLARE_string(log_link); + +DECLARE_int32(v); // in vlog_is_on.cc + +// Sets the maximum log file size (in MB). +DECLARE_int32(max_log_size); + +// Sets whether to avoid logging to the disk if the disk is full. +DECLARE_bool(stop_logging_if_full_disk); + +#ifdef MUST_UNDEF_GFLAGS_DECLARE_MACROS +#undef MUST_UNDEF_GFLAGS_DECLARE_MACROS +#undef DECLARE_VARIABLE +#undef DECLARE_bool +#undef DECLARE_int32 +#undef DECLARE_string +#endif + +// Log messages below the GOOGLE_STRIP_LOG level will be compiled away for +// security reasons. See LOG(severtiy) below. + +// A few definitions of macros that don't generate much code. Since +// LOG(INFO) and its ilk are used all over our code, it's +// better to have compact code for these operations. + +#if GOOGLE_STRIP_LOG == 0 +#define COMPACT_GOOGLE_LOG_INFO google::LogMessage( \ + __FILE__, __LINE__) +#define LOG_TO_STRING_INFO(message) google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_INFO, message) +#else +#define COMPACT_GOOGLE_LOG_INFO google::NullStream() +#define LOG_TO_STRING_INFO(message) google::NullStream() +#endif + +#if GOOGLE_STRIP_LOG <= 1 +#define COMPACT_GOOGLE_LOG_WARNING google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_WARNING) +#define LOG_TO_STRING_WARNING(message) google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_WARNING, message) +#else +#define COMPACT_GOOGLE_LOG_WARNING google::NullStream() +#define LOG_TO_STRING_WARNING(message) google::NullStream() +#endif + +#if GOOGLE_STRIP_LOG <= 2 +#define COMPACT_GOOGLE_LOG_ERROR google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_ERROR) +#define LOG_TO_STRING_ERROR(message) google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_ERROR, message) +#else +#define COMPACT_GOOGLE_LOG_ERROR google::NullStream() +#define LOG_TO_STRING_ERROR(message) google::NullStream() +#endif + +#if GOOGLE_STRIP_LOG <= 3 +#define COMPACT_GOOGLE_LOG_FATAL google::LogMessageFatal( \ + __FILE__, __LINE__) +#define LOG_TO_STRING_FATAL(message) google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_FATAL, message) +#else +#define COMPACT_GOOGLE_LOG_FATAL google::NullStreamFatal() +#define LOG_TO_STRING_FATAL(message) google::NullStreamFatal() +#endif + +#if defined(NDEBUG) && !defined(DCHECK_ALWAYS_ON) +#define DCHECK_IS_ON() 0 +#else +#define DCHECK_IS_ON() 1 +#endif + +// For DFATAL, we want to use LogMessage (as opposed to +// LogMessageFatal), to be consistent with the original behavior. +#if !DCHECK_IS_ON() +#define COMPACT_GOOGLE_LOG_DFATAL COMPACT_GOOGLE_LOG_ERROR +#elif GOOGLE_STRIP_LOG <= 3 +#define COMPACT_GOOGLE_LOG_DFATAL google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_FATAL) +#else +#define COMPACT_GOOGLE_LOG_DFATAL google::NullStreamFatal() +#endif + +#define GOOGLE_LOG_INFO(counter) google::LogMessage(__FILE__, __LINE__, google::GLOG_INFO, counter, &google::LogMessage::SendToLog) +#define SYSLOG_INFO(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_INFO, counter, \ + &google::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_WARNING(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_WARNING, counter, \ + &google::LogMessage::SendToLog) +#define SYSLOG_WARNING(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_WARNING, counter, \ + &google::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_ERROR(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_ERROR, counter, \ + &google::LogMessage::SendToLog) +#define SYSLOG_ERROR(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_ERROR, counter, \ + &google::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_FATAL(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_FATAL, counter, \ + &google::LogMessage::SendToLog) +#define SYSLOG_FATAL(counter) \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_FATAL, counter, \ + &google::LogMessage::SendToSyslogAndLog) +#define GOOGLE_LOG_DFATAL(counter) \ + google::LogMessage(__FILE__, __LINE__, google::DFATAL_LEVEL, counter, \ + &google::LogMessage::SendToLog) +#define SYSLOG_DFATAL(counter) \ + google::LogMessage(__FILE__, __LINE__, google::DFATAL_LEVEL, counter, \ + &google::LogMessage::SendToSyslogAndLog) + +#if defined(WIN32) || defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) || defined(__CYGWIN32__) +// A very useful logging macro to log windows errors: +#define LOG_SYSRESULT(result) \ + if (FAILED(HRESULT_FROM_WIN32(result))) { \ + LPSTR message = NULL; \ + LPSTR msg = reinterpret_cast<LPSTR>(&message); \ + DWORD message_length = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | \ + FORMAT_MESSAGE_FROM_SYSTEM, \ + 0, result, 0, msg, 100, NULL); \ + if (message_length > 0) { \ + google::LogMessage(__FILE__, __LINE__, google::GLOG_ERROR, 0, \ + &google::LogMessage::SendToLog).stream() \ + << reinterpret_cast<const char*>(message); \ + LocalFree(message); \ + } \ + } +#endif + +// We use the preprocessor's merging operator, "##", so that, e.g., +// LOG(INFO) becomes the token GOOGLE_LOG_INFO. There's some funny +// subtle difference between ostream member streaming functions (e.g., +// ostream::operator<<(int) and ostream non-member streaming functions +// (e.g., ::operator<<(ostream&, string&): it turns out that it's +// impossible to stream something like a string directly to an unnamed +// ostream. We employ a neat hack by calling the stream() member +// function of LogMessage which seems to avoid the problem. +#define LOG(severity) COMPACT_GOOGLE_LOG_ ## severity.stream() +#define SYSLOG(severity) SYSLOG_ ## severity(0).stream() + +namespace google { + +// They need the definitions of integer types. +#include "glog/log_severity.h" +#include "glog/vlog_is_on.h" + +// Initialize google's logging library. You will see the program name +// specified by argv0 in log outputs. +GOOGLE_GLOG_DLL_DECL void InitGoogleLogging(const char* argv0); + +// Shutdown google's logging library. +GOOGLE_GLOG_DLL_DECL void ShutdownGoogleLogging(); + +// Install a function which will be called after LOG(FATAL). +GOOGLE_GLOG_DLL_DECL void InstallFailureFunction(void (*fail_func)()); + +// Enable/Disable old log cleaner. +GOOGLE_GLOG_DLL_DECL void EnableLogCleaner(int overdue_days); +GOOGLE_GLOG_DLL_DECL void DisableLogCleaner(); + + +class LogSink; // defined below + +// If a non-NULL sink pointer is given, we push this message to that sink. +// For LOG_TO_SINK we then do normal LOG(severity) logging as well. +// This is useful for capturing messages and passing/storing them +// somewhere more specific than the global log of the process. +// Argument types: +// LogSink* sink; +// LogSeverity severity; +// The cast is to disambiguate NULL arguments. +#define LOG_TO_SINK(sink, severity) \ + google::LogMessage( \ + __FILE__, __LINE__, \ + google::GLOG_ ## severity, \ + static_cast<google::LogSink*>(sink), true).stream() +#define LOG_TO_SINK_BUT_NOT_TO_LOGFILE(sink, severity) \ + google::LogMessage( \ + __FILE__, __LINE__, \ + google::GLOG_ ## severity, \ + static_cast<google::LogSink*>(sink), false).stream() + +// If a non-NULL string pointer is given, we write this message to that string. +// We then do normal LOG(severity) logging as well. +// This is useful for capturing messages and storing them somewhere more +// specific than the global log of the process. +// Argument types: +// string* message; +// LogSeverity severity; +// The cast is to disambiguate NULL arguments. +// NOTE: LOG(severity) expands to LogMessage().stream() for the specified +// severity. +#define LOG_TO_STRING(severity, message) \ + LOG_TO_STRING_##severity(static_cast<string*>(message)).stream() + +// If a non-NULL pointer is given, we push the message onto the end +// of a vector of strings; otherwise, we report it with LOG(severity). +// This is handy for capturing messages and perhaps passing them back +// to the caller, rather than reporting them immediately. +// Argument types: +// LogSeverity severity; +// vector<string> *outvec; +// The cast is to disambiguate NULL arguments. +#define LOG_STRING(severity, outvec) \ + LOG_TO_STRING_##severity(static_cast<std::vector<std::string>*>(outvec)).stream() + +#define LOG_IF(severity, condition) \ + static_cast<void>(0), \ + !(condition) ? (void) 0 : google::LogMessageVoidify() & LOG(severity) +#define SYSLOG_IF(severity, condition) \ + static_cast<void>(0), \ + !(condition) ? (void) 0 : google::LogMessageVoidify() & SYSLOG(severity) + +#define LOG_ASSERT(condition) \ + LOG_IF(FATAL, !(condition)) << "Assert failed: " #condition +#define SYSLOG_ASSERT(condition) \ + SYSLOG_IF(FATAL, !(condition)) << "Assert failed: " #condition + +// CHECK dies with a fatal error if condition is not true. It is *not* +// controlled by DCHECK_IS_ON(), so the check will be executed regardless of +// compilation mode. Therefore, it is safe to do things like: +// CHECK(fp->Write(x) == 4) +#define CHECK(condition) \ + LOG_IF(FATAL, GOOGLE_PREDICT_BRANCH_NOT_TAKEN(!(condition))) \ + << "Check failed: " #condition " " + +// A container for a string pointer which can be evaluated to a bool - +// true iff the pointer is NULL. +struct CheckOpString { + CheckOpString(std::string* str) : str_(str) { } + // No destructor: if str_ is non-NULL, we're about to LOG(FATAL), + // so there's no point in cleaning up str_. + operator bool() const { + return GOOGLE_PREDICT_BRANCH_NOT_TAKEN(str_ != NULL); + } + std::string* str_; +}; + +// Function is overloaded for integral types to allow static const +// integrals declared in classes and not defined to be used as arguments to +// CHECK* macros. It's not encouraged though. +template <class T> +inline const T& GetReferenceableValue(const T& t) { return t; } +inline char GetReferenceableValue(char t) { return t; } +inline unsigned char GetReferenceableValue(unsigned char t) { return t; } +inline signed char GetReferenceableValue(signed char t) { return t; } +inline short GetReferenceableValue(short t) { return t; } +inline unsigned short GetReferenceableValue(unsigned short t) { return t; } +inline int GetReferenceableValue(int t) { return t; } +inline unsigned int GetReferenceableValue(unsigned int t) { return t; } +inline long GetReferenceableValue(long t) { return t; } +inline unsigned long GetReferenceableValue(unsigned long t) { return t; } +inline long long GetReferenceableValue(long long t) { return t; } +inline unsigned long long GetReferenceableValue(unsigned long long t) { + return t; +} + +// This is a dummy class to define the following operator. +struct DummyClassToDefineOperator {}; + +} + +// Define global operator<< to declare using ::operator<<. +// This declaration will allow use to use CHECK macros for user +// defined classes which have operator<< (e.g., stl_logging.h). +inline std::ostream& operator<<( + std::ostream& out, const google::DummyClassToDefineOperator&) { + return out; +} + +namespace google { + +// This formats a value for a failing CHECK_XX statement. Ordinarily, +// it uses the definition for operator<<, with a few special cases below. +template <typename T> +inline void MakeCheckOpValueString(std::ostream* os, const T& v) { + (*os) << v; +} + +// Overrides for char types provide readable values for unprintable +// characters. +template <> GOOGLE_GLOG_DLL_DECL +void MakeCheckOpValueString(std::ostream* os, const char& v); +template <> GOOGLE_GLOG_DLL_DECL +void MakeCheckOpValueString(std::ostream* os, const signed char& v); +template <> GOOGLE_GLOG_DLL_DECL +void MakeCheckOpValueString(std::ostream* os, const unsigned char& v); + +// Build the error message string. Specify no inlining for code size. +template <typename T1, typename T2> +std::string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) + ; + +namespace base { +namespace internal { + +// If "s" is less than base_logging::INFO, returns base_logging::INFO. +// If "s" is greater than base_logging::FATAL, returns +// base_logging::ERROR. Otherwise, returns "s". +LogSeverity NormalizeSeverity(LogSeverity s); + +} // namespace internal + +// A helper class for formatting "expr (V1 vs. V2)" in a CHECK_XX +// statement. See MakeCheckOpString for sample usage. Other +// approaches were considered: use of a template method (e.g., +// base::BuildCheckOpString(exprtext, base::Print<T1>, &v1, +// base::Print<T2>, &v2), however this approach has complications +// related to volatile arguments and function-pointer arguments). +class GOOGLE_GLOG_DLL_DECL CheckOpMessageBuilder { + public: + // Inserts "exprtext" and " (" to the stream. + explicit CheckOpMessageBuilder(const char *exprtext); + // Deletes "stream_". + ~CheckOpMessageBuilder(); + // For inserting the first variable. + std::ostream* ForVar1() { return stream_; } + // For inserting the second variable (adds an intermediate " vs. "). + std::ostream* ForVar2(); + // Get the result (inserts the closing ")"). + std::string* NewString(); + + private: + std::ostringstream *stream_; +}; + +} // namespace base + +template <typename T1, typename T2> +std::string* MakeCheckOpString(const T1& v1, const T2& v2, const char* exprtext) { + base::CheckOpMessageBuilder comb(exprtext); + MakeCheckOpValueString(comb.ForVar1(), v1); + MakeCheckOpValueString(comb.ForVar2(), v2); + return comb.NewString(); +} + +// Helper functions for CHECK_OP macro. +// The (int, int) specialization works around the issue that the compiler +// will not instantiate the template version of the function on values of +// unnamed enum type - see comment below. +#define DEFINE_CHECK_OP_IMPL(name, op) \ + template <typename T1, typename T2> \ + inline std::string* name##Impl(const T1& v1, const T2& v2, \ + const char* exprtext) { \ + if (GOOGLE_PREDICT_TRUE(v1 op v2)) return NULL; \ + else return MakeCheckOpString(v1, v2, exprtext); \ + } \ + inline std::string* name##Impl(int v1, int v2, const char* exprtext) { \ + return name##Impl<int, int>(v1, v2, exprtext); \ + } + +// We use the full name Check_EQ, Check_NE, etc. in case the file including +// base/logging.h provides its own #defines for the simpler names EQ, NE, etc. +// This happens if, for example, those are used as token names in a +// yacc grammar. +DEFINE_CHECK_OP_IMPL(Check_EQ, ==) // Compilation error with CHECK_EQ(NULL, x)? +DEFINE_CHECK_OP_IMPL(Check_NE, !=) // Use CHECK(x == NULL) instead. +DEFINE_CHECK_OP_IMPL(Check_LE, <=) +DEFINE_CHECK_OP_IMPL(Check_LT, < ) +DEFINE_CHECK_OP_IMPL(Check_GE, >=) +DEFINE_CHECK_OP_IMPL(Check_GT, > ) +#undef DEFINE_CHECK_OP_IMPL + +// Helper macro for binary operators. +// Don't use this macro directly in your code, use CHECK_EQ et al below. + +#if defined(STATIC_ANALYSIS) +// Only for static analysis tool to know that it is equivalent to assert +#define CHECK_OP_LOG(name, op, val1, val2, log) CHECK((val1) op (val2)) +#elif DCHECK_IS_ON() +// In debug mode, avoid constructing CheckOpStrings if possible, +// to reduce the overhead of CHECK statments by 2x. +// Real DCHECK-heavy tests have seen 1.5x speedups. + +// The meaning of "string" might be different between now and +// when this macro gets invoked (e.g., if someone is experimenting +// with other string implementations that get defined after this +// file is included). Save the current meaning now and use it +// in the macro. +typedef std::string _Check_string; +#define CHECK_OP_LOG(name, op, val1, val2, log) \ + while (google::_Check_string* _result = \ + google::Check##name##Impl( \ + google::GetReferenceableValue(val1), \ + google::GetReferenceableValue(val2), \ + #val1 " " #op " " #val2)) \ + log(__FILE__, __LINE__, \ + google::CheckOpString(_result)).stream() +#else +// In optimized mode, use CheckOpString to hint to compiler that +// the while condition is unlikely. +#define CHECK_OP_LOG(name, op, val1, val2, log) \ + while (google::CheckOpString _result = \ + google::Check##name##Impl( \ + google::GetReferenceableValue(val1), \ + google::GetReferenceableValue(val2), \ + #val1 " " #op " " #val2)) \ + log(__FILE__, __LINE__, _result).stream() +#endif // STATIC_ANALYSIS, DCHECK_IS_ON() + +#if GOOGLE_STRIP_LOG <= 3 +#define CHECK_OP(name, op, val1, val2) \ + CHECK_OP_LOG(name, op, val1, val2, google::LogMessageFatal) +#else +#define CHECK_OP(name, op, val1, val2) \ + CHECK_OP_LOG(name, op, val1, val2, google::NullStreamFatal) +#endif // STRIP_LOG <= 3 + +// Equality/Inequality checks - compare two values, and log a FATAL message +// including the two values when the result is not as expected. The values +// must have operator<<(ostream, ...) defined. +// +// You may append to the error message like so: +// CHECK_NE(1, 2) << ": The world must be ending!"; +// +// We are very careful to ensure that each argument is evaluated exactly +// once, and that anything which is legal to pass as a function argument is +// legal here. In particular, the arguments may be temporary expressions +// which will end up being destroyed at the end of the apparent statement, +// for example: +// CHECK_EQ(string("abc")[1], 'b'); +// +// WARNING: These don't compile correctly if one of the arguments is a pointer +// and the other is NULL. To work around this, simply static_cast NULL to the +// type of the desired pointer. + +#define CHECK_EQ(val1, val2) CHECK_OP(_EQ, ==, val1, val2) +#define CHECK_NE(val1, val2) CHECK_OP(_NE, !=, val1, val2) +#define CHECK_LE(val1, val2) CHECK_OP(_LE, <=, val1, val2) +#define CHECK_LT(val1, val2) CHECK_OP(_LT, < , val1, val2) +#define CHECK_GE(val1, val2) CHECK_OP(_GE, >=, val1, val2) +#define CHECK_GT(val1, val2) CHECK_OP(_GT, > , val1, val2) + +// Check that the input is non NULL. This very useful in constructor +// initializer lists. + +#define CHECK_NOTNULL(val) \ + google::CheckNotNull(__FILE__, __LINE__, "'" #val "' Must be non NULL", (val)) + +// Helper functions for string comparisons. +// To avoid bloat, the definitions are in logging.cc. +#define DECLARE_CHECK_STROP_IMPL(func, expected) \ + GOOGLE_GLOG_DLL_DECL std::string* Check##func##expected##Impl( \ + const char* s1, const char* s2, const char* names); +DECLARE_CHECK_STROP_IMPL(strcmp, true) +DECLARE_CHECK_STROP_IMPL(strcmp, false) +DECLARE_CHECK_STROP_IMPL(strcasecmp, true) +DECLARE_CHECK_STROP_IMPL(strcasecmp, false) +#undef DECLARE_CHECK_STROP_IMPL + +// Helper macro for string comparisons. +// Don't use this macro directly in your code, use CHECK_STREQ et al below. +#define CHECK_STROP(func, op, expected, s1, s2) \ + while (google::CheckOpString _result = \ + google::Check##func##expected##Impl((s1), (s2), \ + #s1 " " #op " " #s2)) \ + LOG(FATAL) << *_result.str_ + + +// String (char*) equality/inequality checks. +// CASE versions are case-insensitive. +// +// Note that "s1" and "s2" may be temporary strings which are destroyed +// by the compiler at the end of the current "full expression" +// (e.g. CHECK_STREQ(Foo().c_str(), Bar().c_str())). + +#define CHECK_STREQ(s1, s2) CHECK_STROP(strcmp, ==, true, s1, s2) +#define CHECK_STRNE(s1, s2) CHECK_STROP(strcmp, !=, false, s1, s2) +#define CHECK_STRCASEEQ(s1, s2) CHECK_STROP(strcasecmp, ==, true, s1, s2) +#define CHECK_STRCASENE(s1, s2) CHECK_STROP(strcasecmp, !=, false, s1, s2) + +#define CHECK_INDEX(I,A) CHECK(I < (sizeof(A)/sizeof(A[0]))) +#define CHECK_BOUND(B,A) CHECK(B <= (sizeof(A)/sizeof(A[0]))) + +#define CHECK_DOUBLE_EQ(val1, val2) \ + do { \ + CHECK_LE((val1), (val2)+0.000000000000001L); \ + CHECK_GE((val1), (val2)-0.000000000000001L); \ + } while (0) + +#define CHECK_NEAR(val1, val2, margin) \ + do { \ + CHECK_LE((val1), (val2)+(margin)); \ + CHECK_GE((val1), (val2)-(margin)); \ + } while (0) + +// perror()..googly style! +// +// PLOG() and PLOG_IF() and PCHECK() behave exactly like their LOG* and +// CHECK equivalents with the addition that they postpend a description +// of the current state of errno to their output lines. + +#define PLOG(severity) GOOGLE_PLOG(severity, 0).stream() + +#define GOOGLE_PLOG(severity, counter) \ + google::ErrnoLogMessage( \ + __FILE__, __LINE__, google::GLOG_ ## severity, counter, \ + &google::LogMessage::SendToLog) + +#define PLOG_IF(severity, condition) \ + static_cast<void>(0), \ + !(condition) ? (void) 0 : google::LogMessageVoidify() & PLOG(severity) + +// A CHECK() macro that postpends errno if the condition is false. E.g. +// +// if (poll(fds, nfds, timeout) == -1) { PCHECK(errno == EINTR); ... } +#define PCHECK(condition) \ + PLOG_IF(FATAL, GOOGLE_PREDICT_BRANCH_NOT_TAKEN(!(condition))) \ + << "Check failed: " #condition " " + +// A CHECK() macro that lets you assert the success of a function that +// returns -1 and sets errno in case of an error. E.g. +// +// CHECK_ERR(mkdir(path, 0700)); +// +// or +// +// int fd = open(filename, flags); CHECK_ERR(fd) << ": open " << filename; +#define CHECK_ERR(invocation) \ +PLOG_IF(FATAL, GOOGLE_PREDICT_BRANCH_NOT_TAKEN((invocation) == -1)) \ + << #invocation + +// Use macro expansion to create, for each use of LOG_EVERY_N(), static +// variables with the __LINE__ expansion as part of the variable name. +#define LOG_EVERY_N_VARNAME(base, line) LOG_EVERY_N_VARNAME_CONCAT(base, line) +#define LOG_EVERY_N_VARNAME_CONCAT(base, line) base ## line + +#define LOG_OCCURRENCES LOG_EVERY_N_VARNAME(occurrences_, __LINE__) +#define LOG_OCCURRENCES_MOD_N LOG_EVERY_N_VARNAME(occurrences_mod_n_, __LINE__) + +#define SOME_KIND_OF_LOG_EVERY_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (LOG_OCCURRENCES_MOD_N == 1) \ + google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +#define SOME_KIND_OF_LOG_IF_EVERY_N(severity, condition, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (condition && \ + ((LOG_OCCURRENCES_MOD_N=(LOG_OCCURRENCES_MOD_N + 1) % n) == (1 % n))) \ + google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +#define SOME_KIND_OF_PLOG_EVERY_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0, LOG_OCCURRENCES_MOD_N = 0; \ + ++LOG_OCCURRENCES; \ + if (++LOG_OCCURRENCES_MOD_N > n) LOG_OCCURRENCES_MOD_N -= n; \ + if (LOG_OCCURRENCES_MOD_N == 1) \ + google::ErrnoLogMessage( \ + __FILE__, __LINE__, google::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +#define SOME_KIND_OF_LOG_FIRST_N(severity, n, what_to_do) \ + static int LOG_OCCURRENCES = 0; \ + if (LOG_OCCURRENCES <= n) \ + ++LOG_OCCURRENCES; \ + if (LOG_OCCURRENCES <= n) \ + google::LogMessage( \ + __FILE__, __LINE__, google::GLOG_ ## severity, LOG_OCCURRENCES, \ + &what_to_do).stream() + +namespace glog_internal_namespace_ { +template <bool> +struct CompileAssert { +}; +struct CrashReason; + +// Returns true if FailureSignalHandler is installed. +// Needs to be exported since it's used by the signalhandler_unittest. +GOOGLE_GLOG_DLL_DECL bool IsFailureSignalHandlerInstalled(); +} // namespace glog_internal_namespace_ + +#define LOG_EVERY_N(severity, n) \ + SOME_KIND_OF_LOG_EVERY_N(severity, (n), google::LogMessage::SendToLog) + +#define SYSLOG_EVERY_N(severity, n) \ + SOME_KIND_OF_LOG_EVERY_N(severity, (n), google::LogMessage::SendToSyslogAndLog) + +#define PLOG_EVERY_N(severity, n) \ + SOME_KIND_OF_PLOG_EVERY_N(severity, (n), google::LogMessage::SendToLog) + +#define LOG_FIRST_N(severity, n) \ + SOME_KIND_OF_LOG_FIRST_N(severity, (n), google::LogMessage::SendToLog) + +#define LOG_IF_EVERY_N(severity, condition, n) \ + SOME_KIND_OF_LOG_IF_EVERY_N(severity, (condition), (n), google::LogMessage::SendToLog) + +// We want the special COUNTER value available for LOG_EVERY_X()'ed messages +enum PRIVATE_Counter {COUNTER}; + +#ifdef GLOG_NO_ABBREVIATED_SEVERITIES +// wingdi.h defines ERROR to be 0. When we call LOG(ERROR), it gets +// substituted with 0, and it expands to COMPACT_GOOGLE_LOG_0. To allow us +// to keep using this syntax, we define this macro to do the same thing +// as COMPACT_GOOGLE_LOG_ERROR. +#define COMPACT_GOOGLE_LOG_0 COMPACT_GOOGLE_LOG_ERROR +#define SYSLOG_0 SYSLOG_ERROR +#define LOG_TO_STRING_0 LOG_TO_STRING_ERROR +// Needed for LOG_IS_ON(ERROR). +const LogSeverity GLOG_0 = GLOG_ERROR; +#else +// Users may include windows.h after logging.h without +// GLOG_NO_ABBREVIATED_SEVERITIES nor WIN32_LEAN_AND_MEAN. +// For this case, we cannot detect if ERROR is defined before users +// actually use ERROR. Let's make an undefined symbol to warn users. +# define GLOG_ERROR_MSG ERROR_macro_is_defined_Define_GLOG_NO_ABBREVIATED_SEVERITIES_before_including_logging_h_See_the_document_for_detail +# define COMPACT_GOOGLE_LOG_0 GLOG_ERROR_MSG +# define SYSLOG_0 GLOG_ERROR_MSG +# define LOG_TO_STRING_0 GLOG_ERROR_MSG +# define GLOG_0 GLOG_ERROR_MSG +#endif + +// Plus some debug-logging macros that get compiled to nothing for production + +#if DCHECK_IS_ON() + +#define DLOG(severity) LOG(severity) +#define DVLOG(verboselevel) VLOG(verboselevel) +#define DLOG_IF(severity, condition) LOG_IF(severity, condition) +#define DLOG_EVERY_N(severity, n) LOG_EVERY_N(severity, n) +#define DLOG_IF_EVERY_N(severity, condition, n) \ + LOG_IF_EVERY_N(severity, condition, n) +#define DLOG_ASSERT(condition) LOG_ASSERT(condition) + +// debug-only checking. executed if DCHECK_IS_ON(). +#define DCHECK(condition) CHECK(condition) +#define DCHECK_EQ(val1, val2) CHECK_EQ(val1, val2) +#define DCHECK_NE(val1, val2) CHECK_NE(val1, val2) +#define DCHECK_LE(val1, val2) CHECK_LE(val1, val2) +#define DCHECK_LT(val1, val2) CHECK_LT(val1, val2) +#define DCHECK_GE(val1, val2) CHECK_GE(val1, val2) +#define DCHECK_GT(val1, val2) CHECK_GT(val1, val2) +#define DCHECK_NOTNULL(val) CHECK_NOTNULL(val) +#define DCHECK_STREQ(str1, str2) CHECK_STREQ(str1, str2) +#define DCHECK_STRCASEEQ(str1, str2) CHECK_STRCASEEQ(str1, str2) +#define DCHECK_STRNE(str1, str2) CHECK_STRNE(str1, str2) +#define DCHECK_STRCASENE(str1, str2) CHECK_STRCASENE(str1, str2) + +#else // !DCHECK_IS_ON() + +#define DLOG(severity) \ + static_cast<void>(0), \ + true ? (void) 0 : google::LogMessageVoidify() & LOG(severity) + +#define DVLOG(verboselevel) \ + static_cast<void>(0), \ + (true || !VLOG_IS_ON(verboselevel)) ? \ + (void) 0 : google::LogMessageVoidify() & LOG(INFO) + +#define DLOG_IF(severity, condition) \ + static_cast<void>(0), \ + (true || !(condition)) ? (void) 0 : google::LogMessageVoidify() & LOG(severity) + +#define DLOG_EVERY_N(severity, n) \ + static_cast<void>(0), \ + true ? (void) 0 : google::LogMessageVoidify() & LOG(severity) + +#define DLOG_IF_EVERY_N(severity, condition, n) \ + static_cast<void>(0), \ + (true || !(condition))? (void) 0 : google::LogMessageVoidify() & LOG(severity) + +#define DLOG_ASSERT(condition) \ + static_cast<void>(0), \ + true ? (void) 0 : LOG_ASSERT(condition) + +// MSVC warning C4127: conditional expression is constant +#define DCHECK(condition) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK(condition) + +#define DCHECK_EQ(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_EQ(val1, val2) + +#define DCHECK_NE(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_NE(val1, val2) + +#define DCHECK_LE(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_LE(val1, val2) + +#define DCHECK_LT(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_LT(val1, val2) + +#define DCHECK_GE(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_GE(val1, val2) + +#define DCHECK_GT(val1, val2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_GT(val1, val2) + +// You may see warnings in release mode if you don't use the return +// value of DCHECK_NOTNULL. Please just use DCHECK for such cases. +#define DCHECK_NOTNULL(val) (val) + +#define DCHECK_STREQ(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STREQ(str1, str2) + +#define DCHECK_STRCASEEQ(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STRCASEEQ(str1, str2) + +#define DCHECK_STRNE(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STRNE(str1, str2) + +#define DCHECK_STRCASENE(str1, str2) \ + GLOG_MSVC_PUSH_DISABLE_WARNING(4127) \ + while (false) \ + GLOG_MSVC_POP_WARNING() CHECK_STRCASENE(str1, str2) + +#endif // DCHECK_IS_ON() + +// Log only in verbose mode. + +#define VLOG(verboselevel) LOG_IF(INFO, VLOG_IS_ON(verboselevel)) + +#define VLOG_IF(verboselevel, condition) \ + LOG_IF(INFO, (condition) && VLOG_IS_ON(verboselevel)) + +#define VLOG_EVERY_N(verboselevel, n) \ + LOG_IF_EVERY_N(INFO, VLOG_IS_ON(verboselevel), n) + +#define VLOG_IF_EVERY_N(verboselevel, condition, n) \ + LOG_IF_EVERY_N(INFO, (condition) && VLOG_IS_ON(verboselevel), n) + +namespace base_logging { + +// LogMessage::LogStream is a std::ostream backed by this streambuf. +// This class ignores overflow and leaves two bytes at the end of the +// buffer to allow for a '\n' and '\0'. +class GOOGLE_GLOG_DLL_DECL LogStreamBuf : public std::streambuf { + public: + // REQUIREMENTS: "len" must be >= 2 to account for the '\n' and '\0'. + LogStreamBuf(char *buf, int len) { + setp(buf, buf + len - 2); + } + + // This effectively ignores overflow. + virtual int_type overflow(int_type ch) { + return ch; + } + + // Legacy public ostrstream method. + size_t pcount() const { return pptr() - pbase(); } + char* pbase() const { return std::streambuf::pbase(); } +}; + +} // namespace base_logging + +// +// This class more or less represents a particular log message. You +// create an instance of LogMessage and then stream stuff to it. +// When you finish streaming to it, ~LogMessage is called and the +// full message gets streamed to the appropriate destination. +// +// You shouldn't actually use LogMessage's constructor to log things, +// though. You should use the LOG() macro (and variants thereof) +// above. +class GOOGLE_GLOG_DLL_DECL LogMessage { +public: + enum { + // Passing kNoLogPrefix for the line number disables the + // log-message prefix. Useful for using the LogMessage + // infrastructure as a printing utility. See also the --log_prefix + // flag for controlling the log-message prefix on an + // application-wide basis. + kNoLogPrefix = -1 + }; + + // LogStream inherit from non-DLL-exported class (std::ostrstream) + // and VC++ produces a warning for this situation. + // However, MSDN says "C4275 can be ignored in Microsoft Visual C++ + // 2005 if you are deriving from a type in the Standard C++ Library" + // http://msdn.microsoft.com/en-us/library/3tdb471s(VS.80).aspx + // Let's just ignore the warning. +GLOG_MSVC_PUSH_DISABLE_WARNING(4275) + class GOOGLE_GLOG_DLL_DECL LogStream : public std::ostream { +GLOG_MSVC_POP_WARNING() + public: + LogStream(char *buf, int len, int ctr) + : std::ostream(NULL), + streambuf_(buf, len), + ctr_(ctr), + self_(this) { + rdbuf(&streambuf_); + } + + int ctr() const { return ctr_; } + void set_ctr(int ctr) { ctr_ = ctr; } + LogStream* self() const { return self_; } + + // Legacy std::streambuf methods. + size_t pcount() const { return streambuf_.pcount(); } + char* pbase() const { return streambuf_.pbase(); } + char* str() const { return pbase(); } + + private: + LogStream(const LogStream&); + LogStream& operator=(const LogStream&); + base_logging::LogStreamBuf streambuf_; + int ctr_; // Counter hack (for the LOG_EVERY_X() macro) + LogStream *self_; // Consistency check hack + }; + +public: + // icc 8 requires this typedef to avoid an internal compiler error. + typedef void (LogMessage::*SendMethod)(); + + LogMessage(const char* file, int line, LogSeverity severity, int ctr, + SendMethod send_method); + + // Two special constructors that generate reduced amounts of code at + // LOG call sites for common cases. + + // Used for LOG(INFO): Implied are: + // severity = INFO, ctr = 0, send_method = &LogMessage::SendToLog. + // + // Using this constructor instead of the more complex constructor above + // saves 19 bytes per call site. + LogMessage(const char* file, int line); + + // Used for LOG(severity) where severity != INFO. Implied + // are: ctr = 0, send_method = &LogMessage::SendToLog + // + // Using this constructor instead of the more complex constructor above + // saves 17 bytes per call site. + LogMessage(const char* file, int line, LogSeverity severity); + + // Constructor to log this message to a specified sink (if not NULL). + // Implied are: ctr = 0, send_method = &LogMessage::SendToSinkAndLog if + // also_send_to_log is true, send_method = &LogMessage::SendToSink otherwise. + LogMessage(const char* file, int line, LogSeverity severity, LogSink* sink, + bool also_send_to_log); + + // Constructor where we also give a vector<string> pointer + // for storing the messages (if the pointer is not NULL). + // Implied are: ctr = 0, send_method = &LogMessage::SaveOrSendToLog. + LogMessage(const char* file, int line, LogSeverity severity, + std::vector<std::string>* outvec); + + // Constructor where we also give a string pointer for storing the + // message (if the pointer is not NULL). Implied are: ctr = 0, + // send_method = &LogMessage::WriteToStringAndLog. + LogMessage(const char* file, int line, LogSeverity severity, + std::string* message); + + // A special constructor used for check failures + LogMessage(const char* file, int line, const CheckOpString& result); + + ~LogMessage(); + + // Flush a buffered message to the sink set in the constructor. Always + // called by the destructor, it may also be called from elsewhere if + // needed. Only the first call is actioned; any later ones are ignored. + void Flush(); + + // An arbitrary limit on the length of a single log message. This + // is so that streaming can be done more efficiently. + static const size_t kMaxLogMessageLen; + + // Theses should not be called directly outside of logging.*, + // only passed as SendMethod arguments to other LogMessage methods: + void SendToLog(); // Actually dispatch to the logs + void SendToSyslogAndLog(); // Actually dispatch to syslog and the logs + + // Call abort() or similar to perform LOG(FATAL) crash. + static void __declspec(noreturn) Fail(); + + std::ostream& stream(); + + int preserved_errno() const; + + // Must be called without the log_mutex held. (L < log_mutex) + static int64 num_messages(int severity); + + struct LogMessageData; + +private: + // Fully internal SendMethod cases: + void SendToSinkAndLog(); // Send to sink if provided and dispatch to the logs + void SendToSink(); // Send to sink if provided, do nothing otherwise. + + // Write to string if provided and dispatch to the logs. + void WriteToStringAndLog(); + + void SaveOrSendToLog(); // Save to stringvec if provided, else to logs + + void Init(const char* file, int line, LogSeverity severity, + void (LogMessage::*send_method)()); + + // Used to fill in crash information during LOG(FATAL) failures. + void RecordCrashReason(glog_internal_namespace_::CrashReason* reason); + + // Counts of messages sent at each priority: + static int64 num_messages_[NUM_SEVERITIES]; // under log_mutex + + // We keep the data in a separate struct so that each instance of + // LogMessage uses less stack space. + LogMessageData* allocated_; + LogMessageData* data_; + + friend class LogDestination; + + LogMessage(const LogMessage&); + void operator=(const LogMessage&); +}; + +// This class happens to be thread-hostile because all instances share +// a single data buffer, but since it can only be created just before +// the process dies, we don't worry so much. +class GOOGLE_GLOG_DLL_DECL LogMessageFatal : public LogMessage { + public: + LogMessageFatal(const char* file, int line); + LogMessageFatal(const char* file, int line, const CheckOpString& result); + __declspec(noreturn) ~LogMessageFatal(); +}; + +// A non-macro interface to the log facility; (useful +// when the logging level is not a compile-time constant). +inline void LogAtLevel(int const severity, std::string const &msg) { + LogMessage(__FILE__, __LINE__, severity).stream() << msg; +} + +// A macro alternative of LogAtLevel. New code may want to use this +// version since there are two advantages: 1. this version outputs the +// file name and the line number where this macro is put like other +// LOG macros, 2. this macro can be used as C++ stream. +#define LOG_AT_LEVEL(severity) google::LogMessage(__FILE__, __LINE__, severity).stream() + +// Check if it's compiled in C++11 mode. +// +// GXX_EXPERIMENTAL_CXX0X is defined by gcc and clang up to at least +// gcc-4.7 and clang-3.1 (2011-12-13). __cplusplus was defined to 1 +// in gcc before 4.7 (Crosstool 16) and clang before 3.1, but is +// defined according to the language version in effect thereafter. +// Microsoft Visual Studio 14 (2015) sets __cplusplus==199711 despite +// reasonably good C++11 support, so we set LANG_CXX for it and +// newer versions (_MSC_VER >= 1900). +#if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L || \ + (defined(_MSC_VER) && _MSC_VER >= 1900)) +// Helper for CHECK_NOTNULL(). +// +// In C++11, all cases can be handled by a single function. Since the value +// category of the argument is preserved (also for rvalue references), +// member initializer lists like the one below will compile correctly: +// +// Foo() +// : x_(CHECK_NOTNULL(MethodReturningUniquePtr())) {} +template <typename T> +T CheckNotNull(const char* file, int line, const char* names, T&& t) { + if (t == nullptr) { + LogMessageFatal(file, line, new std::string(names)); + } + return std::forward<T>(t); +} + +#else + +// A small helper for CHECK_NOTNULL(). +template <typename T> +T* CheckNotNull(const char *file, int line, const char *names, T* t) { + if (t == NULL) { + LogMessageFatal(file, line, new std::string(names)); + } + return t; +} +#endif + +// Allow folks to put a counter in the LOG_EVERY_X()'ed messages. This +// only works if ostream is a LogStream. If the ostream is not a +// LogStream you'll get an assert saying as much at runtime. +GOOGLE_GLOG_DLL_DECL std::ostream& operator<<(std::ostream &os, + const PRIVATE_Counter&); + + +// Derived class for PLOG*() above. +class GOOGLE_GLOG_DLL_DECL ErrnoLogMessage : public LogMessage { + public: + + ErrnoLogMessage(const char* file, int line, LogSeverity severity, int ctr, + void (LogMessage::*send_method)()); + + // Postpends ": strerror(errno) [errno]". + ~ErrnoLogMessage(); + + private: + ErrnoLogMessage(const ErrnoLogMessage&); + void operator=(const ErrnoLogMessage&); +}; + + +// This class is used to explicitly ignore values in the conditional +// logging macros. This avoids compiler warnings like "value computed +// is not used" and "statement has no effect". + +class GOOGLE_GLOG_DLL_DECL LogMessageVoidify { + public: + LogMessageVoidify() { } + // This has to be an operator with a precedence lower than << but + // higher than ?: + void operator&(std::ostream&) { } +}; + + +// Flushes all log files that contains messages that are at least of +// the specified severity level. Thread-safe. +GOOGLE_GLOG_DLL_DECL void FlushLogFiles(LogSeverity min_severity); + +// Flushes all log files that contains messages that are at least of +// the specified severity level. Thread-hostile because it ignores +// locking -- used for catastrophic failures. +GOOGLE_GLOG_DLL_DECL void FlushLogFilesUnsafe(LogSeverity min_severity); + +// +// Set the destination to which a particular severity level of log +// messages is sent. If base_filename is "", it means "don't log this +// severity". Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetLogDestination(LogSeverity severity, + const char* base_filename); + +// +// Set the basename of the symlink to the latest log file at a given +// severity. If symlink_basename is empty, do not make a symlink. If +// you don't call this function, the symlink basename is the +// invocation name of the program. Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetLogSymlink(LogSeverity severity, + const char* symlink_basename); + +// +// Used to send logs to some other kind of destination +// Users should subclass LogSink and override send to do whatever they want. +// Implementations must be thread-safe because a shared instance will +// be called from whichever thread ran the LOG(XXX) line. +class GOOGLE_GLOG_DLL_DECL LogSink { + public: + virtual ~LogSink(); + + // Sink's logging logic (message_len is such as to exclude '\n' at the end). + // This method can't use LOG() or CHECK() as logging system mutex(s) are held + // during this call. + virtual void send(LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len, int32 usecs) { + send(severity, full_filename, base_filename, line, + tm_time, message, message_len); + } + // This send() signature is obsolete. + // New implementations should define this in terms of + // the above send() method. + virtual void send(LogSeverity severity, const char* full_filename, + const char* base_filename, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len) = 0; + + // Redefine this to implement waiting for + // the sink's logging logic to complete. + // It will be called after each send() returns, + // but before that LogMessage exits or crashes. + // By default this function does nothing. + // Using this function one can implement complex logic for send() + // that itself involves logging; and do all this w/o causing deadlocks and + // inconsistent rearrangement of log messages. + // E.g. if a LogSink has thread-specific actions, the send() method + // can simply add the message to a queue and wake up another thread that + // handles real logging while itself making some LOG() calls; + // WaitTillSent() can be implemented to wait for that logic to complete. + // See our unittest for an example. + virtual void WaitTillSent(); + + // Returns the normal text output of the log message. + // Can be useful to implement send(). + static std::string ToString(LogSeverity severity, const char* file, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len, + int32 usecs); + + // Obsolete + static std::string ToString(LogSeverity severity, const char* file, int line, + const struct ::tm* tm_time, + const char* message, size_t message_len) { + return ToString(severity, file, line, tm_time, message, message_len, 0); + } +}; + +// Add or remove a LogSink as a consumer of logging data. Thread-safe. +GOOGLE_GLOG_DLL_DECL void AddLogSink(LogSink *destination); +GOOGLE_GLOG_DLL_DECL void RemoveLogSink(LogSink *destination); + +// +// Specify an "extension" added to the filename specified via +// SetLogDestination. This applies to all severity levels. It's +// often used to append the port we're listening on to the logfile +// name. Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetLogFilenameExtension( + const char* filename_extension); + +// +// Make it so that all log messages of at least a particular severity +// are logged to stderr (in addition to logging to the usual log +// file(s)). Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetStderrLogging(LogSeverity min_severity); + +// +// Make it so that all log messages go only to stderr. Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void LogToStderr(); + +// +// Make it so that all log messages of at least a particular severity are +// logged via email to a list of addresses (in addition to logging to the +// usual log file(s)). The list of addresses is just a string containing +// the email addresses to send to (separated by spaces, say). Thread-safe. +// +GOOGLE_GLOG_DLL_DECL void SetEmailLogging(LogSeverity min_severity, + const char* addresses); + +// A simple function that sends email. dest is a commma-separated +// list of addressess. Thread-safe. +GOOGLE_GLOG_DLL_DECL bool SendEmail(const char *dest, + const char *subject, const char *body); + +GOOGLE_GLOG_DLL_DECL const std::vector<std::string>& GetLoggingDirectories(); + +// For tests only: Clear the internal [cached] list of logging directories to +// force a refresh the next time GetLoggingDirectories is called. +// Thread-hostile. +void TestOnly_ClearLoggingDirectoriesList(); + +// Returns a set of existing temporary directories, which will be a +// subset of the directories returned by GetLogginDirectories(). +// Thread-safe. +GOOGLE_GLOG_DLL_DECL void GetExistingTempDirectories( + std::vector<std::string>* list); + +// Print any fatal message again -- useful to call from signal handler +// so that the last thing in the output is the fatal message. +// Thread-hostile, but a race is unlikely. +GOOGLE_GLOG_DLL_DECL void ReprintFatalMessage(); + +// Truncate a log file that may be the append-only output of multiple +// processes and hence can't simply be renamed/reopened (typically a +// stdout/stderr). If the file "path" is > "limit" bytes, copy the +// last "keep" bytes to offset 0 and truncate the rest. Since we could +// be racing with other writers, this approach has the potential to +// lose very small amounts of data. For security, only follow symlinks +// if the path is /proc/self/fd/* +GOOGLE_GLOG_DLL_DECL void TruncateLogFile(const char *path, + int64 limit, int64 keep); + +// Truncate stdout and stderr if they are over the value specified by +// --max_log_size; keep the final 1MB. This function has the same +// race condition as TruncateLogFile. +GOOGLE_GLOG_DLL_DECL void TruncateStdoutStderr(); + +// Return the string representation of the provided LogSeverity level. +// Thread-safe. +GOOGLE_GLOG_DLL_DECL const char* GetLogSeverityName(LogSeverity severity); + +// --------------------------------------------------------------------- +// Implementation details that are not useful to most clients +// --------------------------------------------------------------------- + +// A Logger is the interface used by logging modules to emit entries +// to a log. A typical implementation will dump formatted data to a +// sequence of files. We also provide interfaces that will forward +// the data to another thread so that the invoker never blocks. +// Implementations should be thread-safe since the logging system +// will write to them from multiple threads. + +namespace base { + +class GOOGLE_GLOG_DLL_DECL Logger { + public: + virtual ~Logger(); + + // Writes "message[0,message_len-1]" corresponding to an event that + // occurred at "timestamp". If "force_flush" is true, the log file + // is flushed immediately. + // + // The input message has already been formatted as deemed + // appropriate by the higher level logging facility. For example, + // textual log messages already contain timestamps, and the + // file:linenumber header. + virtual void Write(bool force_flush, + time_t timestamp, + const char* message, + int message_len) = 0; + + // Flush any buffered messages + virtual void Flush() = 0; + + // Get the current LOG file size. + // The returned value is approximate since some + // logged data may not have been flushed to disk yet. + virtual uint32 LogSize() = 0; +}; + +// Get the logger for the specified severity level. The logger +// remains the property of the logging module and should not be +// deleted by the caller. Thread-safe. +extern GOOGLE_GLOG_DLL_DECL Logger* GetLogger(LogSeverity level); + +// Set the logger for the specified severity level. The logger +// becomes the property of the logging module and should not +// be deleted by the caller. Thread-safe. +extern GOOGLE_GLOG_DLL_DECL void SetLogger(LogSeverity level, Logger* logger); + +} + +// glibc has traditionally implemented two incompatible versions of +// strerror_r(). There is a poorly defined convention for picking the +// version that we want, but it is not clear whether it even works with +// all versions of glibc. +// So, instead, we provide this wrapper that automatically detects the +// version that is in use, and then implements POSIX semantics. +// N.B. In addition to what POSIX says, we also guarantee that "buf" will +// be set to an empty string, if this function failed. This means, in most +// cases, you do not need to check the error code and you can directly +// use the value of "buf". It will never have an undefined value. +// DEPRECATED: Use StrError(int) instead. +GOOGLE_GLOG_DLL_DECL int posix_strerror_r(int err, char *buf, size_t len); + +// A thread-safe replacement for strerror(). Returns a string describing the +// given POSIX error code. +GOOGLE_GLOG_DLL_DECL std::string StrError(int err); + +// A class for which we define operator<<, which does nothing. +class GOOGLE_GLOG_DLL_DECL NullStream : public LogMessage::LogStream { + public: + // Initialize the LogStream so the messages can be written somewhere + // (they'll never be actually displayed). This will be needed if a + // NullStream& is implicitly converted to LogStream&, in which case + // the overloaded NullStream::operator<< will not be invoked. + NullStream() : LogMessage::LogStream(message_buffer_, 1, 0) { } + NullStream(const char* /*file*/, int /*line*/, + const CheckOpString& /*result*/) : + LogMessage::LogStream(message_buffer_, 1, 0) { } + NullStream &stream() { return *this; } + private: + // A very short buffer for messages (which we discard anyway). This + // will be needed if NullStream& converted to LogStream& (e.g. as a + // result of a conditional expression). + char message_buffer_[2]; +}; + +// Do nothing. This operator is inline, allowing the message to be +// compiled away. The message will not be compiled away if we do +// something like (flag ? LOG(INFO) : LOG(ERROR)) << message; when +// SKIP_LOG=WARNING. In those cases, NullStream will be implicitly +// converted to LogStream and the message will be computed and then +// quietly discarded. +template<class T> +inline NullStream& operator<<(NullStream &str, const T &) { return str; } + +// Similar to NullStream, but aborts the program (without stack +// trace), like LogMessageFatal. +class GOOGLE_GLOG_DLL_DECL NullStreamFatal : public NullStream { + public: + NullStreamFatal() { } + NullStreamFatal(const char* file, int line, const CheckOpString& result) : + NullStream(file, line, result) { } + __declspec(noreturn) ~NullStreamFatal() throw () { _exit(1); } +}; + +// Install a signal handler that will dump signal information and a stack +// trace when the program crashes on certain signals. We'll install the +// signal handler for the following signals. +// +// SIGSEGV, SIGILL, SIGFPE, SIGABRT, SIGBUS, and SIGTERM. +// +// By default, the signal handler will write the failure dump to the +// standard error. You can customize the destination by installing your +// own writer function by InstallFailureWriter() below. +// +// Note on threading: +// +// The function should be called before threads are created, if you want +// to use the failure signal handler for all threads. The stack trace +// will be shown only for the thread that receives the signal. In other +// words, stack traces of other threads won't be shown. +GOOGLE_GLOG_DLL_DECL void InstallFailureSignalHandler(); + +// Installs a function that is used for writing the failure dump. "data" +// is the pointer to the beginning of a message to be written, and "size" +// is the size of the message. You should not expect the data is +// terminated with '\0'. +GOOGLE_GLOG_DLL_DECL void InstallFailureWriter( + void (*writer)(const char* data, int size)); + +} + +#endif // _LOGGING_H_ diff --git a/third_party/glog/src/windows/glog/raw_logging.h b/third_party/glog/src/windows/glog/raw_logging.h new file mode 100755 index 000000000000..e0e6d6f1a256 --- /dev/null +++ b/third_party/glog/src/windows/glog/raw_logging.h @@ -0,0 +1,184 @@ +// This file is automatically generated from src/glog/raw_logging.h.in +// using src/windows/preprocess.sh. +// DO NOT EDIT! + +// Copyright (c) 2006, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Maxim Lifantsev +// +// Thread-safe logging routines that do not allocate any memory or +// acquire any locks, and can therefore be used by low-level memory +// allocation and synchronization code. + +#ifndef BASE_RAW_LOGGING_H_ +#define BASE_RAW_LOGGING_H_ + +#include <time.h> + +namespace google { + +#include "glog/log_severity.h" +#include "glog/vlog_is_on.h" + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +// This is similar to LOG(severity) << format... and VLOG(level) << format.., +// but +// * it is to be used ONLY by low-level modules that can't use normal LOG() +// * it is desiged to be a low-level logger that does not allocate any +// memory and does not need any locks, hence: +// * it logs straight and ONLY to STDERR w/o buffering +// * it uses an explicit format and arguments list +// * it will silently chop off really long message strings +// Usage example: +// RAW_LOG(ERROR, "Failed foo with %i: %s", status, error); +// RAW_VLOG(3, "status is %i", status); +// These will print an almost standard log lines like this to stderr only: +// E0821 211317 file.cc:123] RAW: Failed foo with 22: bad_file +// I0821 211317 file.cc:142] RAW: status is 20 +#define RAW_LOG(severity, ...) \ + do { \ + switch (google::GLOG_ ## severity) { \ + case 0: \ + RAW_LOG_INFO(__VA_ARGS__); \ + break; \ + case 1: \ + RAW_LOG_WARNING(__VA_ARGS__); \ + break; \ + case 2: \ + RAW_LOG_ERROR(__VA_ARGS__); \ + break; \ + case 3: \ + RAW_LOG_FATAL(__VA_ARGS__); \ + break; \ + default: \ + break; \ + } \ + } while (0) + +// The following STRIP_LOG testing is performed in the header file so that it's +// possible to completely compile out the logging code and the log messages. +#if STRIP_LOG == 0 +#define RAW_VLOG(verboselevel, ...) \ + do { \ + if (VLOG_IS_ON(verboselevel)) { \ + RAW_LOG_INFO(__VA_ARGS__); \ + } \ + } while (0) +#else +#define RAW_VLOG(verboselevel, ...) RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG == 0 + +#if STRIP_LOG == 0 +#define RAW_LOG_INFO(...) google::RawLog__(google::GLOG_INFO, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_INFO(...) google::RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG == 0 + +#if STRIP_LOG <= 1 +#define RAW_LOG_WARNING(...) google::RawLog__(google::GLOG_WARNING, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_WARNING(...) google::RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG <= 1 + +#if STRIP_LOG <= 2 +#define RAW_LOG_ERROR(...) google::RawLog__(google::GLOG_ERROR, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_ERROR(...) google::RawLogStub__(0, __VA_ARGS__) +#endif // STRIP_LOG <= 2 + +#if STRIP_LOG <= 3 +#define RAW_LOG_FATAL(...) google::RawLog__(google::GLOG_FATAL, \ + __FILE__, __LINE__, __VA_ARGS__) +#else +#define RAW_LOG_FATAL(...) \ + do { \ + google::RawLogStub__(0, __VA_ARGS__); \ + exit(1); \ + } while (0) +#endif // STRIP_LOG <= 3 + +// Similar to CHECK(condition) << message, +// but for low-level modules: we use only RAW_LOG that does not allocate memory. +// We do not want to provide args list here to encourage this usage: +// if (!cond) RAW_LOG(FATAL, "foo ...", hard_to_compute_args); +// so that the args are not computed when not needed. +#define RAW_CHECK(condition, message) \ + do { \ + if (!(condition)) { \ + RAW_LOG(FATAL, "Check %s failed: %s", #condition, message); \ + } \ + } while (0) + +// Debug versions of RAW_LOG and RAW_CHECK +#ifndef NDEBUG + +#define RAW_DLOG(severity, ...) RAW_LOG(severity, __VA_ARGS__) +#define RAW_DCHECK(condition, message) RAW_CHECK(condition, message) + +#else // NDEBUG + +#define RAW_DLOG(severity, ...) \ + while (false) \ + RAW_LOG(severity, __VA_ARGS__) +#define RAW_DCHECK(condition, message) \ + while (false) \ + RAW_CHECK(condition, message) + +#endif // NDEBUG + +// Stub log function used to work around for unused variable warnings when +// building with STRIP_LOG > 0. +static inline void RawLogStub__(int /* ignored */, ...) { +} + +// Helper function to implement RAW_LOG and RAW_VLOG +// Logs format... at "severity" level, reporting it +// as called from file:line. +// This does not allocate memory or acquire locks. +GOOGLE_GLOG_DLL_DECL void RawLog__(LogSeverity severity, + const char* file, + int line, + const char* format, ...) + ; + +} + +#endif // BASE_RAW_LOGGING_H_ diff --git a/third_party/glog/src/windows/glog/stl_logging.h b/third_party/glog/src/windows/glog/stl_logging.h new file mode 100755 index 000000000000..a97a90895959 --- /dev/null +++ b/third_party/glog/src/windows/glog/stl_logging.h @@ -0,0 +1,224 @@ +// This file is automatically generated from src/glog/stl_logging.h.in +// using src/windows/preprocess.sh. +// DO NOT EDIT! + +// Copyright (c) 2003, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Stream output operators for STL containers; to be used for logging *only*. +// Inclusion of this file lets you do: +// +// list<string> x; +// LOG(INFO) << "data: " << x; +// vector<int> v1, v2; +// CHECK_EQ(v1, v2); +// +// If you want to use this header file with hash_compare maps or slist, you +// need to define macros before including this file: +// +// - GLOG_STL_LOGGING_FOR_UNORDERED - <unordered_map> and <unordered_set> +// - GLOG_STL_LOGGING_FOR_TR1_UNORDERED - <tr1/unordered_(map|set)> +// - GLOG_STL_LOGGING_FOR_EXT_HASH - <ext/hash_(map|set)> +// - GLOG_STL_LOGGING_FOR_EXT_SLIST - <ext/slist> +// + +#ifndef UTIL_GTL_STL_LOGGING_INL_H_ +#define UTIL_GTL_STL_LOGGING_INL_H_ + +#if !1 +# error We do not support stl_logging for this compiler +#endif + +#include <deque> +#include <list> +#include <map> +#include <ostream> +#include <set> +#include <utility> +#include <vector> + +#ifdef GLOG_STL_LOGGING_FOR_UNORDERED +# include <unordered_map> +# include <unordered_set> +#endif + +#ifdef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +# include <tr1/unordered_map> +# include <tr1/unordered_set> +#endif + +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +# include <ext/hash_set> +# include <ext/hash_map> +#endif +#ifdef GLOG_STL_LOGGING_FOR_EXT_SLIST +# include <ext/slist> +#endif + +// Forward declare these two, and define them after all the container streams +// operators so that we can recurse from pair -> container -> container -> pair +// properly. +template<class First, class Second> +std::ostream& operator<<(std::ostream& out, const std::pair<First, Second>& p); + +namespace google { + +template<class Iter> +void PrintSequence(std::ostream& out, Iter begin, Iter end); + +} + +#define OUTPUT_TWO_ARG_CONTAINER(Sequence) \ +template<class T1, class T2> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2>& seq) { \ + google::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +OUTPUT_TWO_ARG_CONTAINER(std::vector) +OUTPUT_TWO_ARG_CONTAINER(std::deque) +OUTPUT_TWO_ARG_CONTAINER(std::list) +#ifdef GLOG_STL_LOGGING_FOR_EXT_SLIST +OUTPUT_TWO_ARG_CONTAINER(__gnu_cxx::slist) +#endif + +#undef OUTPUT_TWO_ARG_CONTAINER + +#define OUTPUT_THREE_ARG_CONTAINER(Sequence) \ +template<class T1, class T2, class T3> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2, T3>& seq) { \ + google::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +OUTPUT_THREE_ARG_CONTAINER(std::set) +OUTPUT_THREE_ARG_CONTAINER(std::multiset) + +#undef OUTPUT_THREE_ARG_CONTAINER + +#define OUTPUT_FOUR_ARG_CONTAINER(Sequence) \ +template<class T1, class T2, class T3, class T4> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2, T3, T4>& seq) { \ + google::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +OUTPUT_FOUR_ARG_CONTAINER(std::map) +OUTPUT_FOUR_ARG_CONTAINER(std::multimap) +#ifdef GLOG_STL_LOGGING_FOR_UNORDERED +OUTPUT_FOUR_ARG_CONTAINER(std::unordered_set) +OUTPUT_FOUR_ARG_CONTAINER(std::unordered_multiset) +#endif +#ifdef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +OUTPUT_FOUR_ARG_CONTAINER(std::tr1::unordered_set) +OUTPUT_FOUR_ARG_CONTAINER(std::tr1::unordered_multiset) +#endif +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +OUTPUT_FOUR_ARG_CONTAINER(__gnu_cxx::hash_set) +OUTPUT_FOUR_ARG_CONTAINER(__gnu_cxx::hash_multiset) +#endif + +#undef OUTPUT_FOUR_ARG_CONTAINER + +#define OUTPUT_FIVE_ARG_CONTAINER(Sequence) \ +template<class T1, class T2, class T3, class T4, class T5> \ +inline std::ostream& operator<<(std::ostream& out, \ + const Sequence<T1, T2, T3, T4, T5>& seq) { \ + google::PrintSequence(out, seq.begin(), seq.end()); \ + return out; \ +} + +#ifdef GLOG_STL_LOGGING_FOR_UNORDERED +OUTPUT_FIVE_ARG_CONTAINER(std::unordered_map) +OUTPUT_FIVE_ARG_CONTAINER(std::unordered_multimap) +#endif +#ifdef GLOG_STL_LOGGING_FOR_TR1_UNORDERED +OUTPUT_FIVE_ARG_CONTAINER(std::tr1::unordered_map) +OUTPUT_FIVE_ARG_CONTAINER(std::tr1::unordered_multimap) +#endif +#ifdef GLOG_STL_LOGGING_FOR_EXT_HASH +OUTPUT_FIVE_ARG_CONTAINER(__gnu_cxx::hash_map) +OUTPUT_FIVE_ARG_CONTAINER(__gnu_cxx::hash_multimap) +#endif + +#undef OUTPUT_FIVE_ARG_CONTAINER + +template<class First, class Second> +inline std::ostream& operator<<(std::ostream& out, + const std::pair<First, Second>& p) { + out << '(' << p.first << ", " << p.second << ')'; + return out; +} + +namespace google { + +template<class Iter> +inline void PrintSequence(std::ostream& out, Iter begin, Iter end) { + // Output at most 100 elements -- appropriate if used for logging. + for (int i = 0; begin != end && i < 100; ++i, ++begin) { + if (i > 0) out << ' '; + out << *begin; + } + if (begin != end) { + out << " ..."; + } +} + +} + +// Note that this is technically undefined behavior! We are adding things into +// the std namespace for a reason though -- we are providing new operations on +// types which are themselves defined with this namespace. Without this, these +// operator overloads cannot be found via ADL. If these definitions are not +// found via ADL, they must be #included before they're used, which requires +// this header to be included before apparently independent other headers. +// +// For example, base/logging.h defines various template functions to implement +// CHECK_EQ(x, y) and stream x and y into the log in the event the check fails. +// It does so via the function template MakeCheckOpValueString: +// template<class T> +// void MakeCheckOpValueString(strstream* ss, const T& v) { +// (*ss) << v; +// } +// Because 'glog/logging.h' is included before 'glog/stl_logging.h', +// subsequent CHECK_EQ(v1, v2) for vector<...> typed variable v1 and v2 can only +// find these operator definitions via ADL. +// +// Even this solution has problems -- it may pull unintended operators into the +// namespace as well, allowing them to also be found via ADL, and creating code +// that only works with a particular order of includes. Long term, we need to +// move all of the *definitions* into namespace std, bet we need to ensure no +// one references them first. This lets us take that step. We cannot define them +// in both because that would create ambiguous overloads when both are found. +namespace std { using ::operator<<; } + +#endif // UTIL_GTL_STL_LOGGING_INL_H_ diff --git a/third_party/glog/src/windows/glog/vlog_is_on.h b/third_party/glog/src/windows/glog/vlog_is_on.h new file mode 100755 index 000000000000..409a4011b385 --- /dev/null +++ b/third_party/glog/src/windows/glog/vlog_is_on.h @@ -0,0 +1,133 @@ +// This file is automatically generated from src/glog/vlog_is_on.h.in +// using src/windows/preprocess.sh. +// DO NOT EDIT! + +// Copyright (c) 1999, 2007, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Author: Ray Sidney and many others +// +// Defines the VLOG_IS_ON macro that controls the variable-verbosity +// conditional logging. +// +// It's used by VLOG and VLOG_IF in logging.h +// and by RAW_VLOG in raw_logging.h to trigger the logging. +// +// It can also be used directly e.g. like this: +// if (VLOG_IS_ON(2)) { +// // do some logging preparation and logging +// // that can't be accomplished e.g. via just VLOG(2) << ...; +// } +// +// The truth value that VLOG_IS_ON(level) returns is determined by +// the three verbosity level flags: +// --v=<n> Gives the default maximal active V-logging level; +// 0 is the default. +// Normally positive values are used for V-logging levels. +// --vmodule=<str> Gives the per-module maximal V-logging levels to override +// the value given by --v. +// E.g. "my_module=2,foo*=3" would change the logging level +// for all code in source files "my_module.*" and "foo*.*" +// ("-inl" suffixes are also disregarded for this matching). +// +// SetVLOGLevel helper function is provided to do limited dynamic control over +// V-logging by overriding the per-module settings given via --vmodule flag. +// +// CAVEAT: --vmodule functionality is not available in non gcc compilers. +// + +#ifndef BASE_VLOG_IS_ON_H_ +#define BASE_VLOG_IS_ON_H_ + +#include "glog/log_severity.h" + +// Annoying stuff for windows -- makes sure clients can import these functions +#ifndef GOOGLE_GLOG_DLL_DECL +# if defined(_WIN32) && !defined(__CYGWIN__) +# define GOOGLE_GLOG_DLL_DECL __declspec(dllimport) +# else +# define GOOGLE_GLOG_DLL_DECL +# endif +#endif + +#if defined(__GNUC__) +// We emit an anonymous static int* variable at every VLOG_IS_ON(n) site. +// (Normally) the first time every VLOG_IS_ON(n) site is hit, +// we determine what variable will dynamically control logging at this site: +// it's either FLAGS_v or an appropriate internal variable +// matching the current source file that represents results of +// parsing of --vmodule flag and/or SetVLOGLevel calls. +#define VLOG_IS_ON(verboselevel) \ + __extension__ \ + ({ static google::int32* vlocal__ = &google::kLogSiteUninitialized; \ + google::int32 verbose_level__ = (verboselevel); \ + (*vlocal__ >= verbose_level__) && \ + ((vlocal__ != &google::kLogSiteUninitialized) || \ + (google::InitVLOG3__(&vlocal__, &FLAGS_v, \ + __FILE__, verbose_level__))); }) +#else +// GNU extensions not available, so we do not support --vmodule. +// Dynamic value of FLAGS_v always controls the logging level. +#define VLOG_IS_ON(verboselevel) (FLAGS_v >= (verboselevel)) +#endif + +// Set VLOG(_IS_ON) level for module_pattern to log_level. +// This lets us dynamically control what is normally set by the --vmodule flag. +// Returns the level that previously applied to module_pattern. +// NOTE: To change the log level for VLOG(_IS_ON) sites +// that have already executed after/during InitGoogleLogging, +// one needs to supply the exact --vmodule pattern that applied to them. +// (If no --vmodule pattern applied to them +// the value of FLAGS_v will continue to control them.) +extern GOOGLE_GLOG_DLL_DECL int SetVLOGLevel(const char* module_pattern, + int log_level); + +// Various declarations needed for VLOG_IS_ON above: ========================= + +// Special value used to indicate that a VLOG_IS_ON site has not been +// initialized. We make this a large value, so the common-case check +// of "*vlocal__ >= verbose_level__" in VLOG_IS_ON definition +// passes in such cases and InitVLOG3__ is then triggered. +extern google::int32 kLogSiteUninitialized; + +// Helper routine which determines the logging info for a particalur VLOG site. +// site_flag is the address of the site-local pointer to the controlling +// verbosity level +// site_default is the default to use for *site_flag +// fname is the current source file name +// verbose_level is the argument to VLOG_IS_ON +// We will return the return value for VLOG_IS_ON +// and if possible set *site_flag appropriately. +extern GOOGLE_GLOG_DLL_DECL bool InitVLOG3__( + google::int32** site_flag, + google::int32* site_default, + const char* fname, + google::int32 verbose_level); + +#endif // BASE_VLOG_IS_ON_H_ diff --git a/third_party/glog/src/windows/port.cc b/third_party/glog/src/windows/port.cc new file mode 100755 index 000000000000..19bda367c62b --- /dev/null +++ b/third_party/glog/src/windows/port.cc @@ -0,0 +1,65 @@ +/* Copyright (c) 2008, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Craig Silverstein + * Copied from google-perftools and modified by Shinichiro Hamaji + */ + +#ifndef _WIN32 +# error You should only be including windows/port.cc in a windows environment! +#endif + +#include "config.h" +#include <stdarg.h> // for va_list, va_start, va_end +#include "port.h" + +// These call the windows _vsnprintf, but always NUL-terminate. +int safe_vsnprintf(char *str, size_t size, const char *format, va_list ap) { + if (size == 0) // not even room for a \0? + return -1; // not what C99 says to do, but what windows does + str[size-1] = '\0'; + return _vsnprintf(str, size-1, format, ap); +} + +#ifndef HAVE_LOCALTIME_R +struct tm* localtime_r(const time_t* timep, struct tm* result) { + localtime_s(result, timep); + return result; +} +#endif // not HAVE_LOCALTIME_R +#ifndef HAVE_SNPRINTF +int snprintf(char *str, size_t size, const char *format, ...) { + va_list ap; + va_start(ap, format); + const int r = vsnprintf(str, size, format, ap); + va_end(ap); + return r; +} +#endif diff --git a/third_party/glog/src/windows/port.h b/third_party/glog/src/windows/port.h new file mode 100755 index 000000000000..7b4b9c8545b8 --- /dev/null +++ b/third_party/glog/src/windows/port.h @@ -0,0 +1,174 @@ +/* Copyright (c) 2008, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * --- + * Author: Craig Silverstein + * Copied from google-perftools and modified by Shinichiro Hamaji + * + * These are some portability typedefs and defines to make it a bit + * easier to compile this code under VC++. + * + * Several of these are taken from glib: + * http://developer.gnome.org/doc/API/glib/glib-windows-compatability-functions.html + */ + +#ifndef CTEMPLATE_WINDOWS_PORT_H_ +#define CTEMPLATE_WINDOWS_PORT_H_ + +#include "config.h" + +#ifdef _WIN32 + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN /* We always want minimal includes */ +#endif + +#include <windows.h> +#include <winsock.h> /* for gethostname */ +#include <io.h> /* because we so often use open/close/etc */ +#include <direct.h> /* for _getcwd() */ +#include <process.h> /* for _getpid() */ +#include <stdio.h> /* read in vsnprintf decl. before redifining it */ +#include <stdarg.h> /* template_dictionary.cc uses va_copy */ +#include <string.h> /* for _strnicmp(), strerror_s() */ +#include <time.h> /* for localtime_s() */ +/* Note: the C++ #includes are all together at the bottom. This file is + * used by both C and C++ code, so we put all the C++ together. + */ + +#ifdef _MSC_VER + +/* 4244: otherwise we get problems when substracting two size_t's to an int + * 4251: it's complaining about a private struct I've chosen not to dllexport + * 4355: we use this in a constructor, but we do it safely + * 4715: for some reason VC++ stopped realizing you can't return after abort() + * 4800: we know we're casting ints/char*'s to bools, and we're ok with that + * 4996: Yes, we're ok using "unsafe" functions like fopen() and strerror() + * 4312: Converting uint32_t to a pointer when testing %p + * 4267: also subtracting two size_t to int + * 4722: Destructor never returns due to abort() + */ +#pragma warning(disable:4244 4251 4355 4715 4800 4996 4267 4312 4722) + +/* file I/O */ +#define PATH_MAX 1024 +#define access _access +#define getcwd _getcwd +#define open _open +#define read _read +#define write _write +#define lseek _lseek +#define close _close +#define popen _popen +#define pclose _pclose +#define R_OK 04 /* read-only (for access()) */ +#define S_ISDIR(m) (((m) & _S_IFMT) == _S_IFDIR) + +#define O_WRONLY _O_WRONLY +#define O_CREAT _O_CREAT +#define O_EXCL _O_EXCL + +#ifndef __MINGW32__ +enum { STDIN_FILENO = 0, STDOUT_FILENO = 1, STDERR_FILENO = 2 }; +#endif +#define S_IRUSR S_IREAD +#define S_IWUSR S_IWRITE + +/* Not quite as lightweight as a hard-link, but more than good enough for us. */ +#define link(oldpath, newpath) CopyFileA(oldpath, newpath, false) + +#define strcasecmp _stricmp +#define strncasecmp _strnicmp + +/* In windows-land, hash<> is called hash_compare<> (from xhash.h) */ +/* VC11 provides std::hash */ +#if defined(_MSC_VER) && (_MSC_VER < 1700) +#define hash hash_compare +#endif + +/* Sleep is in ms, on windows */ +#define sleep(secs) Sleep((secs) * 1000) + +/* We can't just use _vsnprintf and _snprintf as drop-in-replacements, + * because they don't always NUL-terminate. :-( We also can't use the + * name vsnprintf, since windows defines that (but not snprintf (!)). + */ +#ifndef HAVE_SNPRINTF +extern int GOOGLE_GLOG_DLL_DECL snprintf(char *str, size_t size, + const char *format, ...); +#endif +extern int GOOGLE_GLOG_DLL_DECL safe_vsnprintf(char *str, size_t size, + const char *format, va_list ap); +#define vsnprintf(str, size, format, ap) safe_vsnprintf(str, size, format, ap) +#ifndef va_copy +#define va_copy(dst, src) (dst) = (src) +#endif + +/* Windows doesn't support specifying the number of buckets as a + * hash_map constructor arg, so we leave this blank. + */ +#define CTEMPLATE_SMALL_HASHTABLE + +#define DEFAULT_TEMPLATE_ROOTDIR ".." + +// ----------------------------------- SYSTEM/PROCESS +typedef int pid_t; +#define getpid _getpid + +#endif // _MSC_VER + +// ----------------------------------- THREADS +#if defined(HAVE_PTHREAD) +# include <pthread.h> +#else // no PTHREAD +typedef DWORD pthread_t; +typedef DWORD pthread_key_t; +typedef LONG pthread_once_t; +enum { PTHREAD_ONCE_INIT = 0 }; // important that this be 0! for SpinLock +#define pthread_self GetCurrentThreadId +#define pthread_equal(pthread_t_1, pthread_t_2) ((pthread_t_1)==(pthread_t_2)) +#endif // HAVE_PTHREAD + +#ifndef HAVE_LOCALTIME_R +extern GOOGLE_GLOG_DLL_DECL struct tm* localtime_r(const time_t* timep, struct tm* result); +#endif // not HAVE_LOCALTIME_R + +inline char* strerror_r(int errnum, char* buf, size_t buflen) { + strerror_s(buf, buflen, errnum); + return buf; +} + +#ifndef __cplusplus +/* I don't see how to get inlining for C code in MSVC. Ah well. */ +#define inline +#endif + +#endif /* _WIN32 */ + +#endif /* CTEMPLATE_WINDOWS_PORT_H_ */ diff --git a/third_party/glog/src/windows/preprocess.sh b/third_party/glog/src/windows/preprocess.sh new file mode 100755 index 000000000000..c35e92913b65 --- /dev/null +++ b/third_party/glog/src/windows/preprocess.sh @@ -0,0 +1,119 @@ +#!/bin/sh + +# Copyright (c) 2008, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# --- +# Author: Craig Silverstein +# Copied from google-perftools and modified by Shinichiro Hamaji +# +# This script is meant to be run at distribution-generation time, for +# instance by autogen.sh. It does some of the work configure would +# normally do, for windows systems. In particular, it expands all the +# @...@ variables found in .in files, and puts them here, in the windows +# directory. +# +# This script should be run before any new release. + +if [ -z "$1" ]; then + echo "USAGE: $0 <src/ directory>" + exit 1 +fi + +DLLDEF_MACRO_NAME="GLOG_DLL_DECL" + +# The text we put in every .h files we create. As a courtesy, we'll +# include a helpful comment for windows users as to how to use +# GLOG_DLL_DECL. Apparently sed expands \n into a newline. Good! +DLLDEF_DEFINES="\ +// NOTE: if you are statically linking the template library into your binary\n\ +// (rather than using the template .dll), set '/D $DLLDEF_MACRO_NAME='\n\ +// as a compiler flag in your project file to turn off the dllimports.\n\ +#ifndef $DLLDEF_MACRO_NAME\n\ +# define $DLLDEF_MACRO_NAME __declspec(dllimport)\n\ +#endif" + +# Read all the windows config info into variables +# In order for the 'set' to take, this requires putting all in a subshell. +( + while read define varname value; do + [ "$define" != "#define" ] && continue + eval "$varname='$value'" + done + + # Process all the .in files in the "glog" subdirectory + mkdir -p "$1/windows/glog" + for file in `echo "$1"/glog/*.in`; do + echo "Processing $file" + outfile="$1/windows/glog/`basename $file .in`" + + echo "\ +// This file is automatically generated from $file +// using src/windows/preprocess.sh. +// DO NOT EDIT! +" > "$outfile" + # Besides replacing @...@, we also need to turn on dllimport + # We also need to replace hash by hash_compare (annoying we hard-code :-( ) + sed -e "s!@ac_windows_dllexport@!$DLLDEF_MACRO_NAME!g" \ + -e "s!@ac_windows_dllexport_defines@!$DLLDEF_DEFINES!g" \ + -e "s!@ac_cv_cxx_hash_map@!$HASH_MAP_H!g" \ + -e "s!@ac_cv_cxx_hash_namespace@!$HASH_NAMESPACE!g" \ + -e "s!@ac_cv_cxx_hash_set@!$HASH_SET_H!g" \ + -e "s!@ac_cv_have_stdint_h@!0!g" \ + -e "s!@ac_cv_have_systypes_h@!0!g" \ + -e "s!@ac_cv_have_inttypes_h@!0!g" \ + -e "s!@ac_cv_have_unistd_h@!0!g" \ + -e "s!@ac_cv_have_uint16_t@!0!g" \ + -e "s!@ac_cv_have_u_int16_t@!0!g" \ + -e "s!@ac_cv_have___uint16@!1!g" \ + -e "s!@ac_cv_have_libgflags@!0!g" \ + -e "s!@ac_cv_have___builtin_expect@!0!g" \ + -e "s!@ac_cv_cxx_using_operator@!1!g" \ + -e "s!@ac_cv___attribute___noreturn@!__declspec(noreturn)!g" \ + -e "s!@ac_cv___attribute___noinline@!!g" \ + -e "s!@ac_cv___attribute___printf_4_5@!!g" \ + -e "s!@ac_google_attribute@!${HAVE___ATTRIBUTE__:-0}!g" \ + -e "s!@ac_google_end_namespace@!$_END_GOOGLE_NAMESPACE_!g" \ + -e "s!@ac_google_namespace@!$GOOGLE_NAMESPACE!g" \ + -e "s!@ac_google_start_namespace@!$_START_GOOGLE_NAMESPACE_!g" \ + -e "s!@ac_htmlparser_namespace@!$HTMLPARSER_NAMESPACE!g" \ + -e "s!\\bhash\\b!hash_compare!g" \ + "$file" >> "$outfile" + done +) < "$1/windows/config.h" + +# log_severity.h isn't a .in file. +echo "\ +// This file is automatically generated from $1/glog/log_severity.h +// using src/windows/preprocess.sh. +// DO NOT EDIT! +" > "$1/windows/glog/log_severity.h" +cat "$1/glog/log_severity.h" >> "$1/windows/glog/log_severity.h" + +echo "DONE" diff --git a/third_party/glog/toolchains/.gitignore b/third_party/glog/toolchains/.gitignore new file mode 100644 index 000000000000..13489ce3b5db --- /dev/null +++ b/third_party/glog/toolchains/.gitignore @@ -0,0 +1,2 @@ +# allow toolchain files to be tracked by git +!*.cmake diff --git a/third_party/glog/toolchains/clang-cxx17.cmake b/third_party/glog/toolchains/clang-cxx17.cmake new file mode 100644 index 000000000000..09d077a38d7c --- /dev/null +++ b/third_party/glog/toolchains/clang-cxx17.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/gcc.cmake" + +# set compiler +set(CMAKE_C_COMPILER clang) +set(CMAKE_CXX_COMPILER clang++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/gcc-cxx11.cmake b/third_party/glog/toolchains/gcc-cxx11.cmake new file mode 100644 index 000000000000..da1ab5778a27 --- /dev/null +++ b/third_party/glog/toolchains/gcc-cxx11.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/gcc.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/gcc-cxx17.cmake b/third_party/glog/toolchains/gcc-cxx17.cmake new file mode 100644 index 000000000000..1b834f12015b --- /dev/null +++ b/third_party/glog/toolchains/gcc-cxx17.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/gcc.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/gcc-cxx98.cmake b/third_party/glog/toolchains/gcc-cxx98.cmake new file mode 100644 index 000000000000..5697ab252aae --- /dev/null +++ b/third_party/glog/toolchains/gcc-cxx98.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/gcc.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 98) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/gcc-gnuxx11.cmake b/third_party/glog/toolchains/gcc-gnuxx11.cmake new file mode 100644 index 000000000000..91f0e4a3ce76 --- /dev/null +++ b/third_party/glog/toolchains/gcc-gnuxx11.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/gcc.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS ON) diff --git a/third_party/glog/toolchains/linux-mingw-w64-cxx11.cmake b/third_party/glog/toolchains/linux-mingw-w64-cxx11.cmake new file mode 100644 index 000000000000..9c6e126a68d2 --- /dev/null +++ b/third_party/glog/toolchains/linux-mingw-w64-cxx11.cmake @@ -0,0 +1,33 @@ +# Sample toolchain file for building for Windows from an Ubuntu Linux system. +# +# Typical usage: +# *) install cross compiler: `sudo apt-get install mingw-w64` +# *) cmake -H. -B_build_mingw -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/linux-mingw-w64.cmake" + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) + +# set compiler +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + +# target environment on the build host system +# set 1st to dir with the cross compiler's C/C++ headers/libs +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + +# modify default behavior of FIND_XXX() commands to +# search for headers/libs in the target environment and +# search for programs in the build host environment +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +# use emulator for `try_run` calls +set(CMAKE_CROSSCOMPILING_EMULATOR wine64) + +# set c++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/linux-mingw-w64-cxx17.cmake b/third_party/glog/toolchains/linux-mingw-w64-cxx17.cmake new file mode 100644 index 000000000000..1c363f10eb10 --- /dev/null +++ b/third_party/glog/toolchains/linux-mingw-w64-cxx17.cmake @@ -0,0 +1,33 @@ +# Sample toolchain file for building for Windows from an Ubuntu Linux system. +# +# Typical usage: +# *) install cross compiler: `sudo apt-get install mingw-w64` +# *) cmake -H. -B_build_mingw -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/linux-mingw-w64.cmake" + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) + +# set compiler +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + +# target environment on the build host system +# set 1st to dir with the cross compiler's C/C++ headers/libs +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + +# modify default behavior of FIND_XXX() commands to +# search for headers/libs in the target environment and +# search for programs in the build host environment +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +# use emulator for `try_run` calls +set(CMAKE_CROSSCOMPILING_EMULATOR wine64) + +# set c++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/linux-mingw-w64-gnuxx11.cmake b/third_party/glog/toolchains/linux-mingw-w64-gnuxx11.cmake new file mode 100644 index 000000000000..99029649dfa3 --- /dev/null +++ b/third_party/glog/toolchains/linux-mingw-w64-gnuxx11.cmake @@ -0,0 +1,33 @@ +# Sample toolchain file for building for Windows from an Ubuntu Linux system. +# +# Typical usage: +# *) install cross compiler: `sudo apt-get install mingw-w64` +# *) cmake -H. -B_build_mingw -DCMAKE_TOOLCHAIN_FILE="${PWD}/toolchains/linux-mingw-w64.cmake" + +set(CMAKE_SYSTEM_NAME Windows) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(TOOLCHAIN_PREFIX x86_64-w64-mingw32) + +# set compiler +set(CMAKE_C_COMPILER ${TOOLCHAIN_PREFIX}-gcc) +set(CMAKE_CXX_COMPILER ${TOOLCHAIN_PREFIX}-g++) +set(CMAKE_RC_COMPILER ${TOOLCHAIN_PREFIX}-windres) + +# target environment on the build host system +# set 1st to dir with the cross compiler's C/C++ headers/libs +set(CMAKE_FIND_ROOT_PATH /usr/${TOOLCHAIN_PREFIX}) + +# modify default behavior of FIND_XXX() commands to +# search for headers/libs in the target environment and +# search for programs in the build host environment +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + +# use emulator for `try_run` calls +set(CMAKE_CROSSCOMPILING_EMULATOR wine64) + +# set c++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS ON) diff --git a/third_party/glog/toolchains/mingw-cxx11.cmake b/third_party/glog/toolchains/mingw-cxx11.cmake new file mode 100644 index 000000000000..873f623f6011 --- /dev/null +++ b/third_party/glog/toolchains/mingw-cxx11.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="%cd%\toolchains\mingw.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/mingw-cxx17.cmake b/third_party/glog/toolchains/mingw-cxx17.cmake new file mode 100644 index 000000000000..c0c89cc459f4 --- /dev/null +++ b/third_party/glog/toolchains/mingw-cxx17.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="%cd%\toolchains\mingw.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) diff --git a/third_party/glog/toolchains/mingw-gnuxx11.cmake b/third_party/glog/toolchains/mingw-gnuxx11.cmake new file mode 100644 index 000000000000..66337050c92f --- /dev/null +++ b/third_party/glog/toolchains/mingw-gnuxx11.cmake @@ -0,0 +1,13 @@ +# Sample toolchain file for building with gcc compiler +# +# Typical usage: +# *) cmake -H. -B_build -DCMAKE_TOOLCHAIN_FILE="%cd%\toolchains\mingw.cmake" + +# set compiler +set(CMAKE_C_COMPILER gcc) +set(CMAKE_CXX_COMPILER g++) + +# set c++ standard +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS YES) diff --git a/third_party/glog/toolchains/vs-14-2015-sdk-8-1.cmake b/third_party/glog/toolchains/vs-14-2015-sdk-8-1.cmake new file mode 100644 index 000000000000..463bee052386 --- /dev/null +++ b/third_party/glog/toolchains/vs-14-2015-sdk-8-1.cmake @@ -0,0 +1,2 @@ +# set c++ standard +set(CMAKE_SYSTEM_VERSION 8.1) diff --git a/third_party/glog/toolchains/vs-14-2015-win64.cmake b/third_party/glog/toolchains/vs-14-2015-win64.cmake new file mode 100644 index 000000000000..00beda25efd4 --- /dev/null +++ b/third_party/glog/toolchains/vs-14-2015-win64.cmake @@ -0,0 +1 @@ +# dummy, nothing extra to set diff --git a/third_party/glog/toolchains/vs-15-2017-win64-cxx17.cmake b/third_party/glog/toolchains/vs-15-2017-win64-cxx17.cmake new file mode 100644 index 000000000000..5e4e936872d9 --- /dev/null +++ b/third_party/glog/toolchains/vs-15-2017-win64-cxx17.cmake @@ -0,0 +1,2 @@ +# set c++ standard +set(CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS_INIT} /std:c++17") diff --git a/third_party/glog/toolchains/vs-15-2017-win64.cmake b/third_party/glog/toolchains/vs-15-2017-win64.cmake new file mode 100644 index 000000000000..00beda25efd4 --- /dev/null +++ b/third_party/glog/toolchains/vs-15-2017-win64.cmake @@ -0,0 +1 @@ +# dummy, nothing extra to set diff --git a/third_party/gopkgs/cloud.google.com/go/default.nix b/third_party/gopkgs/cloud.google.com/go/default.nix new file mode 100644 index 000000000000..876585f2d14b --- /dev/null +++ b/third_party/gopkgs/cloud.google.com/go/default.nix @@ -0,0 +1,9 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "cloud.google.com/go"; + src = builtins.fetchGit { + url = "https://code.googlesource.com/gocloud"; + rev = "4f03f8e4ba168c636e1c218da7ab41a1c8c0d8cf"; + }; +} diff --git a/third_party/gopkgs/github.com/cenkalti/backoff/default.nix b/third_party/gopkgs/github.com/cenkalti/backoff/default.nix new file mode 100644 index 000000000000..c0e0335de734 --- /dev/null +++ b/third_party/gopkgs/github.com/cenkalti/backoff/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/cenkalti/backoff/v4"; + + src = pkgs.fetchFromGitHub { + owner = "cenkalti"; + repo = "backoff"; + rev = "18fe4ce5a8550e0d0919b680ad3c080a5455bddf"; + sha256 = "083617p066p77ik0js8wwgb5qzabgvl8wqpkjb8s9alpyqsq2mpg"; + }; +} diff --git a/third_party/gopkgs/github.com/charmbracelet/bubbles/default.nix b/third_party/gopkgs/github.com/charmbracelet/bubbles/default.nix new file mode 100644 index 000000000000..e041edd4b698 --- /dev/null +++ b/third_party/gopkgs/github.com/charmbracelet/bubbles/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/charmbracelet/bubbles"; + src = pkgs.fetchFromGitHub { + owner = "charmbracelet"; + repo = "bubbles"; + # unreleased version required by bubbletea + rev = "v0.7.6"; + sha256 = "1gd4k4f2mj2dnqcbpdrh9plziz0l29ls6mgyy4mfdcdfijfyd30n"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".charmbracelet.bubbletea + ]; +} diff --git a/third_party/gopkgs/github.com/charmbracelet/bubbletea/default.nix b/third_party/gopkgs/github.com/charmbracelet/bubbletea/default.nix new file mode 100644 index 000000000000..75eb5402c17c --- /dev/null +++ b/third_party/gopkgs/github.com/charmbracelet/bubbletea/default.nix @@ -0,0 +1,28 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/charmbracelet/bubbletea"; + src = + let gitSrc = pkgs.fetchFromGitHub { + owner = "charmbracelet"; + repo = "bubbletea"; + rev = "v0.13.1"; + sha256 = "0yf2fjkvx8ym9n6f3qp2z7sxs0qsfpj148sfvbrp38k67s3h20cs"; + }; + # The examples/ directory is fairly extensive, + # but it also adds most of the dependencies. + in pkgs.runCommand gitSrc.name {} '' + mkdir -p $out + ln -s "${gitSrc}"/* $out + rm -r $out/examples + rm -r $out/tutorials + ''; + deps = with depot.third_party; [ + gopkgs."github.com".containerd.console + gopkgs."github.com".mattn.go-isatty + gopkgs."github.com".muesli.reflow.truncate + gopkgs."github.com".muesli.termenv + gopkgs."golang.org".x.sys.unix + gopkgs."golang.org".x.crypto.ssh.terminal + ]; +} diff --git a/third_party/gopkgs/github.com/charmbracelet/lipgloss/default.nix b/third_party/gopkgs/github.com/charmbracelet/lipgloss/default.nix new file mode 100644 index 000000000000..d10332a1d572 --- /dev/null +++ b/third_party/gopkgs/github.com/charmbracelet/lipgloss/default.nix @@ -0,0 +1,21 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/charmbracelet/lipgloss"; + src = pkgs.fetchFromGitHub { + owner = "charmbracelet"; + repo = "lipgloss"; + # unreleased version required by bubbletea + rev = "v0.1.0"; + sha256 = "1chhs492rsq7i4mr6qpjv3d89rvsd23ri6psnmil3ah6i286vl06"; + }; + + deps = with depot.third_party; [ + # gopkgs."github.com".charmbracelet.bubbletea + gopkgs."github.com".lucasb-eyer.go-colorful + gopkgs."github.com".muesli.reflow.ansi + gopkgs."github.com".muesli.reflow.truncate + gopkgs."github.com".muesli.reflow.wordwrap + gopkgs."github.com".muesli.termenv + ]; +} diff --git a/third_party/gopkgs/github.com/containerd/console/default.nix b/third_party/gopkgs/github.com/containerd/console/default.nix new file mode 100644 index 000000000000..3f451019e0fd --- /dev/null +++ b/third_party/gopkgs/github.com/containerd/console/default.nix @@ -0,0 +1,15 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/containerd/console"; + src = pkgs.fetchFromGitHub { + owner = "containerd"; + repo = "console"; + rev = "v1.0.1"; + sha256 = "0s837wj6h80fykk2pdmaji75rw9c3863by0gh0cq51hh0lgyjpvg"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.sys.unix + ]; +} diff --git a/third_party/gopkgs/github.com/davecgh/go-spew/default.nix b/third_party/gopkgs/github.com/davecgh/go-spew/default.nix new file mode 100644 index 000000000000..1395f3dce699 --- /dev/null +++ b/third_party/gopkgs/github.com/davecgh/go-spew/default.nix @@ -0,0 +1,11 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/davecgh/go-spew"; + src = pkgs.fetchFromGitHub { + owner = "davecgh"; + repo = "go-spew"; + rev = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"; + sha256 = "0hka6hmyvp701adzag2g26cxdj47g21x6jz4sc6jjz1mn59d474y"; + }; +} diff --git a/third_party/gopkgs/github.com/emirpasic/gods/default.nix b/third_party/gopkgs/github.com/emirpasic/gods/default.nix new file mode 100644 index 000000000000..a858660f4335 --- /dev/null +++ b/third_party/gopkgs/github.com/emirpasic/gods/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/emirpasic/gods"; + + src = pkgs.fetchFromGitHub { + owner = "emirpasic"; + repo = "gods"; + rev = "4e23915b9a82f35f320a68a395a7a5045c826932"; + sha256 = "00f8ch1rccakc62f9nj97hapvnx84z7wbcdmbmz7p802b9mxk5nl"; + }; +} diff --git a/third_party/gopkgs/github.com/golang/glog/default.nix b/third_party/gopkgs/github.com/golang/glog/default.nix new file mode 100644 index 000000000000..c8426f2b1a2f --- /dev/null +++ b/third_party/gopkgs/github.com/golang/glog/default.nix @@ -0,0 +1,11 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/golang/glog"; + src = pkgs.fetchFromGitHub { + owner = "golang"; + repo = "glog"; + rev = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"; + sha256 = "0jb2834rw5sykfr937fxi8hxi2zy80sj2bdn9b3jb4b26ksqng30"; + }; +} diff --git a/third_party/gopkgs/github.com/golang/groupcache/default.nix b/third_party/gopkgs/github.com/golang/groupcache/default.nix new file mode 100644 index 000000000000..4c12173ed960 --- /dev/null +++ b/third_party/gopkgs/github.com/golang/groupcache/default.nix @@ -0,0 +1,9 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "github.com/golang/groupcache"; + src = builtins.fetchGit { + url = "https://github.com/golang/groupcache"; + rev = "611e8accdfc92c4187d399e95ce826046d4c8d73"; + }; +} diff --git a/third_party/gopkgs/github.com/golang/protobuf/default.nix b/third_party/gopkgs/github.com/golang/protobuf/default.nix new file mode 100644 index 000000000000..5a4290dec2a1 --- /dev/null +++ b/third_party/gopkgs/github.com/golang/protobuf/default.nix @@ -0,0 +1,9 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "github.com/golang/protobuf"; + src = builtins.fetchGit { + url = "https://github.com/golang/protobuf"; + rev = "ed6926b37a637426117ccab59282c3839528a700"; + }; +} diff --git a/third_party/gopkgs/github.com/google/uuid/default.nix b/third_party/gopkgs/github.com/google/uuid/default.nix new file mode 100644 index 000000000000..191863bf0eab --- /dev/null +++ b/third_party/gopkgs/github.com/google/uuid/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/google/uuid"; + + src = pkgs.fetchFromGitHub { + owner = "google"; + repo = "uuid"; + rev = "c2e93f3ae59f2904160ceaab466009f965df46d6"; + sha256 = "0zw8fvl6jqg0fmv6kmvhss0g4gkrbvgyvl2zgy5wdbdlgp4fja0h"; + }; +} diff --git a/third_party/gopkgs/github.com/googleapis/gax-go/default.nix b/third_party/gopkgs/github.com/googleapis/gax-go/default.nix new file mode 100644 index 000000000000..4c2d8dbd39a6 --- /dev/null +++ b/third_party/gopkgs/github.com/googleapis/gax-go/default.nix @@ -0,0 +1,19 @@ +{ depot, ... }: + +let + inherit (depot) buildGo; + inherit (builtins) fetchGit; +in depot.nix.buildGo.external { + path = "github.com/googleapis/gax-go"; + src = fetchGit { + url = "https://github.com/googleapis/gax-go"; + rev = "b443e5a67ec8eeac76f5f384004931878cab24b3"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.net.trace.gopkg + gopkgs."google.golang.org".grpc.gopkg + gopkgs."google.golang.org".grpc.codes.gopkg + gopkgs."google.golang.org".grpc.status.gopkg + ]; +} diff --git a/third_party/gopkgs/github.com/hashicorp/golang-lru/default.nix b/third_party/gopkgs/github.com/hashicorp/golang-lru/default.nix new file mode 100644 index 000000000000..417826adab51 --- /dev/null +++ b/third_party/gopkgs/github.com/hashicorp/golang-lru/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "github.com/hashicorp/golang-lru"; + src = builtins.fetchGit { + url = "https://github.com/hashicorp/golang-lru"; + rev = "7f827b33c0f158ec5dfbba01bb0b14a4541fd81d"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.net.context.ctxhttp + gopkgs."cloud.google.com".go.compute.metadata + ]; +} diff --git a/third_party/gopkgs/github.com/jbenet/go-context/default.nix b/third_party/gopkgs/github.com/jbenet/go-context/default.nix new file mode 100644 index 000000000000..401fc6eb40e7 --- /dev/null +++ b/third_party/gopkgs/github.com/jbenet/go-context/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/jbenet/go-context"; + + src = pkgs.fetchFromGitHub { + owner = "jbenet"; + repo = "go-context"; + rev = "d14ea06fba99483203c19d92cfcd13ebe73135f4"; + sha256 = "0q91f5549n81w3z5927n4a1mdh220bdmgl42zi3h992dcc4ls0sl"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.net.context + ]; +} diff --git a/third_party/gopkgs/github.com/kevinburke/ssh_config/default.nix b/third_party/gopkgs/github.com/kevinburke/ssh_config/default.nix new file mode 100644 index 000000000000..6b4e8f5275bc --- /dev/null +++ b/third_party/gopkgs/github.com/kevinburke/ssh_config/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/kevinburke/ssh_config"; + + src = pkgs.fetchFromGitHub { + owner = "kevinburke"; + repo = "ssh_config"; + rev = "01f96b0aa0cdcaa93f9495f89bbc6cb5a992ce6e"; + sha256 = "1bxfjkjl3ibzdkwyvgdwawmd0skz30ah1ha10rg6fkxvj7lgg4jz"; + }; +} diff --git a/third_party/gopkgs/github.com/lucasb-eyer/go-colorful/default.nix b/third_party/gopkgs/github.com/lucasb-eyer/go-colorful/default.nix new file mode 100644 index 000000000000..decb7f3db944 --- /dev/null +++ b/third_party/gopkgs/github.com/lucasb-eyer/go-colorful/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/lucasb-eyer/go-colorful"; + src = pkgs.fetchFromGitHub { + owner = "lucasb-eyer"; + repo = "go-colorful"; + # unreleased version required by bubbletea + rev = "v1.2.0"; + sha256 = "08c3fkf27r16izjjd4w94xd1z7w1r4mdalbl53ms2ka2j465s3qs"; + }; +} diff --git a/third_party/gopkgs/github.com/mattn/go-isatty/default.nix b/third_party/gopkgs/github.com/mattn/go-isatty/default.nix new file mode 100644 index 000000000000..6ba12afff763 --- /dev/null +++ b/third_party/gopkgs/github.com/mattn/go-isatty/default.nix @@ -0,0 +1,15 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/mattn/go-isatty"; + src = pkgs.fetchFromGitHub { + owner = "mattn"; + repo = "go-isatty"; + rev = "v0.0.12"; + sha256 = "1dfsh27d52wmz0nmmzm2382pfrs2fcijvh6cgir7jbb4pnigr5w4"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.sys.unix + ]; +} diff --git a/third_party/gopkgs/github.com/mattn/go-runewidth/default.nix b/third_party/gopkgs/github.com/mattn/go-runewidth/default.nix new file mode 100644 index 000000000000..3186a066291d --- /dev/null +++ b/third_party/gopkgs/github.com/mattn/go-runewidth/default.nix @@ -0,0 +1,15 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/mattn/go-runewidth"; + src = pkgs.fetchFromGitHub { + owner = "mattn"; + repo = "go-runewidth"; + rev = "v0.0.10"; + sha256 = "0jh9552ppqvkdfni7x623n0x5mbiaqqhjhmr0zkh28x56k4ysii4"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".rivo.uniseg + ]; +} diff --git a/third_party/gopkgs/github.com/mitchellh/go-homedir/default.nix b/third_party/gopkgs/github.com/mitchellh/go-homedir/default.nix new file mode 100644 index 000000000000..8c593eaae8c5 --- /dev/null +++ b/third_party/gopkgs/github.com/mitchellh/go-homedir/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/mitchellh/go-homedir"; + + src = pkgs.fetchFromGitHub { + owner = "mitchellh"; + repo = "go-homedir"; + rev = "af06845cf3004701891bf4fdb884bfe4920b3727"; + sha256 = "0ydzkipf28hwj2bfxqmwlww47khyk6d152xax4bnyh60f4lq3nx1"; + }; +} diff --git a/third_party/gopkgs/github.com/muesli/reflow/default.nix b/third_party/gopkgs/github.com/muesli/reflow/default.nix new file mode 100644 index 000000000000..c7c50795c017 --- /dev/null +++ b/third_party/gopkgs/github.com/muesli/reflow/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/muesli/reflow"; + src = pkgs.fetchFromGitHub { + owner = "muesli"; + repo = "reflow"; + # unreleased version required by bubbletea + rev = "9e1d0d53df68baf262851201166872afafd04e5d"; + sha256 = "08bmkqdn7sb5laqc1mvgk4xj31f600n1y04s1ifppjvszbcsxhid"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".mattn.go-runewidth + ]; +} diff --git a/third_party/gopkgs/github.com/muesli/termenv/default.nix b/third_party/gopkgs/github.com/muesli/termenv/default.nix new file mode 100644 index 000000000000..504d53595440 --- /dev/null +++ b/third_party/gopkgs/github.com/muesli/termenv/default.nix @@ -0,0 +1,19 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/muesli/termenv"; + src = pkgs.fetchFromGitHub { + owner = "muesli"; + repo = "termenv"; + # unreleased version required by bubbletea + rev = "v0.8.1"; + sha256 = "0m24ljq1nq7z933fcvg99fw0fhxj9rb5ll4rlay7z2f2p59mrbdp"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".lucasb-eyer.go-colorful + gopkgs."github.com".mattn.go-isatty + gopkgs."github.com".mattn.go-runewidth + gopkgs."golang.org".x.sys.unix + ]; +} diff --git a/third_party/gopkgs/github.com/pkg/browser/default.nix b/third_party/gopkgs/github.com/pkg/browser/default.nix new file mode 100644 index 000000000000..4588c1b589a5 --- /dev/null +++ b/third_party/gopkgs/github.com/pkg/browser/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/pkg/browser"; + + src = pkgs.fetchFromGitHub { + owner = "pkg"; + repo = "browser"; + rev = "0a3d74bf9ce488f035cf5bc36f753a711bc74334"; + sha256 = "0lv6kwvm31n79mh14a63zslaf4l9bspi2q0i8i9im4njfl42iv1c"; + }; +} diff --git a/third_party/gopkgs/github.com/rivo/uniseg/default.nix b/third_party/gopkgs/github.com/rivo/uniseg/default.nix new file mode 100644 index 000000000000..f37d70bbda03 --- /dev/null +++ b/third_party/gopkgs/github.com/rivo/uniseg/default.nix @@ -0,0 +1,14 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/rivo/uniseg"; + src = pkgs.fetchFromGitHub { + owner = "rivo"; + repo = "uniseg"; + rev = "v0.1.0"; + sha256 = "0flpc1px1l6b1lxzhdxi0mvpkkjchppvgxshxxnlmm40s76i9ww5"; + }; + + deps = with depot.third_party; [ + ]; +} diff --git a/third_party/gopkgs/github.com/sergi/go-diff/default.nix b/third_party/gopkgs/github.com/sergi/go-diff/default.nix new file mode 100644 index 000000000000..72fb96d475ee --- /dev/null +++ b/third_party/gopkgs/github.com/sergi/go-diff/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/sergi/go-diff"; + + src = pkgs.fetchFromGitHub { + owner = "sergi"; + repo = "go-diff"; + rev = "58c5cb1602ee9676b5d3590d782bedde80706fcc"; + sha256 = "0ir8ali2vx0j7pipmlfd6k8c973akyy2nmbjrf008fm800zcp7z2"; + }; +} diff --git a/third_party/gopkgs/github.com/src-d/gcfg/default.nix b/third_party/gopkgs/github.com/src-d/gcfg/default.nix new file mode 100644 index 000000000000..210ab1bc70a2 --- /dev/null +++ b/third_party/gopkgs/github.com/src-d/gcfg/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/src-d/gcfg"; + + src = pkgs.fetchFromGitHub { + owner = "src-d"; + repo = "gcfg"; + rev = "1ac3a1ac202429a54835fe8408a92880156b489d"; + sha256 = "044j95skmyrwjw5fwjk6ka32rjgsg0ar0mfp9np19sh1acwv4x4r"; + }; + + deps = with depot.third_party; [ + gopkgs."gopkg.in".warnings + ]; +} diff --git a/third_party/gopkgs/github.com/xanzy/ssh-agent/default.nix b/third_party/gopkgs/github.com/xanzy/ssh-agent/default.nix new file mode 100644 index 000000000000..078592aa9d5f --- /dev/null +++ b/third_party/gopkgs/github.com/xanzy/ssh-agent/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "github.com/xanzy/ssh-agent"; + + src = pkgs.fetchFromGitHub { + owner = "xanzy"; + repo = "ssh-agent"; + rev = "6a3e2ff9e7c564f36873c2e36413f634534f1c44"; + sha256 = "1chjlnv5d6svpymxgsr62d992m2xi6jb5lybjc5zn1h3hv1m01av"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.crypto.ssh.agent + ]; +} diff --git a/third_party/gopkgs/go.opencensus.io/default.nix b/third_party/gopkgs/go.opencensus.io/default.nix new file mode 100644 index 000000000000..bc1444164590 --- /dev/null +++ b/third_party/gopkgs/go.opencensus.io/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "go.opencensus.io"; + src = builtins.fetchGit { + url = "https://github.com/census-instrumentation/opencensus-go"; + rev = "643eada29081047b355cfaa1ceb9bc307a10423c"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".hashicorp.golang-lru.simplelru + gopkgs."github.com".golang.groupcache.lru + ]; +} diff --git a/third_party/gopkgs/golang.org/x/crypto/default.nix b/third_party/gopkgs/golang.org/x/crypto/default.nix new file mode 100644 index 000000000000..dfaf4ac85672 --- /dev/null +++ b/third_party/gopkgs/golang.org/x/crypto/default.nix @@ -0,0 +1,13 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "golang.org/x/crypto"; + src = builtins.fetchGit { + url = "https://go.googlesource.com/crypto"; + rev = "e9b2fee46413994441b28dfca259d911d963dfed"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.sys.unix.gopkg + ]; +} diff --git a/third_party/gopkgs/golang.org/x/net/default.nix b/third_party/gopkgs/golang.org/x/net/default.nix new file mode 100644 index 000000000000..899600dab017 --- /dev/null +++ b/third_party/gopkgs/golang.org/x/net/default.nix @@ -0,0 +1,15 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "golang.org/x/net"; + src = builtins.fetchGit { + url = "https://go.googlesource.com/net"; + rev = "c0dbc17a35534bf2e581d7a942408dc936316da4"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.text.secure.bidirule.gopkg + gopkgs."golang.org".x.text.unicode.bidi.gopkg + gopkgs."golang.org".x.text.unicode.norm.gopkg + ]; +} diff --git a/third_party/gopkgs/golang.org/x/oauth2/default.nix b/third_party/gopkgs/golang.org/x/oauth2/default.nix new file mode 100644 index 000000000000..7ad94930aa8d --- /dev/null +++ b/third_party/gopkgs/golang.org/x/oauth2/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "golang.org/x/oauth2"; + src = builtins.fetchGit { + url = "https://go.googlesource.com/oauth2"; + rev = "858c2ad4c8b6c5d10852cb89079f6ca1c7309787"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.net.context.ctxhttp + gopkgs."cloud.google.com".go.compute.metadata + ]; +} diff --git a/third_party/gopkgs/golang.org/x/sys/default.nix b/third_party/gopkgs/golang.org/x/sys/default.nix new file mode 100644 index 000000000000..8636a9913e5d --- /dev/null +++ b/third_party/gopkgs/golang.org/x/sys/default.nix @@ -0,0 +1,9 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "golang.org/x/sys"; + src = builtins.fetchGit { + url = "https://go.googlesource.com/sys"; + rev = "ac6580df4449443a05718fd7858c1f91ad5f8d20"; + }; +} diff --git a/third_party/gopkgs/golang.org/x/text/default.nix b/third_party/gopkgs/golang.org/x/text/default.nix new file mode 100644 index 000000000000..33b7444616c6 --- /dev/null +++ b/third_party/gopkgs/golang.org/x/text/default.nix @@ -0,0 +1,9 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "golang.org/x/text"; + src = builtins.fetchGit { + url = "https://go.googlesource.com/text"; + rev = "cbf43d21aaebfdfeb81d91a5f444d13a3046e686"; + }; +} diff --git a/third_party/gopkgs/golang.org/x/time/default.nix b/third_party/gopkgs/golang.org/x/time/default.nix new file mode 100644 index 000000000000..2a9a4da56b8d --- /dev/null +++ b/third_party/gopkgs/golang.org/x/time/default.nix @@ -0,0 +1,10 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "golang.org/x/time"; + + src = builtins.fetchGit { + url = "https://go.googlesource.com/time"; + rev = "555d28b269f0569763d25dbe1a237ae74c6bcc82"; + }; +} diff --git a/third_party/gopkgs/google.golang.org/api/default.nix b/third_party/gopkgs/google.golang.org/api/default.nix new file mode 100644 index 000000000000..0d2f173e3023 --- /dev/null +++ b/third_party/gopkgs/google.golang.org/api/default.nix @@ -0,0 +1,20 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "google.golang.org/api"; + src = builtins.fetchGit { + url = "https://code.googlesource.com/google-api-go-client"; + rev = "8b4e46d953bd748a9ff098644a42389b3d8dab41"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".googleapis.gax-go.v2 + gopkgs."golang.org".x.oauth2.google + gopkgs."golang.org".x.oauth2 + gopkgs."google.golang.org".grpc + gopkgs."google.golang.org".grpc.naming + gopkgs."go.opencensus.io".plugin.ochttp + gopkgs."go.opencensus.io".trace + gopkgs."go.opencensus.io".trace.propagation + ]; +} diff --git a/third_party/gopkgs/google.golang.org/genproto/default.nix b/third_party/gopkgs/google.golang.org/genproto/default.nix new file mode 100644 index 000000000000..fa25d13607b3 --- /dev/null +++ b/third_party/gopkgs/google.golang.org/genproto/default.nix @@ -0,0 +1,14 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "google.golang.org/genproto"; + src = builtins.fetchGit { + url = "https://github.com/google/go-genproto"; + rev = "0243a4be9c8f1264d238fdc2895620b4d9baf9e1"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".golang.protobuf.proto.gopkg + gopkgs."github.com".golang.protobuf.ptypes.any.gopkg + ]; +} diff --git a/third_party/gopkgs/google.golang.org/grpc/default.nix b/third_party/gopkgs/google.golang.org/grpc/default.nix new file mode 100644 index 000000000000..f3ed89d092b8 --- /dev/null +++ b/third_party/gopkgs/google.golang.org/grpc/default.nix @@ -0,0 +1,21 @@ +{ depot, ... }: + +depot.nix.buildGo.external { + path = "google.golang.org/grpc"; + src = builtins.fetchGit { + url = "https://github.com/grpc/grpc-go"; + rev = "085c980048876e2735d4aba8f0d5bca4d7acaaa5"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.net.trace + gopkgs."golang.org".x.net.http2 + gopkgs."golang.org".x.net.http2.hpack + gopkgs."golang.org".x.sys.unix + gopkgs."github.com".golang.protobuf.proto + gopkgs."github.com".golang.protobuf.ptypes + gopkgs."github.com".golang.protobuf.ptypes.duration + gopkgs."github.com".golang.protobuf.ptypes.timestamp + gopkgs."google.golang.org".genproto.googleapis.rpc.status + ]; +} diff --git a/third_party/gopkgs/googlemaps.github.io/maps.nix b/third_party/gopkgs/googlemaps.github.io/maps.nix new file mode 100644 index 000000000000..4d29cc2f8993 --- /dev/null +++ b/third_party/gopkgs/googlemaps.github.io/maps.nix @@ -0,0 +1,17 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "googlemaps.github.io/maps"; + + src = pkgs.fetchFromGitHub { + owner = "googlemaps"; + repo = "google-maps-services-go"; + rev = "a46d9fca56ac82caa79408b2417ea93a75e3b986"; + sha256 = "1zpl85yd3m417060isdlhxzakqkf4f59jgpz3kcjp2i0mkrskkjs"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".google.uuid + gopkgs."golang.org".x.time.rate + ]; +} diff --git a/third_party/gopkgs/gopkg.in/irc.v3/default.nix b/third_party/gopkgs/gopkg.in/irc.v3/default.nix new file mode 100644 index 000000000000..7bfe5500232c --- /dev/null +++ b/third_party/gopkgs/gopkg.in/irc.v3/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "gopkg.in/irc.v3"; + + src = pkgs.fetchFromGitHub { + owner = "go-irc"; + repo = "irc"; + rev = "21a5301d6035ea204b2a7bb522a7b4598e5f6b28"; + sha256 = "1pi5y73pr4prhw5bvmp4babiw02nndizgmpksdgrrg28l9f2wm0n"; + }; +} diff --git a/third_party/gopkgs/gopkg.in/src-d/go-billy/default.nix b/third_party/gopkgs/gopkg.in/src-d/go-billy/default.nix new file mode 100644 index 000000000000..b2773d85d518 --- /dev/null +++ b/third_party/gopkgs/gopkg.in/src-d/go-billy/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "gopkg.in/src-d/go-billy.v4"; + + src = pkgs.fetchFromGitHub { + owner = "src-d"; + repo = "go-billy"; + rev = "fd409ff12f33d0d60af0ce0abeb8d93df360af49"; + sha256 = "1j0pl6ggzmd2lrqj71vmsnl6cqm43145h7yg6sy3j5n7hhd592qv"; + }; + + deps = with depot.third_party; [ + gopkgs."golang.org".x.sys.unix + ]; +} diff --git a/third_party/gopkgs/gopkg.in/src-d/go-git/default.nix b/third_party/gopkgs/gopkg.in/src-d/go-git/default.nix new file mode 100644 index 000000000000..ce5fe1d24073 --- /dev/null +++ b/third_party/gopkgs/gopkg.in/src-d/go-git/default.nix @@ -0,0 +1,31 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + # .v4 is used throughout the codebase and I can't be bothered to do + # anything else about it other than using that package path here. + path = "gopkg.in/src-d/go-git.v4"; + + src = pkgs.fetchFromGitHub { + owner = "src-d"; + repo = "go-git"; + rev = "1a7db85bca7027d90afdb5ce711622aaac9feaed"; + sha256 = "08jl4ljrzzil7c3qcl2y1859nhpgw9ixxy1g40ff7kmq989yhs6v"; + }; + + deps = with depot.third_party; [ + gopkgs."github.com".emirpasic.gods.trees.binaryheap + gopkgs."github.com".jbenet.go-context.io + gopkgs."github.com".kevinburke.ssh_config + gopkgs."github.com".mitchellh.go-homedir + gopkgs."github.com".sergi.go-diff.diffmatchpatch + gopkgs."github.com".src-d.gcfg + gopkgs."github.com".xanzy.ssh-agent + gopkgs."golang.org".x.crypto.openpgp + gopkgs."golang.org".x.crypto.ssh + gopkgs."golang.org".x.crypto.ssh.knownhosts + gopkgs."golang.org".x.net.proxy + gopkgs."gopkg.in".src-d.go-billy + gopkgs."gopkg.in".src-d.go-billy.osfs + gopkgs."gopkg.in".src-d.go-billy.util + ]; +} diff --git a/third_party/gopkgs/gopkg.in/warnings/default.nix b/third_party/gopkgs/gopkg.in/warnings/default.nix new file mode 100644 index 000000000000..1b4659d3d84d --- /dev/null +++ b/third_party/gopkgs/gopkg.in/warnings/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.nix.buildGo.external { + path = "gopkg.in/warnings.v0"; + + src = pkgs.fetchFromGitHub { + owner = "go-warnings"; + repo = "warnings"; + rev = "27b9fabbdaf131d2169ec3ff7db8ffc4d839635e"; + sha256 = "1y276jd9gwvjriz8yd98k3srgbnmbja8f7f7m6lvr0h5sbq3g3w9"; + }; +} diff --git a/third_party/grpc/default.nix b/third_party/grpc/default.nix new file mode 100644 index 000000000000..2ef67d402858 --- /dev/null +++ b/third_party/grpc/default.nix @@ -0,0 +1,13 @@ +{ depot, pkgs, ... }: + +(pkgs.grpc.override { + protobuf = depot.third_party.protobuf; + stdenv = pkgs.fullLlvm11Stdenv; + abseil-cpp = depot.third_party.abseil_cpp; + re2 = depot.third_party.re2; +}).overrideAttrs(orig: rec { + cmakeFlags = orig.cmakeFlags ++ [ + "-DCMAKE_CXX_STANDARD_REQUIRED=ON" + "-DCMAKE_CXX_STANDARD=17" + ]; +}) diff --git a/third_party/gtest/default.nix b/third_party/gtest/default.nix new file mode 100644 index 000000000000..70516853ed5c --- /dev/null +++ b/third_party/gtest/default.nix @@ -0,0 +1,12 @@ +{ pkgs, ... }: + +(pkgs.gtest.override { + stdenv = pkgs.fullLlvm11Stdenv; +}).overrideAttrs(_: { + src = pkgs.fetchFromGitHub { + owner = "google"; + repo = "googletest"; + rev = "9dce5e5d878176dc0054ef381f5c6e705f43ef99"; + sha256 = "05xi61j7j251dzkgk9965lqpbacsy44iblzql941kw9d4nk0q6jl"; + }; +}) diff --git a/third_party/hii/OWNERS b/third_party/hii/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/third_party/hii/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/third_party/impermanence/default.nix b/third_party/impermanence/default.nix new file mode 100644 index 000000000000..6ef72e4943ab --- /dev/null +++ b/third_party/impermanence/default.nix @@ -0,0 +1,12 @@ +# NixOS modules for systems with ephemeral root disks. +# +# https://github.com/nix-community/impermanence + +{ pkgs, ... }: + +pkgs.fetchFromGitHub { + owner = "nix-community"; + repo = "impermanence"; + rev = "58558845bc68dcf2bb32caa80564f7fe3f6cbc61"; + sha256 = "10z3g4knkvq838zbfq71pkfyl8cffrpavna448wf5mjscycp0gnv"; +} diff --git a/third_party/irccat/default.nix b/third_party/irccat/default.nix new file mode 100644 index 000000000000..3181fd2067a8 --- /dev/null +++ b/third_party/irccat/default.nix @@ -0,0 +1,16 @@ +# https://github.com/irccloud/irccat +{ lib, pkgs, ... }: + +pkgs.buildGoModule rec { + pname = "irccat"; + version = "20201108"; + meta.license = lib.licenses.gpl3; + vendorSha256 = "06a985y4alw1rsghgmhfyczns6klz7bbkfn5mnqc9fdfclgg4s3r"; + + src = pkgs.fetchFromGitHub { + owner = "irccloud"; + repo = "irccat"; + rev = "17451e7e267f099e9614ec945541b624520f607e"; + sha256 = "0l99mycxymyslwi8mmyfdcqa8pdp79wcyb04s5j5y4grmlsxw1wx"; + }; +} diff --git a/third_party/josh/0001-replace-mentions-of-master-with-canon.patch b/third_party/josh/0001-replace-mentions-of-master-with-canon.patch new file mode 100644 index 000000000000..6e9a8028f22a --- /dev/null +++ b/third_party/josh/0001-replace-mentions-of-master-with-canon.patch @@ -0,0 +1,47 @@ +From 4245dbdcacc3a4d4801a50071ba84379be4e1d2e Mon Sep 17 00:00:00 2001 +From: Vincent Ambo <mail@tazj.in> +Date: Tue, 1 Jun 2021 23:27:30 +0200 +Subject: [PATCH] replace mentions of 'master' with 'canon' + +While there are areas of josh that hardcode `master`, we can override +this to make it work for the TVL use-case. +--- + josh-proxy/src/bin/josh-proxy.rs | 2 +- + src/housekeeping.rs | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/josh-proxy/src/bin/josh-proxy.rs b/josh-proxy/src/bin/josh-proxy.rs +index 6f15566..76f4a5d 100644 +--- a/josh-proxy/src/bin/josh-proxy.rs ++++ b/josh-proxy/src/bin/josh-proxy.rs +@@ -374,7 +374,7 @@ async fn call_service( + + let mut headref = parsed_url.headref.trim_start_matches("@").to_owned(); + if headref == "" { +- headref = "refs/heads/master".to_string(); ++ headref = "refs/heads/canon".to_string(); + } + + let remote_url = [ +diff --git a/src/housekeeping.rs b/src/housekeeping.rs +index 5d08f81..1d55bc4 100644 +--- a/src/housekeeping.rs ++++ b/src/housekeeping.rs +@@ -44,12 +44,12 @@ pub fn memorize_from_to( + ) -> Vec<(String, String)> { + let mut refs = vec![]; + let glob = format!( +- "refs/josh/upstream/{}/refs/heads/master", ++ "refs/josh/upstream/{}/refs/heads/canon", + &to_ns(upstream_repo) + ); + for refname in repo.references_glob(&glob).unwrap().names() { + let refname = refname.unwrap(); +- let to_ref = format!("refs/{}/heads/master", &namespace); ++ let to_ref = format!("refs/{}/heads/canon", &namespace); + + refs.push((refname.to_owned(), to_ref.clone())); + } +-- +2.31.1 + diff --git a/third_party/josh/default.nix b/third_party/josh/default.nix new file mode 100644 index 000000000000..9b529304ce2f --- /dev/null +++ b/third_party/josh/default.nix @@ -0,0 +1,29 @@ +# https://github.com/esrlabs/josh +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "esrlabs"; + repo = "josh"; + rev = "4a70f2914260066b2db44f4cabe9b80acdd556af"; + sha256 = "0gwn3daj24538h5ddksvnyhq4lyjh0br9k49jhbd7r1sm6ly2cwl"; + }; +in depot.third_party.naersk.buildPackage { + inherit src; + + buildInputs = with pkgs; [ + libgit2 + openssl + pkgconfig + ]; + + cargoBuildOptions = x: x ++ [ + "-p" "josh" + "-p" "josh-proxy" + "-p" "josh-ui" + ]; + + overrideMain = x: { + patches = [ ./0001-replace-mentions-of-master-with-canon.patch ]; + }; +} diff --git a/third_party/kernelPatches/trx40_usb_audio/default.nix b/third_party/kernelPatches/trx40_usb_audio/default.nix new file mode 100644 index 000000000000..f753878f7cf8 --- /dev/null +++ b/third_party/kernelPatches/trx40_usb_audio/default.nix @@ -0,0 +1,9 @@ +# This patch adds the ASUS TRX40 Prime Pro Whatever Edition +# motherboard to the list of boards for which the USB Audio connector +# map has been fixed. +{ ... }: + +{ + name = "trx40_usb_audio"; + patch = ./trx40_usb_audio.patch; +} diff --git a/third_party/kernelPatches/trx40_usb_audio/trx40_usb_audio.patch b/third_party/kernelPatches/trx40_usb_audio/trx40_usb_audio.patch new file mode 100644 index 000000000000..d55b7bc362fe --- /dev/null +++ b/third_party/kernelPatches/trx40_usb_audio/trx40_usb_audio.patch @@ -0,0 +1,16 @@ +diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c +index 0260c750e156..5ee82872e31b 100644 +--- a/sound/usb/mixer_maps.c ++++ b/sound/usb/mixer_maps.c +@@ -539,6 +539,11 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { + .id = USB_ID(0x0b05, 0x1917), + .map = asus_rog_map, + }, ++ { /* ASUS TRX40 Prime */ ++ .id = USB_ID(0x0b05, 0x1918), ++ .map = trx40_mobo_map, ++ .connector_map = trx40_mobo_connector_map, ++ }, + { /* MSI TRX40 Creator */ + .id = USB_ID(0x0db0, 0x0d64), + .map = trx40_mobo_map, diff --git a/third_party/lisp/OWNERS b/third_party/lisp/OWNERS new file mode 100644 index 000000000000..2d7f7e237b76 --- /dev/null +++ b/third_party/lisp/OWNERS @@ -0,0 +1,5 @@ +# -*- mode: yaml; -*- +inherited: true +owners: + - eta + - grfn diff --git a/third_party/lisp/alexandria/.boring b/third_party/lisp/alexandria/.boring new file mode 100644 index 000000000000..dfa9e6dd7bb4 --- /dev/null +++ b/third_party/lisp/alexandria/.boring @@ -0,0 +1,13 @@ +# Boring file regexps: +~$ +^_darcs +^\{arch\} +^.arch-ids +\# +\.dfsl$ +\.ppcf$ +\.fasl$ +\.x86f$ +\.fas$ +\.lib$ +^public_html diff --git a/third_party/lisp/alexandria/.gitignore b/third_party/lisp/alexandria/.gitignore new file mode 100644 index 000000000000..e832e9471833 --- /dev/null +++ b/third_party/lisp/alexandria/.gitignore @@ -0,0 +1,4 @@ +*.fasl +*~ +\#* +*.patch diff --git a/third_party/lisp/alexandria/AUTHORS b/third_party/lisp/alexandria/AUTHORS new file mode 100644 index 000000000000..b550ea503248 --- /dev/null +++ b/third_party/lisp/alexandria/AUTHORS @@ -0,0 +1,9 @@ + +ACTA EST FABULA PLAUDITE + +Nikodemus Siivola +Attila Lendvai +Marco Baringer +Robert Strandh +Luis Oliveira +Tobias C. Rittweiler \ No newline at end of file diff --git a/third_party/lisp/alexandria/LICENCE b/third_party/lisp/alexandria/LICENCE new file mode 100644 index 000000000000..b5140fbb2491 --- /dev/null +++ b/third_party/lisp/alexandria/LICENCE @@ -0,0 +1,37 @@ +Alexandria software and associated documentation are in the public +domain: + + Authors dedicate this work to public domain, for the benefit of the + public at large and to the detriment of the authors' heirs and + successors. Authors intends this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights under + copyright law, whether vested or contingent, in the work. Authors + understands that such relinquishment of all rights includes the + relinquishment of all rights to enforce (by lawsuit or otherwise) + those copyrights in the work. + + Authors recognize that, once placed in the public domain, the work + may be freely reproduced, distributed, transmitted, used, modified, + built upon, or otherwise exploited by anyone for any purpose, + commercial or non-commercial, and in any way, including by methods + that have not yet been invented or conceived. + +In those legislations where public domain dedications are not +recognized or possible, Alexandria is distributed under the following +terms and conditions: + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/lisp/alexandria/README b/third_party/lisp/alexandria/README new file mode 100644 index 000000000000..a5dae9ed1ac7 --- /dev/null +++ b/third_party/lisp/alexandria/README @@ -0,0 +1,52 @@ +Alexandria is a collection of portable public domain utilities that +meet the following constraints: + + * Utilities, not extensions: Alexandria will not contain conceptual + extensions to Common Lisp, instead limiting itself to tools and + utilities that fit well within the framework of standard ANSI + Common Lisp. Test-frameworks, system definitions, logging + facilities, serialization layers, etc. are all outside the scope of + Alexandria as a library, though well within the scope of Alexandria + as a project. + + * Conservative: Alexandria limits itself to what project members + consider conservative utilities. Alexandria does not and will not + include anaphoric constructs, loop-like binding macros, etc. + + * Portable: Alexandria limits itself to portable parts of Common + Lisp. Even apparently conservative and useful functions remain + outside the scope of Alexandria if they cannot be implemented + portably. Portability is here defined as portable within a + conforming implementation: implementation bugs are not considered + portability issues. + +Homepage: + + http://common-lisp.net/project/alexandria/ + +Mailing lists: + + http://lists.common-lisp.net/mailman/listinfo/alexandria-devel + http://lists.common-lisp.net/mailman/listinfo/alexandria-cvs + +Repository: + + git://gitlab.common-lisp.net/alexandria/alexandria.git + +Documentation: + + http://common-lisp.net/project/alexandria/draft/alexandria.html + + (To build docs locally: cd doc && make html pdf info) + +Patches: + + Patches are always welcome! Please send them to the mailing list as + attachments, generated by "git format-patch -1". + + Patches should include a commit message that explains what's being + done and /why/, and when fixing a bug or adding a feature you should + also include a test-case. + + Be advised though that right now new features are unlikely to be + accepted until 1.0 is officially out of the door. diff --git a/third_party/lisp/alexandria/alexandria-tests.asd b/third_party/lisp/alexandria/alexandria-tests.asd new file mode 100644 index 000000000000..445c18cf7f77 --- /dev/null +++ b/third_party/lisp/alexandria/alexandria-tests.asd @@ -0,0 +1,11 @@ +(defsystem "alexandria-tests" + :licence "Public Domain / 0-clause MIT" + :description "Tests for Alexandria, which is a collection of portable public domain utilities." + :author "Nikodemus Siivola <nikodemus@sb-studio.net>, and others." + :depends-on (:alexandria #+sbcl :sb-rt #-sbcl :rt) + :components ((:file "tests")) + :perform (test-op (o c) + (flet ((run-tests (&rest args) + (apply (intern (string '#:run-tests) '#:alexandria-tests) args))) + (run-tests :compiled nil) + (run-tests :compiled t)))) diff --git a/third_party/lisp/alexandria/alexandria.asd b/third_party/lisp/alexandria/alexandria.asd new file mode 100644 index 000000000000..db10e4f53710 --- /dev/null +++ b/third_party/lisp/alexandria/alexandria.asd @@ -0,0 +1,62 @@ +(defsystem "alexandria" + :version "1.0.0" + :licence "Public Domain / 0-clause MIT" + :description "Alexandria is a collection of portable public domain utilities." + :author "Nikodemus Siivola and others." + :long-description + "Alexandria is a project and a library. + +As a project Alexandria's goal is to reduce duplication of effort and improve +portability of Common Lisp code according to its own idiosyncratic and rather +conservative aesthetic. + +As a library Alexandria is one of the means by which the project strives for +its goals. + +Alexandria is a collection of portable public domain utilities that meet +the following constraints: + + * Utilities, not extensions: Alexandria will not contain conceptual + extensions to Common Lisp, instead limiting itself to tools and utilities + that fit well within the framework of standard ANSI Common Lisp. + Test-frameworks, system definitions, logging facilities, serialization + layers, etc. are all outside the scope of Alexandria as a library, though + well within the scope of Alexandria as a project. + + * Conservative: Alexandria limits itself to what project members consider + conservative utilities. Alexandria does not and will not include anaphoric + constructs, loop-like binding macros, etc. + Also, its exported symbols are being imported by many other packages + already, so each new export carries the danger of causing conflicts. + + * Portable: Alexandria limits itself to portable parts of Common Lisp. Even + apparently conservative and useful functions remain outside the scope of + Alexandria if they cannot be implemented portably. Portability is here + defined as portable within a conforming implementation: implementation bugs + are not considered portability issues. + + * Team player: Alexandria will not (initially, at least) subsume or provide + functionality for which good-quality special-purpose packages exist, like + split-sequence. Instead, third party packages such as that may be + \"blessed\"." + :components + ((:static-file "LICENCE") + (:static-file "tests.lisp") + (:file "package") + (:file "definitions" :depends-on ("package")) + (:file "binding" :depends-on ("package")) + (:file "strings" :depends-on ("package")) + (:file "conditions" :depends-on ("package")) + (:file "io" :depends-on ("package" "macros" "lists" "types")) + (:file "macros" :depends-on ("package" "strings" "symbols")) + (:file "hash-tables" :depends-on ("package" "macros")) + (:file "control-flow" :depends-on ("package" "definitions" "macros")) + (:file "symbols" :depends-on ("package")) + (:file "functions" :depends-on ("package" "symbols" "macros")) + (:file "lists" :depends-on ("package" "functions")) + (:file "types" :depends-on ("package" "symbols" "lists")) + (:file "arrays" :depends-on ("package" "types")) + (:file "sequences" :depends-on ("package" "lists" "types")) + (:file "numbers" :depends-on ("package" "sequences")) + (:file "features" :depends-on ("package" "control-flow"))) + :in-order-to ((test-op (test-op "alexandria-tests")))) diff --git a/third_party/lisp/alexandria/arrays.lisp b/third_party/lisp/alexandria/arrays.lisp new file mode 100644 index 000000000000..76c18791ad5f --- /dev/null +++ b/third_party/lisp/alexandria/arrays.lisp @@ -0,0 +1,18 @@ +(in-package :alexandria) + +(defun copy-array (array &key (element-type (array-element-type array)) + (fill-pointer (and (array-has-fill-pointer-p array) + (fill-pointer array))) + (adjustable (adjustable-array-p array))) + "Returns an undisplaced copy of ARRAY, with same fill-pointer and +adjustability (if any) as the original, unless overridden by the keyword +arguments." + (let* ((dimensions (array-dimensions array)) + (new-array (make-array dimensions + :element-type element-type + :adjustable adjustable + :fill-pointer fill-pointer))) + (dotimes (i (array-total-size array)) + (setf (row-major-aref new-array i) + (row-major-aref array i))) + new-array)) diff --git a/third_party/lisp/alexandria/binding.lisp b/third_party/lisp/alexandria/binding.lisp new file mode 100644 index 000000000000..37a3d52fb9f0 --- /dev/null +++ b/third_party/lisp/alexandria/binding.lisp @@ -0,0 +1,90 @@ +(in-package :alexandria) + +(defmacro if-let (bindings &body (then-form &optional else-form)) + "Creates new variable bindings, and conditionally executes either +THEN-FORM or ELSE-FORM. ELSE-FORM defaults to NIL. + +BINDINGS must be either single binding of the form: + + (variable initial-form) + +or a list of bindings of the form: + + ((variable-1 initial-form-1) + (variable-2 initial-form-2) + ... + (variable-n initial-form-n)) + +All initial-forms are executed sequentially in the specified order. Then all +the variables are bound to the corresponding values. + +If all variables were bound to true values, the THEN-FORM is executed with the +bindings in effect, otherwise the ELSE-FORM is executed with the bindings in +effect." + (let* ((binding-list (if (and (consp bindings) (symbolp (car bindings))) + (list bindings) + bindings)) + (variables (mapcar #'car binding-list))) + `(let ,binding-list + (if (and ,@variables) + ,then-form + ,else-form)))) + +(defmacro when-let (bindings &body forms) + "Creates new variable bindings, and conditionally executes FORMS. + +BINDINGS must be either single binding of the form: + + (variable initial-form) + +or a list of bindings of the form: + + ((variable-1 initial-form-1) + (variable-2 initial-form-2) + ... + (variable-n initial-form-n)) + +All initial-forms are executed sequentially in the specified order. Then all +the variables are bound to the corresponding values. + +If all variables were bound to true values, then FORMS are executed as an +implicit PROGN." + (let* ((binding-list (if (and (consp bindings) (symbolp (car bindings))) + (list bindings) + bindings)) + (variables (mapcar #'car binding-list))) + `(let ,binding-list + (when (and ,@variables) + ,@forms)))) + +(defmacro when-let* (bindings &body body) + "Creates new variable bindings, and conditionally executes BODY. + +BINDINGS must be either single binding of the form: + + (variable initial-form) + +or a list of bindings of the form: + + ((variable-1 initial-form-1) + (variable-2 initial-form-2) + ... + (variable-n initial-form-n)) + +Each INITIAL-FORM is executed in turn, and the variable bound to the +corresponding value. INITIAL-FORM expressions can refer to variables +previously bound by the WHEN-LET*. + +Execution of WHEN-LET* stops immediately if any INITIAL-FORM evaluates to NIL. +If all INITIAL-FORMs evaluate to true, then BODY is executed as an implicit +PROGN." + (let ((binding-list (if (and (consp bindings) (symbolp (car bindings))) + (list bindings) + bindings))) + (labels ((bind (bindings body) + (if bindings + `(let (,(car bindings)) + (when ,(caar bindings) + ,(bind (cdr bindings) body))) + `(progn ,@body)))) + (bind binding-list body)))) diff --git a/third_party/lisp/alexandria/conditions.lisp b/third_party/lisp/alexandria/conditions.lisp new file mode 100644 index 000000000000..ac471cca7e4c --- /dev/null +++ b/third_party/lisp/alexandria/conditions.lisp @@ -0,0 +1,91 @@ +(in-package :alexandria) + +(defun required-argument (&optional name) + "Signals an error for a missing argument of NAME. Intended for +use as an initialization form for structure and class-slots, and +a default value for required keyword arguments." + (error "Required argument ~@[~S ~]missing." name)) + +(define-condition simple-style-warning (simple-warning style-warning) + ()) + +(defun simple-style-warning (message &rest args) + (warn 'simple-style-warning :format-control message :format-arguments args)) + +;; We don't specify a :report for simple-reader-error to let the +;; underlying implementation report the line and column position for +;; us. Unfortunately this way the message from simple-error is not +;; displayed, unless there's special support for that in the +;; implementation. But even then it's still inspectable from the +;; debugger... +(define-condition simple-reader-error + #-sbcl(simple-error reader-error) + #+sbcl(sb-int:simple-reader-error) + ()) + +(defun simple-reader-error (stream message &rest args) + (error 'simple-reader-error + :stream stream + :format-control message + :format-arguments args)) + +(define-condition simple-parse-error (simple-error parse-error) + ()) + +(defun simple-parse-error (message &rest args) + (error 'simple-parse-error + :format-control message + :format-arguments args)) + +(define-condition simple-program-error (simple-error program-error) + ()) + +(defun simple-program-error (message &rest args) + (error 'simple-program-error + :format-control message + :format-arguments args)) + +(defmacro ignore-some-conditions ((&rest conditions) &body body) + "Similar to CL:IGNORE-ERRORS but the (unevaluated) CONDITIONS +list determines which specific conditions are to be ignored." + `(handler-case + (progn ,@body) + ,@(loop for condition in conditions collect + `(,condition (c) (values nil c))))) + +(defmacro unwind-protect-case ((&optional abort-flag) protected-form &body clauses) + "Like CL:UNWIND-PROTECT, but you can specify the circumstances that +the cleanup CLAUSES are run. + + clauses ::= (:NORMAL form*)* | (:ABORT form*)* | (:ALWAYS form*)* + +Clauses can be given in any order, and more than one clause can be +given for each circumstance. The clauses whose denoted circumstance +occured, are executed in the order the clauses appear. + +ABORT-FLAG is the name of a variable that will be bound to T in +CLAUSES if the PROTECTED-FORM aborted preemptively, and to NIL +otherwise. + +Examples: + + (unwind-protect-case () + (protected-form) + (:normal (format t \"This is only evaluated if PROTECTED-FORM executed normally.~%\")) + (:abort (format t \"This is only evaluated if PROTECTED-FORM aborted preemptively.~%\")) + (:always (format t \"This is evaluated in either case.~%\"))) + + (unwind-protect-case (aborted-p) + (protected-form) + (:always (perform-cleanup-if aborted-p))) +" + (check-type abort-flag (or null symbol)) + (let ((gflag (gensym "FLAG+"))) + `(let ((,gflag t)) + (unwind-protect (multiple-value-prog1 ,protected-form (setf ,gflag nil)) + (let ,(and abort-flag `((,abort-flag ,gflag))) + ,@(loop for (cleanup-kind . forms) in clauses + collect (ecase cleanup-kind + (:normal `(when (not ,gflag) ,@forms)) + (:abort `(when ,gflag ,@forms)) + (:always `(progn ,@forms))))))))) \ No newline at end of file diff --git a/third_party/lisp/alexandria/control-flow.lisp b/third_party/lisp/alexandria/control-flow.lisp new file mode 100644 index 000000000000..dd00df3e1620 --- /dev/null +++ b/third_party/lisp/alexandria/control-flow.lisp @@ -0,0 +1,106 @@ +(in-package :alexandria) + +(defun extract-function-name (spec) + "Useful for macros that want to mimic the functional interface for functions +like #'eq and 'eq." + (if (and (consp spec) + (member (first spec) '(quote function))) + (second spec) + spec)) + +(defun generate-switch-body (whole object clauses test key &optional default) + (with-gensyms (value) + (setf test (extract-function-name test)) + (setf key (extract-function-name key)) + (when (and (consp default) + (member (first default) '(error cerror))) + (setf default `(,@default "No keys match in SWITCH. Testing against ~S with ~S." + ,value ',test))) + `(let ((,value (,key ,object))) + (cond ,@(mapcar (lambda (clause) + (if (member (first clause) '(t otherwise)) + (progn + (when default + (error "Multiple default clauses or illegal use of a default clause in ~S." + whole)) + (setf default `(progn ,@(rest clause))) + '(())) + (destructuring-bind (key-form &body forms) clause + `((,test ,value ,key-form) + ,@forms)))) + clauses) + (t ,default))))) + +(defmacro switch (&whole whole (object &key (test 'eql) (key 'identity)) + &body clauses) + "Evaluates first matching clause, returning its values, or evaluates and +returns the values of T or OTHERWISE if no keys match." + (generate-switch-body whole object clauses test key)) + +(defmacro eswitch (&whole whole (object &key (test 'eql) (key 'identity)) + &body clauses) + "Like SWITCH, but signals an error if no key matches." + (generate-switch-body whole object clauses test key '(error))) + +(defmacro cswitch (&whole whole (object &key (test 'eql) (key 'identity)) + &body clauses) + "Like SWITCH, but signals a continuable error if no key matches." + (generate-switch-body whole object clauses test key '(cerror "Return NIL from CSWITCH."))) + +(defmacro whichever (&rest possibilities &environment env) + "Evaluates exactly one of POSSIBILITIES, chosen at random." + (setf possibilities (mapcar (lambda (p) (macroexpand p env)) possibilities)) + (if (every (lambda (p) (constantp p)) possibilities) + `(svref (load-time-value (vector ,@possibilities)) (random ,(length possibilities))) + (labels ((expand (possibilities position random-number) + (if (null (cdr possibilities)) + (car possibilities) + (let* ((length (length possibilities)) + (half (truncate length 2)) + (second-half (nthcdr half possibilities)) + (first-half (butlast possibilities (- length half)))) + `(if (< ,random-number ,(+ position half)) + ,(expand first-half position random-number) + ,(expand second-half (+ position half) random-number)))))) + (with-gensyms (random-number) + (let ((length (length possibilities))) + `(let ((,random-number (random ,length))) + ,(expand possibilities 0 random-number))))))) + +(defmacro xor (&rest datums) + "Evaluates its arguments one at a time, from left to right. If more than one +argument evaluates to a true value no further DATUMS are evaluated, and NIL is +returned as both primary and secondary value. If exactly one argument +evaluates to true, its value is returned as the primary value after all the +arguments have been evaluated, and T is returned as the secondary value. If no +arguments evaluate to true NIL is retuned as primary, and T as secondary +value." + (with-gensyms (xor tmp true) + `(let (,tmp ,true) + (block ,xor + ,@(mapcar (lambda (datum) + `(if (setf ,tmp ,datum) + (if ,true + (return-from ,xor (values nil nil)) + (setf ,true ,tmp)))) + datums) + (return-from ,xor (values ,true t)))))) + +(defmacro nth-value-or (nth-value &body forms) + "Evaluates FORM arguments one at a time, until the NTH-VALUE returned by one +of the forms is true. It then returns all the values returned by evaluating +that form. If none of the forms return a true nth value, this form returns +NIL." + (once-only (nth-value) + (with-gensyms (values) + `(let ((,values (multiple-value-list ,(first forms)))) + (if (nth ,nth-value ,values) + (values-list ,values) + ,(if (rest forms) + `(nth-value-or ,nth-value ,@(rest forms)) + nil)))))) + +(defmacro multiple-value-prog2 (first-form second-form &body forms) + "Evaluates FIRST-FORM, then SECOND-FORM, and then FORMS. Yields as its value +all the value returned by SECOND-FORM." + `(progn ,first-form (multiple-value-prog1 ,second-form ,@forms))) diff --git a/third_party/lisp/alexandria/default.nix b/third_party/lisp/alexandria/default.nix new file mode 100644 index 000000000000..2358c898b3ab --- /dev/null +++ b/third_party/lisp/alexandria/default.nix @@ -0,0 +1,28 @@ +# Alexandria is one of the foundational Common Lisp libraries that +# pretty much everything depends on: +# +# Imported from https://common-lisp.net/project/alexandria/ +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "alexandria"; + srcs = [ + ./package.lisp + ./definitions.lisp + ./binding.lisp + ./strings.lisp + ./conditions.lisp + ./symbols.lisp + ./macros.lisp + ./functions.lisp + ./io.lisp + ./hash-tables.lisp + ./control-flow.lisp + ./lists.lisp + ./types.lisp + ./arrays.lisp + ./sequences.lisp + ./numbers.lisp + ./features.lisp + ]; +} diff --git a/third_party/lisp/alexandria/definitions.lisp b/third_party/lisp/alexandria/definitions.lisp new file mode 100644 index 000000000000..863e1f696286 --- /dev/null +++ b/third_party/lisp/alexandria/definitions.lisp @@ -0,0 +1,37 @@ +(in-package :alexandria) + +(defun %reevaluate-constant (name value test) + (if (not (boundp name)) + value + (let ((old (symbol-value name)) + (new value)) + (if (not (constantp name)) + (prog1 new + (cerror "Try to redefine the variable as a constant." + "~@<~S is an already bound non-constant variable ~ + whose value is ~S.~:@>" name old)) + (if (funcall test old new) + old + (restart-case + (error "~@<~S is an already defined constant whose value ~ + ~S is not equal to the provided initial value ~S ~ + under ~S.~:@>" name old new test) + (ignore () + :report "Retain the current value." + old) + (continue () + :report "Try to redefine the constant." + new))))))) + +(defmacro define-constant (name initial-value &key (test ''eql) documentation) + "Ensures that the global variable named by NAME is a constant with a value +that is equal under TEST to the result of evaluating INITIAL-VALUE. TEST is a +/function designator/ that defaults to EQL. If DOCUMENTATION is given, it +becomes the documentation string of the constant. + +Signals an error if NAME is already a bound non-constant variable. + +Signals an error if NAME is already a constant variable whose value is not +equal under TEST to result of evaluating INITIAL-VALUE." + `(defconstant ,name (%reevaluate-constant ',name ,initial-value ,test) + ,@(when documentation `(,documentation)))) diff --git a/third_party/lisp/alexandria/doc/.gitignore b/third_party/lisp/alexandria/doc/.gitignore new file mode 100644 index 000000000000..f22577b3ac86 --- /dev/null +++ b/third_party/lisp/alexandria/doc/.gitignore @@ -0,0 +1,3 @@ +alexandria +include + diff --git a/third_party/lisp/alexandria/doc/Makefile b/third_party/lisp/alexandria/doc/Makefile new file mode 100644 index 000000000000..85eb818220d5 --- /dev/null +++ b/third_party/lisp/alexandria/doc/Makefile @@ -0,0 +1,28 @@ +.PHONY: clean html pdf include clean-include clean-crap info doc + +doc: pdf html info clean-crap + +clean-include: + rm -rf include + +clean-crap: + rm -f *.aux *.cp *.fn *.fns *.ky *.log *.pg *.toc *.tp *.tps *.vr + +clean: clean-include + rm -f *.pdf *.html *.info + +include: + sbcl --no-userinit --eval '(require :asdf)' \ + --eval '(let ((asdf:*central-registry* (list "../"))) (require :alexandria))' \ + --load docstrings.lisp \ + --eval '(sb-texinfo:generate-includes "include/" (list :alexandria) :base-package :alexandria)' \ + --eval '(quit)' + +pdf: include + texi2pdf alexandria.texinfo + +html: include + makeinfo --html --no-split alexandria.texinfo + +info: include + makeinfo alexandria.texinfo diff --git a/third_party/lisp/alexandria/doc/alexandria.texinfo b/third_party/lisp/alexandria/doc/alexandria.texinfo new file mode 100644 index 000000000000..89b03ac34967 --- /dev/null +++ b/third_party/lisp/alexandria/doc/alexandria.texinfo @@ -0,0 +1,277 @@ +\input texinfo @c -*-texinfo-*- +@c %**start of header +@setfilename alexandria.info +@settitle Alexandria Manual +@c %**end of header + +@settitle Alexandria Manual -- draft version + +@c for install-info +@dircategory Software development +@direntry +* alexandria: Common Lisp utilities. +@end direntry + +@copying +Alexandria software and associated documentation are in the public +domain: + +@quotation + Authors dedicate this work to public domain, for the benefit of the + public at large and to the detriment of the authors' heirs and + successors. Authors intends this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights under + copyright law, whether vested or contingent, in the work. Authors + understands that such relinquishment of all rights includes the + relinquishment of all rights to enforce (by lawsuit or otherwise) + those copyrights in the work. + + Authors recognize that, once placed in the public domain, the work + may be freely reproduced, distributed, transmitted, used, modified, + built upon, or otherwise exploited by anyone for any purpose, + commercial or non-commercial, and in any way, including by methods + that have not yet been invented or conceived. +@end quotation + +In those legislations where public domain dedications are not +recognized or possible, Alexandria is distributed under the following +terms and conditions: + +@quotation + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +@end quotation +@end copying + +@titlepage + +@title Alexandria Manual +@subtitle draft version + +@c The following two commands start the copyright page. +@page +@vskip 0pt plus 1filll +@insertcopying + +@end titlepage + +@contents + +@ifnottex + +@include include/ifnottex.texinfo + +@node Top +@comment node-name, next, previous, up +@top Alexandria + +@insertcopying + +@menu +* Hash Tables:: +* Data and Control Flow:: +* Conses:: +* Sequences:: +* IO:: +* Macro Writing:: +* Symbols:: +* Arrays:: +* Types:: +* Numbers:: +@end menu + +@end ifnottex + +@node Hash Tables +@comment node-name, next, previous, up +@chapter Hash Tables + +@include include/macro-alexandria-ensure-gethash.texinfo +@include include/fun-alexandria-copy-hash-table.texinfo +@include include/fun-alexandria-maphash-keys.texinfo +@include include/fun-alexandria-maphash-values.texinfo +@include include/fun-alexandria-hash-table-keys.texinfo +@include include/fun-alexandria-hash-table-values.texinfo +@include include/fun-alexandria-hash-table-alist.texinfo +@include include/fun-alexandria-hash-table-plist.texinfo +@include include/fun-alexandria-alist-hash-table.texinfo +@include include/fun-alexandria-plist-hash-table.texinfo + +@node Data and Control Flow +@comment node-name, next, previous, up +@chapter Data and Control Flow + +@include include/macro-alexandria-define-constant.texinfo +@include include/macro-alexandria-destructuring-case.texinfo +@include include/macro-alexandria-ensure-functionf.texinfo +@include include/macro-alexandria-multiple-value-prog2.texinfo +@include include/macro-alexandria-named-lambda.texinfo +@include include/macro-alexandria-nth-value-or.texinfo +@include include/macro-alexandria-if-let.texinfo +@include include/macro-alexandria-when-let.texinfo +@include include/macro-alexandria-when-let-star.texinfo +@include include/macro-alexandria-switch.texinfo +@include include/macro-alexandria-cswitch.texinfo +@include include/macro-alexandria-eswitch.texinfo +@include include/macro-alexandria-whichever.texinfo +@include include/macro-alexandria-xor.texinfo + +@include include/fun-alexandria-disjoin.texinfo +@include include/fun-alexandria-conjoin.texinfo +@include include/fun-alexandria-compose.texinfo +@include include/fun-alexandria-ensure-function.texinfo +@include include/fun-alexandria-multiple-value-compose.texinfo +@include include/fun-alexandria-curry.texinfo +@include include/fun-alexandria-rcurry.texinfo + +@node Conses +@comment node-name, next, previous, up +@chapter Conses + +@include include/type-alexandria-proper-list.texinfo +@include include/type-alexandria-circular-list.texinfo + +@include include/macro-alexandria-appendf.texinfo +@include include/macro-alexandria-nconcf.texinfo +@include include/macro-alexandria-remove-from-plistf.texinfo +@include include/macro-alexandria-delete-from-plistf.texinfo +@include include/macro-alexandria-reversef.texinfo +@include include/macro-alexandria-nreversef.texinfo +@include include/macro-alexandria-unionf.texinfo +@include include/macro-alexandria-nunionf.texinfo + +@include include/macro-alexandria-doplist.texinfo + +@include include/fun-alexandria-circular-list-p.texinfo +@include include/fun-alexandria-circular-tree-p.texinfo +@include include/fun-alexandria-proper-list-p.texinfo + +@include include/fun-alexandria-alist-plist.texinfo +@include include/fun-alexandria-plist-alist.texinfo +@include include/fun-alexandria-circular-list.texinfo +@include include/fun-alexandria-make-circular-list.texinfo +@include include/fun-alexandria-ensure-car.texinfo +@include include/fun-alexandria-ensure-cons.texinfo +@include include/fun-alexandria-ensure-list.texinfo +@include include/fun-alexandria-flatten.texinfo +@include include/fun-alexandria-lastcar.texinfo +@include include/fun-alexandria-setf-lastcar.texinfo +@include include/fun-alexandria-proper-list-length.texinfo +@include include/fun-alexandria-mappend.texinfo +@include include/fun-alexandria-map-product.texinfo +@include include/fun-alexandria-remove-from-plist.texinfo +@include include/fun-alexandria-delete-from-plist.texinfo +@include include/fun-alexandria-set-equal.texinfo +@include include/fun-alexandria-setp.texinfo + +@node Sequences +@comment node-name, next, previous, up +@chapter Sequences + +@include include/type-alexandria-proper-sequence.texinfo + +@include include/macro-alexandria-deletef.texinfo +@include include/macro-alexandria-removef.texinfo + +@include include/fun-alexandria-rotate.texinfo +@include include/fun-alexandria-shuffle.texinfo +@include include/fun-alexandria-random-elt.texinfo +@include include/fun-alexandria-emptyp.texinfo +@include include/fun-alexandria-sequence-of-length-p.texinfo +@include include/fun-alexandria-length-equals.texinfo +@include include/fun-alexandria-copy-sequence.texinfo +@include include/fun-alexandria-first-elt.texinfo +@include include/fun-alexandria-setf-first-elt.texinfo +@include include/fun-alexandria-last-elt.texinfo +@include include/fun-alexandria-setf-last-elt.texinfo +@include include/fun-alexandria-starts-with.texinfo +@include include/fun-alexandria-starts-with-subseq.texinfo +@include include/fun-alexandria-ends-with.texinfo +@include include/fun-alexandria-ends-with-subseq.texinfo +@include include/fun-alexandria-map-combinations.texinfo +@include include/fun-alexandria-map-derangements.texinfo +@include include/fun-alexandria-map-permutations.texinfo + +@node IO +@comment node-name, next, previous, up +@chapter IO + +@include include/fun-alexandria-read-stream-content-into-string.texinfo +@include include/fun-alexandria-read-file-into-string.texinfo +@include include/fun-alexandria-read-stream-content-into-byte-vector.texinfo +@include include/fun-alexandria-read-file-into-byte-vector.texinfo + +@node Macro Writing +@comment node-name, next, previous, up +@chapter Macro Writing + +@include include/macro-alexandria-once-only.texinfo +@include include/macro-alexandria-with-gensyms.texinfo +@include include/macro-alexandria-with-unique-names.texinfo +@include include/fun-alexandria-featurep.texinfo +@include include/fun-alexandria-parse-body.texinfo +@include include/fun-alexandria-parse-ordinary-lambda-list.texinfo + +@node Symbols +@comment node-name, next, previous, up +@chapter Symbols + +@include include/fun-alexandria-ensure-symbol.texinfo +@include include/fun-alexandria-format-symbol.texinfo +@include include/fun-alexandria-make-keyword.texinfo +@include include/fun-alexandria-make-gensym.texinfo +@include include/fun-alexandria-make-gensym-list.texinfo +@include include/fun-alexandria-symbolicate.texinfo + +@node Arrays +@comment node-name, next, previous, up +@chapter Arrays + +@include include/type-alexandria-array-index.texinfo +@include include/type-alexandria-array-length.texinfo +@include include/fun-alexandria-copy-array.texinfo + +@node Types +@comment node-name, next, previous, up +@chapter Types + +@include include/type-alexandria-string-designator.texinfo +@include include/macro-alexandria-coercef.texinfo +@include include/fun-alexandria-of-type.texinfo +@include include/fun-alexandria-type-equals.texinfo + +@node Numbers +@comment node-name, next, previous, up +@chapter Numbers + +@include include/macro-alexandria-maxf.texinfo +@include include/macro-alexandria-minf.texinfo + +@include include/fun-alexandria-binomial-coefficient.texinfo +@include include/fun-alexandria-count-permutations.texinfo +@include include/fun-alexandria-clamp.texinfo +@include include/fun-alexandria-lerp.texinfo +@include include/fun-alexandria-factorial.texinfo +@include include/fun-alexandria-subfactorial.texinfo +@include include/fun-alexandria-gaussian-random.texinfo +@include include/fun-alexandria-iota.texinfo +@include include/fun-alexandria-map-iota.texinfo +@include include/fun-alexandria-mean.texinfo +@include include/fun-alexandria-median.texinfo +@include include/fun-alexandria-variance.texinfo +@include include/fun-alexandria-standard-deviation.texinfo + +@bye diff --git a/third_party/lisp/alexandria/doc/docstrings.lisp b/third_party/lisp/alexandria/doc/docstrings.lisp new file mode 100644 index 000000000000..51dda07d09b7 --- /dev/null +++ b/third_party/lisp/alexandria/doc/docstrings.lisp @@ -0,0 +1,881 @@ +;;; -*- lisp -*- + +;;;; A docstring extractor for the sbcl manual. Creates +;;;; @include-ready documentation from the docstrings of exported +;;;; symbols of specified packages. + +;;;; This software is part of the SBCL software system. SBCL is in the +;;;; public domain and is provided with absolutely no warranty. See +;;;; the COPYING file for more information. +;;;; +;;;; Written by Rudi Schlatte <rudi@constantly.at>, mangled +;;;; by Nikodemus Siivola. + +;;;; TODO +;;;; * Verbatim text +;;;; * Quotations +;;;; * Method documentation untested +;;;; * Method sorting, somehow +;;;; * Index for macros & constants? +;;;; * This is getting complicated enough that tests would be good +;;;; * Nesting (currently only nested itemizations work) +;;;; * doc -> internal form -> texinfo (so that non-texinfo format are also +;;;; easily generated) + +;;;; FIXME: The description below is no longer complete. This +;;;; should possibly be turned into a contrib with proper documentation. + +;;;; Formatting heuristics (tweaked to format SAVE-LISP-AND-DIE sanely): +;;;; +;;;; Formats SYMBOL as @code{symbol}, or @var{symbol} if symbol is in +;;;; the argument list of the defun / defmacro. +;;;; +;;;; Lines starting with * or - that are followed by intented lines +;;;; are marked up with @itemize. +;;;; +;;;; Lines containing only a SYMBOL that are followed by indented +;;;; lines are marked up as @table @code, with the SYMBOL as the item. + +(eval-when (:compile-toplevel :load-toplevel :execute) + (require 'sb-introspect)) + +(defpackage :sb-texinfo + (:use :cl :sb-mop) + (:shadow #:documentation) + (:export #:generate-includes #:document-package) + (:documentation + "Tools to generate TexInfo documentation from docstrings.")) + +(in-package :sb-texinfo) + +;;;; various specials and parameters + +(defvar *texinfo-output*) +(defvar *texinfo-variables*) +(defvar *documentation-package*) +(defvar *base-package*) + +(defparameter *undocumented-packages* '(sb-pcl sb-int sb-kernel sb-sys sb-c)) + +(defparameter *documentation-types* + '(compiler-macro + function + method-combination + setf + ;;structure ; also handled by `type' + type + variable) + "A list of symbols accepted as second argument of `documentation'") + +(defparameter *character-replacements* + '((#\* . "star") (#\/ . "slash") (#\+ . "plus") + (#\< . "lt") (#\> . "gt") + (#\= . "equals")) + "Characters and their replacement names that `alphanumize' uses. If +the replacements contain any of the chars they're supposed to replace, +you deserve to lose.") + +(defparameter *characters-to-drop* '(#\\ #\` #\') + "Characters that should be removed by `alphanumize'.") + +(defparameter *texinfo-escaped-chars* "@{}" + "Characters that must be escaped with #\@ for Texinfo.") + +(defparameter *itemize-start-characters* '(#\* #\-) + "Characters that might start an itemization in docstrings when + at the start of a line.") + +(defparameter *symbol-characters* "ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890*:-+&#'" + "List of characters that make up symbols in a docstring.") + +(defparameter *symbol-delimiters* " ,.!?;") + +(defparameter *ordered-documentation-kinds* + '(package type structure condition class macro)) + +;;;; utilities + +(defun flatten (list) + (cond ((null list) + nil) + ((consp (car list)) + (nconc (flatten (car list)) (flatten (cdr list)))) + ((null (cdr list)) + (cons (car list) nil)) + (t + (cons (car list) (flatten (cdr list)))))) + +(defun whitespacep (char) + (find char #(#\tab #\space #\page))) + +(defun setf-name-p (name) + (or (symbolp name) + (and (listp name) (= 2 (length name)) (eq (car name) 'setf)))) + +(defgeneric specializer-name (specializer)) + +(defmethod specializer-name ((specializer eql-specializer)) + (list 'eql (eql-specializer-object specializer))) + +(defmethod specializer-name ((specializer class)) + (class-name specializer)) + +(defun ensure-class-precedence-list (class) + (unless (class-finalized-p class) + (finalize-inheritance class)) + (class-precedence-list class)) + +(defun specialized-lambda-list (method) + ;; courtecy of AMOP p. 61 + (let* ((specializers (method-specializers method)) + (lambda-list (method-lambda-list method)) + (n-required (length specializers))) + (append (mapcar (lambda (arg specializer) + (if (eq specializer (find-class 't)) + arg + `(,arg ,(specializer-name specializer)))) + (subseq lambda-list 0 n-required) + specializers) + (subseq lambda-list n-required)))) + +(defun string-lines (string) + "Lines in STRING as a vector." + (coerce (with-input-from-string (s string) + (loop for line = (read-line s nil nil) + while line collect line)) + 'vector)) + +(defun indentation (line) + "Position of first non-SPACE character in LINE." + (position-if-not (lambda (c) (char= c #\Space)) line)) + +(defun docstring (x doc-type) + (cl:documentation x doc-type)) + +(defun flatten-to-string (list) + (format nil "~{~A~^-~}" (flatten list))) + +(defun alphanumize (original) + "Construct a string without characters like *`' that will f-star-ck +up filename handling. See `*character-replacements*' and +`*characters-to-drop*' for customization." + (let ((name (remove-if (lambda (x) (member x *characters-to-drop*)) + (if (listp original) + (flatten-to-string original) + (string original)))) + (chars-to-replace (mapcar #'car *character-replacements*))) + (flet ((replacement-delimiter (index) + (cond ((or (< index 0) (>= index (length name))) "") + ((alphanumericp (char name index)) "-") + (t "")))) + (loop for index = (position-if #'(lambda (x) (member x chars-to-replace)) + name) + while index + do (setf name (concatenate 'string (subseq name 0 index) + (replacement-delimiter (1- index)) + (cdr (assoc (aref name index) + *character-replacements*)) + (replacement-delimiter (1+ index)) + (subseq name (1+ index)))))) + name)) + +;;;; generating various names + +(defgeneric name (thing) + (:documentation "Name for a documented thing. Names are either +symbols or lists of symbols.")) + +(defmethod name ((symbol symbol)) + symbol) + +(defmethod name ((cons cons)) + cons) + +(defmethod name ((package package)) + (short-package-name package)) + +(defmethod name ((method method)) + (list + (generic-function-name (method-generic-function method)) + (method-qualifiers method) + (specialized-lambda-list method))) + +;;; Node names for DOCUMENTATION instances + +(defgeneric name-using-kind/name (kind name doc)) + +(defmethod name-using-kind/name (kind (name string) doc) + (declare (ignore kind doc)) + name) + +(defmethod name-using-kind/name (kind (name symbol) doc) + (declare (ignore kind)) + (format nil "~@[~A:~]~A" (short-package-name (get-package doc)) name)) + +(defmethod name-using-kind/name (kind (name list) doc) + (declare (ignore kind)) + (assert (setf-name-p name)) + (format nil "(setf ~@[~A:~]~A)" (short-package-name (get-package doc)) (second name))) + +(defmethod name-using-kind/name ((kind (eql 'method)) name doc) + (format nil "~A~{ ~A~} ~A" + (name-using-kind/name nil (first name) doc) + (second name) + (third name))) + +(defun node-name (doc) + "Returns TexInfo node name as a string for a DOCUMENTATION instance." + (let ((kind (get-kind doc))) + (format nil "~:(~A~) ~(~A~)" kind (name-using-kind/name kind (get-name doc) doc)))) + +(defun short-package-name (package) + (unless (eq package *base-package*) + (car (sort (copy-list (cons (package-name package) (package-nicknames package))) + #'< :key #'length)))) + +;;; Definition titles for DOCUMENTATION instances + +(defgeneric title-using-kind/name (kind name doc)) + +(defmethod title-using-kind/name (kind (name string) doc) + (declare (ignore kind doc)) + name) + +(defmethod title-using-kind/name (kind (name symbol) doc) + (declare (ignore kind)) + (format nil "~@[~A:~]~A" (short-package-name (get-package doc)) name)) + +(defmethod title-using-kind/name (kind (name list) doc) + (declare (ignore kind)) + (assert (setf-name-p name)) + (format nil "(setf ~@[~A:~]~A)" (short-package-name (get-package doc)) (second name))) + +(defmethod title-using-kind/name ((kind (eql 'method)) name doc) + (format nil "~{~A ~}~A" + (second name) + (title-using-kind/name nil (first name) doc))) + +(defun title-name (doc) + "Returns a string to be used as name of the definition." + (string-downcase (title-using-kind/name (get-kind doc) (get-name doc) doc))) + +(defun include-pathname (doc) + (let* ((kind (get-kind doc)) + (name (nstring-downcase + (if (eq 'package kind) + (format nil "package-~A" (alphanumize (get-name doc))) + (format nil "~A-~A-~A" + (case (get-kind doc) + ((function generic-function) "fun") + (structure "struct") + (variable "var") + (otherwise (symbol-name (get-kind doc)))) + (alphanumize (let ((*base-package* nil)) + (short-package-name (get-package doc)))) + (alphanumize (get-name doc))))))) + (make-pathname :name name :type "texinfo"))) + +;;;; documentation class and related methods + +(defclass documentation () + ((name :initarg :name :reader get-name) + (kind :initarg :kind :reader get-kind) + (string :initarg :string :reader get-string) + (children :initarg :children :initform nil :reader get-children) + (package :initform *documentation-package* :reader get-package))) + +(defmethod print-object ((documentation documentation) stream) + (print-unreadable-object (documentation stream :type t) + (princ (list (get-kind documentation) (get-name documentation)) stream))) + +(defgeneric make-documentation (x doc-type string)) + +(defmethod make-documentation ((x package) doc-type string) + (declare (ignore doc-type)) + (make-instance 'documentation + :name (name x) + :kind 'package + :string string)) + +(defmethod make-documentation (x (doc-type (eql 'function)) string) + (declare (ignore doc-type)) + (let* ((fdef (and (fboundp x) (fdefinition x))) + (name x) + (kind (cond ((and (symbolp x) (special-operator-p x)) + 'special-operator) + ((and (symbolp x) (macro-function x)) + 'macro) + ((typep fdef 'generic-function) + (assert (or (symbolp name) (setf-name-p name))) + 'generic-function) + (fdef + (assert (or (symbolp name) (setf-name-p name))) + 'function))) + (children (when (eq kind 'generic-function) + (collect-gf-documentation fdef)))) + (make-instance 'documentation + :name (name x) + :string string + :kind kind + :children children))) + +(defmethod make-documentation ((x method) doc-type string) + (declare (ignore doc-type)) + (make-instance 'documentation + :name (name x) + :kind 'method + :string string)) + +(defmethod make-documentation (x (doc-type (eql 'type)) string) + (make-instance 'documentation + :name (name x) + :string string + :kind (etypecase (find-class x nil) + (structure-class 'structure) + (standard-class 'class) + (sb-pcl::condition-class 'condition) + ((or built-in-class null) 'type)))) + +(defmethod make-documentation (x (doc-type (eql 'variable)) string) + (make-instance 'documentation + :name (name x) + :string string + :kind (if (constantp x) + 'constant + 'variable))) + +(defmethod make-documentation (x (doc-type (eql 'setf)) string) + (declare (ignore doc-type)) + (make-instance 'documentation + :name (name x) + :kind 'setf-expander + :string string)) + +(defmethod make-documentation (x doc-type string) + (make-instance 'documentation + :name (name x) + :kind doc-type + :string string)) + +(defun maybe-documentation (x doc-type) + "Returns a DOCUMENTATION instance for X and DOC-TYPE, or NIL if +there is no corresponding docstring." + (let ((docstring (docstring x doc-type))) + (when docstring + (make-documentation x doc-type docstring)))) + +(defun lambda-list (doc) + (case (get-kind doc) + ((package constant variable type structure class condition nil) + nil) + (method + (third (get-name doc))) + (t + ;; KLUDGE: Eugh. + ;; + ;; believe it or not, the above comment was written before CSR + ;; came along and obfuscated this. (2005-07-04) + (when (symbolp (get-name doc)) + (labels ((clean (x &key optional key) + (typecase x + (atom x) + ((cons (member &optional)) + (cons (car x) (clean (cdr x) :optional t))) + ((cons (member &key)) + (cons (car x) (clean (cdr x) :key t))) + ((cons (member &whole &environment)) + ;; Skip these + (clean (cdr x) :optional optional :key key)) + ((cons cons) + (cons + (cond (key (if (consp (caar x)) + (caaar x) + (caar x))) + (optional (caar x)) + (t (clean (car x)))) + (clean (cdr x) :key key :optional optional))) + (cons + (cons + (cond ((or key optional) (car x)) + (t (clean (car x)))) + (clean (cdr x) :key key :optional optional)))))) + (clean (sb-introspect:function-lambda-list (get-name doc)))))))) + +(defun get-string-name (x) + (let ((name (get-name x))) + (cond ((symbolp name) + (symbol-name name)) + ((and (consp name) (eq 'setf (car name))) + (symbol-name (second name))) + ((stringp name) + name) + (t + (error "Don't know which symbol to use for name ~S" name))))) + +(defun documentation< (x y) + (let ((p1 (position (get-kind x) *ordered-documentation-kinds*)) + (p2 (position (get-kind y) *ordered-documentation-kinds*))) + (if (or (not (and p1 p2)) (= p1 p2)) + (string< (get-string-name x) (get-string-name y)) + (< p1 p2)))) + +;;;; turning text into texinfo + +(defun escape-for-texinfo (string &optional downcasep) + "Return STRING with characters in *TEXINFO-ESCAPED-CHARS* escaped +with #\@. Optionally downcase the result." + (let ((result (with-output-to-string (s) + (loop for char across string + when (find char *texinfo-escaped-chars*) + do (write-char #\@ s) + do (write-char char s))))) + (if downcasep (nstring-downcase result) result))) + +(defun empty-p (line-number lines) + (and (< -1 line-number (length lines)) + (not (indentation (svref lines line-number))))) + +;;; line markups + +(defvar *not-symbols* '("ANSI" "CLHS")) + +(defun locate-symbols (line) + "Return a list of index pairs of symbol-like parts of LINE." + ;; This would be a good application for a regex ... + (let (result) + (flet ((grab (start end) + (unless (member (subseq line start end) '("ANSI" "CLHS")) + (push (list start end) result)))) + (do ((begin nil) + (maybe-begin t) + (i 0 (1+ i))) + ((= i (length line)) + ;; symbol at end of line + (when (and begin (or (> i (1+ begin)) + (not (member (char line begin) '(#\A #\I))))) + (grab begin i)) + (nreverse result)) + (cond + ((and begin (find (char line i) *symbol-delimiters*)) + ;; symbol end; remember it if it's not "A" or "I" + (when (or (> i (1+ begin)) (not (member (char line begin) '(#\A #\I)))) + (grab begin i)) + (setf begin nil + maybe-begin t)) + ((and begin (not (find (char line i) *symbol-characters*))) + ;; Not a symbol: abort + (setf begin nil)) + ((and maybe-begin (not begin) (find (char line i) *symbol-characters*)) + ;; potential symbol begin at this position + (setf begin i + maybe-begin nil)) + ((find (char line i) *symbol-delimiters*) + ;; potential symbol begin after this position + (setf maybe-begin t)) + (t + ;; Not reading a symbol, not at potential start of symbol + (setf maybe-begin nil))))))) + +(defun texinfo-line (line) + "Format symbols in LINE texinfo-style: either as code or as +variables if the symbol in question is contained in symbols +*TEXINFO-VARIABLES*." + (with-output-to-string (result) + (let ((last 0)) + (dolist (symbol/index (locate-symbols line)) + (write-string (subseq line last (first symbol/index)) result) + (let ((symbol-name (apply #'subseq line symbol/index))) + (format result (if (member symbol-name *texinfo-variables* + :test #'string=) + "@var{~A}" + "@code{~A}") + (string-downcase symbol-name))) + (setf last (second symbol/index))) + (write-string (subseq line last) result)))) + +;;; lisp sections + +(defun lisp-section-p (line line-number lines) + "Returns T if the given LINE looks like start of lisp code -- +ie. if it starts with whitespace followed by a paren or +semicolon, and the previous line is empty" + (let ((offset (indentation line))) + (and offset + (plusp offset) + (find (find-if-not #'whitespacep line) "(;") + (empty-p (1- line-number) lines)))) + +(defun collect-lisp-section (lines line-number) + (let ((lisp (loop for index = line-number then (1+ index) + for line = (and (< index (length lines)) (svref lines index)) + while (indentation line) + collect line))) + (values (length lisp) `("@lisp" ,@lisp "@end lisp")))) + +;;; itemized sections + +(defun maybe-itemize-offset (line) + "Return NIL or the indentation offset if LINE looks like it starts +an item in an itemization." + (let* ((offset (indentation line)) + (char (when offset (char line offset)))) + (and offset + (member char *itemize-start-characters* :test #'char=) + (char= #\Space (find-if-not (lambda (c) (char= c char)) + line :start offset)) + offset))) + +(defun collect-maybe-itemized-section (lines starting-line) + ;; Return index of next line to be processed outside + (let ((this-offset (maybe-itemize-offset (svref lines starting-line))) + (result nil) + (lines-consumed 0)) + (loop for line-number from starting-line below (length lines) + for line = (svref lines line-number) + for indentation = (indentation line) + for offset = (maybe-itemize-offset line) + do (cond + ((not indentation) + ;; empty line -- inserts paragraph. + (push "" result) + (incf lines-consumed)) + ((and offset (> indentation this-offset)) + ;; nested itemization -- handle recursively + ;; FIXME: tables in itemizations go wrong + (multiple-value-bind (sub-lines-consumed sub-itemization) + (collect-maybe-itemized-section lines line-number) + (when sub-lines-consumed + (incf line-number (1- sub-lines-consumed)) ; +1 on next loop + (incf lines-consumed sub-lines-consumed) + (setf result (nconc (nreverse sub-itemization) result))))) + ((and offset (= indentation this-offset)) + ;; start of new item + (push (format nil "@item ~A" + (texinfo-line (subseq line (1+ offset)))) + result) + (incf lines-consumed)) + ((and (not offset) (> indentation this-offset)) + ;; continued item from previous line + (push (texinfo-line line) result) + (incf lines-consumed)) + (t + ;; end of itemization + (loop-finish)))) + ;; a single-line itemization isn't. + (if (> (count-if (lambda (line) (> (length line) 0)) result) 1) + (values lines-consumed `("@itemize" ,@(reverse result) "@end itemize")) + nil))) + +;;; table sections + +(defun tabulation-body-p (offset line-number lines) + (when (< line-number (length lines)) + (let ((offset2 (indentation (svref lines line-number)))) + (and offset2 (< offset offset2))))) + +(defun tabulation-p (offset line-number lines direction) + (let ((step (ecase direction + (:backwards (1- line-number)) + (:forwards (1+ line-number))))) + (when (and (plusp line-number) (< line-number (length lines))) + (and (eql offset (indentation (svref lines line-number))) + (or (when (eq direction :backwards) + (empty-p step lines)) + (tabulation-p offset step lines direction) + (tabulation-body-p offset step lines)))))) + +(defun maybe-table-offset (line-number lines) + "Return NIL or the indentation offset if LINE looks like it starts +an item in a tabulation. Ie, if it is (1) indented, (2) preceded by an +empty line, another tabulation label, or a tabulation body, (3) and +followed another tabulation label or a tabulation body." + (let* ((line (svref lines line-number)) + (offset (indentation line)) + (prev (1- line-number)) + (next (1+ line-number))) + (when (and offset (plusp offset)) + (and (or (empty-p prev lines) + (tabulation-body-p offset prev lines) + (tabulation-p offset prev lines :backwards)) + (or (tabulation-body-p offset next lines) + (tabulation-p offset next lines :forwards)) + offset)))) + +;;; FIXME: This and itemization are very similar: could they share +;;; some code, mayhap? + +(defun collect-maybe-table-section (lines starting-line) + ;; Return index of next line to be processed outside + (let ((this-offset (maybe-table-offset starting-line lines)) + (result nil) + (lines-consumed 0)) + (loop for line-number from starting-line below (length lines) + for line = (svref lines line-number) + for indentation = (indentation line) + for offset = (maybe-table-offset line-number lines) + do (cond + ((not indentation) + ;; empty line -- inserts paragraph. + (push "" result) + (incf lines-consumed)) + ((and offset (= indentation this-offset)) + ;; start of new item, or continuation of previous item + (if (and result (search "@item" (car result) :test #'char=)) + (push (format nil "@itemx ~A" (texinfo-line line)) + result) + (progn + (push "" result) + (push (format nil "@item ~A" (texinfo-line line)) + result))) + (incf lines-consumed)) + ((> indentation this-offset) + ;; continued item from previous line + (push (texinfo-line line) result) + (incf lines-consumed)) + (t + ;; end of itemization + (loop-finish)))) + ;; a single-line table isn't. + (if (> (count-if (lambda (line) (> (length line) 0)) result) 1) + (values lines-consumed + `("" "@table @emph" ,@(reverse result) "@end table" "")) + nil))) + +;;; section markup + +(defmacro with-maybe-section (index &rest forms) + `(multiple-value-bind (count collected) (progn ,@forms) + (when count + (dolist (line collected) + (write-line line *texinfo-output*)) + (incf ,index (1- count))))) + +(defun write-texinfo-string (string &optional lambda-list) + "Try to guess as much formatting for a raw docstring as possible." + (let ((*texinfo-variables* (flatten lambda-list)) + (lines (string-lines (escape-for-texinfo string nil)))) + (loop for line-number from 0 below (length lines) + for line = (svref lines line-number) + do (cond + ((with-maybe-section line-number + (and (lisp-section-p line line-number lines) + (collect-lisp-section lines line-number)))) + ((with-maybe-section line-number + (and (maybe-itemize-offset line) + (collect-maybe-itemized-section lines line-number)))) + ((with-maybe-section line-number + (and (maybe-table-offset line-number lines) + (collect-maybe-table-section lines line-number)))) + (t + (write-line (texinfo-line line) *texinfo-output*)))))) + +;;;; texinfo formatting tools + +(defun hide-superclass-p (class-name super-name) + (let ((super-package (symbol-package super-name))) + (or + ;; KLUDGE: We assume that we don't want to advertise internal + ;; classes in CP-lists, unless the symbol we're documenting is + ;; internal as well. + (and (member super-package #.'(mapcar #'find-package *undocumented-packages*)) + (not (eq super-package (symbol-package class-name)))) + ;; KLUDGE: We don't generally want to advertise SIMPLE-ERROR or + ;; SIMPLE-CONDITION in the CPLs of conditions that inherit them + ;; simply as a matter of convenience. The assumption here is that + ;; the inheritance is incidental unless the name of the condition + ;; begins with SIMPLE-. + (and (member super-name '(simple-error simple-condition)) + (let ((prefix "SIMPLE-")) + (mismatch prefix (string class-name) :end2 (length prefix))) + t ; don't return number from MISMATCH + )))) + +(defun hide-slot-p (symbol slot) + ;; FIXME: There is no pricipal reason to avoid the slot docs fo + ;; structures and conditions, but their DOCUMENTATION T doesn't + ;; currently work with them the way we'd like. + (not (and (typep (find-class symbol nil) 'standard-class) + (docstring slot t)))) + +(defun texinfo-anchor (doc) + (format *texinfo-output* "@anchor{~A}~%" (node-name doc))) + +;;; KLUDGE: &AUX *PRINT-PRETTY* here means "no linebreaks please" +(defun texinfo-begin (doc &aux *print-pretty*) + (let ((kind (get-kind doc))) + (format *texinfo-output* "@~A {~:(~A~)} ~({~A}~@[ ~{~A~^ ~}~]~)~%" + (case kind + ((package constant variable) + "defvr") + ((structure class condition type) + "deftp") + (t + "deffn")) + (map 'string (lambda (char) (if (eql char #\-) #\Space char)) (string kind)) + (title-name doc) + ;; &foo would be amusingly bold in the pdf thanks to TeX/Texinfo + ;; interactions,so we escape the ampersand -- amusingly for TeX. + ;; sbcl.texinfo defines macros that expand @&key and friends to &key. + (mapcar (lambda (name) + (if (member name lambda-list-keywords) + (format nil "@~A" name) + name)) + (lambda-list doc))))) + +(defun texinfo-index (doc) + (let ((title (title-name doc))) + (case (get-kind doc) + ((structure type class condition) + (format *texinfo-output* "@tindex ~A~%" title)) + ((variable constant) + (format *texinfo-output* "@vindex ~A~%" title)) + ((compiler-macro function method-combination macro generic-function) + (format *texinfo-output* "@findex ~A~%" title))))) + +(defun texinfo-inferred-body (doc) + (when (member (get-kind doc) '(class structure condition)) + (let ((name (get-name doc))) + ;; class precedence list + (format *texinfo-output* "Class precedence list: @code{~(~{@lw{~A}~^, ~}~)}~%~%" + (remove-if (lambda (class) (hide-superclass-p name class)) + (mapcar #'class-name (ensure-class-precedence-list (find-class name))))) + ;; slots + (let ((slots (remove-if (lambda (slot) (hide-slot-p name slot)) + (class-direct-slots (find-class name))))) + (when slots + (format *texinfo-output* "Slots:~%@itemize~%") + (dolist (slot slots) + (format *texinfo-output* + "@item ~(@code{~A}~#[~:; --- ~]~ + ~:{~2*~@[~2:*~A~P: ~{@code{@w{~S}}~^, ~}~]~:^; ~}~)~%~%" + (slot-definition-name slot) + (remove + nil + (mapcar + (lambda (name things) + (if things + (list name (length things) things))) + '("initarg" "reader" "writer") + (list + (slot-definition-initargs slot) + (slot-definition-readers slot) + (slot-definition-writers slot))))) + ;; FIXME: Would be neater to handler as children + (write-texinfo-string (docstring slot t))) + (format *texinfo-output* "@end itemize~%~%")))))) + +(defun texinfo-body (doc) + (write-texinfo-string (get-string doc))) + +(defun texinfo-end (doc) + (write-line (case (get-kind doc) + ((package variable constant) "@end defvr") + ((structure type class condition) "@end deftp") + (t "@end deffn")) + *texinfo-output*)) + +(defun write-texinfo (doc) + "Writes TexInfo for a DOCUMENTATION instance to *TEXINFO-OUTPUT*." + (texinfo-anchor doc) + (texinfo-begin doc) + (texinfo-index doc) + (texinfo-inferred-body doc) + (texinfo-body doc) + (texinfo-end doc) + ;; FIXME: Children should be sorted one way or another + (mapc #'write-texinfo (get-children doc))) + +;;;; main logic + +(defun collect-gf-documentation (gf) + "Collects method documentation for the generic function GF" + (loop for method in (generic-function-methods gf) + for doc = (maybe-documentation method t) + when doc + collect doc)) + +(defun collect-name-documentation (name) + (loop for type in *documentation-types* + for doc = (maybe-documentation name type) + when doc + collect doc)) + +(defun collect-symbol-documentation (symbol) + "Collects all docs for a SYMBOL and (SETF SYMBOL), returns a list of +the form DOC instances. See `*documentation-types*' for the possible +values of doc-type." + (nconc (collect-name-documentation symbol) + (collect-name-documentation (list 'setf symbol)))) + +(defun collect-documentation (package) + "Collects all documentation for all external symbols of the given +package, as well as for the package itself." + (let* ((*documentation-package* (find-package package)) + (docs nil)) + (check-type package package) + (do-external-symbols (symbol package) + (setf docs (nconc (collect-symbol-documentation symbol) docs))) + (let ((doc (maybe-documentation *documentation-package* t))) + (when doc + (push doc docs))) + docs)) + +(defmacro with-texinfo-file (pathname &body forms) + `(with-open-file (*texinfo-output* ,pathname + :direction :output + :if-does-not-exist :create + :if-exists :supersede) + ,@forms)) + +(defun write-ifnottex () + ;; We use @&key, etc to escape & from TeX in lambda lists -- so we need to + ;; define them for info as well. + (flet ((macro (name) + (let ((string (string-downcase name))) + (format *texinfo-output* "@macro ~A~%~A~%@end macro~%" string string)))) + (macro '&allow-other-keys) + (macro '&optional) + (macro '&rest) + (macro '&key) + (macro '&body))) + +(defun generate-includes (directory packages &key (base-package :cl-user)) + "Create files in `directory' containing Texinfo markup of all +docstrings of each exported symbol in `packages'. `directory' is +created if necessary. If you supply a namestring that doesn't end in a +slash, you lose. The generated files are of the form +\"<doc-type>_<packagename>_<symbol-name>.texinfo\" and can be included +via @include statements. Texinfo syntax-significant characters are +escaped in symbol names, but if a docstring contains invalid Texinfo +markup, you lose." + (handler-bind ((warning #'muffle-warning)) + (let ((directory (merge-pathnames (pathname directory))) + (*base-package* (find-package base-package))) + (ensure-directories-exist directory) + (dolist (package packages) + (dolist (doc (collect-documentation (find-package package))) + (with-texinfo-file (merge-pathnames (include-pathname doc) directory) + (write-texinfo doc)))) + (with-texinfo-file (merge-pathnames "ifnottex.texinfo" directory) + (write-ifnottex)) + directory))) + +(defun document-package (package &optional filename) + "Create a file containing all available documentation for the +exported symbols of `package' in Texinfo format. If `filename' is not +supplied, a file \"<packagename>.texinfo\" is generated. + +The definitions can be referenced using Texinfo statements like +@ref{<doc-type>_<packagename>_<symbol-name>.texinfo}. Texinfo +syntax-significant characters are escaped in symbol names, but if a +docstring contains invalid Texinfo markup, you lose." + (handler-bind ((warning #'muffle-warning)) + (let* ((package (find-package package)) + (filename (or filename (make-pathname + :name (string-downcase (short-package-name package)) + :type "texinfo"))) + (docs (sort (collect-documentation package) #'documentation<))) + (with-texinfo-file filename + (dolist (doc docs) + (write-texinfo doc))) + filename))) diff --git a/third_party/lisp/alexandria/features.lisp b/third_party/lisp/alexandria/features.lisp new file mode 100644 index 000000000000..67348dbba43b --- /dev/null +++ b/third_party/lisp/alexandria/features.lisp @@ -0,0 +1,14 @@ +(in-package :alexandria) + +(defun featurep (feature-expression) + "Returns T if the argument matches the state of the *FEATURES* +list and NIL if it does not. FEATURE-EXPRESSION can be any atom +or list acceptable to the reader macros #+ and #-." + (etypecase feature-expression + (symbol (not (null (member feature-expression *features*)))) + (cons (check-type (first feature-expression) symbol) + (eswitch ((first feature-expression) :test 'string=) + (:and (every #'featurep (rest feature-expression))) + (:or (some #'featurep (rest feature-expression))) + (:not (assert (= 2 (length feature-expression))) + (not (featurep (second feature-expression)))))))) diff --git a/third_party/lisp/alexandria/functions.lisp b/third_party/lisp/alexandria/functions.lisp new file mode 100644 index 000000000000..dd83e38b4ebc --- /dev/null +++ b/third_party/lisp/alexandria/functions.lisp @@ -0,0 +1,161 @@ +(in-package :alexandria) + +;;; To propagate return type and allow the compiler to eliminate the IF when +;;; it is known if the argument is function or not. +(declaim (inline ensure-function)) + +(declaim (ftype (function (t) (values function &optional)) + ensure-function)) +(defun ensure-function (function-designator) + "Returns the function designated by FUNCTION-DESIGNATOR: +if FUNCTION-DESIGNATOR is a function, it is returned, otherwise +it must be a function name and its FDEFINITION is returned." + (if (functionp function-designator) + function-designator + (fdefinition function-designator))) + +(define-modify-macro ensure-functionf/1 () ensure-function) + +(defmacro ensure-functionf (&rest places) + "Multiple-place modify macro for ENSURE-FUNCTION: ensures that each of +PLACES contains a function." + `(progn ,@(mapcar (lambda (x) `(ensure-functionf/1 ,x)) places))) + +(defun disjoin (predicate &rest more-predicates) + "Returns a function that applies each of PREDICATE and MORE-PREDICATE +functions in turn to its arguments, returning the primary value of the first +predicate that returns true, without calling the remaining predicates. +If none of the predicates returns true, NIL is returned." + (declare (optimize (speed 3) (safety 1) (debug 1))) + (let ((predicate (ensure-function predicate)) + (more-predicates (mapcar #'ensure-function more-predicates))) + (lambda (&rest arguments) + (or (apply predicate arguments) + (some (lambda (p) + (declare (type function p)) + (apply p arguments)) + more-predicates))))) + +(defun conjoin (predicate &rest more-predicates) + "Returns a function that applies each of PREDICATE and MORE-PREDICATE +functions in turn to its arguments, returning NIL if any of the predicates +returns false, without calling the remaining predicates. If none of the +predicates returns false, returns the primary value of the last predicate." + (if (null more-predicates) + predicate + (lambda (&rest arguments) + (and (apply predicate arguments) + ;; Cannot simply use CL:EVERY because we want to return the + ;; non-NIL value of the last predicate if all succeed. + (do ((tail (cdr more-predicates) (cdr tail)) + (head (car more-predicates) (car tail))) + ((not tail) + (apply head arguments)) + (unless (apply head arguments) + (return nil))))))) + + +(defun compose (function &rest more-functions) + "Returns a function composed of FUNCTION and MORE-FUNCTIONS that applies its +arguments to to each in turn, starting from the rightmost of MORE-FUNCTIONS, +and then calling the next one with the primary value of the last." + (declare (optimize (speed 3) (safety 1) (debug 1))) + (reduce (lambda (f g) + (let ((f (ensure-function f)) + (g (ensure-function g))) + (lambda (&rest arguments) + (declare (dynamic-extent arguments)) + (funcall f (apply g arguments))))) + more-functions + :initial-value function)) + +(define-compiler-macro compose (function &rest more-functions) + (labels ((compose-1 (funs) + (if (cdr funs) + `(funcall ,(car funs) ,(compose-1 (cdr funs))) + `(apply ,(car funs) arguments)))) + (let* ((args (cons function more-functions)) + (funs (make-gensym-list (length args) "COMPOSE"))) + `(let ,(loop for f in funs for arg in args + collect `(,f (ensure-function ,arg))) + (declare (optimize (speed 3) (safety 1) (debug 1))) + (lambda (&rest arguments) + (declare (dynamic-extent arguments)) + ,(compose-1 funs)))))) + +(defun multiple-value-compose (function &rest more-functions) + "Returns a function composed of FUNCTION and MORE-FUNCTIONS that applies +its arguments to each in turn, starting from the rightmost of +MORE-FUNCTIONS, and then calling the next one with all the return values of +the last." + (declare (optimize (speed 3) (safety 1) (debug 1))) + (reduce (lambda (f g) + (let ((f (ensure-function f)) + (g (ensure-function g))) + (lambda (&rest arguments) + (declare (dynamic-extent arguments)) + (multiple-value-call f (apply g arguments))))) + more-functions + :initial-value function)) + +(define-compiler-macro multiple-value-compose (function &rest more-functions) + (labels ((compose-1 (funs) + (if (cdr funs) + `(multiple-value-call ,(car funs) ,(compose-1 (cdr funs))) + `(apply ,(car funs) arguments)))) + (let* ((args (cons function more-functions)) + (funs (make-gensym-list (length args) "MV-COMPOSE"))) + `(let ,(mapcar #'list funs args) + (declare (optimize (speed 3) (safety 1) (debug 1))) + (lambda (&rest arguments) + (declare (dynamic-extent arguments)) + ,(compose-1 funs)))))) + +(declaim (inline curry rcurry)) + +(defun curry (function &rest arguments) + "Returns a function that applies ARGUMENTS and the arguments +it is called with to FUNCTION." + (declare (optimize (speed 3) (safety 1))) + (let ((fn (ensure-function function))) + (lambda (&rest more) + (declare (dynamic-extent more)) + ;; Using M-V-C we don't need to append the arguments. + (multiple-value-call fn (values-list arguments) (values-list more))))) + +(define-compiler-macro curry (function &rest arguments) + (let ((curries (make-gensym-list (length arguments) "CURRY")) + (fun (gensym "FUN"))) + `(let ((,fun (ensure-function ,function)) + ,@(mapcar #'list curries arguments)) + (declare (optimize (speed 3) (safety 1))) + (lambda (&rest more) + (declare (dynamic-extent more)) + (apply ,fun ,@curries more))))) + +(defun rcurry (function &rest arguments) + "Returns a function that applies the arguments it is called +with and ARGUMENTS to FUNCTION." + (declare (optimize (speed 3) (safety 1))) + (let ((fn (ensure-function function))) + (lambda (&rest more) + (declare (dynamic-extent more)) + (multiple-value-call fn (values-list more) (values-list arguments))))) + +(define-compiler-macro rcurry (function &rest arguments) + (let ((rcurries (make-gensym-list (length arguments) "RCURRY")) + (fun (gensym "FUN"))) + `(let ((,fun (ensure-function ,function)) + ,@(mapcar #'list rcurries arguments)) + (declare (optimize (speed 3) (safety 1))) + (lambda (&rest more) + (declare (dynamic-extent more)) + (multiple-value-call ,fun (values-list more) ,@rcurries))))) + +(declaim (notinline curry rcurry)) + +(defmacro named-lambda (name lambda-list &body body) + "Expands into a lambda-expression within whose BODY NAME denotes the +corresponding function." + `(labels ((,name ,lambda-list ,@body)) + #',name)) diff --git a/third_party/lisp/alexandria/hash-tables.lisp b/third_party/lisp/alexandria/hash-tables.lisp new file mode 100644 index 000000000000..a9f790220405 --- /dev/null +++ b/third_party/lisp/alexandria/hash-tables.lisp @@ -0,0 +1,101 @@ +(in-package :alexandria) + +(defmacro ensure-gethash (key hash-table &optional default) + "Like GETHASH, but if KEY is not found in the HASH-TABLE saves the DEFAULT +under key before returning it. Secondary return value is true if key was +already in the table." + (once-only (key hash-table) + (with-unique-names (value presentp) + `(multiple-value-bind (,value ,presentp) (gethash ,key ,hash-table) + (if ,presentp + (values ,value ,presentp) + (values (setf (gethash ,key ,hash-table) ,default) nil)))))) + +(defun copy-hash-table (table &key key test size + rehash-size rehash-threshold) + "Returns a copy of hash table TABLE, with the same keys and values +as the TABLE. The copy has the same properties as the original, unless +overridden by the keyword arguments. + +Before each of the original values is set into the new hash-table, KEY +is invoked on the value. As KEY defaults to CL:IDENTITY, a shallow +copy is returned by default." + (setf key (or key 'identity)) + (setf test (or test (hash-table-test table))) + (setf size (or size (hash-table-size table))) + (setf rehash-size (or rehash-size (hash-table-rehash-size table))) + (setf rehash-threshold (or rehash-threshold (hash-table-rehash-threshold table))) + (let ((copy (make-hash-table :test test :size size + :rehash-size rehash-size + :rehash-threshold rehash-threshold))) + (maphash (lambda (k v) + (setf (gethash k copy) (funcall key v))) + table) + copy)) + +(declaim (inline maphash-keys)) +(defun maphash-keys (function table) + "Like MAPHASH, but calls FUNCTION with each key in the hash table TABLE." + (maphash (lambda (k v) + (declare (ignore v)) + (funcall function k)) + table)) + +(declaim (inline maphash-values)) +(defun maphash-values (function table) + "Like MAPHASH, but calls FUNCTION with each value in the hash table TABLE." + (maphash (lambda (k v) + (declare (ignore k)) + (funcall function v)) + table)) + +(defun hash-table-keys (table) + "Returns a list containing the keys of hash table TABLE." + (let ((keys nil)) + (maphash-keys (lambda (k) + (push k keys)) + table) + keys)) + +(defun hash-table-values (table) + "Returns a list containing the values of hash table TABLE." + (let ((values nil)) + (maphash-values (lambda (v) + (push v values)) + table) + values)) + +(defun hash-table-alist (table) + "Returns an association list containing the keys and values of hash table +TABLE." + (let ((alist nil)) + (maphash (lambda (k v) + (push (cons k v) alist)) + table) + alist)) + +(defun hash-table-plist (table) + "Returns a property list containing the keys and values of hash table +TABLE." + (let ((plist nil)) + (maphash (lambda (k v) + (setf plist (list* k v plist))) + table) + plist)) + +(defun alist-hash-table (alist &rest hash-table-initargs) + "Returns a hash table containing the keys and values of the association list +ALIST. Hash table is initialized using the HASH-TABLE-INITARGS." + (let ((table (apply #'make-hash-table hash-table-initargs))) + (dolist (cons alist) + (ensure-gethash (car cons) table (cdr cons))) + table)) + +(defun plist-hash-table (plist &rest hash-table-initargs) + "Returns a hash table containing the keys and values of the property list +PLIST. Hash table is initialized using the HASH-TABLE-INITARGS." + (let ((table (apply #'make-hash-table hash-table-initargs))) + (do ((tail plist (cddr tail))) + ((not tail)) + (ensure-gethash (car tail) table (cadr tail))) + table)) diff --git a/third_party/lisp/alexandria/io.lisp b/third_party/lisp/alexandria/io.lisp new file mode 100644 index 000000000000..28bf5e6d82c7 --- /dev/null +++ b/third_party/lisp/alexandria/io.lisp @@ -0,0 +1,172 @@ +;; Copyright (c) 2002-2006, Edward Marco Baringer +;; All rights reserved. + +(in-package :alexandria) + +(defmacro with-open-file* ((stream filespec &key direction element-type + if-exists if-does-not-exist external-format) + &body body) + "Just like WITH-OPEN-FILE, but NIL values in the keyword arguments mean to use +the default value specified for OPEN." + (once-only (direction element-type if-exists if-does-not-exist external-format) + `(with-open-stream + (,stream (apply #'open ,filespec + (append + (when ,direction + (list :direction ,direction)) + (when ,element-type + (list :element-type ,element-type)) + (when ,if-exists + (list :if-exists ,if-exists)) + (when ,if-does-not-exist + (list :if-does-not-exist ,if-does-not-exist)) + (when ,external-format + (list :external-format ,external-format))))) + ,@body))) + +(defmacro with-input-from-file ((stream-name file-name &rest args + &key (direction nil direction-p) + &allow-other-keys) + &body body) + "Evaluate BODY with STREAM-NAME to an input stream on the file +FILE-NAME. ARGS is sent as is to the call to OPEN except EXTERNAL-FORMAT, +which is only sent to WITH-OPEN-FILE when it's not NIL." + (declare (ignore direction)) + (when direction-p + (error "Can't specify :DIRECTION for WITH-INPUT-FROM-FILE.")) + `(with-open-file* (,stream-name ,file-name :direction :input ,@args) + ,@body)) + +(defmacro with-output-to-file ((stream-name file-name &rest args + &key (direction nil direction-p) + &allow-other-keys) + &body body) + "Evaluate BODY with STREAM-NAME to an output stream on the file +FILE-NAME. ARGS is sent as is to the call to OPEN except EXTERNAL-FORMAT, +which is only sent to WITH-OPEN-FILE when it's not NIL." + (declare (ignore direction)) + (when direction-p + (error "Can't specify :DIRECTION for WITH-OUTPUT-TO-FILE.")) + `(with-open-file* (,stream-name ,file-name :direction :output ,@args) + ,@body)) + +(defun read-stream-content-into-string (stream &key (buffer-size 4096)) + "Return the \"content\" of STREAM as a fresh string." + (check-type buffer-size positive-integer) + (let ((*print-pretty* nil)) + (with-output-to-string (datum) + (let ((buffer (make-array buffer-size :element-type 'character))) + (loop + :for bytes-read = (read-sequence buffer stream) + :do (write-sequence buffer datum :start 0 :end bytes-read) + :while (= bytes-read buffer-size)))))) + +(defun read-file-into-string (pathname &key (buffer-size 4096) external-format) + "Return the contents of the file denoted by PATHNAME as a fresh string. + +The EXTERNAL-FORMAT parameter will be passed directly to WITH-OPEN-FILE +unless it's NIL, which means the system default." + (with-input-from-file + (file-stream pathname :external-format external-format) + (read-stream-content-into-string file-stream :buffer-size buffer-size))) + +(defun write-string-into-file (string pathname &key (if-exists :error) + if-does-not-exist + external-format) + "Write STRING to PATHNAME. + +The EXTERNAL-FORMAT parameter will be passed directly to WITH-OPEN-FILE +unless it's NIL, which means the system default." + (with-output-to-file (file-stream pathname :if-exists if-exists + :if-does-not-exist if-does-not-exist + :external-format external-format) + (write-sequence string file-stream))) + +(defun read-stream-content-into-byte-vector (stream &key ((%length length)) + (initial-size 4096)) + "Return \"content\" of STREAM as freshly allocated (unsigned-byte 8) vector." + (check-type length (or null non-negative-integer)) + (check-type initial-size positive-integer) + (do ((buffer (make-array (or length initial-size) + :element-type '(unsigned-byte 8))) + (offset 0) + (offset-wanted 0)) + ((or (/= offset-wanted offset) + (and length (>= offset length))) + (if (= offset (length buffer)) + buffer + (subseq buffer 0 offset))) + (unless (zerop offset) + (let ((new-buffer (make-array (* 2 (length buffer)) + :element-type '(unsigned-byte 8)))) + (replace new-buffer buffer) + (setf buffer new-buffer))) + (setf offset-wanted (length buffer) + offset (read-sequence buffer stream :start offset)))) + +(defun read-file-into-byte-vector (pathname) + "Read PATHNAME into a freshly allocated (unsigned-byte 8) vector." + (with-input-from-file (stream pathname :element-type '(unsigned-byte 8)) + (read-stream-content-into-byte-vector stream '%length (file-length stream)))) + +(defun write-byte-vector-into-file (bytes pathname &key (if-exists :error) + if-does-not-exist) + "Write BYTES to PATHNAME." + (check-type bytes (vector (unsigned-byte 8))) + (with-output-to-file (stream pathname :if-exists if-exists + :if-does-not-exist if-does-not-exist + :element-type '(unsigned-byte 8)) + (write-sequence bytes stream))) + +(defun copy-file (from to &key (if-to-exists :supersede) + (element-type '(unsigned-byte 8)) finish-output) + (with-input-from-file (input from :element-type element-type) + (with-output-to-file (output to :element-type element-type + :if-exists if-to-exists) + (copy-stream input output + :element-type element-type + :finish-output finish-output)))) + +(defun copy-stream (input output &key (element-type (stream-element-type input)) + (buffer-size 4096) + (buffer (make-array buffer-size :element-type element-type)) + (start 0) end + finish-output) + "Reads data from INPUT and writes it to OUTPUT. Both INPUT and OUTPUT must +be streams, they will be passed to READ-SEQUENCE and WRITE-SEQUENCE and must have +compatible element-types." + (check-type start non-negative-integer) + (check-type end (or null non-negative-integer)) + (check-type buffer-size positive-integer) + (when (and end + (< end start)) + (error "END is smaller than START in ~S" 'copy-stream)) + (let ((output-position 0) + (input-position 0)) + (unless (zerop start) + ;; FIXME add platform specific optimization to skip seekable streams + (loop while (< input-position start) + do (let ((n (read-sequence buffer input + :end (min (length buffer) + (- start input-position))))) + (when (zerop n) + (error "~@<Could not read enough bytes from the input to fulfill ~ + the :START ~S requirement in ~S.~:@>" 'copy-stream start)) + (incf input-position n)))) + (assert (= input-position start)) + (loop while (or (null end) (< input-position end)) + do (let ((n (read-sequence buffer input + :end (when end + (min (length buffer) + (- end input-position)))))) + (when (zerop n) + (if end + (error "~@<Could not read enough bytes from the input to fulfill ~ + the :END ~S requirement in ~S.~:@>" 'copy-stream end) + (return))) + (incf input-position n) + (write-sequence buffer output :end n) + (incf output-position n))) + (when finish-output + (finish-output output)) + output-position)) diff --git a/third_party/lisp/alexandria/lists.lisp b/third_party/lisp/alexandria/lists.lisp new file mode 100644 index 000000000000..51286071ebf2 --- /dev/null +++ b/third_party/lisp/alexandria/lists.lisp @@ -0,0 +1,367 @@ +(in-package :alexandria) + +(declaim (inline safe-endp)) +(defun safe-endp (x) + (declare (optimize safety)) + (endp x)) + +(defun alist-plist (alist) + "Returns a property list containing the same keys and values as the +association list ALIST in the same order." + (let (plist) + (dolist (pair alist) + (push (car pair) plist) + (push (cdr pair) plist)) + (nreverse plist))) + +(defun plist-alist (plist) + "Returns an association list containing the same keys and values as the +property list PLIST in the same order." + (let (alist) + (do ((tail plist (cddr tail))) + ((safe-endp tail) (nreverse alist)) + (push (cons (car tail) (cadr tail)) alist)))) + +(declaim (inline racons)) +(defun racons (key value ralist) + (acons value key ralist)) + +(macrolet + ((define-alist-get (name get-entry get-value-from-entry add doc) + `(progn + (declaim (inline ,name)) + (defun ,name (alist key &key (test 'eql)) + ,doc + (let ((entry (,get-entry key alist :test test))) + (values (,get-value-from-entry entry) entry))) + (define-setf-expander ,name (place key &key (test ''eql) + &environment env) + (multiple-value-bind + (temporary-variables initforms newvals setter getter) + (get-setf-expansion place env) + (when (cdr newvals) + (error "~A cannot store multiple values in one place" ',name)) + (with-unique-names (new-value key-val test-val alist entry) + (values + (append temporary-variables + (list alist + key-val + test-val + entry)) + (append initforms + (list getter + key + test + `(,',get-entry ,key-val ,alist :test ,test-val))) + `(,new-value) + `(cond + (,entry + (setf (,',get-value-from-entry ,entry) ,new-value)) + (t + (let ,newvals + (setf ,(first newvals) (,',add ,key ,new-value ,alist)) + ,setter + ,new-value))) + `(,',get-value-from-entry ,entry)))))))) + (define-alist-get assoc-value assoc cdr acons +"ASSOC-VALUE is an alist accessor very much like ASSOC, but it can +be used with SETF.") + (define-alist-get rassoc-value rassoc car racons +"RASSOC-VALUE is an alist accessor very much like RASSOC, but it can +be used with SETF.")) + +(defun malformed-plist (plist) + (error "Malformed plist: ~S" plist)) + +(defmacro doplist ((key val plist &optional values) &body body) + "Iterates over elements of PLIST. BODY can be preceded by +declarations, and is like a TAGBODY. RETURN may be used to terminate +the iteration early. If RETURN is not used, returns VALUES." + (multiple-value-bind (forms declarations) (parse-body body) + (with-gensyms (tail loop results) + `(block nil + (flet ((,results () + (let (,key ,val) + (declare (ignorable ,key ,val)) + (return ,values)))) + (let* ((,tail ,plist) + (,key (if ,tail + (pop ,tail) + (,results))) + (,val (if ,tail + (pop ,tail) + (malformed-plist ',plist)))) + (declare (ignorable ,key ,val)) + ,@declarations + (tagbody + ,loop + ,@forms + (setf ,key (if ,tail + (pop ,tail) + (,results)) + ,val (if ,tail + (pop ,tail) + (malformed-plist ',plist))) + (go ,loop)))))))) + +(define-modify-macro appendf (&rest lists) append + "Modify-macro for APPEND. Appends LISTS to the place designated by the first +argument.") + +(define-modify-macro nconcf (&rest lists) nconc + "Modify-macro for NCONC. Concatenates LISTS to place designated by the first +argument.") + +(define-modify-macro unionf (list &rest args) union + "Modify-macro for UNION. Saves the union of LIST and the contents of the +place designated by the first argument to the designated place.") + +(define-modify-macro nunionf (list &rest args) nunion + "Modify-macro for NUNION. Saves the union of LIST and the contents of the +place designated by the first argument to the designated place. May modify +either argument.") + +(define-modify-macro reversef () reverse + "Modify-macro for REVERSE. Copies and reverses the list stored in the given +place and saves back the result into the place.") + +(define-modify-macro nreversef () nreverse + "Modify-macro for NREVERSE. Reverses the list stored in the given place by +destructively modifying it and saves back the result into the place.") + +(defun circular-list (&rest elements) + "Creates a circular list of ELEMENTS." + (let ((cycle (copy-list elements))) + (nconc cycle cycle))) + +(defun circular-list-p (object) + "Returns true if OBJECT is a circular list, NIL otherwise." + (and (listp object) + (do ((fast object (cddr fast)) + (slow (cons (car object) (cdr object)) (cdr slow))) + (nil) + (unless (and (consp fast) (listp (cdr fast))) + (return nil)) + (when (eq fast slow) + (return t))))) + +(defun circular-tree-p (object) + "Returns true if OBJECT is a circular tree, NIL otherwise." + (labels ((circularp (object seen) + (and (consp object) + (do ((fast (cons (car object) (cdr object)) (cddr fast)) + (slow object (cdr slow))) + (nil) + (when (or (eq fast slow) (member slow seen)) + (return-from circular-tree-p t)) + (when (or (not (consp fast)) (not (consp (cdr slow)))) + (return + (do ((tail object (cdr tail))) + ((not (consp tail)) + nil) + (let ((elt (car tail))) + (circularp elt (cons object seen)))))))))) + (circularp object nil))) + +(defun proper-list-p (object) + "Returns true if OBJECT is a proper list." + (cond ((not object) + t) + ((consp object) + (do ((fast object (cddr fast)) + (slow (cons (car object) (cdr object)) (cdr slow))) + (nil) + (unless (and (listp fast) (consp (cdr fast))) + (return (and (listp fast) (not (cdr fast))))) + (when (eq fast slow) + (return nil)))) + (t + nil))) + +(deftype proper-list () + "Type designator for proper lists. Implemented as a SATISFIES type, hence +not recommended for performance intensive use. Main usefullness as a type +designator of the expected type in a TYPE-ERROR." + `(and list (satisfies proper-list-p))) + +(defun circular-list-error (list) + (error 'type-error + :datum list + :expected-type '(and list (not circular-list)))) + +(macrolet ((def (name lambda-list doc step declare ret1 ret2) + (assert (member 'list lambda-list)) + `(defun ,name ,lambda-list + ,doc + (do ((last list fast) + (fast list (cddr fast)) + (slow (cons (car list) (cdr list)) (cdr slow)) + ,@(when step (list step))) + (nil) + (declare (dynamic-extent slow) ,@(when declare (list declare)) + (ignorable last)) + (when (safe-endp fast) + (return ,ret1)) + (when (safe-endp (cdr fast)) + (return ,ret2)) + (when (eq fast slow) + (circular-list-error list)))))) + (def proper-list-length (list) + "Returns length of LIST, signalling an error if it is not a proper list." + (n 1 (+ n 2)) + ;; KLUDGE: Most implementations don't actually support lists with bignum + ;; elements -- and this is WAY faster on most implementations then declaring + ;; N to be an UNSIGNED-BYTE. + (fixnum n) + (1- n) + n) + + (def lastcar (list) + "Returns the last element of LIST. Signals a type-error if LIST is not a +proper list." + nil + nil + (cadr last) + (car fast)) + + (def (setf lastcar) (object list) + "Sets the last element of LIST. Signals a type-error if LIST is not a proper +list." + nil + nil + (setf (cadr last) object) + (setf (car fast) object))) + +(defun make-circular-list (length &key initial-element) + "Creates a circular list of LENGTH with the given INITIAL-ELEMENT." + (let ((cycle (make-list length :initial-element initial-element))) + (nconc cycle cycle))) + +(deftype circular-list () + "Type designator for circular lists. Implemented as a SATISFIES type, so not +recommended for performance intensive use. Main usefullness as the +expected-type designator of a TYPE-ERROR." + `(satisfies circular-list-p)) + +(defun ensure-car (thing) + "If THING is a CONS, its CAR is returned. Otherwise THING is returned." + (if (consp thing) + (car thing) + thing)) + +(defun ensure-cons (cons) + "If CONS is a cons, it is returned. Otherwise returns a fresh cons with CONS + in the car, and NIL in the cdr." + (if (consp cons) + cons + (cons cons nil))) + +(defun ensure-list (list) + "If LIST is a list, it is returned. Otherwise returns the list designated by LIST." + (if (listp list) + list + (list list))) + +(defun remove-from-plist (plist &rest keys) + "Returns a propery-list with same keys and values as PLIST, except that keys +in the list designated by KEYS and values corresponding to them are removed. +The returned property-list may share structure with the PLIST, but PLIST is +not destructively modified. Keys are compared using EQ." + (declare (optimize (speed 3))) + ;; FIXME: possible optimization: (remove-from-plist '(:x 0 :a 1 :b 2) :a) + ;; could return the tail without consing up a new list. + (loop for (key . rest) on plist by #'cddr + do (assert rest () "Expected a proper plist, got ~S" plist) + unless (member key keys :test #'eq) + collect key and collect (first rest))) + +(defun delete-from-plist (plist &rest keys) + "Just like REMOVE-FROM-PLIST, but this version may destructively modify the +provided PLIST." + (declare (optimize speed)) + (loop with head = plist + with tail = nil ; a nil tail means an empty result so far + for (key . rest) on plist by #'cddr + do (assert rest () "Expected a proper plist, got ~S" plist) + (if (member key keys :test #'eq) + ;; skip over this pair + (let ((next (cdr rest))) + (if tail + (setf (cdr tail) next) + (setf head next))) + ;; keep this pair + (setf tail rest)) + finally (return head))) + +(define-modify-macro remove-from-plistf (&rest keys) remove-from-plist + "Modify macro for REMOVE-FROM-PLIST.") +(define-modify-macro delete-from-plistf (&rest keys) delete-from-plist + "Modify macro for DELETE-FROM-PLIST.") + +(declaim (inline sans)) +(defun sans (plist &rest keys) + "Alias of REMOVE-FROM-PLIST for backward compatibility." + (apply #'remove-from-plist plist keys)) + +(defun mappend (function &rest lists) + "Applies FUNCTION to respective element(s) of each LIST, appending all the +all the result list to a single list. FUNCTION must return a list." + (loop for results in (apply #'mapcar function lists) + append results)) + +(defun setp (object &key (test #'eql) (key #'identity)) + "Returns true if OBJECT is a list that denotes a set, NIL otherwise. A list +denotes a set if each element of the list is unique under KEY and TEST." + (and (listp object) + (let (seen) + (dolist (elt object t) + (let ((key (funcall key elt))) + (if (member key seen :test test) + (return nil) + (push key seen))))))) + +(defun set-equal (list1 list2 &key (test #'eql) (key nil keyp)) + "Returns true if every element of LIST1 matches some element of LIST2 and +every element of LIST2 matches some element of LIST1. Otherwise returns false." + (let ((keylist1 (if keyp (mapcar key list1) list1)) + (keylist2 (if keyp (mapcar key list2) list2))) + (and (dolist (elt keylist1 t) + (or (member elt keylist2 :test test) + (return nil))) + (dolist (elt keylist2 t) + (or (member elt keylist1 :test test) + (return nil)))))) + +(defun map-product (function list &rest more-lists) + "Returns a list containing the results of calling FUNCTION with one argument +from LIST, and one from each of MORE-LISTS for each combination of arguments. +In other words, returns the product of LIST and MORE-LISTS using FUNCTION. + +Example: + + (map-product 'list '(1 2) '(3 4) '(5 6)) + => ((1 3 5) (1 3 6) (1 4 5) (1 4 6) + (2 3 5) (2 3 6) (2 4 5) (2 4 6)) +" + (labels ((%map-product (f lists) + (let ((more (cdr lists)) + (one (car lists))) + (if (not more) + (mapcar f one) + (mappend (lambda (x) + (%map-product (curry f x) more)) + one))))) + (%map-product (ensure-function function) (cons list more-lists)))) + +(defun flatten (tree) + "Traverses the tree in order, collecting non-null leaves into a list." + (let (list) + (labels ((traverse (subtree) + (when subtree + (if (consp subtree) + (progn + (traverse (car subtree)) + (traverse (cdr subtree))) + (push subtree list))))) + (traverse tree)) + (nreverse list))) diff --git a/third_party/lisp/alexandria/macros.lisp b/third_party/lisp/alexandria/macros.lisp new file mode 100644 index 000000000000..4364ad63b82a --- /dev/null +++ b/third_party/lisp/alexandria/macros.lisp @@ -0,0 +1,370 @@ +(in-package :alexandria) + +(defmacro with-gensyms (names &body forms) + "Binds a set of variables to gensyms and evaluates the implicit progn FORMS. + +Each element within NAMES is either a symbol SYMBOL or a pair (SYMBOL +STRING-DESIGNATOR). Bare symbols are equivalent to the pair (SYMBOL SYMBOL). + +Each pair (SYMBOL STRING-DESIGNATOR) specifies that the variable named by SYMBOL +should be bound to a symbol constructed using GENSYM with the string designated +by STRING-DESIGNATOR being its first argument." + `(let ,(mapcar (lambda (name) + (multiple-value-bind (symbol string) + (etypecase name + (symbol + (values name (symbol-name name))) + ((cons symbol (cons string-designator null)) + (values (first name) (string (second name))))) + `(,symbol (gensym ,string)))) + names) + ,@forms)) + +(defmacro with-unique-names (names &body forms) + "Alias for WITH-GENSYMS." + `(with-gensyms ,names ,@forms)) + +(defmacro once-only (specs &body forms) + "Constructs code whose primary goal is to help automate the handling of +multiple evaluation within macros. Multiple evaluation is handled by introducing +intermediate variables, in order to reuse the result of an expression. + +The returned value is a list of the form + + (let ((<gensym-1> <expr-1>) + ... + (<gensym-n> <expr-n>)) + <res>) + +where GENSYM-1, ..., GENSYM-N are the intermediate variables introduced in order +to evaluate EXPR-1, ..., EXPR-N once, only. RES is code that is the result of +evaluating the implicit progn FORMS within a special context determined by +SPECS. RES should make use of (reference) the intermediate variables. + +Each element within SPECS is either a symbol SYMBOL or a pair (SYMBOL INITFORM). +Bare symbols are equivalent to the pair (SYMBOL SYMBOL). + +Each pair (SYMBOL INITFORM) specifies a single intermediate variable: + +- INITFORM is an expression evaluated to produce EXPR-i + +- SYMBOL is the name of the variable that will be bound around FORMS to the + corresponding gensym GENSYM-i, in order for FORMS to generate RES that + references the intermediate variable + +The evaluation of INITFORMs and binding of SYMBOLs resembles LET. INITFORMs of +all the pairs are evaluated before binding SYMBOLs and evaluating FORMS. + +Example: + + The following expression + + (let ((x '(incf y))) + (once-only (x) + `(cons ,x ,x))) + + ;;; => + ;;; (let ((#1=#:X123 (incf y))) + ;;; (cons #1# #1#)) + + could be used within a macro to avoid multiple evaluation like so + + (defmacro cons1 (x) + (once-only (x) + `(cons ,x ,x))) + + (let ((y 0)) + (cons1 (incf y))) + + ;;; => (1 . 1) + +Example: + + The following expression demonstrates the usage of the INITFORM field + + (let ((expr '(incf y))) + (once-only ((var `(1+ ,expr))) + `(list ',expr ,var ,var))) + + ;;; => + ;;; (let ((#1=#:VAR123 (1+ (incf y)))) + ;;; (list '(incf y) #1# #1)) + + which could be used like so + + (defmacro print-succ-twice (expr) + (once-only ((var `(1+ ,expr))) + `(format t \"Expr: ~s, Once: ~s, Twice: ~s~%\" ',expr ,var ,var))) + + (let ((y 10)) + (print-succ-twice (incf y))) + + ;;; >> + ;;; Expr: (INCF Y), Once: 12, Twice: 12" + (let ((gensyms (make-gensym-list (length specs) "ONCE-ONLY")) + (names-and-forms (mapcar (lambda (spec) + (etypecase spec + (list + (destructuring-bind (name form) spec + (cons name form))) + (symbol + (cons spec spec)))) + specs))) + ;; bind in user-macro + `(let ,(mapcar (lambda (g n) (list g `(gensym ,(string (car n))))) + gensyms names-and-forms) + ;; bind in final expansion + `(let (,,@(mapcar (lambda (g n) + ``(,,g ,,(cdr n))) + gensyms names-and-forms)) + ;; bind in user-macro + ,(let ,(mapcar (lambda (n g) (list (car n) g)) + names-and-forms gensyms) + ,@forms))))) + +(defun parse-body (body &key documentation whole) + "Parses BODY into (values remaining-forms declarations doc-string). +Documentation strings are recognized only if DOCUMENTATION is true. +Syntax errors in body are signalled and WHOLE is used in the signal +arguments when given." + (let ((doc nil) + (decls nil) + (current nil)) + (tagbody + :declarations + (setf current (car body)) + (when (and documentation (stringp current) (cdr body)) + (if doc + (error "Too many documentation strings in ~S." (or whole body)) + (setf doc (pop body))) + (go :declarations)) + (when (and (listp current) (eql (first current) 'declare)) + (push (pop body) decls) + (go :declarations))) + (values body (nreverse decls) doc))) + +(defun parse-ordinary-lambda-list (lambda-list &key (normalize t) + allow-specializers + (normalize-optional normalize) + (normalize-keyword normalize) + (normalize-auxilary normalize)) + "Parses an ordinary lambda-list, returning as multiple values: + +1. Required parameters. + +2. Optional parameter specifications, normalized into form: + + (name init suppliedp) + +3. Name of the rest parameter, or NIL. + +4. Keyword parameter specifications, normalized into form: + + ((keyword-name name) init suppliedp) + +5. Boolean indicating &ALLOW-OTHER-KEYS presence. + +6. &AUX parameter specifications, normalized into form + + (name init). + +7. Existence of &KEY in the lambda-list. + +Signals a PROGRAM-ERROR is the lambda-list is malformed." + (let ((state :required) + (allow-other-keys nil) + (auxp nil) + (required nil) + (optional nil) + (rest nil) + (keys nil) + (keyp nil) + (aux nil)) + (labels ((fail (elt) + (simple-program-error "Misplaced ~S in ordinary lambda-list:~% ~S" + elt lambda-list)) + (check-variable (elt what &optional (allow-specializers allow-specializers)) + (unless (and (or (symbolp elt) + (and allow-specializers + (consp elt) (= 2 (length elt)) (symbolp (first elt)))) + (not (constantp elt))) + (simple-program-error "Invalid ~A ~S in ordinary lambda-list:~% ~S" + what elt lambda-list))) + (check-spec (spec what) + (destructuring-bind (init suppliedp) spec + (declare (ignore init)) + (check-variable suppliedp what nil)))) + (dolist (elt lambda-list) + (case elt + (&optional + (if (eq state :required) + (setf state elt) + (fail elt))) + (&rest + (if (member state '(:required &optional)) + (setf state elt) + (fail elt))) + (&key + (if (member state '(:required &optional :after-rest)) + (setf state elt) + (fail elt)) + (setf keyp t)) + (&allow-other-keys + (if (eq state '&key) + (setf allow-other-keys t + state elt) + (fail elt))) + (&aux + (cond ((eq state '&rest) + (fail elt)) + (auxp + (simple-program-error "Multiple ~S in ordinary lambda-list:~% ~S" + elt lambda-list)) + (t + (setf auxp t + state elt)) + )) + (otherwise + (when (member elt '#.(set-difference lambda-list-keywords + '(&optional &rest &key &allow-other-keys &aux))) + (simple-program-error + "Bad lambda-list keyword ~S in ordinary lambda-list:~% ~S" + elt lambda-list)) + (case state + (:required + (check-variable elt "required parameter") + (push elt required)) + (&optional + (cond ((consp elt) + (destructuring-bind (name &rest tail) elt + (check-variable name "optional parameter") + (cond ((cdr tail) + (check-spec tail "optional-supplied-p parameter")) + ((and normalize-optional tail) + (setf elt (append elt '(nil)))) + (normalize-optional + (setf elt (append elt '(nil nil))))))) + (t + (check-variable elt "optional parameter") + (when normalize-optional + (setf elt (cons elt '(nil nil)))))) + (push (ensure-list elt) optional)) + (&rest + (check-variable elt "rest parameter") + (setf rest elt + state :after-rest)) + (&key + (cond ((consp elt) + (destructuring-bind (var-or-kv &rest tail) elt + (cond ((consp var-or-kv) + (destructuring-bind (keyword var) var-or-kv + (unless (symbolp keyword) + (simple-program-error "Invalid keyword name ~S in ordinary ~ + lambda-list:~% ~S" + keyword lambda-list)) + (check-variable var "keyword parameter"))) + (t + (check-variable var-or-kv "keyword parameter") + (when normalize-keyword + (setf var-or-kv (list (make-keyword var-or-kv) var-or-kv))))) + (cond ((cdr tail) + (check-spec tail "keyword-supplied-p parameter")) + ((and normalize-keyword tail) + (setf tail (append tail '(nil)))) + (normalize-keyword + (setf tail '(nil nil)))) + (setf elt (cons var-or-kv tail)))) + (t + (check-variable elt "keyword parameter") + (setf elt (if normalize-keyword + (list (list (make-keyword elt) elt) nil nil) + elt)))) + (push elt keys)) + (&aux + (if (consp elt) + (destructuring-bind (var &optional init) elt + (declare (ignore init)) + (check-variable var "&aux parameter")) + (progn + (check-variable elt "&aux parameter") + (setf elt (list* elt (when normalize-auxilary + '(nil)))))) + (push elt aux)) + (t + (simple-program-error "Invalid ordinary lambda-list:~% ~S" lambda-list))))))) + (values (nreverse required) (nreverse optional) rest (nreverse keys) + allow-other-keys (nreverse aux) keyp))) + +;;;; DESTRUCTURING-*CASE + +(defun expand-destructuring-case (key clauses case) + (once-only (key) + `(if (typep ,key 'cons) + (,case (car ,key) + ,@(mapcar (lambda (clause) + (destructuring-bind ((keys . lambda-list) &body body) clause + `(,keys + (destructuring-bind ,lambda-list (cdr ,key) + ,@body)))) + clauses)) + (error "Invalid key to DESTRUCTURING-~S: ~S" ',case ,key)))) + +(defmacro destructuring-case (keyform &body clauses) + "DESTRUCTURING-CASE, -CCASE, and -ECASE are a combination of CASE and DESTRUCTURING-BIND. +KEYFORM must evaluate to a CONS. + +Clauses are of the form: + + ((CASE-KEYS . DESTRUCTURING-LAMBDA-LIST) FORM*) + +The clause whose CASE-KEYS matches CAR of KEY, as if by CASE, CCASE, or ECASE, +is selected, and FORMs are then executed with CDR of KEY is destructured and +bound by the DESTRUCTURING-LAMBDA-LIST. + +Example: + + (defun dcase (x) + (destructuring-case x + ((:foo a b) + (format nil \"foo: ~S, ~S\" a b)) + ((:bar &key a b) + (format nil \"bar: ~S, ~S\" a b)) + (((:alt1 :alt2) a) + (format nil \"alt: ~S\" a)) + ((t &rest rest) + (format nil \"unknown: ~S\" rest)))) + + (dcase (list :foo 1 2)) ; => \"foo: 1, 2\" + (dcase (list :bar :a 1 :b 2)) ; => \"bar: 1, 2\" + (dcase (list :alt1 1)) ; => \"alt: 1\" + (dcase (list :alt2 2)) ; => \"alt: 2\" + (dcase (list :quux 1 2 3)) ; => \"unknown: 1, 2, 3\" + + (defun decase (x) + (destructuring-case x + ((:foo a b) + (format nil \"foo: ~S, ~S\" a b)) + ((:bar &key a b) + (format nil \"bar: ~S, ~S\" a b)) + (((:alt1 :alt2) a) + (format nil \"alt: ~S\" a)))) + + (decase (list :foo 1 2)) ; => \"foo: 1, 2\" + (decase (list :bar :a 1 :b 2)) ; => \"bar: 1, 2\" + (decase (list :alt1 1)) ; => \"alt: 1\" + (decase (list :alt2 2)) ; => \"alt: 2\" + (decase (list :quux 1 2 3)) ; =| error +" + (expand-destructuring-case keyform clauses 'case)) + +(defmacro destructuring-ccase (keyform &body clauses) + (expand-destructuring-case keyform clauses 'ccase)) + +(defmacro destructuring-ecase (keyform &body clauses) + (expand-destructuring-case keyform clauses 'ecase)) + +(dolist (name '(destructuring-ccase destructuring-ecase)) + (setf (documentation name 'function) (documentation 'destructuring-case 'function))) + + + diff --git a/third_party/lisp/alexandria/numbers.lisp b/third_party/lisp/alexandria/numbers.lisp new file mode 100644 index 000000000000..1c06f71d508f --- /dev/null +++ b/third_party/lisp/alexandria/numbers.lisp @@ -0,0 +1,295 @@ +(in-package :alexandria) + +(declaim (inline clamp)) +(defun clamp (number min max) + "Clamps the NUMBER into [min, max] range. Returns MIN if NUMBER is lesser then +MIN and MAX if NUMBER is greater then MAX, otherwise returns NUMBER." + (if (< number min) + min + (if (> number max) + max + number))) + +(defun gaussian-random (&optional min max) + "Returns two gaussian random double floats as the primary and secondary value, +optionally constrained by MIN and MAX. Gaussian random numbers form a standard +normal distribution around 0.0d0. + +Sufficiently positive MIN or negative MAX will cause the algorithm used to +take a very long time. If MIN is positive it should be close to zero, and +similarly if MAX is negative it should be close to zero." + (macrolet + ((valid (x) + `(<= (or min ,x) ,x (or max ,x)) )) + (labels + ((gauss () + (loop + for x1 = (- (random 2.0d0) 1.0d0) + for x2 = (- (random 2.0d0) 1.0d0) + for w = (+ (expt x1 2) (expt x2 2)) + when (< w 1.0d0) + do (let ((v (sqrt (/ (* -2.0d0 (log w)) w)))) + (return (values (* x1 v) (* x2 v)))))) + (guard (x) + (unless (valid x) + (tagbody + :retry + (multiple-value-bind (x1 x2) (gauss) + (when (valid x1) + (setf x x1) + (go :done)) + (when (valid x2) + (setf x x2) + (go :done)) + (go :retry)) + :done)) + x)) + (multiple-value-bind + (g1 g2) (gauss) + (values (guard g1) (guard g2)))))) + +(declaim (inline iota)) +(defun iota (n &key (start 0) (step 1)) + "Return a list of n numbers, starting from START (with numeric contagion +from STEP applied), each consequtive number being the sum of the previous one +and STEP. START defaults to 0 and STEP to 1. + +Examples: + + (iota 4) => (0 1 2 3) + (iota 3 :start 1 :step 1.0) => (1.0 2.0 3.0) + (iota 3 :start -1 :step -1/2) => (-1 -3/2 -2) +" + (declare (type (integer 0) n) (number start step)) + (loop ;; KLUDGE: get numeric contagion right for the first element too + for i = (+ (- (+ start step) step)) then (+ i step) + repeat n + collect i)) + +(declaim (inline map-iota)) +(defun map-iota (function n &key (start 0) (step 1)) + "Calls FUNCTION with N numbers, starting from START (with numeric contagion +from STEP applied), each consequtive number being the sum of the previous one +and STEP. START defaults to 0 and STEP to 1. Returns N. + +Examples: + + (map-iota #'print 3 :start 1 :step 1.0) => 3 + ;;; 1.0 + ;;; 2.0 + ;;; 3.0 +" + (declare (type (integer 0) n) (number start step)) + (loop ;; KLUDGE: get numeric contagion right for the first element too + for i = (+ start (- step step)) then (+ i step) + repeat n + do (funcall function i)) + n) + +(declaim (inline lerp)) +(defun lerp (v a b) + "Returns the result of linear interpolation between A and B, using the +interpolation coefficient V." + ;; The correct version is numerically stable, at the expense of an + ;; extra multiply. See (lerp 0.1 4 25) with (+ a (* v (- b a))). The + ;; unstable version can often be converted to a fast instruction on + ;; a lot of machines, though this is machine/implementation + ;; specific. As alexandria is more about correct code, than + ;; efficiency, and we're only talking about a single extra multiply, + ;; many would prefer the stable version + (+ (* (- 1.0 v) a) (* v b))) + +(declaim (inline mean)) +(defun mean (sample) + "Returns the mean of SAMPLE. SAMPLE must be a sequence of numbers." + (/ (reduce #'+ sample) (length sample))) + +(defun median (sample) + "Returns median of SAMPLE. SAMPLE must be a sequence of real numbers." + ;; Implements and uses the quick-select algorithm to find the median + ;; https://en.wikipedia.org/wiki/Quickselect + + (labels ((randint-in-range (start-int end-int) + "Returns a random integer in the specified range, inclusive" + (+ start-int (random (1+ (- end-int start-int))))) + (partition (vec start-i end-i) + "Implements the partition function, which performs a partial + sort of vec around the (randomly) chosen pivot. + Returns the index where the pivot element would be located + in a correctly-sorted array" + (if (= start-i end-i) + start-i + (let ((pivot-i (randint-in-range start-i end-i))) + (rotatef (aref vec start-i) (aref vec pivot-i)) + (let ((swap-i end-i)) + (loop for i from swap-i downto (1+ start-i) do + (when (>= (aref vec i) (aref vec start-i)) + (rotatef (aref vec i) (aref vec swap-i)) + (decf swap-i))) + (rotatef (aref vec swap-i) (aref vec start-i)) + swap-i))))) + + (let* ((vector (copy-sequence 'vector sample)) + (len (length vector)) + (mid-i (ash len -1)) + (i 0) + (j (1- len))) + + (loop for correct-pos = (partition vector i j) + while (/= correct-pos mid-i) do + (if (< correct-pos mid-i) + (setf i (1+ correct-pos)) + (setf j (1- correct-pos)))) + + (if (oddp len) + (aref vector mid-i) + (* 1/2 + (+ (aref vector mid-i) + (reduce #'max (make-array + mid-i + :displaced-to vector)))))))) + +(declaim (inline variance)) +(defun variance (sample &key (biased t)) + "Variance of SAMPLE. Returns the biased variance if BIASED is true (the default), +and the unbiased estimator of variance if BIASED is false. SAMPLE must be a +sequence of numbers." + (let ((mean (mean sample))) + (/ (reduce (lambda (a b) + (+ a (expt (- b mean) 2))) + sample + :initial-value 0) + (- (length sample) (if biased 0 1))))) + +(declaim (inline standard-deviation)) +(defun standard-deviation (sample &key (biased t)) + "Standard deviation of SAMPLE. Returns the biased standard deviation if +BIASED is true (the default), and the square root of the unbiased estimator +for variance if BIASED is false (which is not the same as the unbiased +estimator for standard deviation). SAMPLE must be a sequence of numbers." + (sqrt (variance sample :biased biased))) + +(define-modify-macro maxf (&rest numbers) max + "Modify-macro for MAX. Sets place designated by the first argument to the +maximum of its original value and NUMBERS.") + +(define-modify-macro minf (&rest numbers) min + "Modify-macro for MIN. Sets place designated by the first argument to the +minimum of its original value and NUMBERS.") + +;;;; Factorial + +;;; KLUDGE: This is really dependant on the numbers in question: for +;;; small numbers this is larger, and vice versa. Ideally instead of a +;;; constant we would have RANGE-FAST-TO-MULTIPLY-DIRECTLY-P. +(defconstant +factorial-bisection-range-limit+ 8) + +;;; KLUDGE: This is really platform dependant: ideally we would use +;;; (load-time-value (find-good-direct-multiplication-limit)) instead. +(defconstant +factorial-direct-multiplication-limit+ 13) + +(defun %multiply-range (i j) + ;; We use a a bit of cleverness here: + ;; + ;; 1. For large factorials we bisect in order to avoid expensive bignum + ;; multiplications: 1 x 2 x 3 x ... runs into bignums pretty soon, + ;; and once it does that all further multiplications will be with bignums. + ;; + ;; By instead doing the multiplication in a tree like + ;; ((1 x 2) x (3 x 4)) x ((5 x 6) x (7 x 8)) + ;; we manage to get less bignums. + ;; + ;; 2. Division isn't exactly free either, however, so we don't bisect + ;; all the way down, but multiply ranges of integers close to each + ;; other directly. + ;; + ;; For even better results it should be possible to use prime + ;; factorization magic, but Nikodemus ran out of steam. + ;; + ;; KLUDGE: We support factorials of bignums, but it seems quite + ;; unlikely anyone would ever be able to use them on a modern lisp, + ;; since the resulting numbers are unlikely to fit in memory... but + ;; it would be extremely unelegant to define FACTORIAL only on + ;; fixnums, _and_ on lisps with 16 bit fixnums this can actually be + ;; needed. + (labels ((bisect (j k) + (declare (type (integer 1 #.most-positive-fixnum) j k)) + (if (< (- k j) +factorial-bisection-range-limit+) + (multiply-range j k) + (let ((middle (+ j (truncate (- k j) 2)))) + (* (bisect j middle) + (bisect (+ middle 1) k))))) + (bisect-big (j k) + (declare (type (integer 1) j k)) + (if (= j k) + j + (let ((middle (+ j (truncate (- k j) 2)))) + (* (if (<= middle most-positive-fixnum) + (bisect j middle) + (bisect-big j middle)) + (bisect-big (+ middle 1) k))))) + (multiply-range (j k) + (declare (type (integer 1 #.most-positive-fixnum) j k)) + (do ((f k (* f m)) + (m (1- k) (1- m))) + ((< m j) f) + (declare (type (integer 0 (#.most-positive-fixnum)) m) + (type unsigned-byte f))))) + (if (and (typep i 'fixnum) (typep j 'fixnum)) + (bisect i j) + (bisect-big i j)))) + +(declaim (inline factorial)) +(defun %factorial (n) + (if (< n 2) + 1 + (%multiply-range 1 n))) + +(defun factorial (n) + "Factorial of non-negative integer N." + (check-type n (integer 0)) + (%factorial n)) + +;;;; Combinatorics + +(defun binomial-coefficient (n k) + "Binomial coefficient of N and K, also expressed as N choose K. This is the +number of K element combinations given N choises. N must be equal to or +greater then K." + (check-type n (integer 0)) + (check-type k (integer 0)) + (assert (>= n k)) + (if (or (zerop k) (= n k)) + 1 + (let ((n-k (- n k))) + ;; Swaps K and N-K if K < N-K because the algorithm + ;; below is faster for bigger K and smaller N-K + (when (< k n-k) + (rotatef k n-k)) + (if (= 1 n-k) + n + ;; General case, avoid computing the 1x...xK twice: + ;; + ;; N! 1x...xN (K+1)x...xN + ;; -------- = ---------------- = ------------, N>1 + ;; K!(N-K)! 1x...xK x (N-K)! (N-K)! + (/ (%multiply-range (+ k 1) n) + (%factorial n-k)))))) + +(defun subfactorial (n) + "Subfactorial of the non-negative integer N." + (check-type n (integer 0)) + (if (zerop n) + 1 + (do ((x 1 (1+ x)) + (a 0 (* x (+ a b))) + (b 1 a)) + ((= n x) a)))) + +(defun count-permutations (n &optional (k n)) + "Number of K element permutations for a sequence of N objects. +K defaults to N" + (check-type n (integer 0)) + (check-type k (integer 0)) + (assert (>= n k)) + (%multiply-range (1+ (- n k)) n)) diff --git a/third_party/lisp/alexandria/package.lisp b/third_party/lisp/alexandria/package.lisp new file mode 100644 index 000000000000..f9d2014cd7b5 --- /dev/null +++ b/third_party/lisp/alexandria/package.lisp @@ -0,0 +1,243 @@ +(defpackage :alexandria.1.0.0 + (:nicknames :alexandria) + (:use :cl) + #+sb-package-locks + (:lock t) + (:export + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; BLESSED + ;; + ;; Binding constructs + #:if-let + #:when-let + #:when-let* + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; REVIEW IN PROGRESS + ;; + ;; Control flow + ;; + ;; -- no clear consensus yet -- + #:cswitch + #:eswitch + #:switch + ;; -- problem free? -- + #:multiple-value-prog2 + #:nth-value-or + #:whichever + #:xor + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + ;; REVIEW PENDING + ;; + ;; Definitions + #:define-constant + ;; Hash tables + #:alist-hash-table + #:copy-hash-table + #:ensure-gethash + #:hash-table-alist + #:hash-table-keys + #:hash-table-plist + #:hash-table-values + #:maphash-keys + #:maphash-values + #:plist-hash-table + ;; Functions + #:compose + #:conjoin + #:curry + #:disjoin + #:ensure-function + #:ensure-functionf + #:multiple-value-compose + #:named-lambda + #:rcurry + ;; Lists + #:alist-plist + #:appendf + #:nconcf + #:reversef + #:nreversef + #:circular-list + #:circular-list-p + #:circular-tree-p + #:doplist + #:ensure-car + #:ensure-cons + #:ensure-list + #:flatten + #:lastcar + #:make-circular-list + #:map-product + #:mappend + #:nunionf + #:plist-alist + #:proper-list + #:proper-list-length + #:proper-list-p + #:remove-from-plist + #:remove-from-plistf + #:delete-from-plist + #:delete-from-plistf + #:set-equal + #:setp + #:unionf + ;; Numbers + #:binomial-coefficient + #:clamp + #:count-permutations + #:factorial + #:gaussian-random + #:iota + #:lerp + #:map-iota + #:maxf + #:mean + #:median + #:minf + #:standard-deviation + #:subfactorial + #:variance + ;; Arrays + #:array-index + #:array-length + #:copy-array + ;; Sequences + #:copy-sequence + #:deletef + #:emptyp + #:ends-with + #:ends-with-subseq + #:extremum + #:first-elt + #:last-elt + #:length= + #:map-combinations + #:map-derangements + #:map-permutations + #:proper-sequence + #:random-elt + #:removef + #:rotate + #:sequence-of-length-p + #:shuffle + #:starts-with + #:starts-with-subseq + ;; Macros + #:once-only + #:parse-body + #:parse-ordinary-lambda-list + #:with-gensyms + #:with-unique-names + ;; Symbols + #:ensure-symbol + #:format-symbol + #:make-gensym + #:make-gensym-list + #:make-keyword + ;; Strings + #:string-designator + ;; Types + #:negative-double-float + #:negative-fixnum-p + #:negative-float + #:negative-float-p + #:negative-long-float + #:negative-long-float-p + #:negative-rational + #:negative-rational-p + #:negative-real + #:negative-single-float-p + #:non-negative-double-float + #:non-negative-double-float-p + #:non-negative-fixnum + #:non-negative-fixnum-p + #:non-negative-float + #:non-negative-float-p + #:non-negative-integer-p + #:non-negative-long-float + #:non-negative-rational + #:non-negative-real-p + #:non-negative-short-float-p + #:non-negative-single-float + #:non-negative-single-float-p + #:non-positive-double-float + #:non-positive-double-float-p + #:non-positive-fixnum + #:non-positive-fixnum-p + #:non-positive-float + #:non-positive-float-p + #:non-positive-integer + #:non-positive-rational + #:non-positive-real + #:non-positive-real-p + #:non-positive-short-float + #:non-positive-short-float-p + #:non-positive-single-float-p + #:positive-double-float + #:positive-double-float-p + #:positive-fixnum + #:positive-fixnum-p + #:positive-float + #:positive-float-p + #:positive-integer + #:positive-rational + #:positive-real + #:positive-real-p + #:positive-short-float + #:positive-short-float-p + #:positive-single-float + #:positive-single-float-p + #:coercef + #:negative-double-float-p + #:negative-fixnum + #:negative-integer + #:negative-integer-p + #:negative-real-p + #:negative-short-float + #:negative-short-float-p + #:negative-single-float + #:non-negative-integer + #:non-negative-long-float-p + #:non-negative-rational-p + #:non-negative-real + #:non-negative-short-float + #:non-positive-integer-p + #:non-positive-long-float + #:non-positive-long-float-p + #:non-positive-rational-p + #:non-positive-single-float + #:of-type + #:positive-integer-p + #:positive-long-float + #:positive-long-float-p + #:positive-rational-p + #:type= + ;; Conditions + #:required-argument + #:ignore-some-conditions + #:simple-style-warning + #:simple-reader-error + #:simple-parse-error + #:simple-program-error + #:unwind-protect-case + ;; Features + #:featurep + ;; io + #:with-input-from-file + #:with-output-to-file + #:read-stream-content-into-string + #:read-file-into-string + #:write-string-into-file + #:read-stream-content-into-byte-vector + #:read-file-into-byte-vector + #:write-byte-vector-into-file + #:copy-stream + #:copy-file + ;; new additions collected at the end (subject to removal or further changes) + #:symbolicate + #:assoc-value + #:rassoc-value + #:destructuring-case + #:destructuring-ccase + #:destructuring-ecase + )) diff --git a/third_party/lisp/alexandria/sequences.lisp b/third_party/lisp/alexandria/sequences.lisp new file mode 100644 index 000000000000..21464f537610 --- /dev/null +++ b/third_party/lisp/alexandria/sequences.lisp @@ -0,0 +1,555 @@ +(in-package :alexandria) + +;; Make these inlinable by declaiming them INLINE here and some of them +;; NOTINLINE at the end of the file. Exclude functions that have a compiler +;; macro, because NOTINLINE is required to prevent compiler-macro expansion. +(declaim (inline copy-sequence sequence-of-length-p)) + +(defun sequence-of-length-p (sequence length) + "Return true if SEQUENCE is a sequence of length LENGTH. Signals an error if +SEQUENCE is not a sequence. Returns FALSE for circular lists." + (declare (type array-index length) + #-lispworks (inline length) + (optimize speed)) + (etypecase sequence + (null + (zerop length)) + (cons + (let ((n (1- length))) + (unless (minusp n) + (let ((tail (nthcdr n sequence))) + (and tail + (null (cdr tail))))))) + (vector + (= length (length sequence))) + (sequence + (= length (length sequence))))) + +(defun rotate-tail-to-head (sequence n) + (declare (type (integer 1) n)) + (if (listp sequence) + (let ((m (mod n (proper-list-length sequence)))) + (if (null (cdr sequence)) + sequence + (let* ((tail (last sequence (+ m 1))) + (last (cdr tail))) + (setf (cdr tail) nil) + (nconc last sequence)))) + (let* ((len (length sequence)) + (m (mod n len)) + (tail (subseq sequence (- len m)))) + (replace sequence sequence :start1 m :start2 0) + (replace sequence tail) + sequence))) + +(defun rotate-head-to-tail (sequence n) + (declare (type (integer 1) n)) + (if (listp sequence) + (let ((m (mod (1- n) (proper-list-length sequence)))) + (if (null (cdr sequence)) + sequence + (let* ((headtail (nthcdr m sequence)) + (tail (cdr headtail))) + (setf (cdr headtail) nil) + (nconc tail sequence)))) + (let* ((len (length sequence)) + (m (mod n len)) + (head (subseq sequence 0 m))) + (replace sequence sequence :start1 0 :start2 m) + (replace sequence head :start1 (- len m)) + sequence))) + +(defun rotate (sequence &optional (n 1)) + "Returns a sequence of the same type as SEQUENCE, with the elements of +SEQUENCE rotated by N: N elements are moved from the end of the sequence to +the front if N is positive, and -N elements moved from the front to the end if +N is negative. SEQUENCE must be a proper sequence. N must be an integer, +defaulting to 1. + +If absolute value of N is greater then the length of the sequence, the results +are identical to calling ROTATE with + + (* (signum n) (mod n (length sequence))). + +Note: the original sequence may be destructively altered, and result sequence may +share structure with it." + (if (plusp n) + (rotate-tail-to-head sequence n) + (if (minusp n) + (rotate-head-to-tail sequence (- n)) + sequence))) + +(defun shuffle (sequence &key (start 0) end) + "Returns a random permutation of SEQUENCE bounded by START and END. +Original sequence may be destructively modified, and (if it contains +CONS or lists themselv) share storage with the original one. +Signals an error if SEQUENCE is not a proper sequence." + (declare (type fixnum start) + (type (or fixnum null) end)) + (etypecase sequence + (list + (let* ((end (or end (proper-list-length sequence))) + (n (- end start))) + (do ((tail (nthcdr start sequence) (cdr tail))) + ((zerop n)) + (rotatef (car tail) (car (nthcdr (random n) tail))) + (decf n)))) + (vector + (let ((end (or end (length sequence)))) + (loop for i from start below end + do (rotatef (aref sequence i) + (aref sequence (+ i (random (- end i)))))))) + (sequence + (let ((end (or end (length sequence)))) + (loop for i from (- end 1) downto start + do (rotatef (elt sequence i) + (elt sequence (+ i (random (- end i))))))))) + sequence) + +(defun random-elt (sequence &key (start 0) end) + "Returns a random element from SEQUENCE bounded by START and END. Signals an +error if the SEQUENCE is not a proper non-empty sequence, or if END and START +are not proper bounding index designators for SEQUENCE." + (declare (sequence sequence) (fixnum start) (type (or fixnum null) end)) + (let* ((size (if (listp sequence) + (proper-list-length sequence) + (length sequence))) + (end2 (or end size))) + (cond ((zerop size) + (error 'type-error + :datum sequence + :expected-type `(and sequence (not (satisfies emptyp))))) + ((not (and (<= 0 start) (< start end2) (<= end2 size))) + (error 'simple-type-error + :datum (cons start end) + :expected-type `(cons (integer 0 (,end2)) + (or null (integer (,start) ,size))) + :format-control "~@<~S and ~S are not valid bounding index designators for ~ + a sequence of length ~S.~:@>" + :format-arguments (list start end size))) + (t + (let ((index (+ start (random (- end2 start))))) + (elt sequence index)))))) + +(declaim (inline remove/swapped-arguments)) +(defun remove/swapped-arguments (sequence item &rest keyword-arguments) + (apply #'remove item sequence keyword-arguments)) + +(define-modify-macro removef (item &rest keyword-arguments) + remove/swapped-arguments + "Modify-macro for REMOVE. Sets place designated by the first argument to +the result of calling REMOVE with ITEM, place, and the KEYWORD-ARGUMENTS.") + +(declaim (inline delete/swapped-arguments)) +(defun delete/swapped-arguments (sequence item &rest keyword-arguments) + (apply #'delete item sequence keyword-arguments)) + +(define-modify-macro deletef (item &rest keyword-arguments) + delete/swapped-arguments + "Modify-macro for DELETE. Sets place designated by the first argument to +the result of calling DELETE with ITEM, place, and the KEYWORD-ARGUMENTS.") + +(deftype proper-sequence () + "Type designator for proper sequences, that is proper lists and sequences +that are not lists." + `(or proper-list + (and (not list) sequence))) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (when (and (find-package '#:sequence) + (find-symbol (string '#:emptyp) '#:sequence)) + (pushnew 'sequence-emptyp *features*))) + +#-alexandria::sequence-emptyp +(defun emptyp (sequence) + "Returns true if SEQUENCE is an empty sequence. Signals an error if SEQUENCE +is not a sequence." + (etypecase sequence + (list (null sequence)) + (sequence (zerop (length sequence))))) + +#+alexandria::sequence-emptyp +(declaim (ftype (function (sequence) (values boolean &optional)) emptyp)) +#+alexandria::sequence-emptyp +(setf (symbol-function 'emptyp) (symbol-function 'sequence:emptyp)) +#+alexandria::sequence-emptyp +(define-compiler-macro emptyp (sequence) + `(sequence:emptyp ,sequence)) + +(defun length= (&rest sequences) + "Takes any number of sequences or integers in any order. Returns true iff +the length of all the sequences and the integers are equal. Hint: there's a +compiler macro that expands into more efficient code if the first argument +is a literal integer." + (declare (dynamic-extent sequences) + (inline sequence-of-length-p) + (optimize speed)) + (unless (cdr sequences) + (error "You must call LENGTH= with at least two arguments")) + ;; There's room for optimization here: multiple list arguments could be + ;; traversed in parallel. + (let* ((first (pop sequences)) + (current (if (integerp first) + first + (length first)))) + (declare (type array-index current)) + (dolist (el sequences) + (if (integerp el) + (unless (= el current) + (return-from length= nil)) + (unless (sequence-of-length-p el current) + (return-from length= nil))))) + t) + +(define-compiler-macro length= (&whole form length &rest sequences) + (cond + ((zerop (length sequences)) + form) + (t + (let ((optimizedp (integerp length))) + (with-unique-names (tmp current) + (declare (ignorable current)) + `(locally + (declare (inline sequence-of-length-p)) + (let ((,tmp) + ,@(unless optimizedp + `((,current ,length)))) + ,@(unless optimizedp + `((unless (integerp ,current) + (setf ,current (length ,current))))) + (and + ,@(loop + :for sequence :in sequences + :collect `(progn + (setf ,tmp ,sequence) + (if (integerp ,tmp) + (= ,tmp ,(if optimizedp + length + current)) + (sequence-of-length-p ,tmp ,(if optimizedp + length + current))))))))))))) + +(defun copy-sequence (type sequence) + "Returns a fresh sequence of TYPE, which has the same elements as +SEQUENCE." + (if (typep sequence type) + (copy-seq sequence) + (coerce sequence type))) + +(defun first-elt (sequence) + "Returns the first element of SEQUENCE. Signals a type-error if SEQUENCE is +not a sequence, or is an empty sequence." + ;; Can't just directly use ELT, as it is not guaranteed to signal the + ;; type-error. + (cond ((consp sequence) + (car sequence)) + ((and (typep sequence 'sequence) (not (emptyp sequence))) + (elt sequence 0)) + (t + (error 'type-error + :datum sequence + :expected-type '(and sequence (not (satisfies emptyp))))))) + +(defun (setf first-elt) (object sequence) + "Sets the first element of SEQUENCE. Signals a type-error if SEQUENCE is +not a sequence, is an empty sequence, or if OBJECT cannot be stored in SEQUENCE." + ;; Can't just directly use ELT, as it is not guaranteed to signal the + ;; type-error. + (cond ((consp sequence) + (setf (car sequence) object)) + ((and (typep sequence 'sequence) (not (emptyp sequence))) + (setf (elt sequence 0) object)) + (t + (error 'type-error + :datum sequence + :expected-type '(and sequence (not (satisfies emptyp))))))) + +(defun last-elt (sequence) + "Returns the last element of SEQUENCE. Signals a type-error if SEQUENCE is +not a proper sequence, or is an empty sequence." + ;; Can't just directly use ELT, as it is not guaranteed to signal the + ;; type-error. + (let ((len 0)) + (cond ((consp sequence) + (lastcar sequence)) + ((and (typep sequence '(and sequence (not list))) (plusp (setf len (length sequence)))) + (elt sequence (1- len))) + (t + (error 'type-error + :datum sequence + :expected-type '(and proper-sequence (not (satisfies emptyp)))))))) + +(defun (setf last-elt) (object sequence) + "Sets the last element of SEQUENCE. Signals a type-error if SEQUENCE is not a proper +sequence, is an empty sequence, or if OBJECT cannot be stored in SEQUENCE." + (let ((len 0)) + (cond ((consp sequence) + (setf (lastcar sequence) object)) + ((and (typep sequence '(and sequence (not list))) (plusp (setf len (length sequence)))) + (setf (elt sequence (1- len)) object)) + (t + (error 'type-error + :datum sequence + :expected-type '(and proper-sequence (not (satisfies emptyp)))))))) + +(defun starts-with-subseq (prefix sequence &rest args + &key + (return-suffix nil return-suffix-supplied-p) + &allow-other-keys) + "Test whether the first elements of SEQUENCE are the same (as per TEST) as the elements of PREFIX. + +If RETURN-SUFFIX is T the function returns, as a second value, a +sub-sequence or displaced array pointing to the sequence after PREFIX." + (declare (dynamic-extent args)) + (let ((sequence-length (length sequence)) + (prefix-length (length prefix))) + (when (< sequence-length prefix-length) + (return-from starts-with-subseq (values nil nil))) + (flet ((make-suffix (start) + (when return-suffix + (cond + ((not (arrayp sequence)) + (if start + (subseq sequence start) + (subseq sequence 0 0))) + ((not start) + (make-array 0 + :element-type (array-element-type sequence) + :adjustable nil)) + (t + (make-array (- sequence-length start) + :element-type (array-element-type sequence) + :displaced-to sequence + :displaced-index-offset start + :adjustable nil)))))) + (let ((mismatch (apply #'mismatch prefix sequence + (if return-suffix-supplied-p + (remove-from-plist args :return-suffix) + args)))) + (cond + ((not mismatch) + (values t (make-suffix nil))) + ((= mismatch prefix-length) + (values t (make-suffix mismatch))) + (t + (values nil nil))))))) + +(defun ends-with-subseq (suffix sequence &key (test #'eql)) + "Test whether SEQUENCE ends with SUFFIX. In other words: return true if +the last (length SUFFIX) elements of SEQUENCE are equal to SUFFIX." + (let ((sequence-length (length sequence)) + (suffix-length (length suffix))) + (when (< sequence-length suffix-length) + ;; if SEQUENCE is shorter than SUFFIX, then SEQUENCE can't end with SUFFIX. + (return-from ends-with-subseq nil)) + (loop for sequence-index from (- sequence-length suffix-length) below sequence-length + for suffix-index from 0 below suffix-length + when (not (funcall test (elt sequence sequence-index) (elt suffix suffix-index))) + do (return-from ends-with-subseq nil) + finally (return t)))) + +(defun starts-with (object sequence &key (test #'eql) (key #'identity)) + "Returns true if SEQUENCE is a sequence whose first element is EQL to OBJECT. +Returns NIL if the SEQUENCE is not a sequence or is an empty sequence." + (let ((first-elt (typecase sequence + (cons (car sequence)) + (sequence + (if (emptyp sequence) + (return-from starts-with nil) + (elt sequence 0))) + (t + (return-from starts-with nil))))) + (funcall test (funcall key first-elt) object))) + +(defun ends-with (object sequence &key (test #'eql) (key #'identity)) + "Returns true if SEQUENCE is a sequence whose last element is EQL to OBJECT. +Returns NIL if the SEQUENCE is not a sequence or is an empty sequence. Signals +an error if SEQUENCE is an improper list." + (let ((last-elt (typecase sequence + (cons + (lastcar sequence)) ; signals for improper lists + (sequence + ;; Can't use last-elt, as that signals an error + ;; for empty sequences + (let ((len (length sequence))) + (if (plusp len) + (elt sequence (1- len)) + (return-from ends-with nil)))) + (t + (return-from ends-with nil))))) + (funcall test (funcall key last-elt) object))) + +(defun map-combinations (function sequence &key (start 0) end length (copy t)) + "Calls FUNCTION with each combination of LENGTH constructable from the +elements of the subsequence of SEQUENCE delimited by START and END. START +defaults to 0, END to length of SEQUENCE, and LENGTH to the length of the +delimited subsequence. (So unless LENGTH is specified there is only a single +combination, which has the same elements as the delimited subsequence.) If +COPY is true (the default) each combination is freshly allocated. If COPY is +false all combinations are EQ to each other, in which case consequences are +unspecified if a combination is modified by FUNCTION." + (let* ((end (or end (length sequence))) + (size (- end start)) + (length (or length size)) + (combination (subseq sequence 0 length)) + (function (ensure-function function))) + (if (= length size) + (funcall function combination) + (flet ((call () + (funcall function (if copy + (copy-seq combination) + combination)))) + (etypecase sequence + ;; When dealing with lists we prefer walking back and + ;; forth instead of using indexes. + (list + (labels ((combine-list (c-tail o-tail) + (if (not c-tail) + (call) + (do ((tail o-tail (cdr tail))) + ((not tail)) + (setf (car c-tail) (car tail)) + (combine-list (cdr c-tail) (cdr tail)))))) + (combine-list combination (nthcdr start sequence)))) + (vector + (labels ((combine (count start) + (if (zerop count) + (call) + (loop for i from start below end + do (let ((j (- count 1))) + (setf (aref combination j) (aref sequence i)) + (combine j (+ i 1))))))) + (combine length start))) + (sequence + (labels ((combine (count start) + (if (zerop count) + (call) + (loop for i from start below end + do (let ((j (- count 1))) + (setf (elt combination j) (elt sequence i)) + (combine j (+ i 1))))))) + (combine length start))))))) + sequence) + +(defun map-permutations (function sequence &key (start 0) end length (copy t)) + "Calls function with each permutation of LENGTH constructable +from the subsequence of SEQUENCE delimited by START and END. START +defaults to 0, END to length of the sequence, and LENGTH to the +length of the delimited subsequence." + (let* ((end (or end (length sequence))) + (size (- end start)) + (length (or length size))) + (labels ((permute (seq n) + (let ((n-1 (- n 1))) + (if (zerop n-1) + (funcall function (if copy + (copy-seq seq) + seq)) + (loop for i from 0 upto n-1 + do (permute seq n-1) + (if (evenp n-1) + (rotatef (elt seq 0) (elt seq n-1)) + (rotatef (elt seq i) (elt seq n-1))))))) + (permute-sequence (seq) + (permute seq length))) + (if (= length size) + ;; Things are simple if we need to just permute the + ;; full START-END range. + (permute-sequence (subseq sequence start end)) + ;; Otherwise we need to generate all the combinations + ;; of LENGTH in the START-END range, and then permute + ;; a copy of the result: can't permute the combination + ;; directly, as they share structure with each other. + (let ((permutation (subseq sequence 0 length))) + (flet ((permute-combination (combination) + (permute-sequence (replace permutation combination)))) + (declare (dynamic-extent #'permute-combination)) + (map-combinations #'permute-combination sequence + :start start + :end end + :length length + :copy nil))))))) + +(defun map-derangements (function sequence &key (start 0) end (copy t)) + "Calls FUNCTION with each derangement of the subsequence of SEQUENCE denoted +by the bounding index designators START and END. Derangement is a permutation +of the sequence where no element remains in place. SEQUENCE is not modified, +but individual derangements are EQ to each other. Consequences are unspecified +if calling FUNCTION modifies either the derangement or SEQUENCE." + (let* ((end (or end (length sequence))) + (size (- end start)) + ;; We don't really care about the elements here. + (derangement (subseq sequence 0 size)) + ;; Bitvector that has 1 for elements that have been deranged. + (mask (make-array size :element-type 'bit :initial-element 0))) + (declare (dynamic-extent mask)) + ;; ad hoc algorith + (labels ((derange (place n) + ;; Perform one recursive step in deranging the + ;; sequence: PLACE is index of the original sequence + ;; to derange to another index, and N is the number of + ;; indexes not yet deranged. + (if (zerop n) + (funcall function (if copy + (copy-seq derangement) + derangement)) + ;; Itarate over the indexes I of the subsequence to + ;; derange: if I != PLACE and I has not yet been + ;; deranged by an earlier call put the element from + ;; PLACE to I, mark I as deranged, and recurse, + ;; finally removing the mark. + (loop for i from 0 below size + do + (unless (or (= place (+ i start)) (not (zerop (bit mask i)))) + (setf (elt derangement i) (elt sequence place) + (bit mask i) 1) + (derange (1+ place) (1- n)) + (setf (bit mask i) 0)))))) + (derange start size) + sequence))) + +(declaim (notinline sequence-of-length-p)) + +(defun extremum (sequence predicate &key key (start 0) end) + "Returns the element of SEQUENCE that would appear first if the subsequence +bounded by START and END was sorted using PREDICATE and KEY. + +EXTREMUM determines the relationship between two elements of SEQUENCE by using +the PREDICATE function. PREDICATE should return true if and only if the first +argument is strictly less than the second one (in some appropriate sense). Two +arguments X and Y are considered to be equal if (FUNCALL PREDICATE X Y) +and (FUNCALL PREDICATE Y X) are both false. + +The arguments to the PREDICATE function are computed from elements of SEQUENCE +using the KEY function, if supplied. If KEY is not supplied or is NIL, the +sequence element itself is used. + +If SEQUENCE is empty, NIL is returned." + (let* ((pred-fun (ensure-function predicate)) + (key-fun (unless (or (not key) (eq key 'identity) (eq key #'identity)) + (ensure-function key))) + (real-end (or end (length sequence)))) + (cond ((> real-end start) + (if key-fun + (flet ((reduce-keys (a b) + (if (funcall pred-fun + (funcall key-fun a) + (funcall key-fun b)) + a + b))) + (declare (dynamic-extent #'reduce-keys)) + (reduce #'reduce-keys sequence :start start :end real-end)) + (flet ((reduce-elts (a b) + (if (funcall pred-fun a b) + a + b))) + (declare (dynamic-extent #'reduce-elts)) + (reduce #'reduce-elts sequence :start start :end real-end)))) + ((= real-end start) + nil) + (t + (error "Invalid bounding indexes for sequence of length ~S: ~S ~S, ~S ~S" + (length sequence) + :start start + :end end))))) diff --git a/third_party/lisp/alexandria/strings.lisp b/third_party/lisp/alexandria/strings.lisp new file mode 100644 index 000000000000..e9fd91c96155 --- /dev/null +++ b/third_party/lisp/alexandria/strings.lisp @@ -0,0 +1,6 @@ +(in-package :alexandria) + +(deftype string-designator () + "A string designator type. A string designator is either a string, a symbol, +or a character." + `(or symbol string character)) diff --git a/third_party/lisp/alexandria/symbols.lisp b/third_party/lisp/alexandria/symbols.lisp new file mode 100644 index 000000000000..5733d3e1cc50 --- /dev/null +++ b/third_party/lisp/alexandria/symbols.lisp @@ -0,0 +1,65 @@ +(in-package :alexandria) + +(declaim (inline ensure-symbol)) +(defun ensure-symbol (name &optional (package *package*)) + "Returns a symbol with name designated by NAME, accessible in package +designated by PACKAGE. If symbol is not already accessible in PACKAGE, it is +interned there. Returns a secondary value reflecting the status of the symbol +in the package, which matches the secondary return value of INTERN. + +Example: + + (ensure-symbol :cons :cl) => cl:cons, :external +" + (intern (string name) package)) + +(defun maybe-intern (name package) + (values + (if package + (intern name (if (eq t package) *package* package)) + (make-symbol name)))) + +(declaim (inline format-symbol)) +(defun format-symbol (package control &rest arguments) + "Constructs a string by applying ARGUMENTS to string designator CONTROL as +if by FORMAT within WITH-STANDARD-IO-SYNTAX, and then creates a symbol named +by that string. + +If PACKAGE is NIL, returns an uninterned symbol, if package is T, returns a +symbol interned in the current package, and otherwise returns a symbol +interned in the package designated by PACKAGE." + (maybe-intern (with-standard-io-syntax + (apply #'format nil (string control) arguments)) + package)) + +(defun make-keyword (name) + "Interns the string designated by NAME in the KEYWORD package." + (intern (string name) :keyword)) + +(defun make-gensym (name) + "If NAME is a non-negative integer, calls GENSYM using it. Otherwise NAME +must be a string designator, in which case calls GENSYM using the designated +string as the argument." + (gensym (if (typep name '(integer 0)) + name + (string name)))) + +(defun make-gensym-list (length &optional (x "G")) + "Returns a list of LENGTH gensyms, each generated as if with a call to MAKE-GENSYM, +using the second (optional, defaulting to \"G\") argument." + (let ((g (if (typep x '(integer 0)) x (string x)))) + (loop repeat length + collect (gensym g)))) + +(defun symbolicate (&rest things) + "Concatenate together the names of some strings and symbols, +producing a symbol in the current package." + (let* ((length (reduce #'+ things + :key (lambda (x) (length (string x))))) + (name (make-array length :element-type 'character))) + (let ((index 0)) + (dolist (thing things (values (intern name))) + (let* ((x (string thing)) + (len (length x))) + (replace name x :start1 index) + (incf index len)))))) diff --git a/third_party/lisp/alexandria/tests.lisp b/third_party/lisp/alexandria/tests.lisp new file mode 100644 index 000000000000..b70ef0475e81 --- /dev/null +++ b/third_party/lisp/alexandria/tests.lisp @@ -0,0 +1,2047 @@ +(in-package :cl-user) + +(defpackage :alexandria-tests + (:use :cl :alexandria #+sbcl :sb-rt #-sbcl :rtest) + (:import-from #+sbcl :sb-rt #-sbcl :rtest + #:*compile-tests* #:*expected-failures*)) + +(in-package :alexandria-tests) + +(defun run-tests (&key ((:compiled *compile-tests*))) + (do-tests)) + +(defun hash-table-test-name (name) + ;; Workaround for Clisp calling EQL in a hash-table FASTHASH-EQL. + (hash-table-test (make-hash-table :test name))) + +;;;; Arrays + +(deftest copy-array.1 + (let* ((orig (vector 1 2 3)) + (copy (copy-array orig))) + (values (eq orig copy) (equalp orig copy))) + nil t) + +(deftest copy-array.2 + (let ((orig (make-array 1024 :fill-pointer 0))) + (vector-push-extend 1 orig) + (vector-push-extend 2 orig) + (vector-push-extend 3 orig) + (let ((copy (copy-array orig))) + (values (eq orig copy) (equalp orig copy) + (array-has-fill-pointer-p copy) + (eql (fill-pointer orig) (fill-pointer copy))))) + nil t t t) + +(deftest copy-array.3 + (let* ((orig (vector 1 2 3)) + (copy (copy-array orig))) + (typep copy 'simple-array)) + t) + +(deftest copy-array.4 + (let ((orig (make-array 21 + :adjustable t + :fill-pointer 0))) + (dotimes (n 42) + (vector-push-extend n orig)) + (let ((copy (copy-array orig + :adjustable nil + :fill-pointer nil))) + (typep copy 'simple-array))) + t) + +(deftest array-index.1 + (typep 0 'array-index) + t) + +;;;; Conditions + +(deftest unwind-protect-case.1 + (let (result) + (unwind-protect-case () + (random 10) + (:normal (push :normal result)) + (:abort (push :abort result)) + (:always (push :always result))) + result) + (:always :normal)) + +(deftest unwind-protect-case.2 + (let (result) + (unwind-protect-case () + (random 10) + (:always (push :always result)) + (:normal (push :normal result)) + (:abort (push :abort result))) + result) + (:normal :always)) + +(deftest unwind-protect-case.3 + (let (result1 result2 result3) + (ignore-errors + (unwind-protect-case () + (error "FOOF!") + (:normal (push :normal result1)) + (:abort (push :abort result1)) + (:always (push :always result1)))) + (catch 'foof + (unwind-protect-case () + (throw 'foof 42) + (:normal (push :normal result2)) + (:abort (push :abort result2)) + (:always (push :always result2)))) + (block foof + (unwind-protect-case () + (return-from foof 42) + (:normal (push :normal result3)) + (:abort (push :abort result3)) + (:always (push :always result3)))) + (values result1 result2 result3)) + (:always :abort) + (:always :abort) + (:always :abort)) + +(deftest unwind-protect-case.4 + (let (result) + (unwind-protect-case (aborted-p) + (random 42) + (:always (setq result aborted-p))) + result) + nil) + +(deftest unwind-protect-case.5 + (let (result) + (block foof + (unwind-protect-case (aborted-p) + (return-from foof) + (:always (setq result aborted-p)))) + result) + t) + +;;;; Control flow + +(deftest switch.1 + (switch (13 :test =) + (12 :oops) + (13.0 :yay)) + :yay) + +(deftest switch.2 + (switch (13) + ((+ 12 2) :oops) + ((- 13 1) :oops2) + (t :yay)) + :yay) + +(deftest eswitch.1 + (let ((x 13)) + (eswitch (x :test =) + (12 :oops) + (13.0 :yay))) + :yay) + +(deftest eswitch.2 + (let ((x 13)) + (eswitch (x :key 1+) + (11 :oops) + (14 :yay))) + :yay) + +(deftest cswitch.1 + (cswitch (13 :test =) + (12 :oops) + (13.0 :yay)) + :yay) + +(deftest cswitch.2 + (cswitch (13 :key 1-) + (12 :yay) + (13.0 :oops)) + :yay) + +(deftest multiple-value-prog2.1 + (multiple-value-prog2 + (values 1 1 1) + (values 2 20 200) + (values 3 3 3)) + 2 20 200) + +(deftest nth-value-or.1 + (multiple-value-bind (a b c) + (nth-value-or 1 + (values 1 nil 1) + (values 2 2 2)) + (= a b c 2)) + t) + +(deftest whichever.1 + (let ((x (whichever 1 2 3))) + (and (member x '(1 2 3)) t)) + t) + +(deftest whichever.2 + (let* ((a 1) + (b 2) + (c 3) + (x (whichever a b c))) + (and (member x '(1 2 3)) t)) + t) + +(deftest xor.1 + (xor nil nil 1 nil) + 1 + t) + +(deftest xor.2 + (xor nil nil 1 2) + nil + nil) + +(deftest xor.3 + (xor nil nil nil) + nil + t) + +;;;; Definitions + +(deftest define-constant.1 + (let ((name (gensym))) + (eval `(define-constant ,name "FOO" :test 'equal)) + (eval `(define-constant ,name "FOO" :test 'equal)) + (values (equal "FOO" (symbol-value name)) + (constantp name))) + t + t) + +(deftest define-constant.2 + (let ((name (gensym))) + (eval `(define-constant ,name 13)) + (eval `(define-constant ,name 13)) + (values (eql 13 (symbol-value name)) + (constantp name))) + t + t) + +;;;; Errors + +;;; TYPEP is specified to return a generalized boolean and, for +;;; example, ECL exploits this by returning the superclasses of ERROR +;;; in this case. +(defun errorp (x) + (not (null (typep x 'error)))) + +(deftest required-argument.1 + (multiple-value-bind (res err) + (ignore-errors (required-argument)) + (errorp err)) + t) + +;;;; Hash tables + +(deftest ensure-gethash.1 + (let ((table (make-hash-table)) + (x (list 1))) + (multiple-value-bind (value already-there) + (ensure-gethash x table 42) + (and (= value 42) + (not already-there) + (= 42 (gethash x table)) + (multiple-value-bind (value2 already-there2) + (ensure-gethash x table 13) + (and (= value2 42) + already-there2 + (= 42 (gethash x table))))))) + t) + +(deftest ensure-gethash.2 + (let ((table (make-hash-table)) + (count 0)) + (multiple-value-call #'values + (ensure-gethash (progn (incf count) :foo) + (progn (incf count) table) + (progn (incf count) :bar)) + (gethash :foo table) + count)) + :bar nil :bar t 3) + +(deftest copy-hash-table.1 + (let ((orig (make-hash-table :test 'eq :size 123)) + (foo "foo")) + (setf (gethash orig orig) t + (gethash foo orig) t) + (let ((eq-copy (copy-hash-table orig)) + (eql-copy (copy-hash-table orig :test 'eql)) + (equal-copy (copy-hash-table orig :test 'equal)) + (equalp-copy (copy-hash-table orig :test 'equalp))) + (list (eql (hash-table-size eq-copy) (hash-table-size orig)) + (eql (hash-table-rehash-size eq-copy) + (hash-table-rehash-size orig)) + (hash-table-count eql-copy) + (gethash orig eq-copy) + (gethash (copy-seq foo) eql-copy) + (gethash foo eql-copy) + (gethash (copy-seq foo) equal-copy) + (gethash "FOO" equal-copy) + (gethash "FOO" equalp-copy)))) + (t t 2 t nil t t nil t)) + +(deftest copy-hash-table.2 + (let ((ht (make-hash-table)) + (list (list :list (vector :A :B :C)))) + (setf (gethash 'list ht) list) + (let* ((shallow-copy (copy-hash-table ht)) + (deep1-copy (copy-hash-table ht :key 'copy-list)) + (list (gethash 'list ht)) + (shallow-list (gethash 'list shallow-copy)) + (deep1-list (gethash 'list deep1-copy))) + (list (eq ht shallow-copy) + (eq ht deep1-copy) + (eq list shallow-list) + (eq list deep1-list) ; outer list was copied. + (eq (second list) (second shallow-list)) + (eq (second list) (second deep1-list)) ; inner vector wasn't copied. + ))) + (nil nil t nil t t)) + +(deftest maphash-keys.1 + (let ((keys nil) + (table (make-hash-table))) + (declare (notinline maphash-keys)) + (dotimes (i 10) + (setf (gethash i table) t)) + (maphash-keys (lambda (k) (push k keys)) table) + (set-equal keys '(0 1 2 3 4 5 6 7 8 9))) + t) + +(deftest maphash-values.1 + (let ((vals nil) + (table (make-hash-table))) + (declare (notinline maphash-values)) + (dotimes (i 10) + (setf (gethash i table) (- i))) + (maphash-values (lambda (v) (push v vals)) table) + (set-equal vals '(0 -1 -2 -3 -4 -5 -6 -7 -8 -9))) + t) + +(deftest hash-table-keys.1 + (let ((table (make-hash-table))) + (dotimes (i 10) + (setf (gethash i table) t)) + (set-equal (hash-table-keys table) '(0 1 2 3 4 5 6 7 8 9))) + t) + +(deftest hash-table-values.1 + (let ((table (make-hash-table))) + (dotimes (i 10) + (setf (gethash (gensym) table) i)) + (set-equal (hash-table-values table) '(0 1 2 3 4 5 6 7 8 9))) + t) + +(deftest hash-table-alist.1 + (let ((table (make-hash-table))) + (dotimes (i 10) + (setf (gethash i table) (- i))) + (let ((alist (hash-table-alist table))) + (list (length alist) + (assoc 0 alist) + (assoc 3 alist) + (assoc 9 alist) + (assoc nil alist)))) + (10 (0 . 0) (3 . -3) (9 . -9) nil)) + +(deftest hash-table-plist.1 + (let ((table (make-hash-table))) + (dotimes (i 10) + (setf (gethash i table) (- i))) + (let ((plist (hash-table-plist table))) + (list (length plist) + (getf plist 0) + (getf plist 2) + (getf plist 7) + (getf plist nil)))) + (20 0 -2 -7 nil)) + +(deftest alist-hash-table.1 + (let* ((alist '((0 a) (1 b) (2 c))) + (table (alist-hash-table alist))) + (list (hash-table-count table) + (gethash 0 table) + (gethash 1 table) + (gethash 2 table) + (eq (hash-table-test-name 'eql) + (hash-table-test table)))) + (3 (a) (b) (c) t)) + +(deftest alist-hash-table.duplicate-keys + (let* ((alist '((0 a) (1 b) (0 c) (1 d) (2 e))) + (table (alist-hash-table alist))) + (list (hash-table-count table) + (gethash 0 table) + (gethash 1 table) + (gethash 2 table))) + (3 (a) (b) (e))) + +(deftest plist-hash-table.1 + (let* ((plist '(:a 1 :b 2 :c 3)) + (table (plist-hash-table plist :test 'eq))) + (list (hash-table-count table) + (gethash :a table) + (gethash :b table) + (gethash :c table) + (gethash 2 table) + (gethash nil table) + (eq (hash-table-test-name 'eq) + (hash-table-test table)))) + (3 1 2 3 nil nil t)) + +(deftest plist-hash-table.duplicate-keys + (let* ((plist '(:a 1 :b 2 :a 3 :b 4 :c 5)) + (table (plist-hash-table plist))) + (list (hash-table-count table) + (gethash :a table) + (gethash :b table) + (gethash :c table))) + (3 1 2 5)) + +;;;; Functions + +(deftest disjoin.1 + (let ((disjunction (disjoin (lambda (x) + (and (consp x) :cons)) + (lambda (x) + (and (stringp x) :string))))) + (list (funcall disjunction 'zot) + (funcall disjunction '(foo bar)) + (funcall disjunction "test"))) + (nil :cons :string)) + +(deftest disjoin.2 + (let ((disjunction (disjoin #'zerop))) + (list (funcall disjunction 0) + (funcall disjunction 1))) + (t nil)) + +(deftest conjoin.1 + (let ((conjunction (conjoin #'consp + (lambda (x) + (stringp (car x))) + (lambda (x) + (char (car x) 0))))) + (list (funcall conjunction 'zot) + (funcall conjunction '(foo)) + (funcall conjunction '("foo")))) + (nil nil #\f)) + +(deftest conjoin.2 + (let ((conjunction (conjoin #'zerop))) + (list (funcall conjunction 0) + (funcall conjunction 1))) + (t nil)) + +(deftest compose.1 + (let ((composite (compose '1+ + (lambda (x) + (* x 2)) + #'read-from-string))) + (funcall composite "1")) + 3) + +(deftest compose.2 + (let ((composite + (locally (declare (notinline compose)) + (compose '1+ + (lambda (x) + (* x 2)) + #'read-from-string)))) + (funcall composite "2")) + 5) + +(deftest compose.3 + (let ((compose-form (funcall (compiler-macro-function 'compose) + '(compose '1+ + (lambda (x) + (* x 2)) + #'read-from-string) + nil))) + (let ((fun (funcall (compile nil `(lambda () ,compose-form))))) + (funcall fun "3"))) + 7) + +(deftest compose.4 + (let ((composite (compose #'zerop))) + (list (funcall composite 0) + (funcall composite 1))) + (t nil)) + +(deftest multiple-value-compose.1 + (let ((composite (multiple-value-compose + #'truncate + (lambda (x y) + (values y x)) + (lambda (x) + (with-input-from-string (s x) + (values (read s) (read s))))))) + (multiple-value-list (funcall composite "2 7"))) + (3 1)) + +(deftest multiple-value-compose.2 + (let ((composite (locally (declare (notinline multiple-value-compose)) + (multiple-value-compose + #'truncate + (lambda (x y) + (values y x)) + (lambda (x) + (with-input-from-string (s x) + (values (read s) (read s)))))))) + (multiple-value-list (funcall composite "2 11"))) + (5 1)) + +(deftest multiple-value-compose.3 + (let ((compose-form (funcall (compiler-macro-function 'multiple-value-compose) + '(multiple-value-compose + #'truncate + (lambda (x y) + (values y x)) + (lambda (x) + (with-input-from-string (s x) + (values (read s) (read s))))) + nil))) + (let ((fun (funcall (compile nil `(lambda () ,compose-form))))) + (multiple-value-list (funcall fun "2 9")))) + (4 1)) + +(deftest multiple-value-compose.4 + (let ((composite (multiple-value-compose #'truncate))) + (multiple-value-list (funcall composite 9 2))) + (4 1)) + +(deftest curry.1 + (let ((curried (curry '+ 3))) + (funcall curried 1 5)) + 9) + +(deftest curry.2 + (let ((curried (locally (declare (notinline curry)) + (curry '* 2 3)))) + (funcall curried 7)) + 42) + +(deftest curry.3 + (let ((curried-form (funcall (compiler-macro-function 'curry) + '(curry '/ 8) + nil))) + (let ((fun (funcall (compile nil `(lambda () ,curried-form))))) + (funcall fun 2))) + 4) + +(deftest curry.4 + (let* ((x 1) + (curried (curry (progn + (incf x) + (lambda (y z) (* x y z))) + 3))) + (list (funcall curried 7) + (funcall curried 7) + x)) + (42 42 2)) + +(deftest rcurry.1 + (let ((r (rcurry '/ 2))) + (funcall r 8)) + 4) + +(deftest rcurry.2 + (let* ((x 1) + (curried (rcurry (progn + (incf x) + (lambda (y z) (* x y z))) + 3))) + (list (funcall curried 7) + (funcall curried 7) + x)) + (42 42 2)) + +(deftest named-lambda.1 + (let ((fac (named-lambda fac (x) + (if (> x 1) + (* x (fac (- x 1))) + x)))) + (funcall fac 5)) + 120) + +(deftest named-lambda.2 + (let ((fac (named-lambda fac (&key x) + (if (> x 1) + (* x (fac :x (- x 1))) + x)))) + (funcall fac :x 5)) + 120) + +;;;; Lists + +(deftest alist-plist.1 + (alist-plist '((a . 1) (b . 2) (c . 3))) + (a 1 b 2 c 3)) + +(deftest plist-alist.1 + (plist-alist '(a 1 b 2 c 3)) + ((a . 1) (b . 2) (c . 3))) + +(deftest unionf.1 + (let* ((list (list 1 2 3)) + (orig list)) + (unionf list (list 1 2 4)) + (values (equal orig (list 1 2 3)) + (eql (length list) 4) + (set-difference list (list 1 2 3 4)) + (set-difference (list 1 2 3 4) list))) + t + t + nil + nil) + +(deftest nunionf.1 + (let ((list (list 1 2 3))) + (nunionf list (list 1 2 4)) + (values (eql (length list) 4) + (set-difference (list 1 2 3 4) list) + (set-difference list (list 1 2 3 4)))) + t + nil + nil) + +(deftest appendf.1 + (let* ((list (list 1 2 3)) + (orig list)) + (appendf list '(4 5 6) '(7 8)) + (list list (eq list orig))) + ((1 2 3 4 5 6 7 8) nil)) + +(deftest nconcf.1 + (let ((list1 (list 1 2 3)) + (list2 (list 4 5 6))) + (nconcf list1 list2 (list 7 8 9)) + list1) + (1 2 3 4 5 6 7 8 9)) + +(deftest circular-list.1 + (let ((circle (circular-list 1 2 3))) + (list (first circle) + (second circle) + (third circle) + (fourth circle) + (eq circle (nthcdr 3 circle)))) + (1 2 3 1 t)) + +(deftest circular-list-p.1 + (let* ((circle (circular-list 1 2 3 4)) + (tree (list circle circle)) + (dotted (cons circle t)) + (proper (list 1 2 3 circle)) + (tailcirc (list* 1 2 3 circle))) + (list (circular-list-p circle) + (circular-list-p tree) + (circular-list-p dotted) + (circular-list-p proper) + (circular-list-p tailcirc))) + (t nil nil nil t)) + +(deftest circular-list-p.2 + (circular-list-p 'foo) + nil) + +(deftest circular-tree-p.1 + (let* ((circle (circular-list 1 2 3 4)) + (tree1 (list circle circle)) + (tree2 (let* ((level2 (list 1 nil 2)) + (level1 (list level2))) + (setf (second level2) level1) + level1)) + (dotted (cons circle t)) + (proper (list 1 2 3 circle)) + (tailcirc (list* 1 2 3 circle)) + (quite-proper (list 1 2 3)) + (quite-dotted (list 1 (cons 2 3)))) + (list (circular-tree-p circle) + (circular-tree-p tree1) + (circular-tree-p tree2) + (circular-tree-p dotted) + (circular-tree-p proper) + (circular-tree-p tailcirc) + (circular-tree-p quite-proper) + (circular-tree-p quite-dotted))) + (t t t t t t nil nil)) + +(deftest circular-tree-p.2 + (alexandria:circular-tree-p '#1=(#1#)) + t) + +(deftest proper-list-p.1 + (let ((l1 (list 1)) + (l2 (list 1 2)) + (l3 (cons 1 2)) + (l4 (list (cons 1 2) 3)) + (l5 (circular-list 1 2))) + (list (proper-list-p l1) + (proper-list-p l2) + (proper-list-p l3) + (proper-list-p l4) + (proper-list-p l5))) + (t t nil t nil)) + +(deftest proper-list-p.2 + (proper-list-p '(1 2 . 3)) + nil) + +(deftest proper-list.type.1 + (let ((l1 (list 1)) + (l2 (list 1 2)) + (l3 (cons 1 2)) + (l4 (list (cons 1 2) 3)) + (l5 (circular-list 1 2))) + (list (typep l1 'proper-list) + (typep l2 'proper-list) + (typep l3 'proper-list) + (typep l4 'proper-list) + (typep l5 'proper-list))) + (t t nil t nil)) + +(deftest proper-list-length.1 + (values + (proper-list-length nil) + (proper-list-length (list 1)) + (proper-list-length (list 2 2)) + (proper-list-length (list 3 3 3)) + (proper-list-length (list 4 4 4 4)) + (proper-list-length (list 5 5 5 5 5)) + (proper-list-length (list 6 6 6 6 6 6)) + (proper-list-length (list 7 7 7 7 7 7 7)) + (proper-list-length (list 8 8 8 8 8 8 8 8)) + (proper-list-length (list 9 9 9 9 9 9 9 9 9))) + 0 1 2 3 4 5 6 7 8 9) + +(deftest proper-list-length.2 + (flet ((plength (x) + (handler-case + (proper-list-length x) + (type-error () + :ok)))) + (values + (plength (list* 1)) + (plength (list* 2 2)) + (plength (list* 3 3 3)) + (plength (list* 4 4 4 4)) + (plength (list* 5 5 5 5 5)) + (plength (list* 6 6 6 6 6 6)) + (plength (list* 7 7 7 7 7 7 7)) + (plength (list* 8 8 8 8 8 8 8 8)) + (plength (list* 9 9 9 9 9 9 9 9 9)))) + :ok :ok :ok + :ok :ok :ok + :ok :ok :ok) + +(deftest lastcar.1 + (let ((l1 (list 1)) + (l2 (list 1 2))) + (list (lastcar l1) + (lastcar l2))) + (1 2)) + +(deftest lastcar.error.2 + (handler-case + (progn + (lastcar (circular-list 1 2 3)) + nil) + (error () + t)) + t) + +(deftest setf-lastcar.1 + (let ((l (list 1 2 3 4))) + (values (lastcar l) + (progn + (setf (lastcar l) 42) + (lastcar l)))) + 4 + 42) + +(deftest setf-lastcar.2 + (let ((l (circular-list 1 2 3))) + (multiple-value-bind (res err) + (ignore-errors (setf (lastcar l) 4)) + (typep err 'type-error))) + t) + +(deftest make-circular-list.1 + (let ((l (make-circular-list 3 :initial-element :x))) + (setf (car l) :y) + (list (eq l (nthcdr 3 l)) + (first l) + (second l) + (third l) + (fourth l))) + (t :y :x :x :y)) + +(deftest circular-list.type.1 + (let* ((l1 (list 1 2 3)) + (l2 (circular-list 1 2 3)) + (l3 (list* 1 2 3 l2))) + (list (typep l1 'circular-list) + (typep l2 'circular-list) + (typep l3 'circular-list))) + (nil t t)) + +(deftest ensure-list.1 + (let ((x (list 1)) + (y 2)) + (list (ensure-list x) + (ensure-list y))) + ((1) (2))) + +(deftest ensure-cons.1 + (let ((x (cons 1 2)) + (y nil) + (z "foo")) + (values (ensure-cons x) + (ensure-cons y) + (ensure-cons z))) + (1 . 2) + (nil) + ("foo")) + +(deftest setp.1 + (setp '(1)) + t) + +(deftest setp.2 + (setp nil) + t) + +(deftest setp.3 + (setp "foo") + nil) + +(deftest setp.4 + (setp '(1 2 3 1)) + nil) + +(deftest setp.5 + (setp '(1 2 3)) + t) + +(deftest setp.6 + (setp '(a :a)) + t) + +(deftest setp.7 + (setp '(a :a) :key 'character) + nil) + +(deftest setp.8 + (setp '(a :a) :key 'character :test (constantly nil)) + t) + +(deftest set-equal.1 + (set-equal '(1 2 3) '(3 1 2)) + t) + +(deftest set-equal.2 + (set-equal '("Xa") '("Xb") + :test (lambda (a b) (eql (char a 0) (char b 0)))) + t) + +(deftest set-equal.3 + (set-equal '(1 2) '(4 2)) + nil) + +(deftest set-equal.4 + (set-equal '(a b c) '(:a :b :c) :key 'string :test 'equal) + t) + +(deftest set-equal.5 + (set-equal '(a d c) '(:a :b :c) :key 'string :test 'equal) + nil) + +(deftest set-equal.6 + (set-equal '(a b c) '(a b c d)) + nil) + +(deftest map-product.1 + (map-product 'cons '(2 3) '(1 4)) + ((2 . 1) (2 . 4) (3 . 1) (3 . 4))) + +(deftest map-product.2 + (map-product #'cons '(2 3) '(1 4)) + ((2 . 1) (2 . 4) (3 . 1) (3 . 4))) + +(deftest flatten.1 + (flatten '((1) 2 (((3 4))) ((((5)) 6)) 7)) + (1 2 3 4 5 6 7)) + +(deftest remove-from-plist.1 + (let ((orig '(a 1 b 2 c 3 d 4))) + (list (remove-from-plist orig 'a 'c) + (remove-from-plist orig 'b 'd) + (remove-from-plist orig 'b) + (remove-from-plist orig 'a) + (remove-from-plist orig 'd 42 "zot") + (remove-from-plist orig 'a 'b 'c 'd) + (remove-from-plist orig 'a 'b 'c 'd 'x) + (equal orig '(a 1 b 2 c 3 d 4)))) + ((b 2 d 4) + (a 1 c 3) + (a 1 c 3 d 4) + (b 2 c 3 d 4) + (a 1 b 2 c 3) + nil + nil + t)) + +(deftest delete-from-plist.1 + (let ((orig '(a 1 b 2 c 3 d 4 d 5))) + (list (delete-from-plist (copy-list orig) 'a 'c) + (delete-from-plist (copy-list orig) 'b 'd) + (delete-from-plist (copy-list orig) 'b) + (delete-from-plist (copy-list orig) 'a) + (delete-from-plist (copy-list orig) 'd 42 "zot") + (delete-from-plist (copy-list orig) 'a 'b 'c 'd) + (delete-from-plist (copy-list orig) 'a 'b 'c 'd 'x) + (equal orig (delete-from-plist orig)) + (eq orig (delete-from-plist orig)))) + ((b 2 d 4 d 5) + (a 1 c 3) + (a 1 c 3 d 4 d 5) + (b 2 c 3 d 4 d 5) + (a 1 b 2 c 3) + nil + nil + t + t)) + +(deftest mappend.1 + (mappend (compose 'list '*) '(1 2 3) '(1 2 3)) + (1 4 9)) + +(deftest assoc-value.1 + (let ((key1 '(complex key)) + (key2 'simple-key) + (alist '()) + (result '())) + (push 1 (assoc-value alist key1 :test #'equal)) + (push 2 (assoc-value alist key1 :test 'equal)) + (push 42 (assoc-value alist key2)) + (push 43 (assoc-value alist key2 :test 'eq)) + (push (assoc-value alist key1 :test #'equal) result) + (push (assoc-value alist key2) result) + + (push 'very (rassoc-value alist (list 2 1) :test #'equal)) + (push (cdr (assoc '(very complex key) alist :test #'equal)) result) + result) + ((2 1) (43 42) (2 1))) + +;;;; Numbers + +(deftest clamp.1 + (list (clamp 1.5 1 2) + (clamp 2.0 1 2) + (clamp 1.0 1 2) + (clamp 3 1 2) + (clamp 0 1 2)) + (1.5 2.0 1.0 2 1)) + +(deftest gaussian-random.1 + (let ((min -0.2) + (max +0.2)) + (multiple-value-bind (g1 g2) + (gaussian-random min max) + (values (<= min g1 max) + (<= min g2 max) + (/= g1 g2) ;uh + ))) + t + t + t) + +#+sbcl +(deftest gaussian-random.2 + (handler-case + (sb-ext:with-timeout 2 + (progn + (loop + :repeat 10000 + :do (gaussian-random 0 nil)) + 'done)) + (sb-ext:timeout () + 'timed-out)) + done) + +(deftest iota.1 + (iota 3) + (0 1 2)) + +(deftest iota.2 + (iota 3 :start 0.0d0) + (0.0d0 1.0d0 2.0d0)) + +(deftest iota.3 + (iota 3 :start 2 :step 3.0) + (2.0 5.0 8.0)) + +(deftest map-iota.1 + (let (all) + (declare (notinline map-iota)) + (values (map-iota (lambda (x) (push x all)) + 3 + :start 2 + :step 1.1d0) + all)) + 3 + (4.2d0 3.1d0 2.0d0)) + +(deftest lerp.1 + (lerp 0.5 1 2) + 1.5) + +(deftest lerp.2 + (lerp 0.1 1 2) + 1.1) + +(deftest lerp.3 + (lerp 0.1 4 25) + 6.1) + +(deftest mean.1 + (mean '(1 2 3)) + 2) + +(deftest mean.2 + (mean '(1 2 3 4)) + 5/2) + +(deftest mean.3 + (mean '(1 2 10)) + 13/3) + +(deftest median.1 + (median '(100 0 99 1 98 2 97)) + 97) + +(deftest median.2 + (median '(100 0 99 1 98 2 97 96)) + 193/2) + +(deftest variance.1 + (variance (list 1 2 3)) + 2/3) + +(deftest standard-deviation.1 + (< 0 (standard-deviation (list 1 2 3)) 1) + t) + +(deftest maxf.1 + (let ((x 1)) + (maxf x 2) + x) + 2) + +(deftest maxf.2 + (let ((x 1)) + (maxf x 0) + x) + 1) + +(deftest maxf.3 + (let ((x 1) + (c 0)) + (maxf x (incf c)) + (list x c)) + (1 1)) + +(deftest maxf.4 + (let ((xv (vector 0 0 0)) + (p 0)) + (maxf (svref xv (incf p)) (incf p)) + (list p xv)) + (2 #(0 2 0))) + +(deftest minf.1 + (let ((y 1)) + (minf y 0) + y) + 0) + +(deftest minf.2 + (let ((xv (vector 10 10 10)) + (p 0)) + (minf (svref xv (incf p)) (incf p)) + (list p xv)) + (2 #(10 2 10))) + +(deftest subfactorial.1 + (mapcar #'subfactorial (iota 22)) + (1 + 0 + 1 + 2 + 9 + 44 + 265 + 1854 + 14833 + 133496 + 1334961 + 14684570 + 176214841 + 2290792932 + 32071101049 + 481066515734 + 7697064251745 + 130850092279664 + 2355301661033953 + 44750731559645106 + 895014631192902121 + 18795307255050944540)) + +;;;; Arrays + +#+nil +(deftest array-index.type) + +#+nil +(deftest copy-array) + +;;;; Sequences + +(deftest rotate.1 + (list (rotate (list 1 2 3) 0) + (rotate (list 1 2 3) 1) + (rotate (list 1 2 3) 2) + (rotate (list 1 2 3) 3) + (rotate (list 1 2 3) 4)) + ((1 2 3) + (3 1 2) + (2 3 1) + (1 2 3) + (3 1 2))) + +(deftest rotate.2 + (list (rotate (vector 1 2 3 4) 0) + (rotate (vector 1 2 3 4)) + (rotate (vector 1 2 3 4) 2) + (rotate (vector 1 2 3 4) 3) + (rotate (vector 1 2 3 4) 4) + (rotate (vector 1 2 3 4) 5)) + (#(1 2 3 4) + #(4 1 2 3) + #(3 4 1 2) + #(2 3 4 1) + #(1 2 3 4) + #(4 1 2 3))) + +(deftest rotate.3 + (list (rotate (list 1 2 3) 0) + (rotate (list 1 2 3) -1) + (rotate (list 1 2 3) -2) + (rotate (list 1 2 3) -3) + (rotate (list 1 2 3) -4)) + ((1 2 3) + (2 3 1) + (3 1 2) + (1 2 3) + (2 3 1))) + +(deftest rotate.4 + (list (rotate (vector 1 2 3 4) 0) + (rotate (vector 1 2 3 4) -1) + (rotate (vector 1 2 3 4) -2) + (rotate (vector 1 2 3 4) -3) + (rotate (vector 1 2 3 4) -4) + (rotate (vector 1 2 3 4) -5)) + (#(1 2 3 4) + #(2 3 4 1) + #(3 4 1 2) + #(4 1 2 3) + #(1 2 3 4) + #(2 3 4 1))) + +(deftest rotate.5 + (values (rotate (list 1) 17) + (rotate (list 1) -5)) + (1) + (1)) + +(deftest shuffle.1 + (let ((s (shuffle (iota 100)))) + (list (equal s (iota 100)) + (every (lambda (x) + (member x s)) + (iota 100)) + (every (lambda (x) + (typep x '(integer 0 99))) + s))) + (nil t t)) + +(deftest shuffle.2 + (let ((s (shuffle (coerce (iota 100) 'vector)))) + (list (equal s (coerce (iota 100) 'vector)) + (every (lambda (x) + (find x s)) + (iota 100)) + (every (lambda (x) + (typep x '(integer 0 99))) + s))) + (nil t t)) + +(deftest shuffle.3 + (let* ((orig (coerce (iota 21) 'vector)) + (copy (copy-seq orig))) + (shuffle copy :start 10 :end 15) + (list (every #'eql (subseq copy 0 10) (subseq orig 0 10)) + (every #'eql (subseq copy 15) (subseq orig 15)))) + (t t)) + +(deftest random-elt.1 + (let ((s1 #(1 2 3 4)) + (s2 '(1 2 3 4))) + (list (dotimes (i 1000 nil) + (unless (member (random-elt s1) s2) + (return nil)) + (when (/= (random-elt s1) (random-elt s1)) + (return t))) + (dotimes (i 1000 nil) + (unless (member (random-elt s2) s2) + (return nil)) + (when (/= (random-elt s2) (random-elt s2)) + (return t))))) + (t t)) + +(deftest removef.1 + (let* ((x '(1 2 3)) + (x* x) + (y #(1 2 3)) + (y* y)) + (removef x 1) + (removef y 3) + (list x x* y y*)) + ((2 3) + (1 2 3) + #(1 2) + #(1 2 3))) + +(deftest deletef.1 + (let* ((x (list 1 2 3)) + (x* x) + (y (vector 1 2 3))) + (deletef x 2) + (deletef y 1) + (list x x* y)) + ((1 3) + (1 3) + #(2 3))) + +(deftest map-permutations.1 + (let ((seq (list 1 2 3)) + (seen nil) + (ok t)) + (map-permutations (lambda (s) + (unless (set-equal s seq) + (setf ok nil)) + (when (member s seen :test 'equal) + (setf ok nil)) + (push s seen)) + seq + :copy t) + (values ok (length seen))) + t + 6) + +(deftest proper-sequence.type.1 + (mapcar (lambda (x) + (typep x 'proper-sequence)) + (list (list 1 2 3) + (vector 1 2 3) + #2a((1 2) (3 4)) + (circular-list 1 2 3 4))) + (t t nil nil)) + +(deftest emptyp.1 + (mapcar #'emptyp + (list (list 1) + (circular-list 1) + nil + (vector) + (vector 1))) + (nil nil t t nil)) + +(deftest sequence-of-length-p.1 + (mapcar #'sequence-of-length-p + (list nil + #() + (list 1) + (vector 1) + (list 1 2) + (vector 1 2) + (list 1 2) + (vector 1 2) + (list 1 2) + (vector 1 2)) + (list 0 + 0 + 1 + 1 + 2 + 2 + 1 + 1 + 4 + 4)) + (t t t t t t nil nil nil nil)) + +(deftest length=.1 + (mapcar #'length= + (list nil + #() + (list 1) + (vector 1) + (list 1 2) + (vector 1 2) + (list 1 2) + (vector 1 2) + (list 1 2) + (vector 1 2)) + (list 0 + 0 + 1 + 1 + 2 + 2 + 1 + 1 + 4 + 4)) + (t t t t t t nil nil nil nil)) + +(deftest length=.2 + ;; test the compiler macro + (macrolet ((x (&rest args) + (funcall + (compile nil + `(lambda () + (length= ,@args)))))) + (list (x 2 '(1 2)) + (x '(1 2) '(3 4)) + (x '(1 2) 2) + (x '(1 2) 2 '(3 4)) + (x 1 2 3))) + (t t t t nil)) + +(deftest copy-sequence.1 + (let ((l (list 1 2 3)) + (v (vector #\a #\b #\c))) + (declare (notinline copy-sequence)) + (let ((l.list (copy-sequence 'list l)) + (l.vector (copy-sequence 'vector l)) + (l.spec-v (copy-sequence '(vector fixnum) l)) + (v.vector (copy-sequence 'vector v)) + (v.list (copy-sequence 'list v)) + (v.string (copy-sequence 'string v))) + (list (member l (list l.list l.vector l.spec-v)) + (member v (list v.vector v.list v.string)) + (equal l.list l) + (equalp l.vector #(1 2 3)) + (type= (upgraded-array-element-type 'fixnum) + (array-element-type l.spec-v)) + (equalp v.vector v) + (equal v.list '(#\a #\b #\c)) + (equal "abc" v.string)))) + (nil nil t t t t t t)) + +(deftest first-elt.1 + (mapcar #'first-elt + (list (list 1 2 3) + "abc" + (vector :a :b :c))) + (1 #\a :a)) + +(deftest first-elt.error.1 + (mapcar (lambda (x) + (handler-case + (first-elt x) + (type-error () + :type-error))) + (list nil + #() + 12 + :zot)) + (:type-error + :type-error + :type-error + :type-error)) + +(deftest setf-first-elt.1 + (let ((l (list 1 2 3)) + (s (copy-seq "foobar")) + (v (vector :a :b :c))) + (setf (first-elt l) -1 + (first-elt s) #\x + (first-elt v) 'zot) + (values l s v)) + (-1 2 3) + "xoobar" + #(zot :b :c)) + +(deftest setf-first-elt.error.1 + (let ((l 'foo)) + (multiple-value-bind (res err) + (ignore-errors (setf (first-elt l) 4)) + (typep err 'type-error))) + t) + +(deftest last-elt.1 + (mapcar #'last-elt + (list (list 1 2 3) + (vector :a :b :c) + "FOOBAR" + #*001 + #*010)) + (3 :c #\R 1 0)) + +(deftest last-elt.error.1 + (mapcar (lambda (x) + (handler-case + (last-elt x) + (type-error () + :type-error))) + (list nil + #() + 12 + :zot + (circular-list 1 2 3) + (list* 1 2 3 (circular-list 4 5)))) + (:type-error + :type-error + :type-error + :type-error + :type-error + :type-error)) + +(deftest setf-last-elt.1 + (let ((l (list 1 2 3)) + (s (copy-seq "foobar")) + (b (copy-seq #*010101001))) + (setf (last-elt l) '??? + (last-elt s) #\? + (last-elt b) 0) + (values l s b)) + (1 2 ???) + "fooba?" + #*010101000) + +(deftest setf-last-elt.error.1 + (handler-case + (setf (last-elt 'foo) 13) + (type-error () + :type-error)) + :type-error) + +(deftest starts-with.1 + (list (starts-with 1 '(1 2 3)) + (starts-with 1 #(1 2 3)) + (starts-with #\x "xyz") + (starts-with 2 '(1 2 3)) + (starts-with 3 #(1 2 3)) + (starts-with 1 1) + (starts-with nil nil)) + (t t t nil nil nil nil)) + +(deftest starts-with.2 + (values (starts-with 1 '(-1 2 3) :key '-) + (starts-with "foo" '("foo" "bar") :test 'equal) + (starts-with "f" '(#\f) :key 'string :test 'equal) + (starts-with -1 '(0 1 2) :key #'1+) + (starts-with "zot" '("ZOT") :test 'equal)) + t + t + t + nil + nil) + +(deftest ends-with.1 + (list (ends-with 3 '(1 2 3)) + (ends-with 3 #(1 2 3)) + (ends-with #\z "xyz") + (ends-with 2 '(1 2 3)) + (ends-with 1 #(1 2 3)) + (ends-with 1 1) + (ends-with nil nil)) + (t t t nil nil nil nil)) + +(deftest ends-with.2 + (values (ends-with 2 '(0 13 1) :key '1+) + (ends-with "foo" (vector "bar" "foo") :test 'equal) + (ends-with "X" (vector 1 2 #\X) :key 'string :test 'equal) + (ends-with "foo" "foo" :test 'equal)) + t + t + t + nil) + +(deftest ends-with.error.1 + (handler-case + (ends-with 3 (circular-list 3 3 3 1 3 3)) + (type-error () + :type-error)) + :type-error) + +(deftest sequences.passing-improper-lists + (macrolet ((signals-error-p (form) + `(handler-case + (progn ,form nil) + (type-error (e) + t))) + (cut (fn &rest args) + (with-gensyms (arg) + (print`(lambda (,arg) + (apply ,fn (list ,@(substitute arg '_ args)))))))) + (let ((circular-list (make-circular-list 5 :initial-element :foo)) + (dotted-list (list* 'a 'b 'c 'd))) + (loop for nth from 0 + for fn in (list + (cut #'lastcar _) + (cut #'rotate _ 3) + (cut #'rotate _ -3) + (cut #'shuffle _) + (cut #'random-elt _) + (cut #'last-elt _) + (cut #'ends-with :foo _)) + nconcing + (let ((on-circular-p (signals-error-p (funcall fn circular-list))) + (on-dotted-p (signals-error-p (funcall fn dotted-list)))) + (when (or (not on-circular-p) (not on-dotted-p)) + (append + (unless on-circular-p + (let ((*print-circle* t)) + (list + (format nil + "No appropriate error signalled when passing ~S to ~Ath entry." + circular-list nth)))) + (unless on-dotted-p + (list + (format nil + "No appropriate error signalled when passing ~S to ~Ath entry." + dotted-list nth))))))))) + nil) + +;;;; IO + +(deftest read-stream-content-into-string.1 + (values (with-input-from-string (stream "foo bar") + (read-stream-content-into-string stream)) + (with-input-from-string (stream "foo bar") + (read-stream-content-into-string stream :buffer-size 1)) + (with-input-from-string (stream "foo bar") + (read-stream-content-into-string stream :buffer-size 6)) + (with-input-from-string (stream "foo bar") + (read-stream-content-into-string stream :buffer-size 7))) + "foo bar" + "foo bar" + "foo bar" + "foo bar") + +(deftest read-stream-content-into-string.2 + (handler-case + (let ((stream (make-broadcast-stream))) + (read-stream-content-into-string stream :buffer-size 0)) + (type-error () + :type-error)) + :type-error) + +#+(or) +(defvar *octets* + (map '(simple-array (unsigned-byte 8) (7)) #'char-code "foo bar")) + +#+(or) +(deftest read-stream-content-into-byte-vector.1 + (values (with-input-from-byte-vector (stream *octets*) + (read-stream-content-into-byte-vector stream)) + (with-input-from-byte-vector (stream *octets*) + (read-stream-content-into-byte-vector stream :initial-size 1)) + (with-input-from-byte-vector (stream *octets*) + (read-stream-content-into-byte-vector stream 'alexandria::%length 6)) + (with-input-from-byte-vector (stream *octets*) + (read-stream-content-into-byte-vector stream 'alexandria::%length 3))) + *octets* + *octets* + *octets* + (subseq *octets* 0 3)) + +(deftest read-stream-content-into-byte-vector.2 + (handler-case + (let ((stream (make-broadcast-stream))) + (read-stream-content-into-byte-vector stream :initial-size 0)) + (type-error () + :type-error)) + :type-error) + +;;;; Macros + +(deftest with-unique-names.1 + (let ((*gensym-counter* 0)) + (let ((syms (with-unique-names (foo bar quux) + (list foo bar quux)))) + (list (find-if #'symbol-package syms) + (equal '("FOO0" "BAR1" "QUUX2") + (mapcar #'symbol-name syms))))) + (nil t)) + +(deftest with-unique-names.2 + (let ((*gensym-counter* 0)) + (let ((syms (with-unique-names ((foo "_foo_") (bar -bar-) (quux #\q)) + (list foo bar quux)))) + (list (find-if #'symbol-package syms) + (equal '("_foo_0" "-BAR-1" "q2") + (mapcar #'symbol-name syms))))) + (nil t)) + +(deftest with-unique-names.3 + (let ((*gensym-counter* 0)) + (multiple-value-bind (res err) + (ignore-errors + (eval + '(let ((syms + (with-unique-names ((foo "_foo_") (bar -bar-) (quux 42)) + (list foo bar quux)))) + (list (find-if #'symbol-package syms) + (equal '("_foo_0" "-BAR-1" "q2") + (mapcar #'symbol-name syms)))))) + (errorp err))) + t) + +(deftest once-only.1 + (macrolet ((cons1.good (x) + (once-only (x) + `(cons ,x ,x))) + (cons1.bad (x) + `(cons ,x ,x))) + (let ((y 0)) + (list (cons1.good (incf y)) + y + (cons1.bad (incf y)) + y))) + ((1 . 1) 1 (2 . 3) 3)) + +(deftest once-only.2 + (macrolet ((cons1 (x) + (once-only ((y x)) + `(cons ,y ,y)))) + (let ((z 0)) + (list (cons1 (incf z)) + z + (cons1 (incf z))))) + ((1 . 1) 1 (2 . 2))) + +(deftest parse-body.1 + (parse-body '("doc" "body") :documentation t) + ("body") + nil + "doc") + +(deftest parse-body.2 + (parse-body '("body") :documentation t) + ("body") + nil + nil) + +(deftest parse-body.3 + (parse-body '("doc" "body")) + ("doc" "body") + nil + nil) + +(deftest parse-body.4 + (parse-body '((declare (foo)) "doc" (declare (bar)) body) :documentation t) + (body) + ((declare (foo)) (declare (bar))) + "doc") + +(deftest parse-body.5 + (parse-body '((declare (foo)) "doc" (declare (bar)) body)) + ("doc" (declare (bar)) body) + ((declare (foo))) + nil) + +(deftest parse-body.6 + (multiple-value-bind (res err) + (ignore-errors + (parse-body '("foo" "bar" "quux") + :documentation t)) + (errorp err)) + t) + +;;;; Symbols + +(deftest ensure-symbol.1 + (ensure-symbol :cons :cl) + cons + :external) + +(deftest ensure-symbol.2 + (ensure-symbol "CONS" :alexandria) + cons + :inherited) + +(deftest ensure-symbol.3 + (ensure-symbol 'foo :keyword) + :foo + :external) + +(deftest ensure-symbol.4 + (ensure-symbol #\* :alexandria) + * + :inherited) + +(deftest format-symbol.1 + (let ((s (format-symbol nil '#:x-~d 13))) + (list (symbol-package s) + (string= (string '#:x-13) (symbol-name s)))) + (nil t)) + +(deftest format-symbol.2 + (format-symbol :keyword '#:sym-~a (string :bolic)) + :sym-bolic) + +(deftest format-symbol.3 + (let ((*package* (find-package :cl))) + (format-symbol t '#:find-~a (string 'package))) + find-package) + +(deftest make-keyword.1 + (list (make-keyword 'zot) + (make-keyword "FOO") + (make-keyword #\Q)) + (:zot :foo :q)) + +(deftest make-gensym-list.1 + (let ((*gensym-counter* 0)) + (let ((syms (make-gensym-list 3 "FOO"))) + (list (find-if 'symbol-package syms) + (equal '("FOO0" "FOO1" "FOO2") + (mapcar 'symbol-name syms))))) + (nil t)) + +(deftest make-gensym-list.2 + (let ((*gensym-counter* 0)) + (let ((syms (make-gensym-list 3))) + (list (find-if 'symbol-package syms) + (equal '("G0" "G1" "G2") + (mapcar 'symbol-name syms))))) + (nil t)) + +;;;; Type-system + +(deftest of-type.1 + (locally + (declare (notinline of-type)) + (let ((f (of-type 'string))) + (list (funcall f "foo") + (funcall f 'bar)))) + (t nil)) + +(deftest type=.1 + (type= 'string 'string) + t + t) + +(deftest type=.2 + (type= 'list '(or null cons)) + t + t) + +(deftest type=.3 + (type= 'null '(and symbol list)) + t + t) + +(deftest type=.4 + (type= 'string '(satisfies emptyp)) + nil + nil) + +(deftest type=.5 + (type= 'string 'list) + nil + t) + +(macrolet + ((test (type numbers) + `(deftest ,(format-symbol t '#:cdr5.~a (string type)) + (let ((numbers ,numbers)) + (values (mapcar (of-type ',(format-symbol t '#:negative-~a (string type))) numbers) + (mapcar (of-type ',(format-symbol t '#:non-positive-~a (string type))) numbers) + (mapcar (of-type ',(format-symbol t '#:non-negative-~a (string type))) numbers) + (mapcar (of-type ',(format-symbol t '#:positive-~a (string type))) numbers))) + (t t t nil nil nil nil) + (t t t t nil nil nil) + (nil nil nil t t t t) + (nil nil nil nil t t t)))) + (test fixnum (list most-negative-fixnum -42 -1 0 1 42 most-positive-fixnum)) + (test integer (list (1- most-negative-fixnum) -42 -1 0 1 42 (1+ most-positive-fixnum))) + (test rational (list (1- most-negative-fixnum) -42/13 -1 0 1 42/13 (1+ most-positive-fixnum))) + (test real (list most-negative-long-float -42/13 -1 0 1 42/13 most-positive-long-float)) + (test float (list most-negative-short-float -42.02 -1.0 0.0 1.0 42.02 most-positive-short-float)) + (test short-float (list most-negative-short-float -42.02s0 -1.0s0 0.0s0 1.0s0 42.02s0 most-positive-short-float)) + (test single-float (list most-negative-single-float -42.02f0 -1.0f0 0.0f0 1.0f0 42.02f0 most-positive-single-float)) + (test double-float (list most-negative-double-float -42.02d0 -1.0d0 0.0d0 1.0d0 42.02d0 most-positive-double-float)) + (test long-float (list most-negative-long-float -42.02l0 -1.0l0 0.0l0 1.0l0 42.02l0 most-positive-long-float))) + +;;;; Bindings + +(declaim (notinline opaque)) +(defun opaque (x) + x) + +(deftest if-let.1 + (if-let (x (opaque :ok)) + x + :bad) + :ok) + +(deftest if-let.2 + (if-let (x (opaque nil)) + :bad + (and (not x) :ok)) + :ok) + +(deftest if-let.3 + (let ((x 1)) + (if-let ((x 2) + (y x)) + (+ x y) + :oops)) + 3) + +(deftest if-let.4 + (if-let ((x 1) + (y nil)) + :oops + (and (not y) x)) + 1) + +(deftest if-let.5 + (if-let (x) + :oops + (not x)) + t) + +(deftest if-let.error.1 + (handler-case + (eval '(if-let x + :oops + :oops)) + (type-error () + :type-error)) + :type-error) + +(deftest when-let.1 + (when-let (x (opaque :ok)) + (setf x (cons x x)) + x) + (:ok . :ok)) + +(deftest when-let.2 + (when-let ((x 1) + (y nil) + (z 3)) + :oops) + nil) + +(deftest when-let.3 + (let ((x 1)) + (when-let ((x 2) + (y x)) + (+ x y))) + 3) + +(deftest when-let.error.1 + (handler-case + (eval '(when-let x :oops)) + (type-error () + :type-error)) + :type-error) + +(deftest when-let*.1 + (let ((x 1)) + (when-let* ((x 2) + (y x)) + (+ x y))) + 4) + +(deftest when-let*.2 + (let ((y 1)) + (when-let* (x y) + (1+ x))) + 2) + +(deftest when-let*.3 + (when-let* ((x t) + (y (consp x)) + (z (error "OOPS"))) + t) + nil) + +(deftest when-let*.error.1 + (handler-case + (eval '(when-let* x :oops)) + (type-error () + :type-error)) + :type-error) + +(deftest doplist.1 + (let (keys values) + (doplist (k v '(a 1 b 2 c 3) (values t (reverse keys) (reverse values) k v)) + (push k keys) + (push v values))) + t + (a b c) + (1 2 3) + nil + nil) + +(deftest count-permutations.1 + (values (count-permutations 31 7) + (count-permutations 1 1) + (count-permutations 2 1) + (count-permutations 2 2) + (count-permutations 3 2) + (count-permutations 3 1)) + 13253058000 + 1 + 2 + 2 + 6 + 3) + +(deftest binomial-coefficient.1 + (alexandria:binomial-coefficient 1239 139) + 28794902202288970200771694600561826718847179309929858835480006683522184441358211423695124921058123706380656375919763349913245306834194782172712255592710204598527867804110129489943080460154) + +;; Exercise bignum case (at least on x86). +(deftest binomial-coefficient.2 + (alexandria:binomial-coefficient 2000000000000 20) + 430998041177272843950422879590338454856322722740402365741730748431530623813012487773080486408378680853987520854296499536311275320016878730999689934464711239072435565454954447356845336730100919970769793030177499999999900000000000) + +(deftest copy-stream.1 + (let ((data "sdkfjhsakfh weior763495ewofhsdfk sdfadlkfjhsadf woif sdlkjfhslkdfh sdklfjh")) + (values (equal data + (with-input-from-string (in data) + (with-output-to-string (out) + (alexandria:copy-stream in out)))) + (equal (subseq data 10 20) + (with-input-from-string (in data) + (with-output-to-string (out) + (alexandria:copy-stream in out :start 10 :end 20)))) + (equal (subseq data 10) + (with-input-from-string (in data) + (with-output-to-string (out) + (alexandria:copy-stream in out :start 10)))) + (equal (subseq data 0 20) + (with-input-from-string (in data) + (with-output-to-string (out) + (alexandria:copy-stream in out :end 20)))))) + t + t + t + t) + +(deftest extremum.1 + (let ((n 0)) + (dotimes (i 10) + (let ((data (shuffle (coerce (iota 10000 :start i) 'vector))) + (ok t)) + (unless (eql i (extremum data #'<)) + (setf ok nil)) + (unless (eql i (extremum (coerce data 'list) #'<)) + (setf ok nil)) + (unless (eql (+ 9999 i) (extremum data #'>)) + (setf ok nil)) + (unless (eql (+ 9999 i) (extremum (coerce data 'list) #'>)) + (setf ok nil)) + (when ok + (incf n)))) + (when (eql 10 (extremum #(100 1 10 1000) #'> :start 1 :end 3)) + (incf n)) + (when (eql -1000 (extremum #(100 1 10 -1000) #'> :key 'abs)) + (incf n)) + (when (eq nil (extremum "" (lambda (a b) (error "wtf? ~S, ~S" a b)))) + (incf n)) + n) + 13) + +(deftest starts-with-subseq.string + (starts-with-subseq "f" "foo" :return-suffix t) + t + "oo") + +(deftest starts-with-subseq.vector + (starts-with-subseq #(1) #(1 2 3) :return-suffix t) + t + #(2 3)) + +(deftest starts-with-subseq.list + (starts-with-subseq '(1) '(1 2 3) :return-suffix t) + t + (2 3)) + +(deftest starts-with-subseq.start1 + (starts-with-subseq "foo" "oop" :start1 1) + t + nil) + +(deftest starts-with-subseq.start2 + (starts-with-subseq "foo" "xfoop" :start2 1) + t + nil) + +(deftest format-symbol.print-case-bound + (let ((upper (intern "FOO-BAR")) + (lower (intern "foo-bar")) + (*print-escape* nil)) + (values + (let ((*print-case* :downcase)) + (and (eq upper (format-symbol t "~A" upper)) + (eq lower (format-symbol t "~A" lower)))) + (let ((*print-case* :upcase)) + (and (eq upper (format-symbol t "~A" upper)) + (eq lower (format-symbol t "~A" lower)))) + (let ((*print-case* :capitalize)) + (and (eq upper (format-symbol t "~A" upper)) + (eq lower (format-symbol t "~A" lower)))))) + t + t + t) + +(deftest iota.fp-start-and-complex-integer-step + (equal '(#C(0.0 0.0) #C(0.0 2.0) #C(0.0 4.0)) + (iota 3 :start 0.0 :step #C(0 2))) + t) + +(deftest parse-ordinary-lambda-list.1 + (multiple-value-bind (req opt rest keys allowp aux keyp) + (parse-ordinary-lambda-list '(a b c + &optional o1 (o2 42) (o3 42 o3-supplied?) + &key (k1) ((:key k2)) (k3 42 k3-supplied?)) + :normalize t) + (and (equal '(a b c) req) + (equal '((o1 nil nil) + (o2 42 nil) + (o3 42 o3-supplied?)) + opt) + (equal '(((:k1 k1) nil nil) + ((:key k2) nil nil) + ((:k3 k3) 42 k3-supplied?)) + keys) + (not allowp) + (not aux) + (eq t keyp))) + t) diff --git a/third_party/lisp/alexandria/types.lisp b/third_party/lisp/alexandria/types.lisp new file mode 100644 index 000000000000..1942d0ecdf2a --- /dev/null +++ b/third_party/lisp/alexandria/types.lisp @@ -0,0 +1,137 @@ +(in-package :alexandria) + +(deftype array-index (&optional (length (1- array-dimension-limit))) + "Type designator for an index into array of LENGTH: an integer between +0 (inclusive) and LENGTH (exclusive). LENGTH defaults to one less than +ARRAY-DIMENSION-LIMIT." + `(integer 0 (,length))) + +(deftype array-length (&optional (length (1- array-dimension-limit))) + "Type designator for a dimension of an array of LENGTH: an integer between +0 (inclusive) and LENGTH (inclusive). LENGTH defaults to one less than +ARRAY-DIMENSION-LIMIT." + `(integer 0 ,length)) + +;; This MACROLET will generate most of CDR5 (http://cdr.eurolisp.org/document/5/) +;; except the RATIO related definitions and ARRAY-INDEX. +(macrolet + ((frob (type &optional (base-type type)) + (let ((subtype-names (list)) + (predicate-names (list))) + (flet ((make-subtype-name (format-control) + (let ((result (format-symbol :alexandria format-control + (symbol-name type)))) + (push result subtype-names) + result)) + (make-predicate-name (sybtype-name) + (let ((result (format-symbol :alexandria '#:~A-p + (symbol-name sybtype-name)))) + (push result predicate-names) + result)) + (make-docstring (range-beg range-end range-type) + (let ((inf (ecase range-type (:negative "-inf") (:positive "+inf")))) + (format nil "Type specifier denoting the ~(~A~) range from ~A to ~A." + type + (if (equal range-beg ''*) inf (ensure-car range-beg)) + (if (equal range-end ''*) inf (ensure-car range-end)))))) + (let* ((negative-name (make-subtype-name '#:negative-~a)) + (non-positive-name (make-subtype-name '#:non-positive-~a)) + (non-negative-name (make-subtype-name '#:non-negative-~a)) + (positive-name (make-subtype-name '#:positive-~a)) + (negative-p-name (make-predicate-name negative-name)) + (non-positive-p-name (make-predicate-name non-positive-name)) + (non-negative-p-name (make-predicate-name non-negative-name)) + (positive-p-name (make-predicate-name positive-name)) + (negative-extremum) + (positive-extremum) + (below-zero) + (above-zero) + (zero)) + (setf (values negative-extremum below-zero + above-zero positive-extremum zero) + (ecase type + (fixnum (values 'most-negative-fixnum -1 1 'most-positive-fixnum 0)) + (integer (values ''* -1 1 ''* 0)) + (rational (values ''* '(0) '(0) ''* 0)) + (real (values ''* '(0) '(0) ''* 0)) + (float (values ''* '(0.0E0) '(0.0E0) ''* 0.0E0)) + (short-float (values ''* '(0.0S0) '(0.0S0) ''* 0.0S0)) + (single-float (values ''* '(0.0F0) '(0.0F0) ''* 0.0F0)) + (double-float (values ''* '(0.0D0) '(0.0D0) ''* 0.0D0)) + (long-float (values ''* '(0.0L0) '(0.0L0) ''* 0.0L0)))) + `(progn + (deftype ,negative-name () + ,(make-docstring negative-extremum below-zero :negative) + `(,',base-type ,,negative-extremum ,',below-zero)) + + (deftype ,non-positive-name () + ,(make-docstring negative-extremum zero :negative) + `(,',base-type ,,negative-extremum ,',zero)) + + (deftype ,non-negative-name () + ,(make-docstring zero positive-extremum :positive) + `(,',base-type ,',zero ,,positive-extremum)) + + (deftype ,positive-name () + ,(make-docstring above-zero positive-extremum :positive) + `(,',base-type ,',above-zero ,,positive-extremum)) + + (declaim (inline ,@predicate-names)) + + (defun ,negative-p-name (n) + (and (typep n ',type) + (< n ,zero))) + + (defun ,non-positive-p-name (n) + (and (typep n ',type) + (<= n ,zero))) + + (defun ,non-negative-p-name (n) + (and (typep n ',type) + (<= ,zero n))) + + (defun ,positive-p-name (n) + (and (typep n ',type) + (< ,zero n))))))))) + (frob fixnum integer) + (frob integer) + (frob rational) + (frob real) + (frob float) + (frob short-float) + (frob single-float) + (frob double-float) + (frob long-float)) + +(defun of-type (type) + "Returns a function of one argument, which returns true when its argument is +of TYPE." + (lambda (thing) (typep thing type))) + +(define-compiler-macro of-type (&whole form type &environment env) + ;; This can yeild a big benefit, but no point inlining the function + ;; all over the place if TYPE is not constant. + (if (constantp type env) + (with-gensyms (thing) + `(lambda (,thing) + (typep ,thing ,type))) + form)) + +(declaim (inline type=)) +(defun type= (type1 type2) + "Returns a primary value of T is TYPE1 and TYPE2 are the same type, +and a secondary value that is true is the type equality could be reliably +determined: primary value of NIL and secondary value of T indicates that the +types are not equivalent." + (multiple-value-bind (sub ok) (subtypep type1 type2) + (cond ((and ok sub) + (subtypep type2 type1)) + (ok + (values nil ok)) + (t + (multiple-value-bind (sub ok) (subtypep type2 type1) + (declare (ignore sub)) + (values nil ok)))))) + +(define-modify-macro coercef (type-spec) coerce + "Modify-macro for COERCE.") diff --git a/third_party/lisp/anaphora.nix b/third_party/lisp/anaphora.nix new file mode 100644 index 000000000000..d2356f7b05ea --- /dev/null +++ b/third_party/lisp/anaphora.nix @@ -0,0 +1,18 @@ +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "tokenrove"; + repo = "anaphora"; + rev = "018590df36ffb30ece561fb28ea6521363efc6e2"; + sha256 = "0pq6y5swvrjd0kjs2dl2648s13s0pzxin0chrq35jam8jrci3kd1"; + }; +in depot.nix.buildLisp.library { + name = "anaphora"; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "early.lisp" + "symbolic.lisp" + "anaphora.lisp" + ]; +} diff --git a/third_party/lisp/asdf-flv/.gitattributes b/third_party/lisp/asdf-flv/.gitattributes new file mode 100644 index 000000000000..2b45716e4709 --- /dev/null +++ b/third_party/lisp/asdf-flv/.gitattributes @@ -0,0 +1,2 @@ +.gitignore export-ignore +.gitattributes export-ignore diff --git a/third_party/lisp/asdf-flv/.gitignore b/third_party/lisp/asdf-flv/.gitignore new file mode 100644 index 000000000000..bdf4ad2ae6dd --- /dev/null +++ b/third_party/lisp/asdf-flv/.gitignore @@ -0,0 +1,3 @@ +sbcl-*/ +cmu-*/ +openmcl-*/ diff --git a/third_party/lisp/asdf-flv/Makefile b/third_party/lisp/asdf-flv/Makefile new file mode 100644 index 000000000000..b4c74feefe82 --- /dev/null +++ b/third_party/lisp/asdf-flv/Makefile @@ -0,0 +1,77 @@ +### Makefile --- Toplevel directory + +## Copyright (C) 2011, 2015 Didier Verna + +## Author: Didier Verna <didier@didierverna.net> + +## This file is part of ASDF-FLV. + +## Copying and distribution of this file, with or without modification, +## are permitted in any medium without royalty provided the copyright +## notice and this notice are preserved. This file is offered as-is, +## without any warranty. + + +### Commentary: + +## Contents management by FCM version 0.1. + + +### Code: + +PROJECT := asdf-flv +VERSION := 2.1 + +W3DIR := $(HOME)/www/software/lisp/$(PROJECT) + +DIST_NAME := $(PROJECT)-$(VERSION) +TARBALL := $(DIST_NAME).tar.gz +SIGNATURE := $(TARBALL).asc + + +all: + +clean: + -rm *~ + +distclean: clean + -rm *.tar.gz *.tar.gz.asc + +tag: + git tag -a -m 'Version $(VERSION)' 'version-$(VERSION)' + +tar: $(TARBALL) +gpg: $(SIGNATURE) +dist: tar gpg + +install-www: dist + -install -m 644 $(TARBALL) "$(W3DIR)/attic/" + -install -m 644 $(SIGNATURE) "$(W3DIR)/attic/" + echo "\ +<? lref (\"$(PROJECT)/attic/$(PROJECT)-$(VERSION).tar.gz\", \ + contents (\"Dernire version\", \"Latest version\")); ?> \ +| \ +<? lref (\"$(PROJECT)/attic/$(PROJECT)-$(VERSION).tar.gz.asc\", \ + contents (\"Signature GPG\", \"GPG Signature\")); ?>" \ + > "$(W3DIR)/latest.txt" + chmod 644 "$(W3DIR)/latest.txt" + cd "$(W3DIR)" \ + && ln -fs attic/$(TARBALL) latest.tar.gz \ + && ln -fs attic/$(SIGNATURE) latest.tar.gz.asc + +update-version: + perl -pi -e 's/:version ".*"/:version "$(VERSION)"/' \ + net.didierverna.$(PROJECT).asd + +$(TARBALL): + git archive --format=tar --prefix=$(DIST_NAME)/ \ + --worktree-attributes HEAD \ + | gzip -c > $@ + +$(SIGNATURE): $(TARBALL) + gpg -b -a $< + + +.PHONY: all clean distclean tag tar gpg dist install-www update-version + +### Makefile ends here diff --git a/third_party/lisp/asdf-flv/README.md b/third_party/lisp/asdf-flv/README.md new file mode 100644 index 000000000000..7ccdd1888163 --- /dev/null +++ b/third_party/lisp/asdf-flv/README.md @@ -0,0 +1,7 @@ +ASDF-FLV provides support for file-local variables through ASDF. A file-local +variable behaves like `*PACKAGE*` and `*READTABLE*` with respect to `LOAD` and +`COMPILE-FILE`: a new dynamic binding is created before processing the file, +so that any modification to the variable essentially becomes file-local. + +In order to make one or several variables file-local, use the macros +`SET-FILE-LOCAL-VARIABLE(S)`. diff --git a/third_party/lisp/asdf-flv/asdf-flv.lisp b/third_party/lisp/asdf-flv/asdf-flv.lisp new file mode 100644 index 000000000000..76c6845b82b3 --- /dev/null +++ b/third_party/lisp/asdf-flv/asdf-flv.lisp @@ -0,0 +1,64 @@ +;;; asdf-flv.lisp --- Implementation + +;; Copyright (C) 2011, 2015 Didier Verna + +;; Author: Didier Verna <didier@didierverna.net> + +;; This file is part of ASDF-FLV. + +;; Copying and distribution of this file, with or without modification, +;; are permitted in any medium without royalty provided the copyright +;; notice and this notice are preserved. This file is offered as-is, +;; without any warranty. + + +;;; Commentary: + +;; Contents management by FCM version 0.1. + + +;;; Code: + +(in-package :net.didierverna.asdf-flv) + + +(defvar *file-local-variables* () + "List of file-local special variables.") + + +(defun make-variable-file-local (symbol) + "Make special variable named by SYMBOL have a file-local value." + (pushnew symbol *file-local-variables*)) + +(defmacro set-file-local-variable (symbol) + "Set special variable named by SYMBOL as file-local. +SYMBOL need not be quoted." + `(make-variable-file-local ',symbol)) + +(defun make-variables-file-local (&rest symbols) + "Make special variables named by SYMBOLS have a file-local value." + (dolist (symbol symbols) + (pushnew symbol *file-local-variables*))) + +(defmacro set-file-local-variables (&rest symbols) + "Set special variables named by SYMBOLS as file-local. +SYMBOLS need not be quoted." + `(make-variables-file-local ,@(mapcar (lambda (symbol) (list 'quote symbol)) + symbols))) + + +(defmethod asdf:perform :around + ((operation asdf:load-op) (file asdf:cl-source-file)) + "Establish new dynamic bindings for file-local variables." + (progv *file-local-variables* + (mapcar #'symbol-value *file-local-variables*) + (call-next-method))) + +(defmethod asdf:perform :around + ((operation asdf:compile-op) (file asdf:cl-source-file)) + "Establish new dynamic bindings for file-local variables." + (progv *file-local-variables* + (mapcar #'symbol-value *file-local-variables*) + (call-next-method))) + +;;; asdf-flv.lisp ends here diff --git a/third_party/lisp/asdf-flv/default.nix b/third_party/lisp/asdf-flv/default.nix new file mode 100644 index 000000000000..e8ec4aa8f85c --- /dev/null +++ b/third_party/lisp/asdf-flv/default.nix @@ -0,0 +1,13 @@ +# Imported from https://github.com/didierverna/asdf-flv +{ depot, ... }: + +with depot.nix; +buildLisp.library { + name = "asdf-flv"; + deps = [ (buildLisp.bundled "asdf") ]; + + srcs = [ + ./package.lisp + ./asdf-flv.lisp + ]; +} diff --git a/third_party/lisp/asdf-flv/net.didierverna.asdf-flv.asd b/third_party/lisp/asdf-flv/net.didierverna.asdf-flv.asd new file mode 100644 index 000000000000..41202746d019 --- /dev/null +++ b/third_party/lisp/asdf-flv/net.didierverna.asdf-flv.asd @@ -0,0 +1,43 @@ +;;; net.didierverna.asdf-flv.asd --- ASDF system definition + +;; Copyright (C) 2011, 2015 Didier Verna + +;; Author: Didier Verna <didier@didierverna.net> + +;; This file is part of ASDF-FLV. + +;; Copying and distribution of this file, with or without modification, +;; are permitted in any medium without royalty provided the copyright +;; notice and this notice are preserved. This file is offered as-is, +;; without any warranty. + + +;;; Commentary: + +;; Contents management by FCM version 0.1. + + +;;; Code: + +(asdf:defsystem :net.didierverna.asdf-flv + :long-name "ASDF File Local Variables" + :description "ASDF extension to provide support for file-local variables." + :long-description "\ +ASDF-FLV provides support for file-local variables through ASDF. A file-local +variable behaves like *PACKAGE* and *READTABLE* with respect to LOAD and +COMPILE-FILE: a new dynamic binding is created before processing the file, so +that any modification to the variable becomes essentially file-local. + +In order to make one or several variables file-local, use the macros +SET-FILE-LOCAL-VARIABLE(S)." + :author "Didier Verna" + :mailto "didier@didierverna.net" + :homepage "http://www.lrde.epita.fr/~didier/software/lisp/misc.php#asdf-flv" + :source-control "https://github.com/didierverna/asdf-flv" + :license "GNU All Permissive" + :version "2.1" + :serial t + :components ((:file "package") + (:file "asdf-flv"))) + +;;; net.didierverna.asdf-flv.asd ends here diff --git a/third_party/lisp/asdf-flv/package.lisp b/third_party/lisp/asdf-flv/package.lisp new file mode 100644 index 000000000000..1d7fb2bab43d --- /dev/null +++ b/third_party/lisp/asdf-flv/package.lisp @@ -0,0 +1,28 @@ +;;; package.lisp --- Package definition + +;; Copyright (C) 2011, 2015 Didier Verna + +;; Author: Didier Verna <didier@didierverna.net> + +;; This file is part of ASDF-FLV. + +;; Copying and distribution of this file, with or without modification, +;; are permitted in any medium without royalty provided the copyright +;; notice and this notice are preserved. This file is offered as-is, +;; without any warranty. + + +;;; Commentary: + +;; Contents management by FCM version 0.1. + + +;;; Code: + +(in-package :cl-user) + +(defpackage :net.didierverna.asdf-flv + (:use :cl) + (:export :set-file-local-variable :set-file-local-variables)) + +;;; package.lisp ends here diff --git a/third_party/lisp/babel.nix b/third_party/lisp/babel.nix new file mode 100644 index 000000000000..c4a49e833f76 --- /dev/null +++ b/third_party/lisp/babel.nix @@ -0,0 +1,34 @@ +# Babel is an encoding conversion library for Common Lisp. +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/cl-babel/babel.git"; + rev = "f892d0587c7f3a1e6c0899425921b48008c29ee3"; # 2020-07-19 +}; +in depot.nix.buildLisp.library { + name = "babel"; + deps = [ + depot.third_party.lisp.alexandria + depot.third_party.lisp.trivial-features + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "packages.lisp" + "encodings.lisp" + "enc-ascii.lisp" + "enc-ebcdic.lisp" + "enc-ebcdic-int.lisp" + "enc-iso-8859.lisp" + "enc-unicode.lisp" + "enc-cp1251.lisp" + "enc-cp1252.lisp" + "jpn-table.lisp" + "enc-jpn.lisp" + "enc-gbk.lisp" + "enc-koi8.lisp" + "external-format.lisp" + "strings.lisp" + "gbk-map.lisp" + "sharp-backslash.lisp" + ]; +} diff --git a/third_party/lisp/bordeaux-threads.nix b/third_party/lisp/bordeaux-threads.nix new file mode 100644 index 000000000000..92bc1f2629c4 --- /dev/null +++ b/third_party/lisp/bordeaux-threads.nix @@ -0,0 +1,26 @@ +# This library is meant to make writing portable multi-threaded apps +# in Common Lisp simple. +{ depot, ... }: + +let + src = builtins.fetchGit { + url = "https://github.com/sionescu/bordeaux-threads.git"; + rev = "499b6d3f0ce635417d6096acf0a671d8bf3f6e5f"; + }; + getSrc = f: "${src}/src/${f}"; +in depot.nix.buildLisp.library { + name = "bordeaux-threads"; + deps = [ depot.third_party.lisp.alexandria ]; + + srcs = map getSrc [ + "pkgdcl.lisp" + "bordeaux-threads.lisp" + ] ++ [ + { + sbcl = getSrc "impl-sbcl.lisp"; + ecl = getSrc "impl-ecl.lisp"; + } + ] ++ map getSrc [ + "default-implementations.lisp" + ]; +} diff --git a/third_party/lisp/cffi.nix b/third_party/lisp/cffi.nix new file mode 100644 index 000000000000..c8d240c8cd0e --- /dev/null +++ b/third_party/lisp/cffi.nix @@ -0,0 +1,37 @@ +# CFFI purports to be the Common Foreign Function Interface. +{ depot, ... }: + +with depot.nix; +let src = builtins.fetchGit { + url = "https://github.com/cffi/cffi.git"; + rev = "a49ff36a95cb62ffa6cb069d98378d665769926b"; +}; +in buildLisp.library { + name = "cffi"; + deps = with depot.third_party.lisp; [ + alexandria + babel + trivial-features + (buildLisp.bundled "asdf") + ]; + + srcs = [ + { + ecl = src + "/src/cffi-ecl.lisp"; + sbcl = src + "/src/cffi-sbcl.lisp"; + ccl = src + "/src/cffi-openmcl.lisp"; + } + ] ++ map (f: src + ("/src/" + f)) [ + "package.lisp" + "utils.lisp" + "libraries.lisp" + "early-types.lisp" + "types.lisp" + "enum.lisp" + "strings.lisp" + "structures.lisp" + "functions.lisp" + "foreign-vars.lisp" + "features.lisp" + ]; +} diff --git a/third_party/lisp/checkl.nix b/third_party/lisp/checkl.nix new file mode 100644 index 000000000000..4fb92cb37937 --- /dev/null +++ b/third_party/lisp/checkl.nix @@ -0,0 +1,26 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.nix.buildLisp) bundled; + + src = pkgs.fetchFromGitHub { + owner = "rpav"; + repo = "CheckL"; + rev = "80328800d047fef9b6e32dfe6bdc98396aee3cc9"; + sha256 = "0bpisihx1gay44xmyr1dmhlwh00j0zzi04rp9fy35i95l2r4xdlx"; + }; + +in depot.nix.buildLisp.library { + name = "checkl"; + deps = with depot.third_party.lisp; [ + (bundled "asdf") + marshal + fiveam + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "checkl.lisp" + "formalize.lisp" + ]; +} diff --git a/third_party/lisp/chipz.nix b/third_party/lisp/chipz.nix new file mode 100644 index 000000000000..2a68d3b1f8e9 --- /dev/null +++ b/third_party/lisp/chipz.nix @@ -0,0 +1,31 @@ +# Common Lisp library for decompressing deflate, zlib, gzip, and bzip2 data +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "froydnj"; + repo = "chipz"; + rev = "75dfbc660a5a28161c57f115adf74c8a926bfc4d"; + sha256 = "0plx4rs39zbs4gjk77h4a2q11zpy75fh9v8hnxrvsf8fnakajhwg"; +}; +in depot.nix.buildLisp.library { + name = "chipz"; + deps = [ (depot.nix.buildLisp.bundled "asdf") ]; + + srcs = map (f: src + ("/" + f)) [ + "chipz.asd" + "package.lisp" + "constants.lisp" + "conditions.lisp" + "dstate.lisp" + "types-and-tables.lisp" + "crc32.lisp" + "adler32.lisp" + "inflate-state.lisp" + "gzip.lisp" + "zlib.lisp" + "inflate.lisp" + "bzip2.lisp" + "decompress.lisp" + "stream.lisp" + ]; +} diff --git a/third_party/lisp/chunga.nix b/third_party/lisp/chunga.nix new file mode 100644 index 000000000000..d40208063372 --- /dev/null +++ b/third_party/lisp/chunga.nix @@ -0,0 +1,27 @@ +# Portable chunked streams for Common Lisp +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "edicl"; + repo = "chunga"; + rev = "16330852d01dfde4dd97dee7cd985a88ea571e7e"; + sha256 = "0jzn3nyb3f22gm983rfk99smqs3mhb9ivjmasvhq9qla5cl9pyhd"; +}; +in depot.nix.buildLisp.library { + name = "chunga"; + deps = with depot.third_party.lisp; [ + trivial-gray-streams + ]; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "specials.lisp" + "util.lisp" + "known-words.lisp" + "conditions.lisp" + "read.lisp" + "streams.lisp" + "input.lisp" + "output.lisp" + ]; +} diff --git a/third_party/lisp/cl-ansi-text.nix b/third_party/lisp/cl-ansi-text.nix new file mode 100644 index 000000000000..5c01e023265a --- /dev/null +++ b/third_party/lisp/cl-ansi-text.nix @@ -0,0 +1,19 @@ +# Enables ANSI colors for printing. +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/pnathan/cl-ansi-text.git"; + rev = "257a5f19a2dc92d22f8fd772c0a78923b99b36a8"; +}; +in depot.nix.buildLisp.library { + name = "cl-ansi-text"; + deps = with depot.third_party.lisp; [ + alexandria + cl-colors2 + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "cl-ansi-text.lisp" + "define-colors.lisp" + ]; +} diff --git a/third_party/lisp/cl-base64.nix b/third_party/lisp/cl-base64.nix new file mode 100644 index 000000000000..1152601a81d3 --- /dev/null +++ b/third_party/lisp/cl-base64.nix @@ -0,0 +1,17 @@ +# Base64 encoding for Common Lisp +{ depot, ... }: + +let src = builtins.fetchGit { + url = "http://git.kpe.io/cl-base64.git"; + rev = "fc62a5342445d4ec1dd44e95f7dc513473a8c89a"; +}; +in depot.nix.buildLisp.library { + name = "cl-base64"; + srcs = [ + (src + "/package.lisp") + (src + "/encode.lisp") + (src + "/decode.lisp") + ]; +} + + diff --git a/third_party/lisp/cl-colors2.nix b/third_party/lisp/cl-colors2.nix new file mode 100644 index 000000000000..c90b8eae0118 --- /dev/null +++ b/third_party/lisp/cl-colors2.nix @@ -0,0 +1,21 @@ + +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://notabug.org/cage/cl-colors2.git"; + rev = "795aedee593b095fecde574bd999b520dd03ed24"; +}; +in depot.nix.buildLisp.library { + name = "cl-colors2"; + deps = with depot.third_party.lisp; [ + alexandria + cl-ppcre + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "colors.lisp" + "colornames.lisp" + "hexcolors.lisp" + ]; +} diff --git a/third_party/lisp/cl-fad.nix b/third_party/lisp/cl-fad.nix new file mode 100644 index 000000000000..905e0821cac7 --- /dev/null +++ b/third_party/lisp/cl-fad.nix @@ -0,0 +1,32 @@ +# Portable pathname library +{ depot, pkgs, ...}: + +with depot.nix; + +let src = pkgs.fetchFromGitHub { + owner = "edicl"; + repo = "cl-fad"; + rev = "13cbffe08fc660041359302f4057f8fc20d09402"; # 2021-01-10 + sha256 = "049laj8an6g9bh0m0cn0bxhq313p8qq1h37cil15l66147ad8slc"; +}; +in buildLisp.library { + name = "cl-fad"; + + deps = with depot.third_party.lisp; [ + alexandria + bordeaux-threads + { + sbcl = buildLisp.bundled "sb-posix"; + } + ]; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + ] ++ [ + { ccl = "${src}/openmcl.lisp"; } + ] ++ map (f: src + ("/" + f)) [ + "fad.lisp" + "path.lisp" + "temporary-files.lisp" + ]; +} diff --git a/third_party/lisp/cl-json.nix b/third_party/lisp/cl-json.nix new file mode 100644 index 000000000000..5d1450a3e9a1 --- /dev/null +++ b/third_party/lisp/cl-json.nix @@ -0,0 +1,28 @@ +# JSON encoder & decoder +{ depot, pkgs, ... }: + +let + inherit (depot.nix) buildLisp; + + src = pkgs.fetchFromGitHub { + owner = "hankhero"; + repo = "cl-json"; + rev = "6dfebb9540bfc3cc33582d0c03c9ec27cb913e79"; + sha256 = "0fx3m3x3s5ji950yzpazz4s0img3l6b3d6l3jrfjv0lr702496lh"; + }; +in buildLisp.library { + name = "cl-json"; + deps = [ (buildLisp.bundled "asdf") ]; + + srcs = [ "${src}/cl-json.asd" ] ++ + (map (f: src + ("/src/" + f)) [ + "package.lisp" + "common.lisp" + "objects.lisp" + "camel-case.lisp" + "decoder.lisp" + "encoder.lisp" + "utils.lisp" + "json-rpc.lisp" + ]); +} diff --git a/third_party/lisp/cl-plus-ssl.nix b/third_party/lisp/cl-plus-ssl.nix new file mode 100644 index 000000000000..e6a616b95e46 --- /dev/null +++ b/third_party/lisp/cl-plus-ssl.nix @@ -0,0 +1,47 @@ +# Common Lisp bindings to OpenSSL +{ depot, pkgs, ... }: + +with depot.nix; + +let src = builtins.fetchGit { + url = "https://github.com/cl-plus-ssl/cl-plus-ssl.git"; + rev = "29081992f6d7b4e3aa2c5eeece4cd92b745071f4"; +}; +in buildLisp.library { + name = "cl-plus-ssl"; + deps = with depot.third_party.lisp; [ + alexandria + bordeaux-threads + cffi + flexi-streams + trivial-features + trivial-garbage + trivial-gray-streams + { + scbl = buildLisp.bundled "uiop"; + default = buildLisp.bundled "asdf"; + } + { sbcl = buildLisp.bundled "sb-posix"; } + ]; + + native = [ pkgs.openssl ]; + + srcs = map (f: src + ("/src/" + f)) [ + "package.lisp" + "reload.lisp" + "conditions.lisp" + "ffi.lisp" + "x509.lisp" + "ffi-buffer-all.lisp" + "ffi-buffer.lisp" + "streams.lisp" + "bio.lisp" + "random.lisp" + "context.lisp" + "verify-hostname.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/cl-ppcre.nix b/third_party/lisp/cl-ppcre.nix new file mode 100644 index 000000000000..1dc9eb553118 --- /dev/null +++ b/third_party/lisp/cl-ppcre.nix @@ -0,0 +1,30 @@ +# cl-ppcre is a Common Lisp regular expression library. +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/edicl/cl-ppcre"; + rev = "1ca0cd9ca0d161acd49c463d6cb5fff897596e2f"; +}; +in depot.nix.buildLisp.library { + name = "cl-ppcre"; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "specials.lisp" + "util.lisp" + "errors.lisp" + "charset.lisp" + "charmap.lisp" + "chartest.lisp" + "lexer.lisp" + "parser.lisp" + "regex-class.lisp" + "regex-class-util.lisp" + "convert.lisp" + "optimize.lisp" + "closures.lisp" + "repetition-closures.lisp" + "scanner.lisp" + "api.lisp" + ]; +} diff --git a/third_party/lisp/cl-prevalence.nix b/third_party/lisp/cl-prevalence.nix new file mode 100644 index 000000000000..f793136194cb --- /dev/null +++ b/third_party/lisp/cl-prevalence.nix @@ -0,0 +1,30 @@ +# cl-prevalence is an implementation of object prevalence for CL (i.e. +# an in-memory database) +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "40ants"; + repo = "cl-prevalence"; + rev = "b1f90a525f37be0335a8761051fa5661aa74b696"; + sha256 = "1svw58pp7jxb9l08cgnqxf0cf8qa9qsb0z2fnv86a51z7pfz4c0g"; +}; +in depot.nix.buildLisp.library { + name = "cl-prevalence"; + + deps = with depot.third_party.lisp; [ + bordeaux-threads + s-xml + s-sysdeps + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "package.lisp" + "serialization/serialization.lisp" + "serialization/xml.lisp" + "serialization/sexp.lisp" + "prevalence.lisp" + "managed-prevalence.lisp" + "master-slave.lisp" + "blob.lisp" + ]; +} diff --git a/third_party/lisp/cl-smtp.nix b/third_party/lisp/cl-smtp.nix new file mode 100644 index 000000000000..a9905b5ef6f5 --- /dev/null +++ b/third_party/lisp/cl-smtp.nix @@ -0,0 +1,32 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitLab { + domain = "gitlab.common-lisp.net"; + owner = "cl-smtp"; + repo = "cl-smtp"; + rev = "ed47d326fad867ee11323fa3a0f307b5d40e8f2b"; + sha256 = "0vjjfapcrdc5671jz2d24h8zvpz7skq1x6pi9fvw6ls5sgms6fr0"; + }; + +in depot.nix.buildLisp.library { + name = "cl-smtp"; + deps = with depot.third_party.lisp; [ + usocket + trivial-gray-streams + flexi-streams + cl-base64 + cl-plus-ssl + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "attachments.lisp" + "cl-smtp.lisp" + "mime-types.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/cl-unicode.nix b/third_party/lisp/cl-unicode.nix new file mode 100644 index 000000000000..5fff1fbe6bb2 --- /dev/null +++ b/third_party/lisp/cl-unicode.nix @@ -0,0 +1,80 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs) sbcl runCommand writeText; + inherit (depot.nix.buildLisp) bundled; + + src = pkgs.fetchFromGitHub { + owner = "edicl"; + repo = "cl-unicode"; + rev = "8073fc5634c9d4802888ac03abf11dfe383e16fa"; + sha256 = "0ykx2s9lqfl74p1px0ik3l2izd1fc9jd1b4ra68s5x34rvjy0hza"; + }; + + cl-unicode-base = depot.nix.buildLisp.library { + name = "cl-unicode-base"; + deps = with depot.third_party.lisp; [ + cl-ppcre + ]; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "specials.lisp" + "util.lisp" + ]; + }; + + cl-unicode-build = depot.nix.buildLisp.program { + name = "cl-unicode-build"; + deps = with depot.third_party.lisp; [ + cl-unicode-base + flexi-streams + { + ecl = bundled "asdf"; + default = bundled "uiop"; + } + ]; + + srcs = (map (f: src + ("/build/" + f)) [ + "util.lisp" + "char-info.lisp" + "read.lisp" + ]) ++ [ + (runCommand "dump.lisp" {} '' + substitute ${src}/build/dump.lisp $out \ + --replace ':defaults *this-file*' ":defaults (uiop:getcwd)" + '') + + (writeText "export-create-source-files.lisp" '' + (in-package :cl-unicode) + (export 'create-source-files) + '') + ]; + + main = "cl-unicode:create-source-files"; + }; + + + generated = runCommand "cl-unicode-generated" {} '' + mkdir -p $out/build + mkdir -p $out/test + cd $out/build + pwd + ${cl-unicode-build}/bin/cl-unicode-build + ''; + +in +depot.nix.buildLisp.library { + name = "cl-unicode"; + deps = [cl-unicode-base]; + srcs = [ + "${src}/conditions.lisp" + "${generated}/lists.lisp" + "${generated}/hash-tables.lisp" + "${src}/api.lisp" + "${generated}/methods.lisp" + "${src}/test-functions.lisp" + "${src}/derived.lisp" + "${src}/alias.lisp" + ]; +} diff --git a/third_party/lisp/cl-who.nix b/third_party/lisp/cl-who.nix new file mode 100644 index 000000000000..50e4e68c0348 --- /dev/null +++ b/third_party/lisp/cl-who.nix @@ -0,0 +1,21 @@ +{ depot, pkgs, ... }: + +let + + src = pkgs.fetchFromGitHub { + owner = "edicl"; + repo = "cl-who"; + rev = "0d3826475133271ee8c590937136c1bc41b8cbe0"; + sha256 = "0sc8nji9q1df04lhsiwsjy1a35996bibl31w5hp5sh8q6sa122dy"; + }; + +in depot.nix.buildLisp.library { + name = "cl-who"; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "specials.lisp" + "util.lisp" + "who.lisp" + ]; +} diff --git a/third_party/lisp/cl-yacc.nix b/third_party/lisp/cl-yacc.nix new file mode 100644 index 000000000000..d2ceb81103e2 --- /dev/null +++ b/third_party/lisp/cl-yacc.nix @@ -0,0 +1,15 @@ +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "jech"; + repo = "cl-yacc"; + rev = "1334f5469251ffb3f8738a682dc8ee646cb26635"; + sha256 = "16946pzf8vvadnyfayvj8rbh4zjzw90h0azz2qk1mxrvhh5wklib"; + }; +in depot.nix.buildLisp.library { + name = "cl-yacc"; + + srcs = map (f: src + ("/" + f)) [ + "yacc.lisp" + ]; +} diff --git a/third_party/lisp/closer-mop.nix b/third_party/lisp/closer-mop.nix new file mode 100644 index 000000000000..21fb3ba14bef --- /dev/null +++ b/third_party/lisp/closer-mop.nix @@ -0,0 +1,24 @@ +# Closer to MOP is a compatibility layer that rectifies many of the +# absent or incorrect CLOS MOP features across a broad range of Common +# Lisp implementations +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "pcostanza"; + repo = "closer-mop"; + rev = "8ec9577029b08ade5978236121c9ac276f78d8be"; # 2021-07-30 + sha256 = "0dm8xsa3hzpxjd7x248pbzd8blw01a8ls7spalzgbg1g7vbn6zg5"; +}; +in depot.nix.buildLisp.library { + name = "closer-mop"; + + srcs = [ + "${src}/closer-mop-packages.lisp" + "${src}/closer-mop-shared.lisp" + { + sbcl = "${src}/closer-sbcl.lisp"; + ecl = "${src}/closer-ecl.lisp"; + ccl = "${src}/closer-clozure.lisp"; + } + ]; +} diff --git a/third_party/lisp/closure-common.nix b/third_party/lisp/closure-common.nix new file mode 100644 index 000000000000..591620e48705 --- /dev/null +++ b/third_party/lisp/closure-common.nix @@ -0,0 +1,43 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "sharplispers"; + repo = "closure-common"; + rev = "e3c5f5f454b72b01b89115e581c3c52a7e201e5c"; # 2018-09-09 + sha256 = "0k5r2qxn122pxi301ijir3nayi9sg4d7yiy276l36qmzwhp4mg5n"; + }; + + getSrcs = builtins.map (p: "${src}/${p}"); +in + +depot.nix.buildLisp.library { + name = "closure-common"; + + # closure-common.asd surpresses some warnings otherwise breaking + # compilation. Feature macros across implementations: + # + # ECL #+rune-is-character #-rune-is-integer #-x&y-streams-are-stream + # CCL #+rune-is-character #-rune-is-integer #-x&y-streams-are-stream + # SBCL #+rune-is-character #-rune-is-integer #-x&y-streams-are-stream + # + # Since all implementations agree, the alternative files aren't encoded here. + srcs = getSrcs [ + "closure-common.asd" + "package.lisp" + "definline.lisp" + "characters.lisp" #+rune-is-character + "syntax.lisp" + "encodings.lisp" #-x&y-streams-are-stream + "encodings-data.lisp" #-x&y-streams-are-stream + "xstream.lisp" #-x&y-streams-are-stream + "ystream.lisp" #-x&y-streams-are-stream + "hax.lisp" + ]; + + deps = [ + (depot.nix.buildLisp.bundled "asdf") + depot.third_party.lisp.trivial-gray-streams + depot.third_party.lisp.babel #+rune-is-character + ]; +} diff --git a/third_party/lisp/closure-html/default.nix b/third_party/lisp/closure-html/default.nix new file mode 100644 index 000000000000..32ab49f8d479 --- /dev/null +++ b/third_party/lisp/closure-html/default.nix @@ -0,0 +1,70 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.applyPatches { + name = "closure-html-source"; + src = pkgs.fetchFromGitHub { + owner = "bluelisp"; + repo = "closure-html"; + rev = "fee42604ae36884d2f7c5e8ffc3441fdb8ec77b7"; # 2017-04-19 + sha256 = "105vm29qnxh6zj3rh4jwpm8dyp3b9bsva64c8a78cr270p28d032"; + }; + + patches = [ + # delete unexported and unused double defun in sgml-dtd.lisp + # which reference undefined CL-USER:*HTML-DTD* (!) which + # unlike CLOSURE-HTML:*HTML-DTD* is not involved in the + # packages operation. + ./no-double-defun.patch + # Patches html-parser.lisp to look for the distributed + # dtd files and catalog in this source derivations out + # path in the nix store instead of the same directory + # relatively to the (built) system. + ./dtds-from-store.patch + ]; + + postPatch = '' + # Inject file which defines CLOSURE-HTML:*HTML-DTD* + # early in the package's build since SBCL otherwise + # fails due to the undefined variable. Need to inject + # this via postPatch since using a nix file results + # in failure to look up the file's true name which + # is done for … reasons, apparently. + cat > src/define-html-dtd.lisp << EOF + (in-package :closure-html) + (defvar *html-dtd*) + EOF + + # Substitute reference to @out@ of this source + # directory in this patched file. + substituteAllInPlace src/parse/html-parser.lisp + ''; + }; + + getSrcs = builtins.map (p: "${src}/${p}"); +in + +depot.nix.buildLisp.library { + name = "closure-html"; + + srcs = getSrcs [ + "src/defpack.lisp" + "src/define-html-dtd.lisp" + "src/glisp/util.lisp" + "src/util/clex.lisp" + "src/util/lalr.lisp" + "src/net/mime.lisp" + "src/parse/pt.lisp" + "src/parse/sgml-dtd.lisp" + "src/parse/sgml-parse.lisp" + "src/parse/html-parser.lisp" + "src/parse/lhtml.lisp" + "src/parse/unparse.lisp" + "src/parse/documentation.lisp" + ]; + + deps = [ + depot.third_party.lisp.flexi-streams + depot.third_party.lisp.closure-common + ]; +} diff --git a/third_party/lisp/closure-html/dtds-from-store.patch b/third_party/lisp/closure-html/dtds-from-store.patch new file mode 100644 index 000000000000..a9ffd8085e89 --- /dev/null +++ b/third_party/lisp/closure-html/dtds-from-store.patch @@ -0,0 +1,16 @@ +diff --git a/src/parse/html-parser.lisp b/src/parse/html-parser.lisp +index 4e45b81..5025a26 100644 +--- a/src/parse/html-parser.lisp ++++ b/src/parse/html-parser.lisp +@@ -36,10 +36,7 @@ + (make-pathname + :name nil + :type nil +- :defaults (merge-pathnames +- "resources/" +- (asdf:component-relative-pathname +- (asdf:find-system :closure-html)))))) ++ :defaults "@out@/resources/"))) + (loop + :for (name . filename) + :in '(("-//W3O//DTD W3 HTML 3.0//EN" . "dtd/HTML-3.0") diff --git a/third_party/lisp/closure-html/no-double-defun.patch b/third_party/lisp/closure-html/no-double-defun.patch new file mode 100644 index 000000000000..ce7fb33abff1 --- /dev/null +++ b/third_party/lisp/closure-html/no-double-defun.patch @@ -0,0 +1,78 @@ +diff --git a/src/parse/sgml-dtd.lisp b/src/parse/sgml-dtd.lisp +index de774c0..dbee852 100644 +--- a/src/parse/sgml-dtd.lisp ++++ b/src/parse/sgml-dtd.lisp +@@ -624,73 +624,6 @@ + (return)))) + classes)) + +-;;;; ---------------------------------------------------------------------------------------------------- +-;;;; Compiled DTDs +-;;;; +- +-;; Since parsing and 'compiling' DTDs is slow, I'll provide for a way +-;; to (un)dump compiled DTD to stream. +- +-(defun dump-dtd (dtd sink) +- (let ((*print-pretty* nil) +- (*print-readably* t) +- (*print-circle* t)) +- (princ "#." sink) +- (prin1 +- `(MAKE-DTD :NAME ',(dtd-name dtd) +- :ELEMENTS (LET ((R (MAKE-HASH-TABLE :TEST #'EQ))) +- (SETF ,@(let ((q nil)) +- (maphash (lambda (key value) +- (push `',value q) +- (push `(GETHASH ',key R) q)) +- (dtd-elements dtd)) +- q)) +- R) +- :ENTITIES ',(dtd-entities dtd) +- :RESOLVE-INFO (LET ((R (MAKE-HASH-TABLE :TEST #'EQUAL))) +- (SETF ,@(let ((q nil)) +- (maphash (lambda (key value) +- (push `',value q) +- (push `(GETHASH ',key R) q)) +- (dtd-resolve-info dtd)) +- q)) +- R) +- ;; XXX surclusion-cache fehlt +- ) +- sink))) +- +-;;XXX +-(defun save-html-dtd () +- (with-open-file (sink "html-dtd.lisp" :direction :output :if-exists :new-version) +- (print `(in-package :sgml) sink) +- (let ((*package* (find-package :sgml))) +- (princ "(SETQ " sink) +- (prin1 'cl-user::*html-dtd* sink) +- (princ " '" sink) +- (dump-dtd cl-user::*html-dtd* sink) +- (princ ")" sink)))) +- +-;;; -------------------------------------------------------------------------------- +-;;; dumping DTDs +- +- +-(defun dump-dtd (dtd filename) +- (let ((*foo* dtd)) +- (declare (special *foo*)) +- (with-open-file (sink (merge-pathnames filename "*.lisp") +- :direction :output +- :if-exists :new-version) +- (format sink "(in-package :sgml)(locally (declare (special *foo*))(setq *foo* '#.*foo*))")) +- (compile-file (merge-pathnames filename "*.lisp")))) +- +-(defun undump-dtd (filename) +- (let (*foo*) +- (declare (special *foo*)) +- (load (compile-file-pathname (merge-pathnames filename "*.lisp")) +- :verbose nil +- :print nil) +- *foo*)) +- + (defmethod make-load-form ((self dtd) &optional env) + (declare (ignore env)) + `(make-dtd :name ',(dtd-name self) diff --git a/third_party/lisp/data-sift.nix b/third_party/lisp/data-sift.nix new file mode 100644 index 000000000000..18593421e8ed --- /dev/null +++ b/third_party/lisp/data-sift.nix @@ -0,0 +1,27 @@ +{ depot, pkgs, ... }: + +let + + src = pkgs.fetchFromGitHub { + owner = "archimag"; + repo = "data-sift"; + rev = "fd617d8200cdcc1b87ecf45ab59bb38e8b16ef7e"; + sha256 = "1v7gf0x4ibjzp0c56n9m77hxdgwcm9356zlk5n4l3fx4i0hj6146"; + }; + +in depot.nix.buildLisp.library { + name = "data-sift"; + deps = with depot.third_party.lisp; [ + cl-ppcre + parse-number + alexandria + puri + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "packages.lisp" + "conditions.lisp" + "sift.lisp" + ]; + +} diff --git a/third_party/lisp/defclass-std.nix b/third_party/lisp/defclass-std.nix new file mode 100644 index 000000000000..781fd107f59d --- /dev/null +++ b/third_party/lisp/defclass-std.nix @@ -0,0 +1,21 @@ +# A shortcut macro to write DEFCLASS forms quickly +# Seems to be unmaintained (since early 2021) +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "EuAndreh"; + repo = "defclass-std"; + rev = "a4d32260a619eddf3a3e49df3af304f3c07ccec6"; + sha256 = "1c0ymb49wd205lzxmnmsrpqyv0pn61snn2xvsbk5iis135r4fr18"; + }; +in depot.nix.buildLisp.library { + name = "defclass-std"; + deps = with depot.third_party.lisp; [ + alexandria + anaphora + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "defclass-std.lisp" + ]; +} diff --git a/third_party/lisp/drakma.nix b/third_party/lisp/drakma.nix new file mode 100644 index 000000000000..3757aad7b144 --- /dev/null +++ b/third_party/lisp/drakma.nix @@ -0,0 +1,39 @@ +# Drakma is an HTTP client for Common Lisp. +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "edicl"; + repo = "drakma"; + rev = "87feb02bef00b11a753d5fb21a5fec526b0d0bbb"; + sha256 = "01b80am2vrw94xmdj7f21qm7p5ys08mmpzv4nc4icql81hqr1w2m"; +}; +in depot.nix.buildLisp.library { + name = "drakma"; + deps = with depot.third_party.lisp; [ + chipz + chunga + cl-base64 + cl-plus-ssl + cl-ppcre + flexi-streams + puri + usocket + (depot.nix.buildLisp.bundled "asdf") + ]; + + srcs = map (f: src + ("/" + f)) [ + "drakma.asd" # Required because the system definition is used + "packages.lisp" + "specials.lisp" + "conditions.lisp" + "util.lisp" + "read.lisp" + "cookies.lisp" + "encoding.lisp" + "request.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/easy-routes.nix b/third_party/lisp/easy-routes.nix new file mode 100644 index 000000000000..93aed8a66765 --- /dev/null +++ b/third_party/lisp/easy-routes.nix @@ -0,0 +1,29 @@ +{ depot, pkgs, ... }: + +let + + src = pkgs.fetchFromGitHub { + owner = "mmontone"; + repo = "easy-routes"; + rev = "dab613ff419a655036a00beecee026ab6e0ba430"; + sha256 = "06lnipwc6mmg0v5gybcnr7wn5xmn5xfd1gs19vbima777245bfka"; + }; + +in depot.nix.buildLisp.library { + name = "easy-routes"; + deps = with depot.third_party.lisp; [ + hunchentoot + routes + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "util.lisp" + "easy-routes.lisp" + "routes-map-printer.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/fiveam/.boring b/third_party/lisp/fiveam/.boring new file mode 100644 index 000000000000..662944f765b3 --- /dev/null +++ b/third_party/lisp/fiveam/.boring @@ -0,0 +1,14 @@ +# Boring file regexps: +\# +~$ +(^|/)_darcs($|/) +\.dfsl$ +\.ppcf$ +\.fasl$ +\.x86f$ +\.fas$ +\.lib$ +^docs/html($|/) +^docs/pdf($|/) +^\{arch\}$ +(^|/).arch-ids($|/) diff --git a/third_party/lisp/fiveam/.travis.yml b/third_party/lisp/fiveam/.travis.yml new file mode 100644 index 000000000000..6f6559189f27 --- /dev/null +++ b/third_party/lisp/fiveam/.travis.yml @@ -0,0 +1,47 @@ +dist: bionic +language: lisp + +env: + matrix: + - LISP=abcl + - LISP=allegro + - LISP=ccl + - LISP=ccl32 + - LISP=ecl + - LISP=sbcl + - LISP=sbcl32 + - LISP=cmucl + +matrix: + allow_failures: + - env: LISP=allegro + - env: LISP=ccl32 + - env: LISP=cmucl + - env: LISP=sbcl32 + +notifications: + email: + on_success: change + on_failure: always + irc: + channels: + - "chat.freenode.net#iolib" + on_success: change + on_failure: always + use_notice: true + skip_join: true + +install: + - curl -L https://raw.githubusercontent.com/sionescu/cl-travis/master/install.sh | sh + - cl -e "(cl:in-package :cl-user) + (dolist (p '(:alexandria)) + (ql:quickload p :verbose t))" + +script: + - cl -e "(cl:in-package :cl-user) + (ql:quickload :fiveam/test :verbose t) + (uiop:quit (if (some (lambda (x) (typep x '5am::test-failure)) + (5am:run :it.bese.fiveam)) + 1 0))" + +sudo: required diff --git a/third_party/lisp/fiveam/COPYING b/third_party/lisp/fiveam/COPYING new file mode 100644 index 000000000000..91adf85a5a64 --- /dev/null +++ b/third_party/lisp/fiveam/COPYING @@ -0,0 +1,30 @@ +Copyright (c) 2003-2006, Edward Marco Baringer +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of Edward Marco Baringer, nor BESE, nor the names +of its contributors may be used to endorse or promote products derived +from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/third_party/lisp/fiveam/README b/third_party/lisp/fiveam/README new file mode 100644 index 000000000000..32a205fa5f5a --- /dev/null +++ b/third_party/lisp/fiveam/README @@ -0,0 +1,8 @@ +This is FiveAM, a common lisp testing framework. + +The documentation can be found in the docstrings, start with the +package :it.bese.fiveam (nicknamed 5AM). + +The mailing list for FiveAM is fiveam-devel@common-lisp.net + +All the code is Copyright (C) 2002-2006 Edward Marco Baringer. diff --git a/third_party/lisp/fiveam/default.nix b/third_party/lisp/fiveam/default.nix new file mode 100644 index 000000000000..4236b93bc9c5 --- /dev/null +++ b/third_party/lisp/fiveam/default.nix @@ -0,0 +1,28 @@ +# FiveAM is a Common Lisp testing framework. +# +# Imported from https://github.com/sionescu/fiveam.git + +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "fiveam"; + + deps = with depot.third_party.lisp; [ + alexandria + asdf-flv + trivial-backtrace + ]; + + srcs = [ + ./src/package.lisp + ./src/utils.lisp + ./src/check.lisp + ./src/fixture.lisp + ./src/classes.lisp + ./src/random.lisp + ./src/test.lisp + ./src/explain.lisp + ./src/suite.lisp + ./src/run.lisp + ]; +} diff --git a/third_party/lisp/fiveam/docs/make-qbook.lisp b/third_party/lisp/fiveam/docs/make-qbook.lisp new file mode 100644 index 000000000000..8144c94f020e --- /dev/null +++ b/third_party/lisp/fiveam/docs/make-qbook.lisp @@ -0,0 +1,13 @@ +(asdf:oos 'asdf:load-op :FiveAM) +(asdf:oos 'asdf:load-op :qbook) + +(asdf:oos 'qbook:publish-op :FiveAM + :generator (make-instance 'qbook:html-generator + :title "FiveAM" + :output-directory + (merge-pathnames + (make-pathname :directory '(:relative "docs" "html")) + (asdf:component-pathname (asdf:find-system :FiveAM))))) + + + diff --git a/third_party/lisp/fiveam/fiveam.asd b/third_party/lisp/fiveam/fiveam.asd new file mode 100644 index 000000000000..7607e33372fd --- /dev/null +++ b/third_party/lisp/fiveam/fiveam.asd @@ -0,0 +1,36 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +#.(unless (or #+asdf3.1 (version<= "3.1" (asdf-version))) + (error "You need ASDF >= 3.1 to load this system correctly.")) + +(defsystem :fiveam + :author "Edward Marco Baringer <mb@bese.it>" + :version (:read-file-form "version.sexp") + :description "A simple regression testing framework" + :license "BSD" + :depends-on (:alexandria :net.didierverna.asdf-flv :trivial-backtrace) + :pathname "src/" + :components ((:file "package") + (:file "utils" :depends-on ("package")) + (:file "check" :depends-on ("package" "utils")) + (:file "fixture" :depends-on ("package")) + (:file "classes" :depends-on ("package")) + (:file "random" :depends-on ("package" "check")) + (:file "test" :depends-on ("package" "fixture" "classes")) + (:file "explain" :depends-on ("package" "utils" "check" "classes" "random")) + (:file "suite" :depends-on ("package" "test" "classes")) + (:file "run" :depends-on ("package" "check" "classes" "test" "explain" "suite"))) + :in-order-to ((test-op (test-op :fiveam/test)))) + +(defsystem :fiveam/test + :author "Edward Marco Baringer <mb@bese.it>" + :description "FiveAM's own test suite" + :license "BSD" + :depends-on (:fiveam) + :pathname "t/" + :components ((:file "tests")) + :perform (test-op (o c) (symbol-call :5am :run! :it.bese.fiveam))) + +;;;;@include "src/package.lisp" + +;;;;@include "t/example.lisp" diff --git a/third_party/lisp/fiveam/src/check.lisp b/third_party/lisp/fiveam/src/check.lisp new file mode 100644 index 000000000000..b3808c5cf04c --- /dev/null +++ b/third_party/lisp/fiveam/src/check.lisp @@ -0,0 +1,311 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +;;;; * Checks + +;;;; At the lowest level testing the system requires that certain +;;;; forms be evaluated and that certain post conditions are met: the +;;;; value returned must satisfy a certain predicate, the form must +;;;; (or must not) signal a certain condition, etc. In FiveAM these +;;;; low level operations are called 'checks' and are defined using +;;;; the various checking macros. + +;;;; Checks are the basic operators for collecting results. Tests and +;;;; test suites on the other hand allow grouping multiple checks into +;;;; logic collections. + +(defvar *test-dribble* t) + +(defmacro with-*test-dribble* (stream &body body) + `(let ((*test-dribble* ,stream)) + (declare (special *test-dribble*)) + ,@body)) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (def-special-environment run-state () + result-list + current-test)) + +;;;; ** Types of test results + +;;;; Every check produces a result object. + +(defclass test-result () + ((reason :accessor reason :initarg :reason :initform "no reason given") + (test-case :accessor test-case :initarg :test-case) + (test-expr :accessor test-expr :initarg :test-expr)) + (:documentation "All checking macros will generate an object of + type TEST-RESULT.")) + +(defclass test-passed (test-result) + () + (:documentation "Class for successful checks.")) + +(defgeneric test-passed-p (object) + (:method ((o t)) nil) + (:method ((o test-passed)) t)) + +(define-condition check-failure (error) + ((reason :accessor reason :initarg :reason :initform "no reason given") + (test-case :accessor test-case :initarg :test-case) + (test-expr :accessor test-expr :initarg :test-expr)) + (:documentation "Signaled when a check fails.") + (:report (lambda (c stream) + (format stream "The following check failed: ~S~%~A." + (test-expr c) + (reason c))))) + +(defun process-failure (test-expr &optional reason-format &rest format-args) + (let ((reason (and reason-format + (apply #'format nil reason-format format-args)))) + (with-simple-restart (ignore-failure "Continue the test run.") + (error 'check-failure :test-expr test-expr + :reason reason)) + (add-result 'test-failure :test-expr test-expr + :reason reason))) + +(defclass test-failure (test-result) + () + (:documentation "Class for unsuccessful checks.")) + +(defgeneric test-failure-p (object) + (:method ((o t)) nil) + (:method ((o test-failure)) t)) + +(defclass unexpected-test-failure (test-failure) + ((actual-condition :accessor actual-condition :initarg :condition)) + (:documentation "Represents the result of a test which neither +passed nor failed, but signaled an error we couldn't deal +with. + +Note: This is very different than a SIGNALS check which instead +creates a TEST-PASSED or TEST-FAILURE object.")) + +(defclass test-skipped (test-result) + () + (:documentation "A test which was not run. Usually this is due to +unsatisfied dependencies, but users can decide to skip the test when +appropriate.")) + +(defgeneric test-skipped-p (object) + (:method ((o t)) nil) + (:method ((o test-skipped)) t)) + +(defun add-result (result-type &rest make-instance-args) + "Create a TEST-RESULT object of type RESULT-TYPE passing it the + initialize args MAKE-INSTANCE-ARGS and add the resulting + object to the list of test results." + (with-run-state (result-list current-test) + (let ((result (apply #'make-instance result-type + (append make-instance-args (list :test-case current-test))))) + (etypecase result + (test-passed (format *test-dribble* ".")) + (unexpected-test-failure (format *test-dribble* "X")) + (test-failure (format *test-dribble* "f")) + (test-skipped (format *test-dribble* "s"))) + (push result result-list)))) + +;;;; ** The check operators + +;;;; *** The IS check + +(defmacro is (test &rest reason-args) + "The DWIM checking operator. + +If TEST returns a true value a test-passed result is generated, +otherwise a test-failure result is generated. The reason, unless +REASON-ARGS is provided, is generated based on the form of TEST: + + (predicate expected actual) - Means that we want to check + whether, according to PREDICATE, the ACTUAL value is + in fact what we EXPECTED. + + (predicate value) - Means that we want to ensure that VALUE + satisfies PREDICATE. + + Wrapping the TEST form in a NOT simply produces a negated reason + string." + (assert (listp test) + (test) + "Argument to IS must be a list, not ~S" test) + (let (bindings effective-test default-reason-args) + (with-gensyms (e a v) + (flet ((process-entry (predicate expected actual &optional negatedp) + ;; make sure EXPECTED is holding the entry that starts with 'values + (when (and (consp actual) + (eq (car actual) 'values)) + (assert (not (and (consp expected) + (eq (car expected) 'values))) () + "Both the expected and actual part is a values expression.") + (rotatef expected actual)) + (let ((setf-forms)) + (if (and (consp expected) + (eq (car expected) 'values)) + (progn + (setf expected (copy-list expected)) + (setf setf-forms (loop for cell = (rest expected) then (cdr cell) + for i from 0 + while cell + when (eq (car cell) '*) + collect `(setf (elt ,a ,i) nil) + and do (setf (car cell) nil))) + (setf bindings (list (list e `(list ,@(rest expected))) + (list a `(multiple-value-list ,actual))))) + (setf bindings (list (list e expected) + (list a actual)))) + (setf effective-test `(progn + ,@setf-forms + ,(if negatedp + `(not (,predicate ,e ,a)) + `(,predicate ,e ,a))))))) + (list-match-case test + ((not (?predicate ?expected ?actual)) + (process-entry ?predicate ?expected ?actual t) + (setf default-reason-args + (list "~2&~S~2% evaluated to ~2&~S~2% which is ~2&~S~2%to ~2&~S~2% (it should not be)" + `',?actual a `',?predicate e))) + ((not (?satisfies ?value)) + (setf bindings (list (list v ?value)) + effective-test `(not (,?satisfies ,v)) + default-reason-args + (list "~2&~S~2% evaluated to ~2&~S~2% which satisfies ~2&~S~2% (it should not)" + `',?value v `',?satisfies))) + ((?predicate ?expected ?actual) + (process-entry ?predicate ?expected ?actual) + (setf default-reason-args + (list "~2&~S~2% evaluated to ~2&~S~2% which is not ~2&~S~2% to ~2&~S~2%." + `',?actual a `',?predicate e))) + ((?satisfies ?value) + (setf bindings (list (list v ?value)) + effective-test `(,?satisfies ,v) + default-reason-args + (list "~2&~S~2% evaluated to ~2&~S~2% which does not satisfy ~2&~S~2%" + `',?value v `',?satisfies))) + (?_ + (setf bindings '() + effective-test test + default-reason-args (list "~2&~S~2% was NIL." `',test))))) + `(let ,bindings + (if ,effective-test + (add-result 'test-passed :test-expr ',test) + (process-failure ',test + ,@(or reason-args default-reason-args))))))) + +;;;; *** Other checks + +(defmacro skip (&rest reason) + "Generates a TEST-SKIPPED result." + `(progn + (format *test-dribble* "s") + (add-result 'test-skipped :reason (format nil ,@reason)))) + +(defmacro is-every (predicate &body clauses) + "The input is either a list of lists, or a list of pairs. Generates (is (,predicate ,expr ,value)) + for each pair of elements or (is (,predicate ,expr ,value) ,@reason) for each list." + `(progn + ,@(if (every #'consp clauses) + (loop for (expected actual . reason) in clauses + collect `(is (,predicate ,expected ,actual) ,@reason)) + (progn + (assert (evenp (list-length clauses))) + (loop for (expr value) on clauses by #'cddr + collect `(is (,predicate ,expr ,value))))))) + +(defmacro is-true (condition &rest reason-args) + "Like IS this check generates a pass if CONDITION returns true + and a failure if CONDITION returns false. Unlike IS this check + does not inspect CONDITION to determine how to report the + failure." + `(if ,condition + (add-result 'test-passed :test-expr ',condition) + (process-failure ',condition + ,@(or reason-args + `("~S did not return a true value" ',condition))))) + +(defmacro is-false (condition &rest reason-args) + "Generates a pass if CONDITION returns false, generates a + failure otherwise. Like IS-TRUE, and unlike IS, IS-FALSE does + not inspect CONDITION to determine what reason to give it case + of test failure" + (with-gensyms (value) + `(let ((,value ,condition)) + (if ,value + (process-failure ',condition + ,@(or reason-args + `("~S returned the value ~S, which is true" ',condition ,value))) + (add-result 'test-passed :test-expr ',condition))))) + +(defmacro signals (condition-spec + &body body) + "Generates a pass if BODY signals a condition of type +CONDITION. BODY is evaluated in a block named NIL, CONDITION is +not evaluated." + (let ((block-name (gensym))) + (destructuring-bind (condition &optional reason-control reason-args) + (ensure-list condition-spec) + `(block ,block-name + (handler-bind ((,condition (lambda (c) + (declare (ignore c)) + ;; ok, body threw condition + (add-result 'test-passed + :test-expr ',condition) + (return-from ,block-name t)))) + (block nil + ,@body)) + (process-failure + ',condition + ,@(if reason-control + `(,reason-control ,@reason-args) + `("Failed to signal a ~S" ',condition))) + (return-from ,block-name nil))))) + +(defmacro finishes (&body body) + "Generates a pass if BODY executes to normal completion. In +other words if body does signal, return-from or throw this test +fails." + `(unwind-protect-case () (progn ,@body) + (:normal (add-result 'test-passed :test-expr ',body)) + (:abort (process-failure ',body "Test didn't finish")))) + +(defmacro pass (&rest message-args) + "Simply generate a PASS." + `(add-result 'test-passed + :test-expr ',message-args + ,@(when message-args + `(:reason (format nil ,@message-args))))) + +(defmacro fail (&rest message-args) + "Simply generate a FAIL." + `(process-failure ',message-args + ,@message-args)) + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/third_party/lisp/fiveam/src/classes.lisp b/third_party/lisp/fiveam/src/classes.lisp new file mode 100644 index 000000000000..fc4dc782e8cb --- /dev/null +++ b/third_party/lisp/fiveam/src/classes.lisp @@ -0,0 +1,128 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +(defclass testable-object () + ((name :initarg :name :accessor name + :documentation "A symbol naming this test object.") + (description :initarg :description :accessor description :initform nil + :documentation "The textual description of this test object.") + (depends-on :initarg :depends-on :accessor depends-on :initform nil + :documentation "The list of AND, OR, NOT forms specifying when to run this test.") + (status :initarg :status :accessor status :initform :unknown + :documentation "A symbol specifying the current status + of this test. Either: T - this test (and all its + dependencies, have passed. NIL - this test + failed (either it failed or its dependecies weren't + met. :circular this test has a circular dependency + and was skipped. Or :depends-not-satisfied or :resolving") + (profiling-info :accessor profiling-info + :initform nil + :documentation "An object representing how + much time and memory where used by the + test.") + (collect-profiling-info :accessor collect-profiling-info + :initarg :collect-profiling-info + :initform nil + :documentation "When T profiling + information will be collected when the + test is run."))) + +(defmethod print-object ((test testable-object) stream) + (print-unreadable-object (test stream :type t :identity t) + (format stream "~S" (name test)))) + +(defclass test-suite (testable-object) + ((tests :accessor tests :initform (make-hash-table :test 'eql) + :documentation "The hash table mapping names to test + objects in this suite. The values in this hash table + can be either test-cases or other test-suites.")) + (:documentation "A test suite is a collection of tests or test suites. + +Test suites serve to organize tests into groups so that the +developer can chose to run some tests and not just one or +all. Like tests test suites have a name and a description. + +Test suites, like tests, can be part of other test suites, this +allows the developer to create a hierarchy of tests where sub +trees can be singularly run. + +Running a test suite has the effect of running every test (or +suite) in the suite.")) + +(defclass test-case (testable-object) + ((test-lambda :initarg :test-lambda :accessor test-lambda + :documentation "The function to run.") + (runtime-package :initarg :runtime-package :accessor runtime-package + :documentation "By default it stores *package* from the time this test was defined (macroexpanded).")) + (:documentation "A test case is a single, named, collection of +checks. + +A test case is the smallest organizational element which can be +run individually. Every test case has a name, which is a symbol, +a description and a test lambda. The test lambda is a regular +funcall'able function which should use the various checking +macros to collect results. + +Every test case is part of a suite, when a suite is not +explicitly specified (either via the :SUITE parameter to the TEST +macro or the global variable *SUITE*) the test is inserted into +the global suite named NIL. + +Sometimes we want to run a certain test only if another test has +passed. FiveAM allows us to specify the ways in which one test is +dependent on another. + +- AND Run this test only if all the named tests passed. + +- OR Run this test if at least one of the named tests passed. + +- NOT Run this test only if another test has failed. + +FiveAM considers a test to have passed if all the checks executed +were successful, otherwise we consider the test a failure. + +When a test is not run due to it's dependencies having failed a +test-skipped result is added to the results.")) + +(defclass explainer () + ()) + +(defclass text-explainer (explainer) + ()) + +(defclass simple-text-explainer (text-explainer) + ()) + +(defclass detailed-text-explainer (text-explainer) + ()) + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/third_party/lisp/fiveam/src/explain.lisp b/third_party/lisp/fiveam/src/explain.lisp new file mode 100644 index 000000000000..015cdf45521a --- /dev/null +++ b/third_party/lisp/fiveam/src/explain.lisp @@ -0,0 +1,133 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +;;;; * Analyzing the results + +(defparameter *verbose-failures* nil + "T if we should print the expression failing, NIL otherwise.") + +;;;; Just as important as defining and runnig the tests is +;;;; understanding the results. FiveAM provides the function EXPLAIN +;;;; which prints a human readable summary (number passed, number +;;;; failed, what failed and why, etc.) of a list of test results. + +(defgeneric explain (explainer results &optional stream recursive-depth) + (:documentation "Given a list of test results report write to stream detailed + human readable statistics regarding the results.")) + +(defmethod explain ((exp detailed-text-explainer) results + &optional (stream *test-dribble*) (recursive-depth 0)) + (multiple-value-bind (num-checks passed num-passed passed% + skipped num-skipped skipped% + failed num-failed failed% + unknown num-unknown unknown%) + (partition-results results) + (declare (ignore passed)) + (flet ((output (&rest format-args) + (format stream "~&~vT" recursive-depth) + (apply #'format stream format-args))) + + (when (zerop num-checks) + (output "Didn't run anything...huh?") + (return-from explain nil)) + (output "Did ~D check~P.~%" num-checks num-checks) + (output " Pass: ~D (~2D%)~%" num-passed passed%) + (output " Skip: ~D (~2D%)~%" num-skipped skipped%) + (output " Fail: ~D (~2D%)~%" num-failed failed%) + (when unknown + (output " UNKNOWN RESULTS: ~D (~2D)~%" num-unknown unknown%)) + (terpri stream) + (when failed + (output "Failure Details:~%") + (dolist (f (reverse failed)) + (output "--------------------------------~%") + (output "~A ~@{[~A]~}: ~%" + (name (test-case f)) + (description (test-case f))) + (output " ~A.~%" (reason f)) + (when (for-all-test-failed-p f) + (output "Results collected with failure data:~%") + (explain exp (slot-value f 'result-list) + stream (+ 4 recursive-depth))) + (when (and *verbose-failures* (test-expr f)) + (output " ~S~%" (test-expr f))) + (output "--------------------------------~%")) + (terpri stream)) + (when skipped + (output "Skip Details:~%") + (dolist (f skipped) + (output "~A ~@{[~A]~}: ~%" + (name (test-case f)) + (description (test-case f))) + (output " ~A.~%" (reason f))) + (terpri stream))))) + +(defmethod explain ((exp simple-text-explainer) results + &optional (stream *test-dribble*) (recursive-depth 0)) + (multiple-value-bind (num-checks passed num-passed passed% + skipped num-skipped skipped% + failed num-failed failed% + unknown num-unknown unknown%) + (partition-results results) + (declare (ignore passed passed% skipped skipped% failed failed% unknown unknown%)) + (format stream "~&~vTRan ~D checks, ~D passed" recursive-depth num-checks num-passed) + (when (plusp num-skipped) + (format stream ", ~D skipped " num-skipped)) + (format stream " and ~D failed.~%" num-failed) + (when (plusp num-unknown) + (format stream "~vT~D UNKNOWN RESULTS.~%" recursive-depth num-unknown)))) + +(defun partition-results (results-list) + (let ((num-checks (length results-list))) + (destructuring-bind (passed skipped failed unknown) + (partitionx results-list + (lambda (res) + (typep res 'test-passed)) + (lambda (res) + (typep res 'test-skipped)) + (lambda (res) + (typep res 'test-failure)) + t) + (if (zerop num-checks) + (values 0 + nil 0 0 + nil 0 0 + nil 0 0 + nil 0 0) + (values + num-checks + passed (length passed) (floor (* 100 (/ (length passed) num-checks))) + skipped (length skipped) (floor (* 100 (/ (length skipped) num-checks))) + failed (length failed) (floor (* 100 (/ (length failed) num-checks))) + unknown (length unknown) (floor (* 100 (/ (length failed) num-checks)))))))) + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/third_party/lisp/fiveam/src/fixture.lisp b/third_party/lisp/fiveam/src/fixture.lisp new file mode 100644 index 000000000000..26e993304fd9 --- /dev/null +++ b/third_party/lisp/fiveam/src/fixture.lisp @@ -0,0 +1,82 @@ +;; -*- lisp -*- + +(in-package :it.bese.fiveam) + +;;;; ** Fixtures + +;;;; When running tests we often need to setup some kind of context +;;;; (create dummy db connections, simulate an http request, +;;;; etc.). Fixtures provide a way to conviently hide this context +;;;; into a macro and allow the test to focus on testing. + +;;;; NB: A FiveAM fixture is nothing more than a macro. Since the term +;;;; 'fixture' is so common in testing frameworks we've provided a +;;;; wrapper around defmacro for this purpose. + +(defvar *fixture* + (make-hash-table :test 'eql) + "Lookup table mapping fixture names to fixture + objects.") + +(defun get-fixture (key &optional default) + (gethash key *fixture* default)) + +(defun (setf get-fixture) (value key) + (setf (gethash key *fixture*) value)) + +(defun rem-fixture (key) + (remhash key *fixture*)) + +(defmacro def-fixture (name (&rest args) &body body) + "Defines a fixture named NAME. A fixture is very much like a +macro but is used only for simple templating. A fixture created +with DEF-FIXTURE is a macro which can use the special macrolet +&BODY to specify where the body should go. + +See Also: WITH-FIXTURE +" + `(eval-when (:compile-toplevel :load-toplevel :execute) + (setf (get-fixture ',name) (cons ',args ',body)) + ',name)) + +(defmacro with-fixture (fixture-name (&rest args) &body body) + "Insert BODY into the fixture named FIXTURE-NAME. + +See Also: DEF-FIXTURE" + (assert (get-fixture fixture-name) + (fixture-name) + "Unknown fixture ~S." fixture-name) + (destructuring-bind ((&rest largs) &rest lbody) + (get-fixture fixture-name) + `(macrolet ((&body () '(progn ,@body))) + (funcall (lambda (,@largs) ,@lbody) ,@args)))) + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/lisp/fiveam/src/package.lisp b/third_party/lisp/fiveam/src/package.lisp new file mode 100644 index 000000000000..3279a9a4f7fc --- /dev/null +++ b/third_party/lisp/fiveam/src/package.lisp @@ -0,0 +1,139 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +;;;; * Introduction + +;;;; FiveAM is a testing framework. It takes care of all the boring +;;;; bookkeeping associated with managing a test framework allowing +;;;; the developer to focus on writing tests and code. + +;;;; FiveAM was designed with the following premises: + +;;;; - Defining tests should be about writing tests, not +;;;; infrastructure. The developer should be able to focus on what +;;;; they're testing, not the testing framework. + +;;;; - Interactive testing is the norm. Common Lisp is an interactive +;;;; development environment, the testing environment should allow the +;;;; developer to quickly and easily redefine, change, remove and run +;;;; tests. + +(defpackage :it.bese.fiveam + (:use :common-lisp :alexandria) + (:nicknames :5am :fiveam) + #+sb-package-locks + (:lock t) + (:export + ;; creating tests and test-suites + #:make-suite + #:def-suite + #:def-suite* + #:in-suite + #:in-suite* + #:test + #:def-test + #:get-test + #:rem-test + #:test-names + #:*default-test-compilation-time* + ;; fixtures + #:def-fixture + #:with-fixture + #:get-fixture + #:rem-fixture + ;; running checks + #:is + #:is-every + #:is-true + #:is-false + #:signals + #:finishes + #:skip + #:pass + #:fail + #:*test-dribble* + #:for-all + #:*num-trials* + #:*max-trials* + #:gen-integer + #:gen-float + #:gen-character + #:gen-string + #:gen-list + #:gen-tree + #:gen-buffer + #:gen-one-element + ;; running tests + #:run + #:run-all-tests + #:explain + #:explain! + #:run! + #:debug! + #:! + #:!! + #:!!! + #:*run-test-when-defined* + #:*debug-on-error* + #:*debug-on-failure* + #:*on-error* + #:*on-failure* + #:*verbose-failures* + #:*print-names* + #:results-status)) + +;;;; You can use #+5am to put your test-defining code inline with your +;;;; other code - and not require people to have fiveam to run your +;;;; package. + +(pushnew :5am *features*) + +;;;;@include "check.lisp" + +;;;;@include "random.lisp" + +;;;;@include "fixture.lisp" + +;;;;@include "test.lisp" + +;;;;@include "suite.lisp" + +;;;;@include "run.lisp" + +;;;;@include "explain.lisp" + +;;;; * Colophon + +;;;; This documentaion was written by Edward Marco Baringer +;;;; <mb@bese.it> and generated by qbook. + +;;;; ** COPYRIGHT + +;;;; Copyright (c) 2002-2003, Edward Marco Baringer +;;;; All rights reserved. + +;;;; Redistribution and use in source and binary forms, with or without +;;;; modification, are permitted provided that the following conditions are +;;;; met: + +;;;; - Redistributions of source code must retain the above copyright +;;;; notice, this list of conditions and the following disclaimer. + +;;;; - Redistributions in binary form must reproduce the above copyright +;;;; notice, this list of conditions and the following disclaimer in the +;;;; documentation and/or other materials provided with the distribution. + +;;;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;;;; of its contributors may be used to endorse or promote products +;;;; derived from this software without specific prior written permission. + +;;;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;;;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;;;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;;;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;;;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;;;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;;;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;;;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;;;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;;;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;;;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/third_party/lisp/fiveam/src/random.lisp b/third_party/lisp/fiveam/src/random.lisp new file mode 100644 index 000000000000..49e14bc8a880 --- /dev/null +++ b/third_party/lisp/fiveam/src/random.lisp @@ -0,0 +1,265 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +;;;; ** Random (QuickCheck-ish) testing + +;;;; FiveAM provides the ability to automatically generate a +;;;; collection of random input data for a specific test and run a +;;;; test multiple times. + +;;;; Specification testing is done through the FOR-ALL macro. This +;;;; macro will bind variables to random data and run a test body a +;;;; certain number of times. Should the test body ever signal a +;;;; failure we stop running and report what values of the variables +;;;; caused the code to fail. + +;;;; The generation of the random data is done using "generator +;;;; functions" (see below for details). A generator function is a +;;;; function which creates, based on user supplied parameters, a +;;;; function which returns random data. In order to facilitate +;;;; generating good random data the FOR-ALL macro also supports guard +;;;; conditions and creating one random input based on the values of +;;;; another (see the FOR-ALL macro for details). + +;;;; *** Public Interface to the Random Tester + +(defparameter *num-trials* 100 + "Number of times we attempt to run the body of the FOR-ALL test.") + +(defparameter *max-trials* 10000 + "Number of total times we attempt to run the body of the + FOR-ALL test including when the body is skipped due to failed + guard conditions. + +Since we have guard conditions we may get into infinite loops +where the test code is never run due to the guards never +returning true. This second run limit prevents that.") + +(defmacro for-all (bindings &body body) + "Bind BINDINGS to random variables and test BODY *num-trials* times. + +BINDINGS is a list of binding forms, each element is a list +of (BINDING VALUE &optional GUARD). Value, which is evaluated +once when the for-all is evaluated, must return a generator which +be called each time BODY is evaluated. BINDING is either a symbol +or a list which will be passed to destructuring-bind. GUARD is a +form which, if present, stops BODY from executing when IT returns +NIL. The GUARDS are evaluated after all the random data has been +generated and they can refer to the current value of any +binding. NB: Generator forms, unlike guard forms, can not contain +references to the bound variables. + +Examples: + + (for-all ((a (gen-integer))) + (is (integerp a))) + + (for-all ((a (gen-integer) (plusp a))) + (is (integerp a)) + (is (plusp a))) + + (for-all ((less (gen-integer)) + (more (gen-integer) (< less more))) + (is (<= less more))) + + (for-all (((a b) (gen-two-integers))) + (is (integerp a)) + (is (integerp b)))" + (with-gensyms (test-lambda-args) + `(perform-random-testing + (list ,@(mapcar #'second bindings)) + (lambda (,test-lambda-args) + (destructuring-bind ,(mapcar #'first bindings) + ,test-lambda-args + (if (and ,@(delete-if #'null (mapcar #'third bindings))) + (progn ,@body) + (throw 'run-once + (list :guard-conditions-failed)))))))) + +;;;; *** Implementation + +;;;; We could just make FOR-ALL a monster macro, but having FOR-ALL be +;;;; a preproccessor for the perform-random-testing function is +;;;; actually much easier. + +(defun perform-random-testing (generators body) + (loop + with random-state = *random-state* + with total-counter = *max-trials* + with counter = *num-trials* + with run-at-least-once = nil + until (or (zerop total-counter) + (zerop counter)) + do (let ((result (perform-random-testing/run-once generators body))) + (ecase (first result) + (:pass + (decf counter) + (decf total-counter) + (setf run-at-least-once t)) + (:no-tests + (add-result 'for-all-test-no-tests + :reason "No tests" + :random-state random-state) + (return-from perform-random-testing nil)) + (:guard-conditions-failed + (decf total-counter)) + (:fail + (add-result 'for-all-test-failed + :reason "Found failing test data" + :random-state random-state + :failure-values (second result) + :result-list (third result)) + (return-from perform-random-testing nil)))) + finally (if run-at-least-once + (add-result 'for-all-test-passed) + (add-result 'for-all-test-never-run + :reason "Guard conditions never passed")))) + +(defun perform-random-testing/run-once (generators body) + (catch 'run-once + (bind-run-state ((result-list '())) + (let ((values (mapcar #'funcall generators))) + (funcall body values) + (cond + ((null result-list) + (throw 'run-once (list :no-tests))) + ((every #'test-passed-p result-list) + (throw 'run-once (list :pass))) + ((notevery #'test-passed-p result-list) + (throw 'run-once (list :fail values result-list)))))))) + +(defclass for-all-test-result () + ((random-state :initarg :random-state))) + +(defclass for-all-test-passed (test-passed for-all-test-result) + ()) + +(defclass for-all-test-failed (test-failure for-all-test-result) + ((failure-values :initarg :failure-values) + (result-list :initarg :result-list))) + +(defgeneric for-all-test-failed-p (object) + (:method ((object for-all-test-failed)) t) + (:method ((object t)) nil)) + +(defmethod reason ((result for-all-test-failed)) + (format nil "Falsifiable with ~S" (slot-value result 'failure-values))) + +(defclass for-all-test-no-tests (test-failure for-all-test-result) + ()) + +(defclass for-all-test-never-run (test-failure for-all-test-result) + ()) + +;;;; *** Generators + +;;;; Since this is random testing we need some way of creating random +;;;; data to feed to our code. Generators are regular functions which +;;;; create this random data. + +;;;; We provide a set of built-in generators. + +(defun gen-integer (&key (max (1+ most-positive-fixnum)) + (min (1- most-negative-fixnum))) + "Returns a generator which produces random integers greater +than or equal to MIN and less than or equal to MAX." + (lambda () + (+ min (random (1+ (- max min)))))) + +(defun gen-float (&key bound (type 'short-float)) + "Returns a generator which produces floats of type TYPE. BOUND, +if specified, constrains the results to be in the range (-BOUND, +BOUND)." + (lambda () + (let* ((most-negative (ecase type + (short-float most-negative-short-float) + (single-float most-negative-single-float) + (double-float most-negative-double-float) + (long-float most-negative-long-float))) + (most-positive (ecase type + (short-float most-positive-short-float) + (single-float most-positive-single-float) + (double-float most-positive-double-float) + (long-float most-positive-long-float))) + (bound (or bound (max most-positive (- most-negative))))) + (coerce + (ecase (random 2) + (0 ;; generate a positive number + (random (min most-positive bound))) + (1 ;; generate a negative number + (- (random (min (- most-negative) bound))))) + type)))) + +(defun gen-character (&key (code-limit char-code-limit) + (code (gen-integer :min 0 :max (1- code-limit))) + (alphanumericp nil)) + "Returns a generator of characters. + +CODE must be a generator of random integers. ALPHANUMERICP, if +non-NIL, limits the returned chars to those which pass +alphanumericp." + (lambda () + (loop + for count upfrom 0 + for char = (code-char (funcall code)) + until (and char + (or (not alphanumericp) + (alphanumericp char))) + when (= 1000 count) + do (error "After 1000 iterations ~S has still not generated ~:[a valid~;an alphanumeric~] character :(." + code alphanumericp) + finally (return char)))) + +(defun gen-string (&key (length (gen-integer :min 0 :max 80)) + (elements (gen-character)) + (element-type 'character)) + "Returns a generator which produces random strings. LENGTH must +be a generator which produces integers, ELEMENTS must be a +generator which produces characters of type ELEMENT-TYPE." + (lambda () + (loop + with length = (funcall length) + with string = (make-string length :element-type element-type) + for index below length + do (setf (aref string index) (funcall elements)) + finally (return string)))) + +(defun gen-list (&key (length (gen-integer :min 0 :max 10)) + (elements (gen-integer :min -10 :max 10))) + "Returns a generator which produces random lists. LENGTH must be +an integer generator and ELEMENTS must be a generator which +produces objects." + (lambda () + (loop + repeat (funcall length) + collect (funcall elements)))) + +(defun gen-tree (&key (size 20) + (elements (gen-integer :min -10 :max 10))) + "Returns a generator which produces random trees. SIZE controls +the approximate size of the tree, but don't try anything above + 30, you have been warned. ELEMENTS must be a generator which +will produce the elements." + (labels ((rec (&optional (current-depth 0)) + (let ((key (random (+ 3 (- size current-depth))))) + (cond ((> key 2) + (list (rec (+ current-depth 1)) + (rec (+ current-depth 1)))) + (t (funcall elements)))))) + (lambda () + (rec)))) + +(defun gen-buffer (&key (length (gen-integer :min 0 :max 50)) + (element-type '(unsigned-byte 8)) + (elements (gen-integer :min 0 :max (1- (expt 2 8))))) + (lambda () + (let ((buffer (make-array (funcall length) :element-type element-type))) + (map-into buffer elements)))) + +(defun gen-one-element (&rest elements) + (lambda () + (nth (random (length elements)) elements))) + +;;;; The trivial always-produce-the-same-thing generator is done using +;;;; cl:constantly. diff --git a/third_party/lisp/fiveam/src/run.lisp b/third_party/lisp/fiveam/src/run.lisp new file mode 100644 index 000000000000..89c522351504 --- /dev/null +++ b/third_party/lisp/fiveam/src/run.lisp @@ -0,0 +1,385 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +;;;; * Running Tests + +;;;; Once the programmer has defined what the tests are these need to +;;;; be run and the expected effects should be compared with the +;;;; actual effects. FiveAM provides the function RUN for this +;;;; purpose, RUN executes a number of tests and collects the results +;;;; of each individual check into a list which is then +;;;; returned. There are three types of test results: passed, failed +;;;; and skipped, these are represented by TEST-RESULT objects. + +;;;; Generally running a test will return normally, but there are two +;;;; exceptional situations which can occur: + +;;;; - An exception is signaled while running the test. If the +;;;; variable *on-error* is :DEBUG than FiveAM will enter the +;;;; debugger, otherwise a test failure (of type +;;;; unexpected-test-failure) is returned. When entering the +;;;; debugger two restarts are made available, one simply reruns the +;;;; current test and another signals a test-failure and continues +;;;; with the remaining tests. + +;;;; - A circular dependency is detected. An error is signaled and a +;;;; restart is made available which signals a test-skipped and +;;;; continues with the remaining tests. This restart also sets the +;;;; dependency status of the test to nil, so any tests which depend +;;;; on this one (even if the dependency is not circular) will be +;;;; skipped. + +;;;; The functions RUN!, !, !! and !!! are convenient wrappers around +;;;; RUN and EXPLAIN. + +(deftype on-problem-action () + '(member :debug :backtrace nil)) + +(declaim (type on-problem-action *on-error* *on-failure*)) + +(defvar *on-error* nil + "The action to perform on error: +- :DEBUG if we should drop into the debugger +- :BACKTRACE to print a backtrace +- NIL to simply continue") + +(defvar *on-failure* nil + "The action to perform on check failure: +- :DEBUG if we should drop into the debugger +- :BACKTRACE to print a backtrace +- NIL to simply continue") + +(defvar *debug-on-error* nil + "T if we should drop into the debugger on error, NIL otherwise. +OBSOLETE: superseded by *ON-ERROR*") + +(defvar *debug-on-failure* nil + "T if we should drop into the debugger on a failing check, NIL otherwise. +OBSOLETE: superseded by *ON-FAILURE*") + +(defparameter *print-names* t + "T if we should print test running progress, NIL otherwise.") + +(defparameter *test-dribble-indent* (make-array 0 + :element-type 'character + :fill-pointer 0 + :adjustable t) + "Used to indent tests and test suites in their parent suite") + +(defun import-testing-symbols (package-designator) + (import '(5am::is 5am::is-true 5am::is-false 5am::signals 5am::finishes) + package-designator)) + +(defparameter *run-queue* '() + "List of test waiting to be run.") + +(define-condition circular-dependency (error) + ((test-case :initarg :test-case)) + (:report (lambda (cd stream) + (format stream "A circular dependency wes detected in ~S." (slot-value cd 'test-case)))) + (:documentation "Condition signaled when a circular dependency +between test-cases has been detected.")) + +(defgeneric run-resolving-dependencies (test) + (:documentation "Given a dependency spec determine if the spec +is satisfied or not, this will generally involve running other +tests. If the dependency spec can be satisfied the test is also +run.")) + +(defmethod run-resolving-dependencies ((test test-case)) + "Return true if this test, and its dependencies, are satisfied, + NIL otherwise." + (case (status test) + (:unknown + (setf (status test) :resolving) + (if (or (not (depends-on test)) + (eql t (resolve-dependencies (depends-on test)))) + (progn + (run-test-lambda test) + (status test)) + (with-run-state (result-list) + (unless (eql :circular (status test)) + (push (make-instance 'test-skipped + :test-case test + :reason "Dependencies not satisfied") + result-list) + (setf (status test) :depends-not-satisfied))))) + (:resolving + (restart-case + (error 'circular-dependency :test-case test) + (skip () + :report (lambda (s) + (format s "Skip the test ~S and all its dependencies." (name test))) + (with-run-state (result-list) + (push (make-instance 'test-skipped :reason "Circular dependencies" :test-case test) + result-list)) + (setf (status test) :circular)))) + (t (status test)))) + +(defgeneric resolve-dependencies (depends-on)) + +(defmethod resolve-dependencies ((depends-on symbol)) + "A test which depends on a symbol is interpreted as `(AND + ,DEPENDS-ON)." + (run-resolving-dependencies (get-test depends-on))) + +(defmethod resolve-dependencies ((depends-on list)) + "Return true if the dependency spec DEPENDS-ON is satisfied, + nil otherwise." + (if (null depends-on) + t + (flet ((satisfies-depends-p (test) + (funcall test (lambda (dep) + (eql t (resolve-dependencies dep))) + (cdr depends-on)))) + (ecase (car depends-on) + (and (satisfies-depends-p #'every)) + (or (satisfies-depends-p #'some)) + (not (satisfies-depends-p #'notany)) + (:before (every #'(lambda (dep) + (let ((status (status (get-test dep)))) + (if (eql :unknown status) + (run-resolving-dependencies (get-test dep)) + status))) + (cdr depends-on))))))) + +(defun results-status (result-list) + "Given a list of test results (generated while running a test) + return true if no results are of type TEST-FAILURE. Returns second + and third values, which are the set of failed tests and skipped + tests respectively." + (let ((failed-tests + (remove-if-not #'test-failure-p result-list)) + (skipped-tests + (remove-if-not #'test-skipped-p result-list))) + (values (null failed-tests) + failed-tests + skipped-tests))) + +(defun return-result-list (test-lambda) + "Run the test function TEST-LAMBDA and return a list of all + test results generated, does not modify the special environment + variable RESULT-LIST." + (bind-run-state ((result-list '())) + (funcall test-lambda) + result-list)) + +(defgeneric run-test-lambda (test)) + +(defmethod run-test-lambda ((test test-case)) + (with-run-state (result-list) + (bind-run-state ((current-test test)) + (labels ((abort-test (e &optional (reason (format nil "Unexpected Error: ~S~%~A." e e))) + (add-result 'unexpected-test-failure + :test-expr nil + :test-case test + :reason reason + :condition e)) + (run-it () + (let ((result-list '())) + (declare (special result-list)) + (handler-bind ((check-failure (lambda (e) + (declare (ignore e)) + (cond + ((eql *on-failure* :debug) + nil) + (t + (when (eql *on-failure* :backtrace) + (trivial-backtrace:print-backtrace-to-stream + *test-dribble*)) + (invoke-restart + (find-restart 'ignore-failure)))))) + (error (lambda (e) + (unless (or (eql *on-error* :debug) + (typep e 'check-failure)) + (when (eql *on-error* :backtrace) + (trivial-backtrace:print-backtrace-to-stream + *test-dribble*)) + (abort-test e) + (return-from run-it result-list))))) + (restart-case + (handler-case + (let ((*readtable* (copy-readtable)) + (*package* (runtime-package test))) + (when *print-names* + (format *test-dribble* "~%~ARunning test ~A " *test-dribble-indent* (name test))) + (if (collect-profiling-info test) + ;; Timing info doesn't get collected ATM, we need a portable library + ;; (setf (profiling-info test) (collect-timing (test-lambda test))) + (funcall (test-lambda test)) + (funcall (test-lambda test)))) + (storage-condition (e) + ;; heap-exhausted/constrol-stack-exhausted + ;; handler-case unwinds the stack (unlike handler-bind) + (abort-test e (format nil "STORAGE-CONDITION: aborted for safety. ~S~%~A." e e)) + (return-from run-it result-list))) + (retest () + :report (lambda (stream) + (format stream "~@<Rerun the test ~S~@:>" test)) + (return-from run-it (run-it))) + (ignore () + :report (lambda (stream) + (format stream "~@<Signal an exceptional test failure and abort the test ~S.~@:>" test)) + (abort-test (make-instance 'test-failure :test-case test + :reason "Failure restart.")))) + result-list)))) + (let ((results (run-it))) + (setf (status test) (results-status results) + result-list (nconc result-list results))))))) + +(defgeneric %run (test-spec) + (:documentation "Internal method for running a test. Does not + update the status of the tests nor the special variables !, + !!, !!!")) + +(defmethod %run ((test test-case)) + (run-resolving-dependencies test)) + +(defmethod %run ((tests list)) + (mapc #'%run tests)) + +(defmethod %run ((suite test-suite)) + (when *print-names* + (format *test-dribble* "~%~ARunning test suite ~A" *test-dribble-indent* (name suite))) + (let ((suite-results '())) + (flet ((run-tests () + (loop + for test being the hash-values of (tests suite) + do (%run test)))) + (vector-push-extend #\space *test-dribble-indent*) + (unwind-protect + (bind-run-state ((result-list '())) + (unwind-protect + (if (collect-profiling-info suite) + ;; Timing info doesn't get collected ATM, we need a portable library + ;; (setf (profiling-info suite) (collect-timing #'run-tests)) + (run-tests) + (run-tests))) + (setf suite-results result-list + (status suite) (every #'test-passed-p suite-results))) + (vector-pop *test-dribble-indent*) + (with-run-state (result-list) + (setf result-list (nconc result-list suite-results))))))) + +(defmethod %run ((test-name symbol)) + (when-let (test (get-test test-name)) + (%run test))) + +(defvar *initial-!* (lambda () (format t "Haven't run that many tests yet.~%"))) + +(defvar *!* *initial-!*) +(defvar *!!* *initial-!*) +(defvar *!!!* *initial-!*) + +;;;; ** Public entry points + +(defun run! (&optional (test-spec *suite*) + &key ((:print-names *print-names*) *print-names*)) + "Equivalent to (explain! (run TEST-SPEC))." + (explain! (run test-spec))) + +(defun explain! (result-list) + "Explain the results of RESULT-LIST using a +detailed-text-explainer with output going to *test-dribble*. +Return a boolean indicating whether no tests failed." + (explain (make-instance 'detailed-text-explainer) result-list *test-dribble*) + (results-status result-list)) + +(defun debug! (&optional (test-spec *suite*)) + "Calls (run! test-spec) but enters the debugger if any kind of error happens." + (let ((*on-error* :debug) + (*on-failure* :debug)) + (run! test-spec))) + +(defun run (test-spec &key ((:print-names *print-names*) *print-names*)) + "Run the test specified by TEST-SPEC. + +TEST-SPEC can be either a symbol naming a test or test suite, or +a testable-object object. This function changes the operations +performed by the !, !! and !!! functions." + (psetf *!* (lambda () + (loop :for test :being :the :hash-keys :of *test* + :do (setf (status (get-test test)) :unknown)) + (bind-run-state ((result-list '())) + (with-simple-restart (explain "Ignore the rest of the tests and explain current results") + (%run test-spec)) + result-list)) + *!!* *!* + *!!!* *!!*) + (let ((*on-error* + (or *on-error* (cond + (*debug-on-error* + (format *test-dribble* "*DEBUG-ON-ERROR* is obsolete. Use *ON-ERROR*.") + :debug) + (t nil)))) + (*on-failure* + (or *on-failure* (cond + (*debug-on-failure* + (format *test-dribble* "*DEBUG-ON-FAILURE* is obsolete. Use *ON-FAILURE*.") + :debug) + (t nil))))) + (funcall *!*))) + +(defun ! () + "Rerun the most recently run test and explain the results." + (explain! (funcall *!*))) + +(defun !! () + "Rerun the second most recently run test and explain the results." + (explain! (funcall *!!*))) + +(defun !!! () + "Rerun the third most recently run test and explain the results." + (explain! (funcall *!!!*))) + +(defun run-all-tests (&key (summary :end)) + "Runs all defined test suites, T if all tests passed and NIL otherwise. +SUMMARY can be :END to print a summary at the end, :SUITE to print it +after each suite or NIL to skip explanations." + (check-type summary (member nil :suite :end)) + (loop :for suite :in (cons 'nil (sort (copy-list *toplevel-suites*) #'string<=)) + :for results := (if (suite-emptyp suite) nil (run suite)) + :when (consp results) + :collect results :into all-results + :do (cond + ((not (eql summary :suite)) + nil) + (results + (explain! results)) + (suite + (format *test-dribble* "Suite ~A is empty~%" suite))) + :finally (progn + (when (eql summary :end) + (explain! (alexandria:flatten all-results))) + (return (every #'results-status all-results))))) + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/lisp/fiveam/src/style.css b/third_party/lisp/fiveam/src/style.css new file mode 100644 index 000000000000..4a1e6010dce5 --- /dev/null +++ b/third_party/lisp/fiveam/src/style.css @@ -0,0 +1,64 @@ +body { + background-color: #FFFFFF; + color: #000000; + padding: 0px; margin: 0px; +} + +.qbook { width: 600px; background-color: #FFFFFF; margin: 0px; + border-left: 3em solid #660000; padding: 3px; } + +h1 { text-align: center; margin: 0px; + color: #333333; + border-bottom: 0.3em solid #660000; +} + +p { padding-left: 1em; } + +h2 { border-bottom: 0.2em solid #000000; font-family: verdana; } + +h3 { border-bottom: 0.1em solid #000000; } + +pre.code { + background-color: #eeeeee; + border: solid 1px #d0d0d0; + overflow: auto; +} + +pre.code * .paren { color: #666666; } + +pre.code a:active { color: #000000; } +pre.code a:link { color: #000000; } +pre.code a:visited { color: #000000; } + +pre.code .first-line { font-weight: bold; } + +div.contents { font-family: verdana; } + +div.contents a:active { color: #000000; } +div.contents a:link { color: #000000; } +div.contents a:visited { color: #000000; } + +div.contents div.contents-heading-1 { padding-left: 0.5em; font-weight: bold; } +div.contents div.contents-heading-1 a:active { color: #660000; } +div.contents div.contents-heading-1 a:link { color: #660000; } +div.contents div.contents-heading-1 a:visited { color: #660000; } + +div.contents div.contents-heading-2 { padding-left: 1.0em; } +div.contents div.contents-heading-2 a:active { color: #660000; } +div.contents div.contents-heading-2 a:link { color: #660000; } +div.contents div.contents-heading-2 a:visited { color: #660000; } + +div.contents div.contents-heading-3 { padding-left: 1.5em; } +div.contents div.contents-heading-3 a:active { color: #660000; } +div.contents div.contents-heading-3 a:link { color: #660000; } +div.contents div.contents-heading-3 a:visited { color: #660000; } + +div.contents div.contents-heading-4 { padding-left: 2em; } +div.contents div.contents-heading-4 a:active { color: #660000; } +div.contents div.contents-heading-4 a:link { color: #660000; } +div.contents div.contents-heading-4 a:visited { color: #660000; } + +div.contents div.contents-heading-5 { padding-left: 2.5em; } +div.contents div.contents-heading-5 a:active { color: #660000; } +div.contents div.contents-heading-5 a:link { color: #660000; } +div.contents div.contents-heading-5 a:visited { color: #660000; } diff --git a/third_party/lisp/fiveam/src/suite.lisp b/third_party/lisp/fiveam/src/suite.lisp new file mode 100644 index 000000000000..8497a9d12ddc --- /dev/null +++ b/third_party/lisp/fiveam/src/suite.lisp @@ -0,0 +1,140 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +;;;; * Test Suites + +;;;; Test suites allow us to collect multiple tests into a single +;;;; object and run them all using asingle name. Test suites do not +;;;; affect the way test are run nor the way the results are handled, +;;;; they are simply a test organizing group. + +;;;; Test suites can contain both tests and other test suites. Running +;;;; a test suite causes all of its tests and test suites to be +;;;; run. Suites do not affect test dependencies, running a test suite +;;;; can cause tests which are not in the suite to be run. + +;;;; ** Current Suite + +(defvar *suite* nil + "The current test suite object") +(net.didierverna.asdf-flv:set-file-local-variable *suite*) + +;;;; ** Creating Suits + +;; Suites that have no parent suites. +(defvar *toplevel-suites* nil) + +(defgeneric suite-emptyp (suite) + (:method ((suite symbol)) + (suite-emptyp (get-test suite))) + (:method ((suite test-suite)) + (= 0 (hash-table-count (tests suite))))) + +(defmacro def-suite (name &key description in) + "Define a new test-suite named NAME. + +IN (a symbol), if provided, causes this suite te be nested in the +suite named by IN. NB: This macro is built on top of make-suite, +as such it, like make-suite, will overrwrite any existing suite +named NAME." + `(eval-when (:compile-toplevel :load-toplevel :execute) + (make-suite ',name + ,@(when description `(:description ,description)) + ,@(when in `(:in ',in))) + ',name)) + +(defmacro def-suite* (name &rest def-suite-args) + `(progn + (def-suite ,name ,@def-suite-args) + (in-suite ,name))) + +(defun make-suite (name &key description ((:in parent-suite))) + "Create a new test suite object. + +Overrides any existing suite named NAME." + (let ((suite (make-instance 'test-suite :name name))) + (when description + (setf (description suite) description)) + (when (and name + (null (name *suite*)) + (null parent-suite)) + (pushnew name *toplevel-suites*)) + (loop for i in (ensure-list parent-suite) + for in-suite = (get-test i) + do (progn + (when (null in-suite) + (cerror "Create a new suite named ~A." "Unknown suite ~A." i) + (setf (get-test in-suite) (make-suite i) + in-suite (get-test in-suite))) + (setf (gethash name (tests in-suite)) suite))) + (setf (get-test name) suite) + suite)) + +(eval-when (:load-toplevel :execute) + (setf *suite* + (setf (get-test 'nil) + (make-suite 'nil :description "Global Suite")))) + +(defun list-all-suites () + "Returns an unordered LIST of all suites." + (hash-table-values *suite*)) + +;;;; ** Managing the Current Suite + +(defmacro in-suite (suite-name) + "Set the *suite* special variable so that all tests defined +after the execution of this form are, unless specified otherwise, +in the test-suite named SUITE-NAME. + +See also: DEF-SUITE *SUITE*" + `(eval-when (:compile-toplevel :load-toplevel :execute) + (%in-suite ,suite-name))) + +(defmacro in-suite* (suite-name &key in) + "Just like in-suite, but silently creates missing suites." + `(eval-when (:compile-toplevel :load-toplevel :execute) + (%in-suite ,suite-name :in ,in :fail-on-error nil))) + +(defmacro %in-suite (suite-name &key (fail-on-error t) in) + (with-gensyms (suite) + `(progn + (if-let (,suite (get-test ',suite-name)) + (setf *suite* ,suite) + (progn + (when ,fail-on-error + (cerror "Create a new suite named ~A." + "Unknown suite ~A." ',suite-name)) + (setf (get-test ',suite-name) (make-suite ',suite-name :in ',in) + *suite* (get-test ',suite-name)))) + ',suite-name))) + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/third_party/lisp/fiveam/src/test.lisp b/third_party/lisp/fiveam/src/test.lisp new file mode 100644 index 000000000000..4a6f2fee9a0a --- /dev/null +++ b/third_party/lisp/fiveam/src/test.lisp @@ -0,0 +1,167 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +;;;; * Tests + +;;;; While executing checks and collecting the results is the core job +;;;; of a testing framework it is also important to be able to +;;;; organize checks into groups, fiveam provides two mechanisms for +;;;; organizing checks: tests and test suites. A test is a named +;;;; collection of checks which can be run and a test suite is a named +;;;; collection of tests and test suites. + +(declaim (special *suite*)) + +(defvar *test* + (make-hash-table :test 'eql) + "Lookup table mapping test (and test suite) + names to objects.") + +(defun get-test (key &optional default) + (gethash key *test* default)) + +(defun (setf get-test) (value key) + (setf (gethash key *test*) value)) + +(defun rem-test (key) + (remhash key *test*)) + +(defun test-names () + (hash-table-keys *test*)) + +(defmacro test (name &body body) + "Create a test named NAME. If NAME is a list it must be of the +form: + + (name &key depends-on suite fixture compile-at profile) + +NAME is the symbol which names the test. + +DEPENDS-ON is a list of the form: + + (AND . test-names) - This test is run only if all of the tests + in TEST-NAMES have passed, otherwise a single test-skipped + result is generated. + + (OR . test-names) - If any of TEST-NAMES has passed this test is + run, otherwise a test-skipped result is generated. + + (NOT test-name) - This is test is run only if TEST-NAME failed. + +AND, OR and NOT can be combined to produce complex dependencies. + +If DEPENDS-ON is a symbol it is interpreted as `(AND +,depends-on), this is accomadate the common case of one test +depending on another. + +FIXTURE specifies a fixture to wrap the body in. + +If PROFILE is T profiling information will be collected as well." + (destructuring-bind (name &rest args) + (ensure-list name) + `(def-test ,name (,@args) ,@body))) + +(defvar *default-test-compilation-time* :definition-time) + +(defmacro def-test (name (&key depends-on (suite '*suite* suite-p) fixture + (compile-at *default-test-compilation-time*) profile) + &body body) + "Create a test named NAME. + +NAME is the symbol which names the test. + +DEPENDS-ON is a list of the form: + + (AND . test-names) - This test is run only if all of the tests + in TEST-NAMES have passed, otherwise a single test-skipped + result is generated. + + (OR . test-names) - If any of TEST-NAMES has passed this test is + run, otherwise a test-skipped result is generated. + + (NOT test-name) - This is test is run only if TEST-NAME failed. + +AND, OR and NOT can be combined to produce complex dependencies. + +If DEPENDS-ON is a symbol it is interpreted as `(AND +,depends-on), this is accomadate the common case of one test +depending on another. + +FIXTURE specifies a fixture to wrap the body in. + +If PROFILE is T profiling information will be collected as well." + (check-type compile-at (member :run-time :definition-time)) + (multiple-value-bind (forms decls docstring) + (parse-body body :documentation t :whole name) + (let* ((description (or docstring "")) + (body-forms (append decls forms)) + (suite-form (if suite-p + `(get-test ',suite) + (or suite '*suite*))) + (effective-body (if fixture + (destructuring-bind (name &rest args) + (ensure-list fixture) + `((with-fixture ,name ,args ,@body-forms))) + body-forms))) + `(progn + (register-test ',name ,description ',effective-body ,suite-form ',depends-on ,compile-at ,profile) + (when *run-test-when-defined* + (run! ',name)) + ',name)))) + +(defun register-test (name description body suite depends-on compile-at profile) + (let ((lambda-name + (format-symbol t "%~A-~A" '#:test name)) + (inner-lambda-name + (format-symbol t "%~A-~A" '#:inner-test name))) + (setf (get-test name) + (make-instance 'test-case + :name name + :runtime-package (find-package (package-name *package*)) + :test-lambda + (eval + `(named-lambda ,lambda-name () + ,@(ecase compile-at + (:run-time `((funcall + (let ((*package* (find-package ',(package-name *package*)))) + (compile ',inner-lambda-name + '(lambda () ,@body)))))) + (:definition-time body)))) + :description description + :depends-on depends-on + :collect-profiling-info profile)) + (setf (gethash name (tests suite)) name))) + +(defvar *run-test-when-defined* nil + "When non-NIL tests are run as soon as they are defined.") + +;; Copyright (c) 2002-2003, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/third_party/lisp/fiveam/src/utils.lisp b/third_party/lisp/fiveam/src/utils.lisp new file mode 100644 index 000000000000..49d552fa000e --- /dev/null +++ b/third_party/lisp/fiveam/src/utils.lisp @@ -0,0 +1,226 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +(defmacro dolist* ((iterator list &optional return-value) &body body) + "Like DOLIST but destructuring-binds the elements of LIST. + +If ITERATOR is a symbol then dolist* is just like dolist EXCEPT +that it creates a fresh binding." + (if (listp iterator) + (let ((i (gensym "DOLIST*-I-"))) + `(dolist (,i ,list ,return-value) + (destructuring-bind ,iterator ,i + ,@body))) + `(dolist (,iterator ,list ,return-value) + (let ((,iterator ,iterator)) + ,@body)))) + +(defun make-collector (&optional initial-value) + "Create a collector function. + +A Collector function will collect, into a list, all the values +passed to it in the order in which they were passed. If the +callector function is called without arguments it returns the +current list of values." + (let ((value initial-value) + (cdr (last initial-value))) + (lambda (&rest items) + (if items + (progn + (if value + (if cdr + (setf (cdr cdr) items + cdr (last items)) + (setf cdr (last items))) + (setf value items + cdr (last items))) + items) + value)))) + +(defun partitionx (list &rest lambdas) + (let ((collectors (mapcar (lambda (l) + (cons (if (and (symbolp l) + (member l (list :otherwise t) + :test #'string=)) + (constantly t) + l) + (make-collector))) + lambdas))) + (dolist (item list) + (block item + (dolist* ((test-func . collector-func) collectors) + (when (funcall test-func item) + (funcall collector-func item) + (return-from item))))) + (mapcar #'funcall (mapcar #'cdr collectors)))) + +;;;; ** Anaphoric conditionals + +(defmacro if-bind (var test &body then/else) + "Anaphoric IF control structure. + +VAR (a symbol) will be bound to the primary value of TEST. If +TEST returns a true value then THEN will be executed, otherwise +ELSE will be executed." + (assert (first then/else) + (then/else) + "IF-BIND missing THEN clause.") + (destructuring-bind (then &optional else) + then/else + `(let ((,var ,test)) + (if ,var ,then ,else)))) + +(defmacro aif (test then &optional else) + "Just like IF-BIND but the var is always IT." + `(if-bind it ,test ,then ,else)) + +;;;; ** Simple list matching based on code from Paul Graham's On Lisp. + +(defmacro acond2 (&rest clauses) + (if (null clauses) + nil + (with-gensyms (val foundp) + (destructuring-bind ((test &rest progn) &rest others) + clauses + `(multiple-value-bind (,val ,foundp) + ,test + (if (or ,val ,foundp) + (let ((it ,val)) + (declare (ignorable it)) + ,@progn) + (acond2 ,@others))))))) + +(defun varsymp (x) + (and (symbolp x) + (let ((name (symbol-name x))) + (and (>= (length name) 2) + (char= (char name 0) #\?))))) + +(defun binding (x binds) + (labels ((recbind (x binds) + (aif (assoc x binds) + (or (recbind (cdr it) binds) + it)))) + (let ((b (recbind x binds))) + (values (cdr b) b)))) + +(defun list-match (x y &optional binds) + (acond2 + ((or (eql x y) (eql x '_) (eql y '_)) + (values binds t)) + ((binding x binds) (list-match it y binds)) + ((binding y binds) (list-match x it binds)) + ((varsymp x) (values (cons (cons x y) binds) t)) + ((varsymp y) (values (cons (cons y x) binds) t)) + ((and (consp x) (consp y) (list-match (car x) (car y) binds)) + (list-match (cdr x) (cdr y) it)) + (t (values nil nil)))) + +(defun vars (match-spec) + (let ((vars nil)) + (labels ((find-vars (spec) + (cond + ((null spec) nil) + ((varsymp spec) (push spec vars)) + ((consp spec) + (find-vars (car spec)) + (find-vars (cdr spec)))))) + (find-vars match-spec)) + (delete-duplicates vars))) + +(defmacro list-match-case (target &body clauses) + (if clauses + (destructuring-bind ((test &rest progn) &rest others) + clauses + (with-gensyms (tgt binds success) + `(let ((,tgt ,target)) + (multiple-value-bind (,binds ,success) + (list-match ,tgt ',test) + (declare (ignorable ,binds)) + (if ,success + (let ,(mapcar (lambda (var) + `(,var (cdr (assoc ',var ,binds)))) + (vars test)) + (declare (ignorable ,@(vars test))) + ,@progn) + (list-match-case ,tgt ,@others)))))) + nil)) + +;;;; * def-special-environment + +(defun check-required (name vars required) + (dolist (var required) + (assert (member var vars) + (var) + "Unrecognized symbol ~S in ~S." var name))) + +(defmacro def-special-environment (name (&key accessor binder binder*) + &rest vars) + "Define two macros for dealing with groups or related special variables. + +ACCESSOR is defined as a macro: (defmacro ACCESSOR (VARS &rest +BODY)). Each element of VARS will be bound to the +current (dynamic) value of the special variable. + +BINDER is defined as a macro for introducing (and binding new) +special variables. It is basically a readable LET form with the +prorpe declarations appended to the body. The first argument to +BINDER must be a form suitable as the first argument to LET. + +ACCESSOR defaults to a new symbol in the same package as NAME +which is the concatenation of \"WITH-\" NAME. BINDER is built as +\"BIND-\" and BINDER* is BINDER \"*\"." + (unless accessor + (setf accessor (format-symbol (symbol-package name) "~A-~A" '#:with name))) + (unless binder + (setf binder (format-symbol (symbol-package name) "~A-~A" '#:bind name))) + (unless binder* + (setf binder* (format-symbol (symbol-package binder) "~A~A" binder '#:*))) + `(eval-when (:compile-toplevel :load-toplevel :execute) + (flet () + (defmacro ,binder (requested-vars &body body) + (check-required ',name ',vars (mapcar #'car requested-vars)) + `(let ,requested-vars + (declare (special ,@(mapcar #'car requested-vars))) + ,@body)) + (defmacro ,binder* (requested-vars &body body) + (check-required ',name ',vars (mapcar #'car requested-vars)) + `(let* ,requested-vars + (declare (special ,@(mapcar #'car requested-vars))) + ,@body)) + (defmacro ,accessor (requested-vars &body body) + (check-required ',name ',vars requested-vars) + `(locally (declare (special ,@requested-vars)) + ,@body)) + ',name))) + +;; Copyright (c) 2002-2006, Edward Marco Baringer +;; All rights reserved. +;; +;; Redistribution and use in source and binary forms, with or without +;; modification, are permitted provided that the following conditions are +;; met: +;; +;; - Redistributions of source code must retain the above copyright +;; notice, this list of conditions and the following disclaimer. +;; +;; - Redistributions in binary form must reproduce the above copyright +;; notice, this list of conditions and the following disclaimer in the +;; documentation and/or other materials provided with the distribution. +;; +;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names +;; of its contributors may be used to endorse or promote products +;; derived from this software without specific prior written permission. +;; +;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE diff --git a/third_party/lisp/fiveam/t/example.lisp b/third_party/lisp/fiveam/t/example.lisp new file mode 100644 index 000000000000..c949511a28cd --- /dev/null +++ b/third_party/lisp/fiveam/t/example.lisp @@ -0,0 +1,126 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +;;;; * FiveAM Example (poor man's tutorial) + +(asdf:oos 'asdf:load-op :fiveam) + +(defpackage :it.bese.fiveam.example + (:use :common-lisp + :it.bese.fiveam)) + +(in-package :it.bese.fiveam.example) + +;;;; First we need some functions to test. + +(defun add-2 (n) + (+ n 2)) + +(defun add-4 (n) + (+ n 4)) + +;;;; Now we need to create a test which makes sure that add-2 and add-4 +;;;; work as specified. + +;;;; we create a test named ADD-2 and supply a short description. +(test add-2 + "Test the ADD-2 function" ;; a short description + ;; the checks + (is (= 2 (add-2 0))) + (is (= 0 (add-2 -2)))) + +;;;; we can already run add-2. This will return the list of test +;;;; results, it should be a list of two test-passed objects. + +(run 'add-2) + +;;;; since we'd like to have some kind of readbale output we'll explain +;;;; the results + +(explain! (run 'add-2)) + +;;;; or we could do both at once: + +(run! 'add-2) + +;;;; So now we've defined and run a single test. Since we plan on +;;;; having more than one test and we'd like to run them together let's +;;;; create a simple test suite. + +(def-suite example-suite :description "The example test suite.") + +;;;; we could explictly specify that every test we create is in the the +;;;; example-suite suite, but it's easier to just change the default +;;;; suite: + +(in-suite example-suite) + +;;;; now we'll create a new test for the add-4 function. + +(test add-4 + (is (= 0 (add-4 -4)))) + +;;;; now let's run the test + +(run! 'add-4) + +;;;; we can get the same effect by running the suite: + +(run! 'example-suite) + +;;;; since we'd like both add-2 and add-4 to be in the same suite, let's +;;;; redefine add-2 to be in this suite: + +(test add-2 "Test the ADD-2 function" + (is (= 2 (add-2 0))) + (is (= 0 (add-2 -2)))) + +;;;; now we can run the suite and we'll see that both add-2 and add-4 +;;;; have been run (we know this since we no get 4 checks as opposed to +;;;; 2 as before. + +(run! 'example-suite) + +;;;; Just for fun let's see what happens when a test fails. Again we'll +;;;; redefine add-2, but add in a third, failing, check: + +(test add-2 "Test the ADD-2 function" + (is (= 2 (add-2 0))) + (is (= 0 (add-2 -2))) + (is (= 0 (add-2 0)))) + +;;;; Finally let's try out the specification based testing. + +(defun dummy-add (a b) + (+ a b)) + +(defun dummy-strcat (a b) + (concatenate 'string a b)) + +(test dummy-add + (for-all ((a (gen-integer)) + (b (gen-integer))) + ;; assuming we have an "oracle" to compare our function results to + ;; we can use it: + (is (= (+ a b) (dummy-add a b))) + ;; if we don't have an oracle (as in most cases) we just ensure + ;; that certain properties hold: + (is (= (dummy-add a b) + (dummy-add b a))) + (is (= a (dummy-add a 0))) + (is (= 0 (dummy-add a (- a)))) + (is (< a (dummy-add a 1))) + (is (= (* 2 a) (dummy-add a a))))) + +(test dummy-strcat + (for-all ((result (gen-string)) + (split-point (gen-integer :min 0 :max 10000) + (< split-point (length result)))) + (is (string= result (dummy-strcat (subseq result 0 split-point) + (subseq result split-point)))))) + +(test random-failure + (for-all ((result (gen-integer :min 0 :max 1))) + (is (plusp result)) + (is (= result 0)))) + +(run! 'example-suite) diff --git a/third_party/lisp/fiveam/t/tests.lisp b/third_party/lisp/fiveam/t/tests.lisp new file mode 100644 index 000000000000..ed1c565e7d4a --- /dev/null +++ b/third_party/lisp/fiveam/t/tests.lisp @@ -0,0 +1,280 @@ +;;;; -*- Mode: Lisp; indent-tabs-mode: nil -*- + +(in-package :it.bese.fiveam) + +(in-suite* :it.bese.fiveam) + +(def-suite test-suite :description "Suite for tests which should fail.") + +(defmacro with-test-results ((results test-name) &body body) + `(let ((,results (with-*test-dribble* nil (run ',test-name)))) + ,@body)) + +(def-fixture null-fixture () + `(progn ,@(&body))) + +;;;; Test the checks + +(def-test is1 (:suite test-suite) + (is (plusp 1)) + (is (< 0 1)) + (is (not (plusp -1))) + (is (not (< 1 0))) + (is-true t) + (is-false nil)) + +(def-test is2 (:suite test-suite :fixture null-fixture) + (is (plusp 0)) + (is (< 0 -1)) + (is (not (plusp 1))) + (is (not (< 0 1))) + (is-true nil) + (is-false t)) + +(def-test is (:profile t) + (with-test-results (results is1) + (is (= 6 (length results))) + (is (every #'test-passed-p results))) + (with-test-results (results is2) + (is (= 6 (length results))) + (is (every #'test-failure-p results)))) + +(def-test signals/finishes () + (signals error + (error "an error")) + (finishes + (signals error + (error "an error")))) + +(def-test pass () + (pass)) + +(def-test fail1 (:suite test-suite) + (fail "This is supposed to fail")) + +(def-test fail () + (with-test-results (results fail1) + (is (= 1 (length results))) + (is (test-failure-p (first results))))) + +;;;; non top level checks + +(def-test foo-bar () + (let ((state 0)) + (is (= 0 state)) + (is (= 1 (incf state))))) + +;;;; Test dependencies + +(def-test ok (:suite test-suite) + (pass)) + +(def-test not-ok (:suite test-suite) + (fail "This is supposed to fail.")) + +(def-test and1 (:depends-on (and ok not-ok) :suite test-suite) + (fail)) + +(def-test and2 (:depends-on (and ok) :suite test-suite) + (pass)) + +(def-test dep-and () + (with-test-results (results and1) + (is (= 3 (length results))) + ;; we should have one skippedw one failed and one passed + (is (some #'test-passed-p results)) + (is (some #'test-skipped-p results)) + (is (some #'test-failure-p results))) + (with-test-results (results and2) + (is (= 2 (length results))) + (is (every #'test-passed-p results)))) + +(def-test or1 (:depends-on (or ok not-ok) :suite test-suite) + (pass)) + +(def-test or2 (:depends-on (or not-ok ok) :suite test-suite) + (pass)) + +(def-test dep-or () + (with-test-results (results or1) + (is (= 2 (length results))) + (is (every #'test-passed-p results))) + (with-test-results (results or2) + (is (= 3 (length results))) + (is (= 2 (length (remove-if-not #'test-passed-p results)))))) + +(def-test not1 (:depends-on (not not-ok) :suite test-suite) + (pass)) + +(def-test not2 (:depends-on (not ok) :suite test-suite) + (fail)) + +(def-test not () + (with-test-results (results not1) + (is (= 2 (length results))) + (is (some #'test-passed-p results)) + (is (some #'test-failure-p results))) + (with-test-results (results not2) + (is (= 2 (length results))) + (is (some #'test-passed-p results)) + (is (some #'test-skipped-p results)))) + +(def-test nested-logic (:depends-on (and ok (not not-ok) (not not-ok)) + :suite test-suite) + (pass)) + +(def-test dep-nested () + (with-test-results (results nested-logic) + (is (= 3 (length results))) + (is (= 2 (length (remove-if-not #'test-passed-p results)))) + (is (= 1 (length (remove-if-not #'test-failure-p results)))))) + +(def-test circular-0 (:depends-on (and circular-1 circular-2 or1) + :suite test-suite) + (fail "we depend on a circular dependency, we should not be tested.")) + +(def-test circular-1 (:depends-on (and circular-2) + :suite test-suite) + (fail "we have a circular depednency, we should not be tested.")) + +(def-test circular-2 (:depends-on (and circular-1) + :suite test-suite) + (fail "we have a circular depednency, we should not be tested.")) + +(def-test circular () + (signals circular-dependency + (run 'circular-0)) + (signals circular-dependency + (run 'circular-1)) + (signals circular-dependency + (run 'circular-2))) + + +(defun stack-exhaust () + (declare (optimize (debug 3) (speed 0) (space 0) (safety 3))) + (cons 42 (stack-exhaust))) + +;; Disable until we determine on which implementations it's actually safe +;; to exhaust the stack. +#| +(def-test stack-exhaust (:suite test-suite) + (stack-exhaust)) + +(def-test test-stack-exhaust () + (with-test-results (results stack-exhaust) + (is (= 1 (length results))) + (is (test-failure-p (first results))))) +|# + +(def-suite before-test-suite :description "Suite for before test") + +(def-test before-0 (:suite before-test-suite) + (fail)) + +(def-test before-1 (:depends-on (:before before-0) + :suite before-test-suite) + (pass)) + +(def-suite before-test-suite-2 :description "Suite for before test") + +(def-test before-2 (:depends-on (:before before-3) + :suite before-test-suite-2) + (pass)) + +(def-test before-3 (:suite before-test-suite-2) + (pass)) + +(def-test before () + (with-test-results (results before-test-suite) + (is (some #'test-skipped-p results))) + + (with-test-results (results before-test-suite-2) + (is (every #'test-passed-p results)))) + + +;;;; dependencies with symbol +(def-test dep-with-symbol-first (:suite test-suite) + (pass)) + +(def-test dep-with-symbol-dependencies-not-met (:depends-on (not dep-with-symbol-first) + :suite test-suite) + (fail "Error in the test of the test, this should not ever happen")) + +(def-test dep-with-symbol-depends-on-ok (:depends-on dep-with-symbol-first :suite test-suite) + (pass)) + +(def-test dep-with-symbol-depends-on-failed-dependency (:depends-on dep-with-symbol-dependencies-not-met + :suite test-suite) + (fail "No, I should not be tested because I depend on a test that in its turn has a failed dependecy.")) + +(def-test dependencies-with-symbol () + (with-test-results (results dep-with-symbol-first) + (is (some #'test-passed-p results))) + + (with-test-results (results dep-with-symbol-depends-on-ok) + (is (some #'test-passed-p results))) + + (with-test-results (results dep-with-symbol-dependencies-not-met) + (is (some #'test-skipped-p results))) + + ;; No failure here, because it means the test was run. + (with-test-results (results dep-with-symbol-depends-on-failed-dependency) + (is (not (some #'test-failure-p results))))) + + +;;;; test for-all + +(def-test gen-integer () + (for-all ((a (gen-integer))) + (is (integerp a)))) + +(def-test for-all-guarded () + (for-all ((less (gen-integer)) + (more (gen-integer) (< less more))) + (is (< less more)))) + +(def-test gen-float () + (macrolet ((test-gen-float (type) + `(for-all ((unbounded (gen-float :type ',type)) + (bounded (gen-float :type ',type :bound 42))) + (is (typep unbounded ',type)) + (is (typep bounded ',type)) + (is (<= (abs bounded) 42))))) + (test-gen-float single-float) + (test-gen-float short-float) + (test-gen-float double-float) + (test-gen-float long-float))) + +(def-test gen-character () + (for-all ((c (gen-character))) + (is (characterp c))) + (for-all ((c (gen-character :code (gen-integer :min 32 :max 40)))) + (is (characterp c)) + (member c (list #\Space #\! #\" #\# #\$ #\% #\& #\' #\()))) + +(def-test gen-string () + (for-all ((s (gen-string))) + (is (stringp s))) + (for-all ((s (gen-string :length (gen-integer :min 0 :max 2)))) + (is (<= (length s) 2))) + (for-all ((s (gen-string :elements (gen-character :code (gen-integer :min 0 :max 0)) + :length (constantly 2)))) + (is (= 2 (length s))) + (is (every (curry #'char= #\Null) s)))) + +(defun dummy-mv-generator () + (lambda () + (list 1 1))) + +(def-test for-all-destructuring-bind () + (for-all (((a b) (dummy-mv-generator))) + (is (= 1 a)) + (is (= 1 b)))) + +(def-test return-values () + "Return values indicate test failures." + (is-true (with-*test-dribble* nil (explain! (run 'is1)))) + (is-true (with-*test-dribble* nil (run! 'is1))) + + (is-false (with-*test-dribble* nil (explain! (run 'is2)))) + (is-false (with-*test-dribble* nil (run! 'is2)))) diff --git a/third_party/lisp/fiveam/version.sexp b/third_party/lisp/fiveam/version.sexp new file mode 100644 index 000000000000..e0e0284e6761 --- /dev/null +++ b/third_party/lisp/fiveam/version.sexp @@ -0,0 +1,2 @@ +;; -*- lisp -*- +"1.4.1" diff --git a/third_party/lisp/flexi-streams.nix b/third_party/lisp/flexi-streams.nix new file mode 100644 index 000000000000..8cdf062f1cf2 --- /dev/null +++ b/third_party/lisp/flexi-streams.nix @@ -0,0 +1,34 @@ +# Flexible bivalent streams for Common Lisp +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/edicl/flexi-streams.git"; + rev = "0fd872ae32022e834ef861a67d86879cf33a6b64"; +}; +in depot.nix.buildLisp.library { + name = "flexi-streams"; + deps = [ depot.third_party.lisp.trivial-gray-streams ]; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "mapping.lisp" + "ascii.lisp" + "koi8-r.lisp" + "iso-8859.lisp" + "code-pages.lisp" + "specials.lisp" + "util.lisp" + "conditions.lisp" + "external-format.lisp" + "length.lisp" + "encode.lisp" + "decode.lisp" + "in-memory.lisp" + "stream.lisp" + "output.lisp" + "input.lisp" + "io.lisp" + "strings.lisp" + ]; +} + diff --git a/third_party/lisp/global-vars.nix b/third_party/lisp/global-vars.nix new file mode 100644 index 000000000000..2b4078f588cd --- /dev/null +++ b/third_party/lisp/global-vars.nix @@ -0,0 +1,14 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "lmj"; + repo = "global-vars"; + rev = "c749f32c9b606a1457daa47d59630708ac0c266e"; + sha256 = "06m3xc8l3pgsapl8fvsi9wf6y46zs75cp9zn7zh6dc65v4s5wz3d"; + }; + +in depot.nix.buildLisp.library { + name = "global-vars"; + srcs = [ "${src}/global-vars.lisp" ] ; +} diff --git a/third_party/lisp/hunchentoot.nix b/third_party/lisp/hunchentoot.nix new file mode 100644 index 000000000000..24eae6a348cf --- /dev/null +++ b/third_party/lisp/hunchentoot.nix @@ -0,0 +1,65 @@ +# Hunchentoot is a web framework for Common Lisp. +{ depot, pkgs, ...}: + +let + src = pkgs.fetchFromGitHub { + owner = "edicl"; + repo = "hunchentoot"; + rev = "585b45b6b873f2da421fdf456b61860ab5868207"; + sha256 = "13nazwix067mdclq9vgjhsi2vpr57a8dz51dd5d3h99ccsq4mik5"; + }; + url-rewrite = depot.nix.buildLisp.library { + name = "url-rewrite"; + + srcs = map (f: src + ("/url-rewrite/" + f)) [ + "packages.lisp" + "specials.lisp" + "primitives.lisp" + "util.lisp" + "url-rewrite.lisp" + ]; + }; +in depot.nix.buildLisp.library { + name = "hunchentoot"; + + deps = with depot.third_party.lisp; [ + alexandria + bordeaux-threads + chunga + cl-base64 + cl-fad + rfc2388 + cl-plus-ssl + cl-ppcre + flexi-streams + md5 + trivial-backtrace + usocket + url-rewrite + ]; + + srcs = map (f: src + ("/" + f)) [ + "hunchentoot.asd" + "packages.lisp" + "compat.lisp" + "specials.lisp" + "conditions.lisp" + "mime-types.lisp" + "util.lisp" + "log.lisp" + "cookie.lisp" + "reply.lisp" + "request.lisp" + "session.lisp" + "misc.lisp" + "headers.lisp" + "set-timeouts.lisp" + "taskmaster.lisp" + "acceptor.lisp" + "easy-handlers.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/ironclad.nix b/third_party/lisp/ironclad.nix new file mode 100644 index 000000000000..cdd829924ee5 --- /dev/null +++ b/third_party/lisp/ironclad.nix @@ -0,0 +1,155 @@ +{ depot, pkgs, ...}: + +let + inherit (pkgs) runCommand; + inherit (depot.nix.buildLisp) bundled; + src = pkgs.fetchFromGitHub { + owner = "sharplispers"; + repo = "ironclad"; + rev = "c3aa33080621abc10fdb0f34acc4655cc4e982a6"; + sha256 = "0k4bib9mbrzalbl9ivkw4a7g4c7bbad1l5jw4pzkifqszy2swkr5"; + }; + + getSrc = f: "${src}/src/${f}"; + +in depot.nix.buildLisp.library { + name = "ironclad"; + + deps = with depot.third_party.lisp; [ + (bundled "asdf") + { sbcl = bundled "sb-rotate-byte"; } + { sbcl = bundled "sb-posix"; } + alexandria + bordeaux-threads + nibbles + ]; + + srcs = [ + { + # TODO(grfn): Figure out how to get this compiling with the assembly + # optimization eventually - see https://cl.tvl.fyi/c/depot/+/1333 + sbcl = runCommand "package.lisp" {} '' + substitute ${src}/src/package.lisp $out \ + --replace \#-ecl-bytecmp "" \ + --replace '(pushnew :ironclad-assembly *features*)' "" + ''; + default = getSrc "package.lisp"; + } + ] ++ map getSrc [ + "macro-utils.lisp" + ] ++ [ + { sbcl = getSrc "opt/sbcl/fndb.lisp"; } + { sbcl = getSrc "opt/sbcl/cpu-features.lisp"; } + { sbcl = getSrc "opt/sbcl/x86oid-vm.lisp"; } + + { ecl = getSrc "opt/ecl/c-functions.lisp"; } + + { ccl = getSrc "opt/ccl/x86oid-vm.lisp"; } + ] ++ map getSrc [ + + "common.lisp" + "conditions.lisp" + "generic.lisp" + "util.lisp" + + "ciphers/padding.lisp" + "ciphers/cipher.lisp" + "ciphers/chacha.lisp" + "ciphers/modes.lisp" + "ciphers/salsa20.lisp" + "ciphers/xchacha.lisp" + "ciphers/xsalsa20.lisp" + "ciphers/aes.lisp" + "ciphers/arcfour.lisp" + "ciphers/arcfour.lisp" + "ciphers/aria.lisp" + "ciphers/blowfish.lisp" + "ciphers/camellia.lisp" + "ciphers/cast5.lisp" + "ciphers/des.lisp" + "ciphers/idea.lisp" + "ciphers/keystream.lisp" + "ciphers/kalyna.lisp" + "ciphers/kuznyechik.lisp" + "ciphers/make-cipher.lisp" + "ciphers/misty1.lisp" + "ciphers/rc2.lisp" + "ciphers/rc5.lisp" + "ciphers/rc6.lisp" + "ciphers/seed.lisp" + "ciphers/serpent.lisp" + "ciphers/sm4.lisp" + "ciphers/sosemanuk.lisp" + "ciphers/square.lisp" + "ciphers/tea.lisp" + "ciphers/threefish.lisp" + "ciphers/twofish.lisp" + "ciphers/xor.lisp" + "ciphers/xtea.lisp" + + "digests/digest.lisp" + "digests/adler32.lisp" + "digests/blake2.lisp" + "digests/blake2s.lisp" + "digests/crc24.lisp" + "digests/crc32.lisp" + "digests/groestl.lisp" + "digests/jh.lisp" + "digests/kupyna.lisp" + "digests/md2.lisp" + "digests/md4.lisp" + "digests/md5.lisp" + "digests/md5-lispworks-int32.lisp" + "digests/ripemd-128.lisp" + "digests/ripemd-160.lisp" + "digests/sha1.lisp" + "digests/sha256.lisp" + "digests/sha3.lisp" + "digests/sha512.lisp" + "digests/skein.lisp" + "digests/sm3.lisp" + "digests/streebog.lisp" + "digests/tiger.lisp" + "digests/tree-hash.lisp" + "digests/whirlpool.lisp" + + "prng/prng.lisp" + "prng/generator.lisp" + "prng/fortuna.lisp" + "prng/os-prng.lisp" + + "math.lisp" + + "macs/mac.lisp" + "macs/blake2-mac.lisp" + "macs/blake2s-mac.lisp" + "macs/cmac.lisp" + "macs/hmac.lisp" + "macs/gmac.lisp" + "macs/poly1305.lisp" + "macs/siphash.lisp" + "macs/skein-mac.lisp" + + "kdf/kdf-common.lisp" + "kdf/argon2.lisp" + "kdf/password-hash.lisp" + "kdf/pkcs5.lisp" + "kdf/scrypt.lisp" + "kdf/hmac.lisp" + + "aead/aead.lisp" + "aead/eax.lisp" + "aead/etm.lisp" + "aead/gcm.lisp" + + "public-key/public-key.lisp" + "public-key/curve25519.lisp" + "public-key/curve448.lisp" + "public-key/dsa.lisp" + "public-key/ed25519.lisp" + "public-key/ed448.lisp" + "public-key/elgamal.lisp" + "public-key/pkcs1.lisp" + "public-key/rsa.lisp" + ]; +} diff --git a/third_party/lisp/iterate.nix b/third_party/lisp/iterate.nix new file mode 100644 index 000000000000..5875be8d5c88 --- /dev/null +++ b/third_party/lisp/iterate.nix @@ -0,0 +1,15 @@ +# iterate is an iteration construct for Common Lisp, similar to the +# LOOP macro. +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://gitlab.common-lisp.net/iterate/iterate.git"; + rev = "c24f6715bb3b962499bb4643636baaac2df4a957"; # 2021-05-23, 1.5.3 +}; +in depot.nix.buildLisp.library { + name = "iterate"; + srcs = [ + "${src}/package.lisp" + "${src}/iterate.lisp" + ]; +} diff --git a/third_party/lisp/lass.nix b/third_party/lisp/lass.nix new file mode 100644 index 000000000000..457e25c7e532 --- /dev/null +++ b/third_party/lisp/lass.nix @@ -0,0 +1,34 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "Shinmera"; + repo = "LASS"; + rev = "f51b9e941ee0a2a1f76ba814dcef22f9fb5f69bf"; + sha256 = "11mxzyx34ynsfsrs8pgrarqi9s442vkpmh7kdpzvarhj7i97g8yx"; + }; + +in depot.nix.buildLisp.library { + name = "lass"; + + deps = with depot.third_party.lisp; [ + trivial-indent + trivial-mimes + physical-quantities + parse-float + cl-base64 + (depot.nix.buildLisp.bundled "asdf") + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "readable-list.lisp" + "compiler.lisp" + "property-funcs.lisp" + "writer.lisp" + "lass.lisp" + "special.lisp" + "units.lisp" + "asdf.lisp" + ]; +} diff --git a/third_party/lisp/lisp-binary.nix b/third_party/lisp/lisp-binary.nix new file mode 100644 index 000000000000..3e7a43b8ac67 --- /dev/null +++ b/third_party/lisp/lisp-binary.nix @@ -0,0 +1,35 @@ +# A library to easily read and write complex binary formats. +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "j3pic"; + repo = "lisp-binary"; + rev = "052df578900dea59bf951e0a6749281fa73432e4"; + sha256 = "1i1s5g01aimfq6lndcl1pnw7ly5hdh0wmjp2dj9cjjwbkz9lnwcf"; +}; +in depot.nix.buildLisp.library { + name = "lisp-binary"; + + deps = with depot.third_party.lisp; [ + cffi + quasiquote_2 + moptilities + flexi-streams + closer-mop + ]; + + srcs = map (f: src + ("/" + f)) [ + "utils.lisp" + "integer.lisp" + "float.lisp" + "simple-bit-stream.lisp" + "reverse-stream.lisp" + "binary-1.lisp" + "binary-2.lisp" + "types.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/local-time.nix b/third_party/lisp/local-time.nix new file mode 100644 index 000000000000..65fb9c37fbf2 --- /dev/null +++ b/third_party/lisp/local-time.nix @@ -0,0 +1,18 @@ +# Library for manipulating dates & times +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "dlowe-net"; + repo = "local-time"; + rev = "dc54f61415c76ee755a6f69d4154a3a282f2789f"; + sha256 = "1l9v07ghx7g9p2gp003fki4j8bsa1w2gbm40qc41i94mdzikc0ry"; +}; +in depot.nix.buildLisp.library { + name = "local-time"; + deps = [ depot.third_party.lisp.cl-fad ]; + + srcs = [ + "${src}/src/package.lisp" + "${src}/src/local-time.lisp" + ]; +} diff --git a/third_party/lisp/marshal.nix b/third_party/lisp/marshal.nix new file mode 100644 index 000000000000..711e6e082d61 --- /dev/null +++ b/third_party/lisp/marshal.nix @@ -0,0 +1,20 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "wlbr"; + repo = "cl-marshal"; + rev = "eff1b15f2b0af2f26f71ad6a4dd5c4beab9299ec"; + sha256 = "08qs6fhk38xpkkjkpcj92mxx0lgy4ygrbbzrmnivdx281syr0gwh"; + }; + +in depot.nix.buildLisp.library { + name = "marshal"; + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "serialization-format.lisp" + "coding-idiom.lisp" + "marshal.lisp" + "unmarshal.lisp" + ]; +} diff --git a/third_party/lisp/md5.nix b/third_party/lisp/md5.nix new file mode 100644 index 000000000000..ef265d5b6e8f --- /dev/null +++ b/third_party/lisp/md5.nix @@ -0,0 +1,21 @@ +# MD5 hash implementation +{ depot, pkgs, ... }: + +with depot.nix; + +let src = pkgs.fetchFromGitHub { + owner = "pmai"; + repo = "md5"; + rev = "b1412600f60d526ee34a7ba1596ec483da7894ab"; + sha256 = "0lzip6b6xg7gd70xl1xmqp24fvxqj6ywjnz9lmx7988zpj20nhl2"; +}; +in buildLisp.library { + name = "md5"; + deps = [ + { + sbcl = buildLisp.bundled "sb-rotate-byte"; + default = depot.third_party.lisp.flexi-streams; + } + ]; + srcs = [ (src + "/md5.lisp") ]; +} diff --git a/third_party/lisp/mime4cl/.skip-subtree b/third_party/lisp/mime4cl/.skip-subtree new file mode 100644 index 000000000000..5051f60d6b86 --- /dev/null +++ b/third_party/lisp/mime4cl/.skip-subtree @@ -0,0 +1 @@ +prevent readTree from creating entries for subdirs that don't contain an .nix files diff --git a/third_party/lisp/mime4cl/OWNERS b/third_party/lisp/mime4cl/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/third_party/lisp/mime4cl/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/third_party/lisp/mime4cl/README b/third_party/lisp/mime4cl/README new file mode 100644 index 000000000000..73f0efbda9d5 --- /dev/null +++ b/third_party/lisp/mime4cl/README @@ -0,0 +1,7 @@ +MIME4CL is a Common Lisp library for dealing with MIME messages. +It has originally been written by Walter C. Pelissero and vendored +into depot as upstream has become inactive and provides no repo +of any kind. Upstream and depot version may diverge. + +Upstream Website: http://wcp.sdf-eu.org/software/#mime4cl +Vendored Tarball: http://wcp.sdf-eu.org/software/mime4cl-20150207T211851.tbz diff --git a/third_party/lisp/mime4cl/address.lisp b/third_party/lisp/mime4cl/address.lisp new file mode 100644 index 000000000000..9a3bec9b2078 --- /dev/null +++ b/third_party/lisp/mime4cl/address.lisp @@ -0,0 +1,301 @@ +;;; address.lisp --- e-mail address parser + +;;; Copyright (C) 2007, 2008, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +#+cmu (ext:file-comment "$Module: address.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +;;; Although not MIME specific, this parser is often useful together +;;; with the MIME primitives. It should be able to parse the address +;;; syntax described in RFC2822 excluding the obsolete syntax (see +;;; RFC822). Have a look at the test suite to get an idea of what +;;; kind of addresses it can parse. + +(in-package :mime4cl) + +(defstruct (mailbox (:conc-name mbx-)) + description + user + host + domain) + +(defstruct (mailbox-group (:conc-name mbxg-)) + name + mailboxes) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun write-mailbox-domain-name (addr &optional (stream *standard-output*)) + (when (eq :internet (mbx-domain addr)) + (write-char #\[ stream)) + (write-string (mbx-host addr) stream) + (when (eq :internet (mbx-domain addr)) + (write-char #\] stream)) + (when (stringp (mbx-domain addr)) + (write-char #\. stream) + (write-string (mbx-domain addr) stream))) + +(defun write-mailbox-address (addr &optional (stream *standard-output*)) + (write-string (mbx-user addr) stream) + (when (mbx-host addr) + (write-char #\@ stream) + (write-mailbox-domain-name addr stream))) + +(defmethod mbx-domain-name ((MBX mailbox)) + "Return the complete domain name string of MBX, in the form +\"host.domain\"." + (with-output-to-string (out) + (write-mailbox-domain-name mbx out))) + +(defmethod mbx-address ((mbx mailbox)) + "Return the e-mail address string of MBX, in the form +\"user@host.domain\"." + (with-output-to-string (out) + (write-mailbox-address mbx out))) + +(defun write-mailbox (addr &optional (stream *standard-output*)) + (awhen (mbx-description addr) + (write it :stream stream :readably t) + (write-string " <" stream)) + (write-mailbox-address addr stream) + (awhen (mbx-description addr) + (write-char #\> stream))) + +(defun write-mailbox-group (grp &optional (stream *standard-output*)) + (write-string (mbxg-name grp) stream) + (write-string ": " stream) + (loop + for mailboxes on (mbxg-mailboxes grp) + for mailbox = (car mailboxes) + do (write-mailbox mailbox stream) + unless (endp (cdr mailboxes)) + do (write-string ", " stream)) + (write-char #\; stream)) + +(defmethod print-object ((mbx mailbox) stream) + (if (or *print-readably* *print-escape*) + (call-next-method) + (write-mailbox mbx stream))) + +(defmethod print-object ((grp mailbox-group) stream) + (if (or *print-readably* *print-escape*) + (call-next-method) + (write-mailbox-group grp stream))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun parser-make-mailbox (description address-list) + (make-mailbox :description description + :user (car address-list) + :host (cadr address-list) + :domain (when (cddr address-list) + (string-concat (cddr address-list) ".")))) + + +(defun populate-grammar () + (defrule address-list + := (+ address ",")) + + (defrule address + := mailbox + := group) + + (defrule mailbox + := display-name? angle-addr comment? + :reduce (parser-make-mailbox (or display-name comment) angle-addr) + := addr-spec comment? + :reduce (parser-make-mailbox comment addr-spec)) + + (defrule angle-addr + := "<" addr-spec ">") + + (defrule group + := display-name ":" mailbox-list ";" + :reduce (make-mailbox-group :name display-name :mailboxes mailbox-list)) + + (defrule display-name + := phrase + :reduce (string-concat phrase " ")) + + (defrule phrase + := word+) + + (defrule word + := atext + := string) + + (defrule mailbox-list + := (+ mailbox ",")) + + (defrule addr-spec + := local-part "@" domain :reduce (cons local-part domain)) + + (defrule local-part + := dot-atom :reduce (string-concat dot-atom ".") + := string) + + (defrule domain + := dot-atom + := domain-literal :reduce (list domain-literal :internet)) + + ;; actually, according to the RFC, dot-atoms don't allow spaces in + ;; between but these rules do + (defrule dot-atom + := (+ atom ".")) + + (defrule atom + := atext+ + :reduce (apply #'concatenate 'string atext))) + +(deflazy define-grammar + (let ((*package* #.*package*) + (*compile-print* (when npg::*debug* t))) + (reset-grammar) + (format t "~&creating e-mail address grammar...~%") + (populate-grammar) + (let ((grammar (npg:generate-grammar #'string=))) + (reset-grammar) + (npg:print-grammar-figures grammar) + grammar))) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; The lexical analyser + +(defstruct cursor + stream + (position 0)) + +(defun read-delimited-string (stream end-char &key nesting-start-char (escape-char #\\)) + (labels ((collect () + (with-output-to-string (out) + (loop + for c = (read-char stream nil) + while (and c (not (char= c end-char))) + do (cond ((char= c escape-char) + (awhen (read-char stream nil) + (write-char it out))) + ((and nesting-start-char + (char= c nesting-start-char)) + (write-char nesting-start-char out) + (write-string (collect) out) + (write-char end-char out)) + (t (write-char c out))))))) + (collect))) + + +(defun read-string (cursor) + (make-token :type 'string + :value (read-delimited-string (cursor-stream cursor) #\") + :position (incf (cursor-position cursor)))) + +(defun read-domain-literal (cursor) + (make-token :type 'domain-literal + :value (read-delimited-string (cursor-stream cursor) #\]) + :position (incf (cursor-position cursor)))) + +(defun read-comment (cursor) + (make-token :type 'comment + :value (read-delimited-string (cursor-stream cursor) #\) :nesting-start-char #\() + :position (incf (cursor-position cursor)))) + +(declaim (inline atom-component-p)) +(defun atom-component-p (c) + (declare (type character c)) + (not (find c " ()\"[]@.<>:;,"))) + +(defun read-atext (first-character cursor) + (be string (with-output-to-string (out) + (write-char first-character out) + (loop + for c = (read-char (cursor-stream cursor) nil) + while (and c (atom-component-p c)) + do (write-char c out) + finally (when c + (unread-char c (cursor-stream cursor))))) + (make-token :type 'atext + :value string + :position (incf (cursor-position cursor))))) + +(defmethod read-next-tokens ((cursor cursor)) + (flet ((make-keyword (c) + (make-token :type 'keyword + :value (string c) + :position (incf (cursor-position cursor))))) + (be in (cursor-stream cursor) + (loop + for c = (read-char in nil) + while c + unless (whitespace-p c) + return (list + (cond ((char= #\( c) + (read-comment cursor)) + ((char= #\" c) + (read-string cursor)) + ((char= #\[ c) + (read-domain-literal cursor)) + ((find c "@.<>:;,") + (make-keyword c)) + (t + ;; anything else is considered a text atom even + ;; though it's just a single character + (read-atext c cursor)))))))) + +(defun analyse-string (string) + "Return the list of tokens produced by a lexical analysis of +STRING. These are the tokens that would be seen by the parser." + (with-input-from-string (stream string) + (be cursor (make-cursor :stream stream) + (loop + for tokens = (read-next-tokens cursor) + until (endp tokens) + append tokens)))) + +(defun mailboxes-only (list-of-mailboxes-and-groups) + "Return a flat list of MAILBOX-ADDRESSes from +LIST-OF-MAILBOXES-AND-GROUPS, which is the kind of list returned +by PARSE-ADDRESSES. This turns out to be useful when your +program is not interested in mailbox groups and expects the user +addresses only." + (mapcan #'(lambda (mbx) + (if (typep mbx 'mailbox-group) + (mbxg-mailboxes mbx) + (list mbx))) + list-of-mailboxes-and-groups)) + +(defun parse-addresses (string &key no-groups) + "Parse STRING and return a list of MAILBOX-ADDRESSes or +MAILBOX-GROUPs. If STRING is unparsable return NIL. If +NO-GROUPS is true, return a flat list of mailboxes throwing away +the group containers, if any." + (be grammar (force define-grammar) + (with-input-from-string (stream string) + (be* cursor (make-cursor :stream stream) + mailboxes (ignore-errors ; ignore parsing errors + (parse grammar 'address-list cursor)) + (if no-groups + (mailboxes-only mailboxes) + mailboxes))))) + +(defun debug-addresses (string) + "More or less like PARSE-ADDRESSES, but don't ignore parsing errors." + (be grammar (force define-grammar) + (with-input-from-string (stream string) + (be cursor (make-cursor :stream stream) + (parse grammar 'address-list cursor))))) + diff --git a/third_party/lisp/mime4cl/default.nix b/third_party/lisp/mime4cl/default.nix new file mode 100644 index 000000000000..5165774074c1 --- /dev/null +++ b/third_party/lisp/mime4cl/default.nix @@ -0,0 +1,48 @@ +# Copyright (C) 2021 by the TVL Authors +# SPDX-License-Identifier: LGPL-2.1-or-later +{ depot, pkgs, ... }: + +depot.nix.buildLisp.library { + name = "mime4cl"; + + deps = [ + depot.third_party.lisp.sclf + depot.third_party.lisp.npg + ]; + + srcs = [ + ./package.lisp + ./endec.lisp + ./streams.lisp + ./mime.lisp + ./address.lisp + ]; + + tests = { + name = "mime4cl-tests"; + + srcs = [ + ./test/rt.lisp + ./test/package.lisp + (pkgs.writeText "nix-samples.lisp" '' + (in-package :mime4cl-tests) + + ;; missing from the tarball completely + (defvar *samples-directory* (pathname "/this/does/not/exist")) + ;; override auto discovery which doesn't work in store + (defvar *sample1-file* (pathname "${./test/sample1.msg}")) + '') + ./test/endec.lisp + ./test/address.lisp + ./test/mime.lisp + ]; + + expression = "(rtest:do-tests)"; + }; + + # limited by sclf + brokenOn = [ + "ccl" + "ecl" + ]; +} diff --git a/third_party/lisp/mime4cl/endec.lisp b/third_party/lisp/mime4cl/endec.lisp new file mode 100644 index 000000000000..f63eb3c22e1d --- /dev/null +++ b/third_party/lisp/mime4cl/endec.lisp @@ -0,0 +1,683 @@ +;;; endec.lisp --- encoder/decoder functions + +;;; Copyright (C) 2005-2008, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + + +(in-package :mime4cl) + + +;; Thank you SBCL for rendering constants totally useless! +(defparameter +base64-encode-table+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=") + +(defparameter +base64-decode-table+ + (let ((da (make-array 256 :element-type '(unsigned-byte 8) :initial-element 65))) + (dotimes (i 64) + (setf (aref da (char-code (char +base64-encode-table+ i))) i)) + da)) + +(declaim (type (simple-array (unsigned-byte 8)) +base64-decode-table+) + (type simple-string +base64-encode-table+)) + +(defvar *base64-line-length* 76 + "Maximum length of the encoded base64 line. NIL means it can +be of unlimited length \(no line breaks will be done by the +encoding function).") + +(defvar *quoted-printable-line-length* 72 + "Maximum length of the encoded quoted printable line. NIL +means it can be of unlimited length \(no line breaks will be done +by the encoding function).") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass decoder () + ((input-function :initarg :input-function + :reader decoder-input-function + :type function + :documentation + "Function is called repeatedly by the decoder methods to get the next character. +It should return a character os NIL (indicating EOF).")) + (:documentation + "Abstract base class for decoders.")) + +(defclass parsing-decoder (decoder) + ((parser-errors :initform nil + :initarg :parser-errors + :reader decoder-parser-errors + :type boolean)) + (:documentation + "Abstract base class for decoders that do parsing.")) + +(defclass encoder () + ((output-function :initarg :output-function + :reader encoder-output-function + :type function + :documentation + "Function is called repeatedly by the encoder methods to output a character. +It should expect a character as its only argument.")) + (:documentation + "Abstract base class for encoders.")) + +(defclass line-encoder (encoder) + ((column :initform 0 + :type fixnum) + (line-length :initarg :line-length + :initform nil + :reader encoder-line-length + :type (or fixnum null))) + (:documentation + "Abstract base class for line encoders.")) + +(defclass 8bit-decoder (decoder) + () + (:documentation + "Class for decoders that do nothing.")) + +(defclass 8bit-encoder (encoder) + () + (:documentation + "Class for encoders that do nothing.")) + +(defclass 7bit-decoder (decoder) + () + (:documentation + "Class for decoders that do nothing.")) + +(defclass 7bit-encoder (encoder) + () + (:documentation + "Class for encoders that do nothing.")) + +(defclass byte-decoder (decoder) + () + (:documentation + "Class for decoders that turns chars to bytes.")) + +(defclass byte-encoder (encoder) + () + (:documentation + "Class for encoders that turns bytes to chars.")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric encoder-write-byte (encoder byte)) +(defgeneric encoder-finish-output (encoder)) +(defgeneric decoder-read-byte (decoder)) + +(defmethod encoder-finish-output ((encoder encoder)) + (values)) + +(defmethod encoder-write-byte ((encoder 8bit-encoder) byte) + (funcall (slot-value encoder 'output-function) + (code-char byte)) + (values)) + +(defmethod decoder-read-byte ((decoder 8bit-decoder)) + (awhen (funcall (slot-value decoder 'input-function)) + (char-code it))) + +(defmethod encoder-write-byte ((encoder 7bit-encoder) byte) + (funcall (slot-value encoder 'output-function) + (code-char (logand #x7F byte))) + (values)) + +(defmethod decoder-read-byte ((decoder 7bit-decoder)) + (awhen (funcall (slot-value decoder 'input-function)) + (logand #x7F (char-code it)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun decoder-read-sequence (sequence decoder &key (start 0) (end (length sequence))) + (declare (optimize (speed 3) (safety 0) (debug 0)) + (type fixnum start end) + (type vector sequence)) + (loop + for i fixnum from start below end + for byte = (decoder-read-byte decoder) + while byte + do (setf (aref sequence i) byte) + finally (return i))) + +(defun decoder-read-line (decoder) + (with-output-to-string (str) + (loop + for byte = (decoder-read-byte decoder) + unless byte + do (return-from decoder-read-line nil) + do (be c (code-char byte) + (cond ((char= c #\return) + ;; skip the newline + (decoder-read-byte decoder) + (return nil)) + ((char= c #\newline) + ;; the #\return was missing + (return nil)) + (t (write-char c str))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(declaim (inline parse-hex)) +(defun parse-hex (c1 c2) + "Parse two characters as hexadecimal and return their combined +value." + (declare (optimize (speed 3) (safety 0) (debug 0)) + (type character c1 c2)) + (flet ((digit-value (char) + (or (position char "0123456789ABCDEF") + (return-from parse-hex nil)))) + (+ (* 16 (digit-value c1)) + (digit-value c2)))) + +(defclass quoted-printable-decoder (parsing-decoder) + ((saved-bytes :initform (make-queue)))) + +(defmethod decoder-read-byte ((decoder quoted-printable-decoder)) + (declare (optimize (speed 3) (safety 0) (debug 0))) + (with-slots (input-function saved-bytes parser-errors) decoder + (declare (type function input-function)) + (labels ((saveb (b) + (queue-append saved-bytes b) + (values)) + (save (c) + (saveb (char-code c))) + (push-next () + (be c (funcall input-function) + (declare (type (or null character) c)) + (cond ((not c)) + ((or (char= c #\space) + (char= c #\tab)) + (save c) + (push-next)) + ((char= c #\=) + (be c1 (funcall input-function) + (cond ((not c1) + (save #\=)) + ((char= c1 #\return) + ;; soft line break: skip the next + ;; character which we assume to be a + ;; newline (pity if it isn't) + (funcall input-function) + (push-next)) + ((char= c1 #\newline) + ;; soft line break: the #\return is + ;; missing, but we are tolerant + (push-next)) + (t + ;; hexadecimal sequence: get the 2nd digit + (be c2 (funcall input-function) + (if c2 + (aif (parse-hex c1 c2) + (saveb it) + (if parser-errors + (error "invalid hex sequence ~A~A" c1 c2) + (progn + (save #\=) + (save c1) + (save c2)))) + (progn + (save c) + (save c1)))))))) + (t + (save c)))))) + (or (queue-pop saved-bytes) + (progn + (push-next) + (queue-pop saved-bytes)))))) + +(defmacro make-encoder-loop (encoder-class input-form output-form) + (with-gensyms (encoder byte) + `(loop + with ,encoder = (make-instance ',encoder-class + :output-function #'(lambda (char) ,output-form)) + for ,byte = ,input-form + while ,byte + do (encoder-write-byte ,encoder ,byte) + finally (encoder-finish-output ,encoder)))) + +(defmacro make-decoder-loop (decoder-class input-form output-form &key parser-errors) + (with-gensyms (decoder) + `(loop + with ,decoder = (make-instance ',decoder-class + :input-function #'(lambda () ,input-form) + :parser-errors ,parser-errors) + for byte = (decoder-read-byte ,decoder) + while byte + do ,output-form))) + +(defun decode-quoted-printable-stream (in out &key parser-errors) + "Read from stream IN a quoted printable text and write to +binary output OUT the decoded stream of bytes." + (make-decoder-loop quoted-printable-decoder + (read-byte in nil) (write-byte byte out) + :parser-errors parser-errors)) + +(defmacro make-stream-to-sequence-decoder (decoder-class input-form &key parser-errors) + "Decode the character stream STREAM and return a sequence of bytes." + (with-gensyms (output-sequence) + `(be ,output-sequence (make-array 0 + :element-type '(unsigned-byte 8) + :fill-pointer 0 + :adjustable t) + (make-decoder-loop ,decoder-class ,input-form + (vector-push-extend byte ,output-sequence) + :parser-errors ,parser-errors) + ,output-sequence))) + +(defun decode-quoted-printable-stream-to-sequence (stream &key parser-errors) + "Read from STREAM a quoted printable text and return a vector of +bytes." + (make-stream-to-sequence-decoder quoted-printable-decoder + (read-char stream nil) + :parser-errors parser-errors)) + +(defun decode-quoted-printable-string (string &key (start 0) (end (length string)) parser-errors) + "Decode STRING as quoted printable sequence of characters and +return a decoded sequence of bytes." + (with-input-from-string (in string :start start :end end) + (decode-quoted-printable-stream-to-sequence in :parser-errors parser-errors))) + +(defclass quoted-printable-encoder (line-encoder) + ((line-length :initform *quoted-printable-line-length* + :type (or fixnum null)) + (pending-space :initform nil + :type boolean))) + +(defmethod encoder-write-byte ((encoder quoted-printable-encoder) byte) + (declare (optimize (speed 3) (safety 0) (debug 0)) + (type (unsigned-byte 8) byte)) + (with-slots (output-function column pending-space line-length) encoder + (declare (type function output-function) + (type fixnum column) + (type (or fixnum null) line-length) + (type boolean pending-space)) + (labels ((out (c) + (funcall output-function c) + (values)) + (outs (str) + (declare (type simple-string str)) + (loop + for c across str + do (out c)) + (values)) + (out2hex (x) + (declare (type fixnum x)) + (multiple-value-bind (a b) (truncate x 16) + (out (digit-char a 16)) + (out (digit-char b 16))))) + (cond ((= byte #.(char-code #\newline)) + (when pending-space + (outs "=20") + (setf pending-space nil)) + (out #\newline) + (setf column 0)) + ((= byte #.(char-code #\space)) + (if pending-space + (progn + (out #\space) + (f++ column)) + (setf pending-space t))) + (t + (when pending-space + (out #\space) + (f++ column) + (setf pending-space nil)) + (cond ((or (< byte 32) + (= byte #.(char-code #\=)) + (> byte 126)) + (out #\=) + (out2hex byte) + (f++ column 3)) + (t + (out (code-char byte)) + (f++ column))))) + (when (and line-length + (>= column line-length)) + ;; soft line break + (outs #.(coerce '(#\= #\newline) 'string)) + (setf column 0))))) + +(defmethod encoder-finish-output ((encoder quoted-printable-encoder)) + (declare (optimize (speed 3) (safety 0) (debug 0))) + (with-slots (pending-space output-function) encoder + (declare (type boolean pending-space) + (type function output-function)) + (when pending-space + (flet ((outs (s) + (declare (type simple-string s)) + (loop + for c across s + do (funcall output-function c)))) + (setf pending-space nil) + (outs "=20"))))) + +(defun encode-quoted-printable-stream (in out) + "Read from IN a stream of bytes and write to OUT a stream of +characters quoted printables encoded." + (make-encoder-loop quoted-printable-encoder + (read-byte in nil) + (write-char char out))) + +(defun encode-quoted-printable-sequence-to-stream (sequence stream &key (start 0) (end (length sequence))) + "Encode the sequence of bytes SEQUENCE and write to STREAM a +quoted printable sequence of characters." + (be i start + (make-encoder-loop quoted-printable-encoder + (when (< i end) + (prog1 (elt sequence i) + (f++ i))) + (write-char char stream)))) + +(defun encode-quoted-printable-sequence (sequence &key (start 0) (end (length sequence))) + "Encode the sequence of bytes SEQUENCE into a quoted printable +string and return it." + (with-output-to-string (out) + (encode-quoted-printable-sequence-to-stream sequence out :start start :end end))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass base64-encoder (line-encoder) + ((line-length :initform *base64-line-length*) + (bitstore :initform 0 + :type fixnum) + (bytecount :initform 0 + :type fixnum)) + (:documentation + "Class for Base64 encoder output streams.")) + + +(eval-when (:load-toplevel :compile-toplevel) + (unless (> most-positive-fixnum (expt 2 (* 8 3))))) + +(macrolet ((with-encoder (encoder &body forms) + `(with-slots (bitstore line-length column bytecount output-function) ,encoder + (declare (type fixnum column) + (type fixnum bitstore bytecount) + (type (or fixnum null) line-length) + (type function output-function)) + (labels ((emitr (i b) + (declare (type fixnum i b)) + (unless (zerop i) + (emitr (1- i) (ash b -6))) + (emitc + (char +base64-encode-table+ (logand b #x3F))) + (values)) + (out (c) + (funcall output-function c)) + (eol () + (progn + (out #\return) + (out #\newline))) + (emitc (char) + (out char) + (f++ column) + (when (and line-length + (>= column line-length)) + (setf column 0) + (eol)))) + (declare (inline out eol emitc) + (ignorable (function emitr) (function out) (function eol) (function emitc))) + ,@forms)))) + ;; For this function to work correctly, the FIXNUM must be at least + ;; 24 bits. + (defmethod encoder-write-byte ((encoder base64-encoder) byte) + (declare (optimize (speed 3) (safety 0) (debug 0)) + (type (unsigned-byte 8) byte)) + (with-encoder encoder + (setf bitstore (logior byte (the fixnum (ash bitstore 8)))) + (f++ bytecount) + (when (= 3 bytecount) + (emitr 3 bitstore) + (setf bitstore 0 + bytecount 0))) + (values)) + + (defmethod encoder-finish-output ((encoder base64-encoder)) + (with-encoder encoder + (unless (zerop bytecount) + (multiple-value-bind (saved6 rest) (truncate (* bytecount 8) 6) + (setf bitstore (ash bitstore (- 6 rest))) + (emitr saved6 bitstore) + (dotimes (x (- 3 saved6)) + (emitc #\=)))) + (when (and line-length + (not (zerop column))) + (eol))) + (values))) + +(defun encode-base64-stream (in out) + "Read a byte stream from IN and write to OUT the encoded Base64 +character stream." + (make-encoder-loop base64-encoder (read-byte in nil) + (write-char char out))) + +(defun encode-base64-sequence-to-stream (sequence stream &key (start 0) (end (length sequence))) + "Encode the sequence of bytes SEQUENCE and write to STREAM the +Base64 character sequence." + (be i start + (make-encoder-loop base64-encoder + (when (< i end) + (prog1 (elt sequence i) + (incf i))) + (write-char char stream)))) + +(defun encode-base64-sequence (sequence &key (start 0) (end (length sequence))) + "Encode the sequence of bytes SEQUENCE into a Base64 string and +return it." + (with-output-to-string (out) + (encode-base64-sequence-to-stream sequence out :start start :end end))) + +(defclass base64-decoder (parsing-decoder) + ((bitstore :initform 0 + :type fixnum) + (bytecount :initform 0 :type fixnum)) + (:documentation + "Class for Base64 decoder input streams.")) + +(defmethod decoder-read-byte ((decoder base64-decoder)) + (declare (optimize (speed 3) (safety 0) (debug 0))) + (with-slots (bitstore bytecount input-function) decoder + (declare (type fixnum bitstore bytecount) + (type function input-function)) + (labels ((in6 () + (loop + for c = (funcall input-function) + when (or (not c) (char= #\= c)) + do (return-from decoder-read-byte nil) + do (be sextet (aref +base64-decode-table+ (char-code c)) + (unless (= sextet 65) ; ignore unrecognised characters + (return sextet))))) + (push6 (sextet) + (declare (type fixnum sextet)) + (setf bitstore + (logior sextet (the fixnum (ash bitstore 6)))))) + (case bytecount + (0 + (setf bitstore (in6)) + (push6 (in6)) + (setf bytecount 1) + (ash bitstore -4)) + (1 + (push6 (in6)) + (setf bytecount 2) + (logand #xFF (ash bitstore -2))) + (2 + (push6 (in6)) + (setf bytecount 0) + (logand #xFF bitstore)))))) + +(defun decode-base64-stream (in out &key parser-errors) + "Read from IN a stream of characters Base64 encoded and write +to OUT a stream of decoded bytes." + (make-decoder-loop base64-decoder + (read-byte in nil) (write-byte byte out) + :parser-errors parser-errors)) + +(defun decode-base64-stream-to-sequence (stream &key parser-errors) + (make-stream-to-sequence-decoder base64-decoder + (read-char stream nil) + :parser-errors parser-errors)) + +(defun decode-base64-string (string &key (start 0) (end (length string)) parser-errors) + (with-input-from-string (in string :start start :end end) + (decode-base64-stream-to-sequence in :parser-errors parser-errors))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun dump-stream-binary (in out) + "Write content of IN character stream to OUT binary stream." + (loop + for c = (read-char in nil) + while c + do (write-byte (char-code c) out))) + +(defun decode-stream (in out encoding &key parser-errors-p) + (gcase (encoding string-equal) + (:quoted-printable + (decode-quoted-printable-stream in out + :parser-errors parser-errors-p)) + (:base64 + (decode-base64-stream in out + :parser-errors parser-errors-p)) + (otherwise + (dump-stream-binary in out)))) + +(defun decode-string (string encoding &key parser-errors-p) + (gcase (encoding string-equal) + (:quoted-printable + (decode-quoted-printable-string string + :parser-errors parser-errors-p)) + (:base64 + (decode-base64-string string + :parser-errors parser-errors-p)) + (otherwise + (map '(vector (unsigned-byte 8)) #'char-code string)))) + +(defun decode-stream-to-sequence (stream encoding &key parser-errors-p) + (gcase (encoding string-equal) + (:quoted-printable + (decode-quoted-printable-stream-to-sequence stream + :parser-errors parser-errors-p)) + (:base64 + (decode-base64-stream-to-sequence stream + :parser-errors parser-errors-p)) + (otherwise + (loop + with output-sequence = (make-array 0 :fill-pointer 0 + :element-type '(unsigned-byte 8) + :adjustable t) + for c = (read-char stream nil) + while c + do (vector-push-extend (char-code c) output-sequence) + finally (return output-sequence))))) + +(defun encode-stream (in out encoding) + (gcase (encoding string-equal) + (:quoted-printable + (encode-quoted-printable-stream in out)) + (:base64 + (encode-base64-stream in out)) + (otherwise + (loop + for byte = (read-byte in nil) + while byte + do (write-char (code-char byte) out))))) + +(defun encode-sequence-to-stream (sequence out encoding) + (gcase (encoding string-equal) + (:quoted-printable + (encode-quoted-printable-sequence-to-stream sequence out)) + (:base64 + (encode-base64-sequence-to-stream sequence out)) + (otherwise + (loop + for byte across sequence + do (write-char (code-char byte) out))))) + +(defun encode-sequence (sequence encoding) + (gcase (encoding string-equal) + (:quoted-printable + (encode-quoted-printable-sequence sequence)) + (:base64 + (encode-base64-sequence sequence)) + (otherwise + (map 'string #'code-char sequence)))) + +;; This is similar to decode-quoted-printable-string but #\_ is used +;; instead of space +(defun decode-quoted-printable-RFC2047-string (string &key (start 0) (end (length string))) + "Decode a string encoded according to the quoted printable +method of RFC2047 and return a sequence of bytes." + (declare (optimize (speed 3) (debug 0) (safety 0)) + (type simple-string string)) + (loop + with output-sequence = (make-array (length string) + :element-type '(unsigned-byte 8) + :fill-pointer 0) + for i fixnum from start by 1 below end + for c = (char string i) + do (case c + (#\= + (vector-push-extend (or (parse-hex (char string (1+ i)) (char string (+ 2 i))) + ;; the char code was malformed + #.(char-code #\?)) + output-sequence) + (f++ i 2)) + (#\_ (vector-push-extend #.(char-code #\space) output-sequence)) + (otherwise + (vector-push-extend (char-code c) output-sequence))) + finally (return output-sequence))) + +(defun decode-RFC2047-string (encoding string &key (start 0) (end (length string))) + "Decode STRING according to RFC2047 and return a sequence of +bytes." + (gcase (encoding string-equal) + ("Q" (decode-quoted-printable-RFC2047-string string :start start :end end)) + ("B" (decode-base64-string string :start start :end end)) + (t string))) + +(defun parse-RFC2047-text (text) + "Parse the string TEXT according to RFC2047 rules and return a list +of pairs and strings. The strings are the bits interposed between the +actually encoded text. The pairs are composed of: a decoded byte +sequence, a charset string indicating the original coding." + (loop + with result = '() + with previous-end = 0 + for start = (search "=?" text :start2 previous-end) + while start + for first-? = (position #\? text :start (+ 2 start)) + while first-? + for second-? = (position #\? text :start (1+ first-?)) + while second-? + for end = (search "?=" text :start2 (1+ second-?)) + while end + do (let ((charset (string-upcase (subseq text (+ 2 start) first-?))) + (encoding (subseq text (1+ first-?) second-?))) + (unless (= previous-end start) + (push (subseq text previous-end start) + result)) + (setf previous-end (+ end 2)) + (push (cons (decode-RFC2047-string encoding text :start (1+ second-?) :end end) + charset) + result)) + finally (unless (= previous-end (length text)) + (push (subseq text previous-end (length text)) + result)) + (return (nreverse result)))) diff --git a/third_party/lisp/mime4cl/mime.lisp b/third_party/lisp/mime4cl/mime.lisp new file mode 100644 index 000000000000..e35ae6bea547 --- /dev/null +++ b/third_party/lisp/mime4cl/mime.lisp @@ -0,0 +1,1061 @@ +;;; mime4cl.lisp --- MIME primitives for Common Lisp + +;;; Copyright (C) 2005-2008, 2010 by Walter C. Pelissero +;;; Copyright (C) 2021 by the TVL Authors + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :mime4cl) + +(defclass mime-part () + ((subtype + :type (or string null) + :initarg :subtype + :accessor mime-subtype + ;; some mime types don't require a subtype + :initform nil) + (type-parameters + :type list + :initarg :type-parameters + :initform '() + :accessor mime-type-parameters) + (version + :type (or string null) + :initarg :mime-version + :initform "1.0" + :accessor mime-version) + (id + :initform nil + :initarg :id + :reader mime-id) + (description + :initform nil + :initarg :description + :accessor mime-description) + (encoding + :initform :7bit + :initarg :encoding + :reader mime-encoding + :documentation + "It's supposed to be either: + :7BIT, :8BIT, :BINARY, :QUOTED-PRINTABLE, :BASE64, a + X-token or an ietf-token (whatever that means).") + (disposition + :type (or string null) + :initarg :disposition + :initform nil + :accessor mime-disposition) + (disposition-parameters + :type list + :initarg :disposition-parameters + :initform '() + :accessor mime-disposition-parameters)) + (:documentation + "Abstract base class for all types of MIME parts.")) + +(defclass mime-bodily-part (mime-part) + ((body + :initarg :body + :accessor mime-body)) + (:documentation + "Abstract base class for MIME parts with a body.")) + +(defclass mime-unknown-part (mime-bodily-part) + ((type + :initarg :type + :reader mime-type + :documentation + "The original type string from the MIME header.")) + (:documentation + "MIME part unknown to this library. Accepted but not handled.")) + +(defclass mime-text (mime-bodily-part) ()) + +;; This turns out to be handy when making methods specialised +;; non-textual attachments. +(defclass mime-binary (mime-bodily-part) ()) + +(defclass mime-image (mime-binary) ()) + +(defclass mime-audio (mime-binary) ()) + +(defclass mime-video (mime-binary) ()) + +(defclass mime-application (mime-binary) ()) + +(defclass mime-multipart (mime-part) + ((parts :initarg :parts + :accessor mime-parts))) + +(defclass mime-message (mime-part) + ((headers :initarg :headers + :initform '() + :type list + :accessor mime-message-headers) + (real-message :initarg :body + :accessor mime-body))) + +(defun mime-part-p (object) + (typep object 'mime-part)) + +(defmethod initialize-instance ((part mime-multipart) &key &allow-other-keys) + (call-next-method) + ;; The initialization argument of the PARTS slot of a mime-multipart + ;; is expected to be a list of mime-parts. Thus, we implicitly + ;; create the mime parts using the arguments found in this list. + (with-slots (parts) part + (when (slot-boundp part 'parts) + (setf parts + (mapcar #'(lambda (subpart) + (if (mime-part-p subpart) + subpart + (apply #'make-instance subpart))) + parts))))) + +(defmethod initialize-instance ((part mime-message) &key &allow-other-keys) + (call-next-method) + ;; Allow a list of mime parts to be specified as body of a + ;; mime-message. In that case we implicitly create a mime-multipart + ;; and assign to the body slot. + (with-slots (real-message) part + (when (and (slot-boundp part 'real-message) + (consp real-message)) + (setf real-message + (make-instance 'mime-multipart :parts real-message))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun alist= (alist1 alist2 &key (test #'eql)) + (null + (set-difference alist1 alist2 + :test #'(lambda (x y) + (and (funcall test (car x) (car y)) + (funcall test (cdr x) (cdr y))))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric mime= (mime1 mime2) + (:documentation + "Return true if MIME1 and MIME2 have equivalent structure and identical bodies (as for EQ).")) + +(defmethod mime= ((part1 mime-part) (part2 mime-part)) + (macrolet ((null-or (compare x y) + `(or (and (not ,x) + (not ,y)) + (and ,x ,y + (,compare ,x ,y)))) + (cmp-slot (compare reader) + `(null-or ,compare (,reader part1) (,reader part2)))) + (and (eq (class-of part1) (class-of part2)) + (cmp-slot string-equal mime-subtype) + (alist= (mime-type-parameters part1) + (mime-type-parameters part2) + :test #'string-equal) + (cmp-slot string= mime-id) + (cmp-slot string= mime-description) + (cmp-slot eq mime-encoding) + (cmp-slot equal mime-disposition) + (alist= (mime-disposition-parameters part1) + (mime-disposition-parameters part2) + :test #'string-equal)))) + +(defmethod mime= ((part1 mime-multipart) (part2 mime-multipart)) + (and (call-next-method) + (every #'mime= (mime-parts part1) (mime-parts part2)))) + +(defmethod mime= ((part1 mime-message) (part2 mime-message)) + (and (call-next-method) + (alist= (mime-message-headers part1) (mime-message-headers part2) + :test #'string=) + (mime= (mime-body part1) (mime-body part2)))) + +(defun mime-body-stream (mime-part &key (binary t)) + (make-instance (if binary + 'binary-input-adapter-stream + 'character-input-adapter-stream) + :source (mime-body mime-part))) + +(defun mime-body-length (mime-part) + (be body (mime-body mime-part) + ;; here the stream type is missing on purpose, because we may not + ;; be able to size the length of a stream + (etypecase body + (string + (length body)) + (vector + (length body)) + (pathname + (file-size body)) + (file-portion + (with-open-stream (in (open-decoded-file-portion body)) + (loop + for byte = (read-byte in nil) + while byte + count byte)))))) + +(defmacro with-input-from-mime-body-stream ((stream part &key (binary t)) &body forms) + `(with-open-stream (,stream (mime-body-stream ,part :binary ,binary)) + ,@forms)) + +(defmethod mime= ((part1 mime-bodily-part) (part2 mime-bodily-part)) + (and (call-next-method) + (with-input-from-mime-body-stream (in1 part1) + (with-input-from-mime-body-stream (in2 part2) + (loop + for b1 = (read-byte in1 nil) + for b2 = (read-byte in2 nil) + always (eq b1 b2) + while (and b1 b2)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric get-mime-type-parameter (part name) + (:documentation + "Return the MIME type parameter associated to NAME of PART.")) + +(defgeneric (setf get-mime-type-parameter) (value part name) + (:documentation + "Set the MIME type parameter associated to NAME of PART.")) + +(defmethod get-mime-type-parameter ((part mime-part) name) + (cdr (assoc name (mime-type-parameters part) :test #'string-equal))) + +(defmethod (setf get-mime-type-parameter) (value part name) + (aif (assoc name (mime-type-parameters part) :test #'string-equal) + (setf (cdr it) value) + (push (cons name value) + (mime-type-parameters part))) + value) + +(defgeneric get-mime-disposition-parameter (part name) + (:documentation + "Return the MIME disposition parameter associated to NAME of PART.")) + +(defmethod get-mime-disposition-parameter ((part mime-part) name) + (cdr (assoc name (mime-disposition-parameters part) :test #'string-equal))) + +(defmethod (setf get-mime-disposition-parameter) (value part name) + (aif (assoc name (mime-disposition-parameters part) :test #'string-equal) + (setf (cdr it) value) + (push (cons name value) + (mime-disposition-parameters part)))) + +(defmethod mime-part-file-name ((part mime-part)) + "Return the filename associated to mime PART or NIL if the mime +part doesn't have a file name." + (or (get-mime-disposition-parameter part :filename) + (get-mime-type-parameter part :name))) + +(defmethod (setf mime-part-file-name) (value (part mime-part)) + "Set the filename associated to mime PART." + (setf (get-mime-disposition-parameter part :filename) value + (get-mime-type-parameter part :name) value)) + +(defun mime-text-charset (part) + (get-mime-type-parameter part :charset)) + +(defun split-header-parts (string) + "Split parts of a MIME headers. These are divided by +semi-colons not within strings or comments." + (labels ((skip-comment (pos) + (loop + while (< pos (length string)) + do (case (elt string pos) + (#\( (setf pos (skip-comment (1+ pos)))) + (#\\ (incf pos 2)) + (#\) (return (1+ pos))) + (otherwise (incf pos))) + finally (return pos))) + (skip-string (pos) + (loop + while (< pos (length string)) + do (case (elt string pos) + (#\\ (incf pos 2)) + (#\" (return (1+ pos))) + (otherwise (incf pos))) + finally (return pos)))) + (loop + with start = 0 and i = 0 and parts = '() + while (< i (length string)) + do (case (elt string i) + (#\; (push (subseq string start i) parts) + (setf start (incf i))) + (#\" (setf i (skip-string i))) + (#\( (setf i (skip-comment (1+ i)))) + (otherwise (incf i))) + finally (return (mapcar #'string-trim-whitespace (nreverse (cons (subseq string start) parts))))))) + +(defun parse-parameter (string) + "Given a string like \"foo=bar\" return a pair (\"foo\" . +\"bar\"). Return NIL if string is not parsable." + (be equal-position (position #\= string) + (when equal-position + (be key (subseq string 0 equal-position) + (if (= equal-position (1- (length string))) + (cons key "") + (be value (string-trim-whitespace (subseq string (1+ equal-position))) + (cons key + (if (and (> (length value) 1) + (char= #\" (elt value 0))) + ;; the syntax of a RFC822 string is more or + ;; less the same as the Lisp one: use the Lisp + ;; reader + (or (ignore-errors (read-from-string value)) + (subseq value 1)) + (be end (or (position-if #'whitespace-p value) + (length value)) + (subseq value 0 end)))))))))) + +(defun parse-content-type (string) + "Parse string as a Content-Type MIME header and return a list +of three elements. The first is the type, the second is the +subtype and the third is an alist of parameters and their values. +Example: (\"text\" \"plain\" ((\"charset\" . \"us-ascii\")...))." + (let* ((parts (split-header-parts string)) + (content-type-string (car parts)) + (slash (position #\/ content-type-string))) + ;; You'd be amazed to know how many MUA can't produce an RFC + ;; compliant message. + (when slash + (let ((type (subseq content-type-string 0 slash)) + (subtype (subseq content-type-string (1+ slash)))) + (list type subtype (remove nil (mapcar #'parse-parameter (cdr parts)))))))) + +(defun parse-content-disposition (string) + "Parse string as a Content-Disposition MIME header and return a +list. The first element is the layout, the other elements are +the optional parameters alist. +Example: (\"inline\" (\"filename\" . \"doggy.jpg\"))." + (be parts (split-header-parts string) + (cons (car parts) (mapcan #'(lambda (parameter-string) + (awhen (parse-parameter parameter-string) + (list it))) + (cdr parts))))) + +(defun parse-RFC822-header (string) + "Parse STRING which should be a valid RFC822 message header and +return two values: a string of the header name and a string of +the header value." + (be colon (position #\: string) + (when colon + (values (string-trim-whitespace (subseq string 0 colon)) + (string-trim-whitespace (subseq string (1+ colon))))))) + + +(defvar *default-type* '("text" "plain" (("charset" . "us-ascii"))) + "Internal special variable that contains the default MIME type at +any given time of the parsing phase. There are MIME container parts +that may change this.") + +(defvar *mime-types* + '((:text mime-text) + (:image mime-image) + (:audio mime-audio) + (:video mime-video) + (:application mime-application) + (:multipart mime-multipart) + (:message mime-message))) + +(defgeneric mime-part-size (part) + (:documentation + "Return the size in bytes of the body of a MIME part.")) + +(defgeneric print-mime-part (part stream) + (:documentation + "Output to STREAM one of the possible human-readable representation +of mime PART. Binary parts are omitted. This function can be used to +quote messages, for instance.")) + +(defun do-multipart-parts (body-stream part-boundary contents-function end-part-function) + "Read through BODY-STREAM. Call CONTENTS-FUNCTION at +each (non-boundary) line or END-PART-FUNCTION at each PART-BOUNDARY." + (let* ((boundary (s+ "--" part-boundary)) + (boundary-length (length boundary))) + (labels ((output-line (line) + (funcall contents-function line)) + (end-part () + (funcall end-part-function)) + (last-part () + (end-part) + (return-from do-multipart-parts)) + (process-line (line) + (cond ((not (string-starts-with boundary line)) + ;; normal line + (output-line line)) + ((and (= (length (string-trim-whitespace line)) + (+ 2 boundary-length)) + (string= "--" line :start2 boundary-length)) + ;; end of the last part + (last-part)) + ;; according to RFC2046 "the boundary may be followed + ;; by zero or more characters of linear whitespace" + ((= (length (string-trim-whitespace line)) boundary-length) + ;; beginning of the next part + (end-part)) + (t + ;; the line boundary is followed by some + ;; garbage; we treat it as a normal line + (output-line line))))) + (loop + for line = (read-line body-stream nil) + ;; we should never reach the end of a proper multipart MIME + ;; stream, but we don't want to be fooled by corrupted ones, + ;; so we check for EOF + unless line + do (last-part) + do (process-line line))))) + +;; This awkward handling of newlines is due to RFC2046: "The CRLF +;; preceding the boundary delimiter line is conceptually attached to +;; the boundary so that it is possible to have a part that does not +;; end with a CRLF (line break). Body parts that must be considered +;; to end with line breaks, therefore, must have two CRLFs preceding +;; the boundary delimiter line, the first of which is part of the +;; preceding body part, and the second of which is part of the +;; encapsulation boundary". +(defun split-multipart-parts (body-stream part-boundary) + "Read from BODY-STREAM and split MIME parts separated by +PART-BOUNDARY. Return a list of strings." + (let ((part (make-string-output-stream)) + (parts '()) + (beginning-of-part-p t)) + (flet ((output-line (line) + (if beginning-of-part-p + (setf beginning-of-part-p nil) + (terpri part)) + (write-string line part)) + (end-part () + (setf beginning-of-part-p t) + (push (get-output-stream-string part) parts))) + (do-multipart-parts body-stream part-boundary #'output-line #'end-part) + (close part) + ;; the first part is empty or contains all the junk + ;; to the first boundary + (cdr (nreverse parts))))) + +(defun index-multipart-parts (body-stream part-boundary) + "Read from BODY-STREAM and return the file offset of the MIME parts +separated by PART-BOUNDARY." + (let ((parts '()) + (start 0) + (len 0) + (beginning-of-part-p t)) + (flet ((sum-chars (line) + (incf len (length line)) + ;; account for the #\newline + (if beginning-of-part-p + (setf beginning-of-part-p nil) + (incf len))) + (end-part () + (setf beginning-of-part-p t) + (push (cons start (+ start len)) parts) + (setf start (file-position body-stream) + len 0))) + (do-multipart-parts body-stream part-boundary #'sum-chars #'end-part) + ;; the first part is all the stuff up to the first boundary; + ;; just junk + (cdr (nreverse parts))))) + +(defgeneric encode-mime-part (part stream)) +(defgeneric encode-mime-body (part stream)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun write-mime-header (part stream) + (when (mime-version part) + (format stream "~&MIME-Version: ~A~%" (mime-version part))) + (format stream "~&Content-Type: ~A~:{; ~A=~S~}~%" (mime-type-string part) + (mapcar #'(lambda (pair) + (list (car pair) (cdr pair))) + (mime-type-parameters part))) + (awhen (mime-encoding part) + (format stream "Content-Transfer-Encoding: ~A~%" it)) + (awhen (mime-description part) + (format stream "Content-Description: ~A~%" it)) + (when (mime-disposition part) + (format stream "Content-Disposition: ~A~:{; ~A=~S~}~%" + (mime-disposition part) + (mapcar #'(lambda (pair) + (list (car pair) (cdr pair))) + (mime-disposition-parameters part)))) + (awhen (mime-id part) + (format stream "Content-ID: ~A~%" it)) + (terpri stream)) + +(defmethod encode-mime-part ((part mime-part) stream) + (write-mime-header part stream) + (encode-mime-body part stream)) + +(defmethod encode-mime-part ((part mime-message) stream) + ;; tricky: we have to mix the MIME headers with the message headers + (dolist (h (mime-message-headers part)) + (unless (stringp (car h)) + (setf (car h) + (string-capitalize (car h)))) + (unless (or (string-starts-with "content-" (car h) #'string-equal) + (string-equal "mime-version" (car h))) + (format stream "~A: ~A~%" + (car h) (cdr h)))) + (encode-mime-part (mime-body part) stream)) + +(defmethod encode-mime-part ((part mime-multipart) stream) + ;; choose a boundary if not already set + (let* ((original-boundary (get-mime-type-parameter part :boundary)) + (boundary (choose-boundary (mime-parts part) original-boundary))) + (unless (and original-boundary + (string= boundary original-boundary)) + (setf (get-mime-type-parameter part :boundary) boundary)) + (call-next-method))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmethod encode-mime-body ((part mime-part) stream) + (with-input-from-mime-body-stream (in part) + (encode-stream in stream (mime-encoding part)))) + +(defmethod encode-mime-body ((part mime-message) stream) + (encode-mime-body (mime-body part) stream)) + +(defmethod encode-mime-body ((part mime-multipart) stream) + (be boundary (or (get-mime-type-parameter part :boundary) + (setf (get-mime-type-parameter part :boundary) + (choose-boundary (mime-parts part)))) + (dolist (p (mime-parts part)) + (format stream "~%--~A~%" boundary) + (encode-mime-part p stream)) + (format stream "~%--~A--~%" boundary))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun time-RFC822-string (&optional (epoch (get-universal-time))) + "Return a string describing the current time according to +the RFC822." + (multiple-value-bind (ss mm hh day month year week-day dst tz) (decode-universal-time epoch) + (declare (ignore dst)) + (format nil "~A, ~A ~A ~2,'0D ~2,'0D:~2,'0D:~2,'0D ~:[-~;+~]~2,'0D~2,'0D" + (subseq (week-day->string week-day) 0 3) + day (subseq (month->string month) 0 3) (mod year 100) hh mm ss + (plusp tz) (abs (truncate tz)) (mod (* 60 tz) 60)))) + +(defun parse-RFC822-date (date-string) + "Parse a RFC822 compliant date string and return an universal +time." + ;; if we can't parse it, just return NIL + (ignore-errors + ;; skip the optional DoW + (awhen (position #\, date-string) + (setf date-string (subseq date-string (1+ it)))) + (destructuring-bind (day month year time &optional tz &rest rubbish) + (split-at '(#\space #\tab) date-string) + (declare (ignore rubbish)) + (destructuring-bind (hh mm &optional ss) (split-string-at-char time #\:) + (encode-universal-time + (if ss + (read-from-string ss) + 0) + (read-from-string mm) + (read-from-string hh) + (read-from-string day) + (1+ (position month + '("Jan" "Feb" "Mar" "Apr" "May" "Jun" + "Jul" "Aug" "Sep" "Oct" "Nov" "Dec") + :test #'string-equal)) + (read-from-string year) + (when (and tz (or (char= #\+ (elt tz 0)) + (char= #\- (elt tz 0)))) + (/ (read-from-string tz) 100))))))) + +(defun read-RFC822-headers (stream &optional required-headers) + "Read RFC822 compliant headers from STREAM and return them in a +alist of keyword and string pairs. REQUIRED-HEADERS is a list of +header names we are interested in; if NIL return all headers +found in STREAM." + ;; the skip-header variable is to avoid the mistake of appending a + ;; continuation line of a header we don't want to a header we want + (loop + with headers = '() and skip-header = nil + for line = (be line (read-line stream nil) + ;; skip the Unix "From " header if present + (if (string-starts-with "From " line) + (read-line stream nil) + line)) + then (read-line stream nil) + while (and line + (not (zerop (length line)))) + do (if (whitespace-p (elt line 0)) + (unless (or skip-header + (null headers)) + (setf (cdar headers) (s+ (cdar headers) '(#\newline) line))) + (multiple-value-bind (name value) (parse-RFC822-header line) + ;; the line contained rubbish instead of an header: we + ;; play nice and return as we were at the end of the + ;; headers + (unless name + (return (nreverse headers))) + (if (or (null required-headers) + (member name required-headers :test #'string-equal)) + (progn + (push (cons name value) headers) + (setf skip-header nil)) + (setf skip-header t)))) + finally (return (nreverse headers)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric mime-message (thing) + (:documentation + "Convert THING to a MIME-MESSAGE object.")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defvar *lazy-mime-decode* t + "If true don't decode mime bodies in memory.") + +(defgeneric decode-mime-body (part input-stream)) + +(defmethod decode-mime-body ((part mime-part) (stream delimited-input-stream)) + (be base (base-stream stream) + (if *lazy-mime-decode* + (setf (mime-body part) + (make-file-portion :data (etypecase base + (my-string-input-stream + (stream-string base)) + (file-stream + (pathname base))) + :encoding (mime-encoding part) + :start (file-position stream) + :end (stream-end stream))) + (call-next-method)))) + +(defmethod decode-mime-body ((part mime-part) (stream file-stream)) + (if *lazy-mime-decode* + (setf (mime-body part) + (make-file-portion :data (pathname stream) + :encoding (mime-encoding part) + :start (file-position stream))) + (call-next-method))) + +(defmethod decode-mime-body ((part mime-part) (stream my-string-input-stream)) + (if *lazy-mime-decode* + (setf (mime-body part) + (make-file-portion :data (stream-string stream) + :encoding (mime-encoding part) + :start (file-position stream))) + (call-next-method))) + +(defmethod decode-mime-body ((part mime-part) stream) + (setf (mime-body part) + (decode-stream-to-sequence stream (mime-encoding part)))) + +(defmethod decode-mime-body ((part mime-multipart) stream) + "Decode STREAM according to PART characteristics and return a +list of MIME parts." + (save-file-excursion (stream) + (be offsets (index-multipart-parts stream (get-mime-type-parameter part :boundary)) + (setf (mime-parts part) + (mapcar #'(lambda (p) + (destructuring-bind (start . end) p + (be *default-type* (if (eq :digest (mime-subtype part)) + '("message" "rfc822" ()) + '("text" "plain" (("charset" . "us-ascii")))) + in (make-instance 'delimited-input-stream + :stream stream + :dont-close t + :start start + :end end) + (read-mime-part in)))) + offsets))))) + +(defmethod decode-mime-body ((part mime-message) stream) + "Read from STREAM the body of PART. Return the decoded MIME +body." + (setf (mime-body part) + (read-mime-message stream))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defconst +known-encodings+ '(:7BIT :8BIT :BINARY :QUOTED-PRINTABLE :BASE64) + "List of known content encodings.") + +(defun keywordify-encoding (string) + "Return a keyword for a content transfer encoding string. +Return STRING itself if STRING is an unkown encoding." + (aif (member string +known-encodings+ :test #'string-equal) + (car it) + string)) + +(defun header (name headers) + (be elt (assoc name headers :test #'string-equal) + (values (cdr elt) (car elt)))) + +(defun (setf header) (value name headers) + (be entry (assoc name headers :test #'string-equal) + (unless entry + (error "missing header ~A can't be set" name)) + (setf (cdr entry) value))) + +(defun make-mime-part (headers stream) + "Create a MIME-PART object based on HEADERS and a body which +has to be read from STREAM. If the mime part type can't be +guessed from the headers, use the *DEFAULT-TYPE*." + (flet ((hdr (what) + (header what headers))) + (destructuring-bind (type subtype parms) + (or + (aand (hdr :content-type) + (parse-content-type it)) + *default-type*) + (let* ((class (or (cadr (assoc type *mime-types* :test #'string-equal)) + 'mime-unknown-part)) + (disp (aif (hdr :content-disposition) + (parse-content-disposition it) + (values nil nil))) + (part (make-instance class + :type (hdr :content-type) + :subtype subtype + :type-parameters parms + :disposition (car disp) + :disposition-parameters (cdr disp) + :mime-version (hdr :mime-version) + :encoding (keywordify-encoding + (hdr :content-transfer-encoding)) + :description (hdr :content-description) + :id (hdr :content-id) + :allow-other-keys t))) + (decode-mime-body part stream) + part)))) + +(defun read-mime-part (stream) + "Read mime part from STREAM. Return a MIME-PART object." + (be headers (read-rfc822-headers stream + '(:mime-version :content-transfer-encoding :content-type + :content-disposition :content-description :content-id)) + (make-mime-part headers stream))) + +(defun read-mime-message (stream) + "Main function to read a MIME message from a stream. It +returns a MIME-MESSAGE object." + (be headers (read-rfc822-headers stream) + *default-type* '("text" "plain" (("charset" . "us-ascii"))) + (flet ((hdr (what) + (header what headers))) + (destructuring-bind (type subtype parms) + (or (aand (hdr :content-type) + (parse-content-type it)) + *default-type*) + (declare (ignore type subtype)) + (make-instance 'mime-message + :headers headers + ;; this is just for easy access + :type-parameters parms + :body (make-mime-part headers stream)))))) + +(defmethod mime-message ((msg mime-message)) + msg) + +(defmethod mime-message ((msg string)) + (with-open-stream (in (make-instance 'my-string-input-stream :string msg)) + (read-mime-message in))) + +(defmethod mime-message ((msg stream)) + (read-mime-message msg)) + +(defmethod mime-message ((msg pathname)) + (let (#+sbcl(sb-impl::*default-external-format* :latin-1) + #+sbcl(sb-alien::*default-c-string-external-format* :latin-1)) + (with-open-file (in msg) + (read-mime-message in)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric mime-part (object) + (:documentation + "Promote object, if necessary, to MIME-PART.")) + +(defmethod mime-part ((object string)) + (make-instance 'mime-text :subtype "plain" :body object)) + +(defmethod mime-part ((object pathname)) + (make-instance 'mime-application + :subtype "octect-stream" + :content-transfer-encoding :base64 + :body (read-file object :element-type '(unsigned-byte 8)))) + +(defmethod mime-part ((object mime-part)) + object) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmethod make-encoded-body-stream ((part mime-bodily-part)) + (be body (mime-body part) + (make-instance (case (mime-encoding part) + (:base64 + 'base64-encoder-input-stream) + (:quoted-printable + 'quoted-printable-encoder-input-stream) + (t + '8bit-encoder-input-stream)) + :stream (make-instance 'binary-input-adapter-stream :source body)))) + +(defun choose-boundary (parts &optional default) + (labels ((match-in-parts (boundary parts) + (loop + for p in parts + thereis (typecase p + (mime-multipart + (match-in-parts boundary (mime-parts p))) + (mime-bodily-part + (match-in-body p boundary))))) + (match-in-body (part boundary) + (with-open-stream (in (make-encoded-body-stream part)) + (loop + for line = (read-line in nil) + while line + when (string= line boundary) + return t + finally (return nil))))) + (do ((boundary (if default + (format nil "--~A" default) + #1=(format nil "--~{~36R~}" + (loop + for i from 0 below 20 + collect (random 36)))) + #1#)) + ((not (match-in-parts boundary parts)) (subseq boundary 2))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; fall back method +(defmethod mime-part-size ((part mime-part)) + (be body (mime-body part) + (typecase body + (pathname + (file-size body)) + (string + (length body)) + (vector + (length body)) + (t nil)))) + +(defmethod mime-part-size ((part mime-multipart)) + (loop + for p in (mime-parts part) + for size = (mime-part-size p) + unless size + return nil + sum size)) + +(defmethod mime-part-size ((part mime-message)) + (mime-part-size (mime-body part))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmethod print-mime-part ((part mime-multipart) (out stream)) + (case (mime-subtype part) + (:alternative + ;; try to choose something simple to print or the first thing + (be parts (mime-parts part) + (print-mime-part (or (find-if #'(lambda (part) + (and (eq (class-of part) (find-class 'mime-text)) + (eq (mime-subtype part) :plain))) + parts) + (car parts)) out))) + (otherwise + (dolist (subpart (mime-parts part)) + (print-mime-part subpart out))))) + +;; This is WRONG. Here we don't use any special character encoding +;; because we don't know which one we should use. Messages written in +;; anything but ASCII will likely be unreadable -wcp11/10/07. +(defmethod print-mime-part ((part mime-text) (out stream)) + (be body (mime-body part) + (etypecase body + (string + (write-string body out)) + (vector + (loop + for byte across body + do (write-char (code-char byte) out))) + (pathname + (with-open-file (in body) + (loop + for c = (read-char in nil) + while c + do (write-char c out))))))) + +(defmethod print-mime-part ((part mime-message) (out stream)) + (flet ((hdr (name) + (multiple-value-bind (value tag) + (header name (mime-message-headers part)) + (cons tag value)))) + (dolist (h (mapcar #'hdr '("from" "subject" "to" "date" "x-march-archive-id"))) + (when h + (format out "~&~A: ~A" (car h) (cdr h)))) + (format out "~2%") + (print-mime-part (mime-body part) out))) + +(defmethod print-mime-part ((part mime-part) (out stream)) + (format out "~&[ ~A subtype=~A ~@[description=~S ~]~@[size=~A~] ]~%" + (type-of part) (mime-subtype part) (mime-description part) (mime-part-size part))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric find-mime-part-by-path (mime path) + (:documentation + "Return a subpart of MIME identified by PATH, which is a list of +integers. For example '(2 3 1) is the first part of the third of the +second in MIME.")) + +(defmethod find-mime-part-by-path ((part mime-part) path) + (if (null path) + part + (error "~S doesn't have subparts" part))) + +(defmethod find-mime-part-by-path ((part mime-message) path) + (if (null path) + part + (if (= 1 (car path)) + (find-mime-part-by-path (mime-body part) (cdr path)) + (error "~S may have just one subpart, but part ~D was requested (parts are enumerated base 1)." + part (car path))))) + +(defmethod find-mime-part-by-path ((part mime-multipart) path) + (if (null path) + part + (be parts (mime-parts part) + part-number (car path) + (if (<= 1 part-number (length parts)) + (find-mime-part-by-path (nth (1- (car path)) (mime-parts part)) (cdr path)) + (error "~S has just ~D subparts, but part ~D was requested (parts are enumerated base 1)." + part (length parts) part-number))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric find-mime-part-by-id (part id) + (:documentation + "Return a subpart of PAR, whose Content-ID is the same as ID, which +is a string.")) + +(defmethod find-mime-part-by-id ((part mime-part) id) + (when (string= id (mime-id part)) + part)) + +(defmethod find-mime-part-by-id ((part mime-message) id) + (find-mime-part-by-id (mime-body part) id)) + +(defmethod find-mime-part-by-id ((part mime-multipart) id) + (or (call-next-method) + (some #'(lambda (p) + (find-mime-part-by-id p id)) + (mime-parts part)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmethod find-mime-text-part (msg) + (:documentation + "Return message if it is a text message or first text part. + If no suitable text part is found, return NIL.")) + +(defmethod find-mime-text-part ((part mime-text)) + part) ; found our target + +(defmethod find-mime-text-part ((msg mime-message)) + ;; mime-body is either a mime-part or mime-multipart + (find-mime-text-part (mime-body msg))) + +(defmethod find-mime-text-part ((parts mime-multipart)) + ;; multipart messages may have a body, otherwise we + ;; search for the first text part + (or (call-next-method) + (find-if #'find-mime-text-part (mime-parts parts)))) + +(defmethod find-mime-text-part ((part mime-part)) + nil) ; default case + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric mime-type-string (mime-part) + (:documentation + "Return the string describing the MIME part.")) + +(defmethod mime-type-string ((part mime-unknown-part)) + (mime-type part)) + +(defmethod mime-type-string ((part mime-text)) + (format nil "text/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-image)) + (format nil "image/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-audio)) + (format nil "audio/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-video)) + (format nil "video/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-application)) + (format nil "application/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-multipart)) + (format nil "multipart/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-message)) + (format nil "message/~A" (mime-subtype part))) + +(defmethod mime-type-string ((part mime-unknown-part)) + (mime-type part)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defgeneric map-parts (function mime-part) + (:documentation + "Recursively map FUNCTION to MIME-PART or its components.")) + +;; Here we wrongly assume that we'll never want to replace messages +;; and multiparts altogether. If you need to do so you have to write +;; your own mapping functions. + +(defmethod map-parts ((function function) (part mime-part)) + (funcall function part)) + +(defmethod map-parts ((function function) (part mime-message)) + (setf (mime-body part) (map-parts function (mime-body part))) + part) + +(defmethod map-parts ((function function) (part mime-multipart)) + (setf (mime-parts part) (mapcar #'(lambda (p) + (map-parts function p)) + (mime-parts part))) + part) + +;; apply-on-parts is like map-parts but doesn't modify the parts (at least +;; not implicitly) + +(defgeneric apply-on-parts (function part)) + +(defmethod apply-on-parts ((function function) (part mime-part)) + (funcall function part)) + +(defmethod apply-on-parts ((function function) (part mime-multipart)) + (dolist (p (mime-parts part)) + (apply-on-parts function p))) + +(defmethod apply-on-parts ((function function) (part mime-message)) + (apply-on-parts function (mime-body part))) + +(defmacro do-parts ((var mime-part) &body body) + `(apply-on-parts #'(lambda (,var) ,@body) ,mime-part)) diff --git a/third_party/lisp/mime4cl/mime4cl-tests.asd b/third_party/lisp/mime4cl/mime4cl-tests.asd new file mode 100644 index 000000000000..e4d983c05760 --- /dev/null +++ b/third_party/lisp/mime4cl/mime4cl-tests.asd @@ -0,0 +1,54 @@ +;;; mime4cl-tests.asd --- system description for the regression tests + +;;; Copyright (C) 2006, 2007, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +#-(or sbcl cmu) +(warn "This code hasn't been tested on your Lisp system.") + +(defpackage :mime4cl-tests-system + (:use :common-lisp :asdf #+asdfa :asdfa) + (:export #:*base-directory* + #:*compilation-epoch*)) + +(in-package :mime4cl-tests-system) + +(defsystem mime4cl-tests + :name "MIME4CL-tests" + :author "Walter C. Pelissero <walter@pelissero.de>" + :maintainer "Walter C. Pelissero <walter@pelissero.de>" + :description "Test suite for the MIME4CL library" + :long-description + "These regression tests require rt.lisp from MIT. It is included." + :licence "LGPL" + :depends-on (:mime4cl) + :components + ((:module test + :components + ((:file "rt") + (:file "package" :depends-on ("rt")) + (:file "endec" :depends-on ("rt" "package")) + (:file "address" :depends-on ("rt" "package")) + (:file "mime" :depends-on ("rt" "package")))))) + +;; when loading this form the regression-test, the package is yet to +;; be loaded so we cannot use rt:do-tests directly or we would get a +;; reader error (unknown package) +(defmethod perform ((o test-op) (c (eql (find-system :mime4cl-tests)))) + (or (funcall (intern "DO-TESTS" "REGRESSION-TEST")) + (error "test-op failed"))) diff --git a/third_party/lisp/mime4cl/mime4cl.asd b/third_party/lisp/mime4cl/mime4cl.asd new file mode 100644 index 000000000000..2761a00d5283 --- /dev/null +++ b/third_party/lisp/mime4cl/mime4cl.asd @@ -0,0 +1,53 @@ +;;; mime4cl.asd --- system definition + +;;; Copyright (C) 2005-2007, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +;;; This program is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU General Public License as +;;; published by the Free Software Foundation; either version 2, or (at +;;; your option) any later version. +;;; This program is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; General Public License for more details. +;;; You should have received a copy of the GNU General Public License +;;; along with this program; see the file COPYING. If not, write to +;;; the Free Software Foundation, Inc., 59 Temple Place - Suite 330, +;;; Boston, MA 02111-1307, USA. + +(in-package :cl-user) + +#+(and cmu (not gray-streams)) +(eval-when (:compile-toplevel :load-toplevel :execute) + (ext:without-package-locks + (load "library:subsystems/gray-streams-library"))) + +(defpackage :mime4cl-system + (:use :common-lisp :asdf)) + +(in-package :mime4cl-system) + +(defsystem mime4cl + :name "MIME4CL" + :author "Walter C. Pelissero <walter@pelissero.de>" + :maintainer "Walter C. Pelissero <walter@pelissero.de>" + ;; :version "0.0" + :description "MIME primitives for Common Lisp" + :long-description + "A collection of Common Lisp primitives to forge and handle +MIME mail contents." + :licence "LGPL" + :depends-on (:npg :sclf) + :components + ((:file "package") + (:file "mime" :depends-on ("package" "endec" "streams")) + (:file "endec" :depends-on ("package")) + (:file "streams" :depends-on ("package" "endec")) + (:file "address" :depends-on ("package")))) + +(defmethod perform ((o test-op) (c (eql (find-system 'mime4cl)))) + (oos 'load-op 'mime4cl-tests) + (oos 'test-op 'mime4cl-tests :force t)) diff --git a/third_party/lisp/mime4cl/package.lisp b/third_party/lisp/mime4cl/package.lisp new file mode 100644 index 000000000000..a6e7e7d8ef10 --- /dev/null +++ b/third_party/lisp/mime4cl/package.lisp @@ -0,0 +1,108 @@ +;;; package.lisp --- package declaration + +;;; Copyright (C) 2005-2007, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :cl-user) + +(defpackage :mime4cl + (:nicknames :mime) + (:use :common-lisp :npg :sclf + ;; for Gray streams + #+cmu :extensions #+sbcl :sb-gray) + ;; this is stuff that comes from SCLF and clashes with CMUCL's EXT + ;; package + (:shadowing-import-from :sclf + #:process-wait + #:process-alive-p + #:run-program) + (:export #:*lazy-mime-decode* + #:print-mime-part + #:read-mime-message + #:mime-part + #:mime-text + #:mime-binary + #:mime-id + #:mime-image + #:mime-message + #:mime-multipart + #:mime-audio + #:mime-unknown-part + #:get-mime-disposition-parameter + #:get-mime-type-parameter + #:mime-disposition + #:mime-disposition-parameters + #:mime-encoding + #:mime-application + #:mime-video + #:mime-description + #:mime-part-size + #:mime-subtype + #:mime-body + #:mime-body-stream + #:mime-body-length + #:mime-parts + #:mime-part-p + #:mime-type + #:mime-type-string + #:mime-type-parameters + #:mime-message-headers + #:mime= + #:find-mime-part-by-path + #:find-mime-part-by-id + #:find-mime-text-part + #:encode-mime-part + #:encode-mime-body + #:decode-quoted-printable-stream + #:decode-quoted-printable-string + #:encode-quoted-printable-stream + #:encode-quoted-printable-sequence + #:decode-base64-stream + #:decode-base64-string + #:encode-base64-stream + #:encode-base64-sequence + #:parse-RFC2047-text + #:parse-RFC822-header + #:read-RFC822-headers + #:time-RFC822-string + #:parse-RFC822-date + #:map-parts + #:do-parts + #:apply-on-parts + #:mime-part-file-name + #:mime-text-charset + #:with-input-from-mime-body-stream + ;; endec.lisp + #:base64-encoder + #:base64-decoder + #:null-encoder + #:null-decoder + #:byte-encoder + #:byte-decoder + #:quoted-printable-encoder + #:quoted-printable-decoder + #:encoder-write-byte + #:encoder-finish-output + #:decoder-read-byte + #:decoder-read-sequence + #:*base64-line-length* + #:*quoted-printable-line-length* + ;; address.lisp + #:parse-addresses #:mailboxes-only + #:mailbox #:mbx-description #:mbx-user #:mbx-host #:mbx-domain #:mbx-domain-name #:mbx-address + #:mailbox-group #:mbxg-name #:mbxg-mailboxes)) diff --git a/third_party/lisp/mime4cl/streams.lisp b/third_party/lisp/mime4cl/streams.lisp new file mode 100644 index 000000000000..64c7adeec571 --- /dev/null +++ b/third_party/lisp/mime4cl/streams.lisp @@ -0,0 +1,368 @@ + ;;; eds.lisp --- En/De-coding Streams + + ;;; Copyright (C) 2012 by Walter C. Pelissero + ;;; Copyright (C) 2021 by the TVL Authors + + ;;; Author: Walter C. Pelissero <walter@pelissero.de> + ;;; Project: mime4cl + +#+cmu (ext:file-comment "$Module: eds.lisp") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :mime4cl) + +#+cmu +(eval-when (:load-toplevel :compile-toplevel :execute) + ;; CMUCL doesn't provide the STREAM-FILE-POSITION method in its + ;; implementation of Gray streams. We patch it in ourselves. + (defgeneric stream-file-position (stream &optional position)) + (defun my-file-position (stream &optional position) + (stream-file-position stream position)) + (defvar *original-file-position-function* + (prog1 + (symbol-function 'file-position) + (setf (symbol-function 'file-position) (symbol-function 'my-file-position)))) + (defmethod stream-file-position (stream &optional position) + (if position + (funcall *original-file-position-function* stream position) + (funcall *original-file-position-function* stream))) + + ;; oddly CMUCL doesn't seem to provide a default for STREAM-READ-SEQUENCE + (defmacro make-read-sequence (stream-type element-reader) + `(defmethod stream-read-sequence ((stream ,stream-type) seq &optional start end) + (unless start + (setf start 0)) + (unless end + (setf end (length seq))) + (loop + for i from start below end + for b = (,element-reader stream) + until (eq b :eof) + do (setf (elt seq i) b) + finally (return i)))) + + (make-read-sequence fundamental-binary-input-stream stream-read-byte) + (make-read-sequence fundamental-character-input-stream stream-read-char)) + +(defclass coder-stream-mixin () + ((real-stream :type stream + :initarg :stream + :reader real-stream) + (dont-close :initform nil + :initarg :dont-close))) + +(defmethod stream-file-position ((stream coder-stream-mixin) &optional position) + (apply #'file-position (remove nil (list (slot-value stream 'real-stream) + position)))) + +(defclass coder-input-stream-mixin (fundamental-binary-input-stream coder-stream-mixin) + ()) +(defclass coder-output-stream-mixin (fundamental-binary-output-stream coder-stream-mixin) + ()) + + +(defclass quoted-printable-decoder-stream (coder-input-stream-mixin quoted-printable-decoder) ()) +(defclass base64-decoder-stream (coder-input-stream-mixin base64-decoder) ()) +(defclass 8bit-decoder-stream (coder-input-stream-mixin 8bit-decoder) ()) + +(defclass quoted-printable-encoder-stream (coder-output-stream-mixin quoted-printable-encoder) ()) +(defclass base64-encoder-stream (coder-output-stream-mixin base64-encoder) ()) +(defclass 8bit-encoder-stream (coder-output-stream-mixin 8bit-encoder) ()) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmethod initialize-instance :after ((stream coder-stream-mixin) &key &allow-other-keys) + (unless (slot-boundp stream 'real-stream) + (error "REAL-STREAM is unbound. Must provide a :STREAM argument."))) + +(defmethod initialize-instance ((stream coder-output-stream-mixin) &key &allow-other-keys) + (call-next-method) + (unless (slot-boundp stream 'output-function) + (setf (slot-value stream 'output-function) + #'(lambda (char) + (write-char char (slot-value stream 'real-stream)))))) + +(defmethod initialize-instance ((stream coder-input-stream-mixin) &key &allow-other-keys) + (call-next-method) + (unless (slot-boundp stream 'input-function) + (setf (slot-value stream 'input-function) + #'(lambda () + (read-char (slot-value stream 'real-stream) nil))))) + +(defmethod stream-read-byte ((stream coder-input-stream-mixin)) + (or (decoder-read-byte stream) + :eof)) + +(defmethod stream-write-byte ((stream coder-output-stream-mixin) byte) + (encoder-write-byte stream byte)) + +(defmethod close ((stream coder-stream-mixin) &key abort) + (with-slots (real-stream dont-close) stream + (unless dont-close + (close real-stream :abort abort)))) + +(defmethod close ((stream coder-output-stream-mixin) &key abort) + (unless abort + (encoder-finish-output stream)) + (call-next-method)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass encoder-input-stream (fundamental-character-input-stream coder-stream-mixin) + ((encoder) + (buffer-queue :initform (make-queue))) + (:documentation + "This is the base class for encoders with the direction swapped. It +reads from REAL-STREAM a stream of bytes, encodes it and returnes it +in a stream of character.")) + +(defclass quoted-printable-encoder-input-stream (encoder-input-stream) ()) +(defclass base64-encoder-input-stream (encoder-input-stream) ()) +(defclass 8bit-encoder-input-stream (fundamental-character-input-stream coder-stream-mixin) ()) + +(defmethod initialize-instance ((stream quoted-printable-encoder-input-stream) &key &allow-other-keys) + (call-next-method) + (with-slots (encoder buffer-queue) stream + (setf encoder + (make-instance 'quoted-printable-encoder + :output-function #'(lambda (char) + (queue-append buffer-queue char)))))) + +(defmethod initialize-instance ((stream base64-encoder-input-stream) &key &allow-other-keys) + (call-next-method) + (with-slots (encoder buffer-queue) stream + (setf encoder + (make-instance 'base64-encoder + :output-function #'(lambda (char) + (queue-append buffer-queue char)))))) + +(defmethod stream-read-char ((stream encoder-input-stream)) + (with-slots (encoder buffer-queue real-stream) stream + (loop + while (queue-empty-p buffer-queue) + do (be byte (read-byte real-stream nil) + (if byte + (encoder-write-byte encoder byte) + (progn + (encoder-finish-output encoder) + (queue-append buffer-queue :eof))))) + (queue-pop buffer-queue))) + + +(defmethod stream-read-char ((stream 8bit-encoder-input-stream)) + (with-slots (real-stream) stream + (aif (read-byte real-stream nil) + (code-char it) + :eof))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass input-adapter-stream () + ((source :initarg :source) + (real-stream) + (input-function))) + +(defclass binary-input-adapter-stream (fundamental-binary-input-stream input-adapter-stream) ()) + +(defclass character-input-adapter-stream (fundamental-character-input-stream input-adapter-stream) ()) + +(defmethod stream-element-type ((stream binary-input-adapter-stream)) + '(unsigned-byte 8)) + +(defmethod initialize-instance ((stream input-adapter-stream) &key &allow-other-keys) + (call-next-method) + (assert (slot-boundp stream 'source))) + +(defmethod initialize-instance ((stream binary-input-adapter-stream) &key &allow-other-keys) + (call-next-method) + ;; REAL-STREAM slot is set only if we are going to close it later on + (with-slots (source real-stream input-function) stream + (etypecase source + (string + (setf real-stream (make-string-input-stream source) + input-function #'(lambda () + (awhen (read-char real-stream nil) + (char-code it))))) + ((vector (unsigned-byte 8)) + (be i 0 + (setf input-function #'(lambda () + (when (< i (length source)) + (prog1 (aref source i) + (incf i))))))) + (stream + (assert (input-stream-p source)) + (setf input-function (if (subtypep (stream-element-type source) 'character) + #'(lambda () + (awhen (read-char source nil) + (char-code it))) + #'(lambda () + (read-byte source nil))))) + (pathname + (setf real-stream (open source :element-type '(unsigned-byte 8)) + input-function #'(lambda () + (read-byte real-stream nil)))) + (file-portion + (setf real-stream (open-decoded-file-portion source) + input-function #'(lambda () + (read-byte real-stream nil))))))) + +(defmethod initialize-instance ((stream character-input-adapter-stream) &key &allow-other-keys) + (call-next-method) + ;; REAL-STREAM slot is set only if we are going to close later on + (with-slots (source real-stream input-function) stream + (etypecase source + (string + (setf real-stream (make-string-input-stream source) + input-function #'(lambda () + (read-char real-stream nil)))) + ((vector (unsigned-byte 8)) + (be i 0 + (setf input-function #'(lambda () + (when (< i (length source)) + (prog1 (code-char (aref source i)) + (incf i))))))) + (stream + (assert (input-stream-p source)) + (setf input-function (if (subtypep (stream-element-type source) 'character) + #'(lambda () + (read-char source nil)) + #'(lambda () + (awhen (read-byte source nil) + (code-char it)))))) + (pathname + (setf real-stream (open source :element-type 'character) + input-function #'(lambda () + (read-char real-stream nil)))) + (file-portion + (setf real-stream (open-decoded-file-portion source) + input-function #'(lambda () + (awhen (read-byte real-stream nil) + (code-char it)))))))) + +(defmethod close ((stream input-adapter-stream) &key abort) + (when (slot-boundp stream 'real-stream) + (with-slots (real-stream) stream + (close real-stream :abort abort)))) + +(defmethod stream-read-byte ((stream binary-input-adapter-stream)) + (with-slots (input-function) stream + (or (funcall input-function) + :eof))) + +(defmethod stream-read-char ((stream character-input-adapter-stream)) + (with-slots (input-function) stream + (or (funcall input-function) + :eof))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass delimited-input-stream (fundamental-character-input-stream coder-stream-mixin) + ((start-offset :initarg :start + :initform 0 + :reader stream-start + :type integer) + (end-offset :initarg :end + :initform nil + :reader stream-end + :type (or null integer)))) + +(defmethod print-object ((object delimited-input-stream) stream) + (if *print-readably* + (call-next-method) + (with-slots (start-offset end-offset) object + (print-unreadable-object (object stream :type t :identity t) + (format stream "start=~A end=~A" start-offset end-offset))))) + +(defun base-stream (stream) + (if (typep stream 'delimited-input-stream) + (base-stream (real-stream stream)) + stream)) + +(defmethod initialize-instance ((stream delimited-input-stream) &key &allow-other-keys) + (call-next-method) + (unless (slot-boundp stream 'real-stream) + (error "REAL-STREAM is unbound. Must provide a :STREAM argument.")) + (with-slots (start-offset) stream + (when start-offset + (file-position stream start-offset)))) + +(defmethod stream-read-char ((stream delimited-input-stream)) + (with-slots (real-stream end-offset) stream + (if (or (not end-offset) + (< (file-position real-stream) end-offset)) + (or (read-char real-stream nil) + :eof) + :eof))) + +#+(OR)(defmethod stream-read-byte ((stream delimited-input-stream)) + (with-slots (real-stream end-offset) stream + (if (or (not end-offset) + (< (file-position real-stream) end-offset)) + (or (read-byte real-stream nil) + :eof) + :eof))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass my-string-input-stream (fundamental-character-input-stream coder-stream-mixin) + ((string :initarg :string + :reader stream-string))) + +(defmethod initialize-instance ((stream my-string-input-stream) &key &allow-other-keys) + (call-next-method) + (assert (slot-boundp stream 'string)) + (with-slots (string real-stream) stream + (setf real-stream (make-string-input-stream string)))) + +(defmethod stream-read-char ((stream my-string-input-stream)) + (with-slots (real-stream) stream + (or (read-char real-stream nil) + :eof))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defstruct file-portion + data ; string or a pathname + encoding + start + end) + +(defun open-file-portion (file-portion) + (be data (file-portion-data file-portion) + (etypecase data + (pathname + (be stream (open data) + (make-instance 'delimited-input-stream + :stream stream + :start (file-portion-start file-portion) + :end (file-portion-end file-portion)))) + (string + (make-instance 'delimited-input-stream + :stream (make-string-input-stream data) + :start (file-portion-start file-portion) + :end (file-portion-end file-portion))) + (stream + (make-instance 'delimited-input-stream + :stream data + :dont-close t + :start (file-portion-start file-portion) + :end (file-portion-end file-portion)))))) + +(defun open-decoded-file-portion (file-portion) + (make-instance (case (file-portion-encoding file-portion) + (:quoted-printable 'quoted-printable-decoder-stream) + (:base64 'base64-decoder-stream) + (t '8bit-decoder-stream)) + :stream (open-file-portion file-portion))) diff --git a/third_party/lisp/mime4cl/test/address.lisp b/third_party/lisp/mime4cl/test/address.lisp new file mode 100644 index 000000000000..aaa2d231f11c --- /dev/null +++ b/third_party/lisp/mime4cl/test/address.lisp @@ -0,0 +1,124 @@ +;;; address.lisp --- tests for the e-mail address parser + +;;; Copyright (C) 2007, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +#+cmu (ext:file-comment "$Module: address.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :mime4cl-tests) + +(defun test-parsing (string) + (format nil "~{~A~^, ~}" (parse-addresses string))) + +(deftest address-parse-simple.1 + (test-parsing "foo@bar") + "foo@bar") + +(deftest address-parse-simple.2 + (test-parsing "foo@bar.com") + "foo@bar.com") + +(deftest address-parse-simple.3 + (test-parsing "foo@bar.baz.com") + "foo@bar.baz.com") + +(deftest address-parse-simple.4 + (test-parsing "foo.ooo@bar.baz.com") + "foo.ooo@bar.baz.com") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest address-parse-simple-commented.1 + (test-parsing "foo@bar (Some Comment)") + "\"Some Comment\" <foo@bar>") + +(deftest address-parse-simple-commented.2 + (test-parsing "foo@bar (Some, Comment)") + "\"Some, Comment\" <foo@bar>") + +(deftest address-parse-simple-commented.3 + (test-parsing "foo@bar (Some Comment (yes, indeed))") + "\"Some Comment (yes, indeed)\" <foo@bar>") + +(deftest address-parse-simple-commented.4 + (test-parsing "foo.bar@host.complicated.domain.net (Some Comment (yes, indeed))") + "\"Some Comment (yes, indeed)\" <foo.bar@host.complicated.domain.net>") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest address-parse-angle.1 + (test-parsing "<foo@bar.baz.net>") + "foo@bar.baz.net") + +(deftest address-parse-angle.2 + (test-parsing "My far far friend <foo@bar.baz.net>") + "\"My far far friend\" <foo@bar.baz.net>") + +(deftest address-parse-angle.3 + (test-parsing "\"someone, I don't like\" <foo@bar.baz.net>") + "\"someone, I don't like\" <foo@bar.baz.net>") + +(deftest address-parse-angle.4 + (test-parsing "\"this could (be a comment)\" <foo@bar.net>") + "\"this could (be a comment)\" <foo@bar.net>") + +(deftest address-parse-angle.5 + (test-parsing "don't be fooled <foo@bar.net>") + "\"don't be fooled\" <foo@bar.net>") + +(deftest address-parse-angle.6 + (test-parsing "<foo@bar>") + "foo@bar") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest address-parse-domain-literal.1 + (test-parsing "<foo@[bar]>") + "foo@[bar]") + +(deftest address-parse-domain-literal.2 + (test-parsing "<foo@[bar.net]>") + "foo@[bar.net]") + +(deftest address-parse-domain-literal.3 + (test-parsing "<foo@[10.0.0.2]>") + "foo@[10.0.0.2]") + +(deftest address-parse-domain-literal.4 + (test-parsing "<foo.bar@[10.0.0.2]>") + "foo.bar@[10.0.0.2]") + +(deftest address-parse-domain-literal.5 + (test-parsing "somewhere unkown <foo.bar@[10.0.0.2]>") + "\"somewhere unkown\" <foo.bar@[10.0.0.2]>") + +(deftest address-parse-domain-literal.6 + (test-parsing "\"Some--One\" <foo.bar@[10.0.0.23]>") + "\"Some--One\" <foo.bar@[10.0.0.23]>") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest address-parse-group.1 + (test-parsing "friends:john@bar.in.soho, jack@pub.round.the.corner, jim@[10.0.1.2];") + "friends: john@bar.in.soho, jack@pub.round.the.corner, jim@[10.0.1.2];") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest address-parse-mixed.1 + (test-parsing "Foo BAR <foo@bar.com>, \"John, Smith (that one!)\" <john.smith@host.domain.org>, friends:john@bar,jack@pub;, foo.bar.baz@wow.mail.mine, dont.bark@me (Fierce Dog)") + "\"Foo BAR\" <foo@bar.com>, \"John, Smith (that one!)\" <john.smith@host.domain.org>, friends: john@bar, jack@pub;, foo.bar.baz@wow.mail.mine, \"Fierce Dog\" <dont.bark@me>") diff --git a/third_party/lisp/mime4cl/test/endec.lisp b/third_party/lisp/mime4cl/test/endec.lisp new file mode 100644 index 000000000000..7b6763c99035 --- /dev/null +++ b/third_party/lisp/mime4cl/test/endec.lisp @@ -0,0 +1,167 @@ +;;; endec.lisp --- test suite for the MIME encoder/decoder functions + +;;; Copyright (C) 2006, 2007, 2009, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +#+cmu (ext:file-comment "$Module: endec.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :mime4cl-tests) + +(deftest quoted-printable.1 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "Français, Español, böse, skøl")) + "Fran=E7ais, Espa=F1ol, b=F6se, sk=F8l") + +(deftest quoted-printable.2 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "Français, Español, böse, skøl") + :start 10 :end 17) + "Espa=F1ol") + +(deftest quoted-printable.3 + (map 'string #'code-char + (decode-quoted-printable-string "Fran=E7ais, Espa=F1ol, b=F6se, sk=F8l")) + "Français, Español, böse, skøl") + +(deftest quoted-printable.4 + (map 'string #'code-char + (decode-quoted-printable-string "Fran=E7ais, Espa=F1ol, b=F6se, sk=F8l" + :start 12 :end 21)) + "Español") + +(deftest quoted-printable.5 + (map 'string #'code-char + (decode-quoted-printable-string "this = wrong")) + "this = wrong") + +(deftest quoted-printable.6 + (map 'string #'code-char + (decode-quoted-printable-string "this is wrong=")) + "this is wrong=") + +(deftest quoted-printable.7 + (map 'string #'code-char + (decode-quoted-printable-string "this is wrong=1")) + "this is wrong=1") + +(deftest quoted-printable.8 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "x = x + 1")) + "x =3D x + 1") + +(deftest quoted-printable.9 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "x = x + 1 ")) + "x =3D x + 1 =20") + +(deftest quoted-printable.10 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "this string is very very very very very very very very very very very very very very very very very very very very long")) + "this string is very very very very very very very very very very very ve= +ry very very very very very very very very long") + +(deftest quoted-printable.11 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "this string is very very very very long")) + "this string is very very = + very very long") + +(deftest quoted-printable.12 + (encode-quoted-printable-sequence (map '(vector (unsigned-byte 8)) #'char-code + "please read the next +line")) + "please read the next =20 +line") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest base64.1 + (let ((*base64-line-length* nil)) + (encode-base64-sequence (map '(vector (unsigned-byte 8)) #'char-code + "Some random string."))) + "U29tZSByYW5kb20gc3RyaW5nLg==") + +(deftest base64.2 + (let ((*base64-line-length* nil)) + (encode-base64-sequence (map '(vector (unsigned-byte 8)) #'char-code + "Some random string.") :start 5 :end 11)) + "cmFuZG9t") + +(deftest base64.3 + (map 'string #'code-char + (decode-base64-string "U29tZSByYW5kb20gc3RyaW5nLg==")) + "Some random string.") + +(deftest base64.4 + (map 'string #'code-char + (decode-base64-string "some rubbish U29tZSByYW5kb20gc3RyaW5nLg== more rubbish" + :start 13 :end 41)) + "Some random string.") + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(deftest RFC2047.1 + (parse-RFC2047-text "foo bar") + ("foo bar")) + +(defun perftest-encoder (encoder-class &optional (megs 100)) + (declare (optimize (speed 3) (debug 0) (safety 0)) + (type fixnum megs)) + (with-open-file (in #P"/dev/random" :element-type '(unsigned-byte 8)) + (let* ((meg (* 1024 1024)) + (buffer (make-sequence '(vector (unsigned-byte 8)) meg)) + (encoder (make-instance encoder-class + :output-function #'(lambda (c) (declare (ignore c)))))) + (declare (type fixnum meg)) + (time + (progn + (dotimes (x megs) + (read-sequence buffer in) + (dotimes (i meg) + (mime4cl:encoder-write-byte encoder (aref buffer i)))) + (mime4cl:encoder-finish-output encoder)))))) + +(defun perftest-decoder (decoder-class &optional (megs 100)) + (declare (optimize (speed 3) (debug 0) (safety 0)) + (type fixnum megs)) + (with-open-file (in #P"/dev/random" :element-type '(unsigned-byte 8)) + (let ((sclf:*tmp-file-defaults* (make-pathname :defaults #.(or *load-pathname* *compile-file-pathname*) + :type "encoded-data"))) + (sclf:with-temp-file (tmp nil :direction :io) + (let* ((meg (* 1024 1024)) + (buffer (make-sequence '(vector (unsigned-byte 8)) meg)) + (encoder-class (ecase decoder-class + (mime4cl:base64-decoder 'mime4cl:base64-encoder) + (mime4cl:quoted-printable-decoder 'mime4cl:quoted-printable-encoder))) + (encoder (make-instance encoder-class + :output-function #'(lambda (c) + (write-char c tmp)))) + (decoder (make-instance decoder-class + :input-function #'(lambda () + (read-char tmp nil))))) + (declare (type fixnum meg)) + (dotimes (x megs) + (read-sequence buffer in) + (dotimes (i meg) + (mime4cl:encoder-write-byte encoder (aref buffer i)))) + (mime4cl:encoder-finish-output encoder) + (file-position tmp 0) + (time + (loop + for b = (mime4cl:decoder-read-byte decoder) + while b))))))) diff --git a/third_party/lisp/mime4cl/test/mime.lisp b/third_party/lisp/mime4cl/test/mime.lisp new file mode 100644 index 000000000000..1488f927fcd3 --- /dev/null +++ b/third_party/lisp/mime4cl/test/mime.lisp @@ -0,0 +1,56 @@ + ;;; mime.lisp --- MIME regression tests + + ;;; Copyright (C) 2012 by Walter C. Pelissero + ;;; Copyright (C) 2021 by the TVL Authors + + ;;; Author: Walter C. Pelissero <walter@pelissero.de> + ;;; Project: mime4cl + +#+cmu (ext:file-comment "$Module: mime.lisp") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :mime4cl-tests) + +(defvar *samples-directory* + (merge-pathnames (make-pathname :directory '(:relative "samples")) + #.(or *compile-file-pathname* + *load-pathname* + #P""))) + +(defvar *sample1-file* (make-pathname :defaults #.(or *compile-file-pathname* + *load-pathname*) + :name "sample1" + :type "msg")) + +(deftest mime.1 + (let* ((orig (mime-message *sample1-file*)) + (dup (mime-message (with-output-to-string (out) (encode-mime-part orig out))))) + (mime= orig dup)) + t) + +(deftest mime.2 + (loop + for f in (directory (make-pathname :defaults *samples-directory* + :name :wild + :type "txt")) + do + (format t "~A:~%" f) + (finish-output) + (let* ((orig (mime-message f)) + (dup (mime-message (with-output-to-string (out) (encode-mime-part orig out))))) + (unless (mime= orig dup) + (return nil))) + finally (return t)) + t) diff --git a/third_party/lisp/mime4cl/test/package.lisp b/third_party/lisp/mime4cl/test/package.lisp new file mode 100644 index 000000000000..bde0bf25d5b1 --- /dev/null +++ b/third_party/lisp/mime4cl/test/package.lisp @@ -0,0 +1,28 @@ +;;; package.lisp --- package description for the regression tests + +;;; Copyright (C) 2006, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: mime4cl + +#+cmu (ext:file-comment "$Module: package.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(cl:in-package :common-lisp) + +(defpackage :mime4cl-tests + (:use :common-lisp + :rtest :mime4cl) + (:export)) diff --git a/third_party/lisp/mime4cl/test/rt.lisp b/third_party/lisp/mime4cl/test/rt.lisp new file mode 100644 index 000000000000..d4dd2aedb677 --- /dev/null +++ b/third_party/lisp/mime4cl/test/rt.lisp @@ -0,0 +1,254 @@ +#|----------------------------------------------------------------------------| + | Copyright 1990 by the Massachusetts Institute of Technology, Cambridge MA. | + | | + | Permission to use, copy, modify, and distribute this software and its | + | documentation for any purpose and without fee is hereby granted, provided | + | that this copyright and permission notice appear in all copies and | + | supporting documentation, and that the name of M.I.T. not be used in | + | advertising or publicity pertaining to distribution of the software | + | without specific, written prior permission. M.I.T. makes no | + | representations about the suitability of this software for any purpose. | + | It is provided "as is" without express or implied warranty. | + | | + | M.I.T. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING | + | ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL | + | M.I.T. BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR | + | ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, | + | WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, | + | ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS | + | SOFTWARE. | + |----------------------------------------------------------------------------|# + +(defpackage #:regression-test + (:nicknames #:rtest #-lispworks #:rt) + (:use #:cl) + (:export #:*do-tests-when-defined* #:*test* #:continue-testing + #:deftest #:do-test #:do-tests #:get-test #:pending-tests + #:rem-all-tests #:rem-test) + (:documentation "The MIT regression tester with pfdietz's modifications")) + +(in-package :regression-test) + +(defvar *test* nil "Current test name") +(defvar *do-tests-when-defined* nil) +(defvar *entries* '(nil) "Test database") +(defvar *in-test* nil "Used by TEST") +(defvar *debug* nil "For debugging") +(defvar *catch-errors* t + "When true, causes errors in a test to be caught.") +(defvar *print-circle-on-failure* nil + "Failure reports are printed with *PRINT-CIRCLE* bound to this value.") +(defvar *compile-tests* nil + "When true, compile the tests before running them.") +(defvar *optimization-settings* '((safety 3))) +(defvar *expected-failures* nil + "A list of test names that are expected to fail.") + +(defstruct (entry (:conc-name nil) + (:type list)) + pend name form) + +(defmacro vals (entry) `(cdddr ,entry)) + +(defmacro defn (entry) `(cdr ,entry)) + +(defun pending-tests () + (do ((l (cdr *entries*) (cdr l)) + (r nil)) + ((null l) (nreverse r)) + (when (pend (car l)) + (push (name (car l)) r)))) + +(defun rem-all-tests () + (setq *entries* (list nil)) + nil) + +(defun rem-test (&optional (name *test*)) + (do ((l *entries* (cdr l))) + ((null (cdr l)) nil) + (when (equal (name (cadr l)) name) + (setf (cdr l) (cddr l)) + (return name)))) + +(defun get-test (&optional (name *test*)) + (defn (get-entry name))) + +(defun get-entry (name) + (let ((entry (find name (cdr *entries*) + :key #'name + :test #'equal))) + (when (null entry) + (report-error t + "~%No test with name ~:@(~S~)." + name)) + entry)) + +(defmacro deftest (name form &rest values) + `(add-entry '(t ,name ,form .,values))) + +(defun add-entry (entry) + (setq entry (copy-list entry)) + (do ((l *entries* (cdr l))) (nil) + (when (null (cdr l)) + (setf (cdr l) (list entry)) + (return nil)) + (when (equal (name (cadr l)) + (name entry)) + (setf (cadr l) entry) + (report-error nil + "Redefining test ~:@(~S~)" + (name entry)) + (return nil))) + (when *do-tests-when-defined* + (do-entry entry)) + (setq *test* (name entry))) + +(defun report-error (error? &rest args) + (cond (*debug* + (apply #'format t args) + (if error? (throw '*debug* nil))) + (error? (apply #'error args)) + (t (apply #'warn args)))) + +(defun do-test (&optional (name *test*)) + (do-entry (get-entry name))) + +(defun equalp-with-case (x y) + "Like EQUALP, but doesn't do case conversion of characters." + (cond + ((eq x y) t) + ((consp x) + (and (consp y) + (equalp-with-case (car x) (car y)) + (equalp-with-case (cdr x) (cdr y)))) + ((and (typep x 'array) + (= (array-rank x) 0)) + (equalp-with-case (aref x) (aref y))) + ((typep x 'vector) + (and (typep y 'vector) + (let ((x-len (length x)) + (y-len (length y))) + (and (eql x-len y-len) + (loop + for e1 across x + for e2 across y + always (equalp-with-case e1 e2)))))) + ((and (typep x 'array) + (typep y 'array) + (not (equal (array-dimensions x) + (array-dimensions y)))) + nil) + ((typep x 'array) + (and (typep y 'array) + (let ((size (array-total-size x))) + (loop for i from 0 below size + always (equalp-with-case (row-major-aref x i) + (row-major-aref y i)))))) + (t (eql x y)))) + +(defun do-entry (entry &optional + (s *standard-output*)) + (catch '*in-test* + (setq *test* (name entry)) + (setf (pend entry) t) + (let* ((*in-test* t) + ;; (*break-on-warnings* t) + (aborted nil) + r) + ;; (declare (special *break-on-warnings*)) + + (block aborted + (setf r + (flet ((%do + () + (if *compile-tests* + (multiple-value-list + (funcall (compile + nil + `(lambda () + (declare + (optimize ,@*optimization-settings*)) + ,(form entry))))) + (multiple-value-list + (eval (form entry)))))) + (if *catch-errors* + (handler-bind + ((style-warning #'muffle-warning) + (error #'(lambda (c) + (setf aborted t) + (setf r (list c)) + (return-from aborted nil)))) + (%do)) + (%do))))) + + (setf (pend entry) + (or aborted + (not (equalp-with-case r (vals entry))))) + + (when (pend entry) + (let ((*print-circle* *print-circle-on-failure*)) + (format s "~&Test ~:@(~S~) failed~ + ~%Form: ~S~ + ~%Expected value~P: ~ + ~{~S~^~%~17t~}~%" + *test* (form entry) + (length (vals entry)) + (vals entry)) + (format s "Actual value~P: ~ + ~{~S~^~%~15t~}.~%" + (length r) r))))) + (when (not (pend entry)) *test*)) + +(defun continue-testing () + (if *in-test* + (throw '*in-test* nil) + (do-entries *standard-output*))) + +(defun do-tests (&optional + (out *standard-output*)) + (dolist (entry (cdr *entries*)) + (setf (pend entry) t)) + (if (streamp out) + (do-entries out) + (with-open-file + (stream out :direction :output) + (do-entries stream)))) + +(defun do-entries (s) + (format s "~&Doing ~A pending test~:P ~ + of ~A tests total.~%" + (count t (cdr *entries*) + :key #'pend) + (length (cdr *entries*))) + (dolist (entry (cdr *entries*)) + (when (pend entry) + (format s "~@[~<~%~:; ~:@(~S~)~>~]" + (do-entry entry s)))) + (let ((pending (pending-tests)) + (expected-table (make-hash-table :test #'equal))) + (dolist (ex *expected-failures*) + (setf (gethash ex expected-table) t)) + (let ((new-failures + (loop for pend in pending + unless (gethash pend expected-table) + collect pend))) + (if (null pending) + (format s "~&No tests failed.") + (progn + (format s "~&~A out of ~A ~ + total tests failed: ~ + ~:@(~{~<~% ~1:;~S~>~ + ~^, ~}~)." + (length pending) + (length (cdr *entries*)) + pending) + (if (null new-failures) + (format s "~&No unexpected failures.") + (when *expected-failures* + (format s "~&~A unexpected failures: ~ + ~:@(~{~<~% ~1:;~S~>~ + ~^, ~}~)." + (length new-failures) + new-failures))) + )) + (null pending)))) diff --git a/third_party/lisp/mime4cl/test/sample1.msg b/third_party/lisp/mime4cl/test/sample1.msg new file mode 100644 index 000000000000..662a9fab341e --- /dev/null +++ b/third_party/lisp/mime4cl/test/sample1.msg @@ -0,0 +1,86 @@ +From wcp@scylla.home.lan Fri Feb 17 11:02:28 2012 +Status: RO +X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil] + ["1133" "Friday" "17" "February" "2012" "11:02:27" "+0100" "Walter C. Pelissero" "walter@pelissero.de" nil "56" "test" "^From:" nil nil "2" nil nil nil nil nil nil nil nil nil nil] + nil) +X-Clpmr-Processed: 2012-02-17T11:02:31 +X-Clpmr-Version: 2011-10-23T12:55:20, SBCL 1.0.49 +Received: from scylla.home.lan (localhost [127.0.0.1]) + by scylla.home.lan (8.14.5/8.14.5) with ESMTP id q1HA2Sik004513 + for <wcp@scylla.home.lan>; Fri, 17 Feb 2012 11:02:28 +0100 (CET) + (envelope-from wcp@scylla.home.lan) +Received: (from wcp@localhost) + by scylla.home.lan (8.14.5/8.14.5/Submit) id q1HA2SqU004512; + Fri, 17 Feb 2012 11:02:28 +0100 (CET) + (envelope-from wcp) +Message-ID: <20286.9651.890757.323027@scylla.home.lan> +X-Mailer: VM 8.1.1 under 23.3.1 (amd64-portbld-freebsd8.2) +Reply-To: walter@pelissero.de +X-Attribution: WP +X-For-Spammers: blacklistme@pelissero.de +X-MArch-Processing-Time: 0.552s +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="615CiWUaGO" +Content-Transfer-Encoding: 7BIT +From: walter@pelissero.de (Walter C. Pelissero) +To: wcp@scylla.home.lan +Subject: test +Date: Fri, 17 Feb 2012 11:02:27 +0100 + + +--615CiWUaGO +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7BIT +Content-Description: message body text + +Hereafter three attachments. + +The first: + +--615CiWUaGO +Content-Type: application/octet-stream; name="attach1" +Content-Transfer-Encoding: BASE64 +Content-Disposition: attachment; filename="attach1" + +YXR0YWNoMQo= + +--615CiWUaGO +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7BIT +Content-Description: message body text + + +The second: + +--615CiWUaGO +Content-Type: application/octet-stream; name="attach2" +Content-Transfer-Encoding: BASE64 +Content-Disposition: attachment; filename="attach2" + +YXR0YWNoMgo= + +--615CiWUaGO +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7BIT +Content-Description: message body text + + +The third: + +--615CiWUaGO +Content-Type: application/octet-stream; name="attach3" +Content-Transfer-Encoding: BASE64 +Content-Disposition: attachment; filename="attach3" + +YXR0YWNoMwo= + +--615CiWUaGO +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7BIT +Content-Description: .signature + + +-- +http://pelissero.de +--615CiWUaGO-- + diff --git a/third_party/lisp/moptilities.nix b/third_party/lisp/moptilities.nix new file mode 100644 index 000000000000..a8a387ab914f --- /dev/null +++ b/third_party/lisp/moptilities.nix @@ -0,0 +1,18 @@ +# Compatibility layer for minor MOP implementation differences +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "gwkkwg"; + repo = "moptilities"; + rev = "a436f16b357c96b82397ec018ea469574c10dd41"; + sha256 = "1q12bqjbj47lx98yim1kfnnhgfhkl80102fkgp9pdqxg0fp6g5fc"; +}; +in depot.nix.buildLisp.library { + name = "moptilities"; + deps = [ depot.third_party.lisp.closer-mop ]; + srcs = [ "${src}/dev/moptilities.lisp" ]; + + brokenOn = [ + "ecl" # TODO(sterni): https://gitlab.com/embeddable-common-lisp/ecl/-/issues/651 + ]; +} diff --git a/third_party/lisp/nibbles.nix b/third_party/lisp/nibbles.nix new file mode 100644 index 000000000000..da542fe91130 --- /dev/null +++ b/third_party/lisp/nibbles.nix @@ -0,0 +1,32 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.nix.buildLisp) bundled; + src = pkgs.fetchFromGitHub { + owner = "sharplispers"; + repo = "nibbles"; + rev = "dad25240928d5cf8f7df69c4398244e03570bb35"; + sha256 = "0r6ljlpgjmkf87pmvdwzva8qj15bhznc3ylgcjjqyy4frbx9lygz"; + name = "nibbles-source"; + }; + +in depot.nix.buildLisp.library { + name = "nibbles"; + + deps = with depot.third_party.lisp; [ + (bundled "asdf") + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "types.lisp" + "macro-utils.lisp" + "vectors.lisp" + "streams.lisp" + ] ++ [ + { sbcl = "${src}/sbcl-opt/fndb.lisp"; } + { sbcl = "${src}/sbcl-opt/nib-tran.lisp"; } + { sbcl = "${src}/sbcl-opt/x86-vm.lisp"; } + { sbcl = "${src}/sbcl-opt/x86-64-vm.lisp"; } + ]; +} diff --git a/third_party/lisp/npg/.project b/third_party/lisp/npg/.project new file mode 100644 index 000000000000..82a8fe48bbfb --- /dev/null +++ b/third_party/lisp/npg/.project @@ -0,0 +1 @@ +NPG a Naive Parser Generator diff --git a/third_party/lisp/npg/.skip-subtree b/third_party/lisp/npg/.skip-subtree new file mode 100644 index 000000000000..5051f60d6b86 --- /dev/null +++ b/third_party/lisp/npg/.skip-subtree @@ -0,0 +1 @@ +prevent readTree from creating entries for subdirs that don't contain an .nix files diff --git a/third_party/lisp/npg/COPYING b/third_party/lisp/npg/COPYING new file mode 100644 index 000000000000..223ede7de3ec --- /dev/null +++ b/third_party/lisp/npg/COPYING @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/third_party/lisp/npg/OWNERS b/third_party/lisp/npg/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/third_party/lisp/npg/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/third_party/lisp/npg/README b/third_party/lisp/npg/README new file mode 100644 index 000000000000..a1661e744a37 --- /dev/null +++ b/third_party/lisp/npg/README @@ -0,0 +1,48 @@ + + NPG a Naive Parser Generator + for Common Lisp + + Copyright (C) 2003-2006, 2010 by Walter C. Pelissero + Copyright (C) 2021 by the TVL Authors + +Vendored into depot as it is a dependency of mime4cl and upstream has +become inactive. Upstream and depot version may diverge. + +Upstream Website: http://wcp.sdf-eu.org/software/#npg +Vendored Tarball: http://wcp.sdf-eu.org/software/npg-20150517T144652.tbz + +This library is free software; you can redistribute it and/or modify +it under the terms of the GNU Lesser General Public License as +published by the Free Software Foundation; either version 2.1 of the +License, or (at your option) any later version. This library is +distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +License for more details. You should have received a copy of the GNU +Lesser General Public License along with this library; if not, write +to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, +Boston, MA 02111-1307 USA + + +This library generates on the fly (no external representation of the +parser is produced) a recursive descent parser based on the grammar +rules you have fed it with. The parser object can then be used to +scan tokenised input. Although a facility to produce a lexical +analiser is not provided, to write such a library is fairly easy for +most languages. NPG parsers require your lexer to adhere to a certain +protocol to be able to communicate with them. Examples are provided +that explain these requirements. + +While quite possibly not producing the fastest parsers in town, it's +fairly simple and hopefully easy to debug. It accepts a lispy EBNF +grammar description of arbitrary complexity with the exception of +mutually left recursive rules (watch out, they produce undetected +infinite recursion) and produces a backtracking recursive descent +parser. Immediate left recursive rules are properly simplified, +though. + +Multiple concurrent parsers are supported. + +To compile, an ASDF and nix file are provided. + +See the examples directory for clues on how to use it. diff --git a/third_party/lisp/npg/default.nix b/third_party/lisp/npg/default.nix new file mode 100644 index 000000000000..af7ec53eaf93 --- /dev/null +++ b/third_party/lisp/npg/default.nix @@ -0,0 +1,14 @@ +# Copyright (C) 2021 by the TVL Authors +# SPDX-License-Identifier: LGPL-2.1-or-later +{ depot, pkgs, ... }: + +depot.nix.buildLisp.library { + name = "npg"; + + srcs = [ + ./src/package.lisp + ./src/common.lisp + ./src/define.lisp + ./src/parser.lisp + ]; +} diff --git a/third_party/lisp/npg/examples/python.lisp b/third_party/lisp/npg/examples/python.lisp new file mode 100644 index 000000000000..68d794ddec03 --- /dev/null +++ b/third_party/lisp/npg/examples/python.lisp @@ -0,0 +1,336 @@ +;;; python.lisp --- sample grammar definition for the Python language + +;;; Copyright (C) 2003 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator +;;; $Id: F-C1A8CD5961889C584B22F05E8B956006.lisp,v 1.3 2004/03/09 10:33:06 wcp Exp $ + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +;;; Commentary: +;;; +;;; This is far from being a complete Python grammar. Actually I +;;; haven't even read a Python book before starting to write this +;;; stuff, so the code below comes mostly from wild guessing while +;;; reading a Python source file. +;;; +;;; It's a design decision to avoid writing any transformation in this +;;; module; only tagging is done at this level. This improves the +;;; separation between parsing and transformation, making the grammar +;;; reusable for other purposes. + + +#+cmu (ext:file-comment "$Id: F-C1A8CD5961889C584B22F05E8B956006.lisp,v 1.3 2004/03/09 10:33:06 wcp Exp $") + +(in-package :grammar) + +(deflazy define-grammar + (let ((*package* #.*package*) + (*compile-print* (and parser::*debug* t))) + (reset-grammar) + (format t "~&creating Python grammar...~%") + (populate-grammar) + (let ((grammar (parser:generate-grammar))) + (reset-grammar) + (parser:print-grammar-figures grammar) + grammar))) + +(defun populate-grammar () + +(defrule program + := comment-string? statement+) + +(defrule comment-string + := string eol + :reduce string) + +;;; BOB = Beginning Of Block, EOB = End Of Block. It's lexical +;;; analyzer's task to find out where a statement or block starts/ends. + +(defrule suite + := statement-list eol + :reduce statement-list + := statement-block) + +(defrule commentable-suite + := statement-list eol + :reduce statement-list + := commented-statement-block) + +(defrule statement-block + := bob statement+ eob + :reduce $2) + +(defrule commented-statement-block + := bob comment-string? statement* eob + :reduce (cons comment-string statement)) + +(defrule statement-list + := (+ simple-statement ";") + :reduce (if (cdr $1) + (cons :statement-list $1) + (car $1))) + +(defrule statement + := statement-list eol + :reduce statement-list + := compound-statement) + +(defrule simple-statement + := import-statement + := raise-statement + := assignment + := function-call + := return-statement + := assert-statement + := pass-statement + := break-statement + := continue-statement) + +(defrule compound-statement + := class-definition + := method-definition + := try-statement + := if-statement + := while-statement + := for-statement) + +(defrule import-statement + := "import" (+ package-name ",") + :tag :import + := "from" package-name "import" (+ symbol-name ",") + :tag :import-from) + +(defrule package-name := identifier) + +(defrule symbol-name + := identifier + := "*") + +(defrule try-statement + := "try" ":" suite try-except-part* try-finally-part? + :tag :try) + +(defrule try-except-part + := "except" exception-subject? ":" suite) + +(defrule try-finally-part + := "finally" ":" suite) + +(defrule exception-subject + := exception-name exception-variable?) + +(defrule exception-variable + := "," identifier) + +(defrule exception-name := class-name) + +(defrule class-name := identifier) + +(defrule raise-statement + := "raise" + :tag :raise-same + := "raise" exception-name + :tag :raise + := "raise" exception-name "," expression + :tag :raise + := "raise" exception-name "(" expression ")" + :tag :raise) + +(defrule assignment + := (+ variable-with-optional-subscript ",") "=" more-assignment + :tag :set) + +(defrule more-assignment + := expression + := assignment) + +(defrule variable-with-optional-subscript + := variable-name subscript + :tag :subscript + := variable-name) + +(defrule variable-name + := (+ identifier ".") + :tag :varef) + +(defrule expression + := expression "or" expression1 + :tag :or + := expression1) + +(defrule expression1 + := expression1 "and" expression2 + :tag :and + := expression2) + +(defrule expression2 + := expression2 "==" expression3 + :tag :equal + := expression2 ">=" expression3 + :tag :more-equal + := expression2 "<=" expression3 + :tag :less-equal + := expression2 "!=" expression3 + :tag :not-equal + := expression2 ">" expression3 + :tag :more + := expression2 "<" expression3 + :tag :less + := expression2 "is" expression3 + :tag :equal + := expression2 "is" "not" expression3 + :tag :not-equal + := expression3) + +(defrule expression3 + := expression3 "+" expression4 + :tag :plus + := expression3 "-" expression4 + :tag :minus + := expression3 "|" expression4 + :tag :bit-or + := expression4) + +;; high priority expression +(defrule expression4 + := expression4 "*" expression5 + :tag :mult + := expression4 "/" expression5 + :tag :div + := expression4 "%" expression5 + :tag :modulo + := expression4 "&" expression5 + :tag :bit-and + := expression4 "in" expression5 + :tag :in + := expression5) + +(defrule expression5 + := "~" expression5 + :tag :bit-not + := "not" expression5 + :tag :not + := "(" expression ")" + := expression6) + +(defrule expression6 + := simple-expression subscript + :tag :subscript + := simple-expression) + +(defrule simple-expression + := function-call + := variable-name + := constant + := string-conversion + := list-constructor) + +(defrule subscript + := "[" expression "]" + := "[" expression ":" expression "]" + := "[" expression ":" "]" + :reduce (list expression nil) + := "[" ":" expression "]" + :reduce (list nil expression)) + +(defrule string-conversion + := "`" expression "`" + :tag :to-string) + +(defrule constant + := number + := string + := lambda-expression) + +(defrule number + := float + := integer) + +(defrule list-constructor + := "[" (* expression ",") "]" + :tag :make-list) + +(defrule class-definition + := "class" class-name superclasses? ":" commentable-suite + :tag :defclass) + +(defrule superclasses + := "(" class-name+ ")") + +(defrule method-definition + := "def" method-name "(" method-arguments ")" ":" commentable-suite + :tag :defmethod) + +(defrule method-arguments + := (* method-argument ",")) + +(defrule method-argument + := identifier argument-default?) + +(defrule argument-default + := "=" expression) + +(defrule method-name := identifier) + +(defrule if-statement + := "if" expression ":" suite elif-part* else-part? + :tag :if) + +(defrule else-part + := "else" ":" suite) + +(defrule elif-part + := "elif" expression ":" suite) + +(defrule lambda-expression + := "lambda" method-arguments ":" expression + :tag :lambda) + +(defrule function-call + := (+ identifier ".") "(" (* expression ",") ")" + :tag :funcall) + +(defrule for-statement + := "for" identifier "in" expression ":" suite + :tag :do-list + := "for" identifier "in" "range" "(" expression "," expression ")" ":" suite + :tag :do-range) + +(defrule while-statement + := "while" expression ":" suite + :tag :while) + +(defrule return-statement + := "return" expression? + :tag :return) + +(defrule assert-statement + := "assert" expression "," string + :tag :assert) + +(defrule pass-statement + := "pass" + :tag :pass) + +(defrule break-statement + := "break" + :tag :break) + +(defrule continue-statement + := "continue" + :tag :continue) + +) ; end of POPULATE-GRAMMAR diff --git a/third_party/lisp/npg/examples/vs-cobol-ii.lisp b/third_party/lisp/npg/examples/vs-cobol-ii.lisp new file mode 100644 index 000000000000..2edf1292da53 --- /dev/null +++ b/third_party/lisp/npg/examples/vs-cobol-ii.lisp @@ -0,0 +1,1901 @@ +;;; vs-cobol-ii.lisp --- sample grammar for VS-Cobol II + +;;; Copyright (C) 2003 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator +;;; $Id: F-1D03709AEB30BA7644C1CFA2DF60FE8C.lisp,v 1.2 2004/03/09 10:33:07 wcp Exp $ + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +;;; Commentary: +;;; +;;; A fairly incomplete VS-Cobol II grammar fro NPG. It's probably +;;; not very accurate either. + +#+cmu (ext:file-comment "$Id: F-1D03709AEB30BA7644C1CFA2DF60FE8C.lisp,v 1.2 2004/03/09 10:33:07 wcp Exp $") + +(in-package :grammar) + +(defun make-keyword (string) + "Create a keyword from STRING." + (intern (string-upcase string) :keyword)) + +(defun flatten-list (list) + "Remove one depth level in LIST." + (mapcan #'identity list)) + +(deflazy define-grammar + (let ((*package* #.*package*) + (*compile-print* (and parser::*debug* t))) + (reset-grammar) + (format t "creating Cobol grammar...~%") + (populate-grammar) + (let ((grammar (parser:generate-grammar))) + (reset-grammar) + (parser:print-grammar-figures grammar) + grammar))) + +(defun populate-grammar () +;;; +;;; Hereafter PP means Partial Program +;;; + +#+nil +(defrule pp--declarations + := identification-division environment-division? data-division? "PROCEDURE" "DIVISION" using-phrase? "." :rest) + +;;; We need to split the parsing of the declarations from the rest +;;; because the declarations may change the lexical rules (ie decimal +;;; point) + +(defrule pp--declarations + := identification-division environment-division? data-division-head-or-procedure-division-head :rest) + +(defrule data-division-head-or-procedure-division-head + := data-division-head + :reduce :data-division + := procedure-division-head + :reduce (list :procedure-division $1)) + +(defrule pp--data-division + := data-division-content procedure-division-head :rest) + +(defrule pp--sentence + := sentence :rest + := :eof) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; The real grammar +;;; + +(defrule cobol-source-program + := identification-division environment-division? data-division procedure-division end-program?) + +(defrule identification-division + := identification "DIVISION" "." program-id-cobol-source-program identification-division-content + :reduce program-id-cobol-source-program) + +(defrule priority-number + := integer) + +(defrule level-number + := integer) + +(defrule to-id-or-lit + := "TO" id-or-lit) + +(defrule inspect-by-argument + := variable-identifier + := string + := figurative-constant-simple) + +(defrule figurative-constant-simple + := "ZERO" + :reduce :zero + := "ZEROS" + :reduce :zero + := "ZEROES" + :reduce :zero + := "SPACE" + :reduce :space + := "SPACES" + :reduce :space + := "HIGH-VALUE" + :reduce :high + := "HIGH-VALUES" + :reduce :high + := "LOW-VALUE" + :reduce :low + := "LOW-VALUES" + :reduce :low + := "QUOTE" + :reduce :quote + := "QUOTES" + :reduce :quote + := "NULL" + :reduce :null + := "NULLS" + :reduce :null) + +(defrule write-exceptions + := at-end-of-page-statement-list? not-at-end-of-page-statement-list? invalid-key-statement-list? not-invalid-key-statement-list?) + +(defrule set-statement-phrase + := variable-identifier+ set-oper set-src) + +(defrule set-src + := variable-identifier + := literal + := "TRUE" + := "ON" + := "OFF") + +(defrule set-oper + := "TO" + :reduce :to + := "UP" "BY" + :reduce :up + := "DOWN" "BY" + :reduce :down) + +(defrule fce-phrase + := reserve-clause + := fce-organization + := fce-access-mode + := record-key-clause + := password-clause + := alternate-record-key-clause + := file-status-clause + := padding-character-clause + := record-delimiter-clause) + +(defrule fce-organization + := organization-is? alt-indexed-relative-sequential + :reduce (list :organization (make-keyword alt-indexed-relative-sequential))) + +(defrule fce-access-mode + := "ACCESS" "MODE"? "IS"? alt-sequential-random-dynamic relative-key-clause? + :reduce (list :access-mode (make-keyword alt-sequential-random-dynamic))) + +(defrule alt-indexed-relative-sequential + := "INDEXED" + := "RELATIVE" + := "SEQUENTIAL") + +(defrule is-not + := "IS"? "NOT"?) + +(defrule all-procedures + := "ALL" "PROCEDURES") + +(defrule next-sentence + := "NEXT" "SENTENCE") + +(defrule no-rewind + := "NO" "REWIND") + +(defrule for-removal + := "FOR"? "REMOVAL") + +(defrule values + := "VALUE" + := "VALUES") + +(defrule records + := "RECORD" + := "RECORDS") + +(defrule end-program + := "END" "PROGRAM" program-name ".") + +(defrule environment-division + := "ENVIRONMENT" "DIVISION" "." environment-division-content) + +(defrule data-division-head + := "DATA" "DIVISION" ".") + +(defrule data-division + := data-division-head data-division-content + :reduce data-division-content) + +(defrule identification + := "IDENTIFICATION" + := "ID") + +(defrule identification-division-content + := identification-division-phrase*) + +(defrule author + := "AUTHOR" ".") + +(defrule installation + := "INSTALLATION" ".") + +(defrule date-written + := "DATE-WRITTEN" ".") + +(defrule date-compiled + := "DATE-COMPILED" ".") + +(defrule security + := "SECURITY" ".") + +(defrule remarks + := "REMARKS" ".") + +(defrule identification-division-phrase + := author + := installation + := date-written + := date-compiled + := security + := remarks) + +(defrule program-id-cobol-source-program + := "PROGRAM-ID" "."? program-name initial-program? "." + :reduce program-name) + +(defrule initial-program + := "IS"? "INITIAL" "PROGRAM"?) + +(defrule environment-division-content + := configuration-section? input-output-section?) + +(defrule input-output-section + := "INPUT-OUTPUT" "SECTION" "." file-control-paragraph? i-o-control-paragraph? + :reduce file-control-paragraph) + +(defrule file-control-paragraph + := "FILE-CONTROL" "." file-control-entry*) + +(defrule file-control-entry + := select-clause assign-clause fce-phrase* "." + :reduce (append select-clause + assign-clause + (flatten-list fce-phrase))) + +(defrule organization-is + := "ORGANIZATION" "IS"?) + +(defrule alt-sequential-random-dynamic + := "SEQUENTIAL" + := "RANDOM" + := "DYNAMIC") + +(defrule select-clause + := "SELECT" "OPTIONAL"? file-name + :reduce (list file-name :optional (and $2 t))) + +(defrule assign-clause + := "ASSIGN" "TO"? alt-assignment-name-literal+ + :reduce (list :assign alt-assignment-name-literal)) + +(defrule alt-assignment-name-literal + := assignment-name + := literal) + +(defrule reserve-clause + := "RESERVE" integer areas?) + +(defrule areas + := "AREA" + := "AREAS") + +(defrule padding-character-clause + := "PADDING" "CHARACTER"? "IS"? alt-qualified-data-name-literal) + +(defrule record-delimiter-clause + := "RECORD" "DELIMITER" "IS"? record-delimiter-name) + +(defrule record-delimiter-name + := "STANDARD-1" + := assignment-name) + +(defrule password-clause + := "PASSWORD" "IS"? data-name) + +(defrule file-status-clause + := "FILE"? "STATUS" "IS"? qualified-data-name qualified-data-name? + :reduce (list :file-status qualified-data-name)) + +(defrule relative-key-clause + := "RELATIVE" "KEY"? "IS"? qualified-data-name + :reduce (list :relative-key qualified-data-name)) + +(defrule record-key-clause + := "RECORD" "KEY"? "IS"? qualified-data-name + :reduce (list :key qualified-data-name)) + +(defrule alternate-record-key-clause + := "ALTERNATE" "RECORD"? "KEY"? "IS"? qualified-data-name password-clause? with-duplicates? + :reduce (list :alternate-key qualified-data-name with-duplicates)) + +(defrule with-duplicates + := "WITH"? "DUPLICATES") + +(defrule i-o-control-paragraph + := "I-O-CONTROL" "." i-o-sam? i-o-sort-merge?) + +(defrule i-o-sam + := qsam-or-sam-or-vsam-i-o-control-entries+ ".") + +(defrule i-o-sort-merge + := sort-merge-i-o-control-entries ".") + +(defrule qsam-or-sam-or-vsam-i-o-control-entries + := qsam-or-sam-or-vsam-i-o-control-entries-1 + := qsam-or-sam-or-vsam-i-o-control-entries-2 + := qsam-or-sam-or-vsam-i-o-control-entries-3 + := qsam-or-sam-or-vsam-i-o-control-entries-4) + +(defrule qsam-or-sam-or-vsam-i-o-control-entries-1 + := "RERUN" "ON" alt-assignment-name-file-name "EVERY"? every-phrase "OF"? file-name) + +(defrule every-phrase-1 + := integer "RECORDS") + +(defrule every-phrase-2 + := "END" "OF"? alt-reel-unit) + +(defrule every-phrase + := every-phrase-1 + := every-phrase-2) + +(defrule alt-assignment-name-file-name + := assignment-name + := file-name) + +(defrule qsam-or-sam-or-vsam-i-o-control-entries-2 + := "SAME" "RECORD"? "AREA"? "FOR"? file-name file-name+) + +(defrule qsam-or-sam-or-vsam-i-o-control-entries-3 + := "MULTIPLE" "FILE" "TAPE"? "CONTAINS"? file-name-position+) + +(defrule position + := "POSITION" integer) + +(defrule file-name-position + := file-name position?) + +(defrule qsam-or-sam-or-vsam-i-o-control-entries-4 + := "APPLY" "WRITE-ONLY" "ON"? file-name+) + +(defrule sort-merge-i-o-control-entries + := rerun-on? same-area+) + +(defrule rerun-on + := "RERUN" "ON" assignment-name) + +(defrule record-sort + := "RECORD" + := "SORT" + := "SORT-MERGE") + +(defrule same-area + := "SAME" record-sort "AREA"? "FOR"? file-name file-name+) + +(defrule configuration-section + := "CONFIGURATION" "SECTION" "." configuration-section-paragraph* + :reduce (flatten-list configuration-section-paragraph)) + +(defrule configuration-section-paragraph + := source-computer-paragraph + := object-computer-paragraph + := special-names-paragraph) + +(defrule source-computer-paragraph + := "SOURCE-COMPUTER" "." source-computer-name + :reduce (list :source-computer source-computer-name)) + +(defrule with-debugging-mode + := "WITH"? "DEBUGGING" "MODE") + +(defrule source-computer-name + := computer-name with-debugging-mode? "." + :reduce computer-name) + +(defrule object-computer-paragraph + := "OBJECT-COMPUTER" "." object-computer-name + :reduce (list :object-computer object-computer-name)) + +(defrule memory-size-type + := "WORDS" + := "CHARACTERS" + := "MODULES") + +(defrule memory-size + := "MEMORY" "SIZE"? integer memory-size-type) + +(defrule object-computer-name + := computer-name memory-size? object-computer-paragraph-sequence-phrase "." + :reduce computer-name) + +(defrule object-computer-paragraph-sequence-phrase + := program-collating-sequence? segment-limit?) + +(defrule program-collating-sequence + := "PROGRAM"? "COLLATING"? "SEQUENCE" "IS"? alphabet-name) + +(defrule segment-limit + := "SEGMENT-LIMIT" "IS"? priority-number) + +(defrule special-names-paragraph + := "SPECIAL-NAMES" "." special-names-paragraph-phrase* special-names-paragraph-clause* "." + :reduce (flatten-list special-names-paragraph-clause)) + +(defrule is-mnemonic-name + := "IS"? mnemonic-name special-names-paragraph-status-phrase?) + +(defrule special-names-paragraph-phrase-tail + := is-mnemonic-name + := special-names-paragraph-status-phrase) + +(defrule special-names-paragraph-phrase + := environment-name special-names-paragraph-phrase-tail) + +(defrule special-names-paragraph-status-phrase + := special-names-paragraph-status-phrase-1 + := special-names-paragraph-status-phrase-2) + +(defrule special-names-paragraph-status-phrase-1 + := "ON" "STATUS"? "IS"? condition off-status?) + +(defrule off-status + := "OFF" "STATUS"? "IS"? condition) + +(defrule special-names-paragraph-status-phrase-2 + := "OFF" "STATUS"? "IS"? condition on-status?) + +(defrule on-status + := "ON" "STATUS"? "IS"? condition) + +(defrule special-names-paragraph-clause + ;; := alphabet-clause + ;; := symbolic-characters-clause + := currency-sign-clause + := decimal-point-clause) + +(defrule alphabet-clause + := "ALPHABET" alphabet-name "IS"? alphabet-type) + +(defrule alphabet-type-also + := "ALSO" literal) + +(defrule alphabet-type-alsos + := alphabet-type-also+) + +(defrule alphabet-type-also-through + := through-literal + := alphabet-type-alsos) + +(defrule alphabet-type-other + := literal alphabet-type-also-through?) + +(defrule alphabet-type-others + := alphabet-type-other+) + +(defrule alphabet-type + := "STANDARD-1" + := "STANDARD-2" + := "NATIVE" + := "EBCDIC" + := alphabet-type-others) + +(defrule symbolic-characters-clause + := "SYMBOLIC" "CHARACTERS"? symbolic-character-mapping+ in-alphabet-name?) + +(defrule are + := "ARE" + := "IS") + +(defrule symbolic-character-mapping + := symbolic-character+ are? integer+) + +(defrule in-alphabet-name + := "IN" alphabet-name) + +(defrule currency-sign-clause + := "CURRENCY" "SIGN"? "IS"? literal + :reduce (list :currency-sign literal)) + +(defrule decimal-point-clause + := "DECIMAL-POINT" "IS"? "COMMA" + :reduce (list :decimal-point #\,)) + +(defrule data-division-content + := file-section? working-storage-section? linkage-section?) + +(defrule file-section-entry + := file-and-sort-description-entry data-description-entry+ + :reduce (cons file-and-sort-description-entry data-description-entry)) + +(defrule file-section-head + := "FILE" "SECTION" ".") + +(defrule file-section + := file-section-head file-section-entry* + :reduce $2) + +(defrule working-storage-section-head + := "WORKING-STORAGE" "SECTION" ".") + +(defrule working-storage-section + := working-storage-section-head data-description-entry* + :reduce $2) + +(defrule linkage-section-head + := "LINKAGE" "SECTION" ".") + +(defrule linkage-section + := linkage-section-head data-description-entry* + :reduce $2) + +(defrule file-and-sort-description-entry + := alt-fd-sd file-name file-and-sort-description-entry-clause* "." + :reduce (list (make-keyword alt-fd-sd) file-name file-and-sort-description-entry-clause)) + +(defrule alt-fd-sd + := "FD" + := "SD") + +(defrule file-and-sort-description-entry-clause + := external-clause + := global-clause + := block-contains-clause + := record-clause + := label-records-clause + := value-of-clause + := data-records-clause + := linage-clause + := recording-mode-clause + := code-set-clause) + +(defrule integer-to + := integer "TO") + +(defrule block-contains-clause + := "BLOCK" "CONTAINS"? integer-to? integer alt-characters-records?) + +(defrule alt-characters-records + := "CHARACTERS" + := "RECORDS" + := "RECORD") + +(defrule record-clause + := "RECORD" record-clause-tail) + +(defrule depending-on + := "DEPENDING" "ON"? data-name) + +(defrule record-clause-tail-1 + := "CONTAINS"? integer "CHARACTERS"?) + +(defrule record-clause-tail-2 + := "CONTAINS"? integer "TO" integer "CHARACTERS"?) + +(defrule record-clause-tail-3 + := record-varying-phrase depending-on?) + +(defrule record-clause-tail + := record-clause-tail-2 + := record-clause-tail-1 + := record-clause-tail-3) + +(defrule record-varying-phrase + := "IS"? "VARYING" "IN"? "SIZE"? from-integer? to-integer? "CHARACTERS"?) + +(defrule from-integer + := "FROM"? integer) + +(defrule to-integer + := "TO" integer) + +(defrule label-records-clause + := "LABEL" records-are label-records-clause-tail + :reduce (list :label-record label-records-clause-tail)) + +(defrule data-names + := data-name+) + +(defrule label-records-clause-tail + := "STANDARD" :reduce :standard + := "OMITTED" :reduce :omitted + := data-names) + +(defrule value-of-clause + := "VALUE" "OF" value-of-clause-tail+) + +(defrule alt-qualified-data-name-literal + := qualified-data-name + := literal) + +(defrule value-of-clause-tail + := variable-identifier "IS"? alt-qualified-data-name-literal) + +(defrule data-records-clause + := "DATA" records-are data-name+) + +(defrule records-are + := records are?) + +(defrule linage-clause + := "LINAGE" "IS"? alt-data-name-integer "LINES"? linage-footing-phrase) + +(defrule linage-footing-phrase + := footing? lines-top? lines-bottom?) + +(defrule alt-data-name-integer + := data-name + := integer) + +(defrule footing + := "WITH"? "FOOTING" "AT"? alt-data-name-integer) + +(defrule lines-top + := "LINES"? "AT"? "TOP" alt-data-name-integer) + +(defrule lines-bottom + := "LINES"? "AT"? "BOTTOM" alt-data-name-integer) + +(defrule recording-mode-clause + := "RECORDING" "MODE"? "IS"? variable-identifier) + +(defrule code-set-clause + := "CODE-SET" "IS"? alphabet-name) + +(defrule data-description-entry + := level-number alt-data-name-filler? data-description-entry-clause* "." + :reduce (append (list level-number alt-data-name-filler) + (flatten-list data-description-entry-clause))) + +(defrule alt-data-name-filler + := data-name + := "FILLER" + :reduce (list)) + +(defrule data-description-entry-clause + := picture-clause + := redefines-clause + := blank-when-zero-clause + := external-clause + := global-clause + := justified-clause + := occurs-clause + := sign-clause + := synchronized-clause + := usage-clause + := renames-clause + := value-clause) + +(defrule value-clause + := "VALUE" "IS"? literal + :reduce (list :value literal)) + +(defrule redefines-clause + := "REDEFINES" data-name + :reduce `(:redefines ,data-name)) + +(defrule blank-when-zero-clause + := "BLANK" "WHEN"? zeroes + :reduce '(:blank-when-zero t)) + +(defrule zeroes + := "ZERO" + := "ZEROS" + := "ZEROES") + +(defrule external-clause + := "IS"? "EXTERNAL" + :reduce '(:external t)) + +(defrule global-clause + := "IS"? "GLOBAL" + :reduce '(:global t)) + +(defrule justified-clause + := justified "RIGHT"? + :reduce `(:justified ,(if $2 :right :left))) + +(defrule justified + := "JUSTIFIED" + := "JUST") + +(defrule occurs-clause + := "OCCURS" integer "TIMES"? occurs-clause-key* indexed-by? + ;; to be completed -wcp16/7/03. + :reduce `(:times ,integer) + := "OCCURS" integer "TO" integer "TIMES"? "DEPENDING" "ON"? qualified-data-name occurs-clause-key* indexed-by? + ;; to be completed -wcp16/7/03. + :reduce `(:times (,integer ,integer2 ,qualified-data-name))) + +(defrule occurs-clause-key + := alt-ascending-descending "KEY"? "IS"? qualified-data-name+) + +(defrule indexed-by + := "INDEXED" "BY"? index-name+) + +(defrule picture-clause + := picture "IS"? picture-string + :reduce `(:picture ,picture-string)) + +(defrule picture + := "PICTURE" + := "PIC") + +(defrule sign-clause + := sign-is? alt-leading-trailing separate-character? + :reduce `(:separate-sign ,separate-character :sign-position ,alt-leading-trailing)) + +(defrule sign-is + := "SIGN" "IS"?) + +(defrule separate-character + := "SEPARATE" "CHARACTER"? + :reduce t) + +(defrule alt-leading-trailing + := "LEADING" + :reduce :leading + := "TRAILING" + :reduce :trailing) + +(defrule synchronized-clause + := synchronized alt-left-right? + :reduce `(:synchronized ,(if alt-left-right + alt-left-right + t))) + +(defrule alt-left-right + := "LEFT" + :reduce :left + := "RIGHT" + :reduce :right) + +(defrule synchronized + := "SYNCHRONIZED" + := "SYNC") + +(defrule usage-clause + := usage-is? usage + :reduce (list :encoding usage)) + +(defrule usage-is + := "USAGE" "IS"?) + +(defrule usage + := "BINARY" + :reduce :binary + := "COMP" + :reduce :comp + := "COMP-1" + :reduce :comp1 + := "COMP-2" + :reduce :comp2 + := "COMP-3" + :reduce :comp3 + := "COMP-4" + :reduce :comp4 + := "COMPUTATIONAL" + :reduce :comp + := "COMPUTATIONAL-1" + :reduce :comp1 + := "COMPUTATIONAL-2" + :reduce :comp2 + := "COMPUTATIONAL-3" + :reduce :comp3 + := "COMPUTATIONAL-4" + :reduce :comp4 + := "DISPLAY" + :reduce :display + := "DISPLAY-1" + :reduce :display1 + := "INDEX" + :reduce :index + := "PACKED-DECIMAL" + :reduce :packed-decimal + := "POINTER" + :reduce :pointer) + +(defrule renames-clause + := "RENAMES" qualified-data-name through-qualified-data-name? + :reduce `(:renames ,qualified-data-name ,through-qualified-data-name)) + +(defrule through-qualified-data-name + := through qualified-data-name + :reduce qualified-data-name) + +(defrule condition-value-clause + := values-are literal-through-literal+) + +(defrule through-literal + := through literal) + +(defrule literal-through-literal + := literal through-literal?) + +(defrule values-are + := values are?) + +(defrule procedure-division-head + := "PROCEDURE" "DIVISION" using-phrase? ".") + +(defrule procedure-division + := procedure-division-head sentence+) + +(defrule using-phrase + := "USING" data-name+) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +(defrule declaratives + := "DECLARATIVES" "." declaratives-content+ "END" "DECLARATIVES" ".") + +(defrule declaratives-content + := cobol-identifier "SECTION" "." use-statement "." sentence*) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +(defrule paragraph-header + := cobol-identifier "SECTION"? + :reduce (list (if $2 :section :label) $1)) + +(defrule sentence + := declaratives + := statement* "." + :reduce $1 + := paragraph-header "." + :reduce $1) + +(defrule statement + := move-statement + := if-statement + := perform-statement + := go-to-statement + := accept-statement + := add-statement + := alter-statement + := call-statement + := cancel-statement + := close-statement + := compute-statement + := continue-statement + := delete-statement + := display-statement + := divide-statement + := entry-statement + := evaluate-statement + := exit-program-statement + := exit-statement + := goback-statement + := initialize-statement + := inspect-statement + := merge-statement + := multiply-statement + := open-statement + := read-statement + := release-statement + := return-statement + := rewrite-statement + := search-statement + := set-statement + := sort-statement + := start-statement + := stop-statement + := string-statement + := subtract-statement + := unstring-statement + := write-statement + := paragraph-header) + +(defrule accept-statement + := "ACCEPT" variable-identifier "FROM" date + := "ACCEPT" variable-identifier "AT" screen-coordinates + :reduce (apply #'list 'accept-at variable-identifier screen-coordinates) + := "ACCEPT" variable-identifier from-environment-name?) + +(defrule from-environment-name + := "FROM" cobol-identifier) + + +(defrule date + := "DATE" + := "DAY" + := "DAY-OF-WEEK" + := "TIME") + +(defrule add-statement + := "ADD" id-or-lit+ to-id-or-lit? "GIVING" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-ADD"? + := "ADD" id-or-lit+ "TO" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-ADD"? + := "ADD" corresponding variable-identifier "TO" variable-identifier "ROUNDED"? on-size-error-statement-list? not-on-size-error-statement-list? "END-ADD"?) + +(defrule statement-list + := statement+) + +(defrule alter-statement + := "ALTER" procedure-to-procedure+) + +(defrule proceed-to + := "PROCEED" "TO") + +(defrule procedure-to-procedure + := procedure-name "TO" proceed-to? procedure-name) + +(defrule call-statement + := "CALL" id-or-lit using-parameters? call-rest-phrase "END-CALL"? + :reduce (list 'call id-or-lit (cons 'list using-parameters))) + +(defrule by-reference + := "BY"? "REFERENCE") + +(defrule content-parameter-value + := cobol-identifier + := literal) + +(defrule reference-parameter + := by-reference? variable-identifier) + +(defrule content-parameter + := "BY"? "CONTENT" content-parameter-value+) + +(defrule parameter + := reference-parameter + := content-parameter + := literal) + +(defrule using-parameters + := "USING" parameter+) + +(defrule call-rest-phrase + := on-exception-statement-list? not-on-exception-statement-list? on-overflow-statement-list?) + +(defrule on-exception-statement-list + := "ON"? "EXCEPTION" statement-list) + +(defrule not-on-exception-statement-list + := "NOT" "ON"? "EXCEPTION" statement-list) + +(defrule cancel-statement + := "CANCEL" id-or-lit+) + +(defrule close-statement + := "CLOSE" close-statement-file-name+ + :reduce (list 'close close-statement-file-name)) + +(defrule alt-removal-no-rewind + := for-removal + := with-no-rewind) + +(defrule alt-reel-unit + := "REEL" + := "UNIT") + +(defrule alt-no-rewind-lock + := no-rewind + := "LOCK") + +(defrule close-statement-options-1 + := alt-reel-unit alt-removal-no-rewind?) + +(defrule close-statement-options-2 + := "WITH"? alt-no-rewind-lock) + +(defrule close-statement-options + := close-statement-options-1 + := close-statement-options-2) + +(defrule close-statement-file-name + := file-name close-statement-options?) + +(defrule compute-statement + := "COMPUTE" cobword-rounded+ equal arithmetic-expression on-size-error-statement-list? not-on-size-error-statement-list? "END-COMPUTE"? + :reduce (list 'compute cobword-rounded arithmetic-expression :on-size-error on-size-error-statement-list + :not-on-size-error not-on-size-error-statement-list)) + +(defrule equal + := "=" + := "EQUAL") + +(defrule continue-statement + := "CONTINUE") + +(defrule delete-statement + := "DELETE" file-name "RECORD"? invalid-key-statement-list? not-invalid-key-statement-list? "END-DELETE"? + :reduce (list 'delete file-name :invalid invalid-key-statement-list :not-invalid not-invalid-key-statement-list)) + +(defrule display-statement + := "DISPLAY" id-or-lit+ upon-environment-name? with-no-advancing? + :reduce (list 'display (cons 'list id-or-lit) :upon upon-environment-name :advance (not with-no-advancing)) + := "DISPLAY" id-or-lit "AT" screen-coordinates + :reduce (apply #'list 'display-at id-or-lit screen-coordinates)) + +(defrule screen-coordinates + := integer + :reduce (multiple-value-list (truncate integer 100))) + +(defrule upon-environment-name + := "UPON" cobol-identifier) + +(defrule with-no-advancing + := "WITH"? "NO" "ADVANCING") + +(defrule divide-statement + := "DIVIDE" id-or-lit "INTO" id-or-lit "GIVING" variable-identifier "ROUNDED"? "REMAINDER" variable-identifier on-size-error-statement-list? not-on-size-error-statement-list? "END-DIVIDE"? + := "DIVIDE" id-or-lit "BY" id-or-lit "GIVING" variable-identifier "ROUNDED"? "REMAINDER" variable-identifier on-size-error-statement-list? not-on-size-error-statement-list? "END-DIVIDE"? + := "DIVIDE" id-or-lit "INTO" id-or-lit "GIVING" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-DIVIDE"? + := "DIVIDE" id-or-lit "BY" id-or-lit "GIVING" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-DIVIDE"? + := "DIVIDE" id-or-lit "INTO" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-DIVIDE"?) + +(defrule entry-statement + := "ENTRY" literal using-phrase?) + +(defrule evaluate-statement + := "EVALUATE" evaluate-condition also-phrase* when-phrases+ when-other-phrase? "END-EVALUATE"?) + +(defrule evaluate-condition + := condition + := "TRUE" + := "FALSE") + +(defrule also-phrase + := "ALSO" evaluate-condition) + +(defrule when-phrase-also-phrase + := "ALSO" evaluate-phrase) + +(defrule when-phrase + := "WHEN" evaluate-phrase when-phrase-also-phrase*) + +(defrule when-phrases + := when-phrase+ statement-list) + +(defrule when-other-phrase + := "WHEN" "OTHER" statement-list) + +(defrule evaluate-phrase + := "ANY" + := condition + := "TRUE" + := "FALSE" + := evaluate-phrase-1) + +(defrule evaluate-phrase-1 + := "NOT"? arithmetic-expression through-arithmetic-expression?) + +(defrule through-arithmetic-expression + := through arithmetic-expression) + +(defrule exit-statement + := "EXIT" + :reduce '(exit-paragraph)) + +(defrule exit-program-statement + := "EXIT" "PROGRAM" + :reduce '(exit-program)) + +(defrule goback-statement + := "GOBACK" + :reduce '(go-back)) + +(defrule go-to-statement + := "GO" "TO"? procedure-name+ "DEPENDING" "ON"? variable-identifier + :reduce (list 'goto-depending variable-identifier procedure-name) + := "GO" "TO"? procedure-name + :reduce (list 'goto procedure-name)) + +(defrule if-phrase + := "IF" condition "THEN"? alt-statement-list-next-sentence "ELSE" alt-statement-list-next-sentence + :reduce (list 'if condition + (if (cdr alt-statement-list-next-sentence) + (cons 'progn alt-statement-list-next-sentence) + (car alt-statement-list-next-sentence)) + (if (cdr alt-statement-list-next-sentence2) + (cons 'progn alt-statement-list-next-sentence2) + (car alt-statement-list-next-sentence2))) + := "IF" condition "THEN"? alt-statement-list-next-sentence + :reduce (append (list 'when condition) alt-statement-list-next-sentence)) + +(defrule if-statement + := if-phrase "END-IF"? + :reduce $1) + +(defrule initialize-statement + := "INITIALIZE" variable-identifier+ initialize-replacing-phrase?) + +(defrule initialize-replacing-type + := "ALPHABETIC" + := "ALPHANUMERIC" + := "NUMERIC" + := "ALPHANUMERIC-EDITED" + := "NUMERIC-EDITED" + := "DBCS" + := "EGCS") + +(defrule initialize-replacing-argument + := initialize-replacing-type "DATA"? "BY" id-or-lit) + +(defrule initialize-replacing-phrase + := "REPLACING" initialize-replacing-argument+) + +(defrule inspect-statement + := inspect-statement-1 + := inspect-statement-2 + := inspect-statement-3 + := inspect-statement-4) + +(defrule inspect-statement-1 + := "INSPECT" variable-identifier "TALLYING" tallying-argument+) + +(defrule inspect-statement-2 + := "INSPECT" variable-identifier "CONVERTING" id-or-lit "TO" id-or-lit before-after-phrase*) + +(defrule inspect-statement-3 + := "INSPECT" variable-identifier "TALLYING" tallying-argument+ "REPLACING" inspect-replacing-phrase+) + +(defrule tallying-for-id-or-lit + := id-or-lit before-after-phrase*) + +(defrule alt-all-leading + := "ALL" + := "LEADING") + +(defrule tallying-for-argument-1 + := "CHARACTERS" before-after-phrase*) + +(defrule tallying-for-argument-2 + := alt-all-leading tallying-for-id-or-lit+) + +(defrule tallying-for-argument + := tallying-for-argument-1 + := tallying-for-argument-2) + +(defrule tallying-argument + := variable-identifier "FOR" tallying-for-argument+) + +(defrule inspect-statement-4 + := "INSPECT" variable-identifier "REPLACING" inspect-replacing-phrase+) + +(defrule inspect-replacing-argument + := inspect-by-argument "BY" inspect-by-argument before-after-phrase*) + +(defrule alt-all-leading-first + := "ALL" + := "LEADING" + := "FIRST") + +(defrule inspect-replacing-phrase-1 + := "CHARACTERS" "BY" id-or-lit before-after-phrase*) + +(defrule inspect-replacing-phrase-2 + := alt-all-leading-first inspect-replacing-argument+) + +(defrule inspect-replacing-phrase + := inspect-replacing-phrase-1 + := inspect-replacing-phrase-2) + +(defrule before-after-phrase + := alt-before-after "INITIAL"? id-or-lit) + +(defrule merge-statement + := "MERGE" file-name on-key-phrase+ collating-sequence? "USING" file-name file-name+ merge-statement-tail) + +(defrule on-key-phrase + := "ON"? alt-ascending-descending "KEY"? qualified-data-name+) + +(defrule merge-statement-tail + := output-procedure + := giving-file-names) + +(defrule move-statement + := "MOVE" id-or-lit "TO" variable-identifier+ + :reduce (apply #'list 'move id-or-lit variable-identifier) + := "MOVE" corresponding variable-identifier "TO" variable-identifier+ + :reduce (apply #'list 'move-corresponding variable-identifier variable-identifier2)) + +(defrule multiply-statement + := "MULTIPLY" id-or-lit "BY" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-MULTIPLY"? + :reduce (list 'multiply id-or-lit cobword-rounded :on-size-error on-size-error-statement-list + :not-on-size-error not-on-size-error-statement-list) + := "MULTIPLY" id-or-lit "BY" id-or-lit "GIVING" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-MULTIPLY"? + :reduce (list 'multiply id-or-lit id-or-lit2 :giving cobword-rounded + :on-size-error on-size-error-statement-list + :not-on-size-error not-on-size-error-statement-list)) + +(defrule open-statement + := "OPEN" open-statement-phrase+ + :reduce (list 'open open-statement-phrase)) + +(defrule alt-reversed-with-no-rewind + := "REVERSED" + := with-no-rewind) + +(defrule open-statement-input-file-name + := file-name alt-reversed-with-no-rewind?) + +(defrule with-no-rewind + := "WITH"? "NO" "REWIND") + +(defrule open-statement-output-file-name + := file-name with-no-rewind?) + +(defrule open-statement-input + := "INPUT" open-statement-input-file-name+) + +(defrule open-statement-output + := "OUTPUT" open-statement-output-file-name+) + +(defrule open-statement-i-o + := "I-O" file-name+) + +(defrule open-statement-extend + := "EXTEND" file-name+) + +(defrule open-statement-phrase + := open-statement-input + := open-statement-output + := open-statement-i-o + := open-statement-extend) + +(defrule perform-statement + := "PERFORM" procedure-name through-procedure-name? perform-until-phrase + :reduce `(perform-until ,procedure-name ,through-procedure-name ,perform-until-phrase) + := "PERFORM" procedure-name through-procedure-name? perform-varying-phrase perform-after-phrase* + :reduce `(perform-varying ,perform-varying-phrase ,procedure-name ,through-procedure-name ,perform-after-phrase) + := "PERFORM" procedure-name through-procedure-name? cobword-int "TIMES" + :reduce `(perform-times ,cobword-int ,procedure-name ,through-procedure-name) + := "PERFORM" procedure-name through-procedure-name? + :reduce (append (list 'perform procedure-name) through-procedure-name)) + +(defrule perform-varying-phrase + := with-test? "VARYING" variable-identifier "FROM" id-or-lit "BY" id-or-lit "UNTIL" condition) + +(defrule perform-after-phrase + := "AFTER" variable-identifier "FROM" id-or-lit "BY" id-or-lit "UNTIL" condition) + +(defrule perform-until-phrase + := with-test? "UNTIL" condition) + +(defrule with-test + := "WITH"? "TEST" alt-before-after + :reduce alt-before-after) + +(defrule read-statement + := "READ" file-name "NEXT"? "RECORD"? into-identifier? key-is-qualified-data-name? invalid-key-statement-list? not-invalid-key-statement-list? at-end-statement-list? not-at-end-statement-list? "END-READ"?) + +(defrule key-is-qualified-data-name + := "KEY" "IS"? qualified-data-name) + +(defrule release-statement + := "RELEASE" record-name from-identifier?) + +(defrule return-statement + := "RETURN" file-name "RECORD"? into-identifier? "AT"? "END" statement-list not-at-end-statement-list? "END-RETURN"?) + +(defrule into-identifier + := "INTO" variable-identifier) + +(defrule not-at-end-statement-list + := "NOT" "AT"? "END" statement-list) + +(defrule rewrite-statement + := "REWRITE" record-name from-identifier? invalid-key-statement-list? not-invalid-key-statement-list? "END-REWRITE"?) + +(defrule search-statement + := search-statement-1 + := search-statement-2) + +(defrule search-statement-1 + := "SEARCH" cobol-identifier varying-identifier? at-end-statement-list? when-condition-stats+ "END-SEARCH"?) + +(defrule varying-identifier + := "VARYING" variable-identifier) + +(defrule when-condition-stats + := "WHEN" condition alt-statement-list-next-sentence) + +(defrule search-statement-2 + := "SEARCH" "ALL" variable-identifier at-end-statement-list? "WHEN" search-statement-condition search-statement-condition-tail* alt-statement-list-next-sentence "END-SEARCH"?) + +(defrule at-end-statement-list + := "AT"? "END" statement-list) + +(defrule search-statement-equal-expression + := variable-identifier "IS"? equal-to arithmetic-expression + :reduce (list '= variable-identifier arithmetic-expression)) + +(defrule search-statement-condition + := search-statement-equal-expression + := condition-name-reference) + +(defrule search-statement-condition-tail + := "AND" search-statement-condition) + +(defrule alt-statement-list-next-sentence + := statement+ + := next-sentence + :reduce :next-sentence) + +(defrule set-statement + := "SET" set-statement-phrase+) + +(defrule sort-statement + := "SORT" file-name on-key-is-phrase+ with-duplicates-in-order? collating-sequence? sort-statement-in sort-statement-out) + +(defrule key-is + := "KEY" "IS"?) + +(defrule alt-ascending-descending + := "ASCENDING" + := "DESCENDING") + +(defrule on-key-is-phrase + := "ON"? alt-ascending-descending key-is? qualified-data-name+) + +(defrule with-duplicates-in-order + := "WITH"? "DUPLICATES" "IN"? "ORDER"?) + +(defrule collating-sequence + := "COLLATING"? "SEQUENCE" "IS"? alphabet-name) + +(defrule through + := "THROUGH" + := "THRU") + +(defrule through-procedure-name + := through procedure-name + :reduce procedure-name) + +(defrule using-file-names + := "USING" file-name+) + +(defrule input-procedure + := "INPUT" "PROCEDURE" "IS"? procedure-name through-procedure-name?) + +(defrule giving-file-names + := "GIVING" file-name+) + +(defrule output-procedure + := "OUTPUT" "PROCEDURE" "IS"? procedure-name through-procedure-name?) + +(defrule sort-statement-in + := using-file-names + := input-procedure) + +(defrule sort-statement-out + := giving-file-names + := output-procedure) + +(defrule start-statement + := "START" file-name key-is-rel-op-qualified-data-name? invalid-key-statement-list? not-invalid-key-statement-list? "END-START"?) + +(defrule rel-op + := equal-to + :reduce '= + := greater-than + :reduce '> + := greater-equal + :reduce '>=) + +(defrule key-is-rel-op-qualified-data-name + := "KEY" "IS"? rel-op qualified-data-name + :reduce (list rel-op qualified-data-name)) + +(defrule stop-statement + := "STOP" alt-run-literal + :reduce '(stop)) + +(defrule alt-run-literal + := "RUN" + := literal) + +(defrule string-statement + := "STRING" delimited-by-phrase+ "INTO" variable-identifier with-pointer-identifier? on-overflow-statement-list? not-on-overflow-statement-list? "END-STRING"? + :reduce (list 'string-concat delimited-by-phrase variable-identifier :with-pointer with-pointer-identifier :on-overflow on-overflow-statement-list :not-on-overflow not-on-overflow-statement-list)) + +(defrule id-or-lit-size + := literal + := variable-identifier + := "SIZE") + +(defrule delimited-by-phrase + := id-or-lit+ "DELIMITED" "BY"? id-or-lit-size + :reduce (list id-or-lit id-or-lit-size)) + +(defrule subtract-statement + := "SUBTRACT" id-or-lit+ "FROM" id-or-lit "GIVING" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-SUBTRACT"? + :reduce (list 'subtract-giving id-or-lit id-or-lit2 cobword-rounded + :on-size-error on-size-error-statement-list + :not-on-size-error not-on-size-error-statement-list) + := "SUBTRACT" id-or-lit+ "FROM" cobword-rounded+ on-size-error-statement-list? not-on-size-error-statement-list? "END-SUBTRACT"? + :reduce (list 'subtract id-or-lit cobword-rounded + :on-size-error on-size-error-statement-list + :not-on-size-error not-on-size-error-statement-list) + := "SUBTRACT" corresponding variable-identifier "FROM" variable-identifier "ROUNDED"? on-size-error-statement-list? not-on-size-error-statement-list? "END-SUBTRACT"? + :reduce (list 'subtract-corr variable-identifier variable-identifier + :rounded (and $5 t) + :on-size-error on-size-error-statement-list + :not-on-size-error not-on-size-error-statement-list)) + +(defrule cobword-rounded + := variable-identifier "ROUNDED"? + :reduce (list variable-identifier (and $2 t))) + +(defrule on-size-error-statement-list + := "ON"? "SIZE" "ERROR" statement-list + :reduce statement-list) + +(defrule not-on-size-error-statement-list + := "NOT" "ON"? "SIZE" "ERROR" statement-list + :reduce statement-list) + +(defrule corresponding + := "CORRESPONDING" + := "CORR") + +(defrule unstring-statement + := "UNSTRING" variable-identifier delimited-by-all-phrase? "INTO" unstring-statement-dst+ with-pointer-identifier? tallying-in-identifier? on-overflow-statement-list? not-on-overflow-statement-list? "END-UNSTRING"? + :reduce (list 'unstring variable-identifier unstring-statement-dst + :delimited-by-all delimited-by-all-phrase + :with-pointer with-pointer-identifier + :tallying tallying-in-identifier + :on-overflow on-overflow-statement-list + :not-on-overflow not-on-overflow-statement-list)) + +(defrule id-or-lit + := literal + := variable-identifier) + +(defrule or-all-id-or-lit + := "OR" "ALL"? id-or-lit) + +(defrule delimited-by-all-phrase + := "DELIMITED" "BY"? "ALL"? id-or-lit or-all-id-or-lit*) + +(defrule delimiter-in-identifier + := "DELIMITER" "IN"? variable-identifier) + +(defrule count-in-identifier + := "COUNT" "IN"? variable-identifier) + +(defrule unstring-statement-dst + := variable-identifier delimiter-in-identifier? count-in-identifier?) + +(defrule with-pointer-identifier + := "WITH"? "POINTER" variable-identifier) + +(defrule tallying-in-identifier + := "TALLYING" "IN"? variable-identifier) + +(defrule on-overflow-statement-list + := "ON"? "OVERFLOW" statement-list) + +(defrule not-on-overflow-statement-list + := "NOT" "ON"? "OVERFLOW" statement-list) + +(defrule write-statement + := "WRITE" record-name from-identifier? advancing-phrase? write-exceptions "END-WRITE"?) + +(defrule lines + := "LINE" + := "LINES") + +(defrule cobword-int + := cobol-identifier + := integer) + +(defrule nr-lines-phrase + := cobword-int lines?) + +(defrule page-phrase + := nr-lines-phrase + := "PAGE") + +(defrule alt-before-after + := "BEFORE" + := "AFTER") + +(defrule advancing-phrase + := alt-before-after "ADVANCING"? page-phrase) + +(defrule from-identifier + := "FROM" variable-identifier) + +(defrule invalid-key-statement-list + := "INVALID" "KEY"? statement-list + :reduce statement-list) + +(defrule not-invalid-key-statement-list + := "NOT" "INVALID" "KEY"? statement-list + :reduce statement-list) + +(defrule end-of-page + := "END-OF-PAGE" + := "EOP") + +(defrule at-end-of-page-statement-list + := "AT"? end-of-page statement-list + :reduce statement-list) + +(defrule not-at-end-of-page-statement-list + := "NOT" "AT"? end-of-page statement-list + :reduce statement-list) + +;; This is left in the grammar but is not used. COPYs are handled by +;; the lexical scanner. +(defrule copy-statement + := "COPY" alt-text-name-literal in-library? "SUPPRESS"? copy-statement-replacing-phrase?) + +(defrule in + := "OF" + := "IN") + +(defrule alt-library-name-literal + := library-name + := literal) + +(defrule in-library + := in alt-library-name-literal) + +(defrule copy-statement-by-phrase + := copy-operand "BY" copy-operand) + +(defrule copy-statement-replacing-phrase + := "REPLACING" copy-statement-by-phrase+) + +(defrule alt-text-name-literal + := text-name + := literal) + +(defrule copy-operand + := cobol-identifier + := literal) + +(defrule use-statement + := use-statement-1 + := use-statement-2 + := use-statement-3) + +(defrule use-statement-1 + := "USE" "GLOBAL"? "AFTER" "STANDARD"? alt-exception-error "PROCEDURE" "ON"? alt-file-names-i-o) + +(defrule alt-exception-error + := "EXCEPTION" + := "ERROR") + +(defrule use-statement-2 + := "USE" "GLOBAL"? "AFTER" "STANDARD"? alt-beginning-ending? alt-file-reel-unit? "LABEL" "PROCEDURE" "ON"? alt-file-names-i-o) + +(defrule alt-beginning-ending + := "BEGINNING" + := "ENDING") + +(defrule alt-file-reel-unit + := "FILE" + := "REEL" + := "UNIT") + +(defrule file-names + := file-name+) + +(defrule alt-file-names-i-o + := file-names + := "INPUT" + := "OUTPUT" + := "I-O" + := "EXTEND") + +(defrule use-statement-3 + := "USE" "FOR"? "DEBUGGING" "ON"? alt-procedures-all-procedures) + +(defrule procedure-names + := procedure-name+) + +(defrule alt-procedures-all-procedures + := procedure-names + := all-procedures) + +(defrule condition + := combinable-condition + := combinable-condition "AND" condition + :reduce `(and ,combinable-condition ,condition) + := combinable-condition "OR" condition + :reduce `(or ,combinable-condition ,condition) + := combinable-condition "AND" id-or-lit + :reduce `(and ,combinable-condition (,(car combinable-condition) ,(cadr combinable-condition) ,id-or-lit)) + := combinable-condition "OR" id-or-lit + :reduce `(or ,combinable-condition (,(car combinable-condition) ,(cadr combinable-condition) ,id-or-lit))) + +(defrule combinable-condition + := "NOT"? simple-condition + :reduce (if $1 + (list 'not simple-condition) + simple-condition)) + +(defrule simple-condition + := class-condition + := relation-condition + := sign-condition + := "(" condition ")" + ;; not sure if it's necessary -wcp15/7/03. + ;; := arithmetic-expression + ) + +(defrule class-condition + := variable-identifier "IS"? "NOT"? class-type + :reduce (if $3 + (list 'not (list 'type-of variable-identifier (make-keyword class-type))) + (list 'type-of variable-identifier (make-keyword class-type)))) + +(defrule class-type + := "NUMERIC" + := "ALPHABETIC" + := "ALPHABETIC-LOWER" + := "ALPHABETIC-UPPER" + := "DBCS") + +(defun unfold-subrelations (main-relation subs) + (destructuring-bind (main-operator main-variable other-variable) main-relation + (declare (ignore other-variable)) + (labels ((unfold (subs) + (if (null subs) + main-relation + (destructuring-bind (connection operator variable) (car subs) + (list connection + (list (or operator main-operator) main-variable variable) + (unfold (cdr subs))))))) + (unfold subs)))) + +(defrule relation-condition + ;; This is too complex + ;; := arithmetic-expression relational-operator simple-condition + := id-or-lit relational-operator id-or-lit subordinate-relation* + :reduce (unfold-subrelations (list relational-operator id-or-lit id-or-lit2) subordinate-relation)) + +(defrule or-and + := "OR" :reduce 'or + := "AND" :reduce 'and) + +(defrule subordinate-relation + := or-and relational-operator? id-or-lit + :reduce (list or-and relational-operator id-or-lit)) + +(defrule relational-operator + := "IS"? relational-operator-type + :reduce relational-operator-type) + +(defrule less-than + := "LESS" "THAN"? + := "<") + +(defrule greater-equal + := "GREATER" "THAN"? "OR" "EQUAL" "TO"? + := ">=" + := ">" "=" + := "NOT" "<" + := "NOT" "LESS" "THAN"?) + +(defrule less-equal + := "LESS" "THAN"? "OR" "EQUAL" "TO"? + := "<=" + := "<" "=" + := "NOT" ">" + := "NOT" "GREATER" "THAN"?) + +(defrule greater-than + := "GREATER" "THAN"? + := ">") + +(defrule equal-to + := "EQUAL" "TO"? + := "=") + +(defrule relational-operator-type + := greater-equal + :reduce 'cob>= + := less-equal + :reduce 'cob<= + := greater-than + :reduce 'cob> + := less-than + :reduce 'cob< + := equal-to + :reduce 'cob= + := "NOT" equal-to + :reduce 'cob-not=) + +(defrule sign-condition + := arithmetic-expression "IS"? "NOT"? sign-type + :reduce (if $3 + `(not (,sign-type ,arithmetic-expression)) + `(,sign-type ,arithmetic-expression))) + +(defrule sign-type + := "POSITIVE" :reduce '> + := "NEGATIVE" :reduce '< + := "ZERO" :reduce '= + := "ZEROES" :reduce '= + := "ZEROS" :reduce '=) + +(defrule procedure-name + := paragraph-or-section-name in-section-name + :reduce (list paragraph-or-section-name in-section-name) + := paragraph-or-section-name + :reduce paragraph-or-section-name) + +(defrule in-section-name + := in cobol-identifier + :reduce cobol-identifier) + +(defrule variable-identifier + := qualified-data-name subscript-parentheses* ;; reference-modification? + :reduce (if subscript-parentheses + (list :aref qualified-data-name subscript-parentheses) + qualified-data-name)) + +(defrule reference-modification + := "(" leftmost-character-position ":" length? ")" + :reduce (if length + (list :range leftmost-character-position length) + leftmost-character-position)) + +(defrule condition-name-reference + := condition-name in-data-or-file-or-mnemonic-name* subscript-parentheses*) + +(defrule in-data-or-file-or-mnemonic-name + := in data-or-file-or-mnemonic-name) + +(defrule subscript-parentheses + := "(" subscript ")") + +(defrule subscript + := subscript-expression+) + +(defrule plus-minus-integer + := plus-or-minus integer) + +(defrule subscript-expression-ambiguous + := qualified-data-name plus-minus-integer?) + +(defrule subscript-expression + := literal + := subscript-expression-ambiguous) + +(defrule qualified-data-name + := data-name in-data-or-file-name* + :reduce (if in-data-or-file-name + (list data-name in-data-or-file-name) ; incomplete -wcp15/7/03. + data-name) + := "ADDRESS" "OF" data-name + :reduce (list 'address-of data-name) + := "LENGTH" "OF" cobol-identifier + :reduce (list 'length-of cobol-identifier)) + +(defrule in-data-or-file-name + := in data-or-file-name) + +(defrule leftmost-character-position + := arithmetic-expression) + +(defrule length + := arithmetic-expression) + +(defrule arithmetic-expression + := times-div + := times-div "+" arithmetic-expression + :reduce `(+ ,times-div ,arithmetic-expression) + := times-div "-" arithmetic-expression + :reduce `(- ,times-div ,arithmetic-expression)) + +(defrule times-div + := power + := power "*" times-div + :reduce `(* ,power ,times-div) + := power "/" times-div + :reduce `(/ ,power ,times-div)) + +(defrule power + := plus-or-minus? basis + := plus-or-minus? basis "**" power + :reduce (if plus-or-minus + `(plus-or-minus (expt basis basis2)) + `(expt basis basis2))) + +(defrule plus-or-minus + := "+" + :reduce '+ + := "-" + :reduce '-) + +;; (defrule power-tail +;; := "**" basis) + +(defrule basis + := literal + := variable-identifier + := "(" arithmetic-expression ")") + +(defrule alphabet-name + := cobol-identifier) + +(defrule condition-name + := cobol-identifier) + +(defrule data-name + := cobol-identifier) + +(defrule cobol-identifier + := identifier + :reduce (intern (string-upcase identifier))) + +(defrule file-name + := cobol-identifier) + +(defrule data-or-file-name + := cobol-identifier) + +(defrule index-name + := cobol-identifier) + +(defrule mnemonic-name + := cobol-identifier) + +(defrule data-or-file-or-mnemonic-name + := cobol-identifier) + +(defrule record-name + := qualified-data-name) + +(defrule symbolic-character + := cobol-identifier) + +(defrule library-name + := cobol-identifier) + +(defrule program-name + := cobol-identifier + := string) + +(defrule text-name + := cobol-identifier) + +(defrule paragraph-or-section-name + := cobol-identifier + := integer) + +(defrule computer-name + := identifier) + +(defrule environment-name + := cobol-identifier) + +(defrule assignment-name + := cobol-identifier) + +(defrule figurative-constant + := figurative-constant-simple + := figurative-constant-all) + +(defrule figurative-constant-all + := "ALL" literal) + +(defrule literal + := string + := float + := integer + := figurative-constant) + +) ; defun populate-grammar diff --git a/third_party/lisp/npg/npg.asd b/third_party/lisp/npg/npg.asd new file mode 100644 index 000000000000..addb7c6932af --- /dev/null +++ b/third_party/lisp/npg/npg.asd @@ -0,0 +1,55 @@ +;;; npg.asd --- declaration of this system + +;;; Copyright (C) 2003, 2006 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator + +#+cmu (ext:file-comment "$Module: npg.asd, Time-stamp: <2006-01-03 17:20:21 wcp> $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(defpackage :npg-system + (:use :common-lisp :asdf)) + +(in-package :npg-system) + +(defclass sample-file (doc-file) ()) +(defmethod source-file-type ((c sample-file) (s module)) + "lisp") + +(defsystem npg + :name "NPG" + :author "Walter C. Pelissero <walter@pelissero.de>" + :maintainer "Walter C. Pelissero <walter@pelissero.de>" + :licence "Lesser General Public License" + :description "NPG a Naive Parser Generator" + :long-description + "NPG is a backtracking recursive descent parser generator for +Common Lisp. It accepts rules in a Lispy EBNF syntax without indirect +left recursive rules." + :components + ((:doc-file "README") + (:doc-file "COPYING") + (:doc-file ".project") + (:module :examples + :components + ((:sample-file "python") + (:sample-file "vs-cobol-ii"))) + (:module :src + :components + ((:file "package") + (:file "common" :depends-on ("package")) + (:file "define" :depends-on ("package" "common")) + (:file "parser" :depends-on ("package" "common")))))) diff --git a/third_party/lisp/npg/src/common.lisp b/third_party/lisp/npg/src/common.lisp new file mode 100644 index 000000000000..8b64f5cc0a7b --- /dev/null +++ b/third_party/lisp/npg/src/common.lisp @@ -0,0 +1,79 @@ +;;; common.lisp --- common stuff + +;;; Copyright (C) 2003-2006, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator + +#+cmu (ext:file-comment "$Module: common.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :naive-parser-generator) + +(eval-when (:compile-toplevel :load-toplevel) + (defstruct grammar + rules + keywords + equal-p) + + (defstruct rule + name + productions) + + (defstruct (production (:conc-name prod-)) + tokens + (tokens-length 0 :type fixnum) + action) + + (defstruct token + type ; type of token (identifier, number, ...) + value ; its actual value + position) ; line/column in the input stream + ) ; eval-when + +(defmethod print-object ((obj rule) stream) + (format stream "#R(~A)" (rule-name obj))) + +(defmethod print-object ((obj production) stream) + (format stream "#P(action: ~S)" (prod-action obj))) + +(defmethod print-object ((obj token) stream) + (format stream "#T:~A=~S" (token-type obj) (token-value obj))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(declaim (inline make-rules-table find-rule add-rule)) + +(defun make-rules-table () + (make-hash-table)) + +(defun find-rule (rule-name rules) + (gethash rule-name rules)) + +(defun add-rule (rule-name rule rules) + (setf (gethash rule-name rules) rule)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(declaim (inline make-keywords-table find-keyword add-keyword)) + +(defun make-keywords-table () + (make-hash-table :test 'equal)) + +(defun find-keyword (keyword-name keywords) + (gethash keyword-name keywords)) + +(defun add-keyword (keyword keywords) + (setf (gethash keyword keywords) t)) diff --git a/third_party/lisp/npg/src/define.lisp b/third_party/lisp/npg/src/define.lisp new file mode 100644 index 000000000000..f52f0381a2de --- /dev/null +++ b/third_party/lisp/npg/src/define.lisp @@ -0,0 +1,408 @@ +;;; define.lisp --- grammar rules definition + +;;; Copyright (C) 2003-2006, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator + +#+cmu (ext:file-comment "$Module: define.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :naive-parser-generator) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defvar *smart-default-reduction* t + "If true the default reductions take only the non-static tokens - +those that are not declared as strings in the grammar.") + +;; These two are filled with DEFRULE. +(defvar *rules* (make-rules-table)) +(defvar *keywords* (make-keywords-table)) + +(defun make-action-arguments (tokens) + "Given a list of tokens making up a production, return three values: +the list of variables for the function reducing this production, those +that are non static and their unambiguous user-friendly names." + (flet ((unique (sym list) + (if (not (assoc sym list)) + sym + (loop + for i of-type fixnum from 2 + for x = (intern (format nil "~:@(~A~)~A" sym i)) + while (assoc x list) + finally (return x))))) + (loop + for tok in tokens + for i of-type fixnum from 1 + for arg = (intern (format nil "$~A" i) (find-package #.*package*)) + collect arg into args + unless (const-terminal-p tok) + collect arg into vars + and when (symbolp tok) + collect (list (unique tok named-vars) arg) into named-vars + when (and (listp tok) + (symbolp (cadr tok))) + collect (list (unique (cadr tok) named-vars) arg) into named-vars + finally + (return (values args vars named-vars))))) + +(defun make-action-function (name tokens action) + "Create a function with name NAME, arguments derived from TOKENS and +body ACTION. Return it's definition." + (let ((function + (multiple-value-bind (args vars named-vars) + (make-action-arguments tokens) + `(lambda ,args + (declare (ignorable ,@args)) + (let (($vars (list ,@vars)) + ($all (list ,@args)) + ,@named-vars + ($alist (list ,@(mapcar #'(lambda (v) + `(cons ',(intern (symbol-name (car v))) + ,(cadr v))) + named-vars)))) + (declare (ignorable $vars $all $alist ,@(mapcar #'car named-vars))) + (flet ((make-object (&optional type args) + (apply #'make-instance (or type ',name) + (append args $alist)))) + ,action)))))) + (when *compile-print* + (if *compile-verbose* + (format t "; Compiling ~S:~% ~S~%" name function) + (format t "; Compiling ~S~%" name))) + (compile name function))) + +(defun define-rule (name productions) + "Accept a rule in EBNF-like syntax, translate it into a sexp and a +call to INSERT-RULE-IN-CURRENT-GRAMMAR." + (flet ((transform (productions) + (loop + for tok in productions + with prod = nil + with action = nil + with phase = nil + with new-prods = nil + while tok + do (cond ((eq tok :=) + (push (list (nreverse prod) action) new-prods) + (setf prod nil + action nil + phase :prod)) + ((eq tok :reduce) + (setf phase :action)) + ((eq tok :tag) + (setf phase :tag)) + ((eq phase :tag) + (setf action `(cons ,tok $vars))) + ((eq phase :action) + (setf action tok)) + ((eq phase :prod) + (push tok prod))) + finally + (return (cdr (nreverse (cons (list (nreverse prod) action) new-prods))))))) + (insert-rule-in-current-grammar name (transform productions)))) + +(defmacro defrule (name &rest productions) + "Wrapper macro for DEFINE-RULE." + `(define-rule ',name ',productions)) + +(defun make-optional-rule (token) + "Make a rule for a possibly missing (non)terminal (? syntax) and +return it." + (insert-rule-in-current-grammar + (gensym (concatenate 'string "OPT-" + (if (rule-p token) + (symbol-name (rule-name token)) + (string-upcase token)))) + `(((,token)) (())))) + +(defun make-alternative-rule (tokens) + "Make a rule for a list of alternatives (\"or\" syntax) and return it." + (insert-rule-in-current-grammar + (gensym "ALT") + (mapcar #'(lambda (alternative) + `((,alternative))) + tokens))) + +(defun make-nonempty-list-rule (token &optional separator) + "Make a rule for a non-empty list (+ syntax) and return it." + (let ((rule-name (gensym (concatenate 'string "NELST-" + (if (rule-p token) + (symbol-name (rule-name token)) + (string-upcase token)))))) + (insert-rule-in-current-grammar + rule-name + (if separator + `(((,token ,separator ,rule-name) + (cons $1 $3)) + ((,token) ,#'list)) + `(((,token ,rule-name) + (cons $1 $2)) + ((,token) ,#'list)))))) + +(defun make-list-rule (token &optional separator) + "Make a rule for a possibly empty list (* syntax) return it." + (make-optional-rule (make-nonempty-list-rule token separator))) + +(defun const-terminal-p (object) + (or (stringp object) + (keywordp object))) + +(defun expand-production-token (tok) + "Translate token of the type NAME? or NAME* or NAME+ into (? NAME) +or (* NAME) or (+ NAME). This is used by the DEFRULE macro." + (if (symbolp tok) + (let* ((name (symbol-name tok)) + (last (char name (1- (length name)))) + ;; this looks silly but we need to make sure that we + ;; return symbols interned in this package, no one else + (op (cadr (assoc last '((#\? ?) (#\+ +) (#\* *)))))) + (if (and (> (length name) 1) op) + (list op + (intern (subseq name 0 (1- (length name))))) + tok)) + tok)) + +(defun EBNF-to-SEBNF (tokens) + "Take a production as a list of TOKENS and expand it. This turns a +EBNF syntax into a sexp-based EBNF syntax or SEBNF." + (loop + for tok in tokens + for token = (expand-production-token tok) + with new-tokens = '() + do (cond ((member token '(* + ?)) + (setf (car new-tokens) + (list token (car new-tokens)))) + (t + (push token new-tokens))) + finally (return (nreverse new-tokens)))) + +(defun SEBNF-to-BNF (tokens) + "Take a production in SEBNF (Symbolic Extended BNF) syntax and turn +it into BNF. The production is simplified but the current grammar is +populated with additional rules." + (flet ((make-complex-token-rule (tok) + (ecase (car tok) + (* (apply #'make-list-rule (cdr tok))) + (+ (apply #'make-nonempty-list-rule (cdr tok))) + (? (make-optional-rule (cadr tok))) + (or (make-alternative-rule (cdr tok)))))) + (loop + for token in tokens + with new-tokens = '() + with keywords = '() + do (cond ((listp token) + (push (make-complex-token-rule token) new-tokens)) + (t + (push token new-tokens) + (when (const-terminal-p token) + (push token keywords)))) + finally (return (values (nreverse new-tokens) keywords))))) + +(defun make-default-action-function (name tokens) + "Create a sexp to be used as default action in case one is not +supplied in the production. This is usually a quite sensible +one. That is, only the non-constant tokens are returned in a +list and in case only a variable token is available that one is +returned (not included in a list). If all the tokens are +constant, then all of them are returned in a list." + (cond ((null tokens) + ;; if the production matched the empty list (no tokens) we + ;; return always nil, that is the function LIST applied to no + ;; arguments + #'list) + ((null (cdr tokens)) + ;; if the production matches just one token we simply return + ;; that + #'identity) + (*smart-default-reduction* + ;; If we are required to be "smart" then create a function + ;; that simply returns the non static tokens of the + ;; production. If the production doesn't have nonterminal, + ;; then return all the tokens. If the production has only + ;; one argument then return that one only. + (make-action-function name tokens '(cond + ((null $vars) $all) + ((null (cdr $vars)) (car $vars)) + (t $vars)))) + (t + ;; in all the other cases we return all the token matching + ;; the production + #'list))) + +(defun make-production-from-descr (name production-description) + "Take a production NAME and its description in the form of a sexp +and return a production structure object together with a list of used +keywords." + (destructuring-bind (tokens &optional action) production-description + (let ((expanded-tokens (EBNF-to-SEBNF tokens))) + (multiple-value-bind (production-tokens keywords) + (sebnf-to-bnf expanded-tokens) + (let ((funct + (cond ((not action) + (make-default-action-function name expanded-tokens)) + ((or (listp action) + ;; the case when the action is simply to + ;; return a token (ie $2) or a constant value + (symbolp action)) + (make-action-function name expanded-tokens action)) + ((functionp action) + action) + (t ; action is a constant + #'(lambda (&rest args) + (declare (ignore args)) + action))))) + (values + ;; Make a promise instead of actually resolving the + ;; nonterminals. This avoids endless recursion. + (make-production :tokens production-tokens + :tokens-length (length production-tokens) + :action funct) + keywords)))))) + +(defun remove-immediate-left-recursivity (rule) + "Turn left recursive rules of the type + A -> A x | y +into + A -> y A2 + A2 -> x A2 | E +where E is the empty production." + (let ((name (rule-name rule)) + (productions (rule-productions rule))) + (loop + for prod in productions + for tokens = (prod-tokens prod) + ;; when immediately left recursive + when (eq (car tokens) rule) + collect prod into left-recursive + else + collect prod into non-left-recursive + finally + ;; found any left recursive production? + (when left-recursive + (warn "rule ~S is left recursive" name) + (let ((new-rule (make-rule :name (gensym "REWRITE")))) + ;; A -> y A2 + (setf (rule-productions rule) + (mapcar #'(lambda (p) + (let ((tokens (prod-tokens p)) + (action (prod-action p))) + (make-production :tokens (append tokens (list new-rule)) + :tokens-length (1+ (prod-tokens-length p)) + :action #'(lambda (&rest args) + (let ((f-A2 (car (last args))) + (head (butlast args))) + (funcall f-A2 (apply action head))))))) + non-left-recursive)) + ;; A2 -> x A2 | E + (setf (rule-productions new-rule) + (append + (mapcar #'(lambda (p) + (let ((tokens (prod-tokens p)) + (action (prod-action p))) + (make-production :tokens (append (cdr tokens) (list new-rule)) + :tokens-length (prod-tokens-length p) + :action #'(lambda (&rest args) + (let ((f-A2 (car (last args))) + (head (butlast args))) + #'(lambda (x) + (funcall f-A2 (apply action x head)))))))) + left-recursive) + (list + (make-production :tokens nil + :tokens-length 0 + :action #'(lambda () #'(lambda (arg) arg))))))))))) + +(defun remove-left-recursivity-from-rules (rules) + (loop + for rule being each hash-value in rules + do + ;; More to be done here. For now only the trivial immediate left + ;; recursivity is removed -wcp18/11/03. + (remove-immediate-left-recursivity rule))) + +(defun resolve-all-nonterminals (rules) + (loop + for rule being each hash-value in rules + do (loop + for production in (rule-productions rule) + do (setf (prod-tokens production) + (resolve-nonterminals (prod-tokens production) rules))))) + +(defun make-rule-productions (rule-name production-descriptions) + "Return a production object that belongs to RULE-NAME made according +to PRODUCTION-DESCRIPTIONS. See also MAKE-PRODUCTION-FROM-DESCR." + (loop + for descr in production-descriptions + for i of-type fixnum from 1 by 1 + for prod-name = (intern (format nil "~:@(~A~)-PROD~A" rule-name i)) + with productions = '() + with keywords = '() + do (progn + (multiple-value-bind (production keyws) + (make-production-from-descr prod-name descr) + (push production productions) + (setf keywords (append keyws keywords)))) + finally (return + (values (nreverse productions) keywords)))) + +(defun create-rule (name production-descriptions) + "Return a new rule object together with a list of keywords making up +the production definitions." + (multiple-value-bind (productions keywords) + (make-rule-productions name production-descriptions) + (values (make-rule :name name :productions productions) + keywords))) + +(defun insert-rule-in-current-grammar (name productions) + "Add rule to the current grammar and its keywords to the keywords +hash table. You don't want to use this directly. See DEFRULE macro +instead." + (when (find-rule name *rules*) + (error "redefining rule ~A" name)) + (multiple-value-bind (rule keywords) + (create-rule name productions) + (add-rule name rule *rules*) + (dolist (term keywords) + (add-keyword term *keywords*)) + rule)) + +(defun resolve-nonterminals (tokens rules) + "Given a list of production tokens, try to expand the nonterminal +ones with their respective rule from the the RULES pool." + (flet ((resolve-symbol (sym) + (or (find-rule sym rules) + sym))) + (mapcar #'(lambda (tok) + (if (symbolp tok) + (resolve-symbol tok) + tok)) + tokens))) + +(defun reset-grammar () + "Empty the current grammar from any existing rule." + (setf *rules* (make-rules-table) + *keywords* (make-keywords-table))) + +(defun generate-grammar (&optional (equal-p #'string-equal)) + "Return a GRAMMAR structure suitable for the PARSE function, using +the current rules. EQUAL-P, if present, is a function to be used to +match the input tokens; it defaults to STRING-EQUAL." + (resolve-all-nonterminals *rules*) + (remove-left-recursivity-from-rules *rules*) + (make-grammar :rules *rules* + :keywords *keywords* + :equal-p equal-p)) diff --git a/third_party/lisp/npg/src/package.lisp b/third_party/lisp/npg/src/package.lisp new file mode 100644 index 000000000000..b405f7b5f19e --- /dev/null +++ b/third_party/lisp/npg/src/package.lisp @@ -0,0 +1,50 @@ +;;; package.lisp --- backtracking parser package definition + +;;; Copyright (C) 2003-2006, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator + +#+cmu (ext:file-comment "$Module: package.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :cl-user) + +(defpackage :naive-parser-generator + (:nicknames :npg) + (:use :common-lisp) + (:export + #:parse ; The Parser + #:reset-grammar + #:generate-grammar + #:print-grammar-figures + #:grammar-keyword-p + #:keyword + #:grammar + #:make-token + #:token-value + #:token-type + #:token-position + #:later-position + #:defrule ; to define grammars + #:deftoken ; to define a lexer + #:input-cursor-mixin + #:copy-input-cursor-slots + #:dup-input-cursor + #:read-next-tokens + #:end-of-input + #:? #:+ #:* #:or + #:$vars #:$all #:$alist + #:$1 #:$2 #:$3 #:$4 #:$5 #:$6 #:$7 #:$8 #:$9 #:$10)) diff --git a/third_party/lisp/npg/src/parser.lisp b/third_party/lisp/npg/src/parser.lisp new file mode 100644 index 000000000000..328be1dcf30f --- /dev/null +++ b/third_party/lisp/npg/src/parser.lisp @@ -0,0 +1,234 @@ +;;; parser.lisp --- runtime parser + +;;; Copyright (C) 2003-2006, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: NPG a Naive Parser Generator + +#+cmu (ext:file-comment "$Module: parser.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +;;; Commentary: +;;; +;;; This is the runtime part of the parser. The code that is +;;; responsible to execute the parser defined with the primitives +;;; found in define.lisp. + +(in-package :naive-parser-generator) + +(defvar *debug* nil + "Either nil or a stream where to write the debug informations.") +#+debug (declaim (fixnum *maximum-recursion-depth*)) +#+debug (defvar *maximum-recursion-depth* 1000 + "Maximum depth the parser is allowed to recursively call itself. +This is the only way for the parser to detect a loop in the grammar. +Tune this if your grammar is unusually complex.") + +(declaim (inline reduce-production)) +(defun reduce-production (production arguments) + "Apply PRODUCTION's action on ARGUMENTS. This has the effect of + \"reducing\" the production." + (when *debug* + (format *debug* "reducing ~S on ~S~%" production arguments)) + (flet ((safe-token-value (token) + (if (token-p token) + (token-value token) + token))) + (apply (prod-action production) (mapcar #'safe-token-value arguments)))) + +(defgeneric later-position (pos1 pos2) + (:documentation + "Compare two file postions and return true if POS1 is later than +POS2 in the input stream.")) + +;; This is meant to be overloaded in the lexer +(defmethod later-position ((pos1 integer) (pos2 integer)) + (> pos1 pos2)) + +;; this looks silly but turns out to be useful (see below) +(defmethod later-position (pos1 pos2) + (and (eq pos1 :eof) (not (eq pos2 :eof)))) + +(defgeneric read-next-tokens (tokens-source) + (:documentation "Read next token from a lexical analysed stream. The nature of +TOKENS-SOURCE is implementation dependent and any lexical analyzer is +supposed to specialise this method.")) + +;; This is the actual parser. the algorithm is pretty +;; straightforward, the execution of the reductions a bit less. Error +;; recovery is rather clumsy. + +(defun parse (grammar start tokenizer) + "Match a GRAMMAR against the list of input tokens coming from TOKENIZER. +Return the reduced values according to the nonterminal actions. Raise +an error on failure." + (declare (type grammar grammar) + (type symbol start)) + (labels + ((match-token (expected token) + (when *debug* + (format *debug* "match-token ~S ~S -> " expected token)) + (let ((res (cond ((symbolp expected) + ;; non-costant terminal (like identifiers) + (eq expected (token-type token))) + ((and (stringp expected) + (stringp (token-value token))) + ;; string costant terminal + (funcall (the function (grammar-equal-p grammar)) expected (token-value token))) + ((functionp expected) + ;; custom equality predicate (must be able + ;; to deal with token objects) + (funcall expected token)) + ;; all the rest + (t (equal expected (token-value token)))))) + (when *debug* + (format *debug* "~Amatched~%" (if res "" "not "))) + res)) + (match (expected matched #+debug depth) + (declare (list expected matched) + #+debug (fixnum depth)) + (let ((first-expected (car expected))) + (cond #+debug ((> depth *maximum-recursion-depth*) + (error "endless recursion on ~A ~A at ~A expecting ~S" + (token-type (car matched)) (token-value (car matched)) + (token-position (car matched)) expected)) + ((eq first-expected :any) + (match (cdr expected) (cdr matched) #+debug depth)) + ;; This is a trick to obtain partial parses. When we + ;; reach this expected token we assume we succeeded + ;; the parsing and return the remaining tokens as + ;; part of the match. + ((eq first-expected :rest) + ;; we could be at the end of input so we check this + (unless (cdr matched) + (setf (cdr matched) (list :rest))) + (list nil nil)) + ((rule-p first-expected) + ;; If it's a rule, then we try to match all its + ;; productions. We return the first that succeeds. + (loop + for production in (rule-productions first-expected) + for production-tokens of-type list = (prod-tokens production) + with last-error-position = nil + with last-error = nil + for (error-position error-descr) = + (progn + (when *debug* + (format *debug* "trying to match ~A: ~S~%" + (rule-name first-expected) production-tokens)) + (match (append production-tokens (cdr expected)) matched #+debug (1+ depth))) + do (cond ((not error-position) + (return (let ((args-count (prod-tokens-length production))) + (setf (cdr matched) + (cons (reduce-production + production + (subseq (the list (cdr matched)) 0 args-count)) + (nthcdr (1+ args-count) matched))) + (list nil nil)))) + ((or (not last-error) + (later-position error-position last-error-position)) + (setf last-error-position error-position + last-error error-descr))) + ;; if everything fails return the "best" error + finally (return (list last-error-position + (if *debug* + #'(lambda () + (format nil "~A, trying to match ~A" + (funcall (the function last-error)) + (rule-name first-expected))) + last-error))))) + (t + ;; if necessary load the next tokens + (when (null (cdr matched)) + (setf (cdr matched) (read-next-tokens tokenizer))) + (cond ((and (or (null expected) (eq first-expected :eof)) + (null (cdr matched))) + ;; This point is reached only once for each complete + ;; parsing. The expected tokens and the input + ;; tokens have been exhausted at the same time. + ;; Hence we succeeded the parsing. + (setf (cdr matched) (list :eof)) + (list nil nil)) + ((null expected) + ;; Garbage at end of parsing. This may mean that we + ;; have considered a production completed too soon. + (list (token-position (car matched)) + #'(lambda () + "garbage at end of parsing"))) + ((null (cdr matched)) + ;; EOF error + (list :eof + #'(lambda () + (format nil "end of input expecting ~S" expected)))) + (t ;; normal token + (let ((first-token (cadr matched))) + (if (match-token first-expected first-token) + (match (cdr expected) (cdr matched) #+debug depth) + ;; failed: we return the error + (list (token-position first-token) + #'(lambda () + (format nil "expected ~S but got ~S ~S" + first-expected (token-type first-token) + (token-value first-token))))))))))))) + (declare (inline match-token)) + (let ((result (list :head))) + (destructuring-bind (error-position error) + (match (list (find-rule start (grammar-rules grammar))) result #+debug 0) + (when error-position + (error "~A at ~A~%" (funcall (the function error)) error-position)) + (cadr result))))) + +(defgeneric terminals-in-grammar (grammar-or-hashtable) + (:documentation + "Find non constant terminal symbols in GRAMMAR.")) + +(defmethod terminals-in-grammar ((grammar hash-table)) + (loop + for rule being each hash-value of grammar + with terminals = '() + do (loop + for prod in (rule-productions rule) + do (loop + for tok in (prod-tokens prod) + when (symbolp tok) + do (pushnew tok terminals))) + finally (return terminals))) + +(defmethod terminals-in-grammar ((grammar grammar)) + (terminals-in-grammar (grammar-rules grammar))) + +(defun print-grammar-figures (grammar &optional (stream *standard-output*)) + (format stream "rules: ~A~%constant terminals: ~A~%variable terminals: ~S~%" + (hash-table-count (grammar-rules grammar)) + (hash-table-count (grammar-keywords grammar)) + (terminals-in-grammar (grammar-rules grammar)))) + + +(defun grammar-keyword-p (keyword grammar) + "Check if KEYWORD is part of this grammar." + (find-keyword keyword (grammar-keywords grammar))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defvar *grammars* (make-hash-table)) + +(defun find-grammar (name) + (gethash name *grammars*)) + +(defun delete-grammar (name) + (remhash name *grammars*)) + +(defun add-grammar (name grammar) + (setf (gethash name *grammars*) grammar)) diff --git a/third_party/lisp/parse-float.nix b/third_party/lisp/parse-float.nix new file mode 100644 index 000000000000..4e36e69c7d98 --- /dev/null +++ b/third_party/lisp/parse-float.nix @@ -0,0 +1,22 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "soemraws"; + repo = "parse-float"; + rev = "3074765101e41222b6b624a66aaf1e6416379f9c"; + sha256 = "0jd2spawc3v8vzqf8ky4cngl45jm65fhkrdf20mf6dcbn3mzpkmr"; + }; + +in depot.nix.buildLisp.library { + name = "parse-float"; + + deps = with depot.third_party.lisp; [ + alexandria + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "parse-float.lisp" + ]; +} diff --git a/third_party/lisp/parse-number.nix b/third_party/lisp/parse-number.nix new file mode 100644 index 000000000000..1ceba2863d8d --- /dev/null +++ b/third_party/lisp/parse-number.nix @@ -0,0 +1,17 @@ +{ depot, pkgs, ... }: + +let + + src = pkgs.fetchFromGitHub { + owner = "sharplispers"; + repo = "parse-number"; + rev = "7707b224c4b941c2cbd28459113534242cee3a31"; + sha256 = "0sk06ib1bhqv9y39vwnnw44vmbc4b0kvqm37xxmkxd4dwchq82d7"; + }; + +in depot.nix.buildLisp.library { + name = "parse-number"; + srcs = map (f: src + ("/" + f)) [ + "parse-number.lisp" + ]; +} diff --git a/third_party/lisp/parseq.nix b/third_party/lisp/parseq.nix new file mode 100644 index 000000000000..fe045d52996f --- /dev/null +++ b/third_party/lisp/parseq.nix @@ -0,0 +1,20 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "mrossini-ethz"; + repo = "parseq"; + rev = "5cd95b324b68255d89f27f8065f4c29674558b26"; + sha256 = "1f3vvxgyiv0xn2hzafhh63l3gnvn2vaxr5pi3ld7d340mka2ndg0"; + }; + +in depot.nix.buildLisp.library { + name = "parseq"; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "conditions.lisp" + "utils.lisp" + "defrule.lisp" + ]; +} diff --git a/third_party/lisp/physical-quantities.nix b/third_party/lisp/physical-quantities.nix new file mode 100644 index 000000000000..b8079e8ebaaf --- /dev/null +++ b/third_party/lisp/physical-quantities.nix @@ -0,0 +1,31 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "mrossini-ethz"; + repo = "physical-quantities"; + rev = "v0.2.1"; + sha256 = "0mb2s94s6fhw5vfa89naalw7ld11sdsszlqpz0c65dvpfyfmmdmh"; + }; + +in depot.nix.buildLisp.library { + name = "physical-quantities"; + + deps = with depot.third_party.lisp; [ + parseq + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "utils.lisp" + "conditions.lisp" + "unit-factor.lisp" + "unit-database.lisp" + "units.lisp" + "quantity.lisp" + "numeric.lisp" + "parse-rules.lisp" + "read-macro.lisp" + "si-units.lisp" + ]; +} diff --git a/third_party/lisp/postmodern.nix b/third_party/lisp/postmodern.nix new file mode 100644 index 000000000000..333a9d9b770f --- /dev/null +++ b/third_party/lisp/postmodern.nix @@ -0,0 +1,94 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.nix.buildLisp) bundled; + + src = pkgs.fetchFromGitHub { + owner = "marijnh"; + repo = "Postmodern"; + rev = "v1.32"; + sha256 = "0prwmpixcqpzqd67v77cs4zgbs73a10m6hs7q0rpv0z1qm7mqfcb"; + }; + + cl-postgres = depot.nix.buildLisp.library { + name = "cl-postgres"; + deps = with depot.third_party.lisp; [ + md5 + split-sequence + ironclad + cl-base64 + uax-15 + usocket + ]; + + srcs = map (f: src + ("/cl-postgres/" + f)) [ + "package.lisp" + "features.lisp" + "errors.lisp" + "sql-string.lisp" + "trivial-utf-8.lisp" + "strings-utf-8.lisp" + "communicate.lisp" + "messages.lisp" + "oid.lisp" + "ieee-floats.lisp" + "interpret.lisp" + "saslprep.lisp" + "scram.lisp" + "protocol.lisp" + "public.lisp" + "bulk-copy.lisp" + ]; + }; + + s-sql = depot.nix.buildLisp.library { + name = "s-sql"; + deps = with depot.third_party.lisp; [ + cl-postgres + alexandria + ]; + + srcs = map (f: src + ("/s-sql/" + f)) [ + "package.lisp" + "s-sql.lisp" + ]; + }; + + postmodern = depot.nix.buildLisp.library { + name = "postmodern"; + + deps = with depot.third_party.lisp; [ + alexandria + cl-postgres + s-sql + global-vars + split-sequence + cl-unicode + closer-mop + bordeaux-threads + ]; + + srcs = [ + "${src}/postmodern.asd" + ] ++ (map (f: src + ("/postmodern/" + f)) [ + "package.lisp" + "connect.lisp" + "query.lisp" + "prepare.lisp" + "roles.lisp" + "util.lisp" + "transaction.lisp" + "namespace.lisp" + "execute-file.lisp" + "table.lisp" + "deftable.lisp" + ]); + + brokenOn = [ + "ecl" # TODO(sterni): https://gitlab.com/embeddable-common-lisp/ecl/-/issues/651 + ]; + }; + +in postmodern // { + inherit s-sql cl-postgres; +} diff --git a/third_party/lisp/puri.nix b/third_party/lisp/puri.nix new file mode 100644 index 000000000000..925b457f9f03 --- /dev/null +++ b/third_party/lisp/puri.nix @@ -0,0 +1,14 @@ +# Portable URI library +{ depot, pkgs, ... }: + +let src = pkgs.fetchgit { + url = "http://git.kpe.io/puri.git"; + rev = "4bbab89d9ccbb26346899d1f496c97604fec567b"; + sha256 = "0gq2rsr0aihs0z20v4zqvmdl4szq53b52rh97pvnmwrlbn4mapmd"; +}; +in depot.nix.buildLisp.library { + name = "puri"; + srcs = [ + (src + "/src.lisp") + ]; +} diff --git a/third_party/lisp/quasiquote_2/README.md b/third_party/lisp/quasiquote_2/README.md new file mode 100644 index 000000000000..2d590a0564ae --- /dev/null +++ b/third_party/lisp/quasiquote_2/README.md @@ -0,0 +1,258 @@ +quasiquote-2.0 +============== + +Why should it be hard to write macros that write other macros? +Well, it shouldn't! + +quasiquote-2.0 defines slightly different rules for quasiquotation, +that make writing macro-writing macros very smooth experience. + +NOTE: quasiquote-2.0 does horrible things to shared structure!!! +(it does a lot of COPY-TREE's, so shared-ness is destroyed). +So, it's indeed a tool to construct code (where it does not matter much if the +structure is shared or not) and not the data (or, at least, not the data with shared structure) + + +```lisp +(quasiquote-2.0:enable-quasiquote-2.0) + +(defmacro define-my-macro (name args &body body) + `(defmacro ,name ,args + `(sample-thing-to-expand-to + ,,@body))) ; note the difference from usual way + +(define-my-macro foo (x y) + ,x ; now here injections of quotation constructs work + ,y) + +(define-my-macro bar (&body body) + ,@body) ; splicing is also easy +``` + +The "injections" in macros FOO and BAR work as naively expected, as if I had written +```lisp +(defmacro foo (x y) + `(sample-thing-to-expand-to ,x ,y)) + +(defmacro bar (&body body) + `(sample-thing-to-expand-to ,@body)) + +(macroexpand-1 '(foo a b)) + + '(SAMPLE-THING-TO-EXPAND-TO A B) + +(macroexpand-1 '(bar a b c)) + + '(SAMPLE-THING-TO-EXPAND-TO A B C) +``` + + +So, how is this effect achieved? + + +DIG, INJECT and SPLICE +------------------------- + +The transformations of backquote occur at macroexpansion-time and not at read-time. +It is totally possible not to use any special reader syntax, but just +underlying macros directly! + +At the core is a macro DIG, which expands to the code that generates the +expression according to the rules, which are roughly these: + * each DIG increases "depth" by one (hence the name) + * each INJECT or SPLICE decreases "depth" by one + * if depth is 0, evaluation is turned on + * if depth if not zero (even if it's negative!) evaluation is off + * SPLICE splices the form, similarly to ordinary `,@`, INJECT simply injects, same as `,` + +```lisp +;; The example using macros, without special reader syntax + +(dig ; depth is 1 here + (a b + (dig ; depth is 2 here + ((inject c) ; this inject is not evaluated, because depth is nonzero + (inject (d ;depth becomes 1 here again + (inject e) ; and this inject is evaluated, because depth becomes zero + )) + (inject 2 f) ; this inject with level specification is evaluated, because it + ; decreases depth by 2 + )))) + + +;; the same example using ENABLE-QUASIQUOTE-2.0 syntax is written as +`(a b `(,c ,(d ,e) ,,f)) ; note double comma acts different than usually +``` + + +The ENABLE-QUASIQUOTE-2.0 macro just installs reader that reads +`FORM as (DIG FORM), ,FORM as (INJECT FORM) and ,@FORM as (SPLICE FORM). +You can just as well type DIG's, INJECT's and SPLICE's directly, +(in particular, when writing utility functions that generate macro-generating code) +or roll your own convenient reader syntax (pull requests are welcome). + +So, these two lines (with ENABLE-QUASIQUOTE-2.0) read the same +```lisp +`(a (,b `,,c) d) + +(dig (a ((inject b) (dig (inject 2 c))) d)) +``` + +You may notice the (INJECT 2 ...) form appearing, which is described below. + + +At "level 1", i.e. when only \` , and ,@ are used, and not, say \`\` ,, ,', ,,@ ,',@ +this behaves exactly as usual quasiquotation. + + +The optional N argument +-------------- + +All quasiquote-2.0 operators accept optional "depth" argument, +which goes before the form for human readability. + +Namely, (DIG N FORM) increases depth by N instead of one and +(INJECT N FORM) decreases depth by N instead of one. + +```lisp +(DIG 2 (INJECT 2 A)) + +; gives the same result as + +(DIG (INJECT A)) +``` + + +In fact, with ENABLE-QUASIQUOTE-2.0, say, ,,,,,FORM (5 quotes) reads as (INJECT 5 FORM) +and ,,,,,@FORM as (SPLICE 5 FORM) + + +More examples +------------- + +For fairly complicated example, which uses ,,,@ and OINJECT (see below), + see DEFINE-BINOP-DEFINER macro +in CG-LLVM (https://github.com/mabragor/cg-llvm/src/basics.lisp), +desire to write which was the initial impulse for this project. + + +For macro, that is not a macro-writing macro, yet benefits from +ability to inject using `,` and `,@`, consider JOINING-WITH-COMMA-SPACE macro +(also from CG-LLVM) + +```lisp +(defmacro joining-with-comma-space (&body body) + ;; joinl just joins strings in the list with specified string + `(joinl ", " (mapcar #'emit-text-repr + (remove-if-not #'identity `(,,@body))))) + +;; the macro can be then used uniformly over strings and lists of strings +(defun foo (x y &rest z) + (joining-with-comma-space ,x ,y ,@z)) + +(foo "a" "b" "c" "d") + ;; produces + "a, b, c, d" +``` + + +ODIG and OINJECT and OSPLICE +---------------------------- + +Sometimes you don't want DIG's macroexpansion to look further into the structure of +some INJECT or SPLICE or DIG in its subform, +if the depth does not match. In these cases you need "opaque" versions of +DIG, INJECT and SPLICE, named, respectively, ODIG, OINJECT and OSPLICE. + +```lisp +;; here injection of B would occur +(defun foo (b) + (dig (dig (inject (a (inject b)))))) + +;; and here not, because macroexpansion does not look into OINJECT form +(defun bar (b) + (dig (dig (oinject (a (inject b)))))) + +(foo 1) + + '(DIG (INJECT (A 1))) + +(bar 1) + + '(DIG (OINJECT (A (INJECT B)))) +``` + +MACRO-INJECT and MACRO-SPLICE +----------------------------- + +Sometimes you just want to abstract-out some common injection patterns... +That is, you want macros, that expand into common injection patterns. +However, you want this only sometimes, and only in special circumstances. +So it won't do, if INJECT and SPLICE just expanded something, whenever it +turned out to be macro. For that, use MACRO-INJECT and MACRO-SPLICE. + +```lisp +;; with quasiquote-2.0 syntax turned on +(defmacro inject-n-times (form n) + (make-list n :initial-element `(inject ,form))) + +(let (x 0) + `(dig (a (macro-inject (inject-n-times (incf x) 3))))) +;; yields +'(a (1 2 3)) + +;;and same with MACRO-SPLICE +(let (x 0) + `(dig (a (macro-splice (inject-n-times (incf x) 3))))) +;; yields +'(a 1 2 3) +``` + +OMACRO-INJECT and OMACRO-SPLICE are, as usual, opaque variants of MACRO-INJECT and MACRO-SPLICE. + +Both MACRO-INJECT and MACRO-SPLICE expand their subform exactly once (using MACROEXPAND-1), +before plugging it into list. +If you want to expand as much as it's possible, use MACRO-INJECT-ALL and MACRO-SPLICE-ALL, +which expand using MACROEXPAND before injecting/splicing, respectively. +That implies, that while subform of MACRO-INJECT and MACRO-SPLICE is checked to be +macro-form, the subform of MACRO-INJECT-ALL is not. + + +Terse syntax of the ENABLE-QUASIQUOTE-2.0 +----------------------------------------- + +Of course, typing all those MACRO-INJECT-ALL, or OMACRO-SPLICE-ALL or whatever explicitly +every time you want this special things is kind of clumsy. For that, default reader +of quasiquote-2.0 provides extended syntax + +```lisp +',,,,!oma@x + +;; reads as +'(OMACRO-SPLICE-ALL 4 X) +``` + +That is, the regexp of the syntax is +[,]+![o][m][a][@]<whatever> + +As usual, number of commas determine the anti-depth of the injector, exclamation mark +turns on the syntax, if `o` is present, opaque version of injector will be used, +if `m` is present, macro-expanding version of injector will be used and if +`a` is present, macro-all version of injector will be used. + +Note: it's possible to write ,!ax, which will read as (INJECT-ALL X), but +this will not correspond to the actual macro name. + +Note: it was necessary to introduce special escape-char for extended syntax, +since usual idioms like `,args` would otherwise be completely screwed. + + +TODO +---- + +* WITH-QUASIQUOTE-2.0 read-macro-token for local enabling of ` and , overloading +* wrappers for convenient definition of custom overloading schemes +* some syntax for opaque operations + +P.S. Name "quasiquote-2.0" comes from "patronus 2.0" spell from www.hpmor.com + and has nothing to do with being "the 2.0" version of quasiquote. \ No newline at end of file diff --git a/third_party/lisp/quasiquote_2/default.nix b/third_party/lisp/quasiquote_2/default.nix new file mode 100644 index 000000000000..521c384787fe --- /dev/null +++ b/third_party/lisp/quasiquote_2/default.nix @@ -0,0 +1,17 @@ +# Quasiquote more suitable for macros that define other macros +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "quasiquote-2.0"; + + deps = [ + depot.third_party.lisp.iterate + ]; + + srcs = [ + ./package.lisp + ./quasiquote-2.0.lisp + ./macros.lisp + ./readers.lisp + ]; +} diff --git a/third_party/lisp/quasiquote_2/macros.lisp b/third_party/lisp/quasiquote_2/macros.lisp new file mode 100644 index 000000000000..6ebeb47d081e --- /dev/null +++ b/third_party/lisp/quasiquote_2/macros.lisp @@ -0,0 +1,15 @@ + +(in-package #:quasiquote-2.0) + +(defmacro define-dig-like-macro (name) + `(defmacro ,name (n-or-form &optional (form nil form-p) &environment env) + (if (not form-p) + `(,',name 1 ,n-or-form) + (let ((*env* env)) + (transform-dig-form `(,',name ,n-or-form ,form)))))) + + +(define-dig-like-macro dig) +(define-dig-like-macro odig) + + diff --git a/third_party/lisp/quasiquote_2/package.lisp b/third_party/lisp/quasiquote_2/package.lisp new file mode 100644 index 000000000000..9b140ef84c32 --- /dev/null +++ b/third_party/lisp/quasiquote_2/package.lisp @@ -0,0 +1,11 @@ +;;;; package.lisp + +(defpackage #:quasiquote-2.0 + (:use #:cl #:iterate) + (:export #:%codewalk-dig-form #:transform-dig-form + #:dig #:inject #:splice #:odig #:oinject #:osplice + #:macro-inject #:omacro-inject #:macro-splice #:omacro-splice + #:macro-inject-all #:omacro-inject-all #:macro-splice-all #:omacro-splice-all + #:enable-quasiquote-2.0 #:disable-quasiquote-2.0)) + + diff --git a/third_party/lisp/quasiquote_2/quasiquote-2.0.asd b/third_party/lisp/quasiquote_2/quasiquote-2.0.asd new file mode 100644 index 000000000000..3acfd32b80e6 --- /dev/null +++ b/third_party/lisp/quasiquote_2/quasiquote-2.0.asd @@ -0,0 +1,30 @@ +;;;; quasiquote-2.0.asd + +(defpackage :quasiquote-2.0-system + (:use :cl :asdf)) + +(in-package quasiquote-2.0-system) + +(asdf:defsystem #:quasiquote-2.0 + :serial t + :description "Writing macros that write macros. Effortless." + :author "Alexandr Popolitov <popolit@gmail.com>" + :license "MIT" + :version "0.3" + :depends-on (#:iterate) + :components ((:file "package") + (:file "quasiquote-2.0") + (:file "macros") + (:file "readers"))) + +(defsystem :quasiquote-2.0-tests + :description "Tests for QUASIQUOTE-2.0" + :licence "MIT" + :depends-on (:quasiquote-2.0 :fiveam) + :components ((:file "tests") + (:file "tests-macro") + )) + +(defmethod perform ((op test-op) (sys (eql (find-system :quasiquote-2.0)))) + (load-system :quasiquote-2.0) + (funcall (intern "RUN-TESTS" :quasiquote-2.0))) diff --git a/third_party/lisp/quasiquote_2/quasiquote-2.0.lisp b/third_party/lisp/quasiquote_2/quasiquote-2.0.lisp new file mode 100644 index 000000000000..10043fe0ecbc --- /dev/null +++ b/third_party/lisp/quasiquote_2/quasiquote-2.0.lisp @@ -0,0 +1,340 @@ +;;;; quasiquote-2.0.lisp + +(in-package #:quasiquote-2.0) + +(defparameter *env* nil) + +(defmacro nonsense-error (str) + `(error ,(concatenate 'string + str + " appears as a bare, non DIG-enclosed form. " + "For now I don't know how to make sense of this."))) + +(defmacro define-nonsense-when-bare (name) + `(defmacro ,name (n-or-form &optional form) + (declare (ignore n-or-form form)) + (nonsense-error ,(string name)))) + +(define-nonsense-when-bare inject) +(define-nonsense-when-bare oinject) +(define-nonsense-when-bare splice) +(define-nonsense-when-bare osplice) +(define-nonsense-when-bare macro-inject) + +(defparameter *depth* 0) + + +(defparameter *injectors* nil) + +(defparameter *void-elt* nil) +(defparameter *void-filter-needed* nil) + +;; (defmacro with-injector-parsed (form) +;; `(let ((kwd (intern (string + +(defun reset-injectors () + (setf *injectors* nil)) + +(defparameter *known-injectors* '(inject splice oinject osplice + macro-inject omacro-inject + macro-splice omacro-splice + macro-inject-all omacro-inject-all + macro-splice-all omacro-splice-all)) + +(defun injector-form-p (form) + (and (consp form) + (find (car form) *known-injectors* :test #'eq))) + +(defun injector-level (form) + (if (equal 2 (length form)) + 1 + (cadr form))) + +(defun injector-subform (form) + (if (equal 2 (length form)) + (values (cdr form) '(cdr)) + (values (cddr form) '(cddr)))) + +(defparameter *opaque-injectors* '(odig oinject osplice omacro-inject)) + +(defun transparent-p (form) + (not (find (car form) *opaque-injectors* :test #'eq))) + +(defun look-into-injector (form path) + (let ((*depth* (- *depth* (injector-level form)))) + (multiple-value-bind (subform subpath) (injector-subform form) + (search-all-active-sites subform (append subpath path) nil)))) + +(defparameter *known-diggers* '(dig odig)) + +(defun dig-form-p (form) + (and (consp form) + (find (car form) *known-diggers* :test #'eq))) + +(defun look-into-dig (form path) + (let ((*depth* (+ *depth* (injector-level form)))) + (multiple-value-bind (subform subpath) (injector-subform form) + (search-all-active-sites subform (append subpath path) nil)))) + +(defun handle-macro-1 (form) + (if (atom form) + (error "Sorry, symbol-macros are not implemented for now") + (let ((fun (macro-function (car form) *env*))) + (if (not fun) + (error "The subform of MACRO-1 injector is supposed to be macro, perhaps, something went wrong...")) + (macroexpand-1 form *env*)))) + +(defun handle-macro-all (form) + (if (atom form) + (error "Sorry, symbol-macros are not implemented for now") + (macroexpand form *env*))) + + +(defparameter *macro-handlers* `((macro-inject . ,#'handle-macro-1) + (omacro-inject . ,#'handle-macro-1) + (macro-splice . ,#'handle-macro-1) + (omacro-splice . ,#'handle-macro-1) + (macro-inject-all . ,#'handle-macro-all) + (omacro-inject-all . ,#'handle-macro-all) + (macro-splice-all . ,#'handle-macro-all) + (omacro-splice-all . ,#'handle-macro-all))) + +(defun get-macro-handler (sym) + (or (cdr (assoc sym *macro-handlers*)) + (error "Don't know how to handle this macro injector: ~a" sym))) + + + +(defun macroexpand-macroinjector (place) + (if (not (splicing-injector (car place))) + (progn (setf (car place) (funcall (get-macro-handler (caar place)) + (car (injector-subform (car place))))) + nil) + (let ((new-forms (funcall (get-macro-handler (caar place)) + (car (injector-subform (car place)))))) + (cond ((not new-forms) + (setf *void-filter-needed* t + (car place) *void-elt*)) + ((atom new-forms) (error "We need to splice the macroexpansion, but got atom: ~a" new-forms)) + (t (setf (car place) (car new-forms)) + (let ((tail (cdr place))) + (setf (cdr place) (cdr new-forms) + (cdr (last new-forms)) tail)))) + t))) + + +(defun search-all-active-sites (form path toplevel-p) + ;; (format t "SEARCH-ALL-ACTIVE-SITES: got form ~a~%" form) + (if (not form) + nil + (if toplevel-p + (cond ((atom (car form)) :just-quote-it!) + ((injector-form-p (car form)) (if (equal *depth* (injector-level (car form))) + :just-form-it! + (if (transparent-p (car form)) + (look-into-injector (car form) (cons 'car path))))) + ((dig-form-p (car form)) + ;; (format t "Got dig form ~a~%" form) + (if (transparent-p (car form)) + (look-into-dig (car form) (cons 'car path)))) + (t (search-all-active-sites (car form) (cons 'car path) nil) + (search-all-active-sites (cdr form) (cons 'cdr path) nil))) + (when (consp form) + (cond ((dig-form-p (car form)) + ;; (format t "Got dig form ~a~%" form) + (if (transparent-p (car form)) + (look-into-dig (car form) (cons 'car path)))) + ((injector-form-p (car form)) + ;; (format t "Got injector form ~a ~a ~a~%" form *depth* (injector-level (car form))) + (if (equal *depth* (injector-level (car form))) + (if (macro-injector-p (car form)) + (progn (macroexpand-macroinjector form) + (return-from search-all-active-sites + (search-all-active-sites form path nil))) + (progn (push (cons form (cons 'car path)) *injectors*) + nil)) + (if (transparent-p (car form)) + (look-into-injector (car form) (cons 'car path))))) + (t (search-all-active-sites (car form) (cons 'car path) nil))) + (search-all-active-sites (cdr form) (cons 'cdr path) nil))))) + + + +(defun codewalk-dig-form (form) + (reset-injectors) + (let ((it (search-all-active-sites form nil t))) + (values (nreverse *injectors*) it))) + +(defun %codewalk-dig-form (form) + (if (not (dig-form-p form)) + (error "Supposed to be called on dig form") + (let ((*depth* (+ (injector-level form) *depth*))) + (codewalk-dig-form (injector-subform form))))) + +(defun path->setfable (path var) + (let ((res var)) + ;; First element is artifact of extra CAR-ing + (dolist (spec (cdr (reverse path))) + (setf res (list spec res))) + res)) + +(defun tree->cons-code (tree) + (if (atom tree) + `(quote ,tree) + `(cons ,(tree->cons-code (car tree)) + ,(tree->cons-code (cdr tree))))) + +(defparameter *known-splicers* '(splice osplice + macro-splice omacro-splice + macro-splice-all omacro-splice-all)) + +(defun splicing-injector (form) + (and (consp form) + (find (car form) *known-splicers* :test #'eq))) + +(defparameter *known-macro-injectors* '(macro-inject omacro-inject + macro-splice omacro-splice + macro-inject-all omacro-inject-all + macro-splice-all omacro-splice-all + )) + +(defun macro-injector-p (form) + (and (consp form) + (find (car form) *known-macro-injectors* :test #'eq))) + +(defun filter-out-voids (lst void-sym) + (let (caars cadrs cdars cddrs) + ;; search for all occurences of VOID + (labels ((rec (x) + (if (consp x) + (progn (cond ((consp (car x)) + (cond ((eq void-sym (caar x)) (push x caars)) + ((eq void-sym (cdar x)) (push x cdars)))) + ((consp (cdr x)) + (cond ((eq void-sym (cadr x)) (push x cadrs)) + ((eq void-sym (cddr x)) (push x cddrs))))) + (rec (car x)) + (rec (cdr x)))))) + (rec lst)) + (if (or cdars cddrs) + (error "Void sym found on CDR position, which should not have happened")) + ;; destructively transform LST + (dolist (elt caars) + (setf (car elt) (cdar elt))) + (dolist (elt cadrs) + (setf (cdr elt) (cddr elt))) + ;; check that we indeed filtered-out all VOIDs + (labels ((rec (x) + (if (not (atom x)) + (progn (rec (car x)) + (rec (cdr x))) + (if (eq void-sym x) + (error "Not all VOIDs were filtered"))))) + (rec lst)) + lst)) + +(defun transform-dig-form (form) + (let ((the-form (copy-tree form))) + (let ((*void-filter-needed* nil) + (*void-elt* (gensym "VOID"))) + (multiple-value-bind (site-paths cmd) (%codewalk-dig-form the-form) + (cond ((eq cmd :just-quote-it!) + `(quote ,(car (injector-subform the-form)))) + ((eq cmd :just-form-it!) + (car (injector-subform (car (injector-subform the-form))))) + (t (let ((cons-code (if (not site-paths) + (tree->cons-code (car (injector-subform the-form))) + (really-transform-dig-form the-form site-paths)))) + (if (not *void-filter-needed*) + cons-code + `(filter-out-voids ,cons-code ',*void-elt*))))))))) + +(defmacro make-list-form (o!-n form) + (let ((g!-n (gensym "N")) + (g!-i (gensym "I")) + (g!-res (gensym "RES"))) + `(let ((,g!-n ,o!-n) + (,g!-res nil)) + (dotimes (,g!-i ,g!-n) + (push ,form ,g!-res)) + (nreverse ,g!-res)))) + +(defun mk-splicing-injector-let (x) + `(let ((it ,(car (injector-subform x)))) + (assert (listp it)) + (copy-list it))) + + + +(defun mk-splicing-injector-setf (path g!-list g!-splicee) + (assert (eq 'car (car path))) + (let ((g!-rest (gensym "REST"))) + `(let ((,g!-rest ,(path->setfable (cons 'cdr (cdr path)) g!-list))) + (assert (or (not ,g!-rest) (consp ,g!-rest))) + (if (not ,g!-splicee) + (setf ,(path->setfable (cdr path) g!-list) + ,g!-rest) + (progn (setf ,(path->setfable (cdr path) g!-list) ,g!-splicee) + (setf (cdr (last ,g!-splicee)) ,g!-rest)))))) + + +(defun really-transform-dig-form (the-form site-paths) + (let ((gensyms (make-list-form (length site-paths) (gensym "INJECTEE")))) + (let ((g!-list (gensym "LIST"))) + (let ((lets nil) + (splicing-setfs nil) + (setfs nil)) + (do ((site-path site-paths (cdr site-path)) + (gensym gensyms (cdr gensym))) + ((not site-path)) + (destructuring-bind (site . path) (car site-path) + (push `(,(car gensym) ,(if (not (splicing-injector (car site))) + (car (injector-subform (car site))) + (mk-splicing-injector-let (car site)))) + lets) + (if (not (splicing-injector (car site))) + (push `(setf ,(path->setfable path g!-list) ,(car gensym)) setfs) + (push (mk-splicing-injector-setf path g!-list (car gensym)) splicing-setfs)) + (setf (car site) nil))) + `(let ,(nreverse lets) + (let ((,g!-list ,(tree->cons-code (car (injector-subform the-form))))) + ,@(nreverse setfs) + ;; we apply splicing setf in reverse order for them not to bork the paths of each other + ,@splicing-setfs + ,g!-list)))))) + + +;; There are few types of recursive injection that may happen: +;; * compile-time injection: +;; (dig (inject (dig (inject a)))) -- this type will be handled automatically by subsequent macroexpansions +;; * run-time injection: +;; (dig (dig (inject 2 a))) +;; and A is '(dig (inject 3 'foo)) -- this one we guard against ? (probably, first we just ignore it +;; -- do not warn about it, and then it wont really happen. +;; * macroexpanded compile-time injection: +;; (dig (inject (my-macro a b c))), +;; where MY-MACRO expands into, say (splice (list 'a 'b 'c)) +;; This is *not* handled automatically, and therefore we must do it by hand. + + +;; OK, now how to implement splicing ? +;; (dig (a (splice (list b c)) d)) +;; should transform into code that yields +;; (a b c d) +;; what this code is? +;; (let ((#:a (copy-list (list b c)))) +;; (let ((#:res (cons 'a nil 'd))) +;; ;; all non-splicing injects go here, as they do not spoil the path-structure +;; (setf (cdr #:res) #:a) +;; (setf (cdr (last #:a)) (cdr (cdr #:res))) +;; #:res))) + + +;; How this macroexpansion should work in general? +;; * We go over the cons-tree, keeping track of the depth level, which is +;; controlled by DIG's +;; * Once we find the INJECT with matching level, we remember the place, where +;; this happens +;; * We have two special cases: +;; * cons-tree is an atom +;; * cons-tree is just a single INJECT diff --git a/third_party/lisp/quasiquote_2/readers.lisp b/third_party/lisp/quasiquote_2/readers.lisp new file mode 100644 index 000000000000..7c4c5a30c98e --- /dev/null +++ b/third_party/lisp/quasiquote_2/readers.lisp @@ -0,0 +1,77 @@ + + +(in-package #:quasiquote-2.0) + +(defun read-n-chars (stream char) + (let (new-char + (n 0)) + (loop + (setf new-char (read-char stream nil :eof t)) + (if (not (char= new-char char)) + (progn (unread-char new-char stream) + (return n)) + (incf n))))) + +(defmacro define-dig-reader (name symbol) + `(defun ,name (stream char) + (let ((depth (1+ (read-n-chars stream char)))) + (if (equal 1 depth) + (list ',symbol (read stream t nil t)) + (list ',symbol + depth + (read stream t nil t)))))) + +(define-dig-reader dig-reader dig) +(define-dig-reader odig-reader odig) + +(defun expect-char (char stream) + (let ((new-char (read-char stream t nil t))) + (if (char= char new-char) + t + (unread-char new-char stream)))) + +(defun guess-injector-name (opaque-p macro-p all-p splicing-p) + (intern (concatenate 'string + (if opaque-p "O" "") + (if macro-p "MACRO-" "") + (if splicing-p "SPLICE" "INJECT") + (if all-p "-ALL" "")) + "QUASIQUOTE-2.0")) + +(defun inject-reader (stream char) + (let ((anti-depth (1+ (read-n-chars stream char))) + (extended-syntax (expect-char #\! stream))) + (let ((injector-name (if (not extended-syntax) + (guess-injector-name nil nil nil (expect-char #\@ stream)) + (guess-injector-name (expect-char #\o stream) + (expect-char #\m stream) + (expect-char #\a stream) + (expect-char #\@ stream))))) + `(,injector-name ,@(if (not (equal 1 anti-depth)) `(,anti-depth)) + ,(read stream t nil t))))) + + + +(defvar *previous-readtables* nil) + +(defun %enable-quasiquote-2.0 () + (push *readtable* + *previous-readtables*) + (setq *readtable* (copy-readtable)) + (set-macro-character #\` #'dig-reader) + (set-macro-character #\, #'inject-reader) + (values)) + +(defun %disable-quasiquote-2.0 () + (if *previous-readtables* + (setf *readtable* (pop *previous-readtables*)) + (setf *readtable* (copy-readtable nil))) + (values)) + +(defmacro enable-quasiquote-2.0 () + `(eval-when (:compile-toplevel :load-toplevel :execute) + (%enable-quasiquote-2.0))) +(defmacro disable-quasiquote-2.0 () + `(eval-when (:compile-toplevel :load-toplevel :execute) + (%disable-quasiquote-2.0))) + diff --git a/third_party/lisp/quasiquote_2/tests-macro.lisp b/third_party/lisp/quasiquote_2/tests-macro.lisp new file mode 100644 index 000000000000..df6c43e21d77 --- /dev/null +++ b/third_party/lisp/quasiquote_2/tests-macro.lisp @@ -0,0 +1,21 @@ + +(in-package #:quasiquote-2.0-tests) + +(in-suite quasiquote-2.0) + +(enable-quasiquote-2.0) + +(defmacro define-sample-macro (name args &body body) + `(defmacro ,name ,args + `(sample-thing-to-macroexpand-to + ,,@body))) + +(define-sample-macro sample-macro-1 (x y) + ,x ,y) + +(define-sample-macro sample-macro-2 (&body body) + ,@body) + +(test macro-defined-macroexpansions + (is (equal '(sample-thing-to-macroexpand-to a b) (macroexpand-1 '(sample-macro-1 a b)))) + (is (equal '(sample-thing-to-macroexpand-to a b c) (macroexpand-1 '(sample-macro-2 a b c))))) \ No newline at end of file diff --git a/third_party/lisp/quasiquote_2/tests.lisp b/third_party/lisp/quasiquote_2/tests.lisp new file mode 100644 index 000000000000..6c8ab08cc1af --- /dev/null +++ b/third_party/lisp/quasiquote_2/tests.lisp @@ -0,0 +1,143 @@ +(in-package :cl-user) + +(defpackage :quasiquote-2.0-tests + (:use :cl :quasiquote-2.0 :fiveam) + (:export #:run-tests)) + +(in-package :quasiquote-2.0-tests) + +(def-suite quasiquote-2.0) +(in-suite quasiquote-2.0) + +(defun run-tests () + (let ((results (run 'quasiquote-2.0))) + (fiveam:explain! results) + (unless (fiveam:results-status results) + (error "Tests failed.")))) + +(test basic + (is (equal '(nil :just-quote-it!) (multiple-value-list (%codewalk-dig-form '(dig nil))))) + (is (equal '(nil :just-form-it!) (multiple-value-list (%codewalk-dig-form '(dig (inject a)))))) + (is (equal '(nil :just-form-it!) (multiple-value-list (%codewalk-dig-form '(dig 2 (inject 2 a)))))) + (is (equal '(((((inject b) c (inject d)) car cdr car) (((inject d)) car cdr cdr cdr car)) nil) + (multiple-value-list (%codewalk-dig-form '(dig (a (inject b) c (inject d))))))) + (is (equal '(nil nil) + (multiple-value-list (%codewalk-dig-form '(dig (dig (a (inject b) c (inject d)))))))) + (is (equal '(((((inject 2 d)) car cdr cdr cdr car cdr car)) nil) + (multiple-value-list (%codewalk-dig-form '(dig (dig (a (inject b) c (inject 2 d))))))))) + +(test transform + (is (equal '(quote a) (transform-dig-form '(dig a)))) + (is (equal '(quote a) (transform-dig-form '(dig 2 a)))) + (is (equal 'a (transform-dig-form '(dig (inject a))))) + (is (equal 'a (transform-dig-form '(dig 2 (inject 2 a)))))) + +(defun foo (b d) + (dig (a (inject b) c (inject d)))) + +(defun foo1-transparent (x) + (declare (ignorable x)) + (dig (dig (a (inject (b (inject x) c)))))) + +(defun foo1-opaque (x) + (declare (ignorable x)) + (dig (dig (a (oinject (b (inject x) c)))))) + +(defun foo-recursive (x y) + (dig (a (inject (list x (dig (c (inject y)))))))) + + +(test foos + (is (equal '(a 1 c 2) (foo 1 2))) + (is (equal '(a 100 c 200) (foo 100 200)))) + +(test opaque-vs-transparent + (is (equal '(quote a) (transform-dig-form '(odig a)))) + (is (equal '(quote a) (transform-dig-form '(odig 2 a)))) + (is (equal 'a (transform-dig-form '(odig (inject a))))) + (is (equal 'a (transform-dig-form '(odig 2 (inject 2 a))))) + (is (equal '(odig (inject 2 a)) (eval (transform-dig-form '(dig (odig (inject 2 a))))))) + (is (equal '(dig (a (inject (b 3 c)))) (foo1-transparent 3))) + (is (equal '(dig (a (oinject (b (inject x) c)))) (foo1-opaque 3)))) + +(test recursive-compile-time + (is (equal '(a (1 (c 2))) (foo-recursive 1 2)))) + + +(test splicing + (is (equal '(a b c d) (eval (transform-dig-form '(dig (a (splice '(b c)) d)))))) + (is (equal '(b c d) (eval (transform-dig-form '(dig ((splice '(b c)) d)))))) + (is (equal '(a b c) (eval (transform-dig-form '(dig (a (splice '(b c)))))))) + (is (equal '(a b) (eval (transform-dig-form '(dig (a (splice nil) b)))))) + (is (equal '(b) (eval (transform-dig-form '(dig ((splice nil) b)))))) + (is (equal '(a) (eval (transform-dig-form '(dig (a (splice nil))))))) + (is (equal '() (eval (transform-dig-form '(dig ((splice nil))))))) + (is (equal '(a b) (eval (transform-dig-form '(dig ((splice '(a b))))))))) + + +(test are-they-macro + (is (not (equal '(dig (a b)) (macroexpand-1 '(dig (a b)))))) + (is (not (equal '(odig (a b)) (macroexpand-1 '(odig (a b))))))) + + +(defmacro triple-var (x) + `((inject ,x) (inject ,x) (inject ,x))) + +(test correct-order-of-effects + (is (equal '(a 1 2 3) (let ((x 0)) + (dig (a (inject (incf x)) (inject (incf x)) (inject (incf x))))))) + (is (equal '(a (((1))) 2) + (let ((x 0)) + (dig (a ((((inject (incf x))))) (inject (incf x)))))))) + +(test macro-injects + (is (equal '(a (3 3 3)) (let ((x 3)) + (dig (a (macro-inject (triple-var x))))))) + (is (equal '(a (1 2 3)) (let ((x 0)) + (dig (a (macro-inject (triple-var (incf x)))))))) + (macrolet ((frob (form n) + (mapcar (lambda (x) + `(inject ,x)) + (make-list n :initial-element form))) + (frob1 (form) + `(frob ,form 4))) + (is (equal '(a (1 2 3 4 5)) + (let ((x 0)) + (dig (a (macro-inject (frob (incf x) 5))))))) + (is (equal '(a 1 2 3 4 5) + (let ((x 0)) + (dig (a (macro-splice (frob (incf x) 5))))))) + (is (equal '(a) + (let ((x 0)) + (declare (ignorable x)) + (dig (a (macro-splice (frob (incf x) 0))))))) + (is (equal '(a frob (incf x) 4) + (let ((x 0)) + (declare (ignorable x)) + (dig (a (macro-splice (frob1 (incf x)))))))) + (is (equal '(a 1 2 3 4) + (let ((x 0)) + (dig (a (macro-splice-all (frob1 (incf x)))))))))) + + +(quasiquote-2.0:enable-quasiquote-2.0) + +(test reader + (is (equal '(inject x) ',x)) + (is (equal '(inject 3 x) ',,,x)) + (is (equal '(splice x) ',@x)) + (is (equal '(splice 3 x) ',,,@x)) + (is (equal '(omacro-splice-all 4 x) ',,,,!oma@x)) + (is (equal '(inject 4 oma@x) ',,,,oma@x))) + +(test macro-splices + (macrolet ((splicer (x) + ``(splice ,x))) + (is (equal '(a 1 2 3) (let ((x '(1 2 3))) + `(a ,!m(splicer x))))))) + +(test repeated-splices + (is (equal '(a) `(a ,@nil ,@nil ,@nil ,@nil))) + (is (equal '(a b c d e f g) `(a ,@(list 'b 'c) ,@(list 'd 'e) ,@nil ,@(list 'f 'g))))) + + \ No newline at end of file diff --git a/third_party/lisp/restas.nix b/third_party/lisp/restas.nix new file mode 100644 index 000000000000..cf231286e79a --- /dev/null +++ b/third_party/lisp/restas.nix @@ -0,0 +1,41 @@ +{ depot, pkgs, ... }: + +let + + src = pkgs.fetchFromGitHub { + owner = "archimag"; + repo = "restas"; + rev = "81bbbab6b36f81f846f78e71232e9d3d15f6d952"; + sha256 = "00ng6jik1lwjw3bbxhijy8s0ml24lgm73liwrr01gcsb0r6wrjjn"; + }; + +in depot.nix.buildLisp.library { + name = "restas"; + deps = with depot.third_party.lisp; [ + cffi + hunchentoot + bordeaux-threads + routes + alexandria + data-sift + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "packages.lisp" + "special.lisp" + "declarations.lisp" + "errors.lisp" + "render.lisp" + "context.lisp" + "module.lisp" + "route.lisp" + "decorators.lisp" + "vhost.lisp" + "hunchentoot.lisp" + "policy.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/rfc2388.nix b/third_party/lisp/rfc2388.nix new file mode 100644 index 000000000000..6af55f927001 --- /dev/null +++ b/third_party/lisp/rfc2388.nix @@ -0,0 +1,17 @@ +# Implementation of RFC2388 (multipart/form-data) +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "jdz"; + repo = "rfc2388"; + rev = "591bcf7e77f2c222c43953a80f8c297751dc0c4e"; + sha256 = "0phh5n3clhl9ji8jaxrajidn22d3f0aq87mlbfkkxlnx2pnw694k"; +}; +in depot.nix.buildLisp.library { + name = "rfc2388"; + + srcs = map (f: src + ("/" + f)) [ + "packages.lisp" + "rfc2388.lisp" + ]; +} diff --git a/third_party/lisp/routes.nix b/third_party/lisp/routes.nix new file mode 100644 index 000000000000..a76912c651a9 --- /dev/null +++ b/third_party/lisp/routes.nix @@ -0,0 +1,38 @@ +{ depot, pkgs, ... }: + +let + + src = pkgs.applyPatches { + name = "routes-source"; + src = pkgs.fetchFromGitHub { + owner = "archimag"; + repo = "cl-routes"; + rev = "1b79e85aa653e1ec87e21ca745abe51547866fa9"; + sha256 = "1zpk3cp2v8hm50ppjl10yxr437vv4552r8hylvizglzrq2ibsbr1"; + }; + + patches = [ + (pkgs.fetchpatch { + name = "fix-build-with-ccl.patch"; + url = "https://github.com/archimag/cl-routes/commit/2296cdc316ef8e34310f2718b5d35a30040deee0.patch"; + sha256 = "007c19kmymalam3v6l6y2qzch8xs3xnphrcclk1jrpggvigcmhax"; + }) + ]; + }; + +in depot.nix.buildLisp.library { + name = "routes"; + + deps = with depot.third_party.lisp; [ + puri + iterate + split-sequence + ]; + + srcs = map (f: src + ("/src/" + f)) [ + "package.lisp" + "uri-template.lisp" + "route.lisp" + "mapper.lisp" + ]; +} diff --git a/third_party/lisp/s-sysdeps.nix b/third_party/lisp/s-sysdeps.nix new file mode 100644 index 000000000000..571eb147c724 --- /dev/null +++ b/third_party/lisp/s-sysdeps.nix @@ -0,0 +1,17 @@ +# A Common Lisp abstraction layer over platform dependent functionality. +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "svenvc"; + repo = "s-sysdeps"; + rev = "d28246b5dffef9e73a0e0e6cfbc4e878006fe34d"; + sha256 = "14b69b81yrxmjlvmm3lfxk04x5v7hqz4fql121334wh72czznfh9"; +}; +in depot.nix.buildLisp.library { + name = "s-sysdeps"; + + srcs = [ + "${src}/src/package.lisp" + "${src}/src/sysdeps.lisp" + ]; +} diff --git a/third_party/lisp/s-xml/.gitignore b/third_party/lisp/s-xml/.gitignore new file mode 100644 index 000000000000..40caffa8e257 --- /dev/null +++ b/third_party/lisp/s-xml/.gitignore @@ -0,0 +1,28 @@ +# CVS default ignores begin +tags +TAGS +.make.state +.nse_depinfo +*~ +#* +.#* +,* +_$* +*$ +*.old +*.bak +*.BAK +*.orig +*.rej +.del-* +*.a +*.olb +*.o +*.obj +*.so +*.exe +*.Z +*.elc +*.ln +core +# CVS default ignores end diff --git a/third_party/lisp/s-xml/ChangeLog b/third_party/lisp/s-xml/ChangeLog new file mode 100644 index 000000000000..ac196619c0aa --- /dev/null +++ b/third_party/lisp/s-xml/ChangeLog @@ -0,0 +1,66 @@ +2006-01-19 Sven Van Caekenberghe <svc@mac.com> + + * added a set of patches contributed by David Tolpin dvd@davidashen.net : we're now using char of type + Character and #\Null instead of null, read/unread instead of peek/read and some more declarations for + more efficiency - added hooks for customizing parsing attribute names and values + +2005-11-20 Sven Van Caekenberghe <svc@mac.com> + + * added xml prefix namespace as per REC-xml-names-19990114 (by Rudi Schlatte) + +2005-11-06 Sven Van Caekenberghe <svc@mac.com> + + * removed Debian packaging directory (on Luca's request) + * added CDATA support (patch contributed by Peter Van Eynde pvaneynd@mailworks.org) + +2005-08-30 Sven Van Caekenberghe <svc@mac.com> + + * added Debian packaging directory (contributed by Luca Capello luca@pca.it) + * added experimental XML namespace support + +2005-02-03 Sven Van Caekenberghe <svc@mac.com> + + * release 5 (cvs tag RELEASE_5) + * added :start and :end keywords to print-string-xml + * fixed a bug: in a tag containing whitespace, like <foo> </foo> the parser collapsed + and ingnored all whitespace and considered the tag to be empty! + this is now fixed and a unit test has been added + * cleaned up xml character escaping a bit: single quotes and all normal whitespace + (newline, return and tab) is preserved a unit test for this has been added + * IE doesn't understand the ' XML entity, so I've commented that out for now. + Also, using actual newlines for newlines is probably better than using #xA, + which won't get any end of line conversion by the server or user agent. + +June 2004 Sven Van Caekenberghe <svc@mac.com> + + * release 4 + * project moved to common-lisp.net, renamed to s-xml, + * added examples counter, tracer and remove-markup, improved documentation + +13 Jan 2004 Sven Van Caekenberghe <svc@mac.com> + + * release 3 + * added ASDF systems + * optimized print-string-xml + +10 Jun 2003 Sven Van Caekenberghe <svc@mac.com> + + * release 2 + * added echo-xml function: we are no longer taking the car when + the last seed is returned from start-parse-xml + +25 May 2003 Sven Van Caekenberghe <svc@mac.com> + + * release 1 + * first public release of working code + * tested on OpenMCL + * rewritten to be event-based, to improve efficiency and + to optionally use different DOM representations + * more documentation + +end of 2002 Sven Van Caekenberghe <svc@mac.com> + + * release 0 + * as part of an XML-RPC implementation + +$Id: ChangeLog,v 1.5 2005/11/20 14:24:33 scaekenberghe Exp $ diff --git a/third_party/lisp/s-xml/Makefile b/third_party/lisp/s-xml/Makefile new file mode 100644 index 000000000000..0c7292ea9fb5 --- /dev/null +++ b/third_party/lisp/s-xml/Makefile @@ -0,0 +1,35 @@ +# $Id: Makefile,v 1.2 2004/06/11 13:46:48 scaekenberghe Exp $ + +default: + @echo Possible targets: + @echo clean-openmcl --- remove all '*.dfsl' recursively + @echo clean-lw --- remove all '*.nfasl' recursively + @echo clean-emacs --- remove all '*~' recursively + @echo clean --- all of the above + +clean-openmcl: + find . -name "*.dfsl" | xargs rm + +clean-lw: + find . -name "*.nfasl" | xargs rm + +clean-emacs: + find . -name "*~" | xargs rm + +clean: clean-openmcl clean-lw clean-emacs + +# +# This can obviously only be done by a specific person in a very specific context ;-) +# + +PRJ=s-xml +ACCOUNT=scaekenberghe +CVSRT=:ext:$(ACCOUNT)@common-lisp.net:/project/$(PRJ)/cvsroot + +release: + rm -rf /tmp/$(PRJ) /tmp/public_html /tmp/$(PRJ).tgz /tmp/$(PRJ).tgz.asc + cd /tmp; cvs -d$(CVSRT) export -r HEAD $(PRJ); cvs -d$(CVSRT) export -r HEAD public_html + mv /tmp/public_html /tmp/$(PRJ)/doc + cd /tmp; gnutar cvfz $(PRJ).tgz $(PRJ); gpg -a -b $(PRJ).tgz + scp /tmp/$(PRJ).tgz $(ACCOUNT)@common-lisp.net:/project/$(PRJ)/public_html + scp /tmp/$(PRJ).tgz.asc $(ACCOUNT)@common-lisp.net:/project/$(PRJ)/public_html diff --git a/third_party/lisp/s-xml/default.nix b/third_party/lisp/s-xml/default.nix new file mode 100644 index 000000000000..82b6317f372c --- /dev/null +++ b/third_party/lisp/s-xml/default.nix @@ -0,0 +1,17 @@ +# XML serialiser for Common Lisp. +# +# This system was imported from a Quicklisp tarball at 's-xml-20150608'. +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "s-xml"; + + srcs = [ + ./src/package.lisp + ./src/xml.lisp + ./src/dom.lisp + ./src/lxml-dom.lisp + ./src/sxml-dom.lisp + ./src/xml-struct-dom.lisp + ]; +} diff --git a/third_party/lisp/s-xml/examples/counter.lisp b/third_party/lisp/s-xml/examples/counter.lisp new file mode 100644 index 000000000000..b26453e6ea66 --- /dev/null +++ b/third_party/lisp/s-xml/examples/counter.lisp @@ -0,0 +1,47 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: counter.lisp,v 1.2 2004/06/11 11:14:43 scaekenberghe Exp $ +;;;; +;;;; A simple SSAX counter example that can be used as a performance test +;;;; +;;;; Copyright (C) 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(defclass count-xml-seed () + ((elements :initform 0) + (attributes :initform 0) + (characters :initform 0))) + +(defun count-xml-new-element-hook (name attributes seed) + (declare (ignore name)) + (incf (slot-value seed 'elements)) + (incf (slot-value seed 'attributes) (length attributes)) + seed) + +(defun count-xml-text-hook (string seed) + (incf (slot-value seed 'characters) (length string)) + seed) + +(defun count-xml (in) + "Parse a toplevel XML element from stream in, counting elements, attributes and characters" + (start-parse-xml in + (make-instance 'xml-parser-state + :seed (make-instance 'count-xml-seed) + :new-element-hook #'count-xml-new-element-hook + :text-hook #'count-xml-text-hook))) + +(defun count-xml-file (pathname) + "Parse XMl from the file at pathname, counting elements, attributes and characters" + (with-open-file (in pathname) + (let ((result (count-xml in))) + (with-slots (elements attributes characters) result + (format t + "~a contains ~d XML elements, ~d attributes and ~d characters.~%" + pathname elements attributes characters))))) + +;;;; eof diff --git a/third_party/lisp/s-xml/examples/echo.lisp b/third_party/lisp/s-xml/examples/echo.lisp new file mode 100644 index 000000000000..a0befe2cbbbb --- /dev/null +++ b/third_party/lisp/s-xml/examples/echo.lisp @@ -0,0 +1,64 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: echo.lisp,v 1.1 2005/08/17 13:44:30 scaekenberghe Exp $ +;;;; +;;;; A simple example as well as a useful tool: parse, echo and pretty print XML +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(defun indent (stream count) + (loop :repeat (* count 2) :do (write-char #\space stream))) + +(defclass echo-xml-seed () + ((stream :initarg :stream) + (level :initarg :level :initform 0))) + +#+NIL +(defmethod print-object ((seed echo-xml-seed) stream) + (with-slots (stream level) seed + (print-unreadable-object (seed stream :type t) + (format stream "level=~d" level)))) + +(defun echo-xml-new-element-hook (name attributes seed) + (with-slots (stream level) seed + (indent stream level) + (format stream "<~a" name) + (dolist (attribute (reverse attributes)) + (format stream " ~a=\'" (car attribute)) + (print-string-xml (cdr attribute) stream) + (write-char #\' stream)) + (format stream ">~%") + (incf level) + seed)) + +(defun echo-xml-finish-element-hook (name attributes parent-seed seed) + (declare (ignore attributes parent-seed)) + (with-slots (stream level) seed + (decf level) + (indent stream level) + (format stream "</~a>~%" name) + seed)) + +(defun echo-xml-text-hook (string seed) + (with-slots (stream level) seed + (indent stream level) + (print-string-xml string stream) + (terpri stream) + seed)) + +(defun echo-xml (in out) + "Parse a toplevel XML element from stream in, echoing and pretty printing the result to stream out" + (start-parse-xml in + (make-instance 'xml-parser-state + :seed (make-instance 'echo-xml-seed :stream out) + :new-element-hook #'echo-xml-new-element-hook + :finish-element-hook #'echo-xml-finish-element-hook + :text-hook #'echo-xml-text-hook))) + +;;;; eof diff --git a/third_party/lisp/s-xml/examples/remove-markup.lisp b/third_party/lisp/s-xml/examples/remove-markup.lisp new file mode 100644 index 000000000000..41d858b4a8c5 --- /dev/null +++ b/third_party/lisp/s-xml/examples/remove-markup.lisp @@ -0,0 +1,21 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: remove-markup.lisp,v 1.1 2004/06/11 11:14:43 scaekenberghe Exp $ +;;;; +;;;; Remove markup from an XML document using the SSAX interface +;;;; +;;;; Copyright (C) 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(defun remove-xml-markup (in) + (let* ((state (make-instance 'xml-parser-state + :text-hook #'(lambda (string seed) (cons string seed)))) + (result (start-parse-xml in state))) + (apply #'concatenate 'string (nreverse result)))) + +;;;; eof \ No newline at end of file diff --git a/third_party/lisp/s-xml/examples/tracer.lisp b/third_party/lisp/s-xml/examples/tracer.lisp new file mode 100644 index 000000000000..c8a3eaec1f2b --- /dev/null +++ b/third_party/lisp/s-xml/examples/tracer.lisp @@ -0,0 +1,57 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: tracer.lisp,v 1.2 2004/06/11 11:14:43 scaekenberghe Exp $ +;;;; +;;;; A simple SSAX tracer example that can be used to understand how the hooks are called +;;;; +;;;; Copyright (C) 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(defun trace-xml-log (level msg &rest args) + (indent *standard-output* level) + (apply #'format *standard-output* msg args) + (terpri *standard-output*)) + +(defun trace-xml-new-element-hook (name attributes seed) + (let ((new-seed (cons (1+ (car seed)) (1+ (cdr seed))))) + (trace-xml-log (car seed) + "(new-element :name ~s :attributes ~:[()~;~:*~s~] :seed ~s) => ~s" + name attributes seed new-seed) + new-seed)) + +(defun trace-xml-finish-element-hook (name attributes parent-seed seed) + (let ((new-seed (cons (1- (car seed)) (1+ (cdr seed))))) + (trace-xml-log (car parent-seed) + "(finish-element :name ~s :attributes ~:[()~;~:*~s~] :parent-seed ~s :seed ~s) => ~s" + name attributes parent-seed seed new-seed) + new-seed)) + +(defun trace-xml-text-hook (string seed) + (let ((new-seed (cons (car seed) (1+ (cdr seed))))) + (trace-xml-log (car seed) + "(text :string ~s :seed ~s) => ~s" + string seed new-seed) + new-seed)) + +(defun trace-xml (in) + "Parse and trace a toplevel XML element from stream in" + (start-parse-xml in + (make-instance 'xml-parser-state + :seed (cons 0 0) + ;; seed car is xml element nesting level + ;; seed cdr is ever increasing from element to element + :new-element-hook #'trace-xml-new-element-hook + :finish-element-hook #'trace-xml-finish-element-hook + :text-hook #'trace-xml-text-hook))) + +(defun trace-xml-file (pathname) + "Parse and trace XMl from the file at pathname" + (with-open-file (in pathname) + (trace-xml in))) + +;;;; eof diff --git a/third_party/lisp/s-xml/s-xml.asd b/third_party/lisp/s-xml/s-xml.asd new file mode 100644 index 000000000000..651f5e5844c2 --- /dev/null +++ b/third_party/lisp/s-xml/s-xml.asd @@ -0,0 +1,49 @@ +;;;; -*- Mode: LISP -*- +;;;; +;;;; $Id: s-xml.asd,v 1.2 2005/12/14 21:49:04 scaekenberghe Exp $ +;;;; +;;;; The S-XML ASDF system definition +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :asdf) + +(defsystem :s-xml + :name "S-XML" + :author "Sven Van Caekenberghe <svc@mac.com>" + :version "3" + :maintainer "Sven Van Caekenberghe <svc@mac.com>, Brian Mastenbrook <>, Rudi Schlatte <>" + :licence "Lisp Lesser General Public License (LLGPL)" + :description "Simple Common Lisp XML Parser" + :long-description "S-XML is a Common Lisp implementation of a simple XML parser, with a SAX-like and DOM interface" + + :components + ((:module + :src + :components ((:file "package") + (:file "xml" :depends-on ("package")) + (:file "dom" :depends-on ("package" "xml")) + (:file "lxml-dom" :depends-on ("dom")) + (:file "sxml-dom" :depends-on ("dom")) + (:file "xml-struct-dom" :depends-on ("dom")))))) + +(defsystem :s-xml.test + :depends-on (:s-xml) + :components ((:module :test + :components ((:file "test-xml") + (:file "test-xml-struct-dom") + (:file "test-lxml-dom") + (:file "test-sxml-dom"))))) + +(defsystem :s-xml.examples + :depends-on (:s-xml) + :components ((:module :examples + :components ((:file "counter") + (:file "echo") + (:file "remove-markup") + (:file "tracer"))))) +;;;; eof diff --git a/third_party/lisp/s-xml/src/dom.lisp b/third_party/lisp/s-xml/src/dom.lisp new file mode 100644 index 000000000000..74d1c371db22 --- /dev/null +++ b/third_party/lisp/s-xml/src/dom.lisp @@ -0,0 +1,75 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: dom.lisp,v 1.1.1.1 2004/06/07 18:49:56 scaekenberghe Exp $ +;;;; +;;;; This is the generic simple DOM parser and printer interface. +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +;;; top level DOM parser interface + +(defgeneric parse-xml-dom (stream output-type) + (:documentation "Parse a character stream as XML and generate a DOM of output-type")) + +(defun parse-xml (stream &key (output-type :lxml)) + "Parse a character stream as XML and generate a DOM of output-type, defaulting to :lxml" + (parse-xml-dom stream output-type)) + +(defun parse-xml-string (string &key (output-type :lxml)) + "Parse a string as XML and generate a DOM of output-type, defaulting to :lxml" + (with-input-from-string (stream string) + (parse-xml-dom stream output-type))) + +(defun parse-xml-file (filename &key (output-type :lxml)) + "Parse a character file as XML and generate a DOM of output-type, defaulting to :lxml" + (with-open-file (in filename :direction :input) + (parse-xml-dom in output-type))) + +;;; top level DOM printer interface + +(defgeneric print-xml-dom (dom input-type stream pretty level) + (:documentation "Generate XML output on a character stream from a DOM of input-type, optionally pretty printing using level")) + +(defun print-xml (dom &key (stream t) (pretty nil) (input-type :lxml) (header)) + "Generate XML output on a character stream (t by default) from a DOM of input-type (:lxml by default), optionally pretty printing (off by default), or adding a header (none by default)" + (when header (format stream header)) + (when pretty (terpri stream)) + (print-xml-dom dom input-type stream pretty 1)) + +(defun print-xml-string (dom &key (pretty nil) (input-type :lxml)) + "Generate XML output to a string from a DOM of input-type (:lxml by default), optionally pretty printing (off by default)" + (with-output-to-string (stream) + (print-xml dom :stream stream :pretty pretty :input-type input-type))) + +;;; shared/common support functions + +(defun print-spaces (n stream &optional (preceding-newline t)) + (when preceding-newline + (terpri stream)) + (loop :repeat n + :do (write-char #\Space stream))) + +(defun print-solitary-tag (tag stream) + (write-char #\< stream) + (print-identifier tag stream) + (write-string "/>" stream)) + +(defun print-closing-tag (tag stream) + (write-string "</" stream) + (print-identifier tag stream) + (write-char #\> stream)) + +(defun print-attribute (name value stream) + (write-char #\space stream) + (print-identifier name stream t) + (write-string "=\"" stream) + (print-string-xml value stream) + (write-char #\" stream)) + +;;;; eof diff --git a/third_party/lisp/s-xml/src/lxml-dom.lisp b/third_party/lisp/s-xml/src/lxml-dom.lisp new file mode 100644 index 000000000000..d43df6cf8171 --- /dev/null +++ b/third_party/lisp/s-xml/src/lxml-dom.lisp @@ -0,0 +1,83 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: lxml-dom.lisp,v 1.5 2005/09/20 09:57:44 scaekenberghe Exp $ +;;;; +;;;; LXML implementation of the generic DOM parser and printer. +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +;;; the lxml hooks to generate lxml + +(defun lxml-new-element-hook (name attributes seed) + (declare (ignore name attributes seed)) + '()) + +(defun lxml-finish-element-hook (name attributes parent-seed seed) + (let ((xml-element + (cond ((and (null seed) (null attributes)) + name) + (attributes + `((,name ,@(let (list) + (dolist (attribute attributes list) + (push (cdr attribute) list) + (push (car attribute) list)))) + ,@(nreverse seed))) + (t + `(,name ,@(nreverse seed)))))) + (cons xml-element parent-seed))) + +(defun lxml-text-hook (string seed) + (cons string seed)) + +;;; standard DOM interfaces + +(defmethod parse-xml-dom (stream (output-type (eql :lxml))) + (car (start-parse-xml stream + (make-instance 'xml-parser-state + :new-element-hook #'lxml-new-element-hook + :finish-element-hook #'lxml-finish-element-hook + :text-hook #'lxml-text-hook)))) + +(defun plist->alist (plist) + (when plist + (cons (cons (first plist) (second plist)) + (plist->alist (rest (rest plist)))))) + +(defmethod print-xml-dom (dom (input-type (eql :lxml)) stream pretty level) + (declare (special *namespaces*)) + (cond ((symbolp dom) (print-solitary-tag dom stream)) + ((stringp dom) (print-string-xml dom stream)) + ((consp dom) + (let (tag attributes) + (cond ((symbolp (first dom)) (setf tag (first dom))) + ((consp (first dom)) (setf tag (first (first dom)) + attributes (plist->alist (rest (first dom))))) + (t (error "Input not recognized as LXML ~s" dom))) + (let ((*namespaces* (extend-namespaces attributes *namespaces*))) + (write-char #\< stream) + (print-identifier tag stream) + (loop :for (name . value) :in attributes + :do (print-attribute name value stream)) + (if (rest dom) + (let ((children (rest dom))) + (write-char #\> stream) + (if (and (= (length children) 1) (stringp (first children))) + (print-string-xml (first children) stream) + (progn + (dolist (child children) + (when pretty (print-spaces (* 2 level) stream)) + (if (stringp child) + (print-string-xml child stream) + (print-xml-dom child input-type stream pretty (1+ level)))) + (when pretty (print-spaces (* 2 (1- level)) stream)))) + (print-closing-tag tag stream)) + (write-string "/>" stream))))) + (t (error "Input not recognized as LXML ~s" dom)))) + +;;;; eof \ No newline at end of file diff --git a/third_party/lisp/s-xml/src/package.lisp b/third_party/lisp/s-xml/src/package.lisp new file mode 100644 index 000000000000..f90f0f49a166 --- /dev/null +++ b/third_party/lisp/s-xml/src/package.lisp @@ -0,0 +1,46 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: package.lisp,v 1.7 2006/01/19 20:00:06 scaekenberghe Exp $ +;;;; +;;;; This is a Common Lisp implementation of a very basic XML parser. +;;;; The parser is non-validating. +;;;; The API into the parser is pure functional parser hook model that comes from SSAX, +;;;; see also http://pobox.com/~oleg/ftp/Scheme/xml.html or http://ssax.sourceforge.net +;;;; Different DOM models are provided, an XSML, an LXML and a xml-element struct based one. +;;;; +;;;; Copyright (C) 2002, 2003, 2004, 2005, 2006 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(defpackage s-xml + (:use common-lisp) + (:export + ;; main parser interface + #:start-parse-xml + #:print-string-xml + #:xml-parser-error #:xml-parser-error-message #:xml-parser-error-args #:xml-parser-error-stream + #:xml-parser-state #:get-entities #:get-seed + #:get-new-element-hook #:get-finish-element-hook #:get-text-hook + ;; callbacks + #:*attribute-name-parser* + #:*attribute-value-parser* + #:parse-attribute-name + #:parse-attribute-value + ;; dom parser and printer + #:parse-xml-dom #:parse-xml #:parse-xml-string #:parse-xml-file + #:print-xml-dom #:print-xml #:print-xml-string + ;; xml-element structure + #:make-xml-element #:xml-element-children #:xml-element-name + #:xml-element-attribute #:xml-element-attributes + #:xml-element-p #:new-xml-element #:first-xml-element-child + ;; namespaces + #:*ignore-namespaces* #:*local-namespace* #:*namespaces* + #:*require-existing-symbols* #:*auto-export-symbols* #:*auto-create-namespace-packages* + #:find-namespace #:register-namespace #:get-prefix #:get-uri #:get-package + #:resolve-identifier #:extend-namespaces #:print-identifier #:split-identifier) + (:documentation + "A simple XML parser with an efficient, purely functional, event-based interface as well as a DOM interface")) + +;;;; eof diff --git a/third_party/lisp/s-xml/src/sxml-dom.lisp b/third_party/lisp/s-xml/src/sxml-dom.lisp new file mode 100644 index 000000000000..c9e0f9e0db4e --- /dev/null +++ b/third_party/lisp/s-xml/src/sxml-dom.lisp @@ -0,0 +1,76 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: sxml-dom.lisp,v 1.4 2005/09/20 09:57:48 scaekenberghe Exp $ +;;;; +;;;; LXML implementation of the generic DOM parser and printer. +;;;; +;;;; Copyright (C) 2003, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +;;; the sxml hooks to generate sxml + +(defun sxml-new-element-hook (name attributes seed) + (declare (ignore name attributes seed)) + '()) + +(defun sxml-finish-element-hook (name attributes parent-seed seed) + (let ((xml-element (append (list name) + (when attributes + (list (let (list) + (dolist (attribute attributes (cons :@ list)) + (push (list (car attribute) (cdr attribute)) list))))) + (nreverse seed)))) + (cons xml-element parent-seed))) + +(defun sxml-text-hook (string seed) + (cons string seed)) + +;;; the standard DOM interfaces + +(defmethod parse-xml-dom (stream (output-type (eql :sxml))) + (car (start-parse-xml stream + (make-instance 'xml-parser-state + :new-element-hook #'sxml-new-element-hook + :finish-element-hook #'sxml-finish-element-hook + :text-hook #'sxml-text-hook)))) + +(defmethod print-xml-dom (dom (input-type (eql :sxml)) stream pretty level) + (declare (special *namespaces*)) + (cond ((stringp dom) (print-string-xml dom stream)) + ((consp dom) + (let ((tag (first dom)) + attributes + children) + (if (and (consp (second dom)) (eq (first (second dom)) :@)) + (setf attributes (rest (second dom)) + children (rest (rest dom))) + (setf children (rest dom))) + (let ((*namespaces* (extend-namespaces (loop :for (name value) :in attributes + :collect (cons name value)) + *namespaces*))) + (write-char #\< stream) + (print-identifier tag stream) + (loop :for (name value) :in attributes + :do (print-attribute name value stream)) + (if children + (progn + (write-char #\> stream) + (if (and (= (length children) 1) (stringp (first children))) + (print-string-xml (first children) stream) + (progn + (dolist (child children) + (when pretty (print-spaces (* 2 level) stream)) + (if (stringp child) + (print-string-xml child stream) + (print-xml-dom child input-type stream pretty (1+ level)))) + (when pretty (print-spaces (* 2 (1- level)) stream)))) + (print-closing-tag tag stream)) + (write-string "/>" stream))))) + (t (error "Input not recognized as SXML ~s" dom)))) + +;;;; eof diff --git a/third_party/lisp/s-xml/src/xml-struct-dom.lisp b/third_party/lisp/s-xml/src/xml-struct-dom.lisp new file mode 100644 index 000000000000..70373889152f --- /dev/null +++ b/third_party/lisp/s-xml/src/xml-struct-dom.lisp @@ -0,0 +1,125 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: xml-struct-dom.lisp,v 1.2 2005/08/29 15:01:47 scaekenberghe Exp $ +;;;; +;;;; XML-STRUCT implementation of the generic DOM parser and printer. +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +;;; xml-element struct datastructure and API + +(defstruct xml-element + name ; :tag-name + attributes ; a assoc list of (:attribute-name . "attribute-value") + children ; a list of children/content either text strings or xml-elements + ) + +(setf (documentation 'xml-element-p 'function) + "Return T when the argument is an xml-element struct" + (documentation 'xml-element-attributes 'function) + "Return the alist of attribute names and values dotted pairs from an xml-element struct" + (documentation 'xml-element-children 'function) + "Return the list of children from an xml-element struct" + (documentation 'xml-element-name 'function) + "Return the name from an xml-element struct" + (documentation 'make-xml-element 'function) + "Make and return a new xml-element struct") + +(defun xml-element-attribute (xml-element key) + "Return the string value of the attribute with name the keyword :key + of xml-element if any, return null if not found" + (let ((pair (assoc key (xml-element-attributes xml-element) :test #'eq))) + (when pair (cdr pair)))) + +(defun (setf xml-element-attribute) (value xml-element key) + "Set the string value of the attribute with name the keyword :key of + xml-element, creating a new attribute if necessary or overwriting an + existing one, returning the value" + (let ((attributes (xml-element-attributes xml-element))) + (if (null attributes) + (push (cons key value) (xml-element-attributes xml-element)) + (let ((pair (assoc key attributes :test #'eq))) + (if pair + (setf (cdr pair) value) + (push (cons key value) (xml-element-attributes xml-element))))) + value)) + +(defun new-xml-element (name &rest children) + "Make a new xml-element with name and children" + (make-xml-element :name name :children children)) + +(defun first-xml-element-child (xml-element) + "Get the first child of an xml-element" + (first (xml-element-children xml-element))) + +(defun xml-equal (xml-1 xml-2) + (and (xml-element-p xml-1) + (xml-element-p xml-2) + (eq (xml-element-name xml-1) + (xml-element-name xml-2)) + (equal (xml-element-attributes xml-1) + (xml-element-attributes xml-2)) + (reduce #'(lambda (&optional (x t) (y t)) (and x y)) + (mapcar #'(lambda (x y) + (or (and (stringp x) (stringp y) (string= x y)) + (xml-equal x y))) + (xml-element-children xml-1) + (xml-element-children xml-2))))) + +;;; printing xml structures + +(defmethod print-xml-dom (xml-element (input-type (eql :xml-struct)) stream pretty level) + (declare (special *namespaces*)) + (let ((*namespaces* (extend-namespaces (xml-element-attributes xml-element) + *namespaces*))) + (write-char #\< stream) + (print-identifier (xml-element-name xml-element) stream) + (loop :for (name . value) :in (xml-element-attributes xml-element) + :do (print-attribute name value stream)) + (let ((children (xml-element-children xml-element))) + (if children + (progn + (write-char #\> stream) + (if (and (= (length children) 1) (stringp (first children))) + (print-string-xml (first children) stream) + (progn + (dolist (child children) + (when pretty (print-spaces (* 2 level) stream)) + (if (stringp child) + (print-string-xml child stream) + (print-xml-dom child input-type stream pretty (1+ level)))) + (when pretty (print-spaces (* 2 (1- level)) stream)))) + (print-closing-tag (xml-element-name xml-element) stream)) + (write-string "/>" stream))))) + +;;; the standard hooks to generate xml-element structs + +(defun standard-new-element-hook (name attributes seed) + (declare (ignore name attributes seed)) + '()) + +(defun standard-finish-element-hook (name attributes parent-seed seed) + (let ((xml-element (make-xml-element :name name + :attributes attributes + :children (nreverse seed)))) + (cons xml-element parent-seed))) + +(defun standard-text-hook (string seed) + (cons string seed)) + +;;; top level standard parser interfaces + +(defmethod parse-xml-dom (stream (output-type (eql :xml-struct))) + (car (start-parse-xml stream + (make-instance 'xml-parser-state + :new-element-hook #'standard-new-element-hook + :finish-element-hook #'standard-finish-element-hook + :text-hook #'standard-text-hook)))) + +;;;; eof diff --git a/third_party/lisp/s-xml/src/xml.lisp b/third_party/lisp/s-xml/src/xml.lisp new file mode 100644 index 000000000000..8a2076985a49 --- /dev/null +++ b/third_party/lisp/s-xml/src/xml.lisp @@ -0,0 +1,702 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: xml.lisp,v 1.15 2006/01/19 20:00:06 scaekenberghe Exp $ +;;;; +;;;; This is a Common Lisp implementation of a basic but usable XML parser. +;;;; The parser is non-validating and not complete (no PI). +;;;; Namespace and entities are handled. +;;;; The API into the parser is a pure functional parser hook model that comes from SSAX, +;;;; see also http://pobox.com/~oleg/ftp/Scheme/xml.html or http://ssax.sourceforge.net +;;;; Different DOM models are provided, an XSML, an LXML and a xml-element struct based one. +;;;; +;;;; Copyright (C) 2002, 2003, 2004, 2005, 2006 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +;;; (tazjin): moved up here because something was wonky with the +;;; definition order +(defvar *ignore-namespaces* nil + "When t, namespaces are ignored like in the old version of S-XML") + +;;; error reporting + +(define-condition xml-parser-error (error) + ((message :initarg :message :reader xml-parser-error-message) + (args :initarg :args :reader xml-parser-error-args) + (stream :initarg :stream :reader xml-parser-error-stream :initform nil)) + (:report (lambda (condition stream) + (format stream + "XML parser ~?~@[ near stream position ~d~]." + (xml-parser-error-message condition) + (xml-parser-error-args condition) + (and (xml-parser-error-stream condition) + (file-position (xml-parser-error-stream condition)))))) + (:documentation "Thrown by the XML parser to indicate errorneous input")) + +(setf (documentation 'xml-parser-error-message 'function) + "Get the message from an XML parser error" + (documentation 'xml-parser-error-args 'function) + "Get the error arguments from an XML parser error" + (documentation 'xml-parser-error-stream 'function) + "Get the stream from an XML parser error") + +(defun parser-error (message &optional args stream) + (make-condition 'xml-parser-error + :message message + :args args + :stream stream)) + +;; attribute parsing hooks +;; this is a bit complicated, refer to the mailing lists for a more detailed explanation + +(defun parse-attribute-name (string) + "Default parser for the attribute name" + (declare (special *namespaces*)) + (resolve-identifier string *namespaces* t)) + +(defun parse-attribute-value (name string) + "Default parser for the attribute value" + (declare (ignore name) + (special *ignore-namespace*)) + (if *ignore-namespaces* + (copy-seq string) + string)) + +(defparameter *attribute-name-parser* #'parse-attribute-name + "Called to compute interned attribute name from a buffer that will be reused") + +(defparameter *attribute-value-parser* #'parse-attribute-value + "Called to compute an element of an attribute list from a buffer that will be reused") + +;;; utilities + +(defun whitespace-char-p (char) + "Is char an XML whitespace character ?" + (declare (type character char)) + (or (char= char #\space) + (char= char #\tab) + (char= char #\return) + (char= char #\linefeed))) + +(defun identifier-char-p (char) + "Is char an XML identifier character ?" + (declare (type character char)) + (or (and (char<= #\A char) (char<= char #\Z)) + (and (char<= #\a char) (char<= char #\z)) + (and (char<= #\0 char) (char<= char #\9)) + (char= char #\-) + (char= char #\_) + (char= char #\.) + (char= char #\:))) + +(defun skip-whitespace (stream) + "Skip over XML whitespace in stream, return first non-whitespace + character which was peeked but not read, return nil on eof" + (loop + (let ((char (peek-char nil stream nil #\Null))) + (declare (type character char)) + (if (whitespace-char-p char) + (read-char stream) + (return char))))) + +(defun make-extendable-string (&optional (size 10)) + "Make an extendable string which is a one-dimensional character + array which is adjustable and has a fill pointer" + (make-array size + :element-type 'character + :adjustable t + :fill-pointer 0)) + +(defun print-string-xml (string stream &key (start 0) end) + "Write the characters of string to stream using basic XML conventions" + (loop for offset upfrom start below (or end (length string)) + for char = (char string offset) + do (case char + (#\& (write-string "&" stream)) + (#\< (write-string "<" stream)) + (#\> (write-string ">" stream)) + (#\" (write-string """ stream)) + ((#\newline #\return #\tab) (write-char char stream)) + (t (if (and (<= 32 (char-code char)) + (<= (char-code char) 126)) + (write-char char stream) + (progn + (write-string "&#x" stream) + (write (char-code char) :stream stream :base 16) + (write-char #\; stream))))))) + +(defun make-standard-entities () + "A hashtable mapping XML entity names to their replacement strings, + filled with the standard set" + (let ((entities (make-hash-table :test #'equal))) + (setf (gethash "amp" entities) (string #\&) + (gethash "quot" entities) (string #\") + (gethash "apos" entities) (string #\') + (gethash "lt" entities) (string #\<) + (gethash "gt" entities) (string #\>) + (gethash "nbsp" entities) (string #\space)) + entities)) + +(defun resolve-entity (stream extendable-string entities entity) + "Read and resolve an XML entity from stream, positioned after the '&' entity marker, + accepting &name; &#DEC; and &#xHEX; formats, + destructively modifying string, which is also returned, + destructively modifying entity, incorrect entity formats result in errors" + (declare (type (vector character) entity)) + (loop + (let ((char (read-char stream nil #\Null))) + (declare (type character char)) + (cond ((char= char #\Null) (error (parser-error "encountered eof before end of entity"))) + ((char= #\; char) (return)) + (t (vector-push-extend char entity))))) + (if (char= (char entity 0) #\#) + (let ((code (if (char= (char entity 1) #\x) + (parse-integer entity :start 2 :radix 16 :junk-allowed t) + (parse-integer entity :start 1 :radix 10 :junk-allowed t)))) + (when (null code) + (error (parser-error "encountered incorrect entity &~s;" (list entity) stream))) + (vector-push-extend (code-char code) extendable-string)) + (let ((value (gethash entity entities))) + (if value + (loop :for char :across value + :do (vector-push-extend char extendable-string)) + (error (parser-error "encountered unknown entity &~s;" (list entity) stream))))) + extendable-string) + +;;; namespace support + +(defclass xml-namespace () + ((uri :documentation "The URI used to identify this namespace" + :accessor get-uri + :initarg :uri) + (prefix :documentation "The preferred prefix assigned to this namespace" + :accessor get-prefix + :initarg :prefix + :initform nil) + (package :documentation "The Common Lisp package where this namespace's symbols are interned" + :accessor get-package + :initarg :package + :initform nil)) + (:documentation "Describes an XML namespace and how it is handled")) + +(setf (documentation 'get-uri 'function) + "The URI used to identify this namespace" + (documentation 'get-prefix 'function) + "The preferred prefix assigned to this namespace" + (documentation 'get-package 'function) + "The Common Lisp package where this namespace's symbols are interned") + +(defmethod print-object ((object xml-namespace) stream) + (print-unreadable-object (object stream :type t :identity t) + (format stream "~A - ~A" (get-prefix object) (get-uri object)))) + +(defvar *local-namespace* (make-instance 'xml-namespace + :uri "local" + :prefix "" + :package (find-package :keyword)) + "The local (global default) XML namespace") + +(defvar *xml-namespace* (make-instance 'xml-namespace + :uri "http://www.w3.org/XML/1998/namespace" + :prefix "xml" + :package (or (find-package :xml) + (make-package :xml :nicknames '("XML")))) + "REC-xml-names-19990114 says the prefix xml is bound to the namespace http://www.w3.org/XML/1998/namespace.") + +(defvar *known-namespaces* (list *local-namespace* *xml-namespace*) + "The list of known/defined namespaces") + +(defvar *namespaces* `(("xml" . ,*xml-namespace*) ("" . ,*local-namespace*)) + "Ordered list of (prefix . XML-namespace) bindings currently in effect - special variable") + +(defun find-namespace (uri) + "Find a registered XML namespace identified by uri" + (find uri *known-namespaces* :key #'get-uri :test #'string-equal)) + +(defun register-namespace (uri prefix package) + "Register a new or redefine an existing XML namespace defined by uri with prefix and package" + (let ((namespace (find-namespace uri))) + (if namespace + (setf (get-prefix namespace) prefix + (get-package namespace) (find-package package)) + (push (setf namespace (make-instance 'xml-namespace + :uri uri + :prefix prefix + :package (find-package package))) + *known-namespaces*)) + namespace)) + +(defun find-namespace-binding (prefix namespaces) + "Find the XML namespace currently bound to prefix in the namespaces bindings" + (cdr (assoc prefix namespaces :test #'string-equal))) + +(defun split-identifier (identifier) + "Split an identifier 'prefix:name' and return (values prefix name)" + (when (symbolp identifier) + (setf identifier (symbol-name identifier))) + (let ((colon-position (position #\: identifier :test #'char=))) + (if colon-position + (values (subseq identifier 0 colon-position) + (subseq identifier (1+ colon-position))) + (values nil identifier)))) + +(defvar *require-existing-symbols* nil + "If t, each XML identifier must exist as symbol already") + +(defvar *auto-export-symbols* t + "If t, export newly interned symbols form their packages") + +(defun resolve-identifier (identifier namespaces &optional as-attribute) + "Resolve the string identifier in the list of namespace bindings" + (if *ignore-namespaces* + (intern identifier :keyword) + (flet ((intern-symbol (string package) ; intern string as a symbol in package + (if *require-existing-symbols* + (let ((symbol (find-symbol string package))) + (or symbol + (error "Symbol ~s does not exist in ~s" string package))) + (let ((symbol (intern string package))) + (when (and *auto-export-symbols* + (not (eql package (find-package :keyword)))) + (export symbol package)) + symbol)))) + (multiple-value-bind (prefix name) + (split-identifier identifier) + (if (or (null prefix) (string= prefix "xmlns")) + (if as-attribute + (intern (if (string= prefix "xmlns") identifier name) (get-package *local-namespace*)) + (let ((default-namespace (find-namespace-binding "" namespaces))) + (intern-symbol name (get-package default-namespace)))) + (let ((namespace (find-namespace-binding prefix namespaces))) + (if namespace + (intern-symbol name (get-package namespace)) + (error "namespace not found for prefix ~s" prefix)))))))) + +(defvar *auto-create-namespace-packages* t + "If t, new packages will be created for namespaces, if needed, named by the prefix") + +(defun new-namespace (uri &optional prefix) + "Register a new namespace for uri and prefix, creating a package if necessary" + (if prefix + (register-namespace uri + prefix + (or (find-package prefix) + (if *auto-create-namespace-packages* + (make-package prefix :nicknames `(,(string-upcase prefix))) + (error "Cannot find or create package ~s" prefix)))) + (let ((unique-name (loop :for i :upfrom 0 + :do (let ((name (format nil "ns-~d" i))) + (when (not (find-package name)) + (return name)))))) + (register-namespace uri + unique-name + (if *auto-create-namespace-packages* + (make-package (string-upcase unique-name) :nicknames `(,unique-name)) + (error "Cannot create package ~s" unique-name)))))) + +(defun extend-namespaces (attributes namespaces) + "Given possible 'xmlns[:prefix]' attributes, extend the namespaces bindings" + (unless *ignore-namespaces* + (let (default-namespace-uri) + (loop :for (key . value) :in attributes + :do (if (string= key "xmlns") + (setf default-namespace-uri value) + (multiple-value-bind (prefix name) + (split-identifier key) + (when (string= prefix "xmlns") + (let* ((uri value) + (prefix name) + (namespace (find-namespace uri))) + (unless namespace + (setf namespace (new-namespace uri prefix))) + (push `(,prefix . ,namespace) namespaces)))))) + (when default-namespace-uri + (let ((namespace (find-namespace default-namespace-uri))) + (unless namespace + (setf namespace (new-namespace default-namespace-uri))) + (push `("" . ,namespace) namespaces))))) + namespaces) + +(defun print-identifier (identifier stream &optional as-attribute) + "Print identifier on stream using namespace conventions" + (declare (ignore as-attribute) (special *namespaces*)) + (if *ignore-namespaces* + (princ identifier stream) + (if (symbolp identifier) + (let ((package (symbol-package identifier)) + (name (symbol-name identifier))) + (let* ((namespace (find package *known-namespaces* :key #'get-package)) + (prefix (or (car (find namespace *namespaces* :key #'cdr)) + (get-prefix namespace)))) + (if (string= prefix "") + (princ name stream) + (format stream "~a:~a" prefix name)))) + (princ identifier stream)))) + +;;; the parser state + +(defclass xml-parser-state () + ((entities :documentation "A hashtable mapping XML entity names to their replacement stings" + :accessor get-entities + :initarg :entities + :initform (make-standard-entities)) + (seed :documentation "The user seed object" + :accessor get-seed + :initarg :seed + :initform nil) + (buffer :documentation "The main reusable character buffer" + :accessor get-buffer + :initform (make-extendable-string)) + (mini-buffer :documentation "The secondary, smaller reusable character buffer" + :accessor get-mini-buffer + :initform (make-extendable-string)) + (new-element-hook :documentation "Called when new element starts" + ;; Handle the start of a new xml element with name and attributes, + ;; receiving seed from previous element (sibling or parent) + ;; return seed to be used for first child (content) + ;; or directly to finish-element-hook + :accessor get-new-element-hook + :initarg :new-element-hook + :initform #'(lambda (name attributes seed) + (declare (ignore name attributes)) + seed)) + (finish-element-hook :documentation "Called when element ends" + ;; Handle the end of an xml element with name and attributes, + ;; receiving parent-seed, the seed passed to us when this element started, + ;; i.e. passed to our corresponding new-element-hook + ;; and receiving seed from last child (content) + ;; or directly from new-element-hook + ;; return final seed for this element to next element (sibling or parent) + :accessor get-finish-element-hook + :initarg :finish-element-hook + :initform #'(lambda (name attributes parent-seed seed) + (declare (ignore name attributes parent-seed)) + seed)) + (text-hook :documentation "Called when text is found" + ;; Handle text in string, found as contents, + ;; receiving seed from previous element (sibling or parent), + ;; return final seed for this element to next element (sibling or parent) + :accessor get-text-hook + :initarg :text-hook + :initform #'(lambda (string seed) + (declare (ignore string)) + seed))) + (:documentation "The XML parser state passed along all code making up the parser")) + +(setf (documentation 'get-seed 'function) + "Get the initial user seed of an XML parser state" + (documentation 'get-entities 'function) + "Get the entities hashtable of an XML parser state" + (documentation 'get-new-element-hook 'function) + "Get the new element hook of an XML parser state" + (documentation 'get-finish-element-hook 'function) + "Get the finish element hook of an XML parser state" + (documentation 'get-text-hook 'function) + "Get the text hook of an XML parser state") + +#-allegro +(setf (documentation '(setf get-seed) 'function) + "Set the initial user seed of an XML parser state" + (documentation '(setf get-entities) 'function) + "Set the entities hashtable of an XML parser state" + (documentation '(setf get-new-element-hook) 'function) + "Set the new element hook of an XML parser state" + (documentation '(setf get-finish-element-hook) 'function) + "Set the finish element hook of an XML parser state" + (documentation '(setf get-text-hook) 'function) + "Set the text hook of an XML parser state") + +(defmethod get-mini-buffer :after ((state xml-parser-state)) + "Reset and return the reusable mini buffer" + (with-slots (mini-buffer) state + (setf (fill-pointer mini-buffer) 0))) + +(defmethod get-buffer :after ((state xml-parser-state)) + "Reset and return the main reusable buffer" + (with-slots (buffer) state + (setf (fill-pointer buffer) 0))) + +;;; parser support + +(defun parse-whitespace (stream extendable-string) + "Read and collect XML whitespace from stream in string which is + destructively modified, return first non-whitespace character which + was peeked but not read, return #\Null on eof" + (declare (type (vector character) extendable-string)) + (loop + (let ((char (peek-char nil stream nil #\Null))) + (declare (type character char)) + (if (whitespace-char-p char) + (vector-push-extend (read-char stream) extendable-string) + (return char))))) + +(defun parse-string (stream state string) + "Read and return an XML string from stream, delimited by either + single or double quotes, the stream is expected to be on the opening + delimiter, at the end the closing delimiter is also read, entities + are resolved, eof before end of string is an error" + (declare (type (vector character) string)) + (let ((delimiter (read-char stream nil #\Null)) + (char #\Null)) + (declare (type character delimiter char)) + (unless (or (char= delimiter #\') (char= delimiter #\")) + (error (parser-error "expected string delimiter" nil stream))) + (loop + (setf char (read-char stream nil #\Null)) + (cond ((char= char #\Null) (error (parser-error "encountered eof before end of string"))) + ((char= char delimiter) (return)) + ((char= char #\&) (resolve-entity stream string (get-entities state) (get-mini-buffer state))) + (t (vector-push-extend char string)))) + string)) + +(defun parse-text (stream state extendable-string) + "Read and collect XML text from stream in string which is + destructively modified, the text ends with a '<', which is peeked and + returned, entities are resolved, eof is considered an error" + (declare (type (vector character) extendable-string)) + (let ((char #\Null)) + (declare (type character char)) + (loop + (setf char (peek-char nil stream nil #\Null)) + (when (char= char #\Null) (error (parser-error "encountered unexpected eof in text"))) + (when (char= char #\<) (return)) + (read-char stream) + (if (char= char #\&) + (resolve-entity stream extendable-string (get-entities state) (get-mini-buffer state)) + (vector-push-extend char extendable-string))) + char)) + +(defun parse-identifier (stream identifier) + "Read and returns an XML identifier from stream, positioned at the + start of the identifier, ending with the first non-identifier + character, which is peeked, the identifier is written destructively + into identifier which is also returned" + (declare (type (vector character) identifier)) + (loop + (let ((char (read-char stream nil #\Null))) + (declare (type character char)) + (cond ((identifier-char-p char) + (vector-push-extend char identifier)) + (t + (when (char/= char #\Null) (unread-char char stream)) + (return identifier)))))) + +(defun skip-comment (stream) + "Skip an XML comment in stream, positioned after the opening '<!--', + consumes the closing '-->' sequence, unexpected eof or a malformed + closing sequence result in a error" + (let ((dashes-to-read 2)) + (loop + (if (zerop dashes-to-read) (return)) + (let ((char (read-char stream nil #\Null))) + (declare (type character char)) + (if (char= char #\Null) + (error (parser-error "encountered unexpected eof for comment"))) + (if (char= char #\-) + (decf dashes-to-read) + (setf dashes-to-read 2))))) + (if (char/= (read-char stream nil #\Null) #\>) + (error (parser-error "expected > ending comment" nil stream)))) + +(defun read-cdata (stream state string) + "Reads in the CDATA and calls the callback for CDATA if it exists" + ;; we already read the <![CDATA[ stuff + ;; continue to read until we hit ]]> + (let ((char #\space) + (last-3-characters (list #\[ #\A #\T)) + (pattern (list #\> #\] #\]))) + (declare (type character char)) + (loop + (setf char (read-char stream nil #\Null)) + (when (char= char #\Null) (error (parser-error "encountered unexpected eof in text"))) + (push char last-3-characters) + (setf (cdddr last-3-characters) nil) + (cond + ((equal last-3-characters + pattern) + (setf (fill-pointer string) + (- (fill-pointer string) 2)) + (setf (get-seed state) + (funcall (get-text-hook state) + (copy-seq string) + (get-seed state))) + (return-from read-cdata)) + (t + (vector-push-extend char string)))))) + +(defun skip-special-tag (stream state) + "Skip an XML special tag (comments and processing instructions) in + stream, positioned after the opening '<', unexpected eof is an error" + ;; opening < has been read, consume ? or ! + (read-char stream) + (let ((char (read-char stream nil #\Null))) + (declare (type character char)) + ;; see if we are dealing with a comment + (when (char= char #\-) + (setf char (read-char stream nil #\Null)) + (when (char= char #\-) + (skip-comment stream) + (return-from skip-special-tag))) + ;; maybe we are dealing with CDATA? + (when (and (char= char #\[) + (loop :for pattern :across "CDATA[" + :for char = (read-char stream nil #\Null) + :when (char= char #\Null) :do + (error (parser-error "encountered unexpected eof in cdata")) + :always (char= char pattern))) + (read-cdata stream state (get-buffer state)) + (return-from skip-special-tag)) + ;; loop over chars, dealing with strings (skipping their content) + ;; and counting opening and closing < and > chars + (let ((taglevel 1) + (string-delimiter #\Null)) + (declare (type character string-delimiter)) + (loop + (when (zerop taglevel) (return)) + (setf char (read-char stream nil #\Null)) + (when (char= char #\Null) + (error (parser-error "encountered unexpected eof for special (! or ?) tag" nil stream))) + (if (char/= string-delimiter #\Null) + ;; inside a string we only look for a closing string delimiter + (when (char= char string-delimiter) + (setf string-delimiter #\Null)) + ;; outside a string we count < and > and watch out for strings + (cond ((or (char= char #\') (char= char #\")) (setf string-delimiter char)) + ((char= char #\<) (incf taglevel)) + ((char= char #\>) (decf taglevel)))))))) + +;;; the XML parser proper + +(defun parse-xml-element-attributes (stream state) + "Parse XML element attributes from stream positioned after the tag + identifier, returning the attributes as an assoc list, ending at + either a '>' or a '/' which is peeked and also returned" + (declare (special *namespaces*)) + (let ((char #\Null) attributes) + (declare (type character char)) + (loop + ;; skip whitespace separating items + (setf char (skip-whitespace stream)) + ;; start tag attributes ends with > or /> + (when (or (char= char #\>) (char= char #\/)) (return)) + ;; read the attribute key + (let ((key (let ((string (parse-identifier stream (get-mini-buffer state)))) + (if *ignore-namespaces* + (funcall *attribute-name-parser* string) + (copy-seq string))))) + ;; skip separating whitespace + (setf char (skip-whitespace stream)) + ;; require = sign (and consume it if present) + (if (char= char #\=) + (read-char stream) + (error (parser-error "expected =" nil stream))) + ;; skip separating whitespace + (skip-whitespace stream) + ;; read the attribute value as a string + (push (cons key (let ((string (parse-string stream state (get-buffer state)))) + (if *ignore-namespaces* + (funcall *attribute-value-parser* key string) + (copy-seq string)))) + attributes))) + ;; return attributes peek char ending loop + (values attributes char))) + +(defun parse-xml-element (stream state) + "Parse and return an XML element from stream, positioned after the opening '<'" + (declare (special *namespaces*)) + ;; opening < has been read + (when (char= (peek-char nil stream nil #\Null) #\!) + (skip-special-tag stream state) + (return-from parse-xml-element)) + (let ((char #\Null) buffer open-tag parent-seed has-children) + (declare (type character char)) + (setf parent-seed (get-seed state)) + ;; read tag name (no whitespace between < and name ?) + (setf open-tag (copy-seq (parse-identifier stream (get-mini-buffer state)))) + ;; tag has been read, read attributes if any + (multiple-value-bind (attributes peeked-char) + (parse-xml-element-attributes stream state) + (let ((*namespaces* (extend-namespaces attributes *namespaces*))) + (setf open-tag (resolve-identifier open-tag *namespaces*)) + (unless *ignore-namespaces* + (dolist (attribute attributes) + (setf (car attribute) (funcall *attribute-name-parser* (car attribute)) + (cdr attribute) (funcall *attribute-value-parser* (car attribute) (cdr attribute))))) + (setf (get-seed state) (funcall (get-new-element-hook state) + open-tag attributes (get-seed state))) + (setf char peeked-char) + (when (char= char #\/) + ;; handle solitary tag of the form <tag .. /> + (read-char stream) + (setf char (read-char stream nil #\Null)) + (if (char= #\> char) + (progn + (setf (get-seed state) (funcall (get-finish-element-hook state) + open-tag attributes parent-seed (get-seed state))) + (return-from parse-xml-element)) + (error (parser-error "expected >" nil stream)))) + ;; consume > + (read-char stream) + (loop + (setf buffer (get-buffer state)) + ;; read whitespace into buffer + (setf char (parse-whitespace stream buffer)) + ;; see what ended the whitespace scan + (cond ((char= char #\Null) (error (parser-error "encountered unexpected eof handling ~a" + (list open-tag)))) + ((char= char #\<) + ;; consume the < + (read-char stream) + (if (char= (peek-char nil stream nil #\Null) #\/) + (progn + ;; handle the matching closing tag </tag> and done + ;; if we read whitespace as this (leaf) element's contents, it is significant + (when (and (not has-children) (plusp (length buffer))) + (setf (get-seed state) (funcall (get-text-hook state) + (copy-seq buffer) (get-seed state)))) + (read-char stream) + (let ((close-tag (resolve-identifier (parse-identifier stream (get-mini-buffer state)) + *namespaces*))) + (unless (eq open-tag close-tag) + (error (parser-error "found <~a> not matched by </~a> but by <~a>" + (list open-tag open-tag close-tag) stream))) + (unless (char= (read-char stream nil #\Null) #\>) + (error (parser-error "expected >" nil stream))) + (setf (get-seed state) (funcall (get-finish-element-hook state) + open-tag attributes parent-seed (get-seed state)))) + (return)) + ;; handle child tag and loop, no hooks to call here + ;; whitespace between child elements is skipped + (progn + (setf has-children t) + (parse-xml-element stream state)))) + (t + ;; no child tag, concatenate text to whitespace in buffer + ;; handle text content and loop + (setf char (parse-text stream state buffer)) + (setf (get-seed state) (funcall (get-text-hook state) + (copy-seq buffer) (get-seed state)))))))))) + +(defun start-parse-xml (stream &optional (state (make-instance 'xml-parser-state))) + "Parse and return a toplevel XML element from stream, using parser state" + (loop + (let ((char (skip-whitespace stream))) + (when (char= char #\Null) (return-from start-parse-xml)) + ;; skip whitespace until start tag + (unless (char= char #\<) + (error (parser-error "expected <" nil stream))) + (read-char stream) ; consume peeked char + (setf char (peek-char nil stream nil #\Null)) + (if (or (char= char #\!) (char= char #\?)) + ;; deal with special tags + (skip-special-tag stream state) + (progn + ;; read the main element + (parse-xml-element stream state) + (return-from start-parse-xml (get-seed state))))))) + +;;;; eof diff --git a/third_party/lisp/s-xml/test/ant-build-file.xml b/third_party/lisp/s-xml/test/ant-build-file.xml new file mode 100644 index 000000000000..91d78707b8a1 --- /dev/null +++ b/third_party/lisp/s-xml/test/ant-build-file.xml @@ -0,0 +1,252 @@ +<!-- $Id: ant-build-file.xml,v 1.1 2003/03/18 08:22:09 sven Exp $ --> +<!-- Ant 1.2 build file --> + +<project name="Libretto" default="compile" basedir="."> + + <!-- set global properties for this build --> + <property name="src" value="${basedir}/src" /> + <property name="rsrc" value="${basedir}/rsrc" /> + <property name="build" value="${basedir}/bin" /> + <property name="api" value="${basedir}/api" /> + <property name="lib" value="${basedir}/lib" /> + <property name="junit" value="${basedir}/junit" /> + <property name="rsrc" value="${basedir}/rsrc" /> + + <target name="prepare"> + <!-- Create the time stamp --> + <tstamp/> + <!-- Create the build directory structure used by compile --> + <mkdir dir="${build}" /> + <mkdir dir="${api}" /> + <mkdir dir="${junit}" /> + <copy file="${rsrc}/build/build.version" tofile="${build}/build.properties"/> + <replace file="${build}/build.properties" token="@@@BUILD_ID@@@" value="${DSTAMP}-${TSTAMP}"/> + </target> + + <target name="compile" depends="copy-rsrc"> + <!-- Compile the java code from ${src} into ${build} --> + <javac srcdir="${src}" destdir="${build}" debug="on"> + <classpath> + <fileset dir="${lib}"> + <include name="log4j-core.jar" /> + <include name="jaxp.jar" /> + <include name="crimson.jar" /> + <include name="jdom.jar" /> + <include name="beanshell.jar" /> + </fileset> + </classpath> + </javac> + </target> + + <target name="compile-junit" depends="copy-rsrc"> + <!-- Compile the java code from ${src} into ${build} --> + <javac srcdir="${junit}" destdir="${build}" debug="on"> + <classpath> + <fileset dir="${lib}"> + <include name="*.jar" /> + </fileset> + </classpath> + </javac> + </target> + + <target name="copy-rsrc" depends="prepare"> + <!-- Copy various resource files into ${build} --> + <copy todir="${build}"> + <fileset + dir="${basedir}" + includes="images/*.gif, images/*.jpg" /> + </copy> + <copy todir="${build}"> + <fileset + dir="${src}" + includes="be/beta9/libretto/data/*.txt" /> + </copy> + <copy todir="${build}"> + <fileset + dir="${rsrc}/log4j" + includes="log4j.properties" /> + </copy> + </target> + + <target name="c-header" depends="compile"> + <javah destdir="${rsrc}/VC_source" class="be.beta9.libretto.io.ParallelPort"> + <classpath> + <pathelement location="${build}" /> + </classpath> + </javah> + </target> + + <target name="test-parport" depends="compile"> + <java + classname="be.beta9.libretto.io.ParallelPortWriter" + fork="yes"> + <classpath> + <pathelement location="${build}" /> + <fileset dir="${lib}"> + <include name="*.jar" /> + </fileset> + </classpath> + </java> + </target> + + <target name="jar-simple" depends="compile"> + <!-- Put everything in ${build} into the a jar file --> + <jar + jarfile="${basedir}/libretto.jar" + basedir="${build}" + manifest="${rsrc}/manifest/libretto.mf"/> + </target> + + <target name="jar" depends="compile"> + <!-- Put everything in ${build} into the a jar file including all dependecies --> + <unjar src="${lib}/jaxp.jar" dest="${build}" /> + <unjar src="${lib}/crimson.jar" dest="${build}" /> + <unjar src="${lib}/jdom.jar" dest="${build}" /> + <unjar src="${lib}/log4j-core.jar" dest="${build}" /> + <jar + jarfile="${basedir}/libretto.jar" + basedir="${build}" + manifest="${rsrc}/manifest/libretto.mf"/> + </target> + + <target name="client-jar" depends="background-jar"> + <!-- Put everything in ${build} into the a jar file including all dependecies --> + <unjar src="${lib}/log4j-core.jar" dest="${build}" /> + <jar jarfile="${basedir}/libretto-client.jar" manifest="${rsrc}/manifest/libretto-client.mf"> + <fileset dir="${build}"> + <include name="build.properties"/> + <include name="log4j.properties"/> + <include name="be/beta9/libretto/io/*.class"/> + <include name="be/beta9/libretto/application/Build.class"/> + <include name="be/beta9/libretto/net/LibrettoTextClient*.class"/> + <include name="be/beta9/libretto/net/TestClientMessage.class"/> + <include name="be/beta9/libretto/net/ClientStatusMessageResult.class"/> + <include name="be/beta9/libretto/net/Client*.class"/> + <include name="be/beta9/libretto/net/Constants.class"/> + <include name="be/beta9/libretto/net/TextMessage.class"/> + <include name="be/beta9/libretto/net/MessageResult.class"/> + <include name="be/beta9/libretto/net/MessageException.class"/> + <include name="be/beta9/libretto/net/SingleTextMessage.class"/> + <include name="be/beta9/libretto/net/Message.class"/> + <include name="be/beta9/libretto/net/Util.class"/> + <include name="be/beta9/libretto/gui/ShowSingleTextFrame*.class"/> + <include name="be/beta9/libretto/gui/AWTTextView*.class"/> + <include name="be/beta9/libretto/model/AttributedString*.class"/> + <include name="be/beta9/libretto/model/AWTTextStyle.class"/> + <include name="be/beta9/libretto/model/LTextStyle.class"/> + <include name="be/beta9/libretto/model/AWTCharacterAttributes.class"/> + <include name="be/beta9/libretto/model/Java2DTextStyle.class"/> + <include name="be/beta9/libretto/model/LCharacterAttributes.class"/> + <include name="be/beta9/libretto/model/Java2DCharacterAttributes.class"/> + <include name="be/beta9/libretto/util/TextStyleManager.class"/> + <include name="be/beta9/libretto/util/Bean.class"/> + <include name="be/beta9/libretto/util/LibrettoSaxReader.class"/> + <include name="be/beta9/libretto/util/Preferences.class"/> + <include name="be/beta9/libretto/util/Utilities.class"/> + <include name="org/apache/log4j/**"/> + </fileset> + </jar> + </target> + + <target name="background-jar" depends="compile"> + <!-- Put everything in ${build} into the a jar file including all dependecies --> + <jar jarfile="${basedir}/background.jar" manifest="${rsrc}/manifest/background-black-window.mf"> + <fileset dir="${build}"> + <include name="be/beta9/libretto/gui/BackgroundBlackWindow.class"/> + </fileset> + </jar> + </target> + + <target name="run" depends="compile"> + <!-- Execute the main application --> + <java + classname="be.beta9.libretto.application.Libretto" + fork="yes"> + <classpath> + <pathelement location="${build}" /> + <fileset dir="${lib}"> + <include name="log4j-core.jar" /> + <include name="jaxp.jar" /> + <include name="crimson.jar" /> + <include name="jdom.jar" /> + </fileset> + </classpath> + </java> + </target> + + <target name="debug" depends="compile"> + <!-- Execute the main application in debug mode --> + <java + classname="be.beta9.libretto.application.LibrettoDebug" + fork="yes"> + <classpath> + <pathelement location="${build}" /> + <fileset dir="${lib}"> + <include name="*.jar" /> + </fileset> + </classpath> + </java> + </target> + + <target name="junit" depends="compile-junit"> + <!-- Execute all junit tests --> + <java + classname="be.beta9.libretto.AllTests" + fork="yes"> + <classpath> + <pathelement location="${build}" /> + <fileset dir="${lib}"> + <include name="*.jar" /> + </fileset> + </classpath> + </java> + </target> + + <target name="clean"> + <!-- Delete the ${build} directory trees --> + <delete dir="${build}" /> + <delete dir="${api}" /> + </target> + + <target name="api" depends="prepare"> + <!-- Generate javadoc --> + <javadoc + packagenames="be.beta9.libretto.*" + sourcepath="${src}" + destdir="${api}" + windowtitle="Libretto" + author="true" + version="true" + use="true"/> + </target> + + <target name="zip-all" depends="jar, client-jar"> + <zip zipfile="libretto.zip"> + <fileset dir="${basedir}"> + <include name="libretto.jar"/> + <include name="libretto-client.jar"/> + </fileset> + </zip> + </target> + + <target name="upload" depends="clean, zip-all"> + <ftp + server="users.pandora.be" + userid="a002458" + password="bast0s" + remotedir="libretto" + verbose="true" + passive="true"> + <fileset dir="${basedir}"> + <include name="libretto.jar" /> + <include name="libretto-client.jar" /> + <include name="libretto.zip" /> + </fileset> + </ftp> + </target> + +</project> + + + + diff --git a/third_party/lisp/s-xml/test/plist.xml b/third_party/lisp/s-xml/test/plist.xml new file mode 100644 index 000000000000..910e6326ea63 --- /dev/null +++ b/third_party/lisp/s-xml/test/plist.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>AppleDockIconEnabled</key> + <true/> + <key>AppleNavServices:GetFile:0:Path</key> + <string>file://localhost/Users/sven/Pictures/</string> + <key>AppleNavServices:GetFile:0:Position</key> + <data> + AOUBXw== + </data> + <key>AppleNavServices:GetFile:0:Size</key> + <data> + AAAAAAFeAcI= + </data> + <key>AppleNavServices:PutFile:0:Disclosure</key> + <data> + AQ== + </data> + <key>AppleNavServices:PutFile:0:Path</key> + <string>file://localhost/Users/sven/Desktop/</string> + <key>AppleNavServices:PutFile:0:Position</key> + <data> + AUIBVQ== + </data> + <key>AppleNavServices:PutFile:0:Size</key> + <data> + AAAAAACkAdY= + </data> + <key>AppleSavePanelExpanded</key> + <string>YES</string> + <key>NSDefaultOpenDirectory</key> + <string>~/Desktop</string> + <key>NSNoBigString</key> + <true/> +</dict> +</plist> diff --git a/third_party/lisp/s-xml/test/simple.xml b/third_party/lisp/s-xml/test/simple.xml new file mode 100644 index 000000000000..08ad9424e3ae --- /dev/null +++ b/third_party/lisp/s-xml/test/simple.xml @@ -0,0 +1,5 @@ +<?xml version="1.0"?> +<!-- This is a very simple XML document --> +<root id="123"> + <text>Hello World!</text> +</root> diff --git a/third_party/lisp/s-xml/test/test-lxml-dom.lisp b/third_party/lisp/s-xml/test/test-lxml-dom.lisp new file mode 100644 index 000000000000..248e1e4b907f --- /dev/null +++ b/third_party/lisp/s-xml/test/test-lxml-dom.lisp @@ -0,0 +1,86 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: test-lxml-dom.lisp,v 1.2 2005/11/06 12:44:48 scaekenberghe Exp $ +;;;; +;;;; Unit and functional tests for lxml-dom.lisp +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(assert + (equal (with-input-from-string (stream " <foo/>") + (parse-xml stream :output-type :lxml)) + :|foo|)) + +(assert + (equal (parse-xml-string "<tag1><tag2 att1='one'/>this is some text</tag1>" + :output-type :lxml) + '(:|tag1| + ((:|tag2| :|att1| "one")) + "this is some text"))) + +(assert + (equal (parse-xml-string "<TAG><foo></TAG>" + :output-type :lxml) + '(:TAG "<foo>"))) + +(assert + (equal (parse-xml-string + "<P><INDEX ITEM='one'/> This is some <B>bold</B> text, with a leading & trailing space </P>" + :output-type :lxml) + '(:p + ((:index :item "one")) + " This is some " + (:b "bold") + " text, with a leading & trailing space "))) + +(assert + (consp (parse-xml-file (merge-pathnames "test/xhtml-page.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :lxml))) + +(assert + (consp (parse-xml-file (merge-pathnames "test/ant-build-file.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :lxml))) + +(assert + (consp (parse-xml-file (merge-pathnames "test/plist.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :lxml))) + +(assert + (string-equal (print-xml-string :|foo| :input-type :lxml) + "<foo/>")) + +(assert + (string-equal (print-xml-string '((:|foo| :|bar| "1")) :input-type :lxml) + "<foo bar=\"1\"/>")) + +(assert + (string-equal (print-xml-string '(:foo "some text") :input-type :lxml) + "<FOO>some text</FOO>")) + +(assert + (string-equal (print-xml-string '(:|foo| :|bar|) :input-type :lxml) + "<foo><bar/></foo>")) + +(assert (string-equal (second + (with-input-from-string (stream "<foo><![CDATA[<greeting>Hello, world!</greeting>]]></foo>") + (parse-xml stream :output-type :lxml))) + "<greeting>Hello, world!</greeting>")) + +(assert (string-equal (second + (with-input-from-string (stream "<foo><![CDATA[<greeting>Hello, < world!</greeting>]]></foo>") + (parse-xml stream :output-type :lxml))) + "<greeting>Hello, < world!</greeting>")) + +;;;; eof diff --git a/third_party/lisp/s-xml/test/test-sxml-dom.lisp b/third_party/lisp/s-xml/test/test-sxml-dom.lisp new file mode 100644 index 000000000000..7164d5ef0d66 --- /dev/null +++ b/third_party/lisp/s-xml/test/test-sxml-dom.lisp @@ -0,0 +1,76 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: test-sxml-dom.lisp,v 1.1.1.1 2004/06/07 18:49:59 scaekenberghe Exp $ +;;;; +;;;; Unit and functional tests for sxml-dom.lisp +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(assert + (equal (with-input-from-string (stream " <foo/>") + (parse-xml stream :output-type :sxml)) + '(:|foo|))) + +(assert + (equal (parse-xml-string "<tag1><tag2 att1='one'/>this is some text</tag1>" + :output-type :sxml) + '(:|tag1| + (:|tag2| (:@ (:|att1| "one"))) + "this is some text"))) + +(assert + (equal (parse-xml-string "<TAG><foo></TAG>" + :output-type :sxml) + '(:TAG "<foo>"))) + +(assert + (equal (parse-xml-string + "<P><INDEX ITEM='one'/> This is some <B>bold</B> text, with a leading & trailing space </P>" + :output-type :sxml) + '(:p + (:index (:@ (:item "one"))) + " This is some " + (:b "bold") + " text, with a leading & trailing space "))) + +(assert + (consp (parse-xml-file (merge-pathnames "test/xhtml-page.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :sxml))) + +(assert + (consp (parse-xml-file (merge-pathnames "test/ant-build-file.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :sxml))) + +(assert + (consp (parse-xml-file (merge-pathnames "test/plist.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :sxml))) + +(assert + (string-equal (print-xml-string '(:|foo|) :input-type :sxml) + "<foo/>")) + +(assert + (string-equal (print-xml-string '(:|foo| (:@ (:|bar| "1"))) :input-type :sxml) + "<foo bar=\"1\"/>")) + +(assert + (string-equal (print-xml-string '(:foo "some text") :input-type :sxml) + "<FOO>some text</FOO>")) + +(assert + (string-equal (print-xml-string '(:|foo| (:|bar|)) :input-type :sxml) + "<foo><bar/></foo>")) + +;;;; eof \ No newline at end of file diff --git a/third_party/lisp/s-xml/test/test-xml-struct-dom.lisp b/third_party/lisp/s-xml/test/test-xml-struct-dom.lisp new file mode 100644 index 000000000000..f5ee1cc92583 --- /dev/null +++ b/third_party/lisp/s-xml/test/test-xml-struct-dom.lisp @@ -0,0 +1,84 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: test-xml-struct-dom.lisp,v 1.2 2005/08/29 15:01:49 scaekenberghe Exp $ +;;;; +;;;; Unit and functional tests for xml-struct-dom.lisp +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(assert + (xml-equal (with-input-from-string (stream " <foo/>") + (parse-xml stream :output-type :xml-struct)) + (make-xml-element :name :|foo|))) + +(assert + (xml-equal (parse-xml-string "<tag1><tag2 att1='one'/>this is some text</tag1>" + :output-type :xml-struct) + (make-xml-element :name :|tag1| + :children (list (make-xml-element :name :|tag2| + :attributes '((:|att1| . "one"))) + "this is some text")))) + +(assert + (xml-equal (parse-xml-string "<tag><foo></tag>" + :output-type :xml-struct) + (make-xml-element :name :|tag| + :children (list "<foo>")))) + +(assert + (xml-equal (parse-xml-string + "<P><INDEX ITEM='one'/> This is some <B>bold</B> text, with a leading & trailing space </P>" + :output-type :xml-struct) + (make-xml-element :name :p + :children (list (make-xml-element :name :index + :attributes '((:item . "one"))) + " This is some " + (make-xml-element :name :b + :children (list "bold")) + " text, with a leading & trailing space ")))) + +(assert + (xml-element-p (parse-xml-file (merge-pathnames "test/xhtml-page.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :xml-struct))) + +(assert + (xml-element-p (parse-xml-file (merge-pathnames "test/ant-build-file.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :xml-struct))) + +(assert + (xml-element-p (parse-xml-file (merge-pathnames "test/plist.xml" + (asdf:component-pathname + (asdf:find-system :s-xml.test))) + :output-type :xml-struct))) + +(assert + (string-equal (print-xml-string (make-xml-element :name "foo") + :input-type :xml-struct) + "<foo/>")) + +(assert + (string-equal (print-xml-string (make-xml-element :name "foo" :attributes '((:|bar| . "1"))) + :input-type :xml-struct) + "<foo bar=\"1\"/>")) + +(assert + (string-equal (print-xml-string (make-xml-element :name "foo" :children (list "some text")) + :input-type :xml-struct) + "<foo>some text</foo>")) + +(assert + (string-equal (print-xml-string (make-xml-element :name "foo" :children (list (make-xml-element :name "bar"))) + :input-type :xml-struct) + "<foo><bar/></foo>")) + +;;;; eof \ No newline at end of file diff --git a/third_party/lisp/s-xml/test/test-xml.lisp b/third_party/lisp/s-xml/test/test-xml.lisp new file mode 100644 index 000000000000..daef58ea4639 --- /dev/null +++ b/third_party/lisp/s-xml/test/test-xml.lisp @@ -0,0 +1,86 @@ +;;;; -*- mode: lisp -*- +;;;; +;;;; $Id: test-xml.lisp,v 1.3 2005/11/06 12:44:48 scaekenberghe Exp $ +;;;; +;;;; Unit and functional tests for xml.lisp +;;;; +;;;; Copyright (C) 2002, 2004 Sven Van Caekenberghe, Beta Nine BVBA. +;;;; +;;;; You are granted the rights to distribute and use this software +;;;; as governed by the terms of the Lisp Lesser General Public License +;;;; (http://opensource.franz.com/preamble.html), also known as the LLGPL. + +(in-package :s-xml) + +(assert + (whitespace-char-p (character " "))) + +(assert + (whitespace-char-p (character " "))) + +(assert + (whitespace-char-p (code-char 10))) + +(assert + (whitespace-char-p (code-char 13))) + +(assert + (not (whitespace-char-p #\A))) + +(assert + (char= (with-input-from-string (stream " ABC") + (skip-whitespace stream)) + #\A)) + +(assert + (char= (with-input-from-string (stream "ABC") + (skip-whitespace stream)) + #\A)) + +(assert + (string-equal (with-output-to-string (stream) (print-string-xml "<foo>" stream)) + "<foo>")) + +(assert + (string-equal (with-output-to-string (stream) (print-string-xml "' '" stream)) + "' '")) + +(assert + (let ((string (map 'string #'identity '(#\return #\tab #\newline)))) + (string-equal (with-output-to-string (stream) (print-string-xml string stream)) + string))) + +(defun simple-echo-xml (in out) + (start-parse-xml + in + (make-instance 'xml-parser-state + :new-element-hook #'(lambda (name attributes seed) + (declare (ignore seed)) + (format out "<~a~:{ ~a='~a'~}>" + name + (mapcar #'(lambda (p) (list (car p) (cdr p))) + (reverse attributes)))) + :finish-element-hook #'(lambda (name attributes parent-seed seed) + (declare (ignore attributes parent-seed seed)) + (format out "</~a>" name)) + :text-hook #'(lambda (string seed) + (declare (ignore seed)) + (princ string out))))) + +(defun simple-echo-xml-string (string) + (with-input-from-string (in string) + (with-output-to-string (out) + (simple-echo-xml in out)))) + +(dolist (*ignore-namespaces* '(nil t)) + (assert + (let ((xml "<FOO ATT1='1' ATT2='2'><B>Text</B><EMPTY></EMPTY>More text!<SUB><SUB></SUB></SUB></FOO>")) + (equal (simple-echo-xml-string xml) + xml)))) + +(assert + (let ((xml "<p> </p>")) + (equal (simple-echo-xml-string xml) + xml))) + +;;;; eof \ No newline at end of file diff --git a/third_party/lisp/s-xml/test/xhtml-page.xml b/third_party/lisp/s-xml/test/xhtml-page.xml new file mode 100644 index 000000000000..79f3ae3bade6 --- /dev/null +++ b/third_party/lisp/s-xml/test/xhtml-page.xml @@ -0,0 +1,271 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> + +<html> +<head> + +<title>XHTML Tutorial</title> +<meta http-equiv="Content-Type" content="text/html; charset=windows-1252" /> +<meta name="Keywords" content="XML,tutorial,HTML,DHTML,CSS,XSL,XHTML,JavaScript,ASP,ADO,VBScript,DOM,authoring,programming,learning,beginner's guide,primer,lessons,school,howto,reference,examples,samples,source code,demos,tips,links,FAQ,tag list,forms,frames,color table,W3C,Cascading Style Sheets,Active Server Pages,Dynamic HTML,Internet database development,Webbuilder,Sitebuilder,Webmaster,HTMLGuide,SiteExpert" /> +<meta name="Description" content="HTML,CSS,JavaScript,DHTML,XML,XHTML,ASP,ADO and VBScript tutorial from W3Schools." /> +<meta http-equiv="pragma" content="no-cache" /> +<meta http-equiv="cache-control" content="no-cache" /> + +<link rel="stylesheet" type="text/css" href="../stdtheme.css" /> + +</head> +<body> + +<table border="0" cellpadding="0" cellspacing="0" width="775"> +<tr> +<td width="140" class="content" valign="top"> +<br /> +<a class="left" href="../default.asp" target="_top"><b>HOME</b></a><br /> +<br /> +<b>XHTML Tutorial</b><br /> +<a class="left" target="_top" href="default.asp" style='font-weight:bold;color:#000000;background-color:transparent;'>XHTML HOME</a><br /> +<a class="left" target="_top" href="xhtml_intro.asp" >XHTML Introduction</a><br /> +<a class="left" target="_top" href="xhtml_why.asp" >XHTML Why</a><br /> +<a class="left" target="_top" href="xhtml_html.asp" >XHTML v HTML</a><br /> +<a class="left" target="_top" href="xhtml_syntax.asp" >XHTML Syntax</a><br /> +<a class="left" target="_top" href="xhtml_dtd.asp" >XHTML DTD</a><br /> +<a class="left" target="_top" href="xhtml_howto.asp" >XHTML HowTo</a><br /> +<a class="left" target="_top" href="xhtml_validate.asp" >XHTML Validation</a><br /> +<br /> +<b>Quiz</b> +<br /> +<a class="left" target="_top" href="xhtml_quiz.asp" >XHTML Quiz</a><br /> +<br /> +<b>References</b> +<br /> +<a class="left" target="_top" href="xhtml_reference.asp" >XHTML Tag List</a><br /> +<a class="left" target="_top" href="xhtml_standardattributes.asp" >XHTML Attributes</a><br /> +<a class="left" target="_top" href="xhtml_eventattributes.asp" >XHTML Events</a><br /> +</td> +<td width="490" valign="top"> +<table width="100%" bgcolor="#FFFFFF" border="1" cellpadding="7" cellspacing="0"> +<tr> +<td> +<center> +<a href="http://ad.doubleclick.net/jump/N1951.w3schools/B1097963;sz=468x60;ord=[timestamp]?" target="_new"> +<img src="http://ad.doubleclick.net/ad/N1951.w3schools/B1097963;sz=468x60;ord=[timestamp]?" +border="0" width="468" height="60" alt="Corel XMetal 3" /></a> + + +<br />Please Visit Our Sponsors ! +</center> +<h1>XHTML Tutorial</h1> +<a href="../default.asp"><img border="0" src="../images/btn_previous.gif" alt="Previous" /></a> +<a href="xhtml_intro.asp"><img border="0" src="../images/btn_next.gif" width="100" height="20" alt="Next" /></a> + +<hr /> + +<h2>XHTML Tutorial</h2> +<p>XHTML is the next generation of HTML! In our XHTML tutorial you will learn the difference between HTML and XHTML, and how to use XHTML in your future +applications. You will also see how we converted this Web site into XHTML. <a href="xhtml_intro.asp">Start Learning +XHTML!</a></p> + +<h2>XHTML Quiz Test</h2> +<p>Test your XHTML skills at W3Schools! <a href="xhtml_quiz.asp">Start XHTML +Quiz!</a> </p> + +<h2>XHTML References</h2> +<p>At W3Schools you will find complete XHTML references about tags, attributes +and events. <a href="xhtml_reference.asp">XHTML 1.0 References</a>.</p> +<hr /> +<h2>Table of Contents</h2> +<p><a href="xhtml_intro.asp">Introduction to XHTML</a><br /> +This chapter gives a brief introduction to XHTML and explains what XHTML is.</p> +<p><a href="xhtml_why.asp">XHTML - Why?</a><br /> +This chapter explains why we needed a new language like XHTML.</p> +<p><a href="xhtml_html.asp">Differences between XHTML and HTML</a><br /> +This chapter explains the main differences in syntax between XHTML and HTML.</p> +<p><a href="xhtml_syntax.asp">XHTML Syntax</a> <br /> +This chapter explains the basic syntax of XHTML.</p> +<p><a href="xhtml_dtd.asp">XHTML DTD</a> <br /> +This chapter explains the three different XHTML Document Type Definitions.</p> +<p><a href="xhtml_howto.asp">XHTML HowTo</a><br /> +This chapter explains how this web site was converted from HTML to XHTML.</p> +<p><a href="xhtml_validate.asp">XHTML Validation</a><br /> +This chapter explains how to validate XHTML documents.</p> +<hr /> +<h2>XHTML References</h2> +<p><a href="xhtml_reference.asp">XHTML 1.0 Reference<br /> +</a>Our complete XHTML 1.0 reference is an alphabetical list of all XHTML tags +with lots of examples and tips.</p> +<p><a href="xhtml_standardattributes.asp">XHTML 1.0 Standard Attributes<br /> +</a>All the tags have attributes. The attributes for each tag are listed in the +examples in the "XHTML 1.0 Reference" page. The attributes listed here +are the core and language attributes all the tags has as standard (with +few exceptions). This reference describes the attributes, and shows possible +values for each.</p> +<p><a href="xhtml_eventattributes.asp">XHTML 1.0 Event Attributes<br /> +</a>All the standard event attributes of the tags. This reference describes the attributes, and shows possible +values for each.</p> +<hr /> +<a href="../default.asp"><img border="0" src="../images/btn_previous.gif" width="100" height="20" alt="Previous" /></a> +<a href="xhtml_intro.asp"><img border="0" src="../images/btn_next.gif" width="100" height="20" alt="Next" /></a> + + +<hr /> +<p> +Jump to: <a href="#top" target="_top"><b>Top of Page</b></a> +or <a href="/" target="_top"><b>HOME</b></a> or +<a href='/xhtml/default.asp?output=print' target="_blank"> +<img src="../images/print.gif" alt="Printer Friendly" border="0" /> +<b>Printer friendly page</b></a> +</p> +<hr /> + +<h2>Search W3Schools:</h2> +<form method="get" name="searchform" action="http://www.google.com/search" target="_blank"> +<input type="hidden" name="as_sitesearch" value="www.w3schools.com" /> +<input type="text" size="30" name="as_q" /> +<input type="submit" value=" Go! " /> +</form> + +<hr /> +<h2>What Others Say About Us</h2> +<p>Does the world know about us? Check out these places:</p> +<p> +<a href="http://search.dogpile.com/texis/search?q=W3schools" target="_blank">Dogpile</a> +<a href="http://www.altavista.com/cgi-bin/query?q=W3Schools" target="_blank">Alta Vista</a> +<a href="http://search.msn.com/results.asp?q=W3Schools" target="_blank">MSN</a> +<a href="http://www.google.com/search?q=W3Schools" target="_blank">Google</a> +<a href="http://search.excite.com/search.gw?search=W3Schools" target="_blank">Excite</a> +<a href="http://search.lycos.com/main/?query=W3Schools" target="_blank">Lycos</a> +<a href="http://search.yahoo.com/search?p=w3schools" target="_blank">Yahoo</a> +<a href="http://www.ask.com/main/askJeeves.asp?ask=W3Schools" target="_blank">Ask Jeeves</a> +</p> +<hr /> +<h2>We Help You For Free. You Can Help Us!</h2> +<ul> +<li><a href="../tellyourgroup.htm" target="blank">Tell your newsgroup or mailing list</a></li> +<li><a href="../about/about_linking.asp">Link to us from your pages</a></li> +<li><a href="../about/about_helpers.asp">Help us correct errors and broken links</a></li> +<li><a href="../about/about_helpers.asp">Help us with spelling and grammar</a></li> +<li><a href="http://validator.w3.org/check/referer" target="_blank">Validate the XHTML code of this page</a></li> +</ul> + +<hr /> +<p> +W3Schools is for training only. We do not warrant its correctness or its fitness for use. +The risk of using it remains entirely with the user. While using this site, you agree to have read and accepted our +<a href="../about/about_copyright.asp">terms of use</a> and +<a href="../about/about_privacy.asp">privacy policy</a>.</p> +<p> +<a href="../about/about_copyright.asp">Copyright 1999-2002</a> by Refsnes Data. All Rights Reserved</p> +<hr /> +<table border="0" width="100%" cellspacing="0" cellpadding="0"><tr> +<td width="25%" align="left"> +<a href="http://validator.w3.org/check/referer" target="_blank"> +<img src="../images/vxhtml.gif" alt="Validate" width="88" height="31" border="0" /></a> +</td> +<td width="50%" align="center"> +<a href="../xhtml/" target="_top">How we converted to XHTML</a> +</td> +<td width="25%" align="right"> +<a href="http://jigsaw.w3.org/css-validator/check/referer" target="_blank"> +<img src="../images/vcss.gif" alt="Validate" width="88" height="31" border="0" /></a> +</td> +</tr></table> +</td> +</tr> +</table> +</td> + + + +<td width="144" align="center" valign="top"> + +<table border="1" width="100%" bgcolor="#ffffff" cellpadding="0" cellspacing="0"><tr> +<td align="center" class="right"><br /> + +<a href="http://www.dotnetcharting.com" target="_blank"><img src="../images/dnc-icon.gif" alt="Web charting" border="0" /></a> +<br /> +<a class="right" href="http://www.dotnetcharting.com" target="_blank">Web based charting<br />for ASP.NET</a> + +<br /><br /> +</td></tr></table> + +<table border="1" width="100%" bgcolor="#ffffff" cellpadding="0" cellspacing="0"><tr> +<td align="center" class="right"> +<br /> +<a href="../hosting/default.asp"> +Your own Web Site?<br /> +<br />Read W3Schools +<br />Hosting Tutorial</a> +<br /> +<br /> +</td></tr></table> + +<table border="1" width="100%" bgcolor="#ffffff" cellpadding="0" cellspacing="0"><tr> +<td align="center" class="right"> +<br /> +<a class="red" href="http://www.dotdnr.com" target="_blank">$15 Domain Name<br />Registration<br />Save $20 / year!</a> +<br /> +<br /> +</td></tr></table> + + + +<table border="1" width="100%" bgcolor="#ffffff" cellpadding="0" cellspacing="0"> +<tr><td align="center" class="right"> +<br /> +<b>SELECTED LINKS</b> +<br /><br /> +<a class="right" href="http://opogee.com/clk/dangtingcentiaonie" target="_blank">University Online<br /> +Master Degree<br />Bachelor Degree</a> +<br /><br /> +<a class="right" href="../software/default.asp" target="_top">Web Software</a> +<br /><br /> +<a class="right" href="../appml/default.asp" target="_top">The Future of<br />Web Development</a> +<br /><br /> +<a class="right" href="../careers/default.asp" target="_top">Jobs and Careers</a> +<br /><br /> +<a class="right" href="../site/site_security.asp" target="_top">Web Security</a> +<br /> +<a class="right" href="../browsers/browsers_stats.asp" target="_top">Web Statistics</a> +<br /> +<a class="right" href="../w3c" target="_top">Web Standards</a> +<br /><br /> +</td></tr></table> + + +<table border="1" width="100%" bgcolor="#ffffff" cellpadding="0" cellspacing="0"><tr> +<td align="center" class="right"> +<br /> + +<b>Recommended<br /> +Reading:</b><br /><br /> + +<a class="right" target="_blank" +href="http://www.amazon.com/exec/obidos/ASIN/059600026X/w3schools03"> +<img src="../images/book_amazon_xhtml.jpg" border="0" alt="HTML XHTML" /></a> + + +<br /><br /></td> +</tr></table> + +<table border="1" width="100%" bgcolor="#ffffff" cellpadding="0" cellspacing="0"><tr> +<td align="center" class="right"> +<br /> +<b>PARTNERS</b><br /> +<br /> +<a class="right" href="http://www.W3Schools.com" target="_blank">W3Schools</a><br /> +<a class="right" href="http://www.topxml.com" target="_blank">TopXML</a><br /> +<a class="right" href="http://www.visualbuilder.com" target="_blank">VisualBuilder</a><br /> +<a class="right" href="http://www.xmlpitstop.com" target="_blank">XMLPitstop</a><br /> +<a class="right" href="http://www.developersdex.com" target="_blank">DevelopersDex</a><br /> +<a class="right" href="http://www.devguru.com" target="_blank">DevGuru</a><br /> +<a class="right" href="http://www.programmersheaven.com/" target="_blank">Programmers Heaven</a><br /> +<a class="right" href="http://www.codeproject.com" target="_blank">The Code Project</a><br /> +<a class="right" href="http://www.tek-tips.com" target="_blank">Tek Tips Forum</a><br /> +<a class="right" href="http://www.zvon.ORG/" target="_blank">ZVON.ORG</a><br /> +<a class="right" href="http://www.topxml.com/search.asp" target="_blank">TopXML Search</a><br /> +<br /> +</td> +</tr></table> +</td></tr></table> + +</body> +</html> diff --git a/third_party/lisp/sclf/.skip-subtree b/third_party/lisp/sclf/.skip-subtree new file mode 100644 index 000000000000..5051f60d6b86 --- /dev/null +++ b/third_party/lisp/sclf/.skip-subtree @@ -0,0 +1 @@ +prevent readTree from creating entries for subdirs that don't contain an .nix files diff --git a/third_party/lisp/sclf/OWNERS b/third_party/lisp/sclf/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/third_party/lisp/sclf/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/third_party/lisp/sclf/README b/third_party/lisp/sclf/README new file mode 100644 index 000000000000..2a1c2c3c5c1c --- /dev/null +++ b/third_party/lisp/sclf/README @@ -0,0 +1,6 @@ +SCLF has originally been written by Walter C. Pelissero and vendored +into depot since it is a dependency of mime4cl. Upstream and depot version +may diverge. + +Upstream Website: http://wcp.sdf-eu.org/software/#sclf +Vendored Tarball: http://wcp.sdf-eu.org/software/sclf-20150207T213551.tbz diff --git a/third_party/lisp/sclf/default.nix b/third_party/lisp/sclf/default.nix new file mode 100644 index 000000000000..fb07f8f764e5 --- /dev/null +++ b/third_party/lisp/sclf/default.nix @@ -0,0 +1,28 @@ +# Copyright (C) 2021 by the TVL Authors +# SPDX-License-Identifier: LGPL-2.1-or-later +{ depot, pkgs, ... }: + +depot.nix.buildLisp.library { + name = "sclf"; + + deps = [ + (depot.nix.buildLisp.bundled "sb-posix") + ]; + + srcs = [ + ./package.lisp + ./sclf.lisp + ./sysproc.lisp + ./lazy.lisp + ./time.lisp + ./directory.lisp + ./serial.lisp + ./mp/sbcl.lisp + ]; + + # TODO(sterni): implement OS interaction for ECL and CCL + brokenOn = [ + "ecl" + "ccl" + ]; +} diff --git a/third_party/lisp/sclf/directory.lisp b/third_party/lisp/sclf/directory.lisp new file mode 100644 index 000000000000..4684a8e7056a --- /dev/null +++ b/third_party/lisp/sclf/directory.lisp @@ -0,0 +1,404 @@ +;;; directory.lisp --- filesystem directory access + +;;; Copyright (C) 2006, 2007, 2008, 2009, 2010 by Walter C. Pelissero +;;; Copyright (C) 2021 by the TVL Authors + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: sclf + +#+cmu (ext:file-comment "$Module: directory.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + + +(cl:in-package :sclf) + +(defun pathname-as-directory (pathname) + "Converts PATHNAME to directory form and return it." + (setf pathname (pathname pathname)) + (if (pathname-name pathname) + (make-pathname :directory (append (or (pathname-directory pathname) + '(:relative)) + (list (file-namestring pathname))) + :name nil + :type nil + :defaults pathname) + pathname)) + +(defun d+ (path &rest rest) + "Concatenate directory pathname parts and return a pathname." + (make-pathname :defaults path + :directory (append (pathname-directory path) rest))) + +(defun delete-directory (pathname) + "Remove directory PATHNAME. Return PATHNAME." + #+cmu (multiple-value-bind (done errno) + (unix:unix-rmdir (namestring pathname)) + (unless done + (error "Unable to delete directory ~A (errno=~A)" + pathname errno))) + #+sbcl (sb-posix:rmdir pathname) + #+lispworks (lw:delete-directory pathname) + #-(or cmu sbcl) + (error "DELETE-DIRECTORY not implemented for you lisp system.") + pathname) + +(defun list-directory (pathname &key truenamep) + "List content of directory PATHNAME. If TRUENAMEP is true don't try +to follow symbolic links." + #-(or sbcl cmu) (declare (ignore truenamep)) + (let (#+cmu (lisp::*ignore-wildcards* t)) + (directory (make-pathname :defaults (pathname-as-directory pathname) + :name :wild + :type :wild + :version :wild) + #+cmu :truenamep #+cmu truenamep + #+sbcl :resolve-symlinks #+sbcl truenamep))) + +(defun traverse-directory-tree (root-pathname proc &key truenamep test depth-first) + "Call PROC on all pathnames under ROOT-PATHNAME, both files and +directories. Unless TRUENAMEP is true, this function doesn't try +to lookup the truename of files, as finding the truename may be a +superfluous and noxious activity expecially when you expect +broken symbolic links in your filesystem." + (check-type root-pathname pathname) + (check-type proc (or function symbol)) + (check-type test (or function symbol null)) + (labels ((ls (dir) + (declare (type pathname dir)) + (list-directory dir :truenamep truenamep)) + (traverse? (file) + (declare (type pathname file)) + (and (not (pathname-name file)) + (or truenamep + (not (symbolic-link-p file))) + (or (not test) + (funcall test file)))) + (traverse-pre-order (dir) + (declare (type pathname dir)) + (loop + for file in (ls dir) + do (funcall proc file) + when (traverse? file) + do (traverse-pre-order file))) + (traverse-post-order (dir) + (declare (type pathname dir)) + (loop + for file in (ls dir) + when (traverse? file) + do (traverse-post-order file) + do (funcall proc file)))) + (if depth-first + (traverse-post-order root-pathname) + (traverse-pre-order root-pathname)) + (values))) + +(defmacro do-directory-tree ((file root-pathname &key truenamep test depth-first) &body body) + "Call TRAVERSE-DIRECTORY-TREE with BODY es procedure." + `(traverse-directory-tree ,root-pathname + #'(lambda (,file) + ,@body) + :truenamep ,truenamep + :test ,test + :depth-first ,depth-first)) + +(defun empty-directory-p (pathname) + (and (directory-p pathname) + (endp (list-directory pathname)))) + +(defun remove-empty-directories (root) + (do-directory-tree (pathname root :depth-first t) + (when (empty-directory-p pathname) + (delete-directory pathname)))) + +(defun map-directory-tree (pathname function) + "Apply FUNCTION to every file in a directory tree starting from +PATHNAME. Return the list of results." + (be return-list '() + (do-directory-tree (directory-entry pathname) + (push (funcall function directory-entry) return-list)) + (nreverse return-list))) + +(defun find-files (root-pathname matcher-function &key truenamep) + "In the directory tree rooted at ROOT-PATHNAME, find files that +when the pathname is applied to MATCHER-FUNCTION will return +true. Return the list of files found. Unless TRUENAMEP is true +this function doesn't try to lookup the truename of +files. Finding the truename may be a superfluous and noxious +activity expecially when you expect broken symbolic links in your +filesystem. (This may not apply to your particular lisp +system.)" + (be files '() + (do-directory-tree (file root-pathname :truenamep truenamep) + (when (funcall matcher-function file) + (push file files))) + (nreverse files))) + +(defun delete-directory-tree (pathname) + "Recursively delete PATHNAME and all the directory structure below +it. + +WARNING: depending on the way the DIRECTORY function is implemented on +your Lisp system this function may follow Unix symbolic links and thus +delete files outside the PATHNAME hierarchy. Check this before using +this function in your programs." + (if (pathname-name pathname) + (delete-file pathname) + (progn + (dolist (file (list-directory pathname)) + (delete-directory-tree file)) + (delete-directory pathname)))) + +(defun make-directory (pathname &optional (mode #o777)) + "Create a new directory in the filesystem. Permissions MODE +will be assigned to it. Return PATHNAME." + #+cmu (multiple-value-bind (done errno) + (unix:unix-mkdir (native-namestring pathname) mode) + (unless done + (error "Unable to create directory ~A (errno=~A)." pathname errno))) + #+sbcl (sb-posix:mkdir pathname mode) + #-(or cmu sbcl) + (error "MAKE-DIRECTORY is not implemented for this Lisp system.") + pathname) + +;; At least on SBCL/CMUCL + Unix + NFS this function is faster than +;; ENSURE-DIRECTORIES-EXIST, because it doesn't check all the pathname +;; components starting from the root; it proceeds from the leaf and +;; crawls the directory tree upward only if necessary." +(defun ensure-directory (pathname &key verbose (mode #o777)) + "Just like ENSURE-DIRECTORIES-EXIST but, in some situations, +it's faster." + (labels ((ensure (path) + (unless (probe-file path) + (be* tail (last (pathname-directory path) 2) + last (cdr tail) + (setf (cdr tail) nil) + (unwind-protect + (ensure path) + (setf (cdr tail) last)) + (make-directory path mode) + (when verbose + (format t "Created ~S~%" path)))))) + (ensure (make-pathname :defaults pathname + :name nil :type nil + :version nil)))) + +(defun make-temp-directory (&optional (default-pathname *tmp-file-defaults*) (mode #o777)) + "Create a new directory and return its pathname. +If DEFAULT-PATHNAME is specified and not NIL it's used as +defaults to produce the pathname of the directory. Return the +pathname of the temporary directory." + (loop + for name = (pathname-as-directory (temp-file-name default-pathname)) + when (ignore-errors (make-directory name mode)) + return name)) + +(defmacro with-temp-directory ((path &rest make-temp-directory-args) &body body) + "Execute BODY with PATH bound to the pathname of a new unique +temporary directory. On exit of BODY the directory tree starting from +PATH will be automatically removed from the filesystem. Return what +BODY returns. BODY is _not_ executed within the PATH directory; the +working directory is never changed." + `(be ,path (make-temp-directory ,@make-temp-directory-args) + (unwind-protect + (progn ,@body) + (delete-directory-tree ,path)))) + +(defun current-directory () + "Return the pathname of the current directory." + (truename (make-pathname :directory '(:relative)))) + +(defun ensure-home-translations () + "Ensure that the logical pathname translations for the host \"home\" +are defined." + ;; CMUCL already defines a HOME translation of its own and gets + ;; angry if we try to redefine it + #-cmu + (be home (user-homedir-pathname) + ;; we should discard and replace whatever has been defined in any + ;; rc file during compilation + (setf (logical-pathname-translations "home") + (list + (list "**;*.*.*" + (make-pathname :defaults home + :directory (append (pathname-directory home) + '(:wild-inferiors)) + :name :wild + :type :wild)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun parse-native-namestring (string &optional host (defaults *default-pathname-defaults*) + &key (start 0) end junk-allowed) + #+sbcl (sb-ext:parse-native-namestring string host defaults + :start start + :end end + :junk-allowed junk-allowed) + #-sbcl (let (#+cmu(lisp::*ignore-wildcards* t)) + (parse-namestring string host defaults + :start start + :end end + :junk-allowed junk-allowed))) + +(defun native-namestring (pathname) + #+sbcl (sb-ext:native-namestring pathname) + #-sbcl (let (#+cmu (lisp::*ignore-wildcards* t)) + (namestring pathname))) + +(defun native-file-namestring (pathname) + #+sbcl (sb-ext:native-namestring + (make-pathname :name (pathname-name pathname) + :type (pathname-type pathname))) + #+cmu (be lisp::*ignore-wildcards* t + (file-namestring pathname))) + +(defun native-pathname (thing) + #+sbcl (sb-ext:native-pathname thing) + #+cmu (be lisp::*ignore-wildcards* t + (pathname thing))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun bits-set-p (x bits) + (= (logand x bits) + bits)) + +(defun directory-p (pathname) + "Return true if PATHNAME names a directory on the filesystem." + #-clisp (awhen (unix-stat (native-namestring pathname)) + (bits-set-p (stat-mode it) + #+sbcl sb-posix:s-ifdir + #+cmu unix:s-ifdir)) + #+clisp (ext:probe-directory (pathname-as-directory pathname))) + +(defun regular-file-p (pathname) + "Return true if PATHNAME names a regular file on the filesystem." + #-(or sbcl cmu) (error "don't know how to check whether a file might be a regular file") + (awhen (unix-stat (native-namestring pathname)) + (bits-set-p (stat-mode it) + #+sbcl sb-posix:s-ifreg + #+cmu unix:s-ifreg))) + +(defun file-readable-p (pathname) + #+sbcl (sb-unix:unix-access (native-namestring pathname) sb-unix:r_ok) + #+cmu (unix:unix-access (native-namestring pathname) unix:r_ok) + #-(or sbcl cmu) (error "don't know how to check whether a file might be readable")) + +(defun file-writable-p (pathname) + #+sbcl (sb-unix:unix-access (native-namestring pathname) sb-unix:w_ok) + #+cmu (unix:unix-access (native-namestring pathname) unix:w_ok) + #-(or sbcl cmu) (error "don't know how to check whether a file might be writable")) + +(defun file-executable-p (pathname) + #+sbcl (sb-unix:unix-access (native-namestring pathname) sb-unix:x_ok) + #+cmu (unix:unix-access (native-namestring pathname) unix:x_ok) + #-(or sbcl cmu) (error "don't know how to check whether a file might be executable")) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defstruct (unix-file-stat (:conc-name stat-)) + device + inode + links + atime + mtime + ctime + size + blksize + blocks + uid + gid + mode) + +(defun unix-stat (pathname) + ;; this could be different depending on the unix systems + (multiple-value-bind (ok? device inode mode links uid gid rdev + size atime mtime ctime + blksize blocks) + (#+cmu unix:unix-lstat + #+sbcl sb-unix:unix-lstat + (if (stringp pathname) + pathname + (native-namestring pathname))) + (declare (ignore rdev)) + (when ok? + (make-unix-file-stat :device device + :inode inode + :links links + :atime atime + :mtime mtime + :ctime ctime + :size size + :blksize blksize + :blocks blocks + :uid uid + :gid gid + :mode mode)))) + +(defun stat-modification-time (stat) + "Return the modification time of the STAT structure as Lisp +Universal Time, which is not the same as the Unix time." + (unix->universal-time (stat-mtime stat))) + +(defun stat-creation-time (stat) + "Return the creation time of the STAT structure as Lisp +Universal Time, which is not the same as the Unix time." + (unix->universal-time (stat-ctime stat))) + +(defun file-modification-time (file) + "Return the modification time of FILE as Lisp Universal Time, which +is not the same as the Unix time." + (awhen (unix-stat file) + (stat-modification-time it))) + +(defun file-creation-time (file) + "Return the creation time of FILE as Lisp Universal Time, which +is not the same as the Unix time." + (awhen (unix-stat file) + (stat-creation-time it))) + +(defun read-symbolic-link (symlink) + "Return the pathname the SYMLINK points to. That is, it's +contents." + #+sbcl (sb-posix:readlink (native-namestring symlink)) + #+cmu (unix:unix-readlink (native-namestring symlink))) + +;; FILE-LENGTH is a bit idiosyncratic in this respect. Besides, Unix +;; allows to get to know the file size without being able to open a +;; file; just ask politely. +(defun file-size (pathname) + (stat-size (unix-stat pathname))) + +(defun symbolic-link-p (pathname) + #-(or sbcl cmu) (error "don't know hot to test for symbolic links.") + (aand (unix-stat pathname) + (bits-set-p (stat-mode it) + #+sbcl sb-posix:s-iflnk + #+cmu unix:s-iflnk))) + +(defun broken-link-p (pathname) + (when (symbolic-link-p pathname) + #+cmu (not (ignore-errors (truename pathname))) + ;; On a broken symlink SBCL returns the link path without resolving + ;; the link itself. De gustibus non est disputandum. + #+sbcl (equalp pathname (probe-file pathname)))) + +(defun move-file (old new) + "Just like RENAME-FILE, but doesn't carry on to NEW file the type of +OLD file, if NEW doesn't specify one. It does what most people would +expect from a rename function, which RENAME-FILE doesn't do. +So (MOVE-FILE \"foo.bar\" \"foo\") does rename foo.bar to foo, losing +the \"bar\" type; RENAME-FILE wouldn't allow you that." + #+sbcl (sb-posix:rename (native-namestring old) (native-namestring new)) + #+cmu (unix:unix-rename (native-namestring old) (native-namestring new))) diff --git a/third_party/lisp/sclf/lazy.lisp b/third_party/lisp/sclf/lazy.lisp new file mode 100644 index 000000000000..18f6bfdb7109 --- /dev/null +++ b/third_party/lisp/sclf/lazy.lisp @@ -0,0 +1,134 @@ +;;; lazy.lisp --- lazy primitives + +;;; Copyright (C) 2008, 2009, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: sclf + +#+cmu (ext:file-comment "$Module: lazy.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;; +;;; Lazy primitives +;;; + +(in-package :sclf) + +(defstruct promise + procedure + value) + +(defmacro lazy (form) + `(make-promise :procedure #'(lambda () ,form))) + +(defun forced-p (promise) + (null (promise-procedure promise))) + +(defun force (promise) + (if (forced-p promise) + (promise-value promise) + (prog1 (setf (promise-value promise) + (funcall (promise-procedure promise))) + (setf (promise-procedure promise) nil)))) + +(defmacro deflazy (name value &optional documentation) + `(defparameter ,name (lazy ,value) + ,@(when documentation + (list documentation)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass lazy-metaclass (standard-class) + () + (:documentation "Metaclass for object having lazy slots. Lazy slots +should be specified with the :LAZY keyword which must be a function of +one argument. If required this function will be called once to get +the value to memoize in the slot. Lazy slots can also be set/read as +any other.")) + +(defmethod validate-superclass ((class lazy-metaclass) (super standard-class)) + "Lazy classes may inherit from ordinary classes." + (declare (ignore class super)) + t) + +(defmethod validate-superclass ((class standard-class) (super lazy-metaclass)) + "Ordinary classes may inherit from lazy classes." + (declare (ignore class super)) + t) + +(defclass lazy-slot-mixin () + ((lazy-function :initarg :lazy + :reader lazy-slot-function + :initform nil)) + (:documentation + "Slot for LAZY-METACLASS classes. Lazy slots must be declared with +the argument :LAZY which must be a function accepting the object +instance as argument.")) + +(defclass lazy-direct-slot-definition (lazy-slot-mixin standard-direct-slot-definition) + ()) + +(defclass lazy-effective-slot-definition (lazy-slot-mixin standard-effective-slot-definition) + ()) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmethod direct-slot-definition-class ((class lazy-metaclass) &rest initargs) + (if (getf initargs :lazy nil) + (find-class 'lazy-direct-slot-definition) + (call-next-method))) + +(defmethod effective-slot-definition-class ((class lazy-metaclass) &rest initargs) + (if (getf initargs :lazy nil) + (find-class 'lazy-effective-slot-definition) + (call-next-method))) + +(defmethod compute-effective-slot-definition-initargs ((class lazy-metaclass) direct-slots) + (let ((ds (car direct-slots))) + (if (typep ds 'lazy-direct-slot-definition) + (let ((form (lazy-slot-function ds)) + (args (call-next-method))) + (when (or (getf args :initarg) + (getf args :initform)) + (error "Lazy slot ~S cannot have :INITARG nor :INITFORM arguments." ds)) + (list* :lazy + (cond ((and (listp form) + (eq 'lambda (car form))) + (compile nil form)) + ((symbolp form) + form) + (t (compile nil `(lambda (self) + (declare (ignorable self)) + ,form)))) + args)) + (call-next-method)))) + +(defmethod slot-value-using-class ((class lazy-metaclass) instance (slot lazy-slot-mixin)) + (declare (ignore class)) + ;; If the slot is unbound, call the lazy function passing the + ;; instance and memoize the value in the slot. + (unless (slot-boundp-using-class class instance slot) + (setf (slot-value-using-class class instance slot) + (funcall (lazy-slot-function slot) instance))) + (call-next-method)) + +(defun reset-lazy-slots (object) + "Unbind all the lazy slots in OBJECT so that they will be +re-evaluated next time their value is requested again." + (be* class (class-of object) + (dolist (slot (class-slots class)) + (when (typep slot 'lazy-effective-slot-definition) + (slot-makunbound object (slot-definition-name slot)))))) \ No newline at end of file diff --git a/third_party/lisp/sclf/mp/README b/third_party/lisp/sclf/mp/README new file mode 100644 index 000000000000..a0732c029453 --- /dev/null +++ b/third_party/lisp/sclf/mp/README @@ -0,0 +1,6 @@ +This directory contains an uniforming layer for multiprocessing in the +style supported by Allegro Common Lisp and CMUCL. Almost nothing of +this has been written by me. It's mostly the work of Gilbert Baumann +(unk6@rz.uni-karlsruhe.de) and I've shamelessly lifted it from McCLIM. +The copyright disclaimer in this code is compatible with the one of +SCLF, so I believe there should be no legal issues. diff --git a/third_party/lisp/sclf/mp/cmu.lisp b/third_party/lisp/sclf/mp/cmu.lisp new file mode 100644 index 000000000000..6617f6dadd5d --- /dev/null +++ b/third_party/lisp/sclf/mp/cmu.lisp @@ -0,0 +1,115 @@ +;;; +;;; Code freely lifted from various places with compatible license +;;; terms. Most of this code is copyright Gilbert Baumann +;;; <unk6@rz.uni-karlsruhe.de>. The bugs are copyright Walter +;;; C. Pelissero <walter@pelissero.de>. +;;; + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Library General Public +;;; License as published by the Free Software Foundation; either +;;; version 2 of the License, or (at your option) any later version. +;;; +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Library General Public License for more details. +;;; +;;; You should have received a copy of the GNU Library General Public +;;; License along with this library; if not, write to the +;;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, +;;; Boston, MA 02111-1307 USA. + +(in-package :sclf) + +(defun make-lock (&optional name) + (mp:make-lock name)) + +(defun make-recursive-lock (&optional name) + (mp:make-lock name :kind :recursive)) + +(defmacro with-lock-held ((lock &key whostate (wait t) timeout) &body forms) + `(mp:with-lock-held (,lock ,(or whostate "Lock Wait") + :wait wait + ,@(when timeout (list :timeout timeout))) + ,@forms)) + +(defmacro with-recursive-lock-held ((lock &key wait timeout) &body forms) + `(mp:with-lock-held (,lock + ,@(when wait (list :wait wait)) + ,@(when timeout (list :timeout timeout))) + ,@forms)) + +(defstruct condition-variable + (lock (make-lock "condition variable")) + (value nil) + (process-queue nil)) + +(defun %release-lock (lock) ; copied from with-lock-held in multiproc.lisp + #+i486 (kernel:%instance-set-conditional + lock 2 mp:*current-process* nil) + #-i486 (when (eq (lock-process lock) mp:*current-process*) + (setf (lock-process lock) nil))) + +(defun condition-wait (cv lock &optional timeout) + (declare (ignore timeout)) ;For now + (loop + (let ((cv-lock (condition-variable-lock cv))) + (with-lock-held (cv-lock) + (when (condition-variable-value cv) + (setf (condition-variable-value cv) nil) + (return-from condition-wait t)) + (setf (condition-variable-process-queue cv) + (nconc (condition-variable-process-queue cv) + (list mp:*current-process*))) + (%release-lock lock)) + (mp:process-add-arrest-reason mp:*current-process* cv) + (let ((cv-val nil)) + (with-lock-held (cv-lock) + (setq cv-val (condition-variable-value cv)) + (when cv-val + (setf (condition-variable-value cv) nil))) + (when cv-val + (mp::lock-wait lock "waiting for condition variable lock") + (return-from condition-wait t)))))) + +(defun condition-notify (cv) + (with-lock-held ((condition-variable-lock cv)) + (let ((proc (pop (condition-variable-process-queue cv)))) + ;; The waiting process may have released the CV lock but not + ;; suspended itself yet + (when proc + (loop + for activep = (mp:process-active-p proc) + while activep + do (mp:process-yield)) + (setf (condition-variable-value cv) t) + (mp:process-revoke-arrest-reason proc cv)))) + ;; Give the other process a chance + (mp:process-yield)) + +(defun process-execute (process function) + (mp:process-preset process function) + ;; For some obscure reason process-preset doesn't make the process + ;; runnable. I'm sure it's me who didn't understand how + ;; multiprocessing works under CMUCL, despite the vast documentation + ;; available. + (mp:enable-process process) + (mp:process-add-run-reason process :enable)) + +(defun destroy-process (process) + ;; silnetly ignore a process that is trying to destroy itself + (unless (eq (mp:current-process) + process) + (mp:destroy-process process))) + +(defun restart-process (process) + (mp:restart-process process) + (mp:enable-process process) + (mp:process-add-run-reason process :enable)) + +(defun process-alive-p (process) + (mp:process-alive-p process)) + +(defun process-join (process) + (error "PROCESS-JOIN not support under CMUCL.")) diff --git a/third_party/lisp/sclf/mp/sbcl.lisp b/third_party/lisp/sclf/mp/sbcl.lisp new file mode 100644 index 000000000000..7f47ec9c615f --- /dev/null +++ b/third_party/lisp/sclf/mp/sbcl.lisp @@ -0,0 +1,235 @@ +;;; +;;; Code freely lifted from various places with compatible license +;;; terms. Most of this code is copyright Daniel Barlow +;;; <dan@metacircles.com> or Gilbert Baumann +;;; <unk6@rz.uni-karlsruhe.de>. The bugs are copyright Walter +;;; C. Pelissero <walter@pelissero.de>. +;;; + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Library General Public +;;; License as published by the Free Software Foundation; either +;;; version 2 of the License, or (at your option) any later version. +;;; +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Library General Public License for more details. +;;; +;;; You should have received a copy of the GNU Library General Public +;;; License along with this library; if not, write to the +;;; Free Software Foundation, Inc., 59 Temple Place - Suite 330, +;;; Boston, MA 02111-1307 USA. + +(in-package :sclf) + +(defstruct (process + (:constructor %make-process) + (:predicate processp)) + name + state + whostate + function + thread) + +(defvar *current-process* + (%make-process + :name "initial process" :function nil + :thread + #+#.(cl:if (cl:find-symbol "THREAD-NAME" "SB-THREAD") '(and) '(or)) + sb-thread:*current-thread* + #-#.(cl:if (cl:find-symbol "THREAD-NAME" "SB-THREAD") '(and) '(or)) + (sb-thread:current-thread-id))) + +(defvar *all-processes* (list *current-process*)) + +(defvar *all-processes-lock* + (sb-thread:make-mutex :name "Lock around *ALL-PROCESSES*")) + +;; we implement disable-process by making the disablee attempt to lock +;; *permanent-queue*, which is already locked because we locked it +;; here. enable-process just interrupts the lock attempt. + +(defmacro get-mutex (mutex &optional (wait t)) + `( + #+#.(cl:if (cl:find-symbol "GRAB-MUTEX" "SB-THREAD") '(and) '(or)) + sb-thread:grab-mutex + #-#.(cl:if (cl:find-symbol "GRAB-MUTEX" "SB-THREAD") '(and) '(or)) + sb-thread:get-mutex + ,mutex :waitp ,wait)) + +(defvar *permanent-queue* + (sb-thread:make-mutex :name "Lock for disabled threads")) +(unless (sb-thread:mutex-owner *permanent-queue*) + (get-mutex *permanent-queue* nil)) + +(defun make-process (function &key name) + (let ((p (%make-process :name name + :function function))) + (sb-thread:with-mutex (*all-processes-lock*) + (pushnew p *all-processes*)) + (restart-process p))) + +(defun process-kill-thread (process) + (let ((thread (process-thread process))) + (when (and thread + (sb-thread:thread-alive-p thread)) + (assert (not (eq thread sb-thread:*current-thread*))) + (sb-thread:terminate-thread thread) + ;; Wait until all the clean-up forms are done. + (sb-thread:join-thread thread :default nil)) + (setf (process-thread process) nil))) + +(defun process-join (process) + (sb-thread:join-thread (process-thread process))) + +(defun restart-process (p) + (labels ((boing () + (let ((*current-process* p) + (function (process-function p))) + (when function + (funcall function))))) + (process-kill-thread p) + (when (setf (process-thread p) + (sb-thread:make-thread #'boing :name (process-name p))) + p))) + +(defun destroy-process (process) + (sb-thread:with-mutex (*all-processes-lock*) + (setf *all-processes* (delete process *all-processes*))) + (process-kill-thread process)) + +(defun current-process () + *current-process*) + +(defun all-processes () + ;; we're calling DELETE on *ALL-PROCESSES*. If we look up the value + ;; while that delete is executing, we could end up with nonsense. + ;; Better use a lock (or call REMOVE instead in DESTROY-PROCESS). + (sb-thread:with-mutex (*all-processes-lock*) + *all-processes*)) + +(defun process-yield () + (sb-thread:thread-yield)) + +(defun process-wait (reason predicate) + (let ((old-state (process-whostate *current-process*))) + (unwind-protect + (progn + (setf old-state (process-whostate *current-process*) + (process-whostate *current-process*) reason) + (until (funcall predicate) + (process-yield))) + (setf (process-whostate *current-process*) old-state)))) + +(defun process-wait-with-timeout (reason timeout predicate) + (let ((old-state (process-whostate *current-process*)) + (end-time (+ (get-universal-time) timeout))) + (unwind-protect + (progn + (setf old-state (process-whostate *current-process*) + (process-whostate *current-process*) reason) + (loop + for result = (funcall predicate) + until (or result + (> (get-universal-time) end-time)) + do (process-yield) + finally (return result))) + (setf (process-whostate *current-process*) old-state)))) + +(defun process-interrupt (process function) + (sb-thread:interrupt-thread (process-thread process) function)) + +(defun disable-process (process) + (sb-thread:interrupt-thread + (process-thread process) + (lambda () + (catch 'interrupted-wait (get-mutex *permanent-queue*))))) + +(defun enable-process (process) + (sb-thread:interrupt-thread + (process-thread process) (lambda () (throw 'interrupted-wait nil)))) + +(defmacro without-scheduling (&body body) + (declare (ignore body)) + (error "WITHOUT-SCHEDULING is not supported on this platform.")) + +(defparameter *atomic-lock* + (sb-thread:make-mutex :name "atomic incf/decf")) + +(defmacro atomic-incf (place) + `(sb-thread:with-mutex (*atomic-lock*) + (incf ,place))) + +(defmacro atomic-decf (place) + `(sb-thread:with-mutex (*atomic-lock*) + (decf ,place))) + +;;; 32.3 Locks + +(defun make-lock (&optional name) + (sb-thread:make-mutex :name name)) + +(defmacro with-lock-held ((place &key state (wait t) timeout) &body body) + (declare (ignore timeout)) + (let ((old-state (gensym "OLD-STATE"))) + `(sb-thread:with-mutex (,place :wait-p ,wait) + (let (,old-state) + (unwind-protect + (progn + (when ,state + (setf ,old-state (process-state *current-process*)) + (setf (process-state *current-process*) ,state)) + ,@body) + (setf (process-state *current-process*) ,old-state)))))) + + +(defun make-recursive-lock (&optional name) + (sb-thread:make-mutex :name name)) + +(defmacro with-recursive-lock-held ((place &optional state (wait t) timeout) &body body) + (declare (ignore wait timeout)) + (let ((old-state (gensym "OLD-STATE"))) + `(sb-thread:with-recursive-lock (,place) + (let (,old-state) + (unwind-protect + (progn + (when ,state + (setf ,old-state (process-state *current-process*)) + (setf (process-state *current-process*) ,state)) + ,@body) + (setf (process-state *current-process*) ,old-state)))))) + +(defun make-condition-variable () (sb-thread:make-waitqueue)) + +(defun condition-wait (cv lock &optional timeout) + (if timeout + (handler-case + (sb-ext:with-timeout timeout + (sb-thread:condition-wait cv lock) + t) + (sb-ext:timeout (c) + (declare (ignore c)) + nil)) + (progn (sb-thread:condition-wait cv lock) t))) + +(defun condition-notify (cv) + (sb-thread:condition-notify cv)) + + +(defvar *process-plists* (make-hash-table) + "Hash table mapping processes to a property list. This is used by +PROCESS-PLIST.") + +(defun process-property-list (process) + (gethash process *process-plists*)) + +(defun (setf process-property-list) (value process) + (setf (gethash process *process-plists*) value)) + +(defun process-execute (process function) + (setf (process-function process) function) + (restart-process process)) + +(defun process-alive-p (process) + (sb-thread:thread-alive-p (process-thread process))) diff --git a/third_party/lisp/sclf/package.lisp b/third_party/lisp/sclf/package.lisp new file mode 100644 index 000000000000..652194f93cad --- /dev/null +++ b/third_party/lisp/sclf/package.lisp @@ -0,0 +1,258 @@ +;;; package.lisp --- packages description + +;;; Copyright (C) 2006, 2007, 2008, 2009, 2010 by Walter C. Pelissero +;;; Copyright (C) 2021 by the TVL Authors + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: sclf + +#+cmu (ext:file-comment "$Module: package.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :cl-user) + +(defpackage :sclf + (:use :common-lisp + ;; we need the MOP for lazy.lisp and serial.lisp + #+cmu :pcl + #+sbcl :sb-mop) + ;; Don't know why but compute-effective-slot-definition-initargs is + ;; internal in both CMUCL and SBCL + (:import-from #+cmu"PCL" #+sbcl"SB-PCL" + #-(or cmu sbcl) "CLOS" + "COMPUTE-EFFECTIVE-SLOT-DEFINITION-INITARGS") + #+cmu (:import-from :mp + #:make-process + #:current-process + #:all-processes + #:processp + #:process-name + #:process-state + #:process-whostate + #:process-wait + #:process-wait-with-timeout + #:process-yield + #:process-interrupt + #:disable-process + #:enable-process + #:without-scheduling + #:atomic-incf + #:atomic-decf + #:process-property-list) + (:export #:be #:be* + #:defconst + #:with-gensyms + #:d+ + #:s+ + #:f++ + #:list->string + #:string-starts-with #:string-ends-with + #:aif #:awhen #:acond #:aand #:acase #:it + #:+whitespace+ + #:string-trim-whitespace + #:string-right-trim-whitespace + #:string-left-trim-whitespace + #:whitespace-p #:seq-whitespace-p + #:not-empty + #:position-any + #:+month-names+ + #:find-any + #:split-at + #:split-string-at-char + #:week-day->string + #:month->string + #:month-string->number + #:add-months #:add-days + #:read-whole-stream + #:read-file #:write-file #:read-lines + #:read-from-file #:write-to-file + #:string-concat + #:gcase + #:string-truncate + #:promise #:force #:forced-p #:lazy #:deflazy #:lazy-metaclass #:self #:reset-lazy-slots + #:copy-stream #:copy-file + #:symlink-file + #:keywordify + #:until + #:year #:month #:day #:hour #:minute #:week-day #:week #:day-of-the-year + #:beginning-of-week #:end-of-week + #:next-week-day #:next-monday #:full-weeks-in-span + #:beginning-of-first-week #:end-of-last-week + #:beginning-of-month #:end-of-month + #:locate-system-program + #:*tmp-file-defaults* + #:temp-file-name + #:open-temp-file + #:with-temp-file + #:file-size + #:getenv + #:with-system-environment + #:time-string #:iso-time-string #:parse-iso-time-string + #:soundex + #:string-soundex= + #:lru-cache + #:getcache #:cached + #:print-time-span + #:double-linked-list #:limited-list #:sorted-list + #:insert #:size + #:heap #:heap-add #:heap-pop #:heap-empty-p + #:double-linked-element #:make-double-linked-element #:double-linked-element-p + #:dle-previous #:dle-next #:dle-value + #:cons-dle #:dle-remove #:dle-map #:do-dle :do-dle* + #:sl-map #:do-dll #:do-dll* + #:dll-find #:dll-find-cursor + #:push-first #:push-last #:dll-remove + #:pop-first #:pop-last + #:leap-year-p #:last-day-of-month + #:getuid #:setuid #:with-euid + #:get-logname #:get-user-name #:get-user-home #:find-uid + #:super-user-p + #:pathname-as-directory #:pathname-as-file + #:alist->plist #:plist->alist + #:byte-vector->string + #:string->byte-vector + #:outdated-p + #:with-hidden-temp-file + #:let-places #:let-slots + #:*decimal-point* + #:*thousands-comma* + #:format-amount #:parse-amount + #:with-package + #:make-directory #:ensure-directory + #:make-temp-directory + #:with-temp-directory + #:delete-directory + #:delete-directory-tree + #:do-directory-tree + #:traverse-directory-tree + #:empty-directory-p + #:remove-empty-directories + #:map-directory-tree + #:find-files + #:directory-p + #:regular-file-p + #:file-readable-p + #:file-writable-p + #:file-executable-p + #:current-directory + #:ensure-home-translations + #:list-directory + #:string-escape + #:string-substitute + #:bytes-simple-string + #:make-lock-files + #:with-lock-files + #:getpid + #:on-error + #:floor-to + #:round-to + #:ceiling-to + #:insert-in-order + #:forget-documentation + #:load-compiled + #:swap + #:queue #:make-queue #:queue-append #:queue-pop #:queue-empty-p + #:unix-stat #:unix-file-stat + #:stat-device + #:stat-inode + #:stat-links + #:stat-atime + #:stat-mtime + #:stat-ctime + #:stat-birthtime + #:stat-size + #:stat-blksize + #:stat-blocks + #:stat-uid + #:stat-gid + #:stat-mode + #:save-file-excursion + #:stat-modification-time + #:stat-creation-time + #:file-modification-time + #:file-creation-time + #:show + #:memoize-function + #:memoized + #:defun-memoized + #:parse-native-namestring + #:native-file-namestring + #:native-namestring + #:native-pathname + #:read-symbolic-link + #:symbolic-link-p + #:broken-link-p + #:circular-list + #:last-member + #:glob->regex + #:universal->unix-time #:unix->universal-time + #:get-unix-time + #:move-file + + ;; sysproc.lisp + #:*run-verbose* + #:run-pipe + #:run-program + #:run-shell-command + #:run-async-shell-command + #:exit-code + #:with-open-pipe + #:*bourne-shell* + #:sysproc-kill + #:sysproc-input + #:sysproc-output + #:sysproc-alive-p + #:sysproc-pid + #:sysproc-p + #:sysproc-wait + #:sysproc-exit-code + #:sysproc-set-signal-callback + + ;; MP + #:make-process + #:destroy-process + #:current-process + #:all-processes + #:processp + #:process-name + #:process-state + #:process-whostate + #:process-wait + #:process-wait-with-timeout + #:process-yield + #:process-interrupt + #:disable-process + #:enable-process + #:restart-process + #:without-scheduling + #:atomic-incf + #:atomic-decf + #:process-property-list + #:process-alive-p + #:process-join + ;; + #:make-lock + #:with-lock-held + #:make-recursive-lock + #:with-recursive-lock-held + ;; + #:make-condition-variable + #:condition-wait + #:condition-notify + #:process-property-list + #:process-execute + ;; mop.lisp + #:printable-object-mixin + )) diff --git a/third_party/lisp/sclf/sclf.asd b/third_party/lisp/sclf/sclf.asd new file mode 100644 index 000000000000..dfb56a8ded0e --- /dev/null +++ b/third_party/lisp/sclf/sclf.asd @@ -0,0 +1,58 @@ +;;; sclf.asd --- system definition + +;;; Copyright (C) 2005, 2006, 2008, 2009 by Walter C. Pelissero +;;; Copyright (C) 2021 by the TVL Authors + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: SCLF + +#+cmu (ext:file-comment "$Module: sclf.asd, Time-stamp: <2013-06-17 15:32:29 wcp> $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :cl-user) + +(defpackage :sclf-system + (:use :common-lisp :asdf #+asdfa :asdfa)) + +(in-package :sclf-system) + +(defsystem sclf + :name "SCLF" + :author "Walter C. Pelissero <walter@pelissero.de>" + :maintainer "Walter C. Pelissero <walter@pelissero.de>" + ;; :version "0.0" + :description "Stray Common Lisp Functions" + :long-description + "A collection of Common Lisp functions for the most disparate +uses, too small to fit anywhere else." + :licence "LGPL" + :depends-on (#+sbcl :sb-posix) + :components + ((:doc-file "README") + (:file "package") + (:file "sclf" :depends-on ("package")) + (:file "sysproc" :depends-on ("package" "sclf")) + (:file "lazy" :depends-on ("package" "sclf")) + (:file "time" :depends-on ("package" "sclf")) + (:file "directory" :depends-on ("package" "sclf" "time")) + (:file "serial" :depends-on ("package" "sclf")) + (:module "mp" + :depends-on ("package" "sclf") + :components + ((:doc-file "README") + (:file #.(first + (list #+cmu "cmu" + #+sbcl "sbcl" + "unknown"))))))) diff --git a/third_party/lisp/sclf/sclf.lisp b/third_party/lisp/sclf/sclf.lisp new file mode 100644 index 000000000000..0d587da8eb69 --- /dev/null +++ b/third_party/lisp/sclf/sclf.lisp @@ -0,0 +1,1717 @@ +;;; sclf.lisp --- miscellanea + +;;; Copyright (C) 2005, 2006, 2007, 2008, 2009, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: SCLF + +#+cmu (ext:file-comment "$Module: sclf.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +;;; Commentary: + +;;; This is a collection of Common Lisp functions of the most disparate +;;; uses and purposes. These functions are too small or too unrelated +;;; to each other to deserve an own module. +;;; +;;; If you want to indent properly the following macros you should add +;;; the following lines to your .emacs file: +;;; +;;; (defun cl-indent-be (path state indent-point sexp-column normal-indent) +;;; (let ((sexp-start (cadr state)) +;;; (i 0)) +;;; (save-excursion +;;; (goto-char sexp-start) +;;; (forward-char) +;;; (+ sexp-column +;;; (block indentation +;;; (condition-case nil +;;; (while (< (point) indent-point) +;;; (setq i (1+ i)) +;;; (when (and (= 0 (logand i 1)) +;;; (looking-at "[\t\n ]*\\s(")) +;;; (return-from indentation 2)) +;;; (forward-sexp)) +;;; (error nil)) +;;; (if (= 1 (logand i 1)) +;;; 6 4)))))) +;;; +;;; (put 'be 'common-lisp-indent-function 'cl-indent-be) +;;; (put 'be* 'common-lisp-indent-function 'cl-indent-be) +;;; (put 'awhen 'lisp-indent-function 1) +;;; (put 'gcase 'lisp-indent-function 1) +;;; (put 'acase 'lisp-indent-function 1) +;;; (put 'acond 'lisp-indent-function 1) +;;; (put 'until 'lisp-indent-function 1) + + + +(cl:in-package :sclf) + +(defmacro be (&rest bindings-and-body) + "Less-parenthetic let." + (let ((bindings + (loop + while (and (symbolp (car bindings-and-body)) + (cdr bindings-and-body)) + collect (list (pop bindings-and-body) + (pop bindings-and-body))))) + `(let ,bindings + ,@bindings-and-body))) + +(defmacro be* (&rest bindings-and-body) + "Less-parenthetic let*." + (let ((bindings + (loop + while (and (symbolp (car bindings-and-body)) + (cdr bindings-and-body)) + collect (list (pop bindings-and-body) + (pop bindings-and-body))))) + `(let* ,bindings + ,@bindings-and-body))) + +(defmacro defconst (name value &rest etc) + "For some reason SBCL, between usefulness and adherence to the ANSI +standard, has chosen the latter, thus rendering the DEFCONSTANT pretty +useless. This macro works around that problem." + #+sbcl (list* 'defvar name value etc) + #-sbcl (list* 'defconstant name value etc)) + +(defmacro with-gensyms ((&rest symbols) &body body) + "Gensym all SYMBOLS and make them available in BODY. +See also LET-GENSYMS." + `(let ,(mapcar #'(lambda (s) + (list s '(gensym))) symbols) + ,@body)) + +(defun s+ (&rest strings) + "Return a string which is made of the concatenation of STRINGS." + (apply #'concatenate 'string strings)) + +(defun string-starts-with (prefix string &optional (compare #'string=)) + (be prefix-length (length prefix) + (and (>= (length string) prefix-length) + (funcall compare prefix string :end2 prefix-length)))) + +(defun string-ends-with (postfix string &optional (compare #'string=)) + "Return true if STRING's last characters are the same as POSTFIX." + (be postfix-length (length postfix) + string-length (length string) + (and (>= string-length postfix-length) + (funcall compare postfix string :start2 (- string-length postfix-length))))) + +(defun string-substitute (from to sequence &key (start 0) end (test #'eql)) + "Replace in SEQUENCE occurrences of FROM with TO. FROM and TO don't +need to be the same length." + (be from-length (length from) + (with-output-to-string (out) + (write-string sequence out :start 0 :end start) + (loop + for position = (search from sequence :start2 start :end2 end :test test) + while position + do + (write-string sequence out :start start :end position) + (write-string to out) + (setf start (+ position from-length)) + finally (write-string (subseq sequence start) out))))) + +(defun string-escape (string character &key (escape-character #\\) (escape-escape t)) + "Prepend all occurences of CHARACTER in STRING with a +ESCAPE-CHARACTER." + (with-output-to-string (stream) + (loop + for c across string + when (or (char= c character) + (and escape-escape + (char= c escape-character))) + do (write-char escape-character stream) + do (write-char c stream)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmacro aif (test then &optional else) + `(be it ,test + (if it + ,then + ,else))) + +(defmacro awhen (test &body then) + `(be it ,test + (when it + ,@then))) + +(defmacro acond (&body forms) + (when forms + `(aif ,(caar forms) + (progn ,@(cdar forms)) + (acond ,@(cdr forms))))) + +(defmacro aand (&rest args) + (cond ((null args) t) + ((null (cdr args)) (car args)) + (t `(aif ,(car args) (aand ,@(cdr args)))))) + +(defmacro acase (condition &body forms) + `(be it ,condition + (case it ,@forms))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defconst +whitespace+ '(#\return #\newline #\tab #\space #\page)) + +(defun string-trim-whitespace (string) + (string-trim +whitespace+ string)) + +(defun string-right-trim-whitespace (string) + (string-right-trim +whitespace+ string)) + +(defun string-left-trim-whitespace (string) + (string-left-trim +whitespace+ string)) + +(defun whitespace-p (char) + (member char +whitespace+)) + +(defun seq-whitespace-p (sequence) + (every #'whitespace-p sequence)) + +(defun not-empty (sequence) + "Return SEQUENCE if it's not empty, otherwise NIL. +NIL is indeed empty." + (when (or (listp sequence) + (not (zerop (length sequence)))) + sequence)) + +(defun position-any (bag sequence &rest position-args) + "Find any element of bag in sequence and return its position. +Accept any argument accepted by the POSITION function." + (apply #'position-if #'(lambda (element) + (find element bag)) sequence position-args)) + +(defun find-any (bag sequence &rest find-args) + "Find any element of bag in sequence. Accept any argument +accepted by the FIND function." + (apply #'find-if #'(lambda (element) + (find element bag)) sequence find-args)) + +(defun split-at (bag sequence &key (start 0) key) + "Split SEQUENCE at occurence of any element from BAG. +Contiguous occurences of elements from BAG are considered atomic; +so no empty sequence is returned." + (be len (length sequence) + (labels ((split-from (start) + (unless (>= start len) + (be sep (position-any bag sequence :start start :key key) + (cond ((not sep) + (list (subseq sequence start))) + ((> sep start) + (cons (subseq sequence start sep) + (split-from (1+ sep)))) + (t + (split-from (1+ start)))))))) + (split-from start)))) + +(defun split-string-at-char (string separator &key escape skip-empty) + "Split STRING at SEPARATORs and return a list of the substrings. If +SKIP-EMPTY is true then filter out the empty substrings. If ESCAPE is +not nil then split at SEPARATOR only if it's not preceded by ESCAPE." + (declare (type string string) (type character separator)) + (labels ((next-separator (beg) + (be pos (position separator string :start beg) + (if (and escape + pos + (plusp pos) + (char= escape (char string (1- pos)))) + (next-separator (1+ pos)) + pos))) + (parse (beg) + (cond ((< beg (length string)) + (let* ((end (next-separator beg)) + (substring (subseq string beg end))) + (cond ((and skip-empty (string= "" substring)) + (parse (1+ end))) + ((not end) + (list substring)) + (t + (cons substring (parse (1+ end))))))) + (skip-empty + '()) + (t + (list ""))))) + (parse 0))) + +(defun copy-stream (in out) + (loop + for c = (read-char in nil) + while c + do (write-char c out))) + +(defun pathname-as-file (pathname) + "Converts PATHNAME to file form and return it." + (unless (pathnamep pathname) + (setf pathname (pathname pathname))) + (cond ((pathname-name pathname) + pathname) + ((stringp (car (last (pathname-directory pathname)))) + (be name (parse-native-namestring (car (last (pathname-directory pathname)))) + (make-pathname :directory (butlast (pathname-directory pathname)) + :name (pathname-name name) + :type (pathname-type name) + :defaults pathname))) + ;; it can't be done? + (t pathname))) + +(defun copy-file (file copy-file &key (if-exists :error)) + (with-open-file (in file) + (with-open-file (out copy-file :direction :output :if-exists if-exists) + (copy-stream in out)))) + +(defun symlink-file (src dst &key (if-exists :error)) + (when (and (eq :supersede if-exists) + (probe-file dst)) + (delete-file dst)) + #+sbcl (sb-posix:symlink src dst) + #+cmu(unix:unix-symlink (native-namestring src) (native-namestring dst)) + #-(or sbcl cmu) (error "don't know how to symlink files")) + +(defun read-whole-stream (stream) + "Read stream until the end and return it as a string." + (with-output-to-string (string) + (loop + for line = (read-line stream nil) + while line + do (write-line line string)))) + +(defun read-lines (stream &optional n) + "Read N lines from stream and return them as a list of strings. If +N is NIL, read the whole stream til the end. If the stream ends +before N lines a read, this function will return those without +signalling an error." + (loop + for line = (read-line stream nil) + for i from 0 + while (and line + (or (not n) + (< i n))) + collect line)) + +(defun read-file (pathname &key (element-type 'character) (if-does-not-exist :error) default) + "Read the whole content of file and return it as a sequence which +can be a string, a vector of bytes, or whatever you specify as +ELEMENT-TYPE." + (with-open-file (in pathname + :element-type element-type + :if-does-not-exist (unless (eq :value if-does-not-exist) + :error)) + (if in + (be seq (make-array (file-length in) :element-type element-type) + (read-sequence seq in) + seq) + default))) + +(defun write-file (pathname contents &key (if-exists :error)) + "Read the whole content of file and return it as a sequence which +can be a string, a vector of bytes, or whatever you specify as +ELEMENT-TYPE." + (with-open-file (out pathname + :element-type (if (stringp contents) + 'character + (array-element-type contents)) + :if-exists if-exists) + (write-sequence contents out))) + +(defun read-from-file (pathname &key (on-error :error) default) + "Similar to READ-FROM-STRING but for files. Read the first Lisp +object in file and return it. If file does not exist or does not +contain a readable Lisp object, ON-ERROR tells what to do. If +ON-ERROR is :ERROR, an error is signalled. If ON-ERROR is :VALUE, +DEFAULT is returned." + (ecase on-error + (:error + (with-open-file (in pathname) + (read in))) + (:value + (handler-case (with-open-file (in pathname) + (read in)) + (t () + default))))) + +(defun write-to-file (object pathname &key (if-exists :error) pretty) + "Similar to WRITE-TO-STRING but for files. Write OBJECT to a file +with pathname PATHNAME." + (with-open-file (out pathname :direction :output :if-exists if-exists) + (write object :stream out :escape t :readably t :pretty pretty))) + +(defun string-concat (list &optional (separator "")) + "Concatenate the strings in LIST interposing SEPARATOR (default +nothing) between them." + (reduce #'(lambda (&rest args) + (if args + (s+ (car args) separator (cadr args)) + "")) + list)) + +;; to indent it properly: (put 'gcase 'lisp-indent-function 1) +(defmacro gcase ((value &optional (test 'equalp)) &rest cases) + "Generic CASE macro. Match VALUE to CASES as if by the normal CASE +but use TEST as the comparison function, which defaults to EQUALP." + (with-gensyms (val) + `(be ,val ,value + ,(cons 'cond + (mapcar #'(lambda (case-desc) + (destructuring-bind (vals &rest forms) case-desc + `(,(cond ((consp vals) + (cons 'or (mapcar #'(lambda (v) + (list test val v)) + vals))) + ((or (eq vals 'otherwise) + (eq vals t)) + t) + (t (list test val vals))) + ,@forms))) + cases))))) + +(defun string-truncate (string max-length) + "If STRING is longer than MAX-LENGTH, return a shorter version. +Otherwise return the same string unchanged." + (if (> (length string) max-length) + (subseq string 0 max-length) + string)) + +;; to indent properly: (put 'until 'lisp-indent-function 1) +(defmacro until (test &body body) + (with-gensyms (result) + `(loop + for ,result = ,test + until ,result + do (progn ,@body) + finally (return ,result)))) + +(defun keywordify (string) + (intern (string-upcase string) :keyword)) + +(defun locate-system-program (name) + "Given the NAME of a system program try to find it through the +search of the environment variable PATH. Return the full +pathname." + (loop + for dir in (split-string-at-char (getenv "PATH") #\:) + for pathname = (merge-pathnames name (pathname-as-directory dir)) + when (probe-file pathname) + return pathname)) + +(defvar *tmp-file-defaults* #P"/tmp/") + +(defun temp-file-name (&optional (default *tmp-file-defaults*)) + "Create a random pathname based on DEFAULT. No effort is made +to make sure that the returned pathname doesn't identify an +already existing file. If missing DEFAULT defaults to +*TMP-FILE-DEFAULTS*." + (make-pathname :defaults default + :name (format nil "~36R" (random #.(expt 36 10))))) + +(defun open-temp-file (&optional default-pathname &rest open-args) + "Open a new temporary file and return a stream to it. This function +makes sure the pathname of the temporary file is unique. OPEN-ARGS +are arguments passed verbatim to OPEN. If OPEN-ARGS specify +the :DIRECTION it should be either :OUTPUT (default) or :IO; +any other value causes an error. If DEFAULT-PATHNAME is specified and +not NIL it's used as defaults to produce the pathname of the temporary +file, otherwise *TMP-FILE-DEFAULTS* is used." + (unless default-pathname + (setf default-pathname *tmp-file-defaults*)) + ;; if :DIRECTION is specified check that it's compatible with the + ;; purpose of this function, otherwise make it default to :OUTPUT + (aif (getf open-args :direction) + (unless (member it '(:output :io)) + (error "Can't create temporary file with open direction ~A." it)) + (setf open-args (append '(:direction :output) + open-args))) + (do* ((name #1=(temp-file-name default-pathname) #1#) + (stream #2=(apply #'open name + :if-exists nil + :if-does-not-exist :create + open-args) #2#)) + (stream stream))) + +(defmacro with-temp-file ((stream &rest open-temp-args) &body body) + "Execute BODY within a dynamic extent where STREAM is bound to +a STREAM open on a unique temporary file name. OPEN-TEMP-ARGS are +passed verbatim to OPEN-TEMP-FILE." + `(be ,stream (open-temp-file ,@open-temp-args) + (unwind-protect + (progn ,@body) + (close ,stream) + ;; body may decide to rename the file so we must ignore the errors + (ignore-errors + (delete-file (pathname ,stream)))))) + +(defmacro with-hidden-temp-file ((stream &rest open-args) &body body) + "Just like WITH-TEMP-FILE but unlink (delete) the temporary file +before the execution of BODY. As such BODY won't be able to +manipulate the file but through STREAM, and no other program is able +to see it. Once STREAM is closed the temporary file blocks are +automatically relinquished by the operating system. This works at +least on Unix filesystems. I don't know about MS-OSs where the system +may likely decide to crash, take all your data with it and, in the +meanwhile, report you to the NSA as terrorist." + `(be ,stream (open-temp-file ,@open-args) + (unwind-protect + (progn (delete-file (pathname ,stream)) + ,@body) + (close ,stream)))) + +(defun insert-in-order (item seq &key (test #'<) key) + "Destructively insert ITEM in LIST in order by TEST. Return +the new list. This is a simple wrapper around MERGE." + (merge (if seq + (type-of seq) + 'list) + (list item) seq test :key key)) + +(defmacro f++ (x &optional (delta 1)) + "Same as INCF but hopefully optimised for fixnums." + `(setf ,x (+ (the fixnum ,x) (the fixnum ,delta)))) + +(defun soundex (word &optional (key-length 4)) + "Knuth's Soundex algorithm. Returns a string representing the +sound of a certain word (English). Different words will thus +yield the same output string. To compare two string by the +sound, simply do: + + (string= (soundex str1) (soundex str2)) + +Examples: + + (soundex \"Knuth\") => \"K530\" + (soundex \"Kant\") => \"K530\" + (soundex \"Lloyd\") => \"L300\" + (soundex \"Ladd\") => \"L300\"" + (declare (type string word)) + (flet ((translate-char (char) + (awhen (position char "BFPVCGJKQSXZDTLMNR") + (elt "111122222222334556" it)))) + (let ((key (make-string key-length :initial-element #\0)) + (word-length (length word))) + (setf (elt key 0) (elt word 0)) + (loop + with previous-sound = (translate-char (char-upcase (elt word 0))) + with j = 1 + for i from 1 by 1 below word-length + for c = (char-upcase (elt word i)) + while (< j key-length) + do (be sound (translate-char c) + (cond ((not (eq sound previous-sound)) + (unless (member c '(#\H #\W)) + (setf previous-sound sound)) + (when sound + (setf (elt key j) sound) + (incf j)))))) + key))) + +(defun string-soundex= (string1 string2) + (let ((l1 (split-at +whitespace+ string1)) + (l2 (split-at +whitespace+ string2))) + (and (= (length l1) (length l2)) + (every #'string= (mapcar #'soundex l1) (mapcar #'soundex l2))))) + +#+(OR) +(defun soundex-test () + (let* ((words1 '("Euler" "Gauss" "Hilbert" "Knuth" "Lloyd" "Lukasiewicz" "Wachs")) + (words2 '("Ellery" "Ghosh" "Heilbronn" "Kant" "Ladd" "Lissajous" "Waugh")) + (results '("E460" "G200" "H416" "K530" "L300" "L222" "W200"))) + (mapc #'(lambda (w1 w2 r) + (let ((r1 (soundex w1)) + (r2 (soundex w2))) + (format t "~A = ~A, ~A = ~A => ~A~%" w1 r1 w2 r2 + (if (and (string= r1 r2) + (string= r r1)) + "OK" + (format nil "ERROR (expected ~A)" r))))) + words1 words2 results) + (values))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; (defstruct cache-slot () +;; ((previous :type (or cache-slot null) +;; :initarg :previous +;; :initform nil +;; :accessor cslot-previous) +;; (key :initarg :key +;; :accessor cslot-key) +;; (value :initarg :value +;; :accessor cslot-value) +;; (next :type (or cache-slot null) +;; :initarg :next +;; :initform nil +;; :accessor cslot-next))) + +;; (defmethod print-object ((object cache-slot) stream) +;; (print-unreadable-object (object stream :type t) +;; (if (slot-boundp object 'key) +;; (format stream "key=~S, value=~S" (cslot-key object) (cslot-value object)) +;; (format stream "NULL")))) + + +(defstruct (double-linked-element (:conc-name dle-)) + (previous nil :type (or double-linked-element null)) + value + (next nil :type (or double-linked-element null))) + +(defmethod print-object ((object double-linked-element) stream) + (print-unreadable-object (object stream :type t) + (format stream "value=~S" (dle-value object)))) + +(defun cons-dle (value previous next) + (declare (type (or double-linked-element null) previous next)) + (be new-element (make-double-linked-element :previous previous :next next :value value) + (when previous + (setf (dle-next previous) new-element)) + (when next + (setf (dle-previous next) new-element)) + new-element)) + +(defun dle-remove (dle-object) + "Remove the DLE-OBJECT from its current position in the list of +elements agjusting the pointer of dle-objects before and after this +one (if any)." + (declare (type double-linked-element dle-object)) + (awhen (dle-next dle-object) + (setf (dle-previous it) (dle-previous dle-object))) + (awhen (dle-previous dle-object) + (setf (dle-next it) (dle-next dle-object)))) + +(defun dle-map (function dle-object) + (when dle-object + (make-double-linked-element :value (funcall function (dle-value dle-object)) + :previous (dle-previous dle-object) + :next (dle-map function (dle-next dle-object))))) + +(defmacro do-dle ((var dle &optional (result nil)) &body body) + "Iterate over a list of DOUBLE-LINKED-ELEMENTs and map body to +each element's value. Bind VAR to the value on each iteration." + (be cursor (gensym) + `(do ((,cursor ,dle (dle-next ,cursor))) + ((not ,cursor) ,result) + (be ,var (dle-value ,cursor) + ,@body)))) + +(defmacro do-dle* ((var dle &optional (result nil)) &body body) + "Same as DO-DLE but VAR is a symbol macro, so that BODY can +modify the element's value." + (be cursor (gensym) + `(symbol-macrolet ((,var (dle-value ,cursor))) + (do ((,cursor ,dle (dle-next ,cursor))) + ((not ,cursor) ,result) + ,@body)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass double-linked-list () + ((elements :type double-linked-element + :documentation "The actual list of elements held by this object.") + (last-element :type double-linked-element)) + (:documentation + "A double linked list where elements can be added or removed +from either end.")) + +(defmethod initialize-instance ((object double-linked-list) &rest rest) + (declare (ignorable rest)) + (call-next-method) + (with-slots (last-element elements) object + (setf last-element (make-double-linked-element) + elements last-element))) + +(defmethod print-object ((object double-linked-list) stream) + (print-unreadable-object (object stream :type t) + (be elements '() + (do-dle (e (slot-value object 'elements)) + (push e elements)) + (format stream "elements=~S" (nreverse elements))))) + +(defgeneric pop-first (double-linked-list) + (:documentation + "Pop the first element of a double-linked-list.")) +(defgeneric pop-last (double-linked-list) + (:documentation + "Pop the last element of a double-linked-list.")) +(defgeneric push-first (item double-linked-list) + (:documentation + "Push an item in front of a double-linked-list.")) +(defgeneric push-last (item double-linked-list) + (:documentation + "Append an item to a double-linked-list.")) +(defgeneric list-map (function double-linked-list) + (:documentation + "Map a function to a double-linked-list.")) +(defgeneric dll-find-cursor (object dll &key test key)) +(defgeneric dll-find (object dll &key test key)) +(defgeneric dll-remove (cursor dll)) + +(defmethod pop-last ((list double-linked-list)) + "Drop the last element in the dl list." + (with-slots (last-element) list + (awhen (dle-previous last-element) + (dle-remove it) + (dle-value it)))) + +(defmethod pop-first ((list double-linked-list)) + "Drop the first element in the dl list." + (with-slots (elements) list + (when (dle-next elements) + (prog1 (dle-value elements) + (setf (dle-previous (dle-next elements)) nil + elements (dle-next elements)))))) + +(defmethod push-first (value (list double-linked-list)) + (with-slots (elements) list + (setf elements (cons-dle value nil elements))) + list) + +(defmethod push-last (value (list double-linked-list)) + (with-slots (last-element) list + (cons-dle value (dle-previous last-element) last-element)) + list) + +(defmethod list-map (function (list double-linked-list)) + (labels ((map-dll (dle) + (when (dle-next dle) + (make-double-linked-element + :value (funcall function (dle-value dle)) + :previous (dle-previous dle) + :next (map-dll (dle-next dle)))))) + (map-dll (slot-value list 'elements)))) + +(defmethod dll-find-cursor (object (list double-linked-list) &key (test #'eql) (key #'identity)) + (do ((cursor (slot-value list 'elements) (dle-next cursor))) + ((not (dle-next cursor))) + (be value (dle-value cursor) + (when (funcall test (funcall key value) object) + (return cursor))))) + +(defmethod dll-find (object (list double-linked-list) &key (test #'eql) (key #'identity)) + (awhen (dll-find-cursor object list :test test :key key) + (dle-value it))) + +(defmethod dll-remove ((cursor double-linked-element) (list double-linked-list)) + (with-slots (elements) list + (if (dle-previous cursor) + (dle-remove cursor) + (setf (dle-previous (dle-next elements)) nil + elements (dle-next elements)))) + list) + +(defmacro do-dll ((var list &optional (result nil)) &body body) + "Iterate over a dll and map body to each element's +value. Bind VAR to the value on each iteration." + (be cursor (gensym) + `(do ((,cursor (slot-value ,list 'elements) (dle-next ,cursor))) + ((not (dle-next ,cursor)) ,result) + (be ,var (dle-value ,cursor) + ,@body)))) + +(defmacro do-dll* ((var list &optional (result nil)) &body body) + "Same as DO-DLL but VAR is a symbol macro, so that BODY can +modify the element's value." + (be cursor (gensym) + `(symbol-macrolet ((,var (dle-value ,cursor))) + (do ((,cursor (slot-value ,list 'elements) (dle-next ,cursor))) + ((not (dle-next ,cursor)) ,result) + ,@body)))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass limited-list (double-linked-list) + ((max-size :initform nil + :initarg :size + :reader max-size + :type (or integer null) + :documentation "Size limit to which the list is allowed to grow to. NIL = no limit.") + (size :initform 0 + :reader size + :type integer + :documentation "Current number of elements in the list.")) + (:documentation + "A double linked list where the maximum number of elements can +be limited.")) + +(defun dll-member-p (dle list) + (with-slots (elements size) list + (do ((e elements (dle-next e))) + ((not e)) + (when (eq e dle) + (return t))))) + +(defmethod dll-remove ((cursor double-linked-element) (list limited-list)) + (with-slots (size) list + (unless (zerop size) + (decf size) + (call-next-method))) + list) + +(defmethod pop-first ((list limited-list)) + (with-slots (size) list + (unless (zerop size) + (decf size) + (call-next-method)))) + +(defmethod pop-last ((list limited-list)) + (with-slots (size) list + (unless (zerop size) + (decf size) + (call-next-method)))) + +(defmethod push-first (value (list limited-list)) + "Add in front of the list and drop the last element if list is +full." + (declare (ignore value)) + (prog1 (call-next-method) + (with-slots (max-size size last-element) list + (if (or (not max-size) + (< size max-size)) + (incf size) + (dle-remove (dle-previous last-element)))))) + +(defmethod push-last (value (list limited-list)) + "Add at the end of the list and drop the first element if list +is full." + (declare (ignore value)) + (prog1 (call-next-method) + (with-slots (max-size size elements) list + (if (or (not max-size) + (< size max-size)) + (incf size) + (setf (dle-previous (dle-next elements)) nil + elements (dle-next elements)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass sorted-list (limited-list) + ((test :type function + :initarg :test)) + (:documentation + "A double linked list where elements are inserted in a +sorted order.")) + +(defgeneric insert (item sorted-list) + (:documentation + "Insert an item in a sorted-list.")) + +(defmethod insert (item (sl sorted-list)) + "Insert ITEM in SL, which is a sorted double linked list, +before the item for which TEST is true or at the end of the list. +Returns two values, the modified list and the cursor to the new +element." + (with-slots (max-size size elements test last-element) sl + (do ((cursor elements (dle-next cursor))) + ((or (not (dle-next cursor)) + (funcall test item (dle-value cursor))) + (if (dle-previous cursor) + (cons-dle item (dle-previous cursor) cursor) + (setf elements (cons-dle item nil cursor))) + (if (or (not max-size) + (< size max-size)) + (incf size) + (dle-remove (dle-previous last-element))) + (values sl (dle-previous cursor)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defclass heap () + ((less-than :type function + :initarg :test + :documentation "The heap invariant.") + (data :type array + :documentation "The heap tree representation."))) + +(defmethod initialize-instance ((heap heap) &rest args) + (declare (ignore args)) + (call-next-method) + (with-slots (data) heap + (setf data (make-array 0 :fill-pointer 0 :adjustable t)))) + +(defgeneric heap-add (heap item)) + +(defun bubble-up (heap pos) + (with-slots (data less-than) heap + (loop + for current = pos then parent + for parent = (truncate (1- current) 2) + until (or (zerop current) + (funcall less-than (aref data parent) (aref data current))) + do (rotatef (aref data current) (aref data parent))))) + +(defmethod heap-add ((heap heap) item) + (with-slots (data) heap + (vector-push-extend item data) + (bubble-up heap (1- (fill-pointer data))))) + +(defgeneric heap-size (heap)) + +(defmethod heap-size ((heap heap)) + (fill-pointer (slot-value heap 'data))) + +(defgeneric heap-empty-p (heap)) + +(defmethod heap-empty-p ((heap heap)) + (zerop (heap-size heap))) + + +(defgeneric heap-pop (heap)) + +(defun percolate-down (heap pos) + (with-slots (data less-than) heap + (loop + with end = (fill-pointer data) + for current = pos then child + for left-child = (+ 1 (* 2 current)) + for right-child = (+ 2 (* 2 current)) + for child = (cond ((>= left-child end) + (return)) + ((>= right-child end) + left-child) + ((funcall less-than (aref data left-child) (aref data right-child)) + left-child) + (t + right-child)) + while (funcall less-than (aref data child) (aref data current)) + do (rotatef (aref data current) (aref data child))))) + +(defmethod heap-pop ((heap heap)) + (assert (not (heap-empty-p heap))) + (with-slots (data) heap + (be root (aref data 0) + (setf (aref data 0) (vector-pop data)) + (percolate-down heap 0) + root))) + + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defstruct (lru-cache-slot (:include double-linked-element) + (:conc-name lruc-slot-)) + key) + +(defmethod print-object ((object lru-cache-slot) stream) + (print-unreadable-object (object stream :type t) + (format stream "key=~S value=~S" (lruc-slot-key object) (lruc-slot-value object)))) + +(defvar *default-cache-size* 100 + "Default size of a LRU cache if it's not specified at instantiation +time.") + +(defclass lru-cache () + ((max-size :initform *default-cache-size* + :initarg :size + :reader max-size + :type (or integer null) + :documentation + "Maximum number of elements that the cache can fit.") + (elements-list :type lru-cache-slot + :documentation "The list of elements held by the cache.") + (elements-hash :type hash-table + :documentation "The hash table of the elements held bye the cache.") + (last-element :type lru-cache-slot) + (size :initform 0 + :reader size + :type integer + :documentation "Current number of elements in the cache.") + (finalizer :initform nil + :initarg :finalizer + :documentation + "Procedure to call when elements are dropped from cache.")) + (:documentation + "An objects cache that keeps the elements used more often and +drops those that are used less often. The usage is similar to an +hash table. Elements are added to the list up to MAX-SIZE, then +any new element will drop the less used one in the cache. Every +time an element is set or retrieved it goes in front of a list. +Those which get at the end of the list are dropped when more room +is required.")) + +(defmethod initialize-instance ((object lru-cache) &key test &allow-other-keys) + (call-next-method) + (with-slots (last-element elements-list elements-hash) object + (setf last-element (make-lru-cache-slot) + elements-list last-element + elements-hash (if test + (make-hash-table :test test) + (make-hash-table))))) + +(defgeneric getcache (key cache) + (:documentation + "Get an item with KEY from a CACHE.")) + +(defgeneric (setf getcache) (value key cache) + (:documentation + "Set or add an item with KEY in a CACHE.")) + +(defgeneric remcache (key cache) + (:documentation + "Remove an item with KEY from a CACHE.")) + +(defun move-in-front-of-cache-list (slot cache) + "Relocate slot to the front of the elements list in cache. +This will stretch its lifespan in the cache." + (declare (type lru-cache-slot slot) + (type lru-cache cache)) + (with-slots (elements-list) cache + ;; unless it's already the first + (unless (eq slot elements-list) + ;; remove the slot from its original place... + (dle-remove slot) + ;; ... and add it in front of the list + (setf (lruc-slot-next slot) elements-list + (lruc-slot-previous slot) nil + (lruc-slot-previous elements-list) slot + elements-list slot)))) + +(defun drop-last-cache-element (cache) + "Drop the last element in the list of the cache object." + (declare (type lru-cache cache)) + (with-slots (last-element elements-hash finalizer) cache + (let ((second-last (lruc-slot-previous last-element))) + (assert second-last) + (when finalizer + (funcall finalizer (lruc-slot-value second-last))) + (dle-remove second-last) + (remhash (lruc-slot-key second-last) elements-hash)))) + +(defun add-to-cache (slot cache) + (declare (type lru-cache-slot slot) + (type lru-cache cache)) + (move-in-front-of-cache-list slot cache) + (with-slots (max-size size elements-hash) cache + (setf (gethash (lruc-slot-key slot) elements-hash) slot) + (if (and max-size + (< size max-size)) + (incf size) + (drop-last-cache-element cache)))) + +(defmethod getcache (key (cache lru-cache)) + (multiple-value-bind (slot found?) (gethash key (slot-value cache 'elements-hash)) + (when found? + (move-in-front-of-cache-list slot cache) + (values (lruc-slot-value slot) t)))) + +(defmethod (setf getcache) (value key (cache lru-cache)) + (with-slots (elements-hash elements-list) cache + (multiple-value-bind (slot found?) (gethash key elements-hash) + (if found? + (progn + (move-in-front-of-cache-list slot cache) + (setf (lruc-slot-value slot) value)) + (add-to-cache (make-lru-cache-slot :key key :value value) cache)) + value))) + +(defmethod remcache (key (cache lru-cache)) + (with-slots (elements-hash size elements-list finalizer) cache + (multiple-value-bind (slot found?) (gethash key elements-hash) + (when found? + (remhash key elements-hash) + (when finalizer + (funcall finalizer (lruc-slot-value slot))) + (when (eq slot elements-list) + (setf elements-list (dle-next slot))) + (dle-remove slot) + (decf size) + t)))) + +(defmacro cached (cache key value) + "If KEY is found in CACHE return the associated object. Otherwise +store VALUE for later re-use." + (with-gensyms (object my-cache my-key my-value found?) + `(let* ((,my-cache ,cache) + (,my-key ,key)) + (multiple-value-bind (,object ,found?) (getcache ,my-key ,my-cache) + (if ,found? + ,object + (let ((,my-value ,value)) + (setf (getcache ,my-key ,my-cache) ,my-value) + ,my-value)))))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + + +(declaim (inline list->string)) +(defun list->string (list) + "Coerce a list of characters into a string." + (coerce list 'string)) + +(defun setuid (id) + "Set the Unix real user id." + (when (stringp id) + (setf id (find-uid id))) + #+sbcl (sb-posix:setuid id) + #+cmu (unix:unix-setuid id) + #+clisp (posix::%setuid id) ; not verified -wcp26/8/09. + #-(or cmu sbcl clisp) + (error "setuid unsupported under this Lisp implementation")) + +(defun seteuid (id) + "Set the Unix effective user id." + (when (stringp id) + (setf id (find-uid id))) + #+sbcl (sb-posix:seteuid id) + #+cmu (unix:unix-setreuid -1 id) + #+clisp (posix::%seteuid id) ; not verified -wcp26/8/09. + #-(or cmu sbcl clisp) + (error "seteuid unsupported under this Lisp implementation")) + +(defun find-uid (name) + "Find the user id of NAME. Return an integer." + #+sbcl (awhen (sb-posix:getpwnam name) + (sb-posix:passwd-uid it)) + #+cmu (awhen (unix:unix-getpwnam name) + (unix:user-info-uid it)) + #-(or cmu sbcl) + (error "Unable to find a UID on this Lisp system.")) + +#+clisp (ffi:def-call-out %getuid + (:name "getuid") + (:arguments) + (:return-type ffi:int) + (:library "libc.so")) + +(defun getuid () + "Return the Unix user id. This is an integer." + #+sbcl (sb-unix:unix-getuid) + #+cmu (unix:unix-getuid) + #+clisp (%getuid) + #-(or cmu sbcl clisp) + (error "getuid unsupported under this Lisp implementation")) + +(defun super-user-p (&optional id) + "Return true if the user ID is zero. ID defaults to the current +user id." + (zerop (or id (getuid)))) + +(defmacro with-euid (uid &body forms) + "Switch temporarely to Unix user id UID, while performing FORMS." + (with-gensyms (ruid) + `(be ,ruid (getuid) + (seteuid ,uid) + (unwind-protect (progn ,@forms) + (seteuid ,ruid))))) + +(defun get-logname (&optional uid) + "Return the login id of the user. This is a string and it is not +the Unix uid, which is a number." + (unless uid + (setf uid (getuid))) + (when (stringp uid) + (setf uid (find-uid uid))) + (when uid + #+sbcl (sb-unix:uid-username uid) + #+cmu (unix:user-info-name (unix:unix-getpwuid uid)) + #+clisp (posix:user-info-login-id (posix:user-info uid)) + #-(or cmu sbcl clisp) + (error "get-logname unsupported under this Lisp implementation"))) + +(defun get-user-name (&optional uid) + "Return the user name, taken from the GCOS field of the /etc/passwd +file." + (unless uid + (setf uid (getuid))) + (when (stringp uid) + (setf uid (find-uid uid))) + (when uid + (car (split-string-at-char #+cmu (unix:user-info-gecos (unix:unix-getpwuid uid)) + #+sbcl (sb-posix:passwd-gecos (sb-posix:getpwuid uid)) + #-(or cmu sbcl) (error "can't getpwuid() on this Lisp system.") + #\,)))) + +(defun get-user-home (&optional uid) + (unless uid + (setf uid (getuid))) + (when (stringp uid) + (setf uid (find-uid uid))) + (when uid + #+cmu (unix:user-info-dir (unix:unix-getpwuid uid)) + #+sbcl (sb-posix:passwd-dir (sb-posix:getpwuid uid)))) + +;; Rather stupid, but the mnemonic is worth it +(declaim (inline alist->plist)) +(defun alist->plist (alist) + "Convert an association list into a property list. The alist +elements are assumed to be lists of just two elements: the key +and the value. If the element list is longer this function +doesn't work." + (mapcan #'identity alist)) + +(defun plist->alist (plist &optional pairs-p) + "Convert a property list into an association list. The alist +elements wiil be lists of just two elements: the key and the +value. If PAIRS-P is true the alist elements will be pairs." + (loop + for (key val) on plist by #'cddr + collect (if pairs-p + (cons key val) + (list key val)))) + +(defun string->byte-vector (string &key start end) + "Convert a string of characters into a vector of (unsigned-byte +8) elements." + (map '(vector (unsigned-byte 8)) #'char-code + (if (or start end) + (subseq string (or start 0) end) + string))) + +(defun byte-vector->string (vector &key start end) + "Convert a vector of (unsigned-byte 8) elements into a string +of characters." + (map 'string #'code-char + (if (or start end) + (subseq vector (or start 0) end) + vector))) + +(defun outdated-p (file dependencies) + "Check if FILE has been modified before any of its +DEPENDENCIES." + (be epoch (and (probe-file file) + (file-write-date file)) + ;; if file is missing altogether, we consider it outdated + (or (not epoch) + (loop + for dep in dependencies + thereis (aand (probe-file dep) + (file-write-date dep) + (> it epoch)))))) + +(defmacro let-places (places-and-values &body body) + "Execute BODY binding temporarily some places to new values and +restoring the original values of these places on exit of BODY. The +syntax of this macro is identical to LET. The difference is that +instead of new variable names this macro binds values to existing +places (variables)." + (be tmp-variables (loop for x in places-and-values collect (gensym)) + `(let ,(mapcar #'(lambda (tmp-var place-and-value) + (list tmp-var (car place-and-value))) + tmp-variables places-and-values) + (unwind-protect + (progn + ;; as some assignments could signal an error, we assign + ;; within the unwind-protect block so that we can always + ;; guarantee a consistent state on exit + ,@(mapcar #'(lambda (place-and-value) + `(setf ,(car place-and-value) ,(cadr place-and-value))) + places-and-values) + ,@body) + ,@(mapcar #'(lambda (tmp-var place-and-value) + `(setf ,(car place-and-value) ,tmp-var)) + tmp-variables + places-and-values))))) + +(defmacro let-slots (accessor/new-value-pairs object &body body) + "Execute BODY with some OBJECT's slots temporary sets to new +values as described in ACCESSOR/NEW-VALUE-PAIRS. The latter +should be an alist of accessor names and the value to be assigned +to that slot. On exit from BODY, those slots are restored to +their original value. See LET-PLACES." + (with-gensyms (obj) + `(be ,obj ,object + (let-places ,(mapcar #'(lambda (av) + `((,(car av) ,obj) ,(cadr av))) + accessor/new-value-pairs) + ,@body)))) + +(defvar *decimal-point* #\.) +(defvar *thousands-comma* #\,) + +(defun format-amount (number &key (decimals 2) (rounder #'round) + (comma *thousands-comma*) (comma-stance 3) + (decimal-point *decimal-point*)) + "Return a string formatted as fixed decimal point number of DECIMALS +adding commas every COMMA-STANCE places before the decimal point." + (declare (type number number) + (type fixnum decimals comma-stance) + (type function rounder) + (type character comma decimal-point) + (optimize (speed 3) (safety 0) (debug 0))) + (let* ((int (funcall rounder (* number (expt 10 decimals)))) + (negative (< int 0))) + (declare (integer int)) + (when negative + (setf int (- int))) + (let* ((digits (max (1+ decimals) + (1+ (if (zerop int) + 0 + (truncate (log int 10)))))) + (string-length (+ digits + ;; the minus sign + (if negative 1 0) + ;; the decimal point + (if (zerop decimals) 0 1) + ;; the thousands commas + (1- (ceiling (- digits decimals) comma-stance)))) + (string (make-string string-length)) + (pos (1- string-length))) + (declare (type fixnum pos digits)) + (labels ((add-char (char) + (setf (schar string pos) char) + (decf pos)) + (add-digit () + (add-char (digit-char (mod int 10))) + (setf int (truncate int 10)))) + (unless (zerop decimals) + (loop + for i fixnum from 0 below decimals + do (add-digit)) + (add-char decimal-point)) + (loop + for i fixnum from 1 + do (add-digit) + while (>= pos (if negative 1 0)) + when (zerop (mod i comma-stance)) + do (add-char comma)) + (when negative + (add-char #\-))) + string))) + +(defun parse-amount (string &key (start 0) end) + "Parse STRING as if it was formatted with FORMAT-AMOUNT and return +the parsed number. Return NIL if STRING is malformed. Leading or +trailing spaces must be removed from the string in advance." + (loop + with amount = 0 + with decimals = nil + with negative = (when (and (not (zerop (length string))) + (char= #\- (char string 0))) + (incf start) + t) + for i from start below (or end (length string)) + for c = (char string i) + do (cond ((char= c *decimal-point*) + (if decimals + (return nil) + (setf decimals 0))) + ((char= c *thousands-comma*)) + (t + (be d (digit-char-p c) + (cond ((not d) + (return nil)) + (decimals + (incf decimals) + (incf amount (/ d (expt 10 decimals)))) + (t + (setf amount (+ d (* amount 10)))))))) + finally (return (if negative + (- amount) + amount)))) + +(defmacro with-package (name &body body) + `(let ((*package* (find-package ,name))) + ,@body)) + +(defun bytes-simple-string (n &optional imply-bytes) + "Return a string describing N using a unit of measure multiple +of a byte that is most apporpriate for the magnitude of N. A +kilobyte is 1024 not 1000 bytes, everything follows." + (let* ((kilo 1024) + (mega (* kilo kilo)) + (giga (* kilo mega)) + (tera (* mega mega)) + (peta (* kilo tera))) + (apply #'format nil "~,1F~A" + (cond ((> n (* 2 peta)) + (list (/ n peta) (if imply-bytes "P" "PB"))) + ((> n (* 2 tera)) + (list (/ n tera) (if imply-bytes "T" "TB"))) + ((> n (* 2 giga)) + (list (/ n giga) (if imply-bytes "G" "GB"))) + ((> n (* 2 mega)) + (list (/ n mega) (if imply-bytes "M" "MB"))) + ((> n (* 2 kilo)) + (list (/ n kilo) (if imply-bytes "K" "KB"))) + (t (list n (if imply-bytes "" " bytes"))))))) + +;; WARNING: This function may or may not work on your Lisp system. It +;; all depends on how the OPEN function has been implemented regarding +;; the :IF-EXISTS option. This function requires that OPEN be +;; implemented in a way so that the checking of the existence of file +;; and its open attempt be atomic. If the Lisp OPEN first checks that +;; the file exists and then tries to open it, this function won't be +;; reliable. CMUCL seems to use the O_EXCL open() flag in the right +;; way. So at least on CMUCL this function will work. Same goes for +;; SBCL. +(defun make-lock-files (pathnames &key (sleep-time 7) retries (suspend 13) expiration) + "Create semaphore files. If it can't create all the specified +files in the specified order, it waits SLEEP-TIME seconds and +retries the last file that didn't succeed. You can specify the +number of RETRIES to do until failure is returned. If the number +of retries is NIL this function will retry forever. + +If it tries RETRIES times without success, this function signal +an error and removes all the lock files it created until then. + +All files created by lock file will be read-only. + +If you specify a EXPIRATION then an existing lock file will be +removed by force after EXPIRATION seconds have passed since the +lock file was last modified/created (most likely by some other +program that unexpectedly died without cleaning up its lock +files). After a lock file has been removed by force, a +suspension of SUSPEND seconds is taken into account, in order to +prevent the inadvertent immediate removal of any newly created +lock file by another program." + (be locked '() + (flet ((lock (file) + (when (and expiration + (> (get-universal-time) + (+ (file-write-date file) expiration))) + (delete-file file) + (when suspend + (sleep suspend))) + (do ((i 0 (1+ i)) + (done nil)) + (done) + (unless (or (not retries) + (< i retries)) + (error "Can't create lock file ~S: tried ~A time~:P." file retries)) + (with-open-file (out file :direction :output :if-exists nil) + (cond (out + (format out "Lock file created on ~A~%" (time-string (get-universal-time))) + (setf done t)) + (sleep-time + (sleep sleep-time))))))) + (unwind-protect + (progn + (dolist (file pathnames) + (lock file) + (push file locked)) + (setf locked '())) + (mapc #'delete-file locked))))) + +(defmacro with-lock-files ((lock-files &rest lock-args) &body body) + "Execute BODY after creating LOCK-FILES. Remove the lock files +on exit. LOCK-ARGS are passed to MAKE-LOCK-FILES." + (with-gensyms (files) + `(be ,files (list ,@lock-files) + (make-lock-files ,files ,@lock-args) + (unwind-protect (progn ,@body) + (mapc #'delete-file ,files))))) + +(defun getpid () + #+cmu (unix:unix-getpid) + #+sbcl (sb-unix:unix-getpid) + #+clisp (ext:process-id) + #-(or cmu sbcl clisp) + (error "getpid unsupported under this Lisp implementation")) + +(defmacro on-error (form &body error-forms) + "Execute FORM and in case of error execute ERROR-FORMS too. +This does _not_ stop the error from propagating." + (be done-p (gensym) + `(be ,done-p nil + (unwind-protect + (prog1 + ,form + (setf ,done-p t)) + (unless ,done-p + ,@error-forms))))) + +(defun floor-to (x aim) + "Round X down to the nearest multiple of AIM." + (* (floor x aim) aim)) + +(defun round-to (x aim) + "Round X to the nearest multiple of AIM." + (* (round x aim) aim)) + +(defun ceiling-to (x aim) + "Round X up to the nearest multiple of AIM." + (* (ceiling x aim) aim)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defstruct queue + first + last) + +(defgeneric queue-append (queue objects)) +(defgeneric queue-pop (queue)) +(defgeneric queue-empty-p (queue)) + +(defmethod queue-append ((queue queue) (objects list)) + (cond ((null (queue-first queue)) + (setf (queue-first queue) objects + (queue-last queue) (last objects))) + (t + (setf (cdr (queue-last queue)) objects + (queue-last queue) (last objects)))) + queue) + +(defmethod queue-append ((queue queue) object) + (queue-append queue (list object))) + +(defmethod queue-pop ((queue queue)) + (prog1 (car (queue-first queue)) + (setf (queue-first queue) (cdr (queue-first queue))))) + +(defmethod queue-empty-p ((queue queue)) + (null (queue-first queue))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun package-locked-p (package) + #+sbcl (sb-ext:package-locked-p package) + #+cmu (ext:package-definition-lock package) + #+clisp (ext:package-lock package) + #-(or sbcl cmu clisp) (error "Don't know how to check whether a package might be locked.")) + +(defun forget-documentation (packages) + "Remove documentation from all known symbols in PACKAGES. If +PACKAGES is NIL remove documentations from all packages. This may not +make sense if your Lisp image has been built so that existing objects +don't get garbage collected. It may work for your own code, though. +Locked packages are left alone. If you need to do those too, unlock +them first." + (flet ((forget (symbol) + (dolist (type '(compiler-macro function method-combination setf structure type variable)) + (when (ignore-errors (documentation symbol type)) + (setf (documentation symbol type) nil))))) + (setf packages (mapcar #'(lambda (pkg) + (if (packagep pkg) + (package-name pkg) + (package-name (find-package pkg)))) + packages)) + (setf packages + ;; don't try to modify locked packages + (remove-if #'package-locked-p + (mapcar #'find-package + (or packages + (list-all-packages))))) + (dolist (package packages) + (with-package-iterator (next package :internal :external) + (loop + (multiple-value-bind (more? symbol) (next) + (unless more? + (return)) + (forget symbol))))) + #+(OR) (do-all-symbols (symbol) + (when (member (symbol-package symbol) packages) + (forget symbol)))) + (values)) + +(defun load-compiled (pathname &optional compiled-pathname) + "Make sure to compile PATHNAME before loading it. Don't compile if +the compiled version is more recent than its source." + ;; be tolerant if we didn't get a type + (unless (probe-file pathname) + (setf pathname (merge-pathnames pathname (make-pathname :type "lisp")))) + (if (probe-file pathname) + (progn + (setf compiled-pathname (or compiled-pathname + (compile-file-pathname pathname))) + (when (or (not (probe-file compiled-pathname)) + (< (file-write-date compiled-pathname) + (file-write-date pathname))) + (compile-file pathname)) + (load compiled-pathname)) + (error "Can't load ~A as it doesn't exist." pathname))) + +;; Just a silly mnemonic for those used to lesser languages +(defmacro swap (x y) + "Swap values of places X and Y." + `(rotatef ,x ,y)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defmacro show (&rest things) + "Debugging macro to show the name and content of variables. You can +also specify forms, not just variables." + (let ((*print-pretty* nil)) + `(let ((*print-circle* t)) + (format t ,(format nil "~~&~{~A=~~:W~~%~}" things) + ,@things) + (finish-output) + (values)))) + +(defmacro memoize-function (name &key test) + "Make function NAME memoized. TEST is passed to MAKE-HASH-TABLE." + `(setf (get ',name 'results-hash-table) + (make-hash-table ,@(when test (list :test test))))) + +(defmacro defun-memoized (name args &body forms) + "Define function NAME and make it memoizable. Then the MEMOIZED +macro can be used to call this function and memoize its results. The +function NAME must accept only one argument and return just one +argument; more complicated cases are not considered. The hash table +test function is the default 'EQL." + `(eval-when (:load-toplevel :compile-toplevel) + (defun ,name ,args ,@forms) + (memoize-function ,name))) + +(defmacro memoized (function arg) + "If necessary call FUNCTION passing ARG so that its return value is +memoized. The next time this form is executed with the same argument +value, the memoized result is returned instead of executing FUNCTION." + (with-gensyms (table key result not-found) + `(be* ,key ,arg + ,table (get ',function 'results-hash-table) + ,not-found (list nil) + ,result (gethash ,key ,table ,not-found) + (if (eq ,not-found ,result) + (setf (gethash ,key ,table) + (,function ,key)) + ,result)))) + + +(defmacro save-file-excursion ((stream &optional position) &body forms) + "Execute FORMS returning, on exit, STREAM to the position it was +before FORMS. Optionally POSITION can be set to the starting offset." + (unless position + (setf position (gensym))) + `(be ,position (file-position ,stream) + (unwind-protect (progn ,@forms) + (file-position ,stream ,position)))) + +(defun circular-list (&rest elements) + "Return a circular list of ELEMENTS." + (setf (cdr (last elements)) elements)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun getenv (var) + "Return the string associate to VAR in the system environment." + #+cmu (cdr (assoc (if (symbolp var) + var + (intern var :keyword)) + ext:*environment-list*)) + #+sbcl (sb-ext:posix-getenv (string var)) + #+lispworks (hcl:getenv var) + #+clisp (ext:getenv (string var)) + #-(or cmu sbcl lispworks clisp) + (error "GETENV not implemented for your Lisp system.")) + +#+clisp (ffi:def-call-out %setenv + (:name "setenv") + (:arguments (name ffi:c-string) (value ffi:c-string) (overwrite ffi:int)) + (:return-type ffi:int) + (:library "libc.so")) + +#+clisp (ffi:def-call-out %unsetenv + (:name "unsetenv") + (:arguments (name ffi:c-string)) + (:return-type ffi:int) + (:library "libc.so")) + +(defun setenv (name value &optional (overwrite t)) + (typecase value + (string) + (pathname + (setf value (native-namestring value))) + (t + (setf value (format nil "~A" value)))) + #+sbcl (unless (zerop (sb-posix:setenv name value (if overwrite 1 0))) + (error "unable to setenv ~A: errno=~A." name + (sb-alien:get-errno))) + #+cmu (be key (keywordify name) + (aif (assoc key + ext:*environment-list*) + (when overwrite + (setf (cdr it) value)) + (setf ext:*environment-list* + (cons (cons key value) + ext:*environment-list*)))) + #-(or cmu sbcl) (unless (zerop (%setenv name value (if overwrite 1 0))) + (error "unable to setenv ~A." name))) + +(defun unsetenv (name) + #+sbcl (unless (zerop (sb-posix:unsetenv name)) + (error "unable to unsetenv ~A: errno=~A." name + (sb-alien:get-errno))) + #+cmu (be key (keywordify name) + (setf ext:*environment-list* + (delete-if #'(lambda (e) + (eq (car e) key)) + ext:*environment-list*))) + #-(or cmu sbcl) (unless (zerop (%unsetenv name)) + (error "unable to unsetenv ~A." name))) + +(defun (setf getenv) (value name) + (if value + (setenv name value t) + (unsetenv name))) + +;; in CMUCL it's much easier (see below) +#-cmu +(defmacro with-system-environment ((&rest var-and-values) &body body) + (be gensym-alist (mapcar #'(lambda (vv) + (list (gensym) (string (car vv)) (cadr vv))) + var-and-values) + `(let ,(mapcar #'(lambda (vv) + (destructuring-bind (varsym var value) vv + (declare (ignore value)) + `(,varsym (getenv ,var)))) + gensym-alist) + (unwind-protect + (progn + ,@(mapcar #'(lambda (vv) + (destructuring-bind (varsym var value) vv + (declare (ignore varsym)) + `(setenv ,var ,value))) + gensym-alist) + ,@body) + ,@(mapcar #'(lambda (vv) + (destructuring-bind (varsym var value) vv + (declare (ignore value)) + `(if ,varsym + (setenv ,var ,varsym) + (unsetenv ,var)))) + gensym-alist))))) + +#+cmu +(defmacro with-system-environment ((&rest var-and-values) &body body) + `(let ((ext:*environment-list* + (append (list ,@(mapcar #'(lambda (vv) + (destructuring-bind (variable value) vv + `(cons ,(keywordify variable) + ,value))) + var-and-values)) + ext:*environment-list*))) + ,@body)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(defun last-member (item list &key key (test #'eq)) + "Return the last sublist in LIST that is prefixed by ITEM." + (loop + with l = list and result = nil + for l2 = (member item l :key key :test test) + while l2 + do (setf result l2 + l (cdr l2)) + finally (return result))) + + +(defun glob->regex (string) + "Convert a shell glob expression into a regular expression string." + (with-output-to-string (out) + ;; globs are always anchored to beginning and end + (write-char #\^ out) + (loop + for i from 0 below (length string) + do (be c (char string i) + (cond ((char= c #\\) + (setf c (char string (incf i)))) + ((find c ".+()|^$") + (write-char #\\ out)) + ((char= c #\*) + (write-char #\. out)) + ((char= c #\?) + (setf c #\.))) + (write-char c out))) + (write-char #\$ out))) diff --git a/third_party/lisp/sclf/serial.lisp b/third_party/lisp/sclf/serial.lisp new file mode 100644 index 000000000000..936c61606386 --- /dev/null +++ b/third_party/lisp/sclf/serial.lisp @@ -0,0 +1,62 @@ + ;;; serial.lisp --- serialisation of CLOS objects + + ;;; Copyright (C) 2009 by Walter C. Pelissero + + ;;; Author: Walter C. Pelissero <walter@pelissero.de> + ;;; Project: sclf + +#+cmu (ext:file-comment "$Module: serial.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :sclf) + +(defclass printable-object-mixin () ()) + +(defmacro reconstruct-object (class &rest args) + `(apply #'make-instance ',class ',args)) + +(defun print-readable-instance (object &optional stream) + (unless stream + (setf stream *standard-output*)) + (be class (class-of object) + (pprint-logical-block (stream (copy-list (class-slots class)) :prefix "#.(" :suffix ")") + (flet ((spc () + (write-char #\space stream))) + (write 'reconstruct-object :stream stream) + (spc) + (write (class-name class) :stream stream :escape t :readably t :pretty t) + (pprint-exit-if-list-exhausted) + (spc) + (loop + (be* slot (pprint-pop) + slot-name (slot-definition-name slot) + initarg (car (slot-definition-initargs slot)) + (when (and initarg + (slot-boundp object slot-name)) + (write initarg :stream stream) + (spc) + (when *print-pretty* + (pprint-newline :miser stream)) + (write (slot-value object slot-name) + :stream stream) + (pprint-exit-if-list-exhausted) + (if *print-pretty* + (pprint-newline :linear stream) + (spc))))))))) + +(defmethod print-object ((object printable-object-mixin) stream) + (if *print-readably* + (print-readable-instance object stream) + (call-next-method))) diff --git a/third_party/lisp/sclf/sysproc.lisp b/third_party/lisp/sclf/sysproc.lisp new file mode 100644 index 000000000000..85c2517e0002 --- /dev/null +++ b/third_party/lisp/sclf/sysproc.lisp @@ -0,0 +1,295 @@ +;;; sysproc.lisp --- system processes + +;;; Copyright (C) 2008, 2009, 2010 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: sclf + +#+cmu (ext:file-comment "$Module: sysproc.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :sclf) + +(defvar *bourne-shell* "/bin/sh") + +(defvar *run-verbose* nil + "If true system commands are displayed before execution and standard +error is not discarded.") + +;; +;; SIGINFO is missing in both CMUCL and SBCL +;; + +#+cmu +(eval-when (:compile-toplevel :load-toplevel :execute) + (defconstant unix::siginfo 29) + (defvar siginfo (unix::make-unix-signal :siginfo unix::siginfo "Information")) + (export '(unix::siginfo) "UNIX") + (pushnew siginfo unix::*unix-signals*)) + +#+sbcl (in-package :sb-posix) +#+sbcl +(eval-when (:execute :compile-toplevel :load-toplevel) + (unless (find-symbol "SIGINFO" :sb-posix) + (sb-ext:with-unlocked-packages (:sb-posix) + (defvar siginfo 29) + (export '(SIGINFO))))) +#+sbcl (in-package :sclf) + +(defun signal-number (signal-name) + (ecase signal-name + ((:abrt :abort) + #+cmu unix:sigabrt + #+sbcl sb-posix:sigabrt) + ((:alrm :alarm) + #+cmu unix:sigalrm + #+sbcl sb-posix:sigalrm) + ((:bus :bus-error) + #+cmu unix:sigbus + #+sbcl sb-posix:sigbus) + ((:chld :child) + #+cmu unix:sigchld + #+sbcl sb-posix:sigchld) + ((:cont :continue) + #+cmu unix:sigcont + #+sbcl sb-posix:sigcont) + #+freebsd((:emt :emulate-instruction) + #+cmu unix:sigemt + #+sbcl sb-posix:sigemt) + ((:fpe :floating-point-exception) + #+cmu unix:sigfpe + #+sbcl sb-posix:sigfpe) + ((:hup :hangup) + #+cmu unix:sighup + #+sbcl sb-posix:sighup) + ((:ill :illegal :illegal-instruction) + #+cmu unix:sigill + #+sbcl sb-posix:sigill) + ((:int :interrupt) + #+cmu unix:sigint + #+sbcl sb-posix:sigint) + ((:io :input-output) + #+cmu unix:sigio + #+sbcl sb-posix:sigio) + (:kill + #+cmu unix:sigkill + #+sbcl sb-posix:sigkill) + ((:pipe :broke-pipe) + #+cmu unix:sigpipe + #+sbcl sb-posix:sigpipe) + ((:prof :profiler) + #+cmu unix:sigprof + #+sbcl sb-posix:sigprof) + (:quit + #+cmu unix:sigquit + #+sbcl sb-posix:sigquit) + ((:segv :segmentation-violation) + #+cmu unix:sigsegv + #+sbcl sb-posix:sigsegv) + (:stop + #+cmu unix:sigstop + #+sbcl sb-posix:sigstop) + ((:sys :system-call) + #+cmu unix:sigsys + #+sbcl sb-posix:sigsys) + ((:term :terminate) + #+cmu unix:sigterm + #+sbcl sb-posix:sigterm) + ((:trap) + #+cmu unix:sigtrap + #+sbcl sb-posix:sigtrap) + ((:tstp :terminal-stop) + #+cmu unix:sigtstp + #+sbcl sb-posix:sigtstp) + ((:ttin :tty-input) + #+cmu unix:sigttin + #+sbcl sb-posix:sigttin) + ((:ttou :tty-output) + #+cmu unix:sigttou + #+sbcl sb-posix:sigttou) + ((:urg :urgent) + #+cmu unix:sigurg + #+sbcl sb-posix:sigurg) + ((:usr1 :user1) + #+cmu unix:sigusr1 + #+sbcl sb-posix:sigusr1) + ((:usr2 :user2) + #+cmu unix:sigusr2 + #+sbcl sb-posix:sigusr2) + ((:vtalrm :virtual-timer-alarm) + #+cmu unix:sigvtalrm + #+sbcl sb-posix:sigvtalrm) + ((:winch :window-change :window-size-change) + #+cmu unix:sigwinch + #+sbcl sb-posix:sigwinch) + ((:xcpu :exceeded-cpu) + #+cmu unix:sigxcpu + #+sbcl sb-posix:sigxcpu) + ((:xfsz :exceeded-file-size) + #+cmu unix:sigxfsz + #+sbcl sb-posix:sigxfsz) + ;; oddly this is not defined by neither CMUCL nor SBCL + (:info 29))) + +(defun sysproc-kill (process signal) + (when (keywordp signal) + (setf signal (signal-number signal))) + #+cmu (ext:process-kill process signal) + #+sbcl (sb-ext:process-kill process signal) + #-(or sbcl cmu) (error "Don't know how to kill a process")) + +(defun sysproc-exit-code (process) + #+cmu (ext:process-exit-code process) + #+sbcl (sb-ext:process-exit-code process) + #-(or sbcl cmu) (error "Don't know how to get a process exit code")) + +(defun sysproc-wait (process) + #+cmu (ext:process-wait process) + #+sbcl (sb-ext:process-wait process) + #-(or sbcl cmu) (error "Don't know how to wait a process")) + +(defun sysproc-input (process) + #+cmu (ext:process-input process) + #+sbcl (sb-ext:process-input process) + #-(or sbcl cmu) (error "Don't know how to get the process input")) + +(defun sysproc-output (process) + #+cmu (ext:process-output process) + #+sbcl (sb-ext:process-output process) + #-(or sbcl cmu) (error "Don't know how to get the process output")) + +(defun sysproc-alive-p (process) + #+cmu (ext:process-alive-p process) + #+sbcl (sb-ext:process-alive-p process) + #-(or sbcl cmu) (error "Don't know how to test wether a process might be running")) + +(defun sysproc-pid (process) + #+cmu (ext:process-pid process) + #+sbcl (sb-ext:process-pid process) + #-(or sbcl cmu) (error "Don't know how to get the id of a process")) + +(defun sysproc-p (thing) + #+sbcl (sb-ext:process-p thing) + #+cmu (ext:process-p thing) + #-(or sbcl cmu) (error "Don't know how to figure out whether something is a system process")) + +(defun run-program (program arguments &key (wait t) pty input output error) + "Run PROGRAM with ARGUMENTS (a list) and return a process object." + ;; convert arguments to strings + (setf arguments + (mapcar #'(lambda (item) + (typecase item + (string item) + (pathname (native-namestring item)) + (t (format nil "~A" item)))) + arguments)) + (when *run-verbose* + (unless error + (setf error t)) + (format t "~&; run-pipe ~A~{ ~S~}~%" program arguments)) + #+cmu (ext:run-program program arguments + :wait wait + :pty pty + :input input + :output output + :error (or error *run-verbose*)) + #+sbcl (sb-ext:run-program program arguments + :search t + :wait wait + :pty pty + :input input + :output output + :error (or error *run-verbose*)) + #-(or sbcl cmu) + (error "Unsupported Lisp system.")) + +(defun run-pipe (direction program arguments &key error) + "Run PROGRAM with a list of ARGUMENTS and according to DIRECTION +return the input and output streams and process object of that +process." + (be process (run-program program arguments + :wait nil + :pty nil + :input (when (member direction '(:output :input-output :io)) + :stream) + :output (when (member direction '(:input :input-output :io)) + :stream) + :error error) + (values (sysproc-output process) + (sysproc-input process) + process)) + #-(or sbcl cmu) + (error "Unsupported Lisp system.")) + +(defun exit-code (process) + (sysproc-wait process) + (sysproc-exit-code process)) + +(defun run-shell-command (fmt &rest args) + "Run a Bourne Shell command. Return the exit status of the command." + (run-program *bourne-shell* (list "-c" (apply #'format nil fmt args)))) + +(defun run-async-shell-command (fmt &rest args) + "Run a Bourne Shell command asynchronously. Return a process +object if provided by your Lisp implementation." + (run-program *bourne-shell* (list "-c" (apply #'format nil fmt args)) + :wait nil)) + +(defmacro with-open-pipe ((in out program arguments &key (process (gensym)) error pty) &body forms) + "Run BODY with IN and OUT bound respectively to an input and an +output stream connected to a system process created by running PROGRAM +with ARGUMENTS. If IN or OUT are NIL, then don't create that stream." + (with-gensyms (prg args) + `(be* ,prg ,program + ,args ,arguments + ,process (run-program ,prg ,args + :output ,(case in + ((t nil) in) + (t :stream)) + :input ,(case out + ((t nil) out) + (t :stream)) + :wait nil + :pty ,pty + ,@(when error `(:error ,error))) + (if ,process + (let (,@(case in + ((t nil)) + (t `((,in (sysproc-output ,process))))) + ,@(case out + ((t nil)) + (t `((,out (sysproc-input ,process)))))) + (unwind-protect + (progn + ,@forms) + ,@(case in + ((t nil)) + (t `((close ,in)))) + ,@(case out + ((t nil)) + (t `((close ,out)))) + (when (sysproc-alive-p ,process) + (sysproc-kill ,process :term)))) + (error "unable to run ~A~{ ~A~}." ,prg ,args))))) + + +(defun sysproc-set-signal-callback (signal handler) + "Arrange HANDLER function to be called when receiving the system +signal SIGNAL." + (when (keywordp signal) + (setf signal (signal-number signal))) + #+cmu (system:enable-interrupt signal handler) + #+sbcl (sb-sys:enable-interrupt signal handler) + #-(or cmu sbcl) (error "Don't know how to set a system signal callback.")) diff --git a/third_party/lisp/sclf/time.lisp b/third_party/lisp/sclf/time.lisp new file mode 100644 index 000000000000..ca1e1902a92c --- /dev/null +++ b/third_party/lisp/sclf/time.lisp @@ -0,0 +1,311 @@ +;;; time.lisp --- time primitives + +;;; Copyright (C) 2006, 2007, 2009 by Walter C. Pelissero + +;;; Author: Walter C. Pelissero <walter@pelissero.de> +;;; Project: sclf + +#+cmu (ext:file-comment "$Module: time.lisp $") + +;;; This library is free software; you can redistribute it and/or +;;; modify it under the terms of the GNU Lesser General Public License +;;; as published by the Free Software Foundation; either version 2.1 +;;; of the License, or (at your option) any later version. +;;; This library is distributed in the hope that it will be useful, +;;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +;;; Lesser General Public License for more details. +;;; You should have received a copy of the GNU Lesser General Public +;;; License along with this library; if not, write to the Free +;;; Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA +;;; 02111-1307 USA + +(in-package :sclf) + +(defun year (epoch &optional time-zone) + "Return the year of EPOCH." + (sixth (multiple-value-list (decode-universal-time epoch time-zone)))) + +(defun month (epoch &optional time-zone) + "Return the month of EPOCH." + (fifth (multiple-value-list (decode-universal-time epoch time-zone)))) + +(defun day (epoch &optional time-zone) + "Return the day of EPOCH." + (fourth (multiple-value-list (decode-universal-time epoch time-zone)))) + +(defun week-day (epoch &optional time-zone) + "Return the day of the week of EPOCH." + (seventh (multiple-value-list (decode-universal-time epoch time-zone)))) + +(defun hour (epoch &optional time-zone) + "Return the hour of EPOCH." + (third (multiple-value-list (decode-universal-time epoch time-zone)))) + +(defun minute (epoch &optional time-zone) + "Return the minute of EPOCH." + (second (multiple-value-list (decode-universal-time epoch time-zone)))) + +(defun leap-year-p (year) + "Return true if YEAR is a leap year." + (and (zerop (mod year 4)) + (or (not (zerop (mod year 100))) + (zerop (mod year 400))))) + +(defun last-day-of-month (month year) + "Return the last day of the month as integer." + (be last (elt #(31 28 31 30 31 30 31 31 30 31 30 31) (1- month)) + (if (and (= last 28) + (leap-year-p year)) + (1+ last) + last))) + +(defun add-months (months epoch &optional time-zone) + "Add MONTHS to EPOCH, which is a universal time. MONTHS can be +negative." + (multiple-value-bind (ss mm hh day month year) (decode-universal-time epoch time-zone) + (multiple-value-bind (y m) (floor (+ month months -1) 12) + (let ((new-month (1+ m)) + (new-year (+ year y))) + (encode-universal-time ss mm hh + (min day (last-day-of-month new-month (year epoch))) + new-month + new-year + time-zone))))) + +(defun add-days (days epoch) + "Add DAYS to EPOCH, which is an universal time. DAYS can be +negative." + (+ (* 60 60 24 days) epoch)) + +;; The following two functions are based on Thomas Russ <tar@isi.edu> +;; code which didn't carry any copyright notice, so I assume it was in +;; the public domain. + +(defun iso-time-string (time &key time-zone with-timezone-p basic) + "Return an ISO 8601 string representing TIME. The time zone is +included if WITH-TIMEZONE-P is true." + (flet ((format-timezone (zone) + (if (zerop zone) + "Z" + (multiple-value-bind (h m) (truncate (abs zone) 1.0) + ;; Sign of time zone is reversed in ISO 8601 relative + ;; to Common Lisp convention! + (format nil "~:[+~;-~]~2,'0D:~2,'0D" + (> zone 0) h (round m)))))) + (multiple-value-bind (second minute hour day month year dow dst zone) + (decode-universal-time time time-zone) + (declare (ignore dow dst)) + (if basic + (format nil "~4,'0D~2,'0D~2,'0DT~2,'0D~2,'0D~2,'0D~[~*~;~A~]" + year month day hour minute second + with-timezone-p (format-timezone zone)) + (format nil "~4,'0D-~2,'0D-~2,'0DT~2,'0D:~2,'0D:~2,'0D~:[~*~;~A~]" + year month day hour minute second + with-timezone-p (format-timezone zone)))))) + +(defun parse-iso-time-string (time-string) + "Parse an ISO 8601 formated string and return the universal time. +It can parse the basic and the extended format, but may not be able to +cover all the cases." + (labels ((parse-delimited-string (string delimiter n) + ;; Parses a delimited string and returns a list of + ;; n integers found in that string. + (let ((answer (make-list n :initial-element 0))) + (loop + for i upfrom 0 + for start = 0 then (1+ end) + for end = (position delimiter string :start (1+ start)) + do (setf (nth i answer) + (parse-integer (subseq string start end))) + when (null end) return t) + (values-list answer))) + (parse-fixed-field-string (string field-sizes) + ;; Parses a string with fixed length fields and returns + ;; a list of integers found in that string. + (let ((answer (make-list (length field-sizes) :initial-element 0))) + (loop + with len = (length string) + for start = 0 then (+ start field-size) + for field-size in field-sizes + for i upfrom 0 + while (< start len) + do (setf (nth i answer) + (parse-integer (subseq string start (+ start field-size))))) + (values-list answer))) + (parse-iso8601-date (date-string) + (let ((hyphen-pos (position #\- date-string))) + (if hyphen-pos + (parse-delimited-string date-string #\- 3) + (parse-fixed-field-string date-string '(4 2 2))))) + (parse-iso8601-timeonly (time-string) + (let* ((colon-pos (position #\: time-string)) + (zone-pos (or (position #\- time-string) + (position #\+ time-string))) + (timeonly-string (subseq time-string 0 zone-pos)) + (zone-string (when zone-pos (subseq time-string (1+ zone-pos)))) + (time-zone nil)) + (when zone-pos + (multiple-value-bind (zone-h zone-m) + (parse-delimited-string zone-string #\: 2) + (setq time-zone (+ zone-h (/ zone-m 60))) + (when (char= (char time-string zone-pos) #\-) + (setq time-zone (- time-zone))))) + (multiple-value-bind (hh mm ss) + (if colon-pos + (parse-delimited-string timeonly-string #\: 3) + (parse-fixed-field-string timeonly-string '(2 2 2))) + (values hh mm ss time-zone))))) + (let ((time-separator (position #\T time-string))) + (multiple-value-bind (year month date) + (parse-iso8601-date + (subseq time-string 0 time-separator)) + (if time-separator + (multiple-value-bind (hh mm ss zone) + (parse-iso8601-timeonly + (subseq time-string (1+ time-separator))) + (if zone + ;; Sign of time zone is reversed in ISO 8601 + ;; relative to Common Lisp convention! + (encode-universal-time ss mm hh date month year (- zone)) + (encode-universal-time ss mm hh date month year))) + (encode-universal-time 0 0 0 date month year)))))) + +(defun time-string (time &optional time-zone) + "Return a string representing TIME in the form: + Tue Jan 25 12:55:40 2005" + (multiple-value-bind (ss mm hh day month year week-day) + (decode-universal-time time time-zone) + (format nil "~A ~A ~A ~D:~2,'0D:~2,'0D ~A" + (subseq (week-day->string week-day) 0 3) + (subseq (month->string month) 0 3) + day + hh mm ss + year))) + +(defun beginning-of-month (month year &optional time-zone) + (encode-universal-time 0 0 0 1 month year time-zone)) + +(defun end-of-month (month year &optional time-zone) + (1- (add-months 1 (encode-universal-time 0 0 0 1 month year time-zone)))) + +(defun beginning-of-first-week (year &optional time-zone) + "Return the epoch of the first week of YEAR. As the first week +of the year needs to have Thursday in this YEAR, the returned +time can actually fall in the previous year." + (let* ((Jan-1st (encode-universal-time 0 0 0 1 1 year time-zone)) + (start (- 4 (week-day (add-days 4 Jan-1st))))) + (add-days start Jan-1st))) + +(defun beginning-of-week (week year &optional time-zone) + "Return the epoch of the beginning of WEEK of YEAR." + (add-days (* (1- week) 7) (beginning-of-first-week year time-zone))) + +(defun end-of-week (week year &optional time-zone) + "Return the epoch of the beginning of WEEK of YEAR." + (1- (beginning-of-week (1+ week) year time-zone))) + +(defun end-of-last-week (year &optional time-zone) + "Return the epoch of the last week of YEAR. As the last week +of the year needs to have Thursday in this YEAR, the returned +time can fall in the next year." + (1- (beginning-of-first-week (1+ year) time-zone))) + +(defun seconds-from-beginning-of-the-year (time &optional time-zone) + (- time (encode-universal-time 0 0 0 1 1 (year time) time-zone))) + +(defun day-of-the-year (time &optional time-zone) + "Return the day within the year of TIME starting from 1 up to +365 (or 366)." + (1+ (truncate (seconds-from-beginning-of-the-year time time-zone) + (* 60 60 24)))) + +(defun week (time &optional time-zone) + "Return the number of the week and the year TIME referes to. +Week is an integer from 1 to 52. Due to the way the first week +of the year is calculated a day in one year could actually be in +the last week of the previous or next year." + (let* ((year (year time)) + (start (beginning-of-first-week year time-zone)) + (days-from-start (truncate (- time start) (* 60 60 24))) + (weeks (truncate days-from-start 7)) + (week-number (mod weeks 52))) + (values (1+ week-number) + (cond ((< weeks 0) + (1- year)) + ((> weeks 51) + (1+ year)) + (t year))))) + +(defun week-day->string (day &optional sunday-first) + "Return the weekday string corresponding to DAY number." + (elt (if sunday-first + #("Sunday" "Monday" "Tuesday" "Wednesday" "Thursday" "Friday" "Saturday") + #("Monday" "Tuesday" "Wednesday" "Thursday" "Friday" "Saturday" "Sunday")) + day)) + +(defconst +month-names+ #("January" "February" "March" "April" "May" "June" "July" + "August" "September" "October" "November" "December")) + +(defun month->string (month) + "Return the month string corresponding to MONTH number." + (elt +month-names+ (1- month))) + +(defun month-string->number (month) + (1+ (position month +month-names+ :test #'string-equal))) + +(defun print-time-span (span &optional stream) + "Print in English the time SPAN expressed in seconds." + (let* ((minute 60) + (hour (* minute 60)) + (day (* hour 24)) + (seconds span)) + (macrolet ((split (divisor) + `(when (>= seconds ,divisor) + (prog1 (truncate seconds ,divisor) + (setf seconds (mod seconds ,divisor)))))) + (let* ((days (split day)) + (hours (split hour)) + (minutes (split minute))) + (format stream "~{~A~^ ~}" (remove nil + (list + (when days + (format nil "~D day~:P" days)) + (when hours + (format nil "~D hour~:P" hours)) + (when minutes + (format nil "~D minute~:P" minutes)) + (when (or (> seconds 0) + (= span 0)) + (format nil "~D second~:P" seconds))))))))) + +(defun next-week-day (epoch week-day &optional time-zone) + "Return the universal time of the next WEEK-DAY starting from epoch." + (add-days (mod (- week-day (week-day epoch time-zone)) 7) + epoch)) + +(defun next-monday (epoch &optional time-zone) + "Return the universal time of the next Monday starting from +EPOCH." + (next-week-day epoch 0 time-zone)) + +(defun full-weeks-in-span (start end &optional time-zone) + "Return the number of full weeks in time span START to END. A +full week starts on Monday and ends on Sunday." + (be first-monday (next-monday start time-zone) + (truncate (- end first-monday) (* 7 24 60 60)))) + +(defconst +unix-lisp-time-difference+ + (encode-universal-time 0 0 0 1 1 1970 0) + "Time difference between Unix epoch and Common Lisp epoch. The +former is 1st January 1970, while the latter is the beginning of the +XX century.") + +(defun universal->unix-time (time) + (- time +unix-lisp-time-difference+)) + +(defun unix->universal-time (time) + (+ time +unix-lisp-time-difference+)) + +(defun get-unix-time () + (universal->unix-time (get-universal-time))) diff --git a/third_party/lisp/split-sequence.nix b/third_party/lisp/split-sequence.nix new file mode 100644 index 000000000000..105646386fd3 --- /dev/null +++ b/third_party/lisp/split-sequence.nix @@ -0,0 +1,18 @@ +# split-sequence is a library for, well, splitting sequences apparently. +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/sharplispers/split-sequence.git"; + rev = "41c0fc79a5a2871d16e5727969a8f699ef44d791"; +}; +in depot.nix.buildLisp.library { + name = "split-sequence"; + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "vector.lisp" + "list.lisp" + "extended-sequence.lisp" + "api.lisp" + "documentation.lisp" + ]; +} diff --git a/third_party/lisp/trivial-backtrace/.gitignore b/third_party/lisp/trivial-backtrace/.gitignore new file mode 100644 index 000000000000..391b10e5db65 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/.gitignore @@ -0,0 +1,15 @@ +# really this is private to my build process +make/ +common-lisp.net +.vcs +GNUmakefile +init-lisp.lisp +website/changelog.xml + + +trivial-backtrace.tar.gz +website/output/ +test-results/ +lift-local.config +*.dribble +*.fasl diff --git a/third_party/lisp/trivial-backtrace/COPYING b/third_party/lisp/trivial-backtrace/COPYING new file mode 100644 index 000000000000..3798a6664a3d --- /dev/null +++ b/third_party/lisp/trivial-backtrace/COPYING @@ -0,0 +1,25 @@ +Copyright (c) 2008-2008 Gary Warren King (gwking@metabang.com) + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + + +Copyright (c) 2005-2007 Dr. Edi Weitz + +BSD style license: http://www.opensource.org/licenses/bsd-license.php diff --git a/third_party/lisp/trivial-backtrace/default.nix b/third_party/lisp/trivial-backtrace/default.nix new file mode 100644 index 000000000000..bdd057cade27 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/default.nix @@ -0,0 +1,14 @@ +# Imported from http://common-lisp.net/project/trivial-backtrace/trivial-backtrace.git +{ depot, ... }: + +depot.nix.buildLisp.library { + name = "trivial-backtrace"; + + srcs = [ + ./dev/packages.lisp + ./dev/utilities.lisp + ./dev/backtrace.lisp + ./dev/map-backtrace.lisp + ./dev/fallback.lisp + ]; +} diff --git a/third_party/lisp/trivial-backtrace/dev/backtrace.lisp b/third_party/lisp/trivial-backtrace/dev/backtrace.lisp new file mode 100644 index 000000000000..aa3951e30f9f --- /dev/null +++ b/third_party/lisp/trivial-backtrace/dev/backtrace.lisp @@ -0,0 +1,127 @@ +(in-package #:trivial-backtrace) + +(defun print-condition (condition stream) + "Print `condition` to `stream` using the pretty printer." + (format + stream + "~@<An unhandled error condition has been signalled:~3I ~a~I~:@>~%~%" + condition)) + +(defun print-backtrace (error &key (output *debug-io*) + (if-exists :append) + (verbose nil)) + "Send a backtrace for the error `error` to `output`. + +The keywords arguments are: + + * :output - where to send the output. This can be: + + * a string (which is assumed to designate a pathname) + * an open stream + * nil to indicate that the backtrace information should be + returned as a string + + * if-exists - what to do if output designates a pathname and + the pathname already exists. Defaults to :append. + + * verbose - if true, then a message about the backtrace is sent + to \\*terminal-io\\*. Defaults to `nil`. + +If the `output` is nil, the returns the backtrace output as a +string. Otherwise, returns nil. +" + (when verbose + (print-condition error *terminal-io*)) + (multiple-value-bind (stream close?) + (typecase output + (null (values (make-string-output-stream) nil)) + (string (values (open output :if-exists if-exists + :if-does-not-exist :create + :direction :output) t)) + (stream (values output nil))) + (unwind-protect + (progn + (format stream "~&Date/time: ~a" (date-time-string)) + (print-condition error stream) + (terpri stream) + (print-backtrace-to-stream stream) + (terpri stream) + (when (typep stream 'string-stream) + (get-output-stream-string stream))) + ;; cleanup + (when close? + (close stream))))) + +#+(or mcl ccl) +(defun print-backtrace-to-stream (stream) + (let ((*debug-io* stream)) + (ccl:print-call-history :detailed-p nil))) + +#+allegro +(defun print-backtrace-to-stream (stream) + (with-standard-io-syntax + (let ((*print-readably* nil) + (*print-miser-width* 40) + (*print-pretty* t) + (tpl:*zoom-print-circle* t) + (tpl:*zoom-print-level* nil) + (tpl:*zoom-print-length* nil)) + (cl:ignore-errors + (let ((*terminal-io* stream) + (*standard-output* stream)) + (tpl:do-command "zoom" + :from-read-eval-print-loop nil + :count t + :all t)))))) + +#+lispworks +(defun print-backtrace-to-stream (stream) + (let ((dbg::*debugger-stack* + (dbg::grab-stack nil :how-many most-positive-fixnum)) + (*debug-io* stream) + (dbg:*debug-print-level* nil) + (dbg:*debug-print-length* nil)) + (dbg:bug-backtrace nil))) + +#+sbcl +;; determine how we're going to access the backtrace in the next +;; function +(eval-when (:compile-toplevel :load-toplevel :execute) + (when (find-symbol "*DEBUG-PRINT-VARIABLE-ALIST*" :sb-debug) + (pushnew :sbcl-debug-print-variable-alist *features*))) + +#+sbcl +(defun print-backtrace-to-stream (stream) + (let (#+:sbcl-debug-print-variable-alist + (sb-debug:*debug-print-variable-alist* + (list* '(*print-level* . nil) + '(*print-length* . nil) + sb-debug:*debug-print-variable-alist*)) + #-:sbcl-debug-print-variable-alist + (sb-debug:*debug-print-level* nil) + #-:sbcl-debug-print-variable-alist + (sb-debug:*debug-print-length* nil)) + (sb-debug:backtrace most-positive-fixnum stream))) + +#+clisp +(defun print-backtrace-to-stream (stream) + (system::print-backtrace :out stream)) + +#+(or cmucl scl) +(defun print-backtrace-to-stream (stream) + (let ((debug:*debug-print-level* nil) + (debug:*debug-print-length* nil)) + (debug:backtrace most-positive-fixnum stream))) + + +;; must be after the defun above or the docstring may be wiped out +(setf (documentation 'print-backtrace-to-stream 'function) + "Send a backtrace of the current error to stream. + +Stream is assumed to be an open writable file stream or a +string-output-stream. Note that `print-backtrace-to-stream` +will print a backtrace for whatever the Lisp deems to be the +*current* error. +") + + diff --git a/third_party/lisp/trivial-backtrace/dev/fallback.lisp b/third_party/lisp/trivial-backtrace/dev/fallback.lisp new file mode 100644 index 000000000000..40a5219824e5 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/dev/fallback.lisp @@ -0,0 +1,10 @@ +(in-package #:trivial-backtrace) + +(eval-when (:compile-toplevel :load-toplevel :execute) + (unless (fboundp 'map-backtrace) + (defun map-backtrace (func) + (declare (ignore func)))) + + (unless (fboundp 'print-backtrace-to-stream) + (defun print-backtrace-to-stream (stream) + (format stream "~&backtrace output unavailable.~%")))) diff --git a/third_party/lisp/trivial-backtrace/dev/map-backtrace.lisp b/third_party/lisp/trivial-backtrace/dev/map-backtrace.lisp new file mode 100644 index 000000000000..43eddda47579 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/dev/map-backtrace.lisp @@ -0,0 +1,105 @@ +(in-package #:trivial-backtrace) + +(defstruct frame + func + source-filename + source-pos + vars) + +(defstruct var + name + value) + +(defstruct pos-form-number + number) + +(defmethod print-object ((pos-form-number pos-form-number) stream) + (cond + (*print-readably* (call-next-method)) + (t + (format stream "f~A" (pos-form-number-number pos-form-number))))) + + +(defvar *trivial-backtrace-frame-print-specials* + '((*print-length* . 100) + (*print-level* . 20) + (*print-lines* . 5) + (*print-pretty* . t) + (*print-readably* . nil))) + +(defun print-frame (frame stream) + (format stream "~A:~@[~A:~] ~A: ~%" + (or (ignore-errors (translate-logical-pathname (frame-source-filename frame))) (frame-source-filename frame) "<unknown>") + (frame-source-pos frame) + (frame-func frame)) + (loop for var in (frame-vars frame) + do + (format stream " ~A = ~A~%" (var-name var) + (or (ignore-errors + (progv + (mapcar #'car *trivial-backtrace-frame-print-specials*) + (mapcar #'cdr *trivial-backtrace-frame-print-specials*) + (prin1-to-string + (var-value var)))) + "<error>")))) + +(defun map-backtrace (function) + (impl-map-backtrace function)) + +(defun print-map-backtrace (&optional (stream *debug-io*) &rest args) + (apply 'map-backtrace + (lambda (frame) + (print-frame frame stream)) args)) + +(defun backtrace-string (&rest args) + (with-output-to-string (stream) + (apply 'print-map-backtrace stream args))) + + +#+ccl +(defun impl-map-backtrace (func) + (ccl::map-call-frames (lambda (ptr) + (multiple-value-bind (lfun pc) + (ccl::cfp-lfun ptr) + (let ((source-note (ccl:function-source-note lfun))) + (funcall func + (make-frame :func (ccl::lfun-name lfun) + :source-filename (ccl:source-note-filename source-note) + :source-pos (let ((form-number (ccl:source-note-start-pos source-note))) + (when form-number (make-pos-form-number :number form-number))) + :vars (loop for (name . value) in (ccl::arguments-and-locals nil ptr lfun pc) + collect (make-var :name name :value value))))))))) + +#+sbcl +(defun impl-map-backtrace (func) + (loop for f = (or sb-debug:*stack-top-hint* (sb-di:top-frame)) then (sb-di:frame-down f) + while f + do (funcall func + (make-frame :func + (ignore-errors + (sb-di:debug-fun-name + (sb-di:frame-debug-fun f))) + :source-filename + (ignore-errors + (sb-di:debug-source-namestring (sb-di:code-location-debug-source (sb-di:frame-code-location f)))) + :source-pos + (ignore-errors ;;; XXX does not work + (let ((cloc (sb-di:frame-code-location f))) + (unless (sb-di:code-location-unknown-p cloc) + (format nil "tlf~Dfn~D" + (sb-di:code-location-toplevel-form-offset cloc) + (sb-di:code-location-form-number cloc))))) + :vars + (remove-if 'not + (map 'list (lambda(v) + (ignore-errors + (when (eq :valid + (sb-di:debug-var-validity v (sb-di:frame-code-location f))) + (make-var :name (sb-di:debug-var-symbol v) + :value (sb-di:debug-var-value v f))))) + (ignore-errors (sb-di::debug-fun-debug-vars (sb-di:frame-debug-fun f))))))))) + +#-(or ccl sbcl) +(defun impl-map-backtrace (func) + (declare (ignore func)) + (warn "unable to map backtrace for ~a" (lisp-implementation-type))) \ No newline at end of file diff --git a/third_party/lisp/trivial-backtrace/dev/mucking.lisp b/third_party/lisp/trivial-backtrace/dev/mucking.lisp new file mode 100644 index 000000000000..2be26a5a870e --- /dev/null +++ b/third_party/lisp/trivial-backtrace/dev/mucking.lisp @@ -0,0 +1,75 @@ +(in-package #:metabang.gsn) + +#| +Need to account for different kinds of links + in gsn-nodes-from-json, need to return pairs of node and attributes + +hash-table for nodes to prevent duplicates +queue or stack for nodes to expand +hash-table for links (triples of A link B?) to handle duplicates +|# + +(defgeneric expand-node (context node) + ) + +(defgeneric find-neighbors (context node) + ) + +(defgeneric expand-node-p (context node) + ) + +(defgeneric add-node (context node) + ) + +(defgeneric add-link (context node neighbor direction) + ) + +(defgeneric update-node-data (context node data) + ) + +(defclass abstract-context () + ()) + +(defclass gsn-context (abstract-context) + ()) + +(defparameter +gsn-root+ "http://socialgraph.apis.google.com/") + +(defmethod expand-node ((context abstract-context) node) + (bind (((to from) (find-neighbors context node))) + (dolist (neighbor to) + (add-node context neighbor) + (add-link context node neighbor :to)) + (dolist (neighbor from) + (add-node context neighbor) + (add-link context node neighbor :from)))) + + + +(defmethod find-neighbors ((context gsn-context) node) + (bind (((result headers stream) + (http-get + (format nil "~alookup?edo=1&edi=1&pretty=1&q=~a" + +gsn-root+ node))) + json) + (unwind-protect + (setf json (json:decode-json stream)) + (close strea)) + (update-node-data context node json) + (list (gsn-nodes-from-json json :to) + (gsn-nodes-from-json json :from)))) + +(gsn-nodes-from-json x :from) + +(defun gsn-test (who) + (destructuring-bind (result headers stream) + (http-get + (format nil "http://socialgraph.apis.google.com/lookup?edo=1&edi=1&pretty=1&q=~a" who)) + (declare (ignore result headers)) + (json:decode-json stream))) + +(assoc :nodes_referenced + (assoc :nodes (gsn-test "TWITTER.COM/GWKING") :key #'first)) + + +(setf x (gsn-test "TWITTER.COM/GWKING")) diff --git a/third_party/lisp/trivial-backtrace/dev/packages.lisp b/third_party/lisp/trivial-backtrace/dev/packages.lisp new file mode 100644 index 000000000000..2da49d3d9ba5 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/dev/packages.lisp @@ -0,0 +1,13 @@ +(in-package #:common-lisp-user) + +(defpackage #:trivial-backtrace + (:use #:common-lisp) + (:export #:print-backtrace + #:print-backtrace-to-stream + #:print-condition + #:*date-time-format* + + + #:backtrace-string + #:map-backtrace)) + diff --git a/third_party/lisp/trivial-backtrace/dev/utilities.lisp b/third_party/lisp/trivial-backtrace/dev/utilities.lisp new file mode 100644 index 000000000000..b0a249867aa9 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/dev/utilities.lisp @@ -0,0 +1,104 @@ +(in-package #:trivial-backtrace) + +(defparameter *date-time-format* "%Y-%m-%d-%H:%M" + "The default format to use when printing dates and times. + +* %% - A '%' character +* %d - Day of the month as a decimal number [01-31] +* %e - Same as %d but does not print the leading 0 for days 1 through 9 + [unlike strftime[], does not print a leading space] +* %H - Hour based on a 24-hour clock as a decimal number [00-23] +*%I - Hour based on a 12-hour clock as a decimal number [01-12] +* %m - Month as a decimal number [01-12] +* %M - Minute as a decimal number [00-59] +* %S - Second as a decimal number [00-59] +* %w - Weekday as a decimal number [0-6], where Sunday is 0 +* %y - Year without century [00-99] +* %Y - Year with century [such as 1990] + +This code is borrowed from the `format-date` function in +[metatilities-base][].") + +;; modified from metatilities-base +(eval-when (:compile-toplevel :load-toplevel :execute) + (defmacro generate-time-part-function (part-name position) + (let ((function-name + (intern + (concatenate 'string + (symbol-name 'time) "-" (symbol-name part-name)) + :trivial-backtrace))) + `(eval-when (:compile-toplevel :load-toplevel :execute) + (defun ,function-name + (&optional (universal-time (get-universal-time)) + (time-zone nil)) + ,(format nil "Returns the ~(~A~) part of the given time." part-name) + (nth-value ,position + (apply #'decode-universal-time + universal-time time-zone)))))) + + (generate-time-part-function second 0) + (generate-time-part-function minute 1) + (generate-time-part-function hour 2) + (generate-time-part-function date 3) + (generate-time-part-function month 4) + (generate-time-part-function year 5) + (generate-time-part-function day-of-week 6) + (generate-time-part-function daylight-savings-time-p 7)) + +(defun date-time-string (&key (date/time (get-universal-time)) + (format *date-time-format*)) + (format-date format date/time nil)) + +(defun format-date (format date &optional stream time-zone) + (declare (ignore time-zone)) + (let ((format-length (length format))) + (format + stream "~{~A~}" + (loop for index = 0 then (1+ index) + while (< index format-length) collect + (let ((char (aref format index))) + (cond + ((char= #\% char) + (setf char (aref format (incf index))) + (cond + ;; %% - A '%' character + ((char= char #\%) #\%) + + ;; %d - Day of the month as a decimal number [01-31] + ((char= char #\d) (format nil "~2,'0D" (time-date date))) + + ;; %e - Same as %d but does not print the leading 0 for + ;; days 1 through 9. Unlike strftime, does not print a + ;; leading space + ((char= char #\e) (format nil "~D" (time-date date))) + + ;; %H - Hour based on a 24-hour clock as a decimal number [00-23] + ((char= char #\H) (format nil "~2,'0D" (time-hour date))) + + ;; %I - Hour based on a 12-hour clock as a decimal number [01-12] + ((char= char #\I) (format nil "~2,'0D" + (1+ (mod (time-hour date) 12)))) + + ;; %m - Month as a decimal number [01-12] + ((char= char #\m) (format nil "~2,'0D" (time-month date))) + + ;; %M - Minute as a decimal number [00-59] + ((char= char #\M) (format nil "~2,'0D" (time-minute date))) + + ;; %S - Second as a decimal number [00-59] + ((char= char #\S) (format nil "~2,'0D" (time-second date))) + + ;; %w - Weekday as a decimal number [0-6], where Sunday is 0 + ((char= char #\w) (format nil "~D" (time-day-of-week date))) + + ;; %y - Year without century [00-99] + ((char= char #\y) + (let ((year-string (format nil "~,2A" (time-year date)))) + (subseq year-string (- (length year-string) 2)))) + + ;; %Y - Year with century [such as 1990] + ((char= char #\Y) (format nil "~D" (time-year date))) + + (t + (error "Ouch - unknown formatter '%~c" char)))) + (t char))))))) diff --git a/third_party/lisp/trivial-backtrace/lift-standard.config b/third_party/lisp/trivial-backtrace/lift-standard.config new file mode 100644 index 000000000000..0f22312080cf --- /dev/null +++ b/third_party/lisp/trivial-backtrace/lift-standard.config @@ -0,0 +1,35 @@ +;;; configuration for LIFT tests + +;; settings +(:if-dribble-exists :supersede) +(:dribble "lift.dribble") +(:print-length 10) +(:print-level 5) +(:print-test-case-names t) + +;; suites to run +(trivial-backtrace-test) + +;; report properties +(:report-property :title "Trivial-Backtrace | Test results") +(:report-property :relative-to trivial-backtrace-test) + +(:report-property :style-sheet "test-style.css") +(:report-property :if-exists :supersede) +(:report-property :format :html) +(:report-property :full-pathname "test-results/test-report.html") +(:report-property :unique-name t) +(:build-report) + +(:report-property :unique-name t) +(:report-property :format :describe) +(:report-property :full-pathname "test-results/test-report.txt") +(:build-report) + +(:report-property :format :save) +(:report-property :full-pathname "test-results/test-report.sav") +(:build-report) + +(:report-property :format :describe) +(:report-property :full-pathname *standard-output*) +(:build-report) diff --git a/third_party/lisp/trivial-backtrace/test/packages.lisp b/third_party/lisp/trivial-backtrace/test/packages.lisp new file mode 100644 index 000000000000..7dc3eae57682 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/test/packages.lisp @@ -0,0 +1,5 @@ +(in-package #:common-lisp-user) + +(defpackage #:trivial-backtrace-test + (:use #:common-lisp #:lift #:trivial-backtrace)) + diff --git a/third_party/lisp/trivial-backtrace/test/test-setup.lisp b/third_party/lisp/trivial-backtrace/test/test-setup.lisp new file mode 100644 index 000000000000..a46b3a196649 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/test/test-setup.lisp @@ -0,0 +1,4 @@ +(in-package #:trivial-backtrace-test) + +(deftestsuite trivial-backtrace-test () + ()) diff --git a/third_party/lisp/trivial-backtrace/test/tests.lisp b/third_party/lisp/trivial-backtrace/test/tests.lisp new file mode 100644 index 000000000000..9b32090f13e0 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/test/tests.lisp @@ -0,0 +1,17 @@ +(in-package #:trivial-backtrace-test) + +(deftestsuite generates-backtrace (trivial-backtrace-test) + ()) + +(addtest (generates-backtrace) + test-1 + (let ((output nil)) + (handler-case + (let ((x 1)) + (let ((y (- x (expt 1024 0)))) + (declare (optimize (safety 3))) + (/ 2 y))) + (error (c) + (setf output (print-backtrace c :output nil)))) + (ensure (stringp output)) + (ensure (plusp (length output))))) diff --git a/third_party/lisp/trivial-backtrace/trivial-backtrace-test.asd b/third_party/lisp/trivial-backtrace/trivial-backtrace-test.asd new file mode 100644 index 000000000000..cb088434a2c1 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/trivial-backtrace-test.asd @@ -0,0 +1,22 @@ +(defpackage #:trivial-backtrace-test-system (:use #:asdf #:cl)) +(in-package #:trivial-backtrace-test-system) + +(defsystem trivial-backtrace-test + :author "Gary Warren King <gwking@metabang.com>" + :maintainer "Gary Warren King <gwking@metabang.com>" + :licence "MIT Style License; see file COPYING for details" + :components ((:module + "setup" + :pathname "test/" + :components ((:file "packages") + (:file "test-setup" + :depends-on ("packages")))) + (:module + "test" + :pathname "test/" + :depends-on ("setup") + :components ((:file "tests")))) + :depends-on (:lift :trivial-backtrace)) + + + diff --git a/third_party/lisp/trivial-backtrace/trivial-backtrace.asd b/third_party/lisp/trivial-backtrace/trivial-backtrace.asd new file mode 100644 index 000000000000..843b6cc39a3c --- /dev/null +++ b/third_party/lisp/trivial-backtrace/trivial-backtrace.asd @@ -0,0 +1,35 @@ +(in-package #:common-lisp-user) + +(defpackage #:trivial-backtrace-system (:use #:asdf #:cl)) +(in-package #:trivial-backtrace-system) + +(defsystem trivial-backtrace + :version "1.1.0" + :author "Gary Warren King <gwking@metabang.com> and contributors" + :maintainer "Gary Warren King <gwking@metabang.com> and contributors" + :licence "MIT Style license " + :description "trivial-backtrace" + :depends-on () + :components + ((:static-file "COPYING") + (:module + "setup" + :pathname "dev/" + :components ((:file "packages"))) + (:module + "dev" + :depends-on ("setup") + :components ((:file "utilities") + (:file "backtrace") + (:file "map-backtrace") + (:file "fallback" :depends-on ("backtrace" "map-backtrace"))))) + :in-order-to ((test-op (load-op trivial-backtrace-test))) + :perform (test-op :after (op c) + (funcall + (intern (symbol-name '#:run-tests) :lift) + :config :generic))) + +(defmethod operation-done-p + ((o test-op) + (c (eql (find-system 'trivial-backtrace)))) + (values nil)) diff --git a/third_party/lisp/trivial-backtrace/website/source/index.md b/third_party/lisp/trivial-backtrace/website/source/index.md new file mode 100644 index 000000000000..93a5df3b91db --- /dev/null +++ b/third_party/lisp/trivial-backtrace/website/source/index.md @@ -0,0 +1,88 @@ +{include resources/header.md} + +<div class="contents"> +<div class="system-links"> + + * [Mailing Lists][mailing-list] + * [Getting it][downloads] + * [Documentation][] + * [News][] + * [Test results][tr] + * [Changelog][] + +</div> +<div class="system-description"> + +### What it is + +On of the many things that didn't quite get into the Common +Lisp standard was how to get a Lisp to output its call stack +when something has gone wrong. As such, each Lisp has +developed its own notion of what to display, how to display +it, and what sort of arguments can be used to customize it. +`trivial-backtrace` is a simple solution to generating a +backtrace portably. As of {today}, it supports Allegro Common +Lisp, LispWorks, ECL, MCL, SCL, SBCL and CMUCL. Its +interface consists of three functions and one variable: + + * print-backtrace + * print-backtrace-to-stream + * print-condition + * \*date-time-format\* + +You can probably already guess what they do, but they are +described in more detail below. + +{anchor mailing-lists} + +### Mailing Lists + + * [trivial-backtrace-devel][devel-list]: A list for + announcements, questions, patches, bug reports, and so + on; It's for anything and everything + +### API + +{set-property docs-package trivial-backtrace} +{docs print-backtrace} +{docs print-backtrace-to-stream} +{docs print-condition} +{docs *date-time-format*} + +{anchor downloads} + +### Where is it + +A [git][] repository is available using + + git clone http://common-lisp.net/project/trivial-backtrace/trivial-backtrace.git + +The [darcs][] repository is still around but is **not** being updated. +The command to get it is below: + + ;;; WARNING: out of date + darcs get http://common-lisp.net/project/trivial-backtrace/ + +trivial-backtrace is also [ASDF installable][asdf-install]. +Its CLiki home is right [where][cliki-home] you'd expect. + +There's also a handy [gzipped tar file][tarball]. + +{anchor news} + +### What is happening + +<dl> + <dt>14 May 2009</dt> + <dd>Moved to [git][]; John Fremlin adds map-backtrace + </dd> + +<dt>1 June 2008</dt> +<dd>Release version 1.0 + </dd> + </dl> +</div> +</div> + +{include resources/footer.md} + diff --git a/third_party/lisp/trivial-backtrace/website/source/resources/footer.md b/third_party/lisp/trivial-backtrace/website/source/resources/footer.md new file mode 100644 index 000000000000..c5bf3c4ec399 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/website/source/resources/footer.md @@ -0,0 +1,15 @@ +<div id="footer" class="footer"> +<div id="buttons"> +<a class="nav" href="http://validator.w3.org/check/referer" title="xhtml1.1"><img src="http://common-lisp.net/project/cl-containers/shared/buttons/xhtml.gif" width="80" height="15" title="valid xhtml button" alt="valid xhtml" /></a> +<a class="nav" href="http://common-lisp.net/project/cl-markdown/" title="Mark with CL-Markdown"><img src="http://common-lisp.net/project/cl-containers/shared/buttons/cl-markdown.png" width="80" height="15" title="Made with CL-Markdown" alt="CL-Markdown" /></a> +<a class="nav" href="http://www.catb.org/hacker-emblem/" title="hacker"><img src="http://common-lisp.net/project/cl-containers/shared/buttons/hacker.png" width="80" height="15" title="hacker emblem" alt="hacker button" /></a> +<a class="nav" href="http://www.lisp.org/" title="Association of Lisp Users"><img src="http://common-lisp.net/project/cl-containers/shared/buttons/lambda-lisp.png" width="80" height="15" title="ALU emblem" alt="ALU button" /></a> +<a class="nav" href="http://common-lisp.net/" title="Common-Lisp.net"><img src="http://common-lisp.net/project/cl-containers/shared/buttons/lisp-lizard.png" width="80" height="15" title="Common-Lisp.net" alt="Common-Lisp.net button" /></a> +</div> + +### Copyright (c) 2009 - 2011 Gary Warren King (gwking@metabang.com) + +trivial-backtrace has an [MIT style][mit-license] license + +<div id="timestamp">Last updated {today} at {now}</div> +</div> diff --git a/third_party/lisp/trivial-backtrace/website/source/resources/header.md b/third_party/lisp/trivial-backtrace/website/source/resources/header.md new file mode 100644 index 000000000000..2738c471378c --- /dev/null +++ b/third_party/lisp/trivial-backtrace/website/source/resources/header.md @@ -0,0 +1,19 @@ +{include shared-links.md} + +{set-property html yes} +{set-property style-sheet "styles.css"} +{set-property author "Gary Warren King"} +{set-property title "trivial-backtrace | watch where you've been"} + + [devel-list]: http://common-lisp.net/cgi-bin/mailman/listinfo/trivial-backtrace-devel + [cliki-home]: http://www.cliki.net//trivial-backtrace + [tarball]: http://common-lisp.net/project/trivial-backtrace/trivial-backtrace.tar.gz + +<div id="header"> + <span class="logo"><a href="http://www.metabang.com/" title="metabang.com"><img src="http://common-lisp.net/project/cl-containers/shared/metabang-2.png" title="metabang.com" width="100" alt="Metabang Logo" /></a></span> + +## trivial-backtrace + +#### watch where you've been + +</div> diff --git a/third_party/lisp/trivial-backtrace/website/source/resources/navigation.md b/third_party/lisp/trivial-backtrace/website/source/resources/navigation.md new file mode 100644 index 000000000000..a734edfb8323 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/website/source/resources/navigation.md @@ -0,0 +1,2 @@ +<div id="navigation"> +</div> diff --git a/third_party/lisp/trivial-backtrace/website/website.tmproj b/third_party/lisp/trivial-backtrace/website/website.tmproj new file mode 100644 index 000000000000..01b745ba44c0 --- /dev/null +++ b/third_party/lisp/trivial-backtrace/website/website.tmproj @@ -0,0 +1,93 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>currentDocument</key> + <string>source/resources/header.md</string> + <key>documents</key> + <array> + <dict> + <key>expanded</key> + <true/> + <key>name</key> + <string>source</string> + <key>regexFolderFilter</key> + <string>!.*/(\.[^/]*|CVS|_darcs|_MTN|\{arch\}|blib|.*~\.nib|.*\.(framework|app|pbproj|pbxproj|xcode(proj)?|bundle))$</string> + <key>sourceDirectory</key> + <string>source</string> + </dict> + </array> + <key>fileHierarchyDrawerWidth</key> + <integer>190</integer> + <key>metaData</key> + <dict> + <key>source/index.md</key> + <dict> + <key>caret</key> + <dict> + <key>column</key> + <integer>0</integer> + <key>line</key> + <integer>0</integer> + </dict> + <key>firstVisibleColumn</key> + <integer>0</integer> + <key>firstVisibleLine</key> + <integer>0</integer> + </dict> + <key>source/resources/footer.md</key> + <dict> + <key>caret</key> + <dict> + <key>column</key> + <integer>29</integer> + <key>line</key> + <integer>9</integer> + </dict> + <key>firstVisibleColumn</key> + <integer>0</integer> + <key>firstVisibleLine</key> + <integer>0</integer> + </dict> + <key>source/resources/header.md</key> + <dict> + <key>caret</key> + <dict> + <key>column</key> + <integer>27</integer> + <key>line</key> + <integer>3</integer> + </dict> + <key>firstVisibleColumn</key> + <integer>0</integer> + <key>firstVisibleLine</key> + <integer>0</integer> + </dict> + <key>source/resources/navigation.md</key> + <dict> + <key>caret</key> + <dict> + <key>column</key> + <integer>0</integer> + <key>line</key> + <integer>1</integer> + </dict> + <key>firstVisibleColumn</key> + <integer>0</integer> + <key>firstVisibleLine</key> + <integer>0</integer> + </dict> + </dict> + <key>openDocuments</key> + <array> + <string>source/resources/header.md</string> + <string>source/index.md</string> + <string>source/resources/navigation.md</string> + <string>source/resources/footer.md</string> + </array> + <key>showFileHierarchyDrawer</key> + <true/> + <key>windowFrame</key> + <string>{{615, 0}, {578, 778}}</string> +</dict> +</plist> diff --git a/third_party/lisp/trivial-features.nix b/third_party/lisp/trivial-features.nix new file mode 100644 index 000000000000..13a63bab1aba --- /dev/null +++ b/third_party/lisp/trivial-features.nix @@ -0,0 +1,16 @@ +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/trivial-features/trivial-features.git"; + rev = "f6e8dd7268ae0137dbde4be469101a7f735f6416"; # 2021-02-28 +}; +in depot.nix.buildLisp.library { + name = "trivial-features"; + srcs = [ + { + sbcl = src + "/src/tf-sbcl.lisp"; + ecl = src + "/src/tf-ecl.lisp"; + ccl = src + "/src/tf-openmcl.lisp"; + } + ]; +} diff --git a/third_party/lisp/trivial-garbage.nix b/third_party/lisp/trivial-garbage.nix new file mode 100644 index 000000000000..e5b3550de7ba --- /dev/null +++ b/third_party/lisp/trivial-garbage.nix @@ -0,0 +1,12 @@ +# trivial-garbage provides a portable API to finalizers, weak +# hash-tables and weak pointers +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/trivial-garbage/trivial-garbage.git"; + rev = "dbc8e35acb0176b9a14fdc1027f5ebea93435a84"; +}; +in depot.nix.buildLisp.library { + name = "trivial-garbage"; + srcs = [ (src + "/trivial-garbage.lisp") ]; +} diff --git a/third_party/lisp/trivial-gray-streams.nix b/third_party/lisp/trivial-gray-streams.nix new file mode 100644 index 000000000000..b5722f9a685a --- /dev/null +++ b/third_party/lisp/trivial-gray-streams.nix @@ -0,0 +1,16 @@ +# Portability library for CL gray streams. +{ depot, ... }: + +let src = builtins.fetchGit { + url = "https://github.com/trivial-gray-streams/trivial-gray-streams.git"; + rev = "ebd59b1afed03b9dc8544320f8f432fdf92ab010"; +}; +in depot.nix.buildLisp.library { + name = "trivial-gray-streams"; + srcs = [ + (src + "/package.lisp") + (src + "/streams.lisp") + ]; +} + + diff --git a/third_party/lisp/trivial-indent.nix b/third_party/lisp/trivial-indent.nix new file mode 100644 index 000000000000..65d98604d78c --- /dev/null +++ b/third_party/lisp/trivial-indent.nix @@ -0,0 +1,17 @@ + +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "Shinmera"; + repo = "trivial-indent"; + rev = "2d016941751647c6cc5bd471751c2cf68861c94a"; + sha256 = "1sj90nqz17w4jq0ixz00gb9g5g6d2s7l8r17zdby27gxxh51w266"; + }; +in depot.nix.buildLisp.library { + name = "trivial-indent"; + + srcs = map (f: src + ("/" + f)) [ + "indent.lisp" + ]; +} diff --git a/third_party/lisp/trivial-ldap.nix b/third_party/lisp/trivial-ldap.nix new file mode 100644 index 000000000000..c8a27431c687 --- /dev/null +++ b/third_party/lisp/trivial-ldap.nix @@ -0,0 +1,26 @@ +{ depot, pkgs, ... }: + +let src = pkgs.fetchFromGitHub { + owner = "rwiker"; + repo = "trivial-ldap"; + rev = "3b8f1ff85f29ea63e6ab2d0d27029d68b046faf8"; + sha256 = "1zaa4wnk5y5ff211pkg6dl27j4pjwh56hq0246slxsdxv6kvp1z9"; + }; +in depot.nix.buildLisp.library { + name = "trivial-ldap"; + + deps = with depot.third_party.lisp; [ + usocket + cl-plus-ssl + cl-yacc + ]; + + srcs = map (f: src + ("/" + f)) [ + "package.lisp" + "trivial-ldap.lisp" + ]; + + brokenOn = [ + "ecl" # dynamic cffi + ]; +} diff --git a/third_party/lisp/trivial-mimes.nix b/third_party/lisp/trivial-mimes.nix new file mode 100644 index 000000000000..ce45993d05e9 --- /dev/null +++ b/third_party/lisp/trivial-mimes.nix @@ -0,0 +1,30 @@ +{ depot, pkgs, ... }: + +let + src = pkgs.fetchFromGitHub { + owner = "Shinmera"; + repo = "trivial-mimes"; + rev = "a741fc2f567a4f86b853fd4677d75e62c03e51d9"; + sha256 = "00kcm17q5plpzdj1qwg83ldhxksilgpcdkf3m9azxcdr968xs9di"; + }; + + mime-types = pkgs.runCommand "mime-types.lisp" {} '' + substitute ${src}/mime-types.lisp $out \ + --replace /etc/mime.types ${src}/mime.types \ + --replace "(asdf:system-source-directory :trivial-mimes)" '"/bogus-dir"' + # We want to prevent an ASDF lookup at build time since this will + # generally fail — we are not using ASDF after all. + ''; + +in depot.nix.buildLisp.library { + name = "trivial-mimes"; + + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + ]; + + srcs = [ mime-types ]; +} diff --git a/third_party/lisp/uax-15.nix b/third_party/lisp/uax-15.nix new file mode 100644 index 000000000000..a13e5c1690d3 --- /dev/null +++ b/third_party/lisp/uax-15.nix @@ -0,0 +1,45 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs) runCommand; + inherit (depot.nix.buildLisp) bundled; + + src = pkgs.fetchFromGitHub { + owner = "sabracrolleton"; + repo = "uax-15"; + rev = "a62fc7253663fac6467fc6b6bb69a73a0e8dfaa0"; + sha256 = "028kc3yfi29qjxs2nyr7dbrr2rcrr8lwpvbxvrq3g8bcwamc4jz4"; + }; + + # src = ../../../uax-15; + +in depot.nix.buildLisp.library { + name = "uax-15"; + + deps = with depot.third_party.lisp; [ + split-sequence + cl-ppcre + (bundled "asdf") + ]; + + srcs = [ + "${src}/uax-15.asd" + "${src}/src/package.lisp" + "${src}/src/utilities.lisp" + "${src}/src/trivial-utf-16.lisp" + (runCommand "precomputed-tables.lisp" {} '' + substitute ${src}/src/precomputed-tables.lisp precomputed-tables.lisp \ + --replace "(asdf:system-source-directory (asdf:find-system 'uax-15 nil))" \ + '"${src}/"' + + sed -i precomputed-tables.lisp \ + -e '17i(defvar *canonical-decomp-map*)' \ + -e '17i(defvar *compatible-decomp-map*)' \ + -e '17i(defvar *canonical-combining-class*)' + + cp precomputed-tables.lisp $out + '') + "${src}/src/normalize-backend.lisp" + "${src}/src/uax-15.lisp" + ]; +} diff --git a/third_party/lisp/unix-opts.nix b/third_party/lisp/unix-opts.nix new file mode 100644 index 000000000000..389de25eff4f --- /dev/null +++ b/third_party/lisp/unix-opts.nix @@ -0,0 +1,17 @@ +# unix-opts is a portable command line argument parser +{ depot, pkgs, ...}: + +let + src = pkgs.fetchFromGitHub { + owner = "libre-man"; + repo = "unix-opts"; + rev = "b805050b074bd860edd18cfc8776fdec666ec36e"; + sha256 = "0j93dkc9f77wz1zfspm7q1scx6wwbm6jhk8vl2rm6bfd0n8scxla"; + }; +in depot.nix.buildLisp.library { + name = "unix-opts"; + + srcs = [ + "${src}/unix-opts.lisp" + ]; +} diff --git a/third_party/lisp/usocket.nix b/third_party/lisp/usocket.nix new file mode 100644 index 000000000000..dc4281c795c0 --- /dev/null +++ b/third_party/lisp/usocket.nix @@ -0,0 +1,51 @@ +# Usocket is a portable socket library +{ depot, pkgs, ... }: + +let + inherit (depot.nix) buildLisp; + + src = pkgs.fetchFromGitHub { + owner = "usocket"; + repo = "usocket"; + rev = "fdf4fd1e0051ce83340ccfbbc8a43a462bb19cf2"; + sha256 = "0x746wr2324l6bn7skqzgkzcbj5kd0zp2ck0c8rldrw0rzabg826"; + }; +in buildLisp.library { + name = "usocket"; + deps = with depot.third_party.lisp; [ + (buildLisp.bundled "asdf") + { + ecl = buildLisp.bundled "sb-bsd-sockets"; + sbcl = buildLisp.bundled "sb-bsd-sockets"; + } + split-sequence + ]; + + srcs = [ + # usocket also reads its version from ASDF, but there's further + # shenanigans happening there that I don't intend to support right + # now. Behold: + (builtins.toFile "usocket.asd" '' + (in-package :asdf) + (defsystem usocket + :version "0.8.3") + '') + ] ++ + # Now for the regularly scheduled programming: + (map (f: src + ("/" + f)) [ + "package.lisp" + "usocket.lisp" + "condition.lisp" + ] ++ [ + { sbcl = "${src}/backend/sbcl.lisp"; } + + # ECL actually has two files, it supports the SBCL backend, + # but usocket also has some ECL specific code + { ecl = "${src}/backend/sbcl.lisp"; } + { ecl = "${src}/backend/ecl.lisp"; } + + # Same for CCL + { ccl = "${src}/backend/openmcl.lisp"; } + { ccl = "${src}/backend/clozure.lisp"; } + ]); +} diff --git a/third_party/litestream/default.nix b/third_party/litestream/default.nix new file mode 100644 index 000000000000..20b094ee84ba --- /dev/null +++ b/third_party/litestream/default.nix @@ -0,0 +1,23 @@ +# TODO(tazjin): Upstream this derivation if litestream is good. +{ pkgs, lib, ... }: + +let inherit (pkgs) buildGoModule fetchFromGitHub; +in pkgs.buildGoModule rec { + pname = "litestream"; + version = "0.3.4"; + vendorSha256 = "1dcb5f06cq484i9m9pi5l4banygh3s4bz5964hy7ypmm1z2pcmrv"; + + src = pkgs.fetchFromGitHub { + owner = "benbjohnson"; + repo = "litestream"; + rev = "v${version}"; + sha256 = "1ivyxgmbysir9qsbsz6nksjqzw6ba0zrymacsw12ab9bvbldjxv7"; + }; + + meta = { + description = "Streaming S3 replication for SQLite"; + homepage = "https://litestream.io/"; + license = lib.licenses.asl20; + maintainers = [ lib.maintainers.tazjin ]; + }; +} diff --git a/third_party/naersk/default.nix b/third_party/naersk/default.nix new file mode 100644 index 000000000000..865fcf04d23f --- /dev/null +++ b/third_party/naersk/default.nix @@ -0,0 +1,8 @@ +{ pkgs, ... }: + +pkgs.callPackage (pkgs.fetchFromGitHub { + owner = "nmattia"; + repo = "naersk"; + rev = "a3f40fe42cc6d267ff7518fa3199e99ff1444ac4"; + sha256 = "1nf7fn8anghwf6p5p58ywbcwdkjxq112qv663rn52jq9k95iakdi"; +}) {} diff --git a/third_party/nix/.clang-format b/third_party/nix/.clang-format new file mode 100644 index 000000000000..b8c36e122b04 --- /dev/null +++ b/third_party/nix/.clang-format @@ -0,0 +1,11 @@ +# Use the Google style in this project. +BasedOnStyle: Google +DerivePointerAlignment: false +PointerAlignment: Left +IncludeCategories: + - Regex: '^<.*\.h>' + Priority: 2 + - Regex: '^<.*' + Priority: 1 + - Regex: '.*' + Priority: 3 diff --git a/third_party/nix/.clang-tidy b/third_party/nix/.clang-tidy new file mode 100644 index 000000000000..5b22be767fea --- /dev/null +++ b/third_party/nix/.clang-tidy @@ -0,0 +1,4 @@ +--- +Checks: 'abseil-c*,clang-analyzer-security-*,bugprone-*,google-*,modernize-*,cppcoreguidelines-*,misc-*,-modernize-use-trailing-return-type' +WarningsAsErrors: 'abseil-*,clang-analyzer-security*' +... diff --git a/third_party/nix/.dir-locals.el b/third_party/nix/.dir-locals.el new file mode 100644 index 000000000000..92aa816f100e --- /dev/null +++ b/third_party/nix/.dir-locals.el @@ -0,0 +1 @@ +((c++-mode . ((c-file-style . "google")))) diff --git a/third_party/nix/.github/ISSUE_TEMPLATE.md b/third_party/nix/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000000..3372b1f03f7d --- /dev/null +++ b/third_party/nix/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,27 @@ +<!-- + +# Filing a Nix issue + +*WAIT* Are you sure you're filing your issue in the right repository? + +We appreciate you taking the time to tell us about issues you encounter, but routing the issue to the right place will get you help sooner and save everyone time. + +This is the Nix repository, and issues here should be about Nix the build and package management *_tool_*. + +If you have a problem with a specific package on NixOS or when using Nix, you probably want to file an issue with _nixpkgs_, whose issue tracker is over at https://github.com/NixOS/nixpkgs/issues. + +Examples of _Nix_ issues: + +- Nix segfaults when I run `nix-build -A blahblah` +- The Nix language needs a new builtin: `builtins.foobar` +- Regression in the behavior of `nix-env` in Nix 2.0 + +Examples of _nixpkgs_ issues: + +- glibc is b0rked on aarch64 +- chromium in NixOS doesn't support U2F but google-chrome does! +- The OpenJDK package on macOS is missing a key component + +Chances are if you're a newcomer to the Nix world, you'll probably want the [nixpkgs tracker](https://github.com/NixOS/nixpkgs/issues). It also gets a lot more eyeball traffic so you'll probably get a response a lot more quickly. + +--> diff --git a/third_party/nix/.gitignore b/third_party/nix/.gitignore new file mode 100644 index 000000000000..7f2d47713132 --- /dev/null +++ b/third_party/nix/.gitignore @@ -0,0 +1,119 @@ +Makefile.config +perl/Makefile.config + +# / +/aclocal.m4 +/autom4te.cache +/configure +/nix.spec +/stamp-h1 +/svn-revision +/build-gcc +/libtool + +/corepkgs/config.nix + +# /corepkgs/channels/ +/corepkgs/channels/unpack.sh + +# /corepkgs/nar/ +/corepkgs/nar/nar.sh +/corepkgs/nar/unnar.sh + +# /doc/manual/ +/doc/manual/manual.html +/doc/manual/manual.xmli +/doc/manual/manual.pdf +/doc/manual/manual.is-valid +/doc/manual/*.1 +/doc/manual/*.5 +/doc/manual/*.8 +/doc/manual/version.txt + +# /scripts/ +/scripts/nix-profile.sh +/scripts/nix-copy-closure +/scripts/nix-reduce-build +/scripts/nix-http-export.cgi +/scripts/nix-profile-daemon.sh + +# /src/libexpr/ +/src/libexpr/lexer-tab.cc +/src/libexpr/lexer-tab.hh +/src/libexpr/parser-tab.cc +/src/libexpr/parser-tab.hh +/src/libexpr/parser-tab.output +/src/libexpr/nix.tbl + +# /src/libstore/ +/src/libstore/*.gen.hh + +/src/nix/nix + +# /src/nix-env/ +/src/nix-env/nix-env + +# /src/nix-instantiate/ +/src/nix-instantiate/nix-instantiate + +# /src/nix-store/ +/src/nix-store/nix-store + +/src/nix-prefetch-url/nix-prefetch-url + +# /src/nix-daemon/ +/src/nix-daemon/nix-daemon + +/src/nix-collect-garbage/nix-collect-garbage + +# /src/nix-channel/ +/src/nix-channel/nix-channel + +# /src/nix-build/ +/src/nix-build/nix-build + +/src/nix-copy-closure/nix-copy-closure + +/src/build-remote/build-remote + +# /tests/ +/tests/test-tmp +/tests/common.sh +/tests/dummy +/tests/result* +/tests/restricted-innocent +/tests/shell +/tests/shell.drv + +# /tests/lang/ +/tests/lang/*.out +/tests/lang/*.out.xml +/tests/lang/*.ast + +/perl/lib/Nix/Config.pm +/perl/lib/Nix/Store.cc + +/misc/systemd/nix-daemon.service +/misc/systemd/nix-daemon.socket +/misc/upstart/nix-daemon.conf + +/src/resolve-system-dependencies/resolve-system-dependencies + +inst/ + +*.a +*.o +*.so +*.dylib +*.dll +*.exe +*.dep +*~ +*.pc +*.plist + +# GNU Global +GPATH +GRTAGS +GSYMS +GTAGS diff --git a/third_party/nix/.skip-subtree b/third_party/nix/.skip-subtree new file mode 100644 index 000000000000..d49b47f75a16 --- /dev/null +++ b/third_party/nix/.skip-subtree @@ -0,0 +1 @@ +Third-party code with non-depot layout. diff --git a/third_party/nix/.travis.yml b/third_party/nix/.travis.yml new file mode 100644 index 000000000000..99218a963cbd --- /dev/null +++ b/third_party/nix/.travis.yml @@ -0,0 +1,2 @@ +os: osx +script: ./tests/install-darwin.sh diff --git a/third_party/nix/.version b/third_party/nix/.version new file mode 100644 index 000000000000..fd06a9268de4 --- /dev/null +++ b/third_party/nix/.version @@ -0,0 +1 @@ +2.3.4 \ No newline at end of file diff --git a/third_party/nix/CMakeLists.txt b/third_party/nix/CMakeLists.txt new file mode 100644 index 000000000000..5d89572f16a2 --- /dev/null +++ b/third_party/nix/CMakeLists.txt @@ -0,0 +1,77 @@ +# -*- mode: cmake; -*- +cmake_minimum_required(VERSION 3.16) +project(nix CXX) +set(CMAKE_CXX_STANDARD 17) + +# Export compile_commands.json which can be used by tools such as +# clangd and clang-tidy. +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +# Enable warnings +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror") + +# Provide an output path for pkgconfig. +include(GNUInstallDirs) +set(PKGCONFIG_INSTALL_DIR ${CMAKE_INSTALL_LIBDIR}/pkgconfig) + +# The following lines import CMake-native dependencies which may +# contain useful definitions. Other dependencies are not treated +# specially by CMake and are only linked into the resulting binary. +find_package(BZip2) +find_package(Boost COMPONENTS context) +find_package(CURL) +find_package(LibLZMA) +find_package(Protobuf REQUIRED) +find_package(SQLite3) +find_package(Threads) +find_package(absl REQUIRED) +find_package(gRPC REQUIRED) +find_package(glog REQUIRED) + +find_program(CLANG_TIDY_PATH clang-tidy) +if (CLANG_TIDY_PATH) + # TODO(kanepyork): figure out how to reenable + #message("Found clang-tidy: ${CLANG_TIDY_PATH}") + #set(CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH};--line-filter=[{\"name\":\"src/cpptoml/cpptoml.h\"},{\"name\":\"google/protobuf/metadata_lite.h\"}]") + + # nix's toolchain has a problem with system header includes, so clang-tidy requires a manual -isystem + if (DEFINED ENV{LIBCXX_INCLUDE}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isystem $ENV{LIBCXX_INCLUDE}") + endif() +endif() + +if (DEFINED ENV{SANDBOX_SHELL}) + message("Using SANDBOX_SHELL = $ENV{SANDBOX_SHELL}") + set(SANDBOX_SHELL "$ENV{SANDBOX_SHELL}") +else() + find_program(BUSYBOX busybox) + if (BUSYBOX) + set(SANDBOX_SHELL "${BUSYBOX}") + else() + message(FATAL_ERROR "Could not find busybox and SANDBOX_SHELL is not set") + endif() +endif() + +# generate a configuration file (autoheader-style) to configure +# certain symbols that Nix depends on. +configure_file(config.h.in nix_config.h @ONLY) +INSTALL(FILES "${PROJECT_BINARY_DIR}/nix_config.h" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/nix") + +# install corepkgs +configure_file(corepkgs/config.nix.in config.nix @ONLY) +INSTALL(DIRECTORY corepkgs + DESTINATION ${CMAKE_INSTALL_DATADIR}/nix + FILES_MATCHING + PATTERN "*.nix") +INSTALL(FILES "${PROJECT_BINARY_DIR}/config.nix" DESTINATION "${CMAKE_INSTALL_DATADIR}/nix/corepkgs") + +# Conditionally run tests +option(PACKAGE_TESTS "Build the tests" ON) +if (PACKAGE_TESTS) + enable_testing() + find_package(GTest) + find_package(rapidcheck) + include(GoogleTest) +endif() + +add_subdirectory(src) diff --git a/third_party/nix/COPYING b/third_party/nix/COPYING new file mode 100644 index 000000000000..5ab7695ab8ca --- /dev/null +++ b/third_party/nix/COPYING @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + <one line to give the library's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + <signature of Ty Coon>, 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/third_party/nix/OWNERS b/third_party/nix/OWNERS new file mode 100644 index 000000000000..886f766d0c25 --- /dev/null +++ b/third_party/nix/OWNERS @@ -0,0 +1,5 @@ +inherited: true +owners: + - grfn + - tazjin + - kanepyork diff --git a/third_party/nix/README.md b/third_party/nix/README.md new file mode 100644 index 000000000000..4bb58968313d --- /dev/null +++ b/third_party/nix/README.md @@ -0,0 +1,179 @@ +Tvix, also known as TVL's fork of Nix +------------------------------------- + +Nix is a new take on package management that is fairly unique. Because +of its purity aspects, a lot of issues found in traditional package +managers don't appear with Nix. + +To find out more about the tool, usage and installation instructions, +please read the manual, which is available on the Nix website at +<http://nixos.org/nix/manual>. + +This repository is [TVL's](https://tvl.fyi)'s fork of Nix, which we lovingly +refer to as Tvix. + +## Fork background + +Nix is a fantastic project with over a decade of demonstrated +real-world usage, but also with quite a few problems. + +First of all, the project consists of two main components: The Nix +package collection ("[nixpkgs][]") and the package manager itself. + +The package collection is an enormous effort with hundreds of +thousands of commits, encoding expert knowledge about lots of +different software and ways of building and managing it. It is a very +valuable piece of software. + +The package manager however is an old C++ project with severe code +quality issues, little to no documentation, no consistent style and no +unit test coverage. + +Its codebase is larger than it needs to be (often due to custom +reimplementations of basic functionality) and is mostly ad-hoc +structured, making it difficult to correctly implement large-scale +improvements. + +In addition, the upstream Nix project is diverging from the opinions +of some community members via the introduction of concepts such as Nix +flakes. + +To counteract these things we have decided to fork Nix. + +## Fork goals + +The things listed here are explicitly in-scope for work on the fork. +This list is not exhaustive, and it is very likely that many other +smaller things will be discovered along the way. + +### nixpkgs compatibility + +This fork will maintain compatibility with nixpkgs as much as +possible. If at any point we do need to diverge, we will do it in a +way that is backwards compatible. + +### Code quality improvements + +Code quality encompasses several different issues. + +One goal is to slowly bring the codebase in line with the [Google C++ +style guide][google-style]. Apart from the trivial reformatting (which +is already done), this means slowly chipping away at incorrectly +structured type hierarchies, usage of exceptions, usage of raw +pointers, global mutability and so on. + +Another goal is to reduce the amount of code in Nix by removing custom +reimplementations of basic functionality (such as string splitting or +reading files). + +For functionality that is not part of the C++17 standard library, +[Abseil][] will be the primary external library used. + +### Explicit RPC mechanisms + +Nix currently uses homegrown mechanisms of interacting with other Nix +binaries, for example for remote builds or interaction between the CLI +and the Nix daemon. + +This will be replaced with [gRPC][]. + +### New sandboxing mechanism + +Nix implements its own sandboxing mechanism. This was probably the +correct decision at the time, but is not necessary anymore because +Linux containers have become massively popular and lots of new tooling +is now available. + +The goal is to replace the custom sandboxing implementation with +pluggable [OCI runtimes][oci], which will make it possible to use +arbitrary container runtimes such as [gVisor][] or [systemd-nspawn][] + +### Pluggable Nix store backends + +The current Nix store implementation will be removed from Nix' core +and instead be refactored into a gRPC API that can be implemented by +different backends. + +### Builds as graph reductions + +A Nix derivation that should be instantiated describes a build graph. +This graph will become a first-class citizen, making it possible to +distribute different parts of the computation to different nodes. + +Implementing this properly will also allow us to improve the +implementation of import-from-derivation by explicitly moving through +different graph reduction stages. + +## Fork non-goals + +To set expectations, there are some explicit non-goals, too. + +* Merging these changes back into upstream is not a goal, and maybe + not even feasible. The core work has not even started yet and just + basic cleanup has already created a diff of over 40 000 lines. + + This would likely also turn into a political effort, which we have + no interest in. + +* Improved performance is not an (initial) goal. Nix performance is + very unevenly distributed across the codebase (some things have seen + a lot of ad-hoc optimisation, others are written like inefficient + toy implementations) and we simply don't know what effect the + cleanup will have. + + Once the codebase is in a better state we will be able to start + optimising it again while retaining readability, but this is not a + goal until a later point in time. + +* Compatibility with new upstream features is not a goal. Specifically + we do not want Nix flakes, but other changes upstream makes will be + considered for inclusion. + +* Support for non-Linux systems. Currently Nix support Mac OS and + potentially other systems, but this support will be dropped. + + Once we have OCI-compatible sandboxes and a store protocol it will + be possible to reintroduce these with less friction. + +## Building + +To build the project, set up an out-of-tree cmake directory and run cmake in +nix-shell. + +``` +mkdir ~/build/tvix +cd ~/build/tvix + +nix-shell $DEPOT_PATH -A third_party.nix.build-shell + +# Disable clang-tidy for quicker builds +cmake $DEPOT_PATH/third_party/nix/ -DCLANG_TIDY_PATH="" +make -j16 -l12 + +# Run tests +make test +``` + +## Contributing to the fork + +The TVL depot's default [contribution guidelines][contributing] apply. + +In addition, please make sure that submitted code builds and is +formatted with `clang-format`, using the configuration found in this +folder. + +## License + +Nix is released under the LGPL v2.1 + +This product includes software developed by the OpenSSL Project for +use in the [OpenSSL Toolkit](http://www.OpenSSL.org/). + +[nixpkgs]: https://github.com/NixOS/nixpkgs +[google-style]: https://google.github.io/styleguide/cppguide.html +[Abseil]: https://abseil.io/ +[gRPC]: https://grpc.io/ +[oci]: https://www.opencontainers.org/ +[gVisor]: https://gvisor.dev/ +[systemd-nspawn]: https://www.freedesktop.org/software/systemd/man/systemd-nspawn.html +[contributing]: https://cs.tvl.fyi/depot/-/blob/docs/CONTRIBUTING.md diff --git a/third_party/nix/clangd.nix b/third_party/nix/clangd.nix new file mode 100644 index 000000000000..7a29819b10ee --- /dev/null +++ b/third_party/nix/clangd.nix @@ -0,0 +1,30 @@ +# Create a clangd wrapper script that can be used with this project. +# The default Nix wrapper only works with C projects, not C++ +# projects. +# +# The CPATH construction logic is lifted from the original wrapper +# script. + +pkgs: + +pkgs.writeShellScriptBin "nix-clangd" '' + buildcpath() { + local path + while (( $# )); do + case $1 in + -isystem) + shift + path=$path''${path:+':'}$1 + esac + shift + done + echo $path + } + + export CPATH=''${CPATH}''${CPATH:+':'}:$(buildcpath ''${NIX_CFLAGS_COMPILE}) + export CPATH=${pkgs.glibc.dev}/include''${CPATH:+':'}''${CPATH} + export CPLUS_INCLUDE_PATH=${pkgs.llvmPackages_11.libcxx}/include/c++/v1:''${CPATH} + + # TODO(tazjin): Configurable commands directory? + exec -a clangd ${pkgs.llvmPackages_11.clang-unwrapped}/bin/clangd -cross-file-rename $@ +'' diff --git a/third_party/nix/config.h.in b/third_party/nix/config.h.in new file mode 100644 index 000000000000..986969705b82 --- /dev/null +++ b/third_party/nix/config.h.in @@ -0,0 +1,130 @@ +// This file configures various build-time settings in Nix. In +// previous iterations it was mostly responsible for configuring +// OS-dependent settings, which are still preserved below but should +// be removed. + +#ifndef NIX_CONFIG_H +#define NIX_CONFIG_H + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "2.3.4" + +/* Platform identifier (`cpu-os`) */ +// TODO(tazjin): generate +#define SYSTEM "x86_64-linux" + +// TODO(tazjin): some of these values are nonsensical for Nix +#define NIX_PREFIX "@CMAKE_INSTALL_PREFIX@" +#define NIX_STORE_DIR "/nix/store" +#define NIX_DATA_DIR "@CMAKE_INSTALL_FULL_DATADIR@" +#define NIX_LOG_DIR "/nix/var/log/nix" +#define NIX_STATE_DIR "/nix/var/nix" +#define NIX_CONF_DIR "/etc/nix" +#define NIX_LIBEXEC_DIR "@CMAKE_INSTALL_FULL_LIBEXECDIR@" +#define NIX_BIN_DIR "@CMAKE_INSTALL_FULL_BINDIR@" +#define NIX_MAN_DIR "@CMAKE_INSTALL_FULL_MANDIR@" +#define SANDBOX_SHELL "@SANDBOX_SHELL@" + +// Defines used only in tests (e.g. to access data) +#define NIX_SRC_DIR "@CMAKE_SOURCE_DIR@" + +// These are hardcoded either because support for non-Linux is being +// dropped, or because a decision was made to make inclusion of these +// libraries mandatory. + +#define HAVE_STRUCT_DIRENT_D_TYPE 1 + +#define HAVE_LUTIMES 1 + +// Whether link() works on symlinks +#define CAN_LINK_SYMLINK 1 + +/* Whether to use the Boehm garbage collector. */ +#define HAVE_BOEHMGC 1 + +/* Define if the Boost library is available. */ +#define HAVE_BOOST 1 + +/* Define to 1 if you have the <bzlib.h> header file. */ +#define HAVE_BZLIB_H 1 + +/* Define if the compiler supports basic C++17 syntax */ +#define HAVE_CXX17 1 + +/* Define to 1 if you have the <dirent.h> header file, and it defines `DIR` */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the <sys/dir.h> header file, and it defines `DIR` */ +#define HAVE_DIR_H 1 + +/* Define to 1 if you have the <editline.h> header file. */ +#define HAVE_EDITLINE_H 1 + +/* Define to 1 if you have the <inttypes.h> header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `lchown` function. */ +#define HAVE_LCHOWN 1 + +/* Define to 1 if you have the <locale.h> header file. */ +#define HAVE_LOCALE 1 + +/* Define to 1 if you have the `lutimes` function. */ +#define HAVE_LUTIMES 1 + +/* Define to 1 if you have the <memory.h> header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `pipe2` function. */ +#define HAVE_PIPE2 1 + +/* Define to 1 if you have the `posix_fallocate` function. */ +#define HAVE_POSIX_FALLOCATE 1 + +/* Define to 1 if you have the `pubsetbuf` function. */ +#define HAVE_PUBSETBUF 1 + +/* Whether seccomp is available and should be used for sandboxing. */ +#define HAVE_SECCOMP 1 + +/* Define to 1 if you have the `setresuid` function. */ +#define HAVE_SETRESUID 1 + +/* Define to 1 if you have the `setreuid` function. */ +#define HAVE_SETREUID 1 + +/* Whether to use libsodium for cryptography. */ +#define HAVE_SODIUM 1 + +/* Define to 1 if you have the `statvfs` function. */ +#define HAVE_STATVFS 1 + +/* Define to 1 if you have the <stdint.h> header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the <stdlib.h> header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the <strings.h> header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the `strsignal` function. */ +#define HAVE_STRSIGNAL 1 + +/* Define to 1 if you have the `sysconf` function. */ +#define HAVE_SYSCONF 1 + +/* Define to 1 if you have the <sys/stat.h> header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the <sys/types.h> header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the <unistd.h> header file. */ +#define HAVE_UNISTD_H 1 + + +#endif diff --git a/third_party/nix/config/config.sub b/third_party/nix/config/config.sub new file mode 100755 index 000000000000..c19e671805a8 --- /dev/null +++ b/third_party/nix/config/config.sub @@ -0,0 +1,1818 @@ +#! /bin/sh +# Configuration validation subroutine script. +# Copyright 1992-2018 Free Software Foundation, Inc. + +timestamp='2018-08-13' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see <https://www.gnu.org/licenses/>. +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). + + +# Please send patches to <config-patches@gnu.org>. +# +# Configuration subroutine to validate and canonicalize a configuration type. +# Supply the specified configuration type as an argument. +# If it is invalid, we print an error message on stderr and exit with code 1. +# Otherwise, we print the canonical config type on stdout and succeed. + +# You can get the latest version of this script from: +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub + +# This file is supposed to be the same for all GNU packages +# and recognize all the CPU types, system types and aliases +# that are meaningful with *any* GNU software. +# Each package is responsible for reporting which valid configurations +# it does not support. The user should be able to distinguish +# a failure to support a valid configuration from a meaningless +# configuration. + +# The goal of this file is to map all the various variations of a given +# machine specification into a single specification in the form: +# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM +# or in some cases, the newer four-part form: +# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM +# It is wrong to echo any other type of specification. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS + +Canonicalize a configuration name. + +Options: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to <config-patches@gnu.org>." + +version="\ +GNU config.sub ($timestamp) + +Copyright 1992-2018 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" + exit 1 ;; + + *local*) + # First pass through any local machine types. + echo "$1" + exit ;; + + * ) + break ;; + esac +done + +case $# in + 0) echo "$me: missing argument$help" >&2 + exit 1;; + 1) ;; + *) echo "$me: too many arguments$help" >&2 + exit 1;; +esac + +# Split fields of configuration type +IFS="-" read -r field1 field2 field3 field4 <<EOF +$1 +EOF + +# Separate into logical components for further validation +case $1 in + *-*-*-*-*) + echo Invalid configuration \`"$1"\': more than four components >&2 + exit 1 + ;; + *-*-*-*) + basic_machine=$field1-$field2 + os=$field3-$field4 + ;; + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ + | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + os=linux-android + ;; + *) + basic_machine=$field1-$field2 + os=$field3 + ;; + esac + ;; + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc532* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* | hitachi* \ + | c[123]* | convex* | sun | crds | omron* | dg | ultra | tti* \ + | harris | dolphin | highlevel | gould | cbm | ns | masscomp \ + | apple | axis | knuth | cray | microblaze* \ + | sim | cisco | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + os= + ;; + *) + basic_machine=$field1 + os=$field2 + ;; + esac + ;; + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + os=bsd + ;; + a29khif) + basic_machine=a29k-amd + os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + os=scout + ;; + alliant) + basic_machine=fx80-alliant + os= + ;; + altos | altos3068) + basic_machine=m68k-altos + os= + ;; + am29k) + basic_machine=a29k-none + os=bsd + ;; + amdahl) + basic_machine=580-amdahl + os=sysv + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=bsd + ;; + aros) + basic_machine=i386-pc + os=aros + ;; + aux) + basic_machine=m68k-apple + os=aux + ;; + balance) + basic_machine=ns32k-sequent + os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=linux + ;; + cegcc) + basic_machine=arm-unknown + os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=bsd + ;; + convex-c2) + basic_machine=c2-convex + os=bsd + ;; + convex-c32) + basic_machine=c32-convex + os=bsd + ;; + convex-c34) + basic_machine=c34-convex + os=bsd + ;; + convex-c38) + basic_machine=c38-convex + os=bsd + ;; + cray) + basic_machine=j90-cray + os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + os= + ;; + delta88) + basic_machine=m88k-motorola + os=sysv3 + ;; + dicos) + basic_machine=i686-pc + os=dicos + ;; + djgpp) + basic_machine=i586-pc + os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=ose + ;; + gmicro) + basic_machine=tron-gmicro + os=sysv + ;; + go32) + basic_machine=i386-pc + os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=hms + ;; + harris) + basic_machine=m88k-harris + os=sysv3 + ;; + hp300bsd) + basic_machine=m68k-hp + os=bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=hpux + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=proelf + ;; + i386mach) + basic_machine=i386-mach + os=mach + ;; + vsta) + basic_machine=i386-pc + os=vsta + ;; + isi68 | isi) + basic_machine=m68k-isi + os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + os=sysv + ;; + merlin) + basic_machine=ns32k-utek + os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + os=coff + ;; + morphos) + basic_machine=powerpc-unknown + os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=moxiebox + ;; + msdos) + basic_machine=i386-pc + os=msdos + ;; + msys) + basic_machine=i686-pc + os=msys + ;; + mvs) + basic_machine=i370-ibm + os=mvs + ;; + nacl) + basic_machine=le32-unknown + os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=newsos + ;; + news1000) + basic_machine=m68030-sony + os=newsos + ;; + necv70) + basic_machine=v70-nec + os=sysv + ;; + nh3000) + basic_machine=m68k-harris + os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=cxux + ;; + nindy960) + basic_machine=i960-intel + os=nindy + ;; + mon960) + basic_machine=i960-intel + os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=ose + ;; + os68k) + basic_machine=m68k-none + os=os68k + ;; + paragon) + basic_machine=i860-intel + os=osf + ;; + parisc) + basic_machine=hppa-unknown + os=linux + ;; + pw32) + basic_machine=i586-unknown + os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=rdos + ;; + rdos32) + basic_machine=i386-pc + os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=coff + ;; + sa29200) + basic_machine=a29k-amd + os=udi + ;; + sei) + basic_machine=mips-sei + os=seiux + ;; + sps7) + basic_machine=m68k-bull + os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + os= + ;; + stratus) + basic_machine=i860-stratus + os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + os= + ;; + sun2os3) + basic_machine=m68000-sun + os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + os= + ;; + sun3os3) + basic_machine=m68k-sun + os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + os= + ;; + sun4os3) + basic_machine=sparc-sun + os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + os= + ;; + sv1) + basic_machine=sv1-cray + os=unicos + ;; + symmetry) + basic_machine=i386-sequent + os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=unicos + ;; + t90) + basic_machine=t90-cray + os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + os=tpf + ;; + udi29k) + basic_machine=a29k-amd + os=udi + ;; + ultra3) + basic_machine=a29k-nyu + os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=none + ;; + vaxv) + basic_machine=vax-dec + os=sysv + ;; + vms) + basic_machine=vax-dec + os=vms + ;; + vxworks960) + basic_machine=i960-wrs + os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=vxworks + ;; + xbox) + basic_machine=i686-pc + os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + os=unicos + ;; + *) + basic_machine=$1 + os= + ;; + esac + ;; +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv) + basic_machine=craynv-cray + os=${os:-unicosmp} + ;; + fx80) + basic_machine=fx80-alliant + ;; + w89k) + basic_machine=hppa1.1-winbond + ;; + op50n) + basic_machine=hppa1.1-oki + ;; + op60c) + basic_machine=hppa1.1-oki + ;; + romp) + basic_machine=romp-ibm + ;; + mmix) + basic_machine=mmix-knuth + ;; + rs6000) + basic_machine=rs6000-ibm + ;; + vax) + basic_machine=vax-dec + ;; + pdp11) + basic_machine=pdp11-dec + ;; + we32k) + basic_machine=we32k-att + ;; + cydra) + basic_machine=cydra-cydrome + ;; + i370-ibm* | ibm*) + basic_machine=i370-ibm + ;; + orion) + basic_machine=orion-highlevel + ;; + orion105) + basic_machine=clipper-highlevel + ;; + mac | mpw | mac-mpw) + basic_machine=m68k-apple + ;; + pmac | pmac-mpw) + basic_machine=powerpc-apple + ;; + xps | xps100) + basic_machine=xps100-honeywell + ;; + + # Recognize the basic CPU types without company name. + # Some are omitted here because they have special meanings below. + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ + | am33_2.0 \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin \ + | c4x | c8051 | clipper | csky \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ + | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | ns16k | ns32k \ + | open8 | or1k | or1knd | or32 \ + | pdp10 | pj | pjl \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv64 \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh[23]ele \ + | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | visium \ + | wasm32 \ + | x86 | xc16x | xstormy16 | xgate | xtensa \ + | z8k | z80) + basic_machine=$basic_machine-unknown + ;; + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) + ;; + m9s12z | m68hcs12z | hcs12z | s12z) + basic_machine=s12z-unknown + ;; + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + basic_machine=s12z-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ms1) + basic_machine=mt-unknown + ;; + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; + + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + basic_machine=$basic_machine-pc + ;; + # Recognize the basic CPU types with company name. + 1750a-* | 580-* \ + | a29k-* \ + | aarch64-* | aarch64_be-* \ + | abacus-* \ + | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ + | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ + | alphapca5[67]-* | alpha64pca5[67]-* \ + | am33_2.0-* \ + | arc-* | arceb-* \ + | arm-* | arm[lb]e-* | arme[lb]-* | armv*-* \ + | avr-* | avr32-* \ + | asmjs-* \ + | ba-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | c8051-* | clipper-* | craynv-* | csky-* | cydra-* \ + | d10v-* | d30v-* | dlx-* | dsp16xx-* \ + | e2k-* | elxsi-* | epiphany-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | ft32-* | fx80-* \ + | h8300-* | h8500-* \ + | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ + | i370-* | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ + | ip2k-* | iq2000-* \ + | k1om-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ + | m5200-* | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* | v70-* | w65-* \ + | m6811-* | m68hc11-* | m6812-* | m68hc12-* | m68hcs12x-* | nvptx-* | picochip-* \ + | m88110-* | m88k-* | maxq-* | mb-* | mcore-* | mep-* | metag-* \ + | microblaze-* | microblazeel-* \ + | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ + | mips16-* \ + | mips64-* | mips64el-* \ + | mips64octeon-* | mips64octeonel-* \ + | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ + | mips64vr4100-* | mips64vr4100el-* \ + | mips64vr4300-* | mips64vr4300el-* \ + | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ + | mipsisa32-* | mipsisa32el-* \ + | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ + | mipsisa64-* | mipsisa64el-* \ + | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ + | mipsisa64sb1-* | mipsisa64sb1el-* \ + | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ + | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mn10200-* | mn10300-* \ + | moxie-* \ + | mt-* \ + | msp430-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nfp-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ + | or1k*-* \ + | or32-* \ + | orion-* \ + | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ + | pyramid-* \ + | riscv-* | riscv32-* | riscv64-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | score-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]ae[lb]-* | sh[23]e-* | she[lb]-* | sh[lb]e-* \ + | sh[1234]e[lb]-* | sh[12345][lb]e-* | sh[23]ele-* | sh64-* | sh64le-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ + | spu-* \ + | tahoe-* \ + | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tron-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ + | visium-* \ + | wasm32-* \ + | we32k-* \ + | x86-* | x86_64-* | xc16x-* | xgate-* | xps100-* \ + | xstormy16-* | xtensa*-* \ + | ymp-* \ + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown + ;; + # Recognize the various machine names and aliases which stand + # for a CPU type and a company and sometimes even an OS. + 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) + basic_machine=m68000-att + ;; + 3b*) + basic_machine=we32k-att + ;; + amd64) + basic_machine=x86_64-pc + ;; + amd64-*) + basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + amiga | amiga-*) + basic_machine=m68k-unknown + ;; + blackfin-*) + basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=cnk + ;; + c54x-*) + basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + c90) + basic_machine=c90-cray + os=${os:-unicos} + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown + os=${os:-elf} + ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; + cris | cris-* | etrax*) + basic_machine=cris-axis + ;; + crx) + basic_machine=crx-unknown + os=${os:-elf} + ;; + da30 | da30-*) + basic_machine=m68k-da30 + ;; + decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) + basic_machine=mips-dec + ;; + decsystem10* | dec10*) + basic_machine=pdp10-dec + os=tops10 + ;; + decsystem20* | dec20*) + basic_machine=pdp10-dec + os=tops20 + ;; + delta | 3300 | motorola-3300 | motorola-delta \ + | 3300-motorola | delta-motorola) + basic_machine=m68k-motorola + ;; + dpx20 | dpx20-*) + basic_machine=rs6000-bull + os=${os:-bosx} + ;; + dpx2*) + basic_machine=m68k-bull + os=sysv3 + ;; + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" + ;; + e500v[12]-*) + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=$os"spe" + ;; + encore | umax | mmax) + basic_machine=ns32k-encore + ;; + elxsi) + basic_machine=elxsi-elxsi + os=${os:-bsd} + ;; + fx2800) + basic_machine=i860-alliant + ;; + genix) + basic_machine=ns32k-ns + ;; + h3050r* | hiux*) + basic_machine=hppa1.1-hitachi + os=hiuxwe2 + ;; + hp300-*) + basic_machine=m68k-hp + ;; + hp3k9[0-9][0-9] | hp9[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k2[0-9][0-9] | hp9k31[0-9]) + basic_machine=m68000-hp + ;; + hp9k3[2-9][0-9]) + basic_machine=m68k-hp + ;; + hp9k6[0-9][0-9] | hp6[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k7[0-79][0-9] | hp7[0-79][0-9]) + basic_machine=hppa1.1-hp + ;; + hp9k78[0-9] | hp78[0-9]) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][13679] | hp8[0-9][13679]) + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][0-9] | hp8[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + i*86v32) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv32 + ;; + i*86v4*) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv4 + ;; + i*86v) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv + ;; + i*86sol2) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=solaris2 + ;; + j90 | j90-cray) + basic_machine=j90-cray + os=${os:-unicos} + ;; + iris | iris4d) + basic_machine=mips-sgi + case $os in + irix*) + ;; + *) + os=irix4 + ;; + esac + ;; + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` + ;; + m68knommu-*) + basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux + ;; + microblaze*) + basic_machine=microblaze-xilinx + ;; + miniframe) + basic_machine=m68000-convergent + ;; + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) + basic_machine=m68k-atari + os=mint + ;; + mips3*-*) + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` + ;; + mips3*) + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown + ;; + ms1-*) + basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` + ;; + news-3600 | risc-news) + basic_machine=mips-sony + os=newsos + ;; + next | m*-next) + basic_machine=m68k-next + case $os in + nextstep* ) + ;; + ns2*) + os=nextstep2 + ;; + *) + os=nextstep3 + ;; + esac + ;; + np1) + basic_machine=np1-gould + ;; + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem + ;; + nsr-tandem) + basic_machine=nsr-tandem + ;; + nsv-tandem) + basic_machine=nsv-tandem + ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; + op50n-* | op60c-*) + basic_machine=hppa1.1-oki + os=proelf + ;; + openrisc | openrisc-*) + basic_machine=or32-unknown + ;; + pa-hitachi) + basic_machine=hppa1.1-hitachi + os=hiuxwe2 + ;; + parisc-*) + basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux + ;; + pbd) + basic_machine=sparc-tti + ;; + pbb) + basic_machine=m68k-tti + ;; + pc532 | pc532-*) + basic_machine=ns32k-pc532 + ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentium | p5 | k5 | k6 | nexgen | viac3) + basic_machine=i586-pc + ;; + pentiumpro | p6 | 6x86 | athlon | athlon_*) + basic_machine=i686-pc + ;; + pentiumii | pentium2 | pentiumiii | pentium3) + basic_machine=i686-pc + ;; + pentium4) + basic_machine=i786-pc + ;; + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentiumpro-* | p6-* | 6x86-* | athlon-*) + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentium4-*) + basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pn) + basic_machine=pn-gould + ;; + power) basic_machine=power-ibm + ;; + ppc | ppcbe) basic_machine=powerpc-unknown + ;; + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ppcle | powerpclittle) + basic_machine=powerpcle-unknown + ;; + ppcle-* | powerpclittle-*) + basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ppc64) basic_machine=powerpc64-unknown + ;; + ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ppc64le | powerpc64little) + basic_machine=powerpc64le-unknown + ;; + ppc64le-* | powerpc64little-*) + basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ps2) + basic_machine=i386-ibm + ;; + rm[46]00) + basic_machine=mips-siemens + ;; + rtpc | rtpc-*) + basic_machine=romp-ibm + ;; + s390 | s390-*) + basic_machine=s390-ibm + ;; + s390x | s390x-*) + basic_machine=s390x-ibm + ;; + sb1) + basic_machine=mipsisa64sb1-unknown + ;; + sb1el) + basic_machine=mipsisa64sb1el-unknown + ;; + sde) + basic_machine=mipsisa32-sde + os=${os:-elf} + ;; + sequent) + basic_machine=i386-sequent + ;; + sh5el) + basic_machine=sh5le-unknown + ;; + sh5el-*) + basic_machine=sh5le-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + simso-wrs) + basic_machine=sparclite-wrs + os=vxworks + ;; + spur) + basic_machine=spur-unknown + ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + tile*-*) + ;; + tile*) + basic_machine=$basic_machine-unknown + os=${os:-linux-gnu} + ;; + tx39) + basic_machine=mipstx39-unknown + ;; + tx39el) + basic_machine=mipstx39el-unknown + ;; + tower | tower-32) + basic_machine=m68k-ncr + ;; + vpp*|vx|vx-*) + basic_machine=f301-fujitsu + ;; + w65*) + basic_machine=w65-wdc + os=none + ;; + w89k-*) + basic_machine=hppa1.1-winbond + os=proelf + ;; + x64) + basic_machine=x86_64-pc + ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` + ;; + none) + basic_machine=none-none + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 + exit 1 + ;; +esac + +# Here we canonicalize certain aliases for manufacturers. +case $basic_machine in + *-digital*) + basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` + ;; + *-commodore*) + basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` + ;; + *) + ;; +esac + +# Decode manufacturer-specific aliases for certain operating systems. + +if [ x$os != x ] +then +case $os in + # First match some system type aliases that might get confused + # with valid system types. + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux + ;; + bluegene*) + os=cnk + ;; + solaris1 | solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` + ;; + solaris) + os=solaris2 + ;; + unixware*) + os=sysv4.2uw + ;; + gnu/linux*) + os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + ;; + # es1800 is here to avoid being matched by es* (a different OS) + es1800*) + os=ose + ;; + # Some version numbers need modification + chorusos*) + os=chorusos + ;; + isc) + os=isc2.2 + ;; + sco6) + os=sco5v6 + ;; + sco5) + os=sco3.2v5 + ;; + sco4) + os=sco3.2v4 + ;; + sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + ;; + sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + scout) + # Don't match below + ;; + sco*) + os=sco3.2v2 + ;; + psos*) + os=psos + ;; + # Now accept the basic system types. + # The portable systems comes first. + # Each alternative MUST end in a * to match a version number. + # sysv* is not here because it comes later, after sysvr4. + gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | kopensolaris* | plan9* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | knetbsd* | mirbsd* | netbsd* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* \ + | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \ + | linux-newlib* | linux-musl* | linux-uclibc* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* \ + | morphos* | superux* | rtmk* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd*) + # Remember, each alternative MUST END IN *, to match a version number. + ;; + qnx*) + case $basic_machine in + x86-* | i*86-*) + ;; + *) + os=nto-$os + ;; + esac + ;; + hiux*) + os=hiuxwe2 + ;; + nto-qnx*) + ;; + nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` + ;; + sim | xray | os68k* | v88r* \ + | windows* | osx | abug | netware* | os9* \ + | macos* | mpw* | magic* | mmixware* | mon960* | lnews*) + ;; + linux-dietlibc) + os=linux-dietlibc + ;; + linux*) + os=`echo $os | sed -e 's|linux|linux-gnu|'` + ;; + lynx*178) + os=lynxos178 + ;; + lynx*5) + os=lynxos5 + ;; + lynx*) + os=lynxos + ;; + mac*) + os=`echo "$os" | sed -e 's|mac|macos|'` + ;; + opened*) + os=openedition + ;; + os400*) + os=os400 + ;; + sunos5*) + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` + ;; + sunos6*) + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` + ;; + wince*) + os=wince + ;; + utek*) + os=bsd + ;; + dynix*) + os=bsd + ;; + acis*) + os=aos + ;; + atheos*) + os=atheos + ;; + syllable*) + os=syllable + ;; + 386bsd) + os=bsd + ;; + ctix* | uts*) + os=sysv + ;; + nova*) + os=rtmk-nova + ;; + ns2) + os=nextstep2 + ;; + nsk*) + os=nsk + ;; + # Preserve the version number of sinix5. + sinix5.*) + os=`echo $os | sed -e 's|sinix|sysv|'` + ;; + sinix*) + os=sysv4 + ;; + tpf*) + os=tpf + ;; + triton*) + os=sysv3 + ;; + oss*) + os=sysv3 + ;; + svr4*) + os=sysv4 + ;; + svr3) + os=sysv3 + ;; + sysvr4) + os=sysv4 + ;; + # This must come after sysvr4. + sysv*) + ;; + ose*) + os=ose + ;; + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint + ;; + zvmoe) + os=zvmoe + ;; + dicos*) + os=dicos + ;; + pikeos*) + # Until real need of OS specific support for + # particular features comes up, bare metal + # configurations are quite functional. + case $basic_machine in + arm*) + os=eabi + ;; + *) + os=elf + ;; + esac + ;; + nacl*) + ;; + ios) + ;; + none) + ;; + *-eabi) + ;; + *) + echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 + exit 1 + ;; +esac +else + +# Here we handle the default operating systems that come with various machines. +# The value should be what the vendor currently ships out the door with their +# machine or put another way, the most popular os provided with the machine. + +# Note that if you're going to try to match "-MANUFACTURER" here (say, +# "-sun"), then you have to tell the case statement up towards the top +# that MANUFACTURER isn't an operating system. Otherwise, code above +# will signal an error saying that MANUFACTURER isn't an operating +# system, and we'll never get to this point. + +case $basic_machine in + score-*) + os=elf + ;; + spu-*) + os=elf + ;; + *-acorn) + os=riscix1.2 + ;; + arm*-rebel) + os=linux + ;; + arm*-semi) + os=aout + ;; + c4x-* | tic4x-*) + os=coff + ;; + c8051-*) + os=elf + ;; + clipper-intergraph) + os=clix + ;; + hexagon-*) + os=elf + ;; + tic54x-*) + os=coff + ;; + tic55x-*) + os=coff + ;; + tic6x-*) + os=coff + ;; + # This must come before the *-dec entry. + pdp10-*) + os=tops20 + ;; + pdp11-*) + os=none + ;; + *-dec | vax-*) + os=ultrix4.2 + ;; + m68*-apollo) + os=domain + ;; + i386-sun) + os=sunos4.0.2 + ;; + m68000-sun) + os=sunos3 + ;; + m68*-cisco) + os=aout + ;; + mep-*) + os=elf + ;; + mips*-cisco) + os=elf + ;; + mips*-*) + os=elf + ;; + or32-*) + os=coff + ;; + *-tti) # must be before sparc entry or we get the wrong os. + os=sysv3 + ;; + sparc-* | *-sun) + os=sunos4.1.1 + ;; + pru-*) + os=elf + ;; + *-be) + os=beos + ;; + *-ibm) + os=aix + ;; + *-knuth) + os=mmixware + ;; + *-wec) + os=proelf + ;; + *-winbond) + os=proelf + ;; + *-oki) + os=proelf + ;; + *-hp) + os=hpux + ;; + *-hitachi) + os=hiux + ;; + i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) + os=sysv + ;; + *-cbm) + os=amigaos + ;; + *-dg) + os=dgux + ;; + *-dolphin) + os=sysv3 + ;; + m68k-ccur) + os=rtu + ;; + m88k-omron*) + os=luna + ;; + *-next) + os=nextstep + ;; + *-sequent) + os=ptx + ;; + *-crds) + os=unos + ;; + *-ns) + os=genix + ;; + i370-*) + os=mvs + ;; + *-gould) + os=sysv + ;; + *-highlevel) + os=bsd + ;; + *-encore) + os=bsd + ;; + *-sgi) + os=irix + ;; + *-siemens) + os=sysv4 + ;; + *-masscomp) + os=rtu + ;; + f30[01]-fujitsu | f700-fujitsu) + os=uxpv + ;; + *-rom68k) + os=coff + ;; + *-*bug) + os=coff + ;; + *-apple) + os=macos + ;; + *-atari*) + os=mint + ;; + *-wrs) + os=vxworks + ;; + *) + os=none + ;; +esac +fi + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +vendor=unknown +case $basic_machine in + *-unknown) + case $os in + riscix*) + vendor=acorn + ;; + sunos*) + vendor=sun + ;; + cnk*|-aix*) + vendor=ibm + ;; + beos*) + vendor=be + ;; + hpux*) + vendor=hp + ;; + mpeix*) + vendor=hp + ;; + hiux*) + vendor=hitachi + ;; + unos*) + vendor=crds + ;; + dgux*) + vendor=dg + ;; + luna*) + vendor=omron + ;; + genix*) + vendor=ns + ;; + clix*) + vendor=intergraph + ;; + mvs* | opened*) + vendor=ibm + ;; + os400*) + vendor=ibm + ;; + ptx*) + vendor=sequent + ;; + tpf*) + vendor=ibm + ;; + vxsim* | vxworks* | windiss*) + vendor=wrs + ;; + aux*) + vendor=apple + ;; + hms*) + vendor=hitachi + ;; + mpw* | macos*) + vendor=apple + ;; + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + vendor=atari + ;; + vos*) + vendor=stratus + ;; + esac + basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` + ;; +esac + +echo "$basic_machine-$os" +exit + +# Local variables: +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/third_party/nix/config/install-sh b/third_party/nix/config/install-sh new file mode 100755 index 000000000000..377bb8687ffe --- /dev/null +++ b/third_party/nix/config/install-sh @@ -0,0 +1,527 @@ +#!/bin/sh +# install - install a program, script, or datafile + +scriptversion=2011-11-20.07; # UTC + +# This originates from X11R5 (mit/util/scripts/install.sh), which was +# later released in X11R6 (xc/config/util/install.sh) with the +# following copyright and license. +# +# Copyright (C) 1994 X Consortium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- +# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of the X Consortium shall not +# be used in advertising or otherwise to promote the sale, use or other deal- +# ings in this Software without prior written authorization from the X Consor- +# tium. +# +# +# FSF changes to this file are in the public domain. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# 'make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. + +nl=' +' +IFS=" "" $nl" + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit=${DOITPROG-} +if test -z "$doit"; then + doit_exec=exec +else + doit_exec=$doit +fi + +# Put in absolute file names if you don't have them in your path; +# or use environment vars. + +chgrpprog=${CHGRPPROG-chgrp} +chmodprog=${CHMODPROG-chmod} +chownprog=${CHOWNPROG-chown} +cmpprog=${CMPPROG-cmp} +cpprog=${CPPROG-cp} +mkdirprog=${MKDIRPROG-mkdir} +mvprog=${MVPROG-mv} +rmprog=${RMPROG-rm} +stripprog=${STRIPPROG-strip} + +posix_glob='?' +initialize_posix_glob=' + test "$posix_glob" != "?" || { + if (set -f) 2>/dev/null; then + posix_glob= + else + posix_glob=: + fi + } +' + +posix_mkdir= + +# Desired mode of installed file. +mode=0755 + +chgrpcmd= +chmodcmd=$chmodprog +chowncmd= +mvcmd=$mvprog +rmcmd="$rmprog -f" +stripcmd= + +src= +dst= +dir_arg= +dst_arg= + +copy_on_change=false +no_target_directory= + +usage="\ +Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE + or: $0 [OPTION]... SRCFILES... DIRECTORY + or: $0 [OPTION]... -t DIRECTORY SRCFILES... + or: $0 [OPTION]... -d DIRECTORIES... + +In the 1st form, copy SRCFILE to DSTFILE. +In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. +In the 4th, create DIRECTORIES. + +Options: + --help display this help and exit. + --version display version info and exit. + + -c (ignored) + -C install only if different (preserve the last data modification time) + -d create directories instead of installing files. + -g GROUP $chgrpprog installed files to GROUP. + -m MODE $chmodprog installed files to MODE. + -o USER $chownprog installed files to USER. + -s $stripprog installed files. + -t DIRECTORY install into DIRECTORY. + -T report an error if DSTFILE is a directory. + +Environment variables override the default commands: + CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG + RMPROG STRIPPROG +" + +while test $# -ne 0; do + case $1 in + -c) ;; + + -C) copy_on_change=true;; + + -d) dir_arg=true;; + + -g) chgrpcmd="$chgrpprog $2" + shift;; + + --help) echo "$usage"; exit $?;; + + -m) mode=$2 + case $mode in + *' '* | *' '* | *' +'* | *'*'* | *'?'* | *'['*) + echo "$0: invalid mode: $mode" >&2 + exit 1;; + esac + shift;; + + -o) chowncmd="$chownprog $2" + shift;; + + -s) stripcmd=$stripprog;; + + -t) dst_arg=$2 + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + shift;; + + -T) no_target_directory=true;; + + --version) echo "$0 $scriptversion"; exit $?;; + + --) shift + break;; + + -*) echo "$0: invalid option: $1" >&2 + exit 1;; + + *) break;; + esac + shift +done + +if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then + # When -d is used, all remaining arguments are directories to create. + # When -t is used, the destination is already specified. + # Otherwise, the last argument is the destination. Remove it from $@. + for arg + do + if test -n "$dst_arg"; then + # $@ is not empty: it contains at least $arg. + set fnord "$@" "$dst_arg" + shift # fnord + fi + shift # arg + dst_arg=$arg + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + done +fi + +if test $# -eq 0; then + if test -z "$dir_arg"; then + echo "$0: no input file specified." >&2 + exit 1 + fi + # It's OK to call 'install-sh -d' without argument. + # This can happen when creating conditional directories. + exit 0 +fi + +if test -z "$dir_arg"; then + do_exit='(exit $ret); exit $ret' + trap "ret=129; $do_exit" 1 + trap "ret=130; $do_exit" 2 + trap "ret=141; $do_exit" 13 + trap "ret=143; $do_exit" 15 + + # Set umask so as not to create temps with too-generous modes. + # However, 'strip' requires both read and write access to temps. + case $mode in + # Optimize common cases. + *644) cp_umask=133;; + *755) cp_umask=22;; + + *[0-7]) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw='% 200' + fi + cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; + *) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw=,u+rw + fi + cp_umask=$mode$u_plus_rw;; + esac +fi + +for src +do + # Protect names problematic for 'test' and other utilities. + case $src in + -* | [=\(\)!]) src=./$src;; + esac + + if test -n "$dir_arg"; then + dst=$src + dstdir=$dst + test -d "$dstdir" + dstdir_status=$? + else + + # Waiting for this to be detected by the "$cpprog $src $dsttmp" command + # might cause directories to be created, which would be especially bad + # if $src (and thus $dsttmp) contains '*'. + if test ! -f "$src" && test ! -d "$src"; then + echo "$0: $src does not exist." >&2 + exit 1 + fi + + if test -z "$dst_arg"; then + echo "$0: no destination specified." >&2 + exit 1 + fi + dst=$dst_arg + + # If destination is a directory, append the input filename; won't work + # if double slashes aren't ignored. + if test -d "$dst"; then + if test -n "$no_target_directory"; then + echo "$0: $dst_arg: Is a directory" >&2 + exit 1 + fi + dstdir=$dst + dst=$dstdir/`basename "$src"` + dstdir_status=0 + else + # Prefer dirname, but fall back on a substitute if dirname fails. + dstdir=` + (dirname "$dst") 2>/dev/null || + expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$dst" : 'X\(//\)[^/]' \| \ + X"$dst" : 'X\(//\)$' \| \ + X"$dst" : 'X\(/\)' \| . 2>/dev/null || + echo X"$dst" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q' + ` + + test -d "$dstdir" + dstdir_status=$? + fi + fi + + obsolete_mkdir_used=false + + if test $dstdir_status != 0; then + case $posix_mkdir in + '') + # Create intermediate dirs using mode 755 as modified by the umask. + # This is like FreeBSD 'install' as of 1997-10-28. + umask=`umask` + case $stripcmd.$umask in + # Optimize common cases. + *[2367][2367]) mkdir_umask=$umask;; + .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; + + *[0-7]) + mkdir_umask=`expr $umask + 22 \ + - $umask % 100 % 40 + $umask % 20 \ + - $umask % 10 % 4 + $umask % 2 + `;; + *) mkdir_umask=$umask,go-w;; + esac + + # With -d, create the new directory with the user-specified mode. + # Otherwise, rely on $mkdir_umask. + if test -n "$dir_arg"; then + mkdir_mode=-m$mode + else + mkdir_mode= + fi + + posix_mkdir=false + case $umask in + *[123567][0-7][0-7]) + # POSIX mkdir -p sets u+wx bits regardless of umask, which + # is incompatible with FreeBSD 'install' when (umask & 300) != 0. + ;; + *) + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 + + if (umask $mkdir_umask && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + ls_ld_tmpdir=`ls -ld "$tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/d" "$tmpdir" + else + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null + fi + trap '' 0;; + esac;; + esac + + if + $posix_mkdir && ( + umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" + ) + then : + else + + # The umask is ridiculous, or mkdir does not conform to POSIX, + # or it failed possibly due to a race condition. Create the + # directory the slow way, step by step, checking for races as we go. + + case $dstdir in + /*) prefix='/';; + [-=\(\)!]*) prefix='./';; + *) prefix='';; + esac + + eval "$initialize_posix_glob" + + oIFS=$IFS + IFS=/ + $posix_glob set -f + set fnord $dstdir + shift + $posix_glob set +f + IFS=$oIFS + + prefixes= + + for d + do + test X"$d" = X && continue + + prefix=$prefix$d + if test -d "$prefix"; then + prefixes= + else + if $posix_mkdir; then + (umask=$mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break + # Don't fail if two instances are running concurrently. + test -d "$prefix" || exit 1 + else + case $prefix in + *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; + *) qprefix=$prefix;; + esac + prefixes="$prefixes '$qprefix'" + fi + fi + prefix=$prefix/ + done + + if test -n "$prefixes"; then + # Don't fail if two instances are running concurrently. + (umask $mkdir_umask && + eval "\$doit_exec \$mkdirprog $prefixes") || + test -d "$dstdir" || exit 1 + obsolete_mkdir_used=true + fi + fi + fi + + if test -n "$dir_arg"; then + { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && + { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || + test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 + else + + # Make a couple of temp file names in the proper directory. + dsttmp=$dstdir/_inst.$$_ + rmtmp=$dstdir/_rm.$$_ + + # Trap to clean up those temp files at exit. + trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 + + # Copy the file name to the temp name. + (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + + # and set any options; do chmod last to preserve setuid bits. + # + # If any of these fail, we abort the whole thing. If we want to + # ignore errors from any of these, just make sure not to ignore + # errors from the above "$doit $cpprog $src $dsttmp" command. + # + { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && + { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && + { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && + + # If -C, don't bother to copy if it wouldn't change the file. + if $copy_on_change && + old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && + new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && + + eval "$initialize_posix_glob" && + $posix_glob set -f && + set X $old && old=:$2:$4:$5:$6 && + set X $new && new=:$2:$4:$5:$6 && + $posix_glob set +f && + + test "$old" = "$new" && + $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 + then + rm -f "$dsttmp" + else + # Rename the file to the real destination. + $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || + + # The rename failed, perhaps because mv can't rename something else + # to itself, or perhaps because mv is so ancient that it does not + # support -f. + { + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + test ! -f "$dst" || + $doit $rmcmd -f "$dst" 2>/dev/null || + { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && + { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } + } || + { echo "$0: cannot unlink or rename $dst" >&2 + (exit 1); exit 1 + } + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dst" + } + fi || exit 1 + + trap '' 0 + fi +done + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/third_party/nix/contrib/stack-collapse.py b/third_party/nix/contrib/stack-collapse.py new file mode 100755 index 000000000000..f5602c95c48e --- /dev/null +++ b/third_party/nix/contrib/stack-collapse.py @@ -0,0 +1,38 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls '<nixpkgs>' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/third_party/nix/corepkgs/buildenv.nix b/third_party/nix/corepkgs/buildenv.nix new file mode 100644 index 000000000000..0bac4c44b48a --- /dev/null +++ b/third_party/nix/corepkgs/buildenv.nix @@ -0,0 +1,25 @@ +{ derivations, manifest }: + +derivation { + name = "user-environment"; + system = "builtin"; + builder = "builtin:buildenv"; + + inherit manifest; + + # !!! grmbl, need structured data for passing this in a clean way. + derivations = + map (d: + [ (d.meta.active or "true") + (d.meta.priority or 5) + (builtins.length d.outputs) + ] ++ map (output: builtins.getAttr output d) d.outputs) + derivations; + + # Building user environments remotely just causes huge amounts of + # network traffic, so don't do that. + preferLocalBuild = true; + + # Also don't bother substituting. + allowSubstitutes = false; +} diff --git a/third_party/nix/corepkgs/config.nix.in b/third_party/nix/corepkgs/config.nix.in new file mode 100644 index 000000000000..0e4a2f0c90c7 --- /dev/null +++ b/third_party/nix/corepkgs/config.nix.in @@ -0,0 +1,29 @@ +let + fromEnv = var: def: + let val = builtins.getEnv var; in + if val != "" then val else def; +in rec { + shell = "@bash@"; + coreutils = "@coreutils@"; + bzip2 = "@bzip2@"; + gzip = "@gzip@"; + xz = "@xz@"; + tar = "@tar@"; + tarFlags = "@tarFlags@"; + tr = "@tr@"; + nixBinDir = fromEnv "NIX_BIN_DIR" "@CMAKE_INSTALL_FULL_BINDIR@"; + nixPrefix = "@CMAKE_INSTALL_PREFIX@"; + nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@CMAKE_INSTALL_FULL_LIBEXECDIR@"; + nixLocalstateDir = "/nix/var"; + nixSysconfDir = "/etc"; + nixStoreDir = fromEnv "NIX_STORE_DIR" "/nix/store"; + + # If Nix is installed in the Nix store, then automatically add it as + # a dependency to the core packages. This ensures that they work + # properly in a chroot. + chrootDeps = + if dirOf nixPrefix == builtins.storeDir then + [ (builtins.storePath nixPrefix) ] + else + [ ]; +} diff --git a/third_party/nix/corepkgs/derivation.nix b/third_party/nix/corepkgs/derivation.nix new file mode 100644 index 000000000000..c0fbe8082cd3 --- /dev/null +++ b/third_party/nix/corepkgs/derivation.nix @@ -0,0 +1,27 @@ +/* This is the implementation of the ‘derivation’ builtin function. + It's actually a wrapper around the ‘derivationStrict’ primop. */ + +drvAttrs @ { outputs ? [ "out" ], ... }: + +let + + strict = derivationStrict drvAttrs; + + commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvAttrs; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName strict; + drvPath = strict.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/third_party/nix/corepkgs/fetchurl.nix b/third_party/nix/corepkgs/fetchurl.nix new file mode 100644 index 000000000000..a84777f57448 --- /dev/null +++ b/third_party/nix/corepkgs/fetchurl.nix @@ -0,0 +1,41 @@ +{ system ? "" # obsolete +, url +, hash ? "" # an SRI ash + +# Legacy hash specification +, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" +, outputHash ? + if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 +, outputHashAlgo ? + if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + +, executable ? false +, unpack ? false +, name ? baseNameOf (toString url) +}: + +derivation { + builder = "builtin:fetchurl"; + + # New-style output content requirements. + inherit outputHashAlgo outputHash; + outputHashMode = if unpack || executable then "recursive" else "flat"; + + inherit name url executable unpack; + + system = "builtin"; + + # No need to double the amount of network traffic + preferLocalBuild = true; + + impureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + # To make "nix-prefetch-url" work. + urls = [ url ]; +} diff --git a/third_party/nix/corepkgs/imported-drv-to-derivation.nix b/third_party/nix/corepkgs/imported-drv-to-derivation.nix new file mode 100644 index 000000000000..eab8b050e8ff --- /dev/null +++ b/third_party/nix/corepkgs/imported-drv-to-derivation.nix @@ -0,0 +1,21 @@ +attrs @ { drvPath, outputs, name, ... }: + +let + + commonAttrs = (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvPath name; + type = "derivation"; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName attrs; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/third_party/nix/corepkgs/unpack-channel.nix b/third_party/nix/corepkgs/unpack-channel.nix new file mode 100644 index 000000000000..d39a20637818 --- /dev/null +++ b/third_party/nix/corepkgs/unpack-channel.nix @@ -0,0 +1,39 @@ +with import <nix/config.nix>; + +let + + builder = builtins.toFile "unpack-channel.sh" + '' + mkdir $out + cd $out + xzpat="\.xz\$" + gzpat="\.gz\$" + if [[ "$src" =~ $xzpat ]]; then + ${xz} -d < $src | ${tar} xf - ${tarFlags} + elif [[ "$src" =~ $gzpat ]]; then + ${gzip} -d < $src | ${tar} xf - ${tarFlags} + else + ${bzip2} -d < $src | ${tar} xf - ${tarFlags} + fi + if [ * != $channelName ]; then + mv * $out/$channelName + fi + ''; + +in + +{ name, channelName, src }: + +derivation { + system = builtins.currentSystem; + builder = shell; + args = [ "-e" builder ]; + inherit name channelName src; + + PATH = "${nixBinDir}:${coreutils}"; + + # No point in doing this remotely. + preferLocalBuild = true; + + inherit chrootDeps; +} diff --git a/third_party/nix/default.nix b/third_party/nix/default.nix new file mode 100644 index 000000000000..3fc89fc7ebdc --- /dev/null +++ b/third_party/nix/default.nix @@ -0,0 +1,206 @@ +args@{ + depot ? (import ../.. {}) +, pkgs ? depot.third_party.nixpkgs +, lib +, buildType ? "release" +, ... +}: + +let + aws-s3-cpp = pkgs.aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }; + + src = let + srcDir = ./.; + # create relative paths for all the sources we are filtering + asRelative = path: + let + srcS = toString srcDir; + pathS = toString path; + in + if ! lib.hasPrefix srcS pathS then + throw "Path is outside of the working directory." + else + lib.removePrefix srcS pathS; + + in builtins.filterSource (path: type: + # Strip out .nix files that are in the root of the repository. Changing + # the expression of tvix shouldn't cause a rebuild of tvix unless really + # required. + !(dirOf (asRelative path) == "/" && lib.hasSuffix ".nix" path) && + + # remove the proto files from the repo as those are compiled separately + !(lib.hasPrefix "src/proto" (asRelative path)) && + + # ignore result symlinks + !(type == "symlink" && lib.hasPrefix "result" (baseNameOf path)) + ) srcDir; + + # Proto generation in CMake is theoretically possible, but that is + # very theoretical - this does it in Nix instead. + protoSrcs = pkgs.runCommand "nix-proto-srcs" {} '' + export PROTO_SRCS=${./src/proto} + mkdir -p $out/libproto + ${depot.third_party.protobuf}/bin/protoc -I=$PROTO_SRCS \ + --cpp_out=$out/libproto \ + --plugin=protoc-gen-grpc=${depot.third_party.grpc}/bin/grpc_cpp_plugin \ + --grpc_out=$out/libproto \ + $PROTO_SRCS/*.proto + ''; + + # Derivation for busybox that just has the `busybox` binary in bin/, not all + # the symlinks, so cmake can find it + busybox = pkgs.runCommand "busybox" {} '' + mkdir -p $out/bin + cp ${pkgs.busybox}/bin/busybox $out/bin + ''; + +in lib.fix (self: pkgs.fullLlvm11Stdenv.mkDerivation { + pname = "tvix"; + version = "2.3.4"; + inherit src; + + nativeBuildInputs = with pkgs; [ + bison + clang-tools_11 + cmake + libxml2 + libxslt + pkgconfig + (import ./clangd.nix pkgs) + ]; + + # TODO(tazjin): Some of these might only be required for native inputs + buildInputs = (with pkgs; [ + aws-s3-cpp + brotli + bzip2 + c-ares + curl + editline + flex + libseccomp + libsodium + systemd.dev + openssl + sqlite + xz + ]) ++ (with depot.third_party; [ + abseil_cpp + glog + grpc + protobuf + ]); + + doCheck = false; + doInstallCheck = true; + + # Preserve debug symbols, for core dumps + other live debugging + dontStrip = true; + + installCheckInputs = with depot.third_party; [ + gtest + pkgs.fd + rapidcheck + ]; + + propagatedBuildInputs = with pkgs; [ + boost + ]; + + configurePhase = '' + mkdir build + cd build + cmake .. \ + -DCMAKE_INSTALL_PREFIX=$out \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF \ + -DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF \ + -DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON + ''; + + installCheckPhase = '' + export NIX_DATA_DIR=$out/share + export NIX_TEST_VAR=foo # this is required by a language test + make test + + # Ensure formatting is coherent, but do this after the rest of the + # tests run so that developers get all the useful feedback + fd . $src -e hh -e cc | xargs clang-format --dry-run --Werror + ''; + + preBuild = '' + if [ -n "$NIX_BUILD_CORES" ]; then + makeFlags+="-j$NIX_BUILD_CORES " + makeFlags+="-l$NIX_BUILD_CORES " + fi + ''; + + # Forward the location of the generated Protobuf / gRPC files so + # that they can be included by CMake. + NIX_PROTO_SRCS = protoSrcs; + + # Work around broken system header include flags in the cxx toolchain. + LIBCXX_INCLUDE = "${pkgs.llvmPackages_11.libcxx}/include/c++/v1"; + + SANDBOX_SHELL="${pkgs.busybox}/bin/busybox"; + + # Install the various symlinks to the Nix binary which users expect + # to exist. + postInstall = '' + ln -s $out/bin/nix $out/bin/nix-build + ln -s $out/bin/nix $out/bin/nix-channel + ln -s $out/bin/nix $out/bin/nix-collect-garbage + ln -s $out/bin/nix $out/bin/nix-copy-closure + ln -s $out/bin/nix $out/bin/nix-env + ln -s $out/bin/nix $out/bin/nix-hash + ln -s $out/bin/nix $out/bin/nix-instantiate + ln -s $out/bin/nix $out/bin/nix-prefetch-url + ln -s $out/bin/nix $out/bin/nix-shell + ln -s $out/bin/nix $out/bin/nix-store + + mkdir -p $out/libexec/nix + ln -s $out/bin/nix $out/libexec/nix/build-remote + + # configuration variables for templated files + export storedir=/nix/store + export localstatedir=/nix/var + export bindir=$out/bin + + mkdir -p $out/lib/systemd/system + substituteAll \ + ${src}/misc/systemd/nix-daemon.service.in \ + $out/lib/systemd/system/nix-daemon.service + substituteAll \ + ${src}/misc/systemd/nix-daemon.socket.in \ + $out/lib/systemd/system/nix-daemon.socket + + mkdir -p $out/etc/profile.d + substituteAll \ + ${src}/scripts/nix-profile.sh.in $out/etc/profile.d/nix.sh + substituteAll \ + ${src}/scripts/nix-profile-daemon.sh.in $out/etc/profile.d/nix-daemon.sh + ''; + + # TODO(tazjin): integration test setup? + # TODO(tazjin): docs generation? + + passthru = { + build-shell = self.overrideAttrs (up: rec { + run_clang_tidy = pkgs.writeShellScriptBin "run-clang-tidy" '' + test -f compile_commands.json || (echo "run from build output directory"; exit 1) || exit 1 + ${pkgs.jq}/bin/jq < compile_commands.json -r 'map(.file)|.[]' | grep -v '/generated/' | ${pkgs.parallel}/bin/parallel ${pkgs.clang-tools}/bin/clang-tidy -p compile_commands.json $@ + ''; + + installCheckInputs = up.installCheckInputs ++ [run_clang_tidy]; + + shellHook = '' + export NIX_DATA_DIR="${toString depot.path}/third_party" + export NIX_TEST_VAR=foo + ''; + }); + test-vm = import ./test-vm.nix args; + }; +}) diff --git a/third_party/nix/doc/manual/advanced-topics/advanced-topics.xml b/third_party/nix/doc/manual/advanced-topics/advanced-topics.xml new file mode 100644 index 000000000000..871b7eb1d37b --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/advanced-topics.xml @@ -0,0 +1,14 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="part-advanced-topics" + version="5.0"> + +<title>Advanced Topics</title> + +<xi:include href="distributed-builds.xml" /> +<xi:include href="cores-vs-jobs.xml" /> +<xi:include href="diff-hook.xml" /> +<xi:include href="post-build-hook.xml" /> + +</part> diff --git a/third_party/nix/doc/manual/advanced-topics/cores-vs-jobs.xml b/third_party/nix/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 000000000000..eba645faf879 --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="chap-tuning-cores-and-jobs"> + +<title>Tuning Cores and Jobs</title> + +<para>Nix has two relevant settings with regards to how your CPU cores +will be utilized: <xref linkend="conf-cores" /> and +<xref linkend="conf-max-jobs" />. This chapter will talk about what +they are, how they interact, and their configuration trade-offs.</para> + +<variablelist> + <varlistentry> + <term><xref linkend="conf-max-jobs" /></term> + <listitem><para> + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + </para></listitem> + </varlistentry> + <varlistentry> + <term><xref linkend="conf-cores" /></term> + <listitem><para> + Suggests how many cores each derivation should use. Similar to + <command>make -j</command>. + </para></listitem> + </varlistentry> +</variablelist> + +<para>The <xref linkend="conf-cores" /> setting determines the value of +<envar>NIX_BUILD_CORES</envar>. <envar>NIX_BUILD_CORES</envar> is equal +to <xref linkend="conf-cores" />, unless <xref linkend="conf-cores" /> +equals <literal>0</literal>, in which case <envar>NIX_BUILD_CORES</envar> +will be the total number of cores in the system.</para> + +<para>The total number of consumed cores is a simple multiplication, +<xref linkend="conf-cores" /> * <envar>NIX_BUILD_CORES</envar>.</para> + +<para>The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores:</para> + +<table> + <caption>Balancing 24 Build Cores</caption> + <thead> + <tr> + <th><xref linkend="conf-max-jobs" /></th> + <th><xref linkend="conf-cores" /></th> + <th><envar>NIX_BUILD_CORES</envar></th> + <th>Maximum Processes</th> + <th>Result</th> + </tr> + </thead> + <tbody> + <tr> + <td>1</td> + <td>24</td> + <td>24</td> + <td>24</td> + <td> + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. + </td> + </tr> + + <tr> + <td>4</td> + <td>6</td> + <td>6</td> + <td>24</td> + <td> + Four derivations will be built at once, each given access to + six cores. + </td> + </tr> + <tr> + <td>12</td> + <td>6</td> + <td>6</td> + <td>72</td> + <td> + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. + </td> + </tr> + <tr> + <td>24</td> + <td>1</td> + <td>1</td> + <td>24</td> + <td> + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. + </td> + </tr> + <tr> + <td>24</td> + <td>0</td> + <td>24</td> + <td>576</td> + <td> + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. + </td> + </tr> + </tbody> +</table> + +<para>It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +<envar>NIX_BUILD_CORES</envar> environment variable.</para> + +</chapter> diff --git a/third_party/nix/doc/manual/advanced-topics/diff-hook.xml b/third_party/nix/doc/manual/advanced-topics/diff-hook.xml new file mode 100644 index 000000000000..fb4bf819f94b --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/diff-hook.xml @@ -0,0 +1,205 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="chap-diff-hook" + version="5.0" + > + +<title>Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option></title> + +<subtitle>Check build reproducibility by running builds multiple times +and comparing their results.</subtitle> + +<para>Specify a program with Nix's <xref linkend="conf-diff-hook" /> to +compare build results when two builds produce different results. Note: +this hook is only executed if the results are not the same, this hook +is not used for determining if the results are the same.</para> + +<para>For purposes of demonstration, we'll use the following Nix file, +<filename>deterministic.nix</filename> for testing:</para> + +<programlisting> +let + inherit (import <nixpkgs> {}) runCommand; +in { + stable = runCommand "stable" {} '' + touch $out + ''; + + unstable = runCommand "unstable" {} '' + echo $RANDOM > $out + ''; +} +</programlisting> + +<para>Additionally, <filename>nix.conf</filename> contains: + +<programlisting> +diff-hook = /etc/nix/my-diff-hook +run-diff-hook = true +</programlisting> + +where <filename>/etc/nix/my-diff-hook</filename> is an executable +file containing: + +<programlisting> +#!/bin/sh +exec >&2 +echo "For derivation $3:" +/run/current-system/sw/bin/diff -r "$1" "$2" +</programlisting> + +</para> + +<para>The diff hook is executed by the same user and group who ran the +build. However, the diff hook does not have write access to the store +path just built.</para> + +<section> + <title> + Spot-Checking Build Determinism + </title> + + <para> + Verify a path which already exists in the Nix store by passing + <option>--check</option> to the build command. + </para> + + <para>If the build passes and is deterministic, Nix will exit with a + status code of 0:</para> + + <screen> +$ nix-build ./deterministic.nix -A stable +these derivations will be built: + /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv +building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + +$ nix-build ./deterministic.nix -A stable --check +checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable +</screen> + + <para>If the build is not deterministic, Nix will exit with a status + code of 1:</para> + + <screen> +$ nix-build ./deterministic.nix -A unstable +these derivations will be built: + /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv +building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable + +$ nix-build ./deterministic.nix -A unstable --check +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs +</screen> + +<para>In the Nix daemon's log, we will now see: +<screen> +For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv: +1c1 +< 8108 +--- +> 30204 +</screen> +</para> + + <para>Using <option>--check</option> with <option>--keep-failed</option> + will cause Nix to keep the second build's output in a special, + <literal>.check</literal> path:</para> + + <screen> +$ nix-build ./deterministic.nix -A unstable --check --keep-failed +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +note: keeping build directory '/tmp/nix-build-unstable.drv-0' +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check' +</screen> + + <para>In particular, notice the + <literal>/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check</literal> + output. Nix has copied the build results to that directory where you + can examine it.</para> + + <note xml:id="check-dirs-are-unregistered"> + <title><literal>.check</literal> paths are not registered store paths</title> + + <para>Check paths are not protected against garbage collection, + and this path will be deleted on the next garbage collection.</para> + + <para>The path is guaranteed to be alive for the duration of + <xref linkend="conf-diff-hook" />'s execution, but may be deleted + any time after.</para> + + <para>If the comparison is performed as part of automated tooling, + please use the diff-hook or author your tooling to handle the case + where the build was not deterministic and also a check path does + not exist.</para> + </note> + + <para> + <option>--check</option> is only usable if the derivation has + been built on the system already. If the derivation has not been + built Nix will fail with the error: + <screen> +error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible +</screen> + + Run the build without <option>--check</option>, and then try with + <option>--check</option> again. + </para> +</section> + +<section> + <title> + Automatic and Optionally Enforced Determinism Verification + </title> + + <para> + Automatically verify every build at build time by executing the + build multiple times. + </para> + + <para> + Setting <xref linkend="conf-repeat" /> and + <xref linkend="conf-enforce-determinism" /> in your + <filename>nix.conf</filename> permits the automated verification + of every build Nix performs. + </para> + + <para> + The following configuration will run each build three times, and + will require the build to be deterministic: + + <programlisting> +enforce-determinism = true +repeat = 2 +</programlisting> + </para> + + <para> + Setting <xref linkend="conf-enforce-determinism" /> to false as in + the following configuration will run the build multiple times, + execute the build hook, but will allow the build to succeed even + if it does not build reproducibly: + + <programlisting> +enforce-determinism = false +repeat = 1 +</programlisting> + </para> + + <para> + An example output of this configuration: + <screen> +$ nix-build ./test.nix -A unstable +these derivations will be built: + /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... +output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round +/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable +</screen> + </para> +</section> +</chapter> diff --git a/third_party/nix/doc/manual/advanced-topics/distributed-builds.xml b/third_party/nix/doc/manual/advanced-topics/distributed-builds.xml new file mode 100644 index 000000000000..9ac4a92cd5b1 --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/distributed-builds.xml @@ -0,0 +1,190 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='chap-distributed-builds'> + +<title>Remote Builds</title> + +<para>Nix supports remote builds, where a local Nix installation can +forward Nix builds to other machines. This allows multiple builds to +be performed in parallel and allows Nix to perform multi-platform +builds in a semi-transparent way. For instance, if you perform a +build for a <literal>x86_64-darwin</literal> on an +<literal>i686-linux</literal> machine, Nix can automatically forward +the build to a <literal>x86_64-darwin</literal> machine, if +available.</para> + +<para>To forward a build to a remote machine, it’s required that the +remote machine is accessible via SSH and that it has Nix +installed. You can test whether connecting to the remote Nix instance +works, e.g. + +<screen> +$ nix ping-store --store ssh://mac +</screen> + +will try to connect to the machine named <literal>mac</literal>. It is +possible to specify an SSH identity file as part of the remote store +URI, e.g. + +<screen> +$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key +</screen> + +Since builds should be non-interactive, the key should not have a +passphrase. Alternatively, you can load identities ahead of time into +<command>ssh-agent</command> or <command>gpg-agent</command>.</para> + +<para>If you get the error + +<screen> +bash: nix-store: command not found +error: cannot connect to 'mac' +</screen> + +then you need to ensure that the <envar>PATH</envar> of +non-interactive login shells contains Nix.</para> + +<warning><para>If you are building via the Nix daemon, it is the Nix +daemon user account (that is, <literal>root</literal>) that should +have SSH access to the remote machine. If you can’t or don’t want to +configure <literal>root</literal> to be able to access to remote +machine, you can use a private Nix store instead by passing +e.g. <literal>--store ~/my-nix</literal>.</para></warning> + +<para>The list of remote machines can be specified on the command line +or in the Nix configuration file. The former is convenient for +testing. For example, the following command allows you to build a +derivation for <literal>x86_64-darwin</literal> on a Linux machine: + +<screen> +$ uname +Linux + +$ nix build \ + '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ + --builders 'ssh://mac x86_64-darwin' +[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac + +$ cat ./result +Darwin +</screen> + +It is possible to specify multiple builders separated by a semicolon +or a newline, e.g. + +<screen> + --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' +</screen> +</para> + +<para>Each machine specification consists of the following elements, +separated by spaces. Only the first element is required. +To leave a field at its default, set it to <literal>-</literal>. + +<orderedlist> + + <listitem><para>The URI of the remote store in the format + <literal>ssh://[<replaceable>username</replaceable>@]<replaceable>hostname</replaceable></literal>, + e.g. <literal>ssh://nix@mac</literal> or + <literal>ssh://mac</literal>. For backward compatibility, + <literal>ssh://</literal> may be omitted. The hostname may be an + alias defined in your + <filename>~/.ssh/config</filename>.</para></listitem> + + <listitem><para>A comma-separated list of Nix platform type + identifiers, such as <literal>x86_64-darwin</literal>. It is + possible for a machine to support multiple platform types, e.g., + <literal>i686-linux,x86_64-linux</literal>. If omitted, this + defaults to the local platform type.</para></listitem> + + <listitem><para>The SSH identity file to be used to log in to the + remote machine. If omitted, SSH will use its regular + identities.</para></listitem> + + <listitem><para>The maximum number of builds that Nix will execute + in parallel on the machine. Typically this should be equal to the + number of CPU cores. For instance, the machine + <literal>itchy</literal> in the example will execute up to 8 builds + in parallel.</para></listitem> + + <listitem><para>The “speed factor”, indicating the relative speed of + the machine. If there are multiple machines of the right type, Nix + will prefer the fastest, taking load into account.</para></listitem> + + <listitem><para>A comma-separated list of <emphasis>supported + features</emphasis>. If a derivation has the + <varname>requiredSystemFeatures</varname> attribute, then Nix will + only perform the derivation on a machine that has the specified + features. For instance, the attribute + +<programlisting> +requiredSystemFeatures = [ "kvm" ]; +</programlisting> + + will cause the build to be performed on a machine that has the + <literal>kvm</literal> feature.</para></listitem> + + <listitem><para>A comma-separated list of <emphasis>mandatory + features</emphasis>. A machine will only be used to build a + derivation if all of the machine’s mandatory features appear in the + derivation’s <varname>requiredSystemFeatures</varname> + attribute..</para></listitem> + +</orderedlist> + +For example, the machine specification + +<programlisting> +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark +</programlisting> + +specifies several machines that can perform +<literal>i686-linux</literal> builds. However, +<literal>poochie</literal> will only do builds that have the attribute + +<programlisting> +requiredSystemFeatures = [ "benchmark" ]; +</programlisting> + +or + +<programlisting> +requiredSystemFeatures = [ "benchmark" "kvm" ]; +</programlisting> + +<literal>itchy</literal> cannot do builds that require +<literal>kvm</literal>, but <literal>scratchy</literal> does support +such builds. For regular builds, <literal>itchy</literal> will be +preferred over <literal>scratchy</literal> because it has a higher +speed factor.</para> + +<para>Remote builders can also be configured in +<filename>nix.conf</filename>, e.g. + +<programlisting> +builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd +</programlisting> + +Finally, remote builders can be configured in a separate configuration +file included in <option>builders</option> via the syntax +<literal>@<replaceable>file</replaceable></literal>. For example, + +<programlisting> +builders = @/etc/nix/machines +</programlisting> + +causes the list of machines in <filename>/etc/nix/machines</filename> +to be included. (This is the default.)</para> + +<para>If you want the builders to use caches, you likely want to set +the option <link linkend='conf-builders-use-substitutes'><literal>builders-use-substitutes</literal></link> +in your local <filename>nix.conf</filename>.</para> + +<para>To build only on remote builders and disable building on the local machine, +you can use the option <option>--max-jobs 0</option>.</para> + +</chapter> diff --git a/third_party/nix/doc/manual/advanced-topics/post-build-hook.xml b/third_party/nix/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000000..3dc43ee795b1 --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="chap-post-build-hook" + version="5.0" + > + +<title>Using the <xref linkend="conf-post-build-hook" /></title> +<subtitle>Uploading to an S3-compatible binary cache after each build</subtitle> + + +<section xml:id="chap-post-build-hook-caveats"> + <title>Implementation Caveats</title> + <para>Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases.</para> + + <para>The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails.</para> + + <para>Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable.</para> + + <para>A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop.</para> +</section> + +<section> + <title>Prerequisites</title> + + <para> + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + <xref linkend="ssec-s3-substituter-authenticated-writes" />, and + that the <literal>root</literal> user's default AWS profile can + upload to the bucket. + </para> +</section> + +<section> + <title>Set up a Signing Key</title> + <para>Use <command>nix-store --generate-binary-cache-key</command> to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths.</para> + + <screen> +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +</screen> + +<para>Then, add the public key and the cache URL to your +<filename>nix.conf</filename>'s <xref linkend="conf-trusted-public-keys" /> +and <xref linkend="conf-substituters" /> like:</para> + +<programlisting> +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= +</programlisting> + +<para>we will restart the Nix daemon a later step.</para> +</section> + +<section> + <title>Implementing the build hook</title> + <para>Write the following script to + <filename>/etc/nix/upload-to-cache.sh</filename>: + </para> + + <programlisting> +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS +</programlisting> + + <note> + <title>Should <literal>$OUT_PATHS</literal> be quoted?</title> + <para> + The <literal>$OUT_PATHS</literal> variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to <command>nix sign-paths</command>. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The <command>set -f</command> + disables globbing in the shell. + </para> + </note> + <para> + Then make sure the hook program is executable by the <literal>root</literal> user: + <screen> +# chmod +x /etc/nix/upload-to-cache.sh +</screen></para> +</section> + +<section> + <title>Updating Nix Configuration</title> + + <para>Edit <filename>/etc/nix/nix.conf</filename> to run our hook, + by adding the following configuration snippet at the end:</para> + + <programlisting> +post-build-hook = /etc/nix/upload-to-cache.sh +</programlisting> + +<para>Then, restart the <command>nix-daemon</command>.</para> +</section> + +<section> + <title>Testing</title> + + <para>Build any derivation, for example:</para> + + <screen> +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +</screen> + + <para>Then delete the path from the store, and try substituting it from the binary cache:</para> + <screen> +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +</screen> + +<para>Now, copy the path back from the cache:</para> +<screen> +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example +</screen> +</section> +<section> + <title>Conclusion</title> + <para> + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + </para> + + <para> + Before deploying this to production, be sure to consider the + implementation caveats in <xref linkend="chap-post-build-hook-caveats" />. + </para> +</section> +</chapter> diff --git a/third_party/nix/doc/manual/command-ref/command-ref.xml b/third_party/nix/doc/manual/command-ref/command-ref.xml new file mode 100644 index 000000000000..cfad9b7d7966 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/command-ref.xml @@ -0,0 +1,20 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='part-command-ref'> + +<title>Command Reference</title> + +<partintro> +<para>This section lists commands and options that you can use when you +work with Nix.</para> +</partintro> + +<xi:include href="opt-common.xml" /> +<xi:include href="env-common.xml" /> +<xi:include href="main-commands.xml" /> +<xi:include href="utilities.xml" /> +<xi:include href="files.xml" /> + +</part> diff --git a/third_party/nix/doc/manual/command-ref/conf-file.xml b/third_party/nix/doc/manual/command-ref/conf-file.xml new file mode 100644 index 000000000000..4a5400b1936f --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/conf-file.xml @@ -0,0 +1,1202 @@ +<?xml version="1.0" encoding="utf-8"?> +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="sec-conf-file" + version="5"> + +<refmeta> + <refentrytitle>nix.conf</refentrytitle> + <manvolnum>5</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix.conf</refname> + <refpurpose>Nix configuration file</refpurpose> +</refnamediv> + +<refsection><title>Description</title> + +<para>Nix reads settings from two configuration files:</para> + +<itemizedlist> + + <listitem> + <para>The system-wide configuration file + <filename><replaceable>sysconfdir</replaceable>/nix/nix.conf</filename> + (i.e. <filename>/etc/nix/nix.conf</filename> on most systems), or + <filename>$NIX_CONF_DIR/nix.conf</filename> if + <envar>NIX_CONF_DIR</envar> is set.</para> + </listitem> + + <listitem> + <para>The user configuration file + <filename>$XDG_CONFIG_HOME/nix/nix.conf</filename>, or + <filename>~/.config/nix/nix.conf</filename> if + <envar>XDG_CONFIG_HOME</envar> is not set.</para> + </listitem> + +</itemizedlist> + +<para>The configuration files consist of +<literal><replaceable>name</replaceable> = +<replaceable>value</replaceable></literal> pairs, one per line. Other +files can be included with a line like <literal>include +<replaceable>path</replaceable></literal>, where +<replaceable>path</replaceable> is interpreted relative to the current +conf file and a missing file is an error unless +<literal>!include</literal> is used instead. +Comments start with a <literal>#</literal> character. Here is an +example configuration file:</para> + +<programlisting> +keep-outputs = true # Nice for developers +keep-derivations = true # Idem +</programlisting> + +<para>You can override settings on the command line using the +<option>--option</option> flag, e.g. <literal>--option keep-outputs +false</literal>.</para> + +<para>The following settings are currently available: + +<variablelist> + + + <varlistentry xml:id="conf-allowed-uris"><term><literal>allowed-uris</literal></term> + + <listitem> + + <para>A list of URI prefixes to which access is allowed in + restricted evaluation mode. For example, when set to + <literal>https://github.com/NixOS</literal>, builtin functions + such as <function>fetchGit</function> are allowed to access + <literal>https://github.com/NixOS/patchelf.git</literal>.</para> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-allow-import-from-derivation"><term><literal>allow-import-from-derivation</literal></term> + + <listitem><para>By default, Nix allows you to <function>import</function> from a derivation, + allowing building at evaluation time. With this option set to false, Nix will throw an error + when evaluating an expression that uses this feature, allowing users to ensure their evaluation + will not require any builds to take place.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-allow-new-privileges"><term><literal>allow-new-privileges</literal></term> + + <listitem><para>(Linux-specific.) By default, builders on Linux + cannot acquire new privileges by calling setuid/setgid programs or + programs that have file capabilities. For example, programs such + as <command>sudo</command> or <command>ping</command> will + fail. (Note that in sandbox builds, no such programs are available + unless you bind-mount them into the sandbox via the + <option>sandbox-paths</option> option.) You can allow the + use of such programs by enabling this option. This is impure and + usually undesirable, but may be useful in certain scenarios + (e.g. to spin up containers or set up userspace network interfaces + in tests).</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-allowed-users"><term><literal>allowed-users</literal></term> + + <listitem> + + <para>A list of names of users (separated by whitespace) that + are allowed to connect to the Nix daemon. As with the + <option>trusted-users</option> option, you can specify groups by + prefixing them with <literal>@</literal>. Also, you can allow + all users by specifying <literal>*</literal>. The default is + <literal>*</literal>.</para> + + <para>Note that trusted users are always allowed to connect.</para> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-auto-optimise-store"><term><literal>auto-optimise-store</literal></term> + + <listitem><para>If set to <literal>true</literal>, Nix + automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to <literal>false</literal> (the + default), you can still run <command>nix-store + --optimise</command> to get rid of duplicate + files.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-builders"> + <term><literal>builders</literal></term> + <listitem> + <para>A list of machines on which to perform builds. <phrase + condition="manual">See <xref linkend="chap-distributed-builds" + /> for details.</phrase></para> + </listitem> + </varlistentry> + + + <varlistentry xml:id="conf-builders-use-substitutes"><term><literal>builders-use-substitutes</literal></term> + + <listitem><para>If set to <literal>true</literal>, Nix will instruct + remote build machines to use their own binary substitutes if available. In + practical terms, this means that remote hosts will fetch as many build + dependencies as possible from their own substitutes (e.g, from + <literal>cache.nixos.org</literal>), instead of waiting for this host to + upload them all. This can drastically reduce build times if the network + connection between this computer and the remote build host is slow. Defaults + to <literal>false</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-build-users-group"><term><literal>build-users-group</literal></term> + + <listitem><para>This options specifies the Unix group containing + the Nix build user accounts. In multi-user Nix installations, + builds should not be performed by the Nix account since that would + allow users to arbitrarily modify the Nix store and database by + supplying specially crafted builders; and they cannot be performed + by the calling user since that would allow him/her to influence + the build result.</para> + + <para>Therefore, if this option is non-empty and specifies a valid + group, builds will be performed under the user accounts that are a + member of the group specified here (as listed in + <filename>/etc/group</filename>). Those user accounts should not + be used for any other purpose!</para> + + <para>Nix will never run two builds under the same user account at + the same time. This is to prevent an obvious security hole: a + malicious user writing a Nix expression that modifies the build + result of a legitimate Nix expression being built by another user. + Therefore it is good to have as many Nix build user accounts as + you can spare. (Remember: uids are cheap.)</para> + + <para>The build users should have permission to create files in + the Nix store, but not delete them. Therefore, + <filename>/nix/store</filename> should be owned by the Nix + account, its group should be the group specified here, and its + mode should be <literal>1775</literal>.</para> + + <para>If the build users group is empty, builds will be performed + under the uid of the Nix process (that is, the uid of the caller + if <envar>NIX_REMOTE</envar> is empty, the uid under which the Nix + daemon runs if <envar>NIX_REMOTE</envar> is + <literal>daemon</literal>). Obviously, this should not be used in + multi-user settings with untrusted users.</para> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-compress-build-log"><term><literal>compress-build-log</literal></term> + + <listitem><para>If set to <literal>true</literal> (the default), + build logs written to <filename>/nix/var/log/nix/drvs</filename> + will be compressed on the fly using bzip2. Otherwise, they will + not be compressed.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-connect-timeout"><term><literal>connect-timeout</literal></term> + + <listitem> + + <para>The timeout (in seconds) for establishing connections in + the binary cache substituter. It corresponds to + <command>curl</command>’s <option>--connect-timeout</option> + option.</para> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-cores"><term><literal>cores</literal></term> + + <listitem><para>Sets the value of the + <envar>NIX_BUILD_CORES</envar> environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + <varname>enableParallelBuilding</varname> is set to + <literal>true</literal>, the builder passes the + <option>-j<replaceable>N</replaceable></option> flag to GNU Make. + It can be overridden using the <option + linkend='opt-cores'>--cores</option> command line switch and + defaults to <literal>1</literal>. The value <literal>0</literal> + means that the builder should use all available CPU cores in the + system.</para> + + <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-diff-hook"><term><literal>diff-hook</literal></term> + <listitem> + <para> + Absolute path to an executable capable of diffing build results. + The hook executes if <xref linkend="conf-run-diff-hook" /> is + true, and the output of a build is known to not be the same. + This program is not executed to determine if two results are the + same. + </para> + + <para> + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + </para> + + <para>The diff hook program receives three parameters:</para> + + <orderedlist> + <listitem> + <para> + A path to the previous build's results + </para> + </listitem> + + <listitem> + <para> + A path to the current build's results + </para> + </listitem> + + <listitem> + <para> + The path to the build's derivation + </para> + </listitem> + + <listitem> + <para> + The path to the build's scratch directory. This directory + will exist only if the build was run with + <option>--keep-failed</option>. + </para> + </listitem> + </orderedlist> + + <para> + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. + </para> + + <para>When using the Nix daemon, <literal>diff-hook</literal> must + be set in the <filename>nix.conf</filename> configuration file, and + cannot be passed at the command line. + </para> + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-enforce-determinism"> + <term><literal>enforce-determinism</literal></term> + + <listitem><para>See <xref linkend="conf-repeat" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-extra-sandbox-paths"> + <term><literal>extra-sandbox-paths</literal></term> + + <listitem><para>A list of additional paths appended to + <option>sandbox-paths</option>. Useful if you want to extend + its default value.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-extra-platforms"><term><literal>extra-platforms</literal></term> + + <listitem><para>Platforms other than the native one which + this machine is capable of building for. This can be useful for + supporting additional architectures on compatible machines: + i686-linux can be built on x86_64-linux machines (and the default + for this setting reflects this); armv7 is backwards-compatible with + armv6 and armv5tel; some aarch64 machines can also natively run + 32-bit ARM code; and qemu-user may be used to support non-native + platforms (though this may be slow and buggy). Most values for this + are not enabled by default because build systems will often + misdetect the target platform and generate incompatible code, so you + may wish to cross-check the results of using this option against + proper natively-built versions of your + derivations.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-extra-substituters"><term><literal>extra-substituters</literal></term> + + <listitem><para>Additional binary caches appended to those + specified in <option>substituters</option>. When used by + unprivileged users, untrusted substituters (i.e. those not listed + in <option>trusted-substituters</option>) are silently + ignored.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-fallback"><term><literal>fallback</literal></term> + + <listitem><para>If set to <literal>true</literal>, Nix will fall + back to building from source if a binary substitute fails. This + is equivalent to the <option>--fallback</option> flag. The + default is <literal>false</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-fsync-metadata"><term><literal>fsync-metadata</literal></term> + + <listitem><para>If set to <literal>true</literal>, changes to the + Nix store metadata (in <filename>/nix/var/nix/db</filename>) are + synchronously flushed to disk. This improves robustness in case + of system crashes, but reduces performance. The default is + <literal>true</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-hashed-mirrors"><term><literal>hashed-mirrors</literal></term> + + <listitem><para>A list of web servers used by + <function>builtins.fetchurl</function> to obtain files by + hash. The default is + <literal>http://tarballs.nixos.org/</literal>. Given a hash type + <replaceable>ht</replaceable> and a base-16 hash + <replaceable>h</replaceable>, Nix will try to download the file + from + <literal>hashed-mirror/<replaceable>ht</replaceable>/<replaceable>h</replaceable></literal>. + This allows files to be downloaded even if they have disappeared + from their original URI. For example, given the default mirror + <literal>http://tarballs.nixos.org/</literal>, when building the derivation + +<programlisting> +builtins.fetchurl { + url = https://example.org/foo-1.2.3.tar.xz; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; +} +</programlisting> + + Nix will attempt to download this file from + <literal>http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae</literal> + first. If it is not available there, if will try the original URI.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-http-connections"><term><literal>http-connections</literal></term> + + <listitem><para>The maximum number of parallel TCP connections + used to fetch files from binary caches and by other downloads. It + defaults to 25. 0 means no limit.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-keep-build-log"><term><literal>keep-build-log</literal></term> + + <listitem><para>If set to <literal>true</literal> (the default), + Nix will write the build log of a derivation (i.e. the standard + output and error of its builder) to the directory + <filename>/nix/var/log/nix/drvs</filename>. The build log can be + retrieved using the command <command>nix-store -l + <replaceable>path</replaceable></command>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-keep-derivations"><term><literal>keep-derivations</literal></term> + + <listitem><para>If <literal>true</literal> (default), the garbage + collector will keep the derivations from which non-garbage store + paths were built. If <literal>false</literal>, they will be + deleted unless explicitly registered as a root (or reachable from + other roots).</para> + + <para>Keeping derivation around is useful for querying and + traceability (e.g., it allows you to ask with what dependencies or + options a store path was built), so by default this option is on. + Turn it off to save a bit of disk space (or a lot if + <literal>keep-outputs</literal> is also turned on).</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-keep-env-derivations"><term><literal>keep-env-derivations</literal></term> + + <listitem><para>If <literal>false</literal> (default), derivations + are not stored in Nix user environments. That is, the derivations of + any build-time-only dependencies may be garbage-collected.</para> + + <para>If <literal>true</literal>, when you add a Nix derivation to + a user environment, the path of the derivation is stored in the + user environment. Thus, the derivation will not be + garbage-collected until the user environment generation is deleted + (<command>nix-env --delete-generations</command>). To prevent + build-time-only dependencies from being collected, you should also + turn on <literal>keep-outputs</literal>.</para> + + <para>The difference between this option and + <literal>keep-derivations</literal> is that this one is + “sticky”: it applies to any user environment created while this + option was enabled, while <literal>keep-derivations</literal> + only applies at the moment the garbage collector is + run.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-keep-outputs"><term><literal>keep-outputs</literal></term> + + <listitem><para>If <literal>true</literal>, the garbage collector + will keep the outputs of non-garbage derivations. If + <literal>false</literal> (default), outputs will be deleted unless + they are GC roots themselves (or reachable from other roots).</para> + + <para>In general, outputs must be registered as roots separately. + However, even if the output of a derivation is registered as a + root, the collector will still delete store paths that are used + only at build time (e.g., the C compiler, or source tarballs + downloaded from the network). To prevent it from doing so, set + this option to <literal>true</literal>.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-max-build-log-size"><term><literal>max-build-log-size</literal></term> + + <listitem> + + <para>This option defines the maximum number of bytes that a + builder can write to its stdout/stderr. If the builder exceeds + this limit, it’s killed. A value of <literal>0</literal> (the + default) means that there is no limit.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-max-free"><term><literal>max-free</literal></term> + + <listitem><para>When a garbage collection is triggered by the + <literal>min-free</literal> option, it stops as soon as + <literal>max-free</literal> bytes are available. The default is + infinity (i.e. delete all garbage).</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-max-jobs"><term><literal>max-jobs</literal></term> + + <listitem><para>This option defines the maximum number of jobs + that Nix will try to build in parallel. The default is + <literal>1</literal>. The special value <literal>auto</literal> + causes Nix to use the number of CPUs in your system. <literal>0</literal> + is useful when using remote builders to prevent any local builds (except for + <literal>preferLocalBuild</literal> derivation attribute which executes locally + regardless). It can be + overridden using the <option + linkend='opt-max-jobs'>--max-jobs</option> (<option>-j</option>) + command line switch.</para> + + <para>See also <xref linkend="chap-tuning-cores-and-jobs" />.</para> + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-max-silent-time"><term><literal>max-silent-time</literal></term> + + <listitem> + + <para>This option defines the maximum number of seconds that a + builder can go without producing any data on standard output or + standard error. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite + loop, or to catch remote builds that are hanging due to network + problems. It can be overridden using the <option + linkend="opt-max-silent-time">--max-silent-time</option> command + line switch.</para> + + <para>The value <literal>0</literal> means that there is no + timeout. This is also the default.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-min-free"><term><literal>min-free</literal></term> + + <listitem> + <para>When free disk space in <filename>/nix/store</filename> + drops below <literal>min-free</literal> during a build, Nix + performs a garbage-collection until <literal>max-free</literal> + bytes are available or there is no more garbage. A value of + <literal>0</literal> (the default) disables this feature.</para> + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-narinfo-cache-negative-ttl"><term><literal>narinfo-cache-negative-ttl</literal></term> + + <listitem> + + <para>The TTL in seconds for negative lookups. If a store path is + queried from a substituter but was not found, there will be a + negative lookup cached in the local disk cache database for the + specified duration.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-narinfo-cache-positive-ttl"><term><literal>narinfo-cache-positive-ttl</literal></term> + + <listitem> + + <para>The TTL in seconds for positive lookups. If a store path is + queried from a substituter, the result of the query will be cached + in the local disk cache database including some of the NAR + metadata. The default TTL is a month, setting a shorter TTL for + positive lookups can be useful for binary caches that have + frequent garbage collection, in which case having a more frequent + cache invalidation would prevent trying to pull the path again and + failing with a hash mismatch if the build isn't reproducible. + </para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-netrc-file"><term><literal>netrc-file</literal></term> + + <listitem><para>If set to an absolute path to a <filename>netrc</filename> + file, Nix will use the HTTP authentication credentials in this file when + trying to download from a remote host through HTTP or HTTPS. Defaults to + <filename>$NIX_CONF_DIR/netrc</filename>.</para> + + <para>The <filename>netrc</filename> file consists of a list of + accounts in the following format: + +<screen> +machine <replaceable>my-machine</replaceable> +login <replaceable>my-username</replaceable> +password <replaceable>my-password</replaceable> +</screen> + + For the exact syntax, see <link + xlink:href="https://ec.haxx.se/usingcurl-netrc.html">the + <literal>curl</literal> documentation.</link></para> + + <note><para>This must be an absolute path, and <literal>~</literal> + is not resolved. For example, <filename>~/.netrc</filename> won't + resolve to your home directory's <filename>.netrc</filename>.</para></note> + </listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-pre-build-hook"><term><literal>pre-build-hook</literal></term> + + <listitem> + + + <para>If set, the path to a program that can set extra + derivation-specific settings for this system. This is used for settings + that can't be captured by the derivation model itself and are too variable + between different versions of the same system to be hard-coded into nix. + </para> + + <para>The hook is passed the derivation path and, if sandboxes are enabled, + the sandbox directory. It can then modify the sandbox and send a series of + commands to modify various settings to stdout. The currently recognized + commands are:</para> + + <variablelist> + <varlistentry xml:id="extra-sandbox-paths"> + <term><literal>extra-sandbox-paths</literal></term> + + <listitem> + + <para>Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an empty + line. Entries have the same format as + <literal>sandbox-paths</literal>.</para> + + </listitem> + + </varlistentry> + </variablelist> + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-post-build-hook"> + <term><literal>post-build-hook</literal></term> + <listitem> + <para>Optional. The path to a program to execute after each build.</para> + + <para>This option is only settable in the global + <filename>nix.conf</filename>, or on the command line by trusted + users.</para> + + <para>When using the nix-daemon, the daemon executes the hook as + <literal>root</literal>. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build.</para> + + <itemizedlist> + <listitem><para>The hook executes after an evaluation-time build.</para></listitem> + <listitem><para>The hook does not execute on substituted paths.</para></listitem> + <listitem><para>The hook's output always goes to the user's terminal.</para></listitem> + <listitem><para>If the hook fails, the build succeeds but no further builds execute.</para></listitem> + <listitem><para>The hook executes synchronously, and blocks other builds from progressing while it runs.</para></listitem> + </itemizedlist> + + <para>The program executes with no arguments. The program's environment + contains the following environment variables:</para> + + <variablelist> + <varlistentry> + <term><envar>DRV_PATH</envar></term> + <listitem> + <para>The derivation for the built paths.</para> + <para>Example: + <literal>/nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv</literal> + </para> + </listitem> + </varlistentry> + + <varlistentry> + <term><envar>OUT_PATHS</envar></term> + <listitem> + <para>Output paths of the built derivation, separated by a space character.</para> + <para>Example: + <literal>/nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <para>See <xref linkend="chap-post-build-hook" /> for an example + implementation.</para> + + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-repeat"><term><literal>repeat</literal></term> + + <listitem><para>How many times to repeat builds to check whether + they are deterministic. The default value is 0. If the value is + non-zero, every build is repeated the specified number of + times. If the contents of any of the runs differs from the + previous ones and <xref linkend="conf-enforce-determinism" /> is + true, the build is rejected and the resulting store paths are not + registered as “valid” in Nix’s database.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-require-sigs"><term><literal>require-sigs</literal></term> + + <listitem><para>If set to <literal>true</literal> (the default), + any non-content-addressed path added or copied to the Nix store + (e.g. when substituting from a binary cache) must have a valid + signature, that is, be signed using one of the keys listed in + <option>trusted-public-keys</option> or + <option>secret-key-files</option>. Set to <literal>false</literal> + to disable signature checking.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-restrict-eval"><term><literal>restrict-eval</literal></term> + + <listitem> + + <para>If set to <literal>true</literal>, the Nix evaluator will + not allow access to any files outside of the Nix search path (as + set via the <envar>NIX_PATH</envar> environment variable or the + <option>-I</option> option), or to URIs outside of + <option>allowed-uri</option>. The default is + <literal>false</literal>.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-run-diff-hook"><term><literal>run-diff-hook</literal></term> + <listitem> + <para> + If true, enable the execution of <xref linkend="conf-diff-hook" />. + </para> + + <para> + When using the Nix daemon, <literal>run-diff-hook</literal> must + be set in the <filename>nix.conf</filename> configuration file, + and cannot be passed at the command line. + </para> + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-sandbox"><term><literal>sandbox</literal></term> + + <listitem><para>If set to <literal>true</literal>, builds will be + performed in a <emphasis>sandboxed environment</emphasis>, i.e., + they’re isolated from the normal file system hierarchy and will + only see their dependencies in the Nix store, the temporary build + directory, private versions of <filename>/proc</filename>, + <filename>/dev</filename>, <filename>/dev/shm</filename> and + <filename>/dev/pts</filename> (on Linux), and the paths configured with the + <link linkend='conf-sandbox-paths'><literal>sandbox-paths</literal> + option</link>. This is useful to prevent undeclared dependencies + on files in directories such as <filename>/usr/bin</filename>. In + addition, on Linux, builds run in private PID, mount, network, IPC + and UTS namespaces to isolate them from other processes in the + system (except that fixed-output derivations do not run in private + network namespace to ensure they can access the network).</para> + + <para>Currently, sandboxing only work on Linux and macOS. The use + of a sandbox requires that Nix is run as root (so you should use + the <link linkend='conf-build-users-group'>“build users” + feature</link> to perform the actual builds under different users + than root).</para> + + <para>If this option is set to <literal>relaxed</literal>, then + fixed-output derivations and derivations that have the + <varname>__noChroot</varname> attribute set to + <literal>true</literal> do not run in sandboxes.</para> + + <para>The default is <literal>true</literal> on Linux and + <literal>false</literal> on all other platforms.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-sandbox-dev-shm-size"><term><literal>sandbox-dev-shm-size</literal></term> + + <listitem><para>This option determines the maximum size of the + <literal>tmpfs</literal> filesystem mounted on + <filename>/dev/shm</filename> in Linux sandboxes. For the format, + see the description of the <option>size</option> option of + <literal>tmpfs</literal> in + <citerefentry><refentrytitle>mount</refentrytitle><manvolnum>8</manvolnum></citerefentry>. The + default is <literal>50%</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-sandbox-paths"> + <term><literal>sandbox-paths</literal></term> + + <listitem><para>A list of paths bind-mounted into Nix sandbox + environments. You can use the syntax + <literal><replaceable>target</replaceable>=<replaceable>source</replaceable></literal> + to mount a path in a different location in the sandbox; for + instance, <literal>/bin=/nix-bin</literal> will mount the path + <literal>/nix-bin</literal> as <literal>/bin</literal> inside the + sandbox. If <replaceable>source</replaceable> is followed by + <literal>?</literal>, then it is not an error if + <replaceable>source</replaceable> does not exist; for example, + <literal>/dev/nvidiactl?</literal> specifies that + <filename>/dev/nvidiactl</filename> will only be mounted in the + sandbox if it exists in the host filesystem.</para> + + <para>Depending on how Nix was built, the default value for this option + may be empty or provide <filename>/bin/sh</filename> as a + bind-mount of <command>bash</command>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-secret-key-files"><term><literal>secret-key-files</literal></term> + + <listitem><para>A whitespace-separated list of files containing + secret (private) keys. These are used to sign locally-built + paths. They can be generated using <command>nix-store + --generate-binary-cache-key</command>. The corresponding public + key can be distributed to other users, who can add it to + <option>trusted-public-keys</option> in their + <filename>nix.conf</filename>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-show-trace"><term><literal>show-trace</literal></term> + + <listitem><para>Causes Nix to print out a stack trace in case of Nix + expression evaluation errors.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-substitute"><term><literal>substitute</literal></term> + + <listitem><para>If set to <literal>true</literal> (default), Nix + will use binary substitutes if available. This option can be + disabled to force building from source.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-stalled-download-timeout"><term><literal>stalled-download-timeout</literal></term> + <listitem> + <para>The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration.</para> + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-substituters"><term><literal>substituters</literal></term> + + <listitem><para>A list of URLs of substituters, separated by + whitespace. The default is + <literal>https://cache.nixos.org</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-system"><term><literal>system</literal></term> + + <listitem><para>This option specifies the canonical Nix system + name of the current installation, such as + <literal>i686-linux</literal> or + <literal>x86_64-darwin</literal>. Nix can only build derivations + whose <literal>system</literal> attribute equals the value + specified here. In general, it never makes sense to modify this + value from its default, since you can use it to ‘lie’ about the + platform you are building on (e.g., perform a Mac OS build on a + Linux machine; the result would obviously be wrong). It only + makes sense if the Nix binaries can run on multiple platforms, + e.g., ‘universal binaries’ that run on <literal>x86_64-linux</literal> and + <literal>i686-linux</literal>.</para> + + <para>It defaults to the canonical Nix system name detected by + <filename>configure</filename> at build time.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id="conf-system-features"><term><literal>system-features</literal></term> + + <listitem><para>A set of system “features” supported by this + machine, e.g. <literal>kvm</literal>. Derivations can express a + dependency on such features through the derivation attribute + <varname>requiredSystemFeatures</varname>. For example, the + attribute + +<programlisting> +requiredSystemFeatures = [ "kvm" ]; +</programlisting> + + ensures that the derivation can only be built on a machine with + the <literal>kvm</literal> feature.</para> + + <para>This setting by default includes <literal>kvm</literal> if + <filename>/dev/kvm</filename> is accessible, and the + pseudo-features <literal>nixos-test</literal>, + <literal>benchmark</literal> and <literal>big-parallel</literal> + that are used in Nixpkgs to route builds to specific + machines.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-tarball-ttl"><term><literal>tarball-ttl</literal></term> + + <listitem> + <para>Default: <literal>3600</literal> seconds.</para> + + <para>The number of seconds a downloaded tarball is considered + fresh. If the cached tarball is stale, Nix will check whether + it is still up to date using the ETag header. Nix will download + a new version if the ETag header is unsupported, or the + cached ETag doesn't match. + </para> + + <para>Setting the TTL to <literal>0</literal> forces Nix to always + check if the tarball is up to date.</para> + + <para>Nix caches tarballs in + <filename>$XDG_CACHE_HOME/nix/tarballs</filename>.</para> + + <para>Files fetched via <envar>NIX_PATH</envar>, + <function>fetchGit</function>, <function>fetchMercurial</function>, + <function>fetchTarball</function>, and <function>fetchurl</function> + respect this TTL. + </para> + </listitem> + </varlistentry> + + <varlistentry xml:id="conf-timeout"><term><literal>timeout</literal></term> + + <listitem> + + <para>This option defines the maximum number of seconds that a + builder can run. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite loop + but keep writing to their standard output or standard error. It + can be overridden using the <option + linkend="opt-timeout">--timeout</option> command line + switch.</para> + + <para>The value <literal>0</literal> means that there is no + timeout. This is also the default.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-trace-function-calls"><term><literal>trace-function-calls</literal></term> + + <listitem> + + <para>Default: <literal>false</literal>.</para> + + <para>If set to <literal>true</literal>, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit.</para> + + <informalexample><screen> +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 +</screen></informalexample> + + <para>The <literal>undefined position</literal> means the function + call is a builtin.</para> + + <para>Use the <literal>contrib/stack-collapse.py</literal> script + distributed with the Nix source code to convert the trace logs + in to a format suitable for <command>flamegraph.pl</command>.</para> + + </listitem> + + </varlistentry> + + <varlistentry xml:id="conf-trusted-public-keys"><term><literal>trusted-public-keys</literal></term> + + <listitem><para>A whitespace-separated list of public keys. When + paths are copied from another Nix store (such as a binary cache), + they must be signed with one of these keys. For example: + <literal>cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-trusted-substituters"><term><literal>trusted-substituters</literal></term> + + <listitem><para>A list of URLs of substituters, separated by + whitespace. These are not used by default, but can be enabled by + users of the Nix daemon by specifying <literal>--option + substituters <replaceable>urls</replaceable></literal> on the + command line. Unprivileged users are only allowed to pass a + subset of the URLs listed in <literal>substituters</literal> and + <literal>trusted-substituters</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id="conf-trusted-users"><term><literal>trusted-users</literal></term> + + <listitem> + + <para>A list of names of users (separated by whitespace) that + have additional rights when connecting to the Nix daemon, such + as the ability to specify additional binary caches, or to import + unsigned NARs. You can also specify groups by prefixing them + with <literal>@</literal>; for instance, + <literal>@wheel</literal> means all users in the + <literal>wheel</literal> group. The default is + <literal>root</literal>.</para> + + <warning><para>Adding a user to <option>trusted-users</option> + is essentially equivalent to giving that user root access to the + system. For example, the user can set + <option>sandbox-paths</option> and thereby obtain read access to + directories that are otherwise inacessible to + them.</para></warning> + + </listitem> + + </varlistentry> + +</variablelist> +</para> + +<refsection> + <title>Deprecated Settings</title> + +<para> + +<variablelist> + + <varlistentry xml:id="conf-binary-caches"> + <term><literal>binary-caches</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>binary-caches</literal> is now an alias to + <xref linkend="conf-substituters" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-binary-cache-public-keys"> + <term><literal>binary-cache-public-keys</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>binary-cache-public-keys</literal> is now an alias to + <xref linkend="conf-trusted-public-keys" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-compress-log"> + <term><literal>build-compress-log</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-compress-log</literal> is now an alias to + <xref linkend="conf-compress-build-log" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-cores"> + <term><literal>build-cores</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-cores</literal> is now an alias to + <xref linkend="conf-cores" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-extra-chroot-dirs"> + <term><literal>build-extra-chroot-dirs</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-extra-chroot-dirs</literal> is now an alias to + <xref linkend="conf-extra-sandbox-paths" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-extra-sandbox-paths"> + <term><literal>build-extra-sandbox-paths</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-extra-sandbox-paths</literal> is now an alias to + <xref linkend="conf-extra-sandbox-paths" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-fallback"> + <term><literal>build-fallback</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-fallback</literal> is now an alias to + <xref linkend="conf-fallback" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-max-jobs"> + <term><literal>build-max-jobs</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-max-jobs</literal> is now an alias to + <xref linkend="conf-max-jobs" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-max-log-size"> + <term><literal>build-max-log-size</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-max-log-size</literal> is now an alias to + <xref linkend="conf-max-build-log-size" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-max-silent-time"> + <term><literal>build-max-silent-time</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-max-silent-time</literal> is now an alias to + <xref linkend="conf-max-silent-time" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-repeat"> + <term><literal>build-repeat</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-repeat</literal> is now an alias to + <xref linkend="conf-repeat" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-timeout"> + <term><literal>build-timeout</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-timeout</literal> is now an alias to + <xref linkend="conf-timeout" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-use-chroot"> + <term><literal>build-use-chroot</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-use-chroot</literal> is now an alias to + <xref linkend="conf-sandbox" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-use-sandbox"> + <term><literal>build-use-sandbox</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-use-sandbox</literal> is now an alias to + <xref linkend="conf-sandbox" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-build-use-substitutes"> + <term><literal>build-use-substitutes</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>build-use-substitutes</literal> is now an alias to + <xref linkend="conf-substitute" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-gc-keep-derivations"> + <term><literal>gc-keep-derivations</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>gc-keep-derivations</literal> is now an alias to + <xref linkend="conf-keep-derivations" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-gc-keep-outputs"> + <term><literal>gc-keep-outputs</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>gc-keep-outputs</literal> is now an alias to + <xref linkend="conf-keep-outputs" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-env-keep-derivations"> + <term><literal>env-keep-derivations</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>env-keep-derivations</literal> is now an alias to + <xref linkend="conf-keep-env-derivations" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-extra-binary-caches"> + <term><literal>extra-binary-caches</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>extra-binary-caches</literal> is now an alias to + <xref linkend="conf-extra-substituters" />.</para></listitem> + </varlistentry> + + <varlistentry xml:id="conf-trusted-binary-caches"> + <term><literal>trusted-binary-caches</literal></term> + + <listitem><para><emphasis>Deprecated:</emphasis> + <literal>trusted-binary-caches</literal> is now an alias to + <xref linkend="conf-trusted-substituters" />.</para></listitem> + </varlistentry> +</variablelist> +</para> +</refsection> + +</refsection> + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/env-common.xml b/third_party/nix/doc/manual/command-ref/env-common.xml new file mode 100644 index 000000000000..696d68c34516 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/env-common.xml @@ -0,0 +1,202 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-common-env"> + +<title>Common Environment Variables</title> + + +<para>Most Nix commands interpret the following environment variables:</para> + +<variablelist xml:id="env-common"> + +<varlistentry><term><envar>IN_NIX_SHELL</envar></term> + + <listitem><para>Indicator that tells if the current environment was set up by + <command>nix-shell</command>. Since Nix 2.0 the values are + <literal>"pure"</literal> and <literal>"impure"</literal></para></listitem> + +</varlistentry> + +<varlistentry xml:id="env-NIX_PATH"><term><envar>NIX_PATH</envar></term> + + <listitem> + + <para>A colon-separated list of directories used to look up Nix + expressions enclosed in angle brackets (i.e., + <literal><<replaceable>path</replaceable>></literal>). For + instance, the value + + <screen> +/home/eelco/Dev:/etc/nixos</screen> + + will cause Nix to look for paths relative to + <filename>/home/eelco/Dev</filename> and + <filename>/etc/nixos</filename>, in that order. It is also + possible to match paths against a prefix. For example, the value + + <screen> +nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos</screen> + + will cause Nix to search for + <literal><nixpkgs/<replaceable>path</replaceable>></literal> in + <filename>/home/eelco/Dev/nixpkgs-branch/<replaceable>path</replaceable></filename> + and + <filename>/etc/nixos/nixpkgs/<replaceable>path</replaceable></filename>.</para> + + <para>If a path in the Nix search path starts with + <literal>http://</literal> or <literal>https://</literal>, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must consist of a + single top-level directory. For example, setting + <envar>NIX_PATH</envar> to + + <screen> +nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz</screen> + + tells Nix to download the latest revision in the Nixpkgs/NixOS + 15.09 channel.</para> + + <para>A following shorthand can be used to refer to the official channels: + + <screen>nixpkgs=channel:nixos-15.09</screen> + </para> + + <para>The search path can be extended using the <option + linkend="opt-I">-I</option> option, which takes precedence over + <envar>NIX_PATH</envar>.</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_IGNORE_SYMLINK_STORE</envar></term> + + <listitem> + + <para>Normally, the Nix store directory (typically + <filename>/nix/store</filename>) is not allowed to contain any + symlink components. This is to prevent “impure” builds. Builders + sometimes “canonicalise” paths by resolving all symlink components. + Thus, builds on different machines (with + <filename>/nix/store</filename> resolving to different locations) + could yield different results. This is generally not a problem, + except when builds are deployed to machines where + <filename>/nix/store</filename> resolves differently. If you are + sure that you’re not going to do that, you can set + <envar>NIX_IGNORE_SYMLINK_STORE</envar> to <envar>1</envar>.</para> + + <para>Note that if you’re symlinking the Nix store so that you can + put it on another file system than the root file system, on Linux + you’re better off using <literal>bind</literal> mount points, e.g., + + <screen> +$ mkdir /nix +$ mount -o bind /mnt/otherdisk/nix /nix</screen> + + Consult the <citerefentry><refentrytitle>mount</refentrytitle> + <manvolnum>8</manvolnum></citerefentry> manual page for details.</para> + + </listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_STORE_DIR</envar></term> + + <listitem><para>Overrides the location of the Nix store (default + <filename><replaceable>prefix</replaceable>/store</filename>).</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_DATA_DIR</envar></term> + + <listitem><para>Overrides the location of the Nix static data + directory (default + <filename><replaceable>prefix</replaceable>/share</filename>).</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_LOG_DIR</envar></term> + + <listitem><para>Overrides the location of the Nix log directory + (default <filename><replaceable>prefix</replaceable>/var/log/nix</filename>).</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_STATE_DIR</envar></term> + + <listitem><para>Overrides the location of the Nix state directory + (default <filename><replaceable>prefix</replaceable>/var/nix</filename>).</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_CONF_DIR</envar></term> + + <listitem><para>Overrides the location of the Nix configuration + directory (default + <filename><replaceable>prefix</replaceable>/etc/nix</filename>).</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>TMPDIR</envar></term> + + <listitem><para>Use the specified directory to store temporary + files. In particular, this includes temporary build directories; + these can take up substantial amounts of disk space. The default is + <filename>/tmp</filename>.</para></listitem> + +</varlistentry> + + +<varlistentry xml:id="envar-remote"><term><envar>NIX_REMOTE</envar></term> + + <listitem><para>This variable should be set to + <literal>daemon</literal> if you want to use the Nix daemon to + execute Nix operations. This is necessary in <link + linkend="ssec-multi-user">multi-user Nix installations</link>. + If the Nix daemon's Unix socket is at some non-standard path, + this variable should be set to <literal>unix://path/to/socket</literal>. + Otherwise, it should be left unset.</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_SHOW_STATS</envar></term> + + <listitem><para>If set to <literal>1</literal>, Nix will print some + evaluation statistics, such as the number of values + allocated.</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>NIX_COUNT_CALLS</envar></term> + + <listitem><para>If set to <literal>1</literal>, Nix will print how + often functions were called during Nix expression evaluation. This + is useful for profiling your Nix expressions.</para></listitem> + +</varlistentry> + + +<varlistentry><term><envar>GC_INITIAL_HEAP_SIZE</envar></term> + + <listitem><para>If Nix has been configured to use the Boehm garbage + collector, this variable sets the initial size of the heap in bytes. + It defaults to 384 MiB. Setting it to a low value reduces memory + consumption, but will increase runtime due to the overhead of + garbage collection.</para></listitem> + +</varlistentry> + + +</variablelist> + + +</chapter> diff --git a/third_party/nix/doc/manual/command-ref/files.xml b/third_party/nix/doc/manual/command-ref/files.xml new file mode 100644 index 000000000000..7bbc96e89982 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/files.xml @@ -0,0 +1,14 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='ch-files'> + +<title>Files</title> + +<para>This section lists configuration files that you can use when you +work with Nix.</para> + +<xi:include href="conf-file.xml" /> + +</chapter> \ No newline at end of file diff --git a/third_party/nix/doc/manual/command-ref/main-commands.xml b/third_party/nix/doc/manual/command-ref/main-commands.xml new file mode 100644 index 000000000000..0f4169243ca9 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/main-commands.xml @@ -0,0 +1,17 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='ch-main-commands'> + +<title>Main Commands</title> + +<para>This section lists commands and options that you can use when you +work with Nix.</para> + +<xi:include href="nix-env.xml" /> +<xi:include href="nix-build.xml" /> +<xi:include href="nix-shell.xml" /> +<xi:include href="nix-store.xml" /> + +</chapter> \ No newline at end of file diff --git a/third_party/nix/doc/manual/command-ref/nix-build.xml b/third_party/nix/doc/manual/command-ref/nix-build.xml new file mode 100644 index 000000000000..c1b783c87d6d --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-build.xml @@ -0,0 +1,190 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-build"> + +<refmeta> + <refentrytitle>nix-build</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-build</refname> + <refpurpose>build a Nix expression</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-build</command> + <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" /> + <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--attr</option></arg> + <arg choice='plain'><option>-A</option></arg> + </group> + <replaceable>attrPath</replaceable> + </arg> + <arg><option>--no-out-link</option></arg> + <arg><option>--dry-run</option></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--out-link</option></arg> + <arg choice='plain'><option>-o</option></arg> + </group> + <replaceable>outlink</replaceable> + </arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + +<refsection><title>Description</title> + +<para>The <command>nix-build</command> command builds the derivations +described by the Nix expressions in <replaceable>paths</replaceable>. +If the build succeeds, it places a symlink to the result in the +current directory. The symlink is called <filename>result</filename>. +If there are multiple Nix expressions, or the Nix expressions evaluate +to multiple derivations, multiple sequentially numbered symlinks are +created (<filename>result</filename>, <filename>result-2</filename>, +and so on).</para> + +<para>If no <replaceable>paths</replaceable> are specified, then +<command>nix-build</command> will use <filename>default.nix</filename> +in the current directory, if it exists.</para> + +<para>If an element of <replaceable>paths</replaceable> starts with +<literal>http://</literal> or <literal>https://</literal>, it is +interpreted as the URL of a tarball that will be downloaded and +unpacked to a temporary location. The tarball must include a single +top-level directory containing at least a file named +<filename>default.nix</filename>.</para> + +<para><command>nix-build</command> is essentially a wrapper around +<link +linkend="sec-nix-instantiate"><command>nix-instantiate</command></link> +(to translate a high-level Nix expression to a low-level store +derivation) and <link +linkend="rsec-nix-store-realise"><command>nix-store +--realise</command></link> (to build the store derivation).</para> + +<warning><para>The result of the build is automatically registered as +a root of the Nix garbage collector. This root disappears +automatically when the <filename>result</filename> symlink is deleted +or renamed. So don’t rename the symlink.</para></warning> + +</refsection> + + +<refsection><title>Options</title> + +<para>All options not listed here are passed to <command>nix-store +--realise</command>, except for <option>--arg</option> and +<option>--attr</option> / <option>-A</option> which are passed to +<command>nix-instantiate</command>. <phrase condition="manual">See +also <xref linkend="sec-common-options" />.</phrase></para> + +<variablelist> + + <varlistentry><term><option>--no-out-link</option></term> + + <listitem><para>Do not create a symlink to the output path. Note + that as a result the output does not become a root of the garbage + collector, and so might be deleted by <command>nix-store + --gc</command>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--dry-run</option></term> + <listitem><para>Show what store paths would be built or downloaded</para></listitem> + </varlistentry> + + <varlistentry xml:id='opt-out-link'><term><option>--out-link</option> / + <option>-o</option> <replaceable>outlink</replaceable></term> + + <listitem><para>Change the name of the symlink to the output path + created from <filename>result</filename> to + <replaceable>outlink</replaceable>.</para></listitem> + + </varlistentry> + +</variablelist> + +<para>The following common options are supported:</para> + +<variablelist condition="manpage"> + <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" /> +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<screen> +$ nix-build '<nixpkgs>' -A firefox +store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv +/nix/store/d18hyl92g30l...-firefox-1.5.0.7 + +$ ls -l result +lrwxrwxrwx <replaceable>...</replaceable> result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7 + +$ ls ./result/bin/ +firefox firefox-config</screen> + +<para>If a derivation has multiple outputs, +<command>nix-build</command> will build the default (first) output. +You can also build all outputs: +<screen> +$ nix-build '<nixpkgs>' -A openssl.all +</screen> +This will create a symlink for each output named +<filename>result-<replaceable>outputname</replaceable></filename>. +The suffix is omitted if the output name is <literal>out</literal>. +So if <literal>openssl</literal> has outputs <literal>out</literal>, +<literal>bin</literal> and <literal>man</literal>, +<command>nix-build</command> will create symlinks +<literal>result</literal>, <literal>result-bin</literal> and +<literal>result-man</literal>. It’s also possible to build a specific +output: +<screen> +$ nix-build '<nixpkgs>' -A openssl.man +</screen> +This will create a symlink <literal>result-man</literal>.</para> + +<para>Build a Nix expression given on the command line: + +<screen> +$ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"' +$ cat ./result +bar +</screen> + +</para> + +<para>Build the GNU Hello package from the latest revision of the +master branch of Nixpkgs: + +<screen> +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello +</screen> + +</para> + +</refsection> + + +<refsection condition="manpage"><title>Environment variables</title> + +<variablelist> + <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" /> +</variablelist> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-channel.xml b/third_party/nix/doc/manual/command-ref/nix-channel.xml new file mode 100644 index 000000000000..5a2866e6bc4b --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-channel.xml @@ -0,0 +1,178 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-channel"> + +<refmeta> + <refentrytitle>nix-channel</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-channel</refname> + <refpurpose>manage Nix channels</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-channel</command> + <group choice='req'> + <arg choice='plain'><option>--add</option> <replaceable>url</replaceable> <arg choice='opt'><replaceable>name</replaceable></arg></arg> + <arg choice='plain'><option>--remove</option> <replaceable>name</replaceable></arg> + <arg choice='plain'><option>--list</option></arg> + <arg choice='plain'><option>--update</option> <arg rep='repeat'><replaceable>names</replaceable></arg></arg> + <arg choice='plain'><option>--rollback</option> <arg choice='opt'><replaceable>generation</replaceable></arg></arg> + </group> + </cmdsynopsis> +</refsynopsisdiv> + +<refsection><title>Description</title> + +<para>A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. <phrase condition="manual">See also <xref +linkend="sec-channels" />.</phrase></para> + +<para>This command has the following operations: + +<variablelist> + + <varlistentry><term><option>--add</option> <replaceable>url</replaceable> [<replaceable>name</replaceable>]</term> + + <listitem><para>Adds a channel named + <replaceable>name</replaceable> with URL + <replaceable>url</replaceable> to the list of subscribed channels. + If <replaceable>name</replaceable> is omitted, it defaults to the + last component of <replaceable>url</replaceable>, with the + suffixes <literal>-stable</literal> or + <literal>-unstable</literal> removed.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--remove</option> <replaceable>name</replaceable></term> + + <listitem><para>Removes the channel named + <replaceable>name</replaceable> from the list of subscribed + channels.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--list</option></term> + + <listitem><para>Prints the names and URLs of all subscribed + channels on standard output.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--update</option> [<replaceable>names</replaceable>…]</term> + + <listitem><para>Downloads the Nix expressions of all subscribed + channels (or only those included in + <replaceable>names</replaceable> if specified) and makes them the + default for <command>nix-env</command> operations (by symlinking + them from the directory + <filename>~/.nix-defexpr</filename>).</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--rollback</option> [<replaceable>generation</replaceable>]</term> + + <listitem><para>Reverts the previous call to <command>nix-channel + --update</command>. Optionally, you can specify a specific channel + generation number to restore.</para></listitem> + + </varlistentry> + +</variablelist> + +</para> + +<para>Note that <option>--add</option> does not automatically perform +an update.</para> + +<para>The list of subscribed channels is stored in +<filename>~/.nix-channels</filename>.</para> + +</refsection> + +<refsection><title>Examples</title> + +<para>To subscribe to the Nixpkgs channel and install the GNU Hello package:</para> + +<screen> +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --update +$ nix-env -iA nixpkgs.hello</screen> + +<para>You can revert channel updates using <option>--rollback</option>:</para> + +<screen> +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +"14.04.527.0e935f1" + +$ nix-channel --rollback +switching from generation 483 to 482 + +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +"14.04.526.dbadfad" +</screen> + +</refsection> + +<refsection><title>Files</title> + +<variablelist> + + <varlistentry><term><filename>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels</filename></term> + + <listitem><para><command>nix-channel</command> uses a + <command>nix-env</command> profile to keep track of previous + versions of the subscribed channels. Every time you run + <command>nix-channel --update</command>, a new channel generation + (that is, a symlink to the channel Nix expressions in the Nix store) + is created. This enables <command>nix-channel --rollback</command> + to revert to previous versions.</para></listitem> + + </varlistentry> + + <varlistentry><term><filename>~/.nix-defexpr/channels</filename></term> + + <listitem><para>This is a symlink to + <filename>/nix/var/nix/profiles/per-user/<replaceable>username</replaceable>/channels</filename>. It + ensures that <command>nix-env</command> can find your channels. In + a multi-user installation, you may also have + <filename>~/.nix-defexpr/channels_root</filename>, which links to + the channels of the root user.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + +<refsection><title>Channel format</title> + +<para>A channel URL should point to a directory containing the +following files:</para> + +<variablelist> + + <varlistentry><term><filename>nixexprs.tar.xz</filename></term> + + <listitem><para>A tarball containing Nix expressions and files + referenced by them (such as build scripts and patches). At the + top level, the tarball should contain a single directory. That + directory must contain a file <filename>default.nix</filename> + that serves as the channel’s “entry point”.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-collect-garbage.xml b/third_party/nix/doc/manual/command-ref/nix-collect-garbage.xml new file mode 100644 index 000000000000..43e06879691c --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-collect-garbage.xml @@ -0,0 +1,63 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-collect-garbage"> + +<refmeta> + <refentrytitle>nix-collect-garbage</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-collect-garbage</refname> + <refpurpose>delete unreachable store paths</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-collect-garbage</command> + <arg><option>--delete-old</option></arg> + <arg><option>-d</option></arg> + <arg><option>--delete-older-than</option> <replaceable>period</replaceable></arg> + <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg> + <arg><option>--dry-run</option></arg> + </cmdsynopsis> +</refsynopsisdiv> + +<refsection><title>Description</title> + +<para>The command <command>nix-collect-garbage</command> is mostly an +alias of <link linkend="rsec-nix-store-gc"><command>nix-store +--gc</command></link>, that is, it deletes all unreachable paths in +the Nix store to clean up your system. However, it provides two +additional options: <option>-d</option> (<option>--delete-old</option>), +which deletes all old generations of all profiles in +<filename>/nix/var/nix/profiles</filename> by invoking +<literal>nix-env --delete-generations old</literal> on all profiles +(of course, this makes rollbacks to previous configurations +impossible); and +<option>--delete-older-than</option> <replaceable>period</replaceable>, +where period is a value such as <literal>30d</literal>, which deletes +all generations older than the specified number of days in all profiles +in <filename>/nix/var/nix/profiles</filename> (except for the generations +that were active at that point in time). +</para> + +</refsection> + +<refsection><title>Example</title> + +<para>To delete from the Nix store everything that is not used by the +current generations of each profile, do + +<screen> +$ nix-collect-garbage -d</screen> + +</para> + +</refsection> + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-copy-closure.xml b/third_party/nix/doc/manual/command-ref/nix-copy-closure.xml new file mode 100644 index 000000000000..e6dcf180ad69 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-copy-closure.xml @@ -0,0 +1,169 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + xml:id="sec-nix-copy-closure"> + +<refmeta> + <refentrytitle>nix-copy-closure</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-copy-closure</refname> + <refpurpose>copy a closure to or from a remote machine via SSH</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-copy-closure</command> + <group> + <arg choice='plain'><option>--to</option></arg> + <arg choice='plain'><option>--from</option></arg> + </group> + <arg><option>--gzip</option></arg> + <!-- + <arg><option>- -show-progress</option></arg> + --> + <arg><option>--include-outputs</option></arg> + <group> + <arg choice='plain'><option>--use-substitutes</option></arg> + <arg choice='plain'><option>-s</option></arg> + </group> + <arg><option>-v</option></arg> + <arg choice='plain'> + <replaceable>user@</replaceable><replaceable>machine</replaceable> + </arg> + <arg choice='plain'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + + +<refsection><title>Description</title> + +<para><command>nix-copy-closure</command> gives you an easy and +efficient way to exchange software between machines. Given one or +more Nix store <replaceable>paths</replaceable> on the local +machine, <command>nix-copy-closure</command> computes the closure of +those paths (i.e. all their dependencies in the Nix store), and copies +all paths in the closure to the remote machine via the +<command>ssh</command> (Secure Shell) command. With the +<option>--from</option>, the direction is reversed: +the closure of <replaceable>paths</replaceable> on a remote machine is +copied to the Nix store on the local machine.</para> + +<para>This command is efficient because it only sends the store paths +that are missing on the target machine.</para> + +<para>Since <command>nix-copy-closure</command> calls +<command>ssh</command>, you may be asked to type in the appropriate +password or passphrase. In fact, you may be asked +<emphasis>twice</emphasis> because <command>nix-copy-closure</command> +currently connects twice to the remote machine, first to get the set +of paths missing on the target machine, and second to send the dump of +those paths. If this bothers you, use +<command>ssh-agent</command>.</para> + + +<refsection><title>Options</title> + +<variablelist> + + <varlistentry><term><option>--to</option></term> + + <listitem><para>Copy the closure of + <replaceable>paths</replaceable> from the local Nix store to the + Nix store on <replaceable>machine</replaceable>. This is the + default.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--from</option></term> + + <listitem><para>Copy the closure of + <replaceable>paths</replaceable> from the Nix store on + <replaceable>machine</replaceable> to the local Nix + store.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--gzip</option></term> + + <listitem><para>Enable compression of the SSH + connection.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--include-outputs</option></term> + + <listitem><para>Also copy the outputs of store derivations + included in the closure.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--use-substitutes</option> / <option>-s</option></term> + + <listitem><para>Attempt to download missing paths on the target + machine using Nix’s substitute mechanism. Any paths that cannot + be substituted on the target are still copied normally from the + source. This is useful, for instance, if the connection between + the source and target machine is slow, but the connection between + the target machine and <literal>nixos.org</literal> (the default + binary cache server) is fast.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>-v</option></term> + + <listitem><para>Show verbose output.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Environment variables</title> + +<variablelist> + + <varlistentry><term><envar>NIX_SSHOPTS</envar></term> + + <listitem><para>Additional options to be passed to + <command>ssh</command> on the command line.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>Copy Firefox with all its dependencies to a remote machine: + +<screen> +$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox)</screen> + +</para> + +<para>Copy Subversion from a remote machine and then install it into a +user environment: + +<screen> +$ nix-copy-closure --from alice@itchy.labs \ + /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 +$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 +</screen> + +</para> + +</refsection> + + +</refsection> + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-daemon.xml b/third_party/nix/doc/manual/command-ref/nix-daemon.xml new file mode 100644 index 000000000000..9159d15d1c3a --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-daemon.xml @@ -0,0 +1,51 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-daemon"> + +<refmeta> + <refentrytitle>nix-daemon</refentrytitle> + <manvolnum>8</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-daemon</refname> + <refpurpose>Nix multi-user support daemon</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-daemon</command> + </cmdsynopsis> +</refsynopsisdiv> + + +<refsection><title>Description</title> + +<para>The Nix daemon is necessary in multi-user Nix installations. It +performs build actions and other operations on the Nix store on behalf +of unprivileged users.</para> + + +</refsection> + +<refsection><title>Options</title> + + <variablelist> + + <varlistentry><term><option>--pipe</option></term> + + <listitem><para>Causes the nix daemon to forward stdin and stdout to and + from the actual daemon socket. This is used when communicating with a remote + store over SSH</para></listitem> + + </varlistentry> + + </variablelist> + +</refsection> + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-env.xml b/third_party/nix/doc/manual/command-ref/nix-env.xml new file mode 100644 index 000000000000..d257a5e49c70 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-env.xml @@ -0,0 +1,1505 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-env"> + +<refmeta> + <refentrytitle>nix-env</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-env</refname> + <refpurpose>manipulate or query Nix user environments</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-env</command> + <xi:include href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" /> + <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--file</option></arg> + <arg choice='plain'><option>-f</option></arg> + </group> + <replaceable>path</replaceable> + </arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--profile</option></arg> + <arg choice='plain'><option>-p</option></arg> + </group> + <replaceable>path</replaceable> + </arg> + <arg> + <arg choice='plain'><option>--system-filter</option></arg> + <replaceable>system</replaceable> + </arg> + <arg><option>--dry-run</option></arg> + <arg choice='plain'><replaceable>operation</replaceable></arg> + <arg rep='repeat'><replaceable>options</replaceable></arg> + <arg rep='repeat'><replaceable>arguments</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + + +<refsection><title>Description</title> + +<para>The command <command>nix-env</command> is used to manipulate Nix +user environments. User environments are sets of software packages +available to a user at some point in time. In other words, they are a +synthesised view of the programs available in the Nix store. There +may be many user environments: different users can have different +environments, and individual users can switch between different +environments.</para> + +<para><command>nix-env</command> takes exactly one +<emphasis>operation</emphasis> flag which indicates the subcommand to +be performed. These are documented below.</para> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Selectors</title> + +<para>Several commands, such as <command>nix-env -q</command> and +<command>nix-env -i</command>, take a list of arguments that specify +the packages on which to operate. These are extended regular +expressions that must match the entire name of the package. (For +details on regular expressions, see +<citerefentry><refentrytitle>regex</refentrytitle><manvolnum>7</manvolnum></citerefentry>.) +The match is case-sensitive. The regular expression can optionally be +followed by a dash and a version number; if omitted, any version of +the package will match. Here are some examples: + +<variablelist> + + <varlistentry> + <term><literal>firefox</literal></term> + <listitem><para>Matches the package name + <literal>firefox</literal> and any version.</para></listitem> + </varlistentry> + + <varlistentry> + <term><literal>firefox-32.0</literal></term> + <listitem><para>Matches the package name + <literal>firefox</literal> and version + <literal>32.0</literal>.</para></listitem> + </varlistentry> + + <varlistentry> + <term><literal>gtk\\+</literal></term> + <listitem><para>Matches the package name + <literal>gtk+</literal>. The <literal>+</literal> character must + be escaped using a backslash to prevent it from being interpreted + as a quantifier, and the backslash must be escaped in turn with + another backslash to ensure that the shell passes it + on.</para></listitem> + </varlistentry> + + <varlistentry> + <term><literal>.\*</literal></term> + <listitem><para>Matches any package name. This is the default for + most commands.</para></listitem> + </varlistentry> + + <varlistentry> + <term><literal>'.*zip.*'</literal></term> + <listitem><para>Matches any package name containing the string + <literal>zip</literal>. Note the dots: <literal>'*zip*'</literal> + does not work, because in a regular expression, the character + <literal>*</literal> is interpreted as a + quantifier.</para></listitem> + </varlistentry> + + <varlistentry> + <term><literal>'.*(firefox|chromium).*'</literal></term> + <listitem><para>Matches any package name containing the strings + <literal>firefox</literal> or + <literal>chromium</literal>.</para></listitem> + </varlistentry> + +</variablelist> + +</para> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Common options</title> + +<para>This section lists the options that are common to all +operations. These options are allowed for every subcommand, though +they may not always have an effect. <phrase condition="manual">See +also <xref linkend="sec-common-options" />.</phrase></para> + +<variablelist> + + <varlistentry><term><option>--file</option> / <option>-f</option> <replaceable>path</replaceable></term> + + <listitem><para>Specifies the Nix expression (designated below as + the <emphasis>active Nix expression</emphasis>) used by the + <option>--install</option>, <option>--upgrade</option>, and + <option>--query --available</option> operations to obtain + derivations. The default is + <filename>~/.nix-defexpr</filename>.</para> + + <para>If the argument starts with <literal>http://</literal> or + <literal>https://</literal>, it is interpreted as the URL of a + tarball that will be downloaded and unpacked to a temporary + location. The tarball must include a single top-level directory + containing at least a file named <filename>default.nix</filename>.</para> + + </listitem> + + </varlistentry> + + <varlistentry><term><option>--profile</option> / <option>-p</option> <replaceable>path</replaceable></term> + + <listitem><para>Specifies the profile to be used by those + operations that operate on a profile (designated below as the + <emphasis>active profile</emphasis>). A profile is a sequence of + user environments called <emphasis>generations</emphasis>, one of + which is the <emphasis>current + generation</emphasis>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--dry-run</option></term> + + <listitem><para>For the <option>--install</option>, + <option>--upgrade</option>, <option>--uninstall</option>, + <option>--switch-generation</option>, + <option>--delete-generations</option> and + <option>--rollback</option> operations, this flag will cause + <command>nix-env</command> to print what + <emphasis>would</emphasis> be done if this flag had not been + specified, without actually doing it.</para> + + <para><option>--dry-run</option> also prints out which paths will + be <link linkend="gloss-substitute">substituted</link> (i.e., + downloaded) and which paths will be built from source (because no + substitute is available).</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--system-filter</option> <replaceable>system</replaceable></term> + + <listitem><para>By default, operations such as <option>--query + --available</option> show derivations matching any platform. This + option allows you to use derivations for the specified platform + <replaceable>system</replaceable>.</para></listitem> + + </varlistentry> + +</variablelist> + +<variablelist condition="manpage"> + <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" /> +</variablelist> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Files</title> + +<variablelist> + + <varlistentry><term><filename>~/.nix-defexpr</filename></term> + + <listitem><para>The source for the default Nix + expressions used by the <option>--install</option>, + <option>--upgrade</option>, and <option>--query + --available</option> operations to obtain derivations. The + <option>--file</option> option may be used to override this + default.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory + containing a <filename>default.nix</filename> file, that file + is loaded as in the above paragraph.</para> + + <para>If <filename>~/.nix-defexpr</filename> is a directory without + a <filename>default.nix</filename> file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + </para> + + <para>For example, if <filename>~/.nix-defexpr</filename> contains + two files, <filename>foo.nix</filename> and <filename>bar.nix</filename>, + then the default Nix expression will essentially be + +<programlisting> +{ + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; +}</programlisting> + + </para> + + <para>The file <filename>manifest.nix</filename> is always ignored. + Subdirectories without a <filename>default.nix</filename> file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression.</para> + + <para>The command <command>nix-channel</command> places symlinks + to the downloaded Nix expressions from each subscribed channel in + this directory.</para> + </listitem> + + </varlistentry> + + <varlistentry><term><filename>~/.nix-profile</filename></term> + + <listitem><para>A symbolic link to the user's current profile. By + default, this symlink points to + <filename><replaceable>prefix</replaceable>/var/nix/profiles/default</filename>. + The <envar>PATH</envar> environment variable should include + <filename>~/.nix-profile/bin</filename> for the user environment + to be visible to the user.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id="rsec-nix-env-install"><title>Operation <option>--install</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <group choice='req'> + <arg choice='plain'><option>--install</option></arg> + <arg choice='plain'><option>-i</option></arg> + </group> + <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-inst-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" /> + <group choice='opt'> + <arg choice='plain'><option>--preserve-installed</option></arg> + <arg choice='plain'><option>-P</option></arg> + </group> + <group choice='opt'> + <arg choice='plain'><option>--remove-all</option></arg> + <arg choice='plain'><option>-r</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>args</replaceable></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>The install operation creates a new user environment, based on +the current generation of the active profile, to which a set of store +paths described by <replaceable>args</replaceable> is added. The +arguments <replaceable>args</replaceable> map to store paths in a +number of possible ways: + +<itemizedlist> + + <listitem><para>By default, <replaceable>args</replaceable> is a set + of derivation names denoting derivations in the active Nix + expression. These are realised, and the resulting output paths are + installed. Currently installed derivations with a name equal to the + name of a derivation being added are removed unless the option + <option>--preserve-installed</option> is + specified.</para> + + <para>If there are multiple derivations matching a name in + <replaceable>args</replaceable> that have the same name (e.g., + <literal>gcc-3.3.6</literal> and <literal>gcc-4.1.1</literal>), then + the derivation with the highest <emphasis>priority</emphasis> is + used. A derivation can define a priority by declaring the + <varname>meta.priority</varname> attribute. This attribute should + be a number, with a higher value denoting a lower priority. The + default priority is <literal>0</literal>.</para> + + <para>If there are multiple matching derivations with the same + priority, then the derivation with the highest version will be + installed.</para> + + <para>You can force the installation of multiple derivations with + the same name by being specific about the versions. For instance, + <literal>nix-env -i gcc-3.3.6 gcc-4.1.1</literal> will install both + version of GCC (and will probably cause a user environment + conflict!).</para></listitem> + + <listitem><para>If <link + linkend='opt-attr'><option>--attr</option></link> + (<option>-A</option>) is specified, the arguments are + <emphasis>attribute paths</emphasis> that select attributes from the + top-level Nix expression. This is faster than using derivation + names and unambiguous. To find out the attribute paths of available + packages, use <literal>nix-env -qaP</literal>.</para></listitem> + + <listitem><para>If <option>--from-profile</option> + <replaceable>path</replaceable> is given, + <replaceable>args</replaceable> is a set of names denoting installed + store paths in the profile <replaceable>path</replaceable>. This is + an easy way to copy user environment elements from one profile to + another.</para></listitem> + + <listitem><para>If <option>--from-expression</option> is given, + <replaceable>args</replaceable> are Nix <link + linkend="ss-functions">functions</link> that are called with the + active Nix expression as their single argument. The derivations + returned by those function calls are installed. This allows + derivations to be specified in an unambiguous way, which is necessary + if there are multiple derivations with the same + name.</para></listitem> + + <listitem><para>If <replaceable>args</replaceable> are store + derivations, then these are <link + linkend="rsec-nix-store-realise">realised</link>, and the resulting + output paths are installed.</para></listitem> + + <listitem><para>If <replaceable>args</replaceable> are store paths + that are not store derivations, then these are <link + linkend="rsec-nix-store-realise">realised</link> and + installed.</para></listitem> + + <listitem><para>By default all outputs are installed for each derivation. + That can be reduced by setting <literal>meta.outputsToInstall</literal>. + </para></listitem> <!-- TODO: link nixpkgs docs on the ability to override those. --> + +</itemizedlist> + +</para> + +</refsection> + + +<refsection><title>Flags</title> + +<variablelist> + + <varlistentry><term><option>--prebuilt-only</option> / <option>-b</option></term> + + <listitem><para>Use only derivations for which a substitute is + registered, i.e., there is a pre-built binary available that can + be downloaded in lieu of building the derivation. Thus, no + packages will be built from source.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--preserve-installed</option></term> + <term><option>-P</option></term> + + <listitem><para>Do not remove derivations with a name matching one + of the derivations being installed. Usually, trying to have two + versions of the same package installed in the same generation of a + profile will lead to an error in building the generation, due to + file name clashes between the two versions. However, this is not + the case for all packages.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--remove-all</option></term> + <term><option>-r</option></term> + + <listitem><para>Remove all previously installed packages first. + This is equivalent to running <literal>nix-env -e '.*'</literal> + first, except that everything happens in a single + transaction.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection xml:id='refsec-nix-env-install-examples'><title>Examples</title> + +<para>To install a specific version of <command>gcc</command> from the +active Nix expression: + +<screen> +$ nix-env --install gcc-3.3.2 +installing `gcc-3.3.2' +uninstalling `gcc-3.1'</screen> + +Note the previously installed version is removed, since +<option>--preserve-installed</option> was not specified.</para> + +<para>To install an arbitrary version: + +<screen> +$ nix-env --install gcc +installing `gcc-3.3.2'</screen> + +</para> + +<para>To install using a specific attribute: + +<screen> +$ nix-env -i -A gcc40mips +$ nix-env -i -A xorg.xorgserver</screen> + +</para> + +<para>To install all derivations in the Nix expression <filename>foo.nix</filename>: + +<screen> +$ nix-env -f ~/foo.nix -i '.*'</screen> + +</para> + +<para>To copy the store path with symbolic name <literal>gcc</literal> +from another profile: + +<screen> +$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc</screen> + +</para> + +<para>To install a specific store derivation (typically created by +<command>nix-instantiate</command>): + +<screen> +$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv</screen> + +</para> + +<para>To install a specific output path: + +<screen> +$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3</screen> + +</para> + +<para>To install from a Nix expression specified on the command-line: + +<screen> +$ nix-env -f ./foo.nix -i -E \ + 'f: (f {system = "i686-linux";}).subversionWithJava'</screen> + +I.e., this evaluates to <literal>(f: (f {system = +"i686-linux";}).subversionWithJava) (import ./foo.nix)</literal>, thus +selecting the <literal>subversionWithJava</literal> attribute from the +set returned by calling the function defined in +<filename>./foo.nix</filename>.</para> + +<para>A dry-run tells you which paths will be downloaded or built from +source: + +<screen> +$ nix-env -f '<nixpkgs>' -iA hello --dry-run +(dry run; not doing anything) +installing ‘hello-2.10’ +these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): + /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 + <replaceable>...</replaceable></screen> + +</para> + +<para>To install Firefox from the latest revision in the Nixpkgs/NixOS +14.12 channel: + +<screen> +$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox +</screen> + +(The GitHub repository <literal>nixpkgs-channels</literal> is updated +automatically from the main <literal>nixpkgs</literal> repository +after certain tests have succeeded and binaries have been built and +uploaded to the binary cache at <uri>cache.nixos.org</uri>.)</para> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id="rsec-nix-env-upgrade"><title>Operation <option>--upgrade</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <group choice='req'> + <arg choice='plain'><option>--upgrade</option></arg> + <arg choice='plain'><option>-u</option></arg> + </group> + <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-inst-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" /> + <group choice='opt'> + <arg choice='plain'><option>--lt</option></arg> + <arg choice='plain'><option>--leq</option></arg> + <arg choice='plain'><option>--eq</option></arg> + <arg choice='plain'><option>--always</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>args</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The upgrade operation creates a new user environment, based on +the current generation of the active profile, in which all store paths +are replaced for which there are newer versions in the set of paths +described by <replaceable>args</replaceable>. Paths for which there +are no newer versions are left untouched; this is not an error. It is +also not an error if an element of <replaceable>args</replaceable> +matches no installed derivations.</para> + +<para>For a description of how <replaceable>args</replaceable> is +mapped to a set of store paths, see <link +linkend="rsec-nix-env-install"><option>--install</option></link>. If +<replaceable>args</replaceable> describes multiple store paths with +the same symbolic name, only the one with the highest version is +installed.</para> + +</refsection> + +<refsection><title>Flags</title> + +<variablelist> + + <varlistentry><term><option>--lt</option></term> + + <listitem><para>Only upgrade a derivation to newer versions. This + is the default.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--leq</option></term> + + <listitem><para>In addition to upgrading to newer versions, also + “upgrade” to derivations that have the same version. Version are + not a unique identification of a derivation, so there may be many + derivations that have the same version. This flag may be useful + to force “synchronisation” between the installed and available + derivations.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--eq</option></term> + + <listitem><para><emphasis>Only</emphasis> “upgrade” to derivations + that have the same version. This may not seem very useful, but it + actually is, e.g., when there is a new release of Nixpkgs and you + want to replace installed applications with the same versions + built against newer dependencies (to reduce the number of + dependencies floating around on your system).</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--always</option></term> + + <listitem><para>In addition to upgrading to newer versions, also + “upgrade” to derivations that have the same or a lower version. + I.e., derivations may actually be downgraded depending on what is + available in the active Nix expression.</para></listitem> + + </varlistentry> + +</variablelist> + +<para>For the other flags, see <option +linkend="rsec-nix-env-install">--install</option>.</para> + +</refsection> + +<refsection><title>Examples</title> + +<screen> +$ nix-env --upgrade gcc +upgrading `gcc-3.3.1' to `gcc-3.4' + +$ nix-env -u gcc-3.3.2 --always <lineannotation>(switch to a specific version)</lineannotation> +upgrading `gcc-3.4' to `gcc-3.3.2' + +$ nix-env --upgrade pan +<lineannotation>(no upgrades available, so nothing happens)</lineannotation> + +$ nix-env -u <lineannotation>(try to upgrade everything)</lineannotation> +upgrading `hello-2.1.2' to `hello-2.1.3' +upgrading `mozilla-1.2' to `mozilla-1.4'</screen> + +</refsection> + +<refsection xml:id="ssec-version-comparisons"><title>Versions</title> + +<para>The upgrade operation determines whether a derivation +<varname>y</varname> is an upgrade of a derivation +<varname>x</varname> by looking at their respective +<literal>name</literal> attributes. The names (e.g., +<literal>gcc-3.3.1</literal> are split into two parts: the package +name (<literal>gcc</literal>), and the version +(<literal>3.3.1</literal>). The version part starts after the first +dash not followed by a letter. <varname>x</varname> is considered an +upgrade of <varname>y</varname> if their package names match, and the +version of <varname>y</varname> is higher that that of +<varname>x</varname>.</para> + +<para>The versions are compared by splitting them into contiguous +components of numbers and letters. E.g., <literal>3.3.1pre5</literal> +is split into <literal>[3, 3, 1, "pre", 5]</literal>. These lists are +then compared lexicographically (from left to right). Corresponding +components <varname>a</varname> and <varname>b</varname> are compared +as follows. If they are both numbers, integer comparison is used. If +<varname>a</varname> is an empty string and <varname>b</varname> is a +number, <varname>a</varname> is considered less than +<varname>b</varname>. The special string component +<literal>pre</literal> (for <emphasis>pre-release</emphasis>) is +considered to be less than other components. String components are +considered less than number components. Otherwise, they are compared +lexicographically (i.e., using case-sensitive string comparison).</para> + +<para>This is illustrated by the following examples: + +<screen> +1.0 < 2.3 +2.1 < 2.3 +2.3 = 2.3 +2.5 > 2.3 +3.1 > 2.3 +2.3.1 > 2.3 +2.3.1 > 2.3a +2.3pre1 < 2.3 +2.3pre3 < 2.3pre12 +2.3a < 2.3c +2.3pre1 < 2.3c +2.3pre1 < 2.3q</screen> + +</para> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--uninstall</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <group choice='req'> + <arg choice='plain'><option>--uninstall</option></arg> + <arg choice='plain'><option>-e</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>drvnames</replaceable></arg> +</cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The uninstall operation creates a new user environment, based on +the current generation of the active profile, from which the store +paths designated by the symbolic names +<replaceable>names</replaceable> are removed.</para> + +</refsection> + +<refsection><title>Examples</title> + +<screen> +$ nix-env --uninstall gcc +$ nix-env -e '.*' <lineannotation>(remove everything)</lineannotation></screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id="rsec-nix-env-set"><title>Operation <option>--set</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <arg choice='plain'><option>--set</option></arg> + <arg choice='plain'><replaceable>drvname</replaceable></arg> +</cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The <option>--set</option> operation modifies the current generation of a +profile so that it contains exactly the specified derivation, and nothing else. +</para> + +</refsection> + +<refsection><title>Examples</title> + +<para> +The following updates a profile such that its current generation will contain +just Firefox: + +<screen> +$ nix-env -p /nix/var/nix/profiles/browser --set firefox</screen> + +</para> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id="rsec-nix-env-set-flag"><title>Operation <option>--set-flag</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <arg choice='plain'><option>--set-flag</option></arg> + <arg choice='plain'><replaceable>name</replaceable></arg> + <arg choice='plain'><replaceable>value</replaceable></arg> + <arg choice='plain' rep='repeat'><replaceable>drvnames</replaceable></arg> +</cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The <option>--set-flag</option> operation allows meta attributes +of installed packages to be modified. There are several attributes +that can be usefully modified, because they affect the behaviour of +<command>nix-env</command> or the user environment build +script: + +<itemizedlist> + + <listitem><para><varname>priority</varname> can be changed to + resolve filename clashes. The user environment build script uses + the <varname>meta.priority</varname> attribute of derivations to + resolve filename collisions between packages. Lower priority values + denote a higher priority. For instance, the GCC wrapper package and + the Binutils package in Nixpkgs both have a file + <filename>bin/ld</filename>, so previously if you tried to install + both you would get a collision. Now, on the other hand, the GCC + wrapper declares a higher priority than Binutils, so the former’s + <filename>bin/ld</filename> is symlinked in the user + environment.</para></listitem> + + <listitem><para><varname>keep</varname> can be set to + <literal>true</literal> to prevent the package from being upgraded + or replaced. This is useful if you want to hang on to an older + version of a package.</para></listitem> + + <listitem><para><varname>active</varname> can be set to + <literal>false</literal> to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to <literal>true</literal> to re-enable the + package.</para></listitem> + +</itemizedlist> + +</para> + +</refsection> + +<refsection><title>Examples</title> + +<para>To prevent the currently installed Firefox from being upgraded: + +<screen> +$ nix-env --set-flag keep true firefox</screen> + +After this, <command>nix-env -u</command> will ignore Firefox.</para> + +<para>To disable the currently installed Firefox, then install a new +Firefox while the old remains part of the profile: + +<screen> +$ nix-env -q +firefox-2.0.0.9 <lineannotation>(the current one)</lineannotation> + +$ nix-env --preserve-installed -i firefox-2.0.0.11 +installing `firefox-2.0.0.11' +building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' +collision between `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.11/bin/firefox' + and `/nix/store/<replaceable>...</replaceable>-firefox-2.0.0.9/bin/firefox'. +<lineannotation>(i.e., can’t have two active at the same time)</lineannotation> + +$ nix-env --set-flag active false firefox +setting flag on `firefox-2.0.0.9' + +$ nix-env --preserve-installed -i firefox-2.0.0.11 +installing `firefox-2.0.0.11' + +$ nix-env -q +firefox-2.0.0.11 <lineannotation>(the enabled one)</lineannotation> +firefox-2.0.0.9 <lineannotation>(the disabled one)</lineannotation></screen> + +</para> + +<para>To make files from <literal>binutils</literal> take precedence +over files from <literal>gcc</literal>: + +<screen> +$ nix-env --set-flag priority 5 binutils +$ nix-env --set-flag priority 10 gcc</screen> + +</para> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--query</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <group choice='req'> + <arg choice='plain'><option>--query</option></arg> + <arg choice='plain'><option>-q</option></arg> + </group> + <group choice='opt'> + <arg choice='plain'><option>--installed</option></arg> + <arg choice='plain'><option>--available</option></arg> + <arg choice='plain'><option>-a</option></arg> + </group> + + <sbr /> + + <arg> + <group choice='req'> + <arg choice='plain'><option>--status</option></arg> + <arg choice='plain'><option>-s</option></arg> + </group> + </arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--attr-path</option></arg> + <arg choice='plain'><option>-P</option></arg> + </group> + </arg> + <arg><option>--no-name</option></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--compare-versions</option></arg> + <arg choice='plain'><option>-c</option></arg> + </group> + </arg> + <arg><option>--system</option></arg> + <arg><option>--drv-path</option></arg> + <arg><option>--out-path</option></arg> + <arg><option>--description</option></arg> + <arg><option>--meta</option></arg> + + <sbr /> + + <arg><option>--xml</option></arg> + <arg><option>--json</option></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--prebuilt-only</option></arg> + <arg choice='plain'><option>-b</option></arg> + </group> + </arg> + + <arg> + <group choice='req'> + <arg choice='plain'><option>--attr</option></arg> + <arg choice='plain'><option>-A</option></arg> + </group> + <replaceable>attribute-path</replaceable> + </arg> + + <sbr /> + + <arg choice='plain' rep='repeat'><replaceable>names</replaceable></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>The query operation displays information about either the store +paths that are installed in the current generation of the active +profile (<option>--installed</option>), or the derivations that are +available for installation in the active Nix expression +(<option>--available</option>). It only prints information about +derivations whose symbolic name matches one of +<replaceable>names</replaceable>.</para> + +<para>The derivations are sorted by their <literal>name</literal> +attributes.</para> + +</refsection> + + +<refsection><title>Source selection</title> + +<para>The following flags specify the set of things on which the query +operates.</para> + +<variablelist> + + <varlistentry><term><option>--installed</option></term> + + <listitem><para>The query operates on the store paths that are + installed in the current generation of the active profile. This + is the default.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--available</option></term> + <term><option>-a</option></term> + + <listitem><para>The query operates on the derivations that are + available in the active Nix expression.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Queries</title> + +<para>The following flags specify what information to display about +the selected derivations. Multiple flags may be specified, in which +case the information is shown in the order given here. Note that the +name of the derivation is shown unless <option>--no-name</option> is +specified.</para> + +<!-- TODO: fix the terminology here; i.e., derivations, store paths, +user environment elements, etc. --> + +<variablelist> + + <varlistentry><term><option>--xml</option></term> + + <listitem><para>Print the result in an XML representation suitable + for automatic processing by other tools. The root element is + called <literal>items</literal>, which contains a + <literal>item</literal> element for each available or installed + derivation. The fields discussed below are all stored in + attributes of the <literal>item</literal> + elements.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--json</option></term> + + <listitem><para>Print the result in a JSON representation suitable + for automatic processing by other tools.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--prebuilt-only</option> / <option>-b</option></term> + + <listitem><para>Show only derivations for which a substitute is + registered, i.e., there is a pre-built binary available that can + be downloaded in lieu of building the derivation. Thus, this + shows all packages that probably can be installed + quickly.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--status</option></term> + <term><option>-s</option></term> + + <listitem><para>Print the <emphasis>status</emphasis> of the + derivation. The status consists of three characters. The first + is <literal>I</literal> or <literal>-</literal>, indicating + whether the derivation is currently installed in the current + generation of the active profile. This is by definition the case + for <option>--installed</option>, but not for + <option>--available</option>. The second is <literal>P</literal> + or <literal>-</literal>, indicating whether the derivation is + present on the system. This indicates whether installation of an + available derivation will require the derivation to be built. The + third is <literal>S</literal> or <literal>-</literal>, indicating + whether a substitute is available for the + derivation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--attr-path</option></term> + <term><option>-P</option></term> + + <listitem><para>Print the <emphasis>attribute path</emphasis> of + the derivation, which can be used to unambiguously select it using + the <link linkend="opt-attr"><option>--attr</option> option</link> + available in commands that install derivations like + <literal>nix-env --install</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--no-name</option></term> + + <listitem><para>Suppress printing of the <literal>name</literal> + attribute of each derivation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--compare-versions</option> / + <option>-c</option></term> + + <listitem><para>Compare installed versions to available versions, + or vice versa (if <option>--available</option> is given). This is + useful for quickly seeing whether upgrades for installed + packages are available in a Nix expression. A column is added + with the following meaning: + + <variablelist> + + <varlistentry><term><literal><</literal> <replaceable>version</replaceable></term> + + <listitem><para>A newer version of the package is available + or installed.</para></listitem> + + </varlistentry> + + <varlistentry><term><literal>=</literal> <replaceable>version</replaceable></term> + + <listitem><para>At most the same version of the package is + available or installed.</para></listitem> + + </varlistentry> + + <varlistentry><term><literal>></literal> <replaceable>version</replaceable></term> + + <listitem><para>Only older versions of the package are + available or installed.</para></listitem> + + </varlistentry> + + <varlistentry><term><literal>- ?</literal></term> + + <listitem><para>No version of the package is available or + installed.</para></listitem> + + </varlistentry> + + </variablelist> + + </para></listitem> + + </varlistentry> + + <varlistentry><term><option>--system</option></term> + + <listitem><para>Print the <literal>system</literal> attribute of + the derivation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--drv-path</option></term> + + <listitem><para>Print the path of the store + derivation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--out-path</option></term> + + <listitem><para>Print the output path of the + derivation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--description</option></term> + + <listitem><para>Print a short (one-line) description of the + derivation, if available. The description is taken from the + <literal>meta.description</literal> attribute of the + derivation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--meta</option></term> + + <listitem><para>Print all of the meta-attributes of the + derivation. This option is only available with + <option>--xml</option> or <option>--json</option>.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>To show installed packages: + +<screen> +$ nix-env -q +bison-1.875c +docbook-xml-4.2 +firefox-1.0.4 +MPlayer-1.0pre7 +ORBit2-2.8.3 +<replaceable>…</replaceable> +</screen> + +</para> + +<para>To show available packages: + +<screen> +$ nix-env -qa +firefox-1.0.7 +GConf-2.4.0.1 +MPlayer-1.0pre7 +ORBit2-2.8.3 +<replaceable>…</replaceable> +</screen> + +</para> + +<para>To show the status of available packages: + +<screen> +$ nix-env -qas +-P- firefox-1.0.7 <lineannotation>(not installed but present)</lineannotation> +--S GConf-2.4.0.1 <lineannotation>(not present, but there is a substitute for fast installation)</lineannotation> +--S MPlayer-1.0pre3 <lineannotation>(i.e., this is not the installed MPlayer, even though the version is the same!)</lineannotation> +IP- ORBit2-2.8.3 <lineannotation>(installed and by definition present)</lineannotation> +<replaceable>…</replaceable> +</screen> + +</para> + +<para>To show available packages in the Nix expression <filename>foo.nix</filename>: + +<screen> +$ nix-env -f ./foo.nix -qa +foo-1.2.3 +</screen> + +</para> + +<para>To compare installed versions to what’s available: + +<screen> +$ nix-env -qc +<replaceable>...</replaceable> +acrobat-reader-7.0 - ? <lineannotation>(package is not available at all)</lineannotation> +autoconf-2.59 = 2.59 <lineannotation>(same version)</lineannotation> +firefox-1.0.4 < 1.0.7 <lineannotation>(a more recent version is available)</lineannotation> +<replaceable>...</replaceable> +</screen> + +</para> + +<para>To show all packages with “<literal>zip</literal>” in the name: + +<screen> +$ nix-env -qa '.*zip.*' +bzip2-1.0.6 +gzip-1.6 +zip-3.0 +<replaceable>…</replaceable> +</screen> + +</para> + +<para>To show all packages with “<literal>firefox</literal>” or +“<literal>chromium</literal>” in the name: + +<screen> +$ nix-env -qa '.*(firefox|chromium).*' +chromium-37.0.2062.94 +chromium-beta-38.0.2125.24 +firefox-32.0.3 +firefox-with-plugins-13.0.1 +<replaceable>…</replaceable> +</screen> + +</para> + +<para>To show all packages in the latest revision of the Nixpkgs +repository: + +<screen> +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa +</screen> + +</para> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--switch-profile</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <group choice='req'> + <arg choice='plain'><option>--switch-profile</option></arg> + <arg choice='plain'><option>-S</option></arg> + </group> + <arg choice='req'><replaceable>path</replaceable></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>This operation makes <replaceable>path</replaceable> the current +profile for the user. That is, the symlink +<filename>~/.nix-profile</filename> is made to point to +<replaceable>path</replaceable>.</para> + +</refsection> + +<refsection><title>Examples</title> + +<screen> +$ nix-env -S ~/my-profile</screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--list-generations</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <arg choice='plain'><option>--list-generations</option></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>This operation print a list of all the currently existing +generations for the active profile. These may be switched to using +the <option>--switch-generation</option> operation. It also prints +the creation date of the generation, and indicates the current +generation.</para> + +</refsection> + + +<refsection><title>Examples</title> + +<screen> +$ nix-env --list-generations + 95 2004-02-06 11:48:24 + 96 2004-02-06 11:49:01 + 97 2004-02-06 16:22:45 + 98 2004-02-06 16:24:33 (current)</screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--delete-generations</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <arg choice='plain'><option>--delete-generations</option></arg> + <arg choice='plain' rep='repeat'><replaceable>generations</replaceable></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>This operation deletes the specified generations of the current +profile. The generations can be a list of generation numbers, the +special value <literal>old</literal> to delete all non-current +generations, a value such as <literal>30d</literal> to delete all +generations older than the specified number of days (except for the +generation that was active at that point in time), or a value such as +<literal>+5</literal> to keep the last <literal>5</literal> generations +ignoring any newer than current, e.g., if <literal>30</literal> is the current +generation <literal>+5</literal> will delete generation <literal>25</literal> +and all older generations. +Periodically deleting old generations is important to make garbage collection +effective.</para> + +</refsection> + +<refsection><title>Examples</title> + +<screen> +$ nix-env --delete-generations 3 4 8 + +$ nix-env --delete-generations +5 + +$ nix-env --delete-generations 30d + +$ nix-env -p other_profile --delete-generations old</screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--switch-generation</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <group choice='req'> + <arg choice='plain'><option>--switch-generation</option></arg> + <arg choice='plain'><option>-G</option></arg> + </group> + <arg choice='req'><replaceable>generation</replaceable></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>This operation makes generation number +<replaceable>generation</replaceable> the current generation of the +active profile. That is, if the +<filename><replaceable>profile</replaceable></filename> is the path to +the active profile, then the symlink +<filename><replaceable>profile</replaceable></filename> is made to +point to +<filename><replaceable>profile</replaceable>-<replaceable>generation</replaceable>-link</filename>, +which is in turn a symlink to the actual user environment in the Nix +store.</para> + +<para>Switching will fail if the specified generation does not exist.</para> + +</refsection> + + +<refsection><title>Examples</title> + +<screen> +$ nix-env -G 42 +switching from generation 50 to 42</screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--rollback</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-env</command> + <arg choice='plain'><option>--rollback</option></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>This operation switches to the “previous” generation of the +active profile, that is, the highest numbered generation lower than +the current generation, if it exists. It is just a convenience +wrapper around <option>--list-generations</option> and +<option>--switch-generation</option>.</para> + +</refsection> + + +<refsection><title>Examples</title> + +<screen> +$ nix-env --rollback +switching from generation 92 to 91 + +$ nix-env --rollback +error: no generation older than the current (91) exists</screen> + +</refsection> + +</refsection> + + +<refsection condition="manpage"><title>Environment variables</title> + +<variablelist> + + <varlistentry><term><envar>NIX_PROFILE</envar></term> + + <listitem><para>Location of the Nix profile. Defaults to the + target of the symlink <filename>~/.nix-profile</filename>, if it + exists, or <filename>/nix/var/nix/profiles/default</filename> + otherwise.</para></listitem> + + </varlistentry> + + <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" /> +</variablelist> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-hash.xml b/third_party/nix/doc/manual/command-ref/nix-hash.xml new file mode 100644 index 000000000000..80263e18e339 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-hash.xml @@ -0,0 +1,176 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-hash"> + +<refmeta> + <refentrytitle>nix-hash</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-hash</refname> + <refpurpose>compute the cryptographic hash of a path</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-hash</command> + <arg><option>--flat</option></arg> + <arg><option>--base32</option></arg> + <arg><option>--truncate</option></arg> + <arg><option>--type</option> <replaceable>hashAlgo</replaceable></arg> + <arg choice='plain' rep='repeat'><replaceable>path</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <command>nix-hash</command> + <arg choice='plain'><option>--to-base16</option></arg> + <arg choice='plain' rep='repeat'><replaceable>hash</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <command>nix-hash</command> + <arg choice='plain'><option>--to-base32</option></arg> + <arg choice='plain' rep='repeat'><replaceable>hash</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + + +<refsection><title>Description</title> + +<para>The command <command>nix-hash</command> computes the +cryptographic hash of the contents of each +<replaceable>path</replaceable> and prints it on standard output. By +default, it computes an MD5 hash, but other hash algorithms are +available as well. The hash is printed in hexadecimal. To generate +the same hash as <command>nix-prefetch-url</command> you have to +specify multiple arguments, see below for an example.</para> + +<para>The hash is computed over a <emphasis>serialisation</emphasis> +of each path: a dump of the file system tree rooted at the path. This +allows directories and symlinks to be hashed as well as regular files. +The dump is in the <emphasis>NAR format</emphasis> produced by <link +linkend="refsec-nix-store-dump"><command>nix-store</command> +<option>--dump</option></link>. Thus, <literal>nix-hash +<replaceable>path</replaceable></literal> yields the same +cryptographic hash as <literal>nix-store --dump +<replaceable>path</replaceable> | md5sum</literal>.</para> + +</refsection> + + +<refsection><title>Options</title> + +<variablelist> + + <varlistentry><term><option>--flat</option></term> + + <listitem><para>Print the cryptographic hash of the contents of + each regular file <replaceable>path</replaceable>. That is, do + not compute the hash over the dump of + <replaceable>path</replaceable>. The result is identical to that + produced by the GNU commands <command>md5sum</command> and + <command>sha1sum</command>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--base32</option></term> + + <listitem><para>Print the hash in a base-32 representation rather + than hexadecimal. This base-32 representation is more compact and + can be used in Nix expressions (such as in calls to + <function>fetchurl</function>).</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--truncate</option></term> + + <listitem><para>Truncate hashes longer than 160 bits (such as + SHA-256) to 160 bits.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--type</option> <replaceable>hashAlgo</replaceable></term> + + <listitem><para>Use the specified cryptographic hash algorithm, + which can be one of <literal>md5</literal>, + <literal>sha1</literal>, and + <literal>sha256</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--to-base16</option></term> + + <listitem><para>Don’t hash anything, but convert the base-32 hash + representation <replaceable>hash</replaceable> to + hexadecimal.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--to-base32</option></term> + + <listitem><para>Don’t hash anything, but convert the hexadecimal + hash representation <replaceable>hash</replaceable> to + base-32.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>Computing the same hash as <command>nix-prefetch-url</command>: +<screen> +$ nix-prefetch-url file://<(echo test) +1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj +$ nix-hash --type sha256 --flat --base32 <(echo test) +1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj +</screen> +</para> + +<para>Computing hashes: + +<screen> +$ mkdir test +$ echo "hello" > test/world + +$ nix-hash test/ <lineannotation>(MD5 hash; default)</lineannotation> +8179d3caeff1869b5ba1744e5a245c04 + +$ nix-store --dump test/ | md5sum <lineannotation>(for comparison)</lineannotation> +8179d3caeff1869b5ba1744e5a245c04 - + +$ nix-hash --type sha1 test/ +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + +$ nix-hash --type sha1 --base32 test/ +nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 + +$ nix-hash --type sha256 --flat test/ +error: reading file `test/': Is a directory + +$ nix-hash --type sha256 --flat test/world +5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03</screen> + +</para> + +<para>Converting between hexadecimal and base-32: + +<screen> +$ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 +nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 + +$ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6</screen> + +</para> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-instantiate.xml b/third_party/nix/doc/manual/command-ref/nix-instantiate.xml new file mode 100644 index 000000000000..3fd2ef2a9507 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-instantiate.xml @@ -0,0 +1,278 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-instantiate"> + +<refmeta> + <refentrytitle>nix-instantiate</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-instantiate</refname> + <refpurpose>instantiate store derivations from Nix expressions</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-instantiate</command> + <group> + <arg choice='plain'><option>--parse</option></arg> + <arg choice='plain'> + <option>--eval</option> + <arg><option>--strict</option></arg> + <arg><option>--json</option></arg> + <arg><option>--xml</option></arg> + </arg> + </group> + <arg><option>--read-write-mode</option></arg> + <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--attr</option></arg> + <arg choice='plain'><option>-A</option></arg> + </group> + <replaceable>attrPath</replaceable> + </arg> + <arg><option>--add-root</option> <replaceable>path</replaceable></arg> + <arg><option>--indirect</option></arg> + <arg><option>--<arg>no</arg>trace-file-access</option></arg> + <group> + <arg choice='plain'><option>--expr</option></arg> + <arg choice='plain'><option>-E</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg> + </cmdsynopsis> + <cmdsynopsis> + <command>nix-instantiate</command> + <arg choice='plain'><option>--find-file</option></arg> + <arg choice='plain' rep='repeat'><replaceable>files</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + + +<refsection><title>Description</title> + +<para>The command <command>nix-instantiate</command> generates <link +linkend="gloss-derivation">store derivations</link> from (high-level) +Nix expressions. It evaluates the Nix expressions in each of +<replaceable>files</replaceable> (which defaults to +<replaceable>./default.nix</replaceable>). Each top-level expression +should evaluate to a derivation, a list of derivations, or a set of +derivations. The paths of the resulting store derivations are printed +on standard output.</para> + +<para>If <replaceable>files</replaceable> is the character +<literal>-</literal>, then a Nix expression will be read from standard +input.</para> + +<para condition="manual">See also <xref linkend="sec-common-options" +/> for a list of common options.</para> + +</refsection> + + +<refsection><title>Options</title> + +<variablelist> + + <varlistentry> + <term><option>--add-root</option> <replaceable>path</replaceable></term> + <term><option>--indirect</option></term> + + <listitem><para>See the <link linkend="opt-add-root">corresponding + options</link> in <command>nix-store</command>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--parse</option></term> + + <listitem><para>Just parse the input files, and print their + abstract syntax trees on standard output in ATerm + format.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--eval</option></term> + + <listitem><para>Just parse and evaluate the input files, and print + the resulting values on standard output. No instantiation of + store derivations takes place.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--find-file</option></term> + + <listitem><para>Look up the given files in Nix’s search path (as + specified by the <envar linkend="env-NIX_PATH">NIX_PATH</envar> + environment variable). If found, print the corresponding absolute + paths on standard output. For instance, if + <envar>NIX_PATH</envar> is + <literal>nixpkgs=/home/alice/nixpkgs</literal>, then + <literal>nix-instantiate --find-file nixpkgs/default.nix</literal> + will print + <literal>/home/alice/nixpkgs/default.nix</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--strict</option></term> + + <listitem><para>When used with <option>--eval</option>, + recursively evaluate list elements and attributes. Normally, such + sub-expressions are left unevaluated (since the Nix expression + language is lazy).</para> + + <warning><para>This option can cause non-termination, because lazy + data structures can be infinitely large.</para></warning> + + </listitem> + + </varlistentry> + + <varlistentry><term><option>--<arg>no</arg>trace-file-access</option></term> + <listitem><para>While instantiating the expression, the evaluator will + print the full path to any files it reads with the prefix + <envar>trace-file-access: </envar> to the standard error.</para> + </listitem> + </varlistentry> + + <varlistentry><term><option>--json</option></term> + + <listitem><para>When used with <option>--eval</option>, print the resulting + value as an JSON representation of the abstract syntax tree rather + than as an ATerm.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--xml</option></term> + + <listitem><para>When used with <option>--eval</option>, print the resulting + value as an XML representation of the abstract syntax tree rather than as + an ATerm. The schema is the same as that used by the <link + linkend="builtin-toXML"><function>toXML</function> built-in</link>. + </para></listitem> + + </varlistentry> + + <varlistentry><term><option>--read-write-mode</option></term> + + <listitem><para>When used with <option>--eval</option>, perform + evaluation in read/write mode so nix language features that + require it will still work (at the cost of needing to do + instantiation of every evaluated derivation). If this option is + not enabled, there may be uninstantiated store paths in the final + output.</para> + + </listitem> + + </varlistentry> + +</variablelist> + +<variablelist condition="manpage"> + <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" /> +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>Instantiating store derivations from a Nix expression, and +building them using <command>nix-store</command>: + +<screen> +$ nix-instantiate test.nix <lineannotation>(instantiate)</lineannotation> +/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv + +$ nix-store -r $(nix-instantiate test.nix) <lineannotation>(build)</lineannotation> +<replaceable>...</replaceable> +/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 <lineannotation>(output path)</lineannotation> + +$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 +dr-xr-xr-x 2 eelco users 4096 1970-01-01 01:00 lib +...</screen> + +</para> + +<para>You can also give a Nix expression on the command line: + +<screen> +$ nix-instantiate -E 'with import <nixpkgs> { }; hello' +/nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv +</screen> + +This is equivalent to: + +<screen> +$ nix-instantiate '<nixpkgs>' -A hello +</screen> + +</para> + +<para>Parsing and evaluating Nix expressions: + +<screen> +$ nix-instantiate --parse -E '1 + 2' +1 + 2 + +$ nix-instantiate --eval -E '1 + 2' +3 + +$ nix-instantiate --eval --xml -E '1 + 2' +<![CDATA[<?xml version='1.0' encoding='utf-8'?> +<expr> + <int value="3" /> +</expr>]]></screen> + +</para> + +<para>The difference between non-strict and strict evaluation: + +<screen> +$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }' +<replaceable>...</replaceable><![CDATA[ + <attr name="x"> + <string value="foo" /> + </attr> + <attr name="y"> + <unevaluated /> + </attr>]]> +<replaceable>...</replaceable></screen> + +Note that <varname>y</varname> is left unevaluated (the XML +representation doesn’t attempt to show non-normal forms). + +<screen> +$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }' +<replaceable>...</replaceable><![CDATA[ + <attr name="x"> + <string value="foo" /> + </attr> + <attr name="y"> + <string value="foo" /> + </attr>]]> +<replaceable>...</replaceable></screen> + +</para> + +</refsection> + +<refsection><title>Conformance</title> + <para>The <option>--trace-file-access</option> option is a nonstandard + extension added by Tvix in 2020.</para> +</refsection> + +<refsection condition="manpage"><title>Environment variables</title> + +<variablelist> + <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" /> +</variablelist> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-prefetch-url.xml b/third_party/nix/doc/manual/command-ref/nix-prefetch-url.xml new file mode 100644 index 000000000000..621ded72ec26 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-prefetch-url.xml @@ -0,0 +1,131 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-prefetch-url"> + +<refmeta> + <refentrytitle>nix-prefetch-url</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-prefetch-url</refname> + <refpurpose>copy a file from a URL into the store and print its hash</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-prefetch-url</command> + <arg><option>--version</option></arg> + <arg><option>--type</option> <replaceable>hashAlgo</replaceable></arg> + <arg><option>--print-path</option></arg> + <arg><option>--unpack</option></arg> + <arg><option>--name</option> <replaceable>name</replaceable></arg> + <arg choice='plain'><replaceable>url</replaceable></arg> + <arg><replaceable>hash</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + +<refsection><title>Description</title> + +<para>The command <command>nix-prefetch-url</command> downloads the +file referenced by the URL <replaceable>url</replaceable>, prints its +cryptographic hash, and copies it into the Nix store. The file name +in the store is +<filename><replaceable>hash</replaceable>-<replaceable>baseName</replaceable></filename>, +where <replaceable>baseName</replaceable> is everything following the +final slash in <replaceable>url</replaceable>.</para> + +<para>This command is just a convenience for Nix expression writers. +Often a Nix expression fetches some source distribution from the +network using the <literal>fetchurl</literal> expression contained in +Nixpkgs. However, <literal>fetchurl</literal> requires a +cryptographic hash. If you don't know the hash, you would have to +download the file first, and then <literal>fetchurl</literal> would +download it again when you build your Nix expression. Since +<literal>fetchurl</literal> uses the same name for the downloaded file +as <command>nix-prefetch-url</command>, the redundant download can be +avoided.</para> + +<para>If <replaceable>hash</replaceable> is specified, then a download +is not performed if the Nix store already contains a file with the +same hash and base name. Otherwise, the file is downloaded, and an +error is signaled if the actual hash of the file does not match the +specified hash.</para> + +<para>This command prints the hash on standard output. Additionally, +if the option <option>--print-path</option> is used, the path of the +downloaded file in the Nix store is also printed.</para> + +</refsection> + + +<refsection><title>Options</title> + +<variablelist> + + <varlistentry><term><option>--type</option> <replaceable>hashAlgo</replaceable></term> + + <listitem><para>Use the specified cryptographic hash algorithm, + which can be one of <literal>md5</literal>, + <literal>sha1</literal>, and + <literal>sha256</literal>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--print-path</option></term> + + <listitem><para>Print the store path of the downloaded file on + standard output.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--unpack</option></term> + + <listitem><para>Unpack the archive (which must be a tarball or zip + file) and add the result to the Nix store. The resulting hash can + be used with functions such as Nixpkgs’s + <varname>fetchzip</varname> or + <varname>fetchFromGitHub</varname>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--name</option> <replaceable>name</replaceable></term> + + <listitem><para>Override the name of the file in the Nix store. By + default, this is + <literal><replaceable>hash</replaceable>-<replaceable>basename</replaceable></literal>, + where <replaceable>basename</replaceable> is the last component of + <replaceable>url</replaceable>. Overriding the name is necessary + when <replaceable>basename</replaceable> contains characters that + are not allowed in Nix store paths.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<screen> +$ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz +0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i + +$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz +0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + +$ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz +079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7 +/nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz +</screen> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-shell.xml b/third_party/nix/doc/manual/command-ref/nix-shell.xml new file mode 100644 index 000000000000..bb4a4e420122 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-shell.xml @@ -0,0 +1,397 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-shell"> + +<refmeta> + <refentrytitle>nix-shell</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-shell</refname> + <refpurpose>start an interactive shell based on a Nix expression</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-shell</command> + <arg><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></arg> + <arg> + <group choice='req'> + <arg choice='plain'><option>--attr</option></arg> + <arg choice='plain'><option>-A</option></arg> + </group> + <replaceable>attrPath</replaceable> + </arg> + <arg><option>--command</option> <replaceable>cmd</replaceable></arg> + <arg><option>--run</option> <replaceable>cmd</replaceable></arg> + <arg><option>--exclude</option> <replaceable>regexp</replaceable></arg> + <arg><option>--pure</option></arg> + <arg><option>--keep</option> <replaceable>name</replaceable></arg> + <group choice='req'> + <arg choice='plain'> + <group choice='req'> + <arg choice='plain'><option>--packages</option></arg> + <arg choice='plain'><option>-p</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>packages</replaceable></arg> + </arg> + <arg><replaceable>path</replaceable></arg> + </group> + </cmdsynopsis> +</refsynopsisdiv> + +<refsection><title>Description</title> + +<para>The command <command>nix-shell</command> will build the +dependencies of the specified derivation, but not the derivation +itself. It will then start an interactive shell in which all +environment variables defined by the derivation +<replaceable>path</replaceable> have been set to their corresponding +values, and the script <literal>$stdenv/setup</literal> has been +sourced. This is useful for reproducing the environment of a +derivation for development.</para> + +<para>If <replaceable>path</replaceable> is not given, +<command>nix-shell</command> defaults to +<filename>shell.nix</filename> if it exists, and +<filename>default.nix</filename> otherwise.</para> + +<para>If <replaceable>path</replaceable> starts with +<literal>http://</literal> or <literal>https://</literal>, it is +interpreted as the URL of a tarball that will be downloaded and +unpacked to a temporary location. The tarball must include a single +top-level directory containing at least a file named +<filename>default.nix</filename>.</para> + +<para>If the derivation defines the variable +<varname>shellHook</varname>, it will be evaluated after +<literal>$stdenv/setup</literal> has been sourced. Since this hook is +not executed by regular Nix builds, it allows you to perform +initialisation specific to <command>nix-shell</command>. For example, +the derivation attribute + +<programlisting> +shellHook = + '' + echo "Hello shell" + ''; +</programlisting> + +will cause <command>nix-shell</command> to print <literal>Hello shell</literal>.</para> + +</refsection> + + +<refsection><title>Options</title> + +<para>All options not listed here are passed to <command>nix-store +--realise</command>, except for <option>--arg</option> and +<option>--attr</option> / <option>-A</option> which are passed to +<command>nix-instantiate</command>. <phrase condition="manual">See +also <xref linkend="sec-common-options" />.</phrase></para> + +<variablelist> + + <varlistentry><term><option>--command</option> <replaceable>cmd</replaceable></term> + + <listitem><para>In the environment of the derivation, run the + shell command <replaceable>cmd</replaceable>. This command is + executed in an interactive shell. (Use <option>--run</option> to + use a non-interactive shell instead.) However, a call to + <literal>exit</literal> is implicitly added to the command, so the + shell will exit after running the command. To prevent this, add + <literal>return</literal> at the end; e.g. <literal>--command + "echo Hello; return"</literal> will print <literal>Hello</literal> + and then drop you into the interactive shell. This can be useful + for doing any additional initialisation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--run</option> <replaceable>cmd</replaceable></term> + + <listitem><para>Like <option>--command</option>, but executes the + command in a non-interactive shell. This means (among other + things) that if you hit Ctrl-C while the command is running, the + shell exits.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--exclude</option> <replaceable>regexp</replaceable></term> + + <listitem><para>Do not build any dependencies whose store path + matches the regular expression <replaceable>regexp</replaceable>. + This option may be specified multiple times.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--pure</option></term> + + <listitem><para>If this flag is specified, the environment is + almost entirely cleared before the interactive shell is started, + so you get an environment that more closely corresponds to the + “real” Nix build. A few variables, in particular + <envar>HOME</envar>, <envar>USER</envar> and + <envar>DISPLAY</envar>, are retained. Note that + <filename>~/.bashrc</filename> and (depending on your Bash + installation) <filename>/etc/bashrc</filename> are still sourced, + so any variables set there will affect the interactive + shell.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--packages</option> / <option>-p</option> <replaceable>packages</replaceable>…</term> + + <listitem><para>Set up an environment in which the specified + packages are present. The command line arguments are interpreted + as attribute names inside the Nix Packages collection. Thus, + <literal>nix-shell -p libjpeg openjdk</literal> will start a shell + in which the packages denoted by the attribute names + <varname>libjpeg</varname> and <varname>openjdk</varname> are + present.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>-i</option> <replaceable>interpreter</replaceable></term> + + <listitem><para>The chained script interpreter to be invoked by + <command>nix-shell</command>. Only applicable in + <literal>#!</literal>-scripts (described <link + linkend="ssec-nix-shell-shebang">below</link>).</para> + + </listitem></varlistentry> + + <varlistentry><term><option>--keep</option> <replaceable>name</replaceable></term> + + <listitem><para>When a <option>--pure</option> shell is started, + keep the listed environment variables.</para></listitem> + + </varlistentry> + +</variablelist> + +<para>The following common options are supported:</para> + +<variablelist condition="manpage"> + <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" /> +</variablelist> + +</refsection> + + +<refsection><title>Environment variables</title> + +<variablelist> + + <varlistentry><term><envar>NIX_BUILD_SHELL</envar></term> + + <listitem><para>Shell used to start the interactive environment. + Defaults to the <command>bash</command> found in <envar>PATH</envar>.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>To build the dependencies of the package Pan, and start an +interactive shell in which to build it: + +<screen> +$ nix-shell '<nixpkgs>' -A pan +[nix-shell]$ unpackPhase +[nix-shell]$ cd pan-* +[nix-shell]$ configurePhase +[nix-shell]$ buildPhase +[nix-shell]$ ./pan/gui/pan +</screen> + +To clear the environment first, and do some additional automatic +initialisation of the interactive shell: + +<screen> +$ nix-shell '<nixpkgs>' -A pan --pure \ + --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' +</screen> + +Nix expressions can also be given on the command line. For instance, +the following starts a shell containing the packages +<literal>sqlite</literal> and <literal>libX11</literal>: + +<screen> +$ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' +</screen> + +A shorter way to do the same is: + +<screen> +$ nix-shell -p sqlite xorg.libX11 +[nix-shell]$ echo $NIX_LDFLAGS +… -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … +</screen> + +The <command>-p</command> flag looks up Nixpkgs in the Nix search +path. You can override it by passing <option>-I</option> or setting +<envar>NIX_PATH</envar>. For example, the following gives you a shell +containing the Pan package from a specific revision of Nixpkgs: + +<screen> +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz + +[nix-shell:~]$ pan --version +Pan 0.139 +</screen> + +</para> + +</refsection> + + +<refsection xml:id="ssec-nix-shell-shebang"><title>Use as a <literal>#!</literal>-interpreter</title> + +<para>You can use <command>nix-shell</command> as a script interpreter +to allow scripts written in arbitrary languages to obtain their own +dependencies via Nix. This is done by starting the script with the +following lines: + +<programlisting> +#! /usr/bin/env nix-shell +#! nix-shell -i <replaceable>real-interpreter</replaceable> -p <replaceable>packages</replaceable> +</programlisting> + +where <replaceable>real-interpreter</replaceable> is the “real” script +interpreter that will be invoked by <command>nix-shell</command> after +it has obtained the dependencies and initialised the environment, and +<replaceable>packages</replaceable> are the attribute names of the +dependencies in Nixpkgs.</para> + +<para>The lines starting with <literal>#! nix-shell</literal> specify +<command>nix-shell</command> options (see above). Note that you cannot +write <literal>#! /usr/bin/env nix-shell -i ...</literal> because +many operating systems only allow one argument in +<literal>#!</literal> lines.</para> + +<para>For example, here is a Python script that depends on Python and +the <literal>prettytable</literal> package: + +<programlisting> +#! /usr/bin/env nix-shell +#! nix-shell -i python -p python pythonPackages.prettytable + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +</programlisting> + +</para> + +<para>Similarly, the following is a Perl script that specifies that it +requires Perl and the <literal>HTML::TokeParser::Simple</literal> and +<literal>LWP</literal> packages: + +<programlisting> +#! /usr/bin/env nix-shell +#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +</programlisting> + +</para> + +<para>Sometimes you need to pass a simple Nix expression to customize +a package like Terraform: + +<programlisting><![CDATA[ +#! /usr/bin/env nix-shell +#! nix-shell -i bash -p "terraform.withPlugins (plugins: [ plugins.openstack ])" + +terraform apply +]]></programlisting> + +<note><para>You must use double quotes (<literal>"</literal>) when +passing a simple Nix expression in a nix-shell shebang.</para></note> +</para> + +<para>Finally, using the merging of multiple nix-shell shebangs the +following Haskell script uses a specific branch of Nixpkgs/NixOS (the +18.03 stable branch): + +<programlisting><![CDATA[ +#! /usr/bin/env nix-shell +#! nix-shell -i runghc -p "haskellPackages.ghcWithPackages (ps: [ps.HTTP ps.tagsoup])" +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-18.03.tar.gz + +import Network.HTTP +import Text.HTML.TagSoup + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/") + body <- getResponseBody resp + let tags = filter (isTagOpenName "a") $ parseTags body + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +]]></programlisting> + +If you want to be even more precise, you can specify a specific +revision of Nixpkgs: + +<programlisting> +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz +</programlisting> + +</para> + +<para>The examples above all used <option>-p</option> to get +dependencies from Nixpkgs. You can also use a Nix expression to build +your own dependencies. For example, the Python example could have been +written as: + +<programlisting> +#! /usr/bin/env nix-shell +#! nix-shell deps.nix -i python +</programlisting> + +where the file <filename>deps.nix</filename> in the same directory +as the <literal>#!</literal>-script contains: + +<programlisting> +with import <nixpkgs> {}; + +runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" +</programlisting> + +</para> + +</refsection> + + +<refsection condition="manpage"><title>Environment variables</title> + +<variablelist> + <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" /> +</variablelist> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/nix-store.xml b/third_party/nix/doc/manual/command-ref/nix-store.xml new file mode 100644 index 000000000000..113a3c2e41ed --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-store.xml @@ -0,0 +1,1525 @@ +<refentry xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-nix-store"> + +<refmeta> + <refentrytitle>nix-store</refentrytitle> + <manvolnum>1</manvolnum> + <refmiscinfo class="source">Nix</refmiscinfo> + <refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo> +</refmeta> + +<refnamediv> + <refname>nix-store</refname> + <refpurpose>manipulate or query the Nix store</refpurpose> +</refnamediv> + +<refsynopsisdiv> + <cmdsynopsis> + <command>nix-store</command> + <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="opt-common-syn.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(/db:nop/*)" /> + <arg><option>--add-root</option> <replaceable>path</replaceable></arg> + <arg><option>--indirect</option></arg> + <arg choice='plain'><replaceable>operation</replaceable></arg> + <arg rep='repeat'><replaceable>options</replaceable></arg> + <arg rep='repeat'><replaceable>arguments</replaceable></arg> + </cmdsynopsis> +</refsynopsisdiv> + + +<refsection><title>Description</title> + +<para>The command <command>nix-store</command> performs primitive +operations on the Nix store. You generally do not need to run this +command manually.</para> + +<para><command>nix-store</command> takes exactly one +<emphasis>operation</emphasis> flag which indicates the subcommand to +be performed. These are documented below.</para> + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Common options</title> + +<para>This section lists the options that are common to all +operations. These options are allowed for every subcommand, though +they may not always have an effect. <phrase condition="manual">See +also <xref linkend="sec-common-options" /> for a list of common +options.</phrase></para> + +<variablelist> + + <varlistentry xml:id="opt-add-root"><term><option>--add-root</option> <replaceable>path</replaceable></term> + + <listitem><para>Causes the result of a realisation + (<option>--realise</option> and <option>--force-realise</option>) + to be registered as a root of the garbage collector<phrase + condition="manual"> (see <xref linkend="ssec-gc-roots" + />)</phrase>. The root is stored in + <replaceable>path</replaceable>, which must be inside a directory + that is scanned for roots by the garbage collector (i.e., + typically in a subdirectory of + <filename>/nix/var/nix/gcroots/</filename>) + <emphasis>unless</emphasis> the <option>--indirect</option> flag + is used.</para> + + <para>If there are multiple results, then multiple symlinks will + be created by sequentially numbering symlinks beyond the first one + (e.g., <filename>foo</filename>, <filename>foo-2</filename>, + <filename>foo-3</filename>, and so on).</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--indirect</option></term> + + <listitem> + + <para>In conjunction with <option>--add-root</option>, this option + allows roots to be stored <emphasis>outside</emphasis> of the GC + roots directory. This is useful for commands such as + <command>nix-build</command> that place a symlink to the build + result in the current directory; such a build result should not be + garbage-collected unless the symlink is removed.</para> + + <para>The <option>--indirect</option> flag causes a uniquely named + symlink to <replaceable>path</replaceable> to be stored in + <filename>/nix/var/nix/gcroots/auto/</filename>. For instance, + + <screen> +$ nix-store --add-root /home/eelco/bla/result --indirect -r <replaceable>...</replaceable> + +$ ls -l /nix/var/nix/gcroots/auto +lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result + +$ ls -l /home/eelco/bla/result +lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10</screen> + + Thus, when <filename>/home/eelco/bla/result</filename> is removed, + the GC root in the <filename>auto</filename> directory becomes a + dangling symlink and will be ignored by the collector.</para> + + <warning><para>Note that it is not possible to move or rename + indirect GC roots, since the symlink in the + <filename>auto</filename> directory will still point to the old + location.</para></warning> + + </listitem> + + </varlistentry> + +</variablelist> + +<variablelist condition="manpage"> + <xi:include href="opt-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='opt-common']/*)" /> +</variablelist> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id='rsec-nix-store-realise'><title>Operation <option>--realise</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <group choice='req'> + <arg choice='plain'><option>--realise</option></arg> + <arg choice='plain'><option>-r</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> + <arg><option>--dry-run</option></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--realise</option> essentially “builds” +the specified store paths. Realisation is a somewhat overloaded term: + +<itemizedlist> + + <listitem><para>If the store path is a + <emphasis>derivation</emphasis>, realisation ensures that the output + paths of the derivation are <link + linkend="gloss-validity">valid</link> (i.e., the output path and its + closure exist in the file system). This can be done in several + ways. First, it is possible that the outputs are already valid, in + which case we are done immediately. Otherwise, there may be <link + linkend="gloss-substitute">substitutes</link> that produce the + outputs (e.g., by downloading them). Finally, the outputs can be + produced by performing the build action described by the + derivation.</para></listitem> + + <listitem><para>If the store path is not a derivation, realisation + ensures that the specified path is valid (i.e., it and its closure + exist in the file system). If the path is already valid, we are + done immediately. Otherwise, the path and any missing paths in its + closure may be produced through substitutes. If there are no + (successful) subsitutes, realisation fails.</para></listitem> + +</itemizedlist> + +</para> + +<para>The output path of each derivation is printed on standard +output. (For non-derivations argument, the argument itself is +printed.)</para> + +<para>The following flags are available:</para> + +<variablelist> + + <varlistentry><term><option>--dry-run</option></term> + + <listitem><para>Print on standard error a description of what + packages would be built or downloaded, without actually performing + the operation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--ignore-unknown</option></term> + + <listitem><para>If a non-derivation path does not have a + substitute, then silently ignore it.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--check</option></term> + + <listitem><para>This option allows you to check whether a + derivation is deterministic. It rebuilds the specified derivation + and checks whether the result is bitwise-identical with the + existing outputs, printing an error if that’s not the case. The + outputs of the specified derivation must already exist. When used + with <option>-K</option>, if an output path is not identical to + the corresponding output from the previous build, the new output + path is left in + <filename>/nix/store/<replaceable>name</replaceable>.check.</filename></para> + + <para>See also the <option>build-repeat</option> configuration + option, which repeats a derivation a number of times and prevents + its outputs from being registered as “valid” in the Nix store + unless they are identical.</para></listitem> + + </varlistentry> + +</variablelist> + +<para>Special exit codes:</para> + +<variablelist> + + <varlistentry><term><literal>100</literal></term> + <listitem><para>Generic build failure, the builder process + returned with a non-zero exit code.</para></listitem> + </varlistentry> + + <varlistentry><term><literal>101</literal></term> + <listitem><para>Build timeout, the build was aborted because it + did not complete within the specified <link + linkend='conf-timeout'><literal>timeout</literal></link>. + </para></listitem> + </varlistentry> + + <varlistentry><term><literal>102</literal></term> + <listitem><para>Hash mismatch, the build output was rejected + because it does not match the specified <link + linkend="fixed-output-drvs"><varname>outputHash</varname></link>. + </para></listitem> + </varlistentry> + + <varlistentry><term><literal>104</literal></term> + <listitem><para>Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable.</para> + </listitem> + </varlistentry> + +</variablelist> + +<para>With the <option>--keep-going</option> flag it's possible for +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. <screen> +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic +</screen></para> + +</refsection> + + +<refsection><title>Examples</title> + +<para>This operation is typically used to build store derivations +produced by <link +linkend="sec-nix-instantiate"><command>nix-instantiate</command></link>: + +<screen> +$ nix-store -r $(nix-instantiate ./test.nix) +/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1</screen> + +This is essentially what <link +linkend="sec-nix-build"><command>nix-build</command></link> does.</para> + +<para>To test whether a previously-built derivation is deterministic: + +<screen> +$ nix-build '<nixpkgs>' -A hello --check -K +</screen> + +</para> + +</refsection> + + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id='rsec-nix-store-serve'><title>Operation <option>--serve</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--serve</option></arg> + <arg><option>--write</option></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--serve</option> provides access to +the Nix store over stdin and stdout, and is intended to be used +as a means of providing Nix store access to a restricted ssh user. +</para> + +<para>The following flags are available:</para> + +<variablelist> + + <varlistentry><term><option>--write</option></term> + + <listitem><para>Allow the connected client to request the realization + of derivations. In effect, this can be used to make the host act + as a remote builder.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>To turn a host into a build server, the +<filename>authorized_keys</filename> file can be used to provide build +access to a given SSH public key: + +<screen> +$ cat <<EOF >>/root/.ssh/authorized_keys +command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... +EOF +</screen> + +</para> + +</refsection> + + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id='rsec-nix-store-gc'><title>Operation <option>--gc</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--gc</option></arg> + <group> + <arg choice='plain'><option>--print-roots</option></arg> + <arg choice='plain'><option>--print-live</option></arg> + <arg choice='plain'><option>--print-dead</option></arg> + <arg choice='plain'><option>--delete</option></arg> + </group> + <arg><option>--max-freed</option> <replaceable>bytes</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>Without additional flags, the operation <option>--gc</option> +performs a garbage collection on the Nix store. That is, all paths in +the Nix store not reachable via file system references from a set of +“roots”, are deleted.</para> + +<para>The following suboperations may be specified:</para> + +<variablelist> + + <varlistentry><term><option>--print-roots</option></term> + + <listitem><para>This operation prints on standard output the set + of roots used by the garbage collector. What constitutes a root + is described in <xref linkend="ssec-gc-roots" + />.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--print-live</option></term> + + <listitem><para>This operation prints on standard output the set + of “live” store paths, which are all the store paths reachable + from the roots. Live paths should never be deleted, since that + would break consistency — it would become possible that + applications are installed that reference things that are no + longer present in the store.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--print-dead</option></term> + + <listitem><para>This operation prints out on standard output the + set of “dead” store paths, which is just the opposite of the set + of live paths: any path in the store that is not live (with + respect to the roots) is dead.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--delete</option></term> + + <listitem><para>This operation performs an actual garbage + collection. All dead paths are removed from the + store. This is the default.</para></listitem> + + </varlistentry> + +</variablelist> + +<para>By default, all unreachable paths are deleted. The following +options control what gets deleted and in what order: + +<variablelist> + + <varlistentry><term><option>--max-freed</option> <replaceable>bytes</replaceable></term> + + <listitem><para>Keep deleting paths until at least + <replaceable>bytes</replaceable> bytes have been deleted, then + stop. The argument <replaceable>bytes</replaceable> can be + followed by the multiplicative suffix <literal>K</literal>, + <literal>M</literal>, <literal>G</literal> or + <literal>T</literal>, denoting KiB, MiB, GiB or TiB + units.</para></listitem> + + </varlistentry> + +</variablelist> + +</para> + +<para>The behaviour of the collector is also influenced by the <link +linkend="conf-keep-outputs"><literal>keep-outputs</literal></link> +and <link +linkend="conf-keep-derivations"><literal>keep-derivations</literal></link> +variables in the Nix configuration file.</para> + +<para>With <option>--delete</option>, the collector prints the total +number of freed bytes when it finishes (or when it is interrupted). +With <option>--print-dead</option>, it prints the number of bytes that +would be freed.</para> + +</refsection> + + +<refsection><title>Examples</title> + +<para>To delete all unreachable paths, just do: + +<screen> +$ nix-store --gc +deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' +<replaceable>...</replaceable> +8825586 bytes freed (8.42 MiB)</screen> + +</para> + +<para>To delete at least 100 MiBs of unreachable paths: + +<screen> +$ nix-store --gc --max-freed $((100 * 1024 * 1024))</screen> + +</para> + +</refsection> + + +</refsection> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--delete</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--delete</option></arg> + <arg><option>--ignore-liveness</option></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--delete</option> deletes the store paths +<replaceable>paths</replaceable> from the Nix store, but only if it is +safe to do so; that is, when the path is not reachable from a root of +the garbage collector. This means that you can only delete paths that +would also be deleted by <literal>nix-store --gc</literal>. Thus, +<literal>--delete</literal> is a more targeted version of +<literal>--gc</literal>.</para> + +<para>With the option <option>--ignore-liveness</option>, reachability +from the roots is ignored. However, the path still won’t be deleted +if there are other paths in the store that refer to it (i.e., depend +on it).</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 +0 bytes freed (0.00 MiB) +error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive</screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id='refsec-nix-store-query'><title>Operation <option>--query</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <group choice='req'> + <arg choice='plain'><option>--query</option></arg> + <arg choice='plain'><option>-q</option></arg> + </group> + <group choice='req'> + <arg choice='plain'><option>--outputs</option></arg> + <arg choice='plain'><option>--requisites</option></arg> + <arg choice='plain'><option>-R</option></arg> + <arg choice='plain'><option>--references</option></arg> + <arg choice='plain'><option>--referrers</option></arg> + <arg choice='plain'><option>--referrers-closure</option></arg> + <arg choice='plain'><option>--deriver</option></arg> + <arg choice='plain'><option>-d</option></arg> + <arg choice='plain'><option>--graph</option></arg> + <arg choice='plain'><option>--tree</option></arg> + <arg choice='plain'><option>--binding</option> <replaceable>name</replaceable></arg> + <arg choice='plain'><option>-b</option> <replaceable>name</replaceable></arg> + <arg choice='plain'><option>--hash</option></arg> + <arg choice='plain'><option>--size</option></arg> + <arg choice='plain'><option>--roots</option></arg> + </group> + <arg><option>--use-output</option></arg> + <arg><option>-u</option></arg> + <arg><option>--force-realise</option></arg> + <arg><option>-f</option></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> +</cmdsynopsis> + +</refsection> + + +<refsection><title>Description</title> + +<para>The operation <option>--query</option> displays various bits of +information about the store paths . The queries are described below. At +most one query can be specified. The default query is +<option>--outputs</option>.</para> + +<para>The paths <replaceable>paths</replaceable> may also be symlinks +from outside of the Nix store, to the Nix store. In that case, the +query is applied to the target of the symlink.</para> + + +</refsection> + + +<refsection><title>Common query options</title> + +<variablelist> + + <varlistentry><term><option>--use-output</option></term> + <term><option>-u</option></term> + + <listitem><para>For each argument to the query that is a store + derivation, apply the query to the output path of the derivation + instead.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--force-realise</option></term> + <term><option>-f</option></term> + + <listitem><para>Realise each argument to the query first (see + <link linkend="rsec-nix-store-realise"><command>nix-store + --realise</command></link>).</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection xml:id='nixref-queries'><title>Queries</title> + +<variablelist> + + <varlistentry><term><option>--outputs</option></term> + + <listitem><para>Prints out the <link + linkend="gloss-output-path">output paths</link> of the store + derivations <replaceable>paths</replaceable>. These are the paths + that will be produced when the derivation is + built.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--requisites</option></term> + <term><option>-R</option></term> + + <listitem><para>Prints out the <link + linkend="gloss-closure">closure</link> of the store path + <replaceable>paths</replaceable>.</para> + + <para>This query has one option:</para> + + <variablelist> + + <varlistentry><term><option>--include-outputs</option></term> + + <listitem><para>Also include the output path of store + derivations, and their closures.</para></listitem> + + </varlistentry> + + </variablelist> + + <para>This query can be used to implement various kinds of + deployment. A <emphasis>source deployment</emphasis> is obtained + by distributing the closure of a store derivation. A + <emphasis>binary deployment</emphasis> is obtained by distributing + the closure of an output path. A <emphasis>cache + deployment</emphasis> (combined source/binary deployment, + including binaries of build-time-only dependencies) is obtained by + distributing the closure of a store derivation and specifying the + option <option>--include-outputs</option>.</para> + + </listitem> + + </varlistentry> + + <varlistentry><term><option>--references</option></term> + + <listitem><para>Prints the set of <link + linkend="gloss-reference">references</link> of the store paths + <replaceable>paths</replaceable>, that is, their immediate + dependencies. (For <emphasis>all</emphasis> dependencies, use + <option>--requisites</option>.)</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--referrers</option></term> + + <listitem><para>Prints the set of <emphasis>referrers</emphasis> of + the store paths <replaceable>paths</replaceable>, that is, the + store paths currently existing in the Nix store that refer to one + of <replaceable>paths</replaceable>. Note that contrary to the + references, the set of referrers is not constant; it can change as + store paths are added or removed.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--referrers-closure</option></term> + + <listitem><para>Prints the closure of the set of store paths + <replaceable>paths</replaceable> under the referrers relation; that + is, all store paths that directly or indirectly refer to one of + <replaceable>paths</replaceable>. These are all the path currently + in the Nix store that are dependent on + <replaceable>paths</replaceable>.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--deriver</option></term> + <term><option>-d</option></term> + + <listitem><para>Prints the <link + linkend="gloss-deriver">deriver</link> of the store paths + <replaceable>paths</replaceable>. If the path has no deriver + (e.g., if it is a source file), or if the deriver is not known + (e.g., in the case of a binary-only deployment), the string + <literal>unknown-deriver</literal> is printed.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--graph</option></term> + + <listitem><para>Prints the references graph of the store paths + <replaceable>paths</replaceable> in the format of the + <command>dot</command> tool of AT&T's <link + xlink:href="http://www.graphviz.org/">Graphviz package</link>. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--tree</option></term> + + <listitem><para>Prints the references graph of the store paths + <replaceable>paths</replaceable> as a nested ASCII tree. + References are ordered by descending closure size; this tends to + flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the + graph.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--graphml</option></term> + + <listitem><para>Prints the references graph of the store paths + <replaceable>paths</replaceable> in the <link + xlink:href="http://graphml.graphdrawing.org/">GraphML</link> file format. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--binding</option> <replaceable>name</replaceable></term> + <term><option>-b</option> <replaceable>name</replaceable></term> + + <listitem><para>Prints the value of the attribute + <replaceable>name</replaceable> (i.e., environment variable) of + the store derivations <replaceable>paths</replaceable>. It is an + error for a derivation to not have the specified + attribute.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--hash</option></term> + + <listitem><para>Prints the SHA-256 hash of the contents of the + store paths <replaceable>paths</replaceable> (that is, the hash of + the output of <command>nix-store --dump</command> on the given + paths). Since the hash is stored in the Nix database, this is a + fast operation.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--size</option></term> + + <listitem><para>Prints the size in bytes of the contents of the + store paths <replaceable>paths</replaceable> — to be precise, the + size of the output of <command>nix-store --dump</command> on the + given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--roots</option></term> + + <listitem><para>Prints the garbage collector roots that point, + directly or indirectly, at the store paths + <replaceable>paths</replaceable>.</para></listitem> + + </varlistentry> + +</variablelist> + +</refsection> + + +<refsection><title>Examples</title> + +<para>Print the closure (runtime dependencies) of the +<command>svn</command> program in the current user environment: + +<screen> +$ nix-store -qR $(which svn) +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 +<replaceable>...</replaceable></screen> + +</para> + +<para>Print the build-time dependencies of <command>svn</command>: + +<screen> +$ nix-store -qR $(nix-store -qd $(which svn)) +/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv +/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh +/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv +<replaceable>... lots of other paths ...</replaceable></screen> + +The difference with the previous example is that we ask the closure of +the derivation (<option>-qd</option>), not the closure of the output +path that contains <command>svn</command>.</para> + +<para>Show the build-time dependencies as a tree: + +<screen> +$ nix-store -q --tree $(nix-store -qd $(which svn)) +/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv ++---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh ++---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv +| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash +| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh +<replaceable>...</replaceable></screen> + +</para> + +<para>Show all paths that depend on the same OpenSSL library as +<command>svn</command>: + +<screen> +$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) +/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 +/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5</screen> + +</para> + +<para>Show all paths that directly or indirectly depend on the Glibc +(C library) used by <command>svn</command>: + +<screen> +$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') +/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 +/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 +<replaceable>...</replaceable></screen> + +Note that <command>ldd</command> is a command that prints out the +dynamic libraries used by an ELF executable.</para> + +<para>Make a picture of the runtime dependency graph of the current +user environment: + +<screen> +$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps +$ gv graph.ps</screen> + +</para> + +<para>Show every garbage collector root that points to a store path +that depends on <command>svn</command>: + +<screen> +$ nix-store -q --roots $(which svn) +/nix/var/nix/profiles/default-81-link +/nix/var/nix/profiles/default-82-link +/nix/var/nix/profiles/per-user/eelco/profile-97-link +</screen> + +</para> + +</refsection> + + +</refsection> + + + +<!--######################################################################--> + +<!-- +<refsection xml:id="rsec-nix-store-reg-val"><title>Operation <option>-XXX-register-validity</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>-XXX-register-validity</option></arg> +</cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>TODO</para> + +</refsection> + +</refsection> +--> + + + +<!--######################################################################--> + +<refsection><title>Operation <option>--add</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--add</option></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--add</option> adds the specified paths to +the Nix store. It prints the resulting paths in the Nix store on +standard output.</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --add ./foo.c +/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c</screen> + +</refsection> + +</refsection> + +<!--######################################################################--> + +<refsection><title>Operation <option>--add-fixed</option></title> + +<refsection><title>Synopsis</title> + +<cmdsynopsis> + <command>nix-store</command> + <arg><option>--recursive</option></arg> + <arg choice='plain'><option>--add-fixed</option></arg> + <arg choice='plain'><replaceable>algorithm</replaceable></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> +</cmdsynopsis> + +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--add-fixed</option> adds the specified paths to +the Nix store. Unlike <option>--add</option> paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. +</para> + +<para>This operation has the following options: + +<variablelist> + + <varlistentry><term><option>--recursive</option></term> + + <listitem><para> + Use recursive instead of flat hashing mode, used when adding directories + to the store. + </para></listitem> + + </varlistentry> + +</variablelist> + +</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz</screen> + +</refsection> + +</refsection> + + + +<!--######################################################################--> + +<refsection xml:id='refsec-nix-store-verify'><title>Operation <option>--verify</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--verify</option></arg> + <arg><option>--check-contents</option></arg> + <arg><option>--repair</option></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--verify</option> verifies the internal +consistency of the Nix database, and the consistency between the Nix +database and the Nix store. Any inconsistencies encountered are +automatically repaired. Inconsistencies are generally the result of +the Nix store or database being modified by non-Nix tools, or of bugs +in Nix itself.</para> + +<para>This operation has the following options: + +<variablelist> + + <varlistentry><term><option>--check-contents</option></term> + + <listitem><para>Checks that the contents of every valid store path + has not been altered by computing a SHA-256 hash of the contents + and comparing it with the hash stored in the Nix database at build + time. Paths that have been modified are printed out. For large + stores, <option>--check-contents</option> is obviously quite + slow.</para></listitem> + + </varlistentry> + + <varlistentry><term><option>--repair</option></term> + + <listitem><para>If any valid path is missing from the store, or + (if <option>--check-contents</option> is given) the contents of a + valid path has been modified, then try to repair the path by + redownloading it. See <command>nix-store --repair-path</command> + for details.</para></listitem> + + </varlistentry> + +</variablelist> + +</para> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--verify-path</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--verify-path</option></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--verify-path</option> compares the +contents of the given store paths to their cryptographic hashes stored +in Nix’s database. For every changed path, it prints a warning +message. The exit status is 0 if no path has changed, and 1 +otherwise.</para> + +</refsection> + +<refsection><title>Example</title> + +<para>To verify the integrity of the <command>svn</command> command and all its dependencies: + +<screen> +$ nix-store --verify-path $(nix-store -qR $(which svn)) +</screen> + +</para> + +</refsection> + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--repair-path</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--repair-path</option></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--repair-path</option> attempts to +“repair” the specified paths by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible.</para> + +<warning><para>During repair, there is a very small time window during +which the old path (if it exists) is moved out of the way and replaced +with the new path. If repair is interrupted in between, then the +system may be left in a broken state (e.g., if the path contains a +critical system component like the GNU C Library).</para></warning> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! + expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', + got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' + +$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... +… +</screen> + +</refsection> + +</refsection> + + +<!--######################################################################--> + +<refsection xml:id='refsec-nix-store-dump'><title>Operation <option>--dump</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--dump</option></arg> + <arg choice='plain'><replaceable>path</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--dump</option> produces a NAR (Nix +ARchive) file containing the contents of the file system tree rooted +at <replaceable>path</replaceable>. The archive is written to +standard output.</para> + +<para>A NAR archive is like a TAR or Zip archive, but it contains only +the information that Nix considers important. For instance, +timestamps are elided because all files in the Nix store have their +timestamp set to 0 anyway. Likewise, all permissions are left out +except for the execute bit, because all files in the Nix store have +644 or 755 permission.</para> + +<para>Also, a NAR archive is <emphasis>canonical</emphasis>, meaning +that “equal” paths always produce the same NAR archive. For instance, +directory entries are always sorted so that the actual on-disk order +doesn’t influence the result. This means that the cryptographic hash +of a NAR dump of a path is usable as a fingerprint of the contents of +the path. Indeed, the hashes of store paths stored in Nix’s database +(see <link linkend="refsec-nix-store-query"><literal>nix-store -q +--hash</literal></link>) are SHA-256 hashes of the NAR dump of each +store path.</para> + +<para>NAR archives support filenames of unlimited length and 64-bit +file sizes. They can contain regular files, directories, and symbolic +links, but not other types of files (such as device nodes).</para> + +<para>A Nix archive can be unpacked using <literal>nix-store +--restore</literal>.</para> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--restore</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--restore</option></arg> + <arg choice='plain'><replaceable>path</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--restore</option> unpacks a NAR archive +to <replaceable>path</replaceable>, which must not already exist. The +archive is read from standard input.</para> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection xml:id='refsec-nix-store-export'><title>Operation <option>--export</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--export</option></arg> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--export</option> writes a serialisation +of the specified store paths to standard output in a format that can +be imported into another Nix store with <command +linkend="refsec-nix-store-import">nix-store --import</command>. This +is like <command linkend="refsec-nix-store-dump">nix-store +--dump</command>, except that the NAR archive produced by that command +doesn’t contain the necessary meta-information to allow it to be +imported into another Nix store (namely, the set of references of the +path).</para> + +<para>This command does not produce a <emphasis>closure</emphasis> of +the specified paths, so if a store path references other store paths +that are missing in the target Nix store, the import will fail. To +copy a whole closure, do something like: + +<screen> +$ nix-store --export $(nix-store -qR <replaceable>paths</replaceable>) > out</screen> + +To import the whole closure again, run: + +<screen> +$ nix-store --import < out</screen> + +</para> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection xml:id='refsec-nix-store-import'><title>Operation <option>--import</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--import</option></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--import</option> reads a serialisation of +a set of store paths produced by <command +linkend="refsec-nix-store-export">nix-store --export</command> from +standard input and adds those store paths to the Nix store. Paths +that already exist in the Nix store are ignored. If a path refers to +another path that doesn’t exist in the Nix store, the import +fails.</para> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--optimise</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--optimise</option></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--optimise</option> reduces Nix store disk +space usage by finding identical files in the store and hard-linking +them to each other. It typically reduces the size of the store by +something like 25-35%. Only regular files and symlinks are +hard-linked in this manner. Files are considered identical when they +have the same NAR archive serialisation: that is, regular files must +have the same contents and permission (executable or non-executable), +and symlinks must have the same contents.</para> + +<para>After completion, or when the command is interrupted, a report +on the achieved savings is printed on standard error.</para> + +<para>Use <option>-vv</option> or <option>-vvv</option> to get some +progress indication.</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --optimise +hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' +<replaceable>...</replaceable> +541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; +there are 114486 files with equal contents out of 215894 files in total +</screen> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--read-log</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <group choice='req'> + <arg choice='plain'><option>--read-log</option></arg> + <arg choice='plain'><option>-l</option></arg> + </group> + <arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--read-log</option> prints the build log +of the specified store paths on standard output. The build log is +whatever the builder of a derivation wrote to standard output and +standard error. If a store path is not a derivation, the deriver of +the store path is used.</para> + +<para>Build logs are kept in +<filename>/nix/var/log/nix/drvs</filename>. However, there is no +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through +a substitute, then the log is unavailable.</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store -l $(which ktorrent) +building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 +unpacking sources +unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz +ktorrent-2.2.1/ +ktorrent-2.2.1/NEWS +<replaceable>...</replaceable> +</screen> + +</refsection> + + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--dump-db</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--dump-db</option></arg> + <arg rep='repeat'><replaceable>paths</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--dump-db</option> writes a dump of the +Nix database to standard output. It can be loaded into an empty Nix +store using <option>--load-db</option>. This is useful for making +backups and when migrating to different database schemas.</para> + +<para>By default, <option>--dump-db</option> will dump the entire Nix +database. When one or more store paths is passed, only the subset of +the Nix database for those store paths is dumped. As with +<option>--export</option>, the user is responsible for passing all the +store paths for a closure. See <option>--export</option> for an +example.</para> + +</refsection> + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--load-db</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--load-db</option></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--load-db</option> reads a dump of the Nix +database created by <option>--dump-db</option> from standard input and +loads it into the Nix database.</para> + +</refsection> + +</refsection> + + +<!--######################################################################--> + +<refsection><title>Operation <option>--print-env</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'><option>--print-env</option></arg> + <arg choice='plain'><replaceable>drvpath</replaceable></arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>The operation <option>--print-env</option> prints out the +environment of a derivation in a format that can be evaluated by a +shell. The command line arguments of the builder are placed in the +variable <envar>_args</envar>.</para> + +</refsection> + +<refsection><title>Example</title> + +<screen> +$ nix-store --print-env $(nix-instantiate '<nixpkgs>' -A firefox) +<replaceable>…</replaceable> +export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' +export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' +export system; system='x86_64-linux' +export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' +</screen> + +</refsection> + +</refsection> + + +<!--######################################################################--> + +<refsection xml:id='rsec-nix-store-generate-binary-cache-key'><title>Operation <option>--generate-binary-cache-key</option></title> + +<refsection> + <title>Synopsis</title> + <cmdsynopsis> + <command>nix-store</command> + <arg choice='plain'> + <option>--generate-binary-cache-key</option> + <option>key-name</option> + <option>secret-key-file</option> + <option>public-key-file</option> + </arg> + </cmdsynopsis> +</refsection> + +<refsection><title>Description</title> + +<para>This command generates an <link +xlink:href="http://ed25519.cr.yp.to/">Ed25519 key pair</link> that can +be used to create a signed binary cache. It takes three mandatory +parameters: + +<orderedlist> + + <listitem><para>A key name, such as + <literal>cache.example.org-1</literal>, that is used to look up keys + on the client when it verifies signatures. It can be anything, but + it’s suggested to use the host name of your cache + (e.g. <literal>cache.example.org</literal>) with a suffix denoting + the number of the key (to be incremented every time you need to + revoke a key).</para></listitem> + + <listitem><para>The file name where the secret key is to be + stored.</para></listitem> + + <listitem><para>The file name where the public key is to be + stored.</para></listitem> + +</orderedlist> + +</para> + +</refsection> + +</refsection> + + +<!--######################################################################--> + +<refsection condition="manpage"><title>Environment variables</title> + +<variablelist> + <xi:include href="env-common.xml#xmlns(db=http://docbook.org/ns/docbook)xpointer(//db:variablelist[@xml:id='env-common']/*)" /> +</variablelist> + +</refsection> + + +</refentry> diff --git a/third_party/nix/doc/manual/command-ref/opt-common-syn.xml b/third_party/nix/doc/manual/command-ref/opt-common-syn.xml new file mode 100644 index 000000000000..b610b54b9620 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/opt-common-syn.xml @@ -0,0 +1,64 @@ +<nop xmlns="http://docbook.org/ns/docbook"> + +<arg><option>--help</option></arg> +<arg><option>--version</option></arg> +<arg rep='repeat'> + <group choice='req'> + <arg choice='plain'><option>--verbose</option></arg> + <arg choice='plain'><option>-v</option></arg> + </group> +</arg> +<arg> + <arg choice='plain'><option>--quiet</option></arg> +</arg> +<arg> + <group choice='plain'> + <arg choice='plain'><option>--no-build-output</option></arg> + <arg choice='plain'><option>-Q</option></arg> + </group> +</arg> +<arg> + <group choice='req'> + <arg choice='plain'><option>--max-jobs</option></arg> + <arg choice='plain'><option>-j</option></arg> + </group> + <replaceable>number</replaceable> +</arg> +<arg> + <option>--cores</option> + <replaceable>number</replaceable> +</arg> +<arg> + <option>--max-silent-time</option> + <replaceable>number</replaceable> +</arg> +<arg> + <option>--timeout</option> + <replaceable>number</replaceable> +</arg> +<arg> + <group choice='plain'> + <arg choice='plain'><option>--keep-going</option></arg> + <arg choice='plain'><option>-k</option></arg> + </group> +</arg> +<arg> + <group choice='plain'> + <arg choice='plain'><option>--keep-failed</option></arg> + <arg choice='plain'><option>-K</option></arg> + </group> +</arg> +<arg><option>--fallback</option></arg> +<arg><option>--readonly-mode</option></arg> +<arg> + <option>-I</option> + <replaceable>path</replaceable> +</arg> +<arg> + <option>--option</option> + <replaceable>name</replaceable> + <replaceable>value</replaceable> +</arg> +<sbr /> + +</nop> diff --git a/third_party/nix/doc/manual/command-ref/opt-common.xml b/third_party/nix/doc/manual/command-ref/opt-common.xml new file mode 100644 index 000000000000..b8a2f260e8fe --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/opt-common.xml @@ -0,0 +1,366 @@ +<chapter xmlns="http://docbook.org/ns/docbook" xml:id="sec-common-options"> + +<title>Common Options</title> + + +<para>Most Nix commands accept the following command-line options:</para> + +<variablelist xml:id="opt-common"> + +<varlistentry><term><option>--help</option></term> + + <listitem><para>Prints out a summary of the command syntax and + exits.</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--version</option></term> + + <listitem><para>Prints out the Nix version number on standard output + and exits.</para></listitem> +</varlistentry> + + +<varlistentry><term><option>--verbose</option> / <option>-v</option></term> + + <listitem> + + <para>Increases the level of verbosity of diagnostic messages + printed on standard error. For each Nix operation, the information + printed on standard output is well-defined; any diagnostic + information is printed on standard error, never on standard + output.</para> + + <para>This option may be specified repeatedly. Currently, the + following verbosity levels exist:</para> + + <variablelist> + + <varlistentry><term>0</term> + <listitem><para>“Errors only”: only print messages + explaining why the Nix invocation failed.</para></listitem> + </varlistentry> + + <varlistentry><term>1</term> + <listitem><para>“Informational”: print + <emphasis>useful</emphasis> messages about what Nix is doing. + This is the default.</para></listitem> + </varlistentry> + + <varlistentry><term>2</term> + <listitem><para>“Talkative”: print more informational + messages.</para></listitem> + </varlistentry> + + <varlistentry><term>3</term> + <listitem><para>“Chatty”: print even more + informational messages.</para></listitem> + </varlistentry> + + <varlistentry><term>4</term> + <listitem><para>“Debug”: print debug + information.</para></listitem> + </varlistentry> + + <varlistentry><term>5</term> + <listitem><para>“Vomit”: print vast amounts of debug + information.</para></listitem> + </varlistentry> + + </variablelist> + + </listitem> + +</varlistentry> + + +<varlistentry><term><option>--quiet</option></term> + + <listitem> + + <para>Decreases the level of verbosity of diagnostic messages + printed on standard error. This is the inverse option to + <option>-v</option> / <option>--verbose</option>. + </para> + + <para>This option may be specified repeatedly. See the previous + verbosity levels list.</para> + + </listitem> + +</varlistentry> + + +<varlistentry><term><option>--no-build-output</option> / <option>-Q</option></term> + + <listitem><para>By default, output written by builders to standard + output and standard error is echoed to the Nix command's standard + error. This option suppresses this behaviour. Note that the + builder's standard output and error are always written to a log file + in + <filename><replaceable>prefix</replaceable>/nix/var/log/nix</filename>.</para></listitem> + +</varlistentry> + + +<varlistentry xml:id="opt-max-jobs"><term><option>--max-jobs</option> / <option>-j</option> +<replaceable>number</replaceable></term> + + <listitem> + + <para>Sets the maximum number of build jobs that Nix will + perform in parallel to the specified number. Specify + <literal>auto</literal> to use the number of CPUs in the system. + The default is specified by the <link + linkend='conf-max-jobs'><literal>max-jobs</literal></link> + configuration setting, which itself defaults to + <literal>1</literal>. A higher value is useful on SMP systems or to + exploit I/O latency.</para> + + <para> Setting it to <literal>0</literal> disallows building on the local + machine, which is useful when you want builds to happen only on remote + builders.</para> + + </listitem> + +</varlistentry> + + +<varlistentry xml:id="opt-cores"><term><option>--cores</option></term> + + <listitem><para>Sets the value of the <envar>NIX_BUILD_CORES</envar> + environment variable in the invocation of builders. Builders can + use this variable at their discretion to control the maximum amount + of parallelism. For instance, in Nixpkgs, if the derivation + attribute <varname>enableParallelBuilding</varname> is set to + <literal>true</literal>, the builder passes the + <option>-j<replaceable>N</replaceable></option> flag to GNU Make. + It defaults to the value of the <link + linkend='conf-cores'><literal>cores</literal></link> + configuration setting, if set, or <literal>1</literal> otherwise. + The value <literal>0</literal> means that the builder should use all + available CPU cores in the system.</para></listitem> + +</varlistentry> + + +<varlistentry xml:id="opt-max-silent-time"><term><option>--max-silent-time</option></term> + + <listitem><para>Sets the maximum number of seconds that a builder + can go without producing any data on standard output or standard + error. The default is specified by the <link + linkend='conf-max-silent-time'><literal>max-silent-time</literal></link> + configuration setting. <literal>0</literal> means no + time-out.</para></listitem> + +</varlistentry> + +<varlistentry xml:id="opt-timeout"><term><option>--timeout</option></term> + + <listitem><para>Sets the maximum number of seconds that a builder + can run. The default is specified by the <link + linkend='conf-timeout'><literal>timeout</literal></link> + configuration setting. <literal>0</literal> means no + timeout.</para></listitem> + +</varlistentry> + +<varlistentry><term><option>--keep-going</option> / <option>-k</option></term> + + <listitem><para>Keep going in case of failed builds, to the + greatest extent possible. That is, if building an input of some + derivation fails, Nix will still build the other inputs, but not the + derivation itself. Without this option, Nix stops if any build + fails (except for builds of substitutes), possibly killing builds in + progress (in case of parallel or distributed builds).</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--keep-failed</option> / <option>-K</option></term> + + <listitem><para>Specifies that in case of a build failure, the + temporary directory (usually in <filename>/tmp</filename>) in which + the build takes place should not be deleted. The path of the build + directory is printed as an informational message. + </para> + </listitem> +</varlistentry> + + +<varlistentry><term><option>--fallback</option></term> + + <listitem> + + <para>Whenever Nix attempts to build a derivation for which + substitutes are known for each output path, but realising the output + paths through the substitutes fails, fall back on building the + derivation.</para> + + <para>The most common scenario in which this is useful is when we + have registered substitutes in order to perform binary distribution + from, say, a network repository. If the repository is down, the + realisation of the derivation will fail. When this option is + specified, Nix will build the derivation instead. Thus, + installation from binaries falls back on installation from source. + This option is not the default since it is generally not desirable + for a transient failure in obtaining the substitutes to lead to a + full build from source (with the related consumption of + resources).</para> + + </listitem> + +</varlistentry> + +<varlistentry><term><option>--no-build-hook</option></term> + + <listitem> + + <para>Disables the build hook mechanism. This allows to ignore remote + builders if they are setup on the machine.</para> + + <para>It's useful in cases where the bandwidth between the client and the + remote builder is too low. In that case it can take more time to upload the + sources to the remote builder and fetch back the result than to do the + computation locally.</para> + + </listitem> + +</varlistentry> + + + +<varlistentry><term><option>--readonly-mode</option></term> + + <listitem><para>When this option is used, no attempt is made to open + the Nix database. Most Nix operations do need database access, so + those operations will fail.</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--arg</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term> + + <listitem><para>This option is accepted by + <command>nix-env</command>, <command>nix-instantiate</command> and + <command>nix-build</command>. When evaluating Nix expressions, the + expression evaluator will automatically try to call functions that + it encounters. It can automatically call functions for which every + argument has a <link linkend='ss-functions'>default value</link> + (e.g., <literal>{ <replaceable>argName</replaceable> ? + <replaceable>defaultValue</replaceable> }: + <replaceable>...</replaceable></literal>). With + <option>--arg</option>, you can also call functions that have + arguments without a default value (or override a default value). + That is, if the evaluator encounters a function with an argument + named <replaceable>name</replaceable>, it will call it with value + <replaceable>value</replaceable>.</para> + + <para>For instance, the top-level <literal>default.nix</literal> in + Nixpkgs is actually a function: + +<programlisting> +{ # The system (e.g., `i686-linux') for which to build the packages. + system ? builtins.currentSystem + <replaceable>...</replaceable> +}: <replaceable>...</replaceable></programlisting> + + So if you call this Nix expression (e.g., when you do + <literal>nix-env -i <replaceable>pkgname</replaceable></literal>), + the function will be called automatically using the value <link + linkend='builtin-currentSystem'><literal>builtins.currentSystem</literal></link> + for the <literal>system</literal> argument. You can override this + using <option>--arg</option>, e.g., <literal>nix-env -i + <replaceable>pkgname</replaceable> --arg system + \"i686-freebsd\"</literal>. (Note that since the argument is a Nix + string literal, you have to escape the quotes.)</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--argstr</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term> + + <listitem><para>This option is like <option>--arg</option>, only the + value is not a Nix expression but a string. So instead of + <literal>--arg system \"i686-linux\"</literal> (the outer quotes are + to keep the shell happy) you can say <literal>--argstr system + i686-linux</literal>.</para></listitem> + +</varlistentry> + + +<varlistentry xml:id="opt-attr"><term><option>--attr</option> / <option>-A</option> +<replaceable>attrPath</replaceable></term> + + <listitem><para>Select an attribute from the top-level Nix + expression being evaluated. (<command>nix-env</command>, + <command>nix-instantiate</command>, <command>nix-build</command> and + <command>nix-shell</command> only.) The <emphasis>attribute + path</emphasis> <replaceable>attrPath</replaceable> is a sequence of + attribute names separated by dots. For instance, given a top-level + Nix expression <replaceable>e</replaceable>, the attribute path + <literal>xorg.xorgserver</literal> would cause the expression + <literal><replaceable>e</replaceable>.xorg.xorgserver</literal> to + be used. See <link + linkend='refsec-nix-env-install-examples'><command>nix-env + --install</command></link> for some concrete examples.</para> + + <para>In addition to attribute names, you can also specify array + indices. For instance, the attribute path + <literal>foo.3.bar</literal> selects the <literal>bar</literal> + attribute of the fourth element of the array in the + <literal>foo</literal> attribute of the top-level + expression.</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--expr</option> / <option>-E</option></term> + + <listitem><para>Interpret the command line arguments as a list of + Nix expressions to be parsed and evaluated, rather than as a list + of file names of Nix expressions. + (<command>nix-instantiate</command>, <command>nix-build</command> + and <command>nix-shell</command> only.)</para></listitem> + +</varlistentry> + + +<varlistentry xml:id="opt-I"><term><option>-I</option> <replaceable>path</replaceable></term> + + <listitem><para>Add a path to the Nix expression search path. This + option may be given multiple times. See the <envar + linkend="env-NIX_PATH">NIX_PATH</envar> environment variable for + information on the semantics of the Nix search path. Paths added + through <option>-I</option> take precedence over + <envar>NIX_PATH</envar>.</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--option</option> <replaceable>name</replaceable> <replaceable>value</replaceable></term> + + <listitem><para>Set the Nix configuration option + <replaceable>name</replaceable> to <replaceable>value</replaceable>. + This overrides settings in the Nix configuration file (see + <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>).</para></listitem> + +</varlistentry> + + +<varlistentry><term><option>--repair</option></term> + + <listitem><para>Fix corrupted or missing store paths by + redownloading or rebuilding them. Note that this is slow because it + requires computing a cryptographic hash of the contents of every + path in the closure of the build. Also note the warning under + <command>nix-store --repair-path</command>.</para></listitem> + +</varlistentry> + + +</variablelist> + + +</chapter> diff --git a/third_party/nix/doc/manual/command-ref/opt-inst-syn.xml b/third_party/nix/doc/manual/command-ref/opt-inst-syn.xml new file mode 100644 index 000000000000..e8c3f1ec6f04 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/opt-inst-syn.xml @@ -0,0 +1,22 @@ +<nop xmlns="http://docbook.org/ns/docbook"> + + <arg> + <group choice='req'> + <arg choice='plain'><option>--prebuilt-only</option></arg> + <arg choice='plain'><option>-b</option></arg> + </group> + </arg> + + <arg> + <group choice='req'> + <arg choice='plain'><option>--attr</option></arg> + <arg choice='plain'><option>-A</option></arg> + </group> + </arg> + + <arg><option>--from-expression</option></arg> + <arg><option>-E</option></arg> + + <arg><option>--from-profile</option> <replaceable>path</replaceable></arg> + +</nop> diff --git a/third_party/nix/doc/manual/command-ref/utilities.xml b/third_party/nix/doc/manual/command-ref/utilities.xml new file mode 100644 index 000000000000..893f5b5b5260 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/utilities.xml @@ -0,0 +1,20 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='ch-utilities'> + +<title>Utilities</title> + +<para>This section lists utilities that you can use when you +work with Nix.</para> + +<xi:include href="nix-channel.xml" /> +<xi:include href="nix-collect-garbage.xml" /> +<xi:include href="nix-copy-closure.xml" /> +<xi:include href="nix-daemon.xml" /> +<xi:include href="nix-hash.xml" /> +<xi:include href="nix-instantiate.xml" /> +<xi:include href="nix-prefetch-url.xml" /> + +</chapter> diff --git a/third_party/nix/doc/manual/expressions/advanced-attributes.xml b/third_party/nix/doc/manual/expressions/advanced-attributes.xml new file mode 100644 index 000000000000..07b0d97d3f7d --- /dev/null +++ b/third_party/nix/doc/manual/expressions/advanced-attributes.xml @@ -0,0 +1,340 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-advanced-attributes"> + +<title>Advanced Attributes</title> + +<para>Derivations can declare some infrequently used optional +attributes.</para> + +<variablelist> + + <varlistentry><term><varname>allowedReferences</varname></term> + + <listitem><para>The optional attribute + <varname>allowedReferences</varname> specifies a list of legal + references (dependencies) of the output of the builder. For + example, + +<programlisting> +allowedReferences = []; +</programlisting> + + enforces that the output of a derivation cannot have any runtime + dependencies on its inputs. To allow an output to have a runtime + dependency on itself, use <literal>"out"</literal> as a list item. + This is used in NixOS to check that generated files such as + initial ramdisks for booting Linux don’t have accidental + dependencies on other paths in the Nix store.</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>allowedRequisites</varname></term> + + <listitem><para>This attribute is similar to + <varname>allowedReferences</varname>, but it specifies the legal + requisites of the whole closure, so all the dependencies + recursively. For example, + +<programlisting> +allowedRequisites = [ foobar ]; +</programlisting> + + enforces that the output of a derivation cannot have any other + runtime dependency than <varname>foobar</varname>, and in addition + it enforces that <varname>foobar</varname> itself doesn't + introduce any other dependency itself.</para></listitem> + + </varlistentry> + + <varlistentry><term><varname>disallowedReferences</varname></term> + + <listitem><para>The optional attribute + <varname>disallowedReferences</varname> specifies a list of illegal + references (dependencies) of the output of the builder. For + example, + +<programlisting> +disallowedReferences = [ foo ]; +</programlisting> + + enforces that the output of a derivation cannot have a direct runtime + dependencies on the derivation <varname>foo</varname>.</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>disallowedRequisites</varname></term> + + <listitem><para>This attribute is similar to + <varname>disallowedReferences</varname>, but it specifies illegal + requisites for the whole closure, so all the dependencies + recursively. For example, + +<programlisting> +disallowedRequisites = [ foobar ]; +</programlisting> + + enforces that the output of a derivation cannot have any + runtime dependency on <varname>foobar</varname> or any other derivation + depending recursively on <varname>foobar</varname>.</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>exportReferencesGraph</varname></term> + + <listitem><para>This attribute allows builders access to the + references graph of their inputs. The attribute is a list of + inputs in the Nix store whose references graph the builder needs + to know. The value of this attribute should be a list of pairs + <literal>[ <replaceable>name1</replaceable> + <replaceable>path1</replaceable> <replaceable>name2</replaceable> + <replaceable>path2</replaceable> <replaceable>...</replaceable> + ]</literal>. The references graph of each + <replaceable>pathN</replaceable> will be stored in a text file + <replaceable>nameN</replaceable> in the temporary build directory. + The text files have the format used by <command>nix-store + --register-validity</command> (with the deriver fields left + empty). For example, when the following derivation is built: + +<programlisting> +derivation { + ... + exportReferencesGraph = [ "libfoo-graph" libfoo ]; +}; +</programlisting> + + the references graph of <literal>libfoo</literal> is placed in the + file <filename>libfoo-graph</filename> in the temporary build + directory.</para> + + <para><varname>exportReferencesGraph</varname> is useful for + builders that want to do something with the closure of a store + path. Examples include the builders in NixOS that generate the + initial ramdisk for booting Linux (a <command>cpio</command> + archive containing the closure of the boot script) and the + ISO-9660 image for the installation CD (which is populated with a + Nix store containing the closure of a bootable NixOS + configuration).</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>impureEnvVars</varname></term> + + <listitem><para>This attribute allows you to specify a list of + environment variables that should be passed from the environment + of the calling user to the builder. Usually, the environment is + cleared completely when the builder is executed, but with this + attribute you can allow specific environment variables to be + passed unmodified. For example, <function>fetchurl</function> in + Nixpkgs has the line + +<programlisting> +impureEnvVars = [ "http_proxy" "https_proxy" <replaceable>...</replaceable> ]; +</programlisting> + + to make it use the proxy server configuration specified by the + user in the environment variables <envar>http_proxy</envar> and + friends.</para> + + <para>This attribute is only allowed in <link + linkend="fixed-output-drvs">fixed-output derivations</link>, where + impurities such as these are okay since (the hash of) the output + is known in advance. It is ignored for all other + derivations.</para> + + <warning><para><varname>impureEnvVars</varname> implementation takes + environment variables from the current builder process. When a daemon is + building its environmental variables are used. Without the daemon, the + environmental variables come from the environment of the + <command>nix-build</command>.</para></warning></listitem> + + </varlistentry> + + + <varlistentry xml:id="fixed-output-drvs"> + <term><varname>outputHash</varname></term> + <term><varname>outputHashAlgo</varname></term> + <term><varname>outputHashMode</varname></term> + + <listitem><para>These attributes declare that the derivation is a + so-called <emphasis>fixed-output derivation</emphasis>, which + means that a cryptographic hash of the output is already known in + advance. When the build of a fixed-output derivation finishes, + Nix computes the cryptographic hash of the output and compares it + to the hash declared with these attributes. If there is a + mismatch, the build fails.</para> + + <para>The rationale for fixed-output derivations is derivations + such as those produced by the <function>fetchurl</function> + function. This function downloads a file from a given URL. To + ensure that the downloaded file has not been modified, the caller + must also specify a cryptographic hash of the file. For example, + +<programlisting> +fetchurl { + url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; +} +</programlisting> + + It sometimes happens that the URL of the file changes, e.g., + because servers are reorganised or no longer available. We then + must update the call to <function>fetchurl</function>, e.g., + +<programlisting> +fetchurl { + url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; +} +</programlisting> + + If a <function>fetchurl</function> derivation was treated like a + normal derivation, the output paths of the derivation and + <emphasis>all derivations depending on it</emphasis> would change. + For instance, if we were to change the URL of the Glibc source + distribution in Nixpkgs (a package on which almost all other + packages depend) massive rebuilds would be needed. This is + unfortunate for a change which we know cannot have a real effect + as it propagates upwards through the dependency graph.</para> + + <para>For fixed-output derivations, on the other hand, the name of + the output path only depends on the <varname>outputHash*</varname> + and <varname>name</varname> attributes, while all other attributes + are ignored for the purpose of computing the output path. (The + <varname>name</varname> attribute is included because it is part + of the path.)</para> + + <para>As an example, here is the (simplified) Nix expression for + <varname>fetchurl</varname>: + +<programlisting> +{ stdenv, curl }: # The <command>curl</command> program is used for downloading. + +{ url, sha256 }: + +stdenv.mkDerivation { + name = baseNameOf (toString url); + builder = ./builder.sh; + buildInputs = [ curl ]; + + # This is a fixed-output derivation; the output must be a regular + # file with SHA256 hash <varname>sha256</varname>. + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = sha256; + + inherit url; +} +</programlisting> + + </para> + + <para>The <varname>outputHashAlgo</varname> attribute specifies + the hash algorithm used to compute the hash. It can currently be + <literal>"sha1"</literal>, <literal>"sha256"</literal> or + <literal>"sha512"</literal>.</para> + + <para>The <varname>outputHashMode</varname> attribute determines + how the hash is computed. It must be one of the following two + values: + + <variablelist> + + <varlistentry><term><literal>"flat"</literal></term> + + <listitem><para>The output must be a non-executable regular + file. If it isn’t, the build fails. The hash is simply + computed over the contents of that file (so it’s equal to what + Unix commands like <command>sha256sum</command> or + <command>sha1sum</command> produce).</para> + + <para>This is the default.</para></listitem> + + </varlistentry> + + <varlistentry><term><literal>"recursive"</literal></term> + + <listitem><para>The hash is computed over the NAR archive dump + of the output (i.e., the result of <link + linkend="refsec-nix-store-dump"><command>nix-store + --dump</command></link>). In this case, the output can be + anything, including a directory tree.</para></listitem> + + </varlistentry> + + </variablelist> + + </para> + + <para>The <varname>outputHash</varname> attribute, finally, must + be a string containing the hash in either hexadecimal or base-32 + notation. (See the <link + linkend="sec-nix-hash"><command>nix-hash</command> command</link> + for information about converting to and from base-32 + notation.)</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>passAsFile</varname></term> + + <listitem><para>A list of names of attributes that should be + passed via files rather than environment variables. For example, + if you have + + <programlisting> +passAsFile = ["big"]; +big = "a very long string"; + </programlisting> + + then when the builder runs, the environment variable + <envar>bigPath</envar> will contain the absolute path to a + temporary file containing <literal>a very long + string</literal>. That is, for any attribute + <replaceable>x</replaceable> listed in + <varname>passAsFile</varname>, Nix will pass an environment + variable <envar><replaceable>x</replaceable>Path</envar> holding + the path of the file containing the value of attribute + <replaceable>x</replaceable>. This is useful when you need to pass + large strings to a builder, since most operating systems impose a + limit on the size of the environment (typically, a few hundred + kilobyte).</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>preferLocalBuild</varname></term> + + <listitem><para>If this attribute is set to + <literal>true</literal> and <link + linkend="chap-distributed-builds">distributed building is + enabled</link>, then, if possible, the derivaton will be built + locally instead of forwarded to a remote machine. This is + appropriate for trivial builders where the cost of doing a + download or remote build would exceed the cost of building + locally.</para></listitem> + + </varlistentry> + + + <varlistentry><term><varname>allowSubstitutes</varname></term> + + <listitem><para>If this attribute is set to + <literal>false</literal>, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as + <function>writeText</function> in Nixpkgs) that are cheaper to + build than to substitute from a binary cache.</para></listitem> + + </varlistentry> + + +</variablelist> + +</section> diff --git a/third_party/nix/doc/manual/expressions/arguments-variables.xml b/third_party/nix/doc/manual/expressions/arguments-variables.xml new file mode 100644 index 000000000000..bf60cb7eef8a --- /dev/null +++ b/third_party/nix/doc/manual/expressions/arguments-variables.xml @@ -0,0 +1,121 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-arguments'> + +<title>Arguments and Variables</title> + +<example xml:id='ex-hello-composition'> + +<title>Composing GNU Hello +(<filename>all-packages.nix</filename>)</title> +<programlisting> +... + +rec { <co xml:id='ex-hello-composition-co-1' /> + + hello = import ../applications/misc/hello/ex-1 <co xml:id='ex-hello-composition-co-2' /> { <co xml:id='ex-hello-composition-co-3' /> + inherit fetchurl stdenv perl; + }; + + perl = import ../development/interpreters/perl { <co xml:id='ex-hello-composition-co-4' /> + inherit fetchurl stdenv; + }; + + fetchurl = import ../build-support/fetchurl { + inherit stdenv; ... + }; + + stdenv = ...; + +} +</programlisting> +</example> + +<para>The Nix expression in <xref linkend='ex-hello-nix' /> is a +function; it is missing some arguments that have to be filled in +somewhere. In the Nix Packages collection this is done in the file +<filename>pkgs/top-level/all-packages.nix</filename>, where all +Nix expressions for packages are imported and called with the +appropriate arguments. <xref linkend='ex-hello-composition' /> shows +some fragments of +<filename>all-packages.nix</filename>.</para> + +<calloutlist> + + <callout arearefs='ex-hello-composition-co-1'> + + <para>This file defines a set of attributes, all of which are + concrete derivations (i.e., not functions). In fact, we define a + <emphasis>mutually recursive</emphasis> set of attributes. That + is, the attributes can refer to each other. This is precisely + what we want since we want to <quote>plug</quote> the + various packages into each other.</para> + + </callout> + + <callout arearefs='ex-hello-composition-co-2'> + + <para>Here we <emphasis>import</emphasis> the Nix expression for + GNU Hello. The import operation just loads and returns the + specified Nix expression. In fact, we could just have put the + contents of <xref linkend='ex-hello-nix' /> in + <filename>all-packages.nix</filename> at this point. That + would be completely equivalent, but it would make the file rather + bulky.</para> + + <para>Note that we refer to + <filename>../applications/misc/hello/ex-1</filename>, not + <filename>../applications/misc/hello/ex-1/default.nix</filename>. + When you try to import a directory, Nix automatically appends + <filename>/default.nix</filename> to the file name.</para> + + </callout> + + <callout arearefs='ex-hello-composition-co-3'> + + <para>This is where the actual composition takes place. Here we + <emphasis>call</emphasis> the function imported from + <filename>../applications/misc/hello/ex-1</filename> with a set + containing the things that the function expects, namely + <varname>fetchurl</varname>, <varname>stdenv</varname>, and + <varname>perl</varname>. We use inherit again to use the + attributes defined in the surrounding scope (we could also have + written <literal>fetchurl = fetchurl;</literal>, etc.).</para> + + <para>The result of this function call is an actual derivation + that can be built by Nix (since when we fill in the arguments of + the function, what we get is its body, which is the call to + <varname>stdenv.mkDerivation</varname> in <xref + linkend='ex-hello-nix' />).</para> + + <note><para>Nixpkgs has a convenience function + <function>callPackage</function> that imports and calls a + function, filling in any missing arguments by passing the + corresponding attribute from the Nixpkgs set, like this: + +<programlisting> +hello = callPackage ../applications/misc/hello/ex-1 { }; +</programlisting> + + If necessary, you can set or override arguments: + +<programlisting> +hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; }; +</programlisting> + + </para></note> + + </callout> + + <callout arearefs='ex-hello-composition-co-4'> + + <para>Likewise, we have to instantiate Perl, + <varname>fetchurl</varname>, and the standard environment.</para> + + </callout> + +</calloutlist> + +</section> \ No newline at end of file diff --git a/third_party/nix/doc/manual/expressions/build-script.xml b/third_party/nix/doc/manual/expressions/build-script.xml new file mode 100644 index 000000000000..7bad8f808d58 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/build-script.xml @@ -0,0 +1,119 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-build-script'> + +<title>Build Script</title> + +<example xml:id='ex-hello-builder'><title>Build script for GNU Hello +(<filename>builder.sh</filename>)</title> +<programlisting> +source $stdenv/setup <co xml:id='ex-hello-builder-co-1' /> + +PATH=$perl/bin:$PATH <co xml:id='ex-hello-builder-co-2' /> + +tar xvfz $src <co xml:id='ex-hello-builder-co-3' /> +cd hello-* +./configure --prefix=$out <co xml:id='ex-hello-builder-co-4' /> +make <co xml:id='ex-hello-builder-co-5' /> +make install</programlisting> +</example> + +<para><xref linkend='ex-hello-builder' /> shows the builder referenced +from Hello's Nix expression (stored in +<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>). +The builder can actually be made a lot shorter by using the +<emphasis>generic builder</emphasis> functions provided by +<varname>stdenv</varname>, but here we write out the build steps to +elucidate what a builder does. It performs the following +steps:</para> + +<calloutlist> + + <callout arearefs='ex-hello-builder-co-1'> + + <para>When Nix runs a builder, it initially completely clears the + environment (except for the attributes declared in the + derivation). For instance, the <envar>PATH</envar> variable is + empty<footnote><para>Actually, it's initialised to + <filename>/path-not-set</filename> to prevent Bash from setting it + to a default value.</para></footnote>. This is done to prevent + undeclared inputs from being used in the build process. If for + example the <envar>PATH</envar> contained + <filename>/usr/bin</filename>, then you might accidentally use + <filename>/usr/bin/gcc</filename>.</para> + + <para>So the first step is to set up the environment. This is + done by calling the <filename>setup</filename> script of the + standard environment. The environment variable + <envar>stdenv</envar> points to the location of the standard + environment being used. (It wasn't specified explicitly as an + attribute in <xref linkend='ex-hello-nix' />, but + <varname>mkDerivation</varname> adds it automatically.)</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-2'> + + <para>Since Hello needs Perl, we have to make sure that Perl is in + the <envar>PATH</envar>. The <envar>perl</envar> environment + variable points to the location of the Perl package (since it + was passed in as an attribute to the derivation), so + <filename><replaceable>$perl</replaceable>/bin</filename> is the + directory containing the Perl interpreter.</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-3'> + + <para>Now we have to unpack the sources. The + <varname>src</varname> attribute was bound to the result of + fetching the Hello source tarball from the network, so the + <envar>src</envar> environment variable points to the location in + the Nix store to which the tarball was downloaded. After + unpacking, we <command>cd</command> to the resulting source + directory.</para> + + <para>The whole build is performed in a temporary directory + created in <varname>/tmp</varname>, by the way. This directory is + removed after the builder finishes, so there is no need to clean + up the sources afterwards. Also, the temporary directory is + always newly created, so you don't have to worry about files from + previous builds interfering with the current build.</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-4'> + + <para>GNU Hello is a typical Autoconf-based package, so we first + have to run its <filename>configure</filename> script. In Nix + every package is stored in a separate location in the Nix store, + for instance + <filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>. + Nix computes this path by cryptographically hashing all attributes + of the derivation. The path is passed to the builder through the + <envar>out</envar> environment variable. So here we give + <filename>configure</filename> the parameter + <literal>--prefix=$out</literal> to cause Hello to be installed in + the expected location.</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-5'> + + <para>Finally we build Hello (<literal>make</literal>) and install + it into the location specified by <envar>out</envar> + (<literal>make install</literal>).</para> + + </callout> + +</calloutlist> + +<para>If you are wondering about the absence of error checking on the +result of various commands called in the builder: this is because the +shell script is evaluated with Bash's <option>-e</option> option, +which causes the script to be aborted if any command fails without an +error check.</para> + +</section> \ No newline at end of file diff --git a/third_party/nix/doc/manual/expressions/builder-syntax.xml b/third_party/nix/doc/manual/expressions/builder-syntax.xml new file mode 100644 index 000000000000..e51bade44e59 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/builder-syntax.xml @@ -0,0 +1,119 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-builder-syntax'> + +<title>Builder Syntax</title> + +<example xml:id='ex-hello-builder'><title>Build script for GNU Hello +(<filename>builder.sh</filename>)</title> +<programlisting> +source $stdenv/setup <co xml:id='ex-hello-builder-co-1' /> + +PATH=$perl/bin:$PATH <co xml:id='ex-hello-builder-co-2' /> + +tar xvfz $src <co xml:id='ex-hello-builder-co-3' /> +cd hello-* +./configure --prefix=$out <co xml:id='ex-hello-builder-co-4' /> +make <co xml:id='ex-hello-builder-co-5' /> +make install</programlisting> +</example> + +<para><xref linkend='ex-hello-builder' /> shows the builder referenced +from Hello's Nix expression (stored in +<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>). +The builder can actually be made a lot shorter by using the +<emphasis>generic builder</emphasis> functions provided by +<varname>stdenv</varname>, but here we write out the build steps to +elucidate what a builder does. It performs the following +steps:</para> + +<calloutlist> + + <callout arearefs='ex-hello-builder-co-1'> + + <para>When Nix runs a builder, it initially completely clears the + environment (except for the attributes declared in the + derivation). For instance, the <envar>PATH</envar> variable is + empty<footnote><para>Actually, it's initialised to + <filename>/path-not-set</filename> to prevent Bash from setting it + to a default value.</para></footnote>. This is done to prevent + undeclared inputs from being used in the build process. If for + example the <envar>PATH</envar> contained + <filename>/usr/bin</filename>, then you might accidentally use + <filename>/usr/bin/gcc</filename>.</para> + + <para>So the first step is to set up the environment. This is + done by calling the <filename>setup</filename> script of the + standard environment. The environment variable + <envar>stdenv</envar> points to the location of the standard + environment being used. (It wasn't specified explicitly as an + attribute in <xref linkend='ex-hello-nix' />, but + <varname>mkDerivation</varname> adds it automatically.)</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-2'> + + <para>Since Hello needs Perl, we have to make sure that Perl is in + the <envar>PATH</envar>. The <envar>perl</envar> environment + variable points to the location of the Perl package (since it + was passed in as an attribute to the derivation), so + <filename><replaceable>$perl</replaceable>/bin</filename> is the + directory containing the Perl interpreter.</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-3'> + + <para>Now we have to unpack the sources. The + <varname>src</varname> attribute was bound to the result of + fetching the Hello source tarball from the network, so the + <envar>src</envar> environment variable points to the location in + the Nix store to which the tarball was downloaded. After + unpacking, we <command>cd</command> to the resulting source + directory.</para> + + <para>The whole build is performed in a temporary directory + created in <varname>/tmp</varname>, by the way. This directory is + removed after the builder finishes, so there is no need to clean + up the sources afterwards. Also, the temporary directory is + always newly created, so you don't have to worry about files from + previous builds interfering with the current build.</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-4'> + + <para>GNU Hello is a typical Autoconf-based package, so we first + have to run its <filename>configure</filename> script. In Nix + every package is stored in a separate location in the Nix store, + for instance + <filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>. + Nix computes this path by cryptographically hashing all attributes + of the derivation. The path is passed to the builder through the + <envar>out</envar> environment variable. So here we give + <filename>configure</filename> the parameter + <literal>--prefix=$out</literal> to cause Hello to be installed in + the expected location.</para> + + </callout> + + <callout arearefs='ex-hello-builder-co-5'> + + <para>Finally we build Hello (<literal>make</literal>) and install + it into the location specified by <envar>out</envar> + (<literal>make install</literal>).</para> + + </callout> + +</calloutlist> + +<para>If you are wondering about the absence of error checking on the +result of various commands called in the builder: this is because the +shell script is evaluated with Bash's <option>-e</option> option, +which causes the script to be aborted if any command fails without an +error check.</para> + +</section> \ No newline at end of file diff --git a/third_party/nix/doc/manual/expressions/builtins.xml b/third_party/nix/doc/manual/expressions/builtins.xml new file mode 100644 index 000000000000..394e1fc32c95 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/builtins.xml @@ -0,0 +1,1658 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='ssec-builtins'> + +<title>Built-in Functions</title> + +<para>This section lists the functions and constants built into the +Nix expression evaluator. (The built-in function +<function>derivation</function> is discussed above.) Some built-ins, +such as <function>derivation</function>, are always in scope of every +Nix expression; you can just access them right away. But to prevent +polluting the namespace too much, most built-ins are not in scope. +Instead, you can access them through the <varname>builtins</varname> +built-in value, which is a set that contains all built-in functions +and values. For instance, <function>derivation</function> is also +available as <function>builtins.derivation</function>.</para> + + +<variablelist> + + + <varlistentry xml:id='builtin-abort'> + <term><function>abort</function> <replaceable>s</replaceable></term> + <term><function>builtins.abort</function> <replaceable>s</replaceable></term> + + <listitem><para>Abort Nix expression evaluation, print error + message <replaceable>s</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-add'> + <term><function>builtins.add</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable> + </term> + + <listitem><para>Return the sum of the numbers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-all'> + <term><function>builtins.all</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Return <literal>true</literal> if the function + <replaceable>pred</replaceable> returns <literal>true</literal> + for all elements of <replaceable>list</replaceable>, + and <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-any'> + <term><function>builtins.any</function> + <replaceable>pred</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Return <literal>true</literal> if the function + <replaceable>pred</replaceable> returns <literal>true</literal> + for at least one element of <replaceable>list</replaceable>, + and <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-attrNames'> + <term><function>builtins.attrNames</function> + <replaceable>set</replaceable></term> + + <listitem><para>Return the names of the attributes in the set + <replaceable>set</replaceable> in an alphabetically sorted list. For instance, + <literal>builtins.attrNames { y = 1; x = "foo"; }</literal> + evaluates to <literal>[ "x" "y" ]</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-attrValues'> + <term><function>builtins.attrValues</function> + <replaceable>set</replaceable></term> + + <listitem><para>Return the values of the attributes in the set + <replaceable>set</replaceable> in the order corresponding to the + sorted attribute names.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-baseNameOf'> + <term><function>baseNameOf</function> <replaceable>s</replaceable></term> + + <listitem><para>Return the <emphasis>base name</emphasis> of the + string <replaceable>s</replaceable>, that is, everything following + the final slash in the string. This is similar to the GNU + <command>basename</command> command.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-bitAnd'> + <term><function>builtins.bitAnd</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise AND of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-bitOr'> + <term><function>builtins.bitOr</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise OR of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-bitXor'> + <term><function>builtins.bitXor</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the bitwise XOR of the integers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-builtins'> + <term><varname>builtins</varname></term> + + <listitem><para>The set <varname>builtins</varname> contains all + the built-in functions and values. You can use + <varname>builtins</varname> to test for the availability of + features in the Nix installation, e.g., + +<programlisting> +if builtins ? getEnv then builtins.getEnv "PATH" else ""</programlisting> + + This allows a Nix expression to fall back gracefully on older Nix + installations that don’t have the desired built-in + function.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-compareVersions'> + <term><function>builtins.compareVersions</function> + <replaceable>s1</replaceable> <replaceable>s2</replaceable></term> + + <listitem><para>Compare two strings representing versions and + return <literal>-1</literal> if version + <replaceable>s1</replaceable> is older than version + <replaceable>s2</replaceable>, <literal>0</literal> if they are + the same, and <literal>1</literal> if + <replaceable>s1</replaceable> is newer than + <replaceable>s2</replaceable>. The version comparison algorithm + is the same as the one used by <link + linkend="ssec-version-comparisons"><command>nix-env + -u</command></link>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-concatLists'> + <term><function>builtins.concatLists</function> + <replaceable>lists</replaceable></term> + + <listitem><para>Concatenate a list of lists into a single + list.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-concatStringsSep'> + <term><function>builtins.concatStringsSep</function> + <replaceable>separator</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Concatenate a list of strings with a separator + between each element, e.g. <literal>concatStringsSep "/" + ["usr" "local" "bin"] == "usr/local/bin"</literal></para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-currentSystem'> + <term><varname>builtins.currentSystem</varname></term> + + <listitem><para>The built-in value <varname>currentSystem</varname> + evaluates to the Nix platform identifier for the Nix installation + on which the expression is being evaluated, such as + <literal>"i686-linux"</literal> or + <literal>"x86_64-darwin"</literal>.</para></listitem> + + </varlistentry> + + + <!-- + <varlistentry><term><function>currentTime</function></term> + + <listitem><para>The built-in value <varname>currentTime</varname> + returns the current system time in seconds since 00:00:00 1/1/1970 + UTC. Due to the evaluation model of Nix expressions + (<emphasis>maximal laziness</emphasis>), it always yields the same + value within an execution of Nix.</para></listitem> + + </varlistentry> + --> + + + <!-- + <varlistentry><term><function>dependencyClosure</function></term> + + <listitem><para>TODO</para></listitem> + + </varlistentry> + --> + + + <varlistentry xml:id='builtin-deepSeq'> + <term><function>builtins.deepSeq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>This is like <literal>seq + <replaceable>e1</replaceable> + <replaceable>e2</replaceable></literal>, except that + <replaceable>e1</replaceable> is evaluated + <emphasis>deeply</emphasis>: if it’s a list or set, its elements + or attributes are also evaluated recursively.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-derivation'> + <term><function>derivation</function> + <replaceable>attrs</replaceable></term> + <term><function>builtins.derivation</function> + <replaceable>attrs</replaceable></term> + + <listitem><para><function>derivation</function> is described in + <xref linkend='ssec-derivation' />.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-dirOf'> + <term><function>dirOf</function> <replaceable>s</replaceable></term> + <term><function>builtins.dirOf</function> <replaceable>s</replaceable></term> + + <listitem><para>Return the directory part of the string + <replaceable>s</replaceable>, that is, everything before the final + slash in the string. This is similar to the GNU + <command>dirname</command> command.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-div'> + <term><function>builtins.div</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the quotient of the numbers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-elem'> + <term><function>builtins.elem</function> + <replaceable>x</replaceable> <replaceable>xs</replaceable></term> + + <listitem><para>Return <literal>true</literal> if a value equal to + <replaceable>x</replaceable> occurs in the list + <replaceable>xs</replaceable>, and <literal>false</literal> + otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-elemAt'> + <term><function>builtins.elemAt</function> + <replaceable>xs</replaceable> <replaceable>n</replaceable></term> + + <listitem><para>Return element <replaceable>n</replaceable> from + the list <replaceable>xs</replaceable>. Elements are counted + starting from 0. A fatal error occurs if the index is out of + bounds.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-fetchurl'> + <term><function>builtins.fetchurl</function> + <replaceable>url</replaceable></term> + + <listitem><para>Download the specified URL and return the path of + the downloaded file. This function is not available if <link + linkend="conf-restrict-eval">restricted evaluation mode</link> is + enabled.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-fetchTarball'> + <term><function>fetchTarball</function> + <replaceable>url</replaceable></term> + <term><function>builtins.fetchTarball</function> + <replaceable>url</replaceable></term> + + <listitem><para>Download the specified URL, unpack it and return + the path of the unpacked tree. The file must be a tape archive + (<filename>.tar</filename>) compressed with + <literal>gzip</literal>, <literal>bzip2</literal> or + <literal>xz</literal>. The top-level path component of the files + in the tarball is removed, so it is best if the tarball contains a + single directory at top level. The typical use of the function is + to obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. + +<programlisting> +with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; + +stdenv.mkDerivation { … } +</programlisting> + </para> + + <para>The fetched tarball is cached for a certain amount of time + (1 hour by default) in <filename>~/.cache/nix/tarballs/</filename>. + You can change the cache timeout either on the command line with + <option>--option tarball-ttl <replaceable>number of seconds</replaceable></option> or + in the Nix configuration file with this option: + <literal><xref linkend="conf-tarball-ttl" /> <replaceable>number of seconds to cache</replaceable></literal>. + </para> + + <para>Note that when obtaining the hash with <varname>nix-prefetch-url + </varname> the option <varname>--unpack</varname> is required. + </para> + + <para>This function can also verify the contents against a hash. + In that case, the function takes a set instead of a URL. The set + requires the attribute <varname>url</varname> and the attribute + <varname>sha256</varname>, e.g. + +<programlisting> +with import (fetchTarball { + url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz; + sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; +}) {}; + +stdenv.mkDerivation { … } +</programlisting> + + </para> + + <para>This function is not available if <link + linkend="conf-restrict-eval">restricted evaluation mode</link> is + enabled.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-fetchGit'> + <term> + <function>builtins.fetchGit</function> + <replaceable>args</replaceable> + </term> + + <listitem> + <para> + Fetch a path from git. <replaceable>args</replaceable> can be + a URL, in which case the HEAD of the repo at that URL is + fetched. Otherwise, it can be an attribute with the following + attributes (all except <varname>url</varname> optional): + </para> + + <variablelist> + <varlistentry> + <term>url</term> + <listitem> + <para> + The URL of the repo. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>name</term> + <listitem> + <para> + The name of the directory the repo should be exported to + in the store. Defaults to the basename of the URL. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>rev</term> + <listitem> + <para> + The git revision to fetch. Defaults to the tip of + <varname>ref</varname>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>ref</term> + <listitem> + <para> + The git ref to look for the requested revision under. + This is often a branch or tag name. Defaults to + <literal>HEAD</literal>. + </para> + + <para> + By default, the <varname>ref</varname> value is prefixed + with <literal>refs/heads/</literal>. As of Nix 2.3.0 + Nix will not prefix <literal>refs/heads/</literal> if + <varname>ref</varname> starts with <literal>refs/</literal>. + </para> + </listitem> + </varlistentry> + </variablelist> + + <example> + <title>Fetching a private repository over SSH</title> + <programlisting>builtins.fetchGit { + url = "git@github.com:my-secret/repository.git"; + ref = "master"; + rev = "adab8b916a45068c044658c4158d81878f9ed1c3"; +}</programlisting> + </example> + + <example> + <title>Fetching an arbitrary ref</title> + <programlisting>builtins.fetchGit { + url = "https://github.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +}</programlisting> + </example> + + <example> + <title>Fetching a repository's specific commit on an arbitrary branch</title> + <para> + If the revision you're looking for is in the default branch + of the git repository you don't strictly need to specify + the branch name in the <varname>ref</varname> attribute. + </para> + <para> + However, if the revision you're looking for is in a future + branch for the non-default branch you will need to specify + the the <varname>ref</varname> attribute as well. + </para> + <programlisting>builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + ref = "1.11-maintenance"; +}</programlisting> + <note> + <para> + It is nice to always specify the branch which a revision + belongs to. Without the branch being specified, the + fetcher might fail if the default branch changes. + Additionally, it can be confusing to try a commit from a + non-default branch and see the fetch fail. If the branch + is specified the fault is much more obvious. + </para> + </note> + </example> + + <example> + <title>Fetching a repository's specific commit on the default branch</title> + <para> + If the revision you're looking for is in the default branch + of the git repository you may omit the + <varname>ref</varname> attribute. + </para> + <programlisting>builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; +}</programlisting> + </example> + + <example> + <title>Fetching a tag</title> + <programlisting>builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + ref = "refs/tags/1.9"; +}</programlisting> + </example> + + <example> + <title>Fetching the latest version of a remote branch</title> + <para> + <function>builtins.fetchGit</function> can behave impurely + fetch the latest version of a remote branch. + </para> + <note><para>Nix will refetch the branch in accordance to + <xref linkend="conf-tarball-ttl" />.</para></note> + <note><para>This behavior is disabled in + <emphasis>Pure evaluation mode</emphasis>.</para></note> + <programlisting>builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + ref = "master"; +}</programlisting> + </example> + </listitem> + </varlistentry> + + + <varlistentry><term><function>builtins.filter</function> + <replaceable>f</replaceable> <replaceable>xs</replaceable></term> + + <listitem><para>Return a list consisting of the elements of + <replaceable>xs</replaceable> for which the function + <replaceable>f</replaceable> returns + <literal>true</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-filterSource'> + <term><function>builtins.filterSource</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem> + + <para>This function allows you to copy sources into the Nix + store while filtering certain files. For instance, suppose that + you want to use the directory <filename>source-dir</filename> as + an input to a Nix expression, e.g. + +<programlisting> +stdenv.mkDerivation { + ... + src = ./source-dir; +} +</programlisting> + + However, if <filename>source-dir</filename> is a Subversion + working copy, then all those annoying <filename>.svn</filename> + subdirectories will also be copied to the store. Worse, the + contents of those directories may change a lot, causing lots of + spurious rebuilds. With <function>filterSource</function> you + can filter out the <filename>.svn</filename> directories: + +<programlisting> + src = builtins.filterSource + (path: type: type != "directory" || baseNameOf path != ".svn") + ./source-dir; +</programlisting> + + </para> + + <para>Thus, the first argument <replaceable>e1</replaceable> + must be a predicate function that is called for each regular + file, directory or symlink in the source tree + <replaceable>e2</replaceable>. If the function returns + <literal>true</literal>, the file is copied to the Nix store, + otherwise it is omitted. The function is called with two + arguments. The first is the full path of the file. The second + is a string that identifies the type of the file, which is + either <literal>"regular"</literal>, + <literal>"directory"</literal>, <literal>"symlink"</literal> or + <literal>"unknown"</literal> (for other kinds of files such as + device nodes or fifos — but note that those cannot be copied to + the Nix store, so if the predicate returns + <literal>true</literal> for them, the copy will fail). If you + exclude a directory, the entire corresponding subtree of + <replaceable>e2</replaceable> will be excluded.</para> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-foldl-prime'> + <term><function>builtins.foldl’</function> + <replaceable>op</replaceable> <replaceable>nul</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Reduce a list by applying a binary operator, from + left to right, e.g. <literal>foldl’ op nul [x0 x1 x2 ...] = op (op + (op nul x0) x1) x2) ...</literal>. The operator is applied + strictly, i.e., its arguments are evaluated first. For example, + <literal>foldl’ (x: y: x + y) 0 [1 2 3]</literal> evaluates to + 6.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-functionArgs'> + <term><function>builtins.functionArgs</function> + <replaceable>f</replaceable></term> + + <listitem><para> + Return a set containing the names of the formal arguments expected + by the function <replaceable>f</replaceable>. + The value of each attribute is a Boolean denoting whether the corresponding + argument has a default value. For instance, + <literal>functionArgs ({ x, y ? 123}: ...) = { x = false; y = true; }</literal>. + </para> + + <para>"Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + <literal>functionArgs (x: ...) = { }</literal>. + </para></listitem> + </varlistentry> + + + <varlistentry xml:id='builtin-fromJSON'> + <term><function>builtins.fromJSON</function> <replaceable>e</replaceable></term> + + <listitem><para>Convert a JSON string to a Nix + value. For example, + +<programlisting> +builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' +</programlisting> + + returns the value <literal>{ x = [ 1 2 3 ]; y = null; + }</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-genList'> + <term><function>builtins.genList</function> + <replaceable>generator</replaceable> <replaceable>length</replaceable></term> + + <listitem><para>Generate list of size + <replaceable>length</replaceable>, with each element + <replaceable>i</replaceable> equal to the value returned by + <replaceable>generator</replaceable> <literal>i</literal>. For + example, + +<programlisting> +builtins.genList (x: x * x) 5 +</programlisting> + + returns the list <literal>[ 0 1 4 9 16 ]</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-getAttr'> + <term><function>builtins.getAttr</function> + <replaceable>s</replaceable> <replaceable>set</replaceable></term> + + <listitem><para><function>getAttr</function> returns the attribute + named <replaceable>s</replaceable> from + <replaceable>set</replaceable>. Evaluation aborts if the + attribute doesn’t exist. This is a dynamic version of the + <literal>.</literal> operator, since <replaceable>s</replaceable> + is an expression rather than an identifier.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-getEnv'> + <term><function>builtins.getEnv</function> + <replaceable>s</replaceable></term> + + <listitem><para><function>getEnv</function> returns the value of + the environment variable <replaceable>s</replaceable>, or an empty + string if the variable doesn’t exist. This function should be + used with care, as it can introduce all sorts of nasty environment + dependencies in your Nix expression.</para> + + <para><function>getEnv</function> is used in Nix Packages to + locate the file <filename>~/.nixpkgs/config.nix</filename>, which + contains user-local settings for Nix Packages. (That is, it does + a <literal>getEnv "HOME"</literal> to locate the user’s home + directory.)</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-hasAttr'> + <term><function>builtins.hasAttr</function> + <replaceable>s</replaceable> <replaceable>set</replaceable></term> + + <listitem><para><function>hasAttr</function> returns + <literal>true</literal> if <replaceable>set</replaceable> has an + attribute named <replaceable>s</replaceable>, and + <literal>false</literal> otherwise. This is a dynamic version of + the <literal>?</literal> operator, since + <replaceable>s</replaceable> is an expression rather than an + identifier.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-hashString'> + <term><function>builtins.hashString</function> + <replaceable>type</replaceable> <replaceable>s</replaceable></term> + + <listitem><para>Return a base-16 representation of the + cryptographic hash of string <replaceable>s</replaceable>. The + hash algorithm specified by <replaceable>type</replaceable> must + be one of <literal>"md5"</literal>, <literal>"sha1"</literal>, + <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-hashFile'> + <term><function>builtins.hashFile</function> + <replaceable>type</replaceable> <replaceable>p</replaceable></term> + + <listitem><para>Return a base-16 representation of the + cryptographic hash of the file at path <replaceable>p</replaceable>. The + hash algorithm specified by <replaceable>type</replaceable> must + be one of <literal>"md5"</literal>, <literal>"sha1"</literal>, + <literal>"sha256"</literal> or <literal>"sha512"</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-head'> + <term><function>builtins.head</function> + <replaceable>list</replaceable></term> + + <listitem><para>Return the first element of a list; abort + evaluation if the argument isn’t a list or is an empty list. You + can test whether a list is empty by comparing it with + <literal>[]</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-import'> + <term><function>import</function> + <replaceable>path</replaceable></term> + <term><function>builtins.import</function> + <replaceable>path</replaceable></term> + + <listitem><para>Load, parse and return the Nix expression in the + file <replaceable>path</replaceable>. If <replaceable>path + </replaceable> is a directory, the file <filename>default.nix + </filename> in that directory is loaded. Evaluation aborts if the + file doesn’t exist or contains an incorrect Nix expression. + <function>import</function> implements Nix’s module system: you + can put any Nix expression (such as a set or a function) in a + separate file, and use it from Nix expressions in other + files.</para> + + <note><para>Unlike some languages, <function>import</function> is a regular + function in Nix. Paths using the angle bracket syntax (e.g., <function> + import</function> <replaceable><foo></replaceable>) are normal path + values (see <xref linkend='ssec-values' />).</para></note> + + <para>A Nix expression loaded by <function>import</function> must + not contain any <emphasis>free variables</emphasis> (identifiers + that are not defined in the Nix expression itself and are not + built-in). Therefore, it cannot refer to variables that are in + scope at the call site. For instance, if you have a calling + expression + +<programlisting> +rec { + x = 123; + y = import ./foo.nix; +}</programlisting> + + then the following <filename>foo.nix</filename> will give an + error: + +<programlisting> +x + 456</programlisting> + + since <varname>x</varname> is not in scope in + <filename>foo.nix</filename>. If you want <varname>x</varname> + to be available in <filename>foo.nix</filename>, you should pass + it as a function argument: + +<programlisting> +rec { + x = 123; + y = import ./foo.nix x; +}</programlisting> + + and + +<programlisting> +x: x + 456</programlisting> + + (The function argument doesn’t have to be called + <varname>x</varname> in <filename>foo.nix</filename>; any name + would work.)</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-intersectAttrs'> + <term><function>builtins.intersectAttrs</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return a set consisting of the attributes in the + set <replaceable>e2</replaceable> that also exist in the set + <replaceable>e1</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isAttrs'> + <term><function>builtins.isAttrs</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a set, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isList'> + <term><function>builtins.isList</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a list, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isFunction'><term><function>builtins.isFunction</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a function, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isString'> + <term><function>builtins.isString</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a string, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isInt'> + <term><function>builtins.isInt</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to an int, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isFloat'> + <term><function>builtins.isFloat</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a float, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-isBool'> + <term><function>builtins.isBool</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a bool, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + <varlistentry><term><function>builtins.isPath</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to a path, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-isNull'> + <term><function>isNull</function> + <replaceable>e</replaceable></term> + <term><function>builtins.isNull</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return <literal>true</literal> if + <replaceable>e</replaceable> evaluates to <literal>null</literal>, + and <literal>false</literal> otherwise.</para> + + <warning><para>This function is <emphasis>deprecated</emphasis>; + just write <literal>e == null</literal> instead.</para></warning> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-length'> + <term><function>builtins.length</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return the length of the list + <replaceable>e</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-lessThan'> + <term><function>builtins.lessThan</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return <literal>true</literal> if the number + <replaceable>e1</replaceable> is less than the number + <replaceable>e2</replaceable>, and <literal>false</literal> + otherwise. Evaluation aborts if either + <replaceable>e1</replaceable> or <replaceable>e2</replaceable> + does not evaluate to a number.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-listToAttrs'> + <term><function>builtins.listToAttrs</function> + <replaceable>e</replaceable></term> + + <listitem><para>Construct a set from a list specifying the names + and values of each attribute. Each element of the list should be + a set consisting of a string-valued attribute + <varname>name</varname> specifying the name of the attribute, and + an attribute <varname>value</varname> specifying its value. + Example: + +<programlisting> +builtins.listToAttrs + [ { name = "foo"; value = 123; } + { name = "bar"; value = 456; } + ] +</programlisting> + + evaluates to + +<programlisting> +{ foo = 123; bar = 456; } +</programlisting> + + </para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-map'> + <term><function>map</function> + <replaceable>f</replaceable> <replaceable>list</replaceable></term> + <term><function>builtins.map</function> + <replaceable>f</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Apply the function <replaceable>f</replaceable> to + each element in the list <replaceable>list</replaceable>. For + example, + +<programlisting> +map (x: "foo" + x) [ "bar" "bla" "abc" ]</programlisting> + + evaluates to <literal>[ "foobar" "foobla" "fooabc" + ]</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-match'> + <term><function>builtins.match</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + + <listitem><para>Returns a list if the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> + matches <replaceable>str</replaceable> precisely, otherwise returns + <literal>null</literal>. Each item in the list is a regex group. + +<programlisting> +builtins.match "ab" "abc" +</programlisting> + +Evaluates to <literal>null</literal>. + +<programlisting> +builtins.match "abc" "abc" +</programlisting> + +Evaluates to <literal>[ ]</literal>. + +<programlisting> +builtins.match "a(b)(c)" "abc" +</programlisting> + +Evaluates to <literal>[ "b" "c" ]</literal>. + +<programlisting> +builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " +</programlisting> + +Evaluates to <literal>[ "foo" ]</literal>. + + </para></listitem> + </varlistentry> + + <varlistentry xml:id='builtin-mul'> + <term><function>builtins.mul</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the product of the numbers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-parseDrvName'> + <term><function>builtins.parseDrvName</function> + <replaceable>s</replaceable></term> + + <listitem><para>Split the string <replaceable>s</replaceable> into + a package name and version. The package name is everything up to + but not including the first dash followed by a digit, and the + version is everything following that dash. The result is returned + in a set <literal>{ name, version }</literal>. Thus, + <literal>builtins.parseDrvName "nix-0.12pre12876"</literal> + returns <literal>{ name = "nix"; version = "0.12pre12876"; + }</literal>.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-path'> + <term> + <function>builtins.path</function> + <replaceable>args</replaceable> + </term> + + <listitem> + <para> + An enrichment of the built-in path type, based on the attributes + present in <replaceable>args</replaceable>. All are optional + except <varname>path</varname>: + </para> + + <variablelist> + <varlistentry> + <term>path</term> + <listitem> + <para>The underlying path.</para> + </listitem> + </varlistentry> + <varlistentry> + <term>name</term> + <listitem> + <para> + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like <literal>@</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>filter</term> + <listitem> + <para> + A function of the type expected by + <link linkend="builtin-filterSource">builtins.filterSource</link>, + with the same semantics. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>recursive</term> + <listitem> + <para> + When <literal>false</literal>, when + <varname>path</varname> is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, <varname>path</varname> must refer to a + regular file, not a directory. This allows similar + behavior to <literal>fetchurl</literal>. Defaults to + <literal>true</literal>. + </para> + </listitem> + </varlistentry> + <varlistentry> + <term>sha256</term> + <listitem> + <para> + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + <literal>builtins.path</literal> to be used even when the + <literal>pure-eval</literal> nix config option is on. + </para> + </listitem> + </varlistentry> + </variablelist> + </listitem> + </varlistentry> + + <varlistentry xml:id='builtin-pathExists'> + <term><function>builtins.pathExists</function> + <replaceable>path</replaceable></term> + + <listitem><para>Return <literal>true</literal> if the path + <replaceable>path</replaceable> exists at evaluation time, and + <literal>false</literal> otherwise.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-placeholder'> + <term><function>builtins.placeholder</function> + <replaceable>output</replaceable></term> + + <listitem><para>Return a placeholder string for the specified + <replaceable>output</replaceable> that will be substituted by the + corresponding output path at build time. Typical outputs would be + <literal>"out"</literal>, <literal>"bin"</literal> or + <literal>"dev"</literal>.</para></listitem> + </varlistentry> + + <varlistentry xml:id='builtin-readDir'> + <term><function>builtins.readDir</function> + <replaceable>path</replaceable></term> + + <listitem><para>Return the contents of the directory + <replaceable>path</replaceable> as a set mapping directory entries + to the corresponding file type. For instance, if directory + <filename>A</filename> contains a regular file + <filename>B</filename> and another directory + <filename>C</filename>, then <literal>builtins.readDir + ./A</literal> will return the set + +<programlisting> +{ B = "regular"; C = "directory"; }</programlisting> + + The possible values for the file type are + <literal>"regular"</literal>, <literal>"directory"</literal>, + <literal>"symlink"</literal> and + <literal>"unknown"</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-readFile'> + <term><function>builtins.readFile</function> + <replaceable>path</replaceable></term> + + <listitem><para>Return the contents of the file + <replaceable>path</replaceable> as a string.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-removeAttrs'> + <term><function>removeAttrs</function> + <replaceable>set</replaceable> <replaceable>list</replaceable></term> + <term><function>builtins.removeAttrs</function> + <replaceable>set</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Remove the attributes listed in + <replaceable>list</replaceable> from + <replaceable>set</replaceable>. The attributes don’t have to + exist in <replaceable>set</replaceable>. For instance, + +<programlisting> +removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ]</programlisting> + + evaluates to <literal>{ y = 2; }</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-replaceStrings'> + <term><function>builtins.replaceStrings</function> + <replaceable>from</replaceable> <replaceable>to</replaceable> <replaceable>s</replaceable></term> + + <listitem><para>Given string <replaceable>s</replaceable>, replace + every occurrence of the strings in <replaceable>from</replaceable> + with the corresponding string in + <replaceable>to</replaceable>. For example, + +<programlisting> +builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" +</programlisting> + + evaluates to <literal>"fabir"</literal>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-seq'> + <term><function>builtins.seq</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Evaluate <replaceable>e1</replaceable>, then + evaluate and return <replaceable>e2</replaceable>. This ensures + that a computation is strict in the value of + <replaceable>e1</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-sort'> + <term><function>builtins.sort</function> + <replaceable>comparator</replaceable> <replaceable>list</replaceable></term> + + <listitem><para>Return <replaceable>list</replaceable> in sorted + order. It repeatedly calls the function + <replaceable>comparator</replaceable> with two elements. The + comparator should return <literal>true</literal> if the first + element is less than the second, and <literal>false</literal> + otherwise. For example, + +<programlisting> +builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] +</programlisting> + + produces the list <literal>[ 42 77 147 249 483 526 + ]</literal>.</para> + + <para>This is a stable sort: it preserves the relative order of + elements deemed equal by the comparator.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-split'> + <term><function>builtins.split</function> + <replaceable>regex</replaceable> <replaceable>str</replaceable></term> + + <listitem><para>Returns a list composed of non matched strings interleaved + with the lists of the <link + xlink:href="http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_04">extended + POSIX regular expression</link> <replaceable>regex</replaceable> matches + of <replaceable>str</replaceable>. Each item in the lists of matched + sequences is a regex group. + +<programlisting> +builtins.split "(a)b" "abc" +</programlisting> + +Evaluates to <literal>[ "" [ "a" ] "c" ]</literal>. + +<programlisting> +builtins.split "([ac])" "abc" +</programlisting> + +Evaluates to <literal>[ "" [ "a" ] "b" [ "c" ] "" ]</literal>. + +<programlisting> +builtins.split "(a)|(c)" "abc" +</programlisting> + +Evaluates to <literal>[ "" [ "a" null ] "b" [ null "c" ] "" ]</literal>. + +<programlisting> +builtins.split "([[:upper:]]+)" " FOO " +</programlisting> + +Evaluates to <literal>[ " " [ "FOO" ] " " ]</literal>. + + </para></listitem> + </varlistentry> + + + <varlistentry xml:id='builtin-splitVersion'> + <term><function>builtins.splitVersion</function> + <replaceable>s</replaceable></term> + + <listitem><para>Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in <link linkend="ssec-version-comparisons"> + <command>nix-env -u</command></link>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-stringLength'> + <term><function>builtins.stringLength</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return the length of the string + <replaceable>e</replaceable>. If <replaceable>e</replaceable> is + not a string, evaluation is aborted.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-sub'> + <term><function>builtins.sub</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Return the difference between the numbers + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable>.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-substring'> + <term><function>builtins.substring</function> + <replaceable>start</replaceable> <replaceable>len</replaceable> + <replaceable>s</replaceable></term> + + <listitem><para>Return the substring of + <replaceable>s</replaceable> from character position + <replaceable>start</replaceable> (zero-based) up to but not + including <replaceable>start + len</replaceable>. If + <replaceable>start</replaceable> is greater than the length of the + string, an empty string is returned, and if <replaceable>start + + len</replaceable> lies beyond the end of the string, only the + substring up to the end of the string is returned. + <replaceable>start</replaceable> must be + non-negative. For example, + +<programlisting> +builtins.substring 0 3 "nixos" +</programlisting> + + evaluates to <literal>"nix"</literal>. + </para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-tail'> + <term><function>builtins.tail</function> + <replaceable>list</replaceable></term> + + <listitem><para>Return the second to last elements of a list; + abort evaluation if the argument isn’t a list or is an empty + list.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-throw'> + <term><function>throw</function> + <replaceable>s</replaceable></term> + <term><function>builtins.throw</function> + <replaceable>s</replaceable></term> + + <listitem><para>Throw an error message + <replaceable>s</replaceable>. This usually aborts Nix expression + evaluation, but in <command>nix-env -qa</command> and other + commands that try to evaluate a set of derivations to get + information about those derivations, a derivation that throws an + error is silently skipped (which is not the case for + <function>abort</function>).</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-toFile'> + <term><function>builtins.toFile</function> + <replaceable>name</replaceable> + <replaceable>s</replaceable></term> + + <listitem><para>Store the string <replaceable>s</replaceable> in a + file in the Nix store and return its path. The file has suffix + <replaceable>name</replaceable>. This file can be used as an + input to derivations. One application is to write builders + “inline”. For instance, the following Nix expression combines + <xref linkend='ex-hello-nix' /> and <xref + linkend='ex-hello-builder' /> into one file: + +<programlisting> +{ stdenv, fetchurl, perl }: + +stdenv.mkDerivation { + name = "hello-2.1.1"; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + + PATH=$perl/bin:$PATH + + tar xvfz $src + cd hello-* + ./configure --prefix=$out + make + make install + "; + + src = fetchurl { + url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + }; + inherit perl; +}</programlisting> + + </para> + + <para>It is even possible for one file to refer to another, e.g., + +<programlisting> + builder = let + configFile = builtins.toFile "foo.conf" " + # This is some dummy configuration file. + <replaceable>...</replaceable> + "; + in builtins.toFile "builder.sh" " + source $stdenv/setup + <replaceable>...</replaceable> + cp ${configFile} $out/etc/foo.conf + ";</programlisting> + + Note that <literal>${configFile}</literal> is an antiquotation + (see <xref linkend='ssec-values' />), so the result of the + expression <literal>configFile</literal> (i.e., a path like + <filename>/nix/store/m7p7jfny445k...-foo.conf</filename>) will be + spliced into the resulting string.</para> + + <para>It is however <emphasis>not</emphasis> allowed to have files + mutually referring to each other, like so: + +<programlisting> +let + foo = builtins.toFile "foo" "...${bar}..."; + bar = builtins.toFile "bar" "...${foo}..."; +in foo</programlisting> + + This is not allowed because it would cause a cyclic dependency in + the computation of the cryptographic hashes for + <varname>foo</varname> and <varname>bar</varname>.</para> + <para>It is also not possible to reference the result of a derivation. + If you are using Nixpkgs, the <literal>writeTextFile</literal> function is able to + do that.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-toJSON'> + <term><function>builtins.toJSON</function> <replaceable>e</replaceable></term> + + <listitem><para>Return a string containing a JSON representation + of <replaceable>e</replaceable>. Strings, integers, floats, booleans, + nulls and lists are mapped to their JSON equivalents. Sets + (except derivations) are represented as objects. Derivations are + translated to a JSON string containing the derivation’s output + path. Paths are copied to the store and represented as a JSON + string of the resulting store path.</para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-toPath'> + <term><function>builtins.toPath</function> <replaceable>s</replaceable></term> + + <listitem><para> DEPRECATED. Use <literal>/. + "/path"</literal> + to convert a string into an absolute path. For relative paths, + use <literal>./. + "/path"</literal>. + </para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-toString'> + <term><function>toString</function> <replaceable>e</replaceable></term> + <term><function>builtins.toString</function> <replaceable>e</replaceable></term> + + <listitem><para>Convert the expression + <replaceable>e</replaceable> to a string. + <replaceable>e</replaceable> can be:</para> + <itemizedlist> + <listitem><para>A string (in which case the string is returned unmodified).</para></listitem> + <listitem><para>A path (e.g., <literal>toString /foo/bar</literal> yields <literal>"/foo/bar"</literal>.</para></listitem> + <listitem><para>A set containing <literal>{ __toString = self: ...; }</literal>.</para></listitem> + <listitem><para>An integer.</para></listitem> + <listitem><para>A list, in which case the string representations of its elements are joined with spaces.</para></listitem> + <listitem><para>A Boolean (<literal>false</literal> yields <literal>""</literal>, <literal>true</literal> yields <literal>"1"</literal>).</para></listitem> + <listitem><para><literal>null</literal>, which yields the empty string.</para></listitem> + </itemizedlist> + </listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-toXML'> + <term><function>builtins.toXML</function> <replaceable>e</replaceable></term> + + <listitem><para>Return a string containing an XML representation + of <replaceable>e</replaceable>. The main application for + <function>toXML</function> is to communicate information with the + builder in a more structured format than plain environment + variables.</para> + + <!-- TODO: more formally describe the schema of the XML + representation --> + + <para><xref linkend='ex-toxml' /> shows an example where this is + the case. The builder is supposed to generate the configuration + file for a <link xlink:href='http://jetty.mortbay.org/'>Jetty + servlet container</link>. A servlet container contains a number + of servlets (<filename>*.war</filename> files) each exported under + a specific URI prefix. So the servlet configuration is a list of + sets containing the <varname>path</varname> and + <varname>war</varname> of the servlet (<xref + linkend='ex-toxml-co-servlets' />). This kind of information is + difficult to communicate with the normal method of passing + information through an environment variable, which just + concatenates everything together into a string (which might just + work in this case, but wouldn’t work if fields are optional or + contain lists themselves). Instead the Nix expression is + converted to an XML representation with + <function>toXML</function>, which is unambiguous and can easily be + processed with the appropriate tools. For instance, in the + example an XSLT stylesheet (<xref linkend='ex-toxml-co-stylesheet' + />) is applied to it (<xref linkend='ex-toxml-co-apply' />) to + generate the XML configuration file for the Jetty server. The XML + representation produced from <xref linkend='ex-toxml-co-servlets' + /> by <function>toXML</function> is shown in <xref + linkend='ex-toxml-result' />.</para> + + <para>Note that <xref linkend='ex-toxml' /> uses the <function + linkend='builtin-toFile'>toFile</function> built-in to write the + builder and the stylesheet “inline” in the Nix expression. The + path of the stylesheet is spliced into the builder at + <literal>xsltproc ${stylesheet} + <replaceable>...</replaceable></literal>.</para> + + <example xml:id='ex-toxml'><title>Passing information to a builder + using <function>toXML</function></title> + +<programlisting><![CDATA[ +{ stdenv, fetchurl, libxslt, jira, uberwiki }: + +stdenv.mkDerivation (rec { + name = "web-server"; + + buildInputs = [ libxslt ]; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + mkdir $out + echo "$servlets" | xsltproc ${stylesheet} - > $out/server-conf.xml]]> <co xml:id='ex-toxml-co-apply' /> <![CDATA[ + "; + + stylesheet = builtins.toFile "stylesheet.xsl"]]> <co xml:id='ex-toxml-co-stylesheet' /> <![CDATA[ + "<?xml version='1.0' encoding='UTF-8'?> + <xsl:stylesheet xmlns:xsl='http://www.w3.org/1999/XSL/Transform' version='1.0'> + <xsl:template match='/'> + <Configure> + <xsl:for-each select='/expr/list/attrs'> + <Call name='addWebApplication'> + <Arg><xsl:value-of select=\"attr[@name = 'path']/string/@value\" /></Arg> + <Arg><xsl:value-of select=\"attr[@name = 'war']/path/@value\" /></Arg> + </Call> + </xsl:for-each> + </Configure> + </xsl:template> + </xsl:stylesheet> + "; + + servlets = builtins.toXML []]> <co xml:id='ex-toxml-co-servlets' /> <![CDATA[ + { path = "/bugtracker"; war = jira + "/lib/atlassian-jira.war"; } + { path = "/wiki"; war = uberwiki + "/uberwiki.war"; } + ]; +})]]></programlisting> + + </example> + + <example xml:id='ex-toxml-result'><title>XML representation produced by + <function>toXML</function></title> + +<programlisting><![CDATA[<?xml version='1.0' encoding='utf-8'?> +<expr> + <list> + <attrs> + <attr name="path"> + <string value="/bugtracker" /> + </attr> + <attr name="war"> + <path value="/nix/store/d1jh9pasa7k2...-jira/lib/atlassian-jira.war" /> + </attr> + </attrs> + <attrs> + <attr name="path"> + <string value="/wiki" /> + </attr> + <attr name="war"> + <path value="/nix/store/y6423b1yi4sx...-uberwiki/uberwiki.war" /> + </attr> + </attrs> + </list> +</expr>]]></programlisting> + + </example> + + </listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-trace'> + <term><function>builtins.trace</function> + <replaceable>e1</replaceable> <replaceable>e2</replaceable></term> + + <listitem><para>Evaluate <replaceable>e1</replaceable> and print its + abstract syntax representation on standard error. Then return + <replaceable>e2</replaceable>. This function is useful for + debugging.</para></listitem> + + </varlistentry> + + <varlistentry xml:id='builtin-tryEval'> + <term><function>builtins.tryEval</function> + <replaceable>e</replaceable></term> + + <listitem><para>Try to shallowly evaluate <replaceable>e</replaceable>. + Return a set containing the attributes <literal>success</literal> + (<literal>true</literal> if <replaceable>e</replaceable> evaluated + successfully, <literal>false</literal> if an error was thrown) and + <literal>value</literal>, equalling <replaceable>e</replaceable> + if successful and <literal>false</literal> otherwise. Note that this + doesn't evaluate <replaceable>e</replaceable> deeply, so + <literal>let e = { x = throw ""; }; in (builtins.tryEval e).success + </literal> will be <literal>true</literal>. Using <literal>builtins.deepSeq + </literal> one can get the expected result: <literal>let e = { x = throw ""; + }; in (builtins.tryEval (builtins.deepSeq e e)).success</literal> will be + <literal>false</literal>. + </para></listitem> + + </varlistentry> + + + <varlistentry xml:id='builtin-typeOf'> + <term><function>builtins.typeOf</function> + <replaceable>e</replaceable></term> + + <listitem><para>Return a string representing the type of the value + <replaceable>e</replaceable>, namely <literal>"int"</literal>, + <literal>"bool"</literal>, <literal>"string"</literal>, + <literal>"path"</literal>, <literal>"null"</literal>, + <literal>"set"</literal>, <literal>"list"</literal>, + <literal>"lambda"</literal> or + <literal>"float"</literal>.</para></listitem> + + </varlistentry> + + +</variablelist> + + +</section> diff --git a/third_party/nix/doc/manual/expressions/derivations.xml b/third_party/nix/doc/manual/expressions/derivations.xml new file mode 100644 index 000000000000..6f6297565ca2 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/derivations.xml @@ -0,0 +1,211 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-derivation"> + +<title>Derivations</title> + +<para>The most important built-in function is +<function>derivation</function>, which is used to describe a single +derivation (a build action). It takes as input a set, the attributes +of which specify the inputs of the build.</para> + +<itemizedlist> + + <listitem xml:id="attr-system"><para>There must be an attribute named + <varname>system</varname> whose value must be a string specifying a + Nix platform identifier, such as <literal>"i686-linux"</literal> or + <literal>"x86_64-darwin"</literal><footnote><para>To figure out + your platform identifier, look at the line <quote>Checking for the + canonical Nix system name</quote> in the output of Nix's + <filename>configure</filename> script.</para></footnote> The build + can only be performed on a machine and operating system matching the + platform identifier. (Nix can automatically forward builds for + other platforms by forwarding them to other machines; see <xref + linkend='chap-distributed-builds' />.)</para></listitem> + + <listitem><para>There must be an attribute named + <varname>name</varname> whose value must be a string. This is used + as a symbolic name for the package by <command>nix-env</command>, + and it is appended to the output paths of the + derivation.</para></listitem> + + <listitem><para>There must be an attribute named + <varname>builder</varname> that identifies the program that is + executed to perform the build. It can be either a derivation or a + source (a local file reference, e.g., + <filename>./builder.sh</filename>).</para></listitem> + + <listitem><para>Every attribute is passed as an environment variable + to the builder. Attribute values are translated to environment + variables as follows: + + <itemizedlist> + + <listitem><para>Strings and numbers are just passed + verbatim.</para></listitem> + + <listitem><para>A <emphasis>path</emphasis> (e.g., + <filename>../foo/sources.tar</filename>) causes the referenced + file to be copied to the store; its location in the store is put + in the environment variable. The idea is that all sources + should reside in the Nix store, since all inputs to a derivation + should reside in the Nix store.</para></listitem> + + <listitem><para>A <emphasis>derivation</emphasis> causes that + derivation to be built prior to the present derivation; its + default output path is put in the environment + variable.</para></listitem> + + <listitem><para>Lists of the previous types are also allowed. + They are simply concatenated, separated by + spaces.</para></listitem> + + <listitem><para><literal>true</literal> is passed as the string + <literal>1</literal>, <literal>false</literal> and + <literal>null</literal> are passed as an empty string. + </para></listitem> + </itemizedlist> + + </para></listitem> + + <listitem><para>The optional attribute <varname>args</varname> + specifies command-line arguments to be passed to the builder. It + should be a list.</para></listitem> + + <listitem><para>The optional attribute <varname>outputs</varname> + specifies a list of symbolic outputs of the derivation. By default, + a derivation produces a single output path, denoted as + <literal>out</literal>. However, derivations can produce multiple + output paths. This is useful because it allows outputs to be + downloaded or garbage-collected separately. For instance, imagine a + library package that provides a dynamic library, header files, and + documentation. A program that links against the library doesn’t + need the header files and documentation at runtime, and it doesn’t + need the documentation at build time. Thus, the library package + could specify: +<programlisting> +outputs = [ "lib" "headers" "doc" ]; +</programlisting> + This will cause Nix to pass environment variables + <literal>lib</literal>, <literal>headers</literal> and + <literal>doc</literal> to the builder containing the intended store + paths of each output. The builder would typically do something like +<programlisting> +./configure --libdir=$lib/lib --includedir=$headers/include --docdir=$doc/share/doc +</programlisting> + for an Autoconf-style package. You can refer to each output of a + derivation by selecting it as an attribute, e.g. +<programlisting> +buildInputs = [ pkg.lib pkg.headers ]; +</programlisting> + The first element of <varname>outputs</varname> determines the + <emphasis>default output</emphasis>. Thus, you could also write +<programlisting> +buildInputs = [ pkg pkg.headers ]; +</programlisting> + since <literal>pkg</literal> is equivalent to + <literal>pkg.lib</literal>.</para></listitem> + +</itemizedlist> + +<para>The function <function>mkDerivation</function> in the Nixpkgs +standard environment is a wrapper around +<function>derivation</function> that adds a default value for +<varname>system</varname> and always uses Bash as the builder, to +which the supplied builder is passed as a command-line argument. See +the Nixpkgs manual for details.</para> + +<para>The builder is executed as follows: + +<itemizedlist> + + <listitem><para>A temporary directory is created under the directory + specified by <envar>TMPDIR</envar> (default + <filename>/tmp</filename>) where the build will take place. The + current directory is changed to this directory.</para></listitem> + + <listitem><para>The environment is cleared and set to the derivation + attributes, as specified above.</para></listitem> + + <listitem><para>In addition, the following variables are set: + + <itemizedlist> + + <listitem><para><envar>NIX_BUILD_TOP</envar> contains the path of + the temporary directory for this build.</para></listitem> + + <listitem><para>Also, <envar>TMPDIR</envar>, + <envar>TEMPDIR</envar>, <envar>TMP</envar>, <envar>TEMP</envar> + are set to point to the temporary directory. This is to prevent + the builder from accidentally writing temporary files anywhere + else. Doing so might cause interference by other + processes.</para></listitem> + + <listitem><para><envar>PATH</envar> is set to + <filename>/path-not-set</filename> to prevent shells from + initialising it to their built-in default value.</para></listitem> + + <listitem><para><envar>HOME</envar> is set to + <filename>/homeless-shelter</filename> to prevent programs from + using <filename>/etc/passwd</filename> or the like to find the + user's home directory, which could cause impurity. Usually, when + <envar>HOME</envar> is set, it is used as the location of the home + directory, even if it points to a non-existent + path.</para></listitem> + + <listitem><para><envar>NIX_STORE</envar> is set to the path of the + top-level Nix store directory (typically, + <filename>/nix/store</filename>).</para></listitem> + + <listitem><para>For each output declared in + <varname>outputs</varname>, the corresponding environment variable + is set to point to the intended path in the Nix store for that + output. Each output path is a concatenation of the cryptographic + hash of all build inputs, the <varname>name</varname> attribute + and the output name. (The output name is omitted if it’s + <literal>out</literal>.)</para></listitem> + + </itemizedlist> + + </para></listitem> + + <listitem><para>If an output path already exists, it is removed. + Also, locks are acquired to prevent multiple Nix instances from + performing the same build at the same time.</para></listitem> + + <listitem><para>A log of the combined standard output and error is + written to <filename>/nix/var/log/nix</filename>.</para></listitem> + + <listitem><para>The builder is executed with the arguments specified + by the attribute <varname>args</varname>. If it exits with exit + code 0, it is considered to have succeeded.</para></listitem> + + <listitem><para>The temporary directory is removed (unless the + <option>-K</option> option was specified).</para></listitem> + + <listitem><para>If the build was successful, Nix scans each output + path for references to input paths by looking for the hash parts of + the input paths. Since these are potential runtime dependencies, + Nix registers them as dependencies of the output + paths.</para></listitem> + + <listitem><para>After the build, Nix sets the last-modified + timestamp on all files in the build result to 1 (00:00:01 1/1/1970 + UTC), sets the group to the default group, and sets the mode of the + file to 0444 or 0555 (i.e., read-only, with execute permission + enabled if the file was originally executable). Note that possible + <literal>setuid</literal> and <literal>setgid</literal> bits are + cleared. Setuid and setgid programs are not currently supported by + Nix. This is because the Nix archives used in deployment have no + concept of ownership information, and because it makes the build + result dependent on the user performing the build.</para></listitem> + +</itemizedlist> + +</para> + +<xi:include href="advanced-attributes.xml" /> + +</section> diff --git a/third_party/nix/doc/manual/expressions/expression-language.xml b/third_party/nix/doc/manual/expressions/expression-language.xml new file mode 100644 index 000000000000..240ef80f1463 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/expression-language.xml @@ -0,0 +1,30 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-expression-language"> + +<title>Nix Expression Language</title> + +<para>The Nix expression language is a pure, lazy, functional +language. Purity means that operations in the language don't have +side-effects (for instance, there is no variable assignment). +Laziness means that arguments to functions are evaluated only when +they are needed. Functional means that functions are +<quote>normal</quote> values that can be passed around and manipulated +in interesting ways. The language is not a full-featured, general +purpose language. Its main job is to describe packages, +compositions of packages, and the variability within +packages.</para> + +<para>This section presents the various features of the +language.</para> + +<xi:include href="language-values.xml" /> +<xi:include href="language-constructs.xml" /> +<xi:include href="language-operators.xml" /> +<xi:include href="derivations.xml" /> +<xi:include href="builtins.xml" /> + + +</chapter> diff --git a/third_party/nix/doc/manual/expressions/expression-syntax.xml b/third_party/nix/doc/manual/expressions/expression-syntax.xml new file mode 100644 index 000000000000..42b9dca36239 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/expression-syntax.xml @@ -0,0 +1,148 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-expression-syntax'> + +<title>Expression Syntax</title> + +<example xml:id='ex-hello-nix'><title>Nix expression for GNU Hello +(<filename>default.nix</filename>)</title> +<programlisting> +{ stdenv, fetchurl, perl }: <co xml:id='ex-hello-nix-co-1' /> + +stdenv.mkDerivation { <co xml:id='ex-hello-nix-co-2' /> + name = "hello-2.1.1"; <co xml:id='ex-hello-nix-co-3' /> + builder = ./builder.sh; <co xml:id='ex-hello-nix-co-4' /> + src = fetchurl { <co xml:id='ex-hello-nix-co-5' /> + url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + }; + inherit perl; <co xml:id='ex-hello-nix-co-6' /> +}</programlisting> +</example> + +<para><xref linkend='ex-hello-nix' /> shows a Nix expression for GNU +Hello. It's actually already in the Nix Packages collection in +<filename>pkgs/applications/misc/hello/ex-1/default.nix</filename>. +It is customary to place each package in a separate directory and call +the single Nix expression in that directory +<filename>default.nix</filename>. The file has the following elements +(referenced from the figure by number): + +<calloutlist> + + <callout arearefs='ex-hello-nix-co-1'> + + <para>This states that the expression is a + <emphasis>function</emphasis> that expects to be called with three + arguments: <varname>stdenv</varname>, <varname>fetchurl</varname>, + and <varname>perl</varname>. They are needed to build Hello, but + we don't know how to build them here; that's why they are function + arguments. <varname>stdenv</varname> is a package that is used + by almost all Nix Packages packages; it provides a + <quote>standard</quote> environment consisting of the things you + would expect in a basic Unix environment: a C/C++ compiler (GCC, + to be precise), the Bash shell, fundamental Unix tools such as + <command>cp</command>, <command>grep</command>, + <command>tar</command>, etc. <varname>fetchurl</varname> is a + function that downloads files. <varname>perl</varname> is the + Perl interpreter.</para> + + <para>Nix functions generally have the form <literal>{ x, y, ..., + z }: e</literal> where <varname>x</varname>, <varname>y</varname>, + etc. are the names of the expected arguments, and where + <replaceable>e</replaceable> is the body of the function. So + here, the entire remainder of the file is the body of the + function; when given the required arguments, the body should + describe how to build an instance of the Hello package.</para> + + </callout> + + <callout arearefs='ex-hello-nix-co-2'> + + <para>So we have to build a package. Building something from + other stuff is called a <emphasis>derivation</emphasis> in Nix (as + opposed to sources, which are built by humans instead of + computers). We perform a derivation by calling + <varname>stdenv.mkDerivation</varname>. + <varname>mkDerivation</varname> is a function provided by + <varname>stdenv</varname> that builds a package from a set of + <emphasis>attributes</emphasis>. A set is just a list of + key/value pairs where each key is a string and each value is an + arbitrary Nix expression. They take the general form <literal>{ + <replaceable>name1</replaceable> = + <replaceable>expr1</replaceable>; <replaceable>...</replaceable> + <replaceable>nameN</replaceable> = + <replaceable>exprN</replaceable>; }</literal>.</para> + + </callout> + + <callout arearefs='ex-hello-nix-co-3'> + + <para>The attribute <varname>name</varname> specifies the symbolic + name and version of the package. Nix doesn't really care about + these things, but they are used by for instance <command>nix-env + -q</command> to show a <quote>human-readable</quote> name for + packages. This attribute is required by + <varname>mkDerivation</varname>.</para> + + </callout> + + <callout arearefs='ex-hello-nix-co-4'> + + <para>The attribute <varname>builder</varname> specifies the + builder. This attribute can sometimes be omitted, in which case + <varname>mkDerivation</varname> will fill in a default builder + (which does a <literal>configure; make; make install</literal>, in + essence). Hello is sufficiently simple that the default builder + would suffice, but in this case, we will show an actual builder + for educational purposes. The value + <command>./builder.sh</command> refers to the shell script shown + in <xref linkend='ex-hello-builder' />, discussed below.</para> + + </callout> + + <callout arearefs='ex-hello-nix-co-5'> + + <para>The builder has to know what the sources of the package + are. Here, the attribute <varname>src</varname> is bound to the + result of a call to the <command>fetchurl</command> function. + Given a URL and a SHA-256 hash of the expected contents of the file + at that URL, this function builds a derivation that downloads the + file and checks its hash. So the sources are a dependency that + like all other dependencies is built before Hello itself is + built.</para> + + <para>Instead of <varname>src</varname> any other name could have + been used, and in fact there can be any number of sources (bound + to different attributes). However, <varname>src</varname> is + customary, and it's also expected by the default builder (which we + don't use in this example).</para> + + </callout> + + <callout arearefs='ex-hello-nix-co-6'> + + <para>Since the derivation requires Perl, we have to pass the + value of the <varname>perl</varname> function argument to the + builder. All attributes in the set are actually passed as + environment variables to the builder, so declaring an attribute + + <programlisting> +perl = perl;</programlisting> + + will do the trick: it binds an attribute <varname>perl</varname> + to the function argument which also happens to be called + <varname>perl</varname>. However, it looks a bit silly, so there + is a shorter syntax. The <literal>inherit</literal> keyword + causes the specified attributes to be bound to whatever variables + with the same name happen to be in scope.</para> + + </callout> + +</calloutlist> + +</para> + +</section> diff --git a/third_party/nix/doc/manual/expressions/generic-builder.xml b/third_party/nix/doc/manual/expressions/generic-builder.xml new file mode 100644 index 000000000000..db7ff405d8b1 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/generic-builder.xml @@ -0,0 +1,98 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-generic-builder'> + +<title>Generic Builder Syntax</title> + +<para>Recall from <xref linkend='ex-hello-builder' /> that the builder +looked something like this: + +<programlisting> +PATH=$perl/bin:$PATH +tar xvfz $src +cd hello-* +./configure --prefix=$out +make +make install</programlisting> + +The builders for almost all Unix packages look like this — set up some +environment variables, unpack the sources, configure, build, and +install. For this reason the standard environment provides some Bash +functions that automate the build process. A builder using the +generic build facilities in shown in <xref linkend='ex-hello-builder2' +/>.</para> + +<example xml:id='ex-hello-builder2'><title>Build script using the generic +build functions</title> +<programlisting> +buildInputs="$perl" <co xml:id='ex-hello-builder2-co-1' /> + +source $stdenv/setup <co xml:id='ex-hello-builder2-co-2' /> + +genericBuild <co xml:id='ex-hello-builder2-co-3' /></programlisting> +</example> + +<calloutlist> + + <callout arearefs='ex-hello-builder2-co-1'> + + <para>The <envar>buildInputs</envar> variable tells + <filename>setup</filename> to use the indicated packages as + <quote>inputs</quote>. This means that if a package provides a + <filename>bin</filename> subdirectory, it's added to + <envar>PATH</envar>; if it has a <filename>include</filename> + subdirectory, it's added to GCC's header search path; and so + on.<footnote><para>How does it work? <filename>setup</filename> + tries to source the file + <filename><replaceable>pkg</replaceable>/nix-support/setup-hook</filename> + of all dependencies. These “setup hooks” can then set up whatever + environment variables they want; for instance, the setup hook for + Perl sets the <envar>PERL5LIB</envar> environment variable to + contain the <filename>lib/site_perl</filename> directories of all + inputs.</para></footnote> + </para> + + </callout> + + <callout arearefs='ex-hello-builder2-co-2'> + + <para>The function <function>genericBuild</function> is defined in + the file <literal>$stdenv/setup</literal>.</para> + + </callout> + + <callout arearefs='ex-hello-builder2-co-3'> + + <para>The final step calls the shell function + <function>genericBuild</function>, which performs the steps that + were done explicitly in <xref linkend='ex-hello-builder' />. The + generic builder is smart enough to figure out whether to unpack + the sources using <command>gzip</command>, + <command>bzip2</command>, etc. It can be customised in many ways; + see the Nixpkgs manual for details.</para> + + </callout> + +</calloutlist> + +<para>Discerning readers will note that the +<envar>buildInputs</envar> could just as well have been set in the Nix +expression, like this: + +<programlisting> + buildInputs = [ perl ];</programlisting> + +The <varname>perl</varname> attribute can then be removed, and the +builder becomes even shorter: + +<programlisting> +source $stdenv/setup +genericBuild</programlisting> + +In fact, <varname>mkDerivation</varname> provides a default builder +that looks exactly like that, so it is actually possible to omit the +builder for Hello entirely.</para> + +</section> diff --git a/third_party/nix/doc/manual/expressions/language-constructs.xml b/third_party/nix/doc/manual/expressions/language-constructs.xml new file mode 100644 index 000000000000..0d0cbbe1553e --- /dev/null +++ b/third_party/nix/doc/manual/expressions/language-constructs.xml @@ -0,0 +1,409 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-constructs"> + +<title>Language Constructs</title> + +<simplesect><title>Recursive sets</title> + +<para>Recursive sets are just normal sets, but the attributes can +refer to each other. For example, + +<programlisting> +rec { + x = y; + y = 123; +}.x +</programlisting> + +evaluates to <literal>123</literal>. Note that without +<literal>rec</literal> the binding <literal>x = y;</literal> would +refer to the variable <varname>y</varname> in the surrounding scope, +if one exists, and would be invalid if no such variable exists. That +is, in a normal (non-recursive) set, attributes are not added to the +lexical scope; in a recursive set, they are.</para> + +<para>Recursive sets of course introduce the danger of infinite +recursion. For example, + +<programlisting> +rec { + x = y; + y = x; +}.x</programlisting> + +does not terminate<footnote><para>Actually, Nix detects infinite +recursion in this case and aborts (<quote>infinite recursion +encountered</quote>).</para></footnote>.</para> + +</simplesect> + + +<simplesect xml:id="sect-let-expressions"><title>Let-expressions</title> + +<para>A let-expression allows you to define local variables for an +expression. For instance, + +<programlisting> +let + x = "foo"; + y = "bar"; +in x + y</programlisting> + +evaluates to <literal>"foobar"</literal>. + +</para> + +</simplesect> + + +<simplesect><title>Inheriting attributes</title> + +<para>When defining a set or in a let-expression it is often convenient to copy variables +from the surrounding lexical scope (e.g., when you want to propagate +attributes). This can be shortened using the +<literal>inherit</literal> keyword. For instance, + +<programlisting> +let x = 123; in +{ inherit x; + y = 456; +}</programlisting> + +is equivalent to + +<programlisting> +let x = 123; in +{ x = x; + y = 456; +}</programlisting> + +and both evaluate to <literal>{ x = 123; y = 456; }</literal>. (Note that +this works because <varname>x</varname> is added to the lexical scope +by the <literal>let</literal> construct.) It is also possible to +inherit attributes from another set. For instance, in this fragment +from <filename>all-packages.nix</filename>, + +<programlisting> + graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xlibs) libXaw; + }; + + xlibs = { + libX11 = ...; + libXaw = ...; + ... + } + + libpng = ...; + libjpg = ...; + ...</programlisting> + +the set used in the function call to the function defined in +<filename>../tools/graphics/graphviz</filename> inherits a number of +variables from the surrounding scope (<varname>fetchurl</varname> +... <varname>yacc</varname>), but also inherits +<varname>libXaw</varname> (the X Athena Widgets) from the +<varname>xlibs</varname> (X11 client-side libraries) set.</para> + +<para> +Summarizing the fragment + +<programlisting> +... +inherit x y z; +inherit (src-set) a b c; +...</programlisting> + +is equivalent to + +<programlisting> +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +...</programlisting> + +when used while defining local variables in a let-expression or +while defining a set.</para> + +</simplesect> + + +<simplesect xml:id="ss-functions"><title>Functions</title> + +<para>Functions have the following form: + +<programlisting> +<replaceable>pattern</replaceable>: <replaceable>body</replaceable></programlisting> + +The pattern specifies what the argument of the function must look +like, and binds variables in the body to (parts of) the +argument. There are three kinds of patterns:</para> + +<itemizedlist> + + + <listitem><para>If a pattern is a single identifier, then the + function matches any argument. Example: + + <programlisting> +let negate = x: !x; + concat = x: y: x + y; +in if negate true then concat "foo" "bar" else ""</programlisting> + + Note that <function>concat</function> is a function that takes one + argument and returns a function that takes another argument. This + allows partial parameterisation (i.e., only filling some of the + arguments of a function); e.g., + + <programlisting> +map (concat "foo") [ "bar" "bla" "abc" ]</programlisting> + + evaluates to <literal>[ "foobar" "foobla" + "fooabc" ]</literal>.</para></listitem> + + + <listitem><para>A <emphasis>set pattern</emphasis> of the form + <literal>{ name1, name2, …, nameN }</literal> matches a set + containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + +<programlisting> +{ x, y, z }: z + y + x</programlisting> + + can only be called with a set containing exactly the attributes + <varname>x</varname>, <varname>y</varname> and + <varname>z</varname>. No other attributes are allowed. If you want + to allow additional arguments, you can use an ellipsis + (<literal>...</literal>): + +<programlisting> +{ x, y, z, ... }: z + y + x</programlisting> + + This works on any set that contains at least the three named + attributes.</para> + + <para>It is possible to provide <emphasis>default values</emphasis> + for attributes, in which case they are allowed to be missing. A + default value is specified by writing + <literal><replaceable>name</replaceable> ? + <replaceable>e</replaceable></literal>, where + <replaceable>e</replaceable> is an arbitrary expression. For example, + +<programlisting> +{ x, y ? "foo", z ? "bar" }: z + y + x</programlisting> + + specifies a function that only requires an attribute named + <varname>x</varname>, but optionally accepts <varname>y</varname> + and <varname>z</varname>.</para></listitem> + + + <listitem><para>An <literal>@</literal>-pattern provides a means of referring + to the whole value being matched: + +<programlisting> args@{ x, y, z, ... }: z + y + x + args.a</programlisting> + +but can also be written as: + +<programlisting> { x, y, z, ... } @ args: z + y + x + args.a</programlisting> + + Here <varname>args</varname> is bound to the entire argument, which + is further matched against the pattern <literal>{ x, y, z, + ... }</literal>. <literal>@</literal>-pattern makes mainly sense with an + ellipsis(<literal>...</literal>) as you can access attribute names as + <literal>a</literal>, using <literal>args.a</literal>, which was given as an + additional attribute to the function. + </para> + + <warning> + <para> + The <literal>args@</literal> expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in <literal>args</literal>. + </para> + <para> + For instance +<programlisting> +let + function = args@{ a ? 23, ... }: args; +in + function {} +</programlisting> + will evaluate to an empty attribute set. + </para> + </warning></listitem> + +</itemizedlist> + +<para>Note that functions do not have names. If you want to give them +a name, you can bind them to an attribute, e.g., + +<programlisting> +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; }</programlisting> + +</para> + +</simplesect> + + +<simplesect><title>Conditionals</title> + +<para>Conditionals look like this: + +<programlisting> +if <replaceable>e1</replaceable> then <replaceable>e2</replaceable> else <replaceable>e3</replaceable></programlisting> + +where <replaceable>e1</replaceable> is an expression that should +evaluate to a Boolean value (<literal>true</literal> or +<literal>false</literal>).</para> + +</simplesect> + + +<simplesect><title>Assertions</title> + +<para>Assertions are generally used to check that certain requirements +on or between features and dependencies hold. They look like this: + +<programlisting> +assert <replaceable>e1</replaceable>; <replaceable>e2</replaceable></programlisting> + +where <replaceable>e1</replaceable> is an expression that should +evaluate to a Boolean value. If it evaluates to +<literal>true</literal>, <replaceable>e2</replaceable> is returned; +otherwise expression evaluation is aborted and a backtrace is printed.</para> + +<example xml:id='ex-subversion-nix'><title>Nix expression for Subversion</title> +<programlisting> +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; <co xml:id='ex-subversion-nix-co-1' /> +assert httpServer -> httpd != null && httpd.expat == expat; <co xml:id='ex-subversion-nix-co-2' /> +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); <co xml:id='ex-subversion-nix-co-3' /> +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; <co xml:id='ex-subversion-nix-co-4' /> + ... +}</programlisting> +</example> + +<para><xref linkend='ex-subversion-nix' /> show how assertions are +used in the Nix expression for Subversion.</para> + +<calloutlist> + + <callout arearefs='ex-subversion-nix-co-1'> + <para>This assertion states that if Subversion is to have support + for local repositories, then Berkeley DB is needed. So if the + Subversion function is called with the + <varname>localServer</varname> argument set to + <literal>true</literal> but the <varname>db4</varname> argument + set to <literal>null</literal>, then the evaluation fails.</para> + </callout> + + <callout arearefs='ex-subversion-nix-co-2'> + <para>This is a more subtle condition: if Subversion is built with + Apache (<literal>httpServer</literal>) support, then the Expat + library (an XML library) used by Subversion should be same as the + one used by Apache. This is because in this configuration + Subversion code ends up being linked with Apache code, and if the + Expat libraries do not match, a build- or runtime link error or + incompatibility might occur.</para> + </callout> + + <callout arearefs='ex-subversion-nix-co-3'> + <para>This assertion says that in order for Subversion to have SSL + support (so that it can access <literal>https</literal> URLs), an + OpenSSL library must be passed. Additionally, it says that + <emphasis>if</emphasis> Apache support is enabled, then Apache's + OpenSSL should match Subversion's. (Note that if Apache support + is not enabled, we don't care about Apache's OpenSSL.)</para> + </callout> + + <callout arearefs='ex-subversion-nix-co-4'> + <para>The conditional here is not really related to assertions, + but is worth pointing out: it ensures that if SSL support is + disabled, then the Subversion derivation is not dependent on + OpenSSL, even if a non-<literal>null</literal> value was passed. + This prevents an unnecessary rebuild of Subversion if OpenSSL + changes.</para> + </callout> + +</calloutlist> + +</simplesect> + + + +<simplesect><title>With-expressions</title> + +<para>A <emphasis>with-expression</emphasis>, + +<programlisting> +with <replaceable>e1</replaceable>; <replaceable>e2</replaceable></programlisting> + +introduces the set <replaceable>e1</replaceable> into the lexical +scope of the expression <replaceable>e2</replaceable>. For instance, + +<programlisting> +let as = { x = "foo"; y = "bar"; }; +in with as; x + y</programlisting> + +evaluates to <literal>"foobar"</literal> since the +<literal>with</literal> adds the <varname>x</varname> and +<varname>y</varname> attributes of <varname>as</varname> to the +lexical scope in the expression <literal>x + y</literal>. The most +common use of <literal>with</literal> is in conjunction with the +<function>import</function> function. E.g., + +<programlisting> +with (import ./definitions.nix); ...</programlisting> + +makes all attributes defined in the file +<filename>definitions.nix</filename> available as if they were defined +locally in a <literal>let</literal>-expression.</para> + +<para>The bindings introduced by <literal>with</literal> do not shadow bindings +introduced by other means, e.g. + +<programlisting> +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ...</programlisting> + +establishes the same scope as + +<programlisting> +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ...</programlisting> + +</para> + +</simplesect> + + +<simplesect><title>Comments</title> + +<para>Comments can be single-line, started with a <literal>#</literal> +character, or inline/multi-line, enclosed within <literal>/* +... */</literal>.</para> + +</simplesect> + + +</section> diff --git a/third_party/nix/doc/manual/expressions/language-operators.xml b/third_party/nix/doc/manual/expressions/language-operators.xml new file mode 100644 index 000000000000..4f11bf52938f --- /dev/null +++ b/third_party/nix/doc/manual/expressions/language-operators.xml @@ -0,0 +1,222 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-language-operators"> + +<title>Operators</title> + +<para><xref linkend='table-operators' /> lists the operators in the +Nix expression language, in order of precedence (from strongest to +weakest binding).</para> + +<table xml:id='table-operators'> + <title>Operators</title> + <tgroup cols='3'> + <thead> + <row> + <entry>Name</entry> + <entry>Syntax</entry> + <entry>Associativity</entry> + <entry>Description</entry> + <entry>Precedence</entry> + </row> + </thead> + <tbody> + <row> + <entry>Select</entry> + <entry><replaceable>e</replaceable> <literal>.</literal> + <replaceable>attrpath</replaceable> + [ <literal>or</literal> <replaceable>def</replaceable> ] + </entry> + <entry>none</entry> + <entry>Select attribute denoted by the attribute path + <replaceable>attrpath</replaceable> from set + <replaceable>e</replaceable>. (An attribute path is a + dot-separated list of attribute names.) If the attribute + doesn’t exist, return <replaceable>def</replaceable> if + provided, otherwise abort evaluation.</entry> + <entry>1</entry> + </row> + <row> + <entry>Application</entry> + <entry><replaceable>e1</replaceable> <replaceable>e2</replaceable></entry> + <entry>left</entry> + <entry>Call function <replaceable>e1</replaceable> with + argument <replaceable>e2</replaceable>.</entry> + <entry>2</entry> + </row> + <row> + <entry>Arithmetic Negation</entry> + <entry><literal>-</literal> <replaceable>e</replaceable></entry> + <entry>none</entry> + <entry>Arithmetic negation.</entry> + <entry>3</entry> + </row> + <row> + <entry>Has Attribute</entry> + <entry><replaceable>e</replaceable> <literal>?</literal> + <replaceable>attrpath</replaceable></entry> + <entry>none</entry> + <entry>Test whether set <replaceable>e</replaceable> contains + the attribute denoted by <replaceable>attrpath</replaceable>; + return <literal>true</literal> or + <literal>false</literal>.</entry> + <entry>4</entry> + </row> + <row> + <entry>List Concatenation</entry> + <entry><replaceable>e1</replaceable> <literal>++</literal> <replaceable>e2</replaceable></entry> + <entry>right</entry> + <entry>List concatenation.</entry> + <entry>5</entry> + </row> + <row> + <entry>Multiplication</entry> + <entry> + <replaceable>e1</replaceable> <literal>*</literal> <replaceable>e2</replaceable>, + </entry> + <entry>left</entry> + <entry>Arithmetic multiplication.</entry> + <entry>6</entry> + </row> + <row> + <entry>Division</entry> + <entry> + <replaceable>e1</replaceable> <literal>/</literal> <replaceable>e2</replaceable> + </entry> + <entry>left</entry> + <entry>Arithmetic division.</entry> + <entry>6</entry> + </row> + <row> + <entry>Addition</entry> + <entry> + <replaceable>e1</replaceable> <literal>+</literal> <replaceable>e2</replaceable> + </entry> + <entry>left</entry> + <entry>Arithmetic addition.</entry> + <entry>7</entry> + </row> + <row> + <entry>Subtraction</entry> + <entry> + <replaceable>e1</replaceable> <literal>-</literal> <replaceable>e2</replaceable> + </entry> + <entry>left</entry> + <entry>Arithmetic subtraction.</entry> + <entry>7</entry> + </row> + <row> + <entry>String Concatenation</entry> + <entry> + <replaceable>string1</replaceable> <literal>+</literal> <replaceable>string2</replaceable> + </entry> + <entry>left</entry> + <entry>String concatenation.</entry> + <entry>7</entry> + </row> + <row> + <entry>Not</entry> + <entry><literal>!</literal> <replaceable>e</replaceable></entry> + <entry>none</entry> + <entry>Boolean negation.</entry> + <entry>8</entry> + </row> + <row> + <entry>Update</entry> + <entry><replaceable>e1</replaceable> <literal>//</literal> + <replaceable>e2</replaceable></entry> + <entry>right</entry> + <entry>Return a set consisting of the attributes in + <replaceable>e1</replaceable> and + <replaceable>e2</replaceable> (with the latter taking + precedence over the former in case of equally named + attributes).</entry> + <entry>9</entry> + </row> + <row> + <entry>Less Than</entry> + <entry> + <replaceable>e1</replaceable> <literal><</literal> <replaceable>e2</replaceable>, + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Less Than or Equal To</entry> + <entry> + <replaceable>e1</replaceable> <literal><=</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Greater Than</entry> + <entry> + <replaceable>e1</replaceable> <literal>></literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Greater Than or Equal To</entry> + <entry> + <replaceable>e1</replaceable> <literal>>=</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Arithmetic comparison.</entry> + <entry>10</entry> + </row> + <row> + <entry>Equality</entry> + <entry> + <replaceable>e1</replaceable> <literal>==</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Equality.</entry> + <entry>11</entry> + </row> + <row> + <entry>Inequality</entry> + <entry> + <replaceable>e1</replaceable> <literal>!=</literal> <replaceable>e2</replaceable> + </entry> + <entry>none</entry> + <entry>Inequality.</entry> + <entry>11</entry> + </row> + <row> + <entry>Logical AND</entry> + <entry><replaceable>e1</replaceable> <literal>&&</literal> + <replaceable>e2</replaceable></entry> + <entry>left</entry> + <entry>Logical AND.</entry> + <entry>12</entry> + </row> + <row> + <entry>Logical OR</entry> + <entry><replaceable>e1</replaceable> <literal>||</literal> + <replaceable>e2</replaceable></entry> + <entry>left</entry> + <entry>Logical OR.</entry> + <entry>13</entry> + </row> + <row> + <entry>Logical Implication</entry> + <entry><replaceable>e1</replaceable> <literal>-></literal> + <replaceable>e2</replaceable></entry> + <entry>none</entry> + <entry>Logical implication (equivalent to + <literal>!<replaceable>e1</replaceable> || + <replaceable>e2</replaceable></literal>).</entry> + <entry>14</entry> + </row> + </tbody> + </tgroup> +</table> + +</section> diff --git a/third_party/nix/doc/manual/expressions/language-values.xml b/third_party/nix/doc/manual/expressions/language-values.xml new file mode 100644 index 000000000000..bb2090c881fd --- /dev/null +++ b/third_party/nix/doc/manual/expressions/language-values.xml @@ -0,0 +1,313 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='ssec-values'> + +<title>Values</title> + + +<simplesect><title>Simple Values</title> + +<para>Nix has the following basic data types: + +<itemizedlist> + + <listitem> + + <para><emphasis>Strings</emphasis> can be written in three + ways.</para> + + <para>The most common way is to enclose the string between double + quotes, e.g., <literal>"foo bar"</literal>. Strings can span + multiple lines. The special characters <literal>"</literal> and + <literal>\</literal> and the character sequence + <literal>${</literal> must be escaped by prefixing them with a + backslash (<literal>\</literal>). Newlines, carriage returns and + tabs can be written as <literal>\n</literal>, + <literal>\r</literal> and <literal>\t</literal>, + respectively.</para> + + <para>You can include the result of an expression into a string by + enclosing it in + <literal>${<replaceable>...</replaceable>}</literal>, a feature + known as <emphasis>antiquotation</emphasis>. The enclosed + expression must evaluate to something that can be coerced into a + string (meaning that it must be a string, a path, or a + derivation). For instance, rather than writing + +<programlisting> +"--with-freetype2-library=" + freetype + "/lib"</programlisting> + + (where <varname>freetype</varname> is a derivation), you can + instead write the more natural + +<programlisting> +"--with-freetype2-library=${freetype}/lib"</programlisting> + + The latter is automatically translated to the former. A more + complicated example (from the Nix expression for <link + xlink:href='http://www.trolltech.com/products/qt'>Qt</link>): + +<programlisting> +configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} +";</programlisting> + + Note that Nix expressions and strings can be arbitrarily nested; + in this case the outer string contains various antiquotations that + themselves contain strings (e.g., <literal>"-thread"</literal>), + some of which in turn contain expressions (e.g., + <literal>${mesa}</literal>).</para> + + <para>The second way to write string literals is as an + <emphasis>indented string</emphasis>, which is enclosed between + pairs of <emphasis>double single-quotes</emphasis>, like so: + +<programlisting> +'' + This is the first line. + This is the second line. + This is the third line. +''</programlisting> + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as + a whole (disregarding the indentation of empty lines). For + instance, the first and second line are indented two space, while + the third line is indented four spaces. Thus, two spaces are + stripped from each line, so the resulting string is + +<programlisting> +"This is the first line.\nThis is the second line.\n This is the third line.\n"</programlisting> + + </para> + + <para>Note that the whitespace and newline following the opening + <literal>''</literal> is ignored if there is no non-whitespace + text on the initial line.</para> + + <para>Antiquotation + (<literal>${<replaceable>expr</replaceable>}</literal>) is + supported in indented strings.</para> + + <para>Since <literal>${</literal> and <literal>''</literal> have + special meaning in indented strings, you need a way to quote them. + <literal>$</literal> can be escaped by prefixing it with + <literal>''</literal> (that is, two single quotes), i.e., + <literal>''$</literal>. <literal>''</literal> can be escaped by + prefixing it with <literal>'</literal>, i.e., + <literal>'''</literal>. <literal>$</literal> removes any special meaning + from the following <literal>$</literal>. Linefeed, carriage-return and tab + characters can be written as <literal>''\n</literal>, + <literal>''\r</literal>, <literal>''\t</literal>, and <literal>''\</literal> + escapes any other character. + + </para> + + <para>Indented strings are primarily useful in that they allow + multi-line string literals to follow the indentation of the + enclosing Nix expression, and that less escaping is typically + necessary for strings representing languages such as shell scripts + and configuration files because <literal>''</literal> is much less + common than <literal>"</literal>. Example: + +<programlisting> +stdenv.mkDerivation { + <replaceable>...</replaceable> + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + <replaceable>...</replaceable> +} +</programlisting> + + </para> + + <para>Finally, as a convenience, <emphasis>URIs</emphasis> as + defined in appendix B of <link + xlink:href='http://www.ietf.org/rfc/rfc2396.txt'>RFC 2396</link> + can be written <emphasis>as is</emphasis>, without quotes. For + instance, the string + <literal>"http://example.org/foo.tar.bz2"</literal> + can also be written as + <literal>http://example.org/foo.tar.bz2</literal>.</para> + + </listitem> + + <listitem><para>Numbers, which can be <emphasis>integers</emphasis> (like + <literal>123</literal>) or <emphasis>floating point</emphasis> (like + <literal>123.43</literal> or <literal>.27e13</literal>).</para> + + <para>Numbers are type-compatible: pure integer operations will always + return integers, whereas any operation involving at least one floating point + number will have a floating point number as a result.</para></listitem> + + <listitem><para><emphasis>Paths</emphasis>, e.g., + <filename>/bin/sh</filename> or <filename>./builder.sh</filename>. + A path must contain at least one slash to be recognised as such; for + instance, <filename>builder.sh</filename> is not a + path<footnote><para>It's parsed as an expression that selects the + attribute <varname>sh</varname> from the variable + <varname>builder</varname>.</para></footnote>. If the file name is + relative, i.e., if it does not begin with a slash, it is made + absolute at parse time relative to the directory of the Nix + expression that contained it. For instance, if a Nix expression in + <filename>/foo/bar/bla.nix</filename> refers to + <filename>../xyzzy/fnord.nix</filename>, the absolute path is + <filename>/foo/xyzzy/fnord.nix</filename>.</para> + + <para>If the first component of a path is a <literal>~</literal>, + it is interpreted as if the rest of the path were relative to the + user's home directory. e.g. <filename>~/foo</filename> would be + equivalent to <filename>/home/edolstra/foo</filename> for a user + whose home directory is <filename>/home/edolstra</filename>. + </para> + + <para>Paths can also be specified between angle brackets, e.g. + <literal><nixpkgs></literal>. This means that the directories + listed in the environment variable + <envar linkend="env-NIX_PATH">NIX_PATH</envar> will be searched + for the given file or directory name. + </para> + + </listitem> + + <listitem><para><emphasis>Booleans</emphasis> with values + <literal>true</literal> and + <literal>false</literal>.</para></listitem> + + <listitem><para>The null value, denoted as + <literal>null</literal>.</para></listitem> + +</itemizedlist> + +</para> + +</simplesect> + + +<simplesect><title>Lists</title> + +<para>Lists are formed by enclosing a whitespace-separated list of +values between square brackets. For example, + +<programlisting> +[ 123 ./foo.nix "abc" (f { x = y; }) ]</programlisting> + +defines a list of four elements, the last being the result of a call +to the function <varname>f</varname>. Note that function calls have +to be enclosed in parentheses. If they had been omitted, e.g., + +<programlisting> +[ 123 ./foo.nix "abc" f { x = y; } ]</programlisting> + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set.</para> + +<para>Note that lists are only lazy in values, and they are strict in length. +</para> + +</simplesect> + + +<simplesect><title>Sets</title> + +<para>Sets are really the core of the language, since ultimately the +Nix language is all about creating derivations, which are really just +sets of attributes to be passed to build scripts.</para> + +<para>Sets are just a list of name/value pairs (called +<emphasis>attributes</emphasis>) enclosed in curly brackets, where +each value is an arbitrary expression terminated by a semicolon. For +example: + +<programlisting> +{ x = 123; + text = "Hello"; + y = f { bla = 456; }; +}</programlisting> + +This defines a set with attributes named <varname>x</varname>, +<varname>text</varname>, <varname>y</varname>. The order of the +attributes is irrelevant. An attribute name may only occur +once.</para> + +<para>Attributes can be selected from a set using the +<literal>.</literal> operator. For instance, + +<programlisting> +{ a = "Foo"; b = "Bar"; }.a</programlisting> + +evaluates to <literal>"Foo"</literal>. It is possible to provide a +default value in an attribute selection using the +<literal>or</literal> keyword. For example, + +<programlisting> +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy"</programlisting> + +will evaluate to <literal>"Xyzzy"</literal> because there is no +<varname>c</varname> attribute in the set.</para> + +<para>You can use arbitrary double-quoted strings as attribute +names: + +<programlisting> +{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}" +</programlisting> + +This will evaluate to <literal>123</literal> (Assuming +<literal>bar</literal> is antiquotable). In the case where an +attribute name is just a single antiquotation, the quotes can be +dropped: + +<programlisting> +{ foo = 123; }.${bar} or 456 </programlisting> + +This will evaluate to <literal>123</literal> if +<literal>bar</literal> evaluates to <literal>"foo"</literal> when +coerced to a string and <literal>456</literal> otherwise (again +assuming <literal>bar</literal> is antiquotable).</para> + +<para>In the special case where an attribute name inside of a set declaration +evaluates to <literal>null</literal> (which is normally an error, as +<literal>null</literal> is not antiquotable), that attribute is simply not +added to the set: + +<programlisting> +{ ${if foo then "bar" else null} = true; }</programlisting> + +This will evaluate to <literal>{}</literal> if <literal>foo</literal> +evaluates to <literal>false</literal>.</para> + +<para>A set that has a <literal>__functor</literal> attribute whose value +is callable (i.e. is itself a function or a set with a +<literal>__functor</literal> attribute whose value is callable) can be +applied as if it were a function, with the set itself passed in first +, e.g., + +<programlisting> +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +</programlisting> + +evaluates to <literal>2</literal>. This can be used to attach metadata to a +function without the caller needing to treat it specially, or to implement +a form of object-oriented programming, for example. + +</para> + +</simplesect> + + +</section> diff --git a/third_party/nix/doc/manual/expressions/simple-building-testing.xml b/third_party/nix/doc/manual/expressions/simple-building-testing.xml new file mode 100644 index 000000000000..7326a3e76ad5 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/simple-building-testing.xml @@ -0,0 +1,84 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-building-simple'> + +<title>Building and Testing</title> + +<para>You can now try to build Hello. Of course, you could do +<literal>nix-env -i hello</literal>, but you may not want to install a +possibly broken package just yet. The best way to test the package is by +using the command <command linkend="sec-nix-build">nix-build</command>, +which builds a Nix expression and creates a symlink named +<filename>result</filename> in the current directory: + +<screen> +$ nix-build -A hello +building path `/nix/store/632d2b22514d...-hello-2.1.1' +hello-2.1.1/ +hello-2.1.1/intl/ +hello-2.1.1/intl/ChangeLog +<replaceable>...</replaceable> + +$ ls -l result +lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1 + +$ ./result/bin/hello +Hello, world!</screen> + +The <link linkend='opt-attr'><option>-A</option></link> option selects +the <literal>hello</literal> attribute. This is faster than using the +symbolic package name specified by the <literal>name</literal> +attribute (which also happens to be <literal>hello</literal>) and is +unambiguous (there can be multiple packages with the symbolic name +<literal>hello</literal>, but there can be only one attribute in a set +named <literal>hello</literal>).</para> + +<para><command>nix-build</command> registers the +<filename>./result</filename> symlink as a garbage collection root, so +unless and until you delete the <filename>./result</filename> symlink, +the output of the build will be safely kept on your system. You can +use <command>nix-build</command>’s <option +linkend='opt-out-link'>-o</option> switch to give the symlink another +name.</para> + +<para>Nix has transactional semantics. Once a build finishes +successfully, Nix makes a note of this in its database: it registers +that the path denoted by <envar>out</envar> is now +<quote>valid</quote>. If you try to build the derivation again, Nix +will see that the path is already valid and finish immediately. If a +build fails, either because it returns a non-zero exit code, because +Nix or the builder are killed, or because the machine crashes, then +the output paths will not be registered as valid. If you try to build +the derivation again, Nix will remove the output paths if they exist +(e.g., because the builder died half-way through <literal>make +install</literal>) and try again. Note that there is no +<quote>negative caching</quote>: Nix doesn't remember that a build +failed, and so a failed build can always be repeated. This is because +Nix cannot distinguish between permanent failures (e.g., a compiler +error due to a syntax error in the source) and transient failures +(e.g., a disk full condition).</para> + +<para>Nix also performs locking. If you run multiple Nix builds +simultaneously, and they try to build the same derivation, the first +Nix instance that gets there will perform the build, while the others +block (or perform other derivations if available) until the build +finishes: + +<screen> +$ nix-build -A hello +waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x'</screen> + +So it is always safe to run multiple instances of Nix in parallel +(which isn’t the case with, say, <command>make</command>).</para> + +<para>If you have a system with multiple CPUs, you may want to have +Nix build different derivations in parallel (insofar as possible). +Just pass the option <link linkend='opt-max-jobs'><option>-j +<replaceable>N</replaceable></option></link>, where +<replaceable>N</replaceable> is the maximum number of jobs to be run +in parallel, or set. Typically this should be the number of +CPUs.</para> + +</section> diff --git a/third_party/nix/doc/manual/expressions/simple-expression.xml b/third_party/nix/doc/manual/expressions/simple-expression.xml new file mode 100644 index 000000000000..29fd872eea19 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/simple-expression.xml @@ -0,0 +1,47 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-simple-expression"> + +<title>A Simple Nix Expression</title> + +<para>This section shows how to add and test the <link +xlink:href='http://www.gnu.org/software/hello/hello.html'>GNU Hello +package</link> to the Nix Packages collection. Hello is a program +that prints out the text <quote>Hello, world!</quote>.</para> + +<para>To add a package to the Nix Packages collection, you generally +need to do three things: + +<orderedlist> + + <listitem><para>Write a Nix expression for the package. This is a + file that describes all the inputs involved in building the package, + such as dependencies, sources, and so on.</para></listitem> + + <listitem><para>Write a <emphasis>builder</emphasis>. This is a + shell script<footnote><para>In fact, it can be written in any + language, but typically it's a <command>bash</command> shell + script.</para></footnote> that actually builds the package from + the inputs.</para></listitem> + + <listitem><para>Add the package to the file + <filename>pkgs/top-level/all-packages.nix</filename>. The Nix + expression written in the first step is a + <emphasis>function</emphasis>; it requires other packages in order + to build it. In this step you put it all together, i.e., you call + the function with the right arguments to build the actual + package.</para></listitem> + +</orderedlist> + +</para> + +<xi:include href="expression-syntax.xml" /> +<xi:include href="build-script.xml" /> +<xi:include href="arguments-variables.xml" /> +<xi:include href="simple-building-testing.xml" /> +<xi:include href="generic-builder.xml" /> + +</chapter> diff --git a/third_party/nix/doc/manual/expressions/writing-nix-expressions.xml b/third_party/nix/doc/manual/expressions/writing-nix-expressions.xml new file mode 100644 index 000000000000..6646dddf0842 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/writing-nix-expressions.xml @@ -0,0 +1,26 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='chap-writing-nix-expressions'> + +<title>Writing Nix Expressions</title> + +<partintro> +<para>This chapter shows you how to write Nix expressions, which +instruct Nix how to build packages. It starts with a +simple example (a Nix expression for GNU Hello), and then moves +on to a more in-depth look at the Nix expression language.</para> + +<note><para>This chapter is mostly about the Nix expression language. +For more extensive information on adding packages to the Nix Packages +collection (such as functions in the standard environment and coding +conventions), please consult <link +xlink:href="http://nixos.org/nixpkgs/manual/">its +manual</link>.</para></note> +</partintro> + +<xi:include href="simple-expression.xml" /> +<xi:include href="expression-language.xml" /> + +</part> diff --git a/third_party/nix/doc/manual/figures/user-environments.png b/third_party/nix/doc/manual/figures/user-environments.png new file mode 100644 index 000000000000..1f781cf23ce5 --- /dev/null +++ b/third_party/nix/doc/manual/figures/user-environments.png Binary files differdiff --git a/third_party/nix/doc/manual/figures/user-environments.sxd b/third_party/nix/doc/manual/figures/user-environments.sxd new file mode 100644 index 000000000000..bc661b6406fb --- /dev/null +++ b/third_party/nix/doc/manual/figures/user-environments.sxd Binary files differdiff --git a/third_party/nix/doc/manual/glossary/glossary.xml b/third_party/nix/doc/manual/glossary/glossary.xml new file mode 100644 index 000000000000..e3162ed8d469 --- /dev/null +++ b/third_party/nix/doc/manual/glossary/glossary.xml @@ -0,0 +1,199 @@ +<appendix xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xml:id="part-glossary"> + +<title>Glossary</title> + + +<glosslist> + + +<glossentry xml:id="gloss-derivation"><glossterm>derivation</glossterm> + + <glossdef><para>A description of a build action. The result of a + derivation is a store object. Derivations are typically specified + in Nix expressions using the <link + linkend="ssec-derivation"><function>derivation</function> + primitive</link>. These are translated into low-level + <emphasis>store derivations</emphasis> (implicitly by + <command>nix-env</command> and <command>nix-build</command>, or + explicitly by <command>nix-instantiate</command>).</para></glossdef> + +</glossentry> + + +<glossentry><glossterm>store</glossterm> + + <glossdef><para>The location in the file system where store objects + live. Typically <filename>/nix/store</filename>.</para></glossdef> + +</glossentry> + + +<glossentry><glossterm>store path</glossterm> + + <glossdef><para>The location in the file system of a store object, + i.e., an immediate child of the Nix store + directory.</para></glossdef> + +</glossentry> + + +<glossentry><glossterm>store object</glossterm> + + <glossdef><para>A file that is an immediate child of the Nix store + directory. These can be regular files, but also entire directory + trees. Store objects can be sources (objects copied from outside of + the store), derivation outputs (objects produced by running a build + action), or derivations (files describing a build + action).</para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-substitute"><glossterm>substitute</glossterm> + + <glossdef><para>A substitute is a command invocation stored in the + Nix database that describes how to build a store object, bypassing + the normal build mechanism (i.e., derivations). Typically, the + substitute builds the store object by downloading a pre-built + version of the store object from some server.</para></glossdef> + +</glossentry> + + +<glossentry><glossterm>purity</glossterm> + + <glossdef><para>The assumption that equal Nix derivations when run + always produce the same output. This cannot be guaranteed in + general (e.g., a builder can rely on external inputs such as the + network or the system time) but the Nix model assumes + it.</para></glossdef> + +</glossentry> + + +<glossentry><glossterm>Nix expression</glossterm> + + <glossdef><para>A high-level description of software packages and + compositions thereof. Deploying software using Nix entails writing + Nix expressions for your packages. Nix expressions are translated + to derivations that are stored in the Nix store. These derivations + can then be built.</para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-reference"><glossterm>reference</glossterm> + + <glossdef> + <para>A store path <varname>P</varname> is said to have a + reference to a store path <varname>Q</varname> if the store object + at <varname>P</varname> contains the path <varname>Q</varname> + somewhere. The <emphasis>references</emphasis> of a store path are + the set of store paths to which it has a reference. + </para> + <para>A derivation can reference other derivations and sources + (but not output paths), whereas an output path only references other + output paths. + </para> + </glossdef> + +</glossentry> + +<glossentry xml:id="gloss-reachable"><glossterm>reachable</glossterm> + + <glossdef><para>A store path <varname>Q</varname> is reachable from + another store path <varname>P</varname> if <varname>Q</varname> is in the + <link linkend="gloss-closure">closure</link> of the + <link linkend="gloss-reference">references</link> relation. + </para></glossdef> +</glossentry> + +<glossentry xml:id="gloss-closure"><glossterm>closure</glossterm> + + <glossdef><para>The closure of a store path is the set of store + paths that are directly or indirectly “reachable” from that store + path; that is, it’s the closure of the path under the <link + linkend="gloss-reference">references</link> relation. For a package, the + closure of its derivation is equivalent to the build-time + dependencies, while the closure of its output path is equivalent to its + runtime dependencies. For correct deployment it is necessary to deploy whole + closures, since otherwise at runtime files could be missing. The command + <command>nix-store -qR</command> prints out closures of store paths. + </para> + <para>As an example, if the store object at path <varname>P</varname> contains + a reference to path <varname>Q</varname>, then <varname>Q</varname> is + in the closure of <varname>P</varname>. Further, if <varname>Q</varname> + references <varname>R</varname> then <varname>R</varname> is also in + the closure of <varname>P</varname>. + </para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-output-path"><glossterm>output path</glossterm> + + <glossdef><para>A store path produced by a derivation.</para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-deriver"><glossterm>deriver</glossterm> + + <glossdef><para>The deriver of an <link + linkend="gloss-output-path">output path</link> is the store + derivation that built it.</para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-validity"><glossterm>validity</glossterm> + + <glossdef><para>A store path is considered + <emphasis>valid</emphasis> if it exists in the file system, is + listed in the Nix database as being valid, and if all paths in its + closure are also valid.</para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-user-env"><glossterm>user environment</glossterm> + + <glossdef><para>An automatically generated store object that + consists of a set of symlinks to “active” applications, i.e., other + store paths. These are generated automatically by <link + linkend="sec-nix-env"><command>nix-env</command></link>. See <xref + linkend="sec-profiles" />.</para> + + </glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-profile"><glossterm>profile</glossterm> + + <glossdef><para>A symlink to the current <link + linkend="gloss-user-env">user environment</link> of a user, e.g., + <filename>/nix/var/nix/profiles/default</filename>.</para></glossdef> + +</glossentry> + + +<glossentry xml:id="gloss-nar"><glossterm>NAR</glossterm> + + <glossdef><para>A <emphasis>N</emphasis>ix + <emphasis>AR</emphasis>chive. This is a serialisation of a path in + the Nix store. It can contain regular files, directories and + symbolic links. NARs are generated and unpacked using + <command>nix-store --dump</command> and <command>nix-store + --restore</command>.</para></glossdef> + +</glossentry> + + + +</glosslist> + + +</appendix> diff --git a/third_party/nix/doc/manual/hacking.xml b/third_party/nix/doc/manual/hacking.xml new file mode 100644 index 000000000000..b671811d3a30 --- /dev/null +++ b/third_party/nix/doc/manual/hacking.xml @@ -0,0 +1,41 @@ +<appendix xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xml:id="chap-hacking"> + +<title>Hacking</title> + +<para>This section provides some notes on how to hack on Nix. To get +the latest version of Nix from GitHub: +<screen> +$ git clone git://github.com/NixOS/nix.git +$ cd nix +</screen> +</para> + +<para>To build it and its dependencies: +<screen> +$ nix-build release.nix -A build.x86_64-linux +</screen> +</para> + +<para>To build all dependencies and start a shell in which all +environment variables are set up so that those dependencies can be +found: +<screen> +$ nix-shell +</screen> +To build Nix itself in this shell: +<screen> +[nix-shell]$ ./bootstrap.sh +[nix-shell]$ configurePhase +[nix-shell]$ make +</screen> +To install it in <literal>$(pwd)/inst</literal> and test it: +<screen> +[nix-shell]$ make install +[nix-shell]$ make installcheck +</screen> + +</para> + +</appendix> diff --git a/third_party/nix/doc/manual/images/callouts/1.gif b/third_party/nix/doc/manual/images/callouts/1.gif new file mode 100644 index 000000000000..9e7a87f75461 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/1.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/10.gif b/third_party/nix/doc/manual/images/callouts/10.gif new file mode 100644 index 000000000000..e80f7f8e632e --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/10.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/11.gif b/third_party/nix/doc/manual/images/callouts/11.gif new file mode 100644 index 000000000000..67f91a239d66 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/11.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/12.gif b/third_party/nix/doc/manual/images/callouts/12.gif new file mode 100644 index 000000000000..54c4b42f1901 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/12.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/13.gif b/third_party/nix/doc/manual/images/callouts/13.gif new file mode 100644 index 000000000000..dd5d7d9b6439 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/13.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/14.gif b/third_party/nix/doc/manual/images/callouts/14.gif new file mode 100644 index 000000000000..3d7a952a3105 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/14.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/15.gif b/third_party/nix/doc/manual/images/callouts/15.gif new file mode 100644 index 000000000000..1c9183d5bb61 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/15.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/2.gif b/third_party/nix/doc/manual/images/callouts/2.gif new file mode 100644 index 000000000000..94d42a30f99b --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/2.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/3.gif b/third_party/nix/doc/manual/images/callouts/3.gif new file mode 100644 index 000000000000..dd3541a1bc25 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/3.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/4.gif b/third_party/nix/doc/manual/images/callouts/4.gif new file mode 100644 index 000000000000..4bcbf7e31a17 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/4.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/5.gif b/third_party/nix/doc/manual/images/callouts/5.gif new file mode 100644 index 000000000000..1c62b4f92093 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/5.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/6.gif b/third_party/nix/doc/manual/images/callouts/6.gif new file mode 100644 index 000000000000..23bc5555d2a4 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/6.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/7.gif b/third_party/nix/doc/manual/images/callouts/7.gif new file mode 100644 index 000000000000..e55ce89585a8 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/7.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/8.gif b/third_party/nix/doc/manual/images/callouts/8.gif new file mode 100644 index 000000000000..49375e09f4cc --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/8.gif Binary files differdiff --git a/third_party/nix/doc/manual/images/callouts/9.gif b/third_party/nix/doc/manual/images/callouts/9.gif new file mode 100644 index 000000000000..da12a4fe2825 --- /dev/null +++ b/third_party/nix/doc/manual/images/callouts/9.gif Binary files differdiff --git a/third_party/nix/doc/manual/installation/building-source.xml b/third_party/nix/doc/manual/installation/building-source.xml new file mode 100644 index 000000000000..772cda9cc36c --- /dev/null +++ b/third_party/nix/doc/manual/installation/building-source.xml @@ -0,0 +1,49 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-building-source"> + +<title>Building Nix from Source</title> + +<para>After unpacking or checking out the Nix sources, issue the +following commands: + +<screen> +$ ./configure <replaceable>options...</replaceable> +$ make +$ make install</screen> + +Nix requires GNU Make so you may need to invoke +<command>gmake</command> instead.</para> + +<para>When building from the Git repository, these should be preceded +by the command: + +<screen> +$ ./bootstrap.sh</screen> + +</para> + +<para>The installation path can be specified by passing the +<option>--prefix=<replaceable>prefix</replaceable></option> to +<command>configure</command>. The default installation directory is +<filename>/usr/local</filename>. You can change this to any location +you like. You must have write permission to the +<replaceable>prefix</replaceable> path.</para> + +<para>Nix keeps its <emphasis>store</emphasis> (the place where +packages are stored) in <filename>/nix/store</filename> by default. +This can be changed using +<option>--with-store-dir=<replaceable>path</replaceable></option>.</para> + +<warning><para>It is best <emphasis>not</emphasis> to change the Nix +store from its default, since doing so makes it impossible to use +pre-built binaries from the standard Nixpkgs channels — that is, all +packages will need to be built from source.</para></warning> + +<para>Nix keeps state (such as its database and log files) in +<filename>/nix/var</filename> by default. This can be changed using +<option>--localstatedir=<replaceable>path</replaceable></option>.</para> + +</section> diff --git a/third_party/nix/doc/manual/installation/env-variables.xml b/third_party/nix/doc/manual/installation/env-variables.xml new file mode 100644 index 000000000000..e2b8fc867cd3 --- /dev/null +++ b/third_party/nix/doc/manual/installation/env-variables.xml @@ -0,0 +1,89 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-env-variables"> + +<title>Environment Variables</title> + +<para>To use Nix, some environment variables should be set. In +particular, <envar>PATH</envar> should contain the directories +<filename><replaceable>prefix</replaceable>/bin</filename> and +<filename>~/.nix-profile/bin</filename>. The first directory contains +the Nix tools themselves, while <filename>~/.nix-profile</filename> is +a symbolic link to the current <emphasis>user environment</emphasis> +(an automatically generated package consisting of symlinks to +installed packages). The simplest way to set the required environment +variables is to include the file +<filename><replaceable>prefix</replaceable>/etc/profile.d/nix.sh</filename> +in your <filename>~/.profile</filename> (or similar), like this:</para> + +<screen> +source <replaceable>prefix</replaceable>/etc/profile.d/nix.sh</screen> + +<section xml:id="sec-nix-ssl-cert-file"> + +<title><envar>NIX_SSL_CERT_FILE</envar></title> + +<para>If you need to specify a custom certificate bundle to account +for an HTTPS-intercepting man in the middle proxy, you must specify +the path to the certificate bundle in the environment variable +<envar>NIX_SSL_CERT_FILE</envar>.</para> + + +<para>If you don't specify a <envar>NIX_SSL_CERT_FILE</envar> +manually, Nix will install and use its own certificate +bundle.</para> + +<procedure> + <step><para>Set the environment variable and install Nix</para> + <screen> +$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +$ sh <(curl https://nixos.org/nix/install) +</screen></step> + + <step><para>In the shell profile and rc files (for example, + <filename>/etc/bashrc</filename>, <filename>/etc/zshrc</filename>), + add the following line:</para> +<programlisting> +export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +</programlisting> +</step> +</procedure> + +<note><para>You must not add the export and then do the install, as +the Nix installer will detect the presense of Nix configuration, and +abort.</para></note> + +<section xml:id="sec-nix-ssl-cert-file-with-nix-daemon-and-macos"> +<title><envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon</title> + +<para>On macOS you must specify the environment variable for the Nix +daemon service, then restart it:</para> + +<screen> +$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt +$ sudo launchctl kickstart -k system/org.nixos.nix-daemon +</screen> +</section> + +<section xml:id="sec-installer-proxy-settings"> + +<title>Proxy Environment Variables</title> + +<para>The Nix installer has special handling for these proxy-related +environment variables: +<varname>http_proxy</varname>, <varname>https_proxy</varname>, +<varname>ftp_proxy</varname>, <varname>no_proxy</varname>, +<varname>HTTP_PROXY</varname>, <varname>HTTPS_PROXY</varname>, +<varname>FTP_PROXY</varname>, <varname>NO_PROXY</varname>. +</para> +<para>If any of these variables are set when running the Nix installer, +then the installer will create an override file at +<filename>/etc/systemd/system/nix-daemon.service.d/override.conf</filename> +so <command>nix-daemon</command> will use them. +</para> +</section> + +</section> +</chapter> diff --git a/third_party/nix/doc/manual/installation/installation.xml b/third_party/nix/doc/manual/installation/installation.xml new file mode 100644 index 000000000000..87895935288d --- /dev/null +++ b/third_party/nix/doc/manual/installation/installation.xml @@ -0,0 +1,34 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="chap-installation"> + +<title>Installation</title> + +<partintro> +<para>This section describes how to install and configure Nix for first-time use.</para> +</partintro> + +<xi:include href="supported-platforms.xml" /> +<xi:include href="installing-binary.xml" /> +<xi:include href="installing-source.xml" /> +<xi:include href="nix-security.xml" /> +<xi:include href="env-variables.xml" /> + +<!-- TODO: should be updated +<section><title>Upgrading Nix through Nix</title> + +<para>You can install the latest stable version of Nix through Nix +itself by subscribing to the channel <link +xlink:href="http://nixos.org/releases/nix/channels/nix-stable" />, +or the latest unstable version by subscribing to the channel <link +xlink:href="http://nixos.org/releases/nix/channels/nix-unstable" />. +You can also do a <link linkend="sec-one-click">one-click +installation</link> by clicking on the package links at <link +xlink:href="http://nixos.org/releases/full-index-nix.html" />.</para> + +</section> +--> + +</part> diff --git a/third_party/nix/doc/manual/installation/installing-binary.xml b/third_party/nix/doc/manual/installation/installing-binary.xml new file mode 100644 index 000000000000..394d8053b942 --- /dev/null +++ b/third_party/nix/doc/manual/installation/installing-binary.xml @@ -0,0 +1,190 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-installing-binary"> + +<title>Installing a Binary Distribution</title> + +<para>If you are using Linux or macOS, the easiest way to install Nix +is to run the following command: + +<screen> + $ sh <(curl https://nixos.org/nix/install) +</screen> + +As of Nix 2.1.0, the Nix installer will always default to creating a +single-user installation, however opting in to the multi-user +installation is highly recommended. +</para> + +<section xml:id="sect-single-user-installation"> + <title>Single User Installation</title> + + <para> + To explicitly select a single-user installation on your system: + + <screen> + sh <(curl https://nixos.org/nix/install) --no-daemon +</screen> + </para> + +<para> +This will perform a single-user installation of Nix, meaning that +<filename>/nix</filename> is owned by the invoking user. You should +run this under your usual user account, <emphasis>not</emphasis> as +root. The script will invoke <command>sudo</command> to create +<filename>/nix</filename> if it doesn’t already exist. If you don’t +have <command>sudo</command>, you should manually create +<command>/nix</command> first as root, e.g.: + +<screen> +$ mkdir /nix +$ chown alice /nix +</screen> + +The install script will modify the first writable file from amongst +<filename>.bash_profile</filename>, <filename>.bash_login</filename> +and <filename>.profile</filename> to source +<filename>~/.nix-profile/etc/profile.d/nix.sh</filename>. You can set +the <command>NIX_INSTALLER_NO_MODIFY_PROFILE</command> environment +variable before executing the install script to disable this +behaviour. +</para> + + +<para>You can uninstall Nix simply by running: + +<screen> +$ rm -rf /nix +</screen> + +</para> +</section> + +<section xml:id="sect-multi-user-installation"> + <title>Multi User Installation</title> + <para> + The multi-user Nix installation creates system users, and a system + service for the Nix daemon. + </para> + + <itemizedlist> + <title>Supported Systems</title> + + <listitem> + <para>Linux running systemd, with SELinux disabled</para> + </listitem> + <listitem><para>macOS</para></listitem> + </itemizedlist> + + <para> + You can instruct the installer to perform a multi-user + installation on your system: + + <screen> + sh <(curl https://nixos.org/nix/install) --daemon +</screen> + </para> + + <para> + The multi-user installation of Nix will create build users between + the user IDs 30001 and 30032, and a group with the group ID 30000. + + You should run this under your usual user account, + <emphasis>not</emphasis> as root. The script will invoke + <command>sudo</command> as needed. + </para> + + <note><para> + If you need Nix to use a different group ID or user ID set, you + will have to download the tarball manually and <link + linkend="sect-nix-install-binary-tarball">edit the install + script</link>. + </para></note> + + <para> + The installer will modify <filename>/etc/bashrc</filename>, and + <filename>/etc/zshrc</filename> if they exist. The installer will + first back up these files with a + <literal>.backup-before-nix</literal> extension. The installer + will also create <filename>/etc/profile.d/nix.sh</filename>. + </para> + + <para>You can uninstall Nix with the following commands: + +<screen> +sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + +# If you are on Linux with systemd, you will need to run: +sudo systemctl stop nix-daemon.socket +sudo systemctl stop nix-daemon.service +sudo systemctl disable nix-daemon.socket +sudo systemctl disable nix-daemon.service +sudo systemctl daemon-reload + +# If you are on macOS, you will need to run: +sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist +sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist +</screen> + + There may also be references to Nix in + <filename>/etc/profile</filename>, + <filename>/etc/bashrc</filename>, and + <filename>/etc/zshrc</filename> which you may remove. + </para> + +</section> + +<section xml:id="sect-nix-install-pinned-version-url"> + <title>Installing a pinned Nix version from a URL</title> + + <para> + NixOS.org hosts version-specific installation URLs for all Nix + versions since 1.11.16, at + <literal>https://nixos.org/releases/nix/nix-VERSION/install</literal>. + </para> + + <para> + These install scripts can be used the same as the main + NixOS.org installation script: + + <screen> + sh <(curl https://nixos.org/nix/install) +</screen> + </para> + + <para> + In the same directory of the install script are sha256 sums, and + gpg signature files. + </para> +</section> + +<section xml:id="sect-nix-install-binary-tarball"> + <title>Installing from a binary tarball</title> + + <para> + You can also download a binary tarball that contains Nix and all + its dependencies. (This is what the install script at + <uri>https://nixos.org/nix/install</uri> does automatically.) You + should unpack it somewhere (e.g. in <filename>/tmp</filename>), + and then run the script named <command>install</command> inside + the binary tarball: + + +<screen> +alice$ cd /tmp +alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +alice$ cd nix-1.8-x86_64-darwin +alice$ ./install +</screen> + </para> + + <para> + If you need to edit the multi-user installation script to use + different group ID or a different user ID range, modify the + variables set in the file named + <filename>install-multi-user</filename>. + </para> +</section> +</chapter> diff --git a/third_party/nix/doc/manual/installation/installing-source.xml b/third_party/nix/doc/manual/installation/installing-source.xml new file mode 100644 index 000000000000..c261a109d6b1 --- /dev/null +++ b/third_party/nix/doc/manual/installation/installing-source.xml @@ -0,0 +1,16 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-installing-source"> + +<title>Installing Nix from Source</title> + +<para>If no binary package is available, you can download and compile +a source distribution.</para> + +<xi:include href="prerequisites-source.xml" /> +<xi:include href="obtaining-source.xml" /> +<xi:include href="building-source.xml" /> + +</chapter> diff --git a/third_party/nix/doc/manual/installation/multi-user.xml b/third_party/nix/doc/manual/installation/multi-user.xml new file mode 100644 index 000000000000..69ae1ef27041 --- /dev/null +++ b/third_party/nix/doc/manual/installation/multi-user.xml @@ -0,0 +1,107 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-multi-user"> + +<title>Multi-User Mode</title> + +<para>To allow a Nix store to be shared safely among multiple users, +it is important that users are not able to run builders that modify +the Nix store or database in arbitrary ways, or that interfere with +builds started by other users. If they could do so, they could +install a Trojan horse in some package and compromise the accounts of +other users.</para> + +<para>To prevent this, the Nix store and database are owned by some +privileged user (usually <literal>root</literal>) and builders are +executed under special user accounts (usually named +<literal>nixbld1</literal>, <literal>nixbld2</literal>, etc.). When a +unprivileged user runs a Nix command, actions that operate on the Nix +store (such as builds) are forwarded to a <emphasis>Nix +daemon</emphasis> running under the owner of the Nix store/database +that performs the operation.</para> + +<note><para>Multi-user mode has one important limitation: only +<systemitem class="username">root</systemitem> and a set of trusted +users specified in <filename>nix.conf</filename> can specify arbitrary +binary caches. So while unprivileged users may install packages from +arbitrary Nix expressions, they may not get pre-built +binaries.</para></note> + + +<simplesect> + +<title>Setting up the build users</title> + +<para>The <emphasis>build users</emphasis> are the special UIDs under +which builds are performed. They should all be members of the +<emphasis>build users group</emphasis> <literal>nixbld</literal>. +This group should have no other members. The build users should not +be members of any other group. On Linux, you can create the group and +users as follows: + +<screen> +$ groupadd -r nixbld +$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \ + -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \ + nixbld$n; done +</screen> + +This creates 10 build users. There can never be more concurrent builds +than the number of build users, so you may want to increase this if +you expect to do many builds at the same time.</para> + +</simplesect> + + +<simplesect> + +<title>Running the daemon</title> + +<para>The <link linkend="sec-nix-daemon">Nix daemon</link> should be +started as follows (as <literal>root</literal>): + +<screen> +$ nix-daemon</screen> + +You’ll want to put that line somewhere in your system’s boot +scripts.</para> + +<para>To let unprivileged users use the daemon, they should set the +<link linkend="envar-remote"><envar>NIX_REMOTE</envar> environment +variable</link> to <literal>daemon</literal>. So you should put a +line like + +<programlisting> +export NIX_REMOTE=daemon</programlisting> + +into the users’ login scripts.</para> + +</simplesect> + + +<simplesect> + +<title>Restricting access</title> + +<para>To limit which users can perform Nix operations, you can use the +permissions on the directory +<filename>/nix/var/nix/daemon-socket</filename>. For instance, if you +want to restrict the use of Nix to the members of a group called +<literal>nix-users</literal>, do + +<screen> +$ chgrp nix-users /nix/var/nix/daemon-socket +$ chmod ug=rwx,o= /nix/var/nix/daemon-socket +</screen> + +This way, users who are not in the <literal>nix-users</literal> group +cannot connect to the Unix domain socket +<filename>/nix/var/nix/daemon-socket/socket</filename>, so they cannot +perform Nix operations.</para> + +</simplesect> + + +</section> diff --git a/third_party/nix/doc/manual/installation/nix-security.xml b/third_party/nix/doc/manual/installation/nix-security.xml new file mode 100644 index 000000000000..d888ff14d457 --- /dev/null +++ b/third_party/nix/doc/manual/installation/nix-security.xml @@ -0,0 +1,27 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-nix-security"> + +<title>Security</title> + +<para>Nix has two basic security models. First, it can be used in +“single-user mode”, which is similar to what most other package +management tools do: there is a single user (typically <systemitem +class="username">root</systemitem>) who performs all package +management operations. All other users can then use the installed +packages, but they cannot perform package management operations +themselves.</para> + +<para>Alternatively, you can configure Nix in “multi-user mode”. In +this model, all users can perform package management operations — for +instance, every user can install software without requiring root +privileges. Nix ensures that this is secure. For instance, it’s not +possible for one user to overwrite a package used by another user with +a Trojan horse.</para> + +<xi:include href="single-user.xml" /> +<xi:include href="multi-user.xml" /> + +</chapter> \ No newline at end of file diff --git a/third_party/nix/doc/manual/installation/obtaining-source.xml b/third_party/nix/doc/manual/installation/obtaining-source.xml new file mode 100644 index 000000000000..968822cc066e --- /dev/null +++ b/third_party/nix/doc/manual/installation/obtaining-source.xml @@ -0,0 +1,30 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-obtaining-source"> + +<title>Obtaining a Source Distribution</title> + +<para>The source tarball of the most recent stable release can be +downloaded from the <link +xlink:href="http://nixos.org/nix/download.html">Nix homepage</link>. +You can also grab the <link +xlink:href="http://hydra.nixos.org/job/nix/master/release/latest-finished#tabs-constituents">most +recent development release</link>.</para> + +<para>Alternatively, the most recent sources of Nix can be obtained +from its <link +xlink:href="https://github.com/NixOS/nix">Git +repository</link>. For example, the following command will check out +the latest revision into a directory called +<filename>nix</filename>:</para> + +<screen> +$ git clone https://github.com/NixOS/nix</screen> + +<para>Likewise, specific releases can be obtained from the <link +xlink:href="https://github.com/NixOS/nix/tags">tags</link> of the +repository.</para> + +</section> \ No newline at end of file diff --git a/third_party/nix/doc/manual/installation/prerequisites-source.xml b/third_party/nix/doc/manual/installation/prerequisites-source.xml new file mode 100644 index 000000000000..e7bdcf966cf6 --- /dev/null +++ b/third_party/nix/doc/manual/installation/prerequisites-source.xml @@ -0,0 +1,105 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-prerequisites-source"> + +<title>Prerequisites</title> + +<itemizedlist> + + <listitem><para>GNU Make.</para></listitem> + + <listitem><para>Bash Shell. The <literal>./configure</literal> script + relies on bashisms, so Bash is required.</para></listitem> + + <listitem><para>A version of GCC or Clang that supports C++17.</para></listitem> + + <listitem><para><command>pkg-config</command> to locate + dependencies. If your distribution does not provide it, you can get + it from <link + xlink:href="http://www.freedesktop.org/wiki/Software/pkg-config" + />.</para></listitem> + + <listitem><para>The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from <link + xlink:href="https://www.openssl.org"/>.</para></listitem> + + <listitem><para>The <literal>libbrotlienc</literal> and + <literal>libbrotlidec</literal> libraries to provide implementation + of the Brotli compression algorithm. They are available for download + from the official repository <link + xlink:href="https://github.com/google/brotli" />.</para></listitem> + + <listitem><para>The bzip2 compressor program and the + <literal>libbz2</literal> library. Thus you must have bzip2 + installed, including development headers and libraries. If your + distribution does not provide these, you can obtain bzip2 from <link + xlink:href="https://web.archive.org/web/20180624184756/http://www.bzip.org/" + />.</para></listitem> + + <listitem><para><literal>liblzma</literal>, which is provided by + XZ Utils. If your distribution does not provide this, you can + get it from <link xlink:href="https://tukaani.org/xz/"/>.</para></listitem> + + <listitem><para>cURL and its library. If your distribution does not + provide it, you can get it from <link + xlink:href="https://curl.haxx.se/"/>.</para></listitem> + + <listitem><para>The SQLite embedded database library, version 3.6.19 + or higher. If your distribution does not provide it, please install + it from <link xlink:href="http://www.sqlite.org/" />.</para></listitem> + + <listitem><para>The <link + xlink:href="http://www.hboehm.info/gc/">Boehm + garbage collector</link> to reduce the evaluator’s memory + consumption (optional). To enable it, install + <literal>pkgconfig</literal> and the Boehm garbage collector, and + pass the flag <option>--enable-gc</option> to + <command>configure</command>.</para></listitem> + + <listitem><para>The <literal>boost</literal> library of version + 1.66.0 or higher. It can be obtained from the official web site + <link xlink:href="https://www.boost.org/" />.</para></listitem> + + <listitem><para>The <literal>editline</literal> library of version + 1.14.0 or higher. It can be obtained from the its repository + <link xlink:href="https://github.com/troglobit/editline" />.</para></listitem> + + <listitem><para>The <command>xmllint</command> and + <command>xsltproc</command> programs to build this manual and the + man-pages. These are part of the <literal>libxml2</literal> and + <literal>libxslt</literal> packages, respectively. You also need + the <link + xlink:href="http://docbook.sourceforge.net/projects/xsl/">DocBook + XSL stylesheets</link> and optionally the <link + xlink:href="http://www.docbook.org/schemas/5x"> DocBook 5.0 RELAX NG + schemas</link>. Note that these are only required if you modify the + manual sources or when you are building from the Git + repository.</para></listitem> + + <listitem><para>Recent versions of Bison and Flex to build the + parser. (This is because Nix needs GLR support in Bison and + reentrancy support in Flex.) For Bison, you need version 2.6, which + can be obtained from the <link + xlink:href="ftp://alpha.gnu.org/pub/gnu/bison">GNU FTP + server</link>. For Flex, you need version 2.5.35, which is + available on <link + xlink:href="http://lex.sourceforge.net/">SourceForge</link>. + Slightly older versions may also work, but ancient versions like the + ubiquitous 2.5.4a won't. Note that these are only required if you + modify the parser or when you are building from the Git + repository.</para></listitem> + + <listitem><para>The <literal>libseccomp</literal> is used to provide + syscall filtering on Linux. This is an optional dependency and can + be disabled passing a <option>--disable-seccomp-sandboxing</option> + option to the <command>configure</command> script (Not recommended + unless your system doesn't support + <literal>libseccomp</literal>). To get the library, visit <link + xlink:href="https://github.com/seccomp/libseccomp" + />.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/installation/single-user.xml b/third_party/nix/doc/manual/installation/single-user.xml new file mode 100644 index 000000000000..09cdaa5d4898 --- /dev/null +++ b/third_party/nix/doc/manual/installation/single-user.xml @@ -0,0 +1,21 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-single-user"> + +<title>Single-User Mode</title> + +<para>In single-user mode, all Nix operations that access the database +in <filename><replaceable>prefix</replaceable>/var/nix/db</filename> +or modify the Nix store in +<filename><replaceable>prefix</replaceable>/store</filename> must be +performed under the user ID that owns those directories. This is +typically <systemitem class="username">root</systemitem>. (If you +install from RPM packages, that’s in fact the default ownership.) +However, on single-user machines, it is often convenient to +<command>chown</command> those directories to your normal user account +so that you don’t have to <command>su</command> to <systemitem +class="username">root</systemitem> all the time.</para> + +</section> \ No newline at end of file diff --git a/third_party/nix/doc/manual/installation/supported-platforms.xml b/third_party/nix/doc/manual/installation/supported-platforms.xml new file mode 100644 index 000000000000..3e74be49d1f7 --- /dev/null +++ b/third_party/nix/doc/manual/installation/supported-platforms.xml @@ -0,0 +1,36 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-supported-platforms"> + +<title>Supported Platforms</title> + +<para>Nix is currently supported on the following platforms: + +<itemizedlist> + + <listitem><para>Linux (i686, x86_64, aarch64).</para></listitem> + + <listitem><para>macOS (x86_64).</para></listitem> + + <!-- + <listitem><para>FreeBSD (only tested on Intel).</para></listitem> + --> + + <!-- + <listitem><para>Windows through <link + xlink:href="http://www.cygwin.com/">Cygwin</link>.</para> + + <warning><para>On Cygwin, Nix <emphasis>must</emphasis> be installed + on an NTFS partition. It will not work correctly on a FAT + partition.</para></warning> + + </listitem> + --> + +</itemizedlist> + +</para> + +</chapter> diff --git a/third_party/nix/doc/manual/installation/upgrading.xml b/third_party/nix/doc/manual/installation/upgrading.xml new file mode 100644 index 000000000000..30670d7fec9c --- /dev/null +++ b/third_party/nix/doc/manual/installation/upgrading.xml @@ -0,0 +1,22 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-upgrading-nix"> + + <title>Upgrading Nix</title> + + <para> + Multi-user Nix users on macOS can upgrade Nix by running: + <command>sudo -i sh -c 'nix-channel --update && + nix-env -iA nixpkgs.nix && + launchctl remove org.nixos.nix-daemon && + launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist'</command> + </para> + + + <para> + Single-user installations of Nix should run this: + <command>nix-channel --update; nix-env -iA nixpkgs.nix</command> + </para> +</chapter> diff --git a/third_party/nix/doc/manual/introduction/about-nix.xml b/third_party/nix/doc/manual/introduction/about-nix.xml new file mode 100644 index 000000000000..c21ed34ddc74 --- /dev/null +++ b/third_party/nix/doc/manual/introduction/about-nix.xml @@ -0,0 +1,268 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-about-nix"> + +<title>About Nix</title> + +<para>Nix is a <emphasis>purely functional package manager</emphasis>. +This means that it treats packages like values in purely functional +programming languages such as Haskell — they are built by functions +that don’t have side-effects, and they never change after they have +been built. Nix stores packages in the <emphasis>Nix +store</emphasis>, usually the directory +<filename>/nix/store</filename>, where each package has its own unique +subdirectory such as + +<programlisting> +/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ +</programlisting> + +where <literal>b6gvzjyb2pg0…</literal> is a unique identifier for the +package that captures all its dependencies (it’s a cryptographic hash +of the package’s build dependency graph). This enables many powerful +features.</para> + + +<simplesect><title>Multiple versions</title> + +<para>You can have multiple versions or variants of a package +installed at the same time. This is especially important when +different applications have dependencies on different versions of the +same package — it prevents the “DLL hell”. Because of the hashing +scheme, different versions of a package end up in different paths in +the Nix store, so they don’t interfere with each other.</para> + +<para>An important consequence is that operations like upgrading or +uninstalling an application cannot break other applications, since +these operations never “destructively” update or delete files that are +used by other packages.</para> + +</simplesect> + + +<simplesect><title>Complete dependencies</title> + +<para>Nix helps you make sure that package dependency specifications +are complete. In general, when you’re making a package for a package +management system like RPM, you have to specify for each package what +its dependencies are, but there are no guarantees that this +specification is complete. If you forget a dependency, then the +package will build and work correctly on <emphasis>your</emphasis> +machine if you have the dependency installed, but not on the end +user's machine if it's not there.</para> + +<para>Since Nix on the other hand doesn’t install packages in “global” +locations like <filename>/usr/bin</filename> but in package-specific +directories, the risk of incomplete dependencies is greatly reduced. +This is because tools such as compilers don’t search in per-packages +directories such as +<filename>/nix/store/5lbfaxb722zp…-openssl-0.9.8d/include</filename>, +so if a package builds correctly on your system, this is because you +specified the dependency explicitly. This takes care of the build-time +dependencies.</para> + +<para>Once a package is built, runtime dependencies are found by +scanning binaries for the hash parts of Nix store paths (such as +<literal>r8vvq9kq…</literal>). This sounds risky, but it works +extremely well.</para> + +</simplesect> + + +<simplesect><title>Multi-user support</title> + +<para>Nix has multi-user support. This means that non-privileged +users can securely install software. Each user can have a different +<emphasis>profile</emphasis>, a set of packages in the Nix store that +appear in the user’s <envar>PATH</envar>. If a user installs a +package that another user has already installed previously, the +package won’t be built or downloaded a second time. At the same time, +it is not possible for one user to inject a Trojan horse into a +package that might be used by another user.</para> + +</simplesect> + + +<simplesect><title>Atomic upgrades and rollbacks</title> + +<para>Since package management operations never overwrite packages in +the Nix store but just add new versions in different paths, they are +<emphasis>atomic</emphasis>. So during a package upgrade, there is no +time window in which the package has some files from the old version +and some files from the new version — which would be bad because a +program might well crash if it’s started during that period.</para> + +<para>And since packages aren’t overwritten, the old versions are still +there after an upgrade. This means that you can <emphasis>roll +back</emphasis> to the old version:</para> + +<screen> +$ nix-env --upgrade <replaceable>some-packages</replaceable> +$ nix-env --rollback +</screen> + +</simplesect> + + +<simplesect><title>Garbage collection</title> + +<para>When you uninstall a package like this… + +<screen> +$ nix-env --uninstall firefox +</screen> + +the package isn’t deleted from the system right away (after all, you +might want to do a rollback, or it might be in the profiles of other +users). Instead, unused packages can be deleted safely by running the +<emphasis>garbage collector</emphasis>: + +<screen> +$ nix-collect-garbage +</screen> + +This deletes all packages that aren’t in use by any user profile or by +a currently running program.</para> + +</simplesect> + + +<simplesect><title>Functional package language</title> + +<para>Packages are built from <emphasis>Nix expressions</emphasis>, +which is a simple functional language. A Nix expression describes +everything that goes into a package build action (a “derivation”): +other packages, sources, the build script, environment variables for +the build script, etc. Nix tries very hard to ensure that Nix +expressions are <emphasis>deterministic</emphasis>: building a Nix +expression twice should yield the same result.</para> + +<para>Because it’s a functional language, it’s easy to support +building variants of a package: turn the Nix expression into a +function and call it any number of times with the appropriate +arguments. Due to the hashing scheme, variants don’t conflict with +each other in the Nix store.</para> + +</simplesect> + + +<simplesect><title>Transparent source/binary deployment</title> + +<para>Nix expressions generally describe how to build a package from +source, so an installation action like + +<screen> +$ nix-env --install firefox +</screen> + +<emphasis>could</emphasis> cause quite a bit of build activity, as not +only Firefox but also all its dependencies (all the way up to the C +library and the compiler) would have to built, at least if they are +not already in the Nix store. This is a <emphasis>source deployment +model</emphasis>. For most users, building from source is not very +pleasant as it takes far too long. However, Nix can automatically +skip building from source and instead use a <emphasis>binary +cache</emphasis>, a web server that provides pre-built binaries. For +instance, when asked to build +<literal>/nix/store/b6gvzjyb2pg0…-firefox-33.1</literal> from source, +Nix would first check if the file +<uri>https://cache.nixos.org/b6gvzjyb2pg0….narinfo</uri> exists, and +if so, fetch the pre-built binary referenced from there; otherwise, it +would fall back to building from source.</para> + +</simplesect> + + +<!-- +<simplesect><title>Binary patching</title> + +<para>In addition to downloading binaries automatically if they’re +available, Nix can download binary deltas that patch an existing +package in the Nix store into a new version. This speeds up +upgrades.</para> + +</simplesect> +--> + + +<simplesect><title>Nix Packages collection</title> + +<para>We provide a large set of Nix expressions containing hundreds of +existing Unix packages, the <emphasis>Nix Packages +collection</emphasis> (Nixpkgs).</para> + +</simplesect> + + +<simplesect><title>Managing build environments</title> + +<para>Nix is extremely useful for developers as it makes it easy to +automatically set up the build environment for a package. Given a +Nix expression that describes the dependencies of your package, the +command <command>nix-shell</command> will build or download those +dependencies if they’re not already in your Nix store, and then start +a Bash shell in which all necessary environment variables (such as +compiler search paths) are set.</para> + +<para>For example, the following command gets all dependencies of the +Pan newsreader, as described by <link +xlink:href="https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/networking/newsreaders/pan/default.nix">its +Nix expression</link>:</para> + +<screen> +$ nix-shell '<nixpkgs>' -A pan +</screen> + +<para>You’re then dropped into a shell where you can edit, build and test +the package:</para> + +<screen> +[nix-shell]$ tar xf $src +[nix-shell]$ cd pan-* +[nix-shell]$ ./configure +[nix-shell]$ make +[nix-shell]$ ./pan/gui/pan +</screen> + +<!-- +<para>Since Nix packages are reproducible and have complete dependency +specifications, Nix makes an excellent basis for <a +href="[%root%]hydra">a continuous build system</a>.</para> +--> + +</simplesect> + + +<simplesect><title>Portability</title> + +<para>Nix runs on Linux and macOS.</para> + +</simplesect> + + +<simplesect><title>NixOS</title> + +<para>NixOS is a Linux distribution based on Nix. It uses Nix not +just for package management but also to manage the system +configuration (e.g., to build configuration files in +<filename>/etc</filename>). This means, among other things, that it +is easy to roll back the entire configuration of the system to an +earlier state. Also, users can install software without root +privileges. For more information and downloads, see the <link +xlink:href="http://nixos.org/">NixOS homepage</link>.</para> + +</simplesect> + + +<simplesect><title>License</title> + +<para>Nix is released under the terms of the <link +xlink:href="http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html">GNU +LGPLv2.1 or (at your option) any later version</link>.</para> + +</simplesect> + + +</chapter> diff --git a/third_party/nix/doc/manual/introduction/introduction.xml b/third_party/nix/doc/manual/introduction/introduction.xml new file mode 100644 index 000000000000..12b2cc761063 --- /dev/null +++ b/third_party/nix/doc/manual/introduction/introduction.xml @@ -0,0 +1,12 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="chap-introduction"> + +<title>Introduction</title> + +<xi:include href="about-nix.xml" /> +<xi:include href="quick-start.xml" /> + +</part> diff --git a/third_party/nix/doc/manual/introduction/quick-start.xml b/third_party/nix/doc/manual/introduction/quick-start.xml new file mode 100644 index 000000000000..1ce6c8d50a1b --- /dev/null +++ b/third_party/nix/doc/manual/introduction/quick-start.xml @@ -0,0 +1,124 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="chap-quick-start"> + +<title>Quick Start</title> + +<para>This chapter is for impatient people who don't like reading +documentation. For more in-depth information you are kindly referred +to subsequent chapters.</para> + +<procedure> + +<step><para>Install single-user Nix by running the following: + +<screen> +$ bash <(curl https://nixos.org/nix/install) +</screen> + +This will install Nix in <filename>/nix</filename>. The install script +will create <filename>/nix</filename> using <command>sudo</command>, +so make sure you have sufficient rights. (For other installation +methods, see <xref linkend="chap-installation"/>.)</para></step> + +<step><para>See what installable packages are currently available +in the channel: + +<screen> +$ nix-env -qa +docbook-xml-4.3 +docbook-xml-4.5 +firefox-33.0.2 +hello-2.9 +libxslt-1.1.28 +<replaceable>...</replaceable></screen> + +</para></step> + +<step><para>Install some packages from the channel: + +<screen> +$ nix-env -i hello</screen> + +This should download pre-built packages; it should not build them +locally (if it does, something went wrong).</para></step> + +<step><para>Test that they work: + +<screen> +$ which hello +/home/eelco/.nix-profile/bin/hello +$ hello +Hello, world! +</screen> + +</para></step> + +<step><para>Uninstall a package: + +<screen> +$ nix-env -e hello</screen> + +</para></step> + +<step><para>You can also test a package without installing it: + +<screen> +$ nix-shell -p hello +</screen> + +This builds or downloads GNU Hello and its dependencies, then drops +you into a Bash shell where the <command>hello</command> command is +present, all without affecting your normal environment: + +<screen> +[nix-shell:~]$ hello +Hello, world! + +[nix-shell:~]$ exit + +$ hello +hello: command not found +</screen> + +</para></step> + +<step><para>To keep up-to-date with the channel, do: + +<screen> +$ nix-channel --update nixpkgs +$ nix-env -u '*'</screen> + +The latter command will upgrade each installed package for which there +is a “newer” version (as determined by comparing the version +numbers).</para></step> + +<step><para>If you're unhappy with the result of a +<command>nix-env</command> action (e.g., an upgraded package turned +out not to work properly), you can go back: + +<screen> +$ nix-env --rollback</screen> + +</para></step> + +<step><para>You should periodically run the Nix garbage collector +to get rid of unused packages, since uninstalls or upgrades don't +actually delete them: + +<screen> +$ nix-collect-garbage -d</screen> + +<!-- +The first command deletes old “generations” of your profile (making +rollbacks impossible, but also making the packages in those old +generations available for garbage collection), while the second +command actually deletes them.--> + +</para></step> + +</procedure> + +</chapter> diff --git a/third_party/nix/doc/manual/manual.xml b/third_party/nix/doc/manual/manual.xml new file mode 100644 index 000000000000..87d9de28ab14 --- /dev/null +++ b/third_party/nix/doc/manual/manual.xml @@ -0,0 +1,52 @@ +<book xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0"> + + <info> + <title>Nix Package Manager Guide</title> + <subtitle>Version <xi:include href="version.txt" parse="text" /></subtitle> + + <author> + <personname> + <firstname>Eelco</firstname> + <surname>Dolstra</surname> + </personname> + <contrib>Author</contrib> + </author> + + <copyright> + <year>2004-2018</year> + <holder>Eelco Dolstra</holder> + </copyright> + + </info> + + <!-- + <preface> + <title>Preface</title> + <para>This manual describes how to set up and use the Nix package + manager.</para> + </preface> + --> + + <xi:include href="introduction/introduction.xml" /> + <xi:include href="installation/installation.xml" /> + <xi:include href="installation/upgrading.xml" /> + <xi:include href="packages/package-management.xml" /> + <xi:include href="expressions/writing-nix-expressions.xml" /> + <xi:include href="advanced-topics/advanced-topics.xml" /> + <xi:include href="command-ref/command-ref.xml" /> + <xi:include href="glossary/glossary.xml" /> + <xi:include href="hacking.xml" /> + <xi:include href="release-notes/release-notes.xml" /> + +<!-- +<appendix> + <title>Nix Release Notes</title> + <xi:include href="release-notes/release-notes.xml" + xpointer="xmlns(x=http://docbook.org/ns/docbook)xpointer(x:article/x:section)" /> + </appendix> +--> + +</book> diff --git a/third_party/nix/doc/manual/nix-lang-ref.xml b/third_party/nix/doc/manual/nix-lang-ref.xml new file mode 100644 index 000000000000..86273ac3d016 --- /dev/null +++ b/third_party/nix/doc/manual/nix-lang-ref.xml @@ -0,0 +1,182 @@ +<appendix> + <title>Nix Language Reference</title> + + <sect1> + <title>Grammar</title> + + <productionset> + <title>Expressions</title> + + <production id="nix.expr"> + <lhs>Expr</lhs> + <rhs> + <nonterminal def="#nix.expr_function" /> + </rhs> + </production> + + <production id="nix.expr_function"> + <lhs>ExprFunction</lhs> + <rhs> + '{' <nonterminal def="#nix.formals" /> '}' ':' <nonterminal def="#nix.expr_function" /> + <sbr />| + <nonterminal def="#nix.expr_assert" /> + </rhs> + </production> + + <production id="nix.expr_assert"> + <lhs>ExprAssert</lhs> + <rhs> + 'assert' <nonterminal def="#nix.expr" /> ';' <nonterminal def="#nix.expr_assert" /> + <sbr />| + <nonterminal def="#nix.expr_if" /> + </rhs> + </production> + + <production id="nix.expr_if"> + <lhs>ExprIf</lhs> + <rhs> + 'if' <nonterminal def="#nix.expr" /> 'then' <nonterminal def="#nix.expr" /> + 'else' <nonterminal def="#nix.expr" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> + </rhs> + </production> + + <production id="nix.expr_op"> + <lhs>ExprOp</lhs> + <rhs> + '!' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '==' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '!=' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '&&' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '||' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '->' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '//' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '~' <nonterminal def="#nix.expr_op" /> + <sbr />| + <nonterminal def="#nix.expr_op" /> '?' <nonterminal def="#nix.id" /> + <sbr />| + <nonterminal def="#nix.expr_app" /> + </rhs> + </production> + + <production id="nix.expr_app"> + <lhs>ExprApp</lhs> + <rhs> + <nonterminal def="#nix.expr_app" /> '.' <nonterminal def="#nix.expr_select" /> + <sbr />| + <nonterminal def="#nix.expr_select" /> + </rhs> + </production> + + <production id="nix.expr_select"> + <lhs>ExprSelect</lhs> + <rhs> + <nonterminal def="#nix.expr_select" /> <nonterminal def="#nix.id" /> + <sbr />| + <nonterminal def="#nix.expr_simple" /> + </rhs> + </production> + + <production id="nix.expr_simple"> + <lhs>ExprSimple</lhs> + <rhs> + <nonterminal def="#nix.id" /> | + <nonterminal def="#nix.int" /> | + <nonterminal def="#nix.str" /> | + <nonterminal def="#nix.path" /> | + <nonterminal def="#nix.uri" /> + <sbr />| + 'true' | 'false' | 'null' + <sbr />| + '(' <nonterminal def="#nix.expr" /> ')' + <sbr />| + '{' <nonterminal def="#nix.bind" />* '}' + <sbr />| + 'let' '{' <nonterminal def="#nix.bind" />* '}' + <sbr />| + 'rec' '{' <nonterminal def="#nix.bind" />* '}' + <sbr />| + '[' <nonterminal def="#nix.expr_select" />* ']' + </rhs> + </production> + + <production id="nix.bind"> + <lhs>Bind</lhs> + <rhs> + <nonterminal def="#nix.id" /> '=' <nonterminal def="#nix.expr" /> ';' + <sbr />| + 'inherit' ('(' <nonterminal def="#nix.expr" /> ')')? <nonterminal def="#nix.id" />* ';' + </rhs> + </production> + + <production id="nix.formals"> + <lhs>Formals</lhs> + <rhs> + <nonterminal def="#nix.formal" /> ',' <nonterminal def="#nix.formals" /> + | <nonterminal def="#nix.formal" /> + </rhs> + </production> + + <production id="nix.formal"> + <lhs>Formal</lhs> + <rhs> + <nonterminal def="#nix.id" /> + <sbr />| + <nonterminal def="#nix.id" /> '?' <nonterminal def="#nix.expr" /> + </rhs> + </production> + + </productionset> + + <productionset> + <title>Terminals</title> + + <production id="nix.id"> + <lhs>Id</lhs> + <rhs>[a-zA-Z\_][a-zA-Z0-9\_\']*</rhs> + </production> + + <production id="nix.int"> + <lhs>Int</lhs> + <rhs>[0-9]+</rhs> + </production> + + <production id="nix.str"> + <lhs>Str</lhs> + <rhs>\"[^\n\"]*\"</rhs> + </production> + + <production id="nix.path"> + <lhs>Path</lhs> + <rhs>[a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+</rhs> + </production> + + <production id="nix.uri"> + <lhs>Uri</lhs> + <rhs>[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+</rhs> + </production> + + <production id="nix.ws"> + <lhs>Whitespace</lhs> + <rhs> + [ \t\n]+ + <sbr />| + \#[^\n]* + <sbr />| + \/\*(.|\n)*\*\/ + </rhs> + </production> + + </productionset> + + </sect1> + +</appendix> diff --git a/third_party/nix/doc/manual/packages/basic-package-mgmt.xml b/third_party/nix/doc/manual/packages/basic-package-mgmt.xml new file mode 100644 index 000000000000..0f21297f31b9 --- /dev/null +++ b/third_party/nix/doc/manual/packages/basic-package-mgmt.xml @@ -0,0 +1,194 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-basic-package-mgmt"> + +<title>Basic Package Management</title> + +<para>The main command for package management is <link +linkend="sec-nix-env"><command>nix-env</command></link>. You can use +it to install, upgrade, and erase packages, and to query what +packages are installed or are available for installation.</para> + +<para>In Nix, different users can have different “views” +on the set of installed applications. That is, there might be lots of +applications present on the system (possibly in many different +versions), but users can have a specific selection of those active — +where “active” just means that it appears in a directory +in the user’s <envar>PATH</envar>. Such a view on the set of +installed applications is called a <emphasis>user +environment</emphasis>, which is just a directory tree consisting of +symlinks to the files of the active applications. </para> + +<para>Components are installed from a set of <emphasis>Nix +expressions</emphasis> that tell Nix how to build those packages, +including, if necessary, their dependencies. There is a collection of +Nix expressions called the Nixpkgs package collection that contains +packages ranging from basic development stuff such as GCC and Glibc, +to end-user applications like Mozilla Firefox. (Nix is however not +tied to the Nixpkgs package collection; you could write your own Nix +expressions based on Nixpkgs, or completely new ones.)</para> + +<para>You can manually download the latest version of Nixpkgs from +<link xlink:href='http://nixos.org/nixpkgs/download.html'/>. However, +it’s much more convenient to use the Nixpkgs +<emphasis>channel</emphasis>, since it makes it easy to stay up to +date with new versions of Nixpkgs. (Channels are described in more +detail in <xref linkend="sec-channels"/>.) Nixpkgs is automatically +added to your list of “subscribed” channels when you install +Nix. If this is not the case for some reason, you can add it as +follows: + +<screen> +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --update +</screen> + +</para> + +<note><para>On NixOS, you’re automatically subscribed to a NixOS +channel corresponding to your NixOS major release +(e.g. <uri>http://nixos.org/channels/nixos-14.12</uri>). A NixOS +channel is identical to the Nixpkgs channel, except that it contains +only Linux binaries and is updated only if a set of regression tests +succeed.</para></note> + +<para>You can view the set of available packages in Nixpkgs: + +<screen> +$ nix-env -qa +aterm-2.2 +bash-3.0 +binutils-2.15 +bison-1.875d +blackdown-1.4.2 +bzip2-1.0.2 +…</screen> + +The flag <option>-q</option> specifies a query operation, and +<option>-a</option> means that you want to show the “available” (i.e., +installable) packages, as opposed to the installed packages. If you +downloaded Nixpkgs yourself, or if you checked it out from GitHub, +then you need to pass the path to your Nixpkgs tree using the +<option>-f</option> flag: + +<screen> +$ nix-env -qaf <replaceable>/path/to/nixpkgs</replaceable> +</screen> + +where <replaceable>/path/to/nixpkgs</replaceable> is where you’ve +unpacked or checked out Nixpkgs.</para> + +<para>You can select specific packages by name: + +<screen> +$ nix-env -qa firefox +firefox-34.0.5 +firefox-with-plugins-34.0.5 +</screen> + +and using regular expressions: + +<screen> +$ nix-env -qa 'firefox.*' +</screen> + +</para> + +<para>It is also possible to see the <emphasis>status</emphasis> of +available packages, i.e., whether they are installed into the user +environment and/or present in the system: + +<screen> +$ nix-env -qas +… +-PS bash-3.0 +--S binutils-2.15 +IPS bison-1.875d +…</screen> + +The first character (<literal>I</literal>) indicates whether the +package is installed in your current user environment. The second +(<literal>P</literal>) indicates whether it is present on your system +(in which case installing it into your user environment would be a +very quick operation). The last one (<literal>S</literal>) indicates +whether there is a so-called <emphasis>substitute</emphasis> for the +package, which is Nix’s mechanism for doing binary deployment. It +just means that Nix knows that it can fetch a pre-built package from +somewhere (typically a network server) instead of building it +locally.</para> + +<para>You can install a package using <literal>nix-env -i</literal>. +For instance, + +<screen> +$ nix-env -i subversion</screen> + +will install the package called <literal>subversion</literal> (which +is, of course, the <link +xlink:href='http://subversion.tigris.org/'>Subversion version +management system</link>).</para> + +<note><para>When you ask Nix to install a package, it will first try +to get it in pre-compiled form from a <emphasis>binary +cache</emphasis>. By default, Nix will use the binary cache +<uri>https://cache.nixos.org</uri>; it contains binaries for most +packages in Nixpkgs. Only if no binary is available in the binary +cache, Nix will build the package from source. So if <literal>nix-env +-i subversion</literal> results in Nix building stuff from source, +then either the package is not built for your platform by the Nixpkgs +build servers, or your version of Nixpkgs is too old or too new. For +instance, if you have a very recent checkout of Nixpkgs, then the +Nixpkgs build servers may not have had a chance to build everything +and upload the resulting binaries to +<uri>https://cache.nixos.org</uri>. The Nixpkgs channel is only +updated after all binaries have been uploaded to the cache, so if you +stick to the Nixpkgs channel (rather than using a Git checkout of the +Nixpkgs tree), you will get binaries for most packages.</para></note> + +<para>Naturally, packages can also be uninstalled: + +<screen> +$ nix-env -e subversion</screen> + +</para> + +<para>Upgrading to a new version is just as easy. If you have a new +release of Nix Packages, you can do: + +<screen> +$ nix-env -u subversion</screen> + +This will <emphasis>only</emphasis> upgrade Subversion if there is a +“newer” version in the new set of Nix expressions, as +defined by some pretty arbitrary rules regarding ordering of version +numbers (which generally do what you’d expect of them). To just +unconditionally replace Subversion with whatever version is in the Nix +expressions, use <parameter>-i</parameter> instead of +<parameter>-u</parameter>; <parameter>-i</parameter> will remove +whatever version is already installed.</para> + +<para>You can also upgrade all packages for which there are newer +versions: + +<screen> +$ nix-env -u</screen> + +</para> + +<para>Sometimes it’s useful to be able to ask what +<command>nix-env</command> would do, without actually doing it. For +instance, to find out what packages would be upgraded by +<literal>nix-env -u</literal>, you can do + +<screen> +$ nix-env -u --dry-run +(dry run; not doing anything) +upgrading `libxslt-1.1.0' to `libxslt-1.1.10' +upgrading `graphviz-1.10' to `graphviz-1.12' +upgrading `coreutils-5.0' to `coreutils-5.2.1'</screen> + +</para> + +</chapter> diff --git a/third_party/nix/doc/manual/packages/binary-cache-substituter.xml b/third_party/nix/doc/manual/packages/binary-cache-substituter.xml new file mode 100644 index 000000000000..c6ceb9c80610 --- /dev/null +++ b/third_party/nix/doc/manual/packages/binary-cache-substituter.xml @@ -0,0 +1,70 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-binary-cache-substituter"> + +<title>Serving a Nix store via HTTP</title> + +<para>You can easily share the Nix store of a machine via HTTP. This +allows other machines to fetch store paths from that machine to speed +up installations. It uses the same <emphasis>binary cache</emphasis> +mechanism that Nix usually uses to fetch pre-built binaries from +<uri>https://cache.nixos.org</uri>.</para> + +<para>The daemon that handles binary cache requests via HTTP, +<command>nix-serve</command>, is not part of the Nix distribution, but +you can install it from Nixpkgs: + +<screen> +$ nix-env -i nix-serve +</screen> + +You can then start the server, listening for HTTP connections on +whatever port you like: + +<screen> +$ nix-serve -p 8080 +</screen> + +To check whether it works, try the following on the client: + +<screen> +$ curl http://avalon:8080/nix-cache-info +</screen> + +which should print something like: + +<screen> +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 30 +</screen> + +</para> + +<para>On the client side, you can tell Nix to use your binary cache +using <option>--option extra-binary-caches</option>, e.g.: + +<screen> +$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/ +</screen> + +The option <option>extra-binary-caches</option> tells Nix to use this +binary cache in addition to your default caches, such as +<uri>https://cache.nixos.org</uri>. Thus, for any path in the closure +of Firefox, Nix will first check if the path is available on the +server <literal>avalon</literal> or another binary caches. If not, it +will fall back to building from source.</para> + +<para>You can also tell Nix to always use your binary cache by adding +a line to the <filename linkend="sec-conf-file">nix.conf</filename> +configuration file like this: + +<programlisting> +binary-caches = http://avalon:8080/ https://cache.nixos.org/ +</programlisting> + +</para> + +</section> diff --git a/third_party/nix/doc/manual/packages/channels.xml b/third_party/nix/doc/manual/packages/channels.xml new file mode 100644 index 000000000000..15c119fcb1f9 --- /dev/null +++ b/third_party/nix/doc/manual/packages/channels.xml @@ -0,0 +1,57 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-channels"> + +<title>Channels</title> + +<para>If you want to stay up to date with a set of packages, it’s not +very convenient to manually download the latest set of Nix expressions +for those packages and upgrade using <command>nix-env</command>. +Fortunately, there’s a better way: <emphasis>Nix +channels</emphasis>.</para> + +<para>A Nix channel is just a URL that points to a place that contains +a set of Nix expressions and a manifest. Using the command <link +linkend="sec-nix-channel"><command>nix-channel</command></link> you +can automatically stay up to date with whatever is available at that +URL.</para> + +<para>You can “subscribe” to a channel using +<command>nix-channel --add</command>, e.g., + +<screen> +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable</screen> + +subscribes you to a channel that always contains that latest version +of the Nix Packages collection. (Subscribing really just means that +the URL is added to the file <filename>~/.nix-channels</filename>, +where it is read by subsequent calls to <command>nix-channel +--update</command>.) You can “unsubscribe” using <command>nix-channel +--remove</command>: + +<screen> +$ nix-channel --remove nixpkgs +</screen> +</para> + +<para>To obtain the latest Nix expressions available in a channel, do + +<screen> +$ nix-channel --update</screen> + +This downloads and unpacks the Nix expressions in every channel +(downloaded from <literal><replaceable>url</replaceable>/nixexprs.tar.bz2</literal>). +It also makes the union of each channel’s Nix expressions available by +default to <command>nix-env</command> operations (via the symlink +<filename>~/.nix-defexpr/channels</filename>). Consequently, you can +then say + +<screen> +$ nix-env -u</screen> + +to upgrade all packages in your profile to the latest versions +available in the subscribed channels.</para> + +</chapter> diff --git a/third_party/nix/doc/manual/packages/copy-closure.xml b/third_party/nix/doc/manual/packages/copy-closure.xml new file mode 100644 index 000000000000..012030e3eb67 --- /dev/null +++ b/third_party/nix/doc/manual/packages/copy-closure.xml @@ -0,0 +1,50 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-copy-closure"> + +<title>Copying Closures Via SSH</title> + +<para>The command <command +linkend="sec-nix-copy-closure">nix-copy-closure</command> copies a Nix +store path along with all its dependencies to or from another machine +via the SSH protocol. It doesn’t copy store paths that are already +present on the target machine. For example, the following command +copies Firefox with all its dependencies: + +<screen> +$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox)</screen> + +See <xref linkend='sec-nix-copy-closure' /> for details.</para> + +<para>With <command linkend='refsec-nix-store-export'>nix-store +--export</command> and <command +linkend='refsec-nix-store-import'>nix-store --import</command> you can +write the closure of a store path (that is, the path and all its +dependencies) to a file, and then unpack that file into another Nix +store. For example, + +<screen> +$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure</screen> + +writes the closure of Firefox to a file. You can then copy this file +to another machine and install the closure: + +<screen> +$ nix-store --import < firefox.closure</screen> + +Any store paths in the closure that are already present in the target +store are ignored. It is also possible to pipe the export into +another command, e.g. to copy and install a closure directly to/on +another machine: + +<screen> +$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ + ssh alice@itchy.example.org "bunzip2 | nix-store --import"</screen> + +However, <command>nix-copy-closure</command> is generally more +efficient because it only copies paths that are not already present in +the target Nix store.</para> + +</section> diff --git a/third_party/nix/doc/manual/packages/garbage-collection.xml b/third_party/nix/doc/manual/packages/garbage-collection.xml new file mode 100644 index 000000000000..b506f22b0302 --- /dev/null +++ b/third_party/nix/doc/manual/packages/garbage-collection.xml @@ -0,0 +1,86 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='sec-garbage-collection'> + +<title>Garbage Collection</title> + +<para><command>nix-env</command> operations such as upgrades +(<option>-u</option>) and uninstall (<option>-e</option>) never +actually delete packages from the system. All they do (as shown +above) is to create a new user environment that no longer contains +symlinks to the “deleted” packages.</para> + +<para>Of course, since disk space is not infinite, unused packages +should be removed at some point. You can do this by running the Nix +garbage collector. It will remove from the Nix store any package +not used (directly or indirectly) by any generation of any +profile.</para> + +<para>Note however that as long as old generations reference a +package, it will not be deleted. After all, we wouldn’t be able to +do a rollback otherwise. So in order for garbage collection to be +effective, you should also delete (some) old generations. Of course, +this should only be done if you are certain that you will not need to +roll back.</para> + +<para>To delete all old (non-current) generations of your current +profile: + +<screen> +$ nix-env --delete-generations old</screen> + +Instead of <literal>old</literal> you can also specify a list of +generations, e.g., + +<screen> +$ nix-env --delete-generations 10 11 14</screen> + +To delete all generations older than a specified number of days +(except the current generation), use the <literal>d</literal> +suffix. For example, + +<screen> +$ nix-env --delete-generations 14d</screen> + +deletes all generations older than two weeks.</para> + +<para>After removing appropriate old generations you can run the +garbage collector as follows: + +<screen> +$ nix-store --gc</screen> + +The behaviour of the gargage collector is affected by the +<literal>keep-derivations</literal> (default: true) and <literal>keep-outputs</literal> +(default: false) options in the Nix configuration file. The defaults will ensure +that all derivations that are build-time dependencies of garbage collector roots +will be kept and that all output paths that are runtime dependencies +will be kept as well. All other derivations or paths will be collected. +(This is usually what you want, but while you are developing +it may make sense to keep outputs to ensure that rebuild times are quick.) + +If you are feeling uncertain, you can also first view what files would +be deleted: + +<screen> +$ nix-store --gc --print-dead</screen> + +Likewise, the option <option>--print-live</option> will show the paths +that <emphasis>won’t</emphasis> be deleted.</para> + +<para>There is also a convenient little utility +<command>nix-collect-garbage</command>, which when invoked with the +<option>-d</option> (<option>--delete-old</option>) switch deletes all +old generations of all profiles in +<filename>/nix/var/nix/profiles</filename>. So + +<screen> +$ nix-collect-garbage -d</screen> + +is a quick and easy way to clean up your system.</para> + +<xi:include href="garbage-collector-roots.xml" /> + +</chapter> diff --git a/third_party/nix/doc/manual/packages/garbage-collector-roots.xml b/third_party/nix/doc/manual/packages/garbage-collector-roots.xml new file mode 100644 index 000000000000..8338e539202f --- /dev/null +++ b/third_party/nix/doc/manual/packages/garbage-collector-roots.xml @@ -0,0 +1,29 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-gc-roots"> + +<title>Garbage Collector Roots</title> + +<para>The roots of the garbage collector are all store paths to which +there are symlinks in the directory +<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename>. +For instance, the following command makes the path +<filename>/nix/store/d718ef...-foo</filename> a root of the collector: + +<screen> +$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar</screen> + +That is, after this command, the garbage collector will not remove +<filename>/nix/store/d718ef...-foo</filename> or any of its +dependencies.</para> + +<para>Subdirectories of +<filename><replaceable>prefix</replaceable>/nix/var/nix/gcroots</filename> +are also searched for symlinks. Symlinks to non-store paths are +followed and searched for roots, but symlinks to non-store paths +<emphasis>inside</emphasis> the paths reached in that way are not +followed to prevent infinite recursion.</para> + +</section> \ No newline at end of file diff --git a/third_party/nix/doc/manual/packages/package-management.xml b/third_party/nix/doc/manual/packages/package-management.xml new file mode 100644 index 000000000000..61e55faeb311 --- /dev/null +++ b/third_party/nix/doc/manual/packages/package-management.xml @@ -0,0 +1,23 @@ +<part xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id='chap-package-management'> + +<title>Package Management</title> + +<partintro> +<para>This chapter discusses how to do package management with Nix, +i.e., how to obtain, install, upgrade, and erase packages. This is +the “user’s” perspective of the Nix system — people +who want to <emphasis>create</emphasis> packages should consult +<xref linkend='chap-writing-nix-expressions' />.</para> +</partintro> + +<xi:include href="basic-package-mgmt.xml" /> +<xi:include href="profiles.xml" /> +<xi:include href="garbage-collection.xml" /> +<xi:include href="channels.xml" /> +<xi:include href="sharing-packages.xml" /> + +</part> diff --git a/third_party/nix/doc/manual/packages/profiles.xml b/third_party/nix/doc/manual/packages/profiles.xml new file mode 100644 index 000000000000..4d10319abe1c --- /dev/null +++ b/third_party/nix/doc/manual/packages/profiles.xml @@ -0,0 +1,158 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-profiles"> + +<title>Profiles</title> + +<para>Profiles and user environments are Nix’s mechanism for +implementing the ability to allow different users to have different +configurations, and to do atomic upgrades and rollbacks. To +understand how they work, it’s useful to know a bit about how Nix +works. In Nix, packages are stored in unique locations in the +<emphasis>Nix store</emphasis> (typically, +<filename>/nix/store</filename>). For instance, a particular version +of the Subversion package might be stored in a directory +<filename>/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/</filename>, +while another version might be stored in +<filename>/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2</filename>. +The long strings prefixed to the directory names are cryptographic +hashes<footnote><para>160-bit truncations of SHA-256 hashes encoded in +a base-32 notation, to be precise.</para></footnote> of +<emphasis>all</emphasis> inputs involved in building the package — +sources, dependencies, compiler flags, and so on. So if two +packages differ in any way, they end up in different locations in +the file system, so they don’t interfere with each other. <xref +linkend='fig-user-environments' /> shows a part of a typical Nix +store.</para> + +<figure xml:id='fig-user-environments'><title>User environments</title> + <mediaobject> + <imageobject> + <imagedata fileref='../figures/user-environments.png' format='PNG' /> + </imageobject> + </mediaobject> +</figure> + +<para>Of course, you wouldn’t want to type + +<screen> +$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn</screen> + +every time you want to run Subversion. Of course we could set up the +<envar>PATH</envar> environment variable to include the +<filename>bin</filename> directory of every package we want to use, +but this is not very convenient since changing <envar>PATH</envar> +doesn’t take effect for already existing processes. The solution Nix +uses is to create directory trees of symlinks to +<emphasis>activated</emphasis> packages. These are called +<emphasis>user environments</emphasis> and they are packages +themselves (though automatically generated by +<command>nix-env</command>), so they too reside in the Nix store. For +instance, in <xref linkend='fig-user-environments' /> the user +environment <filename>/nix/store/0c1p5z4kda11...-user-env</filename> +contains a symlink to just Subversion 1.1.2 (arrows in the figure +indicate symlinks). This would be what we would obtain if we had done + +<screen> +$ nix-env -i subversion</screen> + +on a set of Nix expressions that contained Subversion 1.1.2.</para> + +<para>This doesn’t in itself solve the problem, of course; you +wouldn’t want to type +<filename>/nix/store/0c1p5z4kda11...-user-env/bin/svn</filename> +either. That’s why there are symlinks outside of the store that point +to the user environments in the store; for instance, the symlinks +<filename>default-42-link</filename> and +<filename>default-43-link</filename> in the example. These are called +<emphasis>generations</emphasis> since every time you perform a +<command>nix-env</command> operation, a new user environment is +generated based on the current one. For instance, generation 43 was +created from generation 42 when we did + +<screen> +$ nix-env -i subversion firefox</screen> + +on a set of Nix expressions that contained Firefox and a new version +of Subversion.</para> + +<para>Generations are grouped together into +<emphasis>profiles</emphasis> so that different users don’t interfere +with each other if they don’t want to. For example: + +<screen> +$ ls -l /nix/var/nix/profiles/ +... +lrwxrwxrwx 1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env +lrwxrwxrwx 1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env +lrwxrwxrwx 1 eelco ... default -> default-43-link</screen> + +This shows a profile called <filename>default</filename>. The file +<filename>default</filename> itself is actually a symlink that points +to the current generation. When we do a <command>nix-env</command> +operation, a new user environment and generation link are created +based on the current one, and finally the <filename>default</filename> +symlink is made to point at the new generation. This last step is +atomic on Unix, which explains how we can do atomic upgrades. (Note +that the building/installing of new packages doesn’t interfere in +any way with old packages, since they are stored in different +locations in the Nix store.)</para> + +<para>If you find that you want to undo a <command>nix-env</command> +operation, you can just do + +<screen> +$ nix-env --rollback</screen> + +which will just make the current generation link point at the previous +link. E.g., <filename>default</filename> would be made to point at +<filename>default-42-link</filename>. You can also switch to a +specific generation: + +<screen> +$ nix-env --switch-generation 43</screen> + +which in this example would roll forward to generation 43 again. You +can also see all available generations: + +<screen> +$ nix-env --list-generations</screen></para> + +<para>You generally wouldn’t have +<filename>/nix/var/nix/profiles/<replaceable>some-profile</replaceable>/bin</filename> +in your <envar>PATH</envar>. Rather, there is a symlink +<filename>~/.nix-profile</filename> that points to your current +profile. This means that you should put +<filename>~/.nix-profile/bin</filename> in your <envar>PATH</envar> +(and indeed, that’s what the initialisation script +<filename>/nix/etc/profile.d/nix.sh</filename> does). This makes it +easier to switch to a different profile. You can do that using the +command <command>nix-env --switch-profile</command>: + +<screen> +$ nix-env --switch-profile /nix/var/nix/profiles/my-profile + +$ nix-env --switch-profile /nix/var/nix/profiles/default</screen> + +These commands switch to the <filename>my-profile</filename> and +default profile, respectively. If the profile doesn’t exist, it will +be created automatically. You should be careful about storing a +profile in another location than the <filename>profiles</filename> +directory, since otherwise it might not be used as a root of the +garbage collector (see <xref linkend='sec-garbage-collection' +/>).</para> + +<para>All <command>nix-env</command> operations work on the profile +pointed to by <command>~/.nix-profile</command>, but you can override +this using the <option>--profile</option> option (abbreviation +<option>-p</option>): + +<screen> +$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion</screen> + +This will <emphasis>not</emphasis> change the +<command>~/.nix-profile</command> symlink.</para> + +</chapter> diff --git a/third_party/nix/doc/manual/packages/s3-substituter.xml b/third_party/nix/doc/manual/packages/s3-substituter.xml new file mode 100644 index 000000000000..868b5a66dc3c --- /dev/null +++ b/third_party/nix/doc/manual/packages/s3-substituter.xml @@ -0,0 +1,182 @@ +<?xml version="1.0" encoding="utf-8"?> +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-s3-substituter"> + +<title>Serving a Nix store via AWS S3 or S3-compatible Service</title> + +<para>Nix has built-in support for storing and fetching store paths +from Amazon S3 and S3 compatible services. This uses the same +<emphasis>binary</emphasis> cache mechanism that Nix usually uses to +fetch prebuilt binaries from <uri>cache.nixos.org</uri>.</para> + +<para>The following options can be specified as URL parameters to +the S3 URL:</para> + +<variablelist> + <varlistentry><term><literal>profile</literal></term> + <listitem> + <para> + The name of the AWS configuration profile to use. By default + Nix will use the <literal>default</literal> profile. + </para> + </listitem> + </varlistentry> + + <varlistentry><term><literal>region</literal></term> + <listitem> + <para> + The region of the S3 bucket. <literal>us–east-1</literal> by + default. + </para> + + <para> + If your bucket is not in <literal>us–east-1</literal>, you + should always explicitly specify the region parameter. + </para> + </listitem> + </varlistentry> + + <varlistentry><term><literal>endpoint</literal></term> + <listitem> + <para> + The URL to your S3-compatible service, for when not using + Amazon S3. Do not specify this value if you're using Amazon + S3. + </para> + <note><para>This endpoint must support HTTPS and will use + path-based addressing instead of virtual host based + addressing.</para></note> + </listitem> + </varlistentry> + + <varlistentry><term><literal>scheme</literal></term> + <listitem> + <para> + The scheme used for S3 requests, <literal>https</literal> + (default) or <literal>http</literal>. This option allows you to + disable HTTPS for binary caches which don't support it. + </para> + <note><para>HTTPS should be used if the cache might contain + sensitive information.</para></note> + </listitem> + </varlistentry> +</variablelist> + +<para>In this example we will use the bucket named +<literal>example-nix-cache</literal>.</para> + +<section xml:id="ssec-s3-substituter-anonymous-reads"> + <title>Anonymous Reads to your S3-compatible binary cache</title> + + <para>If your binary cache is publicly accessible and does not + require authentication, the simplest and easiest way to use Nix with + your S3 compatible binary cache is to use the HTTP URL for that + cache.</para> + + <para>For AWS S3 the binary cache URL for example bucket will be + exactly <uri>https://example-nix-cache.s3.amazonaws.com</uri> or + <uri>s3://example-nix-cache</uri>. For S3 compatible binary caches, + consult that cache's documentation.</para> + + <para>Your bucket will need the following bucket policy:</para> + + <programlisting><![CDATA[ +{ + "Id": "DirectReads", + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowDirectReads", + "Action": [ + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ], + "Principal": "*" + } + ] +} +]]></programlisting> +</section> + +<section xml:id="ssec-s3-substituter-authenticated-reads"> + <title>Authenticated Reads to your S3 binary cache</title> + + <para>For AWS S3 the binary cache URL for example bucket will be + exactly <uri>s3://example-nix-cache</uri>.</para> + + <para>Nix will use the <link + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default + credential provider chain</link> for authenticating requests to + Amazon S3.</para> + + <para>Nix supports authenticated reads from Amazon S3 and S3 + compatible binary caches.</para> + + <para>Your bucket will need a bucket policy allowing the desired + users to perform the <literal>s3:GetObject</literal> and + <literal>s3:GetBucketLocation</literal> action on all objects in the + bucket. The anonymous policy in <xref + linkend="ssec-s3-substituter-anonymous-reads" /> can be updated to + have a restricted <literal>Principal</literal> to support + this.</para> +</section> + + +<section xml:id="ssec-s3-substituter-authenticated-writes"> + <title>Authenticated Writes to your S3-compatible binary cache</title> + + <para>Nix support fully supports writing to Amazon S3 and S3 + compatible buckets. The binary cache URL for our example bucket will + be <uri>s3://example-nix-cache</uri>.</para> + + <para>Nix will use the <link + xlink:href="https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html">default + credential provider chain</link> for authenticating requests to + Amazon S3.</para> + + <para>Your account will need the following IAM policy to + upload to the cache:</para> + + <programlisting><![CDATA[ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "UploadToCache", + "Effect": "Allow", + "Action": [ + "s3:AbortMultipartUpload", + "s3:GetBucketLocation", + "s3:GetObject", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:ListMultipartUploadParts", + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::example-nix-cache", + "arn:aws:s3:::example-nix-cache/*" + ] + } + ] +} +]]></programlisting> + + + <example><title>Uploading with a specific credential profile for Amazon S3</title> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2' nixpkgs.hello</command></para> + </example> + + <example><title>Uploading to an S3-Compatible Binary Cache</title> + <para><command>nix copy --to 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' nixpkgs.hello</command></para> + </example> +</section> +</section> diff --git a/third_party/nix/doc/manual/packages/sharing-packages.xml b/third_party/nix/doc/manual/packages/sharing-packages.xml new file mode 100644 index 000000000000..bb6c52b8f8c1 --- /dev/null +++ b/third_party/nix/doc/manual/packages/sharing-packages.xml @@ -0,0 +1,20 @@ +<chapter xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-sharing-packages"> + +<title>Sharing Packages Between Machines</title> + +<para>Sometimes you want to copy a package from one machine to +another. Or, you want to install some packages and you know that +another machine already has some or all of those packages or their +dependencies. In that case there are mechanisms to quickly copy +packages between machines.</para> + +<xi:include href="binary-cache-substituter.xml" /> +<xi:include href="copy-closure.xml" /> +<xi:include href="ssh-substituter.xml" /> +<xi:include href="s3-substituter.xml" /> + +</chapter> diff --git a/third_party/nix/doc/manual/packages/ssh-substituter.xml b/third_party/nix/doc/manual/packages/ssh-substituter.xml new file mode 100644 index 000000000000..8db3f96625d3 --- /dev/null +++ b/third_party/nix/doc/manual/packages/ssh-substituter.xml @@ -0,0 +1,73 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-ssh-substituter"> + +<title>Serving a Nix store via SSH</title> + +<para>You can tell Nix to automatically fetch needed binaries from a +remote Nix store via SSH. For example, the following installs Firefox, +automatically fetching any store paths in Firefox’s closure if they +are available on the server <literal>avalon</literal>: + +<screen> +$ nix-env -i firefox --substituters ssh://alice@avalon +</screen> + +This works similar to the binary cache substituter that Nix usually +uses, only using SSH instead of HTTP: if a store path +<literal>P</literal> is needed, Nix will first check if it’s available +in the Nix store on <literal>avalon</literal>. If not, it will fall +back to using the binary cache substituter, and then to building from +source.</para> + +<note><para>The SSH substituter currently does not allow you to enter +an SSH passphrase interactively. Therefore, you should use +<command>ssh-add</command> to load the decrypted private key into +<command>ssh-agent</command>.</para></note> + +<para>You can also copy the closure of some store path, without +installing it into your profile, e.g. + +<screen> +$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon +</screen> + +This is essentially equivalent to doing + +<screen> +$ nix-copy-closure --from alice@avalon /nix/store/m85bxg…-firefox-34.0.5 +</screen> + +</para> + +<para>You can use SSH’s <emphasis>forced command</emphasis> feature to +set up a restricted user account for SSH substituter access, allowing +read-only access to the local Nix store, but nothing more. For +example, add the following lines to <filename>sshd_config</filename> +to restrict the user <literal>nix-ssh</literal>: + +<programlisting> +Match User nix-ssh + AllowAgentForwarding no + AllowTcpForwarding no + PermitTTY no + PermitTunnel no + X11Forwarding no + ForceCommand nix-store --serve +Match All +</programlisting> + +On NixOS, you can accomplish the same by adding the following to your +<filename>configuration.nix</filename>: + +<programlisting> +nix.sshServe.enable = true; +nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; +</programlisting> + +where the latter line lists the public keys of users that are allowed +to connect.</para> + +</section> diff --git a/third_party/nix/doc/manual/quote-literals.xsl b/third_party/nix/doc/manual/quote-literals.xsl new file mode 100644 index 000000000000..5002643dbda4 --- /dev/null +++ b/third_party/nix/doc/manual/quote-literals.xsl @@ -0,0 +1,40 @@ +<?xml version="1.0"?> + +<xsl:stylesheet + version="1.0" + xmlns:xsl="http://www.w3.org/1999/XSL/Transform" + xmlns:str="http://exslt.org/strings" + extension-element-prefixes="str"> + + <xsl:output method="xml"/> + + <xsl:template match="function|command|literal|varname|filename|option|quote">`<xsl:apply-templates/>'</xsl:template> + + <xsl:template match="token"><xsl:text> </xsl:text><xsl:apply-templates /><xsl:text> +</xsl:text></xsl:template> + + <xsl:template match="screen|programlisting"> + <screen><xsl:apply-templates select="str:split(., '
')" /></screen> + </xsl:template> + + <xsl:template match="section[following::section]"> + <section> + <xsl:apply-templates /> + <screen><xsl:text> + </xsl:text></screen> + </section> + </xsl:template> + + <xsl:template match="*"> + <xsl:element name="{name(.)}" namespace="{namespace-uri(.)}"> + <xsl:copy-of select="namespace::*" /> + <xsl:for-each select="@*"> + <xsl:attribute name="{name(.)}" namespace="{namespace-uri(.)}"> + <xsl:value-of select="."/> + </xsl:attribute> + </xsl:for-each> + <xsl:apply-templates/> + </xsl:element> + </xsl:template> + +</xsl:stylesheet> diff --git a/third_party/nix/doc/manual/release-notes/release-notes.xml b/third_party/nix/doc/manual/release-notes/release-notes.xml new file mode 100644 index 000000000000..2655d68e354b --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/release-notes.xml @@ -0,0 +1,51 @@ +<appendix xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="sec-relnotes"> + +<title>Nix Release Notes</title> + +<!-- +<partintro> +<para>This section lists the release notes for each stable version of Nix.</para> +</partintro> +--> + +<xi:include href="rl-2.3.xml" /> +<xi:include href="rl-2.2.xml" /> +<xi:include href="rl-2.1.xml" /> +<xi:include href="rl-2.0.xml" /> +<xi:include href="rl-1.11.10.xml" /> +<xi:include href="rl-1.11.xml" /> +<xi:include href="rl-1.10.xml" /> +<xi:include href="rl-1.9.xml" /> +<xi:include href="rl-1.8.xml" /> +<xi:include href="rl-1.7.xml" /> +<xi:include href="rl-1.6.1.xml" /> +<xi:include href="rl-1.6.xml" /> +<xi:include href="rl-1.5.2.xml" /> +<xi:include href="rl-1.5.xml" /> +<xi:include href="rl-1.4.xml" /> +<xi:include href="rl-1.3.xml" /> +<xi:include href="rl-1.2.xml" /> +<xi:include href="rl-1.1.xml" /> +<xi:include href="rl-1.0.xml" /> +<xi:include href="rl-0.16.xml" /> +<xi:include href="rl-0.15.xml" /> +<xi:include href="rl-0.14.xml" /> +<xi:include href="rl-0.13.xml" /> +<xi:include href="rl-0.12.xml" /> +<xi:include href="rl-0.11.xml" /> +<xi:include href="rl-0.10.1.xml" /> +<xi:include href="rl-0.10.xml" /> +<xi:include href="rl-0.9.2.xml" /> +<xi:include href="rl-0.9.1.xml" /> +<xi:include href="rl-0.9.xml" /> +<xi:include href="rl-0.8.1.xml" /> +<xi:include href="rl-0.8.xml" /> +<xi:include href="rl-0.7.xml" /> +<xi:include href="rl-0.6.xml" /> +<xi:include href="rl-0.5.xml" /> + +</appendix> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.10.1.xml b/third_party/nix/doc/manual/release-notes/rl-0.10.1.xml new file mode 100644 index 000000000000..95829323d4fb --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.10.1.xml @@ -0,0 +1,13 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.10.1"> + +<title>Release 0.10.1 (2006-10-11)</title> + +<para>This release fixes two somewhat obscure bugs that occur when +evaluating Nix expressions that are stored inside the Nix store +(<literal>NIX-67</literal>). These do not affect most users.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.10.xml b/third_party/nix/doc/manual/release-notes/rl-0.10.xml new file mode 100644 index 000000000000..9afec4de94de --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.10.xml @@ -0,0 +1,323 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.10"> + +<title>Release 0.10 (2006-10-06)</title> + +<note><para>This version of Nix uses Berkeley DB 4.4 instead of 4.3. +The database is upgraded automatically, but you should be careful not +to use old versions of Nix that still use Berkeley DB 4.3. In +particular, if you use a Nix installed through Nix, you should run + +<screen> +$ nix-store --clear-substitutes</screen> + +first.</para></note> + +<warning><para>Also, the database schema has changed slighted to fix a +performance issue (see below). When you run any Nix 0.10 command for +the first time, the database will be upgraded automatically. This is +irreversible.</para></warning> + +<itemizedlist> + + + <!-- Usability / features --> + + + <listitem><para><command>nix-env</command> usability improvements: + + <itemizedlist> + + <listitem><para>An option <option>--compare-versions</option> + (or <option>-c</option>) has been added to <command>nix-env + --query</command> to allow you to compare installed versions of + packages to available versions, or vice versa. An easy way to + see if you are up to date with what’s in your subscribed + channels is <literal>nix-env -qc \*</literal>.</para></listitem> + + <listitem><para><literal>nix-env --query</literal> now takes as + arguments a list of package names about which to show + information, just like <option>--install</option>, etc.: for + example, <literal>nix-env -q gcc</literal>. Note that to show + all derivations, you need to specify + <literal>\*</literal>.</para></listitem> + + <listitem><para><literal>nix-env -i + <replaceable>pkgname</replaceable></literal> will now install + the highest available version of + <replaceable>pkgname</replaceable>, rather than installing all + available versions (which would probably give collisions) + (<literal>NIX-31</literal>).</para></listitem> + + <listitem><para><literal>nix-env (-i|-u) --dry-run</literal> now + shows exactly which missing paths will be built or + substituted.</para></listitem> + + <listitem><para><literal>nix-env -qa --description</literal> + shows human-readable descriptions of packages, provided that + they have a <literal>meta.description</literal> attribute (which + most packages in Nixpkgs don’t have yet).</para></listitem> + + </itemizedlist> + + </para></listitem> + + + <listitem><para>New language features: + + <itemizedlist> + + <listitem><para>Reference scanning (which happens after each + build) is much faster and takes a constant amount of + memory.</para></listitem> + + <listitem><para>String interpolation. Expressions like + +<programlisting> +"--with-freetype2-library=" + freetype + "/lib"</programlisting> + + can now be written as + +<programlisting> +"--with-freetype2-library=${freetype}/lib"</programlisting> + + You can write arbitrary expressions within + <literal>${<replaceable>...</replaceable>}</literal>, not just + identifiers.</para></listitem> + + <listitem><para>Multi-line string literals.</para></listitem> + + <listitem><para>String concatenations can now involve + derivations, as in the example <code>"--with-freetype2-library=" + + freetype + "/lib"</code>. This was not previously possible + because we need to register that a derivation that uses such a + string is dependent on <literal>freetype</literal>. The + evaluator now properly propagates this information. + Consequently, the subpath operator (<literal>~</literal>) has + been deprecated.</para></listitem> + + <listitem><para>Default values of function arguments can now + refer to other function arguments; that is, all arguments are in + scope in the default values + (<literal>NIX-45</literal>).</para></listitem> + + <!-- + <listitem><para>TODO: domain checks (r5895).</para></listitem> + --> + + <listitem><para>Lots of new built-in primitives, such as + functions for list manipulation and integer arithmetic. See the + manual for a complete list. All primops are now available in + the set <varname>builtins</varname>, allowing one to test for + the availability of primop in a backwards-compatible + way.</para></listitem> + + <listitem><para>Real let-expressions: <literal>let x = ...; + ... z = ...; in ...</literal>.</para></listitem> + + </itemizedlist> + + </para></listitem> + + + <listitem><para>New commands <command>nix-pack-closure</command> and + <command>nix-unpack-closure</command> than can be used to easily + transfer a store path with all its dependencies to another machine. + Very convenient whenever you have some package on your machine and + you want to copy it somewhere else.</para></listitem> + + + <listitem><para>XML support: + + <itemizedlist> + + <listitem><para><literal>nix-env -q --xml</literal> prints the + installed or available packages in an XML representation for + easy processing by other tools.</para></listitem> + + <listitem><para><literal>nix-instantiate --eval-only + --xml</literal> prints an XML representation of the resulting + term. (The new flag <option>--strict</option> forces ‘deep’ + evaluation of the result, i.e., list elements and attributes are + evaluated recursively.)</para></listitem> + + <listitem><para>In Nix expressions, the primop + <function>builtins.toXML</function> converts a term to an XML + representation. This is primarily useful for passing structured + information to builders.</para></listitem> + + </itemizedlist> + + </para></listitem> + + + <listitem><para>You can now unambiguously specify which derivation to + build or install in <command>nix-env</command>, + <command>nix-instantiate</command> and <command>nix-build</command> + using the <option>--attr</option> / <option>-A</option> flags, which + takes an attribute name as argument. (Unlike symbolic package names + such as <literal>subversion-1.4.0</literal>, attribute names in an + attribute set are unique.) For instance, a quick way to perform a + test build of a package in Nixpkgs is <literal>nix-build + pkgs/top-level/all-packages.nix -A + <replaceable>foo</replaceable></literal>. <literal>nix-env -q + --attr</literal> shows the attribute names corresponding to each + derivation.</para></listitem> + + + <listitem><para>If the top-level Nix expression used by + <command>nix-env</command>, <command>nix-instantiate</command> or + <command>nix-build</command> evaluates to a function whose arguments + all have default values, the function will be called automatically. + Also, the new command-line switch <option>--arg + <replaceable>name</replaceable> + <replaceable>value</replaceable></option> can be used to specify + function arguments on the command line.</para></listitem> + + + <listitem><para><literal>nix-install-package --url + <replaceable>URL</replaceable></literal> allows a package to be + installed directly from the given URL.</para></listitem> + + + <listitem><para>Nix now works behind an HTTP proxy server; just set + the standard environment variables <envar>http_proxy</envar>, + <envar>https_proxy</envar>, <envar>ftp_proxy</envar> or + <envar>all_proxy</envar> appropriately. Functions such as + <function>fetchurl</function> in Nixpkgs also respect these + variables.</para></listitem> + + + <listitem><para><literal>nix-build -o + <replaceable>symlink</replaceable></literal> allows the symlink to + the build result to be named something other than + <literal>result</literal>.</para></listitem> + + + <!-- Stability / performance / etc. --> + + + <listitem><para>Platform support: + + <itemizedlist> + + <listitem><para>Support for 64-bit platforms, provided a <link + xlink:href="http://bugzilla.sen.cwi.nl:8080/show_bug.cgi?id=606">suitably + patched ATerm library</link> is used. Also, files larger than 2 + GiB are now supported.</para></listitem> + + <listitem><para>Added support for Cygwin (Windows, + <literal>i686-cygwin</literal>), Mac OS X on Intel + (<literal>i686-darwin</literal>) and Linux on PowerPC + (<literal>powerpc-linux</literal>).</para></listitem> + + <listitem><para>Users of SMP and multicore machines will + appreciate that the number of builds to be performed in parallel + can now be specified in the configuration file in the + <literal>build-max-jobs</literal> setting.</para></listitem> + + </itemizedlist> + + </para></listitem> + + + <listitem><para>Garbage collector improvements: + + <itemizedlist> + + <listitem><para>Open files (such as running programs) are now + used as roots of the garbage collector. This prevents programs + that have been uninstalled from being garbage collected while + they are still running. The script that detects these + additional runtime roots + (<filename>find-runtime-roots.pl</filename>) is inherently + system-specific, but it should work on Linux and on all + platforms that have the <command>lsof</command> + utility.</para></listitem> + + <listitem><para><literal>nix-store --gc</literal> + (a.k.a. <command>nix-collect-garbage</command>) prints out the + number of bytes freed on standard output. <literal>nix-store + --gc --print-dead</literal> shows how many bytes would be freed + by an actual garbage collection.</para></listitem> + + <listitem><para><literal>nix-collect-garbage -d</literal> + removes all old generations of <emphasis>all</emphasis> profiles + before calling the actual garbage collector (<literal>nix-store + --gc</literal>). This is an easy way to get rid of all old + packages in the Nix store.</para></listitem> + + <listitem><para><command>nix-store</command> now has an + operation <option>--delete</option> to delete specific paths + from the Nix store. It won’t delete reachable (non-garbage) + paths unless <option>--ignore-liveness</option> is + specified.</para></listitem> + + </itemizedlist> + + </para></listitem> + + + <listitem><para>Berkeley DB 4.4’s process registry feature is used + to recover from crashed Nix processes.</para></listitem> + + <!-- <listitem><para>TODO: shared stores.</para></listitem> --> + + <listitem><para>A performance issue has been fixed with the + <literal>referer</literal> table, which stores the inverse of the + <literal>references</literal> table (i.e., it tells you what store + paths refer to a given path). Maintaining this table could take a + quadratic amount of time, as well as a quadratic amount of Berkeley + DB log file space (in particular when running the garbage collector) + (<literal>NIX-23</literal>).</para></listitem> + + <listitem><para>Nix now catches the <literal>TERM</literal> and + <literal>HUP</literal> signals in addition to the + <literal>INT</literal> signal. So you can now do a <literal>killall + nix-store</literal> without triggering a database + recovery.</para></listitem> + + <listitem><para><command>bsdiff</command> updated to version + 4.3.</para></listitem> + + <listitem><para>Substantial performance improvements in expression + evaluation and <literal>nix-env -qa</literal>, all thanks to <link + xlink:href="http://valgrind.org/">Valgrind</link>. Memory use has + been reduced by a factor 8 or so. Big speedup by memoisation of + path hashing.</para></listitem> + + <listitem><para>Lots of bug fixes, notably: + + <itemizedlist> + + <listitem><para>Make sure that the garbage collector can run + successfully when the disk is full + (<literal>NIX-18</literal>).</para></listitem> + + <listitem><para><command>nix-env</command> now locks the profile + to prevent races between concurrent <command>nix-env</command> + operations on the same profile + (<literal>NIX-7</literal>).</para></listitem> + + <listitem><para>Removed misleading messages from + <literal>nix-env -i</literal> (e.g., <literal>installing + `foo'</literal> followed by <literal>uninstalling + `foo'</literal>) (<literal>NIX-17</literal>).</para></listitem> + + </itemizedlist> + + </para></listitem> + + <listitem><para>Nix source distributions are a lot smaller now since + we no longer include a full copy of the Berkeley DB source + distribution (but only the bits we need).</para></listitem> + + <listitem><para>Header files are now installed so that external + programs can use the Nix libraries.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.11.xml b/third_party/nix/doc/manual/release-notes/rl-0.11.xml new file mode 100644 index 000000000000..7ad0ab5b71ad --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.11.xml @@ -0,0 +1,261 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-0.11"> + +<title>Release 0.11 (2007-12-31)</title> + +<para>Nix 0.11 has many improvements over the previous stable release. +The most important improvement is secure multi-user support. It also +features many usability enhancements and language extensions, many of +them prompted by NixOS, the purely functional Linux distribution based +on Nix. Here is an (incomplete) list:</para> + + +<itemizedlist> + + + <listitem><para>Secure multi-user support. A single Nix store can + now be shared between multiple (possible untrusted) users. This is + an important feature for NixOS, where it allows non-root users to + install software. The old setuid method for sharing a store between + multiple users has been removed. Details for setting up a + multi-user store can be found in the manual.</para></listitem> + + + <listitem><para>The new command <command>nix-copy-closure</command> + gives you an easy and efficient way to exchange software between + machines. It copies the missing parts of the closure of a set of + store path to or from a remote machine via + <command>ssh</command>.</para></listitem> + + + <listitem><para>A new kind of string literal: strings between double + single-quotes (<literal>''</literal>) have indentation + “intelligently” removed. This allows large strings (such as shell + scripts or configuration file fragments in NixOS) to cleanly follow + the indentation of the surrounding expression. It also requires + much less escaping, since <literal>''</literal> is less common in + most languages than <literal>"</literal>.</para></listitem> + + + <listitem><para><command>nix-env</command> <option>--set</option> + modifies the current generation of a profile so that it contains + exactly the specified derivation, and nothing else. For example, + <literal>nix-env -p /nix/var/nix/profiles/browser --set + firefox</literal> lets the profile named + <filename>browser</filename> contain just Firefox.</para></listitem> + + + <listitem><para><command>nix-env</command> now maintains + meta-information about installed packages in profiles. The + meta-information is the contents of the <varname>meta</varname> + attribute of derivations, such as <varname>description</varname> or + <varname>homepage</varname>. The command <literal>nix-env -q --xml + --meta</literal> shows all meta-information.</para></listitem> + + + <listitem><para><command>nix-env</command> now uses the + <varname>meta.priority</varname> attribute of derivations to resolve + filename collisions between packages. Lower priority values denote + a higher priority. For instance, the GCC wrapper package and the + Binutils package in Nixpkgs both have a file + <filename>bin/ld</filename>, so previously if you tried to install + both you would get a collision. Now, on the other hand, the GCC + wrapper declares a higher priority than Binutils, so the former’s + <filename>bin/ld</filename> is symlinked in the user + environment.</para></listitem> + + + <listitem><para><command>nix-env -i / -u</command>: instead of + breaking package ties by version, break them by priority and version + number. That is, if there are multiple packages with the same name, + then pick the package with the highest priority, and only use the + version if there are multiple packages with the same + priority.</para> + + <para>This makes it possible to mark specific versions/variant in + Nixpkgs more or less desirable than others. A typical example would + be a beta version of some package (e.g., + <literal>gcc-4.2.0rc1</literal>) which should not be installed even + though it is the highest version, except when it is explicitly + selected (e.g., <literal>nix-env -i + gcc-4.2.0rc1</literal>).</para></listitem> + + + <listitem><para><command>nix-env --set-flag</command> allows meta + attributes of installed packages to be modified. There are several + attributes that can be usefully modified, because they affect the + behaviour of <command>nix-env</command> or the user environment + build script: + + <itemizedlist> + + <listitem><para><varname>meta.priority</varname> can be changed + to resolve filename clashes (see above).</para></listitem> + + <listitem><para><varname>meta.keep</varname> can be set to + <literal>true</literal> to prevent the package from being + upgraded or replaced. Useful if you want to hang on to an older + version of a package.</para></listitem> + + <listitem><para><varname>meta.active</varname> can be set to + <literal>false</literal> to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). + Set it back to <literal>true</literal> to re-enable the + package.</para></listitem> + + </itemizedlist> + + </para></listitem> + + + <listitem><para><command>nix-env -q</command> now has a flag + <option>--prebuilt-only</option> (<option>-b</option>) that causes + <command>nix-env</command> to show only those derivations whose + output is already in the Nix store or that can be substituted (i.e., + downloaded from somewhere). In other words, it shows the packages + that can be installed “quickly”, i.e., don’t need to be built from + source. The <option>-b</option> flag is also available in + <command>nix-env -i</command> and <command>nix-env -u</command> to + filter out derivations for which no pre-built binary is + available.</para></listitem> + + + <listitem><para>The new option <option>--argstr</option> (in + <command>nix-env</command>, <command>nix-instantiate</command> and + <command>nix-build</command>) is like <option>--arg</option>, except + that the value is a string. For example, <literal>--argstr system + i686-linux</literal> is equivalent to <literal>--arg system + \"i686-linux\"</literal> (note that <option>--argstr</option> + prevents annoying quoting around shell arguments).</para></listitem> + + + <listitem><para><command>nix-store</command> has a new operation + <option>--read-log</option> (<option>-l</option>) + <parameter>paths</parameter> that shows the build log of the given + paths.</para></listitem> + + + <!-- + <listitem><para>TODO: semantic cleanups of string concatenation + etc. (mostly in r6740).</para></listitem> + --> + + + <listitem><para>Nix now uses Berkeley DB 4.5. The database is + upgraded automatically, but you should be careful not to use old + versions of Nix that still use Berkeley DB 4.4.</para></listitem> + + + <!-- foo + <listitem><para>TODO: option <option>- -reregister</option> in + <command>nix-store - -register-validity</command>.</para></listitem> + --> + + + <listitem><para>The option <option>--max-silent-time</option> + (corresponding to the configuration setting + <literal>build-max-silent-time</literal>) allows you to set a + timeout on builds — if a build produces no output on + <literal>stdout</literal> or <literal>stderr</literal> for the given + number of seconds, it is terminated. This is useful for recovering + automatically from builds that are stuck in an infinite + loop.</para></listitem> + + + <listitem><para><command>nix-channel</command>: each subscribed + channel is its own attribute in the top-level expression generated + for the channel. This allows disambiguation (e.g. <literal>nix-env + -i -A nixpkgs_unstable.firefox</literal>).</para></listitem> + + + <listitem><para>The substitutes table has been removed from the + database. This makes operations such as <command>nix-pull</command> + and <command>nix-channel --update</command> much, much + faster.</para></listitem> + + + <listitem><para><command>nix-pull</command> now supports + bzip2-compressed manifests. This speeds up + channels.</para></listitem> + + + <listitem><para><command>nix-prefetch-url</command> now has a + limited form of caching. This is used by + <command>nix-channel</command> to prevent unnecessary downloads when + the channel hasn’t changed.</para></listitem> + + + <listitem><para><command>nix-prefetch-url</command> now by default + computes the SHA-256 hash of the file instead of the MD5 hash. In + calls to <function>fetchurl</function> you should pass the + <literal>sha256</literal> attribute instead of + <literal>md5</literal>. You can pass either a hexadecimal or a + base-32 encoding of the hash.</para></listitem> + + + <listitem><para>Nix can now perform builds in an automatically + generated “chroot”. This prevents a builder from accessing stuff + outside of the Nix store, and thus helps ensure purity. This is an + experimental feature.</para></listitem> + + + <listitem><para>The new command <command>nix-store + --optimise</command> reduces Nix store disk space usage by finding + identical files in the store and hard-linking them to each other. + It typically reduces the size of the store by something like + 25-35%.</para></listitem> + + + <listitem><para><filename>~/.nix-defexpr</filename> can now be a + directory, in which case the Nix expressions in that directory are + combined into an attribute set, with the file names used as the + names of the attributes. The command <command>nix-env + --import</command> (which set the + <filename>~/.nix-defexpr</filename> symlink) is + removed.</para></listitem> + + + <listitem><para>Derivations can specify the new special attribute + <varname>allowedReferences</varname> to enforce that the references + in the output of a derivation are a subset of a declared set of + paths. For example, if <varname>allowedReferences</varname> is an + empty list, then the output must not have any references. This is + used in NixOS to check that generated files such as initial ramdisks + for booting Linux don’t have any dependencies.</para></listitem> + + + <listitem><para>The new attribute + <varname>exportReferencesGraph</varname> allows builders access to + the references graph of their inputs. This is used in NixOS for + tasks such as generating ISO-9660 images that contain a Nix store + populated with the closure of certain paths.</para></listitem> + + + <listitem><para>Fixed-output derivations (like + <function>fetchurl</function>) can define the attribute + <varname>impureEnvVars</varname> to allow external environment + variables to be passed to builders. This is used in Nixpkgs to + support proxy configuration, among other things.</para></listitem> + + + <listitem><para>Several new built-in functions: + <function>builtins.attrNames</function>, + <function>builtins.filterSource</function>, + <function>builtins.isAttrs</function>, + <function>builtins.isFunction</function>, + <function>builtins.listToAttrs</function>, + <function>builtins.stringLength</function>, + <function>builtins.sub</function>, + <function>builtins.substring</function>, + <function>throw</function>, + <function>builtins.trace</function>, + <function>builtins.readFile</function>.</para></listitem> + + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.12.xml b/third_party/nix/doc/manual/release-notes/rl-0.12.xml new file mode 100644 index 000000000000..fdba8c4d577f --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.12.xml @@ -0,0 +1,175 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-0.12"> + +<title>Release 0.12 (2008-11-20)</title> + +<itemizedlist> + + <listitem> + <para>Nix no longer uses Berkeley DB to store Nix store metadata. + The principal advantages of the new storage scheme are: it works + properly over decent implementations of NFS (allowing Nix stores + to be shared between multiple machines); no recovery is needed + when a Nix process crashes; no write access is needed for + read-only operations; no more running out of Berkeley DB locks on + certain operations.</para> + + <para>You still need to compile Nix with Berkeley DB support if + you want Nix to automatically convert your old Nix store to the + new schema. If you don’t need this, you can build Nix with the + <filename>configure</filename> option + <option>--disable-old-db-compat</option>.</para> + + <para>After the automatic conversion to the new schema, you can + delete the old Berkeley DB files: + + <screen> +$ cd /nix/var/nix/db +$ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG</screen> + + The new metadata is stored in the directories + <filename>/nix/var/nix/db/info</filename> and + <filename>/nix/var/nix/db/referrer</filename>. Though the + metadata is stored in human-readable plain-text files, they are + not intended to be human-editable, as Nix is rather strict about + the format.</para> + + <para>The new storage schema may or may not require less disk + space than the Berkeley DB environment, mostly depending on the + cluster size of your file system. With 1 KiB clusters (which + seems to be the <literal>ext3</literal> default nowadays) it + usually takes up much less space.</para> + </listitem> + + <listitem><para>There is a new substituter that copies paths + directly from other (remote) Nix stores mounted somewhere in the + filesystem. For instance, you can speed up an installation by + mounting some remote Nix store that already has the packages in + question via NFS or <literal>sshfs</literal>. The environment + variable <envar>NIX_OTHER_STORES</envar> specifies the locations of + the remote Nix directories, + e.g. <literal>/mnt/remote-fs/nix</literal>.</para></listitem> + + <listitem><para>New <command>nix-store</command> operations + <option>--dump-db</option> and <option>--load-db</option> to dump + and reload the Nix database.</para></listitem> + + <listitem><para>The garbage collector has a number of new options to + allow only some of the garbage to be deleted. The option + <option>--max-freed <replaceable>N</replaceable></option> tells the + collector to stop after at least <replaceable>N</replaceable> bytes + have been deleted. The option <option>--max-links + <replaceable>N</replaceable></option> tells it to stop after the + link count on <filename>/nix/store</filename> has dropped below + <replaceable>N</replaceable>. This is useful for very large Nix + stores on filesystems with a 32000 subdirectories limit (like + <literal>ext3</literal>). The option <option>--use-atime</option> + causes store paths to be deleted in order of ascending last access + time. This allows non-recently used stuff to be deleted. The + option <option>--max-atime <replaceable>time</replaceable></option> + specifies an upper limit to the last accessed time of paths that may + be deleted. For instance, + + <screen> + $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago")</screen> + + deletes everything that hasn’t been accessed in two months.</para></listitem> + + <listitem><para><command>nix-env</command> now uses optimistic + profile locking when performing an operation like installing or + upgrading, instead of setting an exclusive lock on the profile. + This allows multiple <command>nix-env -i / -u / -e</command> + operations on the same profile in parallel. If a + <command>nix-env</command> operation sees at the end that the profile + was changed in the meantime by another process, it will just + restart. This is generally cheap because the build results are + still in the Nix store.</para></listitem> + + <listitem><para>The option <option>--dry-run</option> is now + supported by <command>nix-store -r</command> and + <command>nix-build</command>.</para></listitem> + + <listitem><para>The information previously shown by + <option>--dry-run</option> (i.e., which derivations will be built + and which paths will be substituted) is now always shown by + <command>nix-env</command>, <command>nix-store -r</command> and + <command>nix-build</command>. The total download size of + substitutable paths is now also shown. For instance, a build will + show something like + + <screen> +the following derivations will be built: + /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv + /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv + ... +the following paths will be downloaded/copied (30.02 MiB): + /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4 + /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6 + ...</screen> + + </para></listitem> + + <listitem><para>Language features: + + <itemizedlist> + + <listitem><para>@-patterns as in Haskell. For instance, in a + function definition + + <programlisting>f = args @ {x, y, z}: <replaceable>...</replaceable>;</programlisting> + + <varname>args</varname> refers to the argument as a whole, which + is further pattern-matched against the attribute set pattern + <literal>{x, y, z}</literal>.</para></listitem> + + <listitem><para>“<literal>...</literal>” (ellipsis) patterns. + An attribute set pattern can now say <literal>...</literal> at + the end of the attribute name list to specify that the function + takes <emphasis>at least</emphasis> the listed attributes, while + ignoring additional attributes. For instance, + + <programlisting>{stdenv, fetchurl, fuse, ...}: <replaceable>...</replaceable></programlisting> + + defines a function that accepts any attribute set that includes + at least the three listed attributes.</para></listitem> + + <listitem><para>New primops: + <varname>builtins.parseDrvName</varname> (split a package name + string like <literal>"nix-0.12pre12876"</literal> into its name + and version components, e.g. <literal>"nix"</literal> and + <literal>"0.12pre12876"</literal>), + <varname>builtins.compareVersions</varname> (compare two version + strings using the same algorithm that <command>nix-env</command> + uses), <varname>builtins.length</varname> (efficiently compute + the length of a list), <varname>builtins.mul</varname> (integer + multiplication), <varname>builtins.div</varname> (integer + division). + <!-- <varname>builtins.genericClosure</varname> --> + </para></listitem> + + </itemizedlist> + + </para></listitem> + + <listitem><para><command>nix-prefetch-url</command> now supports + <literal>mirror://</literal> URLs, provided that the environment + variable <envar>NIXPKGS_ALL</envar> points at a Nixpkgs + tree.</para></listitem> + + <listitem><para>Removed the commands + <command>nix-pack-closure</command> and + <command>nix-unpack-closure</command>. You can do almost the same + thing but much more efficiently by doing <literal>nix-store --export + $(nix-store -qR <replaceable>paths</replaceable>) > closure</literal> and + <literal>nix-store --import < + closure</literal>.</para></listitem> + + <listitem><para>Lots of bug fixes, including a big performance bug in + the handling of <literal>with</literal>-expressions.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.13.xml b/third_party/nix/doc/manual/release-notes/rl-0.13.xml new file mode 100644 index 000000000000..cce2e4a26b05 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.13.xml @@ -0,0 +1,106 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-0.13"> + +<title>Release 0.13 (2009-11-05)</title> + +<para>This is primarily a bug fix release. It has some new +features:</para> + +<itemizedlist> + + <listitem> + <para>Syntactic sugar for writing nested attribute sets. Instead of + +<programlisting> +{ + foo = { + bar = 123; + xyzzy = true; + }; + a = { b = { c = "d"; }; }; +} +</programlisting> + + you can write + +<programlisting> +{ + foo.bar = 123; + foo.xyzzy = true; + a.b.c = "d"; +} +</programlisting> + + This is useful, for instance, in NixOS configuration files.</para> + + </listitem> + + <listitem> + <para>Support for Nix channels generated by Hydra, the Nix-based + continuous build system. (Hydra generates NAR archives on the + fly, so the size and hash of these archives isn’t known in + advance.)</para> + </listitem> + + <listitem> + <para>Support <literal>i686-linux</literal> builds directly on + <literal>x86_64-linux</literal> Nix installations. This is + implemented using the <function>personality()</function> syscall, + which causes <command>uname</command> to return + <literal>i686</literal> in child processes.</para> + </listitem> + + <listitem> + <para>Various improvements to the <literal>chroot</literal> + support. Building in a <literal>chroot</literal> works quite well + now.</para> + </listitem> + + <listitem> + <para>Nix no longer blocks if it tries to build a path and another + process is already building the same path. Instead it tries to + build another buildable path first. This improves + parallelism.</para> + </listitem> + + <listitem> + <para>Support for large (> 4 GiB) files in NAR archives.</para> + </listitem> + + <listitem> + <para>Various (performance) improvements to the remote build + mechanism.</para> + </listitem> + + <listitem> + <para>New primops: <varname>builtins.addErrorContext</varname> (to + add a string to stack traces — useful for debugging), + <varname>builtins.isBool</varname>, + <varname>builtins.isString</varname>, + <varname>builtins.isInt</varname>, + <varname>builtins.intersectAttrs</varname>.</para> + </listitem> + + <listitem> + <para>OpenSolaris support (Sander van der Burg).</para> + </listitem> + + <listitem> + <para>Stack traces are no longer displayed unless the + <option>--show-trace</option> option is used.</para> + </listitem> + + <listitem> + <para>The scoping rules for <literal>inherit + (<replaceable>e</replaceable>) ...</literal> in recursive + attribute sets have changed. The expression + <replaceable>e</replaceable> can now refer to the attributes + defined in the containing set.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.14.xml b/third_party/nix/doc/manual/release-notes/rl-0.14.xml new file mode 100644 index 000000000000..e5fe9da78e7e --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.14.xml @@ -0,0 +1,46 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-0.14"> + +<title>Release 0.14 (2010-02-04)</title> + +<para>This release has the following improvements:</para> + +<itemizedlist> + + <listitem> + <para>The garbage collector now starts deleting garbage much + faster than before. It no longer determines liveness of all paths + in the store, but does so on demand.</para> + </listitem> + + <listitem> + <para>Added a new operation, <command>nix-store --query + --roots</command>, that shows the garbage collector roots that + directly or indirectly point to the given store paths.</para> + </listitem> + + <listitem> + <para>Removed support for converting Berkeley DB-based Nix + databases to the new schema.</para> + </listitem> + + <listitem> + <para>Removed the <option>--use-atime</option> and + <option>--max-atime</option> garbage collector options. They were + not very useful in practice.</para> + </listitem> + + <listitem> + <para>On Windows, Nix now requires Cygwin 1.7.x.</para> + </listitem> + + <listitem> + <para>A few bug fixes.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.15.xml b/third_party/nix/doc/manual/release-notes/rl-0.15.xml new file mode 100644 index 000000000000..9f58a8efc5d4 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.15.xml @@ -0,0 +1,14 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-0.15"> + +<title>Release 0.15 (2010-03-17)</title> + +<para>This is a bug-fix release. Among other things, it fixes +building on Mac OS X (Snow Leopard), and improves the contents of +<filename>/etc/passwd</filename> and <filename>/etc/group</filename> +in <literal>chroot</literal> builds.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.16.xml b/third_party/nix/doc/manual/release-notes/rl-0.16.xml new file mode 100644 index 000000000000..af1edc0ebbe0 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.16.xml @@ -0,0 +1,55 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-0.16"> + +<title>Release 0.16 (2010-08-17)</title> + +<para>This release has the following improvements:</para> + +<itemizedlist> + + <listitem> + <para>The Nix expression evaluator is now much faster in most + cases: typically, <link + xlink:href="http://www.mail-archive.com/nix-dev@cs.uu.nl/msg04113.html">3 + to 8 times compared to the old implementation</link>. It also + uses less memory. It no longer depends on the ATerm + library.</para> + </listitem> + + <listitem> + <para> + Support for configurable parallelism inside builders. Build + scripts have always had the ability to perform multiple build + actions in parallel (for instance, by running <command>make -j + 2</command>), but this was not desirable because the number of + actions to be performed in parallel was not configurable. Nix + now has an option <option>--cores + <replaceable>N</replaceable></option> as well as a configuration + setting <varname>build-cores = + <replaceable>N</replaceable></varname> that causes the + environment variable <envar>NIX_BUILD_CORES</envar> to be set to + <replaceable>N</replaceable> when the builder is invoked. The + builder can use this at its discretion to perform a parallel + build, e.g., by calling <command>make -j + <replaceable>N</replaceable></command>. In Nixpkgs, this can be + enabled on a per-package basis by setting the derivation + attribute <varname>enableParallelBuilding</varname> to + <literal>true</literal>. + </para> + </listitem> + + <listitem> + <para><command>nix-store -q</command> now supports XML output + through the <option>--xml</option> flag.</para> + </listitem> + + <listitem> + <para>Several bug fixes.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.5.xml b/third_party/nix/doc/manual/release-notes/rl-0.5.xml new file mode 100644 index 000000000000..e9f8bf2701b8 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.5.xml @@ -0,0 +1,11 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.5"> + +<title>Release 0.5 and earlier</title> + +<para>Please refer to the Subversion commit log messages.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.6.xml b/third_party/nix/doc/manual/release-notes/rl-0.6.xml new file mode 100644 index 000000000000..6dc6521d3c2a --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.6.xml @@ -0,0 +1,122 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.6"> + +<title>Release 0.6 (2004-11-14)</title> + +<itemizedlist> + + <listitem> + <para>Rewrite of the normalisation engine. + + <itemizedlist> + + <listitem><para>Multiple builds can now be performed in parallel + (option <option>-j</option>).</para></listitem> + + <listitem><para>Distributed builds. Nix can now call a shell + script to forward builds to Nix installations on remote + machines, which may or may not be of the same platform + type.</para></listitem> + + <listitem><para>Option <option>--fallback</option> allows + recovery from broken substitutes.</para></listitem> + + <listitem><para>Option <option>--keep-going</option> causes + building of other (unaffected) derivations to continue if one + failed.</para></listitem> + + </itemizedlist> + + </para> + + </listitem> + + <listitem><para>Improvements to the garbage collector (i.e., it + should actually work now).</para></listitem> + + <listitem><para>Setuid Nix installations allow a Nix store to be + shared among multiple users.</para></listitem> + + <listitem><para>Substitute registration is much faster + now.</para></listitem> + + <listitem><para>A utility <command>nix-build</command> to build a + Nix expression and create a symlink to the result int the current + directory; useful for testing Nix derivations.</para></listitem> + + <listitem><para>Manual updates.</para></listitem> + + <listitem> + + <para><command>nix-env</command> changes: + + <itemizedlist> + + <listitem><para>Derivations for other platforms are filtered out + (which can be overridden using + <option>--system-filter</option>).</para></listitem> + + <listitem><para><option>--install</option> by default now + uninstall previous derivations with the same + name.</para></listitem> + + <listitem><para><option>--upgrade</option> allows upgrading to a + specific version.</para></listitem> + + <listitem><para>New operation + <option>--delete-generations</option> to remove profile + generations (necessary for effective garbage + collection).</para></listitem> + + <listitem><para>Nicer output (sorted, + columnised).</para></listitem> + + </itemizedlist> + + </para> + + </listitem> + + <listitem><para>More sensible verbosity levels all around (builder + output is now shown always, unless <option>-Q</option> is + given).</para></listitem> + + <listitem> + + <para>Nix expression language changes: + + <itemizedlist> + + <listitem><para>New language construct: <literal>with + <replaceable>E1</replaceable>; + <replaceable>E2</replaceable></literal> brings all attributes + defined in the attribute set <replaceable>E1</replaceable> in + scope in <replaceable>E2</replaceable>.</para></listitem> + + <listitem><para>Added a <function>map</function> + function.</para></listitem> + + <listitem><para>Various new operators (e.g., string + concatenation).</para></listitem> + + </itemizedlist> + + </para> + + </listitem> + + <listitem><para>Expression evaluation is much + faster.</para></listitem> + + <listitem><para>An Emacs mode for editing Nix expressions (with + syntax highlighting and indentation) has been + added.</para></listitem> + + <listitem><para>Many bug fixes.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.7.xml b/third_party/nix/doc/manual/release-notes/rl-0.7.xml new file mode 100644 index 000000000000..6f95db4367db --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.7.xml @@ -0,0 +1,35 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.7"> + +<title>Release 0.7 (2005-01-12)</title> + +<itemizedlist> + + <listitem><para>Binary patching. When upgrading components using + pre-built binaries (through nix-pull / nix-channel), Nix can + automatically download and apply binary patches to already installed + components instead of full downloads. Patching is “smart”: if there + is a <emphasis>sequence</emphasis> of patches to an installed + component, Nix will use it. Patches are currently generated + automatically between Nixpkgs (pre-)releases.</para></listitem> + + <listitem><para>Simplifications to the substitute + mechanism.</para></listitem> + + <listitem><para>Nix-pull now stores downloaded manifests in + <filename>/nix/var/nix/manifests</filename>.</para></listitem> + + <listitem><para>Metadata on files in the Nix store is canonicalised + after builds: the last-modified timestamp is set to 0 (00:00:00 + 1/1/1970), the mode is set to 0444 or 0555 (readable and possibly + executable by all; setuid/setgid bits are dropped), and the group is + set to the default. This ensures that the result of a build and an + installation through a substitute is the same; and that timestamp + dependencies are revealed.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.8.1.xml b/third_party/nix/doc/manual/release-notes/rl-0.8.1.xml new file mode 100644 index 000000000000..f7ffca0f8d59 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.8.1.xml @@ -0,0 +1,21 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.8.1"> + +<title>Release 0.8.1 (2005-04-13)</title> + +<para>This is a bug fix release.</para> + +<itemizedlist> + + <listitem><para>Patch downloading was broken.</para></listitem> + + <listitem><para>The garbage collector would not delete paths that + had references from invalid (but substitutable) + paths.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.8.xml b/third_party/nix/doc/manual/release-notes/rl-0.8.xml new file mode 100644 index 000000000000..784b26c6b7d3 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.8.xml @@ -0,0 +1,246 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.8"> + +<title>Release 0.8 (2005-04-11)</title> + +<para>NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). +As a result, <command>nix-pull</command> manifests and channels built +for Nix 0.7 and below will now work anymore. However, the Nix +expression language has not changed, so you can still build from +source. Also, existing user environments continue to work. Nix 0.8 +will automatically upgrade the database schema of previous +installations when it is first run.</para> + +<para>If you get the error message + +<screen> +you have an old-style manifest `/nix/var/nix/manifests/[...]'; please +delete it</screen> + +you should delete previously downloaded manifests: + +<screen> +$ rm /nix/var/nix/manifests/*</screen> + +If <command>nix-channel</command> gives the error message + +<screen> +manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST' +is too old (i.e., for Nix <= 0.7)</screen> + +then you should unsubscribe from the offending channel +(<command>nix-channel --remove +<replaceable>URL</replaceable></command>; leave out +<literal>/MANIFEST</literal>), and subscribe to the same URL, with +<literal>channels</literal> replaced by <literal>channels-v3</literal> +(e.g., <link +xlink:href='http://catamaran.labs.cs.uu.nl/dist/nix/channels-v3/nixpkgs-unstable' +/>).</para> + +<para>Nix 0.8 has the following improvements: + +<itemizedlist> + + <listitem><para>The cryptographic hashes used in store paths are now + 160 bits long, but encoded in base-32 so that they are still only 32 + characters long (e.g., + <filename>/nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0</filename>). + (This is actually a 160 bit truncation of a SHA-256 + hash.)</para></listitem> + + <listitem><para>Big cleanups and simplifications of the basic store + semantics. The notion of “closure store expressions” is gone (and + so is the notion of “successors”); the file system references of a + store path are now just stored in the database.</para> + + <para>For instance, given any store path, you can query its closure: + + <screen> +$ nix-store -qR $(which firefox) +... lots of paths ...</screen> + + Also, Nix now remembers for each store path the derivation that + built it (the “deriver”): + + <screen> +$ nix-store -qR $(which firefox) +/nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv</screen> + + So to see the build-time dependencies, you can do + + <screen> +$ nix-store -qR $(nix-store -qd $(which firefox))</screen> + + or, in a nicer format: + + <screen> +$ nix-store -q --tree $(nix-store -qd $(which firefox))</screen> + + </para> + + <para>File system references are also stored in reverse. For + instance, you can query all paths that directly or indirectly use a + certain Glibc: + + <screen> +$ nix-store -q --referrers-closure \ + /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4</screen> + + </para> + + </listitem> + + <listitem><para>The concept of fixed-output derivations has been + formalised. Previously, functions such as + <function>fetchurl</function> in Nixpkgs used a hack (namely, + explicitly specifying a store path hash) to prevent changes to, say, + the URL of the file from propagating upwards through the dependency + graph, causing rebuilds of everything. This can now be done cleanly + by specifying the <varname>outputHash</varname> and + <varname>outputHashAlgo</varname> attributes. Nix itself checks + that the content of the output has the specified hash. (This is + important for maintaining certain invariants necessary for future + work on secure shared stores.)</para></listitem> + + <listitem><para>One-click installation :-) It is now possible to + install any top-level component in Nixpkgs directly, through the web + — see, e.g., <link + xlink:href='http://catamaran.labs.cs.uu.nl/dist/nixpkgs-0.8/' />. + All you have to do is associate + <filename>/nix/bin/nix-install-package</filename> with the MIME type + <literal>application/nix-package</literal> (or the extension + <filename>.nixpkg</filename>), and clicking on a package link will + cause it to be installed, with all appropriate dependencies. If you + just want to install some specific application, this is easier than + subscribing to a channel.</para></listitem> + + <listitem><para><command>nix-store -r + <replaceable>PATHS</replaceable></command> now builds all the + derivations PATHS in parallel. Previously it did them sequentially + (though exploiting possible parallelism between subderivations). + This is nice for build farms.</para></listitem> + + <listitem><para><command>nix-channel</command> has new operations + <option>--list</option> and + <option>--remove</option>.</para></listitem> + + <listitem><para>New ways of installing components into user + environments: + + <itemizedlist> + + <listitem><para>Copy from another user environment: + + <screen> +$ nix-env -i --from-profile .../other-profile firefox</screen> + + </para></listitem> + + <listitem><para>Install a store derivation directly (bypassing the + Nix expression language entirely): + + <screen> +$ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv</screen> + + (This is used to implement <command>nix-install-package</command>, + which is therefore immune to evolution in the Nix expression + language.)</para></listitem> + + <listitem><para>Install an already built store path directly: + + <screen> +$ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1</screen> + + </para></listitem> + + <listitem><para>Install the result of a Nix expression specified + as a command-line argument: + + <screen> +$ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper'</screen> + + The difference with the normal installation mode is that + <option>-E</option> does not use the <varname>name</varname> + attributes of derivations. Therefore, this can be used to + disambiguate multiple derivations with the same + name.</para></listitem> + + </itemizedlist></para></listitem> + + <listitem><para>A hash of the contents of a store path is now stored + in the database after a successful build. This allows you to check + whether store paths have been tampered with: <command>nix-store + --verify --check-contents</command>.</para></listitem> + + <listitem> + + <para>Implemented a concurrent garbage collector. It is now + always safe to run the garbage collector, even if other Nix + operations are happening simultaneously.</para> + + <para>However, there can still be GC races if you use + <command>nix-instantiate</command> and <command>nix-store + --realise</command> directly to build things. To prevent races, + use the <option>--add-root</option> flag of those commands.</para> + + </listitem> + + <listitem><para>The garbage collector now finally deletes paths in + the right order (i.e., topologically sorted under the “references” + relation), thus making it safe to interrupt the collector without + risking a store that violates the closure + invariant.</para></listitem> + + <listitem><para>Likewise, the substitute mechanism now downloads + files in the right order, thus preserving the closure invariant at + all times.</para></listitem> + + <listitem><para>The result of <command>nix-build</command> is now + registered as a root of the garbage collector. If the + <filename>./result</filename> link is deleted, the GC root + disappears automatically.</para></listitem> + + <listitem> + + <para>The behaviour of the garbage collector can be changed + globally by setting options in + <filename>/nix/etc/nix/nix.conf</filename>. + + <itemizedlist> + + <listitem><para><literal>gc-keep-derivations</literal> specifies + whether deriver links should be followed when searching for live + paths.</para></listitem> + + <listitem><para><literal>gc-keep-outputs</literal> specifies + whether outputs of derivations should be followed when searching + for live paths.</para></listitem> + + <listitem><para><literal>env-keep-derivations</literal> + specifies whether user environments should store the paths of + derivations when they are added (thus keeping the derivations + alive).</para></listitem> + + </itemizedlist> + + </para></listitem> + + <listitem><para>New <command>nix-env</command> query flags + <option>--drv-path</option> and + <option>--out-path</option>.</para></listitem> + + <listitem><para><command>fetchurl</command> allows SHA-1 and SHA-256 + in addition to MD5. Just specify the attribute + <varname>sha1</varname> or <varname>sha256</varname> instead of + <varname>md5</varname>.</para></listitem> + + <listitem><para>Manual updates.</para></listitem> + +</itemizedlist> + +</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.9.1.xml b/third_party/nix/doc/manual/release-notes/rl-0.9.1.xml new file mode 100644 index 000000000000..85d11f416877 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.9.1.xml @@ -0,0 +1,13 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.9.1"> + +<title>Release 0.9.1 (2005-09-20)</title> + +<para>This bug fix release addresses a problem with the ATerm library +when the <option>--with-aterm</option> flag in +<command>configure</command> was <emphasis>not</emphasis> used.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.9.2.xml b/third_party/nix/doc/manual/release-notes/rl-0.9.2.xml new file mode 100644 index 000000000000..cb705e98ac25 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.9.2.xml @@ -0,0 +1,28 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.9.2"> + +<title>Release 0.9.2 (2005-09-21)</title> + +<para>This bug fix release fixes two problems on Mac OS X: + +<itemizedlist> + + <listitem><para>If Nix was linked against statically linked versions + of the ATerm or Berkeley DB library, there would be dynamic link + errors at runtime.</para></listitem> + + <listitem><para><command>nix-pull</command> and + <command>nix-push</command> intermittently failed due to race + conditions involving pipes and child processes with error messages + such as <literal>open2: open(GLOB(0x180b2e4), >&=9) failed: Bad + file descriptor at /nix/bin/nix-pull line 77</literal> (issue + <literal>NIX-14</literal>).</para></listitem> + +</itemizedlist> + +</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-0.9.xml b/third_party/nix/doc/manual/release-notes/rl-0.9.xml new file mode 100644 index 000000000000..fd1e633f78ea --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.9.xml @@ -0,0 +1,98 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ch-relnotes-0.9"> + +<title>Release 0.9 (2005-09-16)</title> + +<para>NOTE: this version of Nix uses Berkeley DB 4.3 instead of 4.2. +The database is upgraded automatically, but you should be careful not +to use old versions of Nix that still use Berkeley DB 4.2. In +particular, if you use a Nix installed through Nix, you should run + +<screen> +$ nix-store --clear-substitutes</screen> + +first.</para> + + +<itemizedlist> + + <listitem><para>Unpacking of patch sequences is much faster now + since we no longer do redundant unpacking and repacking of + intermediate paths.</para></listitem> + + <listitem><para>Nix now uses Berkeley DB 4.3.</para></listitem> + + <listitem><para>The <function>derivation</function> primitive is + lazier. Attributes of dependent derivations can mutually refer to + each other (as long as there are no data dependencies on the + <varname>outPath</varname> and <varname>drvPath</varname> attributes + computed by <function>derivation</function>).</para> + + <para>For example, the expression <literal>derivation + attrs</literal> now evaluates to (essentially) + + <programlisting> +attrs // { + type = "derivation"; + outPath = derivation! attrs; + drvPath = derivation! attrs; +}</programlisting> + + where <function>derivation!</function> is a primop that does the + actual derivation instantiation (i.e., it does what + <function>derivation</function> used to do). The advantage is that + it allows commands such as <command>nix-env -qa</command> and + <command>nix-env -i</command> to be much faster since they no longer + need to instantiate all derivations, just the + <varname>name</varname> attribute.</para> + + <para>Also, it allows derivations to cyclically reference each + other, for example, + + <programlisting> +webServer = derivation { + ... + hostName = "svn.cs.uu.nl"; + services = [svnService]; +}; +  +svnService = derivation { + ... + hostName = webServer.hostName; +};</programlisting> + + Previously, this would yield a black hole (infinite recursion).</para> + + </listitem> + + <listitem><para><command>nix-build</command> now defaults to using + <filename>./default.nix</filename> if no Nix expression is + specified.</para></listitem> + + <listitem><para><command>nix-instantiate</command>, when applied to + a Nix expression that evaluates to a function, will call the + function automatically if all its arguments have + defaults.</para></listitem> + + <listitem><para>Nix now uses libtool to build dynamic libraries. + This reduces the size of executables.</para></listitem> + + <listitem><para>A new list concatenation operator + <literal>++</literal>. For example, <literal>[1 2 3] ++ [4 5 + 6]</literal> evaluates to <literal>[1 2 3 4 5 + 6]</literal>.</para></listitem> + + <listitem><para>Some currently undocumented primops to support + low-level build management using Nix (i.e., using Nix as a Make + replacement). See the commit messages for <literal>r3578</literal> + and <literal>r3580</literal>.</para></listitem> + + <listitem><para>Various bug fixes and performance + improvements.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.0.xml b/third_party/nix/doc/manual/release-notes/rl-1.0.xml new file mode 100644 index 000000000000..ff11168d0932 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.0.xml @@ -0,0 +1,119 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.0"> + +<title>Release 1.0 (2012-05-11)</title> + +<para>There have been numerous improvements and bug fixes since the +previous release. Here are the most significant:</para> + +<itemizedlist> + + <listitem> + <para>Nix can now optionally use the Boehm garbage collector. + This significantly reduces the Nix evaluator’s memory footprint, + especially when evaluating large NixOS system configurations. It + can be enabled using the <option>--enable-gc</option> configure + option.</para> + </listitem> + + <listitem> + <para>Nix now uses SQLite for its database. This is faster and + more flexible than the old <emphasis>ad hoc</emphasis> format. + SQLite is also used to cache the manifests in + <filename>/nix/var/nix/manifests</filename>, resulting in a + significant speedup.</para> + </listitem> + + <listitem> + <para>Nix now has an search path for expressions. The search path + is set using the environment variable <envar>NIX_PATH</envar> and + the <option>-I</option> command line option. In Nix expressions, + paths between angle brackets are used to specify files that must + be looked up in the search path. For instance, the expression + <literal><nixpkgs/default.nix></literal> looks for a file + <filename>nixpkgs/default.nix</filename> relative to every element + in the search path.</para> + </listitem> + + <listitem> + <para>The new command <command>nix-build --run-env</command> + builds all dependencies of a derivation, then starts a shell in an + environment containing all variables from the derivation. This is + useful for reproducing the environment of a derivation for + development.</para> + </listitem> + + <listitem> + <para>The new command <command>nix-store --verify-path</command> + verifies that the contents of a store path have not + changed.</para> + </listitem> + + <listitem> + <para>The new command <command>nix-store --print-env</command> + prints out the environment of a derivation in a format that can be + evaluated by a shell.</para> + </listitem> + + <listitem> + <para>Attribute names can now be arbitrary strings. For instance, + you can write <literal>{ "foo-1.2" = …; "bla bla" = …; }."bla + bla"</literal>.</para> + </listitem> + + <listitem> + <para>Attribute selection can now provide a default value using + the <literal>or</literal> operator. For instance, the expression + <literal>x.y.z or e</literal> evaluates to the attribute + <literal>x.y.z</literal> if it exists, and <literal>e</literal> + otherwise.</para> + </listitem> + + <listitem> + <para>The right-hand side of the <literal>?</literal> operator can + now be an attribute path, e.g., <literal>attrs ? + a.b.c</literal>.</para> + </listitem> + + <listitem> + <para>On Linux, Nix will now make files in the Nix store immutable + on filesystems that support it. This prevents accidental + modification of files in the store by the root user.</para> + </listitem> + + <listitem> + <para>Nix has preliminary support for derivations with multiple + outputs. This is useful because it allows parts of a package to + be deployed and garbage-collected separately. For instance, + development parts of a package such as header files or static + libraries would typically not be part of the closure of an + application, resulting in reduced disk usage and installation + time.</para> + </listitem> + + <listitem> + <para>The Nix store garbage collector is faster and holds the + global lock for a shorter amount of time.</para> + </listitem> + + <listitem> + <para>The option <option>--timeout</option> (corresponding to the + configuration setting <literal>build-timeout</literal>) allows you + to set an absolute timeout on builds — if a build runs for more than + the given number of seconds, it is terminated. This is useful for + recovering automatically from builds that are stuck in an infinite + loop but keep producing output, and for which + <literal>--max-silent-time</literal> is ineffective.</para> + </listitem> + + <listitem> + <para>Nix development has moved to GitHub (<link + xlink:href="https://github.com/NixOS/nix" />).</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.1.xml b/third_party/nix/doc/manual/release-notes/rl-1.1.xml new file mode 100644 index 000000000000..2f26e7a24273 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.1.xml @@ -0,0 +1,100 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.1"> + +<title>Release 1.1 (2012-07-18)</title> + +<para>This release has the following improvements:</para> + +<itemizedlist> + + <listitem> + <para>On Linux, when doing a chroot build, Nix now uses various + namespace features provided by the Linux kernel to improve + build isolation. Namely: + <itemizedlist> + <listitem><para>The private network namespace ensures that + builders cannot talk to the outside world (or vice versa): each + build only sees a private loopback interface. This also means + that two concurrent builds can listen on the same port (e.g. as + part of a test) without conflicting with each + other.</para></listitem> + <listitem><para>The PID namespace causes each build to start as + PID 1. Processes outside of the chroot are not visible to those + on the inside. On the other hand, processes inside the chroot + <emphasis>are</emphasis> visible from the outside (though with + different PIDs).</para></listitem> + <listitem><para>The IPC namespace prevents the builder from + communicating with outside processes using SysV IPC mechanisms + (shared memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits.</para></listitem> + <listitem><para>The UTS namespace ensures that builders see a + hostname of <literal>localhost</literal> rather than the actual + hostname.</para></listitem> + <listitem><para>The private mount namespace was already used by + Nix to ensure that the bind-mounts used to set up the chroot are + cleaned up automatically.</para></listitem> + </itemizedlist> + </para> + </listitem> + + <listitem> + <para>Build logs are now compressed using + <command>bzip2</command>. The command <command>nix-store + -l</command> decompresses them on the fly. This can be disabled + by setting the option <literal>build-compress-log</literal> to + <literal>false</literal>.</para> + </listitem> + + <listitem> + <para>The creation of build logs in + <filename>/nix/var/log/nix/drvs</filename> can be disabled by + setting the new option <literal>build-keep-log</literal> to + <literal>false</literal>. This is useful, for instance, for Hydra + build machines.</para> + </listitem> + + <listitem> + <para>Nix now reserves some space in + <filename>/nix/var/nix/db/reserved</filename> to ensure that the + garbage collector can run successfully if the disk is full. This + is necessary because SQLite transactions fail if the disk is + full.</para> + </listitem> + + <listitem> + <para>Added a basic <function>fetchurl</function> function. This + is not intended to replace the <function>fetchurl</function> in + Nixpkgs, but is useful for bootstrapping; e.g., it will allow us + to get rid of the bootstrap binaries in the Nixpkgs source tree + and download them instead. You can use it by doing + <literal>import <nix/fetchurl.nix> { url = + <replaceable>url</replaceable>; sha256 = + "<replaceable>hash</replaceable>"; }</literal>. (Shea Levy)</para> + </listitem> + + <listitem> + <para>Improved RPM spec file. (Michel Alexandre Salim)</para> + </listitem> + + <listitem> + <para>Support for on-demand socket-based activation in the Nix + daemon with <command>systemd</command>.</para> + </listitem> + + <listitem> + <para>Added a manpage for + <citerefentry><refentrytitle>nix.conf</refentrytitle><manvolnum>5</manvolnum></citerefentry>.</para> + </listitem> + + <listitem> + <para>When using the Nix daemon, the <option>-s</option> flag in + <command>nix-env -qa</command> is now much faster.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.10.xml b/third_party/nix/doc/manual/release-notes/rl-1.10.xml new file mode 100644 index 000000000000..689a95466343 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.10.xml @@ -0,0 +1,64 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.10"> + +<title>Release 1.10 (2015-09-03)</title> + +<para>This is primarily a bug fix release. It also has a number of new +features:</para> + +<itemizedlist> + + <listitem> + <para>A number of builtin functions have been added to reduce + Nixpkgs/NixOS evaluation time and memory consumption: + <function>all</function>, + <function>any</function>, + <function>concatStringsSep</function>, + <function>foldl’</function>, + <function>genList</function>, + <function>replaceStrings</function>, + <function>sort</function>. + </para> + </listitem> + + <listitem> + <para>The garbage collector is more robust when the disk is full.</para> + </listitem> + + <listitem> + <para>Nix supports a new API for building derivations that doesn’t + require a <literal>.drv</literal> file to be present on disk; it + only requires an in-memory representation of the derivation. This + is used by the Hydra continuous build system to make remote builds + more efficient.</para> + </listitem> + + <listitem> + <para>The function <literal><nix/fetchurl.nix></literal> now + uses a <emphasis>builtin</emphasis> builder (i.e. it doesn’t + require starting an external process; the download is performed by + Nix itself). This ensures that derivation paths don’t change when + Nix is upgraded, and obviates the need for ugly hacks to support + chroot execution.</para> + </listitem> + + <listitem> + <para><option>--version -v</option> now prints some configuration + information, in particular what compile-time optional features are + enabled, and the paths of various directories.</para> + </listitem> + + <listitem> + <para>Build users have their supplementary groups set correctly.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Eelco Dolstra, Guillaume +Maudoux, Iwan Aucamp, Jaka Hudoklin, Kirill Elagin, Ludovic Courtès, +Manolis Ragkousis, Nicolas B. Pierron and Shea Levy.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.11.10.xml b/third_party/nix/doc/manual/release-notes/rl-1.11.10.xml new file mode 100644 index 000000000000..415388b3e2d1 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.11.10.xml @@ -0,0 +1,31 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.11.10"> + +<title>Release 1.11.10 (2017-06-12)</title> + +<para>This release fixes a security bug in Nix’s “build user” build +isolation mechanism. Previously, Nix builders had the ability to +create setuid binaries owned by a <literal>nixbld</literal> +user. Such a binary could then be used by an attacker to assume a +<literal>nixbld</literal> identity and interfere with subsequent +builds running under the same UID.</para> + +<para>To prevent this issue, Nix now disallows builders to create +setuid and setgid binaries. On Linux, this is done using a seccomp BPF +filter. Note that this imposes a small performance penalty (e.g. 1% +when building GNU Hello). Using seccomp, we now also prevent the +creation of extended attributes and POSIX ACLs since these cannot be +represented in the NAR format and (in the case of POSIX ACLs) allow +bypassing regular Nix store permissions. On macOS, the restriction is +implemented using the existing sandbox mechanism, which now uses a +minimal “allow all except the creation of setuid/setgid binaries” +profile when regular sandboxing is disabled. On other platforms, the +“build user” mechanism is now disabled.</para> + +<para>Thanks go to Linus Heckemann for discovering and reporting this +bug.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.11.xml b/third_party/nix/doc/manual/release-notes/rl-1.11.xml new file mode 100644 index 000000000000..fe422dd1f893 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.11.xml @@ -0,0 +1,141 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.11"> + +<title>Release 1.11 (2016-01-19)</title> + +<para>This is primarily a bug fix release. It also has a number of new +features:</para> + +<itemizedlist> + + <listitem> + <para><command>nix-prefetch-url</command> can now download URLs + specified in a Nix expression. For example, + +<screen> +$ nix-prefetch-url -A hello.src +</screen> + + will prefetch the file specified by the + <function>fetchurl</function> call in the attribute + <literal>hello.src</literal> from the Nix expression in the + current directory, and print the cryptographic hash of the + resulting file on stdout. This differs from <literal>nix-build -A + hello.src</literal> in that it doesn't verify the hash, and is + thus useful when you’re updating a Nix expression.</para> + + <para>You can also prefetch the result of functions that unpack a + tarball, such as <function>fetchFromGitHub</function>. For example: + +<screen> +$ nix-prefetch-url --unpack https://github.com/NixOS/patchelf/archive/0.8.tar.gz +</screen> + + or from a Nix expression: + +<screen> +$ nix-prefetch-url -A nix-repl.src +</screen> + + </para> + + </listitem> + + <listitem> + <para>The builtin function + <function><nix/fetchurl.nix></function> now supports + downloading and unpacking NARs. This removes the need to have + multiple downloads in the Nixpkgs stdenv bootstrap process (like a + separate busybox binary for Linux, or curl/mkdir/sh/bzip2 for + Darwin). Now all those files can be combined into a single NAR, + optionally compressed using <command>xz</command>.</para> + </listitem> + + <listitem> + <para>Nix now supports SHA-512 hashes for verifying fixed-output + derivations, and in <function>builtins.hashString</function>.</para> + </listitem> + + <listitem> + <para> + The new flag <option>--option build-repeat + <replaceable>N</replaceable></option> will cause every build to + be executed <replaceable>N</replaceable>+1 times. If the build + output differs between any round, the build is rejected, and the + output paths are not registered as valid. This is primarily + useful to verify build determinism. (We already had a + <option>--check</option> option to repeat a previously succeeded + build. However, with <option>--check</option>, non-deterministic + builds are registered in the DB. Preventing that is useful for + Hydra to ensure that non-deterministic builds don't end up + getting published to the binary cache.) + </para> + </listitem> + + <listitem> + <para> + The options <option>--check</option> and <option>--option + build-repeat <replaceable>N</replaceable></option>, if they + detect a difference between two runs of the same derivation and + <option>-K</option> is given, will make the output of the other + run available under + <filename><replaceable>store-path</replaceable>-check</filename>. This + makes it easier to investigate the non-determinism using tools + like <command>diffoscope</command>, e.g., + +<screen> +$ nix-build pkgs/stdenv/linux -A stage1.pkgs.zlib --check -K +error: derivation ‘/nix/store/l54i8wlw2265…-zlib-1.2.8.drv’ may not +be deterministic: output ‘/nix/store/11a27shh6n2i…-zlib-1.2.8’ +differs from ‘/nix/store/11a27shh6n2i…-zlib-1.2.8-check’ + +$ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zlib-1.2.8-check +… +├── lib/libz.a +│ ├── metadata +│ │ @@ -1,15 +1,15 @@ +│ │ -rw-r--r-- 30001/30000 3096 Jan 12 15:20 2016 adler32.o +… +│ │ +rw-r--r-- 30001/30000 3096 Jan 12 15:28 2016 adler32.o +… +</screen> + + </para></listitem> + + <listitem> + <para>Improved FreeBSD support.</para> + </listitem> + + <listitem> + <para><command>nix-env -qa --xml --meta</command> now prints + license information.</para> + </listitem> + + <listitem> + <para>The maximum number of parallel TCP connections that the + binary cache substituter will use has been decreased from 150 to + 25. This should prevent upsetting some broken NAT routers, and + also improves performance.</para> + </listitem> + + <listitem> + <para>All "chroot"-containing strings got renamed to "sandbox". + In particular, some Nix options got renamed, but the old names + are still accepted as lower-priority aliases. + </para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Anders Claesson, Anthony +Cowley, Bjørn Forsman, Brian McKenna, Danny Wilson, davidak, Eelco Dolstra, +Fabian Schmitthenner, FrankHB, Ilya Novoselov, janus, Jim Garrison, John +Ericson, Jude Taylor, Ludovic Courtès, Manuel Jacob, Mathnerd314, +Pascal Wittmann, Peter Simons, Philip Potter, Preston Bennes, Rommel +M. Martinez, Sander van der Burg, Shea Levy, Tim Cuthbertson, Tuomas +Tynkkynen, Utku Demir and Vladimír Čunát.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.2.xml b/third_party/nix/doc/manual/release-notes/rl-1.2.xml new file mode 100644 index 000000000000..748fd9e67022 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.2.xml @@ -0,0 +1,157 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.2"> + +<title>Release 1.2 (2012-12-06)</title> + +<para>This release has the following improvements and changes:</para> + +<itemizedlist> + + <listitem> + <para>Nix has a new binary substituter mechanism: the + <emphasis>binary cache</emphasis>. A binary cache contains + pre-built binaries of Nix packages. Whenever Nix wants to build a + missing Nix store path, it will check a set of binary caches to + see if any of them has a pre-built binary of that path. The + configuration setting <option>binary-caches</option> contains a + list of URLs of binary caches. For instance, doing +<screen> +$ nix-env -i thunderbird --option binary-caches http://cache.nixos.org +</screen> + will install Thunderbird and its dependencies, using the available + pre-built binaries in <uri>http://cache.nixos.org</uri>. + The main advantage over the old “manifest”-based method of getting + pre-built binaries is that you don’t have to worry about your + manifest being in sync with the Nix expressions you’re installing + from; i.e., you don’t need to run <command>nix-pull</command> to + update your manifest. It’s also more scalable because you don’t + need to redownload a giant manifest file every time. + </para> + + <para>A Nix channel can provide a binary cache URL that will be + used automatically if you subscribe to that channel. If you use + the Nixpkgs or NixOS channels + (<uri>http://nixos.org/channels</uri>) you automatically get the + cache <uri>http://cache.nixos.org</uri>.</para> + + <para>Binary caches are created using <command>nix-push</command>. + For details on the operation and format of binary caches, see the + <command>nix-push</command> manpage. More details are provided in + <link xlink:href="https://nixos.org/nix-dev/2012-September/009826.html">this + nix-dev posting</link>.</para> + </listitem> + + <listitem> + <para>Multiple output support should now be usable. A derivation + can declare that it wants to produce multiple store paths by + saying something like +<programlisting> +outputs = [ "lib" "headers" "doc" ]; +</programlisting> + This will cause Nix to pass the intended store path of each output + to the builder through the environment variables + <literal>lib</literal>, <literal>headers</literal> and + <literal>doc</literal>. Other packages can refer to a specific + output by referring to + <literal><replaceable>pkg</replaceable>.<replaceable>output</replaceable></literal>, + e.g. +<programlisting> +buildInputs = [ pkg.lib pkg.headers ]; +</programlisting> + If you install a package with multiple outputs using + <command>nix-env</command>, each output path will be symlinked + into the user environment.</para> + </listitem> + + <listitem> + <para>Dashes are now valid as part of identifiers and attribute + names.</para> + </listitem> + + <listitem> + <para>The new operation <command>nix-store --repair-path</command> + allows corrupted or missing store paths to be repaired by + redownloading them. <command>nix-store --verify --check-contents + --repair</command> will scan and repair all paths in the Nix + store. Similarly, <command>nix-env</command>, + <command>nix-build</command>, <command>nix-instantiate</command> + and <command>nix-store --realise</command> have a + <option>--repair</option> flag to detect and fix bad paths by + rebuilding or redownloading them.</para> + </listitem> + + <listitem> + <para>Nix no longer sets the immutable bit on files in the Nix + store. Instead, the recommended way to guard the Nix store + against accidental modification on Linux is to make it a read-only + bind mount, like this: + +<screen> +$ mount --bind /nix/store /nix/store +$ mount -o remount,ro,bind /nix/store +</screen> + + Nix will automatically make <filename>/nix/store</filename> + writable as needed (using a private mount namespace) to allow + modifications.</para> + </listitem> + + <listitem> + <para>Store optimisation (replacing identical files in the store + with hard links) can now be done automatically every time a path + is added to the store. This is enabled by setting the + configuration option <literal>auto-optimise-store</literal> to + <literal>true</literal> (disabled by default).</para> + </listitem> + + <listitem> + <para>Nix now supports <command>xz</command> compression for NARs + in addition to <command>bzip2</command>. It compresses about 30% + better on typical archives and decompresses about twice as + fast.</para> + </listitem> + + <listitem> + <para>Basic Nix expression evaluation profiling: setting the + environment variable <envar>NIX_COUNT_CALLS</envar> to + <literal>1</literal> will cause Nix to print how many times each + primop or function was executed.</para> + </listitem> + + <listitem> + <para>New primops: <varname>concatLists</varname>, + <varname>elem</varname>, <varname>elemAt</varname> and + <varname>filter</varname>.</para> + </listitem> + + <listitem> + <para>The command <command>nix-copy-closure</command> has a new + flag <option>--use-substitutes</option> (<option>-s</option>) to + download missing paths on the target machine using the substitute + mechanism.</para> + </listitem> + + <listitem> + <para>The command <command>nix-worker</command> has been renamed + to <command>nix-daemon</command>. Support for running the Nix + worker in “slave” mode has been removed.</para> + </listitem> + + <listitem> + <para>The <option>--help</option> flag of every Nix command now + invokes <command>man</command>.</para> + </listitem> + + <listitem> + <para>Chroot builds are now supported on systemd machines.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Eelco Dolstra, Florian +Friesdorf, Mats Erik Andersson and Shea Levy.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.3.xml b/third_party/nix/doc/manual/release-notes/rl-1.3.xml new file mode 100644 index 000000000000..e2009ee3ba4b --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.3.xml @@ -0,0 +1,19 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.3"> + +<title>Release 1.3 (2013-01-04)</title> + +<para>This is primarily a bug fix release. When this version is first +run on Linux, it removes any immutable bits from the Nix store and +increases the schema version of the Nix store. (The previous release +removed support for setting the immutable bit; this release clears any +remaining immutable bits to make certain operations more +efficient.)</para> + +<para>This release has contributions from Eelco Dolstra and Stuart +Pernsteiner.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.4.xml b/third_party/nix/doc/manual/release-notes/rl-1.4.xml new file mode 100644 index 000000000000..aefb22f2b934 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.4.xml @@ -0,0 +1,39 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.4"> + +<title>Release 1.4 (2013-02-26)</title> + +<para>This release fixes a security bug in multi-user operation. It +was possible for derivations to cause the mode of files outside of the +Nix store to be changed to 444 (read-only but world-readable) by +creating hard links to those files (<link +xlink:href="https://github.com/NixOS/nix/commit/5526a282b5b44e9296e61e07d7d2626a79141ac4">details</link>).</para> + +<para>There are also the following improvements:</para> + +<itemizedlist> + + <listitem><para>New built-in function: + <function>builtins.hashString</function>.</para></listitem> + + <listitem><para>Build logs are now stored in + <filename>/nix/var/log/nix/drvs/<replaceable>XX</replaceable>/</filename>, + where <replaceable>XX</replaceable> is the first two characters of + the derivation. This is useful on machines that keep a lot of build + logs (such as Hydra servers).</para></listitem> + + <listitem><para>The function <function>corepkgs/fetchurl</function> + can now make the downloaded file executable. This will allow + getting rid of all bootstrap binaries in the Nixpkgs source + tree.</para></listitem> + + <listitem><para>Language change: The expression <literal>"${./path} + ..."</literal> now evaluates to a string instead of a + path.</para></listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.5.1.xml b/third_party/nix/doc/manual/release-notes/rl-1.5.1.xml new file mode 100644 index 000000000000..035c8dbcbb16 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.5.1.xml @@ -0,0 +1,12 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.5.1"> + +<title>Release 1.5.1 (2013-02-28)</title> + +<para>The bug fix to the bug fix had a bug itself, of course. But +this time it will work for sure!</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.5.2.xml b/third_party/nix/doc/manual/release-notes/rl-1.5.2.xml new file mode 100644 index 000000000000..7e81dd243284 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.5.2.xml @@ -0,0 +1,12 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.5.2"> + +<title>Release 1.5.2 (2013-05-13)</title> + +<para>This is primarily a bug fix release. It has contributions from +Eelco Dolstra, Lluís Batlle i Rossell and Shea Levy.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.5.xml b/third_party/nix/doc/manual/release-notes/rl-1.5.xml new file mode 100644 index 000000000000..8e279d7693e0 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.5.xml @@ -0,0 +1,12 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.5"> + +<title>Release 1.5 (2013-02-27)</title> + +<para>This is a brown paper bag release to fix a regression introduced +by the hard link security fix in 1.4.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.6.1.xml b/third_party/nix/doc/manual/release-notes/rl-1.6.1.xml new file mode 100644 index 000000000000..9ecc52734737 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.6.1.xml @@ -0,0 +1,69 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.6.1"> + +<title>Release 1.6.1 (2013-10-28)</title> + +<para>This is primarily a bug fix release. Changes of interest +are:</para> + +<itemizedlist> + + <listitem> + <para>Nix 1.6 accidentally changed the semantics of antiquoted + paths in strings, such as <literal>"${/foo}/bar"</literal>. This + release reverts to the Nix 1.5.3 behaviour.</para> + </listitem> + + <listitem> + <para>Previously, Nix optimised expressions such as + <literal>"${<replaceable>expr</replaceable>}"</literal> to + <replaceable>expr</replaceable>. Thus it neither checked whether + <replaceable>expr</replaceable> could be coerced to a string, nor + applied such coercions. This meant that + <literal>"${123}"</literal> evaluatued to <literal>123</literal>, + and <literal>"${./foo}"</literal> evaluated to + <literal>./foo</literal> (even though + <literal>"${./foo} "</literal> evaluates to + <literal>"/nix/store/<replaceable>hash</replaceable>-foo "</literal>). + Nix now checks the type of antiquoted expressions and + applies coercions.</para> + </listitem> + + <listitem> + <para>Nix now shows the exact position of undefined variables. In + particular, undefined variable errors in a <literal>with</literal> + previously didn't show <emphasis>any</emphasis> position + information, so this makes it a lot easier to fix such + errors.</para> + </listitem> + + <listitem> + <para>Undefined variables are now treated consistently. + Previously, the <function>tryEval</function> function would catch + undefined variables inside a <literal>with</literal> but not + outside. Now <function>tryEval</function> never catches undefined + variables.</para> + </listitem> + + <listitem> + <para>Bash completion in <command>nix-shell</command> now works + correctly.</para> + </listitem> + + <listitem> + <para>Stack traces are less verbose: they no longer show calls to + builtin functions and only show a single line for each derivation + on the call stack.</para> + </listitem> + + <listitem> + <para>New built-in function: <function>builtins.typeOf</function>, + which returns the type of its argument as a string.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.6.xml b/third_party/nix/doc/manual/release-notes/rl-1.6.xml new file mode 100644 index 000000000000..580563420949 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.6.xml @@ -0,0 +1,127 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.6.0"> + +<title>Release 1.6 (2013-09-10)</title> + +<para>In addition to the usual bug fixes, this release has several new +features:</para> + +<itemizedlist> + + <listitem> + <para>The command <command>nix-build --run-env</command> has been + renamed to <command>nix-shell</command>.</para> + </listitem> + + <listitem> + <para><command>nix-shell</command> now sources + <filename>$stdenv/setup</filename> <emphasis>inside</emphasis> the + interactive shell, rather than in a parent shell. This ensures + that shell functions defined by <literal>stdenv</literal> can be + used in the interactive shell.</para> + </listitem> + + <listitem> + <para><command>nix-shell</command> has a new flag + <option>--pure</option> to clear the environment, so you get an + environment that more closely corresponds to the “real” Nix build. + </para> + </listitem> + + <listitem> + <para><command>nix-shell</command> now sets the shell prompt + (<envar>PS1</envar>) to ensure that Nix shells are distinguishable + from your regular shells.</para> + </listitem> + + <listitem> + <para><command>nix-env</command> no longer requires a + <literal>*</literal> argument to match all packages, so + <literal>nix-env -qa</literal> is equivalent to <literal>nix-env + -qa '*'</literal>.</para> + </listitem> + + <listitem> + <para><command>nix-env -i</command> has a new flag + <option>--remove-all</option> (<option>-r</option>) to remove all + previous packages from the profile. This makes it easier to do + declarative package management similar to NixOS’s + <option>environment.systemPackages</option>. For instance, if you + have a specification <filename>my-packages.nix</filename> like this: + +<programlisting> +with import <nixpkgs> {}; +[ thunderbird + geeqie + ... +] +</programlisting> + + then after any change to this file, you can run: + +<screen> +$ nix-env -f my-packages.nix -ir +</screen> + + to update your profile to match the specification.</para> + </listitem> + + <listitem> + <para>The ‘<literal>with</literal>’ language construct is now more + lazy. It only evaluates its argument if a variable might actually + refer to an attribute in the argument. For instance, this now + works: + +<programlisting> +let + pkgs = with pkgs; { foo = "old"; bar = foo; } // overrides; + overrides = { foo = "new"; }; +in pkgs.bar +</programlisting> + + This evaluates to <literal>"new"</literal>, while previously it + gave an “infinite recursion” error.</para> + </listitem> + + <listitem> + <para>Nix now has proper integer arithmetic operators. For + instance, you can write <literal>x + y</literal> instead of + <literal>builtins.add x y</literal>, or <literal>x < + y</literal> instead of <literal>builtins.lessThan x y</literal>. + The comparison operators also work on strings.</para> + </listitem> + + <listitem> + <para>On 64-bit systems, Nix integers are now 64 bits rather than + 32 bits.</para> + </listitem> + + <listitem> + <para>When using the Nix daemon, the <command>nix-daemon</command> + worker process now runs on the same CPU as the client, on systems + that support setting CPU affinity. This gives a significant speedup + on some systems.</para> + </listitem> + + <listitem> + <para>If a stack overflow occurs in the Nix evaluator, you now get + a proper error message (rather than “Segmentation fault”) on some + systems.</para> + </listitem> + + <listitem> + <para>In addition to directories, you can now bind-mount regular + files in chroots through the (now misnamed) option + <option>build-chroot-dirs</option>.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Domen Kožar, Eelco Dolstra, +Florian Friesdorf, Gergely Risko, Ivan Kozik, Ludovic Courtès and Shea +Levy.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.7.xml b/third_party/nix/doc/manual/release-notes/rl-1.7.xml new file mode 100644 index 000000000000..44ecaa78da5f --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.7.xml @@ -0,0 +1,263 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.7"> + +<title>Release 1.7 (2014-04-11)</title> + +<para>In addition to the usual bug fixes, this release has the +following new features:</para> + +<itemizedlist> + + <listitem> + <para>Antiquotation is now allowed inside of quoted attribute + names (e.g. <literal>set."${foo}"</literal>). In the case where + the attribute name is just a single antiquotation, the quotes can + be dropped (e.g. the above example can be written + <literal>set.${foo}</literal>). If an attribute name inside of a + set declaration evaluates to <literal>null</literal> (e.g. + <literal>{ ${null} = false; }</literal>), then that attribute is + not added to the set.</para> + </listitem> + + <listitem> + <para>Experimental support for cryptographically signed binary + caches. See <link + xlink:href="https://github.com/NixOS/nix/commit/0fdf4da0e979f992db75cc17376e455ddc5a96d8">the + commit for details</link>.</para> + </listitem> + + <listitem> + <para>An experimental new substituter, + <command>download-via-ssh</command>, that fetches binaries from + remote machines via SSH. Specifying the flags <literal>--option + use-ssh-substituter true --option ssh-substituter-hosts + <replaceable>user@hostname</replaceable></literal> will cause Nix + to download binaries from the specified machine, if it has + them.</para> + </listitem> + + <listitem> + <para><command>nix-store -r</command> and + <command>nix-build</command> have a new flag, + <option>--check</option>, that builds a previously built + derivation again, and prints an error message if the output is not + exactly the same. This helps to verify whether a derivation is + truly deterministic. For example: + +<screen> +$ nix-build '<nixpkgs>' -A patchelf +<replaceable>…</replaceable> +$ nix-build '<nixpkgs>' -A patchelf --check +<replaceable>…</replaceable> +error: derivation `/nix/store/1ipvxs…-patchelf-0.6' may not be deterministic: + hash mismatch in output `/nix/store/4pc1dm…-patchelf-0.6.drv' +</screen> + + </para> + + </listitem> + + <listitem> + <para>The <command>nix-instantiate</command> flags + <option>--eval-only</option> and <option>--parse-only</option> + have been renamed to <option>--eval</option> and + <option>--parse</option>, respectively.</para> + </listitem> + + <listitem> + <para><command>nix-instantiate</command>, + <command>nix-build</command> and <command>nix-shell</command> now + have a flag <option>--expr</option> (or <option>-E</option>) that + allows you to specify the expression to be evaluated as a command + line argument. For instance, <literal>nix-instantiate --eval -E + '1 + 2'</literal> will print <literal>3</literal>.</para> + </listitem> + + <listitem> + <para><command>nix-shell</command> improvements:</para> + + <itemizedlist> + + <listitem> + <para>It has a new flag, <option>--packages</option> (or + <option>-p</option>), that sets up a build environment + containing the specified packages from Nixpkgs. For example, + the command + +<screen> +$ nix-shell -p sqlite xorg.libX11 hello +</screen> + + will start a shell in which the given packages are + present.</para> + </listitem> + + <listitem> + <para>It now uses <filename>shell.nix</filename> as the + default expression, falling back to + <filename>default.nix</filename> if the former doesn’t + exist. This makes it convenient to have a + <filename>shell.nix</filename> in your project to set up a + nice development environment.</para> + </listitem> + + <listitem> + <para>It evaluates the derivation attribute + <varname>shellHook</varname>, if set. Since + <literal>stdenv</literal> does not normally execute this hook, + it allows you to do <command>nix-shell</command>-specific + setup.</para> + </listitem> + + <listitem> + <para>It preserves the user’s timezone setting.</para> + </listitem> + + </itemizedlist> + + </listitem> + + <listitem> + <para>In chroots, Nix now sets up a <filename>/dev</filename> + containing only a minimal set of devices (such as + <filename>/dev/null</filename>). Note that it only does this if + you <emphasis>don’t</emphasis> have <filename>/dev</filename> + listed in your <option>build-chroot-dirs</option> setting; + otherwise, it will bind-mount the <literal>/dev</literal> from + outside the chroot.</para> + + <para>Similarly, if you don’t have <filename>/dev/pts</filename> listed + in <option>build-chroot-dirs</option>, Nix will mount a private + <literal>devpts</literal> filesystem on the chroot’s + <filename>/dev/pts</filename>.</para> + + </listitem> + + <listitem> + <para>New built-in function: <function>builtins.toJSON</function>, + which returns a JSON representation of a value.</para> + </listitem> + + <listitem> + <para><command>nix-env -q</command> has a new flag + <option>--json</option> to print a JSON representation of the + installed or available packages.</para> + </listitem> + + <listitem> + <para><command>nix-env</command> now supports meta attributes with + more complex values, such as attribute sets.</para> + </listitem> + + <listitem> + <para>The <option>-A</option> flag now allows attribute names with + dots in them, e.g. + +<screen> +$ nix-instantiate --eval '<nixos>' -A 'config.systemd.units."nscd.service".text' +</screen> + + </para> + </listitem> + + <listitem> + <para>The <option>--max-freed</option> option to + <command>nix-store --gc</command> now accepts a unit + specifier. For example, <literal>nix-store --gc --max-freed + 1G</literal> will free up to 1 gigabyte of disk space.</para> + </listitem> + + <listitem> + <para><command>nix-collect-garbage</command> has a new flag + <option>--delete-older-than</option> + <replaceable>N</replaceable><literal>d</literal>, which deletes + all user environment generations older than + <replaceable>N</replaceable> days. Likewise, <command>nix-env + --delete-generations</command> accepts a + <replaceable>N</replaceable><literal>d</literal> age limit.</para> + </listitem> + + <listitem> + <para>Nix now heuristically detects whether a build failure was + due to a disk-full condition. In that case, the build is not + flagged as “permanently failed”. This is mostly useful for Hydra, + which needs to distinguish between permanent and transient build + failures.</para> + </listitem> + + <listitem> + <para>There is a new symbol <literal>__curPos</literal> that + expands to an attribute set containing its file name and line and + column numbers, e.g. <literal>{ file = "foo.nix"; line = 10; + column = 5; }</literal>. There also is a new builtin function, + <varname>unsafeGetAttrPos</varname>, that returns the position of + an attribute. This is used by Nixpkgs to provide location + information in error messages, e.g. + +<screen> +$ nix-build '<nixpkgs>' -A libreoffice --argstr system x86_64-darwin +error: the package ‘libreoffice-4.0.5.2’ in ‘.../applications/office/libreoffice/default.nix:263’ + is not supported on ‘x86_64-darwin’ +</screen> + + </para> + </listitem> + + <listitem> + <para>The garbage collector is now more concurrent with other Nix + processes because it releases certain locks earlier.</para> + </listitem> + + <listitem> + <para>The binary tarball installer has been improved. You can now + install Nix by running: + +<screen> +$ bash <(curl https://nixos.org/nix/install) +</screen> + + </para> + </listitem> + + <listitem> + <para>More evaluation errors include position information. For + instance, selecting a missing attribute will print something like + +<screen> +error: attribute `nixUnstabl' missing, at /etc/nixos/configurations/misc/eelco/mandark.nix:216:15 +</screen> + + </para> + </listitem> + + <listitem> + <para>The command <command>nix-setuid-helper</command> is + gone.</para> + </listitem> + + <listitem> + <para>Nix no longer uses Automake, but instead has a + non-recursive, GNU Make-based build system.</para> + </listitem> + + <listitem> + <para>All installed libraries now have the prefix + <literal>libnix</literal>. In particular, this gets rid of + <literal>libutil</literal>, which could clash with libraries with + the same name from other packages.</para> + </listitem> + + <listitem> + <para>Nix now requires a compiler that supports C++11.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from Danny Wilson, Domen Kožar, +Eelco Dolstra, Ian-Woo Kim, Ludovic Courtès, Maxim Ivanov, Petr +Rockai, Ricardo M. Correia and Shea Levy.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.8.xml b/third_party/nix/doc/manual/release-notes/rl-1.8.xml new file mode 100644 index 000000000000..c854c5c5f850 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.8.xml @@ -0,0 +1,123 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.8"> + +<title>Release 1.8 (2014-12-14)</title> + +<itemizedlist> + + <listitem><para>Breaking change: to address a race condition, the + remote build hook mechanism now uses <command>nix-store + --serve</command> on the remote machine. This requires build slaves + to be updated to Nix 1.8.</para></listitem> + + <listitem><para>Nix now uses HTTPS instead of HTTP to access the + default binary cache, + <literal>cache.nixos.org</literal>.</para></listitem> + + <listitem><para><command>nix-env</command> selectors are now regular + expressions. For instance, you can do + +<screen> +$ nix-env -qa '.*zip.*' +</screen> + + to query all packages with a name containing + <literal>zip</literal>.</para></listitem> + + <listitem><para><command>nix-store --read-log</command> can now + fetch remote build logs. If a build log is not available locally, + then ‘nix-store -l’ will now try to download it from the servers + listed in the ‘log-servers’ option in nix.conf. For instance, if you + have the configuration option + +<programlisting> +log-servers = http://hydra.nixos.org/log +</programlisting> + +then it will try to get logs from +<literal>http://hydra.nixos.org/log/<replaceable>base name of the +store path</replaceable></literal>. This allows you to do things like: + +<screen> +$ nix-store -l $(which xterm) +</screen> + + and get a log even if <command>xterm</command> wasn't built + locally.</para></listitem> + + <listitem><para>New builtin functions: + <function>attrValues</function>, <function>deepSeq</function>, + <function>fromJSON</function>, <function>readDir</function>, + <function>seq</function>.</para></listitem> + + <listitem><para><command>nix-instantiate --eval</command> now has a + <option>--json</option> flag to print the resulting value in JSON + format.</para></listitem> + + <listitem><para><command>nix-copy-closure</command> now uses + <command>nix-store --serve</command> on the remote side to send or + receive closures. This fixes a race condition between + <command>nix-copy-closure</command> and the garbage + collector.</para></listitem> + + <listitem><para>Derivations can specify the new special attribute + <varname>allowedRequisites</varname>, which has a similar meaning to + <varname>allowedReferences</varname>. But instead of only enforcing + to explicitly specify the immediate references, it requires the + derivation to specify all the dependencies recursively (hence the + name, requisites) that are used by the resulting + output.</para></listitem> + + <listitem><para>On Mac OS X, Nix now handles case collisions when + importing closures from case-sensitive file systems. This is mostly + useful for running NixOps on Mac OS X.</para></listitem> + + <listitem><para>The Nix daemon has new configuration options + <option>allowed-users</option> (specifying the users and groups that + are allowed to connect to the daemon) and + <option>trusted-users</option> (specifying the users and groups that + can perform privileged operations like specifying untrusted binary + caches).</para></listitem> + + <listitem><para>The configuration option + <option>build-cores</option> now defaults to the number of available + CPU cores.</para></listitem> + + <listitem><para>Build users are now used by default when Nix is + invoked as root. This prevents builds from accidentally running as + root.</para></listitem> + + <listitem><para>Nix now includes systemd units and Upstart + jobs.</para></listitem> + + <listitem><para>Speed improvements to <command>nix-store + --optimise</command>.</para></listitem> + + <listitem><para>Language change: the <literal>==</literal> operator + now ignores string contexts (the “dependencies” of a + string).</para></listitem> + + <listitem><para>Nix now filters out Nix-specific ANSI escape + sequences on standard error. They are supposed to be invisible, but + some terminals show them anyway.</para></listitem> + + <listitem><para>Various commands now automatically pipe their output + into the pager as specified by the <envar>PAGER</envar> environment + variable.</para></listitem> + + <listitem><para>Several improvements to reduce memory consumption in + the evaluator.</para></listitem> + +</itemizedlist> + +<para>This release has contributions from Adam Szkoda, Aristid +Breitkreuz, Bob van der Linden, Charles Strahan, darealshinji, Eelco +Dolstra, Gergely Risko, Joel Taylor, Ludovic Courtès, Marko Durkovic, +Mikey Ariel, Paul Colomiets, Ricardo M. Correia, Ricky Elrod, Robert +Helgesson, Rob Vermaas, Russell O'Connor, Shea Levy, Shell Turner, +Sönke Hahn, Steve Purcell, Vladimír Čunát and Wout Mertens.</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-1.9.xml b/third_party/nix/doc/manual/release-notes/rl-1.9.xml new file mode 100644 index 000000000000..c8406bd2077c --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.9.xml @@ -0,0 +1,216 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-1.9"> + +<title>Release 1.9 (2015-06-12)</title> + +<para>In addition to the usual bug fixes, this release has the +following new features:</para> + +<itemizedlist> + + <listitem> + <para>Signed binary cache support. You can enable signature + checking by adding the following to <filename>nix.conf</filename>: + +<programlisting> +signed-binary-caches = * +binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= +</programlisting> + + This will prevent Nix from downloading any binary from the cache + that is not signed by one of the keys listed in + <option>binary-cache-public-keys</option>.</para> + + <para>Signature checking is only supported if you built Nix with + the <literal>libsodium</literal> package.</para> + + <para>Note that while Nix has had experimental support for signed + binary caches since version 1.7, this release changes the + signature format in a backwards-incompatible way.</para> + + </listitem> + + <listitem> + + <para>Automatic downloading of Nix expression tarballs. In various + places, you can now specify the URL of a tarball containing Nix + expressions (such as Nixpkgs), which will be downloaded and + unpacked automatically. For example:</para> + + <itemizedlist> + + <listitem><para>In <command>nix-env</command>: + +<screen> +$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox +</screen> + + This installs Firefox from the latest tested and built revision + of the NixOS 14.12 channel.</para></listitem> + + <listitem><para>In <command>nix-build</command> and + <command>nix-shell</command>: + +<screen> +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello +</screen> + + This builds GNU Hello from the latest revision of the Nixpkgs + master branch.</para></listitem> + + <listitem><para>In the Nix search path (as specified via + <envar>NIX_PATH</envar> or <option>-I</option>). For example, to + start a shell containing the Pan package from a specific version + of Nixpkgs: + +<screen> +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz +</screen> + + </para></listitem> + + <listitem><para>In <command>nixos-rebuild</command> (on NixOS): + +<screen> +$ nixos-rebuild test -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz +</screen> + + </para></listitem> + + <listitem><para>In Nix expressions, via the new builtin function <function>fetchTarball</function>: + +<programlisting> +with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; … +</programlisting> + + (This is not allowed in restricted mode.)</para></listitem> + + </itemizedlist> + + </listitem> + + <listitem> + + <para><command>nix-shell</command> improvements:</para> + + <itemizedlist> + + <listitem><para><command>nix-shell</command> now has a flag + <option>--run</option> to execute a command in the + <command>nix-shell</command> environment, + e.g. <literal>nix-shell --run make</literal>. This is like + the existing <option>--command</option> flag, except that it + uses a non-interactive shell (ensuring that hitting Ctrl-C won’t + drop you into the child shell).</para></listitem> + + <listitem><para><command>nix-shell</command> can now be used as + a <literal>#!</literal>-interpreter. This allows you to write + scripts that dynamically fetch their own dependencies. For + example, here is a Haskell script that, when invoked, first + downloads GHC and the Haskell packages on which it depends: + +<programlisting> +#! /usr/bin/env nix-shell +#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP + +import Network.HTTP + +main = do + resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/") + body <- getResponseBody resp + print (take 100 body) +</programlisting> + + Of course, the dependencies are cached in the Nix store, so the + second invocation of this script will be much + faster.</para></listitem> + + </itemizedlist> + + </listitem> + + <listitem> + + <para>Chroot improvements:</para> + + <itemizedlist> + + <listitem><para>Chroot builds are now supported on Mac OS X + (using its sandbox mechanism).</para></listitem> + + <listitem><para>If chroots are enabled, they are now used for + all derivations, including fixed-output derivations (such as + <function>fetchurl</function>). The latter do have network + access, but can no longer access the host filesystem. If you + need the old behaviour, you can set the option + <option>build-use-chroot</option> to + <literal>relaxed</literal>.</para></listitem> + + <listitem><para>On Linux, if chroots are enabled, builds are + performed in a private PID namespace once again. (This + functionality was lost in Nix 1.8.)</para></listitem> + + <listitem><para>Store paths listed in + <option>build-chroot-dirs</option> are now automatically + expanded to their closure. For instance, if you want + <filename>/nix/store/…-bash/bin/sh</filename> mounted in your + chroot as <filename>/bin/sh</filename>, you only need to say + <literal>build-chroot-dirs = + /bin/sh=/nix/store/…-bash/bin/sh</literal>; it is no longer + necessary to specify the dependencies of Bash.</para></listitem> + + </itemizedlist> + + </listitem> + + <listitem><para>The new derivation attribute + <varname>passAsFile</varname> allows you to specify that the + contents of derivation attributes should be passed via files rather + than environment variables. This is useful if you need to pass very + long strings that exceed the size limit of the environment. The + Nixpkgs function <function>writeTextFile</function> uses + this.</para></listitem> + + <listitem><para>You can now use <literal>~</literal> in Nix file + names to refer to your home directory, e.g. <literal>import + ~/.nixpkgs/config.nix</literal>.</para></listitem> + + <listitem><para>Nix has a new option <option>restrict-eval</option> + that allows limiting what paths the Nix evaluator has access to. By + passing <literal>--option restrict-eval true</literal> to Nix, the + evaluator will throw an exception if an attempt is made to access + any file outside of the Nix search path. This is primarily intended + for Hydra to ensure that a Hydra jobset only refers to its declared + inputs (and is therefore reproducible).</para></listitem> + + <listitem><para><command>nix-env</command> now only creates a new + “generation” symlink in <filename>/nix/var/nix/profiles</filename> + if something actually changed.</para></listitem> + + <listitem><para>The environment variable <envar>NIX_PAGER</envar> + can now be set to override <envar>PAGER</envar>. You can set it to + <literal>cat</literal> to disable paging for Nix commands + only.</para></listitem> + + <listitem><para>Failing <literal><...></literal> + lookups now show position information.</para></listitem> + + <listitem><para>Improved Boehm GC use: we disabled scanning for + interior pointers, which should reduce the “<literal>Repeated + allocation of very large block</literal>” warnings and associated + retention of memory.</para></listitem> + +</itemizedlist> + +<para>This release has contributions from aszlig, Benjamin Staffin, +Charles Strahan, Christian Theune, Daniel Hahler, Danylo Hlynskyi +Daniel Peebles, Dan Peebles, Domen Kožar, Eelco Dolstra, Harald van +Dijk, Hoang Xuan Phu, Jaka Hudoklin, Jeff Ramnani, j-keck, Linquize, +Luca Bruno, Michael Merickel, Oliver Dunkl, Rob Vermaas, Rok Garbas, +Shea Levy, Tobias Geerinckx-Rice and William A. Kennington III.</para> + +</section> + diff --git a/third_party/nix/doc/manual/release-notes/rl-2.0.xml b/third_party/nix/doc/manual/release-notes/rl-2.0.xml new file mode 100644 index 000000000000..fc9a77b08b60 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.0.xml @@ -0,0 +1,1012 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.0"> + +<title>Release 2.0 (2018-02-22)</title> + +<para>The following incompatible changes have been made:</para> + +<itemizedlist> + + <listitem> + <para>The manifest-based substituter mechanism + (<command>download-using-manifests</command>) has been <link + xlink:href="https://github.com/NixOS/nix/commit/867967265b80946dfe1db72d40324b4f9af988ed">removed</link>. It + has been superseded by the binary cache substituter mechanism + since several years. As a result, the following programs have been + removed: + + <itemizedlist> + <listitem><para><command>nix-pull</command></para></listitem> + <listitem><para><command>nix-generate-patches</command></para></listitem> + <listitem><para><command>bsdiff</command></para></listitem> + <listitem><para><command>bspatch</command></para></listitem> + </itemizedlist> + </para> + </listitem> + + <listitem> + <para>The “copy from other stores” substituter mechanism + (<command>copy-from-other-stores</command> and the + <envar>NIX_OTHER_STORES</envar> environment variable) has been + removed. It was primarily used by the NixOS installer to copy + available paths from the installation medium. The replacement is + to use a chroot store as a substituter + (e.g. <literal>--substituters /mnt</literal>), or to build into a + chroot store (e.g. <literal>--store /mnt --substituters /</literal>).</para> + </listitem> + + <listitem> + <para>The command <command>nix-push</command> has been removed as + part of the effort to eliminate Nix's dependency on Perl. You can + use <command>nix copy</command> instead, e.g. <literal>nix copy + --to file:///tmp/my-binary-cache <replaceable>paths…</replaceable></literal></para> + </listitem> + + <listitem> + <para>The “nested” log output feature (<option>--log-type + pretty</option>) has been removed. As a result, + <command>nix-log2xml</command> was also removed.</para> + </listitem> + + <listitem> + <para>OpenSSL-based signing has been <link + xlink:href="https://github.com/NixOS/nix/commit/f435f8247553656774dd1b2c88e9de5d59cab203">removed</link>. This + feature was never well-supported. A better alternative is provided + by the <option>secret-key-files</option> and + <option>trusted-public-keys</option> options.</para> + </listitem> + + <listitem> + <para>Failed build caching has been <link + xlink:href="https://github.com/NixOS/nix/commit/8cffec84859cec8b610a2a22ab0c4d462a9351ff">removed</link>. This + feature was introduced to support the Hydra continuous build + system, but Hydra no longer uses it.</para> + </listitem> + + <listitem> + <para><filename>nix-mode.el</filename> has been removed from + Nix. It is now <link + xlink:href="https://github.com/NixOS/nix-mode">a separate + repository</link> and can be installed through the MELPA package + repository.</para> + </listitem> + +</itemizedlist> + +<para>This release has the following new features:</para> + +<itemizedlist> + + <listitem> + <para>It introduces a new command named <command>nix</command>, + which is intended to eventually replace all + <command>nix-*</command> commands with a more consistent and + better designed user interface. It currently provides replacements + for some (but not all) of the functionality provided by + <command>nix-store</command>, <command>nix-build</command>, + <command>nix-shell -p</command>, <command>nix-env -qa</command>, + <command>nix-instantiate --eval</command>, + <command>nix-push</command> and + <command>nix-copy-closure</command>. It has the following major + features:</para> + + <itemizedlist> + + <listitem> + <para>Unlike the legacy commands, it has a consistent way to + refer to packages and package-like arguments (like store + paths). For example, the following commands all copy the GNU + Hello package to a remote machine: + + <screen>nix copy --to ssh://machine nixpkgs.hello</screen> + <screen>nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen> + <screen>nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)'</screen> + + By contrast, <command>nix-copy-closure</command> only accepted + store paths as arguments.</para> + </listitem> + + <listitem> + <para>It is self-documenting: <option>--help</option> shows + all available command-line arguments. If + <option>--help</option> is given after a subcommand, it shows + examples for that subcommand. <command>nix + --help-config</command> shows all configuration + options.</para> + </listitem> + + <listitem> + <para>It is much less verbose. By default, it displays a + single-line progress indicator that shows how many packages + are left to be built or downloaded, and (if there are running + builds) the most recent line of builder output. If a build + fails, it shows the last few lines of builder output. The full + build log can be retrieved using <command>nix + log</command>.</para> + </listitem> + + <listitem> + <para>It <link + xlink:href="https://github.com/NixOS/nix/commit/b8283773bd64d7da6859ed520ee19867742a03ba">provides</link> + all <filename>nix.conf</filename> configuration options as + command line flags. For example, instead of <literal>--option + http-connections 100</literal> you can write + <literal>--http-connections 100</literal>. Boolean options can + be written as + <literal>--<replaceable>foo</replaceable></literal> or + <literal>--no-<replaceable>foo</replaceable></literal> + (e.g. <option>--no-auto-optimise-store</option>).</para> + </listitem> + + <listitem> + <para>Many subcommands have a <option>--json</option> flag to + write results to stdout in JSON format.</para> + </listitem> + + </itemizedlist> + + <warning><para>Please note that the <command>nix</command> command + is a work in progress and the interface is subject to + change.</para></warning> + + <para>It provides the following high-level (“porcelain”) + subcommands:</para> + + <itemizedlist> + + <listitem> + <para><command>nix build</command> is a replacement for + <command>nix-build</command>.</para> + </listitem> + + <listitem> + <para><command>nix run</command> executes a command in an + environment in which the specified packages are available. It + is (roughly) a replacement for <command>nix-shell + -p</command>. Unlike that command, it does not execute the + command in a shell, and has a flag (<command>-c</command>) + that specifies the unquoted command line to be + executed.</para> + + <para>It is particularly useful in conjunction with chroot + stores, allowing Linux users who do not have permission to + install Nix in <command>/nix/store</command> to still use + binary substitutes that assume + <command>/nix/store</command>. For example, + + <screen>nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!'</screen> + + downloads (or if not substitutes are available, builds) the + GNU Hello package into + <filename>~/my-nix/nix/store</filename>, then runs + <command>hello</command> in a mount namespace where + <filename>~/my-nix/nix/store</filename> is mounted onto + <command>/nix/store</command>.</para> + </listitem> + + <listitem> + <para><command>nix search</command> replaces <command>nix-env + -qa</command>. It searches the available packages for + occurrences of a search string in the attribute name, package + name or description. Unlike <command>nix-env -qa</command>, it + has a cache to speed up subsequent searches.</para> + </listitem> + + <listitem> + <para><command>nix copy</command> copies paths between + arbitrary Nix stores, generalising + <command>nix-copy-closure</command> and + <command>nix-push</command>.</para> + </listitem> + + <listitem> + <para><command>nix repl</command> replaces the external + program <command>nix-repl</command>. It provides an + interactive environment for evaluating and building Nix + expressions. Note that it uses <literal>linenoise-ng</literal> + instead of GNU Readline.</para> + </listitem> + + <listitem> + <para><command>nix upgrade-nix</command> upgrades Nix to the + latest stable version. This requires that Nix is installed in + a profile. (Thus it won’t work on NixOS, or if it’s installed + outside of the Nix store.)</para> + </listitem> + + <listitem> + <para><command>nix verify</command> checks whether store paths + are unmodified and/or “trusted” (see below). It replaces + <command>nix-store --verify</command> and <command>nix-store + --verify-path</command>.</para> + </listitem> + + <listitem> + <para><command>nix log</command> shows the build log of a + package or path. If the build log is not available locally, it + will try to obtain it from the configured substituters (such + as <uri>cache.nixos.org</uri>, which now provides build + logs).</para> + </listitem> + + <listitem> + <para><command>nix edit</command> opens the source code of a + package in your editor.</para> + </listitem> + + <listitem> + <para><command>nix eval</command> replaces + <command>nix-instantiate --eval</command>.</para> + </listitem> + + <listitem> + <para><command + xlink:href="https://github.com/NixOS/nix/commit/d41c5eb13f4f3a37d80dbc6d3888644170c3b44a">nix + why-depends</command> shows why one store path has another in + its closure. This is primarily useful to finding the causes of + closure bloat. For example, + + <screen>nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev</screen> + + shows a chain of files and fragments of file contents that + cause the VLC package to have the “dev” output of + <literal>libdrm</literal> in its closure — an undesirable + situation.</para> + </listitem> + + <listitem> + <para><command>nix path-info</command> shows information about + store paths, replacing <command>nix-store -q</command>. A + useful feature is the option <option>--closure-size</option> + (<option>-S</option>). For example, the following command show + the closure sizes of every path in the current NixOS system + closure, sorted by size: + + <screen>nix path-info -rS /run/current-system | sort -nk2</screen> + + </para> + </listitem> + + <listitem> + <para><command>nix optimise-store</command> replaces + <command>nix-store --optimise</command>. The main difference + is that it has a progress indicator.</para> + </listitem> + + </itemizedlist> + + <para>A number of low-level (“plumbing”) commands are also + available:</para> + + <itemizedlist> + + <listitem> + <para><command>nix ls-store</command> and <command>nix + ls-nar</command> list the contents of a store path or NAR + file. The former is primarily useful in conjunction with + remote stores, e.g. + + <screen>nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10</screen> + + lists the contents of path in a binary cache.</para> + </listitem> + + <listitem> + <para><command>nix cat-store</command> and <command>nix + cat-nar</command> allow extracting a file from a store path or + NAR file.</para> + </listitem> + + <listitem> + <para><command>nix dump-path</command> writes the contents of + a store path to stdout in NAR format. This replaces + <command>nix-store --dump</command>.</para> + </listitem> + + <listitem> + <para><command + xlink:href="https://github.com/NixOS/nix/commit/e8d6ee7c1b90a2fe6d824f1a875acc56799ae6e2">nix + show-derivation</command> displays a store derivation in JSON + format. This is an alternative to + <command>pp-aterm</command>.</para> + </listitem> + + <listitem> + <para><command + xlink:href="https://github.com/NixOS/nix/commit/970366266b8df712f5f9cedb45af183ef5a8357f">nix + add-to-store</command> replaces <command>nix-store + --add</command>.</para> + </listitem> + + <listitem> + <para><command>nix sign-paths</command> signs store + paths.</para> + </listitem> + + <listitem> + <para><command>nix copy-sigs</command> copies signatures from + one store to another.</para> + </listitem> + + <listitem> + <para><command>nix show-config</command> shows all + configuration options and their current values.</para> + </listitem> + + </itemizedlist> + + </listitem> + + <listitem> + <para>The store abstraction that Nix has had for a long time to + support store access via the Nix daemon has been extended + significantly. In particular, substituters (which used to be + external programs such as + <command>download-from-binary-cache</command>) are now subclasses + of the abstract <classname>Store</classname> class. This allows + many Nix commands to operate on such store types. For example, + <command>nix path-info</command> shows information about paths in + your local Nix store, while <command>nix path-info --store + https://cache.nixos.org/</command> shows information about paths + in the specified binary cache. Similarly, + <command>nix-copy-closure</command>, <command>nix-push</command> + and substitution are all instances of the general notion of + copying paths between different kinds of Nix stores.</para> + + <para>Stores are specified using an URI-like syntax, + e.g. <uri>https://cache.nixos.org/</uri> or + <uri>ssh://machine</uri>. The following store types are supported: + + <itemizedlist> + + <listitem> + + <para><classname>LocalStore</classname> (stori URI + <literal>local</literal> or an absolute path) and the misnamed + <classname>RemoteStore</classname> (<literal>daemon</literal>) + provide access to a local Nix store, the latter via the Nix + daemon. You can use <literal>auto</literal> or the empty + string to auto-select a local or daemon store depending on + whether you have write permission to the Nix store. It is no + longer necessary to set the <envar>NIX_REMOTE</envar> + environment variable to use the Nix daemon.</para> + + <para>As noted above, <classname>LocalStore</classname> now + supports chroot builds, allowing the “physical” location of + the Nix store + (e.g. <filename>/home/alice/nix/store</filename>) to differ + from its “logical” location (typically + <filename>/nix/store</filename>). This allows non-root users + to use Nix while still getting the benefits from prebuilt + binaries from <uri>cache.nixos.org</uri>.</para> + + </listitem> + + <listitem> + + <para><classname>BinaryCacheStore</classname> is the abstract + superclass of all binary cache stores. It supports writing + build logs and NAR content listings in JSON format.</para> + + </listitem> + + <listitem> + + <para><classname>HttpBinaryCacheStore</classname> + (<literal>http://</literal>, <literal>https://</literal>) + supports binary caches via HTTP or HTTPS. If the server + supports <literal>PUT</literal> requests, it supports + uploading store paths via commands such as <command>nix + copy</command>.</para> + + </listitem> + + <listitem> + + <para><classname>LocalBinaryCacheStore</classname> + (<literal>file://</literal>) supports binary caches in the + local filesystem.</para> + + </listitem> + + <listitem> + + <para><classname>S3BinaryCacheStore</classname> + (<literal>s3://</literal>) supports binary caches stored in + Amazon S3, if enabled at compile time.</para> + + </listitem> + + <listitem> + + <para><classname>LegacySSHStore</classname> (<literal>ssh://</literal>) + is used to implement remote builds and + <command>nix-copy-closure</command>.</para> + + </listitem> + + <listitem> + + <para><classname>SSHStore</classname> + (<literal>ssh-ng://</literal>) supports arbitrary Nix + operations on a remote machine via the same protocol used by + <command>nix-daemon</command>.</para> + + </listitem> + + </itemizedlist> + + </para> + + </listitem> + + <listitem> + + <para>Security has been improved in various ways: + + <itemizedlist> + + <listitem> + <para>Nix now stores signatures for local store + paths. When paths are copied between stores (e.g., copied from + a binary cache to a local store), signatures are + propagated.</para> + + <para>Locally-built paths are signed automatically using the + secret keys specified by the <option>secret-key-files</option> + store option. Secret/public key pairs can be generated using + <command>nix-store + --generate-binary-cache-key</command>.</para> + + <para>In addition, locally-built store paths are marked as + “ultimately trusted”, but this bit is not propagated when + paths are copied between stores.</para> + </listitem> + + <listitem> + <para>Content-addressable store paths no longer require + signatures — they can be imported into a store by unprivileged + users even if they lack signatures.</para> + </listitem> + + <listitem> + <para>The command <command>nix verify</command> checks whether + the specified paths are trusted, i.e., have a certain number + of trusted signatures, are ultimately trusted, or are + content-addressed.</para> + </listitem> + + <listitem> + <para>Substitutions from binary caches <link + xlink:href="https://github.com/NixOS/nix/commit/ecbc3fedd3d5bdc5a0e1a0a51b29062f2874ac8b">now</link> + require signatures by default. This was already the case on + NixOS.</para> + </listitem> + + <listitem> + <para>In Linux sandbox builds, we <link + xlink:href="https://github.com/NixOS/nix/commit/eba840c8a13b465ace90172ff76a0db2899ab11b">now</link> + use <filename>/build</filename> instead of + <filename>/tmp</filename> as the temporary build + directory. This fixes potential security problems when a build + accidentally stores its <envar>TMPDIR</envar> in some + security-sensitive place, such as an RPATH.</para> + </listitem> + + </itemizedlist> + + </para> + + </listitem> + + <listitem> + <para><emphasis>Pure evaluation mode</emphasis>. This is a variant + of the existing restricted evaluation mode. In pure mode, the Nix + evaluator forbids access to anything that could cause different + evaluations of the same command line arguments to produce a + different result. This includes builtin functions such as + <function>builtins.getEnv</function>, but more importantly, + <emphasis>all</emphasis> filesystem or network access unless a + content hash or commit hash is specified. For example, calls to + <function>builtins.fetchGit</function> are only allowed if a + <varname>rev</varname> attribute is specified.</para> + + <para>The goal of this feature is to enable true reproducibility + and traceability of builds (including NixOS system configurations) + at the evaluation level. For example, in the future, + <command>nixos-rebuild</command> might build configurations from a + Nix expression in a Git repository in pure mode. That expression + might fetch other repositories such as Nixpkgs via + <function>builtins.fetchGit</function>. The commit hash of the + top-level repository then uniquely identifies a running system, + and, in conjunction with that repository, allows it to be + reproduced or modified.</para> + + </listitem> + + <listitem> + <para>There are several new features to support binary + reproducibility (i.e. to help ensure that multiple builds of the + same derivation produce exactly the same output). When + <option>enforce-determinism</option> is set to + <literal>false</literal>, it’s <link + xlink:href="https://github.com/NixOS/nix/commit/8bdf83f936adae6f2c907a6d2541e80d4120f051">no + longer</link> a fatal error if build rounds produce different + output. Also, a hook named <option>diff-hook</option> is <link + xlink:href="https://github.com/NixOS/nix/commit/9a313469a4bdea2d1e8df24d16289dc2a172a169">provided</link> + to allow you to run tools such as <command>diffoscope</command> + when build rounds produce different output.</para> + </listitem> + + <listitem> + <para>Configuring remote builds is a lot easier now. Provided you + are not using the Nix daemon, you can now just specify a remote + build machine on the command line, e.g. <literal>--option builders + 'ssh://my-mac x86_64-darwin'</literal>. The environment variable + <envar>NIX_BUILD_HOOK</envar> has been removed and is no longer + needed. The environment variable <envar>NIX_REMOTE_SYSTEMS</envar> + is still supported for compatibility, but it is also possible to + specify builders in <command>nix.conf</command> by setting the + option <literal>builders = + @<replaceable>path</replaceable></literal>.</para> + </listitem> + + <listitem> + <para>If a fixed-output derivation produces a result with an + incorrect hash, the output path is moved to the location + corresponding to the actual hash and registered as valid. Thus, a + subsequent build of the fixed-output derivation with the correct + hash is unnecessary.</para> + </listitem> + + <listitem> + <para><command>nix-shell</command> <link + xlink:href="https://github.com/NixOS/nix/commit/ea59f39326c8e9dc42dfed4bcbf597fbce58797c">now</link> + sets the <varname>IN_NIX_SHELL</varname> environment variable + during evaluation and in the shell itself. This can be used to + perform different actions depending on whether you’re in a Nix + shell or in a regular build. Nixpkgs provides + <varname>lib.inNixShell</varname> to check this variable during + evaluation.</para> + </listitem> + + <listitem> + <para><envar>NIX_PATH</envar> is now lazy, so URIs in the path are + only downloaded if they are needed for evaluation.</para> + </listitem> + + <listitem> + <para>You can now use + <uri>channel:<replaceable>channel-name</replaceable></uri> as a + short-hand for + <uri>https://nixos.org/channels/<replaceable>channel-name</replaceable>/nixexprs.tar.xz</uri>. For + example, <literal>nix-build channel:nixos-15.09 -A hello</literal> + will build the GNU Hello package from the + <literal>nixos-15.09</literal> channel. In the future, this may + use Git to fetch updates more efficiently.</para> + </listitem> + + <listitem> + <para>When <option>--no-build-output</option> is given, the last + 10 lines of the build log will be shown if a build + fails.</para> + </listitem> + + <listitem> + <para>Networking has been improved: + + <itemizedlist> + + <listitem> + <para>HTTP/2 is now supported. This makes binary cache lookups + <link + xlink:href="https://github.com/NixOS/nix/commit/90ad02bf626b885a5dd8967894e2eafc953bdf92">much + more efficient</link>.</para> + </listitem> + + <listitem> + <para>We now retry downloads on many HTTP errors, making + binary caches substituters more resilient to temporary + failures.</para> + </listitem> + + <listitem> + <para>HTTP credentials can now be configured via the standard + <filename>netrc</filename> mechanism.</para> + </listitem> + + <listitem> + <para>If S3 support is enabled at compile time, + <uri>s3://</uri> URIs are <link + xlink:href="https://github.com/NixOS/nix/commit/9ff9c3f2f80ba4108e9c945bbfda2c64735f987b">supported</link> + in all places where Nix allows URIs.</para> + </listitem> + + <listitem> + <para>Brotli compression is now supported. In particular, + <uri>cache.nixos.org</uri> build logs are now compressed using + Brotli.</para> + </listitem> + + </itemizedlist> + + </para> + + </listitem> + + <listitem> + <para><command>nix-env</command> <link + xlink:href="https://github.com/NixOS/nix/commit/b0cb11722626e906a73f10dd9a0c9eea29faf43a">now</link> + ignores packages with bad derivation names (in particular those + starting with a digit or containing a dot).</para> + </listitem> + + <listitem> + <para>Many configuration options have been renamed, either because + they were unnecessarily verbose + (e.g. <option>build-use-sandbox</option> is now just + <option>sandbox</option>) or to reflect generalised behaviour + (e.g. <option>binary-caches</option> is now + <option>substituters</option> because it allows arbitrary store + URIs). The old names are still supported for compatibility.</para> + </listitem> + + <listitem> + <para>The <option>max-jobs</option> option can <link + xlink:href="https://github.com/NixOS/nix/commit/7251d048fa812d2551b7003bc9f13a8f5d4c95a5">now</link> + be set to <literal>auto</literal> to use the number of CPUs in the + system.</para> + </listitem> + + <listitem> + <para>Hashes can <link + xlink:href="https://github.com/NixOS/nix/commit/c0015e87af70f539f24d2aa2bc224a9d8b84276b">now</link> + be specified in base-64 format, in addition to base-16 and the + non-standard base-32.</para> + </listitem> + + <listitem> + <para><command>nix-shell</command> now uses + <varname>bashInteractive</varname> from Nixpkgs, rather than the + <command>bash</command> command that happens to be in the caller’s + <envar>PATH</envar>. This is especially important on macOS where + the <command>bash</command> provided by the system is seriously + outdated and cannot execute <literal>stdenv</literal>’s setup + script.</para> + </listitem> + + <listitem> + <para>Nix can now automatically trigger a garbage collection if + free disk space drops below a certain level during a build. This + is configured using the <option>min-free</option> and + <option>max-free</option> options.</para> + </listitem> + + <listitem> + <para><command>nix-store -q --roots</command> and + <command>nix-store --gc --print-roots</command> now show temporary + and in-memory roots.</para> + </listitem> + + <listitem> + <para> + Nix can now be extended with plugins. See the documentation of + the <option>plugin-files</option> option for more details. + </para> + </listitem> + +</itemizedlist> + +<para>The Nix language has the following new features: + +<itemizedlist> + + <listitem> + <para>It supports floating point numbers. They are based on the + C++ <literal>float</literal> type and are supported by the + existing numerical operators. Export and import to and from JSON + and XML works, too.</para> + </listitem> + + <listitem> + <para>Derivation attributes can now reference the outputs of the + derivation using the <function>placeholder</function> builtin + function. For example, the attribute + +<programlisting> +configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}"; +</programlisting> + + will cause the <envar>configureFlags</envar> environment variable + to contain the actual store paths corresponding to the + <literal>out</literal> and <literal>dev</literal> outputs.</para> + </listitem> + +</itemizedlist> + +</para> + +<para>The following builtin functions are new or extended: + +<itemizedlist> + + <listitem> + <para><function + xlink:href="https://github.com/NixOS/nix/commit/38539b943a060d9cdfc24d6e5d997c0885b8aa2f">builtins.fetchGit</function> + allows Git repositories to be fetched at evaluation time. Thus it + differs from the <function>fetchgit</function> function in + Nixpkgs, which fetches at build time and cannot be used to fetch + Nix expressions during evaluation. A typical use case is to import + external NixOS modules from your configuration, e.g. + + <programlisting>imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ];</programlisting> + + </para> + </listitem> + + <listitem> + <para>Similarly, <function>builtins.fetchMercurial</function> + allows you to fetch Mercurial repositories.</para> + </listitem> + + <listitem> + <para><function>builtins.path</function> generalises + <function>builtins.filterSource</function> and path literals + (e.g. <literal>./foo</literal>). It allows specifying a store path + name that differs from the source path name + (e.g. <literal>builtins.path { path = ./foo; name = "bar"; + }</literal>) and also supports filtering out unwanted + files.</para> + </listitem> + + <listitem> + <para><function>builtins.fetchurl</function> and + <function>builtins.fetchTarball</function> now support + <varname>sha256</varname> and <varname>name</varname> + attributes.</para> + </listitem> + + <listitem> + <para><function + xlink:href="https://github.com/NixOS/nix/commit/b8867a0239b1930a16f9ef3f7f3e864b01416dff">builtins.split</function> + splits a string using a POSIX extended regular expression as the + separator.</para> + </listitem> + + <listitem> + <para><function + xlink:href="https://github.com/NixOS/nix/commit/26d92017d3b36cff940dcb7d1611c42232edb81a">builtins.partition</function> + partitions the elements of a list into two lists, depending on a + Boolean predicate.</para> + </listitem> + + <listitem> + <para><literal><nix/fetchurl.nix></literal> now uses the + content-addressable tarball cache at + <uri>http://tarballs.nixos.org/</uri>, just like + <function>fetchurl</function> in + Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197)</para> + </listitem> + + <listitem> + <para>In restricted and pure evaluation mode, builtin functions + that download from the network (such as + <function>fetchGit</function>) are permitted to fetch underneath a + list of URI prefixes specified in the option + <option>allowed-uris</option>.</para> + </listitem> + +</itemizedlist> + +</para> + +<para>The Nix build environment has the following changes: + +<itemizedlist> + + <listitem> + <para>Values such as Booleans, integers, (nested) lists and + attribute sets can <link + xlink:href="https://github.com/NixOS/nix/commit/6de33a9c675b187437a2e1abbcb290981a89ecb1">now</link> + be passed to builders in a non-lossy way. If the special attribute + <varname>__structuredAttrs</varname> is set to + <literal>true</literal>, the other derivation attributes are + serialised in JSON format and made available to the builder via + the file <envar>.attrs.json</envar> in the builder’s temporary + directory. This obviates the need for + <varname>passAsFile</varname> since JSON files have no size + restrictions, unlike process environments.</para> + + <para><link + xlink:href="https://github.com/NixOS/nix/commit/2d5b1b24bf70a498e4c0b378704cfdb6471cc699">As + a convenience to Bash builders</link>, Nix writes a script named + <envar>.attrs.sh</envar> to the builder’s directory that + initialises shell variables corresponding to all attributes that + are representable in Bash. This includes non-nested (associative) + arrays. For example, the attribute <literal>hardening.format = + true</literal> ends up as the Bash associative array element + <literal>${hardening[format]}</literal>.</para> + </listitem> + + <listitem> + <para>Builders can <link + xlink:href="https://github.com/NixOS/nix/commit/88e6bb76de5564b3217be9688677d1c89101b2a3">now</link> + communicate what build phase they are in by writing messages to + the file descriptor specified in <envar>NIX_LOG_FD</envar>. The + current phase is shown by the <command>nix</command> progress + indicator. + </para> + </listitem> + + <listitem> + <para>In Linux sandbox builds, we <link + xlink:href="https://github.com/NixOS/nix/commit/a2d92bb20e82a0957067ede60e91fab256948b41">now</link> + provide a default <filename>/bin/sh</filename> (namely + <filename>ash</filename> from BusyBox).</para> + </listitem> + + <listitem> + <para>In structured attribute mode, + <varname>exportReferencesGraph</varname> <link + xlink:href="https://github.com/NixOS/nix/commit/c2b0d8749f7e77afc1c4b3e8dd36b7ee9720af4a">exports</link> + extended information about closures in JSON format. In particular, + it includes the sizes and hashes of paths. This is primarily + useful for NixOS image builders.</para> + </listitem> + + <listitem> + <para>Builds are <link + xlink:href="https://github.com/NixOS/nix/commit/21948deed99a3295e4d5666e027a6ca42dc00b40">now</link> + killed as soon as Nix receives EOF on the builder’s stdout or + stderr. This fixes a bug that allowed builds to hang Nix + indefinitely, regardless of + timeouts.</para> + </listitem> + + <listitem> + <para>The <option>sandbox-paths</option> configuration + option can now specify optional paths by appending a + <literal>?</literal>, e.g. <literal>/dev/nvidiactl?</literal> will + bind-mount <varname>/dev/nvidiactl</varname> only if it + exists.</para> + </listitem> + + <listitem> + <para>On Linux, builds are now executed in a user + namespace with UID 1000 and GID 100.</para> + </listitem> + +</itemizedlist> + +</para> + +<para>A number of significant internal changes were made: + +<itemizedlist> + + <listitem> + <para>Nix no longer depends on Perl and all Perl components have + been rewritten in C++ or removed. The Perl bindings that used to + be part of Nix have been moved to a separate package, + <literal>nix-perl</literal>.</para> + </listitem> + + <listitem> + <para>All <classname>Store</classname> classes are now + thread-safe. <classname>RemoteStore</classname> supports multiple + concurrent connections to the daemon. This is primarily useful in + multi-threaded programs such as + <command>hydra-queue-runner</command>.</para> + </listitem> + +</itemizedlist> + +</para> + +<para>This release has contributions from + +Adrien Devresse, +Alexander Ried, +Alex Cruice, +Alexey Shmalko, +AmineChikhaoui, +Andy Wingo, +Aneesh Agrawal, +Anthony Cowley, +Armijn Hemel, +aszlig, +Ben Gamari, +Benjamin Hipple, +Benjamin Staffin, +Benno Fünfstück, +Bjørn Forsman, +Brian McKenna, +Charles Strahan, +Chase Adams, +Chris Martin, +Christian Theune, +Chris Warburton, +Daiderd Jordan, +Dan Connolly, +Daniel Peebles, +Dan Peebles, +davidak, +David McFarland, +Dmitry Kalinkin, +Domen Kožar, +Eelco Dolstra, +Emery Hemingway, +Eric Litak, +Eric Wolf, +Fabian Schmitthenner, +Frederik Rietdijk, +Gabriel Gonzalez, +Giorgio Gallo, +Graham Christensen, +Guillaume Maudoux, +Harmen, +Iavael, +James Broadhead, +James Earl Douglas, +Janus Troelsen, +Jeremy Shaw, +Joachim Schiele, +Joe Hermaszewski, +Joel Moberg, +Johannes 'fish' Ziemke, +Jörg Thalheim, +Jude Taylor, +kballou, +Keshav Kini, +Kjetil Orbekk, +Langston Barrett, +Linus Heckemann, +Ludovic Courtès, +Manav Rathi, +Marc Scholten, +Markus Hauck, +Matt Audesse, +Matthew Bauer, +Matthias Beyer, +Matthieu Coudron, +N1X, +Nathan Zadoks, +Neil Mayhew, +Nicolas B. Pierron, +Niklas Hambüchen, +Nikolay Amiantov, +Ole Jørgen Brønner, +Orivej Desh, +Peter Simons, +Peter Stuart, +Pyry Jahkola, +regnat, +Renzo Carbonara, +Rhys, +Robert Vollmert, +Scott Olson, +Scott R. Parish, +Sergei Trofimovich, +Shea Levy, +Sheena Artrip, +Spencer Baugh, +Stefan Junker, +Susan Potter, +Thomas Tuegel, +Timothy Allen, +Tristan Hume, +Tuomas Tynkkynen, +tv, +Tyson Whitehead, +Vladimír Čunát, +Will Dietz, +wmertens, +Wout Mertens, +zimbatm and +Zoran Plesivčak. +</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-2.1.xml b/third_party/nix/doc/manual/release-notes/rl-2.1.xml new file mode 100644 index 000000000000..16c243fc191a --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.1.xml @@ -0,0 +1,133 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.1"> + +<title>Release 2.1 (2018-09-02)</title> + +<para>This is primarily a bug fix release. It also reduces memory +consumption in certain situations. In addition, it has the following +new features:</para> + +<itemizedlist> + + <listitem> + <para>The Nix installer will no longer default to the Multi-User + installation for macOS. You can still <link + linkend="sect-multi-user-installation">instruct the installer to + run in multi-user mode</link>. + </para> + </listitem> + + <listitem> + <para>The Nix installer now supports performing a Multi-User + installation for Linux computers which are running systemd. You + can <link + linkend="sect-multi-user-installation">select a Multi-User installation</link> by passing the + <option>--daemon</option> flag to the installer: <command>sh <(curl + https://nixos.org/nix/install) --daemon</command>. + </para> + + <para>The multi-user installer cannot handle systems with SELinux. + If your system has SELinux enabled, you can <link + linkend="sect-single-user-installation">force the installer to run + in single-user mode</link>.</para> + </listitem> + + <listitem> + <para>New builtin functions: + <literal>builtins.bitAnd</literal>, + <literal>builtins.bitOr</literal>, + <literal>builtins.bitXor</literal>, + <literal>builtins.fromTOML</literal>, + <literal>builtins.concatMap</literal>, + <literal>builtins.mapAttrs</literal>. + </para> + </listitem> + + <listitem> + <para>The S3 binary cache store now supports uploading NARs larger + than 5 GiB.</para> + </listitem> + + <listitem> + <para>The S3 binary cache store now supports uploading to + S3-compatible services with the <literal>endpoint</literal> + option.</para> + </listitem> + + <listitem> + <para>The flag <option>--fallback</option> is no longer required + to recover from disappeared NARs in binary caches.</para> + </listitem> + + <listitem> + <para><command>nix-daemon</command> now respects + <option>--store</option>.</para> + </listitem> + + <listitem> + <para><command>nix run</command> now respects + <varname>nix-support/propagated-user-env-packages</varname>.</para> + </listitem> + +</itemizedlist> + +<para>This release has contributions from + +Adrien Devresse, +Aleksandr Pashkov, +Alexandre Esteves, +Amine Chikhaoui, +Andrew Dunham, +Asad Saeeduddin, +aszlig, +Ben Challenor, +Ben Gamari, +Benjamin Hipple, +Bogdan Seniuc, +Corey O'Connor, +Daiderd Jordan, +Daniel Peebles, +Daniel Poelzleithner, +Danylo Hlynskyi, +Dmitry Kalinkin, +Domen Kožar, +Doug Beardsley, +Eelco Dolstra, +Erik Arvstedt, +Félix Baylac-Jacqué, +Gleb Peregud, +Graham Christensen, +Guillaume Maudoux, +Ivan Kozik, +John Arnold, +Justin Humm, +Linus Heckemann, +Lorenzo Manacorda, +Matthew Justin Bauer, +Matthew O'Gorman, +Maximilian Bosch, +Michael Bishop, +Michael Fiano, +Michael Mercier, +Michael Raskin, +Michael Weiss, +Nicolas Dudebout, +Peter Simons, +Ryan Trinkle, +Samuel Dionne-Riel, +Sean Seefried, +Shea Levy, +Symphorien Gibol, +Tim Engler, +Tim Sears, +Tuomas Tynkkynen, +volth, +Will Dietz, +Yorick van Pelt and +zimbatm. +</para> + +</section> diff --git a/third_party/nix/doc/manual/release-notes/rl-2.2.xml b/third_party/nix/doc/manual/release-notes/rl-2.2.xml new file mode 100644 index 000000000000..d29eb87e82c8 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.2.xml @@ -0,0 +1,143 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.2"> + +<title>Release 2.2 (2019-01-11)</title> + +<para>This is primarily a bug fix release. It also has the following +changes:</para> + +<itemizedlist> + + <listitem> + <para>In derivations that use structured attributes (i.e. that + specify set the <varname>__structuredAttrs</varname> attribute to + <literal>true</literal> to cause all attributes to be passed to + the builder in JSON format), you can now specify closure checks + per output, e.g.: + +<programlisting> +outputChecks."out" = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; +}; + +outputChecks."dev" = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; +}; +</programlisting> + + </para> + </listitem> + + + <listitem> + <para>The derivation attribute + <varname>requiredSystemFeatures</varname> is now enforced for + local builds, and not just to route builds to remote builders. + The supported features of a machine can be specified through the + configuration setting <varname>system-features</varname>.</para> + + <para>By default, <varname>system-features</varname> includes + <literal>kvm</literal> if <filename>/dev/kvm</filename> + exists. For compatibility, it also includes the pseudo-features + <literal>nixos-test</literal>, <literal>benchmark</literal> and + <literal>big-parallel</literal> which are used by Nixpkgs to route + builds to particular Hydra build machines.</para> + + </listitem> + + <listitem> + <para>Sandbox builds are now enabled by default on Linux.</para> + </listitem> + + <listitem> + <para>The new command <command>nix doctor</command> shows + potential issues with your Nix installation.</para> + </listitem> + + <listitem> + <para>The <literal>fetchGit</literal> builtin function now uses a + caching scheme that puts different remote repositories in distinct + local repositories, rather than a single shared repository. This + may require more disk space but is faster.</para> + </listitem> + + <listitem> + <para>The <literal>dirOf</literal> builtin function now works on + relative paths.</para> + </listitem> + + <listitem> + <para>Nix now supports <link + xlink:href="https://www.w3.org/TR/SRI/">SRI hashes</link>, + allowing the hash algorithm and hash to be specified in a single + string. For example, you can write: + +<programlisting> +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ="; +}; +</programlisting> + + instead of + +<programlisting> +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4"; +}; +</programlisting> + + </para> + + <para>In fixed-output derivations, the + <varname>outputHashAlgo</varname> attribute is no longer mandatory + if <varname>outputHash</varname> specifies the hash.</para> + + <para><command>nix hash-file</command> and <command>nix + hash-path</command> now print hashes in SRI format by + default. They also use SHA-256 by default instead of SHA-512 + because that's what we use most of the time in Nixpkgs.</para> + </listitem> + + <listitem> + <para>Integers are now 64 bits on all platforms.</para> + </listitem> + + <listitem> + <para>The evaluator now prints profiling statistics (enabled via + the <envar>NIX_SHOW_STATS</envar> and + <envar>NIX_COUNT_CALLS</envar> environment variables) in JSON + format.</para> + </listitem> + + <listitem> + <para>The option <option>--xml</option> in <command>nix-store + --query</command> has been removed. Instead, there now is an + option <option>--graphml</option> to output the dependency graph + in GraphML format.</para> + </listitem> + + <listitem> + <para>All <filename>nix-*</filename> commands are now symlinks to + <filename>nix</filename>. This saves a bit of disk space.</para> + </listitem> + + <listitem> + <para><command>nix repl</command> now uses + <literal>libeditline</literal> or + <literal>libreadline</literal>.</para> + </listitem> + +</itemizedlist> + +</section> + diff --git a/third_party/nix/doc/manual/release-notes/rl-2.3.xml b/third_party/nix/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 000000000000..0ad7d641f879 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,91 @@ +<section xmlns="http://docbook.org/ns/docbook" + xmlns:xlink="http://www.w3.org/1999/xlink" + xmlns:xi="http://www.w3.org/2001/XInclude" + version="5.0" + xml:id="ssec-relnotes-2.3"> + +<title>Release 2.3 (2019-09-04)</title> + +<para>This is primarily a bug fix release. However, it makes some +incompatible changes:</para> + +<itemizedlist> + + <listitem> + <para>Nix now uses BSD file locks instead of POSIX file + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store.</para> + </listitem> + +</itemizedlist> + +<para>It also has the following changes:</para> + +<itemizedlist> + + <listitem> + <para><function>builtins.fetchGit</function>'s <varname>ref</varname> + argument now allows specifying an absolute remote ref. + Nix will automatically prefix <varname>ref</varname> with + <literal>refs/heads</literal> only if <varname>ref</varname> doesn't + already begin with <literal>refs/</literal>. + </para> + </listitem> + + <listitem> + <para>The installer now enables sandboxing by default on Linux when the + system has the necessary kernel support. + </para> + </listitem> + + <listitem> + <para>The <literal>max-jobs</literal> setting now defaults to 1.</para> + </listitem> + + <listitem> + <para>New builtin functions: + <literal>builtins.isPath</literal>, + <literal>builtins.hashFile</literal>. + </para> + </listitem> + + <listitem> + <para>The <command>nix</command> command has a new + <option>--print-build-logs</option> (<option>-L</option>) flag to + print build log output to stderr, rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + </para> + </listitem> + + <listitem> + <para>Builds are now executed in a pseudo-terminal, and the + <envar>TERM</envar> environment variable is set to + <literal>xterm-256color</literal>. This allows many programs + (e.g. <command>gcc</command>, <command>clang</command>, + <command>cmake</command>) to print colorized log output.</para> + </listitem> + + <listitem> + <para>Add <option>--no-net</option> convenience flag. This flag + disables substituters; sets the <literal>tarball-ttl</literal> + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces.</para> + </listitem> + + <listitem> + <para>Add a <literal>post-build-hook</literal> setting to run a + program after a build has succeeded.</para> + </listitem> + + <listitem> + <para>Add a <literal>trace-function-calls</literal> setting to log + the duration of Nix function calls to stderr.</para> + </listitem> + +</itemizedlist> + +</section> diff --git a/third_party/nix/doc/manual/schemas.xml b/third_party/nix/doc/manual/schemas.xml new file mode 100644 index 000000000000..691a517b9c26 --- /dev/null +++ b/third_party/nix/doc/manual/schemas.xml @@ -0,0 +1,4 @@ +<?xml version="1.0"?> +<locatingRules xmlns="http://thaiopensource.com/ns/locating-rules/1.0"> + <uri pattern="*.xml" typeId="DocBook"/> +</locatingRules> diff --git a/third_party/nix/misc/systemd/nix-daemon.service.in b/third_party/nix/misc/systemd/nix-daemon.service.in new file mode 100644 index 000000000000..c3d2a4a39e57 --- /dev/null +++ b/third_party/nix/misc/systemd/nix-daemon.service.in @@ -0,0 +1,12 @@ +[Unit] +Description=Nix Daemon +RequiresMountsFor=@storedir@ +RequiresMountsFor=@localstatedir@ +ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket + +[Service] +ExecStart=@@bindir@/nix-daemon nix-daemon +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/third_party/nix/misc/systemd/nix-daemon.socket.in b/third_party/nix/misc/systemd/nix-daemon.socket.in new file mode 100644 index 000000000000..9ed39ffe6eb2 --- /dev/null +++ b/third_party/nix/misc/systemd/nix-daemon.socket.in @@ -0,0 +1,11 @@ +[Unit] +Description=Nix Daemon Socket +Before=multi-user.target +RequiresMountsFor=@storedir@ +ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket + +[Socket] +ListenStream=@localstatedir@/nix/daemon-socket/socket + +[Install] +WantedBy=sockets.target diff --git a/third_party/nix/scripts/build.sh b/third_party/nix/scripts/build.sh new file mode 100755 index 000000000000..759c9e9f2c12 --- /dev/null +++ b/third_party/nix/scripts/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Run `nix build` using a local store, for use during development. Intended to +# be run from the cmake build directory + +set -eo pipefail + +if [ $1 = "--debug" ]; then + run=(gdb --args) + shift 1 +elif [ "$1" = "--rr" ]; then + run=(rr record) + shift 1 +else + run=() +fi + +make -j 10 +NIX_STORE_DIR=$(pwd)/nix/store \ + NIX_LOG_DIR=$(pwd)/nix/var/log/nix \ + NIX_STATE_DIR=$(pwd)/nix/var/nix \ + XDG_CACHE_HOME=$(pwd)/cache \ + NIX_REMOTE=daemon \ + ${run[*]} ./src/nix build "$@" diff --git a/third_party/nix/scripts/daemon.sh b/third_party/nix/scripts/daemon.sh new file mode 100755 index 000000000000..3daa0f139097 --- /dev/null +++ b/third_party/nix/scripts/daemon.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Run a nix daemon using a local store, for use during development. Intended to +# be run from the cmake build directory + +if [ $1 = "--debug" ]; then + run=(gdb --args) + shift 1 +elif [ "$1" = "--rr" ]; then + run=(rr record) + shift 1 +else + run=() +fi + +make -j 10 +NIX_STORE_DIR=$(pwd)/nix/store \ + NIX_LOG_DIR=$(pwd)/nix/var/log/nix \ + NIX_STATE_DIR=$(pwd)/nix/var/nix \ + XDG_CACHE_HOME=$(pwd)/cache \ + NIX_LIBEXEC_DIR=$(pwd) \ + GRPC_TRACE=all \ + ${gdb[*]} ./src/nix-daemon/nix-daemon diff --git a/third_party/nix/scripts/eval.sh b/third_party/nix/scripts/eval.sh new file mode 100755 index 000000000000..f71d9f7931fe --- /dev/null +++ b/third_party/nix/scripts/eval.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Run `nix eval` using a local store, for use during development. Intended to +# be run from the cmake build directory + +if [ "$#" -gt 0 ] && [ "$1" = "--debug" ]; then + gdb=(gdb --args) + shift 1 +elif [ "$1" = "--rr" ]; then + gdb=(rr record) + shift 1 +else + gdb=() +fi + +make -j 10 +NIX_STORE_DIR=$(pwd)/nix/store \ + NIX_LOG_DIR=$(pwd)/nix/var/log/nix \ + NIX_STATE_DIR=$(pwd)/nix/var/nix \ + XDG_CACHE_HOME=$(pwd)/cache \ + NIX_REMOTE=daemon \ + ${gdb[*]} ./src/nix eval "$@" diff --git a/third_party/nix/scripts/install-darwin-multi-user.sh b/third_party/nix/scripts/install-darwin-multi-user.sh new file mode 100644 index 000000000000..49076bd5c03f --- /dev/null +++ b/third_party/nix/scripts/install-darwin-multi-user.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist + +dsclattr() { + /usr/bin/dscl . -read "$1" \ + | awk "/$2/ { print \$2 }" +} + +poly_validate_assumptions() { + if [ "$(uname -s)" != "Darwin" ]; then + failure "This script is for use with macOS!" + fi +} + +poly_service_installed_check() { + [ -e "$PLIST_DEST" ] +} + +poly_service_uninstall_directions() { + cat <<EOF +$1. Delete $PLIST_DEST + + sudo launchctl unload $PLIST_DEST + sudo rm $PLIST_DEST + +EOF +} + +poly_service_setup_note() { + cat <<EOF + - load and start a LaunchDaemon (at $PLIST_DEST) for nix-daemon + +EOF +} + +poly_configure_nix_daemon_service() { + _sudo "to set up the nix-daemon as a LaunchDaemon" \ + cp -f "/nix/var/nix/profiles/default$PLIST_DEST" "$PLIST_DEST" + + _sudo "to load the LaunchDaemon plist for nix-daemon" \ + launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist + + _sudo "to start the nix-daemon" \ + launchctl start org.nixos.nix-daemon + +} + +poly_group_exists() { + /usr/bin/dscl . -read "/Groups/$1" > /dev/null 2>&1 +} + +poly_group_id_get() { + dsclattr "/Groups/$1" "PrimaryGroupID" +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + /usr/sbin/dseditgroup -o create \ + -r "Nix build group for nix-daemon" \ + -i "$NIX_BUILD_GROUP_ID" \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + /usr/bin/dscl . -read "/Users/$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + dsclattr "/Users/$1" "UniqueID" +} + +poly_user_hidden_get() { + dsclattr "/Users/$1" "IsHidden" +} + +poly_user_hidden_set() { + _sudo "in order to make $1 a hidden user" \ + /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1" +} + +poly_user_home_get() { + dsclattr "/Users/$1" "NFSHomeDirectory" +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2" +} + +poly_user_note_get() { + dsclattr "/Users/$1" "RealName" +} + +poly_user_note_set() { + _sudo "in order to give $username a useful note" \ + /usr/bin/dscl . -create "/Users/$1" "RealName" "$2" +} + +poly_user_shell_get() { + dsclattr "/Users/$1" "UserShell" +} + +poly_user_shell_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" +} + +poly_user_in_group_check() { + username=$1 + group=$2 + dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + username=$1 + group=$2 + + _sudo "Add $username to the $group group"\ + /usr/sbin/dseditgroup -o edit -t user \ + -a "$username" "$group" +} + +poly_user_primary_group_get() { + dsclattr "/Users/$1" "PrimaryGroupID" +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + /usr/bin/dscl . -create "/Users/$1" "PrimaryGroupID" "$2" +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user (#$builder_num), $username" \ + /usr/bin/dscl . create "/Users/$username" \ + UniqueID "${uid}" +} diff --git a/third_party/nix/scripts/install-multi-user.sh b/third_party/nix/scripts/install-multi-user.sh new file mode 100644 index 000000000000..5233762fa649 --- /dev/null +++ b/third_party/nix/scripts/install-multi-user.sh @@ -0,0 +1,798 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +# Sourced from: +# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh +# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 +# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh +# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh +# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a +# +# however tracking which bits came from which would be impossible. + +readonly ESC='\033[0m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' + +readonly NIX_USER_COUNT="32" +readonly NIX_BUILD_GROUP_ID="30000" +readonly NIX_BUILD_GROUP_NAME="nixbld" +readonly NIX_FIRST_BUILD_UID="30001" +# Please don't change this. We don't support it, because the +# default shell profile that comes with Nix doesn't support it. +readonly NIX_ROOT="/nix" + +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" +readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + +readonly NIX_INSTALLED_NIX="@nix@" +readonly NIX_INSTALLED_CACERT="@cacert@" +readonly EXTRACTED_NIX_PATH="$(dirname "$0")" + +readonly ROOT_HOME=$(echo ~root) + +if [ -t 0 ]; then + readonly IS_HEADLESS='no' +else + readonly IS_HEADLESS='yes' +fi + +headless() { + if [ "$IS_HEADLESS" = "yes" ]; then + return 0 + else + return 1 + fi +} + +contactme() { + echo "We'd love to help if you need it." + echo "" + echo "If you can, open an issue at https://github.com/nixos/nix/issues" + echo "" + echo "Or feel free to contact the team," + echo " - on IRC #nixos on irc.freenode.net" + echo " - on twitter @nixos_org" +} + +uninstall_directions() { + subheader "Uninstalling nix:" + local step=0 + + if poly_service_installed_check; then + step=$((step + 1)) + poly_service_uninstall_directions "$step" + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + step=$((step + 1)) + cat <<EOF +$step. Restore $profile_target$PROFILE_BACKUP_SUFFIX back to $profile_target + + sudo mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target + +(after this one, you may need to re-open any terminals that were +opened while it existed.) + +EOF + fi + done + + step=$((step + 1)) + cat <<EOF +$step. Delete the files Nix added to your system: + + sudo rm -rf /etc/nix $NIX_ROOT $ROOT_HOME/.nix-profile $ROOT_HOME/.nix-defexpr $ROOT_HOME/.nix-channels $HOME/.nix-profile $HOME/.nix-defexpr $HOME/.nix-channels + +and that is it. + +EOF + +} + +nix_user_for_core() { + printf "nixbld%d" "$1" +} + +nix_uid_for_core() { + echo $((NIX_FIRST_BUILD_UID + $1 - 1)) +} + +_textout() { + echo -en "$1" + shift + if [ "$*" = "" ]; then + cat + else + echo "$@" + fi + echo -en "$ESC" +} + +header() { + follow="---------------------------------------------------------" + header=$(echo "---- $* $follow$follow$follow" | head -c 80) + echo "" + _textout "$BLUE" "$header" +} + +warningheader() { + follow="---------------------------------------------------------" + header=$(echo "---- $* $follow$follow$follow" | head -c 80) + echo "" + _textout "$RED" "$header" +} + +subheader() { + echo "" + _textout "$BLUE_UL" "$*" +} + +row() { + printf "$BOLD%s$ESC:\\t%s\\n" "$1" "$2" +} + +task() { + echo "" + ok "~~> $1" +} + +bold() { + echo "$BOLD$*$ESC" +} + +ok() { + _textout "$GREEN" "$@" +} + +warning() { + warningheader "warning!" + cat + echo "" +} + +failure() { + header "oh no!" + _textout "$RED" "$@" + echo "" + _textout "$RED" "$(contactme)" + trap finish_cleanup EXIT + exit 1 +} + +ui_confirm() { + _textout "$GREEN$GREEN_UL" "$1" + + if headless; then + echo "No TTY, assuming you would say yes :)" + return 0 + fi + + local prompt="[y/n] " + echo -n "$prompt" + while read -r y; do + if [ "$y" = "y" ]; then + echo "" + return 0 + elif [ "$y" = "n" ]; then + echo "" + return 1 + else + _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" + echo -n "$prompt" + fi + done + echo "" + return 1 +} + +__sudo() { + local expl="$1" + local cmd="$2" + shift + header "sudo execution" + + echo "I am executing:" + echo "" + printf " $ sudo %s\\n" "$cmd" + echo "" + echo "$expl" + echo "" + + return 0 +} + +_sudo() { + local expl="$1" + shift + if ! headless; then + __sudo "$expl" "$*" + fi + sudo "$@" +} + + +readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish_cleanup { + rm -rf "$SCRATCH" +} + +function finish_fail { + finish_cleanup + + failure <<EOF +Jeeze, something went wrong. If you can take all the output and open +an issue, we'd love to fix the problem so nobody else has this issue. + +:( +EOF +} +trap finish_fail EXIT + +channel_update_failed=0 +function finish_success { + finish_cleanup + + ok "Alright! We're done!" + if [ "x$channel_update_failed" = x1 ]; then + echo "" + echo "But fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"sudo -i nix-channel --update nixpkgs\"." + fi + cat <<EOF + +Before Nix will work in your existing shells, you'll need to close +them and open them again. Other than that, you should be ready to go. + +Try it! Open a new terminal, and type: + + $ nix-shell -p nix-info --run "nix-info -m" + +Thank you for using this installer. If you have any feedback, don't +hesitate: + +$(contactme) +EOF +} + + +validate_starting_assumptions() { + poly_validate_assumptions + + if [ $EUID -eq 0 ]; then + failure <<EOF +Please do not run this script with root privileges. We will call sudo +when we need to. +EOF + fi + + if type nix-env 2> /dev/null >&2; then + failure <<EOF +Nix already appears to be installed, and this tool assumes it is +_not_ yet installed. + +$(uninstall_directions) +EOF + fi + + if [ "${NIX_REMOTE:-}" != "" ]; then + failure <<EOF +For some reason, \$NIX_REMOTE is set. It really should not be set +before this installer runs, and it hints that Nix is currently +installed. Please delete the old Nix installation and start again. + +Note: You might need to close your shell window and open a new shell +to clear the variable. +EOF + fi + + if echo "${SSL_CERT_FILE:-}" | grep -qE "(nix/var/nix|nix-profile)"; then + failure <<EOF +It looks like \$SSL_CERT_FILE is set to a path that used to be part of +the old Nix installation. Please unset that variable and try again: + + $ unset SSL_CERT_FILE + +EOF + fi + + for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do + if [ -f "$file" ]; then + if grep -l "^[^#].*.nix-profile" "$file"; then + failure <<EOF +I found a reference to a ".nix-profile" in $file. +This has a high chance of breaking a new nix installation. It was most +likely put there by a previous Nix installer. + +Please remove this reference and try running this again. You should +also look for similar references in: + + - ~/.bash_profile + - ~/.bash_login + - ~/.profile + +or other shell init files that you may have. + +$(uninstall_directions) +EOF + fi + fi + done + + if [ -d /nix/store ] || [ -d /nix/var ]; then + failure <<EOF +There are some relics of a previous installation of Nix at /nix, and +this scripts assumes Nix is _not_ yet installed. Please delete the old +Nix installation and start again. + +$(uninstall_directions) +EOF + fi + + if [ -d /etc/nix ]; then + failure <<EOF +There are some relics of a previous installation of Nix at /etc/nix, and +this scripts assumes Nix is _not_ yet installed. Please delete the old +Nix installation and start again. + +$(uninstall_directions) +EOF + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + failure <<EOF +When this script runs, it backs up the current $profile_target to +$profile_target$PROFILE_BACKUP_SUFFIX. This backup file already exists, though. + +Please follow these instructions to clean up the old backup file: + +1. Copy $profile_target and $profile_target$PROFILE_BACKUP_SUFFIX to another place, just +in case. + +2. Take care to make sure that $profile_target$PROFILE_BACKUP_SUFFIX doesn't look like +it has anything nix-related in it. If it does, something is probably +quite wrong. Please open an issue or get in touch immediately. + +3. Take care to make sure that $profile_target doesn't look like it has +anything nix-related in it. If it does, and $profile_target _did not_, +run: + + $ /usr/bin/sudo /bin/mv $profile_target$PROFILE_BACKUP_SUFFIX $profile_target + +and try again. +EOF + fi + + if [ -e "$profile_target" ] && grep -qi "nix" "$profile_target"; then + failure <<EOF +It looks like $profile_target already has some Nix configuration in +there. There should be no reason to run this again. If you're having +trouble, please open an issue. +EOF + fi + done + + danger_paths=("$ROOT_HOME/.nix-defexpr" "$ROOT_HOME/.nix-channels" "$ROOT_HOME/.nix-profile") + for danger_path in "${danger_paths[@]}"; do + if _sudo "making sure that $danger_path doesn't exist" \ + test -e "$danger_path"; then + failure <<EOF +I found a file at $danger_path, which is a relic of a previous +installation. You must first delete this file before continuing. + +$(uninstall_directions) +EOF + fi + done +} + +setup_report() { + header "Nix config report" + row " Temp Dir" "$SCRATCH" + row " Nix Root" "$NIX_ROOT" + row " Build Users" "$NIX_USER_COUNT" + row " Build Group ID" "$NIX_BUILD_GROUP_ID" + row "Build Group Name" "$NIX_BUILD_GROUP_NAME" + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" != "" ]; then + row "Preexisting Install" "Allowed" + fi + + subheader "build users:" + + row " Username" "UID" + for i in $(seq 1 "$NIX_USER_COUNT"); do + row " $(nix_user_for_core "$i")" "$(nix_uid_for_core "$i")" + done + echo "" +} + +create_build_group() { + local primary_group_id + + task "Setting up the build group $NIX_BUILD_GROUP_NAME" + if ! poly_group_exists "$NIX_BUILD_GROUP_NAME"; then + poly_create_build_group + row " Created" "Yes" + else + primary_group_id=$(poly_group_id_get "$NIX_BUILD_GROUP_NAME") + if [ "$primary_group_id" -ne "$NIX_BUILD_GROUP_ID" ]; then + failure <<EOF +It seems the build group $NIX_BUILD_GROUP_NAME already exists, but +with the UID $primary_group_id. This script can't really handle +that right now, so I'm going to give up. + +You can fix this by editing this script and changing the +NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID +to $primary_group_id and re-run. +EOF + else + row " Exists" "Yes" + fi + fi +} + +create_build_user_for_core() { + local coreid + local username + local uid + + coreid="$1" + username=$(nix_user_for_core "$coreid") + uid=$(nix_uid_for_core "$coreid") + + task "Setting up the build user $username" + + if ! poly_user_exists "$username"; then + poly_create_build_user "$username" "$uid" "$coreid" + row " Created" "Yes" + else + actual_uid=$(poly_user_id_get "$username") + if [ "$actual_uid" != "$uid" ]; then + failure <<EOF +It seems the build user $username already exists, but with the UID +with the UID '$actual_uid'. This script can't really handle that right +now, so I'm going to give up. + +If you already created the users and you know they start from +$actual_uid and go up from there, you can edit this script and change +NIX_FIRST_BUILD_UID near the top of the file to $actual_uid and try +again. +EOF + else + row " Exists" "Yes" + fi + fi + + if [ "$(poly_user_hidden_get "$username")" = "1" ]; then + row " Hidden" "Yes" + else + poly_user_hidden_set "$username" + row " Hidden" "Yes" + fi + + if [ "$(poly_user_home_get "$username")" = "/var/empty" ]; then + row " Home Directory" "/var/empty" + else + poly_user_home_set "$username" "/var/empty" + row " Home Directory" "/var/empty" + fi + + # We use grep instead of an equality check because it is difficult + # to extract _just_ the user's note, instead it is prefixed with + # some plist junk. This was causing the user note to always be set, + # even if there was no reason for it. + if ! poly_user_note_get "$username" | grep -q "Nix build user $coreid"; then + row " Note" "Nix build user $coreid" + else + poly_user_note_set "$username" "Nix build user $coreid" + row " Note" "Nix build user $coreid" + fi + + if [ "$(poly_user_shell_get "$username")" = "/sbin/nologin" ]; then + row " Logins Disabled" "Yes" + else + poly_user_shell_set "$username" "/sbin/nologin" + row " Logins Disabled" "Yes" + fi + + if poly_user_in_group_check "$username" "$NIX_BUILD_GROUP_NAME"; then + row " Member of $NIX_BUILD_GROUP_NAME" "Yes" + else + poly_user_in_group_set "$username" "$NIX_BUILD_GROUP_NAME" + row " Member of $NIX_BUILD_GROUP_NAME" "Yes" + fi + + if [ "$(poly_user_primary_group_get "$username")" = "$NIX_BUILD_GROUP_ID" ]; then + row " PrimaryGroupID" "$NIX_BUILD_GROUP_ID" + else + poly_user_primary_group_set "$username" "$NIX_BUILD_GROUP_ID" + row " PrimaryGroupID" "$NIX_BUILD_GROUP_ID" + fi +} + +create_build_users() { + for i in $(seq 1 "$NIX_USER_COUNT"); do + create_build_user_for_core "$i" + done +} + +create_directories() { + # FIXME: remove all of this because it duplicates LocalStore::LocalStore(). + + _sudo "to make the basic directory structure of Nix (part 1)" \ + mkdir -pv -m 0755 /nix /nix/var /nix/var/log /nix/var/log/nix /nix/var/log/nix/drvs /nix/var/nix{,/db,/gcroots,/profiles,/temproots,/userpool} /nix/var/nix/{gcroots,profiles}/per-user + + _sudo "to make the basic directory structure of Nix (part 2)" \ + mkdir -pv -m 1775 /nix/store + + _sudo "to make the basic directory structure of Nix (part 3)" \ + chgrp "$NIX_BUILD_GROUP_NAME" /nix/store + + _sudo "to place the default nix daemon configuration (part 1)" \ + mkdir -pv -m 0555 /etc/nix +} + +place_channel_configuration() { + echo "https://nixos.org/channels/nixpkgs-unstable nixpkgs" > "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" +} + +welcome_to_nix() { + ok "Welcome to the Multi-User Nix Installation" + + cat <<EOF + +This installation tool will set up your computer with the Nix package +manager. This will happen in a few stages: + +1. Make sure your computer doesn't already have Nix. If it does, I + will show you instructions on how to clean up your old one. + +2. Show you what we are going to install and where. Then we will ask + if you are ready to continue. + +3. Create the system users and groups that the Nix daemon uses to run + builds. + +4. Perform the basic installation of the Nix files daemon. + +5. Configure your shell to import special Nix Profile files, so you + can use Nix. + +6. Start the Nix daemon. + +EOF + + if ui_confirm "Would you like to see a more detailed list of what we will do?"; then + cat <<EOF + +We will: + + - make sure your computer doesn't already have Nix files + (if it does, I will tell you how to clean them up.) + - create local users (see the list above for the users we'll make) + - create a local group ($NIX_BUILD_GROUP_NAME) + - install Nix in to $NIX_ROOT + - create a configuration file in /etc/nix + - set up the "default profile" by creating some Nix-related files in + $ROOT_HOME +EOF + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ]; then + cat <<EOF + - back up $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX + - update $profile_target to include some Nix configuration +EOF + fi + done + poly_service_setup_note + if ! ui_confirm "Ready to continue?"; then + failure <<EOF +Okay, maybe you would like to talk to the team. +EOF + fi + fi +} + +chat_about_sudo() { + header "let's talk about sudo" + + if headless; then + cat <<EOF +This script is going to call sudo a lot. Normally, it would show you +exactly what commands it is running and why. However, the script is +run in a headless fashion, like this: + + $ curl https://nixos.org/nix/install | sh + +or maybe in a CI pipeline. Because of that, we're going to skip the +verbose output in the interest of brevity. + +If you would like to +see the output, try like this: + + $ curl -o install-nix https://nixos.org/nix/install + $ sh ./install-nix + +EOF + return 0 + fi + + cat <<EOF +This script is going to call sudo a lot. Every time we do, it'll +output exactly what it'll do, and why. + +Just like this: +EOF + + __sudo "to demonstrate how our sudo prompts look" \ + echo "this is a sudo prompt" + + cat <<EOF + +This might look scary, but everything can be undone by running just a +few commands. We used to ask you to confirm each time sudo ran, but it +was too many times. Instead, I'll just ask you this one time: + +EOF + if ui_confirm "Can we use sudo?"; then + ok "Yay! Thanks! Let's get going!" + else + failure <<EOF +That is okay, but we can't install. +EOF + fi +} + +install_from_extracted_nix() { + ( + cd "$EXTRACTED_NIX_PATH" + + _sudo "to copy the basic Nix files to the new store at $NIX_ROOT/store" \ + rsync -rlpt ./store/* "$NIX_ROOT/store/" + + if [ -d "$NIX_INSTALLED_NIX" ]; then + echo " Alright! We have our first nix at $NIX_INSTALLED_NIX" + else + failure <<EOF +Something went wrong, and I didn't find Nix installed at +$NIX_INSTALLED_NIX. +EOF + fi + + cat ./.reginfo \ + | _sudo "to load data for the first time in to the Nix Database" \ + "$NIX_INSTALLED_NIX/bin/nix-store" --load-db + + echo " Just finished getting the nix database ready." + ) +} + +shell_source_lines() { + cat <<EOF + +# Nix +if [ -e '$PROFILE_NIX_FILE' ]; then + . '$PROFILE_NIX_FILE' +fi +# End Nix + +EOF +} + +configure_shell_profile() { + # If there is an /etc/profile.d directory, we want to ensure there + # is a nix.sh within it, so we can use the following loop to add + # the source lines to it. Note that I'm _not_ adding the source + # lines here, because we want to be using the regular machinery. + # + # If we go around that machinery, it becomes more complicated and + # adds complications to the uninstall instruction generator and + # old instruction sniffer as well. + if [ -d /etc/profile.d ]; then + _sudo "create a stub /etc/profile.d/nix.sh which will be updated" \ + touch /etc/profile.d/nix.sh + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ]; then + _sudo "to back up your current $profile_target to $profile_target$PROFILE_BACKUP_SUFFIX" \ + cp "$profile_target" "$profile_target$PROFILE_BACKUP_SUFFIX" + + shell_source_lines \ + | _sudo "extend your $profile_target with nix-daemon settings" \ + tee -a "$profile_target" + fi + done +} + +setup_default_profile() { + _sudo "to installing a bootstrapping Nix in to the default Profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_NIX" + + if [ -z "${NIX_SSL_CERT_FILE:-}" ] || ! [ -f "${NIX_SSL_CERT_FILE:-}" ]; then + _sudo "to installing a bootstrapping SSL certificate just for Nix in to the default Profile" \ + HOME="$ROOT_HOME" "$NIX_INSTALLED_NIX/bin/nix-env" -i "$NIX_INSTALLED_CACERT" + export NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt + fi + + # Have to explicitly pass NIX_SSL_CERT_FILE as part of the sudo call, + # otherwise it will be lost in environments where sudo doesn't pass + # all the environment variables by default. + _sudo "to update the default channel in the default profile" \ + HOME="$ROOT_HOME" NIX_SSL_CERT_FILE="$NIX_SSL_CERT_FILE" "$NIX_INSTALLED_NIX/bin/nix-channel" --update nixpkgs \ + || channel_update_failed=1 + +} + + +place_nix_configuration() { + cat <<EOF > "$SCRATCH/nix.conf" +build-users-group = $NIX_BUILD_GROUP_NAME +EOF + _sudo "to place the default nix daemon configuration (part 2)" \ + install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +} + +main() { + if [ "$(uname -s)" = "Darwin" ]; then + # shellcheck source=./install-darwin-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi + else + failure "Sorry, I don't know what to do on $(uname)" + fi + + welcome_to_nix + chat_about_sudo + + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then + validate_starting_assumptions + fi + + setup_report + + if ! ui_confirm "Ready to continue?"; then + ok "Alright, no changes have been made :)" + contactme + trap finish_cleanup EXIT + exit 1 + fi + + create_build_group + create_build_users + create_directories + place_channel_configuration + install_from_extracted_nix + + configure_shell_profile + + set +eu + . /etc/profile + set -eu + + setup_default_profile + place_nix_configuration + poly_configure_nix_daemon_service + + trap finish_success EXIT +} + + +main diff --git a/third_party/nix/scripts/install-nix-from-closure.sh b/third_party/nix/scripts/install-nix-from-closure.sh new file mode 100644 index 000000000000..3f15818547ab --- /dev/null +++ b/third_party/nix/scripts/install-nix-from-closure.sh @@ -0,0 +1,180 @@ +#!/bin/sh + +set -e + +dest="/nix" +self="$(dirname "$0")" +nix="@nix@" +cacert="@cacert@" + + +if ! [ -e "$self/.reginfo" ]; then + echo "$0: incomplete installer (.reginfo is missing)" >&2 +fi + +if [ -z "$USER" ] && ! USER=$(id -u -n); then + echo "$0: \$USER is not set" >&2 + exit 1 +fi + +if [ -z "$HOME" ]; then + echo "$0: \$HOME is not set" >&2 + exit 1 +fi + +# macOS support for 10.12.6 or higher +if [ "$(uname -s)" = "Darwin" ]; then + macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) + macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) + if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + exit 1 + fi +fi + +# Determine if we could use the multi-user installer or not +if [ "$(uname -s)" = "Darwin" ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +fi + +INSTALL_MODE=no-daemon +# Trivially handle the --daemon / --no-daemon options +if [ "x${1:-}" = "x--no-daemon" ]; then + INSTALL_MODE=no-daemon +elif [ "x${1:-}" = "x--daemon" ]; then + INSTALL_MODE=daemon +elif [ "x${1:-}" != "x" ]; then + ( + echo "Nix Installer [--daemon|--no-daemon]" + + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + ) >&2 + exit +fi + +if [ "$INSTALL_MODE" = "daemon" ]; then + printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' + exec "$self/install-multi-user" + exit 0 +fi + +if [ "$(id -u)" -eq 0 ]; then + printf '\e[1;31mwarning: installing Nix as root is not supported by this script!\e[0m\n' +fi + +echo "performing a single-user installation of Nix..." >&2 + +if ! [ -e $dest ]; then + cmd="mkdir -m 0755 $dest && chown $USER $dest" + echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 + if ! sudo sh -c "$cmd"; then + echo "$0: please manually run '$cmd' as root to create $dest" >&2 + exit 1 + fi +fi + +if ! [ -w $dest ]; then + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 + exit 1 +fi + +mkdir -p $dest/store + +printf "copying Nix to %s..." "${dest}/store" >&2 + +for i in $(cd "$self/store" >/dev/null && echo ./*); do + printf "." >&2 + i_tmp="$dest/store/$i.$$" + if [ -e "$i_tmp" ]; then + rm -rf "$i_tmp" + fi + if ! [ -e "$dest/store/$i" ]; then + cp -Rp "$self/store/$i" "$i_tmp" + chmod -R a-w "$i_tmp" + chmod +w "$i_tmp" + mv "$i_tmp" "$dest/store/$i" + chmod -w "$dest/store/$i" + fi +done +echo "" >&2 + +if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then + echo "$0: unable to register valid paths" >&2 + exit 1 +fi + +. "$nix/etc/profile.d/nix.sh" + +if ! "$nix/bin/nix-env" -i "$nix"; then + echo "$0: unable to install Nix into your default profile" >&2 + exit 1 +fi + +# Install an SSL certificate bundle. +if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then + $nix/bin/nix-env -i "$cacert" + export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" +fi + +# Subscribe the user to the Nixpkgs channel and fetch it. +if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then + $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable +fi +if [ -z "$_NIX_INSTALLER_TEST" ]; then + if ! $nix/bin/nix-channel --update nixpkgs; then + echo "Fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"nix-channel --update nixpkgs\"." + fi +fi + +added= +p=$HOME/.nix-profile/etc/profile.d/nix.sh +if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then + # Make the shell source nix.sh during login. + for i in .bash_profile .bash_login .profile; do + fn="$HOME/$i" + if [ -w "$fn" ]; then + if ! grep -q "$p" "$fn"; then + echo "modifying $fn..." >&2 + echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + fi + added=1 + break + fi + done +fi + +if [ -z "$added" ]; then + cat >&2 <<EOF + +Installation finished! To ensure that the necessary environment +variables are set, please add the line + + . $p + +to your shell profile (e.g. ~/.profile). +EOF +else + cat >&2 <<EOF + +Installation finished! To ensure that the necessary environment +variables are set, either log in again, or type + + . $p + +in your shell. +EOF +fi diff --git a/third_party/nix/scripts/install-systemd-multi-user.sh b/third_party/nix/scripts/install-systemd-multi-user.sh new file mode 100755 index 000000000000..bef3ac4f991f --- /dev/null +++ b/third_party/nix/scripts/install-systemd-multi-user.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service +readonly SERVICE_DEST=/etc/systemd/system/nix-daemon.service + +readonly SOCKET_SRC=/lib/systemd/system/nix-daemon.socket +readonly SOCKET_DEST=/etc/systemd/system/nix-daemon.socket + + +# Path for the systemd override unit file to contain the proxy settings +readonly SERVICE_OVERRIDE=${SERVICE_DEST}.d/override.conf + +create_systemd_override() { + header "Configuring proxy for the nix-daemon service" + _sudo "create directory for systemd unit override" mkdir -p "$(dirname $SERVICE_OVERRIDE)" + cat <<EOF | _sudo "create systemd unit override" tee "$SERVICE_OVERRIDE" +[Service] +$1 +EOF +} + +# Gather all non-empty proxy environment variables into a string +create_systemd_proxy_env() { + vars="http_proxy https_proxy ftp_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY NO_PROXY" + for v in $vars; do + if [ "x${!v:-}" != "x" ]; then + echo "Environment=${v}=${!v}" + fi + done +} + +handle_network_proxy() { + # Create a systemd unit override with proxy environment variables + # if any proxy environment variables are not empty. + PROXY_ENV_STRING=$(create_systemd_proxy_env) + if [ -n "${PROXY_ENV_STRING}" ]; then + create_systemd_override "${PROXY_ENV_STRING}" + fi +} + +poly_validate_assumptions() { + if [ "$(uname -s)" != "Linux" ]; then + failure "This script is for use with Linux!" + fi +} + +poly_service_installed_check() { + [ "$(systemctl is-enabled nix-daemon.service)" = "linked" ] \ + || [ "$(systemctl is-enabled nix-daemon.socket)" = "enabled" ] +} + +poly_service_uninstall_directions() { + cat <<EOF +$1. Delete the systemd service and socket units + + sudo systemctl stop nix-daemon.socket + sudo systemctl stop nix-daemon.service + sudo systemctl disable nix-daemon.socket + sudo systemctl disable nix-daemon.service + sudo systemctl daemon-reload +EOF +} + +poly_service_setup_note() { + cat <<EOF + - load and start a service (at $SERVICE_DEST + and $SOCKET_DEST) for nix-daemon + +EOF +} + +poly_configure_nix_daemon_service() { + _sudo "to set up the nix-daemon service" \ + systemctl link "/nix/var/nix/profiles/default$SERVICE_SRC" + + _sudo "to set up the nix-daemon socket service" \ + systemctl enable "/nix/var/nix/profiles/default$SOCKET_SRC" + + handle_network_proxy + + _sudo "to load the systemd unit for nix-daemon" \ + systemctl daemon-reload + + _sudo "to start the nix-daemon.socket" \ + systemctl start nix-daemon.socket + + _sudo "to start the nix-daemon.service" \ + systemctl start nix-daemon.service + +} + +poly_group_exists() { + getent group "$1" > /dev/null 2>&1 +} + +poly_group_id_get() { + getent group "$1" | cut -d: -f3 +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + groupadd -g "$NIX_BUILD_GROUP_ID" --system \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + getent passwd "$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + getent passwd "$1" | cut -d: -f3 +} + +poly_user_hidden_get() { + echo "1" +} + +poly_user_hidden_set() { + true +} + +poly_user_home_get() { + getent passwd "$1" | cut -d: -f6 +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + usermod --home "$2" "$1" +} + +poly_user_note_get() { + getent passwd "$1" | cut -d: -f5 +} + +poly_user_note_set() { + _sudo "in order to give $1 a useful comment" \ + usermod --comment "$2" "$1" +} + +poly_user_shell_get() { + getent passwd "$1" | cut -d: -f7 +} + +poly_user_shell_set() { + _sudo "in order to prevent $1 from logging in" \ + usermod --shell "$2" "$1" +} + +poly_user_in_group_check() { + groups "$1" | grep -q "$2" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + _sudo "Add $1 to the $2 group"\ + usermod --append --groups "$2" "$1" +} + +poly_user_primary_group_get() { + getent passwd "$1" | cut -d: -f4 +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + usermod --gid "$2" "$1" + +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user, $username" \ + useradd \ + --home-dir /var/empty \ + --comment "Nix build user $builder_num" \ + --gid "$NIX_BUILD_GROUP_ID" \ + --groups "$NIX_BUILD_GROUP_NAME" \ + --no-user-group \ + --system \ + --shell /sbin/nologin \ + --uid "$uid" \ + --password "!" \ + "$username" +} diff --git a/third_party/nix/scripts/install.in b/third_party/nix/scripts/install.in new file mode 100644 index 000000000000..902758b138a5 --- /dev/null +++ b/third_party/nix/scripts/install.in @@ -0,0 +1,66 @@ +#!/bin/sh + +# This script installs the Nix package manager on your system by +# downloading a binary distribution and running its installer script +# (which in turn creates and populates /nix). + +{ # Prevent execution if this script was only partially downloaded +oops() { + echo "$0:" "$@" >&2 + exit 1 +} + +tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ + oops "Can't create temporary directory for downloading the Nix binary tarball")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +require_util() { + command -v "$1" > /dev/null 2>&1 || + oops "you do not have '$1' installed, which I need to $2" +} + +case "$(uname -s).$(uname -m)" in + Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;; + Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; + Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; + Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + *) oops "sorry, there is no binary distribution of Nix for your platform";; +esac + +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" + +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" + +require_util curl "download the binary tarball" +require_util tar "unpack the binary tarball" + +echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." +curl -L "$url" -o "$tarball" || oops "failed to download '$url'" + +if command -v sha256sum > /dev/null 2>&1; then + hash2="$(sha256sum -b "$tarball" | cut -c1-64)" +elif command -v shasum > /dev/null 2>&1; then + hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" +elif command -v openssl > /dev/null 2>&1; then + hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" +else + oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" +fi + +if [ "$hash" != "$hash2" ]; then + oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2" +fi + +unpack=$tmpDir/unpack +mkdir -p "$unpack" +tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" + +script=$(echo "$unpack"/*/install) + +[ -e "$script" ] || oops "installation script is missing from the binary tarball!" +"$script" "$@" + +} # End of wrapping diff --git a/third_party/nix/scripts/nix-http-export.cgi.in b/third_party/nix/scripts/nix-http-export.cgi.in new file mode 100755 index 000000000000..19a505af1c50 --- /dev/null +++ b/third_party/nix/scripts/nix-http-export.cgi.in @@ -0,0 +1,51 @@ +#! /bin/sh + +export HOME=/tmp +export NIX_REMOTE=daemon + +TMP_DIR="${TMP_DIR:-/tmp/nix-export}" + +@coreutils@/mkdir -p "$TMP_DIR" || true +@coreutils@/chmod a+r "$TMP_DIR" + +needed_path="?$QUERY_STRING" +needed_path="${needed_path#*[?&]needed_path=}" +needed_path="${needed_path%%&*}" +#needed_path="$(echo $needed_path | ./unhttp)" +needed_path="${needed_path//%2B/+}" +needed_path="${needed_path//%3D/=}" + +echo needed_path: "$needed_path" >&2 + +NIX_STORE="${NIX_STORE_DIR:-/nix/store}" + +echo NIX_STORE: "${NIX_STORE}" >&2 + +full_path="${NIX_STORE}"/"$needed_path" + +if [ "$needed_path" != "${needed_path%.drv}" ]; then + echo "Status: 403 You should create the derivation file yourself" + echo "Content-Type: text/plain" + echo + echo "Refusing to disclose derivation contents" + exit +fi + +if @bindir@/nix-store --check-validity "$full_path"; then + if ! [ -e nix-export/"$needed_path".nar.gz ]; then + @bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz + @coreutils@/ln -fs "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz + fi; + echo "Status: 301 Moved" + echo "Location: nix-export/"$needed_path".nar.gz" + echo +else + echo "Status: 404 No such path found" + echo "Content-Type: text/plain" + echo + echo "Path not found:" + echo "$needed_path" + echo "checked:" + echo "$full_path" +fi + diff --git a/third_party/nix/scripts/nix-profile-daemon.sh.in b/third_party/nix/scripts/nix-profile-daemon.sh.in new file mode 100644 index 000000000000..47655080a674 --- /dev/null +++ b/third_party/nix/scripts/nix-profile-daemon.sh.in @@ -0,0 +1,29 @@ +# Only execute this file once per shell. +if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi +__ETC_PROFILE_NIX_SOURCED=1 + +export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" +export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then + : # Allow users to override the NIX_SSL_CERT_FILE +elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem +elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done +fi + +export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" +export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" diff --git a/third_party/nix/scripts/nix-profile.sh.in b/third_party/nix/scripts/nix-profile.sh.in new file mode 100644 index 000000000000..e15f7cd46b76 --- /dev/null +++ b/third_party/nix/scripts/nix-profile.sh.in @@ -0,0 +1,39 @@ +if [ -n "$HOME" ] && [ -n "$USER" ]; then + + # Set up the per-user profile. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix + + NIX_LINK=$HOME/.nix-profile + + NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER + + # Append ~/.nix-defexpr/channels to $NIX_PATH so that <nixpkgs> + # paths work when the user has fetched the Nixpkgs channel. + export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. + if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem + elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt + elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt + elif [ -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in Nix profile + export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ssl/certs/ca-bundle.crt" + elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # old cacert in Nix profile + export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" + fi + + if [ -n "${MANPATH-}" ]; then + export MANPATH="$NIX_LINK/share/man:$MANPATH" + fi + + export PATH="$NIX_LINK/bin:$PATH" + unset NIX_LINK NIX_USER_PROFILE_DIR +fi diff --git a/third_party/nix/scripts/nix-reduce-build.in b/third_party/nix/scripts/nix-reduce-build.in new file mode 100755 index 000000000000..50beb9d10b16 --- /dev/null +++ b/third_party/nix/scripts/nix-reduce-build.in @@ -0,0 +1,171 @@ +#! @bash@ + +WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX); +cd "$WORKING_DIRECTORY"; + +if test -z "$1" || test "a--help" = "a$1" ; then + echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2 + echo As in: >&2 + echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2 + echo nix-reduce-build /etc/nixos/nixos -- \\ + echo " " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2 + echo " store path name will be added into the end of the URL" >&2 + echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2 + echo " that should be a directory where gzipped 'nix-store --export' ">&2 + echo " files are located (they should have .nar.gz extension)" >&2 + echo " Or all together: " >&2 + echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\ + " ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2 + echo " Also supports best-effort local builds of failing expression set:" >&2 + echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2 + echo " nix-daemon:// builds using daemon" + echo " nix-self:// builds directly using nix-store from current installation" >&2 + echo " nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2; + echo "derivations with specified output hash (sha256, sha1 or md5)." >&2 + echo " nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2; + echo "maximum amount of paths" >&2; + echo " nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2; + echo "maximum amount of paths" >&2; + echo " If no package sources are specified, required paths are listed." >&2; + exit; +fi; + +while ! test "$1" = "--" || test "$1" = "" ; do + echo "$1" >> initial; >&2 + shift; +done +shift; +echo Will work on $(cat initial | wc -l) targets. >&2 + +while read ; do + case "$REPLY" in + ${NIX_STORE_DIR:-/nix/store}/*) + echo "$REPLY" >> paths; >&2 + ;; + *) + ( + IFS=: ; + nix-instantiate $REPLY >> paths; + ); + ;; + esac; +done < initial; +echo Proceeding $(cat paths | wc -l) paths. >&2 + +while read; do + case "$REPLY" in + *.drv) + echo "$REPLY" >> derivers; >&2 + ;; + *) + nix-store --query --deriver "$REPLY" >>derivers; + ;; + esac; +done < paths; +echo Found $(cat derivers | wc -l) derivers. >&2 + +cat derivers | xargs nix-store --query -R > derivers-closure; +echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2 + +cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths; +cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths; +echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2 + +cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; +echo We need $(cat needed-paths | wc -l) paths. >&2 + +egrep '[.]drv$' derivers-closure > critical-derivers; + +if test -z "$1" ; then + cat needed-paths; +fi; + +refresh_critical_derivers() { + echo "Finding needed derivers..." >&2; + cat critical-derivers | while read; do + if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then + echo "$REPLY"; + fi; + done > new-critical-derivers; + mv new-critical-derivers critical-derivers; + echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2 +} + +build_here() { + cat critical-derivers | while read; do + echo "Realising $REPLY using nix-daemon" >&2 + @bindir@/nix-store -r "${REPLY}" + done; +} + +try_to_substitute(){ + cat needed-paths | while read ; do + echo "Building $REPLY using nix-daemon" >&2 + @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}" + done; +} + +for i in "$@"; do + sshHost="${i#ssh://}"; + httpHost="${i#http://}"; + httpsHost="${i#https://}"; + filePath="${i#file:/}"; + if [ "$i" != "$sshHost" ]; then + cat needed-paths | while read; do + echo "Getting $REPLY and its closure over ssh" >&2 + nix-copy-closure --from "$sshHost" --gzip "$REPLY" </dev/null || true; + done; + elif [ "$i" != "$httpHost" ] || [ "$i" != "$httpsHost" ]; then + cat needed-paths | while read; do + echo "Getting $REPLY over http/https" >&2 + curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import; + done; + elif [ "$i" != "$filePath" ] ; then + cat needed-paths | while read; do + echo "Installing $REPLY from file" >&2 + gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import; + done; + elif [ "$i" = "nix-daemon://" ] ; then + NIX_REMOTE=daemon try_to_substitute; + refresh_critical_derivers; + NIX_REMOTE=daemon build_here; + elif [ "$i" = "nix-self://" ] ; then + NIX_REMOTE= try_to_substitute; + refresh_critical_derivers; + NIX_REMOTE= build_here; + elif [ "$i" = "nix-daemon-fixed://" ] ; then + refresh_critical_derivers; + + cat critical-derivers | while read; do + if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then + echo "Realising $REPLY using nix-daemon" >&2 + NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}" + fi; + done; + elif [ "$i" = "nix-self-fixed://" ] ; then + refresh_critical_derivers; + + cat critical-derivers | while read; do + if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then + echo "Realising $REPLY using direct Nix build" >&2 + NIX_REMOTE= @bindir@/nix-store -r "${REPLY}" + fi; + done; + elif [ "$i" = "nix-daemon-substitute://" ] ; then + NIX_REMOTE=daemon try_to_substitute; + elif [ "$i" = "nix-self-substitute://" ] ; then + NIX_REMOTE= try_to_substitute; + elif [ "$i" = "nix-daemon-build://" ] ; then + refresh_critical_derivers; + NIX_REMOTE=daemon build_here; + elif [ "$i" = "nix-self-build://" ] ; then + refresh_critical_derivers; + NIX_REMOTE= build_here; + fi; + mv needed-paths wanted-paths; + cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; + echo We still need $(cat needed-paths | wc -l) paths. >&2 +done; + +cd / +rm -r "$WORKING_DIRECTORY" diff --git a/third_party/nix/scripts/repl.sh b/third_party/nix/scripts/repl.sh new file mode 100755 index 000000000000..d068e807903a --- /dev/null +++ b/third_party/nix/scripts/repl.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Run `nix repl` using a local store, for use during development. Intended to +# be run from the cmake build directory + +if [ "$#" -gt 0 ] && [ "$1" = "--debug" ]; then + gdb=(gdb --args) + shift 1 +elif [ "$1" = "--rr" ]; then + gdb=(rr record) + shift 1 +else + gdb=() +fi + +make -j 10 +NIX_STORE_DIR=$(pwd)/nix/store \ + NIX_LOG_DIR=$(pwd)/nix/var/log/nix \ + NIX_STATE_DIR=$(pwd)/nix/var/nix \ + XDG_CACHE_HOME=$(pwd)/cache \ + NIX_REMOTE=daemon \ + ${gdb[*]} ./src/nix repl "$@" diff --git a/third_party/nix/scripts/setup_store.sh b/third_party/nix/scripts/setup_store.sh new file mode 100755 index 000000000000..ee96c8d3b853 --- /dev/null +++ b/third_party/nix/scripts/setup_store.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Setup a store for local development rooted at the current directory, and +# compatible with the scripts in this directory (repl.sh, build.sh, eval.sh, +# daemon.sh, etc). Intended to be run from the cmake build directory + +mkdir -p nix/store nix/var/nix nix/var/log/nix +ln -s $(pwd)/src/nix ./nix/build-remote +mkdir -p $(dirname "$(pwd)${SANDBOX_SHELL}") +cp "${SANDBOX_SHELL}" "$(pwd)${SANDBOX_SHELL}" diff --git a/third_party/nix/src/CMakeLists.txt b/third_party/nix/src/CMakeLists.txt new file mode 100644 index 000000000000..486c69fa2a3c --- /dev/null +++ b/third_party/nix/src/CMakeLists.txt @@ -0,0 +1,85 @@ +# -*- mode: cmake; -*- + +# The 'nix' binary is composed of various sources below this +# directory. In the previous build system, they were all built from +# this location and this setup mimics that (with the exception of the +# various Nix libraries). + +add_subdirectory(proto) +add_subdirectory(libutil) +add_subdirectory(libstore) +add_subdirectory(libmain) +add_subdirectory(libexpr) +add_subdirectory(nix-daemon) + +if (PACKAGE_TESTS) + add_subdirectory(tests) +endif() + +add_executable(nix) +set_property(TARGET nix PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) +target_include_directories(nix PUBLIC "${nix_SOURCE_DIR}/src") + +target_sources(nix + PRIVATE + nix/command.hh + nix/legacy.hh + nix-env/user-env.hh + nix-store/dotgraph.hh + nix-store/graphml.hh + + nix/add-to-store.cc + nix/build.cc + nix/cat.cc + nix/command.cc + nix/copy.cc + nix/doctor.cc + nix/dump-path.cc + nix/edit.cc + nix/eval.cc + nix/hash.cc + nix/installables.cc + nix/legacy.cc + nix/log.cc + nix/ls.cc + nix/main.cc + nix/optimise-store.cc + nix/path-info.cc + nix/ping-store.cc + nix/repl.cc + nix/run.cc + nix/search.cc + nix/show-config.cc + nix/show-derivation.cc + nix/sigs.cc + nix/upgrade-nix.cc + nix/verify.cc + nix/why-depends.cc + + build-remote/build-remote.cc + nix-build/nix-build.cc + nix-channel/nix-channel.cc + nix-collect-garbage/nix-collect-garbage.cc + nix-copy-closure/nix-copy-closure.cc + nix-env/nix-env.cc + nix-env/user-env.cc + nix-instantiate/nix-instantiate.cc + nix-prefetch-url/nix-prefetch-url.cc + nix-store/dotgraph.cc + nix-store/graphml.cc + nix-store/nix-store.cc +) + +target_link_libraries(nix + nixexpr + nixmain + nixstore + nixutil + + absl::strings + editline + glog +) + +INSTALL(TARGETS nix DESTINATION bin) diff --git a/third_party/nix/src/build-remote/build-remote.cc b/third_party/nix/src/build-remote/build-remote.cc new file mode 100644 index 000000000000..43564a5eb74a --- /dev/null +++ b/third_party/nix/src/build-remote/build-remote.cc @@ -0,0 +1,274 @@ +#include <algorithm> +#include <cstdlib> +#include <cstring> +#include <iomanip> +#include <memory> +#include <set> +#include <tuple> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_cat.h> +#include <glog/logging.h> + +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libstore/machines.hh" +#include "libstore/pathlocks.hh" +#include "libstore/store-api.hh" +#include "libutil/serialise.hh" +#include "nix/legacy.hh" + +using namespace nix; +using std::cin; + +static void handleAlarm(int sig) {} + +std::string escapeUri(std::string uri) { + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; +} + +static std::string currentLoad; + +static AutoCloseFD openSlotLock(const Machine& m, unsigned long long slot) { + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), + true); +} + +static bool allSupportedLocally(const std::set<std::string>& requiredFeatures) { + for (auto& feature : requiredFeatures) { + if (settings.systemFeatures.get().count(feature) == 0u) { + return false; + } + } + return true; +} + +static int _main(int argc, char* argv[]) { + { + /* Ensure we don't get any SSH passphrase or host key popups. */ + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); + + FdSource source(STDIN_FILENO); + + /* Read the parent's settings. */ + while (readInt(source) != 0u) { + auto name = readString(source); + auto value = readString(source); + settings.set(name, value); + } + + settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + + auto store = openStore().cast<LocalStore>(); + + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on macOS. */ + currentLoad = store->stateDir + "/current-load"; + + std::shared_ptr<Store> sshStore; + AutoCloseFD bestSlotLock; + + auto machines = getMachines(); + DLOG(INFO) << "got " << machines.size() << " remote builders"; + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return 0; + } + + std::string drvPath; + std::string storeUri; + + while (true) { + try { + auto s = readString(source); + if (s != "try") { + return 0; + } + } catch (EndOfFile&) { + return 0; + } + + auto amWilling = readInt(source); + auto neededSystem = readString(source); + source >> drvPath; + auto requiredFeatures = readStrings<std::set<std::string>>(source); + + auto canBuildLocally = + (amWilling != 0u) && + (neededSystem == settings.thisSystem || + settings.extraPlatforms.get().count(neededSystem) > 0) && + allSupportedLocally(requiredFeatures); + + /* Error ignored here, will be caught later */ + mkdir(currentLoad.c_str(), 0777); + + while (true) { + bestSlotLock = AutoCloseFD(-1); + AutoCloseFD lock(openLockFile(currentLoad + "/main-lock", true)); + lockFile(lock.get(), ltWrite, true); + + bool rightType = false; + + Machine* bestMachine = nullptr; + unsigned long long bestLoad = 0; + for (auto& m : machines) { + DLOG(INFO) << "considering building on remote machine '" << m.storeUri + << "'"; + + if (m.enabled && + std::find(m.systemTypes.begin(), m.systemTypes.end(), + neededSystem) != m.systemTypes.end() && + m.allSupported(requiredFeatures) && + m.mandatoryMet(requiredFeatures)) { + rightType = true; + AutoCloseFD free; + unsigned long long load = 0; + for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + auto slotLock = openSlotLock(m, slot); + if (lockFile(slotLock.get(), ltWrite, false)) { + if (!free) { + free = std::move(slotLock); + } + } else { + ++load; + } + } + if (!free) { + continue; + } + bool best = false; + if (!bestMachine || !bestSlotLock) { + best = true; + } else if (load / m.speedFactor < + bestLoad / bestMachine->speedFactor) { + best = true; + } else if (load / m.speedFactor == + bestLoad / bestMachine->speedFactor) { + if (m.speedFactor > bestMachine->speedFactor) { + best = true; + } else if (m.speedFactor == bestMachine->speedFactor) { + if (load < bestLoad) { + best = true; + } + } + } + if (best) { + bestLoad = load; + bestSlotLock = std::move(free); + bestMachine = &m; + } + } + } + + if (!bestSlotLock || !bestMachine) { + if (rightType && !canBuildLocally) { + std::cerr << "# postpone\n"; + } else { + std::cerr << "# decline\n"; + } + break; + } + + futimens(bestSlotLock.get(), nullptr); + + lock = AutoCloseFD(-1); + + try { + DLOG(INFO) << "connecting to '" << bestMachine->storeUri << "'"; + + Store::Params storeParams; + if (absl::StartsWith(bestMachine->storeUri, "ssh://")) { + storeParams["max-connections"] = "1"; + storeParams["log-fd"] = "4"; + if (!bestMachine->sshKey.empty()) { + storeParams["ssh-key"] = bestMachine->sshKey; + } + } + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + + } catch (std::exception& e) { + auto msg = absl::StripTrailingAsciiWhitespace(drainFD(5, false)); + LOG(ERROR) << "cannot build on '" << bestMachine->storeUri + << "': " << e.what() + << (msg.empty() ? "" : absl::StrCat(": ", msg)); + bestMachine->enabled = false; + continue; + } + + goto connected; + } + } + + connected: + close(5); + + std::cerr << "# accept\n" << storeUri << "\n"; + + auto inputs = readStrings<PathSet>(source); + auto outputs = readStrings<PathSet>(source); + + AutoCloseFD uploadLock = openLockFile( + currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + + { + DLOG(INFO) << "waiting for the upload lock to '" << storeUri << "'"; + + auto old = signal(SIGALRM, handleAlarm); + alarm(15 * 60); + if (!lockFile(uploadLock.get(), ltWrite, true)) { + LOG(ERROR) << "somebody is hogging the upload lock, continuing..."; + } + alarm(0); + signal(SIGALRM, old); + } + + auto substitute = + settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + + { + DLOG(INFO) << "copying dependencies to '" << storeUri << "'"; + copyPaths(store, ref<Store>(sshStore), inputs, NoRepair, NoCheckSigs, + substitute); + } + + uploadLock = AutoCloseFD(-1); + + BasicDerivation drv( + readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); + drv.inputSrcs = inputs; + + auto result = sshStore->buildDerivation(std::cerr, drvPath, drv); + + if (!result.success()) { + throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, + result.errorMsg); + } + + PathSet missing; + for (auto& path : outputs) { + if (!store->isValidPath(path)) { + missing.insert(path); + } + } + + if (!missing.empty()) { + DLOG(INFO) << "copying outputs from '" << storeUri << "'"; + store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ + copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, + NoSubstitute); + } + + return 0; + } +} + +static RegisterLegacyCommand s1("build-remote", _main); diff --git a/third_party/nix/src/cpptoml/LICENSE b/third_party/nix/src/cpptoml/LICENSE new file mode 100644 index 000000000000..8802c4fa5a36 --- /dev/null +++ b/third_party/nix/src/cpptoml/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2014 Chase Geigle + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/nix/src/cpptoml/cpptoml.h b/third_party/nix/src/cpptoml/cpptoml.h new file mode 100644 index 000000000000..150b53ff863c --- /dev/null +++ b/third_party/nix/src/cpptoml/cpptoml.h @@ -0,0 +1,3668 @@ +/** + * @file cpptoml.h + * @author Chase Geigle + * @date May 2013 + */ + +#ifndef CPPTOML_H +#define CPPTOML_H + +#include <algorithm> +#include <cassert> +#include <clocale> +#include <cstdint> +#include <cstring> +#include <fstream> +#include <iomanip> +#include <map> +#include <memory> +#include <sstream> +#include <stdexcept> +#include <string> +#include <unordered_map> +#include <vector> + +#if __cplusplus > 201103L +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(__clang__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason))) +#elif defined(__GNUG__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated)) +#elif defined(_MSC_VER) +#if _MSC_VER < 1910 +#define CPPTOML_DEPRECATED(reason) __declspec(deprecated) +#else +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#endif +#endif + +namespace cpptoml +{ +class writer; // forward declaration +class base; // forward declaration +#if defined(CPPTOML_USE_MAP) +// a std::map will ensure that entries a sorted, albeit at a slight +// performance penalty relative to the (default) unordered_map +using string_to_base_map = std::map<std::string, std::shared_ptr<base>>; +#else +// by default an unordered_map is used for best performance as the +// toml specification does not require entries to be sorted +using string_to_base_map + = std::unordered_map<std::string, std::shared_ptr<base>>; +#endif + +// if defined, `base` will retain type information in form of an enum class +// such that static_cast can be used instead of dynamic_cast +// #define CPPTOML_NO_RTTI + +template <class T> +class option +{ + public: + option() : empty_{true} + { + // nothing + } + + option(T value) : empty_{false}, value_(std::move(value)) + { + // nothing + } + + explicit operator bool() const + { + return !empty_; + } + + const T& operator*() const + { + return value_; + } + + const T* operator->() const + { + return &value_; + } + + template <class U> + T value_or(U&& alternative) const + { + if (!empty_) + return value_; + return static_cast<T>(std::forward<U>(alternative)); + } + + private: + bool empty_; + T value_; +}; + +struct local_date +{ + int year = 0; + int month = 0; + int day = 0; +}; + +struct local_time +{ + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; +}; + +struct zone_offset +{ + int hour_offset = 0; + int minute_offset = 0; +}; + +struct local_datetime : local_date, local_time +{ +}; + +struct offset_datetime : local_datetime, zone_offset +{ + static inline struct offset_datetime from_zoned(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + + char buf[16]; + strftime(buf, 16, "%z", &t); + + int offset = std::stoi(buf); + dt.hour_offset = offset / 100; + dt.minute_offset = offset % 100; + return dt; + } + + CPPTOML_DEPRECATED("from_local has been renamed to from_zoned") + static inline struct offset_datetime from_local(const struct tm& t) + { + return from_zoned(t); + } + + static inline struct offset_datetime from_utc(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + return dt; + } +}; + +CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime") +typedef offset_datetime datetime; + +class fill_guard +{ + public: + fill_guard(std::ostream& os) : os_(os), fill_{os.fill()} + { + // nothing + } + + ~fill_guard() + { + os_.fill(fill_); + } + + private: + std::ostream& os_; + std::ostream::char_type fill_; +}; + +inline std::ostream& operator<<(std::ostream& os, const local_date& dt) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2) + << dt.day; + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_time& ltime) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":" + << setw(2) << ltime.second; + + if (ltime.microsecond > 0) + { + os << "."; + int power = 100000; + for (int curr_us = ltime.microsecond; curr_us; power /= 10) + { + auto num = curr_us / power; + os << num; + curr_us -= num * power; + } + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + + if (zo.hour_offset != 0 || zo.minute_offset != 0) + { + if (zo.hour_offset > 0) + { + os << "+"; + } + else + { + os << "-"; + } + os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2) + << std::abs(zo.minute_offset); + } + else + { + os << "Z"; + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt) +{ + return os << static_cast<const local_date&>(dt) << "T" + << static_cast<const local_time&>(dt); +} + +inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt) +{ + return os << static_cast<const local_datetime&>(dt) + << static_cast<const zone_offset&>(dt); +} + +template <class T, class... Ts> +struct is_one_of; + +template <class T, class V> +struct is_one_of<T, V> : std::is_same<T, V> +{ +}; + +template <class T, class V, class... Ts> +struct is_one_of<T, V, Ts...> +{ + const static bool value + = std::is_same<T, V>::value || is_one_of<T, Ts...>::value; +}; + +template <class T> +class value; + +template <class T> +struct valid_value + : is_one_of<T, std::string, int64_t, double, bool, local_date, local_time, + local_datetime, offset_datetime> +{ +}; + +template <class T, class Enable = void> +struct value_traits; + +template <class T> +struct valid_value_or_string_convertible +{ + + const static bool value = valid_value<typename std::decay<T>::type>::value + || std::is_convertible<T, std::string>::value; +}; + +template <class T> +struct value_traits<T, typename std::enable_if< + valid_value_or_string_convertible<T>::value>::type> +{ + using value_type = typename std::conditional< + valid_value<typename std::decay<T>::type>::value, + typename std::decay<T>::type, std::string>::type; + + using type = value<value_type>; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template <class T> +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_floating_point<typename std::decay<T>::type>::value>::type> +{ + using value_type = typename std::decay<T>::type; + + using type = value<double>; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template <class T> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && !std::is_floating_point<typename std::decay<T>::type>::value + && std::is_signed<typename std::decay<T>::type>::value>::type> +{ + using value_type = int64_t; + + using type = value<int64_t>; + + static value_type construct(T&& val) + { + if (val < (std::numeric_limits<int64_t>::min)()) + throw std::underflow_error{"constructed value cannot be " + "represented by a 64-bit signed " + "integer"}; + + if (val > (std::numeric_limits<int64_t>::max)()) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast<int64_t>(val); + } +}; + +template <class T> +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible<T>::value + && std::is_unsigned<typename std::decay<T>::type>::value>::type> +{ + using value_type = int64_t; + + using type = value<int64_t>; + + static value_type construct(T&& val) + { + if (val > static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast<int64_t>(val); + } +}; + +class array; +class table; +class table_array; + +template <class T> +struct array_of_trait +{ + using return_type = option<std::vector<T>>; +}; + +template <> +struct array_of_trait<array> +{ + using return_type = option<std::vector<std::shared_ptr<array>>>; +}; + +template <class T> +inline std::shared_ptr<typename value_traits<T>::type> make_value(T&& val); +inline std::shared_ptr<array> make_array(); + +namespace detail +{ +template <class T> +inline std::shared_ptr<T> make_element(); +} + +inline std::shared_ptr<table> make_table(); +inline std::shared_ptr<table_array> make_table_array(bool is_inline = false); + +#if defined(CPPTOML_NO_RTTI) +/// Base type used to store underlying data type explicitly if RTTI is disabled +enum class base_type +{ + NONE, + STRING, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + TABLE, + ARRAY, + TABLE_ARRAY +}; + +/// Type traits class to convert C++ types to enum member +template <class T> +struct base_type_traits; + +template <> +struct base_type_traits<std::string> +{ + static const base_type type = base_type::STRING; +}; + +template <> +struct base_type_traits<local_time> +{ + static const base_type type = base_type::LOCAL_TIME; +}; + +template <> +struct base_type_traits<local_date> +{ + static const base_type type = base_type::LOCAL_DATE; +}; + +template <> +struct base_type_traits<local_datetime> +{ + static const base_type type = base_type::LOCAL_DATETIME; +}; + +template <> +struct base_type_traits<offset_datetime> +{ + static const base_type type = base_type::OFFSET_DATETIME; +}; + +template <> +struct base_type_traits<int64_t> +{ + static const base_type type = base_type::INT; +}; + +template <> +struct base_type_traits<double> +{ + static const base_type type = base_type::FLOAT; +}; + +template <> +struct base_type_traits<bool> +{ + static const base_type type = base_type::BOOL; +}; + +template <> +struct base_type_traits<table> +{ + static const base_type type = base_type::TABLE; +}; + +template <> +struct base_type_traits<array> +{ + static const base_type type = base_type::ARRAY; +}; + +template <> +struct base_type_traits<table_array> +{ + static const base_type type = base_type::TABLE_ARRAY; +}; +#endif + +/** + * A generic base TOML value used for type erasure. + */ +class base : public std::enable_shared_from_this<base> +{ + public: + virtual ~base() = default; + + virtual std::shared_ptr<base> clone() const = 0; + + /** + * Determines if the given TOML element is a value. + */ + virtual bool is_value() const + { + return false; + } + + /** + * Determines if the given TOML element is a table. + */ + virtual bool is_table() const + { + return false; + } + + /** + * Converts the TOML element into a table. + */ + std::shared_ptr<table> as_table() + { + if (is_table()) + return std::static_pointer_cast<table>(shared_from_this()); + return nullptr; + } + /** + * Determines if the TOML element is an array of "leaf" elements. + */ + virtual bool is_array() const + { + return false; + } + + /** + * Converts the TOML element to an array. + */ + std::shared_ptr<array> as_array() + { + if (is_array()) + return std::static_pointer_cast<array>(shared_from_this()); + return nullptr; + } + + /** + * Determines if the given TOML element is an array of tables. + */ + virtual bool is_table_array() const + { + return false; + } + + /** + * Converts the TOML element into a table array. + */ + std::shared_ptr<table_array> as_table_array() + { + if (is_table_array()) + return std::static_pointer_cast<table_array>(shared_from_this()); + return nullptr; + } + + /** + * Attempts to coerce the TOML element into a concrete TOML value + * of type T. + */ + template <class T> + std::shared_ptr<value<T>> as(); + + template <class T> + std::shared_ptr<const value<T>> as() const; + + template <class Visitor, class... Args> + void accept(Visitor&& visitor, Args&&... args) const; + +#if defined(CPPTOML_NO_RTTI) + base_type type() const + { + return type_; + } + + protected: + base(const base_type t) : type_(t) + { + // nothing + } + + private: + const base_type type_ = base_type::NONE; + +#else + protected: + base() + { + // nothing + } +#endif +}; + +/** + * A concrete TOML value representing the "leaves" of the "tree". + */ +template <class T> +class value : public base +{ + struct make_shared_enabler + { + // nothing; this is a private key accessible only to friends + }; + + template <class U> + friend std::shared_ptr<typename value_traits<U>::type> + cpptoml::make_value(U&& val); + + public: + static_assert(valid_value<T>::value, "invalid value type"); + + std::shared_ptr<base> clone() const override; + + value(const make_shared_enabler&, const T& val) : value(val) + { + // nothing; note that users cannot actually invoke this function + // because they lack access to the make_shared_enabler. + } + + bool is_value() const override + { + return true; + } + + /** + * Gets the data associated with this value. + */ + T& get() + { + return data_; + } + + /** + * Gets the data associated with this value. Const version. + */ + const T& get() const + { + return data_; + } + + private: + T data_; + + /** + * Constructs a value from the given data. + */ +#if defined(CPPTOML_NO_RTTI) + value(const T& val) : base(base_type_traits<T>::type), data_(val) + { + } +#else + value(const T& val) : data_(val) + { + } +#endif + + value(const value& val) = delete; + value& operator=(const value& val) = delete; +}; + +template <class T> +std::shared_ptr<typename value_traits<T>::type> make_value(T&& val) +{ + using value_type = typename value_traits<T>::type; + using enabler = typename value_type::make_shared_enabler; + return std::make_shared<value_type>( + enabler{}, value_traits<T>::construct(std::forward<T>(val))); +} + +template <class T> +inline std::shared_ptr<value<T>> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits<T>::type) + return std::static_pointer_cast<value<T>>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast<value<T>>(shared_from_this()); +#endif +} + +// special case value<double> to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr<value<double>> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast<value<double>>(shared_from_this()); + + if (type() == base_type::INT) + { + auto v = std::static_pointer_cast<value<int64_t>>(shared_from_this()); + return make_value<double>(static_cast<double>(v->get())); + } +#else + if (auto v = std::dynamic_pointer_cast<value<double>>(shared_from_this())) + return v; + + if (auto v = std::dynamic_pointer_cast<value<int64_t>>(shared_from_this())) + return make_value<double>(static_cast<double>(v->get())); +#endif + + return nullptr; +} + +template <class T> +inline std::shared_ptr<const value<T>> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits<T>::type) + return std::static_pointer_cast<const value<T>>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast<const value<T>>(shared_from_this()); +#endif +} + +// special case value<double> to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr<const value<double>> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast<const value<double>>( + shared_from_this()); + + if (type() == base_type::INT) + { + auto v = as<int64_t>(); + // the below has to be a non-const value<double> due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value<double>(static_cast<double>(v->get())); + } +#else + if (auto v + = std::dynamic_pointer_cast<const value<double>>(shared_from_this())) + return v; + + if (auto v = as<int64_t>()) + { + // the below has to be a non-const value<double> due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value<double>(static_cast<double>(v->get())); + } +#endif + + return nullptr; +} + +/** + * Exception class for array insertion errors. + */ +class array_exception : public std::runtime_error +{ + public: + array_exception(const std::string& err) : std::runtime_error{err} + { + } +}; + +class array : public base +{ + public: + friend std::shared_ptr<array> make_array(); + + std::shared_ptr<base> clone() const override; + + virtual bool is_array() const override + { + return true; + } + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector<std::shared_ptr<base>>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector<std::shared_ptr<base>>::const_iterator; + + iterator begin() + { + return values_.begin(); + } + + const_iterator begin() const + { + return values_.begin(); + } + + iterator end() + { + return values_.end(); + } + + const_iterator end() const + { + return values_.end(); + } + + /** + * Obtains the array (vector) of base values. + */ + std::vector<std::shared_ptr<base>>& get() + { + return values_; + } + + /** + * Obtains the array (vector) of base values. Const version. + */ + const std::vector<std::shared_ptr<base>>& get() const + { + return values_; + } + + std::shared_ptr<base> at(size_t idx) const + { + return values_.at(idx); + } + + /** + * Obtains an array of value<T>s. Note that elements may be + * nullptr if they cannot be converted to a value<T>. + */ + template <class T> + std::vector<std::shared_ptr<value<T>>> array_of() const + { + std::vector<std::shared_ptr<value<T>>> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr<base> v) { return v->as<T>(); }); + + return result; + } + + /** + * Obtains a option<std::vector<T>>. The option will be empty if the array + * contains values that are not of type T. + */ + template <class T> + inline typename array_of_trait<T>::return_type get_array_of() const + { + std::vector<T> result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as<T>()) + result.push_back(v->get()); + else + return {}; + } + + return {std::move(result)}; + } + + /** + * Obtains an array of arrays. Note that elements may be nullptr + * if they cannot be converted to a array. + */ + std::vector<std::shared_ptr<array>> nested_array() const + { + std::vector<std::shared_ptr<array>> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr<base> v) -> std::shared_ptr<array> { + if (v->is_array()) + return std::static_pointer_cast<array>(v); + return std::shared_ptr<array>{}; + }); + + return result; + } + + /** + * Add a value to the end of the array + */ + template <class T> + void push_back(const std::shared_ptr<value<T>>& val) + { + if (values_.empty() || values_[0]->as<T>()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Add an array to the end of the array + */ + void push_back(const std::shared_ptr<array>& val) + { + if (values_.empty() || values_[0]->is_array()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for adding a simple element to the end + * of the array. + */ + template <class T> + void push_back(T&& val, typename value_traits<T>::type* = 0) + { + push_back(make_value(std::forward<T>(val))); + } + + /** + * Insert a value into the array + */ + template <class T> + iterator insert(iterator position, const std::shared_ptr<value<T>>& value) + { + if (values_.empty() || values_[0]->as<T>()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Insert an array into the array + */ + iterator insert(iterator position, const std::shared_ptr<array>& value) + { + if (values_.empty() || values_[0]->is_array()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for inserting a simple element in the array + */ + template <class T> + iterator insert(iterator position, T&& val, + typename value_traits<T>::type* = 0) + { + return insert(position, make_value(std::forward<T>(val))); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return values_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + values_.clear(); + } + + /** + * Reserve space for n values. + */ + void reserve(size_type n) + { + values_.reserve(n); + } + + private: +#if defined(CPPTOML_NO_RTTI) + array() : base(base_type::ARRAY) + { + // empty + } +#else + array() = default; +#endif + + template <class InputIterator> + array(InputIterator begin, InputIterator end) : values_{begin, end} + { + // nothing + } + + array(const array& obj) = delete; + array& operator=(const array& obj) = delete; + + std::vector<std::shared_ptr<base>> values_; +}; + +inline std::shared_ptr<array> make_array() +{ + struct make_shared_enabler : public array + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared<make_shared_enabler>(); +} + +namespace detail +{ +template <> +inline std::shared_ptr<array> make_element<array>() +{ + return make_array(); +} +} // namespace detail + +/** + * Obtains a option<std::vector<T>>. The option will be empty if the array + * contains values that are not of type T. + */ +template <> +inline typename array_of_trait<array>::return_type +array::get_array_of<array>() const +{ + std::vector<std::shared_ptr<array>> result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as_array()) + result.push_back(v); + else + return {}; + } + + return {std::move(result)}; +} + +class table; + +class table_array : public base +{ + friend class table; + friend std::shared_ptr<table_array> make_table_array(bool); + + public: + std::shared_ptr<base> clone() const override; + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector<std::shared_ptr<table>>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector<std::shared_ptr<table>>::const_iterator; + + iterator begin() + { + return array_.begin(); + } + + const_iterator begin() const + { + return array_.begin(); + } + + iterator end() + { + return array_.end(); + } + + const_iterator end() const + { + return array_.end(); + } + + virtual bool is_table_array() const override + { + return true; + } + + std::vector<std::shared_ptr<table>>& get() + { + return array_; + } + + const std::vector<std::shared_ptr<table>>& get() const + { + return array_; + } + + /** + * Add a table to the end of the array + */ + void push_back(const std::shared_ptr<table>& val) + { + array_.push_back(val); + } + + /** + * Insert a table into the array + */ + iterator insert(iterator position, const std::shared_ptr<table>& value) + { + return array_.insert(position, value); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return array_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + array_.clear(); + } + + /** + * Reserve space for n tables. + */ + void reserve(size_type n) + { + array_.reserve(n); + } + + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + + private: +#if defined(CPPTOML_NO_RTTI) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) + { + // nothing + } +#else + table_array(bool is_inline = false) : is_inline_(is_inline) + { + // nothing + } +#endif + + table_array(const table_array& obj) = delete; + table_array& operator=(const table_array& rhs) = delete; + + std::vector<std::shared_ptr<table>> array_; + const bool is_inline_ = false; +}; + +inline std::shared_ptr<table_array> make_table_array(bool is_inline) +{ + struct make_shared_enabler : public table_array + { + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) + { + // nothing + } + }; + + return std::make_shared<make_shared_enabler>(is_inline); +} + +namespace detail +{ +template <> +inline std::shared_ptr<table_array> make_element<table_array>() +{ + return make_table_array(true); +} +} // namespace detail + +// The below are overloads for fetching specific value types out of a value +// where special casting behavior (like bounds checking) is desired + +template <class T> +typename std::enable_if<!std::is_floating_point<T>::value + && std::is_signed<T>::value, + option<T>>::type +get_impl(const std::shared_ptr<base>& elem) +{ + if (auto v = elem->as<int64_t>()) + { + if (v->get() < (std::numeric_limits<T>::min)()) + throw std::underflow_error{ + "T cannot represent the value requested in get"}; + + if (v->get() > (std::numeric_limits<T>::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast<T>(v->get())}; + } + else + { + return {}; + } +} + +template <class T> +typename std::enable_if<!std::is_same<T, bool>::value + && std::is_unsigned<T>::value, + option<T>>::type +get_impl(const std::shared_ptr<base>& elem) +{ + if (auto v = elem->as<int64_t>()) + { + if (v->get() < 0) + throw std::underflow_error{"T cannot store negative value in get"}; + + if (static_cast<uint64_t>(v->get()) > (std::numeric_limits<T>::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast<T>(v->get())}; + } + else + { + return {}; + } +} + +template <class T> +typename std::enable_if<!std::is_integral<T>::value + || std::is_same<T, bool>::value, + option<T>>::type +get_impl(const std::shared_ptr<base>& elem) +{ + if (auto v = elem->as<T>()) + { + return {v->get()}; + } + else + { + return {}; + } +} + +/** + * Represents a TOML keytable. + */ +class table : public base +{ + public: + friend class table_array; + friend std::shared_ptr<table> make_table(); + + std::shared_ptr<base> clone() const override; + + /** + * tables can be iterated over. + */ + using iterator = string_to_base_map::iterator; + + /** + * tables can be iterated over. Const version. + */ + using const_iterator = string_to_base_map::const_iterator; + + iterator begin() + { + return map_.begin(); + } + + const_iterator begin() const + { + return map_.begin(); + } + + iterator end() + { + return map_.end(); + } + + const_iterator end() const + { + return map_.end(); + } + + bool is_table() const override + { + return true; + } + + bool empty() const + { + return map_.empty(); + } + + /** + * Determines if this key table contains the given key. + */ + bool contains(const std::string& key) const + { + return map_.find(key) != map_.end(); + } + + /** + * Determines if this key table contains the given key. Will + * resolve "qualified keys". Qualified keys are the full access + * path separated with dots like "grandparent.parent.child". + */ + bool contains_qualified(const std::string& key) const + { + return resolve_qualified(key); + } + + /** + * Obtains the base for a given key. + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr<base> get(const std::string& key) const + { + return map_.at(key); + } + + /** + * Obtains the base for a given key. Will resolve "qualified + * keys". Qualified keys are the full access path separated with + * dots like "grandparent.parent.child". + * + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr<base> get_qualified(const std::string& key) const + { + std::shared_ptr<base> p; + resolve_qualified(key, &p); + return p; + } + + /** + * Obtains a table for a given key, if possible. + */ + std::shared_ptr<table> get_table(const std::string& key) const + { + if (contains(key) && get(key)->is_table()) + return std::static_pointer_cast<table>(get(key)); + return nullptr; + } + + /** + * Obtains a table for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr<table> get_table_qualified(const std::string& key) const + { + if (contains_qualified(key) && get_qualified(key)->is_table()) + return std::static_pointer_cast<table>(get_qualified(key)); + return nullptr; + } + + /** + * Obtains an array for a given key. + */ + std::shared_ptr<array> get_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_array(); + } + + /** + * Obtains an array for a given key. Will resolve "qualified keys". + */ + std::shared_ptr<array> get_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_array(); + } + + /** + * Obtains a table_array for a given key, if possible. + */ + std::shared_ptr<table_array> get_table_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_table_array(); + } + + /** + * Obtains a table_array for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr<table_array> + get_table_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_table_array(); + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. + */ + template <class T> + option<T> get_as(const std::string& key) const + { + try + { + return get_impl<T>(get(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. Will resolve "qualified + * keys". + */ + template <class T> + option<T> get_qualified_as(const std::string& key) const + { + try + { + return get_impl<T>(get_qualified(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template <class T> + inline typename array_of_trait<T>::return_type + get_array_of(const std::string& key) const + { + if (auto v = get_array(key)) + { + std::vector<T> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as<T>()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. Will + * resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template <class T> + inline typename array_of_trait<T>::return_type + get_qualified_array_of(const std::string& key) const + { + if (auto v = get_array_qualified(key)) + { + std::vector<T> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as<T>()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Adds an element to the keytable. + */ + void insert(const std::string& key, const std::shared_ptr<base>& value) + { + map_[key] = value; + } + + /** + * Convenience shorthand for adding a simple element to the + * keytable. + */ + template <class T> + void insert(const std::string& key, T&& val, + typename value_traits<T>::type* = 0) + { + insert(key, make_value(std::forward<T>(val))); + } + + /** + * Removes an element from the table. + */ + void erase(const std::string& key) + { + map_.erase(key); + } + + private: +#if defined(CPPTOML_NO_RTTI) + table() : base(base_type::TABLE) + { + // nothing + } +#else + table() + { + // nothing + } +#endif + + table(const table& obj) = delete; + table& operator=(const table& rhs) = delete; + + std::vector<std::string> split(const std::string& value, + char separator) const + { + std::vector<std::string> result; + std::string::size_type p = 0; + std::string::size_type q; + while ((q = value.find(separator, p)) != std::string::npos) + { + result.emplace_back(value, p, q - p); + p = q + 1; + } + result.emplace_back(value, p); + return result; + } + + // If output parameter p is specified, fill it with the pointer to the + // specified entry and throw std::out_of_range if it couldn't be found. + // + // Otherwise, just return true if the entry could be found or false + // otherwise and do not throw. + bool resolve_qualified(const std::string& key, + std::shared_ptr<base>* p = nullptr) const + { + auto parts = split(key, '.'); + auto last_key = parts.back(); + parts.pop_back(); + + auto cur_table = this; + for (const auto& part : parts) + { + cur_table = cur_table->get_table(part).get(); + if (!cur_table) + { + if (!p) + return false; + + throw std::out_of_range{key + " is not a valid key"}; + } + } + + if (!p) + return cur_table->map_.count(last_key) != 0; + + *p = cur_table->map_.at(last_key); + return true; + } + + string_to_base_map map_; +}; + +/** + * Helper function that attempts to get an array of arrays for a given + * key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait<array>::return_type +table::get_array_of<array>(const std::string& key) const +{ + if (auto v = get_array(key)) + { + std::vector<std::shared_ptr<array>> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +/** + * Helper function that attempts to get an array of arrays for a given + * key. Will resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait<array>::return_type +table::get_qualified_array_of<array>(const std::string& key) const +{ + if (auto v = get_array_qualified(key)) + { + std::vector<std::shared_ptr<array>> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +std::shared_ptr<table> make_table() +{ + struct make_shared_enabler : public table + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared<make_shared_enabler>(); +} + +namespace detail +{ +template <> +inline std::shared_ptr<table> make_element<table>() +{ + return make_table(); +} +} // namespace detail + +template <class T> +std::shared_ptr<base> value<T>::clone() const +{ + return make_value(data_); +} + +inline std::shared_ptr<base> array::clone() const +{ + auto result = make_array(); + result->reserve(values_.size()); + for (const auto& ptr : values_) + result->values_.push_back(ptr->clone()); + return result; +} + +inline std::shared_ptr<base> table_array::clone() const +{ + auto result = make_table_array(is_inline()); + result->reserve(array_.size()); + for (const auto& ptr : array_) + result->array_.push_back(ptr->clone()->as_table()); + return result; +} + +inline std::shared_ptr<base> table::clone() const +{ + auto result = make_table(); + for (const auto& pr : map_) + result->insert(pr.first, pr.second->clone()); + return result; +} + +/** + * Exception class for all TOML parsing errors. + */ +class parse_exception : public std::runtime_error +{ + public: + parse_exception(const std::string& err) : std::runtime_error{err} + { + } + + parse_exception(const std::string& err, std::size_t line_number) + : std::runtime_error{err + " at line " + std::to_string(line_number)} + { + } +}; + +inline bool is_number(char c) +{ + return c >= '0' && c <= '9'; +} + +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +/** + * Helper object for consuming expected characters. + */ +template <class OnError> +class consumer +{ + public: + consumer(std::string::iterator& it, const std::string::iterator& end, + OnError&& on_error) + : it_(it), end_(end), on_error_(std::forward<OnError>(on_error)) + { + // nothing + } + + void operator()(char c) + { + if (it_ == end_ || *it_ != c) + on_error_(); + ++it_; + } + + template <std::size_t N> + void operator()(const char (&str)[N]) + { + std::for_each(std::begin(str), std::end(str) - 1, + [&](char c) { (*this)(c); }); + } + + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + + int eat_digits(int len) + { + int val = 0; + for (int i = 0; i < len; ++i) + { + if (!is_number(*it_) || it_ == end_) + on_error_(); + val = 10 * val + (*it_++ - '0'); + } + return val; + } + + void error() + { + on_error_(); + } + + private: + std::string::iterator& it_; + const std::string::iterator& end_; + OnError on_error_; +}; + +template <class OnError> +consumer<OnError> make_consumer(std::string::iterator& it, + const std::string::iterator& end, + OnError&& on_error) +{ + return consumer<OnError>(it, end, std::forward<OnError>(on_error)); +} + +// replacement for std::getline to handle incorrectly line-ended files +// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf +namespace detail +{ +inline std::istream& getline(std::istream& input, std::string& line) +{ + line.clear(); + + std::istream::sentry sentry{input, true}; + auto sb = input.rdbuf(); + + while (true) + { + auto c = sb->sbumpc(); + if (c == '\r') + { + if (sb->sgetc() == '\n') + c = sb->sbumpc(); + } + + if (c == '\n') + return input; + + if (c == std::istream::traits_type::eof()) + { + if (line.empty()) + input.setstate(std::ios::eofbit); + return input; + } + + line.push_back(static_cast<char>(c)); + } +} +} // namespace detail + +/** + * The parser class. + */ +class parser +{ + public: + /** + * Parsers are constructed from streams. + */ + parser(std::istream& stream) : input_(stream) + { + // nothing + } + + parser& operator=(const parser& parser) = delete; + + /** + * Parses the stream this parser was created on until EOF. + * @throw parse_exception if there are errors in parsing + */ + std::shared_ptr<table> parse() + { + std::shared_ptr<table> root = make_table(); + + table* curr_table = root.get(); + + while (detail::getline(input_, line_)) + { + line_number_++; + auto it = line_.begin(); + auto end = line_.end(); + consume_whitespace(it, end); + if (it == end || *it == '#') + continue; + if (*it == '[') + { + curr_table = root.get(); + parse_table(it, end, curr_table); + } + else + { + parse_key_value(it, end, curr_table); + consume_whitespace(it, end); + eol_or_comment(it, end); + } + } + return root; + } + + private: +#if defined _MSC_VER + __declspec(noreturn) +#elif defined __GNUC__ + __attribute__((noreturn)) +#endif + void throw_parse_exception(const std::string& err) + { + throw parse_exception{err, line_number_}; + } + + void parse_table(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + // remove the beginning keytable marker + ++it; + if (it == end) + throw_parse_exception("Unexpected end of table"); + if (*it == '[') + parse_table_array(it, end, curr_table); + else + parse_single_table(it, end, curr_table); + } + + void parse_single_table(std::string::iterator& it, + const std::string::iterator& end, + table*& curr_table) + { + if (it == end || *it == ']') + throw_parse_exception("Table name cannot be empty"); + + std::string full_table_name; + bool inserted = false; + + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { + if (part.empty()) + throw_parse_exception("Empty component of table name"); + + if (!full_table_name.empty()) + full_table_name += '.'; + full_table_name += part; + + if (curr_table->contains(part)) + { +#if !defined(__PGI) + auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif + if (b->is_table()) + curr_table = static_cast<table*>(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast<table_array>(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_table_name + + "already exists as a value"); + } + else + { + inserted = true; + curr_table->insert(part, make_table()); + curr_table = static_cast<table*>(curr_table->get(part).get()); + } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); + + if (it == end) + throw_parse_exception( + "Unterminated table declaration; did you forget a ']'?"); + + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + // table already existed + if (!inserted) + { + auto is_value + = [](const std::pair<const std::string&, + const std::shared_ptr<base>&>& p) { + return p.second->is_value(); + }; + + // if there are any values, we can't add values to this table + // since it has already been defined. If there aren't any + // values, then it was implicitly created by something like + // [a.b] + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) + { + throw_parse_exception("Redefinition of table " + + full_table_name); + } + } + + ++it; + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_table_array(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + ++it; + if (it == end || *it == ']') + throw_parse_exception("Table array name cannot be empty"); + + auto key_end = [](char c) { return c == ']'; }; + + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { + if (part.empty()) + throw_parse_exception("Empty component of table array name"); + + if (!full_ta_name.empty()) + full_ta_name += '.'; + full_ta_name += part; + + if (curr_table->contains(part)) + { +#if !defined(__PGI) + auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr<base> b = curr_table->get(part); +#endif + + // if this is the end of the table array name, add an + // element to the table array that we just looked up, + // provided it was not declared inline + if (it != end && *it == ']') + { + if (!b->is_table_array()) + { + throw_parse_exception("Key " + full_ta_name + + " is not a table array"); + } + + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + + v->get().push_back(make_table()); + curr_table = v->get().back().get(); + } + // otherwise, just keep traversing down the key name + else + { + if (b->is_table()) + curr_table = static_cast<table*>(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast<table_array>(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_ta_name + + " already exists as a value"); + } + } + else + { + // if this is the end of the table array name, add a new + // table array and a new table inside that array for us to + // add keys to next + if (it != end && *it == ']') + { + curr_table->insert(part, make_table_array()); + auto arr = std::static_pointer_cast<table_array>( + curr_table->get(part)); + arr->get().push_back(make_table()); + curr_table = arr->get().back().get(); + } + // otherwise, create the implicitly defined table and move + // down to it + else + { + curr_table->insert(part, make_table()); + curr_table + = static_cast<table*>(curr_table->get(part).get()); + } + } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); + + // consume the last "]]" + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Unterminated table array name"); + }); + eat(']'); + eat(']'); + + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_key_value(std::string::iterator& it, std::string::iterator& end, + table* curr_table) + { + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast<table*>(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + + if (curr_table->contains(key)) + throw_parse_exception("Key " + key + " already present"); + if (it == end || *it != '=') + throw_parse_exception("Value must follow after a '='"); + ++it; + consume_whitespace(it, end); + curr_table->insert(key, parse_value(it, end)); + consume_whitespace(it, end); + } + + template <class KeyEndFinder, class KeyPartHandler> + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) + { + consume_whitespace(it, end); + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') + { + return string_literal(it, end, *it); + } + else + { + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); + return parse_bare_key(it, bke); + } + } + + std::string parse_bare_key(std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Bare key missing name"); + } + + auto key_end = end; + --key_end; + consume_backwards_whitespace(key_end, it); + ++key_end; + std::string key{it, key_end}; + + if (std::find(it, key_end, '#') != key_end) + { + throw_parse_exception("Bare key " + key + " cannot contain #"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == ' ' || c == '\t'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain whitespace"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == '[' || c == ']'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain '[' or ']'"); + } + + it = end; + return key; + } + + enum class parse_type + { + STRING = 1, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + ARRAY, + INLINE_TABLE + }; + + std::shared_ptr<base> parse_value(std::string::iterator& it, + std::string::iterator& end) + { + parse_type type = determine_value_type(it, end); + switch (type) + { + case parse_type::STRING: + return parse_string(it, end); + case parse_type::LOCAL_TIME: + return parse_time(it, end); + case parse_type::LOCAL_DATE: + case parse_type::LOCAL_DATETIME: + case parse_type::OFFSET_DATETIME: + return parse_date(it, end); + case parse_type::INT: + case parse_type::FLOAT: + return parse_number(it, end); + case parse_type::BOOL: + return parse_bool(it, end); + case parse_type::ARRAY: + return parse_array(it, end); + case parse_type::INLINE_TABLE: + return parse_inline_table(it, end); + default: + throw_parse_exception("Failed to parse value"); + } + } + + parse_type determine_value_type(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Failed to parse value type"); + } + if (*it == '"' || *it == '\'') + { + return parse_type::STRING; + } + else if (is_time(it, end)) + { + return parse_type::LOCAL_TIME; + } + else if (auto dtype = date_type(it, end)) + { + return *dtype; + } + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) + { + return determine_number_type(it, end); + } + else if (*it == 't' || *it == 'f') + { + return parse_type::BOOL; + } + else if (*it == '[') + { + return parse_type::ARRAY; + } + else if (*it == '{') + { + return parse_type::INLINE_TABLE; + } + throw_parse_exception("Failed to parse value type"); + } + + parse_type determine_number_type(const std::string::iterator& it, + const std::string::iterator& end) + { + // determine if we are an integer or a float + auto check_it = it; + if (*check_it == '-' || *check_it == '+') + ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + + while (check_it != end && is_number(*check_it)) + ++check_it; + if (check_it != end && *check_it == '.') + { + ++check_it; + while (check_it != end && is_number(*check_it)) + ++check_it; + return parse_type::FLOAT; + } + else + { + return parse_type::INT; + } + } + + std::shared_ptr<value<std::string>> parse_string(std::string::iterator& it, + std::string::iterator& end) + { + auto delim = *it; + assert(delim == '"' || delim == '\''); + + // end is non-const here because we have to be able to potentially + // parse multiple lines in a string, not just one + auto check_it = it; + ++check_it; + if (check_it != end && *check_it == delim) + { + ++check_it; + if (check_it != end && *check_it == delim) + { + it = ++check_it; + return parse_multiline_string(it, end, delim); + } + } + return make_value<std::string>(string_literal(it, end, delim)); + } + + std::shared_ptr<value<std::string>> + parse_multiline_string(std::string::iterator& it, + std::string::iterator& end, char delim) + { + std::stringstream ss; + + auto is_ws = [](char c) { return c == ' ' || c == '\t'; }; + + bool consuming = false; + std::shared_ptr<value<std::string>> ret; + + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value<std::string>(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; + + // handle the remainder of the current line + handle_line(it, end); + if (ret) + return ret; + + // start eating lines + while (detail::getline(input_, line_)) + { + ++line_number_; + + it = line_.begin(); + end = line_.end(); + + handle_line(it, end); + + if (ret) + return ret; + + if (!consuming) + ss << std::endl; + } + + throw_parse_exception("Unterminated multi-line basic string"); + } + + std::string string_literal(std::string::iterator& it, + const std::string::iterator& end, char delim) + { + ++it; + std::string val; + while (it != end) + { + // handle escaped characters + if (delim == '"' && *it == '\\') + { + val += parse_escape_code(it, end); + } + else if (*it == delim) + { + ++it; + consume_whitespace(it, end); + return val; + } + else + { + val += *it++; + } + } + throw_parse_exception("Unterminated string literal"); + } + + std::string parse_escape_code(std::string::iterator& it, + const std::string::iterator& end) + { + ++it; + if (it == end) + throw_parse_exception("Invalid escape sequence"); + char value; + if (*it == 'b') + { + value = '\b'; + } + else if (*it == 't') + { + value = '\t'; + } + else if (*it == 'n') + { + value = '\n'; + } + else if (*it == 'f') + { + value = '\f'; + } + else if (*it == 'r') + { + value = '\r'; + } + else if (*it == '"') + { + value = '"'; + } + else if (*it == '\\') + { + value = '\\'; + } + else if (*it == 'u' || *it == 'U') + { + return parse_unicode(it, end); + } + else + { + throw_parse_exception("Invalid escape sequence"); + } + ++it; + return std::string(1, value); + } + + std::string parse_unicode(std::string::iterator& it, + const std::string::iterator& end) + { + bool large = *it++ == 'U'; + auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000); + + if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) + { + throw_parse_exception( + "Unicode escape sequence is not a Unicode scalar value"); + } + + std::string result; + // See Table 3-6 of the Unicode standard + if (codepoint <= 0x7f) + { + // 1-byte codepoints: 00000000 0xxxxxxx + // repr: 0xxxxxxx + result += static_cast<char>(codepoint & 0x7f); + } + else if (codepoint <= 0x7ff) + { + // 2-byte codepoints: 00000yyy yyxxxxxx + // repr: 110yyyyy 10xxxxxx + // + // 0x1f = 00011111 + // 0xc0 = 11000000 + // + result += static_cast<char>(0xc0 | ((codepoint >> 6) & 0x1f)); + // + // 0x80 = 10000000 + // 0x3f = 00111111 + // + result += static_cast<char>(0x80 | (codepoint & 0x3f)); + } + else if (codepoint <= 0xffff) + { + // 3-byte codepoints: zzzzyyyy yyxxxxxx + // repr: 1110zzzz 10yyyyyy 10xxxxxx + // + // 0xe0 = 11100000 + // 0x0f = 00001111 + // + result += static_cast<char>(0xe0 | ((codepoint >> 12) & 0x0f)); + result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x1f)); + result += static_cast<char>(0x80 | (codepoint & 0x3f)); + } + else + { + // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx + // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx + // + // 0xf0 = 11110000 + // 0x07 = 00000111 + // + result += static_cast<char>(0xf0 | ((codepoint >> 18) & 0x07)); + result += static_cast<char>(0x80 | ((codepoint >> 12) & 0x3f)); + result += static_cast<char>(0x80 | ((codepoint >> 6) & 0x3f)); + result += static_cast<char>(0x80 | (codepoint & 0x3f)); + } + return result; + } + + uint32_t parse_hex(std::string::iterator& it, + const std::string::iterator& end, uint32_t place) + { + uint32_t value = 0; + while (place > 0) + { + if (it == end) + throw_parse_exception("Unexpected end of unicode sequence"); + + if (!is_hex(*it)) + throw_parse_exception("Invalid unicode escape sequence"); + + value += place * hex_to_digit(*it++); + place /= 16; + } + return value; + } + + uint32_t hex_to_digit(char c) + { + if (is_number(c)) + return static_cast<uint32_t>(c - '0'); + return 10 + + static_cast<uint32_t>(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + } + + std::shared_ptr<base> parse_number(std::string::iterator& it, + const std::string::iterator& end) + { + auto check_it = it; + auto check_end = find_end_of_number(it, end); + + auto eat_sign = [&]() { + if (check_it != end && (*check_it == '-' || *check_it == '+')) + ++check_it; + }; + + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; + + auto eat_digits = [&](bool (*check_char)(char)) { + auto beg = check_it; + while (check_it != end && check_char(*check_it)) + { + ++check_it; + if (check_it != end && *check_it == '_') + { + ++check_it; + if (check_it == end || !check_char(*check_it)) + throw_parse_exception("Malformed number"); + } + } + + if (check_it == beg) + throw_parse_exception("Malformed number"); + }; + + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') + { + eat_hex(); + return parse_int(it, check_it, 16); + } + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + + eat_sign(); + check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits<double>::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits<double>::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + + eat_numbers(); + + if (check_it != end + && (*check_it == '.' || *check_it == 'e' || *check_it == 'E')) + { + bool is_exp = *check_it == 'e' || *check_it == 'E'; + + ++check_it; + if (check_it == end) + throw_parse_exception("Floats must have trailing digits"); + + auto eat_exp = [&]() { + eat_sign(); + check_no_leading_zero(); + eat_numbers(); + }; + + if (is_exp) + eat_exp(); + else + eat_numbers(); + + if (!is_exp && check_it != end + && (*check_it == 'e' || *check_it == 'E')) + { + ++check_it; + eat_exp(); + } + + return parse_float(it, check_it); + } + else + { + return parse_int(it, check_it); + } + } + + std::shared_ptr<value<int64_t>> parse_int(std::string::iterator& it, + const std::string::iterator& end, + int base = 10, + const char* prefix = "") + { + std::string v{it, end}; + v = prefix + v; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + try + { + return make_value<int64_t>(std::stoll(v, nullptr, base)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr<value<double>> parse_float(std::string::iterator& it, + const std::string::iterator& end) + { + std::string v{it, end}; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + char decimal_point = std::localeconv()->decimal_point[0]; + std::replace(v.begin(), v.end(), '.', decimal_point); + try + { + return make_value<double>(std::stod(v)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr<value<bool>> parse_bool(std::string::iterator& it, + const std::string::iterator& end) + { + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Attempted to parse invalid boolean value"); + }); + + if (*it == 't') + { + eat("true"); + return make_value<bool>(true); + } + else if (*it == 'f') + { + eat("false"); + return make_value<bool>(false); + } + + eat.error(); + return nullptr; + } + + std::string::iterator find_end_of_number(std::string::iterator it, + std::string::iterator end) + { + auto ret = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; + }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; + } + + std::string::iterator find_end_of_date(std::string::iterator it, + std::string::iterator end) + { + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; + }); + } + + std::string::iterator find_end_of_time(std::string::iterator it, + std::string::iterator end) + { + return std::find_if(it, end, [](char c) { + return !is_number(c) && c != ':' && c != '.'; + }); + } + + local_time read_time(std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + + auto eat = make_consumer( + it, time_end, [&]() { throw_parse_exception("Malformed time"); }); + + local_time ltime; + + ltime.hour = eat.eat_digits(2); + eat(':'); + ltime.minute = eat.eat_digits(2); + eat(':'); + ltime.second = eat.eat_digits(2); + + int power = 100000; + if (it != time_end && *it == '.') + { + ++it; + while (it != time_end && is_number(*it)) + { + ltime.microsecond += power * (*it++ - '0'); + power /= 10; + } + } + + if (it != time_end) + throw_parse_exception("Malformed time"); + + return ltime; + } + + std::shared_ptr<value<local_time>> + parse_time(std::string::iterator& it, const std::string::iterator& end) + { + return make_value(read_time(it, end)); + } + + std::shared_ptr<base> parse_date(std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + + auto eat = make_consumer( + it, date_end, [&]() { throw_parse_exception("Malformed date"); }); + + local_date ldate; + ldate.year = eat.eat_digits(4); + eat('-'); + ldate.month = eat.eat_digits(2); + eat('-'); + ldate.day = eat.eat_digits(2); + + if (it == date_end) + return make_value(ldate); + + eat.eat_or('T', ' '); + + local_datetime ldt; + static_cast<local_date&>(ldt) = ldate; + static_cast<local_time&>(ldt) = read_time(it, date_end); + + if (it == date_end) + return make_value(ldt); + + offset_datetime dt; + static_cast<local_datetime&>(dt) = ldt; + + int hoff = 0; + int moff = 0; + if (*it == '+' || *it == '-') + { + auto plus = *it == '+'; + ++it; + + hoff = eat.eat_digits(2); + dt.hour_offset = (plus) ? hoff : -hoff; + eat(':'); + moff = eat.eat_digits(2); + dt.minute_offset = (plus) ? moff : -moff; + } + else if (*it == 'Z') + { + ++it; + } + + if (it != date_end) + throw_parse_exception("Malformed date"); + + return make_value(dt); + } + + std::shared_ptr<base> parse_array(std::string::iterator& it, + std::string::iterator& end) + { + // this gets ugly because of the "homogeneity" restriction: + // arrays can either be of only one type, or contain arrays + // (each of those arrays could be of different types, though) + // + // because of the latter portion, we don't really have a choice + // but to represent them as arrays of base values... + ++it; + + // ugh---have to read the first value to determine array type... + skip_whitespace_and_comments(it, end); + + // edge case---empty array + if (*it == ']') + { + ++it; + return make_array(); + } + + auto val_end = std::find_if( + it, end, [](char c) { return c == ',' || c == ']' || c == '#'; }); + parse_type type = determine_value_type(it, val_end); + switch (type) + { + case parse_type::STRING: + return parse_value_array<std::string>(it, end); + case parse_type::LOCAL_TIME: + return parse_value_array<local_time>(it, end); + case parse_type::LOCAL_DATE: + return parse_value_array<local_date>(it, end); + case parse_type::LOCAL_DATETIME: + return parse_value_array<local_datetime>(it, end); + case parse_type::OFFSET_DATETIME: + return parse_value_array<offset_datetime>(it, end); + case parse_type::INT: + return parse_value_array<int64_t>(it, end); + case parse_type::FLOAT: + return parse_value_array<double>(it, end); + case parse_type::BOOL: + return parse_value_array<bool>(it, end); + case parse_type::ARRAY: + return parse_object_array<array>(&parser::parse_array, '[', it, + end); + case parse_type::INLINE_TABLE: + return parse_object_array<table_array>( + &parser::parse_inline_table, '{', it, end); + default: + throw_parse_exception("Unable to parse array"); + } + } + + template <class Value> + std::shared_ptr<array> parse_value_array(std::string::iterator& it, + std::string::iterator& end) + { + auto arr = make_array(); + while (it != end && *it != ']') + { + auto val = parse_value(it, end); + if (auto v = val->as<Value>()) + arr->get().push_back(val); + else + throw_parse_exception("Arrays must be homogeneous"); + skip_whitespace_and_comments(it, end); + if (*it != ',') + break; + ++it; + skip_whitespace_and_comments(it, end); + } + if (it != end) + ++it; + return arr; + } + + template <class Object, class Function> + std::shared_ptr<Object> parse_object_array(Function&& fun, char delim, + std::string::iterator& it, + std::string::iterator& end) + { + auto arr = detail::make_element<Object>(); + + while (it != end && *it != ']') + { + if (*it != delim) + throw_parse_exception("Unexpected character in array"); + + arr->get().push_back(((*this).*fun)(it, end)); + skip_whitespace_and_comments(it, end); + + if (it == end || *it != ',') + break; + + ++it; + skip_whitespace_and_comments(it, end); + } + + if (it == end || *it != ']') + throw_parse_exception("Unterminated array"); + + ++it; + return arr; + } + + std::shared_ptr<table> parse_inline_table(std::string::iterator& it, + std::string::iterator& end) + { + auto tbl = make_table(); + do + { + ++it; + if (it == end) + throw_parse_exception("Unterminated inline table"); + + consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } + } while (*it == ','); + + if (it == end || *it != '}') + throw_parse_exception("Unterminated inline table"); + + ++it; + consume_whitespace(it, end); + + return tbl; + } + + void skip_whitespace_and_comments(std::string::iterator& start, + std::string::iterator& end) + { + consume_whitespace(start, end); + while (start == end || *start == '#') + { + if (!detail::getline(input_, line_)) + throw_parse_exception("Unclosed array"); + line_number_++; + start = line_.begin(); + end = line_.end(); + consume_whitespace(start, end); + } + } + + void consume_whitespace(std::string::iterator& it, + const std::string::iterator& end) + { + while (it != end && (*it == ' ' || *it == '\t')) + ++it; + } + + void consume_backwards_whitespace(std::string::iterator& back, + const std::string::iterator& front) + { + while (back != front && (*back == ' ' || *back == '\t')) + --back; + } + + void eol_or_comment(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it != end && *it != '#') + throw_parse_exception("Unidentified trailing character '" + + std::string{*it} + + "'---did you forget a '#'?"); + } + + bool is_time(const std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + auto len = std::distance(it, time_end); + + if (len < 8) + return false; + + if (it[2] != ':' || it[5] != ':') + return false; + + if (len > 8) + return it[8] == '.' && len > 9; + + return true; + } + + option<parse_type> date_type(const std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + auto len = std::distance(it, date_end); + + if (len < 10) + return {}; + + if (it[4] != '-' || it[7] != '-') + return {}; + + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) + { + // datetime type + auto time_end = find_end_of_time(it + 11, date_end); + if (time_end == date_end) + return {parse_type::LOCAL_DATETIME}; + else + return {parse_type::OFFSET_DATETIME}; + } + else if (len == 10) + { + // just a regular date + return {parse_type::LOCAL_DATE}; + } + + return {}; + } + + std::istream& input_; + std::string line_; + std::size_t line_number_ = 0; +}; + +/** + * Utility function to parse a file as a TOML file. Returns the root table. + * Throws a parse_exception if the file cannot be opened. + */ +inline std::shared_ptr<table> parse_file(const std::string& filename) +{ +#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP) + boost::nowide::ifstream file{filename.c_str()}; +#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP) + nowide::ifstream file{filename.c_str()}; +#else + std::ifstream file{filename}; +#endif + if (!file.is_open()) + throw parse_exception{filename + " could not be opened for parsing"}; + parser p{file}; + return p.parse(); +} + +template <class... Ts> +struct value_accept; + +template <> +struct value_accept<> +{ + template <class Visitor, class... Args> + static void accept(const base&, Visitor&&, Args&&...) + { + // nothing + } +}; + +template <class T, class... Ts> +struct value_accept<T, Ts...> +{ + template <class Visitor, class... Args> + static void accept(const base& b, Visitor&& visitor, Args&&... args) + { + if (auto v = b.as<T>()) + { + visitor.visit(*v, std::forward<Args>(args)...); + } + else + { + value_accept<Ts...>::accept(b, std::forward<Visitor>(visitor), + std::forward<Args>(args)...); + } + } +}; + +/** + * base implementation of accept() that calls visitor.visit() on the concrete + * class. + */ +template <class Visitor, class... Args> +void base::accept(Visitor&& visitor, Args&&... args) const +{ + if (is_value()) + { + using value_acceptor + = value_accept<std::string, int64_t, double, bool, local_date, + local_time, local_datetime, offset_datetime>; + value_acceptor::accept(*this, std::forward<Visitor>(visitor), + std::forward<Args>(args)...); + } + else if (is_table()) + { + visitor.visit(static_cast<const table&>(*this), + std::forward<Args>(args)...); + } + else if (is_array()) + { + visitor.visit(static_cast<const array&>(*this), + std::forward<Args>(args)...); + } + else if (is_table_array()) + { + visitor.visit(static_cast<const table_array&>(*this), + std::forward<Args>(args)...); + } +} + +/** + * Writer that can be passed to accept() functions of cpptoml objects and + * will output valid TOML to a stream. + */ +class toml_writer +{ + public: + /** + * Construct a toml_writer that will write to the given stream + */ + toml_writer(std::ostream& s, const std::string& indent_space = "\t") + : stream_(s), indent_(indent_space), has_naked_endline_(false) + { + // nothing + } + + public: + /** + * Output a base value of the TOML tree. + */ + template <class T> + void visit(const value<T>& v, bool = false) + { + write(v); + } + + /** + * Output a table element of the TOML tree + */ + void visit(const table& t, bool in_array = false) + { + write_table_header(in_array); + std::vector<std::string> values; + std::vector<std::string> tables; + + for (const auto& i : t) + { + if (i.second->is_table() || i.second->is_table_array()) + { + tables.push_back(i.first); + } + else + { + values.push_back(i.first); + } + } + + for (unsigned int i = 0; i < values.size(); ++i) + { + path_.push_back(values[i]); + + if (i > 0) + endline(); + + write_table_item_header(*t.get(values[i])); + t.get(values[i])->accept(*this, false); + path_.pop_back(); + } + + for (unsigned int i = 0; i < tables.size(); ++i) + { + path_.push_back(tables[i]); + + if (values.size() > 0 || i > 0) + endline(); + + write_table_item_header(*t.get(tables[i])); + t.get(tables[i])->accept(*this, false); + path_.pop_back(); + } + + endline(); + } + + /** + * Output an array element of the TOML tree + */ + void visit(const array& a, bool = false) + { + write("["); + + for (unsigned int i = 0; i < a.get().size(); ++i) + { + if (i > 0) + write(", "); + + if (a.get()[i]->is_array()) + { + a.get()[i]->as_array()->accept(*this, true); + } + else + { + a.get()[i]->accept(*this, true); + } + } + + write("]"); + } + + /** + * Output a table_array element of the TOML tree + */ + void visit(const table_array& t, bool = false) + { + for (unsigned int j = 0; j < t.get().size(); ++j) + { + if (j > 0) + endline(); + + t.get()[j]->accept(*this, true); + } + + endline(); + } + + /** + * Escape a string for output. + */ + static std::string escape_string(const std::string& str) + { + std::string res; + for (auto it = str.begin(); it != str.end(); ++it) + { + if (*it == '\b') + { + res += "\\b"; + } + else if (*it == '\t') + { + res += "\\t"; + } + else if (*it == '\n') + { + res += "\\n"; + } + else if (*it == '\f') + { + res += "\\f"; + } + else if (*it == '\r') + { + res += "\\r"; + } + else if (*it == '"') + { + res += "\\\""; + } + else if (*it == '\\') + { + res += "\\\\"; + } + else if (static_cast<uint32_t>(*it) <= UINT32_C(0x001f)) + { + res += "\\u"; + std::stringstream ss; + ss << std::hex << static_cast<uint32_t>(*it); + res += ss.str(); + } + else + { + res += *it; + } + } + return res; + } + + protected: + /** + * Write out a string. + */ + void write(const value<std::string>& v) + { + write("\""); + write(escape_string(v.get())); + write("\""); + } + + /** + * Write out a double. + */ + void write(const value<double>& v) + { + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits<double>::max_digits10) + << v.get(); + + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); + + stream_ << double_str; + has_naked_endline_ = false; + } + + /** + * Write out an integer, local_date, local_time, local_datetime, or + * offset_datetime. + */ + template <class T> + typename std::enable_if< + is_one_of<T, int64_t, local_date, local_time, local_datetime, + offset_datetime>::value>::type + write(const value<T>& v) + { + write(v.get()); + } + + /** + * Write out a boolean. + */ + void write(const value<bool>& v) + { + write((v.get() ? "true" : "false")); + } + + /** + * Write out the header of a table. + */ + void write_table_header(bool in_array = false) + { + if (!path_.empty()) + { + indent(); + + write("["); + + if (in_array) + { + write("["); + } + + for (unsigned int i = 0; i < path_.size(); ++i) + { + if (i > 0) + { + write("."); + } + + if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_[i]); + } + else + { + write("\""); + write(escape_string(path_[i])); + write("\""); + } + } + + if (in_array) + { + write("]"); + } + + write("]"); + endline(); + } + } + + /** + * Write out the identifier for an item in a table. + */ + void write_table_item_header(const base& b) + { + if (!b.is_table() && !b.is_table_array()) + { + indent(); + + if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_.back()); + } + else + { + write("\""); + write(escape_string(path_.back())); + write("\""); + } + + write(" = "); + } + } + + private: + /** + * Indent the proper number of tabs given the size of + * the path. + */ + void indent() + { + for (std::size_t i = 1; i < path_.size(); ++i) + write(indent_); + } + + /** + * Write a value out to the stream. + */ + template <class T> + void write(const T& v) + { + stream_ << v; + has_naked_endline_ = false; + } + + /** + * Write an endline out to the stream + */ + void endline() + { + if (!has_naked_endline_) + { + stream_ << "\n"; + has_naked_endline_ = true; + } + } + + private: + std::ostream& stream_; + const std::string indent_; + std::vector<std::string> path_; + bool has_naked_endline_; +}; + +inline std::ostream& operator<<(std::ostream& stream, const base& b) +{ + toml_writer writer{stream}; + b.accept(writer); + return stream; +} + +template <class T> +std::ostream& operator<<(std::ostream& stream, const value<T>& v) +{ + toml_writer writer{stream}; + v.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table_array& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const array& a) +{ + toml_writer writer{stream}; + a.accept(writer); + return stream; +} +} // namespace cpptoml +#endif // CPPTOML_H diff --git a/third_party/nix/src/libexpr/CMakeLists.txt b/third_party/nix/src/libexpr/CMakeLists.txt new file mode 100644 index 000000000000..8cb7143d2c4b --- /dev/null +++ b/third_party/nix/src/libexpr/CMakeLists.txt @@ -0,0 +1,85 @@ +# -*- mode: cmake; -*- +add_library(nixexpr SHARED) +set_property(TARGET nixexpr PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for 'generated/' +target_include_directories(nixexpr PUBLIC "${nix_SOURCE_DIR}/src") + +# Generate lexer & parser for inclusion: +find_package(BISON) +find_package(FLEX) + +BISON_TARGET(NixParser parser.y + ${PROJECT_BINARY_DIR}/generated/parser-tab.cc + DEFINES_FILE ${PROJECT_BINARY_DIR}/generated/parser-tab.hh) + +FLEX_TARGET(NixLexer lexer.l + ${PROJECT_BINARY_DIR}/generated/lexer-tab.cc + DEFINES_FILE ${PROJECT_BINARY_DIR}/generated/lexer-tab.hh) + +ADD_FLEX_BISON_DEPENDENCY(NixLexer NixParser) + +set(HEADER_FILES + attr-path.hh + attr-set.hh + common-eval-args.hh + eval.hh + eval-inline.hh + function-trace.hh + get-drvs.hh + json-to-value.hh + names.hh + nixexpr.hh + parser.hh + primops.hh + symbol-table.hh + value.hh + value-to-json.hh + value-to-xml.hh +) + +target_sources(nixexpr + PUBLIC + ${HEADER_FILES} + + PRIVATE + ${PROJECT_BINARY_DIR}/generated/parser-tab.hh + ${PROJECT_BINARY_DIR}/generated/parser-tab.cc + ${PROJECT_BINARY_DIR}/generated/lexer-tab.hh + ${PROJECT_BINARY_DIR}/generated/lexer-tab.cc + primops/context.cc + primops/fetchGit.cc + primops/fetchMercurial.cc + primops/fromTOML.cc + attr-path.cc + attr-set.cc + common-eval-args.cc + eval.cc + function-trace.cc + get-drvs.cc + json-to-value.cc + names.cc + nixexpr.cc + parser.cc + primops.cc + symbol-table.cc + value.cc + value-to-json.cc + value-to-xml.cc +) + +target_link_libraries(nixexpr + nixmain + nixstore + nixutil + + absl::btree + absl::flat_hash_set + absl::node_hash_set + absl::strings +) + +configure_file("nix-expr.pc.in" "${PROJECT_BINARY_DIR}/nix-expr.pc" @ONLY) +INSTALL(FILES "${PROJECT_BINARY_DIR}/nix-expr.pc" DESTINATION "${PKGCONFIG_INSTALL_DIR}") + +INSTALL(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nix/libexpr) +INSTALL(TARGETS nixexpr DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/third_party/nix/src/libexpr/attr-path.cc b/third_party/nix/src/libexpr/attr-path.cc new file mode 100644 index 000000000000..86ebeec2fb15 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-path.cc @@ -0,0 +1,109 @@ +#include "libexpr/attr-path.hh" + +#include <absl/strings/numbers.h> + +#include "libexpr/eval-inline.hh" +#include "libutil/util.hh" + +namespace nix { + +static Strings parseAttrPath(const std::string& s) { + Strings res; + std::string cur; + std::string::const_iterator i = s.begin(); + while (i != s.end()) { + if (*i == '.') { + res.push_back(cur); + cur.clear(); + } else if (*i == '"') { + ++i; + while (true) { + if (i == s.end()) { + throw Error(format("missing closing quote in selection path '%1%'") % + s); + } + if (*i == '"') { + break; + } + cur.push_back(*i++); + } + } else { + cur.push_back(*i); + } + ++i; + } + if (!cur.empty()) { + res.push_back(cur); + } + return res; +} + +Value* findAlongAttrPath(EvalState& state, const std::string& attrPath, + Bindings* autoArgs, Value& vIn) { + Strings tokens = parseAttrPath(attrPath); + + Error attrError = + Error(format("attribute selection path '%1%' does not match expression") % + attrPath); + + Value* v = &vIn; + + for (auto& attr : tokens) { + /* Is i an index (integer) or a normal attribute name? */ + enum { apAttr, apIndex } apType = apAttr; + unsigned int attrIndex; + if (absl::SimpleAtoi(attr, &attrIndex)) { + apType = apIndex; + } + + /* Evaluate the expression. */ + Value* vNew = state.allocValue(); + state.autoCallFunction(autoArgs, *v, *vNew); + v = vNew; + state.forceValue(*v); + + /* It should evaluate to either a set or an expression, + according to what is specified in the attrPath. */ + + if (apType == apAttr) { + if (v->type != tAttrs) { + throw TypeError(format("the expression selected by the selection path " + "'%1%' should be a set but is %2%") % + attrPath % showType(*v)); + } + + if (attr.empty()) { + throw Error(format("empty attribute name in selection path '%1%'") % + attrPath); + } + + Bindings::iterator a = v->attrs->find(state.symbols.Create(attr)); + if (a == v->attrs->end()) { + throw Error( + format("attribute '%1%' in selection path '%2%' not found") % attr % + attrPath); + } + v = &*(a->second).value; + } + + else if (apType == apIndex) { + if (!v->isList()) { + throw TypeError(format("the expression selected by the selection path " + "'%1%' should be a list but is %2%") % + attrPath % showType(*v)); + } + + if (attrIndex >= v->listSize()) { + throw Error( + format("list index %1% in selection path '%2%' is out of range") % + attrIndex % attrPath); + } + + v = (*v->list)[attrIndex]; + } + } + + return v; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/attr-path.hh b/third_party/nix/src/libexpr/attr-path.hh new file mode 100644 index 000000000000..97170be84098 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-path.hh @@ -0,0 +1,13 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" + +namespace nix { + +Value* findAlongAttrPath(EvalState& state, const std::string& attrPath, + Bindings* autoArgs, Value& vIn); + +} diff --git a/third_party/nix/src/libexpr/attr-set.cc b/third_party/nix/src/libexpr/attr-set.cc new file mode 100644 index 000000000000..b1617c981f51 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-set.cc @@ -0,0 +1,111 @@ +#include "libexpr/attr-set.hh" + +#include <new> + +#include <absl/container/btree_map.h> +#include <glog/logging.h> + +#include "libexpr/eval-inline.hh" + +namespace nix { + +// This function inherits its name from previous implementations, in +// which Bindings was backed by an array of elements which was scanned +// linearly. +// +// In that setup, inserting duplicate elements would always yield the +// first element (until the next sort, which wasn't stable, after +// which things are more or less undefined). +// +// This behaviour is mimicked by using .insert(), which will *not* +// override existing values. +void Bindings::push_back(const Attr& attr) { + auto [_, inserted] = attributes_.insert({attr.name, attr}); + + if (!inserted) { + DLOG(WARNING) << "attempted to insert duplicate attribute for key '" + << attr.name << "'"; + } +} + +size_t Bindings::size() const { return attributes_.size(); } + +bool Bindings::empty() { return attributes_.empty(); } + +Bindings::iterator Bindings::find(const Symbol& name) { + return attributes_.find(name); +} + +bool Bindings::Equal(const Bindings* other, EvalState& state) const { + if (this == other) { + return true; + } + + if (this->attributes_.size() != other->attributes_.size()) { + return false; + } + + Bindings::const_iterator i; + Bindings::const_iterator j; + for (i = this->cbegin(), j = other->cbegin(); i != this->cend(); ++i, ++j) { + if (i->second.name != j->second.name || + !state.eqValues(*i->second.value, *j->second.value)) { + return false; + } + } + + return true; +} + +Bindings::iterator Bindings::begin() { return attributes_.begin(); } +Bindings::iterator Bindings::end() { return attributes_.end(); } + +Bindings::const_iterator Bindings::cbegin() const { + return attributes_.cbegin(); +} + +Bindings::const_iterator Bindings::cend() const { return attributes_.cend(); } + +std::unique_ptr<Bindings> Bindings::New(size_t capacity) { + if (capacity == 0) { + // TODO(tazjin): A lot of 0-capacity Bindings are allocated. + // It would be nice to optimize that. + } + + return std::make_unique<Bindings>(); +} + +std::unique_ptr<Bindings> Bindings::Merge(const Bindings& lhs, + const Bindings& rhs) { + auto bindings = New(lhs.size() + rhs.size()); + + // Values are merged by inserting the entire iterator range of both + // input sets. The right-hand set (the values of which take + // precedence) is inserted *first* because the range insertion + // method does not override values. + bindings->attributes_.insert(rhs.attributes_.cbegin(), + rhs.attributes_.cend()); + bindings->attributes_.insert(lhs.attributes_.cbegin(), + lhs.attributes_.cend()); + + return bindings; +} + +void EvalState::mkAttrs(Value& v, size_t capacity) { + clearValue(v); + v.type = tAttrs; + v.attrs = Bindings::New(capacity); + nrAttrsets++; + nrAttrsInAttrsets += capacity; +} + +/* Create a new attribute named 'name' on an existing attribute set stored + in 'vAttrs' and return the newly allocated Value which is associated with + this attribute. */ +Value* EvalState::allocAttr(Value& vAttrs, const Symbol& name) { + Value* v = allocValue(); + vAttrs.attrs->push_back(Attr(name, v)); + return v; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/attr-set.hh b/third_party/nix/src/libexpr/attr-set.hh new file mode 100644 index 000000000000..5d77e0907cd6 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-set.hh @@ -0,0 +1,69 @@ +// This file implements the underlying structure of Nix attribute sets. +#pragma once + +#include <absl/container/btree_map.h> + +#include "libexpr/nixexpr.hh" +#include "libexpr/symbol-table.hh" +#include "libutil/types.hh" + +namespace nix { // TODO(tazjin): ::expr + +class EvalState; +struct Value; + +/* Map one attribute name to its value. */ +struct Attr { + Symbol name; + Value* value; // TODO(tazjin): Who owns this? + Pos* pos; // TODO(tazjin): Who owns this? + Attr(Symbol name, Value* value, Pos* pos = &noPos) + : name(name), value(value), pos(pos){}; +}; + +using AttributeMap = absl::btree_map<Symbol, Attr>; + +class Bindings { + public: + using iterator = AttributeMap::iterator; + using const_iterator = AttributeMap::const_iterator; + + // Allocate a new attribute set that is visible to the garbage + // collector. + static std::unique_ptr<Bindings> New(size_t capacity = 0); + + // Create a new attribute set by merging two others. This is used to + // implement the `//` operator in Nix. + static std::unique_ptr<Bindings> Merge(const Bindings& lhs, + const Bindings& rhs); + + // Return the number of contained elements. + size_t size() const; + + // Is this attribute set empty? + bool empty(); + + // Insert, but do not replace, values in the attribute set. + void push_back(const Attr& attr); + + // Are these two attribute sets deeply equal? + // Note: Does not special-case derivations. Use state.eqValues() to check + // attrsets that may be derivations. + bool Equal(const Bindings* other, EvalState& state) const; + + // Look up a specific element of the attribute set. + iterator find(const Symbol& name); + + iterator begin(); + const_iterator cbegin() const; + iterator end(); + const_iterator cend() const; + + // oh no + friend class EvalState; + + private: + AttributeMap attributes_; +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/common-eval-args.cc b/third_party/nix/src/libexpr/common-eval-args.cc new file mode 100644 index 000000000000..f63d3f8276ec --- /dev/null +++ b/third_party/nix/src/libexpr/common-eval-args.cc @@ -0,0 +1,72 @@ +#include "libexpr/common-eval-args.hh" + +#include "libexpr/eval.hh" +#include "libmain/shared.hh" +#include "libstore/download.hh" +#include "libutil/util.hh" + +namespace nix { + +MixEvalArgs::MixEvalArgs() { + mkFlag() + .longName("arg") + .description("argument to be passed to Nix functions") + .labels({"name", "expr"}) + .handler([&](std::vector<std::string> ss) { + auto_args_[ss[0]] = std::make_pair(kArgTypeExpr, ss[1]); + }); + + mkFlag() + .longName("argstr") + .description("string-valued argument to be passed to Nix functions") + .labels({"name", "string"}) + .handler([&](std::vector<std::string> ss) { + auto_args_[ss[0]] = std::make_pair(kArgTypeString, ss[1]); + }); + + mkFlag() + .shortName('I') + .longName("include") + .description( + "add a path to the list of locations used to look up <...> file " + "names") + .label("path") + .handler([&](const std::string& s) { searchPath.push_back(s); }); +} + +std::unique_ptr<Bindings> MixEvalArgs::getAutoArgs(EvalState& state) { + auto res = Bindings::New(auto_args_.size()); + for (auto& [arg, arg_value] : auto_args_) { + Value* v = state.allocValue(); + switch (arg_value.first) { + case kArgTypeExpr: { + state.mkThunk_( + *v, state.parseExprFromString(arg_value.second, absPath("."))); + break; + } + case kArgTypeString: { + mkString(*v, arg_value.second); + break; + } + } + + res->push_back(Attr(state.symbols.Create(arg), v)); + } + return res; +} + +Path lookupFileArg(EvalState& state, std::string s) { + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } + if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + return state.findFile(p); + } else { + return absPath(s); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/common-eval-args.hh b/third_party/nix/src/libexpr/common-eval-args.hh new file mode 100644 index 000000000000..5e0e8af79cbe --- /dev/null +++ b/third_party/nix/src/libexpr/common-eval-args.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "libutil/args.hh" + +namespace nix { + +class Store; +class EvalState; +class Bindings; + +enum ArgType { kArgTypeString, kArgTypeExpr }; + +struct MixEvalArgs : virtual Args { + MixEvalArgs(); + + std::unique_ptr<Bindings> getAutoArgs(EvalState& state); + + Strings searchPath; + + private: + std::map<std::string, std::pair<ArgType, std::string>> auto_args_; +}; + +Path lookupFileArg(EvalState& state, std::string s); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval-inline.hh b/third_party/nix/src/libexpr/eval-inline.hh new file mode 100644 index 000000000000..5162ab3971a3 --- /dev/null +++ b/third_party/nix/src/libexpr/eval-inline.hh @@ -0,0 +1,90 @@ +#pragma once + +#include "libexpr/eval.hh" + +#define LocalNoInline(f) \ + static f __attribute__((noinline)); \ + f +#define LocalNoInlineNoReturn(f) \ + static f __attribute__((noinline, noreturn)); \ + f + +namespace nix { + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const Pos& pos)) { + throw EvalError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v)) { + throw TypeError(format(s) % showType(v)); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Value& v, + const Pos& pos)) { + throw TypeError(format(s) % showType(v) % pos); +} + +void EvalState::forceValue(Value& v, const Pos& pos) { + if (v.type == tThunk) { + Env* env = v.thunk.env; + Expr* expr = v.thunk.expr; + try { + v.type = tBlackhole; + // checkInterrupt(); + expr->eval(*this, *env, v); + } catch (...) { + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = expr; + throw; + } + } else if (v.type == tApp) { + callFunction(*v.app.left, *v.app.right, v, noPos); + } else if (v.type == tBlackhole) { + throwEvalError("infinite recursion encountered, at %1%", pos); + } +} + +inline void EvalState::forceAttrs(Value& v) { + forceValue(v); + if (v.type != tAttrs) { + throwTypeError("value is %1% while a set was expected", v); + } +} + +inline void EvalState::forceAttrs(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tAttrs) { + throwTypeError("value is %1% while a set was expected, at %2%", v, pos); + } +} + +inline void EvalState::forceList(Value& v) { + forceValue(v); + if (!v.isList()) { + throwTypeError("value is %1% while a list was expected", v); + } +} + +inline void EvalState::forceList(Value& v, const Pos& pos) { + forceValue(v); + if (!v.isList()) { + throwTypeError("value is %1% while a list was expected, at %2%", v, pos); + } +} + +/* Note: Various places expect the allocated memory to be zeroed. */ +inline void* allocBytes(size_t n) { + void* p; +#if HAVE_BOEHMGC + p = GC_MALLOC(n); +#else + p = calloc(n, 1); +#endif + if (!p) { + throw std::bad_alloc(); + } + return p; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval.cc b/third_party/nix/src/libexpr/eval.cc new file mode 100644 index 000000000000..682ea6483213 --- /dev/null +++ b/third_party/nix/src/libexpr/eval.cc @@ -0,0 +1,1878 @@ +#include "libexpr/eval.hh" + +#include <algorithm> +#include <chrono> +#include <cstdint> +#include <cstring> +#include <fstream> +#include <iostream> +#include <memory> +#include <new> +#include <optional> +#include <variant> + +#include <absl/base/call_once.h> +#include <absl/container/flat_hash_set.h> +#include <absl/strings/match.h> +#include <glog/logging.h> +#include <sys/resource.h> +#include <sys/time.h> +#include <unistd.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/function-trace.hh" +#include "libexpr/value.hh" +#include "libstore/derivations.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/hash.hh" +#include "libutil/json.hh" +#include "libutil/util.hh" +#include "libutil/visitor.hh" + +namespace nix { +namespace { + +void ConfigureGc() { /* This function intentionally left blank. */ +} + +} // namespace + +namespace expr { + +absl::once_flag gc_flag; + +void InitGC() { absl::call_once(gc_flag, &ConfigureGc); } + +} // namespace expr + +static char* dupString(const char* s) { + char* t; + t = strdup(s); + if (t == nullptr) { + throw std::bad_alloc(); + } + return t; +} + +std::shared_ptr<Value*> allocRootValue(Value* v) { + return std::make_shared<Value*>(v); +} + +static void printValue(std::ostream& str, std::set<const Value*>& active, + const Value& v) { + checkInterrupt(); + + if (active.find(&v) != active.end()) { + str << "<CYCLE>"; + return; + } + active.insert(&v); + + switch (v.type) { + case tInt: + str << v.integer; + break; + case tBool: + str << (v.boolean ? "true" : "false"); + break; + case tString: + str << "\""; + for (const char* i = v.string.s; *i != 0; i++) { + if (*i == '\"' || *i == '\\') { + str << "\\" << *i; + } else if (*i == '\n') { + str << "\\n"; + } else if (*i == '\r') { + str << "\\r"; + } else if (*i == '\t') { + str << "\\t"; + } else { + str << *i; + } + } + str << "\""; + break; + case tPath: + str << v.path; // !!! escaping? + break; + case tNull: + str << "null"; + break; + case tAttrs: { + str << "{ "; + for (const auto& [key, value] : *v.attrs) { + str << key << " = "; + printValue(str, active, *value.value); + str << "; "; + } + str << "}"; + break; + } + case tList: + str << "[ "; + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValue(str, active, *(*v.list)[n]); + str << " "; + } + str << "]"; + break; + case tThunk: + case tApp: + str << "<CODE>"; + break; + case tLambda: + str << "<LAMBDA>"; + break; + case tPrimOp: + str << "<PRIMOP>"; + break; + case tPrimOpApp: + str << "<PRIMOP-APP>"; + break; + case tFloat: + str << v.fpoint; + break; + default: + throw Error( + absl::StrCat("invalid value of type ", static_cast<int>(v.type))); + } + + active.erase(&v); +} + +std::ostream& operator<<(std::ostream& str, const Value& v) { + std::set<const Value*> active; + printValue(str, active, v); + return str; +} + +const Value* getPrimOp(const Value& v) { + const Value* primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + +std::string showType(const Value& v) { + switch (v.type) { + case tInt: + return "an integer"; + case tBool: + return "a boolean"; + case tString: + return v.string.context != nullptr ? "a string with context" : "a string"; + case tPath: + return "a path"; + case tNull: + return "null"; + case tAttrs: + return "a set"; + case tList: + return "a list"; + case tThunk: + return "a thunk"; + case tApp: + return "a function application"; + case tLambda: + return "a function"; + case tBlackhole: + return "a black hole"; + case tPrimOp: + return fmt("the built-in function '%s'", std::string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", + std::string(getPrimOp(v)->primOp->name)); + case _reserved1: + LOG(FATAL) << "attempted to show the type string of the deprecated " + "tExternal value"; + break; + case tFloat: + return "a float"; + } + LOG(FATAL) + << "attempted to determine the type string of an unknown type number (" + << static_cast<int>(v.type) << ")"; + abort(); +} + +static Symbol getName(const AttrName& name, EvalState& state, Env& env) { + return std::visit( + util::overloaded{[&](const Symbol& name) -> Symbol { return name; }, + [&](Expr* expr) -> Symbol { + Value nameValue; + expr->eval(state, env, nameValue); + state.forceStringNoCtx(nameValue); + return state.symbols.Create(nameValue.string.s); + }}, + name); +} + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const std::string& s) { + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') { + start2 = p + 1; + } + ++p; + } + + if (p == s.end()) { + if (p != start) { + res.push_back(std::string(start, p)); + } + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') { + ++p; + } + } + res.push_back(std::string(start, p)); + if (p == s.end()) { + break; + } + } + + ++p; + } + + return res; +} + +EvalState::EvalState(const Strings& _searchPath, const ref<Store>& store) + : sWith(symbols.Create("<with>")), + sOutPath(symbols.Create("outPath")), + sDrvPath(symbols.Create("drvPath")), + sType(symbols.Create("type")), + sMeta(symbols.Create("meta")), + sName(symbols.Create("name")), + sValue(symbols.Create("value")), + sSystem(symbols.Create("system")), + sOutputs(symbols.Create("outputs")), + sOutputName(symbols.Create("outputName")), + sIgnoreNulls(symbols.Create("__ignoreNulls")), + sFile(symbols.Create("file")), + sLine(symbols.Create("line")), + sColumn(symbols.Create("column")), + sFunctor(symbols.Create("__functor")), + sToString(symbols.Create("__toString")), + sRight(symbols.Create("right")), + sWrong(symbols.Create("wrong")), + sStructuredAttrs(symbols.Create("__structuredAttrs")), + sBuilder(symbols.Create("builder")), + sArgs(symbols.Create("args")), + sOutputHash(symbols.Create("outputHash")), + sOutputHashAlgo(symbols.Create("outputHashAlgo")), + sOutputHashMode(symbols.Create("outputHashMode")), + sDerivationNix(std::nullopt), + repair(NoRepair), + store(store), + baseEnv(allocEnv(128)), + staticBaseEnv(false, nullptr) { + expr::InitGC(); + + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; + + /* Initialise the Nix expression search path. */ + if (!evalSettings.pureEval) { + Strings paths = parseNixPath(getEnv("NIX_PATH").value_or("")); + for (auto& i : _searchPath) { + addToSearchPath(i); + } + for (auto& i : paths) { + addToSearchPath(i); + } + } + addToSearchPath("nix=" + + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + if (evalSettings.restrictEval || evalSettings.pureEval) { + allowedPaths = PathSet(); + + for (auto& i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) { + continue; + } + + auto path = r.second; + + if (store->isInStore(r.second)) { + PathSet closure; + store->computeFSClosure(store->toStorePath(r.second), closure); + for (auto& path : closure) { + allowedPaths->insert(path); + } + } else { + allowedPaths->insert(r.second); + } + } + } + + createBaseEnv(); +} + +EvalState::~EvalState() = default; + +Path EvalState::checkSourcePath(const Path& path_) { + TraceFileAccess(path_); + if (!allowedPaths) { + return path_; + } + + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) { + return i->second; + } + + bool found = false; + + /* First canonicalize the path without symlinks, so we make sure an + * attacker can't append ../../... to a path that would be in allowedPaths + * and thus leak symlink targets. + */ + Path abspath = canonPath(path_); + + for (auto& i : *allowedPaths) { + if (isDirOrInDir(abspath, i)) { + found = true; + break; + } + } + + if (!found) { + throw RestrictedPathError( + "access to path '%1%' is forbidden in restricted mode", abspath); + } + + /* Resolve symlinks. */ + DLOG(INFO) << "checking access to '" << abspath << "'"; + Path path = canonPath(abspath, true); + + for (auto& i : *allowedPaths) { + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; + return path; + } + } + + throw RestrictedPathError( + "access to path '%1%' is forbidden in restricted mode", path); +} + +void EvalState::checkURI(const std::string& uri) { + if (!evalSettings.restrictEval) { + return; + } + + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. Note: this allows 'http://' and + 'https://' as prefixes for any http/https URI. */ + for (auto& prefix : evalSettings.allowedUris.get()) { + if (uri == prefix || + (uri.size() > prefix.size() && !prefix.empty() && + absl::StartsWith(uri, prefix) && + (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) { + return; + } + } + + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (absl::StartsWith(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (absl::StartsWith(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + + throw RestrictedPathError( + "access to URI '%s' is forbidden in restricted mode", uri); +} + +Path EvalState::toRealPath(const Path& path, const PathSet& context) { + // FIXME: check whether 'path' is in 'context'. + return !context.empty() && store->isInStore(path) ? store->toRealPath(path) + : path; +}; + +Value* EvalState::addConstant(const std::string& name, Value& v) { + Value* v2 = allocValue(); + *v2 = v; + staticBaseEnv.vars[symbols.Create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v2; + std::string name2 = + std::string(name, 0, 2) == "__" ? std::string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.Create(name2), v2)); + return v2; +} + +Value* EvalState::addPrimOp(const std::string& name, size_t arity, + PrimOpFun primOp) { + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } + std::string name2 = + std::string(name, 0, 2) == "__" ? std::string(name, 2) : name; + Symbol sym = symbols.Create(name2); + Value* v = allocValue(); + v->type = tPrimOp; + v->primOp = std::make_shared<PrimOp>(primOp, arity, sym); + staticBaseEnv.vars[symbols.Create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + return v; +} + +Value& EvalState::getBuiltin(const std::string& name) { + return *baseEnv.values[0]->attrs->find(symbols.Create(name))->second.value; +} + +/* Every "format" object (even temporary) takes up a few hundred bytes + of stack space, which is a real killer in the recursive + evaluator. So here are some helper functions for throwing + exceptions. */ + +LocalNoInlineNoReturn(void throwEvalError(const char* s, + const std::string& s2)) { + throw EvalError(format(s) % s2); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const std::string& s2, + const Pos& pos)) { + throw EvalError(format(s) % s2 % pos); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const std::string& s2, + const std::string& s3)) { + throw EvalError(format(s) % s2 % s3); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const std::string& s2, + const std::string& s3, + const Pos& pos)) { + throw EvalError(format(s) % s2 % s3 % pos); +} + +LocalNoInlineNoReturn(void throwEvalError(const char* s, const Symbol& sym, + const Pos& p1, const Pos& p2)) { + throw EvalError(format(s) % sym % p1 % p2); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const Pos& pos)) { + throw TypeError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, + const std::string& s1)) { + throw TypeError(format(s) % s1); +} + +LocalNoInlineNoReturn(void throwTypeError(const char* s, const ExprLambda& fun, + const Symbol& s2, const Pos& pos)) { + throw TypeError(format(s) % fun.showNamePos() % s2 % pos); +} + +LocalNoInlineNoReturn(void throwAssertionError(const char* s, + const std::string& s1, + const Pos& pos)) { + throw AssertionError(format(s) % s1 % pos); +} + +LocalNoInlineNoReturn(void throwUndefinedVarError(const char* s, + const std::string& s1, + const Pos& pos)) { + throw UndefinedVarError(format(s) % s1 % pos); +} + +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const std::string& s2)) { + e.addPrefix(format(s) % s2); +} + +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const ExprLambda& fun, const Pos& pos)) { + e.addPrefix(format(s) % fun.showNamePos() % pos); +} + +LocalNoInline(void addErrorPrefix(Error& e, const char* s, + const std::string& s2, const Pos& pos)) { + e.addPrefix(format(s) % s2 % pos); +} + +void mkString(Value& v, const char* s) { mkStringNoCopy(v, dupString(s)); } + +Value& mkString(Value& v, const std::string& s, const PathSet& context) { + mkString(v, s.c_str()); + if (!context.empty()) { + size_t n = 0; + v.string.context = static_cast<const char**>( + allocBytes((context.size() + 1) * sizeof(char*))); + for (auto& i : context) { + v.string.context[n++] = dupString(i.c_str()); + } + v.string.context[n] = nullptr; + } + return v; +} + +void mkPath(Value& v, const char* s) { mkPathNoCopy(v, dupString(s)); } + +inline Value* EvalState::lookupVar(Env* env, const ExprVar& var, bool noEval) { + for (size_t l = var.level; l != 0u; --l, env = env->up) { + ; + } + + if (!var.fromWith) { + return env->values[var.displ]; + } + + while (true) { + if (env->type == Env::HasWithExpr) { + if (noEval) { + return nullptr; + } + if (!env->withAttrsExpr) { + CHECK(false) << "HasWithExpr evaluated twice"; + } + Value* v = allocValue(); + evalAttrs(*env->up, env->withAttrsExpr, *v); + env->values[0] = v; + env->withAttrsExpr = nullptr; + env->type = Env::HasWithAttrs; + } + Bindings::iterator j = env->values[0]->attrs->find(var.name); + if (j != env->values[0]->attrs->end()) { + if (countCalls && (j->second.pos != nullptr)) { + attrSelects[*j->second.pos]++; + } + return j->second.value; + } + if (env->prevWith == 0u) { + throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, + var.pos); + } + for (size_t l = env->prevWith; l != 0u; --l, env = env->up) { + } + } +} + +Value* EvalState::allocValue() { + nrValues++; + return new Value; +} + +Env& EvalState::allocEnv(size_t size) { + if (size > std::numeric_limits<decltype(Env::size)>::max()) { + throw Error("environment size %d is too big", size); + } + + nrEnvs++; + nrValuesInEnvs += size; + Env* env = new Env(size); + env->type = Env::Plain; + + return *env; +} + +void EvalState::mkList(Value& v, std::shared_ptr<NixList> list) { + nrListElems += list->size(); + clearValue(v); + v.type = tList; + v.list = list; +} + +void EvalState::mkList(Value& v, size_t size) { + EvalState::mkList(v, std::make_shared<NixList>(size)); +} + +unsigned long nrThunks = 0; + +static inline void mkThunk(Value& v, Env& env, Expr* expr) { + v.type = tThunk; + v.thunk.env = &env; + v.thunk.expr = expr; + nrThunks++; +} + +void EvalState::mkThunk_(Value& v, Expr* expr) { mkThunk(v, baseEnv, expr); } + +void EvalState::mkPos(Value& v, Pos* pos) { + if ((pos != nullptr) && pos->file.has_value() && pos->file.value().set()) { + mkAttrs(v, 3); + mkString(*allocAttr(v, sFile), pos->file.value()); + mkInt(*allocAttr(v, sLine), pos->line); + mkInt(*allocAttr(v, sColumn), pos->column); + } else { + mkNull(v); + } +} + +/* Create a thunk for the delayed computation of the given expression + in the given environment. But if the expression is a variable, + then look it up right away. This significantly reduces the number + of thunks allocated. */ +Value* Expr::maybeThunk(EvalState& state, Env& env) { + Value* v = state.allocValue(); + mkThunk(*v, env, this); + return v; +} + +unsigned long nrAvoided = 0; + +Value* ExprVar::maybeThunk(EvalState& state, Env& env) { + Value* v = state.lookupVar(&env, *this, true); + /* The value might not be initialised in the environment yet. + In that case, ignore it. */ + if (v != nullptr) { + nrAvoided++; + return v; + } + return Expr::maybeThunk(state, env); +} + +Value* ExprString::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +Value* ExprInt::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +Value* ExprFloat::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +Value* ExprPath::maybeThunk(EvalState& state, Env& env) { + nrAvoided++; + return &v; +} + +void EvalState::evalFile(const Path& path_, Value& v) { + auto path = checkSourcePath(path_); + + FileEvalCache::iterator i; + if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { + v = i->second; + return; + } + + Path path2 = resolveExprPath(path); + if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + v = i->second; + return; + } + + VLOG(2) << "evaluating file '" << path2 << "'"; + Expr* e = nullptr; + + auto j = fileParseCache.find(path2); + if (j != fileParseCache.end()) { + e = j->second; + } + + if (e == nullptr) { + e = parseExprFromFile(checkSourcePath(path2)); + } + + fileParseCache[path2] = e; + + try { + eval(e, v); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); + throw; + } + + fileEvalCache[path2] = v; + if (path != path2) { + fileEvalCache[path] = v; + } +} + +void EvalState::resetFileCache() { + fileEvalCache.clear(); + fileParseCache.clear(); +} + +void EvalState::eval(Expr* e, Value& v) { e->eval(*this, baseEnv, v); } + +inline bool EvalState::evalBool(Env& env, Expr* e) { + Value v; + e->eval(*this, env, v); + if (v.type != tBool) { + throwTypeError("value is %1% while a Boolean was expected", v); + } + return v.boolean; +} + +inline bool EvalState::evalBool(Env& env, Expr* e, const Pos& pos) { + Value v; + e->eval(*this, env, v); + if (v.type != tBool) { + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + } + return v.boolean; +} + +inline void EvalState::evalAttrs(Env& env, Expr* e, Value& v) { + e->eval(*this, env, v); + if (v.type != tAttrs) { + throwTypeError("value is %1% while a set was expected", v); + } +} + +void Expr::eval(EvalState& state, Env& env, Value& v) { abort(); } + +void ExprInt::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprFloat::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprString::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprPath::eval(EvalState& state, Env& env, Value& v) { v = this->v; } + +void ExprAttrs::eval(EvalState& state, Env& env, Value& value) { + state.mkAttrs(value, attrs.size() + dynamicAttrs.size()); + Env* dynamicEnv = &env; + + if (recursive) { + /* Create a new environment that contains the attributes in + this `rec'. */ + Env& env2(state.allocEnv(attrs.size())); + env2.up = &env; + dynamicEnv = &env2; + + /* The recursive attributes are evaluated in the new + environment, while the inherited attributes are evaluated + in the original environment. */ + size_t displ = 0; + for (auto& attr : attrs) { + Value* vAttr; + vAttr = + attr.second.e->maybeThunk(state, attr.second.inherited ? env : env2); + env2.values[displ++] = vAttr; + value.attrs->push_back(Attr(attr.first, vAttr, &attr.second.pos)); + } + } else { + // TODO(tazjin): insert range + for (auto& i : attrs) { + value.attrs->push_back( + Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); + } + } + + /* Dynamic attrs apply *after* rec. */ + for (auto& i : dynamicAttrs) { + Value nameVal; + i.nameExpr->eval(state, *dynamicEnv, nameVal); + state.forceValue(nameVal, i.pos); + if (nameVal.type == tNull) { + continue; + } + state.forceStringNoCtx(nameVal); + Symbol nameSym = state.symbols.Create(nameVal.string.s); + Bindings::iterator j = value.attrs->find(nameSym); + if (j != value.attrs->end()) { + throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", + nameSym, i.pos, *j->second.pos); + } + + value.attrs->push_back( + Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); + } +} + +void ExprLet::eval(EvalState& state, Env& env, Value& v) { + /* Create a new environment that contains the attributes in this + `let'. */ + Env& env2(state.allocEnv(attrs->attrs.size())); + env2.up = &env; + + /* The recursive attributes are evaluated in the new environment, + while the inherited attributes are evaluated in the original + environment. */ + size_t displ = 0; + for (auto& i : attrs->attrs) { + env2.values[displ++] = + i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + } + + body->eval(state, env2, v); +} + +void ExprList::eval(EvalState& state, Env& env, Value& v) { + state.mkList(v, elems.size()); + for (size_t n = 0; n < elems.size(); ++n) { + (*v.list)[n] = elems[n]->maybeThunk(state, env); + } +} + +void ExprVar::eval(EvalState& state, Env& env, Value& v) { + Value* v2 = state.lookupVar(&env, *this, false); + state.forceValue(*v2, pos); + v = *v2; +} + +static std::string showAttrPath(EvalState& state, Env& env, + const AttrPath& attrPath) { + std::ostringstream out; + bool first = true; + for (auto& i : attrPath) { + if (!first) { + out << '.'; + } else { + first = false; + } + out << getName(i, state, env); + } + return out.str(); +} + +uint64_t nrLookups = 0; + +void ExprSelect::eval(EvalState& state, Env& env, Value& v) { + Value vTmp; + Pos* pos2 = nullptr; + Value* vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + try { + for (auto& i : attrPath) { + nrLookups++; + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (def != nullptr) { + state.forceValue(*vAttrs, pos); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + def->eval(state, env, v); + return; + } + } else { + state.forceAttrs(*vAttrs, pos); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + throwEvalError("attribute '%1%' missing, at %2%", name, pos); + } + } + vAttrs = j->second.value; + pos2 = j->second.pos; + if (state.countCalls && (pos2 != nullptr)) { + state.attrSelects[*pos2]++; + } + } + + state.forceValue(*vAttrs, (pos2 != nullptr ? *pos2 : this->pos)); + + } catch (Error& e) { + // This code relies on 'sDerivationNix' being correcty mutated at + // some prior point (it would previously otherwise have been a + // nullptr). + // + // We haven't seen this fail, so for now the contained value is + // just accessed at the risk of potentially crashing. + if ((pos2 != nullptr) && pos2->file != state.sDerivationNix.value()) { + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + showAttrPath(state, env, attrPath), *pos2); + } + throw; + } + + v = *vAttrs; +} + +void ExprOpHasAttr::eval(EvalState& state, Env& env, Value& v) { + Value vTmp; + Value* vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + for (auto& i : attrPath) { + state.forceValue(*vAttrs); + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { + mkBool(v, false); + return; + } + vAttrs = j->second.value; + } + + mkBool(v, true); +} + +void ExprLambda::eval(EvalState& state, Env& env, Value& v) { + v.type = tLambda; + v.lambda.env = &env; + v.lambda.fun = this; +} + +void ExprApp::eval(EvalState& state, Env& env, Value& v) { + /* FIXME: vFun prevents GCC from doing tail call optimisation. */ + Value vFun; + e1->eval(state, env, vFun); + state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); +} + +void EvalState::callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos) { + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value* primOp = &fun; + while (primOp->type == tPrimOpApp) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (argsLeft == 1) { + /* We have all the arguments, so call the primop. */ + + /* Put all the arguments in an array. */ + Value* vArgs[arity]; + auto n = arity - 1; + vArgs[n--] = &arg; + for (Value* arg = &fun; arg->type == tPrimOpApp; + arg = arg->primOpApp.left) { + vArgs[n--] = arg->primOpApp.right; + } + + /* And call the primop. */ + nrPrimOpCalls++; + if (countCalls) { + primOpCalls[primOp->primOp->name]++; + } + primOp->primOp->fun(*this, pos, vArgs, v); + } else { + Value* fun2 = allocValue(); + *fun2 = fun; + v.type = tPrimOpApp; + v.primOpApp.left = fun2; + v.primOpApp.right = &arg; + } +} + +void EvalState::callFunction(Value& fun, Value& arg, Value& v, const Pos& pos) { + auto trace = evalSettings.traceFunctionCalls + ? std::make_unique<FunctionCallTrace>(pos) + : nullptr; + + forceValue(fun, pos); + + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + callPrimOp(fun, arg, v, pos); + return; + } + + // If the value to be called is an attribute set, check whether it + // contains an appropriate function in the '__functor' element and + // use that. + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + // fun may be allocated on the stack of the calling function, + // but for functors we may keep a reference, so heap-allocate a + // copy and use that instead + auto& fun2 = *allocValue(); + fun2 = fun; + /* !!! Should we use the attr pos here? */ + Value v2; + // functors are called with the element itself as the first + // parameter, which is partially applied here + callFunction(*found->second.value, fun2, v2, pos); + return callFunction(v2, arg, v, pos); + } + } + + if (fun.type != tLambda) { + throwTypeError( + "attempt to call something which is not a function but %1%, at %2%", + fun, pos); + } + + ExprLambda& lambda(*fun.lambda.fun); + + auto size = (lambda.arg.empty() ? 0 : 1) + + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + Env& env2(allocEnv(size)); + env2.up = fun.lambda.env; + + size_t displ = 0; + + if (!lambda.matchAttrs) { + env2.values[displ++] = &arg; + + } else { + forceAttrs(arg, pos); + + if (!lambda.arg.empty()) { + env2.values[displ++] = &arg; + } + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto& i : lambda.formals->formals) { + Bindings::iterator j = arg.attrs->find(i.name); + if (j == arg.attrs->end()) { + if (i.def == nullptr) { + throwTypeError("%1% called without required argument '%2%', at %3%", + lambda, i.name, pos); + } + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->second.value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto& i : *arg.attrs) { + if (lambda.formals->argNames.find(i.second.name) == + lambda.formals->argNames.end()) { + throwTypeError("%1% called with unexpected argument '%2%', at %3%", + lambda, i.second.name, pos); + } + } + abort(); // shouldn't happen + } + } + + nrFunctionCalls++; + if (countCalls) { + incrFunctionCall(&lambda); + } + + /* Evaluate the body. This is conditional on showTrace, because + catching exceptions makes this function not tail-recursive. */ + if (settings.showTrace) { + try { + lambda.body->eval(*this, env2, v); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, + pos); + throw; + } + } else { + fun.lambda.fun->body->eval(*this, env2, v); + } +} + +// Lifted out of callFunction() because it creates a temporary that +// prevents tail-call optimisation. +void EvalState::incrFunctionCall(ExprLambda* fun) { functionCalls[fun]++; } + +void EvalState::autoCallFunction(Bindings* args, Value& fun, Value& res) { + forceValue(fun); + + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + Value* v = allocValue(); + callFunction(*found->second.value, fun, *v, noPos); + forceValue(*v); + return autoCallFunction(args, *v, res); + } + } + + if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + res = fun; + return; + } + + Value* actualArgs = allocValue(); + mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); + + if (fun.lambda.fun->formals->ellipsis) { + // If the formals have an ellipsis (eg the function accepts extra args) pass + // all available automatic arguments (which includes arguments specified on + // the command line via --arg/--argstr) + for (auto& [_, v] : *args) { + actualArgs->attrs->push_back(v); + } + } else { + // Otherwise, only pass the arguments that the function accepts + for (auto& i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args->find(i.name); + if (j != args->end()) { + actualArgs->attrs->push_back(j->second); + } else if (i.def == nullptr) { + throwTypeError( + "cannot auto-call a function that has an argument without a " + "default " + "value ('%1%')", + i.name); + } + } + } + + callFunction(fun, *actualArgs, res, noPos); +} + +void ExprWith::eval(EvalState& state, Env& env, Value& v) { + Env& env2(state.allocEnv(1)); + env2.up = &env; + env2.prevWith = prevWith; + env2.type = Env::HasWithExpr; + /* placeholder for result of attrs */ + env2.values[0] = nullptr; + env2.withAttrsExpr = this->attrs; + + body->eval(state, env2, v); +} + +void ExprIf::eval(EvalState& state, Env& env, Value& v) { + (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); +} + +void ExprAssert::eval(EvalState& state, Env& env, Value& v) { + if (!state.evalBool(env, cond, pos)) { + std::ostringstream out; + cond->show(out); + throwAssertionError("assertion %1% failed at %2%", out.str(), pos); + } + body->eval(state, env, v); +} + +void ExprOpNot::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, !state.evalBool(env, e)); +} + +void ExprOpEq::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + mkBool(v, state.eqValues(v1, v2)); +} + +void ExprOpNEq::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + mkBool(v, !state.eqValues(v1, v2)); +} + +void ExprOpAnd::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); +} + +void ExprOpOr::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +} + +void ExprOpImpl::eval(EvalState& state, Env& env, Value& v) { + mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +} + +void ExprOpUpdate::eval(EvalState& state, Env& env, Value& dest) { + Value v1; + Value v2; + state.evalAttrs(env, e1, v1); + state.evalAttrs(env, e2, v2); + + state.nrOpUpdates++; + + clearValue(dest); + dest.type = tAttrs; + dest.attrs = Bindings::Merge(*v1.attrs, *v2.attrs); +} + +void ExprOpConcatLists::eval(EvalState& state, Env& env, Value& v) { + Value v1; + e1->eval(state, env, v1); + Value v2; + e2->eval(state, env, v2); + state.concatLists(v, {&v1, &v2}, pos); +} + +void EvalState::concatLists(Value& v, const NixList& lists, const Pos& pos) { + nrListConcats++; + + auto outlist = std::make_shared<NixList>(); + + for (Value* list : lists) { + forceList(*list, pos); + outlist->insert(outlist->end(), list->list->begin(), list->list->end()); + } + + mkList(v, outlist); +} + +void ExprConcatStrings::eval(EvalState& state, Env& env, Value& v) { + PathSet context; + std::ostringstream s; + NixInt n = 0; + NixFloat nf = 0; + + bool first = !forceString; + ValueType firstType = tString; + + for (auto& i : *es) { + Value vTmp; + i->eval(state, env, vTmp); + + /* If the first element is a path, then the result will also + be a path, we don't copy anything (yet - that's done later, + since paths are copied when they are used in a derivation), + and none of the strings are allowed to have contexts. */ + if (first) { + firstType = vTmp.type; + first = false; + } + + if (firstType == tInt) { + if (vTmp.type == tInt) { + n += vTmp.integer; + } else if (vTmp.type == tFloat) { + // Upgrade the type from int to float; + firstType = tFloat; + nf = n; + nf += vTmp.fpoint; + } else { + throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), + pos); + } + } else if (firstType == tFloat) { + if (vTmp.type == tInt) { + nf += vTmp.integer; + } else if (vTmp.type == tFloat) { + nf += vTmp.fpoint; + } else { + throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), + pos); + } + } else { + s << state.coerceToString(pos, vTmp, context, false, + firstType == tString); + } + } + + if (firstType == tInt) { + mkInt(v, n); + } else if (firstType == tFloat) { + mkFloat(v, nf); + } else if (firstType == tPath) { + if (!context.empty()) { + throwEvalError( + "a string that refers to a store path cannot be appended to a path, " + "at %1%", + pos); + } + auto path = canonPath(s.str()); + mkPath(v, path.c_str()); + } else { + mkString(v, s.str(), context); + } +} + +void ExprPos::eval(EvalState& state, Env& env, Value& v) { + state.mkPos(v, &pos); +} + +template <typename T> +using traceable_flat_hash_set = absl::flat_hash_set<T>; + +void EvalState::forceValueDeep(Value& v) { + traceable_flat_hash_set<const Value*> seen; + + std::function<void(Value & v)> recurse; + + recurse = [&](Value& v) { + if (seen.find(&v) != seen.end()) { + return; + } + seen.insert(&v); + + forceValue(v); + + if (v.type == tAttrs) { + for (auto& i : *v.attrs) { + try { + recurse(*i.second.value); + } catch (Error& e) { + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + i.second.name, *i.second.pos); + throw; + } + } + } else if (v.isList()) { + for (size_t n = 0; n < v.listSize(); ++n) { + recurse(*(*v.list)[n]); + } + } + }; + + recurse(v); +} + +NixInt EvalState::forceInt(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type != tInt) { + throwTypeError("value is %1% while an integer was expected, at %2%", v, + pos); + } + return v.integer; +} + +NixFloat EvalState::forceFloat(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type == tInt) { + return static_cast<NixFloat>(v.integer); + } + if (v.type != tFloat) { + throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + } + return v.fpoint; +} + +bool EvalState::forceBool(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tBool) { + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + } + return v.boolean; +} + +bool EvalState::isFunctor(Value& fun) { + return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); +} + +void EvalState::forceFunction(Value& v, const Pos& pos) { + forceValue(v); + if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && + !isFunctor(v)) { + throwTypeError("value is %1% while a function was expected, at %2%", v, + pos); + } +} + +std::string EvalState::forceString(Value& v, const Pos& pos) { + forceValue(v, pos); + if (v.type != tString) { + if (pos) { + throwTypeError("value is %1% while a string was expected, at %2%", v, + pos); + } else { + throwTypeError("value is %1% while a string was expected", v); + } + } + return std::string(v.string.s); +} + +void copyContext(const Value& v, PathSet& context) { + if (v.string.context != nullptr) { + for (const char** p = v.string.context; *p != nullptr; ++p) { + context.insert(*p); + } + } +} + +std::string EvalState::forceString(Value& v, PathSet& context, const Pos& pos) { + std::string s = forceString(v, pos); + copyContext(v, context); + return s; +} + +std::string EvalState::forceStringNoCtx(Value& v, const Pos& pos) { + std::string s = forceString(v, pos); + if (v.string.context != nullptr) { + if (pos) { + throwEvalError( + "the string '%1%' is not allowed to refer to a store path (such as " + "'%2%'), at %3%", + v.string.s, v.string.context[0], pos); + } else { + throwEvalError( + "the string '%1%' is not allowed to refer to a store path (such as " + "'%2%')", + v.string.s, v.string.context[0]); + } + } + return s; +} + +bool EvalState::isDerivation(Value& v) { + if (v.type != tAttrs) { + return false; + } + Bindings::iterator i = v.attrs->find(sType); + if (i == v.attrs->end()) { + return false; + } + forceValue(*i->second.value); + if (i->second.value->type != tString) { + return false; + } + return strcmp(i->second.value->string.s, "derivation") == 0; +} + +std::optional<std::string> EvalState::tryAttrsToString(const Pos& pos, Value& v, + PathSet& context, + bool coerceMore, + bool copyToStore) { + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->second.value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + + return {}; +} + +std::string EvalState::coerceToString(const Pos& pos, Value& v, + PathSet& context, bool coerceMore, + bool copyToStore) { + forceValue(v); + + std::string s; + + if (v.type == tString) { + copyContext(v, context); + return v.string.s; + } + + if (v.type == tPath) { + Path path(canonPath(v.path)); + return copyToStore ? copyPathToStore(context, path) : path; + } + + if (v.type == tAttrs) { + auto maybeString = + tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; + } + auto i = v.attrs->find(sOutPath); + if (i == v.attrs->end()) { + throwTypeError("cannot coerce a set to a string, at %1%", pos); + } + return coerceToString(pos, *i->second.value, context, coerceMore, + copyToStore); + } + + if (coerceMore) { + /* Note that `false' is represented as an empty string for + shell scripting convenience, just like `null'. */ + if (v.type == tBool && v.boolean) { + return "1"; + } + if (v.type == tBool && !v.boolean) { + return ""; + } + if (v.type == tInt) { + return std::to_string(v.integer); + } + if (v.type == tFloat) { + return std::to_string(v.fpoint); + } + if (v.type == tNull) { + return ""; + } + + if (v.isList()) { + std::string result; + for (size_t n = 0; n < v.listSize(); ++n) { + result += coerceToString(pos, *(*v.list)[n], context, coerceMore, + copyToStore); + if (n < v.listSize() - 1 + /* !!! not quite correct */ + && (!(*v.list)[n]->isList() || (*v.list)[n]->listSize() != 0)) { + result += " "; + } + } + return result; + } + } + + throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); +} + +std::string EvalState::copyPathToStore(PathSet& context, const Path& path) { + if (nix::isDerivation(path)) { + throwEvalError("file names are not allowed to end in '%1%'", drvExtension); + } + + Path dstPath; + if (!srcToStore[path].empty()) { + dstPath = srcToStore[path]; + } else { + dstPath = + settings.readOnlyMode + ? store + ->computeStorePathForPath(baseNameOf(path), + checkSourcePath(path)) + .first + : store->addToStore(baseNameOf(path), checkSourcePath(path), true, + htSHA256, defaultPathFilter, repair); + srcToStore[path] = dstPath; + VLOG(2) << "copied source '" << path << "' -> '" << dstPath << "'"; + } + + context.insert(dstPath); + return dstPath; +} + +Path EvalState::coerceToPath(const Pos& pos, Value& v, PathSet& context) { + std::string path = coerceToString(pos, v, context, false, false); + if (path.empty() || path[0] != '/') { + throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", + path, pos); + } + return path; +} + +bool EvalState::eqValues(Value& v1, Value& v2) { + forceValue(v1); + forceValue(v2); + + /* !!! Hack to support some old broken code that relies on pointer + equality tests between sets. (Specifically, builderDefs calls + uniqList on a list of sets.) Will remove this eventually. */ + if (&v1 == &v2) { + return true; + } + + // Special case type-compatibility between float and int + if (v1.type == tInt && v2.type == tFloat) { + return v1.integer == v2.fpoint; + } + if (v1.type == tFloat && v2.type == tInt) { + return v1.fpoint == v2.integer; + } + + // All other types are not compatible with each other. + if (v1.type != v2.type) { + return false; + } + + switch (v1.type) { + case tInt: + return v1.integer == v2.integer; + + case tBool: + return v1.boolean == v2.boolean; + + case tString: + return strcmp(v1.string.s, v2.string.s) == 0; + + case tPath: + return strcmp(v1.path, v2.path) == 0; + + case tNull: + return true; + + case tList: + if (v1.listSize() != v2.listSize()) { + return false; + } + for (size_t n = 0; n < v1.listSize(); ++n) { + if (!eqValues(*(*v1.list)[n], *(*v2.list)[n])) { + return false; + } + } + return true; + + case tAttrs: { + // As an optimisation if both values are pointing towards the + // same attribute set, we can skip all this extra work. + if (v1.attrs == v2.attrs) { + return true; + } + + /* If both sets denote a derivation (type = "derivation"), + then compare their outPaths. */ + if (isDerivation(v1) && isDerivation(v2)) { + Bindings::iterator i = v1.attrs->find(sOutPath); + Bindings::iterator j = v2.attrs->find(sOutPath); + if (i != v1.attrs->end() && j != v2.attrs->end()) { + return eqValues(*i->second.value, *j->second.value); + } + } + + return v1.attrs->Equal(v2.attrs.get(), *this); + } + + /* Functions are incomparable. */ + case tLambda: + case tPrimOp: + case tPrimOpApp: + return false; + + case tFloat: + return v1.fpoint == v2.fpoint; + + default: + throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); + } +} + +void EvalState::printStats() { + bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + + struct rusage buf; + getrusage(RUSAGE_SELF, &buf); + float cpuTime = buf.ru_utime.tv_sec + + (static_cast<float>(buf.ru_utime.tv_usec) / 1000000); + + uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value*); + uint64_t bLists = nrListElems * sizeof(Value*); + uint64_t bValues = nrValues * sizeof(Value); + uint64_t bAttrsets = + nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); + + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); + std::fstream fs; + if (outPath != "-") { + fs.open(outPath, std::fstream::out); + } + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime", cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.Size()); + syms.attr("bytes", symbols.TotalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); + + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto& i : primOpCalls) { + obj.attr(i.first, i.second); + } + } + { + auto list = topObj.list("functions"); + for (auto& i : functionCalls) { + auto obj = list.object(); + if (i.first->name.has_value()) { + obj.attr("name", (const std::string&)i.first->name.value()); + } else { + obj.attr("name", nullptr); + } + if (i.first->pos) { + obj.attr("file", (const std::string&)i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); + } + } + { + auto list = topObj.list("attributes"); + for (auto& i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const std::string&)i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); + } + } + } + + // TODO(tazjin): what is this? commented out because .dump() is gone. + // if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + // auto list = topObj.list("symbols"); + // symbols.dump([&](const std::string& s) { list.elem(s); }); + // } + } +} + +void EvalState::TraceFileAccess(const Path& realPath) { + if (file_access_trace_fn) { + if (last_traced_file != realPath) { + file_access_trace_fn(realPath); + // Basic deduplication. + last_traced_file = std::string(realPath); + } + } +} + +void EvalState::EnableFileAccessTracing(std::function<void(const Path&)> fn) { + file_access_trace_fn = fn; +} + +size_t valueSize(const Value& v) { + traceable_flat_hash_set<const Bindings*> seenBindings; + traceable_flat_hash_set<const Env*> seenEnvs; + traceable_flat_hash_set<const NixList*> seenLists; + traceable_flat_hash_set<const char*> seenStrings; + traceable_flat_hash_set<const Value*> seenValues; + + auto doString = [&](const char* s) -> size_t { + if (seenStrings.find(s) != seenStrings.end()) { + return 0; + } + seenStrings.insert(s); + return strlen(s) + 1; + }; + + std::function<size_t(const Value& v)> doValue; + std::function<size_t(const Env& v)> doEnv; + + doValue = [&](const Value& v) -> size_t { + if (seenValues.find(&v) != seenValues.end()) { + return 0; + } + seenValues.insert(&v); + + size_t sz = sizeof(Value); + + switch (v.type) { + case tString: + sz += doString(v.string.s); + if (v.string.context != nullptr) { + for (const char** p = v.string.context; *p != nullptr; ++p) { + sz += doString(*p); + } + } + break; + case tPath: + sz += doString(v.path); + break; + case tAttrs: + if (seenBindings.find(v.attrs.get()) == seenBindings.end()) { + seenBindings.insert(v.attrs.get()); + sz += sizeof(Bindings); + for (const auto& i : *v.attrs) { + sz += doValue(*i.second.value); + } + } + break; + case tList: + if (seenLists.find(v.list.get()) == seenLists.end()) { + seenLists.insert(v.list.get()); + sz += v.listSize() * sizeof(Value*); + for (const Value* v : *v.list) { + sz += doValue(*v); + } + } + break; + case tThunk: + sz += doEnv(*v.thunk.env); + break; + case tApp: + sz += doValue(*v.app.left); + sz += doValue(*v.app.right); + break; + case tLambda: + sz += doEnv(*v.lambda.env); + break; + case tPrimOpApp: + sz += doValue(*v.primOpApp.left); + sz += doValue(*v.primOpApp.right); + break; + default:; + } + + return sz; + }; + + doEnv = [&](const Env& env) -> size_t { + if (seenEnvs.find(&env) != seenEnvs.end()) { + return 0; + } + seenEnvs.insert(&env); + + size_t sz = sizeof(Env) + sizeof(Value*) * env.size; + + if (env.type != Env::HasWithExpr) { + for (const Value* v : env.values) { + if (v != nullptr) { + sz += doValue(*v); + } + } + } else { + // TODO(kanepyork): trace ExprWith? how important is this accounting? + } + + if (env.up != nullptr) { + sz += doEnv(*env.up); + } + + return sz; + }; + + return doValue(v); +} + +EvalSettings evalSettings; + +static GlobalConfig::Register r1(&evalSettings); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/eval.hh b/third_party/nix/src/libexpr/eval.hh new file mode 100644 index 000000000000..0352a89a2ab8 --- /dev/null +++ b/third_party/nix/src/libexpr/eval.hh @@ -0,0 +1,365 @@ +#pragma once + +#include <map> +#include <optional> +#include <unordered_map> +#include <vector> + +#include "libexpr/attr-set.hh" +#include "libexpr/nixexpr.hh" +#include "libexpr/symbol-table.hh" +#include "libexpr/value.hh" +#include "libutil/config.hh" +#include "libutil/hash.hh" + +namespace nix { +namespace expr { + +// Initialise the Boehm GC once per program instance. This should be +// called in places that require the garbage collector. +void InitGC(); + +} // namespace expr + +class Store; +class EvalState; +enum RepairFlag : bool; + +typedef void (*PrimOpFun)(EvalState& state, const Pos& pos, Value** args, + Value& v); + +struct PrimOp { + PrimOpFun fun; + size_t arity; + Symbol name; + PrimOp(PrimOpFun fun, size_t arity, Symbol name) + : fun(fun), arity(arity), name(name) {} +}; + +struct Env { + Env(unsigned short size) : size(size) { values = std::vector<Value*>(size); } + + Env* up; + unsigned short size; // used by ‘valueSize’ + unsigned short prevWith : 14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type : 2; + std::vector<Value*> values; + Expr* withAttrsExpr = nullptr; +}; + +Value& mkString(Value& v, const std::string& s, + const PathSet& context = PathSet()); + +void copyContext(const Value& v, PathSet& context); + +/* Cache for calls to addToStore(); maps source paths to the store + paths. */ +using SrcToStore = std::map<Path, Path>; + +std::ostream& operator<<(std::ostream& str, const Value& v); + +using SearchPathElem = std::pair<std::string, std::string>; +using SearchPath = std::list<SearchPathElem>; + +using FileParseCache = std::map<Path, Expr*>; + +class EvalState { + public: + SymbolTable symbols; + + const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, + sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, + sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, sOutputHash, + sOutputHashAlgo, sOutputHashMode; + + // Symbol representing the path to the built-in 'derivation.nix' + // file, set during primops initialisation. + std::optional<Symbol> sDerivationNix; + + /* If set, force copying files to the Nix store even if they + already exist there. */ + RepairFlag repair; + + /* The allowed filesystem paths in restricted or pure evaluation + mode. */ + std::optional<PathSet> allowedPaths; + + const ref<Store> store; + + private: + SrcToStore srcToStore; + + /* A cache from path names to parse trees. */ + FileParseCache fileParseCache; + + /* A cache from path names to values. */ + using FileEvalCache = std::map<Path, Value>; + FileEvalCache fileEvalCache; + + SearchPath searchPath; + + std::map<std::string, std::pair<bool, std::string>> searchPathResolved; + + /* Cache used by checkSourcePath(). */ + std::unordered_map<Path, Path> resolvedPaths; + + public: + EvalState(const Strings& _searchPath, const ref<Store>& store); + ~EvalState(); + + void addToSearchPath(const std::string& s); + + SearchPath getSearchPath() { return searchPath; } + + Path checkSourcePath(const Path& path); + + void checkURI(const std::string& uri); + + /* When using a diverted store and 'path' is in the Nix store, map + 'path' to the diverted location (e.g. /nix/store/foo is mapped + to /home/alice/my-nix/nix/store/foo). However, this is only + done if the context is not empty, since otherwise we're + probably trying to read from the actual /nix/store. This is + intended to distinguish between import-from-derivation and + sources stored in the actual /nix/store. */ + Path toRealPath(const Path& path, const PathSet& context); + + /* Parse a Nix expression from the specified file. */ + Expr* parseExprFromFile(const Path& path); + Expr* parseExprFromFile(const Path& path, StaticEnv& staticEnv); + + /* Parse a Nix expression from the specified string. */ + Expr* parseExprFromString(const std::string& s, const Path& basePath, + StaticEnv& staticEnv); + Expr* parseExprFromString(const std::string& s, const Path& basePath); + + Expr* parseStdin(); + + /* Evaluate an expression read from the given file to normal + form. */ + void evalFile(const Path& path, Value& v); + + void resetFileCache(); + + /* Look up a file in the search path. */ + Path findFile(const std::string& path); + Path findFile(SearchPath& searchPath, const std::string& path, + const Pos& pos = noPos); + + /* If the specified search path element is a URI, download it. */ + std::pair<bool, std::string> resolveSearchPathElem( + const SearchPathElem& elem); + + /* Evaluate an expression to normal form, storing the result in + value `v'. */ + void eval(Expr* e, Value& v); + + /* Evaluation the expression, then verify that it has the expected + type. */ + inline bool evalBool(Env& env, Expr* e); + inline bool evalBool(Env& env, Expr* e, const Pos& pos); + inline void evalAttrs(Env& env, Expr* e, Value& v); + + /* If `v' is a thunk, enter it and overwrite `v' with the result + of the evaluation of the thunk. If `v' is a delayed function + application, call the function and overwrite `v' with the + result. Otherwise, this is a no-op. */ + inline void forceValue(Value& v, const Pos& pos = noPos); + + /* Force a value, then recursively force list elements and + attributes. */ + void forceValueDeep(Value& v); + + /* Force `v', and then verify that it has the expected type. */ + NixInt forceInt(Value& v, const Pos& pos); + NixFloat forceFloat(Value& v, const Pos& pos); + bool forceBool(Value& v, const Pos& pos); + inline void forceAttrs(Value& v); + inline void forceAttrs(Value& v, const Pos& pos); + inline void forceList(Value& v); + inline void forceList(Value& v, const Pos& pos); + void forceFunction(Value& v, const Pos& pos); // either lambda or primop + std::string forceString(Value& v, const Pos& pos = noPos); + std::string forceString(Value& v, PathSet& context, const Pos& pos = noPos); + std::string forceStringNoCtx(Value& v, const Pos& pos = noPos); + + /* Return true iff the value `v' denotes a derivation (i.e. a + set with attribute `type = "derivation"'). */ + bool isDerivation(Value& v); + + std::optional<std::string> tryAttrsToString(const Pos& pos, Value& v, + PathSet& context, + bool coerceMore = false, + bool copyToStore = true); + + /* String coercion. Converts strings, paths and derivations to a + string. If `coerceMore' is set, also converts nulls, integers, + booleans and lists to a string. If `copyToStore' is set, + referenced paths are copied to the Nix store as a side effect. */ + std::string coerceToString(const Pos& pos, Value& v, PathSet& context, + bool coerceMore = false, bool copyToStore = true); + + std::string copyPathToStore(PathSet& context, const Path& path); + + /* Path coercion. Converts strings, paths and derivations to a + path. The result is guaranteed to be a canonicalised, absolute + path. Nothing is copied to the store. */ + Path coerceToPath(const Pos& pos, Value& v, PathSet& context); + + public: + /* The base environment, containing the builtin functions and + values. */ + Env& baseEnv; + + /* The same, but used during parsing to resolve variables. */ + StaticEnv staticBaseEnv; // !!! should be private + + private: + unsigned int baseEnvDispl = 0; + + void createBaseEnv(); + + Value* addConstant(const std::string& name, Value& v); + + Value* addPrimOp(const std::string& name, size_t arity, PrimOpFun primOp); + + public: + Value& getBuiltin(const std::string& name); + + private: + inline Value* lookupVar(Env* env, const ExprVar& var, bool noEval); + + friend struct ExprVar; + friend struct ExprAttrs; + friend struct ExprLet; + + Expr* parse(const char* text, const Path& path, const Path& basePath, + StaticEnv& staticEnv); + + public: + /* Do a deep equality test between two values. That is, list + elements and attributes are compared recursively. */ + bool eqValues(Value& v1, Value& v2); + + bool isFunctor(Value& fun); + + void callFunction(Value& fun, Value& arg, Value& v, const Pos& pos); + void callPrimOp(Value& fun, Value& arg, Value& v, const Pos& pos); + + /* Automatically call a function for which each argument has a + default value or has a binding in the `args' map. 'args' need + not live past the end of the call. */ + void autoCallFunction(Bindings* args, Value& fun, Value& res); + + /* Allocation primitives. */ + Value* allocValue(); + Env& allocEnv(size_t size); + + Value* allocAttr(Value& vAttrs, const Symbol& name); + + // Create a list value from the specified vector. + void mkList(Value& v, std::shared_ptr<NixList> list); + + // Create a list value, allocating as many elements as specified in + // size. This is used for the many cases in this codebase where + // assignment happens into the preallocated list. + void mkList(Value& v, size_t size = 0); + + void mkAttrs(Value& v, size_t capacity); + void mkThunk_(Value& v, Expr* expr); + void mkPos(Value& v, Pos* pos); + + void concatLists(Value& v, const NixList& lists, const Pos& pos); + + /* Print statistics. */ + void printStats(); + + void realiseContext(const PathSet& context); + + /* File access tracing. */ + void TraceFileAccess(const Path& path); + void EnableFileAccessTracing(std::function<void(const Path&)> fn); + + private: + unsigned long nrEnvs = 0; + unsigned long nrValuesInEnvs = 0; + unsigned long nrValues = 0; + unsigned long nrListElems = 0; + unsigned long nrAttrsets = 0; + unsigned long nrAttrsInAttrsets = 0; + unsigned long nrOpUpdates = 0; + unsigned long nrOpUpdateValuesCopied = 0; + unsigned long nrListConcats = 0; + unsigned long nrPrimOpCalls = 0; + unsigned long nrFunctionCalls = 0; + + bool countCalls; + + std::function<void(const Path&)> file_access_trace_fn = nullptr; + Path last_traced_file = ""; + + using PrimOpCalls = std::map<Symbol, size_t>; + PrimOpCalls primOpCalls; + + using FunctionCalls = std::map<ExprLambda*, size_t>; + FunctionCalls functionCalls; + + void incrFunctionCall(ExprLambda* fun); + + using AttrSelects = std::map<Pos, size_t>; + AttrSelects attrSelects; + + friend struct ExprOpUpdate; + friend struct ExprOpConcatLists; + friend struct ExprSelect; + friend void prim_getAttr(EvalState& state, const Pos& pos, Value** args, + Value& v); +}; + +/* Return a string representing the type of the value `v'. */ +std::string showType(const Value& v); + +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<std::string, std::string> decodeContext(const std::string& s); + +/* If `path' refers to a directory, then append "/default.nix". */ +Path resolveExprPath(Path path); + +struct InvalidPathError : EvalError { + Path path; + InvalidPathError(const Path& path); +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~InvalidPathError() noexcept {}; +#endif +}; + +struct EvalSettings : Config { + Setting<bool> restrictEval{ + this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting<bool> pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access " + "to files specified by cryptographic hash."}; + + Setting<bool> enableImportFromDerivation{ + this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting<Strings> allowedUris{ + this, + {}, + "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit " + "are allowed to fetch."}; + + Setting<bool> traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry " + "and exit at the 'vomit' log level (-vvvv)"}; +}; + +extern EvalSettings evalSettings; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/function-trace.cc b/third_party/nix/src/libexpr/function-trace.cc new file mode 100644 index 000000000000..b1b856965c2a --- /dev/null +++ b/third_party/nix/src/libexpr/function-trace.cc @@ -0,0 +1,19 @@ +#include "libexpr/function-trace.hh" + +#include <glog/logging.h> + +namespace nix { + +FunctionCallTrace::FunctionCallTrace(const Pos& pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + LOG(INFO) << "function-trace entered " << pos << " at " << ns.count(); +} + +FunctionCallTrace::~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast<std::chrono::nanoseconds>(duration); + LOG(INFO) << "function-trace exited " << pos << " at " << ns.count(); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/function-trace.hh b/third_party/nix/src/libexpr/function-trace.hh new file mode 100644 index 000000000000..6b810159b8fc --- /dev/null +++ b/third_party/nix/src/libexpr/function-trace.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <chrono> + +#include "libexpr/eval.hh" + +namespace nix { + +struct FunctionCallTrace { + const Pos& pos; + FunctionCallTrace(const Pos& pos); + ~FunctionCallTrace(); +}; +} // namespace nix diff --git a/third_party/nix/src/libexpr/get-drvs.cc b/third_party/nix/src/libexpr/get-drvs.cc new file mode 100644 index 000000000000..164c1e54f38e --- /dev/null +++ b/third_party/nix/src/libexpr/get-drvs.cc @@ -0,0 +1,446 @@ +#include "libexpr/get-drvs.hh" + +#include <cstring> +#include <regex> +#include <utility> + +#include <absl/container/flat_hash_set.h> +#include <absl/strings/numbers.h> +#include <glog/logging.h> + +#include "libexpr/eval-inline.hh" +#include "libstore/derivations.hh" +#include "libutil/util.hh" + +namespace nix { + +DrvInfo::DrvInfo(EvalState& state, std::string attrPath, + std::shared_ptr<Bindings> attrs) + : state(&state), attrs(attrs), attrPath(std::move(attrPath)) {} + +DrvInfo::DrvInfo(EvalState& state, const ref<Store>& store, + const std::string& drvPathWithOutputs) + : state(&state), attrPath("") { + auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); + + drvPath = spec.first; + + auto drv = store->derivationFromPath(drvPath); + + name = storePathToName(drvPath); + + if (spec.second.size() > 1) { + throw Error( + "building more than one derivation output is not supported, in '%s'", + drvPathWithOutputs); + } + + outputName = spec.second.empty() ? get(drv.env, "outputName", "out") + : *spec.second.begin(); + + auto i = drv.outputs.find(outputName); + if (i == drv.outputs.end()) { + throw Error("derivation '%s' does not have output '%s'", drvPath, + outputName); + } + + outPath = i->second.path; +} + +std::string DrvInfo::queryName() const { + if (name.empty() && (attrs != nullptr)) { + auto i = attrs->find(state->sName); + if (i == attrs->end()) { + throw TypeError("derivation name missing"); + } + name = state->forceStringNoCtx(*i->second.value); + } + return name; +} + +std::string DrvInfo::querySystem() const { + if (system.empty() && (attrs != nullptr)) { + auto i = attrs->find(state->sSystem); + system = i == attrs->end() + ? "unknown" + : state->forceStringNoCtx(*i->second.value, *i->second.pos); + } + return system; +} + +std::string DrvInfo::queryDrvPath() const { + if (drvPath.empty() && (attrs != nullptr)) { + Bindings::iterator i = attrs->find(state->sDrvPath); + PathSet context; + drvPath = i != attrs->end() ? state->coerceToPath(*i->second.pos, + *i->second.value, context) + : ""; + } + return drvPath; +} + +std::string DrvInfo::queryOutPath() const { + if (outPath.empty() && (attrs != nullptr)) { + Bindings::iterator i = attrs->find(state->sOutPath); + PathSet context; + outPath = i != attrs->end() ? state->coerceToPath(*i->second.pos, + *i->second.value, context) + : ""; + } + return outPath; +} + +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) { + if (outputs.empty()) { + /* Get the ‘outputs’ list. */ + Bindings::iterator i; + if ((attrs != nullptr) && + (i = attrs->find(state->sOutputs)) != attrs->end()) { + state->forceList(*i->second.value, *i->second.pos); + + /* For each output... */ + for (unsigned int j = 0; j < i->second.value->listSize(); ++j) { + /* Evaluate the corresponding set. */ + std::string name = state->forceStringNoCtx(*(*i->second.value->list)[j], + *i->second.pos); + Bindings::iterator out = attrs->find(state->symbols.Create(name)); + if (out == attrs->end()) { + continue; // FIXME: throw error? + } + state->forceAttrs(*out->second.value); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = + out->second.value->attrs->find(state->sOutPath); + if (outPath == out->second.value->attrs->end()) { + continue; // FIXME: throw error? + } + PathSet context; + outputs[name] = state->coerceToPath(*outPath->second.pos, + *outPath->second.value, context); + } + } else { + outputs["out"] = queryOutPath(); + } + } + if (!onlyOutputsToInstall || (attrs == nullptr)) { + return outputs; + } + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value* outTI = queryMeta("outputsToInstall"); + if (outTI == nullptr) { + return outputs; + } + const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) { + throw errMsg; + } + Outputs result; + + for (Value* i : *outTI->list) { + if (i->type != tString) { + throw errMsg; + } + auto out = outputs.find(i->string.s); + if (out == outputs.end()) { + throw errMsg; + } + result.insert(*out); + } + return result; +} + +std::string DrvInfo::queryOutputName() const { + if (outputName.empty() && (attrs != nullptr)) { + Bindings::iterator i = attrs->find(state->sOutputName); + outputName = + i != attrs->end() ? state->forceStringNoCtx(*i->second.value) : ""; + } + return outputName; +} + +Bindings* DrvInfo::getMeta() { + if (meta != nullptr) { + return meta.get(); + } + if (attrs == nullptr) { + return nullptr; + } + Bindings::iterator a = attrs->find(state->sMeta); + if (a == attrs->end()) { + return nullptr; + } + state->forceAttrs(*a->second.value, *a->second.pos); + meta = a->second.value->attrs; + return meta.get(); +} + +StringSet DrvInfo::queryMetaNames() { + StringSet res; + if (getMeta() == nullptr) { + return res; + } + for (auto& i : *meta) { + res.insert(i.second.name); + } + return res; +} + +bool DrvInfo::checkMeta(Value& v) { + state->forceValue(v); + if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (!checkMeta(*(*v.list)[n])) { + return false; + } + } + return true; + } + if (v.type == tAttrs) { + Bindings::iterator i = v.attrs->find(state->sOutPath); + if (i != v.attrs->end()) { + return false; + } + for (auto& i : *v.attrs) { + if (!checkMeta(*i.second.value)) { + return false; + } + } + return true; + } else { + return v.type == tInt || v.type == tBool || v.type == tString || + v.type == tFloat; + } +} + +Value* DrvInfo::queryMeta(const std::string& name) { + if (getMeta() == nullptr) { + return nullptr; + } + Bindings::iterator a = meta->find(state->symbols.Create(name)); + if (a == meta->end() || !checkMeta(*a->second.value)) { + return nullptr; + } + return a->second.value; +} + +std::string DrvInfo::queryMetaString(const std::string& name) { + Value* v = queryMeta(name); + if ((v == nullptr) || v->type != tString) { + return ""; + } + return v->string.s; +} + +NixInt DrvInfo::queryMetaInt(const std::string& name, NixInt def) { + Value* v = queryMeta(name); + if (v == nullptr) { + return def; + } + if (v->type == tInt) { + return v->integer; + } + if (v->type == tString) { + /* Backwards compatibility with before we had support for + integer meta fields. */ + NixInt n; + if (absl::SimpleAtoi(v->string.s, &n)) { + return n; + } + } + return def; +} + +NixFloat DrvInfo::queryMetaFloat(const std::string& name, NixFloat def) { + Value* v = queryMeta(name); + if (v == nullptr) { + return def; + } + if (v->type == tFloat) { + return v->fpoint; + } + if (v->type == tString) { + /* Backwards compatibility with before we had support for + float meta fields. */ + NixFloat n; + if (string2Float(v->string.s, n)) { + return n; + } + } + return def; +} + +bool DrvInfo::queryMetaBool(const std::string& name, bool def) { + Value* v = queryMeta(name); + if (v == nullptr) { + return def; + } + if (v->type == tBool) { + return v->boolean; + } + if (v->type == tString) { + /* Backwards compatibility with before we had support for + Boolean meta fields. */ + if (strcmp(v->string.s, "true") == 0) { + return true; + } + if (strcmp(v->string.s, "false") == 0) { + return false; + } + } + return def; +} + +void DrvInfo::setMeta(const std::string& name, Value* v) { + std::shared_ptr<Bindings> old = meta; + meta = std::shared_ptr<Bindings>(Bindings::New(old->size() + 1).release()); + Symbol sym = state->symbols.Create(name); + if (old != nullptr) { + for (auto i : *old) { + if (i.second.name != sym) { + meta->push_back(i.second); + } + } + } + if (v != nullptr) { + meta->push_back(Attr(sym, v)); + } +} + +/* Cache for already considered attrsets. */ +using Done = absl::flat_hash_set<std::shared_ptr<Bindings>>; + +/* Evaluate value `v'. If it evaluates to a set of type `derivation', + then put information about it in `drvs' (unless it's already in `done'). + The result boolean indicates whether it makes sense + for the caller to recursively search for derivations in `v'. */ +static bool getDerivation(EvalState& state, Value& v, + const std::string& attrPath, DrvInfos& drvs, + Done& done, bool ignoreAssertionFailures) { + try { + state.forceValue(v); + if (!state.isDerivation(v)) { + return true; + } + + /* Remove spurious duplicates (e.g., a set like `rec { x = + derivation {...}; y = x;}'. */ + if (done.find(v.attrs) != done.end()) { + return false; + } + done.insert(v.attrs); + + DrvInfo drv(state, attrPath, v.attrs); + + drv.queryName(); + + drvs.push_back(drv); + + return false; + + } catch (AssertionError& e) { + if (ignoreAssertionFailures) { + return false; + } + throw; + } +} + +std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, + bool ignoreAssertionFailures) { + Done done; + DrvInfos drvs; + getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); + if (drvs.size() != 1) { + return {}; + } + return std::move(drvs.front()); +} + +static std::string addToPath(const std::string& s1, const std::string& s2) { + return s1.empty() ? s2 : s1 + "." + s2; +} + +static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); + +static void getDerivations(EvalState& state, Value& vIn, + const std::string& pathPrefix, Bindings* autoArgs, + DrvInfos& drvs, Done& done, + bool ignoreAssertionFailures) { + Value v; + state.autoCallFunction(autoArgs, vIn, v); + + /* Process the expression. */ + if (!getDerivation(state, v, pathPrefix, drvs, done, + ignoreAssertionFailures)) { + ; + + } else if (v.type == tAttrs) { + /* !!! undocumented hackery to support combining channels in + nix-env.cc. */ + bool combineChannels = + v.attrs->find(state.symbols.Create("_combineChannels")) != + v.attrs->end(); + + /* Consider the attributes in sorted order to get more + deterministic behaviour in nix-env operations (e.g. when + there are names clashes between derivations, the derivation + bound to the attribute with the "lower" name should take + precedence). */ + for (auto& [_, i] : *v.attrs) { + DLOG(INFO) << "evaluating attribute '" << i.name << "'"; + if (!std::regex_match(std::string(i.name), attrRegex)) { + continue; + } + std::string pathPrefix2 = addToPath(pathPrefix, i.name); + if (combineChannels) { + getDerivations(state, *i.value, pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + } else if (getDerivation(state, *i.value, pathPrefix2, drvs, done, + ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i.value->type == tAttrs) { + Bindings::iterator j = i.value->attrs->find( + state.symbols.Create("recurseForDerivations")); + if (j != i.value->attrs->end() && + state.forceBool(*j->second.value, *j->second.pos)) { + getDerivations(state, *i.value, pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + } + } + } + } + } + + else if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + std::string pathPrefix2 = + addToPath(pathPrefix, (format("%1%") % n).str()); + if (getDerivation(state, *(*v.list)[n], pathPrefix2, drvs, done, + ignoreAssertionFailures)) { + getDerivations(state, *(*v.list)[n], pathPrefix2, autoArgs, drvs, done, + ignoreAssertionFailures); + } + } + } + + else { + throw TypeError( + "expression does not evaluate to a derivation (or a set or list of " + "those)"); + } +} + +void getDerivations(EvalState& state, Value& v, const std::string& pathPrefix, + Bindings* autoArgs, DrvInfos& drvs, + bool ignoreAssertionFailures) { + Done done; + getDerivations(state, v, pathPrefix, autoArgs, drvs, done, + ignoreAssertionFailures); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/get-drvs.hh b/third_party/nix/src/libexpr/get-drvs.hh new file mode 100644 index 000000000000..3de266d0c0e7 --- /dev/null +++ b/third_party/nix/src/libexpr/get-drvs.hh @@ -0,0 +1,83 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" + +namespace nix { + +struct DrvInfo { + public: + typedef std::map<std::string, Path> Outputs; + + private: + EvalState* state; + + mutable std::string name; + mutable std::string system; + mutable std::string drvPath; + mutable std::string outPath; + mutable std::string outputName; + Outputs outputs; + + bool failed = false; // set if we get an AssertionError + + std::shared_ptr<Bindings> attrs = nullptr; + std::shared_ptr<Bindings> meta = nullptr; + + Bindings* getMeta(); + + bool checkMeta(Value& v); + + public: + std::string attrPath; /* path towards the derivation */ + + DrvInfo(EvalState& state) : state(&state){}; + DrvInfo(EvalState& state, std::string attrPath, + std::shared_ptr<Bindings> attrs); + DrvInfo(EvalState& state, const ref<Store>& store, + const std::string& drvPathWithOutputs); + + std::string queryName() const; + std::string querySystem() const; + std::string queryDrvPath() const; + std::string queryOutPath() const; + std::string queryOutputName() const; + /** Return the list of outputs. The "outputs to install" are determined by + * `meta.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); + + StringSet queryMetaNames(); + Value* queryMeta(const std::string& name); + std::string queryMetaString(const std::string& name); + NixInt queryMetaInt(const std::string& name, NixInt def); + NixFloat queryMetaFloat(const std::string& name, NixFloat def); + bool queryMetaBool(const std::string& name, bool def); + void setMeta(const std::string& name, Value* v); + + /* + MetaInfo queryMetaInfo(EvalState & state) const; + MetaValue queryMetaInfo(EvalState & state, const std::string & name) const; + */ + + void setName(const std::string& s) { name = s; } + void setDrvPath(const std::string& s) { drvPath = s; } + void setOutPath(const std::string& s) { outPath = s; } + + void setFailed() { failed = true; }; + bool hasFailed() { return failed; }; +}; + +using DrvInfos = std::list<DrvInfo>; + +/* If value `v' denotes a derivation, return a DrvInfo object + describing it. Otherwise return nothing. */ +std::optional<DrvInfo> getDerivation(EvalState& state, Value& v, + bool ignoreAssertionFailures); + +void getDerivations(EvalState& state, Value& v, const std::string& pathPrefix, + Bindings* autoArgs, DrvInfos& drvs, + bool ignoreAssertionFailures); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/json-to-value.cc b/third_party/nix/src/libexpr/json-to-value.cc new file mode 100644 index 000000000000..043f8c64cd30 --- /dev/null +++ b/third_party/nix/src/libexpr/json-to-value.cc @@ -0,0 +1,152 @@ +#include "libexpr/json-to-value.hh" + +#include <nlohmann/json.hpp> +#include <variant> +#include <vector> + +#include "libexpr/value.hh" + +using json = nlohmann::json; + +namespace nix { + +// for more information, refer to +// https://github.com/nlohmann/json/blob/master/include/nlohmann/detail/input/json_sax.hpp +class JSONSax : nlohmann::json_sax<json> { + class JSONState { + protected: + std::unique_ptr<JSONState> parent; + std::shared_ptr<Value*> v; + + public: + virtual std::unique_ptr<JSONState> resolve(EvalState&) { + throw std::logic_error("tried to close toplevel json parser state"); + } + explicit JSONState(std::unique_ptr<JSONState>&& p) : parent(std::move(p)) {} + explicit JSONState(Value* v) : v(allocRootValue(v)) {} + JSONState(JSONState& p) = delete; + Value& value(EvalState& state) { + if (!v) v = allocRootValue(state.allocValue()); + return **v; + } + virtual ~JSONState() {} + virtual void add() {} + }; + + class JSONObjectState : public JSONState { + using JSONState::JSONState; + ValueMap attrs = ValueMap(); + std::unique_ptr<JSONState> resolve(EvalState& state) override { + Value& v = parent->value(state); + state.mkAttrs(v, attrs.size()); + for (auto& i : attrs) v.attrs->push_back(Attr(i.first, i.second)); + return std::move(parent); + } + void add() override { v = nullptr; }; + + public: + void key(string_t& name, EvalState& state) { + attrs[state.symbols.Create(name)] = &value(state); + } + }; + + class JSONListState : public JSONState { + using JSONState::JSONState; + std::vector<Value*> values; + std::unique_ptr<JSONState> resolve(EvalState& state) override { + Value& v = parent->value(state); + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) { + (*v.list)[n] = values[n]; + } + return std::move(parent); + } + void add() override { + values.push_back(*v); + v = nullptr; + }; + + public: + JSONListState(std::unique_ptr<JSONState>&& p, std::size_t reserve) + : JSONState(std::move(p)) { + values.reserve(reserve); + } + }; + + EvalState& state; + std::unique_ptr<JSONState> rs; + + template <typename T, typename... Args> + inline bool handle_value(T f, Args... args) { + f(rs->value(state), args...); + rs->add(); + return true; + } + + public: + JSONSax(EvalState& state, Value& v) : state(state), rs(new JSONState(&v)){}; + + bool null() override { return handle_value(mkNull); } + + bool boolean(bool val) override { return handle_value(mkBool, val); } + + bool number_integer(number_integer_t val) override { + return handle_value(mkInt, val); + } + + bool number_unsigned(number_unsigned_t val) override { + return handle_value(mkInt, val); + } + + bool number_float(number_float_t val, const string_t&) override { + return handle_value(mkFloat, val); + } + + bool string(string_t& val) override { + return handle_value<void(Value&, const char*)>(mkString, val.c_str()); + } + +#if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 + bool binary(binary_t&) { + // This function ought to be unreachable + assert(false); + return true; + } +#endif + + bool start_object(std::size_t) override { + rs = std::make_unique<JSONObjectState>(std::move(rs)); + return true; + } + + bool key(string_t& name) override { + dynamic_cast<JSONObjectState*>(rs.get())->key(name, state); + return true; + } + + bool end_object() override { + rs = rs->resolve(state); + rs->add(); + return true; + } + + bool end_array() override { return end_object(); } + + bool start_array(size_t len) override { + rs = std::make_unique<JSONListState>( + std::move(rs), len != std::numeric_limits<size_t>::max() ? len : 128); + return true; + } + + bool parse_error(std::size_t, const std::string&, + const nlohmann::detail::exception& ex) override { + throw JSONParseError(ex.what()); + } +}; + +void parseJSON(EvalState& state, const std::string& s_, Value& v) { + JSONSax parser(state, v); + bool res = json::sax_parse(s_, &parser); + if (!res) throw JSONParseError("Invalid JSON Value"); +} +} // namespace nix diff --git a/third_party/nix/src/libexpr/json-to-value.hh b/third_party/nix/src/libexpr/json-to-value.hh new file mode 100644 index 000000000000..7f258f2137ba --- /dev/null +++ b/third_party/nix/src/libexpr/json-to-value.hh @@ -0,0 +1,13 @@ +#pragma once + +#include <string> + +#include "libexpr/eval.hh" + +namespace nix { + +MakeError(JSONParseError, EvalError); + +void parseJSON(EvalState& state, const std::string& s, Value& v); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/lexer.l b/third_party/nix/src/libexpr/lexer.l new file mode 100644 index 000000000000..d5b8a459363a --- /dev/null +++ b/third_party/nix/src/libexpr/lexer.l @@ -0,0 +1,193 @@ +%option reentrant bison-bridge bison-locations +%option noyywrap +%option never-interactive +%option stack +%option nodefault +%option nounput noyy_top_state + + +%s DEFAULT +%x STRING +%x IND_STRING + + +%{ +#include <boost/lexical_cast.hpp> + +#include "generated/parser-tab.hh" +#include "libexpr/nixexpr.hh" +#include "libexpr/parser.hh" + +using namespace nix; + +namespace nix { + +static void initLoc(YYLTYPE* loc) { + loc->first_line = loc->last_line = 1; + loc->first_column = loc->last_column = 1; +} + +static void adjustLoc(YYLTYPE* loc, const char* s, size_t len) { + loc->first_line = loc->last_line; + loc->first_column = loc->last_column; + + while (len--) { + switch (*s++) { + case '\r': + if (*s == '\n') /* cr/lf */ + s++; + /* fall through */ + case '\n': + ++loc->last_line; + loc->last_column = 1; + break; + default: + ++loc->last_column; + } + } +} + +} + +#define YY_USER_INIT initLoc(yylloc) +#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); + +#define PUSH_STATE(state) yy_push_state(state, yyscanner) +#define POP_STATE() yy_pop_state(yyscanner) + +%} + + +ANY .|\n +ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* +INT [0-9]+ +FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? +PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> +URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ + + +%% + + +if { return IF; } +then { return THEN; } +else { return ELSE; } +assert { return ASSERT; } +with { return WITH; } +let { return LET; } +in { return IN; } +rec { return REC; } +inherit { return INHERIT; } +or { return OR_KW; } +\.\.\. { return ELLIPSIS; } + +\=\= { return EQ; } +\!\= { return NEQ; } +\<\= { return LEQ; } +\>\= { return GEQ; } +\&\& { return AND; } +\|\| { return OR; } +\-\> { return IMPL; } +\/\/ { return UPDATE; } +\+\+ { return CONCAT; } + +{ID} { yylval->id = strdup(yytext); return ID; } +{INT} { errno = 0; + try { + yylval->n = boost::lexical_cast<int64_t>(yytext); + } catch (const boost::bad_lexical_cast &) { + throw ParseError(format("invalid integer '%1%'") % yytext); + } + return INT; + } +{FLOAT} { errno = 0; + yylval->nf = strtod(yytext, 0); + if (errno != 0) + throw ParseError(format("invalid float '%1%'") % yytext); + return FLOAT; + } + +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } + +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; + } +\{ { PUSH_STATE(DEFAULT); return '{'; } + +\" { PUSH_STATE(STRING); return '"'; } +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | +<STRING>([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext, yyleng); + return STR; + } +<STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +<STRING>\" { POP_STATE(); return '"'; } +<STRING>\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } + +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +<IND_STRING>([^\$\']|\$[^\{\']|\'[^\'\$])+ { + yylval->e = new ExprIndStr(yytext); + return IND_STR; + } +<IND_STRING>\'\'\$ | +<IND_STRING>\$ { + yylval->e = new ExprIndStr("$"); + return IND_STR; + } +<IND_STRING>\'\'\' { + yylval->e = new ExprIndStr("''"); + return IND_STR; + } +<IND_STRING>\'\'\\{ANY} { + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + return IND_STR; + } +<IND_STRING>\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +<IND_STRING>\'\' { POP_STATE(); return IND_STRING_CLOSE; } +<IND_STRING>\' { + yylval->e = new ExprIndStr("'"); + return IND_STR; + } + + +{PATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path '%s' has a trailing slash", yytext); + yylval->path = strdup(yytext); + return PATH; + } +{HPATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path '%s' has a trailing slash", yytext); + yylval->path = strdup(yytext); + return HPATH; + } +{SPATH} { yylval->path = strdup(yytext); return SPATH; } +{URI} { yylval->uri = strdup(yytext); return URI; } + +[ \t\r\n]+ /* eat up whitespace */ +\#[^\r\n]* /* single-line comments */ +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ + +{ANY} { + /* Don't return a negative number, as this will cause + Bison to stop parsing without an error. */ + return (unsigned char) yytext[0]; + } + +%% + diff --git a/third_party/nix/src/libexpr/names.cc b/third_party/nix/src/libexpr/names.cc new file mode 100644 index 000000000000..1e9c2f2f4aac --- /dev/null +++ b/third_party/nix/src/libexpr/names.cc @@ -0,0 +1,121 @@ +#include "libexpr/names.hh" + +#include <memory> + +#include <absl/strings/numbers.h> + +#include "libutil/util.hh" + +namespace nix { + +DrvName::DrvName() { name = ""; } + +/* Parse a derivation name. The `name' part of a derivation name is + everything up to but not including the first dash *not* followed by + a letter. The `version' part is the rest (excluding the separating + dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', + '2.0.48'). */ +DrvName::DrvName(const std::string& s) : hits(0) { + name = fullName = s; + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && (isalpha(s[i + 1]) == 0)) { + name = std::string(s, 0, i); + version = std::string(s, i + 1); + break; + } + } +} + +bool DrvName::matches(DrvName& n) { + if (name != "*") { + if (!regex) { + regex = std::make_unique<std::regex>(name, std::regex::extended); + } + if (!std::regex_match(n.name, *regex)) { + return false; + } + } + return !(!version.empty() && version != n.version); +} + +std::string nextComponent(std::string::const_iterator& p, + const std::string::const_iterator end) { + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) { + ++p; + } + + if (p == end) { + return ""; + } + + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + std::string s; + if (isdigit(*p) != 0) { + while (p != end && (isdigit(*p) != 0)) { + s += *p++; + } + } else { + while (p != end && ((isdigit(*p) == 0) && *p != '.' && *p != '-')) { + s += *p++; + } + } + + return s; +} + +static bool componentsLT(const std::string& c1, const std::string& c2) { + int n1; + int n2; + bool c1Num = absl::SimpleAtoi(c1, &n1); + bool c2Num = absl::SimpleAtoi(c2, &n2); + + if (c1Num && c2Num) { + return n1 < n2; + } + if (c1.empty() && c2Num) { + return true; + } else if (c1 == "pre" && c2 != "pre") { + return true; + } else if (c2 == "pre") { + return false; + /* Assume that `2.3a' < `2.3.1'. */ + } else if (c2Num) { + return true; + } else if (c1Num) { + return false; + } else { + return c1 < c2; + } +} + +int compareVersions(const std::string& v1, const std::string& v2) { + std::string::const_iterator p1 = v1.begin(); + std::string::const_iterator p2 = v2.begin(); + + while (p1 != v1.end() || p2 != v2.end()) { + std::string c1 = nextComponent(p1, v1.end()); + std::string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) { + return -1; + } + if (componentsLT(c2, c1)) { + return 1; + } + } + + return 0; +} + +DrvNames drvNamesFromArgs(const Strings& opArgs) { + DrvNames result; + for (auto& i : opArgs) { + result.push_back(DrvName(i)); + } + return result; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/names.hh b/third_party/nix/src/libexpr/names.hh new file mode 100644 index 000000000000..061388d517cd --- /dev/null +++ b/third_party/nix/src/libexpr/names.hh @@ -0,0 +1,31 @@ +#pragma once + +#include <memory> +#include <regex> + +#include "libutil/types.hh" + +namespace nix { + +struct DrvName { + std::string fullName; + std::string name; + std::string version; + unsigned int hits; + + DrvName(); + DrvName(const std::string& s); + bool matches(DrvName& n); + + private: + std::unique_ptr<std::regex> regex; +}; + +typedef std::list<DrvName> DrvNames; + +std::string nextComponent(std::string::const_iterator& p, + const std::string::const_iterator end); +int compareVersions(const std::string& v1, const std::string& v2); +DrvNames drvNamesFromArgs(const Strings& opArgs); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/nix-expr.pc.in b/third_party/nix/src/libexpr/nix-expr.pc.in new file mode 100644 index 000000000000..99b0ae2c68ec --- /dev/null +++ b/third_party/nix/src/libexpr/nix-expr.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-store bdw-gc +Libs: -L${libdir} -lnixexpr +Cflags: -I${includedir}/nix diff --git a/third_party/nix/src/libexpr/nixexpr.cc b/third_party/nix/src/libexpr/nixexpr.cc new file mode 100644 index 000000000000..391f0682059c --- /dev/null +++ b/third_party/nix/src/libexpr/nixexpr.cc @@ -0,0 +1,414 @@ +#include "libexpr/nixexpr.hh" + +#include <cstdlib> +#include <variant> + +#include "libstore/derivations.hh" +#include "libutil/util.hh" +#include "libutil/visitor.hh" + +namespace nix { + +/* Displaying abstract syntax trees. */ + +std::ostream& operator<<(std::ostream& str, const Expr& e) { + e.show(str); + return str; +} + +static void showString(std::ostream& str, const std::string& s) { + str << '"'; + for (auto c : std::string(s)) { + if (c == '"' || c == '\\' || c == '$') { + str << "\\" << c; + } else if (c == '\n') { + str << "\\n"; + } else if (c == '\r') { + str << "\\r"; + } else if (c == '\t') { + str << "\\t"; + } else { + str << c; + } + } + str << '"'; +} + +static void showId(std::ostream& str, const std::string& s) { + if (s.empty()) { + str << "\"\""; + } else if (s == "if") { // FIXME: handle other keywords + str << '"' << s << '"'; + } else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + showString(str, s); + return; + } + for (auto c : s) { + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || c == '_' || c == '\'' || c == '-')) { + showString(str, s); + return; + } + } + str << s; + } +} + +std::ostream& operator<<(std::ostream& str, const Symbol& sym) { + showId(str, *sym.s); + return str; +} + +void Expr::show(std::ostream& str) const { abort(); } + +void ExprInt::show(std::ostream& str) const { str << n; } + +void ExprFloat::show(std::ostream& str) const { str << nf; } + +void ExprString::show(std::ostream& str) const { showString(str, s); } + +void ExprPath::show(std::ostream& str) const { str << s; } + +void ExprVar::show(std::ostream& str) const { str << name; } + +void ExprSelect::show(std::ostream& str) const { + str << "(" << *e << ")." << showAttrPath(attrPath); + if (def != nullptr) { + str << " or (" << *def << ")"; + } +} + +void ExprOpHasAttr::show(std::ostream& str) const { + str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; +} + +void ExprAttrs::show(std::ostream& str) const { + if (recursive) { + str << "rec "; + } + str << "{ "; + for (auto& i : attrs) { + if (i.second.inherited) { + str << "inherit " << i.first << " " + << "; "; + } else { + str << i.first << " = " << *i.second.e << "; "; + } + } + for (auto& i : dynamicAttrs) { + str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; + } + str << "}"; +} + +void ExprList::show(std::ostream& str) const { + str << "[ "; + for (auto& i : elems) { + str << "(" << *i << ") "; + } + str << "]"; +} + +void ExprLambda::show(std::ostream& str) const { + str << "("; + if (matchAttrs) { + str << "{ "; + bool first = true; + for (auto& i : formals->formals) { + if (first) { + first = false; + } else { + str << ", "; + } + str << i.name; + if (i.def != nullptr) { + str << " ? " << *i.def; + } + } + if (formals->ellipsis) { + if (!first) { + str << ", "; + } + str << "..."; + } + str << " }"; + if (!arg.empty()) { + str << " @ "; + } + } + if (!arg.empty()) { + str << arg; + } + str << ": " << *body << ")"; +} + +void ExprLet::show(std::ostream& str) const { + str << "(let "; + for (auto& i : attrs->attrs) { + if (i.second.inherited) { + str << "inherit " << i.first << "; "; + } else { + str << i.first << " = " << *i.second.e << "; "; + } + } + str << "in " << *body << ")"; +} + +void ExprWith::show(std::ostream& str) const { + str << "(with " << *attrs << "; " << *body << ")"; +} + +void ExprIf::show(std::ostream& str) const { + str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; +} + +void ExprAssert::show(std::ostream& str) const { + str << "assert " << *cond << "; " << *body; +} + +void ExprOpNot::show(std::ostream& str) const { str << "(! " << *e << ")"; } + +void ExprConcatStrings::show(std::ostream& str) const { + bool first = true; + str << "("; + for (auto& i : *es) { + if (first) { + first = false; + } else { + str << " + "; + } + str << *i; + } + str << ")"; +} + +void ExprPos::show(std::ostream& str) const { str << "__curPos"; } + +std::ostream& operator<<(std::ostream& str, const Pos& pos) { + if (!pos || !pos.file.has_value()) { + str << "undefined position"; + } else { + str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % + std::string(pos.file.value()) % pos.line % pos.column) + .str(); + } + return str; +} + +std::string showAttrPath(const AttrPath& attrPath) { + std::ostringstream out; + bool first = true; + for (auto& attr : attrPath) { + if (!first) { + out << '.'; + } else { + first = false; + } + + std::visit(util::overloaded{ + [&](const Symbol& sym) { out << sym; }, + [&](const Expr* expr) { out << "\"${" << *expr << "}\""; }}, + attr); + } + return out.str(); +} + +Pos noPos; + +/* Computing levels/displacements for variables. */ + +void Expr::bindVars(const StaticEnv& env) { abort(); } + +void ExprInt::bindVars(const StaticEnv& env) {} + +void ExprFloat::bindVars(const StaticEnv& env) {} + +void ExprString::bindVars(const StaticEnv& env) {} + +void ExprPath::bindVars(const StaticEnv& env) {} + +void ExprVar::bindVars(const StaticEnv& env) { + /* Check whether the variable appears in the environment. If so, + set its level and displacement. */ + const StaticEnv* curEnv; + unsigned int level; + std::optional<unsigned int> withLevel = std::nullopt; + for (curEnv = &env, level = 0; curEnv != nullptr; + curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + if (!withLevel.has_value()) { + withLevel = level; + } + } else { + auto i = curEnv->vars.find(name); + if (i != curEnv->vars.end()) { + fromWith = false; + this->level = level; + displ = i->second; + return; + } + } + } + + /* Otherwise, the variable must be obtained from the nearest + enclosing `with'. If there is no `with', then we can issue an + "undefined variable" error now. */ + if (!withLevel.has_value()) { + throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % + pos); + } + + fromWith = true; + this->level = withLevel.value(); +} + +void ExprSelect::bindVars(const StaticEnv& env) { + e->bindVars(env); + if (def != nullptr) { + def->bindVars(env); + } + for (auto& i : attrPath) { + if (auto* expr = std::get_if<Expr*>(&i)) { + (*expr)->bindVars(env); + } + } +} + +void ExprOpHasAttr::bindVars(const StaticEnv& env) { + e->bindVars(env); + for (auto& i : attrPath) { + if (auto* expr = std::get_if<Expr*>(&i)) { + (*expr)->bindVars(env); + } + } +} + +void ExprAttrs::bindVars(const StaticEnv& env) { + const StaticEnv* dynamicEnv = &env; + StaticEnv newEnv(/* isWith = */ false, &env); + + if (recursive) { + dynamicEnv = &newEnv; + + unsigned int displ = 0; + for (auto& i : attrs) { + newEnv.vars[i.first] = i.second.displ = displ++; + } + + for (auto& i : attrs) { + i.second.e->bindVars(i.second.inherited ? env : newEnv); + } + } + + else { + for (auto& i : attrs) { + i.second.e->bindVars(env); + } + } + + for (auto& i : dynamicAttrs) { + i.nameExpr->bindVars(*dynamicEnv); + i.valueExpr->bindVars(*dynamicEnv); + } +} + +void ExprList::bindVars(const StaticEnv& env) { + for (auto& i : elems) { + i->bindVars(env); + } +} + +void ExprLambda::bindVars(const StaticEnv& env) { + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + + if (!arg.empty()) { + newEnv.vars[arg] = displ++; + } + + if (matchAttrs) { + for (auto& i : formals->formals) { + newEnv.vars[i.name] = displ++; + } + + for (auto& i : formals->formals) { + if (i.def != nullptr) { + i.def->bindVars(newEnv); + } + } + } + + body->bindVars(newEnv); +} + +void ExprLet::bindVars(const StaticEnv& env) { + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + for (auto& i : attrs->attrs) { + newEnv.vars[i.first] = i.second.displ = displ++; + } + + for (auto& i : attrs->attrs) { + i.second.e->bindVars(i.second.inherited ? env : newEnv); + } + + body->bindVars(newEnv); +} + +void ExprWith::bindVars(const StaticEnv& env) { + /* Does this `with' have an enclosing `with'? If so, record its + level so that `lookupVar' can look up variables in the previous + `with' if this one doesn't contain the desired attribute. */ + const StaticEnv* curEnv; + unsigned int level; + prevWith = 0; + for (curEnv = &env, level = 1; curEnv != nullptr; + curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + prevWith = level; + break; + } + } + + attrs->bindVars(env); + StaticEnv newEnv(true, &env); + body->bindVars(newEnv); +} + +void ExprIf::bindVars(const StaticEnv& env) { + cond->bindVars(env); + then->bindVars(env); + else_->bindVars(env); +} + +void ExprAssert::bindVars(const StaticEnv& env) { + cond->bindVars(env); + body->bindVars(env); +} + +void ExprOpNot::bindVars(const StaticEnv& env) { e->bindVars(env); } + +void ExprConcatStrings::bindVars(const StaticEnv& env) { + for (auto& i : *es) { + i->bindVars(env); + } +} + +void ExprPos::bindVars(const StaticEnv& env) {} + +/* Storing function names. */ +void ExprLambda::setName(Symbol& name) { this->name = name; } + +std::string ExprLambda::showNamePos() const { + return (format("%1% at %2%") % + (name.has_value() ? "'" + std::string(name.value()) + "'" + : "anonymous function") % + pos) + .str(); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/nixexpr.hh b/third_party/nix/src/libexpr/nixexpr.hh new file mode 100644 index 000000000000..16b58dec2e84 --- /dev/null +++ b/third_party/nix/src/libexpr/nixexpr.hh @@ -0,0 +1,361 @@ +#pragma once + +#include <map> +#include <optional> +#include <variant> + +#include <absl/container/flat_hash_map.h> + +#include "libexpr/symbol-table.hh" +#include "libexpr/value.hh" +#include "libutil/types.hh" // TODO(tazjin): audit this include + +namespace nix { + +MakeError(EvalError, Error); +MakeError(ParseError, Error); +MakeError(AssertionError, EvalError); +MakeError(ThrownError, AssertionError); +MakeError(Abort, EvalError); +MakeError(TypeError, EvalError); +MakeError(UndefinedVarError, Error); +MakeError(RestrictedPathError, Error); + +/* Position objects. */ + +struct Pos { + std::optional<Symbol> file; + unsigned int line, column; + Pos(const std::optional<Symbol>& file, unsigned int line, unsigned int column) + : file(file), line(line), column(column){}; + + // TODO(tazjin): remove this - empty pos is never useful + Pos() : file(std::nullopt), line(0), column(0){}; + + operator bool() const { return line != 0; } + + bool operator<(const Pos& p2) const { + if (!file.has_value()) { + return true; + } + + if (!line) { + return p2.line; + } + if (!p2.line) { + return false; + } + int d = ((std::string)file.value()).compare((std::string)p2.file.value()); + if (d < 0) { + return true; + } + if (d > 0) { + return false; + } + if (line < p2.line) { + return true; + } + if (line > p2.line) { + return false; + } + return column < p2.column; + } +}; + +extern Pos noPos; + +std::ostream& operator<<(std::ostream& str, const Pos& pos); + +struct Env; +struct Value; +class EvalState; +struct StaticEnv; + +/* An attribute path is a sequence of attribute names. */ +using AttrName = std::variant<Symbol, Expr*>; +using AttrPath = std::vector<AttrName>; +using AttrNameVector = std::vector<AttrName>; + +using VectorExprs = std::vector<nix::Expr*>; + +std::string showAttrPath(const AttrPath& attrPath); + +/* Abstract syntax of Nix expressions. */ + +struct Expr { + virtual ~Expr(){}; + virtual void show(std::ostream& str) const; + virtual void bindVars(const StaticEnv& env); + virtual void eval(EvalState& state, Env& env, Value& v); + virtual Value* maybeThunk(EvalState& state, Env& env); +}; + +std::ostream& operator<<(std::ostream& str, const Expr& e); + +#define COMMON_METHODS \ + void show(std::ostream& str) const; \ + void eval(EvalState& state, Env& env, Value& v); \ + void bindVars(const StaticEnv& env); + +struct ExprInt : Expr { + NixInt n; + Value v; + ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +struct ExprFloat : Expr { + NixFloat nf; + Value v; + ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +struct ExprString : Expr { + Symbol s; + Value v; + ExprString(const Symbol& s) : s(s) { mkString(v, s); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +/* Temporary class used during parsing of indented strings. */ +struct ExprIndStr : Expr { + std::string s; + ExprIndStr(const std::string& s) : s(s){}; +}; + +struct ExprPath : Expr { + std::string s; + Value v; + ExprPath(const std::string& s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +struct ExprVar : Expr { + Pos pos; + Symbol name; + + /* Whether the variable comes from an environment (e.g. a rec, let + or function argument) or from a "with". */ + bool fromWith; + + /* In the former case, the value is obtained by going `level' + levels up from the current environment and getting the + `displ'th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name' from + the set stored in the environment that is `level' levels up + from the current one.*/ + unsigned int level; + unsigned int displ; + + ExprVar(const Symbol& name) : name(name){}; + ExprVar(const Pos& pos, const Symbol& name) : pos(pos), name(name){}; + COMMON_METHODS + Value* maybeThunk(EvalState& state, Env& env); +}; + +// [tazjin] I *think* that this struct describes the syntactic +// construct for "selecting" something out of an attribute set, e.g. +// `a.b.c` => ExprSelect{"b", "c"}. +// +// Each path element has got a pointer to an expression, which seems +// to be the thing preceding its period, but afaict that is only set +// for the first one in a path. +struct ExprSelect : Expr { + Pos pos; + Expr *e, *def; + AttrPath attrPath; + ExprSelect(const Pos& pos, Expr* e, const AttrPath& attrPath, Expr* def) + : pos(pos), e(e), def(def), attrPath(attrPath){}; + ExprSelect(const Pos& pos, Expr* e, const Symbol& name) + : pos(pos), e(e), def(0) { + attrPath.push_back(AttrName(name)); + }; + COMMON_METHODS +}; + +struct ExprOpHasAttr : Expr { + Pos pos; + Expr* e; + AttrPath attrPath; + ExprOpHasAttr(Expr* e, const AttrPath& attrPath) : e(e), attrPath(attrPath){}; + ExprOpHasAttr(const Pos& pos, Expr* e, const AttrPath& attrPath) + : pos(pos), e(e), attrPath(attrPath){}; + COMMON_METHODS +}; + +struct ExprAttrs : Expr { + bool recursive; + + struct AttrDef { + bool inherited; + Expr* e; + Pos pos; + unsigned int displ; // displacement + AttrDef(Expr* e, const Pos& pos, bool inherited = false) + : inherited(inherited), e(e), pos(pos), displ(0){}; + AttrDef(){}; + }; + + using AttrDefs = absl::flat_hash_map<Symbol, AttrDef>; + AttrDefs attrs; + + struct DynamicAttrDef { + Expr *nameExpr, *valueExpr; + Pos pos; + DynamicAttrDef(Expr* nameExpr, Expr* valueExpr, const Pos& pos) + : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos){}; + }; + + using DynamicAttrDefs = std::vector<DynamicAttrDef>; + DynamicAttrDefs dynamicAttrs; + + ExprAttrs() : recursive(false){}; + COMMON_METHODS +}; + +struct ExprList : Expr { + VectorExprs elems; + ExprList(){}; + COMMON_METHODS +}; + +struct Formal { + Symbol name; + Expr* def; // def = default, not definition + Formal(const Symbol& name, Expr* def) : name(name), def(def){}; +}; + +// Describes structured function arguments (e.g. `{ a }: ...`) +struct Formals { + using Formals_ = std::list<Formal>; + Formals_ formals; + std::set<Symbol> argNames; // used during parsing + bool ellipsis; +}; + +struct ExprLambda : Expr { + public: + Pos pos; + std::optional<Symbol> name; + Symbol arg; + bool matchAttrs; + Formals* formals; + Expr* body; + ExprLambda(const Pos& pos, const Symbol& arg, bool matchAttrs, + Formals* formals, Expr* body) + : pos(pos), + arg(arg), + matchAttrs(matchAttrs), + formals(formals), + body(body) { + if (!arg.empty() && formals && + formals->argNames.find(arg) != formals->argNames.end()) { + throw ParseError( + format("duplicate formal function argument '%1%' at %2%") % arg % + pos); + } + }; + void setName(Symbol& name); + std::string showNamePos() const; + COMMON_METHODS +}; + +struct ExprLet : Expr { + ExprAttrs* attrs; + Expr* body; + ExprLet(ExprAttrs* attrs, Expr* body) : attrs(attrs), body(body){}; + COMMON_METHODS +}; + +struct ExprWith : Expr { + Pos pos; + Expr *attrs, *body; + size_t prevWith; + ExprWith(const Pos& pos, Expr* attrs, Expr* body) + : pos(pos), attrs(attrs), body(body){}; + COMMON_METHODS +}; + +struct ExprIf : Expr { + Pos pos; + Expr *cond, *then, *else_; + ExprIf(Expr* cond, Expr* then, Expr* else_) + : cond(cond), then(then), else_(else_){}; + ExprIf(const Pos& pos, Expr* cond, Expr* then, Expr* else_) + : pos(pos), cond(cond), then(then), else_(else_){}; + COMMON_METHODS +}; + +struct ExprAssert : Expr { + Pos pos; + Expr *cond, *body; + ExprAssert(const Pos& pos, Expr* cond, Expr* body) + : pos(pos), cond(cond), body(body){}; + COMMON_METHODS +}; + +struct ExprOpNot : Expr { + Pos pos; + Expr* e; + explicit ExprOpNot(Expr* e) : e(e){}; + ExprOpNot(const Pos& pos, Expr* e) : pos(pos), e(e){}; + COMMON_METHODS +}; + +#define MakeBinOp(name, s) \ + struct name : Expr { \ + Pos pos; \ + Expr *e1, *e2; \ + name(Expr* e1, Expr* e2) : e1(e1), e2(e2){}; \ + name(const Pos& pos, Expr* e1, Expr* e2) : pos(pos), e1(e1), e2(e2){}; \ + void show(std::ostream& str) const { \ + str << "(" << *e1 << " " s " " << *e2 << ")"; \ + } \ + void bindVars(const StaticEnv& env) { \ + e1->bindVars(env); \ + e2->bindVars(env); \ + } \ + void eval(EvalState& state, Env& env, Value& v); \ + }; + +MakeBinOp(ExprApp, ""); +MakeBinOp(ExprOpEq, "=="); +MakeBinOp(ExprOpNEq, "!="); +MakeBinOp(ExprOpAnd, "&&"); +MakeBinOp(ExprOpOr, "||"); +MakeBinOp(ExprOpImpl, "->"); +MakeBinOp(ExprOpUpdate, "//"); +MakeBinOp(ExprOpConcatLists, "++"); + +struct ExprConcatStrings : Expr { + Pos pos; + bool forceString; + nix::VectorExprs* es; + ExprConcatStrings(const Pos& pos, bool forceString, nix::VectorExprs* es) + : pos(pos), forceString(forceString), es(es){}; + COMMON_METHODS +}; + +struct ExprPos : Expr { + Pos pos; + ExprPos(const Pos& pos) : pos(pos){}; + COMMON_METHODS +}; + +/* Static environments are used to map variable names onto (level, + displacement) pairs used to obtain the value of the variable at + runtime. */ +struct StaticEnv { + bool isWith; + const StaticEnv* up; + typedef absl::flat_hash_map<Symbol, unsigned int> Vars; + Vars vars; + StaticEnv(bool isWith, const StaticEnv* up) : isWith(isWith), up(up){}; +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/parser.cc b/third_party/nix/src/libexpr/parser.cc new file mode 100644 index 000000000000..aea6cec7e445 --- /dev/null +++ b/third_party/nix/src/libexpr/parser.cc @@ -0,0 +1,332 @@ +#include "libexpr/parser.hh" + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libexpr/eval.hh" +#include "libstore/download.hh" +#include "libstore/store-api.hh" + +namespace nix { + +void addAttr(ExprAttrs* attrs, AttrPath& attrPath, Expr* e, const Pos& pos) { + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (const auto* sym = std::get_if<Symbol>(&(*i)); sym && sym->set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*sym); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs* attrs2 = dynamic_cast<ExprAttrs*>(j->second.e); + if (!attrs2) { + dupAttr(attrPath, pos, j->second.pos); + } + attrs = attrs2; + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + ExprAttrs* nested = new ExprAttrs; + attrs->attrs[*sym] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + // Yes, this code does not handle all conditions + // exhaustively. We use std::get to throw if the condition + // that isn't covered happens, which is potentially a + // behaviour change from the previous default constructed + // Symbol. It should alert us about anything untoward going + // on here. + auto* expr = std::get<Expr*>(*i); + + ExprAttrs* nested = new ExprAttrs; + attrs->dynamicAttrs.push_back( + ExprAttrs::DynamicAttrDef(expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (auto* sym = std::get_if<Symbol>(&(*i)); sym && sym->set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(*sym); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast<ExprAttrs*>(e); + auto jAttrs = dynamic_cast<ExprAttrs*>(j->second.e); + if (jAttrs && ae) { + for (auto& ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != + jAttrs->attrs.end()) { // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + } + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs[*sym] = ExprAttrs::AttrDef(e, pos); + } + } else { + // Same caveat as the identical line above. + auto* expr = std::get<Expr*>(*i); + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(expr, e, pos)); + } +} + +void addFormal(const Pos& pos, Formals* formals, const Formal& formal) { + if (formals->argNames.find(formal.name) != formals->argNames.end()) { + throw ParseError(format("duplicate formal function argument '%1%' at %2%") % + formal.name % pos); + } + formals->formals.push_front(formal); + formals->argNames.insert(formal.name); +} + +Expr* stripIndentation(const Pos& pos, SymbolTable& symbols, VectorExprs& es) { + if (es.empty()) { + return new ExprString(symbols.Create("")); + } + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto& i : es) { + ExprIndStr* e = dynamic_cast<ExprIndStr*>(i); + if (!e) { + /* Anti-quotations end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) { + minIndent = curIndent; + } + } + continue; + } + for (size_t j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') { + curIndent++; + } else if (e->s[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) { + minIndent = curIndent; + } + } + } else if (e->s[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + VectorExprs* es2 = new VectorExprs; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + for (VectorExprs::iterator i = es.begin(); i != es.end(); ++i, --n) { + ExprIndStr* e = dynamic_cast<ExprIndStr*>(*i); + if (!e) { + atStartOfLine = false; + curDropped = 0; + es2->push_back(*i); + continue; + } + + std::string s2; + for (size_t j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') { + if (curDropped++ >= minIndent) { + s2 += e->s[j]; + } + } else if (e->s[j] == '\n') { + curDropped = 0; + s2 += e->s[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += e->s[j]; + } + } else { + s2 += e->s[j]; + if (e->s[j] == '\n') { + atStartOfLine = true; + } + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + std::string::size_type p = s2.find_last_of('\n'); + if (p != std::string::npos && + s2.find_first_not_of(' ', p + 1) == std::string::npos) { + s2 = std::string(s2, 0, p + 1); + } + } + + es2->push_back(new ExprString(symbols.Create(s2))); + } + + /* If this is a single string, then don't do a concatenation. */ + return es2->size() == 1 && dynamic_cast<ExprString*>((*es2)[0]) + ? (*es2)[0] + : new ExprConcatStrings(pos, true, es2); +} + +Path resolveExprPath(Path path) { + assert(path[0] == '/'); + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + struct stat st; + while (true) { + if (lstat(path.c_str(), &st)) { + throw SysError(format("getting status of '%1%'") % path); + } + if (!S_ISLNK(st.st_mode)) { + break; + } + path = absPath(readLink(path), dirOf(path)); + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (S_ISDIR(st.st_mode)) { + path = canonPath(path + "/default.nix"); + } + + return path; +} + +// These methods are actually declared in eval.hh, and were - for some +// reason - previously implemented in parser.y. + +Expr* EvalState::parseExprFromFile(const Path& path) { + return parseExprFromFile(path, staticBaseEnv); +} + +Expr* EvalState::parseExprFromFile(const Path& path, StaticEnv& staticEnv) { + return parse(readFile(path).c_str(), path, dirOf(path), staticEnv); +} + +Expr* EvalState::parseExprFromString(const std::string& s, const Path& basePath, + StaticEnv& staticEnv) { + return parse(s.c_str(), "(std::string)", basePath, staticEnv); +} + +Expr* EvalState::parseExprFromString(const std::string& s, + const Path& basePath) { + return parseExprFromString(s, basePath, staticBaseEnv); +} + +Expr* EvalState::parseStdin() { + // Activity act(*logger, lvlTalkative, format("parsing standard input")); + return parseExprFromString(drainFD(0), absPath(".")); +} + +void EvalState::addToSearchPath(const std::string& s) { + size_t pos = s.find('='); + std::string prefix; + Path path; + if (pos == std::string::npos) { + path = s; + } else { + prefix = std::string(s, 0, pos); + path = std::string(s, pos + 1); + } + + searchPath.emplace_back(prefix, path); +} + +Path EvalState::findFile(const std::string& path) { + return findFile(searchPath, path); +} + +Path EvalState::findFile(SearchPath& searchPath, const std::string& path, + const Pos& pos) { + for (auto& i : searchPath) { + std::string suffix; + if (i.first.empty()) { + suffix = "/" + path; + } else { + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) { + continue; + } + suffix = path.size() == s ? "" : "/" + std::string(path, s); + } + auto r = resolveSearchPathElem(i); + if (!r.first) { + continue; + } + Path res = r.second + suffix; + if (pathExists(res)) { + return canonPath(res); + } + } + format f = format( + "file '%1%' was not found in the Nix search path (add it using $NIX_PATH " + "or -I)" + + std::string(pos ? ", at %2%" : "")); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + throw ThrownError(f % path % pos); +} + +std::pair<bool, std::string> EvalState::resolveSearchPathElem( + const SearchPathElem& elem) { + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) { + return i->second; + } + + std::pair<bool, std::string> res; + + if (isUri(elem.second)) { + try { + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = {true, getDownloader()->downloadCached(store, request).path}; + } catch (DownloadError& e) { + LOG(WARNING) << "Nix search path entry '" << elem.second + << "' cannot be downloaded, ignoring"; + res = {false, ""}; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) { + res = {true, path}; + } else { + LOG(WARNING) << "Nix search path entry '" << elem.second + << "' does not exist, ignoring"; + res = {false, ""}; + } + } + + VLOG(2) << "resolved search path element '" << elem.second << "' to '" + << res.second << "'"; + + searchPathResolved[elem.second] = res; + return res; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/parser.hh b/third_party/nix/src/libexpr/parser.hh new file mode 100644 index 000000000000..70b5450b5aa8 --- /dev/null +++ b/third_party/nix/src/libexpr/parser.hh @@ -0,0 +1,100 @@ +// Parser utilities for use in parser.y +#pragma once + +// TODO(tazjin): Audit these includes, they were in parser.y +#include <optional> +#include <variant> + +#include <glog/logging.h> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" +#include "libutil/util.hh" + +#define YY_DECL \ + int yylex(YYSTYPE* yylval_param, YYLTYPE* yylloc_param, yyscan_t yyscanner, \ + nix::ParseData* data) + +#define CUR_POS makeCurPos(*yylocp, data) + +namespace nix { + +struct ParseData { + EvalState& state; + SymbolTable& symbols; + Expr* result; + Path basePath; + std::optional<Symbol> path; + std::string error; + Symbol sLetBody; + + ParseData(EvalState& state) + : state(state), + symbols(state.symbols), + sLetBody(symbols.Create("<let-body>")){}; +}; + +// Clang fails to identify these functions as used, probably because +// of some interaction between the lexer/parser codegen and something +// else. +// +// To avoid warnings for that we disable -Wunused-function in this block. + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-function" + +// TODO(tazjin): move dupAttr to anonymous namespace +static void dupAttr(const AttrPath& attrPath, const Pos& pos, + const Pos& prevPos) { + throw ParseError(format("attribute '%1%' at %2% already defined at %3%") % + showAttrPath(attrPath) % pos % prevPos); +} + +static void dupAttr(Symbol attr, const Pos& pos, const Pos& prevPos) { + throw ParseError(format("attribute '%1%' at %2% already defined at %3%") % + attr % pos % prevPos); +} + +void addAttr(ExprAttrs* attrs, AttrPath& attrPath, Expr* e, const Pos& pos); + +void addFormal(const Pos& pos, Formals* formals, const Formal& formal); + +Expr* stripIndentation(const Pos& pos, SymbolTable& symbols, VectorExprs& es); + +Path resolveExprPath(Path path); + +// implementations originally from lexer.l + +static Expr* unescapeStr(SymbolTable& symbols, const char* s, size_t length) { + std::string t; + t.reserve(length); + char c; + while ((c = *s++)) { + if (c == '\\') { + assert(*s); + c = *s++; + if (c == 'n') { + t += '\n'; + } else if (c == 'r') { + t += '\r'; + } else if (c == 't') { + t += '\t'; + } else { + t += c; + } + } else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + t += '\n'; + if (*s == '\n') { + s++; + } /* cr/lf */ + } else { + t += c; + } + } + return new ExprString(symbols.Create(t)); +} + +#pragma clang diagnostic pop // re-enable -Wunused-function + +} // namespace nix diff --git a/third_party/nix/src/libexpr/parser.y b/third_party/nix/src/libexpr/parser.y new file mode 100644 index 000000000000..a8af06802f16 --- /dev/null +++ b/third_party/nix/src/libexpr/parser.y @@ -0,0 +1,359 @@ +%glr-parser +%locations +%define parse.error verbose +%define api.pure true +%defines +/* %no-lines */ +%parse-param { void * scanner } +%parse-param { nix::ParseData * data } +%lex-param { void * scanner } +%lex-param { nix::ParseData * data } +%expect 1 +%expect-rr 1 + +%code requires { +#define YY_NO_INPUT 1 // disable unused yyinput features +#include "libexpr/parser.hh" + +struct YYSTYPE { + union { + nix::Expr * e; + nix::ExprList * list; + nix::ExprAttrs * attrs; + nix::Formals * formals; + nix::Formal * formal; + nix::NixInt n; + nix::NixFloat nf; + const char * id; // !!! -> Symbol + char * path; + char * uri; + nix::AttrNameVector * attrNames; + nix::VectorExprs * string_parts; + }; +}; + +} + +%{ + +#include "generated/parser-tab.hh" +#include "generated/lexer-tab.hh" + +YY_DECL; + +using namespace nix; + +namespace nix { + +static inline Pos makeCurPos(const YYLTYPE& loc, ParseData* data) { + return Pos(data->path, loc.first_line, loc.first_column); +} + +void yyerror(YYLTYPE* loc, yyscan_t scanner, ParseData* data, + const char* error) { + data->error = (format("%1%, at %2%") % error % makeCurPos(*loc, data)).str(); +} + +} + +%} + +%type <e> start expr expr_function expr_if expr_op +%type <e> expr_app expr_select expr_simple +%type <list> expr_list +%type <attrs> binds +%type <formals> formals +%type <formal> formal +%type <attrNames> attrs attrpath +%type <string_parts> string_parts_interpolated ind_string_parts +%type <e> string_parts string_attr +%type <id> attr +%token <id> ID ATTRPATH +%token <e> STR IND_STR +%token <n> INT +%token <nf> FLOAT +%token <path> PATH HPATH SPATH +%token <uri> URI +%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token DOLLAR_CURLY /* == ${ */ +%token IND_STRING_OPEN IND_STRING_CLOSE +%token ELLIPSIS + +%right IMPL +%left OR +%left AND +%nonassoc EQ NEQ +%nonassoc '<' '>' LEQ GEQ +%right UPDATE +%left NOT +%left '+' '-' +%left '*' '/' +%right CONCAT +%nonassoc '?' +%nonassoc NEGATE + +%% + +start: expr { data->result = $1; }; + +expr: expr_function; + +expr_function + : ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create($1), false, 0, $3); } + | '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create(""), true, $2, $5); } + | '{' formals '}' '@' ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create($5), true, $2, $7); } + | ID '@' '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.Create($1), true, $4, $7); } + | ASSERT expr ';' expr_function + { $$ = new ExprAssert(CUR_POS, $2, $4); } + | WITH expr ';' expr_function + { $$ = new ExprWith(CUR_POS, $2, $4); } + | LET binds IN expr_function + { if (!$2->dynamicAttrs.empty()) + throw ParseError(format("dynamic attributes not allowed in let at %1%") + % CUR_POS); + $$ = new ExprLet($2, $4); + } + | expr_if + ; + +expr_if + : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_op + ; + +expr_op + : '!' expr_op %prec NOT { $$ = new ExprOpNot(CUR_POS, $2); } + | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__sub")), new ExprInt(0)), $2); } + | expr_op EQ expr_op { $$ = new ExprOpEq(CUR_POS, $1, $3); } + | expr_op NEQ expr_op { $$ = new ExprOpNEq(CUR_POS, $1, $3); } + | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $1), $3); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $3), $1)); } + | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $3), $1); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__lessThan")), $1), $3)); } + | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr(CUR_POS, $1, *$3); } + | expr_op '+' expr_op + { $$ = new ExprConcatStrings(CUR_POS, false, new nix::VectorExprs({$1, $3})); } + | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__sub")), $1), $3); } + | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__mul")), $1), $3); } + | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.Create("__div")), $1), $3); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } + | expr_app + ; + +expr_app + : expr_app expr_select + { $$ = new ExprApp(CUR_POS, $1, $2); } + | expr_select { $$ = $1; } + ; + +expr_select + : expr_simple '.' attrpath + { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + | expr_simple '.' attrpath OR_KW expr_select + { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + | /* Backwards compatibility: because Nixpkgs has a rarely used + function named ‘or’, allow stuff like ‘map or [...]’. */ + expr_simple OR_KW + { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.Create("or"))); } + | expr_simple { $$ = $1; } + ; + +expr_simple + : ID { + if (strcmp($1, "__curPos") == 0) + $$ = new ExprPos(CUR_POS); + else + $$ = new ExprVar(CUR_POS, data->symbols.Create($1)); + } + | INT { $$ = new ExprInt($1); } + | FLOAT { $$ = new ExprFloat($1); } + | '"' string_parts '"' { $$ = $2; } + | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { + $$ = stripIndentation(CUR_POS, data->symbols, *$2); + } + | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } + | HPATH { $$ = new ExprPath(getHome() + std::string{$1 + 1}); } + | SPATH { + std::string path($1 + 1, strlen($1) - 2); + $$ = new ExprApp(CUR_POS, + new ExprApp(new ExprVar(data->symbols.Create("__findFile")), + new ExprVar(data->symbols.Create("__nixPath"))), + new ExprString(data->symbols.Create(path))); + } + | URI { $$ = new ExprString(data->symbols.Create($1)); } + | '(' expr ')' { $$ = $2; } + /* Let expressions `let {..., body = ...}' are just desugared + into `(rec {..., body = ...}).body'. */ + | LET '{' binds '}' + { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.Create("body")); } + | REC '{' binds '}' + { $3->recursive = true; $$ = $3; } + | '{' binds '}' + { $$ = $2; } + | '[' expr_list ']' { $$ = $2; } + ; + +string_parts + : STR + | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); } + | { $$ = new ExprString(data->symbols.Create("")); } + ; + +string_parts_interpolated + : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | DOLLAR_CURLY expr '}' { $$ = new nix::VectorExprs; $$->push_back($2); } + | STR DOLLAR_CURLY expr '}' { + $$ = new nix::VectorExprs; + $$->push_back($1); + $$->push_back($3); + } + ; + +ind_string_parts + : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | { $$ = new nix::VectorExprs; } + ; + +binds + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } + | binds INHERIT attrs ';' + { $$ = $1; + for (auto & i : *$3) { + auto sym = std::get<Symbol>(i); + if ($$->attrs.find(sym) != $$->attrs.end()) { + dupAttr(sym, makeCurPos(@3, data), $$->attrs[sym].pos); + } + Pos pos = makeCurPos(@3, data); + $$->attrs[sym] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, sym), pos, true); + } + } + | binds INHERIT '(' expr ')' attrs ';' + { $$ = $1; + /* !!! Should ensure sharing of the expression in $4. */ + for (auto & i : *$6) { + auto sym = std::get<Symbol>(i); + if ($$->attrs.find(sym) != $$->attrs.end()) { + dupAttr(sym, makeCurPos(@6, data), $$->attrs[sym].pos); + } + $$->attrs[sym] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, sym), makeCurPos(@6, data)); + } + } + | { $$ = new ExprAttrs; } + ; + +attrs + : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.Create($2))); } + | attrs string_attr + { $$ = $1; + ExprString * str = dynamic_cast<ExprString *>($2); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + throw ParseError(format("dynamic attributes not allowed in inherit at %1%") + % makeCurPos(@2, data)); + } + | { $$ = new AttrPath; } + ; + +attrpath + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.Create($3))); } + | attrpath '.' string_attr + { $$ = $1; + ExprString * str = dynamic_cast<ExprString *>($3); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else { + $$->push_back(AttrName($3)); + } + } + | attr { $$ = new nix::AttrNameVector; $$->push_back(AttrName(data->symbols.Create($1))); } + | string_attr + { $$ = new nix::AttrNameVector; + ExprString *str = dynamic_cast<ExprString *>($1); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + $$->push_back(AttrName($1)); + } + ; + +attr + : ID { $$ = $1; } + | OR_KW { $$ = "or"; } + ; + +string_attr + : '"' string_parts '"' { $$ = $2; } + | DOLLAR_CURLY expr '}' { $$ = $2; } + ; + +expr_list + : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ } + | { $$ = new ExprList; } + ; + +formals + : formal ',' formals + { $$ = $3; addFormal(CUR_POS, $$, *$1); } + | formal + { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; } + | + { $$ = new Formals; $$->ellipsis = false; } + | ELLIPSIS + { $$ = new Formals; $$->ellipsis = true; } + ; + +formal + : ID { $$ = new Formal(data->symbols.Create($1), 0); } + | ID '?' expr { $$ = new Formal(data->symbols.Create($1), $3); } + ; + +%% + + +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> +#include <unistd.h> + +#include "libexpr/eval.hh" +#include "libstore/store-api.hh" + + +namespace nix { + +Expr* EvalState::parse(const char* text, const Path& path, const Path& basePath, + StaticEnv& staticEnv) { + yyscan_t scanner; + ParseData data(*this); + data.basePath = basePath; + data.path = data.symbols.Create(path); + + yylex_init(&scanner); + yy_scan_string(text, scanner); + int res = yyparse(scanner, &data); + yylex_destroy(scanner); + + if (res) { + throw ParseError(data.error); + } + + data.result->bindVars(staticEnv); + + return data.result; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops.cc b/third_party/nix/src/libexpr/primops.cc new file mode 100644 index 000000000000..710d214ce7c2 --- /dev/null +++ b/third_party/nix/src/libexpr/primops.cc @@ -0,0 +1,2336 @@ +#include "libexpr/primops.hh" + +#include <algorithm> +#include <cstring> +#include <iostream> +#include <regex> + +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/json-to-value.hh" +#include "libexpr/names.hh" +#include "libexpr/value-to-json.hh" +#include "libexpr/value-to-xml.hh" +#include "libstore/derivations.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/json.hh" +#include "libutil/status.hh" +#include "libutil/util.hh" + +namespace nix { + +/************************************************************* + * Miscellaneous + *************************************************************/ + +/* Decode a context string ‘!<name>!<path>’ into a pair <path, + name>. */ +std::pair<std::string, std::string> decodeContext(const std::string& s) { + if (s.at(0) == '!') { + size_t index = s.find('!', 1); + return std::pair<std::string, std::string>(std::string(s, index + 1), + std::string(s, 1, index - 1)); + } + return std::pair<std::string, std::string>( + s.at(0) == '/' ? s : std::string(s, 1), ""); +} + +InvalidPathError::InvalidPathError(const Path& path) + : EvalError(format("path '%1%' is not valid") % path), path(path) {} + +void EvalState::realiseContext(const PathSet& context) { + PathSet drvs; + + for (auto& i : context) { + std::pair<std::string, std::string> decoded = decodeContext(i); + Path ctx = decoded.first; + assert(store->isStorePath(ctx)); + if (!store->isValidPath(ctx)) { + throw InvalidPathError(ctx); + } + if (!decoded.second.empty() && nix::isDerivation(ctx)) { + drvs.insert(decoded.first + "!" + decoded.second); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(decoded.first); + auto i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) { + throw Error("derivation '%s' does not have an output named '%s'", + decoded.first, decoded.second); + } + allowedPaths->insert(i->second.path); + } + } + } + + if (drvs.empty()) { + return; + } + + if (!evalSettings.enableImportFromDerivation) { + throw EvalError(format("attempted to realize '%1%' during evaluation but " + "'allow-import-from-derivation' is false") % + *(drvs.begin())); + } + + /* For performance, prefetch all substitute info. */ + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + unsigned long long downloadSize; + unsigned long long narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, + narSize); + + nix::util::OkOrThrow(store->buildPaths(std::cerr, drvs)); +} + +/* Load and evaluate an expression from path specified by the + argument. */ +static void prim_scopedImport(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot import '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + + if (state.store->isStorePath(path) && state.store->isValidPath(path) && + isDerivation(path)) { + Derivation drv = readDerivation(realPath); + Value& w = *state.allocValue(); + state.mkAttrs(w, 3 + drv.outputs.size()); + Value* v2 = state.allocAttr(w, state.sDrvPath); + mkString(*v2, path, {"=" + path}); + v2 = state.allocAttr(w, state.sName); + mkString(*v2, drv.env["name"]); + Value* outputsVal = state.allocAttr(w, state.symbols.Create("outputs")); + state.mkList(*outputsVal, drv.outputs.size()); + unsigned int outputs_index = 0; + + for (const auto& o : drv.outputs) { + v2 = state.allocAttr(w, state.symbols.Create(o.first)); + mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + (*outputsVal->list)[outputs_index] = state.allocValue(); + mkString(*((*outputsVal->list)[outputs_index++]), o.first); + } + + Value fun; + state.evalFile( + settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", + fun); + state.forceFunction(fun, pos); + mkApp(v, fun, w); + state.forceAttrs(v, pos); + } else { + state.forceAttrs(*args[0]); + if (args[0]->attrs->empty()) { + state.evalFile(realPath, v); + } else { + Env* env = &state.allocEnv(args[0]->attrs->size()); + env->up = &state.baseEnv; + + StaticEnv staticEnv(false, &state.staticBaseEnv); + + unsigned int displ = 0; + for (auto& attr : *args[0]->attrs) { + staticEnv.vars[attr.second.name] = displ; + env->values[displ++] = attr.second.value; + } + + DLOG(INFO) << "evaluating file '" << realPath << "'"; + Expr* e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + + e->eval(state, *env, v); + } + } +} + +/* Return a string representing the type of the expression. */ +static void prim_typeOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + std::string t; + switch (args[0]->type) { + case tInt: + t = "int"; + break; + case tBool: + t = "bool"; + break; + case tString: + t = "string"; + break; + case tPath: + t = "path"; + break; + case tNull: + t = "null"; + break; + case tAttrs: + t = "set"; + break; + case tList: + t = "list"; + break; + case tLambda: + case tPrimOp: + case tPrimOpApp: + t = "lambda"; + break; + case tFloat: + t = "float"; + break; + default: + abort(); + } + mkString(v, state.symbols.Create(t)); +} + +/* Determine whether the argument is the null value. */ +static void prim_isNull(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tNull); +} + +/* Determine whether the argument is a function. */ +static void prim_isFunction(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); +} + +/* Determine whether the argument is an integer. */ +static void prim_isInt(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tInt); +} + +/* Determine whether the argument is a float. */ +static void prim_isFloat(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tFloat); +} + +/* Determine whether the argument is a string. */ +static void prim_isString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tString); +} + +/* Determine whether the argument is a Boolean. */ +static void prim_isBool(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tBool); +} + +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} + +struct CompareValues { + bool operator()(const Value* v1, const Value* v2) const { + if (v1->type == tFloat && v2->type == tInt) { + return v1->fpoint < v2->integer; + } + if (v1->type == tInt && v2->type == tFloat) { + return v1->integer < v2->fpoint; + } + if (v1->type != v2->type) { + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % + showType(*v2)); + } + switch (v1->type) { + case tInt: + return v1->integer < v2->integer; + case tFloat: + return v1->fpoint < v2->fpoint; + case tString: + return strcmp(v1->string.s, v2->string.s) < 0; + case tPath: + return strcmp(v1->path, v2->path) < 0; + default: + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % + showType(*v2)); + } + } +}; + +typedef std::list<Value*> ValueList; + +static void prim_genericClosure(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + + /* Get the start set. */ + Bindings::iterator startSet = + args[0]->attrs->find(state.symbols.Create("startSet")); + if (startSet == args[0]->attrs->end()) { + throw EvalError(format("attribute 'startSet' required, at %1%") % pos); + } + state.forceList(*startSet->second.value, pos); + + ValueList workSet; + for (Value* elem : *startSet->second.value->list) { + workSet.push_back(elem); + } + + /* Get the operator. */ + Bindings::iterator op = + args[0]->attrs->find(state.symbols.Create("operator")); + if (op == args[0]->attrs->end()) { + throw EvalError(format("attribute 'operator' required, at %1%") % pos); + } + state.forceValue(*op->second.value); + + /* Construct the closure by applying the operator to element of + `workSet', adding the result to `workSet', continuing until + no new elements are found. */ + ValueList res; + // `doneKeys' doesn't need to be a GC root, because its values are + // reachable from res. + std::set<Value*, CompareValues> doneKeys; + while (!workSet.empty()) { + Value* e = *(workSet.begin()); + workSet.pop_front(); + + state.forceAttrs(*e, pos); + + Bindings::iterator key = e->attrs->find(state.symbols.Create("key")); + if (key == e->attrs->end()) { + throw EvalError(format("attribute 'key' required, at %1%") % pos); + } + state.forceValue(*key->second.value); + + if (doneKeys.find(key->second.value) != doneKeys.end()) { + continue; + } + doneKeys.insert(key->second.value); + res.push_back(e); + + /* Call the `operator' function with `e' as argument. */ + Value call; + mkApp(call, *op->second.value, *e); + state.forceList(call, pos); + + /* Add the values returned by the operator to the work set. */ + for (unsigned int n = 0; n < call.listSize(); ++n) { + state.forceValue(*(*call.list)[n]); + workSet.push_back((*call.list)[n]); + } + } + + /* Create the result list. */ + state.mkList(v, res.size()); + unsigned int n = 0; + for (auto& i : res) { + (*v.list)[n++] = i; + } +} + +static void prim_abort(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + throw Abort( + format("evaluation aborted with the following error message: '%1%'") % s); +} + +static void prim_throw(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + throw ThrownError(s); +} + +static void prim_addErrorContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + try { + state.forceValue(*args[1]); + v = *args[1]; + } catch (Error& e) { + PathSet context; + e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context)); + throw; + } +} + +/* Try evaluating the argument. Success => {success=true; value=something;}, + * else => {success=false; value=false;} */ +static void prim_tryEval(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.mkAttrs(v, 2); + try { + state.forceValue(*args[0]); + v.attrs->push_back(Attr(state.sValue, args[0])); + mkBool(*state.allocAttr(v, state.symbols.Create("success")), true); + } catch (AssertionError& e) { + mkBool(*state.allocAttr(v, state.sValue), false); + mkBool(*state.allocAttr(v, state.symbols.Create("success")), false); + } +} + +/* Return an environment variable. Use with care. */ +static void prim_getEnv(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string name = state.forceStringNoCtx(*args[0], pos); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval + ? "" + : getEnv(name).value_or("")); +} + +/* Evaluate the first argument, then return the second argument. */ +static void prim_seq(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + +/* Evaluate the first argument deeply (i.e. recursing into lists and + attrsets), then return the second argument. */ +static void prim_deepSeq(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValueDeep(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + +/* Evaluate the first expression and print it on standard error. Then + return the second expression. Useful for debugging. */ +static void prim_trace(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + if (args[0]->type == tString) { + LOG(INFO) << "trace: " << args[0]->string.s; + } else { + LOG(INFO) << "trace: " << *args[0]; + } + state.forceValue(*args[1]); + v = *args[1]; +} + +void prim_valueSize(EvalState& state, const Pos& pos, Value** args, Value& v) { + /* We're not forcing the argument on purpose. */ + mkInt(v, valueSize(*args[0])); +} + +/************************************************************* + * Derivations + *************************************************************/ + +/* Construct (as a unobservable side effect) a Nix derivation + expression that performs the derivation described by the argument + set. Returns the original set extended with the following + attributes: `outPath' containing the primary output path of the + derivation; `drvPath' containing the path of the Nix expression; + and `type' set to `derivation' to indicate that this is a + derivation. */ +static void prim_derivationStrict(EvalState& state, const Pos& pos, + Value** args, Value& v) { + state.forceAttrs(*args[0], pos); + + /* Figure out the name first (for stack backtraces). */ + Bindings::iterator attr = args[0]->attrs->find(state.sName); + if (attr == args[0]->attrs->end()) { + throw EvalError(format("required attribute 'name' missing, at %1%") % pos); + } + std::string drvName; + Pos& posDrvName(*attr->second.pos); + try { + drvName = state.forceStringNoCtx(*attr->second.value, pos); + } catch (Error& e) { + e.addPrefix( + format("while evaluating the derivation attribute 'name' at %1%:\n") % + posDrvName); + throw; + } + + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr<JSONObject> jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && + state.forceBool(*attr->second.value, pos)) { + jsonObject = std::make_unique<JSONObject>(jsonBuf); + } + + /* Check whether null attributes should be ignored. */ + bool ignoreNulls = false; + attr = args[0]->attrs->find(state.sIgnoreNulls); + if (attr != args[0]->attrs->end()) { + ignoreNulls = state.forceBool(*attr->second.value, pos); + } + + /* Build the derivation expression by processing the attributes. */ + Derivation drv; + + PathSet context; + + std::optional<std::string> outputHash; + std::string outputHashAlgo; + bool outputHashRecursive = false; + + StringSet outputs; + outputs.insert("out"); + + for (auto& [_, i] : *args[0]->attrs) { + if (i.name == state.sIgnoreNulls) { + continue; + } + const std::string& key = i.name; + + auto handleHashMode = [&](const std::string& s) { + if (s == "recursive") { + outputHashRecursive = true; + } else if (s == "flat") { + outputHashRecursive = false; + } else { + throw EvalError( + "invalid value '%s' for 'outputHashMode' attribute, at %s", s, + posDrvName); + } + }; + + auto handleOutputs = [&](const Strings& ss) { + outputs.clear(); + for (auto& j : ss) { + if (outputs.find(j) != outputs.end()) { + throw EvalError(format("duplicate derivation output '%1%', at %2%") % + j % posDrvName); + } + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") { + throw EvalError( + format("invalid derivation output name 'drv', at %1%") % + posDrvName); + } + outputs.insert(j); + } + if (outputs.empty()) { + throw EvalError( + format("derivation cannot have an empty set of outputs, at %1%") % + posDrvName); + } + }; + + try { + if (ignoreNulls) { + state.forceValue(*i.value); + if (i.value->type == tNull) { + continue; + } + } + + /* The `args' attribute is special: it supplies the + command-line arguments to the builder. */ + if (i.name == state.sArgs) { + state.forceList(*i.value, pos); + for (unsigned int n = 0; n < i.value->listSize(); ++n) { + std::string s = state.coerceToString(posDrvName, *(*i.value->list)[n], + context, true); + drv.args.push_back(s); + } + } + + /* All other attributes are passed to the builder through + the environment. */ + else { + if (jsonObject) { + if (i.name == state.sStructuredAttrs) { + continue; + } + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i.value, placeholder, context); + + if (i.name == state.sBuilder) { + drv.builder = state.forceString(*i.value, context, posDrvName); + } else if (i.name == state.sSystem) { + drv.platform = state.forceStringNoCtx(*i.value, posDrvName); + } else if (i.name == state.sOutputHash) { + outputHash = state.forceStringNoCtx(*i.value, posDrvName); + } else if (i.name == state.sOutputHashAlgo) { + outputHashAlgo = state.forceStringNoCtx(*i.value, posDrvName); + } else if (i.name == state.sOutputHashMode) { + handleHashMode(state.forceStringNoCtx(*i.value, posDrvName)); + } else if (i.name == state.sOutputs) { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i.value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i.value->listSize(); ++n) { + ss.emplace_back( + state.forceStringNoCtx(*(*i.value->list)[n], posDrvName)); + } + handleOutputs(ss); + } + + } else { + auto s = state.coerceToString(posDrvName, *i.value, context, true); + drv.env.emplace(key, s); + if (i.name == state.sBuilder) { + drv.builder = s; + } else if (i.name == state.sSystem) { + drv.platform = s; + } else if (i.name == state.sOutputHash) { + outputHash = s; + } else if (i.name == state.sOutputHashAlgo) { + outputHashAlgo = s; + } else if (i.name == state.sOutputHashMode) { + handleHashMode(s); + } else if (i.name == state.sOutputs) { + handleOutputs(absl::StrSplit(s, absl::ByAnyChar(" \t\n\r"), + absl::SkipEmpty())); + } + } + } + + } catch (Error& e) { + e.addPrefix(format("while evaluating the attribute '%1%' of the " + "derivation '%2%' at %3%:\n") % + key % drvName % posDrvName); + throw; + } + } + + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + + /* Everything in the context of the strings in the derivation + attributes should be added as dependencies of the resulting + derivation. */ + for (auto& path : context) { + /* Paths marked with `=' denote that the path of a derivation + is explicitly passed to the builder. Since that allows the + builder to gain access to every path in the dependency + graph of the derivation (including all outputs), all paths + in the graph must be added to this derivation's list of + inputs to ensure that they are available when the builder + runs. */ + if (path.at(0) == '=') { + /* !!! This doesn't work if readOnlyMode is set. */ + PathSet refs; + state.store->computeFSClosure(std::string(path, 1), refs); + for (auto& j : refs) { + drv.inputSrcs.insert(j); + if (isDerivation(j)) { + drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); + } + } + } + + /* Handle derivation outputs of the form ‘!<name>!<path>’. */ + else if (path.at(0) == '!') { + std::pair<std::string, std::string> ctx = decodeContext(path); + drv.inputDrvs[ctx.first].insert(ctx.second); + } + + /* Otherwise it's a source file. */ + else { + drv.inputSrcs.insert(path); + } + } + + /* Do we have all required attributes? */ + if (drv.builder.empty()) { + throw EvalError(format("required attribute 'builder' missing, at %1%") % + posDrvName); + } + if (drv.platform.empty()) { + throw EvalError(format("required attribute 'system' missing, at %1%") % + posDrvName); + } + + /* Check whether the derivation name is valid. */ + checkStoreName(drvName); + if (isDerivation(drvName)) { + throw EvalError( + format("derivation names are not allowed to end in '%1%', at %2%") % + drvExtension % posDrvName); + } + + if (outputHash) { + /* Handle fixed-output derivations. */ + if (outputs.size() != 1 || *(outputs.begin()) != "out") { + throw Error(format("multiple outputs are not supported in fixed-output " + "derivations, at %1%") % + posDrvName); + } + + HashType ht = + outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); + auto hash_ = Hash::deserialize(*outputHash, ht); + auto h = Hash::unwrap_throw(hash_); + + Path outPath = + state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + if (!jsonObject) { + drv.env["out"] = outPath; + } + drv.outputs["out"] = DerivationOutput( + outPath, (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false)); + } + + else { + /* Construct the "masked" store derivation, which is the final + one except that in the list of outputs, the output paths + are empty, and the corresponding environment variables have + an empty value. This ensures that changes in the set of + output names do get reflected in the hash. */ + for (auto& i : outputs) { + if (!jsonObject) { + drv.env[i] = ""; + } + drv.outputs[i] = DerivationOutput("", "", ""); + } + + /* Use the masked derivation expression to compute the output + path. */ + Hash h = hashDerivationModulo(*state.store, drv); + + for (auto& i : drv.outputs) { + if (i.second.path.empty()) { + Path outPath = state.store->makeOutputPath(i.first, h, drvName); + if (!jsonObject) { + drv.env[i.first] = outPath; + } + i.second.path = outPath; + } + } + } + + /* Write the resulting term into the Nix store directory. */ + Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); + + VLOG(2) << "instantiated '" << drvName << "' -> '" << drvPath << "'"; + + /* Optimisation, but required in read-only mode! because in that + case we don't actually write store derivations, so we can't + read them later. */ + drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); + + state.mkAttrs(v, 1 + drv.outputs.size()); + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + for (auto& i : drv.outputs) { + mkString(*state.allocAttr(v, state.symbols.Create(i.first)), i.second.path, + {"!" + i.first + "!" + drvPath}); + } +} + +/* Return a placeholder string for the specified output that will be + substituted by the corresponding output path at build time. For + example, 'placeholder "out"' returns the string + /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build + time, any occurence of this string in an derivation attribute will + be replaced with the concrete path in the Nix store of the output + ‘out’. */ +static void prim_placeholder(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); +} + +/************************************************************* + * Paths + *************************************************************/ + +/* Convert the argument to a path. !!! obsolete? */ +static void prim_toPath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + mkString(v, canonPath(path), context); +} + +/* Allow a valid store path to be used in an expression. This is + useful in some generated expressions such as in nix-push, which + generates a call to a function with an already existing store path + as argument. You don't want to use `toPath' here because it copies + the path to the Nix store, which yields a copy like + /nix/store/newhash-oldhash-oldname. In the past, `toPath' had + special case behaviour for store paths, but that created weird + corner cases. */ +static void prim_storePath(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink + directly in the store. The latter condition is necessary so + e.g. nix-push does the right thing. */ + if (!state.store->isStorePath(path)) { + path = canonPath(path, true); + } + if (!state.store->isInStore(path)) { + throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % + path % pos); + } + Path path2 = state.store->toStorePath(path); + if (!settings.readOnlyMode) { + state.store->ensurePath(path2); + } + context.insert(path2); + mkString(v, path, context); +} + +static void prim_pathExists(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError(format("cannot check the existence of '%1%', since path " + "'%2%' is not valid, at %3%") % + path % e.path % pos); + } + + try { + mkBool(v, pathExists(state.checkSourcePath(path))); + } catch (SysError& e) { + /* Don't give away info from errors while canonicalising + ‘path’ in restricted mode. */ + mkBool(v, false); + } catch (RestrictedPathError& e) { + mkBool(v, false); + } +} + +/* Return the base name of the given string, i.e., everything + following the last slash. */ +static void prim_baseNameOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + mkString( + v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), + context); +} + +/* Return the directory of the given path, i.e., everything before the + last slash. Return either a path or a string depending on the type + of the argument. */ +static void prim_dirOf(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); + if (args[0]->type == tPath) { + mkPath(v, dir.c_str()); + } else { + mkString(v, dir, context); + } +} + +/* Return the contents of a file as a string. */ +static void prim_readFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot read '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + std::string s = + readFile(state.checkSourcePath(state.toRealPath(path, context))); + if (s.find(static_cast<char>(0)) != std::string::npos) { + throw Error(format("the contents of the file '%1%' cannot be represented " + "as a Nix string") % + path); + } + mkString(v, s.c_str()); +} + +/* Find a file in the Nix search path. Used to implement <x> paths, + which are desugared to 'findFile __nixPath "x"'. */ +static void prim_findFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + + SearchPath searchPath; + + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + Value& v2(*(*args[0]->list)[n]); + state.forceAttrs(v2, pos); + + std::string prefix; + Bindings::iterator i = v2.attrs->find(state.symbols.Create("prefix")); + if (i != v2.attrs->end()) { + prefix = state.forceStringNoCtx(*i->second.value, pos); + } + + i = v2.attrs->find(state.symbols.Create("path")); + if (i == v2.attrs->end()) { + throw EvalError(format("attribute 'path' missing, at %1%") % pos); + } + + PathSet context; + std::string path = + state.coerceToString(pos, *i->second.value, context, false, false); + + try { + state.realiseContext(context); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot find '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + searchPath.emplace_back(prefix, path); + } + + std::string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, + state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); +} + +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) { + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + } + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), + context); +} + +/* Read a directory (without . or ..) */ +static void prim_readDir(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet ctx; + Path path = state.coerceToPath(pos, *args[0], ctx); + try { + state.realiseContext(ctx); + } catch (InvalidPathError& e) { + throw EvalError( + format("cannot read '%1%', since path '%2%' is not valid, at %3%") % + path % e.path % pos); + } + + DirEntries entries = readDirectory(state.checkSourcePath(path)); + state.mkAttrs(v, entries.size()); + + for (auto& ent : entries) { + Value* ent_val = state.allocAttr(v, state.symbols.Create(ent.name)); + if (ent.type == DT_UNKNOWN) { + ent.type = getFileType(path + "/" + ent.name); + } + mkStringNoCopy(*ent_val, ent.type == DT_REG ? "regular" + : ent.type == DT_DIR ? "directory" + : ent.type == DT_LNK ? "symlink" + : "unknown"); + } +} + +/************************************************************* + * Creating files + *************************************************************/ + +/* Convert the argument (which can be any Nix expression) to an XML + representation returned in a string. Not all Nix expressions can + be sensibly or completely represented (e.g., functions). */ +static void prim_toXML(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::ostringstream out; + PathSet context; + printValueAsXML(state, true, false, *args[0], out, context); + mkString(v, out.str(), context); +} + +/* Convert the argument (which can be any Nix expression) to a JSON + string. Not all Nix expressions can be sensibly or completely + represented (e.g., functions). */ +static void prim_toJSON(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::ostringstream out; + PathSet context; + printValueAsJSON(state, true, *args[0], out, context); + mkString(v, out.str(), context); +} + +/* Parse a JSON string to a value. */ +static void prim_fromJSON(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string s = state.forceStringNoCtx(*args[0], pos); + parseJSON(state, s, v); +} + +/* Store a string in the Nix store as a source file that can be used + as an input by derivations. */ +static void prim_toFile(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string name = state.forceStringNoCtx(*args[0], pos); + std::string contents = state.forceString(*args[1], context, pos); + + PathSet refs; + + for (auto path : context) { + if (path.at(0) != '/') { + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to " + "derivation outputs, at %2%") % + name % pos); + } + refs.insert(path); + } + + Path storePath = + settings.readOnlyMode + ? state.store->computeStorePathForText(name, contents, refs) + : state.store->addTextToStore(name, contents, refs, state.repair); + + /* Note: we don't need to add `context' to the context of the + result, since `storePath' itself has references to the paths + used in args[1]. */ + + mkString(v, storePath, {storePath}); +} + +static void addPath(EvalState& state, const Pos& pos, const std::string& name, + const Path& path_, Value* filterFun, bool recursive, + const Hash& expectedHash, Value& v) { + const auto path = evalSettings.pureEval && expectedHash + ? path_ + : state.checkSourcePath(path_); + PathFilter filter = filterFun != nullptr ? ([&](const Path& path) { + auto st = lstat(path); + + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); + + Value fun2; + state.callFunction(*filterFun, arg1, fun2, noPos); + + Value arg2; + mkString(arg2, S_ISREG(st.st_mode) ? "regular" + : S_ISDIR(st.st_mode) ? "directory" + : S_ISLNK(st.st_mode) + ? "symlink" + : "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res, noPos); + + return state.forceBool(res, pos); + }) + : defaultPathFilter; + + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store + ->computeStorePathForPath(name, path, recursive, + htSHA256, filter) + .first + : state.store->addToStore(name, path, recursive, htSHA256, + filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path " + "added from '%1%'") % + path); + } + } else { + dstPath = expectedStorePath; + } + + mkString(v, dstPath, {dstPath}); +} + +static void prim_filterSource(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) { + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % + path % pos); + } + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) { + throw TypeError(format("first argument in call to 'filterSource' is not a " + "function but %1%, at %2%") % + showType(*args[0]) % pos); + } + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + Path path; + std::string name; + Value* filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto& attr : *args[0]->attrs) { + const std::string& n(attr.second.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.second.pos, *attr.second.value, context); + if (!context.empty()) { + throw EvalError( + format("string '%1%' cannot refer to other paths, at %2%") % path % + *attr.second.pos); + } + } else if (attr.second.name == state.sName) { + name = state.forceStringNoCtx(*attr.second.value, *attr.second.pos); + } else if (n == "filter") { + state.forceValue(*attr.second.value); + filterFun = attr.second.value; + } else if (n == "recursive") { + recursive = state.forceBool(*attr.second.value, *attr.second.pos); + } else if (n == "sha256") { + auto hash_ = Hash::deserialize( + state.forceStringNoCtx(*attr.second.value, *attr.second.pos), + htSHA256); + expectedHash = Hash::unwrap_throw(hash_); + } else { + throw EvalError( + format("unsupported argument '%1%' to 'addPath', at %2%") % + attr.second.name % *attr.second.pos); + } + } + if (path.empty()) { + throw EvalError(format("'path' required, at %1%") % pos); + } + if (name.empty()) { + name = baseNameOf(path); + } + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + +/************************************************************* + * Sets + *************************************************************/ + +/* Return the names of the attributes in a set as a sorted list of + strings. */ +static void prim_attrNames(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + + state.mkList(v, args[0]->attrs->size()); + + unsigned int n = 0; + for (auto& [key, value] : *args[0]->attrs) { + mkString(*((*v.list)[n++] = state.allocValue()), key); + } +} + +/* Return the values of the attributes in a set as a list, in the same + order as attrNames. */ +static void prim_attrValues(EvalState& state, const Pos& pos, Value** input, + Value& output) { + state.forceAttrs(*input[0], pos); + + state.mkList(output, input[0]->attrs->size()); + + unsigned int n = 0; + for (auto& [key, value] : *input[0]->attrs) { + (*output.list)[n++] = value.value; + } +} + +/* Dynamic version of the `.' operator. */ +void prim_getAttr(EvalState& state, const Pos& pos, Value** args, Value& v) { + std::string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + // !!! Should we create a symbol here or just do a lookup? + Bindings::iterator i = args[1]->attrs->find(state.symbols.Create(attr)); + if (i == args[1]->attrs->end()) { + throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); + } + // !!! add to stack trace? + if (state.countCalls && (i->second.pos != nullptr)) { + state.attrSelects[*i->second.pos]++; + } + state.forceValue(*i->second.value); + v = *i->second.value; +} + +/* Return position information of the specified attribute. */ +void prim_unsafeGetAttrPos(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + Bindings::iterator i = args[1]->attrs->find(state.symbols.Create(attr)); + if (i == args[1]->attrs->end()) { + mkNull(v); + } else { + state.mkPos(v, i->second.pos); + } +} + +/* Dynamic version of the `?' operator. */ +static void prim_hasAttr(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + mkBool(v, args[1]->attrs->find(state.symbols.Create(attr)) != + args[1]->attrs->end()); +} + +/* Determine whether the argument is a set. */ +static void prim_isAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tAttrs); +} + +static void prim_removeAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + state.forceList(*args[1], pos); + + /* Get the attribute names to be removed. */ + std::set<Symbol> names; + for (unsigned int i = 0; i < args[1]->listSize(); ++i) { + state.forceStringNoCtx(*(*args[1]->list)[i], pos); + names.insert(state.symbols.Create((*args[1]->list)[i]->string.s)); + } + + /* Copy all attributes not in that set. Note that we don't need + to sort v.attrs because it's a subset of an already sorted + vector. */ + state.mkAttrs(v, args[0]->attrs->size()); + for (auto& i : *args[0]->attrs) { + if (names.find(i.second.name) == names.end()) { + v.attrs->push_back(i.second); + } + } +} + +/* Builds a set from a list specifying (name, value) pairs. To be + precise, a list [{name = "name1"; value = value1;} ... {name = + "nameN"; value = valueN;}] is transformed to {name1 = value1; + ... nameN = valueN;}. In case of duplicate occurences of the same + name, the first takes precedence. */ +static void prim_listToAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + + state.mkAttrs(v, args[0]->listSize()); + + std::set<Symbol> seen; + + for (unsigned int i = 0; i < args[0]->listSize(); ++i) { + Value& v2(*(*args[0]->list)[i]); + state.forceAttrs(v2, pos); + + Bindings::iterator j = v2.attrs->find(state.sName); + if (j == v2.attrs->end()) { + throw TypeError( + format( + "'name' attribute missing in a call to 'listToAttrs', at %1%") % + pos); + } + std::string name = state.forceStringNoCtx(*j->second.value, pos); + + Symbol sym = state.symbols.Create(name); + if (seen.find(sym) == seen.end()) { + Bindings::iterator j2 = + // TODO(tazjin): this line used to construct the symbol again: + // state.symbols.Create(state.sValue)); + // Why? + v2.attrs->find(state.sValue); + if (j2 == v2.attrs->end()) { + throw TypeError(format("'value' attribute missing in a call to " + "'listToAttrs', at %1%") % + pos); + } + + v.attrs->push_back(Attr(sym, j2->second.value, j2->second.pos)); + seen.insert(sym); + } + } +} + +/* Return the right-biased intersection of two sets as1 and as2, + i.e. a set that contains every attribute from as2 that is also a + member of as1. */ +static void prim_intersectAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + + for (auto& i : *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i.second.name); + if (j != args[1]->attrs->end()) { + v.attrs->push_back(j->second); + } + } +} + +/* Collect each attribute named `attr' from a list of attribute sets. + Sets that don't contain the named attribute are ignored. + + Example: + catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + => [1 2] +*/ +static void prim_catAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + Symbol attrName = state.symbols.Create(state.forceStringNoCtx(*args[0], pos)); + state.forceList(*args[1], pos); + + Value* res[args[1]->listSize()]; + unsigned int found = 0; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value& v2(*(*args[1]->list)[n]); + state.forceAttrs(v2, pos); + Bindings::iterator i = v2.attrs->find(attrName); + if (i != v2.attrs->end()) { + res[found++] = i->second.value; + } + } + + state.mkList(v, found); + for (unsigned int n = 0; n < found; ++n) { + (*v.list)[n] = res[n]; + } +} + +/* Return a set containing the names of the formal arguments expected + by the function `f'. The value of each attribute is a Boolean + denoting whether the corresponding argument has a default value. For + instance, + + functionArgs ({ x, y ? 123}: ...) + => { x = false; y = true; } + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + + functionArgs (x: ...) + => { } +*/ +static void prim_functionArgs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + // TODO(sterni): return set of formal arguments for fetch* primops + state.mkAttrs(v, 0); + return; + } + if (args[0]->type != tLambda) { + throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); + } + + if (!args[0]->lambda.fun->matchAttrs) { + state.mkAttrs(v, 0); + return; + } + + state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); + for (auto& i : args[0]->lambda.fun->formals->formals) { + // !!! should optimise booleans (allocate only once) + // TODO(tazjin): figure out what the above comment means + mkBool(*state.allocAttr(v, i.name), i.def != nullptr); + } +} + +/* Apply a function to every element of an attribute set. */ +static void prim_mapAttrs(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, args[1]->attrs->size()); + + for (auto& i : *args[1]->attrs) { + Value* vName = state.allocValue(); + Value* vFun2 = state.allocValue(); + mkString(*vName, i.second.name); + mkApp(*vFun2, *args[0], *vName); + mkApp(*state.allocAttr(v, i.second.name), *vFun2, *i.second.value); + } +} + +/************************************************************* + * Lists + *************************************************************/ + +/* Determine whether the argument is a list. */ +static void prim_isList(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + mkBool(v, args[0]->isList()); +} + +static void elemAt(EvalState& state, const Pos& pos, Value& list, int n, + Value& v) { + state.forceList(list, pos); + if (n < 0 || static_cast<unsigned int>(n) >= list.listSize()) { + throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); + } + state.forceValue(*(*list.list)[n]); + v = *(*list.list)[n]; +} + +/* Return the n-1'th element of a list. */ +static void prim_elemAt(EvalState& state, const Pos& pos, Value** args, + Value& v) { + elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); +} + +/* Return the first element of a list. */ +static void prim_head(EvalState& state, const Pos& pos, Value** args, + Value& v) { + elemAt(state, pos, *args[0], 0, v); +} + +/* Return a list consisting of everything but the first element of + a list. Warning: this function takes O(n) time, so you probably + don't want to use it! */ +static void prim_tail(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + if (args[0]->listSize() == 0) { + throw Error(format("'tail' called on an empty list, at %1%") % pos); + } + state.mkList(v, args[0]->listSize() - 1); + for (unsigned int n = 0; n < v.listSize(); ++n) { + (*v.list)[n] = (*args[0]->list)[n + 1]; + } +} + +/* Apply a function to every element of a list. */ +static void prim_map(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceList(*args[1], pos); + + state.mkList(v, args[1]->listSize()); + + for (unsigned int n = 0; n < v.listSize(); ++n) { + mkApp(*((*v.list)[n] = state.allocValue()), *args[0], *(*args[1]->list)[n]); + } +} + +/* Filter a list using a predicate; that is, return a list containing + every element from the list for which the predicate function + returns true. */ +static void prim_filter(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + // FIXME: putting this on the stack is risky. + Value* vs[args[1]->listSize()]; + unsigned int k = 0; + + bool same = true; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value res; + state.callFunction(*args[0], *(*args[1]->list)[n], res, noPos); + if (state.forceBool(res, pos)) { + vs[k++] = (*args[1]->list)[n]; + } else { + same = false; + } + } + + if (same) { + v = *args[1]; + } else { + state.mkList(v, k); + for (unsigned int n = 0; n < k; ++n) { + (*v.list)[n] = vs[n]; + } + } +} + +/* Return true if a list contains a given element. */ +static void prim_elem(EvalState& state, const Pos& pos, Value** args, + Value& v) { + bool res = false; + state.forceList(*args[1], pos); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (state.eqValues(*args[0], *(*args[1]->list)[n])) { + res = true; + break; + } + } + mkBool(v, res); +} + +/* Concatenate a list of lists. */ +static void prim_concatLists(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + state.concatLists(v, *args[0]->list, pos); +} + +/* Return the length of a list. This is an O(1) time operation. */ +static void prim_length(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + mkInt(v, args[0]->listSize()); +} + +/* Reduce a list by applying a binary operator, from left to + right. The operator is applied strictly. */ +static void prim_foldlStrict(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[2], pos); + + if (args[2]->listSize() != 0u) { + Value* vCur = args[1]; + + for (unsigned int n = 0; n < args[2]->listSize(); ++n) { + Value vTmp; + state.callFunction(*args[0], *vCur, vTmp, pos); + vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); + state.callFunction(vTmp, *(*args[2]->list)[n], *vCur, pos); + } + state.forceValue(v); + } else { + state.forceValue(*args[1]); + v = *args[1]; + } +} + +static void anyOrAll(bool any, EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + Value vTmp; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + state.callFunction(*args[0], *(*args[1]->list)[n], vTmp, pos); + bool res = state.forceBool(vTmp, pos); + if (res == any) { + mkBool(v, any); + return; + } + } + + mkBool(v, !any); +} + +static void prim_any(EvalState& state, const Pos& pos, Value** args, Value& v) { + anyOrAll(true, state, pos, args, v); +} + +static void prim_all(EvalState& state, const Pos& pos, Value** args, Value& v) { + anyOrAll(false, state, pos, args, v); +} + +static void prim_genList(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto len = state.forceInt(*args[1], pos); + + if (len < 0) { + throw EvalError(format("cannot create list of size %1%, at %2%") % len % + pos); + } + + state.mkList(v, len); + + for (unsigned int n = 0; n < static_cast<unsigned int>(len); ++n) { + Value* arg = state.allocValue(); + mkInt(*arg, n); + mkApp(*((*v.list)[n] = state.allocValue()), *args[0], *arg); + } +} + +static void prim_lessThan(EvalState& state, const Pos& pos, Value** args, + Value& v); + +static void prim_sort(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + // Copy of the input list which can be sorted in place. + v.type = tList; + v.list = std::make_shared<NixList>(*args[1]->list); + + std::for_each(v.list->begin(), v.list->end(), + [&](Value* val) { state.forceValue(*val); }); + + auto comparator = [&](Value* a, Value* b) { + /* Optimization: if the comparator is lessThan, bypass + callFunction. */ + if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) { + return CompareValues()(a, b); + } + + Value vTmp1{}; + Value vTmp2{}; + state.callFunction(*args[0], *a, vTmp1, pos); + state.callFunction(vTmp1, *b, vTmp2, pos); + return state.forceBool(vTmp2, pos); + }; + + /* FIXME: std::sort can segfault if the comparator is not a strict + weak ordering. What to do? std::stable_sort() seems more + resilient, but no guarantees... */ + std::stable_sort(v.list->begin(), v.list->end(), comparator); +} + +static void prim_partition(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + std::shared_ptr<NixList> right = std::make_shared<NixList>(); + std::shared_ptr<NixList> wrong = std::make_shared<NixList>(); + + for (Value* elem : *args[1]->list) { + state.forceValue(*elem, pos); + + Value res; + state.callFunction(*args[0], *elem, res, pos); + if (state.forceBool(res, pos)) { + right->push_back(elem); + } else { + wrong->push_back(elem); + } + } + + state.mkAttrs(v, 2); + + Value* vRight = state.allocAttr(v, state.sRight); + state.mkList(*vRight, right); + + Value* vWrong = state.allocAttr(v, state.sWrong); + state.mkList(*vWrong, wrong); +} + +/* concatMap = f: list: concatLists (map f list); */ +/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ +static void prim_concatMap(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + std::shared_ptr<NixList> outlist = std::make_shared<NixList>(); + + for (Value* elem : *args[1]->list) { + auto out = state.allocValue(); + state.callFunction(*args[0], *elem, *out, pos); + state.forceList(*out, pos); + + outlist->insert(outlist->end(), out->list->begin(), out->list->end()); + } + + state.mkList(v, outlist); +} + +/************************************************************* + * Integer arithmetic + *************************************************************/ + +static void prim_add(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + } else { + mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); + } +} + +static void prim_sub(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + } else { + mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); + } +} + +static void prim_mul(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + } else { + mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); + } +} + +static void prim_div(EvalState& state, const Pos& pos, Value** args, Value& v) { + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + + NixFloat f2 = state.forceFloat(*args[1], pos); + if (f2 == 0) { + throw EvalError(format("division by zero, at %1%") % pos); + } + + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, + state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + } else { + NixInt i1 = state.forceInt(*args[0], pos); + NixInt i2 = state.forceInt(*args[1], pos); + /* Avoid division overflow as it might raise SIGFPE. */ + if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1) { + throw EvalError(format("overflow in integer division, at %1%") % pos); + } + mkInt(v, i1 / i2); + } +} + +static void prim_bitAnd(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +} + +static void prim_bitOr(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +} + +static void prim_bitXor(EvalState& state, const Pos& pos, Value** args, + Value& v) { + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +} + +static void prim_lessThan(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceValue(*args[0]); + state.forceValue(*args[1]); + CompareValues comp; + mkBool(v, comp(args[0], args[1])); +} + +/************************************************************* + * String manipulation + *************************************************************/ + +/* Convert the argument to a string. Paths are *not* copied to the + store, so `toString /foo/bar' yields `"/foo/bar"', not + `"/nix/store/whatever..."'. */ +static void prim_toString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context, true, false); + mkString(v, s, context); +} + +/* `substring start len str' returns the substring of `str' starting + at character position `min(start, stringLength str)' inclusive and + ending at `min(start + len, stringLength str)'. `start' must be + non-negative. */ +static void prim_substring(EvalState& state, const Pos& pos, Value** args, + Value& v) { + int start = state.forceInt(*args[0], pos); + int len = state.forceInt(*args[1], pos); + PathSet context; + std::string s = state.coerceToString(pos, *args[2], context); + + if (start < 0) { + throw EvalError(format("negative start position in 'substring', at %1%") % + pos); + } + + mkString(v, + static_cast<unsigned int>(start) >= s.size() + ? "" + : std::string(s, start, len), + context); +} + +static void prim_stringLength(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + mkInt(v, s.size()); +} + +/* Return the cryptographic hash of a string in base-16. */ +static void prim_hashString(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) { + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + } + + PathSet context; // discarded + std::string s = state.forceString(*args[1], context, pos); + + mkString(v, hashString(ht, s).to_string(Base16, false), context); +} + +/* Match a regular expression against a string and return either + ‘null’ or a list containing substring matches. */ +static void prim_match(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + std::smatch match; + if (!std::regex_match(str, match, regex)) { + mkNull(v); + return; + } + + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i + 1].matched) { + mkNull(*((*v.list)[i] = state.allocValue())); + } else { + mkString(*((*v.list)[i] = state.allocValue()), + match[i + 1].str().c_str()); + } + } + + } catch (std::regex_error& e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", + re, pos); + } + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } +} + +/* Split a std::string with a regular expression, and return a list of the + non-matching parts interleaved by the lists of the matching groups. */ +static void prim_split(EvalState& state, const Pos& pos, Value** args, + Value& v) { + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + auto begin = std::sregex_iterator(str.begin(), str.end(), regex); + auto end = std::sregex_iterator(); + + // Any matches results are surrounded by non-matching results. + const size_t len = std::distance(begin, end); + state.mkList(v, 2 * len + 1); + size_t idx = 0; + Value* elem; + + if (len == 0) { + (*v.list)[idx++] = args[1]; + return; + } + + for (std::sregex_iterator i = begin; i != end; ++i) { + assert(idx <= 2 * len + 1 - 3); + std::smatch match = *i; + + // Add a string for non-matched characters. + elem = (*v.list)[idx++] = state.allocValue(); + mkString(*elem, match.prefix().str().c_str()); + + // Add a list for matched substrings. + const size_t slen = match.size() - 1; + elem = (*v.list)[idx++] = state.allocValue(); + + // Start at 1, beacause the first match is the whole string. + state.mkList(*elem, slen); + for (size_t si = 0; si < slen; ++si) { + if (!match[si + 1].matched) { + mkNull(*((*elem->list)[si] = state.allocValue())); + } else { + mkString(*((*elem->list)[si] = state.allocValue()), + match[si + 1].str().c_str()); + } + } + + // Add a string for non-matched suffix characters. + if (idx == 2 * len) { + elem = (*v.list)[idx++] = state.allocValue(); + mkString(*elem, match.suffix().str().c_str()); + } + } + assert(idx == 2 * len + 1); + + } catch (std::regex_error& e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", + re, pos); + } + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } +} + +static void prim_concatStringSep(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + + auto sep = state.forceString(*args[0], context, pos); + state.forceList(*args[1], pos); + + std::string res; + res.reserve((args[1]->listSize() + 32) * sep.size()); + bool first = true; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (first) { + first = false; + } else { + res += sep; + } + + res += state.coerceToString(pos, *(*args[1]->list)[n], context); + } + + mkString(v, res, context); +} + +static void prim_replaceStrings(EvalState& state, const Pos& pos, Value** args, + Value& v) { + state.forceList(*args[0], pos); + state.forceList(*args[1], pos); + if (args[0]->listSize() != args[1]->listSize()) { + throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have " + "different lengths, at %1%") % + pos); + } + + std::vector<std::string> from; + from.reserve(args[0]->listSize()); + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + from.push_back(state.forceString(*(*args[0]->list)[n], pos)); + } + + std::vector<std::pair<std::string, PathSet>> to; + to.reserve(args[1]->listSize()); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + PathSet ctx; + auto s = state.forceString(*(*args[1]->list)[n], ctx, pos); + to.emplace_back(std::move(s), std::move(ctx)); + } + + PathSet context; + auto s = state.forceString(*args[2], context, pos); + + std::string res; + // Loops one past last character to handle the case where 'from' contains an + // empty string. + for (size_t p = 0; p <= s.size();) { + bool found = false; + auto i = from.begin(); + auto j = to.begin(); + for (; i != from.end(); ++i, ++j) { + if (s.compare(p, i->size(), *i) == 0) { + found = true; + res += j->first; + if (i->empty()) { + if (p < s.size()) { + res += s[p]; + } + p++; + } else { + p += i->size(); + } + for (auto& path : j->second) { + context.insert(path); + } + j->second.clear(); + break; + } + } + if (!found) { + if (p < s.size()) { + res += s[p]; + } + p++; + } + } + + mkString(v, res, context); +} + +/************************************************************* + * Versions + *************************************************************/ + +static void prim_parseDrvName(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string name = state.forceStringNoCtx(*args[0], pos); + DrvName parsed(name); + state.mkAttrs(v, 2); + mkString(*state.allocAttr(v, state.sName), parsed.name); + mkString(*state.allocAttr(v, state.symbols.Create("version")), + parsed.version); +} + +static void prim_compareVersions(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string version1 = state.forceStringNoCtx(*args[0], pos); + std::string version2 = state.forceStringNoCtx(*args[1], pos); + mkInt(v, compareVersions(version1, version2)); +} + +static void prim_splitVersion(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) { + break; + } + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto& component : components) { + auto listElem = (*v.list)[n++] = state.allocValue(); + mkString(*listElem, component); + } +} + +/************************************************************* + * Networking + *************************************************************/ + +void fetch(EvalState& state, const Pos& pos, Value** args, Value& v, + const std::string& who, bool unpack, + const std::string& defaultName) { + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr : *args[0]->attrs) { + std::string n(attr.second.name); + if (n == "url") { + request.uri = + state.forceStringNoCtx(*attr.second.value, *attr.second.pos); + } else if (n == "sha256") { + auto hash_ = Hash::deserialize( + state.forceStringNoCtx(*attr.second.value, *attr.second.pos), + htSHA256); + request.expectedHash = Hash::unwrap_throw(hash_); + } else if (n == "name") { + request.name = + state.forceStringNoCtx(*attr.second.value, *attr.second.pos); + } else { + throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % + attr.second.name % who % attr.second.pos); + } + } + + if (request.uri.empty()) { + throw EvalError(format("'url' argument required, at %1%") % pos); + } + + } else { + request.uri = state.forceStringNoCtx(*args[0], pos); + } + + state.checkURI(request.uri); + + if (evalSettings.pureEval && !request.expectedHash) { + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", + who); + } + + auto res = getDownloader()->downloadCached(state.store, request); + + if (state.allowedPaths) { + state.allowedPaths->insert(res.path); + } + + mkString(v, res.storePath, PathSet({res.storePath})); +} + +static void prim_fetchurl(EvalState& state, const Pos& pos, Value** args, + Value& v) { + fetch(state, pos, args, v, "fetchurl", false, ""); +} + +static void prim_fetchTarball(EvalState& state, const Pos& pos, Value** args, + Value& v) { + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + +/************************************************************* + * Primop registration + *************************************************************/ + +RegisterPrimOp::PrimOps* RegisterPrimOp::primOps; + +RegisterPrimOp::RegisterPrimOp(const std::string& name, size_t arity, + PrimOpFun fun) { + if (primOps == nullptr) { + primOps = new PrimOps; + } + primOps->emplace_back(name, arity, fun); +} + +void EvalState::createBaseEnv() { + baseEnv.up = nullptr; + + /* Add global constants such as `true' to the base environment. */ + Value v; + + /* `builtins' must be first! */ + mkAttrs(v, 128); + addConstant("builtins", v); + + mkBool(v, true); + addConstant("true", v); + + mkBool(v, false); + addConstant("false", v); + + mkNull(v); + addConstant("null", v); + + auto vThrow = addPrimOp("throw", 1, prim_throw); + + auto addPurityError = [&](const std::string& name) { + Value* v2 = allocValue(); + mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); + mkApp(v, *vThrow, *v2); + addConstant(name, v); + }; + + if (!evalSettings.pureEval) { + mkInt(v, time(nullptr)); + addConstant("__currentTime", v); + } + + if (!evalSettings.pureEval) { + mkString(v, settings.thisSystem); + addConstant("__currentSystem", v); + } + + mkString(v, nixVersion); + addConstant("__nixVersion", v); + + mkString(v, store->storeDir); + addConstant("__storeDir", v); + + /* Language version. This should be increased every time a new + language feature gets added. It's not necessary to increase it + when primops get added, because you can just use `builtins ? + primOp' to check. */ + mkInt(v, 5); + addConstant("__langVersion", v); + + // Miscellaneous + auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); + Value* v2 = allocValue(); + mkAttrs(*v2, 0); + mkApp(v, *vScopedImport, *v2); + forceValue(v); + addConstant("import", v); + addPrimOp("__typeOf", 1, prim_typeOf); + addPrimOp("isNull", 1, prim_isNull); + addPrimOp("__isFunction", 1, prim_isFunction); + addPrimOp("__isString", 1, prim_isString); + addPrimOp("__isInt", 1, prim_isInt); + addPrimOp("__isFloat", 1, prim_isFloat); + addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); + addPrimOp("__genericClosure", 1, prim_genericClosure); + addPrimOp("abort", 1, prim_abort); + addPrimOp("__addErrorContext", 2, prim_addErrorContext); + addPrimOp("__tryEval", 1, prim_tryEval); + addPrimOp("__getEnv", 1, prim_getEnv); + + // Strictness + addPrimOp("__seq", 2, prim_seq); + addPrimOp("__deepSeq", 2, prim_deepSeq); + + // Debugging + addPrimOp("__trace", 2, prim_trace); + addPrimOp("__valueSize", 1, prim_valueSize); + + // Paths + addPrimOp("__toPath", 1, prim_toPath); + if (evalSettings.pureEval) { + addPurityError("__storePath"); + } else { + addPrimOp("__storePath", 1, prim_storePath); + } + addPrimOp("__pathExists", 1, prim_pathExists); + addPrimOp("baseNameOf", 1, prim_baseNameOf); + addPrimOp("dirOf", 1, prim_dirOf); + addPrimOp("__readFile", 1, prim_readFile); + addPrimOp("__readDir", 1, prim_readDir); + addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); + + // Creating files + addPrimOp("__toXML", 1, prim_toXML); + addPrimOp("__toJSON", 1, prim_toJSON); + addPrimOp("__fromJSON", 1, prim_fromJSON); + addPrimOp("__toFile", 2, prim_toFile); + addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); + + // Sets + addPrimOp("__attrNames", 1, prim_attrNames); + addPrimOp("__attrValues", 1, prim_attrValues); + addPrimOp("__getAttr", 2, prim_getAttr); + addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); + addPrimOp("__hasAttr", 2, prim_hasAttr); + addPrimOp("__isAttrs", 1, prim_isAttrs); + addPrimOp("removeAttrs", 2, prim_removeAttrs); + addPrimOp("__listToAttrs", 1, prim_listToAttrs); + addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); + addPrimOp("__catAttrs", 2, prim_catAttrs); + addPrimOp("__functionArgs", 1, prim_functionArgs); + addPrimOp("__mapAttrs", 2, prim_mapAttrs); + + // Lists + addPrimOp("__isList", 1, prim_isList); + addPrimOp("__elemAt", 2, prim_elemAt); + addPrimOp("__head", 1, prim_head); + addPrimOp("__tail", 1, prim_tail); + addPrimOp("map", 2, prim_map); + addPrimOp("__filter", 2, prim_filter); + addPrimOp("__elem", 2, prim_elem); + addPrimOp("__concatLists", 1, prim_concatLists); + addPrimOp("__length", 1, prim_length); + addPrimOp("__foldl'", 3, prim_foldlStrict); + addPrimOp("__any", 2, prim_any); + addPrimOp("__all", 2, prim_all); + addPrimOp("__genList", 2, prim_genList); + addPrimOp("__sort", 2, prim_sort); + addPrimOp("__partition", 2, prim_partition); + addPrimOp("__concatMap", 2, prim_concatMap); + + // Integer arithmetic + addPrimOp("__add", 2, prim_add); + addPrimOp("__sub", 2, prim_sub); + addPrimOp("__mul", 2, prim_mul); + addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); + addPrimOp("__lessThan", 2, prim_lessThan); + + // String manipulation + addPrimOp("toString", 1, prim_toString); + addPrimOp("__substring", 3, prim_substring); + addPrimOp("__stringLength", 1, prim_stringLength); + addPrimOp("__hashString", 2, prim_hashString); + addPrimOp("__match", 2, prim_match); + addPrimOp("__split", 2, prim_split); + addPrimOp("__concatStringsSep", 2, prim_concatStringSep); + addPrimOp("__replaceStrings", 3, prim_replaceStrings); + + // Versions + addPrimOp("__parseDrvName", 1, prim_parseDrvName); + addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); + + // Derivations + addPrimOp("derivationStrict", 1, prim_derivationStrict); + addPrimOp("placeholder", 1, prim_placeholder); + + // Networking + addPrimOp("__fetchurl", 1, prim_fetchurl); + addPrimOp("fetchTarball", 1, prim_fetchTarball); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + std::string path = + canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); + sDerivationNix = symbols.Create(path); + evalFile(path, v); + addConstant("derivation", v); + + /* Add a value containing the current Nix expression search path. */ + mkList(v, searchPath.size()); + int n = 0; + for (auto& i : searchPath) { + v2 = (*v.list)[n++] = allocValue(); + mkAttrs(*v2, 2); + mkString(*allocAttr(*v2, symbols.Create("path")), i.second); + mkString(*allocAttr(*v2, symbols.Create("prefix")), i.first); + } + addConstant("__nixPath", v); + + if (RegisterPrimOp::primOps != nullptr) { + for (auto& primOp : *RegisterPrimOp::primOps) { + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + } + } +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops.hh b/third_party/nix/src/libexpr/primops.hh new file mode 100644 index 000000000000..ab5f64720273 --- /dev/null +++ b/third_party/nix/src/libexpr/primops.hh @@ -0,0 +1,17 @@ +#include <tuple> +#include <vector> + +#include "libexpr/eval.hh" + +namespace nix { + +struct RegisterPrimOp { + using PrimOps = std::vector<std::tuple<std::string, size_t, PrimOpFun> >; + static PrimOps* primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ + RegisterPrimOp(const std::string& name, size_t arity, PrimOpFun fun); +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/context.cc b/third_party/nix/src/libexpr/primops/context.cc new file mode 100644 index 000000000000..fb8879ead16d --- /dev/null +++ b/third_party/nix/src/libexpr/primops/context.cc @@ -0,0 +1,202 @@ +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" +#include "libstore/derivations.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState& state, const Pos& pos, + Value** args, Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, + prim_unsafeDiscardStringContext); + +static void prim_hasContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState& state, const Pos& pos, + Value** args, Value& v) { + PathSet context; + std::string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto& p : context) { + context2.insert(p.at(0) == '=' ? std::string(p, 1) : p); + } + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, + prim_unsafeDiscardOutputDependency); + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present. +*/ +static void prim_getContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map<Path, ContextInfo>(); + for (const auto& p : context) { + Path drv; + std::string output; + const Path* path = &p; + if (p.at(0) == '=') { + drv = std::string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair<std::string, std::string> ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace( + *path, + ContextInfo{isPath, isAllOutputs, + output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) { + iter->second.path = true; + } else if (isAllOutputs) { + iter->second.allOutputs = true; + } else { + iter->second.outputs.emplace_back(std::move(output)); + } + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.Create("path"); + auto sAllOutputs = state.symbols.Create("allOutputs"); + for (const auto& info : contextInfos) { + auto& infoVal = *state.allocAttr(v, state.symbols.Create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) { + mkBool(*state.allocAttr(infoVal, sPath), true); + } + if (info.second.allOutputs) { + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + } + if (!info.second.outputs.empty()) { + auto& outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto& output : info.second.outputs) { + mkString(*((*outputsVal.list)[i++] = state.allocValue()), output); + } + } + } +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState& state, const Pos& pos, Value** args, + Value& v) { + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.Create("path"); + auto sAllOutputs = state.symbols.Create("allOutputs"); + for (const auto& attr_iter : *args[1]->attrs) { + const Attr* i = &attr_iter.second; // TODO(tazjin): get rid of this + if (!state.store->isStorePath(i->name)) { + throw EvalError("Context key '%s' is not a store path, at %s", i->name, + i->pos); + } + if (!settings.readOnlyMode) { + state.store->ensurePath(i->name); + } + state.forceAttrs(*i->value, *i->pos); + auto iter = i->value->attrs->find(sPath); + if (iter != i->value->attrs->end()) { + if (state.forceBool(*iter->second.value, *iter->second.pos)) { + context.insert(i->name); + } + } + + iter = i->value->attrs->find(sAllOutputs); + if (iter != i->value->attrs->end()) { + if (state.forceBool(*iter->second.value, *iter->second.pos)) { + if (!isDerivation(i->name)) { + throw EvalError( + "Tried to add all-outputs context of %s, which is not a " + "derivation, to a string, at %s", + i->name, i->pos); + } + context.insert("=" + std::string(i->name)); + } + } + + iter = i->value->attrs->find(state.sOutputs); + if (iter != i->value->attrs->end()) { + state.forceList(*iter->second.value, *iter->second.pos); + if (iter->second.value->listSize() && !isDerivation(i->name)) { + throw EvalError( + "Tried to add derivation output context of %s, which is not a " + "derivation, to a string, at %s", + i->name, i->pos); + } + for (unsigned int n = 0; n < iter->second.value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*(*iter->second.value->list)[n], + *iter->second.pos); + context.insert("!" + name + "!" + std::string(i->name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fetchGit.cc b/third_party/nix/src/libexpr/primops/fetchGit.cc new file mode 100644 index 000000000000..da4d683401d7 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fetchGit.cc @@ -0,0 +1,277 @@ +#include <nlohmann/json.hpp> +#include <regex> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sys/time.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" +#include "libstore/download.hh" +#include "libstore/pathlocks.hh" +#include "libstore/store-api.hh" +#include "libutil/hash.hh" + +using namespace std::string_literals; + +namespace nix { + +struct GitInfo { + Path storePath; + std::string rev; + std::string shortRev; + uint64_t revCount = 0; +}; + +std::regex revRegex("^[0-9a-fA-F]{40}$"); + +GitInfo exportGit(ref<Store> store, const std::string& uri, + std::optional<std::string> ref, std::string rev, + const std::string& name) { + if (evalSettings.pureEval && rev == "") { + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + } + + if (!ref && rev == "" && absl::StartsWith(uri, "/") && + pathExists(uri + "/.git")) { + bool clean = true; + + try { + runProgram("git", true, + {"-C", uri, "diff-index", "--quiet", "HEAD", "--"}); + } catch (ExecError& e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) { + throw; + } + clean = false; + } + + if (!clean) { + /* This is an unclean working tree. So copy all tracked + files. */ + + GitInfo gitInfo; + gitInfo.rev = "0000000000000000000000000000000000000000"; + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + std::set<std::string> files = + absl::StrSplit(runProgram("git", true, {"-C", uri, "ls-files", "-z"}), + absl::ByChar('\0'), absl::SkipEmpty()); + + PathFilter filter = [&](const Path& p) -> bool { + assert(absl::StartsWith(p, uri)); + std::string file(p, uri.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && absl::StartsWith(*i, prefix); + } + + return files.count(file); + }; + + gitInfo.storePath = + store->addToStore("source", uri, true, htSHA256, filter); + + return gitInfo; + } + + // clean working tree, but no ref or rev specified. Use 'HEAD'. + rev = absl::StripTrailingAsciiWhitespace( + runProgram("git", true, {"-C", uri, "rev-parse", "HEAD"})); + ref = "HEAD"s; + } + + if (!ref) { + ref = "HEAD"s; + } + + if (rev != "" && !std::regex_match(rev, revRegex)) { + throw Error("invalid Git revision '%s'", rev); + } + + deletePath(getCacheDir() + "/nix/git"); + + Path cacheDir = getCacheDir() + "/nix/gitv2/" + + hashString(htSHA256, uri).to_string(Base32, false); + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, {"init", "--bare", cacheDir}); + } + + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) { + localRefFile = cacheDir + "/" + *ref; + } else { + localRefFile = cacheDir + "/refs/heads/" + *ref; + } + + bool doFetch; + time_t now = time(0); + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev != "") { + try { + runProgram("git", true, {"-C", cacheDir, "cat-file", "-e", rev}); + doFetch = false; + } catch (ExecError& e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + static_cast<uint64_t>(st.st_mtime) + settings.tarballTtl <= + static_cast<uint64_t>(now); + } + if (doFetch) { + DLOG(INFO) << "fetching Git repository '" << uri << "'"; + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, + {"-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, + fmt("%s:%s", *ref, *ref)}); + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + // FIXME: check whether rev is an ancestor of ref. + GitInfo gitInfo; + gitInfo.rev = + rev != "" ? rev + : absl::StripTrailingAsciiWhitespace(readFile(localRefFile)); + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + VLOG(2) << "using revision " << gitInfo.rev << " of repo '" << uri << "'"; + + std::string storeLinkName = + hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev) + .to_string(Base32, false); + Path storeLink = cacheDir + "/" + storeLinkName + ".link"; + PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", + storeLink)); // FIXME: broken + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == gitInfo.rev); + + gitInfo.storePath = json["storePath"]; + + if (store->isValidPath(gitInfo.storePath)) { + gitInfo.revCount = json["revCount"]; + return gitInfo; + } + + } catch (SysError& e) { + if (e.errNo != ENOENT) { + throw; + } + } + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, {"-C", cacheDir, "archive", gitInfo.rev}); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("tar", true, {"x", "-C", tmpDir}, tar); + + gitInfo.storePath = store->addToStore(name, tmpDir); + + gitInfo.revCount = std::stoull(runProgram( + "git", true, {"-C", cacheDir, "rev-list", "--count", gitInfo.rev})); + + nlohmann::json json; + json["storePath"] = gitInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["rev"] = gitInfo.rev; + json["revCount"] = gitInfo.revCount; + + writeFile(storeLink, json.dump()); + + return gitInfo; +} + +static void prim_fetchGit(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string url; + std::optional<std::string> ref; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr_iter : *args[0]->attrs) { + auto& attr = attr_iter.second; + std::string n(attr.name); + if (n == "url") { + url = + state.coerceToString(*attr.pos, *attr.value, context, false, false); + } else if (n == "ref") { + ref = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (n == "rev") { + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (n == "name") { + name = state.forceStringNoCtx(*attr.value, *attr.pos); + } else { + throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", + attr.name, *attr.pos); + } + } + + if (url.empty()) { + throw EvalError(format("'url' argument required, at %1%") % pos); + } + + } else { + url = state.coerceToString(pos, *args[0], context, false, false); + } + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto gitInfo = exportGit(state.store, url, ref, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, + PathSet({gitInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.Create("rev")), gitInfo.rev); + mkString(*state.allocAttr(v, state.symbols.Create("shortRev")), + gitInfo.shortRev); + mkInt(*state.allocAttr(v, state.symbols.Create("revCount")), + gitInfo.revCount); + + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); + } +} + +static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fetchMercurial.cc b/third_party/nix/src/libexpr/primops/fetchMercurial.cc new file mode 100644 index 000000000000..13dc61766fe0 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fetchMercurial.cc @@ -0,0 +1,246 @@ +#include <nlohmann/json.hpp> +#include <regex> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sys/time.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" +#include "libstore/download.hh" +#include "libstore/pathlocks.hh" +#include "libstore/store-api.hh" + +using namespace std::string_literals; + +namespace nix { + +struct HgInfo { + Path storePath; + std::string branch; + std::string rev; + uint64_t revCount = 0; +}; + +std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); + +HgInfo exportMercurial(ref<Store> store, const std::string& uri, + std::string rev, const std::string& name) { + if (evalSettings.pureEval && rev == "") { + throw Error( + "in pure evaluation mode, 'fetchMercurial' requires a Mercurial " + "revision"); + } + + if (rev == "" && absl::StartsWith(uri, "/") && pathExists(uri + "/.hg")) { + bool clean = runProgram("hg", true, + {"status", "-R", uri, "--modified", "--added", + "--removed"}) == ""; + + if (!clean) { + /* This is an unclean working tree. So copy all tracked + files. */ + + DLOG(INFO) << "copying unclean Mercurial working tree '" << uri << "'"; + + HgInfo hgInfo; + hgInfo.rev = "0000000000000000000000000000000000000000"; + hgInfo.branch = absl::StripTrailingAsciiWhitespace( + runProgram("hg", true, {"branch", "-R", uri})); + + std::set<std::string> files = absl::StrSplit( + runProgram("hg", true, + {"status", "-R", uri, "--clean", "--modified", "--added", + "--no-status", "--print0"}), + absl::ByChar('\0'), absl::SkipEmpty()); + + PathFilter filter = [&](const Path& p) -> bool { + assert(absl::StartsWith(p, uri)); + std::string file(p, uri.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && absl::StartsWith(*i, prefix); + } + + return files.count(file); + }; + + hgInfo.storePath = + store->addToStore("source", uri, true, htSHA256, filter); + + return hgInfo; + } + } + + if (rev == "") { + rev = "default"; + } + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), + hashString(htSHA256, uri).to_string(Base32, false)); + + Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, + hashString(htSHA512, rev).to_string(Base32, false)); + + /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, + do so now. */ + time_t now = time(0); + struct stat st; + if (stat(stampFile.c_str(), &st) != 0 || + static_cast<uint64_t>(st.st_mtime) + settings.tarballTtl <= + static_cast<uint64_t>(now)) { + /* Except that if this is a commit hash that we already have, + we don't have to pull again. */ + if (!(std::regex_match(rev, commitHashRegex) && pathExists(cacheDir) && + runProgram(RunOptions("hg", {"log", "-R", cacheDir, "-r", rev, + "--template", "1"}) + .killStderr(true)) + .second == "1")) { + DLOG(INFO) << "fetching Mercurial repository '" << uri << "'"; + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); + } catch (ExecError& e) { + std::string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, {"recover", "-R", cacheDir}); + runProgram("hg", true, {"pull", "-R", cacheDir, "--", uri}); + } else { + throw ExecError(e.status, + fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, {"clone", "--noupdate", "--", uri, cacheDir}); + } + } + + writeFile(stampFile, ""); + } + + std::vector<std::string> tokens = + absl::StrSplit(runProgram("hg", true, + {"log", "-R", cacheDir, "-r", rev, "--template", + "{node} {rev} {branch}"}), + absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + assert(tokens.size() == 3); + + HgInfo hgInfo; + hgInfo.rev = tokens[0]; + hgInfo.revCount = std::stoull(tokens[1]); + hgInfo.branch = tokens[2]; + + std::string storeLinkName = + hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev) + .to_string(Base32, false); + Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == hgInfo.rev); + + hgInfo.storePath = json["storePath"]; + + if (store->isValidPath(hgInfo.storePath)) { + DLOG(INFO) << "using cached Mercurial store path '" << hgInfo.storePath + << "'"; + return hgInfo; + } + + } catch (SysError& e) { + if (e.errNo != ENOENT) { + throw; + } + } + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, {"archive", "-R", cacheDir, "-r", rev, tmpDir}); + + deletePath(tmpDir + "/.hg_archival.txt"); + + hgInfo.storePath = store->addToStore(name, tmpDir); + + nlohmann::json json; + json["storePath"] = hgInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["branch"] = hgInfo.branch; + json["rev"] = hgInfo.rev; + json["revCount"] = hgInfo.revCount; + + writeFile(storeLink, json.dump()); + + return hgInfo; +} + +static void prim_fetchMercurial(EvalState& state, const Pos& pos, Value** args, + Value& v) { + std::string url; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + state.forceAttrs(*args[0], pos); + + for (auto& attr_iter : *args[0]->attrs) { + auto& attr = attr_iter.second; + std::string n(attr.name); + if (n == "url") { + url = + state.coerceToString(*attr.pos, *attr.value, context, false, false); + } else if (n == "rev") { + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + } else if (n == "name") { + name = state.forceStringNoCtx(*attr.value, *attr.pos); + } else { + throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", + attr.name, *attr.pos); + } + } + + if (url.empty()) { + throw EvalError(format("'url' argument required, at %1%") % pos); + } + + } else { + url = state.coerceToString(pos, *args[0], context, false, false); + } + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto hgInfo = exportMercurial(state.store, url, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, + PathSet({hgInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.Create("branch")), hgInfo.branch); + mkString(*state.allocAttr(v, state.symbols.Create("rev")), hgInfo.rev); + mkString(*state.allocAttr(v, state.symbols.Create("shortRev")), + std::string(hgInfo.rev, 0, 12)); + mkInt(*state.allocAttr(v, state.symbols.Create("revCount")), hgInfo.revCount); + + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); + } +} + +static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/primops/fromTOML.cc b/third_party/nix/src/libexpr/primops/fromTOML.cc new file mode 100644 index 000000000000..e3d2a4940769 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fromTOML.cc @@ -0,0 +1,94 @@ +#include "cpptoml/cpptoml.h" +#include "libexpr/eval-inline.hh" +#include "libexpr/primops.hh" + +namespace nix { + +static void prim_fromTOML(EvalState& state, const Pos& pos, Value** args, + Value& v) { + using namespace cpptoml; + + auto toml = state.forceStringNoCtx(*args[0], pos); + + std::istringstream tomlStream(toml); + + std::function<void(Value&, std::shared_ptr<base>)> visit; + + visit = [&](Value& v, std::shared_ptr<base> t) { + if (auto t2 = t->as_table()) { + size_t size = 0; + for (auto& i : *t2) { + (void)i; + size++; + } + + state.mkAttrs(v, size); + + for (auto& i : *t2) { + auto& v2 = *state.allocAttr(v, state.symbols.Create(i.first)); + + if (auto i2 = i.second->as_table_array()) { + size_t size2 = i2->get().size(); + state.mkList(v2, size2); + for (size_t j = 0; j < size2; ++j) { + visit(*((*v2.list)[j] = state.allocValue()), i2->get()[j]); + } + } else { + visit(v2, i.second); + } + } + } + + else if (auto t2 = t->as_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t i = 0; i < size; ++i) { + visit(*((*v.list)[i] = state.allocValue()), t2->get()[i]); + } + } + + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) { + visit(*((*v.list)[j] = state.allocValue()), t2->get()[j]); + } + } + + else if (t->is_value()) { + if (auto val = t->as<int64_t>()) { + mkInt(v, val->get()); + } else if (auto val = t->as<NixFloat>()) { + mkFloat(v, val->get()); + } else if (auto val = t->as<bool>()) { + mkBool(v, val->get()); + } else if (auto val = t->as<std::string>()) { + mkString(v, val->get()); + } else { + throw EvalError("unsupported value type in TOML"); + } + } + + else { + abort(); + } + }; + + try { + visit(v, parser(tomlStream).parse()); + } catch (std::runtime_error& e) { + throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + } +} + +static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/symbol-table.cc b/third_party/nix/src/libexpr/symbol-table.cc new file mode 100644 index 000000000000..2b27ca54c289 --- /dev/null +++ b/third_party/nix/src/libexpr/symbol-table.cc @@ -0,0 +1,24 @@ +#include "libexpr/symbol-table.hh" + +#include <absl/container/node_hash_set.h> +#include <absl/strings/string_view.h> + +namespace nix { + +Symbol SymbolTable::Create(absl::string_view sym) { + auto it = symbols_.emplace(sym); + const std::string* ptr = &(*it.first); + return Symbol(ptr); +} + +size_t SymbolTable::Size() const { return symbols_.size(); } + +size_t SymbolTable::TotalSize() const { + size_t n = 0; + for (auto& i : symbols_) { + n += i.size(); + } + return n; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/symbol-table.hh b/third_party/nix/src/libexpr/symbol-table.hh new file mode 100644 index 000000000000..c2599658859f --- /dev/null +++ b/third_party/nix/src/libexpr/symbol-table.hh @@ -0,0 +1,69 @@ +#pragma once + +#include <absl/container/node_hash_set.h> +#include <absl/strings/string_view.h> + +namespace nix { // TODO(tazjin): ::expr + +// TODO(tazjin): Replace with a simpler struct, or get rid of. +class Symbol { + private: + const std::string* s; // pointer into SymbolTable + Symbol(const std::string* s) : s(s){}; + friend class SymbolTable; + + public: + bool operator==(const Symbol& s2) const { return s == s2.s; } + + bool operator!=(const Symbol& s2) const { return s != s2.s; } + + bool operator<(const Symbol& s2) const { return *s < *s2.s; } + + operator const std::string&() const { return *s; } + + bool set() const { return s; } + + bool empty() const { return s->empty(); } + + friend std::ostream& operator<<(std::ostream& str, const Symbol& sym); + + template <typename H> + friend H AbslHashValue(H h, const Symbol& c) { + return H::combine(std::move(h), c.s); + } +}; + +// SymbolTable is a hash-set based symbol-interning mechanism. +// +// TODO(tazjin): Figure out which things use this. AttrSets, ...? +// Is it possible this only exists because AttrSet wasn't a map? +// +// Original comment: +// +// Symbol table used by the parser and evaluator to represent and look +// up identifiers and attributes efficiently. SymbolTable::create() +// converts a string into a symbol. Symbols have the property that +// they can be compared efficiently (using a pointer equality test), +// because the symbol table stores only one copy of each string. +class SymbolTable { + public: + // Create a new symbol in this table by emplacing the provided + // string into it. + // + // The symbol will reference an existing symbol if the symbol is + // already interned. + Symbol Create(absl::string_view sym); + + // Return the number of symbols interned. + size_t Size() const; + + // Return the total size (in bytes) + size_t TotalSize() const; + + private: + // flat_hash_set does not retain pointer stability on rehashing, + // hence "interned" strings/symbols are stored on the heap. + absl::node_hash_set<std::string> symbols_; +}; + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-json.cc b/third_party/nix/src/libexpr/value-to-json.cc new file mode 100644 index 000000000000..a338d4eed79a --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-json.cc @@ -0,0 +1,91 @@ +#include "libexpr/value-to-json.hh" + +#include <cstdlib> +#include <iomanip> + +#include "libexpr/eval-inline.hh" +#include "libutil/json.hh" +#include "libutil/util.hh" + +namespace nix { + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + JSONPlaceholder& out, PathSet& context) { + checkInterrupt(); + + if (strict) { + state.forceValue(v); + } + + switch (v.type) { + case tInt: + out.write(v.integer); + break; + + case tBool: + out.write(v.boolean); + break; + + case tString: + copyContext(v, context); + out.write(v.string.s); + break; + + case tPath: + out.write(state.copyPathToStore(context, v.path)); + break; + + case tNull: + out.write(nullptr); + break; + + case tAttrs: { + auto maybeString = + state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + auto obj(out.object()); + StringSet names; + for (auto& j : *v.attrs) { + names.insert(j.second.name); + } + for (auto& j : names) { + auto [_, a] = *v.attrs->find(state.symbols.Create(j)); + auto placeholder(obj.placeholder(j)); + printValueAsJSON(state, strict, *a.value, placeholder, context); + } + } else { + printValueAsJSON(state, strict, *i->second.value, out, context); + } + break; + } + + case tList: { + auto list(out.list()); + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto placeholder(list.placeholder()); + printValueAsJSON(state, strict, *(*v.list)[n], placeholder, context); + } + break; + } + + case tFloat: + out.write(v.fpoint); + break; + + default: + throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + } +} + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + std::ostream& str, PathSet& context) { + JSONPlaceholder out(str); + printValueAsJSON(state, strict, v, out, context); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-json.hh b/third_party/nix/src/libexpr/value-to-json.hh new file mode 100644 index 000000000000..294d77604560 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-json.hh @@ -0,0 +1,19 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" + +namespace nix { + +class JSONPlaceholder; + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + JSONPlaceholder& out, PathSet& context); + +void printValueAsJSON(EvalState& state, bool strict, Value& v, + std::ostream& str, PathSet& context); + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-xml.cc b/third_party/nix/src/libexpr/value-to-xml.cc new file mode 100644 index 000000000000..921973881f50 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-xml.cc @@ -0,0 +1,184 @@ +#include "libexpr/value-to-xml.hh" + +#include <cstdlib> + +#include "libexpr/eval-inline.hh" +#include "libutil/util.hh" +#include "libutil/xml-writer.hh" + +namespace nix { + +static XMLAttrs singletonAttrs(const std::string& name, + const std::string& value) { + XMLAttrs attrs; + attrs[name] = value; + return attrs; +} + +static void printValueAsXML(EvalState& state, bool strict, bool location, + Value& v, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen); + +static void posToXML(XMLAttrs& xmlAttrs, const Pos& pos) { + xmlAttrs["path"] = pos.file.value(); + xmlAttrs["line"] = (format("%1%") % pos.line).str(); + xmlAttrs["column"] = (format("%1%") % pos.column).str(); +} + +static void showAttrs(EvalState& state, bool strict, bool location, + Bindings& attrs, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) { + StringSet names; + + for (auto& i : attrs) { + names.insert(i.second.name); + } + + for (auto& i : names) { + auto& [_, a] = *attrs.find(state.symbols.Create(i)); + + XMLAttrs xmlAttrs; + xmlAttrs["name"] = i; + if (location && a.pos != &noPos) { + posToXML(xmlAttrs, *a.pos); + } + + XMLOpenElement elem(doc, "attr", xmlAttrs); + printValueAsXML(state, strict, location, *a.value, doc, context, drvsSeen); + } +} + +static void printValueAsXML(EvalState& state, bool strict, bool location, + Value& v, XMLWriter& doc, PathSet& context, + PathSet& drvsSeen) { + checkInterrupt(); + + if (strict) { + state.forceValue(v); + } + + switch (v.type) { + case tInt: + doc.writeEmptyElement( + "int", singletonAttrs("value", (format("%1%") % v.integer).str())); + break; + + case tBool: + doc.writeEmptyElement( + "bool", singletonAttrs("value", v.boolean ? "true" : "false")); + break; + + case tString: + /* !!! show the context? */ + copyContext(v, context); + doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + break; + + case tPath: + doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + break; + + case tNull: + doc.writeEmptyElement("null"); + break; + + case tAttrs: + if (state.isDerivation(v)) { + XMLAttrs xmlAttrs; + + Bindings::iterator a = + v.attrs->find(state.symbols.Create("derivation")); + + Path drvPath; + a = v.attrs->find(state.sDrvPath); + if (a != v.attrs->end()) { + if (strict) { + state.forceValue(*a->second.value); + } + if (a->second.value->type == tString) { + xmlAttrs["drvPath"] = drvPath = a->second.value->string.s; + } + } + + a = v.attrs->find(state.sOutPath); + if (a != v.attrs->end()) { + if (strict) { + state.forceValue(*a->second.value); + } + if (a->second.value->type == tString) { + xmlAttrs["outPath"] = a->second.value->string.s; + } + } + + XMLOpenElement _(doc, "derivation", xmlAttrs); + + if (!drvPath.empty() && drvsSeen.find(drvPath) == drvsSeen.end()) { + drvsSeen.insert(drvPath); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } else { + doc.writeEmptyElement("repeated"); + } + } + + else { + XMLOpenElement _(doc, "attrs"); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } + + break; + + case tList: { + XMLOpenElement _(doc, "list"); + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValueAsXML(state, strict, location, *(*v.list)[n], doc, context, + drvsSeen); + } + break; + } + + case tLambda: { + XMLAttrs xmlAttrs; + if (location) { + posToXML(xmlAttrs, v.lambda.fun->pos); + } + XMLOpenElement _(doc, "function", xmlAttrs); + + if (v.lambda.fun->matchAttrs) { + XMLAttrs attrs; + if (!v.lambda.fun->arg.empty()) { + attrs["name"] = v.lambda.fun->arg; + } + if (v.lambda.fun->formals->ellipsis) { + attrs["ellipsis"] = "1"; + } + XMLOpenElement _(doc, "attrspat", attrs); + for (auto& i : v.lambda.fun->formals->formals) { + doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); + } + } else { + doc.writeEmptyElement("varpat", + singletonAttrs("name", v.lambda.fun->arg)); + } + + break; + } + + case tFloat: + doc.writeEmptyElement( + "float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + break; + + default: + doc.writeEmptyElement("unevaluated"); + } +} + +void printValueAsXML(EvalState& state, bool strict, bool location, Value& v, + std::ostream& out, PathSet& context) { + XMLWriter doc(true, out); + XMLOpenElement root(doc, "expr"); + PathSet drvsSeen; + printValueAsXML(state, strict, location, v, doc, context, drvsSeen); +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value-to-xml.hh b/third_party/nix/src/libexpr/value-to-xml.hh new file mode 100644 index 000000000000..18c5279236ff --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-xml.hh @@ -0,0 +1,14 @@ +#pragma once + +#include <map> +#include <string> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" + +namespace nix { + +void printValueAsXML(EvalState& state, bool strict, bool location, Value& v, + std::ostream& out, PathSet& context); + +} diff --git a/third_party/nix/src/libexpr/value.cc b/third_party/nix/src/libexpr/value.cc new file mode 100644 index 000000000000..93fe1874786a --- /dev/null +++ b/third_party/nix/src/libexpr/value.cc @@ -0,0 +1,121 @@ +#include "libexpr/value.hh" + +#include <glog/logging.h> + +namespace nix { + +Value::Value(const Value& copy) { *this = copy; } + +Value::Value(Value&& move) { *this = move; } + +Value& Value::operator=(const Value& copy) { + if (type != copy.type) { + memset(this, 0, sizeof(*this)); + } + type = copy.type; + switch (type) { + case tInt: + integer = copy.integer; + break; + case tBool: + boolean = copy.boolean; + break; + case tString: + string = copy.string; + break; + case tPath: + path = copy.path; + break; + case tNull: + /* no fields */ + break; + case tAttrs: + attrs = copy.attrs; + break; + case tList: + list = copy.list; + break; + case tThunk: + thunk = copy.thunk; + break; + case tApp: + app = copy.app; + break; + case tLambda: + lambda = copy.lambda; + break; + case tBlackhole: + /* no fields */ + break; + case tPrimOp: + primOp = copy.primOp; + break; + case tPrimOpApp: + primOpApp = copy.primOpApp; + break; + case _reserved1: + LOG(FATAL) << "attempted to assign a tExternal value"; + break; + case tFloat: + fpoint = copy.fpoint; + break; + } + return *this; +} + +Value& Value::operator=(Value&& move) { + if (type != move.type) { + memset(this, 0, sizeof(*this)); + } + type = move.type; + switch (type) { + case tInt: + integer = move.integer; + break; + case tBool: + boolean = move.boolean; + break; + case tString: + string = move.string; + break; + case tPath: + path = move.path; + break; + case tNull: + /* no fields */ + break; + case tAttrs: + attrs = move.attrs; + break; + case tList: + list = move.list; + break; + case tThunk: + thunk = move.thunk; + break; + case tApp: + app = move.app; + break; + case tLambda: + lambda = move.lambda; + break; + case tBlackhole: + /* no fields */ + break; + case tPrimOp: + primOp = move.primOp; + break; + case tPrimOpApp: + primOpApp = move.primOpApp; + break; + case _reserved1: + LOG(FATAL) << "attempted to assign a tExternal value"; + break; + case tFloat: + fpoint = move.fpoint; + break; + } + return *this; +} + +} // namespace nix diff --git a/third_party/nix/src/libexpr/value.hh b/third_party/nix/src/libexpr/value.hh new file mode 100644 index 000000000000..82021c77c41b --- /dev/null +++ b/third_party/nix/src/libexpr/value.hh @@ -0,0 +1,191 @@ +#pragma once + +#include <tuple> +#include <vector> + +#include "libexpr/symbol-table.hh" +#include "libutil/types.hh" + +namespace nix { + +using ValueType = enum { + tInt = 1, + tBool, + tString, + tPath, + tNull, + tAttrs, + tList, + tThunk, + tApp, + tLambda, + tBlackhole, + tPrimOp, + tPrimOpApp, + _reserved1, // formerly tExternal + tFloat +}; + +class Bindings; +struct Env; +struct Expr; +struct ExprLambda; +struct PrimOp; +struct PrimOp; +class Symbol; + +typedef int64_t NixInt; +typedef double NixFloat; + +// Forward declaration of Value is required because the following +// types are mutually recursive. +// +// TODO(tazjin): Really, these types need some serious refactoring. +struct Value; + +/* Strings in the evaluator carry a so-called `context' which + is a list of strings representing store paths. This is to + allow users to write things like + + "--with-freetype2-library=" + freetype + "/lib" + + where `freetype' is a derivation (or a source to be copied + to the store). If we just concatenated the strings without + keeping track of the referenced store paths, then if the + string is used as a derivation attribute, the derivation + will not have the correct dependencies in its inputDrvs and + inputSrcs. + + The semantics of the context is as follows: when a string + with context C is used as a derivation attribute, then the + derivations in C will be added to the inputDrvs of the + derivation, and the other store paths in C will be added to + the inputSrcs of the derivations. + + For canonicity, the store paths should be in sorted order. */ +struct NixString { + const char* s; + const char** context; // must be in sorted order +}; + +struct NixThunk { + Env* env; + Expr* expr; +}; + +struct NixApp { + Value *left, *right; +}; + +struct NixLambda { + Env* env; + ExprLambda* fun; +}; + +struct NixPrimOpApp { + Value *left, *right; +}; + +using NixList = std::vector<Value*>; + +struct Value { + ValueType type; + union { // TODO(tazjin): std::variant + NixInt integer; + bool boolean; + NixString string; + const char* path; + std::shared_ptr<Bindings> attrs; + std::shared_ptr<NixList> list; + NixThunk thunk; + NixApp app; // TODO(tazjin): "app"? + NixLambda lambda; + std::shared_ptr<PrimOp> primOp; + NixPrimOpApp primOpApp; + NixFloat fpoint; + }; + + Value() : type(tInt), attrs(nullptr) { + static_assert(offsetof(Value, attrs) + sizeof(attrs) == sizeof(Value)); + } + + Value(const Value& copy); + Value(Value&& move); + ~Value() {} + Value& operator=(const Value& copy); + Value& operator=(Value&& move); + + bool isList() const { return type == tList; } + + size_t listSize() const { return list->size(); } +}; + +/* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ +static inline void clearValue(Value& v) { v.app.left = v.app.right = 0; } + +static inline void mkInt(Value& v, NixInt n) { + clearValue(v); + v.type = tInt; + v.integer = n; +} + +static inline void mkFloat(Value& v, NixFloat n) { + clearValue(v); + v.type = tFloat; + v.fpoint = n; +} + +static inline void mkBool(Value& v, bool b) { + clearValue(v); + v.type = tBool; + v.boolean = b; +} + +static inline void mkNull(Value& v) { + clearValue(v); + v.type = tNull; +} + +static inline void mkApp(Value& v, Value& left, Value& right) { + v.type = tApp; + v.app.left = &left; + v.app.right = &right; +} + +static inline void mkPrimOpApp(Value& v, Value& left, Value& right) { + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} + +static inline void mkStringNoCopy(Value& v, const char* s) { + v.type = tString; + v.string.s = s; + v.string.context = 0; +} + +static inline void mkString(Value& v, const Symbol& s) { + mkStringNoCopy(v, ((const std::string&)s).c_str()); +} + +void mkString(Value& v, const char* s); + +static inline void mkPathNoCopy(Value& v, const char* s) { + clearValue(v); + v.type = tPath; + v.path = s; +} + +void mkPath(Value& v, const char* s); + +/* Compute the size in bytes of the given value, including all values + and environments reachable from it. Static expressions (Exprs) are + not included. */ +size_t valueSize(const Value& v); + +using ValueMap = std::map<Symbol, Value*>; + +std::shared_ptr<Value*> allocRootValue(Value* v); + +} // namespace nix diff --git a/third_party/nix/src/libmain/CMakeLists.txt b/third_party/nix/src/libmain/CMakeLists.txt new file mode 100644 index 000000000000..a95128c1318f --- /dev/null +++ b/third_party/nix/src/libmain/CMakeLists.txt @@ -0,0 +1,33 @@ +# -*- mode: cmake; -*- +add_library(nixmain SHARED) +set_property(TARGET nixmain PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +target_include_directories(nixmain PUBLIC "${nix_SOURCE_DIR}/src") + +set(HEADER_FILES + common-args.hh + shared.hh +) + +target_sources(nixmain + PUBLIC + ${HEADER_FILES} + PRIVATE + common-args.cc + shared.cc + stack.cc +) + +target_link_libraries(nixmain + nixstore + nixutil + + absl::strings + glog +) + +configure_file("nix-main.pc.in" "${PROJECT_BINARY_DIR}/nix-main.pc" @ONLY) +INSTALL(FILES "${PROJECT_BINARY_DIR}/nix-main.pc" DESTINATION "${PKGCONFIG_INSTALL_DIR}") + +INSTALL(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nix/libmain) +INSTALL(TARGETS nixmain DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/third_party/nix/src/libmain/common-args.cc b/third_party/nix/src/libmain/common-args.cc new file mode 100644 index 000000000000..729e026f1990 --- /dev/null +++ b/third_party/nix/src/libmain/common-args.cc @@ -0,0 +1,56 @@ +#include "libmain/common-args.hh" + +#include <glog/logging.h> + +#include "libstore/globals.hh" + +namespace nix { + +MixCommonArgs::MixCommonArgs(const std::string& programName) + : programName(programName) { + mkFlag() + .longName("verbose") + .shortName('v') + .description("increase verbosity level") + .handler([]() { + FLAGS_stderrthreshold = google::GLOG_INFO; + FLAGS_v += 1; + }); + + mkFlag() + .longName("quiet") + .description("silence all log output") + .handler([]() { FLAGS_stderrthreshold = google::GLOG_FATAL; }); + + mkFlag() + .longName("option") + .labels({"name", "value"}) + .description("set a Nix configuration option (overriding nix.conf)") + .arity(2) + .handler([](std::vector<std::string> ss) { + try { + globalConfig.set(ss[0], ss[1]); + } catch (UsageError& e) { + LOG(WARNING) << e.what(); + } + }); + + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](const std::string& s) { settings.set("max-jobs", s); }); + + std::string cat = "config"; + globalConfig.convertToArgs(*this, cat); + + // Backward compatibility hack: nix-env already had a --system flag. + if (programName == "nix-env") { + longFlags.erase("system"); + } + + hiddenCategories.insert(cat); +} + +} // namespace nix diff --git a/third_party/nix/src/libmain/common-args.hh b/third_party/nix/src/libmain/common-args.hh new file mode 100644 index 000000000000..f1c7c8481323 --- /dev/null +++ b/third_party/nix/src/libmain/common-args.hh @@ -0,0 +1,27 @@ +#pragma once + +#include "libutil/args.hh" + +namespace nix { + +struct MixCommonArgs : virtual Args { + std::string programName; + MixCommonArgs(const std::string& programName); +}; + +struct MixDryRun : virtual Args { + bool dryRun = false; + + MixDryRun() { + mkFlag(0, "dry-run", "show what this command would do without doing it", + &dryRun); + } +}; + +struct MixJSON : virtual Args { + bool json = false; + + MixJSON() { mkFlag(0, "json", "produce JSON output", &json); } +}; + +} // namespace nix diff --git a/third_party/nix/src/libmain/nix-main.pc.in b/third_party/nix/src/libmain/nix-main.pc.in new file mode 100644 index 000000000000..9876a3d1b7ae --- /dev/null +++ b/third_party/nix/src/libmain/nix-main.pc.in @@ -0,0 +1,9 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixmain +Cflags: -I${includedir}/nix diff --git a/third_party/nix/src/libmain/shared.cc b/third_party/nix/src/libmain/shared.cc new file mode 100644 index 000000000000..331ea6b3a9d2 --- /dev/null +++ b/third_party/nix/src/libmain/shared.cc @@ -0,0 +1,386 @@ +#include "libmain/shared.hh" + +#include <algorithm> +#include <cctype> +#include <csignal> +#include <cstdlib> +#include <exception> +#include <iostream> +#include <mutex> +#include <utility> + +#include <glog/logging.h> +#include <openssl/crypto.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <unistd.h> + +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/util.hh" + +namespace nix { + +static bool gcWarning = true; + +void printGCWarning() { + if (!gcWarning) { + return; + } + + static bool haveWarned = false; + if (!haveWarned) { + haveWarned = true; + LOG(WARNING) << "you did not specify '--add-root'; " + << "the result might be removed by the garbage collector"; + } +} + +void printMissing(const ref<Store>& store, const PathSet& paths) { + unsigned long long downloadSize; + unsigned long long narSize; + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, + narSize); + printMissing(store, willBuild, willSubstitute, unknown, downloadSize, + narSize); +} + +void printMissing(const ref<Store>& store, const PathSet& willBuild, + const PathSet& willSubstitute, const PathSet& unknown, + unsigned long long downloadSize, unsigned long long narSize) { + if (!willBuild.empty()) { + LOG(INFO) << "these derivations will be built:"; + Paths sorted = store->topoSortPaths(willBuild); + reverse(sorted.begin(), sorted.end()); + for (auto& i : sorted) { + LOG(INFO) << " " << i; + } + } + + if (!willSubstitute.empty()) { + LOG(INFO) << "these paths will be fetched (" + << (downloadSize / (1024.0 * 1024.0)) << " MiB download, " + << (narSize / (1024.0 * 1024.0)) << "MiB unpacked):"; + + for (auto& i : willSubstitute) { + LOG(INFO) << i; + } + } + + if (!unknown.empty()) { + LOG(INFO) << "don't know how to build these paths" + << (settings.readOnlyMode + ? " (may be caused by read-only store access)" + : "") + << ":"; + + for (auto& i : unknown) { + LOG(INFO) << i; + } + } +} + +std::string getArg(const std::string& opt, Strings::iterator& i, + const Strings::iterator& end) { + ++i; + if (i == end) { + throw UsageError(format("'%1%' requires an argument") % opt); + } + + return *i; +} + +#if OPENSSL_VERSION_NUMBER < 0x10101000L +/* OpenSSL is not thread-safe by default - it will randomly crash + unless the user supplies a mutex locking function. So let's do + that. */ +static std::vector<std::mutex> opensslLocks; + +static void opensslLockCallback(int mode, int type, const char* file, + int line) { + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); +} +#endif + +static void sigHandler(int signo) {} + +void initNix() { + /* Turn on buffering for cerr. */ +#if HAVE_PUBSETBUF + static char buf[1024]; + std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); +#endif + +#if OPENSSL_VERSION_NUMBER < 0x10101000L + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector<std::mutex>(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); +#endif + + loadConfFile(); + + startSignalHandlerThread(); + + /* Reset SIGCHLD to its default. */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, nullptr) != 0) { + throw SysError("resetting SIGCHLD"); + } + + /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + act.sa_handler = sigHandler; + if (sigaction(SIGUSR1, &act, nullptr) != 0) { + throw SysError("handling SIGUSR1"); + } + + /* Register a SIGSEGV handler to detect stack overflows. */ + detectStackOverflow(); + + /* There is no privacy in the Nix system ;-) At least not for + now. In particular, store objects should be readable by + everybody. */ + umask(0022); + + /* Initialise the PRNG. */ + struct timeval tv; + gettimeofday(&tv, nullptr); + srandom(tv.tv_usec); +} + +LegacyArgs::LegacyArgs( + const std::string& programName, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg) + : MixCommonArgs(programName), parseArg(std::move(parseArg)) { + mkFlag() + .longName("no-build-output") + .shortName('Q') + .description("do not show build output") + .set(&settings.verboseBuild, false); + + mkFlag() + .longName("keep-failed") + .shortName('K') + .description("keep temporary directories of failed builds") + .set(&(bool&)settings.keepFailed, true); + + mkFlag() + .longName("keep-going") + .shortName('k') + .description("keep going after a build fails") + .set(&(bool&)settings.keepGoing, true); + + mkFlag() + .longName("fallback") + .description("build from source if substitution fails") + .set(&(bool&)settings.tryFallback, true); + + auto intSettingAlias = [&](char shortName, const std::string& longName, + const std::string& description, + const std::string& dest) { + mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); + }); + }; + + intSettingAlias(0, "cores", + "maximum number of CPU cores to use inside a build", "cores"); + intSettingAlias(0, "max-silent-time", + "number of seconds of silence before a build is killed", + "max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", + "timeout"); + + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); + + mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + &gcWarning, false); + + mkFlag() + .longName("store") + .label("store-uri") + .description("URI of the Nix store to use") + .dest(&(std::string&)settings.storeUri); +} + +bool LegacyArgs::processFlag(Strings::iterator& pos, Strings::iterator end) { + if (MixCommonArgs::processFlag(pos, end)) { + return true; + } + bool res = parseArg(pos, end); + if (res) { + ++pos; + } + return res; +} + +bool LegacyArgs::processArgs(const Strings& args, bool finish) { + if (args.empty()) { + return true; + } + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) { + throw UsageError(format("unexpected argument '%1%'") % args.front()); + } + return true; +} + +void parseCmdLine( + int argc, char** argv, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg) { + parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), + std::move(parseArg)); +} + +void parseCmdLine( + const std::string& programName, const Strings& args, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg) { + LegacyArgs(programName, std::move(parseArg)).parseCmdline(args); +} + +void printVersion(const std::string& programName) { + std::cout << format("%1% (Tvix) %2%") % programName % nixVersion << std::endl; + + // TODO(tazjin): figure out what the fuck this is + /*if (verbosity > lvlInfo) { + Strings cfg; +#if HAVE_BOEHMGC + cfg.push_back("gc"); +#endif +#if HAVE_SODIUM + cfg.push_back("signed-caches"); +#endif + std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; + std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" + << "\n"; + std::cout << "Store directory: " << settings.nixStore << "\n"; + std::cout << "State directory: " << settings.nixStateDir << "\n"; + } */ + throw Exit(); +} + +void showManPage(const std::string& name) { + restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); + execlp("man", "man", name.c_str(), nullptr); + throw SysError(format("command 'man %1%' failed") % name.c_str()); +} + +int handleExceptions(const std::string& programName, + const std::function<void()>& fun) { + ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + + std::string error = ANSI_RED "error:" ANSI_NORMAL " "; + try { + try { + fun(); + } catch (...) { + /* Subtle: we have to make sure that any `interrupted' + condition is discharged before we reach printMsg() + below, since otherwise it will throw an (uncaught) + exception. */ + setInterruptThrown(); + throw; + } + } catch (Exit& e) { + return e.status; + } catch (UsageError& e) { + LOG(INFO) << e.what(); + LOG(INFO) << "Try '" << programName << " --help' for more information."; + return 1; + } catch (BaseError& e) { + LOG(ERROR) << error << (settings.showTrace ? e.prefix() : "") << e.msg(); + if (!e.prefix().empty() && !settings.showTrace) { + LOG(INFO) << "(use '--show-trace' to show detailed location information)"; + } + return static_cast<int>(e.status); + } catch (std::bad_alloc& e) { + LOG(ERROR) << error << "failed to allocate: " << e.what(); + return 1; + } catch (std::exception& e) { + LOG(ERROR) << error << e.what(); + return 1; + } + + return 0; +} + +RunPager::RunPager() { + if (isatty(STDOUT_FILENO) == 0) { + return; + } + char* pager = getenv("NIX_PAGER"); + if (pager == nullptr) { + pager = getenv("PAGER"); + } + if (pager && (std::string(pager) == "" || std::string(pager) == "cat")) { + return; + } + + Pipe toPager; + toPager.create(); + + pid = startProcess([&]() { + if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) { + throw SysError("dupping stdin"); + } + if (getenv("LESS") == nullptr) { + setenv("LESS", "FRSXMK", 1); + } + restoreSignals(); + if (pager != nullptr) { + execl("/bin/sh", "sh", "-c", pager, nullptr); + } + execlp("pager", "pager", nullptr); + execlp("less", "less", nullptr); + execlp("more", "more", nullptr); + throw SysError(format("executing '%1%'") % pager); + }); + + pid.setKillSignal(SIGINT); + + if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) { + throw SysError("dupping stdout"); + } +} + +RunPager::~RunPager() { + try { + if (pid != Pid(-1)) { + std::cout.flush(); + close(STDOUT_FILENO); + pid.wait(); + } + } catch (...) { + ignoreException(); + } +} + +std::string showBytes(unsigned long long bytes) { + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +} + +PrintFreed::~PrintFreed() { + if (show) { + std::cout << format("%1% store paths deleted, %2% freed\n") % + results.paths.size() % showBytes(results.bytesFreed); + } +} + +Exit::~Exit() = default; + +} // namespace nix diff --git a/third_party/nix/src/libmain/shared.hh b/third_party/nix/src/libmain/shared.hh new file mode 100644 index 000000000000..d1061d5e04e5 --- /dev/null +++ b/third_party/nix/src/libmain/shared.hh @@ -0,0 +1,134 @@ +#pragma once + +#include <locale> + +#include <absl/strings/numbers.h> +#include <signal.h> + +#include "libmain/common-args.hh" +#include "libutil/args.hh" +#include "libutil/util.hh" + +namespace nix { + +class Exit : public std::exception { + public: + int status; + Exit() : status(0) {} + Exit(int status) : status(status) {} + virtual ~Exit(); +}; + +int handleExceptions(const std::string& programName, + const std::function<void()>& fun); + +void initNix(); + +void parseCmdLine( + int argc, char** argv, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg); + +void parseCmdLine( + const std::string& programName, const Strings& args, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg); + +void printVersion(const std::string& programName); + +/* Ugh. No better place to put this. */ +void printGCWarning(); + +class Store; + +void printMissing(const ref<Store>& store, const PathSet& paths); + +void printMissing(const ref<Store>& store, const PathSet& willBuild, + const PathSet& willSubstitute, const PathSet& unknown, + unsigned long long downloadSize, unsigned long long narSize); + +std::string getArg(const std::string& opt, Strings::iterator& i, + const Strings::iterator& end); + +template <class N> +N getIntArg(const std::string& opt, Strings::iterator& i, + const Strings::iterator& end, bool allowUnit) { + ++i; + if (i == end) { + throw UsageError(format("'%1%' requires an argument") % opt); + } + std::string s = *i; + N multiplier = 1; + if (allowUnit && !s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') { + multiplier = 1ULL << 10; + } else if (u == 'M') { + multiplier = 1ULL << 20; + } else if (u == 'G') { + multiplier = 1ULL << 30; + } else if (u == 'T') { + multiplier = 1ULL << 40; + } else { + throw UsageError(format("invalid unit specifier '%1%'") % u); + } + + s.resize(s.size() - 1); + } + } + N n; + if (!absl::SimpleAtoi(s, &n)) { + throw UsageError(format("'%1%' requires an integer argument") % opt); + } + return n * multiplier; +} + +struct LegacyArgs : public MixCommonArgs { + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg; + + LegacyArgs( + const std::string& programName, + std::function<bool(Strings::iterator& arg, const Strings::iterator& end)> + parseArg); + + bool processFlag(Strings::iterator& pos, Strings::iterator end) override; + + bool processArgs(const Strings& args, bool finish) override; +}; + +/* Show the manual page for the specified program. */ +void showManPage(const std::string& name); + +/* The constructor of this class starts a pager if stdout is a + terminal and $PAGER is set. Stdout is redirected to the pager. */ +class RunPager { + public: + RunPager(); + ~RunPager(); + + private: + Pid pid; +}; + +extern volatile ::sig_atomic_t blockInt; + +/* GC helpers. */ + +std::string showBytes(unsigned long long bytes); + +struct GCResults; + +struct PrintFreed { + bool show; + const GCResults& results; + PrintFreed(bool show, const GCResults& results) + : show(show), results(results) {} + ~PrintFreed(); +}; + +/* Install a SIGSEGV handler to detect stack overflows. */ +void detectStackOverflow(); + +} // namespace nix diff --git a/third_party/nix/src/libmain/stack.cc b/third_party/nix/src/libmain/stack.cc new file mode 100644 index 000000000000..628b6313a89c --- /dev/null +++ b/third_party/nix/src/libmain/stack.cc @@ -0,0 +1,75 @@ +#include <csignal> +#include <cstddef> +#include <cstdlib> +#include <cstring> + +#include <unistd.h> + +#include "libutil/types.hh" + +namespace nix { + +static void sigsegvHandler(int signo, siginfo_t* info, void* ctx) { + /* Detect stack overflows by comparing the faulting address with + the stack pointer. Unfortunately, getting the stack pointer is + not portable. */ + bool haveSP = true; + char* sp = nullptr; +#if defined(__x86_64__) && defined(REG_RSP) + sp = (char*)(static_cast<ucontext_t*>(ctx))->uc_mcontext.gregs[REG_RSP]; +#elif defined(REG_ESP) + sp = (char*)((ucontext_t*)ctx)->uc_mcontext.gregs[REG_ESP]; +#else + haveSP = false; +#endif + + if (haveSP) { + ptrdiff_t diff = static_cast<char*>(info->si_addr) - sp; + if (diff < 0) { + diff = -diff; + } + if (diff < 4096) { + char msg[] = "error: stack overflow (possible infinite recursion)\n"; + [[gnu::unused]] auto res = write(2, msg, strlen(msg)); + _exit(1); // maybe abort instead? + } + } + + /* Restore default behaviour (i.e. segfault and dump core). */ + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGSEGV, &act, nullptr) != 0) { + abort(); + } +} + +void detectStackOverflow() { +#if defined(SA_SIGINFO) && defined(SA_ONSTACK) + /* Install a SIGSEGV handler to detect stack overflows. This + requires an alternative stack, otherwise the signal cannot be + delivered when we're out of stack space. */ + stack_t stack; + stack.ss_size = 4096 * 4 + MINSIGSTKSZ; + static auto stackBuf = std::make_unique<std::vector<char>>(stack.ss_size); + stack.ss_sp = stackBuf->data(); + if (stack.ss_sp == nullptr) { + throw Error("cannot allocate alternative stack"); + } + stack.ss_flags = 0; + if (sigaltstack(&stack, nullptr) == -1) { + throw SysError("cannot set alternative stack"); + } + + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_sigaction = sigsegvHandler; + act.sa_flags = SA_SIGINFO | SA_ONSTACK; + if (sigaction(SIGSEGV, &act, nullptr) != 0) { + throw SysError("resetting SIGSEGV"); + } +#endif +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/CMakeLists.txt b/third_party/nix/src/libstore/CMakeLists.txt new file mode 100644 index 000000000000..246377cc9b8d --- /dev/null +++ b/third_party/nix/src/libstore/CMakeLists.txt @@ -0,0 +1,127 @@ +# -*- mode: cmake; -*- +add_library(nixstore SHARED) +add_library(nixstoremock SHARED) +set_property(TARGET nixstore PROPERTY CXX_STANDARD 17) +set_property(TARGET nixstoremock PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +target_include_directories(nixstore PUBLIC "${nix_SOURCE_DIR}/src") +target_include_directories(nixstoremock PUBLIC "${nix_SOURCE_DIR}/src") + +# The database schema is stored in schema.sql, but needs to be +# available during the build as static data. +# +# These commands create an includeable source-file out of it. +file(READ "schema.sql" NIX_SCHEMA) + +string(CONFIGURE + "#pragma once + namespace nix { + constexpr char kNixSqlSchema[] = R\"(${NIX_SCHEMA})\"; + }" + NIX_SCHEMA_GEN) + +file(WRITE ${PROJECT_BINARY_DIR}/generated/schema.sql.hh "${NIX_SCHEMA_GEN}") + +set(HEADER_FILES + binary-cache-store.hh + builtins.hh + crypto.hh + derivations.hh + download.hh + fs-accessor.hh + globals.hh + local-store.hh + machines.hh + nar-accessor.hh + nar-info-disk-cache.hh + nar-info.hh + parsed-derivations.hh + pathlocks.hh + profiles.hh + references.hh + remote-fs-accessor.hh + remote-store.hh + rpc-store.hh + s3-binary-cache-store.hh + s3.hh + serve-protocol.hh + sqlite.hh + ssh.hh + store-api.hh + worker-protocol.hh +) + +target_sources(nixstore + PUBLIC + ${HEADER_FILES} + + PRIVATE + ${PROJECT_BINARY_DIR}/generated/schema.sql.hh + binary-cache-store.cc + build.cc + crypto.cc + derivations.cc + download.cc + export-import.cc + gc.cc + globals.cc + http-binary-cache-store.cc + legacy-ssh-store.cc + local-binary-cache-store.cc + local-fs-store.cc + local-store.cc + machines.cc + misc.cc + nar-accessor.cc + nar-info.cc + nar-info-disk-cache.cc + optimise-store.cc + parsed-derivations.cc + pathlocks.cc + profiles.cc + references.cc + remote-fs-accessor.cc + remote-store.cc + rpc-store.cc + s3-binary-cache-store.cc + sqlite.cc + ssh.cc + ssh-store.cc + store-api.cc + builtins/buildenv.cc + builtins/fetchurl.cc +) + +target_link_libraries(nixstore + nixproto + nixutil + + CURL::libcurl + SQLite::SQLite3 + absl::strings + glog + seccomp + sodium +) + +target_sources(nixstoremock + PUBLIC + mock-binary-cache-store.hh + + PRIVATE + mock-binary-cache-store.cc +) + +target_link_libraries(nixstoremock + nixstore + + absl::btree + absl::flat_hash_map + glog +) + +configure_file("nix-store.pc.in" "${PROJECT_BINARY_DIR}/nix-store.pc" @ONLY) +INSTALL(FILES "${PROJECT_BINARY_DIR}/nix-store.pc" DESTINATION "${PKGCONFIG_INSTALL_DIR}") + +INSTALL(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nix/libstore) +INSTALL(TARGETS nixstore nixstoremock DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/third_party/nix/src/libstore/binary-cache-store.cc b/third_party/nix/src/libstore/binary-cache-store.cc new file mode 100644 index 000000000000..0b04e972da7c --- /dev/null +++ b/third_party/nix/src/libstore/binary-cache-store.cc @@ -0,0 +1,396 @@ +#include "libstore/binary-cache-store.hh" + +#include <chrono> +#include <future> +#include <memory> + +#include <absl/strings/ascii.h> +#include <absl/strings/numbers.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> + +#include "libstore/derivations.hh" +#include "libstore/fs-accessor.hh" +#include "libstore/globals.hh" +#include "libstore/nar-accessor.hh" +#include "libstore/nar-info-disk-cache.hh" +#include "libstore/nar-info.hh" +#include "libstore/remote-fs-accessor.hh" +#include "libutil/archive.hh" +#include "libutil/compression.hh" +#include "libutil/json.hh" +#include "libutil/sync.hh" + +namespace nix { + +BinaryCacheStore::BinaryCacheStore(const Params& params) : Store(params) { + if (secretKeyFile != "") { + const std::string& secret_key_file = secretKeyFile; + secretKey = std::make_unique<SecretKey>(readFile(secret_key_file)); + } + + StringSink sink; + sink << std::string(kNarVersionMagic1); + narMagic = *sink.s; +} + +void BinaryCacheStore::init() { + std::string cacheInfoFile = "nix-cache-info"; + + auto cacheInfo = getFile(cacheInfoFile); + if (!cacheInfo) { + upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", + "text/x-nix-cache-info"); + } else { + for (auto& line : + absl::StrSplit(*cacheInfo, absl::ByChar('\n'), absl::SkipEmpty())) { + size_t colon = line.find(':'); + if (colon == std::string::npos) { + continue; + } + auto name = line.substr(0, colon); + auto value = + absl::StripAsciiWhitespace(line.substr(colon + 1, std::string::npos)); + if (name == "StoreDir") { + if (value != storeDir) { + throw Error(format("binary cache '%s' is for Nix stores with prefix " + "'%s', not '%s'") % + getUri() % value % storeDir); + } + } else if (name == "WantMassQuery") { + wantMassQuery_ = value == "1"; + } else if (name == "Priority") { + if (!absl::SimpleAtoi(value, &priority)) { + LOG(WARNING) << "Invalid 'Priority' value: " << value; + } + } + } + } +} + +void BinaryCacheStore::getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept { + try { + callback(getFile(path)); + } catch (...) { + callback.rethrow(); + } +} + +void BinaryCacheStore::getFile(const std::string& path, Sink& sink) { + std::promise<std::shared_ptr<std::string>> promise; + getFile(path, Callback<std::shared_ptr<std::string>>{ + [&](std::future<std::shared_ptr<std::string>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink(reinterpret_cast<unsigned char*>(data->data()), data->size()); +} + +std::shared_ptr<std::string> BinaryCacheStore::getFile( + const std::string& path) { + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile&) { + return nullptr; + } + return sink.s; +} + +Path BinaryCacheStore::narInfoFileFor(const Path& storePath) { + assertStorePath(storePath); + return storePathToHash(storePath) + ".narinfo"; +} + +void BinaryCacheStore::writeNarInfo(const ref<NarInfo>& narInfo) { + auto narInfoFile = narInfoFileFor(narInfo->path); + + upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); + + auto hashPart = storePathToHash(narInfo->path); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, std::shared_ptr<NarInfo>(narInfo)); + } + + if (diskCache) { + diskCache->upsertNarInfo(getUri(), hashPart, + std::shared_ptr<NarInfo>(narInfo)); + } +} + +void BinaryCacheStore::addToStore(const ValidPathInfo& info, + const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + if ((repair == 0u) && isValidPath(info.path)) { + return; + } + + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto& ref : info.references) { + try { + if (ref != info.path) { + queryPathInfo(ref); + } + } catch (InvalidPath&) { + throw Error(format("cannot add '%s' to the binary cache because the " + "reference '%s' is not valid") % + info.path % ref); + } + } + + assert(nar->compare(0, narMagic.size(), narMagic) == 0); + + auto narInfo = make_ref<NarInfo>(info); + + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); + + if (info.narHash && info.narHash != narInfo->narHash) { + throw Error( + format("refusing to copy corrupted path '%1%' to binary cache") % + info.path); + } + + auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor); + + /* Optionally write a JSON file containing a listing of the + contents of the NAR. */ + if (writeNARListing) { + std::ostringstream jsonOut; + + { + JSONObject jsonRoot(jsonOut); + jsonRoot.attr("version", 1); + + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) { + accessor_->addToCache(info.path, *nar, narAccessor); + } + + { + auto res = jsonRoot.placeholder("root"); + listNar(res, narAccessor, "", true); + } + } + + upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), + "application/json"); + } + + else { + if (accessor_) { + accessor_->addToCache(info.path, *nar, makeNarAccessor(nar)); + } + } + + /* Compress the NAR. */ + narInfo->compression = compression; + auto now1 = std::chrono::steady_clock::now(); + auto narCompressed = compress(compression, *nar, parallelCompression); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); + + auto duration = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); + DLOG(INFO) << "copying path '" << narInfo->path << "' (" << narInfo->narSize + << " bytes, compressed " + << ((1.0 - + static_cast<double>(narCompressed->size()) / nar->size()) * + 100.0) + << "% in " << duration << "ms) to binary cache"; + + /* Atomically write the NAR file. */ + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" + : compression == "bzip2" ? ".bz2" + : compression == "br" ? ".br" + : ""); + if ((repair != 0u) || !fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + } else { + stats.narWriteAverted++; + } + + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) { + narInfo->sign(*secretKey); + } + + writeNarInfo(narInfo); + + stats.narInfoWrite++; +} + +bool BinaryCacheStore::isValidPathUncached(const Path& storePath) { + // FIXME: this only checks whether a .narinfo with a matching hash + // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even + // though they shouldn't. Not easily fixed. + return fileExists(narInfoFileFor(storePath)); +} + +void BinaryCacheStore::narFromPath(const Path& storePath, Sink& sink) { + auto info = queryPathInfo(storePath).cast<const NarInfo>(); + + uint64_t narSize = 0; + + LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { + sink(data, len); + narSize += len; + }); + + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + + try { + getFile(info->url, *decompressor); + } catch (NoSuchBinaryCacheFile& e) { + throw SubstituteGone(e.what()); + } + + decompressor->finish(); + + stats.narRead++; + // stats.narReadCompressedBytes += nar->size(); // FIXME + stats.narReadBytes += narSize; +} + +void BinaryCacheStore::queryPathInfoUncached( + const Path& storePath, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + auto uri = getUri(); + LOG(INFO) << "querying info about '" << storePath << "' on '" << uri << "'"; + + auto narInfoFile = narInfoFileFor(storePath); + + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + + getFile(narInfoFile, + Callback<std::shared_ptr<std::string>>( + [=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); + + if (!data) { + return (*callbackPtr)(nullptr); + } + + stats.narInfoRead++; + + (*callbackPtr)(std::shared_ptr<ValidPathInfo>( + std::make_shared<NarInfo>(*this, *data, narInfoFile))); + + } catch (...) { + callbackPtr->rethrow(); + } + })); +} + +Path BinaryCacheStore::addToStore(const std::string& name, const Path& srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info; + info.path = makeFixedOutputPath(recursive, h, name); + + addToStore(info, sink.s, repair, CheckSigs, nullptr); + + return info.path; +} + +Path BinaryCacheStore::addTextToStore(const std::string& name, + const std::string& s, + const PathSet& references, + RepairFlag repair) { + ValidPathInfo info; + info.path = computeStorePathForText(name, s, references); + info.references = references; + + if ((repair != 0u) || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); + addToStore(info, sink.s, repair, CheckSigs, nullptr); + } + + return info.path; +} + +ref<FSAccessor> BinaryCacheStore::getFSAccessor() { + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), + localNarCache); +} + +void BinaryCacheStore::addSignatures(const Path& storePath, + const StringSet& sigs) { + /* Note: this is inherently racy since there is no locking on + binary caches. In particular, with S3 this unreliable, even + when addSignatures() is called sequentially on a path, because + S3 might return an outdated cached version. */ + + auto narInfo = make_ref<NarInfo>((NarInfo&)*queryPathInfo(storePath)); + + narInfo->sigs.insert(sigs.begin(), sigs.end()); + + auto narInfoFile = narInfoFileFor(narInfo->path); + + writeNarInfo(narInfo); +} + +std::shared_ptr<std::string> BinaryCacheStore::getBuildLog(const Path& path) { + Path drvPath; + + if (isDerivation(path)) { + drvPath = path; + } else { + try { + auto info = queryPathInfo(path); + // FIXME: add a "Log" field to .narinfo + if (info->deriver.empty()) { + return nullptr; + } + drvPath = info->deriver; + } catch (InvalidPath&) { + return nullptr; + } + } + + auto logPath = "log/" + baseNameOf(drvPath); + + DLOG(INFO) << "fetching build log from binary cache '" << getUri() << "/" + << logPath << "'"; + + return getFile(logPath); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/binary-cache-store.hh b/third_party/nix/src/libstore/binary-cache-store.hh new file mode 100644 index 000000000000..40c636f60ac2 --- /dev/null +++ b/third_party/nix/src/libstore/binary-cache-store.hh @@ -0,0 +1,115 @@ +#pragma once + +#include <atomic> + +#include "libstore/crypto.hh" +#include "libstore/store-api.hh" +#include "libutil/pool.hh" + +namespace nix { + +struct NarInfo; + +class BinaryCacheStore : public Store { + public: + const Setting<std::string> compression{ + this, "xz", "compression", + "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting<bool> writeNARListing{ + this, false, "write-nar-listing", + "whether to write a JSON file listing the files in each NAR"}; + const Setting<Path> secretKeyFile{ + this, "", "secret-key", + "path to secret key used to sign the binary cache"}; + const Setting<Path> localNarCache{this, "", "local-nar-cache", + "path to a local cache of NARs"}; + const Setting<bool> parallelCompression{ + this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; + + private: + std::unique_ptr<SecretKey> secretKey; + + protected: + BinaryCacheStore(const Params& params); + + public: + virtual bool fileExists(const std::string& path) = 0; + + virtual void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) = 0; + + /* Note: subclasses must implement at least one of the two + following getFile() methods. */ + + /* Dump the contents of the specified file to a sink. */ + virtual void getFile(const std::string& path, Sink& sink); + + /* Fetch the specified file and call the specified callback with + the result. A subclass may implement this asynchronously. */ + virtual void getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept; + + std::shared_ptr<std::string> getFile(const std::string& path); + + protected: + bool wantMassQuery_ = false; + int priority = 50; + + public: + virtual void init(); + + private: + std::string narMagic; + + std::string narInfoFileFor(const Path& storePath); + + void writeNarInfo(const ref<NarInfo>& narInfo); + + public: + bool isValidPathUncached(const Path& path) override; + + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + + Path queryPathFromHashPart(const std::string& hashPart) override { + unsupported("queryPathFromHashPart"); + } + + bool wantMassQuery() override { return wantMassQuery_; } + + void addToStore(const ValidPathInfo& info, const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override; + + Path addToStore(const std::string& name, const Path& srcPath, bool recursive, + HashType hashAlgo, PathFilter& filter, + RepairFlag repair) override; + + Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair) override; + + void narFromPath(const Path& path, Sink& sink) override; + + BuildResult buildDerivation(std::ostream& /*log_sink*/, const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) override { + unsupported("buildDerivation"); + } + + void ensurePath(const Path& path) override { unsupported("ensurePath"); } + + ref<FSAccessor> getFSAccessor() override; + + void addSignatures(const Path& storePath, const StringSet& sigs) override; + + std::shared_ptr<std::string> getBuildLog(const Path& path) override; + + int getPriority() override { return priority; } +}; + +MakeError(NoSuchBinaryCacheFile, Error); + +} // namespace nix diff --git a/third_party/nix/src/libstore/build.cc b/third_party/nix/src/libstore/build.cc new file mode 100644 index 000000000000..1f5752a168c8 --- /dev/null +++ b/third_party/nix/src/libstore/build.cc @@ -0,0 +1,4820 @@ +#include <algorithm> +#include <cerrno> +#include <chrono> +#include <climits> +#include <cstring> +#include <future> +#include <iostream> +#include <map> +#include <memory> +#include <ostream> +#include <queue> +#include <regex> +#include <sstream> +#include <string> +#include <thread> + +#include <absl/status/status.h> +#include <absl/strings/ascii.h> +#include <absl/strings/numbers.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_format.h> +#include <absl/strings/str_split.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <grp.h> +#include <netdb.h> +#include <pwd.h> +#include <sys/resource.h> +#include <sys/select.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <sys/wait.h> +#include <termios.h> +#include <unistd.h> + +#include "libstore/builtins.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libstore/machines.hh" +#include "libstore/nar-info.hh" +#include "libstore/parsed-derivations.hh" +#include "libstore/pathlocks.hh" +#include "libstore/references.hh" +#include "libstore/store-api.hh" +#include "libutil/affinity.hh" +#include "libutil/archive.hh" +#include "libutil/compression.hh" +#include "libutil/finally.hh" +#include "libutil/json.hh" +#include "libutil/util.hh" + +/* Includes required for chroot support. */ +#if __linux__ +#include <net/if.h> +#include <netinet/ip.h> +#include <sched.h> +#include <sys/ioctl.h> +#include <sys/mman.h> +#include <sys/mount.h> +#include <sys/param.h> +#include <sys/personality.h> +#include <sys/socket.h> +#include <sys/syscall.h> +#if HAVE_SECCOMP +#include <seccomp.h> +#endif +#define pivot_root(new_root, put_old) \ + (syscall(SYS_pivot_root, new_root, put_old)) +#endif + +#if HAVE_STATVFS +#include <sys/statvfs.h> +#endif + +#include <nlohmann/json.hpp> +#include <utility> + +namespace nix { + +constexpr std::string_view kPathNullDevice = "/dev/null"; + +/* Forward definition. */ +class Worker; +struct HookInstance; + +/* A pointer to a goal. */ +class Goal; +class DerivationGoal; +using GoalPtr = std::shared_ptr<Goal>; +using WeakGoalPtr = std::weak_ptr<Goal>; + +struct CompareGoalPtrs { + bool operator()(const GoalPtr& a, const GoalPtr& b) const; +}; + +/* Set of goals. */ +using Goals = std::set<GoalPtr, CompareGoalPtrs>; +using WeakGoals = std::list<WeakGoalPtr>; + +/* A map of paths to goals (and the other way around). */ +using WeakGoalMap = std::map<Path, WeakGoalPtr>; + +class Goal : public std::enable_shared_from_this<Goal> { + public: + using ExitCode = enum { + ecBusy, + ecSuccess, + ecFailed, + ecNoSubstituters, + ecIncompleteClosure + }; + + protected: + /* Backlink to the worker. */ + Worker& worker; + + /* Goals that this goal is waiting for. */ + Goals waitees; + + /* Goals waiting for this one to finish. Must use weak pointers + here to prevent cycles. */ + WeakGoals waiters; + + /* Number of goals we are/were waiting for that have failed. */ + unsigned int nrFailed; + + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; + + /* Number of substitution goals we are/were waiting for that + failed because othey had unsubstitutable references. */ + unsigned int nrIncompleteClosure; + + /* Name of this goal for debugging purposes. */ + std::string name; + + /* Whether the goal is finished. */ + ExitCode exitCode; + + // Output stream for build logs. + // TODO(tazjin): Rename all build_log instances to log_sink. + std::ostream& log_sink() const; + + explicit Goal(Worker& worker) : worker(worker) { + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + exitCode = ecBusy; + } + + virtual ~Goal() { trace("goal destroyed"); } + + public: + virtual void work() = 0; + + void addWaitee(const GoalPtr& waitee); + + virtual void waiteeDone(GoalPtr waitee, ExitCode result); + + virtual void handleChildOutput(int fd, const std::string& data) { abort(); } + + virtual void handleEOF(int fd) { abort(); } + + void trace(const FormatOrString& fs); + + std::string getName() { return name; } + + ExitCode getExitCode() { return exitCode; } + + /* Callback in case of a timeout. It should wake up its waiters, + get rid of any running child processes that are being monitored + by the worker (important!), etc. */ + virtual void timedOut() = 0; + + virtual std::string key() = 0; + + protected: + virtual void amDone(ExitCode result); +}; + +bool CompareGoalPtrs::operator()(const GoalPtr& a, const GoalPtr& b) const { + std::string s1 = a->key(); + std::string s2 = b->key(); + return s1 < s2; +} + +using steady_time_point = std::chrono::time_point<std::chrono::steady_clock>; + +/* A mapping used to remember for each child process to what goal it + belongs, and file descriptors for receiving log data and output + path creation commands. */ +struct Child { + WeakGoalPtr goal; + Goal* goal2; // ugly hackery + std::set<int> fds; + bool respectTimeouts; + bool inBuildSlot; + steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + steady_time_point timeStarted; +}; + +/* The worker class. */ +class Worker { + private: + /* Note: the worker should only have strong pointers to the + top-level goals. */ + + /* The top-level goals of the worker. */ + Goals topGoals; + + /* Goals that are ready to do some work. */ + WeakGoals awake; + + /* Goals waiting for a build slot. */ + WeakGoals wantingToBuild; + + /* Child processes currently running. */ + std::list<Child> children; + + /* Number of build slots occupied. This includes local builds and + substitutions but not remote builds via the build hook. */ + unsigned int nrLocalBuilds; + + /* Maps used to prevent multiple instantiations of a goal for the + same derivation / path. */ + WeakGoalMap derivationGoals; + WeakGoalMap substitutionGoals; + + /* Goals waiting for busy paths to be unlocked. */ + WeakGoals waitingForAnyGoal; + + /* Goals sleeping for a few seconds (polling a lock). */ + WeakGoals waitingForAWhile; + + /* Last time the goals in `waitingForAWhile' where woken up. */ + steady_time_point lastWokenUp; + + /* Cache for pathContentsGood(). */ + std::map<Path, bool> pathContentsGoodCache; + + std::ostream& log_sink_; + + public: + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; + + /* Set if at least one derivation had a timeout. */ + bool timedOut; + + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + + LocalStore& store; + + std::unique_ptr<HookInstance> hook; + + uint64_t expectedBuilds = 0; + uint64_t doneBuilds = 0; + uint64_t failedBuilds = 0; + uint64_t runningBuilds = 0; + + uint64_t expectedSubstitutions = 0; + uint64_t doneSubstitutions = 0; + uint64_t failedSubstitutions = 0; + uint64_t runningSubstitutions = 0; + uint64_t expectedDownloadSize = 0; + uint64_t doneDownloadSize = 0; + uint64_t expectedNarSize = 0; + uint64_t doneNarSize = 0; + + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + + Worker(LocalStore& store, std::ostream& log_sink); + ~Worker(); + + /* Make a goal (with caching). */ + GoalPtr makeDerivationGoal(const Path& drvPath, + const StringSet& wantedOutputs, + BuildMode buildMode); + + std::shared_ptr<DerivationGoal> makeBasicDerivationGoal( + const Path& drvPath, const BasicDerivation& drv, BuildMode buildMode); + + GoalPtr makeSubstitutionGoal(const Path& storePath, + RepairFlag repair = NoRepair); + + /* Remove a dead goal. */ + void removeGoal(const GoalPtr& goal); + + /* Wake up a goal (i.e., there is something for it to do). */ + void wakeUp(const GoalPtr& goal); + + /* Return the number of local build and substitution processes + currently running (but not remote builds via the build + hook). */ + unsigned int getNrLocalBuilds(); + + /* Registers a running child process. `inBuildSlot' means that + the process counts towards the jobs limit. */ + void childStarted(const GoalPtr& goal, const std::set<int>& fds, + bool inBuildSlot, bool respectTimeouts); + + /* Unregisters a running child process. `wakeSleepers' should be + false if there is no sense in waking up goals that are sleeping + because they can't run yet (e.g., there is no free build slot, + or the hook would still say `postpone'). */ + void childTerminated(Goal* goal, bool wakeSleepers = true); + + /* Put `goal' to sleep until a build slot becomes available (which + might be right away). */ + void waitForBuildSlot(const GoalPtr& goal); + + /* Wait for any goal to finish. Pretty indiscriminate way to + wait for some resource that some other goal is holding. */ + void waitForAnyGoal(GoalPtr goal); + + /* Wait for a few seconds and then retry this goal. Used when + waiting for a lock held by another process. This kind of + polling is inefficient, but POSIX doesn't really provide a way + to wait for multiple locks in the main select() loop. */ + void waitForAWhile(GoalPtr goal); + + /* Loop until the specified top-level goals have finished. */ + void run(const Goals& topGoals); + + /* Wait for input to become available. */ + void waitForInput(); + + unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path& path); + + void markContentsGood(const Path& path); + + std::ostream& log_sink() const { return log_sink_; }; +}; + +////////////////////////////////////////////////////////////////////// + +void addToWeakGoals(WeakGoals& goals, const GoalPtr& p) { + // FIXME: necessary? + // FIXME: O(n) + for (auto& i : goals) { + if (i.lock() == p) { + return; + } + } + goals.push_back(p); +} + +std::ostream& Goal::log_sink() const { return worker.log_sink(); } + +void Goal::addWaitee(const GoalPtr& waitee) { + waitees.insert(waitee); + addToWeakGoals(waitee->waiters, shared_from_this()); +} + +void Goal::waiteeDone(GoalPtr waitee, ExitCode result) { + assert(waitees.find(waitee) != waitees.end()); + waitees.erase(waitee); + + trace(format("waitee '%1%' done; %2% left") % waitee->name % waitees.size()); + + if (result == ecFailed || result == ecNoSubstituters || + result == ecIncompleteClosure) { + ++nrFailed; + } + + if (result == ecNoSubstituters) { + ++nrNoSubstituters; + } + + if (result == ecIncompleteClosure) { + ++nrIncompleteClosure; + } + + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (auto& goal : waitees) { + WeakGoals waiters2; + for (auto& j : goal->waiters) { + if (j.lock() != shared_from_this()) { + waiters2.push_back(j); + } + } + goal->waiters = waiters2; + } + waitees.clear(); + + worker.wakeUp(shared_from_this()); + } +} + +void Goal::amDone(ExitCode result) { + trace("done"); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || + result == ecNoSubstituters || result == ecIncompleteClosure); + exitCode = result; + for (auto& i : waiters) { + GoalPtr goal = i.lock(); + if (goal) { + goal->waiteeDone(shared_from_this(), result); + } + } + waiters.clear(); + worker.removeGoal(shared_from_this()); +} + +void Goal::trace(const FormatOrString& fs) { + DLOG(INFO) << name << ": " << fs.s; +} + +////////////////////////////////////////////////////////////////////// + +/* Common initialisation performed in child processes. */ +static void commonChildInit(Pipe& logPipe) { + restoreSignals(); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) { + throw SysError(format("creating a new session")); + } + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) { + throw SysError("cannot pipe standard error into log file"); + } + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) { + throw SysError("cannot dup stderr into stdout"); + } + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(kPathNullDevice.begin(), O_RDWR); + if (fdDevNull == -1) { + throw SysError(format("cannot open '%1%'") % kPathNullDevice); + } + if (dup2(fdDevNull, STDIN_FILENO) == -1) { + throw SysError("cannot dup null device into stdin"); + } + close(fdDevNull); +} + +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, + Path tmpDir, std::ostream& log_sink) { + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions( + diffHook, {std::move(tryA), std::move(tryB), std::move(drvPath), + std::move(tmpDir)}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) { + throw ExecError(diffRes.first, + fmt("diff-hook program '%1%' %2%", diffHook, + statusToString(diffRes.first))); + } + + if (!diffRes.second.empty()) { + log_sink << absl::StripTrailingAsciiWhitespace(diffRes.second); + } + } catch (Error& error) { + log_sink << "diff hook execution failed: " << error.what(); + } + } +} + +////////////////////////////////////////////////////////////////////// + +class UserLock { + private: + /* POSIX locks suck. If we have a lock on a file, and we open and + close that file again (without closing the original file + descriptor), we lose the lock. So we have to be *very* careful + not to open a lock file on which we are holding a lock. */ + static Sync<PathSet> lockedPaths_; + + Path fnUserLock; + AutoCloseFD fdUserLock; + + std::string user; + uid_t uid; + gid_t gid; + std::vector<gid_t> supplementaryGIDs; + + public: + UserLock(); + ~UserLock(); + + void kill(); + + std::string getUser() { return user; } + uid_t getUID() { + assert(uid); + return uid; + } + uid_t getGID() { + assert(gid); + return gid; + } + std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; } + + bool enabled() { return uid != 0; } +}; + +Sync<PathSet> UserLock::lockedPaths_; + +UserLock::UserLock() { + assert(settings.buildUsersGroup != ""); + + /* Get the members of the build-users-group. */ + struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (gr == nullptr) { + throw Error( + format( + "the group '%1%' specified in 'build-users-group' does not exist") % + settings.buildUsersGroup); + } + gid = gr->gr_gid; + + /* Copy the result of getgrnam. */ + Strings users; + for (char** p = gr->gr_mem; *p != nullptr; ++p) { + DLOG(INFO) << "found build user " << *p; + users.push_back(*p); + } + + if (users.empty()) { + throw Error(format("the build users group '%1%' has no members") % + settings.buildUsersGroup); + } + + /* Find a user account that isn't currently in use for another + build. */ + for (auto& i : users) { + DLOG(INFO) << "trying user " << i; + + struct passwd* pw = getpwnam(i.c_str()); + if (pw == nullptr) { + throw Error(format("the user '%1%' in the group '%2%' does not exist") % + i % settings.buildUsersGroup); + } + + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = + (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); + + { + auto lockedPaths(lockedPaths_.lock()); + if (lockedPaths->count(fnUserLock) != 0u) { + /* We already have a lock on this one. */ + continue; + } + lockedPaths->insert(fnUserLock); + } + + try { + AutoCloseFD fd( + open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600)); + if (!fd) { + throw SysError(format("opening user lock '%1%'") % fnUserLock); + } + + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + user = i; + uid = pw->pw_uid; + + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) { + throw Error(format("the Nix user should not be a member of '%1%'") % + settings.buildUsersGroup); + } + +#if __linux__ + /* Get the list of supplementary groups of this build user. This + is usually either empty or contains a group such as "kvm". */ + supplementaryGIDs.resize(10); + int ngroups = supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + supplementaryGIDs.data(), &ngroups); + if (err == -1) { + throw Error( + format("failed to get list of supplementary groups for '%1%'") % + pw->pw_name); + } + + supplementaryGIDs.resize(ngroups); +#endif + + return; + } + + } catch (...) { + lockedPaths_.lock()->erase(fnUserLock); + } + } + + throw Error(format("all build users are currently in use; " + "consider creating additional users and adding them to " + "the '%1%' group") % + settings.buildUsersGroup); +} + +UserLock::~UserLock() { + auto lockedPaths(lockedPaths_.lock()); + assert(lockedPaths->count(fnUserLock)); + lockedPaths->erase(fnUserLock); +} + +void UserLock::kill() { killUser(uid); } + +////////////////////////////////////////////////////////////////////// + +struct HookInstance { + /* Pipes for talking to the build hook. */ + Pipe toHook; + + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* The process ID of the hook. */ + Pid pid; + + FdSink sink; + + HookInstance(); + + ~HookInstance(); +}; + +HookInstance::HookInstance() { + DLOG(INFO) << "starting build hook " << settings.buildHook; + + /* Create a pipe to get the output of the child. */ + fromHook.create(); + + /* Create the communication pipes. */ + toHook.create(); + + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + + /* Fork the hook. */ + pid = startProcess([&]() { + commonChildInit(fromHook); + + if (chdir("/") == -1) { + throw SysError("changing into /"); + } + + /* Dup the communication pipes. */ + if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) { + throw SysError("dupping to-hook read side"); + } + + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide.get(), 4) == -1) { + throw SysError("dupping builder's stdout/stderr"); + } + + /* Hack: pass the read side of that fd to allow build-remote + to read SSH error messages. */ + if (dup2(builderOut.readSide.get(), 5) == -1) { + throw SysError("dupping builder's stdout/stderr"); + } + + Strings args = { + baseNameOf(settings.buildHook), + // std::to_string(verbosity), // TODO(tazjin): what? + }; + + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing '%s'", settings.buildHook); + }); + + pid.setSeparatePG(true); + fromHook.writeSide = AutoCloseFD(-1); + toHook.readSide = AutoCloseFD(-1); + + sink = FdSink(toHook.writeSide.get()); + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto& setting : settings) { + sink << 1 << setting.first << setting.second.value; + } + sink << 0; +} + +HookInstance::~HookInstance() { + try { + toHook.writeSide = AutoCloseFD(-1); + if (pid != Pid(-1)) { + pid.kill(); + } + } catch (...) { + ignoreException(); + } +} + +////////////////////////////////////////////////////////////////////// + +using StringRewrites = std::map<std::string, std::string>; + +std::string rewriteStrings(std::string s, const StringRewrites& rewrites) { + for (auto& i : rewrites) { + size_t j = 0; + while ((j = s.find(i.first, j)) != std::string::npos) { + s.replace(j, i.first.size(), i.second); + } + } + return s; +} + +////////////////////////////////////////////////////////////////////// + +using HookReply = enum { rpAccept, rpDecline, rpPostpone }; + +class SubstitutionGoal; + +class DerivationGoal : public Goal { + private: + /* Whether to use an on-disk .drv file. */ + bool useDerivation; + + /* The path of the derivation. */ + Path drvPath; + + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart = false; + + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; + + /* The derivation stored at drvPath. */ + std::unique_ptr<BasicDerivation> drv; + + std::unique_ptr<ParsedDerivation> parsedDrv; + + /* The remainder is state held during the build. */ + + /* Locks on the output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + PathSet inputPaths; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* Outputs that are already valid. If we're repairing, these are + the outputs that are valid *and* not corrupt. */ + PathSet validPaths; + + /* Outputs that are corrupt or not valid. */ + PathSet missingPaths; + + /* User selected for running the builder. */ + std::unique_ptr<UserLock> buildUser; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + + /* File descriptor for the log file. */ + AutoCloseFD fdLogFile; + std::shared_ptr<BufferedSink> logFileSink, logSink; + + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; + + /* The most recent log lines. */ + std::list<std::string> logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + + std::string currentHookLine; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* Pipe for synchronising updates to the builder user namespace. */ + Pipe userNamespaceSync; + + /* The build hook. */ + std::unique_ptr<HookInstance> hook; + + /* Whether we're currently doing a chroot build. */ + bool useChroot = false; + + Path chrootRootDir; + + /* RAII object to delete the chroot directory. */ + std::shared_ptr<AutoDelete> autoDelChroot; + + /* Whether this is a fixed-output derivation. */ + bool fixedOutput; + + /* Whether to run the build in a private network namespace. */ + bool privateNetwork = false; + + using GoalState = void (DerivationGoal::*)(); + GoalState state; + + /* Stuff we need to pass to initChild(). */ + struct ChrootPath { + Path source; + bool optional; + explicit ChrootPath(Path source = "", bool optional = false) + : source(std::move(source)), optional(optional) {} + }; + using DirsInChroot = + std::map<Path, ChrootPath>; // maps target path to source path + DirsInChroot dirsInChroot; + + using Environment = std::map<std::string, std::string>; + Environment env; + + /* Hash rewriting. */ + StringRewrites inputRewrites, outputRewrites; + using RedirectedOutputs = std::map<Path, Path>; + RedirectedOutputs redirectedOutputs; + + BuildMode buildMode; + + /* If we're repairing without a chroot, there may be outputs that + are valid but corrupt. So we redirect these outputs to + temporary paths. */ + PathSet redirectedBadOutputs; + + BuildResult result; + + /* The current round, if we're building multiple times. */ + size_t curRound = 1; + + size_t nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + std::map<Path, ValidPathInfo> prevInfos; + + const uid_t sandboxUid = 1000; + const gid_t sandboxGid = 100; + + const static Path homeDir; + + std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds; + + /* The remote machine on which we're building. */ + std::string machineName; + + public: + DerivationGoal(Worker& worker, const Path& drvPath, StringSet wantedOutputs, + BuildMode buildMode); + + DerivationGoal(Worker& worker, const Path& drvPath, + const BasicDerivation& drv, BuildMode buildMode); + + ~DerivationGoal() override; + + /* Whether we need to perform hash rewriting if there are valid output paths. + */ + bool needsHashRewrite(); + + void timedOut() override; + + std::string key() override { + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + storePathToName(drvPath) + "$" + drvPath; + } + + void work() override; + + Path getDrvPath() { return drvPath; } + + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet& outputs); + + BuildResult getResult() { return result; } + + private: + /* The states. */ + void getDerivation(); + void loadDerivation(); + void haveDerivation(); + void outputsSubstituted(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void buildDone(); + + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Fill in the environment for the builder. */ + void initEnv(); + + /* Setup tmp dir location. */ + void initTmpDir(); + + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); + + /* Make a file owned by the builder. */ + void chownToBuilder(const Path& path); + + /* Run the builder's process. */ + void runChild(); + + friend int childEntry(void* /*arg*/); + + /* Check that the derivation outputs all exist and register them + as valid. */ + void registerOutputs(); + + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map<std::string, ValidPathInfo>& outputs); + + /* Open a log file and a pipe to it. */ + Path openLogFile(); + + /* Close the log file. */ + void closeLogFile(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const std::string& data) override; + void handleEOF(int fd) override; + void flushLine(); + + /* Return the set of (in)valid paths. */ + PathSet checkPathValidity(bool returnValid, bool checkHash); + + /* Abort the goal if `path' failed to build. */ + bool pathFailed(const Path& path); + + /* Forcibly kill the child process, if any. */ + void killChild(); + + Path addHashRewrite(const Path& path); + + void repairClosure(); + + void amDone(ExitCode result) override { Goal::amDone(result); } + + void done(BuildResult::Status status, const std::string& msg = ""); + + PathSet exportReferences(const PathSet& storePaths); +}; + +const Path DerivationGoal::homeDir = "/homeless-shelter"; + +DerivationGoal::DerivationGoal(Worker& worker, const Path& drvPath, + StringSet wantedOutputs, BuildMode buildMode) + : Goal(worker), + useDerivation(true), + drvPath(drvPath), + wantedOutputs(std::move(wantedOutputs)), + buildMode(buildMode) { + state = &DerivationGoal::getDerivation; + name = (format("building of '%1%'") % drvPath).str(); + trace("created"); + + mcExpectedBuilds = + std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); +} + +DerivationGoal::DerivationGoal(Worker& worker, const Path& drvPath, + const BasicDerivation& drv, BuildMode buildMode) + : Goal(worker), + useDerivation(false), + drvPath(drvPath), + buildMode(buildMode) { + this->drv = std::make_unique<BasicDerivation>(drv); + state = &DerivationGoal::haveDerivation; + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + trace("created"); + + mcExpectedBuilds = + std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(drvPath); +} + +DerivationGoal::~DerivationGoal() { + /* Careful: we should never ever throw an exception from a + destructor. */ + try { + killChild(); + } catch (...) { + ignoreException(); + } + try { + deleteTmpDir(false); + } catch (...) { + ignoreException(); + } + try { + closeLogFile(); + } catch (...) { + ignoreException(); + } +} + +inline bool DerivationGoal::needsHashRewrite() { return !useChroot; } + +void DerivationGoal::killChild() { + if (pid != Pid(-1)) { + worker.childTerminated(this); + + if (buildUser) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-static_cast<pid_t>(pid), SIGKILL); /* ignore the result */ + buildUser->kill(); + pid.wait(); + } else { + pid.kill(); + } + + assert(pid == Pid(-1)); + } + + hook.reset(); +} + +void DerivationGoal::timedOut() { + killChild(); + done(BuildResult::TimedOut); +} + +void DerivationGoal::work() { (this->*state)(); } + +void DerivationGoal::addWantedOutputs(const StringSet& outputs) { + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) { + return; + } + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else { + for (auto& i : outputs) { + if (wantedOutputs.find(i) == wantedOutputs.end()) { + wantedOutputs.insert(i); + needRestart = true; + } + } + } +} + +void DerivationGoal::getDerivation() { + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(worker.makeSubstitutionGoal(drvPath)); + + state = &DerivationGoal::loadDerivation; +} + +void DerivationGoal::loadDerivation() { + trace("loading derivation"); + + if (nrFailed != 0) { + log_sink() << "cannot build missing derivation '" << drvPath << "'" + << std::endl; + done(BuildResult::MiscFailure); + return; + } + + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); + + assert(worker.store.isValidPath(drvPath)); + + /* Get the derivation. */ + drv = std::unique_ptr<BasicDerivation>( + new Derivation(worker.store.derivationFromPath(drvPath))); + + haveDerivation(); +} + +void DerivationGoal::haveDerivation() { + trace("have derivation"); + + retrySubstitution = false; + + for (auto& i : drv->outputs) { + worker.store.addTempRoot(i.second.path); + } + + /* Check what outputs paths are not already valid. */ + PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + + /* If they are all valid, then we're done. */ + if (invalidOutputs.empty() && buildMode == bmNormal) { + done(BuildResult::AlreadyValid); + return; + } + + parsedDrv = std::make_unique<ParsedDerivation>(drvPath, *drv); + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) { + for (auto& i : invalidOutputs) { + addWaitee(worker.makeSubstitutionGoal( + i, buildMode == bmRepair ? Repair : NoRepair)); + } + } + + if (waitees.empty()) { /* to prevent hang (no wake-up event) */ + outputsSubstituted(); + } else { + state = &DerivationGoal::outputsSubstituted; + } +} + +void DerivationGoal::outputsSubstituted() { + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && + !settings.tryFallback) { + done(BuildResult::TransientFailure, + (format("some substitutes for the outputs of derivation '%1%' failed " + "(usually happens due to networking issues); try '--fallback' " + "to build derivation from source ") % + drvPath) + .str()); + return; + } + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. */ + if (nrIncompleteClosure > 0) { + retrySubstitution = true; + } + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + if (buildMode == bmNormal && nrInvalid == 0) { + done(BuildResult::Substituted); + return; + } + if (buildMode == bmRepair && nrInvalid == 0) { + repairClosure(); + return; + } + if (buildMode == bmCheck && nrInvalid > 0) { + throw Error(format("some outputs of '%1%' are not valid, so checking is " + "not possible") % + drvPath); + } + + /* Otherwise, at least one of the output paths could not be + produced using a substitute. So we have to build instead. */ + + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = PathSet(); + + /* The inputs must be built before we can build this goal. */ + if (useDerivation) { + for (auto& i : dynamic_cast<Derivation*>(drv.get())->inputDrvs) { + addWaitee(worker.makeDerivationGoal( + i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + } + } + + for (auto& i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) { + continue; + } + if (!settings.useSubstitutes) { + throw Error(format("dependency '%1%' of '%2%' does not exist, and " + "substitution is disabled") % + i % drvPath); + } + addWaitee(worker.makeSubstitutionGoal(i)); + } + + if (waitees.empty()) { /* to prevent hang (no wake-up event) */ + inputsRealised(); + } else { + state = &DerivationGoal::inputsRealised; + } +} + +void DerivationGoal::repairClosure() { + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + PathSet outputClosure; + for (auto& i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) { + continue; + } + worker.store.computeFSClosure(i.second.path, outputClosure); + } + + /* Filter out our own outputs (which we have already checked). */ + for (auto& i : drv->outputs) { + outputClosure.erase(i.second.path); + } + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + PathSet inputClosure; + if (useDerivation) { + worker.store.computeFSClosure(drvPath, inputClosure); + } + std::map<Path, Path> outputsToDrv; + for (auto& i : inputClosure) { + if (isDerivation(i)) { + Derivation drv = worker.store.derivationFromPath(i); + for (auto& j : drv.outputs) { + outputsToDrv[j.second.path] = i; + } + } + } + + /* Check each path (slow!). */ + PathSet broken; + for (auto& i : outputClosure) { + if (worker.pathContentsGood(i)) { + continue; + } + log_sink() << "found corrupted or missing path '" << i + << "' in the output closure of '" << drvPath << "'" << std::endl; + Path drvPath2 = outputsToDrv[i]; + if (drvPath2.empty()) { + addWaitee(worker.makeSubstitutionGoal(i, Repair)); + } else { + addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); + } + } + + if (waitees.empty()) { + done(BuildResult::AlreadyValid); + return; + } + + state = &DerivationGoal::closureRepaired; +} + +void DerivationGoal::closureRepaired() { + trace("closure repaired"); + if (nrFailed > 0) { + throw Error(format("some paths in the output closure of derivation '%1%' " + "could not be repaired") % + drvPath); + } + done(BuildResult::AlreadyValid); +} + +void DerivationGoal::inputsRealised() { + trace("all inputs realised"); + + if (nrFailed != 0) { + if (!useDerivation) { + throw Error(format("some dependencies of '%1%' are missing") % drvPath); + } + log_sink() << "cannot build derivation '" << drvPath << "': " << nrFailed + << " dependencies couldn't be built" << std::endl; + done(BuildResult::DependencyFailed); + return; + } + + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* The outputs are referenceable paths. */ + for (auto& i : drv->outputs) { + log_sink() << "building path " << i.second.path << std::endl; + allPaths.insert(i.second.path); + } + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + if (useDerivation) { + for (auto& i : dynamic_cast<Derivation*>(drv.get())->inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i.first)); + Derivation inDrv = worker.store.derivationFromPath(i.first); + for (auto& j : i.second) { + if (inDrv.outputs.find(j) != inDrv.outputs.end()) { + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + } else { + throw Error(format("derivation '%1%' requires non-existent output " + "'%2%' from input derivation '%3%'") % + drvPath % j % i.first); + } + } + } + } + + /* Second, the input sources. */ + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + + DLOG(INFO) << "added input paths " << showPaths(inputPaths); + + allPaths.insert(inputPaths.begin(), inputPaths.end()); + + /* Is this a fixed-output derivation? */ + fixedOutput = drv->isFixedOutput(); + + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + + result = BuildResult(); +} + +void DerivationGoal::tryToBuild() { + trace("trying to build"); + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. If we + can't acquire the lock, then continue; hopefully some other + goal can start a build, and if not, the main loop will sleep a + few seconds and then retry this goal. */ + PathSet lockFiles; + for (auto& outPath : drv->outputPaths()) { + lockFiles.insert(worker.store.toRealPath(outPath)); + } + + if (!outputLocks.lockPaths(lockFiles, "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + validPaths = checkPathValidity(true, buildMode == bmRepair); + if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { + DLOG(INFO) << "skipping build of derivation '" << drvPath + << "', someone beat us to it"; + outputLocks.setDeletion(true); + done(BuildResult::AlreadyValid); + return; + } + + missingPaths = drv->outputPaths(); + if (buildMode != bmCheck) { + for (auto& i : validPaths) { + missingPaths.erase(i); + } + } + + /* If any of the outputs already exist but are not valid, delete + them. */ + for (auto& i : drv->outputs) { + Path path = i.second.path; + if (worker.store.isValidPath(path)) { + continue; + } + DLOG(INFO) << "removing invalid path " << path; + deletePath(worker.store.toRealPath(path)); + } + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. Also, check and repair modes are only + supported for local builds. */ + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + + auto started = [&]() { + std::string msg; + if (buildMode == bmRepair) { + msg = absl::StrFormat("repairing outputs of '%s'", drvPath); + } else if (buildMode == bmCheck) { + msg = absl::StrFormat("checking outputs of '%s'", drvPath); + } else if (nrRounds > 1) { + msg = absl::StrFormat("building '%s' (round %d/%d)", drvPath, curRound, + nrRounds); + } else { + msg = absl::StrFormat("building '%s'", drvPath); + } + + if (hook) { + absl::StrAppend(&msg, absl::StrFormat(" on '%s'", machineName)); + } + + log_sink() << msg << std::endl; + mcRunningBuilds = + std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds); + }; + + /* Is the build hook willing to accept this job? */ + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + result.startTime = time(nullptr); // inexact + state = &DerivationGoal::buildDone; + started(); + return; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + try { + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError& e) { + log_sink() << e.msg() << std::endl; + outputLocks.unlock(); + buildUser.reset(); + worker.permanentFailure = true; + done(BuildResult::InputRejected, e.msg()); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; + + started(); +} + +void replaceValidPath(const Path& storePath, const Path& tmpPath) { + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = + (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + if (pathExists(storePath)) { + rename(storePath.c_str(), oldPath.c_str()); + } + if (rename(tmpPath.c_str(), storePath.c_str()) == -1) { + throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath); + } + deletePath(oldPath); +} + +MakeError(NotDeterministic, BuildError); + +void DerivationGoal::buildDone() { + trace("build done"); + + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill() : pid.kill(); + + DLOG(INFO) << "builder process for '" << drvPath << "' finished"; + + result.timesBuilt++; + result.stopTime = time(nullptr); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide = AutoCloseFD(-1); + hook->fromHook.readSide = AutoCloseFD(-1); + } else { + builderOut.readSide = AutoCloseFD(-1); + } + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) { + buildUser->kill(); + } + + bool diskFull = false; + + try { + /* Check the exit status. */ + if (!statusOk(status)) { + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + unsigned long long required = + 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && + static_cast<unsigned long long>(st.f_bavail) * st.f_bsize < + required) { + diskFull = true; + } + if (statvfs(tmpDir.c_str(), &st) == 0 && + static_cast<unsigned long long>(st.f_bavail) * st.f_bsize < + required) { + diskFull = true; + } +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) { + for (auto& i : missingPaths) { + if (pathExists(chrootRootDir + i)) { + rename((chrootRootDir + i).c_str(), i.c_str()); + } + } + } + + std::string msg = + (format("builder for '%1%' %2%") % drvPath % statusToString(status)) + .str(); + + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto& line : logTail) { + msg += "\n " + line; + } + } + + if (diskFull) { + msg += + "\nnote: build failure may have been caused by lack of free disk " + "space"; + } + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + registerOutputs(); + + if (settings.postBuildHook != "") { + log_sink() << "running post-build-hook '" << settings.postBuildHook + << "' [" << drvPath << "]" << std::endl; + auto outputPaths = drv->outputPaths(); + std::map<std::string, std::string> hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", + absl::StripTrailingAsciiWhitespace( + concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + std::string currentLine; + + void operator()(const unsigned char* data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + LOG(ERROR) << "post-build-hook: " << currentLine; + } + currentLine.clear(); + } + + ~LogSink() override { + if (!currentLine.empty()) { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink; + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + + if (buildMode == bmCheck) { + done(BuildResult::Built); + return; + } + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto& i : redirectedOutputs) { + deletePath(i.second); + } + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + } catch (BuildError& e) { + log_sink() << e.msg() << std::endl; + + outputLocks.unlock(); + + BuildResult::Status st = BuildResult::MiscFailure; + + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) { + st = BuildResult::TimedOut; + + } else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { + } + + else { + st = dynamic_cast<NotDeterministic*>(&e) != nullptr + ? BuildResult::NotDeterministic + : statusOk(status) ? BuildResult::OutputRejected + : fixedOutput || diskFull ? BuildResult::TransientFailure + : BuildResult::PermanentFailure; + } + + done(st, e.msg()); + return; + } + + done(BuildResult::Built); +} + +HookReply DerivationGoal::tryBuildHook() { + if (!worker.tryBuildHook || !useDerivation) { + return rpDecline; + } + + if (!worker.hook) { + worker.hook = std::make_unique<HookInstance>(); + } + + try { + /* Send the request to the hook. */ + worker.hook->sink << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 + : 0) + << drv->platform << drvPath + << parsedDrv->getRequiredSystemFeatures(); + worker.hook->sink.flush(); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + std::string reply; + while (true) { + std::string s = readLine(worker.hook->fromHook.readSide.get()); + if (std::string(s, 0, 2) == "# ") { + reply = std::string(s, 2); + break; + } + s += "\n"; + std::cerr << s; + } + + DLOG(INFO) << "hook reply is " << reply; + + if (reply == "decline") { + return rpDecline; + } + if (reply == "decline-permanently") { + worker.tryBuildHook = false; + worker.hook = nullptr; + return rpDecline; + } else if (reply == "postpone") { + return rpPostpone; + } else if (reply != "accept") { + throw Error(format("bad hook reply '%1%'") % reply); + } + } catch (SysError& e) { + if (e.errNo == EPIPE) { + log_sink() << "build hook died unexpectedly: " + << absl::StripTrailingAsciiWhitespace( + drainFD(worker.hook->fromHook.readSide.get())) + << std::endl; + worker.hook = nullptr; + return rpDecline; + } + throw; + } + + hook = std::move(worker.hook); + + machineName = readLine(hook->fromHook.readSide.get()); + + /* Tell the hook all the inputs that have to be copied to the + remote system. */ + hook->sink << inputPaths; + + /* Tell the hooks the missing outputs that have to be copied back + from the remote system. */ + hook->sink << missingPaths; + + hook->sink = FdSink(); + hook->toHook.writeSide = AutoCloseFD(-1); + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + std::set<int> fds; + fds.insert(hook->fromHook.readSide.get()); + fds.insert(hook->builderOut.readSide.get()); + worker.childStarted(shared_from_this(), fds, false, false); + + return rpAccept; +} + +void chmod_(const Path& path, mode_t mode) { + if (chmod(path.c_str(), mode) == -1) { + throw SysError(format("setting permissions on '%1%'") % path); + } +} + +int childEntry(void* arg) { + (static_cast<DerivationGoal*>(arg))->runChild(); + return 1; +} + +PathSet DerivationGoal::exportReferences(const PathSet& storePaths) { + PathSet paths; + + for (auto storePath : storePaths) { + /* Check that the store path is valid. */ + if (!worker.store.isInStore(storePath)) { + throw BuildError( + format("'exportReferencesGraph' contains a non-store path '%1%'") % + storePath); + } + + storePath = worker.store.toStorePath(storePath); + + if (inputPaths.count(storePath) == 0u) { + throw BuildError( + "cannot export references of path '%s' because it is not in the " + "input closure of the derivation", + storePath); + } + + worker.store.computeFSClosure(storePath, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths2(paths); + + for (auto& j : paths2) { + if (isDerivation(j)) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto& k : drv.outputs) { + worker.store.computeFSClosure(k.second.path, paths); + } + } + } + + return paths; +} + +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a + dynamic library load of one of the glibc NSS libraries in a sandboxed + child, which will fail unless the library's already been loaded in the + parent. So we force a lookup of an invalid domain to force the NSS + machinery to + load its lookup libraries in the parent before any child gets a chance to. + */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo* res = nullptr; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", + nullptr, &res) != 0) { + if (res != nullptr) { + freeaddrinfo(res); + } + } + }); +} + +void DerivationGoal::startBuilder() { + /* Right platform? */ + if (!parsedDrv->canBuildLocally()) { + throw Error( + "a '%s' with features {%s} is required to build '%s', but I am a '%s' " + "with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), drvPath, + settings.thisSystem, concatStringsSep(", ", settings.systemFeatures)); + } + + if (drv->isBuiltin()) { + preloadNSS(); + } + + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) { + throw Error(format("derivation '%1%' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'") % + drvPath); + } + useChroot = true; + } else if (settings.sandboxMode == smDisabled) { + useChroot = false; + } else if (settings.sandboxMode == smRelaxed) { + useChroot = !fixedOutput && !noChroot; + } + } + + if (worker.store.storeDir != worker.store.realStoreDir) { + useChroot = true; + } + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { + buildUser = std::make_unique<UserLock>(); + + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); + } + + /* Create a temporary directory where the build will take + place. */ + auto drvName = storePathToName(drvPath); + tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + + chownToBuilder(tmpDir); + + /* Substitute output placeholders with the actual output paths. */ + for (auto& output : drv->outputs) { + inputRewrites[hashPlaceholder(output.first)] = output.second.path; + } + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv->getStructuredAttrs()) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + std::string s = get(drv->env, "exportReferencesGraph"); + std::vector<std::string> ss = + absl::StrSplit(s, absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + if (ss.size() % 2 != 0) { + throw BuildError(absl::StrFormat( + "odd number of tokens %d in 'exportReferencesGraph': '%s'", ss.size(), + s)); + } + for (auto i = ss.begin(); i != ss.end();) { + std::string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + Path storePath = *i++; + + /* Write closure info to <fileName>. */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences({storePath}), false, false)); + } + } + + if (useChroot) { + /* Allow a user-configurable set of directories from the + host file system. */ + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; + dirs.insert(dirs2.begin(), dirs2.end()); + + dirsInChroot.clear(); + + for (auto i : dirs) { + if (i.empty()) { + continue; + } + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == std::string::npos) { + dirsInChroot[i] = ChrootPath(i, optional); + } else { + dirsInChroot[std::string(i, 0, p)] = + ChrootPath(std::string(i, p + 1), optional); + } + } + dirsInChroot[tmpDirInSandbox] = ChrootPath(tmpDir); + + /* Add the closure of store paths to the chroot. */ + PathSet closure; + for (auto& i : dirsInChroot) { + try { + if (worker.store.isInStore(i.second.source)) { + worker.store.computeFSClosure( + worker.store.toStorePath(i.second.source), closure); + } + } catch (InvalidPath& e) { + } catch (Error& e) { + throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + } + } + for (auto& i : closure) { + dirsInChroot[i] = ChrootPath(i); + } + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = + parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); + + for (auto& i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are + * generally going to be pretty small */ + for (auto& a : allowedPaths) { + Path canonA = canonPath(a); + if (canonI == canonA || isInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) { + throw Error(format("derivation '%1%' requested impure path '%2%', but " + "it was not in allowed-impure-host-deps") % + drvPath % i); + } + + dirsInChroot[i] = ChrootPath(i); + } + + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; + deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir); + + DLOG(INFO) << "setting up chroot environment in '" << chrootRootDir << "'"; + + if (mkdir(chrootRootDir.c_str(), 0750) == -1) { + throw SysError(format("cannot create '%1%'") % chrootRootDir); + } + + if (buildUser && + chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) { + throw SysError(format("cannot change ownership of '%1%'") % + chrootRootDir); + } + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + writeFile(chrootRootDir + "/etc/passwd", + fmt("root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", (format("root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n") % + sandboxGid) + .str()); + + /* Create /etc/hosts with localhost entry. */ + if (!fixedOutput) { + writeFile(chrootRootDir + "/etc/hosts", + "127.0.0.1 localhost\n::1 localhost\n"); + } + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && + chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) { + throw SysError(format("cannot change ownership of '%1%'") % + chrootStoreDir); + } + + for (auto& i : inputPaths) { + Path r = worker.store.toRealPath(i); + struct stat st; + if (lstat(r.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % i); + } + if (S_ISDIR(st.st_mode)) { + dirsInChroot[i] = ChrootPath(r); + } else { + Path p = chrootRootDir + i; + DLOG(INFO) << "linking '" << p << "' to '" << r << "'"; + if (link(r.c_str(), p.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum + link count on a file (e.g. 32000 of ext3), + which is quite possible after a `nix-store + --optimise'. */ + if (errno != EMLINK) { + throw SysError(format("linking '%1%' to '%2%'") % p % i); + } + StringSink sink; + dumpPath(r, sink); + StringSource source(*sink.s); + restorePath(p, source); + } + } + } + + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.dirsInChroot + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto& i : drv->outputs) { + dirsInChroot.erase(i.second.path); + } + } + + if (needsHashRewrite()) { + if (pathExists(homeDir)) { + throw Error(format("directory '%1%' exists; please remove it") % homeDir); + } + + /* We're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + if (!validPaths.empty()) { + for (auto& i : validPaths) { + addHashRewrite(i); + } + } + + /* If we're repairing, then we don't want to delete the + corrupt outputs in advance. So rewrite them as well. */ + if (buildMode == bmRepair) { + for (auto& i : missingPaths) { + if (worker.store.isValidPath(i) && pathExists(i)) { + addHashRewrite(i); + redirectedBadOutputs.insert(i); + } + } + } + } + + if (useChroot && settings.preBuildHook != "" && + (dynamic_cast<Derivation*>(drv.get()) != nullptr)) { + DLOG(INFO) << "executing pre-build hook '" << settings.preBuildHook << "'"; + auto args = + useChroot ? Strings({drvPath, chrootRootDir}) : Strings({drvPath}); + enum BuildHookState { stBegin, stExtraChrootDirs }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != std::string::npos; + nlPos = lines.find('\n', lastPos)) { + auto line = std::string{lines, lastPos, nlPos - lastPos}; + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error(format("unknown pre-build hook command '%1%'") % line); + } + } else if (state == stExtraChrootDirs) { + if (line.empty()) { + state = stBegin; + } else { + auto p = line.find('='); + if (p == std::string::npos) { + dirsInChroot[line] = ChrootPath(line); + } else { + dirsInChroot[std::string(line, 0, p)] = + ChrootPath(std::string(line, p + 1)); + } + } + } + } + } + + /* Run the builder. */ + DLOG(INFO) << "executing builder '" << drv->builder << "'"; + + /* Create the log file. */ + Path logFile = openLogFile(); + + /* Create a pipe to get the output of the builder. */ + // builderOut.create(); + + builderOut.readSide = AutoCloseFD(posix_openpt(O_RDWR | O_NOCTTY)); + if (!builderOut.readSide) { + throw SysError("opening pseudoterminal master"); + } + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600) != 0) { + throw SysError("changing mode of pseudoterminal slave"); + } + + if (chown(slaveName.c_str(), buildUser->getUID(), 0) != 0) { + throw SysError("changing owner of pseudoterminal slave"); + } + } else { + if (grantpt(builderOut.readSide.get()) != 0) { + throw SysError("granting access to pseudoterminal slave"); + } + } + +#if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; +#endif + + if (unlockpt(builderOut.readSide.get()) != 0) { + throw SysError("unlocking pseudoterminal"); + } + + builderOut.writeSide = + AutoCloseFD(open(slaveName.c_str(), O_RDWR | O_NOCTTY)); + if (!builderOut.writeSide) { + throw SysError("opening pseudoterminal slave"); + } + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term) != 0) { + throw SysError("getting pseudoterminal attributes"); + } + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term) != 0) { + throw SysError("putting pseudoterminal into raw mode"); + } + + result.startTime = time(nullptr); + + /* Fork a child to build the package. */ + ProcessOptions options; + +#if __linux__ + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + if (!fixedOutput) { + privateNetwork = true; + } + + userNamespaceSync.create(); + + Pid helper(startProcess( + [&]() { + /* Drop additional groups here because we can't do it + after we've created the new user namespace. FIXME: + this means that if we're not root in the parent + namespace, we can't drop additional groups; they will + be mapped to nogroup in the child namespace. There does + not seem to be a workaround for this. (But who can tell + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ + if (getuid() == 0 && setgroups(0, nullptr) == -1) { + throw SysError("setgroups failed"); + } + + size_t stackSize = 1 * 1024 * 1024; + char* stack = static_cast<char*>( + mmap(nullptr, stackSize, PROT_WRITE | PROT_READ, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0)); + if (stack == MAP_FAILED) { + throw SysError("allocating stack"); + } + + int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | + CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) { + flags |= CLONE_NEWNET; + } + + pid_t child = clone(childEntry, stack + stackSize, flags, this); + if (child == -1 && errno == EINVAL) { + /* Fallback for Linux < 2.13 where CLONE_NEWPID and + CLONE_PARENT are not allowed together. */ + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unpriveleged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && + settings.sandboxFallback) { + _exit(1); + } + if (child == -1) { + throw SysError("cloning builder process"); + } + + writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); + _exit(0); + }, + options)); + + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + initTmpDir(); + goto fallback; + } else if (res != 0) { + throw Error("unable to start build process"); + } + + userNamespaceSync.readSide = AutoCloseFD(-1); + + pid_t tmp; + if (!absl::SimpleAtoi(readLine(builderOut.readSide.get()), &tmp)) { + abort(); + } + pid = tmp; + + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + + writeFile("/proc/" + std::to_string(static_cast<pid_t>(pid)) + "/uid_map", + (format("%d %d 1") % sandboxUid % hostUid).str()); + + writeFile("/proc/" + std::to_string(static_cast<pid_t>(pid)) + "/setgroups", + "deny"); + + writeFile("/proc/" + std::to_string(static_cast<pid_t>(pid)) + "/gid_map", + (format("%d %d 1") % sandboxGid % hostGid).str()); + + /* Signal the builder that we've updated its user + namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + userNamespaceSync.writeSide = AutoCloseFD(-1); + + } else +#endif + { + fallback: + pid = startProcess([&]() { runChild(); }, options); + } + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide = AutoCloseFD(-1); + worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, + true); + + /* Check if setting up the build environment failed. */ + while (true) { + std::string msg = readLine(builderOut.readSide.get()); + if (std::string(msg, 0, 1) == "\1") { + if (msg.size() == 1) { + break; + } + throw Error(std::string(msg, 1)); + } + DLOG(INFO) << msg; + } +} + +void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, add all bindings specified in the + derivation via the environment, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!parsedDrv->getStructuredAttrs()) { + std::set<std::string> passAsFile = + absl::StrSplit(get(drv->env, "passAsFile"), absl::ByAnyChar(" \t\n\r"), + absl::SkipEmpty()); + for (auto& i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + auto hash = hashString(htSHA256, i.first); + std::string fn = ".attr-" + hash.to_string(Base32, false); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + +void DerivationGoal::initEnv() { + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) { + env["NIX_OUTPUT_CHECKED"] = "1"; + } + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + for (auto& i : + parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) { + env[i] = getEnv(i).value_or(""); + } + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + +void DerivationGoal::writeStructuredAttrs() { + auto& structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) { + return; + } + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto& i : drv->outputs) { + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + } + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto& p : *i) { + storePaths.insert(p.get<std::string>()); + } + worker.store.pathInfoToJSON(jsonRoot, exportReferences(storePaths), + false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + writeFile(tmpDir + "/.attrs.json", + rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = + [](const nlohmann::json& value) -> std::optional<std::string> { + if (value.is_string()) { + return shellEscape(value); + } + + if (value.is_number()) { + auto f = value.get<float>(); + if (std::ceil(f) == f) { + return std::to_string(value.get<int>()); + } + } + + if (value.is_null()) { + return std::string("''"); + } + + if (value.is_boolean()) { + return value.get<bool>() ? std::string("1") : std::string(""); + } + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + if (!std::regex_match(i.key(), shVarName)) { + continue; + } + + auto& value = i.value(); + + auto s = handleSimpleType(value); + if (s) { + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + } else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { + good = false; + break; + } + s2 += *s3; + s2 += ' '; + } + + if (good) { + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { + good = false; + break; + } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) { + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); +} + +void DerivationGoal::chownToBuilder(const Path& path) { + if (!buildUser) { + return; + } + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) { + throw SysError(format("cannot change ownership of '%1%'") % path); + } +} + +void setupSeccomp() { +#if __linux__ + if (!settings.filterSyscalls) { + return; + } +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if ((ctx = seccomp_init(SCMP_ACT_ALLOW)) == nullptr) { + throw SysError("unable to initialize seccomp mode 2"); + } + + Finally cleanup([&]() { seccomp_release(ctx); }); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) { + throw SysError("unable to add 32-bit seccomp architecture"); + } + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) { + throw SysError("unable to add X32 seccomp architecture"); + } + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) { + LOG(ERROR) << "unable to add ARM seccomp architecture; this may result in " + << "spurious build failures if running 32-bit ARM processes"; + } + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : {S_ISUID, S_ISGID}) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t)perm, + (scmp_datum_t)perm)) != 0) { + throw SysError("unable to add seccomp rule"); + } + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t)perm, + (scmp_datum_t)perm)) != 0) { + throw SysError("unable to add seccomp rule"); + } + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t)perm, + (scmp_datum_t)perm)) != 0) { + throw SysError("unable to add seccomp rule"); + } + } + + /* Prevent builders from creating EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != + 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != + 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != + 0) { + throw SysError("unable to add seccomp rule"); + } + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, + settings.allowNewPrivileges ? 0 : 1) != 0) { + throw SysError("unable to set 'no new privileges' seccomp attribute"); + } + + if (seccomp_load(ctx) != 0) { + throw SysError("unable to load seccomp BPF program"); + } +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to " + "false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif +} + +void DerivationGoal::runChild() { + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + try { /* child */ + + commonChildInit(builderOut); + + try { + setupSeccomp(); + } catch (...) { + if (buildUser) { + throw; + } + } + + bool setUser = true; + + /* Make the contents of netrc available to builtin:fetchurl + (which may run under a different uid and/or in a sandbox). */ + std::string netrcData; + try { + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { + const std::string& netrc_file = settings.netrcFile; + netrcData = readFile(netrc_file); + } + } catch (SysError&) { + } + +#if __linux__ + if (useChroot) { + userNamespaceSync.writeSide = AutoCloseFD(-1); + + if (drainFD(userNamespaceSync.readSide.get()) != "1") { + throw Error("user namespace initialisation failed"); + } + + userNamespaceSync.readSide = AutoCloseFD(-1); + + if (privateNetwork) { + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) { + throw SysError("cannot open IP socket"); + } + + struct ifreq ifr; + strncpy(ifr.ifr_name, "lo", sizeof("lo")); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) { + throw SysError("cannot set loopback interface flags"); + } + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) { + throw SysError("cannot set host name"); + } + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) { + throw SysError("cannot set domain name"); + } + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(nullptr, "/", nullptr, MS_REC | MS_PRIVATE, nullptr) == -1) { + throw SysError("unable to make '/' private mount"); + } + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), nullptr, MS_BIND, + nullptr) == -1) { + throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + } + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if ((settings.systemFeatures.get().count("kvm") != 0u) && + pathExists("/dev/kvm")) { + ss.push_back("/dev/kvm"); + } + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (fixedOutput) { + ss.push_back("/etc/resolv.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", + "hosts: files dns\nservices: files\n"); + + ss.push_back("/etc/services"); + ss.push_back("/etc/hosts"); + if (pathExists("/var/run/nscd/socket")) { + ss.push_back("/var/run/nscd/socket"); + } + } + + for (auto& i : ss) { + dirsInChroot.emplace(i, i); + } + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + auto doBind = [&](const Path& source, const Path& target, + bool optional = false) { + DLOG(INFO) << "bind mounting '" << source << "' to '" << target << "'"; + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) { + return; + } + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) { + createDirs(target); + } else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, + nullptr) == -1) { + throw SysError("bind mount from '%1%' to '%2%' failed", source, + target); + } + }; + + for (auto& i : dirsInChroot) { + if (i.second.source == "/proc") { + continue; + } // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, + nullptr) == -1) { + throw SysError("mounting /proc"); + } + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && + mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) { + throw SysError("mounting /dev/shm"); + } + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") && + (dirsInChroot.count("/dev/pts") == 0u)) { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, + "newinstance,mode=0620") == 0) { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) { + throw SysError("mounting /dev/pts"); + } + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) { + throw SysError(format("cannot change directory to '%1%'") % + chrootRootDir); + } + + if (mkdir("real-root", 0) == -1) { + throw SysError("cannot create real-root directory"); + } + + if (pivot_root(".", "real-root") == -1) { + throw SysError(format("cannot pivot old root directory onto '%1%'") % + (chrootRootDir + "/real-root")); + } + + if (chroot(".") == -1) { + throw SysError(format("cannot change root directory to '%1%'") % + chrootRootDir); + } + + if (umount2("real-root", MNT_DETACH) == -1) { + throw SysError("cannot unmount real root filesystem"); + } + + if (rmdir("real-root") == -1) { + throw SysError("cannot remove real-root directory"); + } + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid) == -1) { + throw SysError("setgid failed"); + } + if (setuid(sandboxUid) == -1) { + throw SysError("setuid failed"); + } + + setUser = false; + } +#endif + + if (chdir(tmpDirInSandbox.c_str()) == -1) { + throw SysError(format("changing into '%1%'") % tmpDir); + } + + /* Close all other file descriptors. */ + closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv->platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + ((strcmp(utsbuf.sysname, "Linux") == 0) && + (strcmp(utsbuf.machine, "x86_64") == 0)))) { + if (personality(PER_LINUX32) == -1) { + throw SysError("cannot set i686-linux personality"); + } + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && + settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) { + personality(cur | 0x0020000 /* == UNAME26 */); + } + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) { + personality(cur | ADDR_NO_RANDOMIZE); + } +#endif + + /* Disable core dumps by default. */ + struct rlimit limit = {0, RLIM_INFINITY}; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto& i : env) { + envStrs.push_back( + rewriteStrings(i.first + "=" + i.second, inputRewrites)); + } + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) { + throw SysError("cannot set supplementary groups of build user"); + } + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || getegid() != buildUser->getGID()) { + throw SysError("setgid failed"); + } + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || geteuid() != buildUser->getUID()) { + throw SysError("setuid failed"); + } + } + + /* Fill in the arguments. */ + Strings args; + + const char* builder = "invalid"; + + if (!drv->isBuiltin()) { + builder = drv->builder.c_str(); + std::string builderBasename = baseNameOf(drv->builder); + args.push_back(builderBasename); + } + + for (auto& i : drv->args) { + args.push_back(rewriteStrings(i, inputRewrites)); + } + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, std::string("\1\n")); + + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + BasicDerivation drv2(*drv); + for (auto& e : drv2.env) { + e.second = rewriteStrings(e.second, inputRewrites); + } + + if (drv->builder == "builtin:fetchurl") { + builtinFetchurl(drv2, netrcData); + } else if (drv->builder == "builtin:buildenv") { + builtinBuildenv(drv2); + } else { + throw Error(format("unsupported builtin function '%1%'") % + std::string(drv->builder, 8)); + } + _exit(0); + } catch (std::exception& e) { + writeFull(STDERR_FILENO, "error: " + std::string(e.what()) + "\n"); + _exit(1); + } + } + + execve(builder, stringsToCharPtrs(args).data(), + stringsToCharPtrs(envStrs).data()); + + throw SysError(format("executing '%1%'") % drv->builder); + + } catch (std::exception& e) { + writeFull(STDERR_FILENO, "\1while setting up the build environment: " + + std::string(e.what()) + "\n"); + _exit(1); + } +} + +/* Parse a list of reference specifiers. Each element must either be + a store path, or the symbolic name of the output of the derivation + (such as `out'). */ +PathSet parseReferenceSpecifiers(Store& store, const BasicDerivation& drv, + const Strings& paths) { + PathSet result; + for (auto& i : paths) { + if (store.isStorePath(i)) { + result.insert(i); + } else if (drv.outputs.find(i) != drv.outputs.end()) { + result.insert(drv.outputs.find(i)->second.path); + } else { + throw BuildError( + format("derivation contains an illegal reference specifier '%1%'") % + i); + } + } + return result; +} + +void DerivationGoal::registerOutputs() { + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. */ + if (hook) { + bool allValid = true; + for (auto& i : drv->outputs) { + if (!worker.store.isValidPath(i.second.path)) { + allValid = false; + } + } + if (allValid) { + return; + } + } + + std::map<std::string, ValidPathInfo> infos; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = ".check"; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; + + std::exception_ptr delayedException; + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + for (auto& i : drv->outputs) { + Path path = i.second.path; + if (missingPaths.find(path) == missingPaths.end()) { + continue; + } + + ValidPathInfo info; + + Path actualPath = path; + if (useChroot) { + actualPath = chrootRootDir + path; + if (pathExists(actualPath)) { + /* Move output paths from the chroot to the Nix store. */ + if (buildMode == bmRepair) { + replaceValidPath(path, actualPath); + } else if (buildMode != bmCheck && + rename(actualPath.c_str(), + worker.store.toRealPath(path).c_str()) == -1) { + throw SysError(format("moving build output '%1%' from the sandbox to " + "the Nix store") % + path); + } + } + if (buildMode != bmCheck) { + actualPath = worker.store.toRealPath(path); + } + } + + if (needsHashRewrite()) { + Path redirected = redirectedOutputs[path]; + if (buildMode == bmRepair && + redirectedBadOutputs.find(path) != redirectedBadOutputs.end() && + pathExists(redirected)) { + replaceValidPath(path, redirected); + } + if (buildMode == bmCheck && !redirected.empty()) { + actualPath = redirected; + } + } + + struct stat st; + if (lstat(actualPath.c_str(), &st) == -1) { + if (errno == ENOENT) { + throw BuildError( + format("builder for '%1%' failed to produce output path '%2%'") % + drvPath % path); + } + throw SysError(format("getting attributes of path '%1%'") % actualPath); + } + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && ((st.st_mode & (S_IWGRP | S_IWOTH)) != 0u)) || + (buildUser && st.st_uid != buildUser->getUID())) { + throw BuildError(format("suspicious ownership or permission on '%1%'; " + "rejecting this build output") % + path); + } +#endif + + /* Apply hash rewriting if necessary. */ + bool rewritten = false; + if (!outputRewrites.empty()) { + LOG(WARNING) << "rewriting hashes in '" << path << "'; cross fingers"; + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, + inodesSeen); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + rewritten = true; + } + + /* Check that fixed-output derivations produced the right + outputs (i.e., the content hash should match the specified + hash). */ + if (fixedOutput) { + bool recursive; + Hash h; + i.second.parseHashInfo(recursive, h); + + if (!recursive) { + /* The output path should be a regular file without + execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) { + throw BuildError( + format( + "output path '%1%' should be a non-executable regular file") % + path); + } + } + + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ + Hash h2 = recursive ? hashPath(h.type, actualPath).first + : hashFile(h.type, actualPath); + + Path dest = worker.store.makeFixedOutputPath(recursive, h2, + storePathToName(path)); + + if (h != h2) { + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n " + "wanted: %s\n got: %s", + dest, h.to_string(), h2.to_string())); + + Path actualDest = worker.store.toRealPath(dest); + + if (worker.store.isValidPath(dest)) { + std::rethrow_exception(delayedException); + } + + if (actualPath != actualDest) { + PathLocks outputLocks({actualDest}); + deletePath(actualDest); + if (rename(actualPath.c_str(), actualDest.c_str()) == -1) { + throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); + } + } + + path = dest; + actualPath = actualDest; + } else { + assert(path == dest); + } + + info.ca = makeFixedOutputCA(recursive, h2); + } + + /* Get rid of all weird permissions. This also checks that + all files are owned by the build user, if applicable. */ + canonicalisePathMetaData(actualPath, + buildUser && !rewritten ? buildUser->getUID() : -1, + inodesSeen); + + /* For this output path, find the references to other paths + contained in it. Compute the SHA-256 NAR hash at the same + time. The hash is stored in the database so that we can + verify later on whether nobody has messed with the store. */ + DLOG(INFO) << "scanning for references inside '" << path << "'"; + HashResult hash; + PathSet references = scanForReferences(actualPath, allPaths, hash); + + if (buildMode == bmCheck) { + if (!worker.store.isValidPath(path)) { + continue; + } + auto info = *worker.store.queryPathInfo(path); + if (hash.first != info.narHash) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { + Path dst = worker.store.toRealPath(path + checkSuffix); + deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str()) != 0) { + throw SysError(format("renaming '%1%' to '%2%'") % actualPath % + dst); + } + + handleDiffHook(buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), path, dst, + drvPath, tmpDir, log_sink()); + + throw NotDeterministic( + format("derivation '%1%' may not be deterministic: output '%2%' " + "differs from '%3%'") % + drvPath % path % dst); + } + throw NotDeterministic(format("derivation '%1%' may not be " + "deterministic: output '%2%' differs") % + drvPath % path); + } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + + continue; + } + + /* For debugging, print out the referenced and unreferenced + paths. */ + for (auto& i : inputPaths) { + auto j = references.find(i); + if (j == references.end()) { + DLOG(INFO) << "unreferenced input: '" << i << "'"; + } else { + DLOG(INFO) << "referenced input: '" << i << "'"; + } + } + + if (curRound == nrRounds) { + worker.store.optimisePath( + actualPath); // FIXME: combine with scanForReferences() + worker.markContentsGood(path); + } + + info.path = path; + info.narHash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + + if (!info.references.empty()) { + info.ca.clear(); + } + + infos[i.first] = info; + } + + if (buildMode == bmCheck) { + return; + } + + /* Apply output checks. */ + checkOutputs(infos); + + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); + ++i, ++j) { + if (!(*i == *j)) { + result.isNonDeterministic = true; + Path prev = i->second.path + checkSuffix; + bool prevExists = keepPreviousRound && pathExists(prev); + auto msg = + prevExists + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous " + "round", + i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", + i->second.path, drvPath); + + handleDiffHook(buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), prev, + i->second.path, drvPath, tmpDir, log_sink()); + + if (settings.enforceDeterminism) { + throw NotDeterministic(msg); + } + + log_sink() << msg << std::endl; + curRound = nrRounds; // we know enough, bail out early + } + } + } + + /* If this is the first round of several, then move the output out + of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && + keepPreviousRound) { + for (auto& i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str()) != 0) { + throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst); + } + } + } + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto& i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + } + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + ValidPathInfos infos2; + for (auto& i : infos) { + infos2.push_back(i.second); + } + worker.store.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) { + std::rethrow_exception(delayedException); + } +} + +void DerivationGoal::checkOutputs( + const std::map<Path, ValidPathInfo>& outputs) { + std::map<Path, const ValidPathInfo&> outputsByPath; + for (auto& output : outputs) { + outputsByPath.emplace(output.second.path, output.second); + } + + for (auto& output : outputs) { + auto& outputName = output.first; + auto& info = output.second; + + struct Checks { + bool ignoreSelfRefs = false; + std::optional<uint64_t> maxSize, maxClosureSize; + std::optional<Strings> allowedReferences, allowedRequisites, + disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path& path) { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue<Path> pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) { + continue; + } + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto& ref : i->second.references) { + pathsLeft.push(ref); + } + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto& ref : info->references) { + pathsLeft.push(ref); + } + } + } + + return std::make_pair(pathsDone, closureSize); + }; + + auto applyChecks = [&](const Checks& checks) { + if (checks.maxSize && info.narSize > *checks.maxSize) { + throw BuildError( + "path '%s' is too large at %d bytes; limit is %d bytes", info.path, + info.narSize, *checks.maxSize); + } + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) { + throw BuildError( + "closure of path '%s' is too large at %d bytes; limit is %d " + "bytes", + info.path, closureSize, *checks.maxClosureSize); + } + } + + auto checkRefs = [&](const std::optional<Strings>& value, bool allowed, + bool recursive) { + if (!value) { + return; + } + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = + recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) { + used.erase(info.path); + } + + PathSet badPaths; + + for (auto& i : used) { + if (allowed) { + if (spec.count(i) == 0u) { + badPaths.insert(i); + } + } else { + if (spec.count(i) != 0u) { + badPaths.insert(i); + } + } + } + + if (!badPaths.empty()) { + std::string badPathsStr; + for (auto& i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError( + "output '%s' is not allowed to refer to the following paths:%s", + info.path, badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) { + checks.maxSize = maxSize->get<uint64_t>(); + } + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) { + checks.maxClosureSize = maxClosureSize->get<uint64_t>(); + } + + auto get = [&](const std::string& name) -> std::optional<Strings> { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto& j : *i) { + if (!j.is_string()) { + throw Error( + "attribute '%s' of derivation '%s' must be a list of " + "strings", + name, drvPath); + } + res.push_back(j.get<std::string>()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = + parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = + parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + +Path DerivationGoal::openLogFile() { + logSize = 0; + + if (!settings.keepLog) { + return ""; + } + + std::string baseName = baseNameOf(drvPath); + + /* Create a log file. */ + Path dir = fmt("%s/%s/%s/", worker.store.logDir, nix::LocalStore::drvsLogDir, + std::string(baseName, 0, 2)); + createDirs(dir); + + Path logFileName = fmt("%s/%s%s", dir, std::string(baseName, 2), + settings.compressLog ? ".bz2" : ""); + + fdLogFile = AutoCloseFD(open(logFileName.c_str(), + O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666)); + if (!fdLogFile) { + throw SysError(format("creating log file '%1%'") % logFileName); + } + + logFileSink = std::make_shared<FdSink>(fdLogFile.get()); + + if (settings.compressLog) { + logSink = std::shared_ptr<CompressionSink>( + makeCompressionSink("bzip2", *logFileSink)); + } else { + logSink = logFileSink; + } + + return logFileName; +} + +void DerivationGoal::closeLogFile() { + auto logSink2 = std::dynamic_pointer_cast<CompressionSink>(logSink); + if (logSink2) { + logSink2->finish(); + } + if (logFileSink) { + logFileSink->flush(); + } + logSink = logFileSink = nullptr; + fdLogFile = AutoCloseFD(-1); +} + +void DerivationGoal::deleteTmpDir(bool force) { + if (!tmpDir.empty()) { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + log_sink() << "note: keeping build directory '" << tmpDir << "'" + << std::endl; + chmod(tmpDir.c_str(), 0755); + } else { + deletePath(tmpDir); + } + tmpDir = ""; + } +} + +// TODO(tazjin): What ... what does this function ... do? +void DerivationGoal::handleChildOutput(int fd, const std::string& data) { + if ((hook && fd == hook->builderOut.readSide.get()) || + (!hook && fd == builderOut.readSide.get())) { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + log_sink() << getName() << " killed after writing more than " + << settings.maxLogSize << " bytes of log output" << std::endl; + killChild(); + done(BuildResult::LogLimitExceeded); + return; + } + + for (auto c : data) { + if (c == '\r') { + currentLogLinePos = 0; + } else if (c == '\n') { + flushLine(); + } else { + if (currentLogLinePos >= currentLogLine.size()) { + currentLogLine.resize(currentLogLinePos + 1); + } + currentLogLine[currentLogLinePos++] = c; + } + } + + if (logSink) { + (*logSink)(data); + } + } + + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) { + if (c == '\n') { + currentHookLine.clear(); + } else { + currentHookLine += c; + } + } + } +} + +void DerivationGoal::handleEOF(int /* fd */) { + if (!currentLogLine.empty()) { + flushLine(); + } + worker.wakeUp(shared_from_this()); +} + +void DerivationGoal::flushLine() { + if (settings.verboseBuild && + (settings.printRepeatedBuilds || curRound == 1)) { + log_sink() << currentLogLine << std::endl; + } else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) { + logTail.pop_front(); + } + } + + currentLogLine = ""; + currentLogLinePos = 0; +} + +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) { + PathSet result; + for (auto& i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) { + continue; + } + bool good = worker.store.isValidPath(i.second.path) && + (!checkHash || worker.pathContentsGood(i.second.path)); + if (good == returnValid) { + result.insert(i.second.path); + } + } + return result; +} + +Path DerivationGoal::addHashRewrite(const Path& path) { + std::string h1 = std::string(path, worker.store.storeDir.size() + 1, 32); + std::string h2 = + std::string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path) + .to_string(Base32, false), + 0, 32); + Path p = worker.store.storeDir + "/" + h2 + + std::string(path, worker.store.storeDir.size() + 33); + deletePath(p); + assert(path.size() == p.size()); + inputRewrites[h1] = h2; + outputRewrites[h2] = h1; + redirectedOutputs[path] = p; + return p; +} + +void DerivationGoal::done(BuildResult::Status status, const std::string& msg) { + result.status = status; + result.errorMsg = msg; + amDone(result.success() ? ecSuccess : ecFailed); + if (result.status == BuildResult::TimedOut) { + worker.timedOut = true; + } + if (result.status == BuildResult::PermanentFailure) { + worker.permanentFailure = true; + } + + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); + + if (result.success()) { + if (status == BuildResult::Built) { + worker.doneBuilds++; + } + } else { + if (status != BuildResult::DependencyFailed) { + worker.failedBuilds++; + } + } +} + +////////////////////////////////////////////////////////////////////// + +class SubstitutionGoal : public Goal { + friend class Worker; + + private: + /* The store path that should be realised through a substitute. */ + Path storePath; + + /* The remaining substituters. */ + std::list<ref<Store>> subs; + + /* The current substituter. */ + std::shared_ptr<Store> sub; + + /* Whether a substituter failed. */ + bool substituterFailed = false; + + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr<const ValidPathInfo> info; + + /* Pipe for the substituter's standard output. */ + Pipe outPipe; + + /* The substituter thread. */ + std::thread thr; + + std::promise<void> promise; + + /* Whether to try to repair a valid path. */ + RepairFlag repair; + + /* Location where we're downloading the substitute. Differs from + storePath when doing a repair. */ + Path destPath; + + std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions, + maintainRunningSubstitutions, maintainExpectedNar, + maintainExpectedDownload; + + using GoalState = void (SubstitutionGoal::*)(); + GoalState state; + + public: + SubstitutionGoal(Worker& worker, const Path& storePath, + RepairFlag repair = NoRepair); + + ~SubstitutionGoal() override; + + void timedOut() override { abort(); }; + + std::string key() override { + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + storePathToName(storePath) + "$" + storePath; + } + + void work() override; + + /* The states. */ + void init(); + void tryNext(); + void gotInfo(); + void referencesValid(); + void tryToRun(); + void finished(); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const std::string& data) override; + void handleEOF(int fd) override; + + Path getStorePath() { return storePath; } + + void amDone(ExitCode result) override { Goal::amDone(result); } +}; + +SubstitutionGoal::SubstitutionGoal(Worker& worker, const Path& storePath, + RepairFlag repair) + : Goal(worker), repair(repair) { + this->storePath = storePath; + state = &SubstitutionGoal::init; + name = absl::StrCat("substitution of ", storePath); + trace("created"); + maintainExpectedSubstitutions = + std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions); +} + +SubstitutionGoal::~SubstitutionGoal() { + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); + } + } catch (...) { + ignoreException(); + } +} + +void SubstitutionGoal::work() { (this->*state)(); } + +void SubstitutionGoal::init() { + trace("init"); + + worker.store.addTempRoot(storePath); + + /* If the path already exists we're done. */ + if ((repair == 0u) && worker.store.isValidPath(storePath)) { + amDone(ecSuccess); + return; + } + + if (settings.readOnlyMode) { + throw Error( + format( + "cannot substitute path '%1%' - no write access to the Nix store") % + storePath); + } + + subs = settings.useSubstitutes ? getDefaultSubstituters() + : std::list<ref<Store>>(); + + tryNext(); +} + +void SubstitutionGoal::tryNext() { + trace("trying next substituter"); + + if (subs.empty()) { + /* None left. Terminate this goal and let someone else deal + with it. */ + DLOG(WARNING) + << "path '" << storePath + << "' is required, but there is no substituter that can build it"; + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(substituterFailed ? ecFailed : ecNoSubstituters); + + if (substituterFailed) { + worker.failedSubstitutions++; + } + + return; + } + + sub = subs.front(); + subs.pop_front(); + + if (sub->storeDir != worker.store.storeDir) { + tryNext(); + return; + } + + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath&) { + tryNext(); + return; + } catch (SubstituterDisabled&) { + if (settings.tryFallback) { + tryNext(); + return; + } + throw; + } catch (Error& e) { + if (settings.tryFallback) { + log_sink() << e.what() << std::endl; + tryNext(); + return; + } + throw; + } + + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast<const NarInfo>(info); + + maintainExpectedNar = std::make_unique<MaintainCount<uint64_t>>( + worker.expectedNarSize, info->narSize); + + maintainExpectedDownload = + narInfo && (narInfo->fileSize != 0u) + ? std::make_unique<MaintainCount<uint64_t>>( + worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (worker.store.requireSigs && !sub->isTrusted && + (info->checkSignatures(worker.store, worker.store.getPublicKeys()) == + 0u)) { + log_sink() << "substituter '" << sub->getUri() + << "' does not have a valid signature for path '" << storePath + << "'" << std::endl; + tryNext(); + return; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto& i : info->references) { + if (i != storePath) { /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(i)); + } + } + + if (waitees.empty()) { /* to prevent hang (no wake-up event) */ + referencesValid(); + } else { + state = &SubstitutionGoal::referencesValid; + } +} + +void SubstitutionGoal::referencesValid() { + trace("all references realised"); + + if (nrFailed > 0) { + DLOG(WARNING) << "some references of path '" << storePath + << "' could not be realised"; + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure + : ecFailed); + return; + } + + for (auto& i : info->references) { + if (i != storePath) { /* ignore self-references */ + assert(worker.store.isValidPath(i)); + } + } + + state = &SubstitutionGoal::tryToRun; + worker.wakeUp(shared_from_this()); +} + +void SubstitutionGoal::tryToRun() { + trace("trying to run"); + + /* Make sure that we are allowed to start a build. Note that even + if maxBuildJobs == 0 (no local builds allowed), we still allow + a substituter to run. This is because substitutions cannot be + distributed to another machine via the build hook. */ + if (worker.getNrLocalBuilds() >= + std::max(1U, (unsigned int)settings.maxBuildJobs)) { + worker.waitForBuildSlot(shared_from_this()); + return; + } + + maintainRunningSubstitutions = + std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions); + + outPipe.create(); + + promise = std::promise<void>(); + + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide = AutoCloseFD(-1); }); + + copyStorePath(ref<Store>(sub), + ref<Store>(worker.store.shared_from_this()), storePath, + repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, + false); + + state = &SubstitutionGoal::finished; +} + +void SubstitutionGoal::finished() { + trace("substitute finished"); + + thr.join(); + worker.childTerminated(this); + + try { + promise.get_future().get(); + } catch (std::exception& e) { + log_sink() << e.what() << std::endl; + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone&) { + } catch (...) { + substituterFailed = true; + } + + /* Try the next substitute. */ + state = &SubstitutionGoal::tryNext; + worker.wakeUp(shared_from_this()); + return; + } + + worker.markContentsGood(storePath); + + DLOG(INFO) << "substitution of path '" << storePath << "' succeeded"; + + maintainRunningSubstitutions.reset(); + + maintainExpectedSubstitutions.reset(); + worker.doneSubstitutions++; + + if (maintainExpectedDownload) { + auto fileSize = maintainExpectedDownload->delta; + maintainExpectedDownload.reset(); + worker.doneDownloadSize += fileSize; + } + + worker.doneNarSize += maintainExpectedNar->delta; + maintainExpectedNar.reset(); + + amDone(ecSuccess); +} + +void SubstitutionGoal::handleChildOutput(int fd, const std::string& data) {} + +void SubstitutionGoal::handleEOF(int fd) { + if (fd == outPipe.readSide.get()) { + worker.wakeUp(shared_from_this()); + } +} + +////////////////////////////////////////////////////////////////////// + +ABSL_CONST_INIT static thread_local bool working = false; + +Worker::Worker(LocalStore& store, std::ostream& log_sink) + : log_sink_(log_sink), store(store) { + // Debugging: prevent recursive workers. + // TODO(grfn): Do we need this? + CHECK(!working) << "Worker initialized during execution of a worker"; + working = true; + nrLocalBuilds = 0; + lastWokenUp = steady_time_point::min(); + permanentFailure = false; + timedOut = false; + hashMismatch = false; + checkMismatch = false; +} + +Worker::~Worker() { + working = false; + + /* Explicitly get rid of all strong pointers now. After this all + goals that refer to this worker should be gone. (Otherwise we + are in trouble, since goals may call childTerminated() etc. in + their destructors). */ + topGoals.clear(); + + assert(expectedSubstitutions == 0); + assert(expectedDownloadSize == 0); + assert(expectedNarSize == 0); +} + +GoalPtr Worker::makeDerivationGoal(const Path& drv_path, + const StringSet& wantedOutputs, + BuildMode buildMode) { + GoalPtr goal = derivationGoals[drv_path].lock(); + if (!goal) { + goal = std::make_shared<DerivationGoal>(*this, drv_path, wantedOutputs, + buildMode); + derivationGoals[drv_path] = goal; + wakeUp(goal); + } else { + (dynamic_cast<DerivationGoal*>(goal.get())) + ->addWantedOutputs(wantedOutputs); + } + return goal; +} + +std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal( + const Path& drvPath, const BasicDerivation& drv, BuildMode buildMode) { + std::shared_ptr<DerivationGoal> goal = + std::make_shared<DerivationGoal>(*this, drvPath, drv, buildMode); + wakeUp(goal); + return goal; +} + +GoalPtr Worker::makeSubstitutionGoal(const Path& path, RepairFlag repair) { + GoalPtr goal = substitutionGoals[path].lock(); + if (!goal) { + goal = std::make_shared<SubstitutionGoal>(*this, path, repair); + substitutionGoals[path] = goal; + wakeUp(goal); + } + return goal; +} + +static void removeGoal(const GoalPtr& goal, WeakGoalMap& goalMap) { + /* !!! inefficient */ + for (auto i = goalMap.begin(); i != goalMap.end();) { + if (i->second.lock() == goal) { + auto j = i; + ++j; + goalMap.erase(i); + i = j; + } else { + ++i; + } + } +} + +void Worker::removeGoal(const GoalPtr& goal) { + nix::removeGoal(goal, derivationGoals); + nix::removeGoal(goal, substitutionGoals); + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) { + topGoals.clear(); + } + } + + /* Wake up goals waiting for any goal to finish. */ + for (auto& i : waitingForAnyGoal) { + GoalPtr goal = i.lock(); + if (goal) { + wakeUp(goal); + } + } + + waitingForAnyGoal.clear(); +} + +void Worker::wakeUp(const GoalPtr& goal) { + goal->trace("woken up"); + addToWeakGoals(awake, goal); +} + +unsigned Worker::getNrLocalBuilds() { return nrLocalBuilds; } + +void Worker::childStarted(const GoalPtr& goal, const std::set<int>& fds, + bool inBuildSlot, bool respectTimeouts) { + Child child; + child.goal = goal; + child.goal2 = goal.get(); + child.fds = fds; + child.timeStarted = child.lastOutput = steady_time_point::clock::now(); + child.inBuildSlot = inBuildSlot; + child.respectTimeouts = respectTimeouts; + children.emplace_back(child); + if (inBuildSlot) { + nrLocalBuilds++; + } +} + +void Worker::childTerminated(Goal* goal, bool wakeSleepers) { + auto i = + std::find_if(children.begin(), children.end(), + [&](const Child& child) { return child.goal2 == goal; }); + if (i == children.end()) { + return; + } + + if (i->inBuildSlot) { + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + } + + children.erase(i); + + if (wakeSleepers) { + /* Wake up goals waiting for a build slot. */ + for (auto& j : wantingToBuild) { + GoalPtr goal = j.lock(); + if (goal) { + wakeUp(goal); + } + } + + wantingToBuild.clear(); + } +} + +void Worker::waitForBuildSlot(const GoalPtr& goal) { + DLOG(INFO) << "wait for build slot"; + if (getNrLocalBuilds() < settings.maxBuildJobs) { + wakeUp(goal); /* we can do it right away */ + } else { + addToWeakGoals(wantingToBuild, goal); + } +} + +void Worker::waitForAnyGoal(GoalPtr goal) { + DLOG(INFO) << "wait for any goal"; + addToWeakGoals(waitingForAnyGoal, std::move(goal)); +} + +void Worker::waitForAWhile(GoalPtr goal) { + DLOG(INFO) << "wait for a while"; + addToWeakGoals(waitingForAWhile, std::move(goal)); +} + +void Worker::run(const Goals& _topGoals) { + for (auto& i : _topGoals) { + topGoals.insert(i); + } + + DLOG(INFO) << "entered goal loop"; + + while (true) { + checkInterrupt(); + + store.autoGC(false); + + /* Call every wake goal (in the ordering established by + CompareGoalPtrs). */ + while (!awake.empty() && !topGoals.empty()) { + Goals awake2; + for (auto& i : awake) { + GoalPtr goal = i.lock(); + if (goal) { + awake2.insert(goal); + } + } + awake.clear(); + for (auto& goal : awake2) { + checkInterrupt(); + goal->work(); + if (topGoals.empty()) { + break; + } // stuff may have been cancelled + } + } + + if (topGoals.empty()) { + break; + } + + /* Wait for input. */ + if (!children.empty() || !waitingForAWhile.empty()) { + waitForInput(); + } else { + if (awake.empty() && 0 == settings.maxBuildJobs) { + throw Error( + "unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); + } + assert(!awake.empty()); + } + } + + /* If --keep-going is not set, it's possible that the main goal + exited while some of its subgoals were still active. But if + --keep-going *is* set, then they must all be finished now. */ + assert(!settings.keepGoing || awake.empty()); + assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || children.empty()); +} + +void Worker::waitForInput() { + DLOG(INFO) << "waiting for children"; + + /* Process output from the file descriptors attached to the + children, namely log output and output path creation commands. + We also use this to detect child termination: if we get EOF on + the logger pipe of a build, we assume that the builder has + terminated. */ + + bool useTimeout = false; + struct timeval timeout; + timeout.tv_usec = 0; + auto before = steady_time_point::clock::now(); + + /* If we're monitoring for silence on stdout/stderr, or if there + is a build timeout, then wait for input until the first + deadline for any child. */ + auto nearest = steady_time_point::max(); // nearest deadline + if (settings.minFree.get() != 0) { + // Periodicallty wake up to see if we need to run the garbage collector. + nearest = before + std::chrono::seconds(10); + } + for (auto& i : children) { + if (!i.respectTimeouts) { + continue; + } + if (0 != settings.maxSilentTime) { + nearest = std::min( + nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); + } + if (0 != settings.buildTimeout) { + nearest = std::min( + nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); + } + } + if (nearest != steady_time_point::max()) { + timeout.tv_sec = std::max( + 1L, static_cast<long>(std::chrono::duration_cast<std::chrono::seconds>( + nearest - before) + .count())); + useTimeout = true; + } + + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + if (!waitingForAWhile.empty()) { + useTimeout = true; + if (lastWokenUp == steady_time_point::min()) { + DLOG(WARNING) << "waiting for locks or build slots..."; + } + if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) { + lastWokenUp = before; + } + timeout.tv_sec = std::max( + 1L, static_cast<long>(std::chrono::duration_cast<std::chrono::seconds>( + lastWokenUp + + std::chrono::seconds(settings.pollInterval) - + before) + .count())); + } else { + lastWokenUp = steady_time_point::min(); + } + + if (useTimeout) { + DLOG(INFO) << "sleeping " << timeout.tv_sec << " seconds"; + } + + /* Use select() to wait for the input side of any logger pipe to + become `available'. Note that `available' (i.e., non-blocking) + includes EOF. */ + fd_set fds; + FD_ZERO(&fds); + int fdMax = 0; + for (auto& i : children) { + for (auto& j : i.fds) { + if (j >= FD_SETSIZE) { + throw Error("reached FD_SETSIZE limit"); + } + FD_SET(j, &fds); + if (j >= fdMax) { + fdMax = j + 1; + } + } + } + + if (select(fdMax, &fds, nullptr, nullptr, useTimeout ? &timeout : nullptr) == + -1) { + if (errno == EINTR) { + return; + } + throw SysError("waiting for input"); + } + + auto after = steady_time_point::clock::now(); + + /* Process all available file descriptors. FIXME: this is + O(children * fds). */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); + + checkInterrupt(); + + GoalPtr goal = j->goal.lock(); + assert(goal); + + std::set<int> fds2(j->fds); + std::vector<unsigned char> buffer(4096); + for (auto& k : fds2) { + if (FD_ISSET(k, &fds)) { + ssize_t rd = read(k, buffer.data(), buffer.size()); + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { + DLOG(WARNING) << goal->getName() << ": got EOF"; + goal->handleEOF(k); + j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) { + throw SysError("%s: read failed", goal->getName()); + } + } else { + DLOG(INFO) << goal->getName() << ": read " << rd << " bytes"; + std::string data(reinterpret_cast<char*>(buffer.data()), rd); + j->lastOutput = after; + goal->handleChildOutput(k, data); + } + } + } + + if (goal->getExitCode() == Goal::ecBusy && 0 != settings.maxSilentTime && + j->respectTimeouts && + after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) { + log_sink_ << goal->getName() << " timed out after " + << settings.maxSilentTime << " seconds of silence"; + goal->timedOut(); + } + + else if (goal->getExitCode() == Goal::ecBusy && + 0 != settings.buildTimeout && j->respectTimeouts && + after - j->timeStarted >= + std::chrono::seconds(settings.buildTimeout)) { + log_sink_ << goal->getName() << " timed out after " + << settings.buildTimeout << " seconds"; + goal->timedOut(); + } + } + + if (!waitingForAWhile.empty() && + lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { + lastWokenUp = after; + for (auto& i : waitingForAWhile) { + GoalPtr goal = i.lock(); + if (goal) { + wakeUp(goal); + } + } + waitingForAWhile.clear(); + } +} + +unsigned int Worker::exitStatus() { + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) { + mask |= 0x04; // 100 + } + if (timedOut) { + mask |= 0x01; // 101 + } + if (hashMismatch) { + mask |= 0x02; // 102 + } + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask != 0u) { + mask |= 0x60; + } + return mask != 0u ? mask : 1; +} + +bool Worker::pathContentsGood(const Path& path) { + auto i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) { + return i->second; + } + log_sink_ << "checking path '" << path << "'..."; + auto info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) { + res = false; + } else { + HashResult current = hashPath(info->narHash.type, path); + Hash nullHash(htSHA256); + res = info->narHash == nullHash || info->narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) { + log_sink_ << "path '" << path << "' is corrupted or missing!"; + } + return res; +} + +void Worker::markContentsGood(const Path& path) { + pathContentsGoodCache[path] = true; +} + +////////////////////////////////////////////////////////////////////// + +static void primeCache(Store& store, const PathSet& paths) { + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + unsigned long long downloadSize; + unsigned long long narSize; + store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, + narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && + getMachines().empty()) { + throw Error( + "%d derivations need to be built, but neither local builds " + "('--max-jobs') " + "nor remote builds ('--builders') are enabled", + willBuild.size()); + } +} + +absl::Status LocalStore::buildPaths(std::ostream& log_sink, + const PathSet& drvPaths, + BuildMode build_mode) { + Worker worker(*this, log_sink); + + primeCache(*this, drvPaths); + + Goals goals; + for (auto& i : drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); + if (isDerivation(i2.first)) { + goals.insert(worker.makeDerivationGoal(i2.first, i2.second, build_mode)); + } else { + goals.insert(worker.makeSubstitutionGoal( + i, build_mode == bmRepair ? Repair : NoRepair)); + } + } + + worker.run(goals); + + PathSet failed; + for (auto& i : goals) { + if (i->getExitCode() != Goal::ecSuccess) { + auto* i2 = dynamic_cast<DerivationGoal*>(i.get()); + if (i2 != nullptr) { + failed.insert(i2->getDrvPath()); + } else { + failed.insert(dynamic_cast<SubstitutionGoal*>(i.get())->getStorePath()); + } + } + } + + if (!failed.empty()) { + return absl::Status( + absl::StatusCode::kInternal, + absl::StrFormat("build of %s failed (exit code %d)", showPaths(failed), + worker.exitStatus())); + } + return absl::OkStatus(); +} + +BuildResult LocalStore::buildDerivation(std::ostream& log_sink, + const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + Worker worker(*this, log_sink); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + + BuildResult result; + + try { + worker.run(Goals{goal}); + result = goal->getResult(); + } catch (Error& e) { + result.status = BuildResult::MiscFailure; + result.errorMsg = e.msg(); + } + + return result; +} + +void LocalStore::ensurePath(const Path& path) { + /* If the path is already valid, we're done. */ + if (isValidPath(path)) { + return; + } + + primeCache(*this, {path}); + + auto discard_logs = DiscardLogsSink(); + Worker worker(*this, discard_logs); + GoalPtr goal = worker.makeSubstitutionGoal(path); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) { + throw Error(worker.exitStatus(), + "path '%s' does not exist and cannot be created", path); + } +} + +void LocalStore::repairPath(const Path& path) { + auto discard_logs = DiscardLogsSink(); + Worker worker(*this, discard_logs); + GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + auto deriver = queryPathInfo(path)->deriver; + if (!deriver.empty() && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else { + throw Error(worker.exitStatus(), "cannot repair path '%s'", path); + } + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/builtins.hh b/third_party/nix/src/libstore/builtins.hh new file mode 100644 index 000000000000..bc53e78ebc33 --- /dev/null +++ b/third_party/nix/src/libstore/builtins.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "libstore/derivations.hh" + +namespace nix { + +// TODO: make pluggable. +void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData); +void builtinBuildenv(const BasicDerivation& drv); + +} // namespace nix diff --git a/third_party/nix/src/libstore/builtins/buildenv.cc b/third_party/nix/src/libstore/builtins/buildenv.cc new file mode 100644 index 000000000000..433082a0f9de --- /dev/null +++ b/third_party/nix/src/libstore/builtins/buildenv.cc @@ -0,0 +1,240 @@ +#include <algorithm> + +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "libstore/builtins.hh" + +namespace nix { + +typedef std::map<Path, int> Priorities; + +// FIXME: change into local variables. + +static Priorities priorities; + +static unsigned long symlinks; + +/* For each activated package, create symlinks */ +static void createLinks(const Path& srcDir, const Path& dstDir, int priority) { + DirEntries srcFiles; + + try { + srcFiles = readDirectory(srcDir); + } catch (SysError& e) { + if (e.errNo == ENOTDIR) { + LOG(ERROR) << "warning: not including '" << srcDir + << "' in the user environment because it's not a directory"; + return; + } + throw; + } + + for (const auto& ent : srcFiles) { + if (ent.name[0] == '.') { /* not matched by glob */ + continue; + } + auto srcFile = srcDir + "/" + ent.name; + auto dstFile = dstDir + "/" + ent.name; + + struct stat srcSt; + try { + if (stat(srcFile.c_str(), &srcSt) == -1) { + throw SysError("getting status of '%1%'", srcFile); + } + } catch (SysError& e) { + if (e.errNo == ENOENT || e.errNo == ENOTDIR) { + LOG(ERROR) << "warning: skipping dangling symlink '" << dstFile << "'"; + continue; + } + throw; + } + + /* The files below are special-cased to that they don't show up + * in user profiles, either because they are useless, or + * because they would cauase pointless collisions (e.g., each + * Python package brings its own + * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) + */ + if (absl::EndsWith(srcFile, "/propagated-build-inputs") || + absl::EndsWith(srcFile, "/nix-support") || + absl::EndsWith(srcFile, "/perllocal.pod") || + absl::EndsWith(srcFile, "/info/dir") || + absl::EndsWith(srcFile, "/log")) { + continue; + + } else if (S_ISDIR(srcSt.st_mode)) { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISDIR(dstSt.st_mode)) { + createLinks(srcFile, dstFile, priority); + continue; + } else if (S_ISLNK(dstSt.st_mode)) { + auto target = canonPath(dstFile, true); + if (!S_ISDIR(lstat(target).st_mode)) { + throw Error("collision between '%1%' and non-directory '%2%'", + srcFile, target); + } + if (unlink(dstFile.c_str()) == -1) { + throw SysError(format("unlinking '%1%'") % dstFile); + } + if (mkdir(dstFile.c_str(), 0755) == -1) { + throw SysError(format("creating directory '%1%'")); + } + createLinks(target, dstFile, priorities[dstFile]); + createLinks(srcFile, dstFile, priority); + continue; + } + } else if (errno != ENOENT) { + throw SysError(format("getting status of '%1%'") % dstFile); + } + } + + else { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISLNK(dstSt.st_mode)) { + auto prevPriority = priorities[dstFile]; + if (prevPriority == priority) { + throw Error( + "packages '%1%' and '%2%' have the same priority %3%; " + "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "to change the priority of one of the conflicting packages" + " (0 being the highest priority)", + srcFile, readLink(dstFile), priority); + } + if (prevPriority < priority) { + continue; + } + if (unlink(dstFile.c_str()) == -1) { + throw SysError(format("unlinking '%1%'") % dstFile); + } + } else if (S_ISDIR(dstSt.st_mode)) { + throw Error( + "collision between non-directory '%1%' and directory '%2%'", + srcFile, dstFile); + } + } else if (errno != ENOENT) { + throw SysError(format("getting status of '%1%'") % dstFile); + } + } + + createSymlink(srcFile, dstFile); + priorities[dstFile] = priority; + symlinks++; + } +} + +using FileProp = std::set<Path>; + +static FileProp done; +static FileProp postponed = FileProp{}; + +static Path out; + +static void addPkg(const Path& pkgDir, int priority) { + if (done.count(pkgDir)) { + return; + } + done.insert(pkgDir); + createLinks(pkgDir, out, priority); + + try { + for (auto p : absl::StrSplit( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), + absl::ByAnyChar(" \n"), absl::SkipEmpty())) { + auto pkg = std::string(p); + if (!done.count(pkg)) { + postponed.insert(pkg); + } + } + } catch (SysError& e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) { + throw; + } + } +} + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) + : path{path}, active{active}, priority{priority} {} +}; + +using Packages = std::vector<Package>; + +void builtinBuildenv(const BasicDerivation& drv) { + auto getAttr = [&](const std::string& name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) { + throw Error("attribute '%s' missing", name); + } + return i->second; + }; + + out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + Strings derivations = absl::StrSplit( + getAttr("derivations"), absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly + */ + auto active = derivations.front(); + derivations.pop_front(); + auto priority = stoi(derivations.front()); + derivations.pop_front(); + auto outputs = stoi(derivations.front()); + derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); + derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package& a, const Package& b) { + return a.priority < b.priority || + (a.priority == b.priority && a.path < b.path); + }); + for (const auto& pkg : pkgs) { + if (pkg.active) { + addPkg(pkg.path, pkg.priority); + } + } + + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + auto pkgDirs = postponed; + postponed = FileProp{}; + for (const auto& pkgDir : pkgDirs) { + addPkg(pkgDir, priorityCounter++); + } + } + + LOG(INFO) << "created " << symlinks << " symlinks in user environment"; + + createSymlink(getAttr("manifest"), out + "/manifest.nix"); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/builtins/fetchurl.cc b/third_party/nix/src/libstore/builtins/fetchurl.cc new file mode 100644 index 000000000000..961d08142347 --- /dev/null +++ b/third_party/nix/src/libstore/builtins/fetchurl.cc @@ -0,0 +1,93 @@ +#include <absl/strings/match.h> +#include <glog/logging.h> + +#include "libstore/builtins.hh" +#include "libstore/download.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/compression.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation& drv, const std::string& netrcData) { + /* Make the host's netrc data available. Too bad curl requires + this to be stored in a file. It would be nice if we could just + pass a pointer to the data. */ + if (netrcData != "") { + settings.netrcFile = "netrc"; + writeFile(settings.netrcFile, netrcData, 0600); + } + + auto getAttr = [&](const std::string& name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) { + throw Error(format("attribute '%s' missing") % name); + } + return i->second; + }; + + Path storePath = getAttr("out"); + auto mainUrl = getAttr("url"); + bool unpack = get(drv.env, "unpack", "") == "1"; + + /* Note: have to use a fresh downloader here because we're in + a forked process. */ + auto downloader = makeDownloader(); + + auto fetch = [&](const std::string& url) { + auto source = sinkToSource([&](Sink& sink) { + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadRequest request(url); + request.verifyTLS = false; + request.decompress = false; + + auto decompressor = makeDecompressionSink( + unpack && absl::EndsWith(mainUrl, ".xz") ? "xz" : "none", sink); + downloader->download(std::move(request), *decompressor); + decompressor->finish(); + }); + + if (unpack) { + restorePath(storePath, *source); + } else { + writeFile(storePath, *source); + } + + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) { + throw SysError(format("making '%1%' executable") % storePath); + } + } + }; + + /* Try the hashed mirrors first. */ + if (getAttr("outputHashMode") == "flat") { + auto hash_ = Hash::deserialize(getAttr("outputHash"), + parseHashType(getAttr("outputHashAlgo"))); + if (hash_.ok()) { + auto h = *hash_; + for (auto hashedMirror : settings.hashedMirrors.get()) { + try { + if (!absl::EndsWith(hashedMirror, "/")) { + hashedMirror += '/'; + } + fetch(hashedMirror + printHashType(h.type) + "/" + + h.to_string(Base16, false)); + return; + } catch (Error& e) { + LOG(ERROR) << e.what(); + } + } + } else { + LOG(ERROR) << "checking mirrors for '" << mainUrl + << "': " << hash_.status().ToString(); + } + } + + /* Otherwise try the specified URL. */ + fetch(mainUrl); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/crypto.cc b/third_party/nix/src/libstore/crypto.cc new file mode 100644 index 000000000000..0a2795cb0a01 --- /dev/null +++ b/third_party/nix/src/libstore/crypto.cc @@ -0,0 +1,138 @@ +#include "libstore/crypto.hh" + +#include <absl/strings/escaping.h> + +#include "libstore/globals.hh" +#include "libutil/util.hh" + +#if HAVE_SODIUM +#include <sodium.h> +#endif + +namespace nix { + +// TODO(riking): convert to string_view to reduce allocations +static std::pair<std::string, std::string> split(const std::string& s) { + size_t colon = s.find(':'); + if (colon == std::string::npos || colon == 0) { + return {"", ""}; + } + return {std::string(s, 0, colon), std::string(s, colon + 1)}; +} + +Key::Key(const std::string& s) { + auto ss = split(s); + + name = ss.first; + std::string keyb64 = ss.second; + + if (name.empty() || keyb64.empty()) { + throw Error("secret key is corrupt"); + } + + if (!absl::Base64Unescape(keyb64, &key)) { + // TODO(grfn): replace this with StatusOr + throw Error("Invalid Base64"); + } +} + +SecretKey::SecretKey(const std::string& s) : Key(s) { +#if HAVE_SODIUM + if (key.size() != crypto_sign_SECRETKEYBYTES) { + throw Error("secret key is not valid"); + } +#endif +} + +#if !HAVE_SODIUM +[[noreturn]] static void noSodium() { + throw Error( + "Nix was not compiled with libsodium, required for signed binary cache " + "support"); +} +#endif + +std::string SecretKey::signDetached(const std::string& data) const { +#if HAVE_SODIUM + unsigned char sig[crypto_sign_BYTES]; + unsigned long long sigLen; + crypto_sign_detached(sig, &sigLen, (unsigned char*)data.data(), data.size(), + (unsigned char*)key.data()); + return name + ":" + + absl::Base64Escape(std::string(reinterpret_cast<char*>(sig), sigLen)); +#else + noSodium(); +#endif +} + +PublicKey SecretKey::toPublicKey() const { +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + crypto_sign_ed25519_sk_to_pk(pk, (unsigned char*)key.data()); + return PublicKey(name, std::string(reinterpret_cast<char*>(pk), + crypto_sign_PUBLICKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(const std::string& s) : Key(s) { +#if HAVE_SODIUM + if (key.size() != crypto_sign_PUBLICKEYBYTES) { + throw Error("public key is not valid"); + } +#endif +} + +bool verifyDetached(const std::string& data, const std::string& sig, + const PublicKeys& publicKeys) { +#if HAVE_SODIUM + auto ss = split(sig); + + auto key = publicKeys.find(ss.first); + if (key == publicKeys.end()) { + return false; + } + + std::string sig2; + if (!absl::Base64Unescape(ss.second, &sig2)) { + // TODO(grfn): replace this with StatusOr + throw Error("Invalid Base64"); + } + if (sig2.size() != crypto_sign_BYTES) { + throw Error("signature is not valid"); + } + + return crypto_sign_verify_detached( + reinterpret_cast<unsigned char*>(sig2.data()), + (unsigned char*)data.data(), data.size(), + (unsigned char*)key->second.key.data()) == 0; +#else + noSodium(); +#endif +} + +PublicKeys getDefaultPublicKeys() { + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (const auto& s : settings.trustedPublicKeys.get()) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (const auto& secretKeyFile : settings.secretKeyFiles.get()) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError& e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/crypto.hh b/third_party/nix/src/libstore/crypto.hh new file mode 100644 index 000000000000..e282f4f8ef69 --- /dev/null +++ b/third_party/nix/src/libstore/crypto.hh @@ -0,0 +1,49 @@ +#pragma once + +#include <map> + +#include "libutil/types.hh" + +namespace nix { + +struct Key { + std::string name; + std::string key; + + /* Construct Key from a string in the format + ‘<name>:<key-in-base64>’. */ + Key(const std::string& s); + + protected: + Key(const std::string& name, const std::string& key) : name(name), key(key) {} +}; + +struct PublicKey; + +struct SecretKey : Key { + SecretKey(const std::string& s); + + /* Return a detached signature of the given string. */ + std::string signDetached(const std::string& data) const; + + PublicKey toPublicKey() const; +}; + +struct PublicKey : Key { + PublicKey(const std::string& s); + + private: + PublicKey(const std::string& name, const std::string& key) : Key(name, key) {} + friend struct SecretKey; +}; + +typedef std::map<std::string, PublicKey> PublicKeys; + +/* Return true iff ‘sig’ is a correct signature over ‘data’ using one + of the given public keys. */ +bool verifyDetached(const std::string& data, const std::string& sig, + const PublicKeys& publicKeys); + +PublicKeys getDefaultPublicKeys(); + +} // namespace nix diff --git a/third_party/nix/src/libstore/derivations.cc b/third_party/nix/src/libstore/derivations.cc new file mode 100644 index 000000000000..9c344502f386 --- /dev/null +++ b/third_party/nix/src/libstore/derivations.cc @@ -0,0 +1,520 @@ +#include "libstore/derivations.hh" + +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <absl/strings/string_view.h> +#include <glog/logging.h> + +#include "libproto/worker.pb.h" +#include "libstore/fs-accessor.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/istringstream_nocopy.hh" +#include "libutil/util.hh" + +namespace nix { + +// TODO(#statusor): looks like easy absl::Status conversion +void DerivationOutput::parseHashInfo(bool& recursive, Hash& hash) const { + recursive = false; + std::string algo = hashAlgo; + + if (std::string(algo, 0, 2) == "r:") { + recursive = true; + algo = std::string(algo, 2); + } + + HashType hashType = parseHashType(algo); + if (hashType == htUnknown) { + throw Error(format("unknown hash algorithm '%1%'") % algo); + } + + auto hash_ = Hash::deserialize(this->hash, hashType); + hash = Hash::unwrap_throw(hash_); +} + +nix::proto::Derivation_DerivationOutput DerivationOutput::to_proto() const { + nix::proto::Derivation_DerivationOutput result; + result.mutable_path()->set_path(path); + result.set_hash_algo(hashAlgo); + result.set_hash(hash); + return result; +} + +BasicDerivation BasicDerivation::from_proto( + const nix::proto::Derivation* proto_derivation) { + BasicDerivation result; + result.platform = proto_derivation->platform(); + result.builder = proto_derivation->builder().path(); + + for (auto [k, v] : proto_derivation->outputs()) { + result.outputs.emplace(k, v); + } + + result.inputSrcs.insert(proto_derivation->input_sources().paths().begin(), + proto_derivation->input_sources().paths().end()); + + result.args.insert(result.args.end(), proto_derivation->args().begin(), + proto_derivation->args().end()); + + for (auto [k, v] : proto_derivation->env()) { + result.env.emplace(k, v); + } + + return result; +} + +nix::proto::Derivation BasicDerivation::to_proto() const { + nix::proto::Derivation result; + for (const auto& [key, output] : outputs) { + result.mutable_outputs()->insert({key, output.to_proto()}); + } + for (const auto& input_src : inputSrcs) { + *result.mutable_input_sources()->add_paths() = input_src; + } + result.set_platform(platform); + result.mutable_builder()->set_path(builder); + for (const auto& arg : args) { + result.add_args(arg); + } + + for (const auto& [key, value] : env) { + result.mutable_env()->insert({key, value}); + } + + return result; +} + +Path BasicDerivation::findOutput(const std::string& id) const { + auto i = outputs.find(id); + if (i == outputs.end()) { + throw Error(format("derivation has no output '%1%'") % id); + } + return i->second.path; +} + +bool BasicDerivation::isBuiltin() const { + return std::string(builder, 0, 8) == "builtin:"; +} + +Path writeDerivation(const ref<Store>& store, const Derivation& drv, + const std::string& name, RepairFlag repair) { + PathSet references; + references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + for (auto& i : drv.inputDrvs) { + references.insert(i.first); + } + /* Note that the outputs of a derivation are *not* references + (that can be missing (of course) and should not necessarily be + held during a garbage collection). */ + std::string suffix = name + drvExtension; + std::string contents = drv.unparse(); + return settings.readOnlyMode + ? store->computeStorePathForText(suffix, contents, references) + : store->addTextToStore(suffix, contents, references, repair); +} + +/* Read string `s' from stream `str'. */ +static void expect(std::istream& str, const std::string& s) { + char s2[s.size()]; + str.read(s2, s.size()); + if (std::string(s2, s.size()) != s) { + throw FormatError(format("expected string '%1%'") % s); + } +} + +/* Read a C-style string from stream `str'. */ +static std::string parseString(std::istream& str) { + std::string res; + expect(str, "\""); + int c; + while ((c = str.get()) != '"' && c != EOF) { + if (c == '\\') { + c = str.get(); + if (c == 'n') { + res += '\n'; + } else if (c == 'r') { + res += '\r'; + } else if (c == 't') { + res += '\t'; + } else if (c == EOF) { + throw FormatError("unexpected EOF while parsing C-style escape"); + } else { + res += static_cast<char>(c); + } + } else { + res += static_cast<char>(c); + } + } + return res; +} + +static Path parsePath(std::istream& str) { + std::string s = parseString(str); + if (s.empty() || s[0] != '/') { + throw FormatError(format("bad path '%1%' in derivation") % s); + } + return s; +} + +static bool endOfList(std::istream& str) { + if (str.peek() == ',') { + str.get(); + return false; + } + if (str.peek() == ']') { + str.get(); + return true; + } + return false; +} + +static StringSet parseStrings(std::istream& str, bool arePaths) { + StringSet res; + while (!endOfList(str)) { + res.insert(arePaths ? parsePath(str) : parseString(str)); + } + return res; +} + +Derivation parseDerivation(const std::string& s) { + Derivation drv; + istringstream_nocopy str(s); + expect(str, "Derive(["); + + /* Parse the list of outputs. */ + while (!endOfList(str)) { + DerivationOutput out; + expect(str, "("); + std::string id = parseString(str); + expect(str, ","); + out.path = parsePath(str); + expect(str, ","); + out.hashAlgo = parseString(str); + expect(str, ","); + out.hash = parseString(str); + expect(str, ")"); + drv.outputs[id] = out; + } + + /* Parse the list of input derivations. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + Path drvPath = parsePath(str); + expect(str, ",["); + drv.inputDrvs[drvPath] = parseStrings(str, false); + expect(str, ")"); + } + + expect(str, ",["); + drv.inputSrcs = parseStrings(str, true); + expect(str, ","); + drv.platform = parseString(str); + expect(str, ","); + drv.builder = parseString(str); + + /* Parse the builder arguments. */ + expect(str, ",["); + while (!endOfList(str)) { + drv.args.push_back(parseString(str)); + } + + /* Parse the environment variables. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + std::string name = parseString(str); + expect(str, ","); + std::string value = parseString(str); + expect(str, ")"); + drv.env[name] = value; + } + + expect(str, ")"); + return drv; +} + +Derivation readDerivation(const Path& drvPath) { + try { + return parseDerivation(readFile(drvPath)); + } catch (FormatError& e) { + throw Error(format("error parsing derivation '%1%': %2%") % drvPath % + e.msg()); + } +} + +Derivation Store::derivationFromPath(const Path& drvPath) { + assertStorePath(drvPath); + ensurePath(drvPath); + auto accessor = getFSAccessor(); + try { + return parseDerivation(accessor->readFile(drvPath)); + } catch (FormatError& e) { + throw Error(format("error parsing derivation '%1%': %2%") % drvPath % + e.msg()); + } +} + +const char* findChunk(const char* begin) { + while (*begin != 0 && *begin != '\"' && *begin != '\\' && *begin != '\n' && + *begin != '\r' && *begin != '\t') { + begin++; + } + + return begin; +} + +static void printString(std::string& res, const std::string& s) { + res += '"'; + + const char* it = s.c_str(); + while (*it != 0) { + const char* end = findChunk(it); + std::copy(it, end, std::back_inserter(res)); + + it = end; + + switch (*it) { + case '"': + case '\\': + res += "\\"; + res += *it; + break; + case '\n': + res += "\\n"; + break; + case '\r': + res += "\\r"; + break; + case '\t': + res += "\\t"; + break; + default: + continue; + } + + it++; + } + + res += '"'; +} + +template <class ForwardIterator> +static void printStrings(std::string& res, ForwardIterator i, + ForwardIterator j) { + res += '['; + bool first = true; + for (; i != j; ++i) { + if (first) { + first = false; + } else { + res += ','; + } + printString(res, *i); + } + res += ']'; +} + +std::string Derivation::unparse() const { + std::string s; + s.reserve(65536); + s += "Derive(["; + + bool first = true; + for (auto& i : outputs) { + if (first) { + first = false; + } else { + s += ','; + } + s += '('; + printString(s, i.first); + s += ','; + printString(s, i.second.path); + s += ','; + printString(s, i.second.hashAlgo); + s += ','; + printString(s, i.second.hash); + s += ')'; + } + + s += "],["; + first = true; + for (auto& i : inputDrvs) { + if (first) { + first = false; + } else { + s += ','; + } + s += '('; + printString(s, i.first); + s += ','; + printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } + + s += "],"; + printStrings(s, inputSrcs.begin(), inputSrcs.end()); + + s += ','; + printString(s, platform); + s += ','; + printString(s, builder); + s += ','; + printStrings(s, args.begin(), args.end()); + + s += ",["; + first = true; + for (auto& i : env) { + if (first) { + first = false; + } else { + s += ','; + } + s += '('; + printString(s, i.first); + s += ','; + printString(s, i.second); + s += ')'; + } + + s += "])"; + + return s; +} + +bool isDerivation(const std::string& fileName) { + return absl::EndsWith(fileName, drvExtension); +} + +bool BasicDerivation::isFixedOutput() const { + return outputs.size() == 1 && outputs.begin()->first == "out" && + !outputs.begin()->second.hash.empty(); +} + +DrvHashes drvHashes; + +/* Returns the hash of a derivation modulo fixed-output + subderivations. A fixed-output derivation is a derivation with one + output (`out') for which an expected hash and hash algorithm are + specified (using the `outputHash' and `outputHashAlgo' + attributes). We don't want changes to such derivations to + propagate upwards through the dependency graph, changing output + paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it + (after all, (the hash of) the file being downloaded is unchanged). + So the *output paths* should not change. On the other hand, the + *derivation paths* should change to reflect the new dependency + graph. + + That's what this function does: it returns a hash which is just the + hash of the derivation ATerm, except that any input derivation + paths have been replaced by the result of a recursive call to this + function, and that for fixed-output derivations we return a hash of + its output path. */ +Hash hashDerivationModulo(Store& store, Derivation drv) { + /* Return a fixed hash for fixed-output derivations. */ + if (drv.isFixedOutput()) { + auto i = drv.outputs.begin(); + return hashString(htSHA256, "fixed:out:" + i->second.hashAlgo + ":" + + i->second.hash + ":" + i->second.path); + } + + /* For other derivations, replace the inputs paths with recursive + calls to this function.*/ + DerivationInputs inputs2; + for (auto& i : drv.inputDrvs) { + Hash h = drvHashes[i.first]; + if (!h) { + assert(store.isValidPath(i.first)); + Derivation drv2 = readDerivation(store.toRealPath(i.first)); + h = hashDerivationModulo(store, drv2); + drvHashes[i.first] = h; + } + inputs2[h.to_string(Base16, false)] = i.second; + } + drv.inputDrvs = inputs2; + + return hashString(htSHA256, drv.unparse()); +} + +DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path) { + auto pos = path.find('!'); + if (pos == absl::string_view::npos) { + return DrvPathWithOutputs(path, std::set<std::string>()); + } + + return DrvPathWithOutputs( + path.substr(0, pos), + absl::StrSplit(path.substr(pos + 1), absl::ByChar(','), + absl::SkipEmpty())); +} + +Path makeDrvPathWithOutputs(const Path& drvPath, + const std::set<std::string>& outputs) { + return outputs.empty() ? drvPath + : drvPath + "!" + concatStringsSep(",", outputs); +} + +bool wantOutput(const std::string& output, + const std::set<std::string>& wanted) { + return wanted.empty() || wanted.find(output) != wanted.end(); +} + +PathSet BasicDerivation::outputPaths() const { + PathSet paths; + for (auto& i : outputs) { + paths.insert(i.second.path); + } + return paths; +} + +Source& readDerivation(Source& in, Store& store, BasicDerivation& drv) { + drv.outputs.clear(); + auto nr = readNum<size_t>(in); + for (size_t n = 0; n < nr; n++) { + auto name = readString(in); + DerivationOutput o; + in >> o.path >> o.hashAlgo >> o.hash; + store.assertStorePath(o.path); + drv.outputs[name] = o; + } + + drv.inputSrcs = readStorePaths<PathSet>(store, in); + in >> drv.platform >> drv.builder; + drv.args = readStrings<Strings>(in); + + nr = readNum<size_t>(in); + for (size_t n = 0; n < nr; n++) { + auto key = readString(in); + auto value = readString(in); + drv.env[key] = value; + } + + return in; +} + +Sink& operator<<(Sink& out, const BasicDerivation& drv) { + out << drv.outputs.size(); + for (auto& i : drv.outputs) { + out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; + } + out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << drv.env.size(); + for (auto& i : drv.env) { + out << i.first << i.second; + } + return out; +} + +std::string hashPlaceholder(const std::string& outputName) { + // FIXME: memoize? + return "/" + hashString(htSHA256, "nix-output:" + outputName) + .to_string(Base32, false); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/derivations.hh b/third_party/nix/src/libstore/derivations.hh new file mode 100644 index 000000000000..4966b858d3b3 --- /dev/null +++ b/third_party/nix/src/libstore/derivations.hh @@ -0,0 +1,130 @@ +#pragma once + +#include <map> + +#include <absl/container/btree_map.h> + +#include "libproto/worker.pb.h" +#include "libstore/store-api.hh" +#include "libutil/hash.hh" +#include "libutil/types.hh" + +namespace nix { + +/* Extension of derivations in the Nix store. */ +const std::string drvExtension = ".drv"; + +/* Abstract syntax of derivations. */ + +struct DerivationOutput { + Path path; + // TODO(grfn): make these two fields a Hash + std::string hashAlgo; /* hash used for expected hash computation */ + std::string hash; /* expected hash, may be null */ + DerivationOutput() {} + DerivationOutput(Path path, std::string hashAlgo, std::string hash) { + this->path = path; + this->hashAlgo = hashAlgo; + this->hash = hash; + } + + explicit DerivationOutput( + const nix::proto::Derivation_DerivationOutput& proto_derivation_output) + : path(proto_derivation_output.path().path()), + hashAlgo(proto_derivation_output.hash_algo()), + hash(proto_derivation_output.hash()) {} + + void parseHashInfo(bool& recursive, Hash& hash) const; + + [[nodiscard]] nix::proto::Derivation_DerivationOutput to_proto() const; +}; + +// TODO(tazjin): Determine whether this actually needs to be ordered. +using DerivationOutputs = absl::btree_map<std::string, DerivationOutput>; + +/* For inputs that are sub-derivations, we specify exactly which + output IDs we are interested in. */ +using DerivationInputs = absl::btree_map<Path, StringSet>; + +using StringPairs = absl::btree_map<std::string, std::string>; + +struct BasicDerivation { + DerivationOutputs outputs; /* keyed on symbolic IDs */ + PathSet inputSrcs; /* inputs that are sources */ + std::string platform; + Path builder; + Strings args; + StringPairs env; + + BasicDerivation() = default; + + // Convert the given proto derivation to a BasicDerivation + static BasicDerivation from_proto( + const nix::proto::Derivation* proto_derivation); + + [[nodiscard]] nix::proto::Derivation to_proto() const; + + virtual ~BasicDerivation(){}; + + /* Return the path corresponding to the output identifier `id' in + the given derivation. */ + Path findOutput(const std::string& id) const; + + bool isBuiltin() const; + + /* Return true iff this is a fixed-output derivation. */ + bool isFixedOutput() const; + + /* Return the output paths of a derivation. */ + PathSet outputPaths() const; +}; + +struct Derivation : BasicDerivation { + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + + /* Print a derivation. */ + std::string unparse() const; +}; + +class Store; + +/* Write a derivation to the Nix store, and return its path. */ +Path writeDerivation(const ref<Store>& store, const Derivation& drv, + const std::string& name, RepairFlag repair = NoRepair); + +/* Read a derivation from a file. */ +Derivation readDerivation(const Path& drvPath); + +Derivation parseDerivation(const std::string& s); + +/* Check whether a file name ends with the extension for + derivations. */ +bool isDerivation(const std::string& fileName); + +Hash hashDerivationModulo(Store& store, Derivation drv); + +/* Memoisation of hashDerivationModulo(). */ +typedef std::map<Path, Hash> DrvHashes; + +extern DrvHashes drvHashes; // FIXME: global, not thread-safe + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path and + the outputs. */ +using DrvPathWithOutputs = std::pair<std::string, std::set<std::string> >; +DrvPathWithOutputs parseDrvPathWithOutputs(absl::string_view path); + +Path makeDrvPathWithOutputs(const Path& drvPath, + const std::set<std::string>& outputs); + +bool wantOutput(const std::string& output, const std::set<std::string>& wanted); + +struct Source; +struct Sink; + +Source& readDerivation(Source& in, Store& store, BasicDerivation& drv); +Sink& operator<<(Sink& out, const BasicDerivation& drv); + +std::string hashPlaceholder(const std::string& outputName); + +} // namespace nix diff --git a/third_party/nix/src/libstore/download.cc b/third_party/nix/src/libstore/download.cc new file mode 100644 index 000000000000..fd472713e6a7 --- /dev/null +++ b/third_party/nix/src/libstore/download.cc @@ -0,0 +1,1024 @@ +#include "libstore/download.hh" + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/numbers.h> +#include <absl/strings/str_split.h> + +#include "libstore/globals.hh" +#include "libstore/pathlocks.hh" +#include "libstore/s3.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/compression.hh" +#include "libutil/finally.hh" +#include "libutil/hash.hh" +#include "libutil/util.hh" + +#ifdef ENABLE_S3 +#include <aws/core/client/ClientConfiguration.h> +#endif + +#include <algorithm> +#include <cmath> +#include <cstring> +#include <iostream> +#include <queue> +#include <random> +#include <thread> + +#include <curl/curl.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <unistd.h> + +using namespace std::string_literals; + +namespace nix { + +DownloadSettings downloadSettings; + +static GlobalConfig::Register r1(&downloadSettings); + +std::string resolveUri(const std::string& uri) { + if (uri.compare(0, 8, "channel:") == 0) { + return "https://nixos.org/channels/" + std::string(uri, 8) + + "/nixexprs.tar.xz"; + } + return uri; +} + +struct CurlDownloader : public Downloader { + CURLM* curlm = nullptr; + + std::random_device rd; + std::mt19937 mt19937; + + struct DownloadItem : public std::enable_shared_from_this<DownloadItem> { + CurlDownloader& downloader; + DownloadRequest request; + DownloadResult result; + bool done = false; // whether either the success or failure function has + // been called + Callback<DownloadResult> callback; + CURL* req = nullptr; + bool active = + false; // whether the handle has been added to the multi object + std::string status; + + unsigned int attempt = 0; + + /* Don't start this download until the specified time point + has been reached. */ + std::chrono::steady_clock::time_point embargo; + + struct curl_slist* requestHeaders = nullptr; + + std::string encoding; + + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + + DownloadItem(CurlDownloader& downloader, const DownloadRequest& request, + Callback<DownloadResult>&& callback) + : downloader(downloader), + request(request), + callback(std::move(callback)), + finalSink([this](const unsigned char* data, size_t len) { + if (this->request.dataCallback) { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || + httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char*)data, len); + } + } else { + this->result.data->append((char*)data, len); + } + }) { + LOG(INFO) << (request.data ? "uploading '" : "downloading '") + << request.uri << "'"; + + if (!request.expectedETag.empty()) { + requestHeaders = curl_slist_append( + requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + } + if (!request.mimeType.empty()) { + requestHeaders = curl_slist_append( + requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + } + } + + ~DownloadItem() { + if (req != nullptr) { + if (active) { + curl_multi_remove_handle(downloader.curlm, req); + } + curl_easy_cleanup(req); + } + if (requestHeaders != nullptr) { + curl_slist_free_all(requestHeaders); + } + try { + if (!done) { + fail(DownloadError( + Interrupted, + format("download of '%s' was interrupted") % request.uri)); + } + } catch (...) { + ignoreException(); + } + } + + void failEx(const std::exception_ptr& ex) { + assert(!done); + done = true; + callback.rethrow(ex); + } + + template <class T> + void fail(const T& e) { + failEx(std::make_exception_ptr(e)); + } + + LambdaSink finalSink; + std::shared_ptr<CompressionSink> decompressionSink; + + std::exception_ptr writeException; + + size_t writeCallback(void* contents, size_t size, size_t nmemb) { + try { + size_t realSize = size * nmemb; + result.bodySize += realSize; + + if (!decompressionSink) { + decompressionSink = makeDecompressionSink(encoding, finalSink); + } + + (*decompressionSink)(static_cast<unsigned char*>(contents), realSize); + + return realSize; + } catch (...) { + writeException = std::current_exception(); + return 0; + } + } + + static size_t writeCallbackWrapper(void* contents, size_t size, + size_t nmemb, void* userp) { + return (static_cast<DownloadItem*>(userp)) + ->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void* contents, size_t size, size_t nmemb) { + size_t realSize = size * nmemb; + std::string line(static_cast<char*>(contents), realSize); + DLOG(INFO) << "got header for '" << request.uri + << "': " << absl::StripAsciiWhitespace(line); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + result.etag = ""; + std::vector<std::string> ss = + absl::StrSplit(line, absl::ByChar(' '), absl::SkipEmpty()); + status = ss.size() >= 2 ? ss[1] : ""; + result.data = std::make_shared<std::string>(); + result.bodySize = 0; + acceptRanges = false; + encoding = ""; + } else { + auto i = line.find(':'); + if (i != std::string::npos) { + std::string name = absl::AsciiStrToLower( + absl::StripAsciiWhitespace(std::string(line, 0, i))); + if (name == "etag") { + result.etag = absl::StripAsciiWhitespace(std::string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + if (result.etag == request.expectedETag && status == "200") { + DLOG(INFO) + << "shutting down on 200 HTTP response with expected ETag"; + return 0; + } + } else if (name == "content-encoding") { + encoding = absl::StripAsciiWhitespace(std::string(line, i + 1)); + } else if (name == "accept-ranges" && + absl::AsciiStrToLower(absl::StripAsciiWhitespace( + std::string(line, i + 1))) == "bytes") { + acceptRanges = true; + } + } + } + return realSize; + } + + static size_t headerCallbackWrapper(void* contents, size_t size, + size_t nmemb, void* userp) { + return (static_cast<DownloadItem*>(userp)) + ->headerCallback(contents, size, nmemb); + } + + static int debugCallback(CURL* handle, curl_infotype type, char* data, + size_t size, void* userptr) { + if (type == CURLINFO_TEXT) { + DLOG(INFO) << "curl: " + << absl::StripTrailingAsciiWhitespace( + std::string(data, size)); + } + return 0; + } + + size_t readOffset = 0; + size_t readCallback(char* buffer, size_t size, size_t nitems) { + if (readOffset == request.data->length()) { + return 0; + } + auto count = std::min(size * nitems, request.data->length() - readOffset); + assert(count); + memcpy(buffer, request.data->data() + readOffset, count); + readOffset += count; + return count; + } + + static size_t readCallbackWrapper(char* buffer, size_t size, size_t nitems, + void* userp) { + return (static_cast<DownloadItem*>(userp)) + ->readCallback(buffer, size, nitems); + } + + void init() { + if (req == nullptr) { + req = curl_easy_init(); + } + + curl_easy_reset(req); + + // TODO(tazjin): Add an Abseil flag for this + // if (verbosity >= lvlVomit) { + // curl_easy_setopt(req, CURLOPT_VERBOSE, 1); + // curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, + // DownloadItem::debugCallback); + // } + + curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); + curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); + curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(req, CURLOPT_USERAGENT, + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + (downloadSettings.userAgentSuffix != "" + ? " " + downloadSettings.userAgentSuffix.get() + : "")) + .c_str()); +#if LIBCURL_VERSION_NUM >= 0x072b00 + curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); +#endif +#if LIBCURL_VERSION_NUM >= 0x072f00 + if (downloadSettings.enableHttp2) { + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + } else { + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + } +#endif + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, + DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEDATA, this); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, + DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERDATA, this); + + curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + + if (request.head) { + curl_easy_setopt(req, CURLOPT_NOBODY, 1); + } + + if (request.data) { + curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); + curl_easy_setopt(req, CURLOPT_READDATA, this); + curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, + (curl_off_t)request.data->length()); + } + + if (request.verifyTLS) { + if (!settings.caFile.empty()) { + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + } + } else { + curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); + } + + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, + downloadSettings.connectTimeout.get()); + + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, + downloadSettings.stalledDownloadTimeout.get()); + + /* If no file exist in the specified path, curl continues to work + anyway as if netrc support was disabled. */ + curl_easy_setopt(req, CURLOPT_NETRC_FILE, + settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + + if (writtenToSink != 0) { + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + } + + result.data = std::make_shared<std::string>(); + result.bodySize = 0; + } + + void finish(CURLcode code) { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + char* effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr != nullptr) { + result.effectiveUri = effectiveUriCStr; + } + + DLOG(INFO) << "finished " << request.verb() << " of " << request.uri + << "; curl status = " << code + << ", HTTP status = " << httpStatus + << ", body = " << result.bodySize << " bytes"; + + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } + + if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { + code = CURLE_OK; + httpStatus = 304; + } + + if (writeException) { + failEx(writeException); + + } else if (code == CURLE_OK && + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || + httpStatus == 206 || httpStatus == 304 || + httpStatus == 226 /* FTP */ || + httpStatus == 0 /* other protocol */)) { + result.cached = httpStatus == 304; + done = true; + callback(std::move(result)); + } + + else { + // We treat most errors as transient, but won't retry when hopeless + Error err = Transient; + + if (httpStatus == 404 || httpStatus == 410 || + code == CURLE_FILE_COULDNT_READ_FILE) { + // The file is definitely not there + err = NotFound; + } else if (httpStatus == 401 || httpStatus == 403 || + httpStatus == 407) { + // Don't retry on authentication/authorization failures + err = Forbidden; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && + httpStatus != 429) { + // Most 4xx errors are client errors and are probably not worth + // retrying: + // * 408 means the server timed out waiting for us, so we try again + // * 429 means too many requests, so we retry (with a delay) + err = Misc; + } else if (httpStatus == 501 || httpStatus == 505 || + httpStatus == 511) { + // Let's treat most 5xx (server) errors as transient, except for a + // handful: + // * 501 not implemented + // * 505 http version not supported + // * 511 we're behind a captive portal + err = Misc; + } else { + // Don't bother retrying on certain cURL errors either + switch (code) { + case CURLE_FAILED_INIT: + case CURLE_URL_MALFORMAT: + case CURLE_NOT_BUILT_IN: + case CURLE_REMOTE_ACCESS_DENIED: + case CURLE_FILE_COULDNT_READ_FILE: + case CURLE_FUNCTION_NOT_FOUND: + case CURLE_ABORTED_BY_CALLBACK: + case CURLE_BAD_FUNCTION_ARGUMENT: + case CURLE_INTERFACE_FAILED: + case CURLE_UNKNOWN_OPTION: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_TOO_MANY_REDIRECTS: + case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: + err = Misc; + break; + default: // Shut up warnings + break; + } + } + + attempt++; + + auto exc = + code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted + ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", + request.verb(), request.uri)) + : httpStatus != 0 + ? DownloadError( + err, + fmt("unable to %s '%s': HTTP error %d", request.verb(), + request.uri, httpStatus) + + (code == CURLE_OK ? "" + : fmt(" (curl error: %s)", + curl_easy_strerror(code)))) + : DownloadError( + err, fmt("unable to %s '%s': %s (%d)", request.verb(), + request.uri, curl_easy_strerror(code), code)); + + /* If this is a transient error, then maybe retry the + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient && attempt < request.tries && + (!this->request.dataCallback || writtenToSink == 0 || + (acceptRanges && encoding.empty()))) { + int ms = request.baseRetryTimeMs * + std::pow(2.0F, attempt - 1 + + std::uniform_real_distribution<>( + 0.0, 0.5)(downloader.mt19937)); + if (writtenToSink != 0) { + LOG(WARNING) << exc.what() << "; retrying from offset " + << writtenToSink << " in " << ms << "ms"; + } else { + LOG(WARNING) << exc.what() << "; retrying in " << ms << "ms"; + } + embargo = + std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + downloader.enqueueItem(shared_from_this()); + } else { + fail(exc); + } + } + } + }; + + struct State { + struct EmbargoComparator { + bool operator()(const std::shared_ptr<DownloadItem>& i1, + const std::shared_ptr<DownloadItem>& i2) { + return i1->embargo > i2->embargo; + } + }; + bool quit = false; + std::priority_queue<std::shared_ptr<DownloadItem>, + std::vector<std::shared_ptr<DownloadItem>>, + EmbargoComparator> + incoming; + }; + + Sync<State> state_; + + /* We can't use a std::condition_variable to wake up the curl + thread, because it only monitors file descriptors. So use a + pipe instead. */ + Pipe wakeupPipe; + + std::thread workerThread; + + CurlDownloader() : mt19937(rd()) { + static std::once_flag globalInit; + std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); + + curlm = curl_multi_init(); + +#if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0 + curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); +#endif +#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 + curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, + downloadSettings.httpConnections.get()); +#endif + + wakeupPipe.create(); + fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); + + workerThread = std::thread([&]() { workerThreadEntry(); }); + } + + ~CurlDownloader() override { + stopWorkerThread(); + + workerThread.join(); + + if (curlm != nullptr) { + curl_multi_cleanup(curlm); + } + } + + void stopWorkerThread() { + /* Signal the worker thread to exit. */ + { + auto state(state_.lock()); + state->quit = true; + } + writeFull(wakeupPipe.writeSide.get(), " ", false); + } + + void workerThreadMain() { + /* Cause this thread to be notified on SIGINT. */ + auto callback = createInterruptCallback([&]() { stopWorkerThread(); }); + + std::map<CURL*, std::shared_ptr<DownloadItem>> items; + + bool quit = false; + + std::chrono::steady_clock::time_point nextWakeup; + + while (!quit) { + checkInterrupt(); + + /* Let curl do its thing. */ + int running; + CURLMcode mc = curl_multi_perform(curlm, &running); + if (mc != CURLM_OK) { + throw nix::Error( + format("unexpected error from curl_multi_perform(): %s") % + curl_multi_strerror(mc)); + } + + /* Set the promises of any finished requests. */ + CURLMsg* msg; + int left; + while ((msg = curl_multi_info_read(curlm, &left)) != nullptr) { + if (msg->msg == CURLMSG_DONE) { + auto i = items.find(msg->easy_handle); + assert(i != items.end()); + i->second->finish(msg->data.result); + curl_multi_remove_handle(curlm, i->second->req); + i->second->active = false; + items.erase(i); + } + } + + /* Wait for activity, including wakeup events. */ + int numfds = 0; + struct curl_waitfd extraFDs[1]; + extraFDs[0].fd = wakeupPipe.readSide.get(); + extraFDs[0].events = CURL_WAIT_POLLIN; + extraFDs[0].revents = 0; + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = + nextWakeup != std::chrono::steady_clock::time_point() + ? std::max( + 0, + static_cast<int>( + std::chrono::duration_cast<std::chrono::milliseconds>( + nextWakeup - std::chrono::steady_clock::now()) + .count())) + : maxSleepTimeMs; + VLOG(2) << "download thread waiting for " << sleepTimeMs << " ms"; + mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); + if (mc != CURLM_OK) { + throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % + curl_multi_strerror(mc)); + } + + nextWakeup = std::chrono::steady_clock::time_point(); + + /* Add new curl requests from the incoming requests queue, + except for requests that are embargoed (waiting for a + retry timeout to expire). */ + if ((extraFDs[0].revents & CURL_WAIT_POLLIN) != 0) { + char buf[1024]; + auto res = read(extraFDs[0].fd, buf, sizeof(buf)); + if (res == -1 && errno != EINTR) { + throw SysError("reading curl wakeup socket"); + } + } + + std::vector<std::shared_ptr<DownloadItem>> incoming; + auto now = std::chrono::steady_clock::now(); + + { + auto state(state_.lock()); + while (!state->incoming.empty()) { + auto item = state->incoming.top(); + if (item->embargo <= now) { + incoming.push_back(item); + state->incoming.pop(); + } else { + if (nextWakeup == std::chrono::steady_clock::time_point() || + item->embargo < nextWakeup) { + nextWakeup = item->embargo; + } + break; + } + } + quit = state->quit; + } + + for (auto& item : incoming) { + DLOG(INFO) << "starting " << item->request.verb() << " of " + << item->request.uri; + item->init(); + curl_multi_add_handle(curlm, item->req); + item->active = true; + items[item->req] = item; + } + } + + DLOG(INFO) << "download thread shutting down"; + } + + void workerThreadEntry() { + try { + workerThreadMain(); + } catch (nix::Interrupted& e) { + } catch (std::exception& e) { + LOG(ERROR) << "unexpected error in download thread: " << e.what(); + } + + { + auto state(state_.lock()); + while (!state->incoming.empty()) { + state->incoming.pop(); + } + state->quit = true; + } + } + + void enqueueItem(const std::shared_ptr<DownloadItem>& item) { + if (item->request.data && !absl::StartsWith(item->request.uri, "http://") && + !absl::StartsWith(item->request.uri, "https://")) { + throw nix::Error("uploading to '%s' is not supported", item->request.uri); + } + + { + auto state(state_.lock()); + if (state->quit) { + throw nix::Error( + "cannot enqueue download request because the download thread is " + "shutting down"); + } + state->incoming.push(item); + } + writeFull(wakeupPipe.writeSide.get(), " "); + } + +#ifdef ENABLE_S3 + std::tuple<std::string, std::string, Store::Params> parseS3Uri( + std::string uri) { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) { + throw nix::Error("bad S3 URI '%s'", path); + } + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + + void enqueueDownload(const DownloadRequest& request, + Callback<DownloadResult> callback) override { + /* Ugly hack to support s3:// URIs. */ + if (absl::StartsWith(request.uri, "s3://")) { + // FIXME: do this on a worker thread + try { +#ifdef ENABLE_S3 + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + + // FIXME: implement ETag + auto s3Res = s3Helper.getObject(bucketName, key); + DownloadResult res; + if (!s3Res.data) + throw DownloadError( + NotFound, fmt("S3 object '%s' does not exist", request.uri)); + res.data = s3Res.data; + callback(std::move(res)); +#else + throw nix::Error( + "cannot download '%s' because Nix is not built with S3 support", + request.uri); +#endif + } catch (...) { + callback.rethrow(); + } + return; + } + + enqueueItem( + std::make_shared<DownloadItem>(*this, request, std::move(callback))); + } +}; + +ref<Downloader> getDownloader() { + static ref<Downloader> downloader = makeDownloader(); + return downloader; +} + +ref<Downloader> makeDownloader() { return make_ref<CurlDownloader>(); } + +std::future<DownloadResult> Downloader::enqueueDownload( + const DownloadRequest& request) { + auto promise = std::make_shared<std::promise<DownloadResult>>(); + enqueueDownload( + request, + Callback<DownloadResult>([promise](std::future<DownloadResult> fut) { + try { + promise->set_value(fut.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + })); + return promise->get_future(); +} + +DownloadResult Downloader::download(const DownloadRequest& request) { + return enqueueDownload(request).get(); +} + +void Downloader::download(DownloadRequest&& request, Sink& sink) { + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the downloader + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + }; + + auto _state = std::make_shared<Sync<State>>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + request.dataCallback = [_state](char* buf, size_t len) { + auto state(_state->lock()); + + if (state->quit) { + return; + } + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { + DLOG(INFO) << "download buffer is full; going to sleep"; + state.wait_for(state->request, std::chrono::seconds(10)); + } + + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(buf, len); + state->avail.notify_one(); + }; + + enqueueDownload(request, Callback<DownloadResult>( + [_state](std::future<DownloadResult> fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + })); + + while (true) { + checkInterrupt(); + + std::string chunk; + + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + while (state->data.empty()) { + if (state->quit) { + if (state->exc) { + std::rethrow_exception(state->exc); + } + return; + } + + state.wait(state->avail); + } + + chunk = std::move(state->data); + state->data = std::string(); + + state->request.notify_one(); + } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink(reinterpret_cast<unsigned char*>(chunk.data()), chunk.size()); + } +} + +CachedDownloadResult Downloader::downloadCached( + const ref<Store>& store, const CachedDownloadRequest& request) { + auto url = resolveUri(request.uri); + + auto name = request.name; + if (name.empty()) { + auto p = url.rfind('/'); + if (p != std::string::npos) { + name = std::string(url, p + 1); + } + } + + Path expectedStorePath; + if (request.expectedHash) { + expectedStorePath = + store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + return result; + } + } + + Path cacheDir = getCacheDir() + "/nix/tarballs"; + createDirs(cacheDir); + + std::string urlHash = hashString(htSHA256, name + std::string("\0"s) + url) + .to_string(Base32, false); + + Path dataFile = cacheDir + "/" + urlHash + ".info"; + Path fileLink = cacheDir + "/" + urlHash + "-file"; + + PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); + + Path storePath; + + std::string expectedETag; + + bool skip = false; + + CachedDownloadResult result; + + if (pathExists(fileLink) && pathExists(dataFile)) { + storePath = readLink(fileLink); + store->addTempRoot(storePath); + if (store->isValidPath(storePath)) { + std::vector<std::string> ss = absl::StrSplit( + readFile(dataFile), absl::ByChar('\n'), absl::SkipEmpty()); + if (ss.size() >= 3 && ss[0] == url) { + time_t lastChecked; + if (absl::SimpleAtoi(ss[2], &lastChecked) && + static_cast<uint64_t>(lastChecked) + request.ttl >= + static_cast<uint64_t>(time(nullptr))) { + skip = true; + result.effectiveUri = request.uri; + result.etag = ss[1]; + } else if (!ss[1].empty()) { + DLOG(INFO) << "verifying previous ETag: " << ss[1]; + expectedETag = ss[1]; + } + } + } else { + storePath = ""; + } + } + + if (!skip) { + try { + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; + + if (!res.cached) { + ValidPathInfo info; + StringSink sink; + dumpString(*res.data, sink); + Hash hash = hashString( + request.expectedHash ? request.expectedHash.type : htSHA256, + *res.data); + info.path = store->makeFixedOutputPath(false, hash, name); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = info.path; + } + + assert(!storePath.empty()); + replaceSymlink(storePath, fileLink); + + writeFile(dataFile, url + "\n" + res.etag + "\n" + + std::to_string(time(nullptr)) + "\n"); + } catch (DownloadError& e) { + if (storePath.empty()) { + throw; + } + LOG(WARNING) << e.msg() << "; using cached result"; + result.etag = expectedETag; + } + } + + if (request.unpack) { + Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + PathLocks lock2({unpackedLink}, + fmt("waiting for lock on '%1%'...", unpackedLink)); + Path unpackedStorePath; + if (pathExists(unpackedLink)) { + unpackedStorePath = readLink(unpackedLink); + store->addTempRoot(unpackedStorePath); + if (!store->isValidPath(unpackedStorePath)) { + unpackedStorePath = ""; + } + } + if (unpackedStorePath.empty()) { + LOG(INFO) << "unpacking '" << url << "' ..."; + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, + {"xf", store->toRealPath(storePath), "-C", tmpDir, + "--strip-components", "1"}); + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, + defaultPathFilter, NoRepair); + } + replaceSymlink(unpackedStorePath, unpackedLink); + storePath = unpackedStorePath; + } + + if (!expectedStorePath.empty() && storePath != expectedStorePath) { + unsigned int statusCode = 102; + Hash gotHash = + request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)) + .first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + throw nix::Error(statusCode, + "hash mismatch in file downloaded from '%s':\n wanted: " + "%s\n got: %s", + url, request.expectedHash.to_string(), + gotHash.to_string()); + } + + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; +} + +bool isUri(const std::string& s) { + if (s.compare(0, 8, "channel:") == 0) { + return true; + } + size_t pos = s.find("://"); + if (pos == std::string::npos) { + return false; + } + std::string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || + scheme == "channel" || scheme == "git" || scheme == "s3" || + scheme == "ssh"; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/download.hh b/third_party/nix/src/libstore/download.hh new file mode 100644 index 000000000000..cbfab5f40d22 --- /dev/null +++ b/third_party/nix/src/libstore/download.hh @@ -0,0 +1,133 @@ +#pragma once + +#include <future> +#include <string> + +#include "libstore/globals.hh" +#include "libutil/hash.hh" +#include "libutil/types.hh" + +namespace nix { + +struct DownloadSettings : Config { + Setting<bool> enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting<std::string> userAgentSuffix{ + this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting<size_t> httpConnections{this, + 25, + "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting<unsigned long> connectTimeout{ + this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's " + "builtin default."}; + + Setting<unsigned long> stalledDownloadTimeout{ + this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. " + "Nix cancels idle downloads after this timeout's duration."}; + + Setting<unsigned int> tries{ + this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + +struct DownloadRequest { + std::string uri; + std::string expectedETag; + bool verifyTLS = true; + bool head = false; + size_t tries = downloadSettings.tries; + unsigned int baseRetryTimeMs = 250; + bool decompress = true; + std::shared_ptr<std::string> data; + std::string mimeType; + std::function<void(char*, size_t)> dataCallback; + + DownloadRequest(const std::string& uri) : uri(uri) {} + + std::string verb() { return data ? "upload" : "download"; } +}; + +struct DownloadResult { + bool cached = false; + std::string etag; + std::string effectiveUri; + std::shared_ptr<std::string> data; + uint64_t bodySize = 0; +}; + +struct CachedDownloadRequest { + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string& uri) : uri(uri) {} +}; + +struct CachedDownloadResult { + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional<std::string> etag; + std::string effectiveUri; +}; + +class Store; + +struct Downloader { + virtual ~Downloader() {} + + /* Enqueue a download request, returning a future to the result of + the download. The future may throw a DownloadError + exception. */ + virtual void enqueueDownload(const DownloadRequest& request, + Callback<DownloadResult> callback) = 0; + + std::future<DownloadResult> enqueueDownload(const DownloadRequest& request); + + /* Synchronously download a file. */ + DownloadResult download(const DownloadRequest& request); + + /* Download a file, writing its data to a sink. The sink will be + invoked on the thread of the caller. */ + void download(DownloadRequest&& request, Sink& sink); + + /* Check if the specified file is already in ~/.cache/nix/tarballs + and is more recent than ‘tarball-ttl’ seconds. Otherwise, + use the recorded ETag to verify if the server has a more + recent version, and if so, download it to the Nix store. */ + CachedDownloadResult downloadCached(const ref<Store>& store, + const CachedDownloadRequest& request); + + enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; +}; + +/* Return a shared Downloader object. Using this object is preferred + because it enables connection reuse and HTTP/2 multiplexing. */ +ref<Downloader> getDownloader(); + +/* Return a new Downloader object. */ +ref<Downloader> makeDownloader(); + +class DownloadError : public Error { + public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString& fs) + : Error(fs), error(error) {} +}; + +bool isUri(const std::string& s); + +} // namespace nix diff --git a/third_party/nix/src/libstore/export-import.cc b/third_party/nix/src/libstore/export-import.cc new file mode 100644 index 000000000000..8e9314433990 --- /dev/null +++ b/third_party/nix/src/libstore/export-import.cc @@ -0,0 +1,111 @@ +#include <algorithm> + +#include "libstore/store-api.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/archive.hh" + +namespace nix { + +struct HashAndWriteSink : Sink { + Sink& writeSink; + HashSink hashSink; + explicit HashAndWriteSink(Sink& writeSink) + : writeSink(writeSink), hashSink(htSHA256) {} + void operator()(const unsigned char* data, size_t len) override { + writeSink(data, len); + hashSink(data, len); + } + Hash currentHash() { return hashSink.currentHash().first; } +}; + +void Store::exportPaths(const Paths& paths, Sink& sink) { + Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + std::reverse(sorted.begin(), sorted.end()); + + std::string doneLabel("paths exported"); + // logger->incExpected(doneLabel, sorted.size()); + + for (auto& path : sorted) { + // Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + sink << 1; + exportPath(path, sink); + // logger->incProgress(doneLabel); + } + + sink << 0; +} + +void Store::exportPath(const Path& path, Sink& sink) { + auto info = queryPathInfo(path); + + HashAndWriteSink hashAndWriteSink(sink); + + narFromPath(path, hashAndWriteSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashAndWriteSink.currentHash(); + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) { + throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % + path % info->narHash.to_string() % hash.to_string()); + } + + hashAndWriteSink << exportMagic << path << info->references << info->deriver + << 0; +} + +Paths Store::importPaths(Source& source, + const std::shared_ptr<FSAccessor>& accessor, + CheckSigsFlag checkSigs) { + Paths res; + while (true) { + auto n = readNum<uint64_t>(source); + if (n == 0) { + break; + } + if (n != 1) { + throw Error( + "input doesn't look like something created by 'nix-store --export'"); + } + + /* Extract the NAR from the source. */ + TeeSink tee(source); + parseDump(tee, tee.source); + + uint32_t magic = readInt(source); + if (magic != exportMagic) { + throw Error("Nix archive cannot be imported; wrong format"); + } + + ValidPathInfo info; + + info.path = readStorePath(*this, source); + + // Activity act(*logger, lvlInfo, format("importing path '%s'") % + // info.path); + + info.references = readStorePaths<PathSet>(*this, source); + + info.deriver = readString(source); + if (!info.deriver.empty()) { + assertStorePath(info.deriver); + } + + info.narHash = hashString(htSHA256, *tee.source.data); + info.narSize = tee.source.data->size(); + + // Ignore optional legacy signature. + if (readInt(source) == 1) { + readString(source); + } + + addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); + + res.push_back(info.path); + } + + return res; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/fs-accessor.hh b/third_party/nix/src/libstore/fs-accessor.hh new file mode 100644 index 000000000000..1bc1373dcb2f --- /dev/null +++ b/third_party/nix/src/libstore/fs-accessor.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "libutil/types.hh" + +namespace nix { + +/* An abstract class for accessing a filesystem-like structure, such + as a (possibly remote) Nix store or the contents of a NAR file. */ +class FSAccessor { + public: + enum Type { tMissing, tRegular, tSymlink, tDirectory }; + + struct Stat { + Type type = tMissing; + uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + uint64_t narOffset = 0; // regular files only + }; + + virtual ~FSAccessor() {} + + virtual Stat stat(const Path& path) = 0; + + virtual StringSet readDirectory(const Path& path) = 0; + + virtual std::string readFile(const Path& path) = 0; + + virtual std::string readLink(const Path& path) = 0; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/gc.cc b/third_party/nix/src/libstore/gc.cc new file mode 100644 index 000000000000..07dc10629a91 --- /dev/null +++ b/third_party/nix/src/libstore/gc.cc @@ -0,0 +1,997 @@ +#include <algorithm> +#include <cerrno> +#include <climits> +#include <functional> +#include <queue> +#include <random> +#include <regex> + +#include <absl/strings/match.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libutil/finally.hh" + +namespace nix { + +constexpr std::string_view kGcLockName = "gc.lock"; +constexpr std::string_view kGcRootsDir = "gcroots"; + +/* Acquire the global GC lock. This is used to prevent new Nix + processes from starting after the temporary root files have been + read. To be precise: when they try to create a new temporary root + file, they will block until the garbage collector has finished / + yielded the GC lock. */ +AutoCloseFD LocalStore::openGCLock(LockType lockType) { + Path fnGCLock = absl::StrCat(stateDir.get(), "/", kGcLockName); + + DLOG(INFO) << "acquiring global GC lock " << fnGCLock; + + AutoCloseFD fdGCLock( + open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600)); + + if (!fdGCLock) { + throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + } + + if (!lockFile(fdGCLock.get(), lockType, false)) { + LOG(ERROR) << "waiting for the big garbage collector lock..."; + lockFile(fdGCLock.get(), lockType, true); + } + + /* !!! Restrict read permission on the GC root. Otherwise any + process that can open the file for reading can DoS the + collector. */ + + return fdGCLock; +} + +static void makeSymlink(const Path& link, const Path& target) { + /* Create directories up to `gcRoot'. */ + createDirs(dirOf(link)); + + /* Create the new symlink. */ + Path tempLink = + (format("%1%.tmp-%2%-%3%") % link % getpid() % random()).str(); + createSymlink(target, tempLink); + + /* Atomically replace the old one. */ + if (rename(tempLink.c_str(), link.c_str()) == -1) { + throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % link); + } +} + +void LocalStore::syncWithGC() { AutoCloseFD fdGCLock = openGCLock(ltRead); } + +void LocalStore::addIndirectRoot(const Path& path) { + std::string hash = hashString(htSHA1, path).to_string(Base32, false); + Path realRoot = + canonPath(absl::StrCat(stateDir.get(), "/", kGcRootsDir, "/auto/", hash)); + makeSymlink(realRoot, path); +} + +Path LocalFSStore::addPermRoot(const Path& _storePath, const Path& _gcRoot, + bool indirect, bool allowOutsideRootsDir) { + Path storePath(canonPath(_storePath)); + Path gcRoot(canonPath(_gcRoot)); + assertStorePath(storePath); + + if (isInStore(gcRoot)) { + throw Error(format("creating a garbage collector root (%1%) in the Nix " + "store is forbidden " + "(are you running nix-build inside the store?)") % + gcRoot); + } + + if (indirect) { + /* Don't clobber the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && + (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) { + throw Error(format("cannot create symlink '%1%'; already exists") % + gcRoot); + } + makeSymlink(gcRoot, storePath); + addIndirectRoot(gcRoot); + } + + else { + if (!allowOutsideRootsDir) { + Path rootsDir = canonPath(absl::StrCat(stateDir.get(), "/", kGcRootsDir)); + + if (std::string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") { + throw Error(format("path '%1%' is not a valid garbage collector root; " + "it's not in the directory '%2%'") % + gcRoot % rootsDir); + } + } + + if (baseNameOf(gcRoot) == baseNameOf(storePath)) { + writeFile(gcRoot, ""); + } else { + makeSymlink(gcRoot, storePath); + } + } + + /* Check that the root can be found by the garbage collector. + !!! This can be very slow on machines that have many roots. + Instead of reading all the roots, it would be more efficient to + check if the root is in a directory in or linked from the + gcroots directory. */ + if (settings.checkRootReachability) { + Roots roots = findRoots(false); + if (roots[storePath].count(gcRoot) == 0) { + LOG(ERROR) << "warning: '" << gcRoot + << "' is not in a directory where the garbage " + << "collector looks for roots; therefore, '" << storePath + << "' might be removed by the garbage collector"; + } + } + + /* Grab the global GC root, causing us to block while a GC is in + progress. This prevents the set of permanent roots from + increasing while a GC is in progress. */ + syncWithGC(); + + return gcRoot; +} + +void LocalStore::addTempRoot(const Path& path) { + auto state(_state.lock()); + + /* Create the temporary roots file for this process. */ + if (!state->fdTempRoots) { + while (true) { + AutoCloseFD fdGCLock = openGCLock(ltRead); + + if (pathExists(fnTempRoots)) { + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); + } + + state->fdTempRoots = openLockFile(fnTempRoots, true); + + fdGCLock = AutoCloseFD(-1); + + DLOG(INFO) << "acquiring read lock on " << fnTempRoots; + lockFile(state->fdTempRoots.get(), ltRead, true); + + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(state->fdTempRoots.get(), &st) == -1) { + throw SysError(format("statting '%1%'") % fnTempRoots); + } + if (st.st_size == 0) { + break; + } + + /* The garbage collector deleted this file before we could + get a lock. (It won't delete the file after we get a + lock.) Try again. */ + } + } + + /* Upgrade the lock to a write lock. This will cause us to block + if the garbage collector is holding our lock. */ + DLOG(INFO) << "acquiring write lock on " << fnTempRoots; + lockFile(state->fdTempRoots.get(), ltWrite, true); + + std::string s = path + '\0'; + writeFull(state->fdTempRoots.get(), s); + + /* Downgrade to a read lock. */ + DLOG(INFO) << "downgrading to read lock on " << fnTempRoots; + lockFile(state->fdTempRoots.get(), ltRead, true); +} + +constexpr std::string_view kCensored = "{censored}"; + +void LocalStore::findTempRoots(FDs& fds, Roots& tempRoots, bool censor) { + /* Read the `temproots' directory for per-process temporary root + files. */ + for (auto& i : readDirectory(tempRootsDir)) { + Path path = tempRootsDir + "/" + i.name; + + pid_t pid = std::stoi(i.name); + + DLOG(INFO) << "reading temporary root file " << path; + FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); + if (!*fd) { + /* It's okay if the file has disappeared. */ + if (errno == ENOENT) { + continue; + } + throw SysError(format("opening temporary roots file '%1%'") % path); + } + + /* This should work, but doesn't, for some reason. */ + // FDPtr fd(new AutoCloseFD(openLockFile(path, false))); + // if (*fd == -1) { continue; } + + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + LOG(ERROR) << "removing stale temporary roots file " << path; + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; + } + + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + DLOG(INFO) << "waiting for read lock on " << path; + lockFile(fd->get(), ltRead, true); + + /* Read the entire file. */ + std::string contents = readFile(fd->get()); + + /* Extract the roots. */ + std::string::size_type pos = 0; + std::string::size_type end; + + while ((end = contents.find(static_cast<char>(0), pos)) != + std::string::npos) { + Path root(contents, pos, end - pos); + DLOG(INFO) << "got temporary root " << root; + assertStorePath(root); + tempRoots[root].emplace(censor ? kCensored : fmt("{temp:%d}", pid)); + pos = end + 1; + } + + fds.push_back(fd); /* keep open */ + } +} + +void LocalStore::findRoots(const Path& path, unsigned char type, Roots& roots) { + auto foundRoot = [&](const Path& path, const Path& target) { + Path storePath = toStorePath(target); + if (isStorePath(storePath) && isValidPath(storePath)) { + roots[storePath].emplace(path); + } else { + LOG(INFO) << "skipping invalid root from '" << path << "' to '" + << storePath << "'"; + } + }; + + try { + if (type == DT_UNKNOWN) { + type = getFileType(path); + } + + if (type == DT_DIR) { + for (auto& i : readDirectory(path)) { + findRoots(path + "/" + i.name, i.type, roots); + } + } + + else if (type == DT_LNK) { + Path target = readLink(path); + if (isInStore(target)) { + foundRoot(path, target); + } + + /* Handle indirect roots. */ + else { + target = absPath(target, dirOf(path)); + if (!pathExists(target)) { + if (isInDir(path, absl::StrCat(stateDir.get(), "/", kGcRootsDir, + "/auto"))) { + LOG(INFO) << "removing stale link from '" << path << "' to '" + << target << "'"; + unlink(path.c_str()); + } + } else { + struct stat st2 = lstat(target); + if (!S_ISLNK(st2.st_mode)) { + return; + } + Path target2 = readLink(target); + if (isInStore(target2)) { + foundRoot(target, target2); + } + } + } + } + + else if (type == DT_REG) { + Path storePath = storeDir + "/" + baseNameOf(path); + if (isStorePath(storePath) && isValidPath(storePath)) { + roots[storePath].emplace(path); + } + } + + } + + catch (SysError& e) { + /* We only ignore permanent failures. */ + if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) { + LOG(INFO) << "cannot read potential root '" << path << "'"; + } else { + throw; + } + } +} + +void LocalStore::findRootsNoTemp(Roots& roots, bool censor) { + /* Process direct roots in {gcroots,profiles}. */ + findRoots(absl::StrCat(stateDir.get(), "/", kGcRootsDir), DT_UNKNOWN, roots); + findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); + + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ + findRuntimeRoots(roots, censor); +} + +Roots LocalStore::findRoots(bool censor) { + Roots roots; + findRootsNoTemp(roots, censor); + + FDs fds; + findTempRoots(fds, roots, censor); + + return roots; +} + +static void readProcLink(const std::string& file, Roots& roots) { + /* 64 is the starting buffer size gnu readlink uses... */ + auto bufsiz = ssize_t{64}; +try_again: + char buf[bufsiz]; + auto res = readlink(file.c_str(), buf, bufsiz); + if (res == -1) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) { + return; + } + throw SysError("reading symlink"); + } + if (res == bufsiz) { + if (SSIZE_MAX / 2 < bufsiz) { + throw Error("stupidly long symlink"); + } + bufsiz *= 2; + goto try_again; + } + if (res > 0 && buf[0] == '/') { + roots[std::string(static_cast<char*>(buf), res)].emplace(file); + } +} + +static std::string quoteRegexChars(const std::string& raw) { + static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); + return std::regex_replace(raw, specialRegex, R"(\$&)"); +} + +static void readFileRoots(const char* path, Roots& roots) { + try { + roots[readFile(path)].emplace(path); + } catch (SysError& e) { + if (e.errNo != ENOENT && e.errNo != EACCES) { + throw; + } + } +} + +void LocalStore::findRuntimeRoots(Roots& roots, bool censor) { + Roots unchecked; + + auto procDir = AutoCloseDir{opendir("/proc")}; + if (procDir) { + struct dirent* ent; + auto digitsRegex = std::regex(R"(^\d+$)"); + auto mapRegex = + std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); + auto storePathRegex = std::regex(quoteRegexChars(storeDir) + + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + while (errno = 0, ent = readdir(procDir.get())) { + checkInterrupt(); + if (std::regex_match(ent->d_name, digitsRegex)) { + readProcLink(fmt("/proc/%s/exe", ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) { + continue; + } + throw SysError(format("opening %1%") % fdStr); + } + struct dirent* fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') { + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + } + if (errno) { + if (errno == ESRCH) { + continue; + } + throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + } + fdDir.reset(); + + try { + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + std::vector<std::string> mapLines = absl::StrSplit( + readFile(mapFile, true), absl::ByChar('\n'), absl::SkipEmpty()); + for (const auto& line : mapLines) { + auto match = std::smatch{}; + if (std::regex_match(line, match, mapRegex)) { + unchecked[match[1]].emplace(mapFile); + } + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile, true); + auto env_end = std::sregex_iterator{}; + for (auto i = std::sregex_iterator{envString.begin(), envString.end(), + storePathRegex}; + i != env_end; ++i) { + unchecked[i->str()].emplace(envFile); + } + } catch (SysError& e) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) { + continue; + } + throw; + } + } + } + if (errno) { + throw SysError("iterating /proc"); + } + } + + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); + + for (auto& [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); + if (isStorePath(path) && isValidPath(path)) { + DLOG(INFO) << "got additional root " << path; + if (censor) { + roots[path].insert(std::string(kCensored)); + } else { + roots[path].insert(links.begin(), links.end()); + } + } + } + } +} + +struct GCLimitReached {}; + +struct LocalStore::GCState { + GCOptions options; + GCResults& results; + PathSet roots; + PathSet tempRoots; + PathSet dead; + PathSet alive; + bool gcKeepOutputs; + bool gcKeepDerivations; + unsigned long long bytesInvalidated; + bool moveToTrash = true; + bool shouldDelete; + explicit GCState(GCResults& results_) + : results(results_), bytesInvalidated(0) {} +}; + +bool LocalStore::isActiveTempFile(const GCState& state, const Path& path, + const std::string& suffix) { + return absl::EndsWith(path, suffix) && + state.tempRoots.find(std::string( + path, 0, path.size() - suffix.size())) != state.tempRoots.end(); +} + +void LocalStore::deleteGarbage(GCState& state, const Path& path) { + unsigned long long bytesFreed; + deletePath(path, bytesFreed); + state.results.bytesFreed += bytesFreed; +} + +void LocalStore::deletePathRecursive(GCState& state, const Path& path) { + checkInterrupt(); + + unsigned long long size = 0; + + if (isStorePath(path) && isValidPath(path)) { + PathSet referrers; + queryReferrers(path, referrers); + for (auto& i : referrers) { + if (i != path) { + deletePathRecursive(state, i); + } + } + size = queryPathInfo(path)->narSize; + invalidatePathChecked(path); + } + + Path realPath = realStoreDir + "/" + baseNameOf(path); + + struct stat st; + if (lstat(realPath.c_str(), &st) != 0) { + if (errno == ENOENT) { + return; + } + throw SysError(format("getting status of %1%") % realPath); + } + + LOG(INFO) << "deleting '" << path << "'"; + + state.results.paths.insert(path); + + /* If the path is not a regular file or symlink, move it to the + trash directory. The move is to ensure that later (when we're + not holding the global GC lock) we can delete the path without + being afraid that the path has become alive again. Otherwise + delete it right away. */ + if (state.moveToTrash && S_ISDIR(st.st_mode)) { + // Estimate the amount freed using the narSize field. FIXME: + // if the path was not valid, need to determine the actual + // size. + try { + if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) { + throw SysError(format("making '%1%' writable") % realPath); + } + Path tmp = trashDir + "/" + baseNameOf(path); + if (rename(realPath.c_str(), tmp.c_str()) != 0) { + throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % + tmp); + } + state.bytesInvalidated += size; + } catch (SysError& e) { + if (e.errNo == ENOSPC) { + LOG(INFO) << "note: can't create move '" << realPath + << "': " << e.msg(); + deleteGarbage(state, realPath); + } + } + } else { + deleteGarbage(state, realPath); + } + + if (state.results.bytesFreed + state.bytesInvalidated > + state.options.maxFreed) { + LOG(INFO) << "deleted or invalidated more than " << state.options.maxFreed + << " bytes; stopping"; + throw GCLimitReached(); + } +} + +bool LocalStore::canReachRoot(GCState& state, PathSet& visited, + const Path& path) { + if (visited.count(path) != 0u) { + return false; + } + + if (state.alive.count(path) != 0u) { + return true; + } + + if (state.dead.count(path) != 0u) { + return false; + } + + if (state.roots.count(path) != 0u) { + DLOG(INFO) << "cannot delete '" << path << "' because it's a root"; + state.alive.insert(path); + return true; + } + + visited.insert(path); + + if (!isStorePath(path) || !isValidPath(path)) { + return false; + } + + PathSet incoming; + + /* Don't delete this path if any of its referrers are alive. */ + queryReferrers(path, incoming); + + /* If keep-derivations is set and this is a derivation, then + don't delete the derivation if any of the outputs are alive. */ + if (state.gcKeepDerivations && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + for (auto& i : outputs) { + if (isValidPath(i) && queryPathInfo(i)->deriver == path) { + incoming.insert(i); + } + } + } + + /* If keep-outputs is set, then don't delete this path if there + are derivers of this path that are not garbage. */ + if (state.gcKeepOutputs) { + PathSet derivers = queryValidDerivers(path); + for (auto& i : derivers) { + incoming.insert(i); + } + } + + for (auto& i : incoming) { + if (i != path) { + if (canReachRoot(state, visited, i)) { + state.alive.insert(path); + return true; + } + } + } + + return false; +} + +void LocalStore::tryToDelete(GCState& state, const Path& path) { + checkInterrupt(); + + auto realPath = realStoreDir + "/" + baseNameOf(path); + if (realPath == linksDir || realPath == trashDir) { + return; + } + + // Activity act(*logger, lvlDebug, format("considering whether to delete + // '%1%'") % path); + + if (!isStorePath(path) || !isValidPath(path)) { + /* A lock file belonging to a path that we're building right + now isn't garbage. */ + if (isActiveTempFile(state, path, ".lock")) { + return; + } + + /* Don't delete .chroot directories for derivations that are + currently being built. */ + if (isActiveTempFile(state, path, ".chroot")) { + return; + } + + /* Don't delete .check directories for derivations that are + currently being built, because we may need to run + diff-hook. */ + if (isActiveTempFile(state, path, ".check")) { + return; + } + } + + PathSet visited; + + if (canReachRoot(state, visited, path)) { + DLOG(INFO) << "cannot delete '" << path << "' because it's still reachable"; + } else { + /* No path we visited was a root, so everything is garbage. + But we only delete ‘path’ and its referrers here so that + ‘nix-store --delete’ doesn't have the unexpected effect of + recursing into derivations and outputs. */ + state.dead.insert(visited.begin(), visited.end()); + if (state.shouldDelete) { + deletePathRecursive(state, path); + } + } +} + +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks(const GCState& state) { + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) { + throw SysError(format("opening directory '%1%'") % linksDir); + } + + long long actualSize = 0; + long long unsharedSize = 0; + + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") { + continue; + } + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + throw SysError(format("statting '%1%'") % path); + } + + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } + + LOG(INFO) << "deleting unused link " << path; + + if (unlink(path.c_str()) == -1) { + throw SysError(format("deleting '%1%'") % path); + } + + state.results.bytesFreed += st.st_size; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) { + throw SysError(format("statting '%1%'") % linksDir); + } + + long long overhead = st.st_blocks * 512ULL; + + // TODO(tazjin): absl::StrFormat %.2f + LOG(INFO) << "note: currently hard linking saves " + << ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)) + << " MiB"; +} + +void LocalStore::collectGarbage(const GCOptions& options, GCResults& results) { + GCState state(results); + state.options = options; + state.gcKeepOutputs = settings.gcKeepOutputs; + state.gcKeepDerivations = settings.gcKeepDerivations; + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } + + state.shouldDelete = options.action == GCOptions::gcDeleteDead || + options.action == GCOptions::gcDeleteSpecific; + + if (state.shouldDelete) { + deletePath(reservedPath); + } + + /* Acquire the global GC root. This prevents + a) New roots from being added. + b) Processes from creating new temporary root files. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + LOG(INFO) << "finding garbage collector roots..."; + Roots rootMap; + if (!options.ignoreLiveness) { + findRootsNoTemp(rootMap, true); + } + + for (auto& i : rootMap) { + state.roots.insert(i.first); + } + + /* Read the temporary roots. This acquires read locks on all + per-process temporary root files. So after this point no paths + can be added to the set of temporary roots. */ + FDs fds; + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto& root : tempRoots) { + state.tempRoots.insert(root.first); + } + state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + + /* After this point the set of roots or temporary roots cannot + increase, since we hold locks on everything. So everything + that is not reachable from `roots' is garbage. */ + + if (state.shouldDelete) { + if (pathExists(trashDir)) { + deleteGarbage(state, trashDir); + } + try { + createDirs(trashDir); + } catch (SysError& e) { + if (e.errNo == ENOSPC) { + LOG(INFO) << "note: can't create trash directory: " << e.msg(); + state.moveToTrash = false; + } + } + } + + /* Now either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ + + if (options.action == GCOptions::gcDeleteSpecific) { + for (auto& i : options.pathsToDelete) { + assertStorePath(i); + tryToDelete(state, i); + if (state.dead.find(i) == state.dead.end()) { + throw Error(format("cannot delete path '%1%' since it is still alive") % + i); + } + } + + } else if (options.maxFreed > 0) { + if (state.shouldDelete) { + LOG(INFO) << "deleting garbage..."; + } else { + LOG(ERROR) << "determining live/dead paths..."; + } + + try { + AutoCloseDir dir(opendir(realStoreDir.c_str())); + if (!dir) { + throw SysError(format("opening directory '%1%'") % realStoreDir); + } + + /* Read the store and immediately delete all paths that + aren't valid. When using --max-freed etc., deleting + invalid paths is preferred over deleting unreachable + paths, since unreachable paths could become reachable + again. We don't use readDirectory() here so that GCing + can start faster. */ + Paths entries; + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") { + continue; + } + Path path = storeDir + "/" + name; + if (isStorePath(path) && isValidPath(path)) { + entries.push_back(path); + } else { + tryToDelete(state, path); + } + } + + dir.reset(); + + /* Now delete the unreachable valid paths. Randomise the + order in which we delete entries to make the collector + less biased towards deleting paths that come + alphabetically first (e.g. /nix/store/000...). This + matters when using --max-freed etc. */ + std::vector<Path> entries_(entries.begin(), entries.end()); + std::mt19937 gen(1); + std::shuffle(entries_.begin(), entries_.end(), gen); + + for (auto& i : entries_) { + tryToDelete(state, i); + } + + } catch (GCLimitReached& e) { + } + } + + if (state.options.action == GCOptions::gcReturnLive) { + state.results.paths = state.alive; + return; + } + + if (state.options.action == GCOptions::gcReturnDead) { + state.results.paths = state.dead; + return; + } + + /* Allow other processes to add to the store from here on. */ + fdGCLock = AutoCloseFD(-1); + fds.clear(); + + /* Delete the trash directory. */ + LOG(INFO) << "deleting " << trashDir; + deleteGarbage(state, trashDir); + + /* Clean up the links directory. */ + if (options.action == GCOptions::gcDeleteDead || + options.action == GCOptions::gcDeleteSpecific) { + LOG(INFO) << "deleting unused links..."; + removeUnusedLinks(state); + } + + /* While we're at it, vacuum the database. */ + // if (options.action == GCOptions::gcDeleteDead) { vacuumDB(); } +} + +void LocalStore::autoGC(bool sync) { + static auto fakeFreeSpaceFile = + getEnv("_NIX_TEST_FREE_SPACE_FILE").value_or(""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) { + return std::stoll(readFile(fakeFreeSpaceFile)); + } + + struct statvfs st; + if (statvfs(realStoreDir.c_str(), &st) != 0) { + throw SysError("getting filesystem info about '%s'", realStoreDir); + } + + return static_cast<uint64_t>(st.f_bavail) * st.f_bsize; + }; + + std::shared_future<void> future; + + { + auto state(_state.lock()); + + if (state->gcRunning) { + future = state->gcFuture; + DLOG(INFO) << "waiting for auto-GC to finish"; + goto sync; + } + + auto now = std::chrono::steady_clock::now(); + + if (now < state->lastGCCheck + + std::chrono::seconds(settings.minFreeCheckInterval)) { + return; + } + + auto avail = getAvail(); + + state->lastGCCheck = now; + + if (avail >= settings.minFree || avail >= settings.maxFree) { + return; + } + + if (avail > state->availAfterGC * 0.97) { + return; + } + + state->gcRunning = true; + + std::promise<void> promise; + future = state->gcFuture = promise.get_future().share(); + + std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + try { + /* Wake up any threads waiting for the auto-GC to finish. */ + Finally wakeup([&]() { + auto state(_state.lock()); + state->gcRunning = false; + state->lastGCCheck = std::chrono::steady_clock::now(); + promise.set_value(); + }); + + GCOptions options; + options.maxFreed = settings.maxFree - avail; + + LOG(INFO) << "running auto-GC to free " << options.maxFreed << " bytes"; + + GCResults results; + + collectGarbage(options, results); + + _state.lock()->availAfterGC = getAvail(); + + } catch (...) { + // FIXME: we could propagate the exception to the + // future, but we don't really care. + ignoreException(); + } + }).detach(); + } + +sync: + // Wait for the future outside of the state lock. + if (sync) { + future.get(); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/globals.cc b/third_party/nix/src/libstore/globals.cc new file mode 100644 index 000000000000..6babb4589fa0 --- /dev/null +++ b/third_party/nix/src/libstore/globals.cc @@ -0,0 +1,178 @@ +#include "libstore/globals.hh" + +#include <algorithm> +#include <filesystem> +#include <map> +#include <thread> + +#include <absl/strings/numbers.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <dlfcn.h> + +#include "libutil/archive.hh" +#include "libutil/args.hh" +#include "libutil/util.hh" +#include "nix_config.h" + +namespace nix { + +/* The default location of the daemon socket, relative to nixStateDir. + The socket is in a directory to allow you to control access to the + Nix daemon by setting the mode/ownership of the directory + appropriately. (This wouldn't work on the socket itself since it + must be deleted and recreated on startup.) */ +#define DEFAULT_SOCKET_PATH "/daemon-socket/socket" + +Settings settings; + +static GlobalConfig::Register r1(&settings); + +Settings::Settings() + : nixPrefix(NIX_PREFIX), + nixStore(canonPath( + getEnv("NIX_STORE_DIR") + .value_or(getEnv("NIX_STORE").value_or(NIX_STORE_DIR)))), + nixDataDir(canonPath(getEnv("NIX_DATA_DIR").value_or(NIX_DATA_DIR))), + nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR))), + nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR))), + nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR))), + nixLibexecDir( + canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR))), + nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR))), + nixManDir(canonPath(NIX_MAN_DIR)), + nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) { + buildUsersGroup = getuid() == 0 ? "nixbld" : ""; + lockCPU = getEnv("NIX_AFFINITY_HACK").value_or("1") == "1"; + + caFile = getEnv("NIX_SSL_CERT_FILE") + .value_or(getEnv("SSL_CERT_FILE").value_or("")); + if (caFile.empty()) { + for (auto& fn : + {"/etc/ssl/certs/ca-certificates.crt", + "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) { + if (pathExists(fn)) { + caFile = fn; + break; + } + } + } + + /* Backwards compatibility. */ + // TODO(tazjin): still? + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s) { + Strings ss; + for (auto p : absl::StrSplit(*s, absl::ByChar(':'), absl::SkipEmpty())) { + ss.push_back(absl::StrCat("@", p)); + } + builders = concatStringsSep(" ", ss); + } + + sandboxPaths = absl::StrSplit("/bin/sh=" SANDBOX_SHELL, + absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); +} + +void loadConfFile() { + if (std::filesystem::exists(settings.nixConfDir + "/nix.conf")) { + globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + } + + /* We only want to send overrides to the daemon, i.e. stuff from + ~/.nix/nix.conf or the command line. */ + globalConfig.resetOverriden(); + + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path + // take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + if (std::filesystem::exists(*dir + "/nix.conf")) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } + } +} + +unsigned int Settings::getDefaultCores() { + return std::max(1U, std::thread::hardware_concurrency()); +} + +StringSet Settings::getDefaultSystemFeatures() { + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel"}; + +#if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) { + features.insert("kvm"); + } +#endif + + return features; +} + +const std::string nixVersion = PACKAGE_VERSION; + +template <> +void BaseSetting<SandboxMode>::set(const std::string& str) { + if (str == "true") { + value = smEnabled; + } else if (str == "relaxed") { + value = smRelaxed; + } else if (str == "false") { + value = smDisabled; + } else { + throw UsageError("option '%s' has invalid value '%s'", name, str); + } +} + +template <> +std::string BaseSetting<SandboxMode>::to_string() { + if (value == smEnabled) { + return "true"; + } + if (value == smRelaxed) { + return "relaxed"; + } else if (value == smDisabled) { + return "false"; + } else { + abort(); + } +} + +template <> +void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder& out) { + AbstractSetting::toJSON(out); +} + +template <> +void BaseSetting<SandboxMode>::convertToArg(Args& args, + const std::string& category) { + args.mkFlag() + .longName(name) + .description("Enable sandboxing.") + .handler([=](const std::vector<std::string>& ss) { override(smEnabled); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description("Disable sandboxing.") + .handler( + [=](const std::vector<std::string>& ss) { override(smDisabled); }) + .category(category); + args.mkFlag() + .longName("relaxed-" + name) + .description("Enable sandboxing, but allow builds to disable it.") + .handler([=](const std::vector<std::string>& ss) { override(smRelaxed); }) + .category(category); +} + +void MaxBuildJobsSetting::set(const std::string& str) { + if (str == "auto") { + value = std::max(1U, std::thread::hardware_concurrency()); + } else if (!absl::SimpleAtoi(str, &value)) { + throw UsageError( + "configuration setting '%s' should be 'auto' or an integer", name); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/globals.hh b/third_party/nix/src/libstore/globals.hh new file mode 100644 index 000000000000..ed9b6a338e96 --- /dev/null +++ b/third_party/nix/src/libstore/globals.hh @@ -0,0 +1,464 @@ +#pragma once + +#include <limits> +#include <map> + +#include <sys/types.h> + +#include "libutil/config.hh" +#include "libutil/types.hh" +#include "libutil/util.hh" +#include "nix_config.h" + +namespace nix { + +typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; + +struct MaxBuildJobsSetting : public BaseSetting<unsigned int> { + MaxBuildJobsSetting(Config* options, unsigned int def, + const std::string& name, const std::string& description, + const std::set<std::string>& aliases = {}) + : BaseSetting<unsigned int>(def, name, description, aliases) { + options->addSetting(this); + } + + void set(const std::string& str) override; +}; + +class Settings : public Config { + static unsigned int getDefaultCores(); + + static StringSet getDefaultSystemFeatures(); + + public: + Settings(); + + Path nixPrefix; + + /* The directory where we store sources and derived files. */ + Path nixStore; + + Path nixDataDir; /* !!! fix */ + + /* The directory where we log various operations. */ + Path nixLogDir; + + /* The directory where state is stored. */ + Path nixStateDir; + + /* The directory where configuration files are stored. */ + Path nixConfDir; + + /* The directory where internal helper programs are stored. */ + Path nixLibexecDir; + + /* The directory where the main programs are stored. */ + Path nixBinDir; + + /* The directory where the man pages are stored. */ + Path nixManDir; + + /* File name of the socket the daemon listens to. */ + Path nixDaemonSocketFile; + + Setting<std::string> storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), + "store", "The default Nix store to use."}; + + Setting<bool> keepFailed{ + this, false, "keep-failed", + "Whether to keep temporary directories of failed builds."}; + + Setting<bool> keepGoing{ + this, false, "keep-going", + "Whether to keep building derivations when another build fails."}; + + Setting<bool> tryFallback{ + this, + false, + "fallback", + "Whether to fall back to building when substitution fails.", + {"build-fallback"}}; + + /* Whether to show build log output in real time. */ + bool verboseBuild = true; + + Setting<size_t> logLines{ + this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; + + MaxBuildJobsSetting maxBuildJobs{this, + 1, + "max-jobs", + "Maximum number of parallel build jobs. " + "\"auto\" means use number of cores.", + {"build-max-jobs"}}; + + Setting<unsigned int> buildCores{ + this, + getDefaultCores(), + "cores", + "Number of CPU cores to utilize in parallel within a build, " + "i.e. by passing this number to Make via '-j'. 0 means that the " + "number of actual CPU cores on the local host ought to be " + "auto-detected.", + {"build-cores"}}; + + /* Read-only mode. Don't copy stuff to the store, don't change + the database. */ + bool readOnlyMode = false; + + Setting<std::string> thisSystem{this, SYSTEM, "system", + "The canonical Nix system name."}; + + Setting<time_t> maxSilentTime{ + this, + 0, + "max-silent-time", + "The maximum time in seconds that a builer can go without " + "producing any output on stdout/stderr before it is killed. " + "0 means infinity.", + {"build-max-silent-time"}}; + + Setting<time_t> buildTimeout{ + this, + 0, + "timeout", + "The maximum duration in seconds that a builder can run. " + "0 means infinity.", + {"build-timeout"}}; + + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", + "build-hook", + "The path of the helper program that executes builds " + "to remote machines."}; + + Setting<std::string> builders{this, "@" + nixConfDir + "/machines", + "builders", + "A semicolon-separated list of build machines, " + "in the format of nix.machines."}; + + Setting<bool> buildersUseSubstitutes{ + this, false, "builders-use-substitutes", + "Whether build machines should use their own substitutes for obtaining " + "build dependencies if possible, rather than waiting for this host to " + "upload them."}; + + Setting<off_t> reservedSize{ + this, 8 * 1024 * 1024, "gc-reserved-space", + "Amount of reserved disk space for the garbage collector."}; + + Setting<bool> fsyncMetadata{this, true, "fsync-metadata", + "Whether SQLite should use fsync()."}; + + Setting<bool> useSQLiteWAL{this, true, "use-sqlite-wal", + "Whether SQLite should use WAL mode."}; + + Setting<bool> syncBeforeRegistering{ + this, false, "sync-before-registering", + "Whether to call sync() before registering a path as valid."}; + + Setting<bool> useSubstitutes{this, + true, + "substitute", + "Whether to use substitutes.", + {"build-use-substitutes"}}; + + Setting<std::string> buildUsersGroup{ + this, "", "build-users-group", + "The Unix group that contains the build users."}; + + Setting<bool> impersonateLinux26{ + this, + false, + "impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels.", + {"build-impersonate-linux-26"}}; + + Setting<bool> keepLog{this, + true, + "keep-build-log", + "Whether to store build logs.", + {"build-keep-log"}}; + + Setting<bool> compressLog{this, + true, + "compress-build-log", + "Whether to compress logs.", + {"build-compress-log"}}; + + Setting<unsigned long> maxLogSize{ + this, + 0, + "max-build-log-size", + "Maximum number of bytes a builder can write to stdout/stderr " + "before being killed (0 means no limit).", + {"build-max-log-size"}}; + + /* When buildRepeat > 0 and verboseBuild == true, whether to print + repeated builds (i.e. builds other than the first one) to + stderr. Hack to prevent Hydra logs from being polluted. */ + bool printRepeatedBuilds = true; + + Setting<unsigned int> pollInterval{ + this, 5, "build-poll-interval", + "How often (in seconds) to poll for locks."}; + + Setting<bool> checkRootReachability{ + this, false, "gc-check-reachability", + "Whether to check if new GC roots can in fact be found by the " + "garbage collector."}; + + Setting<bool> gcKeepOutputs{ + this, + false, + "keep-outputs", + "Whether the garbage collector should keep outputs of live derivations.", + {"gc-keep-outputs"}}; + + Setting<bool> gcKeepDerivations{ + this, + true, + "keep-derivations", + "Whether the garbage collector should keep derivers of live paths.", + {"gc-keep-derivations"}}; + + Setting<bool> autoOptimiseStore{this, false, "auto-optimise-store", + "Whether to automatically replace files with " + "identical contents with hard links."}; + + Setting<bool> envKeepDerivations{ + this, + false, + "keep-env-derivations", + "Whether to add derivations as a dependency of user environments " + "(to prevent them from being GCed).", + {"env-keep-derivations"}}; + + /* Whether to lock the Nix client and worker to the same CPU. */ + bool lockCPU; + + /* Whether to show a stack trace if Nix evaluation fails. */ + Setting<bool> showTrace{ + this, false, "show-trace", + "Whether to show a stack trace on evaluation errors."}; + + Setting<SandboxMode> sandboxMode { + this, +#if __linux__ + smEnabled +#else + smDisabled +#endif + , + "sandbox", + "Whether to enable sandboxed builds. Can be \"true\", \"false\" or " + "\"relaxed\".", + { + "build-use-chroot", "build-use-sandbox" + } + }; + + Setting<PathSet> sandboxPaths{ + this, + {}, + "sandbox-paths", + "The paths to make available inside the build sandbox.", + {"build-chroot-dirs", "build-sandbox-paths"}}; + + Setting<bool> sandboxFallback{ + this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + + Setting<PathSet> extraSandboxPaths{ + this, + {}, + "extra-sandbox-paths", + "Additional paths to make available inside the build sandbox.", + {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; + + Setting<size_t> buildRepeat{ + this, + 0, + "repeat", + "The number of times to repeat a build in order to verify determinism.", + {"build-repeat"}}; + +#if __linux__ + Setting<std::string> sandboxShmSize{ + this, "50%", "sandbox-dev-shm-size", + "The size of /dev/shm in the build sandbox."}; + + Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir", + "The build directory inside the sandbox."}; +#endif + + Setting<PathSet> allowedImpureHostPrefixes{ + this, + {}, + "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for " + "Darwin)."}; + + Setting<bool> runDiffHook{ + this, false, "run-diff-hook", + "Whether to run the program specified by the diff-hook setting " + "repeated builds produce a different result. Typically used to " + "plug in diffoscope."}; + + PathSetting diffHook{ + this, true, "", "diff-hook", + "A program that prints out the differences between the two paths " + "specified on its command line."}; + + Setting<bool> enforceDeterminism{ + this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output."}; + + Setting<Strings> trustedPublicKeys{ + this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "trusted-public-keys", + "Trusted public keys for secure substitution.", + {"binary-cache-public-keys"}}; + + Setting<Strings> secretKeyFiles{ + this, + {}, + "secret-key-files", + "Secret keys with which to sign local builds."}; + + Setting<unsigned int> tarballTtl{ + this, 60 * 60, "tarball-ttl", + "How long downloaded files are considered up-to-date."}; + + Setting<bool> requireSigs{ + this, true, "require-sigs", + "Whether to check that any non-content-addressed path added to the " + "Nix store has a valid signature (that is, one signed using a key " + "listed in 'trusted-public-keys'."}; + + Setting<StringSet> extraPlatforms{ + this, + std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} + : StringSet{}, + "extra-platforms", + "Additional platforms that can be built on the local system. " + "These may be supported natively (e.g. armv7 on some aarch64 CPUs " + "or using hacks like qemu-user."}; + + Setting<StringSet> systemFeatures{ + this, getDefaultSystemFeatures(), "system-features", + "Optional features that this system implements (like \"kvm\")."}; + + Setting<Strings> substituters{ + this, + nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} + : Strings(), + "substituters", + "The URIs of substituters (such as https://cache.nixos.org/).", + {"binary-caches"}}; + + // FIXME: provide a way to add to option values. + Setting<Strings> extraSubstituters{this, + {}, + "extra-substituters", + "Additional URIs of substituters.", + {"extra-binary-caches"}}; + + Setting<StringSet> trustedSubstituters{ + this, + {}, + "trusted-substituters", + "Disabled substituters that may be enabled via the substituters option " + "by untrusted users.", + {"trusted-binary-caches"}}; + + Setting<Strings> trustedUsers{this, + {"root"}, + "trusted-users", + "Which users or groups are trusted to ask the " + "daemon to do unsafe things."}; + + Setting<unsigned int> ttlNegativeNarInfoCache{ + this, 3600, "narinfo-cache-negative-ttl", + "The TTL in seconds for negative lookups in the disk cache i.e binary " + "cache lookups that " + "return an invalid path result"}; + + Setting<unsigned int> ttlPositiveNarInfoCache{ + this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + "The TTL in seconds for positive lookups in the disk cache i.e binary " + "cache lookups that " + "return a valid path result."}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting<Strings> allowedUsers{ + this, + {"*"}, + "allowed-users", + "Which users or groups are allowed to connect to the daemon."}; + + Setting<bool> printMissing{ + this, true, "print-missing", + "Whether to print what paths need to be built or downloaded."}; + + Setting<std::string> preBuildHook{ + this, "", "pre-build-hook", + "A program to run just before a build to set derivation-specific build " + "settings."}; + + Setting<std::string> postBuildHook{ + this, "", "post-build-hook", + "A program to run just after each successful build."}; + + Setting<std::string> netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), + "netrc-file", + "Path to the netrc file used to obtain " + "usernames/passwords for downloads."}; + + /* Path to the SSL CA file used */ + Path caFile; + +#if __linux__ + Setting<bool> filterSyscalls{ + this, true, "filter-syscalls", + "Whether to prevent certain dangerous system calls, such as " + "creation of setuid/setgid files or adding ACLs or extended " + "attributes. Only disable this if you're aware of the " + "security implications."}; + + Setting<bool> allowNewPrivileges{ + this, false, "allow-new-privileges", + "Whether builders can acquire new privileges by calling programs with " + "setuid/setgid bits or with file capabilities."}; +#endif + + Setting<Strings> hashedMirrors{ + this, + {"http://tarballs.nixos.org/"}, + "hashed-mirrors", + "A list of servers used by builtins.fetchurl to fetch files by hash."}; + + Setting<uint64_t> minFree{this, 0, "min-free", + "Automatically run the garbage collector when free " + "disk space drops below the specified amount."}; + + Setting<uint64_t> maxFree{this, std::numeric_limits<uint64_t>::max(), + "max-free", + "Stop deleting garbage when free disk space is " + "above the specified amount."}; + + Setting<uint64_t> minFreeCheckInterval{ + this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; +}; + +// FIXME: don't use a global variable. +extern Settings settings; + +void loadConfFile(); + +extern const std::string nixVersion; + +} // namespace nix diff --git a/third_party/nix/src/libstore/http-binary-cache-store.cc b/third_party/nix/src/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000000..c713ac43c47a --- /dev/null +++ b/third_party/nix/src/libstore/http-binary-cache-store.cc @@ -0,0 +1,171 @@ +#include <utility> + +#include <glog/logging.h> + +#include "libstore/binary-cache-store.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/nar-info-disk-cache.hh" + +namespace nix { + +MakeError(UploadToHTTP, Error); + +class HttpBinaryCacheStore : public BinaryCacheStore { + private: + Path cacheUri; + + struct State { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync<State> _state; + + public: + HttpBinaryCacheStore(const Params& params, Path _cacheUri) + : BinaryCacheStore(params), cacheUri(std::move(_cacheUri)) { + if (cacheUri.back() == '/') { + cacheUri.pop_back(); + } + + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override { return cacheUri; } + + void init() override { + // FIXME: do this lazily? + if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { + try { + BinaryCacheStore::init(); + } catch (UploadToHTTP&) { + throw Error("'%s' does not appear to be a binary cache", cacheUri); + } + diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); + } + } + + protected: + void maybeDisable() { + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + LOG(WARNING) << "disabling binary cache '" << getUri() << "' for " << t + << " seconds"; + state->enabled = false; + state->disabledUntil = + std::chrono::steady_clock::now() + std::chrono::seconds(t); + } + } + + void checkEnabled() { + auto state(_state.lock()); + if (state->enabled) { + return; + } + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + DLOG(INFO) << "re-enabling binary cache '" << getUri() << "'"; + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + + bool fileExists(const std::string& path) override { + checkEnabled(); + + try { + DownloadRequest request(cacheUri + "/" + path); + request.head = true; + getDownloader()->download(request); + return true; + } catch (DownloadError& e) { + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) { + return false; + } + maybeDisable(); + throw; + } + } + + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override { + auto req = DownloadRequest(cacheUri + "/" + path); + req.data = std::make_shared<std::string>(data); // FIXME: inefficient + req.mimeType = mimeType; + try { + getDownloader()->download(req); + } catch (DownloadError& e) { + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", + cacheUri, e.msg()); + } + } + + DownloadRequest makeRequest(const std::string& path) { + DownloadRequest request(cacheUri + "/" + path); + return request; + } + + void getFile(const std::string& path, Sink& sink) override { + checkEnabled(); + auto request(makeRequest(path)); + try { + getDownloader()->download(std::move(request), sink); + } catch (DownloadError& e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) { + throw NoSuchBinaryCacheFile( + "file '%s' does not exist in binary cache '%s'", path, getUri()); + } + maybeDisable(); + throw; + } + } + + void getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept override { + checkEnabled(); + + auto request(makeRequest(path)); + + auto callbackPtr = + std::make_shared<decltype(callback)>(std::move(callback)); + + getDownloader()->enqueueDownload( + request, + Callback<DownloadResult>{ + [callbackPtr, this](std::future<DownloadResult> result) { + try { + (*callbackPtr)(result.get().data); + } catch (DownloadError& e) { + if (e.error == Downloader::NotFound || + e.error == Downloader::Forbidden) { + return (*callbackPtr)(std::shared_ptr<std::string>()); + } + maybeDisable(); + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } +}; + +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://" && + (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || + std::string(uri, 0, 7) != "file://")) { + return nullptr; + } + auto store = std::make_shared<HttpBinaryCacheStore>(params, uri); + store->init(); + return store; + }); + +} // namespace nix diff --git a/third_party/nix/src/libstore/legacy-ssh-store.cc b/third_party/nix/src/libstore/legacy-ssh-store.cc new file mode 100644 index 000000000000..8163258179a1 --- /dev/null +++ b/third_party/nix/src/libstore/legacy-ssh-store.cc @@ -0,0 +1,282 @@ +#include <absl/strings/match.h> +#include <absl/strings/str_cat.h> +#include <glog/logging.h> + +#include "libstore/derivations.hh" +#include "libstore/remote-store.hh" +#include "libstore/serve-protocol.hh" +#include "libstore/ssh.hh" +#include "libstore/store-api.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/archive.hh" +#include "libutil/pool.hh" + +namespace nix { + +constexpr std::string_view kUriScheme = "ssh://"; + +struct LegacySSHStore : public Store { + const Setting<int> maxConnections{ + this, 1, "max-connections", + "maximum number of concurrent SSH connections"}; + const Setting<Path> sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting<bool> compress{this, false, "compress", + "whether to compress the connection"}; + const Setting<Path> remoteProgram{ + this, "nix-store", "remote-program", + "path to the nix-store executable on the remote system"}; + const Setting<std::string> remoteStore{ + this, "", "remote-store", "URI of the store on the remote system"}; + + // Hack for getting remote build log output. + const Setting<int> logFD{ + this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection { + std::unique_ptr<SSHMaster::Connection> sshConn; + FdSink to; + FdSource from; + int remoteVersion; + bool good = true; + }; + + std::string host; + + ref<Pool<Connection>> connections; + + SSHMaster master; + + LegacySSHStore(const std::string& host, const Params& params) + : Store(params), + host(host), + connections(make_ref<Pool<Connection>>( + std::max(1, (int)maxConnections), + [this]() { return openConnection(); }, + [](const ref<Connection>& r) { return r->good; })), + master(host, sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, compress, logFD) {} + + ref<Connection> openConnection() { + auto conn = make_ref<Connection>(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get().empty() + ? "" + : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + unsigned int magic = readInt(conn->from); + if (magic != SERVE_MAGIC_2) { + throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); + } + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) { + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", + host); + } + + } catch (EndOfFile& e) { + throw Error("cannot connect to '%1%'", host); + } + + return conn; + }; + + std::string getUri() override { return absl::StrCat(kUriScheme, host); } + + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override { + try { + auto conn(connections->get()); + + DLOG(INFO) << "querying remote host '" << host << "' for info on '" + << path << "'"; + + conn->to << cmdQueryPathInfos << PathSet{path}; + conn->to.flush(); + + auto info = std::make_shared<ValidPathInfo>(); + conn->from >> info->path; + if (info->path.empty()) { + return callback(nullptr); + } + assert(path == info->path); + + PathSet references; + conn->from >> info->deriver; + info->references = readStorePaths<PathSet>(*this, conn->from); + readLongLong(conn->from); // download size + info->narSize = readLongLong(conn->from); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + auto s = readString(conn->from); + if (s.empty()) { + info->narHash = Hash(); + } else { + auto hash_ = Hash::deserialize(s); + info->narHash = Hash::unwrap_throw(hash_); + } + conn->from >> info->ca; + info->sigs = readStrings<StringSet>(conn->from); + } + + auto s = readString(conn->from); + assert(s.empty()); + + callback(std::move(info)); + } catch (...) { + callback.rethrow(); + } + } + + void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override { + DLOG(INFO) << "adding path '" << info.path << "' to remote host '" << host + << "'"; + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + conn->to << cmdAddToStoreNar << info.path << info.deriver + << info.narHash.to_string(Base16, false) << info.references + << info.registrationTime << info.narSize + << static_cast<uint64_t>(info.ultimate) << info.sigs << info.ca; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + conn->to << cmdImportPaths << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to << exportMagic << info.path << info.references << info.deriver + << 0 << 0; + conn->to.flush(); + } + + if (readInt(conn->from) != 1) { + throw Error( + "failed to add path '%s' to remote host '%s', info.path, host"); + } + } + + void narFromPath(const Path& path, Sink& sink) override { + auto conn(connections->get()); + + conn->to << cmdDumpStorePath << path; + conn->to.flush(); + copyNAR(conn->from, sink); + } + + Path queryPathFromHashPart(const std::string& hashPart) override { + unsupported("queryPathFromHashPart"); + } + + Path addToStore(const std::string& name, const Path& srcPath, bool recursive, + HashType hashAlgo, PathFilter& filter, + RepairFlag repair) override { + unsupported("addToStore"); + } + + Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair) override { + unsupported("addTextToStore"); + } + + BuildResult buildDerivation(std::ostream& /*log_sink*/, const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) override { + auto conn(connections->get()); + + conn->to << cmdBuildDerivation << drvPath << drv << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) { + conn->to << settings.maxLogSize; + } + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) { + conn->to << settings.buildRepeat + << static_cast<uint64_t>(settings.enforceDeterminism); + } + + conn->to.flush(); + + BuildResult status; + status.status = static_cast<BuildResult::Status>(readInt(conn->from)); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) { + conn->from >> status.timesBuilt >> status.isNonDeterministic >> + status.startTime >> status.stopTime; + } + + return status; + } + + void ensurePath(const Path& path) override { unsupported("ensurePath"); } + + void computeFSClosure(const PathSet& paths, PathSet& out, + bool flipDirection = false, bool includeOutputs = false, + bool includeDerivers = false) override { + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, + includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to << cmdQueryClosure << static_cast<uint64_t>(includeOutputs) + << paths; + conn->to.flush(); + + auto res = readStorePaths<PathSet>(*this, conn->from); + + out.insert(res.begin(), res.end()); + } + + PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute = + NoSubstitute) override { + auto conn(connections->get()); + + conn->to << cmdQueryValidPaths << 0u // lock + << maybeSubstitute << paths; + conn->to.flush(); + + return readStorePaths<PathSet>(*this, conn->from); + } + + void connect() override { auto conn(connections->get()); } + + unsigned int getProtocol() override { + auto conn(connections->get()); + return conn->remoteVersion; + } +}; + +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (!absl::StartsWith(uri, kUriScheme)) { + return nullptr; + } + return std::make_shared<LegacySSHStore>( + std::string(uri, kUriScheme.size()), params); + }); + +} // namespace nix diff --git a/third_party/nix/src/libstore/local-binary-cache-store.cc b/third_party/nix/src/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000000..4555de504783 --- /dev/null +++ b/third_party/nix/src/libstore/local-binary-cache-store.cc @@ -0,0 +1,93 @@ +#include <utility> + +#include <absl/strings/match.h> + +#include "libstore/binary-cache-store.hh" +#include "libstore/globals.hh" +#include "libstore/nar-info-disk-cache.hh" + +namespace nix { + +class LocalBinaryCacheStore : public BinaryCacheStore { + private: + Path binaryCacheDir; + + public: + LocalBinaryCacheStore(const Params& params, Path binaryCacheDir) + : BinaryCacheStore(params), binaryCacheDir(std::move(binaryCacheDir)) {} + + void init() override; + + std::string getUri() override { return "file://" + binaryCacheDir; } + + protected: + bool fileExists(const std::string& path) override; + + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override; + + void getFile(const std::string& path, Sink& sink) override { + try { + readFile(binaryCacheDir + "/" + path, sink); + } catch (SysError& e) { + if (e.errNo == ENOENT) { + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", + path); + } + } + } + + PathSet queryAllValidPaths() override { + PathSet paths; + + for (auto& entry : readDirectory(binaryCacheDir)) { + if (entry.name.size() != 40 || !absl::EndsWith(entry.name, ".narinfo")) { + continue; + } + paths.insert(storeDir + "/" + + entry.name.substr(0, entry.name.size() - 8)); + } + + return paths; + } +}; + +void LocalBinaryCacheStore::init() { + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); +} + +static void atomicWrite(const Path& path, const std::string& s) { + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str()) != 0) { + throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + } + del.cancel(); +} + +bool LocalBinaryCacheStore::fileExists(const std::string& path) { + return pathExists(binaryCacheDir + "/" + path); +} + +void LocalBinaryCacheStore::upsertFile(const std::string& path, + const std::string& data, + const std::string& mimeType) { + atomicWrite(binaryCacheDir + "/" + path, data); +} + +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || + std::string(uri, 0, 7) != "file://") { + return nullptr; + } + auto store = + std::make_shared<LocalBinaryCacheStore>(params, std::string(uri, 7)); + store->init(); + return store; + }); + +} // namespace nix diff --git a/third_party/nix/src/libstore/local-fs-store.cc b/third_party/nix/src/libstore/local-fs-store.cc new file mode 100644 index 000000000000..f2235bad7677 --- /dev/null +++ b/third_party/nix/src/libstore/local-fs-store.cc @@ -0,0 +1,123 @@ +#include "libstore/derivations.hh" +#include "libstore/fs-accessor.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/compression.hh" + +namespace nix { + +LocalFSStore::LocalFSStore(const Params& params) : Store(params) {} + +struct LocalStoreAccessor : public FSAccessor { + ref<LocalFSStore> store; + + explicit LocalStoreAccessor(const ref<LocalFSStore>& store) : store(store) {} + + Path toRealPath(const Path& path) { + Path storePath = store->toStorePath(path); + if (!store->isValidPath(storePath)) { + throw InvalidPath(format("path '%1%' is not a valid store path") % + storePath); + } + return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + } + + FSAccessor::Stat stat(const Path& path) override { + auto realPath = toRealPath(path); + + struct stat st; + if (lstat(realPath.c_str(), &st) != 0) { + if (errno == ENOENT || errno == ENOTDIR) { + return {Type::tMissing, 0, false}; + } + throw SysError(format("getting status of '%1%'") % path); + } + + if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) { + throw Error(format("file '%1%' has unsupported type") % path); + } + + return {S_ISREG(st.st_mode) ? Type::tRegular + : S_ISLNK(st.st_mode) ? Type::tSymlink + : Type::tDirectory, + S_ISREG(st.st_mode) ? static_cast<uint64_t>(st.st_size) : 0, + S_ISREG(st.st_mode) && ((st.st_mode & S_IXUSR) != 0u)}; + } + + StringSet readDirectory(const Path& path) override { + auto realPath = toRealPath(path); + + auto entries = nix::readDirectory(realPath); + + StringSet res; + for (auto& entry : entries) { + res.insert(entry.name); + } + + return res; + } + + std::string readFile(const Path& path) override { + return nix::readFile(toRealPath(path)); + } + + std::string readLink(const Path& path) override { + return nix::readLink(toRealPath(path)); + } +}; + +ref<FSAccessor> LocalFSStore::getFSAccessor() { + return make_ref<LocalStoreAccessor>(ref<LocalFSStore>( + std::dynamic_pointer_cast<LocalFSStore>(shared_from_this()))); +} + +void LocalFSStore::narFromPath(const Path& path, Sink& sink) { + if (!isValidPath(path)) { + throw Error(format("path '%s' is not valid") % path); + } + dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); +} + +const std::string LocalFSStore::drvsLogDir = "drvs"; + +std::shared_ptr<std::string> LocalFSStore::getBuildLog(const Path& path_) { + auto path(path_); + + assertStorePath(path); + + if (!isDerivation(path)) { + try { + path = queryPathInfo(path)->deriver; + } catch (InvalidPath&) { + return nullptr; + } + if (path.empty()) { + return nullptr; + } + } + + std::string baseName = baseNameOf(path); + + for (int j = 0; j < 2; j++) { + Path logPath = + j == 0 ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, + std::string(baseName, 0, 2), std::string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + Path logBz2Path = logPath + ".bz2"; + + if (pathExists(logPath)) { + return std::make_shared<std::string>(readFile(logPath)); + } + if (pathExists(logBz2Path)) { + try { + return decompress("bzip2", readFile(logBz2Path)); + } catch (Error&) { + } + } + } + + return nullptr; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc new file mode 100644 index 000000000000..aca305e1a52f --- /dev/null +++ b/third_party/nix/src/libstore/local-store.cc @@ -0,0 +1,1519 @@ +#include "libstore/local-store.hh" + +#include <algorithm> +#include <cerrno> +#include <cstdio> +#include <cstring> +#include <ctime> +#include <iostream> + +#include <absl/strings/numbers.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <grp.h> +#include <sched.h> +#include <sqlite3.h> +#include <sys/ioctl.h> +#include <sys/mount.h> +#include <sys/select.h> +#include <sys/stat.h> +#include <sys/statvfs.h> +#include <sys/time.h> +#include <sys/types.h> +#include <sys/xattr.h> +#include <unistd.h> +#include <utime.h> + +#include "generated/schema.sql.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/nar-info.hh" +#include "libstore/pathlocks.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/archive.hh" + +namespace nix { + +LocalStore::LocalStore(const Params& params) + : Store(params), + LocalFSStore(params), + realStoreDir_{this, false, + rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"}, + realStoreDir(realStoreDir_), + dbDir(stateDir + "/db"), + linksDir(realStoreDir + "/.links"), + reservedPath(dbDir + "/reserved"), + schemaPath(dbDir + "/schema"), + trashDir(realStoreDir + "/trash"), + tempRootsDir(stateDir + "/temproots"), + fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) { + auto state(_state.lock()); + + /* Create missing state directories if they don't already exist. */ + createDirs(realStoreDir); + makeStoreWritable(); + createDirs(linksDir); + Path profilesDir = stateDir + "/profiles"; + createDirs(profilesDir); + createDirs(tempRootsDir); + createDirs(dbDir); + Path gcRootsDir = stateDir + "/gcroots"; + if (!pathExists(gcRootsDir)) { + createDirs(gcRootsDir); + createSymlink(profilesDir, gcRootsDir + "/profiles"); + } + + for (auto& perUserDir : + {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) { + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + } + + // TODO(kanepyork): migrate to external constructor, this bypasses virtual + // dispatch + // NOLINTNEXTLINE clang-analyzer-optin.cplusplus.VirtualCall + createUser(getUserName(), getuid()); + + /* Optionally, create directories and set permissions for a + multi-user install. */ + if (getuid() == 0 && settings.buildUsersGroup != "") { + mode_t perm = 01775; + + struct group* gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (gr == nullptr) { + LOG(ERROR) << "warning: the group '" << settings.buildUsersGroup + << "' specified in 'build-users-group' does not exist"; + } else { + struct stat st; + if (stat(realStoreDir.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % + realStoreDir); + } + + if (st.st_uid != 0 || st.st_gid != gr->gr_gid || + (st.st_mode & ~S_IFMT) != perm) { + if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) { + throw SysError(format("changing ownership of path '%1%'") % + realStoreDir); + } + if (chmod(realStoreDir.c_str(), perm) == -1) { + throw SysError(format("changing permissions on path '%1%'") % + realStoreDir); + } + } + } + } + + /* Ensure that the store and its parents are not symlinks. */ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + Path path = realStoreDir; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting status of '%1%'") % path); + } + if (S_ISLNK(st.st_mode)) { + throw Error(format("the path '%1%' is a symlink; " + "this is not allowed for the Nix store and its " + "parent directories") % + path); + } + path = dirOf(path); + } + } + + /* We can't open a SQLite database if the disk is full. Since + this prevents the garbage collector from running when it's most + needed, we reserve some dummy space that we can free just + before doing a garbage collection. */ + try { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) { + AutoCloseFD fd( + open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600)); + int res = -1; +#if HAVE_POSIX_FALLOCATE + res = posix_fallocate(fd.get(), 0, settings.reservedSize); +#endif + if (res == -1) { + writeFull(fd.get(), std::string(settings.reservedSize, 'X')); + [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); + } + } + } catch (SysError& e) { /* don't care about errors */ + } + + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath, true); + + if (!lockFile(globalLock.get(), ltRead, false)) { + LOG(INFO) << "waiting for the big Nix store lock..."; + lockFile(globalLock.get(), ltRead, true); + } + + /* Check the current database schema and if necessary do an + upgrade. */ + int curSchema = getSchema(); + if (curSchema > nixSchemaVersion) { + throw Error( + format( + "current Nix store schema is version %1%, but I only support %2%") % + curSchema % nixSchemaVersion); + } + if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + openDB(*state, true); + writeFile(schemaPath, (format("%1%") % curSchema).str()); + } else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) { + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + } + + if (curSchema < 6) { + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + } + + if (!lockFile(globalLock.get(), ltWrite, false)) { + LOG(INFO) << "waiting for exclusive access to the Nix store..."; + lockFile(globalLock.get(), ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 7) { + upgradeStore7(); + } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ultimate integer"); + state->db.exec("alter table ValidPaths add column sigs text"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + state->db.exec("drop table FailedPaths"); + txn.commit(); + } + + if (curSchema < 10) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ca text"); + txn.commit(); + } + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock.get(), ltRead, true); + } else { + openDB(*state, false); + } + + /* Prepare SQL statements. */ + state->stmtRegisterValidPath.create( + state->db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, " + "ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); + state->stmtUpdatePathInfo.create( + state->db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca " + "= ? where path = ?;"); + state->stmtAddReference.create( + state->db, + "insert or replace into Refs (referrer, reference) values (?, ?);"); + state->stmtQueryPathInfo.create( + state->db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca " + "from ValidPaths where path = ?;"); + state->stmtQueryReferences.create(state->db, + "select path from Refs join ValidPaths on " + "reference = id where referrer = ?;"); + state->stmtQueryReferrers.create( + state->db, + "select path from Refs join ValidPaths on referrer = id where reference " + "= (select id from ValidPaths where path = ?);"); + state->stmtInvalidatePath.create(state->db, + "delete from ValidPaths where path = ?;"); + state->stmtAddDerivationOutput.create( + state->db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, " + "?);"); + state->stmtQueryValidDerivers.create( + state->db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv " + "= v.id where d.path = ?;"); + state->stmtQueryDerivationOutputs.create( + state->db, "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + state->stmtQueryPathFromHashPart.create( + state->db, "select path from ValidPaths where path >= ? limit 1;"); + state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); +} + +LocalStore::~LocalStore() { + std::shared_future<void> future; + + { + auto state(_state.lock()); + if (state->gcRunning) { + future = state->gcFuture; + } + } + + if (future.valid()) { + LOG(INFO) << "waiting for auto-GC to finish on exit..."; + future.get(); + } + + try { + auto state(_state.lock()); + if (state->fdTempRoots) { + state->fdTempRoots = AutoCloseFD(-1); + unlink(fnTempRoots.c_str()); + } + } catch (...) { + ignoreException(); + } +} + +std::string LocalStore::getUri() { return "local"; } + +int LocalStore::getSchema() { + int curSchema = 0; + if (pathExists(schemaPath)) { + std::string s = readFile(schemaPath); + if (!absl::SimpleAtoi(s, &curSchema)) { + throw Error(format("'%1%' is corrupt") % schemaPath); + } + } + return curSchema; +} + +void LocalStore::openDB(State& state, bool create) { + if (access(dbDir.c_str(), R_OK | W_OK) != 0) { + throw SysError(format("Nix database directory '%1%' is not writable") % + dbDir); + } + + /* Open the Nix database. */ + std::string dbPath = dbDir + "/db.sqlite"; + auto& db(state.db); + if (sqlite3_open_v2(dbPath.c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), + nullptr) != SQLITE_OK) { + throw Error(format("cannot open Nix database '%1%'") % dbPath); + } + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) { + throwSQLiteError(db, "setting timeout"); + } + + db.exec("pragma foreign_keys = 1"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + std::string syncMode = settings.fsyncMetadata ? "normal" : "off"; + db.exec("pragma synchronous = " + syncMode); + + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + std::string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + std::string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) { + throwSQLiteError(db, "querying journal mode"); + } + prevMode = std::string( + reinterpret_cast<const char*>(sqlite3_column_text(stmt, 0))); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), + nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "setting journal mode"); + } + + /* Increase the auto-checkpoint interval to 40000 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", + nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "setting autocheckpoint interval"); + } + + /* Initialise the database schema, if necessary. */ + if (create) { + db.exec(kNixSqlSchema); + } +} + +/* To improve purity, users may want to make the Nix store a read-only + bind mount. So make the Nix store writable for this process. */ +void LocalStore::makeStoreWritable() { + if (getuid() != 0) { + return; + } + /* Check if /nix/store is on a read-only mount. */ + struct statvfs stat; + if (statvfs(realStoreDir.c_str(), &stat) != 0) { + throw SysError("getting info about the Nix store mount point"); + } + + if ((stat.f_flag & ST_RDONLY) != 0u) { + if (unshare(CLONE_NEWNS) == -1) { + throw SysError("setting up a private mount namespace"); + } + + if (mount(nullptr, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, + nullptr) == -1) { + throw SysError(format("remounting %1% writable") % realStoreDir); + } + } +} + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + +static void canonicaliseTimestampAndPermissions(const Path& path, + const struct stat& st) { + if (!S_ISLNK(st.st_mode)) { + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) | 0444 | + ((st.st_mode & S_IXUSR) != 0u ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) { + throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + } + } + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) { +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) { +#endif + throw SysError(format("changing modification time of '%1%'") % path); + } + } + } // namespace nix +} // namespace nix + +void canonicaliseTimestampAndPermissions(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + canonicaliseTimestampAndPermissions(path, st); +} + +static void canonicalisePathMetaData_(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen) { + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) { + throw Error(format("file '%1%' has an unsupported type") % path); + } + + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) { + throw SysError("querying extended attributes of '%s'", path); + } + } else if (eaSize > 0) { + std::vector<char> eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) { + throw SysError("querying extended attributes of '%s'", path); + } + + for (auto& eaName : absl::StrSplit(std::string(eaBuf.data(), eaSize), + absl::ByString(std::string("\000", 1)), + absl::SkipEmpty())) { + /* Ignore SELinux security labels since these cannot be + removed even by root. */ + if (eaName == "security.selinux") { + continue; + } + if (lremovexattr(path.c_str(), std::string(eaName).c_str()) == -1) { + throw SysError("removing extended attribute '%s' from '%s'", eaName, + path); + } + } + } + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (fromUid != static_cast<uid_t>(-1) && st.st_uid != fromUid) { + if (S_ISDIR(st.st_mode)) { + throw BuildError(format("invalid file '%1%': is a directory") % path); + } + if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) { + throw BuildError(format("invalid ownership on file '%1%'") % path); + } + if (!(S_ISLNK(st.st_mode) || + (st.st_uid == geteuid() && + ((st.st_mode & ~S_IFMT) == 0444 || (st.st_mode & ~S_IFMT) == 0555) && + st.st_mtime == mtimeStore))) { + throw BuildError( + format("invalid permissions on file '%1%', should be 0444/0555") % + path); + } + + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) { +#else + if (!S_ISLNK(st.st_mode) && chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError(format("changing owner of '%1%' to %2%") % path % + geteuid()); + } + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto& i : entries) { + canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + } + } +} + +void canonicalisePathMetaData(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen) { + canonicalisePathMetaData_(path, fromUid, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error(format("wrong ownership of top-level store path '%1%'") % path); + } +} + +void canonicalisePathMetaData(const Path& path, uid_t fromUid) { + InodesSeen inodesSeen; + canonicalisePathMetaData(path, fromUid, inodesSeen); +} + +void LocalStore::checkDerivationOutputs(const Path& drvPath, + const Derivation& drv) { + std::string drvName = storePathToName(drvPath); + assert(isDerivation(drvName)); + drvName = std::string(drvName, 0, drvName.size() - drvExtension.size()); + + if (drv.isFixedOutput()) { + auto out = drv.outputs.find("out"); + if (out == drv.outputs.end()) { + throw Error( + format("derivation '%1%' does not have an output named 'out'") % + drvPath); + } + + bool recursive; + Hash h; + out->second.parseHashInfo(recursive, h); + Path outPath = makeFixedOutputPath(recursive, h, drvName); + + auto j = drv.env.find("out"); + if (out->second.path != outPath || j == drv.env.end() || + j->second != outPath) { + throw Error( + format( + "derivation '%1%' has incorrect output '%2%', should be '%3%'") % + drvPath % out->second.path % outPath); + } + } + + else { + Derivation drvCopy(drv); + for (auto& i : drvCopy.outputs) { + i.second.path = ""; + drvCopy.env[i.first] = ""; + } + + Hash h = hashDerivationModulo(*this, drvCopy); + + for (auto& i : drv.outputs) { + Path outPath = makeOutputPath(i.first, h, drvName); + auto j = drv.env.find(i.first); + if (i.second.path != outPath || j == drv.env.end() || + j->second != outPath) { + throw Error(format("derivation '%1%' has incorrect output '%2%', " + "should be '%3%'") % + drvPath % i.second.path % outPath); + } + } + } +} + +uint64_t LocalStore::addValidPath(State& state, const ValidPathInfo& info, + bool checkOutputs) { + if (!info.ca.empty() && !info.isContentAddressed(*this)) { + throw Error( + "cannot add path '%s' to the Nix store because it claims to be " + "content-addressed but isn't", + info.path); + } + + state.stmtRegisterValidPath + .use()(info.path)(info.narHash.to_string(Base16))( + info.registrationTime == 0 ? time(nullptr) : info.registrationTime)( + info.deriver, !info.deriver.empty())(info.narSize, info.narSize != 0)( + info.ultimate ? 1 : 0, info.ultimate)( + concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ca, !info.ca.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + + /* Verify that the output paths in the derivation are correct + (i.e., follow the scheme for computing output paths from + derivations). Note that if this throws an error, then the + DB transaction is rolled back, so the path validity + registration above is undone. */ + if (checkOutputs) { + checkDerivationOutputs(info.path, drv); + } + + for (auto& i : drv.outputs) { + state.stmtAddDerivationOutput.use()(id)(i.first)(i.second.path).exec(); + } + } + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), + std::make_shared<ValidPathInfo>(info)); + } + + return id; +} + +void LocalStore::queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + try { + auto info = std::make_shared<ValidPathInfo>(); + info->path = path; + + assertStorePath(path); + + callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + auto state(_state.lock()); + + /* Get the path info. */ + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + + if (!useQueryPathInfo.next()) { + return std::shared_ptr<ValidPathInfo>(); + } + + info->id = useQueryPathInfo.getInt(0); + + auto hash_ = Hash::deserialize(useQueryPathInfo.getStr(1)); + if (!hash_.ok()) { + throw Error(absl::StrCat("in valid-path entry for '", path, + "': ", hash_.status().ToString())); + } + info->narHash = *hash_; + + info->registrationTime = useQueryPathInfo.getInt(2); + + auto s = reinterpret_cast<const char*>( + sqlite3_column_text(state->stmtQueryPathInfo, 3)); + if (s != nullptr) { + info->deriver = s; + } + + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); + + info->ultimate = useQueryPathInfo.getInt(5) == 1; + + s = reinterpret_cast<const char*>( + sqlite3_column_text(state->stmtQueryPathInfo, 6)); + if (s != nullptr) { + info->sigs = absl::StrSplit(s, absl::ByChar(' '), absl::SkipEmpty()); + } + + s = reinterpret_cast<const char*>( + sqlite3_column_text(state->stmtQueryPathInfo, 7)); + if (s != nullptr) { + info->ca = s; + } + + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + + while (useQueryReferences.next()) { + info->references.insert(useQueryReferences.getStr(0)); + } + + return info; + })); + + } catch (...) { + callback.rethrow(); + } +} + +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State& state, const ValidPathInfo& info) { + state.stmtUpdatePathInfo + .use()(info.narSize, info.narSize != 0)(info.narHash.to_string(Base16))( + info.ultimate ? 1 : 0, info.ultimate)( + concatStringsSep(" ", info.sigs), !info.sigs.empty())( + info.ca, !info.ca.empty())(info.path) + .exec(); +} + +uint64_t LocalStore::queryValidPathId(State& state, const Path& path) { + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) { + throw Error(format("path '%1%' is not valid") % path); + } + return use.getInt(0); +} + +bool LocalStore::isValidPath_(State& state, const Path& path) { + return state.stmtQueryPathInfo.use()(path).next(); +} + +bool LocalStore::isValidPathUncached(const Path& path) { + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); +} + +PathSet LocalStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + PathSet res; + for (auto& i : paths) { + if (isValidPath(i)) { + res.insert(i); + } + } + return res; +} + +PathSet LocalStore::queryAllValidPaths() { + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); + PathSet res; + while (use.next()) { + res.insert(use.getStr(0)); + } + return res; + }); +} + +void LocalStore::queryReferrers(State& state, const Path& path, + PathSet& referrers) { + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); + + while (useQueryReferrers.next()) { + referrers.insert(useQueryReferrers.getStr(0)); + } +} + +void LocalStore::queryReferrers(const Path& path, PathSet& referrers) { + assertStorePath(path); + return retrySQLite<void>([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); +} + +PathSet LocalStore::queryValidDerivers(const Path& path) { + assertStorePath(path); + + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + + PathSet derivers; + while (useQueryValidDerivers.next()) { + derivers.insert(useQueryValidDerivers.getStr(1)); + } + + return derivers; + }); +} + +PathSet LocalStore::queryDerivationOutputs(const Path& path) { + return retrySQLite<PathSet>([&]() { + auto state(_state.lock()); + + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( + queryValidPathId(*state, path))); + + PathSet outputs; + while (useQueryDerivationOutputs.next()) { + outputs.insert(useQueryDerivationOutputs.getStr(1)); + } + + return outputs; + }); +} + +StringSet LocalStore::queryDerivationOutputNames(const Path& path) { + return retrySQLite<StringSet>([&]() { + auto state(_state.lock()); + + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()( + queryValidPathId(*state, path))); + + StringSet outputNames; + while (useQueryDerivationOutputs.next()) { + outputNames.insert(useQueryDerivationOutputs.getStr(0)); + } + + return outputNames; + }); +} + +Path LocalStore::queryPathFromHashPart(const std::string& hashPart) { + if (hashPart.size() != storePathHashLen) { + throw Error("invalid hash part"); + } + + Path prefix = storeDir + "/" + hashPart; + + return retrySQLite<Path>([&]() -> std::string { + auto state(_state.lock()); + + auto useQueryPathFromHashPart( + state->stmtQueryPathFromHashPart.use()(prefix)); + + if (!useQueryPathFromHashPart.next()) { + return ""; + } + + const char* s = reinterpret_cast<const char*>( + sqlite3_column_text(state->stmtQueryPathFromHashPart, 0)); + return (s != nullptr) && + prefix.compare(0, prefix.size(), s, prefix.size()) == 0 + ? s + : ""; + }); +} + +PathSet LocalStore::querySubstitutablePaths(const PathSet& paths) { + if (!settings.useSubstitutes) { + return PathSet(); + } + + auto remaining = paths; + PathSet res; + + for (auto& sub : getDefaultSubstituters()) { + if (remaining.empty()) { + break; + } + if (sub->storeDir != storeDir) { + continue; + } + if (!sub->wantMassQuery()) { + continue; + } + + auto valid = sub->queryValidPaths(remaining); + + PathSet remaining2; + for (auto& path : remaining) { + if (valid.count(path) != 0u) { + res.insert(path); + } else { + remaining2.insert(path); + } + } + + std::swap(remaining, remaining2); + } + + return res; +} + +void LocalStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + if (!settings.useSubstitutes) { + return; + } + for (auto& sub : getDefaultSubstituters()) { + if (sub->storeDir != storeDir) { + continue; + } + for (auto& path : paths) { + if (infos.count(path) != 0u) { + continue; + } + DLOG(INFO) << "checking substituter '" << sub->getUri() << "' for path '" + << path << "'"; + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + infos[path] = SubstitutablePathInfo{info->deriver, info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath&) { + } catch (SubstituterDisabled&) { + } catch (Error& e) { + if (settings.tryFallback) { + LOG(ERROR) << e.what(); + } else { + throw; + } + } + } + } +} + +void LocalStore::registerValidPath(const ValidPathInfo& info) { + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + +void LocalStore::registerValidPaths(const ValidPathInfos& infos) { + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ + if (settings.syncBeforeRegistering) { + sync(); + } + + return retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + PathSet paths; + + for (auto& i : infos) { + assert(i.narHash.type == htSHA256); + if (isValidPath_(*state, i.path)) { + updatePathInfo(*state, i); + } else { + addValidPath(*state, i, false); + } + paths.insert(i.path); + } + + for (auto& i : infos) { + auto referrer = queryValidPathId(*state, i.path); + for (auto& j : i.references) { + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)) + .exec(); + } + } + + /* Check that the derivation outputs are correct. We can't do + this in addValidPath() above, because the references might + not be valid yet. */ + for (auto& i : infos) { + if (isDerivation(i.path)) { + // FIXME: inefficient; we already loaded the + // derivation in addValidPath(). + Derivation drv = + readDerivation(realStoreDir + "/" + baseNameOf(i.path)); + checkDerivationOutputs(i.path, drv); + } + } + + /* Do a topological sort of the paths. This will throw an + error if a cycle is detected and roll back the + transaction. Cycles can only occur when a derivation + has multiple outputs. */ + topoSortPaths(paths); + + txn.commit(); + }); +} + +/* Invalidate a path. The caller is responsible for checking that + there are no referrers. */ +void LocalStore::invalidatePath(State& state, const Path& path) { + LOG(INFO) << "invalidating path '" << path << "'"; + + state.stmtInvalidatePath.use()(path).exec(); + + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } +} + +const PublicKeys& LocalStore::getPublicKeys() { + auto state(_state.lock()); + if (!state->publicKeys) { + state->publicKeys = std::make_unique<PublicKeys>(getDefaultPublicKeys()); + } + return *state->publicKeys; +} + +void LocalStore::addToStore(const ValidPathInfo& info, Source& source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + if (!info.narHash) { + throw Error("cannot add path '%s' because it lacks a hash", info.path); + } + + if (requireSigs && (checkSigs != 0u) && + (info.checkSignatures(*this, getPublicKeys()) == 0u)) { + throw Error("cannot add path '%s' because it lacks a valid signature", + info.path); + } + + addTempRoot(info.path); + + if ((repair != 0u) || !isValidPath(info.path)) { + PathLocks outputLock; + + Path realPath = realStoreDir + "/" + baseNameOf(info.path); + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + if (locksHeld.count(info.path) == 0u) { + outputLock.lockPaths({realPath}); + } + + if ((repair != 0u) || !isValidPath(info.path)) { + deletePath(realPath); + + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(htSHA256); + + LambdaSource wrapperSource( + [&](unsigned char* data, size_t len) -> size_t { + size_t n = source.read(data, len); + hashSink(data, n); + return n; + }); + + restorePath(realPath, wrapperSource); + + auto hashResult = hashSink.finish(); + + if (hashResult.first != info.narHash) { + throw Error( + "hash mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narHash.to_string(), hashResult.first.to_string()); + } + + if (hashResult.second != info.narSize) { + throw Error( + "size mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narSize, hashResult.second); + } + + autoGC(); + + canonicalisePathMetaData(realPath, -1); + + optimisePath(realPath); // FIXME: combine with hashPath() + + registerValidPath(info); + } + + outputLock.setDeletion(true); + } +} + +Path LocalStore::addToStoreFromDump(const std::string& dump, + const std::string& name, bool recursive, + HashType hashAlgo, RepairFlag repair) { + Hash h = hashString(hashAlgo, dump); + + Path dstPath = makeFixedOutputPath(recursive, h, name); + + addTempRoot(dstPath); + + if ((repair != 0u) || !isValidPath(dstPath)) { + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if ((repair != 0u) || !isValidPath(dstPath)) { + deletePath(realPath); + + autoGC(); + + if (recursive) { + StringSource source(dump); + restorePath(realPath, source); + } else { + writeFile(realPath, dump); + } + + canonicalisePathMetaData(realPath, -1); + + /* Register the SHA-256 hash of the NAR serialisation of + the path in the database. We may just have computed it + above (if called with recursive == true and hashAlgo == + sha256); otherwise, compute it here. */ + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else { + hash = hashPath(htSHA256, realPath); + } + + optimisePath(realPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.narHash = hash.first; + info.narSize = hash.second; + info.ca = makeFixedOutputCA(recursive, h); + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + +Path LocalStore::addToStore(const std::string& name, const Path& _srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + Path srcPath(absPath(_srcPath)); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) { + dumpPath(srcPath, sink, filter); + } else { + sink.s = make_ref<std::string>(readFile(srcPath)); + } + + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); +} + +Path LocalStore::addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair) { + auto hash = hashString(htSHA256, s); + auto dstPath = makeTextPath(name, hash, references); + + addTempRoot(dstPath); + + if ((repair != 0u) || !isValidPath(dstPath)) { + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if ((repair != 0u) || !isValidPath(dstPath)) { + deletePath(realPath); + + autoGC(); + + writeFile(realPath, s); + + canonicalisePathMetaData(realPath, -1); + + StringSink sink; + dumpString(s, sink); + auto narHash = hashString(htSHA256, *sink.s); + + optimisePath(realPath); + + ValidPathInfo info; + info.path = dstPath; + info.narHash = narHash; + info.narSize = sink.s->size(); + info.references = references; + info.ca = "text:" + hash.to_string(); + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() { + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(realStoreDir); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + +void LocalStore::invalidatePathChecked(const Path& path) { + assertStorePath(path); + + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + if (isValidPath_(*state, path)) { + PathSet referrers; + queryReferrers(*state, path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) { + throw PathInUse( + format("cannot delete path '%1%' because it is in use by %2%") % + path % showPaths(referrers)); + } + invalidatePath(*state, path); + } + + txn.commit(); + }); +} + +bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { + LOG(INFO) << "reading the Nix store..."; + + bool errors = false; + + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + PathSet store; + for (auto& i : readDirectory(realStoreDir)) { + store.insert(i.name); + } + + /* Check whether all valid paths actually exist. */ + LOG(INFO) << "checking path existence..."; + + PathSet validPaths2 = queryAllValidPaths(); + PathSet validPaths; + PathSet done; + + fdGCLock = AutoCloseFD(-1); + + for (auto& i : validPaths2) { + verifyPath(i, store, done, validPaths, repair, errors); + } + + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + LOG(INFO) << "checking hashes..."; + + Hash nullHash(htSHA256); + + for (auto& i : validPaths) { + try { + auto info = std::const_pointer_cast<ValidPathInfo>( + std::shared_ptr<const ValidPathInfo>(queryPathInfo(i))); + + /* Check the content hash (optionally - slow). */ + DLOG(INFO) << "checking contents of '" << i << "'"; + HashResult current = hashPath(info->narHash.type, toRealPath(i)); + + if (info->narHash != nullHash && info->narHash != current.first) { + LOG(ERROR) << "path '" << i << "' was modified! expected hash '" + << info->narHash.to_string() << "', got '" + << current.first.to_string() << "'"; + if (repair != 0u) { + repairPath(i); + } else { + errors = true; + } + } else { + bool update = false; + + /* Fill in missing hashes. */ + if (info->narHash == nullHash) { + LOG(WARNING) << "fixing missing hash on '" << i << "'"; + info->narHash = current.first; + update = true; + } + + /* Fill in missing narSize fields (from old stores). */ + if (info->narSize == 0) { + LOG(ERROR) << "updating size field on '" << i << "' to " + << current.second; + info->narSize = current.second; + update = true; + } + + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } + } + + } catch (Error& e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(i)) { + LOG(ERROR) << e.msg(); + } else { + LOG(WARNING) << e.msg(); + } + errors = true; + } + } + } + + return errors; +} + +void LocalStore::verifyPath(const Path& path, const PathSet& store, + PathSet& done, PathSet& validPaths, + RepairFlag repair, bool& errors) { + checkInterrupt(); + + if (done.find(path) != done.end()) { + return; + } + done.insert(path); + + if (!isStorePath(path)) { + LOG(ERROR) << "path '" << path << "' is not in the Nix store"; + auto state(_state.lock()); + invalidatePath(*state, path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; + queryReferrers(path, referrers); + for (auto& i : referrers) { + if (i != path) { + verifyPath(i, store, done, validPaths, repair, errors); + if (validPaths.find(i) != validPaths.end()) { + canInvalidate = false; + } + } + } + + if (canInvalidate) { + LOG(WARNING) << "path '" << path + << "' disappeared, removing from database..."; + auto state(_state.lock()); + invalidatePath(*state, path); + } else { + LOG(ERROR) << "path '" << path + << "' disappeared, but it still has valid referrers!"; + if (repair != 0u) { + try { + repairPath(path); + } catch (Error& e) { + LOG(WARNING) << e.msg(); + errors = true; + } + } else { + errors = true; + } + } + + return; + } + + validPaths.insert(path); +} + +unsigned int LocalStore::getProtocol() { return PROTOCOL_VERSION; } + +#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && \ + defined(FS_IMMUTABLE_FL) + +static void makeMutable(const Path& path) { + checkInterrupt(); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) { + return; + } + + if (S_ISDIR(st.st_mode)) { + for (auto& i : readDirectory(path)) { + makeMutable(path + "/" + i.name); + } + } + + /* The O_NOFOLLOW is important to prevent us from changing the + mutable bit on the target of a symlink (which would be a + security hole). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd == -1) { + if (errno == ELOOP) { + return; + } // it's a symlink + throw SysError(format("opening file '%1%'") % path); + } + + unsigned int flags = 0, old; + + /* Silently ignore errors getting/setting the immutable flag so + that we work correctly on filesystems that don't support it. */ + if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) { + return; + } + old = flags; + flags &= ~FS_IMMUTABLE_FL; + if (old == flags) { + return; + } + if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) { + return; + } +} + +/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ +void LocalStore::upgradeStore7() { + if (getuid() != 0) { + return; + } + printError( + "removing immutable bits from the Nix store (this may take a while)..."); + makeMutable(realStoreDir); +} + +#else + +void LocalStore::upgradeStore7() {} + +#endif + +void LocalStore::vacuumDB() { + auto state(_state.lock()); + state->db.exec("vacuum"); +} + +void LocalStore::addSignatures(const Path& storePath, const StringSet& sigs) { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = std::const_pointer_cast<ValidPathInfo>( + std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + + info->sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, *info); + + txn.commit(); + }); +} + +void LocalStore::signPathInfo(ValidPathInfo& info) { + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto& secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } +} + +void LocalStore::createUser(const std::string& userName, uid_t userId) { + for (auto& dir : {fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName)}) { + createDirs(dir); + if (chmod(dir.c_str(), 0755) == -1) { + throw SysError("changing permissions of directory '%s'", dir); + } + if (chown(dir.c_str(), userId, getgid()) == -1) { + throw SysError("changing owner of directory '%s'", dir); + } + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/local-store.hh b/third_party/nix/src/libstore/local-store.hh new file mode 100644 index 000000000000..a7c49079d22f --- /dev/null +++ b/third_party/nix/src/libstore/local-store.hh @@ -0,0 +1,319 @@ +#pragma once + +#include <chrono> +#include <future> +#include <string> +#include <unordered_set> + +#include <absl/status/status.h> +#include <absl/strings/str_split.h> + +#include "libstore/pathlocks.hh" +#include "libstore/sqlite.hh" +#include "libstore/store-api.hh" +#include "libutil/sync.hh" +#include "libutil/util.hh" + +namespace nix { + +/* Nix store and database schema version. Version 1 (or 0) was Nix <= + 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +const int nixSchemaVersion = 10; + +struct Derivation; + +struct OptimiseStats { + unsigned long filesLinked = 0; + unsigned long long bytesFreed = 0; + unsigned long long blocksFreed = 0; +}; + +class LocalStore : public LocalFSStore { + private: + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; + + struct State { + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; + + /* The file to which we write our temporary roots. */ + AutoCloseFD fdTempRoots; + + /* The last time we checked whether to do an auto-GC, or an + auto-GC finished. */ + std::chrono::time_point<std::chrono::steady_clock> lastGCCheck; + + /* Whether auto-GC is running. If so, get gcFuture to wait for + the GC to finish. */ + bool gcRunning = false; + std::shared_future<void> gcFuture; + + /* How much disk space was available after the previous + auto-GC. If the current available disk space is below + minFree but not much below availAfterGC, then there is no + point in starting a new GC. */ + uint64_t availAfterGC = std::numeric_limits<uint64_t>::max(); + + std::unique_ptr<PublicKeys> publicKeys; + }; + + Sync<State, std::recursive_mutex> _state; + + public: + PathSetting realStoreDir_; + + const Path realStoreDir; + const Path dbDir; + const Path linksDir; + const Path reservedPath; + const Path schemaPath; + const Path trashDir; + const Path tempRootsDir; + const Path fnTempRoots; + + private: + Setting<bool> requireSigs{ + (Store*)this, settings.requireSigs, "require-sigs", + "whether store paths should have a trusted signature on import"}; + + const PublicKeys& getPublicKeys(); + + public: + // Hack for build-remote.cc. + // TODO(tazjin): remove this when we've got gRPC + PathSet locksHeld = + absl::StrSplit(getEnv("NIX_HELD_LOCKS").value_or(""), + absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + + /* Initialise the local store, upgrading the schema if + necessary. */ + LocalStore(const Params& params); + + ~LocalStore(); + + /* Implementations of abstract store API methods. */ + + std::string getUri() override; + + bool isValidPathUncached(const Path& path) override; + + PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute = + NoSubstitute) override; + + PathSet queryAllValidPaths() override; + + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + + void queryReferrers(const Path& path, PathSet& referrers) override; + + PathSet queryValidDerivers(const Path& path) override; + + PathSet queryDerivationOutputs(const Path& path) override; + + StringSet queryDerivationOutputNames(const Path& path) override; + + Path queryPathFromHashPart(const std::string& hashPart) override; + + PathSet querySubstitutablePaths(const PathSet& paths) override; + + void querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) override; + + void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override; + + Path addToStore(const std::string& name, const Path& srcPath, bool recursive, + HashType hashAlgo, PathFilter& filter, + RepairFlag repair) override; + + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ + Path addToStoreFromDump(const std::string& dump, const std::string& name, + bool recursive = true, HashType hashAlgo = htSHA256, + RepairFlag repair = NoRepair); + + Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair) override; + + absl::Status buildPaths(std::ostream& log_sink, const PathSet& paths, + BuildMode build_mode) override; + + BuildResult buildDerivation(std::ostream& log_sink, const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) override; + + void ensurePath(const Path& path) override; + + void addTempRoot(const Path& path) override; + + void addIndirectRoot(const Path& path) override; + + void syncWithGC() override; + + private: + typedef std::shared_ptr<AutoCloseFD> FDPtr; + using FDs = std::list<FDPtr>; + + void findTempRoots(FDs& fds, Roots& roots, bool censor); + + public: + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions& options, GCResults& results) override; + + /* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ + void optimiseStore(OptimiseStats& stats); + + void optimiseStore() override; + + /* Optimise a single store path. */ + void optimisePath(const Path& path); + + bool verifyStore(bool checkContents, RepairFlag repair) override; + + /* Register the validity of a path, i.e., that `path' exists, that + the paths referenced by it exists, and in the case of an output + path of a derivation, that it has been produced by a successful + execution of the derivation (or something equivalent). Also + register the hash of the file system contents of the path. The + hash must be a SHA-256 hash. */ + void registerValidPath(const ValidPathInfo& info); + + void registerValidPaths(const ValidPathInfos& infos); + + unsigned int getProtocol() override; + + void vacuumDB(); + + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + void repairPath(const Path& path); + + void addSignatures(const Path& storePath, const StringSet& sigs) override; + + /* If free disk space in /nix/store if below minFree, delete + garbage until it exceeds maxFree. */ + void autoGC(bool sync = true); + + private: + int getSchema(); + + void openDB(State& state, bool create); + + void makeStoreWritable(); + + static uint64_t queryValidPathId(State& state, const Path& path); + + uint64_t addValidPath(State& state, const ValidPathInfo& info, + bool checkOutputs = true); + + void invalidatePath(State& state, const Path& path); + + /* Delete a path from the Nix store. */ + void invalidatePathChecked(const Path& path); + + void verifyPath(const Path& path, const PathSet& store, PathSet& done, + PathSet& validPaths, RepairFlag repair, bool& errors); + + static void updatePathInfo(State& state, const ValidPathInfo& info); + + void upgradeStore6(); + void upgradeStore7(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path& path); + + struct GCState; + + static void deleteGarbage(GCState& state, const Path& path); + + void tryToDelete(GCState& state, const Path& path); + + bool canReachRoot(GCState& state, PathSet& visited, const Path& path); + + void deletePathRecursive(GCState& state, const Path& path); + + static bool isActiveTempFile(const GCState& state, const Path& path, + const std::string& suffix); + + AutoCloseFD openGCLock(LockType lockType); + + void findRoots(const Path& path, unsigned char type, Roots& roots); + + void findRootsNoTemp(Roots& roots, bool censor); + + void findRuntimeRoots(Roots& roots, bool censor); + + void removeUnusedLinks(const GCState& state); + + Path createTempDirInStore(); + + void checkDerivationOutputs(const Path& drvPath, const Derivation& drv); + + using InodeHash = std::unordered_set<ino_t>; + + InodeHash loadInodeHash(); + static Strings readDirectoryIgnoringInodes(const Path& path, + const InodeHash& inodeHash); + void optimisePath_(OptimiseStats& stats, const Path& path, + InodeHash& inodeHash); + + // Internal versions that are not wrapped in retry_sqlite. + static bool isValidPath_(State& state, const Path& path); + static void queryReferrers(State& state, const Path& path, + PathSet& referrers); + + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + static void signPathInfo(ValidPathInfo& info); + + Path getRealStoreDir() override { return realStoreDir; } + + void createUser(const std::string& userName, uid_t userId) override; + + friend class DerivationGoal; + friend class SubstitutionGoal; +}; + +using Inode = std::pair<dev_t, ino_t>; +using InodesSeen = std::set<Inode>; + +/* "Fix", or canonicalise, the meta-data of the files in a store path + after it has been built. In particular: + - the last modification date on each file is set to 1 (i.e., + 00:00:01 1/1/1970 UTC) + - the permissions are set of 444 or 555 (i.e., read-only with or + without execute permission; setuid bits etc. are cleared) + - the owner and group are set to the Nix user and group, if we're + running as root. */ +void canonicalisePathMetaData(const Path& path, uid_t fromUid, + InodesSeen& inodesSeen); +void canonicalisePathMetaData(const Path& path, uid_t fromUid); + +void canonicaliseTimestampAndPermissions(const Path& path); + +MakeError(PathInUse, Error); + +} // namespace nix diff --git a/third_party/nix/src/libstore/machines.cc b/third_party/nix/src/libstore/machines.cc new file mode 100644 index 000000000000..57c89e06924b --- /dev/null +++ b/third_party/nix/src/libstore/machines.cc @@ -0,0 +1,114 @@ +#include "libstore/machines.hh" + +#include <algorithm> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> +#include <absl/strings/string_view.h> +#include <glog/logging.h> + +#include "libstore/globals.hh" +#include "libutil/util.hh" + +namespace nix { + +Machine::Machine(decltype(storeUri)& storeUri, + decltype(systemTypes)& systemTypes, decltype(sshKey)& sshKey, + decltype(maxJobs) maxJobs, decltype(speedFactor) speedFactor, + decltype(supportedFeatures)& supportedFeatures, + decltype(mandatoryFeatures)& mandatoryFeatures, + decltype(sshPublicHostKey)& sshPublicHostKey) + : storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos || + absl::StartsWith(storeUri, "local") || + absl::StartsWith(storeUri, "remote") || + absl::StartsWith(storeUri, "auto") || + absl::StartsWith(storeUri, "/") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) {} + +bool Machine::allSupported(const std::set<std::string>& features) const { + return std::all_of(features.begin(), features.end(), + [&](const std::string& feature) { + return (supportedFeatures.count(feature) != 0u) || + (mandatoryFeatures.count(feature) != 0u); + }); +} + +bool Machine::mandatoryMet(const std::set<std::string>& features) const { + return std::all_of( + mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const std::string& feature) { return features.count(feature); }); +} + +void parseMachines(const std::string& s, Machines& machines) { + for (auto line : + absl::StrSplit(s, absl::ByAnyChar("\n;"), absl::SkipEmpty())) { + // Skip empty lines & comments + line = absl::StripAsciiWhitespace(line); + if (line.empty() || line[line.find_first_not_of(" \t")] == '#') { + continue; + } + + if (line[0] == '@') { + auto file = absl::StripAsciiWhitespace(line.substr(1)); + try { + parseMachines(readFile(file), machines); + } catch (const SysError& e) { + if (e.errNo != ENOENT) { + throw; + } + DLOG(INFO) << "cannot find machines file: " << file; + } + continue; + } + + std::vector<std::string> tokens = + absl::StrSplit(line, absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + auto sz = tokens.size(); + if (sz < 1) { + throw FormatError("bad machine specification '%s'", line); + } + + auto isSet = [&](size_t n) { + return tokens.size() > n && !tokens[n].empty() && tokens[n] != "-"; + }; + + // TODO(tazjin): what??? + machines.emplace_back( + tokens[0], + isSet(1) + ? absl::StrSplit(tokens[1], absl::ByChar(','), absl::SkipEmpty()) + : std::vector<std::string>{settings.thisSystem}, + isSet(2) ? tokens[2] : "", isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) + ? absl::StrSplit(tokens[5], absl::ByChar(','), absl::SkipEmpty()) + : std::set<std::string>{}, + isSet(6) + ? absl::StrSplit(tokens[6], absl::ByChar(','), absl::SkipEmpty()) + : std::set<std::string>{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() { + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); + return machines; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/machines.hh b/third_party/nix/src/libstore/machines.hh new file mode 100644 index 000000000000..0e7269723707 --- /dev/null +++ b/third_party/nix/src/libstore/machines.hh @@ -0,0 +1,36 @@ +#pragma once + +#include "libutil/types.hh" + +namespace nix { + +struct Machine { + const std::string storeUri; + const std::vector<std::string> systemTypes; + const std::string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set<std::string> supportedFeatures; + const std::set<std::string> mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set<std::string>& features) const; + + bool mandatoryMet(const std::set<std::string>& features) const; + + Machine(decltype(storeUri)& storeUri, decltype(systemTypes)& systemTypes, + decltype(sshKey)& sshKey, decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures)& supportedFeatures, + decltype(mandatoryFeatures)& mandatoryFeatures, + decltype(sshPublicHostKey)& sshPublicHostKey); +}; + +typedef std::vector<Machine> Machines; + +void parseMachines(const std::string& s, Machines& machines); + +Machines getMachines(); + +} // namespace nix diff --git a/third_party/nix/src/libstore/misc.cc b/third_party/nix/src/libstore/misc.cc new file mode 100644 index 000000000000..44e67ada369c --- /dev/null +++ b/third_party/nix/src/libstore/misc.cc @@ -0,0 +1,331 @@ +#include <glog/logging.h> + +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libstore/parsed-derivations.hh" +#include "libstore/store-api.hh" +#include "libutil/thread-pool.hh" + +namespace nix { + +void Store::computeFSClosure(const PathSet& startPaths, PathSet& paths_, + bool flipDirection, bool includeOutputs, + bool includeDerivers) { + struct State { + size_t pending; + PathSet& paths; + std::exception_ptr exc; + }; + + Sync<State> state_(State{0, paths_, nullptr}); + + std::function<void(const Path&)> enqueue; + + std::condition_variable done; + + enqueue = [&](const Path& path) -> void { + { + auto state(state_.lock()); + if (state->exc) { + return; + } + if (state->paths.count(path) != 0u) { + return; + } + state->paths.insert(path); + state->pending++; + } + + queryPathInfo( + path, + Callback<ref<ValidPathInfo>>( + [&, path](std::future<ref<ValidPathInfo>> fut) { + // FIXME: calls to isValidPath() should be async + + try { + auto info = fut.get(); + + if (flipDirection) { + PathSet referrers; + queryReferrers(path, referrers); + for (auto& ref : referrers) { + if (ref != path) { + enqueue(ref); + } + } + + if (includeOutputs) { + for (auto& i : queryValidDerivers(path)) { + enqueue(i); + } + } + + if (includeDerivers && isDerivation(path)) { + for (auto& i : queryDerivationOutputs(path)) { + if (isValidPath(i) && queryPathInfo(i)->deriver == path) { + enqueue(i); + } + } + } + + } else { + for (auto& ref : info->references) { + if (ref != path) { + enqueue(ref); + } + } + + if (includeOutputs && isDerivation(path)) { + for (auto& i : queryDerivationOutputs(path)) { + if (isValidPath(i)) { + enqueue(i); + } + } + } + + if (includeDerivers && isValidPath(info->deriver)) { + enqueue(info->deriver); + } + } + + { + auto state(state_.lock()); + assert(state->pending); + if (--state->pending == 0u) { + done.notify_one(); + } + } + + } catch (...) { + auto state(state_.lock()); + if (!state->exc) { + state->exc = std::current_exception(); + } + assert(state->pending); + if (--state->pending == 0u) { + done.notify_one(); + } + }; + })); + }; + + for (auto& startPath : startPaths) { + enqueue(startPath); + } + + { + auto state(state_.lock()); + while (state->pending != 0u) { + state.wait(done); + } + if (state->exc) { + std::rethrow_exception(state->exc); + } + } +} + +void Store::computeFSClosure(const Path& startPath, PathSet& paths_, + bool flipDirection, bool includeOutputs, + bool includeDerivers) { + computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, + includeDerivers); +} + +void Store::queryMissing(const PathSet& targets, PathSet& willBuild_, + PathSet& willSubstitute_, PathSet& unknown_, + unsigned long long& downloadSize_, + unsigned long long& narSize_) { + LOG(INFO) << "querying info about missing paths"; + + downloadSize_ = narSize_ = 0; + + ThreadPool pool; + + struct State { + PathSet done; + PathSet &unknown, &willSubstitute, &willBuild; + unsigned long long& downloadSize; + unsigned long long& narSize; + }; + + struct DrvState { + size_t left; + bool done = false; + PathSet outPaths; + explicit DrvState(size_t left) : left(left) {} + }; + + Sync<State> state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, + downloadSize_, narSize_}); + + std::function<void(Path)> doPath; + + auto mustBuildDrv = [&](const Path& drvPath, const Derivation& drv) { + { + auto state(state_.lock()); + state->willBuild.insert(drvPath); + } + + for (auto& i : drv.inputDrvs) { + pool.enqueue( + std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + } + }; + + auto checkOutput = [&](const Path& drvPath, const ref<Derivation>& drv, + const Path& outPath, + const ref<Sync<DrvState>>& drvState_) { + if (drvState_->lock()->done) { + return; + } + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({outPath}, infos); + + if (infos.empty()) { + drvState_->lock()->done = true; + mustBuildDrv(drvPath, *drv); + } else { + { + auto drvState(drvState_->lock()); + if (drvState->done) { + return; + } + assert(drvState->left); + drvState->left--; + drvState->outPaths.insert(outPath); + if (drvState->left == 0u) { + for (auto& path : drvState->outPaths) { + pool.enqueue(std::bind(doPath, path)); + } + } + } + } + }; + + doPath = [&](const Path& path) { + { + auto state(state_.lock()); + if (state->done.count(path) != 0u) { + return; + } + state->done.insert(path); + } + + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); + + if (isDerivation(i2.first)) { + if (!isValidPath(i2.first)) { + // FIXME: we could try to substitute the derivation. + auto state(state_.lock()); + state->unknown.insert(path); + return; + } + + Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); + + PathSet invalid; + for (auto& j : drv.outputs) { + if (wantOutput(j.first, i2.second) && !isValidPath(j.second.path)) { + invalid.insert(j.second.path); + } + } + if (invalid.empty()) { + return; + } + + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size())); + for (auto& output : invalid) { + pool.enqueue(std::bind(checkOutput, i2.first, + make_ref<Derivation>(drv), output, drvState)); + } + } else { + mustBuildDrv(i2.first, drv); + } + + } else { + if (isValidPath(path)) { + return; + } + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({path}, infos); + + if (infos.empty()) { + auto state(state_.lock()); + state->unknown.insert(path); + return; + } + + auto info = infos.find(path); + assert(info != infos.end()); + + { + auto state(state_.lock()); + state->willSubstitute.insert(path); + state->downloadSize += info->second.downloadSize; + state->narSize += info->second.narSize; + } + + for (auto& ref : info->second.references) { + pool.enqueue(std::bind(doPath, ref)); + } + } + }; + + for (auto& path : targets) { + pool.enqueue(std::bind(doPath, path)); + } + + pool.process(); +} + +Paths Store::topoSortPaths(const PathSet& paths) { + Paths sorted; + PathSet visited; + PathSet parents; + + std::function<void(const Path& path, const Path* parent)> dfsVisit; + + dfsVisit = [&](const Path& path, const Path* parent) { + if (parents.find(path) != parents.end()) { + throw BuildError( + format("cycle detected in the references of '%1%' from '%2%'") % + path % *parent); + } + + if (visited.find(path) != visited.end()) { + return; + } + visited.insert(path); + parents.insert(path); + + PathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath&) { + } + + for (auto& i : references) { + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (i != path && paths.find(i) != paths.end()) { + dfsVisit(i, &path); + } + } + + sorted.push_front(path); + parents.erase(path); + }; + + for (auto& i : paths) { + dfsVisit(i, nullptr); + } + + return sorted; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/mock-binary-cache-store.cc b/third_party/nix/src/libstore/mock-binary-cache-store.cc new file mode 100644 index 000000000000..995d61521c01 --- /dev/null +++ b/third_party/nix/src/libstore/mock-binary-cache-store.cc @@ -0,0 +1,91 @@ +#include "libstore/mock-binary-cache-store.hh" + +#include <glog/logging.h> + +namespace nix { + +MockBinaryCacheStore::MockBinaryCacheStore(const Params& params) + : BinaryCacheStore(params), contents_(), errorInjections_() {} + +std::string MockBinaryCacheStore::getUri() { return "mock://1"; } + +bool MockBinaryCacheStore::fileExists(const std::string& path) { + ThrowInjectedErrors(path); + + return contents_.find(path) != contents_.end(); +}; + +void MockBinaryCacheStore::upsertFile(const std::string& path, + const std::string& data, + const std::string& mimeType) { + ThrowInjectedErrors(path); + + contents_[path] = MemoryFile{data, mimeType}; +} + +void MockBinaryCacheStore::getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept { + auto eit = errorInjections_.find(path); + if (eit != errorInjections_.end()) { + try { + eit->second(); + LOG(FATAL) << "thrower failed to throw"; + } catch (...) { + callback.rethrow(); + } + return; + } + + auto it = contents_.find(path); + if (it == contents_.end()) { + try { + throw NoSuchBinaryCacheFile(absl::StrCat( + "file '", path, "' was not added to the MockBinaryCache")); + } catch (...) { + callback.rethrow(); + } + return; + } + callback(std::make_shared<std::string>(it->second.data)); +} + +PathSet MockBinaryCacheStore::queryAllValidPaths() { + PathSet paths; + + for (auto it : contents_) { + paths.insert(it.first); + } + + return paths; +} + +void MockBinaryCacheStore::DeleteFile(const std::string& path) { + contents_.erase(path); +} + +// Same as upsert, but bypasses injected errors. +void MockBinaryCacheStore::SetFileContentsForTest(const std::string& path, + const std::string& data, + const std::string& mimeType) { + contents_[path] = MemoryFile{data, mimeType}; +} + +void MockBinaryCacheStore::PrepareErrorInjection( + const std::string& path, std::function<void()> err_factory) { + errorInjections_[path] = err_factory; +} + +void MockBinaryCacheStore::CancelErrorInjection(const std::string& path) { + errorInjections_.erase(path); +} + +void MockBinaryCacheStore::ThrowInjectedErrors(const std::string& path) { + auto it = errorInjections_.find(path); + if (it != errorInjections_.end()) { + it->second(); + LOG(FATAL) << "thrower failed to throw"; + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/mock-binary-cache-store.hh b/third_party/nix/src/libstore/mock-binary-cache-store.hh new file mode 100644 index 000000000000..419077b6bbb0 --- /dev/null +++ b/third_party/nix/src/libstore/mock-binary-cache-store.hh @@ -0,0 +1,59 @@ +#pragma once + +#include <absl/container/btree_map.h> +#include <absl/container/flat_hash_map.h> + +#include "libstore/binary-cache-store.hh" + +namespace nix { + +// MockBinaryCacheStore implements a memory-based BinaryCacheStore, for use in +// tests. +class MockBinaryCacheStore : public BinaryCacheStore { + public: + MockBinaryCacheStore(const Params& params); + + // Store API + + std::string getUri() override; + + bool fileExists(const std::string& path) override; + + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override; + + void getFile( + const std::string& path, + Callback<std::shared_ptr<std::string>> callback) noexcept override; + + PathSet queryAllValidPaths() override; + + // Test API + + // Remove a file from the store. + void DeleteFile(const std::string& path); + + // Same as upsert, but bypasses injected errors. + void SetFileContentsForTest(const std::string& path, const std::string& data, + const std::string& mimeType); + + void PrepareErrorInjection(const std::string& path, + std::function<void()> throw_func); + + void CancelErrorInjection(const std::string& path); + + // Internals + + private: + void ThrowInjectedErrors(const std::string& path); + + struct MemoryFile { + std::string data; + std::string mimeType; + }; + + absl::btree_map<std::string, MemoryFile> contents_; + absl::flat_hash_map<std::string, std::function<void()>> errorInjections_; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-accessor.cc b/third_party/nix/src/libstore/nar-accessor.cc new file mode 100644 index 000000000000..cfd3d50b32bc --- /dev/null +++ b/third_party/nix/src/libstore/nar-accessor.cc @@ -0,0 +1,268 @@ +#include "libstore/nar-accessor.hh" + +#include <algorithm> +#include <map> +#include <nlohmann/json.hpp> +#include <stack> +#include <utility> + +#include "libutil/archive.hh" +#include "libutil/json.hh" + +namespace nix { + +struct NarMember { + FSAccessor::Type type = FSAccessor::Type::tMissing; + + bool isExecutable = false; + + /* If this is a regular file, position of the contents of this + file in the NAR. */ + size_t start = 0, size = 0; + + std::string target; + + /* If this is a directory, all the children of the directory. */ + std::map<std::string, NarMember> children; +}; + +struct NarAccessor : public FSAccessor { + std::shared_ptr<const std::string> nar; + + GetNarBytes getNarBytes; + + NarMember root; + + struct NarIndexer : ParseSink, StringSource { + NarAccessor& acc; + + std::stack<NarMember*> parents; + + std::string currentStart; + bool isExec = false; + + NarIndexer(NarAccessor& acc, const std::string& nar) + : StringSource(nar), acc(acc) {} + + void createMember(const Path& path, NarMember member) { + size_t level = std::count(path.begin(), path.end(), '/'); + while (parents.size() > level) { + parents.pop(); + } + + if (parents.empty()) { + acc.root = std::move(member); + parents.push(&acc.root); + } else { + if (parents.top()->type != FSAccessor::Type::tDirectory) { + throw Error("NAR file missing parent directory of path '%s'", path); + } + auto result = parents.top()->children.emplace(baseNameOf(path), + std::move(member)); + parents.push(&result.first->second); + } + } + + void createDirectory(const Path& path) override { + createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + } + + void createRegularFile(const Path& path) override { + createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + } + + void isExecutable() override { parents.top()->isExecutable = true; } + + void preallocateContents(unsigned long long size) override { + currentStart = std::string(s, pos, 16); + assert(size <= std::numeric_limits<size_t>::max()); + parents.top()->size = static_cast<size_t>(size); + parents.top()->start = pos; + } + + void receiveContents(unsigned char* data, unsigned int len) override { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == std::string((char*)data, 16)); + currentStart.clear(); + } + } + + void createSymlink(const Path& path, const std::string& target) override { + createMember(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + } + }; + + explicit NarAccessor(const ref<const std::string>& nar) : nar(nar) { + NarIndexer indexer(*this, *nar); + parseDump(indexer, indexer); + } + + NarAccessor(const std::string& listing, GetNarBytes getNarBytes) + : getNarBytes(std::move(getNarBytes)) { + using json = nlohmann::json; + + std::function<void(NarMember&, json&)> recurse; + + recurse = [&](NarMember& member, json& v) { + std::string type = v["type"]; + + if (type == "directory") { + member.type = FSAccessor::Type::tDirectory; + for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { + const std::string& name = i.key(); + recurse(member.children[name], i.value()); + } + } else if (type == "regular") { + member.type = FSAccessor::Type::tRegular; + member.size = v["size"]; + member.isExecutable = v.value("executable", false); + member.start = v["narOffset"]; + } else if (type == "symlink") { + member.type = FSAccessor::Type::tSymlink; + member.target = v.value("target", ""); + } else { + return; + } + }; + + json v = json::parse(listing); + recurse(root, v); + } + + NarMember* find(const Path& path) { + Path canon = path.empty() ? "" : canonPath(path); + NarMember* current = &root; + auto end = path.end(); + for (auto it = path.begin(); it != end;) { + // because it != end, the remaining component is non-empty so we need + // a directory + if (current->type != FSAccessor::Type::tDirectory) { + return nullptr; + } + + // skip slash (canonPath above ensures that this is always a slash) + assert(*it == '/'); + it += 1; + + // lookup current component + auto next = std::find(it, end, '/'); + auto child = current->children.find(std::string(it, next)); + if (child == current->children.end()) { + return nullptr; + } + current = &child->second; + + it = next; + } + + return current; + } + + NarMember& get(const Path& path) { + auto result = find(path); + if (result == nullptr) { + throw Error("NAR file does not contain path '%1%'", path); + } + return *result; + } + + Stat stat(const Path& path) override { + auto i = find(path); + if (i == nullptr) { + return {FSAccessor::Type::tMissing, 0, false}; + } + return {i->type, i->size, i->isExecutable, i->start}; + } + + StringSet readDirectory(const Path& path) override { + auto i = get(path); + + if (i.type != FSAccessor::Type::tDirectory) { + throw Error(format("path '%1%' inside NAR file is not a directory") % + path); + } + + StringSet res; + for (auto& child : i.children) { + res.insert(child.first); + } + + return res; + } + + std::string readFile(const Path& path) override { + auto i = get(path); + if (i.type != FSAccessor::Type::tRegular) { + throw Error(format("path '%1%' inside NAR file is not a regular file") % + path); + } + + if (getNarBytes) { + return getNarBytes(i.start, i.size); + } + + assert(nar); + return std::string(*nar, i.start, i.size); + } + + std::string readLink(const Path& path) override { + auto i = get(path); + if (i.type != FSAccessor::Type::tSymlink) { + throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + } + return i.target; + } +}; + +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar) { + return make_ref<NarAccessor>(nar); +} + +ref<FSAccessor> makeLazyNarAccessor(const std::string& listing, + GetNarBytes getNarBytes) { + return make_ref<NarAccessor>(listing, getNarBytes); +} + +void listNar(JSONPlaceholder& res, const ref<FSAccessor>& accessor, + const Path& path, bool recurse) { + auto st = accessor->stat(path); + + auto obj = res.object(); + + switch (st.type) { + case FSAccessor::Type::tRegular: + obj.attr("type", "regular"); + obj.attr("size", st.fileSize); + if (st.isExecutable) { + obj.attr("executable", true); + } + if (st.narOffset != 0u) { + obj.attr("narOffset", st.narOffset); + } + break; + case FSAccessor::Type::tDirectory: + obj.attr("type", "directory"); + { + auto res2 = obj.object("entries"); + for (auto& name : accessor->readDirectory(path)) { + if (recurse) { + auto res3 = res2.placeholder(name); + listNar(res3, accessor, path + "/" + name, true); + } else { + res2.object(name); + } + } + } + break; + case FSAccessor::Type::tSymlink: + obj.attr("type", "symlink"); + obj.attr("target", accessor->readLink(path)); + break; + default: + throw Error("path '%s' does not exist in NAR", path); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-accessor.hh b/third_party/nix/src/libstore/nar-accessor.hh new file mode 100644 index 000000000000..0906a4606eb4 --- /dev/null +++ b/third_party/nix/src/libstore/nar-accessor.hh @@ -0,0 +1,29 @@ +#pragma once + +#include <functional> + +#include "libstore/fs-accessor.hh" + +namespace nix { + +/* Return an object that provides access to the contents of a NAR + file. */ +ref<FSAccessor> makeNarAccessor(ref<const std::string> nar); + +/* Create a NAR accessor from a NAR listing (in the format produced by + listNar()). The callback getNarBytes(offset, length) is used by the + readFile() method of the accessor to get the contents of files + inside the NAR. */ +typedef std::function<std::string(uint64_t, uint64_t)> GetNarBytes; + +ref<FSAccessor> makeLazyNarAccessor(const std::string& listing, + GetNarBytes getNarBytes); + +class JSONPlaceholder; + +/* Write a JSON representation of the contents of a NAR (except file + contents). */ +void listNar(JSONPlaceholder& res, const ref<FSAccessor>& accessor, + const Path& path, bool recurse); + +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.cc b/third_party/nix/src/libstore/nar-info-disk-cache.cc new file mode 100644 index 000000000000..90ea20a8936b --- /dev/null +++ b/third_party/nix/src/libstore/nar-info-disk-cache.cc @@ -0,0 +1,295 @@ +#include "libstore/nar-info-disk-cache.hh" + +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <sqlite3.h> + +#include "libstore/globals.hh" +#include "libstore/sqlite.hh" +#include "libutil/sync.hh" + +namespace nix { + +static const char* schema = R"sql( + +create table if not exists BinaryCaches ( + id integer primary key autoincrement not null, + url text unique not null, + timestamp integer not null, + storeDir text not null, + wantMassQuery integer not null, + priority integer not null +); + +create table if not exists NARs ( + cache integer not null, + hashPart text not null, + namePart text, + url text, + compression text, + fileHash text, + fileSize integer, + narHash text, + narSize integer, + refs text, + deriver text, + sigs text, + ca text, + timestamp integer not null, + present integer not null, + primary key (cache, hashPart), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +create table if not exists LastPurge ( + dummy text primary key, + value integer +); + +)sql"; + +class NarInfoDiskCacheImpl final : public NarInfoDiskCache { + public: + /* How often to purge expired entries from the cache. */ + const int purgeInterval = 24 * 3600; + + struct Cache { + int id; + Path storeDir; + bool wantMassQuery; + int priority; + }; + + struct State { + SQLite db; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, + purgeCache; + std::map<std::string, Cache> caches; + }; + + Sync<State> _state; + + NarInfoDiskCacheImpl() { + auto state(_state.lock()); + + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + + if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) { + throwSQLiteError(state->db, "setting timeout"); + } + + // We can always reproduce the cache. + state->db.exec("pragma synchronous = off"); + state->db.exec("pragma main.journal_mode = truncate"); + + state->db.exec(schema); + + state->insertCache.create( + state->db, + "insert or replace into BinaryCaches(url, timestamp, storeDir, " + "wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + + state->queryCache.create(state->db, + "select id, storeDir, wantMassQuery, priority " + "from BinaryCaches where url = ?"); + + state->insertNAR.create( + state->db, + "insert or replace into NARs(cache, hashPart, namePart, url, " + "compression, fileHash, fileSize, narHash, " + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, " + "?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + + state->insertMissingNAR.create( + state->db, + "insert or replace into NARs(cache, hashPart, timestamp, present) " + "values (?, ?, ?, 0)"); + + state->queryNAR.create( + state->db, + "select present, namePart, url, compression, fileHash, fileSize, " + "narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? " + "and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 " + "and timestamp > ?))"); + + /* Periodically purge expired entries from the database. */ + retrySQLite<void>([&]() { + auto now = time(nullptr); + + SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); + auto queryLastPurge_(queryLastPurge.use()); + + if (!queryLastPurge_.next() || + queryLastPurge_.getInt(0) < now - purgeInterval) { + SQLiteStmt(state->db, + "delete from NARs where ((present = 0 and timestamp < ?) or " + "(present = 1 and timestamp < ?))") + .use()(now - settings.ttlNegativeNarInfoCache)( + now - settings.ttlPositiveNarInfoCache) + .exec(); + + DLOG(INFO) << "deleted " << sqlite3_changes(state->db) + << " entries from the NAR info disk cache"; + + SQLiteStmt( + state->db, + "insert or replace into LastPurge(dummy, value) values ('', ?)") + .use()(now) + .exec(); + } + }); + } + + static Cache& getCache(State& state, const std::string& uri) { + auto i = state.caches.find(uri); + if (i == state.caches.end()) { + abort(); + } + return i->second; + } + + void createCache(const std::string& uri, const Path& storeDir, + bool wantMassQuery, int priority) override { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + // FIXME: race + + state->insertCache + .use()(uri)(time(nullptr))(storeDir)( + static_cast<int64_t>(wantMassQuery))(priority) + .exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = + Cache{static_cast<int>(sqlite3_last_insert_rowid(state->db)), + storeDir, wantMassQuery, priority}; + }); + } + + bool cacheExists(const std::string& uri, bool& wantMassQuery, + int& priority) override { + return retrySQLite<bool>([&]() { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i == state->caches.end()) { + auto queryCache(state->queryCache.use()(uri)); + if (!queryCache.next()) { + return false; + } + state->caches.emplace( + uri, Cache{static_cast<int>(queryCache.getInt(0)), + queryCache.getStr(1), queryCache.getInt(2) != 0, + static_cast<int>(queryCache.getInt(3))}); + } + + auto& cache(getCache(*state, uri)); + + wantMassQuery = cache.wantMassQuery; + priority = cache.priority; + + return true; + }); + } + + std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string& uri, const std::string& hashPart) override { + return retrySQLite<std::pair<Outcome, std::shared_ptr<NarInfo>>>( + [&]() -> std::pair<Outcome, std::shared_ptr<NarInfo>> { + auto state(_state.lock()); + + auto& cache(getCache(*state, uri)); + + auto now = time(nullptr); + + auto queryNAR(state->queryNAR.use()(cache.id)(hashPart)( + now - settings.ttlNegativeNarInfoCache)( + now - settings.ttlPositiveNarInfoCache)); + + if (!queryNAR.next()) { + return {oUnknown, nullptr}; + } + + if (queryNAR.getInt(0) == 0) { + return {oInvalid, nullptr}; + } + + auto narInfo = make_ref<NarInfo>(); + + auto namePart = queryNAR.getStr(1); + narInfo->path = cache.storeDir + "/" + hashPart + + (namePart.empty() ? "" : "-" + namePart); + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) { + auto hash_ = Hash::deserialize(queryNAR.getStr(4)); + // TODO(#statusor): does this throw mess with retrySQLite? + narInfo->fileHash = Hash::unwrap_throw(hash_); + } + narInfo->fileSize = queryNAR.getInt(5); + auto hash_ = Hash::deserialize(queryNAR.getStr(6)); + narInfo->narHash = Hash::unwrap_throw(hash_); + narInfo->narSize = queryNAR.getInt(7); + for (auto r : absl::StrSplit(queryNAR.getStr(8), absl::ByChar(' '), + absl::SkipEmpty())) { + narInfo->references.insert(absl::StrCat(cache.storeDir, "/", r)); + } + if (!queryNAR.isNull(9)) { + narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + } + for (auto& sig : absl::StrSplit( + queryNAR.getStr(10), absl::ByChar(' '), absl::SkipEmpty())) { + narInfo->sigs.insert(std::string(sig)); + } + narInfo->ca = queryNAR.getStr(11); + + return {oValid, narInfo}; + }); + } + + void upsertNarInfo(const std::string& uri, const std::string& hashPart, + std::shared_ptr<ValidPathInfo> info) override { + retrySQLite<void>([&]() { + auto state(_state.lock()); + + auto& cache(getCache(*state, uri)); + + if (info) { + auto narInfo = std::dynamic_pointer_cast<NarInfo>(info); + + assert(hashPart == storePathToHash(info->path)); + + state->insertNAR + .use()(cache.id)(hashPart)(storePathToName(info->path))( + narInfo ? narInfo->url : "", narInfo != nullptr)( + narInfo ? narInfo->compression : "", narInfo != nullptr)( + narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() + : "", + narInfo && narInfo->fileHash)( + narInfo ? narInfo->fileSize : 0, + narInfo != nullptr && + (narInfo->fileSize != 0u))(info->narHash.to_string())( + info->narSize)(concatStringsSep(" ", info->shortRefs()))( + !info->deriver.empty() ? baseNameOf(info->deriver) : "", + !info->deriver.empty())(concatStringsSep(" ", info->sigs))( + info->ca)(time(nullptr)) + .exec(); + + } else { + state->insertMissingNAR.use()(cache.id)(hashPart)(time(nullptr)).exec(); + } + }); + } +}; + +std::shared_ptr<NarInfoDiskCache> getNarInfoDiskCache() { + static std::shared_ptr<NarInfoDiskCache> cache = + std::make_shared<NarInfoDiskCacheImpl>(); + return cache; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.hh b/third_party/nix/src/libstore/nar-info-disk-cache.hh new file mode 100644 index 000000000000..8eeab7635a4e --- /dev/null +++ b/third_party/nix/src/libstore/nar-info-disk-cache.hh @@ -0,0 +1,30 @@ +#pragma once + +#include "libstore/nar-info.hh" +#include "libutil/ref.hh" + +namespace nix { + +class NarInfoDiskCache { + public: + typedef enum { oValid, oInvalid, oUnknown } Outcome; + + virtual void createCache(const std::string& uri, const Path& storeDir, + bool wantMassQuery, int priority) = 0; + + virtual bool cacheExists(const std::string& uri, bool& wantMassQuery, + int& priority) = 0; + + virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo( + const std::string& uri, const std::string& hashPart) = 0; + + virtual void upsertNarInfo(const std::string& uri, + const std::string& hashPart, + std::shared_ptr<ValidPathInfo> info) = 0; +}; + +/* Return a singleton cache object that can be used concurrently by + multiple threads. */ +std::shared_ptr<NarInfoDiskCache> getNarInfoDiskCache(); + +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info.cc b/third_party/nix/src/libstore/nar-info.cc new file mode 100644 index 000000000000..d42167dbfa69 --- /dev/null +++ b/third_party/nix/src/libstore/nar-info.cc @@ -0,0 +1,142 @@ +#include "libstore/nar-info.hh" + +#include <absl/strings/numbers.h> +#include <absl/strings/str_split.h> + +#include "libstore/globals.hh" + +namespace nix { + +NarInfo::NarInfo(const Store& store, const std::string& s, + const std::string& whence) { + auto corrupt = [&]() { + throw Error(format("NAR info file '%1%' is corrupt") % whence); + }; + + auto parseHashField = [&](const std::string& s) { + auto hash_ = Hash::deserialize(s); + if (hash_.ok()) { + return *hash_; + } else { + // TODO(#statusor): return an actual error + corrupt(); + return Hash(); + } + }; + + size_t pos = 0; + while (pos < s.size()) { + size_t colon = s.find(':', pos); + if (colon == std::string::npos) { + corrupt(); + } + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) { + corrupt(); + } + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "StorePath") { + if (!store.isStorePath(value)) { + corrupt(); + } + path = value; + } else if (name == "URL") { + url = value; + } else if (name == "Compression") { + compression = value; + } else if (name == "FileHash") { + fileHash = parseHashField(value); + } else if (name == "FileSize") { + if (!absl::SimpleAtoi(value, &fileSize)) { + corrupt(); + } + } else if (name == "NarHash") { + narHash = parseHashField(value); + } else if (name == "NarSize") { + if (!absl::SimpleAtoi(value, &narSize)) { + corrupt(); + } + } else if (name == "References") { + std::vector<std::string> refs = + absl::StrSplit(value, absl::ByChar(' '), absl::SkipEmpty()); + if (!references.empty()) { + corrupt(); + } + for (auto& r : refs) { + auto r2 = store.storeDir + "/" + r; + if (!store.isStorePath(r2)) { + corrupt(); + } + references.insert(r2); + } + } else if (name == "Deriver") { + if (value != "unknown-deriver") { + auto p = store.storeDir + "/" + value; + if (!store.isStorePath(p)) { + corrupt(); + } + deriver = p; + } + } else if (name == "System") { + system = value; + } else if (name == "Sig") { + sigs.insert(value); + } else if (name == "CA") { + if (!ca.empty()) { + corrupt(); + } + ca = value; + } + + pos = eol + 1; + } + + if (compression.empty()) { + compression = "bzip2"; + } + + if (path.empty() || url.empty() || narSize == 0 || !narHash) { + corrupt(); + } +} + +std::string NarInfo::to_string() const { + std::string res; + res += "StorePath: " + path + "\n"; + res += "URL: " + url + "\n"; + assert(!compression.empty()); + res += "Compression: " + compression + "\n"; + assert(fileHash.type == htSHA256); + res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileSize: " + std::to_string(fileSize) + "\n"; + assert(narHash.type == htSHA256); + res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarSize: " + std::to_string(narSize) + "\n"; + + res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + + if (!deriver.empty()) { + res += "Deriver: " + baseNameOf(deriver) + "\n"; + } + + if (!system.empty()) { + res += "System: " + system + "\n"; + } + + for (const auto& sig : sigs) { + res += "Sig: " + sig + "\n"; + } + + if (!ca.empty()) { + res += "CA: " + ca + "\n"; + } + + return res; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/nar-info.hh b/third_party/nix/src/libstore/nar-info.hh new file mode 100644 index 000000000000..48eccf830270 --- /dev/null +++ b/third_party/nix/src/libstore/nar-info.hh @@ -0,0 +1,23 @@ +#pragma once + +#include "libstore/store-api.hh" +#include "libutil/hash.hh" +#include "libutil/types.hh" + +namespace nix { + +struct NarInfo : ValidPathInfo { + std::string url; + std::string compression; + Hash fileHash; + uint64_t fileSize = 0; + std::string system; + + NarInfo() {} + NarInfo(const ValidPathInfo& info) : ValidPathInfo(info) {} + NarInfo(const Store& store, const std::string& s, const std::string& whence); + + std::string to_string() const; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/nix-store.pc.in b/third_party/nix/src/libstore/nix-store.pc.in new file mode 100644 index 000000000000..b204776b372a --- /dev/null +++ b/third_party/nix/src/libstore/nix-store.pc.in @@ -0,0 +1,9 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=@CMAKE_INSTALL_FULL_LIBDIR@ +includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixstore -lnixutil +Cflags: -I${includedir}/nix diff --git a/third_party/nix/src/libstore/optimise-store.cc b/third_party/nix/src/libstore/optimise-store.cc new file mode 100644 index 000000000000..eb24633c181c --- /dev/null +++ b/third_party/nix/src/libstore/optimise-store.cc @@ -0,0 +1,296 @@ +#include <cerrno> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <regex> +#include <utility> + +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libutil/util.hh" + +namespace nix { + +static void makeWritable(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) { + throw SysError(format("changing writability of '%1%'") % path); + } +} + +struct MakeReadOnly { + Path path; + explicit MakeReadOnly(Path path) : path(std::move(path)) {} + ~MakeReadOnly() { + try { + /* This will make the path read-only. */ + if (!path.empty()) { + canonicaliseTimestampAndPermissions(path); + } + } catch (...) { + ignoreException(); + } + } +}; + +LocalStore::InodeHash LocalStore::loadInodeHash() { + DLOG(INFO) << "loading hash inodes in memory"; + InodeHash inodeHash; + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) { + throw SysError(format("opening directory '%1%'") % linksDir); + } + + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + // We don't care if we hit non-hash files, anything goes + inodeHash.insert(dirent->d_ino); + } + if (errno) { + throw SysError(format("reading directory '%1%'") % linksDir); + } + + DLOG(INFO) << "loaded " << inodeHash.size() << " hash inodes"; + + return inodeHash; +} + +Strings LocalStore::readDirectoryIgnoringInodes(const Path& path, + const InodeHash& inodeHash) { + Strings names; + + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) { + throw SysError(format("opening directory '%1%'") % path); + } + + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + + if (inodeHash.count(dirent->d_ino) != 0u) { + DLOG(WARNING) << dirent->d_name << " is already linked"; + continue; + } + + std::string name = dirent->d_name; + if (name == "." || name == "..") { + continue; + } + names.push_back(name); + } + if (errno) { + throw SysError(format("reading directory '%1%'") % path); + } + + return names; +} + +void LocalStore::optimisePath_(OptimiseStats& stats, const Path& path, + InodeHash& inodeHash) { + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectoryIgnoringInodes(path, inodeHash); + for (auto& i : names) { + optimisePath_(stats, path + "/" + i, inodeHash); + } + return; + } + + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + && !S_ISLNK(st.st_mode) +#endif + ) + return; + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. FIXME: check the modification time. */ + if (S_ISREG(st.st_mode) && ((st.st_mode & S_IWUSR) != 0u)) { + LOG(WARNING) << "skipping suspicious writable file '" << path << "'"; + return; + } + + /* This can still happen on top-level files. */ + if (st.st_nlink > 1 && (inodeHash.count(st.st_ino) != 0u)) { + DLOG(INFO) << path << " is already linked, with " << (st.st_nlink - 2) + << " other file(s)"; + return; + } + + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + LOG(INFO) << path << " has hash " << hash.to_string(); + + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + +retry: + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + if (link(path.c_str(), linkPath.c_str()) == 0) { + inodeHash.insert(st.st_ino); + return; + } + + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + LOG(WARNING) << "cannot link '" << linkPath << " to " << path << ": " + << strerror(errno); + + return; + + default: + throw SysError("cannot link '%1%' to '%2%'", linkPath, path); + } + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink) != 0) { + throw SysError(format("getting attributes of path '%1%'") % linkPath); + } + + if (st.st_ino == stLink.st_ino) { + DLOG(INFO) << path << " is already linked to " << linkPath; + return; + } + + if (st.st_size != stLink.st_size) { + LOG(WARNING) << "removing corrupted link '" << linkPath << "'"; + unlink(linkPath.c_str()); + goto retry; + } + + DLOG(INFO) << "linking '" << path << "' to '" << linkPath << "'"; + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = dirOf(path) != realStoreDir; + if (mustToggle) { + makeWritable(dirOf(path)); + } + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + Path tempLink = + (format("%1%/.tmp-link-%2%-%3%") % realStoreDir % getpid() % random()) + .str(); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + if (st.st_size != 0) { + LOG(WARNING) << linkPath << " has maximum number of links"; + } + return; + } + throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (unlink(tempLink.c_str()) == -1) { + LOG(ERROR) << "unable to unlink '" << tempLink << "'"; + } + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + DLOG(WARNING) << "'" << linkPath + << "' has reached maximum number of links"; + return; + } + throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; +} + +void LocalStore::optimiseStore(OptimiseStats& stats) { + PathSet paths = queryAllValidPaths(); + InodeHash inodeHash = loadInodeHash(); + + uint64_t done = 0; + + for (auto& i : paths) { + addTempRoot(i); + if (!isValidPath(i)) { + continue; + } /* path was GC'ed, probably */ + { + LOG(INFO) << "optimising path '" << i << "'"; + optimisePath_(stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + } + done++; + } +} + +static std::string showBytes(unsigned long long bytes) { + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +} + +void LocalStore::optimiseStore() { + OptimiseStats stats; + + optimiseStore(stats); + + LOG(INFO) << showBytes(stats.bytesFreed) << " freed by hard-linking " + << stats.filesLinked << " files"; +} + +void LocalStore::optimisePath(const Path& path) { + OptimiseStats stats; + InodeHash inodeHash; + + if (settings.autoOptimiseStore) { + optimisePath_(stats, path, inodeHash); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/parsed-derivations.cc b/third_party/nix/src/libstore/parsed-derivations.cc new file mode 100644 index 000000000000..6989a21fee5c --- /dev/null +++ b/third_party/nix/src/libstore/parsed-derivations.cc @@ -0,0 +1,128 @@ +#include "libstore/parsed-derivations.hh" + +#include <absl/strings/str_split.h> + +namespace nix { + +ParsedDerivation::ParsedDerivation(const Path& drvPath, BasicDerivation& drv) + : drvPath(drvPath), drv(drv) { + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = nlohmann::json::parse(jsonAttr->second); + } catch (std::exception& e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, + e.what()); + } + } +} + +std::optional<std::string> ParsedDerivation::getStringAttr( + const std::string& name) const { + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) { + return {}; + } + if (!i->is_string()) { + throw Error("attribute '%s' of derivation '%s' must be a string", name, + drvPath); + } + return i->get<std::string>(); + + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) { + return {}; + } + return i->second; + } +} + +bool ParsedDerivation::getBoolAttr(const std::string& name, bool def) const { + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) { + return def; + } + if (!i->is_boolean()) { + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, + drvPath); + } + return i->get<bool>(); + + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) { + return def; + } + return i->second == "1"; + } +} + +std::optional<Strings> ParsedDerivation::getStringsAttr( + const std::string& name) const { + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) { + return {}; + } + if (!i->is_array()) { + throw Error("attribute '%s' of derivation '%s' must be a list of strings", + name, drvPath); + } + Strings res; + for (const auto& j : *i) { + if (!j.is_string()) { + throw Error( + "attribute '%s' of derivation '%s' must be a list of strings", name, + drvPath); + } + res.push_back(j.get<std::string>()); + } + return res; + + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) { + return {}; + } + return absl::StrSplit(i->second, absl::ByAnyChar(" \t\n\r"), + absl::SkipEmpty()); + } +} + +StringSet ParsedDerivation::getRequiredSystemFeatures() const { + StringSet res; + for (auto& i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) { + res.insert(i); + } + return res; +} + +bool ParsedDerivation::canBuildLocally() const { + if (drv.platform != settings.thisSystem.get() && + (settings.extraPlatforms.get().count(drv.platform) == 0u) && + !drv.isBuiltin()) { + return false; + } + + for (auto& feature : getRequiredSystemFeatures()) { + if (settings.systemFeatures.get().count(feature) == 0u) { + return false; + } + } + + return true; +} + +bool ParsedDerivation::willBuildLocally() const { + return getBoolAttr("preferLocalBuild") && canBuildLocally(); +} + +bool ParsedDerivation::substitutesAllowed() const { + return getBoolAttr("allowSubstitutes", true); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/parsed-derivations.hh b/third_party/nix/src/libstore/parsed-derivations.hh new file mode 100644 index 000000000000..7cd3d36f67bf --- /dev/null +++ b/third_party/nix/src/libstore/parsed-derivations.hh @@ -0,0 +1,34 @@ +#include <nlohmann/json.hpp> + +#include "libstore/derivations.hh" + +namespace nix { + +class ParsedDerivation { + Path drvPath; + BasicDerivation& drv; + std::optional<nlohmann::json> structuredAttrs; + + public: + ParsedDerivation(const Path& drvPath, BasicDerivation& drv); + + const std::optional<nlohmann::json>& getStructuredAttrs() const { + return structuredAttrs; + } + + std::optional<std::string> getStringAttr(const std::string& name) const; + + bool getBoolAttr(const std::string& name, bool def = false) const; + + std::optional<Strings> getStringsAttr(const std::string& name) const; + + StringSet getRequiredSystemFeatures() const; + + bool canBuildLocally() const; + + bool willBuildLocally() const; + + bool substitutesAllowed() const; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/pathlocks.cc b/third_party/nix/src/libstore/pathlocks.cc new file mode 100644 index 000000000000..09dec08c458c --- /dev/null +++ b/third_party/nix/src/libstore/pathlocks.cc @@ -0,0 +1,172 @@ +#include "libstore/pathlocks.hh" + +#include <cerrno> +#include <cstdlib> + +#include <fcntl.h> +#include <glog/logging.h> +#include <sys/file.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "libutil/sync.hh" +#include "libutil/util.hh" + +namespace nix { + +AutoCloseFD openLockFile(const Path& path, bool create) { + AutoCloseFD fd( + open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600)); + + if (!fd && (create || errno != ENOENT)) { + throw SysError(format("opening lock file '%1%'") % path); + } + + return fd; +} + +void deleteLockFile(const Path& path, int fd) { + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(fd, "d"); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ +} + +bool lockFile(int fd, LockType lockType, bool wait) { + int type; + if (lockType == ltRead) { + type = LOCK_SH; + } else if (lockType == ltWrite) { + type = LOCK_EX; + } else if (lockType == ltNone) { + type = LOCK_UN; + } else { + abort(); + } + + if (wait) { + while (flock(fd, type) != 0) { + checkInterrupt(); + if (errno != EINTR) { + throw SysError(format("acquiring/releasing lock")); + } + return false; + } + } else { + while (flock(fd, type | LOCK_NB) != 0) { + checkInterrupt(); + if (errno == EWOULDBLOCK) { + return false; + } + if (errno != EINTR) { + throw SysError(format("acquiring/releasing lock")); + } + } + } + + return true; +} + +PathLocks::PathLocks() : deletePaths(false) {} + +PathLocks::PathLocks(const PathSet& paths, const std::string& waitMsg) + : deletePaths(false) { + lockPaths(paths, waitMsg); +} + +bool PathLocks::lockPaths(const PathSet& paths, const std::string& waitMsg, + bool wait) { + assert(fds.empty()); + + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ + for (auto& path : paths) { + checkInterrupt(); + Path lockPath = path + ".lock"; + + VLOG(2) << "locking path '" << path << "'"; + + AutoCloseFD fd; + + while (true) { + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (!waitMsg.empty()) { + LOG(WARNING) << waitMsg; + } + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + VLOG(2) << "lock acquired on '" << lockPath << "'"; + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) { + throw SysError(format("statting lock file '%1%'") % lockPath); + } + if (st.st_size != 0) { + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + DLOG(INFO) << "open lock file '" << lockPath << "' has become stale"; + } else { + break; + } + } + + /* Use borrow so that the descriptor isn't closed. */ + fds.emplace_back(fd.release(), lockPath); + } + + return true; +} + +PathLocks::~PathLocks() { + try { + unlock(); + } catch (...) { + ignoreException(); + } +} + +void PathLocks::unlock() { + for (auto& i : fds) { + if (deletePaths) { + deleteLockFile(i.second, i.first); + } + + if (close(i.first) == -1) { + LOG(WARNING) << "cannot close lock file on '" << i.second << "'"; + } + + VLOG(2) << "lock released on '" << i.second << "'"; + } + + fds.clear(); +} + +void PathLocks::setDeletion(bool deletePaths) { + this->deletePaths = deletePaths; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/pathlocks.hh b/third_party/nix/src/libstore/pathlocks.hh new file mode 100644 index 000000000000..d515963e76e9 --- /dev/null +++ b/third_party/nix/src/libstore/pathlocks.hh @@ -0,0 +1,35 @@ +#pragma once + +#include "libutil/util.hh" + +namespace nix { + +/* Open (possibly create) a lock file and return the file descriptor. + -1 is returned if create is false and the lock could not be opened + because it doesn't exist. Any other error throws an exception. */ +AutoCloseFD openLockFile(const Path& path, bool create); + +/* Delete an open lock file. */ +void deleteLockFile(const Path& path, int fd); + +enum LockType { ltRead, ltWrite, ltNone }; + +bool lockFile(int fd, LockType lockType, bool wait); + +class PathLocks { + private: + typedef std::pair<int, Path> FDPair; + std::list<FDPair> fds; + bool deletePaths; + + public: + PathLocks(); + PathLocks(const PathSet& paths, const std::string& waitMsg = ""); + bool lockPaths(const PathSet& _paths, const std::string& waitMsg = "", + bool wait = true); + ~PathLocks(); + void unlock(); + void setDeletion(bool deletePaths); +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/profiles.cc b/third_party/nix/src/libstore/profiles.cc new file mode 100644 index 000000000000..0d44c60cc4e9 --- /dev/null +++ b/third_party/nix/src/libstore/profiles.cc @@ -0,0 +1,252 @@ +#include "libstore/profiles.hh" + +#include <cerrno> +#include <cstdio> + +#include <absl/strings/numbers.h> +#include <absl/strings/string_view.h> +#include <absl/strings/strip.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libstore/store-api.hh" +#include "libutil/util.hh" + +namespace nix { + +static bool cmpGensByNumber(const Generation& a, const Generation& b) { + return a.number < b.number; +} + +// Parse a generation out of the format +// `<profilename>-<generation>-link'. +static int parseName(absl::string_view profileName, absl::string_view name) { + // Consume the `<profilename>-' prefix and and `-link' suffix. + if (!(absl::ConsumePrefix(&name, profileName) && + absl::ConsumePrefix(&name, "-") && + absl::ConsumeSuffix(&name, "-link"))) { + return -1; + } + + int n; + if (!absl::SimpleAtoi(name, &n) || n < 0) { + return -1; + } + + return n; +} + +Generations findGenerations(const Path& profile, int& curGen) { + Generations gens; + + Path profileDir = dirOf(profile); + std::string profileName = baseNameOf(profile); + + for (auto& i : readDirectory(profileDir)) { + int n; + if ((n = parseName(profileName, i.name)) != -1) { + Generation gen; + gen.path = profileDir + "/" + i.name; + gen.number = n; + struct stat st; + if (lstat(gen.path.c_str(), &st) != 0) { + throw SysError(format("statting '%1%'") % gen.path); + } + gen.creationTime = st.st_mtime; + gens.push_back(gen); + } + } + + gens.sort(cmpGensByNumber); + + curGen = pathExists(profile) ? parseName(profileName, readLink(profile)) : -1; + + return gens; +} + +static void makeName(const Path& profile, unsigned int num, Path& outLink) { + Path prefix = (format("%1%-%2%") % profile % num).str(); + outLink = prefix + "-link"; +} + +Path createGeneration(const ref<LocalFSStore>& store, const Path& profile, + const Path& outPath) { + /* The new generation number should be higher than old the + previous ones. */ + int dummy; + Generations gens = findGenerations(profile, dummy); + + unsigned int num; + if (!gens.empty()) { + Generation last = gens.back(); + + if (readLink(last.path) == outPath) { + /* We only create a new generation symlink if it differs + from the last one. + + This helps keeping gratuitous installs/rebuilds from piling + up uncontrolled numbers of generations, cluttering up the + UI like grub. */ + return last.path; + } + + num = gens.back().number; + } else { + num = 0; + } + + /* Create the new generation. Note that addPermRoot() blocks if + the garbage collector is running to prevent the stuff we've + built from moving from the temporary roots (which the GC knows) + to the permanent roots (of which the GC would have a stale + view). If we didn't do it this way, the GC might remove the + user environment etc. we've just built. */ + Path generation; + makeName(profile, num + 1, generation); + store->addPermRoot(outPath, generation, false, true); + + return generation; +} + +static void removeFile(const Path& path) { + if (remove(path.c_str()) == -1) { + throw SysError(format("cannot unlink '%1%'") % path); + } +} + +void deleteGeneration(const Path& profile, unsigned int gen) { + Path generation; + makeName(profile, gen, generation); + removeFile(generation); +} + +static void deleteGeneration2(const Path& profile, unsigned int gen, + bool dryRun) { + if (dryRun) { + LOG(INFO) << "would remove generation " << gen; + } else { + LOG(INFO) << "removing generation " << gen; + deleteGeneration(profile, gen); + } +} + +void deleteGenerations(const Path& profile, + const std::set<unsigned int>& gensToDelete, + bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + Generations gens = findGenerations(profile, curGen); + + if (gensToDelete.find(curGen) != gensToDelete.end()) { + throw Error(format("cannot delete current generation of profile %1%'") % + profile); + } + + for (auto& i : gens) { + if (gensToDelete.find(i.number) == gensToDelete.end()) { + continue; + } + deleteGeneration2(profile, i.number, dryRun); + } +} + +void deleteGenerationsGreaterThan(const Path& profile, int max, bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + bool fromCurGen = false; + Generations gens = findGenerations(profile, curGen); + for (auto i = gens.rbegin(); i != gens.rend(); ++i) { + if (i->number == curGen) { + fromCurGen = true; + max--; + continue; + } + if (fromCurGen) { + if (max != 0) { + max--; + continue; + } + deleteGeneration2(profile, i->number, dryRun); + } + } +} + +void deleteOldGenerations(const Path& profile, bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + Generations gens = findGenerations(profile, curGen); + + for (auto& i : gens) { + if (i.number != curGen) { + deleteGeneration2(profile, i.number, dryRun); + } + } +} + +void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun) { + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + Generations gens = findGenerations(profile, curGen); + + bool canDelete = false; + for (auto i = gens.rbegin(); i != gens.rend(); ++i) { + if (canDelete) { + assert(i->creationTime < t); + if (i->number != curGen) { + deleteGeneration2(profile, i->number, dryRun); + } + } else if (i->creationTime < t) { + /* We may now start deleting generations, but we don't + delete this generation yet, because this generation was + still the one that was active at the requested point in + time. */ + canDelete = true; + } + } +} + +void deleteGenerationsOlderThan(const Path& profile, + const std::string& timeSpec, bool dryRun) { + time_t curTime = time(nullptr); + std::string strDays = std::string(timeSpec, 0, timeSpec.size() - 1); + int days; + + if (!absl::SimpleAtoi(strDays, &days) || days < 1) { + throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + } + + time_t oldTime = curTime - days * 24 * 3600; + + deleteGenerationsOlderThan(profile, oldTime, dryRun); +} + +void switchLink(const Path& link, Path target) { + /* Hacky. */ + if (dirOf(target) == dirOf(link)) { + target = baseNameOf(target); + } + + replaceSymlink(target, link); +} + +void lockProfile(PathLocks& lock, const Path& profile) { + lock.lockPaths({profile}, + (format("waiting for lock on profile '%1%'") % profile).str()); + lock.setDeletion(true); +} + +std::string optimisticLockProfile(const Path& profile) { + return pathExists(profile) ? readLink(profile) : ""; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/profiles.hh b/third_party/nix/src/libstore/profiles.hh new file mode 100644 index 000000000000..ff6399040928 --- /dev/null +++ b/third_party/nix/src/libstore/profiles.hh @@ -0,0 +1,61 @@ +#pragma once + +#include <time.h> + +#include "libstore/pathlocks.hh" +#include "libutil/types.hh" + +namespace nix { + +struct Generation { + int number; + Path path; + time_t creationTime; + Generation() { number = -1; } + operator bool() const { return number != -1; } +}; + +typedef std::list<Generation> Generations; + +/* Returns the list of currently present generations for the specified + profile, sorted by generation number. */ +Generations findGenerations(const Path& profile, int& curGen); + +class LocalFSStore; + +Path createGeneration(const ref<LocalFSStore>& store, const Path& profile, + const Path& outPath); + +void deleteGeneration(const Path& profile, unsigned int gen); + +void deleteGenerations(const Path& profile, + const std::set<unsigned int>& gensToDelete, bool dryRun); + +void deleteGenerationsGreaterThan(const Path& profile, const int max, + bool dryRun); + +void deleteOldGenerations(const Path& profile, bool dryRun); + +void deleteGenerationsOlderThan(const Path& profile, time_t t, bool dryRun); + +void deleteGenerationsOlderThan(const Path& profile, + const std::string& timeSpec, bool dryRun); + +void switchLink(const Path& link, Path target); + +/* Ensure exclusive access to a profile. Any command that modifies + the profile first acquires this lock. */ +void lockProfile(PathLocks& lock, const Path& profile); + +/* Optimistic locking is used by long-running operations like `nix-env + -i'. Instead of acquiring the exclusive lock for the entire + duration of the operation, we just perform the operation + optimistically (without an exclusive lock), and check at the end + whether the profile changed while we were busy (i.e., the symlink + target changed). If so, the operation is restarted. Restarting is + generally cheap, since the build results are still in the Nix + store. Most of the time, only the user environment has to be + rebuilt. */ +std::string optimisticLockProfile(const Path& profile); + +} // namespace nix diff --git a/third_party/nix/src/libstore/references.cc b/third_party/nix/src/libstore/references.cc new file mode 100644 index 000000000000..f120439c1060 --- /dev/null +++ b/third_party/nix/src/libstore/references.cc @@ -0,0 +1,126 @@ +#include "libstore/references.hh" + +#include <cstdlib> +#include <map> + +#include <glog/logging.h> + +#include "libutil/archive.hh" +#include "libutil/hash.hh" +#include "libutil/util.hh" + +namespace nix { + +constexpr unsigned int kRefLength = 32; /* characters */ + +static void search(const unsigned char* s, size_t len, StringSet& hashes, + StringSet& seen) { + static bool initialised = false; + static bool isBase32[256]; + if (!initialised) { + for (bool& i : isBase32) { + i = false; + } + for (char base32Char : base32Chars) { + isBase32[static_cast<unsigned char>(base32Char)] = true; + } + initialised = true; + } + + for (size_t i = 0; i + kRefLength <= len;) { + int j = 0; + bool match = true; + for (j = kRefLength - 1; j >= 0; --j) { + if (!isBase32[s[i + j]]) { + i += j + 1; + match = false; + break; + } + } + if (!match) { + continue; + } + std::string ref(reinterpret_cast<const char*>(s) + i, kRefLength); + if (hashes.find(ref) != hashes.end()) { + DLOG(INFO) << "found reference to '" << ref << "' at offset " << i; + seen.insert(ref); + hashes.erase(ref); + } + ++i; + } +} + +struct RefScanSink : Sink { + HashSink hashSink; + StringSet hashes; + StringSet seen; + + std::string tail; + + RefScanSink() : hashSink(htSHA256) {} + + void operator()(const unsigned char* data, size_t len) override; +}; + +void RefScanSink::operator()(const unsigned char* data, size_t len) { + hashSink(data, len); + + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + std::string s = tail + std::string(reinterpret_cast<const char*>(data), + len > kRefLength ? kRefLength : len); + search(reinterpret_cast<const unsigned char*>(s.data()), s.size(), hashes, + seen); + + search(data, len, hashes, seen); + + size_t tailLen = len <= kRefLength ? len : kRefLength; + tail = + std::string(tail, tail.size() < kRefLength - tailLen + ? 0 + : tail.size() - (kRefLength - tailLen)) + + std::string(reinterpret_cast<const char*>(data) + len - tailLen, tailLen); +} + +PathSet scanForReferences(const std::string& path, const PathSet& refs, + HashResult& hash) { + RefScanSink sink; + std::map<std::string, Path> backMap; + + /* For efficiency (and a higher hit rate), just search for the + hash part of the file name. (This assumes that all references + have the form `HASH-bla'). */ + for (auto& i : refs) { + std::string baseName = baseNameOf(i); + std::string::size_type pos = baseName.find('-'); + if (pos == std::string::npos) { + throw Error(format("bad reference '%1%'") % i); + } + std::string s = std::string(baseName, 0, pos); + assert(s.size() == kRefLength); + assert(backMap.find(s) == backMap.end()); + // parseHash(htSHA256, s); + sink.hashes.insert(s); + backMap[s] = i; + } + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + /* Map the hashes found back to their store paths. */ + PathSet found; + for (auto& i : sink.seen) { + std::map<std::string, Path>::iterator j; + if ((j = backMap.find(i)) == backMap.end()) { + abort(); + } + found.insert(j->second); + } + + hash = sink.hashSink.finish(); + + return found; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/references.hh b/third_party/nix/src/libstore/references.hh new file mode 100644 index 000000000000..94ac5200bdff --- /dev/null +++ b/third_party/nix/src/libstore/references.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "libutil/hash.hh" +#include "libutil/types.hh" + +namespace nix { + +PathSet scanForReferences(const Path& path, const PathSet& refs, + HashResult& hash); + +} diff --git a/third_party/nix/src/libstore/remote-fs-accessor.cc b/third_party/nix/src/libstore/remote-fs-accessor.cc new file mode 100644 index 000000000000..4178030b55d6 --- /dev/null +++ b/third_party/nix/src/libstore/remote-fs-accessor.cc @@ -0,0 +1,133 @@ +#include "libstore/remote-fs-accessor.hh" + +#include <fcntl.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "libstore/nar-accessor.hh" +#include "libutil/json.hh" + +namespace nix { + +RemoteFSAccessor::RemoteFSAccessor(const ref<Store>& store, + const Path& cacheDir) + : store(store), cacheDir(cacheDir) { + if (!cacheDir.empty()) { + createDirs(cacheDir); + } +} + +Path RemoteFSAccessor::makeCacheFile(const Path& storePath, + const std::string& ext) { + assert(!cacheDir.empty()); + return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); +} + +void RemoteFSAccessor::addToCache(const Path& storePath, const std::string& nar, + const ref<FSAccessor>& narAccessor) { + nars.emplace(storePath, narAccessor); + + if (!cacheDir.empty()) { + try { + std::ostringstream str; + JSONPlaceholder jsonRoot(str); + listNar(jsonRoot, narAccessor, "", true); + writeFile(makeCacheFile(storePath, "ls"), str.str()); + + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(storePath, "nar"), nar); + + } catch (...) { + ignoreException(); + } + } +} + +std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path& path_) { + auto path = canonPath(path_); + + auto storePath = store->toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) { + throw InvalidPath(format("path '%1%' is not a valid store path") % + storePath); + } + + auto i = nars.find(storePath); + if (i != nars.end()) { + return {i->second, restPath}; + } + + StringSink sink; + std::string listing; + Path cacheFile; + + if (!cacheDir.empty() && + pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + try { + listing = nix::readFile(makeCacheFile(storePath, "ls")); + + auto narAccessor = makeLazyNarAccessor( + listing, [cacheFile](uint64_t offset, uint64_t length) { + AutoCloseFD fd(open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC)); + if (!fd) { + throw SysError("opening NAR cache file '%s'", cacheFile); + } + + if (lseek(fd.get(), offset, SEEK_SET) != + static_cast<off_t>(offset)) { + throw SysError("seeking in '%s'", cacheFile); + } + + std::string buf(length, 0); + readFull(fd.get(), reinterpret_cast<unsigned char*>(buf.data()), + length); + + return buf; + }); + + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; + + } catch (SysError&) { + } + + try { + *sink.s = nix::readFile(cacheFile); + + auto narAccessor = makeNarAccessor(sink.s); + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; + + } catch (SysError&) { + } + } + + store->narFromPath(storePath, sink); + auto narAccessor = makeNarAccessor(sink.s); + addToCache(storePath, *sink.s, narAccessor); + return {narAccessor, restPath}; +} + +FSAccessor::Stat RemoteFSAccessor::stat(const Path& path) { + auto res = fetch(path); + return res.first->stat(res.second); +} + +StringSet RemoteFSAccessor::readDirectory(const Path& path) { + auto res = fetch(path); + return res.first->readDirectory(res.second); +} + +std::string RemoteFSAccessor::readFile(const Path& path) { + auto res = fetch(path); + return res.first->readFile(res.second); +} + +std::string RemoteFSAccessor::readLink(const Path& path) { + auto res = fetch(path); + return res.first->readLink(res.second); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/remote-fs-accessor.hh b/third_party/nix/src/libstore/remote-fs-accessor.hh new file mode 100644 index 000000000000..c4f6e89c97b5 --- /dev/null +++ b/third_party/nix/src/libstore/remote-fs-accessor.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "libstore/fs-accessor.hh" +#include "libstore/store-api.hh" +#include "libutil/ref.hh" + +namespace nix { + +class RemoteFSAccessor : public FSAccessor { + ref<Store> store; + + std::map<Path, ref<FSAccessor>> nars; + + Path cacheDir; + + std::pair<ref<FSAccessor>, Path> fetch(const Path& path_); + + friend class BinaryCacheStore; + + Path makeCacheFile(const Path& storePath, const std::string& ext); + + void addToCache(const Path& storePath, const std::string& nar, + const ref<FSAccessor>& narAccessor); + + public: + RemoteFSAccessor(const ref<Store>& store, + const /* FIXME: use std::optional */ Path& cacheDir = ""); + + Stat stat(const Path& path) override; + + StringSet readDirectory(const Path& path) override; + + std::string readFile(const Path& path) override; + + std::string readLink(const Path& path) override; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/remote-store.cc b/third_party/nix/src/libstore/remote-store.cc new file mode 100644 index 000000000000..cb6cc808c610 --- /dev/null +++ b/third_party/nix/src/libstore/remote-store.cc @@ -0,0 +1,686 @@ +#include "libstore/remote-store.hh" + +#include <cerrno> +#include <cstring> + +#include <absl/status/status.h> +#include <absl/strings/ascii.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/un.h> +#include <unistd.h> + +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/affinity.hh" +#include "libutil/archive.hh" +#include "libutil/finally.hh" +#include "libutil/pool.hh" +#include "libutil/serialise.hh" +#include "libutil/util.hh" + +namespace nix { + +Path readStorePath(Store& store, Source& from) { + Path path = readString(from); + store.assertStorePath(path); + return path; +} + +template <class T> +T readStorePaths(Store& store, Source& from) { + T paths = readStrings<T>(from); + for (auto& i : paths) { + store.assertStorePath(i); + } + return paths; +} + +template PathSet readStorePaths(Store& store, Source& from); +template Paths readStorePaths(Store& store, Source& from); + +/* TODO: Separate these store impls into different files, give them better names + */ +RemoteStore::RemoteStore(const Params& params) + : Store(params), + connections(make_ref<Pool<Connection>>( + std::max(1, (int)maxConnections), + [this]() { return openConnectionWrapper(); }, + [this](const ref<Connection>& r) { + return r->to.good() && r->from.good() && + std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::steady_clock::now() - r->startTime) + .count() < maxConnectionAge; + })) {} + +ref<RemoteStore::Connection> RemoteStore::openConnectionWrapper() { + if (failed) { + throw Error("opening a connection to remote store '%s' previously failed", + getUri()); + } + try { + return openConnection(); + } catch (...) { + failed = true; + throw; + } +} + +void RemoteStore::initConnection(Connection& conn) { + /* Send the magic greeting, check for the reply. */ + try { + conn.to << WORKER_MAGIC_1; + conn.to.flush(); + unsigned int magic = readInt(conn.from); + if (magic != WORKER_MAGIC_2) { + throw Error("protocol mismatch"); + } + + conn.from >> conn.daemonVersion; + if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != + GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) { + throw Error("Nix daemon protocol version not supported"); + } + if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) { + throw Error("the Nix daemon version is too old"); + } + conn.to << PROTOCOL_VERSION; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) { + conn.to << 1 << cpu; + } else { + conn.to << 0; + } + } + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) { + conn.to << 0u; + } + + auto ex = conn.processStderr(); + if (ex) { + std::rethrow_exception(ex); + } + } catch (Error& e) { + throw Error("cannot open connection to remote store '%s': %s", getUri(), + e.what()); + } + + setOptions(conn); +} + +void RemoteStore::setOptions(Connection& conn) { + conn.to << wopSetOptions << static_cast<uint64_t>(settings.keepFailed) + << static_cast<uint64_t>(settings.keepGoing) + << static_cast<uint64_t>(settings.tryFallback) + << /* previously: verbosity = */ 0 << settings.maxBuildJobs + << settings.maxSilentTime << 1u + << /* previously: remote verbosity = */ 0 << 0 // obsolete log type + << 0 /* obsolete print build trace */ + << settings.buildCores + << static_cast<uint64_t>(settings.useSubstitutes); + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + std::map<std::string, Config::SettingInfo> overrides; + globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); + conn.to << overrides.size(); + for (auto& i : overrides) { + conn.to << i.first << i.second.value; + } + } + + auto ex = conn.processStderr(); + if (ex) { + std::rethrow_exception(ex); + } +} + +/* A wrapper around Pool<RemoteStore::Connection>::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle { + Pool<RemoteStore::Connection>::Handle handle; + bool daemonException = false; + + explicit ConnectionHandle(Pool<RemoteStore::Connection>::Handle&& handle) + : handle(std::move(handle)) {} + + ConnectionHandle(ConnectionHandle&& h) : handle(std::move(h.handle)) {} + + ~ConnectionHandle() { + if (!daemonException && (std::uncaught_exceptions() != 0)) { + handle.markBad(); + // TODO(tazjin): are these types of things supposed to be DEBUG? + DLOG(INFO) << "closing daemon connection because of an exception"; + } + } + + RemoteStore::Connection* operator->() { return &*handle; } + + void processStderr(Sink* sink = nullptr, Source* source = nullptr) { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } +}; + +ConnectionHandle RemoteStore::getConnection() { + return ConnectionHandle(connections->get()); +} + +bool RemoteStore::isValidPathUncached(const Path& path) { + auto conn(getConnection()); + conn->to << wopIsValidPath << path; + conn.processStderr(); + return readInt(conn->from) != 0u; +} + +PathSet RemoteStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + PathSet res; + for (auto& i : paths) { + if (isValidPath(i)) { + res.insert(i); + } + } + return res; + } + conn->to << wopQueryValidPaths << paths; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +PathSet RemoteStore::queryAllValidPaths() { + auto conn(getConnection()); + conn->to << wopQueryAllValidPaths; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +PathSet RemoteStore::querySubstitutablePaths(const PathSet& paths) { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + PathSet res; + for (auto& i : paths) { + conn->to << wopHasSubstitutes << i; + conn.processStderr(); + if (readInt(conn->from) != 0u) { + res.insert(i); + } + } + return res; + } + conn->to << wopQuerySubstitutablePaths << paths; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +void RemoteStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + if (paths.empty()) { + return; + } + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + for (auto& i : paths) { + SubstitutablePathInfo info; + conn->to << wopQuerySubstitutablePathInfo << i; + conn.processStderr(); + unsigned int reply = readInt(conn->from); + if (reply == 0) { + continue; + } + info.deriver = readString(conn->from); + if (!info.deriver.empty()) { + assertStorePath(info.deriver); + } + info.references = readStorePaths<PathSet>(*this, conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + infos[i] = info; + } + + } else { + conn->to << wopQuerySubstitutablePathInfos << paths; + conn.processStderr(); + auto count = readNum<size_t>(conn->from); + for (size_t n = 0; n < count; n++) { + Path path = readStorePath(*this, conn->from); + SubstitutablePathInfo& info(infos[path]); + info.deriver = readString(conn->from); + if (!info.deriver.empty()) { + assertStorePath(info.deriver); + } + info.references = readStorePaths<PathSet>(*this, conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + } + } +} + +void RemoteStore::queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + try { + std::shared_ptr<ValidPathInfo> info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << path; + try { + conn.processStderr(); + } catch (Error& e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) { + throw InvalidPath(e.what()); + } + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; + conn->from >> valid; + if (!valid) { + throw InvalidPath(format("path '%s' is not valid") % path); + } + } + info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = readString(conn->from); + if (!info->deriver.empty()) { + assertStorePath(info->deriver); + } + auto hash_ = Hash::deserialize(readString(conn->from), htSHA256); + info->narHash = Hash::unwrap_throw(hash_); + info->references = readStorePaths<PathSet>(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings<StringSet>(conn->from); + conn->from >> info->ca; + } + } + callback(std::move(info)); + } catch (...) { + callback.rethrow(); + } +} + +void RemoteStore::queryReferrers(const Path& path, PathSet& referrers) { + auto conn(getConnection()); + conn->to << wopQueryReferrers << path; + conn.processStderr(); + auto referrers2 = readStorePaths<PathSet>(*this, conn->from); + referrers.insert(referrers2.begin(), referrers2.end()); +} + +PathSet RemoteStore::queryValidDerivers(const Path& path) { + auto conn(getConnection()); + conn->to << wopQueryValidDerivers << path; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +PathSet RemoteStore::queryDerivationOutputs(const Path& path) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputs << path; + conn.processStderr(); + return readStorePaths<PathSet>(*this, conn->from); +} + +PathSet RemoteStore::queryDerivationOutputNames(const Path& path) { + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputNames << path; + conn.processStderr(); + return readStrings<PathSet>(conn->from); +} + +Path RemoteStore::queryPathFromHashPart(const std::string& hashPart) { + auto conn(getConnection()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn.processStderr(); + Path path = readString(conn->from); + if (!path.empty()) { + assertStorePath(path); + } + return path; +} + +void RemoteStore::addToStore(const ValidPathInfo& info, Source& source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + conn->to << wopImportPaths; + + auto source2 = sinkToSource([&](Sink& sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink << exportMagic << info.path << info.references << info.deriver + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn.processStderr(nullptr, source2.get()); + + auto importedPaths = readStorePaths<PathSet>(*this, conn->from); + assert(importedPaths.size() <= 1); + } + + else { + conn->to << wopAddToStoreNar << info.path << info.deriver + << info.narHash.to_string(Base16, false) << info.references + << info.registrationTime << info.narSize << info.ultimate + << info.sigs << info.ca << repair << !checkSigs; + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) { + copyNAR(source, conn->to); + } + conn.processStderr(nullptr, tunnel ? &source : nullptr); + } +} + +Path RemoteStore::addToStore(const std::string& name, const Path& _srcPath, + bool recursive, HashType hashAlgo, + PathFilter& filter, RepairFlag repair) { + if (repair != 0u) { + throw Error( + "repairing is not supported when building through the Nix daemon"); + } + + auto conn(getConnection()); + + Path srcPath(absPath(_srcPath)); + + conn->to << wopAddToStore << name + << ((hashAlgo == htSHA256 && recursive) + ? 0 + : 1) /* backwards compatibility hack */ + << (recursive ? 1 : 0) << printHashType(hashAlgo); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + dumpPath(srcPath, conn->to, filter); + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError& e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) { + try { + conn.processStderr(); + } catch (EndOfFile& e) { + } + } + throw; + } + + return readStorePath(*this, conn->from); +} + +Path RemoteStore::addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair) { + if (repair != 0u) { + throw Error( + "repairing is not supported when building through the Nix daemon"); + } + + auto conn(getConnection()); + conn->to << wopAddTextToStore << name << s << references; + + conn.processStderr(); + return readStorePath(*this, conn->from); +} + +absl::Status RemoteStore::buildPaths(std::ostream& /* log_sink */, + const PathSet& drvPaths, + BuildMode build_mode) { + auto conn(getConnection()); + conn->to << wopBuildPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { + conn->to << drvPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) { + conn->to << build_mode; + } else if (build_mode != bmNormal) { + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + return absl::Status( + absl::StatusCode::kInvalidArgument, + "repairing or checking is not supported when building through the " + "Nix daemon"); + } + } else { + /* For backwards compatibility with old daemons, strip output + identifiers. */ + PathSet drvPaths2; + for (auto& i : drvPaths) { + drvPaths2.insert(std::string(i, 0, i.find('!'))); + } + conn->to << drvPaths2; + } + conn.processStderr(); + readInt(conn->from); + + return absl::OkStatus(); +} + +BuildResult RemoteStore::buildDerivation(std::ostream& /*log_sink*/, + const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + auto conn(getConnection()); + conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn.processStderr(); + BuildResult res; + unsigned int status; + conn->from >> status >> res.errorMsg; + res.status = static_cast<BuildResult::Status>(status); + return res; +} + +void RemoteStore::ensurePath(const Path& path) { + auto conn(getConnection()); + conn->to << wopEnsurePath << path; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::addTempRoot(const Path& path) { + auto conn(getConnection()); + conn->to << wopAddTempRoot << path; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::addIndirectRoot(const Path& path) { + auto conn(getConnection()); + conn->to << wopAddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::syncWithGC() { + auto conn(getConnection()); + conn->to << wopSyncWithGC; + conn.processStderr(); + readInt(conn->from); +} + +Roots RemoteStore::findRoots(bool censor) { + auto conn(getConnection()); + conn->to << wopFindRoots; + conn.processStderr(); + auto count = readNum<size_t>(conn->from); + Roots result; + while ((count--) != 0u) { + Path link = readString(conn->from); + Path target = readStorePath(*this, conn->from); + result[target].emplace(link); + } + return result; +} + +void RemoteStore::collectGarbage(const GCOptions& options, GCResults& results) { + auto conn(getConnection()); + + conn->to << wopCollectGarbage << options.action << options.pathsToDelete + << static_cast<uint64_t>(options.ignoreLiveness) + << options.maxFreed + /* removed options */ + << 0 << 0 << 0; + + conn.processStderr(); + + results.paths = readStrings<PathSet>(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } +} + +void RemoteStore::optimiseStore() { + auto conn(getConnection()); + conn->to << wopOptimiseStore; + conn.processStderr(); + readInt(conn->from); +} + +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) { + auto conn(getConnection()); + conn->to << wopVerifyStore << static_cast<uint64_t>(checkContents) << repair; + conn.processStderr(); + return readInt(conn->from) != 0u; +} + +void RemoteStore::addSignatures(const Path& storePath, const StringSet& sigs) { + auto conn(getConnection()); + conn->to << wopAddSignatures << storePath << sigs; + conn.processStderr(); + readInt(conn->from); +} + +void RemoteStore::queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { + { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) { + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + } + conn->to << wopQueryMissing << targets; + conn.processStderr(); + willBuild = readStorePaths<PathSet>(*this, conn->from); + willSubstitute = readStorePaths<PathSet>(*this, conn->from); + unknown = readStorePaths<PathSet>(*this, conn->from); + conn->from >> downloadSize >> narSize; + return; + } + +fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); +} + +void RemoteStore::connect() { auto conn(getConnection()); } + +unsigned int RemoteStore::getProtocol() { + auto conn(connections->get()); + return conn->daemonVersion; +} + +void RemoteStore::flushBadConnections() { connections->flushBad(); } + +RemoteStore::Connection::~Connection() { + try { + to.flush(); + } catch (...) { + ignoreException(); + } +} + +std::exception_ptr RemoteStore::Connection::processStderr(Sink* sink, + Source* source) { + to.flush(); + + while (true) { + auto msg = readNum<uint64_t>(from); + + if (msg == STDERR_WRITE) { + std::string s = readString(from); + if (sink == nullptr) { + throw Error("no sink"); + } + (*sink)(s); + } + + else if (msg == STDERR_READ) { + if (source == nullptr) { + throw Error("no source"); + } + auto len = readNum<size_t>(from); + auto buf = std::make_unique<unsigned char[]>(len); + writeString(buf.get(), source->read(buf.get(), len), to); + to.flush(); + } + + else if (msg == STDERR_ERROR) { + std::string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } + + else if (msg == STDERR_NEXT) { + LOG(ERROR) << absl::StripTrailingAsciiWhitespace(readString(from)); + } + + else if (msg == STDERR_START_ACTIVITY) { + LOG(INFO) << readString(from); + } + + else if (msg == STDERR_LAST) { + break; + } + + else { + throw Error("got unknown message type %x from Nix daemon", msg); + } + } + + return nullptr; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/remote-store.hh b/third_party/nix/src/libstore/remote-store.hh new file mode 100644 index 000000000000..c360055b6e6b --- /dev/null +++ b/third_party/nix/src/libstore/remote-store.hh @@ -0,0 +1,141 @@ +#pragma once + +#include <limits> +#include <string> + +#include "libstore/store-api.hh" + +namespace nix { + +class Pipe; +class Pid; +struct FdSink; +struct FdSource; +template <typename T> +class Pool; +struct ConnectionHandle; + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store { + public: + const Setting<int> maxConnections{ + (Store*)this, 1, "max-connections", + "maximum number of concurrent connections to the Nix daemon"}; + + const Setting<unsigned int> maxConnectionAge{ + (Store*)this, std::numeric_limits<unsigned int>::max(), + "max-connection-age", "number of seconds to reuse a connection"}; + + virtual bool sameMachine() = 0; + + RemoteStore(const Params& params); + + /* Implementations of abstract store API methods. */ + + bool isValidPathUncached(const Path& path) override; + + PathSet queryValidPaths(const PathSet& paths, SubstituteFlag maybeSubstitute = + NoSubstitute) override; + + PathSet queryAllValidPaths() override; + + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + + void queryReferrers(const Path& path, PathSet& referrers) override; + + PathSet queryValidDerivers(const Path& path) override; + + PathSet queryDerivationOutputs(const Path& path) override; + + StringSet queryDerivationOutputNames(const Path& path) override; + + Path queryPathFromHashPart(const std::string& hashPart) override; + + PathSet querySubstitutablePaths(const PathSet& paths) override; + + void querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) override; + + void addToStore(const ValidPathInfo& info, Source& source, RepairFlag repair, + CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) override; + + Path addToStore(const std::string& name, const Path& srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter, + RepairFlag repair = NoRepair) override; + + Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair) override; + + absl::Status buildPaths(std::ostream& log_sink, const PathSet& paths, + BuildMode build_mode) override; + + BuildResult buildDerivation(std::ostream& log_sink, const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) override; + + void ensurePath(const Path& path) override; + + void addTempRoot(const Path& path) override; + + void addIndirectRoot(const Path& path) override; + + void syncWithGC() override; + + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions& options, GCResults& results) override; + + void optimiseStore() override; + + bool verifyStore(bool checkContents, RepairFlag repair) override; + + void addSignatures(const Path& storePath, const StringSet& sigs) override; + + void queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) override; + + void connect() override; + + unsigned int getProtocol() override; + + void flushBadConnections(); + + protected: + struct Connection { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + std::chrono::time_point<std::chrono::steady_clock> startTime; + + virtual ~Connection(); + + std::exception_ptr processStderr(Sink* sink = 0, Source* source = 0); + }; + + ref<Connection> openConnectionWrapper(); + + virtual ref<Connection> openConnection() = 0; + + void initConnection(Connection& conn); + + ref<Pool<Connection>> connections; + + virtual void setOptions(Connection& conn); + + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + + private: + std::atomic_bool failed{false}; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/rpc-store.cc b/third_party/nix/src/libstore/rpc-store.cc new file mode 100644 index 000000000000..c29bd059de9b --- /dev/null +++ b/third_party/nix/src/libstore/rpc-store.cc @@ -0,0 +1,549 @@ +#include "rpc-store.hh" + +#include <algorithm> +#include <filesystem> +#include <memory> +#include <optional> +#include <ostream> +#include <string_view> + +#include <absl/status/status.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_format.h> +#include <absl/strings/string_view.h> +#include <glog/logging.h> +#include <google/protobuf/empty.pb.h> +#include <google/protobuf/util/time_util.h> +#include <grpcpp/create_channel.h> +#include <grpcpp/impl/codegen/async_unary_call.h> +#include <grpcpp/impl/codegen/client_context.h> +#include <grpcpp/impl/codegen/completion_queue.h> +#include <grpcpp/impl/codegen/status.h> +#include <grpcpp/impl/codegen/status_code_enum.h> +#include <grpcpp/impl/codegen/sync_stream.h> +#include <grpcpp/security/credentials.h> +#include <sys/ucontext.h> + +#include "libproto/worker.grpc.pb.h" +#include "libproto/worker.pb.h" +#include "libstore/derivations.hh" +#include "libstore/store-api.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/archive.hh" +#include "libutil/hash.hh" +#include "libutil/proto.hh" +#include "libutil/types.hh" + +namespace nix { + +namespace store { + +// Should be set to the bandwidth delay product between the client and the +// daemon. The current value, which should eventually be determined dynamically, +// has currently been set to a developer's deskop computer, rounded up +constexpr size_t kChunkSize = 1024 * 64; + +using google::protobuf::util::TimeUtil; +using grpc::ClientContext; +using nix::proto::WorkerService; + +static google::protobuf::Empty kEmpty; + +template <typename Request> +class RPCSink : public BufferedSink { + public: + using Writer = grpc::ClientWriter<Request>; + explicit RPCSink(std::unique_ptr<Writer>&& writer) + : writer_(std::move(writer)), good_(true) {} + + bool good() override { return good_; } + + void write(const unsigned char* data, size_t len) override { + Request req; + req.set_data(data, len); + if (!writer_->Write(req)) { + good_ = false; + } + } + + ~RPCSink() override { flush(); } + + grpc::Status Finish() { + flush(); + return writer_->Finish(); + } + + private: + std::unique_ptr<Writer> writer_; + bool good_; +}; + +// TODO(grfn): Obviously this should go away and be replaced by StatusOr... but +// that would require refactoring the entire store api, which we don't feel like +// doing right now. We should at some point though +void const RpcStore::SuccessOrThrow(const grpc::Status& status, + const absl::string_view& call) const { + if (!status.ok()) { + auto uri = uri_.value_or("unknown URI"); + switch (status.error_code()) { + case grpc::StatusCode::UNIMPLEMENTED: + throw Unsupported( + absl::StrFormat("operation %s is not supported by store at %s: %s", + call, uri, status.error_message())); + default: + throw Error(absl::StrFormat( + "Rpc call %s to %s failed (%s): %s ", call, uri, + util::proto::GRPCStatusCodeDescription(status.error_code()), + status.error_message())); + } + } +} + +bool RpcStore::isValidPathUncached(const Path& path) { + ClientContext ctx; + proto::IsValidPathResponse resp; + SuccessOrThrow(stub_->IsValidPath(&ctx, util::proto::StorePath(path), &resp), + __FUNCTION__); + return resp.is_valid(); +} + +PathSet RpcStore::queryAllValidPaths() { + ClientContext ctx; + proto::StorePaths paths; + SuccessOrThrow(stub_->QueryAllValidPaths(&ctx, kEmpty, &paths), __FUNCTION__); + return util::proto::FillFrom<PathSet>(paths.paths()); +} + +PathSet RpcStore::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + ClientContext ctx; + proto::StorePaths store_paths; + for (const auto& path : paths) { + store_paths.add_paths(path); + } + proto::StorePaths result_paths; + SuccessOrThrow(stub_->QueryValidPaths(&ctx, store_paths, &result_paths), + __FUNCTION__); + return util::proto::FillFrom<PathSet>(result_paths.paths()); +} + +void RpcStore::queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept { + ClientContext ctx; + proto::StorePath store_path; + store_path.set_path(path); + + try { + proto::PathInfo path_info; + auto result = stub_->QueryPathInfo(&ctx, store_path, &path_info); + if (result.error_code() == grpc::INVALID_ARGUMENT) { + throw InvalidPath(absl::StrFormat("path '%s' is not valid", path)); + } + SuccessOrThrow(result); + + std::shared_ptr<ValidPathInfo> info; + + if (!path_info.is_valid()) { + throw InvalidPath(absl::StrFormat("path '%s' is not valid", path)); + } + + info = std::make_shared<ValidPathInfo>(); + info->path = path; + info->deriver = path_info.deriver().path(); + if (!info->deriver.empty()) { + assertStorePath(info->deriver); + } + auto hash_ = Hash::deserialize(path_info.nar_hash(), htSHA256); + info->narHash = Hash::unwrap_throw(hash_); + info->references.insert(path_info.references().begin(), + path_info.references().end()); + info->registrationTime = + TimeUtil::TimestampToTimeT(path_info.registration_time()); + info->narSize = path_info.nar_size(); + info->ultimate = path_info.ultimate(); + info->sigs.insert(path_info.sigs().begin(), path_info.sigs().end()); + info->ca = path_info.ca(); + + callback(std::move(info)); + } catch (...) { + callback.rethrow(); + } +} + +void RpcStore::queryReferrers(const Path& path, PathSet& referrers) { + ClientContext ctx; + proto::StorePaths paths; + SuccessOrThrow( + stub_->QueryReferrers(&ctx, util::proto::StorePath(path), &paths), + __FUNCTION__); + referrers.insert(paths.paths().begin(), paths.paths().end()); +} + +PathSet RpcStore::queryValidDerivers(const Path& path) { + ClientContext ctx; + proto::StorePaths paths; + SuccessOrThrow( + stub_->QueryValidDerivers(&ctx, util::proto::StorePath(path), &paths), + __FUNCTION__); + return util::proto::FillFrom<PathSet>(paths.paths()); +} + +PathSet RpcStore::queryDerivationOutputs(const Path& path) { + ClientContext ctx; + proto::StorePaths paths; + SuccessOrThrow( + stub_->QueryDerivationOutputs(&ctx, util::proto::StorePath(path), &paths), + __FUNCTION__); + return util::proto::FillFrom<PathSet>(paths.paths()); +} + +StringSet RpcStore::queryDerivationOutputNames(const Path& path) { + ClientContext ctx; + proto::DerivationOutputNames output_names; + SuccessOrThrow(stub_->QueryDerivationOutputNames( + &ctx, util::proto::StorePath(path), &output_names)); + return util::proto::FillFrom<StringSet>(output_names.names()); +} + +Path RpcStore::queryPathFromHashPart(const std::string& hashPart) { + ClientContext ctx; + proto::StorePath path; + proto::HashPart proto_hash_part; + proto_hash_part.set_hash_part(hashPart); + SuccessOrThrow(stub_->QueryPathFromHashPart(&ctx, proto_hash_part, &path), + __FUNCTION__); + return path.path(); +} + +PathSet RpcStore::querySubstitutablePaths(const PathSet& paths) { + ClientContext ctx; + proto::StorePaths result; + SuccessOrThrow(stub_->QuerySubstitutablePaths( + &ctx, util::proto::StorePaths(paths), &result)); + return util::proto::FillFrom<PathSet>(result.paths()); +} + +void RpcStore::querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + ClientContext ctx; + proto::SubstitutablePathInfos result; + SuccessOrThrow(stub_->QuerySubstitutablePathInfos( + &ctx, util::proto::StorePaths(paths), &result)); + + for (const auto& path_info : result.path_infos()) { + auto path = path_info.path().path(); + SubstitutablePathInfo& info(infos[path]); + info.deriver = path_info.deriver().path(); + if (!info.deriver.empty()) { + assertStorePath(info.deriver); + } + info.references = util::proto::FillFrom<PathSet>(path_info.references()); + info.downloadSize = path_info.download_size(); + info.narSize = path_info.nar_size(); + } +} + +void RpcStore::addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + ClientContext ctx; + google::protobuf::Empty response; + auto writer = stub_->AddToStoreNar(&ctx, &response); + + proto::AddToStoreNarRequest path_info_req; + path_info_req.mutable_path_info()->mutable_path()->set_path(info.path); + path_info_req.mutable_path_info()->mutable_deriver()->set_path(info.deriver); + path_info_req.mutable_path_info()->set_nar_hash( + info.narHash.to_string(Base16, false)); + for (const auto& ref : info.references) { + path_info_req.mutable_path_info()->add_references(ref); + } + *path_info_req.mutable_path_info()->mutable_registration_time() = + TimeUtil::TimeTToTimestamp(info.registrationTime); + path_info_req.mutable_path_info()->set_nar_size(info.narSize); + path_info_req.mutable_path_info()->set_ultimate(info.ultimate); + for (const auto& sig : info.sigs) { + path_info_req.mutable_path_info()->add_sigs(sig); + } + path_info_req.mutable_path_info()->set_ca(info.ca); + path_info_req.mutable_path_info()->set_repair(repair); + path_info_req.mutable_path_info()->set_check_sigs(checkSigs); + + if (!writer->Write(path_info_req)) { + throw Error("Could not write to nix daemon"); + } + + RPCSink sink(std::move(writer)); + copyNAR(narSource, sink); + SuccessOrThrow(sink.Finish(), __FUNCTION__); +} + +Path RpcStore::addToStore(const std::string& name, const Path& srcPath, + bool recursive, HashType hashAlgo, PathFilter& filter, + RepairFlag repair) { + if (repair != 0u) { + throw Error( + "repairing is not supported when building through the Nix daemon"); + } + + ClientContext ctx; + proto::StorePath response; + auto writer = stub_->AddToStore(&ctx, &response); + + proto::AddToStoreRequest metadata_req; + metadata_req.mutable_meta()->set_base_name(name); + // TODO(grfn): what is fixed? + metadata_req.mutable_meta()->set_fixed(!(hashAlgo == htSHA256 && recursive)); + metadata_req.mutable_meta()->set_recursive(recursive); + metadata_req.mutable_meta()->set_hash_type(HashTypeToProto(hashAlgo)); + + if (!writer->Write(metadata_req)) { + throw Error("Could not write to nix daemon"); + } + + RPCSink sink(std::move(writer)); + dumpPath(std::filesystem::absolute(srcPath), sink); + sink.flush(); + SuccessOrThrow(sink.Finish(), __FUNCTION__); + + return response.path(); +} + +Path RpcStore::addTextToStore(const std::string& name, + const std::string& content, + const PathSet& references, RepairFlag repair) { + if (repair != 0u) { + throw Error( + "repairing is not supported when building through the Nix daemon"); + } + ClientContext ctx; + proto::StorePath result; + auto writer = stub_->AddTextToStore(&ctx, &result); + + proto::AddTextToStoreRequest meta; + meta.mutable_meta()->set_name(name); + meta.mutable_meta()->set_size(content.size()); + for (const auto& ref : references) { + meta.mutable_meta()->add_references(ref); + } + writer->Write(meta); + + for (int i = 0; i <= content.size(); i += kChunkSize) { + auto len = std::min(kChunkSize, content.size() - i); + proto::AddTextToStoreRequest data; + data.set_data(content.data() + i, len); + if (!writer->Write(data)) { + // Finish() below will error + break; + } + } + + writer->WritesDone(); + SuccessOrThrow(writer->Finish(), __FUNCTION__); + return result.path(); +} + +absl::Status RpcStore::buildPaths(std::ostream& log_sink, const PathSet& paths, + BuildMode build_mode) { + ClientContext ctx; + proto::BuildPathsRequest request; + for (const auto& path : paths) { + request.add_drvs(path); + } + + google::protobuf::Empty response; + request.set_mode(nix::BuildModeToProto(build_mode)); + + std::unique_ptr<grpc::ClientReader<proto::BuildEvent>> reader = + stub_->BuildPaths(&ctx, request); + + proto::BuildEvent event; + while (reader->Read(&event)) { + if (event.has_build_log()) { + // TODO(tazjin): Include .path()? + log_sink << event.build_log().line(); + } else { + log_sink << "Building path: " << event.building_path().path() + << std::endl; + } + + // has_result() is not in use in this call (for now) + } + + return nix::util::proto::GRPCStatusToAbsl(reader->Finish()); +} + +BuildResult RpcStore::buildDerivation(std::ostream& log_sink, + const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) { + ClientContext ctx; + proto::BuildDerivationRequest request; + request.mutable_drv_path()->set_path(drvPath); + proto::Derivation proto_drv = drv.to_proto(); + *request.mutable_derivation() = proto_drv; + request.set_build_mode(BuildModeToProto(buildMode)); + + std::unique_ptr<grpc::ClientReader<proto::BuildEvent>> reader = + stub_->BuildDerivation(&ctx, request); + + std::optional<BuildResult> result; + + proto::BuildEvent event; + while (reader->Read(&event)) { + if (event.has_build_log()) { + log_sink << event.build_log().line(); + } else if (event.has_result()) { + result = BuildResult::FromProto(event.result()); + } + } + SuccessOrThrow(reader->Finish(), __FUNCTION__); + + if (!result.has_value()) { + throw Error("Invalid response from daemon for buildDerivation"); + } + return result.value(); +} + +void RpcStore::ensurePath(const Path& path) { + ClientContext ctx; + google::protobuf::Empty response; + SuccessOrThrow( + stub_->EnsurePath(&ctx, util::proto::StorePath(path), &response), + __FUNCTION__); +} + +void RpcStore::addTempRoot(const Path& path) { + ClientContext ctx; + google::protobuf::Empty response; + SuccessOrThrow( + stub_->AddTempRoot(&ctx, util::proto::StorePath(path), &response), + __FUNCTION__); +} + +void RpcStore::addIndirectRoot(const Path& path) { + ClientContext ctx; + google::protobuf::Empty response; + SuccessOrThrow( + stub_->AddIndirectRoot(&ctx, util::proto::StorePath(path), &response), + __FUNCTION__); +} + +void RpcStore::syncWithGC() { + ClientContext ctx; + google::protobuf::Empty response; + SuccessOrThrow(stub_->SyncWithGC(&ctx, kEmpty, &response), __FUNCTION__); +} + +Roots RpcStore::findRoots(bool censor) { + ClientContext ctx; + proto::FindRootsResponse response; + SuccessOrThrow(stub_->FindRoots(&ctx, kEmpty, &response), __FUNCTION__); + Roots result; + + for (const auto& [target, links] : response.roots()) { + auto link_paths = + util::proto::FillFrom<std::unordered_set<std::string>>(links.paths()); + result.insert({target, link_paths}); + } + + return result; +} + +void RpcStore::collectGarbage(const GCOptions& options, GCResults& results) { + ClientContext ctx; + proto::CollectGarbageRequest request; + request.set_action(options.ActionToProto()); + for (const auto& path : options.pathsToDelete) { + request.add_paths_to_delete(path); + } + request.set_ignore_liveness(options.ignoreLiveness); + request.set_max_freed(options.maxFreed); + + proto::CollectGarbageResponse response; + SuccessOrThrow(stub_->CollectGarbage(&ctx, request, &response), __FUNCTION__); + + for (const auto& path : response.deleted_paths()) { + results.paths.insert(path); + } + results.bytesFreed = response.bytes_freed(); +} + +void RpcStore::optimiseStore() { + ClientContext ctx; + google::protobuf::Empty response; + SuccessOrThrow(stub_->OptimiseStore(&ctx, kEmpty, &response), __FUNCTION__); +} + +bool RpcStore::verifyStore(bool checkContents, RepairFlag repair) { + ClientContext ctx; + proto::VerifyStoreRequest request; + request.set_check_contents(checkContents); + request.set_repair(repair); + proto::VerifyStoreResponse response; + SuccessOrThrow(stub_->VerifyStore(&ctx, request, &response), __FUNCTION__); + return response.errors(); +} + +void RpcStore::addSignatures(const Path& storePath, const StringSet& sigs) { + ClientContext ctx; + proto::AddSignaturesRequest request; + request.mutable_path()->set_path(storePath); + for (const auto& sig : sigs) { + request.mutable_sigs()->add_sigs(sig); + } + google::protobuf::Empty response; + SuccessOrThrow(stub_->AddSignatures(&ctx, request, &response), __FUNCTION__); +} + +void RpcStore::queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) { + ClientContext ctx; + proto::QueryMissingResponse response; + SuccessOrThrow( + stub_->QueryMissing(&ctx, util::proto::StorePaths(targets), &response), + __FUNCTION__); + + willBuild = util::proto::FillFrom<PathSet>(response.will_build()); + willSubstitute = util::proto::FillFrom<PathSet>(response.will_substitute()); + unknown = util::proto::FillFrom<PathSet>(response.unknown()); + downloadSize = response.download_size(); + narSize = response.nar_size(); +} + +std::shared_ptr<std::string> RpcStore::getBuildLog(const Path& path) { + ClientContext ctx; + proto::BuildLog response; + SuccessOrThrow( + stub_->GetBuildLog(&ctx, util::proto::StorePath(path), &response), + __FUNCTION__); + + auto build_log = response.build_log(); + if (build_log.empty()) { + return nullptr; + } + return std::make_shared<std::string>(build_log); +} + +unsigned int RpcStore::getProtocol() { return PROTOCOL_VERSION; } + +} // namespace store + +constexpr std::string_view kUriScheme = "unix://"; + +// TODO(grfn): Make this a function that we call from main rather than... this +static RegisterStoreImplementation regStore([](const std::string& uri, + const Store::Params& params) + -> std::shared_ptr<Store> { + if (std::string(uri, 0, kUriScheme.size()) != kUriScheme) { + return nullptr; + } + auto channel = grpc::CreateChannel(uri, grpc::InsecureChannelCredentials()); + return std::make_shared<store::RpcStore>( + uri, params, proto::WorkerService::NewStub(channel)); +}); + +} // namespace nix diff --git a/third_party/nix/src/libstore/rpc-store.hh b/third_party/nix/src/libstore/rpc-store.hh new file mode 100644 index 000000000000..679ceac7af9c --- /dev/null +++ b/third_party/nix/src/libstore/rpc-store.hh @@ -0,0 +1,129 @@ +#pragma once + +#include <absl/strings/string_view.h> + +#include "libproto/worker.grpc.pb.h" +#include "libproto/worker.pb.h" +#include "libstore/remote-store.hh" +#include "libstore/store-api.hh" + +namespace nix::store { + +// TODO(grfn): Currently, since the RPCStore is only used for the connection to +// the nix daemon over a unix socket, it inherits from the LocalFSStore since it +// shares a filesystem with the daemon. This will not always be the case, at +// which point we should tease these two things apart. +class RpcStore : public LocalFSStore, public virtual Store { + public: + RpcStore(const Params& params, + std::unique_ptr<nix::proto::WorkerService::Stub> stub) + : Store(params), LocalFSStore(params), stub_(std::move(stub)) {} + + RpcStore(std::string uri, const Params& params, + std::unique_ptr<nix::proto::WorkerService::Stub> stub) + : Store(params), + LocalFSStore(params), + uri_(uri), + stub_(std::move(stub)) {} + + std::string getUri() override { + if (uri_.has_value()) { + return uri_.value(); + } else { + return "daemon"; + } + }; + + virtual PathSet queryAllValidPaths() override; + + virtual void queryReferrers(const Path& path, PathSet& referrers) override; + + virtual PathSet queryValidDerivers(const Path& path) override; + + virtual PathSet queryDerivationOutputs(const Path& path) override; + + virtual StringSet queryDerivationOutputNames(const Path& path) override; + + virtual Path queryPathFromHashPart(const std::string& hashPart) override; + + virtual PathSet querySubstitutablePaths(const PathSet& paths) override; + + virtual void querySubstitutablePathInfos( + const PathSet& paths, SubstitutablePathInfos& infos) override; + + virtual bool wantMassQuery() override { return true; } + + virtual void addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr<FSAccessor> accessor = 0) override; + + virtual Path addToStore(const std::string& name, const Path& srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter, + RepairFlag repair = NoRepair) override; + + virtual Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, + RepairFlag repair = NoRepair) override; + + absl::Status buildPaths(std::ostream& log_sink, const PathSet& paths, + BuildMode build_mode) override; + + virtual BuildResult buildDerivation(std::ostream& log_sink, + const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) override; + + virtual void ensurePath(const Path& path) override; + + virtual void addTempRoot(const Path& path) override; + + virtual void addIndirectRoot(const Path& path) override; + + virtual void syncWithGC() override; + + virtual Roots findRoots(bool censor) override; + + virtual void collectGarbage(const GCOptions& options, + GCResults& results) override; + + virtual void optimiseStore() override; + + virtual bool verifyStore(bool checkContents, + RepairFlag repair = NoRepair) override; + + virtual void addSignatures(const Path& storePath, + const StringSet& sigs) override; + + virtual void queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize) override; + + virtual std::shared_ptr<std::string> getBuildLog(const Path& path) override; + + void connect() override{}; + + virtual unsigned int getProtocol() override; + + protected: + virtual bool isValidPathUncached(const Path& path) override; + + virtual PathSet queryValidPaths( + const PathSet& paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + virtual void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept override; + + private: + std::optional<std::string> uri_; + std::unique_ptr<nix::proto::WorkerService::Stub> stub_; + + void const SuccessOrThrow(const grpc::Status& status, + const absl::string_view& call = "") const; +}; + +} // namespace nix::store diff --git a/third_party/nix/src/libstore/s3-binary-cache-store.cc b/third_party/nix/src/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000000..0c13039b52d4 --- /dev/null +++ b/third_party/nix/src/libstore/s3-binary-cache-store.cc @@ -0,0 +1,431 @@ +#if ENABLE_S3 + +#include "libstore/s3-binary-cache-store.hh" + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <aws/core/Aws.h> +#include <aws/core/VersionConfig.h> +#include <aws/core/auth/AWSCredentialsProvider.h> +#include <aws/core/auth/AWSCredentialsProviderChain.h> +#include <aws/core/client/ClientConfiguration.h> +#include <aws/core/client/DefaultRetryStrategy.h> +#include <aws/core/utils/logging/FormattedLogSystem.h> +#include <aws/core/utils/logging/LogMacros.h> +#include <aws/core/utils/threading/Executor.h> +#include <aws/s3/S3Client.h> +#include <aws/s3/model/GetObjectRequest.h> +#include <aws/s3/model/HeadObjectRequest.h> +#include <aws/s3/model/ListObjectsRequest.h> +#include <aws/s3/model/PutObjectRequest.h> +#include <aws/transfer/TransferManager.h> + +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/nar-info-disk-cache.hh" +#include "libstore/nar-info.hh" +#include "libstore/s3.hh" +#include "libutil/compression.hh" +#include "libutil/istringstream_nocopy.hh" + +using namespace Aws::Transfer; + +namespace nix { + +struct S3Error : public Error { + Aws::S3::S3Errors err; + S3Error(Aws::S3::S3Errors err, const FormatOrString& fs) + : Error(fs), err(err){}; +}; + +/* Helper: given an Outcome<R, E>, return R in case of success, or + throw an exception in case of an error. */ +template <typename R, typename E> +R&& checkAws(const FormatOrString& fs, Aws::Utils::Outcome<R, E>&& outcome) { + if (!outcome.IsSuccess()) + throw S3Error(outcome.GetError().GetErrorType(), + fs.s + ": " + outcome.GetError().GetMessage()); + return outcome.GetResultWithOwnership(); +} + +class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem { + using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; + + void ProcessFormattedStatement(Aws::String&& statement) override { + debug("AWS: %s", absl::StripTrailingAsciiWhitespace(statement)); + } +}; + +static void initAWS() { + static std::once_flag flag; + std::call_once(flag, []() { + Aws::SDKOptions options; + + /* We install our own OpenSSL locking function (see + shared.cc), so don't let aws-sdk-cpp override it. */ + options.cryptoOptions.initAndCleanupOpenSSL = false; + + if (verbosity >= lvlDebug) { + options.loggingOptions.logLevel = + verbosity == lvlDebug ? Aws::Utils::Logging::LogLevel::Debug + : Aws::Utils::Logging::LogLevel::Trace; + options.loggingOptions.logger_create_fn = [options]() { + return std::make_shared<AwsLogger>(options.loggingOptions.logLevel); + }; + } + + Aws::InitAPI(options); + }); +} + +S3Helper::S3Helper(const std::string& profile, const std::string& region, + const std::string& scheme, const std::string& endpoint) + : config(makeConfig(region, scheme, endpoint)), + client(make_ref<Aws::S3::S3Client>( + profile == "" + ? std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( + std::make_shared< + Aws::Auth::DefaultAWSCredentialsProviderChain>()) + : std::dynamic_pointer_cast<Aws::Auth::AWSCredentialsProvider>( + std::make_shared< + Aws::Auth::ProfileConfigFileAWSCredentialsProvider>( + profile.c_str())), + *config, +// FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 +#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 + false, +#else + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, +#endif + endpoint.empty())) { +} + +/* Log AWS retries. */ +class RetryStrategy : public Aws::Client::DefaultRetryStrategy { + bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>& error, + long attemptedRetries) const override { + auto retry = + Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); + if (retry) + printError("AWS error '%s' (%s), will retry in %d ms", + error.GetExceptionName(), error.GetMessage(), + CalculateDelayBeforeNextRetry(error, attemptedRetries)); + return retry; + } +}; + +ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig( + const std::string& region, const std::string& scheme, + const std::string& endpoint) { + initAWS(); + auto res = make_ref<Aws::Client::ClientConfiguration>(); + res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } + if (!endpoint.empty()) { + res->endpointOverride = endpoint; + } + res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; + res->retryStrategy = std::make_shared<RetryStrategy>(); + res->caFile = settings.caFile; + return res; +} + +S3Helper::DownloadResult S3Helper::getObject(const std::string& bucketName, + const std::string& key) { + debug("fetching 's3://%s/%s'...", bucketName, key); + + auto request = + Aws::S3::Model::GetObjectRequest().WithBucket(bucketName).WithKey(key); + + request.SetResponseStreamFactory( + [&]() { return Aws::New<std::stringstream>("STRINGSTREAM"); }); + + DownloadResult res; + + auto now1 = std::chrono::steady_clock::now(); + + try { + auto result = checkAws(fmt("AWS error fetching '%s'", key), + client->GetObject(request)); + + res.data = + decompress(result.GetContentEncoding(), + dynamic_cast<std::stringstream&>(result.GetBody()).str()); + + } catch (S3Error& e) { + if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) { + throw; + } + } + + auto now2 = std::chrono::steady_clock::now(); + + res.durationMs = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); + + return res; +} + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore { + const Setting<std::string> profile{ + this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting<std::string> region{ + this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting<std::string> scheme{ + this, "", "scheme", + "The scheme to use for S3 requests, https by default."}; + const Setting<std::string> endpoint{ + this, "", "endpoint", + "An optional override of the endpoint to use when talking to S3."}; + const Setting<std::string> narinfoCompression{ + this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting<std::string> lsCompression{this, "", "ls-compression", + "compression method for .ls files"}; + const Setting<std::string> logCompression{ + this, "", "log-compression", "compression method for log/* files"}; + const Setting<bool> multipartUpload{this, false, "multipart-upload", + "whether to use multi-part uploads"}; + const Setting<uint64_t> bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", + "size (in bytes) of each part in multi-part uploads"}; + + std::string bucketName; + + Stats stats; + + S3Helper s3Helper; + + S3BinaryCacheStoreImpl(const Params& params, const std::string& bucketName) + : S3BinaryCacheStore(params), + bucketName(bucketName), + s3Helper(profile, region, scheme, endpoint) { + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override { return "s3://" + bucketName; } + + void init() override { + if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { + BinaryCacheStore::init(); + + diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); + } + } + + const Stats& getS3Stats() override { return stats; } + + /* This is a specialisation of isValidPath() that optimistically + fetches the .narinfo file, rather than first checking for its + existence via a HEAD request. Since .narinfos are small, doing + a GET is unlikely to be slower than HEAD. */ + bool isValidPathUncached(const Path& storePath) override { + try { + queryPathInfo(storePath); + return true; + } catch (InvalidPath& e) { + return false; + } + } + + bool fileExists(const std::string& path) override { + stats.head++; + + auto res = s3Helper.client->HeadObject(Aws::S3::Model::HeadObjectRequest() + .WithBucket(bucketName) + .WithKey(path)); + + if (!res.IsSuccess()) { + auto& error = res.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND || + error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY + // If bucket listing is disabled, 404s turn into 403s + || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) + return false; + throw Error(format("AWS error fetching '%s': %s") % path % + error.GetMessage()); + } + + return true; + } + + std::shared_ptr<TransferManager> transferManager; + std::once_flag transferManagerCreated; + + void uploadFile(const std::string& path, const std::string& data, + const std::string& mimeType, + const std::string& contentEncoding) { + auto stream = std::make_shared<istringstream_nocopy>(data); + + auto maxThreads = std::thread::hardware_concurrency(); + + static std::shared_ptr<Aws::Utils::Threading::PooledThreadExecutor> + executor = + std::make_shared<Aws::Utils::Threading::PooledThreadExecutor>( + maxThreads); + + std::call_once(transferManagerCreated, [&]() { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager* transferManager, + const std::shared_ptr<const TransferHandle>& transferHandle) { + // FIXME: find a way to properly abort the multipart upload. + // checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); + + auto now1 = std::chrono::steady_clock::now(); + + if (transferManager) { + if (contentEncoding != "") + throw Error( + "setting a content encoding is not supported with S3 multi-part " + "uploads"); + + std::shared_ptr<TransferHandle> transferHandle = + transferManager->UploadFile(stream, bucketName, path, mimeType, + Aws::Map<Aws::String, Aws::String>(), + nullptr /*, contentEncoding */); + + transferHandle->WaitUntilFinished(); + + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", bucketName, + path, transferHandle->GetLastError().GetMessage()); + + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error( + "AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); + + } else { + auto request = Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetContentType(mimeType); + + if (contentEncoding != "") { + request.SetContentEncoding(contentEncoding); + } + + auto stream = std::make_shared<istringstream_nocopy>(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto duration = + std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1) + .count(); + + printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % + bucketName % path % data.size() % duration); + + stats.putTimeMs += duration; + stats.putBytes += data.size(); + stats.put++; + } + + void upsertFile(const std::string& path, const std::string& data, + const std::string& mimeType) override { + if (narinfoCompression != "" && absl::EndsWith(path, ".narinfo")) + uploadFile(path, *compress(narinfoCompression, data), mimeType, + narinfoCompression); + else if (lsCompression != "" && absl::EndsWith(path, ".ls")) + uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + else if (logCompression != "" && absl::StartsWith(path, "log/")) + uploadFile(path, *compress(logCompression, data), mimeType, + logCompression); + else + uploadFile(path, data, mimeType, ""); + } + + void getFile(const std::string& path, Sink& sink) override { + stats.get++; + + // FIXME: stream output to sink. + auto res = s3Helper.getObject(bucketName, path); + + stats.getBytes += res.data ? res.data->size() : 0; + stats.getTimeMs += res.durationMs; + + if (res.data) { + printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", bucketName, + path, res.data->size(), res.durationMs); + + sink((unsigned char*)res.data->data(), res.data->size()); + } else + throw NoSuchBinaryCacheFile( + "file '%s' does not exist in binary cache '%s'", path, getUri()); + } + + PathSet queryAllValidPaths() override { + PathSet paths; + std::string marker; + + do { + debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % + marker); + + auto res = checkAws( + format("AWS error listing bucket '%s'") % bucketName, + s3Helper.client->ListObjects(Aws::S3::Model::ListObjectsRequest() + .WithBucket(bucketName) + .WithDelimiter("/") + .WithMarker(marker))); + + auto& contents = res.GetContents(); + + debug(format("got %d keys, next marker '%s'") % contents.size() % + res.GetNextMarker()); + + for (auto object : contents) { + auto& key = object.GetKey(); + if (key.size() != 40 || !absl::EndsWith(key, ".narinfo")) { + continue; + } + paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); + } + + marker = res.GetNextMarker(); + } while (!marker.empty()); + + return paths; + } +}; + +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, 5) != "s3://") { + return 0; + } + auto store = + std::make_shared<S3BinaryCacheStoreImpl>(params, std::string(uri, 5)); + store->init(); + return store; + }); + +} // namespace nix + +#endif diff --git a/third_party/nix/src/libstore/s3-binary-cache-store.hh b/third_party/nix/src/libstore/s3-binary-cache-store.hh new file mode 100644 index 000000000000..3d0d0b3c4496 --- /dev/null +++ b/third_party/nix/src/libstore/s3-binary-cache-store.hh @@ -0,0 +1,27 @@ +#pragma once + +#include <atomic> + +#include "libstore/binary-cache-store.hh" + +namespace nix { + +class S3BinaryCacheStore : public BinaryCacheStore { + protected: + S3BinaryCacheStore(const Params& params) : BinaryCacheStore(params) {} + + public: + struct Stats { + std::atomic<uint64_t> put{0}; + std::atomic<uint64_t> putBytes{0}; + std::atomic<uint64_t> putTimeMs{0}; + std::atomic<uint64_t> get{0}; + std::atomic<uint64_t> getBytes{0}; + std::atomic<uint64_t> getTimeMs{0}; + std::atomic<uint64_t> head{0}; + }; + + virtual const Stats& getS3Stats() = 0; +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/s3.hh b/third_party/nix/src/libstore/s3.hh new file mode 100644 index 000000000000..4f1852dc3d72 --- /dev/null +++ b/third_party/nix/src/libstore/s3.hh @@ -0,0 +1,42 @@ +#pragma once + +#if ENABLE_S3 + +#include "libutil/ref.hh" + +namespace Aws { +namespace Client { +class ClientConfiguration; +} +} // namespace Aws +namespace Aws { +namespace S3 { +class S3Client; +} +} // namespace Aws + +namespace nix { + +struct S3Helper { + ref<Aws::Client::ClientConfiguration> config; + ref<Aws::S3::S3Client> client; + + S3Helper(const std::string& profile, const std::string& region, + const std::string& scheme, const std::string& endpoint); + + ref<Aws::Client::ClientConfiguration> makeConfig(const std::string& region, + const std::string& scheme, + const std::string& endpoint); + + struct DownloadResult { + std::shared_ptr<std::string> data; + unsigned int durationMs; + }; + + DownloadResult getObject(const std::string& bucketName, + const std::string& key); +}; + +} // namespace nix + +#endif diff --git a/third_party/nix/src/libstore/sandbox-defaults.sb b/third_party/nix/src/libstore/sandbox-defaults.sb new file mode 100644 index 000000000000..0299d1ee45d2 --- /dev/null +++ b/third_party/nix/src/libstore/sandbox-defaults.sb @@ -0,0 +1,87 @@ +(define TMPDIR (param "_GLOBAL_TMP_DIR")) + +(deny default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +; Allow forking. +(allow process-fork) + +; Allow reading system information like #CPUs, etc. +(allow sysctl-read) + +; Allow POSIX semaphores and shared memory. +(allow ipc-posix*) + +; Allow socket creation. +(allow system-socket) + +; Allow sending signals within the sandbox. +(allow signal (target same-sandbox)) + +; Allow getpwuid. +(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) + +; Access to /tmp. +; The network-outbound/network-inbound ones are for unix domain sockets, which +; we allow access to in TMPDIR (but if we allow them more broadly, you could in +; theory escape the sandbox) +(allow file* process-exec network-outbound network-inbound + (literal "/tmp") (subpath TMPDIR)) + +; Some packages like to read the system version. +(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) + +; Without this line clang cannot write to /dev/null, breaking some configure tests. +(allow file-read-metadata (literal "/dev")) + +; Many packages like to do local networking in their test suites, but let's only +; allow it if the package explicitly asks for it. +(if (param "_ALLOW_LOCAL_NETWORKING") + (begin + (allow network* (local ip) (local tcp) (local udp)) + + ; Allow access to /etc/resolv.conf (which is a symlink to + ; /private/var/run/resolv.conf). + ; TODO: deduplicate with sandbox-network.sb + (allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + + (allow file-read* + (literal "/private/var/run/resolv.conf")) + + ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on + (allow file-read-metadata (literal "/etc/hosts")) + (allow file-read* (literal "/private/etc/hosts")) + (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) + +; Standard devices. +(allow file* + (literal "/dev/null") + (literal "/dev/random") + (literal "/dev/stdin") + (literal "/dev/stdout") + (literal "/dev/tty") + (literal "/dev/urandom") + (literal "/dev/zero") + (subpath "/dev/fd")) + +; Does nothing, but reduces build noise. +(allow file* (literal "/dev/dtracehelper")) + +; Allow access to zoneinfo since libSystem needs it. +(allow file-read* (subpath "/usr/share/zoneinfo")) + +(allow file-read* (subpath "/usr/share/locale")) + +; This is mostly to get more specific log messages when builds try to +; access something in /etc or /var. +(allow file-read-metadata + (literal "/etc") + (literal "/var") + (literal "/private/var/tmp")) diff --git a/third_party/nix/src/libstore/sandbox-minimal.sb b/third_party/nix/src/libstore/sandbox-minimal.sb new file mode 100644 index 000000000000..65f5108b3990 --- /dev/null +++ b/third_party/nix/src/libstore/sandbox-minimal.sb @@ -0,0 +1,5 @@ +(allow default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) diff --git a/third_party/nix/src/libstore/sandbox-network.sb b/third_party/nix/src/libstore/sandbox-network.sb new file mode 100644 index 000000000000..56beec761fa8 --- /dev/null +++ b/third_party/nix/src/libstore/sandbox-network.sb @@ -0,0 +1,16 @@ +; Allow local and remote network traffic. +(allow network* (local ip) (remote ip)) + +; Allow access to /etc/resolv.conf (which is a symlink to +; /private/var/run/resolv.conf). +(allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private/var/run/resolv.conf")) + +; Allow DNS lookups. +(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) diff --git a/third_party/nix/src/libstore/schema.sql b/third_party/nix/src/libstore/schema.sql new file mode 100644 index 000000000000..09c71a2b8dd7 --- /dev/null +++ b/third_party/nix/src/libstore/schema.sql @@ -0,0 +1,42 @@ +create table if not exists ValidPaths ( + id integer primary key autoincrement not null, + path text unique not null, + hash text not null, + registrationTime integer not null, + deriver text, + narSize integer, + ultimate integer, -- null implies "false" + sigs text, -- space-separated + ca text -- if not null, an assertion that the path is content-addressed; see ValidPathInfo +); + +create table if not exists Refs ( + referrer integer not null, + reference integer not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict +); + +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger if not exists DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); diff --git a/third_party/nix/src/libstore/serve-protocol.hh b/third_party/nix/src/libstore/serve-protocol.hh new file mode 100644 index 000000000000..04c92e63f6de --- /dev/null +++ b/third_party/nix/src/libstore/serve-protocol.hh @@ -0,0 +1,24 @@ +#pragma once + +namespace nix { + +#define SERVE_MAGIC_1 0x390c9deb +#define SERVE_MAGIC_2 0x5452eecb + +#define SERVE_PROTOCOL_VERSION 0x205 +#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00) +#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff) + +using ServeCommand = enum { + cmdQueryValidPaths = 1, + cmdQueryPathInfos = 2, + cmdDumpStorePath = 3, + cmdImportPaths = 4, + cmdExportPaths = 5, + cmdBuildPaths = 6, + cmdQueryClosure = 7, + cmdBuildDerivation = 8, + cmdAddToStoreNar = 9, +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/sqlite.cc b/third_party/nix/src/libstore/sqlite.cc new file mode 100644 index 000000000000..0fb32326f5c5 --- /dev/null +++ b/third_party/nix/src/libstore/sqlite.cc @@ -0,0 +1,195 @@ +#include "libstore/sqlite.hh" + +#include <atomic> + +#include <glog/logging.h> +#include <sqlite3.h> + +#include "libutil/util.hh" + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs) { + int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); + + auto path = sqlite3_db_filename(db, nullptr); + if (path == nullptr) { + path = "(in-memory)"; + } + + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + throw SQLiteBusy( + err == SQLITE_PROTOCOL + ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) + : fmt("SQLite database '%s' is busy", path)); + } + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); +} + +SQLite::SQLite(const Path& path) { + if (sqlite3_open_v2(path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + nullptr) != SQLITE_OK) { + throw Error(format("cannot open SQLite database '%s'") % path); + } +} + +SQLite::~SQLite() { + try { + if ((db != nullptr) && sqlite3_close(db) != SQLITE_OK) { + throwSQLiteError(db, "closing database"); + } + } catch (...) { + ignoreException(); + } +} + +void SQLite::exec(const std::string& stmt) { + retrySQLite<void>([&]() { + if (sqlite3_exec(db, stmt.c_str(), nullptr, nullptr, nullptr) != + SQLITE_OK) { + throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + } + }); +} + +void SQLiteStmt::create(sqlite3* db, const std::string& sql) { + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, nullptr) != SQLITE_OK) { + throwSQLiteError(db, fmt("creating statement '%s'", sql)); + } + this->db = db; + this->sql = sql; +} + +SQLiteStmt::~SQLiteStmt() { + try { + if ((stmt != nullptr) && sqlite3_finalize(stmt) != SQLITE_OK) { + throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); + } + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt& stmt) : stmt(stmt) { + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() { sqlite3_reset(stmt); } + +SQLiteStmt::Use& SQLiteStmt::Use::operator()(const std::string& value, + bool notNull) { + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, + SQLITE_TRANSIENT) != SQLITE_OK) { + throwSQLiteError(stmt.db, "binding argument"); + } + } else { + bind(); + } + return *this; +} + +SQLiteStmt::Use& SQLiteStmt::Use::operator()(int64_t value, bool notNull) { + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) { + throwSQLiteError(stmt.db, "binding argument"); + } + } else { + bind(); + } + return *this; +} + +SQLiteStmt::Use& SQLiteStmt::Use::bind() { + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) { + throwSQLiteError(stmt.db, "binding argument"); + } + return *this; +} + +int SQLiteStmt::Use::step() { return sqlite3_step(stmt); } + +void SQLiteStmt::Use::exec() { + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) { + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); + } +} + +bool SQLiteStmt::Use::next() { + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) { + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); + } + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) { + auto s = reinterpret_cast<const char*>(sqlite3_column_text(stmt, col)); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) { + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +bool SQLiteStmt::Use::isNull(int col) { + return sqlite3_column_type(stmt, col) == SQLITE_NULL; +} + +SQLiteTxn::SQLiteTxn(sqlite3* db) { + this->db = db; + if (sqlite3_exec(db, "begin;", nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "starting transaction"); + } + active = true; +} + +void SQLiteTxn::commit() { + if (sqlite3_exec(db, "commit;", nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "committing transaction"); + } + active = false; +} + +SQLiteTxn::~SQLiteTxn() { + try { + if (active && + sqlite3_exec(db, "rollback;", nullptr, nullptr, nullptr) != SQLITE_OK) { + throwSQLiteError(db, "aborting transaction"); + } + } catch (...) { + ignoreException(); + } +} + +void handleSQLiteBusy(const SQLiteBusy& e) { + static std::atomic<time_t> lastWarned{0}; + + time_t now = time(nullptr); + + if (now > lastWarned + 10) { + lastWarned = now; + LOG(ERROR) << e.what(); + } + + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, nullptr); +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/sqlite.hh b/third_party/nix/src/libstore/sqlite.hh new file mode 100644 index 000000000000..cad78aed45be --- /dev/null +++ b/third_party/nix/src/libstore/sqlite.hh @@ -0,0 +1,109 @@ +#pragma once + +#include <functional> +#include <string> + +#include "libutil/types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite { + sqlite3* db = 0; + SQLite() {} + SQLite(const Path& path); + SQLite(const SQLite& from) = delete; + SQLite& operator=(const SQLite& from) = delete; + SQLite& operator=(SQLite&& from) { + db = from.db; + from.db = 0; + return *this; + } + ~SQLite(); + operator sqlite3*() { return db; } + + void exec(const std::string& stmt); +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt { + sqlite3* db = 0; + sqlite3_stmt* stmt = 0; + std::string sql; + SQLiteStmt() {} + SQLiteStmt(sqlite3* db, const std::string& sql) { create(db, sql); } + void create(sqlite3* db, const std::string& s); + ~SQLiteStmt(); + operator sqlite3_stmt*() { return stmt; } + + /* Helper for binding / executing statements. */ + class Use { + friend struct SQLiteStmt; + + private: + SQLiteStmt& stmt; + int curArg = 1; + Use(SQLiteStmt& stmt); + + public: + ~Use(); + + /* Bind the next parameter. */ + Use& operator()(const std::string& value, bool notNull = true); + Use& operator()(int64_t value, bool notNull = true); + Use& bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + bool isNull(int col); + }; + + Use use() { return Use(*this); } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn { + bool active = false; + sqlite3* db; + + SQLiteTxn(sqlite3* db); + + void commit(); + + ~SQLiteTxn(); +}; + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3* db, const FormatOrString& fs); + +void handleSQLiteBusy(const SQLiteBusy& e); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template <typename T> +T retrySQLite(std::function<T()> fun) { + while (true) { + try { + return fun(); + } catch (SQLiteBusy& e) { + handleSQLiteBusy(e); + } + } +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/ssh-store.cc b/third_party/nix/src/libstore/ssh-store.cc new file mode 100644 index 000000000000..96adb3660d42 --- /dev/null +++ b/third_party/nix/src/libstore/ssh-store.cc @@ -0,0 +1,89 @@ +#include <absl/strings/str_cat.h> + +#include "libstore/remote-fs-accessor.hh" +#include "libstore/remote-store.hh" +#include "libstore/ssh.hh" +#include "libstore/store-api.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/archive.hh" +#include "libutil/pool.hh" + +namespace nix { + +constexpr std::string_view kUriScheme = "ssh-ng://"; + +class SSHStore : public RemoteStore { + public: + const Setting<Path> sshKey{(Store*)this, "", "ssh-key", + "path to an SSH private key"}; + const Setting<bool> compress{(Store*)this, false, "compress", + "whether to compress the connection"}; + + SSHStore(const std::string& host, const Params& params) + : Store(params), + RemoteStore(params), + host(host), + master(host, sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, compress) {} + + std::string getUri() override { return absl::StrCat(kUriScheme, host); } + + bool sameMachine() override { return false; } + + void narFromPath(const Path& path, Sink& sink) override; + + ref<FSAccessor> getFSAccessor() override; + + private: + struct Connection : RemoteStore::Connection { + std::unique_ptr<SSHMaster::Connection> sshConn; + }; + + ref<RemoteStore::Connection> openConnection() override; + + std::string host; + + SSHMaster master; + + void setOptions(RemoteStore::Connection& conn) override{ + /* TODO Add a way to explicitly ask for some options to be + forwarded. One option: A way to query the daemon for its + settings, and then a series of params to SSHStore like + forward-cores or forward-overridden-cores that only + override the requested settings. + */ + }; +}; + +void SSHStore::narFromPath(const Path& path, Sink& sink) { + auto conn(connections->get()); + conn->to << wopNarFromPath << path; + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref<FSAccessor> SSHStore::getFSAccessor() { + return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this())); +} + +ref<RemoteStore::Connection> SSHStore::openConnection() { + auto conn = make_ref<Connection>(); + conn->sshConn = master.startCommand("nix-daemon --pipe"); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + initConnection(*conn); + return conn; +} + +static RegisterStoreImplementation regStore( + [](const std::string& uri, + const Store::Params& params) -> std::shared_ptr<Store> { + if (std::string(uri, 0, kUriScheme.size()) != kUriScheme) { + return nullptr; + } + return std::make_shared<SSHStore>(std::string(uri, kUriScheme.size()), + params); + }); + +} // namespace nix diff --git a/third_party/nix/src/libstore/ssh.cc b/third_party/nix/src/libstore/ssh.cc new file mode 100644 index 000000000000..6043e584ddbe --- /dev/null +++ b/third_party/nix/src/libstore/ssh.cc @@ -0,0 +1,160 @@ +#include "libstore/ssh.hh" + +#include <utility> + +#include <absl/strings/match.h> +#include <absl/strings/str_split.h> + +namespace nix { + +SSHMaster::SSHMaster(const std::string& host, std::string keyFile, + bool useMaster, bool compress, int logFD) + : host(host), + fakeSSH(host == "localhost"), + keyFile(std::move(keyFile)), + useMaster(useMaster && !fakeSSH), + compress(compress), + logFD(logFD) { + if (host.empty() || absl::StartsWith(host, "-")) { + throw Error("invalid SSH host name '%s'", host); + } +} + +void SSHMaster::addCommonSSHOpts(Strings& args) { + for (auto& i : + absl::StrSplit(getEnv("NIX_SSHOPTS").value_or(""), + absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty())) { + args.push_back(std::string(i)); + } + if (!keyFile.empty()) { + args.insert(args.end(), {"-i", keyFile}); + } + if (compress) { + args.push_back("-C"); + } +} + +std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand( + const std::string& command) { + Path socketPath = startMaster(); + + Pipe in; + Pipe out; + in.create(); + out.create(); + + auto conn = std::make_unique<Connection>(); + ProcessOptions options; + options.dieWithParent = false; + + conn->sshPid = startProcess( + [&]() { + restoreSignals(); + + close(in.writeSide.get()); + close(out.readSide.get()); + + if (dup2(in.readSide.get(), STDIN_FILENO) == -1) { + throw SysError("duping over stdin"); + } + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) { + throw SysError("duping over stdout"); + } + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) { + throw SysError("duping over stderr"); + } + + Strings args; + + if (fakeSSH) { + args = {"bash", "-c"}; + } else { + args = {"ssh", host, "-x", "-a"}; + addCommonSSHOpts(args); + if (!socketPath.empty()) { + args.insert(args.end(), {"-S", socketPath}); + } + // TODO(tazjin): Abseil verbosity flag + /*if (verbosity >= lvlChatty) { + args.push_back("-v"); + }*/ + } + + args.push_back(command); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + + // could not exec ssh/bash + throw SysError("unable to execute '%s'", args.front()); + }, + options); + + in.readSide = AutoCloseFD(-1); + out.writeSide = AutoCloseFD(-1); + + conn->out = std::move(out.readSide); + conn->in = std::move(in.writeSide); + + return conn; +} + +Path SSHMaster::startMaster() { + if (!useMaster) { + return ""; + } + + auto state(state_.lock()); + + if (state->sshMaster != Pid(-1)) { + return state->socketPath; + } + + state->tmpDir = + std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700)); + + state->socketPath = Path(*state->tmpDir) + "/ssh.sock"; + + Pipe out; + out.create(); + + ProcessOptions options; + options.dieWithParent = false; + + state->sshMaster = startProcess( + [&]() { + restoreSignals(); + + close(out.readSide.get()); + + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) { + throw SysError("duping over stdout"); + } + + Strings args = {"ssh", host, + "-M", "-N", + "-S", state->socketPath, + "-o", "LocalCommand=echo started", + "-o", "PermitLocalCommand=yes"}; + // if (verbosity >= lvlChatty) { args.push_back("-v"); } + addCommonSSHOpts(args); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", args.front()); + }, + options); + + out.writeSide = AutoCloseFD(-1); + + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile& e) { + } + + if (reply != "started") { + throw Error("failed to start SSH master connection to '%s'", host); + } + + return state->socketPath; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/ssh.hh b/third_party/nix/src/libstore/ssh.hh new file mode 100644 index 000000000000..9844f89d3599 --- /dev/null +++ b/third_party/nix/src/libstore/ssh.hh @@ -0,0 +1,41 @@ +#pragma once + +#include "libutil/sync.hh" +#include "libutil/util.hh" + +namespace nix { + +class SSHMaster { + private: + const std::string host; + bool fakeSSH; + const std::string keyFile; + const bool useMaster; + const bool compress; + const int logFD; + + struct State { + Pid sshMaster; + std::unique_ptr<AutoDelete> tmpDir; + Path socketPath; + }; + + Sync<State> state_; + + void addCommonSSHOpts(Strings& args); + + public: + SSHMaster(const std::string& host, std::string keyFile, bool useMaster, + bool compress, int logFD = -1); + + struct Connection { + Pid sshPid; + AutoCloseFD out, in; + }; + + std::unique_ptr<Connection> startCommand(const std::string& command); + + Path startMaster(); +}; + +} // namespace nix diff --git a/third_party/nix/src/libstore/store-api.cc b/third_party/nix/src/libstore/store-api.cc new file mode 100644 index 000000000000..d8dbea18e953 --- /dev/null +++ b/third_party/nix/src/libstore/store-api.cc @@ -0,0 +1,1167 @@ +#include "libstore/store-api.hh" + +#include <future> +#include <ostream> +#include <streambuf> +#include <utility> + +#include <absl/status/status.h> +#include <absl/strings/match.h> +#include <absl/strings/numbers.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <grpcpp/create_channel.h> + +#include "libproto/worker.pb.h" +#include "libstore/crypto.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/nar-info-disk-cache.hh" +#include "libstore/rpc-store.hh" +#include "libutil/json.hh" +#include "libutil/thread-pool.hh" +#include "libutil/util.hh" + +namespace nix { + +namespace { +class NullStream : public std::streambuf { + public: + int overflow(int c) override { return c; } +}; + +static NullStream NULL_STREAM{}; +} // namespace + +std::ostream DiscardLogsSink() { return std::ostream(&NULL_STREAM); } + +std::optional<BuildMode> BuildModeFrom(nix::proto::BuildMode mode) { + switch (mode) { + case nix::proto::BuildMode::Normal: + return BuildMode::bmNormal; + case nix::proto::BuildMode::Repair: + return BuildMode::bmRepair; + case nix::proto::BuildMode::Check: + return BuildMode::bmCheck; + default: + return {}; + } +} + +nix::proto::BuildMode BuildModeToProto(BuildMode mode) { + switch (mode) { + case BuildMode::bmNormal: + return nix::proto::BuildMode::Normal; + case BuildMode::bmRepair: + return nix::proto::BuildMode::Repair; + case BuildMode::bmCheck: + return nix::proto::BuildMode::Check; + } +} + +nix::proto::BuildStatus BuildResult::status_to_proto() { + switch (status) { + case BuildResult::Status::Built: + return proto::BuildStatus::Built; + case BuildResult::Status::Substituted: + return proto::BuildStatus::Substituted; + case BuildResult::Status::AlreadyValid: + return proto::BuildStatus::AlreadyValid; + case BuildResult::Status::PermanentFailure: + return proto::BuildStatus::PermanentFailure; + case BuildResult::Status::InputRejected: + return proto::BuildStatus::InputRejected; + case BuildResult::Status::OutputRejected: + return proto::BuildStatus::OutputRejected; + case BuildResult::Status::TransientFailure: + return proto::BuildStatus::TransientFailure; + case BuildResult::Status::CachedFailure: + return proto::BuildStatus::CachedFailure; + case BuildResult::Status::TimedOut: + return proto::BuildStatus::TimedOut; + case BuildResult::Status::MiscFailure: + return proto::BuildStatus::MiscFailure; + case BuildResult::Status::DependencyFailed: + return proto::BuildStatus::DependencyFailed; + case BuildResult::Status::LogLimitExceeded: + return proto::BuildStatus::LogLimitExceeded; + case BuildResult::Status::NotDeterministic: + return proto::BuildStatus::NotDeterministic; + } +} + +std::optional<BuildResult> BuildResult::FromProto( + const nix::proto::BuildResult& resp) { + BuildResult result; + switch (resp.status()) { + case proto::BuildStatus::Built: + result.status = BuildResult::Status::Built; + break; + case proto::BuildStatus::Substituted: + result.status = BuildResult::Status::Substituted; + break; + case proto::BuildStatus::AlreadyValid: + result.status = BuildResult::Status::AlreadyValid; + break; + case proto::BuildStatus::PermanentFailure: + result.status = BuildResult::Status::PermanentFailure; + break; + case proto::BuildStatus::InputRejected: + result.status = BuildResult::Status::InputRejected; + break; + case proto::BuildStatus::OutputRejected: + result.status = BuildResult::Status::OutputRejected; + break; + case proto::BuildStatus::TransientFailure: + result.status = BuildResult::Status::TransientFailure; + break; + case proto::BuildStatus::CachedFailure: + result.status = BuildResult::Status::CachedFailure; + break; + case proto::BuildStatus::TimedOut: + result.status = BuildResult::Status::TimedOut; + break; + case proto::BuildStatus::MiscFailure: + result.status = BuildResult::Status::MiscFailure; + break; + case proto::BuildStatus::DependencyFailed: + result.status = BuildResult::Status::DependencyFailed; + break; + case proto::BuildStatus::LogLimitExceeded: + result.status = BuildResult::Status::LogLimitExceeded; + break; + case proto::BuildStatus::NotDeterministic: + result.status = BuildResult::Status::NotDeterministic; + break; + default: + return {}; + } + + result.errorMsg = resp.msg(); + return result; +} + +std::optional<GCOptions::GCAction> GCActionFromProto( + nix::proto::GCAction gc_action) { + switch (gc_action) { + case nix::proto::GCAction::ReturnLive: + return GCOptions::GCAction::gcReturnLive; + case nix::proto::GCAction::ReturnDead: + return GCOptions::GCAction::gcReturnDead; + case nix::proto::GCAction::DeleteDead: + return GCOptions::GCAction::gcDeleteDead; + case nix::proto::GCAction::DeleteSpecific: + return GCOptions::GCAction::gcDeleteSpecific; + default: + return {}; + } +} + +[[nodiscard]] const proto::GCAction GCOptions::ActionToProto() const { + switch (action) { + case GCOptions::GCAction::gcReturnLive: + return nix::proto::GCAction::ReturnLive; + case GCOptions::GCAction::gcReturnDead: + return nix::proto::GCAction::ReturnDead; + case GCOptions::GCAction::gcDeleteDead: + return nix::proto::GCAction::DeleteDead; + case GCOptions::GCAction::gcDeleteSpecific: + return nix::proto::GCAction::DeleteSpecific; + } +} + +bool Store::isInStore(const Path& path) const { + return isInDir(path, storeDir); +} + +bool Store::isStorePath(const Path& path) const { + return isInStore(path) && + path.size() >= storeDir.size() + 1 + storePathHashLen && + path.find('/', storeDir.size() + 1) == Path::npos; +} + +void Store::assertStorePath(const Path& path) const { + if (!isStorePath(path)) { + throw Error(format("path '%1%' is not in the Nix store") % path); + } +} + +Path Store::toStorePath(const Path& path) const { + if (!isInStore(path)) { + throw Error(format("path '%1%' is not in the Nix store") % path); + } + Path::size_type slash = path.find('/', storeDir.size() + 1); + if (slash == Path::npos) { + return path; + } + return Path(path, 0, slash); +} + +Path Store::followLinksToStore(const Path& _path) const { + Path path = absPath(_path); + while (!isInStore(path)) { + if (!isLink(path)) { + break; + } + std::string target = readLink(path); + path = absPath(target, dirOf(path)); + } + if (!isInStore(path)) { + throw Error(format("path '%1%' is not in the Nix store") % path); + } + return path; +} + +Path Store::followLinksToStorePath(const Path& path) const { + return toStorePath(followLinksToStore(path)); +} + +std::string storePathToName(const Path& path) { + auto base = baseNameOf(path); + + // The base name of the store path must be `storePathHashLen` characters long, + // if it is not `storePathHashLen` long then the next character, following + // the hash part, MUST be a dash (`-`). + const bool hasLengthMismatch = base.size() != storePathHashLen; + const bool hasInvalidSuffix = + base.size() > storePathHashLen && base[storePathHashLen] != '-'; + if (hasLengthMismatch && hasInvalidSuffix) { + throw Error(format("path '%1%' is not a valid store path") % path); + } + + return base.size() == storePathHashLen + ? "" + : std::string(base, storePathHashLen + 1); +} + +std::string storePathToHash(const Path& path) { + auto base = baseNameOf(path); + assert(base.size() >= storePathHashLen); + return std::string(base, 0, storePathHashLen); +} + +void checkStoreName(const std::string& name) { + std::string validChars = "+-._?="; + + auto baseError = + format( + "The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % + validChars % name; + + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (std::string(name, 0, 1) == ".") { + throw Error(baseError % "it is illegal to start the name with a period"); + } + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) { + throw Error(baseError % "name must be less than 212 characters"); + } + for (auto& i : name) { + if (!((i >= 'A' && i <= 'Z') || (i >= 'a' && i <= 'z') || + (i >= '0' && i <= '9') || validChars.find(i) != std::string::npos)) { + throw Error(baseError % (format("the '%1%' character is invalid") % i)); + } + } +} + +/* Store paths have the following form: + + <store>/<h>-<name> + + where + + <store> = the location of the Nix store, usually /nix/store + + <name> = a human readable name for the path, typically obtained + from the name attribute of the derivation, or the name of the + source file from which the store path is created. For derivation + outputs other than the default "out" output, the string "-<id>" + is suffixed to <name>. + + <h> = base-32 representation of the first 160 bits of a SHA-256 + hash of <s>; the hash part of the store name + + <s> = the string "<type>:sha256:<h2>:<store>:<name>"; + note that it includes the location of the store as well as the + name to make sure that changes to either of those are reflected + in the hash (e.g. you won't get /nix/store/<h>-name1 and + /nix/store/<h>-name2 with equal hash parts). + + <type> = one of: + "text:<r1>:<r2>:...<rN>" + for plain text files written to the store using + addTextToStore(); <r1> ... <rN> are the references of the + path. + "source" + for paths copied to the store using addToStore() when recursive + = true and hashAlgo = "sha256" + "output:<id>" + for either the outputs created by derivations, OR paths copied + to the store using addToStore() with recursive != true or + hashAlgo != "sha256" (in that case "source" is used; it's + silly, but it's done that way for compatibility). <id> is the + name of the output (usually, "out"). + + <h2> = base-16 representation of a SHA-256 hash of: + if <type> = "text:...": + the string written to the resulting store path + if <type> = "source": + the serialisation of the path from which this store path is + copied, as returned by hashPath() + if <type> = "output:<id>": + for non-fixed derivation outputs: + the derivation (see hashDerivationModulo() in + primops.cc) + for paths copied by addToStore() or produced by fixed-output + derivations: + the string "fixed:out:<rec><algo>:<hash>:", where + <rec> = "r:" for recursive (path) hashes, or "" for flat + (file) hashes + <algo> = "md5", "sha1" or "sha256" + <hash> = base-16 representation of the path or flat hash of + the contents of the path (or expected contents of the + path for fixed-output derivations) + + It would have been nicer to handle fixed-output derivations under + "source", e.g. have something like "source:<rec><algo>", but we're + stuck with this for now... + + The main reason for this way of computing names is to prevent name + collisions (for security). For instance, it shouldn't be feasible + to come up with a derivation whose output path collides with the + path for a copied source. The former would have a <s> starting with + "output:out:", while the latter would have a <s> starting with + "source:". +*/ + +Path Store::makeStorePath(const std::string& type, const Hash& hash, + const std::string& name) const { + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + std::string s = + type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; + + checkStoreName(name); + + return absl::StrCat(storeDir, "/", hashString(htSHA256, s).ToStorePathHash(), + "-", name); +} + +Path Store::makeOutputPath(const std::string& id, const Hash& hash, + const std::string& name) const { + return makeStorePath("output:" + id, hash, + name + (id == "out" ? "" : "-" + id)); +} + +Path Store::makeFixedOutputPath(bool recursive, const Hash& hash, + const std::string& name) const { + return hash.type == htSHA256 && recursive + ? makeStorePath("source", hash, name) + : makeStorePath( + "output:out", + hashString( + htSHA256, + absl::StrCat("fixed:out:", (recursive ? "r:" : ""), + hash.to_string(Base16), ":")), + name); +} + +Path Store::makeTextPath(const std::string& name, const Hash& hash, + const PathSet& references) const { + assert(hash.type == htSHA256); + /* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ + std::string type = "text"; + for (auto& i : references) { + type += ":"; + type += i; + } + return makeStorePath(type, hash, name); +} + +std::pair<Path, Hash> Store::computeStorePathForPath(const std::string& name, + const Path& srcPath, + bool recursive, + HashType hashAlgo, + PathFilter& filter) const { + Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first + : hashFile(hashAlgo, srcPath); + Path dstPath = makeFixedOutputPath(recursive, h, name); + return std::pair<Path, Hash>(dstPath, h); +} + +Path Store::computeStorePathForText(const std::string& name, + const std::string& s, + const PathSet& references) const { + return makeTextPath(name, hashString(htSHA256, s), references); +} + +Store::Store(const Params& params) + : Config(params), + state(Sync<State>{ + State{LRUCache<std::string, std::shared_ptr<ValidPathInfo>>( + (size_t)pathInfoCacheSize)}}) {} + +std::string Store::getUri() { return ""; } + +bool Store::isValidPath(const Path& storePath) { + assertStorePath(storePath); + + auto hashPart = storePathToHash(storePath); + + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + return *res != nullptr; + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert( + hashPart, + res.first == NarInfoDiskCache::oInvalid ? nullptr : res.second); + return res.first == NarInfoDiskCache::oValid; + } + } + + bool valid = isValidPathUncached(storePath); + + if (diskCache && !valid) { + // FIXME: handle valid = true case. + diskCache->upsertNarInfo(getUri(), hashPart, nullptr); + } + + return valid; +} + +/* Default implementation for stores that only implement + queryPathInfoUncached(). */ +bool Store::isValidPathUncached(const Path& path) { + try { + queryPathInfo(path); + return true; + } catch (InvalidPath&) { + return false; + } +} + +ref<const ValidPathInfo> Store::queryPathInfo(const Path& storePath) { + std::promise<ref<ValidPathInfo>> promise; + + queryPathInfo( + storePath, + Callback<ref<ValidPathInfo>>([&](std::future<ref<ValidPathInfo>> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + })); + + return promise.get_future().get(); +} + +void Store::queryPathInfo(const Path& storePath, + Callback<ref<ValidPathInfo>> callback) noexcept { + std::string hashPart; + + try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); + + { + auto res = state.lock()->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + if (!*res) { + throw InvalidPath(format("path '%s' is not valid") % storePath); + } + return callback(ref<ValidPathInfo>(*res)); + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + { + auto state_(state.lock()); + state_->pathInfoCache.upsert( + hashPart, + res.first == NarInfoDiskCache::oInvalid ? nullptr : res.second); + if (res.first == NarInfoDiskCache::oInvalid || + (res.second->path != storePath && + !storePathToName(storePath).empty())) { + throw InvalidPath(format("path '%s' is not valid") % storePath); + } + } + return callback(ref<ValidPathInfo>(res.second)); + } + } + + } catch (...) { + return callback.rethrow(); + } + + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + + queryPathInfoUncached( + storePath, + Callback<std::shared_ptr<ValidPathInfo>>{ + [this, storePath, hashPart, + callbackPtr](std::future<std::shared_ptr<ValidPathInfo>> fut) { + try { + auto info = fut.get(); + + if (diskCache) { + diskCache->upsertNarInfo(getUri(), hashPart, info); + } + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } + + if (!info || (info->path != storePath && + !storePathToName(storePath).empty())) { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } + + (*callbackPtr)(ref<ValidPathInfo>(info)); + } catch (...) { + callbackPtr->rethrow(); + } + }}); +} + +PathSet Store::queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute) { + struct State { + size_t left; + PathSet valid; + std::exception_ptr exc; + }; + + Sync<State> state_(State{paths.size(), PathSet()}); + + std::condition_variable wakeup; + ThreadPool pool; + + auto doQuery = [&](const Path& path) { + checkInterrupt(); + queryPathInfo(path, Callback<ref<ValidPathInfo>>( + [path, &state_, + &wakeup](std::future<ref<ValidPathInfo>> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); + state->valid.insert(path); + } catch (InvalidPath&) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (--state->left == 0u) { + wakeup.notify_one(); + } + })); + }; + + for (auto& path : paths) { + pool.enqueue(std::bind(doQuery, path)); + } + + pool.process(); + + while (true) { + auto state(state_.lock()); + if (state->left == 0u) { + if (state->exc) { + std::rethrow_exception(state->exc); + } + return state->valid; + } + state.wait(wakeup); + } +} + +/* Return a string accepted by decodeValidPathInfo() that + registers the specified paths as valid. Note: it's the + responsibility of the caller to provide a closure. */ +std::string Store::makeValidityRegistration(const PathSet& paths, + bool showDerivers, bool showHash) { + std::string s; + + for (auto& i : paths) { + s += i + "\n"; + + auto info = queryPathInfo(i); + + if (showHash) { + s += info->narHash.to_string(Base16, false) + "\n"; + s += (format("%1%\n") % info->narSize).str(); + } + + Path deriver = showDerivers ? info->deriver : ""; + s += deriver + "\n"; + + s += (format("%1%\n") % info->references.size()).str(); + + for (auto& j : info->references) { + s += j + "\n"; + } + } + + return s; +} + +void Store::pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths, + bool includeImpureInfo, bool showClosureSize, + AllowInvalidFlag allowInvalid) { + auto jsonList = jsonOut.list(); + + for (auto storePath : storePaths) { + auto jsonPath = jsonList.object(); + jsonPath.attr("path", storePath); + + try { + auto info = queryPathInfo(storePath); + storePath = info->path; + + jsonPath.attr("narHash", info->narHash.to_string()) + .attr("narSize", info->narSize); + + { + auto jsonRefs = jsonPath.list("references"); + for (auto& ref : info->references) { + jsonRefs.elem(ref); + } + } + + if (!info->ca.empty()) { + jsonPath.attr("ca", info->ca); + } + + std::pair<uint64_t, uint64_t> closureSizes; + + if (showClosureSize) { + closureSizes = getClosureSize(storePath); + jsonPath.attr("closureSize", closureSizes.first); + } + + if (includeImpureInfo) { + if (!info->deriver.empty()) { + jsonPath.attr("deriver", info->deriver); + } + + if (info->registrationTime != 0) { + jsonPath.attr("registrationTime", info->registrationTime); + } + + if (info->ultimate) { + jsonPath.attr("ultimate", info->ultimate); + } + + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto& sig : info->sigs) { + jsonSigs.elem(sig); + } + } + + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + + if (narInfo) { + if (!narInfo->url.empty()) { + jsonPath.attr("url", narInfo->url); + } + if (narInfo->fileHash) { + jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + } + if (narInfo->fileSize != 0u) { + jsonPath.attr("downloadSize", narInfo->fileSize); + } + if (showClosureSize) { + jsonPath.attr("closureDownloadSize", closureSizes.second); + } + } + } + + } catch (InvalidPath&) { + jsonPath.attr("valid", false); + } + } +} + +std::pair<uint64_t, uint64_t> Store::getClosureSize(const Path& storePath) { + uint64_t totalNarSize = 0; + uint64_t totalDownloadSize = 0; + PathSet closure; + computeFSClosure(storePath, closure, false, false); + for (auto& p : closure) { + auto info = queryPathInfo(p); + totalNarSize += info->narSize; + auto narInfo = std::dynamic_pointer_cast<const NarInfo>( + std::shared_ptr<const ValidPathInfo>(info)); + if (narInfo) { + totalDownloadSize += narInfo->fileSize; + } + } + return {totalNarSize, totalDownloadSize}; +} + +const Store::Stats& Store::getStats() { + { + auto state_(state.lock()); + stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } + return stats; +} + +absl::Status Store::buildPaths(std::ostream& /* log_sink */, + const PathSet& paths, BuildMode) { + for (auto& path : paths) { + if (isDerivation(path)) { + return absl::Status(absl::StatusCode::kUnimplemented, + "buildPaths is unsupported"); + } + } + + if (queryValidPaths(paths).size() != paths.size()) { + return absl::Status(absl::StatusCode::kUnimplemented, + "buildPaths is unsupported"); + } + + return absl::OkStatus(); +} + +void copyStorePath(ref<Store> srcStore, const ref<Store>& dstStore, + const Path& storePath, RepairFlag repair, + CheckSigsFlag checkSigs) { + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + if (srcUri == "local" || srcUri == "daemon") { + LOG(INFO) << "copying path '" << storePath << "' to '" << dstUri << "'"; + } else { + if (dstUri == "local" || dstUri == "daemon") { + LOG(INFO) << "copying path '" << storePath << "' from '" << srcUri << "'"; + } else { + LOG(INFO) << "copying path '" << storePath << "' from '" << srcUri + << "' to '" << dstUri << "'"; + } + } + + auto info = srcStore->queryPathInfo(storePath); + + uint64_t total = 0; + + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref<ValidPathInfo>(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (info->narSize == 0u) { + info2->narSize = sink.s->size(); + } + if (info->ultimate) { + info2->ultimate = false; + } + info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref<ValidPathInfo>(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource( + [&](Sink& sink) { + LambdaSink wrapperSink([&](const unsigned char* data, size_t len) { + sink(data, len); + total += len; + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, + [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", + storePath, srcStore->getUri()); + }); + + dstStore->addToStore(*info, *source, repair, checkSigs); +} + +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const PathSet& storePaths, RepairFlag repair, + CheckSigsFlag checkSigs, SubstituteFlag substitute) { + PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + + PathSet missing; + for (auto& path : storePaths) { + if (valid.count(path) == 0u) { + missing.insert(path); + } + } + + if (missing.empty()) { + return; + } + + LOG(INFO) << "copying " << missing.size() << " paths"; + + std::atomic<size_t> nrDone{0}; + std::atomic<size_t> nrFailed{0}; + std::atomic<uint64_t> bytesExpected{0}; + std::atomic<uint64_t> nrRunning{0}; + + ThreadPool pool; + + processGraph<Path>( + pool, PathSet(missing.begin(), missing.end()), + + [&](const Path& storePath) { + if (dstStore->isValidPath(storePath)) { + nrDone++; + return PathSet(); + } + + auto info = srcStore->queryPathInfo(storePath); + + bytesExpected += info->narSize; + + return info->references; + }, + + [&](const Path& storePath) { + checkInterrupt(); + + if (!dstStore->isValidPath(storePath)) { + MaintainCount<decltype(nrRunning)> mc(nrRunning); + try { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } catch (Error& e) { + nrFailed++; + if (!settings.keepGoing) { + throw e; + } + LOG(ERROR) << "could not copy " << storePath << ": " << e.what(); + return; + } + } + + nrDone++; + }); +} + +void copyClosure(const ref<Store>& srcStore, const ref<Store>& dstStore, + const PathSet& storePaths, RepairFlag repair, + CheckSigsFlag checkSigs, SubstituteFlag substitute) { + PathSet closure; + srcStore->computeFSClosure({storePaths}, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + +ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven) { + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { + info.path = ""; + return info; + } + if (hashGiven) { + std::string s; + getline(str, s); + auto hash_ = Hash::deserialize(s, htSHA256); + info.narHash = Hash::unwrap_throw(hash_); + getline(str, s); + if (!absl::SimpleAtoi(s, &info.narSize)) { + throw Error("number expected"); + } + } + getline(str, info.deriver); + std::string s; + int n; + getline(str, s); + if (!absl::SimpleAtoi(s, &n)) { + throw Error("number expected"); + } + while ((n--) != 0) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) { + throw Error("missing input"); + } + return info; +} + +std::string showPaths(const PathSet& paths) { + std::string s; + for (auto& i : paths) { + if (!s.empty()) { + s += ", "; + } + s += "'" + i + "'"; + } + return s; +} + +std::string ValidPathInfo::fingerprint() const { + if (narSize == 0 || !narHash) { + throw Error(format("cannot calculate fingerprint of path '%s' because its " + "size/hash is not known") % + path); + } + return "1;" + path + ";" + narHash.to_string(Base32) + ";" + + std::to_string(narSize) + ";" + concatStringsSep(",", references); +} + +void ValidPathInfo::sign(const SecretKey& secretKey) { + sigs.insert(secretKey.signDetached(fingerprint())); +} + +bool ValidPathInfo::isContentAddressed(const Store& store) const { + auto warn = [&]() { + LOG(ERROR) << "warning: path '" << path + << "' claims to be content-addressed but isn't"; + }; + + if (absl::StartsWith(ca, "text:")) { + auto hash_ = Hash::deserialize(std::string_view(ca).substr(5)); + Hash hash = Hash::unwrap_throw(hash_); + if (store.makeTextPath(storePathToName(path), hash, references) == path) { + return true; + } + warn(); + + } + + else if (absl::StartsWith(ca, "fixed:")) { + bool recursive = ca.compare(6, 2, "r:") == 0; + auto hash_ = + Hash::deserialize(std::string_view(ca).substr(recursive ? 8 : 6)); + Hash hash = Hash::unwrap_throw(hash_); + if (references.empty() && + store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == + path) { + return true; + } + warn(); + } + + return false; +} + +size_t ValidPathInfo::checkSignatures(const Store& store, + const PublicKeys& publicKeys) const { + if (isContentAddressed(store)) { + return maxSigs; + } + + size_t good = 0; + for (auto& sig : sigs) { + if (checkSignature(publicKeys, sig)) { + good++; + } + } + return good; +} + +bool ValidPathInfo::checkSignature(const PublicKeys& publicKeys, + const std::string& sig) const { + return verifyDetached(fingerprint(), sig, publicKeys); +} + +Strings ValidPathInfo::shortRefs() const { + Strings refs; + for (auto& r : references) { + refs.push_back(baseNameOf(r)); + } + return refs; +} + +std::string makeFixedOutputCA(bool recursive, const Hash& hash) { + return "fixed:" + (recursive ? std::string("r:") : "") + hash.to_string(); +} + +void Store::addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + addToStore(info, make_ref<std::string>(narSource.drain()), repair, checkSigs, + std::move(accessor)); +} + +void Store::addToStore(const ValidPathInfo& info, const ref<std::string>& nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr<FSAccessor> accessor) { + StringSource source(*nar); + addToStore(info, source, repair, checkSigs, std::move(accessor)); +} + +} // namespace nix + +#include "libstore/local-store.hh" +#include "libstore/remote-store.hh" + +namespace nix { + +RegisterStoreImplementation::Implementations* + RegisterStoreImplementation::implementations = nullptr; + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams( + const std::string& uri_) { + auto uri(uri_); + Store::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + Strings parts = + absl::StrSplit(uri.substr(q + 1), absl::ByChar('&'), absl::SkipEmpty()); + for (const auto& s : parts) { + auto e = s.find('='); + if (e != std::string::npos) { + auto value = s.substr(e + 1); + std::string decoded; + for (size_t i = 0; i < value.size();) { + if (value[i] == '%') { + if (i + 2 >= value.size()) { + throw Error("invalid URI parameter '%s'", value); + } + try { + decoded += std::stoul(std::string(value, i + 1, 2), nullptr, 16); + i += 3; + } catch (...) { + throw Error("invalid URI parameter '%s'", value); + } + } else { + decoded += value[i++]; + } + } + params[s.substr(0, e)] = decoded; + } + } + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +ref<Store> openStore(const std::string& uri_, + const Store::Params& extraParams) { + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); + + for (const auto& fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri, params); + if (store) { + store->warnUnknownSettings(); + return ref<Store>(store); + } + } + + throw Error("don't know how to open Nix store '%s'", uri); +} + +StoreType getStoreType(const std::string& uri, const std::string& stateDir) { + if (uri == "daemon") { + return tDaemon; + } + if (uri == "local" || absl::StartsWith(uri, "/")) { + return tLocal; + } else if (uri.empty() || uri == "auto") { + if (access(stateDir.c_str(), R_OK | W_OK) == 0) { + return tLocal; + } + if (pathExists(settings.nixDaemonSocketFile)) { + return tDaemon; + } else { + return tLocal; + } + } else { + return tOther; + } +} + +static RegisterStoreImplementation regStore([](const std::string& uri, + const Store::Params& params) + -> std::shared_ptr<Store> { + switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + case tDaemon: { + auto daemon_socket_uri = + absl::StrCat("unix://", settings.nixDaemonSocketFile); + auto channel = grpc::CreateChannel(daemon_socket_uri, + grpc::InsecureChannelCredentials()); + return std::shared_ptr<Store>(std::make_shared<nix::store::RpcStore>( + daemon_socket_uri, params, proto::WorkerService::NewStub(channel))); + } + case tLocal: { + Store::Params params2 = params; + if (absl::StartsWith(uri, "/")) { + params2["root"] = uri; + } + return std::shared_ptr<Store>(std::make_shared<LocalStore>(params2)); + } + default: + return nullptr; + } +}); + +std::list<ref<Store>> getDefaultSubstituters() { + static auto stores([]() { + std::list<ref<Store>> stores; + + StringSet done; + + auto addStore = [&](const std::string& uri) { + if (done.count(uri) != 0u) { + return; + } + done.insert(uri); + try { + stores.push_back(openStore(uri)); + } catch (Error& e) { + LOG(WARNING) << e.what(); + } + }; + + for (const auto& uri : settings.substituters.get()) { + addStore(uri); + } + + for (const auto& uri : settings.extraSubstituters.get()) { + addStore(uri); + } + + stores.sort([](ref<Store>& a, ref<Store>& b) { + return a->getPriority() < b->getPriority(); + }); + + return stores; + }()); + + return stores; +} + +} // namespace nix diff --git a/third_party/nix/src/libstore/store-api.hh b/third_party/nix/src/libstore/store-api.hh new file mode 100644 index 000000000000..eb18511e60df --- /dev/null +++ b/third_party/nix/src/libstore/store-api.hh @@ -0,0 +1,816 @@ +#pragma once + +#include <atomic> +#include <limits> +#include <map> +#include <memory> +#include <string> +#include <unordered_map> +#include <unordered_set> + +#include "libproto/worker.pb.h" +#include "libstore/crypto.hh" +#include "libstore/globals.hh" +#include "libutil/config.hh" +#include "libutil/hash.hh" +#include "libutil/lru-cache.hh" +#include "libutil/serialise.hh" +#include "libutil/sync.hh" + +namespace nix { + +// Create a no-op stream buffer used to discard build output in cases +// where we don't have a build log sink to thread through. +// +// TODO(tazjin): Get rid of this and do *something* with those logs. +std::ostream DiscardLogsSink(); + +MakeError(SubstError, Error); +MakeError(BuildError, Error); /* denotes a permanent build failure */ +MakeError(InvalidPath, Error); +MakeError(Unsupported, Error); +MakeError(SubstituteGone, Error); +MakeError(SubstituterDisabled, Error); + +struct BasicDerivation; +struct Derivation; +class FSAccessor; +class NarInfoDiskCache; +class Store; +class JSONPlaceholder; + +enum RepairFlag : bool { NoRepair = false, Repair = true }; +enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; +enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; +enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; + +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Magic header of exportPath() output (obsolete). */ +const uint32_t exportMagic = 0x4558494e; + +using Roots = std::unordered_map<Path, std::unordered_set<std::string>>; + +struct GCOptions { + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + using GCAction = enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + }; + + GCAction action{gcDeleteDead}; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; + + /* For `gcDeleteSpecific', the paths to delete. */ + PathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + unsigned long long maxFreed{std::numeric_limits<unsigned long long>::max()}; + + [[nodiscard]] const proto::GCAction ActionToProto() const; +}; + +std::optional<GCOptions::GCAction> GCActionFromProto( + nix::proto::GCAction gc_action); + +struct GCResults { + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + unsigned long long bytesFreed = 0; +}; + +struct SubstitutablePathInfo { + Path deriver; + PathSet references; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +using SubstitutablePathInfos = std::map<Path, SubstitutablePathInfo>; + +struct ValidPathInfo { + Path path; + Path deriver; + Hash narHash; + PathSet references; + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + /* If non-empty, an assertion that the path is content-addressed, + i.e., that the store path is computed from a cryptographic hash + of the contents of the path, plus some other bits of data like + the "name" part of the path. Such a path doesn't need + signatures, since we don't have to trust anybody's claim that + the path is the output of a particular derivation. (In the + extensional store model, we have to trust that the *contents* + of an output path of a derivation were actually produced by + that derivation. In the intensional model, we have to trust + that a particular output path was produced by a derivation; the + path then implies the contents.) + + Ideally, the content-addressability assertion would just be a + Boolean, and the store path would be computed from + ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, + 1) we've accumulated several types of content-addressed paths + over the years; and 2) fixed-output derivations support + multiple hash algorithms and serialisation methods (flat file + vs NAR). Thus, ‘ca’ has one of the following forms: + + * ‘text:sha256:<sha256 hash of file contents>’: For paths + computed by makeTextPath() / addTextToStore(). + + * ‘fixed:<r?>:<ht>:<h>’: For paths computed by + makeFixedOutputPath() / addToStore(). + */ + std::string ca; + + bool operator==(const ValidPathInfo& i) const { + return path == i.path && narHash == i.narHash && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey& secretKey); + + /* Return true iff the path is verifiably content-addressed. */ + bool isContentAddressed(const Store& store) const; + + static const size_t maxSigs = std::numeric_limits<size_t>::max(); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys, or maxSigs if the path + is content-addressed. */ + size_t checkSignatures(const Store& store, + const PublicKeys& publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys& publicKeys, + const std::string& sig) const; + + Strings shortRefs() const; + + virtual ~ValidPathInfo() {} +}; + +using ValidPathInfos = std::list<ValidPathInfo>; + +enum BuildMode { bmNormal, bmRepair, bmCheck }; + +// Convert the proto version of a `nix::proto::BuildMode` to its corresponding +// nix `BuildMode` +std::optional<BuildMode> BuildModeFrom(nix::proto::BuildMode mode); + +// Convert a `nix::BuildMode` to its corresponding proto representation +nix::proto::BuildMode BuildModeToProto(BuildMode mode); + +struct BuildResult { + /* Note: don't remove status codes, and only add new status codes + at the end of the list, to prevent client/server + incompatibilities in the nix-store --serve protocol. */ + enum Status { + Built = 0, + Substituted, + AlreadyValid, + PermanentFailure, + InputRejected, + OutputRejected, + TransientFailure, // possibly transient + CachedFailure, // no longer used + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + } status = MiscFailure; + std::string errorMsg; + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + + bool success() { + return status == Built || status == Substituted || status == AlreadyValid; + } + + // Convert the status of this `BuildResult` to its corresponding + // `nix::proto::BuildStatus` + nix::proto::BuildStatus status_to_proto(); + + static std::optional<BuildResult> FromProto( + const nix::proto::BuildResult& resp); +}; + +class Store : public std::enable_shared_from_this<Store>, public Config { + public: + using Params = std::map<std::string, std::string>; + + const PathSetting storeDir_{this, false, settings.nixStore, "store", + "path to the Nix store"}; + const Path storeDir = storeDir_; + + const Setting<int> pathInfoCacheSize{ + this, 65536, "path-info-cache-size", + "size of the in-memory store path information cache"}; + + const Setting<bool> isTrusted{ + this, false, "trusted", + "whether paths from this store can be used as substitutes even when they " + "lack trusted signatures"}; + + protected: + struct State { + LRUCache<std::string, std::shared_ptr<ValidPathInfo>> pathInfoCache; + }; + + Sync<State> state; + + std::shared_ptr<NarInfoDiskCache> diskCache; + + Store(const Params& params); + + public: + virtual ~Store() {} + + virtual std::string getUri() = 0; + + /* Return true if ‘path’ is in the Nix store (but not the Nix + store itself). */ + bool isInStore(const Path& path) const; + + /* Return true if ‘path’ is a store path, i.e. a direct child of + the Nix store. */ + bool isStorePath(const Path& path) const; + + /* Throw an exception if ‘path’ is not a store path. */ + void assertStorePath(const Path& path) const; + + /* Chop off the parts after the top-level store name, e.g., + /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ + Path toStorePath(const Path& path) const; + + /* Follow symlinks until we end up with a path in the Nix store. */ + Path followLinksToStore(const Path& path) const; + + /* Same as followLinksToStore(), but apply toStorePath() to the + result. */ + Path followLinksToStorePath(const Path& path) const; + + /* Constructs a unique store path name. */ + Path makeStorePath(const std::string& type, const Hash& hash, + const std::string& name) const; + + Path makeOutputPath(const std::string& id, const Hash& hash, + const std::string& name) const; + + Path makeFixedOutputPath(bool recursive, const Hash& hash, + const std::string& name) const; + + Path makeTextPath(const std::string& name, const Hash& hash, + const PathSet& references) const; + + /* This is the preparatory part of addToStore(); it computes the + store path to which srcPath is to be copied. Returns the store + path and the cryptographic hash of the contents of srcPath. */ + std::pair<Path, Hash> computeStorePathForPath( + const std::string& name, const Path& srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter) const; + + /* Preparatory part of addTextToStore(). + + !!! Computation of the path should take the references given to + addTextToStore() into account, otherwise we have a (relatively + minor) security hole: a caller can register a source file with + bogus references. If there are too many references, the path may + not be garbage collected when it has to be (not really a problem, + the caller could create a root anyway), or it may be garbage + collected when it shouldn't be (more serious). + + Hashing the references would solve this (bogus references would + simply yield a different store path, so other users wouldn't be + affected), but it has some backwards compatibility issues (the + hashing scheme changes), so I'm not doing that for now. */ + Path computeStorePathForText(const std::string& name, const std::string& s, + const PathSet& references) const; + + /* Check whether a path is valid. */ + bool isValidPath(const Path& path); + + protected: + virtual bool isValidPathUncached(const Path& path); + + public: + /* Query which of the given paths is valid. Optionally, try to + substitute missing paths. */ + virtual PathSet queryValidPaths(const PathSet& paths, + SubstituteFlag maybeSubstitute); + + PathSet queryValidPaths(const PathSet& paths) { + return queryValidPaths(paths, NoSubstitute); + } + + /* Query the set of all valid paths. Note that for some store + backends, the name part of store paths may be omitted + (i.e. you'll get /nix/store/<hash> rather than + /nix/store/<hash>-<name>). Use queryPathInfo() to obtain the + full store path. */ + virtual PathSet queryAllValidPaths() { unsupported("queryAllValidPaths"); } + + /* Query information about a valid path. It is permitted to omit + the name part of the store path. */ + ref<const ValidPathInfo> queryPathInfo(const Path& path); + + /* Asynchronous version of queryPathInfo(). */ + void queryPathInfo(const Path& path, + Callback<ref<ValidPathInfo>> callback) noexcept; + + protected: + virtual void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept = 0; + + public: + /* Queries the set of incoming FS references for a store path. + The result is not cleared. */ + virtual void queryReferrers(const Path& path, PathSet& referrers) { + unsupported("queryReferrers"); + } + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + virtual PathSet queryValidDerivers(const Path& path) { return {}; }; + + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path& path) { + unsupported("queryDerivationOutputs"); + } + + /* Query the output names of the derivation denoted by `path'. */ + virtual StringSet queryDerivationOutputNames(const Path& path) { + unsupported("queryDerivationOutputNames"); + } + + /* Query the full store path given the hash part of a valid store + path, or "" if the path doesn't exist. */ + virtual Path queryPathFromHashPart(const std::string& hashPart) = 0; + + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet& paths) { return {}; }; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet& paths, + SubstitutablePathInfos& infos) { + return; + }; + + virtual bool wantMassQuery() { return false; } + + /* Import a path into the store. */ + virtual void addToStore(const ValidPathInfo& info, Source& narSource, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr<FSAccessor> accessor = 0); + + // FIXME: remove + virtual void addToStore(const ValidPathInfo& info, + const ref<std::string>& nar, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr<FSAccessor> accessor = 0); + + /* Copy the contents of a path to the store and register the + validity of the resulting path. The resulting path is returned. + The function object `filter' can be used to exclude files (see + libutil/archive.hh). If recursive is set to true, the path will be treated + as a directory (eg cp -r vs cp) */ + virtual Path addToStore(const std::string& name, const Path& srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter, + RepairFlag repair = NoRepair) = 0; + + /* Like addToStore, but the contents written to the output path is + a regular file containing the given string. */ + virtual Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, + RepairFlag repair = NoRepair) = 0; + + /* Write a NAR dump of a store path. */ + virtual void narFromPath(const Path& path, Sink& sink) = 0; + + /* For each path, if it's a derivation, build it. Building a + derivation means ensuring that the output paths are valid. If + they are already valid, this is a no-op. Otherwise, validity + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. For inputs that are + not derivations, substitute them. */ + [[nodiscard]] virtual absl::Status buildPaths(std::ostream& log_sink, + const PathSet& paths, + BuildMode build_mode); + + [[nodiscard]] absl::Status buildPaths(std::ostream& log_sink, + const PathSet& paths) { + return buildPaths(log_sink, paths, bmNormal); + } + + /* Build a single non-materialized derivation (i.e. not from an + on-disk .drv file). Note that ‘drvPath’ is only used for + informational purposes. */ + // TODO(tazjin): Thread std::ostream through here, too. + virtual BuildResult buildDerivation(std::ostream& log_sink, + const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode) = 0; + + BuildResult buildDerivation(std::ostream& log_sink, const Path& drvPath, + const BasicDerivation& drv) { + return buildDerivation(log_sink, drvPath, drv, bmNormal); + } + + /* Ensure that a path is valid. If it is not currently valid, it + may be made valid by running a substitute (if defined for the + path). */ + virtual void ensurePath(const Path& path) = 0; + + /* Add a store path as a temporary root of the garbage collector. + The root disappears as soon as we exit. */ + virtual void addTempRoot(const Path& path) { unsupported("addTempRoot"); } + + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path& path) { + unsupported("addIndirectRoot"); + } + + /* Acquire the global GC lock, then immediately release it. This + function must be called after registering a new permanent root, + but before exiting. Otherwise, it is possible that a running + garbage collector doesn't see the new root and deletes the + stuff we've just built. By acquiring the lock briefly, we + ensure that either: + + - The collector is already running, and so we block until the + collector is finished. The collector will know about our + *temporary* locks, which should include whatever it is we + want to register as a permanent lock. + + - The collector isn't running, or it's just started but hasn't + acquired the GC lock yet. In that case we get and release + the lock right away, then exit. The collector scans the + permanent root and sees our's. + + In either case the permanent root is seen by the collector. */ + virtual void syncWithGC(){}; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) { unsupported("findRoots"); } + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions& options, GCResults& results) { + unsupported("collectGarbage"); + } + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + std::string makeValidityRegistration(const PathSet& paths, bool showDerivers, + bool showHash); + + /* Write a JSON representation of store path metadata, such as the + hash and the references. If ‘includeImpureInfo’ is true, + variable elements such as the registration time are + included. If ‘showClosureSize’ is true, the closure size of + each path is included. */ + void pathInfoToJSON(JSONPlaceholder& jsonOut, const PathSet& storePaths, + bool includeImpureInfo, bool showClosureSize, + AllowInvalidFlag allowInvalid = DisallowInvalid); + + /* Return the size of the closure of the specified path, that is, + the sum of the size of the NAR serialisation of each path in + the closure. */ + std::pair<uint64_t, uint64_t> getClosureSize(const Path& storePath); + + /* Optimise the disk space usage of the Nix store by hard-linking files + with the same contents. */ + virtual void optimiseStore(){}; + + /* Check the integrity of the Nix store. Returns true if errors + remain. */ + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { + return false; + }; + + /* Return an object to access files in the Nix store. */ + virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); } + + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path& storePath, const StringSet& sigs) { + unsupported("addSignatures"); + } + + /* Utility functions. */ + + /* Read a derivation, after ensuring its existence through + ensurePath(). */ + Derivation derivationFromPath(const Path& drvPath); + + /* Place in `out' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly + or indirectly reached from it. `out' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ + virtual void computeFSClosure(const PathSet& paths, PathSet& paths_, + bool flipDirection = false, + bool includeOutputs = false, + bool includeDerivers = false); + + void computeFSClosure(const Path& path, PathSet& paths_, + bool flipDirection = false, bool includeOutputs = false, + bool includeDerivers = false); + + /* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths + that will be substituted. */ + virtual void queryMissing(const PathSet& targets, PathSet& willBuild, + PathSet& willSubstitute, PathSet& unknown, + unsigned long long& downloadSize, + unsigned long long& narSize); + + /* Sort a set of paths topologically under the references + relation. If p refers to q, then p precedes q in this list. */ + Paths topoSortPaths(const PathSet& paths); + + /* Export multiple paths in the format expected by ‘nix-store + --import’. */ + void exportPaths(const Paths& paths, Sink& sink); + + void exportPath(const Path& path, Sink& sink); + + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + Paths importPaths(Source& source, const std::shared_ptr<FSAccessor>& accessor, + CheckSigsFlag checkSigs = CheckSigs); + + struct Stats { + std::atomic<uint64_t> narInfoRead{0}; + std::atomic<uint64_t> narInfoReadAverted{0}; + std::atomic<uint64_t> narInfoMissing{0}; + std::atomic<uint64_t> narInfoWrite{0}; + std::atomic<uint64_t> pathInfoCacheSize{0}; + std::atomic<uint64_t> narRead{0}; + std::atomic<uint64_t> narReadBytes{0}; + std::atomic<uint64_t> narReadCompressedBytes{0}; + std::atomic<uint64_t> narWrite{0}; + std::atomic<uint64_t> narWriteAverted{0}; + std::atomic<uint64_t> narWriteBytes{0}; + std::atomic<uint64_t> narWriteCompressedBytes{0}; + std::atomic<uint64_t> narWriteCompressionTimeMs{0}; + }; + + const Stats& getStats(); + + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::shared_ptr<std::string> getBuildLog(const Path& path) { + return nullptr; + } + + /* Hack to allow long-running processes like hydra-queue-runner to + occasionally flush their path info cache. */ + void clearPathInfoCache() { state.lock()->pathInfoCache.clear(); } + + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect(){}; + + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() { return 0; }; + + /* Get the priority of the store, used to order substituters. In + particular, binary caches can specify a priority field in their + "nix-cache-info" file. Lower value means higher priority. */ + virtual int getPriority() { return 0; } + + virtual Path toRealPath(const Path& storePath) { return storePath; } + + virtual void createUser(const std::string& userName, uid_t userId) {} + + protected: + Stats stats; + + /* Unsupported methods. */ + [[noreturn]] void unsupported(const std::string& op) { + throw Unsupported("operation '%s' is not supported by store '%s'", op, + getUri()); + } +}; + +class LocalFSStore : public virtual Store { + public: + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to emit the call to the Option constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*)this, true, "", "root", + "directory prefixed to all other paths"}; + const PathSetting stateDir{ + (Store*)this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, "state", + "directory where Nix will store state"}; + const PathSetting logDir{ + (Store*)this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, "log", + "directory where Nix will store state"}; + + const static std::string drvsLogDir; + + LocalFSStore(const Params& params); + + void narFromPath(const Path& path, Sink& sink) override; + ref<FSAccessor> getFSAccessor() override; + + /* Register a permanent GC root. */ + Path addPermRoot(const Path& storePath, const Path& gcRoot, bool indirect, + bool allowOutsideRootsDir = false); + + virtual Path getRealStoreDir() { return storeDir; } + + Path toRealPath(const Path& storePath) override { + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + + std::string(storePath, storeDir.size() + 1); + } + + std::shared_ptr<std::string> getBuildLog(const Path& path) override; +}; + +/* Extract the name part of the given store path. */ +std::string storePathToName(const Path& path); + +/* Extract the hash part of the given store path. */ +std::string storePathToHash(const Path& path); + +/* Check whether ‘name’ is a valid store path name part, i.e. contains + only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with + a dot. */ +void checkStoreName(const std::string& name); + +/* Copy a path from one store to another. */ +void copyStorePath(ref<Store> srcStore, const ref<Store>& dstStore, + const Path& storePath, RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs); + +/* Copy store paths from one store to another. The paths may be copied + in parallel. They are copied in a topologically sorted order + (i.e. if A is a reference of B, then A is copied before B), but + the set of store paths is not automatically closed; use + copyClosure() for that. */ +void copyPaths(ref<Store> srcStore, ref<Store> dstStore, + const PathSet& storePaths, RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + +/* Copy the closure of the specified paths from one store to another. */ +void copyClosure(const ref<Store>& srcStore, const ref<Store>& dstStore, + const PathSet& storePaths, RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + +/* Remove the temporary roots file for this process. Any temporary + root becomes garbage after this point unless it has been registered + as a (permanent) root. */ +void removeTempRoots(); + +/* Return a Store object to access the Nix store denoted by + ‘uri’ (slight misnomer...). Supported values are: + + * ‘local’: The Nix store in /nix/store and database in + /nix/var/nix/db, accessed directly. + + * ‘daemon’: The Nix store accessed via a Unix domain socket + connection to nix-daemon. + + * ‘unix://<path>’: The Nix store accessed via a Unix domain socket + connection to nix-daemon, with the socket located at <path>. + + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + + * ‘file://<path>’: A binary cache stored in <path>. + + * ‘https://<path>’: A binary cache accessed via HTTP. + + * ‘s3://<path>’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]<host>’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. + + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref<Store> openStore(const std::string& uri = settings.storeUri.get(), + const Store::Params& extraParams = Store::Params()); + +enum StoreType { tDaemon, tLocal, tOther }; + +StoreType getStoreType(const std::string& uri = settings.storeUri.get(), + const std::string& stateDir = settings.nixStateDir); + +/* Return the default substituter stores, defined by the + ‘substituters’ option and various legacy options. */ +std::list<ref<Store>> getDefaultSubstituters(); + +/* Store implementation registration. */ +using OpenStore = std::function<std::shared_ptr<Store>(const std::string&, + const Store::Params&)>; + +struct RegisterStoreImplementation { + using Implementations = std::vector<OpenStore>; + static Implementations* implementations; + + RegisterStoreImplementation(OpenStore fun) { + if (!implementations) { + implementations = new Implementations; + } + implementations->push_back(fun); + } +}; + +/* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ +std::string showPaths(const PathSet& paths); + +ValidPathInfo decodeValidPathInfo(std::istream& str, bool hashGiven = false); + +/* Compute the content-addressability assertion (ValidPathInfo::ca) + for paths created by makeFixedOutputPath() / addToStore(). */ +std::string makeFixedOutputCA(bool recursive, const Hash& hash); + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair<std::string, Store::Params> splitUriAndParams(const std::string& uri); + +} // namespace nix diff --git a/third_party/nix/src/libstore/worker-protocol.hh b/third_party/nix/src/libstore/worker-protocol.hh new file mode 100644 index 000000000000..e2f40a449d86 --- /dev/null +++ b/third_party/nix/src/libstore/worker-protocol.hh @@ -0,0 +1,68 @@ +#pragma once + +#include "libstore/store-api.hh" +#include "libutil/types.hh" + +namespace nix { + +#define WORKER_MAGIC_1 0x6e697863 +#define WORKER_MAGIC_2 0x6478696f + +#define PROTOCOL_VERSION 0x115 +#define GET_PROTOCOL_MAJOR(x) ((x)&0xff00) +#define GET_PROTOCOL_MINOR(x) ((x)&0x00ff) + +typedef enum { + wopIsValidPath = 1, + wopHasSubstitutes = 3, + wopQueryPathHash = 4, // obsolete + wopQueryReferences = 5, // obsolete + wopQueryReferrers = 6, + wopAddToStore = 7, + wopAddTextToStore = 8, + wopBuildPaths = 9, + wopEnsurePath = 10, + wopAddTempRoot = 11, + wopAddIndirectRoot = 12, + wopSyncWithGC = 13, + wopFindRoots = 14, + wopExportPath = 16, // obsolete + wopQueryDeriver = 18, // obsolete + wopSetOptions = 19, + wopCollectGarbage = 20, + wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, + wopQueryAllValidPaths = 23, + wopQueryFailedPaths = 24, // obsolete + wopClearFailedPaths = 25, // obsolete + wopQueryPathInfo = 26, + wopImportPaths = 27, // obsolete + wopQueryDerivationOutputNames = 28, + wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, + wopQueryValidDerivers = 33, + wopOptimiseStore = 34, + wopVerifyStore = 35, + wopBuildDerivation = 36, + wopAddSignatures = 37, + wopNarFromPath = 38, + wopAddToStoreNar = 39, + wopQueryMissing = 40, +} WorkerOp; + +#define STDERR_NEXT 0x6f6c6d67 +#define STDERR_READ 0x64617461 // data needed from source +#define STDERR_WRITE 0x64617416 // data for sink +#define STDERR_LAST 0x616c7473 +#define STDERR_ERROR 0x63787470 +#define STDERR_START_ACTIVITY 0x53545254 +#define STDERR_STOP_ACTIVITY 0x53544f50 +#define STDERR_RESULT 0x52534c54 + +Path readStorePath(Store& store, Source& from); +template <class T> +T readStorePaths(Store& store, Source& from); + +} // namespace nix diff --git a/third_party/nix/src/libutil/CMakeLists.txt b/third_party/nix/src/libutil/CMakeLists.txt new file mode 100644 index 000000000000..0b36929218e8 --- /dev/null +++ b/third_party/nix/src/libutil/CMakeLists.txt @@ -0,0 +1,68 @@ +# -*- mode: cmake; -*- +add_library(nixutil SHARED) +set_property(TARGET nixutil PROPERTY CXX_STANDARD 17) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +target_compile_features(nixutil PUBLIC cxx_std_17) + +set(HEADER_FILES + affinity.hh + archive.hh + args.hh + compression.hh + config.hh + finally.hh + hash.hh + istringstream_nocopy.hh + json.hh + lazy.hh + lru-cache.hh + monitor-fd.hh + pool.hh + proto.hh + ref.hh + serialise.hh + status.hh + sync.hh + thread-pool.hh + types.hh + util.hh + visitor.hh + xml-writer.hh +) + +target_sources(nixutil + PUBLIC + ${HEADER_FILES} + + PRIVATE + affinity.cc + archive.cc + args.cc + compression.cc + config.cc + hash.cc + json.cc + serialise.cc + thread-pool.cc + util.cc + xml-writer.cc +) + +target_link_libraries(nixutil + nixproto + absl::strings + absl::statusor + glog + BZip2::BZip2 + LibLZMA::LibLZMA + Boost::context + brotlienc + brotlidec + ssl +) + +# Install header files to include/libutil and mark them for automatic +# inclusion in targets that link to this one. +target_include_directories(nixutil PUBLIC "${nix_SOURCE_DIR}/src") +INSTALL(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nix/libutil) +INSTALL(TARGETS nixutil DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/third_party/nix/src/libutil/affinity.cc b/third_party/nix/src/libutil/affinity.cc new file mode 100644 index 000000000000..03fbe1243992 --- /dev/null +++ b/third_party/nix/src/libutil/affinity.cc @@ -0,0 +1,60 @@ +#include "libutil/affinity.hh" + +#include <glog/logging.h> + +#include "libutil/types.hh" +#include "libutil/util.hh" + +#if __linux__ +#include <sched.h> +#endif + +namespace nix { + +#if __linux__ +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; +#endif + +void setAffinityTo(int cpu) { +#if __linux__ + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) { + return; + } + + didSaveAffinity = true; + DLOG(INFO) << "locking this thread to CPU " << cpu; + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) { + LOG(ERROR) << "failed to lock thread to CPU " << cpu; + } +#endif +} + +int lockToCurrentCPU() { +#if __linux__ + int cpu = sched_getcpu(); + if (cpu != -1) { + setAffinityTo(cpu); + } + return cpu; +#else + return -1; +#endif +} + +void restoreAffinity() { +#if __linux__ + if (!didSaveAffinity) { + return; + } + + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) { + LOG(ERROR) << "failed to restore affinity"; + } +#endif +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/affinity.hh b/third_party/nix/src/libutil/affinity.hh new file mode 100644 index 000000000000..5e5ef9b0de0d --- /dev/null +++ b/third_party/nix/src/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} // namespace nix diff --git a/third_party/nix/src/libutil/archive.cc b/third_party/nix/src/libutil/archive.cc new file mode 100644 index 000000000000..e470ad7be6ce --- /dev/null +++ b/third_party/nix/src/libutil/archive.cc @@ -0,0 +1,398 @@ +#include "libutil/archive.hh" + +#include <algorithm> +#include <cerrno> +#include <map> +#include <vector> + +#include <dirent.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <strings.h> // for strcasecmp +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libutil/config.hh" +#include "libutil/util.hh" + +namespace nix { + +struct ArchiveSettings : Config { + Setting<bool> useCaseHack { + this, +#if __APPLE__ + true, +#else + false, +#endif + "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name " + "collisions." + }; +}; + +static ArchiveSettings archiveSettings; + +static GlobalConfig::Register r1(&archiveSettings); + +constexpr std::string_view kCaseHackSuffix = "~nix~case~hack~"; + +PathFilter defaultPathFilter = [](const Path& /*unused*/) { return true; }; + +static void dumpContents(const Path& path, size_t size, Sink& sink) { + sink << "contents" << size; + + AutoCloseFD fd(open(path.c_str(), O_RDONLY | O_CLOEXEC)); + if (!fd) { + throw SysError(format("opening file '%1%'") % path); + } + + std::vector<unsigned char> buf(65536); + size_t left = size; + + while (left > 0) { + auto n = std::min(left, buf.size()); + readFull(fd.get(), buf.data(), n); + left -= n; + sink(buf.data(), n); + } + + writePadding(size, sink); +} + +static void dump(const Path& path, Sink& sink, PathFilter& filter) { + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting attributes of path '%1%'") % path); + } + + sink << "("; + + if (S_ISREG(st.st_mode)) { + sink << "type" + << "regular"; + if ((st.st_mode & S_IXUSR) != 0u) { + sink << "executable" + << ""; + } + dumpContents(path, static_cast<size_t>(st.st_size), sink); + } + + else if (S_ISDIR(st.st_mode)) { + sink << "type" + << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map<std::string, std::string> unhacked; + for (auto& i : readDirectory(path)) { + if (archiveSettings.useCaseHack) { + std::string name(i.name); + size_t pos = i.name.find(kCaseHackSuffix); + if (pos != std::string::npos) { + DLOG(INFO) << "removing case hack suffix from " << path << "/" + << i.name; + + name.erase(pos); + } + if (unhacked.find(name) != unhacked.end()) { + throw Error(format("file name collision in between '%1%' and '%2%'") % + (path + "/" + unhacked[name]) % (path + "/" + i.name)); + } + unhacked[name] = i.name; + } else { + unhacked[i.name] = i.name; + } + } + + for (auto& i : unhacked) { + if (filter(path + "/" + i.first)) { + sink << "entry" + << "(" + << "name" << i.first << "node"; + dump(path + "/" + i.second, sink, filter); + sink << ")"; + } + } + } + + else if (S_ISLNK(st.st_mode)) { + sink << "type" + << "symlink" + << "target" << readLink(path); + + } else { + throw Error(format("file '%1%' has an unsupported type") % path); + } + + sink << ")"; +} + +void dumpPath(const Path& path, Sink& sink, PathFilter& filter) { + sink << std::string(kNarVersionMagic1); + dump(path, sink, filter); +} + +void dumpString(const std::string& s, Sink& sink) { + sink << std::string(kNarVersionMagic1) << "(" + << "type" + << "regular" + << "contents" << s << ")"; +} + +static SerialisationError badArchive(const std::string& s) { + return SerialisationError("bad archive: " + s); +} + +#if 0 +static void skipGeneric(Source & source) +{ + if (readString(source) == "(") { + while (readString(source) != ")") + skipGeneric(source); + } +} +#endif + +static void parseContents(ParseSink& sink, Source& source, const Path& path) { + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector<unsigned char> buf(65536); + + while (left != 0u) { + checkInterrupt(); + auto n = buf.size(); + if (static_cast<unsigned long long>(n) > left) { + n = left; + } + source(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + + readPadding(size, source); +} + +struct CaseInsensitiveCompare { + bool operator()(const std::string& a, const std::string& b) const { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; + +static void parse(ParseSink& sink, Source& source, const Path& path) { + std::string s; + + s = readString(source); + if (s != "(") { + throw badArchive("expected open tag"); + } + + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + + std::map<Path, int, CaseInsensitiveCompare> names; + + while (true) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + + if (s == "type") { + if (type != tpUnknown) { + throw badArchive("multiple type fields"); + } + std::string t = readString(source); + + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } + + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } + + else if (t == "symlink") { + type = tpSymlink; + } + + else { + throw badArchive("unknown file type " + t); + } + + } + + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } + + else if (s == "executable" && type == tpRegular) { + auto s = readString(source); + if (!s.empty()) { + throw badArchive("executable marker has non-empty value"); + } + sink.isExecutable(); + } + + else if (s == "entry" && type == tpDirectory) { + std::string name; + std::string prevName; + + s = readString(source); + if (s != "(") { + throw badArchive("expected open tag"); + } + + while (true) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + if (s == "name") { + name = readString(source); + if (name.empty() || name == "." || name == ".." || + name.find('/') != std::string::npos || + name.find(static_cast<char>(0)) != std::string::npos) { + throw Error(format("NAR contains invalid file name '%1%'") % name); + } + if (name <= prevName) { + throw Error("NAR directory is not sorted"); + } + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + DLOG(INFO) << "case collision between '" << i->first << "' and '" + << name << "'"; + name += kCaseHackSuffix; + name += std::to_string(++i->second); + } else { + names[name] = 0; + } + } + } else if (s == "node") { + if (s.empty()) { + throw badArchive("entry name missing"); + } + parse(sink, source, path + "/" + name); + } else { + throw badArchive("unknown field " + s); + } + } + } + + else if (s == "target" && type == tpSymlink) { + std::string target = readString(source); + sink.createSymlink(path, target); + } + + else { + throw badArchive("unknown field " + s); + } + } +} + +void parseDump(ParseSink& sink, Source& source) { + std::string version; + try { + version = readString(source, kNarVersionMagic1.size()); + } catch (SerialisationError& e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != kNarVersionMagic1) { + throw badArchive("input doesn't look like a Nix archive"); + } + parse(sink, source, ""); +} + +struct RestoreSink : ParseSink { + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path& path) override { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) { + throw SysError(format("creating directory '%1%'") % p); + } + }; + + void createRegularFile(const Path& path) override { + Path p = dstPath + path; + fd = AutoCloseFD( + open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666)); + if (!fd) { + throw SysError(format("creating file '%1%'") % p); + } + } + + void isExecutable() override { + struct stat st; + if (fstat(fd.get(), &st) == -1) { + throw SysError("fstat"); + } + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) { + throw SysError("fchmod"); + } + } + + void preallocateContents(unsigned long long len) override { +#if HAVE_POSIX_FALLOCATE + if (len != 0u) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) { + throw SysError(format("preallocating file of %1% bytes") % len); + } + } +#endif + } + + void receiveContents(unsigned char* data, unsigned int len) override { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path& path, const std::string& target) override { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + +void restorePath(const Path& path, Source& source) { + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); +} + +void copyNAR(Source& source, Sink& sink) { + // FIXME: if 'source' is the output of dumpPath() followed by EOF, + // we should just forward all data directly without parsing. + + ParseSink parseSink; /* null sink; just parse the NAR */ + + LambdaSource wrapper([&](unsigned char* data, size_t len) { + auto n = source.read(data, len); + sink(data, n); + return n; + }); + + parseDump(parseSink, wrapper); +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/archive.hh b/third_party/nix/src/libutil/archive.hh new file mode 100644 index 000000000000..396627878541 --- /dev/null +++ b/third_party/nix/src/libutil/archive.hh @@ -0,0 +1,77 @@ +#pragma once + +#include "libutil/serialise.hh" +#include "libutil/types.hh" + +namespace nix { + +/* dumpPath creates a Nix archive of the specified path. The format + is as follows: + + IF path points to a REGULAR FILE: + dump(path) = attrs( + [ ("type", "regular") + , ("contents", contents(path)) + ]) + + IF path points to a DIRECTORY: + dump(path) = attrs( + [ ("type", "directory") + , ("entries", concat(map(f, sort(entries(path))))) + ]) + where f(fn) = attrs( + [ ("name", fn) + , ("file", dump(path + "/" + fn)) + ]) + + where: + + attrs(as) = concat(map(attr, as)) + encN(0) + attrs((a, b)) = encS(a) + encS(b) + + encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + + encN(n) = 64-bit little-endian encoding of n. + + contents(path) = the contents of a regular file. + + sort(strings) = lexicographic sort by 8-bit value (strcmp). + + entries(path) = the entries of a directory, without `.' and + `..'. + + `+' denotes string concatenation. */ + +void dumpPath(const Path& path, Sink& sink, + PathFilter& filter = defaultPathFilter); + +void dumpString(const std::string& s, Sink& sink); + +/* FIXME: fix this API, it sucks. */ +struct ParseSink { + virtual void createDirectory(const Path& path){}; + + virtual void createRegularFile(const Path& path){}; + virtual void isExecutable(){}; + virtual void preallocateContents(unsigned long long size){}; + virtual void receiveContents(unsigned char* data, unsigned int len){}; + + virtual void createSymlink(const Path& path, const std::string& target){}; +}; + +struct TeeSink : ParseSink { + TeeSource source; + + explicit TeeSink(Source& source) : source(source) {} +}; + +void parseDump(ParseSink& sink, Source& source); + +void restorePath(const Path& path, Source& source); + +/* Read a NAR from 'source' and write it to 'sink'. */ +void copyNAR(Source& source, Sink& sink); + +constexpr std::string_view kNarVersionMagic1 = "nix-archive-1"; + +} // namespace nix diff --git a/third_party/nix/src/libutil/args.cc b/third_party/nix/src/libutil/args.cc new file mode 100644 index 000000000000..2be8a1b0ce2a --- /dev/null +++ b/third_party/nix/src/libutil/args.cc @@ -0,0 +1,219 @@ +#include "libutil/args.hh" + +#include "libutil/hash.hh" + +namespace nix { + +Args::FlagMaker Args::mkFlag() { return FlagMaker(*this); } + +Args::FlagMaker::~FlagMaker() { + assert(!flag->longName.empty()); + args.longFlags[flag->longName] = flag; + if (flag->shortName != 0) { + args.shortFlags[flag->shortName] = flag; + } +} + +void Args::parseCmdline(const Strings& _cmdline) { + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end();) { + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && + (isalpha(arg[1]) != 0)) { + *pos = std::string("-") + arg[1]; + auto next = pos; + ++next; + for (unsigned int j = 2; j < arg.length(); j++) { + if (isalpha(arg[j]) != 0) { + cmdline.insert(next, std::string("-") + arg[j]); + } else { + cmdline.insert(next, std::string(arg, j)); + break; + } + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) { + throw UsageError(format("unrecognised flag '%1%'") % arg); + } + } else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) { + pendingArgs.clear(); + } + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const std::string& programName, std::ostream& out) { + std::cout << "Usage: " << programName << " <FLAGS>..."; + for (auto& exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) { + std::cout << "..."; + } + if (exp.optional) { + std::cout << "?"; + } + } + std::cout << "\n"; + + auto s = description(); + if (!s.empty()) { + std::cout << "\nSummary: " << s << ".\n"; + } + + if (!longFlags.empty() != 0u) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream& out) { + Table2 table; + for (auto& flag : longFlags) { + if (hiddenCategories.count(flag.second->category) != 0u) { + continue; + } + table.push_back(std::make_pair( + (flag.second->shortName != 0 + ? std::string("-") + flag.second->shortName + ", " + : " ") + + "--" + flag.first + renderLabels(flag.second->labels), + flag.second->description)); + } + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator& pos, Strings::iterator end) { + assert(pos != end); + + auto process = [&](const std::string& name, const Flag& flag) -> bool { + ++pos; + std::vector<std::string> args; + for (size_t n = 0; n < flag.arity; ++n) { + if (pos == end) { + if (flag.arity == ArityAny) { + break; + } + throw UsageError(format("flag '%1%' requires %2% argument(s)") % name % + flag.arity); + } + args.push_back(*pos++); + } + flag.handler(std::move(args)); + return true; + }; + + if (std::string(*pos, 0, 2) == "--") { + auto i = longFlags.find(std::string(*pos, 2)); + if (i == longFlags.end()) { + return false; + } + return process("--" + i->first, *i->second); + } + + if (std::string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) { + return false; + } + return process(std::string("-") + c, *i->second); + } + + return false; +} + +bool Args::processArgs(const Strings& args, bool finish) { + if (expectedArgs.empty()) { + if (!args.empty()) { + throw UsageError(format("unexpected argument '%1%'") % args.front()); + } + return true; + } + + auto& exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) { + std::vector<std::string> ss; + for (auto& s : args) { + ss.push_back(s); + } + exp.handler(std::move(ss)); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty() && !expectedArgs.front().optional) { + throw UsageError("more arguments are required"); + } + + return res; +} + +Args::FlagMaker& Args::FlagMaker::mkHashTypeFlag(HashType* ht) { + arity(1); + label("type"); + description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); + handler([ht](const std::string& s) { + *ht = parseHashType(s); + if (*ht == htUnknown) { + throw UsageError("unknown hash type '%1%'", s); + } + }); + return *this; +} + +Strings argvToStrings(int argc, char** argv) { + Strings args; + argc--; + argv++; + while ((argc--) != 0) { + args.push_back(*argv++); + } + return args; +} + +std::string renderLabels(const Strings& labels) { + std::string res; + for (auto label : labels) { + for (auto& c : label) { + c = std::toupper(c); + } + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream& out, const Table2& table) { + size_t max = 0; + for (auto& row : table) { + max = std::max(max, row.first.size()); + } + for (auto& row : table) { + out << " " << row.first << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/args.hh b/third_party/nix/src/libutil/args.hh new file mode 100644 index 000000000000..bb1ef43912bb --- /dev/null +++ b/third_party/nix/src/libutil/args.hh @@ -0,0 +1,221 @@ +#pragma once + +#include <iostream> +#include <map> +#include <memory> + +#include <absl/strings/numbers.h> + +#include "libutil/util.hh" + +namespace nix { + +MakeError(UsageError, Error); + +enum HashType : char; + +class Args { + public: + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings& cmdline); + + virtual void printHelp(const std::string& programName, std::ostream& out); + + virtual std::string description() { return ""; } + + protected: + static const size_t ArityAny = std::numeric_limits<size_t>::max(); + + /* Flags. */ + struct Flag { + typedef std::shared_ptr<Flag> ptr; + std::string longName; + char shortName = 0; + std::string description; + Strings labels; + size_t arity = 0; + std::function<void(std::vector<std::string>)> handler; + std::string category; + }; + + std::map<std::string, Flag::ptr> longFlags; + std::map<char, Flag::ptr> shortFlags; + + virtual bool processFlag(Strings::iterator& pos, Strings::iterator end); + + virtual void printFlags(std::ostream& out); + + /* Positional arguments. */ + struct ExpectedArg { + std::string label; + size_t arity; // 0 = any + bool optional; + std::function<void(std::vector<std::string>)> handler; + }; + + std::list<ExpectedArg> expectedArgs; + + virtual bool processArgs(const Strings& args, bool finish); + + std::set<std::string> hiddenCategories; + + public: + class FlagMaker { + Args& args; + Flag::ptr flag; + friend class Args; + explicit FlagMaker(Args& args) + : args(args), flag(std::make_shared<Flag>()){}; + + public: + ~FlagMaker(); + FlagMaker& longName(const std::string& s) { + flag->longName = s; + return *this; + }; + FlagMaker& shortName(char s) { + flag->shortName = s; + return *this; + }; + FlagMaker& description(const std::string& s) { + flag->description = s; + return *this; + }; + FlagMaker& label(const std::string& l) { + flag->arity = 1; + flag->labels = {l}; + return *this; + }; + FlagMaker& labels(const Strings& ls) { + flag->arity = ls.size(); + flag->labels = ls; + return *this; + }; + FlagMaker& arity(size_t arity) { + flag->arity = arity; + return *this; + }; + FlagMaker& handler(std::function<void(std::vector<std::string>)> handler) { + flag->handler = handler; + return *this; + }; + FlagMaker& handler(std::function<void()> handler) { + flag->handler = [handler](std::vector<std::string>) { handler(); }; + return *this; + }; + FlagMaker& handler(std::function<void(std::string)> handler) { + flag->arity = 1; + flag->handler = [handler](std::vector<std::string> ss) { + handler(std::move(ss[0])); + }; + return *this; + }; + FlagMaker& category(const std::string& s) { + flag->category = s; + return *this; + }; + + template <class T> + FlagMaker& dest(T* dest) { + flag->arity = 1; + flag->handler = [=](std::vector<std::string> ss) { *dest = ss[0]; }; + return *this; + } + + template <class T> + FlagMaker& set(T* dest, const T& val) { + flag->arity = 0; + flag->handler = [=](std::vector<std::string> ss) { *dest = val; }; + return *this; + } + + FlagMaker& mkHashTypeFlag(HashType* ht); + }; + + FlagMaker mkFlag(); + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag1(char shortName, const std::string& longName, + const std::string& label, const std::string& description, + std::function<void(std::string)> fun) { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({label}) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { fun(ss[0]); }); + } + + void mkFlag(char shortName, const std::string& name, + const std::string& description, bool* dest) { + mkFlag(shortName, name, description, dest, true); + } + + template <class T> + void mkFlag(char shortName, const std::string& longName, + const std::string& description, T* dest, const T& value) { + mkFlag() + .shortName(shortName) + .longName(longName) + .description(description) + .handler([=](std::vector<std::string> ss) { *dest = value; }); + } + + template <class I> + void mkIntFlag(char shortName, const std::string& longName, + const std::string& description, I* dest) { + mkFlag<I>(shortName, longName, description, [=](I n) { *dest = n; }); + } + + template <class I> + void mkFlag(char shortName, const std::string& longName, + const std::string& description, std::function<void(I)> fun) { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({"N"}) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { + I n; + if (!absl::SimpleAtoi(ss[0], &n)) { + throw UsageError("flag '--%s' requires a integer argument", + longName); + } + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string& label, std::string* dest, + bool optional = false) { + expectedArgs.push_back( + ExpectedArg{label, 1, optional, + [=](std::vector<std::string> ss) { *dest = ss[0]; }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string& label, std::vector<std::string>* dest) { + expectedArgs.push_back(ExpectedArg{ + label, 0, false, + [=](std::vector<std::string> ss) { *dest = std::move(ss); }}); + } + + friend class MultiCommand; +}; + +Strings argvToStrings(int argc, char** argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings& labels); + +/* Helper function for printing 2-column tables. */ +using Table2 = std::vector<std::pair<std::string, std::string> >; + +void printTable(std::ostream& out, const Table2& table); + +} // namespace nix diff --git a/third_party/nix/src/libutil/compression.cc b/third_party/nix/src/libutil/compression.cc new file mode 100644 index 000000000000..d0895ca5fda4 --- /dev/null +++ b/third_party/nix/src/libutil/compression.cc @@ -0,0 +1,400 @@ +#include "libutil/compression.hh" + +#include <cstdio> +#include <cstring> +#include <iostream> + +#include <brotli/decode.h> +#include <brotli/encode.h> +#include <bzlib.h> +#include <glog/logging.h> +#include <lzma.h> + +#include "libutil/finally.hh" +#include "libutil/util.hh" + +namespace nix { + +// Don't feed brotli too much at once. +struct ChunkedCompressionSink : CompressionSink { + uint8_t outbuf[32 * 1024]; + + void write(const unsigned char* data, size_t len) override { + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len != 0u) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; + } + } + + virtual void writeInternal(const unsigned char* data, size_t len) = 0; +}; + +struct NoneSink : CompressionSink { + Sink& nextSink; + explicit NoneSink(Sink& nextSink) : nextSink(nextSink) {} + void finish() override { flush(); } + void write(const unsigned char* data, size_t len) override { + nextSink(data, len); + } +}; + +struct XzDecompressionSink : CompressionSink { + Sink& nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + explicit XzDecompressionSink(Sink& nextSink) : nextSink(nextSink) { + lzma_ret ret = lzma_stream_decoder(&strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) { + throw CompressionError("unable to initialise lzma decoder"); + } + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzDecompressionSink() override { lzma_end(&strm); } + + void finish() override { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char* data, size_t len) override { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && ((data == nullptr) || (strm.avail_in != 0u))) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data != nullptr ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) { + throw CompressionError("error %d while decompressing xz file", ret); + } + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipDecompressionSink : ChunkedCompressionSink { + Sink& nextSink; + bz_stream strm; + bool finished = false; + + explicit BzipDecompressionSink(Sink& nextSink) : nextSink(nextSink) { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) { + throw CompressionError("unable to initialise bzip2 decoder"); + } + + strm.next_out = reinterpret_cast<char*>(outbuf); + strm.avail_out = sizeof(outbuf); + } + + ~BzipDecompressionSink() override { BZ2_bzDecompressEnd(&strm); } + + void finish() override { + flush(); + write(nullptr, 0); + } + + void writeInternal(const unsigned char* data, size_t len) override { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + + strm.next_in = (char*)data; + strm.avail_in = len; + + while (strm.avail_in != 0u) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + if (ret != BZ_OK && ret != BZ_STREAM_END) { + throw CompressionError("error while decompressing bzip2 file"); + } + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = reinterpret_cast<char*>(outbuf); + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BrotliDecompressionSink : ChunkedCompressionSink { + Sink& nextSink; + BrotliDecoderState* state; + bool finished = false; + + explicit BrotliDecompressionSink(Sink& nextSink) : nextSink(nextSink) { + state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (state == nullptr) { + throw CompressionError("unable to initialize brotli decoder"); + } + } + + ~BrotliDecompressionSink() override { BrotliDecoderDestroyInstance(state); } + + void finish() override { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char* data, size_t len) override { + const uint8_t* next_in = data; + size_t avail_in = len; + uint8_t* next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && ((data == nullptr) || (avail_in != 0u))) { + checkInterrupt(); + + if (BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, + &next_out, nullptr) == 0u) { + throw CompressionError("error while decompressing brotli file"); + } + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = (BrotliDecoderIsFinished(state) != 0); + } + } +}; + +ref<std::string> decompress(const std::string& method, const std::string& in) { + StringSink ssink; + auto sink = makeDecompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +ref<CompressionSink> makeDecompressionSink(const std::string& method, + Sink& nextSink) { + if (method == "none" || method.empty()) { + return make_ref<NoneSink>(nextSink); + } + if (method == "xz") { + return make_ref<XzDecompressionSink>(nextSink); + } else if (method == "bzip2") { + return make_ref<BzipDecompressionSink>(nextSink); + } else if (method == "br") { + return make_ref<BrotliDecompressionSink>(nextSink); + } else { + throw UnknownCompressionMethod("unknown compression method '%s'", method); + } +} + +struct XzCompressionSink : CompressionSink { + Sink& nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzCompressionSink(Sink& nextSink, bool parallel) : nextSink(nextSink) { + lzma_ret ret; + bool done = false; + + if (parallel) { + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) { + mt_options.threads = 1; + } + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + ret = lzma_stream_encoder_mt(&strm, &mt_options); + done = true; + } + + if (!done) { + ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + } + + if (ret != LZMA_OK) { + throw CompressionError("unable to initialise lzma encoder"); + } + + // FIXME: apply the x86 BCJ filter? + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzCompressionSink() override { lzma_end(&strm); } + + void finish() override { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char* data, size_t len) override { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && ((data == nullptr) || (strm.avail_in != 0u))) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data != nullptr ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) { + throw CompressionError("error %d while compressing xz file", ret); + } + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipCompressionSink : ChunkedCompressionSink { + Sink& nextSink; + bz_stream strm; + bool finished = false; + + explicit BzipCompressionSink(Sink& nextSink) : nextSink(nextSink) { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) { + throw CompressionError("unable to initialise bzip2 encoder"); + } + + strm.next_out = reinterpret_cast<char*>(outbuf); + strm.avail_out = sizeof(outbuf); + } + + ~BzipCompressionSink() override { BZ2_bzCompressEnd(&strm); } + + void finish() override { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char* data, size_t len) override { + assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max()); + + strm.next_in = (char*)data; + strm.avail_in = len; + + while (!finished && ((data == nullptr) || (strm.avail_in != 0u))) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, data != nullptr ? BZ_RUN : BZ_FINISH); + if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) { + throw CompressionError("error %d while compressing bzip2 file", ret); + } + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = reinterpret_cast<char*>(outbuf); + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BrotliCompressionSink : ChunkedCompressionSink { + Sink& nextSink; + uint8_t outbuf[BUFSIZ]; + BrotliEncoderState* state; + bool finished = false; + + explicit BrotliCompressionSink(Sink& nextSink) : nextSink(nextSink) { + state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (state == nullptr) { + throw CompressionError("unable to initialise brotli encoder"); + } + } + + ~BrotliCompressionSink() override { BrotliEncoderDestroyInstance(state); } + + void finish() override { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char* data, size_t len) override { + const uint8_t* next_in = data; + size_t avail_in = len; + uint8_t* next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && ((data == nullptr) || (avail_in != 0u))) { + checkInterrupt(); + + if (BrotliEncoderCompressStream(state, + data != nullptr ? BROTLI_OPERATION_PROCESS + : BROTLI_OPERATION_FINISH, + &avail_in, &next_in, &avail_out, + &next_out, nullptr) == 0) { + throw CompressionError("error while compressing brotli compression"); + } + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = (BrotliEncoderIsFinished(state) != 0); + } + } +}; + +ref<CompressionSink> makeCompressionSink(const std::string& method, + Sink& nextSink, const bool parallel) { + if (method == "none") { + return make_ref<NoneSink>(nextSink); + } + if (method == "xz") { + return make_ref<XzCompressionSink>(nextSink, parallel); + } else if (method == "bzip2") { + return make_ref<BzipCompressionSink>(nextSink); + } else if (method == "br") { + return make_ref<BrotliCompressionSink>(nextSink); + } else { + throw UnknownCompressionMethod(format("unknown compression method '%s'") % + method); + } +} + +ref<std::string> compress(const std::string& method, const std::string& in, + const bool parallel) { + StringSink ssink; + auto sink = makeCompressionSink(method, ssink, parallel); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/compression.hh b/third_party/nix/src/libutil/compression.hh new file mode 100644 index 000000000000..8ec340ab7447 --- /dev/null +++ b/third_party/nix/src/libutil/compression.hh @@ -0,0 +1,31 @@ +#pragma once + +#include <string> + +#include "libutil/ref.hh" +#include "libutil/serialise.hh" +#include "libutil/types.hh" + +namespace nix { + +struct CompressionSink : BufferedSink { + virtual void finish() = 0; +}; + +ref<std::string> decompress(const std::string& method, const std::string& in); + +ref<CompressionSink> makeDecompressionSink(const std::string& method, + Sink& nextSink); + +ref<std::string> compress(const std::string& method, const std::string& in, + const bool parallel = false); + +ref<CompressionSink> makeCompressionSink(const std::string& method, + Sink& nextSink, + const bool parallel = false); + +MakeError(UnknownCompressionMethod, Error); + +MakeError(CompressionError, Error); + +} // namespace nix diff --git a/third_party/nix/src/libutil/config.cc b/third_party/nix/src/libutil/config.cc new file mode 100644 index 000000000000..7c6e7af48752 --- /dev/null +++ b/third_party/nix/src/libutil/config.cc @@ -0,0 +1,370 @@ +#include "libutil/config.hh" + +#include <string> +#include <utility> +#include <vector> + +#include <absl/strings/numbers.h> +#include <absl/strings/str_split.h> +#include <absl/strings/string_view.h> +#include <glog/logging.h> + +#include "libutil/args.hh" +#include "libutil/json.hh" + +namespace nix { + +bool Config::set(const std::string& name, const std::string& value) { + auto i = _settings.find(name); + if (i == _settings.end()) { + return false; + } + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; +} + +void Config::addSetting(AbstractSetting* setting) { + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto& alias : setting->aliases) { + _settings.emplace(alias, Config::SettingData(true, setting)); + } + + bool set = false; + + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + + for (auto& alias : setting->aliases) { + auto i = unknownSettings.find(alias); + if (i != unknownSettings.end()) { + if (set) { + LOG(WARNING) << "setting '" << alias + << "' is set, but it's an alias of '" << setting->name + << "', which is also set"; + } + + else { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + } + } +} + +void AbstractConfig::warnUnknownSettings() { + for (auto& s : unknownSettings) { + LOG(WARNING) << "unknown setting: " << s.first; + } +} + +void AbstractConfig::reapplyUnknownSettings() { + auto unknownSettings2 = std::move(unknownSettings); + for (auto& s : unknownSettings2) { + set(s.first, s.second); + } +} + +void Config::getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly) { + for (auto& opt : _settings) { + if (!opt.second.isAlias && + (!overridenOnly || opt.second.setting->overriden)) { + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), + opt.second.setting->description}); + } + } +} + +void AbstractConfig::applyConfigFile(const Path& path) { + try { + std::string contents = readFile(path); + + unsigned int pos = 0; + + while (pos < contents.size()) { + std::string line; + while (pos < contents.size() && contents[pos] != '\n') { + line += contents[pos++]; + } + pos++; + + std::string::size_type hash = line.find('#'); + if (hash != std::string::npos) { + line = std::string(line, 0, hash); + } + + // TODO(tazjin): absl::string_view after path functions are fixed. + std::vector<std::string> tokens = absl::StrSplit( + line, absl::ByAnyChar(" \t\n\r"), absl::SkipWhitespace()); + if (tokens.empty()) { + continue; + } + + if (tokens.size() < 2) { + throw UsageError("illegal configuration line '%1%' in '%2%'", line, + path); + } + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") { + include = true; + } else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) { + throw UsageError("illegal configuration line '%1%' in '%2%'", line, + path); + } + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") { + throw UsageError("illegal configuration line '%1%' in '%2%'", line, + path); + } + + std::string name = tokens[0]; + + auto i = tokens.begin(); + advance(i, 2); + + set(name, + concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; + } catch (SysError&) { + } +} + +void Config::resetOverriden() { + for (auto& s : _settings) { + s.second.setting->overriden = false; + } +} + +void Config::toJSON(JSONObject& out) { + for (auto& s : _settings) { + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } + } +} + +void Config::convertToArgs(Args& args, const std::string& category) { + for (auto& s : _settings) { + if (!s.second.isAlias) { + s.second.setting->convertToArg(args, category); + } + } +} + +AbstractSetting::AbstractSetting(std::string name, std::string description, + std::set<std::string> aliases) + : name(std::move(name)), + description(std::move(description)), + aliases(std::move(aliases)) {} + +void AbstractSetting::toJSON(JSONPlaceholder& out) { out.write(to_string()); } + +void AbstractSetting::convertToArg(Args& args, const std::string& category) {} + +template <typename T> +void BaseSetting<T>::toJSON(JSONPlaceholder& out) { + out.write(value); +} + +template <typename T> +void BaseSetting<T>::convertToArg(Args& args, const std::string& category) { + args.mkFlag() + .longName(name) + .description(description) + .arity(1) + .handler([=](std::vector<std::string> ss) { + overriden = true; + set(ss[0]); + }) + .category(category); +} + +template <> +void BaseSetting<std::string>::set(const std::string& str) { + value = str; +} + +template <> +std::string BaseSetting<std::string>::to_string() { + return value; +} + +template <typename T> +void BaseSetting<T>::set(const std::string& str) { + static_assert(std::is_integral<T>::value, "Integer required."); + if (!absl::SimpleAtoi(str, &value)) { + throw UsageError("setting '%s' has invalid value '%s'", name, str); + } +} + +template <typename T> +std::string BaseSetting<T>::to_string() { + static_assert(std::is_integral<T>::value, "Integer required."); + return std::to_string(value); +} + +template <> +void BaseSetting<bool>::set(const std::string& str) { + if (str == "true" || str == "yes" || str == "1") { + value = true; + } else if (str == "false" || str == "no" || str == "0") { + value = false; + } else { + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); + } +} + +template <> +std::string BaseSetting<bool>::to_string() { + return value ? "true" : "false"; +} + +template <> +void BaseSetting<bool>::convertToArg(Args& args, const std::string& category) { + args.mkFlag() + .longName(name) + .description(description) + .handler([=](const std::vector<std::string>& ss) { override(true); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description(description) + .handler([=](const std::vector<std::string>& ss) { override(false); }) + .category(category); +} + +template <> +void BaseSetting<Strings>::set(const std::string& str) { + value = absl::StrSplit(str, absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); +} + +template <> +std::string BaseSetting<Strings>::to_string() { + return concatStringsSep(" ", value); +} + +template <> +void BaseSetting<Strings>::toJSON(JSONPlaceholder& out) { + JSONList list(out.list()); + for (auto& s : value) { + list.elem(s); + } +} + +template <> +void BaseSetting<StringSet>::set(const std::string& str) { + value = absl::StrSplit(str, absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); +} + +template <> +std::string BaseSetting<StringSet>::to_string() { + return concatStringsSep(" ", value); +} + +template <> +void BaseSetting<StringSet>::toJSON(JSONPlaceholder& out) { + JSONList list(out.list()); + for (auto& s : value) { + list.elem(s); + } +} + +template class BaseSetting<int>; +template class BaseSetting<unsigned int>; +template class BaseSetting<long>; +template class BaseSetting<unsigned long>; +template class BaseSetting<long long>; +template class BaseSetting<unsigned long long>; +template class BaseSetting<bool>; +template class BaseSetting<std::string>; +template class BaseSetting<Strings>; +template class BaseSetting<StringSet>; + +void PathSetting::set(const std::string& str) { + if (str.empty()) { + if (allowEmpty) { + value = ""; + } else { + throw UsageError("setting '%s' cannot be empty", name); + } + } else { + value = canonPath(str); + } +} + +bool GlobalConfig::set(const std::string& name, const std::string& value) { + for (auto& config : *configRegistrations) { + if (config->set(name, value)) { + return true; + } + } + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly) { + for (auto& config : *configRegistrations) { + config->getSettings(res, overridenOnly); + } +} + +void GlobalConfig::resetOverriden() { + for (auto& config : *configRegistrations) { + config->resetOverriden(); + } +} + +void GlobalConfig::toJSON(JSONObject& out) { + for (auto& config : *configRegistrations) { + config->toJSON(out); + } +} + +void GlobalConfig::convertToArgs(Args& args, const std::string& category) { + for (auto& config : *configRegistrations) { + config->convertToArgs(args, category); + } +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations* GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config* config) { + if (configRegistrations == nullptr) { + configRegistrations = new ConfigRegistrations; + } + configRegistrations->emplace_back(config); +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/config.hh b/third_party/nix/src/libutil/config.hh new file mode 100644 index 000000000000..81b1c80e0e97 --- /dev/null +++ b/third_party/nix/src/libutil/config.hh @@ -0,0 +1,228 @@ +#include <map> +#include <set> + +#include "libutil/types.hh" + +#pragma once + +namespace nix { + +class Args; +class AbstractSetting; +class JSONPlaceholder; +class JSONObject; + +class AbstractConfig { + protected: + StringMap unknownSettings; + + explicit AbstractConfig(const StringMap& initials = {}) + : unknownSettings(initials) {} + + public: + virtual bool set(const std::string& name, const std::string& value) = 0; + + struct SettingInfo { + std::string value; + std::string description; + }; + + virtual void getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly = false) = 0; + + void applyConfigFile(const Path& path); + + virtual void resetOverriden() = 0; + + virtual void toJSON(JSONObject& out) = 0; + + virtual void convertToArgs(Args& args, const std::string& category) = 0; + + void warnUnknownSettings(); + + void reapplyUnknownSettings(); +}; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting<T> members: + + class MyClass : private Config + { + Setting<int> foo{this, 123, "foo", "the number of foos to use"}; + Setting<std::string> bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config : public AbstractConfig { + friend class AbstractSetting; + + public: + struct SettingData { + bool isAlias; + AbstractSetting* setting; + SettingData(bool isAlias, AbstractSetting* setting) + : isAlias(isAlias), setting(setting) {} + }; + + typedef std::map<std::string, SettingData> Settings; + + private: + Settings _settings; + + public: + explicit Config(const StringMap& initials = {}) : AbstractConfig(initials) {} + + bool set(const std::string& name, const std::string& value) override; + + void addSetting(AbstractSetting* setting); + + void getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject& out) override; + + void convertToArgs(Args& args, const std::string& category) override; +}; + +class AbstractSetting { + friend class Config; + + public: + const std::string name; + const std::string description; + const std::set<std::string> aliases; + + int created = 123; + + bool overriden = false; + + protected: + AbstractSetting(std::string name, std::string description, + std::set<std::string> aliases); + + virtual ~AbstractSetting() { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } + + virtual void set(const std::string& value) = 0; + + virtual std::string to_string() = 0; + + virtual void toJSON(JSONPlaceholder& out); + + virtual void convertToArg(Args& args, const std::string& category); + + bool isOverriden() { return overriden; } +}; + +/* A setting of type T. */ +template <typename T> +class BaseSetting : public AbstractSetting { + protected: + T value; + + public: + BaseSetting(const T& def, const std::string& name, + const std::string& description, + const std::set<std::string>& aliases = {}) + : AbstractSetting(name, description, aliases), value(def) {} + + operator const T&() const { return value; } + operator T&() { return value; } + const T& get() const { return value; } + bool operator==(const T& v2) const { return value == v2; } + bool operator!=(const T& v2) const { return value != v2; } + void operator=(const T& v) { assign(v); } + virtual void assign(const T& v) { value = v; } + + void set(const std::string& str) override; + + virtual void override(const T& v) { + overriden = true; + value = v; + } + + std::string to_string() override; + + void convertToArg(Args& args, const std::string& category) override; + + void toJSON(JSONPlaceholder& out) override; +}; + +template <typename T> +std::ostream& operator<<(std::ostream& str, const BaseSetting<T>& opt) { + str << (const T&)opt; + return str; +} + +template <typename T> +bool operator==(const T& v1, const BaseSetting<T>& v2) { + return v1 == (const T&)v2; +} + +template <typename T> +class Setting : public BaseSetting<T> { + public: + Setting(Config* options, const T& def, const std::string& name, + const std::string& description, + const std::set<std::string>& aliases = {}) + : BaseSetting<T>(def, name, description, aliases) { + options->addSetting(this); + } + + void operator=(const T& v) { this->assign(v); } +}; + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public BaseSetting<Path> { + bool allowEmpty; + + public: + PathSetting(Config* options, bool allowEmpty, const Path& def, + const std::string& name, const std::string& description, + const std::set<std::string>& aliases = {}) + : BaseSetting<Path>(def, name, description, aliases), + allowEmpty(allowEmpty) { + options->addSetting(this); + } + + void set(const std::string& str) override; + + Path operator+(const char* p) const { return value + p; } + + void operator=(const Path& v) { this->assign(v); } +}; + +struct GlobalConfig : public AbstractConfig { + using ConfigRegistrations = std::vector<Config*>; + static ConfigRegistrations* configRegistrations; + + bool set(const std::string& name, const std::string& value) override; + + void getSettings(std::map<std::string, SettingInfo>& res, + bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject& out) override; + + void convertToArgs(Args& args, const std::string& category) override; + + struct Register { + explicit Register(Config* config); + }; +}; + +extern GlobalConfig globalConfig; + +} // namespace nix diff --git a/third_party/nix/src/libutil/finally.hh b/third_party/nix/src/libutil/finally.hh new file mode 100644 index 000000000000..2ead8661a6e7 --- /dev/null +++ b/third_party/nix/src/libutil/finally.hh @@ -0,0 +1,13 @@ +#pragma once + +#include <functional> + +/* A trivial class to run a function at the end of a scope. */ +class Finally { + private: + std::function<void()> fun; + + public: + explicit Finally(std::function<void()> fun) : fun(fun) {} + ~Finally() { fun(); } +}; diff --git a/third_party/nix/src/libutil/hash.cc b/third_party/nix/src/libutil/hash.cc new file mode 100644 index 000000000000..ba61254392e0 --- /dev/null +++ b/third_party/nix/src/libutil/hash.cc @@ -0,0 +1,484 @@ +#include "libutil/hash.hh" + +#include <cstring> +#include <iostream> + +#include <absl/strings/escaping.h> +#include <absl/strings/str_format.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <openssl/md5.h> +#include <openssl/sha.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "libutil/archive.hh" +#include "libutil/istringstream_nocopy.hh" +#include "libutil/util.hh" + +namespace nix { + +std::optional<HashType> hash_type_from(nix::proto::HashType hash_type) { + switch (hash_type) { + case nix::proto::HashType::UNKNOWN: + return HashType::htUnknown; + case nix::proto::HashType::MD5: + return HashType::htMD5; + case nix::proto::HashType::SHA1: + return HashType::htSHA1; + case nix::proto::HashType::SHA256: + return HashType::htSHA256; + case nix::proto::HashType::SHA512: + return HashType::htSHA512; + default: + return {}; + } +} + +nix::proto::HashType HashTypeToProto(HashType hash_type) { + switch (hash_type) { + case HashType::htMD5: + return nix::proto::HashType::MD5; + case HashType::htSHA1: + return nix::proto::HashType::SHA1; + case HashType::htSHA256: + return nix::proto::HashType::SHA256; + case HashType::htSHA512: + return nix::proto::HashType::SHA512; + default: + return nix::proto::HashType::UNKNOWN; + } +} + +void Hash::init() { + if (type == htMD5) { + hashSize = md5HashSize; + } else if (type == htSHA1) { + hashSize = sha1HashSize; + } else if (type == htSHA256) { + hashSize = sha256HashSize; + } else if (type == htSHA512) { + hashSize = sha512HashSize; + } else { + abort(); + } + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); +} + +bool Hash::operator==(const Hash& h2) const { + if (hashSize != h2.hashSize) { + return false; + } + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] != h2.hash[i]) { + return false; + } + } + return true; +} + +bool Hash::operator!=(const Hash& h2) const { return !(*this == h2); } + +bool Hash::operator<(const Hash& h) const { + if (hashSize < h.hashSize) { + return true; + } + if (hashSize > h.hashSize) { + return false; + } + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) { + return true; + } + if (hash[i] > h.hash[i]) { + return false; + } + } + return false; +} + +const std::string base16Chars = "0123456789abcdef"; + +static std::string printHash16(const Hash& hash) { + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return std::string(buf, hash.hashSize * 2); +} + +bool Hash::IsValidBase16(absl::string_view s) { + for (char c : s) { + if ('0' <= c && c <= '9') { + continue; + } + if ('a' <= c && c <= 'f') { + continue; + } + if ('A' <= c && c <= 'F') { + continue; + } + return false; + } + return true; +} + +constexpr signed char kUnBase32[] = { + -1, -1, -1, -1, -1, -1, -1, -1, /* unprintables */ + -1, -1, -1, -1, -1, -1, -1, -1, /* unprintables */ + -1, -1, -1, -1, -1, -1, -1, -1, /* unprintables */ + -1, -1, -1, -1, -1, -1, -1, -1, /* unprintables */ + -1, -1, -1, -1, -1, -1, -1, -1, /* SP..' */ + -1, -1, -1, -1, -1, -1, -1, -1, /* (../ */ + 0, 1, 2, 3, 4, 5, 6, 7, /* 0..7 */ + 8, 9, -1, -1, -1, -1, -1, -1, /* 8..? */ + -1, -1, -1, -1, -1, -1, -1, -1, /* @..G */ + -1, -1, -1, -1, -1, -1, -1, -1, /* H..O */ + -1, -1, -1, -1, -1, -1, -1, -1, /* P..W */ + -1, -1, -1, -1, -1, -1, -1, -1, /* X.._ */ + -1, 10, 11, 12, 13, -1, 14, 15, /* `..g */ + 16, 17, 18, 19, 20, 21, 22, -1, /* h..o */ + 23, 24, 25, 26, -1, -1, 27, 28, /* p..w */ + 29, 30, 31, -1, -1, -1, -1, -1, /* x..DEL */ + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, /* high */ +}; + +bool Hash::IsValidBase32(absl::string_view s) { + static_assert(sizeof(kUnBase32) == 256); + + for (char c : s) { + if (kUnBase32[static_cast<unsigned char>(c)] == -1) { + return false; + } + } + return true; +} + +std::string Hash::ToStorePathHash() const { + return compressHash(*this, kStorePathHashSize).to_string(Base32, false); +} + +static std::string printHash32(const Hash& hash) { + assert(hash.hashSize); + size_t len = hash.base32Len(); + assert(len); + + std::string s; + s.reserve(len); + + for (int n = static_cast<int>(len) - 1; n >= 0; n--) { + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + unsigned char c = + (hash.hash[i] >> j) | + (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); + s.push_back(base32Chars[c & 0x1f]); + } + + return s; +} + +std::string printHash16or32(const Hash& hash) { + return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); +} + +std::string Hash::to_string(Base base, bool includeType) const { + std::string s; + if (base == SRI || includeType) { + s += printHashType(type); + s += base == SRI ? '-' : ':'; + } + switch (base) { + case Base16: + s += printHash16(*this); + break; + case Base32: + s += printHash32(*this); + break; + case Base64: + case SRI: + std::string b64; + absl::Base64Escape( + std::string(reinterpret_cast<const char*>(hash), hashSize), &b64); + s += b64; + break; + } + return s; +} + +Hash::Hash(std::string_view s, HashType type) : type(type) { + absl::StatusOr<Hash> result = deserialize(s, type); + *this = unwrap_throw(result); +} + +// TODO(riking): change ht to an optional +absl::StatusOr<Hash> Hash::deserialize(std::string_view s, HashType type) { + size_t pos = 0; + bool isSRI = false; + + auto sep = s.find(':'); + if (sep == std::string::npos) { + sep = s.find('-'); + if (sep != std::string::npos) { + isSRI = true; + } else if (type == htUnknown) { + return absl::InvalidArgumentError( + absl::StrCat("hash string '", s, " does not include a type")); + } + } + + HashType parsedType = type; + if (sep != std::string::npos) { + std::string hts = std::string(s, 0, sep); + parsedType = parseHashType(hts); + if (type != htUnknown && parsedType != type) { + return absl::InvalidArgumentError( + absl::StrCat("hash '", s, "' should have type '", printHashType(type), + "', found '", printHashType(parsedType), "'")); + } + pos = sep + 1; + } + + Hash dest(parsedType); + + size_t size = s.size() - pos; + absl::string_view sv(s.data() + pos, size); + + if (!isSRI && size == dest.base16Len()) { + std::string bytes; + if (!IsValidBase16(sv)) { + return absl::InvalidArgumentError( + absl::StrCat("invalid base-16 hash: bad character in '", s, "'")); + } + bytes = absl::HexStringToBytes(sv); + if (bytes.size() != dest.hashSize) { + return absl::InvalidArgumentError( + absl::StrCat("hash '", s, "' has wrong length for base16 ", + printHashType(dest.type))); + } + memcpy(dest.hash, bytes.data(), dest.hashSize); + } + + else if (!isSRI && size == dest.base32Len()) { + for (unsigned int n = 0; n < size; ++n) { + char c = sv[size - n - 1]; + // range: -1, 0..31 + signed char digit = kUnBase32[static_cast<unsigned char>(c)]; + if (digit < 0) { + return absl::InvalidArgumentError( + absl::StrCat("invalid base-32 hash: bad character ", + absl::CEscape(absl::string_view(&c, 1)))); + } + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + dest.hash[i] |= digit << j; + + if (i < dest.hashSize - 1) { + dest.hash[i + 1] |= digit >> (8 - j); + } else { + if ((digit >> (8 - j)) != 0) { + return absl::InvalidArgumentError( + absl::StrCat("invalid base-32 hash '", s, "'")); + } + } + } + } + + else if (isSRI || size == dest.base64Len()) { + std::string decoded; + if (!absl::Base64Unescape(sv, &decoded)) { + return absl::InvalidArgumentError("invalid base-64 hash"); + } + if (decoded.size() != dest.hashSize) { + return absl::InvalidArgumentError( + absl::StrCat("hash '", s, "' has wrong length for base64 ", + printHashType(dest.type))); + } + memcpy(dest.hash, decoded.data(), dest.hashSize); + } + + else { + return absl::InvalidArgumentError(absl::StrCat( + "hash '", s, "' has wrong length for ", printHashType(dest.type))); + } + + return dest; +} + +Hash Hash::unwrap_throw(absl::StatusOr<Hash> hash) { + if (hash.ok()) { + return *hash; + } else { + throw BadHash(hash.status().message()); + } +} + +namespace hash { + +union Ctx { + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; + SHA512_CTX sha512; +}; + +static void start(HashType ht, Ctx& ctx) { + if (ht == htMD5) { + MD5_Init(&ctx.md5); + } else if (ht == htSHA1) { + SHA1_Init(&ctx.sha1); + } else if (ht == htSHA256) { + SHA256_Init(&ctx.sha256); + } else if (ht == htSHA512) { + SHA512_Init(&ctx.sha512); + } +} + +static void update(HashType ht, Ctx& ctx, const unsigned char* bytes, + size_t len) { + if (ht == htMD5) { + MD5_Update(&ctx.md5, bytes, len); + } else if (ht == htSHA1) { + SHA1_Update(&ctx.sha1, bytes, len); + } else if (ht == htSHA256) { + SHA256_Update(&ctx.sha256, bytes, len); + } else if (ht == htSHA512) { + SHA512_Update(&ctx.sha512, bytes, len); + } +} + +static void finish(HashType ht, Ctx& ctx, unsigned char* hash) { + if (ht == htMD5) { + MD5_Final(hash, &ctx.md5); + } else if (ht == htSHA1) { + SHA1_Final(hash, &ctx.sha1); + } else if (ht == htSHA256) { + SHA256_Final(hash, &ctx.sha256); + } else if (ht == htSHA512) { + SHA512_Final(hash, &ctx.sha512); + } +} + +} // namespace hash + +Hash hashString(HashType ht, const std::string& s) { + hash::Ctx ctx{}; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, reinterpret_cast<const unsigned char*>(s.data()), s.length()); + finish(ht, ctx, hash.hash); + return hash; +} + +Hash hashFile(HashType ht, const Path& path) { + hash::Ctx ctx{}; + Hash hash(ht); + start(ht, ctx); + + AutoCloseFD fd(open(path.c_str(), O_RDONLY | O_CLOEXEC)); + if (!fd) { + throw SysError(format("opening file '%1%'") % path); + } + + std::vector<unsigned char> buf(8192); + ssize_t n; + while ((n = read(fd.get(), buf.data(), buf.size())) != 0) { + checkInterrupt(); + if (n == -1) { + throw SysError(format("reading file '%1%'") % path); + } + update(ht, ctx, buf.data(), n); + } + + finish(ht, ctx, hash.hash); + return hash; +} + +HashSink::HashSink(HashType ht) + : ht(ht), ctx(std::make_unique<hash::Ctx>()), bytes(0) { + start(ht, *ctx); +} + +HashSink::~HashSink() { bufPos = 0; } + +void HashSink::write(const unsigned char* data, size_t len) { + bytes += len; + nix::hash::update(ht, *ctx, data, len); +} + +HashResult HashSink::finish() { + flush(); + Hash hash(ht); + nix::hash::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); +} + +HashResult HashSink::currentHash() { + flush(); + nix::hash::Ctx ctx2 = *ctx; + Hash hash(ht); + nix::hash::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + +HashResult hashPath(HashType ht, const Path& path, PathFilter& filter) { + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); +} + +Hash compressHash(const Hash& hash, unsigned int newSize) { + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) { + h.hash[i % newSize] ^= hash.hash[i]; + } + return h; +} + +HashType parseHashType(const std::string& s) { + if (s == "md5") { + return htMD5; + } + if (s == "sha1") { + return htSHA1; + } else if (s == "sha256") { + return htSHA256; + } else if (s == "sha512") { + return htSHA512; + } else { + return htUnknown; + } +} + +std::string printHashType(HashType ht) { + if (ht == htMD5) { + return "md5"; + } + if (ht == htSHA1) { + return "sha1"; + } else if (ht == htSHA256) { + return "sha256"; + } else if (ht == htSHA512) { + return "sha512"; + } else if (ht == htUnknown) { + return "<unknown>"; + } else { + LOG(FATAL) << "Unrecognized hash type: " << static_cast<int>(ht); + abort(); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/hash.hh b/third_party/nix/src/libutil/hash.hh new file mode 100644 index 000000000000..8b52ac657e7f --- /dev/null +++ b/third_party/nix/src/libutil/hash.hh @@ -0,0 +1,147 @@ +#pragma once + +#include <absl/status/statusor.h> + +#include "libproto/worker.grpc.pb.h" +#include "libutil/serialise.hh" +#include "libutil/types.hh" + +namespace nix { + +// Size of the hashes rendered in store paths, in bytes +constexpr unsigned int kStorePathHashSize = 20; + +MakeError(BadHash, Error); + +// TODO(grfn): Replace this with the hash type enum from the daemon proto so we +// don't have to juggle two different types +enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; + +std::optional<HashType> hash_type_from(nix::proto::HashType hash_type); + +nix::proto::HashType HashTypeToProto(HashType hash_type); + +const int md5HashSize = 16; +const int sha1HashSize = 20; +const int sha256HashSize = 32; +const int sha512HashSize = 64; + +// omitted: E O U T +constexpr char base32Chars[] = "0123456789abcdfghijklmnpqrsvwxyz"; + +enum Base : int { Base64, Base32, Base16, SRI }; + +struct Hash { + static const unsigned int maxHashSize = 64; + unsigned int hashSize = 0; + unsigned char hash[maxHashSize] = {}; + + HashType type = htUnknown; + + /* Create an unset hash object. */ + Hash(){}; + + /* Create a zero-filled hash object. */ + explicit Hash(HashType type) : type(type) { init(); }; + + /* Initialize the hash from a string representation, in the format + "[<type>:]<base16|base32|base64>" or "<type>-<base64>" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the + string. */ + explicit Hash(std::string_view s, HashType type = htUnknown); + + /* Status-returning version of above constructor */ + static absl::StatusOr<Hash> deserialize(std::string_view s, + HashType type = htUnknown); + + // Legacy unwrapper for StatusOr. Throws BadHash. + static Hash unwrap_throw(absl::StatusOr<Hash> hash) noexcept(false); + + void init(); + + /* Check whether a hash is set. */ + explicit operator bool() const { return type != htUnknown; } + + /* Check whether two hash are equal. */ + bool operator==(const Hash& h2) const; + + /* Check whether two hash are not equal. */ + bool operator!=(const Hash& h2) const; + + /* For sorting. */ + bool operator<(const Hash& h) const; + + /* Returns the length of a base-16 representation of this hash. */ + size_t base16Len() const { return hashSize * 2; } + + /* Returns the length of a base-32 representation of this hash. */ + size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } + + /* Returns the length of a base-64 representation of this hash. */ + size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } + + /* Return a string representation of the hash, in base-16, base-32 + or base-64. By default, this is prefixed by the hash type + (e.g. "sha256:"). */ + std::string to_string(Base base = Base32, bool includeType = true) const; + + /* Returns whether the passed string contains entirely valid base16 + characters. */ + static bool IsValidBase16(absl::string_view s); + + /* Returns whether the passed string contains entirely valid base32 + characters. */ + static bool IsValidBase32(absl::string_view s); + + // Convert this Hash to the format expected in store paths + [[nodiscard]] std::string ToStorePathHash() const; +}; + +/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +std::string printHash16or32(const Hash& hash); + +/* Compute the hash of the given string. */ +Hash hashString(HashType ht, const std::string& s); + +/* Compute the hash of the given file. */ +Hash hashFile(HashType ht, const Path& path); + +/* A pair of the Hash, and the number of bytes consumed. */ +typedef std::pair<Hash, unsigned long long> HashResult; + +/* Compute the hash of the given path. The hash is defined as + (essentially) hashString(ht, dumpPath(path)). */ +HashResult hashPath(HashType ht, const Path& path, + PathFilter& filter = defaultPathFilter); + +/* Compress a hash to the specified number of bytes by cyclically + XORing bytes together. */ +Hash compressHash(const Hash& hash, unsigned int newSize); + +/* Parse a string representing a hash type. */ +HashType parseHashType(const std::string& s); + +/* And the reverse. */ +std::string printHashType(HashType ht); + +namespace hash { +union Ctx; +} + +class HashSink : public BufferedSink { + private: + HashType ht; + std::unique_ptr<hash::Ctx> ctx; + unsigned long long bytes; + + public: + explicit HashSink(HashType ht); + HashSink(const HashSink& h); + ~HashSink(); + void write(const unsigned char* data, size_t len); + HashResult finish(); + HashResult currentHash(); +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/istringstream_nocopy.hh b/third_party/nix/src/libutil/istringstream_nocopy.hh new file mode 100644 index 000000000000..31683d37c91b --- /dev/null +++ b/third_party/nix/src/libutil/istringstream_nocopy.hh @@ -0,0 +1,85 @@ +/* This file provides a variant of std::istringstream that doesn't + copy its string argument. This is useful for large strings. The + caller must ensure that the string object is not destroyed while + it's referenced by this object. */ + +#pragma once + +#include <iostream> +#include <string> + +template <class CharT, class Traits = std::char_traits<CharT>, + class Allocator = std::allocator<CharT>> +class basic_istringbuf_nocopy : public std::basic_streambuf<CharT, Traits> { + public: + using string_type = std::basic_string<CharT, Traits, Allocator>; + + using off_type = typename std::basic_streambuf<CharT, Traits>::off_type; + + using pos_type = typename std::basic_streambuf<CharT, Traits>::pos_type; + + using int_type = typename std::basic_streambuf<CharT, Traits>::int_type; + + using traits_type = typename std::basic_streambuf<CharT, Traits>::traits_type; + + private: + const string_type& s; + + off_type off; + + public: + explicit basic_istringbuf_nocopy(const string_type& s) : s{s}, off{0} {} + + private: + pos_type seekoff(off_type off, std::ios_base::seekdir dir, + std::ios_base::openmode which) { + if (which & std::ios_base::in) { + this->off = + dir == std::ios_base::beg + ? off + : (dir == std::ios_base::end ? s.size() + off : this->off + off); + } + return pos_type(this->off); + } + + pos_type seekpos(pos_type pos, std::ios_base::openmode which) { + return seekoff(pos, std::ios_base::beg, which); + } + + std::streamsize showmanyc() { return s.size() - off; } + + int_type underflow() { + if (typename string_type::size_type(off) == s.size()) { + return traits_type::eof(); + } + return traits_type::to_int_type(s[off]); + } + + int_type uflow() { + if (typename string_type::size_type(off) == s.size()) { + return traits_type::eof(); + } + return traits_type::to_int_type(s[off++]); + } + + int_type pbackfail(int_type ch) { + if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) { + return traits_type::eof(); + } + + return traits_type::to_int_type(s[--off]); + } +}; + +template <class CharT, class Traits = std::char_traits<CharT>, + class Allocator = std::allocator<CharT>> +class basic_istringstream_nocopy : public std::basic_iostream<CharT, Traits> { + using buf_type = basic_istringbuf_nocopy<CharT, Traits, Allocator>; + buf_type buf; + + public: + explicit basic_istringstream_nocopy(const typename buf_type::string_type& s) + : std::basic_iostream<CharT, Traits>(&buf), buf(s){}; +}; + +using istringstream_nocopy = basic_istringstream_nocopy<char>; diff --git a/third_party/nix/src/libutil/json.cc b/third_party/nix/src/libutil/json.cc new file mode 100644 index 000000000000..59ff74f5796f --- /dev/null +++ b/third_party/nix/src/libutil/json.cc @@ -0,0 +1,198 @@ +#include "libutil/json.hh" + +#include <cstring> +#include <iomanip> + +namespace nix { + +void toJSON(std::ostream& str, const char* start, const char* end) { + str << '"'; + for (auto i = start; i != end; i++) { + if (*i == '\"' || *i == '\\') { + str << '\\' << *i; + } else if (*i == '\n') { + str << "\\n"; + } else if (*i == '\r') { + str << "\\r"; + } else if (*i == '\t') { + str << "\\t"; + } else if (*i >= 0 && *i < 32) { + str << "\\u" << std::setfill('0') << std::setw(4) << std::hex + << static_cast<uint16_t>(*i) << std::dec; + } else { + str << *i; + } + } + str << '"'; +} + +void toJSON(std::ostream& str, const char* s) { + if (s == nullptr) { + str << "null"; + } else { + toJSON(str, s, s + strlen(s)); + } +} + +template <> +void toJSON<int>(std::ostream& str, const int& n) { + str << n; +} +template <> +void toJSON<unsigned int>(std::ostream& str, const unsigned int& n) { + str << n; +} +template <> +void toJSON<long>(std::ostream& str, const long& n) { + str << n; +} +template <> +void toJSON<unsigned long>(std::ostream& str, const unsigned long& n) { + str << n; +} +template <> +void toJSON<long long>(std::ostream& str, const long long& n) { + str << n; +} +template <> +void toJSON<unsigned long long>(std::ostream& str, + const unsigned long long& n) { + str << n; +} +template <> +void toJSON<float>(std::ostream& str, const float& n) { + str << n; +} +template <> +void toJSON<double>(std::ostream& str, const double& n) { + str << n; +} + +template <> +void toJSON<std::string>(std::ostream& str, const std::string& s) { + toJSON(str, s.c_str(), s.c_str() + s.size()); +} + +template <> +void toJSON<bool>(std::ostream& str, const bool& b) { + str << (b ? "true" : "false"); +} + +template <> +void toJSON<std::nullptr_t>(std::ostream& str, const std::nullptr_t& b) { + str << "null"; +} + +JSONWriter::JSONWriter(std::ostream& str, bool indent) + : state(new JSONState(str, indent)) { + state->stack++; +} + +JSONWriter::JSONWriter(JSONState* state) : state(state) { state->stack++; } + +JSONWriter::~JSONWriter() { + if (state != nullptr) { + assertActive(); + state->stack--; + if (state->stack == 0) { + delete state; + } + } +} + +void JSONWriter::comma() { + assertActive(); + if (first) { + first = false; + } else { + state->str << ','; + } + if (state->indent) { + indent(); + } +} + +void JSONWriter::indent() { + state->str << '\n' << std::string(state->depth * 2, ' '); +} + +void JSONList::open() { + state->depth++; + state->str << '['; +} + +JSONList::~JSONList() { + state->depth--; + if (state->indent && !first) { + indent(); + } + state->str << "]"; +} + +JSONList JSONList::list() { + comma(); + return JSONList(state); +} + +JSONObject JSONList::object() { + comma(); + return JSONObject(state); +} + +JSONPlaceholder JSONList::placeholder() { + comma(); + return JSONPlaceholder(state); +} + +void JSONObject::open() { + state->depth++; + state->str << '{'; +} + +JSONObject::~JSONObject() { + if (state != nullptr) { + state->depth--; + if (state->indent && !first) { + indent(); + } + state->str << "}"; + } +} + +void JSONObject::attr(const std::string& s) { + comma(); + toJSON(state->str, s); + state->str << ':'; + if (state->indent) { + state->str << ' '; + } +} + +JSONList JSONObject::list(const std::string& name) { + attr(name); + return JSONList(state); +} + +JSONObject JSONObject::object(const std::string& name) { + attr(name); + return JSONObject(state); +} + +JSONPlaceholder JSONObject::placeholder(const std::string& name) { + attr(name); + return JSONPlaceholder(state); +} + +JSONList JSONPlaceholder::list() { + assertValid(); + first = false; + return JSONList(state); +} + +JSONObject JSONPlaceholder::object() { + assertValid(); + first = false; + return JSONObject(state); +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/json.hh b/third_party/nix/src/libutil/json.hh new file mode 100644 index 000000000000..14d61d8a5716 --- /dev/null +++ b/third_party/nix/src/libutil/json.hh @@ -0,0 +1,144 @@ +#pragma once + +#include <cassert> +#include <iostream> +#include <vector> + +namespace nix { + +void toJSON(std::ostream& str, const char* start, const char* end); +void toJSON(std::ostream& str, const char* s); + +template <typename T> +void toJSON(std::ostream& str, const T& n); + +class JSONWriter { + protected: + struct JSONState { + std::ostream& str; + bool indent; + size_t depth = 0; + size_t stack = 0; + JSONState(std::ostream& str, bool indent) : str(str), indent(indent) {} + ~JSONState() { assert(stack == 0); } + }; + + JSONState* state; + + bool first = true; + + JSONWriter(std::ostream& str, bool indent); + + explicit JSONWriter(JSONState* state); + + ~JSONWriter(); + + void assertActive() { assert(state->stack != 0); } + + void comma(); + + void indent(); +}; + +class JSONObject; +class JSONPlaceholder; + +class JSONList : JSONWriter { + private: + friend class JSONObject; + friend class JSONPlaceholder; + + void open(); + + explicit JSONList(JSONState* state) : JSONWriter(state) { open(); } + + public: + explicit JSONList(std::ostream& str, bool indent = false) + : JSONWriter(str, indent) { + open(); + } + + ~JSONList(); + + template <typename T> + JSONList& elem(const T& v) { + comma(); + toJSON(state->str, v); + return *this; + } + + JSONList list(); + + JSONObject object(); + + JSONPlaceholder placeholder(); +}; + +class JSONObject : JSONWriter { + private: + friend class JSONList; + friend class JSONPlaceholder; + + void open(); + + explicit JSONObject(JSONState* state) : JSONWriter(state) { open(); } + + void attr(const std::string& s); + + public: + explicit JSONObject(std::ostream& str, bool indent = false) + : JSONWriter(str, indent) { + open(); + } + + JSONObject(const JSONObject& obj) = delete; + + JSONObject(JSONObject&& obj) : JSONWriter(obj.state) { obj.state = 0; } + + ~JSONObject(); + + template <typename T> + JSONObject& attr(const std::string& name, const T& v) { + attr(name); + toJSON(state->str, v); + return *this; + } + + JSONList list(const std::string& name); + + JSONObject object(const std::string& name); + + JSONPlaceholder placeholder(const std::string& name); +}; + +class JSONPlaceholder : JSONWriter { + private: + friend class JSONList; + friend class JSONObject; + + explicit JSONPlaceholder(JSONState* state) : JSONWriter(state) {} + + void assertValid() { + assertActive(); + assert(first); + } + + public: + explicit JSONPlaceholder(std::ostream& str, bool indent = false) + : JSONWriter(str, indent) {} + + ~JSONPlaceholder() { assert(!first || std::uncaught_exception()); } + + template <typename T> + void write(const T& v) { + assertValid(); + first = false; + toJSON(state->str, v); + } + + JSONList list(); + + JSONObject object(); +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/lazy.hh b/third_party/nix/src/libutil/lazy.hh new file mode 100644 index 000000000000..5c6ff5d8df6a --- /dev/null +++ b/third_party/nix/src/libutil/lazy.hh @@ -0,0 +1,45 @@ +#include <exception> +#include <functional> +#include <mutex> + +namespace nix { + +/* A helper class for lazily-initialized variables. + + Lazy<T> var([]() { return value; }); + + declares a variable of type T that is initialized to 'value' (in a + thread-safe way) on first use, that is, when var() is first + called. If the initialiser code throws an exception, then all + subsequent calls to var() will rethrow that exception. */ +template <typename T> +class Lazy { + typedef std::function<T()> Init; + + Init init; + + std::once_flag done; + + T value; + + std::exception_ptr ex; + + public: + explicit Lazy(Init init) : init(init) {} + + const T& operator()() { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) { + std::rethrow_exception(ex); + } + return value; + } +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/lru-cache.hh b/third_party/nix/src/libutil/lru-cache.hh new file mode 100644 index 000000000000..1832c542449e --- /dev/null +++ b/third_party/nix/src/libutil/lru-cache.hh @@ -0,0 +1,90 @@ +#pragma once + +#include <list> +#include <map> +#include <optional> + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template <typename Key, typename Value> +class LRUCache { + private: + size_t capacity; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map<Key, std::pair<LRUIterator, Value>>; + using LRU = std::list<typename Data::iterator>; + + struct LRUIterator { + typename LRU::iterator it; + }; + + Data data; + LRU lru; + + public: + explicit LRUCache(size_t capacity) : capacity(capacity) {} + + /* Insert or upsert an item in the cache. */ + void upsert(const Key& key, const Value& value) { + if (capacity == 0) { + return; + } + + erase(key); + + if (data.size() >= capacity) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto& i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key& key) { + auto i = data.find(key); + if (i == data.end()) { + return false; + } + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + std::optional<Value> get(const Key& key) { + auto i = data.find(key); + if (i == data.end()) { + return {}; + } + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return i->second.second; + } + + size_t size() { return data.size(); } + + void clear() { + data.clear(); + lru.clear(); + } +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/monitor-fd.hh b/third_party/nix/src/libutil/monitor-fd.hh new file mode 100644 index 000000000000..c818c5826116 --- /dev/null +++ b/third_party/nix/src/libutil/monitor-fd.hh @@ -0,0 +1,57 @@ +#pragma once + +#include <atomic> +#include <cstdlib> +#include <thread> + +#include <poll.h> +#include <signal.h> +#include <sys/types.h> +#include <unistd.h> + +namespace nix { + +class MonitorFdHup { + private: + std::thread thread; + + public: + MonitorFdHup(int fd) { + thread = std::thread([fd]() { + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) { + abort(); + } // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) { + continue; + } + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } + }); + }; + + ~MonitorFdHup() { + pthread_cancel(thread.native_handle()); + thread.join(); + } +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/pool.hh b/third_party/nix/src/libutil/pool.hh new file mode 100644 index 000000000000..b5c3c4b5c43f --- /dev/null +++ b/third_party/nix/src/libutil/pool.hh @@ -0,0 +1,176 @@ +#pragma once + +#include <cassert> +#include <functional> +#include <limits> +#include <list> +#include <memory> + +#include "libutil/ref.hh" +#include "libutil/sync.hh" + +namespace nix { + +/* This template class implements a simple pool manager of resources + of some type R, such as database connections. It is used as + follows: + + class Connection { ... }; + + Pool<Connection> pool; + + { + auto conn(pool.get()); + conn->exec("select ..."); + } + + Here, the Connection object referenced by ‘conn’ is automatically + returned to the pool when ‘conn’ goes out of scope. +*/ + +template <class R> +class Pool { + public: + /* A function that produces new instances of R on demand. */ + typedef std::function<ref<R>()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + using Validator = std::function<bool(const ref<R>&)>; + + private: + Factory factory; + Validator validator; + + struct State { + size_t inUse = 0; + size_t max; + std::vector<ref<R>> idle; + }; + + Sync<State> state; + + std::condition_variable wakeup; + + public: + explicit Pool( + size_t max = std::numeric_limits<size_t>::max(), + const Factory& factory = []() { return make_ref<R>(); }, + const Validator& validator = [](ref<R> r) { return true; }) + : factory(factory), validator(validator) { + auto state_(state.lock()); + state_->max = max; + } + + void incCapacity() { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() { + auto state_(state.lock()); + state_->max--; + } + + ~Pool() { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle { + private: + Pool& pool; + std::shared_ptr<R> r; + bool bad = false; + + friend Pool; + + Handle(Pool& pool, std::shared_ptr<R> r) : pool(pool), r(r) {} + + public: + Handle(Handle&& h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle& l) = delete; + + ~Handle() { + if (!r) { + return; + } + { + auto state_(pool.state.lock()); + if (!bad) { + state_->idle.push_back(ref<R>(r)); + } + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); + } + + R* operator->() { return &*r; } + R& operator*() { return *r; } + + void markBad() { bad = true; } + }; + + Handle get() { + { + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) { + state_.wait(wakeup); + } + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); + } + } + + state_->inUse++; + } + + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + wakeup.notify_one(); + throw; + } + } + + size_t count() { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } + + size_t capacity() { return state.lock()->max; } + + void flushBad() { + auto state_(state.lock()); + std::vector<ref<R>> left; + for (auto& p : state_->idle) { + if (validator(p)) { + left.push_back(p); + } + } + std::swap(state_->idle, left); + } +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/proto.hh b/third_party/nix/src/libutil/proto.hh new file mode 100644 index 000000000000..058cb7b7b452 --- /dev/null +++ b/third_party/nix/src/libutil/proto.hh @@ -0,0 +1,174 @@ +#pragma once + +#include <absl/status/status.h> +#include <grpcpp/impl/codegen/status.h> +#include <grpcpp/impl/codegen/status_code_enum.h> + +#include "libproto/worker.pb.h" +#include "libutil/types.hh" + +namespace nix::util::proto { + +inline ::nix::proto::StorePath StorePath(const Path& path) { + ::nix::proto::StorePath store_path; + store_path.set_path(path); + return store_path; +} + +inline ::nix::proto::StorePaths StorePaths(const PathSet& paths) { + ::nix::proto::StorePaths result; + for (const auto& path : paths) { + result.add_paths(path); + } + return result; +} + +template <typename T, typename U> +T FillFrom(const U& src) { + T result; + result.insert(src.begin(), src.end()); + return result; +} + +constexpr absl::StatusCode GRPCStatusCodeToAbsl(grpc::StatusCode code) { + switch (code) { + case grpc::StatusCode::OK: + return absl::StatusCode::kOk; + case grpc::StatusCode::CANCELLED: + return absl::StatusCode::kCancelled; + case grpc::StatusCode::UNKNOWN: + return absl::StatusCode::kUnknown; + case grpc::StatusCode::INVALID_ARGUMENT: + return absl::StatusCode::kInvalidArgument; + case grpc::StatusCode::DEADLINE_EXCEEDED: + return absl::StatusCode::kDeadlineExceeded; + case grpc::StatusCode::NOT_FOUND: + return absl::StatusCode::kNotFound; + case grpc::StatusCode::ALREADY_EXISTS: + return absl::StatusCode::kAlreadyExists; + case grpc::StatusCode::PERMISSION_DENIED: + return absl::StatusCode::kPermissionDenied; + case grpc::StatusCode::UNAUTHENTICATED: + return absl::StatusCode::kUnauthenticated; + case grpc::StatusCode::RESOURCE_EXHAUSTED: + return absl::StatusCode::kResourceExhausted; + case grpc::StatusCode::FAILED_PRECONDITION: + return absl::StatusCode::kFailedPrecondition; + case grpc::StatusCode::ABORTED: + return absl::StatusCode::kAborted; + case grpc::StatusCode::OUT_OF_RANGE: + return absl::StatusCode::kOutOfRange; + case grpc::StatusCode::UNIMPLEMENTED: + return absl::StatusCode::kUnimplemented; + case grpc::StatusCode::INTERNAL: + return absl::StatusCode::kInternal; + case grpc::StatusCode::UNAVAILABLE: + return absl::StatusCode::kUnavailable; + case grpc::StatusCode::DATA_LOSS: + return absl::StatusCode::kDataLoss; + default: + return absl::StatusCode::kInternal; + } +} + +constexpr grpc::StatusCode AbslStatusCodeToGRPC(absl::StatusCode code) { + switch (code) { + case absl::StatusCode::kOk: + return grpc::StatusCode::OK; + case absl::StatusCode::kCancelled: + return grpc::StatusCode::CANCELLED; + case absl::StatusCode::kUnknown: + return grpc::StatusCode::UNKNOWN; + case absl::StatusCode::kInvalidArgument: + return grpc::StatusCode::INVALID_ARGUMENT; + case absl::StatusCode::kDeadlineExceeded: + return grpc::StatusCode::DEADLINE_EXCEEDED; + case absl::StatusCode::kNotFound: + return grpc::StatusCode::NOT_FOUND; + case absl::StatusCode::kAlreadyExists: + return grpc::StatusCode::ALREADY_EXISTS; + case absl::StatusCode::kPermissionDenied: + return grpc::StatusCode::PERMISSION_DENIED; + case absl::StatusCode::kUnauthenticated: + return grpc::StatusCode::UNAUTHENTICATED; + case absl::StatusCode::kResourceExhausted: + return grpc::StatusCode::RESOURCE_EXHAUSTED; + case absl::StatusCode::kFailedPrecondition: + return grpc::StatusCode::FAILED_PRECONDITION; + case absl::StatusCode::kAborted: + return grpc::StatusCode::ABORTED; + case absl::StatusCode::kOutOfRange: + return grpc::StatusCode::OUT_OF_RANGE; + case absl::StatusCode::kUnimplemented: + return grpc::StatusCode::UNIMPLEMENTED; + case absl::StatusCode::kInternal: + return grpc::StatusCode::INTERNAL; + case absl::StatusCode::kUnavailable: + return grpc::StatusCode::UNAVAILABLE; + case absl::StatusCode::kDataLoss: + return grpc::StatusCode::DATA_LOSS; + default: + return grpc::StatusCode::INTERNAL; + } +} + +constexpr absl::string_view GRPCStatusCodeDescription(grpc::StatusCode code) { + switch (code) { + case grpc::StatusCode::OK: + return "OK"; + case grpc::StatusCode::CANCELLED: + return "CANCELLED"; + case grpc::StatusCode::UNKNOWN: + return "UNKNOWN"; + case grpc::StatusCode::INVALID_ARGUMENT: + return "INVALID_ARGUMENT"; + case grpc::StatusCode::DEADLINE_EXCEEDED: + return "DEADLINE_EXCEEDED"; + case grpc::StatusCode::NOT_FOUND: + return "NOT_FOUND"; + case grpc::StatusCode::ALREADY_EXISTS: + return "ALREADY_EXISTS"; + case grpc::StatusCode::PERMISSION_DENIED: + return "PERMISSION_DENIED"; + case grpc::StatusCode::UNAUTHENTICATED: + return "UNAUTHENTICATED"; + case grpc::StatusCode::RESOURCE_EXHAUSTED: + return "RESOURCE_EXHAUSTED"; + case grpc::StatusCode::FAILED_PRECONDITION: + return "FAILED_PRECONDITION"; + case grpc::StatusCode::ABORTED: + return "ABORTED"; + case grpc::StatusCode::OUT_OF_RANGE: + return "OUT_OF_RANGE"; + case grpc::StatusCode::UNIMPLEMENTED: + return "UNIMPLEMENTED"; + case grpc::StatusCode::INTERNAL: + return "INTERNAL"; + case grpc::StatusCode::UNAVAILABLE: + return "UNAVAILABLE"; + case grpc::StatusCode::DATA_LOSS: + return "DATA_LOSS"; + default: + return "<BAD ERROR CODE>"; + }; +} + +inline absl::Status GRPCStatusToAbsl(grpc::Status status) { + if (status.ok()) { + return absl::OkStatus(); + } + + return absl::Status(GRPCStatusCodeToAbsl(status.error_code()), + status.error_message()); +} + +inline grpc::Status AbslToGRPCStatus(absl::Status status) { + if (status.ok()) { + return grpc::Status::OK; + } + + return grpc::Status(AbslStatusCodeToGRPC(status.code()), + std::string(status.message())); +} + +} // namespace nix::util::proto diff --git a/third_party/nix/src/libutil/ref.hh b/third_party/nix/src/libutil/ref.hh new file mode 100644 index 000000000000..3c375491fd0b --- /dev/null +++ b/third_party/nix/src/libutil/ref.hh @@ -0,0 +1,65 @@ +#pragma once + +#include <exception> +#include <memory> +#include <stdexcept> + +namespace nix { + +/* A simple non-nullable reference-counted pointer. Actually a wrapper + around std::shared_ptr that prevents non-null constructions. */ +template <typename T> +class ref { // TODO(tazjin): rename to brainworm_ref or something + private: + std::shared_ptr<T> p; + + public: + ref<T>(const ref<T>& r) : p(r.p) {} + + explicit ref<T>(const std::shared_ptr<T>& p) : p(p) { + if (!p) { + throw std::invalid_argument("null pointer cast to ref"); + } + } + + explicit ref<T>(T* p) : p(p) { + if (!p) { + throw std::invalid_argument("null pointer cast to ref"); + } + } + + T* operator->() const { return &*p; } + + T& operator*() const { return *p; } + + operator std::shared_ptr<T>() const { return p; } + + std::shared_ptr<T> get_ptr() const { return p; } + + template <typename T2> + ref<T2> cast() const { + return ref<T2>(std::dynamic_pointer_cast<T2>(p)); + } + + template <typename T2> + std::shared_ptr<T2> dynamic_pointer_cast() const { + return std::dynamic_pointer_cast<T2>(p); + } + + template <typename T2> + operator ref<T2>() const { + return ref<T2>((std::shared_ptr<T2>)p); + } + + private: + template <typename T2, typename... Args> + friend ref<T2> make_ref(Args&&... args); +}; + +template <typename T, typename... Args> +inline ref<T> make_ref(Args&&... args) { + auto p = std::make_shared<T>(std::forward<Args>(args)...); + return ref<T>(p); +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/serialise.cc b/third_party/nix/src/libutil/serialise.cc new file mode 100644 index 000000000000..288255089bb6 --- /dev/null +++ b/third_party/nix/src/libutil/serialise.cc @@ -0,0 +1,311 @@ +#include "libutil/serialise.hh" + +#include <boost/coroutine2/coroutine.hpp> +#include <cerrno> +#include <cstring> +#include <memory> +#include <utility> + +#include <glog/logging.h> + +#include "libutil/util.hh" + +namespace nix { + +void BufferedSink::operator()(const unsigned char* data, size_t len) { + if (!buffer) { + buffer = decltype(buffer)(new unsigned char[bufSize]); + } + + while (len != 0u) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer.get() + bufPos, data, n); + data += n; + bufPos += n; + len -= n; + if (bufPos == bufSize) { + flush(); + } + } +} + +void BufferedSink::flush() { + if (bufPos == 0) { + return; + } + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer.get(), n); +} + +FdSink::~FdSink() { + try { + flush(); + } catch (...) { + ignoreException(); + } +} + +size_t threshold = 256 * 1024 * 1024; + +static void warnLargeDump() { + LOG(WARNING) + << "dumping very large path (> 256 MiB); this may run out of memory"; +} + +void FdSink::write(const unsigned char* data, size_t len) { + written += len; + static bool warned = false; + if (warn && !warned) { + if (written > threshold) { + warnLargeDump(); + warned = true; + } + } + try { + writeFull(fd, data, len); + } catch (SysError& e) { + _good = false; + throw; + } +} + +bool FdSink::good() { return _good; } + +void Source::operator()(unsigned char* data, size_t len) { + while (len != 0u) { + size_t n = read(data, len); + data += n; + len -= n; + } +} + +std::string Source::drain() { + std::string s; + std::vector<unsigned char> buf(8192); + while (true) { + size_t n; + try { + n = read(buf.data(), buf.size()); + s.append(reinterpret_cast<char*>(buf.data()), n); + } catch (EndOfFile&) { + break; + } + } + return s; +} + +size_t BufferedSource::read(unsigned char* data, size_t len) { + if (!buffer) { + buffer = decltype(buffer)(new unsigned char[bufSize]); + } + + if (bufPosIn == 0u) { + bufPosIn = readUnbuffered(buffer.get(), bufSize); + } + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer.get() + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) { + bufPosIn = bufPosOut = 0; + } + return n; +} + +bool BufferedSource::hasData() { return bufPosOut < bufPosIn; } + +size_t FdSource::readUnbuffered(unsigned char* data, size_t len) { + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, reinterpret_cast<char*>(data), len); + } while (n == -1 && errno == EINTR); + if (n == -1) { + _good = false; + throw SysError("reading from file"); + } + if (n == 0) { + _good = false; + throw EndOfFile("unexpected end-of-file"); + } + read += n; + return n; +} + +bool FdSource::good() { return _good; } + +size_t StringSource::read(unsigned char* data, size_t len) { + if (pos == s.size()) { + throw EndOfFile("end of string reached"); + } + size_t n = s.copy(reinterpret_cast<char*>(data), len, pos); + pos += n; + return n; +} + +#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 +#error Coroutines are broken in this version of Boost! +#endif + +std::unique_ptr<Source> sinkToSource(const std::function<void(Sink&)>& fun, + const std::function<void()>& eof) { + struct SinkToSource : Source { + using coro_t = boost::coroutines2::coroutine<std::string>; + + std::function<void(Sink&)> fun; + std::function<void()> eof; + std::optional<coro_t::pull_type> coro; + bool started = false; + + SinkToSource(std::function<void(Sink&)> fun, std::function<void()> eof) + : fun(std::move(fun)), eof(std::move(eof)) {} + + std::string cur; + size_t pos = 0; + + size_t read(unsigned char* data, size_t len) override { + if (!coro) { + coro = coro_t::pull_type([&](coro_t::push_type& yield) { + LambdaSink sink([&](const unsigned char* data, size_t len) { + if (len != 0u) { + yield(std::string(reinterpret_cast<const char*>(data), len)); + } + }); + fun(sink); + }); + } + + if (!*coro) { + eof(); + abort(); + } + + if (pos == cur.size()) { + if (!cur.empty()) { + (*coro)(); + } + cur = coro->get(); + pos = 0; + } + + auto n = std::min(cur.size() - pos, len); + memcpy(data, reinterpret_cast<unsigned char*>(cur.data()) + pos, n); + pos += n; + + return n; + } + }; + + return std::make_unique<SinkToSource>(fun, eof); +} + +void writePadding(size_t len, Sink& sink) { + if ((len % 8) != 0u) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } +} + +void writeString(const unsigned char* buf, size_t len, Sink& sink) { + sink << len; + sink(buf, len); + writePadding(len, sink); +} + +Sink& operator<<(Sink& sink, const std::string& s) { + writeString(reinterpret_cast<const unsigned char*>(s.data()), s.size(), sink); + return sink; +} + +template <class T> +void writeStrings(const T& ss, Sink& sink) { + sink << ss.size(); + for (auto& i : ss) { + sink << i; + } +} + +Sink& operator<<(Sink& sink, const Strings& s) { + writeStrings(s, sink); + return sink; +} + +Sink& operator<<(Sink& sink, const StringSet& s) { + writeStrings(s, sink); + return sink; +} + +void readPadding(size_t len, Source& source) { + if ((len % 8) != 0u) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) { + if (zero[i] != 0u) { + throw SerialisationError("non-zero padding"); + } + } + } +} + +size_t readString(unsigned char* buf, size_t max, Source& source) { + auto len = readNum<size_t>(source); + if (len > max) { + throw SerialisationError("string is too long"); + } + source(buf, len); + readPadding(len, source); + return len; +} + +std::string readString(Source& source, size_t max) { + auto len = readNum<size_t>(source); + if (len > max) { + throw SerialisationError("string is too long"); + } + std::string res(len, 0); + source(reinterpret_cast<unsigned char*>(res.data()), len); + readPadding(len, source); + return res; +} + +Source& operator>>(Source& in, std::string& s) { + s = readString(in); + return in; +} + +template <class T> +T readStrings(Source& source) { + auto count = readNum<size_t>(source); + T ss; + while (count--) { + ss.insert(ss.end(), readString(source)); + } + return ss; +} + +template Paths readStrings(Source& source); +template PathSet readStrings(Source& source); + +void StringSink::operator()(const unsigned char* data, size_t len) { + static bool warned = false; + if (!warned && s->size() > threshold) { + warnLargeDump(); + warned = true; + } + s->append(reinterpret_cast<const char*>(data), len); +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/serialise.hh b/third_party/nix/src/libutil/serialise.hh new file mode 100644 index 000000000000..c6d1d814dbdd --- /dev/null +++ b/third_party/nix/src/libutil/serialise.hh @@ -0,0 +1,289 @@ +#pragma once + +#include <memory> + +#include "libutil/types.hh" +#include "libutil/util.hh" + +namespace nix { + +/* Abstract destination of binary data. */ +struct Sink { + virtual ~Sink() {} + virtual void operator()(const unsigned char* data, size_t len) = 0; + virtual bool good() { return true; } + + void operator()(const std::string& s) { + (*this)((const unsigned char*)s.data(), s.size()); + } +}; + +/* A buffered abstract sink. */ +struct BufferedSink : Sink { + size_t bufSize, bufPos; + std::unique_ptr<unsigned char[]> buffer; + + explicit BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(nullptr) {} + + void operator()(const unsigned char* data, size_t len) override; + + void operator()(const std::string& s) { Sink::operator()(s); } + + void flush(); + + virtual void write(const unsigned char* data, size_t len) = 0; +}; + +/* Abstract source of binary data. */ +struct Source { + virtual ~Source() {} + + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator()(unsigned char* data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. It blocks until at least + one byte is available. */ + virtual size_t read(unsigned char* data, size_t len) = 0; + + virtual bool good() { return true; } + + std::string drain(); +}; + +/* A buffered abstract source. */ +struct BufferedSource : Source { + size_t bufSize, bufPosIn, bufPosOut; + std::unique_ptr<unsigned char[]> buffer; + + explicit BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) {} + + size_t read(unsigned char* data, size_t len) override; + + bool hasData(); + + protected: + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char* data, size_t len) = 0; +}; + +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink { + int fd; + bool warn = false; + size_t written = 0; + + FdSink() : fd(-1) {} + explicit FdSink(int fd) : fd(fd) {} + FdSink(FdSink&&) = default; + + FdSink& operator=(FdSink&& s) { + flush(); + fd = s.fd; + s.fd = -1; + warn = s.warn; + written = s.written; + return *this; + } + + ~FdSink(); + + void write(const unsigned char* data, size_t len) override; + + bool good() override; + + private: + bool _good = true; +}; + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource { + int fd; + size_t read = 0; + + FdSource() : fd(-1) {} + explicit FdSource(int fd) : fd(fd) {} + FdSource(FdSource&&) = default; + + FdSource& operator=(FdSource&& s) { + fd = s.fd; + s.fd = -1; + read = s.read; + return *this; + } + + bool good() override; + + protected: + size_t readUnbuffered(unsigned char* data, size_t len) override; + + private: + bool _good = true; +}; + +/* A sink that writes data to a string. */ +struct StringSink : Sink { + ref<std::string> s; + StringSink() : s(make_ref<std::string>()){}; + explicit StringSink(ref<std::string> s) : s(s){}; + void operator()(const unsigned char* data, size_t len) override; +}; + +/* A source that reads data from a string. */ +struct StringSource : Source { + const std::string& s; + size_t pos; + explicit StringSource(const std::string& _s) : s(_s), pos(0) {} + size_t read(unsigned char* data, size_t len) override; +}; + +/* Adapter class of a Source that saves all data read to `s'. */ +struct TeeSource : Source { + Source& orig; + ref<std::string> data; + explicit TeeSource(Source& orig) + : orig(orig), data(make_ref<std::string>()) {} + size_t read(unsigned char* data, size_t len) { + size_t n = orig.read(data, len); + this->data->append((const char*)data, n); + return n; + } +}; + +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source { + Source& orig; + size_t remain; + SizedSource(Source& orig, size_t size) : orig(orig), remain(size) {} + size_t read(unsigned char* data, size_t len) { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() { + std::vector<unsigned char> buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; + +/* Convert a function into a sink. */ +struct LambdaSink : Sink { + typedef std::function<void(const unsigned char*, size_t)> lambda_t; + + lambda_t lambda; + + explicit LambdaSink(const lambda_t& lambda) : lambda(lambda) {} + + virtual void operator()(const unsigned char* data, size_t len) { + lambda(data, len); + } +}; + +/* Convert a function into a source. */ +struct LambdaSource : Source { + using lambda_t = std::function<size_t(unsigned char*, size_t)>; + + lambda_t lambda; + + explicit LambdaSource(const lambda_t& lambda) : lambda(lambda) {} + + size_t read(unsigned char* data, size_t len) override { + return lambda(data, len); + } +}; + +/* Convert a function that feeds data into a Sink into a Source. The + Source executes the function as a coroutine. */ +std::unique_ptr<Source> sinkToSource( + const std::function<void(Sink&)>& fun, + const std::function<void()>& eof = []() { + throw EndOfFile("coroutine has finished"); + }); + +void writePadding(size_t len, Sink& sink); +void writeString(const unsigned char* buf, size_t len, Sink& sink); + +inline Sink& operator<<(Sink& sink, uint64_t n) { + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (unsigned char)(n >> 56) & 0xff; + sink(buf, sizeof(buf)); + return sink; +} + +Sink& operator<<(Sink& sink, const std::string& s); +Sink& operator<<(Sink& sink, const Strings& s); +Sink& operator<<(Sink& sink, const StringSet& s); + +MakeError(SerialisationError, Error); + +template <typename T> +T readNum(Source& source) { + unsigned char buf[8]; + source(buf, sizeof(buf)); + + uint64_t n = + ((unsigned long long)buf[0]) | ((unsigned long long)buf[1] << 8) | + ((unsigned long long)buf[2] << 16) | ((unsigned long long)buf[3] << 24) | + ((unsigned long long)buf[4] << 32) | ((unsigned long long)buf[5] << 40) | + ((unsigned long long)buf[6] << 48) | ((unsigned long long)buf[7] << 56); + + if (n > std::numeric_limits<T>::max()) { + throw SerialisationError("serialised integer %d is too large for type '%s'", + n, typeid(T).name()); + } + + return (T)n; +} + +inline unsigned int readInt(Source& source) { + return readNum<unsigned int>(source); +} + +inline uint64_t readLongLong(Source& source) { + return readNum<uint64_t>(source); +} + +void readPadding(size_t len, Source& source); +size_t readString(unsigned char* buf, size_t max, Source& source); +std::string readString(Source& source, + size_t max = std::numeric_limits<size_t>::max()); +template <class T> +T readStrings(Source& source); + +Source& operator>>(Source& in, std::string& s); + +template <typename T> +Source& operator>>(Source& in, T& n) { + n = readNum<T>(in); + return in; +} + +template <typename T> +Source& operator>>(Source& in, bool& b) { + b = readNum<uint64_t>(in); + return in; +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/status.hh b/third_party/nix/src/libutil/status.hh new file mode 100644 index 000000000000..aeee0f502257 --- /dev/null +++ b/third_party/nix/src/libutil/status.hh @@ -0,0 +1,17 @@ +#pragma once + +#include <absl/status/status.h> +#include <absl/strings/str_format.h> +#include <absl/strings/string_view.h> + +#include "libutil/types.hh" + +namespace nix::util { + +inline void OkOrThrow(absl::Status status) { + if (!status.ok()) { + throw Error(absl::StrFormat("Operation failed: %s", status.ToString())); + } +} + +} // namespace nix::util diff --git a/third_party/nix/src/libutil/sync.hh b/third_party/nix/src/libutil/sync.hh new file mode 100644 index 000000000000..ef640d5b56ef --- /dev/null +++ b/third_party/nix/src/libutil/sync.hh @@ -0,0 +1,84 @@ +#pragma once + +#include <cassert> +#include <condition_variable> +#include <cstdlib> +#include <mutex> + +namespace nix { + +/* This template class ensures synchronized access to a value of type + T. It is used as follows: + + struct Data { int x; ... }; + + Sync<Data> data; + + { + auto data_(data.lock()); + data_->x = 123; + } + + Here, "data" is automatically unlocked when "data_" goes out of + scope. +*/ + +template <class T, class M = std::mutex> +class Sync { + private: + M mutex; + T data; + + public: + Sync() {} + explicit Sync(const T& data) : data(data) {} + explicit Sync(T&& data) noexcept : data(std::move(data)) {} + + class Lock { + private: + Sync* s; + std::unique_lock<M> lk; + friend Sync; + explicit Lock(Sync* s) : s(s), lk(s->mutex) {} + + public: + Lock(Lock&& l) : s(l.s) { abort(); } + Lock(const Lock& l) = delete; + ~Lock() {} + T* operator->() { return &s->data; } + T& operator*() { return s->data; } + + void wait(std::condition_variable& cv) { + assert(s); + cv.wait(lk); + } + + template <class Rep, class Period> + std::cv_status wait_for( + std::condition_variable& cv, + const std::chrono::duration<Rep, Period>& duration) { + assert(s); + return cv.wait_for(lk, duration); + } + + template <class Rep, class Period, class Predicate> + bool wait_for(std::condition_variable& cv, + const std::chrono::duration<Rep, Period>& duration, + Predicate pred) { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template <class Clock, class Duration> + std::cv_status wait_until( + std::condition_variable& cv, + const std::chrono::time_point<Clock, Duration>& duration) { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } +}; + +} // namespace nix diff --git a/third_party/nix/src/libutil/thread-pool.cc b/third_party/nix/src/libutil/thread-pool.cc new file mode 100644 index 000000000000..7c6b0a1b469b --- /dev/null +++ b/third_party/nix/src/libutil/thread-pool.cc @@ -0,0 +1,163 @@ +#include "libutil/thread-pool.hh" + +#include <glog/logging.h> + +#include "libutil/affinity.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _maxThreads) : maxThreads(_maxThreads) { + restoreAffinity(); // FIXME + + if (maxThreads == 0u) { + maxThreads = std::thread::hardware_concurrency(); + if (maxThreads == 0u) { + maxThreads = 1; + } + } + + DLOG(INFO) << "starting pool of " << maxThreads - 1 << " threads"; +} + +ThreadPool::~ThreadPool() { shutdown(); } + +void ThreadPool::shutdown() { + std::vector<std::thread> workers; + { + auto state(state_.lock()); + quit = true; + std::swap(workers, state->workers); + } + + if (workers.empty()) { + return; + } + + DLOG(INFO) << "reaping " << workers.size() << " worker threads"; + + work.notify_all(); + + for (auto& thr : workers) { + thr.join(); + } +} + +void ThreadPool::enqueue(const work_t& t) { + auto state(state_.lock()); + if (quit) { + throw ThreadPoolShutDown( + "cannot enqueue a work item while the thread pool is shutting down"); + } + state->pending.push(t); + /* Note: process() also executes items, so count it as a worker. */ + if (state->pending.size() > state->workers.size() + 1 && + state->workers.size() + 1 < maxThreads) { + state->workers.emplace_back(&ThreadPool::doWork, this, false); + } + work.notify_one(); +} + +void ThreadPool::process() { + state_.lock()->draining = true; + + /* Do work until no more work is pending or active. */ + try { + doWork(true); + + auto state(state_.lock()); + + assert(quit); + + if (state->exception) { + std::rethrow_exception(state->exception); + } + + } catch (...) { + /* In the exceptional case, some workers may still be + active. They may be referencing the stack frame of the + caller. So wait for them to finish. (~ThreadPool also does + this, but it might be destroyed after objects referenced by + the work item lambdas.) */ + shutdown(); + throw; + } +} + +void ThreadPool::doWork(bool mainThread) { + if (!mainThread) { + interruptCheck = [&]() { return (bool)quit; }; + } + + bool didWork = false; + std::exception_ptr exc; + + while (true) { + work_t w; + { + auto state(state_.lock()); + + if (didWork) { + assert(state->active); + state->active--; + + if (exc) { + if (!state->exception) { + state->exception = exc; + // Tell the other workers to quit. + quit = true; + work.notify_all(); + } else { + /* Print the exception, since we can't + propagate it. */ + try { + std::rethrow_exception(exc); + } catch (std::exception& e) { + if ((dynamic_cast<Interrupted*>(&e) == nullptr) && + (dynamic_cast<ThreadPoolShutDown*>(&e) == nullptr)) { + ignoreException(); + } + } catch (...) { + } + } + } + } + + /* Wait until a work item is available or we're asked to + quit. */ + while (true) { + if (quit) { + return; + } + + if (!state->pending.empty()) { + break; + } + + /* If there are no active or pending items, and the + main thread is running process(), then no new items + can be added. So exit. */ + if ((state->active == 0u) && state->draining) { + quit = true; + work.notify_all(); + return; + } + + state.wait(work); + } + + w = std::move(state->pending.front()); + state->pending.pop(); + state->active++; + } + + try { + w(); + } catch (...) { + exc = std::current_exception(); + } + + didWork = true; + } +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/thread-pool.hh b/third_party/nix/src/libutil/thread-pool.hh new file mode 100644 index 000000000000..0efc4c1bfc67 --- /dev/null +++ b/third_party/nix/src/libutil/thread-pool.hh @@ -0,0 +1,140 @@ +#pragma once + +#include <atomic> +#include <functional> +#include <map> +#include <queue> +#include <thread> + +#include "libutil/sync.hh" +#include "libutil/util.hh" + +namespace nix { + +MakeError(ThreadPoolShutDown, Error); + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool { + public: + explicit ThreadPool(size_t maxThreads = 0); + + ~ThreadPool(); + + // FIXME: use std::packaged_task? + typedef std::function<void()> work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t& t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + + private: + size_t maxThreads; + + struct State { + std::queue<work_t> pending; + size_t active = 0; + std::exception_ptr exception; + std::vector<std::thread> workers; + bool draining = false; + }; + + std::atomic_bool quit{false}; + + Sync<State> state_; + + std::condition_variable work; + + void doWork(bool mainThread); + + void shutdown(); +}; + +/* Process in parallel a set of items of type T that have a partial + ordering between them. Thus, any item is only processed after all + its dependencies have been processed. */ +template <typename T> +void processGraph(ThreadPool& pool, const std::set<T>& nodes, + std::function<std::set<T>(const T&)> getEdges, + std::function<void(const T&)> processNode) { + struct Graph { + std::set<T> left; + std::map<T, std::set<T>> refs, rrefs; + }; + + Sync<Graph> graph_(Graph{nodes, {}, {}}); + + std::function<void(const T&)> worker; + + worker = [&](const T& node) { + { + auto graph(graph_.lock()); + auto i = graph->refs.find(node); + if (i == graph->refs.end()) { + goto getRefs; + } + goto doWork; + } + + getRefs : { + auto refs = getEdges(node); + refs.erase(node); + + { + auto graph(graph_.lock()); + for (auto& ref : refs) { + if (graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + } + if (graph->refs[node].empty()) { + goto doWork; + } + } + } + + return; + + doWork: + processNode(node); + + /* Enqueue work for all nodes that were waiting on this one + and have no unprocessed dependencies. */ + { + auto graph(graph_.lock()); + for (auto& rref : graph->rrefs[node]) { + auto& refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) { + pool.enqueue(std::bind(worker, rref)); + } + } + graph->left.erase(node); + graph->refs.erase(node); + graph->rrefs.erase(node); + } + }; + + for (auto& node : nodes) { + pool.enqueue(std::bind(worker, std::ref(node))); + } + + pool.process(); + + if (!graph_.lock()->left.empty()) { + throw Error("graph processing incomplete (cyclic reference?)"); + } +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/types.hh b/third_party/nix/src/libutil/types.hh new file mode 100644 index 000000000000..bf95206d0824 --- /dev/null +++ b/third_party/nix/src/libutil/types.hh @@ -0,0 +1,118 @@ +#pragma once + +#include <boost/format.hpp> +#include <list> +#include <map> +#include <memory> +#include <set> +#include <string> + +#include "libutil/ref.hh" + +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + +namespace nix { + +/* Inherit some names from other namespaces for convenience. */ +using boost::format; + +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { + template <typename... T> + explicit nop(T...) {} +}; + +struct FormatOrString { + std::string s; + FormatOrString(const std::string& s) : s(s){}; + FormatOrString(const format& f) : s(f.str()){}; + FormatOrString(const char* s) : s(s){}; +}; + +/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + equivalent to ‘boost::format(format) % a_0 % ... % + ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + takes place). */ + +inline std::string fmt(const std::string& s) { return s; } + +inline std::string fmt(std::string_view s) { return std::string(s); } + +inline std::string fmt(const char* s) { return s; } + +inline std::string fmt(const FormatOrString& fs) { return fs.s; } + +template <typename... Args> +inline std::string fmt(const std::string& fs, Args... args) { + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + nop{boost::io::detail::feed(f, args)...}; + return f.str(); +} + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception { + protected: + std::string prefix_; // used for location traces etc. + std::string err; + + public: + unsigned int status = 1; // exit status + + template <typename... Args> + explicit BaseError(unsigned int status, Args... args) + : err(fmt(args...)), status(status) {} + + template <typename... Args> + explicit BaseError(Args... args) : err(fmt(args...)) {} + +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~BaseError() noexcept {}; + const char* what() const noexcept { return err.c_str(); } +#else + const char* what() const noexcept { return err.c_str(); } +#endif + + const std::string& msg() const { return err; } + const std::string& prefix() const { return prefix_; } + BaseError& addPrefix(const FormatOrString& fs); +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass { \ + public: \ + using superClass::superClass; \ + }; + +MakeError(Error, BaseError); + +class SysError : public Error { + public: + int errNo; + + template <typename... Args> + explicit SysError(Args... args) : Error(addErrno(fmt(args...))) {} + + private: + std::string addErrno(const std::string& s); +}; + +typedef std::list<std::string> Strings; +using StringSet = std::set<std::string>; +using StringMap = std::map<std::string, std::string>; + +/* Paths are just strings. */ +using Path = std::string; +using Paths = std::list<Path>; +using PathSet = std::set<Path>; + +} // namespace nix diff --git a/third_party/nix/src/libutil/util.cc b/third_party/nix/src/libutil/util.cc new file mode 100644 index 000000000000..aea1e68e3c77 --- /dev/null +++ b/third_party/nix/src/libutil/util.cc @@ -0,0 +1,1426 @@ +#include "libutil/util.hh" + +#include <cctype> +#include <cerrno> +#include <climits> +#include <cstdio> +#include <cstdlib> +#include <cstring> +#include <future> +#include <iostream> +#include <sstream> +#include <thread> +#include <utility> + +#include <absl/strings/str_split.h> +#include <absl/strings/string_view.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <grp.h> +#include <pwd.h> +#include <sys/ioctl.h> +#include <sys/prctl.h> +#include <sys/types.h> +#include <sys/wait.h> +#include <unistd.h> + +#include "libutil/affinity.hh" +#include "libutil/finally.hh" +#include "libutil/lazy.hh" +#include "libutil/serialise.hh" +#include "libutil/sync.hh" +#include "nix_config.h" + +namespace nix { + +const std::string nativeSystem = SYSTEM; + +BaseError& BaseError::addPrefix(const FormatOrString& fs) { + prefix_ = fs.s + prefix_; + return *this; +} + +std::string SysError::addErrno(const std::string& s) { + errNo = errno; + return s + ": " + strerror(errNo); +} + +std::optional<std::string> getEnv(const std::string& key) { + char* value = getenv(key.c_str()); + if (value == nullptr) return {}; + return std::string(value); +} + +std::map<std::string, std::string> getEnv() { + std::map<std::string, std::string> env; + for (size_t i = 0; environ[i] != nullptr; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (eq == nullptr) { + // invalid env, just keep going + continue; + } + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + +void clearEnv() { + for (auto& name : getEnv()) { + unsetenv(name.first.c_str()); + } +} + +void replaceEnv(const std::map<std::string, std::string>& newEnv) { + clearEnv(); + for (const auto& newEnvVar : newEnv) { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + +Path absPath(Path path, Path dir) { + if (path[0] != '/') { + if (dir.empty()) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char* buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (getcwd(buf, sizeof(buf)) == nullptr) { +#endif + throw SysError("cannot get cwd"); + } + dir = buf; +#ifdef __GNU__ + free(buf); +#endif + } + path = dir + "/" + path; +} +return canonPath(path); +} // namespace nix + +Path canonPath(const Path& path, bool resolveSymlinks) { + assert(!path.empty()); + + std::string s; + + if (path[0] != '/') { + throw Error(format("not an absolute path: '%1%'") % path); + } + + std::string::const_iterator i = path.begin(); + std::string::const_iterator end = path.end(); + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0; + unsigned int maxFollow = 1024; + + while (true) { + /* Skip slashes. */ + while (i != end && *i == '/') { + i++; + } + if (i == end) { + break; + } + + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) { + i++; + } + + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) { + if (!s.empty()) { + s.erase(s.rfind('/')); + } + i += 2; + } + + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') { + s += *i++; + } + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) { + throw Error(format("infinite symlink recursion in path '%1%'") % + path); + } + temp = absPath(readLink(s), dirOf(s)) + std::string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + } + } + } + + return s.empty() ? "/" : s; +} + +// TODO(grfn) remove in favor of std::filesystem::path::parent_path() +Path dirOf(absl::string_view path) { + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) { + return "."; + } + return pos == 0 ? "/" : Path(path, 0, pos); +} + +// TODO(grfn) remove in favor of std::filesystem::path::root_name() +std::string baseNameOf(const Path& path) { + if (path.empty()) { + return ""; + } + + Path::size_type last = path.length() - 1; + if (path[last] == '/' && last > 0) { + last -= 1; + } + + Path::size_type pos = path.rfind('/', last); + if (pos == std::string::npos) { + pos = 0; + } else { + pos += 1; + } + + return std::string(path, pos, last - pos + 1); +} + +bool isInDir(const Path& path, const Path& dir) { + return path[0] == '/' && std::string(path, 0, dir.size()) == dir && + path.size() >= dir.size() + 2 && path[dir.size()] == '/'; +} + +bool isDirOrInDir(const Path& path, const Path& dir) { + return path == dir || isInDir(path, dir); +} + +struct stat lstat(const Path& path) { + struct stat st; + if (lstat(path.c_str(), &st) != 0) { + throw SysError(format("getting status of '%1%'") % path); + } + return st; +} + +bool pathExists(const Path& path) { + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (res == 0) { + return true; + } + if (errno != ENOENT && errno != ENOTDIR) { + throw SysError(format("getting status of %1%") % path); + } + return false; +} + +Path readLink(const Path& path) { + checkInterrupt(); + std::vector<char> buf; + for (ssize_t bufSize = PATH_MAX / 4; true; bufSize += bufSize / 2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) { + if (errno == EINVAL) { + throw Error("'%1%' is not a symlink", path); + } + throw SysError("reading symbolic link '%1%'", path); + + } else if (rlSize < bufSize) { + return std::string(buf.data(), rlSize); + } + } +} + +bool isLink(const Path& path) { + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + +DirEntries readDirectory(DIR* dir, const Path& path) { + DirEntries entries; + entries.reserve(64); + + struct dirent* dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") { + continue; + } + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) { + throw SysError(format("reading directory '%1%'") % path); + } + + return entries; +} + +DirEntries readDirectory(const Path& path) { + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) { + throw SysError(format("opening directory '%1%'") % path); + } + + return readDirectory(dir.get(), path); +} + +unsigned char getFileType(const Path& path) { + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) { + return DT_DIR; + } + if (S_ISLNK(st.st_mode)) { + return DT_LNK; + } + if (S_ISREG(st.st_mode)) { + return DT_REG; + } + return DT_UNKNOWN; +} + +std::string readFile(int fd) { + struct stat st; + if (fstat(fd, &st) == -1) { + throw SysError("statting file"); + } + + std::vector<unsigned char> buf(st.st_size); + readFull(fd, buf.data(), st.st_size); + + return std::string(reinterpret_cast<char*>(buf.data()), st.st_size); +} + +std::string readFile(absl::string_view path, bool drain) { + AutoCloseFD fd(open(std::string(path).c_str(), O_RDONLY | O_CLOEXEC)); + if (!fd) { + throw SysError(format("opening file '%1%'") % path); + } + return drain ? drainFD(fd.get()) : readFile(fd.get()); +} + +void readFile(absl::string_view path, Sink& sink) { + // TODO(tazjin): use stdlib functions for this stuff + AutoCloseFD fd(open(std::string(path).c_str(), O_RDONLY | O_CLOEXEC)); + if (!fd) { + throw SysError("opening file '%s'", path); + } + drainFD(fd.get(), sink); +} + +void writeFile(const Path& path, const std::string& s, mode_t mode) { + AutoCloseFD fd( + open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)); + if (!fd) { + throw SysError(format("opening file '%1%'") % path); + } + writeFull(fd.get(), s); +} + +void writeFile(const Path& path, Source& source, mode_t mode) { + AutoCloseFD fd( + open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode)); + if (!fd) { + throw SysError(format("opening file '%1%'") % path); + } + + std::vector<unsigned char> buf(64 * 1024); + + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), static_cast<unsigned char*>(buf.data()), n); + } catch (EndOfFile&) { + break; + } + } +} + +std::string readLine(int fd) { + std::string s; + while (true) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) { + throw SysError("reading a line"); + } + } else if (rd == 0) { + throw EndOfFile("unexpected EOF reading a line"); + } else { + if (ch == '\n') { + return s; + } + s += ch; + } + } +} + +void writeLine(int fd, std::string s) { + s += '\n'; + writeFull(fd, s); +} + +static void _deletePath(int parentfd, const Path& path, + unsigned long long& bytesFreed) { + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) { + return; + } + throw SysError(format("getting status of '%1%'") % path); + } + + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) { + bytesFreed += st.st_size; + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) { + throw SysError(format("chmod '%1%'") % path); + } + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (!fd) { + throw SysError(format("opening directory '%1%'") % path); + } + AutoCloseDir dir(fdopendir(fd)); + if (!dir) { + throw SysError(format("opening directory '%1%'") % path); + } + for (auto& i : readDirectory(dir.get(), path)) { + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) { + return; + } + throw SysError(format("cannot unlink '%1%'") % path); + } +} + +static void _deletePath(const Path& path, unsigned long long& bytesFreed) { + Path dir = dirOf(path); + if (dir == "") { + dir = "/"; + } + + AutoCloseFD dirfd(open(dir.c_str(), O_RDONLY)); + if (!dirfd) { + // This really shouldn't fail silently, but it's left this way + // for backwards compatibility. + if (errno == ENOENT) { + return; + } + + throw SysError(format("opening directory '%1%'") % path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + +void deletePath(const Path& path) { + unsigned long long dummy; + deletePath(path, dummy); +} + +void deletePath(const Path& path, unsigned long long& bytesFreed) { + // Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % + // path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + +static Path tempName(Path tmpRoot, const Path& prefix, bool includePid, + int& counter) { + tmpRoot = canonPath( + tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + + if (includePid) { + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++) + .str(); + } + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + +Path createTempDir(const Path& tmpRoot, const Path& prefix, bool includePid, + bool useGlobalCounter, mode_t mode) { + static int globalCounter = 0; + int localCounter = 0; + int& counter(useGlobalCounter ? globalCounter : localCounter); + + while (true) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t)-1, getegid()) != 0) + throw SysError(format("setting group of directory '%1%'") % tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) { + throw SysError(format("creating directory '%1%'") % tmpDir); + } + } +} + +std::string getUserName() { + auto pw = getpwuid(geteuid()); + std::optional<std::string> name = + pw != nullptr ? pw->pw_name : getEnv("USER"); + if (!name.has_value()) { + throw Error("cannot figure out user name"); + } + return *name; +} + +static Lazy<Path> getHome2([]() { + std::optional<Path> homeDir = getEnv("HOME"); + if (!homeDir) { + std::vector<char> buf(16384); + struct passwd pwbuf; + struct passwd* pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 || + (pw == nullptr) || (pw->pw_dir == nullptr) || (pw->pw_dir[0] == 0)) { + throw Error("cannot determine user's home directory"); + } + return std::string(pw->pw_dir); + } + return homeDir.value(); +}); + +Path getHome() { return getHome2(); } + +Path getCacheDir() { + return getEnv("XDG_CACHE_HOME").value_or(getHome() + "/.cache"); +} + +Path getConfigDir() { + return getEnv("XDG_CONFIG_HOME").value_or(getHome() + "/.config"); +} + +std::vector<Path> getConfigDirs() { + Path configHome = getConfigDir(); + std::string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + std::vector<std::string> result = + absl::StrSplit(configDirs, absl::ByChar(':'), absl::SkipEmpty()); + result.insert(result.begin(), configHome); + return result; +} + +Path getDataDir() { + return getEnv("XDG_DATA_HOME").value_or(getHome() + "/.local/share"); +} + +// TODO(grfn): Remove in favor of std::filesystem::create_directories +Paths createDirs(const Path& path) { + Paths created; + if (path == "/") { + return created; + } + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) { + throw SysError(format("creating directory '%1%'") % path); + } + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) { + throw SysError(format("statting symlink '%1%'") % path); + } + + if (!S_ISDIR(st.st_mode)) { + throw Error(format("'%1%' is not a directory") % path); + } + + return created; +} + +void createSymlink(const Path& target, const Path& link) { + if (symlink(target.c_str(), link.c_str()) != 0) { + throw SysError(format("creating symlink from '%1%' to '%2%'") % link % + target); + } +} + +void replaceSymlink(const Path& target, const Path& link) { + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError& e) { + if (e.errNo == EEXIST) { + continue; + } + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) { + throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); + } + + break; + } +} + +void readFull(int fd, unsigned char* buf, size_t count) { + while (count != 0u) { + checkInterrupt(); + ssize_t res = read(fd, reinterpret_cast<char*>(buf), count); + if (res == -1) { + if (errno == EINTR) { + continue; + } + throw SysError("reading from file"); + } + if (res == 0) { + throw EndOfFile("unexpected end-of-file"); + } + count -= res; + buf += res; + } +} + +void writeFull(int fd, const unsigned char* buf, size_t count, + bool allowInterrupts) { + while (count != 0u) { + if (allowInterrupts) { + checkInterrupt(); + } + ssize_t res = write(fd, (char*)buf, count); + if (res == -1 && errno != EINTR) { + throw SysError("writing to file"); + } + if (res > 0) { + count -= res; + buf += res; + } + } +} + +void writeFull(int fd, const std::string& s, bool allowInterrupts) { + writeFull(fd, reinterpret_cast<const unsigned char*>(s.data()), s.size(), + allowInterrupts); +} + +std::string drainFD(int fd, bool block) { + StringSink sink; + drainFD(fd, sink, block); + return std::move(*sink.s); +} + +void drainFD(int fd, Sink& sink, bool block) { + int saved; + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) { + throw SysError("making file descriptor blocking"); + } + } + }); + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) { + throw SysError("making file descriptor non-blocking"); + } + } + + std::vector<unsigned char> buf(64 * 1024); + while (true) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) { + break; + } + if (errno != EINTR) { + throw SysError("reading from file"); + } + } else if (rd == 0) { + break; + } else { + sink(buf.data(), rd); + } + } +} + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(std::string p, bool recursive) : path(std::move(p)) { + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() { + try { + if (del) { + if (recursive) { + deletePath(path); + } else { + if (remove(path.c_str()) == -1) { + throw SysError(format("cannot unlink '%1%'") % path); + } + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() { del = false; } + +void AutoDelete::reset(const Path& p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + +AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} { that.fd = -1; } + +AutoCloseFD& AutoCloseFD::operator=(AutoCloseFD&& that) { + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + +AutoCloseFD::~AutoCloseFD() { + try { + close(); + } catch (...) { + ignoreException(); + } +} + +int AutoCloseFD::get() const { return fd; } + +void AutoCloseFD::close() { + if (fd != -1) { + if (::close(fd) == -1) { /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + } + } +} + +AutoCloseFD::operator bool() const { return fd != -1; } + +int AutoCloseFD::release() { + int oldFD = fd; + fd = -1; + return oldFD; +} + +void Pipe::create() { + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) { + throw SysError("creating pipe"); + } +#else + if (pipe(fds) != 0) { + throw SysError("creating pipe"); + } + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = AutoCloseFD(fds[0]); + writeSide = AutoCloseFD(fds[1]); +} + +////////////////////////////////////////////////////////////////////// + +Pid::Pid() = default; + +Pid::Pid(pid_t pid) : pid(pid) {} + +Pid::~Pid() { + if (pid != -1) { + kill(); + } +} + +void Pid::operator=(pid_t pid) { + if (this->pid != -1 && this->pid != pid) { + kill(); + } + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + +Pid::operator pid_t() { return pid; } + +int Pid::kill() { + assert(pid != -1); + + DLOG(INFO) << "killing process " << pid; + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + LOG(ERROR) << SysError("killing process %d", pid).msg(); + } + + return wait(); +} + +int Pid::wait() { + assert(pid != -1); + while (true) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) { + throw SysError("cannot get child exit status"); + } + checkInterrupt(); + } +} + +void Pid::setSeparatePG(bool separatePG) { this->separatePG = separatePG; } + +void Pid::setKillSignal(int signal) { this->killSignal = signal; } + +pid_t Pid::release() { + pid_t p = pid; + pid = -1; + return p; +} + +void killUser(uid_t uid) { + DLOG(INFO) << "killing all processes running under UID " << uid; + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + ProcessOptions options; + + Pid pid(startProcess( + [&]() { + if (setuid(uid) == -1) { + throw SysError("setting uid"); + } + + while (true) { + if (kill(-1, SIGKILL) == 0) { + break; + } + if (errno == ESRCH) { + break; + } /* no more processes */ + if (errno != EINTR) { + throw SysError(format("cannot kill processes for uid '%1%'") % uid); + } + } + + _exit(0); + }, + options)); + + int status = pid.wait(); + if (status != 0) { + throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % + statusToString(status)); + } + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + +////////////////////////////////////////////////////////////////////// + +/* + * Please note that it is not legal for this function to call vfork(). If the + * process created by vfork() returns from the function in which vfork() was + * called, or calls any other function before successfully calling _exit() or + * one of the exec*() family of functions, the behavior is undefined. + */ +static pid_t doFork(const std::function<void()>& fun) __attribute__((noinline)); +static pid_t doFork(const std::function<void()>& fun) { +#ifdef __linux__ + // TODO(kanepyork): call clone() instead for faster forking +#endif + + pid_t pid = fork(); + if (pid != 0) { + return pid; + } + fun(); + abort(); +} + +pid_t startProcess(std::function<void()> fun, const ProcessOptions& options) { + auto wrapper = [&]() { + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) { + throw SysError("setting death signal"); + } +#endif + restoreAffinity(); + fun(); + } catch (std::exception& e) { + try { + LOG(ERROR) << options.errorPrefix << e.what(); + } catch (...) { + } + } catch (...) { + } + if (options.runExitHandlers) { + exit(1); + } else { + _exit(1); + } + }; + + pid_t pid = doFork(wrapper); + if (pid == -1) { + throw SysError("unable to fork"); + } + + return pid; +} + +std::vector<char*> stringsToCharPtrs(const Strings& ss) { + std::vector<char*> res; + for (auto& s : ss) { + res.push_back(const_cast<char*>(s.c_str())); + } + res.push_back(nullptr); + return res; +} + +std::string runProgram(const Path& program, bool searchPath, + const Strings& args, + const std::optional<std::string>& input) { + RunOptions opts(program, args); + opts.searchPath = searchPath; + opts.input = input; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) { + throw ExecError(res.first, fmt("program '%1%' %2%", program, + statusToString(res.first))); + } + + return res.second; +} + +std::pair<int, std::string> runProgram(const RunOptions& options_) { + RunOptions options(options_); + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError& e) { + status = e.status; + } + + return {status, std::move(*sink.s)}; +} + +void runProgram2(const RunOptions& options) { + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr<Source> source_; + Source* source = options.standardIn; + + if (options.input) { + source_ = std::make_unique<StringSource>(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out; + Pipe in; + if (options.standardOut != nullptr) { + out.create(); + } + if (source != nullptr) { + in.create(); + } + + ProcessOptions processOptions; + + /* Fork. */ + Pid pid(startProcess( + [&]() { + if (options.environment) { + replaceEnv(*options.environment); + } + if ((options.standardOut != nullptr) && + dup2(out.writeSide.get(), STDOUT_FILENO) == -1) { + throw SysError("dupping stdout"); + } + if (options.mergeStderrToStdout) { + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) { + throw SysError("cannot dup stdout into stderr"); + } + } + if ((source != nullptr) && + dup2(in.readSide.get(), STDIN_FILENO) == -1) { + throw SysError("dupping stdin"); + } + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) { + throw SysError("chdir failed"); + } + if (options.gid && setgid(*options.gid) == -1) { + throw SysError("setgid failed"); + } + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, nullptr) == -1) { + throw SysError("setgroups failed"); + } + if (options.uid && setuid(*options.uid) == -1) { + throw SysError("setuid failed"); + } + + Strings args_(options.args); + args_.push_front(options.program); + + restoreSignals(); + + if (options.searchPath) { + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + } else { + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + } + + throw SysError("executing '%1%'", options.program); + }, + processOptions)); + + out.writeSide = AutoCloseFD(-1); + + std::thread writerThread; + + std::promise<void> promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) { + writerThread.join(); + } + }); + + if (source != nullptr) { + in.readSide = AutoCloseFD(-1); + writerThread = std::thread([&]() { + try { + std::vector<unsigned char> buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile&) { + break; + } + writeFull(in.writeSide.get(), buf.data(), n); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide = AutoCloseFD(-1); + }); + } + + if (options.standardOut != nullptr) { + drainFD(out.readSide.get(), *options.standardOut); + } + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source != nullptr) { + promise.get_future().get(); + } + + if (status != 0) { + throw ExecError(status, fmt("program '%1%' %2%", options.program, + statusToString(status))); + } +} + +void closeMostFDs(const std::set<int>& exceptions) { +#if __linux__ + try { + for (auto& s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (exceptions.count(fd) == 0u) { + DLOG(INFO) << "closing leaked FD " << fd; + close(fd); + } + } + return; + } catch (SysError&) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) { + if (exceptions.count(fd) == 0u) { + close(fd); + } /* ignore result */ + } +} + +void closeOnExec(int fd) { + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) { + throw SysError("setting close-on-exec flag"); + } +} + +////////////////////////////////////////////////////////////////////// + +bool _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function<bool()> interruptCheck; + +void setInterruptThrown() { interruptThrown = true; } + +void _interrupted() { + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && (std::uncaught_exceptions() == 0)) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + +////////////////////////////////////////////////////////////////////// + +std::string concatStringsSep(const std::string& sep, const Strings& ss) { + std::string s; + for (auto& i : ss) { + if (!s.empty()) { + s += sep; + } + s += i; + } + return s; +} + +std::string concatStringsSep(const std::string& sep, const StringSet& ss) { + std::string s; + for (auto& i : ss) { + if (!s.empty()) { + s += sep; + } + s += i; + } + return s; +} + +std::string replaceStrings(const std::string& s, const std::string& from, + const std::string& to) { + if (from.empty()) { + return s; + } + std::string res = s; + size_t pos = 0; + while ((pos = res.find(from, pos)) != std::string::npos) { + res.replace(pos, from.size(), to); + pos += to.size(); + } + return res; +} + +std::string statusToString(int status) { + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) { + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + } + if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char* description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description) + .str(); +#else + return (format("failed due to signal %1%") % sig).str(); +#endif + } else { + return "died abnormally"; + } + } else { + return "succeeded"; + } +} + +bool statusOk(int status) { + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +std::string toLower(const std::string& s) { + std::string r(s); + for (auto& c : r) { + c = std::tolower(c); + } + return r; +} + +std::string shellEscape(const std::string& s) { + std::string r = "'"; + for (auto& i : s) { + if (i == '\'') { + r += "'\\''"; + } else { + r += i; + } + } + r += '\''; + return r; +} + +void ignoreException() { + try { + throw; + } catch (std::exception& e) { + LOG(ERROR) << "error (ignored): " << e.what(); + } +} + +std::string filterANSIEscapes(const std::string& s, bool filterAll, + unsigned int width) { + std::string t; + std::string e; + size_t w = 0; + auto i = s.begin(); + + while (w < static_cast<size_t>(width) && i != s.end()) { + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) { + e += *i++; + } + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) { + e += *i++; + } + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) { + e += last = *i++; + } + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) { + e += *i++; + } + } + + if (!filterAll && last == 'm') { + t += e; + } + } + + else if (*i == '\t') { + i++; + t += ' '; + w++; + while (w < static_cast<size_t>(width) && ((w % 8) != 0u)) { + t += ' '; + w++; + } + } + + else if (*i == '\r') { + // do nothing for now + i++; + + } else { + t += *i++; + w++; + } + } + + return t; +} + +void callFailure(const std::function<void(std::exception_ptr exc)>& failure, + const std::exception_ptr& exc) { + try { + failure(exc); + } catch (std::exception& e) { + LOG(ERROR) << "uncaught exception: " << e.what(); + abort(); + } +} + +static Sync<std::pair<unsigned short, unsigned short>> windowSize{{0, 0}}; + +static void updateWindowSize() { + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + +std::pair<unsigned short, unsigned short> getWindowSize() { + return *windowSize.lock(); +} + +static Sync<std::list<std::function<void()>>> _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) { + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) { + triggerInterrupt(); + + } else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() { + _isInterrupted = true; + + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto& callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + +static sigset_t savedSignalMask; + +void startSignalHandlerThread() { + updateWindowSize(); + + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask) != 0) { + throw SysError("quering signal mask"); + } + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr) != 0) { + throw SysError("blocking signals"); + } + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr) != 0) { + throw SysError("restoring signals"); + } +} + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback { + std::list<std::function<void()>>::iterator it; + ~InterruptCallbackImpl() override { _interruptCallbacks.lock()->erase(it); } +}; + +std::unique_ptr<InterruptCallback> createInterruptCallback( + const std::function<void()>& callback) { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->push_back(callback); + + auto res = std::make_unique<InterruptCallbackImpl>(); + res->it = interruptCallbacks->end(); + res->it--; + + return std::unique_ptr<InterruptCallback>(res.release()); +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/util.hh b/third_party/nix/src/libutil/util.hh new file mode 100644 index 000000000000..b3349c4f39c4 --- /dev/null +++ b/third_party/nix/src/libutil/util.hh @@ -0,0 +1,476 @@ +#pragma once + +#include <cstdio> +#include <functional> +#include <future> +#include <limits> +#include <map> +#include <optional> +#include <sstream> + +#include <absl/strings/string_view.h> +#include <dirent.h> +#include <signal.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libutil/types.hh" + +namespace nix { + +struct Sink; +struct Source; + +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + +/* Return an environment variable. */ +std::optional<std::string> getEnv(const std::string& key); + +/* Get the entire environment. */ +std::map<std::string, std::string> getEnv(); + +/* Clear the environment. */ +void clearEnv(); + +/* Return an absolutized path, resolving paths relative to the + specified directory, or the current directory otherwise. The path + is also canonicalised. */ +Path absPath(Path path, Path dir = ""); + +/* Canonicalise a path by removing all `.' or `..' components and + double or trailing slashes. Optionally resolves all symlink + components such that each component of the resulting path is *not* + a symbolic link. */ +Path canonPath(const Path& path, bool resolveSymlinks = false); + +/* Return the directory part of the given canonical path, i.e., + everything before the final `/'. If the path is the root or an + immediate child thereof (e.g., `/foo'), this means an empty string + is returned. */ +Path dirOf(absl::string_view path); + +/* Return the base name of the given canonical path, i.e., everything + following the final `/'. */ +std::string baseNameOf(const Path& path); + +/* Check whether 'path' is a descendant of 'dir'. */ +bool isInDir(const Path& path, const Path& dir); + +/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ +bool isDirOrInDir(const Path& path, const Path& dir); + +/* Get status of `path'. */ +struct stat lstat(const Path& path); + +/* Return true iff the given path exists. */ +bool pathExists(const Path& path); + +/* Read the contents (target) of a symbolic link. The result is not + in any way canonicalised. */ +Path readLink(const Path& path); + +bool isLink(const Path& path); + +/* Read the contents of a directory. The entries `.' and `..' are + removed. */ +struct DirEntry { + std::string name; + ino_t ino; + unsigned char type; // one of DT_* + DirEntry(const std::string& name, ino_t ino, unsigned char type) + : name(name), ino(ino), type(type) {} +}; + +typedef std::vector<DirEntry> DirEntries; + +DirEntries readDirectory(const Path& path); + +unsigned char getFileType(const Path& path); + +/* Read the contents of a file into a string. */ +std::string readFile(int fd); +std::string readFile(absl::string_view path, bool drain = false); +void readFile(absl::string_view path, Sink& sink); + +/* Write a string to a file. */ +void writeFile(const Path& path, const std::string& s, mode_t mode = 0666); + +void writeFile(const Path& path, Source& source, mode_t mode = 0666); + +/* Read a line from a file descriptor. */ +std::string readLine(int fd); + +/* Write a line to a file descriptor. */ +void writeLine(int fd, std::string s); + +/* Delete a path; i.e., in the case of a directory, it is deleted + recursively. It's not an error if the path does not exist. The + second variant returns the number of bytes and blocks freed. */ +void deletePath(const Path& path); + +void deletePath(const Path& path, unsigned long long& bytesFreed); + +/* Create a temporary directory. */ +Path createTempDir(const Path& tmpRoot = "", const Path& prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, + mode_t mode = 0755); + +std::string getUserName(); + +/* Return $HOME or the user's home directory from /etc/passwd. */ +Path getHome(); + +/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +Path getCacheDir(); + +/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +Path getConfigDir(); + +/* Return the directories to search for user configuration files */ +std::vector<Path> getConfigDirs(); + +/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +Path getDataDir(); + +/* Create a directory and all its parents, if necessary. Returns the + list of created directories, in order of creation. */ +Paths createDirs(const Path& path); + +/* Create a symlink. */ +void createSymlink(const Path& target, const Path& link); + +/* Atomically create or replace a symlink. */ +void replaceSymlink(const Path& target, const Path& link); + +/* Wrappers arount read()/write() that read/write exactly the + requested number of bytes. */ +void readFull(int fd, unsigned char* buf, size_t count); +void writeFull(int fd, const unsigned char* buf, size_t count, + bool allowInterrupts = true); +void writeFull(int fd, const std::string& s, bool allowInterrupts = true); + +MakeError(EndOfFile, Error); + +/* Read a file descriptor until EOF occurs. */ +std::string drainFD(int fd, bool block = true); + +void drainFD(int fd, Sink& sink, bool block = true); + +/* Automatic cleanup of resources. */ + +class AutoDelete { + Path path; + bool del; + bool recursive; + + public: + AutoDelete(); + explicit AutoDelete(Path p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path& p, bool recursive = true); + explicit operator Path() const { return path; } +}; + +class AutoCloseFD { + int fd; + void close(); + + public: + AutoCloseFD(); + explicit AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD& fd) = delete; + AutoCloseFD(AutoCloseFD&& that); + ~AutoCloseFD(); + AutoCloseFD& operator=(const AutoCloseFD& fd) = delete; + AutoCloseFD& operator=(AutoCloseFD&& that); + int get() const; + explicit operator bool() const; + int release(); +}; + +class Pipe { + public: + AutoCloseFD readSide, writeSide; + void create(); +}; + +struct DIRDeleter { + void operator()(DIR* dir) const { closedir(dir); } +}; + +using AutoCloseDir = std::unique_ptr<DIR, DIRDeleter>; + +class Pid { + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; + + public: + Pid(); + explicit Pid(pid_t pid); + ~Pid(); + void operator=(pid_t pid); + explicit operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); + + friend bool operator==(const Pid& lhs, const Pid& rhs) { + return lhs.pid == rhs.pid; + } + + friend bool operator!=(const Pid& lhs, const Pid& rhs) { + return !(lhs == rhs); + } +}; + +/* Kill all processes running under the specified uid by sending them + a SIGKILL. */ +void killUser(uid_t uid); + +/* Fork a process that runs the given function, and return the child + pid to the caller. */ +struct ProcessOptions { + std::string errorPrefix = "error: "; + bool dieWithParent = true; + bool runExitHandlers = false; +}; + +pid_t startProcess(std::function<void()> fun, + const ProcessOptions& options = ProcessOptions()); + +/* Run a program and return its stdout in a string (i.e., like the + shell backtick operator). */ +std::string runProgram(const Path& program, bool searchPath = false, + const Strings& args = Strings(), + const std::optional<std::string>& input = {}); + +struct RunOptions { + std::optional<uid_t> uid; + std::optional<uid_t> gid; + std::optional<Path> chdir; + std::optional<std::map<std::string, std::string>> environment; + Path program; + bool searchPath = true; + Strings args; + std::optional<std::string> input; + Source* standardIn = nullptr; + Sink* standardOut = nullptr; + bool mergeStderrToStdout = false; + bool _killStderr = false; + + RunOptions(const Path& program, const Strings& args) + : program(program), args(args){}; + + RunOptions& killStderr(bool v) { + _killStderr = true; + return *this; + } +}; + +std::pair<int, std::string> runProgram(const RunOptions& options); + +void runProgram2(const RunOptions& options); + +class ExecError : public Error { + public: + int status; + + template <typename... Args> + explicit ExecError(int status, Args... args) + : Error(args...), status(status) {} +}; + +/* Convert a list of strings to a null-terminated vector of char + *'s. The result must not be accessed beyond the lifetime of the + list of strings. */ +std::vector<char*> stringsToCharPtrs(const Strings& ss); + +/* Close all file descriptors except those listed in the given set. + Good practice in child processes. */ +void closeMostFDs(const std::set<int>& exceptions); + +/* Set the close-on-exec flag for the given file descriptor. */ +void closeOnExec(int fd); + +/* User interruption. */ + +extern bool _isInterrupted; + +extern thread_local std::function<bool()> interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() { + if (_isInterrupted || (interruptCheck && interruptCheck())) { + _interrupted(); + } +} + +MakeError(Interrupted, BaseError); + +MakeError(FormatError, Error); + +/* Concatenate the given strings with a separator between the + elements. */ +std::string concatStringsSep(const std::string& sep, const Strings& ss); +std::string concatStringsSep(const std::string& sep, const StringSet& ss); + +/* Replace all occurrences of a string inside another string. */ +std::string replaceStrings(const std::string& s, const std::string& from, + const std::string& to); + +/* Convert the exit status of a child as returned by wait() into an + error string. */ +std::string statusToString(int status); + +bool statusOk(int status); + +/* Parse a string into a float. */ +template <class N> +bool string2Float(const std::string& s, N& n) { + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + +/* Convert a string to lower case. */ +std::string toLower(const std::string& s); + +/* Escape a string as a shell word. */ +std::string shellEscape(const std::string& s); + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ +void ignoreException(); + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_BLUE "\e[34;1m" + +/* Truncate a string to 'width' printable characters. If 'filterAll' + is true, all ANSI escape sequences are filtered out. Otherwise, + some escape sequences (such as colour setting) are copied but not + included in the character count. Also, tabs are expanded to + spaces. */ +std::string filterANSIEscapes( + const std::string& s, bool filterAll = false, + unsigned int width = std::numeric_limits<unsigned int>::max()); + +/* Get a value for the specified key from an associate container, or a + default value if the key doesn't exist. */ +template <class T> +std::string get(const T& map, const std::string& key, + const std::string& def = "") { + auto i = map.find(key); + return i == map.end() ? def : i->second; +} + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future<T> to pass the value or + exception.) */ +template <typename T> +class Callback { + std::function<void(std::future<T>)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + + public: + explicit Callback(std::function<void(std::future<T>)> fun) : fun(fun) {} + + Callback(Callback&& callback) : fun(std::move(callback.fun)) { + auto prev = callback.done.test_and_set(); + if (prev) { + done.test_and_set(); + } + } + +// The unused-variable assert is disabled in this block because the +// `prev` variables are only used in debug mode (in the asserts). +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-variable" + + void operator()(T&& t) noexcept { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow( + const std::exception_ptr& exc = std::current_exception()) noexcept { + auto prev = done.test_and_set(); + assert(!prev); + std::promise<T> promise; + promise.set_exception(exc); + fun(promise.get_future()); + } + +#pragma clang diagnostic pop +}; + +/* Start a thread that handles various signals. Also block those signals + on the current thread (and thus any threads created by it). */ +void startSignalHandlerThread(); + +/* Restore default signal handling. */ +void restoreSignals(); + +struct InterruptCallback { + virtual ~InterruptCallback(){}; +}; + +/* Register a function that gets called on SIGINT (in a non-signal + context). */ +std::unique_ptr<InterruptCallback> createInterruptCallback( + const std::function<void()>& callback); + +void triggerInterrupt(); + +/* A RAII class that causes the current thread to receive SIGUSR1 when + the signal handler thread receives SIGINT. That is, this allows + SIGINT to be multiplexed to multiple threads. */ +struct ReceiveInterrupts { + pthread_t target; + std::unique_ptr<InterruptCallback> callback; + + ReceiveInterrupts() + : target(pthread_self()), callback(createInterruptCallback([&]() { + pthread_kill(target, SIGUSR1); + })) {} +}; + +/* A RAII helper that increments a counter on construction and + decrements it on destruction. */ +template <typename T> +struct MaintainCount { + T& counter; + long delta; + explicit MaintainCount(T& counter, long delta = 1) + : counter(counter), delta(delta) { + counter += delta; + } + ~MaintainCount() { counter -= delta; } +}; + +/* Return the number of rows and columns of the terminal. */ +std::pair<unsigned short, unsigned short> getWindowSize(); + +/* Used in various places. */ +using PathFilter = std::function<bool(const Path&)>; + +extern PathFilter defaultPathFilter; + +} // namespace nix diff --git a/third_party/nix/src/libutil/visitor.hh b/third_party/nix/src/libutil/visitor.hh new file mode 100644 index 000000000000..bf1d665af77f --- /dev/null +++ b/third_party/nix/src/libutil/visitor.hh @@ -0,0 +1,19 @@ +#pragma once + +namespace nix::util { + +// Helper class used for visiting std::variants by creating a variadic +// list of lambda expressions that delegates calls to each of the +// callables. +// +// See e.g. +// https://dev.to/tmr232/that-overloaded-trick-overloading-lambdas-in-c17 +template <class... Ts> +struct overloaded : Ts... { + using Ts::operator()...; +}; + +template <class... Ts> +overloaded(Ts...) -> overloaded<Ts...>; + +} // namespace nix::util diff --git a/third_party/nix/src/libutil/xml-writer.cc b/third_party/nix/src/libutil/xml-writer.cc new file mode 100644 index 000000000000..8274ed769eb4 --- /dev/null +++ b/third_party/nix/src/libutil/xml-writer.cc @@ -0,0 +1,93 @@ +#include "libutil/xml-writer.hh" + +#include <cassert> + +namespace nix { + +XMLWriter::XMLWriter(bool indent, std::ostream& output) + : output(output), indent(indent) { + output << "<?xml version='1.0' encoding='utf-8'?>" << std::endl; + closed = false; +} + +XMLWriter::~XMLWriter() { close(); } + +void XMLWriter::close() { + if (closed) { + return; + } + while (!pendingElems.empty()) { + closeElement(); + } + closed = true; +} + +void XMLWriter::indent_(size_t depth) { + if (!indent) { + return; + } + output << std::string(depth * 2, ' '); +} + +void XMLWriter::openElement(const std::string& name, const XMLAttrs& attrs) { + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) { + output << std::endl; + } + pendingElems.push_back(name); +} + +void XMLWriter::closeElement() { + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << "</" << pendingElems.back() << ">"; + if (indent) { + output << std::endl; + } + pendingElems.pop_back(); + if (pendingElems.empty()) { + closed = true; + } +} + +void XMLWriter::writeEmptyElement(const std::string& name, + const XMLAttrs& attrs) { + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) { + output << std::endl; + } +} + +void XMLWriter::writeAttrs(const XMLAttrs& attrs) { + for (auto& i : attrs) { + output << " " << i.first << "=\""; + for (char c : i.second) { + if (c == '"') { + output << """; + } else if (c == '<') { + output << "<"; + } else if (c == '>') { + output << ">"; + } else if (c == '&') { + output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + } else if (c == '\n') { + output << "
"; + } else { + output << c; + } + } + output << "\""; + } +} + +} // namespace nix diff --git a/third_party/nix/src/libutil/xml-writer.hh b/third_party/nix/src/libutil/xml-writer.hh new file mode 100644 index 000000000000..d6f7cddb35ac --- /dev/null +++ b/third_party/nix/src/libutil/xml-writer.hh @@ -0,0 +1,52 @@ +#pragma once + +#include <iostream> +#include <list> +#include <map> +#include <string> + +namespace nix { + +typedef std::map<std::string, std::string> XMLAttrs; + +class XMLWriter { + private: + std::ostream& output; + + bool indent; + bool closed; + + std::list<std::string> pendingElems; + + public: + XMLWriter(bool indent, std::ostream& output); + ~XMLWriter(); + + void close(); + + void openElement(const std::string& name, const XMLAttrs& attrs = XMLAttrs()); + void closeElement(); + + void writeEmptyElement(const std::string& name, + const XMLAttrs& attrs = XMLAttrs()); + + private: + void writeAttrs(const XMLAttrs& attrs); + + void indent_(size_t depth); +}; + +class XMLOpenElement { + private: + XMLWriter& writer; + + public: + XMLOpenElement(XMLWriter& writer, const std::string& name, + const XMLAttrs& attrs = XMLAttrs()) + : writer(writer) { + writer.openElement(name, attrs); + } + ~XMLOpenElement() { writer.closeElement(); } +}; + +} // namespace nix diff --git a/third_party/nix/src/nix-build/nix-build.cc b/third_party/nix/src/nix-build/nix-build.cc new file mode 100644 index 000000000000..26c308967724 --- /dev/null +++ b/third_party/nix/src/nix-build/nix-build.cc @@ -0,0 +1,581 @@ +#include <cstring> +#include <fstream> +#include <iostream> +#include <regex> +#include <sstream> +#include <vector> + +#include <absl/strings/ascii.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> + +#include "libexpr/attr-path.hh" +#include "libexpr/common-eval-args.hh" +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/get-drvs.hh" +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/affinity.hh" +#include "libutil/status.hh" +#include "libutil/util.hh" +#include "nix/legacy.hh" + +using namespace nix; +using namespace std::string_literals; + +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes + */ +std::vector<std::string> shellwords(const std::string& s) { + std::regex whitespace("^(\\s+).*"); + auto begin = s.cbegin(); + std::vector<std::string> res; + std::string cur; + enum state { sBegin, sQuote }; + state st = sBegin; + auto it = begin; + for (; it != s.cend(); ++it) { + if (st == sBegin) { + std::smatch match; + if (regex_search(it, s.cend(), match, whitespace)) { + cur.append(begin, it); + res.push_back(cur); + cur.clear(); + it = match[1].second; + begin = it; + } + } + switch (*it) { + case '"': + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sQuote : sBegin; + break; + case '\\': + /* perl shellwords mostly just treats the next char as part of the + * string with no special processing */ + cur.append(begin, it); + begin = ++it; + break; + } + } + cur.append(begin, it); + if (!cur.empty()) { + res.push_back(cur); + } + return res; +} + +static void _main(int argc, char** argv) { + auto dryRun = false; + auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto pure = false; + auto fromArgs = false; + auto packages = false; + // Same condition as bash uses for interactive shells + auto interactive = + (isatty(STDIN_FILENO) != 0) && (isatty(STDERR_FILENO) != 0); + Strings attrPaths; + Strings left; + RepairFlag repair = NoRepair; + Path gcRoot; + BuildMode buildMode = bmNormal; + bool readStdin = false; + + std::string envCommand; // interactive shell + Strings envExclude; + + auto myName = runEnv ? "nix-shell" : "nix-build"; + + auto inShebang = false; + std::string script; + std::vector<std::string> savedArgs; + + AutoDelete tmpDir(createTempDir("", myName)); + + std::string outLink = "./result"; + + // List of environment variables kept for --pure + std::set<std::string> keepVars{ + "HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", + "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"}; + + Strings args; + for (int i = 1; i < argc; ++i) { + args.push_back(argv[i]); + } + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + if (runEnv && argc > 1 && + !std::regex_search(argv[1], std::regex("nix-shell"))) { + script = argv[1]; + try { + Strings lines = absl::StrSplit(readFile(script), absl::ByChar('\n'), + absl::SkipEmpty()); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + inShebang = true; + for (int i = 2; i < argc; ++i) { + savedArgs.emplace_back(argv[i]); + } + args.clear(); + for (auto line : lines) { + line = absl::StripTrailingAsciiWhitespace(line); + std::smatch match; + if (std::regex_match(line, match, + std::regex("^#!\\s*nix-shell (.*)$"))) { + for (const auto& word : shellwords(match[1].str())) { + args.push_back(word); + } + } + } + } + } catch (SysError&) { + } + } + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs( + myName, [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + deletePath(Path(tmpDir)); + showManPage(myName); + } + + else if (*arg == "--version") { + printVersion(myName); + + } else if (*arg == "--add-drv-link" || *arg == "--indirect") { + ; // obsolete + + } else if (*arg == "--no-out-link" || *arg == "--no-link") { + outLink = Path(tmpDir) + "/result"; + + } else if (*arg == "--attr" || *arg == "-A") { + attrPaths.push_back(getArg(*arg, arg, end)); + + } else if (*arg == "--drv-link") { + getArg(*arg, arg, end); // obsolete + + } else if (*arg == "--out-link" || *arg == "-o") { + outLink = getArg(*arg, arg, end); + + } else if (*arg == "--add-root") { + gcRoot = getArg(*arg, arg, end); + + } else if (*arg == "--dry-run") { + dryRun = true; + + } else if (*arg == "--repair") { + repair = Repair; + buildMode = bmRepair; + } + + else if (*arg == "--run-env") { // obsolete + runEnv = true; + + } else if (*arg == "--command" || *arg == "--run") { + if (*arg == "--run") { + interactive = false; + } + envCommand = getArg(*arg, arg, end) + "\nexit"; + } + + else if (*arg == "--check") { + buildMode = bmCheck; + + } else if (*arg == "--exclude") { + envExclude.push_back(getArg(*arg, arg, end)); + + } else if (*arg == "--expr" || *arg == "-E") { + fromArgs = true; + + } else if (runEnv && *arg == "--pure") { + pure = true; + } else if (runEnv && *arg == "--impure") { + pure = false; + + } else if (*arg == "--packages" || *arg == "-p") { + packages = true; + + } else if (inShebang && *arg == "-i") { + auto interpreter = getArg(*arg, arg, end); + interactive = false; + auto execArgs = ""; + + // Überhack to support Perl. Perl examines the shebang and + // executes it unless it contains the string "perl" or "indir", + // or (undocumented) argv[0] does not contain "perl". Exploit + // the latter by doing "exec -a". + if (std::regex_search(interpreter, std::regex("perl"))) { + execArgs = "-a PERL"; + } + + std::ostringstream joined; + for (const auto& i : savedArgs) { + joined << shellEscape(i) << ' '; + } + + if (std::regex_search(interpreter, std::regex("ruby"))) { + // Hack for Ruby. Ruby also examines the shebang. It tries to + // read the shebang to understand which packages to read from. Since + // this is handled via nix-shell -p, we wrap our ruby script + // execution in ruby -e 'load' which ignores the shebangs. + envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % + execArgs % interpreter % script % joined.str()) + .str(); + } else { + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % + interpreter % script % joined.str()) + .str(); + } + } + + else if (*arg == "--keep") { + keepVars.insert(getArg(*arg, arg, end)); + + } else if (*arg == "-") { + readStdin = true; + + } else if (*arg != "" && arg->at(0) == '-') { + return false; + + } else { + left.push_back(*arg); + } + + return true; + }); + + myArgs.parseCmdline(args); + + if (packages && fromArgs) { + throw UsageError("'-p' and '-E' are mutually exclusive"); + } + + auto store = openStore(); + + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + state->repair = repair; + + std::unique_ptr<Bindings> autoArgs = myArgs.getAutoArgs(*state); + + if (packages) { + std::ostringstream joined; + // TODO(grfn): Generate a syntax tree here, not a string + joined << "with import <nixpkgs> { }; (pkgs.runCommandCC or " + "pkgs.runCommand) \"shell\" { buildInputs = [ "; + for (const auto& i : left) { + joined << '(' << i << ") "; + } + joined << "]; } \"\""; + fromArgs = true; + left = {joined.str()}; + } else if (!fromArgs) { + if (left.empty() && runEnv && pathExists("shell.nix")) { + left = {"shell.nix"}; + } + if (left.empty()) { + left = {"default.nix"}; + } + } + + if (runEnv) { + setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); + } + + DrvInfos drvs; + + /* Parse the expressions. */ + std::vector<Expr*> exprs; + + if (readStdin) { + exprs = {state->parseStdin()}; + } else { + for (const auto& i : left) { + if (fromArgs) { + exprs.push_back(state->parseExprFromString(i, absPath("."))); + } else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error& e) { + }; + if (store->isStorePath(absolute) && + std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) { + drvs.push_back(DrvInfo(*state, store, absolute)); + } else { + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back( + state->parseExprFromFile(resolveExprPath(state->checkSourcePath( + lookupFileArg(*state, inShebang && !packages + ? absPath(i, absPath(dirOf(script))) + : i))))); + } + } + } + } + + /* Evaluate them into derivations. */ + if (attrPaths.empty()) { + attrPaths = {""}; + } + + for (auto e : exprs) { + Value vRoot; + state->eval(e, vRoot); + + for (auto& i : attrPaths) { + Value& v(*findAlongAttrPath(*state, i, autoArgs.get(), vRoot)); + state->forceValue(v); + getDerivations(*state, v, "", autoArgs.get(), drvs, false); + } + } + + state->printStats(); + + auto buildPaths = [&](const PathSet& paths) { + /* Note: we do this even when !printMissing to efficiently + fetch binary cache data. */ + unsigned long long downloadSize; + unsigned long long narSize; + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, + narSize); + + if (settings.printMissing) { + printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, + downloadSize, narSize); + } + + if (!dryRun) { + util::OkOrThrow(store->buildPaths(std::cerr, paths, buildMode)); + } + }; + + if (runEnv) { + if (drvs.size() != 1) { + throw UsageError("nix-shell requires a single derivation"); + } + + auto& drvInfo = drvs.front(); + auto drv = store->derivationFromPath(drvInfo.queryDrvPath()); + + PathSet pathsToBuild; + + /* Figure out what bash shell to use. If $NIX_BUILD_SHELL + is not set, then build bashInteractive from + <nixpkgs>. */ + auto opt_shell = getEnv("NIX_BUILD_SHELL"); + std::string shell; + + if (opt_shell.has_value()) { + shell = opt_shell.value(); + } else { + try { + auto expr = state->parseExprFromString( + "(import <nixpkgs> {}).bashInteractive", absPath(".")); + + Value v; + state->eval(expr, v); + + auto drv = getDerivation(*state, v, false); + if (!drv) { + throw Error( + "the 'bashInteractive' attribute in <nixpkgs> did not evaluate " + "to a derivation"); + } + + pathsToBuild.insert(drv->queryDrvPath()); + + shell = drv->queryOutPath() + "/bin/bash"; + + } catch (Error& e) { + LOG(WARNING) << e.what() << "; will use bash from your environment"; + shell = "bash"; + } + } + + // Build or fetch all dependencies of the derivation. + for (const auto& input : drv.inputDrvs) { + if (std::all_of(envExclude.cbegin(), envExclude.cend(), + [&](const std::string& exclude) { + return !std::regex_search(input.first, + std::regex(exclude)); + })) { + pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second)); + } + } + for (const auto& src : drv.inputSrcs) { + pathsToBuild.insert(src); + } + + buildPaths(pathsToBuild); + + if (dryRun) { + return; + } + + // Set the environment. + auto env = getEnv(); + + auto tmp = + getEnv("TMPDIR").value_or(getEnv("XDG_RUNTIME_DIR").value_or("/tmp")); + + if (pure) { + decltype(env) newEnv; + for (auto& i : env) { + if (keepVars.count(i.first) != 0u) { + newEnv.emplace(i); + } + } + env = newEnv; + // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. + env["__ETC_PROFILE_SOURCED"] = "1"; + } + + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = + env["TEMP"] = tmp; + env["NIX_STORE"] = store->storeDir; + env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); + + StringSet passAsFile = + absl::StrSplit(get(drv.env, "passAsFile", ""), + absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + + bool keepTmp = false; + int fileNr = 0; + + for (auto& var : drv.env) { + if (passAsFile.count(var.first) != 0u) { + keepTmp = true; + std::string fn = ".attr-" + std::to_string(fileNr++); + Path p = Path(tmpDir) + "/" + fn; + writeFile(p, var.second); + env[var.first + "Path"] = p; + } else { + env[var.first] = var.second; + } + } + + restoreAffinity(); + + /* Run a shell using the derivation's environment. For + convenience, source $stdenv/setup to setup additional + environment variables and shell functions. Also don't + lose the current $PATH directories. */ + auto rcfile = Path(tmpDir) + "/rc"; + writeFile( + rcfile, + fmt((keepTmp ? "" : "rm -rf '%1%'; "s) + + "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " + "%2%" + "dontAddDisableDepTrack=1; " + "[ -e $stdenv/setup ] && source $stdenv/setup; " + "%3%" + "PATH=\"%4%:$PATH\"; " + "SHELL=%5%; " + "set +e; " + R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + "if [ \"$(type -t runHook)\" = function ]; then runHook " + "shellHook; fi; " + "unset NIX_ENFORCE_PURITY; " + "shopt -u nullglob; " + "unset TZ; %6%" + "%7%", + Path(tmpDir), (pure ? "" : "p=$PATH; "), + (pure ? "" : "PATH=$PATH:$p; unset p; "), dirOf(shell), shell, + (getenv("TZ") != nullptr + ? (std::string("export TZ='") + getenv("TZ") + "'; ") + : ""), + envCommand)); + + Strings envStrs; + for (auto& i : env) { + envStrs.push_back(i.first + "=" + i.second); + } + + auto args = interactive ? Strings{"bash", "--rcfile", rcfile} + : Strings{"bash", rcfile}; + + auto envPtrs = stringsToCharPtrs(envStrs); + + environ = envPtrs.data(); + + auto argPtrs = stringsToCharPtrs(args); + + restoreSignals(); + + execvp(shell.c_str(), argPtrs.data()); + + throw SysError("executing shell '%s'", shell); + } + + PathSet pathsToBuild; + + std::map<Path, Path> drvPrefixes; + std::map<Path, Path> resultSymlinks; + std::vector<Path> outPaths; + + for (auto& drvInfo : drvs) { + auto drvPath = drvInfo.queryDrvPath(); + auto outPath = drvInfo.queryOutPath(); + + auto outputName = drvInfo.queryOutputName(); + if (outputName.empty()) { + throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); + } + + pathsToBuild.insert(drvPath + "!" + outputName); + + std::string drvPrefix; + auto i = drvPrefixes.find(drvPath); + if (i != drvPrefixes.end()) { + drvPrefix = i->second; + } else { + drvPrefix = outLink; + if (!drvPrefixes.empty() != 0u) { + drvPrefix += fmt("-%d", drvPrefixes.size() + 1); + } + drvPrefixes[drvPath] = drvPrefix; + } + + std::string symlink = drvPrefix; + if (outputName != "out") { + symlink += "-" + outputName; + } + + resultSymlinks[symlink] = outPath; + outPaths.push_back(outPath); + } + + buildPaths(pathsToBuild); + + if (dryRun) { + return; + } + + for (auto& symlink : resultSymlinks) { + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { + store2->addPermRoot(symlink.second, absPath(symlink.first), true); + } + } + + for (auto& path : outPaths) { + std::cout << path << '\n'; + } +} + +static RegisterLegacyCommand s1("nix-build", _main); +static RegisterLegacyCommand s2("nix-shell", _main); diff --git a/third_party/nix/src/nix-channel/nix-channel.cc b/third_party/nix/src/nix-channel/nix-channel.cc new file mode 100644 index 000000000000..5cc16d1b4b9a --- /dev/null +++ b/third_party/nix/src/nix-channel/nix-channel.cc @@ -0,0 +1,275 @@ +#include <regex> + +#include <absl/strings/ascii.h> +#include <absl/strings/str_split.h> +#include <fcntl.h> +#include <pwd.h> + +#include "libmain/shared.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "nix/legacy.hh" + +using namespace nix; + +typedef std::map<std::string, std::string> Channels; + +static Channels channels; +static Path channelsList; + +// Reads the list of channels. +static void readChannels() { + if (!pathExists(channelsList)) { + return; + } + auto channelsFile = readFile(channelsList); + + std::vector<std::string> lines = + absl::StrSplit(channelsFile, absl::ByChar('\n'), absl::SkipEmpty()); + + for (auto& line : lines) { + line = absl::StripTrailingAsciiWhitespace(line); + if (std::regex_search(line, std::regex("^\\s*\\#"))) { + continue; + } + std::vector<std::string> split = + absl::StrSplit(line, absl::ByChar(' '), absl::SkipEmpty()); + auto url = std::regex_replace(split[0], std::regex("/*$"), ""); + auto name = split.size() > 1 ? split[1] : baseNameOf(url); + channels[name] = url; + } +} + +// Writes the list of channels. +static void writeChannels() { + auto channelsFD = AutoCloseFD{open( + channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; + if (!channelsFD) { + throw SysError(format("opening '%1%' for writing") % channelsList); + } + for (const auto& channel : channels) { + writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); + } +} + +// Adds a channel. +static void addChannel(const std::string& url, const std::string& name) { + if (!regex_search(url, std::regex("^(file|http|https)://"))) { + throw Error(format("invalid channel URL '%1%'") % url); + } + if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) { + throw Error(format("invalid channel identifier '%1%'") % name); + } + readChannels(); + channels[name] = url; + writeChannels(); +} + +static Path profile; + +// Remove a channel. +static void removeChannel(const std::string& name) { + readChannels(); + channels.erase(name); + writeChannels(); + + runProgram(settings.nixBinDir + "/nix-env", true, + {"--profile", profile, "--uninstall", name}); +} + +static Path nixDefExpr; + +// Fetch Nix expressions and binary cache URLs from the subscribed channels. +static void update(const StringSet& channelNames) { + readChannels(); + + auto store = openStore(); + + // Download each channel. + Strings exprs; + for (const auto& channel : channels) { + auto name = channel.first; + auto url = channel.second; + if (!(channelNames.empty() || (channelNames.count(name) != 0u))) { + continue; + } + + // We want to download the url to a file to see if it's a tarball while also + // checking if we got redirected in the process, so that we can grab the + // various parts of a nix channel definition from a consistent location if + // the redirect changes mid-download. + CachedDownloadRequest request(url); + request.ttl = 0; + auto dl = getDownloader(); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = absl::StripTrailingAsciiWhitespace(result.effectiveUri); + + // If the URL contains a version number, append it to the name + // attribute (so that "nix-env -q" on the channels profile + // shows something useful). + auto cname = name; + std::smatch match; + auto urlBase = baseNameOf(url); + if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { + cname = cname + std::string(match[1]); + } + + std::string extraAttrs; + + bool unpacked = false; + if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { + runProgram(settings.nixBinDir + "/nix-build", false, + {"--no-out-link", "--expr", + "import <nix/unpack-channel.nix> " + "{ name = \"" + + cname + "\"; channelName = \"" + name + + "\"; src = builtins.storePath \"" + filename + "\"; }"}); + unpacked = true; + } + + if (!unpacked) { + // Download the channel tarball. + try { + filename = dl->downloadCached( + store, CachedDownloadRequest(url + "/nixexprs.tar.xz")) + .path; + } catch (DownloadError& e) { + filename = + dl->downloadCached(store, + CachedDownloadRequest(url + "/nixexprs.tar.bz2")) + .path; + } + filename = absl::StripTrailingAsciiWhitespace(filename); + } + + // Regardless of where it came from, add the expression representing this + // channel to accumulated expression + exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + + "\"; src = builtins.storePath \"" + filename + "\"; " + + extraAttrs + " }"); + } + + // Unpack the channel tarballs into the Nix store and install them + // into the channels profile. + std::cerr << "unpacking channels...\n"; + Strings envArgs{"--profile", profile, + "--file", "<nix/unpack-channel.nix>", + "--install", "--from-expression"}; + for (auto& expr : exprs) { + envArgs.push_back(std::move(expr)); + } + envArgs.push_back("--quiet"); + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + + // Make the channels appear in nix-env. + struct stat st; + if (lstat(nixDefExpr.c_str(), &st) == 0) { + if (S_ISLNK(st.st_mode)) { + // old-skool ~/.nix-defexpr + if (unlink(nixDefExpr.c_str()) == -1) { + throw SysError(format("unlinking %1%") % nixDefExpr); + } + } + } else if (errno != ENOENT) { + throw SysError(format("getting status of %1%") % nixDefExpr); + } + createDirs(nixDefExpr); + auto channelLink = nixDefExpr + "/channels"; + replaceSymlink(profile, channelLink); +} + +static int _main(int argc, char** argv) { + { + // Figure out the name of the `.nix-channels' file to use + auto home = getHome(); + channelsList = home + "/.nix-channels"; + nixDefExpr = home + "/.nix-defexpr"; + + // Figure out the name of the channels profile. + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, + getUserName()); + + enum { cNone, cAdd, cRemove, cList, cUpdate, cRollback } cmd = cNone; + std::vector<std::string> args; + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-channel"); + } else if (*arg == "--version") { + printVersion("nix-channel"); + } else if (*arg == "--add") { + cmd = cAdd; + } else if (*arg == "--remove") { + cmd = cRemove; + } else if (*arg == "--list") { + cmd = cList; + } else if (*arg == "--update") { + cmd = cUpdate; + } else if (*arg == "--rollback") { + cmd = cRollback; + } else { + args.push_back(std::move(*arg)); + } + return true; + }); + + switch (cmd) { + case cNone: + throw UsageError("no command specified"); + case cAdd: + if (args.empty() || args.size() > 2) { + throw UsageError("'--add' requires one or two arguments"); + } + { + auto url = args[0]; + std::string name; + if (args.size() == 2) { + name = args[1]; + } else { + name = baseNameOf(url); + name = std::regex_replace(name, std::regex("-unstable$"), ""); + name = std::regex_replace(name, std::regex("-stable$"), ""); + } + addChannel(url, name); + } + break; + case cRemove: + if (args.size() != 1) { + throw UsageError("'--remove' requires one argument"); + } + removeChannel(args[0]); + break; + case cList: + if (!args.empty()) { + throw UsageError("'--list' expects no arguments"); + } + readChannels(); + for (const auto& channel : channels) { + std::cout << channel.first << ' ' << channel.second << '\n'; + } + break; + case cUpdate: + update(StringSet(args.begin(), args.end())); + break; + case cRollback: + if (args.size() > 1) { + throw UsageError("'--rollback' has at most one argument"); + } + Strings envArgs{"--profile", profile}; + if (args.size() == 1) { + envArgs.push_back("--switch-generation"); + envArgs.push_back(args[0]); + } else { + envArgs.push_back("--rollback"); + } + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + break; + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc b/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc new file mode 100644 index 000000000000..ac8c7d9399fb --- /dev/null +++ b/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc @@ -0,0 +1,103 @@ +#include <cerrno> +#include <iostream> + +#include <glog/logging.h> + +#include "libmain/shared.hh" +#include "libstore/globals.hh" +#include "libstore/profiles.hh" +#include "libstore/store-api.hh" +#include "nix/legacy.hh" + +using namespace nix; + +std::string deleteOlderThan; +bool dryRun = false; + +/* If `-d' was specified, remove all old generations of all profiles. + * Of course, this makes rollbacks to before this point in time + * impossible. */ + +void removeOldGenerations(const std::string& dir) { + if (access(dir.c_str(), R_OK) != 0) { + return; + } + + bool canWrite = access(dir.c_str(), W_OK) == 0; + + for (auto& i : readDirectory(dir)) { + checkInterrupt(); + + auto path = dir + "/" + i.name; + auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type; + + if (type == DT_LNK && canWrite) { + std::string link; + try { + link = readLink(path); + } catch (SysError& e) { + if (e.errNo == ENOENT) { + continue; + } + } + if (link.find("link") != std::string::npos) { + LOG(INFO) << "removing old generations of profile " << path; + if (!deleteOlderThan.empty()) { + deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); + } else { + deleteOldGenerations(path, dryRun); + } + } + } else if (type == DT_DIR) { + removeOldGenerations(path); + } + } +} + +static int _main(int argc, char** argv) { + { + bool removeOld = false; + + GCOptions options; + + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-collect-garbage"); + } else if (*arg == "--version") { + printVersion("nix-collect-garbage"); + } else if (*arg == "--delete-old" || *arg == "-d") { + removeOld = true; + } else if (*arg == "--delete-older-than") { + removeOld = true; + deleteOlderThan = getArg(*arg, arg, end); + } else if (*arg == "--dry-run") { + dryRun = true; + } else if (*arg == "--max-freed") { + auto maxFreed = getIntArg<long long>(*arg, arg, end, true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } else { + return false; + } + return true; + }); + + auto profilesDir = settings.nixStateDir + "/profiles"; + if (removeOld) { + removeOldGenerations(profilesDir); + } + + // Run the actual garbage collector. + if (!dryRun) { + auto store = openStore(); + options.action = GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc b/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc new file mode 100644 index 000000000000..3dbe29f2240f --- /dev/null +++ b/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc @@ -0,0 +1,73 @@ +#include <glog/logging.h> + +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "nix/legacy.hh" + +using namespace nix; + +static int _main(int argc, char** argv) { + { + auto gzip = false; + auto toMode = true; + auto includeOutputs = false; + auto dryRun = false; + auto useSubstitutes = NoSubstitute; + std::string sshHost; + PathSet storePaths; + + parseCmdLine( + argc, argv, [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-copy-closure"); + } else if (*arg == "--version") { + printVersion("nix-copy-closure"); + } else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { + if (*arg != "--gzip") { + LOG(WARNING) << "'" << *arg + << "' is not implemented, falling back to gzip"; + } + gzip = true; + } else if (*arg == "--from") { + toMode = false; + } else if (*arg == "--to") { + toMode = true; + } else if (*arg == "--include-outputs") { + includeOutputs = true; + } else if (*arg == "--show-progress") { + LOG(WARNING) << "'--show-progress' is not implemented"; + } else if (*arg == "--dry-run") { + dryRun = true; + } else if (*arg == "--use-substitutes" || *arg == "-s") { + useSubstitutes = Substitute; + } else if (sshHost.empty()) { + sshHost = *arg; + } else { + storePaths.insert(*arg); + } + return true; + }); + + if (sshHost.empty()) { + throw UsageError("no host name specified"); + } + + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto to = toMode ? openStore(remoteUri) : openStore(); + auto from = toMode ? openStore() : openStore(remoteUri); + + PathSet storePaths2; + for (auto& path : storePaths) { + storePaths2.insert(from->followLinksToStorePath(path)); + } + + PathSet closure; + from->computeFSClosure(storePaths2, closure, false, includeOutputs); + + copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/third_party/nix/src/nix-daemon/CMakeLists.txt b/third_party/nix/src/nix-daemon/CMakeLists.txt new file mode 100644 index 000000000000..63125a9b26b2 --- /dev/null +++ b/third_party/nix/src/nix-daemon/CMakeLists.txt @@ -0,0 +1,29 @@ +# -*- mode: cmake; -*- + +# The nix-daemon is the binary running the gRPC server component to +# which other components of Nix talk to perform store and builder +# operations. + +add_executable(nix-daemon) +include_directories(${PROJECT_BINARY_DIR}) # for config.h +set_property(TARGET nix-daemon PROPERTY CXX_STANDARD 17) + +pkg_check_modules(systemd REQUIRED) + +target_sources(nix-daemon + PRIVATE + nix-daemon-proto.hh + nix-daemon-proto.cc + nix-daemon.cc +) + +target_link_libraries(nix-daemon + nixutil + nixstore + nixmain + absl::flags + absl::flags_parse + systemd +) + +install(TARGETS nix-daemon DESTINATION bin) diff --git a/third_party/nix/src/nix-daemon/nix-daemon-legacy.cc b/third_party/nix/src/nix-daemon/nix-daemon-legacy.cc new file mode 100644 index 000000000000..97cf5195d35c --- /dev/null +++ b/third_party/nix/src/nix-daemon/nix-daemon-legacy.cc @@ -0,0 +1,1185 @@ +/* + NOTE: You are looking at the *previous* implementation of the Nix + daemon. This file is not in use, is only left in here for reference + and will be deleted from the codebase eventually. + */ + +#include <algorithm> +#include <cerrno> +#include <climits> +#include <csignal> +#include <cstring> + +#include <fcntl.h> +#include <glog/logging.h> +#include <grp.h> +#include <pwd.h> +#include <sys/socket.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/un.h> +#include <sys/wait.h> +#include <unistd.h> + +#include "libmain/shared.hh" +#include "libproto/worker.pb.h" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/affinity.hh" +#include "libutil/archive.hh" +#include "libutil/finally.hh" +#include "libutil/monitor-fd.hh" +#include "libutil/serialise.hh" +#include "libutil/util.hh" +#include "nix/legacy.hh" + +using namespace nix; + +#ifndef __linux__ +#define SPLICE_F_MOVE 0 +static ssize_t splice(int fd_in, void* off_in, int fd_out, void* off_out, + size_t len, unsigned int flags) { + /* We ignore most parameters, we just have them for conformance with the linux + * syscall */ + std::vector<char> buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); + if (read_count == -1) { + return read_count; + } + auto write_count = decltype(read_count)(0); + while (write_count < read_count) { + auto res = + write(fd_out, buf.data() + write_count, read_count - write_count); + if (res == -1) { + return res; + } + write_count += res; + } + return read_count; +} +#endif + +static FdSource from(STDIN_FILENO); +static FdSink to(STDOUT_FILENO); + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +struct TunnelLogger { + struct State { + bool canSendStderr = false; + std::vector<std::string> pendingMsgs; + }; + + Sync<State> state_; + + unsigned int clientVersion; + + explicit TunnelLogger(unsigned int clientVersion) + : clientVersion(clientVersion) {} + + void enqueueMsg(const std::string& s) { + auto state(state_.lock()); + + if (state->canSendStderr) { + assert(state->pendingMsgs.empty()); + try { + to(s); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + state->canSendStderr = false; + throw; + } + } else { + state->pendingMsgs.push_back(s); + } + } + + void log(const FormatOrString& fs) { + StringSink buf; + buf << STDERR_NEXT << (fs.s + "\n"); + enqueueMsg(*buf.s); + } + + /* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ + void startWork() { + auto state(state_.lock()); + state->canSendStderr = true; + + for (auto& msg : state->pendingMsgs) { + to(msg); + } + + state->pendingMsgs.clear(); + + to.flush(); + } + + /* stopWork() means that we're done; stop sending stderr to the + client. */ + void stopWork(bool success = true, const std::string& msg = "", + unsigned int status = 0) { + auto state(state_.lock()); + + state->canSendStderr = false; + + if (success) { + to << STDERR_LAST; + } else { + to << STDERR_ERROR << msg; + if (status != 0) { + to << status; + } + } + } + + void startActivity(const std::string& s) { + DLOG(INFO) << "startActivity(" << s << ")"; + if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (!s.empty()) { + LOG(INFO) << s; + } + return; + } + + StringSink buf; + buf << STDERR_START_ACTIVITY << s; + enqueueMsg(*buf.s); + } +}; + +struct TunnelSink : Sink { + Sink& to; + explicit TunnelSink(Sink& to) : to(to) {} + void operator()(const unsigned char* data, size_t len) override { + to << STDERR_WRITE; + writeString(data, len, to); + } +}; + +struct TunnelSource : BufferedSource { + Source& from; + explicit TunnelSource(Source& from) : from(from) {} + + protected: + size_t readUnbuffered(unsigned char* data, size_t len) override { + to << STDERR_READ << len; + to.flush(); + size_t n = readString(data, len, from); + if (n == 0) { + throw EndOfFile("unexpected end-of-file"); + } + return n; + } +}; + +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink { + bool regular{true}; + std::string s; + + RetrieveRegularNARSink() {} + + void createDirectory(const Path& path) override { regular = false; } + + void receiveContents(unsigned char* data, unsigned int len) override { + s.append((const char*)data, len); + } + + void createSymlink(const Path& path, const std::string& target) override { + regular = false; + } +}; + +static void performOp(TunnelLogger* logger, const ref<Store>& store, + bool trusted, unsigned int clientVersion, Source& from, + Sink& to, unsigned int op) { + switch (op) { + case wopIsValidPath: { + /* 'readStorePath' could raise an error leading to the connection + being closed. To be able to recover from an invalid path error, + call 'startWork' early, and do 'assertStorePath' afterwards so + that the 'Error' exception handler doesn't close the + connection. */ + Path path = readString(from); + logger->startWork(); + store->assertStorePath(path); + bool result = store->isValidPath(path); + logger->stopWork(); + to << static_cast<uint64_t>(result); + break; + } + + case wopQueryValidPaths: { + auto paths = readStorePaths<PathSet>(*store, from); + logger->startWork(); + PathSet res = store->queryValidPaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopHasSubstitutes: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths({path}); + logger->stopWork(); + to << static_cast<uint64_t>(res.find(path) != res.end()); + break; + } + + case wopQuerySubstitutablePaths: { + auto paths = readStorePaths<PathSet>(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopQueryPathHash: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto hash = store->queryPathInfo(path)->narHash; + logger->stopWork(); + to << hash.to_string(Base16, false); + break; + } + + case wopQueryReferences: + case wopQueryReferrers: + case wopQueryValidDerivers: + case wopQueryDerivationOutputs: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet paths; + if (op == wopQueryReferences) { + paths = store->queryPathInfo(path)->references; + } else if (op == wopQueryReferrers) { + store->queryReferrers(path, paths); + } else if (op == wopQueryValidDerivers) { + paths = store->queryValidDerivers(path); + } else { + paths = store->queryDerivationOutputs(path); + } + logger->stopWork(); + to << paths; + break; + } + + case wopQueryDerivationOutputNames: { + Path path = readStorePath(*store, from); + logger->startWork(); + StringSet names; + names = store->queryDerivationOutputNames(path); + logger->stopWork(); + to << names; + break; + } + + case wopQueryDeriver: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto deriver = store->queryPathInfo(path)->deriver; + logger->stopWork(); + to << deriver; + break; + } + + case wopQueryPathFromHashPart: { + std::string hashPart = readString(from); + logger->startWork(); + Path path = store->queryPathFromHashPart(hashPart); + logger->stopWork(); + to << path; + break; + } + + case wopAddToStore: { + bool fixed = false; + bool recursive = false; + std::string hashType; + std::string baseName; + from >> baseName >> fixed /* obsolete */ >> recursive >> hashType; + /* Compatibility hack. */ + if (!fixed) { + hashType = "sha256"; + recursive = true; + } + HashType hashAlgo = parseHashType(hashType); + + TeeSource savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else { + parseDump(savedRegular, from); + } + + logger->startWork(); + if (!savedRegular.regular) { + throw Error("regular file expected"); + } + + auto store2 = store.dynamic_pointer_cast<LocalStore>(); + if (!store2) { + throw Error("operation is only supported by LocalStore"); + } + + Path path = store2->addToStoreFromDump( + recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, + hashAlgo); + logger->stopWork(); + + to << path; + break; + } + + case wopAddTextToStore: { + std::string suffix = readString(from); + std::string s = readString(from); + auto refs = readStorePaths<PathSet>(*store, from); + logger->startWork(); + Path path = store->addTextToStore(suffix, s, refs, NoRepair); + logger->stopWork(); + to << path; + break; + } + + case wopExportPath: { + Path path = readStorePath(*store, from); + readInt(from); // obsolete + logger->startWork(); + TunnelSink sink(to); + store->exportPath(path, sink); + logger->stopWork(); + to << 1; + break; + } + + case wopImportPaths: { + logger->startWork(); + TunnelSource source(from); + Paths paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); + logger->stopWork(); + to << paths; + break; + } + + case wopBuildPaths: { + auto drvs = readStorePaths<PathSet>(*store, from); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode)readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) { + throw Error( + "repairing is not allowed because you are not in " + "'trusted-users'"); + } + } + logger->startWork(); + store->buildPaths(drvs, mode); + logger->stopWork(); + to << 1; + break; + } + + case wopBuildDerivation: { + Path drvPath = readStorePath(*store, from); + BasicDerivation drv; + readDerivation(from, *store, drv); + auto buildMode = (BuildMode)readInt(from); + logger->startWork(); + if (!trusted) { + throw Error("you are not privileged to build derivations"); + } + auto res = store->buildDerivation(drvPath, drv, buildMode); + logger->stopWork(); + to << res.status << res.errorMsg; + break; + } + + case wopEnsurePath: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->ensurePath(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddTempRoot: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->addTempRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddIndirectRoot: { + Path path = absPath(readString(from)); + logger->startWork(); + store->addIndirectRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopSyncWithGC: { + logger->startWork(); + store->syncWithGC(); + logger->stopWork(); + to << 1; + break; + } + + case wopFindRoots: { + logger->startWork(); + Roots roots = store->findRoots(!trusted); + logger->stopWork(); + + size_t size = 0; + for (auto& i : roots) { + size += i.second.size(); + } + + to << size; + + for (auto& [target, links] : roots) { + for (auto& link : links) { + to << link << target; + } + } + + break; + } + + case wopCollectGarbage: { + GCOptions options; + options.action = (GCOptions::GCAction)readInt(from); + options.pathsToDelete = readStorePaths<PathSet>(*store, from); + from >> options.ignoreLiveness >> options.maxFreed; + // obsolete fields + readInt(from); + readInt(from); + readInt(from); + + GCResults results; + + logger->startWork(); + if (options.ignoreLiveness) { + throw Error("you are not allowed to ignore liveness"); + } + store->collectGarbage(options, results); + logger->stopWork(); + + to << results.paths << results.bytesFreed << 0 /* obsolete */; + + break; + } + + case wopSetOptions: { + settings.keepFailed = readInt(from) != 0u; + settings.keepGoing = readInt(from) != 0u; + settings.tryFallback = readInt(from) != 0u; + readInt(from); // obsolete verbosity + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); + readInt(from); // obsolete useBuildHook + settings.verboseBuild = 0 == readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from) != 0u; + + StringMap overrides; + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + std::string name = readString(from); + std::string value = readString(from); + overrides.emplace(name, value); + } + } + + logger->startWork(); + + for (auto& i : overrides) { + auto& name(i.first); + auto& value(i.second); + + auto setSubstituters = [&](Setting<Strings>& res) { + if (name != res.name && res.aliases.count(name) == 0) { + return false; + } + StringSet trusted = settings.trustedSubstituters; + for (auto& s : settings.substituters.get()) { + trusted.insert(s); + } + Strings subs; + Strings ss = absl::StrSplit(value, absl::ByAnyChar(" \t\n\r"), + absl::SkipEmpty()); + for (auto& s : ss) { + if (trusted.count(s) != 0u) { + subs.push_back(s); + } else { + LOG(WARNING) << "ignoring untrusted substituter '" << s << "'"; + } + } + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") { // obsolete + ; + } else if (trusted || name == settings.buildTimeout.name || + name == "connect-timeout" || + (name == "builders" && value.empty())) { + settings.set(name, value); + } else if (setSubstituters(settings.substituters)) { + ; + } else if (setSubstituters(settings.extraSubstituters)) { + ; + } else { + LOG(WARNING) << "ignoring the user-specified setting '" << name + << "', because it is a " + << "restricted setting and you are not a trusted user"; + } + } catch (UsageError& e) { + LOG(WARNING) << e.what(); + } + } + + logger->stopWork(); + break; + } + + case wopQuerySubstitutablePathInfo: { + Path path = absPath(readString(from)); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos({path}, infos); + logger->stopWork(); + auto i = infos.find(path); + if (i == infos.end()) { + to << 0; + } else { + to << 1 << i->second.deriver << i->second.references + << i->second.downloadSize << i->second.narSize; + } + break; + } + + case wopQuerySubstitutablePathInfos: { + auto paths = readStorePaths<PathSet>(*store, from); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + to << infos.size(); + for (auto& i : infos) { + to << i.first << i.second.deriver << i.second.references + << i.second.downloadSize << i.second.narSize; + } + break; + } + + case wopQueryAllValidPaths: { + logger->startWork(); + PathSet paths = store->queryAllValidPaths(); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryPathInfo: { + Path path = readStorePath(*store, from); + std::shared_ptr<const ValidPathInfo> info; + logger->startWork(); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath&) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) { + throw; + } + } + logger->stopWork(); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) { + to << 1; + } + to << info->deriver << info->narHash.to_string(Base16, false) + << info->references << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << static_cast<uint64_t>(info->ultimate) << info->sigs << info->ca; + } + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } + break; + } + + case wopOptimiseStore: { + logger->startWork(); + store->optimiseStore(); + logger->stopWork(); + to << 1; + break; + } + + case wopVerifyStore: { + bool checkContents; + bool repair; + from >> checkContents >> repair; + logger->startWork(); + if (repair && !trusted) { + throw Error("you are not privileged to repair paths"); + } + bool errors = store->verifyStore(checkContents, (RepairFlag)repair); + logger->stopWork(); + to << static_cast<uint64_t>(errors); + break; + } + + case wopAddSignatures: { + Path path = readStorePath(*store, from); + auto sigs = readStrings<StringSet>(from); + logger->startWork(); + if (!trusted) { + throw Error("you are not privileged to add signatures"); + } + store->addSignatures(path, sigs); + logger->stopWork(); + to << 1; + break; + } + + case wopNarFromPath: { + auto path = readStorePath(*store, from); + logger->startWork(); + logger->stopWork(); + dumpPath(path, to); + break; + } + + case wopAddToStoreNar: { + bool repair; + bool dontCheckSigs; + ValidPathInfo info; + info.path = readStorePath(*store, from); + from >> info.deriver; + if (!info.deriver.empty()) { + store->assertStorePath(info.deriver); + } + info.narHash = Hash(readString(from), htSHA256); + info.references = readStorePaths<PathSet>(*store, from); + from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings<StringSet>(from); + from >> info.ca >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) { + dontCheckSigs = false; + } + if (!trusted) { + info.ultimate = false; + } + + std::string saved; + std::unique_ptr<Source> source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) { + source = std::make_unique<TunnelSource>(from); + } else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique<StringSource>(saved); + } + + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag)repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + + logger->stopWork(); + break; + } + + case wopQueryMissing: { + auto targets = readStorePaths<PathSet>(*store, from); + logger->startWork(); + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + unsigned long long downloadSize; + unsigned long long narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, + downloadSize, narSize); + logger->stopWork(); + to << willBuild << willSubstitute << unknown << downloadSize << narSize; + break; + } + + default: + throw Error(format("invalid operation %1%") % op); + } +} + +static void processConnection(bool trusted, const std::string& userName, + uid_t userId) { + MonitorFdHup monitor(from.fd); + + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) { + throw Error("protocol mismatch"); + } + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; + to.flush(); + unsigned int clientVersion = readInt(from); + + if (clientVersion < 0x10a) { + throw Error("the Nix client version is too old"); + } + + auto tunnelLogger = new TunnelLogger(clientVersion); + // logger = tunnelLogger; + + unsigned int opCount = 0; + + Finally finally([&]() { + _isInterrupted = false; + DLOG(INFO) << opCount << " operations"; + }); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && (readInt(from) != 0u)) { + setAffinityTo(readInt(from)); + } + + readInt(from); // obsolete reserveSpace + + /* Send startup error messages to the client. */ + tunnelLogger->startWork(); + + try { + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ + +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + + /* Open the store. */ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + auto store = openStore(settings.storeUri, params); + + store->createUser(userName, userId); + + tunnelLogger->stopWork(); + to.flush(); + + /* Process client requests. */ + while (true) { + WorkerOp op; + try { + op = (WorkerOp)readInt(from); + } catch (Interrupted& e) { + break; + } catch (EndOfFile& e) { + break; + } + + opCount++; + + try { + performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + } catch (Error& e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; + tunnelLogger->stopWork(false, e.msg(), e.status); + if (!errorAllowed) { + throw; + } + } catch (std::bad_alloc& e) { + tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + throw; + } + + to.flush(); + + assert(!tunnelLogger->state_.lock()->canSendStderr); + }; + + } catch (std::exception& e) { + tunnelLogger->stopWork(false, e.what(), 1); + to.flush(); + return; + } +} + +static void sigChldHandler(int sigNo) { + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + /* Reap all dead children. */ + while (waitpid(-1, nullptr, WNOHANG) > 0) { + ; + } + errno = saved_errno; +} + +static void setSigChldAction(bool autoReap) { + struct sigaction act; + struct sigaction oact; + act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact) != 0) { + throw SysError("setting SIGCHLD handler"); + } +} + +bool matchUser(const std::string& user, const std::string& group, + const Strings& users) { + if (find(users.begin(), users.end(), "*") != users.end()) { + return true; + } + + if (find(users.begin(), users.end(), user) != users.end()) { + return true; + } + + for (auto& i : users) { + if (std::string(i, 0, 1) == "@") { + if (group == std::string(i, 1)) { + return true; + } + struct group* gr = getgrnam(i.c_str() + 1); + if (gr == nullptr) { + continue; + } + for (char** mem = gr->gr_mem; *mem != nullptr; mem++) { + if (user == std::string(*mem)) { + return true; + } + } + } + } + + return false; +} + +struct PeerInfo { + bool pidKnown; + pid_t pid; + bool uidKnown; + uid_t uid; + bool gidKnown; + gid_t gid; +}; + +/* Get the identity of the caller, if possible. */ +static PeerInfo getPeerInfo(int remote) { + PeerInfo peer = {false, 0, false, 0, false, 0}; + +#if defined(SO_PEERCRED) + + ucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) { + throw SysError("getting peer credentials"); + } + peer = {true, cred.pid, true, cred.uid, true, cred.gid}; + +#elif defined(LOCAL_PEERCRED) + +#if !defined(SOL_LOCAL) +#define SOL_LOCAL 0 +#endif + + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = {false, 0, true, cred.cr_uid, false, 0}; + +#endif + + return peer; +} + +#define SD_LISTEN_FDS_START 3 + +static void daemonLoop(char** argv) { + if (chdir("/") == -1) { + throw SysError("cannot change current directory"); + } + + /* Get rid of children automatically; don't let them become + zombies. */ + setSigChldAction(true); + + AutoCloseFD fdSocket; + + /* Handle socket-based activation by systemd. */ + if (!getEnv("LISTEN_FDS").empty()) { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || + getEnv("LISTEN_FDS") != "1") { + throw Error("unexpected systemd environment variables"); + } + fdSocket = SD_LISTEN_FDS_START; + } + + /* Otherwise, create and bind to a Unix domain socket. */ + else { + /* Create and bind to a Unix domain socket. */ + fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); + if (!fdSocket) { + throw SysError("cannot create Unix domain socket"); + } + + std::string socketPath = settings.nixDaemonSocketFile; + + createDirs(dirOf(socketPath)); + + /* Urgh, sockaddr_un allows path names of only 108 characters. + So chdir to the socket directory so that we can pass a + relative path name. */ + if (chdir(dirOf(socketPath).c_str()) == -1) { + throw SysError("cannot change current directory"); + } + Path socketPathRel = "./" + baseNameOf(socketPath); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, socketPathRel.c_str(), sizeof(addr.sun_path)); + if (addr.sun_path[sizeof(addr.sun_path) - 1] != '\0') { + throw Error(format("socket path '%1%' is too long") % socketPathRel); + } + + unlink(socketPath.c_str()); + + /* Make sure that the socket is created with 0666 permission + (everybody can connect --- provided they have access to the + directory containing the socket). */ + mode_t oldMode = umask(0111); + int res = bind(fdSocket.get(), (struct sockaddr*)&addr, sizeof(addr)); + umask(oldMode); + if (res == -1) { + throw SysError(format("cannot bind to socket '%1%'") % socketPath); + } + + if (chdir("/") == -1) { /* back to the root */ + throw SysError("cannot change current directory"); + } + + if (listen(fdSocket.get(), 5) == -1) { + throw SysError(format("cannot listen on socket '%1%'") % socketPath); + } + } + + closeOnExec(fdSocket.get()); + + /* Loop accepting connections. */ + while (true) { + try { + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = + accept(fdSocket.get(), (struct sockaddr*)&remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) { + continue; + } + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + bool trusted = false; + PeerInfo peer = getPeerInfo(remote.get()); + + struct passwd* pw = peer.uidKnown ? getpwuid(peer.uid) : nullptr; + std::string user = pw != nullptr ? pw->pw_name : std::to_string(peer.uid); + + struct group* gr = peer.gidKnown ? getgrgid(peer.gid) : nullptr; + std::string group = + gr != nullptr ? gr->gr_name : std::to_string(peer.gid); + + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) { + trusted = true; + } + + if ((!trusted && !matchUser(user, group, allowedUsers)) || + group == settings.buildUsersGroup) { + throw Error( + format("user '%1%' is not allowed to connect to the Nix daemon") % + user); + } + + LOG(INFO) << "accepted connection from pid " + << (peer.pidKnown ? std::to_string(peer.pid) : "<unknown>") + << ", user " << (peer.uidKnown ? user : "<unknown>") + << (trusted ? " (trusted)" : ""); + + /* Fork a child to handle the connection. */ + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + startProcess( + [&]() { + fdSocket = -1; + + /* Background the daemon. */ + if (setsid() == -1) { + throw SysError(format("creating a new session")); + } + + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); + + /* For debugging, stuff the pid into argv[1]. */ + if (peer.pidKnown && (argv[1] != nullptr)) { + std::string processName = std::to_string(peer.pid); + strncpy(argv[1], processName.c_str(), strlen(argv[1])); + } + + /* Handle the connection. */ + from.fd = remote.get(); + to.fd = remote.get(); + processConnection(trusted, user, peer.uid); + + exit(0); + }, + options); + + } catch (Interrupted& e) { + return; + } catch (Error& e) { + LOG(ERROR) << "error processing connection: " << e.msg(); + } + } +} + +static int _main(int argc, char** argv) { + { + auto stdio = false; + + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--daemon") { + ; /* ignored for backwards compatibility */ + } else if (*arg == "--help") { + showManPage("nix-daemon"); + } else if (*arg == "--version") { + printVersion("nix-daemon"); + } else if (*arg == "--stdio") { + stdio = true; + } else { + return false; + } + return true; + }); + + if (stdio) { + if (getStoreType() == tDaemon) { + /* Forward on this connection to the real daemon */ + auto socketPath = settings.nixDaemonSocketFile; + auto s = socket(PF_UNIX, SOCK_STREAM, 0); + if (s == -1) { + throw SysError("creating Unix domain socket"); + } + + auto socketDir = dirOf(socketPath); + if (chdir(socketDir.c_str()) == -1) { + throw SysError(format("changing to socket directory '%1%'") % + socketDir); + } + + auto socketName = baseNameOf(socketPath); + auto addr = sockaddr_un{}; + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, socketName.c_str(), sizeof(addr.sun_path)); + if (addr.sun_path[sizeof(addr.sun_path) - 1] != '\0') { + throw Error(format("socket name %1% is too long") % socketName); + } + + if (connect(s, (struct sockaddr*)&addr, sizeof(addr)) == -1) { + throw SysError(format("cannot connect to daemon at %1%") % + socketPath); + } + + auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(s, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) { + throw SysError("waiting for data from client or server"); + } + if (FD_ISSET(s, &fds)) { + auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, + SPLICE_F_MOVE); + if (res == -1) { + throw SysError("splicing data from daemon socket to stdout"); + } + if (res == 0) { + throw EndOfFile("unexpected EOF from daemon socket"); + } + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, + SPLICE_F_MOVE); + if (res == -1) { + throw SysError("splicing data from stdin to daemon socket"); + } + if (res == 0) { + return 0; + } + } + } + } else { + processConnection(true, "root", 0); + } + } else { + daemonLoop(argv); + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/third_party/nix/src/nix-daemon/nix-daemon-proto.cc b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc new file mode 100644 index 000000000000..d6498e77c241 --- /dev/null +++ b/third_party/nix/src/nix-daemon/nix-daemon-proto.cc @@ -0,0 +1,799 @@ +#include "nix-daemon-proto.hh" + +#include <filesystem> +#include <ostream> +#include <sstream> +#include <streambuf> +#include <string> + +#include <absl/strings/str_cat.h> +#include <absl/strings/str_format.h> +#include <google/protobuf/empty.pb.h> +#include <google/protobuf/util/time_util.h> +#include <grpcpp/impl/codegen/server_context.h> +#include <grpcpp/impl/codegen/status.h> +#include <grpcpp/impl/codegen/status_code_enum.h> +#include <grpcpp/impl/codegen/sync_stream.h> + +#include "libmain/shared.hh" +#include "libproto/worker.grpc.pb.h" +#include "libproto/worker.pb.h" +#include "libstore/derivations.hh" +#include "libstore/local-store.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/hash.hh" +#include "libutil/proto.hh" +#include "libutil/serialise.hh" +#include "libutil/types.hh" + +namespace nix::daemon { + +using ::google::protobuf::util::TimeUtil; +using ::grpc::Status; +using ::nix::proto::PathInfo; +using ::nix::proto::StorePath; +using ::nix::proto::StorePaths; +using ::nix::proto::WorkerService; + +template <typename Request> +class RPCSource final : public Source { + public: + using Reader = grpc::ServerReader<Request>; + explicit RPCSource(Reader* reader) : reader_(reader) {} + + size_t read(unsigned char* data, size_t len) override { + auto got = buffer_.sgetn(reinterpret_cast<char*>(data), len); + if (got < len) { + Request msg; + if (!reader_->Read(&msg)) { + return got; + } + if (msg.add_oneof_case() != Request::kData) { + // TODO(grfn): Make Source::read return a StatusOr and get rid of this + // throw + throw Error( + "Invalid AddToStoreRequest: all messages except the first must " + "contain data"); + } + buffer_.sputn(msg.data().data(), msg.data().length()); + return got + read(data + got, len - got); + } + return got; + }; + + private: + std::stringbuf buffer_; + Reader* reader_; +}; + +// TODO(grfn): Make this some sort of pipe so we don't have to store data in +// memory +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink { + bool regular{true}; + std::string s; + + RetrieveRegularNARSink() {} + + void createDirectory(const Path& path) override { regular = false; } + + void receiveContents(unsigned char* data, unsigned int len) override { + s.append(reinterpret_cast<const char*>(data), len); + } + + void createSymlink(const Path& path, const std::string& target) override { + regular = false; + } +}; + +#define ASSERT_INPUT_STORE_PATH(path) \ + if (!store_->isStorePath(path)) { \ + return Status(grpc::StatusCode::INVALID_ARGUMENT, \ + absl::StrFormat("path '%s' is not in the Nix store", path)); \ + } + +class BuildLogStreambuf final : public std::streambuf { + public: + using Writer = grpc::ServerWriter<nix::proto::BuildEvent>; + explicit BuildLogStreambuf(Writer* writer) : writer_(writer) {} + + // TODO(grfn): buffer with a timeout so we don't have too many messages + std::streamsize xsputn(const char_type* s, std::streamsize n) override { + nix::proto::BuildEvent event; + event.mutable_build_log()->set_line(s, n); + writer_->Write(event); + return n; + } + + int_type overflow(int_type ch) override { + if (ch != traits_type::eof()) { + nix::proto::BuildEvent event; + event.mutable_build_log()->set_line(std::string(1, ch)); + writer_->Write(event); + } + return ch; + } + + private: + Writer* writer_{}; +}; + +class WorkerServiceImpl final : public WorkerService::Service { + public: + WorkerServiceImpl(nix::Store& store) : store_(&store) {} + + Status IsValidPath(grpc::ServerContext* context, const StorePath* request, + nix::proto::IsValidPathResponse* response) override { + return HandleExceptions( + [&]() -> Status { + const auto& path = request->path(); + response->set_is_valid(store_->isValidPath(path)); + + return Status::OK; + }, + __FUNCTION__); + } + + Status HasSubstitutes(grpc::ServerContext* context, const StorePath* request, + nix::proto::HasSubstitutesResponse* response) override { + return HandleExceptions( + [&]() -> Status { + const auto& path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + PathSet res = store_->querySubstitutablePaths({path}); + response->set_has_substitutes(res.find(path) != res.end()); + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryReferrers(grpc::ServerContext* context, const StorePath* request, + StorePaths* response) override { + return HandleExceptions( + [&]() -> Status { + const auto& path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + + PathSet paths; + store_->queryReferrers(path, paths); + + for (const auto& path : paths) { + response->add_paths(path); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status AddToStore(grpc::ServerContext* context, + grpc::ServerReader<nix::proto::AddToStoreRequest>* reader, + nix::proto::StorePath* response) override { + return HandleExceptions( + [&]() -> Status { + proto::AddToStoreRequest metadata_request; + auto has_metadata = reader->Read(&metadata_request); + + if (!has_metadata || !metadata_request.has_meta()) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "Metadata must be set before sending file content"); + } + + auto meta = metadata_request.meta(); + RPCSource source(reader); + auto opt_hash_type = hash_type_from(meta.hash_type()); + if (!opt_hash_type) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "Invalid hash type"); + } + + std::string* data; + RetrieveRegularNARSink nar; + TeeSource saved_nar(source); + + if (meta.recursive()) { + // TODO(grfn): Don't store the full data in memory, instead just + // make addToStoreFromDump take a Source + ParseSink sink; + parseDump(sink, saved_nar); + data = &(*saved_nar.data); + } else { + parseDump(nar, source); + if (!nar.regular) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "Regular file expected"); + } + data = &nar.s; + } + + auto local_store = store_.dynamic_pointer_cast<LocalStore>(); + if (!local_store) { + return Status(grpc::StatusCode::FAILED_PRECONDITION, + "operation is only supported by LocalStore"); + } + + auto path = local_store->addToStoreFromDump( + *data, meta.base_name(), meta.recursive(), opt_hash_type.value()); + + response->set_path(path); + + return Status::OK; + }, + __FUNCTION__); + } + + Status AddToStoreNar( + grpc::ServerContext* context, + grpc::ServerReader<nix::proto::AddToStoreNarRequest>* reader, + google::protobuf::Empty*) override { + return HandleExceptions( + [&]() -> Status { + proto::AddToStoreNarRequest path_info_request; + auto has_path_info = reader->Read(&path_info_request); + if (!has_path_info || !path_info_request.has_path_info()) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "Path info must be set before sending nar content"); + } + + auto path_info = path_info_request.path_info(); + + ValidPathInfo info; + info.path = path_info.path().path(); + info.deriver = path_info.deriver().path(); + + if (!info.deriver.empty()) { + ASSERT_INPUT_STORE_PATH(info.deriver); + } + + auto nar_hash = Hash::deserialize(path_info.nar_hash(), htSHA256); + + if (!nar_hash.ok()) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + std::string(nar_hash.status().message())); + } + + info.narHash = *nar_hash; + for (const auto& ref : path_info.references()) { + info.references.insert(ref); + } + info.registrationTime = + TimeUtil::TimestampToTimeT(path_info.registration_time()); + info.narSize = path_info.nar_size(); + info.ultimate = path_info.ultimate(); + for (const auto& sig : path_info.sigs()) { + info.sigs.insert(sig); + } + info.ca = path_info.ca(); + + auto repair = path_info.repair(); + auto check_sigs = path_info.check_sigs(); + + std::string saved; + RPCSource source(reader); + store_->addToStore(info, source, repair ? Repair : NoRepair, + check_sigs ? CheckSigs : NoCheckSigs, nullptr); + + return Status::OK; + }, + __FUNCTION__); + } + + Status AddTextToStore( + grpc::ServerContext*, + grpc::ServerReader<nix::proto::AddTextToStoreRequest>* reader, + nix::proto::StorePath* response) override { + return HandleExceptions( + [&]() -> Status { + proto::AddTextToStoreRequest request; + auto has_metadata = reader->Read(&request); + if (!has_metadata || !request.has_meta()) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "Metadata must be set before sending content"); + } + + proto::AddTextToStoreRequest_Metadata meta = request.meta(); + + PathSet references; + for (const auto& ref : meta.references()) { + references.insert(ref); + } + + std::string content; + content.reserve(meta.size()); + while (reader->Read(&request)) { + if (request.add_oneof_case() != request.kData) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "All requests except the first must contain data"); + } + + content.append(request.data()); + } + + auto path = store_->addTextToStore(meta.name(), content, references); + response->set_path(path); + return Status::OK; + }, + __FUNCTION__); + } + + Status BuildPaths( + grpc::ServerContext*, const nix::proto::BuildPathsRequest* request, + grpc::ServerWriter<nix::proto::BuildEvent>* writer) override { + return HandleExceptions( + [&]() -> Status { + PathSet drvs; + for (const auto& drv : request->drvs()) { + drvs.insert(drv); + } + auto mode = BuildModeFrom(request->mode()); + + if (!mode.has_value()) { + return Status(grpc::StatusCode::INTERNAL, "Invalid build mode"); + } + + BuildLogStreambuf log_buffer(writer); + std::ostream log_sink(&log_buffer); + + // TODO(grfn): If mode is repair and not trusted, we need to return an + // error here (but we can't yet because we don't know anything about + // trusted users) + return nix::util::proto::AbslToGRPCStatus( + store_->buildPaths(log_sink, drvs, mode.value())); + }, + __FUNCTION__); + } + + Status EnsurePath(grpc::ServerContext* context, + const nix::proto::StorePath* request, + google::protobuf::Empty*) override { + auto path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + return HandleExceptions( + [&]() -> Status { + store_->ensurePath(path); + return Status::OK; + }, + __FUNCTION__); + } + + Status AddTempRoot(grpc::ServerContext*, const nix::proto::StorePath* request, + google::protobuf::Empty*) override { + auto path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + + return HandleExceptions( + [&]() -> Status { + store_->addTempRoot(path); + return Status::OK; + }, + __FUNCTION__); + } + + Status AddIndirectRoot(grpc::ServerContext*, + const nix::proto::StorePath* request, + google::protobuf::Empty*) override { + auto path = std::filesystem::canonical(request->path()); + ASSERT_INPUT_STORE_PATH(path); + + return HandleExceptions( + [&]() -> Status { + store_->addIndirectRoot(path); + return Status::OK; + }, + __FUNCTION__); + } + + Status SyncWithGC(grpc::ServerContext*, const google::protobuf::Empty*, + google::protobuf::Empty*) override { + return HandleExceptions( + [&]() -> Status { + store_->syncWithGC(); + return Status::OK; + }, + __FUNCTION__); + } + + Status FindRoots(grpc::ServerContext*, const google::protobuf::Empty*, + nix::proto::FindRootsResponse* response) override { + return HandleExceptions( + [&]() -> Status { + auto roots = store_->findRoots(false); + for (const auto& [target, links] : roots) { + StorePaths link_paths; + for (const auto& link : links) { + link_paths.add_paths(link); + } + response->mutable_roots()->insert({target, link_paths}); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status CollectGarbage(grpc::ServerContext*, + const proto::CollectGarbageRequest* request, + proto::CollectGarbageResponse* response) override { + return HandleExceptions( + [&]() -> Status { + GCOptions options; + auto action = GCActionFromProto(request->action()); + if (!action.has_value()) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "Invalid GC action"); + } + + options.action = action.value(); + for (const auto& path : request->paths_to_delete()) { + options.pathsToDelete.insert(path); + } + options.ignoreLiveness = request->ignore_liveness(); + options.maxFreed = request->max_freed(); + + if (options.ignoreLiveness) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, + "you are not allowed to ignore liveness"); + } + + GCResults results; + store_->collectGarbage(options, results); + + for (const auto& path : results.paths) { + response->add_deleted_paths(path); + } + response->set_bytes_freed(results.bytesFreed); + + return Status::OK; + }, + __FUNCTION__); + } + + Status QuerySubstitutablePathInfos( + grpc::ServerContext*, const StorePaths* request, + nix::proto::SubstitutablePathInfos* response) override { + return HandleExceptions( + [&]() -> Status { + SubstitutablePathInfos infos; + PathSet paths; + for (const auto& path : request->paths()) { + paths.insert(path); + } + store_->querySubstitutablePathInfos(paths, infos); + for (const auto& [path, path_info] : infos) { + auto proto_path_info = response->add_path_infos(); + proto_path_info->mutable_path()->set_path(path); + proto_path_info->mutable_deriver()->set_path(path_info.deriver); + for (const auto& ref : path_info.references) { + proto_path_info->add_references(ref); + } + proto_path_info->set_download_size(path_info.downloadSize); + proto_path_info->set_nar_size(path_info.narSize); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryValidDerivers(grpc::ServerContext* context, + const StorePath* request, + StorePaths* response) override { + return HandleExceptions( + [&]() -> Status { + const auto& path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + + PathSet paths = store_->queryValidDerivers(path); + + for (const auto& path : paths) { + response->add_paths(path); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryDerivationOutputs(grpc::ServerContext* context, + const StorePath* request, + StorePaths* response) override { + return HandleExceptions( + [&]() -> Status { + const auto& path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + + PathSet paths = store_->queryDerivationOutputs(path); + + for (const auto& path : paths) { + response->add_paths(path); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryAllValidPaths(grpc::ServerContext* context, + const google::protobuf::Empty* request, + StorePaths* response) override { + return HandleExceptions( + [&]() -> Status { + const auto paths = store_->queryAllValidPaths(); + for (const auto& path : paths) { + response->add_paths(path); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryPathInfo(grpc::ServerContext* context, const StorePath* request, + PathInfo* response) override { + return HandleExceptions( + [&]() -> Status { + auto path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + + response->mutable_path()->set_path(path); + try { + auto info = store_->queryPathInfo(path); + response->mutable_deriver()->set_path(info->deriver); + response->set_nar_hash( + reinterpret_cast<const char*>(&info->narHash.hash[0]), + info->narHash.hashSize); + + for (const auto& reference : info->references) { + response->add_references(reference); + } + + *response->mutable_registration_time() = + google::protobuf::util::TimeUtil::TimeTToTimestamp( + info->registrationTime); + + response->set_nar_size(info->narSize); + response->set_ultimate(info->ultimate); + + for (const auto& sig : info->sigs) { + response->add_sigs(sig); + } + + response->set_ca(info->ca); + + return Status::OK; + } catch (InvalidPath& e) { + return Status(grpc::StatusCode::INVALID_ARGUMENT, e.msg()); + } + }, + __FUNCTION__); + } + + Status QueryDerivationOutputNames( + grpc::ServerContext* context, const StorePath* request, + nix::proto::DerivationOutputNames* response) override { + return HandleExceptions( + [&]() -> Status { + auto path = request->path(); + ASSERT_INPUT_STORE_PATH(path); + auto names = store_->queryDerivationOutputNames(path); + for (const auto& name : names) { + response->add_names(name); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryPathFromHashPart(grpc::ServerContext* context, + const nix::proto::HashPart* request, + StorePath* response) override { + return HandleExceptions( + [&]() -> Status { + auto hash_part = request->hash_part(); + auto path = store_->queryPathFromHashPart(hash_part); + ASSERT_INPUT_STORE_PATH(path); + response->set_path(path); + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryValidPaths(grpc::ServerContext* context, + const StorePaths* request, + StorePaths* response) override { + return HandleExceptions( + [&]() -> Status { + std::set<Path> paths; + for (const auto& path : request->paths()) { + ASSERT_INPUT_STORE_PATH(path); + paths.insert(path); + } + + auto res = store_->queryValidPaths(paths); + + for (const auto& path : res) { + response->add_paths(path); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status QuerySubstitutablePaths(grpc::ServerContext* context, + const StorePaths* request, + StorePaths* response) override { + return HandleExceptions( + [&]() -> Status { + std::set<Path> paths; + for (const auto& path : request->paths()) { + ASSERT_INPUT_STORE_PATH(path); + paths.insert(path); + } + + auto res = store_->querySubstitutablePaths(paths); + + for (const auto& path : res) { + response->add_paths(path); + } + + return Status::OK; + }, + __FUNCTION__); + } + + Status OptimiseStore(grpc::ServerContext* context, + const google::protobuf::Empty* request, + google::protobuf::Empty* response) override { + return HandleExceptions( + [&]() -> Status { + store_->optimiseStore(); + return Status::OK; + }, + __FUNCTION__); + } + + Status VerifyStore(grpc::ServerContext* context, + const nix::proto::VerifyStoreRequest* request, + nix::proto::VerifyStoreResponse* response) override { + return HandleExceptions( + [&]() -> Status { + auto errors = + store_->verifyStore(request->check_contents(), + static_cast<RepairFlag>(request->repair())); + + response->set_errors(errors); + + return Status::OK; + }, + __FUNCTION__); + } + + Status BuildDerivation( + grpc::ServerContext*, const nix::proto::BuildDerivationRequest* request, + grpc::ServerWriter<nix::proto::BuildEvent>* writer) override { + return HandleExceptions( + [&]() -> Status { + auto drv_path = request->drv_path().path(); + ASSERT_INPUT_STORE_PATH(drv_path); + auto drv = BasicDerivation::from_proto(&request->derivation()); + + auto build_mode = nix::BuildModeFrom(request->build_mode()); + if (!build_mode) { + return Status(grpc::StatusCode::INTERNAL, "Invalid build mode"); + } + + BuildLogStreambuf log_buffer(writer); + std::ostream log_sink(&log_buffer); + BuildResult res = + store_->buildDerivation(log_sink, drv_path, drv, *build_mode); + + proto::BuildResult proto_res{}; + proto_res.set_status(res.status_to_proto()); + + if (!res.errorMsg.empty()) { + proto_res.set_msg(res.errorMsg); + } + + proto::BuildEvent event{}; + *event.mutable_result() = proto_res; + + writer->Write(event); + return Status::OK; + }, + __FUNCTION__); + } + + Status AddSignatures(grpc::ServerContext* context, + const nix::proto::AddSignaturesRequest* request, + google::protobuf::Empty* response) override { + return HandleExceptions( + [&]() -> Status { + auto path = request->path().path(); + ASSERT_INPUT_STORE_PATH(path); + + StringSet sigs; + sigs.insert(request->sigs().sigs().begin(), + request->sigs().sigs().end()); + + store_->addSignatures(path, sigs); + + return Status::OK; + }, + __FUNCTION__); + } + + Status QueryMissing(grpc::ServerContext* context, const StorePaths* request, + nix::proto::QueryMissingResponse* response) override { + return HandleExceptions( + [&]() -> Status { + std::set<Path> targets; + for (auto& path : request->paths()) { + ASSERT_INPUT_STORE_PATH(path); + targets.insert(path); + } + PathSet will_build; + PathSet will_substitute; + PathSet unknown; + // TODO(grfn): Switch to concrete size type + unsigned long long download_size; + unsigned long long nar_size; + + store_->queryMissing(targets, will_build, will_substitute, unknown, + download_size, nar_size); + for (auto& path : will_build) { + response->add_will_build(path); + } + for (auto& path : will_substitute) { + response->add_will_substitute(path); + } + for (auto& path : unknown) { + response->add_unknown(path); + } + response->set_download_size(download_size); + response->set_nar_size(nar_size); + + return Status::OK; + }, + __FUNCTION__); + }; + + Status GetBuildLog(grpc::ServerContext* context, const StorePath* request, + proto::BuildLog* response) override { + return HandleExceptions( + [&]() -> Status { + const auto log = store_->getBuildLog(request->path()); + if (log) { + response->set_build_log(*log); + } + return Status::OK; + }, + __FUNCTION__); + } + + private: + Status HandleExceptions(std::function<Status(void)> fn, + absl::string_view methodName) { + try { + return fn(); + } catch (Unsupported& e) { + return Status(grpc::StatusCode::UNIMPLEMENTED, + absl::StrCat(methodName, " is not supported: ", e.what())); + } catch (Error& e) { + return Status(grpc::StatusCode::INTERNAL, e.what()); + } + // add more specific Error-Status mappings above + } + + ref<nix::Store> store_; +}; + +WorkerService::Service* NewWorkerService(nix::Store& store) { + return new WorkerServiceImpl(store); +} + +} // namespace nix::daemon diff --git a/third_party/nix/src/nix-daemon/nix-daemon-proto.hh b/third_party/nix/src/nix-daemon/nix-daemon-proto.hh new file mode 100644 index 000000000000..ca871213eb60 --- /dev/null +++ b/third_party/nix/src/nix-daemon/nix-daemon-proto.hh @@ -0,0 +1,12 @@ +#pragma once + +#include <memory> + +#include "libproto/worker.grpc.pb.h" +#include "libstore/store-api.hh" + +namespace nix::daemon { + +nix::proto::WorkerService::Service* NewWorkerService(nix::Store&); + +} // namespace nix::daemon diff --git a/third_party/nix/src/nix-daemon/nix-daemon.cc b/third_party/nix/src/nix-daemon/nix-daemon.cc new file mode 100644 index 000000000000..0551625a3e13 --- /dev/null +++ b/third_party/nix/src/nix-daemon/nix-daemon.cc @@ -0,0 +1,201 @@ +#include <filesystem> + +#include <absl/flags/flag.h> +#include <absl/flags/parse.h> +#include <absl/flags/usage_config.h> +#include <absl/strings/str_format.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <grpcpp/security/server_credentials.h> +#include <grpcpp/server.h> +#include <grpcpp/server_builder.h> +#include <grpcpp/server_posix.h> +#include <sys/socket.h> +#include <sys/un.h> +#include <systemd/sd-daemon.h> + +#include "libmain/shared.hh" // TODO(tazjin): can this be removed? +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/util.hh" +#include "nix-daemon-proto.hh" +#include "nix-daemon/nix-daemon-proto.hh" +#include "nix/legacy.hh" + +ABSL_FLAG(bool, pipe, false, "Use pipes for daemon communication"); + +namespace nix::daemon { + +using grpc::Server; +using grpc::ServerBuilder; + +namespace { + +// TODO(grfn): There has to be a better way to do this - this was ported +// verbatim from the old daemon implementation without much critical evaluation. +static int ForwardToSocket(nix::Path socket_path) { + // Forward on this connection to the real daemon + int sockfd = socket(PF_UNIX, SOCK_STREAM, 0); + if (sockfd == -1) { + throw SysError("creating Unix domain socket"); + } + + auto socketDir = dirOf(socket_path); + if (chdir(socketDir.c_str()) == -1) { + throw SysError(format("changing to socket directory '%1%'") % socketDir); + } + + auto socketName = baseNameOf(socket_path); + auto addr = sockaddr_un{}; + addr.sun_family = AF_UNIX; + if (socketName.size() + 1 >= sizeof(addr.sun_path)) { + throw Error(format("socket name %1% is too long") % socketName); + } + strncpy(addr.sun_path, socketName.c_str(), sizeof(addr.sun_family)); + + if (connect(sockfd, reinterpret_cast<struct sockaddr*>(&addr), + sizeof(addr)) == -1) { + throw SysError(format("cannot connect to daemon at %1%") % socket_path); + } + + auto nfds = (sockfd > STDIN_FILENO ? sockfd : STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(sockfd, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) { + throw SysError("waiting for data from client or server"); + } + if (FD_ISSET(sockfd, &fds)) { + auto res = splice(sockfd, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, + SPLICE_F_MOVE); + if (res == -1) { + throw SysError("splicing data from daemon socket to stdout"); + } + if (res == 0) { + throw EndOfFile("unexpected EOF from daemon socket"); + } + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, sockfd, nullptr, SSIZE_MAX, + SPLICE_F_MOVE); + if (res == -1) { + throw SysError("splicing data from stdin to daemon socket"); + } + if (res == 0) { + return 0; + } + } + } +} + +void SetNonBlocking(int fd) { + int flags = fcntl(fd, F_GETFL); // NOLINT + PCHECK(flags != 0) << "Error getting socket flags"; + PCHECK(fcntl( // NOLINT + fd, F_SETFL, flags | O_NONBLOCK) == 0) + << "Could not set socket flags"; +} + +} // namespace + +int RunServer() { + Store::Params params; + params["path-info-cache-size"] = "0"; + auto store = openStore(settings.storeUri, params); + auto worker = NewWorkerService(*store); + ServerBuilder builder; + builder.RegisterService(worker); + + auto n_fds = sd_listen_fds(0); + + if (n_fds > 1) { + LOG(FATAL) << "Too many file descriptors (" << n_fds + << ") received from systemd socket activation"; + } + + std::filesystem::path socket_path; + + if (n_fds == 0) { + socket_path = settings.nixDaemonSocketFile; + std::filesystem::create_directories(socket_path.parent_path()); + auto socket_addr = absl::StrFormat("unix://%s", socket_path); + builder.AddListeningPort(socket_addr, grpc::InsecureServerCredentials()); + } + + std::unique_ptr<Server> server(builder.BuildAndStart()); + + if (!server) { + LOG(FATAL) << "Error building server"; + return 1; + } + + // We have been systemd socket-activated - instead of asking grpc to make the + // socket path for us, start our own accept loop and pass file descriptors to + // grpc. + // + // This approach was *somewhat* adapted from + // https://gist.github.com/yorickvP/8d523a4df2b10c5812fa7789e82b7c1b - at some + // point we'd like gRPC to do it for us, though - see + // https://github.com/grpc/grpc/issues/19133 + if (n_fds == 1) { + int socket_fd = SD_LISTEN_FDS_START; + // Only used for logging + socket_path = readLink(absl::StrFormat("/proc/self/fd/%d", socket_fd)); + + PCHECK(sd_notify(0, "READY=1") == 0) << "Error notifying systemd"; + for (;;) { + try { + struct sockaddr_un remote_addr {}; + socklen_t remote_addr_len = sizeof(remote_addr); + int remote_fd = + accept(socket_fd, + reinterpret_cast<struct sockaddr*>(&remote_addr), // NOLINT + &remote_addr_len); + checkInterrupt(); + if (!remote_fd) { + if (errno == EINTR) { + continue; + } + PCHECK(false) << "error accepting connection"; + } + + LOG(INFO) << "Accepted remote connection on fd " << remote_fd; + SetNonBlocking(remote_fd); + grpc::AddInsecureChannelFromFd(server.get(), remote_fd); + } catch (Interrupted& e) { + return -1; + } catch (Error& e) { + LOG(ERROR) << "error processing connection: " << e.msg(); + } + } + } + + LOG(INFO) << "Nix daemon listening at " << socket_path; + server->Wait(); + return 0; +} + +} // namespace nix::daemon + +int main(int argc, char** argv) { // NOLINT + FLAGS_logtostderr = true; + google::InitGoogleLogging(argv[0]); // NOLINT + + absl::SetFlagsUsageConfig({.version_string = [] { return nix::nixVersion; }}); + absl::ParseCommandLine(argc, argv); + + if (absl::GetFlag(FLAGS_pipe)) { + if (nix::getStoreType() == nix::tDaemon) { + return nix::daemon::ForwardToSocket(nix::settings.nixDaemonSocketFile); + } else { + // TODO(grfn): Need to launch a server on stdin here - upstream calls + // processConnection(true, "root", 0); + LOG(ERROR) << "not implemented"; + return 1; + } + } + + return nix::daemon::RunServer(); +} diff --git a/third_party/nix/src/nix-env/nix-env.cc b/third_party/nix/src/nix-env/nix-env.cc new file mode 100644 index 000000000000..15f12abd97e9 --- /dev/null +++ b/third_party/nix/src/nix-env/nix-env.cc @@ -0,0 +1,1543 @@ +#include <algorithm> +#include <cerrno> +#include <ctime> +#include <iostream> +#include <sstream> + +#include <absl/strings/match.h> +#include <absl/strings/numbers.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "libexpr/attr-path.hh" +#include "libexpr/common-eval-args.hh" +#include "libexpr/eval.hh" +#include "libexpr/get-drvs.hh" +#include "libexpr/names.hh" +#include "libexpr/value-to-json.hh" +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/profiles.hh" +#include "libstore/store-api.hh" +#include "libutil/json.hh" +#include "libutil/status.hh" +#include "libutil/util.hh" +#include "libutil/xml-writer.hh" +#include "nix-env/user-env.hh" +#include "nix/legacy.hh" + +using namespace nix; +using std::cout; + +using InstallSourceType = enum { + srcNixExprDrvs, + srcNixExprs, + srcStorePaths, + srcProfile, + srcAttrPath, + srcUnknown +}; + +struct InstallSourceInfo { + InstallSourceType type; + Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + Path profile; /* for srcProfile */ + std::string systemFilter; /* for srcNixExprDrvs */ + std::unique_ptr<Bindings> autoArgs; +}; + +struct Globals { + InstallSourceInfo instSource; + Path profile; + std::shared_ptr<EvalState> state; + bool dryRun; + bool preserveInstalled; + bool removeAll; + std::string forceName; + bool prebuiltOnly; +}; + +using Operation = void (*)(Globals&, Strings, Strings); + +static std::string needArg(Strings::iterator& i, Strings& args, + const std::string& arg) { + if (i == args.end()) { + throw UsageError(format("'%1%' requires an argument") % arg); + } + return *i++; +} + +static bool parseInstallSourceOptions(Globals& globals, Strings::iterator& i, + Strings& args, const std::string& arg) { + if (arg == "--from-expression" || arg == "-E") { + globals.instSource.type = srcNixExprs; + } else if (arg == "--from-profile") { + globals.instSource.type = srcProfile; + globals.instSource.profile = needArg(i, args, arg); + } else if (arg == "--attr" || arg == "-A") { + globals.instSource.type = srcAttrPath; + } else { + return false; + } + return true; +} + +static bool isNixExpr(const Path& path, struct stat& st) { + return S_ISREG(st.st_mode) || + (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); +} + +static void getAllExprs(EvalState& state, const Path& path, StringSet& attrs, + Value& v) { + StringSet namesSorted; + for (auto& i : readDirectory(path)) { + namesSorted.insert(i.name); + } + + for (auto& i : namesSorted) { + /* Ignore the manifest.nix used by profiles. This is + necessary to prevent it from showing up in channels (which + are implemented using profiles). */ + if (i == "manifest.nix") { + continue; + } + + Path path2 = path + "/" + i; + + struct stat st; + if (stat(path2.c_str(), &st) == -1) { + continue; // ignore dangling symlinks in ~/.nix-defexpr + } + + if (isNixExpr(path2, st) && + (!S_ISREG(st.st_mode) || absl::EndsWith(path2, ".nix"))) { + /* Strip off the `.nix' filename suffix (if applicable), + otherwise the attribute cannot be selected with the + `-A' option. Useful if you want to stick a Nix + expression directly in ~/.nix-defexpr. */ + std::string attrName = i; + if (absl::EndsWith(attrName, ".nix")) { + attrName = std::string(attrName, 0, attrName.size() - 4); + } + if (attrs.find(attrName) != attrs.end()) { + LOG(WARNING) << "name collision in input Nix expressions, skipping '" + << path2 << "'"; + continue; + } + attrs.insert(attrName); + /* Load the expression on demand. */ + Value& vFun = state.getBuiltin("import"); + Value& vArg(*state.allocValue()); + mkString(vArg, path2); + mkApp(*state.allocAttr(v, state.symbols.Create(attrName)), vFun, vArg); + } else if (S_ISDIR(st.st_mode)) { + /* `path2' is a directory (with no default.nix in it); + recurse into it. */ + getAllExprs(state, path2, attrs, v); + } + } +} + +static void loadSourceExpr(EvalState& state, const Path& path, Value& v) { + struct stat st; + if (stat(path.c_str(), &st) == -1) { + throw SysError(format("getting information about '%1%'") % path); + } + + if (isNixExpr(path, st)) { + state.evalFile(path, v); + } + + /* The path is a directory. Put the Nix expressions in the + directory in a set, with the file name of each expression as + the attribute name. Recurse into subdirectories (but keep the + set flat, not nested, to make it easier for a user to have a + ~/.nix-defexpr directory that includes some system-wide + directory). */ + else if (S_ISDIR(st.st_mode)) { + state.mkAttrs(v, 1024); + state.mkList(*state.allocAttr(v, state.symbols.Create("_combineChannels"))); + StringSet attrs; + getAllExprs(state, path, attrs, v); + } + + else { + throw Error("path '%s' is not a directory or a Nix expression", path); + } +} + +static void loadDerivations(EvalState& state, const Path& nixExprPath, + const std::string& systemFilter, Bindings* autoArgs, + const std::string& pathPrefix, DrvInfos& elems) { + Value vRoot; + loadSourceExpr(state, nixExprPath, vRoot); + + Value& v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + + getDerivations(state, v, pathPrefix, autoArgs, elems, true); + + /* Filter out all derivations not applicable to the current + system. */ + for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { + j = i; + j++; + if (systemFilter != "*" && i->querySystem() != systemFilter) { + elems.erase(i); + } + } +} + +static long getPriority(EvalState& state, DrvInfo& drv) { + return drv.queryMetaInt("priority", 0); +} + +static long comparePriorities(EvalState& state, DrvInfo& drv1, DrvInfo& drv2) { + return getPriority(state, drv2) - getPriority(state, drv1); +} + +// FIXME: this function is rather slow since it checks a single path +// at a time. +static bool isPrebuilt(EvalState& state, DrvInfo& elem) { + Path path = elem.queryOutPath(); + if (state.store->isValidPath(path)) { + return true; + } + PathSet ps = state.store->querySubstitutablePaths({path}); + return ps.find(path) != ps.end(); +} + +static void checkSelectorUse(DrvNames& selectors) { + /* Check that all selectors have been used. */ + for (auto& i : selectors) { + if (i.hits == 0 && i.fullName != "*") { + throw Error(format("selector '%1%' matches no derivations") % i.fullName); + } + } +} + +static DrvInfos filterBySelector(EvalState& state, const DrvInfos& allElems, + const Strings& args, bool newestOnly) { + DrvNames selectors = drvNamesFromArgs(args); + if (selectors.empty()) { + selectors.push_back(DrvName("*")); + } + + DrvInfos elems; + std::set<unsigned int> done; + + for (auto& i : selectors) { + using Matches = std::list<std::pair<DrvInfo, unsigned int> >; + Matches matches; + unsigned int n = 0; + for (auto j = allElems.begin(); j != allElems.end(); ++j, ++n) { + DrvName drvName(j->queryName()); + if (i.matches(drvName)) { + i.hits++; + matches.push_back(std::pair<DrvInfo, unsigned int>(*j, n)); + } + } + + /* If `newestOnly', if a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ + if (newestOnly) { + /* Map from package names to derivations. */ + using Newest = std::map<std::string, std::pair<DrvInfo, unsigned int> >; + Newest newest; + StringSet multiple; + + for (auto& j : matches) { + DrvName drvName(j.first.queryName()); + long d = 1; + + auto k = newest.find(drvName.name); + + if (k != newest.end()) { + d = j.first.querySystem() == k->second.first.querySystem() ? 0 + : j.first.querySystem() == settings.thisSystem ? 1 + : k->second.first.querySystem() == settings.thisSystem ? -1 + : 0; + if (d == 0) { + d = comparePriorities(state, j.first, k->second.first); + } + if (d == 0) { + d = compareVersions(drvName.version, + DrvName(k->second.first.queryName()).version); + } + } + + if (d > 0) { + newest.erase(drvName.name); + newest.insert(Newest::value_type(drvName.name, j)); + multiple.erase(j.first.queryName()); + } else if (d == 0) { + multiple.insert(j.first.queryName()); + } + } + + matches.clear(); + for (auto& j : newest) { + if (multiple.find(j.second.first.queryName()) != multiple.end()) { + LOG(WARNING) << "warning: there are multiple derivations named '" + << j.second.first.queryName() + << "'; using the first one"; + } + matches.push_back(j.second); + } + } + + /* Insert only those elements in the final list that we + haven't inserted before. */ + for (auto& j : matches) { + if (done.find(j.second) == done.end()) { + done.insert(j.second); + elems.push_back(j.first); + } + } + } + + checkSelectorUse(selectors); + + return elems; +} + +static bool isPath(const std::string& s) { + return s.find('/') != std::string::npos; +} + +static void queryInstSources(EvalState& state, InstallSourceInfo& instSource, + const Strings& args, DrvInfos& elems, + bool newestOnly) { + InstallSourceType type = instSource.type; + if (type == srcUnknown && !args.empty() && isPath(args.front())) { + type = srcStorePaths; + } + + switch (type) { + /* Get the available user environment elements from the + derivations specified in a Nix expression, including only + those with names matching any of the names in `args'. */ + case srcUnknown: + case srcNixExprDrvs: { + /* Load the derivations from the (default or specified) + Nix expression. */ + DrvInfos allElems; + loadDerivations(state, instSource.nixExprPath, instSource.systemFilter, + instSource.autoArgs.get(), "", allElems); + + elems = filterBySelector(state, allElems, args, newestOnly); + + break; + } + + /* Get the available user environment elements from the Nix + expressions specified on the command line; these should be + functions that take the default Nix expression file as + argument, e.g., if the file is `./foo.nix', then the + argument `x: x.bar' is equivalent to `(x: x.bar) + (import ./foo.nix)' = `(import ./foo.nix).bar'. */ + case srcNixExprs: { + Value vArg; + loadSourceExpr(state, instSource.nixExprPath, vArg); + + for (auto& i : args) { + Expr* eFun = state.parseExprFromString(i, absPath(".")); + Value vFun; + Value vTmp; + state.eval(eFun, vFun); + mkApp(vTmp, vFun, vArg); + getDerivations(state, vTmp, "", instSource.autoArgs.get(), elems, true); + } + + break; + } + + /* The available user environment elements are specified as a + list of store paths (which may or may not be + derivations). */ + case srcStorePaths: { + for (auto& i : args) { + Path path = state.store->followLinksToStorePath(i); + + std::string name = baseNameOf(path); + std::string::size_type dash = name.find('-'); + if (dash != std::string::npos) { + name = std::string(name, dash + 1); + } + + DrvInfo elem(state, "", nullptr); + elem.setName(name); + + if (isDerivation(path)) { + elem.setDrvPath(path); + elem.setOutPath( + state.store->derivationFromPath(path).findOutput("out")); + if (name.size() >= drvExtension.size() && + std::string(name, name.size() - drvExtension.size()) == + drvExtension) { + name = std::string(name, 0, name.size() - drvExtension.size()); + } + } else { + elem.setOutPath(path); + } + + elems.push_back(elem); + } + + break; + } + + /* Get the available user environment elements from another + user environment. These are then filtered as in the + `srcNixExprDrvs' case. */ + case srcProfile: { + elems = filterBySelector(state, queryInstalled(state, instSource.profile), + args, newestOnly); + break; + } + + case srcAttrPath: { + Value vRoot; + loadSourceExpr(state, instSource.nixExprPath, vRoot); + for (auto& i : args) { + Value& v( + *findAlongAttrPath(state, i, instSource.autoArgs.get(), vRoot)); + getDerivations(state, v, "", instSource.autoArgs.get(), elems, true); + } + break; + } + } +} + +static void printMissing(EvalState& state, DrvInfos& elems) { + PathSet targets; + for (auto& i : elems) { + Path drvPath = i.queryDrvPath(); + if (!drvPath.empty()) { + targets.insert(drvPath); + } else { + targets.insert(i.queryOutPath()); + } + } + + printMissing(state.store, targets); +} + +static bool keep(DrvInfo& drv) { return drv.queryMetaBool("keep", false); } + +static void installDerivations(Globals& globals, const Strings& args, + const Path& profile) { + DLOG(INFO) << "installing derivations"; + + /* Get the set of user environment elements to be installed. */ + DrvInfos newElems; + DrvInfos newElemsTmp; + queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); + + /* If --prebuilt-only is given, filter out source-only packages. */ + for (auto& i : newElemsTmp) { + if (!globals.prebuiltOnly || isPrebuilt(*globals.state, i)) { + newElems.push_back(i); + } + } + + StringSet newNames; + for (auto& i : newElems) { + /* `forceName' is a hack to get package names right in some + one-click installs, namely those where the name used in the + path is not the one we want (e.g., `java-front' versus + `java-front-0.9pre15899'). */ + if (!globals.forceName.empty()) { + i.setName(globals.forceName); + } + newNames.insert(DrvName(i.queryName()).name); + } + + while (true) { + std::string lockToken = optimisticLockProfile(profile); + + DrvInfos allElems(newElems); + + /* Add in the already installed derivations, unless they have + the same name as a to-be-installed element. */ + if (!globals.removeAll) { + DrvInfos installedElems = queryInstalled(*globals.state, profile); + + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + if (!globals.preserveInstalled && + newNames.find(drvName.name) != newNames.end() && !keep(i)) { + LOG(INFO) << "replacing old '" << i.queryName() << "'"; + } else { + allElems.push_back(i); + } + } + + for (auto& i : newElems) { + LOG(INFO) << "installing " << i.queryName(); + } + } + + printMissing(*globals.state, newElems); + + if (globals.dryRun) { + return; + } + + if (createUserEnv(*globals.state, allElems, profile, + settings.envKeepDerivations, lockToken)) { + break; + } + } +} + +static void opInstall(Globals& globals, Strings opFlags, Strings opArgs) { + for (auto i = opFlags.begin(); i != opFlags.end();) { + std::string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) { + ; + } else if (arg == "--preserve-installed" || arg == "-P") { + globals.preserveInstalled = true; + } else if (arg == "--remove-all" || arg == "-r") { + globals.removeAll = true; + } else { + throw UsageError(format("unknown flag '%1%'") % arg); + } + } + + installDerivations(globals, opArgs, globals.profile); +} + +typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; + +static void upgradeDerivations(Globals& globals, const Strings& args, + UpgradeType upgradeType) { + DLOG(INFO) << "upgrading derivations"; + + /* Upgrade works as follows: we take all currently installed + derivations, and for any derivation matching any selector, look + for a derivation in the input Nix expression that has the same + name and a higher version number. */ + + while (true) { + std::string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + + /* Fetch all derivations from the input file. */ + DrvInfos availElems; + queryInstSources(*globals.state, globals.instSource, args, availElems, + false); + + /* Go through all installed derivations. */ + DrvInfos newElems; + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + + try { + if (keep(i)) { + newElems.push_back(i); + continue; + } + + /* Find the derivation in the input Nix expression + with the same name that satisfies the version + constraints specified by upgradeType. If there are + multiple matches, take the one with the highest + priority. If there are still multiple matches, + take the one with the highest version. + Do not upgrade if it would decrease the priority. */ + auto bestElem = availElems.end(); + std::string bestVersion; + for (auto j = availElems.begin(); j != availElems.end(); ++j) { + if (comparePriorities(*globals.state, i, *j) > 0) { + continue; + } + DrvName newName(j->queryName()); + if (newName.name == drvName.name) { + int d = compareVersions(drvName.version, newName.version); + if ((upgradeType == utLt && d < 0) || + (upgradeType == utLeq && d <= 0) || + (upgradeType == utEq && d == 0) || upgradeType == utAlways) { + long d2 = -1; + if (bestElem != availElems.end()) { + d2 = comparePriorities(*globals.state, *bestElem, *j); + if (d2 == 0) { + d2 = compareVersions(bestVersion, newName.version); + } + } + if (d2 < 0 && + (!globals.prebuiltOnly || isPrebuilt(*globals.state, *j))) { + bestElem = j; + bestVersion = newName.version; + } + } + } + } + + if (bestElem != availElems.end() && + i.queryOutPath() != bestElem->queryOutPath()) { + const char* action = + compareVersions(drvName.version, bestVersion) <= 0 + ? "upgrading" + : "downgrading"; + LOG(INFO) << action << " '" << i.queryName() << "' to '" + << bestElem->queryName() << "'"; + newElems.push_back(*bestElem); + } else { + newElems.push_back(i); + } + + } catch (Error& e) { + e.addPrefix( + fmt("while trying to find an upgrade for '%s':\n", i.queryName())); + throw; + } + } + + printMissing(*globals.state, newElems); + + if (globals.dryRun) { + return; + } + + if (createUserEnv(*globals.state, newElems, globals.profile, + settings.envKeepDerivations, lockToken)) { + break; + } + } +} + +static void opUpgrade(Globals& globals, Strings opFlags, Strings opArgs) { + UpgradeType upgradeType = utLt; + for (auto i = opFlags.begin(); i != opFlags.end();) { + std::string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) { + ; + } else if (arg == "--lt") { + upgradeType = utLt; + } else if (arg == "--leq") { + upgradeType = utLeq; + } else if (arg == "--eq") { + upgradeType = utEq; + } else if (arg == "--always") { + upgradeType = utAlways; + } else { + throw UsageError(format("unknown flag '%1%'") % arg); + } + } + + upgradeDerivations(globals, opArgs, upgradeType); +} + +static void setMetaFlag(EvalState& state, DrvInfo& drv, const std::string& name, + const std::string& value) { + Value* v = state.allocValue(); + mkString(*v, value.c_str()); + drv.setMeta(name, v); +} + +static void opSetFlag(Globals& globals, Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + if (opArgs.size() < 2) { + throw UsageError("not enough arguments to '--set-flag'"); + } + + auto arg = opArgs.begin(); + std::string flagName = *arg++; + std::string flagValue = *arg++; + DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end())); + + while (true) { + std::string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + + /* Update all matching derivations. */ + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + for (auto& j : selectors) { + if (j.matches(drvName)) { + LOG(INFO) << "setting flag on '" << i.queryName() << "'"; + j.hits++; + setMetaFlag(*globals.state, i, flagName, flagValue); + break; + } + } + } + + checkSelectorUse(selectors); + + /* Write the new user environment. */ + if (createUserEnv(*globals.state, installedElems, globals.profile, + settings.envKeepDerivations, lockToken)) { + break; + } + } +} + +static void opSet(Globals& globals, Strings opFlags, Strings opArgs) { + auto store2 = globals.state->store.dynamic_pointer_cast<LocalFSStore>(); + if (!store2) { + throw Error("--set is not supported for this Nix store"); + } + + for (auto i = opFlags.begin(); i != opFlags.end();) { + std::string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) { + ; + } else { + throw UsageError(format("unknown flag '%1%'") % arg); + } + } + + DrvInfos elems; + queryInstSources(*globals.state, globals.instSource, opArgs, elems, true); + + if (elems.size() != 1) { + throw Error("--set requires exactly one derivation"); + } + + DrvInfo& drv(elems.front()); + + if (!globals.forceName.empty()) { + drv.setName(globals.forceName); + } + + if (!drv.queryDrvPath().empty()) { + PathSet paths = {drv.queryDrvPath()}; + printMissing(globals.state->store, paths); + if (globals.dryRun) { + return; + } + nix::util::OkOrThrow(globals.state->store->buildPaths( + std::cerr, paths, globals.state->repair != 0u ? bmRepair : bmNormal)); + } else { + printMissing(globals.state->store, {drv.queryOutPath()}); + if (globals.dryRun) { + return; + } + globals.state->store->ensurePath(drv.queryOutPath()); + } + + DLOG(INFO) << "switching to new user environment"; + Path generation = createGeneration(ref<LocalFSStore>(store2), globals.profile, + drv.queryOutPath()); + switchLink(globals.profile, generation); +} + +static void uninstallDerivations(Globals& globals, Strings& selectors, + Path& profile) { + while (true) { + std::string lockToken = optimisticLockProfile(profile); + + DrvInfos installedElems = queryInstalled(*globals.state, profile); + DrvInfos newElems; + + for (auto& i : installedElems) { + DrvName drvName(i.queryName()); + bool found = false; + for (auto& j : selectors) { + /* !!! the repeated calls to followLinksToStorePath() + are expensive, should pre-compute them. */ + if ((isPath(j) && + i.queryOutPath() == + globals.state->store->followLinksToStorePath(j)) || + DrvName(j).matches(drvName)) { + LOG(INFO) << "uninstalling '" << i.queryName() << "'"; + found = true; + break; + } + } + if (!found) { + newElems.push_back(i); + } + } + + if (globals.dryRun) { + return; + } + + if (createUserEnv(*globals.state, newElems, profile, + settings.envKeepDerivations, lockToken)) { + break; + } + } +} + +static void opUninstall(Globals& globals, Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + uninstallDerivations(globals, opArgs, globals.profile); +} + +static bool cmpChars(char a, char b) { return toupper(a) < toupper(b); } + +static bool cmpElemByName(const DrvInfo& a, const DrvInfo& b) { + auto a_name = a.queryName(); + auto b_name = b.queryName(); + return lexicographical_compare(a_name.begin(), a_name.end(), b_name.begin(), + b_name.end(), cmpChars); +} + +using Table = std::list<Strings>; + +void printTable(Table& table) { + auto nrColumns = !table.empty() ? table.front().size() : 0; + + std::vector<size_t> widths; + widths.resize(nrColumns); + + for (auto& i : table) { + assert(i.size() == nrColumns); + Strings::iterator j; + size_t column; + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { + if (j->size() > widths[column]) { + widths[column] = j->size(); + } + } + } + + for (auto& i : table) { + Strings::iterator j; + size_t column; + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { + std::string s = *j; + replace(s.begin(), s.end(), '\n', ' '); + cout << s; + if (column < nrColumns - 1) { + cout << std::string(widths[column] - s.size() + 2, ' '); + } + } + cout << std::endl; + } +} + +/* This function compares the version of an element against the + versions in the given set of elements. `cvLess' means that only + lower versions are in the set, `cvEqual' means that at most an + equal version is in the set, and `cvGreater' means that there is at + least one element with a higher version in the set. `cvUnavail' + means that there are no elements with the same name in the set. */ + +using VersionDiff = enum { cvLess, cvEqual, cvGreater, cvUnavail }; + +static VersionDiff compareVersionAgainstSet(const DrvInfo& elem, + const DrvInfos& elems, + std::string& version) { + DrvName name(elem.queryName()); + + VersionDiff diff = cvUnavail; + version = "?"; + + for (auto& i : elems) { + DrvName name2(i.queryName()); + if (name.name == name2.name) { + int d = compareVersions(name.version, name2.version); + if (d < 0) { + diff = cvGreater; + version = name2.version; + } else if (diff != cvGreater && d == 0) { + diff = cvEqual; + version = name2.version; + } else if (diff != cvGreater && diff != cvEqual && d > 0) { + diff = cvLess; + if (version.empty() || compareVersions(version, name2.version) < 0) { + version = name2.version; + } + } + } + } + + return diff; +} + +static void queryJSON(Globals& globals, std::vector<DrvInfo>& elems) { + JSONObject topObj(cout, true); + for (auto& i : elems) { + JSONObject pkgObj = topObj.object(i.attrPath); + + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); + pkgObj.attr("system", i.querySystem()); + + JSONObject metaObj = pkgObj.object("meta"); + StringSet metaNames = i.queryMetaNames(); + for (auto& j : metaNames) { + auto placeholder = metaObj.placeholder(j); + Value* v = i.queryMeta(j); + if (v == nullptr) { + LOG(ERROR) << "derivation '" << i.queryName() + << "' has invalid meta attribute '" << j << "'"; + placeholder.write(nullptr); + } else { + PathSet context; + printValueAsJSON(*globals.state, true, *v, placeholder, context); + } + } + } +} + +static void opQuery(Globals& globals, Strings opFlags, Strings opArgs) { + Strings remaining; + std::string attrPath; + + bool printStatus = false; + bool printName = true; + bool printAttrPath = false; + bool printSystem = false; + bool printDrvPath = false; + bool printOutPath = false; + bool printDescription = false; + bool printMeta = false; + bool compareVersions = false; + bool xmlOutput = false; + bool jsonOutput = false; + + enum { sInstalled, sAvailable } source = sInstalled; + + settings.readOnlyMode = true; /* makes evaluation a bit faster */ + + for (auto i = opFlags.begin(); i != opFlags.end();) { + std::string arg = *i++; + if (arg == "--status" || arg == "-s") { + printStatus = true; + } else if (arg == "--no-name") { + printName = false; + } else if (arg == "--system") { + printSystem = true; + } else if (arg == "--description") { + printDescription = true; + } else if (arg == "--compare-versions" || arg == "-c") { + compareVersions = true; + } else if (arg == "--drv-path") { + printDrvPath = true; + } else if (arg == "--out-path") { + printOutPath = true; + } else if (arg == "--meta") { + printMeta = true; + } else if (arg == "--installed") { + source = sInstalled; + } else if (arg == "--available" || arg == "-a") { + source = sAvailable; + } else if (arg == "--xml") { + xmlOutput = true; + } else if (arg == "--json") { + jsonOutput = true; + } else if (arg == "--attr-path" || arg == "-P") { + printAttrPath = true; + } else if (arg == "--attr" || arg == "-A") { + attrPath = needArg(i, opFlags, arg); + } else { + throw UsageError(format("unknown flag '%1%'") % arg); + } + } + + /* Obtain derivation information from the specified source. */ + DrvInfos availElems; + DrvInfos installedElems; + + if (source == sInstalled || compareVersions || printStatus) { + installedElems = queryInstalled(*globals.state, globals.profile); + } + + if (source == sAvailable || compareVersions) { + loadDerivations(*globals.state, globals.instSource.nixExprPath, + globals.instSource.systemFilter, + globals.instSource.autoArgs.get(), attrPath, availElems); + } + + DrvInfos elems_ = filterBySelector( + *globals.state, source == sInstalled ? installedElems : availElems, + opArgs, false); + + DrvInfos& otherElems(source == sInstalled ? availElems : installedElems); + + /* Sort them by name. */ + /* !!! */ + std::vector<DrvInfo> elems; + for (auto& i : elems_) { + elems.push_back(i); + } + sort(elems.begin(), elems.end(), cmpElemByName); + + /* We only need to know the installed paths when we are querying + the status of the derivation. */ + PathSet installed; /* installed paths */ + + if (printStatus) { + for (auto& i : installedElems) { + installed.insert(i.queryOutPath()); + } + } + + /* Query which paths have substitutes. */ + PathSet validPaths; + PathSet substitutablePaths; + if (printStatus || globals.prebuiltOnly) { + PathSet paths; + for (auto& i : elems) { + try { + paths.insert(i.queryOutPath()); + } catch (AssertionError& e) { + DLOG(WARNING) << "skipping derivation named '" << i.queryName() + << "' which gives an assertion failure"; + i.setFailed(); + } + } + validPaths = globals.state->store->queryValidPaths(paths); + substitutablePaths = globals.state->store->querySubstitutablePaths(paths); + } + + /* Print the desired columns, or XML output. */ + if (jsonOutput) { + queryJSON(globals, elems); + return; + } + + bool tty = isatty(STDOUT_FILENO) != 0; + RunPager pager; + + Table table; + std::ostringstream dummy; + XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); + XMLOpenElement xmlRoot(xml, "items"); + + for (auto& i : elems) { + try { + if (i.hasFailed()) { + continue; + } + + // Activity act(*logger, lvlDebug, format("outputting query result '%1%'") + // % i.attrPath); + + if (globals.prebuiltOnly && + validPaths.find(i.queryOutPath()) == validPaths.end() && + substitutablePaths.find(i.queryOutPath()) == + substitutablePaths.end()) { + continue; + } + + /* For table output. */ + Strings columns; + + /* For XML output. */ + XMLAttrs attrs; + + if (printStatus) { + Path outPath = i.queryOutPath(); + bool hasSubs = + substitutablePaths.find(outPath) != substitutablePaths.end(); + bool isInstalled = installed.find(outPath) != installed.end(); + bool isValid = validPaths.find(outPath) != validPaths.end(); + if (xmlOutput) { + attrs["installed"] = isInstalled ? "1" : "0"; + attrs["valid"] = isValid ? "1" : "0"; + attrs["substitutable"] = hasSubs ? "1" : "0"; + } else { + columns.push_back(absl::StrCat((isInstalled ? "I" : "-"), + (isValid ? "P" : "-"), + (hasSubs ? "S" : "-"))); + } + } + + if (xmlOutput) { + attrs["attrPath"] = i.attrPath; + } else if (printAttrPath) { + columns.push_back(i.attrPath); + } + + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { + columns.push_back(i.queryName()); + } + + if (compareVersions) { + /* Compare this element against the versions of the + same named packages in either the set of available + elements, or the set of installed elements. !!! + This is O(N * M), should be O(N * lg M). */ + std::string version; + VersionDiff diff = compareVersionAgainstSet(i, otherElems, version); + + char ch; + switch (diff) { + case cvLess: + ch = '>'; + break; + case cvEqual: + ch = '='; + break; + case cvGreater: + ch = '<'; + break; + case cvUnavail: + ch = '-'; + break; + default: + abort(); + } + + if (xmlOutput) { + if (diff != cvUnavail) { + attrs["versionDiff"] = ch; + attrs["maxComparedVersion"] = version; + } + } else { + std::string column = std::to_string(ch) + " " + version; + if (diff == cvGreater && tty) { + column = ANSI_RED + column + ANSI_NORMAL; + } + columns.push_back(column); + } + } + + if (xmlOutput) { + if (!i.querySystem().empty()) { + attrs["system"] = i.querySystem(); + } + } else if (printSystem) { + columns.push_back(i.querySystem()); + } + + if (printDrvPath) { + std::string drvPath = i.queryDrvPath(); + if (xmlOutput) { + if (!drvPath.empty()) { + attrs["drvPath"] = drvPath; + } + } else { + columns.push_back(drvPath.empty() ? "-" : drvPath); + } + } + + if (printOutPath && !xmlOutput) { + DrvInfo::Outputs outputs = i.queryOutputs(); + std::string s; + for (auto& j : outputs) { + if (!s.empty()) { + s += ';'; + } + if (j.first != "out") { + s += j.first; + s += "="; + } + s += j.second; + } + columns.push_back(s); + } + + if (printDescription) { + std::string descr = i.queryMetaString("description"); + if (xmlOutput) { + if (!descr.empty()) { + attrs["description"] = descr; + } + } else { + columns.push_back(descr); + } + } + + if (xmlOutput) { + if (printOutPath || printMeta) { + XMLOpenElement item(xml, "item", attrs); + if (printOutPath) { + DrvInfo::Outputs outputs = i.queryOutputs(); + for (auto& j : outputs) { + XMLAttrs attrs2; + attrs2["name"] = j.first; + attrs2["path"] = j.second; + xml.writeEmptyElement("output", attrs2); + } + } + if (printMeta) { + StringSet metaNames = i.queryMetaNames(); + for (auto& j : metaNames) { + XMLAttrs attrs2; + attrs2["name"] = j; + Value* v = i.queryMeta(j); + if (v == nullptr) { + LOG(ERROR) << "derivation '" << i.queryName() + << "' has invalid meta attribute '" << j << "'"; + } else { + if (v->type == tString) { + attrs2["type"] = "string"; + attrs2["value"] = v->string.s; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % v->integer).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tFloat) { + attrs2["type"] = "float"; + attrs2["value"] = (format("%1%") % v->fpoint).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tBool) { + attrs2["type"] = "bool"; + attrs2["value"] = v->boolean ? "true" : "false"; + xml.writeEmptyElement("meta", attrs2); + } else if (v->isList()) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + for (unsigned int j = 0; j < v->listSize(); ++j) { + if ((*v->list)[j]->type != tString) { + continue; + } + XMLAttrs attrs3; + attrs3["value"] = (*v->list)[j]->string.s; + xml.writeEmptyElement("string", attrs3); + } + } else if (v->type == tAttrs) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + Bindings& attrs = *v->attrs; + for (auto& [name, a] : attrs) { + if (a.value->type != tString) { + continue; + } + XMLAttrs attrs3; + attrs3["type"] = name; + attrs3["value"] = a.value->string.s; + xml.writeEmptyElement("string", attrs3); + } + } + } + } + } + } else { + xml.writeEmptyElement("item", attrs); + } + } else { + table.push_back(columns); + } + + cout.flush(); + + } catch (AssertionError& e) { + DLOG(WARNING) << "skipping derivation named '" << i.queryName() + << "' which gives an assertion failure"; + } catch (Error& e) { + e.addPrefix( + fmt("while querying the derivation named '%1%':\n", i.queryName())); + throw; + } + } + + if (!xmlOutput) { + printTable(table); + } +} + +static void opSwitchProfile(Globals& globals, Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + if (opArgs.size() != 1) { + throw UsageError(format("exactly one argument expected")); + } + + Path profile = absPath(opArgs.front()); + Path profileLink = getHome() + "/.nix-profile"; + + switchLink(profileLink, profile); +} + +static const int prevGen = -2; + +static void switchGeneration(Globals& globals, int dstGen) { + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + Generation dst; + for (auto& i : gens) { + if ((dstGen == prevGen && i.number < curGen) || + (dstGen >= 0 && i.number == dstGen)) { + dst = i; + } + } + + if (!dst) { + if (dstGen == prevGen) { + throw Error(format("no generation older than the current (%1%) exists") % + curGen); + } + throw Error(format("generation %1% does not exist") % dstGen); + } + + LOG(INFO) << "switching from generation " << curGen << " to " << dst.number; + + if (globals.dryRun) { + return; + } + + switchLink(globals.profile, dst.path); +} + +static void opSwitchGeneration(Globals& globals, Strings opFlags, + Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + if (opArgs.size() != 1) { + throw UsageError(format("exactly one argument expected")); + } + + int dstGen; + if (!absl::SimpleAtoi(opArgs.front(), &dstGen)) { + throw UsageError(format("expected a generation number")); + } + + switchGeneration(globals, dstGen); +} + +static void opRollback(Globals& globals, Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + if (!opArgs.empty()) { + throw UsageError(format("no arguments expected")); + } + + switchGeneration(globals, prevGen); +} + +static void opListGenerations(Globals& globals, Strings opFlags, + Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + if (!opArgs.empty()) { + throw UsageError(format("no arguments expected")); + } + + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + RunPager pager; + + for (auto& i : gens) { + tm t; + if (localtime_r(&i.creationTime, &t) == nullptr) { + throw Error("cannot convert time"); + } + cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") % + i.number % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday % + t.tm_hour % t.tm_min % t.tm_sec % + (i.number == curGen ? "(current)" : ""); + } +} + +static void opDeleteGenerations(Globals& globals, Strings opFlags, + Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + } + + if (opArgs.size() == 1 && opArgs.front() == "old") { + deleteOldGenerations(globals.profile, globals.dryRun); + } else if (opArgs.size() == 1 && + opArgs.front().find('d') != std::string::npos) { + deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + } else if (opArgs.size() == 1 && + opArgs.front().find('+') != std::string::npos) { + if (opArgs.front().size() < 2) { + throw Error(format("invalid number of generations ‘%1%’") % + opArgs.front()); + } + std::string str_max = std::string(opArgs.front(), 1, opArgs.front().size()); + int max; + if (!absl::SimpleAtoi(str_max, &max) || max == 0) { + throw Error(format("invalid number of generations to keep ‘%1%’") % + opArgs.front()); + } + deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + } else { + std::set<unsigned int> gens; + for (auto& i : opArgs) { + unsigned int n; + if (!absl::SimpleAtoi(i, &n)) { + throw UsageError(format("invalid generation number '%1%'") % i); + } + gens.insert(n); + } + deleteGenerations(globals.profile, gens, globals.dryRun); + } +} + +static void opVersion(Globals& globals, Strings opFlags, Strings opArgs) { + printVersion("nix-env"); +} + +static int _main(int argc, char** argv) { + { + Strings opFlags; + Strings opArgs; + Operation op = nullptr; + RepairFlag repair = NoRepair; + std::string file; + + Globals globals; + + globals.instSource.type = srcUnknown; + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; + globals.instSource.systemFilter = "*"; + + if (!pathExists(globals.instSource.nixExprPath)) { + try { + createDirs(globals.instSource.nixExprPath); + replaceSymlink(fmt("%s/profiles/per-user/%s/channels", + settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) { + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } + } catch (Error&) { + } + } + + globals.dryRun = false; + globals.preserveInstalled = false; + globals.removeAll = false; + globals.prebuiltOnly = false; + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator& arg, + const Strings::iterator& end) { + Operation oldOp = op; + + if (*arg == "--help") { + showManPage("nix-env"); + } else if (*arg == "--version") { + op = opVersion; + } else if (*arg == "--install" || *arg == "-i") { + op = opInstall; + } else if (*arg == + "--force-name") { // undocumented flag for nix-install-package + globals.forceName = getArg(*arg, arg, end); + } else if (*arg == "--uninstall" || *arg == "-e") { + op = opUninstall; + } else if (*arg == "--upgrade" || *arg == "-u") { + op = opUpgrade; + } else if (*arg == "--set-flag") { + op = opSetFlag; + } else if (*arg == "--set") { + op = opSet; + } else if (*arg == "--query" || *arg == "-q") { + op = opQuery; + } else if (*arg == "--profile" || *arg == "-p") { + globals.profile = absPath(getArg(*arg, arg, end)); + } else if (*arg == "--file" || *arg == "-f") { + file = getArg(*arg, arg, end); + } else if (*arg == "--switch-profile" || *arg == "-S") { + op = opSwitchProfile; + } else if (*arg == "--switch-generation" || *arg == "-G") { + op = opSwitchGeneration; + } else if (*arg == "--rollback") { + op = opRollback; + } else if (*arg == "--list-generations") { + op = opListGenerations; + } else if (*arg == "--delete-generations") { + op = opDeleteGenerations; + } else if (*arg == "--dry-run") { + LOG(INFO) << "(dry run; not doing anything)"; + globals.dryRun = true; + } else if (*arg == "--system-filter") { + globals.instSource.systemFilter = getArg(*arg, arg, end); + } else if (*arg == "--prebuilt-only" || *arg == "-b") { + globals.prebuiltOnly = true; + } else if (*arg == "--repair") { + repair = Repair; + } else if (*arg != "" && arg->at(0) == '-') { + opFlags.push_back(*arg); + /* FIXME: hacky */ + if (*arg == "--from-profile" || + (op == opQuery && (*arg == "--attr" || *arg == "-A"))) { + opFlags.push_back(getArg(*arg, arg, end)); + } + } else { + opArgs.push_back(*arg); + } + + if ((oldOp != nullptr) && oldOp != op) { + throw UsageError("only one operation may be specified"); + } + + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + if (op == nullptr) { + throw UsageError("no operation specified"); + } + + auto store = openStore(); + + globals.state = + std::shared_ptr<EvalState>(new EvalState(myArgs.searchPath, store)); + globals.state->repair = repair; + + if (!file.empty()) { + globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); + } + + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); + + if (globals.profile.empty()) { + globals.profile = getEnv("NIX_PROFILE").value_or(""); + } + + if (globals.profile.empty()) { + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink(getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", + settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error&) { + globals.profile = profileLink; + } + } + + op(globals, opFlags, opArgs); + + globals.state->printStats(); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-env", _main); diff --git a/third_party/nix/src/nix-env/user-env.cc b/third_party/nix/src/nix-env/user-env.cc new file mode 100644 index 000000000000..bce5c44f9591 --- /dev/null +++ b/third_party/nix/src/nix-env/user-env.cc @@ -0,0 +1,169 @@ +#include "nix-env/user-env.hh" + +#include <iostream> + +#include <glog/logging.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/profiles.hh" +#include "libstore/store-api.hh" +#include "libutil/status.hh" +#include "libutil/util.hh" + +namespace nix { + +DrvInfos queryInstalled(EvalState& state, const Path& userEnv) { + DrvInfos elems; + Path manifestFile = userEnv + "/manifest.nix"; + if (pathExists(manifestFile)) { + Value v; + state.evalFile(manifestFile, v); + std::unique_ptr<Bindings> bindings(Bindings::New()); + getDerivations(state, v, "", bindings.get(), elems, false); + } + return elems; +} + +bool createUserEnv(EvalState& state, DrvInfos& elems, const Path& profile, + bool keepDerivations, const std::string& lockToken) { + /* Build the components in the user environment, if they don't + exist already. */ + PathSet drvsToBuild; + for (auto& i : elems) { + if (!i.queryDrvPath().empty()) { + drvsToBuild.insert(i.queryDrvPath()); + } + } + + DLOG(INFO) << "building user environment dependencies"; + util::OkOrThrow(state.store->buildPaths( + std::cerr, drvsToBuild, state.repair != 0u ? bmRepair : bmNormal)); + + /* Construct the whole top level derivation. */ + PathSet references; + Value manifest; + state.mkList(manifest, elems.size()); + unsigned int n = 0; + for (auto& i : elems) { + /* Create a pseudo-derivation containing the name, system, + output paths, and optionally the derivation path, as well + as the meta attributes. */ + Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + + Value* v = state.allocValue(); + (*manifest.list)[n++] = v; + state.mkAttrs(*v, 16); + + mkString(*state.allocAttr(*v, state.sType), "derivation"); + mkString(*state.allocAttr(*v, state.sName), i.queryName()); + auto system = i.querySystem(); + if (!system.empty()) { + mkString(*state.allocAttr(*v, state.sSystem), system); + } + mkString(*state.allocAttr(*v, state.sOutPath), i.queryOutPath()); + if (!drvPath.empty()) { + mkString(*state.allocAttr(*v, state.sDrvPath), i.queryDrvPath()); + } + + // Copy each output meant for installation. + DrvInfo::Outputs outputs = i.queryOutputs(true); + Value& vOutputs = *state.allocAttr(*v, state.sOutputs); + state.mkList(vOutputs, outputs.size()); + unsigned int m = 0; + for (auto& j : outputs) { + mkString(*((*vOutputs.list)[m++] = state.allocValue()), j.first); + Value& vOutputs = *state.allocAttr(*v, state.symbols.Create(j.first)); + state.mkAttrs(vOutputs, 2); + mkString(*state.allocAttr(vOutputs, state.sOutPath), j.second); + + /* This is only necessary when installing store paths, e.g., + `nix-env -i /nix/store/abcd...-foo'. */ + state.store->addTempRoot(j.second); + state.store->ensurePath(j.second); + + references.insert(j.second); + } + + // Copy the meta attributes. + Value& vMeta = *state.allocAttr(*v, state.sMeta); + state.mkAttrs(vMeta, 16); + StringSet metaNames = i.queryMetaNames(); + for (auto& j : metaNames) { + Value* v = i.queryMeta(j); + if (v == nullptr) { + continue; + } + vMeta.attrs->push_back(Attr(state.symbols.Create(j), v)); + } + + if (!drvPath.empty()) { + references.insert(drvPath); + } + } + + /* Also write a copy of the list of user environment elements to + the store; we need it for future modifications of the + environment. */ + Path manifestFile = state.store->addTextToStore( + "env-manifest.nix", (format("%1%") % manifest).str(), references); + + /* Get the environment builder expression. */ + Value envBuilder; + state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); + + /* Construct a Nix expression that calls the user environment + builder with the manifest as argument. */ + Value args; + Value topLevel; + state.mkAttrs(args, 3); + mkString(*state.allocAttr(args, state.symbols.Create("manifest")), + manifestFile, {manifestFile}); + args.attrs->push_back(Attr(state.symbols.Create("derivations"), &manifest)); + mkApp(topLevel, envBuilder, args); + + /* Evaluate it. */ + DLOG(INFO) << "evaluating user environment builder"; + state.forceValue(topLevel); + PathSet context; + Attr& aDrvPath(topLevel.attrs->find(state.sDrvPath)->second); + Path topLevelDrv = + state.coerceToPath(aDrvPath.pos != nullptr ? *(aDrvPath.pos) : noPos, + *(aDrvPath.value), context); + Attr& aOutPath(topLevel.attrs->find(state.sOutPath)->second); + Path topLevelOut = + state.coerceToPath(aOutPath.pos != nullptr ? *(aOutPath.pos) : noPos, + *(aOutPath.value), context); + + /* Realise the resulting store expression. */ + DLOG(INFO) << "building user environment"; + util::OkOrThrow(state.store->buildPaths( + std::cerr, {topLevelDrv}, state.repair != 0u ? bmRepair : bmNormal)); + + /* Switch the current user environment to the output path. */ + auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); + + if (store2) { + PathLocks lock; + lockProfile(lock, profile); + + Path lockTokenCur = optimisticLockProfile(profile); + if (lockToken != lockTokenCur) { + LOG(WARNING) << "profile '" << profile + << "' changed while we were busy; restarting"; + return false; + } + + DLOG(INFO) << "switching to new user environment"; + Path generation = + createGeneration(ref<LocalFSStore>(store2), profile, topLevelOut); + switchLink(profile, generation); + } + + return true; +} + +} // namespace nix diff --git a/third_party/nix/src/nix-env/user-env.hh b/third_party/nix/src/nix-env/user-env.hh new file mode 100644 index 000000000000..95919a6c8799 --- /dev/null +++ b/third_party/nix/src/nix-env/user-env.hh @@ -0,0 +1,12 @@ +#pragma once + +#include "libexpr/get-drvs.hh" + +namespace nix { + +DrvInfos queryInstalled(EvalState& state, const Path& userEnv); + +bool createUserEnv(EvalState& state, DrvInfos& elems, const Path& profile, + bool keepDerivations, const std::string& lockToken); + +} // namespace nix diff --git a/third_party/nix/src/nix-instantiate/nix-instantiate.cc b/third_party/nix/src/nix-instantiate/nix-instantiate.cc new file mode 100644 index 000000000000..236037299d8b --- /dev/null +++ b/third_party/nix/src/nix-instantiate/nix-instantiate.cc @@ -0,0 +1,219 @@ +#include <iostream> +#include <map> + +#include "libexpr/attr-path.hh" +#include "libexpr/common-eval-args.hh" +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/get-drvs.hh" +#include "libexpr/value-to-json.hh" +#include "libexpr/value-to-xml.hh" +#include "libmain/shared.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/util.hh" +#include "nix/legacy.hh" + +using namespace nix; + +static Path gcRoot; +static int rootNr = 0; +static bool indirectRoot = false; + +enum OutputKind { okPlain, okXML, okJSON }; + +void processExpr(EvalState& state, const Strings& attrPaths, bool parseOnly, + bool strict, Bindings* autoArgs, bool evalOnly, + OutputKind output, bool location, Expr* e) { + if (parseOnly) { + std::cout << format("%1%\n") % *e; + return; + } + + Value vRoot; + state.eval(e, vRoot); + + for (auto& i : attrPaths) { + Value& v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + state.forceValue(v); + + PathSet context; + if (evalOnly) { + Value vRes; + if (autoArgs->empty()) { + vRes = v; + } else { + state.autoCallFunction(autoArgs, v, vRes); + } + if (output == okXML) { + printValueAsXML(state, strict, location, vRes, std::cout, context); + } else if (output == okJSON) { + printValueAsJSON(state, strict, vRes, std::cout, context); + } else { + if (strict) { + state.forceValueDeep(vRes); + } + std::cout << vRes << std::endl; + } + } else { + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + for (auto& i : drvs) { + Path drvPath = i.queryDrvPath(); + + /* What output do we want? */ + std::string outputName = i.queryOutputName(); + if (outputName.empty()) { + throw Error( + format("derivation '%1%' lacks an 'outputName' attribute ") % + drvPath); + } + + if (gcRoot.empty()) { + printGCWarning(); + } else { + Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; + if (++rootNr > 1) { + rootName += "-" + std::to_string(rootNr); + } + auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>(); + if (store2) { + drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot); + } + } + std::cout << format("%1%%2%\n") % drvPath % + (outputName != "out" ? "!" + outputName : ""); + } + } + } +} + +static int _main(int argc, char** argv) { + { + Strings files; + bool readStdin = false; + bool fromArgs = false; + bool findFile = false; + bool evalOnly = false; + bool parseOnly = false; + bool traceFileAccess = false; + OutputKind outputKind = okPlain; + bool xmlOutputSourceLocation = true; + bool strict = false; + Strings attrPaths; + bool wantsReadWrite = false; + RepairFlag repair = NoRepair; + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-instantiate"); + } else if (*arg == "--version") { + printVersion("nix-instantiate"); + } else if (*arg == "-") { + readStdin = true; + } else if (*arg == "--expr" || *arg == "-E") { + fromArgs = true; + } else if (*arg == "--eval" || *arg == "--eval-only") { + evalOnly = true; + } else if (*arg == "--read-write-mode") { + wantsReadWrite = true; + } else if (*arg == "--parse" || *arg == "--parse-only") { + parseOnly = evalOnly = true; + } else if (*arg == "--find-file") { + findFile = true; + } else if (*arg == "--attr" || *arg == "-A") { + attrPaths.push_back(getArg(*arg, arg, end)); + } else if (*arg == "--add-root") { + gcRoot = getArg(*arg, arg, end); + } else if (*arg == "--indirect") { + indirectRoot = true; + } else if (*arg == "--xml") { + outputKind = okXML; + } else if (*arg == "--json") { + outputKind = okJSON; + } else if (*arg == "--no-location") { + xmlOutputSourceLocation = false; + } else if (*arg == "--strict") { + strict = true; + } else if (*arg == "--repair") { + repair = Repair; + } else if (*arg == "--dry-run") { + settings.readOnlyMode = true; + } else if (*arg == "--trace-file-access") { + traceFileAccess = true; + } else if (*arg == "--trace-file-access=true") { + traceFileAccess = true; + } else if (*arg == "--trace-file-access=false") { + traceFileAccess = false; + } else if (*arg == "--notrace-file-access") { + traceFileAccess = false; + } else if (*arg != "" && arg->at(0) == '-') { + return false; + } else { + files.push_back(*arg); + } + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + if (evalOnly && !wantsReadWrite) { + settings.readOnlyMode = true; + } + + auto store = openStore(); + + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + state->repair = repair; + if (traceFileAccess) { + state->EnableFileAccessTracing([](const Path& path) { + std::cerr << "trace: depot-scan: " << path << "\n"; + }); + } + + std::unique_ptr<Bindings> autoArgs = myArgs.getAutoArgs(*state); + + if (attrPaths.empty()) { + attrPaths = {""}; + } + + if (findFile) { + for (auto& i : files) { + Path p = state->findFile(i); + if (p.empty()) { + throw Error(format("unable to find '%1%'") % i); + } + std::cout << p << std::endl; + } + return 0; + } + + if (readStdin) { + Expr* e = state->parseStdin(); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs.get(), + evalOnly, outputKind, xmlOutputSourceLocation, e); + } else if (files.empty() && !fromArgs) { + files.push_back("./default.nix"); + } + + for (auto& i : files) { + Expr* e = fromArgs + ? state->parseExprFromString(i, absPath(".")) + : state->parseExprFromFile(resolveExprPath( + state->checkSourcePath(lookupFileArg(*state, i)))); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs.get(), + evalOnly, outputKind, xmlOutputSourceLocation, e); + } + + state->printStats(); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc b/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc new file mode 100644 index 000000000000..b61a38a7f193 --- /dev/null +++ b/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc @@ -0,0 +1,253 @@ +#include <iostream> + +#include <absl/strings/match.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "libexpr/attr-path.hh" +#include "libexpr/common-eval-args.hh" +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libmain/shared.hh" +#include "libstore/download.hh" +#include "libstore/store-api.hh" +#include "libutil/finally.hh" +#include "libutil/hash.hh" +#include "nix/legacy.hh" + +using namespace nix; + +/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +std::string resolveMirrorUri(EvalState& state, std::string uri) { + if (std::string(uri, 0, 9) != "mirror://") { + return uri; + } + + std::string s(uri, 9); + auto p = s.find('/'); + if (p == std::string::npos) { + throw Error("invalid mirror URI"); + } + std::string mirrorName(s, 0, p); + + Value vMirrors; + state.eval( + state.parseExprFromString( + "import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), + vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.Create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) { + throw Error(format("unknown mirror name '%1%'") % mirrorName); + } + state.forceList(*mirrorList->second.value); + + if (mirrorList->second.value->listSize() < 1) { + throw Error(format("mirror URI '%1%' did not expand to anything") % uri); + } + + std::string mirror = state.forceString(*(*mirrorList->second.value->list)[0]); + return mirror + (absl::EndsWith(mirror, "/") ? "" : "/") + + std::string(s, p + 1); +} + +static int _main(int argc, char** argv) { + { + HashType ht = htSHA256; + std::vector<std::string> args; + bool printPath = getEnv("PRINT_PATH").has_value(); + bool fromExpr = false; + std::string attrPath; + bool unpack = false; + std::string name; + + struct MyArgs : LegacyArgs, MixEvalArgs { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-prefetch-url"); + } else if (*arg == "--version") { + printVersion("nix-prefetch-url"); + } else if (*arg == "--type") { + std::string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) { + throw UsageError(format("unknown hash type '%1%'") % s); + } + } else if (*arg == "--print-path") { + printPath = true; + } else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } else if (*arg == "--unpack") { + unpack = true; + } else if (*arg == "--name") { + name = getArg(*arg, arg, end); + } else if (*arg != "" && arg->at(0) == '-') { + return false; + } else { + args.push_back(*arg); + } + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + if (args.size() > 2) { + throw UsageError("too many arguments"); + } + + auto store = openStore(); + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + + std::unique_ptr<Bindings> autoArgs = myArgs.getAutoArgs(*state); + + /* If -A is given, get the URI from the specified Nix + expression. */ + std::string uri; + if (!fromExpr) { + if (args.empty()) { + throw UsageError("you must specify a URI"); + } + uri = args[0]; + } else { + Path path = + resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value& v(*findAlongAttrPath(*state, attrPath, autoArgs.get(), vRoot)); + state->forceAttrs(v); + + /* Extract the URI. */ + auto attr = v.attrs->find(state->symbols.Create("urls")); + if (attr == v.attrs->end()) { + throw Error("attribute set does not contain a 'urls' attribute"); + } + state->forceList(*attr->second.value); + if (attr->second.value->listSize() < 1) { + throw Error("'urls' list is empty"); + } + uri = state->forceString(*(*attr->second.value->list)[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.Create("outputHashMode")); + if (attr == v.attrs->end()) { + LOG(WARNING) << "this does not look like a fetchurl call"; + } else { + unpack = state->forceString(*attr->second.value) == "recursive"; + } + + /* Extract the name. */ + if (name.empty()) { + attr = v.attrs->find(state->symbols.Create("name")); + if (attr != v.attrs->end()) { + name = state->forceString(*attr->second.value); + } + } + } + + /* Figure out a name in the Nix store. */ + if (name.empty()) { + name = baseNameOf(uri); + } + if (name.empty()) { + throw Error(format("cannot figure out file name for '%1%'") % uri); + } + + /* If an expected hash is given, the file may already exist in + the store. */ + Hash hash; + Hash expectedHash(ht); + Path storePath; + if (args.size() == 2) { + auto expectedHash_ = Hash::deserialize(args[1], ht); + expectedHash = Hash::unwrap_throw(expectedHash); + storePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (store->isValidPath(storePath)) { + hash = expectedHash; + } else { + storePath.clear(); + } + } + + if (storePath.empty()) { + auto actualUri = resolveMirrorUri(*state, uri); + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = Path(tmpDir) + "/tmp"; + + /* Download the file. */ + { + AutoCloseFD fd( + open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600)); + if (!fd) { + throw SysError("creating temporary file '%s'", tmpFile); + } + + FdSink sink(fd.get()); + + DownloadRequest req(actualUri); + req.decompress = false; + getDownloader()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + LOG(INFO) << "unpacking..."; + Path unpacked = Path(tmpDir) + "/unpacked"; + createDirs(unpacked); + if (absl::EndsWith(baseNameOf(uri), ".zip")) { + runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); + } else { + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + } + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) { + tmpFile = unpacked + "/" + entries[0].name; + } else { + tmpFile = unpacked; + } + } + + /* FIXME: inefficient; addToStore() will also hash + this. */ + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); + + if (expectedHash != Hash(ht) && expectedHash != hash) { + throw Error(format("hash mismatch for '%1%'") % uri); + } + + /* Copy the file to the Nix store. FIXME: if RemoteStore + implemented addToStoreFromDump() and downloadFile() + supported a sink, we could stream the download directly + into the Nix store. */ + storePath = store->addToStore(name, tmpFile, unpack, ht); + + assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); + } + + if (!printPath) { + LOG(INFO) << "path is '" << storePath << "'"; + } + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) { + std::cout << storePath << std::endl; + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/third_party/nix/src/nix-store/dotgraph.cc b/third_party/nix/src/nix-store/dotgraph.cc new file mode 100644 index 000000000000..2500b8f4b07d --- /dev/null +++ b/third_party/nix/src/nix-store/dotgraph.cc @@ -0,0 +1,141 @@ +#include "nix-store/dotgraph.hh" + +#include <iostream> + +#include "libstore/store-api.hh" +#include "libutil/util.hh" + +using std::cout; + +namespace nix { + +static std::string dotQuote(const std::string& s) { return "\"" + s + "\""; } + +static std::string nextColour() { + static int n = 0; + static std::string colours[] = {"black", "red", "green", + "blue", "magenta", "burlywood"}; + return colours[n++ % (sizeof(colours) / sizeof(std::string))]; +} + +static std::string makeEdge(const std::string& src, const std::string& dst) { + format f = format("%1% -> %2% [color = %3%];\n") % dotQuote(src) % + dotQuote(dst) % dotQuote(nextColour()); + return f.str(); +} + +static std::string makeNode(const std::string& id, const std::string& label, + const std::string& colour) { + format f = format( + "%1% [label = %2%, shape = box, " + "style = filled, fillcolor = %3%];\n") % + dotQuote(id) % dotQuote(label) % dotQuote(colour); + return f.str(); +} + +static std::string symbolicName(const std::string& path) { + std::string p = baseNameOf(path); + return std::string(p, p.find('-') + 1); +} + +#if 0 +std::string pathLabel(const Path & nePath, const std::string & elemPath) +{ + return (std::string) nePath + "-" + elemPath; +} + + +void printClosure(const Path & nePath, const StoreExpr & fs) +{ + PathSet workList(fs.closure.roots); + PathSet doneSet; + + for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) { + cout << makeEdge(pathLabel(nePath, *i), nePath); + } + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) == doneSet.end()) { + doneSet.insert(path); + + ClosureElems::const_iterator elem = fs.closure.elems.find(path); + if (elem == fs.closure.elems.end()) + throw Error(format("bad closure, missing path '%1%'") % path); + + for (StringSet::const_iterator i = elem->second.refs.begin(); + i != elem->second.refs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); + } + + cout << makeNode(pathLabel(nePath, path), + symbolicName(path), "#ff0000"); + } + } +} +#endif + +void printDotGraph(const ref<Store>& store, const PathSet& roots) { + PathSet workList(roots); + PathSet doneSet; + + cout << "digraph G {\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) != doneSet.end()) { + continue; + } + doneSet.insert(path); + + cout << makeNode(path, symbolicName(path), "#ff0000"); + + for (auto& p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); + } + } + +#if 0 + StoreExpr ne = storeExprFromPath(path); + + string label, colour; + + if (ne.type == StoreExpr::neDerivation) { + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(*i, path); + } + + label = "derivation"; + colour = "#00ff00"; + for (StringPairs::iterator i = ne.derivation.env.begin(); + i != ne.derivation.env.end(); ++i) + if (i->first == "name") { label = i->second; } + } + + else if (ne.type == StoreExpr::neClosure) { + label = "<closure>"; + colour = "#00ffff"; + printClosure(path, ne); + } + + else abort(); + + cout << makeNode(path, label, colour); +#endif + } + + cout << "}\n"; +} + +} // namespace nix diff --git a/third_party/nix/src/nix-store/dotgraph.hh b/third_party/nix/src/nix-store/dotgraph.hh new file mode 100644 index 000000000000..40c268685494 --- /dev/null +++ b/third_party/nix/src/nix-store/dotgraph.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "libutil/types.hh" + +namespace nix { + +class Store; + +void printDotGraph(const ref<Store>& store, const PathSet& roots); + +} // namespace nix diff --git a/third_party/nix/src/nix-store/graphml.cc b/third_party/nix/src/nix-store/graphml.cc new file mode 100644 index 000000000000..ada4aaf6d048 --- /dev/null +++ b/third_party/nix/src/nix-store/graphml.cc @@ -0,0 +1,80 @@ +#include "nix-store/graphml.hh" + +#include <iostream> + +#include "libstore/derivations.hh" +#include "libstore/store-api.hh" +#include "libutil/util.hh" + +using std::cout; + +namespace nix { + +static inline const std::string& xmlQuote(const std::string& s) { + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + +static std::string symbolicName(const std::string& path) { + std::string p = baseNameOf(path); + return std::string(p, p.find('-') + 1); +} + +static std::string makeEdge(const std::string& src, const std::string& dst) { + return fmt(" <edge source=\"%1%\" target=\"%2%\"/>\n", xmlQuote(src), + xmlQuote(dst)); +} + +static std::string makeNode(const ValidPathInfo& info) { + return fmt( + " <node id=\"%1%\">\n" + " <data key=\"narSize\">%2%</data>\n" + " <data key=\"name\">%3%</data>\n" + " <data key=\"type\">%4%</data>\n" + " </node>\n", + info.path, info.narSize, symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); +} + +void printGraphML(const ref<Store>& store, const PathSet& roots) { + PathSet workList(roots); + PathSet doneSet; + std::pair<PathSet::iterator, bool> ret; + + cout << "<?xml version='1.0' encoding='utf-8'?>\n" + << "<graphml xmlns='http://graphml.graphdrawing.org/xmlns'\n" + << " xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'\n" + << " " + "xsi:schemaLocation='http://graphml.graphdrawing.org/xmlns/1.0/" + "graphml.xsd'>\n" + << "<key id='narSize' for='node' attr.name='narSize' attr.type='int'/>" + << "<key id='name' for='node' attr.name='name' attr.type='string'/>" + << "<key id='type' for='node' attr.name='type' attr.type='string'/>" + << "<graph id='G' edgedefault='directed'>\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (!ret.second) { + continue; + } + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto& p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } + } + } + + cout << "</graph>\n"; + cout << "</graphml>\n"; +} + +} // namespace nix diff --git a/third_party/nix/src/nix-store/graphml.hh b/third_party/nix/src/nix-store/graphml.hh new file mode 100644 index 000000000000..be07904d0fc6 --- /dev/null +++ b/third_party/nix/src/nix-store/graphml.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "libutil/types.hh" + +namespace nix { + +class Store; + +void printGraphML(const ref<Store>& store, const PathSet& roots); + +} // namespace nix diff --git a/third_party/nix/src/nix-store/nix-store.cc b/third_party/nix/src/nix-store/nix-store.cc new file mode 100644 index 000000000000..532f60b7b7e3 --- /dev/null +++ b/third_party/nix/src/nix-store/nix-store.cc @@ -0,0 +1,1302 @@ +#include <algorithm> +#include <cstdio> +#include <iostream> + +#include <absl/strings/escaping.h> +#include <fcntl.h> +#include <glog/logging.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/local-store.hh" +#include "libstore/serve-protocol.hh" +#include "libstore/worker-protocol.hh" +#include "libutil/archive.hh" +#include "libutil/monitor-fd.hh" +#include "libutil/status.hh" +#include "libutil/util.hh" +#include "nix-store/dotgraph.hh" +#include "nix-store/graphml.hh" +#include "nix/legacy.hh" + +#if HAVE_SODIUM +#include <sodium.h> +#endif + +using namespace nix; +using std::cin; +using std::cout; + +// TODO(tazjin): clang-tidy's performance lints don't like this, but +// the automatic fixes fail (it seems that some of the ops want to own +// the args for whatever reason) +using Operation = void (*)(Strings, Strings); + +static Path gcRoot; +static int rootNr = 0; +static bool indirectRoot = false; +static bool noOutput = false; +static std::shared_ptr<Store> store; + +ref<LocalStore> ensureLocalStore() { + auto store2 = std::dynamic_pointer_cast<LocalStore>(store); + if (!store2) { + throw Error("you don't have sufficient rights to use this command"); + } + return ref<LocalStore>(store2); +} + +static Path useDeriver(Path path) { + if (isDerivation(path)) { + return path; + } + Path drvPath = store->queryPathInfo(path)->deriver; + if (drvPath.empty()) { + throw Error(format("deriver of path '%1%' is not known") % path); + } + return drvPath; +} + +/* Realise the given path. For a derivation that means build it; for + other paths it means ensure their validity. */ +static PathSet realisePath(Path path, bool build = true) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(path); + + auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store); + + if (isDerivation(p.first)) { + if (build) { + util::OkOrThrow(store->buildPaths(std::cerr, {path})); + } + Derivation drv = store->derivationFromPath(p.first); + rootNr++; + + if (p.second.empty()) { + for (auto& i : drv.outputs) { + p.second.insert(i.first); + } + } + + PathSet outputs; + for (auto& j : p.second) { + auto i = drv.outputs.find(j); + if (i == drv.outputs.end()) { + throw Error( + format("derivation '%1%' does not have an output named '%2%'") % + p.first % j); + } + Path outPath = i->second.path; + if (store2) { + if (gcRoot.empty()) { + printGCWarning(); + } else { + Path rootName = gcRoot; + if (rootNr > 1) { + rootName += "-" + std::to_string(rootNr); + } + if (i->first != "out") { + rootName += "-" + i->first; + } + outPath = store2->addPermRoot(outPath, rootName, indirectRoot); + } + } + outputs.insert(outPath); + } + return outputs; + } + + if (build) { + store->ensurePath(path); + } else if (!store->isValidPath(path)) { + throw Error(format("path '%1%' does not exist and cannot be created") % + path); + } + if (store2) { + if (gcRoot.empty()) { + printGCWarning(); + } else { + Path rootName = gcRoot; + rootNr++; + if (rootNr > 1) { + rootName += "-" + std::to_string(rootNr); + } + path = store2->addPermRoot(path, rootName, indirectRoot); + } + } + return {path}; +} + +/* Realise the given paths. */ +static void opRealise(Strings opFlags, Strings opArgs) { + bool dryRun = false; + BuildMode buildMode = bmNormal; + bool ignoreUnknown = false; + + for (auto& i : opFlags) { + if (i == "--dry-run") { + dryRun = true; + } else if (i == "--repair") { + buildMode = bmRepair; + } else if (i == "--check") { + buildMode = bmCheck; + } else if (i == "--ignore-unknown") { + ignoreUnknown = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + Paths paths; + for (auto& i : opArgs) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(i); + paths.push_back(makeDrvPathWithOutputs( + store->followLinksToStorePath(p.first), p.second)); + } + + unsigned long long downloadSize; + unsigned long long narSize; + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + store->queryMissing(PathSet(paths.begin(), paths.end()), willBuild, + willSubstitute, unknown, downloadSize, narSize); + + if (ignoreUnknown) { + Paths paths2; + for (auto& i : paths) { + if (unknown.find(i) == unknown.end()) { + paths2.push_back(i); + } + } + paths = paths2; + unknown = PathSet(); + } + + if (settings.printMissing) { + printMissing(ref<Store>(store), willBuild, willSubstitute, unknown, + downloadSize, narSize); + } + + if (dryRun) { + return; + } + + /* Build all paths at the same time to exploit parallelism. */ + util::OkOrThrow(store->buildPaths( + std::cerr, PathSet(paths.begin(), paths.end()), buildMode)); + + if (!ignoreUnknown) { + for (auto& i : paths) { + PathSet paths = realisePath(i, false); + if (!noOutput) { + for (auto& j : paths) { + cout << format("%1%\n") % j; + } + } + } + } +} + +/* Add files to the Nix store and print the resulting paths. */ +static void opAdd(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + + for (auto& i : opArgs) { + cout << format("%1%\n") % store->addToStore(baseNameOf(i), i); + } +} + +/* Preload the output of a fixed-output derivation into the Nix + store. */ +static void opAddFixed(Strings opFlags, Strings opArgs) { + bool recursive = false; + + for (auto& i : opFlags) { + if (i == "--recursive") { + recursive = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + if (opArgs.empty()) { + throw UsageError("first argument must be hash algorithm"); + } + + HashType hashAlgo = parseHashType(opArgs.front()); + opArgs.pop_front(); + + for (auto& i : opArgs) { + cout << format("%1%\n") % + store->addToStore(baseNameOf(i), i, recursive, hashAlgo); + } +} + +/* Hack to support caching in `nix-prefetch-url'. */ +static void opPrintFixedPath(Strings opFlags, Strings opArgs) { + bool recursive = false; + + for (auto i : opFlags) { + if (i == "--recursive") { + recursive = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + if (opArgs.size() != 3) { + throw UsageError(format("'--print-fixed-path' requires three arguments")); + } + + auto i = opArgs.begin(); + HashType hashAlgo = parseHashType(*i++); + std::string hash = *i++; + std::string name = *i++; + + auto hash_ = Hash::deserialize(hash, hashAlgo); + Hash::unwrap_throw(hash_); + + cout << absl::StrCat(store->makeFixedOutputPath(recursive, *hash_, name), + "\n"); +} + +static PathSet maybeUseOutputs(const Path& storePath, bool useOutput, + bool forceRealise) { + if (forceRealise) { + realisePath(storePath); + } + if (useOutput && isDerivation(storePath)) { + Derivation drv = store->derivationFromPath(storePath); + PathSet outputs; + for (auto& i : drv.outputs) { + outputs.insert(i.second.path); + } + return outputs; + } + return {storePath}; +} + +/* Some code to print a tree representation of a derivation dependency + graph. Topological sorting is used to keep the tree relatively + flat. */ + +const std::string treeConn = "+---"; +const std::string treeLine = "| "; +const std::string treeNull = " "; + +static void printTree(const Path& path, const std::string& firstPad, + const std::string& tailPad, PathSet& done) { + if (done.find(path) != done.end()) { + cout << format("%1%%2% [...]\n") % firstPad % path; + return; + } + done.insert(path); + + cout << format("%1%%2%\n") % firstPad % path; + + auto references = store->queryPathInfo(path)->references; + + /* Topologically sort under the relation A < B iff A \in + closure(B). That is, if derivation A is an (possibly indirect) + input of B, then A is printed first. This has the effect of + flattening the tree, preventing deeply nested structures. */ + Paths sorted = store->topoSortPaths(references); + reverse(sorted.begin(), sorted.end()); + + for (auto i = sorted.begin(); i != sorted.end(); ++i) { + auto j = i; + ++j; + printTree(*i, tailPad + treeConn, + j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + done); + } +} + +/* Perform various sorts of queries. */ +static void opQuery(Strings opFlags, Strings opArgs) { + enum QueryType { + qDefault, + qOutputs, + qRequisites, + qReferences, + qReferrers, + qReferrersClosure, + qDeriver, + qBinding, + qHash, + qSize, + qTree, + qGraph, + qGraphML, + qResolve, + qRoots + }; + QueryType query = qDefault; + bool useOutput = false; + bool includeOutputs = false; + bool forceRealise = false; + std::string bindingName; + + for (auto& i : opFlags) { + QueryType prev = query; + if (i == "--outputs") { + query = qOutputs; + } else if (i == "--requisites" || i == "-R") { + query = qRequisites; + } else if (i == "--references") { + query = qReferences; + } else if (i == "--referrers" || i == "--referers") { + query = qReferrers; + } else if (i == "--referrers-closure" || i == "--referers-closure") { + query = qReferrersClosure; + } else if (i == "--deriver" || i == "-d") { + query = qDeriver; + } else if (i == "--binding" || i == "-b") { + if (opArgs.empty()) { + throw UsageError("expected binding name"); + } + bindingName = opArgs.front(); + opArgs.pop_front(); + query = qBinding; + } else if (i == "--hash") { + query = qHash; + } else if (i == "--size") { + query = qSize; + } else if (i == "--tree") { + query = qTree; + } else if (i == "--graph") { + query = qGraph; + } else if (i == "--graphml") { + query = qGraphML; + } else if (i == "--resolve") { + query = qResolve; + } else if (i == "--roots") { + query = qRoots; + } else if (i == "--use-output" || i == "-u") { + useOutput = true; + } else if (i == "--force-realise" || i == "--force-realize" || i == "-f") { + forceRealise = true; + } else if (i == "--include-outputs") { + includeOutputs = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + if (prev != qDefault && prev != query) { + throw UsageError(format("query type '%1%' conflicts with earlier flag") % + i); + } + } + + if (query == qDefault) { + query = qOutputs; + } + + RunPager pager; + + switch (query) { + case qOutputs: { + for (auto& i : opArgs) { + i = store->followLinksToStorePath(i); + if (forceRealise) { + realisePath(i); + } + Derivation drv = store->derivationFromPath(i); + for (auto& j : drv.outputs) { + cout << format("%1%\n") % j.second.path; + } + } + break; + } + + case qRequisites: + case qReferences: + case qReferrers: + case qReferrersClosure: { + PathSet paths; + for (auto& i : opArgs) { + PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + for (auto& j : ps) { + if (query == qRequisites) { + store->computeFSClosure(j, paths, false, includeOutputs); + } else if (query == qReferences) { + for (auto& p : store->queryPathInfo(j)->references) { + paths.insert(p); + } + } else if (query == qReferrers) { + store->queryReferrers(j, paths); + } else if (query == qReferrersClosure) { + store->computeFSClosure(j, paths, true); + } + } + } + Paths sorted = store->topoSortPaths(paths); + for (auto i = sorted.rbegin(); i != sorted.rend(); ++i) { + cout << format("%s\n") % *i; + } + break; + } + + case qDeriver: + for (auto& i : opArgs) { + Path deriver = + store->queryPathInfo(store->followLinksToStorePath(i))->deriver; + cout << format("%1%\n") % + (deriver.empty() ? "unknown-deriver" : deriver); + } + break; + + case qBinding: + for (auto& i : opArgs) { + Path path = useDeriver(store->followLinksToStorePath(i)); + Derivation drv = store->derivationFromPath(path); + auto j = drv.env.find(bindingName); + if (j == drv.env.end()) { + throw Error( + format( + "derivation '%1%' has no environment binding named '%2%'") % + path % bindingName); + } + cout << format("%1%\n") % j->second; + } + break; + + case qHash: + case qSize: + for (auto& i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + for (auto& j : paths) { + auto info = store->queryPathInfo(j); + if (query == qHash) { + assert(info->narHash.type == htSHA256); + cout << fmt("%s\n", info->narHash.to_string(Base32)); + } else if (query == qSize) { + cout << fmt("%d\n", info->narSize); + } + } + } + break; + + case qTree: { + PathSet done; + for (auto& i : opArgs) { + printTree(store->followLinksToStorePath(i), "", "", done); + } + break; + } + + case qGraph: { + PathSet roots; + for (auto& i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printDotGraph(ref<Store>(store), roots); + break; + } + + case qGraphML: { + PathSet roots; + for (auto& i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), + useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printGraphML(ref<Store>(store), roots); + break; + } + + case qResolve: { + for (auto& i : opArgs) { + cout << format("%1%\n") % store->followLinksToStorePath(i); + } + break; + } + + case qRoots: { + PathSet referrers; + for (auto& i : opArgs) { + store->computeFSClosure( + maybeUseOutputs(store->followLinksToStorePath(i), useOutput, + forceRealise), + referrers, true, settings.gcKeepOutputs, + settings.gcKeepDerivations); + } + Roots roots = store->findRoots(false); + for (auto& [target, links] : roots) { + if (referrers.find(target) != referrers.end()) { + for (auto& link : links) { + cout << format("%1% -> %2%\n") % link % target; + } + } + } + break; + } + + default: + abort(); + } +} + +static void opPrintEnv(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + if (opArgs.size() != 1) { + throw UsageError("'--print-env' requires one derivation store path"); + } + + Path drvPath = opArgs.front(); + Derivation drv = store->derivationFromPath(drvPath); + + /* Print each environment variable in the derivation in a format + that can be sourced by the shell. */ + for (auto& i : drv.env) { + cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + } + + /* Also output the arguments. This doesn't preserve whitespace in + arguments. */ + cout << "export _args; _args='"; + bool first = true; + for (auto& i : drv.args) { + if (!first) { + cout << ' '; + } + first = false; + cout << shellEscape(i); + } + cout << "'\n"; +} + +static void opReadLog(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + + RunPager pager; + + for (auto& i : opArgs) { + auto path = store->followLinksToStorePath(i); + auto log = store->getBuildLog(path); + if (!log) { + throw Error("build log of derivation '%s' is not available", path); + } + std::cout << *log; + } +} + +static void opDumpDB(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + if (!opArgs.empty()) { + for (auto& i : opArgs) { + i = store->followLinksToStorePath(i); + } + for (auto& i : opArgs) { + cout << store->makeValidityRegistration({i}, true, true); + } + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto& i : validPaths) { + cout << store->makeValidityRegistration({i}, true, true); + } + } +} + +static void registerValidity(bool reregister, bool hashGiven, + bool canonicalise) { + ValidPathInfos infos; + + while (true) { + ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); + if (info.path.empty()) { + break; + } + if (!store->isValidPath(info.path) || reregister) { + /* !!! races */ + if (canonicalise) { + canonicalisePathMetaData(info.path, -1); + } + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.narHash = hash.first; + info.narSize = hash.second; + } + infos.push_back(info); + } + } + + ensureLocalStore()->registerValidPaths(infos); +} + +static void opLoadDB(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + registerValidity(true, true, false); +} + +static void opRegisterValidity(Strings opFlags, Strings opArgs) { + bool reregister = false; // !!! maybe this should be the default + bool hashGiven = false; + + for (auto& i : opFlags) { + if (i == "--reregister") { + reregister = true; + } else if (i == "--hash-given") { + hashGiven = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + + registerValidity(reregister, hashGiven, true); +} + +static void opCheckValidity(Strings opFlags, Strings opArgs) { + bool printInvalid = false; + + for (auto& i : opFlags) { + if (i == "--print-invalid") { + printInvalid = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + for (auto& i : opArgs) { + Path path = store->followLinksToStorePath(i); + if (!store->isValidPath(path)) { + if (printInvalid) { + cout << format("%1%\n") % path; + } else { + throw Error(format("path '%1%' is not valid") % path); + } + } + } +} + +static void opGC(Strings opFlags, Strings opArgs) { + bool printRoots = false; + GCOptions options; + options.action = GCOptions::gcDeleteDead; + + GCResults results; + + /* Do what? */ + for (auto i = opFlags.begin(); i != opFlags.end(); ++i) { + if (*i == "--print-roots") { + printRoots = true; + } else if (*i == "--print-live") { + options.action = GCOptions::gcReturnLive; + } else if (*i == "--print-dead") { + options.action = GCOptions::gcReturnDead; + } else if (*i == "--delete") { + options.action = GCOptions::gcDeleteDead; + } else if (*i == "--max-freed") { + auto maxFreed = getIntArg<long long>(*i, i, opFlags.end(), true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } else { + throw UsageError(format("bad sub-operation '%1%' in GC") % *i); + } + } + + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + + if (printRoots) { + Roots roots = store->findRoots(false); + std::set<std::pair<Path, Path>> roots2; + // Transpose and sort the roots. + for (auto& [target, links] : roots) { + for (auto& link : links) { + roots2.emplace(link, target); + } + } + for (auto& [link, target] : roots2) { + std::cout << link << " -> " << target << "\n"; + } + } + + else { + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + + if (options.action != GCOptions::gcDeleteDead) { + for (auto& i : results.paths) { + cout << i << std::endl; + } + } + } +} + +/* Remove paths from the Nix store if possible (i.e., if they do not + have any remaining referrers and are not reachable from any GC + roots). */ +static void opDelete(Strings opFlags, Strings opArgs) { + GCOptions options; + options.action = GCOptions::gcDeleteSpecific; + + for (auto& i : opFlags) { + if (i == "--ignore-liveness") { + options.ignoreLiveness = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + for (auto& i : opArgs) { + options.pathsToDelete.insert(store->followLinksToStorePath(i)); + } + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); +} + +/* Dump a path as a Nix archive. The archive is written to standard + output. */ +static void opDump(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + if (opArgs.size() != 1) { + throw UsageError("only one argument allowed"); + } + + FdSink sink(STDOUT_FILENO); + std::string path = *opArgs.begin(); + dumpPath(path, sink); + sink.flush(); +} + +/* Restore a value from a Nix archive. The archive is read from + standard input. */ +static void opRestore(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + if (opArgs.size() != 1) { + throw UsageError("only one argument allowed"); + } + + FdSource source(STDIN_FILENO); + restorePath(*opArgs.begin(), source); +} + +static void opExport(Strings opFlags, Strings opArgs) { + for (auto& i : opFlags) { + throw UsageError(format("unknown flag '%1%'") % i); + } + + for (auto& i : opArgs) { + i = store->followLinksToStorePath(i); + } + + FdSink sink(STDOUT_FILENO); + store->exportPaths(opArgs, sink); + sink.flush(); +} + +static void opImport(Strings opFlags, Strings opArgs) { + for (auto& i : opFlags) { + throw UsageError(format("unknown flag '%1%'") % i); + } + + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + + FdSource source(STDIN_FILENO); + Paths paths = store->importPaths(source, nullptr, NoCheckSigs); + + for (auto& i : paths) { + cout << format("%1%\n") % i << std::flush; + } +} + +/* Initialise the Nix databases. */ +static void opInit(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("unknown flag"); + } + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + /* Doesn't do anything right now; database tables are initialised + automatically. */ +} + +/* Verify the consistency of the Nix environment. */ +static void opVerify(Strings opFlags, Strings opArgs) { + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + + bool checkContents = false; + RepairFlag repair = NoRepair; + + for (auto& i : opFlags) { + if (i == "--check-contents") { + checkContents = true; + } else if (i == "--repair") { + repair = Repair; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + if (store->verifyStore(checkContents, repair)) { + LOG(WARNING) << "not all errors were fixed"; + throw Exit(1); + } +} + +/* Verify whether the contents of the given store path have not changed. */ +static void opVerifyPath(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("no flags expected"); + } + + int status = 0; + + for (auto& i : opArgs) { + Path path = store->followLinksToStorePath(i); + LOG(INFO) << "checking path '" << path << "'..."; + auto info = store->queryPathInfo(path); + HashSink sink(info->narHash.type); + store->narFromPath(path, sink); + auto current = sink.finish(); + if (current.first != info->narHash) { + LOG(ERROR) << "path '" << path << "' was modified! expected hash '" + << info->narHash.to_string() << "', got '" + << current.first.to_string() << "'"; + status = 1; + } + } + + throw Exit(status); +} + +/* Repair the contents of the given path by redownloading it using a + substituter (if available). */ +static void opRepairPath(Strings opFlags, Strings opArgs) { + if (!opFlags.empty()) { + throw UsageError("no flags expected"); + } + + for (auto& i : opArgs) { + Path path = store->followLinksToStorePath(i); + ensureLocalStore()->repairPath(path); + } +} + +/* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ +static void opOptimise(Strings opFlags, Strings opArgs) { + if (!opArgs.empty() || !opFlags.empty()) { + throw UsageError("no arguments expected"); + } + + store->optimiseStore(); +} + +/* Serve the nix store in a way usable by a restricted ssh user. */ +static void opServe(Strings opFlags, Strings opArgs) { + bool writeAllowed = false; + for (auto& i : opFlags) { + if (i == "--write") { + writeAllowed = true; + } else { + throw UsageError(format("unknown flag '%1%'") % i); + } + } + + if (!opArgs.empty()) { + throw UsageError("no arguments expected"); + } + + FdSource in(STDIN_FILENO); + FdSink out(STDOUT_FILENO); + + /* Exchange the greeting. */ + unsigned int magic = readInt(in); + if (magic != SERVE_MAGIC_1) { + throw Error("protocol mismatch"); + } + out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; + out.flush(); + unsigned int clientVersion = readInt(in); + + auto getBuildSettings = [&]() { + // FIXME: changing options here doesn't work if we're + // building through the daemon. + settings.keepLog = false; + settings.useSubstitutes = false; + settings.maxSilentTime = readInt(in); + settings.buildTimeout = readInt(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 2) { + settings.maxLogSize = readNum<unsigned long>(in); + } + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { + settings.buildRepeat = readInt(in); + settings.enforceDeterminism = readInt(in) != 0u; + settings.runDiffHook = true; + } + settings.printRepeatedBuilds = false; + }; + + while (true) { + ServeCommand cmd; + try { + cmd = static_cast<ServeCommand>(readInt(in)); + } catch (EndOfFile& e) { + break; + } + + switch (cmd) { + case cmdQueryValidPaths: { + bool lock = readInt(in) != 0u; + bool substitute = readInt(in) != 0u; + auto paths = readStorePaths<PathSet>(*store, in); + if (lock && writeAllowed) { + for (auto& path : paths) { + store->addTempRoot(path); + } + } + + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + if (substitute && writeAllowed) { + /* Filter out .drv files (we don't want to build anything). */ + PathSet paths2; + for (auto& path : paths) { + if (!isDerivation(path)) { + paths2.insert(path); + } + } + unsigned long long downloadSize; + unsigned long long narSize; + PathSet willBuild; + PathSet willSubstitute; + PathSet unknown; + store->queryMissing(PathSet(paths2.begin(), paths2.end()), willBuild, + willSubstitute, unknown, downloadSize, narSize); + /* FIXME: should use ensurePath(), but it only + does one path at a time. */ + if (!willSubstitute.empty()) { + try { + util::OkOrThrow(store->buildPaths(std::cerr, willSubstitute)); + } catch (Error& e) { + LOG(WARNING) << e.msg(); + } + } + } + + out << store->queryValidPaths(paths); + break; + } + + case cmdQueryPathInfos: { + auto paths = readStorePaths<PathSet>(*store, in); + // !!! Maybe we want a queryPathInfos? + for (auto& i : paths) { + try { + auto info = store->queryPathInfo(i); + out << info->path << info->deriver << info->references; + // !!! Maybe we want compression? + out << info->narSize // downloadSize + << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 4) { + out << (info->narHash ? info->narHash.to_string() : "") + << info->ca << info->sigs; + } + } catch (InvalidPath&) { + } + } + out << ""; + break; + } + + case cmdDumpStorePath: + store->narFromPath(readStorePath(*store, in), out); + break; + + case cmdImportPaths: { + if (!writeAllowed) { + throw Error("importing paths is not allowed"); + } + store->importPaths(in, nullptr, + NoCheckSigs); // FIXME: should we skip sig checking? + out << 1; // indicate success + break; + } + + case cmdExportPaths: { + readInt(in); // obsolete + store->exportPaths(readStorePaths<Paths>(*store, in), out); + break; + } + + case cmdBuildPaths: { + if (!writeAllowed) { + throw Error("building paths is not allowed"); + } + auto paths = readStorePaths<PathSet>(*store, in); + + getBuildSettings(); + + try { + MonitorFdHup monitor(in.fd); + util::OkOrThrow(store->buildPaths(std::cerr, paths)); + out << 0; + } catch (Error& e) { + assert(e.status); + out << e.status << e.msg(); + } + break; + } + + case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + + if (!writeAllowed) { + throw Error("building paths is not allowed"); + } + + Path drvPath = readStorePath(*store, in); // informational only + BasicDerivation drv; + readDerivation(in, *store, drv); + + getBuildSettings(); + + MonitorFdHup monitor(in.fd); + auto status = store->buildDerivation(std::cerr, drvPath, drv); + + out << status.status << status.errorMsg; + + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { + out << status.timesBuilt + << static_cast<uint64_t>(status.isNonDeterministic) + << status.startTime << status.stopTime; + } + + break; + } + + case cmdQueryClosure: { + bool includeOutputs = readInt(in) != 0u; + PathSet closure; + store->computeFSClosure(readStorePaths<PathSet>(*store, in), closure, + false, includeOutputs); + out << closure; + break; + } + + case cmdAddToStoreNar: { + if (!writeAllowed) { + throw Error("importing paths is not allowed"); + } + + ValidPathInfo info; + info.path = readStorePath(*store, in); + in >> info.deriver; + if (!info.deriver.empty()) { + store->assertStorePath(info.deriver); + } + auto hash_ = Hash::deserialize(readString(in), htSHA256); + info.narHash = Hash::unwrap_throw(hash_); + info.references = readStorePaths<PathSet>(*store, in); + in >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings<StringSet>(in); + in >> info.ca; + + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); + + out << 1; // indicate success + + break; + } + + default: + throw Error(format("unknown serve command %1%") % cmd); + } + + out.flush(); + } +} + +static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) { + for (auto& i : opFlags) { + throw UsageError(format("unknown flag '%1%'") % i); + } + + if (opArgs.size() != 3) { + throw UsageError("three arguments expected"); + } + auto i = opArgs.begin(); + std::string keyName = *i++; + std::string secretKeyFile = *i++; + std::string publicKeyFile = *i++; + +#if HAVE_SODIUM + if (sodium_init() == -1) { + throw Error("could not initialise libsodium"); + } + + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) { + throw Error("key generation failed"); + } + + writeFile(publicKeyFile, + keyName + ":" + + absl::Base64Escape(std::string(reinterpret_cast<char*>(pk), + crypto_sign_PUBLICKEYBYTES))); + umask(0077); + writeFile(secretKeyFile, + keyName + ":" + + absl::Base64Escape(std::string(reinterpret_cast<char*>(sk), + crypto_sign_SECRETKEYBYTES))); +#else + throw Error( + "Nix was not compiled with libsodium, required for signed binary cache " + "support"); +#endif +} + +static void opVersion(Strings opFlags, Strings opArgs) { + printVersion("nix-store"); +} + +/* Scan the arguments; find the operation, set global flags, put all + other flags in a list, and put all other arguments in another + list. */ +static int _main(int argc, char** argv) { + { + Strings opFlags; + Strings opArgs; + Operation op = nullptr; + + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + Operation oldOp = op; + + if (*arg == "--help") { + showManPage("nix-store"); + } else if (*arg == "--version") { + op = opVersion; + } else if (*arg == "--realise" || *arg == "--realize" || + *arg == "-r") { + op = opRealise; + } else if (*arg == "--add" || *arg == "-A") { + op = opAdd; + } else if (*arg == "--add-fixed") { + op = opAddFixed; + } else if (*arg == "--print-fixed-path") { + op = opPrintFixedPath; + } else if (*arg == "--delete") { + op = opDelete; + } else if (*arg == "--query" || *arg == "-q") { + op = opQuery; + } else if (*arg == "--print-env") { + op = opPrintEnv; + } else if (*arg == "--read-log" || *arg == "-l") { + op = opReadLog; + } else if (*arg == "--dump-db") { + op = opDumpDB; + } else if (*arg == "--load-db") { + op = opLoadDB; + } else if (*arg == "--register-validity") { + op = opRegisterValidity; + } else if (*arg == "--check-validity") { + op = opCheckValidity; + } else if (*arg == "--gc") { + op = opGC; + } else if (*arg == "--dump") { + op = opDump; + } else if (*arg == "--restore") { + op = opRestore; + } else if (*arg == "--export") { + op = opExport; + } else if (*arg == "--import") { + op = opImport; + } else if (*arg == "--init") { + op = opInit; + } else if (*arg == "--verify") { + op = opVerify; + } else if (*arg == "--verify-path") { + op = opVerifyPath; + } else if (*arg == "--repair-path") { + op = opRepairPath; + } else if (*arg == "--optimise" || *arg == "--optimize") { + op = opOptimise; + } else if (*arg == "--serve") { + op = opServe; + } else if (*arg == "--generate-binary-cache-key") { + op = opGenerateBinaryCacheKey; + } else if (*arg == "--add-root") { + gcRoot = absPath(getArg(*arg, arg, end)); + } else if (*arg == "--indirect") { + indirectRoot = true; + } else if (*arg == "--no-output") { + noOutput = true; + } else if (*arg != "" && arg->at(0) == '-') { + opFlags.push_back(*arg); + if (*arg == "--max-freed" || *arg == "--max-links" || + *arg == "--max-atime") { /* !!! hack */ + opFlags.push_back(getArg(*arg, arg, end)); + } + } else { + opArgs.push_back(*arg); + } + + if ((oldOp != nullptr) && oldOp != op) { + throw UsageError("only one operation may be specified"); + } + + return true; + }); + + if (op == nullptr) { + throw UsageError("no operation specified"); + } + + if (op != opDump && op != opRestore) { /* !!! hack */ + store = openStore(); + } + + op(opFlags, opArgs); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-store", _main); diff --git a/third_party/nix/src/nix/add-to-store.cc b/third_party/nix/src/nix/add-to-store.cc new file mode 100644 index 000000000000..53641f120e8f --- /dev/null +++ b/third_party/nix/src/nix/add-to-store.cc @@ -0,0 +1,51 @@ +#include "libmain/common-args.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdAddToStore final : MixDryRun, StoreCommand { + Path path; + std::optional<std::string> namePart; + + CmdAddToStore() { + expectArg("path", &path); + + mkFlag() + .longName("name") + .shortName('n') + .description("name component of the store path") + .labels({"name"}) + .dest(&namePart); + } + + std::string name() override { return "add-to-store"; } + + std::string description() override { return "add a path to the Nix store"; } + + Examples examples() override { return {}; } + + void run(ref<Store> store) override { + if (!namePart) { + namePart = baseNameOf(path); + } + + StringSink sink; + dumpPath(path, sink); + + ValidPathInfo info; + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.path = store->makeFixedOutputPath(true, info.narHash, *namePart); + info.ca = makeFixedOutputCA(true, info.narHash); + + if (!dryRun) { + store->addToStore(info, sink.s); + } + + std::cout << fmt("%s\n", info.path); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdAddToStore>()); diff --git a/third_party/nix/src/nix/build.cc b/third_party/nix/src/nix/build.cc new file mode 100644 index 000000000000..3fe74b7ffdba --- /dev/null +++ b/third_party/nix/src/nix/build.cc @@ -0,0 +1,68 @@ +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdBuild final : MixDryRun, InstallablesCommand { + Path outLink = "result"; + + CmdBuild() { + mkFlag() + .longName("out-link") + .shortName('o') + .description("path of the symlink to the build result") + .labels({"path"}) + .dest(&outLink); + + mkFlag() + .longName("no-link") + .description("do not create a symlink to the build result") + .set(&outLink, Path("")); + } + + std::string name() override { return "build"; } + + std::string description() override { + return "build a derivation or fetch a store path"; + } + + Examples examples() override { + return { + Example{"To build and run GNU Hello from NixOS 17.03:", + "nix build -f channel:nixos-17.03 hello; ./result/bin/hello"}, + Example{"To build the build.x86_64-linux attribute from release.nix:", + "nix build -f release.nix build.x86_64-linux"}, + }; + } + + void run(ref<Store> store) override { + auto buildables = build(store, dryRun ? DryRun : Build, installables); + + if (dryRun) { + return; + } + + for (size_t i = 0; i < buildables.size(); ++i) { + auto& b(buildables[i]); + + if (!outLink.empty()) { + for (auto& output : b.outputs) { + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) { + std::string symlink = outLink; + if (i != 0u) { + symlink += fmt("-%d", i); + } + if (output.first != "out") { + symlink += fmt("-%s", output.first); + } + store2->addPermRoot(output.second, absPath(symlink), true); + } + } + } + } + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdBuild>()); diff --git a/third_party/nix/src/nix/cat.cc b/third_party/nix/src/nix/cat.cc new file mode 100644 index 000000000000..7788707eae56 --- /dev/null +++ b/third_party/nix/src/nix/cat.cc @@ -0,0 +1,56 @@ +#include "libstore/fs-accessor.hh" +#include "libstore/nar-accessor.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct MixCat : virtual Args { + std::string path; + + void cat(const ref<FSAccessor>& accessor) { + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) { + throw Error(format("path '%1%' does not exist") % path); + } + if (st.type != FSAccessor::Type::tRegular) { + throw Error(format("path '%1%' is not a regular file") % path); + } + + std::cout << accessor->readFile(path); + } +}; + +struct CmdCatStore final : StoreCommand, MixCat { + CmdCatStore() { expectArg("path", &path); } + + std::string name() override { return "cat-store"; } + + std::string description() override { + return "print the contents of a store file on stdout"; + } + + void run(ref<Store> store) override { cat(store->getFSAccessor()); } +}; + +struct CmdCatNar final : StoreCommand, MixCat { + Path narPath; + + CmdCatNar() { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override { return "cat-nar"; } + + std::string description() override { + return "print the contents of a file inside a NAR file"; + } + + void run(ref<Store> store) override { + cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdCatStore>()); +static nix::RegisterCommand r2(nix::make_ref<nix::CmdCatNar>()); diff --git a/third_party/nix/src/nix/command.cc b/third_party/nix/src/nix/command.cc new file mode 100644 index 000000000000..f7f183ab0ab7 --- /dev/null +++ b/third_party/nix/src/nix/command.cc @@ -0,0 +1,156 @@ +#include "nix/command.hh" + +#include <utility> + +#include "libstore/derivations.hh" +#include "libstore/store-api.hh" + +namespace nix { + +Commands* RegisterCommand::commands = nullptr; + +void Command::printHelp(const std::string& programName, std::ostream& out) { + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto& ex : exs) { + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } + } +} + +MultiCommand::MultiCommand(Commands _commands) + : commands(std::move(_commands)) { + expectedArgs.push_back(ExpectedArg{ + "command", 1, true, [=](std::vector<std::string> ss) { + assert(!command); + auto i = commands.find(ss[0]); + if (i == commands.end()) { + throw UsageError("'%s' is not a recognised command", ss[0]); + } + command = i->second; + }}); +} + +void MultiCommand::printHelp(const std::string& programName, + std::ostream& out) { + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " <COMMAND> <FLAGS>... <ARGS>...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto& command : commands) { + auto descr = command.second->description(); + if (!descr.empty()) { + table.push_back(std::make_pair(command.second->name(), descr)); + } + } + printTable(out, table); + +#if 0 + out << "\n"; + out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-<COMMAND>'.\n"; +#endif +} + +bool MultiCommand::processFlag(Strings::iterator& pos, Strings::iterator end) { + if (Args::processFlag(pos, end)) { + return true; + } + if (command && command->processFlag(pos, end)) { + return true; + } + return false; +} + +bool MultiCommand::processArgs(const Strings& args, bool finish) { + if (command) { + return command->processArgs(args, finish); + } + return Args::processArgs(args, finish); +} + +StoreCommand::StoreCommand() = default; + +ref<Store> StoreCommand::getStore() { + if (!_store) { + _store = createStore(); + } + return ref<Store>(_store); +} + +ref<Store> StoreCommand::createStore() { return openStore(); } + +void StoreCommand::run() { run(getStore()); } + +StorePathsCommand::StorePathsCommand(bool recursive) : recursive(recursive) { + if (recursive) { + mkFlag() + .longName("no-recursive") + .description("apply operation to specified paths only") + .set(&this->recursive, false); + } else { + mkFlag() + .longName("recursive") + .shortName('r') + .description("apply operation to closure of the specified paths") + .set(&this->recursive, true); + } + + mkFlag(0, "all", "apply operation to the entire store", &all); +} + +void StorePathsCommand::run(ref<Store> store) { + Paths storePaths; + + if (all) { + if (!installables.empty() != 0u) { + throw UsageError("'--all' does not expect arguments"); + } + for (auto& p : store->queryAllValidPaths()) { + storePaths.push_back(p); + } + } + + else { + for (auto& p : toStorePaths(store, NoBuild, installables)) { + storePaths.push_back(p); + } + + if (recursive) { + PathSet closure; + store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), + closure, false, false); + storePaths = Paths(closure.begin(), closure.end()); + } + } + + run(store, storePaths); +} + +void StorePathCommand::run(ref<Store> store) { + auto storePaths = toStorePaths(store, NoBuild, installables); + + if (storePaths.size() != 1) { + throw UsageError("this command requires exactly one store path"); + } + + run(store, *storePaths.begin()); +} + +} // namespace nix diff --git a/third_party/nix/src/nix/command.hh b/third_party/nix/src/nix/command.hh new file mode 100644 index 000000000000..87e2fbe9d28d --- /dev/null +++ b/third_party/nix/src/nix/command.hh @@ -0,0 +1,194 @@ +#pragma once + +#include <memory> + +#include "libexpr/common-eval-args.hh" +#include "libutil/args.hh" + +namespace nix { + +extern std::string programPath; + +struct Value; +class Bindings; +class EvalState; + +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args { + virtual std::string name() = 0; + virtual void prepare(){}; + virtual void run() = 0; + + struct Example { + std::string description; + std::string command; + }; + + typedef std::list<Example> Examples; + + virtual Examples examples() { return Examples(); } + + void printHelp(const std::string& programName, std::ostream& out) override; +}; + +class Store; + +/* A command that require a Nix store. */ +struct StoreCommand : virtual Command { + StoreCommand(); + void run() override; + ref<Store> getStore(); + virtual ref<Store> createStore(); + virtual void run(ref<Store>) = 0; + + private: + std::shared_ptr<Store> _store; +}; + +struct Buildable { + Path drvPath; // may be empty + std::map<std::string, Path> outputs; +}; + +using Buildables = std::vector<Buildable>; + +struct Installable { + virtual std::string what() = 0; + + virtual Buildables toBuildables() { + throw Error("argument '%s' cannot be built", what()); + } + + Buildable toBuildable(); + + virtual Value* toValue(EvalState& state) { + throw Error("argument '%s' cannot be evaluated", what()); + } +}; + +struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs { + Path file; + + SourceExprCommand(); + + /* Return a value representing the Nix expression from which we + are installing. This is either the file specified by ‘--file’, + or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs + = import ...; bla = import ...; }’. */ + Value* getSourceExpr(EvalState& state); + + ref<EvalState> getEvalState(); + + private: + std::shared_ptr<EvalState> evalState; + std::shared_ptr<Value*> vSourceExpr; +}; + +enum RealiseMode { Build, NoBuild, DryRun }; + +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : virtual Args, SourceExprCommand { + std::vector<std::shared_ptr<Installable>> installables; + + InstallablesCommand() { expectArgs("installables", &_installables); } + + void prepare() override; + + virtual bool useDefaultInstallables() { return true; } + + private: + std::vector<std::string> _installables; +}; + +struct InstallableCommand : virtual Args, SourceExprCommand { + std::shared_ptr<Installable> installable; + + InstallableCommand() { expectArg("installable", &_installable); } + + void prepare() override; + + private: + std::string _installable; +}; + +/* A command that operates on zero or more store paths. */ +struct StorePathsCommand : public InstallablesCommand { + private: + bool recursive = false; + bool all = false; + + public: + StorePathsCommand(bool recursive = false); + + using StoreCommand::run; + + virtual void run(ref<Store> store, Paths storePaths) = 0; + + void run(ref<Store> store) override; + + bool useDefaultInstallables() override { return !all; } +}; + +/* A command that operates on exactly one store path. */ +struct StorePathCommand : public InstallablesCommand { + using StoreCommand::run; + + virtual void run(ref<Store> store, const Path& storePath) = 0; + + void run(ref<Store> store) override; +}; + +using Commands = std::map<std::string, ref<Command>>; + +/* An argument parser that supports multiple subcommands, + i.e. ‘<command> <subcommand>’. */ +class MultiCommand : virtual Args { + public: + Commands commands; + + std::shared_ptr<Command> command; + + MultiCommand(Commands commands); + + void printHelp(const std::string& programName, std::ostream& out) override; + + bool processFlag(Strings::iterator& pos, Strings::iterator end) override; + + bool processArgs(const Strings& args, bool finish) override; +}; + +/* A helper class for registering commands globally. */ +struct RegisterCommand { + static Commands* commands; + + RegisterCommand(ref<Command> command) { + if (!commands) { + commands = new Commands; + } + commands->emplace(command->name(), command); + } +}; + +std::shared_ptr<Installable> parseInstallable(SourceExprCommand& cmd, + const ref<Store>& store, + const std::string& installable, + bool useDefaultInstallables); + +Buildables build(const ref<Store>& store, RealiseMode mode, + const std::vector<std::shared_ptr<Installable>>& installables); + +PathSet toStorePaths( + const ref<Store>& store, RealiseMode mode, + const std::vector<std::shared_ptr<Installable>>& installables); + +Path toStorePath(const ref<Store>& store, RealiseMode mode, + const std::shared_ptr<Installable>& installable); + +PathSet toDerivations( + const ref<Store>& store, + const std::vector<std::shared_ptr<Installable>>& installables, + bool useDeriver = false); + +} // namespace nix diff --git a/third_party/nix/src/nix/copy.cc b/third_party/nix/src/nix/copy.cc new file mode 100644 index 000000000000..75c85698d11e --- /dev/null +++ b/third_party/nix/src/nix/copy.cc @@ -0,0 +1,86 @@ +#include <atomic> + +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "libutil/sync.hh" +#include "libutil/thread-pool.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdCopy final : StorePathsCommand { + std::string srcUri, dstUri; + + CheckSigsFlag checkSigs = CheckSigs; + + SubstituteFlag substitute = NoSubstitute; + + CmdCopy() : StorePathsCommand(true) { + mkFlag() + .longName("from") + .labels({"store-uri"}) + .description("URI of the source Nix store") + .dest(&srcUri); + mkFlag() + .longName("to") + .labels({"store-uri"}) + .description("URI of the destination Nix store") + .dest(&dstUri); + + mkFlag() + .longName("no-check-sigs") + .description("do not require that paths are signed by trusted keys") + .set(&checkSigs, NoCheckSigs); + + mkFlag() + .longName("substitute-on-destination") + .shortName('s') + .description( + "whether to try substitutes on the destination store (only " + "supported by SSH)") + .set(&substitute, Substitute); + } + + std::string name() override { return "copy"; } + + std::string description() override { return "copy paths between Nix stores"; } + + Examples examples() override { + return { + Example{"To copy Firefox from the local store to a binary cache in " + "file:///tmp/cache:", + "nix copy --to file:///tmp/cache $(type -p firefox)"}, + Example{"To copy the entire current NixOS system closure to another " + "machine via SSH:", + "nix copy --to ssh://server /run/current-system"}, + Example{"To copy a closure from another machine via SSH:", + "nix copy --from ssh://server " + "/nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"}, +#ifdef ENABLE_S3 + Example{"To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello"}, + Example{"To copy Hello to an S3-compatible binary cache:", + "nix copy --to " + "s3://my-bucket?region=eu-west-1&endpoint=example.com " + "nixpkgs.hello"}, +#endif + }; + } + + ref<Store> createStore() override { + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); + } + + void run(ref<Store> srcStore, Paths storePaths) override { + if (srcUri.empty() && dstUri.empty()) { + throw UsageError("you must pass '--from' and/or '--to'"); + } + + ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + + copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()), + NoRepair, checkSigs, substitute); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdCopy>()); diff --git a/third_party/nix/src/nix/doctor.cc b/third_party/nix/src/nix/doctor.cc new file mode 100644 index 000000000000..d0b4c2b58873 --- /dev/null +++ b/third_party/nix/src/nix/doctor.cc @@ -0,0 +1,142 @@ +#include <absl/strings/match.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> + +#include "libmain/shared.hh" +#include "libstore/serve-protocol.hh" +#include "libstore/store-api.hh" +#include "libstore/worker-protocol.hh" +#include "nix/command.hh" + +namespace nix { +static std::string formatProtocol(unsigned int proto) { + if (proto != 0u) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; +} + +struct CmdDoctor final : StoreCommand { + bool success = true; + + std::string name() override { return "doctor"; } + + std::string description() override { + return "check your system for potential problems"; + } + + void run(ref<Store> store) override { + std::cout << "Store uri: " << store->getUri() << std::endl; + std::cout << std::endl; + + auto type = getStoreType(); + + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); + } + success &= checkStoreProtocol(store->getProtocol()); + + if (!success) { + throw Exit(2); + } + } + + static bool checkNixInPath() { + PathSet dirs; + + for (auto& dir : absl::StrSplit(getEnv("PATH").value_or(""), + absl::ByChar(':'), absl::SkipEmpty())) { + if (pathExists(absl::StrCat(dir, "/nix-env"))) { + dirs.insert(dirOf(canonPath(absl::StrCat(dir, "/nix-env"), true))); + } + } + + if (dirs.size() != 1) { + std::cout << "Warning: multiple versions of nix found in PATH." + << std::endl; + std::cout << std::endl; + for (auto& dir : dirs) { + std::cout << " " << dir << std::endl; + } + std::cout << std::endl; + return false; + } + + return true; + } + + static bool checkProfileRoots(const ref<Store>& store) { + PathSet dirs; + + for (auto dir : absl::StrSplit(getEnv("PATH").value_or(""), + absl::ByChar(':'), absl::SkipEmpty())) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); + + if (store->isStorePath(userEnv) && + absl::EndsWith(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && + isLink(profileDir)) { + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + } + + if (profileDir.find("/profiles/") == std::string::npos) { + dirs.insert(std::string(dir)); + } + } + } catch (SysError&) { + } + } + + if (!dirs.empty()) { + std::cout << "Warning: found profiles outside of " << settings.nixStateDir + << "/profiles." << std::endl; + std::cout << "The generation this profile points to might not have a " + "gcroot and could be" + << std::endl; + std::cout << "garbage collected, resulting in broken symlinks." + << std::endl; + std::cout << std::endl; + for (auto& dir : dirs) { + std::cout << " " << dir << std::endl; + } + std::cout << std::endl; + return false; + } + + return true; + } + + static bool checkStoreProtocol(unsigned int storeProto) { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == + GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::cout << "Warning: protocol version of this client does not match " + "the store." + << std::endl; + std::cout << "While this is not necessarily a problem it's recommended " + "to keep the client in" + << std::endl; + std::cout << "sync with the daemon." << std::endl; + std::cout << std::endl; + std::cout << "Client protocol: " << formatProtocol(clientProto) + << std::endl; + std::cout << "Store protocol: " << formatProtocol(storeProto) + << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdDoctor>()); diff --git a/third_party/nix/src/nix/dump-path.cc b/third_party/nix/src/nix/dump-path.cc new file mode 100644 index 000000000000..1d0a996e561c --- /dev/null +++ b/third_party/nix/src/nix/dump-path.cc @@ -0,0 +1,28 @@ +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdDumpPath final : StorePathCommand { + std::string name() override { return "dump-path"; } + + std::string description() override { + return "dump a store path to stdout (in NAR format)"; + } + + Examples examples() override { + return { + Example{"To get a NAR from the binary cache https://cache.nixos.org/:", + "nix dump-path --store https://cache.nixos.org/ " + "/nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"}, + }; + } + + void run(ref<Store> store, const Path& storePath) override { + FdSink sink(STDOUT_FILENO); + store->narFromPath(storePath, sink); + sink.flush(); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdDumpPath>()); diff --git a/third_party/nix/src/nix/edit.cc b/third_party/nix/src/nix/edit.cc new file mode 100644 index 000000000000..04c67acb94c6 --- /dev/null +++ b/third_party/nix/src/nix/edit.cc @@ -0,0 +1,75 @@ +#include <absl/strings/str_split.h> +#include <glog/logging.h> +#include <unistd.h> + +#include "libexpr/attr-path.hh" +#include "libexpr/eval.hh" +#include "libmain/shared.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdEdit final : InstallableCommand { + std::string name() override { return "edit"; } + + std::string description() override { + return "open the Nix expression of a Nix package in $EDITOR"; + } + + Examples examples() override { + return { + Example{"To open the Nix expression of the GNU Hello package:", + "nix edit nixpkgs.hello"}, + }; + } + + void run(ref<Store> store) override { + auto state = getEvalState(); + + auto v = installable->toValue(*state); + + Value* v2; + try { + auto dummyArgs = Bindings::New(); + v2 = findAlongAttrPath(*state, "meta.position", dummyArgs.get(), *v); + } catch (Error&) { + throw Error("package '%s' has no source location information", + installable->what()); + } + + auto pos = state->forceString(*v2); + DLOG(INFO) << "position is " << pos; + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) { + throw Error("cannot parse meta.position attribute '%s'", pos); + } + + std::string filename(pos, 0, colon); + int lineno; + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument& e) { + throw Error("cannot parse line number '%s'", pos); + } + + auto editor = getEnv("EDITOR").value_or("cat"); + + Strings args = + absl::StrSplit(editor, absl::ByAnyChar(" \t\n\r"), absl::SkipEmpty()); + + if (editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos) { + args.push_back(fmt("+%d", lineno)); + } + + args.push_back(filename); + + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); + + throw SysError("cannot run editor '%s'", editor); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdEdit>()); diff --git a/third_party/nix/src/nix/eval.cc b/third_party/nix/src/nix/eval.cc new file mode 100644 index 000000000000..72fcbd82717d --- /dev/null +++ b/third_party/nix/src/nix/eval.cc @@ -0,0 +1,56 @@ +#include "libexpr/eval.hh" + +#include "libexpr/value-to-json.hh" +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "libutil/json.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdEval final : MixJSON, InstallableCommand { + bool raw = false; + + CmdEval() { mkFlag(0, "raw", "print strings unquoted", &raw); } + + std::string name() override { return "eval"; } + + std::string description() override { return "evaluate a Nix expression"; } + + Examples examples() override { + return { + Example{"To evaluate a Nix expression given on the command line:", + "nix eval '(1 + 2)'"}, + Example{"To evaluate a Nix expression from a file or URI:", + "nix eval -f channel:nixos-17.09 hello.name"}, + Example{"To get the current version of Nixpkgs:", + "nix eval --raw nixpkgs.lib.nixpkgsVersion"}, + Example{"To print the store path of the Hello package:", + "nix eval --raw nixpkgs.hello"}, + }; + } + + void run(ref<Store> store) override { + if (raw && json) { + throw UsageError("--raw and --json are mutually exclusive"); + } + + auto state = getEvalState(); + + auto v = installable->toValue(*state); + PathSet context; + + if (raw) { + std::cout << state->coerceToString(noPos, *v, context); + } else if (json) { + JSONPlaceholder jsonOut(std::cout); + printValueAsJSON(*state, true, *v, jsonOut, context); + } else { + state->forceValueDeep(*v); + std::cout << *v << "\n"; + } + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdEval>()); diff --git a/third_party/nix/src/nix/hash.cc b/third_party/nix/src/nix/hash.cc new file mode 100644 index 000000000000..4fb262f1a8f0 --- /dev/null +++ b/third_party/nix/src/nix/hash.cc @@ -0,0 +1,152 @@ +#include "libutil/hash.hh" + +#include "libmain/shared.hh" +#include "nix/command.hh" +#include "nix/legacy.hh" + +namespace nix { +struct CmdHash final : Command { + enum Mode { mFile, mPath }; + Mode mode; + Base base = SRI; + bool truncate = false; + HashType ht = htSHA256; + std::vector<std::string> paths; + + explicit CmdHash(Mode mode) : mode(mode) { + mkFlag(0, "sri", "print hash in SRI format", &base, SRI); + mkFlag(0, "base64", "print hash in base-64", &base, Base64); + mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); + mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag().longName("type").mkHashTypeFlag(&ht); + expectArgs("paths", &paths); + } + + std::string name() override { + return mode == mFile ? "hash-file" : "hash-path"; + } + + std::string description() override { + return mode == mFile + ? "print cryptographic hash of a regular file" + : "print cryptographic hash of the NAR serialisation of a path"; + } + + void run() override { + for (const auto& path : paths) { + Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + if (truncate && h.hashSize > nix::kStorePathHashSize) { + h = compressHash(h, nix::kStorePathHashSize); + } + std::cout << format("%1%\n") % h.to_string(base, base == SRI); + } + } +}; + +static RegisterCommand r1(make_ref<CmdHash>(CmdHash::mFile)); +static RegisterCommand r2(make_ref<CmdHash>(CmdHash::mPath)); + +struct CmdToBase final : Command { + Base base; + HashType ht = htUnknown; + std::vector<std::string> args; + + explicit CmdToBase(Base base) : base(base) { + mkFlag().longName("type").mkHashTypeFlag(&ht); + expectArgs("strings", &args); + } + + std::string name() override { + return base == Base16 ? "to-base16" + : base == Base32 ? "to-base32" + : base == Base64 ? "to-base64" + : "to-sri"; + } + + std::string description() override { + return fmt("convert a hash to %s representation", + base == Base16 ? "base-16" + : base == Base32 ? "base-32" + : base == Base64 ? "base-64" + : "SRI"); + } + + void run() override { + for (const auto& s : args) { + auto hash_ = Hash::deserialize(s, ht); + if (hash_.ok()) { + std::cout << hash_->to_string(base, base == SRI) << "\n"; + } else { + std::cerr << "failed to parse: " << hash_.status().ToString() << "\n"; + // create a matching blank line, for scripting + std::cout << "\n"; + } + } + } +}; + +static RegisterCommand r3(make_ref<CmdToBase>(Base16)); +static RegisterCommand r4(make_ref<CmdToBase>(Base32)); +static RegisterCommand r5(make_ref<CmdToBase>(Base64)); +static RegisterCommand r6(make_ref<CmdToBase>(SRI)); + +/* Legacy nix-hash command. */ +static int compatNixHash(int argc, char** argv) { + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + std::vector<std::string> ss; + + parseCmdLine(argc, argv, + [&](Strings::iterator& arg, const Strings::iterator& end) { + if (*arg == "--help") { + showManPage("nix-hash"); + } else if (*arg == "--version") { + printVersion("nix-hash"); + } else if (*arg == "--flat") { + flat = true; + } else if (*arg == "--base32") { + base32 = true; + } else if (*arg == "--truncate") { + truncate = true; + } else if (*arg == "--type") { + std::string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) { + throw UsageError(format("unknown hash type '%1%'") % s); + } + } else if (*arg == "--to-base16") { + op = opTo16; + } else if (*arg == "--to-base32") { + op = opTo32; + } else if (*arg != "" && arg->at(0) == '-') { + return false; + } else { + ss.push_back(*arg); + } + return true; + }); + + if (op == opHash) { + CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + cmd.ht = ht; + cmd.base = base32 ? Base32 : Base16; + cmd.truncate = truncate; + cmd.paths = ss; + cmd.run(); + } + + else { + CmdToBase cmd(op == opTo32 ? Base32 : Base16); + cmd.args = ss; + cmd.ht = ht; + cmd.run(); + } + + return 0; +} + +static RegisterLegacyCommand s1("nix-hash", compatNixHash); +} // namespace nix diff --git a/third_party/nix/src/nix/installables.cc b/third_party/nix/src/nix/installables.cc new file mode 100644 index 000000000000..7aa26b0dee8c --- /dev/null +++ b/third_party/nix/src/nix/installables.cc @@ -0,0 +1,349 @@ +#include <iostream> +#include <regex> +#include <utility> + +#include "libexpr/attr-path.hh" +#include "libexpr/common-eval-args.hh" +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/get-drvs.hh" +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/store-api.hh" +#include "libutil/status.hh" +#include "nix/command.hh" + +namespace nix { + +SourceExprCommand::SourceExprCommand() { + mkFlag() + .shortName('f') + .longName("file") + .label("file") + .description("evaluate FILE rather than the default") + .dest(&file); +} + +Value* SourceExprCommand::getSourceExpr(EvalState& state) { + if (vSourceExpr != nullptr) { + return *vSourceExpr; + } + + auto sToplevel = state.symbols.Create("_toplevel"); + + // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't + // consider the member variable "alive" during execution causing it to be + // GC'ed in the middle of evaluation. + vSourceExpr = allocRootValue(state.allocValue()); + + if (!file.empty()) { + state.evalFile(lookupFileArg(state, file), **vSourceExpr); + } else { + /* Construct the installation source from $NIX_PATH. */ + + auto searchPath = state.getSearchPath(); + + state.mkAttrs(**vSourceExpr, 1024); + + mkBool(*state.allocAttr(**vSourceExpr, sToplevel), true); + + std::unordered_set<std::string> seen; + + auto addEntry = [&](const std::string& name) { + if (name.empty()) { + return; + } + if (!seen.insert(name).second) { + return; + } + Value* v1 = state.allocValue(); + mkPrimOpApp(*v1, state.getBuiltin("findFile"), + state.getBuiltin("nixPath")); + Value* v2 = state.allocValue(); + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(**vSourceExpr, state.symbols.Create(name)), + state.getBuiltin("import"), *v2); + }; + + for (auto& i : searchPath) { /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto& j : readDirectory(i.second)) { + if (j.name != "manifest.nix" && + pathExists(fmt("%s/%s/default.nix", i.second, j.name))) { + addEntry(j.name); + } + } + } else { + addEntry(i.first); + } + } + } + + return *vSourceExpr; +} + +ref<EvalState> SourceExprCommand::getEvalState() { + if (!evalState) { + evalState = std::make_shared<EvalState>(searchPath, getStore()); + } + return ref<EvalState>(evalState); +} + +Buildable Installable::toBuildable() { + auto buildables = toBuildables(); + if (buildables.size() != 1) { + throw Error( + "installable '%s' evaluates to %d derivations, where only one is " + "expected", + what(), buildables.size()); + } + return std::move(buildables[0]); +} + +struct InstallableStorePath final : Installable { + Path storePath; + + explicit InstallableStorePath(Path storePath) + : storePath(std::move(storePath)) {} + + std::string what() override { return storePath; } + + Buildables toBuildables() override { + return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; + } +}; + +struct InstallableValue : Installable { + SourceExprCommand& cmd; + + explicit InstallableValue(SourceExprCommand& cmd) : cmd(cmd) {} + + Buildables toBuildables() override { + auto state = cmd.getEvalState(); + + auto v = toValue(*state); + + std::unique_ptr<Bindings> autoArgs = cmd.getAutoArgs(*state); + + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs.get(), drvs, false); + + Buildables res; + + PathSet drvPaths; + + for (auto& drv : drvs) { + Buildable b{drv.queryDrvPath()}; + drvPaths.insert(b.drvPath); + + auto outputName = drv.queryOutputName(); + if (outputName.empty()) { + throw Error("derivation '%s' lacks an 'outputName' attribute", + b.drvPath); + } + + b.outputs.emplace(outputName, drv.queryOutPath()); + + res.push_back(std::move(b)); + } + + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{*drvPaths.begin()}; + for (auto& b2 : res) { + b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); + } + return {b}; + } + return res; + } +}; + +struct InstallableExpr final : InstallableValue { + std::string text; + + InstallableExpr(SourceExprCommand& cmd, std::string text) + : InstallableValue(cmd), text(std::move(text)) {} + + std::string what() override { return text; } + + Value* toValue(EvalState& state) override { + auto v = state.allocValue(); + state.eval(state.parseExprFromString(text, absPath(".")), *v); + return v; + } +}; + +struct InstallableAttrPath final : InstallableValue { + std::string attrPath; + + InstallableAttrPath(SourceExprCommand& cmd, std::string attrPath) + : InstallableValue(cmd), attrPath(std::move(attrPath)) {} + + std::string what() override { return attrPath; } + + Value* toValue(EvalState& state) override { + auto source = cmd.getSourceExpr(state); + + std::unique_ptr<Bindings> autoArgs = cmd.getAutoArgs(state); + + Value* v = findAlongAttrPath(state, attrPath, autoArgs.get(), *source); + state.forceValue(*v); + + return v; + } +}; + +// FIXME: extend +std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; +static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); + +static std::vector<std::shared_ptr<Installable>> parseInstallables( + SourceExprCommand& cmd, const ref<Store>& store, + std::vector<std::string> ss, bool useDefaultInstallables) { + std::vector<std::shared_ptr<Installable>> result; + + if (ss.empty() && useDefaultInstallables) { + if (cmd.file.empty()) { + cmd.file = "."; + } + ss = {""}; + } + + for (auto& s : ss) { + if (s.compare(0, 1, "(") == 0) { + result.push_back(std::make_shared<InstallableExpr>(cmd, s)); + + } else if (s.find('/') != std::string::npos) { + auto path = store->toStorePath(store->followLinksToStore(s)); + + if (store->isStorePath(path)) { + result.push_back(std::make_shared<InstallableStorePath>(path)); + } + } + + else if (s.empty() || std::regex_match(s, attrPathRegex)) { + result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); + + } else { + throw UsageError("don't know what to do with argument '%s'", s); + } + } + + return result; +} + +std::shared_ptr<Installable> parseInstallable(SourceExprCommand& cmd, + const ref<Store>& store, + const std::string& installable, + bool useDefaultInstallables) { + auto installables = parseInstallables(cmd, store, {installable}, false); + assert(installables.size() == 1); + return installables.front(); +} + +Buildables build( + const ref<Store>& store, RealiseMode mode, + const std::vector<std::shared_ptr<Installable>>& installables) { + if (mode != Build) { + settings.readOnlyMode = true; + } + + Buildables buildables; + + PathSet pathsToBuild; + + for (auto& i : installables) { + for (auto& b : i->toBuildables()) { + if (!b.drvPath.empty()) { + StringSet outputNames; + for (auto& output : b.outputs) { + outputNames.insert(output.first); + } + pathsToBuild.insert(b.drvPath + "!" + + concatStringsSep(",", outputNames)); + } else { + for (auto& output : b.outputs) { + pathsToBuild.insert(output.second); + } + } + buildables.push_back(std::move(b)); + } + } + + if (mode == DryRun) { + printMissing(store, pathsToBuild); + } else if (mode == Build) { + util::OkOrThrow(store->buildPaths(std::cerr, pathsToBuild)); + } + + return buildables; +} + +PathSet toStorePaths( + const ref<Store>& store, RealiseMode mode, + const std::vector<std::shared_ptr<Installable>>& installables) { + PathSet outPaths; + + for (auto& b : build(store, mode, installables)) { + for (auto& output : b.outputs) { + outPaths.insert(output.second); + } + } + + return outPaths; +} + +Path toStorePath(const ref<Store>& store, RealiseMode mode, + const std::shared_ptr<Installable>& installable) { + auto paths = toStorePaths(store, mode, {installable}); + + if (paths.size() != 1) { + throw Error("argument '%s' should evaluate to one store path", + installable->what()); + } + + return *paths.begin(); +} + +PathSet toDerivations( + const ref<Store>& store, + const std::vector<std::shared_ptr<Installable>>& installables, + bool useDeriver) { + PathSet drvPaths; + + for (auto& i : installables) { + for (auto& b : i->toBuildables()) { + if (b.drvPath.empty()) { + if (!useDeriver) { + throw Error("argument '%s' did not evaluate to a derivation", + i->what()); + } + for (auto& output : b.outputs) { + auto derivers = store->queryValidDerivers(output.second); + if (derivers.empty()) { + throw Error("'%s' does not have a known deriver", i->what()); + } + // FIXME: use all derivers? + drvPaths.insert(*derivers.begin()); + } + } else { + drvPaths.insert(b.drvPath); + } + } + } + + return drvPaths; +} + +void InstallablesCommand::prepare() { + installables = parseInstallables(*this, getStore(), _installables, + useDefaultInstallables()); +} + +void InstallableCommand::prepare() { + installable = parseInstallable(*this, getStore(), _installable, false); +} + +} // namespace nix diff --git a/third_party/nix/src/nix/legacy.cc b/third_party/nix/src/nix/legacy.cc new file mode 100644 index 000000000000..a0f9fc65b311 --- /dev/null +++ b/third_party/nix/src/nix/legacy.cc @@ -0,0 +1,7 @@ +#include "nix/legacy.hh" + +namespace nix { + +RegisterLegacyCommand::Commands* RegisterLegacyCommand::commands = nullptr; + +} diff --git a/third_party/nix/src/nix/legacy.hh b/third_party/nix/src/nix/legacy.hh new file mode 100644 index 000000000000..a0fc88da244c --- /dev/null +++ b/third_party/nix/src/nix/legacy.hh @@ -0,0 +1,23 @@ +#pragma once + +#include <functional> +#include <map> +#include <string> + +namespace nix { + +typedef std::function<void(int, char**)> MainFunction; + +struct RegisterLegacyCommand { + using Commands = std::map<std::string, MainFunction>; + static Commands* commands; + + RegisterLegacyCommand(const std::string& name, MainFunction fun) { + if (!commands) { + commands = new Commands; + } + (*commands)[name] = fun; + } +}; + +} // namespace nix diff --git a/third_party/nix/src/nix/log.cc b/third_party/nix/src/nix/log.cc new file mode 100644 index 000000000000..84207d8576ea --- /dev/null +++ b/third_party/nix/src/nix/log.cc @@ -0,0 +1,63 @@ +#include <glog/logging.h> + +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdLog final : InstallableCommand { + CmdLog() = default; + + std::string name() override { return "log"; } + + std::string description() override { + return "show the build log of the specified packages or paths, if " + "available"; + } + + Examples examples() override { + return { + Example{"To get the build log of GNU Hello:", "nix log nixpkgs.hello"}, + Example{ + "To get the build log of a specific path:", + "nix log " + "/nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1"}, + Example{"To get a build log from a specific binary cache:", + "nix log --store https://cache.nixos.org nixpkgs.hello"}, + }; + } + + void run(ref<Store> store) override { + settings.readOnlyMode = true; + + auto subs = getDefaultSubstituters(); + + subs.push_front(store); + + auto b = installable->toBuildable(); + + RunPager pager; + for (auto& sub : subs) { + auto log = !b.drvPath.empty() ? sub->getBuildLog(b.drvPath) : nullptr; + for (auto& output : b.outputs) { + if (log) { + break; + } + log = sub->getBuildLog(output.second); + } + if (!log) { + continue; + } + LOG(INFO) << "got build log for '" << installable->what() << "' from '" + << sub->getUri() << "'"; + std::cout << *log; + return; + } + + throw Error("build log of '%s' is not available", installable->what()); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdLog>()); diff --git a/third_party/nix/src/nix/ls.cc b/third_party/nix/src/nix/ls.cc new file mode 100644 index 000000000000..1da722babbd3 --- /dev/null +++ b/third_party/nix/src/nix/ls.cc @@ -0,0 +1,137 @@ +#include "libmain/common-args.hh" +#include "libstore/fs-accessor.hh" +#include "libstore/nar-accessor.hh" +#include "libstore/store-api.hh" +#include "libutil/json.hh" +#include "nix/command.hh" + +namespace nix { +struct MixLs : virtual Args, MixJSON { + std::string path; + + bool recursive = false; + bool verbose = false; + bool showDirectory = false; + + MixLs() { + mkFlag('R', "recursive", "list subdirectories recursively", &recursive); + mkFlag('l', "long", "show more file information", &verbose); + mkFlag('d', "directory", "show directories rather than their contents", + &showDirectory); + } + + void listText(ref<FSAccessor> accessor) { + std::function<void(const FSAccessor::Stat&, const Path&, const std::string&, + bool)> + doPath; + + auto showFile = [&](const Path& curPath, const std::string& relPath) { + if (verbose) { + auto st = accessor->stat(curPath); + std::string tp = st.type == FSAccessor::Type::tRegular + ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") + : st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" + : "dr-xr-xr-x"; + std::cout << (format("%s %20d %s") % tp % st.fileSize % relPath); + if (st.type == FSAccessor::Type::tSymlink) { + std::cout << " -> " << accessor->readLink(curPath); + } + std::cout << "\n"; + if (recursive && st.type == FSAccessor::Type::tDirectory) { + doPath(st, curPath, relPath, false); + } + } else { + std::cout << relPath << "\n"; + if (recursive) { + auto st = accessor->stat(curPath); + if (st.type == FSAccessor::Type::tDirectory) { + doPath(st, curPath, relPath, false); + } + } + } + }; + + doPath = [&](const FSAccessor::Stat& st, const Path& curPath, + const std::string& relPath, bool showDirectory) { + if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + auto names = accessor->readDirectory(curPath); + for (auto& name : names) { + showFile(curPath + "/" + name, relPath + "/" + name); + } + } else { + showFile(curPath, relPath); + } + }; + + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) { + throw Error(format("path '%1%' does not exist") % path); + } + doPath(st, path, + st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + showDirectory); + } + + void list(const ref<FSAccessor>& accessor) { + if (path == "/") { + path = ""; + } + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + listNar(jsonRoot, accessor, path, recursive); + } else { + listText(accessor); + } + } +}; + +struct CmdLsStore final : StoreCommand, MixLs { + CmdLsStore() { expectArg("path", &path); } + + Examples examples() override { + return { + Example{"To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR " + "/nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"}, + }; + } + + std::string name() override { return "ls-store"; } + + std::string description() override { + return "show information about a store path"; + } + + void run(ref<Store> store) override { list(store->getFSAccessor()); } +}; + +struct CmdLsNar final : Command, MixLs { + Path narPath; + + CmdLsNar() { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + Examples examples() override { + return { + Example{"To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello"}, + }; + } + + std::string name() override { return "ls-nar"; } + + std::string description() override { + return "show information about the contents of a NAR file"; + } + + void run() override { + list(makeNarAccessor(make_ref<std::string>(readFile(narPath, true)))); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdLsStore>()); +static nix::RegisterCommand r2(nix::make_ref<nix::CmdLsNar>()); diff --git a/third_party/nix/src/nix/main.cc b/third_party/nix/src/nix/main.cc new file mode 100644 index 000000000000..08390fd24b9f --- /dev/null +++ b/third_party/nix/src/nix/main.cc @@ -0,0 +1,185 @@ +#include <algorithm> + +#include <glog/logging.h> +#include <ifaddrs.h> +#include <netdb.h> +#include <netinet/in.h> +#include <sys/socket.h> +#include <sys/types.h> + +#include "libexpr/eval.hh" +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/download.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/finally.hh" +#include "nix/command.hh" +#include "nix/legacy.hh" + +extern std::string chrootHelperName; + +void chrootHelper(int argc, char** argv); + +namespace nix { + +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() { + struct ifaddrs* addrs; + + if (getifaddrs(&addrs) != 0) { + return true; + } + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i != nullptr; i = i->ifa_next) { + if (i->ifa_addr == nullptr) { + continue; + } + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl( + (reinterpret_cast<sockaddr_in*>(i->ifa_addr))->sin_addr.s_addr) != + INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6*)i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6*)i->ifa_addr)->sin6_addr)) { + return true; + } + } + } + + return false; +} + +std::string programPath; + +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { + bool printBuildLogs = false; + bool useNet = true; + + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { showHelpAndExit(); }); + + mkFlag() + .longName("help-config") + .description("show configuration options") + .handler([&]() { + std::cout << "The following configuration options are available:\n\n"; + Table2 tbl; + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (const auto& s : settings) { + tbl.emplace_back(s.first, s.second.description); + } + printTable(std::cout, tbl); + throw Exit(); + }); + + mkFlag() + .longName("print-build-logs") + .shortName('L') + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + + mkFlag() + .longName("version") + .description("show version information") + .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description( + "disable substituters and consider all previously downloaded files " + "up-to-date") + .handler([&]() { useNet = false; }); + } + + void printFlags(std::ostream& out) override { + Args::printFlags(out); + std::cout << "\n" + "In addition, most configuration settings can be overriden " + "using '--<name> <value>'.\n" + "Boolean settings can be overriden using '--<name>' or " + "'--no-<name>'. See 'nix\n" + "--help-config' for a list of configuration settings.\n"; + } + + void showHelpAndExit() { + printHelp(programName, std::cout); + std::cout + << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + throw Exit(); + } +}; + +void mainWrapped(int argc, char** argv) { + /* The chroot helper needs to be run before any threads have been + started. */ + if (argc > 0 && argv[0] == chrootHelperName) { + chrootHelper(argc, argv); + return; + } + + initNix(); + + programPath = argv[0]; + std::string programName = baseNameOf(programPath); + + { + auto legacy = (*RegisterLegacyCommand::commands)[programName]; + if (legacy) { + return legacy(argc, argv); + } + } + + settings.verboseBuild = false; + + NixArgs args; + + args.parseCmdline(argvToStrings(argc, argv)); + + if (!args.command) { + args.showHelpAndExit(); + } + + if (args.useNet && !haveInternet()) { + LOG(WARNING) << "you don't have Internet access; " + << "disabling some network-dependent features"; + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) { + settings.useSubstitutes = false; + } + if (!settings.tarballTtl.overriden) { + settings.tarballTtl = std::numeric_limits<unsigned int>::max(); + } + if (!downloadSettings.tries.overriden) { + downloadSettings.tries = 0; + } + if (!downloadSettings.connectTimeout.overriden) { + downloadSettings.connectTimeout = 1; + } + } + + args.command->prepare(); + args.command->run(); +} + +} // namespace nix + +int main(int argc, char* argv[]) { + FLAGS_logtostderr = true; + google::InitGoogleLogging(argv[0]); + + return nix::handleExceptions(argv[0], + [&]() { nix::mainWrapped(argc, argv); }); +} diff --git a/third_party/nix/src/nix/optimise-store.cc b/third_party/nix/src/nix/optimise-store.cc new file mode 100644 index 000000000000..ceb53aa77b17 --- /dev/null +++ b/third_party/nix/src/nix/optimise-store.cc @@ -0,0 +1,27 @@ +#include <atomic> + +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdOptimiseStore final : StoreCommand { + CmdOptimiseStore() = default; + + std::string name() override { return "optimise-store"; } + + std::string description() override { + return "replace identical files in the store by hard links"; + } + + Examples examples() override { + return { + Example{"To optimise the Nix store:", "nix optimise-store"}, + }; + } + + void run(ref<Store> store) override { store->optimiseStore(); } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdOptimiseStore>()); diff --git a/third_party/nix/src/nix/path-info.cc b/third_party/nix/src/nix/path-info.cc new file mode 100644 index 000000000000..fcf060d50d08 --- /dev/null +++ b/third_party/nix/src/nix/path-info.cc @@ -0,0 +1,133 @@ +#include <algorithm> +#include <array> + +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "libutil/json.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdPathInfo final : StorePathsCommand, MixJSON { + bool showSize = false; + bool showClosureSize = false; + bool humanReadable = false; + bool showSigs = false; + + CmdPathInfo() { + mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); + mkFlag('S', "closure-size", + "print sum size of the NAR dumps of the closure of each path", + &showClosureSize); + mkFlag('h', "human-readable", + "with -s and -S, print sizes like 1K 234M 5.67G etc.", + &humanReadable); + mkFlag(0, "sigs", "show signatures", &showSigs); + } + + std::string name() override { return "path-info"; } + + std::string description() override { + return "query information about store paths"; + } + + Examples examples() override { + return { + Example{"To show the closure sizes of every path in the current NixOS " + "system closure, sorted by size:", + "nix path-info -rS /run/current-system | sort -nk2"}, + Example{"To show a package's closure size and all its dependencies " + "with human readable sizes:", + "nix path-info -rsSh nixpkgs.rust"}, + Example{"To check the existence of a path in a binary cache:", + "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store " + "https://cache.nixos.org/"}, + Example{"To print the 10 most recently added paths (using --json and " + "the jq(1) command):", + "nix path-info --json --all | jq -r " + "'sort_by(.registrationTime)[-11:-1][].path'"}, + Example{"To show the size of the entire Nix store:", + "nix path-info --json --all | jq 'map(.narSize) | add'"}, + Example{"To show every path whose closure is bigger than 1 GB, sorted " + "by closure size:", + "nix path-info --json --all -S | jq 'map(select(.closureSize > " + "1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'"}, + }; + } + + void printSize(unsigned long long value) { + if (!humanReadable) { + std::cout << fmt("\t%11d", value); + return; + } + + static const std::array<char, 9> idents{ + {' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y'}}; + size_t power = 0; + double res = value; + while (res > 1024 && power < idents.size()) { + ++power; + res /= 1024; + } + std::cout << fmt("\t%6.1f%c", res, idents.at(power)); + } + + void run(ref<Store> store, Paths storePaths) override { + size_t pathLen = 0; + for (auto& storePath : storePaths) { + pathLen = std::max(pathLen, storePath.size()); + } + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + store->pathInfoToJSON(jsonRoot, + // FIXME: preserve order? + PathSet(storePaths.begin(), storePaths.end()), true, + showClosureSize, AllowInvalid); + } + + else { + for (auto storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + storePath = info->path; // FIXME: screws up padding + + std::cout << storePath; + + if (showSize || showClosureSize || showSigs) { + std::cout << std::string( + std::max(0, static_cast<int>(pathLen) - + static_cast<int>(storePath.size())), + ' '); + } + + if (showSize) { + printSize(info->narSize); + } + + if (showClosureSize) { + printSize(store->getClosureSize(storePath).first); + } + + if (showSigs) { + std::cout << '\t'; + Strings ss; + if (info->ultimate) { + ss.push_back("ultimate"); + } + if (!info->ca.empty()) { + ss.push_back("ca:" + info->ca); + } + for (auto& sig : info->sigs) { + ss.push_back(sig); + } + std::cout << concatStringsSep(" ", ss); + } + + std::cout << std::endl; + } + } + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdPathInfo>()); diff --git a/third_party/nix/src/nix/ping-store.cc b/third_party/nix/src/nix/ping-store.cc new file mode 100644 index 000000000000..4a33486bf8c6 --- /dev/null +++ b/third_party/nix/src/nix/ping-store.cc @@ -0,0 +1,25 @@ +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdPingStore final : StoreCommand { + std::string name() override { return "ping-store"; } + + std::string description() override { + return "test whether a store can be opened"; + } + + Examples examples() override { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1"}, + }; + } + + void run(ref<Store> store) override { store->connect(); } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdPingStore>()); diff --git a/third_party/nix/src/nix/repl.cc b/third_party/nix/src/nix/repl.cc new file mode 100644 index 000000000000..b926d195aec1 --- /dev/null +++ b/third_party/nix/src/nix/repl.cc @@ -0,0 +1,819 @@ +#include <climits> +#include <csetjmp> +#include <cstdlib> +#include <cstring> +#include <iostream> +#include <utility> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <editline.h> +#include <glog/logging.h> + +#include "libexpr/common-eval-args.hh" +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/get-drvs.hh" +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/globals.hh" +#include "libstore/store-api.hh" +#include "libutil/affinity.hh" +#include "libutil/finally.hh" +#include "nix/command.hh" + +namespace nix { + +#define ESC_RED "\033[31m" +#define ESC_GRE "\033[32m" +#define ESC_YEL "\033[33m" +#define ESC_BLU "\033[34;1m" +#define ESC_MAG "\033[35m" +#define ESC_CYA "\033[36m" +#define ESC_END "\033[0m" + +struct NixRepl { + std::string curDir; + EvalState state; + std::unique_ptr<Bindings> autoArgs; + + Strings loadedFiles; + + const static int envSize = 32768; + StaticEnv staticEnv; + Env* env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings& searchPath, const nix::ref<Store>& store); + ~NixRepl(); + void mainLoop(const std::vector<std::string>& files); + StringSet completePrefix(const std::string& prefix); + static bool getLine(std::string& input, const std::string& prompt); + Path getDerivationPath(Value& v); + bool processLine(std::string line); + void loadFile(const Path& path); + void initEnv(); + void reloadFiles(); + void addAttrsToScope(Value& attrs); + void addVarToScope(const Symbol& name, Value& v); + Expr* parseString(const std::string& s); + void evalString(std::string s, Value& v); + + using ValuesSeen = std::set<Value*>; + std::ostream& printValue(std::ostream& str, Value& v, unsigned int maxDepth); + std::ostream& printValue(std::ostream& str, Value& v, unsigned int maxDepth, + ValuesSeen& seen); +}; + +void printHelp() { + std::cout << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" + << "\n" + << "nix-repl is a simple read-eval-print loop (REPL) for the Nix " + "package manager.\n" + << "\n" + << "Options:\n" + << " --help\n" + << " Prints out a summary of the command syntax and exits.\n" + << "\n" + << " --version\n" + << " Prints out the Nix version number on standard output " + "and exits.\n" + << "\n" + << " -I path\n" + << " Add a path to the Nix expression search path. This " + "option may be given\n" + << " multiple times. See the NIX_PATH environment variable " + "for information on\n" + << " the semantics of the Nix search path. Paths added " + "through -I take\n" + << " precedence over NIX_PATH.\n" + << "\n" + << " paths...\n" + << " A list of paths to files containing Nix expressions " + "which nix-repl will\n" + << " load and add to its scope.\n" + << "\n" + << " A path surrounded in < and > will be looked up in the " + "Nix expression search\n" + << " path, as in the Nix language itself.\n" + << "\n" + << " If an element of paths starts with http:// or " + "https://, it is interpreted\n" + << " as the URL of a tarball that will be downloaded and " + "unpacked to a temporary\n" + << " location. The tarball must include a single top-level " + "directory containing\n" + << " at least a file named default.nix.\n"; +} + +std::string removeWhitespace(std::string s) { + s = absl::StripTrailingAsciiWhitespace(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != std::string::npos) { + s = std::string(s, n); + } + return s; +} + +NixRepl::NixRepl(const Strings& searchPath, const nix::ref<Store>& store) + : state(searchPath, store), + staticEnv(false, &state.staticBaseEnv), + historyFile(getDataDir() + "/nix/repl-history") { + curDir = absPath("."); +} + +NixRepl::~NixRepl() { write_history(historyFile.c_str()); } + +static NixRepl* curRepl; // ugly + +static char* completionCallback(char* s, int* match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto* res = strdup(possible.begin()->c_str() + strlen(s)); + if (res == nullptr) { + throw Error("allocation failure"); + } + return res; + } + if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto& first = *possible.begin(); + for (auto& p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) { + return false; + } + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) { + ++len; + } + if (len > 0) { + *match = 1; + auto* res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (res == nullptr) { + throw Error("allocation failure"); + } + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char* s, char*** avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) { + throw Error("too many completions"); + } + + int ac = 0; + char** vp = nullptr; + + auto check = [&](auto* p) { + if (!p) { + if (vp) { + while (--ac >= 0) { + free(vp[ac]); + } + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check(static_cast<char**>(malloc(possible.size() * sizeof(char*)))); + + for (auto& p : possible) { + vp[ac++] = check(strdup(p.c_str())); + } + + *avp = vp; + + return ac; +} + +namespace { +// Used to communicate to NixRepl::getLine whether a signal occurred in +// ::readline. +volatile sig_atomic_t g_signal_received = 0; + +void sigintHandler(int signo) { g_signal_received = signo; } +} // namespace + +void NixRepl::mainLoop(const std::vector<std::string>& files) { + std::string error = ANSI_RED "error:" ANSI_NORMAL " "; + std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." + << std::endl + << std::endl; + + for (auto& i : files) { + loadedFiles.push_back(i); + } + + reloadFiles(); + if (!loadedFiles.empty()) { + std::cout << std::endl; + } + + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + createDirs(dirOf(historyFile)); + el_hist_size = 1000; + read_history(historyFile.c_str()); + curRepl = this; + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); + + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just + // align to the same number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { + break; + } + + try { + if (!removeWhitespace(input).empty() && !processLine(input)) { + return; + } + } catch (ParseError& e) { + if (e.msg().find("unexpected $end") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the + // next line of input without clearing the input so far. + continue; + } + LOG(ERROR) << error << (settings.showTrace ? e.prefix() : "") << e.msg(); + + } catch (Error& e) { + LOG(ERROR) << error << (settings.showTrace ? e.prefix() : "") << e.msg(); + } catch (Interrupted& e) { + LOG(ERROR) << error << (settings.showTrace ? e.prefix() : "") << e.msg(); + } + + // We handled the current input fully, so we should clear it + // and read brand new input. + input.clear(); + std::cout << std::endl; + } +} + +bool NixRepl::getLine(std::string& input, const std::string& prompt) { + struct sigaction act; + struct sigaction old; + sigset_t savedSignalMask; + sigset_t set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old) != 0) { + throw SysError("installing handler for SIGINT"); + } + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask) != 0) { + throw SysError("unblocking SIGINT"); + } + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr) != 0) { + throw SysError("restoring signals"); + } + + if (sigaction(SIGINT, &old, nullptr) != 0) { + throw SysError("restoring handler for SIGINT"); + } + }; + + setupSignals(); + char* s = readline(prompt.c_str()); + Finally doFree([&]() { free(s); }); + restoreSignals(); + + if (g_signal_received != 0) { + g_signal_received = 0; + input.clear(); + return true; + } + + if (s == nullptr) { + return false; + } + input += s; + input += '\n'; + return true; +} + +StringSet NixRepl::completePrefix(const std::string& prefix) { + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev; + std::string cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } + + size_t slash; + size_t dot; + + if ((slash = cur.rfind('/')) != std::string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto& entry : readDirectory(dir.empty() ? "/" : dir)) { + if (entry.name[0] != '.' && absl::StartsWith(entry.name, prefix2)) { + completions.insert(prev + dir + "/" + entry.name); + } + } + } catch (Error&) { + } + } else if ((dot = cur.rfind('.')) == std::string::npos) { + /* This is a variable name; look it up in the current scope. */ + auto i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (std::string(*i, 0, cur.size()) != cur) { + break; + } + completions.insert(prev + *i); + i++; + } + } else { + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + std::string expr(cur, 0, dot); + std::string cur2 = std::string(cur, dot + 1); + + Expr* e = parseString(expr); + Value v; + e->eval(state, *env, v); + state.forceAttrs(v); + + for (auto& i : *v.attrs) { + std::string name = i.second.name; + if (std::string(name, 0, cur2.size()) != cur2) { + continue; + } + completions.insert(prev + expr + "." + name); + } + + } catch (ParseError& e) { + // Quietly ignore parse errors. + } catch (EvalError& e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError& e) { + // Quietly ignore undefined variable errors. + } + } + + return completions; +} + +static int runProgram(const std::string& program, const Strings& args) { + Strings args2(args); + args2.push_front(program); + + Pid pid; + pid = fork(); + if (pid == Pid(-1)) { + throw SysError("forking"); + } + if (pid == Pid(0)) { + restoreAffinity(); + execvp(program.c_str(), stringsToCharPtrs(args2).data()); + _exit(1); + } + + return pid.wait(); +} + +bool isVarName(const std::string& s) { + if (s.empty()) { + return false; + } + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') { + return false; + } + for (auto& i : s) { + if (!((i >= 'a' && i <= 'z') || (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || i == '_' || i == '-' || i == '\'')) { + return false; + } + } + return true; +} + +Path NixRepl::getDerivationPath(Value& v) { + auto drvInfo = getDerivation(state, v, false); + if (!drvInfo) { + throw Error( + "expression does not evaluate to a derivation, so I can't build it"); + } + Path drvPath = drvInfo->queryDrvPath(); + if (drvPath.empty() || !state.store->isValidPath(drvPath)) { + throw Error("expression did not evaluate to a valid derivation"); + } + return drvPath; +} + +bool NixRepl::processLine(std::string line) { + if (line.empty()) { + return true; + } + + std::string command; + std::string arg; + + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = std::string(line, 0, p); + if (p != std::string::npos) { + arg = removeWhitespace(std::string(line, p)); + } + } else { + arg = line; + } + + if (command == ":?" || command == ":help") { + std::cout << "The following commands are available:\n" + << "\n" + << " <expr> Evaluate and print expression\n" + << " <x> = <expr> Bind expression to variable\n" + << " :a <expr> Add attributes from resulting set to scope\n" + << " :b <expr> Build derivation\n" + << " :i <expr> Build derivation, then install result into " + "current profile\n" + << " :l <path> Load Nix expression and add it to scope\n" + << " :p <expr> Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :s <expr> Build dependencies of derivation, then start " + "nix-shell\n" + << " :t <expr> Describe result of evaluation\n" + << " :u <expr> Build derivation, then start nix-shell\n"; + } + + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } + + else if (command == ":l" || command == ":load") { + state.resetFileCache(); + loadFile(arg); + } + + else if (command == ":r" || command == ":reload") { + state.resetFileCache(); + reloadFiles(); + } + + else if (command == ":t") { + Value v; + evalString(arg, v); + std::cout << showType(v) << std::endl; + + } else if (command == ":u") { + Value v; + Value f; + Value result; + evalString(arg, v); + evalString( + "drv: (import <nixpkgs> {}).runCommand \"shell\" { buildInputs = [ drv " + "]; } \"\"", + f); + state.callFunction(f, v, result, Pos()); + + Path drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + + else if (command == ":b" || command == ":i" || command == ":s") { + Value v; + evalString(arg, v); + Path drvPath = getDerivationPath(v); + + if (command == ":b") { + /* We could do the build in this process using buildPaths(), + but doing it in a child makes it easier to recover from + problems / SIGINT. */ + if (runProgram(settings.nixBinDir + "/nix", + Strings{"build", "--no-link", drvPath}) == 0) { + Derivation drv = readDerivation(drvPath); + std::cout << std::endl + << "this derivation produced the following outputs:" + << std::endl; + for (auto& i : drv.outputs) { + std::cout << format(" %1% -> %2%") % i.first % i.second.path + << std::endl; + } + } + } else if (command == ":i") { + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + } else { + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + } + + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } + + else if (command == ":q" || command == ":quit") { + return false; + + } else if (!command.empty()) { + throw Error(format("unknown command '%1%'") % command); + + } else { + size_t p = line.find('='); + std::string name; + if (p != std::string::npos && p < line.size() && line[p + 1] != '=' && + isVarName(name = removeWhitespace(std::string(line, 0, p)))) { + Expr* e = parseString(std::string(line, p + 1)); + Value& v(*state.allocValue()); + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = e; + addVarToScope(state.symbols.Create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; + } + } + + return true; +} + +void NixRepl::loadFile(const Path& path) { + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v; + Value v2; + state.evalFile(lookupFileArg(state, path), v); + state.autoCallFunction(autoArgs.get(), v, v2); + addAttrsToScope(v2); +} + +void NixRepl::initEnv() { + env = &state.allocEnv(envSize); + env->up = &state.baseEnv; + displ = 0; + staticEnv.vars.clear(); + + varNames.clear(); + for (auto& i : state.staticBaseEnv.vars) { + varNames.insert(i.first); + } +} + +void NixRepl::reloadFiles() { + initEnv(); + + Strings old = loadedFiles; + loadedFiles.clear(); + + bool first = true; + for (auto& i : old) { + if (!first) { + std::cout << std::endl; + } + first = false; + std::cout << format("Loading '%1%'...") % i << std::endl; + loadFile(i); + } +} + +void NixRepl::addAttrsToScope(Value& attrs) { + state.forceAttrs(attrs); + for (auto& i : *attrs.attrs) { + addVarToScope(i.second.name, *i.second.value); + } + std::cout << format("Added %1% variables.") % attrs.attrs->size() + << std::endl; +} + +void NixRepl::addVarToScope(const Symbol& name, Value& v) { + if (displ >= envSize) { + throw Error("environment full; cannot add more variables"); + } + staticEnv.vars[name] = displ; + env->values[displ++] = &v; + varNames.insert(std::string(name)); +} + +Expr* NixRepl::parseString(const std::string& s) { + Expr* e = state.parseExprFromString(s, curDir, staticEnv); + return e; +} + +void NixRepl::evalString(std::string s, Value& v) { + Expr* e = parseString(std::move(s)); + e->eval(state, *env, v); + state.forceValue(v); +} + +std::ostream& NixRepl::printValue(std::ostream& str, Value& v, + unsigned int maxDepth) { + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); +} + +std::ostream& printStringValue(std::ostream& str, const char* string) { + str << "\""; + for (const char* i = string; *i != 0; i++) { + if (*i == '\"' || *i == '\\') { + str << "\\" << *i; + } else if (*i == '\n') { + str << "\\n"; + } else if (*i == '\r') { + str << "\\r"; + } else if (*i == '\t') { + str << "\\t"; + } else { + str << *i; + } + } + str << "\""; + return str; +} + +// FIXME: lot of cut&paste from Nix's eval.cc. +std::ostream& NixRepl::printValue(std::ostream& str, Value& v, + unsigned int maxDepth, ValuesSeen& seen) { + str.flush(); + checkInterrupt(); + + state.forceValue(v); + + switch (v.type) { + case tInt: + str << ESC_CYA << v.integer << ESC_END; + break; + + case tBool: + str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + break; + + case tString: + str << ESC_YEL; + printStringValue(str, v.string.s); + str << ESC_END; + break; + + case tPath: + str << ESC_GRE << v.path << ESC_END; // !!! escaping? + break; + + case tNull: + str << ESC_CYA "null" ESC_END; + break; + + case tAttrs: { + seen.insert(&v); + + bool isDrv = state.isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state.sDrvPath); + PathSet context; + Path drvPath = + i != v.attrs->end() + ? state.coerceToPath(*i->second.pos, *i->second.value, context) + : "???"; + str << drvPath << "»"; + } + + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map<std::string, Value*> Sorted; + Sorted sorted; + for (auto& i : *v.attrs) { + sorted[i.second.name] = i.second.value; + } + + for (auto& i : sorted) { + if (isVarName(i.first)) { + str << i.first; + } else { + printStringValue(str, i.first.c_str()); + } + str << " = "; + if (seen.find(i.second) != seen.end()) { + str << "«repeated»"; + } else { + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError& e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + } + str << "; "; + } + + str << "}"; + } else { + str << "{ ... }"; + } + + break; + } + + case tList: + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (seen.find((*v.list)[n]) != seen.end()) { + str << "«repeated»"; + } else { + try { + printValue(str, *(*v.list)[n], maxDepth - 1, seen); + } catch (AssertionError& e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + } + str << " "; + } + } else { + str << "... "; + } + + str << "]"; + break; + + case tLambda: { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + break; + } + + case tPrimOp: + str << ESC_MAG "«primop»" ESC_END; + break; + + case tPrimOpApp: + str << ESC_BLU "«primop-app»" ESC_END; + break; + + case tFloat: + str << v.fpoint; + break; + + default: + str << ESC_RED "«unknown»" ESC_END; + break; + } + + return str; +} + +struct CmdRepl final : StoreCommand, MixEvalArgs { + std::vector<std::string> files; + + CmdRepl() { expectArgs("files", &files); } + + std::string name() override { return "repl"; } + + std::string description() override { + return "start an interactive environment for evaluating Nix expressions"; + } + + void run(ref<Store> store) override { + auto repl = std::make_unique<NixRepl>(searchPath, openStore()); + repl->autoArgs = getAutoArgs(repl->state); + repl->mainLoop(files); + } +}; + +static RegisterCommand r1(make_ref<CmdRepl>()); + +} // namespace nix diff --git a/third_party/nix/src/nix/run.cc b/third_party/nix/src/nix/run.cc new file mode 100644 index 000000000000..b3b54f300b4a --- /dev/null +++ b/third_party/nix/src/nix/run.cc @@ -0,0 +1,283 @@ +#include <queue> + +#include <absl/strings/str_split.h> +#include <sys/mount.h> + +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/derivations.hh" +#include "libstore/fs-accessor.hh" +#include "libstore/local-store.hh" +#include "libstore/store-api.hh" +#include "libutil/affinity.hh" +#include "libutil/finally.hh" +#include "nix/command.hh" + +// note: exported in header file +std::string chrootHelperName = "__run_in_chroot"; + +namespace nix { +struct CmdRun final : InstallablesCommand { + std::vector<std::string> command = {"bash"}; + StringSet keep, unset; + bool ignoreEnvironment = false; + + CmdRun() { + mkFlag() + .longName("command") + .shortName('c') + .description("command and arguments to be executed; defaults to 'bash'") + .labels({"command", "args"}) + .arity(ArityAny) + .handler([&](const std::vector<std::string>& ss) { + if (ss.empty()) { + throw UsageError("--command requires at least one argument"); + } + command = ss; + }); + + mkFlag() + .longName("ignore-environment") + .shortName('i') + .description( + "clear the entire environment (except those specified with --keep)") + .set(&ignoreEnvironment, true); + + mkFlag() + .longName("keep") + .shortName('k') + .description("keep specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector<std::string> ss) { keep.insert(ss.front()); }); + + mkFlag() + .longName("unset") + .shortName('u') + .description("unset specified environment variable") + .arity(1) + .labels({"name"}) + .handler( + [&](std::vector<std::string> ss) { unset.insert(ss.front()); }); + } + + std::string name() override { return "run"; } + + std::string description() override { + return "run a shell in which the specified packages are available"; + } + + Examples examples() override { + return { + Example{"To start a shell providing GNU Hello from NixOS 17.03:", + "nix run -f channel:nixos-17.03 hello"}, + Example{"To start a shell providing youtube-dl from your 'nixpkgs' " + "channel:", + "nix run nixpkgs.youtube-dl"}, + Example{"To run GNU Hello:", + "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'"}, + Example{"To run GNU Hello in a chroot store:", + "nix run --store ~/my-nix nixpkgs.hello -c hello"}, + }; + } + + void run(ref<Store> store) override { + auto outPaths = toStorePaths(store, Build, installables); + + auto accessor = store->getFSAccessor(); + + if (ignoreEnvironment) { + if (!unset.empty()) { + throw UsageError( + "--unset does not make sense with --ignore-environment"); + } + + std::map<std::string, std::string> kept; + for (auto& var : keep) { + auto s = getenv(var.c_str()); + if (s != nullptr) { + kept[var] = s; + } + } + + clearEnv(); + + for (auto& var : kept) { + setenv(var.first.c_str(), var.second.c_str(), 1); + } + + } else { + if (!keep.empty()) { + throw UsageError( + "--keep does not make sense without --ignore-environment"); + } + + for (auto& var : unset) { + unsetenv(var.c_str()); + } + } + + std::unordered_set<Path> done; + std::queue<Path> todo; + for (auto& path : outPaths) { + todo.push(path); + } + + Strings unixPath = absl::StrSplit(getEnv("PATH").value_or(""), + absl::ByChar(':'), absl::SkipEmpty()); + + while (!todo.empty()) { + Path path = todo.front(); + todo.pop(); + if (!done.insert(path).second) { + continue; + } + + { unixPath.push_front(path + "/bin"); } + + auto propPath = path + "/nix-support/propagated-user-env-packages"; + if (accessor->stat(propPath).type == FSAccessor::tRegular) { + for (auto p : + absl::StrSplit(readFile(propPath), absl::ByAnyChar(" \t\n\r"), + absl::SkipEmpty())) { + todo.push(std::string(p)); + } + } + } + + setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); + + std::string cmd = *command.begin(); + Strings args; + for (auto& arg : command) { + args.push_back(arg); + } + + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast<LocalStore>(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = {chrootHelperName, store->storeDir, + store2->realStoreDir, cmd}; + for (auto& arg : args) { + helperArgs.push_back(arg); + } + + execv(readLink("/proc/self/exe").c_str(), + stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to exec '%s'", cmd); + } +}; + +static RegisterCommand r1(make_ref<CmdRun>()); +} // namespace nix + +void chrootHelper(int argc, char** argv) { + int p = 1; + std::string storeDir = argv[p++]; + std::string realStoreDir = argv[p++]; + std::string cmd = argv[p++]; + nix::Strings args; + while (p < argc) { + args.push_back(argv[p++]); + } + +#if __linux__ + uid_t uid = getuid(); + uid_t gid = getgid(); + + if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) { + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) { + throw nix::SysError("setting up a private mount namespace"); + } + } + + /* Bind-mount realStoreDir on /nix/store. If the latter mount + point doesn't already exists, we have to create a chroot + environment containing the mount point and bind mounts for the + children of /. Would be nice if we could use overlayfs here, + but that doesn't work in a user namespace yet (Ubuntu has a + patch for this: + https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ + if (!nix::pathExists(storeDir)) { + // FIXME: Use overlayfs? + + nix::Path tmpDir = nix::createTempDir(); + + nix::createDirs(tmpDir + storeDir); + + if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, + nullptr) == -1) { + throw nix::SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + } + + for (const auto& entry : nix::readDirectory("/")) { + auto src = "/" + entry.name; + auto st = nix::lstat(src); + if (!S_ISDIR(st.st_mode)) { + continue; + } + nix::Path dst = tmpDir + "/" + entry.name; + if (nix::pathExists(dst)) { + continue; + } + if (mkdir(dst.c_str(), 0700) == -1) { + throw nix::SysError("creating directory '%s'", dst); + } + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, nullptr) == + -1) { + throw nix::SysError("mounting '%s' on '%s'", src, dst); + } + } + + char* cwd = getcwd(nullptr, 0); + if (cwd == nullptr) { + throw nix::SysError("getting current directory"); + } + ::Finally freeCwd([&]() { free(cwd); }); + + if (chroot(tmpDir.c_str()) == -1) { + throw nix::SysError(nix::format("chrooting into '%s'") % tmpDir); + } + + if (chdir(cwd) == -1) { + throw nix::SysError(nix::format("chdir to '%s' in chroot") % cwd); + } + } else if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, + nullptr) == -1) { + throw nix::SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + } + + nix::writeFile("/proc/self/setgroups", "deny"); + nix::writeFile("/proc/self/uid_map", nix::fmt("%d %d %d", uid, uid, 1)); + nix::writeFile("/proc/self/gid_map", nix::fmt("%d %d %d", gid, gid, 1)); + + execvp(cmd.c_str(), nix::stringsToCharPtrs(args).data()); + + throw nix::SysError("unable to exec '%s'", cmd); + +#else + throw nix::Error( + "mounting the Nix store on '%s' is not supported on this platform", + storeDir); +#endif +} diff --git a/third_party/nix/src/nix/search.cc b/third_party/nix/src/nix/search.cc new file mode 100644 index 000000000000..5a6bae6a1161 --- /dev/null +++ b/third_party/nix/src/nix/search.cc @@ -0,0 +1,276 @@ +#include <fstream> +#include <regex> + +#include <glog/logging.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/get-drvs.hh" +#include "libexpr/json-to-value.hh" +#include "libexpr/names.hh" +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/globals.hh" +#include "libutil/json.hh" +#include "nix/command.hh" + +namespace { +std::string wrap(const std::string& prefix, const std::string& s) { + return prefix + s + ANSI_NORMAL; +} + +std::string hilite(const std::string& s, const std::smatch& m, + const std::string& postfix) { + return m.empty() ? s + : std::string(m.prefix()) + ANSI_RED + std::string(m.str()) + + postfix + std::string(m.suffix()); +} +} // namespace + +namespace nix { +struct CmdSearch final : SourceExprCommand, MixJSON { + std::vector<std::string> res; + + bool writeCache = true; + bool useCache = true; + + CmdSearch() { + expectArgs("regex", &res); + + mkFlag() + .longName("update-cache") + .shortName('u') + .description("update the package search cache") + .handler([&]() { + writeCache = true; + useCache = false; + }); + + mkFlag() + .longName("no-cache") + .description("do not use or update the package search cache") + .handler([&]() { + writeCache = false; + useCache = false; + }); + } + + std::string name() override { return "search"; } + + std::string description() override { return "query available packages"; } + + Examples examples() override { + return {Example{"To show all available packages:", "nix search"}, + Example{"To show any packages containing 'blender' in its name or " + "description:", + "nix search blender"}, + Example{"To search for Firefox or Chromium:", + "nix search 'firefox|chromium'"}, + Example{"To search for git and frontend or gui:", + "nix search git 'frontend|gui'"}}; + } + + void run(ref<Store> store) override { + settings.readOnlyMode = true; + + // Empty search string should match all packages + // Use "^" here instead of ".*" due to differences in resulting highlighting + // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + if (res.empty()) { + res.emplace_back("^"); + } + + std::vector<std::regex> regexes; + regexes.reserve(res.size()); + + for (auto& re : res) { + regexes.emplace_back(re, std::regex::extended | std::regex::icase); + } + + auto state = getEvalState(); + + auto jsonOut = json ? std::make_unique<JSONObject>(std::cout) : nullptr; + + auto sToplevel = state->symbols.Create("_toplevel"); + auto sRecurse = state->symbols.Create("recurseForDerivations"); + + bool fromCache = false; + + std::map<std::string, std::string> results; + + std::function<void(Value*, std::string, bool, JSONObject*)> doExpr; + + doExpr = [&](Value* v, const std::string& attrPath, bool toplevel, + JSONObject* cache) { + DLOG(INFO) << "at attribute '" << attrPath << "'"; + + try { + uint found = 0; + + state->forceValue(*v); + + if (v->type == tLambda && toplevel) { + Value* v2 = state->allocValue(); + auto dummyArgs = Bindings::New(); + state->autoCallFunction(dummyArgs.get(), *v, *v2); + v = v2; + state->forceValue(*v); + } + + if (state->isDerivation(*v)) { + DrvInfo drv(*state, attrPath, v->attrs); + std::string description; + std::smatch attrPathMatch; + std::smatch descriptionMatch; + std::smatch nameMatch; + std::string name; + + DrvName parsed(drv.queryName()); + + for (auto& regex : regexes) { + std::regex_search(attrPath, attrPathMatch, regex); + + name = parsed.name; + std::regex_search(name, nameMatch, regex); + + description = drv.queryMetaString("description"); + std::replace(description.begin(), description.end(), '\n', ' '); + std::regex_search(description, descriptionMatch, regex); + + if (!attrPathMatch.empty() || !nameMatch.empty() || + !descriptionMatch.empty()) { + found++; + } + } + + if (found == res.size()) { + if (json) { + auto jsonElem = jsonOut->object(attrPath); + + jsonElem.attr("pkgName", parsed.name); + jsonElem.attr("version", parsed.version); + jsonElem.attr("description", description); + + } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); + results[attrPath] = fmt( + "* %s (%s)\n %s\n", + wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), + hilite(description, descriptionMatch, ANSI_NORMAL)); + } + } + + if (cache != nullptr) { + cache->attr("type", "derivation"); + cache->attr("name", drv.queryName()); + cache->attr("system", drv.querySystem()); + if (!description.empty()) { + auto meta(cache->object("meta")); + meta.attr("description", description); + } + } + } + + else if (v->type == tAttrs) { + if (!toplevel) { + auto attrs = v->attrs; + Bindings::iterator j = attrs->find(sRecurse); + if (j == attrs->end() || + !state->forceBool(*j->second.value, *j->second.pos)) { + DLOG(INFO) << "skip attribute '" << attrPath << "'"; + return; + } + } + + bool toplevel2 = false; + if (!fromCache) { + Bindings::iterator j = v->attrs->find(sToplevel); + toplevel2 = j != v->attrs->end() && + state->forceBool(*j->second.value, *j->second.pos); + } + + for (auto& i : *v->attrs) { + auto cache2 = + cache != nullptr + ? std::make_unique<JSONObject>(cache->object(i.second.name)) + : nullptr; + doExpr(i.second.value, + attrPath.empty() + ? std::string(i.second.name) + : attrPath + "." + std::string(i.second.name), + toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); + } + } + + } catch (AssertionError& e) { + } catch (Error& e) { + if (!toplevel) { + e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath)); + throw; + } + } + }; + + Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; + + if (useCache && pathExists(jsonCacheFileName)) { + LOG(WARNING) << "using cached results; pass '-u' to update the cache"; + + Value vRoot; + parseJSON(*state, readFile(jsonCacheFileName), vRoot); + + fromCache = true; + + doExpr(&vRoot, "", true, nullptr); + } + + else { + createDirs(dirOf(jsonCacheFileName)); + + Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); + + std::ofstream jsonCacheFile; + + try { + // iostream considered harmful + jsonCacheFile.exceptions(std::ofstream::failbit); + jsonCacheFile.open(tmpFile); + + auto cache = writeCache + ? std::make_unique<JSONObject>(jsonCacheFile, false) + : nullptr; + + doExpr(getSourceExpr(*state), "", true, cache.get()); + + } catch (std::exception&) { + /* Fun fact: catching std::ios::failure does not work + due to C++11 ABI shenanigans. + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ + if (!jsonCacheFile) { + throw Error("error writing to %s", tmpFile); + } + throw; + } + + if (writeCache && + rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) { + throw SysError("cannot rename '%s' to '%s'", tmpFile, + jsonCacheFileName); + } + } + + if (results.empty()) { + throw Error("no results for the given search term(s)!"); + } + + RunPager pager; + for (const auto& el : results) { + std::cout << el.second << "\n"; + } + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdSearch>()); diff --git a/third_party/nix/src/nix/show-config.cc b/third_party/nix/src/nix/show-config.cc new file mode 100644 index 000000000000..fd92e481e89a --- /dev/null +++ b/third_party/nix/src/nix/show-config.cc @@ -0,0 +1,31 @@ +#include "libmain/common-args.hh" +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "libutil/json.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdShowConfig final : Command, MixJSON { + CmdShowConfig() = default; + + std::string name() override { return "show-config"; } + + std::string description() override { return "show the Nix configuration"; } + + void run() override { + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + JSONObject jsonObj(std::cout); + globalConfig.toJSON(jsonObj); + } else { + std::map<std::string, Config::SettingInfo> settings; + globalConfig.getSettings(settings); + for (auto& s : settings) { + std::cout << s.first + " = " + s.second.value + "\n"; + } + } + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdShowConfig>()); diff --git a/third_party/nix/src/nix/show-derivation.cc b/third_party/nix/src/nix/show-derivation.cc new file mode 100644 index 000000000000..efe554710ff8 --- /dev/null +++ b/third_party/nix/src/nix/show-derivation.cc @@ -0,0 +1,113 @@ +// FIXME: integrate this with nix path-info? + +#include "libmain/common-args.hh" +#include "libstore/derivations.hh" +#include "libstore/store-api.hh" +#include "libutil/archive.hh" +#include "libutil/json.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdShowDerivation final : InstallablesCommand { + bool recursive = false; + + CmdShowDerivation() { + mkFlag() + .longName("recursive") + .shortName('r') + .description("include the dependencies of the specified derivations") + .set(&recursive, true); + } + + std::string name() override { return "show-derivation"; } + + std::string description() override { + return "show the contents of a store derivation"; + } + + Examples examples() override { + return { + Example{"To show the store derivation that results from evaluating the " + "Hello package:", + "nix show-derivation nixpkgs.hello"}, + Example{"To show the full derivation graph (if available) that " + "produced your NixOS system:", + "nix show-derivation -r /run/current-system"}, + }; + } + + void run(ref<Store> store) override { + auto drvPaths = toDerivations(store, installables, true); + + if (recursive) { + PathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = closure; + } + + { + JSONObject jsonRoot(std::cout, true); + + for (auto& drvPath : drvPaths) { + if (!isDerivation(drvPath)) { + continue; + } + + auto drvObj(jsonRoot.object(drvPath)); + + auto drv = readDerivation(drvPath); + + { + auto outputsObj(drvObj.object("outputs")); + for (auto& output : drv.outputs) { + auto outputObj(outputsObj.object(output.first)); + outputObj.attr("path", output.second.path); + if (!output.second.hash.empty()) { + outputObj.attr("hashAlgo", output.second.hashAlgo); + outputObj.attr("hash", output.second.hash); + } + } + } + + { + auto inputsList(drvObj.list("inputSrcs")); + for (auto& input : drv.inputSrcs) { + inputsList.elem(input); + } + } + + { + auto inputDrvsObj(drvObj.object("inputDrvs")); + for (auto& input : drv.inputDrvs) { + auto inputList(inputDrvsObj.list(input.first)); + for (auto& outputId : input.second) { + inputList.elem(outputId); + } + } + } + + drvObj.attr("platform", drv.platform); + drvObj.attr("builder", drv.builder); + + { + auto argsList(drvObj.list("args")); + for (auto& arg : drv.args) { + argsList.elem(arg); + } + } + + { + auto envObj(drvObj.object("env")); + for (auto& var : drv.env) { + envObj.attr(var.first, var.second); + } + } + } + } + + std::cout << "\n"; + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdShowDerivation>()); diff --git a/third_party/nix/src/nix/sigs.cc b/third_party/nix/src/nix/sigs.cc new file mode 100644 index 000000000000..cc42613d0713 --- /dev/null +++ b/third_party/nix/src/nix/sigs.cc @@ -0,0 +1,146 @@ +#include <atomic> + +#include <glog/logging.h> + +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "libutil/thread-pool.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdCopySigs final : StorePathsCommand { + Strings substituterUris; + + CmdCopySigs() { + mkFlag() + .longName("substituter") + .shortName('s') + .labels({"store-uri"}) + .description("use signatures from specified store") + .arity(1) + .handler([&](std::vector<std::string> ss) { + substituterUris.push_back(ss[0]); + }); + } + + std::string name() override { return "copy-sigs"; } + + std::string description() override { + return "copy path signatures from substituters (like binary caches)"; + } + + void run(ref<Store> store, Paths storePaths) override { + if (substituterUris.empty()) { + throw UsageError("you must specify at least one substituter using '-s'"); + } + + // FIXME: factor out commonality with MixVerify. + std::vector<ref<Store>> substituters; + for (auto& s : substituterUris) { + substituters.push_back(openStore(s)); + } + + ThreadPool pool; + + std::string doneLabel = "done"; + std::atomic<size_t> added{0}; + + // logger->setExpected(doneLabel, storePaths.size()); + + auto doPath = [&](const Path& storePath) { + // Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % + // storePath); + + checkInterrupt(); + + auto info = store->queryPathInfo(storePath); + + StringSet newSigs; + + for (auto& store2 : substituters) { + try { + auto info2 = store2->queryPathInfo(storePath); + + /* Don't import signatures that don't match this + binary. */ + if (info->narHash != info2->narHash || + info->narSize != info2->narSize || + info->references != info2->references) { + continue; + } + + for (auto& sig : info2->sigs) { + if (info->sigs.count(sig) == 0u) { + newSigs.insert(sig); + } + } + } catch (InvalidPath&) { + } + } + + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } + + // logger->incProgress(doneLabel); + }; + + for (auto& storePath : storePaths) { + pool.enqueue(std::bind(doPath, storePath)); + } + + pool.process(); + + LOG(INFO) << "imported " << added << " signatures"; + } +}; + +static nix::RegisterCommand r1(make_ref<CmdCopySigs>()); + +struct CmdSignPaths final : StorePathsCommand { + Path secretKeyFile; + + CmdSignPaths() { + mkFlag() + .shortName('k') + .longName("key-file") + .label("file") + .description("file containing the secret signing key") + .dest(&secretKeyFile); + } + + std::string name() override { return "sign-paths"; } + + std::string description() override { return "sign the specified paths"; } + + void run(ref<Store> store, Paths storePaths) override { + if (secretKeyFile.empty()) { + throw UsageError("you must specify a secret key file using '-k'"); + } + + SecretKey secretKey(readFile(secretKeyFile)); + + size_t added{0}; + + for (auto& storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + + auto info2(*info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); + + if (info->sigs.count(*info2.sigs.begin()) == 0u) { + store->addSignatures(storePath, info2.sigs); + added++; + } + } + + LOG(INFO) << "added " << added << " signatures"; + } +}; + +static RegisterCommand r3(make_ref<CmdSignPaths>()); + +} // namespace nix diff --git a/third_party/nix/src/nix/upgrade-nix.cc b/third_party/nix/src/nix/upgrade-nix.cc new file mode 100644 index 000000000000..c7f654d64884 --- /dev/null +++ b/third_party/nix/src/nix/upgrade-nix.cc @@ -0,0 +1,167 @@ +#include <absl/strings/match.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <glog/logging.h> + +#include "libexpr/attr-path.hh" +#include "libexpr/eval.hh" +#include "libexpr/names.hh" +#include "libmain/common-args.hh" +#include "libstore/download.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdUpgradeNix final : MixDryRun, StoreCommand { + Path profileDir; + std::string storePathsUrl = + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/" + "tools/nix-fallback-paths.nix"; + + CmdUpgradeNix() { + mkFlag() + .longName("profile") + .shortName('p') + .labels({"profile-dir"}) + .description("the Nix profile to upgrade") + .dest(&profileDir); + + mkFlag() + .longName("nix-store-paths-url") + .labels({"url"}) + .description( + "URL of the file that contains the store paths of the latest Nix " + "release") + .dest(&storePathsUrl); + } + + std::string name() override { return "upgrade-nix"; } + + std::string description() override { + return "upgrade Nix to the latest stable version"; + } + + Examples examples() override { + return { + Example{"To upgrade Nix to the latest stable version:", + "nix upgrade-nix"}, + Example{ + "To upgrade Nix in a specific profile:", + "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile"}, + }; + } + + void run(ref<Store> store) override { + evalSettings.pureEval = true; + + if (profileDir.empty()) { + profileDir = getProfileDir(store); + } + + LOG(INFO) << "upgrading Nix in profile '" << profileDir << "'"; + + Path storePath; + { + LOG(INFO) << "querying latest Nix version"; + storePath = getLatestNix(store); + } + + auto version = DrvName(storePathToName(storePath)).version; + + if (dryRun) { + LOG(ERROR) << "would upgrade to version " << version; + return; + } + + { + LOG(INFO) << "downloading '" << storePath << "'..."; + store->ensurePath(storePath); + } + + { + LOG(INFO) << "verifying that '" << storePath << "' works..."; + auto program = storePath + "/bin/nix-env"; + auto s = runProgram(program, false, {"--version"}); + if (s.find("Nix") == std::string::npos) { + throw Error("could not verify that '%s' works", program); + } + } + + { + LOG(INFO) << "installing '" << storePath << "' into profile '" + << profileDir << "'..."; + runProgram(settings.nixBinDir + "/nix-env", false, + {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); + } + + LOG(INFO) << ANSI_GREEN << "upgrade to version " << version << " done" + << ANSI_NORMAL; + } + + /* Return the profile in which Nix is installed. */ + static Path getProfileDir(const ref<Store>& store) { + Path where; + + for (auto& dir : absl::StrSplit(getEnv("PATH").value_or(""), + absl::ByChar(':'), absl::SkipEmpty())) { + if (pathExists(absl::StrCat(dir, "/nix-env"))) { + where = dir; + break; + } + } + + if (where.empty()) { + throw Error( + "couldn't figure out how Nix is installed, so I can't upgrade it"); + } + + LOG(INFO) << "found Nix in '" << where << "'"; + + if (absl::StartsWith(where, "/run/current-system")) { + throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); + } + + Path profileDir = dirOf(where); + + // Resolve profile to /nix/var/nix/profiles/<name> link. + while (canonPath(profileDir).find("/profiles/") == std::string::npos && + isLink(profileDir)) { + profileDir = readLink(profileDir); + } + + LOG(INFO) << "found profile '" << profileDir << "'"; + + Path userEnv = canonPath(profileDir, true); + + if (baseNameOf(where) != "bin" || + !absl::EndsWith(userEnv, "user-environment")) { + throw Error("directory '%s' does not appear to be part of a Nix profile", + where); + } + + if (!store->isValidPath(userEnv)) { + throw Error("directory '%s' is not in the Nix store", userEnv); + } + + return profileDir; + } + + /* Return the store path of the latest stable Nix. */ + Path getLatestNix(const ref<Store>& store) { + // FIXME: use nixos.org? + auto req = DownloadRequest(storePathsUrl); + auto res = getDownloader()->download(req); + + auto state = std::make_unique<EvalState>(Strings(), store); + auto v = state->allocValue(); + state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); + std::unique_ptr<Bindings> bindings(Bindings::New()); + auto v2 = + findAlongAttrPath(*state, settings.thisSystem, bindings.get(), *v); + + return state->forceString(*v2); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdUpgradeNix>()); diff --git a/third_party/nix/src/nix/verify.cc b/third_party/nix/src/nix/verify.cc new file mode 100644 index 000000000000..7de46f2a9c01 --- /dev/null +++ b/third_party/nix/src/nix/verify.cc @@ -0,0 +1,171 @@ +#include <atomic> + +#include <glog/logging.h> + +#include "libmain/shared.hh" +#include "libstore/store-api.hh" +#include "libutil/sync.hh" +#include "libutil/thread-pool.hh" +#include "nix/command.hh" + +namespace nix { +struct CmdVerify final : StorePathsCommand { + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded = 0; + + CmdVerify() { + mkFlag(0, "no-contents", "do not verify the contents of each store path", + &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", + &noTrust); + mkFlag() + .longName("substituter") + .shortName('s') + .labels({"store-uri"}) + .description("use signatures from specified store") + .arity(1) + .handler([&](std::vector<std::string> ss) { + substituterUris.push_back(ss[0]); + }); + mkIntFlag('n', "sigs-needed", + "require that each path has at least N valid signatures", + &sigsNeeded); + } + + std::string name() override { return "verify"; } + + std::string description() override { + return "verify the integrity of store paths"; + } + + Examples examples() override { + return { + Example{"To verify the entire Nix store:", "nix verify --all"}, + Example{"To check whether each path in the closure of Firefox has at " + "least 2 signatures:", + "nix verify -r -n2 --no-contents $(type -p firefox)"}, + }; + } + + void run(ref<Store> store, Paths storePaths) override { + std::vector<ref<Store>> substituters; + for (auto& s : substituterUris) { + substituters.push_back(openStore(s)); + } + + auto publicKeys = getDefaultPublicKeys(); + + std::atomic<size_t> done{0}; + std::atomic<size_t> untrusted{0}; + std::atomic<size_t> corrupted{0}; + std::atomic<size_t> failed{0}; + std::atomic<size_t> active{0}; + + ThreadPool pool; + + auto doPath = [&](const Path& storePath) { + try { + checkInterrupt(); + + LOG(INFO) << "checking '" << storePath << "'"; + + MaintainCount<std::atomic<size_t>> mcActive(active); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + HashSink sink(info->narHash.type); + store->narFromPath(info->path, sink); + + auto hash = sink.finish(); + + if (hash.first != info->narHash) { + corrupted++; + LOG(WARNING) << "path '" << info->path + << "' was modified! expected hash '" + << info->narHash.to_string() << "', got '" + << hash.first.to_string() << "'"; + } + } + + if (!noTrust) { + bool good = false; + + if (info->ultimate && (sigsNeeded == 0u)) { + good = true; + + } else { + StringSet sigsSeen; + size_t actualSigsNeeded = + std::max(sigsNeeded, static_cast<size_t>(1)); + size_t validSigs = 0; + + auto doSigs = [&](const StringSet& sigs) { + for (const auto& sig : sigs) { + if (sigsSeen.count(sig) != 0u) { + continue; + } + sigsSeen.insert(sig); + if (validSigs < ValidPathInfo::maxSigs && + info->checkSignature(publicKeys, sig)) { + validSigs++; + } + } + }; + + if (info->isContentAddressed(*store)) { + validSigs = ValidPathInfo::maxSigs; + } + + doSigs(info->sigs); + + for (auto& store2 : substituters) { + if (validSigs >= actualSigsNeeded) { + break; + } + try { + auto info2 = store2->queryPathInfo(info->path); + if (info2->isContentAddressed(*store)) { + validSigs = ValidPathInfo::maxSigs; + } + doSigs(info2->sigs); + } catch (InvalidPath&) { + } catch (Error& e) { + LOG(ERROR) << e.what(); + } + } + + if (validSigs >= actualSigsNeeded) { + good = true; + } + } + + if (!good) { + untrusted++; + LOG(WARNING) << "path '" << info->path << "' is untrusted"; + } + } + + done++; + + } catch (Error& e) { + LOG(ERROR) << e.what(); + failed++; + } + }; + + for (auto& storePath : storePaths) { + pool.enqueue(std::bind(doPath, storePath)); + } + + pool.process(); + + throw Exit((corrupted != 0u ? 1 : 0) | (untrusted != 0u ? 2 : 0) | + (failed != 0u ? 4 : 0)); + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdVerify>()); diff --git a/third_party/nix/src/nix/why-depends.cc b/third_party/nix/src/nix/why-depends.cc new file mode 100644 index 000000000000..954d619ef3ce --- /dev/null +++ b/third_party/nix/src/nix/why-depends.cc @@ -0,0 +1,269 @@ +#include <queue> + +#include <glog/logging.h> + +#include "libmain/shared.hh" +#include "libstore/fs-accessor.hh" +#include "libstore/store-api.hh" +#include "nix/command.hh" + +namespace { +static std::string hilite(const std::string& s, size_t pos, size_t len, + const std::string& colour = ANSI_RED) { + return std::string(s, 0, pos) + colour + std::string(s, pos, len) + + ANSI_NORMAL + std::string(s, pos + len); +} + +static std::string filterPrintable(const std::string& s) { + std::string res; + for (char c : s) { + res += isprint(c) != 0 ? c : '.'; + } + return res; +} +} // namespace + +namespace nix { +struct CmdWhyDepends final : SourceExprCommand { + std::string _package, _dependency; + bool all = false; + + CmdWhyDepends() { + expectArg("package", &_package); + expectArg("dependency", &_dependency); + + mkFlag() + .longName("all") + .shortName('a') + .description( + "show all edges in the dependency graph leading from 'package' to " + "'dependency', rather than just a shortest path") + .set(&all, true); + } + + std::string name() override { return "why-depends"; } + + std::string description() override { + return "show why a package has another package in its closure"; + } + + Examples examples() override { + return { + Example{"To show one path through the dependency graph leading from " + "Hello to Glibc:", + "nix why-depends nixpkgs.hello nixpkgs.glibc"}, + Example{ + "To show all files and paths in the dependency graph leading from " + "Thunderbird to libX11:", + "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11"}, + Example{"To show why Glibc depends on itself:", + "nix why-depends nixpkgs.glibc nixpkgs.glibc"}, + }; + } + + void run(ref<Store> store) override { + auto package = parseInstallable(*this, store, _package, false); + auto packagePath = toStorePath(store, Build, package); + auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto dependencyPathHash = storePathToHash(dependencyPath); + + PathSet closure; + store->computeFSClosure({packagePath}, closure, false, false); + + if (closure.count(dependencyPath) == 0u) { + LOG(WARNING) << "'" << package->what() << "' does not depend on '" + << dependency->what() << "'"; + return; + } + + auto accessor = store->getFSAccessor(); + + auto const inf = std::numeric_limits<size_t>::max(); + + struct Node { + Path path; + PathSet refs; + PathSet rrefs; + size_t dist = inf; + Node* prev = nullptr; + bool queued = false; + bool visited = false; + }; + + std::map<Path, Node> graph; + + for (auto& path : closure) { + graph.emplace(path, Node{path, store->queryPathInfo(path)->references}); + } + + // Transpose the graph. + for (auto& node : graph) { + for (auto& ref : node.second.refs) { + graph[ref].rrefs.insert(node.first); + } + } + + /* Run Dijkstra's shortest path algorithm to get the distance + of every path in the closure to 'dependency'. */ + graph[dependencyPath].dist = 0; + + std::priority_queue<Node*> queue; + + queue.push(&graph.at(dependencyPath)); + + while (!queue.empty()) { + auto& node = *queue.top(); + queue.pop(); + + for (auto& rref : node.rrefs) { + auto& node2 = graph.at(rref); + auto dist = node.dist + 1; + if (dist < node2.dist) { + node2.dist = dist; + node2.prev = &node; + if (!node2.queued) { + node2.queued = true; + queue.push(&node2); + } + } + } + } + + /* Print the subgraph of nodes that have 'dependency' in their + closure (i.e., that have a non-infinite distance to + 'dependency'). Print every edge on a path between `package` + and `dependency`. */ + std::function<void(Node&, const std::string&, const std::string&)> + printNode; + + const std::string treeConn = "╠═══"; + const std::string treeLast = "╚═══"; + const std::string treeLine = "║ "; + const std::string treeNull = " "; + + struct BailOut {}; + + printNode = [&](Node& node, const std::string& firstPad, + const std::string& tailPad) { + assert(node.dist != inf); + std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", firstPad, + node.visited ? "\e[38;5;244m" : "", + !firstPad.empty() ? "=> " : "", node.path); + + if (node.path == dependencyPath && !all && + packagePath != dependencyPath) { + throw BailOut(); + } + + if (node.visited) { + return; + } + node.visited = true; + + /* Sort the references by distance to `dependency` to + ensure that the shortest path is printed first. */ + std::multimap<size_t, Node*> refs; + std::set<std::string> hashes; + + for (auto& ref : node.refs) { + if (ref == node.path && packagePath != dependencyPath) { + continue; + } + auto& node2 = graph.at(ref); + if (node2.dist == inf) { + continue; + } + refs.emplace(node2.dist, &node2); + hashes.insert(storePathToHash(node2.path)); + } + + /* For each reference, find the files and symlinks that + contain the reference. */ + std::map<std::string, Strings> hits; + + std::function<void(const Path&)> visitPath; + + visitPath = [&](const Path& p) { + auto st = accessor->stat(p); + + auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1); + + auto getColour = [&](const std::string& hash) { + return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; + }; + + if (st.type == FSAccessor::Type::tDirectory) { + auto names = accessor->readDirectory(p); + for (auto& name : names) { + visitPath(p + "/" + name); + } + } + + else if (st.type == FSAccessor::Type::tRegular) { + auto contents = accessor->readFile(p); + + for (auto& hash : hashes) { + auto pos = contents.find(hash); + if (pos != std::string::npos) { + size_t margin = 32; + auto pos2 = pos >= margin ? pos - margin : 0; + hits[hash].emplace_back(fmt( + "%s: …%s…\n", p2, + hilite( + filterPrintable(std::string( + contents, pos2, pos - pos2 + hash.size() + margin)), + pos - pos2, storePathHashLen, getColour(hash)))); + } + } + } + + else if (st.type == FSAccessor::Type::tSymlink) { + auto target = accessor->readLink(p); + + for (auto& hash : hashes) { + auto pos = target.find(hash); + if (pos != std::string::npos) { + hits[hash].emplace_back( + fmt("%s -> %s\n", p2, + hilite(target, pos, storePathHashLen, getColour(hash)))); + } + } + } + }; + + // FIXME: should use scanForReferences(). + + visitPath(node.path); + + RunPager pager; + for (auto& ref : refs) { + auto hash = storePathToHash(ref.second->path); + + bool last = all ? ref == *refs.rbegin() : true; + + for (auto& hit : hits[hash]) { + bool first = hit == *hits[hash].begin(); + std::cout << tailPad + << (first ? (last ? treeLast : treeConn) + : (last ? treeNull : treeLine)) + << hit; + if (!all) { + break; + } + } + + printNode(*ref.second, tailPad + (last ? treeNull : treeLine), + tailPad + (last ? treeNull : treeLine)); + } + }; + + try { + printNode(graph.at(packagePath), "", ""); + } catch (BailOut&) { + } + } +}; +} // namespace nix + +static nix::RegisterCommand r1(nix::make_ref<nix::CmdWhyDepends>()); diff --git a/third_party/nix/src/nlohmann/json.hpp b/third_party/nix/src/nlohmann/json.hpp new file mode 100644 index 000000000000..c9af0bed36d6 --- /dev/null +++ b/third_party/nix/src/nlohmann/json.hpp @@ -0,0 +1,20406 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.5.0 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License <http://opensource.org/licenses/MIT>. +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann <http://nlohmann.me>. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 5 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include <algorithm> // all_of, find, for_each +#include <cassert> // assert +#include <ciso646> // and, not, or +#include <cstddef> // nullptr_t, ptrdiff_t, size_t +#include <functional> // hash, less +#include <initializer_list> // initializer_list +#include <iosfwd> // istream, ostream +#include <iterator> // random_access_iterator_tag +#include <numeric> // accumulate +#include <string> // string, stoi, to_string +#include <utility> // declval, forward, move, pair, swap + +// #include <nlohmann/json_fwd.hpp> +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include <cstdint> // int64_t, uint64_t +#include <map> // map +#include <memory> // allocator +#include <string> // string +#include <vector> // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template<typename T = void, typename SFINAE = void> +struct adl_serializer; + +template<template<typename U, typename V, typename... Args> class ObjectType = + std::map, + template<typename U, typename... Args> class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template<typename U> class AllocatorType = std::allocator, + template<typename T, typename SFINAE = void> class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template<typename BasicJsonType> +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif + +// #include <nlohmann/detail/macro_scope.hpp> + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// manual branch prediction +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) + #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else + #define JSON_LIKELY(x) x + #define JSON_UNLIKELY(x) x +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template<typename BasicJsonType> \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template<typename BasicJsonType> \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum<ENUM_TYPE>::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair<ENUM_TYPE, BasicJsonType> m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair<ENUM_TYPE, BasicJsonType>& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template<template<typename, typename, typename...> class ObjectType, \ + template<typename, typename...> class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template<typename> class AllocatorType, \ + template<typename, typename = void> class JSONSerializer> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json<ObjectType, ArrayType, StringType, BooleanType, \ + NumberIntegerType, NumberUnsignedType, NumberFloatType, \ + AllocatorType, JSONSerializer> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + + +#include <ciso646> // not +#include <cstddef> // size_t +#include <type_traits> // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template<bool B, typename T = void> +using enable_if_t = typename std::enable_if<B, T>::type; + +template<typename T> +using uncvref_t = typename std::remove_cv<typename std::remove_reference<T>::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template<std::size_t... Ints> +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template<class Sequence1, class Sequence2> +struct merge_and_renumber; + +template<std::size_t... I1, std::size_t... I2> +struct merge_and_renumber<index_sequence<I1...>, index_sequence<I2...>> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template<std::size_t N> +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template<typename... Ts> +using index_sequence_for = make_index_sequence<sizeof...(Ts)>; + +// dispatch utility (taken from ranges-v3) +template<unsigned N> struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template<typename T> +struct static_const +{ + static constexpr T value{}; +}; + +template<typename T> +constexpr T static_const<T>::value; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +#include <ciso646> // not +#include <limits> // numeric_limits +#include <type_traits> // false_type, is_constructible, is_integral, is_same, true_type +#include <utility> // declval + +// #include <nlohmann/json_fwd.hpp> + +// #include <nlohmann/detail/iterators/iterator_traits.hpp> + + +#include <iterator> // random_access_iterator_tag + +// #include <nlohmann/detail/meta/void_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename ...Ts> struct make_void +{ + using type = void; +}; +template <typename ...Ts> using void_t = typename make_void<Ts...>::type; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/meta/cpp_future.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename It, typename = void> +struct iterator_types {}; + +template <typename It> +struct iterator_types < + It, + void_t<typename It::difference_type, typename It::value_type, typename It::pointer, + typename It::reference, typename It::iterator_category >> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template <typename T, typename = void> +struct iterator_traits +{ +}; + +template <typename T> +struct iterator_traits < T, enable_if_t < !std::is_pointer<T>::value >> + : iterator_types<T> +{ +}; + +template <typename T> +struct iterator_traits<T*, enable_if_t<std::is_object<T>::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} +} + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/meta/detected.hpp> + + +#include <type_traits> + +// #include <nlohmann/detail/meta/void_t.hpp> + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template <class Default, + class AlwaysVoid, + template <class...> class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template <class Default, template <class...> class Op, class... Args> +struct detector<Default, void_t<Op<Args...>>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op<Args...>; +}; + +template <template <class...> class Op, class... Args> +using is_detected = typename detector<nonesuch, void, Op, Args...>::value_t; + +template <template <class...> class Op, class... Args> +using detected_t = typename detector<nonesuch, void, Op, Args...>::type; + +template <class Default, template <class...> class Op, class... Args> +using detected_or = detector<Default, void, Op, Args...>; + +template <class Default, template <class...> class Op, class... Args> +using detected_or_t = typename detected_or<Default, Op, Args...>::type; + +template <class Expected, template <class...> class Op, class... Args> +using is_detected_exact = std::is_same<Expected, detected_t<Op, Args...>>; + +template <class To, template <class...> class Op, class... Args> +using is_detected_convertible = + std::is_convertible<detected_t<Op, Args...>, To>; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + + +namespace nlohmann +{ +/*! +@brief detail namespace with internal helper functions + +This namespace collects functions that should not be exposed, +implementations of some @ref basic_json methods, and meta-programming helpers. + +@since version 2.1.0 +*/ +namespace detail +{ +///////////// +// helpers // +///////////// + +// Note to maintainers: +// +// Every trait in this file expects a non CV-qualified type. +// The only exceptions are in the 'aliases for detected' section +// (i.e. those of the form: decltype(T::member_function(std::declval<T>()))) +// +// In this case, T has to be properly CV-qualified to constraint the function arguments +// (e.g. to_json(BasicJsonType&, const T&)) + +template<typename> struct is_basic_json : std::false_type {}; + +NLOHMANN_BASIC_JSON_TPL_DECLARATION +struct is_basic_json<NLOHMANN_BASIC_JSON_TPL> : std::true_type {}; + +////////////////////////// +// aliases for detected // +////////////////////////// + +template <typename T> +using mapped_type_t = typename T::mapped_type; + +template <typename T> +using key_type_t = typename T::key_type; + +template <typename T> +using value_type_t = typename T::value_type; + +template <typename T> +using difference_type_t = typename T::difference_type; + +template <typename T> +using pointer_t = typename T::pointer; + +template <typename T> +using reference_t = typename T::reference; + +template <typename T> +using iterator_category_t = typename T::iterator_category; + +template <typename T> +using iterator_t = typename T::iterator; + +template <typename T, typename... Args> +using to_json_function = decltype(T::to_json(std::declval<Args>()...)); + +template <typename T, typename... Args> +using from_json_function = decltype(T::from_json(std::declval<Args>()...)); + +template <typename T, typename U> +using get_template_function = decltype(std::declval<T>().template get<U>()); + +// trait checking if JSONSerializer<T>::from_json(json const&, udt&) exists +template <typename BasicJsonType, typename T, typename = void> +struct has_from_json : std::false_type {}; + +template <typename BasicJsonType, typename T> +struct has_from_json<BasicJsonType, T, + enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<void, from_json_function, serializer, + const BasicJsonType&, T&>::value; +}; + +// This trait checks if JSONSerializer<T>::from_json(json const&) exists +// this overload is used for non-default-constructible user-defined-types +template <typename BasicJsonType, typename T, typename = void> +struct has_non_default_from_json : std::false_type {}; + +template<typename BasicJsonType, typename T> +struct has_non_default_from_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<T, from_json_function, serializer, + const BasicJsonType&>::value; +}; + +// This trait checks if BasicJsonType::json_serializer<T>::to_json exists +// Do not evaluate the trait when T is a basic_json type, to avoid template instantiation infinite recursion. +template <typename BasicJsonType, typename T, typename = void> +struct has_to_json : std::false_type {}; + +template <typename BasicJsonType, typename T> +struct has_to_json<BasicJsonType, T, enable_if_t<not is_basic_json<T>::value>> +{ + using serializer = typename BasicJsonType::template json_serializer<T, void>; + + static constexpr bool value = + is_detected_exact<void, to_json_function, serializer, BasicJsonType&, + T>::value; +}; + + +/////////////////// +// is_ functions // +/////////////////// + +template <typename T, typename = void> +struct is_iterator_traits : std::false_type {}; + +template <typename T> +struct is_iterator_traits<iterator_traits<T>> +{ + private: + using traits = iterator_traits<T>; + + public: + static constexpr auto value = + is_detected<value_type_t, traits>::value && + is_detected<difference_type_t, traits>::value && + is_detected<pointer_t, traits>::value && + is_detected<iterator_category_t, traits>::value && + is_detected<reference_t, traits>::value; +}; + +// source: https://stackoverflow.com/a/37193089/4116453 + +template <typename T, typename = void> +struct is_complete_type : std::false_type {}; + +template <typename T> +struct is_complete_type<T, decltype(void(sizeof(T)))> : std::true_type {}; + +template <typename BasicJsonType, typename CompatibleObjectType, + typename = void> +struct is_compatible_object_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleObjectType> +struct is_compatible_object_type_impl < + BasicJsonType, CompatibleObjectType, + enable_if_t<is_detected<mapped_type_t, CompatibleObjectType>::value and + is_detected<key_type_t, CompatibleObjectType>::value >> +{ + + using object_t = typename BasicJsonType::object_t; + + // macOS's is_constructible does not play well with nonesuch... + static constexpr bool value = + std::is_constructible<typename object_t::key_type, + typename CompatibleObjectType::key_type>::value and + std::is_constructible<typename object_t::mapped_type, + typename CompatibleObjectType::mapped_type>::value; +}; + +template <typename BasicJsonType, typename CompatibleObjectType> +struct is_compatible_object_type + : is_compatible_object_type_impl<BasicJsonType, CompatibleObjectType> {}; + +template <typename BasicJsonType, typename ConstructibleObjectType, + typename = void> +struct is_constructible_object_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleObjectType> +struct is_constructible_object_type_impl < + BasicJsonType, ConstructibleObjectType, + enable_if_t<is_detected<mapped_type_t, ConstructibleObjectType>::value and + is_detected<key_type_t, ConstructibleObjectType>::value >> +{ + using object_t = typename BasicJsonType::object_t; + + static constexpr bool value = + (std::is_constructible<typename ConstructibleObjectType::key_type, typename object_t::key_type>::value and + std::is_same<typename object_t::mapped_type, typename ConstructibleObjectType::mapped_type>::value) or + (has_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type>::value or + has_non_default_from_json<BasicJsonType, typename ConstructibleObjectType::mapped_type >::value); +}; + +template <typename BasicJsonType, typename ConstructibleObjectType> +struct is_constructible_object_type + : is_constructible_object_type_impl<BasicJsonType, + ConstructibleObjectType> {}; + +template <typename BasicJsonType, typename CompatibleStringType, + typename = void> +struct is_compatible_string_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleStringType> +struct is_compatible_string_type_impl < + BasicJsonType, CompatibleStringType, + enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type, + value_type_t, CompatibleStringType>::value >> +{ + static constexpr auto value = + std::is_constructible<typename BasicJsonType::string_t, CompatibleStringType>::value; +}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_compatible_string_type + : is_compatible_string_type_impl<BasicJsonType, ConstructibleStringType> {}; + +template <typename BasicJsonType, typename ConstructibleStringType, + typename = void> +struct is_constructible_string_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_constructible_string_type_impl < + BasicJsonType, ConstructibleStringType, + enable_if_t<is_detected_exact<typename BasicJsonType::string_t::value_type, + value_type_t, ConstructibleStringType>::value >> +{ + static constexpr auto value = + std::is_constructible<ConstructibleStringType, + typename BasicJsonType::string_t>::value; +}; + +template <typename BasicJsonType, typename ConstructibleStringType> +struct is_constructible_string_type + : is_constructible_string_type_impl<BasicJsonType, ConstructibleStringType> {}; + +template <typename BasicJsonType, typename CompatibleArrayType, typename = void> +struct is_compatible_array_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename CompatibleArrayType> +struct is_compatible_array_type_impl < + BasicJsonType, CompatibleArrayType, + enable_if_t<is_detected<value_type_t, CompatibleArrayType>::value and + is_detected<iterator_t, CompatibleArrayType>::value and +// This is needed because json_reverse_iterator has a ::iterator type... +// Therefore it is detected as a CompatibleArrayType. +// The real fix would be to have an Iterable concept. + not is_iterator_traits< + iterator_traits<CompatibleArrayType>>::value >> +{ + static constexpr bool value = + std::is_constructible<BasicJsonType, + typename CompatibleArrayType::value_type>::value; +}; + +template <typename BasicJsonType, typename CompatibleArrayType> +struct is_compatible_array_type + : is_compatible_array_type_impl<BasicJsonType, CompatibleArrayType> {}; + +template <typename BasicJsonType, typename ConstructibleArrayType, typename = void> +struct is_constructible_array_type_impl : std::false_type {}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t<std::is_same<ConstructibleArrayType, + typename BasicJsonType::value_type>::value >> + : std::true_type {}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type_impl < + BasicJsonType, ConstructibleArrayType, + enable_if_t<not std::is_same<ConstructibleArrayType, + typename BasicJsonType::value_type>::value and + is_detected<value_type_t, ConstructibleArrayType>::value and + is_detected<iterator_t, ConstructibleArrayType>::value and + is_complete_type< + detected_t<value_type_t, ConstructibleArrayType>>::value >> +{ + static constexpr bool value = + // This is needed because json_reverse_iterator has a ::iterator type, + // furthermore, std::back_insert_iterator (and other iterators) have a base class `iterator`... + // Therefore it is detected as a ConstructibleArrayType. + // The real fix would be to have an Iterable concept. + not is_iterator_traits < + iterator_traits<ConstructibleArrayType >>::value and + + (std::is_same<typename ConstructibleArrayType::value_type, typename BasicJsonType::array_t::value_type>::value or + has_from_json<BasicJsonType, + typename ConstructibleArrayType::value_type>::value or + has_non_default_from_json < + BasicJsonType, typename ConstructibleArrayType::value_type >::value); +}; + +template <typename BasicJsonType, typename ConstructibleArrayType> +struct is_constructible_array_type + : is_constructible_array_type_impl<BasicJsonType, ConstructibleArrayType> {}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType, + typename = void> +struct is_compatible_integer_type_impl : std::false_type {}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type_impl < + RealIntegerType, CompatibleNumberIntegerType, + enable_if_t<std::is_integral<RealIntegerType>::value and + std::is_integral<CompatibleNumberIntegerType>::value and + not std::is_same<bool, CompatibleNumberIntegerType>::value >> +{ + // is there an assert somewhere on overflows? + using RealLimits = std::numeric_limits<RealIntegerType>; + using CompatibleLimits = std::numeric_limits<CompatibleNumberIntegerType>; + + static constexpr auto value = + std::is_constructible<RealIntegerType, + CompatibleNumberIntegerType>::value and + CompatibleLimits::is_integer and + RealLimits::is_signed == CompatibleLimits::is_signed; +}; + +template <typename RealIntegerType, typename CompatibleNumberIntegerType> +struct is_compatible_integer_type + : is_compatible_integer_type_impl<RealIntegerType, + CompatibleNumberIntegerType> {}; + +template <typename BasicJsonType, typename CompatibleType, typename = void> +struct is_compatible_type_impl: std::false_type {}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type_impl < + BasicJsonType, CompatibleType, + enable_if_t<is_complete_type<CompatibleType>::value >> +{ + static constexpr bool value = + has_to_json<BasicJsonType, CompatibleType>::value; +}; + +template <typename BasicJsonType, typename CompatibleType> +struct is_compatible_type + : is_compatible_type_impl<BasicJsonType, CompatibleType> {}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/exceptions.hpp> + + +#include <exception> // exception +#include <stdexcept> // runtime_error +#include <string> // to_string + +// #include <nlohmann/detail/input/position_t.hpp> + + +#include <cstddef> // size_t + +namespace nlohmann +{ +namespace detail +{ +/// struct to capture the start position of the current token +struct position_t +{ + /// the total number of characters read + std::size_t chars_read_total = 0; + /// the number of characters read in the current line + std::size_t chars_read_current_line = 0; + /// the number of lines read + std::size_t lines_read = 0; + + /// conversion to size_t to preserve SAX interface + constexpr operator size_t() const + { + return chars_read_total; + } +}; + +} +} + + +namespace nlohmann +{ +namespace detail +{ +//////////////// +// exceptions // +//////////////// + +/*! +@brief general exception of the @ref basic_json class + +This class is an extension of `std::exception` objects with a member @a id for +exception ids. It is used as the base class for all exceptions thrown by the +@ref basic_json class. This class can hence be used as "wildcard" to catch +exceptions. + +Subclasses: +- @ref parse_error for exceptions indicating a parse error +- @ref invalid_iterator for exceptions indicating errors with iterators +- @ref type_error for exceptions indicating executing a member function with + a wrong type +- @ref out_of_range for exceptions indicating access out of the defined range +- @ref other_error for exceptions indicating other library errors + +@internal +@note To have nothrow-copy-constructible exceptions, we internally use + `std::runtime_error` which can cope with arbitrary-length error messages. + Intermediate strings are built with static functions and then passed to + the actual constructor. +@endinternal + +@liveexample{The following code shows how arbitrary library exceptions can be +caught.,exception} + +@since version 3.0.0 +*/ +class exception : public std::exception +{ + public: + /// returns the explanatory string + const char* what() const noexcept override + { + return m.what(); + } + + /// the id of the exception + const int id; + + protected: + exception(int id_, const char* what_arg) : id(id_), m(what_arg) {} + + static std::string name(const std::string& ename, int id_) + { + return "[json.exception." + ename + "." + std::to_string(id_) + "] "; + } + + private: + /// an exception object as storage for error messages + std::runtime_error m; +}; + +/*! +@brief exception indicating a parse error + +This exception is thrown by the library when a parse error occurs. Parse errors +can occur during the deserialization of JSON text, CBOR, MessagePack, as well +as when using JSON Patch. + +Member @a byte holds the byte index of the last read character in the input +file. + +Exceptions have ids 1xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.parse_error.101 | parse error at 2: unexpected end of input; expected string literal | This error indicates a syntax error while deserializing a JSON text. The error message describes that an unexpected token (character) was encountered, and the member @a byte indicates the error position. +json.exception.parse_error.102 | parse error at 14: missing or wrong low surrogate | JSON uses the `\uxxxx` format to describe Unicode characters. Code points above above 0xFFFF are split into two `\uxxxx` entries ("surrogate pairs"). This error indicates that the surrogate pair is incomplete or contains an invalid code point. +json.exception.parse_error.103 | parse error: code points above 0x10FFFF are invalid | Unicode supports code points up to 0x10FFFF. Code points above 0x10FFFF are invalid. +json.exception.parse_error.104 | parse error: JSON patch must be an array of objects | [RFC 6902](https://tools.ietf.org/html/rfc6902) requires a JSON Patch document to be a JSON document that represents an array of objects. +json.exception.parse_error.105 | parse error: operation must have string member 'op' | An operation of a JSON Patch document must contain exactly one "op" member, whose value indicates the operation to perform. Its value must be one of "add", "remove", "replace", "move", "copy", or "test"; other values are errors. +json.exception.parse_error.106 | parse error: array index '01' must not begin with '0' | An array index in a JSON Pointer ([RFC 6901](https://tools.ietf.org/html/rfc6901)) may be `0` or any number without a leading `0`. +json.exception.parse_error.107 | parse error: JSON pointer must be empty or begin with '/' - was: 'foo' | A JSON Pointer must be a Unicode string containing a sequence of zero or more reference tokens, each prefixed by a `/` character. +json.exception.parse_error.108 | parse error: escape character '~' must be followed with '0' or '1' | In a JSON Pointer, only `~0` and `~1` are valid escape sequences. +json.exception.parse_error.109 | parse error: array index 'one' is not a number | A JSON Pointer array index must be a number. +json.exception.parse_error.110 | parse error at 1: cannot read 2 bytes from vector | When parsing CBOR or MessagePack, the byte vector ends before the complete value has been read. +json.exception.parse_error.112 | parse error at 1: error reading CBOR; last byte: 0xF8 | Not all types of CBOR or MessagePack are supported. This exception occurs if an unsupported byte was read. +json.exception.parse_error.113 | parse error at 2: expected a CBOR string; last byte: 0x98 | While parsing a map key, a value that is not a string has been read. +json.exception.parse_error.114 | parse error: Unsupported BSON record type 0x0F | The parsing of the corresponding BSON record type is not implemented (yet). + +@note For an input with n bytes, 1 is the index of the first character and n+1 + is the index of the terminating null byte or the end of file. This also + holds true when reading a byte vector (CBOR or MessagePack). + +@liveexample{The following code shows how a `parse_error` exception can be +caught.,parse_error} + +@sa @ref exception for the base class of the library exceptions +@sa @ref invalid_iterator for exceptions indicating errors with iterators +@sa @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa @ref out_of_range for exceptions indicating access out of the defined range +@sa @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class parse_error : public exception +{ + public: + /*! + @brief create a parse error exception + @param[in] id_ the id of the exception + @param[in] position the position where the error occurred (or with + chars_read_total=0 if the position cannot be + determined) + @param[in] what_arg the explanatory string + @return parse_error object + */ + static parse_error create(int id_, const position_t& pos, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + position_string(pos) + ": " + what_arg; + return parse_error(id_, pos.chars_read_total, w.c_str()); + } + + static parse_error create(int id_, std::size_t byte_, const std::string& what_arg) + { + std::string w = exception::name("parse_error", id_) + "parse error" + + (byte_ != 0 ? (" at byte " + std::to_string(byte_)) : "") + + ": " + what_arg; + return parse_error(id_, byte_, w.c_str()); + } + + /*! + @brief byte index of the parse error + + The byte index of the last read character in the input file. + + @note For an input with n bytes, 1 is the index of the first character and + n+1 is the index of the terminating null byte or the end of file. + This also holds true when reading a byte vector (CBOR or MessagePack). + */ + const std::size_t byte; + + private: + parse_error(int id_, std::size_t byte_, const char* what_arg) + : exception(id_, what_arg), byte(byte_) {} + + static std::string position_string(const position_t& pos) + { + return " at line " + std::to_string(pos.lines_read + 1) + + ", column " + std::to_string(pos.chars_read_current_line); + } +}; + +/*! +@brief exception indicating errors with iterators + +This exception is thrown if iterators passed to a library function do not match +the expected semantics. + +Exceptions have ids 2xx. + +name / id | example message | description +----------------------------------- | --------------- | ------------------------- +json.exception.invalid_iterator.201 | iterators are not compatible | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.202 | iterator does not fit current value | In an erase or insert function, the passed iterator @a pos does not belong to the JSON value for which the function was called. It hence does not define a valid position for the deletion/insertion. +json.exception.invalid_iterator.203 | iterators do not fit current value | Either iterator passed to function @ref erase(IteratorType first, IteratorType last) does not belong to the JSON value from which values shall be erased. It hence does not define a valid range to delete values from. +json.exception.invalid_iterator.204 | iterators out of range | When an iterator range for a primitive type (number, boolean, or string) is passed to a constructor or an erase function, this range has to be exactly (@ref begin(), @ref end()), because this is the only way the single stored value is expressed. All other ranges are invalid. +json.exception.invalid_iterator.205 | iterator out of range | When an iterator for a primitive type (number, boolean, or string) is passed to an erase function, the iterator has to be the @ref begin() iterator, because it is the only way to address the stored value. All other iterators are invalid. +json.exception.invalid_iterator.206 | cannot construct with iterators from null | The iterators passed to constructor @ref basic_json(InputIT first, InputIT last) belong to a JSON null value and hence to not define a valid range. +json.exception.invalid_iterator.207 | cannot use key() for non-object iterators | The key() member function can only be used on iterators belonging to a JSON object, because other types do not have a concept of a key. +json.exception.invalid_iterator.208 | cannot use operator[] for object iterators | The operator[] to specify a concrete offset cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.209 | cannot use offsets with object iterators | The offset operators (+, -, +=, -=) cannot be used on iterators belonging to a JSON object, because JSON objects are unordered. +json.exception.invalid_iterator.210 | iterators do not fit | The iterator range passed to the insert function are not compatible, meaning they do not belong to the same container. Therefore, the range (@a first, @a last) is invalid. +json.exception.invalid_iterator.211 | passed iterators may not belong to container | The iterator range passed to the insert function must not be a subrange of the container to insert to. +json.exception.invalid_iterator.212 | cannot compare iterators of different containers | When two iterators are compared, they must belong to the same container. +json.exception.invalid_iterator.213 | cannot compare order of object iterators | The order of object iterators cannot be compared, because JSON objects are unordered. +json.exception.invalid_iterator.214 | cannot get value | Cannot get value for iterator: Either the iterator belongs to a null value or it is an iterator to a primitive type (number, boolean, or string), but the iterator is different to @ref begin(). + +@liveexample{The following code shows how an `invalid_iterator` exception can be +caught.,invalid_iterator} + +@sa @ref exception for the base class of the library exceptions +@sa @ref parse_error for exceptions indicating a parse error +@sa @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa @ref out_of_range for exceptions indicating access out of the defined range +@sa @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class invalid_iterator : public exception +{ + public: + static invalid_iterator create(int id_, const std::string& what_arg) + { + std::string w = exception::name("invalid_iterator", id_) + what_arg; + return invalid_iterator(id_, w.c_str()); + } + + private: + invalid_iterator(int id_, const char* what_arg) + : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating executing a member function with a wrong type + +This exception is thrown in case of a type error; that is, a library function is +executed on a JSON value whose type does not match the expected semantics. + +Exceptions have ids 3xx. + +name / id | example message | description +----------------------------- | --------------- | ------------------------- +json.exception.type_error.301 | cannot create object from initializer list | To create an object from an initializer list, the initializer list must consist only of a list of pairs whose first element is a string. When this constraint is violated, an array is created instead. +json.exception.type_error.302 | type must be object, but is array | During implicit or explicit value conversion, the JSON type must be compatible to the target type. For instance, a JSON string can only be converted into string types, but not into numbers or boolean types. +json.exception.type_error.303 | incompatible ReferenceType for get_ref, actual type is object | To retrieve a reference to a value stored in a @ref basic_json object with @ref get_ref, the type of the reference must match the value type. For instance, for a JSON array, the @a ReferenceType must be @ref array_t&. +json.exception.type_error.304 | cannot use at() with string | The @ref at() member functions can only be executed for certain JSON types. +json.exception.type_error.305 | cannot use operator[] with string | The @ref operator[] member functions can only be executed for certain JSON types. +json.exception.type_error.306 | cannot use value() with string | The @ref value() member functions can only be executed for certain JSON types. +json.exception.type_error.307 | cannot use erase() with string | The @ref erase() member functions can only be executed for certain JSON types. +json.exception.type_error.308 | cannot use push_back() with string | The @ref push_back() and @ref operator+= member functions can only be executed for certain JSON types. +json.exception.type_error.309 | cannot use insert() with | The @ref insert() member functions can only be executed for certain JSON types. +json.exception.type_error.310 | cannot use swap() with number | The @ref swap() member functions can only be executed for certain JSON types. +json.exception.type_error.311 | cannot use emplace_back() with string | The @ref emplace_back() member function can only be executed for certain JSON types. +json.exception.type_error.312 | cannot use update() with string | The @ref update() member functions can only be executed for certain JSON types. +json.exception.type_error.313 | invalid value to unflatten | The @ref unflatten function converts an object whose keys are JSON Pointers back into an arbitrary nested JSON value. The JSON Pointers must not overlap, because then the resulting value would not be well defined. +json.exception.type_error.314 | only objects can be unflattened | The @ref unflatten function only works for an object whose keys are JSON Pointers. +json.exception.type_error.315 | values in object must be primitive | The @ref unflatten function only works for an object whose keys are JSON Pointers and whose values are primitive. +json.exception.type_error.316 | invalid UTF-8 byte at index 10: 0x7E | The @ref dump function only works with UTF-8 encoded strings; that is, if you assign a `std::string` to a JSON value, make sure it is UTF-8 encoded. | +json.exception.type_error.317 | JSON value cannot be serialized to requested format | The dynamic type of the object cannot be represented in the requested serialization format (e.g. a raw `true` or `null` JSON object cannot be serialized to BSON) | + +@liveexample{The following code shows how a `type_error` exception can be +caught.,type_error} + +@sa @ref exception for the base class of the library exceptions +@sa @ref parse_error for exceptions indicating a parse error +@sa @ref invalid_iterator for exceptions indicating errors with iterators +@sa @ref out_of_range for exceptions indicating access out of the defined range +@sa @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class type_error : public exception +{ + public: + static type_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("type_error", id_) + what_arg; + return type_error(id_, w.c_str()); + } + + private: + type_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating access out of the defined range + +This exception is thrown in case a library function is called on an input +parameter that exceeds the expected range, for instance in case of array +indices or nonexisting object keys. + +Exceptions have ids 4xx. + +name / id | example message | description +------------------------------- | --------------- | ------------------------- +json.exception.out_of_range.401 | array index 3 is out of range | The provided array index @a i is larger than @a size-1. +json.exception.out_of_range.402 | array index '-' (3) is out of range | The special array index `-` in a JSON Pointer never describes a valid element of the array, but the index past the end. That is, it can only be used to add elements at this position, but not to read it. +json.exception.out_of_range.403 | key 'foo' not found | The provided key was not found in the JSON object. +json.exception.out_of_range.404 | unresolved reference token 'foo' | A reference token in a JSON Pointer could not be resolved. +json.exception.out_of_range.405 | JSON pointer has no parent | The JSON Patch operations 'remove' and 'add' can not be applied to the root element of the JSON value. +json.exception.out_of_range.406 | number overflow parsing '10E1000' | A parsed number could not be stored as without changing it to NaN or INF. +json.exception.out_of_range.407 | number overflow serializing '9223372036854775808' | UBJSON and BSON only support integer numbers up to 9223372036854775807. | +json.exception.out_of_range.408 | excessive array size: 8658170730974374167 | The size (following `#`) of an UBJSON array or object exceeds the maximal capacity. | +json.exception.out_of_range.409 | BSON key cannot contain code point U+0000 (at byte 2) | Key identifiers to be serialized to BSON cannot contain code point U+0000, since the key is stored as zero-terminated c-string | + +@liveexample{The following code shows how an `out_of_range` exception can be +caught.,out_of_range} + +@sa @ref exception for the base class of the library exceptions +@sa @ref parse_error for exceptions indicating a parse error +@sa @ref invalid_iterator for exceptions indicating errors with iterators +@sa @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa @ref other_error for exceptions indicating other library errors + +@since version 3.0.0 +*/ +class out_of_range : public exception +{ + public: + static out_of_range create(int id_, const std::string& what_arg) + { + std::string w = exception::name("out_of_range", id_) + what_arg; + return out_of_range(id_, w.c_str()); + } + + private: + out_of_range(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; + +/*! +@brief exception indicating other library errors + +This exception is thrown in case of errors that cannot be classified with the +other exception types. + +Exceptions have ids 5xx. + +name / id | example message | description +------------------------------ | --------------- | ------------------------- +json.exception.other_error.501 | unsuccessful: {"op":"test","path":"/baz", "value":"bar"} | A JSON Patch operation 'test' failed. The unsuccessful operation is also printed. + +@sa @ref exception for the base class of the library exceptions +@sa @ref parse_error for exceptions indicating a parse error +@sa @ref invalid_iterator for exceptions indicating errors with iterators +@sa @ref type_error for exceptions indicating executing a member function with + a wrong type +@sa @ref out_of_range for exceptions indicating access out of the defined range + +@liveexample{The following code shows how an `other_error` exception can be +caught.,other_error} + +@since version 3.0.0 +*/ +class other_error : public exception +{ + public: + static other_error create(int id_, const std::string& what_arg) + { + std::string w = exception::name("other_error", id_) + what_arg; + return other_error(id_, w.c_str()); + } + + private: + other_error(int id_, const char* what_arg) : exception(id_, what_arg) {} +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/value_t.hpp> + + +#include <array> // array +#include <ciso646> // and +#include <cstddef> // size_t +#include <cstdint> // uint8_t + +namespace nlohmann +{ +namespace detail +{ +/////////////////////////// +// JSON type enumeration // +/////////////////////////// + +/*! +@brief the JSON type enumeration + +This enumeration collects the different JSON types. It is internally used to +distinguish the stored values, and the functions @ref basic_json::is_null(), +@ref basic_json::is_object(), @ref basic_json::is_array(), +@ref basic_json::is_string(), @ref basic_json::is_boolean(), +@ref basic_json::is_number() (with @ref basic_json::is_number_integer(), +@ref basic_json::is_number_unsigned(), and @ref basic_json::is_number_float()), +@ref basic_json::is_discarded(), @ref basic_json::is_primitive(), and +@ref basic_json::is_structured() rely on it. + +@note There are three enumeration entries (number_integer, number_unsigned, and +number_float), because the library distinguishes these three types for numbers: +@ref basic_json::number_unsigned_t is used for unsigned integers, +@ref basic_json::number_integer_t is used for signed integers, and +@ref basic_json::number_float_t is used for floating-point numbers or to +approximate integers which do not fit in the limits of their respective type. + +@sa @ref basic_json::basic_json(const value_t value_type) -- create a JSON +value with the default value for a given type + +@since version 1.0.0 +*/ +enum class value_t : std::uint8_t +{ + null, ///< null value + object, ///< object (unordered set of name/value pairs) + array, ///< array (ordered collection of values) + string, ///< string value + boolean, ///< boolean value + number_integer, ///< number value (signed integer) + number_unsigned, ///< number value (unsigned integer) + number_float, ///< number value (floating-point) + discarded ///< discarded by the the parser callback function +}; + +/*! +@brief comparison operator for JSON types + +Returns an ordering that is similar to Python: +- order: null < boolean < number < object < array < string +- furthermore, each type is not smaller than itself +- discarded values are not comparable + +@since version 1.0.0 +*/ +inline bool operator<(const value_t lhs, const value_t rhs) noexcept +{ + static constexpr std::array<std::uint8_t, 8> order = {{ + 0 /* null */, 3 /* object */, 4 /* array */, 5 /* string */, + 1 /* boolean */, 2 /* integer */, 2 /* unsigned */, 2 /* float */ + } + }; + + const auto l_index = static_cast<std::size_t>(lhs); + const auto r_index = static_cast<std::size_t>(rhs); + return l_index < order.size() and r_index < order.size() and order[l_index] < order[r_index]; +} +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/conversions/from_json.hpp> + + +#include <algorithm> // transform +#include <array> // array +#include <ciso646> // and, not +#include <forward_list> // forward_list +#include <iterator> // inserter, front_inserter, end +#include <map> // map +#include <string> // string +#include <tuple> // tuple, make_tuple +#include <type_traits> // is_arithmetic, is_same, is_enum, underlying_type, is_convertible +#include <unordered_map> // unordered_map +#include <utility> // pair, declval +#include <valarray> // valarray + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename std::nullptr_t& n) +{ + if (JSON_UNLIKELY(not j.is_null())) + { + JSON_THROW(type_error::create(302, "type must be null, but is " + std::string(j.type_name()))); + } + n = nullptr; +} + +// overloads for basic_json template parameters +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t<std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void get_arithmetic_value(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::boolean_t& b) +{ + if (JSON_UNLIKELY(not j.is_boolean())) + { + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(j.type_name()))); + } + b = *j.template get_ptr<const typename BasicJsonType::boolean_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::string_t& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template < + typename BasicJsonType, typename ConstructibleStringType, + enable_if_t < + is_constructible_string_type<BasicJsonType, ConstructibleStringType>::value and + not std::is_same<typename BasicJsonType::string_t, + ConstructibleStringType>::value, + int > = 0 > +void from_json(const BasicJsonType& j, ConstructibleStringType& s) +{ + if (JSON_UNLIKELY(not j.is_string())) + { + JSON_THROW(type_error::create(302, "type must be string, but is " + std::string(j.type_name()))); + } + + s = *j.template get_ptr<const typename BasicJsonType::string_t*>(); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_float_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_unsigned_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType> +void from_json(const BasicJsonType& j, typename BasicJsonType::number_integer_t& val) +{ + get_arithmetic_value(j, val); +} + +template<typename BasicJsonType, typename EnumType, + enable_if_t<std::is_enum<EnumType>::value, int> = 0> +void from_json(const BasicJsonType& j, EnumType& e) +{ + typename std::underlying_type<EnumType>::type val; + get_arithmetic_value(j, val); + e = static_cast<EnumType>(val); +} + +// forward_list doesn't have an insert method +template<typename BasicJsonType, typename T, typename Allocator, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::forward_list<T, Allocator>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + std::transform(j.rbegin(), j.rend(), + std::front_inserter(l), [](const BasicJsonType & i) + { + return i.template get<T>(); + }); +} + +// valarray doesn't have an insert method +template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<BasicJsonType, T>::value, int> = 0> +void from_json(const BasicJsonType& j, std::valarray<T>& l) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + l.resize(j.size()); + std::copy(j.m_value.array->begin(), j.m_value.array->end(), std::begin(l)); +} + +template<typename BasicJsonType> +void from_json_array_impl(const BasicJsonType& j, typename BasicJsonType::array_t& arr, priority_tag<3> /*unused*/) +{ + arr = *j.template get_ptr<const typename BasicJsonType::array_t*>(); +} + +template <typename BasicJsonType, typename T, std::size_t N> +auto from_json_array_impl(const BasicJsonType& j, std::array<T, N>& arr, + priority_tag<2> /*unused*/) +-> decltype(j.template get<T>(), void()) +{ + for (std::size_t i = 0; i < N; ++i) + { + arr[i] = j.at(i).template get<T>(); + } +} + +template<typename BasicJsonType, typename ConstructibleArrayType> +auto from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, priority_tag<1> /*unused*/) +-> decltype( + arr.reserve(std::declval<typename ConstructibleArrayType::size_type>()), + j.template get<typename ConstructibleArrayType::value_type>(), + void()) +{ + using std::end; + + arr.reserve(j.size()); + std::transform(j.begin(), j.end(), + std::inserter(arr, end(arr)), [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename ConstructibleArrayType::value_type>(); + }); +} + +template <typename BasicJsonType, typename ConstructibleArrayType> +void from_json_array_impl(const BasicJsonType& j, ConstructibleArrayType& arr, + priority_tag<0> /*unused*/) +{ + using std::end; + + std::transform( + j.begin(), j.end(), std::inserter(arr, end(arr)), + [](const BasicJsonType & i) + { + // get<BasicJsonType>() returns *this, this won't call a from_json + // method when value_type is BasicJsonType + return i.template get<typename ConstructibleArrayType::value_type>(); + }); +} + +template <typename BasicJsonType, typename ConstructibleArrayType, + enable_if_t < + is_constructible_array_type<BasicJsonType, ConstructibleArrayType>::value and + not is_constructible_object_type<BasicJsonType, ConstructibleArrayType>::value and + not is_constructible_string_type<BasicJsonType, ConstructibleArrayType>::value and + not is_basic_json<ConstructibleArrayType>::value, + int > = 0 > + +auto from_json(const BasicJsonType& j, ConstructibleArrayType& arr) +-> decltype(from_json_array_impl(j, arr, priority_tag<3> {}), +j.template get<typename ConstructibleArrayType::value_type>(), +void()) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + + std::string(j.type_name()))); + } + + from_json_array_impl(j, arr, priority_tag<3> {}); +} + +template<typename BasicJsonType, typename ConstructibleObjectType, + enable_if_t<is_constructible_object_type<BasicJsonType, ConstructibleObjectType>::value, int> = 0> +void from_json(const BasicJsonType& j, ConstructibleObjectType& obj) +{ + if (JSON_UNLIKELY(not j.is_object())) + { + JSON_THROW(type_error::create(302, "type must be object, but is " + std::string(j.type_name()))); + } + + auto inner_object = j.template get_ptr<const typename BasicJsonType::object_t*>(); + using value_type = typename ConstructibleObjectType::value_type; + std::transform( + inner_object->begin(), inner_object->end(), + std::inserter(obj, obj.begin()), + [](typename BasicJsonType::object_t::value_type const & p) + { + return value_type(p.first, p.second.template get<typename ConstructibleObjectType::mapped_type>()); + }); +} + +// overload for arithmetic types, not chosen for basic_json template arguments +// (BooleanType, etc..); note: Is it really necessary to provide explicit +// overloads for boolean_t etc. in case of a custom BooleanType which is not +// an arithmetic type? +template<typename BasicJsonType, typename ArithmeticType, + enable_if_t < + std::is_arithmetic<ArithmeticType>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_unsigned_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_integer_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::number_float_t>::value and + not std::is_same<ArithmeticType, typename BasicJsonType::boolean_t>::value, + int> = 0> +void from_json(const BasicJsonType& j, ArithmeticType& val) +{ + switch (static_cast<value_t>(j)) + { + case value_t::number_unsigned: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_unsigned_t*>()); + break; + } + case value_t::number_integer: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_integer_t*>()); + break; + } + case value_t::number_float: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::number_float_t*>()); + break; + } + case value_t::boolean: + { + val = static_cast<ArithmeticType>(*j.template get_ptr<const typename BasicJsonType::boolean_t*>()); + break; + } + + default: + JSON_THROW(type_error::create(302, "type must be number, but is " + std::string(j.type_name()))); + } +} + +template<typename BasicJsonType, typename A1, typename A2> +void from_json(const BasicJsonType& j, std::pair<A1, A2>& p) +{ + p = {j.at(0).template get<A1>(), j.at(1).template get<A2>()}; +} + +template<typename BasicJsonType, typename Tuple, std::size_t... Idx> +void from_json_tuple_impl(const BasicJsonType& j, Tuple& t, index_sequence<Idx...> /*unused*/) +{ + t = std::make_tuple(j.at(Idx).template get<typename std::tuple_element<Idx, Tuple>::type>()...); +} + +template<typename BasicJsonType, typename... Args> +void from_json(const BasicJsonType& j, std::tuple<Args...>& t) +{ + from_json_tuple_impl(j, t, index_sequence_for<Args...> {}); +} + +template <typename BasicJsonType, typename Key, typename Value, typename Compare, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::map<Key, Value, Compare, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +template <typename BasicJsonType, typename Key, typename Value, typename Hash, typename KeyEqual, typename Allocator, + typename = enable_if_t<not std::is_constructible< + typename BasicJsonType::string_t, Key>::value>> +void from_json(const BasicJsonType& j, std::unordered_map<Key, Value, Hash, KeyEqual, Allocator>& m) +{ + if (JSON_UNLIKELY(not j.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(j.type_name()))); + } + for (const auto& p : j) + { + if (JSON_UNLIKELY(not p.is_array())) + { + JSON_THROW(type_error::create(302, "type must be array, but is " + std::string(p.type_name()))); + } + m.emplace(p.at(0).template get<Key>(), p.at(1).template get<Value>()); + } +} + +struct from_json_fn +{ + template<typename BasicJsonType, typename T> + auto operator()(const BasicJsonType& j, T& val) const + noexcept(noexcept(from_json(j, val))) + -> decltype(from_json(j, val), void()) + { + return from_json(j, val); + } +}; +} // namespace detail + +/// namespace to hold default `from_json` function +/// to see why this is required: +/// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2015/n4381.html +namespace +{ +constexpr const auto& from_json = detail::static_const<detail::from_json_fn>::value; +} // namespace +} // namespace nlohmann + +// #include <nlohmann/detail/conversions/to_json.hpp> + + +#include <ciso646> // or, and, not +#include <iterator> // begin, end +#include <tuple> // tuple, get +#include <type_traits> // is_same, is_constructible, is_floating_point, is_enum, underlying_type +#include <utility> // move, forward, declval, pair +#include <valarray> // valarray +#include <vector> // vector + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + +// #include <nlohmann/detail/value_t.hpp> + +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> + + +#include <cstddef> // size_t +#include <string> // string, to_string +#include <iterator> // input_iterator_tag +#include <tuple> // tuple_size, get, tuple_element + +// #include <nlohmann/detail/value_t.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename IteratorType> class iteration_proxy_value +{ + public: + using difference_type = std::ptrdiff_t; + using value_type = iteration_proxy_value; + using pointer = value_type * ; + using reference = value_type & ; + using iterator_category = std::input_iterator_tag; + + private: + /// the iterator + IteratorType anchor; + /// an index for arrays (used to create key names) + std::size_t array_index = 0; + /// last stringified array index + mutable std::size_t array_index_last = 0; + /// a string representation of the array index + mutable std::string array_index_str = "0"; + /// an empty string (to return a reference for primitive values) + const std::string empty_str = ""; + + public: + explicit iteration_proxy_value(IteratorType it) noexcept : anchor(it) {} + + /// dereference operator (needed for range-based for) + iteration_proxy_value& operator*() + { + return *this; + } + + /// increment operator (needed for range-based for) + iteration_proxy_value& operator++() + { + ++anchor; + ++array_index; + + return *this; + } + + /// equality operator (needed for InputIterator) + bool operator==(const iteration_proxy_value& o) const noexcept + { + return anchor == o.anchor; + } + + /// inequality operator (needed for range-based for) + bool operator!=(const iteration_proxy_value& o) const noexcept + { + return anchor != o.anchor; + } + + /// return key of the iterator + const std::string& key() const + { + assert(anchor.m_object != nullptr); + + switch (anchor.m_object->type()) + { + // use integer array index as key + case value_t::array: + { + if (array_index != array_index_last) + { + array_index_str = std::to_string(array_index); + array_index_last = array_index; + } + return array_index_str; + } + + // use key from the object + case value_t::object: + return anchor.key(); + + // use an empty key for all primitive types + default: + return empty_str; + } + } + + /// return value of the iterator + typename IteratorType::reference value() const + { + return anchor.value(); + } +}; + +/// proxy class for the items() function +template<typename IteratorType> class iteration_proxy +{ + private: + /// the container to iterate + typename IteratorType::reference container; + + public: + /// construct iteration proxy from a container + explicit iteration_proxy(typename IteratorType::reference cont) noexcept + : container(cont) {} + + /// return iterator begin (needed for range-based for) + iteration_proxy_value<IteratorType> begin() noexcept + { + return iteration_proxy_value<IteratorType>(container.begin()); + } + + /// return iterator end (needed for range-based for) + iteration_proxy_value<IteratorType> end() noexcept + { + return iteration_proxy_value<IteratorType>(container.end()); + } +}; +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template <std::size_t N, typename IteratorType, enable_if_t<N == 0, int> = 0> +auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.key()) +{ + return i.key(); +} +// Structured Bindings Support +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +template <std::size_t N, typename IteratorType, enable_if_t<N == 1, int> = 0> +auto get(const nlohmann::detail::iteration_proxy_value<IteratorType>& i) -> decltype(i.value()) +{ + return i.value(); +} +} // namespace detail +} // namespace nlohmann + +// The Addition to the STD Namespace is required to add +// Structured Bindings Support to the iteration_proxy_value class +// For further reference see https://blog.tartanllama.xyz/structured-bindings/ +// And see https://github.com/nlohmann/json/pull/1391 +namespace std +{ +template <typename IteratorType> +class tuple_size<::nlohmann::detail::iteration_proxy_value<IteratorType>> + : public std::integral_constant<std::size_t, 2> {}; + +template <std::size_t N, typename IteratorType> +class tuple_element<N, ::nlohmann::detail::iteration_proxy_value<IteratorType >> +{ + public: + using type = decltype( + get<N>(std::declval < + ::nlohmann::detail::iteration_proxy_value<IteratorType >> ())); +}; +} + +namespace nlohmann +{ +namespace detail +{ +////////////////// +// constructors // +////////////////// + +template<value_t> struct external_constructor; + +template<> +struct external_constructor<value_t::boolean> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::boolean_t b) noexcept + { + j.m_type = value_t::boolean; + j.m_value = b; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor<value_t::string> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, const typename BasicJsonType::string_t& s) + { + j.m_type = value_t::string; + j.m_value = s; + j.assert_invariant(); + } + + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::string_t&& s) + { + j.m_type = value_t::string; + j.m_value = std::move(s); + j.assert_invariant(); + } + + template<typename BasicJsonType, typename CompatibleStringType, + enable_if_t<not std::is_same<CompatibleStringType, typename BasicJsonType::string_t>::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleStringType& str) + { + j.m_type = value_t::string; + j.m_value.string = j.template create<typename BasicJsonType::string_t>(str); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor<value_t::number_float> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::number_float_t val) noexcept + { + j.m_type = value_t::number_float; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor<value_t::number_unsigned> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::number_unsigned_t val) noexcept + { + j.m_type = value_t::number_unsigned; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor<value_t::number_integer> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::number_integer_t val) noexcept + { + j.m_type = value_t::number_integer; + j.m_value = val; + j.assert_invariant(); + } +}; + +template<> +struct external_constructor<value_t::array> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, const typename BasicJsonType::array_t& arr) + { + j.m_type = value_t::array; + j.m_value = arr; + j.assert_invariant(); + } + + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::array_t&& arr) + { + j.m_type = value_t::array; + j.m_value = std::move(arr); + j.assert_invariant(); + } + + template<typename BasicJsonType, typename CompatibleArrayType, + enable_if_t<not std::is_same<CompatibleArrayType, typename BasicJsonType::array_t>::value, + int> = 0> + static void construct(BasicJsonType& j, const CompatibleArrayType& arr) + { + using std::begin; + using std::end; + j.m_type = value_t::array; + j.m_value.array = j.template create<typename BasicJsonType::array_t>(begin(arr), end(arr)); + j.assert_invariant(); + } + + template<typename BasicJsonType> + static void construct(BasicJsonType& j, const std::vector<bool>& arr) + { + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->reserve(arr.size()); + for (const bool x : arr) + { + j.m_value.array->push_back(x); + } + j.assert_invariant(); + } + + template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> + static void construct(BasicJsonType& j, const std::valarray<T>& arr) + { + j.m_type = value_t::array; + j.m_value = value_t::array; + j.m_value.array->resize(arr.size()); + std::copy(std::begin(arr), std::end(arr), j.m_value.array->begin()); + j.assert_invariant(); + } +}; + +template<> +struct external_constructor<value_t::object> +{ + template<typename BasicJsonType> + static void construct(BasicJsonType& j, const typename BasicJsonType::object_t& obj) + { + j.m_type = value_t::object; + j.m_value = obj; + j.assert_invariant(); + } + + template<typename BasicJsonType> + static void construct(BasicJsonType& j, typename BasicJsonType::object_t&& obj) + { + j.m_type = value_t::object; + j.m_value = std::move(obj); + j.assert_invariant(); + } + + template<typename BasicJsonType, typename CompatibleObjectType, + enable_if_t<not std::is_same<CompatibleObjectType, typename BasicJsonType::object_t>::value, int> = 0> + static void construct(BasicJsonType& j, const CompatibleObjectType& obj) + { + using std::begin; + using std::end; + + j.m_type = value_t::object; + j.m_value.object = j.template create<typename BasicJsonType::object_t>(begin(obj), end(obj)); + j.assert_invariant(); + } +}; + +///////////// +// to_json // +///////////// + +template<typename BasicJsonType, typename T, + enable_if_t<std::is_same<T, typename BasicJsonType::boolean_t>::value, int> = 0> +void to_json(BasicJsonType& j, T b) noexcept +{ + external_constructor<value_t::boolean>::construct(j, b); +} + +template<typename BasicJsonType, typename CompatibleString, + enable_if_t<std::is_constructible<typename BasicJsonType::string_t, CompatibleString>::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleString& s) +{ + external_constructor<value_t::string>::construct(j, s); +} + +template<typename BasicJsonType> +void to_json(BasicJsonType& j, typename BasicJsonType::string_t&& s) +{ + external_constructor<value_t::string>::construct(j, std::move(s)); +} + +template<typename BasicJsonType, typename FloatType, + enable_if_t<std::is_floating_point<FloatType>::value, int> = 0> +void to_json(BasicJsonType& j, FloatType val) noexcept +{ + external_constructor<value_t::number_float>::construct(j, static_cast<typename BasicJsonType::number_float_t>(val)); +} + +template<typename BasicJsonType, typename CompatibleNumberUnsignedType, + enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_unsigned_t, CompatibleNumberUnsignedType>::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberUnsignedType val) noexcept +{ + external_constructor<value_t::number_unsigned>::construct(j, static_cast<typename BasicJsonType::number_unsigned_t>(val)); +} + +template<typename BasicJsonType, typename CompatibleNumberIntegerType, + enable_if_t<is_compatible_integer_type<typename BasicJsonType::number_integer_t, CompatibleNumberIntegerType>::value, int> = 0> +void to_json(BasicJsonType& j, CompatibleNumberIntegerType val) noexcept +{ + external_constructor<value_t::number_integer>::construct(j, static_cast<typename BasicJsonType::number_integer_t>(val)); +} + +template<typename BasicJsonType, typename EnumType, + enable_if_t<std::is_enum<EnumType>::value, int> = 0> +void to_json(BasicJsonType& j, EnumType e) noexcept +{ + using underlying_type = typename std::underlying_type<EnumType>::type; + external_constructor<value_t::number_integer>::construct(j, static_cast<underlying_type>(e)); +} + +template<typename BasicJsonType> +void to_json(BasicJsonType& j, const std::vector<bool>& e) +{ + external_constructor<value_t::array>::construct(j, e); +} + +template <typename BasicJsonType, typename CompatibleArrayType, + enable_if_t<is_compatible_array_type<BasicJsonType, + CompatibleArrayType>::value and + not is_compatible_object_type< + BasicJsonType, CompatibleArrayType>::value and + not is_compatible_string_type<BasicJsonType, CompatibleArrayType>::value and + not is_basic_json<CompatibleArrayType>::value, + int> = 0> +void to_json(BasicJsonType& j, const CompatibleArrayType& arr) +{ + external_constructor<value_t::array>::construct(j, arr); +} + +template<typename BasicJsonType, typename T, + enable_if_t<std::is_convertible<T, BasicJsonType>::value, int> = 0> +void to_json(BasicJsonType& j, const std::valarray<T>& arr) +{ + external_constructor<value_t::array>::construct(j, std::move(arr)); +} + +template<typename BasicJsonType> +void to_json(BasicJsonType& j, typename BasicJsonType::array_t&& arr) +{ + external_constructor<value_t::array>::construct(j, std::move(arr)); +} + +template<typename BasicJsonType, typename CompatibleObjectType, + enable_if_t<is_compatible_object_type<BasicJsonType, CompatibleObjectType>::value and not is_basic_json<CompatibleObjectType>::value, int> = 0> +void to_json(BasicJsonType& j, const CompatibleObjectType& obj) +{ + external_constructor<value_t::object>::construct(j, obj); +} + +template<typename BasicJsonType> +void to_json(BasicJsonType& j, typename BasicJsonType::object_t&& obj) +{ + external_constructor<value_t::object>::construct(j, std::move(obj)); +} + +template < + typename BasicJsonType, typename T, std::size_t N, + enable_if_t<not std::is_constructible<typename BasicJsonType::string_t, + const T(&)[N]>::value, + int> = 0 > +void to_json(BasicJsonType& j, const T(&arr)[N]) +{ + external_constructor<value_t::array>::construct(j, arr); +} + +template<typename BasicJsonType, typename... Args> +void to_json(BasicJsonType& j, const std::pair<Args...>& p) +{ + j = { p.first, p.second }; +} + +// for https://github.com/nlohmann/json/pull/1134 +template < typename BasicJsonType, typename T, + enable_if_t<std::is_same<T, iteration_proxy_value<typename BasicJsonType::iterator>>::value, int> = 0> +void to_json(BasicJsonType& j, const T& b) +{ + j = { {b.key(), b.value()} }; +} + +template<typename BasicJsonType, typename Tuple, std::size_t... Idx> +void to_json_tuple_impl(BasicJsonType& j, const Tuple& t, index_sequence<Idx...> /*unused*/) +{ + j = { std::get<Idx>(t)... }; +} + +template<typename BasicJsonType, typename... Args> +void to_json(BasicJsonType& j, const std::tuple<Args...>& t) +{ + to_json_tuple_impl(j, t, index_sequence_for<Args...> {}); +} + +struct to_json_fn +{ + template<typename BasicJsonType, typename T> + auto operator()(BasicJsonType& j, T&& val) const noexcept(noexcept(to_json(j, std::forward<T>(val)))) + -> decltype(to_json(j, std::forward<T>(val)), void()) + { + return to_json(j, std::forward<T>(val)); + } +}; +} // namespace detail + +/// namespace to hold default `to_json` function +namespace +{ +constexpr const auto& to_json = detail::static_const<detail::to_json_fn>::value; +} // namespace +} // namespace nlohmann + +// #include <nlohmann/detail/input/input_adapters.hpp> + + +#include <cassert> // assert +#include <cstddef> // size_t +#include <cstring> // strlen +#include <istream> // istream +#include <iterator> // begin, end, iterator_traits, random_access_iterator_tag, distance, next +#include <memory> // shared_ptr, make_shared, addressof +#include <numeric> // accumulate +#include <string> // string, char_traits +#include <type_traits> // enable_if, is_base_of, is_pointer, is_integral, remove_pointer +#include <utility> // pair, declval +#include <cstdio> //FILE * + +// #include <nlohmann/detail/macro_scope.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/// the supported input formats +enum class input_format_t { json, cbor, msgpack, ubjson, bson }; + +//////////////////// +// input adapters // +//////////////////// + +/*! +@brief abstract input adapter interface + +Produces a stream of std::char_traits<char>::int_type characters from a +std::istream, a buffer, or some other input type. Accepts the return of +exactly one non-EOF character for future input. The int_type characters +returned consist of all valid char values as positive values (typically +unsigned char), plus an EOF value outside that range, specified by the value +of the function std::char_traits<char>::eof(). This value is typically -1, but +could be any arbitrary value which is not a valid char value. +*/ +struct input_adapter_protocol +{ + /// get a character [0,255] or std::char_traits<char>::eof(). + virtual std::char_traits<char>::int_type get_character() = 0; + virtual ~input_adapter_protocol() = default; +}; + +/// a type to simplify interfaces +using input_adapter_t = std::shared_ptr<input_adapter_protocol>; + +/*! +Input adapter for stdio file access. This adapter read only 1 byte and do not use any + buffer. This adapter is a very low level adapter. +*/ +class file_input_adapter : public input_adapter_protocol +{ + public: + explicit file_input_adapter(std::FILE* f) noexcept + : m_file(f) + {} + + std::char_traits<char>::int_type get_character() noexcept override + { + return std::fgetc(m_file); + } + private: + /// the file pointer to read from + std::FILE* m_file; +}; + + +/*! +Input adapter for a (caching) istream. Ignores a UFT Byte Order Mark at +beginning of input. Does not support changing the underlying std::streambuf +in mid-input. Maintains underlying std::istream and std::streambuf to support +subsequent use of standard std::istream operations to process any input +characters following those used in parsing the JSON input. Clears the +std::istream flags; any input errors (e.g., EOF) will be detected by the first +subsequent call for input from the std::istream. +*/ +class input_stream_adapter : public input_adapter_protocol +{ + public: + ~input_stream_adapter() override + { + // clear stream flags; we use underlying streambuf I/O, do not + // maintain ifstream flags, except eof + is.clear(is.rdstate() & std::ios::eofbit); + } + + explicit input_stream_adapter(std::istream& i) + : is(i), sb(*i.rdbuf()) + {} + + // delete because of pointer members + input_stream_adapter(const input_stream_adapter&) = delete; + input_stream_adapter& operator=(input_stream_adapter&) = delete; + input_stream_adapter(input_stream_adapter&&) = delete; + input_stream_adapter& operator=(input_stream_adapter&&) = delete; + + // std::istream/std::streambuf use std::char_traits<char>::to_int_type, to + // ensure that std::char_traits<char>::eof() and the character 0xFF do not + // end up as the same value, eg. 0xFFFFFFFF. + std::char_traits<char>::int_type get_character() override + { + auto res = sb.sbumpc(); + // set eof manually, as we don't use the istream interface. + if (res == EOF) + { + is.clear(is.rdstate() | std::ios::eofbit); + } + return res; + } + + private: + /// the associated input stream + std::istream& is; + std::streambuf& sb; +}; + +/// input adapter for buffer input +class input_buffer_adapter : public input_adapter_protocol +{ + public: + input_buffer_adapter(const char* b, const std::size_t l) noexcept + : cursor(b), limit(b + l) + {} + + // delete because of pointer members + input_buffer_adapter(const input_buffer_adapter&) = delete; + input_buffer_adapter& operator=(input_buffer_adapter&) = delete; + input_buffer_adapter(input_buffer_adapter&&) = delete; + input_buffer_adapter& operator=(input_buffer_adapter&&) = delete; + ~input_buffer_adapter() override = default; + + std::char_traits<char>::int_type get_character() noexcept override + { + if (JSON_LIKELY(cursor < limit)) + { + return std::char_traits<char>::to_int_type(*(cursor++)); + } + + return std::char_traits<char>::eof(); + } + + private: + /// pointer to the current character + const char* cursor; + /// pointer past the last character + const char* const limit; +}; + +template<typename WideStringType, size_t T> +struct wide_string_input_helper +{ + // UTF-32 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = static_cast<int>(str[current_wchar++]); + + // UTF-32 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6) & 0x1F); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (wc <= 0xFFFF) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12) & 0x0F); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else if (wc <= 0x10FFFF) + { + utf8_bytes[0] = 0xF0 | ((wc >> 18) & 0x07); + utf8_bytes[1] = 0x80 | ((wc >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + } + } +}; + +template<typename WideStringType> +struct wide_string_input_helper<WideStringType, 2> +{ + // UTF-16 + static void fill_buffer(const WideStringType& str, size_t& current_wchar, std::array<std::char_traits<char>::int_type, 4>& utf8_bytes, size_t& utf8_bytes_index, size_t& utf8_bytes_filled) + { + utf8_bytes_index = 0; + + if (current_wchar == str.size()) + { + utf8_bytes[0] = std::char_traits<char>::eof(); + utf8_bytes_filled = 1; + } + else + { + // get the current character + const auto wc = static_cast<int>(str[current_wchar++]); + + // UTF-16 to UTF-8 encoding + if (wc < 0x80) + { + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + else if (wc <= 0x7FF) + { + utf8_bytes[0] = 0xC0 | ((wc >> 6)); + utf8_bytes[1] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 2; + } + else if (0xD800 > wc or wc >= 0xE000) + { + utf8_bytes[0] = 0xE0 | ((wc >> 12)); + utf8_bytes[1] = 0x80 | ((wc >> 6) & 0x3F); + utf8_bytes[2] = 0x80 | (wc & 0x3F); + utf8_bytes_filled = 3; + } + else + { + if (current_wchar < str.size()) + { + const auto wc2 = static_cast<int>(str[current_wchar++]); + const int charcode = 0x10000 + (((wc & 0x3FF) << 10) | (wc2 & 0x3FF)); + utf8_bytes[0] = 0xf0 | (charcode >> 18); + utf8_bytes[1] = 0x80 | ((charcode >> 12) & 0x3F); + utf8_bytes[2] = 0x80 | ((charcode >> 6) & 0x3F); + utf8_bytes[3] = 0x80 | (charcode & 0x3F); + utf8_bytes_filled = 4; + } + else + { + // unknown character + ++current_wchar; + utf8_bytes[0] = wc; + utf8_bytes_filled = 1; + } + } + } + } +}; + +template<typename WideStringType> +class wide_string_input_adapter : public input_adapter_protocol +{ + public: + explicit wide_string_input_adapter(const WideStringType& w) noexcept + : str(w) + {} + + std::char_traits<char>::int_type get_character() noexcept override + { + // check if buffer needs to be filled + if (utf8_bytes_index == utf8_bytes_filled) + { + fill_buffer<sizeof(typename WideStringType::value_type)>(); + + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index == 0); + } + + // use buffer + assert(utf8_bytes_filled > 0); + assert(utf8_bytes_index < utf8_bytes_filled); + return utf8_bytes[utf8_bytes_index++]; + } + + private: + template<size_t T> + void fill_buffer() + { + wide_string_input_helper<WideStringType, T>::fill_buffer(str, current_wchar, utf8_bytes, utf8_bytes_index, utf8_bytes_filled); + } + + /// the wstring to process + const WideStringType& str; + + /// index of the current wchar in str + std::size_t current_wchar = 0; + + /// a buffer for UTF-8 bytes + std::array<std::char_traits<char>::int_type, 4> utf8_bytes = {{0, 0, 0, 0}}; + + /// index to the utf8_codes array for the next valid byte + std::size_t utf8_bytes_index = 0; + /// number of valid bytes in the utf8_codes array + std::size_t utf8_bytes_filled = 0; +}; + +class input_adapter +{ + public: + // native support + input_adapter(std::FILE* file) + : ia(std::make_shared<file_input_adapter>(file)) {} + /// input adapter for input stream + input_adapter(std::istream& i) + : ia(std::make_shared<input_stream_adapter>(i)) {} + + /// input adapter for input stream + input_adapter(std::istream&& i) + : ia(std::make_shared<input_stream_adapter>(i)) {} + + input_adapter(const std::wstring& ws) + : ia(std::make_shared<wide_string_input_adapter<std::wstring>>(ws)) {} + + input_adapter(const std::u16string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u16string>>(ws)) {} + + input_adapter(const std::u32string& ws) + : ia(std::make_shared<wide_string_input_adapter<std::u32string>>(ws)) {} + + /// input adapter for buffer + template<typename CharT, + typename std::enable_if< + std::is_pointer<CharT>::value and + std::is_integral<typename std::remove_pointer<CharT>::type>::value and + sizeof(typename std::remove_pointer<CharT>::type) == 1, + int>::type = 0> + input_adapter(CharT b, std::size_t l) + : ia(std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(b), l)) {} + + // derived support + + /// input adapter for string literal + template<typename CharT, + typename std::enable_if< + std::is_pointer<CharT>::value and + std::is_integral<typename std::remove_pointer<CharT>::type>::value and + sizeof(typename std::remove_pointer<CharT>::type) == 1, + int>::type = 0> + input_adapter(CharT b) + : input_adapter(reinterpret_cast<const char*>(b), + std::strlen(reinterpret_cast<const char*>(b))) {} + + /// input adapter for iterator range with contiguous storage + template<class IteratorType, + typename std::enable_if< + std::is_same<typename iterator_traits<IteratorType>::iterator_category, std::random_access_iterator_tag>::value, + int>::type = 0> + input_adapter(IteratorType first, IteratorType last) + { +#ifndef NDEBUG + // assertion to check that the iterator range is indeed contiguous, + // see http://stackoverflow.com/a/35008842/266378 for more discussion + const auto is_contiguous = std::accumulate( + first, last, std::pair<bool, int>(true, 0), + [&first](std::pair<bool, int> res, decltype(*first) val) + { + res.first &= (val == *(std::next(std::addressof(*first), res.second++))); + return res; + }).first; + assert(is_contiguous); +#endif + + // assertion to check that each element is 1 byte long + static_assert( + sizeof(typename iterator_traits<IteratorType>::value_type) == 1, + "each element in the iterator range must have the size of 1 byte"); + + const auto len = static_cast<size_t>(std::distance(first, last)); + if (JSON_LIKELY(len > 0)) + { + // there is at least one element: use the address of first + ia = std::make_shared<input_buffer_adapter>(reinterpret_cast<const char*>(&(*first)), len); + } + else + { + // the address of first cannot be used: use nullptr + ia = std::make_shared<input_buffer_adapter>(nullptr, len); + } + } + + /// input adapter for array + template<class T, std::size_t N> + input_adapter(T (&array)[N]) + : input_adapter(std::begin(array), std::end(array)) {} + + /// input adapter for contiguous container + template<class ContiguousContainer, typename + std::enable_if<not std::is_pointer<ContiguousContainer>::value and + std::is_base_of<std::random_access_iterator_tag, typename iterator_traits<decltype(std::begin(std::declval<ContiguousContainer const>()))>::iterator_category>::value, + int>::type = 0> + input_adapter(const ContiguousContainer& c) + : input_adapter(std::begin(c), std::end(c)) {} + + operator input_adapter_t() + { + return ia; + } + + private: + /// the actual adapter + input_adapter_t ia = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/lexer.hpp> + + +#include <clocale> // localeconv +#include <cstddef> // size_t +#include <cstdlib> // strtof, strtod, strtold, strtoll, strtoull +#include <cstdio> // snprintf +#include <initializer_list> // initializer_list +#include <string> // char_traits, string +#include <vector> // vector + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/position_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////// +// lexer // +/////////// + +/*! +@brief lexical analysis + +This class organizes the lexical analysis during JSON deserialization. +*/ +template<typename BasicJsonType> +class lexer +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + public: + /// token types for the parser + enum class token_type + { + uninitialized, ///< indicating the scanner is uninitialized + literal_true, ///< the `true` literal + literal_false, ///< the `false` literal + literal_null, ///< the `null` literal + value_string, ///< a string -- use get_string() for actual value + value_unsigned, ///< an unsigned integer -- use get_number_unsigned() for actual value + value_integer, ///< a signed integer -- use get_number_integer() for actual value + value_float, ///< an floating point number -- use get_number_float() for actual value + begin_array, ///< the character for array begin `[` + begin_object, ///< the character for object begin `{` + end_array, ///< the character for array end `]` + end_object, ///< the character for object end `}` + name_separator, ///< the name separator `:` + value_separator, ///< the value separator `,` + parse_error, ///< indicating a parse error + end_of_input, ///< indicating the end of the input buffer + literal_or_value ///< a literal or the begin of a value (only for diagnostics) + }; + + /// return name of values of type token_type (only used for errors) + static const char* token_type_name(const token_type t) noexcept + { + switch (t) + { + case token_type::uninitialized: + return "<uninitialized>"; + case token_type::literal_true: + return "true literal"; + case token_type::literal_false: + return "false literal"; + case token_type::literal_null: + return "null literal"; + case token_type::value_string: + return "string literal"; + case lexer::token_type::value_unsigned: + case lexer::token_type::value_integer: + case lexer::token_type::value_float: + return "number literal"; + case token_type::begin_array: + return "'['"; + case token_type::begin_object: + return "'{'"; + case token_type::end_array: + return "']'"; + case token_type::end_object: + return "'}'"; + case token_type::name_separator: + return "':'"; + case token_type::value_separator: + return "','"; + case token_type::parse_error: + return "<parse error>"; + case token_type::end_of_input: + return "end of input"; + case token_type::literal_or_value: + return "'[', '{', or a literal"; + // LCOV_EXCL_START + default: // catch non-enum values + return "unknown token"; + // LCOV_EXCL_STOP + } + } + + explicit lexer(detail::input_adapter_t&& adapter) + : ia(std::move(adapter)), decimal_point_char(get_decimal_point()) {} + + // delete because of pointer members + lexer(const lexer&) = delete; + lexer(lexer&&) = delete; + lexer& operator=(lexer&) = delete; + lexer& operator=(lexer&&) = delete; + ~lexer() = default; + + private: + ///////////////////// + // locales + ///////////////////// + + /// return the locale-dependent decimal point + static char get_decimal_point() noexcept + { + const auto loc = localeconv(); + assert(loc != nullptr); + return (loc->decimal_point == nullptr) ? '.' : *(loc->decimal_point); + } + + ///////////////////// + // scan functions + ///////////////////// + + /*! + @brief get codepoint from 4 hex characters following `\u` + + For input "\u c1 c2 c3 c4" the codepoint is: + (c1 * 0x1000) + (c2 * 0x0100) + (c3 * 0x0010) + c4 + = (c1 << 12) + (c2 << 8) + (c3 << 4) + (c4 << 0) + + Furthermore, the possible characters '0'..'9', 'A'..'F', and 'a'..'f' + must be converted to the integers 0x0..0x9, 0xA..0xF, 0xA..0xF, resp. The + conversion is done by subtracting the offset (0x30, 0x37, and 0x57) + between the ASCII value of the character and the desired integer value. + + @return codepoint (0x0000..0xFFFF) or -1 in case of an error (e.g. EOF or + non-hex character) + */ + int get_codepoint() + { + // this function only makes sense after reading `\u` + assert(current == 'u'); + int codepoint = 0; + + const auto factors = { 12, 8, 4, 0 }; + for (const auto factor : factors) + { + get(); + + if (current >= '0' and current <= '9') + { + codepoint += ((current - 0x30) << factor); + } + else if (current >= 'A' and current <= 'F') + { + codepoint += ((current - 0x37) << factor); + } + else if (current >= 'a' and current <= 'f') + { + codepoint += ((current - 0x57) << factor); + } + else + { + return -1; + } + } + + assert(0x0000 <= codepoint and codepoint <= 0xFFFF); + return codepoint; + } + + /*! + @brief check if the next byte(s) are inside a given range + + Adds the current byte and, for each passed range, reads a new byte and + checks if it is inside the range. If a violation was detected, set up an + error message and return false. Otherwise, return true. + + @param[in] ranges list of integers; interpreted as list of pairs of + inclusive lower and upper bound, respectively + + @pre The passed list @a ranges must have 2, 4, or 6 elements; that is, + 1, 2, or 3 pairs. This precondition is enforced by an assertion. + + @return true if and only if no range violation was detected + */ + bool next_byte_in_range(std::initializer_list<int> ranges) + { + assert(ranges.size() == 2 or ranges.size() == 4 or ranges.size() == 6); + add(current); + + for (auto range = ranges.begin(); range != ranges.end(); ++range) + { + get(); + if (JSON_LIKELY(*range <= current and current <= *(++range))) + { + add(current); + } + else + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return false; + } + } + + return true; + } + + /*! + @brief scan a string literal + + This function scans a string according to Sect. 7 of RFC 7159. While + scanning, bytes are escaped and copied into buffer token_buffer. Then the + function returns successfully, token_buffer is *not* null-terminated (as it + may contain \0 bytes), and token_buffer.size() is the number of bytes in the + string. + + @return token_type::value_string if string could be successfully scanned, + token_type::parse_error otherwise + + @note In case of errors, variable error_message contains a textual + description. + */ + token_type scan_string() + { + // reset token_buffer (ignore opening quote) + reset(); + + // we entered the function by reading an open quote + assert(current == '\"'); + + while (true) + { + // get next character + switch (get()) + { + // end of file while parsing string + case std::char_traits<char>::eof(): + { + error_message = "invalid string: missing closing quote"; + return token_type::parse_error; + } + + // closing quote + case '\"': + { + return token_type::value_string; + } + + // escapes + case '\\': + { + switch (get()) + { + // quotation mark + case '\"': + add('\"'); + break; + // reverse solidus + case '\\': + add('\\'); + break; + // solidus + case '/': + add('/'); + break; + // backspace + case 'b': + add('\b'); + break; + // form feed + case 'f': + add('\f'); + break; + // line feed + case 'n': + add('\n'); + break; + // carriage return + case 'r': + add('\r'); + break; + // tab + case 't': + add('\t'); + break; + + // unicode escapes + case 'u': + { + const int codepoint1 = get_codepoint(); + int codepoint = codepoint1; // start with codepoint1 + + if (JSON_UNLIKELY(codepoint1 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if code point is a high surrogate + if (0xD800 <= codepoint1 and codepoint1 <= 0xDBFF) + { + // expect next \uxxxx entry + if (JSON_LIKELY(get() == '\\' and get() == 'u')) + { + const int codepoint2 = get_codepoint(); + + if (JSON_UNLIKELY(codepoint2 == -1)) + { + error_message = "invalid string: '\\u' must be followed by 4 hex digits"; + return token_type::parse_error; + } + + // check if codepoint2 is a low surrogate + if (JSON_LIKELY(0xDC00 <= codepoint2 and codepoint2 <= 0xDFFF)) + { + // overwrite codepoint + codepoint = + // high surrogate occupies the most significant 22 bits + (codepoint1 << 10) + // low surrogate occupies the least significant 15 bits + + codepoint2 + // there is still the 0xD800, 0xDC00 and 0x10000 noise + // in the result so we have to subtract with: + // (0xD800 << 10) + DC00 - 0x10000 = 0x35FDC00 + - 0x35FDC00; + } + else + { + error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + error_message = "invalid string: surrogate U+DC00..U+DFFF must be followed by U+DC00..U+DFFF"; + return token_type::parse_error; + } + } + else + { + if (JSON_UNLIKELY(0xDC00 <= codepoint1 and codepoint1 <= 0xDFFF)) + { + error_message = "invalid string: surrogate U+DC00..U+DFFF must follow U+D800..U+DBFF"; + return token_type::parse_error; + } + } + + // result of the above calculation yields a proper codepoint + assert(0x00 <= codepoint and codepoint <= 0x10FFFF); + + // translate codepoint into bytes + if (codepoint < 0x80) + { + // 1-byte characters: 0xxxxxxx (ASCII) + add(codepoint); + } + else if (codepoint <= 0x7FF) + { + // 2-byte characters: 110xxxxx 10xxxxxx + add(0xC0 | (codepoint >> 6)); + add(0x80 | (codepoint & 0x3F)); + } + else if (codepoint <= 0xFFFF) + { + // 3-byte characters: 1110xxxx 10xxxxxx 10xxxxxx + add(0xE0 | (codepoint >> 12)); + add(0x80 | ((codepoint >> 6) & 0x3F)); + add(0x80 | (codepoint & 0x3F)); + } + else + { + // 4-byte characters: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx + add(0xF0 | (codepoint >> 18)); + add(0x80 | ((codepoint >> 12) & 0x3F)); + add(0x80 | ((codepoint >> 6) & 0x3F)); + add(0x80 | (codepoint & 0x3F)); + } + + break; + } + + // other characters after escape + default: + error_message = "invalid string: forbidden character after backslash"; + return token_type::parse_error; + } + + break; + } + + // invalid control characters + case 0x00: + { + error_message = "invalid string: control character U+0000 (NUL) must be escaped to \\u0000"; + return token_type::parse_error; + } + + case 0x01: + { + error_message = "invalid string: control character U+0001 (SOH) must be escaped to \\u0001"; + return token_type::parse_error; + } + + case 0x02: + { + error_message = "invalid string: control character U+0002 (STX) must be escaped to \\u0002"; + return token_type::parse_error; + } + + case 0x03: + { + error_message = "invalid string: control character U+0003 (ETX) must be escaped to \\u0003"; + return token_type::parse_error; + } + + case 0x04: + { + error_message = "invalid string: control character U+0004 (EOT) must be escaped to \\u0004"; + return token_type::parse_error; + } + + case 0x05: + { + error_message = "invalid string: control character U+0005 (ENQ) must be escaped to \\u0005"; + return token_type::parse_error; + } + + case 0x06: + { + error_message = "invalid string: control character U+0006 (ACK) must be escaped to \\u0006"; + return token_type::parse_error; + } + + case 0x07: + { + error_message = "invalid string: control character U+0007 (BEL) must be escaped to \\u0007"; + return token_type::parse_error; + } + + case 0x08: + { + error_message = "invalid string: control character U+0008 (BS) must be escaped to \\u0008 or \\b"; + return token_type::parse_error; + } + + case 0x09: + { + error_message = "invalid string: control character U+0009 (HT) must be escaped to \\u0009 or \\t"; + return token_type::parse_error; + } + + case 0x0A: + { + error_message = "invalid string: control character U+000A (LF) must be escaped to \\u000A or \\n"; + return token_type::parse_error; + } + + case 0x0B: + { + error_message = "invalid string: control character U+000B (VT) must be escaped to \\u000B"; + return token_type::parse_error; + } + + case 0x0C: + { + error_message = "invalid string: control character U+000C (FF) must be escaped to \\u000C or \\f"; + return token_type::parse_error; + } + + case 0x0D: + { + error_message = "invalid string: control character U+000D (CR) must be escaped to \\u000D or \\r"; + return token_type::parse_error; + } + + case 0x0E: + { + error_message = "invalid string: control character U+000E (SO) must be escaped to \\u000E"; + return token_type::parse_error; + } + + case 0x0F: + { + error_message = "invalid string: control character U+000F (SI) must be escaped to \\u000F"; + return token_type::parse_error; + } + + case 0x10: + { + error_message = "invalid string: control character U+0010 (DLE) must be escaped to \\u0010"; + return token_type::parse_error; + } + + case 0x11: + { + error_message = "invalid string: control character U+0011 (DC1) must be escaped to \\u0011"; + return token_type::parse_error; + } + + case 0x12: + { + error_message = "invalid string: control character U+0012 (DC2) must be escaped to \\u0012"; + return token_type::parse_error; + } + + case 0x13: + { + error_message = "invalid string: control character U+0013 (DC3) must be escaped to \\u0013"; + return token_type::parse_error; + } + + case 0x14: + { + error_message = "invalid string: control character U+0014 (DC4) must be escaped to \\u0014"; + return token_type::parse_error; + } + + case 0x15: + { + error_message = "invalid string: control character U+0015 (NAK) must be escaped to \\u0015"; + return token_type::parse_error; + } + + case 0x16: + { + error_message = "invalid string: control character U+0016 (SYN) must be escaped to \\u0016"; + return token_type::parse_error; + } + + case 0x17: + { + error_message = "invalid string: control character U+0017 (ETB) must be escaped to \\u0017"; + return token_type::parse_error; + } + + case 0x18: + { + error_message = "invalid string: control character U+0018 (CAN) must be escaped to \\u0018"; + return token_type::parse_error; + } + + case 0x19: + { + error_message = "invalid string: control character U+0019 (EM) must be escaped to \\u0019"; + return token_type::parse_error; + } + + case 0x1A: + { + error_message = "invalid string: control character U+001A (SUB) must be escaped to \\u001A"; + return token_type::parse_error; + } + + case 0x1B: + { + error_message = "invalid string: control character U+001B (ESC) must be escaped to \\u001B"; + return token_type::parse_error; + } + + case 0x1C: + { + error_message = "invalid string: control character U+001C (FS) must be escaped to \\u001C"; + return token_type::parse_error; + } + + case 0x1D: + { + error_message = "invalid string: control character U+001D (GS) must be escaped to \\u001D"; + return token_type::parse_error; + } + + case 0x1E: + { + error_message = "invalid string: control character U+001E (RS) must be escaped to \\u001E"; + return token_type::parse_error; + } + + case 0x1F: + { + error_message = "invalid string: control character U+001F (US) must be escaped to \\u001F"; + return token_type::parse_error; + } + + // U+0020..U+007F (except U+0022 (quote) and U+005C (backspace)) + case 0x20: + case 0x21: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + { + add(current); + break; + } + + // U+0080..U+07FF: bytes C2..DF 80..BF + case 0xC2: + case 0xC3: + case 0xC4: + case 0xC5: + case 0xC6: + case 0xC7: + case 0xC8: + case 0xC9: + case 0xCA: + case 0xCB: + case 0xCC: + case 0xCD: + case 0xCE: + case 0xCF: + case 0xD0: + case 0xD1: + case 0xD2: + case 0xD3: + case 0xD4: + case 0xD5: + case 0xD6: + case 0xD7: + case 0xD8: + case 0xD9: + case 0xDA: + case 0xDB: + case 0xDC: + case 0xDD: + case 0xDE: + case 0xDF: + { + if (JSON_UNLIKELY(not next_byte_in_range({0x80, 0xBF}))) + { + return token_type::parse_error; + } + break; + } + + // U+0800..U+0FFF: bytes E0 A0..BF 80..BF + case 0xE0: + { + if (JSON_UNLIKELY(not (next_byte_in_range({0xA0, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+1000..U+CFFF: bytes E1..EC 80..BF 80..BF + // U+E000..U+FFFF: bytes EE..EF 80..BF 80..BF + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xEE: + case 0xEF: + { + if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+D000..U+D7FF: bytes ED 80..9F 80..BF + case 0xED: + { + if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x9F, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+10000..U+3FFFF F0 90..BF 80..BF 80..BF + case 0xF0: + { + if (JSON_UNLIKELY(not (next_byte_in_range({0x90, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF + case 0xF1: + case 0xF2: + case 0xF3: + { + if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0xBF, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // U+100000..U+10FFFF F4 80..8F 80..BF 80..BF + case 0xF4: + { + if (JSON_UNLIKELY(not (next_byte_in_range({0x80, 0x8F, 0x80, 0xBF, 0x80, 0xBF})))) + { + return token_type::parse_error; + } + break; + } + + // remaining bytes (80..C1 and F5..FF) are ill-formed + default: + { + error_message = "invalid string: ill-formed UTF-8 byte"; + return token_type::parse_error; + } + } + } + } + + static void strtof(float& f, const char* str, char** endptr) noexcept + { + f = std::strtof(str, endptr); + } + + static void strtof(double& f, const char* str, char** endptr) noexcept + { + f = std::strtod(str, endptr); + } + + static void strtof(long double& f, const char* str, char** endptr) noexcept + { + f = std::strtold(str, endptr); + } + + /*! + @brief scan a number literal + + This function scans a string according to Sect. 6 of RFC 7159. + + The function is realized with a deterministic finite state machine derived + from the grammar described in RFC 7159. Starting in state "init", the + input is read and used to determined the next state. Only state "done" + accepts the number. State "error" is a trap state to model errors. In the + table below, "anything" means any character but the ones listed before. + + state | 0 | 1-9 | e E | + | - | . | anything + ---------|----------|----------|----------|---------|---------|----------|----------- + init | zero | any1 | [error] | [error] | minus | [error] | [error] + minus | zero | any1 | [error] | [error] | [error] | [error] | [error] + zero | done | done | exponent | done | done | decimal1 | done + any1 | any1 | any1 | exponent | done | done | decimal1 | done + decimal1 | decimal2 | [error] | [error] | [error] | [error] | [error] | [error] + decimal2 | decimal2 | decimal2 | exponent | done | done | done | done + exponent | any2 | any2 | [error] | sign | sign | [error] | [error] + sign | any2 | any2 | [error] | [error] | [error] | [error] | [error] + any2 | any2 | any2 | done | done | done | done | done + + The state machine is realized with one label per state (prefixed with + "scan_number_") and `goto` statements between them. The state machine + contains cycles, but any cycle can be left when EOF is read. Therefore, + the function is guaranteed to terminate. + + During scanning, the read bytes are stored in token_buffer. This string is + then converted to a signed integer, an unsigned integer, or a + floating-point number. + + @return token_type::value_unsigned, token_type::value_integer, or + token_type::value_float if number could be successfully scanned, + token_type::parse_error otherwise + + @note The scanner is independent of the current locale. Internally, the + locale's decimal point is used instead of `.` to work with the + locale-dependent converters. + */ + token_type scan_number() // lgtm [cpp/use-of-goto] + { + // reset token_buffer to store the number's bytes + reset(); + + // the type of the parsed number; initially set to unsigned; will be + // changed if minus sign, decimal point or exponent is read + token_type number_type = token_type::value_unsigned; + + // state (init): we just found out we need to scan a number + switch (current) + { + case '-': + { + add(current); + goto scan_number_minus; + } + + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + // LCOV_EXCL_START + default: + { + // all other characters are rejected outside scan_number() + assert(false); + } + // LCOV_EXCL_STOP + } + +scan_number_minus: + // state: we just parsed a leading minus sign + number_type = token_type::value_integer; + switch (get()) + { + case '0': + { + add(current); + goto scan_number_zero; + } + + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + default: + { + error_message = "invalid number; expected digit after '-'"; + return token_type::parse_error; + } + } + +scan_number_zero: + // state: we just parse a zero (maybe with a leading minus sign) + switch (get()) + { + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_any1: + // state: we just parsed a number 0-9 (maybe with a leading minus sign) + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any1; + } + + case '.': + { + add(decimal_point_char); + goto scan_number_decimal1; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_decimal1: + // state: we just parsed a decimal point + number_type = token_type::value_float; + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + default: + { + error_message = "invalid number; expected digit after '.'"; + return token_type::parse_error; + } + } + +scan_number_decimal2: + // we just parsed at least one number after a decimal point + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_decimal2; + } + + case 'e': + case 'E': + { + add(current); + goto scan_number_exponent; + } + + default: + goto scan_number_done; + } + +scan_number_exponent: + // we just parsed an exponent + number_type = token_type::value_float; + switch (get()) + { + case '+': + case '-': + { + add(current); + goto scan_number_sign; + } + + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = + "invalid number; expected '+', '-', or digit after exponent"; + return token_type::parse_error; + } + } + +scan_number_sign: + // we just parsed an exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + { + error_message = "invalid number; expected digit after exponent sign"; + return token_type::parse_error; + } + } + +scan_number_any2: + // we just parsed a number after the exponent or exponent sign + switch (get()) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + add(current); + goto scan_number_any2; + } + + default: + goto scan_number_done; + } + +scan_number_done: + // unget the character after the number (we only read it to know that + // we are done scanning a number) + unget(); + + char* endptr = nullptr; + errno = 0; + + // try to parse integers first and fall back to floats + if (number_type == token_type::value_unsigned) + { + const auto x = std::strtoull(token_buffer.data(), &endptr, 10); + + // we checked the number format before + assert(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_unsigned = static_cast<number_unsigned_t>(x); + if (value_unsigned == x) + { + return token_type::value_unsigned; + } + } + } + else if (number_type == token_type::value_integer) + { + const auto x = std::strtoll(token_buffer.data(), &endptr, 10); + + // we checked the number format before + assert(endptr == token_buffer.data() + token_buffer.size()); + + if (errno == 0) + { + value_integer = static_cast<number_integer_t>(x); + if (value_integer == x) + { + return token_type::value_integer; + } + } + } + + // this code is reached if we parse a floating-point number or if an + // integer conversion above failed + strtof(value_float, token_buffer.data(), &endptr); + + // we checked the number format before + assert(endptr == token_buffer.data() + token_buffer.size()); + + return token_type::value_float; + } + + /*! + @param[in] literal_text the literal text to expect + @param[in] length the length of the passed literal text + @param[in] return_type the token type to return on success + */ + token_type scan_literal(const char* literal_text, const std::size_t length, + token_type return_type) + { + assert(current == literal_text[0]); + for (std::size_t i = 1; i < length; ++i) + { + if (JSON_UNLIKELY(get() != literal_text[i])) + { + error_message = "invalid literal"; + return token_type::parse_error; + } + } + return return_type; + } + + ///////////////////// + // input management + ///////////////////// + + /// reset token_buffer; current character is beginning of token + void reset() noexcept + { + token_buffer.clear(); + token_string.clear(); + token_string.push_back(std::char_traits<char>::to_char_type(current)); + } + + /* + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a + `std::char_traits<char>::eof()` in that case. Stores the scanned characters + for use in error messages. + + @return character read from the input + */ + std::char_traits<char>::int_type get() + { + ++position.chars_read_total; + ++position.chars_read_current_line; + + if (next_unget) + { + // just reset the next_unget variable and work with current + next_unget = false; + } + else + { + current = ia->get_character(); + } + + if (JSON_LIKELY(current != std::char_traits<char>::eof())) + { + token_string.push_back(std::char_traits<char>::to_char_type(current)); + } + + if (current == '\n') + { + ++position.lines_read; + ++position.chars_read_current_line = 0; + } + + return current; + } + + /*! + @brief unget current character (read it again on next get) + + We implement unget by setting variable next_unget to true. The input is not + changed - we just simulate ungetting by modifying chars_read_total, + chars_read_current_line, and token_string. The next call to get() will + behave as if the unget character is read again. + */ + void unget() + { + next_unget = true; + + --position.chars_read_total; + + // in case we "unget" a newline, we have to also decrement the lines_read + if (position.chars_read_current_line == 0) + { + if (position.lines_read > 0) + { + --position.lines_read; + } + } + else + { + --position.chars_read_current_line; + } + + if (JSON_LIKELY(current != std::char_traits<char>::eof())) + { + assert(token_string.size() != 0); + token_string.pop_back(); + } + } + + /// add a character to token_buffer + void add(int c) + { + token_buffer.push_back(std::char_traits<char>::to_char_type(c)); + } + + public: + ///////////////////// + // value getters + ///////////////////// + + /// return integer value + constexpr number_integer_t get_number_integer() const noexcept + { + return value_integer; + } + + /// return unsigned integer value + constexpr number_unsigned_t get_number_unsigned() const noexcept + { + return value_unsigned; + } + + /// return floating-point value + constexpr number_float_t get_number_float() const noexcept + { + return value_float; + } + + /// return current string value (implicitly resets the token; useful only once) + string_t& get_string() + { + return token_buffer; + } + + ///////////////////// + // diagnostics + ///////////////////// + + /// return position of last read token + constexpr position_t get_position() const noexcept + { + return position; + } + + /// return the last read token (for errors only). Will never contain EOF + /// (an arbitrary value that is not a valid char value, often -1), because + /// 255 may legitimately occur. May contain NUL, which should be escaped. + std::string get_token_string() const + { + // escape control characters + std::string result; + for (const auto c : token_string) + { + if ('\x00' <= c and c <= '\x1F') + { + // escape control characters + char cs[9]; + (std::snprintf)(cs, 9, "<U+%.4X>", static_cast<unsigned char>(c)); + result += cs; + } + else + { + // add character as is + result.push_back(c); + } + } + + return result; + } + + /// return syntax error message + constexpr const char* get_error_message() const noexcept + { + return error_message; + } + + ///////////////////// + // actual scanner + ///////////////////// + + /*! + @brief skip the UTF-8 byte order mark + @return true iff there is no BOM or the correct BOM has been skipped + */ + bool skip_bom() + { + if (get() == 0xEF) + { + // check if we completely parse the BOM + return get() == 0xBB and get() == 0xBF; + } + + // the first character is not the beginning of the BOM; unget it to + // process is later + unget(); + return true; + } + + token_type scan() + { + // initially, skip the BOM + if (position.chars_read_total == 0 and not skip_bom()) + { + error_message = "invalid BOM; must be 0xEF 0xBB 0xBF if given"; + return token_type::parse_error; + } + + // read next character and ignore whitespace + do + { + get(); + } + while (current == ' ' or current == '\t' or current == '\n' or current == '\r'); + + switch (current) + { + // structural characters + case '[': + return token_type::begin_array; + case ']': + return token_type::end_array; + case '{': + return token_type::begin_object; + case '}': + return token_type::end_object; + case ':': + return token_type::name_separator; + case ',': + return token_type::value_separator; + + // literals + case 't': + return scan_literal("true", 4, token_type::literal_true); + case 'f': + return scan_literal("false", 5, token_type::literal_false); + case 'n': + return scan_literal("null", 4, token_type::literal_null); + + // string + case '\"': + return scan_string(); + + // number + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return scan_number(); + + // end of input (the null byte is needed when parsing from + // string literals) + case '\0': + case std::char_traits<char>::eof(): + return token_type::end_of_input; + + // error + default: + error_message = "invalid literal"; + return token_type::parse_error; + } + } + + private: + /// input adapter + detail::input_adapter_t ia = nullptr; + + /// the current character + std::char_traits<char>::int_type current = std::char_traits<char>::eof(); + + /// whether the next get() call should just return current + bool next_unget = false; + + /// the start position of the current token + position_t position; + + /// raw input token string (for error messages) + std::vector<char> token_string {}; + + /// buffer for variable-length tokens (numbers, strings) + string_t token_buffer {}; + + /// a description of occurred lexer errors + const char* error_message = ""; + + // number values + number_integer_t value_integer = 0; + number_unsigned_t value_unsigned = 0; + number_float_t value_float = 0; + + /// the decimal point + const char decimal_point_char = '.'; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/parser.hpp> + + +#include <cassert> // assert +#include <cmath> // isfinite +#include <cstdint> // uint8_t +#include <functional> // function +#include <string> // string +#include <utility> // move + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + + +#include <cstdint> // size_t +#include <utility> // declval + +// #include <nlohmann/detail/meta/detected.hpp> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template <typename T> +using null_function_t = decltype(std::declval<T&>().null()); + +template <typename T> +using boolean_function_t = + decltype(std::declval<T&>().boolean(std::declval<bool>())); + +template <typename T, typename Integer> +using number_integer_function_t = + decltype(std::declval<T&>().number_integer(std::declval<Integer>())); + +template <typename T, typename Unsigned> +using number_unsigned_function_t = + decltype(std::declval<T&>().number_unsigned(std::declval<Unsigned>())); + +template <typename T, typename Float, typename String> +using number_float_function_t = decltype(std::declval<T&>().number_float( + std::declval<Float>(), std::declval<const String&>())); + +template <typename T, typename String> +using string_function_t = + decltype(std::declval<T&>().string(std::declval<String&>())); + +template <typename T> +using start_object_function_t = + decltype(std::declval<T&>().start_object(std::declval<std::size_t>())); + +template <typename T, typename String> +using key_function_t = + decltype(std::declval<T&>().key(std::declval<String&>())); + +template <typename T> +using end_object_function_t = decltype(std::declval<T&>().end_object()); + +template <typename T> +using start_array_function_t = + decltype(std::declval<T&>().start_array(std::declval<std::size_t>())); + +template <typename T> +using end_array_function_t = decltype(std::declval<T&>().end_array()); + +template <typename T, typename Exception> +using parse_error_function_t = decltype(std::declval<T&>().parse_error( + std::declval<std::size_t>(), std::declval<const std::string&>(), + std::declval<const Exception&>())); + +template <typename SAX, typename BasicJsonType> +struct is_sax +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static constexpr bool value = + is_detected_exact<bool, null_function_t, SAX>::value && + is_detected_exact<bool, boolean_function_t, SAX>::value && + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value && + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value && + is_detected_exact<bool, number_float_function_t, SAX, number_float_t, + string_t>::value && + is_detected_exact<bool, string_function_t, SAX, string_t>::value && + is_detected_exact<bool, start_object_function_t, SAX>::value && + is_detected_exact<bool, key_function_t, SAX, string_t>::value && + is_detected_exact<bool, end_object_function_t, SAX>::value && + is_detected_exact<bool, start_array_function_t, SAX>::value && + is_detected_exact<bool, end_array_function_t, SAX>::value && + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value; +}; + +template <typename SAX, typename BasicJsonType> +struct is_sax_static_asserts +{ + private: + static_assert(is_basic_json<BasicJsonType>::value, + "BasicJsonType must be of type basic_json<...>"); + + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using exception_t = typename BasicJsonType::exception; + + public: + static_assert(is_detected_exact<bool, null_function_t, SAX>::value, + "Missing/invalid function: bool null()"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert(is_detected_exact<bool, boolean_function_t, SAX>::value, + "Missing/invalid function: bool boolean(bool)"); + static_assert( + is_detected_exact<bool, number_integer_function_t, SAX, + number_integer_t>::value, + "Missing/invalid function: bool number_integer(number_integer_t)"); + static_assert( + is_detected_exact<bool, number_unsigned_function_t, SAX, + number_unsigned_t>::value, + "Missing/invalid function: bool number_unsigned(number_unsigned_t)"); + static_assert(is_detected_exact<bool, number_float_function_t, SAX, + number_float_t, string_t>::value, + "Missing/invalid function: bool number_float(number_float_t, const string_t&)"); + static_assert( + is_detected_exact<bool, string_function_t, SAX, string_t>::value, + "Missing/invalid function: bool string(string_t&)"); + static_assert(is_detected_exact<bool, start_object_function_t, SAX>::value, + "Missing/invalid function: bool start_object(std::size_t)"); + static_assert(is_detected_exact<bool, key_function_t, SAX, string_t>::value, + "Missing/invalid function: bool key(string_t&)"); + static_assert(is_detected_exact<bool, end_object_function_t, SAX>::value, + "Missing/invalid function: bool end_object()"); + static_assert(is_detected_exact<bool, start_array_function_t, SAX>::value, + "Missing/invalid function: bool start_array(std::size_t)"); + static_assert(is_detected_exact<bool, end_array_function_t, SAX>::value, + "Missing/invalid function: bool end_array()"); + static_assert( + is_detected_exact<bool, parse_error_function_t, SAX, exception_t>::value, + "Missing/invalid function: bool parse_error(std::size_t, const " + "std::string&, const exception&)"); +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + + +#include <cstddef> +#include <string> +#include <vector> + +// #include <nlohmann/detail/input/parser.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + + +namespace nlohmann +{ + +/*! +@brief SAX interface + +This class describes the SAX interface used by @ref nlohmann::json::sax_parse. +Each function is called in different situations while the input is parsed. The +boolean return value informs the parser whether to continue processing the +input. +*/ +template<typename BasicJsonType> +struct json_sax +{ + /// type for (signed) integers + using number_integer_t = typename BasicJsonType::number_integer_t; + /// type for unsigned integers + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + /// type for floating-point numbers + using number_float_t = typename BasicJsonType::number_float_t; + /// type for strings + using string_t = typename BasicJsonType::string_t; + + /*! + @brief a null value was read + @return whether parsing should proceed + */ + virtual bool null() = 0; + + /*! + @brief a boolean value was read + @param[in] val boolean value + @return whether parsing should proceed + */ + virtual bool boolean(bool val) = 0; + + /*! + @brief an integer number was read + @param[in] val integer value + @return whether parsing should proceed + */ + virtual bool number_integer(number_integer_t val) = 0; + + /*! + @brief an unsigned integer number was read + @param[in] val unsigned integer value + @return whether parsing should proceed + */ + virtual bool number_unsigned(number_unsigned_t val) = 0; + + /*! + @brief an floating-point number was read + @param[in] val floating-point value + @param[in] s raw token value + @return whether parsing should proceed + */ + virtual bool number_float(number_float_t val, const string_t& s) = 0; + + /*! + @brief a string was read + @param[in] val string value + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool string(string_t& val) = 0; + + /*! + @brief the beginning of an object was read + @param[in] elements number of object elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_object(std::size_t elements) = 0; + + /*! + @brief an object key was read + @param[in] val object key + @return whether parsing should proceed + @note It is safe to move the passed string. + */ + virtual bool key(string_t& val) = 0; + + /*! + @brief the end of an object was read + @return whether parsing should proceed + */ + virtual bool end_object() = 0; + + /*! + @brief the beginning of an array was read + @param[in] elements number of array elements or -1 if unknown + @return whether parsing should proceed + @note binary formats may report the number of elements + */ + virtual bool start_array(std::size_t elements) = 0; + + /*! + @brief the end of an array was read + @return whether parsing should proceed + */ + virtual bool end_array() = 0; + + /*! + @brief a parse error occurred + @param[in] position the position in the input where the error occurs + @param[in] last_token the last read token + @param[in] ex an exception object describing the error + @return whether parsing should proceed (must return false) + */ + virtual bool parse_error(std::size_t position, + const std::string& last_token, + const detail::exception& ex) = 0; + + virtual ~json_sax() = default; +}; + + +namespace detail +{ +/*! +@brief SAX implementation to create a JSON value from SAX events + +This class implements the @ref json_sax interface and processes the SAX events +to create a JSON value which makes it basically a DOM parser. The structure or +hierarchy of the JSON value is managed by the stack `ref_stack` which contains +a pointer to the respective array or object for each recursion depth. + +After successful parsing, the value that is passed by reference to the +constructor contains the parsed value. + +@tparam BasicJsonType the JSON type +*/ +template<typename BasicJsonType> +class json_sax_dom_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + /*! + @param[in, out] r reference to a JSON value that is manipulated while + parsing + @param[in] allow_exceptions_ whether parse errors yield exceptions + */ + explicit json_sax_dom_parser(BasicJsonType& r, const bool allow_exceptions_ = true) + : root(r), allow_exceptions(allow_exceptions_) + {} + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::object)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + + return true; + } + + bool key(string_t& val) + { + // add null at given key and store the reference for later + object_element = &(ref_stack.back()->m_value.object->operator[](val)); + return true; + } + + bool end_object() + { + ref_stack.pop_back(); + return true; + } + + bool start_array(std::size_t len) + { + ref_stack.push_back(handle_value(BasicJsonType::value_t::array)); + + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + + return true; + } + + bool end_array() + { + ref_stack.pop_back(); + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + */ + template<typename Value> + BasicJsonType* handle_value(Value&& v) + { + if (ref_stack.empty()) + { + root = BasicJsonType(std::forward<Value>(v)); + return &root; + } + + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->emplace_back(std::forward<Value>(v)); + return &(ref_stack.back()->m_value.array->back()); + } + else + { + assert(object_element); + *object_element = BasicJsonType(std::forward<Value>(v)); + return object_element; + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; + +template<typename BasicJsonType> +class json_sax_dom_callback_parser +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using parser_callback_t = typename BasicJsonType::parser_callback_t; + using parse_event_t = typename BasicJsonType::parse_event_t; + + json_sax_dom_callback_parser(BasicJsonType& r, + const parser_callback_t cb, + const bool allow_exceptions_ = true) + : root(r), callback(cb), allow_exceptions(allow_exceptions_) + { + keep_stack.push_back(true); + } + + bool null() + { + handle_value(nullptr); + return true; + } + + bool boolean(bool val) + { + handle_value(val); + return true; + } + + bool number_integer(number_integer_t val) + { + handle_value(val); + return true; + } + + bool number_unsigned(number_unsigned_t val) + { + handle_value(val); + return true; + } + + bool number_float(number_float_t val, const string_t& /*unused*/) + { + handle_value(val); + return true; + } + + bool string(string_t& val) + { + handle_value(val); + return true; + } + + bool start_object(std::size_t len) + { + // check callback for object start + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::object_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::object, true); + ref_stack.push_back(val.second); + + // check object limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive object size: " + std::to_string(len))); + } + } + + return true; + } + + bool key(string_t& val) + { + BasicJsonType k = BasicJsonType(val); + + // check callback for key + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::key, k); + key_keep_stack.push_back(keep); + + // add discarded value at given key and store the reference for later + if (keep and ref_stack.back()) + { + object_element = &(ref_stack.back()->m_value.object->operator[](val) = discarded); + } + + return true; + } + + bool end_object() + { + if (ref_stack.back()) + { + if (not callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::object_end, *ref_stack.back())) + { + // discard object + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + if (not ref_stack.empty() and ref_stack.back()) + { + // remove discarded value + if (ref_stack.back()->is_object()) + { + for (auto it = ref_stack.back()->begin(); it != ref_stack.back()->end(); ++it) + { + if (it->is_discarded()) + { + ref_stack.back()->erase(it); + break; + } + } + } + } + + return true; + } + + bool start_array(std::size_t len) + { + const bool keep = callback(static_cast<int>(ref_stack.size()), parse_event_t::array_start, discarded); + keep_stack.push_back(keep); + + auto val = handle_value(BasicJsonType::value_t::array, true); + ref_stack.push_back(val.second); + + // check array limit + if (ref_stack.back()) + { + if (JSON_UNLIKELY(len != std::size_t(-1) and len > ref_stack.back()->max_size())) + { + JSON_THROW(out_of_range::create(408, + "excessive array size: " + std::to_string(len))); + } + } + + return true; + } + + bool end_array() + { + bool keep = true; + + if (ref_stack.back()) + { + keep = callback(static_cast<int>(ref_stack.size()) - 1, parse_event_t::array_end, *ref_stack.back()); + if (not keep) + { + // discard array + *ref_stack.back() = discarded; + } + } + + assert(not ref_stack.empty()); + assert(not keep_stack.empty()); + ref_stack.pop_back(); + keep_stack.pop_back(); + + // remove discarded value + if (not keep and not ref_stack.empty()) + { + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->pop_back(); + } + } + + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, + const detail::exception& ex) + { + errored = true; + if (allow_exceptions) + { + // determine the proper exception type from the id + switch ((ex.id / 100) % 100) + { + case 1: + JSON_THROW(*reinterpret_cast<const detail::parse_error*>(&ex)); + case 4: + JSON_THROW(*reinterpret_cast<const detail::out_of_range*>(&ex)); + // LCOV_EXCL_START + case 2: + JSON_THROW(*reinterpret_cast<const detail::invalid_iterator*>(&ex)); + case 3: + JSON_THROW(*reinterpret_cast<const detail::type_error*>(&ex)); + case 5: + JSON_THROW(*reinterpret_cast<const detail::other_error*>(&ex)); + default: + assert(false); + // LCOV_EXCL_STOP + } + } + return false; + } + + constexpr bool is_errored() const + { + return errored; + } + + private: + /*! + @param[in] v value to add to the JSON value we build during parsing + @param[in] skip_callback whether we should skip calling the callback + function; this is required after start_array() and + start_object() SAX events, because otherwise we would call the + callback function with an empty array or object, respectively. + + @invariant If the ref stack is empty, then the passed value will be the new + root. + @invariant If the ref stack contains a value, then it is an array or an + object to which we can add elements + + @return pair of boolean (whether value should be kept) and pointer (to the + passed value in the ref_stack hierarchy; nullptr if not kept) + */ + template<typename Value> + std::pair<bool, BasicJsonType*> handle_value(Value&& v, const bool skip_callback = false) + { + assert(not keep_stack.empty()); + + // do not handle this value if we know it would be added to a discarded + // container + if (not keep_stack.back()) + { + return {false, nullptr}; + } + + // create value + auto value = BasicJsonType(std::forward<Value>(v)); + + // check callback + const bool keep = skip_callback or callback(static_cast<int>(ref_stack.size()), parse_event_t::value, value); + + // do not handle this value if we just learnt it shall be discarded + if (not keep) + { + return {false, nullptr}; + } + + if (ref_stack.empty()) + { + root = std::move(value); + return {true, &root}; + } + + // skip this value if we already decided to skip the parent + // (https://github.com/nlohmann/json/issues/971#issuecomment-413678360) + if (not ref_stack.back()) + { + return {false, nullptr}; + } + + // we now only expect arrays and objects + assert(ref_stack.back()->is_array() or ref_stack.back()->is_object()); + + if (ref_stack.back()->is_array()) + { + ref_stack.back()->m_value.array->push_back(std::move(value)); + return {true, &(ref_stack.back()->m_value.array->back())}; + } + else + { + // check if we should store an element for the current key + assert(not key_keep_stack.empty()); + const bool store_element = key_keep_stack.back(); + key_keep_stack.pop_back(); + + if (not store_element) + { + return {false, nullptr}; + } + + assert(object_element); + *object_element = std::move(value); + return {true, object_element}; + } + } + + /// the parsed JSON value + BasicJsonType& root; + /// stack to model hierarchy of values + std::vector<BasicJsonType*> ref_stack; + /// stack to manage which values to keep + std::vector<bool> keep_stack; + /// stack to manage which object keys to keep + std::vector<bool> key_keep_stack; + /// helper to hold the reference for the next object element + BasicJsonType* object_element = nullptr; + /// whether a syntax error occurred + bool errored = false; + /// callback function + const parser_callback_t callback = nullptr; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; + /// a discarded value for the callback + BasicJsonType discarded = BasicJsonType::value_t::discarded; +}; + +template<typename BasicJsonType> +class json_sax_acceptor +{ + public: + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + + bool null() + { + return true; + } + + bool boolean(bool /*unused*/) + { + return true; + } + + bool number_integer(number_integer_t /*unused*/) + { + return true; + } + + bool number_unsigned(number_unsigned_t /*unused*/) + { + return true; + } + + bool number_float(number_float_t /*unused*/, const string_t& /*unused*/) + { + return true; + } + + bool string(string_t& /*unused*/) + { + return true; + } + + bool start_object(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool key(string_t& /*unused*/) + { + return true; + } + + bool end_object() + { + return true; + } + + bool start_array(std::size_t /*unused*/ = std::size_t(-1)) + { + return true; + } + + bool end_array() + { + return true; + } + + bool parse_error(std::size_t /*unused*/, const std::string& /*unused*/, const detail::exception& /*unused*/) + { + return false; + } +}; +} // namespace detail + +} // namespace nlohmann + +// #include <nlohmann/detail/input/lexer.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +//////////// +// parser // +//////////// + +/*! +@brief syntax analysis + +This class implements a recursive decent parser. +*/ +template<typename BasicJsonType> +class parser +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using lexer_t = lexer<BasicJsonType>; + using token_type = typename lexer_t::token_type; + + public: + enum class parse_event_t : uint8_t + { + /// the parser read `{` and started to process a JSON object + object_start, + /// the parser read `}` and finished processing a JSON object + object_end, + /// the parser read `[` and started to process a JSON array + array_start, + /// the parser read `]` and finished processing a JSON array + array_end, + /// the parser read a key of a value in an object + key, + /// the parser finished reading a JSON value + value + }; + + using parser_callback_t = + std::function<bool(int depth, parse_event_t event, BasicJsonType& parsed)>; + + /// a parser reading from an input adapter + explicit parser(detail::input_adapter_t&& adapter, + const parser_callback_t cb = nullptr, + const bool allow_exceptions_ = true) + : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_) + { + // read first token + get_token(); + } + + /*! + @brief public parser interface + + @param[in] strict whether to expect the last token to be EOF + @param[in,out] result parsed JSON value + + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + */ + void parse(const bool strict, BasicJsonType& result) + { + if (callback) + { + json_sax_dom_callback_parser<BasicJsonType> sdp(result, callback, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + + // set top-level value to null if it was discarded by the callback + // function + if (result.is_discarded()) + { + result = nullptr; + } + } + else + { + json_sax_dom_parser<BasicJsonType> sdp(result, allow_exceptions); + sax_parse_internal(&sdp); + result.assert_invariant(); + + // in strict mode, input must be completely read + if (strict and (get_token() != token_type::end_of_input)) + { + sdp.parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } + + // in case of an error, return discarded value + if (sdp.is_errored()) + { + result = value_t::discarded; + return; + } + } + } + + /*! + @brief public accept interface + + @param[in] strict whether to expect the last token to be EOF + @return whether the input is a proper JSON text + */ + bool accept(const bool strict = true) + { + json_sax_acceptor<BasicJsonType> sax_acceptor; + return sax_parse(&sax_acceptor, strict); + } + + template <typename SAX> + bool sax_parse(SAX* sax, const bool strict = true) + { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; + const bool result = sax_parse_internal(sax); + + // strict mode: next byte must be EOF + if (result and strict and (get_token() != token_type::end_of_input)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_of_input, "value"))); + } + + return result; + } + + private: + template <typename SAX> + bool sax_parse_internal(SAX* sax) + { + // stack to remember the hierarchy of structured values we are parsing + // true = array; false = object + std::vector<bool> states; + // value to avoid a goto (see comment where set to true) + bool skip_to_state_evaluation = false; + + while (true) + { + if (not skip_to_state_evaluation) + { + // invariant: get_token() was called before each iteration + switch (last_token) + { + case token_type::begin_object: + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + // closing } -> we are done + if (get_token() == token_type::end_object) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + break; + } + + // parse key + if (JSON_UNLIKELY(last_token != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); + } + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } + + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); + } + + // remember we are now inside an object + states.push_back(false); + + // parse values + get_token(); + continue; + } + + case token_type::begin_array: + { + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + // closing ] -> we are done + if (get_token() == token_type::end_array) + { + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } + break; + } + + // remember we are now inside an array + states.push_back(true); + + // parse values (no need to call get_token) + continue; + } + + case token_type::value_float: + { + const auto res = m_lexer.get_number_float(); + + if (JSON_UNLIKELY(not std::isfinite(res))) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + out_of_range::create(406, "number overflow parsing '" + m_lexer.get_token_string() + "'")); + } + else + { + if (JSON_UNLIKELY(not sax->number_float(res, m_lexer.get_string()))) + { + return false; + } + break; + } + } + + case token_type::literal_false: + { + if (JSON_UNLIKELY(not sax->boolean(false))) + { + return false; + } + break; + } + + case token_type::literal_null: + { + if (JSON_UNLIKELY(not sax->null())) + { + return false; + } + break; + } + + case token_type::literal_true: + { + if (JSON_UNLIKELY(not sax->boolean(true))) + { + return false; + } + break; + } + + case token_type::value_integer: + { + if (JSON_UNLIKELY(not sax->number_integer(m_lexer.get_number_integer()))) + { + return false; + } + break; + } + + case token_type::value_string: + { + if (JSON_UNLIKELY(not sax->string(m_lexer.get_string()))) + { + return false; + } + break; + } + + case token_type::value_unsigned: + { + if (JSON_UNLIKELY(not sax->number_unsigned(m_lexer.get_number_unsigned()))) + { + return false; + } + break; + } + + case token_type::parse_error: + { + // using "uninitialized" to avoid "expected" message + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::uninitialized, "value"))); + } + + default: // the last token was unexpected + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::literal_or_value, "value"))); + } + } + } + else + { + skip_to_state_evaluation = false; + } + + // we reached this line after we successfully parsed a value + if (states.empty()) + { + // empty stack: we reached the end of the hierarchy: done + return true; + } + else + { + if (states.back()) // array + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse a new value + get_token(); + continue; + } + + // closing ] + if (JSON_LIKELY(last_token == token_type::end_array)) + { + if (JSON_UNLIKELY(not sax->end_array())) + { + return false; + } + + // We are done with this array. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + else + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_array, "array"))); + } + } + else // object + { + // comma -> next value + if (get_token() == token_type::value_separator) + { + // parse key + if (JSON_UNLIKELY(get_token() != token_type::value_string)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::value_string, "object key"))); + } + else + { + if (JSON_UNLIKELY(not sax->key(m_lexer.get_string()))) + { + return false; + } + } + + // parse separator (:) + if (JSON_UNLIKELY(get_token() != token_type::name_separator)) + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::name_separator, "object separator"))); + } + + // parse values + get_token(); + continue; + } + + // closing } + if (JSON_LIKELY(last_token == token_type::end_object)) + { + if (JSON_UNLIKELY(not sax->end_object())) + { + return false; + } + + // We are done with this object. Before we can parse a + // new value, we need to evaluate the new state first. + // By setting skip_to_state_evaluation to false, we + // are effectively jumping to the beginning of this if. + assert(not states.empty()); + states.pop_back(); + skip_to_state_evaluation = true; + continue; + } + else + { + return sax->parse_error(m_lexer.get_position(), + m_lexer.get_token_string(), + parse_error::create(101, m_lexer.get_position(), + exception_message(token_type::end_object, "object"))); + } + } + } + } + } + + /// get next token from lexer + token_type get_token() + { + return (last_token = m_lexer.scan()); + } + + std::string exception_message(const token_type expected, const std::string& context) + { + std::string error_msg = "syntax error "; + + if (not context.empty()) + { + error_msg += "while parsing " + context + " "; + } + + error_msg += "- "; + + if (last_token == token_type::parse_error) + { + error_msg += std::string(m_lexer.get_error_message()) + "; last read: '" + + m_lexer.get_token_string() + "'"; + } + else + { + error_msg += "unexpected " + std::string(lexer_t::token_type_name(last_token)); + } + + if (expected != token_type::uninitialized) + { + error_msg += "; expected " + std::string(lexer_t::token_type_name(expected)); + } + + return error_msg; + } + + private: + /// callback function + const parser_callback_t callback = nullptr; + /// the type of the last read token + token_type last_token = token_type::uninitialized; + /// the lexer + lexer_t m_lexer; + /// whether to throw exceptions in case of errors + const bool allow_exceptions = true; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + + +#include <cstddef> // ptrdiff_t +#include <limits> // numeric_limits + +namespace nlohmann +{ +namespace detail +{ +/* +@brief an iterator for primitive JSON types + +This class models an iterator for primitive JSON types (boolean, number, +string). It's only purpose is to allow the iterator/const_iterator classes +to "iterate" over primitive values. Internally, the iterator is modeled by +a `difference_type` variable. Value begin_value (`0`) models the begin, +end_value (`1`) models past the end. +*/ +class primitive_iterator_t +{ + private: + using difference_type = std::ptrdiff_t; + static constexpr difference_type begin_value = 0; + static constexpr difference_type end_value = begin_value + 1; + + /// iterator as signed integer type + difference_type m_it = (std::numeric_limits<std::ptrdiff_t>::min)(); + + public: + constexpr difference_type get_value() const noexcept + { + return m_it; + } + + /// set iterator to a defined beginning + void set_begin() noexcept + { + m_it = begin_value; + } + + /// set iterator to a defined past the end + void set_end() noexcept + { + m_it = end_value; + } + + /// return whether the iterator can be dereferenced + constexpr bool is_begin() const noexcept + { + return m_it == begin_value; + } + + /// return whether the iterator is at end + constexpr bool is_end() const noexcept + { + return m_it == end_value; + } + + friend constexpr bool operator==(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it == rhs.m_it; + } + + friend constexpr bool operator<(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it < rhs.m_it; + } + + primitive_iterator_t operator+(difference_type n) noexcept + { + auto result = *this; + result += n; + return result; + } + + friend constexpr difference_type operator-(primitive_iterator_t lhs, primitive_iterator_t rhs) noexcept + { + return lhs.m_it - rhs.m_it; + } + + primitive_iterator_t& operator++() noexcept + { + ++m_it; + return *this; + } + + primitive_iterator_t const operator++(int) noexcept + { + auto result = *this; + ++m_it; + return result; + } + + primitive_iterator_t& operator--() noexcept + { + --m_it; + return *this; + } + + primitive_iterator_t const operator--(int) noexcept + { + auto result = *this; + --m_it; + return result; + } + + primitive_iterator_t& operator+=(difference_type n) noexcept + { + m_it += n; + return *this; + } + + primitive_iterator_t& operator-=(difference_type n) noexcept + { + m_it -= n; + return *this; + } +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/iterators/internal_iterator.hpp> + + +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/*! +@brief an iterator value + +@note This structure could easily be a union, but MSVC currently does not allow +unions members with complex constructors, see https://github.com/nlohmann/json/pull/105. +*/ +template<typename BasicJsonType> struct internal_iterator +{ + /// iterator for JSON objects + typename BasicJsonType::object_t::iterator object_iterator {}; + /// iterator for JSON arrays + typename BasicJsonType::array_t::iterator array_iterator {}; + /// generic iterator for all other types + primitive_iterator_t primitive_iterator {}; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/iterators/iter_impl.hpp> + + +#include <ciso646> // not +#include <iterator> // iterator, random_access_iterator_tag, bidirectional_iterator_tag, advance, next +#include <type_traits> // conditional, is_const, remove_const + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/iterators/internal_iterator.hpp> + +// #include <nlohmann/detail/iterators/primitive_iterator.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +// forward declare, to be able to friend it later on +template<typename IteratorType> class iteration_proxy; +template<typename IteratorType> class iteration_proxy_value; + +/*! +@brief a template for a bidirectional iterator for the @ref basic_json class +This class implements a both iterators (iterator and const_iterator) for the +@ref basic_json class. +@note An iterator is called *initialized* when a pointer to a JSON value has + been set (e.g., by a constructor or a copy assignment). If the iterator is + default-constructed, it is *uninitialized* and most methods are undefined. + **The library uses assertions to detect calls on uninitialized iterators.** +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +@since version 1.0.0, simplified in version 2.0.9, change to bidirectional + iterators in version 3.0.0 (see https://github.com/nlohmann/json/issues/593) +*/ +template<typename BasicJsonType> +class iter_impl +{ + /// allow basic_json to access private members + friend iter_impl<typename std::conditional<std::is_const<BasicJsonType>::value, typename std::remove_const<BasicJsonType>::type, const BasicJsonType>::type>; + friend BasicJsonType; + friend iteration_proxy<iter_impl>; + friend iteration_proxy_value<iter_impl>; + + using object_t = typename BasicJsonType::object_t; + using array_t = typename BasicJsonType::array_t; + // make sure BasicJsonType is basic_json or const basic_json + static_assert(is_basic_json<typename std::remove_const<BasicJsonType>::type>::value, + "iter_impl only accepts (const) basic_json"); + + public: + + /// The std::iterator class template (used as a base class to provide typedefs) is deprecated in C++17. + /// The C++ Standard has never required user-defined iterators to derive from std::iterator. + /// A user-defined iterator should provide publicly accessible typedefs named + /// iterator_category, value_type, difference_type, pointer, and reference. + /// Note that value_type is required to be non-const, even for constant iterators. + using iterator_category = std::bidirectional_iterator_tag; + + /// the type of the values when the iterator is dereferenced + using value_type = typename BasicJsonType::value_type; + /// a type to represent differences between iterators + using difference_type = typename BasicJsonType::difference_type; + /// defines a pointer to the type iterated over (value_type) + using pointer = typename std::conditional<std::is_const<BasicJsonType>::value, + typename BasicJsonType::const_pointer, + typename BasicJsonType::pointer>::type; + /// defines a reference to the type iterated over (value_type) + using reference = + typename std::conditional<std::is_const<BasicJsonType>::value, + typename BasicJsonType::const_reference, + typename BasicJsonType::reference>::type; + + /// default constructor + iter_impl() = default; + + /*! + @brief constructor for a given JSON instance + @param[in] object pointer to a JSON object for this iterator + @pre object != nullptr + @post The iterator is initialized; i.e. `m_object != nullptr`. + */ + explicit iter_impl(pointer object) noexcept : m_object(object) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = typename object_t::iterator(); + break; + } + + case value_t::array: + { + m_it.array_iterator = typename array_t::iterator(); + break; + } + + default: + { + m_it.primitive_iterator = primitive_iterator_t(); + break; + } + } + } + + /*! + @note The conventional copy constructor and copy assignment are implicitly + defined. Combined with the following converting constructor and + assignment, they support: (1) copy from iterator to iterator, (2) + copy from const iterator to const iterator, and (3) conversion from + iterator to const iterator. However conversion from const iterator + to iterator is not defined. + */ + + /*! + @brief converting constructor + @param[in] other non-const iterator to copy from + @note It is not checked whether @a other is initialized. + */ + iter_impl(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept + : m_object(other.m_object), m_it(other.m_it) {} + + /*! + @brief converting assignment + @param[in,out] other non-const iterator to copy from + @return const/non-const iterator + @note It is not checked whether @a other is initialized. + */ + iter_impl& operator=(const iter_impl<typename std::remove_const<BasicJsonType>::type>& other) noexcept + { + m_object = other.m_object; + m_it = other.m_it; + return *this; + } + + private: + /*! + @brief set the iterator to the first value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_begin() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->begin(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->begin(); + break; + } + + case value_t::null: + { + // set to end so begin()==end() is true: null is empty + m_it.primitive_iterator.set_end(); + break; + } + + default: + { + m_it.primitive_iterator.set_begin(); + break; + } + } + } + + /*! + @brief set the iterator past the last value + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + void set_end() noexcept + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + m_it.object_iterator = m_object->m_value.object->end(); + break; + } + + case value_t::array: + { + m_it.array_iterator = m_object->m_value.array->end(); + break; + } + + default: + { + m_it.primitive_iterator.set_end(); + break; + } + } + } + + public: + /*! + @brief return a reference to the value pointed to by the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator*() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return m_it.object_iterator->second; + } + + case value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return *m_it.array_iterator; + } + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value")); + + default: + { + if (JSON_LIKELY(m_it.primitive_iterator.is_begin())) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); + } + } + } + + /*! + @brief dereference the iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + pointer operator->() const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + assert(m_it.object_iterator != m_object->m_value.object->end()); + return &(m_it.object_iterator->second); + } + + case value_t::array: + { + assert(m_it.array_iterator != m_object->m_value.array->end()); + return &*m_it.array_iterator; + } + + default: + { + if (JSON_LIKELY(m_it.primitive_iterator.is_begin())) + { + return m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); + } + } + } + + /*! + @brief post-increment (it++) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator++(int) + { + auto result = *this; + ++(*this); + return result; + } + + /*! + @brief pre-increment (++it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator++() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, 1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, 1); + break; + } + + default: + { + ++m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief post-decrement (it--) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl const operator--(int) + { + auto result = *this; + --(*this); + return result; + } + + /*! + @brief pre-decrement (--it) + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator--() + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + { + std::advance(m_it.object_iterator, -1); + break; + } + + case value_t::array: + { + std::advance(m_it.array_iterator, -1); + break; + } + + default: + { + --m_it.primitive_iterator; + break; + } + } + + return *this; + } + + /*! + @brief comparison: equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator==(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + return (m_it.object_iterator == other.m_it.object_iterator); + + case value_t::array: + return (m_it.array_iterator == other.m_it.array_iterator); + + default: + return (m_it.primitive_iterator == other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: not equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator!=(const iter_impl& other) const + { + return not operator==(other); + } + + /*! + @brief comparison: smaller + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<(const iter_impl& other) const + { + // if objects are not the same, the comparison is undefined + if (JSON_UNLIKELY(m_object != other.m_object)) + { + JSON_THROW(invalid_iterator::create(212, "cannot compare iterators of different containers")); + } + + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(213, "cannot compare order of object iterators")); + + case value_t::array: + return (m_it.array_iterator < other.m_it.array_iterator); + + default: + return (m_it.primitive_iterator < other.m_it.primitive_iterator); + } + } + + /*! + @brief comparison: less than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator<=(const iter_impl& other) const + { + return not other.operator < (*this); + } + + /*! + @brief comparison: greater than + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>(const iter_impl& other) const + { + return not operator<=(other); + } + + /*! + @brief comparison: greater than or equal + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + bool operator>=(const iter_impl& other) const + { + return not operator<(other); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator+=(difference_type i) + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); + + case value_t::array: + { + std::advance(m_it.array_iterator, i); + break; + } + + default: + { + m_it.primitive_iterator += i; + break; + } + } + + return *this; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl& operator-=(difference_type i) + { + return operator+=(-i); + } + + /*! + @brief add to iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator+(difference_type i) const + { + auto result = *this; + result += i; + return result; + } + + /*! + @brief addition of distance and iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + friend iter_impl operator+(difference_type i, const iter_impl& it) + { + auto result = it; + result += i; + return result; + } + + /*! + @brief subtract from iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + iter_impl operator-(difference_type i) const + { + auto result = *this; + result -= i; + return result; + } + + /*! + @brief return difference + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + difference_type operator-(const iter_impl& other) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(209, "cannot use offsets with object iterators")); + + case value_t::array: + return m_it.array_iterator - other.m_it.array_iterator; + + default: + return m_it.primitive_iterator - other.m_it.primitive_iterator; + } + } + + /*! + @brief access to successor + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference operator[](difference_type n) const + { + assert(m_object != nullptr); + + switch (m_object->m_type) + { + case value_t::object: + JSON_THROW(invalid_iterator::create(208, "cannot use operator[] for object iterators")); + + case value_t::array: + return *std::next(m_it.array_iterator, n); + + case value_t::null: + JSON_THROW(invalid_iterator::create(214, "cannot get value")); + + default: + { + if (JSON_LIKELY(m_it.primitive_iterator.get_value() == -n)) + { + return *m_object; + } + + JSON_THROW(invalid_iterator::create(214, "cannot get value")); + } + } + } + + /*! + @brief return the key of an object iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + const typename object_t::key_type& key() const + { + assert(m_object != nullptr); + + if (JSON_LIKELY(m_object->is_object())) + { + return m_it.object_iterator->first; + } + + JSON_THROW(invalid_iterator::create(207, "cannot use key() for non-object iterators")); + } + + /*! + @brief return the value of an iterator + @pre The iterator is initialized; i.e. `m_object != nullptr`. + */ + reference value() const + { + return operator*(); + } + + private: + /// associated JSON instance + pointer m_object = nullptr; + /// the actual iterator of the associated instance + internal_iterator<typename std::remove_const<BasicJsonType>::type> m_it; +}; +} // namespace detail +} // namespace nlohmann +// #include <nlohmann/detail/iterators/iteration_proxy.hpp> + +// #include <nlohmann/detail/iterators/json_reverse_iterator.hpp> + + +#include <cstddef> // ptrdiff_t +#include <iterator> // reverse_iterator +#include <utility> // declval + +namespace nlohmann +{ +namespace detail +{ +////////////////////// +// reverse_iterator // +////////////////////// + +/*! +@brief a template for a reverse iterator class + +@tparam Base the base iterator type to reverse. Valid types are @ref +iterator (to create @ref reverse_iterator) and @ref const_iterator (to +create @ref const_reverse_iterator). + +@requirement The class satisfies the following concept requirements: +- +[BidirectionalIterator](https://en.cppreference.com/w/cpp/named_req/BidirectionalIterator): + The iterator that can be moved can be moved in both directions (i.e. + incremented and decremented). +- [OutputIterator](https://en.cppreference.com/w/cpp/named_req/OutputIterator): + It is possible to write to the pointed-to element (only if @a Base is + @ref iterator). + +@since version 1.0.0 +*/ +template<typename Base> +class json_reverse_iterator : public std::reverse_iterator<Base> +{ + public: + using difference_type = std::ptrdiff_t; + /// shortcut to the reverse iterator adapter + using base_iterator = std::reverse_iterator<Base>; + /// the reference type for the pointed-to element + using reference = typename Base::reference; + + /// create reverse iterator from iterator + explicit json_reverse_iterator(const typename base_iterator::iterator_type& it) noexcept + : base_iterator(it) {} + + /// create reverse iterator from base class + explicit json_reverse_iterator(const base_iterator& it) noexcept : base_iterator(it) {} + + /// post-increment (it++) + json_reverse_iterator const operator++(int) + { + return static_cast<json_reverse_iterator>(base_iterator::operator++(1)); + } + + /// pre-increment (++it) + json_reverse_iterator& operator++() + { + return static_cast<json_reverse_iterator&>(base_iterator::operator++()); + } + + /// post-decrement (it--) + json_reverse_iterator const operator--(int) + { + return static_cast<json_reverse_iterator>(base_iterator::operator--(1)); + } + + /// pre-decrement (--it) + json_reverse_iterator& operator--() + { + return static_cast<json_reverse_iterator&>(base_iterator::operator--()); + } + + /// add to iterator + json_reverse_iterator& operator+=(difference_type i) + { + return static_cast<json_reverse_iterator&>(base_iterator::operator+=(i)); + } + + /// add to iterator + json_reverse_iterator operator+(difference_type i) const + { + return static_cast<json_reverse_iterator>(base_iterator::operator+(i)); + } + + /// subtract from iterator + json_reverse_iterator operator-(difference_type i) const + { + return static_cast<json_reverse_iterator>(base_iterator::operator-(i)); + } + + /// return difference + difference_type operator-(const json_reverse_iterator& other) const + { + return base_iterator(*this) - base_iterator(other); + } + + /// access to successor + reference operator[](difference_type n) const + { + return *(this->operator+(n)); + } + + /// return the key of an object iterator + auto key() const -> decltype(std::declval<Base>().key()) + { + auto it = --this->base(); + return it.key(); + } + + /// return the value of an iterator + reference value() const + { + auto it = --this->base(); + return it.operator * (); + } +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/output_adapters.hpp> + + +#include <algorithm> // copy +#include <cstddef> // size_t +#include <ios> // streamsize +#include <iterator> // back_inserter +#include <memory> // shared_ptr, make_shared +#include <ostream> // basic_ostream +#include <string> // basic_string +#include <vector> // vector + +namespace nlohmann +{ +namespace detail +{ +/// abstract output adapter interface +template<typename CharType> struct output_adapter_protocol +{ + virtual void write_character(CharType c) = 0; + virtual void write_characters(const CharType* s, std::size_t length) = 0; + virtual ~output_adapter_protocol() = default; +}; + +/// a type to simplify interfaces +template<typename CharType> +using output_adapter_t = std::shared_ptr<output_adapter_protocol<CharType>>; + +/// output adapter for byte vectors +template<typename CharType> +class output_vector_adapter : public output_adapter_protocol<CharType> +{ + public: + explicit output_vector_adapter(std::vector<CharType>& vec) noexcept + : v(vec) + {} + + void write_character(CharType c) override + { + v.push_back(c); + } + + void write_characters(const CharType* s, std::size_t length) override + { + std::copy(s, s + length, std::back_inserter(v)); + } + + private: + std::vector<CharType>& v; +}; + +/// output adapter for output streams +template<typename CharType> +class output_stream_adapter : public output_adapter_protocol<CharType> +{ + public: + explicit output_stream_adapter(std::basic_ostream<CharType>& s) noexcept + : stream(s) + {} + + void write_character(CharType c) override + { + stream.put(c); + } + + void write_characters(const CharType* s, std::size_t length) override + { + stream.write(s, static_cast<std::streamsize>(length)); + } + + private: + std::basic_ostream<CharType>& stream; +}; + +/// output adapter for basic_string +template<typename CharType, typename StringType = std::basic_string<CharType>> +class output_string_adapter : public output_adapter_protocol<CharType> +{ + public: + explicit output_string_adapter(StringType& s) noexcept + : str(s) + {} + + void write_character(CharType c) override + { + str.push_back(c); + } + + void write_characters(const CharType* s, std::size_t length) override + { + str.append(s, length); + } + + private: + StringType& str; +}; + +template<typename CharType, typename StringType = std::basic_string<CharType>> +class output_adapter +{ + public: + output_adapter(std::vector<CharType>& vec) + : oa(std::make_shared<output_vector_adapter<CharType>>(vec)) {} + + output_adapter(std::basic_ostream<CharType>& s) + : oa(std::make_shared<output_stream_adapter<CharType>>(s)) {} + + output_adapter(StringType& s) + : oa(std::make_shared<output_string_adapter<CharType, StringType>>(s)) {} + + operator output_adapter_t<CharType>() + { + return oa; + } + + private: + output_adapter_t<CharType> oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/input/binary_reader.hpp> + + +#include <algorithm> // generate_n +#include <array> // array +#include <cassert> // assert +#include <cmath> // ldexp +#include <cstddef> // size_t +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstdio> // snprintf +#include <cstring> // memcpy +#include <iterator> // back_inserter +#include <limits> // numeric_limits +#include <string> // char_traits, string +#include <utility> // make_pair, move + +// #include <nlohmann/detail/input/input_adapters.hpp> + +// #include <nlohmann/detail/input/json_sax.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/is_sax.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary reader // +/////////////////// + +/*! +@brief deserialization of CBOR, MessagePack, and UBJSON values +*/ +template<typename BasicJsonType, typename SAX = json_sax_dom_parser<BasicJsonType>> +class binary_reader +{ + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using number_float_t = typename BasicJsonType::number_float_t; + using string_t = typename BasicJsonType::string_t; + using json_sax_t = SAX; + + public: + /*! + @brief create a binary reader + + @param[in] adapter input adapter to read from + */ + explicit binary_reader(input_adapter_t adapter) : ia(std::move(adapter)) + { + (void)detail::is_sax_static_asserts<SAX, BasicJsonType> {}; + assert(ia); + } + + /*! + @param[in] format the binary format to parse + @param[in] sax_ a SAX event processor + @param[in] strict whether to expect the input to be consumed completed + + @return + */ + bool sax_parse(const input_format_t format, + json_sax_t* sax_, + const bool strict = true) + { + sax = sax_; + bool result = false; + + switch (format) + { + case input_format_t::bson: + result = parse_bson_internal(); + break; + + case input_format_t::cbor: + result = parse_cbor_internal(); + break; + + case input_format_t::msgpack: + result = parse_msgpack_internal(); + break; + + case input_format_t::ubjson: + result = parse_ubjson_internal(); + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP + } + + // strict mode: next byte must be EOF + if (result and strict) + { + if (format == input_format_t::ubjson) + { + get_ignore_noop(); + } + else + { + get(); + } + + if (JSON_UNLIKELY(current != std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, get_token_string(), + parse_error::create(110, chars_read, exception_message(format, "expected end of input; last byte: 0x" + get_token_string(), "value"))); + } + } + + return result; + } + + /*! + @brief determine system byte order + + @return true if and only if system's byte order is little endian + + @note from http://stackoverflow.com/a/1001328/266378 + */ + static constexpr bool little_endianess(int num = 1) noexcept + { + return (*reinterpret_cast<char*>(&num) == 1); + } + + private: + ////////// + // BSON // + ////////// + + /*! + @brief Reads in a BSON-object and passes it to the SAX-parser. + @return whether a valid BSON-value was passed to the SAX parser + */ + bool parse_bson_internal() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/false))) + { + return false; + } + + return sax->end_object(); + } + + /*! + @brief Parses a C-style string from the BSON input. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @return `true` if the \x00-byte indicating the end of the string was + encountered before the EOF; false` indicates an unexpected EOF. + */ + bool get_bson_cstr(string_t& result) + { + auto out = std::back_inserter(result); + while (true) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "cstring"))) + { + return false; + } + if (current == 0x00) + { + return true; + } + *out++ = static_cast<char>(current); + } + + return true; + } + + /*! + @brief Parses a zero-terminated string of length @a len from the BSON + input. + @param[in] len The length (including the zero-byte at the end) of the + string to be read. + @param[in, out] result A reference to the string variable where the read + string is to be stored. + @tparam NumberType The type of the length @a len + @pre len >= 1 + @return `true` if the string was successfully parsed + */ + template<typename NumberType> + bool get_bson_string(const NumberType len, string_t& result) + { + if (JSON_UNLIKELY(len < 1)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::bson, "string length must be at least 1, is " + std::to_string(len), "string"))); + } + + return get_string(input_format_t::bson, len - static_cast<NumberType>(1), result) and get() != std::char_traits<char>::eof(); + } + + /*! + @brief Read a BSON document element of the given @a element_type. + @param[in] element_type The BSON element type, c.f. http://bsonspec.org/spec.html + @param[in] element_type_parse_position The position in the input stream, + where the `element_type` was read. + @warning Not all BSON element types are supported yet. An unsupported + @a element_type will give rise to a parse_error.114: + Unsupported BSON record type 0x... + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_internal(const int element_type, + const std::size_t element_type_parse_position) + { + switch (element_type) + { + case 0x01: // double + { + double number; + return get_number<double, true>(input_format_t::bson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0x02: // string + { + std::int32_t len; + string_t value; + return get_number<std::int32_t, true>(input_format_t::bson, len) and get_bson_string(len, value) and sax->string(value); + } + + case 0x03: // object + { + return parse_bson_internal(); + } + + case 0x04: // array + { + return parse_bson_array(); + } + + case 0x08: // boolean + { + return sax->boolean(get() != 0); + } + + case 0x0A: // null + { + return sax->null(); + } + + case 0x10: // int32 + { + std::int32_t value; + return get_number<std::int32_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + case 0x12: // int64 + { + std::int64_t value; + return get_number<std::int64_t, true>(input_format_t::bson, value) and sax->number_integer(value); + } + + default: // anything else not supported (yet) + { + char cr[3]; + (std::snprintf)(cr, sizeof(cr), "%.2hhX", static_cast<unsigned char>(element_type)); + return sax->parse_error(element_type_parse_position, std::string(cr), parse_error::create(114, element_type_parse_position, "Unsupported BSON record type 0x" + std::string(cr))); + } + } + } + + /*! + @brief Read a BSON element list (as specified in the BSON-spec) + + The same binary layout is used for objects and arrays, hence it must be + indicated with the argument @a is_array which one is expected + (true --> array, false --> object). + + @param[in] is_array Determines if the element list being read is to be + treated as an object (@a is_array == false), or as an + array (@a is_array == true). + @return whether a valid BSON-object/array was passed to the SAX parser + */ + bool parse_bson_element_list(const bool is_array) + { + string_t key; + while (int element_type = get()) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::bson, "element list"))) + { + return false; + } + + const std::size_t element_type_parse_position = chars_read; + if (JSON_UNLIKELY(not get_bson_cstr(key))) + { + return false; + } + + if (not is_array) + { + if (not sax->key(key)) + { + return false; + } + } + + if (JSON_UNLIKELY(not parse_bson_element_internal(element_type, element_type_parse_position))) + { + return false; + } + + // get_bson_cstr only appends + key.clear(); + } + + return true; + } + + /*! + @brief Reads an array from the BSON input and passes it to the SAX-parser. + @return whether a valid BSON-array was passed to the SAX parser + */ + bool parse_bson_array() + { + std::int32_t document_size; + get_number<std::int32_t, true>(input_format_t::bson, document_size); + + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_bson_element_list(/*is_array*/true))) + { + return false; + } + + return sax->end_array(); + } + + ////////// + // CBOR // + ////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid CBOR value was passed to the SAX parser + */ + bool parse_cbor_internal(const bool get_char = true) + { + switch (get_char ? get() : current) + { + // EOF + case std::char_traits<char>::eof(): + return unexpect_eof(input_format_t::cbor, "value"); + + // Integer 0x00..0x17 (0..23) + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); + + case 0x18: // Unsigned integer (one-byte uint8_t follows) + { + uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x19: // Unsigned integer (two-byte uint16_t follows) + { + uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x1A: // Unsigned integer (four-byte uint32_t follows) + { + uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + case 0x1B: // Unsigned integer (eight-byte uint64_t follows) + { + uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_unsigned(number); + } + + // Negative integer -1-0x00..-1-0x17 (-1..-24) + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + return sax->number_integer(static_cast<int8_t>(0x20 - 1 - current)); + + case 0x38: // Negative integer (one-byte uint8_t follows) + { + uint8_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); + } + + case 0x39: // Negative integer -1-n (two-byte uint16_t follows) + { + uint16_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); + } + + case 0x3A: // Negative integer -1-n (four-byte uint32_t follows) + { + uint32_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) - number); + } + + case 0x3B: // Negative integer -1-n (eight-byte uint64_t follows) + { + uint64_t number; + return get_number(input_format_t::cbor, number) and sax->number_integer(static_cast<number_integer_t>(-1) + - static_cast<number_integer_t>(number)); + } + + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + case 0x7F: // UTF-8 string (indefinite length) + { + string_t s; + return get_cbor_string(s) and sax->string(s); + } + + // array (0x00..0x17 data items follow) + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + return get_cbor_array(static_cast<std::size_t>(current & 0x1F)); + + case 0x98: // array (one-byte uint8_t for n follows) + { + uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x99: // array (two-byte uint16_t for n follow) + { + uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x9A: // array (four-byte uint32_t for n follow) + { + uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x9B: // array (eight-byte uint64_t for n follow) + { + uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_array(static_cast<std::size_t>(len)); + } + + case 0x9F: // array (indefinite length) + return get_cbor_array(std::size_t(-1)); + + // map (0x00..0x17 pairs of data items follow) + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + return get_cbor_object(static_cast<std::size_t>(current & 0x1F)); + + case 0xB8: // map (one-byte uint8_t for n follows) + { + uint8_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xB9: // map (two-byte uint16_t for n follow) + { + uint16_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xBA: // map (four-byte uint32_t for n follow) + { + uint32_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xBB: // map (eight-byte uint64_t for n follow) + { + uint64_t len; + return get_number(input_format_t::cbor, len) and get_cbor_object(static_cast<std::size_t>(len)); + } + + case 0xBF: // map (indefinite length) + return get_cbor_object(std::size_t(-1)); + + case 0xF4: // false + return sax->boolean(false); + + case 0xF5: // true + return sax->boolean(true); + + case 0xF6: // null + return sax->null(); + + case 0xF9: // Half-Precision Float (two-byte IEEE 754) + { + const int byte1_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + const int byte2_raw = get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "number"))) + { + return false; + } + + const auto byte1 = static_cast<unsigned char>(byte1_raw); + const auto byte2 = static_cast<unsigned char>(byte2_raw); + + // code from RFC 7049, Appendix D, Figure 3: + // As half-precision floating-point numbers were only added + // to IEEE 754 in 2008, today's programming platforms often + // still only have limited support for them. It is very + // easy to include at least decoding support for them even + // without such support. An example of a small decoder for + // half-precision floating-point numbers in the C language + // is shown in Fig. 3. + const int half = (byte1 << 8) + byte2; + const double val = [&half] + { + const int exp = (half >> 10) & 0x1F; + const int mant = half & 0x3FF; + assert(0 <= exp and exp <= 32); + assert(0 <= mant and mant <= 1024); + switch (exp) + { + case 0: + return std::ldexp(mant, -24); + case 31: + return (mant == 0) + ? std::numeric_limits<double>::infinity() + : std::numeric_limits<double>::quiet_NaN(); + default: + return std::ldexp(mant + 1024, exp - 25); + } + }(); + return sax->number_float((half & 0x8000) != 0 + ? static_cast<number_float_t>(-val) + : static_cast<number_float_t>(val), ""); + } + + case 0xFA: // Single-Precision Float (four-byte IEEE 754) + { + float number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0xFB: // Double-Precision Float (eight-byte IEEE 754) + { + double number; + return get_number(input_format_t::cbor, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + default: // anything else (0xFF is handled inside the other types) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::cbor, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @brief reads a CBOR string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + Additionally, CBOR's strings with indefinite lengths are supported. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_cbor_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::cbor, "string"))) + { + return false; + } + + switch (current) + { + // UTF-8 string (0x00..0x17 bytes follow) + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + { + return get_string(input_format_t::cbor, current & 0x1F, result); + } + + case 0x78: // UTF-8 string (one-byte uint8_t for n follows) + { + uint8_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x79: // UTF-8 string (two-byte uint16_t for n follow) + { + uint16_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7A: // UTF-8 string (four-byte uint32_t for n follow) + { + uint32_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7B: // UTF-8 string (eight-byte uint64_t for n follow) + { + uint64_t len; + return get_number(input_format_t::cbor, len) and get_string(input_format_t::cbor, len, result); + } + + case 0x7F: // UTF-8 string (indefinite length) + { + while (get() != 0xFF) + { + string_t chunk; + if (not get_cbor_string(chunk)) + { + return false; + } + result.append(chunk); + } + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::cbor, "expected length specification (0x60-0x7B) or indefinite string type (0x7F); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array or std::size_t(-1) for an + array of indefinite size + @return whether array creation completed + */ + bool get_cbor_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not parse_cbor_internal(false))) + { + return false; + } + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object or std::size_t(-1) for an + object of indefinite size + @return whether object creation completed + */ + bool get_cbor_object(const std::size_t len) + { + if (not JSON_UNLIKELY(sax->start_object(len))) + { + return false; + } + + string_t key; + if (len != std::size_t(-1)) + { + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + else + { + while (get() != 0xFF) + { + if (JSON_UNLIKELY(not get_cbor_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_cbor_internal())) + { + return false; + } + key.clear(); + } + } + + return sax->end_object(); + } + + ///////////// + // MsgPack // + ///////////// + + /*! + @return whether a valid MessagePack value was passed to the SAX parser + */ + bool parse_msgpack_internal() + { + switch (get()) + { + // EOF + case std::char_traits<char>::eof(): + return unexpect_eof(input_format_t::msgpack, "value"); + + // positive fixint + case 0x00: + case 0x01: + case 0x02: + case 0x03: + case 0x04: + case 0x05: + case 0x06: + case 0x07: + case 0x08: + case 0x09: + case 0x0A: + case 0x0B: + case 0x0C: + case 0x0D: + case 0x0E: + case 0x0F: + case 0x10: + case 0x11: + case 0x12: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + case 0x18: + case 0x19: + case 0x1A: + case 0x1B: + case 0x1C: + case 0x1D: + case 0x1E: + case 0x1F: + case 0x20: + case 0x21: + case 0x22: + case 0x23: + case 0x24: + case 0x25: + case 0x26: + case 0x27: + case 0x28: + case 0x29: + case 0x2A: + case 0x2B: + case 0x2C: + case 0x2D: + case 0x2E: + case 0x2F: + case 0x30: + case 0x31: + case 0x32: + case 0x33: + case 0x34: + case 0x35: + case 0x36: + case 0x37: + case 0x38: + case 0x39: + case 0x3A: + case 0x3B: + case 0x3C: + case 0x3D: + case 0x3E: + case 0x3F: + case 0x40: + case 0x41: + case 0x42: + case 0x43: + case 0x44: + case 0x45: + case 0x46: + case 0x47: + case 0x48: + case 0x49: + case 0x4A: + case 0x4B: + case 0x4C: + case 0x4D: + case 0x4E: + case 0x4F: + case 0x50: + case 0x51: + case 0x52: + case 0x53: + case 0x54: + case 0x55: + case 0x56: + case 0x57: + case 0x58: + case 0x59: + case 0x5A: + case 0x5B: + case 0x5C: + case 0x5D: + case 0x5E: + case 0x5F: + case 0x60: + case 0x61: + case 0x62: + case 0x63: + case 0x64: + case 0x65: + case 0x66: + case 0x67: + case 0x68: + case 0x69: + case 0x6A: + case 0x6B: + case 0x6C: + case 0x6D: + case 0x6E: + case 0x6F: + case 0x70: + case 0x71: + case 0x72: + case 0x73: + case 0x74: + case 0x75: + case 0x76: + case 0x77: + case 0x78: + case 0x79: + case 0x7A: + case 0x7B: + case 0x7C: + case 0x7D: + case 0x7E: + case 0x7F: + return sax->number_unsigned(static_cast<number_unsigned_t>(current)); + + // fixmap + case 0x80: + case 0x81: + case 0x82: + case 0x83: + case 0x84: + case 0x85: + case 0x86: + case 0x87: + case 0x88: + case 0x89: + case 0x8A: + case 0x8B: + case 0x8C: + case 0x8D: + case 0x8E: + case 0x8F: + return get_msgpack_object(static_cast<std::size_t>(current & 0x0F)); + + // fixarray + case 0x90: + case 0x91: + case 0x92: + case 0x93: + case 0x94: + case 0x95: + case 0x96: + case 0x97: + case 0x98: + case 0x99: + case 0x9A: + case 0x9B: + case 0x9C: + case 0x9D: + case 0x9E: + case 0x9F: + return get_msgpack_array(static_cast<std::size_t>(current & 0x0F)); + + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } + + case 0xC0: // nil + return sax->null(); + + case 0xC2: // false + return sax->boolean(false); + + case 0xC3: // true + return sax->boolean(true); + + case 0xCA: // float 32 + { + float number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0xCB: // float 64 + { + double number; + return get_number(input_format_t::msgpack, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 0xCC: // uint 8 + { + uint8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCD: // uint 16 + { + uint16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCE: // uint 32 + { + uint32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xCF: // uint 64 + { + uint64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_unsigned(number); + } + + case 0xD0: // int 8 + { + int8_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD1: // int 16 + { + int16_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD2: // int 32 + { + int32_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD3: // int 64 + { + int64_t number; + return get_number(input_format_t::msgpack, number) and sax->number_integer(number); + } + + case 0xD9: // str 8 + case 0xDA: // str 16 + case 0xDB: // str 32 + { + string_t s; + return get_msgpack_string(s) and sax->string(s); + } + + case 0xDC: // array 16 + { + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); + } + + case 0xDD: // array 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_array(static_cast<std::size_t>(len)); + } + + case 0xDE: // map 16 + { + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); + } + + case 0xDF: // map 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_msgpack_object(static_cast<std::size_t>(len)); + } + + // negative fixint + case 0xE0: + case 0xE1: + case 0xE2: + case 0xE3: + case 0xE4: + case 0xE5: + case 0xE6: + case 0xE7: + case 0xE8: + case 0xE9: + case 0xEA: + case 0xEB: + case 0xEC: + case 0xED: + case 0xEE: + case 0xEF: + case 0xF0: + case 0xF1: + case 0xF2: + case 0xF3: + case 0xF4: + case 0xF5: + case 0xF6: + case 0xF7: + case 0xF8: + case 0xF9: + case 0xFA: + case 0xFB: + case 0xFC: + case 0xFD: + case 0xFE: + case 0xFF: + return sax->number_integer(static_cast<int8_t>(current)); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::msgpack, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @brief reads a MessagePack string + + This function first reads starting bytes to determine the expected + string length and then copies this number of bytes into a string. + + @param[out] result created string + + @return whether string creation completed + */ + bool get_msgpack_string(string_t& result) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::msgpack, "string"))) + { + return false; + } + + switch (current) + { + // fixstr + case 0xA0: + case 0xA1: + case 0xA2: + case 0xA3: + case 0xA4: + case 0xA5: + case 0xA6: + case 0xA7: + case 0xA8: + case 0xA9: + case 0xAA: + case 0xAB: + case 0xAC: + case 0xAD: + case 0xAE: + case 0xAF: + case 0xB0: + case 0xB1: + case 0xB2: + case 0xB3: + case 0xB4: + case 0xB5: + case 0xB6: + case 0xB7: + case 0xB8: + case 0xB9: + case 0xBA: + case 0xBB: + case 0xBC: + case 0xBD: + case 0xBE: + case 0xBF: + { + return get_string(input_format_t::msgpack, current & 0x1F, result); + } + + case 0xD9: // str 8 + { + uint8_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + case 0xDA: // str 16 + { + uint16_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + case 0xDB: // str 32 + { + uint32_t len; + return get_number(input_format_t::msgpack, len) and get_string(input_format_t::msgpack, len, result); + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::msgpack, "expected length specification (0xA0-0xBF, 0xD9-0xDB); last byte: 0x" + last_token, "string"))); + } + } + } + + /*! + @param[in] len the length of the array + @return whether array creation completed + */ + bool get_msgpack_array(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_array(len))) + { + return false; + } + + for (std::size_t i = 0; i < len; ++i) + { + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + } + + return sax->end_array(); + } + + /*! + @param[in] len the length of the object + @return whether object creation completed + */ + bool get_msgpack_object(const std::size_t len) + { + if (JSON_UNLIKELY(not sax->start_object(len))) + { + return false; + } + + string_t key; + for (std::size_t i = 0; i < len; ++i) + { + get(); + if (JSON_UNLIKELY(not get_msgpack_string(key) or not sax->key(key))) + { + return false; + } + + if (JSON_UNLIKELY(not parse_msgpack_internal())) + { + return false; + } + key.clear(); + } + + return sax->end_object(); + } + + //////////// + // UBJSON // + //////////// + + /*! + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether a valid UBJSON value was passed to the SAX parser + */ + bool parse_ubjson_internal(const bool get_char = true) + { + return get_ubjson_value(get_char ? get_ignore_noop() : current); + } + + /*! + @brief reads a UBJSON string + + This function is either called after reading the 'S' byte explicitly + indicating a string, or in case of an object key where the 'S' byte can be + left out. + + @param[out] result created string + @param[in] get_char whether a new character should be retrieved from the + input (true, default) or whether the last read + character should be considered instead + + @return whether string creation completed + */ + bool get_ubjson_string(string_t& result, const bool get_char = true) + { + if (get_char) + { + get(); // TODO: may we ignore N here? + } + + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + + switch (current) + { + case 'U': + { + uint8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'i': + { + int8_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'I': + { + int16_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'l': + { + int32_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + case 'L': + { + int64_t len; + return get_number(input_format_t::ubjson, len) and get_string(input_format_t::ubjson, len, result); + } + + default: + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L); last byte: 0x" + last_token, "string"))); + } + } + + /*! + @param[out] result determined size + @return whether size determination completed + */ + bool get_ubjson_size_value(std::size_t& result) + { + switch (get_ignore_noop()) + { + case 'U': + { + uint8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'i': + { + int8_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'I': + { + int16_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'l': + { + int32_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + case 'L': + { + int64_t number; + if (JSON_UNLIKELY(not get_number(input_format_t::ubjson, number))) + { + return false; + } + result = static_cast<std::size_t>(number); + return true; + } + + default: + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "expected length type specification (U, i, I, l, L) after '#'; last byte: 0x" + last_token, "size"))); + } + } + } + + /*! + @brief determine the type and size for a container + + In the optimized UBJSON format, a type and a size can be provided to allow + for a more compact representation. + + @param[out] result pair of the size and the type + + @return whether pair creation completed + */ + bool get_ubjson_size_type(std::pair<std::size_t, int>& result) + { + result.first = string_t::npos; // size + result.second = 0; // type + + get_ignore_noop(); + + if (current == '$') + { + result.second = get(); // must not ignore 'N', because 'N' maybe the type + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "type"))) + { + return false; + } + + get_ignore_noop(); + if (JSON_UNLIKELY(current != '#')) + { + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "value"))) + { + return false; + } + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "expected '#' after type information; last byte: 0x" + last_token, "size"))); + } + + return get_ubjson_size_value(result.first); + } + else if (current == '#') + { + return get_ubjson_size_value(result.first); + } + return true; + } + + /*! + @param prefix the previously read or set type prefix + @return whether value creation completed + */ + bool get_ubjson_value(const int prefix) + { + switch (prefix) + { + case std::char_traits<char>::eof(): // EOF + return unexpect_eof(input_format_t::ubjson, "value"); + + case 'T': // true + return sax->boolean(true); + case 'F': // false + return sax->boolean(false); + + case 'Z': // null + return sax->null(); + + case 'U': + { + uint8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_unsigned(number); + } + + case 'i': + { + int8_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'I': + { + int16_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'l': + { + int32_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'L': + { + int64_t number; + return get_number(input_format_t::ubjson, number) and sax->number_integer(number); + } + + case 'd': + { + float number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'D': + { + double number; + return get_number(input_format_t::ubjson, number) and sax->number_float(static_cast<number_float_t>(number), ""); + } + + case 'C': // char + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(input_format_t::ubjson, "char"))) + { + return false; + } + if (JSON_UNLIKELY(current > 127)) + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(113, chars_read, exception_message(input_format_t::ubjson, "byte after 'C' must be in range 0x00..0x7F; last byte: 0x" + last_token, "char"))); + } + string_t s(1, static_cast<char>(current)); + return sax->string(s); + } + + case 'S': // string + { + string_t s; + return get_ubjson_string(s) and sax->string(s); + } + + case '[': // array + return get_ubjson_array(); + + case '{': // object + return get_ubjson_object(); + + default: // anything else + { + auto last_token = get_token_string(); + return sax->parse_error(chars_read, last_token, parse_error::create(112, chars_read, exception_message(input_format_t::ubjson, "invalid byte: 0x" + last_token, "value"))); + } + } + } + + /*! + @return whether array creation completed + */ + bool get_ubjson_array() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_array(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + if (size_and_type.second != 'N') + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + } + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_array(std::size_t(-1)))) + { + return false; + } + + while (current != ']') + { + if (JSON_UNLIKELY(not parse_ubjson_internal(false))) + { + return false; + } + get_ignore_noop(); + } + } + + return sax->end_array(); + } + + /*! + @return whether object creation completed + */ + bool get_ubjson_object() + { + std::pair<std::size_t, int> size_and_type; + if (JSON_UNLIKELY(not get_ubjson_size_type(size_and_type))) + { + return false; + } + + string_t key; + if (size_and_type.first != string_t::npos) + { + if (JSON_UNLIKELY(not sax->start_object(size_and_type.first))) + { + return false; + } + + if (size_and_type.second != 0) + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not get_ubjson_value(size_and_type.second))) + { + return false; + } + key.clear(); + } + } + else + { + for (std::size_t i = 0; i < size_and_type.first; ++i) + { + if (JSON_UNLIKELY(not get_ubjson_string(key) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + key.clear(); + } + } + } + else + { + if (JSON_UNLIKELY(not sax->start_object(std::size_t(-1)))) + { + return false; + } + + while (current != '}') + { + if (JSON_UNLIKELY(not get_ubjson_string(key, false) or not sax->key(key))) + { + return false; + } + if (JSON_UNLIKELY(not parse_ubjson_internal())) + { + return false; + } + get_ignore_noop(); + key.clear(); + } + } + + return sax->end_object(); + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /*! + @brief get next character from the input + + This function provides the interface to the used input adapter. It does + not throw in case the input reached EOF, but returns a -'ve valued + `std::char_traits<char>::eof()` in that case. + + @return character read from the input + */ + int get() + { + ++chars_read; + return (current = ia->get_character()); + } + + /*! + @return character read from the input after ignoring all 'N' entries + */ + int get_ignore_noop() + { + do + { + get(); + } + while (current == 'N'); + + return current; + } + + /* + @brief read a number from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[out] result number of type @a NumberType + + @return whether conversion completed + + @note This function needs to respect the system's endianess, because + bytes in CBOR, MessagePack, and UBJSON are stored in network order + (big endian) and therefore need reordering on little endian systems. + */ + template<typename NumberType, bool InputIsLittleEndian = false> + bool get_number(const input_format_t format, NumberType& result) + { + // step 1: read input into array with system's byte order + std::array<uint8_t, sizeof(NumberType)> vec; + for (std::size_t i = 0; i < sizeof(NumberType); ++i) + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "number"))) + { + return false; + } + + // reverse byte order prior to conversion if necessary + if (is_little_endian && !InputIsLittleEndian) + { + vec[sizeof(NumberType) - i - 1] = static_cast<uint8_t>(current); + } + else + { + vec[i] = static_cast<uint8_t>(current); // LCOV_EXCL_LINE + } + } + + // step 2: convert array into number of type T and return + std::memcpy(&result, vec.data(), sizeof(NumberType)); + return true; + } + + /*! + @brief create a string by reading characters from the input + + @tparam NumberType the type of the number + @param[in] format the current format (for diagnostics) + @param[in] len number of characters to read + @param[out] result string created by reading @a len bytes + + @return whether string creation completed + + @note We can not reserve @a len bytes for the result, because @a len + may be too large. Usually, @ref unexpect_eof() detects the end of + the input before we run out of string memory. + */ + template<typename NumberType> + bool get_string(const input_format_t format, + const NumberType len, + string_t& result) + { + bool success = true; + std::generate_n(std::back_inserter(result), len, [this, &success, &format]() + { + get(); + if (JSON_UNLIKELY(not unexpect_eof(format, "string"))) + { + success = false; + } + return static_cast<char>(current); + }); + return success; + } + + /*! + @param[in] format the current format (for diagnostics) + @param[in] context further context information (for diagnostics) + @return whether the last read character is not EOF + */ + bool unexpect_eof(const input_format_t format, const char* context) const + { + if (JSON_UNLIKELY(current == std::char_traits<char>::eof())) + { + return sax->parse_error(chars_read, "<end of file>", + parse_error::create(110, chars_read, exception_message(format, "unexpected end of input", context))); + } + return true; + } + + /*! + @return a string representation of the last read byte + */ + std::string get_token_string() const + { + char cr[3]; + (std::snprintf)(cr, 3, "%.2hhX", static_cast<unsigned char>(current)); + return std::string{cr}; + } + + /*! + @param[in] format the current format + @param[in] detail a detailed error message + @param[in] context further contect information + @return a message string to use in the parse_error exceptions + */ + std::string exception_message(const input_format_t format, + const std::string& detail, + const std::string& context) const + { + std::string error_msg = "syntax error while parsing "; + + switch (format) + { + case input_format_t::cbor: + error_msg += "CBOR"; + break; + + case input_format_t::msgpack: + error_msg += "MessagePack"; + break; + + case input_format_t::ubjson: + error_msg += "UBJSON"; + break; + + case input_format_t::bson: + error_msg += "BSON"; + break; + + // LCOV_EXCL_START + default: + assert(false); + // LCOV_EXCL_STOP + } + + return error_msg + " " + context + ": " + detail; + } + + private: + /// input adapter + input_adapter_t ia = nullptr; + + /// the current character + int current = std::char_traits<char>::eof(); + + /// the number of characters read + std::size_t chars_read = 0; + + /// whether we can assume little endianess + const bool is_little_endian = little_endianess(); + + /// the SAX parser + json_sax_t* sax = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/binary_writer.hpp> + + +#include <algorithm> // reverse +#include <array> // array +#include <cstdint> // uint8_t, uint16_t, uint32_t, uint64_t +#include <cstring> // memcpy +#include <limits> // numeric_limits + +// #include <nlohmann/detail/input/binary_reader.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// binary writer // +/////////////////// + +/*! +@brief serialization to CBOR and MessagePack values +*/ +template<typename BasicJsonType, typename CharType> +class binary_writer +{ + using string_t = typename BasicJsonType::string_t; + + public: + /*! + @brief create a binary writer + + @param[in] adapter output adapter to write to + */ + explicit binary_writer(output_adapter_t<CharType> adapter) : oa(adapter) + { + assert(oa); + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + { + write_bson_object(*j.m_value.object); + break; + } + + default: + { + JSON_THROW(type_error::create(317, "to serialize to BSON, top-level type must be object, but is " + std::string(j.type_name()))); + } + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_cbor(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: + { + oa->write_character(to_char_type(0xF6)); + break; + } + + case value_t::boolean: + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xF5) + : to_char_type(0xF4)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // CBOR does not differentiate between positive signed + // integers and unsigned integers. Therefore, we used the + // code from the value_t::number_unsigned case here. + if (j.m_value.number_integer <= 0x17) + { + write_number(static_cast<uint8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast<uint8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits<uint16_t>::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast<uint16_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer <= (std::numeric_limits<uint32_t>::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast<uint32_t>(j.m_value.number_integer)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast<uint64_t>(j.m_value.number_integer)); + } + } + else + { + // The conversions below encode the sign in the first + // byte, and the value is converted to a positive number. + const auto positive_number = -1 - j.m_value.number_integer; + if (j.m_value.number_integer >= -24) + { + write_number(static_cast<uint8_t>(0x20 + positive_number)); + } + else if (positive_number <= (std::numeric_limits<uint8_t>::max)()) + { + oa->write_character(to_char_type(0x38)); + write_number(static_cast<uint8_t>(positive_number)); + } + else if (positive_number <= (std::numeric_limits<uint16_t>::max)()) + { + oa->write_character(to_char_type(0x39)); + write_number(static_cast<uint16_t>(positive_number)); + } + else if (positive_number <= (std::numeric_limits<uint32_t>::max)()) + { + oa->write_character(to_char_type(0x3A)); + write_number(static_cast<uint32_t>(positive_number)); + } + else + { + oa->write_character(to_char_type(0x3B)); + write_number(static_cast<uint64_t>(positive_number)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= 0x17) + { + write_number(static_cast<uint8_t>(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + oa->write_character(to_char_type(0x18)); + write_number(static_cast<uint8_t>(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) + { + oa->write_character(to_char_type(0x19)); + write_number(static_cast<uint16_t>(j.m_value.number_unsigned)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) + { + oa->write_character(to_char_type(0x1A)); + write_number(static_cast<uint32_t>(j.m_value.number_unsigned)); + } + else + { + oa->write_character(to_char_type(0x1B)); + write_number(static_cast<uint64_t>(j.m_value.number_unsigned)); + } + break; + } + + case value_t::number_float: + { + oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 0x17) + { + write_number(static_cast<uint8_t>(0x60 + N)); + } + else if (N <= (std::numeric_limits<uint8_t>::max)()) + { + oa->write_character(to_char_type(0x78)); + write_number(static_cast<uint8_t>(N)); + } + else if (N <= (std::numeric_limits<uint16_t>::max)()) + { + oa->write_character(to_char_type(0x79)); + write_number(static_cast<uint16_t>(N)); + } + else if (N <= (std::numeric_limits<uint32_t>::max)()) + { + oa->write_character(to_char_type(0x7A)); + write_number(static_cast<uint32_t>(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits<uint64_t>::max)()) + { + oa->write_character(to_char_type(0x7B)); + write_number(static_cast<uint64_t>(N)); + } + // LCOV_EXCL_STOP + + // step 2: write the string + oa->write_characters( + reinterpret_cast<const CharType*>(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 0x17) + { + write_number(static_cast<uint8_t>(0x80 + N)); + } + else if (N <= (std::numeric_limits<uint8_t>::max)()) + { + oa->write_character(to_char_type(0x98)); + write_number(static_cast<uint8_t>(N)); + } + else if (N <= (std::numeric_limits<uint16_t>::max)()) + { + oa->write_character(to_char_type(0x99)); + write_number(static_cast<uint16_t>(N)); + } + else if (N <= (std::numeric_limits<uint32_t>::max)()) + { + oa->write_character(to_char_type(0x9A)); + write_number(static_cast<uint32_t>(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits<uint64_t>::max)()) + { + oa->write_character(to_char_type(0x9B)); + write_number(static_cast<uint64_t>(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_cbor(el); + } + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 0x17) + { + write_number(static_cast<uint8_t>(0xA0 + N)); + } + else if (N <= (std::numeric_limits<uint8_t>::max)()) + { + oa->write_character(to_char_type(0xB8)); + write_number(static_cast<uint8_t>(N)); + } + else if (N <= (std::numeric_limits<uint16_t>::max)()) + { + oa->write_character(to_char_type(0xB9)); + write_number(static_cast<uint16_t>(N)); + } + else if (N <= (std::numeric_limits<uint32_t>::max)()) + { + oa->write_character(to_char_type(0xBA)); + write_number(static_cast<uint32_t>(N)); + } + // LCOV_EXCL_START + else if (N <= (std::numeric_limits<uint64_t>::max)()) + { + oa->write_character(to_char_type(0xBB)); + write_number(static_cast<uint64_t>(N)); + } + // LCOV_EXCL_STOP + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_cbor(el.first); + write_cbor(el.second); + } + break; + } + + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + */ + void write_msgpack(const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::null: // nil + { + oa->write_character(to_char_type(0xC0)); + break; + } + + case value_t::boolean: // true and false + { + oa->write_character(j.m_value.boolean + ? to_char_type(0xC3) + : to_char_type(0xC2)); + break; + } + + case value_t::number_integer: + { + if (j.m_value.number_integer >= 0) + { + // MessagePack does not differentiate between positive + // signed integers and unsigned integers. Therefore, we used + // the code from the value_t::number_unsigned case here. + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast<uint8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast<uint8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast<uint16_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast<uint32_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast<uint64_t>(j.m_value.number_integer)); + } + } + else + { + if (j.m_value.number_integer >= -32) + { + // negative fixnum + write_number(static_cast<int8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits<int8_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + { + // int 8 + oa->write_character(to_char_type(0xD0)); + write_number(static_cast<int8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits<int16_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + { + // int 16 + oa->write_character(to_char_type(0xD1)); + write_number(static_cast<int16_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits<int32_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + { + // int 32 + oa->write_character(to_char_type(0xD2)); + write_number(static_cast<int32_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_integer >= (std::numeric_limits<int64_t>::min)() and + j.m_value.number_integer <= (std::numeric_limits<int64_t>::max)()) + { + // int 64 + oa->write_character(to_char_type(0xD3)); + write_number(static_cast<int64_t>(j.m_value.number_integer)); + } + } + break; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned < 128) + { + // positive fixnum + write_number(static_cast<uint8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + // uint 8 + oa->write_character(to_char_type(0xCC)); + write_number(static_cast<uint8_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint16_t>::max)()) + { + // uint 16 + oa->write_character(to_char_type(0xCD)); + write_number(static_cast<uint16_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint32_t>::max)()) + { + // uint 32 + oa->write_character(to_char_type(0xCE)); + write_number(static_cast<uint32_t>(j.m_value.number_integer)); + } + else if (j.m_value.number_unsigned <= (std::numeric_limits<uint64_t>::max)()) + { + // uint 64 + oa->write_character(to_char_type(0xCF)); + write_number(static_cast<uint64_t>(j.m_value.number_integer)); + } + break; + } + + case value_t::number_float: + { + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + break; + } + + case value_t::string: + { + // step 1: write control byte and the string length + const auto N = j.m_value.string->size(); + if (N <= 31) + { + // fixstr + write_number(static_cast<uint8_t>(0xA0 | N)); + } + else if (N <= (std::numeric_limits<uint8_t>::max)()) + { + // str 8 + oa->write_character(to_char_type(0xD9)); + write_number(static_cast<uint8_t>(N)); + } + else if (N <= (std::numeric_limits<uint16_t>::max)()) + { + // str 16 + oa->write_character(to_char_type(0xDA)); + write_number(static_cast<uint16_t>(N)); + } + else if (N <= (std::numeric_limits<uint32_t>::max)()) + { + // str 32 + oa->write_character(to_char_type(0xDB)); + write_number(static_cast<uint32_t>(N)); + } + + // step 2: write the string + oa->write_characters( + reinterpret_cast<const CharType*>(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + // step 1: write control byte and the array size + const auto N = j.m_value.array->size(); + if (N <= 15) + { + // fixarray + write_number(static_cast<uint8_t>(0x90 | N)); + } + else if (N <= (std::numeric_limits<uint16_t>::max)()) + { + // array 16 + oa->write_character(to_char_type(0xDC)); + write_number(static_cast<uint16_t>(N)); + } + else if (N <= (std::numeric_limits<uint32_t>::max)()) + { + // array 32 + oa->write_character(to_char_type(0xDD)); + write_number(static_cast<uint32_t>(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.array) + { + write_msgpack(el); + } + break; + } + + case value_t::object: + { + // step 1: write control byte and the object size + const auto N = j.m_value.object->size(); + if (N <= 15) + { + // fixmap + write_number(static_cast<uint8_t>(0x80 | (N & 0xF))); + } + else if (N <= (std::numeric_limits<uint16_t>::max)()) + { + // map 16 + oa->write_character(to_char_type(0xDE)); + write_number(static_cast<uint16_t>(N)); + } + else if (N <= (std::numeric_limits<uint32_t>::max)()) + { + // map 32 + oa->write_character(to_char_type(0xDF)); + write_number(static_cast<uint32_t>(N)); + } + + // step 2: write each element + for (const auto& el : *j.m_value.object) + { + write_msgpack(el.first); + write_msgpack(el.second); + } + break; + } + + default: + break; + } + } + + /*! + @param[in] j JSON value to serialize + @param[in] use_count whether to use '#' prefixes (optimized format) + @param[in] use_type whether to use '$' prefixes (optimized format) + @param[in] add_prefix whether prefixes need to be used for this value + */ + void write_ubjson(const BasicJsonType& j, const bool use_count, + const bool use_type, const bool add_prefix = true) + { + switch (j.type()) + { + case value_t::null: + { + if (add_prefix) + { + oa->write_character(to_char_type('Z')); + } + break; + } + + case value_t::boolean: + { + if (add_prefix) + { + oa->write_character(j.m_value.boolean + ? to_char_type('T') + : to_char_type('F')); + } + break; + } + + case value_t::number_integer: + { + write_number_with_ubjson_prefix(j.m_value.number_integer, add_prefix); + break; + } + + case value_t::number_unsigned: + { + write_number_with_ubjson_prefix(j.m_value.number_unsigned, add_prefix); + break; + } + + case value_t::number_float: + { + write_number_with_ubjson_prefix(j.m_value.number_float, add_prefix); + break; + } + + case value_t::string: + { + if (add_prefix) + { + oa->write_character(to_char_type('S')); + } + write_number_with_ubjson_prefix(j.m_value.string->size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(j.m_value.string->c_str()), + j.m_value.string->size()); + break; + } + + case value_t::array: + { + if (add_prefix) + { + oa->write_character(to_char_type('[')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.array->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin() + 1, j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.array->size(), true); + } + + for (const auto& el : *j.m_value.array) + { + write_ubjson(el, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type(']')); + } + + break; + } + + case value_t::object: + { + if (add_prefix) + { + oa->write_character(to_char_type('{')); + } + + bool prefix_required = true; + if (use_type and not j.m_value.object->empty()) + { + assert(use_count); + const CharType first_prefix = ubjson_prefix(j.front()); + const bool same_prefix = std::all_of(j.begin(), j.end(), + [this, first_prefix](const BasicJsonType & v) + { + return ubjson_prefix(v) == first_prefix; + }); + + if (same_prefix) + { + prefix_required = false; + oa->write_character(to_char_type('$')); + oa->write_character(first_prefix); + } + } + + if (use_count) + { + oa->write_character(to_char_type('#')); + write_number_with_ubjson_prefix(j.m_value.object->size(), true); + } + + for (const auto& el : *j.m_value.object) + { + write_number_with_ubjson_prefix(el.first.size(), true); + oa->write_characters( + reinterpret_cast<const CharType*>(el.first.c_str()), + el.first.size()); + write_ubjson(el.second, use_count, use_type, prefix_required); + } + + if (not use_count) + { + oa->write_character(to_char_type('}')); + } + + break; + } + + default: + break; + } + } + + private: + ////////// + // BSON // + ////////// + + /*! + @return The size of a BSON document entry header, including the id marker + and the entry name size (and its null-terminator). + */ + static std::size_t calc_bson_entry_header_size(const string_t& name) + { + const auto it = name.find(static_cast<typename string_t::value_type>(0)); + if (JSON_UNLIKELY(it != BasicJsonType::string_t::npos)) + { + JSON_THROW(out_of_range::create(409, + "BSON key cannot contain code point U+0000 (at byte " + std::to_string(it) + ")")); + } + + return /*id*/ 1ul + name.size() + /*zero-terminator*/1u; + } + + /*! + @brief Writes the given @a element_type and @a name to the output adapter + */ + void write_bson_entry_header(const string_t& name, + const std::uint8_t element_type) + { + oa->write_character(to_char_type(element_type)); // boolean + oa->write_characters( + reinterpret_cast<const CharType*>(name.c_str()), + name.size() + 1u); + } + + /*! + @brief Writes a BSON element with key @a name and boolean value @a value + */ + void write_bson_boolean(const string_t& name, + const bool value) + { + write_bson_entry_header(name, 0x08); + oa->write_character(value ? to_char_type(0x01) : to_char_type(0x00)); + } + + /*! + @brief Writes a BSON element with key @a name and double value @a value + */ + void write_bson_double(const string_t& name, + const double value) + { + write_bson_entry_header(name, 0x01); + write_number<double, true>(value); + } + + /*! + @return The size of the BSON-encoded string in @a value + */ + static std::size_t calc_bson_string_size(const string_t& value) + { + return sizeof(std::int32_t) + value.size() + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and string value @a value + */ + void write_bson_string(const string_t& name, + const string_t& value) + { + write_bson_entry_header(name, 0x02); + + write_number<std::int32_t, true>(static_cast<std::int32_t>(value.size() + 1ul)); + oa->write_characters( + reinterpret_cast<const CharType*>(value.c_str()), + value.size() + 1); + } + + /*! + @brief Writes a BSON element with key @a name and null value + */ + void write_bson_null(const string_t& name) + { + write_bson_entry_header(name, 0x0A); + } + + /*! + @return The size of the BSON-encoded integer @a value + */ + static std::size_t calc_bson_integer_size(const std::int64_t value) + { + if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) + { + return sizeof(std::int32_t); + } + else + { + return sizeof(std::int64_t); + } + } + + /*! + @brief Writes a BSON element with key @a name and integer @a value + */ + void write_bson_integer(const string_t& name, + const std::int64_t value) + { + if ((std::numeric_limits<std::int32_t>::min)() <= value and value <= (std::numeric_limits<std::int32_t>::max)()) + { + write_bson_entry_header(name, 0x10); // int32 + write_number<std::int32_t, true>(static_cast<std::int32_t>(value)); + } + else + { + write_bson_entry_header(name, 0x12); // int64 + write_number<std::int64_t, true>(static_cast<std::int64_t>(value)); + } + } + + /*! + @return The size of the BSON-encoded unsigned integer in @a j + */ + static constexpr std::size_t calc_bson_unsigned_size(const std::uint64_t value) noexcept + { + return (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) + ? sizeof(std::int32_t) + : sizeof(std::int64_t); + } + + /*! + @brief Writes a BSON element with key @a name and unsigned @a value + */ + void write_bson_unsigned(const string_t& name, + const std::uint64_t value) + { + if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int32_t>::max)())) + { + write_bson_entry_header(name, 0x10 /* int32 */); + write_number<std::int32_t, true>(static_cast<std::int32_t>(value)); + } + else if (value <= static_cast<std::uint64_t>((std::numeric_limits<std::int64_t>::max)())) + { + write_bson_entry_header(name, 0x12 /* int64 */); + write_number<std::int64_t, true>(static_cast<std::int64_t>(value)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(value) + " cannot be represented by BSON as it does not fit int64")); + } + } + + /*! + @brief Writes a BSON element with key @a name and object @a value + */ + void write_bson_object_entry(const string_t& name, + const typename BasicJsonType::object_t& value) + { + write_bson_entry_header(name, 0x03); // object + write_bson_object(value); + } + + /*! + @return The size of the BSON-encoded array @a value + */ + static std::size_t calc_bson_array_size(const typename BasicJsonType::array_t& value) + { + std::size_t embedded_document_size = 0ul; + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + embedded_document_size += calc_bson_element_size(std::to_string(array_index++), el); + } + + return sizeof(std::int32_t) + embedded_document_size + 1ul; + } + + /*! + @brief Writes a BSON element with key @a name and array @a value + */ + void write_bson_array(const string_t& name, + const typename BasicJsonType::array_t& value) + { + write_bson_entry_header(name, 0x04); // array + write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_array_size(value))); + + std::size_t array_index = 0ul; + + for (const auto& el : value) + { + write_bson_element(std::to_string(array_index++), el); + } + + oa->write_character(to_char_type(0x00)); + } + + /*! + @brief Calculates the size necessary to serialize the JSON value @a j with its @a name + @return The calculated size for the BSON document entry for @a j with the given @a name. + */ + static std::size_t calc_bson_element_size(const string_t& name, + const BasicJsonType& j) + { + const auto header_size = calc_bson_entry_header_size(name); + switch (j.type()) + { + case value_t::object: + return header_size + calc_bson_object_size(*j.m_value.object); + + case value_t::array: + return header_size + calc_bson_array_size(*j.m_value.array); + + case value_t::boolean: + return header_size + 1ul; + + case value_t::number_float: + return header_size + 8ul; + + case value_t::number_integer: + return header_size + calc_bson_integer_size(j.m_value.number_integer); + + case value_t::number_unsigned: + return header_size + calc_bson_unsigned_size(j.m_value.number_unsigned); + + case value_t::string: + return header_size + calc_bson_string_size(*j.m_value.string); + + case value_t::null: + return header_size + 0ul; + + // LCOV_EXCL_START + default: + assert(false); + return 0ul; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Serializes the JSON value @a j to BSON and associates it with the + key @a name. + @param name The name to associate with the JSON entity @a j within the + current BSON document + @return The size of the BSON entry + */ + void write_bson_element(const string_t& name, + const BasicJsonType& j) + { + switch (j.type()) + { + case value_t::object: + return write_bson_object_entry(name, *j.m_value.object); + + case value_t::array: + return write_bson_array(name, *j.m_value.array); + + case value_t::boolean: + return write_bson_boolean(name, j.m_value.boolean); + + case value_t::number_float: + return write_bson_double(name, j.m_value.number_float); + + case value_t::number_integer: + return write_bson_integer(name, j.m_value.number_integer); + + case value_t::number_unsigned: + return write_bson_unsigned(name, j.m_value.number_unsigned); + + case value_t::string: + return write_bson_string(name, *j.m_value.string); + + case value_t::null: + return write_bson_null(name); + + // LCOV_EXCL_START + default: + assert(false); + return; + // LCOV_EXCL_STOP + }; + } + + /*! + @brief Calculates the size of the BSON serialization of the given + JSON-object @a j. + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + static std::size_t calc_bson_object_size(const typename BasicJsonType::object_t& value) + { + std::size_t document_size = std::accumulate(value.begin(), value.end(), 0ul, + [](size_t result, const typename BasicJsonType::object_t::value_type & el) + { + return result += calc_bson_element_size(el.first, el.second); + }); + + return sizeof(std::int32_t) + document_size + 1ul; + } + + /*! + @param[in] j JSON value to serialize + @pre j.type() == value_t::object + */ + void write_bson_object(const typename BasicJsonType::object_t& value) + { + write_number<std::int32_t, true>(static_cast<std::int32_t>(calc_bson_object_size(value))); + + for (const auto& el : value) + { + write_bson_element(el.first, el.second); + } + + oa->write_character(to_char_type(0x00)); + } + + ////////// + // CBOR // + ////////// + + static constexpr CharType get_cbor_float_prefix(float /*unused*/) + { + return to_char_type(0xFA); // Single-Precision Float + } + + static constexpr CharType get_cbor_float_prefix(double /*unused*/) + { + return to_char_type(0xFB); // Double-Precision Float + } + + ///////////// + // MsgPack // + ///////////// + + static constexpr CharType get_msgpack_float_prefix(float /*unused*/) + { + return to_char_type(0xCA); // float 32 + } + + static constexpr CharType get_msgpack_float_prefix(double /*unused*/) + { + return to_char_type(0xCB); // float 64 + } + + //////////// + // UBJSON // + //////////// + + // UBJSON: write number (floating point) + template<typename NumberType, typename std::enable_if< + std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (add_prefix) + { + oa->write_character(get_ubjson_float_prefix(n)); + } + write_number(n); + } + + // UBJSON: write number (unsigned integer) + template<typename NumberType, typename std::enable_if< + std::is_unsigned<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if (n <= static_cast<uint64_t>((std::numeric_limits<int8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= (std::numeric_limits<uint8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int16_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int32_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if (n <= static_cast<uint64_t>((std::numeric_limits<int64_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + } + + // UBJSON: write number (signed integer) + template<typename NumberType, typename std::enable_if< + std::is_signed<NumberType>::value and + not std::is_floating_point<NumberType>::value, int>::type = 0> + void write_number_with_ubjson_prefix(const NumberType n, + const bool add_prefix) + { + if ((std::numeric_limits<int8_t>::min)() <= n and n <= (std::numeric_limits<int8_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('i')); // int8 + } + write_number(static_cast<int8_t>(n)); + } + else if (static_cast<int64_t>((std::numeric_limits<uint8_t>::min)()) <= n and n <= static_cast<int64_t>((std::numeric_limits<uint8_t>::max)())) + { + if (add_prefix) + { + oa->write_character(to_char_type('U')); // uint8 + } + write_number(static_cast<uint8_t>(n)); + } + else if ((std::numeric_limits<int16_t>::min)() <= n and n <= (std::numeric_limits<int16_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('I')); // int16 + } + write_number(static_cast<int16_t>(n)); + } + else if ((std::numeric_limits<int32_t>::min)() <= n and n <= (std::numeric_limits<int32_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('l')); // int32 + } + write_number(static_cast<int32_t>(n)); + } + else if ((std::numeric_limits<int64_t>::min)() <= n and n <= (std::numeric_limits<int64_t>::max)()) + { + if (add_prefix) + { + oa->write_character(to_char_type('L')); // int64 + } + write_number(static_cast<int64_t>(n)); + } + // LCOV_EXCL_START + else + { + JSON_THROW(out_of_range::create(407, "integer number " + std::to_string(n) + " cannot be represented by UBJSON as it does not fit int64")); + } + // LCOV_EXCL_STOP + } + + /*! + @brief determine the type prefix of container values + + @note This function does not need to be 100% accurate when it comes to + integer limits. In case a number exceeds the limits of int64_t, + this will be detected by a later call to function + write_number_with_ubjson_prefix. Therefore, we return 'L' for any + value that does not fit the previous limits. + */ + CharType ubjson_prefix(const BasicJsonType& j) const noexcept + { + switch (j.type()) + { + case value_t::null: + return 'Z'; + + case value_t::boolean: + return j.m_value.boolean ? 'T' : 'F'; + + case value_t::number_integer: + { + if ((std::numeric_limits<int8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + if ((std::numeric_limits<uint8_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + if ((std::numeric_limits<int16_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + if ((std::numeric_limits<int32_t>::min)() <= j.m_value.number_integer and j.m_value.number_integer <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_unsigned: + { + if (j.m_value.number_unsigned <= (std::numeric_limits<int8_t>::max)()) + { + return 'i'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<uint8_t>::max)()) + { + return 'U'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<int16_t>::max)()) + { + return 'I'; + } + if (j.m_value.number_unsigned <= (std::numeric_limits<int32_t>::max)()) + { + return 'l'; + } + // no check and assume int64_t (see note above) + return 'L'; + } + + case value_t::number_float: + return get_ubjson_float_prefix(j.m_value.number_float); + + case value_t::string: + return 'S'; + + case value_t::array: + return '['; + + case value_t::object: + return '{'; + + default: // discarded values + return 'N'; + } + } + + static constexpr CharType get_ubjson_float_prefix(float /*unused*/) + { + return 'd'; // float 32 + } + + static constexpr CharType get_ubjson_float_prefix(double /*unused*/) + { + return 'D'; // float 64 + } + + /////////////////////// + // Utility functions // + /////////////////////// + + /* + @brief write a number to output input + @param[in] n number of type @a NumberType + @tparam NumberType the type of the number + @tparam OutputIsLittleEndian Set to true if output data is + required to be little endian + + @note This function needs to respect the system's endianess, because bytes + in CBOR, MessagePack, and UBJSON are stored in network order (big + endian) and therefore need reordering on little endian systems. + */ + template<typename NumberType, bool OutputIsLittleEndian = false> + void write_number(const NumberType n) + { + // step 1: write number to array of length NumberType + std::array<CharType, sizeof(NumberType)> vec; + std::memcpy(vec.data(), &n, sizeof(NumberType)); + + // step 2: write array to output (with possible reordering) + if (is_little_endian and not OutputIsLittleEndian) + { + // reverse byte order prior to conversion if necessary + std::reverse(vec.begin(), vec.end()); + } + + oa->write_characters(vec.data(), sizeof(NumberType)); + } + + public: + // The following to_char_type functions are implement the conversion + // between uint8_t and CharType. In case CharType is not unsigned, + // such a conversion is required to allow values greater than 128. + // See <https://github.com/nlohmann/json/issues/1286> for a discussion. + template < typename C = CharType, + enable_if_t < std::is_signed<C>::value and std::is_signed<char>::value > * = nullptr > + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return *reinterpret_cast<char*>(&x); + } + + template < typename C = CharType, + enable_if_t < std::is_signed<C>::value and std::is_unsigned<char>::value > * = nullptr > + static CharType to_char_type(std::uint8_t x) noexcept + { + static_assert(sizeof(std::uint8_t) == sizeof(CharType), "size of CharType must be equal to std::uint8_t"); + static_assert(std::is_pod<CharType>::value, "CharType must be POD"); + CharType result; + std::memcpy(&result, &x, sizeof(x)); + return result; + } + + template<typename C = CharType, + enable_if_t<std::is_unsigned<C>::value>* = nullptr> + static constexpr CharType to_char_type(std::uint8_t x) noexcept + { + return x; + } + + template < typename InputCharType, typename C = CharType, + enable_if_t < + std::is_signed<C>::value and + std::is_signed<char>::value and + std::is_same<char, typename std::remove_cv<InputCharType>::type>::value + > * = nullptr > + static constexpr CharType to_char_type(InputCharType x) noexcept + { + return x; + } + + private: + /// whether we can assume little endianess + const bool is_little_endian = binary_reader<BasicJsonType>::little_endianess(); + + /// the output + output_adapter_t<CharType> oa = nullptr; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/output/serializer.hpp> + + +#include <algorithm> // reverse, remove, fill, find, none_of +#include <array> // array +#include <cassert> // assert +#include <ciso646> // and, or +#include <clocale> // localeconv, lconv +#include <cmath> // labs, isfinite, isnan, signbit +#include <cstddef> // size_t, ptrdiff_t +#include <cstdint> // uint8_t +#include <cstdio> // snprintf +#include <limits> // numeric_limits +#include <string> // string +#include <type_traits> // is_same + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/conversions/to_chars.hpp> + + +#include <cassert> // assert +#include <ciso646> // or, and, not +#include <cmath> // signbit, isfinite +#include <cstdint> // intN_t, uintN_t +#include <cstring> // memcpy, memmove + +namespace nlohmann +{ +namespace detail +{ + +/*! +@brief implements the Grisu2 algorithm for binary to decimal floating-point +conversion. + +This implementation is a slightly modified version of the reference +implementation which may be obtained from +http://florian.loitsch.com/publications (bench.tar.gz). + +The code is distributed under the MIT license, Copyright (c) 2009 Florian Loitsch. + +For a detailed description of the algorithm see: + +[1] Loitsch, "Printing Floating-Point Numbers Quickly and Accurately with + Integers", Proceedings of the ACM SIGPLAN 2010 Conference on Programming + Language Design and Implementation, PLDI 2010 +[2] Burger, Dybvig, "Printing Floating-Point Numbers Quickly and Accurately", + Proceedings of the ACM SIGPLAN 1996 Conference on Programming Language + Design and Implementation, PLDI 1996 +*/ +namespace dtoa_impl +{ + +template <typename Target, typename Source> +Target reinterpret_bits(const Source source) +{ + static_assert(sizeof(Target) == sizeof(Source), "size mismatch"); + + Target target; + std::memcpy(&target, &source, sizeof(Source)); + return target; +} + +struct diyfp // f * 2^e +{ + static constexpr int kPrecision = 64; // = q + + uint64_t f = 0; + int e = 0; + + constexpr diyfp(uint64_t f_, int e_) noexcept : f(f_), e(e_) {} + + /*! + @brief returns x - y + @pre x.e == y.e and x.f >= y.f + */ + static diyfp sub(const diyfp& x, const diyfp& y) noexcept + { + assert(x.e == y.e); + assert(x.f >= y.f); + + return {x.f - y.f, x.e}; + } + + /*! + @brief returns x * y + @note The result is rounded. (Only the upper q bits are returned.) + */ + static diyfp mul(const diyfp& x, const diyfp& y) noexcept + { + static_assert(kPrecision == 64, "internal error"); + + // Computes: + // f = round((x.f * y.f) / 2^q) + // e = x.e + y.e + q + + // Emulate the 64-bit * 64-bit multiplication: + // + // p = u * v + // = (u_lo + 2^32 u_hi) (v_lo + 2^32 v_hi) + // = (u_lo v_lo ) + 2^32 ((u_lo v_hi ) + (u_hi v_lo )) + 2^64 (u_hi v_hi ) + // = (p0 ) + 2^32 ((p1 ) + (p2 )) + 2^64 (p3 ) + // = (p0_lo + 2^32 p0_hi) + 2^32 ((p1_lo + 2^32 p1_hi) + (p2_lo + 2^32 p2_hi)) + 2^64 (p3 ) + // = (p0_lo ) + 2^32 (p0_hi + p1_lo + p2_lo ) + 2^64 (p1_hi + p2_hi + p3) + // = (p0_lo ) + 2^32 (Q ) + 2^64 (H ) + // = (p0_lo ) + 2^32 (Q_lo + 2^32 Q_hi ) + 2^64 (H ) + // + // (Since Q might be larger than 2^32 - 1) + // + // = (p0_lo + 2^32 Q_lo) + 2^64 (Q_hi + H) + // + // (Q_hi + H does not overflow a 64-bit int) + // + // = p_lo + 2^64 p_hi + + const uint64_t u_lo = x.f & 0xFFFFFFFF; + const uint64_t u_hi = x.f >> 32; + const uint64_t v_lo = y.f & 0xFFFFFFFF; + const uint64_t v_hi = y.f >> 32; + + const uint64_t p0 = u_lo * v_lo; + const uint64_t p1 = u_lo * v_hi; + const uint64_t p2 = u_hi * v_lo; + const uint64_t p3 = u_hi * v_hi; + + const uint64_t p0_hi = p0 >> 32; + const uint64_t p1_lo = p1 & 0xFFFFFFFF; + const uint64_t p1_hi = p1 >> 32; + const uint64_t p2_lo = p2 & 0xFFFFFFFF; + const uint64_t p2_hi = p2 >> 32; + + uint64_t Q = p0_hi + p1_lo + p2_lo; + + // The full product might now be computed as + // + // p_hi = p3 + p2_hi + p1_hi + (Q >> 32) + // p_lo = p0_lo + (Q << 32) + // + // But in this particular case here, the full p_lo is not required. + // Effectively we only need to add the highest bit in p_lo to p_hi (and + // Q_hi + 1 does not overflow). + + Q += uint64_t{1} << (64 - 32 - 1); // round, ties up + + const uint64_t h = p3 + p2_hi + p1_hi + (Q >> 32); + + return {h, x.e + y.e + 64}; + } + + /*! + @brief normalize x such that the significand is >= 2^(q-1) + @pre x.f != 0 + */ + static diyfp normalize(diyfp x) noexcept + { + assert(x.f != 0); + + while ((x.f >> 63) == 0) + { + x.f <<= 1; + x.e--; + } + + return x; + } + + /*! + @brief normalize x such that the result has the exponent E + @pre e >= x.e and the upper e - x.e bits of x.f must be zero. + */ + static diyfp normalize_to(const diyfp& x, const int target_exponent) noexcept + { + const int delta = x.e - target_exponent; + + assert(delta >= 0); + assert(((x.f << delta) >> delta) == x.f); + + return {x.f << delta, target_exponent}; + } +}; + +struct boundaries +{ + diyfp w; + diyfp minus; + diyfp plus; +}; + +/*! +Compute the (normalized) diyfp representing the input number 'value' and its +boundaries. + +@pre value must be finite and positive +*/ +template <typename FloatType> +boundaries compute_boundaries(FloatType value) +{ + assert(std::isfinite(value)); + assert(value > 0); + + // Convert the IEEE representation into a diyfp. + // + // If v is denormal: + // value = 0.F * 2^(1 - bias) = ( F) * 2^(1 - bias - (p-1)) + // If v is normalized: + // value = 1.F * 2^(E - bias) = (2^(p-1) + F) * 2^(E - bias - (p-1)) + + static_assert(std::numeric_limits<FloatType>::is_iec559, + "internal error: dtoa_short requires an IEEE-754 floating-point implementation"); + + constexpr int kPrecision = std::numeric_limits<FloatType>::digits; // = p (includes the hidden bit) + constexpr int kBias = std::numeric_limits<FloatType>::max_exponent - 1 + (kPrecision - 1); + constexpr int kMinExp = 1 - kBias; + constexpr uint64_t kHiddenBit = uint64_t{1} << (kPrecision - 1); // = 2^(p-1) + + using bits_type = typename std::conditional< kPrecision == 24, uint32_t, uint64_t >::type; + + const uint64_t bits = reinterpret_bits<bits_type>(value); + const uint64_t E = bits >> (kPrecision - 1); + const uint64_t F = bits & (kHiddenBit - 1); + + const bool is_denormal = (E == 0); + const diyfp v = is_denormal + ? diyfp(F, kMinExp) + : diyfp(F + kHiddenBit, static_cast<int>(E) - kBias); + + // Compute the boundaries m- and m+ of the floating-point value + // v = f * 2^e. + // + // Determine v- and v+, the floating-point predecessor and successor if v, + // respectively. + // + // v- = v - 2^e if f != 2^(p-1) or e == e_min (A) + // = v - 2^(e-1) if f == 2^(p-1) and e > e_min (B) + // + // v+ = v + 2^e + // + // Let m- = (v- + v) / 2 and m+ = (v + v+) / 2. All real numbers _strictly_ + // between m- and m+ round to v, regardless of how the input rounding + // algorithm breaks ties. + // + // ---+-------------+-------------+-------------+-------------+--- (A) + // v- m- v m+ v+ + // + // -----------------+------+------+-------------+-------------+--- (B) + // v- m- v m+ v+ + + const bool lower_boundary_is_closer = (F == 0 and E > 1); + const diyfp m_plus = diyfp(2 * v.f + 1, v.e - 1); + const diyfp m_minus = lower_boundary_is_closer + ? diyfp(4 * v.f - 1, v.e - 2) // (B) + : diyfp(2 * v.f - 1, v.e - 1); // (A) + + // Determine the normalized w+ = m+. + const diyfp w_plus = diyfp::normalize(m_plus); + + // Determine w- = m- such that e_(w-) = e_(w+). + const diyfp w_minus = diyfp::normalize_to(m_minus, w_plus.e); + + return {diyfp::normalize(v), w_minus, w_plus}; +} + +// Given normalized diyfp w, Grisu needs to find a (normalized) cached +// power-of-ten c, such that the exponent of the product c * w = f * 2^e lies +// within a certain range [alpha, gamma] (Definition 3.2 from [1]) +// +// alpha <= e = e_c + e_w + q <= gamma +// +// or +// +// f_c * f_w * 2^alpha <= f_c 2^(e_c) * f_w 2^(e_w) * 2^q +// <= f_c * f_w * 2^gamma +// +// Since c and w are normalized, i.e. 2^(q-1) <= f < 2^q, this implies +// +// 2^(q-1) * 2^(q-1) * 2^alpha <= c * w * 2^q < 2^q * 2^q * 2^gamma +// +// or +// +// 2^(q - 2 + alpha) <= c * w < 2^(q + gamma) +// +// The choice of (alpha,gamma) determines the size of the table and the form of +// the digit generation procedure. Using (alpha,gamma)=(-60,-32) works out well +// in practice: +// +// The idea is to cut the number c * w = f * 2^e into two parts, which can be +// processed independently: An integral part p1, and a fractional part p2: +// +// f * 2^e = ( (f div 2^-e) * 2^-e + (f mod 2^-e) ) * 2^e +// = (f div 2^-e) + (f mod 2^-e) * 2^e +// = p1 + p2 * 2^e +// +// The conversion of p1 into decimal form requires a series of divisions and +// modulos by (a power of) 10. These operations are faster for 32-bit than for +// 64-bit integers, so p1 should ideally fit into a 32-bit integer. This can be +// achieved by choosing +// +// -e >= 32 or e <= -32 := gamma +// +// In order to convert the fractional part +// +// p2 * 2^e = p2 / 2^-e = d[-1] / 10^1 + d[-2] / 10^2 + ... +// +// into decimal form, the fraction is repeatedly multiplied by 10 and the digits +// d[-i] are extracted in order: +// +// (10 * p2) div 2^-e = d[-1] +// (10 * p2) mod 2^-e = d[-2] / 10^1 + ... +// +// The multiplication by 10 must not overflow. It is sufficient to choose +// +// 10 * p2 < 16 * p2 = 2^4 * p2 <= 2^64. +// +// Since p2 = f mod 2^-e < 2^-e, +// +// -e <= 60 or e >= -60 := alpha + +constexpr int kAlpha = -60; +constexpr int kGamma = -32; + +struct cached_power // c = f * 2^e ~= 10^k +{ + uint64_t f; + int e; + int k; +}; + +/*! +For a normalized diyfp w = f * 2^e, this function returns a (normalized) cached +power-of-ten c = f_c * 2^e_c, such that the exponent of the product w * c +satisfies (Definition 3.2 from [1]) + + alpha <= e_c + e + q <= gamma. +*/ +inline cached_power get_cached_power_for_binary_exponent(int e) +{ + // Now + // + // alpha <= e_c + e + q <= gamma (1) + // ==> f_c * 2^alpha <= c * 2^e * 2^q + // + // and since the c's are normalized, 2^(q-1) <= f_c, + // + // ==> 2^(q - 1 + alpha) <= c * 2^(e + q) + // ==> 2^(alpha - e - 1) <= c + // + // If c were an exakt power of ten, i.e. c = 10^k, one may determine k as + // + // k = ceil( log_10( 2^(alpha - e - 1) ) ) + // = ceil( (alpha - e - 1) * log_10(2) ) + // + // From the paper: + // "In theory the result of the procedure could be wrong since c is rounded, + // and the computation itself is approximated [...]. In practice, however, + // this simple function is sufficient." + // + // For IEEE double precision floating-point numbers converted into + // normalized diyfp's w = f * 2^e, with q = 64, + // + // e >= -1022 (min IEEE exponent) + // -52 (p - 1) + // -52 (p - 1, possibly normalize denormal IEEE numbers) + // -11 (normalize the diyfp) + // = -1137 + // + // and + // + // e <= +1023 (max IEEE exponent) + // -52 (p - 1) + // -11 (normalize the diyfp) + // = 960 + // + // This binary exponent range [-1137,960] results in a decimal exponent + // range [-307,324]. One does not need to store a cached power for each + // k in this range. For each such k it suffices to find a cached power + // such that the exponent of the product lies in [alpha,gamma]. + // This implies that the difference of the decimal exponents of adjacent + // table entries must be less than or equal to + // + // floor( (gamma - alpha) * log_10(2) ) = 8. + // + // (A smaller distance gamma-alpha would require a larger table.) + + // NB: + // Actually this function returns c, such that -60 <= e_c + e + 64 <= -34. + + constexpr int kCachedPowersSize = 79; + constexpr int kCachedPowersMinDecExp = -300; + constexpr int kCachedPowersDecStep = 8; + + static constexpr cached_power kCachedPowers[] = + { + { 0xAB70FE17C79AC6CA, -1060, -300 }, + { 0xFF77B1FCBEBCDC4F, -1034, -292 }, + { 0xBE5691EF416BD60C, -1007, -284 }, + { 0x8DD01FAD907FFC3C, -980, -276 }, + { 0xD3515C2831559A83, -954, -268 }, + { 0x9D71AC8FADA6C9B5, -927, -260 }, + { 0xEA9C227723EE8BCB, -901, -252 }, + { 0xAECC49914078536D, -874, -244 }, + { 0x823C12795DB6CE57, -847, -236 }, + { 0xC21094364DFB5637, -821, -228 }, + { 0x9096EA6F3848984F, -794, -220 }, + { 0xD77485CB25823AC7, -768, -212 }, + { 0xA086CFCD97BF97F4, -741, -204 }, + { 0xEF340A98172AACE5, -715, -196 }, + { 0xB23867FB2A35B28E, -688, -188 }, + { 0x84C8D4DFD2C63F3B, -661, -180 }, + { 0xC5DD44271AD3CDBA, -635, -172 }, + { 0x936B9FCEBB25C996, -608, -164 }, + { 0xDBAC6C247D62A584, -582, -156 }, + { 0xA3AB66580D5FDAF6, -555, -148 }, + { 0xF3E2F893DEC3F126, -529, -140 }, + { 0xB5B5ADA8AAFF80B8, -502, -132 }, + { 0x87625F056C7C4A8B, -475, -124 }, + { 0xC9BCFF6034C13053, -449, -116 }, + { 0x964E858C91BA2655, -422, -108 }, + { 0xDFF9772470297EBD, -396, -100 }, + { 0xA6DFBD9FB8E5B88F, -369, -92 }, + { 0xF8A95FCF88747D94, -343, -84 }, + { 0xB94470938FA89BCF, -316, -76 }, + { 0x8A08F0F8BF0F156B, -289, -68 }, + { 0xCDB02555653131B6, -263, -60 }, + { 0x993FE2C6D07B7FAC, -236, -52 }, + { 0xE45C10C42A2B3B06, -210, -44 }, + { 0xAA242499697392D3, -183, -36 }, + { 0xFD87B5F28300CA0E, -157, -28 }, + { 0xBCE5086492111AEB, -130, -20 }, + { 0x8CBCCC096F5088CC, -103, -12 }, + { 0xD1B71758E219652C, -77, -4 }, + { 0x9C40000000000000, -50, 4 }, + { 0xE8D4A51000000000, -24, 12 }, + { 0xAD78EBC5AC620000, 3, 20 }, + { 0x813F3978F8940984, 30, 28 }, + { 0xC097CE7BC90715B3, 56, 36 }, + { 0x8F7E32CE7BEA5C70, 83, 44 }, + { 0xD5D238A4ABE98068, 109, 52 }, + { 0x9F4F2726179A2245, 136, 60 }, + { 0xED63A231D4C4FB27, 162, 68 }, + { 0xB0DE65388CC8ADA8, 189, 76 }, + { 0x83C7088E1AAB65DB, 216, 84 }, + { 0xC45D1DF942711D9A, 242, 92 }, + { 0x924D692CA61BE758, 269, 100 }, + { 0xDA01EE641A708DEA, 295, 108 }, + { 0xA26DA3999AEF774A, 322, 116 }, + { 0xF209787BB47D6B85, 348, 124 }, + { 0xB454E4A179DD1877, 375, 132 }, + { 0x865B86925B9BC5C2, 402, 140 }, + { 0xC83553C5C8965D3D, 428, 148 }, + { 0x952AB45CFA97A0B3, 455, 156 }, + { 0xDE469FBD99A05FE3, 481, 164 }, + { 0xA59BC234DB398C25, 508, 172 }, + { 0xF6C69A72A3989F5C, 534, 180 }, + { 0xB7DCBF5354E9BECE, 561, 188 }, + { 0x88FCF317F22241E2, 588, 196 }, + { 0xCC20CE9BD35C78A5, 614, 204 }, + { 0x98165AF37B2153DF, 641, 212 }, + { 0xE2A0B5DC971F303A, 667, 220 }, + { 0xA8D9D1535CE3B396, 694, 228 }, + { 0xFB9B7CD9A4A7443C, 720, 236 }, + { 0xBB764C4CA7A44410, 747, 244 }, + { 0x8BAB8EEFB6409C1A, 774, 252 }, + { 0xD01FEF10A657842C, 800, 260 }, + { 0x9B10A4E5E9913129, 827, 268 }, + { 0xE7109BFBA19C0C9D, 853, 276 }, + { 0xAC2820D9623BF429, 880, 284 }, + { 0x80444B5E7AA7CF85, 907, 292 }, + { 0xBF21E44003ACDD2D, 933, 300 }, + { 0x8E679C2F5E44FF8F, 960, 308 }, + { 0xD433179D9C8CB841, 986, 316 }, + { 0x9E19DB92B4E31BA9, 1013, 324 }, + }; + + // This computation gives exactly the same results for k as + // k = ceil((kAlpha - e - 1) * 0.30102999566398114) + // for |e| <= 1500, but doesn't require floating-point operations. + // NB: log_10(2) ~= 78913 / 2^18 + assert(e >= -1500); + assert(e <= 1500); + const int f = kAlpha - e - 1; + const int k = (f * 78913) / (1 << 18) + static_cast<int>(f > 0); + + const int index = (-kCachedPowersMinDecExp + k + (kCachedPowersDecStep - 1)) / kCachedPowersDecStep; + assert(index >= 0); + assert(index < kCachedPowersSize); + static_cast<void>(kCachedPowersSize); // Fix warning. + + const cached_power cached = kCachedPowers[index]; + assert(kAlpha <= cached.e + e + 64); + assert(kGamma >= cached.e + e + 64); + + return cached; +} + +/*! +For n != 0, returns k, such that pow10 := 10^(k-1) <= n < 10^k. +For n == 0, returns 1 and sets pow10 := 1. +*/ +inline int find_largest_pow10(const uint32_t n, uint32_t& pow10) +{ + // LCOV_EXCL_START + if (n >= 1000000000) + { + pow10 = 1000000000; + return 10; + } + // LCOV_EXCL_STOP + else if (n >= 100000000) + { + pow10 = 100000000; + return 9; + } + else if (n >= 10000000) + { + pow10 = 10000000; + return 8; + } + else if (n >= 1000000) + { + pow10 = 1000000; + return 7; + } + else if (n >= 100000) + { + pow10 = 100000; + return 6; + } + else if (n >= 10000) + { + pow10 = 10000; + return 5; + } + else if (n >= 1000) + { + pow10 = 1000; + return 4; + } + else if (n >= 100) + { + pow10 = 100; + return 3; + } + else if (n >= 10) + { + pow10 = 10; + return 2; + } + else + { + pow10 = 1; + return 1; + } +} + +inline void grisu2_round(char* buf, int len, uint64_t dist, uint64_t delta, + uint64_t rest, uint64_t ten_k) +{ + assert(len >= 1); + assert(dist <= delta); + assert(rest <= delta); + assert(ten_k > 0); + + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // ten_k + // <------> + // <---- rest ----> + // --------------[------------------+----+--------------]-------------- + // w V + // = buf * 10^k + // + // ten_k represents a unit-in-the-last-place in the decimal representation + // stored in buf. + // Decrement buf by ten_k while this takes buf closer to w. + + // The tests are written in this order to avoid overflow in unsigned + // integer arithmetic. + + while (rest < dist + and delta - rest >= ten_k + and (rest + ten_k < dist or dist - rest > rest + ten_k - dist)) + { + assert(buf[len - 1] != '0'); + buf[len - 1]--; + rest += ten_k; + } +} + +/*! +Generates V = buffer * 10^decimal_exponent, such that M- <= V <= M+. +M- and M+ must be normalized and share the same exponent -60 <= e <= -32. +*/ +inline void grisu2_digit_gen(char* buffer, int& length, int& decimal_exponent, + diyfp M_minus, diyfp w, diyfp M_plus) +{ + static_assert(kAlpha >= -60, "internal error"); + static_assert(kGamma <= -32, "internal error"); + + // Generates the digits (and the exponent) of a decimal floating-point + // number V = buffer * 10^decimal_exponent in the range [M-, M+]. The diyfp's + // w, M- and M+ share the same exponent e, which satisfies alpha <= e <= gamma. + // + // <--------------------------- delta ----> + // <---- dist ---------> + // --------------[------------------+-------------------]-------------- + // M- w M+ + // + // Grisu2 generates the digits of M+ from left to right and stops as soon as + // V is in [M-,M+]. + + assert(M_plus.e >= kAlpha); + assert(M_plus.e <= kGamma); + + uint64_t delta = diyfp::sub(M_plus, M_minus).f; // (significand of (M+ - M-), implicit exponent is e) + uint64_t dist = diyfp::sub(M_plus, w ).f; // (significand of (M+ - w ), implicit exponent is e) + + // Split M+ = f * 2^e into two parts p1 and p2 (note: e < 0): + // + // M+ = f * 2^e + // = ((f div 2^-e) * 2^-e + (f mod 2^-e)) * 2^e + // = ((p1 ) * 2^-e + (p2 )) * 2^e + // = p1 + p2 * 2^e + + const diyfp one(uint64_t{1} << -M_plus.e, M_plus.e); + + auto p1 = static_cast<uint32_t>(M_plus.f >> -one.e); // p1 = f div 2^-e (Since -e >= 32, p1 fits into a 32-bit int.) + uint64_t p2 = M_plus.f & (one.f - 1); // p2 = f mod 2^-e + + // 1) + // + // Generate the digits of the integral part p1 = d[n-1]...d[1]d[0] + + assert(p1 > 0); + + uint32_t pow10; + const int k = find_largest_pow10(p1, pow10); + + // 10^(k-1) <= p1 < 10^k, pow10 = 10^(k-1) + // + // p1 = (p1 div 10^(k-1)) * 10^(k-1) + (p1 mod 10^(k-1)) + // = (d[k-1] ) * 10^(k-1) + (p1 mod 10^(k-1)) + // + // M+ = p1 + p2 * 2^e + // = d[k-1] * 10^(k-1) + (p1 mod 10^(k-1)) + p2 * 2^e + // = d[k-1] * 10^(k-1) + ((p1 mod 10^(k-1)) * 2^-e + p2) * 2^e + // = d[k-1] * 10^(k-1) + ( rest) * 2^e + // + // Now generate the digits d[n] of p1 from left to right (n = k-1,...,0) + // + // p1 = d[k-1]...d[n] * 10^n + d[n-1]...d[0] + // + // but stop as soon as + // + // rest * 2^e = (d[n-1]...d[0] * 2^-e + p2) * 2^e <= delta * 2^e + + int n = k; + while (n > 0) + { + // Invariants: + // M+ = buffer * 10^n + (p1 + p2 * 2^e) (buffer = 0 for n = k) + // pow10 = 10^(n-1) <= p1 < 10^n + // + const uint32_t d = p1 / pow10; // d = p1 div 10^(n-1) + const uint32_t r = p1 % pow10; // r = p1 mod 10^(n-1) + // + // M+ = buffer * 10^n + (d * 10^(n-1) + r) + p2 * 2^e + // = (buffer * 10 + d) * 10^(n-1) + (r + p2 * 2^e) + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(n-1) + (r + p2 * 2^e) + // + p1 = r; + n--; + // + // M+ = buffer * 10^n + (p1 + p2 * 2^e) + // pow10 = 10^n + // + + // Now check if enough digits have been generated. + // Compute + // + // p1 + p2 * 2^e = (p1 * 2^-e + p2) * 2^e = rest * 2^e + // + // Note: + // Since rest and delta share the same exponent e, it suffices to + // compare the significands. + const uint64_t rest = (uint64_t{p1} << -one.e) + p2; + if (rest <= delta) + { + // V = buffer * 10^n, with M- <= V <= M+. + + decimal_exponent += n; + + // We may now just stop. But instead look if the buffer could be + // decremented to bring V closer to w. + // + // pow10 = 10^n is now 1 ulp in the decimal representation V. + // The rounding procedure works with diyfp's with an implicit + // exponent of e. + // + // 10^n = (10^n * 2^-e) * 2^e = ulp * 2^e + // + const uint64_t ten_n = uint64_t{pow10} << -one.e; + grisu2_round(buffer, length, dist, delta, rest, ten_n); + + return; + } + + pow10 /= 10; + // + // pow10 = 10^(n-1) <= p1 < 10^n + // Invariants restored. + } + + // 2) + // + // The digits of the integral part have been generated: + // + // M+ = d[k-1]...d[1]d[0] + p2 * 2^e + // = buffer + p2 * 2^e + // + // Now generate the digits of the fractional part p2 * 2^e. + // + // Note: + // No decimal point is generated: the exponent is adjusted instead. + // + // p2 actually represents the fraction + // + // p2 * 2^e + // = p2 / 2^-e + // = d[-1] / 10^1 + d[-2] / 10^2 + ... + // + // Now generate the digits d[-m] of p1 from left to right (m = 1,2,...) + // + // p2 * 2^e = d[-1]d[-2]...d[-m] * 10^-m + // + 10^-m * (d[-m-1] / 10^1 + d[-m-2] / 10^2 + ...) + // + // using + // + // 10^m * p2 = ((10^m * p2) div 2^-e) * 2^-e + ((10^m * p2) mod 2^-e) + // = ( d) * 2^-e + ( r) + // + // or + // 10^m * p2 * 2^e = d + r * 2^e + // + // i.e. + // + // M+ = buffer + p2 * 2^e + // = buffer + 10^-m * (d + r * 2^e) + // = (buffer * 10^m + d) * 10^-m + 10^-m * r * 2^e + // + // and stop as soon as 10^-m * r * 2^e <= delta * 2^e + + assert(p2 > delta); + + int m = 0; + for (;;) + { + // Invariant: + // M+ = buffer * 10^-m + 10^-m * (d[-m-1] / 10 + d[-m-2] / 10^2 + ...) * 2^e + // = buffer * 10^-m + 10^-m * (p2 ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (10 * p2) ) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * ((10*p2 div 2^-e) * 2^-e + (10*p2 mod 2^-e)) * 2^e + // + assert(p2 <= UINT64_MAX / 10); + p2 *= 10; + const uint64_t d = p2 >> -one.e; // d = (10 * p2) div 2^-e + const uint64_t r = p2 & (one.f - 1); // r = (10 * p2) mod 2^-e + // + // M+ = buffer * 10^-m + 10^-m * (1/10 * (d * 2^-e + r) * 2^e + // = buffer * 10^-m + 10^-m * (1/10 * (d + r * 2^e)) + // = (buffer * 10 + d) * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + assert(d <= 9); + buffer[length++] = static_cast<char>('0' + d); // buffer := buffer * 10 + d + // + // M+ = buffer * 10^(-m-1) + 10^(-m-1) * r * 2^e + // + p2 = r; + m++; + // + // M+ = buffer * 10^-m + 10^-m * p2 * 2^e + // Invariant restored. + + // Check if enough digits have been generated. + // + // 10^-m * p2 * 2^e <= delta * 2^e + // p2 * 2^e <= 10^m * delta * 2^e + // p2 <= 10^m * delta + delta *= 10; + dist *= 10; + if (p2 <= delta) + { + break; + } + } + + // V = buffer * 10^-m, with M- <= V <= M+. + + decimal_exponent -= m; + + // 1 ulp in the decimal representation is now 10^-m. + // Since delta and dist are now scaled by 10^m, we need to do the + // same with ulp in order to keep the units in sync. + // + // 10^m * 10^-m = 1 = 2^-e * 2^e = ten_m * 2^e + // + const uint64_t ten_m = one.f; + grisu2_round(buffer, length, dist, delta, p2, ten_m); + + // By construction this algorithm generates the shortest possible decimal + // number (Loitsch, Theorem 6.2) which rounds back to w. + // For an input number of precision p, at least + // + // N = 1 + ceil(p * log_10(2)) + // + // decimal digits are sufficient to identify all binary floating-point + // numbers (Matula, "In-and-Out conversions"). + // This implies that the algorithm does not produce more than N decimal + // digits. + // + // N = 17 for p = 53 (IEEE double precision) + // N = 9 for p = 24 (IEEE single precision) +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +inline void grisu2(char* buf, int& len, int& decimal_exponent, + diyfp m_minus, diyfp v, diyfp m_plus) +{ + assert(m_plus.e == m_minus.e); + assert(m_plus.e == v.e); + + // --------(-----------------------+-----------------------)-------- (A) + // m- v m+ + // + // --------------------(-----------+-----------------------)-------- (B) + // m- v m+ + // + // First scale v (and m- and m+) such that the exponent is in the range + // [alpha, gamma]. + + const cached_power cached = get_cached_power_for_binary_exponent(m_plus.e); + + const diyfp c_minus_k(cached.f, cached.e); // = c ~= 10^-k + + // The exponent of the products is = v.e + c_minus_k.e + q and is in the range [alpha,gamma] + const diyfp w = diyfp::mul(v, c_minus_k); + const diyfp w_minus = diyfp::mul(m_minus, c_minus_k); + const diyfp w_plus = diyfp::mul(m_plus, c_minus_k); + + // ----(---+---)---------------(---+---)---------------(---+---)---- + // w- w w+ + // = c*m- = c*v = c*m+ + // + // diyfp::mul rounds its result and c_minus_k is approximated too. w, w- and + // w+ are now off by a small amount. + // In fact: + // + // w - v * 10^k < 1 ulp + // + // To account for this inaccuracy, add resp. subtract 1 ulp. + // + // --------+---[---------------(---+---)---------------]---+-------- + // w- M- w M+ w+ + // + // Now any number in [M-, M+] (bounds included) will round to w when input, + // regardless of how the input rounding algorithm breaks ties. + // + // And digit_gen generates the shortest possible such number in [M-, M+]. + // Note that this does not mean that Grisu2 always generates the shortest + // possible number in the interval (m-, m+). + const diyfp M_minus(w_minus.f + 1, w_minus.e); + const diyfp M_plus (w_plus.f - 1, w_plus.e ); + + decimal_exponent = -cached.k; // = -(-k) = k + + grisu2_digit_gen(buf, len, decimal_exponent, M_minus, w, M_plus); +} + +/*! +v = buf * 10^decimal_exponent +len is the length of the buffer (number of decimal digits) +The buffer must be large enough, i.e. >= max_digits10. +*/ +template <typename FloatType> +void grisu2(char* buf, int& len, int& decimal_exponent, FloatType value) +{ + static_assert(diyfp::kPrecision >= std::numeric_limits<FloatType>::digits + 3, + "internal error: not enough precision"); + + assert(std::isfinite(value)); + assert(value > 0); + + // If the neighbors (and boundaries) of 'value' are always computed for double-precision + // numbers, all float's can be recovered using strtod (and strtof). However, the resulting + // decimal representations are not exactly "short". + // + // The documentation for 'std::to_chars' (https://en.cppreference.com/w/cpp/utility/to_chars) + // says "value is converted to a string as if by std::sprintf in the default ("C") locale" + // and since sprintf promotes float's to double's, I think this is exactly what 'std::to_chars' + // does. + // On the other hand, the documentation for 'std::to_chars' requires that "parsing the + // representation using the corresponding std::from_chars function recovers value exactly". That + // indicates that single precision floating-point numbers should be recovered using + // 'std::strtof'. + // + // NB: If the neighbors are computed for single-precision numbers, there is a single float + // (7.0385307e-26f) which can't be recovered using strtod. The resulting double precision + // value is off by 1 ulp. +#if 0 + const boundaries w = compute_boundaries(static_cast<double>(value)); +#else + const boundaries w = compute_boundaries(value); +#endif + + grisu2(buf, len, decimal_exponent, w.minus, w.w, w.plus); +} + +/*! +@brief appends a decimal representation of e to buf +@return a pointer to the element following the exponent. +@pre -1000 < e < 1000 +*/ +inline char* append_exponent(char* buf, int e) +{ + assert(e > -1000); + assert(e < 1000); + + if (e < 0) + { + e = -e; + *buf++ = '-'; + } + else + { + *buf++ = '+'; + } + + auto k = static_cast<uint32_t>(e); + if (k < 10) + { + // Always print at least two digits in the exponent. + // This is for compatibility with printf("%g"). + *buf++ = '0'; + *buf++ = static_cast<char>('0' + k); + } + else if (k < 100) + { + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + else + { + *buf++ = static_cast<char>('0' + k / 100); + k %= 100; + *buf++ = static_cast<char>('0' + k / 10); + k %= 10; + *buf++ = static_cast<char>('0' + k); + } + + return buf; +} + +/*! +@brief prettify v = buf * 10^decimal_exponent + +If v is in the range [10^min_exp, 10^max_exp) it will be printed in fixed-point +notation. Otherwise it will be printed in exponential notation. + +@pre min_exp < 0 +@pre max_exp > 0 +*/ +inline char* format_buffer(char* buf, int len, int decimal_exponent, + int min_exp, int max_exp) +{ + assert(min_exp < 0); + assert(max_exp > 0); + + const int k = len; + const int n = len + decimal_exponent; + + // v = buf * 10^(n-k) + // k is the length of the buffer (number of decimal digits) + // n is the position of the decimal point relative to the start of the buffer. + + if (k <= n and n <= max_exp) + { + // digits[000] + // len <= max_exp + 2 + + std::memset(buf + k, '0', static_cast<size_t>(n - k)); + // Make it look like a floating-point number (#362, #378) + buf[n + 0] = '.'; + buf[n + 1] = '0'; + return buf + (n + 2); + } + + if (0 < n and n <= max_exp) + { + // dig.its + // len <= max_digits10 + 1 + + assert(k > n); + + std::memmove(buf + (n + 1), buf + n, static_cast<size_t>(k - n)); + buf[n] = '.'; + return buf + (k + 1); + } + + if (min_exp < n and n <= 0) + { + // 0.[000]digits + // len <= 2 + (-min_exp - 1) + max_digits10 + + std::memmove(buf + (2 + -n), buf, static_cast<size_t>(k)); + buf[0] = '0'; + buf[1] = '.'; + std::memset(buf + 2, '0', static_cast<size_t>(-n)); + return buf + (2 + (-n) + k); + } + + if (k == 1) + { + // dE+123 + // len <= 1 + 5 + + buf += 1; + } + else + { + // d.igitsE+123 + // len <= max_digits10 + 1 + 5 + + std::memmove(buf + 2, buf + 1, static_cast<size_t>(k - 1)); + buf[1] = '.'; + buf += 1 + k; + } + + *buf++ = 'e'; + return append_exponent(buf, n - 1); +} + +} // namespace dtoa_impl + +/*! +@brief generates a decimal representation of the floating-point number value in [first, last). + +The format of the resulting decimal representation is similar to printf's %g +format. Returns an iterator pointing past-the-end of the decimal representation. + +@note The input number must be finite, i.e. NaN's and Inf's are not supported. +@note The buffer must be large enough. +@note The result is NOT null-terminated. +*/ +template <typename FloatType> +char* to_chars(char* first, const char* last, FloatType value) +{ + static_cast<void>(last); // maybe unused - fix warning + assert(std::isfinite(value)); + + // Use signbit(value) instead of (value < 0) since signbit works for -0. + if (std::signbit(value)) + { + value = -value; + *first++ = '-'; + } + + if (value == 0) // +-0 + { + *first++ = '0'; + // Make it look like a floating-point number (#362, #378) + *first++ = '.'; + *first++ = '0'; + return first; + } + + assert(last - first >= std::numeric_limits<FloatType>::max_digits10); + + // Compute v = buffer * 10^decimal_exponent. + // The decimal digits are stored in the buffer, which needs to be interpreted + // as an unsigned decimal integer. + // len is the length of the buffer, i.e. the number of decimal digits. + int len = 0; + int decimal_exponent = 0; + dtoa_impl::grisu2(first, len, decimal_exponent, value); + + assert(len <= std::numeric_limits<FloatType>::max_digits10); + + // Format the buffer like printf("%.*g", prec, value) + constexpr int kMinExp = -4; + // Use digits10 here to increase compatibility with version 2. + constexpr int kMaxExp = std::numeric_limits<FloatType>::digits10; + + assert(last - first >= kMaxExp + 2); + assert(last - first >= 2 + (-kMinExp - 1) + std::numeric_limits<FloatType>::max_digits10); + assert(last - first >= std::numeric_limits<FloatType>::max_digits10 + 6); + + return dtoa_impl::format_buffer(first, len, decimal_exponent, kMinExp, kMaxExp); +} + +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/meta/cpp_future.hpp> + +// #include <nlohmann/detail/output/binary_writer.hpp> + +// #include <nlohmann/detail/output/output_adapters.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +namespace detail +{ +/////////////////// +// serialization // +/////////////////// + +/// how to treat decoding errors +enum class error_handler_t +{ + strict, ///< throw a type_error exception in case of invalid UTF-8 + replace, ///< replace invalid UTF-8 sequences with U+FFFD + ignore ///< ignore invalid UTF-8 sequences +}; + +template<typename BasicJsonType> +class serializer +{ + using string_t = typename BasicJsonType::string_t; + using number_float_t = typename BasicJsonType::number_float_t; + using number_integer_t = typename BasicJsonType::number_integer_t; + using number_unsigned_t = typename BasicJsonType::number_unsigned_t; + static constexpr uint8_t UTF8_ACCEPT = 0; + static constexpr uint8_t UTF8_REJECT = 1; + + public: + /*! + @param[in] s output stream to serialize to + @param[in] ichar indentation character to use + @param[in] error_handler_ how to react on decoding errors + */ + serializer(output_adapter_t<char> s, const char ichar, + error_handler_t error_handler_ = error_handler_t::strict) + : o(std::move(s)) + , loc(std::localeconv()) + , thousands_sep(loc->thousands_sep == nullptr ? '\0' : * (loc->thousands_sep)) + , decimal_point(loc->decimal_point == nullptr ? '\0' : * (loc->decimal_point)) + , indent_char(ichar) + , indent_string(512, indent_char) + , error_handler(error_handler_) + {} + + // delete because of pointer members + serializer(const serializer&) = delete; + serializer& operator=(const serializer&) = delete; + serializer(serializer&&) = delete; + serializer& operator=(serializer&&) = delete; + ~serializer() = default; + + /*! + @brief internal implementation of the serialization function + + This function is called by the public member function dump and organizes + the serialization internally. The indentation level is propagated as + additional parameter. In case of arrays and objects, the function is + called recursively. + + - strings and object keys are escaped using `escape_string()` + - integer numbers are converted implicitly via `operator<<` + - floating-point numbers are converted to a string using `"%g"` format + + @param[in] val value to serialize + @param[in] pretty_print whether the output shall be pretty-printed + @param[in] indent_step the indent level + @param[in] current_indent the current indent level (only used internally) + */ + void dump(const BasicJsonType& val, const bool pretty_print, + const bool ensure_ascii, + const unsigned int indent_step, + const unsigned int current_indent = 0) + { + switch (val.m_type) + { + case value_t::object: + { + if (val.m_value.object->empty()) + { + o->write_characters("{}", 2); + return; + } + + if (pretty_print) + { + o->write_characters("{\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + assert(i != val.m_value.object->cend()); + assert(std::next(i) == val.m_value.object->cend()); + o->write_characters(indent_string.c_str(), new_indent); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\": ", 3); + dump(i->second, true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character('}'); + } + else + { + o->write_character('{'); + + // first n-1 elements + auto i = val.m_value.object->cbegin(); + for (std::size_t cnt = 0; cnt < val.m_value.object->size() - 1; ++cnt, ++i) + { + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + assert(i != val.m_value.object->cend()); + assert(std::next(i) == val.m_value.object->cend()); + o->write_character('\"'); + dump_escaped(i->first, ensure_ascii); + o->write_characters("\":", 2); + dump(i->second, false, ensure_ascii, indent_step, current_indent); + + o->write_character('}'); + } + + return; + } + + case value_t::array: + { + if (val.m_value.array->empty()) + { + o->write_characters("[]", 2); + return; + } + + if (pretty_print) + { + o->write_characters("[\n", 2); + + // variable to hold indentation for recursive calls + const auto new_indent = current_indent + indent_step; + if (JSON_UNLIKELY(indent_string.size() < new_indent)) + { + indent_string.resize(indent_string.size() * 2, ' '); + } + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + o->write_characters(indent_string.c_str(), new_indent); + dump(*i, true, ensure_ascii, indent_step, new_indent); + o->write_characters(",\n", 2); + } + + // last element + assert(not val.m_value.array->empty()); + o->write_characters(indent_string.c_str(), new_indent); + dump(val.m_value.array->back(), true, ensure_ascii, indent_step, new_indent); + + o->write_character('\n'); + o->write_characters(indent_string.c_str(), current_indent); + o->write_character(']'); + } + else + { + o->write_character('['); + + // first n-1 elements + for (auto i = val.m_value.array->cbegin(); + i != val.m_value.array->cend() - 1; ++i) + { + dump(*i, false, ensure_ascii, indent_step, current_indent); + o->write_character(','); + } + + // last element + assert(not val.m_value.array->empty()); + dump(val.m_value.array->back(), false, ensure_ascii, indent_step, current_indent); + + o->write_character(']'); + } + + return; + } + + case value_t::string: + { + o->write_character('\"'); + dump_escaped(*val.m_value.string, ensure_ascii); + o->write_character('\"'); + return; + } + + case value_t::boolean: + { + if (val.m_value.boolean) + { + o->write_characters("true", 4); + } + else + { + o->write_characters("false", 5); + } + return; + } + + case value_t::number_integer: + { + dump_integer(val.m_value.number_integer); + return; + } + + case value_t::number_unsigned: + { + dump_integer(val.m_value.number_unsigned); + return; + } + + case value_t::number_float: + { + dump_float(val.m_value.number_float); + return; + } + + case value_t::discarded: + { + o->write_characters("<discarded>", 11); + return; + } + + case value_t::null: + { + o->write_characters("null", 4); + return; + } + } + } + + private: + /*! + @brief dump escaped string + + Escape a string by replacing certain special characters by a sequence of an + escape character (backslash) and another character and other control + characters by a sequence of "\u" followed by a four-digit hex + representation. The escaped string is written to output stream @a o. + + @param[in] s the string to escape + @param[in] ensure_ascii whether to escape non-ASCII characters with + \uXXXX sequences + + @complexity Linear in the length of string @a s. + */ + void dump_escaped(const string_t& s, const bool ensure_ascii) + { + uint32_t codepoint; + uint8_t state = UTF8_ACCEPT; + std::size_t bytes = 0; // number of bytes written to string_buffer + + // number of bytes written at the point of the last valid byte + std::size_t bytes_after_last_accept = 0; + std::size_t undumped_chars = 0; + + for (std::size_t i = 0; i < s.size(); ++i) + { + const auto byte = static_cast<uint8_t>(s[i]); + + switch (decode(state, codepoint, byte)) + { + case UTF8_ACCEPT: // decode found a new code point + { + switch (codepoint) + { + case 0x08: // backspace + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'b'; + break; + } + + case 0x09: // horizontal tab + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 't'; + break; + } + + case 0x0A: // newline + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'n'; + break; + } + + case 0x0C: // formfeed + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'f'; + break; + } + + case 0x0D: // carriage return + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'r'; + break; + } + + case 0x22: // quotation mark + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\"'; + break; + } + + case 0x5C: // reverse solidus + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = '\\'; + break; + } + + default: + { + // escape control characters (0x00..0x1F) or, if + // ensure_ascii parameter is used, non-ASCII characters + if ((codepoint <= 0x1F) or (ensure_ascii and (codepoint >= 0x7F))) + { + if (codepoint <= 0xFFFF) + { + (std::snprintf)(string_buffer.data() + bytes, 7, "\\u%04x", + static_cast<uint16_t>(codepoint)); + bytes += 6; + } + else + { + (std::snprintf)(string_buffer.data() + bytes, 13, "\\u%04x\\u%04x", + static_cast<uint16_t>(0xD7C0 + (codepoint >> 10)), + static_cast<uint16_t>(0xDC00 + (codepoint & 0x3FF))); + bytes += 12; + } + } + else + { + // copy byte to buffer (all previous bytes + // been copied have in default case above) + string_buffer[bytes++] = s[i]; + } + break; + } + } + + // write buffer and reset index; there must be 13 bytes + // left, as this is the maximal number of bytes to be + // written ("\uxxxx\uxxxx\0") for one code point + if (string_buffer.size() - bytes < 13) + { + o->write_characters(string_buffer.data(), bytes); + bytes = 0; + } + + // remember the byte position of this accept + bytes_after_last_accept = bytes; + undumped_chars = 0; + break; + } + + case UTF8_REJECT: // decode found invalid UTF-8 byte + { + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", byte); + JSON_THROW(type_error::create(316, "invalid UTF-8 byte at index " + std::to_string(i) + ": 0x" + sn)); + } + + case error_handler_t::ignore: + case error_handler_t::replace: + { + // in case we saw this character the first time, we + // would like to read it again, because the byte + // may be OK for itself, but just not OK for the + // previous sequence + if (undumped_chars > 0) + { + --i; + } + + // reset length buffer to the last accepted index; + // thus removing/ignoring the invalid characters + bytes = bytes_after_last_accept; + + if (error_handler == error_handler_t::replace) + { + // add a replacement character + if (ensure_ascii) + { + string_buffer[bytes++] = '\\'; + string_buffer[bytes++] = 'u'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'f'; + string_buffer[bytes++] = 'd'; + } + else + { + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xEF'); + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBF'); + string_buffer[bytes++] = detail::binary_writer<BasicJsonType, char>::to_char_type('\xBD'); + } + bytes_after_last_accept = bytes; + } + + undumped_chars = 0; + + // continue processing the string + state = UTF8_ACCEPT; + break; + } + } + break; + } + + default: // decode found yet incomplete multi-byte code point + { + if (not ensure_ascii) + { + // code point will not be escaped - copy byte to buffer + string_buffer[bytes++] = s[i]; + } + ++undumped_chars; + break; + } + } + } + + // we finished processing the string + if (JSON_LIKELY(state == UTF8_ACCEPT)) + { + // write buffer + if (bytes > 0) + { + o->write_characters(string_buffer.data(), bytes); + } + } + else + { + // we finish reading, but do not accept: string was incomplete + switch (error_handler) + { + case error_handler_t::strict: + { + std::string sn(3, '\0'); + (std::snprintf)(&sn[0], sn.size(), "%.2X", static_cast<uint8_t>(s.back())); + JSON_THROW(type_error::create(316, "incomplete UTF-8 string; last byte: 0x" + sn)); + } + + case error_handler_t::ignore: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + break; + } + + case error_handler_t::replace: + { + // write all accepted bytes + o->write_characters(string_buffer.data(), bytes_after_last_accept); + // add a replacement character + if (ensure_ascii) + { + o->write_characters("\\ufffd", 6); + } + else + { + o->write_characters("\xEF\xBF\xBD", 3); + } + break; + } + } + } + } + + /*! + @brief dump an integer + + Dump a given integer to output stream @a o. Works internally with + @a number_buffer. + + @param[in] x integer number (signed or unsigned) to dump + @tparam NumberType either @a number_integer_t or @a number_unsigned_t + */ + template<typename NumberType, detail::enable_if_t< + std::is_same<NumberType, number_unsigned_t>::value or + std::is_same<NumberType, number_integer_t>::value, + int> = 0> + void dump_integer(NumberType x) + { + // special case for "0" + if (x == 0) + { + o->write_character('0'); + return; + } + + const bool is_negative = std::is_same<NumberType, number_integer_t>::value and not (x >= 0); // see issue #755 + std::size_t i = 0; + + while (x != 0) + { + // spare 1 byte for '\0' + assert(i < number_buffer.size() - 1); + + const auto digit = std::labs(static_cast<long>(x % 10)); + number_buffer[i++] = static_cast<char>('0' + digit); + x /= 10; + } + + if (is_negative) + { + // make sure there is capacity for the '-' + assert(i < number_buffer.size() - 2); + number_buffer[i++] = '-'; + } + + std::reverse(number_buffer.begin(), number_buffer.begin() + i); + o->write_characters(number_buffer.data(), i); + } + + /*! + @brief dump a floating-point number + + Dump a given floating-point number to output stream @a o. Works internally + with @a number_buffer. + + @param[in] x floating-point number to dump + */ + void dump_float(number_float_t x) + { + // NaN / inf + if (not std::isfinite(x)) + { + o->write_characters("null", 4); + return; + } + + // If number_float_t is an IEEE-754 single or double precision number, + // use the Grisu2 algorithm to produce short numbers which are + // guaranteed to round-trip, using strtof and strtod, resp. + // + // NB: The test below works if <long double> == <double>. + static constexpr bool is_ieee_single_or_double + = (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 24 and std::numeric_limits<number_float_t>::max_exponent == 128) or + (std::numeric_limits<number_float_t>::is_iec559 and std::numeric_limits<number_float_t>::digits == 53 and std::numeric_limits<number_float_t>::max_exponent == 1024); + + dump_float(x, std::integral_constant<bool, is_ieee_single_or_double>()); + } + + void dump_float(number_float_t x, std::true_type /*is_ieee_single_or_double*/) + { + char* begin = number_buffer.data(); + char* end = ::nlohmann::detail::to_chars(begin, begin + number_buffer.size(), x); + + o->write_characters(begin, static_cast<size_t>(end - begin)); + } + + void dump_float(number_float_t x, std::false_type /*is_ieee_single_or_double*/) + { + // get number of digits for a float -> text -> float round-trip + static constexpr auto d = std::numeric_limits<number_float_t>::max_digits10; + + // the actual conversion + std::ptrdiff_t len = (std::snprintf)(number_buffer.data(), number_buffer.size(), "%.*g", d, x); + + // negative value indicates an error + assert(len > 0); + // check if buffer was large enough + assert(static_cast<std::size_t>(len) < number_buffer.size()); + + // erase thousands separator + if (thousands_sep != '\0') + { + const auto end = std::remove(number_buffer.begin(), + number_buffer.begin() + len, thousands_sep); + std::fill(end, number_buffer.end(), '\0'); + assert((end - number_buffer.begin()) <= len); + len = (end - number_buffer.begin()); + } + + // convert decimal point to '.' + if (decimal_point != '\0' and decimal_point != '.') + { + const auto dec_pos = std::find(number_buffer.begin(), number_buffer.end(), decimal_point); + if (dec_pos != number_buffer.end()) + { + *dec_pos = '.'; + } + } + + o->write_characters(number_buffer.data(), static_cast<std::size_t>(len)); + + // determine if need to append ".0" + const bool value_is_int_like = + std::none_of(number_buffer.begin(), number_buffer.begin() + len + 1, + [](char c) + { + return (c == '.' or c == 'e'); + }); + + if (value_is_int_like) + { + o->write_characters(".0", 2); + } + } + + /*! + @brief check whether a string is UTF-8 encoded + + The function checks each byte of a string whether it is UTF-8 encoded. The + result of the check is stored in the @a state parameter. The function must + be called initially with state 0 (accept). State 1 means the string must + be rejected, because the current byte is not allowed. If the string is + completely processed, but the state is non-zero, the string ended + prematurely; that is, the last byte indicated more bytes should have + followed. + + @param[in,out] state the state of the decoding + @param[in,out] codep codepoint (valid only if resulting state is UTF8_ACCEPT) + @param[in] byte next byte to decode + @return new state + + @note The function has been edited: a std::array is used. + + @copyright Copyright (c) 2008-2009 Bjoern Hoehrmann <bjoern@hoehrmann.de> + @sa http://bjoern.hoehrmann.de/utf-8/decoder/dfa/ + */ + static uint8_t decode(uint8_t& state, uint32_t& codep, const uint8_t byte) noexcept + { + static const std::array<uint8_t, 400> utf8d = + { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 00..1F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20..3F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 40..5F + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 60..7F + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, // 80..9F + 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, // A0..BF + 8, 8, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, // C0..DF + 0xA, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x3, 0x4, 0x3, 0x3, // E0..EF + 0xB, 0x6, 0x6, 0x6, 0x5, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, 0x8, // F0..FF + 0x0, 0x1, 0x2, 0x3, 0x5, 0x8, 0x7, 0x1, 0x1, 0x1, 0x4, 0x6, 0x1, 0x1, 0x1, 0x1, // s0..s0 + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, // s1..s2 + 1, 2, 1, 1, 1, 1, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, // s3..s4 + 1, 2, 1, 1, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, // s5..s6 + 1, 3, 1, 1, 1, 1, 1, 3, 1, 3, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 // s7..s8 + } + }; + + const uint8_t type = utf8d[byte]; + + codep = (state != UTF8_ACCEPT) + ? (byte & 0x3fu) | (codep << 6) + : static_cast<uint32_t>(0xff >> type) & (byte); + + state = utf8d[256u + state * 16u + type]; + return state; + } + + private: + /// the output of the serializer + output_adapter_t<char> o = nullptr; + + /// a (hopefully) large enough character buffer + std::array<char, 64> number_buffer{{}}; + + /// the locale + const std::lconv* loc = nullptr; + /// the locale's thousand separator character + const char thousands_sep = '\0'; + /// the locale's decimal point character + const char decimal_point = '\0'; + + /// string buffer + std::array<char, 512> string_buffer{{}}; + + /// the indentation character + const char indent_char; + /// the indentation string + string_t indent_string; + + /// error_handler how to react on decoding errors + const error_handler_t error_handler; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/json_ref.hpp> + + +#include <initializer_list> +#include <utility> + +// #include <nlohmann/detail/meta/type_traits.hpp> + + +namespace nlohmann +{ +namespace detail +{ +template<typename BasicJsonType> +class json_ref +{ + public: + using value_type = BasicJsonType; + + json_ref(value_type&& value) + : owned_value(std::move(value)), value_ref(&owned_value), is_rvalue(true) + {} + + json_ref(const value_type& value) + : value_ref(const_cast<value_type*>(&value)), is_rvalue(false) + {} + + json_ref(std::initializer_list<json_ref> init) + : owned_value(init), value_ref(&owned_value), is_rvalue(true) + {} + + template < + class... Args, + enable_if_t<std::is_constructible<value_type, Args...>::value, int> = 0 > + json_ref(Args && ... args) + : owned_value(std::forward<Args>(args)...), value_ref(&owned_value), + is_rvalue(true) {} + + // class should be movable only + json_ref(json_ref&&) = default; + json_ref(const json_ref&) = delete; + json_ref& operator=(const json_ref&) = delete; + json_ref& operator=(json_ref&&) = delete; + ~json_ref() = default; + + value_type moved_or_copied() const + { + if (is_rvalue) + { + return std::move(*value_ref); + } + return *value_ref; + } + + value_type const& operator*() const + { + return *static_cast<value_type const*>(value_ref); + } + + value_type const* operator->() const + { + return static_cast<value_type const*>(value_ref); + } + + private: + mutable value_type owned_value = nullptr; + value_type* value_ref = nullptr; + const bool is_rvalue; +}; +} // namespace detail +} // namespace nlohmann + +// #include <nlohmann/detail/json_pointer.hpp> + + +#include <cassert> // assert +#include <numeric> // accumulate +#include <string> // string +#include <vector> // vector + +// #include <nlohmann/detail/macro_scope.hpp> + +// #include <nlohmann/detail/exceptions.hpp> + +// #include <nlohmann/detail/value_t.hpp> + + +namespace nlohmann +{ +template<typename BasicJsonType> +class json_pointer +{ + // allow basic_json to access private members + NLOHMANN_BASIC_JSON_TPL_DECLARATION + friend class basic_json; + + public: + /*! + @brief create JSON pointer + + Create a JSON pointer according to the syntax described in + [Section 3 of RFC6901](https://tools.ietf.org/html/rfc6901#section-3). + + @param[in] s string representing the JSON pointer; if omitted, the empty + string is assumed which references the whole JSON value + + @throw parse_error.107 if the given JSON pointer @a s is nonempty and does + not begin with a slash (`/`); see example below + + @throw parse_error.108 if a tilde (`~`) in the given JSON pointer @a s is + not followed by `0` (representing `~`) or `1` (representing `/`); see + example below + + @liveexample{The example shows the construction several valid JSON pointers + as well as the exceptional behavior.,json_pointer} + + @since version 2.0.0 + */ + explicit json_pointer(const std::string& s = "") + : reference_tokens(split(s)) + {} + + /*! + @brief return a string representation of the JSON pointer + + @invariant For each JSON pointer `ptr`, it holds: + @code {.cpp} + ptr == json_pointer(ptr.to_string()); + @endcode + + @return a string representation of the JSON pointer + + @liveexample{The example shows the result of `to_string`., + json_pointer__to_string} + + @since version 2.0.0 + */ + std::string to_string() const + { + return std::accumulate(reference_tokens.begin(), reference_tokens.end(), + std::string{}, + [](const std::string & a, const std::string & b) + { + return a + "/" + escape(b); + }); + } + + /// @copydoc to_string() + operator std::string() const + { + return to_string(); + } + + /*! + @param[in] s reference token to be converted into an array index + + @return integer representation of @a s + + @throw out_of_range.404 if string @a s could not be converted to an integer + */ + static int array_index(const std::string& s) + { + std::size_t processed_chars = 0; + const int res = std::stoi(s, &processed_chars); + + // check if the string was completely read + if (JSON_UNLIKELY(processed_chars != s.size())) + { + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'")); + } + + return res; + } + + private: + /*! + @brief remove and return last reference pointer + @throw out_of_range.405 if JSON pointer has no parent + */ + std::string pop_back() + { + if (JSON_UNLIKELY(is_root())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); + } + + auto last = reference_tokens.back(); + reference_tokens.pop_back(); + return last; + } + + /// return whether pointer points to the root document + bool is_root() const noexcept + { + return reference_tokens.empty(); + } + + json_pointer top() const + { + if (JSON_UNLIKELY(is_root())) + { + JSON_THROW(detail::out_of_range::create(405, "JSON pointer has no parent")); + } + + json_pointer result = *this; + result.reference_tokens = {reference_tokens[0]}; + return result; + } + + /*! + @brief create and return a reference to the pointed to value + + @complexity Linear in the number of reference tokens. + + @throw parse_error.109 if array index is not a number + @throw type_error.313 if value cannot be unflattened + */ + BasicJsonType& get_and_create(BasicJsonType& j) const + { + using size_type = typename BasicJsonType::size_type; + auto result = &j; + + // in case no reference tokens exist, return a reference to the JSON value + // j which will be overwritten by a primitive value + for (const auto& reference_token : reference_tokens) + { + switch (result->m_type) + { + case detail::value_t::null: + { + if (reference_token == "0") + { + // start a new array if reference token is 0 + result = &result->operator[](0); + } + else + { + // start a new object otherwise + result = &result->operator[](reference_token); + } + break; + } + + case detail::value_t::object: + { + // create an entry in the object + result = &result->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // create an entry in the array + JSON_TRY + { + result = &result->operator[](static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + /* + The following code is only reached if there exists a reference + token _and_ the current value is primitive. In this case, we have + an error situation, because primitive values may only occur as + single value; that is, with an empty list of reference tokens. + */ + default: + JSON_THROW(detail::type_error::create(313, "invalid value to unflatten")); + } + } + + return *result; + } + + /*! + @brief return a reference to the pointed to value + + @note This version does not throw if a value is not present, but tries to + create nested values instead. For instance, calling this function + with pointer `"/this/that"` on a null value is equivalent to calling + `operator[]("this").operator[]("that")` on that value, effectively + changing the null value to an object. + + @param[in] ptr a JSON value + + @return reference to the JSON value pointed to by the JSON pointer + + @complexity Linear in the length of the JSON pointer. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_unchecked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + // convert null values to arrays or objects before continuing + if (ptr->m_type == detail::value_t::null) + { + // check if reference token is a number + const bool nums = + std::all_of(reference_token.begin(), reference_token.end(), + [](const char x) + { + return (x >= '0' and x <= '9'); + }); + + // change value to array for numbers or "-" or to object otherwise + *ptr = (nums or reference_token == "-") + ? detail::value_t::array + : detail::value_t::object; + } + + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + if (reference_token == "-") + { + // explicitly treat "-" as index beyond the end + ptr = &ptr->operator[](ptr->m_value.array->size()); + } + else + { + // convert array index to number; unchecked access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + BasicJsonType& get_checked(BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } + + /*! + @brief return a const reference to the pointed to value + + @param[in] ptr a JSON value + + @return const reference to the JSON value pointed to by the JSON + pointer + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_unchecked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // use unchecked object access + ptr = &ptr->operator[](reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" cannot be used for const access + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // use unchecked array access + JSON_TRY + { + ptr = &ptr->operator[]( + static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } + + /*! + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + */ + const BasicJsonType& get_checked(const BasicJsonType* ptr) const + { + using size_type = typename BasicJsonType::size_type; + for (const auto& reference_token : reference_tokens) + { + switch (ptr->m_type) + { + case detail::value_t::object: + { + // note: at performs range check + ptr = &ptr->at(reference_token); + break; + } + + case detail::value_t::array: + { + if (JSON_UNLIKELY(reference_token == "-")) + { + // "-" always fails the range check + JSON_THROW(detail::out_of_range::create(402, + "array index '-' (" + std::to_string(ptr->m_value.array->size()) + + ") is out of range")); + } + + // error condition (cf. RFC 6901, Sect. 4) + if (JSON_UNLIKELY(reference_token.size() > 1 and reference_token[0] == '0')) + { + JSON_THROW(detail::parse_error::create(106, 0, + "array index '" + reference_token + + "' must not begin with '0'")); + } + + // note: at performs range check + JSON_TRY + { + ptr = &ptr->at(static_cast<size_type>(array_index(reference_token))); + } + JSON_CATCH(std::invalid_argument&) + { + JSON_THROW(detail::parse_error::create(109, 0, "array index '" + reference_token + "' is not a number")); + } + break; + } + + default: + JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + reference_token + "'")); + } + } + + return *ptr; + } + + /*! + @brief split the string input to reference tokens + + @note This function is only called by the json_pointer constructor. + All exceptions below are documented there. + + @throw parse_error.107 if the pointer is not empty or begins with '/' + @throw parse_error.108 if character '~' is not followed by '0' or '1' + */ + static std::vector<std::string> split(const std::string& reference_string) + { + std::vector<std::string> result; + + // special case: empty reference string -> no reference tokens + if (reference_string.empty()) + { + return result; + } + + // check if nonempty reference string begins with slash + if (JSON_UNLIKELY(reference_string[0] != '/')) + { + JSON_THROW(detail::parse_error::create(107, 1, + "JSON pointer must be empty or begin with '/' - was: '" + + reference_string + "'")); + } + + // extract the reference tokens: + // - slash: position of the last read slash (or end of string) + // - start: position after the previous slash + for ( + // search for the first slash after the first character + std::size_t slash = reference_string.find_first_of('/', 1), + // set the beginning of the first reference token + start = 1; + // we can stop if start == 0 (if slash == std::string::npos) + start != 0; + // set the beginning of the next reference token + // (will eventually be 0 if slash == std::string::npos) + start = (slash == std::string::npos) ? 0 : slash + 1, + // find next slash + slash = reference_string.find_first_of('/', start)) + { + // use the text between the beginning of the reference token + // (start) and the last slash (slash). + auto reference_token = reference_string.substr(start, slash - start); + + // check reference tokens are properly escaped + for (std::size_t pos = reference_token.find_first_of('~'); + pos != std::string::npos; + pos = reference_token.find_first_of('~', pos + 1)) + { + assert(reference_token[pos] == '~'); + + // ~ must be followed by 0 or 1 + if (JSON_UNLIKELY(pos == reference_token.size() - 1 or + (reference_token[pos + 1] != '0' and + reference_token[pos + 1] != '1'))) + { + JSON_THROW(detail::parse_error::create(108, 0, "escape character '~' must be followed with '0' or '1'")); + } + } + + // finally, store the reference token + unescape(reference_token); + result.push_back(reference_token); + } + + return result; + } + + /*! + @brief replace all occurrences of a substring by another string + + @param[in,out] s the string to manipulate; changed so that all + occurrences of @a f are replaced with @a t + @param[in] f the substring to replace with @a t + @param[in] t the string to replace @a f + + @pre The search string @a f must not be empty. **This precondition is + enforced with an assertion.** + + @since version 2.0.0 + */ + static void replace_substring(std::string& s, const std::string& f, + const std::string& t) + { + assert(not f.empty()); + for (auto pos = s.find(f); // find first occurrence of f + pos != std::string::npos; // make sure f was found + s.replace(pos, f.size(), t), // replace with t, and + pos = s.find(f, pos + t.size())) // find next occurrence of f + {} + } + + /// escape "~" to "~0" and "/" to "~1" + static std::string escape(std::string s) + { + replace_substring(s, "~", "~0"); + replace_substring(s, "/", "~1"); + return s; + } + + /// unescape "~1" to tilde and "~0" to slash (order is important!) + static void unescape(std::string& s) + { + replace_substring(s, "~1", "/"); + replace_substring(s, "~0", "~"); + } + + /*! + @param[in] reference_string the reference string to the current value + @param[in] value the value to consider + @param[in,out] result the result object to insert values to + + @note Empty objects or arrays are flattened to `null`. + */ + static void flatten(const std::string& reference_string, + const BasicJsonType& value, + BasicJsonType& result) + { + switch (value.m_type) + { + case detail::value_t::array: + { + if (value.m_value.array->empty()) + { + // flatten empty array as null + result[reference_string] = nullptr; + } + else + { + // iterate array and use index as reference string + for (std::size_t i = 0; i < value.m_value.array->size(); ++i) + { + flatten(reference_string + "/" + std::to_string(i), + value.m_value.array->operator[](i), result); + } + } + break; + } + + case detail::value_t::object: + { + if (value.m_value.object->empty()) + { + // flatten empty object as null + result[reference_string] = nullptr; + } + else + { + // iterate object and use keys as reference string + for (const auto& element : *value.m_value.object) + { + flatten(reference_string + "/" + escape(element.first), element.second, result); + } + } + break; + } + + default: + { + // add primitive value with its reference string + result[reference_string] = value; + break; + } + } + } + + /*! + @param[in] value flattened JSON + + @return unflattened JSON + + @throw parse_error.109 if array index is not a number + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + @throw type_error.313 if value cannot be unflattened + */ + static BasicJsonType + unflatten(const BasicJsonType& value) + { + if (JSON_UNLIKELY(not value.is_object())) + { + JSON_THROW(detail::type_error::create(314, "only objects can be unflattened")); + } + + BasicJsonType result; + + // iterate the JSON object values + for (const auto& element : *value.m_value.object) + { + if (JSON_UNLIKELY(not element.second.is_primitive())) + { + JSON_THROW(detail::type_error::create(315, "values in object must be primitive")); + } + + // assign value to reference pointed to by JSON pointer; Note that if + // the JSON pointer is "" (i.e., points to the whole value), function + // get_and_create returns a reference to result itself. An assignment + // will then create a primitive value. + json_pointer(element.first).get_and_create(result) = element.second; + } + + return result; + } + + friend bool operator==(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return (lhs.reference_tokens == rhs.reference_tokens); + } + + friend bool operator!=(json_pointer const& lhs, + json_pointer const& rhs) noexcept + { + return not (lhs == rhs); + } + + /// the reference tokens + std::vector<std::string> reference_tokens; +}; +} // namespace nlohmann + +// #include <nlohmann/adl_serializer.hpp> + + +#include <utility> + +// #include <nlohmann/detail/conversions/from_json.hpp> + +// #include <nlohmann/detail/conversions/to_json.hpp> + + +namespace nlohmann +{ + +template<typename, typename> +struct adl_serializer +{ + /*! + @brief convert a JSON value to any value type + + This function is usually called by the `get()` function of the + @ref basic_json class (either explicit or via conversion operators). + + @param[in] j JSON value to read from + @param[in,out] val value to write to + */ + template<typename BasicJsonType, typename ValueType> + static auto from_json(BasicJsonType&& j, ValueType& val) noexcept( + noexcept(::nlohmann::from_json(std::forward<BasicJsonType>(j), val))) + -> decltype(::nlohmann::from_json(std::forward<BasicJsonType>(j), val), void()) + { + ::nlohmann::from_json(std::forward<BasicJsonType>(j), val); + } + + /*! + @brief convert any value type to a JSON value + + This function is usually called by the constructors of the @ref basic_json + class. + + @param[in,out] j JSON value to write to + @param[in] val value to read from + */ + template <typename BasicJsonType, typename ValueType> + static auto to_json(BasicJsonType& j, ValueType&& val) noexcept( + noexcept(::nlohmann::to_json(j, std::forward<ValueType>(val)))) + -> decltype(::nlohmann::to_json(j, std::forward<ValueType>(val)), void()) + { + ::nlohmann::to_json(j, std::forward<ValueType>(val)); + } +}; + +} // namespace nlohmann + + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ + +/*! +@brief a class to store JSON values + +@tparam ObjectType type for JSON objects (`std::map` by default; will be used +in @ref object_t) +@tparam ArrayType type for JSON arrays (`std::vector` by default; will be used +in @ref array_t) +@tparam StringType type for JSON strings and object keys (`std::string` by +default; will be used in @ref string_t) +@tparam BooleanType type for JSON booleans (`bool` by default; will be used +in @ref boolean_t) +@tparam NumberIntegerType type for JSON integer numbers (`int64_t` by +default; will be used in @ref number_integer_t) +@tparam NumberUnsignedType type for JSON unsigned integer numbers (@c +`uint64_t` by default; will be used in @ref number_unsigned_t) +@tparam NumberFloatType type for JSON floating-point numbers (`double` by +default; will be used in @ref number_float_t) +@tparam AllocatorType type of the allocator to use (`std::allocator` by +default) +@tparam JSONSerializer the serializer to resolve internal calls to `to_json()` +and `from_json()` (@ref adl_serializer by default) + +@requirement The class satisfies the following concept requirements: +- Basic + - [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible): + JSON values can be default constructed. The result will be a JSON null + value. + - [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible): + A JSON value can be constructed from an rvalue argument. + - [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible): + A JSON value can be copy-constructed from an lvalue expression. + - [MoveAssignable](https://en.cppreference.com/w/cpp/named_req/MoveAssignable): + A JSON value van be assigned from an rvalue argument. + - [CopyAssignable](https://en.cppreference.com/w/cpp/named_req/CopyAssignable): + A JSON value can be copy-assigned from an lvalue expression. + - [Destructible](https://en.cppreference.com/w/cpp/named_req/Destructible): + JSON values can be destructed. +- Layout + - [StandardLayoutType](https://en.cppreference.com/w/cpp/named_req/StandardLayoutType): + JSON values have + [standard layout](https://en.cppreference.com/w/cpp/language/data_members#Standard_layout): + All non-static data members are private and standard layout types, the + class has no virtual functions or (virtual) base classes. +- Library-wide + - [EqualityComparable](https://en.cppreference.com/w/cpp/named_req/EqualityComparable): + JSON values can be compared with `==`, see @ref + operator==(const_reference,const_reference). + - [LessThanComparable](https://en.cppreference.com/w/cpp/named_req/LessThanComparable): + JSON values can be compared with `<`, see @ref + operator<(const_reference,const_reference). + - [Swappable](https://en.cppreference.com/w/cpp/named_req/Swappable): + Any JSON lvalue or rvalue of can be swapped with any lvalue or rvalue of + other compatible types, using unqualified function call @ref swap(). + - [NullablePointer](https://en.cppreference.com/w/cpp/named_req/NullablePointer): + JSON values can be compared against `std::nullptr_t` objects which are used + to model the `null` value. +- Container + - [Container](https://en.cppreference.com/w/cpp/named_req/Container): + JSON values can be used like STL containers and provide iterator access. + - [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer); + JSON values can be used like STL containers and provide reverse iterator + access. + +@invariant The member variables @a m_value and @a m_type have the following +relationship: +- If `m_type == value_t::object`, then `m_value.object != nullptr`. +- If `m_type == value_t::array`, then `m_value.array != nullptr`. +- If `m_type == value_t::string`, then `m_value.string != nullptr`. +The invariants are checked by member function assert_invariant(). + +@internal +@note ObjectType trick from http://stackoverflow.com/a/9860911 +@endinternal + +@see [RFC 7159: The JavaScript Object Notation (JSON) Data Interchange +Format](http://rfc7159.net/rfc7159) + +@since version 1.0.0 + +@nosubgrouping +*/ +NLOHMANN_BASIC_JSON_TPL_DECLARATION +class basic_json +{ + private: + template<detail::value_t> friend struct detail::external_constructor; + friend ::nlohmann::json_pointer<basic_json>; + friend ::nlohmann::detail::parser<basic_json>; + friend ::nlohmann::detail::serializer<basic_json>; + template<typename BasicJsonType> + friend class ::nlohmann::detail::iter_impl; + template<typename BasicJsonType, typename CharType> + friend class ::nlohmann::detail::binary_writer; + template<typename BasicJsonType, typename SAX> + friend class ::nlohmann::detail::binary_reader; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_parser; + template<typename BasicJsonType> + friend class ::nlohmann::detail::json_sax_dom_callback_parser; + + /// workaround type for MSVC + using basic_json_t = NLOHMANN_BASIC_JSON_TPL; + + // convenience aliases for types residing in namespace detail; + using lexer = ::nlohmann::detail::lexer<basic_json>; + using parser = ::nlohmann::detail::parser<basic_json>; + + using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; + template<typename BasicJsonType> + using internal_iterator = ::nlohmann::detail::internal_iterator<BasicJsonType>; + template<typename BasicJsonType> + using iter_impl = ::nlohmann::detail::iter_impl<BasicJsonType>; + template<typename Iterator> + using iteration_proxy = ::nlohmann::detail::iteration_proxy<Iterator>; + template<typename Base> using json_reverse_iterator = ::nlohmann::detail::json_reverse_iterator<Base>; + + template<typename CharType> + using output_adapter_t = ::nlohmann::detail::output_adapter_t<CharType>; + + using binary_reader = ::nlohmann::detail::binary_reader<basic_json>; + template<typename CharType> using binary_writer = ::nlohmann::detail::binary_writer<basic_json, CharType>; + + using serializer = ::nlohmann::detail::serializer<basic_json>; + + public: + using value_t = detail::value_t; + /// JSON Pointer, see @ref nlohmann::json_pointer + using json_pointer = ::nlohmann::json_pointer<basic_json>; + template<typename T, typename SFINAE> + using json_serializer = JSONSerializer<T, SFINAE>; + /// how to treat decoding errors + using error_handler_t = detail::error_handler_t; + /// helper type for initializer lists of basic_json values + using initializer_list_t = std::initializer_list<detail::json_ref<basic_json>>; + + using input_format_t = detail::input_format_t; + /// SAX interface type, see @ref nlohmann::json_sax + using json_sax_t = json_sax<basic_json>; + + //////////////// + // exceptions // + //////////////// + + /// @name exceptions + /// Classes to implement user-defined exceptions. + /// @{ + + /// @copydoc detail::exception + using exception = detail::exception; + /// @copydoc detail::parse_error + using parse_error = detail::parse_error; + /// @copydoc detail::invalid_iterator + using invalid_iterator = detail::invalid_iterator; + /// @copydoc detail::type_error + using type_error = detail::type_error; + /// @copydoc detail::out_of_range + using out_of_range = detail::out_of_range; + /// @copydoc detail::other_error + using other_error = detail::other_error; + + /// @} + + + ///////////////////// + // container types // + ///////////////////// + + /// @name container types + /// The canonic container types to use @ref basic_json like any other STL + /// container. + /// @{ + + /// the type of elements in a basic_json container + using value_type = basic_json; + + /// the type of an element reference + using reference = value_type&; + /// the type of an element const reference + using const_reference = const value_type&; + + /// a type to represent differences between iterators + using difference_type = std::ptrdiff_t; + /// a type to represent container sizes + using size_type = std::size_t; + + /// the allocator type + using allocator_type = AllocatorType<basic_json>; + + /// the type of an element pointer + using pointer = typename std::allocator_traits<allocator_type>::pointer; + /// the type of an element const pointer + using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer; + + /// an iterator for a basic_json container + using iterator = iter_impl<basic_json>; + /// a const iterator for a basic_json container + using const_iterator = iter_impl<const basic_json>; + /// a reverse iterator for a basic_json container + using reverse_iterator = json_reverse_iterator<typename basic_json::iterator>; + /// a const reverse iterator for a basic_json container + using const_reverse_iterator = json_reverse_iterator<typename basic_json::const_iterator>; + + /// @} + + + /*! + @brief returns the allocator associated with the container + */ + static allocator_type get_allocator() + { + return allocator_type(); + } + + /*! + @brief returns version information on the library + + This function returns a JSON object with information about the library, + including the version number and information on the platform and compiler. + + @return JSON object holding version information + key | description + ----------- | --------------- + `compiler` | Information on the used compiler. It is an object with the following keys: `c++` (the used C++ standard), `family` (the compiler family; possible values are `clang`, `icc`, `gcc`, `ilecpp`, `msvc`, `pgcpp`, `sunpro`, and `unknown`), and `version` (the compiler version). + `copyright` | The copyright line for the library as string. + `name` | The name of the library as string. + `platform` | The used platform as string. Possible values are `win32`, `linux`, `apple`, `unix`, and `unknown`. + `url` | The URL of the project as string. + `version` | The version of the library. It is an object with the following keys: `major`, `minor`, and `patch` as defined by [Semantic Versioning](http://semver.org), and `string` (the version string). + + @liveexample{The following code shows an example output of the `meta()` + function.,meta} + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @complexity Constant. + + @since 2.1.0 + */ + static basic_json meta() + { + basic_json result; + + result["copyright"] = "(C) 2013-2017 Niels Lohmann"; + result["name"] = "JSON for Modern C++"; + result["url"] = "https://github.com/nlohmann/json"; + result["version"]["string"] = + std::to_string(NLOHMANN_JSON_VERSION_MAJOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_MINOR) + "." + + std::to_string(NLOHMANN_JSON_VERSION_PATCH); + result["version"]["major"] = NLOHMANN_JSON_VERSION_MAJOR; + result["version"]["minor"] = NLOHMANN_JSON_VERSION_MINOR; + result["version"]["patch"] = NLOHMANN_JSON_VERSION_PATCH; + +#ifdef _WIN32 + result["platform"] = "win32"; +#elif defined __linux__ + result["platform"] = "linux"; +#elif defined __APPLE__ + result["platform"] = "apple"; +#elif defined __unix__ + result["platform"] = "unix"; +#else + result["platform"] = "unknown"; +#endif + +#if defined(__ICC) || defined(__INTEL_COMPILER) + result["compiler"] = {{"family", "icc"}, {"version", __INTEL_COMPILER}}; +#elif defined(__clang__) + result["compiler"] = {{"family", "clang"}, {"version", __clang_version__}}; +#elif defined(__GNUC__) || defined(__GNUG__) + result["compiler"] = {{"family", "gcc"}, {"version", std::to_string(__GNUC__) + "." + std::to_string(__GNUC_MINOR__) + "." + std::to_string(__GNUC_PATCHLEVEL__)}}; +#elif defined(__HP_cc) || defined(__HP_aCC) + result["compiler"] = "hp" +#elif defined(__IBMCPP__) + result["compiler"] = {{"family", "ilecpp"}, {"version", __IBMCPP__}}; +#elif defined(_MSC_VER) + result["compiler"] = {{"family", "msvc"}, {"version", _MSC_VER}}; +#elif defined(__PGI) + result["compiler"] = {{"family", "pgcpp"}, {"version", __PGI}}; +#elif defined(__SUNPRO_CC) + result["compiler"] = {{"family", "sunpro"}, {"version", __SUNPRO_CC}}; +#else + result["compiler"] = {{"family", "unknown"}, {"version", "unknown"}}; +#endif + +#ifdef __cplusplus + result["compiler"]["c++"] = std::to_string(__cplusplus); +#else + result["compiler"]["c++"] = "unknown"; +#endif + return result; + } + + + /////////////////////////// + // JSON value data types // + /////////////////////////// + + /// @name JSON value data types + /// The data types to store a JSON value. These types are derived from + /// the template arguments passed to class @ref basic_json. + /// @{ + +#if defined(JSON_HAS_CPP_14) + // Use transparent comparator if possible, combined with perfect forwarding + // on find() and count() calls prevents unnecessary string construction. + using object_comparator_t = std::less<>; +#else + using object_comparator_t = std::less<StringType>; +#endif + + /*! + @brief a type for an object + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON objects as follows: + > An object is an unordered collection of zero or more name/value pairs, + > where a name is a string and a value is a string, number, boolean, null, + > object, or array. + + To store objects in C++, a type is defined by the template parameters + described below. + + @tparam ObjectType the container to store objects (e.g., `std::map` or + `std::unordered_map`) + @tparam StringType the type of the keys or names (e.g., `std::string`). + The comparison function `std::less<StringType>` is used to order elements + inside the container. + @tparam AllocatorType the allocator to use for objects (e.g., + `std::allocator`) + + #### Default type + + With the default values for @a ObjectType (`std::map`), @a StringType + (`std::string`), and @a AllocatorType (`std::allocator`), the default + value for @a object_t is: + + @code {.cpp} + std::map< + std::string, // key_type + basic_json, // value_type + std::less<std::string>, // key_compare + std::allocator<std::pair<const std::string, basic_json>> // allocator_type + > + @endcode + + #### Behavior + + The choice of @a object_t influences the behavior of the JSON class. With + the default type, objects have the following behavior: + + - When all names are unique, objects will be interoperable in the sense + that all software implementations receiving that object will agree on + the name-value mappings. + - When the names within an object are not unique, it is unspecified which + one of the values for a given key will be chosen. For instance, + `{"key": 2, "key": 1}` could be equal to either `{"key": 1}` or + `{"key": 2}`. + - Internally, name/value pairs are stored in lexicographical order of the + names. Objects will also be serialized (see @ref dump) in this order. + For instance, `{"b": 1, "a": 2}` and `{"a": 2, "b": 1}` will be stored + and serialized as `{"a": 2, "b": 1}`. + - When comparing objects, the order of the name/value pairs is irrelevant. + This makes objects interoperable in the sense that they will not be + affected by these differences. For instance, `{"b": 1, "a": 2}` and + `{"a": 2, "b": 1}` will be treated as equal. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the object's limit of nesting is not explicitly constrained. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON object. + + #### Storage + + Objects are stored as pointers in a @ref basic_json type. That is, for any + access to object values, a pointer of type `object_t*` must be + dereferenced. + + @sa @ref array_t -- type for an array value + + @since version 1.0.0 + + @note The order name/value pairs are added to the object is *not* + preserved by the library. Therefore, iterating an object may return + name/value pairs in a different order than they were originally stored. In + fact, keys will be traversed in alphabetical order as `std::map` with + `std::less` is used by default. Please note this behavior conforms to [RFC + 7159](http://rfc7159.net/rfc7159), because any order implements the + specified "unordered" nature of JSON objects. + */ + using object_t = ObjectType<StringType, + basic_json, + object_comparator_t, + AllocatorType<std::pair<const StringType, + basic_json>>>; + + /*! + @brief a type for an array + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON arrays as follows: + > An array is an ordered sequence of zero or more values. + + To store objects in C++, a type is defined by the template parameters + explained below. + + @tparam ArrayType container type to store arrays (e.g., `std::vector` or + `std::list`) + @tparam AllocatorType allocator to use for arrays (e.g., `std::allocator`) + + #### Default type + + With the default values for @a ArrayType (`std::vector`) and @a + AllocatorType (`std::allocator`), the default value for @a array_t is: + + @code {.cpp} + std::vector< + basic_json, // value_type + std::allocator<basic_json> // allocator_type + > + @endcode + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the maximum depth of nesting. + + In this class, the array's limit of nesting is not explicitly constrained. + However, a maximum depth of nesting may be introduced by the compiler or + runtime environment. A theoretical limit can be queried by calling the + @ref max_size function of a JSON array. + + #### Storage + + Arrays are stored as pointers in a @ref basic_json type. That is, for any + access to array values, a pointer of type `array_t*` must be dereferenced. + + @sa @ref object_t -- type for an object value + + @since version 1.0.0 + */ + using array_t = ArrayType<basic_json, AllocatorType<basic_json>>; + + /*! + @brief a type for a string + + [RFC 7159](http://rfc7159.net/rfc7159) describes JSON strings as follows: + > A string is a sequence of zero or more Unicode characters. + + To store objects in C++, a type is defined by the template parameter + described below. Unicode values are split by the JSON class into + byte-sized characters during deserialization. + + @tparam StringType the container to store strings (e.g., `std::string`). + Note this container is used for keys/names in objects, see @ref object_t. + + #### Default type + + With the default values for @a StringType (`std::string`), the default + value for @a string_t is: + + @code {.cpp} + std::string + @endcode + + #### Encoding + + Strings are stored in UTF-8 encoding. Therefore, functions like + `std::string::size()` or `std::string::length()` return the number of + bytes in the string rather than the number of characters or glyphs. + + #### String comparison + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > Software implementations are typically required to test names of object + > members for equality. Implementations that transform the textual + > representation into sequences of Unicode code units and then perform the + > comparison numerically, code unit by code unit, are interoperable in the + > sense that implementations will agree in all cases on equality or + > inequality of two strings. For example, implementations that compare + > strings with escaped characters unconverted may incorrectly find that + > `"a\\b"` and `"a\u005Cb"` are not equal. + + This implementation is interoperable as it does compare strings code unit + by code unit. + + #### Storage + + String values are stored as pointers in a @ref basic_json type. That is, + for any access to string values, a pointer of type `string_t*` must be + dereferenced. + + @since version 1.0.0 + */ + using string_t = StringType; + + /*! + @brief a type for a boolean + + [RFC 7159](http://rfc7159.net/rfc7159) implicitly describes a boolean as a + type which differentiates the two literals `true` and `false`. + + To store objects in C++, a type is defined by the template parameter @a + BooleanType which chooses the type to use. + + #### Default type + + With the default values for @a BooleanType (`bool`), the default value for + @a boolean_t is: + + @code {.cpp} + bool + @endcode + + #### Storage + + Boolean values are stored directly inside a @ref basic_json type. + + @since version 1.0.0 + */ + using boolean_t = BooleanType; + + /*! + @brief a type for a number (integer) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store integer numbers in C++, a type is defined by the template + parameter @a NumberIntegerType which chooses the type to use. + + #### Default type + + With the default values for @a NumberIntegerType (`int64_t`), the default + value for @a number_integer_t is: + + @code {.cpp} + int64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `9223372036854775807` (INT64_MAX) and the minimal integer number + that can be stored is `-9223372036854775808` (INT64_MIN). Integer numbers + that are out of range will yield over/underflow when used in a + constructor. During deserialization, too large or small integer numbers + will be automatically be stored as @ref number_unsigned_t or @ref + number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange of the exactly supported range [INT64_MIN, + INT64_MAX], this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_integer_t = NumberIntegerType; + + /*! + @brief a type for a number (unsigned) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store unsigned integer numbers in C++, a type is defined by the + template parameter @a NumberUnsignedType which chooses the type to use. + + #### Default type + + With the default values for @a NumberUnsignedType (`uint64_t`), the + default value for @a number_unsigned_t is: + + @code {.cpp} + uint64_t + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in integer literals lead to an interpretation as octal + number. Internally, the value will be stored as decimal number. For + instance, the C++ integer literal `010` will be serialized to `8`. + During deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) specifies: + > An implementation may set limits on the range and precision of numbers. + + When the default type is used, the maximal integer number that can be + stored is `18446744073709551615` (UINT64_MAX) and the minimal integer + number that can be stored is `0`. Integer numbers that are out of range + will yield over/underflow when used in a constructor. During + deserialization, too large or small integer numbers will be automatically + be stored as @ref number_integer_t or @ref number_float_t. + + [RFC 7159](http://rfc7159.net/rfc7159) further states: + > Note that when such software is used, numbers that are integers and are + > in the range \f$[-2^{53}+1, 2^{53}-1]\f$ are interoperable in the sense + > that implementations will agree exactly on their numeric values. + + As this range is a subrange (when considered in conjunction with the + number_integer_t type) of the exactly supported range [0, UINT64_MAX], + this class's integer type is interoperable. + + #### Storage + + Integer number values are stored directly inside a @ref basic_json type. + + @sa @ref number_float_t -- type for number values (floating-point) + @sa @ref number_integer_t -- type for number values (integer) + + @since version 2.0.0 + */ + using number_unsigned_t = NumberUnsignedType; + + /*! + @brief a type for a number (floating-point) + + [RFC 7159](http://rfc7159.net/rfc7159) describes numbers as follows: + > The representation of numbers is similar to that used in most + > programming languages. A number is represented in base 10 using decimal + > digits. It contains an integer component that may be prefixed with an + > optional minus sign, which may be followed by a fraction part and/or an + > exponent part. Leading zeros are not allowed. (...) Numeric values that + > cannot be represented in the grammar below (such as Infinity and NaN) + > are not permitted. + + This description includes both integer and floating-point numbers. + However, C++ allows more precise storage if it is known whether the number + is a signed integer, an unsigned integer or a floating-point number. + Therefore, three different types, @ref number_integer_t, @ref + number_unsigned_t and @ref number_float_t are used. + + To store floating-point numbers in C++, a type is defined by the template + parameter @a NumberFloatType which chooses the type to use. + + #### Default type + + With the default values for @a NumberFloatType (`double`), the default + value for @a number_float_t is: + + @code {.cpp} + double + @endcode + + #### Default behavior + + - The restrictions about leading zeros is not enforced in C++. Instead, + leading zeros in floating-point literals will be ignored. Internally, + the value will be stored as decimal number. For instance, the C++ + floating-point literal `01.2` will be serialized to `1.2`. During + deserialization, leading zeros yield an error. + - Not-a-number (NaN) values will be serialized to `null`. + + #### Limits + + [RFC 7159](http://rfc7159.net/rfc7159) states: + > This specification allows implementations to set limits on the range and + > precision of numbers accepted. Since software that implements IEEE + > 754-2008 binary64 (double precision) numbers is generally available and + > widely used, good interoperability can be achieved by implementations + > that expect no more precision or range than these provide, in the sense + > that implementations will approximate JSON numbers within the expected + > precision. + + This implementation does exactly follow this approach, as it uses double + precision floating-point numbers. Note values smaller than + `-1.79769313486232e+308` and values greater than `1.79769313486232e+308` + will be stored as NaN internally and be serialized to `null`. + + #### Storage + + Floating-point number values are stored directly inside a @ref basic_json + type. + + @sa @ref number_integer_t -- type for number values (integer) + + @sa @ref number_unsigned_t -- type for number values (unsigned integer) + + @since version 1.0.0 + */ + using number_float_t = NumberFloatType; + + /// @} + + private: + + /// helper for exception-safe object creation + template<typename T, typename... Args> + static T* create(Args&& ... args) + { + AllocatorType<T> alloc; + using AllocatorTraits = std::allocator_traits<AllocatorType<T>>; + + auto deleter = [&](T * object) + { + AllocatorTraits::deallocate(alloc, object, 1); + }; + std::unique_ptr<T, decltype(deleter)> object(AllocatorTraits::allocate(alloc, 1), deleter); + AllocatorTraits::construct(alloc, object.get(), std::forward<Args>(args)...); + assert(object != nullptr); + return object.release(); + } + + //////////////////////// + // JSON value storage // + //////////////////////// + + /*! + @brief a JSON value + + The actual storage for a JSON value of the @ref basic_json class. This + union combines the different storage types for the JSON value types + defined in @ref value_t. + + JSON type | value_t type | used type + --------- | --------------- | ------------------------ + object | object | pointer to @ref object_t + array | array | pointer to @ref array_t + string | string | pointer to @ref string_t + boolean | boolean | @ref boolean_t + number | number_integer | @ref number_integer_t + number | number_unsigned | @ref number_unsigned_t + number | number_float | @ref number_float_t + null | null | *no value is stored* + + @note Variable-length types (objects, arrays, and strings) are stored as + pointers. The size of the union should not exceed 64 bits if the default + value types are used. + + @since version 1.0.0 + */ + union json_value + { + /// object (stored with pointer to save storage) + object_t* object; + /// array (stored with pointer to save storage) + array_t* array; + /// string (stored with pointer to save storage) + string_t* string; + /// boolean + boolean_t boolean; + /// number (integer) + number_integer_t number_integer; + /// number (unsigned integer) + number_unsigned_t number_unsigned; + /// number (floating-point) + number_float_t number_float; + + /// default constructor (for null values) + json_value() = default; + /// constructor for booleans + json_value(boolean_t v) noexcept : boolean(v) {} + /// constructor for numbers (integer) + json_value(number_integer_t v) noexcept : number_integer(v) {} + /// constructor for numbers (unsigned) + json_value(number_unsigned_t v) noexcept : number_unsigned(v) {} + /// constructor for numbers (floating-point) + json_value(number_float_t v) noexcept : number_float(v) {} + /// constructor for empty values of a given type + json_value(value_t t) + { + switch (t) + { + case value_t::object: + { + object = create<object_t>(); + break; + } + + case value_t::array: + { + array = create<array_t>(); + break; + } + + case value_t::string: + { + string = create<string_t>(""); + break; + } + + case value_t::boolean: + { + boolean = boolean_t(false); + break; + } + + case value_t::number_integer: + { + number_integer = number_integer_t(0); + break; + } + + case value_t::number_unsigned: + { + number_unsigned = number_unsigned_t(0); + break; + } + + case value_t::number_float: + { + number_float = number_float_t(0.0); + break; + } + + case value_t::null: + { + object = nullptr; // silence warning, see #821 + break; + } + + default: + { + object = nullptr; // silence warning, see #821 + if (JSON_UNLIKELY(t == value_t::null)) + { + JSON_THROW(other_error::create(500, "961c151d2e87f2686a955a9be24d316f1362bf21 3.5.0")); // LCOV_EXCL_LINE + } + break; + } + } + } + + /// constructor for strings + json_value(const string_t& value) + { + string = create<string_t>(value); + } + + /// constructor for rvalue strings + json_value(string_t&& value) + { + string = create<string_t>(std::move(value)); + } + + /// constructor for objects + json_value(const object_t& value) + { + object = create<object_t>(value); + } + + /// constructor for rvalue objects + json_value(object_t&& value) + { + object = create<object_t>(std::move(value)); + } + + /// constructor for arrays + json_value(const array_t& value) + { + array = create<array_t>(value); + } + + /// constructor for rvalue arrays + json_value(array_t&& value) + { + array = create<array_t>(std::move(value)); + } + + void destroy(value_t t) noexcept + { + switch (t) + { + case value_t::object: + { + AllocatorType<object_t> alloc; + std::allocator_traits<decltype(alloc)>::destroy(alloc, object); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, object, 1); + break; + } + + case value_t::array: + { + AllocatorType<array_t> alloc; + std::allocator_traits<decltype(alloc)>::destroy(alloc, array); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, array, 1); + break; + } + + case value_t::string: + { + AllocatorType<string_t> alloc; + std::allocator_traits<decltype(alloc)>::destroy(alloc, string); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, string, 1); + break; + } + + default: + { + break; + } + } + } + }; + + /*! + @brief checks the class invariants + + This function asserts the class invariants. It needs to be called at the + end of every constructor to make sure that created objects respect the + invariant. Furthermore, it has to be called each time the type of a JSON + value is changed, because the invariant expresses a relationship between + @a m_type and @a m_value. + */ + void assert_invariant() const noexcept + { + assert(m_type != value_t::object or m_value.object != nullptr); + assert(m_type != value_t::array or m_value.array != nullptr); + assert(m_type != value_t::string or m_value.string != nullptr); + } + + public: + ////////////////////////// + // JSON parser callback // + ////////////////////////// + + /*! + @brief parser event types + + The parser callback distinguishes the following events: + - `object_start`: the parser read `{` and started to process a JSON object + - `key`: the parser read a key of a value in an object + - `object_end`: the parser read `}` and finished processing a JSON object + - `array_start`: the parser read `[` and started to process a JSON array + - `array_end`: the parser read `]` and finished processing a JSON array + - `value`: the parser finished reading a JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + @sa @ref parser_callback_t for more information and examples + */ + using parse_event_t = typename parser::parse_event_t; + + /*! + @brief per-element parser callback type + + With a parser callback function, the result of parsing a JSON text can be + influenced. When passed to @ref parse, it is called on certain events + (passed as @ref parse_event_t via parameter @a event) with a set recursion + depth @a depth and context JSON value @a parsed. The return value of the + callback function is a boolean indicating whether the element that emitted + the callback shall be kept or not. + + We distinguish six scenarios (determined by the event type) in which the + callback function can be called. The following table describes the values + of the parameters @a depth, @a event, and @a parsed. + + parameter @a event | description | parameter @a depth | parameter @a parsed + ------------------ | ----------- | ------------------ | ------------------- + parse_event_t::object_start | the parser read `{` and started to process a JSON object | depth of the parent of the JSON object | a JSON value with type discarded + parse_event_t::key | the parser read a key of a value in an object | depth of the currently parsed JSON object | a JSON string containing the key + parse_event_t::object_end | the parser read `}` and finished processing a JSON object | depth of the parent of the JSON object | the parsed JSON object + parse_event_t::array_start | the parser read `[` and started to process a JSON array | depth of the parent of the JSON array | a JSON value with type discarded + parse_event_t::array_end | the parser read `]` and finished processing a JSON array | depth of the parent of the JSON array | the parsed JSON array + parse_event_t::value | the parser finished reading a JSON value | depth of the value | the parsed JSON value + + @image html callback_events.png "Example when certain parse events are triggered" + + Discarding a value (i.e., returning `false`) has different effects + depending on the context in which function was called: + + - Discarded values in structured types are skipped. That is, the parser + will behave as if the discarded value was never read. + - In case a value outside a structured type is skipped, it is replaced + with `null`. This case happens if the top-level element is skipped. + + @param[in] depth the depth of the recursion during parsing + + @param[in] event an event of type parse_event_t indicating the context in + the callback function has been called + + @param[in,out] parsed the current intermediate parse result; note that + writing to this value has no effect for parse_event_t::key events + + @return Whether the JSON value which called the function during parsing + should be kept (`true`) or not (`false`). In the latter case, it is either + skipped completely or replaced by an empty discarded object. + + @sa @ref parse for examples + + @since version 1.0.0 + */ + using parser_callback_t = typename parser::parser_callback_t; + + ////////////////// + // constructors // + ////////////////// + + /// @name constructors and destructors + /// Constructors of class @ref basic_json, copy/move constructor, copy + /// assignment, static functions creating objects, and the destructor. + /// @{ + + /*! + @brief create an empty value with a given type + + Create an empty JSON value with a given type. The value will be default + initialized with an empty value which depends on the type: + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @param[in] v the type of the value to create + + @complexity Constant. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @liveexample{The following code shows the constructor for different @ref + value_t values,basic_json__value_t} + + @sa @ref clear() -- restores the postcondition of this constructor + + @since version 1.0.0 + */ + basic_json(const value_t v) + : m_type(v), m_value(v) + { + assert_invariant(); + } + + /*! + @brief create a null object + + Create a `null` JSON value. It either takes a null pointer as parameter + (explicitly creating `null`) or no parameter (implicitly creating `null`). + The passed null pointer itself is not read -- it is only used to choose + the right constructor. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @liveexample{The following code shows the constructor with and without a + null pointer parameter.,basic_json__nullptr_t} + + @since version 1.0.0 + */ + basic_json(std::nullptr_t = nullptr) noexcept + : basic_json(value_t::null) + { + assert_invariant(); + } + + /*! + @brief create a JSON value + + This is a "catch all" constructor for all compatible JSON types; that is, + types for which a `to_json()` method exists. The constructor forwards the + parameter @a val to that method (to `json_serializer<U>::to_json` method + with `U = uncvref_t<CompatibleType>`, to be exact). + + Template type @a CompatibleType includes, but is not limited to, the + following types: + - **arrays**: @ref array_t and all kinds of compatible containers such as + `std::vector`, `std::deque`, `std::list`, `std::forward_list`, + `std::array`, `std::valarray`, `std::set`, `std::unordered_set`, + `std::multiset`, and `std::unordered_multiset` with a `value_type` from + which a @ref basic_json value can be constructed. + - **objects**: @ref object_t and all kinds of compatible associative + containers such as `std::map`, `std::unordered_map`, `std::multimap`, + and `std::unordered_multimap` with a `key_type` compatible to + @ref string_t and a `value_type` from which a @ref basic_json value can + be constructed. + - **strings**: @ref string_t, string literals, and all compatible string + containers can be used. + - **numbers**: @ref number_integer_t, @ref number_unsigned_t, + @ref number_float_t, and all convertible number types such as `int`, + `size_t`, `int64_t`, `float` or `double` can be used. + - **boolean**: @ref boolean_t / `bool` can be used. + + See the examples below. + + @tparam CompatibleType a type such that: + - @a CompatibleType is not derived from `std::istream`, + - @a CompatibleType is not @ref basic_json (to avoid hijacking copy/move + constructors), + - @a CompatibleType is not a different @ref basic_json type (i.e. with different template arguments) + - @a CompatibleType is not a @ref basic_json nested type (e.g., + @ref json_pointer, @ref iterator, etc ...) + - @ref @ref json_serializer<U> has a + `to_json(basic_json_t&, CompatibleType&&)` method + + @tparam U = `uncvref_t<CompatibleType>` + + @param[in] val the value to be forwarded to the respective constructor + + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. + + @exceptionsafety Depends on the called constructor. For types directly + supported by the library (i.e., all types for which no `to_json()` function + was provided), strong guarantee holds: if an exception is thrown, there are + no changes to any JSON value. + + @liveexample{The following code shows the constructor with several + compatible types.,basic_json__CompatibleType} + + @since version 2.1.0 + */ + template <typename CompatibleType, + typename U = detail::uncvref_t<CompatibleType>, + detail::enable_if_t< + not detail::is_basic_json<U>::value and detail::is_compatible_type<basic_json_t, U>::value, int> = 0> + basic_json(CompatibleType && val) noexcept(noexcept( + JSONSerializer<U>::to_json(std::declval<basic_json_t&>(), + std::forward<CompatibleType>(val)))) + { + JSONSerializer<U>::to_json(*this, std::forward<CompatibleType>(val)); + assert_invariant(); + } + + /*! + @brief create a JSON value from an existing one + + This is a constructor for existing @ref basic_json types. + It does not hijack copy/move constructors, since the parameter has different + template arguments than the current ones. + + The constructor tries to convert the internal @ref m_value of the parameter. + + @tparam BasicJsonType a type such that: + - @a BasicJsonType is a @ref basic_json type. + - @a BasicJsonType has different template arguments than @ref basic_json_t. + + @param[in] val the @ref basic_json value to be converted. + + @complexity Usually linear in the size of the passed @a val, also + depending on the implementation of the called `to_json()` + method. + + @exceptionsafety Depends on the called constructor. For types directly + supported by the library (i.e., all types for which no `to_json()` function + was provided), strong guarantee holds: if an exception is thrown, there are + no changes to any JSON value. + + @since version 3.2.0 + */ + template <typename BasicJsonType, + detail::enable_if_t< + detail::is_basic_json<BasicJsonType>::value and not std::is_same<basic_json, BasicJsonType>::value, int> = 0> + basic_json(const BasicJsonType& val) + { + using other_boolean_t = typename BasicJsonType::boolean_t; + using other_number_float_t = typename BasicJsonType::number_float_t; + using other_number_integer_t = typename BasicJsonType::number_integer_t; + using other_number_unsigned_t = typename BasicJsonType::number_unsigned_t; + using other_string_t = typename BasicJsonType::string_t; + using other_object_t = typename BasicJsonType::object_t; + using other_array_t = typename BasicJsonType::array_t; + + switch (val.type()) + { + case value_t::boolean: + JSONSerializer<other_boolean_t>::to_json(*this, val.template get<other_boolean_t>()); + break; + case value_t::number_float: + JSONSerializer<other_number_float_t>::to_json(*this, val.template get<other_number_float_t>()); + break; + case value_t::number_integer: + JSONSerializer<other_number_integer_t>::to_json(*this, val.template get<other_number_integer_t>()); + break; + case value_t::number_unsigned: + JSONSerializer<other_number_unsigned_t>::to_json(*this, val.template get<other_number_unsigned_t>()); + break; + case value_t::string: + JSONSerializer<other_string_t>::to_json(*this, val.template get_ref<const other_string_t&>()); + break; + case value_t::object: + JSONSerializer<other_object_t>::to_json(*this, val.template get_ref<const other_object_t&>()); + break; + case value_t::array: + JSONSerializer<other_array_t>::to_json(*this, val.template get_ref<const other_array_t&>()); + break; + case value_t::null: + *this = nullptr; + break; + case value_t::discarded: + m_type = value_t::discarded; + break; + } + assert_invariant(); + } + + /*! + @brief create a container (array or object) from an initializer list + + Creates a JSON value of type array or object from the passed initializer + list @a init. In case @a type_deduction is `true` (default), the type of + the JSON value to be created is deducted from the initializer list @a init + according to the following rules: + + 1. If the list is empty, an empty JSON object value `{}` is created. + 2. If the list consists of pairs whose first element is a string, a JSON + object value is created where the first elements of the pairs are + treated as keys and the second elements are as values. + 3. In all other cases, an array is created. + + The rules aim to create the best fit between a C++ initializer list and + JSON values. The rationale is as follows: + + 1. The empty initializer list is written as `{}` which is exactly an empty + JSON object. + 2. C++ has no way of describing mapped types other than to list a list of + pairs. As JSON requires that keys must be of type string, rule 2 is the + weakest constraint one can pose on initializer lists to interpret them + as an object. + 3. In all other cases, the initializer list could not be interpreted as + JSON object type, so interpreting it as JSON array type is safe. + + With the rules described above, the following JSON values cannot be + expressed by an initializer list: + + - the empty array (`[]`): use @ref array(initializer_list_t) + with an empty initializer list in this case + - arrays whose elements satisfy rule 2: use @ref + array(initializer_list_t) with the same initializer list + in this case + + @note When used without parentheses around an empty initializer list, @ref + basic_json() is called instead of this function, yielding the JSON null + value. + + @param[in] init initializer list with JSON values + + @param[in] type_deduction internal parameter; when set to `true`, the type + of the JSON value is deducted from the initializer list @a init; when set + to `false`, the type provided via @a manual_type is forced. This mode is + used by the functions @ref array(initializer_list_t) and + @ref object(initializer_list_t). + + @param[in] manual_type internal parameter; when @a type_deduction is set + to `false`, the created JSON value will use the provided type (only @ref + value_t::array and @ref value_t::object are valid); when @a type_deduction + is set to `true`, this parameter has no effect + + @throw type_error.301 if @a type_deduction is `false`, @a manual_type is + `value_t::object`, but @a init contains an element which is not a pair + whose first element is a string. In this case, the constructor could not + create an object. If @a type_deduction would have be `true`, an array + would have been created. See @ref object(initializer_list_t) + for an example. + + @complexity Linear in the size of the initializer list @a init. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @liveexample{The example below shows how JSON values are created from + initializer lists.,basic_json__list_init_t} + + @sa @ref array(initializer_list_t) -- create a JSON array + value from an initializer list + @sa @ref object(initializer_list_t) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + basic_json(initializer_list_t init, + bool type_deduction = true, + value_t manual_type = value_t::array) + { + // check if each element is an array with two elements whose first + // element is a string + bool is_an_object = std::all_of(init.begin(), init.end(), + [](const detail::json_ref<basic_json>& element_ref) + { + return (element_ref->is_array() and element_ref->size() == 2 and (*element_ref)[0].is_string()); + }); + + // adjust type if type deduction is not wanted + if (not type_deduction) + { + // if array is wanted, do not create an object though possible + if (manual_type == value_t::array) + { + is_an_object = false; + } + + // if object is wanted but impossible, throw an exception + if (JSON_UNLIKELY(manual_type == value_t::object and not is_an_object)) + { + JSON_THROW(type_error::create(301, "cannot create object from initializer list")); + } + } + + if (is_an_object) + { + // the initializer list is a list of pairs -> create object + m_type = value_t::object; + m_value = value_t::object; + + std::for_each(init.begin(), init.end(), [this](const detail::json_ref<basic_json>& element_ref) + { + auto element = element_ref.moved_or_copied(); + m_value.object->emplace( + std::move(*((*element.m_value.array)[0].m_value.string)), + std::move((*element.m_value.array)[1])); + }); + } + else + { + // the initializer list describes an array -> create array + m_type = value_t::array; + m_value.array = create<array_t>(init.begin(), init.end()); + } + + assert_invariant(); + } + + /*! + @brief explicitly create an array from an initializer list + + Creates a JSON array value from a given initializer list. That is, given a + list of values `a, b, c`, creates the JSON value `[a, b, c]`. If the + initializer list is empty, the empty array `[]` is created. + + @note This function is only needed to express two edge cases that cannot + be realized with the initializer list constructor (@ref + basic_json(initializer_list_t, bool, value_t)). These cases + are: + 1. creating an array whose elements are all pairs whose first element is a + string -- in this case, the initializer list constructor would create an + object, taking the first elements as keys + 2. creating an empty array -- passing the empty initializer list to the + initializer list constructor yields an empty object + + @param[in] init initializer list with JSON values to create an array from + (optional) + + @return JSON array value + + @complexity Linear in the size of @a init. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @liveexample{The following code shows an example for the `array` + function.,array} + + @sa @ref basic_json(initializer_list_t, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref object(initializer_list_t) -- create a JSON object + value from an initializer list + + @since version 1.0.0 + */ + static basic_json array(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::array); + } + + /*! + @brief explicitly create an object from an initializer list + + Creates a JSON object value from a given initializer list. The initializer + lists elements must be pairs, and their first elements must be strings. If + the initializer list is empty, the empty object `{}` is created. + + @note This function is only added for symmetry reasons. In contrast to the + related function @ref array(initializer_list_t), there are + no cases which can only be expressed by this function. That is, any + initializer list @a init can also be passed to the initializer list + constructor @ref basic_json(initializer_list_t, bool, value_t). + + @param[in] init initializer list to create an object from (optional) + + @return JSON object value + + @throw type_error.301 if @a init is not a list of pairs whose first + elements are strings. In this case, no object can be created. When such a + value is passed to @ref basic_json(initializer_list_t, bool, value_t), + an array would have been created from the passed initializer list @a init. + See example below. + + @complexity Linear in the size of @a init. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @liveexample{The following code shows an example for the `object` + function.,object} + + @sa @ref basic_json(initializer_list_t, bool, value_t) -- + create a JSON value from an initializer list + @sa @ref array(initializer_list_t) -- create a JSON array + value from an initializer list + + @since version 1.0.0 + */ + static basic_json object(initializer_list_t init = {}) + { + return basic_json(init, false, value_t::object); + } + + /*! + @brief construct an array with count copies of given value + + Constructs a JSON array value by creating @a cnt copies of a passed value. + In case @a cnt is `0`, an empty array is created. + + @param[in] cnt the number of JSON copies of @a val to create + @param[in] val the JSON value to copy + + @post `std::distance(begin(),end()) == cnt` holds. + + @complexity Linear in @a cnt. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @liveexample{The following code shows examples for the @ref + basic_json(size_type\, const basic_json&) + constructor.,basic_json__size_type_basic_json} + + @since version 1.0.0 + */ + basic_json(size_type cnt, const basic_json& val) + : m_type(value_t::array) + { + m_value.array = create<array_t>(cnt, val); + assert_invariant(); + } + + /*! + @brief construct a JSON container given an iterator range + + Constructs the JSON value with the contents of the range `[first, last)`. + The semantics depends on the different types a JSON value can have: + - In case of a null type, invalid_iterator.206 is thrown. + - In case of other primitive types (number, boolean, or string), @a first + must be `begin()` and @a last must be `end()`. In this case, the value is + copied. Otherwise, invalid_iterator.204 is thrown. + - In case of structured types (array, object), the constructor behaves as + similar versions for `std::vector` or `std::map`; that is, a JSON array + or object is constructed from the values in the range. + + @tparam InputIT an input iterator type (@ref iterator or @ref + const_iterator) + + @param[in] first begin of the range to copy from (included) + @param[in] last end of the range to copy from (excluded) + + @pre Iterators @a first and @a last must be initialized. **This + precondition is enforced with an assertion (see warning).** If + assertions are switched off, a violation of this precondition yields + undefined behavior. + + @pre Range `[first, last)` is valid. Usually, this precondition cannot be + checked efficiently. Only certain edge cases are detected; see the + description of the exceptions below. A violation of this precondition + yields undefined behavior. + + @warning A precondition is enforced with a runtime assertion that will + result in calling `std::abort` if this precondition is not met. + Assertions can be disabled by defining `NDEBUG` at compile time. + See https://en.cppreference.com/w/cpp/error/assert for more + information. + + @throw invalid_iterator.201 if iterators @a first and @a last are not + compatible (i.e., do not belong to the same JSON value). In this case, + the range `[first, last)` is undefined. + @throw invalid_iterator.204 if iterators @a first and @a last belong to a + primitive type (number, boolean, or string), but @a first does not point + to the first element any more. In this case, the range `[first, last)` is + undefined. See example code below. + @throw invalid_iterator.206 if iterators @a first and @a last belong to a + null value. In this case, the range `[first, last)` is undefined. + + @complexity Linear in distance between @a first and @a last. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @liveexample{The example below shows several ways to create JSON values by + specifying a subrange with iterators.,basic_json__InputIt_InputIt} + + @since version 1.0.0 + */ + template<class InputIT, typename std::enable_if< + std::is_same<InputIT, typename basic_json_t::iterator>::value or + std::is_same<InputIT, typename basic_json_t::const_iterator>::value, int>::type = 0> + basic_json(InputIT first, InputIT last) + { + assert(first.m_object != nullptr); + assert(last.m_object != nullptr); + + // make sure iterator fits the current value + if (JSON_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(201, "iterators are not compatible")); + } + + // copy type from first iterator + m_type = first.m_object->m_type; + + // check if iterator range is complete for primitive values + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (JSON_UNLIKELY(not first.m_it.primitive_iterator.is_begin() + or not last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); + } + break; + } + + default: + break; + } + + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = first.m_object->m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = first.m_object->m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value.number_float = first.m_object->m_value.number_float; + break; + } + + case value_t::boolean: + { + m_value.boolean = first.m_object->m_value.boolean; + break; + } + + case value_t::string: + { + m_value = *first.m_object->m_value.string; + break; + } + + case value_t::object: + { + m_value.object = create<object_t>(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + m_value.array = create<array_t>(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + JSON_THROW(invalid_iterator::create(206, "cannot construct with iterators from " + + std::string(first.m_object->type_name()))); + } + + assert_invariant(); + } + + + /////////////////////////////////////// + // other constructors and destructor // + /////////////////////////////////////// + + /// @private + basic_json(const detail::json_ref<basic_json>& ref) + : basic_json(ref.moved_or_copied()) + {} + + /*! + @brief copy constructor + + Creates a copy of a given JSON value. + + @param[in] other the JSON value to copy + + @post `*this == other` + + @complexity Linear in the size of @a other. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes to any JSON value. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is linear. + - As postcondition, it holds: `other == basic_json(other)`. + + @liveexample{The following code shows an example for the copy + constructor.,basic_json__basic_json} + + @since version 1.0.0 + */ + basic_json(const basic_json& other) + : m_type(other.m_type) + { + // check of passed value is valid + other.assert_invariant(); + + switch (m_type) + { + case value_t::object: + { + m_value = *other.m_value.object; + break; + } + + case value_t::array: + { + m_value = *other.m_value.array; + break; + } + + case value_t::string: + { + m_value = *other.m_value.string; + break; + } + + case value_t::boolean: + { + m_value = other.m_value.boolean; + break; + } + + case value_t::number_integer: + { + m_value = other.m_value.number_integer; + break; + } + + case value_t::number_unsigned: + { + m_value = other.m_value.number_unsigned; + break; + } + + case value_t::number_float: + { + m_value = other.m_value.number_float; + break; + } + + default: + break; + } + + assert_invariant(); + } + + /*! + @brief move constructor + + Move constructor. Constructs a JSON value with the contents of the given + value @a other using move semantics. It "steals" the resources from @a + other and leaves it as JSON null value. + + @param[in,out] other value to move to this object + + @post `*this` has the same value as @a other before the call. + @post @a other is a JSON null value. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this constructor never throws + exceptions. + + @requirement This function helps `basic_json` satisfying the + [MoveConstructible](https://en.cppreference.com/w/cpp/named_req/MoveConstructible) + requirements. + + @liveexample{The code below shows the move constructor explicitly called + via std::move.,basic_json__moveconstructor} + + @since version 1.0.0 + */ + basic_json(basic_json&& other) noexcept + : m_type(std::move(other.m_type)), + m_value(std::move(other.m_value)) + { + // check that passed value is valid + other.assert_invariant(); + + // invalidate payload + other.m_type = value_t::null; + other.m_value = {}; + + assert_invariant(); + } + + /*! + @brief copy assignment + + Copy assignment operator. Copies a JSON value via the "copy and swap" + strategy: It is expressed in terms of the copy constructor, destructor, + and the `swap()` member function. + + @param[in] other value to copy from + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is linear. + + @liveexample{The code below shows and example for the copy assignment. It + creates a copy of value `a` which is then swapped with `b`. Finally\, the + copy of `a` (which is the null value after the swap) is + destroyed.,basic_json__copyassignment} + + @since version 1.0.0 + */ + basic_json& operator=(basic_json other) noexcept ( + std::is_nothrow_move_constructible<value_t>::value and + std::is_nothrow_move_assignable<value_t>::value and + std::is_nothrow_move_constructible<json_value>::value and + std::is_nothrow_move_assignable<json_value>::value + ) + { + // check that passed value is valid + other.assert_invariant(); + + using std::swap; + swap(m_type, other.m_type); + swap(m_value, other.m_value); + + assert_invariant(); + return *this; + } + + /*! + @brief destructor + + Destroys the JSON value and frees all allocated memory. + + @complexity Linear. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is linear. + - All stored elements are destroyed and all memory is freed. + + @since version 1.0.0 + */ + ~basic_json() noexcept + { + assert_invariant(); + m_value.destroy(m_type); + } + + /// @} + + public: + /////////////////////// + // object inspection // + /////////////////////// + + /// @name object inspection + /// Functions to inspect the type of a JSON value. + /// @{ + + /*! + @brief serialization + + Serialization function for JSON values. The function tries to mimic + Python's `json.dumps()` function, and currently supports its @a indent + and @a ensure_ascii parameters. + + @param[in] indent If indent is nonnegative, then array elements and object + members will be pretty-printed with that indent level. An indent level of + `0` will only insert newlines. `-1` (the default) selects the most compact + representation. + @param[in] indent_char The character to use for indentation if @a indent is + greater than `0`. The default is ` ` (space). + @param[in] ensure_ascii If @a ensure_ascii is true, all non-ASCII characters + in the output are escaped with `\uXXXX` sequences, and the result consists + of ASCII characters only. + @param[in] error_handler how to react on decoding errors; there are three + possible values: `strict` (throws and exception in case a decoding error + occurs; default), `replace` (replace invalid UTF-8 sequences with U+FFFD), + and `ignore` (ignore invalid UTF-8 sequences during serialization). + + @return string containing the serialization of the JSON value + + @throw type_error.316 if a string stored inside the JSON value is not + UTF-8 encoded + + @complexity Linear. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @liveexample{The following example shows the effect of different @a indent\, + @a indent_char\, and @a ensure_ascii parameters to the result of the + serialization.,dump} + + @see https://docs.python.org/2/library/json.html#json.dump + + @since version 1.0.0; indentation character @a indent_char, option + @a ensure_ascii and exceptions added in version 3.0.0; error + handlers added in version 3.4.0. + */ + string_t dump(const int indent = -1, + const char indent_char = ' ', + const bool ensure_ascii = false, + const error_handler_t error_handler = error_handler_t::strict) const + { + string_t result; + serializer s(detail::output_adapter<char, string_t>(result), indent_char, error_handler); + + if (indent >= 0) + { + s.dump(*this, true, ensure_ascii, static_cast<unsigned int>(indent)); + } + else + { + s.dump(*this, false, ensure_ascii, 0); + } + + return result; + } + + /*! + @brief return the type of the JSON value (explicit) + + Return the type of the JSON value as a value from the @ref value_t + enumeration. + + @return the type of the JSON value + Value type | return value + ------------------------- | ------------------------- + null | value_t::null + boolean | value_t::boolean + string | value_t::string + number (integer) | value_t::number_integer + number (unsigned integer) | value_t::number_unsigned + number (floating-point) | value_t::number_float + object | value_t::object + array | value_t::array + discarded | value_t::discarded + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `type()` for all JSON + types.,type} + + @sa @ref operator value_t() -- return the type of the JSON value (implicit) + @sa @ref type_name() -- return the type as string + + @since version 1.0.0 + */ + constexpr value_t type() const noexcept + { + return m_type; + } + + /*! + @brief return whether type is primitive + + This function returns true if and only if the JSON type is primitive + (string, number, boolean, or null). + + @return `true` if type is primitive (string, number, boolean, or null), + `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_primitive()` for all JSON + types.,is_primitive} + + @sa @ref is_structured() -- returns whether JSON value is structured + @sa @ref is_null() -- returns whether JSON value is `null` + @sa @ref is_string() -- returns whether JSON value is a string + @sa @ref is_boolean() -- returns whether JSON value is a boolean + @sa @ref is_number() -- returns whether JSON value is a number + + @since version 1.0.0 + */ + constexpr bool is_primitive() const noexcept + { + return is_null() or is_string() or is_boolean() or is_number(); + } + + /*! + @brief return whether type is structured + + This function returns true if and only if the JSON type is structured + (array or object). + + @return `true` if type is structured (array or object), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_structured()` for all JSON + types.,is_structured} + + @sa @ref is_primitive() -- returns whether value is primitive + @sa @ref is_array() -- returns whether value is an array + @sa @ref is_object() -- returns whether value is an object + + @since version 1.0.0 + */ + constexpr bool is_structured() const noexcept + { + return is_array() or is_object(); + } + + /*! + @brief return whether value is null + + This function returns true if and only if the JSON value is null. + + @return `true` if type is null, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_null()` for all JSON + types.,is_null} + + @since version 1.0.0 + */ + constexpr bool is_null() const noexcept + { + return (m_type == value_t::null); + } + + /*! + @brief return whether value is a boolean + + This function returns true if and only if the JSON value is a boolean. + + @return `true` if type is boolean, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_boolean()` for all JSON + types.,is_boolean} + + @since version 1.0.0 + */ + constexpr bool is_boolean() const noexcept + { + return (m_type == value_t::boolean); + } + + /*! + @brief return whether value is a number + + This function returns true if and only if the JSON value is a number. This + includes both integer (signed and unsigned) and floating-point values. + + @return `true` if type is number (regardless whether integer, unsigned + integer or floating-type), `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number()` for all JSON + types.,is_number} + + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number() const noexcept + { + return is_number_integer() or is_number_float(); + } + + /*! + @brief return whether value is an integer number + + This function returns true if and only if the JSON value is a signed or + unsigned integer number. This excludes floating-point values. + + @return `true` if type is an integer or unsigned integer number, `false` + otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_integer()` for all + JSON types.,is_number_integer} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 1.0.0 + */ + constexpr bool is_number_integer() const noexcept + { + return (m_type == value_t::number_integer or m_type == value_t::number_unsigned); + } + + /*! + @brief return whether value is an unsigned integer number + + This function returns true if and only if the JSON value is an unsigned + integer number. This excludes floating-point and signed integer values. + + @return `true` if type is an unsigned integer number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_unsigned()` for all + JSON types.,is_number_unsigned} + + @sa @ref is_number() -- check if value is a number + @sa @ref is_number_integer() -- check if value is an integer or unsigned + integer number + @sa @ref is_number_float() -- check if value is a floating-point number + + @since version 2.0.0 + */ + constexpr bool is_number_unsigned() const noexcept + { + return (m_type == value_t::number_unsigned); + } + + /*! + @brief return whether value is a floating-point number + + This function returns true if and only if the JSON value is a + floating-point number. This excludes signed and unsigned integer values. + + @return `true` if type is a floating-point number, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_number_float()` for all + JSON types.,is_number_float} + + @sa @ref is_number() -- check if value is number + @sa @ref is_number_integer() -- check if value is an integer number + @sa @ref is_number_unsigned() -- check if value is an unsigned integer + number + + @since version 1.0.0 + */ + constexpr bool is_number_float() const noexcept + { + return (m_type == value_t::number_float); + } + + /*! + @brief return whether value is an object + + This function returns true if and only if the JSON value is an object. + + @return `true` if type is object, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_object()` for all JSON + types.,is_object} + + @since version 1.0.0 + */ + constexpr bool is_object() const noexcept + { + return (m_type == value_t::object); + } + + /*! + @brief return whether value is an array + + This function returns true if and only if the JSON value is an array. + + @return `true` if type is array, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_array()` for all JSON + types.,is_array} + + @since version 1.0.0 + */ + constexpr bool is_array() const noexcept + { + return (m_type == value_t::array); + } + + /*! + @brief return whether value is a string + + This function returns true if and only if the JSON value is a string. + + @return `true` if type is string, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_string()` for all JSON + types.,is_string} + + @since version 1.0.0 + */ + constexpr bool is_string() const noexcept + { + return (m_type == value_t::string); + } + + /*! + @brief return whether value is discarded + + This function returns true if and only if the JSON value was discarded + during parsing with a callback function (see @ref parser_callback_t). + + @note This function will always be `false` for JSON values after parsing. + That is, discarded values can only occur during parsing, but will be + removed when inside a structured value or replaced by null in other cases. + + @return `true` if type is discarded, `false` otherwise. + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies `is_discarded()` for all JSON + types.,is_discarded} + + @since version 1.0.0 + */ + constexpr bool is_discarded() const noexcept + { + return (m_type == value_t::discarded); + } + + /*! + @brief return the type of the JSON value (implicit) + + Implicitly return the type of the JSON value as a value from the @ref + value_t enumeration. + + @return the type of the JSON value + + @complexity Constant. + + @exceptionsafety No-throw guarantee: this member function never throws + exceptions. + + @liveexample{The following code exemplifies the @ref value_t operator for + all JSON types.,operator__value_t} + + @sa @ref type() -- return the type of the JSON value (explicit) + @sa @ref type_name() -- return the type as string + + @since version 1.0.0 + */ + constexpr operator value_t() const noexcept + { + return m_type; + } + + /// @} + + private: + ////////////////// + // value access // + ////////////////// + + /// get a boolean (explicit) + boolean_t get_impl(boolean_t* /*unused*/) const + { + if (JSON_LIKELY(is_boolean())) + { + return m_value.boolean; + } + + JSON_THROW(type_error::create(302, "type must be boolean, but is " + std::string(type_name()))); + } + + /// get a pointer to the value (object) + object_t* get_impl_ptr(object_t* /*unused*/) noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (object) + constexpr const object_t* get_impl_ptr(const object_t* /*unused*/) const noexcept + { + return is_object() ? m_value.object : nullptr; + } + + /// get a pointer to the value (array) + array_t* get_impl_ptr(array_t* /*unused*/) noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (array) + constexpr const array_t* get_impl_ptr(const array_t* /*unused*/) const noexcept + { + return is_array() ? m_value.array : nullptr; + } + + /// get a pointer to the value (string) + string_t* get_impl_ptr(string_t* /*unused*/) noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (string) + constexpr const string_t* get_impl_ptr(const string_t* /*unused*/) const noexcept + { + return is_string() ? m_value.string : nullptr; + } + + /// get a pointer to the value (boolean) + boolean_t* get_impl_ptr(boolean_t* /*unused*/) noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (boolean) + constexpr const boolean_t* get_impl_ptr(const boolean_t* /*unused*/) const noexcept + { + return is_boolean() ? &m_value.boolean : nullptr; + } + + /// get a pointer to the value (integer number) + number_integer_t* get_impl_ptr(number_integer_t* /*unused*/) noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (integer number) + constexpr const number_integer_t* get_impl_ptr(const number_integer_t* /*unused*/) const noexcept + { + return is_number_integer() ? &m_value.number_integer : nullptr; + } + + /// get a pointer to the value (unsigned number) + number_unsigned_t* get_impl_ptr(number_unsigned_t* /*unused*/) noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (unsigned number) + constexpr const number_unsigned_t* get_impl_ptr(const number_unsigned_t* /*unused*/) const noexcept + { + return is_number_unsigned() ? &m_value.number_unsigned : nullptr; + } + + /// get a pointer to the value (floating-point number) + number_float_t* get_impl_ptr(number_float_t* /*unused*/) noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /// get a pointer to the value (floating-point number) + constexpr const number_float_t* get_impl_ptr(const number_float_t* /*unused*/) const noexcept + { + return is_number_float() ? &m_value.number_float : nullptr; + } + + /*! + @brief helper function to implement get_ref() + + This function helps to implement get_ref() without code duplication for + const and non-const overloads + + @tparam ThisType will be deduced as `basic_json` or `const basic_json` + + @throw type_error.303 if ReferenceType does not match underlying value + type of the current JSON + */ + template<typename ReferenceType, typename ThisType> + static ReferenceType get_ref_impl(ThisType& obj) + { + // delegate the call to get_ptr<>() + auto ptr = obj.template get_ptr<typename std::add_pointer<ReferenceType>::type>(); + + if (JSON_LIKELY(ptr != nullptr)) + { + return *ptr; + } + + JSON_THROW(type_error::create(303, "incompatible ReferenceType for get_ref, actual type is " + std::string(obj.type_name()))); + } + + public: + /// @name value access + /// Direct access to the stored value of a JSON value. + /// @{ + + /*! + @brief get special-case overload + + This overloads avoids a lot of template boilerplate, it can be seen as the + identity method + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this + + @complexity Constant. + + @since version 2.1.0 + */ + template<typename BasicJsonType, detail::enable_if_t< + std::is_same<typename std::remove_const<BasicJsonType>::type, basic_json_t>::value, + int> = 0> + basic_json get() const + { + return *this; + } + + /*! + @brief get special-case overload + + This overloads converts the current @ref basic_json in a different + @ref basic_json type + + @tparam BasicJsonType == @ref basic_json + + @return a copy of *this, converted into @tparam BasicJsonType + + @complexity Depending on the implementation of the called `from_json()` + method. + + @since version 3.2.0 + */ + template<typename BasicJsonType, detail::enable_if_t< + not std::is_same<BasicJsonType, basic_json>::value and + detail::is_basic_json<BasicJsonType>::value, int> = 0> + BasicJsonType get() const + { + return *this; + } + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value + which is [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer<ValueType> + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType ret; + JSONSerializer<ValueType>::from_json(*this, ret); + return ret; + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer<ValueType> has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + - @ref json_serializer<ValueType> does not have a `from_json()` method of + the form `ValueType from_json(const basic_json&)` + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer<ValueType> `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,get__ValueType_const} + + @since version 2.1.0 + */ + template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, + detail::enable_if_t < + not detail::is_basic_json<ValueType>::value and + detail::has_from_json<basic_json_t, ValueType>::value and + not detail::has_non_default_from_json<basic_json_t, ValueType>::value, + int> = 0> + ValueType get() const noexcept(noexcept( + JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), std::declval<ValueType&>()))) + { + // we cannot static_assert on ValueTypeCV being non-const, because + // there is support for get<const basic_json_t>(), which is why we + // still need the uncvref + static_assert(not std::is_reference<ValueTypeCV>::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + static_assert(std::is_default_constructible<ValueType>::value, + "types must be DefaultConstructible when used with get()"); + + ValueType ret; + JSONSerializer<ValueType>::from_json(*this, ret); + return ret; + } + + /*! + @brief get a value (explicit); special case + + Explicit type conversion between the JSON value and a compatible value + which is **not** [CopyConstructible](https://en.cppreference.com/w/cpp/named_req/CopyConstructible) + and **not** [DefaultConstructible](https://en.cppreference.com/w/cpp/named_req/DefaultConstructible). + The value is converted by calling the @ref json_serializer<ValueType> + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + return JSONSerializer<ValueTypeCV>::from_json(*this); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json and + - @ref json_serializer<ValueType> has a `from_json()` method of the form + `ValueType from_json(const basic_json&)` + + @note If @ref json_serializer<ValueType> has both overloads of + `from_json()`, this one is chosen. + + @tparam ValueTypeCV the provided value type + @tparam ValueType the returned value type + + @return copy of the JSON value, converted to @a ValueType + + @throw what @ref json_serializer<ValueType> `from_json()` method throws + + @since version 2.1.0 + */ + template<typename ValueTypeCV, typename ValueType = detail::uncvref_t<ValueTypeCV>, + detail::enable_if_t<not std::is_same<basic_json_t, ValueType>::value and + detail::has_non_default_from_json<basic_json_t, ValueType>::value, + int> = 0> + ValueType get() const noexcept(noexcept( + JSONSerializer<ValueTypeCV>::from_json(std::declval<const basic_json_t&>()))) + { + static_assert(not std::is_reference<ValueTypeCV>::value, + "get() cannot be used with reference types, you might want to use get_ref()"); + return JSONSerializer<ValueTypeCV>::from_json(*this); + } + + /*! + @brief get a value (explicit) + + Explicit type conversion between the JSON value and a compatible value. + The value is filled into the input parameter by calling the @ref json_serializer<ValueType> + `from_json()` method. + + The function is equivalent to executing + @code {.cpp} + ValueType v; + JSONSerializer<ValueType>::from_json(*this, v); + @endcode + + This overloads is chosen if: + - @a ValueType is not @ref basic_json, + - @ref json_serializer<ValueType> has a `from_json()` method of the form + `void from_json(const basic_json&, ValueType&)`, and + + @tparam ValueType the input parameter type. + + @return the input parameter, allowing chaining calls. + + @throw what @ref json_serializer<ValueType> `from_json()` method throws + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,get_to} + + @since version 3.3.0 + */ + template<typename ValueType, + detail::enable_if_t < + not detail::is_basic_json<ValueType>::value and + detail::has_from_json<basic_json_t, ValueType>::value, + int> = 0> + ValueType & get_to(ValueType& v) const noexcept(noexcept( + JSONSerializer<ValueType>::from_json(std::declval<const basic_json_t&>(), v))) + { + JSONSerializer<ValueType>::from_json(*this, v); + return v; + } + + + /*! + @brief get a pointer value (implicit) + + Implicit pointer access to the internally stored JSON value. No copies are + made. + + @warning Writing data to the pointee of the result yields an undefined + state. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. Enforced by a static + assertion. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get_ptr} + + @since version 1.0.0 + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value, int>::type = 0> + auto get_ptr() noexcept -> decltype(std::declval<basic_json_t&>().get_impl_ptr(std::declval<PointerType>())) + { + // delegate the call to get_impl_ptr<>() + return get_impl_ptr(static_cast<PointerType>(nullptr)); + } + + /*! + @brief get a pointer value (implicit) + @copydoc get_ptr() + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value and + std::is_const<typename std::remove_pointer<PointerType>::type>::value, int>::type = 0> + constexpr auto get_ptr() const noexcept -> decltype(std::declval<const basic_json_t&>().get_impl_ptr(std::declval<PointerType>())) + { + // delegate the call to get_impl_ptr<>() const + return get_impl_ptr(static_cast<PointerType>(nullptr)); + } + + /*! + @brief get a pointer value (explicit) + + Explicit pointer access to the internally stored JSON value. No copies are + made. + + @warning The pointer becomes invalid if the underlying JSON object + changes. + + @tparam PointerType pointer type; must be a pointer to @ref array_t, @ref + object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, + @ref number_unsigned_t, or @ref number_float_t. + + @return pointer to the internally stored JSON value if the requested + pointer type @a PointerType fits to the JSON value; `nullptr` otherwise + + @complexity Constant. + + @liveexample{The example below shows how pointers to internal values of a + JSON value can be requested. Note that no type conversions are made and a + `nullptr` is returned if the value and the requested pointer type does not + match.,get__PointerType} + + @sa @ref get_ptr() for explicit pointer-member access + + @since version 1.0.0 + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value, int>::type = 0> + auto get() noexcept -> decltype(std::declval<basic_json_t&>().template get_ptr<PointerType>()) + { + // delegate the call to get_ptr + return get_ptr<PointerType>(); + } + + /*! + @brief get a pointer value (explicit) + @copydoc get() + */ + template<typename PointerType, typename std::enable_if< + std::is_pointer<PointerType>::value, int>::type = 0> + constexpr auto get() const noexcept -> decltype(std::declval<const basic_json_t&>().template get_ptr<PointerType>()) + { + // delegate the call to get_ptr + return get_ptr<PointerType>(); + } + + /*! + @brief get a reference value (implicit) + + Implicit reference access to the internally stored JSON value. No copies + are made. + + @warning Writing data to the referee of the result yields an undefined + state. + + @tparam ReferenceType reference type; must be a reference to @ref array_t, + @ref object_t, @ref string_t, @ref boolean_t, @ref number_integer_t, or + @ref number_float_t. Enforced by static assertion. + + @return reference to the internally stored JSON value if the requested + reference type @a ReferenceType fits to the JSON value; throws + type_error.303 otherwise + + @throw type_error.303 in case passed type @a ReferenceType is incompatible + with the stored JSON value; see example below + + @complexity Constant. + + @liveexample{The example shows several calls to `get_ref()`.,get_ref} + + @since version 1.1.0 + */ + template<typename ReferenceType, typename std::enable_if< + std::is_reference<ReferenceType>::value, int>::type = 0> + ReferenceType get_ref() + { + // delegate call to get_ref_impl + return get_ref_impl<ReferenceType>(*this); + } + + /*! + @brief get a reference value (implicit) + @copydoc get_ref() + */ + template<typename ReferenceType, typename std::enable_if< + std::is_reference<ReferenceType>::value and + std::is_const<typename std::remove_reference<ReferenceType>::type>::value, int>::type = 0> + ReferenceType get_ref() const + { + // delegate call to get_ref_impl + return get_ref_impl<ReferenceType>(*this); + } + + /*! + @brief get a value (implicit) + + Implicit type conversion between the JSON value and a compatible value. + The call is realized by calling @ref get() const. + + @tparam ValueType non-pointer type compatible to the JSON value, for + instance `int` for JSON integer numbers, `bool` for JSON booleans, or + `std::vector` types for JSON arrays. The character type of @ref string_t + as well as an initializer list of this type is excluded to avoid + ambiguities as these types implicitly convert to `std::string`. + + @return copy of the JSON value, converted to type @a ValueType + + @throw type_error.302 in case passed type @a ValueType is incompatible + to the JSON value type (e.g., the JSON value is of type boolean, but a + string is requested); see example below + + @complexity Linear in the size of the JSON value. + + @liveexample{The example below shows several conversions from JSON values + to other types. There a few things to note: (1) Floating-point numbers can + be converted to integers\, (2) A JSON array can be converted to a standard + `std::vector<short>`\, (3) A JSON object can be converted to C++ + associative containers such as `std::unordered_map<std::string\, + json>`.,operator__ValueType} + + @since version 1.0.0 + */ + template < typename ValueType, typename std::enable_if < + not std::is_pointer<ValueType>::value and + not std::is_same<ValueType, detail::json_ref<basic_json>>::value and + not std::is_same<ValueType, typename string_t::value_type>::value and + not detail::is_basic_json<ValueType>::value + +#ifndef _MSC_VER // fix for issue #167 operator<< ambiguity under VS2015 + and not std::is_same<ValueType, std::initializer_list<typename string_t::value_type>>::value +#if defined(JSON_HAS_CPP_17) && defined(_MSC_VER) and _MSC_VER <= 1914 + and not std::is_same<ValueType, typename std::string_view>::value +#endif +#endif + and detail::is_detected<detail::get_template_function, const basic_json_t&, ValueType>::value + , int >::type = 0 > + operator ValueType() const + { + // delegate the call to get<>() const + return get<ValueType>(); + } + + /// @} + + + //////////////////// + // element access // + //////////////////// + + /// @name element access + /// Access to the JSON value. + /// @{ + + /*! + @brief access specified array element with bounds checking + + Returns a reference to the element at specified location @a idx, with + bounds checking. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__size_type} + */ + reference at(size_type idx) + { + // at only works for arrays + if (JSON_LIKELY(is_array())) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()))); + } + } + + /*! + @brief access specified array element with bounds checking + + Returns a const reference to the element at specified location @a idx, + with bounds checking. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw type_error.304 if the JSON value is not an array; in this case, + calling `at` with an index makes no sense. See example below. + @throw out_of_range.401 if the index @a idx is out of range of the array; + that is, `idx >= size()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 1.0.0 + + @liveexample{The example below shows how array elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__size_type_const} + */ + const_reference at(size_type idx) const + { + // at only works for arrays + if (JSON_LIKELY(is_array())) + { + JSON_TRY + { + return m_value.array->at(idx); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()))); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a reference to the element at with specified key @a key, with + bounds checking. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Logarithmic in the size of the container. + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read and + written using `at()`. It also demonstrates the different exceptions that + can be thrown.,at__object_t_key_type} + */ + reference at(const typename object_t::key_type& key) + { + // at only works for objects + if (JSON_LIKELY(is_object())) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()))); + } + } + + /*! + @brief access specified object element with bounds checking + + Returns a const reference to the element at with specified key @a key, + with bounds checking. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @throw type_error.304 if the JSON value is not an object; in this case, + calling `at` with a key makes no sense. See example below. + @throw out_of_range.403 if the key @a key is is not stored in the object; + that is, `find(key) == end()`. See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Logarithmic in the size of the container. + + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + + @liveexample{The example below shows how object elements can be read using + `at()`. It also demonstrates the different exceptions that can be thrown., + at__object_t_key_type_const} + */ + const_reference at(const typename object_t::key_type& key) const + { + // at only works for objects + if (JSON_LIKELY(is_object())) + { + JSON_TRY + { + return m_value.object->at(key); + } + JSON_CATCH (std::out_of_range&) + { + // create better exception explanation + JSON_THROW(out_of_range::create(403, "key '" + key + "' not found")); + } + } + else + { + JSON_THROW(type_error::create(304, "cannot use at() with " + std::string(type_name()))); + } + } + + /*! + @brief access specified array element + + Returns a reference to the element at specified location @a idx. + + @note If @a idx is beyond the range of the array (i.e., `idx >= size()`), + then the array is silently filled up with `null` values to make `idx` a + valid reference to the last stored element. + + @param[in] idx index of the element to access + + @return reference to the element at index @a idx + + @throw type_error.305 if the JSON value is not an array or null; in that + cases, using the [] operator with an index makes no sense. + + @complexity Constant if @a idx is in the range of the array. Otherwise + linear in `idx - size()`. + + @liveexample{The example below shows how array elements can be read and + written using `[]` operator. Note the addition of `null` + values.,operatorarray__size_type} + + @since version 1.0.0 + */ + reference operator[](size_type idx) + { + // implicitly convert null value to an empty array + if (is_null()) + { + m_type = value_t::array; + m_value.array = create<array_t>(); + assert_invariant(); + } + + // operator[] only works for arrays + if (JSON_LIKELY(is_array())) + { + // fill up array with null values if given idx is outside range + if (idx >= m_value.array->size()) + { + m_value.array->insert(m_value.array->end(), + idx - m_value.array->size() + 1, + basic_json()); + } + + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()))); + } + + /*! + @brief access specified array element + + Returns a const reference to the element at specified location @a idx. + + @param[in] idx index of the element to access + + @return const reference to the element at index @a idx + + @throw type_error.305 if the JSON value is not an array; in that case, + using the [] operator with an index makes no sense. + + @complexity Constant. + + @liveexample{The example below shows how array elements can be read using + the `[]` operator.,operatorarray__size_type_const} + + @since version 1.0.0 + */ + const_reference operator[](size_type idx) const + { + // const operator[] only works for arrays + if (JSON_LIKELY(is_array())) + { + return m_value.array->operator[](idx); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a numeric argument with " + std::string(type_name()))); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + reference operator[](const typename object_t::key_type& key) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create<object_t>(); + assert_invariant(); + } + + // operator[] only works for objects + if (JSON_LIKELY(is_object())) + { + return m_value.object->operator[](key); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @pre The element with key @a key must exist. **This precondition is + enforced with an assertion.** + + @throw type_error.305 if the JSON value is not an object; in that case, + using the [] operator with a key makes no sense. + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.0.0 + */ + const_reference operator[](const typename object_t::key_type& key) const + { + // const operator[] only works for objects + if (JSON_LIKELY(is_object())) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); + } + + /*! + @brief access specified object element + + Returns a reference to the element at with specified key @a key. + + @note If @a key is not found in the object, then it is silently added to + the object and filled with a `null` value to make `key` a valid reference. + In case the value was `null` before, it is converted to an object. + + @param[in] key key of the element to access + + @return reference to the element at key @a key + + @throw type_error.305 if the JSON value is not an object or null; in that + cases, using the [] operator with a key makes no sense. + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read and + written using the `[]` operator.,operatorarray__key_type} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template<typename T> + reference operator[](T* key) + { + // implicitly convert null to object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // at only works for objects + if (JSON_LIKELY(is_object())) + { + return m_value.object->operator[](key); + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); + } + + /*! + @brief read-only access specified object element + + Returns a const reference to the element at with specified key @a key. No + bounds checking is performed. + + @warning If the element with key @a key does not exist, the behavior is + undefined. + + @param[in] key key of the element to access + + @return const reference to the element at key @a key + + @pre The element with key @a key must exist. **This precondition is + enforced with an assertion.** + + @throw type_error.305 if the JSON value is not an object; in that case, + using the [] operator with a key makes no sense. + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be read using + the `[]` operator.,operatorarray__key_type_const} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref value() for access by value with a default value + + @since version 1.1.0 + */ + template<typename T> + const_reference operator[](T* key) const + { + // at only works for objects + if (JSON_LIKELY(is_object())) + { + assert(m_value.object->find(key) != m_value.object->end()); + return m_value.object->find(key)->second; + } + + JSON_THROW(type_error::create(305, "cannot use operator[] with a string argument with " + std::string(type_name()))); + } + + /*! + @brief access specified object element with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(key); + } catch(out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const typename object_t::key_type&), this function + does not throw if the given key @a key was not found. + + @note Unlike @ref operator[](const typename object_t::key_type& key), this + function does not implicitly add an element to the position defined by @a + key. This function is furthermore also applicable to const objects. + + @param[in] key key of the element to access + @param[in] default_value the value to return if @a key is not found + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw type_error.306 if the JSON value is not an object; in that case, + using `value()` with a key makes no sense. + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value} + + @sa @ref at(const typename object_t::key_type&) for access by reference + with range checking + @sa @ref operator[](const typename object_t::key_type&) for unchecked + access by reference + + @since version 1.0.0 + */ + template<class ValueType, typename std::enable_if< + std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0> + ValueType value(const typename object_t::key_type& key, const ValueType& default_value) const + { + // at only works for objects + if (JSON_LIKELY(is_object())) + { + // if key is found, return value and given default value otherwise + const auto it = find(key); + if (it != end()) + { + return *it; + } + + return default_value; + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()))); + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const typename object_t::key_type&, const ValueType&) const + */ + string_t value(const typename object_t::key_type& key, const char* default_value) const + { + return value(key, string_t(default_value)); + } + + /*! + @brief access specified object element via JSON Pointer with default value + + Returns either a copy of an object's element at the specified key @a key + or a given default value if no element with key @a key exists. + + The function is basically equivalent to executing + @code {.cpp} + try { + return at(ptr); + } catch(out_of_range) { + return default_value; + } + @endcode + + @note Unlike @ref at(const json_pointer&), this function does not throw + if the given key @a key was not found. + + @param[in] ptr a JSON pointer to the element to access + @param[in] default_value the value to return if @a ptr found no value + + @tparam ValueType type compatible to JSON values, for instance `int` for + JSON integer numbers, `bool` for JSON booleans, or `std::vector` types for + JSON arrays. Note the type of the expected value at @a key and the default + value @a default_value must be compatible. + + @return copy of the element at key @a key or @a default_value if @a key + is not found + + @throw type_error.306 if the JSON value is not an object; in that case, + using `value()` with a key makes no sense. + + @complexity Logarithmic in the size of the container. + + @liveexample{The example below shows how object elements can be queried + with a default value.,basic_json__value_ptr} + + @sa @ref operator[](const json_pointer&) for unchecked access by reference + + @since version 2.0.2 + */ + template<class ValueType, typename std::enable_if< + std::is_convertible<basic_json_t, ValueType>::value, int>::type = 0> + ValueType value(const json_pointer& ptr, const ValueType& default_value) const + { + // at only works for objects + if (JSON_LIKELY(is_object())) + { + // if pointer resolves a value, return it or use default value + JSON_TRY + { + return ptr.get_checked(this); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + return default_value; + } + } + + JSON_THROW(type_error::create(306, "cannot use value() with " + std::string(type_name()))); + } + + /*! + @brief overload for a default value of type const char* + @copydoc basic_json::value(const json_pointer&, ValueType) const + */ + string_t value(const json_pointer& ptr, const char* default_value) const + { + return value(ptr, string_t(default_value)); + } + + /*! + @brief access the first element + + Returns a reference to the first element in the container. For a JSON + container `c`, the expression `c.front()` is equivalent to `*c.begin()`. + + @return In case of a structured type (array or object), a reference to the + first element is returned. In case of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, **guarded by + assertions**). + @post The JSON value remains unchanged. + + @throw invalid_iterator.214 when called on `null` value + + @liveexample{The following code shows an example for `front()`.,front} + + @sa @ref back() -- access the last element + + @since version 1.0.0 + */ + reference front() + { + return *begin(); + } + + /*! + @copydoc basic_json::front() + */ + const_reference front() const + { + return *cbegin(); + } + + /*! + @brief access the last element + + Returns a reference to the last element in the container. For a JSON + container `c`, the expression `c.back()` is equivalent to + @code {.cpp} + auto tmp = c.end(); + --tmp; + return *tmp; + @endcode + + @return In case of a structured type (array or object), a reference to the + last element is returned. In case of number, string, or boolean values, a + reference to the value is returned. + + @complexity Constant. + + @pre The JSON value must not be `null` (would throw `std::out_of_range`) + or an empty array or object (undefined behavior, **guarded by + assertions**). + @post The JSON value remains unchanged. + + @throw invalid_iterator.214 when called on a `null` value. See example + below. + + @liveexample{The following code shows an example for `back()`.,back} + + @sa @ref front() -- access the first element + + @since version 1.0.0 + */ + reference back() + { + auto tmp = end(); + --tmp; + return *tmp; + } + + /*! + @copydoc basic_json::back() + */ + const_reference back() const + { + auto tmp = cend(); + --tmp; + return *tmp; + } + + /*! + @brief remove element given an iterator + + Removes the element specified by iterator @a pos. The iterator @a pos must + be valid and dereferenceable. Thus the `end()` iterator (which is valid, + but is not dereferenceable) cannot be used as a value for @a pos. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] pos iterator to the element to remove + @return Iterator following the last removed element. If the iterator @a + pos refers to the last element, the `end()` iterator is returned. + + @tparam IteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.202 if called on an iterator which does not belong + to the current JSON value; example: `"iterator does not fit current + value"` + @throw invalid_iterator.205 if called on a primitive type with invalid + iterator (i.e., any iterator which is not `begin()`); example: `"iterator + out of range"` + + @complexity The complexity depends on the type: + - objects: amortized constant + - arrays: linear in distance between @a pos and the end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType} + + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template<class IteratorType, typename std::enable_if< + std::is_same<IteratorType, typename basic_json_t::iterator>::value or + std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type + = 0> + IteratorType erase(IteratorType pos) + { + // make sure iterator fits the current value + if (JSON_UNLIKELY(this != pos.m_object)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (JSON_UNLIKELY(not pos.m_it.primitive_iterator.is_begin())) + { + JSON_THROW(invalid_iterator::create(205, "iterator out of range")); + } + + if (is_string()) + { + AllocatorType<string_t> alloc; + std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(pos.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(pos.m_it.array_iterator); + break; + } + + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()))); + } + + return result; + } + + /*! + @brief remove elements given an iterator range + + Removes the element specified by the range `[first; last)`. The iterator + @a first does not need to be dereferenceable if `first == last`: erasing + an empty range is a no-op. + + If called on a primitive type other than `null`, the resulting JSON value + will be `null`. + + @param[in] first iterator to the beginning of the range to remove + @param[in] last iterator past the end of the range to remove + @return Iterator following the last removed element. If the iterator @a + second refers to the last element, the `end()` iterator is returned. + + @tparam IteratorType an @ref iterator or @ref const_iterator + + @post Invalidates iterators and references at or after the point of the + erase, including the `end()` iterator. + + @throw type_error.307 if called on a `null` value; example: `"cannot use + erase() with null"` + @throw invalid_iterator.203 if called on iterators which does not belong + to the current JSON value; example: `"iterators do not fit current value"` + @throw invalid_iterator.204 if called on a primitive type with invalid + iterators (i.e., if `first != begin()` and `last != end()`); example: + `"iterators out of range"` + + @complexity The complexity depends on the type: + - objects: `log(size()) + std::distance(first, last)` + - arrays: linear in the distance between @a first and @a last, plus linear + in the distance between @a last and end of the container + - strings: linear in the length of the string + - other types: constant + + @liveexample{The example shows the result of `erase()` for different JSON + types.,erase__IteratorType_IteratorType} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + template<class IteratorType, typename std::enable_if< + std::is_same<IteratorType, typename basic_json_t::iterator>::value or + std::is_same<IteratorType, typename basic_json_t::const_iterator>::value, int>::type + = 0> + IteratorType erase(IteratorType first, IteratorType last) + { + // make sure iterator fits the current value + if (JSON_UNLIKELY(this != first.m_object or this != last.m_object)) + { + JSON_THROW(invalid_iterator::create(203, "iterators do not fit current value")); + } + + IteratorType result = end(); + + switch (m_type) + { + case value_t::boolean: + case value_t::number_float: + case value_t::number_integer: + case value_t::number_unsigned: + case value_t::string: + { + if (JSON_LIKELY(not first.m_it.primitive_iterator.is_begin() + or not last.m_it.primitive_iterator.is_end())) + { + JSON_THROW(invalid_iterator::create(204, "iterators out of range")); + } + + if (is_string()) + { + AllocatorType<string_t> alloc; + std::allocator_traits<decltype(alloc)>::destroy(alloc, m_value.string); + std::allocator_traits<decltype(alloc)>::deallocate(alloc, m_value.string, 1); + m_value.string = nullptr; + } + + m_type = value_t::null; + assert_invariant(); + break; + } + + case value_t::object: + { + result.m_it.object_iterator = m_value.object->erase(first.m_it.object_iterator, + last.m_it.object_iterator); + break; + } + + case value_t::array: + { + result.m_it.array_iterator = m_value.array->erase(first.m_it.array_iterator, + last.m_it.array_iterator); + break; + } + + default: + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()))); + } + + return result; + } + + /*! + @brief remove element from a JSON object given a key + + Removes elements from a JSON object with the key value @a key. + + @param[in] key value of the elements to remove + + @return Number of elements removed. If @a ObjectType is the default + `std::map` type, the return value will always be `0` (@a key was not + found) or `1` (@a key was found). + + @post References and iterators to the erased elements are invalidated. + Other references and iterators are not affected. + + @throw type_error.307 when called on a type other than JSON object; + example: `"cannot use erase() with null"` + + @complexity `log(size()) + count(key)` + + @liveexample{The example shows the effect of `erase()`.,erase__key_type} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const size_type) -- removes the element from an array at + the given index + + @since version 1.0.0 + */ + size_type erase(const typename object_t::key_type& key) + { + // this erase only works for objects + if (JSON_LIKELY(is_object())) + { + return m_value.object->erase(key); + } + + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()))); + } + + /*! + @brief remove element from a JSON array given an index + + Removes element from a JSON array at the index @a idx. + + @param[in] idx index of the element to remove + + @throw type_error.307 when called on a type other than JSON object; + example: `"cannot use erase() with null"` + @throw out_of_range.401 when `idx >= size()`; example: `"array index 17 + is out of range"` + + @complexity Linear in distance between @a idx and the end of the container. + + @liveexample{The example shows the effect of `erase()`.,erase__size_type} + + @sa @ref erase(IteratorType) -- removes the element at a given position + @sa @ref erase(IteratorType, IteratorType) -- removes the elements in + the given range + @sa @ref erase(const typename object_t::key_type&) -- removes the element + from an object at the given key + + @since version 1.0.0 + */ + void erase(const size_type idx) + { + // this erase only works for arrays + if (JSON_LIKELY(is_array())) + { + if (JSON_UNLIKELY(idx >= size())) + { + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); + } + + m_value.array->erase(m_value.array->begin() + static_cast<difference_type>(idx)); + } + else + { + JSON_THROW(type_error::create(307, "cannot use erase() with " + std::string(type_name()))); + } + } + + /// @} + + + //////////// + // lookup // + //////////// + + /// @name lookup + /// @{ + + /*! + @brief find an element in a JSON object + + Finds an element in a JSON object with key equivalent to @a key. If the + element is not found or the JSON value is not an object, end() is + returned. + + @note This method always returns @ref end() when executed on a JSON type + that is not an object. + + @param[in] key key value of the element to search for. + + @return Iterator to an element with key equivalent to @a key. If no such + element is found or the JSON value is not an object, past-the-end (see + @ref end()) iterator is returned. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `find()` is used.,find__key_type} + + @since version 1.0.0 + */ + template<typename KeyT> + iterator find(KeyT&& key) + { + auto result = end(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key)); + } + + return result; + } + + /*! + @brief find an element in a JSON object + @copydoc find(KeyT&&) + */ + template<typename KeyT> + const_iterator find(KeyT&& key) const + { + auto result = cend(); + + if (is_object()) + { + result.m_it.object_iterator = m_value.object->find(std::forward<KeyT>(key)); + } + + return result; + } + + /*! + @brief returns the number of occurrences of a key in a JSON object + + Returns the number of elements with key @a key. If ObjectType is the + default `std::map` type, the return value will always be `0` (@a key was + not found) or `1` (@a key was found). + + @note This method always returns `0` when executed on a JSON type that is + not an object. + + @param[in] key key value of the element to count + + @return Number of elements with key @a key. If the JSON value is not an + object, the return value will be `0`. + + @complexity Logarithmic in the size of the JSON object. + + @liveexample{The example shows how `count()` is used.,count} + + @since version 1.0.0 + */ + template<typename KeyT> + size_type count(KeyT&& key) const + { + // return 0 for all nonobject types + return is_object() ? m_value.object->count(std::forward<KeyT>(key)) : 0; + } + + /// @} + + + /////////////// + // iterators // + /////////////// + + /// @name iterators + /// @{ + + /*! + @brief returns an iterator to the first element + + Returns an iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `begin()`.,begin} + + @sa @ref cbegin() -- returns a const iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + iterator begin() noexcept + { + iterator result(this); + result.set_begin(); + return result; + } + + /*! + @copydoc basic_json::cbegin() + */ + const_iterator begin() const noexcept + { + return cbegin(); + } + + /*! + @brief returns a const iterator to the first element + + Returns a const iterator to the first element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator to the first element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).begin()`. + + @liveexample{The following code shows an example for `cbegin()`.,cbegin} + + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref end() -- returns an iterator to the end + @sa @ref cend() -- returns a const iterator to the end + + @since version 1.0.0 + */ + const_iterator cbegin() const noexcept + { + const_iterator result(this); + result.set_begin(); + return result; + } + + /*! + @brief returns an iterator to one past the last element + + Returns an iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + + @liveexample{The following code shows an example for `end()`.,end} + + @sa @ref cend() -- returns a const iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + iterator end() noexcept + { + iterator result(this); + result.set_end(); + return result; + } + + /*! + @copydoc basic_json::cend() + */ + const_iterator end() const noexcept + { + return cend(); + } + + /*! + @brief returns a const iterator to one past the last element + + Returns a const iterator to one past the last element. + + @image html range-begin-end.svg "Illustration from cppreference.com" + + @return const iterator one past the last element + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).end()`. + + @liveexample{The following code shows an example for `cend()`.,cend} + + @sa @ref end() -- returns an iterator to the end + @sa @ref begin() -- returns an iterator to the beginning + @sa @ref cbegin() -- returns a const iterator to the beginning + + @since version 1.0.0 + */ + const_iterator cend() const noexcept + { + const_iterator result(this); + result.set_end(); + return result; + } + + /*! + @brief returns an iterator to the reverse-beginning + + Returns an iterator to the reverse-beginning; that is, the last element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(end())`. + + @liveexample{The following code shows an example for `rbegin()`.,rbegin} + + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + reverse_iterator rbegin() noexcept + { + return reverse_iterator(end()); + } + + /*! + @copydoc basic_json::crbegin() + */ + const_reverse_iterator rbegin() const noexcept + { + return crbegin(); + } + + /*! + @brief returns an iterator to the reverse-end + + Returns an iterator to the reverse-end; that is, one before the first + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `reverse_iterator(begin())`. + + @liveexample{The following code shows an example for `rend()`.,rend} + + @sa @ref crend() -- returns a const reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + reverse_iterator rend() noexcept + { + return reverse_iterator(begin()); + } + + /*! + @copydoc basic_json::crend() + */ + const_reverse_iterator rend() const noexcept + { + return crend(); + } + + /*! + @brief returns a const reverse iterator to the last element + + Returns a const iterator to the reverse-beginning; that is, the last + element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).rbegin()`. + + @liveexample{The following code shows an example for `crbegin()`.,crbegin} + + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref crend() -- returns a const reverse iterator to the end + + @since version 1.0.0 + */ + const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator(cend()); + } + + /*! + @brief returns a const reverse iterator to one before the first + + Returns a const reverse iterator to the reverse-end; that is, one before + the first element. + + @image html range-rbegin-rend.svg "Illustration from cppreference.com" + + @complexity Constant. + + @requirement This function helps `basic_json` satisfying the + [ReversibleContainer](https://en.cppreference.com/w/cpp/named_req/ReversibleContainer) + requirements: + - The complexity is constant. + - Has the semantics of `const_cast<const basic_json&>(*this).rend()`. + + @liveexample{The following code shows an example for `crend()`.,crend} + + @sa @ref rend() -- returns a reverse iterator to the end + @sa @ref rbegin() -- returns a reverse iterator to the beginning + @sa @ref crbegin() -- returns a const reverse iterator to the beginning + + @since version 1.0.0 + */ + const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator(cbegin()); + } + + public: + /*! + @brief wrapper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + For loop without iterator_wrapper: + + @code{cpp} + for (auto it = j_object.begin(); it != j_object.end(); ++it) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + Range-based for loop without iterator proxy: + + @code{cpp} + for (auto it : j_object) + { + // "it" is of type json::reference and has no key() member + std::cout << "value: " << it << '\n'; + } + @endcode + + Range-based for loop with iterator proxy: + + @code{cpp} + for (auto it : json::iterator_wrapper(j_object)) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + @note When iterating over an array, `key()` will return the index of the + element as string (see example). + + @param[in] ref reference to a JSON value + @return iteration proxy object wrapping @a ref with an interface to use in + range-based for loops + + @liveexample{The following code shows how the wrapper is used,iterator_wrapper} + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @note The name of this function is not yet final and may change in the + future. + + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use @ref items() instead; + that is, replace `json::iterator_wrapper(j)` with `j.items()`. + */ + JSON_DEPRECATED + static iteration_proxy<iterator> iterator_wrapper(reference ref) noexcept + { + return ref.items(); + } + + /*! + @copydoc iterator_wrapper(reference) + */ + JSON_DEPRECATED + static iteration_proxy<const_iterator> iterator_wrapper(const_reference ref) noexcept + { + return ref.items(); + } + + /*! + @brief helper to access iterator member functions in range-based for + + This function allows to access @ref iterator::key() and @ref + iterator::value() during range-based for loops. In these loops, a + reference to the JSON values is returned, so there is no access to the + underlying iterator. + + For loop without `items()` function: + + @code{cpp} + for (auto it = j_object.begin(); it != j_object.end(); ++it) + { + std::cout << "key: " << it.key() << ", value:" << it.value() << '\n'; + } + @endcode + + Range-based for loop without `items()` function: + + @code{cpp} + for (auto it : j_object) + { + // "it" is of type json::reference and has no key() member + std::cout << "value: " << it << '\n'; + } + @endcode + + Range-based for loop with `items()` function: + + @code{cpp} + for (auto& el : j_object.items()) + { + std::cout << "key: " << el.key() << ", value:" << el.value() << '\n'; + } + @endcode + + The `items()` function also allows to use + [structured bindings](https://en.cppreference.com/w/cpp/language/structured_binding) + (C++17): + + @code{cpp} + for (auto& [key, val] : j_object.items()) + { + std::cout << "key: " << key << ", value:" << val << '\n'; + } + @endcode + + @note When iterating over an array, `key()` will return the index of the + element as string (see example). For primitive types (e.g., numbers), + `key()` returns an empty string. + + @return iteration proxy object wrapping @a ref with an interface to use in + range-based for loops + + @liveexample{The following code shows how the function is used.,items} + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 3.1.0, structured bindings support since 3.5.0. + */ + iteration_proxy<iterator> items() noexcept + { + return iteration_proxy<iterator>(*this); + } + + /*! + @copydoc items() + */ + iteration_proxy<const_iterator> items() const noexcept + { + return iteration_proxy<const_iterator>(*this); + } + + /// @} + + + ////////////// + // capacity // + ////////////// + + /// @name capacity + /// @{ + + /*! + @brief checks whether the container is empty. + + Checks if a JSON value has no elements (i.e. whether its @ref size is `0`). + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `true` + boolean | `false` + string | `false` + number | `false` + object | result of function `object_t::empty()` + array | result of function `array_t::empty()` + + @liveexample{The following code uses `empty()` to check if a JSON + object contains any elements.,empty} + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `empty()` functions have constant + complexity. + + @iterators No changes. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @note This function does not return whether a string stored as JSON value + is empty - it returns whether the JSON container itself is empty which is + false in the case of a string. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + - Has the semantics of `begin() == end()`. + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + bool empty() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return true; + } + + case value_t::array: + { + // delegate call to array_t::empty() + return m_value.array->empty(); + } + + case value_t::object: + { + // delegate call to object_t::empty() + return m_value.object->empty(); + } + + default: + { + // all other types are nonempty + return false; + } + } + } + + /*! + @brief returns the number of elements + + Returns the number of elements in a JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` + boolean | `1` + string | `1` + number | `1` + object | result of function object_t::size() + array | result of function array_t::size() + + @liveexample{The following code calls `size()` on the different value + types.,size} + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their size() functions have constant + complexity. + + @iterators No changes. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @note This function does not return the length of a string stored as JSON + value - it returns the number of elements in the JSON value which is 1 in + the case of a string. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + - Has the semantics of `std::distance(begin(), end())`. + + @sa @ref empty() -- checks whether the container is empty + @sa @ref max_size() -- returns the maximal number of elements + + @since version 1.0.0 + */ + size_type size() const noexcept + { + switch (m_type) + { + case value_t::null: + { + // null values are empty + return 0; + } + + case value_t::array: + { + // delegate call to array_t::size() + return m_value.array->size(); + } + + case value_t::object: + { + // delegate call to object_t::size() + return m_value.object->size(); + } + + default: + { + // all other types have size 1 + return 1; + } + } + } + + /*! + @brief returns the maximum possible number of elements + + Returns the maximum number of elements a JSON value is able to hold due to + system or library implementation limitations, i.e. `std::distance(begin(), + end())` for the JSON value. + + @return The return value depends on the different types and is + defined as follows: + Value type | return value + ----------- | ------------- + null | `0` (same as `size()`) + boolean | `1` (same as `size()`) + string | `1` (same as `size()`) + number | `1` (same as `size()`) + object | result of function `object_t::max_size()` + array | result of function `array_t::max_size()` + + @liveexample{The following code calls `max_size()` on the different value + types. Note the output is implementation specific.,max_size} + + @complexity Constant, as long as @ref array_t and @ref object_t satisfy + the Container concept; that is, their `max_size()` functions have constant + complexity. + + @iterators No changes. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @requirement This function helps `basic_json` satisfying the + [Container](https://en.cppreference.com/w/cpp/named_req/Container) + requirements: + - The complexity is constant. + - Has the semantics of returning `b.size()` where `b` is the largest + possible JSON value. + + @sa @ref size() -- returns the number of elements + + @since version 1.0.0 + */ + size_type max_size() const noexcept + { + switch (m_type) + { + case value_t::array: + { + // delegate call to array_t::max_size() + return m_value.array->max_size(); + } + + case value_t::object: + { + // delegate call to object_t::max_size() + return m_value.object->max_size(); + } + + default: + { + // all other types have max_size() == size() + return size(); + } + } + } + + /// @} + + + /////////////// + // modifiers // + /////////////// + + /// @name modifiers + /// @{ + + /*! + @brief clears the contents + + Clears the content of a JSON value and resets it to the default value as + if @ref basic_json(value_t) would have been called with the current value + type from @ref type(): + + Value type | initial value + ----------- | ------------- + null | `null` + boolean | `false` + string | `""` + number | `0` + object | `{}` + array | `[]` + + @post Has the same effect as calling + @code {.cpp} + *this = basic_json(type()); + @endcode + + @liveexample{The example below shows the effect of `clear()` to different + JSON types.,clear} + + @complexity Linear in the size of the JSON value. + + @iterators All iterators, pointers and references related to this container + are invalidated. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @sa @ref basic_json(value_t) -- constructor that creates an object with the + same value than calling `clear()` + + @since version 1.0.0 + */ + void clear() noexcept + { + switch (m_type) + { + case value_t::number_integer: + { + m_value.number_integer = 0; + break; + } + + case value_t::number_unsigned: + { + m_value.number_unsigned = 0; + break; + } + + case value_t::number_float: + { + m_value.number_float = 0.0; + break; + } + + case value_t::boolean: + { + m_value.boolean = false; + break; + } + + case value_t::string: + { + m_value.string->clear(); + break; + } + + case value_t::array: + { + m_value.array->clear(); + break; + } + + case value_t::object: + { + m_value.object->clear(); + break; + } + + default: + break; + } + } + + /*! + @brief add an object to an array + + Appends the given element @a val to the end of the JSON value. If the + function is called on a JSON null value, an empty array is created before + appending @a val. + + @param[in] val the value to add to the JSON array + + @throw type_error.308 when called on a type other than JSON array or + null; example: `"cannot use push_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON array. Note how the `null` value was silently + converted to a JSON array.,push_back} + + @since version 1.0.0 + */ + void push_back(basic_json&& val) + { + // push_back only works for null objects or arrays + if (JSON_UNLIKELY(not(is_null() or is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()))); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (move semantics) + m_value.array->push_back(std::move(val)); + // invalidate object + val.m_type = value_t::null; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(basic_json&& val) + { + push_back(std::move(val)); + return *this; + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + void push_back(const basic_json& val) + { + // push_back only works for null objects or arrays + if (JSON_UNLIKELY(not(is_null() or is_array()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()))); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array + m_value.array->push_back(val); + } + + /*! + @brief add an object to an array + @copydoc push_back(basic_json&&) + */ + reference operator+=(const basic_json& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + Inserts the given element @a val to the JSON object. If the function is + called on a JSON null value, an empty object is created before inserting + @a val. + + @param[in] val the value to add to the JSON object + + @throw type_error.308 when called on a type other than JSON object or + null; example: `"cannot use push_back() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `push_back()` and `+=` can be used to + add elements to a JSON object. Note how the `null` value was silently + converted to a JSON object.,push_back__object_t__value} + + @since version 1.0.0 + */ + void push_back(const typename object_t::value_type& val) + { + // push_back only works for null objects or objects + if (JSON_UNLIKELY(not(is_null() or is_object()))) + { + JSON_THROW(type_error::create(308, "cannot use push_back() with " + std::string(type_name()))); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array + m_value.object->insert(val); + } + + /*! + @brief add an object to an object + @copydoc push_back(const typename object_t::value_type&) + */ + reference operator+=(const typename object_t::value_type& val) + { + push_back(val); + return *this; + } + + /*! + @brief add an object to an object + + This function allows to use `push_back` with an initializer list. In case + + 1. the current value is an object, + 2. the initializer list @a init contains only two elements, and + 3. the first element of @a init is a string, + + @a init is converted into an object element and added using + @ref push_back(const typename object_t::value_type&). Otherwise, @a init + is converted to a JSON value and added using @ref push_back(basic_json&&). + + @param[in] init an initializer list + + @complexity Linear in the size of the initializer list @a init. + + @note This function is required to resolve an ambiguous overload error, + because pairs like `{"key", "value"}` can be both interpreted as + `object_t::value_type` or `std::initializer_list<basic_json>`, see + https://github.com/nlohmann/json/issues/235 for more information. + + @liveexample{The example shows how initializer lists are treated as + objects when possible.,push_back__initializer_list} + */ + void push_back(initializer_list_t init) + { + if (is_object() and init.size() == 2 and (*init.begin())->is_string()) + { + basic_json&& key = init.begin()->moved_or_copied(); + push_back(typename object_t::value_type( + std::move(key.get_ref<string_t&>()), (init.begin() + 1)->moved_or_copied())); + } + else + { + push_back(basic_json(init)); + } + } + + /*! + @brief add an object to an object + @copydoc push_back(initializer_list_t) + */ + reference operator+=(initializer_list_t init) + { + push_back(init); + return *this; + } + + /*! + @brief add an object to an array + + Creates a JSON value from the passed parameters @a args to the end of the + JSON value. If the function is called on a JSON null value, an empty array + is created before appending the value created from @a args. + + @param[in] args arguments to forward to a constructor of @ref basic_json + @tparam Args compatible types to create a @ref basic_json object + + @throw type_error.311 when called on a type other than JSON array or + null; example: `"cannot use emplace_back() with number"` + + @complexity Amortized constant. + + @liveexample{The example shows how `push_back()` can be used to add + elements to a JSON array. Note how the `null` value was silently converted + to a JSON array.,emplace_back} + + @since version 2.0.8 + */ + template<class... Args> + void emplace_back(Args&& ... args) + { + // emplace_back only works for null objects or arrays + if (JSON_UNLIKELY(not(is_null() or is_array()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace_back() with " + std::string(type_name()))); + } + + // transform null object into an array + if (is_null()) + { + m_type = value_t::array; + m_value = value_t::array; + assert_invariant(); + } + + // add element to array (perfect forwarding) + m_value.array->emplace_back(std::forward<Args>(args)...); + } + + /*! + @brief add an object to an object if key does not exist + + Inserts a new element into a JSON object constructed in-place with the + given @a args if there is no element with the key in the container. If the + function is called on a JSON null value, an empty object is created before + appending the value created from @a args. + + @param[in] args arguments to forward to a constructor of @ref basic_json + @tparam Args compatible types to create a @ref basic_json object + + @return a pair consisting of an iterator to the inserted element, or the + already-existing element if no insertion happened, and a bool + denoting whether the insertion took place. + + @throw type_error.311 when called on a type other than JSON object or + null; example: `"cannot use emplace() with number"` + + @complexity Logarithmic in the size of the container, O(log(`size()`)). + + @liveexample{The example shows how `emplace()` can be used to add elements + to a JSON object. Note how the `null` value was silently converted to a + JSON object. Further note how no value is added if there was already one + value stored with the same key.,emplace} + + @since version 2.0.8 + */ + template<class... Args> + std::pair<iterator, bool> emplace(Args&& ... args) + { + // emplace only works for null objects or arrays + if (JSON_UNLIKELY(not(is_null() or is_object()))) + { + JSON_THROW(type_error::create(311, "cannot use emplace() with " + std::string(type_name()))); + } + + // transform null object into an object + if (is_null()) + { + m_type = value_t::object; + m_value = value_t::object; + assert_invariant(); + } + + // add element to array (perfect forwarding) + auto res = m_value.object->emplace(std::forward<Args>(args)...); + // create result iterator and set iterator to the result of emplace + auto it = begin(); + it.m_it.object_iterator = res.first; + + // return pair of iterator and boolean + return {it, res.second}; + } + + /// Helper for insertion of an iterator + /// @note: This uses std::distance to support GCC 4.8, + /// see https://github.com/nlohmann/json/pull/1257 + template<typename... Args> + iterator insert_iterator(const_iterator pos, Args&& ... args) + { + iterator result(this); + assert(m_value.array != nullptr); + + auto insert_pos = std::distance(m_value.array->begin(), pos.m_it.array_iterator); + m_value.array->insert(pos.m_it.array_iterator, std::forward<Args>(args)...); + result.m_it.array_iterator = m_value.array->begin() + insert_pos; + + // This could have been written as: + // result.m_it.array_iterator = m_value.array->insert(pos.m_it.array_iterator, cnt, val); + // but the return value of insert is missing in GCC 4.8, so it is written this way instead. + + return result; + } + + /*! + @brief inserts element + + Inserts element @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] val element to insert + @return iterator pointing to the inserted @a val. + + @throw type_error.309 if called on JSON values other than arrays; + example: `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + + @complexity Constant plus linear in the distance between @a pos and end of + the container. + + @liveexample{The example shows how `insert()` is used.,insert} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const basic_json& val) + { + // insert only works for arrays + if (JSON_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); + } + + // insert to array and return iterator + return insert_iterator(pos, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); + } + + /*! + @brief inserts element + @copydoc insert(const_iterator, const basic_json&) + */ + iterator insert(const_iterator pos, basic_json&& val) + { + return insert(pos, val); + } + + /*! + @brief inserts elements + + Inserts @a cnt copies of @a val before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] cnt number of copies of @a val to insert + @param[in] val element to insert + @return iterator pointing to the first element inserted, or @a pos if + `cnt==0` + + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + + @complexity Linear in @a cnt plus linear in the distance between @a pos + and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__count} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, size_type cnt, const basic_json& val) + { + // insert only works for arrays + if (JSON_LIKELY(is_array())) + { + // check if iterator pos fits to this JSON value + if (JSON_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); + } + + // insert to array and return iterator + return insert_iterator(pos, cnt, val); + } + + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)` before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + @throw invalid_iterator.211 if @a first or @a last are iterators into + container for which insert is called; example: `"passed iterators may not + belong to container"` + + @return iterator pointing to the first element inserted, or @a pos if + `first==last` + + @complexity Linear in `std::distance(first, last)` plus linear in the + distance between @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__range} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, const_iterator first, const_iterator last) + { + // insert only works for arrays + if (JSON_UNLIKELY(not is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); + } + + // check if iterator pos fits to this JSON value + if (JSON_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); + } + + // check if range iterators belong to the same JSON object + if (JSON_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); + } + + if (JSON_UNLIKELY(first.m_object == this)) + { + JSON_THROW(invalid_iterator::create(211, "passed iterators may not belong to container")); + } + + // insert to array and return iterator + return insert_iterator(pos, first.m_it.array_iterator, last.m_it.array_iterator); + } + + /*! + @brief inserts elements + + Inserts elements from initializer list @a ilist before iterator @a pos. + + @param[in] pos iterator before which the content will be inserted; may be + the end() iterator + @param[in] ilist initializer list to insert the values from + + @throw type_error.309 if called on JSON values other than arrays; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if @a pos is not an iterator of *this; + example: `"iterator does not fit current value"` + + @return iterator pointing to the first element inserted, or @a pos if + `ilist` is empty + + @complexity Linear in `ilist.size()` plus linear in the distance between + @a pos and end of the container. + + @liveexample{The example shows how `insert()` is used.,insert__ilist} + + @since version 1.0.0 + */ + iterator insert(const_iterator pos, initializer_list_t ilist) + { + // insert only works for arrays + if (JSON_UNLIKELY(not is_array())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); + } + + // check if iterator pos fits to this JSON value + if (JSON_UNLIKELY(pos.m_object != this)) + { + JSON_THROW(invalid_iterator::create(202, "iterator does not fit current value")); + } + + // insert to array and return iterator + return insert_iterator(pos, ilist.begin(), ilist.end()); + } + + /*! + @brief inserts elements + + Inserts elements from range `[first, last)`. + + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw type_error.309 if called on JSON values other than objects; example: + `"cannot use insert() with string"` + @throw invalid_iterator.202 if iterator @a first or @a last does does not + point to an object; example: `"iterators first and last must point to + objects"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + + @complexity Logarithmic: `O(N*log(size() + N))`, where `N` is the number + of elements to insert. + + @liveexample{The example shows how `insert()` is used.,insert__range_object} + + @since version 3.0.0 + */ + void insert(const_iterator first, const_iterator last) + { + // insert only works for objects + if (JSON_UNLIKELY(not is_object())) + { + JSON_THROW(type_error::create(309, "cannot use insert() with " + std::string(type_name()))); + } + + // check if range iterators belong to the same JSON object + if (JSON_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); + } + + // passed iterators must belong to objects + if (JSON_UNLIKELY(not first.m_object->is_object())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); + } + + m_value.object->insert(first.m_it.object_iterator, last.m_it.object_iterator); + } + + /*! + @brief updates a JSON object from another object, overwriting existing keys + + Inserts all values from JSON object @a j and overwrites existing keys. + + @param[in] j JSON object to read values from + + @throw type_error.312 if called on JSON values other than objects; example: + `"cannot use update() with string"` + + @complexity O(N*log(size() + N)), where N is the number of elements to + insert. + + @liveexample{The example shows how `update()` is used.,update} + + @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update + + @since version 3.0.0 + */ + void update(const_reference j) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create<object_t>(); + assert_invariant(); + } + + if (JSON_UNLIKELY(not is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()))); + } + if (JSON_UNLIKELY(not j.is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(j.type_name()))); + } + + for (auto it = j.cbegin(); it != j.cend(); ++it) + { + m_value.object->operator[](it.key()) = it.value(); + } + } + + /*! + @brief updates a JSON object from another object, overwriting existing keys + + Inserts all values from from range `[first, last)` and overwrites existing + keys. + + @param[in] first begin of the range of elements to insert + @param[in] last end of the range of elements to insert + + @throw type_error.312 if called on JSON values other than objects; example: + `"cannot use update() with string"` + @throw invalid_iterator.202 if iterator @a first or @a last does does not + point to an object; example: `"iterators first and last must point to + objects"` + @throw invalid_iterator.210 if @a first and @a last do not belong to the + same JSON value; example: `"iterators do not fit"` + + @complexity O(N*log(size() + N)), where N is the number of elements to + insert. + + @liveexample{The example shows how `update()` is used__range.,update} + + @sa https://docs.python.org/3.6/library/stdtypes.html#dict.update + + @since version 3.0.0 + */ + void update(const_iterator first, const_iterator last) + { + // implicitly convert null value to an empty object + if (is_null()) + { + m_type = value_t::object; + m_value.object = create<object_t>(); + assert_invariant(); + } + + if (JSON_UNLIKELY(not is_object())) + { + JSON_THROW(type_error::create(312, "cannot use update() with " + std::string(type_name()))); + } + + // check if range iterators belong to the same JSON object + if (JSON_UNLIKELY(first.m_object != last.m_object)) + { + JSON_THROW(invalid_iterator::create(210, "iterators do not fit")); + } + + // passed iterators must belong to objects + if (JSON_UNLIKELY(not first.m_object->is_object() + or not last.m_object->is_object())) + { + JSON_THROW(invalid_iterator::create(202, "iterators first and last must point to objects")); + } + + for (auto it = first; it != last; ++it) + { + m_value.object->operator[](it.key()) = it.value(); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of the JSON value with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other JSON value to exchange the contents with + + @complexity Constant. + + @liveexample{The example below shows how JSON values can be swapped with + `swap()`.,swap__reference} + + @since version 1.0.0 + */ + void swap(reference other) noexcept ( + std::is_nothrow_move_constructible<value_t>::value and + std::is_nothrow_move_assignable<value_t>::value and + std::is_nothrow_move_constructible<json_value>::value and + std::is_nothrow_move_assignable<json_value>::value + ) + { + std::swap(m_type, other.m_type); + std::swap(m_value, other.m_value); + assert_invariant(); + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON array with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other array to exchange the contents with + + @throw type_error.310 when JSON value is not an array; example: `"cannot + use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how arrays can be swapped with + `swap()`.,swap__array_t} + + @since version 1.0.0 + */ + void swap(array_t& other) + { + // swap only works for arrays + if (JSON_LIKELY(is_array())) + { + std::swap(*(m_value.array), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()))); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON object with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other object to exchange the contents with + + @throw type_error.310 when JSON value is not an object; example: + `"cannot use swap() with string"` + + @complexity Constant. + + @liveexample{The example below shows how objects can be swapped with + `swap()`.,swap__object_t} + + @since version 1.0.0 + */ + void swap(object_t& other) + { + // swap only works for objects + if (JSON_LIKELY(is_object())) + { + std::swap(*(m_value.object), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()))); + } + } + + /*! + @brief exchanges the values + + Exchanges the contents of a JSON string with those of @a other. Does not + invoke any move, copy, or swap operations on individual elements. All + iterators and references remain valid. The past-the-end iterator is + invalidated. + + @param[in,out] other string to exchange the contents with + + @throw type_error.310 when JSON value is not a string; example: `"cannot + use swap() with boolean"` + + @complexity Constant. + + @liveexample{The example below shows how strings can be swapped with + `swap()`.,swap__string_t} + + @since version 1.0.0 + */ + void swap(string_t& other) + { + // swap only works for strings + if (JSON_LIKELY(is_string())) + { + std::swap(*(m_value.string), other); + } + else + { + JSON_THROW(type_error::create(310, "cannot use swap() with " + std::string(type_name()))); + } + } + + /// @} + + public: + ////////////////////////////////////////// + // lexicographical comparison operators // + ////////////////////////////////////////// + + /// @name lexicographical comparison operators + /// @{ + + /*! + @brief comparison: equal + + Compares two JSON values for equality according to the following rules: + - Two JSON values are equal if (1) they are from the same type and (2) + their stored values are the same according to their respective + `operator==`. + - Integer and floating-point numbers are automatically converted before + comparison. Note than two NaN values are always treated as unequal. + - Two JSON null values are equal. + + @note Floating-point inside JSON values numbers are compared with + `json::number_float_t::operator==` which is `double::operator==` by + default. To compare floating-point while respecting an epsilon, an alternative + [comparison function](https://github.com/mariokonrad/marnav/blob/master/src/marnav/math/floatingpoint.hpp#L34-#L39) + could be used, for instance + @code {.cpp} + template<typename T, typename = typename std::enable_if<std::is_floating_point<T>::value, T>::type> + inline bool is_same(T a, T b, T epsilon = std::numeric_limits<T>::epsilon()) noexcept + { + return std::abs(a - b) <= epsilon; + } + @endcode + + @note NaN values never compare equal to themselves or to other NaN values. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are equal + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @complexity Linear. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__equal} + + @since version 1.0.0 + */ + friend bool operator==(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + return (*lhs.m_value.array == *rhs.m_value.array); + + case value_t::object: + return (*lhs.m_value.object == *rhs.m_value.object); + + case value_t::null: + return true; + + case value_t::string: + return (*lhs.m_value.string == *rhs.m_value.string); + + case value_t::boolean: + return (lhs.m_value.boolean == rhs.m_value.boolean); + + case value_t::number_integer: + return (lhs.m_value.number_integer == rhs.m_value.number_integer); + + case value_t::number_unsigned: + return (lhs.m_value.number_unsigned == rhs.m_value.number_unsigned); + + case value_t::number_float: + return (lhs.m_value.number_float == rhs.m_value.number_float); + + default: + return false; + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return (static_cast<number_float_t>(lhs.m_value.number_integer) == rhs.m_value.number_float); + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_integer)); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return (static_cast<number_float_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_float); + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return (lhs.m_value.number_float == static_cast<number_float_t>(rhs.m_value.number_unsigned)); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return (static_cast<number_integer_t>(lhs.m_value.number_unsigned) == rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return (lhs.m_value.number_integer == static_cast<number_integer_t>(rhs.m_value.number_unsigned)); + } + + return false; + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator==(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs == basic_json(rhs)); + } + + /*! + @brief comparison: equal + @copydoc operator==(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator==(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) == rhs); + } + + /*! + @brief comparison: not equal + + Compares two JSON values for inequality by calculating `not (lhs == rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether the values @a lhs and @a rhs are not equal + + @complexity Linear. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__notequal} + + @since version 1.0.0 + */ + friend bool operator!=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs == rhs); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator!=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs != basic_json(rhs)); + } + + /*! + @brief comparison: not equal + @copydoc operator!=(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator!=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) != rhs); + } + + /*! + @brief comparison: less than + + Compares whether one JSON value @a lhs is less than another JSON value @a + rhs according to the following rules: + - If @a lhs and @a rhs have the same type, the values are compared using + the default `<` operator. + - Integer and floating-point numbers are automatically converted before + comparison + - In case @a lhs and @a rhs have different types, the values are ignored + and the order of the types is considered, see + @ref operator<(const value_t, const value_t). + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than @a rhs + + @complexity Linear. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__less} + + @since version 1.0.0 + */ + friend bool operator<(const_reference lhs, const_reference rhs) noexcept + { + const auto lhs_type = lhs.type(); + const auto rhs_type = rhs.type(); + + if (lhs_type == rhs_type) + { + switch (lhs_type) + { + case value_t::array: + return (*lhs.m_value.array) < (*rhs.m_value.array); + + case value_t::object: + return *lhs.m_value.object < *rhs.m_value.object; + + case value_t::null: + return false; + + case value_t::string: + return *lhs.m_value.string < *rhs.m_value.string; + + case value_t::boolean: + return lhs.m_value.boolean < rhs.m_value.boolean; + + case value_t::number_integer: + return lhs.m_value.number_integer < rhs.m_value.number_integer; + + case value_t::number_unsigned: + return lhs.m_value.number_unsigned < rhs.m_value.number_unsigned; + + case value_t::number_float: + return lhs.m_value.number_float < rhs.m_value.number_float; + + default: + return false; + } + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_float) + { + return static_cast<number_float_t>(lhs.m_value.number_integer) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_integer) + { + return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_integer); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_float) + { + return static_cast<number_float_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_float; + } + else if (lhs_type == value_t::number_float and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_float < static_cast<number_float_t>(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_integer and rhs_type == value_t::number_unsigned) + { + return lhs.m_value.number_integer < static_cast<number_integer_t>(rhs.m_value.number_unsigned); + } + else if (lhs_type == value_t::number_unsigned and rhs_type == value_t::number_integer) + { + return static_cast<number_integer_t>(lhs.m_value.number_unsigned) < rhs.m_value.number_integer; + } + + // We only reach this line if we cannot compare values. In that case, + // we compare types. Note we have to call the operator explicitly, + // because MSVC has problems otherwise. + return operator<(lhs_type, rhs_type); + } + + /*! + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator<(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs < basic_json(rhs)); + } + + /*! + @brief comparison: less than + @copydoc operator<(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator<(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) < rhs); + } + + /*! + @brief comparison: less than or equal + + Compares whether one JSON value @a lhs is less than or equal to another + JSON value by calculating `not (rhs < lhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is less than or equal to @a rhs + + @complexity Linear. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greater} + + @since version 1.0.0 + */ + friend bool operator<=(const_reference lhs, const_reference rhs) noexcept + { + return not (rhs < lhs); + } + + /*! + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator<=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs <= basic_json(rhs)); + } + + /*! + @brief comparison: less than or equal + @copydoc operator<=(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator<=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) <= rhs); + } + + /*! + @brief comparison: greater than + + Compares whether one JSON value @a lhs is greater than another + JSON value by calculating `not (lhs <= rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than to @a rhs + + @complexity Linear. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__lessequal} + + @since version 1.0.0 + */ + friend bool operator>(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs <= rhs); + } + + /*! + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator>(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs > basic_json(rhs)); + } + + /*! + @brief comparison: greater than + @copydoc operator>(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator>(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) > rhs); + } + + /*! + @brief comparison: greater than or equal + + Compares whether one JSON value @a lhs is greater than or equal to another + JSON value by calculating `not (lhs < rhs)`. + + @param[in] lhs first JSON value to consider + @param[in] rhs second JSON value to consider + @return whether @a lhs is greater than or equal to @a rhs + + @complexity Linear. + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @liveexample{The example demonstrates comparing several JSON + types.,operator__greaterequal} + + @since version 1.0.0 + */ + friend bool operator>=(const_reference lhs, const_reference rhs) noexcept + { + return not (lhs < rhs); + } + + /*! + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator>=(const_reference lhs, const ScalarType rhs) noexcept + { + return (lhs >= basic_json(rhs)); + } + + /*! + @brief comparison: greater than or equal + @copydoc operator>=(const_reference, const_reference) + */ + template<typename ScalarType, typename std::enable_if< + std::is_scalar<ScalarType>::value, int>::type = 0> + friend bool operator>=(const ScalarType lhs, const_reference rhs) noexcept + { + return (basic_json(lhs) >= rhs); + } + + /// @} + + /////////////////// + // serialization // + /////////////////// + + /// @name serialization + /// @{ + + /*! + @brief serialize to stream + + Serialize the given JSON value @a j to the output stream @a o. The JSON + value will be serialized using the @ref dump member function. + + - The indentation of the output can be controlled with the member variable + `width` of the output stream @a o. For instance, using the manipulator + `std::setw(4)` on @a o sets the indentation level to `4` and the + serialization result is the same as calling `dump(4)`. + + - The indentation character can be controlled with the member variable + `fill` of the output stream @a o. For instance, the manipulator + `std::setfill('\\t')` sets indentation to use a tab character rather than + the default space character. + + @param[in,out] o stream to serialize to + @param[in] j JSON value to serialize + + @return the stream @a o + + @throw type_error.316 if a string stored inside the JSON value is not + UTF-8 encoded + + @complexity Linear. + + @liveexample{The example below shows the serialization with different + parameters to `width` to adjust the indentation level.,operator_serialize} + + @since version 1.0.0; indentation character added in version 3.0.0 + */ + friend std::ostream& operator<<(std::ostream& o, const basic_json& j) + { + // read width member and use it as indentation parameter if nonzero + const bool pretty_print = (o.width() > 0); + const auto indentation = (pretty_print ? o.width() : 0); + + // reset width to 0 for subsequent calls to this stream + o.width(0); + + // do the actual serialization + serializer s(detail::output_adapter<char>(o), o.fill()); + s.dump(j, pretty_print, false, static_cast<unsigned int>(indentation)); + return o; + } + + /*! + @brief serialize to stream + @deprecated This stream operator is deprecated and will be removed in + future 4.0.0 of the library. Please use + @ref operator<<(std::ostream&, const basic_json&) + instead; that is, replace calls like `j >> o;` with `o << j;`. + @since version 1.0.0; deprecated since version 3.0.0 + */ + JSON_DEPRECATED + friend std::ostream& operator>>(const basic_json& j, std::ostream& o) + { + return o << j; + } + + /// @} + + + ///////////////////// + // deserialization // + ///////////////////// + + /// @name deserialization + /// @{ + + /*! + @brief deserialize from a compatible input + + This function reads from a compatible input. Examples are: + - an array of 1-byte values + - strings with character/literal type with size of 1 byte + - input streams + - container with contiguous storage of 1-byte values. Compatible container + types include `std::vector`, `std::string`, `std::array`, + `std::valarray`, and `std::initializer_list`. Furthermore, C-style + arrays can be used with `std::begin()`/`std::end()`. User-defined + containers can be used as long as they implement random-access iterators + and a contiguous storage. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] i input to read from + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return result of the deserialization + + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an array.,parse__array__parser_callback_t} + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__string__parser_callback_t} + + @liveexample{The example below demonstrates the `parse()` function with + and without callback function.,parse__istream__parser_callback_t} + + @liveexample{The example below demonstrates the `parse()` function reading + from a contiguous container.,parse__contiguouscontainer__parser_callback_t} + + @since version 2.0.3 (contiguous containers) + */ + static basic_json parse(detail::input_adapter&& i, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true) + { + basic_json result; + parser(i, cb, allow_exceptions).parse(true, result); + return result; + } + + static bool accept(detail::input_adapter&& i) + { + return parser(i).accept(true); + } + + /*! + @brief generate SAX events + + The SAX event lister must follow the interface of @ref json_sax. + + This function reads from a compatible input. Examples are: + - an array of 1-byte values + - strings with character/literal type with size of 1 byte + - input streams + - container with contiguous storage of 1-byte values. Compatible container + types include `std::vector`, `std::string`, `std::array`, + `std::valarray`, and `std::initializer_list`. Furthermore, C-style + arrays can be used with `std::begin()`/`std::end()`. User-defined + containers can be used as long as they implement random-access iterators + and a contiguous storage. + + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @pre The container storage is contiguous. Violating this precondition + yields undefined behavior. **This precondition is enforced with an + assertion.** + @pre Each element of the container has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with a noncompliant container and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @param[in] i input to read from + @param[in,out] sax SAX event listener + @param[in] format the format to parse (JSON, CBOR, MessagePack, or UBJSON) + @param[in] strict whether the input has to be consumed completely + + @return return value of the last processed SAX event + + @throw parse_error.101 if a parse error occurs; example: `""unexpected end + of input; expected string literal""` + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the SAX consumer @a sax has + a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `sax_parse()` function + reading from string and processing the events with a user-defined SAX + event consumer.,sax_parse} + + @since version 3.2.0 + */ + template <typename SAX> + static bool sax_parse(detail::input_adapter&& i, SAX* sax, + input_format_t format = input_format_t::json, + const bool strict = true) + { + assert(sax); + switch (format) + { + case input_format_t::json: + return parser(std::move(i)).sax_parse(sax, strict); + default: + return detail::binary_reader<basic_json, SAX>(std::move(i)).sax_parse(format, sax, strict); + } + } + + /*! + @brief deserialize from an iterator range with contiguous storage + + This function reads from an iterator range of a container with contiguous + storage of 1-byte values. Compatible container types include + `std::vector`, `std::string`, `std::array`, `std::valarray`, and + `std::initializer_list`. Furthermore, C-style arrays can be used with + `std::begin()`/`std::end()`. User-defined containers can be used as long + as they implement random-access iterators and a contiguous storage. + + @pre The iterator range is contiguous. Violating this precondition yields + undefined behavior. **This precondition is enforced with an assertion.** + @pre Each element in the range has a size of 1 byte. Violating this + precondition yields undefined behavior. **This precondition is enforced + with a static assertion.** + + @warning There is no way to enforce all preconditions at compile-time. If + the function is called with noncompliant iterators and with + assertions switched off, the behavior is undefined and will most + likely yield segmentation violation. + + @tparam IteratorType iterator of container with contiguous storage + @param[in] first begin of the range to parse (included) + @param[in] last end of the range to parse (excluded) + @param[in] cb a parser callback function of type @ref parser_callback_t + which is used to control the deserialization by filtering unwanted values + (optional) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return result of the deserialization + + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. The complexity can be higher if the parser callback function + @a cb has a super-linear complexity. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below demonstrates the `parse()` function reading + from an iterator range.,parse__iteratortype__parser_callback_t} + + @since version 2.0.3 + */ + template<class IteratorType, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static basic_json parse(IteratorType first, IteratorType last, + const parser_callback_t cb = nullptr, + const bool allow_exceptions = true) + { + basic_json result; + parser(detail::input_adapter(first, last), cb, allow_exceptions).parse(true, result); + return result; + } + + template<class IteratorType, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static bool accept(IteratorType first, IteratorType last) + { + return parser(detail::input_adapter(first, last)).accept(true); + } + + template<class IteratorType, class SAX, typename std::enable_if< + std::is_base_of< + std::random_access_iterator_tag, + typename std::iterator_traits<IteratorType>::iterator_category>::value, int>::type = 0> + static bool sax_parse(IteratorType first, IteratorType last, SAX* sax) + { + return parser(detail::input_adapter(first, last)).sax_parse(sax); + } + + /*! + @brief deserialize from stream + @deprecated This stream operator is deprecated and will be removed in + version 4.0.0 of the library. Please use + @ref operator>>(std::istream&, basic_json&) + instead; that is, replace calls like `j << i;` with `i >> j;`. + @since version 1.0.0; deprecated since version 3.0.0 + */ + JSON_DEPRECATED + friend std::istream& operator<<(basic_json& j, std::istream& i) + { + return operator>>(i, j); + } + + /*! + @brief deserialize from stream + + Deserializes an input stream to a JSON value. + + @param[in,out] i input stream to read a serialized JSON value from + @param[in,out] j JSON value to write the deserialized input to + + @throw parse_error.101 in case of an unexpected token + @throw parse_error.102 if to_unicode fails or surrogate error + @throw parse_error.103 if to_unicode fails + + @complexity Linear in the length of the input. The parser is a predictive + LL(1) parser. + + @note A UTF-8 byte order mark is silently ignored. + + @liveexample{The example below shows how a JSON value is constructed by + reading a serialization from a stream.,operator_deserialize} + + @sa parse(std::istream&, const parser_callback_t) for a variant with a + parser callback function to filter values while parsing + + @since version 1.0.0 + */ + friend std::istream& operator>>(std::istream& i, basic_json& j) + { + parser(detail::input_adapter(i)).parse(false, j); + return i; + } + + /// @} + + /////////////////////////// + // convenience functions // + /////////////////////////// + + /*! + @brief return the type as string + + Returns the type name as string to be used in error messages - usually to + indicate that a function was called on a wrong JSON type. + + @return a string representation of a the @a m_type member: + Value type | return value + ----------- | ------------- + null | `"null"` + boolean | `"boolean"` + string | `"string"` + number | `"number"` (for all number types) + object | `"object"` + array | `"array"` + discarded | `"discarded"` + + @exceptionsafety No-throw guarantee: this function never throws exceptions. + + @complexity Constant. + + @liveexample{The following code exemplifies `type_name()` for all JSON + types.,type_name} + + @sa @ref type() -- return the type of the JSON value + @sa @ref operator value_t() -- return the type of the JSON value (implicit) + + @since version 1.0.0, public since 2.1.0, `const char*` and `noexcept` + since 3.0.0 + */ + const char* type_name() const noexcept + { + { + switch (m_type) + { + case value_t::null: + return "null"; + case value_t::object: + return "object"; + case value_t::array: + return "array"; + case value_t::string: + return "string"; + case value_t::boolean: + return "boolean"; + case value_t::discarded: + return "discarded"; + default: + return "number"; + } + } + } + + + private: + ////////////////////// + // member variables // + ////////////////////// + + /// the type of the current element + value_t m_type = value_t::null; + + /// the value of the current element + json_value m_value = {}; + + ////////////////////////////////////////// + // binary serialization/deserialization // + ////////////////////////////////////////// + + /// @name binary serialization/deserialization support + /// @{ + + public: + /*! + @brief create a CBOR serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the CBOR (Concise + Binary Object Representation) serialization format. CBOR is a binary + serialization format which aims to be more compact than JSON itself, yet + more efficient to parse. + + The library uses the following mapping from JSON values types to + CBOR types according to the CBOR specification (RFC 7049): + + JSON value type | value/range | CBOR type | first byte + --------------- | ------------------------------------------ | ---------------------------------- | --------------- + null | `null` | Null | 0xF6 + boolean | `true` | True | 0xF5 + boolean | `false` | False | 0xF4 + number_integer | -9223372036854775808..-2147483649 | Negative integer (8 bytes follow) | 0x3B + number_integer | -2147483648..-32769 | Negative integer (4 bytes follow) | 0x3A + number_integer | -32768..-129 | Negative integer (2 bytes follow) | 0x39 + number_integer | -128..-25 | Negative integer (1 byte follow) | 0x38 + number_integer | -24..-1 | Negative integer | 0x20..0x37 + number_integer | 0..23 | Integer | 0x00..0x17 + number_integer | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_integer | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_integer | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1A + number_integer | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1B + number_unsigned | 0..23 | Integer | 0x00..0x17 + number_unsigned | 24..255 | Unsigned integer (1 byte follow) | 0x18 + number_unsigned | 256..65535 | Unsigned integer (2 bytes follow) | 0x19 + number_unsigned | 65536..4294967295 | Unsigned integer (4 bytes follow) | 0x1A + number_unsigned | 4294967296..18446744073709551615 | Unsigned integer (8 bytes follow) | 0x1B + number_float | *any value* | Double-Precision Float | 0xFB + string | *length*: 0..23 | UTF-8 string | 0x60..0x77 + string | *length*: 23..255 | UTF-8 string (1 byte follow) | 0x78 + string | *length*: 256..65535 | UTF-8 string (2 bytes follow) | 0x79 + string | *length*: 65536..4294967295 | UTF-8 string (4 bytes follow) | 0x7A + string | *length*: 4294967296..18446744073709551615 | UTF-8 string (8 bytes follow) | 0x7B + array | *size*: 0..23 | array | 0x80..0x97 + array | *size*: 23..255 | array (1 byte follow) | 0x98 + array | *size*: 256..65535 | array (2 bytes follow) | 0x99 + array | *size*: 65536..4294967295 | array (4 bytes follow) | 0x9A + array | *size*: 4294967296..18446744073709551615 | array (8 bytes follow) | 0x9B + object | *size*: 0..23 | map | 0xA0..0xB7 + object | *size*: 23..255 | map (1 byte follow) | 0xB8 + object | *size*: 256..65535 | map (2 bytes follow) | 0xB9 + object | *size*: 65536..4294967295 | map (4 bytes follow) | 0xBA + object | *size*: 4294967296..18446744073709551615 | map (8 bytes follow) | 0xBB + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a CBOR value. + + @note If NaN or Infinity are stored inside a JSON number, they are + serialized properly. This behavior differs from the @ref dump() + function which serializes NaN or Infinity to `null`. + + @note The following CBOR types are not used in the conversion: + - byte strings (0x40..0x5F) + - UTF-8 strings terminated by "break" (0x7F) + - arrays terminated by "break" (0x9F) + - maps terminated by "break" (0xBF) + - date/time (0xC0..0xC1) + - bignum (0xC2..0xC3) + - decimal fraction (0xC4) + - bigfloat (0xC5) + - tagged items (0xC6..0xD4, 0xD8..0xDB) + - expected conversions (0xD5..0xD7) + - simple values (0xE0..0xF3, 0xF8) + - undefined (0xF7) + - half and single-precision floats (0xF9-0xFA) + - break (0xFF) + + @param[in] j JSON value to serialize + @return MessagePack serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in CBOR format.,to_cbor} + + @sa http://cbor.io + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + analogous deserialization + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + + @since version 2.0.9 + */ + static std::vector<uint8_t> to_cbor(const basic_json& j) + { + std::vector<uint8_t> result; + to_cbor(j, result); + return result; + } + + static void to_cbor(const basic_json& j, detail::output_adapter<uint8_t> o) + { + binary_writer<uint8_t>(o).write_cbor(j); + } + + static void to_cbor(const basic_json& j, detail::output_adapter<char> o) + { + binary_writer<char>(o).write_cbor(j); + } + + /*! + @brief create a MessagePack serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the MessagePack + serialization format. MessagePack is a binary serialization format which + aims to be more compact than JSON itself, yet more efficient to parse. + + The library uses the following mapping from JSON values types to + MessagePack types according to the MessagePack specification: + + JSON value type | value/range | MessagePack type | first byte + --------------- | --------------------------------- | ---------------- | ---------- + null | `null` | nil | 0xC0 + boolean | `true` | true | 0xC3 + boolean | `false` | false | 0xC2 + number_integer | -9223372036854775808..-2147483649 | int64 | 0xD3 + number_integer | -2147483648..-32769 | int32 | 0xD2 + number_integer | -32768..-129 | int16 | 0xD1 + number_integer | -128..-33 | int8 | 0xD0 + number_integer | -32..-1 | negative fixint | 0xE0..0xFF + number_integer | 0..127 | positive fixint | 0x00..0x7F + number_integer | 128..255 | uint 8 | 0xCC + number_integer | 256..65535 | uint 16 | 0xCD + number_integer | 65536..4294967295 | uint 32 | 0xCE + number_integer | 4294967296..18446744073709551615 | uint 64 | 0xCF + number_unsigned | 0..127 | positive fixint | 0x00..0x7F + number_unsigned | 128..255 | uint 8 | 0xCC + number_unsigned | 256..65535 | uint 16 | 0xCD + number_unsigned | 65536..4294967295 | uint 32 | 0xCE + number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF + number_float | *any value* | float 64 | 0xCB + string | *length*: 0..31 | fixstr | 0xA0..0xBF + string | *length*: 32..255 | str 8 | 0xD9 + string | *length*: 256..65535 | str 16 | 0xDA + string | *length*: 65536..4294967295 | str 32 | 0xDB + array | *size*: 0..15 | fixarray | 0x90..0x9F + array | *size*: 16..65535 | array 16 | 0xDC + array | *size*: 65536..4294967295 | array 32 | 0xDD + object | *size*: 0..15 | fix map | 0x80..0x8F + object | *size*: 16..65535 | map 16 | 0xDE + object | *size*: 65536..4294967295 | map 32 | 0xDF + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a MessagePack value. + + @note The following values can **not** be converted to a MessagePack value: + - strings with more than 4294967295 bytes + - arrays with more than 4294967295 elements + - objects with more than 4294967295 elements + + @note The following MessagePack types are not used in the conversion: + - bin 8 - bin 32 (0xC4..0xC6) + - ext 8 - ext 32 (0xC7..0xC9) + - float 32 (0xCA) + - fixext 1 - fixext 16 (0xD4..0xD8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + + @note If NaN or Infinity are stored inside a JSON number, they are + serialized properly. This behavior differs from the @ref dump() + function which serializes NaN or Infinity to `null`. + + @param[in] j JSON value to serialize + @return MessagePack serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in MessagePack format.,to_msgpack} + + @sa http://msgpack.org + @sa @ref from_msgpack for the analogous deserialization + @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + + @since version 2.0.9 + */ + static std::vector<uint8_t> to_msgpack(const basic_json& j) + { + std::vector<uint8_t> result; + to_msgpack(j, result); + return result; + } + + static void to_msgpack(const basic_json& j, detail::output_adapter<uint8_t> o) + { + binary_writer<uint8_t>(o).write_msgpack(j); + } + + static void to_msgpack(const basic_json& j, detail::output_adapter<char> o) + { + binary_writer<char>(o).write_msgpack(j); + } + + /*! + @brief create a UBJSON serialization of a given JSON value + + Serializes a given JSON value @a j to a byte vector using the UBJSON + (Universal Binary JSON) serialization format. UBJSON aims to be more compact + than JSON itself, yet more efficient to parse. + + The library uses the following mapping from JSON values types to + UBJSON types according to the UBJSON specification: + + JSON value type | value/range | UBJSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | `Z` + boolean | `true` | true | `T` + boolean | `false` | false | `F` + number_integer | -9223372036854775808..-2147483649 | int64 | `L` + number_integer | -2147483648..-32769 | int32 | `l` + number_integer | -32768..-129 | int16 | `I` + number_integer | -128..127 | int8 | `i` + number_integer | 128..255 | uint8 | `U` + number_integer | 256..32767 | int16 | `I` + number_integer | 32768..2147483647 | int32 | `l` + number_integer | 2147483648..9223372036854775807 | int64 | `L` + number_unsigned | 0..127 | int8 | `i` + number_unsigned | 128..255 | uint8 | `U` + number_unsigned | 256..32767 | int16 | `I` + number_unsigned | 32768..2147483647 | int32 | `l` + number_unsigned | 2147483648..9223372036854775807 | int64 | `L` + number_float | *any value* | float64 | `D` + string | *with shortest length indicator* | string | `S` + array | *see notes on optimized format* | array | `[` + object | *see notes on optimized format* | map | `{` + + @note The mapping is **complete** in the sense that any JSON value type + can be converted to a UBJSON value. + + @note The following values can **not** be converted to a UBJSON value: + - strings with more than 9223372036854775807 bytes (theoretical) + - unsigned integer numbers above 9223372036854775807 + + @note The following markers are not used in the conversion: + - `Z`: no-op values are not created. + - `C`: single-byte strings are serialized with `S` markers. + + @note Any UBJSON output created @ref to_ubjson can be successfully parsed + by @ref from_ubjson. + + @note If NaN or Infinity are stored inside a JSON number, they are + serialized properly. This behavior differs from the @ref dump() + function which serializes NaN or Infinity to `null`. + + @note The optimized formats for containers are supported: Parameter + @a use_size adds size information to the beginning of a container and + removes the closing marker. Parameter @a use_type further checks + whether all elements of a container have the same type and adds the + type marker to the beginning of the container. The @a use_type + parameter must only be used together with @a use_size = true. Note + that @a use_size = true alone may result in larger representations - + the benefit of this parameter is that the receiving side is + immediately informed on the number of elements of the container. + + @param[in] j JSON value to serialize + @param[in] use_size whether to add size annotations to container types + @param[in] use_type whether to add type annotations to container types + (must be combined with @a use_size = true) + @return UBJSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in UBJSON format.,to_ubjson} + + @sa http://ubjson.org + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + analogous deserialization + @sa @ref to_cbor(const basic_json& for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + + @since version 3.1.0 + */ + static std::vector<uint8_t> to_ubjson(const basic_json& j, + const bool use_size = false, + const bool use_type = false) + { + std::vector<uint8_t> result; + to_ubjson(j, result, use_size, use_type); + return result; + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<uint8_t> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<uint8_t>(o).write_ubjson(j, use_size, use_type); + } + + static void to_ubjson(const basic_json& j, detail::output_adapter<char> o, + const bool use_size = false, const bool use_type = false) + { + binary_writer<char>(o).write_ubjson(j, use_size, use_type); + } + + + /*! + @brief Serializes the given JSON object `j` to BSON and returns a vector + containing the corresponding BSON-representation. + + BSON (Binary JSON) is a binary format in which zero or more ordered key/value pairs are + stored as a single entity (a so-called document). + + The library uses the following mapping from JSON values types to BSON types: + + JSON value type | value/range | BSON type | marker + --------------- | --------------------------------- | ----------- | ------ + null | `null` | null | 0x0A + boolean | `true`, `false` | boolean | 0x08 + number_integer | -9223372036854775808..-2147483649 | int64 | 0x12 + number_integer | -2147483648..2147483647 | int32 | 0x10 + number_integer | 2147483648..9223372036854775807 | int64 | 0x12 + number_unsigned | 0..2147483647 | int32 | 0x10 + number_unsigned | 2147483648..9223372036854775807 | int64 | 0x12 + number_unsigned | 9223372036854775808..18446744073709551615| -- | -- + number_float | *any value* | double | 0x01 + string | *any value* | string | 0x02 + array | *any value* | document | 0x04 + object | *any value* | document | 0x03 + + @warning The mapping is **incomplete**, since only JSON-objects (and things + contained therein) can be serialized to BSON. + Also, integers larger than 9223372036854775807 cannot be serialized to BSON, + and the keys may not contain U+0000, since they are serialized a + zero-terminated c-strings. + + @throw out_of_range.407 if `j.is_number_unsigned() && j.get<std::uint64_t>() > 9223372036854775807` + @throw out_of_range.409 if a key in `j` contains a NULL (U+0000) + @throw type_error.317 if `!j.is_object()` + + @pre The input `j` is required to be an object: `j.is_object() == true`. + + @note Any BSON output created via @ref to_bson can be successfully parsed + by @ref from_bson. + + @param[in] j JSON value to serialize + @return BSON serialization as byte vector + + @complexity Linear in the size of the JSON value @a j. + + @liveexample{The example shows the serialization of a JSON value to a byte + vector in BSON format.,to_bson} + + @sa http://bsonspec.org/spec.html + @sa @ref from_bson(detail::input_adapter&&, const bool strict) for the + analogous deserialization + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + related UBJSON format + @sa @ref to_cbor(const basic_json&) for the related CBOR format + @sa @ref to_msgpack(const basic_json&) for the related MessagePack format + */ + static std::vector<uint8_t> to_bson(const basic_json& j) + { + std::vector<uint8_t> result; + to_bson(j, result); + return result; + } + + /*! + @brief Serializes the given JSON object `j` to BSON and forwards the + corresponding BSON-representation to the given output_adapter `o`. + @param j The JSON object to convert to BSON. + @param o The output adapter that receives the binary BSON representation. + @pre The input `j` shall be an object: `j.is_object() == true` + @sa @ref to_bson(const basic_json&) + */ + static void to_bson(const basic_json& j, detail::output_adapter<uint8_t> o) + { + binary_writer<uint8_t>(o).write_bson(j); + } + + /*! + @copydoc to_bson(const basic_json&, detail::output_adapter<uint8_t>) + */ + static void to_bson(const basic_json& j, detail::output_adapter<char> o) + { + binary_writer<char>(o).write_bson(j); + } + + + /*! + @brief create a JSON value from an input in CBOR format + + Deserializes a given input @a i to a JSON value using the CBOR (Concise + Binary Object Representation) serialization format. + + The library maps CBOR types to JSON value types as follows: + + CBOR type | JSON value type | first byte + ---------------------- | --------------- | ---------- + Integer | number_unsigned | 0x00..0x17 + Unsigned integer | number_unsigned | 0x18 + Unsigned integer | number_unsigned | 0x19 + Unsigned integer | number_unsigned | 0x1A + Unsigned integer | number_unsigned | 0x1B + Negative integer | number_integer | 0x20..0x37 + Negative integer | number_integer | 0x38 + Negative integer | number_integer | 0x39 + Negative integer | number_integer | 0x3A + Negative integer | number_integer | 0x3B + Negative integer | number_integer | 0x40..0x57 + UTF-8 string | string | 0x60..0x77 + UTF-8 string | string | 0x78 + UTF-8 string | string | 0x79 + UTF-8 string | string | 0x7A + UTF-8 string | string | 0x7B + UTF-8 string | string | 0x7F + array | array | 0x80..0x97 + array | array | 0x98 + array | array | 0x99 + array | array | 0x9A + array | array | 0x9B + array | array | 0x9F + map | object | 0xA0..0xB7 + map | object | 0xB8 + map | object | 0xB9 + map | object | 0xBA + map | object | 0xBB + map | object | 0xBF + False | `false` | 0xF4 + True | `true` | 0xF5 + Null | `null` | 0xF6 + Half-Precision Float | number_float | 0xF9 + Single-Precision Float | number_float | 0xFA + Double-Precision Float | number_float | 0xFB + + @warning The mapping is **incomplete** in the sense that not all CBOR + types can be converted to a JSON value. The following CBOR types + are not supported and will yield parse errors (parse_error.112): + - byte strings (0x40..0x5F) + - date/time (0xC0..0xC1) + - bignum (0xC2..0xC3) + - decimal fraction (0xC4) + - bigfloat (0xC5) + - tagged items (0xC6..0xD4, 0xD8..0xDB) + - expected conversions (0xD5..0xD7) + - simple values (0xE0..0xF3, 0xF8) + - undefined (0xF7) + + @warning CBOR allows map keys of any type, whereas JSON only allows + strings as keys in object values. Therefore, CBOR maps with keys + other than UTF-8 strings are rejected (parse_error.113). + + @note Any CBOR output created @ref to_cbor can be successfully parsed by + @ref from_cbor. + + @param[in] i an input in CBOR format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.110 if the given input ends prematurely or the end of + file was not reached when @a strict was set to true + @throw parse_error.112 if unsupported features from CBOR were + used in the given input @a v or if the input is not valid CBOR + @throw parse_error.113 if a string was expected as map key, but not found + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in CBOR + format to a JSON value.,from_cbor} + + @sa http://cbor.io + @sa @ref to_cbor(const basic_json&) for the analogous serialization + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for the + related MessagePack format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + related UBJSON format + + @since version 2.0.9; parameter @a start_index since 2.1.1; changed to + consume input adapters, removed start_index parameter, and added + @a strict parameter since 3.0.0; added @a allow_exceptions parameter + since 3.2.0 + */ + static basic_json from_cbor(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_cbor(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_cbor(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::cbor, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @brief create a JSON value from an input in MessagePack format + + Deserializes a given input @a i to a JSON value using the MessagePack + serialization format. + + The library maps MessagePack types to JSON value types as follows: + + MessagePack type | JSON value type | first byte + ---------------- | --------------- | ---------- + positive fixint | number_unsigned | 0x00..0x7F + fixmap | object | 0x80..0x8F + fixarray | array | 0x90..0x9F + fixstr | string | 0xA0..0xBF + nil | `null` | 0xC0 + false | `false` | 0xC2 + true | `true` | 0xC3 + float 32 | number_float | 0xCA + float 64 | number_float | 0xCB + uint 8 | number_unsigned | 0xCC + uint 16 | number_unsigned | 0xCD + uint 32 | number_unsigned | 0xCE + uint 64 | number_unsigned | 0xCF + int 8 | number_integer | 0xD0 + int 16 | number_integer | 0xD1 + int 32 | number_integer | 0xD2 + int 64 | number_integer | 0xD3 + str 8 | string | 0xD9 + str 16 | string | 0xDA + str 32 | string | 0xDB + array 16 | array | 0xDC + array 32 | array | 0xDD + map 16 | object | 0xDE + map 32 | object | 0xDF + negative fixint | number_integer | 0xE0-0xFF + + @warning The mapping is **incomplete** in the sense that not all + MessagePack types can be converted to a JSON value. The following + MessagePack types are not supported and will yield parse errors: + - bin 8 - bin 32 (0xC4..0xC6) + - ext 8 - ext 32 (0xC7..0xC9) + - fixext 1 - fixext 16 (0xD4..0xD8) + + @note Any MessagePack output created @ref to_msgpack can be successfully + parsed by @ref from_msgpack. + + @param[in] i an input in MessagePack format convertible to an input + adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.110 if the given input ends prematurely or the end of + file was not reached when @a strict was set to true + @throw parse_error.112 if unsupported features from MessagePack were + used in the given input @a i or if the input is not valid MessagePack + @throw parse_error.113 if a string was expected as map key, but not found + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + MessagePack format to a JSON value.,from_msgpack} + + @sa http://msgpack.org + @sa @ref to_msgpack(const basic_json&) for the analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for + the related UBJSON format + @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for + the related BSON format + + @since version 2.0.9; parameter @a start_index since 2.1.1; changed to + consume input adapters, removed start_index parameter, and added + @a strict parameter since 3.0.0; added @a allow_exceptions parameter + since 3.2.0 + */ + static basic_json from_msgpack(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_msgpack(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_msgpack(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::msgpack, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @brief create a JSON value from an input in UBJSON format + + Deserializes a given input @a i to a JSON value using the UBJSON (Universal + Binary JSON) serialization format. + + The library maps UBJSON types to JSON value types as follows: + + UBJSON type | JSON value type | marker + ----------- | --------------------------------------- | ------ + no-op | *no value, next value is read* | `N` + null | `null` | `Z` + false | `false` | `F` + true | `true` | `T` + float32 | number_float | `d` + float64 | number_float | `D` + uint8 | number_unsigned | `U` + int8 | number_integer | `i` + int16 | number_integer | `I` + int32 | number_integer | `l` + int64 | number_integer | `L` + string | string | `S` + char | string | `C` + array | array (optimized values are supported) | `[` + object | object (optimized values are supported) | `{` + + @note The mapping is **complete** in the sense that any UBJSON value can + be converted to a JSON value. + + @param[in] i an input in UBJSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.110 if the given input ends prematurely or the end of + file was not reached when @a strict was set to true + @throw parse_error.112 if a parse error occurs + @throw parse_error.113 if a string could not be parsed successfully + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + UBJSON format to a JSON value.,from_ubjson} + + @sa http://ubjson.org + @sa @ref to_ubjson(const basic_json&, const bool, const bool) for the + analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for + the related MessagePack format + @sa @ref from_bson(detail::input_adapter&&, const bool, const bool) for + the related BSON format + + @since version 3.1.0; added @a allow_exceptions parameter since 3.2.0 + */ + static basic_json from_ubjson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_ubjson(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_ubjson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::ubjson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @brief Create a JSON value from an input in BSON format + + Deserializes a given input @a i to a JSON value using the BSON (Binary JSON) + serialization format. + + The library maps BSON record types to JSON value types as follows: + + BSON type | BSON marker byte | JSON value type + --------------- | ---------------- | --------------------------- + double | 0x01 | number_float + string | 0x02 | string + document | 0x03 | object + array | 0x04 | array + binary | 0x05 | still unsupported + undefined | 0x06 | still unsupported + ObjectId | 0x07 | still unsupported + boolean | 0x08 | boolean + UTC Date-Time | 0x09 | still unsupported + null | 0x0A | null + Regular Expr. | 0x0B | still unsupported + DB Pointer | 0x0C | still unsupported + JavaScript Code | 0x0D | still unsupported + Symbol | 0x0E | still unsupported + JavaScript Code | 0x0F | still unsupported + int32 | 0x10 | number_integer + Timestamp | 0x11 | still unsupported + 128-bit decimal float | 0x13 | still unsupported + Max Key | 0x7F | still unsupported + Min Key | 0xFF | still unsupported + + @warning The mapping is **incomplete**. The unsupported mappings + are indicated in the table above. + + @param[in] i an input in BSON format convertible to an input adapter + @param[in] strict whether to expect the input to be consumed until EOF + (true by default) + @param[in] allow_exceptions whether to throw exceptions in case of a + parse error (optional, true by default) + + @return deserialized JSON value + + @throw parse_error.114 if an unsupported BSON record type is encountered + + @complexity Linear in the size of the input @a i. + + @liveexample{The example shows the deserialization of a byte vector in + BSON format to a JSON value.,from_bson} + + @sa http://bsonspec.org/spec.html + @sa @ref to_bson(const basic_json&) for the analogous serialization + @sa @ref from_cbor(detail::input_adapter&&, const bool, const bool) for the + related CBOR format + @sa @ref from_msgpack(detail::input_adapter&&, const bool, const bool) for + the related MessagePack format + @sa @ref from_ubjson(detail::input_adapter&&, const bool, const bool) for the + related UBJSON format + */ + static basic_json from_bson(detail::input_adapter&& i, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(i)).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + /*! + @copydoc from_bson(detail::input_adapter&&, const bool, const bool) + */ + template<typename A1, typename A2, + detail::enable_if_t<std::is_constructible<detail::input_adapter, A1, A2>::value, int> = 0> + static basic_json from_bson(A1 && a1, A2 && a2, + const bool strict = true, + const bool allow_exceptions = true) + { + basic_json result; + detail::json_sax_dom_parser<basic_json> sdp(result, allow_exceptions); + const bool res = binary_reader(detail::input_adapter(std::forward<A1>(a1), std::forward<A2>(a2))).sax_parse(input_format_t::bson, &sdp, strict); + return res ? result : basic_json(value_t::discarded); + } + + + + /// @} + + ////////////////////////// + // JSON Pointer support // + ////////////////////////// + + /// @name JSON Pointer functions + /// @{ + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. Similar to @ref operator[](const typename + object_t::key_type&), `null` values are created in arrays and objects if + necessary. + + In particular: + - If the JSON pointer points to an object key that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. + - If the JSON pointer points to an array index that does not exist, it + is created an filled with a `null` value before a reference to it + is returned. All indices between the current maximum and the given + index are also filled with `null`. + - The special value `-` is treated as a synonym for the index past the + end. + + @param[in] ptr a JSON pointer + + @return reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.404 if the JSON pointer can not be resolved + + @liveexample{The behavior is shown in the example.,operatorjson_pointer} + + @since version 2.0.0 + */ + reference operator[](const json_pointer& ptr) + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Uses a JSON pointer to retrieve a reference to the respective JSON value. + No bound checking is performed. The function does not change the JSON + value; no `null` values are created. In particular, the the special value + `-` yields an exception. + + @param[in] ptr JSON pointer to the desired element + + @return const reference to the element pointed to by @a ptr + + @complexity Constant. + + @throw parse_error.106 if an array index begins with '0' + @throw parse_error.109 if an array index was not a number + @throw out_of_range.402 if the array index '-' is used + @throw out_of_range.404 if the JSON pointer can not be resolved + + @liveexample{The behavior is shown in the example.,operatorjson_pointer_const} + + @since version 2.0.0 + */ + const_reference operator[](const json_pointer& ptr) const + { + return ptr.get_unchecked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a reference to the element at with specified JSON pointer @a ptr, + with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. + + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. + + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.403 if the JSON pointer describes a key of an object + which cannot be found. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 2.0.0 + + @liveexample{The behavior is shown in the example.,at_json_pointer} + */ + reference at(const json_pointer& ptr) + { + return ptr.get_checked(this); + } + + /*! + @brief access specified element via JSON Pointer + + Returns a const reference to the element at with specified JSON pointer @a + ptr, with bounds checking. + + @param[in] ptr JSON pointer to the desired element + + @return reference to the element pointed to by @a ptr + + @throw parse_error.106 if an array index in the passed JSON pointer @a ptr + begins with '0'. See example below. + + @throw parse_error.109 if an array index in the passed JSON pointer @a ptr + is not a number. See example below. + + @throw out_of_range.401 if an array index in the passed JSON pointer @a ptr + is out of range. See example below. + + @throw out_of_range.402 if the array index '-' is used in the passed JSON + pointer @a ptr. As `at` provides checked access (and no elements are + implicitly inserted), the index '-' is always invalid. See example below. + + @throw out_of_range.403 if the JSON pointer describes a key of an object + which cannot be found. See example below. + + @throw out_of_range.404 if the JSON pointer @a ptr can not be resolved. + See example below. + + @exceptionsafety Strong guarantee: if an exception is thrown, there are no + changes in the JSON value. + + @complexity Constant. + + @since version 2.0.0 + + @liveexample{The behavior is shown in the example.,at_json_pointer_const} + */ + const_reference at(const json_pointer& ptr) const + { + return ptr.get_checked(this); + } + + /*! + @brief return flattened JSON value + + The function creates a JSON object whose keys are JSON pointers (see [RFC + 6901](https://tools.ietf.org/html/rfc6901)) and whose values are all + primitive. The original JSON value can be restored using the @ref + unflatten() function. + + @return an object that maps JSON pointers to primitive values + + @note Empty objects and arrays are flattened to `null` and will not be + reconstructed correctly by the @ref unflatten() function. + + @complexity Linear in the size the JSON value. + + @liveexample{The following code shows how a JSON object is flattened to an + object whose keys consist of JSON pointers.,flatten} + + @sa @ref unflatten() for the reverse function + + @since version 2.0.0 + */ + basic_json flatten() const + { + basic_json result(value_t::object); + json_pointer::flatten("", *this, result); + return result; + } + + /*! + @brief unflatten a previously flattened JSON value + + The function restores the arbitrary nesting of a JSON value that has been + flattened before using the @ref flatten() function. The JSON value must + meet certain constraints: + 1. The value must be an object. + 2. The keys must be JSON pointers (see + [RFC 6901](https://tools.ietf.org/html/rfc6901)) + 3. The mapped values must be primitive JSON types. + + @return the original JSON from a flattened version + + @note Empty objects and arrays are flattened by @ref flatten() to `null` + values and can not unflattened to their original type. Apart from + this example, for a JSON value `j`, the following is always true: + `j == j.flatten().unflatten()`. + + @complexity Linear in the size the JSON value. + + @throw type_error.314 if value is not an object + @throw type_error.315 if object values are not primitive + + @liveexample{The following code shows how a flattened JSON object is + unflattened into the original nested JSON object.,unflatten} + + @sa @ref flatten() for the reverse function + + @since version 2.0.0 + */ + basic_json unflatten() const + { + return json_pointer::unflatten(*this); + } + + /// @} + + ////////////////////////// + // JSON Patch functions // + ////////////////////////// + + /// @name JSON Patch functions + /// @{ + + /*! + @brief applies a JSON patch + + [JSON Patch](http://jsonpatch.com) defines a JSON document structure for + expressing a sequence of operations to apply to a JSON) document. With + this function, a JSON Patch is applied to the current JSON value by + executing all operations from the patch. + + @param[in] json_patch JSON patch document + @return patched document + + @note The application of a patch is atomic: Either all operations succeed + and the patched document is returned or an exception is thrown. In + any case, the original value is not changed: the patch is applied + to a copy of the value. + + @throw parse_error.104 if the JSON patch does not consist of an array of + objects + + @throw parse_error.105 if the JSON patch is malformed (e.g., mandatory + attributes are missing); example: `"operation add must have member path"` + + @throw out_of_range.401 if an array index is out of range. + + @throw out_of_range.403 if a JSON pointer inside the patch could not be + resolved successfully in the current JSON value; example: `"key baz not + found"` + + @throw out_of_range.405 if JSON pointer has no parent ("add", "remove", + "move") + + @throw other_error.501 if "test" operation was unsuccessful + + @complexity Linear in the size of the JSON value and the length of the + JSON patch. As usually only a fraction of the JSON value is affected by + the patch, the complexity can usually be neglected. + + @liveexample{The following code shows how a JSON patch is applied to a + value.,patch} + + @sa @ref diff -- create a JSON patch by comparing two JSON values + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + @sa [RFC 6901 (JSON Pointer)](https://tools.ietf.org/html/rfc6901) + + @since version 2.0.0 + */ + basic_json patch(const basic_json& json_patch) const + { + // make a working copy to apply the patch to + basic_json result = *this; + + // the valid JSON Patch operations + enum class patch_operations {add, remove, replace, move, copy, test, invalid}; + + const auto get_op = [](const std::string & op) + { + if (op == "add") + { + return patch_operations::add; + } + if (op == "remove") + { + return patch_operations::remove; + } + if (op == "replace") + { + return patch_operations::replace; + } + if (op == "move") + { + return patch_operations::move; + } + if (op == "copy") + { + return patch_operations::copy; + } + if (op == "test") + { + return patch_operations::test; + } + + return patch_operations::invalid; + }; + + // wrapper for "add" operation; add value at ptr + const auto operation_add = [&result](json_pointer & ptr, basic_json val) + { + // adding to the root of the target document means replacing it + if (ptr.is_root()) + { + result = val; + } + else + { + // make sure the top element of the pointer exists + json_pointer top_pointer = ptr.top(); + if (top_pointer != ptr) + { + result.at(top_pointer); + } + + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result[ptr]; + + switch (parent.m_type) + { + case value_t::null: + case value_t::object: + { + // use operator[] to add value + parent[last_path] = val; + break; + } + + case value_t::array: + { + if (last_path == "-") + { + // special case: append to back + parent.push_back(val); + } + else + { + const auto idx = json_pointer::array_index(last_path); + if (JSON_UNLIKELY(static_cast<size_type>(idx) > parent.size())) + { + // avoid undefined behavior + JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range")); + } + + // default case: insert add offset + parent.insert(parent.begin() + static_cast<difference_type>(idx), val); + } + break; + } + + // LCOV_EXCL_START + default: + { + // if there exists a parent it cannot be primitive + assert(false); + } + // LCOV_EXCL_STOP + } + } + }; + + // wrapper for "remove" operation; remove value at ptr + const auto operation_remove = [&result](json_pointer & ptr) + { + // get reference to parent of JSON pointer ptr + const auto last_path = ptr.pop_back(); + basic_json& parent = result.at(ptr); + + // remove child + if (parent.is_object()) + { + // perform range check + auto it = parent.find(last_path); + if (JSON_LIKELY(it != parent.end())) + { + parent.erase(it); + } + else + { + JSON_THROW(out_of_range::create(403, "key '" + last_path + "' not found")); + } + } + else if (parent.is_array()) + { + // note erase performs range check + parent.erase(static_cast<size_type>(json_pointer::array_index(last_path))); + } + }; + + // type check: top level value must be an array + if (JSON_UNLIKELY(not json_patch.is_array())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); + } + + // iterate and apply the operations + for (const auto& val : json_patch) + { + // wrapper to get a value for an operation + const auto get_value = [&val](const std::string & op, + const std::string & member, + bool string_type) -> basic_json & + { + // find value + auto it = val.m_value.object->find(member); + + // context-sensitive error message + const auto error_msg = (op == "op") ? "operation" : "operation '" + op + "'"; + + // check if desired value is present + if (JSON_UNLIKELY(it == val.m_value.object->end())) + { + JSON_THROW(parse_error::create(105, 0, error_msg + " must have member '" + member + "'")); + } + + // check if result is of type string + if (JSON_UNLIKELY(string_type and not it->second.is_string())) + { + JSON_THROW(parse_error::create(105, 0, error_msg + " must have string member '" + member + "'")); + } + + // no error: return value + return it->second; + }; + + // type check: every element of the array must be an object + if (JSON_UNLIKELY(not val.is_object())) + { + JSON_THROW(parse_error::create(104, 0, "JSON patch must be an array of objects")); + } + + // collect mandatory members + const std::string op = get_value("op", "op", true); + const std::string path = get_value(op, "path", true); + json_pointer ptr(path); + + switch (get_op(op)) + { + case patch_operations::add: + { + operation_add(ptr, get_value("add", "value", false)); + break; + } + + case patch_operations::remove: + { + operation_remove(ptr); + break; + } + + case patch_operations::replace: + { + // the "path" location must exist - use at() + result.at(ptr) = get_value("replace", "value", false); + break; + } + + case patch_operations::move: + { + const std::string from_path = get_value("move", "from", true); + json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The move operation is functionally identical to a + // "remove" operation on the "from" location, followed + // immediately by an "add" operation at the target + // location with the value that was just removed. + operation_remove(from_ptr); + operation_add(ptr, v); + break; + } + + case patch_operations::copy: + { + const std::string from_path = get_value("copy", "from", true); + const json_pointer from_ptr(from_path); + + // the "from" location must exist - use at() + basic_json v = result.at(from_ptr); + + // The copy is functionally identical to an "add" + // operation at the target location using the value + // specified in the "from" member. + operation_add(ptr, v); + break; + } + + case patch_operations::test: + { + bool success = false; + JSON_TRY + { + // check if "value" matches the one at "path" + // the "path" location must exist - use at() + success = (result.at(ptr) == get_value("test", "value", false)); + } + JSON_INTERNAL_CATCH (out_of_range&) + { + // ignore out of range errors: success remains false + } + + // throw an exception if test fails + if (JSON_UNLIKELY(not success)) + { + JSON_THROW(other_error::create(501, "unsuccessful: " + val.dump())); + } + + break; + } + + case patch_operations::invalid: + { + // op must be "add", "remove", "replace", "move", "copy", or + // "test" + JSON_THROW(parse_error::create(105, 0, "operation value '" + op + "' is invalid")); + } + } + } + + return result; + } + + /*! + @brief creates a diff as a JSON patch + + Creates a [JSON Patch](http://jsonpatch.com) so that value @a source can + be changed into the value @a target by calling @ref patch function. + + @invariant For two JSON values @a source and @a target, the following code + yields always `true`: + @code {.cpp} + source.patch(diff(source, target)) == target; + @endcode + + @note Currently, only `remove`, `add`, and `replace` operations are + generated. + + @param[in] source JSON value to compare from + @param[in] target JSON value to compare against + @param[in] path helper value to create JSON pointers + + @return a JSON patch to convert the @a source to @a target + + @complexity Linear in the lengths of @a source and @a target. + + @liveexample{The following code shows how a JSON patch is created as a + diff for two JSON values.,diff} + + @sa @ref patch -- apply a JSON patch + @sa @ref merge_patch -- apply a JSON Merge Patch + + @sa [RFC 6902 (JSON Patch)](https://tools.ietf.org/html/rfc6902) + + @since version 2.0.0 + */ + static basic_json diff(const basic_json& source, const basic_json& target, + const std::string& path = "") + { + // the patch + basic_json result(value_t::array); + + // if the values are the same, return empty patch + if (source == target) + { + return result; + } + + if (source.type() != target.type()) + { + // different types: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + } + else + { + switch (source.type()) + { + case value_t::array: + { + // first pass: traverse common elements + std::size_t i = 0; + while (i < source.size() and i < target.size()) + { + // recursive call to compare array values at index i + auto temp_diff = diff(source[i], target[i], path + "/" + std::to_string(i)); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + ++i; + } + + // i now reached the end of at least one array + // in a second pass, traverse the remaining elements + + // remove my remaining elements + const auto end_index = static_cast<difference_type>(result.size()); + while (i < source.size()) + { + // add operations in reverse order to avoid invalid + // indices + result.insert(result.begin() + end_index, object( + { + {"op", "remove"}, + {"path", path + "/" + std::to_string(i)} + })); + ++i; + } + + // add other remaining elements + while (i < target.size()) + { + result.push_back( + { + {"op", "add"}, + {"path", path + "/" + std::to_string(i)}, + {"value", target[i]} + }); + ++i; + } + + break; + } + + case value_t::object: + { + // first pass: traverse this object's elements + for (auto it = source.cbegin(); it != source.cend(); ++it) + { + // escape the key name to be used in a JSON patch + const auto key = json_pointer::escape(it.key()); + + if (target.find(it.key()) != target.end()) + { + // recursive call to compare object values at key it + auto temp_diff = diff(it.value(), target[it.key()], path + "/" + key); + result.insert(result.end(), temp_diff.begin(), temp_diff.end()); + } + else + { + // found a key that is not in o -> remove it + result.push_back(object( + { + {"op", "remove"}, {"path", path + "/" + key} + })); + } + } + + // second pass: traverse other object's elements + for (auto it = target.cbegin(); it != target.cend(); ++it) + { + if (source.find(it.key()) == source.end()) + { + // found a key that is not in this -> add it + const auto key = json_pointer::escape(it.key()); + result.push_back( + { + {"op", "add"}, {"path", path + "/" + key}, + {"value", it.value()} + }); + } + } + + break; + } + + default: + { + // both primitive type: replace value + result.push_back( + { + {"op", "replace"}, {"path", path}, {"value", target} + }); + break; + } + } + } + + return result; + } + + /// @} + + //////////////////////////////// + // JSON Merge Patch functions // + //////////////////////////////// + + /// @name JSON Merge Patch functions + /// @{ + + /*! + @brief applies a JSON Merge Patch + + The merge patch format is primarily intended for use with the HTTP PATCH + method as a means of describing a set of modifications to a target + resource's content. This function applies a merge patch to the current + JSON value. + + The function implements the following algorithm from Section 2 of + [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396): + + ``` + define MergePatch(Target, Patch): + if Patch is an Object: + if Target is not an Object: + Target = {} // Ignore the contents and set it to an empty Object + for each Name/Value pair in Patch: + if Value is null: + if Name exists in Target: + remove the Name/Value pair from Target + else: + Target[Name] = MergePatch(Target[Name], Value) + return Target + else: + return Patch + ``` + + Thereby, `Target` is the current object; that is, the patch is applied to + the current value. + + @param[in] apply_patch the patch to apply + + @complexity Linear in the lengths of @a patch. + + @liveexample{The following code shows how a JSON Merge Patch is applied to + a JSON document.,merge_patch} + + @sa @ref patch -- apply a JSON patch + @sa [RFC 7396 (JSON Merge Patch)](https://tools.ietf.org/html/rfc7396) + + @since version 3.0.0 + */ + void merge_patch(const basic_json& apply_patch) + { + if (apply_patch.is_object()) + { + if (not is_object()) + { + *this = object(); + } + for (auto it = apply_patch.begin(); it != apply_patch.end(); ++it) + { + if (it.value().is_null()) + { + erase(it.key()); + } + else + { + operator[](it.key()).merge_patch(it.value()); + } + } + } + else + { + *this = apply_patch; + } + } + + /// @} +}; +} // namespace nlohmann + +/////////////////////// +// nonmember support // +/////////////////////// + +// specialization of std::swap, and std::hash +namespace std +{ + +/// hash value for JSON objects +template<> +struct hash<nlohmann::json> +{ + /*! + @brief return a hash value for a JSON object + + @since version 1.0.0 + */ + std::size_t operator()(const nlohmann::json& j) const + { + // a naive hashing via the string representation + const auto& h = hash<nlohmann::json::string_t>(); + return h(j.dump()); + } +}; + +/// specialization for std::less<value_t> +/// @note: do not remove the space after '<', +/// see https://github.com/nlohmann/json/pull/679 +template<> +struct less< ::nlohmann::detail::value_t> +{ + /*! + @brief compare two value_t enum values + @since version 3.0.0 + */ + bool operator()(nlohmann::detail::value_t lhs, + nlohmann::detail::value_t rhs) const noexcept + { + return nlohmann::detail::operator<(lhs, rhs); + } +}; + +/*! +@brief exchanges the values of two JSON objects + +@since version 1.0.0 +*/ +template<> +inline void swap<nlohmann::json>(nlohmann::json& j1, nlohmann::json& j2) noexcept( + is_nothrow_move_constructible<nlohmann::json>::value and + is_nothrow_move_assignable<nlohmann::json>::value +) +{ + j1.swap(j2); +} + +} // namespace std + +/*! +@brief user-defined string literal for JSON values + +This operator implements a user-defined string literal for JSON objects. It +can be used by adding `"_json"` to a string literal and returns a JSON object +if no parse error occurred. + +@param[in] s a string representation of a JSON object +@param[in] n the length of string @a s +@return a JSON object + +@since version 1.0.0 +*/ +inline nlohmann::json operator "" _json(const char* s, std::size_t n) +{ + return nlohmann::json::parse(s, s + n); +} + +/*! +@brief user-defined string literal for JSON pointer + +This operator implements a user-defined string literal for JSON Pointers. It +can be used by adding `"_json_pointer"` to a string literal and returns a JSON pointer +object if no parse error occurred. + +@param[in] s a string representation of a JSON Pointer +@param[in] n the length of string @a s +@return a JSON pointer object + +@since version 2.0.0 +*/ +inline nlohmann::json::json_pointer operator "" _json_pointer(const char* s, std::size_t n) +{ + return nlohmann::json::json_pointer(std::string(s, n)); +} + +// #include <nlohmann/detail/macro_unscope.hpp> + + +// restore GCC/clang diagnostic settings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic pop +#endif +#if defined(__clang__) + #pragma GCC diagnostic pop +#endif + +// clean up +#undef JSON_INTERNAL_CATCH +#undef JSON_CATCH +#undef JSON_THROW +#undef JSON_TRY +#undef JSON_LIKELY +#undef JSON_UNLIKELY +#undef JSON_DEPRECATED +#undef JSON_HAS_CPP_14 +#undef JSON_HAS_CPP_17 +#undef NLOHMANN_BASIC_JSON_TPL_DECLARATION +#undef NLOHMANN_BASIC_JSON_TPL + + +#endif diff --git a/third_party/nix/src/proto/CMakeLists.txt b/third_party/nix/src/proto/CMakeLists.txt new file mode 100644 index 000000000000..726ca8742f86 --- /dev/null +++ b/third_party/nix/src/proto/CMakeLists.txt @@ -0,0 +1,37 @@ +# -*- mode: cmake; -*- +# +# The proto generation happens outside of CMake and the path to the +# generated files is passed in via the environment variable +# $NIX_PROTO_SRCS. +# +# This configuration defines a library target that compiles these +# sources and makes the headers available. + +add_library(nixproto SHARED) +set_property(TARGET nixproto PROPERTY CXX_STANDARD 17) + +set(HEADER_FILES + $ENV{NIX_PROTO_SRCS}/libproto/worker.grpc.pb.h + $ENV{NIX_PROTO_SRCS}/libproto/worker.pb.h +) + +target_sources(nixproto + PUBLIC + ${HEADER_FILES} + + PRIVATE + $ENV{NIX_PROTO_SRCS}/libproto/worker.grpc.pb.cc + $ENV{NIX_PROTO_SRCS}/libproto/worker.pb.cc +) + +target_link_libraries(nixproto + gRPC::grpc++_reflection + protobuf::libprotobuf +) + +target_include_directories(nixproto + INTERFACE $ENV{NIX_PROTO_SRCS} +) + +INSTALL(FILES ${HEADER_FILES} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nix/libproto) +INSTALL(TARGETS nixproto DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff --git a/third_party/nix/src/proto/worker.proto b/third_party/nix/src/proto/worker.proto new file mode 100644 index 000000000000..b0dc32afbe5f --- /dev/null +++ b/third_party/nix/src/proto/worker.proto @@ -0,0 +1,374 @@ +syntax = "proto3"; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +package nix.proto; + +// Service representing a worker used for building and interfacing +// with the Nix store. +service WorkerService { + // Validates whether the supplied path is a valid store path. + rpc IsValidPath(StorePath) returns (IsValidPathResponse); + + // Checks whether any substitutes exist for the given path. + rpc HasSubstitutes(StorePath) returns (HasSubstitutesResponse); + + // Query referrers for a given path. + rpc QueryReferrers(StorePath) returns (StorePaths); + + // Add a path to the store. The first stream request + // should be a message indicating metadata, the rest should be file + // chunks. + rpc AddToStore(stream AddToStoreRequest) returns (StorePath); + + // Adds the supplied string to the store, as a text file. + rpc AddTextToStore(stream AddTextToStoreRequest) returns (StorePath); + + // Build the specified derivations in one of the specified build + // modes, defaulting to a normal build. + rpc BuildPaths(BuildPathsRequest) returns (stream BuildEvent); + + // TODO: What does this do? + rpc EnsurePath(StorePath) returns (google.protobuf.Empty); + + // TODO: What does this do? + // TODO(grfn): This should not actually take a StorePath, as it's not a + // StorePath + rpc AddTempRoot(StorePath) returns (google.protobuf.Empty); + + // TODO: What does this do? + rpc AddIndirectRoot(StorePath) returns (google.protobuf.Empty); + + // TODO: What does this do? + rpc SyncWithGC(google.protobuf.Empty) returns (google.protobuf.Empty); + + // TODO: What does this do? + rpc FindRoots(google.protobuf.Empty) returns (FindRootsResponse); + + // TODO: What does this do? + rpc SetOptions(SetOptionsRequest) returns (google.protobuf.Empty); + + // Ask the store to perform a garbage collection, based on the + // specified parameters. See `GCAction` for the possible actions. + rpc CollectGarbage(CollectGarbageRequest) returns (CollectGarbageResponse); + + // TODO: What does this do? + rpc QuerySubstitutablePathInfos(StorePaths) returns (SubstitutablePathInfos); + + // TODO: What does this do? + rpc QueryDerivationOutputs(StorePath) returns (StorePaths); + + // Query all valid paths in the store + rpc QueryAllValidPaths(google.protobuf.Empty) returns (StorePaths); + + // TODO: What does this do? + rpc QueryPathInfo(StorePath) returns (PathInfo); + + // Query the output names of the given derivation + rpc QueryDerivationOutputNames(StorePath) returns (DerivationOutputNames); + + // TODO: What is a HashPart? + rpc QueryPathFromHashPart(HashPart) returns (StorePath); + + // Query which of the given paths is valid. + rpc QueryValidPaths(StorePaths) returns (StorePaths); + + // Query which of the given paths have substitutes. + rpc QuerySubstitutablePaths(StorePaths) returns (StorePaths); + + // Return all currently valid derivations that have the given store path as an + // output. + rpc QueryValidDerivers(StorePath) returns (StorePaths); + + // Optimise the disk space usage of the Nix store by hard-linking files + // with the same contents. + rpc OptimiseStore(google.protobuf.Empty) returns (google.protobuf.Empty); + + // Check the integrity of the Nix store + rpc VerifyStore(VerifyStoreRequest) returns (VerifyStoreResponse); + + // Build a single non-materialized derivation (i.e. not from an + // on-disk .drv file). + rpc BuildDerivation(BuildDerivationRequest) returns (stream BuildEvent); + + // Add signatures to the specified store path. The signatures are not + // verified. + rpc AddSignatures(AddSignaturesRequest) returns (google.protobuf.Empty); + + // TODO: What does this do? + rpc NarFromPath(StorePath) returns (StorePath); + + // Upload & add a NAR to the daemon's Nix store. + rpc AddToStoreNar(stream AddToStoreNarRequest) + returns (google.protobuf.Empty); + + // Given a set of paths that are to be built, return the set of + // derivations that will be built, and the set of output paths that + // will be substituted. + rpc QueryMissing(StorePaths) returns (QueryMissingResponse); + + // Return the build log of the specified store path, if available + rpc GetBuildLog(StorePath) returns (BuildLog); +} + +enum HashType { + UNKNOWN = 0; + MD5 = 1; // TODO(tazjin): still needed? + SHA1 = 2; + SHA256 = 3; + SHA512 = 4; +} + +enum BuildMode { + Normal = 0; + Repair = 1; + Check = 2; +} + +enum GCAction { + // Return the set of paths reachable from (i.e. in the closure of) + // the roots. + ReturnLive = 0; + + // Return the set of paths not reachable from the roots. + ReturnDead = 1; + + // Actually delete the latter set. + DeleteDead = 2; + + // Delete the paths listed in `pathsToDelete', insofar as they are + // not reachable. + DeleteSpecific = 3; +} + +enum BuildStatus { + Built = 0; + Substituted = 1; + AlreadyValid = 2; + PermanentFailure = 3; + InputRejected = 4; + OutputRejected = 5; + TransientFailure = 6; // possibly transient + CachedFailure = 7; // no longer used + TimedOut = 8; + MiscFailure = 9; + DependencyFailed = 10; + LogLimitExceeded = 11; + NotDeterministic = 12; +}; + +// Generic type for any RPC call that just reads or returns a single +// store path. +message StorePath { + string path = 1; +} + +// Generic type for any RPC call that just reads or returns a list of +// store paths. +message StorePaths { + repeated string paths = 1; +} + +message Signatures { + repeated string sigs = 1; +} + +// Represents the outcome of a build for a single store path. +message BuildResult { + StorePath path = 1; + BuildStatus status = 2; + string msg = 3; +} + +// Represents an event occuring during a build. +message BuildEvent { + message LogLine { + string line = 1; + StorePath path = 2; + } + + oneof result_type { + // Build for a store path has finished + BuildResult result = 1; + + // A line of build log output was produced + LogLine build_log = 2; + + // Build for a store path has started + StorePath building_path = 3; + } +} + +message IsValidPathResponse { + bool is_valid = 1; +} + +message HasSubstitutesResponse { + bool has_substitutes = 1; +} + +message AddToStoreRequest { + message Metadata { + bool fixed = 1; + bool recursive = 2; // TODO(tazjin): what is this? "obsolete" comment? + HashType hash_type = 3; + string base_name = 4; + } + + oneof add_oneof { + Metadata meta = 1; + bytes data = 3; + } +} + +message AddTextToStoreRequest { + message Metadata { + string name = 1; + repeated string references = 2; + uint64 size = 3; + } + + oneof add_oneof { + Metadata meta = 4; + bytes data = 5; + } +} + +message BuildPathsRequest { + repeated string drvs = 1; + BuildMode mode = 2; +} + +message FindRootsResponse { + map<string, StorePaths> roots = 1; +} + +message SetOptionsRequest { + bool keep_failed = 1; + bool keep_going = 2; + bool try_fallback = 3; + uint32 max_build_jobs = 4; + uint32 verbose_build = + 5; // TODO(tazjin): Maybe this should be bool, unclear. + uint32 build_cores = 6; // TODO(tazjin): Difference from max_build_jobs? + bool use_substitutes = 7; + map<string, string> overrides = 8; // TODO(tazjin): better name? +} + +message CollectGarbageRequest { + // GC action that should be performed. + GCAction action = 1; + + // For `DeleteSpecific', the paths to delete. + repeated string paths_to_delete = 2; + + // If `ignore_liveness' is set, then reachability from the roots is + // ignored (dangerous!). However, the paths must still be + // unreferenced *within* the store (i.e., there can be no other + // store paths that depend on them). + bool ignore_liveness = 3; + + // Stop after at least `max_freed' bytes have been freed. + uint64 max_freed = 4; +} + +message CollectGarbageResponse { + // Depending on the action, the GC roots, or the paths that would be + // or have been deleted. + repeated string deleted_paths = 1; + + // For `ReturnDead', `DeleteDead' and `DeleteSpecific', the number + // of bytes that would be or was freed. + uint64 bytes_freed = 2; +} + +message PathInfo { + StorePath path = 10; + bool is_valid = 9; + StorePath deriver = 1; + bytes nar_hash = 2; + repeated string references = 3; + google.protobuf.Timestamp registration_time = 4; + uint64 download_size = 11; + uint64 nar_size = 5; + // Whether the path is ultimately trusted, that is, it's a derivation + // output that was built locally. + bool ultimate = 6; + repeated string sigs = 7; + // If non-empty, an assertion that the path is content-addressed + string ca = 8; + + // Only used for AddToStoreNarRequest + bool repair = 12; + bool check_sigs = 13; +} + +message SubstitutablePathInfos { + repeated PathInfo path_infos = 1; +} + +message DerivationOutputNames { + repeated string names = 1; +} + +message HashPart { + string hash_part = 1; +} + +message VerifyStoreRequest { + bool check_contents = 1; + bool repair = 2; + // TODO(grfn): Remove double-negative + bool dont_check_sigs = 3; +} + +message VerifyStoreResponse { + // True if errors remain (???) + bool errors = 1; +} + +message Derivation { + message DerivationOutput { + StorePath path = 1; + string hash_algo = 2; + bytes hash = 3; + } + map<string, DerivationOutput> outputs = 1; + StorePaths input_sources = 2; + string platform = 3; + StorePath builder = 4; + repeated string args = 5; + map<string, string> env = 6; +} + +message BuildDerivationRequest { + // Only used for informational purposes. + StorePath drv_path = 1; + Derivation derivation = 2; + BuildMode build_mode = 3; +} + +message AddSignaturesRequest { + StorePath path = 1; + Signatures sigs = 2; +} + +message AddToStoreNarRequest { + oneof add_oneof { + PathInfo path_info = 1; + bytes data = 2; + } +} + +message QueryMissingResponse { + repeated string will_build = 1; + repeated string will_substitute = 2; + repeated string unknown = 3; + uint64 download_size = 4; + uint64 nar_size = 5; +} + +message BuildLog { + string build_log = 1; +} diff --git a/third_party/nix/src/tests/CMakeLists.txt b/third_party/nix/src/tests/CMakeLists.txt new file mode 100644 index 000000000000..f8158d06c306 --- /dev/null +++ b/third_party/nix/src/tests/CMakeLists.txt @@ -0,0 +1,78 @@ +# -*- mode: cmake; -*- +include_directories(${PROJECT_BINARY_DIR}) # for 'generated/' + +add_executable(attr-set attr-set.cc) +target_link_libraries(attr-set + nixexpr + rapidcheck + rapidcheck_gtest + GTest::gtest_main +) + +gtest_discover_tests(attr-set) + +add_executable(derivations_test derivations_test.cc) +target_link_libraries(derivations_test + nixexpr + nixstore + rapidcheck + rapidcheck_gtest + GTest::gtest_main +) + +gtest_discover_tests(derivations_test) + +add_executable(hash_test hash_test.cc) +target_link_libraries(hash_test + nixutil + GTest::gtest_main +) + +gtest_discover_tests(hash_test) + +add_executable(references_test references_test.cc) +target_link_libraries(references_test + nixstore + rapidcheck + rapidcheck_gtest + GTest::gtest_main +) + +gtest_discover_tests(references_test) + +add_executable(store_test store_tests.cc) +target_link_libraries(store_test + nixstore + nixstoremock + GTest::gtest_main +) + +gtest_discover_tests(store_test) + +add_executable(value-to-json value-to-json.cc) +target_link_libraries(value-to-json + nixexpr + nixstore + GTest::gtest_main +) + +gtest_discover_tests(value-to-json) + +add_executable(language-tests language-tests.cc) +target_link_libraries(language-tests + nixexpr + nixstore + GTest::gtest_main +) + +gtest_discover_tests(language-tests) + +add_executable(store-api-test store-api-test.cc) +target_link_libraries(store-api-test + nixstore + rapidcheck + rapidcheck_gtest + GTest::gtest_main +) + +gtest_discover_tests(store-api-test) diff --git a/third_party/nix/src/tests/arbitrary.hh b/third_party/nix/src/tests/arbitrary.hh new file mode 100644 index 000000000000..026f8522cf71 --- /dev/null +++ b/third_party/nix/src/tests/arbitrary.hh @@ -0,0 +1,176 @@ +#pragma once + +#include <rapidcheck.h> +#include <rapidcheck/Gen.h> +#include <rapidcheck/gen/Arbitrary.h> + +#include "libexpr/attr-set.hh" +#include "libexpr/nixexpr.hh" +#include "libstore/derivations.hh" +#include "libutil/hash.hh" + +namespace nix::tests { +static nix::SymbolTable* symbol_table; +} + +namespace rc { + +using nix::Derivation; +using nix::DerivationOutput; +using nix::Pos; +using nix::Value; + +template <> +struct Arbitrary<nix::Symbol> { + static Gen<nix::Symbol> arbitrary() { + return gen::map(gen::arbitrary<std::string>(), [](std::string s) { + return nix::tests::symbol_table->Create(s); + }); + } +}; + +template <> +struct Arbitrary<Value> { + static Gen<nix::Value> arbitrary() { + return gen::build(gen::construct<Value>(), + // TODO(grfn) generalize to more types + gen::set(&Value::type, gen::just(nix::ValueType::tInt)), + gen::set(&Value::integer, gen::arbitrary<int64_t>())); + } +}; + +template <> +struct Arbitrary<Value*> { + static Gen<nix::Value*> arbitrary() { + return gen::apply( + [](nix::ValueType typ, int i) { + auto ret = new Value(); + ret->type = typ; + ret->integer = i; + return ret; + }, + gen::just(nix::ValueType::tInt), gen::arbitrary<int64_t>()); + } +}; + +template <> +struct Arbitrary<nix::Pos> { + static Gen<nix::Pos> arbitrary() { + return gen::construct<nix::Pos>(gen::arbitrary<nix::Symbol>(), + gen::arbitrary<unsigned int>(), + gen::arbitrary<unsigned int>()); + } +}; + +template <> +struct Arbitrary<nix::Pos*> { + static Gen<nix::Pos*> arbitrary() { + return gen::apply( + [](unsigned int line, unsigned int column) { + return new Pos({}, line, column); + }, + gen::arbitrary<unsigned int>(), gen::arbitrary<unsigned int>()); + } +}; + +template <> +struct Arbitrary<nix::Attr> { + static Gen<nix::Attr> arbitrary() { + return gen::construct<nix::Attr>(gen::arbitrary<nix::Symbol>(), + gen::arbitrary<Value*>(), + gen::arbitrary<nix::Pos*>()); + } +}; + +template <> +struct Arbitrary<nix::Bindings> { + static Gen<nix::Bindings> arbitrary() { + return gen::map(gen::arbitrary<std::vector<nix::Attr>>(), [](auto attrs) { + nix::Bindings res; + for (const auto& attr : attrs) { + res.push_back(attr); + } + return res; + }); + } +}; + +template <class K, class V> +struct Arbitrary<absl::btree_map<K, V>> { + static Gen<absl::btree_map<K, V>> arbitrary() { + return gen::map(gen::arbitrary<std::map<K, V>>(), [](std::map<K, V> map) { + absl::btree_map<K, V> out_map; + out_map.insert(map.begin(), map.end()); + return out_map; + }); + } +}; + +template <> +struct Arbitrary<nix::Base> { + static Gen<nix::Base> arbitrary() { + return gen::element(nix::Base16, nix::Base32, nix::Base64); + } +}; + +template <> +struct Arbitrary<DerivationOutput> { + static Gen<DerivationOutput> arbitrary() { + return gen::apply( + [](std::string content, std::string path, std::string hash_algo, + bool recursive, bool include_algo_in_hash, nix::Base base) { + auto hash_type = nix::parseHashType(hash_algo); + auto hash = nix::hashString(hash_type, content); + return DerivationOutput( + path, recursive ? absl::StrCat("r:", hash_algo) : hash_algo, + hash.to_string(base, include_algo_in_hash)); + }, + gen::arbitrary<std::string>(), + gen::map(gen::arbitrary<std::string>(), + [](std::string s) { return absl::StrCat("/", s); }), + gen::element<std::string>("md5", "sha1", "sha256", "sha512"), + gen::arbitrary<bool>(), gen::arbitrary<bool>(), + gen::arbitrary<nix::Base>()); + } +}; + +template <> +struct Arbitrary<Derivation> { + static Gen<Derivation> arbitrary() { + auto gen_path = gen::map(gen::arbitrary<std::string>(), [](std::string s) { + return absl::StrCat("/", s); + }); + + return gen::build<Derivation>( + gen::set(&nix::BasicDerivation::outputs), + gen::set(&nix::BasicDerivation::inputSrcs, + gen::container<nix::PathSet>(gen_path)), + gen::set(&nix::BasicDerivation::platform), + gen::set(&nix::BasicDerivation::builder, gen_path), + gen::set(&nix::BasicDerivation::args), + gen::set(&nix::BasicDerivation::env), + gen::set(&Derivation::inputDrvs, + gen::container<nix::DerivationInputs>( + gen_path, gen::arbitrary<nix::StringSet>()))); + } +}; + +template <> +struct Arbitrary<nix::BuildResult::Status> { + static Gen<nix::BuildResult::Status> arbitrary() { + return gen::element(nix::BuildResult::Status::Built, + nix::BuildResult::Status::Substituted, + nix::BuildResult::Status::AlreadyValid, + nix::BuildResult::Status::PermanentFailure, + nix::BuildResult::Status::InputRejected, + nix::BuildResult::Status::OutputRejected, + nix::BuildResult::Status::TransientFailure, + nix::BuildResult::Status::CachedFailure, + nix::BuildResult::Status::TimedOut, + nix::BuildResult::Status::MiscFailure, + nix::BuildResult::Status::DependencyFailed, + nix::BuildResult::Status::LogLimitExceeded, + nix::BuildResult::Status::NotDeterministic); + } +}; +} // namespace rc diff --git a/third_party/nix/src/tests/attr-set.cc b/third_party/nix/src/tests/attr-set.cc new file mode 100644 index 000000000000..35932bbeff65 --- /dev/null +++ b/third_party/nix/src/tests/attr-set.cc @@ -0,0 +1,71 @@ +#include "libexpr/attr-set.hh" + +#include <cstdio> +#include <optional> +#include <string> +#include <vector> + +#include <absl/container/btree_map.h> +#include <bits/stdint-intn.h> +#include <gtest/gtest.h> +#include <rapidcheck.h> +#include <rapidcheck/Assertions.h> +#include <rapidcheck/gen/Arbitrary.h> +#include <rapidcheck/gen/Build.h> +#include <rapidcheck/gen/Create.h> +#include <rapidcheck/gen/Transform.h> +#include <rapidcheck/gtest.h> + +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" +#include "libexpr/symbol-table.hh" +#include "libexpr/value.hh" +#include "tests/arbitrary.hh" +#include "tests/dummy-store.hh" + +namespace nix { + +using nix::tests::DummyStore; + +class AttrSetTest : public ::testing::Test { + protected: + EvalState* eval_state_; + void SetUp() override { + nix::expr::InitGC(); + auto store = std::make_shared<DummyStore>(); + eval_state_ = new EvalState({"."}, ref<Store>(store)); + tests::symbol_table = &eval_state_->symbols; + } + + void assert_bindings_equal(nix::Bindings* lhs, nix::Bindings* rhs) { + RC_ASSERT(lhs->Equal(rhs, *eval_state_)); + } +}; + +class AttrSetMonoidTest : public AttrSetTest {}; + +RC_GTEST_FIXTURE_PROP(AttrSetMonoidTest, mergeLeftIdentity, + (nix::Bindings && bindings)) { + auto empty_bindings = nix::Bindings::New(); + auto result = Bindings::Merge(*empty_bindings, bindings); + assert_bindings_equal(result.get(), &bindings); +} + +RC_GTEST_FIXTURE_PROP(AttrSetMonoidTest, mergeRightIdentity, + (nix::Bindings && bindings)) { + auto empty_bindings = nix::Bindings::New(); + auto result = Bindings::Merge(bindings, *empty_bindings); + assert_bindings_equal(result.get(), &bindings); +} + +RC_GTEST_FIXTURE_PROP(AttrSetMonoidTest, mergeAssociative, + (nix::Bindings && bindings_1, nix::Bindings&& bindings_2, + nix::Bindings&& bindings_3)) { + auto b231 = + Bindings::Merge(bindings_1, *Bindings::Merge(bindings_2, bindings_3)); + auto b123 = + Bindings::Merge(*Bindings::Merge(bindings_1, bindings_2), bindings_3); + assert_bindings_equal(b231.get(), b123.get()); +} + +} // namespace nix diff --git a/third_party/nix/src/tests/derivations_test.cc b/third_party/nix/src/tests/derivations_test.cc new file mode 100644 index 000000000000..6ebe17824ce1 --- /dev/null +++ b/third_party/nix/src/tests/derivations_test.cc @@ -0,0 +1,109 @@ +#include "libstore/derivations.hh" + +#include <memory> + +#include <absl/strings/str_cat.h> +#include <gtest/gtest.h> +#include <rapidcheck.h> +#include <rapidcheck/Assertions.h> +#include <rapidcheck/gen/Arbitrary.h> +#include <rapidcheck/gen/Build.h> +#include <rapidcheck/gen/Container.h> +#include <rapidcheck/gen/Tuple.h> +#include <rapidcheck/gtest.h> +#include <rapidcheck/state.h> + +#include "libexpr/eval.hh" +#include "libutil/hash.hh" +#include "libutil/types.hh" +#include "tests/arbitrary.hh" + +namespace nix { + +void AssertDerivationsEqual(const Derivation& lhs, const Derivation& rhs) { + RC_ASSERT(lhs.outputs.size() == rhs.outputs.size()); + for (const auto& [k, lhs_v] : lhs.outputs) { + auto rhs_v = rhs.outputs.find(k); + RC_ASSERT(rhs_v != rhs.outputs.end()); + RC_ASSERT(lhs_v.path == rhs_v->second.path); + RC_ASSERT(lhs_v.hashAlgo == rhs_v->second.hashAlgo); + RC_ASSERT(lhs_v.hash == rhs_v->second.hash); + } + + RC_ASSERT(lhs.inputSrcs == rhs.inputSrcs); + RC_ASSERT(lhs.platform == rhs.platform); + RC_ASSERT(lhs.builder == rhs.builder); + RC_ASSERT(lhs.args == rhs.args); + RC_ASSERT(lhs.env == rhs.env); + RC_ASSERT(lhs.inputDrvs == rhs.inputDrvs); +} + +class DerivationsTest : public ::testing::Test {}; + +// NOLINTNEXTLINE +RC_GTEST_FIXTURE_PROP(DerivationsTest, UnparseParseRoundTrip, + (Derivation && drv)) { + auto unparsed = drv.unparse(); + auto parsed = parseDerivation(unparsed); + AssertDerivationsEqual(drv, parsed); +} + +// NOLINTNEXTLINE +RC_GTEST_FIXTURE_PROP(DerivationsTest, ToProtoPreservesInput, + (Derivation && drv)) { + auto proto = drv.to_proto(); + + RC_ASSERT(proto.outputs_size() == drv.outputs.size()); + RC_ASSERT(proto.input_sources().paths_size() == drv.inputSrcs.size()); + auto paths = proto.input_sources().paths(); + for (const auto& input_src : drv.inputSrcs) { + RC_ASSERT(std::find(paths.begin(), paths.end(), input_src) != paths.end()); + } + + RC_ASSERT(proto.platform() == drv.platform); + RC_ASSERT(proto.builder().path() == drv.builder); + + RC_ASSERT(proto.args_size() == drv.args.size()); + auto args = proto.args(); + for (const auto& arg : drv.args) { + RC_ASSERT(std::find(args.begin(), args.end(), arg) != args.end()); + } + + RC_ASSERT(proto.env_size() == drv.env.size()); + auto env = proto.env(); + for (const auto& [key, value] : drv.env) { + RC_ASSERT(env.at(key) == value); + } +} + +class ParseDrvPathWithOutputsTest : public DerivationsTest {}; + +TEST(ParseDrvPathWithOutputsTest, ParseDrvPathWithOutputs) { + auto input = "/nix/store/my51f75kp056md84gq2v08pd140pcz57-test.drv!out"; + auto result = nix::parseDrvPathWithOutputs(input); + + ASSERT_EQ(result.first, + "/nix/store/my51f75kp056md84gq2v08pd140pcz57-test.drv"); + ASSERT_EQ(result.second, nix::PathSet{"out"}); +} + +TEST(ParseDrvPathWithOutputsTest, ParseDrvPathWithMultipleOutputs) { + auto input = "/nix/store/my51f75kp056md84gq2v08pd140pcz57-test.drv!out,dev"; + auto result = nix::parseDrvPathWithOutputs(input); + + nix::PathSet expected = {"out", "dev"}; + + ASSERT_EQ(result.first, + "/nix/store/my51f75kp056md84gq2v08pd140pcz57-test.drv"); + ASSERT_EQ(result.second, expected); +} + +TEST(ParseDrvPathWithOutputsTest, ParseDrvPathWithNoOutputs) { + auto input = "/nix/store/my51f75kp056md84gq2v08pd140pcz57-test"; + auto result = nix::parseDrvPathWithOutputs(input); + + ASSERT_EQ(result.first, "/nix/store/my51f75kp056md84gq2v08pd140pcz57-test"); + ASSERT_EQ(result.second, nix::PathSet()); +} + +} // namespace nix diff --git a/third_party/nix/src/tests/dummy-store.hh b/third_party/nix/src/tests/dummy-store.hh new file mode 100644 index 000000000000..8047d25727e1 --- /dev/null +++ b/third_party/nix/src/tests/dummy-store.hh @@ -0,0 +1,48 @@ +#pragma once + +#include <filesystem> + +#include "libstore/store-api.hh" + +namespace nix::tests { + +class DummyStore final : public Store { + public: + explicit DummyStore() : Store(Store::Params{}) {} + + std::string getUri() { return ""; } + + void queryPathInfoUncached( + const Path& path, + Callback<std::shared_ptr<ValidPathInfo>> callback) noexcept {} + + Path queryPathFromHashPart(const std::string& hashPart) { return ""; } + + Path addToStore(const std::string& name, const Path& srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter& filter = defaultPathFilter, + RepairFlag repair = NoRepair) { + if (srcPath == "/exists-for-tests") { + return "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + } + + throw SysError("file does not exist"); + } + + Path addTextToStore(const std::string& name, const std::string& s, + const PathSet& references, RepairFlag repair = NoRepair) { + return "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + } + + void narFromPath(const Path& path, Sink& sink) {} + + BuildResult buildDerivation(std::ostream& log_sink, const Path& drvPath, + const BasicDerivation& drv, + BuildMode buildMode = bmNormal) { + return BuildResult{}; + } + + void ensurePath(const Path& path) {} +}; + +} // namespace nix::tests diff --git a/third_party/nix/src/tests/hash_test.cc b/third_party/nix/src/tests/hash_test.cc new file mode 100644 index 000000000000..caa77f5d6b15 --- /dev/null +++ b/third_party/nix/src/tests/hash_test.cc @@ -0,0 +1,101 @@ +#include "libutil/hash.hh" + +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +class HashTest : public ::testing::Test {}; + +using testing::EndsWith; +using testing::HasSubstr; + +namespace nix { + +TEST(HashTest, SHA256) { + auto hash = hashString(HashType::htSHA256, "foo"); + ASSERT_EQ(hash.base64Len(), 44); + ASSERT_EQ(hash.base32Len(), 52); + ASSERT_EQ(hash.base16Len(), 64); + + ASSERT_EQ(hash.to_string(Base16), + "sha256:" + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"); + ASSERT_EQ(hash.to_string(Base32), + "sha256:1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89ic"); + ASSERT_EQ(hash.to_string(Base64), + "sha256:LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564="); +} + +TEST(HashTest, SHA256Decode) { + auto hash = hashString(HashType::htSHA256, "foo"); + + std::unique_ptr<Hash> base16 = std::make_unique<Hash>( + "sha256:" + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae", + HashType::htSHA256); + std::unique_ptr<Hash> base32 = std::make_unique<Hash>( + "sha256:1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89ic", + HashType::htSHA256); + std::unique_ptr<Hash> base64 = std::make_unique<Hash>( + "sha256:LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm564=", + HashType::htSHA256); + + ASSERT_EQ(hash, *base16); + ASSERT_EQ(hash, *base32); + ASSERT_EQ(hash, *base64); +} + +TEST(HashTest, SHA256DecodeFail) { + EXPECT_THAT( + Hash::deserialize("sha256:LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm56==", + HashType::htSHA256) + .status() + .message(), + HasSubstr("wrong length")); + EXPECT_THAT( + Hash::deserialize("sha256:LCa0a2j/xo/5m0U8HTBBNBNCLXBkg7+g+YpeiGJm56,=", + HashType::htSHA256) + .status() + .message(), + HasSubstr("invalid base-64")); + + EXPECT_THAT(Hash::deserialize( + "sha256:1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89i", + HashType::htSHA256) + .status() + .message(), + HasSubstr("wrong length")); + absl::StatusOr<Hash> badB32Char = Hash::deserialize( + "sha256:1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89i,", + HashType::htSHA256); + EXPECT_THAT(badB32Char.status().message(), HasSubstr("invalid base-32")); + EXPECT_THAT(badB32Char.status().message(), EndsWith(",")); + + EXPECT_THAT( + Hash::deserialize( + "sha256:" + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7 ", + HashType::htSHA256) + .status() + .message(), + HasSubstr("invalid base-16")); +} + +TEST(HashSink, SHA256) { + HashSink sink(htSHA256); + + sink.write(reinterpret_cast<const unsigned char*>("fo"), 2); + HashResult partial = sink.currentHash(); + EXPECT_EQ(partial.first.to_string(Base16), + "sha256:" + "9c3aee7110b787f0fb5f81633a36392bd277ea945d44c874a9a23601aefe20cf"); + EXPECT_EQ(partial.second, 2); + + sink.write(reinterpret_cast<const unsigned char*>("o"), 1); + HashResult end = sink.finish(); + EXPECT_EQ(end.first.to_string(Base16), + "sha256:" + "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"); + EXPECT_EQ(end.second, 3); +} + +} // namespace nix diff --git a/third_party/nix/src/tests/lang/binary-data b/third_party/nix/src/tests/lang/binary-data new file mode 100644 index 000000000000..06d740502001 --- /dev/null +++ b/third_party/nix/src/tests/lang/binary-data Binary files differdiff --git a/third_party/nix/src/tests/lang/data b/third_party/nix/src/tests/lang/data new file mode 100644 index 000000000000..257cc5642cb1 --- /dev/null +++ b/third_party/nix/src/tests/lang/data @@ -0,0 +1 @@ +foo diff --git a/third_party/nix/src/tests/lang/dir1/a.nix b/third_party/nix/src/tests/lang/dir1/a.nix new file mode 100644 index 000000000000..231f150c579c --- /dev/null +++ b/third_party/nix/src/tests/lang/dir1/a.nix @@ -0,0 +1 @@ +"a" diff --git a/third_party/nix/src/tests/lang/dir2/a.nix b/third_party/nix/src/tests/lang/dir2/a.nix new file mode 100644 index 000000000000..170df520ab68 --- /dev/null +++ b/third_party/nix/src/tests/lang/dir2/a.nix @@ -0,0 +1 @@ +"X" diff --git a/third_party/nix/src/tests/lang/dir2/b.nix b/third_party/nix/src/tests/lang/dir2/b.nix new file mode 100644 index 000000000000..19010cc35ca6 --- /dev/null +++ b/third_party/nix/src/tests/lang/dir2/b.nix @@ -0,0 +1 @@ +"b" diff --git a/third_party/nix/src/tests/lang/dir3/a.nix b/third_party/nix/src/tests/lang/dir3/a.nix new file mode 100644 index 000000000000..170df520ab68 --- /dev/null +++ b/third_party/nix/src/tests/lang/dir3/a.nix @@ -0,0 +1 @@ +"X" diff --git a/third_party/nix/src/tests/lang/dir3/b.nix b/third_party/nix/src/tests/lang/dir3/b.nix new file mode 100644 index 000000000000..170df520ab68 --- /dev/null +++ b/third_party/nix/src/tests/lang/dir3/b.nix @@ -0,0 +1 @@ +"X" diff --git a/third_party/nix/src/tests/lang/dir3/c.nix b/third_party/nix/src/tests/lang/dir3/c.nix new file mode 100644 index 000000000000..cdf158597eef --- /dev/null +++ b/third_party/nix/src/tests/lang/dir3/c.nix @@ -0,0 +1 @@ +"c" diff --git a/third_party/nix/src/tests/lang/dir4/a.nix b/third_party/nix/src/tests/lang/dir4/a.nix new file mode 100644 index 000000000000..170df520ab68 --- /dev/null +++ b/third_party/nix/src/tests/lang/dir4/a.nix @@ -0,0 +1 @@ +"X" diff --git a/third_party/nix/src/tests/lang/dir4/c.nix b/third_party/nix/src/tests/lang/dir4/c.nix new file mode 100644 index 000000000000..170df520ab68 --- /dev/null +++ b/third_party/nix/src/tests/lang/dir4/c.nix @@ -0,0 +1 @@ +"X" diff --git a/third_party/nix/src/tests/lang/disabled/README.txt b/third_party/nix/src/tests/lang/disabled/README.txt new file mode 100644 index 000000000000..50225deb63a2 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/README.txt @@ -0,0 +1,7 @@ +These tests are disabled primarily because the DummyStore used for +tests does not interact with real files on disk at the moment, but the +tests expect it to. + +Once we have a solution for this (potentially just reading & hashing +the files, but not writing them anywhere) these tests will be enabled +again. diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-path.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-path.nix new file mode 100644 index 000000000000..e67168cf3edf --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-path.nix @@ -0,0 +1,7 @@ +builtins.path + { path = ./.; + filter = path: _: baseNameOf path == "data"; + recursive = true; + sha256 = "1yhm3gwvg5a41yylymgblsclk95fs6jy72w0wv925mmidlhcq4sw"; + name = "output"; + } diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.exp new file mode 100644 index 000000000000..4519bc406db5 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.exp @@ -0,0 +1 @@ +"abccX" diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.flags b/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.flags new file mode 100644 index 000000000000..a28e6821004a --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.flags @@ -0,0 +1 @@ +-I lang/dir1 -I lang/dir2 -I dir5=lang/dir3 \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.nix new file mode 100644 index 000000000000..cca41f821f83 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-search-path.nix @@ -0,0 +1,11 @@ +with import ./lib.nix; +with builtins; + +assert pathExists <nix/buildenv.nix>; + +assert length __nixPath == 6; +assert length (filter (x: x.prefix == "nix") __nixPath) == 1; +assert length (filter (x: baseNameOf x.path == "dir4") __nixPath) == 1; + +import <a.nix> + import <b.nix> + import <c.nix> + import <dir5/c.nix> + + (let __nixPath = [ { path = ./dir2; } { path = ./dir1; } ]; in import <a.nix>) diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-tail-call-1.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-tail-call-1.nix new file mode 100644 index 000000000000..a3962ce3fdb5 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-tail-call-1.nix @@ -0,0 +1,3 @@ +let + f = n: if n == 100000 then n else f (n + 1); +in f 0 diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-xml.exp b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.exp new file mode 100644 index 000000000000..92b75e0b8b17 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.exp @@ -0,0 +1,52 @@ +<?xml version='1.0' encoding='utf-8'?> +<expr> + <attrs> + <attr name="a"> + <string value="foo" /> + </attr> + <attr name="at"> + <function> + <attrspat name="args"> + <attr name="x" /> + <attr name="y" /> + <attr name="z" /> + </attrspat> + </function> + </attr> + <attr name="b"> + <string value="bar" /> + </attr> + <attr name="c"> + <string value="foobar" /> + </attr> + <attr name="ellipsis"> + <function> + <attrspat ellipsis="1"> + <attr name="x" /> + <attr name="y" /> + <attr name="z" /> + </attrspat> + </function> + </attr> + <attr name="f"> + <function> + <attrspat> + <attr name="z" /> + <attr name="x" /> + <attr name="y" /> + </attrspat> + </function> + </attr> + <attr name="id"> + <function> + <varpat name="x" /> + </function> + </attr> + <attr name="x"> + <int value="123" /> + </attr> + <attr name="y"> + <float value="567.89" /> + </attr> + </attrs> +</expr> diff --git a/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix new file mode 100644 index 000000000000..9ee9f8a0b4f5 --- /dev/null +++ b/third_party/nix/src/tests/lang/disabled/eval-okay-xml.nix @@ -0,0 +1,21 @@ +rec { + + x = 123; + + y = 567.890; + + a = "foo"; + + b = "bar"; + + c = "foo" + "bar"; + + f = {z, x, y}: if y then x else z; + + id = x: x; + + at = args@{x, y, z}: x; + + ellipsis = {x, y, z, ...}: x; + +} diff --git a/third_party/nix/src/tests/lang/eval-fail-abort.nix b/third_party/nix/src/tests/lang/eval-fail-abort.nix new file mode 100644 index 000000000000..75c51bceb540 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-abort.nix @@ -0,0 +1 @@ +if true then abort "this should fail" else 1 diff --git a/third_party/nix/src/tests/lang/eval-fail-antiquoted-path.nix b/third_party/nix/src/tests/lang/eval-fail-antiquoted-path.nix new file mode 100644 index 000000000000..f2f08107b516 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-antiquoted-path.nix @@ -0,0 +1,4 @@ +# This must fail to evaluate, since ./fnord doesn't exist. If it did +# exist, it would produce "/nix/store/<hash>-fnord/xyzzy" (with an +# appropriate context). +"${./fnord}/xyzzy" diff --git a/third_party/nix/src/tests/lang/eval-fail-assert.nix b/third_party/nix/src/tests/lang/eval-fail-assert.nix new file mode 100644 index 000000000000..3b7a1e8bf0c2 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-assert.nix @@ -0,0 +1,5 @@ +let { + x = arg: assert arg == "y"; 123; + + body = x "x"; +} \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-1.nix b/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-1.nix new file mode 100644 index 000000000000..ffe9c983c26b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-1.nix @@ -0,0 +1 @@ +"${x: x}" diff --git a/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-2.nix b/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-2.nix new file mode 100644 index 000000000000..3745235ce95e --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-2.nix @@ -0,0 +1 @@ +"${./fnord}" diff --git a/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-3.nix b/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-3.nix new file mode 100644 index 000000000000..65b9d4f505b1 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-bad-antiquote-3.nix @@ -0,0 +1 @@ +''${x: x}'' diff --git a/third_party/nix/src/tests/lang/eval-fail-blackhole.nix b/third_party/nix/src/tests/lang/eval-fail-blackhole.nix new file mode 100644 index 000000000000..81133b511c95 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-blackhole.nix @@ -0,0 +1,5 @@ +let { + body = x; + x = y; + y = x; +} diff --git a/third_party/nix/src/tests/lang/eval-fail-deepseq.nix b/third_party/nix/src/tests/lang/eval-fail-deepseq.nix new file mode 100644 index 000000000000..9baa49b063ec --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-deepseq.nix @@ -0,0 +1 @@ +builtins.deepSeq { x = abort "foo"; } 456 diff --git a/third_party/nix/src/tests/lang/eval-fail-hashfile-missing.nix b/third_party/nix/src/tests/lang/eval-fail-hashfile-missing.nix new file mode 100644 index 000000000000..ce098b82380a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-hashfile-missing.nix @@ -0,0 +1,5 @@ +let + paths = [ ./this-file-is-definitely-not-there-7392097 "/and/neither/is/this/37293620" ]; +in + toString (builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"])) + diff --git a/third_party/nix/src/tests/lang/eval-fail-missing-arg.nix b/third_party/nix/src/tests/lang/eval-fail-missing-arg.nix new file mode 100644 index 000000000000..c4be9797c534 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-missing-arg.nix @@ -0,0 +1 @@ +({x, y, z}: x + y + z) {x = "foo"; z = "bar";} diff --git a/third_party/nix/src/tests/lang/eval-fail-remove.nix b/third_party/nix/src/tests/lang/eval-fail-remove.nix new file mode 100644 index 000000000000..539e0eb0a6f6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-remove.nix @@ -0,0 +1,5 @@ +let { + attrs = {x = 123; y = 456;}; + + body = (removeAttrs attrs ["x"]).x; +} \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/eval-fail-scope-5.nix b/third_party/nix/src/tests/lang/eval-fail-scope-5.nix new file mode 100644 index 000000000000..f89a65a99be3 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-scope-5.nix @@ -0,0 +1,10 @@ +let { + + x = "a"; + y = "b"; + + f = {x ? y, y ? x}: x + y; + + body = f {}; + +} diff --git a/third_party/nix/src/tests/lang/eval-fail-seq.nix b/third_party/nix/src/tests/lang/eval-fail-seq.nix new file mode 100644 index 000000000000..cddbbfd3261e --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-seq.nix @@ -0,0 +1 @@ +builtins.seq (abort "foo") 2 diff --git a/third_party/nix/src/tests/lang/eval-fail-substring.nix b/third_party/nix/src/tests/lang/eval-fail-substring.nix new file mode 100644 index 000000000000..f37c2bc0a160 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-substring.nix @@ -0,0 +1 @@ +builtins.substring (builtins.sub 0 1) 1 "x" diff --git a/third_party/nix/src/tests/lang/eval-fail-to-path.nix b/third_party/nix/src/tests/lang/eval-fail-to-path.nix new file mode 100644 index 000000000000..5e322bc31369 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-to-path.nix @@ -0,0 +1 @@ +builtins.toPath "foo/bar" diff --git a/third_party/nix/src/tests/lang/eval-fail-undeclared-arg.nix b/third_party/nix/src/tests/lang/eval-fail-undeclared-arg.nix new file mode 100644 index 000000000000..cafdf1636272 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-fail-undeclared-arg.nix @@ -0,0 +1 @@ +({x, z}: x + z) {x = "foo"; y = "bla"; z = "bar";} diff --git a/third_party/nix/src/tests/lang/eval-okay-any-all.exp b/third_party/nix/src/tests/lang/eval-okay-any-all.exp new file mode 100644 index 000000000000..eb273f45b2a6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-any-all.exp @@ -0,0 +1 @@ +[ false false true true true true false true ] diff --git a/third_party/nix/src/tests/lang/eval-okay-any-all.nix b/third_party/nix/src/tests/lang/eval-okay-any-all.nix new file mode 100644 index 000000000000..a3f26ea2aa83 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-any-all.nix @@ -0,0 +1,11 @@ +with builtins; + +[ (any (x: x == 1) []) + (any (x: x == 1) [2 3 4]) + (any (x: x == 1) [1 2 3 4]) + (any (x: x == 1) [4 3 2 1]) + (all (x: x == 1) []) + (all (x: x == 1) [1]) + (all (x: x == 1) [1 2 3]) + (all (x: x == 1) [1 1 1]) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-arithmetic.exp b/third_party/nix/src/tests/lang/eval-okay-arithmetic.exp new file mode 100644 index 000000000000..5c54d10b7b47 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-arithmetic.exp @@ -0,0 +1 @@ +2216 diff --git a/third_party/nix/src/tests/lang/eval-okay-arithmetic.nix b/third_party/nix/src/tests/lang/eval-okay-arithmetic.nix new file mode 100644 index 000000000000..7e9e6a0b666e --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-arithmetic.nix @@ -0,0 +1,59 @@ +with import ./lib.nix; + +let { + + /* Supposedly tail recursive version: + + range_ = accum: first: last: + if first == last then ([first] ++ accum) + else range_ ([first] ++ accum) (builtins.add first 1) last; + + range = range_ []; + */ + + x = 12; + + err = abort "urgh"; + + body = sum + [ (sum (range 1 50)) + (123 + 456) + (0 + -10 + -(-11) + -x) + (10 - 7 - -2) + (10 - (6 - -1)) + (10 - 1 + 2) + (3 * 4 * 5) + (56088 / 123 / 2) + (3 + 4 * const 5 0 - 6 / id 2) + + (builtins.bitAnd 12 10) # 0b1100 & 0b1010 = 8 + (builtins.bitOr 12 10) # 0b1100 | 0b1010 = 14 + (builtins.bitXor 12 10) # 0b1100 ^ 0b1010 = 6 + + (if 3 < 7 then 1 else err) + (if 7 < 3 then err else 1) + (if 3 < 3 then err else 1) + + (if 3 <= 7 then 1 else err) + (if 7 <= 3 then err else 1) + (if 3 <= 3 then 1 else err) + + (if 3 > 7 then err else 1) + (if 7 > 3 then 1 else err) + (if 3 > 3 then err else 1) + + (if 3 >= 7 then err else 1) + (if 7 >= 3 then 1 else err) + (if 3 >= 3 then 1 else err) + + (if 2 > 1 == 1 < 2 then 1 else err) + (if 1 + 2 * 3 >= 7 then 1 else err) + (if 1 + 2 * 3 < 7 then err else 1) + + # Not integer, but so what. + (if "aa" < "ab" then 1 else err) + (if "aa" < "aa" then err else 1) + (if "foo" < "foobar" then 1 else err) + ]; + +} diff --git a/third_party/nix/src/tests/lang/eval-okay-attrnames.exp b/third_party/nix/src/tests/lang/eval-okay-attrnames.exp new file mode 100644 index 000000000000..b4aa387e07b8 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrnames.exp @@ -0,0 +1 @@ +"newxfoonewxy" diff --git a/third_party/nix/src/tests/lang/eval-okay-attrnames.nix b/third_party/nix/src/tests/lang/eval-okay-attrnames.nix new file mode 100644 index 000000000000..e5b26e9f2e39 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrnames.nix @@ -0,0 +1,11 @@ +with import ./lib.nix; + +let + + attrs = {y = "y"; x = "x"; foo = "foo";} // rec {x = "newx"; bar = x;}; + + names = builtins.attrNames attrs; + + values = map (name: builtins.getAttr name attrs) names; + +in assert values == builtins.attrValues attrs; concat values diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs.exp b/third_party/nix/src/tests/lang/eval-okay-attrs.exp new file mode 100644 index 000000000000..45b0f829eb33 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs.exp @@ -0,0 +1 @@ +987 diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs.nix b/third_party/nix/src/tests/lang/eval-okay-attrs.nix new file mode 100644 index 000000000000..810b31a5da96 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs.nix @@ -0,0 +1,5 @@ +let { + as = { x = 123; y = 456; } // { z = 789; } // { z = 987; }; + + body = if as ? a then as.a else assert as ? z; as.z; +} diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs2.exp b/third_party/nix/src/tests/lang/eval-okay-attrs2.exp new file mode 100644 index 000000000000..45b0f829eb33 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs2.exp @@ -0,0 +1 @@ +987 diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs2.nix b/third_party/nix/src/tests/lang/eval-okay-attrs2.nix new file mode 100644 index 000000000000..9e06b83ac1fd --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs2.nix @@ -0,0 +1,10 @@ +let { + as = { x = 123; y = 456; } // { z = 789; } // { z = 987; }; + + A = "a"; + Z = "z"; + + body = if builtins.hasAttr A as + then builtins.getAttr A as + else assert builtins.hasAttr Z as; builtins.getAttr Z as; +} diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs3.exp b/third_party/nix/src/tests/lang/eval-okay-attrs3.exp new file mode 100644 index 000000000000..19de4fdf79f7 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs3.exp @@ -0,0 +1 @@ +"foo 22 80 itchyxac" diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs3.nix b/third_party/nix/src/tests/lang/eval-okay-attrs3.nix new file mode 100644 index 000000000000..f29de11fe660 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs3.nix @@ -0,0 +1,22 @@ +let + + config = + { + services.sshd.enable = true; + services.sshd.port = 22; + services.httpd.port = 80; + hostName = "itchy"; + a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z = "x"; + foo = { + a = "a"; + b.c = "c"; + }; + }; + +in + if config.services.sshd.enable + then "foo ${toString config.services.sshd.port} ${toString config.services.httpd.port} ${config.hostName}" + + "${config.a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z}" + + "${config.foo.a}" + + "${config.foo.b.c}" + else "bar" diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs4.exp b/third_party/nix/src/tests/lang/eval-okay-attrs4.exp new file mode 100644 index 000000000000..1851731442d3 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs4.exp @@ -0,0 +1 @@ +[ true false true false false true false false ] diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs4.nix b/third_party/nix/src/tests/lang/eval-okay-attrs4.nix new file mode 100644 index 000000000000..43ec81210f38 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs4.nix @@ -0,0 +1,7 @@ +let + + as = { x.y.z = 123; a.b.c = 456; }; + + bs = null; + +in [ (as ? x) (as ? y) (as ? x.y.z) (as ? x.y.z.a) (as ? x.y.a) (as ? a.b.c) (bs ? x) (bs ? x.y.z) ] diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs5.exp b/third_party/nix/src/tests/lang/eval-okay-attrs5.exp new file mode 100644 index 000000000000..ce0430d78081 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs5.exp @@ -0,0 +1 @@ +[ 123 "foo" 456 456 "foo" "xyzzy" "xyzzy" true ] diff --git a/third_party/nix/src/tests/lang/eval-okay-attrs5.nix b/third_party/nix/src/tests/lang/eval-okay-attrs5.nix new file mode 100644 index 000000000000..a4584cd3b398 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-attrs5.nix @@ -0,0 +1,21 @@ +with import ./lib.nix; + +let + + as = { x.y.z = 123; a.b.c = 456; }; + + bs = { f-o-o.bar = "foo"; }; + + or = x: y: x || y; + +in + [ as.x.y.z + as.foo or "foo" + as.x.y.bla or as.a.b.c + as.a.b.c or as.x.y.z + as.x.y.bla or bs.f-o-o.bar or "xyzzy" + as.x.y.bla or bs.bar.foo or "xyzzy" + (123).bla or null.foo or "xyzzy" + # Backwards compatibility test. + (fold or [] [true false false]) + ] diff --git a/third_party/nix/src/tests/lang/eval-okay-autoargs.flags b/third_party/nix/src/tests/lang/eval-okay-autoargs.flags new file mode 100644 index 000000000000..ae3762254460 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-autoargs.flags @@ -0,0 +1 @@ +--arg lib import(lang/lib.nix) --argstr xyzzy xyzzy! -A result diff --git a/third_party/nix/src/tests/lang/eval-okay-backslash-newline-1.exp b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-1.exp new file mode 100644 index 000000000000..3e754364cc9c --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-1.exp @@ -0,0 +1 @@ +"a\nb" diff --git a/third_party/nix/src/tests/lang/eval-okay-backslash-newline-1.nix b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-1.nix new file mode 100644 index 000000000000..7fef3dddd4dd --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-1.nix @@ -0,0 +1,2 @@ +"a\ +b" diff --git a/third_party/nix/src/tests/lang/eval-okay-backslash-newline-2.exp b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-2.exp new file mode 100644 index 000000000000..3e754364cc9c --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-2.exp @@ -0,0 +1 @@ +"a\nb" diff --git a/third_party/nix/src/tests/lang/eval-okay-backslash-newline-2.nix b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-2.nix new file mode 100644 index 000000000000..35ddf495c63b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-backslash-newline-2.nix @@ -0,0 +1,2 @@ +''a''\ +b'' diff --git a/third_party/nix/src/tests/lang/eval-okay-builtins-add.exp b/third_party/nix/src/tests/lang/eval-okay-builtins-add.exp new file mode 100644 index 000000000000..0350b518a7ec --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-builtins-add.exp @@ -0,0 +1 @@ +[ 5 4 "int" "tt" "float" 4 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-builtins-add.nix b/third_party/nix/src/tests/lang/eval-okay-builtins-add.nix new file mode 100644 index 000000000000..c841816222a5 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-builtins-add.nix @@ -0,0 +1,8 @@ +[ +(builtins.add 2 3) +(builtins.add 2 2) +(builtins.typeOf (builtins.add 2 2)) +("t" + "t") +(builtins.typeOf (builtins.add 2.0 2)) +(builtins.add 2.0 2) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-builtins.exp b/third_party/nix/src/tests/lang/eval-okay-builtins.exp new file mode 100644 index 000000000000..0661686d611d --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-builtins.exp @@ -0,0 +1 @@ +/foo diff --git a/third_party/nix/src/tests/lang/eval-okay-builtins.nix b/third_party/nix/src/tests/lang/eval-okay-builtins.nix new file mode 100644 index 000000000000..e9d65e88a817 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-builtins.nix @@ -0,0 +1,12 @@ +assert builtins ? currentSystem; +assert !builtins ? __currentSystem; + +let { + + x = if builtins ? dirOf then builtins.dirOf /foo/bar else ""; + + y = if builtins ? fnord then builtins.fnord "foo" else ""; + + body = x + y; + +} diff --git a/third_party/nix/src/tests/lang/eval-okay-callable-attrs.exp b/third_party/nix/src/tests/lang/eval-okay-callable-attrs.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-callable-attrs.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-callable-attrs.nix b/third_party/nix/src/tests/lang/eval-okay-callable-attrs.nix new file mode 100644 index 000000000000..310a030df004 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-callable-attrs.nix @@ -0,0 +1 @@ +({ __functor = self: x: self.foo && x; foo = false; } // { foo = true; }) true diff --git a/third_party/nix/src/tests/lang/eval-okay-catattrs.exp b/third_party/nix/src/tests/lang/eval-okay-catattrs.exp new file mode 100644 index 000000000000..b4a1e66d6b8a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-catattrs.exp @@ -0,0 +1 @@ +[ 1 2 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-catattrs.nix b/third_party/nix/src/tests/lang/eval-okay-catattrs.nix new file mode 100644 index 000000000000..2c3dc10da528 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-catattrs.nix @@ -0,0 +1 @@ +builtins.catAttrs "a" [ { a = 1; } { b = 0; } { a = 2; } ] diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.exp b/third_party/nix/src/tests/lang/eval-okay-closure.exp new file mode 100644 index 000000000000..e7dbf978160d --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-closure.exp @@ -0,0 +1 @@ +[ { foo = true; key = -13; } { foo = true; key = -12; } { foo = true; key = -11; } { foo = true; key = -9; } { foo = true; key = -8; } { foo = true; key = -7; } { foo = true; key = -5; } { foo = true; key = -4; } { foo = true; key = -3; } { key = -1; } { foo = true; key = 0; } { foo = true; key = 1; } { foo = true; key = 2; } { foo = true; key = 4; } { foo = true; key = 5; } { foo = true; key = 6; } { key = 8; } { foo = true; key = 9; } { foo = true; key = 10; } { foo = true; key = 13; } { foo = true; key = 14; } { foo = true; key = 15; } { key = 17; } { foo = true; key = 18; } { foo = true; key = 19; } { foo = true; key = 22; } { foo = true; key = 23; } { key = 26; } { foo = true; key = 27; } { foo = true; key = 28; } { foo = true; key = 31; } { foo = true; key = 32; } { key = 35; } { foo = true; key = 36; } { foo = true; key = 40; } { foo = true; key = 41; } { key = 44; } { foo = true; key = 45; } { foo = true; key = 49; } { key = 53; } { foo = true; key = 54; } { foo = true; key = 58; } { key = 62; } { foo = true; key = 67; } { key = 71; } { key = 80; } ] diff --git a/third_party/nix/src/tests/lang/eval-okay-closure.nix b/third_party/nix/src/tests/lang/eval-okay-closure.nix new file mode 100644 index 000000000000..cccd4dc35730 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-closure.nix @@ -0,0 +1,13 @@ +let + + closure = builtins.genericClosure { + startSet = [{key = 80;}]; + operator = {key, foo ? false}: + if builtins.lessThan key 0 + then [] + else [{key = builtins.sub key 9;} {key = builtins.sub key 13; foo = true;}]; + }; + + sort = (import ./lib.nix).sortBy (a: b: builtins.lessThan a.key b.key); + +in sort closure diff --git a/third_party/nix/src/tests/lang/eval-okay-comments.exp b/third_party/nix/src/tests/lang/eval-okay-comments.exp new file mode 100644 index 000000000000..7182dc2f9b8e --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-comments.exp @@ -0,0 +1 @@ +"abcdefghijklmnopqrstuvwxyz" diff --git a/third_party/nix/src/tests/lang/eval-okay-comments.nix b/third_party/nix/src/tests/lang/eval-okay-comments.nix new file mode 100644 index 000000000000..cb2cce218029 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-comments.nix @@ -0,0 +1,59 @@ +# A simple comment +"a"+ # And another +## A double comment +"b"+ ## And another +# Nested # comments # +"c"+ # and # some # other # +# An empty line, following here: + +"d"+ # and a comment not starting the line ! + +"e"+ +/* multiline comments */ +"f" + +/* multiline + comments, + on + multiple + lines +*/ +"g" + +# Small, tricky comments +/**/ "h"+ /*/*/ "i"+ /***/ "j"+ /* /*/ "k"+ /*/* /*/ "l"+ +# Comments with an even number of ending '*' used to fail: +"m"+ +/* */ /* **/ /* ***/ /* ****/ "n"+ +/* */ /** */ /*** */ /**** */ "o"+ +/** **/ /*** ***/ /**** ****/ "p"+ +/* * ** *** **** ***** */ "q"+ +# Random comments +/* ***** ////// * / * / /* */ "r"+ +# Mixed comments +/* # */ +"s"+ +# /* # +"t"+ +# /* # */ +"u"+ +# /*********/ +"v"+ +## */* +"w"+ +/* + * Multiline, decorated comments + * # This ain't a nest'd comm'nt + */ +"x"+ +''${/** with **/"y" + # real + /* comments + inside ! # */ + + # (and empty lines) + +}''+ /* And a multiline comment, + on the same line, + after some spaces +*/ # followed by a one-line comment +"z" +/* EOF */ diff --git a/third_party/nix/src/tests/lang/eval-okay-concat.exp b/third_party/nix/src/tests/lang/eval-okay-concat.exp new file mode 100644 index 000000000000..bb4bbd577410 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-concat.exp @@ -0,0 +1 @@ +[ 1 2 3 4 5 6 7 8 9 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-concat.nix b/third_party/nix/src/tests/lang/eval-okay-concat.nix new file mode 100644 index 000000000000..d158a9bf05b9 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-concat.nix @@ -0,0 +1 @@ +[1 2 3] ++ [4 5 6] ++ [7 8 9] diff --git a/third_party/nix/src/tests/lang/eval-okay-concatmap.exp b/third_party/nix/src/tests/lang/eval-okay-concatmap.exp new file mode 100644 index 000000000000..3b8be7739deb --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-concatmap.exp @@ -0,0 +1 @@ +[ [ 1 3 5 7 9 ] [ "a" "z" "b" "z" ] ] diff --git a/third_party/nix/src/tests/lang/eval-okay-concatmap.nix b/third_party/nix/src/tests/lang/eval-okay-concatmap.nix new file mode 100644 index 000000000000..97da5d37a412 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-concatmap.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +[ (builtins.concatMap (x: if x / 2 * 2 == x then [] else [ x ]) (range 0 10)) + (builtins.concatMap (x: [x] ++ ["z"]) ["a" "b"]) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-concatstringssep.exp b/third_party/nix/src/tests/lang/eval-okay-concatstringssep.exp new file mode 100644 index 000000000000..93987647ffe6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-concatstringssep.exp @@ -0,0 +1 @@ +[ "" "foobarxyzzy" "foo, bar, xyzzy" "foo" "" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-concatstringssep.nix b/third_party/nix/src/tests/lang/eval-okay-concatstringssep.nix new file mode 100644 index 000000000000..adc4c41bd551 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-concatstringssep.nix @@ -0,0 +1,8 @@ +with builtins; + +[ (concatStringsSep "" []) + (concatStringsSep "" ["foo" "bar" "xyzzy"]) + (concatStringsSep ", " ["foo" "bar" "xyzzy"]) + (concatStringsSep ", " ["foo"]) + (concatStringsSep ", " []) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-curpos.exp b/third_party/nix/src/tests/lang/eval-okay-curpos.exp new file mode 100644 index 000000000000..65fd65b4d01f --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-curpos.exp @@ -0,0 +1 @@ +[ 3 7 4 9 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-curpos.nix b/third_party/nix/src/tests/lang/eval-okay-curpos.nix new file mode 100644 index 000000000000..b79553df0bd3 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-curpos.nix @@ -0,0 +1,5 @@ +# Bla +let + x = __curPos; + y = __curPos; +in [ x.line x.column y.line y.column ] diff --git a/third_party/nix/src/tests/lang/eval-okay-deepseq.exp b/third_party/nix/src/tests/lang/eval-okay-deepseq.exp new file mode 100644 index 000000000000..8d38505c1686 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-deepseq.exp @@ -0,0 +1 @@ +456 diff --git a/third_party/nix/src/tests/lang/eval-okay-deepseq.nix b/third_party/nix/src/tests/lang/eval-okay-deepseq.nix new file mode 100644 index 000000000000..53aa4b1dc251 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-deepseq.nix @@ -0,0 +1 @@ +builtins.deepSeq (let as = { x = 123; y = as; }; in as) 456 diff --git a/third_party/nix/src/tests/lang/eval-okay-delayed-with-inherit.exp b/third_party/nix/src/tests/lang/eval-okay-delayed-with-inherit.exp new file mode 100644 index 000000000000..eaacb55c1aff --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-delayed-with-inherit.exp @@ -0,0 +1 @@ +"b-overridden" diff --git a/third_party/nix/src/tests/lang/eval-okay-delayed-with-inherit.nix b/third_party/nix/src/tests/lang/eval-okay-delayed-with-inherit.nix new file mode 100644 index 000000000000..84b388c27130 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-delayed-with-inherit.nix @@ -0,0 +1,24 @@ +let + pkgs_ = with pkgs; { + a = derivation { + name = "a"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "touch $out" ]; + inherit b; + }; + + inherit b; + }; + + packageOverrides = p: { + b = derivation { + name = "b-overridden"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "touch $out" ]; + }; + }; + + pkgs = pkgs_ // (packageOverrides pkgs_); +in pkgs.a.b.name diff --git a/third_party/nix/src/tests/lang/eval-okay-delayed-with.exp b/third_party/nix/src/tests/lang/eval-okay-delayed-with.exp new file mode 100644 index 000000000000..8e7c61ab8e77 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-delayed-with.exp @@ -0,0 +1 @@ +"b-overridden b-overridden a" diff --git a/third_party/nix/src/tests/lang/eval-okay-delayed-with.nix b/third_party/nix/src/tests/lang/eval-okay-delayed-with.nix new file mode 100644 index 000000000000..3fb023e1cd42 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-delayed-with.nix @@ -0,0 +1,29 @@ +let + + pkgs_ = with pkgs; { + a = derivation { + name = "a"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "touch $out" ]; + inherit b; + }; + + b = derivation { + name = "b"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "touch $out" ]; + inherit a; + }; + + c = b; + }; + + packageOverrides = pkgs: with pkgs; { + b = derivation (b.drvAttrs // { name = "${b.name}-overridden"; }); + }; + + pkgs = pkgs_ // (packageOverrides pkgs_); + +in "${pkgs.a.b.name} ${pkgs.c.name} ${pkgs.b.a.name}" diff --git a/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-2.exp b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-2.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-2.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-2.nix b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-2.nix new file mode 100644 index 000000000000..6d57bf854908 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-2.nix @@ -0,0 +1 @@ +{ a."${"b"}" = true; a."${"c"}" = false; }.a.b diff --git a/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-bare.exp b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-bare.exp new file mode 100644 index 000000000000..df8750afc036 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-bare.exp @@ -0,0 +1 @@ +{ binds = true; hasAttrs = true; multiAttrs = true; recBinds = true; selectAttrs = true; selectOrAttrs = true; } diff --git a/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-bare.nix b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-bare.nix new file mode 100644 index 000000000000..0dbe15e6384c --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs-bare.nix @@ -0,0 +1,17 @@ +let + aString = "a"; + + bString = "b"; +in { + hasAttrs = { a.b = null; } ? ${aString}.b; + + selectAttrs = { a.b = true; }.a.${bString}; + + selectOrAttrs = { }.${aString} or true; + + binds = { ${aString}."${bString}c" = true; }.a.bc; + + recBinds = rec { ${bString} = a; a = true; }.b; + + multiAttrs = { ${aString} = true; ${bString} = false; }.a; +} diff --git a/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs.exp b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs.exp new file mode 100644 index 000000000000..df8750afc036 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs.exp @@ -0,0 +1 @@ +{ binds = true; hasAttrs = true; multiAttrs = true; recBinds = true; selectAttrs = true; selectOrAttrs = true; } diff --git a/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs.nix b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs.nix new file mode 100644 index 000000000000..ee02ac7e6579 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-dynamic-attrs.nix @@ -0,0 +1,17 @@ +let + aString = "a"; + + bString = "b"; +in { + hasAttrs = { a.b = null; } ? "${aString}".b; + + selectAttrs = { a.b = true; }.a."${bString}"; + + selectOrAttrs = { }."${aString}" or true; + + binds = { "${aString}"."${bString}c" = true; }.a.bc; + + recBinds = rec { "${bString}" = a; a = true; }.b; + + multiAttrs = { "${aString}" = true; "${bString}" = false; }.a; +} diff --git a/third_party/nix/src/tests/lang/eval-okay-elem.exp b/third_party/nix/src/tests/lang/eval-okay-elem.exp new file mode 100644 index 000000000000..3cf6c0e962f0 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-elem.exp @@ -0,0 +1 @@ +[ true false 30 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-elem.nix b/third_party/nix/src/tests/lang/eval-okay-elem.nix new file mode 100644 index 000000000000..71ea7a4ed03d --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-elem.nix @@ -0,0 +1,6 @@ +with import ./lib.nix; + +let xs = range 10 40; in + +[ (builtins.elem 23 xs) (builtins.elem 42 xs) (builtins.elemAt xs 20) ] + diff --git a/third_party/nix/src/tests/lang/eval-okay-empty-args.exp b/third_party/nix/src/tests/lang/eval-okay-empty-args.exp new file mode 100644 index 000000000000..cb5537d5d7ce --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-empty-args.exp @@ -0,0 +1 @@ +"ab" diff --git a/third_party/nix/src/tests/lang/eval-okay-empty-args.nix b/third_party/nix/src/tests/lang/eval-okay-empty-args.nix new file mode 100644 index 000000000000..78c133afdd94 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-empty-args.nix @@ -0,0 +1 @@ +({}: {x,y,}: "${x}${y}") {} {x = "a"; y = "b";} diff --git a/third_party/nix/src/tests/lang/eval-okay-eq-derivations.exp b/third_party/nix/src/tests/lang/eval-okay-eq-derivations.exp new file mode 100644 index 000000000000..ec04aab6aeec --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq-derivations.exp @@ -0,0 +1 @@ +[ true true true false ] diff --git a/third_party/nix/src/tests/lang/eval-okay-eq-derivations.nix b/third_party/nix/src/tests/lang/eval-okay-eq-derivations.nix new file mode 100644 index 000000000000..d526cb4a2161 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq-derivations.nix @@ -0,0 +1,10 @@ +let + + drvA1 = derivation { name = "a"; builder = "/foo"; system = "i686-linux"; }; + drvA2 = derivation { name = "a"; builder = "/foo"; system = "i686-linux"; }; + drvA3 = derivation { name = "a"; builder = "/foo"; system = "i686-linux"; } // { dummy = 1; }; + + drvC1 = derivation { name = "c"; builder = "/foo"; system = "i686-linux"; }; + drvC2 = derivation { name = "c"; builder = "/bar"; system = "i686-linux"; }; + +in [ (drvA1 == drvA1) (drvA1 == drvA2) (drvA1 == drvA3) (drvC1 == drvC2) ] diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.exp b/third_party/nix/src/tests/lang/eval-okay-eq.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-eq.nix b/third_party/nix/src/tests/lang/eval-okay-eq.nix new file mode 100644 index 000000000000..73d200b38141 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-eq.nix @@ -0,0 +1,3 @@ +["foobar" (rec {x = 1; y = x;})] +== +[("foo" + "bar") ({x = 1; y = 1;})] diff --git a/third_party/nix/src/tests/lang/eval-okay-filter.exp b/third_party/nix/src/tests/lang/eval-okay-filter.exp new file mode 100644 index 000000000000..355d51c27d8f --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-filter.exp @@ -0,0 +1 @@ +[ 0 2 4 6 8 10 100 102 104 106 108 110 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-filter.nix b/third_party/nix/src/tests/lang/eval-okay-filter.nix new file mode 100644 index 000000000000..85109b0d0eb8 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-filter.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +builtins.filter + (x: x / 2 * 2 == x) + (builtins.concatLists [ (range 0 10) (range 100 110) ]) diff --git a/third_party/nix/src/tests/lang/eval-okay-flatten.exp b/third_party/nix/src/tests/lang/eval-okay-flatten.exp new file mode 100644 index 000000000000..b979b2b8b9bc --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-flatten.exp @@ -0,0 +1 @@ +"1234567" diff --git a/third_party/nix/src/tests/lang/eval-okay-flatten.nix b/third_party/nix/src/tests/lang/eval-okay-flatten.nix new file mode 100644 index 000000000000..fe911e9683e2 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-flatten.nix @@ -0,0 +1,8 @@ +with import ./lib.nix; + +let { + + l = ["1" "2" ["3" ["4"] ["5" "6"]] "7"]; + + body = concat (flatten l); +} diff --git a/third_party/nix/src/tests/lang/eval-okay-float.exp b/third_party/nix/src/tests/lang/eval-okay-float.exp new file mode 100644 index 000000000000..3c50a8adce86 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-float.exp @@ -0,0 +1 @@ +[ 3.4 3.5 2.5 1.5 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-float.nix b/third_party/nix/src/tests/lang/eval-okay-float.nix new file mode 100644 index 000000000000..b2702c7b1668 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-float.nix @@ -0,0 +1,6 @@ +[ + (1.1 + 2.3) + (builtins.add (0.5 + 0.5) (2.0 + 0.5)) + ((0.5 + 0.5) * (2.0 + 0.5)) + ((1.5 + 1.5) / (0.5 * 4.0)) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-fromTOML.exp b/third_party/nix/src/tests/lang/eval-okay-fromTOML.exp new file mode 100644 index 000000000000..d0dd3af2c814 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-fromTOML.exp @@ -0,0 +1 @@ +[ { clients = { data = [ [ "gamma" "delta" ] [ 1 2 ] ]; hosts = [ "alpha" "omega" ]; }; database = { connection_max = 5000; enabled = true; ports = [ 8001 8001 8002 ]; server = "192.168.1.1"; }; owner = { name = "Tom Preston-Werner"; }; servers = { alpha = { dc = "eqdc10"; ip = "10.0.0.1"; }; beta = { dc = "eqdc10"; ip = "10.0.0.2"; }; }; title = "TOML Example"; } { "1234" = "value"; "127.0.0.1" = "value"; a = { b = { c = { }; }; }; arr1 = [ 1 2 3 ]; arr2 = [ "red" "yellow" "green" ]; arr3 = [ [ 1 2 ] [ 3 4 5 ] ]; arr4 = [ "all" "strings" "are the same" "type" ]; arr5 = [ [ 1 2 ] [ "a" "b" "c" ] ]; arr7 = [ 1 2 3 ]; arr8 = [ 1 2 ]; bare-key = "value"; bare_key = "value"; bin1 = 214; bool1 = true; bool2 = false; "character encoding" = "value"; d = { e = { f = { }; }; }; dog = { "tater.man" = { type = { name = "pug"; }; }; }; flt1 = 1; flt2 = 3.1415; flt3 = -0.01; flt4 = 5e+22; flt5 = 1e+06; flt6 = -0.02; flt7 = 6.626e-34; flt8 = 9.22462e+06; fruit = [ { name = "apple"; physical = { color = "red"; shape = "round"; }; variety = [ { name = "red delicious"; } { name = "granny smith"; } ]; } { name = "banana"; variety = [ { name = "plantain"; } ]; } ]; g = { h = { i = { }; }; }; hex1 = 3735928559; hex2 = 3735928559; hex3 = 3735928559; int1 = 99; int2 = 42; int3 = 0; int4 = -17; int5 = 1000; int6 = 5349221; int7 = 12345; j = { "ʞ" = { l = { }; }; }; key = "value"; key2 = "value"; name = "Orange"; oct1 = 342391; oct2 = 493; physical = { color = "orange"; shape = "round"; }; products = [ { name = "Hammer"; sku = 738594937; } { } { color = "gray"; name = "Nail"; sku = 284758393; } ]; "quoted \"value\"" = "value"; site = { "google.com" = true; }; str = "I'm a string. \"You can quote me\". Name\tJosé\nLocation\tSF."; table-1 = { key1 = "some string"; key2 = 123; }; table-2 = { key1 = "another string"; key2 = 456; }; x = { y = { z = { w = { animal = { type = { name = "pug"; }; }; name = { first = "Tom"; last = "Preston-Werner"; }; point = { x = 1; y = 2; }; }; }; }; }; "ʎǝʞ" = "value"; } { metadata = { "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"; "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"; "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6"; "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"; }; package = [ { dependencies = [ "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "aho-corasick"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.6.4"; } { name = "ansi_term"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.9.0"; } { dependencies = [ "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)" "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" ]; name = "atty"; source = "registry+https://github.com/rust-lang/crates.io-index"; version = "0.2.10"; } ]; } { a = [ [ { b = true; } ] ]; c = [ [ { d = true; } ] ]; e = [ [ 123 ] ]; } ] diff --git a/third_party/nix/src/tests/lang/eval-okay-fromTOML.nix b/third_party/nix/src/tests/lang/eval-okay-fromTOML.nix new file mode 100644 index 000000000000..963932689942 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-fromTOML.nix @@ -0,0 +1,208 @@ +[ + + (builtins.fromTOML '' + # This is a TOML document. + + title = "TOML Example" + + [owner] + name = "Tom Preston-Werner" + #dob = 1979-05-27T07:32:00-08:00 # First class dates + + [database] + server = "192.168.1.1" + ports = [ 8001, 8001, 8002 ] + connection_max = 5000 + enabled = true + + [servers] + + # Indentation (tabs and/or spaces) is allowed but not required + [servers.alpha] + ip = "10.0.0.1" + dc = "eqdc10" + + [servers.beta] + ip = "10.0.0.2" + dc = "eqdc10" + + [clients] + data = [ ["gamma", "delta"], [1, 2] ] + + # Line breaks are OK when inside arrays + hosts = [ + "alpha", + "omega" + ] + '') + + (builtins.fromTOML '' + key = "value" + bare_key = "value" + bare-key = "value" + 1234 = "value" + + "127.0.0.1" = "value" + "character encoding" = "value" + "ʎǝʞ" = "value" + 'key2' = "value" + 'quoted "value"' = "value" + + name = "Orange" + + physical.color = "orange" + physical.shape = "round" + site."google.com" = true + + # This is legal according to the spec, but cpptoml doesn't handle it. + #a.b.c = 1 + #a.d = 2 + + str = "I'm a string. \"You can quote me\". Name\tJos\u00E9\nLocation\tSF." + + int1 = +99 + int2 = 42 + int3 = 0 + int4 = -17 + int5 = 1_000 + int6 = 5_349_221 + int7 = 1_2_3_4_5 + + hex1 = 0xDEADBEEF + hex2 = 0xdeadbeef + hex3 = 0xdead_beef + + oct1 = 0o01234567 + oct2 = 0o755 + + bin1 = 0b11010110 + + flt1 = +1.0 + flt2 = 3.1415 + flt3 = -0.01 + flt4 = 5e+22 + flt5 = 1e6 + flt6 = -2E-2 + flt7 = 6.626e-34 + flt8 = 9_224_617.445_991_228_313 + + bool1 = true + bool2 = false + + # FIXME: not supported because Nix doesn't have a date/time type. + #odt1 = 1979-05-27T07:32:00Z + #odt2 = 1979-05-27T00:32:00-07:00 + #odt3 = 1979-05-27T00:32:00.999999-07:00 + #odt4 = 1979-05-27 07:32:00Z + #ldt1 = 1979-05-27T07:32:00 + #ldt2 = 1979-05-27T00:32:00.999999 + #ld1 = 1979-05-27 + #lt1 = 07:32:00 + #lt2 = 00:32:00.999999 + + arr1 = [ 1, 2, 3 ] + arr2 = [ "red", "yellow", "green" ] + arr3 = [ [ 1, 2 ], [3, 4, 5] ] + arr4 = [ "all", 'strings', """are the same""", ''''type''''] + arr5 = [ [ 1, 2 ], ["a", "b", "c"] ] + + arr7 = [ + 1, 2, 3 + ] + + arr8 = [ + 1, + 2, # this is ok + ] + + [table-1] + key1 = "some string" + key2 = 123 + + + [table-2] + key1 = "another string" + key2 = 456 + + [dog."tater.man"] + type.name = "pug" + + [a.b.c] + [ d.e.f ] + [ g . h . i ] + [ j . "ʞ" . 'l' ] + [x.y.z.w] + + name = { first = "Tom", last = "Preston-Werner" } + point = { x = 1, y = 2 } + animal = { type.name = "pug" } + + [[products]] + name = "Hammer" + sku = 738594937 + + [[products]] + + [[products]] + name = "Nail" + sku = 284758393 + color = "gray" + + [[fruit]] + name = "apple" + + [fruit.physical] + color = "red" + shape = "round" + + [[fruit.variety]] + name = "red delicious" + + [[fruit.variety]] + name = "granny smith" + + [[fruit]] + name = "banana" + + [[fruit.variety]] + name = "plantain" + '') + + (builtins.fromTOML '' + [[package]] + name = "aho-corasick" + version = "0.6.4" + source = "registry+https://github.com/rust-lang/crates.io-index" + dependencies = [ + "memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [[package]] + name = "ansi_term" + version = "0.9.0" + source = "registry+https://github.com/rust-lang/crates.io-index" + + [[package]] + name = "atty" + version = "0.2.10" + source = "registry+https://github.com/rust-lang/crates.io-index" + dependencies = [ + "libc 0.2.42 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + ] + + [metadata] + "checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4" + "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" + "checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" + "checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef" + '') + + (builtins.fromTOML '' + a = [[{ b = true }]] + c = [ [ { d = true } ] ] + e = [[123]] + '') + +] diff --git a/third_party/nix/src/tests/lang/eval-okay-fromjson.exp b/third_party/nix/src/tests/lang/eval-okay-fromjson.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-fromjson.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-fromjson.nix b/third_party/nix/src/tests/lang/eval-okay-fromjson.nix new file mode 100644 index 000000000000..102ee82b5e6b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-fromjson.nix @@ -0,0 +1,36 @@ +# RFC 7159, section 13. +builtins.fromJSON + '' + { + "Image": { + "Width": 800, + "Height": 600, + "Title": "View from 15th Floor", + "Thumbnail": { + "Url": "http://www.example.com/image/481989943", + "Height": 125, + "Width": 100 + }, + "Animated" : false, + "IDs": [116, 943, 234, 38793, true ,false,null, -100], + "Latitude": 37.7668, + "Longitude": -122.3959 + } + } + '' +== + { Image = + { Width = 800; + Height = 600; + Title = "View from 15th Floor"; + Thumbnail = + { Url = http://www.example.com/image/481989943; + Height = 125; + Width = 100; + }; + Animated = false; + IDs = [ 116 943 234 38793 true false null (0-100) ]; + Latitude = 37.7668; + Longitude = -122.3959; + }; + } diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.exp b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp new file mode 100644 index 000000000000..f8ddea8e0bfa --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.exp @@ -0,0 +1 @@ +[ { aterm = false; fetchurl = false; stdenv = false; } { color = false; name = true; } { } { } { } "stdenv" "fetchurl" "aterm-stdenv" "aterm-stdenv2" "libX11" "libXv" "mplayer-stdenv2.libXv-libX11" "mplayer-stdenv2.libXv-libX11_2" "nix-stdenv-aterm-stdenv" "nix-stdenv2-aterm2-stdenv2" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-functionargs.nix b/third_party/nix/src/tests/lang/eval-okay-functionargs.nix new file mode 100644 index 000000000000..205bbf307aa7 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-functionargs.nix @@ -0,0 +1,89 @@ +let + + stdenvFun = { }: { name = "stdenv"; }; + stdenv2Fun = { }: { name = "stdenv2"; }; + fetchurlFun = { stdenv }: assert stdenv.name == "stdenv"; { name = "fetchurl"; }; + atermFun = { stdenv, fetchurl }: { name = "aterm-${stdenv.name}"; }; + aterm2Fun = { stdenv, fetchurl }: { name = "aterm2-${stdenv.name}"; }; + nixFun = { stdenv, fetchurl, aterm }: { name = "nix-${stdenv.name}-${aterm.name}"; }; + + trivialFunctionArgsUsage = [ + (builtins.functionArgs nixFun) + (builtins.functionArgs ({ name ? "Karl", color }: "${name} is ${color}")) + (builtins.functionArgs (x: y: x + y)) + (builtins.functionArgs builtins.map) + (builtins.functionArgs builtins.fetchurl) + ]; + + mplayerFun = + { stdenv, fetchurl, enableX11 ? false, xorg ? null, enableFoo ? true, foo ? null }: + assert stdenv.name == "stdenv2"; + assert enableX11 -> xorg.libXv.name == "libXv"; + assert enableFoo -> foo != null; + { name = "mplayer-${stdenv.name}.${xorg.libXv.name}-${xorg.libX11.name}"; }; + + makeOverridable = f: origArgs: f origArgs // + { override = newArgs: + makeOverridable f (origArgs // (if builtins.isFunction newArgs then newArgs origArgs else newArgs)); + }; + + callPackage_ = pkgs: f: args: + makeOverridable f ((builtins.intersectAttrs (builtins.functionArgs f) pkgs) // args); + + allPackages = + { overrides ? (pkgs: pkgsPrev: { }) }: + let + callPackage = callPackage_ pkgs; + pkgs = pkgsStd // (overrides pkgs pkgsStd); + pkgsStd = { + inherit pkgs; + stdenv = callPackage stdenvFun { }; + stdenv2 = callPackage stdenv2Fun { }; + fetchurl = callPackage fetchurlFun { }; + aterm = callPackage atermFun { }; + xorg = callPackage xorgFun { }; + mplayer = callPackage mplayerFun { stdenv = pkgs.stdenv2; enableFoo = false; }; + nix = callPackage nixFun { }; + }; + in pkgs; + + libX11Fun = { stdenv, fetchurl }: { name = "libX11"; }; + libX11_2Fun = { stdenv, fetchurl }: { name = "libX11_2"; }; + libXvFun = { stdenv, fetchurl, libX11 }: { name = "libXv"; }; + + xorgFun = + { pkgs }: + let callPackage = callPackage_ (pkgs // pkgs.xorg); in + { + libX11 = callPackage libX11Fun { }; + libXv = callPackage libXvFun { }; + }; + +in + +let + + pkgs = allPackages { }; + + pkgs2 = allPackages { + overrides = pkgs: pkgsPrev: { + stdenv = pkgs.stdenv2; + nix = pkgsPrev.nix.override { aterm = aterm2Fun { inherit (pkgs) stdenv fetchurl; }; }; + xorg = pkgsPrev.xorg // { libX11 = libX11_2Fun { inherit (pkgs) stdenv fetchurl; }; }; + }; + }; + +in + + trivialFunctionArgsUsage ++ [ + pkgs.stdenv.name + pkgs.fetchurl.name + pkgs.aterm.name + pkgs2.aterm.name + pkgs.xorg.libX11.name + pkgs.xorg.libXv.name + pkgs.mplayer.name + pkgs2.mplayer.name + pkgs.nix.name + pkgs2.nix.name + ] diff --git a/third_party/nix/src/tests/lang/eval-okay-getattrpos-undefined.exp b/third_party/nix/src/tests/lang/eval-okay-getattrpos-undefined.exp new file mode 100644 index 000000000000..19765bd501b6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-getattrpos-undefined.exp @@ -0,0 +1 @@ +null diff --git a/third_party/nix/src/tests/lang/eval-okay-getattrpos-undefined.nix b/third_party/nix/src/tests/lang/eval-okay-getattrpos-undefined.nix new file mode 100644 index 000000000000..14dd38f7734c --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-getattrpos-undefined.nix @@ -0,0 +1 @@ +builtins.unsafeGetAttrPos "abort" builtins diff --git a/third_party/nix/src/tests/lang/eval-okay-getattrpos.exp b/third_party/nix/src/tests/lang/eval-okay-getattrpos.exp new file mode 100644 index 000000000000..469249bbc646 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-getattrpos.exp @@ -0,0 +1 @@ +{ column = 5; file = "eval-okay-getattrpos.nix"; line = 3; } diff --git a/third_party/nix/src/tests/lang/eval-okay-getattrpos.nix b/third_party/nix/src/tests/lang/eval-okay-getattrpos.nix new file mode 100644 index 000000000000..ca6b07961547 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-getattrpos.nix @@ -0,0 +1,6 @@ +let + as = { + foo = "bar"; + }; + pos = builtins.unsafeGetAttrPos "foo" as; +in { inherit (pos) column line; file = baseNameOf pos.file; } diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.exp b/third_party/nix/src/tests/lang/eval-okay-getenv.exp new file mode 100644 index 000000000000..14e24d419005 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-getenv.exp @@ -0,0 +1 @@ +"foobar" diff --git a/third_party/nix/src/tests/lang/eval-okay-getenv.nix b/third_party/nix/src/tests/lang/eval-okay-getenv.nix new file mode 100644 index 000000000000..ea8bb9f0a629 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-getenv.nix @@ -0,0 +1 @@ +builtins.getEnv "NIX_TEST_VAR" + (if builtins.getEnv "NO_SUCH_VAR" == "" then "bar" else "bla") diff --git a/third_party/nix/src/tests/lang/eval-okay-hash.exp b/third_party/nix/src/tests/lang/eval-okay-hash.exp new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-hash.exp diff --git a/third_party/nix/src/tests/lang/eval-okay-hashfile.exp b/third_party/nix/src/tests/lang/eval-okay-hashfile.exp new file mode 100644 index 000000000000..ff1e8293ef22 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-hashfile.exp @@ -0,0 +1 @@ +[ "d3b07384d113edec49eaa6238ad5ff00" "0f343b0931126a20f133d67c2b018a3b" "f1d2d2f924e986ac86fdf7b36c94bcdf32beec15" "60cacbf3d72e1e7834203da608037b1bf83b40e8" "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" "5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef" "0cf9180a764aba863a67b6d72f0918bc131c6772642cb2dce5a34f0a702f9470ddc2bf125c12198b1995c233c34b4afd346c54a2334c350a948a51b6e8b4e6b6" "8efb4f73c5655351c444eb109230c556d39e2c7624e9c11abc9e3fb4b9b9254218cc5085b454a9698d085cfa92198491f07a723be4574adc70617b73eb0b6461" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-hashfile.nix b/third_party/nix/src/tests/lang/eval-okay-hashfile.nix new file mode 100644 index 000000000000..aff5a1856814 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-hashfile.nix @@ -0,0 +1,4 @@ +let + paths = [ ./data ./binary-data ]; +in + builtins.concatLists (map (hash: map (builtins.hashFile hash) paths) ["md5" "sha1" "sha256" "sha512"]) diff --git a/third_party/nix/src/tests/lang/eval-okay-hashstring.exp b/third_party/nix/src/tests/lang/eval-okay-hashstring.exp new file mode 100644 index 000000000000..d720a082ddb3 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-hashstring.exp @@ -0,0 +1 @@ +[ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-hashstring.nix b/third_party/nix/src/tests/lang/eval-okay-hashstring.nix new file mode 100644 index 000000000000..b0f62b245ca8 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-hashstring.nix @@ -0,0 +1,4 @@ +let + strings = [ "" "text 1" "text 2" ]; +in + builtins.concatLists (map (hash: map (builtins.hashString hash) strings) ["md5" "sha1" "sha256" "sha512"]) diff --git a/third_party/nix/src/tests/lang/eval-okay-if.exp b/third_party/nix/src/tests/lang/eval-okay-if.exp new file mode 100644 index 000000000000..00750edc07d6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-if.exp @@ -0,0 +1 @@ +3 diff --git a/third_party/nix/src/tests/lang/eval-okay-if.nix b/third_party/nix/src/tests/lang/eval-okay-if.nix new file mode 100644 index 000000000000..23e4c74d5016 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-if.nix @@ -0,0 +1 @@ +if "foo" != "f" + "oo" then 1 else if false then 2 else 3 diff --git a/third_party/nix/src/tests/lang/eval-okay-import.exp b/third_party/nix/src/tests/lang/eval-okay-import.exp new file mode 100644 index 000000000000..c508125b55be --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-import.exp @@ -0,0 +1 @@ +[ 1 2 3 4 5 6 7 8 9 10 ] diff --git a/third_party/nix/src/tests/lang/eval-okay-import.nix b/third_party/nix/src/tests/lang/eval-okay-import.nix new file mode 100644 index 000000000000..0b18d9413122 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-import.nix @@ -0,0 +1,11 @@ +let + + overrides = { + import = fn: scopedImport overrides fn; + + scopedImport = attrs: fn: scopedImport (overrides // attrs) fn; + + builtins = builtins // overrides; + } // import ./lib.nix; + +in scopedImport overrides ./imported.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-ind-string.exp b/third_party/nix/src/tests/lang/eval-okay-ind-string.exp new file mode 100644 index 000000000000..9cf4bd2ee78a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-ind-string.exp @@ -0,0 +1 @@ +"This is an indented multi-line string\nliteral. An amount of whitespace at\nthe start of each line matching the minimum\nindentation of all lines in the string\nliteral together will be removed. Thus,\nin this case four spaces will be\nstripped from each line, even though\n THIS LINE is indented six spaces.\n\nAlso, empty lines don't count in the\ndetermination of the indentation level (the\nprevious empty line has indentation 0, but\nit doesn't matter).\nIf the string starts with whitespace\n followed by a newline, it's stripped, but\n that's not the case here. Two spaces are\n stripped because of the \" \" at the start. \nThis line is indented\na bit further.\nAnti-quotations, like so, are\nalso allowed.\n The \\ is not special here.\n' can be followed by any character except another ', e.g. 'x'.\nLikewise for $, e.g. $$ or $varName.\nBut ' followed by ' is special, as is $ followed by {.\nIf you want them, use anti-quotations: '', ${.\n Tabs are not interpreted as whitespace (since we can't guess\n what tab settings are intended), so don't use them.\n\tThis line starts with a space and a tab, so only one\n space will be stripped from each line.\nAlso note that if the last line (just before the closing ' ')\nconsists only of whitespace, it's ignored. But here there is\nsome non-whitespace stuff, so the line isn't removed. \nThis shows a hacky way to preserve an empty line after the start.\nBut there's no reason to do so: you could just repeat the empty\nline.\n Similarly you can force an indentation level,\n in this case to 2 spaces. This works because the anti-quote\n is significant (not whitespace).\nstart on network-interfaces\n\nstart script\n\n rm -f /var/run/opengl-driver\n ln -sf 123 /var/run/opengl-driver\n\n rm -f /var/log/slim.log\n \nend script\n\nenv SLIM_CFGFILE=abc\nenv SLIM_THEMESDIR=def\nenv FONTCONFIG_FILE=/etc/fonts/fonts.conf \t\t\t\t# !!! cleanup\nenv XKB_BINDIR=foo/bin \t\t\t\t# Needed for the Xkb extension.\nenv LD_LIBRARY_PATH=libX11/lib:libXext/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz)\n\nenv XORG_DRI_DRIVER_PATH=nvidiaDrivers/X11R6/lib/modules/drivers/ \n\nexec slim/bin/slim\nEscaping of ' followed by ': ''\nEscaping of $ followed by {: ${\nAnd finally to interpret \\n etc. as in a string: \n, \r, \t.\nfoo\n'bla'\nbar\ncut -d $'\\t' -f 1\nending dollar $$\n" diff --git a/third_party/nix/src/tests/lang/eval-okay-ind-string.nix b/third_party/nix/src/tests/lang/eval-okay-ind-string.nix new file mode 100644 index 000000000000..1669dc0648ea --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-ind-string.nix @@ -0,0 +1,128 @@ +let + + s1 = '' + This is an indented multi-line string + literal. An amount of whitespace at + the start of each line matching the minimum + indentation of all lines in the string + literal together will be removed. Thus, + in this case four spaces will be + stripped from each line, even though + THIS LINE is indented six spaces. + + Also, empty lines don't count in the + determination of the indentation level (the + previous empty line has indentation 0, but + it doesn't matter). + ''; + + s2 = '' If the string starts with whitespace + followed by a newline, it's stripped, but + that's not the case here. Two spaces are + stripped because of the " " at the start. + ''; + + s3 = '' + This line is indented + a bit further. + ''; # indentation of last line doesn't count if it's empty + + s4 = '' + Anti-quotations, like ${if true then "so" else "not so"}, are + also allowed. + ''; + + s5 = '' + The \ is not special here. + ' can be followed by any character except another ', e.g. 'x'. + Likewise for $, e.g. $$ or $varName. + But ' followed by ' is special, as is $ followed by {. + If you want them, use anti-quotations: ${"''"}, ${"\${"}. + ''; + + s6 = '' + Tabs are not interpreted as whitespace (since we can't guess + what tab settings are intended), so don't use them. + This line starts with a space and a tab, so only one + space will be stripped from each line. + ''; + + s7 = '' + Also note that if the last line (just before the closing ' ') + consists only of whitespace, it's ignored. But here there is + some non-whitespace stuff, so the line isn't removed. ''; + + s8 = '' ${""} + This shows a hacky way to preserve an empty line after the start. + But there's no reason to do so: you could just repeat the empty + line. + ''; + + s9 = '' + ${""} Similarly you can force an indentation level, + in this case to 2 spaces. This works because the anti-quote + is significant (not whitespace). + ''; + + s10 = '' + ''; + + s11 = ''''; + + s12 = '' ''; + + s13 = '' + start on network-interfaces + + start script + + rm -f /var/run/opengl-driver + ${if true + then "ln -sf 123 /var/run/opengl-driver" + else if true + then "ln -sf 456 /var/run/opengl-driver" + else "" + } + + rm -f /var/log/slim.log + + end script + + env SLIM_CFGFILE=${"abc"} + env SLIM_THEMESDIR=${"def"} + env FONTCONFIG_FILE=/etc/fonts/fonts.conf # !!! cleanup + env XKB_BINDIR=${"foo"}/bin # Needed for the Xkb extension. + env LD_LIBRARY_PATH=${"libX11"}/lib:${"libXext"}/lib:/usr/lib/ # related to xorg-sys-opengl - needed to load libglx for (AI)GLX support (for compiz) + + ${if true + then "env XORG_DRI_DRIVER_PATH=${"nvidiaDrivers"}/X11R6/lib/modules/drivers/" + else if true + then "env XORG_DRI_DRIVER_PATH=${"mesa"}/lib/modules/dri" + else "" + } + + exec ${"slim"}/bin/slim + ''; + + s14 = '' + Escaping of ' followed by ': ''' + Escaping of $ followed by {: ''${ + And finally to interpret \n etc. as in a string: ''\n, ''\r, ''\t. + ''; + + # Regression test: antiquotation in '${x}' should work, but didn't. + s15 = let x = "bla"; in '' + foo + '${x}' + bar + ''; + + # Regression test: accept $'. + s16 = '' + cut -d $'\t' -f 1 + ''; + + # Accept dollars at end of strings + s17 = ''ending dollar $'' + ''$'' + "\n"; + +in s1 + s2 + s3 + s4 + s5 + s6 + s7 + s8 + s9 + s10 + s11 + s12 + s13 + s14 + s15 + s16 + s17 diff --git a/third_party/nix/src/tests/lang/eval-okay-let.exp b/third_party/nix/src/tests/lang/eval-okay-let.exp new file mode 100644 index 000000000000..14e24d419005 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-let.exp @@ -0,0 +1 @@ +"foobar" diff --git a/third_party/nix/src/tests/lang/eval-okay-let.nix b/third_party/nix/src/tests/lang/eval-okay-let.nix new file mode 100644 index 000000000000..fe118c5282e3 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-let.nix @@ -0,0 +1,5 @@ +let { + x = "foo"; + y = "bar"; + body = x + y; +} diff --git a/third_party/nix/src/tests/lang/eval-okay-list.exp b/third_party/nix/src/tests/lang/eval-okay-list.exp new file mode 100644 index 000000000000..f784f26d83f4 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-list.exp @@ -0,0 +1 @@ +"foobarblatest" diff --git a/third_party/nix/src/tests/lang/eval-okay-list.nix b/third_party/nix/src/tests/lang/eval-okay-list.nix new file mode 100644 index 000000000000..d433bcf908ba --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-list.nix @@ -0,0 +1,7 @@ +with import ./lib.nix; + +let { + + body = concat ["foo" "bar" "bla" "test"]; + +} \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/eval-okay-listtoattrs.exp b/third_party/nix/src/tests/lang/eval-okay-listtoattrs.exp new file mode 100644 index 000000000000..74abef7bc6ed --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-listtoattrs.exp @@ -0,0 +1 @@ +"AAbar" diff --git a/third_party/nix/src/tests/lang/eval-okay-listtoattrs.nix b/third_party/nix/src/tests/lang/eval-okay-listtoattrs.nix new file mode 100644 index 000000000000..4186e029b538 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-listtoattrs.nix @@ -0,0 +1,11 @@ +# this test shows how to use listToAttrs and that evaluation is still lazy (throw isn't called) +with import ./lib.nix; + +let + asi = name: value : { inherit name value; }; + list = [ ( asi "a" "A" ) ( asi "b" "B" ) ]; + a = builtins.listToAttrs list; + b = builtins.listToAttrs ( list ++ list ); + r = builtins.listToAttrs [ (asi "result" [ a b ]) ( asi "throw" (throw "this should not be thrown")) ]; + x = builtins.listToAttrs [ (asi "foo" "bar") (asi "foo" "bla") ]; +in concat (map (x: x.a) r.result) + x.foo diff --git a/third_party/nix/src/tests/lang/eval-okay-logic.exp b/third_party/nix/src/tests/lang/eval-okay-logic.exp new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-logic.exp @@ -0,0 +1 @@ +1 diff --git a/third_party/nix/src/tests/lang/eval-okay-logic.nix b/third_party/nix/src/tests/lang/eval-okay-logic.nix new file mode 100644 index 000000000000..fbb12794401f --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-logic.nix @@ -0,0 +1 @@ +assert !false && (true || false) -> true; 1 diff --git a/third_party/nix/src/tests/lang/eval-okay-map.exp b/third_party/nix/src/tests/lang/eval-okay-map.exp new file mode 100644 index 000000000000..dbb64f717b96 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-map.exp @@ -0,0 +1 @@ +"foobarblabarxyzzybar" diff --git a/third_party/nix/src/tests/lang/eval-okay-map.nix b/third_party/nix/src/tests/lang/eval-okay-map.nix new file mode 100644 index 000000000000..a76c1d811454 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-map.nix @@ -0,0 +1,3 @@ +with import ./lib.nix; + +concat (map (x: x + "bar") [ "foo" "bla" "xyzzy" ]) \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/eval-okay-mapattrs.exp b/third_party/nix/src/tests/lang/eval-okay-mapattrs.exp new file mode 100644 index 000000000000..3f113f17bab1 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-mapattrs.exp @@ -0,0 +1 @@ +{ x = "x-foo"; y = "y-bar"; } diff --git a/third_party/nix/src/tests/lang/eval-okay-mapattrs.nix b/third_party/nix/src/tests/lang/eval-okay-mapattrs.nix new file mode 100644 index 000000000000..f075b6275e5a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-mapattrs.nix @@ -0,0 +1,3 @@ +with import ./lib.nix; + +builtins.mapAttrs (name: value: name + "-" + value) { x = "foo"; y = "bar"; } diff --git a/third_party/nix/src/tests/lang/eval-okay-nested-with.exp b/third_party/nix/src/tests/lang/eval-okay-nested-with.exp new file mode 100644 index 000000000000..0cfbf08886fc --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-nested-with.exp @@ -0,0 +1 @@ +2 diff --git a/third_party/nix/src/tests/lang/eval-okay-nested-with.nix b/third_party/nix/src/tests/lang/eval-okay-nested-with.nix new file mode 100644 index 000000000000..ba9d79aa79b1 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-nested-with.nix @@ -0,0 +1,3 @@ +with { x = 1; }; +with { x = 2; }; +x diff --git a/third_party/nix/src/tests/lang/eval-okay-new-let.exp b/third_party/nix/src/tests/lang/eval-okay-new-let.exp new file mode 100644 index 000000000000..f98b388071c2 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-new-let.exp @@ -0,0 +1 @@ +"xyzzyfoobar" diff --git a/third_party/nix/src/tests/lang/eval-okay-new-let.nix b/third_party/nix/src/tests/lang/eval-okay-new-let.nix new file mode 100644 index 000000000000..738123141508 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-new-let.nix @@ -0,0 +1,14 @@ +let + + f = z: + + let + x = "foo"; + y = "bar"; + body = 1; # compat test + in + z + x + y; + + arg = "xyzzy"; + +in f arg diff --git a/third_party/nix/src/tests/lang/eval-okay-null-dynamic-attrs.exp b/third_party/nix/src/tests/lang/eval-okay-null-dynamic-attrs.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-null-dynamic-attrs.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-null-dynamic-attrs.nix b/third_party/nix/src/tests/lang/eval-okay-null-dynamic-attrs.nix new file mode 100644 index 000000000000..b060c0bc9850 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-null-dynamic-attrs.nix @@ -0,0 +1 @@ +{ ${null} = true; } == {} diff --git a/third_party/nix/src/tests/lang/eval-okay-partition.exp b/third_party/nix/src/tests/lang/eval-okay-partition.exp new file mode 100644 index 000000000000..cd8b8b020c05 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-partition.exp @@ -0,0 +1 @@ +{ right = [ 0 2 4 6 8 10 100 102 104 106 108 110 ]; wrong = [ 1 3 5 7 9 101 103 105 107 109 ]; } diff --git a/third_party/nix/src/tests/lang/eval-okay-partition.nix b/third_party/nix/src/tests/lang/eval-okay-partition.nix new file mode 100644 index 000000000000..846d2ce49486 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-partition.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +builtins.partition + (x: x / 2 * 2 == x) + (builtins.concatLists [ (range 0 10) (range 100 110) ]) diff --git a/third_party/nix/src/tests/lang/eval-okay-pathexists.exp b/third_party/nix/src/tests/lang/eval-okay-pathexists.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-pathexists.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-pathexists.nix b/third_party/nix/src/tests/lang/eval-okay-pathexists.nix new file mode 100644 index 000000000000..50c28ee0cd30 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-pathexists.nix @@ -0,0 +1,5 @@ +builtins.pathExists (builtins.toPath ./lib.nix) +&& builtins.pathExists (builtins.toPath (builtins.toString ./lib.nix)) +&& !builtins.pathExists (builtins.toPath (builtins.toString ./bla.nix)) +&& builtins.pathExists ./lib.nix +&& !builtins.pathExists ./bla.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-patterns.exp b/third_party/nix/src/tests/lang/eval-okay-patterns.exp new file mode 100644 index 000000000000..a4304010fe80 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-patterns.exp @@ -0,0 +1 @@ +"abcxyzDDDDEFijk" diff --git a/third_party/nix/src/tests/lang/eval-okay-patterns.nix b/third_party/nix/src/tests/lang/eval-okay-patterns.nix new file mode 100644 index 000000000000..96fd25a01517 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-patterns.nix @@ -0,0 +1,16 @@ +let + + f = args@{x, y, z}: x + args.y + z; + + g = {x, y, z}@args: f args; + + h = {x ? "d", y ? x, z ? args.x}@args: x + y + z; + + j = {x, y, z, ...}: x + y + z; + +in + f {x = "a"; y = "b"; z = "c";} + + g {x = "x"; y = "y"; z = "z";} + + h {x = "D";} + + h {x = "D"; y = "E"; z = "F";} + + j {x = "i"; y = "j"; z = "k"; bla = "bla"; foo = "bar";} diff --git a/third_party/nix/src/tests/lang/eval-okay-readDir.exp b/third_party/nix/src/tests/lang/eval-okay-readDir.exp new file mode 100644 index 000000000000..bf8d2c14ea4f --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-readDir.exp @@ -0,0 +1 @@ +{ bar = "regular"; foo = "directory"; } diff --git a/third_party/nix/src/tests/lang/eval-okay-readDir.nix b/third_party/nix/src/tests/lang/eval-okay-readDir.nix new file mode 100644 index 000000000000..a7ec9292aae2 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-readDir.nix @@ -0,0 +1 @@ +builtins.readDir ./readDir diff --git a/third_party/nix/src/tests/lang/eval-okay-readfile.exp b/third_party/nix/src/tests/lang/eval-okay-readfile.exp new file mode 100644 index 000000000000..a2c87d0c439f --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-readfile.exp @@ -0,0 +1 @@ +"builtins.readFile ./eval-okay-readfile.nix\n" diff --git a/third_party/nix/src/tests/lang/eval-okay-readfile.nix b/third_party/nix/src/tests/lang/eval-okay-readfile.nix new file mode 100644 index 000000000000..82f7cb17435a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-readfile.nix @@ -0,0 +1 @@ +builtins.readFile ./eval-okay-readfile.nix diff --git a/third_party/nix/src/tests/lang/eval-okay-redefine-builtin.exp b/third_party/nix/src/tests/lang/eval-okay-redefine-builtin.exp new file mode 100644 index 000000000000..c508d5366f70 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-redefine-builtin.exp @@ -0,0 +1 @@ +false diff --git a/third_party/nix/src/tests/lang/eval-okay-redefine-builtin.nix b/third_party/nix/src/tests/lang/eval-okay-redefine-builtin.nix new file mode 100644 index 000000000000..df9fc3f37d22 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-redefine-builtin.nix @@ -0,0 +1,3 @@ +let + throw = abort "Error!"; +in (builtins.tryEval <foobaz>).success diff --git a/third_party/nix/src/tests/lang/eval-okay-regex-match.exp b/third_party/nix/src/tests/lang/eval-okay-regex-match.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-regex-match.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-regex-match.nix b/third_party/nix/src/tests/lang/eval-okay-regex-match.nix new file mode 100644 index 000000000000..273e2590713e --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-regex-match.nix @@ -0,0 +1,29 @@ +with builtins; + +let + + matches = pat: s: match pat s != null; + + splitFN = match "((.*)/)?([^/]*)\\.(nix|cc)"; + +in + +assert matches "foobar" "foobar"; +assert matches "fo*" "f"; +assert !matches "fo+" "f"; +assert matches "fo*" "fo"; +assert matches "fo*" "foo"; +assert matches "fo+" "foo"; +assert matches "fo{1,2}" "foo"; +assert !matches "fo{1,2}" "fooo"; +assert !matches "fo*" "foobar"; +assert matches "[[:space:]]+([^[:space:]]+)[[:space:]]+" " foo "; +assert !matches "[[:space:]]+([[:upper:]]+)[[:space:]]+" " foo "; + +assert match "(.*)\\.nix" "foobar.nix" == [ "foobar" ]; +assert match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " == [ "FOO" ]; + +assert splitFN "/path/to/foobar.nix" == [ "/path/to/" "/path/to" "foobar" "nix" ]; +assert splitFN "foobar.cc" == [ null null "foobar" "cc" ]; + +true diff --git a/third_party/nix/src/tests/lang/eval-okay-regex-split.exp b/third_party/nix/src/tests/lang/eval-okay-regex-split.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-regex-split.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-regex-split.nix b/third_party/nix/src/tests/lang/eval-okay-regex-split.nix new file mode 100644 index 000000000000..0073e057787d --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-regex-split.nix @@ -0,0 +1,48 @@ +with builtins; + +# Non capturing regex returns empty lists +assert split "foobar" "foobar" == ["" [] ""]; +assert split "fo*" "f" == ["" [] ""]; +assert split "fo+" "f" == ["f"]; +assert split "fo*" "fo" == ["" [] ""]; +assert split "fo*" "foo" == ["" [] ""]; +assert split "fo+" "foo" == ["" [] ""]; +assert split "fo{1,2}" "foo" == ["" [] ""]; +assert split "fo{1,2}" "fooo" == ["" [] "o"]; +assert split "fo*" "foobar" == ["" [] "bar"]; + +# Capturing regex returns a list of sub-matches +assert split "(fo*)" "f" == ["" ["f"] ""]; +assert split "(fo+)" "f" == ["f"]; +assert split "(fo*)" "fo" == ["" ["fo"] ""]; +assert split "(f)(o*)" "f" == ["" ["f" ""] ""]; +assert split "(f)(o*)" "foo" == ["" ["f" "oo"] ""]; +assert split "(fo+)" "foo" == ["" ["foo"] ""]; +assert split "(fo{1,2})" "foo" == ["" ["foo"] ""]; +assert split "(fo{1,2})" "fooo" == ["" ["foo"] "o"]; +assert split "(fo*)" "foobar" == ["" ["foo"] "bar"]; + +# Matches are greedy. +assert split "(o+)" "oooofoooo" == ["" ["oooo"] "f" ["oooo"] ""]; + +# Matches multiple times. +assert split "(b)" "foobarbaz" == ["foo" ["b"] "ar" ["b"] "az"]; + +# Split large strings containing newlines. null are inserted when a +# pattern within the current did not match anything. +assert split "[[:space:]]+|([',.!?])" '' + Nix Rocks! + That's why I use it. +'' == [ + "Nix" [ null ] "Rocks" ["!"] "" [ null ] + "That" ["'"] "s" [ null ] "why" [ null ] "I" [ null ] "use" [ null ] "it" ["."] "" [ null ] + "" +]; + +# Documentation examples +assert split "(a)b" "abc" == [ "" [ "a" ] "c" ]; +assert split "([ac])" "abc" == [ "" [ "a" ] "b" [ "c" ] "" ]; +assert split "(a)|(c)" "abc" == [ "" [ "a" null ] "b" [ null "c" ] "" ]; +assert split "([[:upper:]]+)" " FOO " == [ " " [ "FOO" ] " " ]; + +true diff --git a/third_party/nix/src/tests/lang/eval-okay-remove.exp b/third_party/nix/src/tests/lang/eval-okay-remove.exp new file mode 100644 index 000000000000..8d38505c1686 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-remove.exp @@ -0,0 +1 @@ +456 diff --git a/third_party/nix/src/tests/lang/eval-okay-remove.nix b/third_party/nix/src/tests/lang/eval-okay-remove.nix new file mode 100644 index 000000000000..4ad5ba897fa7 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-remove.nix @@ -0,0 +1,5 @@ +let { + attrs = {x = 123; y = 456;}; + + body = (removeAttrs attrs ["x"]).y; +} \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/eval-okay-replacestrings.exp b/third_party/nix/src/tests/lang/eval-okay-replacestrings.exp new file mode 100644 index 000000000000..72e8274d8c58 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-replacestrings.exp @@ -0,0 +1 @@ +[ "faabar" "fbar" "fubar" "faboor" "fubar" "XaXbXcX" "X" "a_b" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-replacestrings.nix b/third_party/nix/src/tests/lang/eval-okay-replacestrings.nix new file mode 100644 index 000000000000..bd8031fc004e --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-replacestrings.nix @@ -0,0 +1,11 @@ +with builtins; + +[ (replaceStrings ["o"] ["a"] "foobar") + (replaceStrings ["o"] [""] "foobar") + (replaceStrings ["oo"] ["u"] "foobar") + (replaceStrings ["oo" "a"] ["a" "oo"] "foobar") + (replaceStrings ["oo" "oo"] ["u" "i"] "foobar") + (replaceStrings [""] ["X"] "abc") + (replaceStrings [""] ["X"] "") + (replaceStrings ["-"] ["_"] "a-b") +] diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-1.exp b/third_party/nix/src/tests/lang/eval-okay-scope-1.exp new file mode 100644 index 000000000000..00750edc07d6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-1.exp @@ -0,0 +1 @@ +3 diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-1.nix b/third_party/nix/src/tests/lang/eval-okay-scope-1.nix new file mode 100644 index 000000000000..fa38a7174e03 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-1.nix @@ -0,0 +1,6 @@ +(({x}: x: + + { x = 1; + y = x; + } +) {x = 2;} 3).y diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-2.exp b/third_party/nix/src/tests/lang/eval-okay-scope-2.exp new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-2.exp @@ -0,0 +1 @@ +1 diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-2.nix b/third_party/nix/src/tests/lang/eval-okay-scope-2.nix new file mode 100644 index 000000000000..eb8b02bc4994 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-2.nix @@ -0,0 +1,6 @@ +((x: {x}: + rec { + x = 1; + y = x; + } +) 2 {x = 3;}).y diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-3.exp b/third_party/nix/src/tests/lang/eval-okay-scope-3.exp new file mode 100644 index 000000000000..b8626c4cff28 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-3.exp @@ -0,0 +1 @@ +4 diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-3.nix b/third_party/nix/src/tests/lang/eval-okay-scope-3.nix new file mode 100644 index 000000000000..10d6bc04d830 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-3.nix @@ -0,0 +1,6 @@ +((x: as: {x}: + rec { + inherit (as) x; + y = x; + } +) 2 {x = 4;} {x = 3;}).y diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-4.exp b/third_party/nix/src/tests/lang/eval-okay-scope-4.exp new file mode 100644 index 000000000000..00ff03a46c9b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-4.exp @@ -0,0 +1 @@ +"ccdd" diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-4.nix b/third_party/nix/src/tests/lang/eval-okay-scope-4.nix new file mode 100644 index 000000000000..dc8243bc8546 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-4.nix @@ -0,0 +1,10 @@ +let { + + x = "a"; + y = "b"; + + f = {x ? y, y ? x}: x + y; + + body = f {x = "c";} + f {y = "d";}; + +} diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-6.exp b/third_party/nix/src/tests/lang/eval-okay-scope-6.exp new file mode 100644 index 000000000000..00ff03a46c9b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-6.exp @@ -0,0 +1 @@ +"ccdd" diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-6.nix b/third_party/nix/src/tests/lang/eval-okay-scope-6.nix new file mode 100644 index 000000000000..0995d4e7e7e0 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-6.nix @@ -0,0 +1,7 @@ +let { + + f = {x ? y, y ? x}: x + y; + + body = f {x = "c";} + f {y = "d";}; + +} diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-7.exp b/third_party/nix/src/tests/lang/eval-okay-scope-7.exp new file mode 100644 index 000000000000..d00491fd7e5b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-7.exp @@ -0,0 +1 @@ +1 diff --git a/third_party/nix/src/tests/lang/eval-okay-scope-7.nix b/third_party/nix/src/tests/lang/eval-okay-scope-7.nix new file mode 100644 index 000000000000..4da02968f6b7 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-scope-7.nix @@ -0,0 +1,6 @@ +rec { + inherit (x) y; + x = { + y = 1; + }; +}.y diff --git a/third_party/nix/src/tests/lang/eval-okay-seq.exp b/third_party/nix/src/tests/lang/eval-okay-seq.exp new file mode 100644 index 000000000000..0cfbf08886fc --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-seq.exp @@ -0,0 +1 @@ +2 diff --git a/third_party/nix/src/tests/lang/eval-okay-seq.nix b/third_party/nix/src/tests/lang/eval-okay-seq.nix new file mode 100644 index 000000000000..0a9a21c03b62 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-seq.nix @@ -0,0 +1 @@ +builtins.seq 1 2 diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.exp b/third_party/nix/src/tests/lang/eval-okay-sort.exp new file mode 100644 index 000000000000..148b93516394 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-sort.exp @@ -0,0 +1 @@ +[ [ 42 77 147 249 483 526 ] [ 526 483 249 147 77 42 ] [ "bar" "fnord" "foo" "xyzzy" ] [ { key = 1; value = "foo"; } { key = 1; value = "fnord"; } { key = 2; value = "bar"; } ] ] diff --git a/third_party/nix/src/tests/lang/eval-okay-sort.nix b/third_party/nix/src/tests/lang/eval-okay-sort.nix new file mode 100644 index 000000000000..8299c3a4a3aa --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-sort.nix @@ -0,0 +1,8 @@ +with builtins; + +[ (sort lessThan [ 483 249 526 147 42 77 ]) + (sort (x: y: y < x) [ 483 249 526 147 42 77 ]) + (sort lessThan [ "foo" "bar" "xyzzy" "fnord" ]) + (sort (x: y: x.key < y.key) + [ { key = 1; value = "foo"; } { key = 2; value = "bar"; } { key = 1; value = "fnord"; } ]) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-splitversion.exp b/third_party/nix/src/tests/lang/eval-okay-splitversion.exp new file mode 100644 index 000000000000..153ceb8186a0 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-splitversion.exp @@ -0,0 +1 @@ +[ "1" "2" "3" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-splitversion.nix b/third_party/nix/src/tests/lang/eval-okay-splitversion.nix new file mode 100644 index 000000000000..9e5c99d2e7f6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-splitversion.nix @@ -0,0 +1 @@ +builtins.splitVersion "1.2.3" diff --git a/third_party/nix/src/tests/lang/eval-okay-string.exp b/third_party/nix/src/tests/lang/eval-okay-string.exp new file mode 100644 index 000000000000..63f650f73a3a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-string.exp @@ -0,0 +1 @@ +"foobar/a/b/c/d/foo/xyzzy/foo.txt/../foo/x/yescape: \"quote\" \n \\end\nof\nlinefoobarblaatfoo$bar$\"$\"$" diff --git a/third_party/nix/src/tests/lang/eval-okay-string.nix b/third_party/nix/src/tests/lang/eval-okay-string.nix new file mode 100644 index 000000000000..47cc989ad46a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-string.nix @@ -0,0 +1,12 @@ +"foo" + "bar" + + toString (/a/b + /c/d) + + toString (/foo/bar + "/../xyzzy/." + "/foo.txt") + + ("/../foo" + toString /x/y) + + "escape: \"quote\" \n \\" + + "end +of +line" + + "foo${if true then "b${"a" + "r"}" else "xyzzy"}blaat" + + "foo$bar" + + "$\"$\"" + + "$" diff --git a/third_party/nix/src/tests/lang/eval-okay-strings-as-attrs-names.exp b/third_party/nix/src/tests/lang/eval-okay-strings-as-attrs-names.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-strings-as-attrs-names.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-strings-as-attrs-names.nix b/third_party/nix/src/tests/lang/eval-okay-strings-as-attrs-names.nix new file mode 100644 index 000000000000..5e40928dbe31 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-strings-as-attrs-names.nix @@ -0,0 +1,20 @@ +let + + attr = { + "key 1" = "test"; + "key 2" = "caseok"; + }; + + t1 = builtins.getAttr "key 1" attr; + t2 = attr."key 2"; + t3 = attr ? "key 1"; + t4 = builtins.attrNames { inherit (attr) "key 1"; }; + + # This is permitted, but there is currently no way to reference this + # variable. + "foo bar" = 1; + +in t1 == "test" + && t2 == "caseok" + && t3 == true + && t4 == ["key 1"] diff --git a/third_party/nix/src/tests/lang/eval-okay-substring.exp b/third_party/nix/src/tests/lang/eval-okay-substring.exp new file mode 100644 index 000000000000..6aace04b0f57 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-substring.exp @@ -0,0 +1 @@ +"ooxfoobarybarzobaabbc" diff --git a/third_party/nix/src/tests/lang/eval-okay-substring.nix b/third_party/nix/src/tests/lang/eval-okay-substring.nix new file mode 100644 index 000000000000..424af00d9b3b --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-substring.nix @@ -0,0 +1,21 @@ +with builtins; + +let + + s = "foobar"; + +in + +substring 1 2 s ++ "x" ++ substring 0 (stringLength s) s ++ "y" ++ substring 3 100 s ++ "z" ++ substring 2 (sub (stringLength s) 3) s ++ "a" ++ substring 3 0 s ++ "b" ++ substring 3 1 s ++ "c" ++ substring 5 10 "perl" diff --git a/third_party/nix/src/tests/lang/eval-okay-tail-call-1.exp-disabled b/third_party/nix/src/tests/lang/eval-okay-tail-call-1.exp-disabled new file mode 100644 index 000000000000..f7393e847d34 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-tail-call-1.exp-disabled @@ -0,0 +1 @@ +100000 diff --git a/third_party/nix/src/tests/lang/eval-okay-tojson.exp b/third_party/nix/src/tests/lang/eval-okay-tojson.exp new file mode 100644 index 000000000000..e92aae3235f2 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-tojson.exp @@ -0,0 +1 @@ +"{\"a\":123,\"b\":-456,\"c\":\"foo\",\"d\":\"foo\\n\\\"bar\\\"\",\"e\":true,\"f\":false,\"g\":[1,2,3],\"h\":[\"a\",[\"b\",{\"foo\\nbar\":{}}]],\"i\":3,\"j\":1.44,\"k\":\"foo\"}" diff --git a/third_party/nix/src/tests/lang/eval-okay-tojson.nix b/third_party/nix/src/tests/lang/eval-okay-tojson.nix new file mode 100644 index 000000000000..ce67943bead5 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-tojson.nix @@ -0,0 +1,13 @@ +builtins.toJSON + { a = 123; + b = -456; + c = "foo"; + d = "foo\n\"bar\""; + e = true; + f = false; + g = [ 1 2 3 ]; + h = [ "a" [ "b" { "foo\nbar" = {}; } ] ]; + i = 1 + 2; + j = 1.44; + k = { __toString = self: self.a; a = "foo"; }; + } diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml2.exp b/third_party/nix/src/tests/lang/eval-okay-toxml2.exp new file mode 100644 index 000000000000..634a841eb190 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-toxml2.exp @@ -0,0 +1 @@ +"<?xml version='1.0' encoding='utf-8'?>\n<expr>\n <list>\n <string value=\"ab\" />\n <int value=\"10\" />\n <attrs>\n <attr name=\"x\">\n <string value=\"x\" />\n </attr>\n <attr name=\"y\">\n <string value=\"x\" />\n </attr>\n </attrs>\n </list>\n</expr>\n" diff --git a/third_party/nix/src/tests/lang/eval-okay-toxml2.nix b/third_party/nix/src/tests/lang/eval-okay-toxml2.nix new file mode 100644 index 000000000000..ff1791b30eb5 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-toxml2.nix @@ -0,0 +1 @@ +builtins.toXML [("a" + "b") 10 (rec {x = "x"; y = x;})] diff --git a/third_party/nix/src/tests/lang/eval-okay-tryeval.exp b/third_party/nix/src/tests/lang/eval-okay-tryeval.exp new file mode 100644 index 000000000000..2b2e6fa711f4 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-tryeval.exp @@ -0,0 +1 @@ +{ x = { success = true; value = "x"; }; y = { success = false; value = false; }; z = { success = false; value = false; }; } diff --git a/third_party/nix/src/tests/lang/eval-okay-tryeval.nix b/third_party/nix/src/tests/lang/eval-okay-tryeval.nix new file mode 100644 index 000000000000..629bc440a85a --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-tryeval.nix @@ -0,0 +1,5 @@ +{ + x = builtins.tryEval "x"; + y = builtins.tryEval (assert false; "y"); + z = builtins.tryEval (throw "bla"); +} diff --git a/third_party/nix/src/tests/lang/eval-okay-types.exp b/third_party/nix/src/tests/lang/eval-okay-types.exp new file mode 100644 index 000000000000..882c16dbfe36 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-types.exp @@ -0,0 +1 @@ +[ true false true true false true false true false true true true true true true true true true true true false true true true false "int" "bool" "string" "null" "set" "list" "lambda" "lambda" "lambda" "lambda" ] diff --git a/third_party/nix/src/tests/lang/eval-okay-types.nix b/third_party/nix/src/tests/lang/eval-okay-types.nix new file mode 100644 index 000000000000..cc51d8cb7a25 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-types.nix @@ -0,0 +1,38 @@ +with builtins; + +[ (isNull null) + (isNull (x: x)) + (isFunction (x: x)) + (isFunction functionArgs) + (isFunction "fnord") + (isString ("foo" + "bar")) + (isString [ "x" ]) + (isInt (1 + 2)) + (isInt { x = 123; }) + (isInt (1 / 2)) + (isInt (1 + 1)) + (isInt (1 / 2)) + (isInt (1 * 2)) + (isInt (1 - 2)) + (isFloat (1.2)) + (isFloat (1 + 1.0)) + (isFloat (1 / 2.0)) + (isFloat (1 * 2.0)) + (isFloat (1 - 2.0)) + (isBool (true && false)) + (isBool null) + (isPath /nix/store) + (isPath ./.) + (isAttrs { x = 123; }) + (isAttrs null) + (typeOf (3 * 4)) + (typeOf true) + (typeOf "xyzzy") + (typeOf null) + (typeOf { x = 456; }) + (typeOf [ 1 2 3 ]) + (typeOf (x: x)) + (typeOf ((x: y: x) 1)) + (typeOf map) + (typeOf (map (x: x))) +] diff --git a/third_party/nix/src/tests/lang/eval-okay-versions.exp b/third_party/nix/src/tests/lang/eval-okay-versions.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-versions.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/eval-okay-versions.nix b/third_party/nix/src/tests/lang/eval-okay-versions.nix new file mode 100644 index 000000000000..e63c36586bb6 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-versions.nix @@ -0,0 +1,40 @@ +let + + name1 = "hello-1.0.2"; + name2 = "hello"; + name3 = "915resolution-0.5.2"; + name4 = "xf86-video-i810-1.7.4"; + + eq = 0; + lt = builtins.sub 0 1; + gt = 1; + + versionTest = v1: v2: expected: + let d1 = builtins.compareVersions v1 v2; + d2 = builtins.compareVersions v2 v1; + in d1 == builtins.sub 0 d2 && d1 == expected; + + tests = [ + ((builtins.parseDrvName name1).name == "hello") + ((builtins.parseDrvName name1).version == "1.0.2") + ((builtins.parseDrvName name2).name == "hello") + ((builtins.parseDrvName name2).version == "") + ((builtins.parseDrvName name3).name == "915resolution") + ((builtins.parseDrvName name3).version == "0.5.2") + ((builtins.parseDrvName name4).name == "xf86-video-i810") + ((builtins.parseDrvName name4).version == "1.7.4") + (versionTest "1.0" "2.3" lt) + (versionTest "2.1" "2.3" lt) + (versionTest "2.3" "2.3" eq) + (versionTest "2.5" "2.3" gt) + (versionTest "3.1" "2.3" gt) + (versionTest "2.3.1" "2.3" gt) + (versionTest "2.3.1" "2.3a" gt) + (versionTest "2.3pre1" "2.3" lt) + (versionTest "2.3pre3" "2.3pre12" lt) + (versionTest "2.3a" "2.3c" lt) + (versionTest "2.3pre1" "2.3c" lt) + (versionTest "2.3pre1" "2.3q" lt) + ]; + +in (import ./lib.nix).and tests diff --git a/third_party/nix/src/tests/lang/eval-okay-with.exp b/third_party/nix/src/tests/lang/eval-okay-with.exp new file mode 100644 index 000000000000..378c8dc80403 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-with.exp @@ -0,0 +1 @@ +"xyzzybarxyzzybar" diff --git a/third_party/nix/src/tests/lang/eval-okay-with.nix b/third_party/nix/src/tests/lang/eval-okay-with.nix new file mode 100644 index 000000000000..033e8d3aba57 --- /dev/null +++ b/third_party/nix/src/tests/lang/eval-okay-with.nix @@ -0,0 +1,19 @@ +let { + + a = "xyzzy"; + + as = { + a = "foo"; + b = "bar"; + }; + + bs = { + a = "bar"; + }; + + x = with as; a + b; + + y = with as; with bs; a + b; + + body = x + y; +} diff --git a/third_party/nix/src/tests/lang/evalargs-okay-autoargs.nix b/third_party/nix/src/tests/lang/evalargs-okay-autoargs.nix new file mode 100644 index 000000000000..815f51b1d67a --- /dev/null +++ b/third_party/nix/src/tests/lang/evalargs-okay-autoargs.nix @@ -0,0 +1,15 @@ +let + + foobar = "foobar"; + +in + +{ xyzzy2 ? xyzzy # mutually recursive args +, xyzzy ? "blaat" # will be overridden by --argstr +, fb ? foobar +, lib # will be set by --arg +}: + +{ + result = lib.concat [xyzzy xyzzy2 fb]; +} diff --git a/third_party/nix/src/tests/lang/evalstore-okay-autoargs.exp b/third_party/nix/src/tests/lang/evalstore-okay-autoargs.exp new file mode 100644 index 000000000000..7a8391786a09 --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-autoargs.exp @@ -0,0 +1 @@ +"xyzzy!xyzzy!foobar" diff --git a/third_party/nix/src/tests/lang/evalstore-okay-context-introspection.exp b/third_party/nix/src/tests/lang/evalstore-okay-context-introspection.exp new file mode 100644 index 000000000000..27ba77ddaf61 --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-context-introspection.exp @@ -0,0 +1 @@ +true diff --git a/third_party/nix/src/tests/lang/evalstore-okay-context-introspection.nix b/third_party/nix/src/tests/lang/evalstore-okay-context-introspection.nix new file mode 100644 index 000000000000..d11aad38c726 --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-context-introspection.nix @@ -0,0 +1,24 @@ +let + drv = derivation { + name = "fail"; + builder = "/bin/false"; + system = "x86_64-linux"; + outputs = [ "out" "foo" ]; + }; + + path = "${./evalstore-okay-context-introspection.nix}"; + + desired-context = { + "${builtins.unsafeDiscardStringContext path}" = { + path = true; + }; + "${builtins.unsafeDiscardStringContext drv.drvPath}" = { + outputs = [ "foo" "out" ]; + allOutputs = true; + }; + }; + + legit-context = builtins.getContext "${path}${drv.outPath}${drv.foo.outPath}${drv.drvPath}"; + + constructed-context = builtins.getContext (builtins.appendContext "" desired-context); +in legit-context == constructed-context diff --git a/third_party/nix/src/tests/lang/evalstore-okay-context.exp b/third_party/nix/src/tests/lang/evalstore-okay-context.exp new file mode 100644 index 000000000000..f8088f9e172a --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-context.exp @@ -0,0 +1 @@ +"foo evalstore-okay-context.nix bar" diff --git a/third_party/nix/src/tests/lang/evalstore-okay-context.nix b/third_party/nix/src/tests/lang/evalstore-okay-context.nix new file mode 100644 index 000000000000..90f82abe1c36 --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-context.nix @@ -0,0 +1,6 @@ +let s = "foo ${builtins.substring 33 100 (baseNameOf "${./evalstore-okay-context.nix}")} bar"; +in + if s != "foo evalstore-okay-context.nix bar" + then abort "context not discarded" + else builtins.unsafeDiscardStringContext s + diff --git a/third_party/nix/src/tests/lang/evalstore-okay-toxml.exp b/third_party/nix/src/tests/lang/evalstore-okay-toxml.exp new file mode 100644 index 000000000000..828220890ecd --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-toxml.exp @@ -0,0 +1 @@ +"<?xml version='1.0' encoding='utf-8'?>\n<expr>\n <attrs>\n <attr name=\"a\">\n <string value=\"s\" />\n </attr>\n </attrs>\n</expr>\n" diff --git a/third_party/nix/src/tests/lang/evalstore-okay-toxml.nix b/third_party/nix/src/tests/lang/evalstore-okay-toxml.nix new file mode 100644 index 000000000000..068c97a6c1b3 --- /dev/null +++ b/third_party/nix/src/tests/lang/evalstore-okay-toxml.nix @@ -0,0 +1,3 @@ +# Make sure the expected XML output is produced; in particular, make sure it +# doesn't contain source location information. +builtins.toXML { a = "s"; } diff --git a/third_party/nix/src/tests/lang/imported.nix b/third_party/nix/src/tests/lang/imported.nix new file mode 100644 index 000000000000..fb39ee4efacd --- /dev/null +++ b/third_party/nix/src/tests/lang/imported.nix @@ -0,0 +1,3 @@ +# The function ‘range’ comes from lib.nix and was added to the lexical +# scope by scopedImport. +range 1 5 ++ import ./imported2.nix diff --git a/third_party/nix/src/tests/lang/imported2.nix b/third_party/nix/src/tests/lang/imported2.nix new file mode 100644 index 000000000000..6d0a2992b732 --- /dev/null +++ b/third_party/nix/src/tests/lang/imported2.nix @@ -0,0 +1 @@ +range 6 10 diff --git a/third_party/nix/src/tests/lang/lib.nix b/third_party/nix/src/tests/lang/lib.nix new file mode 100644 index 000000000000..028a538314b7 --- /dev/null +++ b/third_party/nix/src/tests/lang/lib.nix @@ -0,0 +1,61 @@ +with builtins; + +rec { + + fold = op: nul: list: + if list == [] + then nul + else op (head list) (fold op nul (tail list)); + + concat = + fold (x: y: x + y) ""; + + and = fold (x: y: x && y) true; + + flatten = x: + if isList x + then fold (x: y: (flatten x) ++ y) [] x + else [x]; + + sum = foldl' (x: y: add x y) 0; + + hasSuffix = ext: fileName: + let lenFileName = stringLength fileName; + lenExt = stringLength ext; + in !(lessThan lenFileName lenExt) && + substring (sub lenFileName lenExt) lenFileName fileName == ext; + + # Split a list at the given position. + splitAt = pos: list: + if pos == 0 then {first = []; second = list;} else + if list == [] then {first = []; second = [];} else + let res = splitAt (sub pos 1) (tail list); + in {first = [(head list)] ++ res.first; second = res.second;}; + + # Stable merge sort. + sortBy = comp: list: + if lessThan 1 (length list) + then + let + split = splitAt (div (length list) 2) list; + first = sortBy comp split.first; + second = sortBy comp split.second; + in mergeLists comp first second + else list; + + mergeLists = comp: list1: list2: + if list1 == [] then list2 else + if list2 == [] then list1 else + if comp (head list2) (head list1) then [(head list2)] ++ mergeLists comp list1 (tail list2) else + [(head list1)] ++ mergeLists comp (tail list1) list2; + + id = x: x; + + const = x: y: x; + + range = first: last: + if first > last + then [] + else genList (n: first + n) (last - first + 1); + +} diff --git a/third_party/nix/src/tests/lang/parse-fail-dup-attrs-1.nix b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-1.nix new file mode 100644 index 000000000000..e590e8a04e5a --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-1.nix @@ -0,0 +1,5 @@ +{ + x = 123; + y = 456; + x = 789; +} diff --git a/third_party/nix/src/tests/lang/parse-fail-dup-attrs-2.nix b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-2.nix new file mode 100644 index 000000000000..864d9865e07d --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-2.nix @@ -0,0 +1,13 @@ +let { + + as = { + x = 123; + y = 456; + }; + + bs = { + x = 789; + inherit (as) x; + }; + +} diff --git a/third_party/nix/src/tests/lang/parse-fail-dup-attrs-3.nix b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-3.nix new file mode 100644 index 000000000000..114d19779f86 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-3.nix @@ -0,0 +1,13 @@ +let { + + as = { + x = 123; + y = 456; + }; + + bs = rec { + x = 789; + inherit (as) x; + }; + +} diff --git a/third_party/nix/src/tests/lang/parse-fail-dup-attrs-4.nix b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-4.nix new file mode 100644 index 000000000000..77417432b347 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-4.nix @@ -0,0 +1,4 @@ +{ + services.ssh.port = 22; + services.ssh.port = 23; +} diff --git a/third_party/nix/src/tests/lang/parse-fail-dup-attrs-7.nix b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-7.nix new file mode 100644 index 000000000000..bbc3eb08c0f6 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-dup-attrs-7.nix @@ -0,0 +1,9 @@ +rec { + + x = 1; + + as = { + inherit x; + inherit x; + }; +} \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/parse-fail-dup-formals.nix b/third_party/nix/src/tests/lang/parse-fail-dup-formals.nix new file mode 100644 index 000000000000..a0edd91a9666 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-dup-formals.nix @@ -0,0 +1 @@ +{x, y, x}: x \ No newline at end of file diff --git a/third_party/nix/src/tests/lang/parse-fail-mixed-nested-attrs1.nix b/third_party/nix/src/tests/lang/parse-fail-mixed-nested-attrs1.nix new file mode 100644 index 000000000000..11e40e66fd1b --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-mixed-nested-attrs1.nix @@ -0,0 +1,4 @@ +{ + x.z = 3; + x = { y = 3; z = 3; }; +} diff --git a/third_party/nix/src/tests/lang/parse-fail-mixed-nested-attrs2.nix b/third_party/nix/src/tests/lang/parse-fail-mixed-nested-attrs2.nix new file mode 100644 index 000000000000..17da82e5f0c7 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-mixed-nested-attrs2.nix @@ -0,0 +1,4 @@ +{ + x.y.y = 3; + x = { y.y= 3; z = 3; }; +} diff --git a/third_party/nix/src/tests/lang/parse-fail-path-slash.nix b/third_party/nix/src/tests/lang/parse-fail-path-slash.nix new file mode 100644 index 000000000000..8c2e104c788f --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-path-slash.nix @@ -0,0 +1,6 @@ +# Trailing slashes in paths are not allowed. +# This restriction could be lifted sometime, +# for example if we make '/' a path concatenation operator. +# See https://github.com/NixOS/nix/issues/1138 +# and https://nixos.org/nix-dev/2016-June/020829.html +/nix/store/ diff --git a/third_party/nix/src/tests/lang/parse-fail-patterns-1.nix b/third_party/nix/src/tests/lang/parse-fail-patterns-1.nix new file mode 100644 index 000000000000..7b40616417b8 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-patterns-1.nix @@ -0,0 +1 @@ +args@{args, x, y, z}: x diff --git a/third_party/nix/src/tests/lang/parse-fail-regression-20060610.nix b/third_party/nix/src/tests/lang/parse-fail-regression-20060610.nix new file mode 100644 index 000000000000..b1934f7e1e82 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-regression-20060610.nix @@ -0,0 +1,11 @@ +let { + x = + {gcc}: + { + inherit gcc; + }; + + body = ({ + inherit gcc; + }).gcc; +} diff --git a/third_party/nix/src/tests/lang/parse-fail-uft8.nix b/third_party/nix/src/tests/lang/parse-fail-uft8.nix new file mode 100644 index 000000000000..34948d48aed2 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-uft8.nix @@ -0,0 +1 @@ +123 é 4 diff --git a/third_party/nix/src/tests/lang/parse-fail-undef-var-2.nix b/third_party/nix/src/tests/lang/parse-fail-undef-var-2.nix new file mode 100644 index 000000000000..c10a52b1ea42 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-undef-var-2.nix @@ -0,0 +1,7 @@ +let { + + f = {x, y : ["baz" "bar" z "bat"]}: x + y; + + body = f {x = "foo"; y = "bar";}; + +} diff --git a/third_party/nix/src/tests/lang/parse-fail-undef-var.nix b/third_party/nix/src/tests/lang/parse-fail-undef-var.nix new file mode 100644 index 000000000000..7b63008110db --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-fail-undef-var.nix @@ -0,0 +1 @@ +x: y diff --git a/third_party/nix/src/tests/lang/parse-okay-1.nix b/third_party/nix/src/tests/lang/parse-okay-1.nix new file mode 100644 index 000000000000..23a58ed109b1 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-1.nix @@ -0,0 +1 @@ +{x, y, z}: x + y + z diff --git a/third_party/nix/src/tests/lang/parse-okay-crlf.nix b/third_party/nix/src/tests/lang/parse-okay-crlf.nix new file mode 100644 index 000000000000..21518d4c6d80 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-crlf.nix @@ -0,0 +1,17 @@ +rec { + + /* Dit is + een test. */ + + x = + # Dit is een test. y; + + y = 123; + + # CR or CR/LF (but not explicit \r's) in strings should be + # translated to LF. + foo = "multi line + string + test\r"; + + z = 456; } diff --git a/third_party/nix/src/tests/lang/parse-okay-dup-attrs-5.nix b/third_party/nix/src/tests/lang/parse-okay-dup-attrs-5.nix new file mode 100644 index 000000000000..f4b9efd0c596 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-dup-attrs-5.nix @@ -0,0 +1,4 @@ +{ + services.ssh = { enable = true; }; + services.ssh.port = 23; +} diff --git a/third_party/nix/src/tests/lang/parse-okay-dup-attrs-6.nix b/third_party/nix/src/tests/lang/parse-okay-dup-attrs-6.nix new file mode 100644 index 000000000000..ae6d7a769305 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-dup-attrs-6.nix @@ -0,0 +1,4 @@ +{ + services.ssh.port = 23; + services.ssh = { enable = true; }; +} diff --git a/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-1.nix b/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-1.nix new file mode 100644 index 000000000000..fd1001c8cafc --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-1.nix @@ -0,0 +1,4 @@ +{ + x = { y = 3; z = 3; }; + x.q = 3; +} diff --git a/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-2.nix b/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-2.nix new file mode 100644 index 000000000000..ad066b680384 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-2.nix @@ -0,0 +1,4 @@ +{ + x.q = 3; + x = { y = 3; z = 3; }; +} diff --git a/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-3.nix b/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-3.nix new file mode 100644 index 000000000000..45a33e480373 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-mixed-nested-attrs-3.nix @@ -0,0 +1,7 @@ +{ + services.ssh.enable = true; + services.ssh = { port = 123; }; + services = { + httpd.enable = true; + }; +} diff --git a/third_party/nix/src/tests/lang/parse-okay-regression-20041027.nix b/third_party/nix/src/tests/lang/parse-okay-regression-20041027.nix new file mode 100644 index 000000000000..ae2e256eeaaa --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-regression-20041027.nix @@ -0,0 +1,11 @@ +{stdenv, fetchurl /* pkgconfig, libX11 */ }: + +stdenv.mkDerivation { + name = "libXi-6.0.1"; + src = fetchurl { + url = http://freedesktop.org/~xlibs/release/libXi-6.0.1.tar.bz2; + md5 = "7e935a42428d63a387b3c048be0f2756"; + }; +/* buildInputs = [pkgconfig]; + propagatedBuildInputs = [libX11]; */ +} diff --git a/third_party/nix/src/tests/lang/parse-okay-regression-751.nix b/third_party/nix/src/tests/lang/parse-okay-regression-751.nix new file mode 100644 index 000000000000..05c78b3016d3 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-regression-751.nix @@ -0,0 +1,2 @@ +let const = a: "const"; in +''${ const { x = "q"; }}'' diff --git a/third_party/nix/src/tests/lang/parse-okay-subversion.nix b/third_party/nix/src/tests/lang/parse-okay-subversion.nix new file mode 100644 index 000000000000..356272815d26 --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-subversion.nix @@ -0,0 +1,43 @@ +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert expat != null; +assert localServer -> db4 != null; +assert httpServer -> httpd != null && httpd.expat == expat; +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + + builder = /foo/bar; + src = fetchurl { + url = http://subversion.tigris.org/tarballs/subversion-1.1.1.tar.bz2; + md5 = "a180c3fe91680389c210c99def54d9e0"; + }; + + # This is a hopefully temporary fix for the problem that + # libsvnjavahl.so isn't linked against libstdc++, which causes + # loading the library into the JVM to fail. + patches = if javahlBindings then [/javahl.patch] else []; + + openssl = if sslSupport then openssl else null; + httpd = if httpServer then httpd else null; + db4 = if localServer then db4 else null; + swig = if pythonBindings || javaSwigBindings then swig else null; + python = if pythonBindings then swig.python else null; + j2sdk = if javaSwigBindings then swig.j2sdk else + if javahlBindings then j2sdk else null; + + inherit expat localServer httpServer sslSupport + pythonBindings javaSwigBindings javahlBindings; +} diff --git a/third_party/nix/src/tests/lang/parse-okay-url.nix b/third_party/nix/src/tests/lang/parse-okay-url.nix new file mode 100644 index 000000000000..fce3b13ee64b --- /dev/null +++ b/third_party/nix/src/tests/lang/parse-okay-url.nix @@ -0,0 +1,7 @@ +[ x:x + https://svn.cs.uu.nl:12443/repos/trace/trunk + http://www2.mplayerhq.hu/MPlayer/releases/fonts/font-arial-iso-8859-1.tar.bz2 + http://losser.st-lab.cs.uu.nl/~armijn/.nix/gcc-3.3.4-static-nix.tar.gz + http://fpdownload.macromedia.com/get/shockwave/flash/english/linux/7.0r25/install_flash_player_7_linux.tar.gz + ftp://ftp.gtk.org/pub/gtk/v1.2/gtk+-1.2.10.tar.gz +] diff --git a/third_party/nix/src/tests/lang/readDir/bar b/third_party/nix/src/tests/lang/readDir/bar new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/third_party/nix/src/tests/lang/readDir/bar diff --git a/third_party/nix/src/tests/lang/readDir/foo/git-hates-directories b/third_party/nix/src/tests/lang/readDir/foo/git-hates-directories new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/third_party/nix/src/tests/lang/readDir/foo/git-hates-directories diff --git a/third_party/nix/src/tests/language-tests.cc b/third_party/nix/src/tests/language-tests.cc new file mode 100644 index 000000000000..46e8c6ea80f7 --- /dev/null +++ b/third_party/nix/src/tests/language-tests.cc @@ -0,0 +1,290 @@ +// This file defines the language test suite. Language tests are run +// by evaluating a small snippet of Nix code (against a fake store), +// serialising it to a string and comparing that to a known output. +// +// This test suite is a port of the previous language integration test +// suite, and it's previous structure is retained. +// +// Test cases are written in nix files under lang/, following one of +// four possible filename patterns which trigger different behaviours: +// +// 1. parse-fail-*.nix: These files contain expressions which should +// cause a parser failure. +// +// 2. parse-okay-*.nix: These files contain expressions which should +// parse fine. +// +// 3. eval-fail-*.nix: These files contain expressions which should +// parse, but fail to evaluate. +// +// 4. eval-okay-*.nix: These files contain expressions which should +// parse and evaluate fine. They have accompanying .exp files which +// contain the expected string representation of the evaluation. + +#include <algorithm> +#include <filesystem> +#include <fstream> +#include <iostream> +#include <iterator> +#include <memory> +#include <optional> +#include <sstream> +#include <string> + +#include <absl/strings/ascii.h> +#include <absl/strings/match.h> +#include <absl/strings/str_cat.h> +#include <absl/strings/str_split.h> +#include <absl/strings/string_view.h> +#include <glog/logging.h> +#include <gtest/gtest-param-test.h> +#include <gtest/gtest.h> +#include <gtest/internal/gtest-param-util.h> + +#include "libexpr/eval-inline.hh" +#include "libexpr/eval.hh" +#include "libexpr/nixexpr.hh" +#include "nix_config.h" +#include "tests/dummy-store.hh" +#include "tests/store-util.hh" + +namespace nix::tests { +namespace { + +// List all the language test .nix files matching the given prefix. +std::vector<std::filesystem::path> TestFilesFor(absl::string_view prefix) { + std::vector<std::filesystem::path> matching_files; + + auto dir_iter = + std::filesystem::directory_iterator(NIX_SRC_DIR "/src/tests/lang"); + + for (auto& entry : dir_iter) { + if (!entry.is_regular_file()) { + continue; + } + + auto filename = entry.path().filename().string(); + if (absl::StartsWith(filename, prefix) && + absl::EndsWith(filename, ".nix")) { + matching_files.push_back(entry.path()); + } + } + + std::sort(matching_files.begin(), matching_files.end()); + return matching_files; +} + +// Construct a test name from a path parameter, re-casing its name to +// PascalCase. Googletest only accepts alphanumeric test-names, but +// the file names are in kebab-case. +std::string TestNameFor( + const testing::TestParamInfo<std::filesystem::path>& info) { + std::string name; + + for (auto part : + absl::StrSplit(info.param.stem().string(), '-', absl::SkipEmpty())) { + std::string part_owned(part); + part_owned[0] = absl::ascii_toupper(part_owned[0]); + absl::StrAppend(&name, part_owned); + } + + return name; +} + +// Load the expected output of a given test as a string. +std::string ExpectedOutputFor(absl::string_view stem) { + std::filesystem::path path( + absl::StrCat(NIX_SRC_DIR, "/src/tests/lang/", stem, ".exp")); + + EXPECT_TRUE(std::filesystem::exists(path)) + << stem << ": expected output file should exist"; + + std::ifstream input(path); + std::stringstream buffer; + buffer << input.rdbuf(); + return std::string(absl::StripTrailingAsciiWhitespace(buffer.str())); +} + +} // namespace + +using nix::tests::DummyStore; + +class NixEnvironment : public testing::Environment { + public: + void SetUp() override { + google::InitGoogleLogging("--logtostderr=false"); + nix::expr::InitGC(); + } +}; + +::testing::Environment* const nix_env = + ::testing::AddGlobalTestEnvironment(new NixEnvironment); + +class ParserFailureTest : public testing::TestWithParam<std::filesystem::path> { +}; + +// Test pattern for files that should fail to parse. +TEST_P(ParserFailureTest, Fails) { + std::shared_ptr<Store> store = std::make_shared<DummyStore>(); + EvalState state({}, ref<Store>(store)); + auto path = GetParam(); + + // There are multiple types of exceptions that the parser can throw, + // and the tests don't define which one they expect, so we need to + // allow all of these - but fail on other errors. + try { + state.parseExprFromFile(GetParam().string()); + FAIL() << path.stem().string() << ": parsing should not succeed"; + } catch (ParseError e) { + SUCCEED(); + } catch (UndefinedVarError e) { + SUCCEED(); + } catch (const std::exception& e) { + FAIL() << path.stem().string() + << ": unexpected parser exception: " << e.what(); + } +} + +INSTANTIATE_TEST_SUITE_P(Parser, ParserFailureTest, + testing::ValuesIn(TestFilesFor("parse-fail-")), + TestNameFor); + +class ParserSuccessTest : public testing::TestWithParam<std::filesystem::path> { +}; + +// Test pattern for files that should parse successfully. +TEST_P(ParserSuccessTest, Parses) { + std::shared_ptr<Store> store = std::make_shared<DummyStore>(); + EvalState state({}, ref<Store>(store)); + auto path = GetParam(); + + EXPECT_NO_THROW(state.parseExprFromFile(GetParam().string())) + << path.stem().string() << ": parsing should succeed"; + + SUCCEED(); +} + +INSTANTIATE_TEST_SUITE_P(Parser, ParserSuccessTest, + testing::ValuesIn(TestFilesFor("parse-okay-")), + TestNameFor); + +class EvalFailureTest : public testing::TestWithParam<std::filesystem::path> {}; + +// Test pattern for files that should fail to evaluate. +TEST_P(EvalFailureTest, Fails) { + std::shared_ptr<Store> store = std::make_shared<DummyStore>(); + EvalState state({}, ref<Store>(store)); + auto path = GetParam(); + + Expr* expr = nullptr; + EXPECT_NO_THROW(expr = state.parseExprFromFile(GetParam().string())) + << path.stem().string() << ": should parse successfully"; + + // Again, there are multiple expected exception types and the tests + // don't specify which ones they are looking for. + try { + Value result; + state.eval(expr, result); + state.forceValue(result); + std::cout << result; + FAIL() << path.stem().string() << ": evaluating should not succeed"; + } catch (AssertionError e) { + SUCCEED(); + } catch (EvalError e) { + SUCCEED(); + } catch (SysError e) { + SUCCEED(); + } catch (ParseError /* sic! */ e) { + SUCCEED(); + } catch (const std::exception& e) { + FAIL() << path.stem().string() + << ": unexpected evaluator exception: " << e.what(); + } +} + +INSTANTIATE_TEST_SUITE_P(Eval, EvalFailureTest, + testing::ValuesIn(TestFilesFor("eval-fail-")), + TestNameFor); + +class EvalSuccessTest : public testing::TestWithParam<std::filesystem::path> {}; + +// Test pattern for files that should evaluate successfully. +TEST_P(EvalSuccessTest, Succeeds) { + std::shared_ptr<Store> store = std::make_shared<DummyStore>(); + EvalState state({}, ref<Store>(store)); + auto path = GetParam(); + + Expr* expr = nullptr; + ASSERT_NO_THROW(expr = state.parseExprFromFile(GetParam().string())) + << path.stem().string() << ": should parse successfully"; + + Value result; + + ASSERT_NO_THROW({ + state.eval(expr, result); + state.forceValueDeep(result); + }) << path.stem().string() + << ": should evaluate successfully"; + + auto expected = ExpectedOutputFor(path.stem().string()); + std::ostringstream value_str; + value_str << result; + + EXPECT_EQ(expected, value_str.str()) << "evaluator output should match"; +} + +INSTANTIATE_TEST_SUITE_P(Eval, EvalSuccessTest, + testing::ValuesIn(TestFilesFor("eval-okay-")), + TestNameFor); + +class BlankStoreTest : public nix::StoreTest { + virtual void TestBody() override{}; +}; + +class EvalStoreSuccessTest + : public testing::TestWithParam<std::filesystem::path> { + public: + virtual void TearDown() { store_test_.TearDown(); } + + absl::StatusOr<std::unique_ptr<nix::LocalStore>> OpenTemporaryStore() { + return store_test_.OpenTemporaryStore(); + } + + private: + BlankStoreTest store_test_; +}; + +// Test pattern for files that should evaluate successfully but require a real +// store. +TEST_P(EvalStoreSuccessTest, Succeeds) { + absl::StatusOr<std::unique_ptr<nix::LocalStore>> store_ = + OpenTemporaryStore(); + CHECK(store_.ok()) << "failed to open temporary store"; + ref<Store> store = ref<Store>(store_->release()); + EvalState state({}, store); + auto path = GetParam(); + + Expr* expr = nullptr; + ASSERT_NO_THROW(expr = state.parseExprFromFile(GetParam().string())) + << path.stem().string() << ": should parse successfully"; + + Value result; + + ASSERT_NO_THROW({ + state.eval(expr, result); + state.forceValueDeep(result); + }) << path.stem().string() + << ": should evaluate successfully"; + + auto expected = ExpectedOutputFor(path.stem().string()); + std::ostringstream value_str; + value_str << result; + + EXPECT_EQ(expected, value_str.str()) << "evaluator output should match"; +} + +INSTANTIATE_TEST_SUITE_P(Eval, EvalStoreSuccessTest, + testing::ValuesIn(TestFilesFor("evalstore-okay-")), + TestNameFor); + +} // namespace nix::tests diff --git a/third_party/nix/src/tests/references_test.cc b/third_party/nix/src/tests/references_test.cc new file mode 100644 index 000000000000..8dcb3ed37a8b --- /dev/null +++ b/third_party/nix/src/tests/references_test.cc @@ -0,0 +1,74 @@ +#include "libstore/references.hh" + +#include <cstdio> +#include <fstream> +#include <ostream> +#include <unordered_set> + +#include <absl/strings/str_format.h> +#include <gtest/gtest.h> +#include <rapidcheck.h> +#include <rapidcheck/gtest.h> + +#include "libutil/hash.hh" + +class ReferencesTest : public ::testing::Test {}; + +namespace nix { + +TEST(ReferencesTest, ScanForOneReferenceNotFound) { + char path[] = "store_XXXXXXX"; + auto f = mkstemp(path); + + auto hash = hashString(htSHA256, "foo"); + auto ref = absl::StrFormat("/nix/store/%s-foo", hash.ToStorePathHash()); + + HashResult hr; + auto result = scanForReferences(path, {ref}, hr); + + ASSERT_EQ(result.find(ref), result.end()); + + EXPECT_EQ(close(f), 0); +} + +TEST(ReferencesTest, ScanForOneReferenceFound) { + char path[] = "store_XXXXXXX"; + auto f = mkstemp(path); + + auto hash = hashString(htSHA256, "foo"); + auto ref = absl::StrFormat("/nix/store/%s-foo", hash.ToStorePathHash()); + + EXPECT_GT(write(f, ref.c_str(), sizeof(char) * ref.size()), 0); + + HashResult hr; + auto result = scanForReferences(path, {ref}, hr); + + ASSERT_NE(result.find(ref), result.end()); + + ASSERT_EQ(close(f), 0); +} + +RC_GTEST_PROP(ReferencesTest, ScanForReferences, + (std::unordered_set<std::string> strs)) { + char path[] = "store_XXXXXXX"; + auto f = mkstemp(path); + + PathSet refs; + for (const auto& s : strs) { + auto hash = hashString(htSHA256, s); + auto ref = absl::StrFormat("/nix/store/%s-foo", hash.ToStorePathHash()); + refs.insert(ref); + RC_ASSERT(write(f, ref.c_str(), sizeof(char) * ref.size()) > 0); + } + + HashResult hr; + auto result = scanForReferences(path, refs, hr); + + for (const auto& ref : refs) { + RC_ASSERT(result.find(ref) != result.end()); + } + + RC_ASSERT(close(f) == 0); +} + +} // namespace nix diff --git a/third_party/nix/src/tests/status_helpers.h b/third_party/nix/src/tests/status_helpers.h new file mode 100644 index 000000000000..ca596dbb5254 --- /dev/null +++ b/third_party/nix/src/tests/status_helpers.h @@ -0,0 +1,83 @@ +#pragma once + +#include <absl/status/status.h> +#include <absl/status/statusor.h> +#include <absl/strings/str_cat.h> +#include <gmock/gmock.h> +#include <gtest/gtest.h> + +namespace testing { + +/* + * This file contains gtest matchers for absl::Status. + * + * Example usage: + * + * EXPECT_OK(status); -- fails the test if 'status' is an error + * ASSERT_OK(status); -- instantly fails the test if error + * + * using ::testing::IsStatusCode; + * EXPECT_THAT(status, IsStatusCode(absl::StatusCode::kInternal)); + */ + +namespace nix_internal { + +using ::testing::MakeMatcher; +using ::testing::Matcher; +using ::testing::MatcherInterface; +using ::testing::MatchResultListener; + +MATCHER_P(IsStatusCode, code, "") { return arg.code() == code; } + +class StatusCodeMatcher { + public: + StatusCodeMatcher(absl::StatusCode code) : code_(code) {} + + // Match on absl::Status. + template <class T, + typename std::enable_if<std::is_same<T, absl::Status>::value, + int>::type int_ = 0> + bool MatchAndExplain(const T& status, + MatchResultListener* /* listener */) const { + return status.code() == code_; + } + + // Match on absl::StatusOr. + // + // note: I check for the return value of ConsumeValueOrDie because it's the + // only non-overloaded member I could figure out how to select. Checking for + // the presence of .status() didn't work because it's overloaded, so + // std::invoke_result can't pick which overload to use. + template <class T, + typename std::enable_if< + std::is_same<typename std::invoke_result< + decltype(&T::ConsumeValueOrDie), T>::type, + typename T::value_type>::value, + int>::type int_ = 0> + bool MatchAndExplain(const T& statusor, + MatchResultListener* /* listener */) const { + return statusor.status().code() == code_; + } + + void DescribeTo(std::ostream* os) const { *os << "is " << code_; } + + void DescribeNegationTo(std::ostream* os) const { *os << "isn't " << code_; } + + private: + absl::StatusCode code_; +}; + +} // namespace nix_internal + +PolymorphicMatcher<nix_internal::StatusCodeMatcher> IsStatusCode( + absl::StatusCode code) { + return MakePolymorphicMatcher(nix_internal::StatusCodeMatcher(code)); +} + +#define EXPECT_OK(status) \ + EXPECT_THAT((status), testing::IsStatusCode(absl::StatusCode::kOk)) + +#define ASSERT_OK(status) \ + ASSERT_THAT((status), testing::IsStatusCode(absl::StatusCode::kOk)) + +} // namespace testing diff --git a/third_party/nix/src/tests/store-api-test.cc b/third_party/nix/src/tests/store-api-test.cc new file mode 100644 index 000000000000..259e4b991b57 --- /dev/null +++ b/third_party/nix/src/tests/store-api-test.cc @@ -0,0 +1,28 @@ +#include "libstore/store-api.hh" + +#include <gtest/gtest.h> +#include <rapidcheck/Assertions.h> +#include <rapidcheck/gtest.h> + +#include "libproto/worker.pb.h" +#include "tests/arbitrary.hh" + +namespace nix { + +class BuildResultTest : public ::testing::Test {}; + +RC_GTEST_PROP(BuildResultTest, StatusToFromProtoRoundTrip, + (BuildResult::Status && status)) { + BuildResult br; + br.status = status; + + auto proto_status = br.status_to_proto(); + nix::proto::BuildResult br_proto; + br_proto.set_status(proto_status); + + auto result = BuildResult::FromProto(br_proto); + + RC_ASSERT(result.value().status == status); +} + +} // namespace nix diff --git a/third_party/nix/src/tests/store-util.hh b/third_party/nix/src/tests/store-util.hh new file mode 100644 index 000000000000..b31bb0edcb8f --- /dev/null +++ b/third_party/nix/src/tests/store-util.hh @@ -0,0 +1,76 @@ +#pragma once + +#include <filesystem> + +#include <absl/status/statusor.h> +#include <absl/strings/escaping.h> +#include <glog/logging.h> +#include <sys/random.h> + +#include "libstore/local-store.hh" + +namespace nix { + +class StoreTest : public ::testing::Test { + public: + virtual void TearDown() { + for (auto fn : cleanup_funcs_) { + try { + fn(); + } catch (std::exception e) { + LOG(ERROR) << e.what(); + } + } + } + + absl::StatusOr<std::filesystem::path> OpenTempDir( + std::filesystem::path parent = std::filesystem::temp_directory_path()) { + for (;;) { + constexpr int kByteCnt = 9; + std::array<char, kByteCnt> randBytes; + if (getrandom(randBytes.data(), kByteCnt, 0) < 0) { + return absl::InternalError("getrandom() failed"); + } + std::string suffix = absl::WebSafeBase64Escape( + absl::string_view(randBytes.data(), kByteCnt)); + CHECK(suffix != ""); + + // Workaround for stdlib bug: use .assign() and ::errc + // https://stackoverflow.com/a/52401295/1210278 + std::error_code ec_exists; + ec_exists.assign(EEXIST, std::system_category()); + + std::error_code ec; + std::filesystem::path candidate = + parent / absl::StrCat("nixtest-", suffix); + if (std::filesystem::create_directory(candidate, ec)) { + cleanup_funcs_.push_back( + [candidate]() { std::filesystem::remove_all(candidate); }); + return candidate; + } else if (ec == ec_exists || ec == std::errc::file_exists) { + // Directory existed, retry + continue; + } else { + return absl::InternalError(absl::StrCat( + "could not create dir ", candidate.c_str(), ": ", ec.message())); + } + } + } + + absl::StatusOr<std::unique_ptr<nix::LocalStore>> OpenTemporaryStore() { + absl::StatusOr<std::filesystem::path> storePath = OpenTempDir(); + if (!storePath.ok()) { + return storePath.status(); + } + + nix::Store::Params params; + params["root"] = *storePath; + + return std::make_unique<nix::LocalStore>(params); + } + + private: + std::vector<std::function<void(void)>> cleanup_funcs_; +}; + +} // namespace nix diff --git a/third_party/nix/src/tests/store_tests.cc b/third_party/nix/src/tests/store_tests.cc new file mode 100644 index 000000000000..a6ffb715a90f --- /dev/null +++ b/third_party/nix/src/tests/store_tests.cc @@ -0,0 +1,122 @@ +#include <filesystem> + +#include <absl/container/btree_map.h> +#include <absl/container/flat_hash_map.h> +#include <absl/strings/escaping.h> +#include <glog/logging.h> +#include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <sys/random.h> + +#include "libstore/binary-cache-store.hh" +#include "libstore/mock-binary-cache-store.hh" +#include "tests/store-util.hh" + +using ::testing::HasSubstr; + +namespace nix { + +MakeError(InjectedError, Error); + +class BinaryCacheStoreTest : public StoreTest {}; + +constexpr absl::string_view kXZHeader = "7zXZ"; + +constexpr absl::string_view kRootFileName = "myRootFile"; +constexpr absl::string_view kDep1FileName = "dep1"; +constexpr absl::string_view kDep1FileContents = "==dep1 contents=="; +constexpr absl::string_view kDep1NarCache = + "nar/0hfdc95cy6mxi4c15pp0frdf97r7yvd8c141qzvpms2f8x17p2ig.nar.xz"; +constexpr absl::string_view kBogusPath = + "/nix/store/g1ghizdg18k0d00000000000000z3v32-doesNotExist"; + +struct TestTree { + Path rootPath; + Path dep1Path; +}; + +TestTree AddTestTreeToStore(Store& store) { + TestTree results; + results.rootPath = + store.addTextToStore(std::string(kRootFileName), "1", PathSet()); + + PathSet onlyRoot; + onlyRoot.insert(results.rootPath); + results.dep1Path = store.addTextToStore( + std::string(kDep1FileName), std::string(kDep1FileContents), onlyRoot); + + return results; +} + +TEST_F(BinaryCacheStoreTest, BasicStorage) { + MockBinaryCacheStore::Params params; + MockBinaryCacheStore store(params); + + store.init(); + + auto tree = AddTestTreeToStore(store); + + EXPECT_TRUE(store.isValidPath(tree.rootPath)); + EXPECT_TRUE(store.isValidPath(tree.dep1Path)); + + StringSink sink; + store.narFromPath(tree.dep1Path, sink); + EXPECT_THAT(*sink.s, HasSubstr(kDep1FileContents)); + + EXPECT_THAT(*store.BinaryCacheStore::getFile(Path(kDep1NarCache)), + HasSubstr(kXZHeader)); +} + +TEST_F(BinaryCacheStoreTest, BasicErrors) { + MockBinaryCacheStore::Params params; + MockBinaryCacheStore store(params); + + store.init(); + + auto tree = AddTestTreeToStore(store); + store.PrepareErrorInjection(std::string(kDep1NarCache), + []() { throw InjectedError("injected"); }); + + { + StringSink sink; + EXPECT_THROW(store.narFromPath(tree.dep1Path, sink), InjectedError); + } + { + StringSink sink; + EXPECT_THROW(store.narFromPath(std::string(kBogusPath), sink), + NoSuchBinaryCacheFile); + } +} + +// ./tests/add.sh +TEST_F(StoreTest, AddFileHashes) { + auto store_ = OpenTemporaryStore(); + CHECK(store_.ok()) << "failed to open temporary store"; + nix::Store* store = store_->release(); + nix::Path dataPath = NIX_SRC_DIR "/src/tests/lang/data"; + std::string dataName = "data"; + + nix::Path path1 = store->addToStore(dataName, dataPath); + + nix::Path path2 = store->addToStore(dataName, dataPath, /*recursive=*/true, + HashType::htSHA256); + + EXPECT_EQ(path1, path2) << "nix-store --add and --add-fixed mismatch"; + + nix::Path path3 = store->addToStore(dataName, dataPath, /*recursive=*/false, + HashType::htSHA256); + EXPECT_NE(path1, path3); + + nix::Path path4 = + store->addToStore(dataName, dataPath, false, HashType::htSHA1); + EXPECT_NE(path1, path4); + + auto info1 = store->queryPathInfo(store->followLinksToStorePath(path1)); + ASSERT_EQ(info1->narHash.type, HashType::htSHA256); + + Hash h = nix::hashPath(HashType::htSHA256, dataPath).first; + + EXPECT_EQ(info1->narHash.to_string(), h.to_string()); +} + +} // namespace nix diff --git a/third_party/nix/src/tests/value-to-json.cc b/third_party/nix/src/tests/value-to-json.cc new file mode 100644 index 000000000000..45427425306f --- /dev/null +++ b/third_party/nix/src/tests/value-to-json.cc @@ -0,0 +1,257 @@ +#include "libexpr/value-to-json.hh" + +#include <set> +#include <sstream> + +#include <gtest/gtest.h> + +#include "libexpr/value-to-xml.hh" +#include "libexpr/value.hh" +#include "libstore/store-api.hh" +#include "tests/dummy-store.hh" + +class ValueTest : public ::testing::Test { + protected: + static void SetUpTestCase() { nix::expr::InitGC(); } + + static void TearDownTestCase() {} +}; + +class JSONValueTest : public ValueTest {}; +class XMLValueTest : public ValueTest {}; + +namespace nix { + +using nix::tests::DummyStore; + +TEST_F(JSONValueTest, null) { + std::stringstream ss; + Value v; + PathSet ps; + std::shared_ptr<Store> store = std::make_shared<DummyStore>(); + EvalState s({}, ref<Store>(store)); + + mkNull(v); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "null"); +} + +TEST_F(JSONValueTest, BoolFalse) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkBool(v, false); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "false"); +} + +TEST_F(JSONValueTest, BoolTrue) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkBool(v, true); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "true"); +} + +TEST_F(JSONValueTest, IntPositive) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkInt(v, 100); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "100"); +} + +TEST_F(JSONValueTest, IntNegative) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkInt(v, -100); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "-100"); +} + +TEST_F(JSONValueTest, String) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkString(v, "test"); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "\"test\""); +} + +TEST_F(JSONValueTest, StringQuotes) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkString(v, "test\""); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "\"test\\\"\""); +} + +TEST_F(JSONValueTest, Path) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkPathNoCopy(v, "/exists-for-tests"); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); +} + +TEST_F(JSONValueTest, PathNoCopy) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkPathNoCopy(v, "/exists-for-tests"); + printValueAsJSON(s, true, v, ss, ps); + ASSERT_EQ(ss.str(), "\"/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x\""); +} + +/* + * Value to XMl tests + */ + +#define XML(v) \ + ("<?xml version='1.0' encoding='utf-8'?>\n<expr>\n" v "\n</expr>\n") + +TEST_F(XMLValueTest, null) { + std::stringstream ss; + Value v; + PathSet ps; + auto store = std::make_shared<DummyStore>(); + EvalState s({}, ref<Store>(store)); + + mkNull(v); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <null />")); +} + +TEST_F(XMLValueTest, BoolFalse) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkBool(v, false); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <bool value=\"false\" />")); +} + +TEST_F(XMLValueTest, BoolTrue) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkBool(v, true); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <bool value=\"true\" />")); +} + +TEST_F(XMLValueTest, IntPositive) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkInt(v, 100); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <int value=\"100\" />")); +} + +TEST_F(XMLValueTest, IntNegative) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkInt(v, -100); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <int value=\"-100\" />")); +} + +TEST_F(XMLValueTest, String) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkString(v, "test-value"); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <string value=\"test-value\" />")); +} + +TEST_F(XMLValueTest, StringQuotes) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkString(v, "test-value\""); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <string value=\"test-value"\" />")); +} + +/* + * FIXME: This function returns the original input path while the JSON version + * of the same actually touches the store to create a /nix/store path. + */ +TEST_F(XMLValueTest, Path) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkPath(v, "some-path"); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <path value=\"some-path\" />")); +} + +/* + * FIXME: This function returns the original input path while the JSON version + * of the same actually touches the store to create a /nix/store path. + */ +TEST_F(XMLValueTest, PathNoCopy) { + std::stringstream ss; + auto store = std::make_shared<DummyStore>(); + EvalState s({"."}, ref<Store>(store)); + Value v; + PathSet ps; + + mkPathNoCopy(v, "some-other-path"); + printValueAsXML(s, true, true, v, ss, ps); + ASSERT_EQ(ss.str(), XML(" <path value=\"some-other-path\" />")); +} +} // namespace nix diff --git a/third_party/nix/test-vm.nix b/third_party/nix/test-vm.nix new file mode 100644 index 000000000000..e5f8690fcb2d --- /dev/null +++ b/third_party/nix/test-vm.nix @@ -0,0 +1,19 @@ +{ depot, pkgs, ... }: + +let + configuration = { ... }: { + imports = [ + "${pkgs.path}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix" + ]; + + nix.package = depot.third_party.nix; + + virtualisation.qemu.options = [ "-nographic" "-curses" ]; + + nix.nixPath = [ + "depot=${depot.path}" + ]; + }; + + system = depot.third_party.nixos { inherit configuration; }; +in system.vm diff --git a/third_party/nix/tests/add.sh b/third_party/nix/tests/add.sh new file mode 100644 index 000000000000..e26e05843d7f --- /dev/null +++ b/third_party/nix/tests/add.sh @@ -0,0 +1,28 @@ +source common.sh + +path1=$(nix-store --add ./dummy) +echo $path1 + +path2=$(nix-store --add-fixed sha256 --recursive ./dummy) +echo $path2 + +if test "$path1" != "$path2"; then + echo "nix-store --add and --add-fixed mismatch" + exit 1 +fi + +path3=$(nix-store --add-fixed sha256 ./dummy) +echo $path3 +test "$path1" != "$path3" || exit 1 + +path4=$(nix-store --add-fixed sha1 --recursive ./dummy) +echo $path4 +test "$path1" != "$path4" || exit 1 + +hash1=$(nix-store -q --hash $path1) +echo $hash1 + +hash2=$(nix-hash --type sha256 --base32 ./dummy) +echo $hash2 + +test "$hash1" = "sha256:$hash2" diff --git a/third_party/nix/tests/binary-cache.sh b/third_party/nix/tests/binary-cache.sh new file mode 100644 index 000000000000..eb58ae7c12a8 --- /dev/null +++ b/third_party/nix/tests/binary-cache.sh @@ -0,0 +1,170 @@ +source common.sh + +clearStore +clearCache + +# Create the binary cache. +outPath=$(nix-build dependencies.nix --no-out-link) + +nix copy --to file://$cacheDir $outPath + + +basicTests() { + + # By default, a binary cache doesn't support "nix-env -qas", but does + # support installation. + clearStore + clearCacheCache + + nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" + + nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + + [ -x $outPath/program ] + + + # But with the right configuration, "nix-env -qas" should also work. + clearStore + clearCacheCache + echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info + + nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" + nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" + + x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) + [ -z "$x" ] + + nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + + nix-store --check-validity $outPath + nix-store -qR $outPath | grep input-2 + + echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info +} + + +# Test LocalBinaryCacheStore. +basicTests + + +# Test HttpBinaryCacheStore. +export _NIX_FORCE_HTTP_BINARY_CACHE_STORE=1 +basicTests + + +# Test whether Nix notices if the NAR doesn't match the hash in the NAR info. +clearStore + +nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) +mv $nar $nar.good +mkdir -p $TEST_ROOT/empty +nix-store --dump $TEST_ROOT/empty | xz > $nar + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grep -q "hash mismatch" $TEST_ROOT/log + +mv $nar.good $nar + + +# Test whether this unsigned cache is rejected if the user requires signed caches. +clearStore +clearCacheCache + +if nix-store --substituters "file://$cacheDir" -r $outPath; then + echo "unsigned binary cache incorrectly accepted" + exit 1 +fi + + +# Test whether fallback works if a NAR has disappeared. This does not require --fallback. +clearStore + +mv $cacheDir/nar $cacheDir/nar2 + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result + +mv $cacheDir/nar2 $cacheDir/nar + + +# Test whether fallback works if a NAR is corrupted. This does require --fallback. +clearStore + +mv $cacheDir/nar $cacheDir/nar2 +mkdir $cacheDir/nar +for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done + +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback + +rm -rf $cacheDir/nar +mv $cacheDir/nar2 $cacheDir/nar + + +# Test whether building works if the binary cache contains an +# incomplete closure. +clearStore + +rm $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) + +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log +grep -q "copying path" $TEST_ROOT/log + + +if [ -n "$HAVE_SODIUM" ]; then + +# Create a signed binary cache. +clearCache +clearCacheCache + +declare -a res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk1 $TEST_ROOT/pk1 )) +publicKey="$(cat $TEST_ROOT/pk1)" + +res=($(nix-store --generate-binary-cache-key test.nixos.org-1 $TEST_ROOT/sk2 $TEST_ROOT/pk2)) +badKey="$(cat $TEST_ROOT/pk2)" + +res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3)) +otherKey="$(cat $TEST_ROOT/pk3)" + +_NIX_FORCE_HTTP_BINARY_CACHE_STORE= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath + + +# Downloading should fail if we don't provide a key. +clearStore +clearCacheCache + +(! nix-store -r $outPath --substituters "file://$cacheDir") + + +# And it should fail if we provide an incorrect key. +clearStore +clearCacheCache + +(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey") + + +# It should succeed if we provide the correct key. +nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" + + +# It should fail if we corrupt the .narinfo. +clearStore + +cacheDir2=$TEST_ROOT/binary-cache-2 +rm -rf $cacheDir2 +cp -r $cacheDir $cacheDir2 + +for i in $cacheDir2/*.narinfo; do + grep -v References $i > $i.tmp + mv $i.tmp $i +done + +clearCacheCache + +(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") + +# If we provide a bad and a good binary cache, it should succeed. + +nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" + +fi # HAVE_LIBSODIUM diff --git a/third_party/nix/tests/brotli.sh b/third_party/nix/tests/brotli.sh new file mode 100644 index 000000000000..a3c6e55a8fad --- /dev/null +++ b/third_party/nix/tests/brotli.sh @@ -0,0 +1,21 @@ +source common.sh + +clearStore +clearCache + +cacheURI="file://$cacheDir?compression=br" + +outPath=$(nix-build dependencies.nix --no-out-link) + +nix copy --to $cacheURI $outPath + +HASH=$(nix hash-path $outPath) + +clearStore +clearCacheCache + +nix copy --from $cacheURI $outPath --no-check-sigs + +HASH2=$(nix hash-path $outPath) + +[[ $HASH = $HASH2 ]] diff --git a/third_party/nix/tests/build-dry.sh b/third_party/nix/tests/build-dry.sh new file mode 100644 index 000000000000..e72533e70614 --- /dev/null +++ b/third_party/nix/tests/build-dry.sh @@ -0,0 +1,52 @@ +source common.sh + +################################################### +# Check that --dry-run isn't confused with read-only mode +# https://github.com/NixOS/nix/issues/1795 + +clearStore +clearCache + +# Ensure this builds successfully first +nix build --no-link -f dependencies.nix + +clearStore +clearCache + +# Try --dry-run using old command first +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" +# Now new command: +nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" + +# TODO: XXX: FIXME: #1793 +# Disable this part of the test until the problem is resolved: +if [ -n "$ISSUE_1795_IS_FIXED" ]; then +clearStore +clearCache + +# Try --dry-run using new command first +nix build -f dependencies.nix --dry-run 2>&1 | grep "will be built" +# Now old command: +nix-build --no-out-link dependencies.nix --dry-run 2>&1 | grep "will be built" +fi + +################################################### +# Check --dry-run doesn't create links with --dry-run +# https://github.com/NixOS/nix/issues/1849 +clearStore +clearCache + +RESULT=$TEST_ROOT/result-link +rm -f $RESULT + +nix-build dependencies.nix -o $RESULT --dry-run + +[[ ! -h $RESULT ]] || fail "nix-build --dry-run created output link" + +nix build -f dependencies.nix -o $RESULT --dry-run + +[[ ! -h $RESULT ]] || fail "nix build --dry-run created output link" + +nix build -f dependencies.nix -o $RESULT + +[[ -h $RESULT ]] diff --git a/third_party/nix/tests/build-hook.nix b/third_party/nix/tests/build-hook.nix new file mode 100644 index 000000000000..8bff0fe79032 --- /dev/null +++ b/third_party/nix/tests/build-hook.nix @@ -0,0 +1,23 @@ +with import ./config.nix; + +let + + input1 = mkDerivation { + name = "build-hook-input-1"; + builder = ./dependencies.builder1.sh; + requiredSystemFeatures = ["foo"]; + }; + + input2 = mkDerivation { + name = "build-hook-input-2"; + builder = ./dependencies.builder2.sh; + }; + +in + + mkDerivation { + name = "build-hook"; + builder = ./dependencies.builder0.sh; + input1 = " " + input1 + "/."; + input2 = " ${input2}/."; + } diff --git a/third_party/nix/tests/build-remote.sh b/third_party/nix/tests/build-remote.sh new file mode 100644 index 000000000000..ddd68f327a15 --- /dev/null +++ b/third_party/nix/tests/build-remote.sh @@ -0,0 +1,24 @@ +source common.sh + +clearStore + +if ! canUseSandbox; then exit; fi +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +chmod -R u+w $TEST_ROOT/store1 || true +rm -rf $TEST_ROOT/store0 $TEST_ROOT/store1 + +nix build -f build-hook.nix -o $TEST_ROOT/result --max-jobs 0 \ + --sandbox-paths /nix/store --sandbox-build-dir /build-tmp \ + --builders "$TEST_ROOT/store0; $TEST_ROOT/store1 - - 1 1 foo" \ + --system-features foo + +outPath=$TEST_ROOT/result + +cat $outPath/foobar | grep FOOBAR + +# Ensure that input1 was built on store1 due to the required feature. +p=$(readlink -f $outPath/input-2) +(! nix path-info --store $TEST_ROOT/store0 --all | grep dependencies.builder1.sh) +nix path-info --store $TEST_ROOT/store1 --all | grep dependencies.builder1.sh diff --git a/third_party/nix/tests/case-hack.sh b/third_party/nix/tests/case-hack.sh new file mode 100644 index 000000000000..61bf9b94bf5c --- /dev/null +++ b/third_party/nix/tests/case-hack.sh @@ -0,0 +1,19 @@ +source common.sh + +clearStore + +rm -rf $TEST_ROOT/case + +opts="--option use-case-hack true" + +# Check whether restoring and dumping a NAR that contains case +# collisions is round-tripping, even on a case-insensitive system. +nix-store $opts --restore $TEST_ROOT/case < case.nar +nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar +cmp case.nar $TEST_ROOT/case.nar +[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] + +# Check whether we detect true collisions (e.g. those remaining after +# removal of the suffix). +touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" +(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) diff --git a/third_party/nix/tests/case.nar b/third_party/nix/tests/case.nar new file mode 100644 index 000000000000..22ff26db5afd --- /dev/null +++ b/third_party/nix/tests/case.nar Binary files differdiff --git a/third_party/nix/tests/check-refs.nix b/third_party/nix/tests/check-refs.nix new file mode 100644 index 000000000000..9d90b0920542 --- /dev/null +++ b/third_party/nix/tests/check-refs.nix @@ -0,0 +1,70 @@ +with import ./config.nix; + +rec { + + dep = import ./dependencies.nix; + + makeTest = nr: args: mkDerivation ({ + name = "check-refs-" + toString nr; + } // args); + + src = builtins.toFile "aux-ref" "bla bla"; + + test1 = makeTest 1 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $dep $out/link"; + inherit dep; + }; + + test2 = makeTest 2 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s ${src} $out/link"; + inherit dep; + }; + + test3 = makeTest 3 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $dep $out/link"; + allowedReferences = []; + inherit dep; + }; + + test4 = makeTest 4 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $dep $out/link"; + allowedReferences = [dep]; + inherit dep; + }; + + test5 = makeTest 5 { + builder = builtins.toFile "builder.sh" "mkdir $out"; + allowedReferences = []; + inherit dep; + }; + + test6 = makeTest 6 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $out $out/link"; + allowedReferences = []; + inherit dep; + }; + + test7 = makeTest 7 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $out $out/link"; + allowedReferences = ["out"]; + inherit dep; + }; + + test8 = makeTest 8 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s ${test1} $out/link"; + inherit dep; + }; + + test9 = makeTest 9 { + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $dep $out/link"; + inherit dep; + disallowedReferences = [dep]; + }; + + test10 = makeTest 10 { + builder = builtins.toFile "builder.sh" "mkdir $out; echo $test5; ln -s $dep $out/link"; + inherit dep test5; + disallowedReferences = [test5]; + }; + +} diff --git a/third_party/nix/tests/check-refs.sh b/third_party/nix/tests/check-refs.sh new file mode 100644 index 000000000000..16bbabc40985 --- /dev/null +++ b/third_party/nix/tests/check-refs.sh @@ -0,0 +1,42 @@ +source common.sh + +clearStore + +RESULT=$TEST_ROOT/result + +dep=$(nix-build -o $RESULT check-refs.nix -A dep) + +# test1 references dep, not itself. +test1=$(nix-build -o $RESULT check-refs.nix -A test1) +(! nix-store -q --references $test1 | grep -q $test1) +nix-store -q --references $test1 | grep -q $dep + +# test2 references src, not itself nor dep. +test2=$(nix-build -o $RESULT check-refs.nix -A test2) +(! nix-store -q --references $test2 | grep -q $test2) +(! nix-store -q --references $test2 | grep -q $dep) +nix-store -q --references $test2 | grep -q aux-ref + +# test3 should fail (unallowed ref). +(! nix-build -o $RESULT check-refs.nix -A test3) + +# test4 should succeed. +nix-build -o $RESULT check-refs.nix -A test4 + +# test5 should succeed. +nix-build -o $RESULT check-refs.nix -A test5 + +# test6 should fail (unallowed self-ref). +(! nix-build -o $RESULT check-refs.nix -A test6) + +# test7 should succeed (allowed self-ref). +nix-build -o $RESULT check-refs.nix -A test7 + +# test8 should fail (toFile depending on derivation output). +(! nix-build -o $RESULT check-refs.nix -A test8) + +# test9 should fail (disallowed reference). +(! nix-build -o $RESULT check-refs.nix -A test9) + +# test10 should succeed (no disallowed references). +nix-build -o $RESULT check-refs.nix -A test10 diff --git a/third_party/nix/tests/check-reqs.nix b/third_party/nix/tests/check-reqs.nix new file mode 100644 index 000000000000..41436cb48e08 --- /dev/null +++ b/third_party/nix/tests/check-reqs.nix @@ -0,0 +1,57 @@ +with import ./config.nix; + +rec { + dep1 = mkDerivation { + name = "check-reqs-dep1"; + builder = builtins.toFile "builder.sh" "mkdir $out; touch $out/file1"; + }; + + dep2 = mkDerivation { + name = "check-reqs-dep2"; + builder = builtins.toFile "builder.sh" "mkdir $out; touch $out/file2"; + }; + + deps = mkDerivation { + name = "check-reqs-deps"; + dep1 = dep1; + dep2 = dep2; + builder = builtins.toFile "builder.sh" '' + mkdir $out + ln -s $dep1/file1 $out/file1 + ln -s $dep2/file2 $out/file2 + ''; + }; + + makeTest = nr: allowreqs: mkDerivation { + name = "check-reqs-" + toString nr; + inherit deps; + builder = builtins.toFile "builder.sh" '' + mkdir $out + ln -s $deps $out/depdir1 + ''; + allowedRequisites = allowreqs; + }; + + # When specifying all the requisites, the build succeeds. + test1 = makeTest 1 [ dep1 dep2 deps ]; + + # But missing anything it fails. + test2 = makeTest 2 [ dep2 deps ]; + test3 = makeTest 3 [ dep1 deps ]; + test4 = makeTest 4 [ deps ]; + test5 = makeTest 5 []; + + test6 = mkDerivation { + name = "check-reqs"; + inherit deps; + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $deps $out/depdir1"; + disallowedRequisites = [dep1]; + }; + + test7 = mkDerivation { + name = "check-reqs"; + inherit deps; + builder = builtins.toFile "builder.sh" "mkdir $out; ln -s $deps $out/depdir1"; + disallowedRequisites = [test1]; + }; +} diff --git a/third_party/nix/tests/check-reqs.sh b/third_party/nix/tests/check-reqs.sh new file mode 100644 index 000000000000..e9f65fc2a6d3 --- /dev/null +++ b/third_party/nix/tests/check-reqs.sh @@ -0,0 +1,16 @@ +source common.sh + +clearStore + +RESULT=$TEST_ROOT/result + +nix-build -o $RESULT check-reqs.nix -A test1 + +(! nix-build -o $RESULT check-reqs.nix -A test2) +(! nix-build -o $RESULT check-reqs.nix -A test3) +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep1' +(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grep -q 'check-reqs-dep2' +(! nix-build -o $RESULT check-reqs.nix -A test5) +(! nix-build -o $RESULT check-reqs.nix -A test6) + +nix-build -o $RESULT check-reqs.nix -A test7 diff --git a/third_party/nix/tests/check.nix b/third_party/nix/tests/check.nix new file mode 100644 index 000000000000..56c82e565a8f --- /dev/null +++ b/third_party/nix/tests/check.nix @@ -0,0 +1,22 @@ +with import ./config.nix; + +{ + nondeterministic = mkDerivation { + name = "nondeterministic"; + buildCommand = + '' + mkdir $out + date +%s.%N > $out/date + ''; + }; + + hashmismatch = import <nix/fetchurl.nix> { + url = "file://" + toString ./dummy; + sha256 = "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73"; + }; + + fetchurl = import <nix/fetchurl.nix> { + url = "file://" + toString ./lang/eval-okay-xml.exp.xml; + sha256 = "0kg4sla7ihm8ijr8cb3117fhl99zrc2bwy1jrngsfmkh8bav4m0v"; + }; +} diff --git a/third_party/nix/tests/check.sh b/third_party/nix/tests/check.sh new file mode 100644 index 000000000000..bc23a6634ca0 --- /dev/null +++ b/third_party/nix/tests/check.sh @@ -0,0 +1,47 @@ +source common.sh + +clearStore + +nix-build dependencies.nix --no-out-link +nix-build dependencies.nix --no-out-link --check + +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic --no-out-link --check 2> $TEST_ROOT/log || status=$? +grep 'may not be deterministic' $TEST_ROOT/log +[ "$status" = "104" ] + +clearStore + +nix-build dependencies.nix --no-out-link --repeat 3 + +nix-build check.nix -A nondeterministic --no-out-link --repeat 1 2> $TEST_ROOT/log || status=$? +[ "$status" = "1" ] +grep 'differs from previous round' $TEST_ROOT/log + +path=$(nix-build check.nix -A fetchurl --no-out-link --hashed-mirrors '') + +chmod +w $path +echo foo > $path +chmod -w $path + +nix-build check.nix -A fetchurl --no-out-link --check --hashed-mirrors '' +# Note: "check" doesn't repair anything, it just compares to the hash stored in the database. +[[ $(cat $path) = foo ]] + +nix-build check.nix -A fetchurl --no-out-link --repair --hashed-mirrors '' +[[ $(cat $path) != foo ]] + +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +echo -n > ./dummy +nix-build check.nix -A hashmismatch --no-out-link --hashed-mirrors '' +echo 'Hello World' > ./dummy + +nix-build check.nix -A hashmismatch --no-out-link --check --hashed-mirrors '' || status=$? +[ "$status" = "102" ] + +# Multiple failures with --keep-going +nix-build check.nix -A nondeterministic --no-out-link +nix-build check.nix -A nondeterministic -A hashmismatch --no-out-link --check --keep-going --hashed-mirrors '' || status=$? +[ "$status" = "110" ] diff --git a/third_party/nix/tests/common.sh.in b/third_party/nix/tests/common.sh.in new file mode 100644 index 000000000000..15d7b1ef9119 --- /dev/null +++ b/third_party/nix/tests/common.sh.in @@ -0,0 +1,118 @@ +set -e + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test) +export NIX_STORE_DIR +if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store +fi +export NIX_LOCALSTATE_DIR=$TEST_ROOT/var +export NIX_LOG_DIR=$TEST_ROOT/var/log/nix +export NIX_STATE_DIR=$TEST_ROOT/var/nix +export NIX_CONF_DIR=$TEST_ROOT/etc +export _NIX_TEST_SHARED=$TEST_ROOT/shared +if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 +fi +export _NIX_IN_TEST=$TEST_ROOT/shared +export _NIX_TEST_NO_LSOF=1 +export NIX_REMOTE=$NIX_REMOTE_ +unset NIX_PATH +export TEST_HOME=$TEST_ROOT/test-home +export HOME=$TEST_HOME +unset XDG_CACHE_HOME +mkdir -p $TEST_HOME + +export PATH=@bindir@:$PATH +coreutils=@coreutils@ + +export dot=@dot@ +export xmllint="@xmllint@" +export SHELL="@bash@" +export PAGER=cat +export HAVE_SODIUM="@HAVE_SODIUM@" + +export version=@PACKAGE_VERSION@ +export system=@system@ + +cacheDir=$TEST_ROOT/binary-cache + +readLink() { + ls -l "$1" | sed 's/.*->\ //' +} + +clearProfiles() { + profiles="$NIX_STATE_DIR"/profiles + rm -rf $profiles +} + +clearStore() { + echo "clearing store..." + chmod -R +w "$NIX_STORE_DIR" + rm -rf "$NIX_STORE_DIR" + mkdir "$NIX_STORE_DIR" + rm -rf "$NIX_STATE_DIR" + mkdir "$NIX_STATE_DIR" + nix-store --init + clearProfiles +} + +clearCache() { + rm -rf "$cacheDir" +} + +clearCacheCache() { + rm -f $TEST_HOME/.cache/nix/binary-cache* +} + +startDaemon() { + # Start the daemon, wait for the socket to appear. !!! + # ‘nix-daemon’ should have an option to fork into the background. + rm -f $NIX_STATE_DIR/daemon-socket/socket + nix-daemon & + for ((i = 0; i < 30; i++)); do + if [ -e $NIX_STATE_DIR/daemon-socket/socket ]; then break; fi + sleep 1 + done + pidDaemon=$! + trap "kill -9 $pidDaemon" EXIT + export NIX_REMOTE=daemon +} + +killDaemon() { + kill -9 $pidDaemon + wait $pidDaemon || true + trap "" EXIT +} + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +canUseSandbox() { + if [[ ! $_canUseSandbox ]]; then + echo "Sandboxing not supported, skipping this test..." + return 1 + fi + + return 0 +} + +fail() { + echo "$1" + exit 1 +} + +expect() { + local expected res + expected="$1" + shift + set +e + "$@" + res="$?" + set -e + [[ $res -eq $expected ]] +} + +set -x diff --git a/third_party/nix/tests/config.nix b/third_party/nix/tests/config.nix new file mode 100644 index 000000000000..6ba91065b83d --- /dev/null +++ b/third_party/nix/tests/config.nix @@ -0,0 +1,20 @@ +with import <nix/config.nix>; + +rec { + inherit shell; + + path = coreutils; + + system = builtins.currentSystem; + + shared = builtins.getEnv "_NIX_TEST_SHARED"; + + mkDerivation = args: + derivation ({ + inherit system; + builder = shell; + args = ["-e" args.builder or (builtins.toFile "builder.sh" "if [ -e .attrs.sh ]; then source .attrs.sh; fi; eval \"$buildCommand\"")]; + PATH = path; + } // removeAttrs args ["builder" "meta"]) + // { meta = args.meta or {}; }; +} diff --git a/third_party/nix/tests/dependencies.builder0.sh b/third_party/nix/tests/dependencies.builder0.sh new file mode 100644 index 000000000000..c37bf909a5f9 --- /dev/null +++ b/third_party/nix/tests/dependencies.builder0.sh @@ -0,0 +1,16 @@ +[ "${input1: -2}" = /. ] +[ "${input2: -2}" = /. ] + +mkdir $out +echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar + +ln -s $input2 $out/input-2 + +# Self-reference. +ln -s $out $out/self + +# Executable. +echo program > $out/program +chmod +x $out/program + +echo FOO diff --git a/third_party/nix/tests/dependencies.builder1.sh b/third_party/nix/tests/dependencies.builder1.sh new file mode 100644 index 000000000000..4b006a17d70f --- /dev/null +++ b/third_party/nix/tests/dependencies.builder1.sh @@ -0,0 +1,2 @@ +mkdir $out +echo FOO > $out/foo diff --git a/third_party/nix/tests/dependencies.builder2.sh b/third_party/nix/tests/dependencies.builder2.sh new file mode 100644 index 000000000000..4f886fdb3a1a --- /dev/null +++ b/third_party/nix/tests/dependencies.builder2.sh @@ -0,0 +1,2 @@ +mkdir $out +echo BAR > $out/bar diff --git a/third_party/nix/tests/dependencies.nix b/third_party/nix/tests/dependencies.nix new file mode 100644 index 000000000000..eca4b2964cfb --- /dev/null +++ b/third_party/nix/tests/dependencies.nix @@ -0,0 +1,24 @@ +with import ./config.nix; + +let { + + input1 = mkDerivation { + name = "dependencies-input-1"; + builder = ./dependencies.builder1.sh; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + builder = "${./dependencies.builder2.sh}"; + }; + + body = mkDerivation { + name = "dependencies"; + builder = ./dependencies.builder0.sh + "/FOOBAR/../."; + input1 = input1 + "/."; + input2 = "${input2}/."; + input1_drv = input1; + meta.description = "Random test package"; + }; + +} diff --git a/third_party/nix/tests/dependencies.sh b/third_party/nix/tests/dependencies.sh new file mode 100644 index 000000000000..df204d185ddc --- /dev/null +++ b/third_party/nix/tests/dependencies.sh @@ -0,0 +1,52 @@ +source common.sh + +clearStore + +drvPath=$(nix-instantiate dependencies.nix) + +echo "derivation is $drvPath" + +nix-store -q --tree "$drvPath" | grep ' +---.*builder1.sh' + +# Test Graphviz graph generation. +nix-store -q --graph "$drvPath" > $TEST_ROOT/graph +if test -n "$dot"; then + # Does it parse? + $dot < $TEST_ROOT/graph +fi + +outPath=$(nix-store -rvv "$drvPath") || fail "build failed" + +# Test Graphviz graph generation. +nix-store -q --graph "$outPath" > $TEST_ROOT/graph +if test -n "$dot"; then + # Does it parse? + $dot < $TEST_ROOT/graph +fi + +nix-store -q --tree "$outPath" | grep '+---.*dependencies-input-2' + +echo "output path is $outPath" + +text=$(cat "$outPath"/foobar) +if test "$text" != "FOOBAR"; then exit 1; fi + +deps=$(nix-store -quR "$drvPath") + +echo "output closure contains $deps" + +# The output path should be in the closure. +echo "$deps" | grep -q "$outPath" + +# Input-1 is not retained. +if echo "$deps" | grep -q "dependencies-input-1"; then exit 1; fi + +# Input-2 is retained. +input2OutPath=$(echo "$deps" | grep "dependencies-input-2") + +# The referrers closure of input-2 should include outPath. +nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" + +# Check that the derivers are set properly. +test $(nix-store -q --deriver "$outPath") = "$drvPath" +nix-store -q --deriver "$input2OutPath" | grep -q -- "-input-2.drv" diff --git a/third_party/nix/tests/dump-db.sh b/third_party/nix/tests/dump-db.sh new file mode 100644 index 000000000000..d6eea42aa04e --- /dev/null +++ b/third_party/nix/tests/dump-db.sh @@ -0,0 +1,20 @@ +source common.sh + +clearStore + +path=$(nix-build dependencies.nix -o $TEST_ROOT/result) + +deps="$(nix-store -qR $TEST_ROOT/result)" + +nix-store --dump-db > $TEST_ROOT/dump + +rm -rf $NIX_STATE_DIR/db + +nix-store --load-db < $TEST_ROOT/dump + +deps2="$(nix-store -qR $TEST_ROOT/result)" + +[ "$deps" = "$deps2" ]; + +nix-store --dump-db > $TEST_ROOT/dump2 +cmp $TEST_ROOT/dump $TEST_ROOT/dump2 diff --git a/third_party/nix/tests/export-graph.nix b/third_party/nix/tests/export-graph.nix new file mode 100644 index 000000000000..fdac9583db2c --- /dev/null +++ b/third_party/nix/tests/export-graph.nix @@ -0,0 +1,29 @@ +with import ./config.nix; + +rec { + + printRefs = + '' + echo $exportReferencesGraph + while read path; do + read drv + read nrRefs + echo "$path has $nrRefs references" + echo "$path" >> $out + for ((n = 0; n < $nrRefs; n++)); do read ref; echo "ref $ref"; test -e "$ref"; done + done < refs + ''; + + foo."bar.runtimeGraph" = mkDerivation { + name = "dependencies"; + builder = builtins.toFile "build-graph-builder" "${printRefs}"; + exportReferencesGraph = ["refs" (import ./dependencies.nix)]; + }; + + foo."bar.buildGraph" = mkDerivation { + name = "dependencies"; + builder = builtins.toFile "build-graph-builder" "${printRefs}"; + exportReferencesGraph = ["refs" (import ./dependencies.nix).drvPath]; + }; + +} diff --git a/third_party/nix/tests/export-graph.sh b/third_party/nix/tests/export-graph.sh new file mode 100644 index 000000000000..a6fd69054425 --- /dev/null +++ b/third_party/nix/tests/export-graph.sh @@ -0,0 +1,30 @@ +source common.sh + +clearStore +clearProfiles + +checkRef() { + nix-store -q --references $TEST_ROOT/result | grep -q "$1" || fail "missing reference $1" +} + +# Test the export of the runtime dependency graph. + +outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result) + +test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 2 || fail "bad nr of references" + +checkRef input-2 +for i in $(cat $outPath); do checkRef $i; done + +# Test the export of the build-time dependency graph. + +nix-store --gc # should force rebuild of input-1 + +outPath=$(nix-build ./export-graph.nix -A 'foo."bar.buildGraph"' -o $TEST_ROOT/result) + +checkRef input-1 +checkRef input-1.drv +checkRef input-2 +checkRef input-2.drv + +for i in $(cat $outPath); do checkRef $i; done diff --git a/third_party/nix/tests/export.sh b/third_party/nix/tests/export.sh new file mode 100644 index 000000000000..2238539bcca9 --- /dev/null +++ b/third_party/nix/tests/export.sh @@ -0,0 +1,36 @@ +source common.sh + +clearStore + +outPath=$(nix-build dependencies.nix --no-out-link) + +nix-store --export $outPath > $TEST_ROOT/exp + +nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all + +if nix-store --export $outPath >/dev/full ; then + echo "exporting to a bad file descriptor should fail" + exit 1 +fi + + +clearStore + +if nix-store --import < $TEST_ROOT/exp; then + echo "importing a non-closure should fail" + exit 1 +fi + + +clearStore + +nix-store --import < $TEST_ROOT/exp_all + +nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all2 + + +clearStore + +# Regression test: the derivers in exp_all2 are empty, which shouldn't +# cause a failure. +nix-store --import < $TEST_ROOT/exp_all2 diff --git a/third_party/nix/tests/fetchGit.sh b/third_party/nix/tests/fetchGit.sh new file mode 100644 index 000000000000..4c46bdf0465b --- /dev/null +++ b/third_party/nix/tests/fetchGit.sh @@ -0,0 +1,141 @@ +source common.sh + +if [[ -z $(type -p git) ]]; then + echo "Git not installed; skipping Git tests" + exit 99 +fi + +clearStore + +repo=$TEST_ROOT/git + +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/gitv2 + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" + +echo utrecht > $repo/hello +touch $repo/.gitignore +git -C $repo add hello .gitignore +git -C $repo commit -m 'Bla1' +rev1=$(git -C $repo rev-parse HEAD) + +echo world > $repo/hello +git -C $repo commit -m 'Bla2' -a +rev2=$(git -C $repo rev-parse HEAD) + +# Fetch the default branch. +path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +[[ $(cat $path/hello) = world ]] + +# In pure eval mode, fetchGit without a revision should fail. +[[ $(nix eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") = world ]] +(! nix eval --pure-eval --raw "(builtins.readFile (fetchGit file://$repo + \"/hello\"))") + +# Fetch using an explicit revision hash. +path2=$(nix eval --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +# In pure eval mode, fetchGit with a revision should succeed. +[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] + +# Fetch again. This should be cached. +mv $repo ${repo}-tmp +path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +[[ $path = $path2 ]] + +[[ $(nix eval "(builtins.fetchGit file://$repo).revCount") = 2 ]] +[[ $(nix eval --raw "(builtins.fetchGit file://$repo).rev") = $rev2 ]] + +# But with TTL 0, it should fail. +(! nix eval --tarball-ttl 0 "(builtins.fetchGit file://$repo)" -vvvvv) + +# Fetching with a explicit hash should succeed. +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") +[[ $(cat $path2/hello) = utrecht ]] + +mv ${repo}-tmp $repo + +# Using a clean working tree should produce the same result. +path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +[[ $path = $path2 ]] + +# Using an unclean tree should yield the tracked but uncommitted changes. +mkdir $repo/dir1 $repo/dir2 +echo foo > $repo/dir1/foo +echo bar > $repo/bar +echo bar > $repo/dir2/bar +git -C $repo add dir1/foo +git -C $repo rm hello + +path2=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +[ ! -e $path2/hello ] +[ ! -e $path2/bar ] +[ ! -e $path2/dir2/bar ] +[ ! -e $path2/.git ] +[[ $(cat $path2/dir1/foo) = foo ]] + +[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# ... unless we're using an explicit ref or rev. +path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") +[[ $path = $path3 ]] + +path3=$(nix eval --raw "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path3 ]] + +# Committing should not affect the store path. +git -C $repo commit -m 'Bla3' -a + +path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchGit file://$repo).outPath") +[[ $path2 = $path4 ]] + +# tarball-ttl should be ignored if we specify a rev +echo delft > $repo/hello +git -C $repo add hello +git -C $repo commit -m 'Bla4' +rev3=$(git -C $repo rev-parse HEAD) +nix eval --tarball-ttl 3600 "(builtins.fetchGit { url = $repo; rev = \"$rev3\"; })" >/dev/null + +# Update 'path' to reflect latest master +path=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") + +# Check behavior when non-master branch is used +git -C $repo checkout $rev2 -b dev +echo dev > $repo/hello + +# File URI uses 'master' unless specified otherwise +path2=$(nix eval --raw "(builtins.fetchGit file://$repo).outPath") +[[ $path = $path2 ]] + +# Using local path with branch other than 'master' should work when clean or dirty +path3=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +# (check dirty-tree handling was used) +[[ $(nix eval --raw "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# Committing shouldn't change store path, or switch to using 'master' +git -C $repo commit -m 'Bla5' -a +path4=$(nix eval --raw "(builtins.fetchGit $repo).outPath") +[[ $(cat $path4/hello) = dev ]] +[[ $path3 = $path4 ]] + +# Confirm same as 'dev' branch +path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +[[ $path3 = $path5 ]] + + +# Nuke the cache +rm -rf $TEST_HOME/.cache/nix/gitv2 + +# Try again, but without 'git' on PATH +NIX=$(command -v nix) +# This should fail +(! PATH= $NIX eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) + +# Try again, with 'git' available. This should work. +path5=$(nix eval --raw "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") +[[ $path3 = $path5 ]] diff --git a/third_party/nix/tests/fetchMercurial.sh b/third_party/nix/tests/fetchMercurial.sh new file mode 100644 index 000000000000..4088dbd39796 --- /dev/null +++ b/third_party/nix/tests/fetchMercurial.sh @@ -0,0 +1,93 @@ +source common.sh + +if [[ -z $(type -p hg) ]]; then + echo "Mercurial not installed; skipping Mercurial tests" + exit 99 +fi + +clearStore + +repo=$TEST_ROOT/hg + +rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix/hg + +hg init $repo +echo '[ui]' >> $repo/.hg/hgrc +echo 'username = Foobar <foobar@example.org>' >> $repo/.hg/hgrc + +echo utrecht > $repo/hello +touch $repo/.hgignore +hg add --cwd $repo hello .hgignore +hg commit --cwd $repo -m 'Bla1' +rev1=$(hg log --cwd $repo -r tip --template '{node}') + +echo world > $repo/hello +hg commit --cwd $repo -m 'Bla2' +rev2=$(hg log --cwd $repo -r tip --template '{node}') + +# Fetch the default branch. +path=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $(cat $path/hello) = world ]] + +# In pure eval mode, fetchGit without a revision should fail. +[[ $(nix eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] +(! nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") + +# Fetch using an explicit revision hash. +path2=$(nix eval --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +# In pure eval mode, fetchGit with a revision should succeed. +[[ $(nix eval --pure-eval --raw "(builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\"))") = world ]] + +# Fetch again. This should be cached. +mv $repo ${repo}-tmp +path2=$(nix eval --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $path = $path2 ]] + +[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).branch") = default ]] +[[ $(nix eval "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] +[[ $(nix eval --raw "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] + +# But with TTL 0, it should fail. +(! nix eval --tarball-ttl 0 "(builtins.fetchMercurial file://$repo)") + +# Fetching with a explicit hash should succeed. +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") +[[ $path = $path2 ]] + +path2=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") +[[ $(cat $path2/hello) = utrecht ]] + +mv ${repo}-tmp $repo + +# Using a clean working tree should produce the same result. +path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +[[ $path = $path2 ]] + +# Using an unclean tree should yield the tracked but uncommitted changes. +mkdir $repo/dir1 $repo/dir2 +echo foo > $repo/dir1/foo +echo bar > $repo/bar +echo bar > $repo/dir2/bar +hg add --cwd $repo dir1/foo +hg rm --cwd $repo hello + +path2=$(nix eval --raw "(builtins.fetchMercurial $repo).outPath") +[ ! -e $path2/hello ] +[ ! -e $path2/bar ] +[ ! -e $path2/dir2/bar ] +[ ! -e $path2/.hg ] +[[ $(cat $path2/dir1/foo) = foo ]] + +[[ $(nix eval --raw "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] + +# ... unless we're using an explicit rev. +path3=$(nix eval --raw "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") +[[ $path = $path3 ]] + +# Committing should not affect the store path. +hg commit --cwd $repo -m 'Bla3' + +path4=$(nix eval --tarball-ttl 0 --raw "(builtins.fetchMercurial file://$repo).outPath") +[[ $path2 = $path4 ]] diff --git a/third_party/nix/tests/fetchurl.sh b/third_party/nix/tests/fetchurl.sh new file mode 100644 index 000000000000..7319ced2b599 --- /dev/null +++ b/third_party/nix/tests/fetchurl.sh @@ -0,0 +1,78 @@ +source common.sh + +clearStore + +# Test fetching a flat file. +hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link --hashed-mirrors '') + +cmp $outPath fetchurl.sh + +# Now using a base-64 hash. +clearStore + +hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors '') + +cmp $outPath fetchurl.sh + +# Now using an SRI hash. +clearStore + +hash=$(nix hash-file ./fetchurl.sh) + +[[ $hash =~ ^sha256- ]] + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link --hashed-mirrors '') + +cmp $outPath fetchurl.sh + +# Test the hashed mirror feature. +clearStore + +hash=$(nix hash-file --type sha512 --base64 ./fetchurl.sh) +hash32=$(nix hash-file --type sha512 --base16 ./fetchurl.sh) + +mirror=$TMPDIR/hashed-mirror +rm -rf $mirror +mkdir -p $mirror/sha512 +ln -s $(pwd)/fetchurl.sh $mirror/sha512/$hash32 + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha512 $hash --no-out-link --hashed-mirrors "file://$mirror") + +# Test hashed mirrors with an SRI hash. +nix-build '<nix/fetchurl.nix>' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix to-sri --type sha512 $hash) \ + --argstr name bla --no-out-link --hashed-mirrors "file://$mirror" + +# Test unpacking a NAR. +rm -rf $TEST_ROOT/archive +mkdir -p $TEST_ROOT/archive +cp ./fetchurl.sh $TEST_ROOT/archive +chmod +x $TEST_ROOT/archive/fetchurl.sh +ln -s foo $TEST_ROOT/archive/symlink +nar=$TEST_ROOT/archive.nar +nix-store --dump $TEST_ROOT/archive > $nar + +hash=$(nix-hash --flat --type sha256 $nar) + +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$nar --argstr sha256 $hash \ + --arg unpack true --argstr name xyzzy --no-out-link) + +echo $outPath | grep -q 'xyzzy' + +test -x $outPath/fetchurl.sh +test -L $outPath/symlink + +nix-store --delete $outPath + +# Test unpacking a compressed NAR. +narxz=$TEST_ROOT/archive.nar.xz +rm -f $narxz +xz --keep $nar +outPath=$(nix-build '<nix/fetchurl.nix>' --argstr url file://$narxz --argstr sha256 $hash \ + --arg unpack true --argstr name xyzzy --no-out-link) + +test -x $outPath/fetchurl.sh +test -L $outPath/symlink diff --git a/third_party/nix/tests/filter-source.nix b/third_party/nix/tests/filter-source.nix new file mode 100644 index 000000000000..9071636394af --- /dev/null +++ b/third_party/nix/tests/filter-source.nix @@ -0,0 +1,12 @@ +with import ./config.nix; + +mkDerivation { + name = "filter"; + builder = builtins.toFile "builder" "ln -s $input $out"; + input = + let filter = path: type: + type != "symlink" + && baseNameOf path != "foo" + && !((import ./lang/lib.nix).hasSuffix ".bak" (baseNameOf path)); + in builtins.filterSource filter ((builtins.getEnv "TEST_ROOT") + "/filterin"); +} diff --git a/third_party/nix/tests/filter-source.sh b/third_party/nix/tests/filter-source.sh new file mode 100644 index 000000000000..1f8dceee5786 --- /dev/null +++ b/third_party/nix/tests/filter-source.sh @@ -0,0 +1,19 @@ +source common.sh + +rm -rf $TEST_ROOT/filterin +mkdir $TEST_ROOT/filterin +mkdir $TEST_ROOT/filterin/foo +touch $TEST_ROOT/filterin/foo/bar +touch $TEST_ROOT/filterin/xyzzy +touch $TEST_ROOT/filterin/b +touch $TEST_ROOT/filterin/bak +touch $TEST_ROOT/filterin/bla.c.bak +ln -s xyzzy $TEST_ROOT/filterin/link + +nix-build ./filter-source.nix -o $TEST_ROOT/filterout + +test ! -e $TEST_ROOT/filterout/foo/bar +test -e $TEST_ROOT/filterout/xyzzy +test -e $TEST_ROOT/filterout/bak +test ! -e $TEST_ROOT/filterout/bla.c.bak +test ! -L $TEST_ROOT/filterout/link diff --git a/third_party/nix/tests/fixed.builder1.sh b/third_party/nix/tests/fixed.builder1.sh new file mode 100644 index 000000000000..c41bb2b9a611 --- /dev/null +++ b/third_party/nix/tests/fixed.builder1.sh @@ -0,0 +1,3 @@ +if test "$IMPURE_VAR1" != "foo"; then exit 1; fi +if test "$IMPURE_VAR2" != "bar"; then exit 1; fi +echo "Hello World!" > $out diff --git a/third_party/nix/tests/fixed.builder2.sh b/third_party/nix/tests/fixed.builder2.sh new file mode 100644 index 000000000000..31ea1579a514 --- /dev/null +++ b/third_party/nix/tests/fixed.builder2.sh @@ -0,0 +1,6 @@ +echo dummy: $dummy +if test -n "$dummy"; then sleep 2; fi +mkdir $out +mkdir $out/bla +echo "Hello World!" > $out/foo +ln -s foo $out/bar diff --git a/third_party/nix/tests/fixed.nix b/third_party/nix/tests/fixed.nix new file mode 100644 index 000000000000..76580ffa19e8 --- /dev/null +++ b/third_party/nix/tests/fixed.nix @@ -0,0 +1,50 @@ +with import ./config.nix; + +rec { + + f2 = dummy: builder: mode: algo: hash: mkDerivation { + name = "fixed"; + inherit builder; + outputHashMode = mode; + outputHashAlgo = algo; + outputHash = hash; + inherit dummy; + impureEnvVars = ["IMPURE_VAR1" "IMPURE_VAR2"]; + }; + + f = f2 ""; + + good = [ + (f ./fixed.builder1.sh "flat" "md5" "8ddd8be4b179a529afa5f2ffae4b9858") + (f ./fixed.builder1.sh "flat" "sha1" "a0b65939670bc2c010f4d5d6a0b3e4e4590fb92b") + (f ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42") + (f ./fixed.builder2.sh "recursive" "sha1" "vw46m23bizj4n8afrc0fj19wrp7mj3c0") + ]; + + good2 = [ + # Yes, this looks fscked up: builder2 doesn't have that result. + # But Nix sees that an output with the desired hash already + # exists, and will refrain from building it. + (f ./fixed.builder2.sh "flat" "md5" "8ddd8be4b179a529afa5f2ffae4b9858") + ]; + + sameAsAdd = + f ./fixed.builder2.sh "recursive" "sha256" "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik"; + + bad = [ + (f ./fixed.builder1.sh "flat" "md5" "0ddd8be4b179a529afa5f2ffae4b9858") + ]; + + reallyBad = [ + # Hash too short, and not base-32 either. + (f ./fixed.builder1.sh "flat" "md5" "ddd8be4b179a529afa5f2ffae4b9858") + ]; + + # Test for building two derivations in parallel that produce the + # same output path because they're fixed-output derivations. + parallelSame = [ + (f2 "foo" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42") + (f2 "bar" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42") + ]; + +} diff --git a/third_party/nix/tests/fixed.sh b/third_party/nix/tests/fixed.sh new file mode 100644 index 000000000000..8f51403a7071 --- /dev/null +++ b/third_party/nix/tests/fixed.sh @@ -0,0 +1,56 @@ +source common.sh + +clearStore + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0)) + +echo 'testing bad...' +nix-build fixed.nix -A bad --no-out-link && fail "should fail" + +# Building with the bad hash should produce the "good" output path as +# a side-effect. +[[ -e $path ]] +nix path-info --json $path | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd + +echo 'testing good...' +nix-build fixed.nix -A good --no-out-link + +echo 'testing good2...' +nix-build fixed.nix -A good2 --no-out-link + +echo 'testing reallyBad...' +nix-instantiate fixed.nix -A reallyBad && fail "should fail" + +# While we're at it, check attribute selection a bit more. +echo 'testing attribute selection...' +test $(nix-instantiate fixed.nix -A good.1 | wc -l) = 1 + +# Test parallel builds of derivations that produce the same output. +# Only one should run at the same time. +echo 'testing parallelSame...' +clearStore +nix-build fixed.nix -A parallelSame --no-out-link -j2 + +# Fixed-output derivations with a recursive SHA-256 hash should +# produce the same path as "nix-store --add". +echo 'testing sameAsAdd...' +out=$(nix-build fixed.nix -A sameAsAdd --no-out-link) + +# This is what fixed.builder2 produces... +rm -rf $TEST_ROOT/fixed +mkdir $TEST_ROOT/fixed +mkdir $TEST_ROOT/fixed/bla +echo "Hello World!" > $TEST_ROOT/fixed/foo +ln -s foo $TEST_ROOT/fixed/bar + +out2=$(nix-store --add $TEST_ROOT/fixed) +[ "$out" = "$out2" ] + +out3=$(nix-store --add-fixed --recursive sha256 $TEST_ROOT/fixed) +[ "$out" = "$out3" ] + +out4=$(nix-store --print-fixed-path --recursive sha256 "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik" fixed) +[ "$out" = "$out4" ] diff --git a/third_party/nix/tests/function-trace.sh b/third_party/nix/tests/function-trace.sh new file mode 100755 index 000000000000..182a4d5c287a --- /dev/null +++ b/third_party/nix/tests/function-trace.sh @@ -0,0 +1,85 @@ +source common.sh + +set +x + +expect_trace() { + expr="$1" + expect="$2" + actual=$( + nix-instantiate \ + --trace-function-calls \ + --expr "$expr" 2>&1 \ + | grep "function-trace" \ + | sed -e 's/ [0-9]*$//' + ); + + echo -n "Tracing expression '$expr'" + set +e + msg=$(diff -swB \ + <(echo "$expect") \ + <(echo "$actual") + ); + result=$? + set -e + if [ $result -eq 0 ]; then + echo " ok." + else + echo " failed. difference:" + echo "$msg" + return $result + fi +} + +# failure inside a tryEval +expect_trace 'builtins.tryEval (throw "example")' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace entered (string):1:19 at +function-trace exited (string):1:19 at +function-trace exited (string):1:1 at +" + +# Missing argument to a formal function +expect_trace '({ x }: x) { }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a formal function +expect_trace '({ x }: x) { x = "x"; y = "y"; }' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not enough arguments to a lambda +expect_trace '(x: y: x + y) 1' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Too many arguments to a lambda +expect_trace '(x: x) 1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +# Not a function +expect_trace '1 2' " +function-trace entered undefined position at +function-trace exited undefined position at +function-trace entered (string):1:1 at +function-trace exited (string):1:1 at +" + +set -e diff --git a/third_party/nix/tests/gc-auto.sh b/third_party/nix/tests/gc-auto.sh new file mode 100644 index 000000000000..de1e2cfe4059 --- /dev/null +++ b/third_party/nix/tests/gc-auto.sh @@ -0,0 +1,70 @@ +source common.sh + +clearStore + +garbage1=$(nix add-to-store --name garbage1 ./nar-access.sh) +garbage2=$(nix add-to-store --name garbage2 ./nar-access.sh) +garbage3=$(nix add-to-store --name garbage3 ./nar-access.sh) + +ls -l $garbage3 +POSIXLY_CORRECT=1 du $garbage3 + +fake_free=$TEST_ROOT/fake-free +export _NIX_TEST_FREE_SPACE_FILE=$fake_free +echo 1100 > $fake_free + +expr=$(cat <<EOF +with import ./config.nix; mkDerivation { + name = "gc-A"; + buildCommand = '' + set -x + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 3 ]] + mkdir \$out + echo foo > \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp1 + mv ${fake_free}.tmp1 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + [[ \$(ls \$NIX_STORE/*-garbage? | wc -l) = 1 ]] + ''; +} +EOF +) + +expr2=$(cat <<EOF +with import ./config.nix; mkDerivation { + name = "gc-B"; + buildCommand = '' + set -x + mkdir \$out + echo foo > \$out/bar + echo 1... + sleep 2 + echo 200 > ${fake_free}.tmp2 + mv ${fake_free}.tmp2 $fake_free + echo 2... + sleep 2 + echo 3... + sleep 2 + echo 4... + ''; +} +EOF +) + +nix build -v -o $TEST_ROOT/result-A -L "($expr)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 & +pid=$! + +nix build -v -o $TEST_ROOT/result-B -L "($expr2)" \ + --min-free 1000 --max-free 2000 --min-free-check-interval 1 + +wait "$pid" + +[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] +[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] diff --git a/third_party/nix/tests/gc-concurrent.builder.sh b/third_party/nix/tests/gc-concurrent.builder.sh new file mode 100644 index 000000000000..0cd67df3aeda --- /dev/null +++ b/third_party/nix/tests/gc-concurrent.builder.sh @@ -0,0 +1,13 @@ +mkdir $out +echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar + +sleep 10 + +# $out should not have been GC'ed while we were sleeping, but just in +# case... +mkdir -p $out + +# Check that the GC hasn't deleted the lock on our output. +test -e "$out.lock" + +ln -s $input2 $out/input-2 diff --git a/third_party/nix/tests/gc-concurrent.nix b/third_party/nix/tests/gc-concurrent.nix new file mode 100644 index 000000000000..c0595cc471b9 --- /dev/null +++ b/third_party/nix/tests/gc-concurrent.nix @@ -0,0 +1,27 @@ +with import ./config.nix; + +rec { + + input1 = mkDerivation { + name = "dependencies-input-1"; + builder = ./dependencies.builder1.sh; + }; + + input2 = mkDerivation { + name = "dependencies-input-2"; + builder = ./dependencies.builder2.sh; + }; + + test1 = mkDerivation { + name = "gc-concurrent"; + builder = ./gc-concurrent.builder.sh; + inherit input1 input2; + }; + + test2 = mkDerivation { + name = "gc-concurrent2"; + builder = ./gc-concurrent2.builder.sh; + inherit input1 input2; + }; + +} diff --git a/third_party/nix/tests/gc-concurrent.sh b/third_party/nix/tests/gc-concurrent.sh new file mode 100644 index 000000000000..d395930ca0dc --- /dev/null +++ b/third_party/nix/tests/gc-concurrent.sh @@ -0,0 +1,58 @@ +source common.sh + +clearStore + +drvPath1=$(nix-instantiate gc-concurrent.nix -A test1) +outPath1=$(nix-store -q $drvPath1) + +drvPath2=$(nix-instantiate gc-concurrent.nix -A test2) +outPath2=$(nix-store -q $drvPath2) + +drvPath3=$(nix-instantiate simple.nix) +outPath3=$(nix-store -r $drvPath3) + +(! test -e $outPath3.lock) +touch $outPath3.lock + +rm -f "$NIX_STATE_DIR"/gcroots/foo* +ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo +ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2 + +# Start build #1 in the background. It starts immediately. +nix-store -rvv "$drvPath1" & +pid1=$! + +# Start build #2 in the background after 10 seconds. +(sleep 10 && nix-store -rvv "$drvPath2") & +pid2=$! + +# Run the garbage collector while the build is running. +sleep 6 +nix-collect-garbage + +# Wait for build #1/#2 to finish. +echo waiting for pid $pid1 to finish... +wait $pid1 +echo waiting for pid $pid2 to finish... +wait $pid2 + +# Check that the root of build #1 and its dependencies haven't been +# deleted. The should not be deleted by the GC because they were +# being built during the GC. +cat $outPath1/foobar +cat $outPath1/input-2/bar + +# Check that build #2 has succeeded. It should succeed because the +# derivation is a GC root. +cat $outPath2/foobar + +rm -f "$NIX_STATE_DIR"/gcroots/foo* + +# The collector should have deleted lock files for paths that have +# been built previously. +(! test -e $outPath3.lock) + +# If we run the collector now, it should delete outPath1/2. +nix-collect-garbage +(! test -e $outPath1) +(! test -e $outPath2) diff --git a/third_party/nix/tests/gc-concurrent2.builder.sh b/third_party/nix/tests/gc-concurrent2.builder.sh new file mode 100644 index 000000000000..4bfb33103e73 --- /dev/null +++ b/third_party/nix/tests/gc-concurrent2.builder.sh @@ -0,0 +1,7 @@ +mkdir $out +echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar + +# Check that the GC hasn't deleted the lock on our output. +test -e "$out.lock" + +sleep 6 diff --git a/third_party/nix/tests/gc-runtime.nix b/third_party/nix/tests/gc-runtime.nix new file mode 100644 index 000000000000..ee5980bdff98 --- /dev/null +++ b/third_party/nix/tests/gc-runtime.nix @@ -0,0 +1,17 @@ +with import ./config.nix; + +mkDerivation { + name = "gc-runtime"; + builder = + # Test inline source file definitions. + builtins.toFile "builder.sh" '' + mkdir $out + + cat > $out/program <<EOF + #! ${shell} + sleep 10000 + EOF + + chmod +x $out/program + ''; +} diff --git a/third_party/nix/tests/gc-runtime.sh b/third_party/nix/tests/gc-runtime.sh new file mode 100644 index 000000000000..4c5028005c57 --- /dev/null +++ b/third_party/nix/tests/gc-runtime.sh @@ -0,0 +1,38 @@ +source common.sh + +case $system in + *linux*) + ;; + *) + exit 0; +esac + +set -m # enable job control, needed for kill + +profiles="$NIX_STATE_DIR"/profiles +rm -rf $profiles + +nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime + +outPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime) +echo $outPath + +echo "backgrounding program..." +$profiles/test/program & +sleep 2 # hack - wait for the program to get started +child=$! +echo PID=$child + +nix-env -p $profiles/test -e gc-runtime +nix-env -p $profiles/test --delete-generations old + +nix-store --gc + +kill -- -$child + +if ! test -e $outPath; then + echo "running program was garbage collected!" + exit 1 +fi + +exit 0 diff --git a/third_party/nix/tests/gc.sh b/third_party/nix/tests/gc.sh new file mode 100644 index 000000000000..8b4f8d282184 --- /dev/null +++ b/third_party/nix/tests/gc.sh @@ -0,0 +1,40 @@ +source common.sh + +drvPath=$(nix-instantiate dependencies.nix) +outPath=$(nix-store -rvv "$drvPath") + +# Set a GC root. +rm -f "$NIX_STATE_DIR"/gcroots/foo +ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo + +[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] + +nix-store --gc --print-roots | grep $outPath +nix-store --gc --print-live | grep $outPath +nix-store --gc --print-dead | grep $drvPath +if nix-store --gc --print-dead | grep $outPath; then false; fi + +nix-store --gc --print-dead + +inUse=$(readLink $outPath/input-2) +if nix-store --delete $inUse; then false; fi +test -e $inUse + +if nix-store --delete $outPath; then false; fi +test -e $outPath + +nix-collect-garbage + +# Check that the root and its dependencies haven't been deleted. +cat $outPath/foobar +cat $outPath/input-2/bar + +# Check that the derivation has been GC'd. +if test -e $drvPath; then false; fi + +rm "$NIX_STATE_DIR"/gcroots/foo + +nix-collect-garbage + +# Check that the output has been GC'd. +if test -e $outPath/foobar; then false; fi diff --git a/third_party/nix/tests/hash-check.nix b/third_party/nix/tests/hash-check.nix new file mode 100644 index 000000000000..4a8e9b8a8df9 --- /dev/null +++ b/third_party/nix/tests/hash-check.nix @@ -0,0 +1,29 @@ +let { + + input1 = derivation { + name = "dependencies-input-1"; + system = "i086-msdos"; + builder = "/bar/sh"; + args = ["-e" "-x" ./dummy]; + }; + + input2 = derivation { + name = "dependencies-input-2"; + system = "i086-msdos"; + builder = "/bar/sh"; + args = ["-e" "-x" ./dummy]; + outputHashMode = "recursive"; + outputHashAlgo = "md5"; + outputHash = "ffffffffffffffffffffffffffffffff"; + }; + + body = derivation { + name = "dependencies"; + system = "i086-msdos"; + builder = "/bar/sh"; + args = ["-e" "-x" (./dummy + "/FOOBAR/../.")]; + input1 = input1 + "/."; + inherit input2; + }; + +} \ No newline at end of file diff --git a/third_party/nix/tests/hash.sh b/third_party/nix/tests/hash.sh new file mode 100644 index 000000000000..4cfc97901012 --- /dev/null +++ b/third_party/nix/tests/hash.sh @@ -0,0 +1,87 @@ +source common.sh + +try () { + printf "%s" "$2" > $TEST_ROOT/vector + hash=$(nix hash-file --base16 $EXTRA --type "$1" $TEST_ROOT/vector) + if test "$hash" != "$3"; then + echo "hash $1, expected $3, got $hash" + exit 1 + fi +} + +try md5 "" "d41d8cd98f00b204e9800998ecf8427e" +try md5 "a" "0cc175b9c0f1b6a831c399e269772661" +try md5 "abc" "900150983cd24fb0d6963f7d28e17f72" +try md5 "message digest" "f96b697d7cb7938d525a2f31aaf161d0" +try md5 "abcdefghijklmnopqrstuvwxyz" "c3fcd3d76192e4007dfb496cca67e13b" +try md5 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" "d174ab98d277d9f5a5611c2c9f419d9f" +try md5 "12345678901234567890123456789012345678901234567890123456789012345678901234567890" "57edf4a22be3c955ac49da2e2107b67a" + +try sha1 "" "da39a3ee5e6b4b0d3255bfef95601890afd80709" +try sha1 "abc" "a9993e364706816aba3e25717850c26c9cd0d89d" +try sha1 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "84983e441c3bd26ebaae4aa1f95129e5e54670f1" + +try sha256 "" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +try sha256 "abc" "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" +try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + +try sha512 "" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" +try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" + +EXTRA=--base32 +try sha256 "abc" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" +EXTRA= + +EXTRA=--sri +try sha512 "" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" +try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" +try sha512 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha512-IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" +try sha256 "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq" "sha256-JI1qYdIGOLjlwCaTDD5gOaM85Flk/yFn9uzt1BnbBsE=" + +try2 () { + hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) + if test "$hash" != "$2"; then + echo "hash $1, expected $2, got $hash" + exit 1 + fi +} + +rm -rf $TEST_ROOT/hash-path +mkdir $TEST_ROOT/hash-path +echo "Hello World" > $TEST_ROOT/hash-path/hello + +try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" + +# Execute bit matters. +chmod +x $TEST_ROOT/hash-path/hello +try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" + +# Mtime and other bits don't. +touch -r . $TEST_ROOT/hash-path/hello +chmod 744 $TEST_ROOT/hash-path/hello +try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" + +# File type (e.g., symlink) does. +rm $TEST_ROOT/hash-path/hello +ln -s x $TEST_ROOT/hash-path/hello +try2 md5 "f78b733a68f5edbdf9413899339eaa4a" + +# Conversion. +try3() { + h64=$(nix to-base64 --type "$1" "$2") + [ "$h64" = "$4" ] + sri=$(nix to-sri --type "$1" "$2") + [ "$sri" = "$1-$4" ] + h32=$(nix-hash --type "$1" --to-base32 "$2") + [ "$h32" = "$3" ] + h16=$(nix-hash --type "$1" --to-base16 "$h32") + [ "$h16" = "$2" ] + h16=$(nix to-base16 --type "$1" "$h64") + [ "$h16" = "$2" ] + h16=$(nix to-base16 "$sri") + [ "$h16" = "$2" ] +} +try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" +try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" +try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/third_party/nix/tests/import-derivation.nix b/third_party/nix/tests/import-derivation.nix new file mode 100644 index 000000000000..44fa9a45d7e1 --- /dev/null +++ b/third_party/nix/tests/import-derivation.nix @@ -0,0 +1,26 @@ +with import ./config.nix; + +let + + bar = mkDerivation { + name = "bar"; + builder = builtins.toFile "builder.sh" + '' + echo 'builtins.add 123 456' > $out + ''; + }; + + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; + +in + +mkDerivation { + name = "foo"; + builder = builtins.toFile "builder.sh" + '' + echo -n FOO${toString value} > $out + ''; +} diff --git a/third_party/nix/tests/import-derivation.sh b/third_party/nix/tests/import-derivation.sh new file mode 100644 index 000000000000..98d61ef49b9c --- /dev/null +++ b/third_party/nix/tests/import-derivation.sh @@ -0,0 +1,12 @@ +source common.sh + +clearStore + +if nix-instantiate --readonly-mode ./import-derivation.nix; then + echo "read-only evaluation of an imported derivation unexpectedly failed" + exit 1 +fi + +outPath=$(nix-build ./import-derivation.nix --no-out-link) + +[ "$(cat $outPath)" = FOO579 ] diff --git a/third_party/nix/tests/init.sh b/third_party/nix/tests/init.sh new file mode 100644 index 000000000000..19a12c1e2d9e --- /dev/null +++ b/third_party/nix/tests/init.sh @@ -0,0 +1,34 @@ +source common.sh + +test -n "$TEST_ROOT" +if test -d "$TEST_ROOT"; then + chmod -R u+w "$TEST_ROOT" + rm -rf "$TEST_ROOT" +fi +mkdir "$TEST_ROOT" + +mkdir "$NIX_STORE_DIR" +mkdir "$NIX_LOCALSTATE_DIR" +mkdir -p "$NIX_LOG_DIR"/drvs +mkdir "$NIX_STATE_DIR" +mkdir "$NIX_CONF_DIR" + +cat > "$NIX_CONF_DIR"/nix.conf <<EOF +build-users-group = +keep-derivations = false +sandbox = false +include nix.conf.extra +EOF + +cat > "$NIX_CONF_DIR"/nix.conf.extra <<EOF +fsync-metadata = false +!include nix.conf.extra.not-there +EOF + +# Initialise the database. +nix-store --init + +# Did anything happen? +test -e "$NIX_STATE_DIR"/db/db.sqlite + +echo 'Hello World' > ./dummy diff --git a/third_party/nix/tests/install-darwin.sh b/third_party/nix/tests/install-darwin.sh new file mode 100755 index 000000000000..9933eba94431 --- /dev/null +++ b/third_party/nix/tests/install-darwin.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +set -eux + +cleanup() { + PLIST="/Library/LaunchDaemons/org.nixos.nix-daemon.plist" + if sudo launchctl list | grep -q nix-daemon; then + sudo launchctl unload "$PLIST" + fi + + if [ -f "$PLIST" ]; then + sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist + fi + + profiles=(/etc/profile /etc/bashrc /etc/zshrc) + for profile in "${profiles[@]}"; do + if [ -f "${profile}.backup-before-nix" ]; then + sudo mv "${profile}.backup-before-nix" "${profile}" + fi + done + + for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do + if [ -e "$file" ]; then + cat "$file" | grep -v nix-profile > "$file.next" + mv "$file.next" "$file" + fi + done + + for i in $(seq 1 $(sysctl -n hw.ncpu)); do + sudo /usr/bin/dscl . -delete "/Users/nixbld$i" || true + done + sudo /usr/bin/dscl . -delete "/Groups/nixbld" || true + + sudo rm -rf /etc/nix \ + /nix \ + /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels \ + "$HOME/.nix-profile" "$HOME/.nix-defexpr" "$HOME/.nix-channels" +} + +verify() { + set +e + output=$(echo "nix-shell -p bash --run 'echo toow | rev'" | bash -l) + set -e + + test "$output" = "woot" +} + +scratch=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish { + rm -rf "$scratch" +} +trap finish EXIT + +# First setup Nix +cleanup +curl -o install https://nixos.org/nix/install +yes | bash ./install +verify + + +( + set +e + ( + echo "cd $(pwd)" + echo nix-build ./release.nix -A binaryTarball.x86_64-darwin + ) | bash -l + set -e + cp ./result/nix-*.tar.bz2 $scratch/nix.tar.bz2 +) + +( + cd $scratch + tar -xf ./nix.tar.bz2 + + cd nix-* + + set -eux + + cleanup + + yes | ./install + verify + cleanup + + echo -n "" | ./install + verify + cleanup + + sudo mkdir -p /nix/store + sudo touch /nix/store/.silly-hint + echo -n "" | ALLOW_PREEXISTING_INSTALLATION=true ./install + verify + test -e /nix/store/.silly-hint + + cleanup +) diff --git a/third_party/nix/tests/lang.sh b/third_party/nix/tests/lang.sh new file mode 100644 index 000000000000..151a71316683 --- /dev/null +++ b/third_party/nix/tests/lang.sh @@ -0,0 +1,68 @@ +export TEST_VAR=foo # for eval-okay-getenv.nix + +nix-instantiate --eval -E 'builtins.trace "Hello" 123' 2>&1 | grep -q Hello +(! nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grep -q Hello) +nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' 2>&1 | grep -q Hello + +set +x + +fail=0 + +for i in lang/parse-fail-*.nix; do + echo "parsing $i (should fail)"; + i=$(basename $i .nix) + if nix-instantiate --parse - < lang/$i.nix; then + echo "FAIL: $i shouldn't parse" + fail=1 + fi +done + +for i in lang/parse-okay-*.nix; do + echo "parsing $i (should succeed)"; + i=$(basename $i .nix) + if ! nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then + echo "FAIL: $i should parse" + fail=1 + fi +done + +for i in lang/eval-fail-*.nix; do + echo "evaluating $i (should fail)"; + i=$(basename $i .nix) + if nix-instantiate --eval lang/$i.nix; then + echo "FAIL: $i shouldn't evaluate" + fail=1 + fi +done + +for i in lang/eval-okay-*.nix; do + echo "evaluating $i (should succeed)"; + i=$(basename $i .nix) + + if test -e lang/$i.exp; then + flags= + if test -e lang/$i.flags; then + flags=$(cat lang/$i.flags) + fi + if ! NIX_PATH=lang/dir3:lang/dir4 nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then + echo "FAIL: $i should evaluate" + fail=1 + elif ! diff lang/$i.out lang/$i.exp; then + echo "FAIL: evaluation result of $i not as expected" + fail=1 + fi + fi + + if test -e lang/$i.exp.xml; then + if ! nix-instantiate --eval --xml --no-location --strict \ + lang/$i.nix > lang/$i.out.xml; then + echo "FAIL: $i should evaluate" + fail=1 + elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then + echo "FAIL: XML evaluation result of $i not as expected" + fail=1 + fi + fi +done + +exit $fail diff --git a/third_party/nix/tests/linux-sandbox.sh b/third_party/nix/tests/linux-sandbox.sh new file mode 100644 index 000000000000..52967d07dda2 --- /dev/null +++ b/third_party/nix/tests/linux-sandbox.sh @@ -0,0 +1,30 @@ +source common.sh + +clearStore + +if ! canUseSandbox; then exit; fi + +# Note: we need to bind-mount $SHELL into the chroot. Currently we +# only support the case where $SHELL is in the Nix store, because +# otherwise things get complicated (e.g. if it's in /bin, do we need +# /lib as well?). +if [[ ! $SHELL =~ /nix/store ]]; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +export NIX_STORE_DIR=/my/store +export NIX_REMOTE=$TEST_ROOT/store0 + +outPath=$(nix-build dependencies.nix --no-out-link --sandbox-paths /nix/store) + +[[ $outPath =~ /my/store/.*-dependencies ]] + +nix path-info -r $outPath | grep input-2 + +nix ls-store -R -l $outPath | grep foobar + +nix cat-store $outPath/foobar | grep FOOBAR + +# Test --check without hash rewriting. +nix-build dependencies.nix --no-out-link --check --sandbox-paths /nix/store diff --git a/third_party/nix/tests/logging.sh b/third_party/nix/tests/logging.sh new file mode 100644 index 000000000000..c894ad3ff079 --- /dev/null +++ b/third_party/nix/tests/logging.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +path=$(nix-build dependencies.nix --no-out-link) + +# Test nix-store -l. +[ "$(nix-store -l $path)" = FOO ] + +# Test compressed logs. +clearStore +rm -rf $NIX_LOG_DIR +(! nix-store -l $path) +nix-build dependencies.nix --no-out-link --compress-build-log +[ "$(nix-store -l $path)" = FOO ] diff --git a/third_party/nix/tests/misc.sh b/third_party/nix/tests/misc.sh new file mode 100644 index 000000000000..eda0164167f2 --- /dev/null +++ b/third_party/nix/tests/misc.sh @@ -0,0 +1,19 @@ +source common.sh + +# Tests miscellaneous commands. + +# Do all commands have help? +#nix-env --help | grep -q install +#nix-store --help | grep -q realise +#nix-instantiate --help | grep -q eval +#nix-hash --help | grep -q base32 + +# Can we ask for the version number? +nix-env --version | grep "$version" + +# Usage errors. +nix-env --foo 2>&1 | grep "no operation" +nix-env -q --foo 2>&1 | grep "unknown flag" + +# Eval Errors. +nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 | grep "infinite recursion encountered, at .*(string).*:1:15$" diff --git a/third_party/nix/tests/multiple-outputs.nix b/third_party/nix/tests/multiple-outputs.nix new file mode 100644 index 000000000000..4a9010d1868e --- /dev/null +++ b/third_party/nix/tests/multiple-outputs.nix @@ -0,0 +1,68 @@ +with import ./config.nix; + +rec { + + a = mkDerivation { + name = "multiple-outputs-a"; + outputs = [ "first" "second" ]; + builder = builtins.toFile "builder.sh" + '' + mkdir $first $second + test -z $all + echo "first" > $first/file + echo "second" > $second/file + ln -s $first $second/link + ''; + helloString = "Hello, world!"; + }; + + b = mkDerivation { + defaultOutput = assert a.second.helloString == "Hello, world!"; a; + firstOutput = assert a.outputName == "first"; a.first.first; + secondOutput = assert a.second.outputName == "second"; a.second.first.first.second.second.first.second; + allOutputs = a.all; + name = "multiple-outputs-b"; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + test "$firstOutput $secondOutput" = "$allOutputs" + test "$defaultOutput" = "$firstOutput" + test "$(cat $firstOutput/file)" = "first" + test "$(cat $secondOutput/file)" = "second" + echo "success" > $out/file + ''; + }; + + c = mkDerivation { + name = "multiple-outputs-c"; + drv = b.drvPath; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + ln -s $drv $out/drv + ''; + }; + + d = mkDerivation { + name = "multiple-outputs-d"; + drv = builtins.unsafeDiscardOutputDependency b.drvPath; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + echo $drv > $out/drv + ''; + }; + + cyclic = (mkDerivation { + name = "cyclic-outputs"; + outputs = [ "a" "b" "c" ]; + builder = builtins.toFile "builder.sh" + '' + mkdir $a $b $c + echo $a > $b/foo + echo $b > $c/bar + echo $c > $a/baz + ''; + }).a; + +} diff --git a/third_party/nix/tests/multiple-outputs.sh b/third_party/nix/tests/multiple-outputs.sh new file mode 100644 index 000000000000..bedbc39a4ebf --- /dev/null +++ b/third_party/nix/tests/multiple-outputs.sh @@ -0,0 +1,76 @@ +source common.sh + +clearStore + +rm -f $TEST_ROOT/result* + +# Test whether read-only evaluation works when referring to the +# ‘drvPath’ attribute. +echo "evaluating c..." +#drvPath=$(nix-instantiate multiple-outputs.nix -A c --readonly-mode) + +# And check whether the resulting derivation explicitly depends on all +# outputs. +drvPath=$(nix-instantiate multiple-outputs.nix -A c) +#[ "$drvPath" = "$drvPath2" ] +grep -q 'multiple-outputs-a.drv",\["first","second"\]' $drvPath +grep -q 'multiple-outputs-b.drv",\["out"\]' $drvPath + +# While we're at it, test the ‘unsafeDiscardOutputDependency’ primop. +outPath=$(nix-build multiple-outputs.nix -A d --no-out-link) +drvPath=$(cat $outPath/drv) +outPath=$(nix-store -q $drvPath) +(! [ -e "$outPath" ]) + +# Do a build of something that depends on a derivation with multiple +# outputs. +echo "building b..." +outPath=$(nix-build multiple-outputs.nix -A b --no-out-link) +echo "output path is $outPath" +[ "$(cat "$outPath"/file)" = "success" ] + +# Test nix-build on a derivation with multiple outputs. +outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) +[ -e $TEST_ROOT/result-first ] +(! [ -e $TEST_ROOT/result-second ]) +nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result +[ "$(cat $TEST_ROOT/result-first/file)" = "first" ] +[ "$(cat $TEST_ROOT/result-second/file)" = "second" ] +[ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] +hash1=$(nix-store -q --hash $TEST_ROOT/result-second) + +outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a) --no-out-link) +[[ $outPath1 = $outPath2 ]] + +outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.first) --no-out-link) +[[ $outPath1 = $outPath2 ]] + +outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-out-link) +[[ $(cat $outPath2/file) = second ]] + +[[ $(nix-build $(nix-instantiate multiple-outputs.nix -A a.all) --no-out-link | wc -l) -eq 2 ]] + +# Delete one of the outputs and rebuild it. This will cause a hash +# rewrite. +nix-store --delete $TEST_ROOT/result-second --ignore-liveness +nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result +[ "$(cat $TEST_ROOT/result-second/file)" = "second" ] +[ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] +hash2=$(nix-store -q --hash $TEST_ROOT/result-second) +[ "$hash1" = "$hash2" ] + +# Make sure that nix-build works on derivations with multiple outputs. +echo "building a.first..." +nix-build multiple-outputs.nix -A a.first --no-out-link + +# Cyclic outputs should be rejected. +echo "building cyclic..." +if nix-build multiple-outputs.nix -A cyclic --no-out-link; then + echo "Cyclic outputs incorrectly accepted!" + exit 1 +fi + +echo "collecting garbage..." +rm $TEST_ROOT/result* +nix-store --gc --keep-derivations --keep-outputs +nix-store --gc --print-roots diff --git a/third_party/nix/tests/nar-access.nix b/third_party/nix/tests/nar-access.nix new file mode 100644 index 000000000000..0e2a7f721135 --- /dev/null +++ b/third_party/nix/tests/nar-access.nix @@ -0,0 +1,23 @@ +with import ./config.nix; + +rec { + a = mkDerivation { + name = "nar-index-a"; + builder = builtins.toFile "builder.sh" + '' + mkdir $out + mkdir $out/foo + touch $out/foo-x + touch $out/foo/bar + touch $out/foo/baz + touch $out/qux + mkdir $out/zyx + + cat >$out/foo/data <<EOF + lasjdöaxnasd +asdom 12398 +ä"§Æẞ¢«»”alsd +EOF + ''; + }; +} \ No newline at end of file diff --git a/third_party/nix/tests/nar-access.sh b/third_party/nix/tests/nar-access.sh new file mode 100644 index 000000000000..553d6ca89d7d --- /dev/null +++ b/third_party/nix/tests/nar-access.sh @@ -0,0 +1,44 @@ +source common.sh + +echo "building test path" +storePath="$(nix-build nar-access.nix -A a --no-out-link)" + +cd "$TEST_ROOT" + +# Dump path to nar. +narFile="$TEST_ROOT/path.nar" +nix-store --dump $storePath > $narFile + +# Check that find and ls-nar match. +( cd $storePath; find . | sort ) > files.find +nix ls-nar -R -d $narFile "" | sort > files.ls-nar +diff -u files.find files.ls-nar + +# Check that file contents of data match. +nix cat-nar $narFile /foo/data > data.cat-nar +diff -u data.cat-nar $storePath/foo/data + +# Check that file contents of baz match. +nix cat-nar $narFile /foo/baz > baz.cat-nar +diff -u baz.cat-nar $storePath/foo/baz + +nix cat-store $storePath/foo/baz > baz.cat-nar +diff -u baz.cat-nar $storePath/foo/baz + +# Test --json. +[[ $(nix ls-nar --json $narFile /) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] +[[ $(nix ls-nar --json -R $narFile /foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' ]] +[[ $(nix ls-nar --json -R $narFile /foo/bar) = '{"type":"regular","size":0,"narOffset":368}' ]] +[[ $(nix ls-store --json $storePath) = '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' ]] +[[ $(nix ls-store --json -R $storePath/foo) = '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' ]] +[[ $(nix ls-store --json -R $storePath/foo/bar) = '{"type":"regular","size":0}' ]] + +# Test missing files. +nix ls-store --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +nix ls-store $storePath/xyzzy 2>&1 | grep 'does not exist' + +# Test failure to dump. +if nix-store --dump $storePath >/dev/full ; then + echo "dumping to /dev/full should fail" + exit -1 +fi diff --git a/third_party/nix/tests/nix-build.sh b/third_party/nix/tests/nix-build.sh new file mode 100644 index 000000000000..395264863196 --- /dev/null +++ b/third_party/nix/tests/nix-build.sh @@ -0,0 +1,25 @@ +source common.sh + +clearStore + +outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) +test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR + +# The result should be retained by a GC. +echo A +target=$(readLink $TEST_ROOT/result) +echo B +echo target is $target +nix-store --gc +test -e $target/foobar + +# But now it should be gone. +rm $TEST_ROOT/result +nix-store --gc +if test -e $target/foobar; then false; fi + +outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link) +[[ $outPath = $outPath2 ]] + +outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link) +[[ $outPath = $outPath2 ]] diff --git a/third_party/nix/tests/nix-channel.sh b/third_party/nix/tests/nix-channel.sh new file mode 100644 index 000000000000..93f837befcec --- /dev/null +++ b/third_party/nix/tests/nix-channel.sh @@ -0,0 +1,59 @@ +source common.sh + +clearProfiles + +rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile + +# Test add/list/remove. +nix-channel --add http://foo/bar xyzzy +nix-channel --list | grep -q http://foo/bar +nix-channel --remove xyzzy + +[ -e $TEST_HOME/.nix-channels ] +[ "$(cat $TEST_HOME/.nix-channels)" = '' ] + +# Create a channel. +rm -rf $TEST_ROOT/foo +mkdir -p $TEST_ROOT/foo +nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix)) +rm -rf $TEST_ROOT/nixexprs +mkdir -p $TEST_ROOT/nixexprs +cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ +ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix +(cd $TEST_ROOT && tar cvf - nixexprs) | bzip2 > $TEST_ROOT/foo/nixexprs.tar.bz2 + +# Test the update action. +nix-channel --add file://$TEST_ROOT/foo +nix-channel --update + +# Do a query. +nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml +if [ "$xmllint" != false ]; then + $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" +fi +grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml + +# Do an install. +nix-env -i dependencies +[ -e $TEST_HOME/.nix-profile/foobar ] + +clearProfiles +rm -f $TEST_HOME/.nix-channels + +# Test updating from a tarball +nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 foo +nix-channel --update + +# Do a query. +nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml +if [ "$xmllint" != false ]; then + $xmllint --noout $TEST_ROOT/meta.xml || fail "malformed XML" +fi +grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="foo".*name="dependencies"' $TEST_ROOT/meta.xml + +# Do an install. +nix-env -i dependencies +[ -e $TEST_HOME/.nix-profile/foobar ] + diff --git a/third_party/nix/tests/nix-copy-closure.nix b/third_party/nix/tests/nix-copy-closure.nix new file mode 100644 index 000000000000..0dc147fb34e9 --- /dev/null +++ b/third_party/nix/tests/nix-copy-closure.nix @@ -0,0 +1,64 @@ +# Test ‘nix-copy-closure’. + +{ nixpkgs, system, nix }: + +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; + +makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in { + + nodes = + { client = + { config, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.pathsInNixDB = [ pkgA ]; + nix.package = nix; + nix.binaryCaches = [ ]; + }; + + server = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + virtualisation.pathsInNixDB = [ pkgB pkgC ]; + nix.package = nix; + }; + }; + + testScript = { nodes }: + '' + startAll; + + # Create an SSH key on the client. + my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; + $client->succeed("mkdir -m 700 /root/.ssh"); + $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); + $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + + # Install the SSH key on the server. + $server->succeed("mkdir -m 700 /root/.ssh"); + $server->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); + $server->waitForUnit("sshd"); + $client->waitForUnit("network.target"); + $client->succeed("ssh -o StrictHostKeyChecking=no " . $server->name() . " 'echo hello world'"); + + # Copy the closure of package A from the client to the server. + $server->fail("nix-store --check-validity ${pkgA}"); + $client->succeed("nix-copy-closure --to server --gzip ${pkgA} >&2"); + $server->succeed("nix-store --check-validity ${pkgA}"); + + # Copy the closure of package B from the server to the client. + $client->fail("nix-store --check-validity ${pkgB}"); + $client->succeed("nix-copy-closure --from server --gzip ${pkgB} >&2"); + $client->succeed("nix-store --check-validity ${pkgB}"); + + # Copy the closure of package C via the SSH substituter. + $client->fail("nix-store -r ${pkgC}"); + # FIXME + #$client->succeed( + # "nix-store --option use-ssh-substituter true" + # . " --option ssh-substituter-hosts root\@server" + # . " -r ${pkgC} >&2"); + #$client->succeed("nix-store --check-validity ${pkgC}"); + ''; + +}) diff --git a/third_party/nix/tests/nix-copy-ssh.sh b/third_party/nix/tests/nix-copy-ssh.sh new file mode 100644 index 000000000000..eb801548d2f1 --- /dev/null +++ b/third_party/nix/tests/nix-copy-ssh.sh @@ -0,0 +1,20 @@ +source common.sh + +clearStore +clearCache + +remoteRoot=$TEST_ROOT/store2 +chmod -R u+w "$remoteRoot" || true +rm -rf "$remoteRoot" + +outPath=$(nix-build --no-out-link dependencies.nix) + +nix copy --to "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath + +[ -f $remoteRoot$outPath/foobar ] + +clearStore + +nix copy --no-check-sigs --from "ssh://localhost?store=$NIX_STORE_DIR&remote-store=$remoteRoot%3fstore=$NIX_STORE_DIR%26real=$remoteRoot$NIX_STORE_DIR" $outPath + +[ -f $outPath/foobar ] diff --git a/third_party/nix/tests/nix-profile.sh b/third_party/nix/tests/nix-profile.sh new file mode 100644 index 000000000000..e2e0d1090804 --- /dev/null +++ b/third_party/nix/tests/nix-profile.sh @@ -0,0 +1,9 @@ +source common.sh + +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh + +user=$(whoami) +rm -rf $TEST_HOME $TEST_ROOT/profile-var +mkdir -p $TEST_HOME +USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" +USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency diff --git a/third_party/nix/tests/nix-shell.sh b/third_party/nix/tests/nix-shell.sh new file mode 100644 index 000000000000..ee502dddb955 --- /dev/null +++ b/third_party/nix/tests/nix-shell.sh @@ -0,0 +1,57 @@ +source common.sh + +clearStore + +# Test nix-shell -A +export IMPURE_VAR=foo +export SELECTED_IMPURE_VAR=baz +export NIX_BUILD_SHELL=$SHELL +output=$(nix-shell --pure shell.nix -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') + +[ "$output" = " - foo - bar" ] + +# Test --keep +output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') + +[ "$output" = " - foo - bar - baz" ] + +# Test nix-shell on a .drv +[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# Test nix-shell on a .drv symlink + +# Legacy: absolute path and .drv extension required +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell.drv +[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# New behaviour: just needs to resolve to a derivation in the store +nix-instantiate shell.nix -A shellDrv --indirect --add-root $TEST_ROOT/shell +[[ $(nix-shell --pure $TEST_ROOT/shell --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] + +# Test nix-shell -p +output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') +[ "$output" = "foo bar" ] + +# Test nix-shell shebang mode +sed -e "s|@ENV_PROG@|$(type -p env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh +chmod a+rx $TEST_ROOT/shell.shebang.sh + +output=$($TEST_ROOT/shell.shebang.sh abc def) +[ "$output" = "foo bar abc def" ] + +# Test nix-shell shebang mode for ruby +# This uses a fake interpreter that returns the arguments passed +# This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. +sed -e "s|@SHELL_PROG@|$(type -p nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb +chmod a+rx $TEST_ROOT/shell.shebang.rb + +output=$($TEST_ROOT/shell.shebang.rb abc ruby) +[ "$output" = '-e load("'"$TEST_ROOT"'/shell.shebang.rb") -- abc ruby' ] diff --git a/third_party/nix/tests/optimise-store.sh b/third_party/nix/tests/optimise-store.sh new file mode 100644 index 000000000000..61e3df2f9f7e --- /dev/null +++ b/third_party/nix/tests/optimise-store.sh @@ -0,0 +1,43 @@ +source common.sh + +clearStore + +outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) + +inode1="$(stat --format=%i $outPath1/foo)" +inode2="$(stat --format=%i $outPath2/foo)" +if [ "$inode1" != "$inode2" ]; then + echo "inodes do not match" + exit 1 +fi + +nlink="$(stat --format=%h $outPath1/foo)" +if [ "$nlink" != 3 ]; then + echo "link count incorrect" + exit 1 +fi + +outPath3=$(echo 'with import ./config.nix; mkDerivation { name = "foo3"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) + +inode3="$(stat --format=%i $outPath3/foo)" +if [ "$inode1" = "$inode3" ]; then + echo "inodes match unexpectedly" + exit 1 +fi + +nix-store --optimise + +inode1="$(stat --format=%i $outPath1/foo)" +inode3="$(stat --format=%i $outPath3/foo)" +if [ "$inode1" != "$inode3" ]; then + echo "inodes do not match" + exit 1 +fi + +nix-store --gc + +if [ -n "$(ls $NIX_STORE_DIR/.links)" ]; then + echo ".links directory not empty after GC" + exit 1 +fi diff --git a/third_party/nix/tests/parallel.builder.sh b/third_party/nix/tests/parallel.builder.sh new file mode 100644 index 000000000000..d092bc5a6bd4 --- /dev/null +++ b/third_party/nix/tests/parallel.builder.sh @@ -0,0 +1,29 @@ +echo "DOING $text" + + +# increase counter +while ! ln -s x $shared.lock 2> /dev/null; do + sleep 1 +done +test -f $shared.cur || echo 0 > $shared.cur +test -f $shared.max || echo 0 > $shared.max +new=$(($(cat $shared.cur) + 1)) +if test $new -gt $(cat $shared.max); then + echo $new > $shared.max +fi +echo $new > $shared.cur +rm $shared.lock + + +echo -n $(cat $inputs)$text > $out + +sleep $sleepTime + + +# decrease counter +while ! ln -s x $shared.lock 2> /dev/null; do + sleep 1 +done +test -f $shared.cur || echo 0 > $shared.cur +echo $(($(cat $shared.cur) - 1)) > $shared.cur +rm $shared.lock diff --git a/third_party/nix/tests/parallel.nix b/third_party/nix/tests/parallel.nix new file mode 100644 index 000000000000..23f142059f58 --- /dev/null +++ b/third_party/nix/tests/parallel.nix @@ -0,0 +1,19 @@ +{sleepTime ? 3}: + +with import ./config.nix; + +let + + mkDrv = text: inputs: mkDerivation { + name = "parallel"; + builder = ./parallel.builder.sh; + inherit text inputs shared sleepTime; + }; + + a = mkDrv "a" []; + b = mkDrv "b" [a]; + c = mkDrv "c" [a]; + d = mkDrv "d" [a]; + e = mkDrv "e" [b c d]; + +in e diff --git a/third_party/nix/tests/parallel.sh b/third_party/nix/tests/parallel.sh new file mode 100644 index 000000000000..3b7bbe5a2251 --- /dev/null +++ b/third_party/nix/tests/parallel.sh @@ -0,0 +1,56 @@ +source common.sh + + +# First, test that -jN performs builds in parallel. +echo "testing nix-build -j..." + +clearStore + +rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max + +outPath=$(nix-build -j10000 parallel.nix --no-out-link) + +echo "output path is $outPath" + +text=$(cat "$outPath") +if test "$text" != "abacade"; then exit 1; fi + +if test "$(cat $_NIX_TEST_SHARED.cur)" != 0; then fail "wrong current process count"; fi +if test "$(cat $_NIX_TEST_SHARED.max)" != 3; then fail "not enough parallelism"; fi + + +# Second, test that parallel invocations of nix-build perform builds +# in parallel, and don't block waiting on locks held by the others. +echo "testing multiple nix-build -j1..." + +clearStore + +rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max + +drvPath=$(nix-instantiate parallel.nix --argstr sleepTime 15) + +cmd="nix-store -j1 -r $drvPath" + +$cmd & +pid1=$! +echo "pid 1 is $pid1" + +$cmd & +pid2=$! +echo "pid 2 is $pid2" + +$cmd & +pid3=$! +echo "pid 3 is $pid3" + +$cmd & +pid4=$! +echo "pid 4 is $pid4" + +wait $pid1 || fail "instance 1 failed: $?" +wait $pid2 || fail "instance 2 failed: $?" +wait $pid3 || fail "instance 3 failed: $?" +wait $pid4 || fail "instance 4 failed: $?" + +if test "$(cat $_NIX_TEST_SHARED.cur)" != 0; then fail "wrong current process count"; fi +if test "$(cat $_NIX_TEST_SHARED.max)" != 3; then fail "not enough parallelism"; fi diff --git a/third_party/nix/tests/pass-as-file.sh b/third_party/nix/tests/pass-as-file.sh new file mode 100644 index 000000000000..2c0bc5031ad7 --- /dev/null +++ b/third_party/nix/tests/pass-as-file.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore + +outPath=$(nix-build --no-out-link -E " +with import ./config.nix; + +mkDerivation { + name = \"pass-as-file\"; + passAsFile = [ \"foo\" ]; + foo = [ \"xyzzy\" ]; + builder = builtins.toFile \"builder.sh\" '' + [ \"\$(basename \$fooPath)\" = .attr-1bp7cri8hplaz6hbz0v4f0nl44rl84q1sg25kgwqzipzd1mv89ic ] + [ \"\$(cat \$fooPath)\" = xyzzy ] + touch \$out + ''; +} +") diff --git a/third_party/nix/tests/placeholders.sh b/third_party/nix/tests/placeholders.sh new file mode 100644 index 000000000000..cd1bb7bc2aac --- /dev/null +++ b/third_party/nix/tests/placeholders.sh @@ -0,0 +1,20 @@ +source common.sh + +clearStore + +nix-build --no-out-link -E ' + with import ./config.nix; + + mkDerivation { + name = "placeholders"; + outputs = [ "out" "bin" "dev" ]; + buildCommand = " + echo foo1 > $out + echo foo2 > $bin + echo foo3 > $dev + [[ $(cat ${placeholder "out"}) = foo1 ]] + [[ $(cat ${placeholder "bin"}) = foo2 ]] + [[ $(cat ${placeholder "dev"}) = foo3 ]] + "; + } +' diff --git a/third_party/nix/tests/post-hook.sh b/third_party/nix/tests/post-hook.sh new file mode 100644 index 000000000000..a026572154db --- /dev/null +++ b/third_party/nix/tests/post-hook.sh @@ -0,0 +1,15 @@ +source common.sh + +clearStore + +export REMOTE_STORE=$TEST_ROOT/remote_store + +# Build the dependencies and push them to the remote store +nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook $PWD/push-to-store.sh + +clearStore + +# Ensure that we the remote store contains both the runtime and buildtime +# closure of what we've just built +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix +nix copy --from "$REMOTE_STORE" --no-require-sigs -f dependencies.nix input1_drv diff --git a/third_party/nix/tests/pure-eval.nix b/third_party/nix/tests/pure-eval.nix new file mode 100644 index 000000000000..ed25b3d45637 --- /dev/null +++ b/third_party/nix/tests/pure-eval.nix @@ -0,0 +1,3 @@ +{ + x = 123; +} diff --git a/third_party/nix/tests/pure-eval.sh b/third_party/nix/tests/pure-eval.sh new file mode 100644 index 000000000000..49c8564487c3 --- /dev/null +++ b/third_party/nix/tests/pure-eval.sh @@ -0,0 +1,18 @@ +source common.sh + +clearStore + +nix eval --pure-eval '(assert 1 + 2 == 3; true)' + +[[ $(nix eval '(builtins.readFile ./pure-eval.sh)') =~ clearStore ]] + +(! nix eval --pure-eval '(builtins.readFile ./pure-eval.sh)') + +(! nix eval --pure-eval '(builtins.currentTime)') +(! nix eval --pure-eval '(builtins.currentSystem)') + +(! nix-instantiate --pure-eval ./simple.nix) + +[[ $(nix eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") == 123 ]] +(! nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x)") +nix eval --pure-eval "((import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash-file pure-eval.nix --type sha256)\"; })).x)" diff --git a/third_party/nix/tests/push-to-store.sh b/third_party/nix/tests/push-to-store.sh new file mode 100755 index 000000000000..6aadb916ba0b --- /dev/null +++ b/third_party/nix/tests/push-to-store.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo Pushing "$@" to "$REMOTE_STORE" +printf "%s" "$OUT_PATHS" | xargs -d: nix copy --to "$REMOTE_STORE" --no-require-sigs diff --git a/third_party/nix/tests/referrers.sh b/third_party/nix/tests/referrers.sh new file mode 100644 index 000000000000..8ab8e5ddfe87 --- /dev/null +++ b/third_party/nix/tests/referrers.sh @@ -0,0 +1,36 @@ +source common.sh + +clearStore + +max=500 + +reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +touch $reference +(echo $reference && echo && echo 0) | nix-store --register-validity + +echo "making registration..." + +set +x +for ((n = 0; n < $max; n++)); do + storePath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$n + echo -n > $storePath + ref2=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$((n+1)) + if test $((n+1)) = $max; then + ref2=$reference + fi + echo $storePath; echo; echo 2; echo $reference; echo $ref2 +done > $TEST_ROOT/reg_info +set -x + +echo "registering..." + +nix-store --register-validity < $TEST_ROOT/reg_info + +echo "collecting garbage..." +ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref +nix-store --gc + +if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_STATE_DIR/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then + echo "referrers not cleaned up" + exit 1 +fi diff --git a/third_party/nix/tests/remote-builds.nix b/third_party/nix/tests/remote-builds.nix new file mode 100644 index 000000000000..b867f13b4995 --- /dev/null +++ b/third_party/nix/tests/remote-builds.nix @@ -0,0 +1,108 @@ +# Test Nix's remote build feature. + +{ nixpkgs, system, nix }: + +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; + +makeTest ( + +let + + # The configuration of the remote builders. + builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.package = nix; + nix.useSandbox = true; + }; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "if [ ${toString nr} = 5 ]; then echo FAIL; exit 1; fi; echo Hello; mkdir $out $foo; cat /proc/sys/kernel/hostname > $out/host; ln -s $out $foo/bar; sleep 10" ]; + outputs = [ "out" "foo" ]; + } + ''; + +in + +{ + + nodes = + { builder1 = builder; + builder2 = builder; + + client = + { config, pkgs, ... }: + { nix.maxJobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder1"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + { hostName = "builder2"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + } + ]; + virtualisation.writableStore = true; + virtualisation.pathsInNixDB = [ config.system.build.extraUtils ]; + nix.package = nix; + nix.binaryCaches = [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: + '' + startAll; + + # Create an SSH key on the client. + my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`; + $client->succeed("mkdir -p -m 700 /root/.ssh"); + $client->copyFileFromHost("key", "/root/.ssh/id_ed25519"); + $client->succeed("chmod 600 /root/.ssh/id_ed25519"); + + # Install the SSH key on the builders. + $client->waitForUnit("network.target"); + foreach my $builder ($builder1, $builder2) { + $builder->succeed("mkdir -p -m 700 /root/.ssh"); + $builder->copyFileFromHost("key.pub", "/root/.ssh/authorized_keys"); + $builder->waitForUnit("sshd"); + $client->succeed("ssh -o StrictHostKeyChecking=no " . $builder->name() . " 'echo hello world'"); + } + + # Perform a build and check that it was performed on the builder. + my $out = $client->succeed( + "nix-build ${expr nodes.client.config 1} 2> build-output", + "grep -q Hello build-output" + ); + $builder1->succeed("test -e $out"); + + # And a parallel build. + my ($out1, $out2) = split /\s/, + $client->succeed('nix-store -r $(nix-instantiate ${expr nodes.client.config 2})\!out $(nix-instantiate ${expr nodes.client.config 3})\!out'); + $builder1->succeed("test -e $out1 -o -e $out2"); + $builder2->succeed("test -e $out1 -o -e $out2"); + + # And a failing build. + $client->fail("nix-build ${expr nodes.client.config 5}"); + + # Test whether the build hook automatically skips unavailable builders. + $builder1->block; + $client->succeed("nix-build ${expr nodes.client.config 4}"); + ''; + +}) diff --git a/third_party/nix/tests/remote-store.sh b/third_party/nix/tests/remote-store.sh new file mode 100644 index 000000000000..77437658ead6 --- /dev/null +++ b/third_party/nix/tests/remote-store.sh @@ -0,0 +1,19 @@ +source common.sh + +clearStore + +startDaemon + +storeCleared=1 $SHELL ./user-envs.sh + +nix-store --dump-db > $TEST_ROOT/d1 +NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 +cmp $TEST_ROOT/d1 $TEST_ROOT/d2 + +nix-store --gc --max-freed 1K + +killDaemon + +user=$(whoami) +[ -e $NIX_STATE_DIR/gcroots/per-user/$user ] +[ -e $NIX_STATE_DIR/profiles/per-user/$user ] diff --git a/third_party/nix/tests/repair.sh b/third_party/nix/tests/repair.sh new file mode 100644 index 000000000000..ec7ad5dcaff4 --- /dev/null +++ b/third_party/nix/tests/repair.sh @@ -0,0 +1,77 @@ +source common.sh + +clearStore + +path=$(nix-build dependencies.nix -o $TEST_ROOT/result) +path2=$(nix-store -qR $path | grep input-2) + +nix-store --verify --check-contents -v + +hash=$(nix-hash $path2) + +# Corrupt a path and check whether nix-build --repair can fix it. +chmod u+w $path2 +touch $path2/bad + +if nix-store --verify --check-contents -v; then + echo "nix-store --verify succeeded unexpectedly" >&2 + exit 1 +fi + +# The path can be repaired by rebuilding the derivation. +nix-store --verify --check-contents --repair + +nix-store --verify-path $path2 + +# Re-corrupt and delete the deriver. Now --verify --repair should +# not work. +chmod u+w $path2 +touch $path2/bad + +nix-store --delete $(nix-store -qd $path2) + +if nix-store --verify --check-contents --repair; then + echo "nix-store --verify --repair succeeded unexpectedly" >&2 + exit 1 +fi + +nix-build dependencies.nix -o $TEST_ROOT/result --repair + +if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then + echo "path not repaired properly" >&2 + exit 1 +fi + +# Corrupt a path that has a substitute and check whether nix-store +# --verify can fix it. +clearCache + +nix copy --to file://$cacheDir $path + +chmod u+w $path2 +rm -rf $path2 + +nix-store --verify --check-contents --repair --substituters "file://$cacheDir" --no-require-sigs + +if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then + echo "path not repaired properly" >&2 + exit 1 +fi + +# Check --verify-path and --repair-path. +nix-store --verify-path $path2 + +chmod u+w $path2 +rm -rf $path2 + +if nix-store --verify-path $path2; then + echo "nix-store --verify-path succeeded unexpectedly" >&2 + exit 1 +fi + +nix-store --repair-path $path2 --substituters "file://$cacheDir" --no-require-sigs + +if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then + echo "path not repaired properly" >&2 + exit 1 +fi diff --git a/third_party/nix/tests/restricted.nix b/third_party/nix/tests/restricted.nix new file mode 100644 index 000000000000..e0ef5840209c --- /dev/null +++ b/third_party/nix/tests/restricted.nix @@ -0,0 +1 @@ +1 + 2 diff --git a/third_party/nix/tests/restricted.sh b/third_party/nix/tests/restricted.sh new file mode 100644 index 000000000000..e02becc60e38 --- /dev/null +++ b/third_party/nix/tests/restricted.sh @@ -0,0 +1,51 @@ +source common.sh + +clearStore + +nix-instantiate --restrict-eval --eval -E '1 + 2' +(! nix-instantiate --restrict-eval ./restricted.nix) +(! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) +nix-instantiate --restrict-eval ./simple.nix -I src=. +nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh + +(! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') +nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=.. + +(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel') +nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../src/nix-channel' -I src=../src + +(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>') +nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in <foo>' -I src=. + +p=$(nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)") +cmp $p restricted.sh + +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval) + +(! nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") + +nix eval --raw "(builtins.fetchurl file://$(pwd)/restricted.sh)" --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" + +(! nix eval --raw "(builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) +(! nix eval --raw "(builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz)" --restrict-eval) +(! nix eval --raw "(fetchGit git://github.com/NixOS/patchelf.git)" --restrict-eval) + +ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix +[[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] +(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix) +(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT) +(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) +nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . + +[[ $(nix eval --raw --restrict-eval -I . '(builtins.readFile "${import ./simple.nix}/hello")') == 'Hello World!' ]] + +# Check whether we can leak symlink information through directory traversal. +traverseDir="$(pwd)/restricted-traverse-me" +ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" +mkdir -p "$traverseDir" +goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')" +output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ + "(builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\")" \ + 2>&1 || :)" +echo "$output" | grep "is forbidden" +! echo "$output" | grep -F restricted-secret diff --git a/third_party/nix/tests/run.nix b/third_party/nix/tests/run.nix new file mode 100644 index 000000000000..77dcbd2a9df0 --- /dev/null +++ b/third_party/nix/tests/run.nix @@ -0,0 +1,17 @@ +with import ./config.nix; + +{ + hello = mkDerivation { + name = "hello"; + buildCommand = + '' + mkdir -p $out/bin + cat > $out/bin/hello <<EOF + #! ${shell} + who=\$1 + echo "Hello \''${who:-World} from $out/bin/hello" + EOF + chmod +x $out/bin/hello + ''; + }; +} diff --git a/third_party/nix/tests/run.sh b/third_party/nix/tests/run.sh new file mode 100644 index 000000000000..d1dbfd6bd4a6 --- /dev/null +++ b/third_party/nix/tests/run.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore +clearCache + +nix run -f run.nix hello -c hello | grep 'Hello World' +nix run -f run.nix hello -c hello NixOS | grep 'Hello NixOS' + +if ! canUseSandbox; then exit; fi + +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 + +clearStore + +path=$(nix eval --raw -f run.nix hello) + +# Note: we need the sandbox paths to ensure that the shell is +# visible in the sandbox. +nix run --sandbox-build-dir /build-tmp \ + --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ + --store $TEST_ROOT/store0 -f run.nix hello -c hello | grep 'Hello World' + +path2=$(nix run --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f run.nix hello -c $SHELL -c 'type -p hello') + +[[ $path/bin/hello = $path2 ]] + +[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] diff --git a/third_party/nix/tests/search.nix b/third_party/nix/tests/search.nix new file mode 100644 index 000000000000..fea6e7a7a647 --- /dev/null +++ b/third_party/nix/tests/search.nix @@ -0,0 +1,25 @@ +with import ./config.nix; + +{ + hello = mkDerivation rec { + name = "hello-${version}"; + version = "0.1"; + buildCommand = "touch $out"; + meta.description = "Empty file"; + }; + foo = mkDerivation rec { + name = "foo-5"; + buildCommand = '' + mkdir -p $out + echo ${name} > $out/${name} + ''; + }; + bar = mkDerivation rec { + name = "bar-3"; + buildCommand = '' + echo "Does not build successfully" + exit 1 + ''; + meta.description = "broken bar"; + }; +} diff --git a/third_party/nix/tests/search.sh b/third_party/nix/tests/search.sh new file mode 100644 index 000000000000..14da3127b0d5 --- /dev/null +++ b/third_party/nix/tests/search.sh @@ -0,0 +1,43 @@ +source common.sh + +clearStore +clearCache + +# No packages +(( $(NIX_PATH= nix search -u|wc -l) == 0 )) + +# Haven't updated cache, still nothing +(( $(nix search -f search.nix hello|wc -l) == 0 )) +(( $(nix search -f search.nix |wc -l) == 0 )) + +# Update cache, search should work +(( $(nix search -f search.nix -u hello|wc -l) > 0 )) + +# Use cache +(( $(nix search -f search.nix foo|wc -l) > 0 )) +(( $(nix search foo|wc -l) > 0 )) + +# Test --no-cache works +# No results from cache +(( $(nix search --no-cache foo |wc -l) == 0 )) +# Does find results from file pointed at +(( $(nix search -f search.nix --no-cache foo |wc -l) > 0 )) + +# Check descriptions are searched +(( $(nix search broken | wc -l) > 0 )) + +# Check search that matches nothing +(( $(nix search nosuchpackageexists | wc -l) == 0 )) + +# Search for multiple arguments +(( $(nix search hello empty | wc -l) == 3 )) + +# Multiple arguments will not exist +(( $(nix search hello broken | wc -l) == 0 )) + +## Search expressions + +# Check that empty search string matches all +nix search|grep -q foo +nix search|grep -q bar +nix search|grep -q hello diff --git a/third_party/nix/tests/secure-drv-outputs.nix b/third_party/nix/tests/secure-drv-outputs.nix new file mode 100644 index 000000000000..b4ac8ff531f8 --- /dev/null +++ b/third_party/nix/tests/secure-drv-outputs.nix @@ -0,0 +1,23 @@ +with import ./config.nix; + +{ + + good = mkDerivation { + name = "good"; + builder = builtins.toFile "builder" + '' + mkdir $out + echo > $out/good + ''; + }; + + bad = mkDerivation { + name = "good"; + builder = builtins.toFile "builder" + '' + mkdir $out + echo > $out/bad + ''; + }; + +} diff --git a/third_party/nix/tests/secure-drv-outputs.sh b/third_party/nix/tests/secure-drv-outputs.sh new file mode 100644 index 000000000000..50a9c4428d30 --- /dev/null +++ b/third_party/nix/tests/secure-drv-outputs.sh @@ -0,0 +1,36 @@ +# Test that users cannot register specially-crafted derivations that +# produce output paths belonging to other derivations. This could be +# used to inject malware into the store. + +source common.sh + +clearStore + +startDaemon + +# Determine the output path of the "good" derivation. +goodOut=$(nix-store -q $(nix-instantiate ./secure-drv-outputs.nix -A good)) + +# Instantiate the "bad" derivation. +badDrv=$(nix-instantiate ./secure-drv-outputs.nix -A bad) +badOut=$(nix-store -q $badDrv) + +# Rewrite the bad derivation to produce the output path of the good +# derivation. +rm -f $TEST_ROOT/bad.drv +sed -e "s|$badOut|$goodOut|g" < $badDrv > $TEST_ROOT/bad.drv + +# Add the manipulated derivation to the store and build it. This +# should fail. +if badDrv2=$(nix-store --add $TEST_ROOT/bad.drv); then + nix-store -r "$badDrv2" +fi + +# Now build the good derivation. +goodOut2=$(nix-build ./secure-drv-outputs.nix -A good --no-out-link) +test "$goodOut" = "$goodOut2" + +if ! test -e "$goodOut"/good; then + echo "Bad derivation stole the output path of the good derivation!" + exit 1 +fi diff --git a/third_party/nix/tests/setuid.nix b/third_party/nix/tests/setuid.nix new file mode 100644 index 000000000000..77e83c8d6c2c --- /dev/null +++ b/third_party/nix/tests/setuid.nix @@ -0,0 +1,108 @@ +# Verify that Linux builds cannot create setuid or setgid binaries. + +{ nixpkgs, system, nix }: + +with import (nixpkgs + "/nixos/lib/testing.nix") { inherit system; }; + +makeTest { + + machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.package = nix; + nix.binaryCaches = [ ]; + nix.nixPath = [ "nixpkgs=${lib.cleanSource pkgs.path}" ]; + virtualisation.pathsInNixDB = [ pkgs.stdenv pkgs.pkgsi686Linux.stdenv ]; + }; + + testScript = { nodes }: + '' + startAll; + + # Copying to /tmp should succeed. + $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # Creating a setuid binary should fail. + $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 4755 /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # Creating a setgid binary should fail. + $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # The checks should also work on 32-bit binaries. + $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> { system = "i686-linux"; }; runCommand "foo" {} " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + chmod 2755 /tmp/id + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # The tests above use fchmodat(). Test chmod() as well. + $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 0666, qw(/tmp/id) or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 666 ]]'); + + $machine->succeed("rm /tmp/id"); + + $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"chmod 04755, qw(/tmp/id) or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + + # And test fchmod(). + $machine->succeed('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 01750, \\\$x or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 1750 ]]'); + + $machine->succeed("rm /tmp/id"); + + $machine->fail('nix-build --no-sandbox -E \'(with import <nixpkgs> {}; runCommand "foo" { buildInputs = [ perl ]; } " + mkdir -p $out + cp ${pkgs.coreutils}/bin/id /tmp/id + perl -e \"my \\\$x; open \\\$x, qw(/tmp/id); chmod 04777, \\\$x or die\" + ")\' '); + + $machine->succeed('[[ $(stat -c %a /tmp/id) = 555 ]]'); + + $machine->succeed("rm /tmp/id"); + ''; + +} diff --git a/third_party/nix/tests/shell.nix b/third_party/nix/tests/shell.nix new file mode 100644 index 000000000000..eb39f9039a88 --- /dev/null +++ b/third_party/nix/tests/shell.nix @@ -0,0 +1,56 @@ +{ }: + +with import ./config.nix; + +let pkgs = rec { + setupSh = builtins.toFile "setup" '' + export VAR_FROM_STDENV_SETUP=foo + for pkg in $buildInputs; do + export PATH=$PATH:$pkg/bin + done + ''; + + stdenv = mkDerivation { + name = "stdenv"; + buildCommand = '' + mkdir -p $out + ln -s ${setupSh} $out/setup + ''; + }; + + shellDrv = mkDerivation { + name = "shellDrv"; + builder = "/does/not/exist"; + VAR_FROM_NIX = "bar"; + inherit stdenv; + }; + + # Used by nix-shell -p + runCommand = name: args: buildCommand: mkDerivation (args // { + inherit name buildCommand stdenv; + }); + + foo = runCommand "foo" {} '' + mkdir -p $out/bin + echo 'echo foo' > $out/bin/foo + chmod a+rx $out/bin/foo + ln -s ${shell} $out/bin/bash + ''; + + bar = runCommand "bar" {} '' + mkdir -p $out/bin + echo 'echo bar' > $out/bin/bar + chmod a+rx $out/bin/bar + ''; + + bash = shell; + + # ruby "interpreter" that outputs "$@" + ruby = runCommand "ruby" {} '' + mkdir -p $out/bin + echo 'printf -- "$*"' > $out/bin/ruby + chmod a+rx $out/bin/ruby + ''; + + inherit pkgs; +}; in pkgs diff --git a/third_party/nix/tests/shell.shebang.rb b/third_party/nix/tests/shell.shebang.rb new file mode 100644 index 000000000000..ea67eb09c1c6 --- /dev/null +++ b/third_party/nix/tests/shell.shebang.rb @@ -0,0 +1,7 @@ +#! @SHELL_PROG@ +#! ruby +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --pure -p ruby -i ruby + +# Contents doesn't matter. +abort("This shouldn't be executed.") diff --git a/third_party/nix/tests/shell.shebang.sh b/third_party/nix/tests/shell.shebang.sh new file mode 100755 index 000000000000..f7132043de44 --- /dev/null +++ b/third_party/nix/tests/shell.shebang.sh @@ -0,0 +1,4 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell -I nixpkgs=shell.nix --no-substitute +#! nix-shell --pure -i bash -p foo bar +echo "$(foo) $(bar) $@" diff --git a/third_party/nix/tests/signing.sh b/third_party/nix/tests/signing.sh new file mode 100644 index 000000000000..9e29e3fbf063 --- /dev/null +++ b/third_party/nix/tests/signing.sh @@ -0,0 +1,105 @@ +source common.sh + +clearStore +clearCache + +nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 +pk1=$(cat $TEST_ROOT/pk1) +nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2 +pk2=$(cat $TEST_ROOT/pk2) + +# Build a path. +outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") + +# Verify that the path got signed. +info=$(nix path-info --json $outPath) +[[ $info =~ '"ultimate":true' ]] +[[ $info =~ 'cache1.example.org' ]] +[[ $info =~ 'cache2.example.org' ]] + +# Test "nix verify". +nix verify -r $outPath + +expect 2 nix verify -r $outPath --sigs-needed 1 + +nix verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 + +expect 2 nix verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 + +nix verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +nix verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +# Build something unsigned. +outPath2=$(nix-build simple.nix --no-out-link) + +nix verify -r $outPath + +# Verify that the path did not get signed but does have the ultimate bit. +info=$(nix path-info --json $outPath2) +[[ $info =~ '"ultimate":true' ]] +(! [[ $info =~ 'signatures' ]]) + +# Test "nix verify". +nix verify -r $outPath2 + +expect 2 nix verify -r $outPath2 --sigs-needed 1 + +expect 2 nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 + +# Test "nix sign-paths". +nix sign-paths --key-file $TEST_ROOT/sk1 $outPath2 + +nix verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 + +# Build something content-addressed. +outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) + +[[ $(nix path-info --json $outPathCA) =~ '"ca":"fixed:md5:' ]] + +# Content-addressed paths don't need signatures, so they verify +# regardless of --sigs-needed. +nix verify $outPathCA +nix verify $outPathCA --sigs-needed 1000 + +# Check that signing a content-addressed path doesn't overflow validSigs +nix sign-paths --key-file $TEST_ROOT/sk1 $outPathCA +nix verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 + +# Copy to a binary cache. +nix copy --to file://$cacheDir $outPath2 + +# Verify that signatures got copied. +info=$(nix path-info --store file://$cacheDir --json $outPath2) +(! [[ $info =~ '"ultimate":true' ]]) +[[ $info =~ 'cache1.example.org' ]] +(! [[ $info =~ 'cache2.example.org' ]]) + +# Verify that adding a signature to a path in a binary cache works. +nix sign-paths --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 +info=$(nix path-info --store file://$cacheDir --json $outPath2) +[[ $info =~ 'cache1.example.org' ]] +[[ $info =~ 'cache2.example.org' ]] + +# Copying to a diverted store should fail due to a lack of valid signatures. +chmod -R u+w $TEST_ROOT/store0 || true +rm -rf $TEST_ROOT/store0 +(! nix copy --to $TEST_ROOT/store0 $outPath) + +# But succeed if we supply the public keys. +nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 + +expect 2 nix verify --store $TEST_ROOT/store0 -r $outPath + +nix verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 +nix verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" + +# It should also succeed if we disable signature checking. +(! nix copy --to $TEST_ROOT/store0 $outPath2) +nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 + +# But signatures should still get copied. +nix verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 + +# Content-addressed stuff can be copied without signatures. +nix copy --to $TEST_ROOT/store0 $outPathCA diff --git a/third_party/nix/tests/simple.builder.sh b/third_party/nix/tests/simple.builder.sh new file mode 100644 index 000000000000..569e8ca88c1e --- /dev/null +++ b/third_party/nix/tests/simple.builder.sh @@ -0,0 +1,11 @@ +echo "PATH=$PATH" + +# Verify that the PATH is empty. +if mkdir foo 2> /dev/null; then exit 1; fi + +# Set a PATH (!!! impure). +export PATH=$goodPath + +mkdir $out + +echo "Hello World!" > $out/hello \ No newline at end of file diff --git a/third_party/nix/tests/simple.nix b/third_party/nix/tests/simple.nix new file mode 100644 index 000000000000..4223c0f23a5b --- /dev/null +++ b/third_party/nix/tests/simple.nix @@ -0,0 +1,8 @@ +with import ./config.nix; + +mkDerivation { + name = "simple"; + builder = ./simple.builder.sh; + PATH = ""; + goodPath = path; +} diff --git a/third_party/nix/tests/simple.sh b/third_party/nix/tests/simple.sh new file mode 100644 index 000000000000..37631b648c67 --- /dev/null +++ b/third_party/nix/tests/simple.sh @@ -0,0 +1,25 @@ +source common.sh + +drvPath=$(nix-instantiate simple.nix) + +test "$(nix-store -q --binding system "$drvPath")" = "$system" + +echo "derivation is $drvPath" + +outPath=$(nix-store -rvv "$drvPath") + +echo "output path is $outPath" + +text=$(cat "$outPath"/hello) +if test "$text" != "Hello World!"; then exit 1; fi + +# Directed delete: $outPath is not reachable from a root, so it should +# be deleteable. +nix-store --delete $outPath +if test -e $outPath/hello; then false; fi + +outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" +if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then + echo "hashDerivationModulo appears broken, got $outPath" + exit 1 +fi diff --git a/third_party/nix/tests/structured-attrs.nix b/third_party/nix/tests/structured-attrs.nix new file mode 100644 index 000000000000..6c77a43913a7 --- /dev/null +++ b/third_party/nix/tests/structured-attrs.nix @@ -0,0 +1,66 @@ +with import ./config.nix; + +let + + dep = mkDerivation { + name = "dep"; + buildCommand = '' + mkdir $out; echo bla > $out/bla + ''; + }; + +in + +mkDerivation { + name = "structured"; + + __structuredAttrs = true; + + buildCommand = '' + set -x + + [[ $int = 123456789 ]] + [[ -z $float ]] + [[ -n $boolTrue ]] + [[ -z $boolFalse ]] + [[ -n ''${hardening[format]} ]] + [[ -z ''${hardening[fortify]} ]] + [[ ''${#buildInputs[@]} = 7 ]] + [[ ''${buildInputs[2]} = c ]] + [[ -v nothing ]] + [[ -z $nothing ]] + + mkdir ''${outputs[out]} + echo bar > $dest + + json=$(cat .attrs.json) + [[ $json =~ '"narHash":"sha256:1r7yc43zqnzl5b0als5vnyp649gk17i37s7mj00xr8kc47rjcybk"' ]] + [[ $json =~ '"narSize":288' ]] + [[ $json =~ '"closureSize":288' ]] + [[ $json =~ '"references":[]' ]] + ''; + + buildInputs = [ "a" "b" "c" 123 "'" "\"" null ]; + + hardening.format = true; + hardening.fortify = false; + + outer.inner = [ 1 2 3 ]; + + int = 123456789; + + float = 123.456; + + boolTrue = true; + boolFalse = false; + + nothing = null; + + dest = "${placeholder "out"}/foo"; + + "foo bar" = "BAD"; + "1foobar" = "BAD"; + "foo$" = "BAD"; + + exportReferencesGraph.refs = [ dep ]; +} diff --git a/third_party/nix/tests/structured-attrs.sh b/third_party/nix/tests/structured-attrs.sh new file mode 100644 index 000000000000..9ba2672b6833 --- /dev/null +++ b/third_party/nix/tests/structured-attrs.sh @@ -0,0 +1,7 @@ +source common.sh + +clearStore + +outPath=$(nix-build structured-attrs.nix --no-out-link) + +[[ $(cat $outPath/foo) = bar ]] diff --git a/third_party/nix/tests/tarball.sh b/third_party/nix/tests/tarball.sh new file mode 100644 index 000000000000..ba534c6261ad --- /dev/null +++ b/third_party/nix/tests/tarball.sh @@ -0,0 +1,28 @@ +source common.sh + +clearStore + +rm -rf $TEST_HOME + +tarroot=$TEST_ROOT/tarball +rm -rf $tarroot +mkdir -p $tarroot +cp dependencies.nix $tarroot/default.nix +cp config.nix dependencies.builder*.sh $tarroot/ + +tarball=$TEST_ROOT/tarball.tar.xz +(cd $TEST_ROOT && tar c tarball) | xz > $tarball + +nix-env -f file://$tarball -qa --out-path | grep -q dependencies + +nix-build -o $TEST_ROOT/result file://$tarball + +nix-build -o $TEST_ROOT/result '<foo>' -I foo=file://$tarball + +nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + +nix-instantiate --eval -E '1 + 2' -I fnord=file://no-such-tarball.tar.xz +nix-instantiate --eval -E 'with <fnord/xyzzy>; 1 + 2' -I fnord=file://no-such-tarball.tar.xz +(! nix-instantiate --eval -E '<fnord/xyzzy> 1' -I fnord=file://no-such-tarball.tar.xz) + +nix-instantiate --eval -E '<fnord/config.nix>' -I fnord=file://no-such-tarball.tar.xz -I fnord=. diff --git a/third_party/nix/tests/timeout.nix b/third_party/nix/tests/timeout.nix new file mode 100644 index 000000000000..d0e949e31498 --- /dev/null +++ b/third_party/nix/tests/timeout.nix @@ -0,0 +1,31 @@ +with import ./config.nix; + +{ + + infiniteLoop = mkDerivation { + name = "timeout"; + buildCommand = '' + touch $out + echo "'timeout' builder entering an infinite loop" + while true ; do echo -n .; done + ''; + }; + + silent = mkDerivation { + name = "silent"; + buildCommand = '' + touch $out + sleep 60 + ''; + }; + + closeLog = mkDerivation { + name = "silent"; + buildCommand = '' + touch $out + exec > /dev/null 2>&1 + sleep 1000000000 + ''; + }; + +} diff --git a/third_party/nix/tests/timeout.sh b/third_party/nix/tests/timeout.sh new file mode 100644 index 000000000000..eea9b5731da0 --- /dev/null +++ b/third_party/nix/tests/timeout.sh @@ -0,0 +1,40 @@ +# Test the `--timeout' option. + +source common.sh + + +set +e +messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) +status=$? +set -e + +if [ $status -ne 101 ]; then + echo "error: 'nix-store' exited with '$status'; should have exited 101" + exit 1 +fi + +if ! echo "$messages" | grep -q "timed out"; then + echo "error: build may have failed for reasons other than timeout; output:" + echo "$messages" >&2 + exit 1 +fi + +if nix-build -Q timeout.nix -A infiniteLoop --max-build-log-size 100; then + echo "build should have failed" + exit 1 +fi + +if nix-build timeout.nix -A silent --max-silent-time 2; then + echo "build should have failed" + exit 1 +fi + +if nix-build timeout.nix -A closeLog; then + echo "build should have failed" + exit 1 +fi + +if nix build -f timeout.nix silent --max-silent-time 2; then + echo "build should have failed" + exit 1 +fi diff --git a/third_party/nix/tests/user-envs.builder.sh b/third_party/nix/tests/user-envs.builder.sh new file mode 100644 index 000000000000..5fafa797f11e --- /dev/null +++ b/third_party/nix/tests/user-envs.builder.sh @@ -0,0 +1,5 @@ +mkdir $out +mkdir $out/bin +echo "#! $shell" > $out/bin/$progName +echo "echo $name" >> $out/bin/$progName +chmod +x $out/bin/$progName diff --git a/third_party/nix/tests/user-envs.nix b/third_party/nix/tests/user-envs.nix new file mode 100644 index 000000000000..1aa410cc9680 --- /dev/null +++ b/third_party/nix/tests/user-envs.nix @@ -0,0 +1,29 @@ +# Some dummy arguments... +{ foo ? "foo" +}: + +with import ./config.nix; + +assert foo == "foo"; + +let + + makeDrv = name: progName: (mkDerivation { + inherit name progName system; + builder = ./user-envs.builder.sh; + } // { + meta = { + description = "A silly test package"; + }; + }); + +in + + [ + (makeDrv "foo-1.0" "foo") + (makeDrv "foo-2.0pre1" "foo") + (makeDrv "bar-0.1" "bar") + (makeDrv "foo-2.0" "foo") + (makeDrv "bar-0.1.1" "bar") + (makeDrv "foo-0.1" "foo" // { meta.priority = 10; }) + ] diff --git a/third_party/nix/tests/user-envs.sh b/third_party/nix/tests/user-envs.sh new file mode 100644 index 000000000000..aebf6a2a2b87 --- /dev/null +++ b/third_party/nix/tests/user-envs.sh @@ -0,0 +1,181 @@ +source common.sh + +if [ -z "$storeCleared" ]; then + clearStore +fi + +clearProfiles + +# Query installed: should be empty. +test "$(nix-env -p $profiles/test -q '*' | wc -l)" -eq 0 + +mkdir -p $TEST_HOME +nix-env --switch-profile $profiles/test + +# Query available: should contain several. +test "$(nix-env -f ./user-envs.nix -qa '*' | wc -l)" -eq 6 +outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1.0) +drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) +[ -n "$outPath10" -a -n "$drvPath10" ] + +# Query descriptions. +nix-env -f ./user-envs.nix -qa '*' --description | grep -q silly +rm -rf $HOME/.nix-defexpr +ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr +nix-env -qa '*' --description | grep -q silly + +# Query the system. +nix-env -qa '*' --system | grep -q $system + +# Install "foo-1.0". +nix-env -i foo-1.0 + +# Query installed: should contain foo-1.0 now (which should be +# executable). +test "$(nix-env -q '*' | wc -l)" -eq 1 +nix-env -q '*' | grep -q foo-1.0 +test "$($profiles/test/bin/foo)" = "foo-1.0" + +# Test nix-env -qc to compare installed against available packages, and vice versa. +nix-env -qc '*' | grep -q '< 2.0' +nix-env -qac '*' | grep -q '> 1.0' + +# Test the -b flag to filter out source-only packages. +[ "$(nix-env -qab | wc -l)" -eq 1 ] + +# Test the -s flag to get package status. +nix-env -qas | grep -q 'IP- foo-1.0' +nix-env -qas | grep -q -- '--- bar-0.1' + +# Disable foo. +nix-env --set-flag active false foo +(! [ -e "$profiles/test/bin/foo" ]) + +# Enable foo. +nix-env --set-flag active true foo +[ -e "$profiles/test/bin/foo" ] + +# Store the path of foo-1.0. +outPath10_=$(nix-env -q --out-path --no-name '*' | grep foo-1.0) +echo "foo-1.0 = $outPath10" +[ "$outPath10" = "$outPath10_" ] + +# Install "foo-2.0pre1": should remove foo-1.0. +nix-env -i foo-2.0pre1 + +# Query installed: should contain foo-2.0pre1 now. +test "$(nix-env -q '*' | wc -l)" -eq 1 +nix-env -q '*' | grep -q foo-2.0pre1 +test "$($profiles/test/bin/foo)" = "foo-2.0pre1" + +# Upgrade "foo": should install foo-2.0. +NIX_PATH=nixpkgs=./user-envs.nix:$NIX_PATH nix-env -f '<nixpkgs>' -u foo + +# Query installed: should contain foo-2.0 now. +test "$(nix-env -q '*' | wc -l)" -eq 1 +nix-env -q '*' | grep -q foo-2.0 +test "$($profiles/test/bin/foo)" = "foo-2.0" + +# Store the path of foo-2.0. +outPath20=$(nix-env -q --out-path --no-name '*' | grep foo-2.0) +test -n "$outPath20" + +# Install bar-0.1, uninstall foo. +nix-env -i bar-0.1 +nix-env -e foo + +# Query installed: should only contain bar-0.1 now. +if nix-env -q '*' | grep -q foo; then false; fi +nix-env -q '*' | grep -q bar + +# Rollback: should bring "foo" back. +oldGen="$(nix-store -q --resolve $profiles/test)" +nix-env --rollback +[ "$(nix-store -q --resolve $profiles/test)" != "$oldGen" ] +nix-env -q '*' | grep -q foo-2.0 +nix-env -q '*' | grep -q bar + +# Rollback again: should remove "bar". +nix-env --rollback +nix-env -q '*' | grep -q foo-2.0 +if nix-env -q '*' | grep -q bar; then false; fi + +# Count generations. +nix-env --list-generations +test "$(nix-env --list-generations | wc -l)" -eq 7 + +# Doing the same operation twice results in the same generation, which triggers +# "lazy" behaviour and does not create a new symlink. + +nix-env -i foo +nix-env -i foo + +# Count generations. +nix-env --list-generations +test "$(nix-env --list-generations | wc -l)" -eq 8 + +# Switch to a specified generation. +nix-env --switch-generation 7 +[ "$(nix-store -q --resolve $profiles/test)" = "$oldGen" ] + +# Install foo-1.0, now using its store path. +nix-env -i "$outPath10" +nix-env -q '*' | grep -q foo-1.0 +nix-store -qR $profiles/test | grep "$outPath10" +nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve $profiles/test)" +[ "$(nix-store -q --deriver "$outPath10")" = $drvPath10 ] + +# Uninstall foo-1.0, using a symlink to its store path. +ln -sfn $outPath10/bin/foo $TEST_ROOT/symlink +nix-env -e $TEST_ROOT/symlink +if nix-env -q '*' | grep -q foo; then false; fi +(! nix-store -qR $profiles/test | grep "$outPath10") + +# Install foo-1.0, now using a symlink to its store path. +nix-env -i $TEST_ROOT/symlink +nix-env -q '*' | grep -q foo + +# Delete all old generations. +nix-env --delete-generations old + +# Run the garbage collector. This should get rid of foo-2.0 but not +# foo-1.0. +nix-collect-garbage +test -e "$outPath10" +(! [ -e "$outPath20" ]) + +# Uninstall everything +nix-env -e '*' +test "$(nix-env -q '*' | wc -l)" -eq 0 + +# Installing "foo" should only install the newest foo. +nix-env -i foo +test "$(nix-env -q '*' | grep foo- | wc -l)" -eq 1 +nix-env -q '*' | grep -q foo-2.0 + +# On the other hand, this should install both (and should fail due to +# a collision). +nix-env -e '*' +(! nix-env -i foo-1.0 foo-2.0) + +# Installing "*" should install one foo and one bar. +nix-env -e '*' +nix-env -i '*' +test "$(nix-env -q '*' | wc -l)" -eq 2 +nix-env -q '*' | grep -q foo-2.0 +nix-env -q '*' | grep -q bar-0.1.1 + +# Test priorities: foo-0.1 has a lower priority than foo-1.0, so it +# should be possible to install both without a collision. Also test +# ‘--set-flag priority’ to manually override the declared priorities. +nix-env -e '*' +nix-env -i foo-0.1 foo-1.0 +[ "$($profiles/test/bin/foo)" = "foo-1.0" ] +nix-env --set-flag priority 1 foo-0.1 +[ "$($profiles/test/bin/foo)" = "foo-0.1" ] + +# Test nix-env --set. +nix-env --set $outPath10 +[ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] +nix-env --set $drvPath10 +[ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] diff --git a/third_party/nixery/default.nix b/third_party/nixery/default.nix new file mode 100644 index 000000000000..9262f1461401 --- /dev/null +++ b/third_party/nixery/default.nix @@ -0,0 +1,18 @@ +# Import the Nixery repository as-is, but pass our own package set +# instead of the pin it has. +{ depot, pkgs, ... }: + +let + inherit (depot.nix.readTree) drvTargets; + + commit = "6c4a69fa4280f0154ce257a1dfd23fb463c1ec5b"; + src = pkgs.fetchFromGitHub { + owner = "google"; + repo = "nixery"; + rev = commit; + sha256 = "0g1v7020vjiqa3k07cfm3jzg3gc6nl3hs3vyw4c9s5v0dmmhhsz4"; + }; +in drvTargets (import src { + inherit pkgs; + commitHash = _: commit; +}) diff --git a/third_party/nixpkgs/default.nix b/third_party/nixpkgs/default.nix new file mode 100644 index 000000000000..9b9afb9262ec --- /dev/null +++ b/third_party/nixpkgs/default.nix @@ -0,0 +1,71 @@ +# This file imports the pinned nixpkgs sets and applies relevant +# modifications, such as our overlays. +# +# Note that the attribute exposed by this (third_party.nixpkgs) is +# "special" in that the fixpoint used as readTree's config parameter +# in //default.nix passes this attribute as the `pkgs` argument to all +# readTree derivations. + +{ depot, externalArgs, ... }: + +let + # This provides the sources of nixpkgs. We track both + # nixos-unstable, and the current stable channel of the latest NixOS + # release. + + # Tracking nixpkgs master as of 2021-12-07. + unstableHashes = { + commit = "f4eddfe2fdef11dcbcc4139c6b91b78a9e1c28c5"; + sha256 = "196agg765k1ai9m2nh7hgc5lzhff9dkv80q2ij0yyacngjbzy5dd"; + }; + + # Tracking nixos-21.05 as of 2021-12-07. + stableHashes = { + commit = "4a8751d694b46340072d95409804f59b98326f44"; + sha256 = "14pmmd82s0anknd6g57i0c1795766bvylqalf9kz730q1v0ransl"; + }; + + # import the nixos-unstable package set, or optionally use the + # source (e.g. a path) specified by the `nixpkgsBisectPath` + # argument. This is intended for use-cases where the depot is + # bisected against nixpkgs to find the root cause of an issue in a + # channel bump. + nixpkgsSrc = externalArgs.nixpkgsBisectPath or (fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/${unstableHashes.commit}.tar.gz"; + sha256 = unstableHashes.sha256; + }); + + stableNixpkgsSrc = fetchTarball { + url = "https://github.com/NixOS/nixpkgs/archive/${stableHashes.commit}.tar.gz"; + sha256 = stableHashes.sha256; + }; + + # Stable package set is imported, but not exposed, to overlay + # required packages into the unstable set. + stableNixpkgs = import stableNixpkgsSrc {}; + + # Overlay for packages that should come from the stable channel + # instead (e.g. because something is broken in unstable). + stableOverlay = self: super: { + # Nothing picked from stable presently. + }; + + # Overlay to expose the nixpkgs commits we are using to other Nix code. + commitsOverlay = _: _: { + nixpkgsCommits = { + unstable = unstableHashes.commit; + stable = stableHashes.commit; + }; + }; +in import nixpkgsSrc { + config.allowUnfree = true; + config.allowBroken = true; + overlays = [ + commitsOverlay + stableOverlay + depot.third_party.overlays.haskell + depot.third_party.overlays.emacs + depot.third_party.overlays.tvl + depot.third_party.overlays.ecl-static + ]; +} diff --git a/third_party/nsfv/default.nix b/third_party/nsfv/default.nix new file mode 100644 index 000000000000..26d5488c42ca --- /dev/null +++ b/third_party/nsfv/default.nix @@ -0,0 +1,23 @@ +# Real-time Noise Suppression Plugin (for PulseAudio). +# +# This should be invoked as a "ladspa" plugin for pulseaudio. +# +# https://github.com/werman/noise-suppression-for-voice +{ pkgs, lib, ... }: + +pkgs.stdenv.mkDerivation { + name = "noise-suppression-for-voice"; + + nativeBuildInputs = [ pkgs.cmake ]; + src = pkgs.fetchFromGitHub { + owner = "werman"; + repo = "noise-suppression-for-voice"; + rev = "cd5c79378ab9819cd85f4fd108f3c77a40fd66ac"; + sha256 = "10zh1al1bys60sjdd4p72qbp9jfb5wq1zaw33b595psgwmqpbckq"; + }; + + meta = with lib; { + description = "Real-time noise suppression LADSPA plugin"; + license = licenses.gpl3; + }; +} diff --git a/third_party/overlays/ecl-static.nix b/third_party/overlays/ecl-static.nix new file mode 100644 index 000000000000..beda6641a0a6 --- /dev/null +++ b/third_party/overlays/ecl-static.nix @@ -0,0 +1,41 @@ +{ ... }: + +self: super: + +{ + # Statically linked ECL with statically linked dependencies. + # Works quite well, but solving this properly in a nixpkgs + # context will require figuring out cross compilation (for + # pkgsStatic), so we're gonna use this override for now. + # + # Note that ecl-static does mean that we have things + # statically linked against GMP and ECL which are LGPL. + # I believe this should be alright: The way ppl are gonna + # interact with the distributed binaries (i. e. the binary + # cache) is Nix in the depot monorepo, so the separability + # requirement should be satisfied: Source code or overriding + # would be available as ways to swap out the used GMP in the + # program. + # See https://www.gnu.org/licenses/gpl-faq.en.html#LGPLStaticVsDynamic + ecl-static = (super.pkgsMusl.ecl.override { + inherit (self.pkgsStatic) gmp libffi boehmgc; + }).overrideAttrs (drv: { + # Patches that make .fasc files concatenable again + patches = drv.patches ++ [ + (self.fetchpatch { + name = "make-bytecode-fasl-concatenatable-1.patch"; + url = "https://gitlab.com/embeddable-common-lisp/ecl/-/commit/fbb75a0fc524e3280d89d8abf3be2ee9924955c8.patch"; + sha256 = "0k6cx1bh835rl0j0wbbi5nj0aa2rwbyfyz5q2jw643iqc62l16kv"; + }) + (self.fetchpatch { + name = "make-bytecode-fasl-concatenatable-2.patch"; + url = "https://gitlab.com/embeddable-common-lisp/ecl/-/commit/a8b1c0da43f89800d09c23a27832d0b4c9dcc1e8.patch"; + sha256 = "18hl79lss0dxglpa34hszqb6ajvs8rs4b4g1qlrqrvgh1gs667n0"; + }) + ]; + configureFlags = drv.configureFlags ++ [ + "--disable-shared" + "--with-dffi=no" # will fail at runtime anyways if statically linked + ]; + }); +} diff --git a/third_party/overlays/emacs.nix b/third_party/overlays/emacs.nix new file mode 100644 index 000000000000..4b55a0081744 --- /dev/null +++ b/third_party/overlays/emacs.nix @@ -0,0 +1,11 @@ +# Emacs overlay from https://github.com/nix-community/emacs-overlay +{ ... }: + +let + # from 2021-12-07 + commit = "2f14e98a92505c517a8364c3a5e614592fbea4f4"; + src = builtins.fetchTarball { + url = "https://github.com/nix-community/emacs-overlay/archive/${commit}.tar.gz"; + sha256 = "16i16pm3bm43fj85n5bafghak0asi7p9xpyshpls6yyh85chcdb5"; + }; +in import src diff --git a/third_party/overlays/haskell/default.nix b/third_party/overlays/haskell/default.nix new file mode 100644 index 000000000000..ccabe0a90e4e --- /dev/null +++ b/third_party/overlays/haskell/default.nix @@ -0,0 +1,26 @@ +# Defines an overlay for overriding Haskell packages, for example to +# avoid breakage currently present in nixpkgs or to modify package +# versions. + +{ lib, ... }: + +self: super: # overlay parameters for the nixpkgs overlay + +let + overrides = hsSelf: hsSuper: with super.haskell.lib; { + generic-arbitrary = appendPatch hsSuper.generic-arbitrary + [ ./patches/generic-arbitrary-export-garbitrary.patch ]; + + nix-diff = overrideSrc hsSuper.nix-diff { + src = builtins.fetchTarball { + # https://github.com/Gabriel439/nix-diff/pull/50 + url = "https://github.com/Profpatsch/nix-diff/archive/6234936b26cd144d321039a44cf1791dcfaee026.tar.gz"; + sha256 = "1i3ahh8jmiayf29i1xrl2n1dkldgdmrllmccxi4yx6hbcd0r2b1l"; + }; + }; + }; +in { + haskellPackages = super.haskellPackages.override { + inherit overrides; + }; +} diff --git a/third_party/overlays/haskell/patches/generic-arbitrary-export-garbitrary.patch b/third_party/overlays/haskell/patches/generic-arbitrary-export-garbitrary.patch new file mode 100644 index 000000000000..f0c936bfca18 --- /dev/null +++ b/third_party/overlays/haskell/patches/generic-arbitrary-export-garbitrary.patch @@ -0,0 +1,12 @@ +diff --git a/src/Test/QuickCheck/Arbitrary/Generic.hs b/src/Test/QuickCheck/Arbitrary/Generic.hs +index fed6ab3..91f59f1 100644 +--- a/src/Test/QuickCheck/Arbitrary/Generic.hs ++++ b/src/Test/QuickCheck/Arbitrary/Generic.hs +@@ -23,6 +23,7 @@ The generated 'arbitrary' method is equivalent to + + module Test.QuickCheck.Arbitrary.Generic + ( Arbitrary(..) ++ , GArbitrary + , genericArbitrary + , genericShrink + ) where diff --git a/third_party/overlays/patches/notmuch-dottime.patch b/third_party/overlays/patches/notmuch-dottime.patch new file mode 100644 index 000000000000..7a9cfc6cc2a9 --- /dev/null +++ b/third_party/overlays/patches/notmuch-dottime.patch @@ -0,0 +1,81 @@ +From 569438172fa0e38129de4e61a72e06eff3330dca Mon Sep 17 00:00:00 2001 +From: Vincent Ambo <tazjin@google.com> +Date: Thu, 10 Dec 2020 10:53:47 +0100 +Subject: [PATCH] time: Use dottime for formatting non-relative timestamps + +--- + notmuch-time.c | 10 +++++----- + util/gmime-extra.c | 7 +++++-- + util/gmime-extra.h | 2 ++ + 3 files changed, 12 insertions(+), 7 deletions(-) + +diff --git a/notmuch-time.c b/notmuch-time.c +index cc7ffc23..3030a667 100644 +--- a/notmuch-time.c ++++ b/notmuch-time.c +@@ -50,8 +50,8 @@ notmuch_time_relative_date (const void *ctx, time_t then) + time_t delta; + char *result; + +- localtime_r (&now, &tm_now); +- localtime_r (&then, &tm_then); ++ gmtime_r (&now, &tm_now); ++ gmtime_r (&then, &tm_then); + + result = talloc_zero_size (ctx, RELATIVE_DATE_MAX); + if (result == NULL) +@@ -78,16 +78,16 @@ notmuch_time_relative_date (const void *ctx, time_t then) + if (tm_then.tm_wday == tm_now.tm_wday && + delta < DAY) { + strftime (result, RELATIVE_DATE_MAX, +- "Today %R", &tm_then); /* Today 12:30 */ ++ "Today %k·%M", &tm_then); /* Today 12·30 */ + return result; + } else if ((tm_now.tm_wday + 7 - tm_then.tm_wday) % 7 == 1) { + strftime (result, RELATIVE_DATE_MAX, +- "Yest. %R", &tm_then); /* Yest. 12:30 */ ++ "Yest. %k·%M", &tm_then); /* Yest. 12·30 */ + return result; + } else { + if (tm_then.tm_wday != tm_now.tm_wday) { + strftime (result, RELATIVE_DATE_MAX, +- "%a. %R", &tm_then); /* Mon. 12:30 */ ++ "%a. %k·%M", &tm_then); /* Mon. 12·30 */ + return result; + } + } +diff --git a/util/gmime-extra.c b/util/gmime-extra.c +index 04d8ed3d..868a2f69 100644 +--- a/util/gmime-extra.c ++++ b/util/gmime-extra.c +@@ -131,10 +131,13 @@ g_mime_message_get_date_string (void *ctx, GMimeMessage *message) + GDateTime *parsed_date = g_mime_message_get_date (message); + + if (parsed_date) { +- char *date = g_mime_utils_header_format_date (parsed_date); ++ char *date = g_date_time_format( ++ parsed_date, ++ "%a, %d %b %Y %H·%M%z" ++ ); + return g_string_talloc_strdup (ctx, date); + } else { +- return talloc_strdup (ctx, "Thu, 01 Jan 1970 00:00:00 +0000"); ++ return talloc_strdup (ctx, "Thu, 01 Jan 1970 00·00:00"); + } + } + +diff --git a/util/gmime-extra.h b/util/gmime-extra.h +index 094309ec..e6c98f8d 100644 +--- a/util/gmime-extra.h ++++ b/util/gmime-extra.h +@@ -1,5 +1,7 @@ + #ifndef _GMIME_EXTRA_H + #define _GMIME_EXTRA_H ++#include <glib.h> ++#include <glib/gprintf.h> + #include <gmime/gmime.h> + #include <talloc.h> + +-- +2.29.2.576.ga3fc446d84-goog + diff --git a/third_party/overlays/tvl.nix b/third_party/overlays/tvl.nix new file mode 100644 index 000000000000..3e9607984ef3 --- /dev/null +++ b/third_party/overlays/tvl.nix @@ -0,0 +1,62 @@ +# This overlay is used to make TVL-specific modifications in the +# nixpkgs tree, where required. +{ depot, ... }: + +self: super: { + # Rollback Nix to a stable version (2.3) while there is lots of + # random ecosystem breakage with the newer versions. + nix = super.nix_2_3; + + # Required for apereo-cas + # TODO(lukegb): Document why? + gradle_6 = self.callPackage (super.gradleGen { + version = "6.5.1"; + nativeVersion = "0.22-milestone-3"; + sha256 = "0jmmipjh4fbsn92zpifa5cqg5ws2a4ha0s4jzqhrg4zs542x79sh"; + }) { + java = self.jdk11; + }; + + clang-tools_11 = self.clang-tools.override { + llvmPackages = self.llvmPackages_11; + }; + + # stdenv which uses clang, lld and libc++; full is a slight exaggeration, + # we for example don't use LLVM's libunwind + fullLlvm11Stdenv = self.overrideCC self.stdenv + (self.llvmPackages_11.libcxxStdenv.cc.override { + inherit (self.llvmPackages_11) bintools; + }); + + # Add our Emacs packages to the fixpoint + emacsPackagesFor = emacs: ( + (super.emacsPackagesFor emacs).overrideScope' (eself: esuper: { + tvlPackages = depot.tools.emacs-pkgs // depot.third_party.emacs; + + # Use the notmuch from nixpkgs instead of from the Emacs + # overlay, to avoid versions being out of sync. + notmuch = super.notmuch.emacs; + + # Build EXWM with the depot sources instead. + exwm = esuper.exwm.overrideAttrs(_: { + src = depot.path.origSrc + "/third_party/exwm"; + }); + }) + ); + + # dottime support for notmuch + notmuch = super.notmuch.overrideAttrs(old: { + passthru = old.passthru // { + patches = old.patches ++ [ ./patches/notmuch-dottime.patch ]; + }; + }); + + # Fix Steam issues with web views (nixpkgs#137279) + steam = super.steam.override { + extraPkgs = pkgs: with pkgs; [ pango harfbuzz libthai ]; + }; + + # nix-serve does not work with nix 2.4 + # https://github.com/edolstra/nix-serve/issues/28 + nix-serve = super.nix-serve.override { nix = super.nix_2_3; }; +} diff --git a/third_party/prometheus-fail2ban-exporter/default.nix b/third_party/prometheus-fail2ban-exporter/default.nix new file mode 100644 index 000000000000..bed5bd630161 --- /dev/null +++ b/third_party/prometheus-fail2ban-exporter/default.nix @@ -0,0 +1,17 @@ +{ pkgs, ... }: + +let + script = pkgs.fetchurl { + url = "https://raw.githubusercontent.com/jangrewe/prometheus-fail2ban-exporter/11066950b47bb2dbef96ea8544f76e46ed829e81/fail2ban-exporter.py"; + sha256 = "049lsvw1nj65bbvp8ygyz3743ayzdawrbjixaxmpm03qbrcfmwc4"; + }; + + python = pkgs.python3.withPackages (p: [ + p.prometheus_client + ]); + +in pkgs.writeShellScriptBin "prometheus-fail2ban-exporter" '' + set -eo pipefail + + exec "${python}/bin/python" "${script}" +'' diff --git a/third_party/protobuf/default.nix b/third_party/protobuf/default.nix new file mode 100644 index 000000000000..b30ff9de918c --- /dev/null +++ b/third_party/protobuf/default.nix @@ -0,0 +1,12 @@ +# Pin protobuf to version 3.12, with LLVM. +{ depot, pkgs, ... }: + +pkgs.callPackage "${pkgs.path}/pkgs/development/libraries/protobuf/generic-v3.nix" { + version = "3.12.2"; + sha256 = "1lp368aa206vpic9fmax4k6llnmf28plfvkkm4vqhgphmjqykvl2"; + stdenv = pkgs.fullLlvm11Stdenv; + buildPackages = { + inherit (pkgs) which; + stdenv = pkgs.fullLlvm11Stdenv; + }; +} diff --git a/third_party/python/broadlink/.gitignore b/third_party/python/broadlink/.gitignore new file mode 100644 index 000000000000..0d20b6487c61 --- /dev/null +++ b/third_party/python/broadlink/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/third_party/python/broadlink/LICENSE b/third_party/python/broadlink/LICENSE new file mode 100644 index 000000000000..d8c801656bdf --- /dev/null +++ b/third_party/python/broadlink/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mike Ryan +Copyright (c) 2016 Matthew Garrett + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/third_party/python/broadlink/README.md b/third_party/python/broadlink/README.md new file mode 100644 index 000000000000..8faba2be75fc --- /dev/null +++ b/third_party/python/broadlink/README.md @@ -0,0 +1,112 @@ +Python control for Broadlink RM2, RM3 and RM4 series controllers +=============================================== + +A simple Python API for controlling IR/RF controllers from [Broadlink](http://www.ibroadlink.com/rm/). At present, the following devices are currently supported: + +* RM Pro (referred to as RM2 in the codebase) +* A1 sensor platform devices are supported +* RM3 mini IR blaster +* RM4 and RM4C mini blasters + +There is currently no support for the cloud API. + +Example use +----------- + +Setup a new device on your local wireless network: + +1. Put the device into AP Mode + 1. Long press the reset button until the blue LED is blinking quickly. + 2. Long press again until blue LED is blinking slowly. + 3. Manually connect to the WiFi SSID named BroadlinkProv. +2. Run setup() and provide your ssid, network password (if secured), and set the security mode + 1. Security mode options are (0 = none, 1 = WEP, 2 = WPA1, 3 = WPA2, 4 = WPA1/2) +``` +import broadlink + +broadlink.setup('myssid', 'mynetworkpass', 3) +``` + +Discover available devices on the local network: +``` +import broadlink + +devices = broadlink.discover(timeout=5) +``` + +Obtain the authentication key required for further communication: +``` +devices[0].auth() +``` + +Enter learning mode: +``` +devices[0].enter_learning() +``` + +Sweep RF frequencies: +``` +devices[0].sweep_frequency() +``` + +Cancel sweep RF frequencies: +``` +devices[0].cancel_sweep_frequency() +``` +Check whether a frequency has been found: +``` +found = devices[0].check_frequency() +``` +(This will return True if the RM has locked onto a frequency, False otherwise) + +Attempt to learn an RF packet: +``` +found = devices[0].find_rf_packet() +``` +(This will return True if a packet has been found, False otherwise) + +Obtain an IR or RF packet while in learning mode: +``` +ir_packet = devices[0].check_data() +``` +(This will return None if the device does not have a packet to return) + +Send an IR or RF packet: +``` +devices[0].send_data(ir_packet) +``` + +Obtain temperature data from an RM2: +``` +devices[0].check_temperature() +``` + +Obtain sensor data from an A1: +``` +data = devices[0].check_sensors() +``` + +Set power state on a SmartPlug SP2/SP3: +``` +devices[0].set_power(True) +``` + +Check power state on a SmartPlug: +``` +state = devices[0].check_power() +``` + +Check energy consumption on a SmartPlug: +``` +state = devices[0].get_energy() +``` + +Set power state for S1 on a SmartPowerStrip MP1: +``` +devices[0].set_power(1, True) +``` + +Check power state on a SmartPowerStrip: +``` +state = devices[0].check_power() +``` diff --git a/third_party/python/broadlink/broadlink/__init__.py b/third_party/python/broadlink/broadlink/__init__.py new file mode 100644 index 000000000000..0c70c4beaed3 --- /dev/null +++ b/third_party/python/broadlink/broadlink/__init__.py @@ -0,0 +1,1118 @@ +#!/usr/bin/python + +import codecs +import json +import random +import socket +import struct +import threading +import time +from datetime import datetime + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + + +def gendevice(devtype, host, mac, name=None, cloud=None): + devices = { + sp1: [0], + sp2: [0x2711, # SP2 + 0x2719, 0x7919, 0x271a, 0x791a, # Honeywell SP2 + 0x2720, # SPMini + 0x753e, # SP3 + 0x7D00, # OEM branded SP3 + 0x947a, 0x9479, # SP3S + 0x2728, # SPMini2 + 0x2733, 0x273e, # OEM branded SPMini + 0x7530, 0x7546, 0x7918, # OEM branded SPMini2 + 0x7D0D, # TMall OEM SPMini3 + 0x2736 # SPMiniPlus + ], + rm: [0x2712, # RM2 + 0x2737, # RM Mini + 0x273d, # RM Pro Phicomm + 0x2783, # RM2 Home Plus + 0x277c, # RM2 Home Plus GDT + 0x272a, # RM2 Pro Plus + 0x2787, # RM2 Pro Plus2 + 0x279d, # RM2 Pro Plus3 + 0x27a9, # RM2 Pro Plus_300 + 0x278b, # RM2 Pro Plus BL + 0x2797, # RM2 Pro Plus HYC + 0x27a1, # RM2 Pro Plus R1 + 0x27a6, # RM2 Pro PP + 0x278f, # RM Mini Shate + 0x27c2, # RM Mini 3 + 0x27d1, # new RM Mini3 + 0x27de # RM Mini 3 (C) + ], + rm4: [0x51da, # RM4 Mini + 0x5f36, # RM Mini 3 + 0x6026, # RM4 Pro + 0x610e, # RM4 Mini + 0x610f, # RM4c + 0x62bc, # RM4 Mini + 0x62be # RM4c + ], + a1: [0x2714], # A1 + mp1: [0x4EB5, # MP1 + 0x4EF7 # Honyar oem mp1 + ], + hysen: [0x4EAD], # Hysen controller + S1C: [0x2722], # S1 (SmartOne Alarm Kit) + dooya: [0x4E4D], # Dooya DT360E (DOOYA_CURTAIN_V2) + bg1: [0x51E3], # BG Electrical Smart Power Socket + lb1 : [0x60c8] # RGB Smart Bulb + } + + # Look for the class associated to devtype in devices + [device_class] = [dev for dev in devices if devtype in devices[dev]] or [None] + if device_class is None: + return device(host, mac, devtype, name=name, cloud=cloud) + return device_class(host, mac, devtype, name=name, cloud=cloud) + + +def discover(timeout=None, local_ip_address=None, discover_ip_address='255.255.255.255', max_devices=100): + if local_ip_address is None: + local_ip_address = socket.gethostbyname(socket.gethostname()) + if local_ip_address.startswith('127.'): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + s.connect(('8.8.8.8', 53)) # connecting to a UDP address doesn't send packets + local_ip_address = s.getsockname()[0] + address = local_ip_address.split('.') + cs = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + cs.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + cs.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + cs.bind((local_ip_address, 0)) + port = cs.getsockname()[1] + starttime = time.time() + + devices = [] + + timezone = int(time.timezone / -3600) + packet = bytearray(0x30) + + year = datetime.now().year + + if timezone < 0: + packet[0x08] = 0xff + timezone - 1 + packet[0x09] = 0xff + packet[0x0a] = 0xff + packet[0x0b] = 0xff + else: + packet[0x08] = timezone + packet[0x09] = 0 + packet[0x0a] = 0 + packet[0x0b] = 0 + packet[0x0c] = year & 0xff + packet[0x0d] = year >> 8 + packet[0x0e] = datetime.now().minute + packet[0x0f] = datetime.now().hour + subyear = str(year)[2:] + packet[0x10] = int(subyear) + packet[0x11] = datetime.now().isoweekday() + packet[0x12] = datetime.now().day + packet[0x13] = datetime.now().month + packet[0x18] = int(address[0]) + packet[0x19] = int(address[1]) + packet[0x1a] = int(address[2]) + packet[0x1b] = int(address[3]) + packet[0x1c] = port & 0xff + packet[0x1d] = port >> 8 + packet[0x26] = 6 + + checksum = 0xbeaf + for b in packet: + checksum = (checksum + b) & 0xffff + + packet[0x20] = checksum & 0xff + packet[0x21] = checksum >> 8 + + cs.sendto(packet, (discover_ip_address, 80)) + if timeout is None: + response = cs.recvfrom(1024) + responsepacket = bytearray(response[0]) + host = response[1] + devtype = responsepacket[0x34] | responsepacket[0x35] << 8 + mac = responsepacket[0x3a:0x40] + name = responsepacket[0x40:].split(b'\x00')[0].decode('utf-8') + cloud = bool(responsepacket[-1]) + device = gendevice(devtype, host, mac, name=name, cloud=cloud) + return device + + while ((time.time() - starttime) < timeout) and (len(devices) < max_devices): + cs.settimeout(timeout - (time.time() - starttime)) + try: + response = cs.recvfrom(1024) + except socket.timeout: + return devices + responsepacket = bytearray(response[0]) + host = response[1] + devtype = responsepacket[0x34] | responsepacket[0x35] << 8 + mac = responsepacket[0x3a:0x40] + name = responsepacket[0x40:].split(b'\x00')[0].decode('utf-8') + cloud = bool(responsepacket[-1]) + device = gendevice(devtype, host, mac, name=name, cloud=cloud) + devices.append(device) + return devices + + +class device: + def __init__(self, host, mac, devtype, timeout=10, name=None, cloud=None): + self.host = host + self.mac = mac.encode() if isinstance(mac, str) else mac + self.devtype = devtype if devtype is not None else 0x272a + self.name = name + self.cloud = cloud + self.timeout = timeout + self.count = random.randrange(0xffff) + self.iv = bytearray( + [0x56, 0x2e, 0x17, 0x99, 0x6d, 0x09, 0x3d, 0x28, 0xdd, 0xb3, 0xba, 0x69, 0x5a, 0x2e, 0x6f, 0x58]) + self.id = bytearray([0, 0, 0, 0]) + self.cs = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.cs.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self.cs.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + self.cs.bind(('', 0)) + self.type = "Unknown" + self.lock = threading.Lock() + + self.aes = None + key = bytearray( + [0x09, 0x76, 0x28, 0x34, 0x3f, 0xe9, 0x9e, 0x23, 0x76, 0x5c, 0x15, 0x13, 0xac, 0xcf, 0x8b, 0x02]) + self.update_aes(key) + + def update_aes(self, key): + self.aes = Cipher(algorithms.AES(key), modes.CBC(self.iv), + backend=default_backend()) + + def encrypt(self, payload): + encryptor = self.aes.encryptor() + return encryptor.update(payload) + encryptor.finalize() + + def decrypt(self, payload): + decryptor = self.aes.decryptor() + return decryptor.update(payload) + decryptor.finalize() + + def auth(self): + payload = bytearray(0x50) + payload[0x04] = 0x31 + payload[0x05] = 0x31 + payload[0x06] = 0x31 + payload[0x07] = 0x31 + payload[0x08] = 0x31 + payload[0x09] = 0x31 + payload[0x0a] = 0x31 + payload[0x0b] = 0x31 + payload[0x0c] = 0x31 + payload[0x0d] = 0x31 + payload[0x0e] = 0x31 + payload[0x0f] = 0x31 + payload[0x10] = 0x31 + payload[0x11] = 0x31 + payload[0x12] = 0x31 + payload[0x1e] = 0x01 + payload[0x2d] = 0x01 + payload[0x30] = ord('T') + payload[0x31] = ord('e') + payload[0x32] = ord('s') + payload[0x33] = ord('t') + payload[0x34] = ord(' ') + payload[0x35] = ord(' ') + payload[0x36] = ord('1') + + response = self.send_packet(0x65, payload) + + if any(response[0x22:0x24]): + return False + + payload = self.decrypt(response[0x38:]) + + key = payload[0x04:0x14] + if len(key) % 16 != 0: + return False + + self.id = payload[0x00:0x04] + self.update_aes(key) + + return True + + def get_type(self): + return self.type + + def send_packet(self, command, payload): + self.count = (self.count + 1) & 0xffff + packet = bytearray(0x38) + packet[0x00] = 0x5a + packet[0x01] = 0xa5 + packet[0x02] = 0xaa + packet[0x03] = 0x55 + packet[0x04] = 0x5a + packet[0x05] = 0xa5 + packet[0x06] = 0xaa + packet[0x07] = 0x55 + packet[0x24] = self.devtype & 0xff + packet[0x25] = self.devtype >> 8 + packet[0x26] = command + packet[0x28] = self.count & 0xff + packet[0x29] = self.count >> 8 + packet[0x2a] = self.mac[0] + packet[0x2b] = self.mac[1] + packet[0x2c] = self.mac[2] + packet[0x2d] = self.mac[3] + packet[0x2e] = self.mac[4] + packet[0x2f] = self.mac[5] + packet[0x30] = self.id[0] + packet[0x31] = self.id[1] + packet[0x32] = self.id[2] + packet[0x33] = self.id[3] + + # pad the payload for AES encryption + if payload: + payload += bytearray((16 - len(payload)) % 16) + + checksum = 0xbeaf + for b in payload: + checksum = (checksum + b) & 0xffff + + packet[0x34] = checksum & 0xff + packet[0x35] = checksum >> 8 + + payload = self.encrypt(payload) + for i in range(len(payload)): + packet.append(payload[i]) + + checksum = 0xbeaf + for b in packet: + checksum = (checksum + b) & 0xffff + + packet[0x20] = checksum & 0xff + packet[0x21] = checksum >> 8 + + start_time = time.time() + with self.lock: + while True: + try: + self.cs.sendto(packet, self.host) + self.cs.settimeout(1) + response = self.cs.recvfrom(2048) + break + except socket.timeout: + if (time.time() - start_time) > self.timeout: + raise + return bytearray(response[0]) + + +class mp1(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "MP1" + + def set_power_mask(self, sid_mask, state): + """Sets the power state of the smart power strip.""" + + packet = bytearray(16) + packet[0x00] = 0x0d + packet[0x02] = 0xa5 + packet[0x03] = 0xa5 + packet[0x04] = 0x5a + packet[0x05] = 0x5a + packet[0x06] = 0xb2 + ((sid_mask << 1) if state else sid_mask) + packet[0x07] = 0xc0 + packet[0x08] = 0x02 + packet[0x0a] = 0x03 + packet[0x0d] = sid_mask + packet[0x0e] = sid_mask if state else 0 + + self.send_packet(0x6a, packet) + + def set_power(self, sid, state): + """Sets the power state of the smart power strip.""" + sid_mask = 0x01 << (sid - 1) + return self.set_power_mask(sid_mask, state) + + def check_power_raw(self): + """Returns the power state of the smart power strip in raw format.""" + packet = bytearray(16) + packet[0x00] = 0x0a + packet[0x02] = 0xa5 + packet[0x03] = 0xa5 + packet[0x04] = 0x5a + packet[0x05] = 0x5a + packet[0x06] = 0xae + packet[0x07] = 0xc0 + packet[0x08] = 0x01 + + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + if isinstance(payload[0x4], int): + state = payload[0x0e] + else: + state = ord(payload[0x0e]) + return state + + def check_power(self): + """Returns the power state of the smart power strip.""" + state = self.check_power_raw() + if state is None: + return {'s1': None, 's2': None, 's3': None, 's4': None} + data = {} + data['s1'] = bool(state & 0x01) + data['s2'] = bool(state & 0x02) + data['s3'] = bool(state & 0x04) + data['s4'] = bool(state & 0x08) + return data + + +class bg1(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "BG1" + + def get_state(self): + """Get state of device. + + Returns: + dict: Dictionary of current state + eg. `{"pwr":1,"pwr1":1,"pwr2":0,"maxworktime":60,"maxworktime1":60,"maxworktime2":0,"idcbrightness":50}`""" + packet = self._encode(1, b'{}') + response = self.send_packet(0x6a, packet) + return self._decode(response) + + def set_state(self, pwr=None, pwr1=None, pwr2=None, maxworktime=None, maxworktime1=None, maxworktime2=None, idcbrightness=None): + data = {} + if pwr is not None: + data['pwr'] = int(bool(pwr)) + if pwr1 is not None: + data['pwr1'] = int(bool(pwr1)) + if pwr2 is not None: + data['pwr2'] = int(bool(pwr2)) + if maxworktime is not None: + data['maxworktime'] = maxworktime + if maxworktime1 is not None: + data['maxworktime1'] = maxworktime1 + if maxworktime2 is not None: + data['maxworktime2'] = maxworktime2 + if idcbrightness is not None: + data['idcbrightness'] = idcbrightness + js = json.dumps(data).encode('utf8') + packet = self._encode(2, js) + response = self.send_packet(0x6a, packet) + return self._decode(response) + + def _encode(self, flag, js): + # packet format is: + # 0x00-0x01 length + # 0x02-0x05 header + # 0x06-0x07 00 + # 0x08 flag (1 for read or 2 write?) + # 0x09 unknown (0xb) + # 0x0a-0x0d length of json + # 0x0e- json data + packet = bytearray(14) + length = 4 + 2 + 2 + 4 + len(js) + struct.pack_into('<HHHHBBI', packet, 0, length, 0xa5a5, 0x5a5a, 0x0000, flag, 0x0b, len(js)) + for i in range(len(js)): + packet.append(js[i]) + + checksum = 0xc0ad + for b in packet[0x08:]: + checksum = (checksum + b) & 0xffff + + packet[0x06] = checksum & 0xff + packet[0x07] = checksum >> 8 + + return packet + + def _decode(self, response): + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + + payload = self.decrypt(bytes(response[0x38:])) + js_len = struct.unpack_from('<I', payload, 0x0a)[0] + state = json.loads(payload[0x0e:0x0e+js_len]) + return state + +class sp1(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "SP1" + + def set_power(self, state): + packet = bytearray(4) + packet[0] = state + self.send_packet(0x66, packet) + + +class sp2(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "SP2" + + def set_power(self, state): + """Sets the power state of the smart plug.""" + packet = bytearray(16) + packet[0] = 2 + if self.check_nightlight(): + packet[4] = 3 if state else 2 + else: + packet[4] = 1 if state else 0 + self.send_packet(0x6a, packet) + + def set_nightlight(self, state): + """Sets the night light state of the smart plug""" + packet = bytearray(16) + packet[0] = 2 + if self.check_power(): + packet[4] = 3 if state else 1 + else: + packet[4] = 2 if state else 0 + self.send_packet(0x6a, packet) + + def check_power(self): + """Returns the power state of the smart plug.""" + packet = bytearray(16) + packet[0] = 1 + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + if isinstance(payload[0x4], int): + return bool(payload[0x4] == 1 or payload[0x4] == 3 or payload[0x4] == 0xFD) + return bool(ord(payload[0x4]) == 1 or ord(payload[0x4]) == 3 or ord(payload[0x4]) == 0xFD) + + def check_nightlight(self): + """Returns the power state of the smart plug.""" + packet = bytearray(16) + packet[0] = 1 + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + if isinstance(payload[0x4], int): + return bool(payload[0x4] == 2 or payload[0x4] == 3 or payload[0x4] == 0xFF) + return bool(ord(payload[0x4]) == 2 or ord(payload[0x4]) == 3 or ord(payload[0x4]) == 0xFF) + + def get_energy(self): + packet = bytearray([8, 0, 254, 1, 5, 1, 0, 0, 0, 45]) + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + if isinstance(payload[0x7], int): + energy = int(hex(payload[0x07] * 256 + payload[0x06])[2:]) + int(hex(payload[0x05])[2:]) / 100.0 + else: + energy = int(hex(ord(payload[0x07]) * 256 + ord(payload[0x06]))[2:]) + int( + hex(ord(payload[0x05]))[2:]) / 100.0 + return energy + + +class a1(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "A1" + + def check_sensors(self): + packet = bytearray(16) + packet[0] = 1 + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + data = {} + payload = self.decrypt(bytes(response[0x38:])) + if isinstance(payload[0x4], int): + data['temperature'] = (payload[0x4] * 10 + payload[0x5]) / 10.0 + data['humidity'] = (payload[0x6] * 10 + payload[0x7]) / 10.0 + light = payload[0x8] + air_quality = payload[0x0a] + noise = payload[0xc] + else: + data['temperature'] = (ord(payload[0x4]) * 10 + ord(payload[0x5])) / 10.0 + data['humidity'] = (ord(payload[0x6]) * 10 + ord(payload[0x7])) / 10.0 + light = ord(payload[0x8]) + air_quality = ord(payload[0x0a]) + noise = ord(payload[0xc]) + if light == 0: + data['light'] = 'dark' + elif light == 1: + data['light'] = 'dim' + elif light == 2: + data['light'] = 'normal' + elif light == 3: + data['light'] = 'bright' + else: + data['light'] = 'unknown' + if air_quality == 0: + data['air_quality'] = 'excellent' + elif air_quality == 1: + data['air_quality'] = 'good' + elif air_quality == 2: + data['air_quality'] = 'normal' + elif air_quality == 3: + data['air_quality'] = 'bad' + else: + data['air_quality'] = 'unknown' + if noise == 0: + data['noise'] = 'quiet' + elif noise == 1: + data['noise'] = 'normal' + elif noise == 2: + data['noise'] = 'noisy' + else: + data['noise'] = 'unknown' + return data + + def check_sensors_raw(self): + packet = bytearray(16) + packet[0] = 1 + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + data = {} + payload = self.decrypt(bytes(response[0x38:])) + if isinstance(payload[0x4], int): + data['temperature'] = (payload[0x4] * 10 + payload[0x5]) / 10.0 + data['humidity'] = (payload[0x6] * 10 + payload[0x7]) / 10.0 + data['light'] = payload[0x8] + data['air_quality'] = payload[0x0a] + data['noise'] = payload[0xc] + else: + data['temperature'] = (ord(payload[0x4]) * 10 + ord(payload[0x5])) / 10.0 + data['humidity'] = (ord(payload[0x6]) * 10 + ord(payload[0x7])) / 10.0 + data['light'] = ord(payload[0x8]) + data['air_quality'] = ord(payload[0x0a]) + data['noise'] = ord(payload[0xc]) + return data + + +class rm(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "RM2" + self._request_header = bytes() + self._code_sending_header = bytes() + + def check_data(self): + packet = bytearray(self._request_header) + packet.append(0x04) + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + return payload[len(self._request_header) + 4:] + + def send_data(self, data): + packet = bytearray(self._code_sending_header) + packet += bytes([0x02, 0x00, 0x00, 0x00]) + packet += data + self.send_packet(0x6a, packet) + + def enter_learning(self): + packet = bytearray(self._request_header) + packet.append(0x03) + self.send_packet(0x6a, packet) + + def sweep_frequency(self): + packet = bytearray(self._request_header) + packet.append(0x19) + self.send_packet(0x6a, packet) + + def cancel_sweep_frequency(self): + packet = bytearray(self._request_header) + packet.append(0x1e) + self.send_packet(0x6a, packet) + + def check_frequency(self): + packet = bytearray(self._request_header) + packet.append(0x1a) + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return False + payload = self.decrypt(bytes(response[0x38:])) + if payload[len(self._request_header) + 4] == 1: + return True + return False + + def find_rf_packet(self): + packet = bytearray(self._request_header) + packet.append(0x1b) + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return False + payload = self.decrypt(bytes(response[0x38:])) + if payload[len(self._request_header) + 4] == 1: + return True + return False + + def _read_sensor(self, type, offset, divider): + packet = bytearray(self._request_header) + packet.append(type) + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return False + payload = self.decrypt(bytes(response[0x38:])) + value_pos = len(self._request_header) + offset + if isinstance(payload[value_pos], int): + value = (payload[value_pos] + payload[value_pos+1] / divider) + else: + value = (ord(payload[value_pos]) + ord(payload[value_pos+1]) / divider) + return value + + def check_temperature(self): + return self._read_sensor( 0x01, 4, 10.0 ) + +class rm4(rm): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "RM4" + self._request_header = b'\x04\x00' + self._code_sending_header = b'\xd0\x00' + + def check_temperature(self): + return self._read_sensor( 0x24, 4, 100.0 ) + + def check_humidity(self): + return self._read_sensor( 0x24, 6, 100.0 ) + + def check_sensors(self): + return { + 'temperature': self.check_temperature(), + 'humidity': self.check_humidity() + } + +# For legacy compatibility - don't use this +class rm2(rm): + def __init__(self): + device.__init__(self, None, None, None) + + def discover(self): + dev = discover() + self.host = dev.host + self.mac = dev.mac + + +class hysen(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "Hysen heating controller" + + # Send a request + # input_payload should be a bytearray, usually 6 bytes, e.g. bytearray([0x01,0x06,0x00,0x02,0x10,0x00]) + # Returns decrypted payload + # New behaviour: raises a ValueError if the device response indicates an error or CRC check fails + # The function prepends length (2 bytes) and appends CRC + + def calculate_crc16(self, input_data): + from ctypes import c_ushort + crc16_tab = [] + crc16_constant = 0xA001 + + for i in range(0, 256): + crc = c_ushort(i).value + for j in range(0, 8): + if (crc & 0x0001): + crc = c_ushort(crc >> 1).value ^ crc16_constant + else: + crc = c_ushort(crc >> 1).value + crc16_tab.append(hex(crc)) + + try: + is_string = isinstance(input_data, str) + is_bytes = isinstance(input_data, bytes) + + if not is_string and not is_bytes: + raise Exception("Please provide a string or a byte sequence " + "as argument for calculation.") + + crcValue = 0xffff + + for c in input_data: + d = ord(c) if is_string else c + tmp = crcValue ^ d + rotated = c_ushort(crcValue >> 8).value + crcValue = rotated ^ int(crc16_tab[(tmp & 0x00ff)], 0) + + return crcValue + except Exception as e: + print("EXCEPTION(calculate): {}".format(e)) + + def send_request(self, input_payload): + + crc = self.calculate_crc16(bytes(input_payload)) + + # first byte is length, +2 for CRC16 + request_payload = bytearray([len(input_payload) + 2, 0x00]) + request_payload.extend(input_payload) + + # append CRC + request_payload.append(crc & 0xFF) + request_payload.append((crc >> 8) & 0xFF) + + # send to device + response = self.send_packet(0x6a, request_payload) + + # check for error + err = response[0x22] | (response[0x23] << 8) + if err: + raise ValueError('broadlink_response_error', err) + + response_payload = bytearray(self.decrypt(bytes(response[0x38:]))) + + # experimental check on CRC in response (first 2 bytes are len, and trailing bytes are crc) + response_payload_len = response_payload[0] + if response_payload_len + 2 > len(response_payload): + raise ValueError('hysen_response_error', 'first byte of response is not length') + crc = self.calculate_crc16(bytes(response_payload[2:response_payload_len])) + if (response_payload[response_payload_len] == crc & 0xFF) and ( + response_payload[response_payload_len + 1] == (crc >> 8) & 0xFF): + return response_payload[2:response_payload_len] + raise ValueError('hysen_response_error', 'CRC check on response failed') + + # Get current room temperature in degrees celsius + def get_temp(self): + payload = self.send_request(bytearray([0x01, 0x03, 0x00, 0x00, 0x00, 0x08])) + return payload[0x05] / 2.0 + + # Get current external temperature in degrees celsius + def get_external_temp(self): + payload = self.send_request(bytearray([0x01, 0x03, 0x00, 0x00, 0x00, 0x08])) + return payload[18] / 2.0 + + # Get full status (including timer schedule) + def get_full_status(self): + payload = self.send_request(bytearray([0x01, 0x03, 0x00, 0x00, 0x00, 0x16])) + data = {} + data['remote_lock'] = payload[3] & 1 + data['power'] = payload[4] & 1 + data['active'] = (payload[4] >> 4) & 1 + data['temp_manual'] = (payload[4] >> 6) & 1 + data['room_temp'] = (payload[5] & 255) / 2.0 + data['thermostat_temp'] = (payload[6] & 255) / 2.0 + data['auto_mode'] = payload[7] & 15 + data['loop_mode'] = (payload[7] >> 4) & 15 + data['sensor'] = payload[8] + data['osv'] = payload[9] + data['dif'] = payload[10] + data['svh'] = payload[11] + data['svl'] = payload[12] + data['room_temp_adj'] = ((payload[13] << 8) + payload[14]) / 2.0 + if data['room_temp_adj'] > 32767: + data['room_temp_adj'] = 32767 - data['room_temp_adj'] + data['fre'] = payload[15] + data['poweron'] = payload[16] + data['unknown'] = payload[17] + data['external_temp'] = (payload[18] & 255) / 2.0 + data['hour'] = payload[19] + data['min'] = payload[20] + data['sec'] = payload[21] + data['dayofweek'] = payload[22] + + weekday = [] + for i in range(0, 6): + weekday.append( + {'start_hour': payload[2 * i + 23], 'start_minute': payload[2 * i + 24], 'temp': payload[i + 39] / 2.0}) + + data['weekday'] = weekday + weekend = [] + for i in range(6, 8): + weekend.append( + {'start_hour': payload[2 * i + 23], 'start_minute': payload[2 * i + 24], 'temp': payload[i + 39] / 2.0}) + + data['weekend'] = weekend + return data + + # Change controller mode + # auto_mode = 1 for auto (scheduled/timed) mode, 0 for manual mode. + # Manual mode will activate last used temperature. + # In typical usage call set_temp to activate manual control and set temp. + # loop_mode refers to index in [ "12345,67", "123456,7", "1234567" ] + # E.g. loop_mode = 0 ("12345,67") means Saturday and Sunday follow the "weekend" schedule + # loop_mode = 2 ("1234567") means every day (including Saturday and Sunday) follows the "weekday" schedule + # The sensor command is currently experimental + def set_mode(self, auto_mode, loop_mode, sensor=0): + mode_byte = ((loop_mode + 1) << 4) + auto_mode + self.send_request(bytearray([0x01, 0x06, 0x00, 0x02, mode_byte, sensor])) + + # Advanced settings + # Sensor mode (SEN) sensor = 0 for internal sensor, 1 for external sensor, + # 2 for internal control temperature, external limit temperature. Factory default: 0. + # Set temperature range for external sensor (OSV) osv = 5..99. Factory default: 42C + # Deadzone for floor temprature (dIF) dif = 1..9. Factory default: 2C + # Upper temperature limit for internal sensor (SVH) svh = 5..99. Factory default: 35C + # Lower temperature limit for internal sensor (SVL) svl = 5..99. Factory default: 5C + # Actual temperature calibration (AdJ) adj = -0.5. Prescision 0.1C + # Anti-freezing function (FrE) fre = 0 for anti-freezing function shut down, + # 1 for anti-freezing function open. Factory default: 0 + # Power on memory (POn) poweron = 0 for power on memory off, 1 for power on memory on. Factory default: 0 + def set_advanced(self, loop_mode, sensor, osv, dif, svh, svl, adj, fre, poweron): + input_payload = bytearray([0x01, 0x10, 0x00, 0x02, 0x00, 0x05, 0x0a, loop_mode, sensor, osv, dif, svh, svl, + (int(adj * 2) >> 8 & 0xff), (int(adj * 2) & 0xff), fre, poweron]) + self.send_request(input_payload) + + # For backwards compatibility only. Prefer calling set_mode directly. + # Note this function invokes loop_mode=0 and sensor=0. + def switch_to_auto(self): + self.set_mode(auto_mode=1, loop_mode=0) + + def switch_to_manual(self): + self.set_mode(auto_mode=0, loop_mode=0) + + # Set temperature for manual mode (also activates manual mode if currently in automatic) + def set_temp(self, temp): + self.send_request(bytearray([0x01, 0x06, 0x00, 0x01, 0x00, int(temp * 2)])) + + # Set device on(1) or off(0), does not deactivate Wifi connectivity. + # Remote lock disables control by buttons on thermostat. + def set_power(self, power=1, remote_lock=0): + self.send_request(bytearray([0x01, 0x06, 0x00, 0x00, remote_lock, power])) + + # set time on device + # n.b. day=1 is Monday, ..., day=7 is Sunday + def set_time(self, hour, minute, second, day): + self.send_request(bytearray([0x01, 0x10, 0x00, 0x08, 0x00, 0x02, 0x04, hour, minute, second, day])) + + # Set timer schedule + # Format is the same as you get from get_full_status. + # weekday is a list (ordered) of 6 dicts like: + # {'start_hour':17, 'start_minute':30, 'temp': 22 } + # Each one specifies the thermostat temp that will become effective at start_hour:start_minute + # weekend is similar but only has 2 (e.g. switch on in morning and off in afternoon) + def set_schedule(self, weekday, weekend): + # Begin with some magic values ... + input_payload = bytearray([0x01, 0x10, 0x00, 0x0a, 0x00, 0x0c, 0x18]) + + # Now simply append times/temps + # weekday times + for i in range(0, 6): + input_payload.append(weekday[i]['start_hour']) + input_payload.append(weekday[i]['start_minute']) + + # weekend times + for i in range(0, 2): + input_payload.append(weekend[i]['start_hour']) + input_payload.append(weekend[i]['start_minute']) + + # weekday temperatures + for i in range(0, 6): + input_payload.append(int(weekday[i]['temp'] * 2)) + + # weekend temperatures + for i in range(0, 2): + input_payload.append(int(weekend[i]['temp'] * 2)) + + self.send_request(input_payload) + + +S1C_SENSORS_TYPES = { + 0x31: 'Door Sensor', # 49 as hex + 0x91: 'Key Fob', # 145 as hex, as serial on fob corpse + 0x21: 'Motion Sensor' # 33 as hex +} + + +class S1C(device): + """ + Its VERY VERY VERY DIRTY IMPLEMENTATION of S1C + """ + + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = 'S1C' + + def get_sensors_status(self): + packet = bytearray(16) + packet[0] = 0x06 # 0x06 - get sensors info, 0x07 - probably add sensors + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + + payload = self.decrypt(bytes(response[0x38:])) + if not payload: + return None + count = payload[0x4] + sensors = payload[0x6:] + sensors_a = [bytearray(sensors[i * 83:(i + 1) * 83]) for i in range(len(sensors) // 83)] + + sens_res = [] + for sens in sensors_a: + status = ord(chr(sens[0])) + _name = str(bytes(sens[4:26]).decode()) + _order = ord(chr(sens[1])) + _type = ord(chr(sens[3])) + _serial = bytes(codecs.encode(sens[26:30], "hex")).decode() + + type_str = S1C_SENSORS_TYPES.get(_type, 'Unknown') + + r = { + 'status': status, + 'name': _name.strip('\x00'), + 'type': type_str, + 'order': _order, + 'serial': _serial, + } + if r['serial'] != '00000000': + sens_res.append(r) + result = { + 'count': count, + 'sensors': sens_res + } + return result + + +class dooya(device): + def __init__(self, *args, **kwargs): + device.__init__(self, *args, **kwargs) + self.type = "Dooya DT360E" + + def _send(self, magic1, magic2): + packet = bytearray(16) + packet[0] = 0x09 + packet[2] = 0xbb + packet[3] = magic1 + packet[4] = magic2 + packet[9] = 0xfa + packet[10] = 0x44 + response = self.send_packet(0x6a, packet) + err = response[0x22] | (response[0x23] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + return ord(payload[4]) + + def open(self): + return self._send(0x01, 0x00) + + def close(self): + return self._send(0x02, 0x00) + + def stop(self): + return self._send(0x03, 0x00) + + def get_percentage(self): + return self._send(0x06, 0x5d) + + def set_percentage_and_wait(self, new_percentage): + current = self.get_percentage() + if current > new_percentage: + self.close() + while current is not None and current > new_percentage: + time.sleep(0.2) + current = self.get_percentage() + + elif current < new_percentage: + self.open() + while current is not None and current < new_percentage: + time.sleep(0.2) + current = self.get_percentage() + self.stop() + +class lb1(device): + state_dict = [] + effect_map_dict = { 'lovely color' : 0, + 'flashlight' : 1, + 'lightning' : 2, + 'color fading' : 3, + 'color breathing' : 4, + 'multicolor breathing' : 5, + 'color jumping' : 6, + 'multicolor jumping' : 7 } + + def __init__(self, host, mac, devtype): + device.__init__(self, host, mac, devtype) + self.type = "SmartBulb" + + def send_command(self,command, type = 'set'): + packet = bytearray(16+(int(len(command)/16) + 1)*16) + packet[0x02] = 0xa5 + packet[0x03] = 0xa5 + packet[0x04] = 0x5a + packet[0x05] = 0x5a + packet[0x08] = 0x02 if type == "set" else 0x01 # 0x01 => query, # 0x02 => set + packet[0x09] = 0x0b + packet[0x0a] = len(command) + packet[0x0e:] = map(ord, command) + + checksum = 0xbeaf + for b in packet: + checksum = (checksum + b) & 0xffff + + packet[0x00] = (0x0c + len(command)) & 0xff + packet[0x06] = checksum & 0xff # Checksum 1 position + packet[0x07] = checksum >> 8 # Checksum 2 position + + response = self.send_packet(0x6a, packet) + + err = response[0x36] | (response[0x37] << 8) + if err != 0: + return None + payload = self.decrypt(bytes(response[0x38:])) + + responseLength = int(payload[0x0a]) | (int(payload[0x0b]) << 8) + if responseLength > 0: + self.state_dict = json.loads(payload[0x0e:0x0e+responseLength]) + + def set_json(self, jsonstr): + reconvert = json.loads(jsonstr) + if 'bulb_sceneidx' in reconvert.keys(): + reconvert['bulb_sceneidx'] = self.effect_map_dict.get(reconvert['bulb_sceneidx'], 255) + + self.send_command(json.dumps(reconvert)) + return json.dumps(self.state_dict) + + def set_state(self, state): + cmd = '{"pwr":%d}' % (1 if state == "ON" or state == 1 else 0) + self.send_command(cmd) + + def get_state(self): + cmd = "{}" + self.send_command(cmd) + return self.state_dict + +# Setup a new Broadlink device via AP Mode. Review the README to see how to enter AP Mode. +# Only tested with Broadlink RM3 Mini (Blackbean) +def setup(ssid, password, security_mode): + # Security mode options are (0 - none, 1 = WEP, 2 = WPA1, 3 = WPA2, 4 = WPA1/2) + payload = bytearray(0x88) + payload[0x26] = 0x14 # This seems to always be set to 14 + # Add the SSID to the payload + ssid_start = 68 + ssid_length = 0 + for letter in ssid: + payload[(ssid_start + ssid_length)] = ord(letter) + ssid_length += 1 + # Add the WiFi password to the payload + pass_start = 100 + pass_length = 0 + for letter in password: + payload[(pass_start + pass_length)] = ord(letter) + pass_length += 1 + + payload[0x84] = ssid_length # Character length of SSID + payload[0x85] = pass_length # Character length of password + payload[0x86] = security_mode # Type of encryption (00 - none, 01 = WEP, 02 = WPA1, 03 = WPA2, 04 = WPA1/2) + + checksum = 0xbeaf + for b in payload: + checksum = (checksum + b) & 0xffff + + payload[0x20] = checksum & 0xff # Checksum 1 position + payload[0x21] = checksum >> 8 # Checksum 2 position + + sock = socket.socket(socket.AF_INET, # Internet + socket.SOCK_DGRAM) # UDP + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + # sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + sock.sendto(payload, ('192.168.10.1', 80)) diff --git a/third_party/python/broadlink/cli/README.md b/third_party/python/broadlink/cli/README.md new file mode 100644 index 000000000000..7e229e3eb557 --- /dev/null +++ b/third_party/python/broadlink/cli/README.md @@ -0,0 +1,85 @@ +Command line interface for python-broadlink +=========================================== + +This is a command line interface for broadlink python library + +Tested with BroadLink RMPRO / RM2 + + +Requirements +------------ +You should have the broadlink python installed, this can be made in many linux distributions using : +``` +sudo pip install broadlink +``` + +Installation +----------- +Just copy this files + + +Programs +-------- + + +* broadlink_discovery +used to run the discovery in the network +this program withh show the command line parameters to be used with +broadlink_cli to select broadlink device + +* broadlink_cli +used to send commands and query the broadlink device + + +device specification formats +---------------------------- + +Using separate parameters for each information: +``` +broadlink_cli --type 0x2712 --host 1.1.1.1 --mac aaaaaaaaaa --temp +``` + +Using all parameters as a single argument: +``` +broadlink_cli --device "0x2712 1.1.1.1 aaaaaaaaaa" --temp +``` + +Using file with parameters: +``` +broadlink_cli --device @BEDROOM.device --temp +``` +This is prefered as the configuration is stored in file and you can change +just a file to point to a different hardware + +Sample usage +------------ + +Learn commands : +``` +# Learn and save to file +broadlink_cli --device @BEDROOM.device --learnfile LG-TV.power +# LEard and show at console +broadlink_cli --device @BEDROOM.device --learn +``` + + +Send command : +``` +broadlink_cli --device @BEDROOM.device --send @LG-TV.power +broadlink_cli --device @BEDROOM.device --send ....datafromlearncommand... +``` + +Get Temperature : +``` +broadlink_cli --device @BEDROOM.device --temperature +``` + +Get Energy Consumption (For a SmartPlug) : +``` +broadlink_cli --device @BEDROOM.device --energy +``` + +Once joined to the Broadlink provisioning Wi-Fi, configure it with your Wi-Fi details: +``` +broadlink_cli --joinwifi MySSID MyWifiPassword +``` diff --git a/third_party/python/broadlink/cli/broadlink_cli b/third_party/python/broadlink/cli/broadlink_cli new file mode 100755 index 000000000000..5045c5c1082f --- /dev/null +++ b/third_party/python/broadlink/cli/broadlink_cli @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 + +import argparse +import base64 +import codecs +import time + +import broadlink + +TICK = 32.84 +IR_TOKEN = 0x26 + + +def auto_int(x): + return int(x, 0) + + +def to_microseconds(bytes): + result = [] + # print bytes[0] # 0x26 = 38for IR + index = 4 + while index < len(bytes): + chunk = bytes[index] + index += 1 + if chunk == 0: + chunk = bytes[index] + chunk = 256 * chunk + bytes[index + 1] + index += 2 + result.append(int(round(chunk * TICK))) + if chunk == 0x0d05: + break + return result + + +def durations_to_broadlink(durations): + result = bytearray() + result.append(IR_TOKEN) + result.append(0) + result.append(len(durations) % 256) + result.append(len(durations) / 256) + for dur in durations: + num = int(round(dur / TICK)) + if num > 255: + result.append(0) + result.append(num / 256) + result.append(num % 256) + return result + + +def format_durations(data): + result = '' + for i in range(0, len(data)): + if len(result) > 0: + result += ' ' + result += ('+' if i % 2 == 0 else '-') + str(data[i]) + return result + + +def parse_durations(str): + result = [] + for s in str.split(): + result.append(abs(int(s))) + return result + + +parser = argparse.ArgumentParser(fromfile_prefix_chars='@') +parser.add_argument("--device", help="device definition as 'type host mac'") +parser.add_argument("--type", type=auto_int, default=0x2712, help="type of device") +parser.add_argument("--host", help="host address") +parser.add_argument("--mac", help="mac address (hex reverse), as used by python-broadlink library") +parser.add_argument("--temperature", action="store_true", help="request temperature from device") +parser.add_argument("--energy", action="store_true", help="request energy consumption from device") +parser.add_argument("--check", action="store_true", help="check current power state") +parser.add_argument("--checknl", action="store_true", help="check current nightlight state") +parser.add_argument("--turnon", action="store_true", help="turn on device") +parser.add_argument("--turnoff", action="store_true", help="turn off device") +parser.add_argument("--turnnlon", action="store_true", help="turn on nightlight on the device") +parser.add_argument("--turnnloff", action="store_true", help="turn off nightlight on the device") +parser.add_argument("--switch", action="store_true", help="switch state from on to off and off to on") +parser.add_argument("--send", action="store_true", help="send command") +parser.add_argument("--sensors", action="store_true", help="check all sensors") +parser.add_argument("--learn", action="store_true", help="learn command") +parser.add_argument("--rfscanlearn", action="store_true", help="rf scan learning") +parser.add_argument("--learnfile", help="save learned command to a specified file") +parser.add_argument("--durations", action="store_true", + help="use durations in micro seconds instead of the Broadlink format") +parser.add_argument("--convert", action="store_true", help="convert input data to durations") +parser.add_argument("--joinwifi", nargs=2, help="Args are SSID PASSPHRASE to configure Broadlink device with"); +parser.add_argument("data", nargs='*', help="Data to send or convert") +args = parser.parse_args() + +if args.device: + values = args.device.split() + type = int(values[0], 0) + host = values[1] + mac = bytearray.fromhex(values[2]) +elif args.mac: + type = args.type + host = args.host + mac = bytearray.fromhex(args.mac) + +if args.host or args.device: + dev = broadlink.gendevice(type, (host, 80), mac) + dev.auth() + +if args.joinwifi: + broadlink.setup(args.joinwifi[0], args.joinwifi[1], 4) + +if args.convert: + data = bytearray.fromhex(''.join(args.data)) + durations = to_microseconds(data) + print(format_durations(durations)) +if args.temperature: + print(dev.check_temperature()) +if args.energy: + print(dev.get_energy()) +if args.sensors: + try: + data = dev.check_sensors() + except: + data = {} + data['temperature'] = dev.check_temperature() + for key in data: + print("{} {}".format(key, data[key])) +if args.send: + data = durations_to_broadlink(parse_durations(' '.join(args.data))) \ + if args.durations else bytearray.fromhex(''.join(args.data)) + dev.send_data(data) +if args.learn or args.learnfile: + dev.enter_learning() + data = None + print("Learning...") + timeout = 30 + while (data is None) and (timeout > 0): + time.sleep(2) + timeout -= 2 + data = dev.check_data() + if data: + learned = format_durations(to_microseconds(bytearray(data))) \ + if args.durations \ + else ''.join(format(x, '02x') for x in bytearray(data)) + if args.learn: + print(learned) + decode_hex = codecs.getdecoder("hex_codec") + print("Base64: " + str(base64.b64encode(decode_hex(learned)[0]))) + if args.learnfile: + print("Saving to {}".format(args.learnfile)) + with open(args.learnfile, "w") as text_file: + text_file.write(learned) + else: + print("No data received...") +if args.check: + if dev.check_power(): + print('* ON *') + else: + print('* OFF *') +if args.checknl: + if dev.check_nightlight(): + print('* ON *') + else: + print('* OFF *') +if args.turnon: + dev.set_power(True) + if dev.check_power(): + print('== Turned * ON * ==') + else: + print('!! Still OFF !!') +if args.turnoff: + dev.set_power(False) + if dev.check_power(): + print('!! Still ON !!') + else: + print('== Turned * OFF * ==') +if args.turnnlon: + dev.set_nightlight(True) + if dev.check_nightlight(): + print('== Turned * ON * ==') + else: + print('!! Still OFF !!') +if args.turnnloff: + dev.set_nightlight(False) + if dev.check_nightlight(): + print('!! Still ON !!') + else: + print('== Turned * OFF * ==') +if args.switch: + if dev.check_power(): + dev.set_power(False) + print('* Switch to OFF *') + else: + dev.set_power(True) + print('* Switch to ON *') +if args.rfscanlearn: + dev.sweep_frequency() + print("Learning RF Frequency, press and hold the button to learn...") + + timeout = 20 + + while (not dev.check_frequency()) and (timeout > 0): + time.sleep(1) + timeout -= 1 + + if timeout <= 0: + print("RF Frequency not found") + dev.cancel_sweep_frequency() + exit(1) + + print("Found RF Frequency - 1 of 2!") + print("You can now let go of the button") + + input("Press enter to continue...") + + print("To complete learning, single press the button you want to learn") + + dev.find_rf_packet() + + data = None + timeout = 20 + + while (data is None) and (timeout > 0): + time.sleep(1) + timeout -= 1 + data = dev.check_data() + + if data: + print("Found RF Frequency - 2 of 2!") + learned = format_durations(to_microseconds(bytearray(data))) \ + if args.durations \ + else ''.join(format(x, '02x') for x in bytearray(data)) + if args.learnfile is None: + print(learned) + decode_hex = codecs.getdecoder("hex_codec") + print("Base64: {}".format(str(base64.b64encode(decode_hex(learned)[0])))) + if args.learnfile is not None: + print("Saving to {}".format(args.learnfile)) + with open(args.learnfile, "w") as text_file: + text_file.write(learned) + else: + print("No data received...") diff --git a/third_party/python/broadlink/cli/broadlink_discovery b/third_party/python/broadlink/cli/broadlink_discovery new file mode 100755 index 000000000000..1c6b80b1483e --- /dev/null +++ b/third_party/python/broadlink/cli/broadlink_discovery @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +import argparse + +import broadlink + +parser = argparse.ArgumentParser(fromfile_prefix_chars='@') +parser.add_argument("--timeout", type=int, default=5, help="timeout to wait for receiving discovery responses") +parser.add_argument("--ip", default=None, help="ip address to use in the discovery") +parser.add_argument("--dst-ip", default=None, help="destination ip address to use in the discovery") +args = parser.parse_args() + +print("Discovering...") +devices = broadlink.discover(timeout=args.timeout, local_ip_address=args.ip, discover_ip_address=args.dst_ip) +for device in devices: + if device.auth(): + print("###########################################") + print(device.type) + print("# broadlink_cli --type {} --host {} --mac {}".format(hex(device.devtype), device.host[0], + ''.join(format(x, '02x') for x in device.mac))) + print("Device file data (to be used with --device @filename in broadlink_cli) : ") + print("{} {} {}".format(hex(device.devtype), device.host[0], ''.join(format(x, '02x') for x in device.mac))) + if hasattr(device, 'check_temperature'): + print("temperature = {}".format(device.check_temperature())) + print("") + else: + print("Error authenticating with device : {}".format(device.host)) diff --git a/third_party/python/broadlink/default.nix b/third_party/python/broadlink/default.nix new file mode 100644 index 000000000000..e316d83d1d71 --- /dev/null +++ b/third_party/python/broadlink/default.nix @@ -0,0 +1,15 @@ +# Python package for controlling the Broadlink RM Pro Infrared +# controller. +# +# https://github.com/mjg59/python-broadlink +{ pkgs, lib, ... }: + +let + inherit (pkgs) fetchFromGitHub; + inherit (pkgs.python3Packages) buildPythonPackage cryptography; +in buildPythonPackage (lib.fix (self: { + pname = "python-broadlink"; + version = "0.13.2"; + src = ./.; + propagatedBuildInputs = [ cryptography ]; +})) diff --git a/third_party/python/broadlink/protocol.md b/third_party/python/broadlink/protocol.md new file mode 100644 index 000000000000..2e388d749963 --- /dev/null +++ b/third_party/python/broadlink/protocol.md @@ -0,0 +1,202 @@ +Broadlink RM2 network protocol +============================== + +Encryption +---------- + +Packets include AES-based encryption in CBC mode. The initial key is 0x09, 0x76, 0x28, 0x34, 0x3f, 0xe9, 0x9e, 0x23, 0x76, 0x5c, 0x15, 0x13, 0xac, 0xcf, 0x8b, 0x02. The IV is 0x56, 0x2e, 0x17, 0x99, 0x6d, 0x09, 0x3d, 0x28, 0xdd, 0xb3, 0xba, 0x69, 0x5a, 0x2e, 0x6f, 0x58. + +Checksum +-------- + +Construct the packet and set checksum bytes to zero. Add each byte to the starting value of 0xbeaf, wrapping after 0xffff. + +New device setup +---------------- + +To setup a new Broadlink device while in AP Mode a 136 byte packet needs to be sent to the device as follows: + +| Offset | Contents | +|---------|----------| +|0x00-0x19|00| +|0x20-0x21|Checksum as a little-endian 16 bit integer| +|0x26|14 (Always 14)| +|0x44-0x63|SSID Name (zero padding is appended)| +|0x64-0x83|Password (zero padding is appended)| +|0x84|Character length of SSID| +|0x85|Character length of password| +|0x86|Wireless security mode (00 - none, 01 = WEP, 02 = WPA1, 03 = WPA2, 04 = WPA1/2)| +|0x87-88|00| + +Send this packet as a UDP broadcast to 255.255.255.255 on port 80. + +Network discovery +----------------- + +To discover Broadlink devices on the local network, send a 48 byte packet with the following contents: + +| Offset | Contents | +|---------|----------| +|0x00-0x07|00| +|0x08-0x0b|Current offset from GMT as a little-endian 32 bit integer| +|0x0c-0x0d|Current year as a little-endian 16 bit integer| +|0x0e|Current number of seconds past the minute| +|0x0f|Current number of minutes past the hour| +|0x10|Current number of hours past midnight| +|0x11|Current day of the week (Monday = 1, Tuesday = 2, etc)| +|0x12|Current day in month| +|0x13|Current month| +|0x14-0x17|00| +|0x18-0x1b|Local IP address| +|0x1c-0x1d|Source port as a little-endian 16 bit integer| +|0x1e-0x1f|00| +|0x20-0x21|Checksum as a little-endian 16 bit integer| +|0x22-0x25|00| +|0x26|06| +|0x27-0x2f|00| + +Send this packet as a UDP broadcast to 255.255.255.255 on port 80. + +Response (any unicast response): + +| Offset | Contents | +|---------|----------| +|0x34-0x35|Device type as a little-endian 16 bit integer (see device type mapping)| +|0x3a-0x3f|MAC address of the target device| + +Device type mapping: + +| Device type in response packet | Device type | Treat as | +|---------|----------|----------| +|0|SP1|SP1| +|0x2711|SP2|SP2| +|0x2719 or 0x7919 or 0x271a or 0x791a|Honeywell SP2|SP2| +|0x2720|SPMini|SP2| +|0x753e|SP3|SP2| +|0x2728|SPMini2|SP2 +|0x2733 or 0x273e|OEM branded SPMini|SP2| +|>= 0x7530 and <= 0x7918|OEM branded SPMini2|SP2| +|0x2736|SPMiniPlus|SP2| +|0x2712|RM2|RM| +|0x2737|RM Mini / RM3 Mini Blackbean|RM| +|0x273d|RM Pro Phicomm|RM| +|0x2783|RM2 Home Plus|RM| +|0x277c|RM2 Home Plus GDT|RM| +|0x272a|RM2 Pro Plus|RM| +|0x2787|RM2 Pro Plus2|RM| +|0x278b|RM2 Pro Plus BL|RM| +|0x278f|RM Mini Shate|RM| +|0x2714|A1|A1| +|0x4EB5|MP1|MP1| + + +Command packet format +--------------------- + +The command packet header is 56 bytes long with the following format: + +|Offset|Contents| +|------|--------| +|0x00|0x5a| +|0x01|0xa5| +|0x02|0xaa| +|0x03|0x55| +|0x04|0x5a| +|0x05|0xa5| +|0x06|0xaa| +|0x07|0x55| +|0x08-0x1f|00| +|0x20-0x21|Checksum of full packet as a little-endian 16 bit integer| +|0x22-0x23|00| +|0x24-0x25|Device type as a little-endian 16 bit integer| +|0x26-0x27|Command code as a little-endian 16 bit integer| +|0x28-0x29|Packet count as a little-endian 16 bit integer| +|0x2a-0x2f|Local MAC address| +|0x30-0x33|Local device ID (obtained during authentication, 00 before authentication)| +|0x34-0x35|Checksum of unencrypted payload as a little-endian 16 bit integer +|0x36-0x37|00| + +The payload is appended immediately after this. The checksum at 0x20 is calculated *after* the payload is appended, and covers the entire packet (including the checksum at 0x34). Therefore: + +1. Generate packet header with checksum values set to 0 +2. Set the checksum initialisation value to 0xbeaf and calculate the checksum of the unencrypted payload. Set 0x34-0x35 to this value. +3. Encrypt and append the payload +4. Set the checksum initialisation value to 0xbeaf and calculate the checksum of the entire packet. Set 0x20-0x21 to this value. + +Authorisation +------------- + +You must obtain an authorisation key from the device before you can communicate. To do so, generate an 80 byte packet with the following contents: + +|Offset|Contents| +|------|--------| +|0x00-0x03|00| +|0x04-0x12|A 15-digit value that represents this device. Broadlink's implementation uses the IMEI.| +|0x13|01| +|0x14-0x2c|00| +|0x2d|0x01| +|0x30-0x7f|NULL-terminated ASCII string containing the device name| + +Send this payload with a command value of 0x0065. The response packet will contain an encrypted payload from byte 0x38 onwards. Decrypt this using the default key and IV. The format of the decrypted payload is: + +|Offset|Contents| +|------|--------| +|0x00-0x03|Device ID| +|0x04-0x13|Device encryption key| + +All further command packets must use this encryption key and device ID. + +Entering learning mode +---------------------- + +Send the following 16 byte payload with a command value of 0x006a: + +|Offset|Contents| +|------|--------| +|0x00|0x03| +|0x01-0x0f|0x00| + +Reading back data from learning mode +------------------------------------ + +Send the following 16 byte payload with a command value of 0x006a: + +|Offset|Contents| +|------|--------| +|0x00|0x04| +|0x01-0x0f|0x00| + +Byte 0x22 of the response contains a little-endian 16 bit error code. If this is 0, a code has been obtained. Bytes 0x38 and onward of the response are encrypted. Decrypt them. Bytes 0x04 and onward of the decrypted payload contain the captured data. + +Sending data +------------ + +Send the following payload with a command byte of 0x006a + +|Offset|Contents| +|------|--------| +|0x00|0x02| +|0x01-0x03|0x00| +|0x04|0x26 = IR, 0xb2 for RF 433Mhz, 0xd7 for RF 315Mhz| +|0x05|repeat count, (0 = no repeat, 1 send twice, .....)| +|0x06-0x07|Length of the following data in little endian| +|0x08 ....|Pulse lengths in 2^-15 s units (µs * 269 / 8192 works very well)| +|....|0x0d 0x05 at the end for IR only| + +Each value is represented by one byte. If the length exceeds one byte +then it is stored big endian with a leading 0. + +Example: The header for my Optoma projector is 8920 4450 +8920 * 269 / 8192 = 0x124 +4450 * 269 / 8192 = 0x92 + +So the data starts with `0x00 0x1 0x24 0x92 ....` + + +Todo +---- + +* Support for other devices using the Broadlink protocol (various smart home devices) +* Figure out what the format of the data packets actually is. +* Deal with the response after AP Mode WiFi network setup. + diff --git a/third_party/python/broadlink/requirements.txt b/third_party/python/broadlink/requirements.txt new file mode 100644 index 000000000000..09f445bfd823 --- /dev/null +++ b/third_party/python/broadlink/requirements.txt @@ -0,0 +1 @@ +cryptography==2.6.1 diff --git a/third_party/python/broadlink/setup.py b/third_party/python/broadlink/setup.py new file mode 100644 index 000000000000..778f495fb58a --- /dev/null +++ b/third_party/python/broadlink/setup.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + + +from setuptools import setup, find_packages + + +version = '0.13.2' + +setup( + name='broadlink', + version=version, + author='Matthew Garrett', + author_email='mjg59@srcf.ucam.org', + url='http://github.com/mjg59/python-broadlink', + packages=find_packages(), + scripts=[], + install_requires=['cryptography>=2.1.1'], + description='Python API for controlling Broadlink IR controllers', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + ], + include_package_data=True, + zip_safe=False, +) diff --git a/third_party/rapidcheck/default.nix b/third_party/rapidcheck/default.nix new file mode 100644 index 000000000000..ec8c01694da6 --- /dev/null +++ b/third_party/rapidcheck/default.nix @@ -0,0 +1,21 @@ +{ pkgs, ... }: + +(pkgs.callPackage "${pkgs.path}/pkgs/development/libraries/rapidcheck" { + stdenv = pkgs.fullLlvm11Stdenv; +}).overrideAttrs (attrs: rec { + # follows the versioning scheme of nixpkgs, since rapidcheck does not + # provide versioned releases + version = "unstable-2020-05-04"; + + src = pkgs.fetchFromGitHub { + owner = "emil-e"; + repo = "rapidcheck"; + rev = "7bc7d302191a4f3d0bf005692677126136e02f60"; + sha256 = "0khawy2n007yk97ls2qqpna4ly09v6rb6hw72nm16kzk3zbyzh17"; + }; + + cmakeFlags = [ + "-DRC_ENABLE_GTEST=ON" + "-DRC_ENABLE_GMOCK=ON" + ]; +}) diff --git a/third_party/re2/default.nix b/third_party/re2/default.nix new file mode 100644 index 000000000000..62bce9e2ee72 --- /dev/null +++ b/third_party/re2/default.nix @@ -0,0 +1,5 @@ +{ pkgs, ... }: + +pkgs.re2.override { + stdenv = pkgs.fullLlvm11Stdenv; +} diff --git a/third_party/rust-crates/OWNERS b/third_party/rust-crates/OWNERS new file mode 100644 index 000000000000..30cd81721c16 --- /dev/null +++ b/third_party/rust-crates/OWNERS @@ -0,0 +1,4 @@ +inherited: true +owners: + - sterni + - Profpatsch diff --git a/third_party/rust-crates/default.nix b/third_party/rust-crates/default.nix new file mode 100644 index 000000000000..66db9331c52b --- /dev/null +++ b/third_party/rust-crates/default.nix @@ -0,0 +1,313 @@ +{ depot, pkgs, ... }: + +# TVL tool rust crate dependencies, where tools like carnix are not used. +# Intended for manual updates, which keeps us honest with what we pull into our closure. + +let + buildRustCrate = attrs@{ + edition ? "2018", + ... + }: pkgs.buildRustCrate (attrs // { + inherit + edition + ; + }); +in + +# TODO: remove this giant with because it screws with the static analyzer +with depot.third_party.rust-crates; + +{ + cfg-if = buildRustCrate { + pname = "cfg-if"; + crateName = "cfg-if"; + version = "1.0.0"; + sha256 = "1fzidq152hnxhg4lj6r2gv4jpnn8yivp27z6q6xy7w6v0dp6bai9"; + }; + + cc = buildRustCrate { + pname = "cc"; + crateName = "cc"; + version = "1.0.66"; + sha256 = "12q71z6ck8wlqrwgi25x3lrryyks9djymswn9b1c6qq0i01jpc1p"; + }; + + ascii = buildRustCrate { + pname = "ascii"; + crateName = "ascii"; + version = "1.0.0"; + edition = "2015"; + sha256 = "0gam8xsn981wfa40srsniivffjsfz1pg0xnigmczk9k7azb1ks1m"; + }; + + regex-syntax = buildRustCrate { + pname = "regex-syntax"; + crateName = "regex-syntax"; + version = "0.6.22"; + edition = "2015"; + sha256 = "0r00n2dgyixacl1sczqp18gxf0xh7x272hcdp62412lypba2gqyg"; + }; + + regex = buildRustCrate { + pname = "regex"; + crateName = "regex"; + version = "1.4.3"; + features = [ "std" ]; + dependencies = [ regex-syntax ]; + edition = "2015"; + sha256 = "0w0b4bh0ng20lf5y8raaxmxj46ikjqpgwy1iggzpby9lhv9vydkp"; + }; + + libloading = buildRustCrate { + pname = "libloading"; + crateName = "libloading"; + version = "0.6.7"; + dependencies = [ cfg-if ]; + edition = "2015"; + sha256 = "111d8zsizswnxiqn43vcgnc2ym9spsx1i6pcfp35ca3yw2ixq95j"; + }; + + tree-sitter = buildRustCrate { + pname = "tree_sitter"; + crateName = "tree-sitter"; + # buildRustCrate isn’t really smart enough to detect the subdir + libPath = "binding_rust/lib.rs"; + # and the build.rs is also not where buildRustCrate would find it + build = "binding_rust/build.rs"; + version = "0.17.1"; + dependencies = [ regex ]; + buildDependencies = [ cc ]; + sha256 = "0jwwbvs4icpra7m1ycvnyri5h3sbw4qrfvgnnvnk72h4w93qhzhr"; + }; + + libc = buildRustCrate { + pname = "libc"; + crateName = "libc"; + version = "0.2.82"; + edition = "2015"; + sha256 = "02zgn6c0xwh331hky417lbr29kmvrw3ylxs8822syyhjfjqszvsx"; + }; + + bitflags = buildRustCrate { + pname = "bitflags"; + crateName = "bitflags"; + version = "1.2.1"; + sha256 = "0b77awhpn7yaqjjibm69ginfn996azx5vkzfjj39g3wbsqs7mkxg"; + }; + + inotify-sys = buildRustCrate { + pname = "inotify-sys"; + crateName = "inotify-sys"; + version = "0.1.5"; + dependencies = [ libc ]; + sha256 = "1yiy577xxhi0j90nbg9nkd8cqwc1xix62rz55jjngvxa5jl5613v"; + }; + + inotify = buildRustCrate { + pname = "inotify"; + crateName = "inotify"; + version = "0.9.2"; + dependencies = [ bitflags libc inotify-sys ]; + sha256 = "0fcknyvknglwwk1pdzdlb4m0ry2dym1yx8r5prf2v00pxnjk0hv2"; + }; + + httparse = buildRustCrate { + pname = "httparse"; + version = "1.3.4"; + crateName = "httparse"; + edition = "2015"; + sha256 = "0dggj4s0cq69bn63q9nqzzay5acmwl33nrbhjjsh5xys8sk2x4jw"; + }; + + version-check = buildRustCrate { + pname = "version-check"; + version = "0.9.2"; + crateName = "version-check"; + edition = "2015"; + sha256 = "1vwvc1mzwv8ana9jv8z933p2xzgj1533qwwl5zr8mi89azyhq21v"; + }; + + memchr = buildRustCrate { + pname = "memchr"; + version = "2.3.3"; + crateName = "memchr"; + edition = "2015"; + sha256 = "1ivxvlswglk6wd46gadkbbsknr94gwryk6y21v64ja7x4icrpihw"; + }; + nom = buildRustCrate { + pname = "nom"; + version = "5.1.1"; + crateName = "nom"; + sha256 = "1gb4r6mjwd645jqh02nhn60i7qkw8cgy3xq1r4clnmvz3cmkv1l0"; + dependencies = [ memchr ]; + buildDependencies = [ version-check ]; + features = [ "std" "alloc" ]; + }; + + base64 = buildRustCrate { + pname = "base64"; + version = "0.13.0"; + crateName = "base64"; + sha256 = "0i0jk5sgq37kc4c90d1g7dp7zvphbg0dbqc1ajnn0vffjxblgamg"; + features = [ "alloc" "std" ]; + }; + + bufstream = buildRustCrate { + pname = "bufstream"; + version = "0.1.4"; + crateName = "bufstream"; + sha256 = "10rqm7jly5jjx7wcc19q6q4m2zsrw3l2v3m1054wnbwvdh42xxf1"; + }; + + autocfg = buildRustCrate { + pname = "autocfg"; + version = "1.0.1"; + crateName = "autocfg"; + edition = "2015"; + sha256 = "1lsjz23jdcchcqbsmlzd4iksg3hc8bdvy677jy0938i2gp24frw1"; + }; + + num-traits = buildRustCrate { + pname = "num-traits"; + version = "0.2.14"; + crateName = "num-traits"; + edition = "2015"; + buildDependencies = [ autocfg ]; + sha256 = "09ac9dcp6cr57vjzyiy213y7312jqcy84mkamp99zr40qd1gwnyk"; + }; + + num-integer = buildRustCrate { + pname = "num-integer"; + version = "0.1.44"; + crateName = "num-integer"; + edition = "2015"; + dependencies = [ num-traits ]; + buildDependencies = [ autocfg ]; + sha256 = "1gdbnfgnivp90h644wmqj4a20yfmdga2xxxacx53pjbcazvfvajc"; + }; + + chrono = buildRustCrate { + pname = "chrono"; + version = "0.4.19"; + crateName = "chrono"; + edition = "2015"; + dependencies = [ num-traits num-integer ]; + features = [ "alloc" "std" ]; + sha256 = "0cjf5dnfbk99607vz6n5r6bhwykcypq5psihvk845sxrhnzadsar"; + }; + + imap-proto = buildRustCrate { + pname = "imap-proto"; + version = "0.10.2"; + crateName = "imap-proto"; + dependencies = [ nom ]; + sha256 = "1bf5r4d0z7c8wxrvr7kjy26500wr7cd4sxz49ix3b3yzc6ayyqv1"; + }; + + lazy_static = buildRustCrate { + pname = "lazy_static"; + version = "1.4.0"; + crateName = "lazy_static"; + sha256 = "13h6sdghdcy7vcqsm2gasfw3qg7ssa0fl3sw7lq6pdkbk52wbyfr"; + }; + + imap = buildRustCrate { + pname = "imap"; + version = "2.4.0"; + crateName = "imap"; + dependencies = [ + base64 + bufstream + chrono + imap-proto + lazy_static + nom + regex + ]; + sha256 = "1nj6x45qnid98nv637623rrh7imcxk0kad89ry8j5dkkgccvjyc0"; + }; + + epoll = buildRustCrate { + pname = "epoll"; + version = "4.3.1"; + crateName = "epoll"; + dependencies = [ bitflags libc ]; + sha256 = "0dgmgdmrfbjkpxn1w3xmmwsm2a623a9qdwn90s8yl78n4a36kbh9"; + }; + + serde = buildRustCrate { + pname = "serde"; + crateName = "serde"; + version = "1.0.123"; + edition = "2015"; + sha256 = "05xl2s1vpf3p7fi2yc9qlzw88d5ap0z3qmhmd7axa6pp9pn1s5xc"; + features = [ "std" ]; + }; + + ryu = buildRustCrate { + pname = "ryu"; + version = "1.0.5"; + crateName = "ryu"; + sha256 = "060y2ln1csix593ingwxr2y3wl236ls0ly1ffkv39h5im7xydhrc"; + }; + + itoa = buildRustCrate { + pname = "itoa"; + version = "0.4.7"; + crateName = "itoa"; + sha256 = "0079jlkcmcaw37wljrvb6r3dqq15nfahkqnl5npvlpdvkg31k11x"; + }; + + serde_json = buildRustCrate { + pname = "serde_json"; + version = "1.0.62"; + crateName = "serde_json"; + sha256 = "0sgc8dycigq0nxr4j613m4q733alfb2i10s6nz80lsbbqgrka21q"; + dependencies = [ serde ryu itoa ]; + features = [ "std" ]; + }; + + log = buildRustCrate { + pname = "log"; + version = "0.4.11"; + crateName = "log"; + sha256 = "0m6xhqxsps5mgd7r91g5mqkndbh8zbjd58p7w75r330zl4n40l07"; + dependencies = [ cfg-if ]; + }; + + mustache = buildRustCrate { + pname = "mustache"; + version = "0.9.0"; + crateName = "mustache"; + edition = "2015"; + sha256 = "1zgl8l15i19lzp90icgwyi6zqdd31b9vm8w129f41d1zd0hs7ayq"; + dependencies = [ log serde ]; + }; + + semver-parser = buildRustCrate { + pname = "semver-parser"; + version = "0.7.0"; + crateName = "semver-parser"; + edition = "2015"; + sha256 = "1da66c8413yakx0y15k8c055yna5lyb6fr0fw9318kdwkrk5k12h"; + dependencies = [ ]; + }; + + semver = buildRustCrate { + pname = "semver"; + version = "0.10.0"; + crateName = "semver"; + edition = "2015"; + sha256 = "0pbkdwlpq4d0hgdrymm2rcw31plni2siwd882gbcbscjvyvrrrqa"; + dependencies = [ semver-parser ]; + }; + + toml = buildRustCrate { + pname = "toml"; + version = "0.5.8"; + crateName = "toml"; + sha256 = "1vwjwmwsy83pbgvvm11a6grbhb09zkcrv9v95wfwv48wjm01wdj4"; + edition = "2018"; + dependencies = [ serde ]; + }; +} diff --git a/third_party/rustsec-advisory-db/default.nix b/third_party/rustsec-advisory-db/default.nix new file mode 100644 index 000000000000..3d3b387be84f --- /dev/null +++ b/third_party/rustsec-advisory-db/default.nix @@ -0,0 +1,24 @@ +# RustSec's advisory db for crates +# +# Update using: +# +# nix-prefetch-git --quiet --url https://github.com/RustSec/advisory-db.git > third_party/rustsec-advisory-db/pin.json +# +# TODO(Profpatsch): automatically update in regular intervals +{ pkgs, ... }: + +let + pin = builtins.fromJSON (builtins.readFile ./pin.json); + + date = builtins.head (builtins.split "T" pin.date); +in + +pkgs.fetchFromGitHub { + name = "advisory-db-${date}"; + owner = "RustSec"; + repo = "advisory-db"; + inherit (pin) + rev + sha256 + ; +} diff --git a/third_party/rustsec-advisory-db/pin.json b/third_party/rustsec-advisory-db/pin.json new file mode 100644 index 000000000000..1155625cffbb --- /dev/null +++ b/third_party/rustsec-advisory-db/pin.json @@ -0,0 +1,11 @@ +{ + "url": "https://github.com/RustSec/advisory-db.git", + "rev": "d29205a680bb8b3a22eaba6e9b2a5a6580274af0", + "date": "2021-10-08T18:17:22+02:00", + "path": "/nix/store/nm8nwgdyrs6mi9dydf6vylc833i3alnn-advisory-db", + "sha256": "0h08kfn2878k5l0qdsxikakrjbqbn6fb8f95zxpqfh5hqzn7mb6b", + "fetchLFS": false, + "fetchSubmodules": false, + "deepClone": false, + "leaveDotGit": false +} diff --git a/third_party/smtprelay/default.nix b/third_party/smtprelay/default.nix new file mode 100644 index 000000000000..6a8b063a5208 --- /dev/null +++ b/third_party/smtprelay/default.nix @@ -0,0 +1,20 @@ +# A simple SMTP relay without the kitchen sink. +{ pkgs, lib, ... }: + +pkgs.buildGoModule { + name = "smtprelay"; + vendorSha256 = "0kv9cv2jca2r90qsf40qmqpw84kgxvbxlf39bfw8rvs2lnmqc2dg"; + + src = pkgs.fetchFromGitHub { + owner = "decke"; + repo = "smtprelay"; + rev = "ed1c3a98889e752291aaca6c64149e48452d0583"; + sha256 = "16q2d2ja2cipjvsnfxmdzixkg85sh15rh9r95w6bw2r1gjqr65hr"; + }; + + meta = with lib; { + description = "Simple Golang SMTP relay/proxy server"; + homepage = https://github.com/decke/smtprelay; + license = licenses.mit; + }; +} diff --git a/tools/cheddar/.gitignore b/tools/cheddar/.gitignore new file mode 100644 index 000000000000..2f7896d1d136 --- /dev/null +++ b/tools/cheddar/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/tools/cheddar/.skip-subtree b/tools/cheddar/.skip-subtree new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/tools/cheddar/.skip-subtree diff --git a/tools/cheddar/Cargo.lock b/tools/cheddar/Cargo.lock new file mode 100644 index 000000000000..0635209a1d0a --- /dev/null +++ b/tools/cheddar/Cargo.lock @@ -0,0 +1,1207 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "ascii" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr", + "safemem", +] + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cheddar" +version = "0.2.0" +dependencies = [ + "clap", + "comrak", + "lazy_static", + "regex", + "rouille", + "serde", + "serde_json", + "syntect", +] + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time 0.1.43", + "winapi", +] + +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "comrak" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b423acba50d5016684beaf643f9991e622633a4c858be6885653071c2da2b0c6" +dependencies = [ + "clap", + "entities", + "lazy_static", + "pest", + "pest_derive", + "regex", + "shell-words", + "twoway 0.2.2", + "typed-arena", + "unicode_categories", + "xdg", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if", + "lazy_static", +] + +[[package]] +name = "deflate" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f95bf05dffba6e6cce8dfbb30def788154949ccd9aed761b472119c21e01c70" +dependencies = [ + "adler32", + "gzip-header", +] + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "entities" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "filetime" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "975ccf83d8d9d0d84682850a38c8169027be83368805971cc4f238c2b245bc98" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.10", + "winapi", +] + +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "gzip-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0131feb3d3bb2a5a238d8a4d09f6353b7ebfdc52e77bccbf4ea6eaa751dde639" +dependencies = [ + "crc32fast", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6" + +[[package]] +name = "line-wrap" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +dependencies = [ + "safemem", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime", + "unicase", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "multipart" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" +dependencies = [ + "buf_redux", + "httparse", + "log", + "mime", + "mime_guess", + "quick-error", + "rand", + "safemem", + "tempfile", + "twoway 0.1.8", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "onig" +version = "6.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b17403cf40f61e3ee059e3e90b7fc0a2953297168d4379b160f80d18fed848a4" +dependencies = [ + "bitflags", + "lazy_static", + "libc", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd3eee045c84695b53b20255bb7317063df090b68e18bfac0abb6c39cf7f33e" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + +[[package]] +name = "pkg-config" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb" + +[[package]] +name = "plist" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38d026d73eeaf2ade76309d0c65db5a35ecf649e3cec428db316243ea9d6711" +dependencies = [ + "base64", + "chrono", + "indexmap", + "line-wrap", + "serde", + "xml-rs", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro2" +version = "1.0.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f5105d4fdaab20335ca9565e106a5d9b82b6219b5ba735731124ac6711d23d" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom 0.2.3", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rouille" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc1bcf3b32bd9ef568402e750404c369ff172a6a34597c858f8ccf5f3bed013" +dependencies = [ + "base64", + "brotli2", + "chrono", + "deflate", + "filetime", + "multipart", + "num_cpus", + "percent-encoding", + "rand", + "serde", + "serde_derive", + "serde_json", + "sha1", + "threadpool", + "time 0.3.3", + "tiny_http", + "url", +] + +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "shell-words" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fa3938c99da4914afedd13bf3d79bcb6c277d1b2c398d23257a304d9e1b074" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "syntect" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b20815bbe80ee0be06e6957450a841185fcf690fe0178f14d77a05ce2caa031" +dependencies = [ + "bincode", + "bitflags", + "flate2", + "fnv", + "lazy_static", + "lazycell", + "onig", + "plist", + "regex-syntax", + "serde", + "serde_derive", + "serde_json", + "walkdir", + "yaml-rust", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall 0.2.10", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "time" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde1cf55178e0293453ba2cca0d5f8392a922e52aa958aee9c28ed02becc6d03" +dependencies = [ + "libc", +] + +[[package]] +name = "tiny_http" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce51b50006056f590c9b7c3808c3bd70f0d1101666629713866c227d6e58d39" +dependencies = [ + "ascii", + "chrono", + "chunked_transfer", + "log", + "url", +] + +[[package]] +name = "tinyvec" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "twoway" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +dependencies = [ + "memchr", +] + +[[package]] +name = "twoway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57ffb460d7c24cd6eda43694110189030a3d1dfe418416d9468fd1c1d290b47" +dependencies = [ + "memchr", + "unchecked-index", +] + +[[package]] +name = "typed-arena" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unchecked-index" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xdg" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de4cfc7dc9727713f386aadce9496f1ed64ea368d9f1f813a54d0f98f8741286" +dependencies = [ + "dirs", +] + +[[package]] +name = "xml-rs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/tools/cheddar/Cargo.toml b/tools/cheddar/Cargo.toml new file mode 100644 index 000000000000..6cc8163c730a --- /dev/null +++ b/tools/cheddar/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "cheddar" +version = "0.2.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +edition = "2018" + +[dependencies] +clap = "2.33" +comrak = "0.10" +lazy_static = "1.4" +rouille = "3.0" +syntect = "4.5.0" +serde_json = "1.0" +regex = "1.4" + +[dependencies.serde] +version = "1.0" +features = [ "derive" ] diff --git a/tools/cheddar/README.md b/tools/cheddar/README.md new file mode 100644 index 000000000000..706f3b62d552 --- /dev/null +++ b/tools/cheddar/README.md @@ -0,0 +1,21 @@ +cheddar +======= + +Cheddar is a tiny Rust tool that uses [syntect][] to render source code to +syntax-highlighted HTML. + +It's invocation is compatible with `cgit` filters, i.e. data is read from +`stdin` and the filename is taken from `argv`: + +```shell +cat README.md | cheddar README.md > README.html + +``` + +In fact, if you are looking at this file on git.tazj.in chances are that it was +rendered by cheddar. + +The name was chosen because I was eyeing a pack of cheddar-flavoured crisps +while thinking about name selection. + +[syntect]: https://github.com/trishume/syntect diff --git a/tools/cheddar/build.rs b/tools/cheddar/build.rs new file mode 100644 index 000000000000..b63b2e337851 --- /dev/null +++ b/tools/cheddar/build.rs @@ -0,0 +1,50 @@ +//! Build script that can be used outside of Nix builds to inject the +//! BAT_SYNTAXES variable when building in development mode. +//! +//! Note that this script assumes that cheddar is in a checkout of the +//! TVL depot. + +use std::process::Command; + +static BAT_SYNTAXES: &str = "BAT_SYNTAXES"; +static ERROR_MESSAGE: &str = r#"Failed to build syntax set. + +When building during development, cheddar expects to be in a checkout +of the TVL depot. This is required to automatically build the syntax +highlighting files that are needed at compile time. + +As cheddar can not automatically detect the location of the syntax +files, you must set the `BAT_SYNTAXES` environment variable to the +right path. + +The expected syntax files are at //third_party/bat_syntaxes in the +depot."#; + +fn main() { + // Do nothing if the variable is already set (e.g. via Nix) + if let Ok(_) = std::env::var(BAT_SYNTAXES) { + return; + } + + // Otherwise ask Nix to build it and inject the result. + let output = Command::new("nix-build") + .arg("-A").arg("third_party.bat_syntaxes") + // ... assuming cheddar is at //tools/cheddar ... + .arg("../..") + .output() + .expect(ERROR_MESSAGE); + + if !output.status.success() { + eprintln!("{}\nNix output: {}", ERROR_MESSAGE, String::from_utf8_lossy(&output.stderr)); + return; + } + + let out_path = String::from_utf8(output.stdout) + .expect("Nix returned invalid output after building syntax set"); + + // Return an instruction to Cargo that will set the environment + // variable during rustc calls. + // + // https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorustc-envvarvalue + println!("cargo:rustc-env={}={}", BAT_SYNTAXES, out_path.trim()); +} diff --git a/tools/cheddar/default.nix b/tools/cheddar/default.nix new file mode 100644 index 000000000000..c8d7ba5ffef2 --- /dev/null +++ b/tools/cheddar/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; + doDoc = false; + + override = x: { + # Use our custom bat syntax set, which is everything from upstream, + # plus additional languages we care about. + BAT_SYNTAXES = "${depot.third_party.bat_syntaxes}"; + }; +} diff --git a/tools/cheddar/src/bin/cheddar.rs b/tools/cheddar/src/bin/cheddar.rs new file mode 100644 index 000000000000..58ef32a1b432 --- /dev/null +++ b/tools/cheddar/src/bin/cheddar.rs @@ -0,0 +1,135 @@ +//! This file defines the binary for cheddar, which can be interacted +//! with in two different ways: +//! +//! 1. As a CLI tool that acts as a cgit filter. +//! 2. As a long-running HTTP server that handles rendering requests +//! (matching the SourceGraph protocol). +use clap::{App, Arg}; +use rouille::Response; +use rouille::{router, try_or_400}; +use serde::Deserialize; +use serde_json::json; +use std::collections::HashMap; +use std::io; + +use cheddar::{THEMES, format_code, format_markdown}; + +// Server endpoint for rendering the syntax of source code. This +// replaces the 'syntect_server' component of Sourcegraph. +fn code_endpoint(request: &rouille::Request) -> rouille::Response { + #[derive(Deserialize)] + struct SourcegraphQuery { + filepath: String, + theme: String, + code: String, + } + + let query: SourcegraphQuery = try_or_400!(rouille::input::json_input(request)); + let mut buf: Vec<u8> = Vec::new(); + + // We don't use syntect with the sourcegraph themes bundled + // currently, so let's fall back to something that is kind of + // similar (tm). + let theme = &THEMES.themes[match query.theme.as_str() { + "Sourcegraph (light)" => "Solarized (light)", + _ => "Solarized (dark)", + }]; + + format_code(theme, &mut query.code.as_bytes(), &mut buf, &query.filepath); + + Response::json(&json!({ + "is_plaintext": false, + "data": String::from_utf8_lossy(&buf) + })) +} + +// Server endpoint for rendering a Markdown file. +fn markdown_endpoint(request: &rouille::Request) -> rouille::Response { + let mut texts: HashMap<String, String> = try_or_400!(rouille::input::json_input(request)); + + for text in texts.values_mut() { + let mut buf: Vec<u8> = Vec::new(); + format_markdown(&mut text.as_bytes(), &mut buf); + *text = String::from_utf8_lossy(&buf).to_string(); + } + + Response::json(&texts) +} + +fn highlighting_server(listen: &str) { + println!("Starting syntax highlighting server on '{}'", listen); + + rouille::start_server(listen, move |request| { + router!(request, + // Markdown rendering route + (POST) (/markdown) => { + markdown_endpoint(request) + }, + + // Code rendering route + (POST) (/) => { + code_endpoint(request) + }, + + _ => { + rouille::Response::empty_404() + }, + ) + }); +} + +fn main() { + // Parse the command-line flags passed to cheddar to determine + // whether it is running in about-filter mode (`--about-filter`) + // and what file extension has been supplied. + let matches = App::new("cheddar") + .about("TVL's syntax highlighter") + .arg( + Arg::with_name("about-filter") + .help("Run as a cgit about-filter (renders Markdown)") + .long("about-filter") + .takes_value(false), + ) + .arg( + Arg::with_name("sourcegraph-server") + .help("Run as a Sourcegraph compatible web-server") + .long("sourcegraph-server") + .takes_value(false), + ) + .arg( + Arg::with_name("listen") + .help("Address to listen on") + .long("listen") + .takes_value(true), + ) + .arg(Arg::with_name("filename").help("File to render").index(1)) + .get_matches(); + + if matches.is_present("sourcegraph-server") { + highlighting_server( + matches + .value_of("listen") + .expect("Listening address is required for server mode"), + ); + return; + } + + let filename = matches.value_of("filename").expect("filename is required"); + + let stdin = io::stdin(); + let mut in_handle = stdin.lock(); + + let stdout = io::stdout(); + let mut out_handle = stdout.lock(); + + if matches.is_present("about-filter") && filename.ends_with(".md") { + format_markdown(&mut in_handle, &mut out_handle); + } else { + format_code( + &THEMES.themes["InspiredGitHub"], + &mut in_handle, + &mut out_handle, + filename, + ); + } +} diff --git a/tools/cheddar/src/lib.rs b/tools/cheddar/src/lib.rs new file mode 100644 index 000000000000..e5d7aad1e62c --- /dev/null +++ b/tools/cheddar/src/lib.rs @@ -0,0 +1,337 @@ +//! This file implements the rendering logic of cheddar with public +//! functions for syntax-highlighting code and for turning Markdown +//! into HTML with TVL extensions. +use comrak::arena_tree::Node; +use comrak::nodes::{Ast, AstNode, NodeCodeBlock, NodeHtmlBlock, NodeValue}; +use comrak::{format_html, parse_document, Arena, ComrakOptions}; +use lazy_static::lazy_static; +use regex::Regex; +use std::cell::RefCell; +use std::collections::HashMap; +use std::env; +use std::ffi::OsStr; +use std::io; +use std::io::BufRead; +use std::io::Write; +use std::path::Path; +use syntect::dumps::from_binary; +use syntect::easy::HighlightLines; +use syntect::highlighting::{Theme, ThemeSet}; +use syntect::parsing::{SyntaxReference, SyntaxSet}; +use syntect::util::LinesWithEndings; + +use syntect::html::{ + append_highlighted_html_for_styled_line, start_highlighted_html_snippet, IncludeBackground, +}; + +#[cfg(test)] +mod tests; + +lazy_static! { + // Load syntaxes lazily. Initialisation might not be required in + // the case of Markdown rendering (if there's no code blocks + // within the document). + // + // Note that the syntax set is included from the path pointed to + // by the BAT_SYNTAXES environment variable at compile time. This + // variable is populated by Nix and points to TVL's syntax set. + static ref SYNTAXES: SyntaxSet = from_binary(include_bytes!(env!("BAT_SYNTAXES"))); + pub static ref THEMES: ThemeSet = ThemeSet::load_defaults(); + + // Configure Comrak's Markdown rendering with all the bells & + // whistles! + static ref MD_OPTS: ComrakOptions = { + let mut options = ComrakOptions::default(); + + // Enable non-standard Markdown features: + options.extension.strikethrough = true; + options.extension.tagfilter = true; + options.extension.table = true; + options.extension.autolink = true; + options.extension.tasklist = true; + options.extension.header_ids = Some(String::new()); // yyeeesss! + options.extension.footnotes = true; + options.extension.description_lists = true; + options.extension.front_matter_delimiter = Some("---".to_owned()); + + // Required for tagfilter + options.render.unsafe_ = true; + + options + }; + + // Configures a map of specific filenames to languages, for cases + // where the detection by extension or other heuristics fails. + static ref FILENAME_OVERRIDES: HashMap<&'static str, &'static str> = { + let mut map = HashMap::new(); + // rules.pl is the canonical name of the submit rule file in + // Gerrit, which is written in Prolog. + map.insert("rules.pl", "Prolog"); + map + }; + + // Default shortlink set used in cheddar (i.e. TVL's shortlinks) + static ref TVL_LINKS: Vec<Shortlink> = vec", + } + ]; +} + +/// Structure that describes a single shortlink that should be +/// automatically highlighted. Highlighting is performed as a string +/// replacement over input Markdown. +pub struct Shortlink { + /// Short link pattern to recognise. Make sure to anchor these + /// correctly. + pub pattern: Regex, + + /// Replacement string, as per the documentation of + /// [`Regex::replace`]. + pub replacement: &'static str, +} + +// HTML fragment used when rendering inline blocks in Markdown documents. +// Emulates the GitHub style (subtle background hue and padding). +const BLOCK_PRE: &str = "<pre style=\"background-color:#f6f8fa;padding:16px;\">\n"; + +fn should_continue(res: &io::Result<usize>) -> bool { + match *res { + Ok(n) => n > 0, + Err(_) => false, + } +} + +// This function is taken from the Comrak documentation. +fn iter_nodes<'a, F>(node: &'a AstNode<'a>, f: &F) +where + F: Fn(&'a AstNode<'a>), +{ + f(node); + for c in node.children() { + iter_nodes(c, f); + } +} + +// Many of the syntaxes in the syntax list have random capitalisations, which +// means that name matching for the block info of a code block in HTML fails. +// +// Instead, try finding a syntax match by comparing case insensitively (for +// ASCII characters, anyways). +fn find_syntax_case_insensitive(info: &str) -> Option<&'static SyntaxReference> { + // TODO(tazjin): memoize this lookup + SYNTAXES + .syntaxes() + .iter() + .rev() + .find(|&s| info.eq_ignore_ascii_case(&s.name)) +} + +// Replaces code-block inside of a Markdown AST with HTML blocks rendered by +// syntect. This enables static (i.e. no JavaScript) syntax highlighting, even +// of complex languages. +fn highlight_code_block(code_block: &NodeCodeBlock) -> NodeValue { + let theme = &THEMES.themes["InspiredGitHub"]; + let info = String::from_utf8_lossy(&code_block.info); + + let syntax = find_syntax_case_insensitive(&info) + .or_else(|| SYNTAXES.find_syntax_by_extension(&info)) + .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text()); + + let code = String::from_utf8_lossy(&code_block.literal); + + let rendered = { + // Write the block preamble manually to get exactly the + // desired layout: + let mut hl = HighlightLines::new(syntax, theme); + let mut buf = BLOCK_PRE.to_string(); + + for line in LinesWithEndings::from(&code) { + let regions = hl.highlight(line, &SYNTAXES); + append_highlighted_html_for_styled_line(®ions[..], IncludeBackground::No, &mut buf); + } + + buf.push_str("</pre>"); + buf + }; + + let mut block = NodeHtmlBlock::default(); + block.literal = rendered.into_bytes(); + + NodeValue::HtmlBlock(block) +} + +// Supported callout elements (which each have their own distinct rendering): +enum Callout { + Todo, + Warning, + Question, + Tip, +} + +// Determine whether the first child of the supplied node contains a text that +// should cause a callout section to be rendered. +fn has_callout<'a>(node: &Node<'a, RefCell<Ast>>) -> Option<Callout> { + match node.first_child().map(|c| c.data.borrow()) { + Some(child) => match &child.value { + NodeValue::Text(text) => { + if text.starts_with(b"TODO") { + return Some(Callout::Todo); + } else if text.starts_with(b"WARNING") { + return Some(Callout::Warning); + } else if text.starts_with(b"QUESTION") { + return Some(Callout::Question); + } else if text.starts_with(b"TIP") { + return Some(Callout::Tip); + } + + None + } + _ => None, + }, + _ => None, + } +} + +// Replace instances of known shortlinks in the input document with +// Markdown syntax for a highlighted link. +fn linkify_shortlinks(mut text: String, shortlinks: &[Shortlink]) -> String { + for link in shortlinks { + text = link + .pattern + .replace_all(&text, link.replacement) + .to_string(); + } + + return text; +} + +fn format_callout_paragraph(callout: Callout) -> NodeValue { + let class = match callout { + Callout::Todo => "cheddar-todo", + Callout::Warning => "cheddar-warning", + Callout::Question => "cheddar-question", + Callout::Tip => "cheddar-tip", + }; + + let mut block = NodeHtmlBlock::default(); + block.literal = format!("<p class=\"cheddar-callout {}\">", class).into_bytes(); + NodeValue::HtmlBlock(block) +} + +pub fn format_markdown_with_shortlinks<R: BufRead, W: Write>( + reader: &mut R, + writer: &mut W, + shortlinks: &[Shortlink], +) { + let document = { + let mut buffer = String::new(); + reader + .read_to_string(&mut buffer) + .expect("reading should work"); + buffer + }; + + let arena = Arena::new(); + let root = parse_document(&arena, &linkify_shortlinks(document, shortlinks), &MD_OPTS); + + // This node must exist with a lifetime greater than that of the parsed AST + // in case that callouts are encountered (otherwise insertion into the tree + // is not possible). + let mut p_close_value = NodeHtmlBlock::default(); + p_close_value.literal = b"</p>".to_vec(); + + let p_close_node = Ast::new(NodeValue::HtmlBlock(p_close_value)); + let p_close = Node::new(RefCell::new(p_close_node)); + + // Special features of Cheddar are implemented by traversing the + // arena and reacting on nodes that we might want to modify. + iter_nodes(root, &|node| { + let mut ast = node.data.borrow_mut(); + let new = match &ast.value { + // Syntax highlighting is implemented by replacing the + // code block node with literal HTML. + NodeValue::CodeBlock(code) => Some(highlight_code_block(code)), + + NodeValue::Paragraph => { + if let Some(callout) = has_callout(node) { + node.insert_after(&p_close); + Some(format_callout_paragraph(callout)) + } else { + None + } + } + _ => None, + }; + + if let Some(new_value) = new { + ast.value = new_value + } + }); + + format_html(root, &MD_OPTS, writer).expect("Markdown rendering failed"); +} + +pub fn format_markdown<R: BufRead, W: Write>(reader: &mut R, writer: &mut W) { + format_markdown_with_shortlinks(reader, writer, &TVL_LINKS) +} + +fn find_syntax_for_file(filename: &str) -> &'static SyntaxReference { + (*FILENAME_OVERRIDES) + .get(filename) + .and_then(|name| SYNTAXES.find_syntax_by_name(name)) + .or_else(|| { + Path::new(filename) + .extension() + .and_then(OsStr::to_str) + .and_then(|s| SYNTAXES.find_syntax_by_extension(s)) + }) + .unwrap_or_else(|| SYNTAXES.find_syntax_plain_text()) +} + +pub fn format_code<R: BufRead, W: Write>( + theme: &Theme, + reader: &mut R, + writer: &mut W, + filename: &str, +) { + let mut linebuf = String::new(); + + // Get the first line, we might need it for syntax identification. + let mut read_result = reader.read_line(&mut linebuf); + let syntax = find_syntax_for_file(filename); + + let mut hl = HighlightLines::new(syntax, theme); + let (mut outbuf, bg) = start_highlighted_html_snippet(theme); + + // Rather than using the `lines` iterator, read each line manually + // and maintain buffer state. + // + // This is done because the syntax highlighter requires trailing + // newlines to be efficient, and those are stripped in the lines + // iterator. + while should_continue(&read_result) { + let regions = hl.highlight(&linebuf, &SYNTAXES); + + append_highlighted_html_for_styled_line( + ®ions[..], + IncludeBackground::IfDifferent(bg), + &mut outbuf, + ); + + // immediately output the current state to avoid keeping + // things in memory + write!(writer, "{}", outbuf).expect("write should not fail"); + + // merry go round again + linebuf.clear(); + outbuf.clear(); + read_result = reader.read_line(&mut linebuf); + } + + writeln!(writer, "</pre>").expect("write should not fail"); +} diff --git a/tools/cheddar/src/tests.rs b/tools/cheddar/src/tests.rs new file mode 100644 index 000000000000..5b7b1cc52a95 --- /dev/null +++ b/tools/cheddar/src/tests.rs @@ -0,0 +1,97 @@ +use super::*; +use std::io::BufReader; + +// Markdown rendering expectation, ignoring leading and trailing +// whitespace in the input and output. +fn expect_markdown(input: &str, expected: &str) { + let mut input_buf = BufReader::new(input.trim().as_bytes()); + let mut out_buf: Vec<u8> = vec![]; + format_markdown(&mut input_buf, &mut out_buf); + + let out_string = String::from_utf8(out_buf).expect("output should be UTF8"); + assert_eq!(out_string.trim(), expected.trim()); +} + +#[test] +fn renders_simple_markdown() { + expect_markdown("hello", "<p>hello</p>\n"); +} + +#[test] +fn renders_callouts() { + expect_markdown( + "TODO some task.", + r#"<p class="cheddar-callout cheddar-todo"> +TODO some task. +</p> +"#, + ); + + expect_markdown( + "WARNING: be careful", + r#"<p class="cheddar-callout cheddar-warning"> +WARNING: be careful +</p> +"#, + ); + + expect_markdown( + "TIP: note the thing", + r#"<p class="cheddar-callout cheddar-tip"> +TIP: note the thing +</p> +"#, + ); +} + +#[test] +fn renders_code_snippets() { + expect_markdown( + r#" +Code: +```nix +toString 42 +``` +"#, + r#" +<p>Code:</p> +<pre style="background-color:#f6f8fa;padding:16px;"> +<span style="color:#62a35c;">toString </span><span style="color:#0086b3;">42 +</span></pre> +"#, + ); +} + +#[test] +fn highlights_bug_link() { + expect_markdown( + "Please look at b/123.", + "<p>Please look at <a href=\"https://b.tvl.fyi/123\">b/123</a>.</p>", + ); +} + +#[test] +fn highlights_cl_link() { + expect_markdown( + "Please look at cl/420.", + "<p>Please look at <a href=\"https://cl.tvl.fyi/420\">cl/420</a>.</p>", + ); +} + +#[test] +fn highlights_multiple_shortlinks() { + expect_markdown( + "Please look at cl/420, b/123.", + "<p>Please look at <a href=\"https://cl.tvl.fyi/420\">cl/420</a>, <a href=\"https://b.tvl.fyi/123\">b/123</a>.</p>", + ); + + expect_markdown( + "b/213/cl/213 are different things", + "<p><a href=\"https://b.tvl.fyi/213\">b/213</a>/<a href=\"https://cl.tvl.fyi/213\">cl/213</a> are different things</p>", + ); +} + +#[test] +fn ignores_invalid_shortlinks() { + expect_markdown("b/abc is not a real bug", "<p>b/abc is not a real bug</p>"); +} diff --git a/tools/depot-build.nix b/tools/depot-build.nix new file mode 100644 index 000000000000..62b4c7fc4476 --- /dev/null +++ b/tools/depot-build.nix @@ -0,0 +1,8 @@ +# Utility script for building any arbitrary depot path in its folder. +{ pkgs, ... }: + +pkgs.writeShellScriptBin "depot-build" '' + TARGET=$(git rev-parse --show-prefix | sed 's|/$||') + echo "Building //$TARGET" + nix-build -A $(echo $TARGET | sed 's|/|.|g') $(${pkgs.git}/bin/git rev-parse --show-toplevel) +'' diff --git a/tools/depot-nixpkgs-update.nix b/tools/depot-nixpkgs-update.nix new file mode 100644 index 000000000000..2475ca2e2330 --- /dev/null +++ b/tools/depot-nixpkgs-update.nix @@ -0,0 +1,44 @@ +{ pkgs, depot, ... }: + +let + inherit (depot.nix) + getBins + ; + + stableRelease = "21.05"; + + channelsUrl = "https://channels.nixos.org"; + archiveUrl = "https://github.com/NixOS/nixpkgs/archive/"; + + bins = getBins pkgs.nix [ "nix-prefetch-url" ] + // getBins pkgs.curl [ "curl" ] + ; + +in + +pkgs.writers.writeDashBin "depot-nixpkgs-update" '' + set -e + + printSet() { + setname="$1" + shift + channel="$1" + shift + + commit="$(${bins.curl} -L "${channelsUrl}/$channel/git-revision")" + date="$(curl -i -L "${channelsUrl}/$channel/git-revision" \ + | grep ^last-modified \ + | sed 's/^last-modified: \(.\+\)\r/\1/')" + hash="$(${bins.nix-prefetch-url} --unpack --type sha256 "${archiveUrl}/$commit.tar.gz")" + + printf '%s\n' " + # Tracking $channel as of $(date --rfc-3339=date --date="$date"). + $setname = { + commit = \"$commit\"; + sha256 = \"$hash\"; + };" + } + + printSet unstableHashes nixos-unstable + printSet stableHashes nixos-${stableRelease} +'' diff --git a/tools/depot-scanner/OWNERS b/tools/depot-scanner/OWNERS new file mode 100644 index 000000000000..cefacea4d049 --- /dev/null +++ b/tools/depot-scanner/OWNERS @@ -0,0 +1,3 @@ +inherit: true +owners: + - riking diff --git a/tools/depot-scanner/default.nix b/tools/depot-scanner/default.nix new file mode 100644 index 000000000000..e6fd5dec292c --- /dev/null +++ b/tools/depot-scanner/default.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ...}: + +let + localProto = depot.nix.buildGo.grpc { + name = "code.tvl.fyi/tools/depot-scanner/proto"; + proto = ./depot_scanner.proto; + }; +in depot.nix.buildGo.program { + name = "depot-scanner"; + srcs = [ + ./main.go + ]; + deps = [ + localProto + ]; +} // { inherit localProto; meta.ci = false; } diff --git a/tools/depot-scanner/depot_scanner.proto b/tools/depot-scanner/depot_scanner.proto new file mode 100644 index 000000000000..5249daebf495 --- /dev/null +++ b/tools/depot-scanner/depot_scanner.proto @@ -0,0 +1,46 @@ +// Copyright 2020 TVL +// SPDX-License-Identifier: MIT + +syntax = "proto3"; +package tvl.tools.depot_scanner; +option go_package = "code.tvl.fyi/tools/depot-scanner/proto"; + +enum PathType { + UNKNOWN = 0; + DEPOT = 1; + STORE = 2; + CORE = 3; +} + +message ScanRequest { + // Which revision of the depot + string revision = 1; + string attr = 2; + // Optionally, the attr to evaluate can be provided as a path to a folder or a + // .nix file. This is used by the HTTP service. + string attrAsPath = 3; +} + +message ScanResponse { + repeated string depotPath = 1; + repeated string nixStorePath = 2; + repeated string corePkgsPath = 4; + repeated string otherPath = 3; + + bytes derivation = 5; +} + +message ArchiveRequest { + repeated string depotPath = 1; +} + +message ArchiveChunk { + bytes chunk = 1; +} + +service DepotScanService { + rpc Scan(ScanRequest) returns (ScanResponse); + + rpc MakeArchive(ArchiveRequest) returns (stream ArchiveChunk); +} + diff --git a/tools/depot-scanner/go.mod b/tools/depot-scanner/go.mod new file mode 100644 index 000000000000..bdd22fc1ef01 --- /dev/null +++ b/tools/depot-scanner/go.mod @@ -0,0 +1,3 @@ +module code.tvl.fyi/tools/depot-scanner + +go 1.14 diff --git a/tools/depot-scanner/main.go b/tools/depot-scanner/main.go new file mode 100644 index 000000000000..273190258958 --- /dev/null +++ b/tools/depot-scanner/main.go @@ -0,0 +1,222 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "os/exec" + "strings" + + pb "code.tvl.fyi/tools/depot-scanner/proto" +) + +var nixInstantiatePath = flag.String("nix-bin", "/run/current-system/sw/bin/nix-instantiate", "path to nix-instantiate") +var depotRoot = flag.String("depot", envOr("DEPOT_ROOT", "/depot/"), "path to tvl.fyi depot at current canon") +var nixStoreRoot = flag.String("store-path", "/nix/store/", "prefix for all valid nix store paths") + +var modeFlag = flag.String("mode", modeArchive, "operation mode. valid values: tar, print") +var onlyFlag = flag.String("only", "", "only enable the listed output types, comma separated. valid values: DEPOT, STORE, CORE, UNKNOWN") +var relativeFlag = flag.Bool("relpath", false, "when printing paths, print them relative to the root of their path type") + +const ( + modeArchive = "tar" + modePrint = "print" +) + +const ( + // String that identifies a path as belonging to nix corepkgs. + corePkgsString = "/share/nix/corepkgs/" + + depotTraceString = "trace: depot-scan: " +) + +type fileScanType int + +const ( + unknownPath fileScanType = iota + depotPath + nixStorePath + corePkgsPath +) + +func launchNix(attr string) (*exec.Cmd, io.ReadCloser, io.ReadCloser, error) { + cmd := exec.Command(*nixInstantiatePath, "--trace-file-access", "-A", attr) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, nil, nil, err + } + stderr, err := cmd.StderrPipe() + if err != nil { + stdout.Close() + return nil, nil, nil, err + } + + err = cmd.Start() + if err != nil { + stdout.Close() + stderr.Close() + return nil, nil, nil, err + } + + return cmd, stdout, stderr, nil +} + +func categorizePath(path string) fileScanType { + if strings.HasPrefix(path, *nixStoreRoot) { + if strings.Contains(path, corePkgsString) { + return corePkgsPath + } + return nixStorePath + } else if strings.HasPrefix(path, *depotRoot) { + return depotPath + } else if strings.Contains(path, corePkgsString) { + return corePkgsPath + } + return unknownPath +} + +func addPath(path string, out map[fileScanType]map[string]struct{}) { + cat := categorizePath(path) + if out[cat] == nil { + out[cat] = make(map[string]struct{}) + } + + out[cat][path] = struct{}{} +} + +func consumeOutput(stdout, stderr io.ReadCloser) (map[fileScanType]map[string]struct{}, string, error) { + result := make(map[fileScanType]map[string]struct{}) + + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, depotTraceString) { + addPath(strings.TrimPrefix(line, depotTraceString), result) + } + } + if scanner.Err() != nil { + return nil, "", scanner.Err() + } + + // Get derivation path + derivPath := "" + scanner = bufio.NewScanner(stdout) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, *nixStoreRoot) { + derivPath = line + // consume the rest of the output + } + } + if scanner.Err() != nil { + return nil, "", scanner.Err() + } + + return result, derivPath, nil +} + +func main() { + flag.Parse() + + checkDepotRoot() + + enabledPathTypes := make(map[pb.PathType]bool, 4) + if len(*onlyFlag) > 0 { + enabledOutputs := strings.Split(*onlyFlag, ",") + for _, v := range enabledOutputs { + i, ok := pb.PathType_value[strings.ToUpper(v)] + if !ok { + fmt.Fprintln(os.Stderr, "warning: unrecognized PathType name: ", v) + continue + } + enabledPathTypes[pb.PathType(i)] = true + } + } else { + // Default + enabledPathTypes = map[pb.PathType]bool{ + pb.PathType_UNKNOWN: true, + pb.PathType_DEPOT: true, + pb.PathType_STORE: true, + pb.PathType_CORE: true, + } + } + + cmd, stdout, stderr, err := launchNix(flag.Arg(0)) + if err != nil { + panic(fmt.Errorf("could not launch nix: %w", err)) + } + results, derivPath, err := consumeOutput(stdout, stderr) + if err != nil { + err2 := cmd.Wait() + if err2 != nil { + panic(fmt.Errorf("nix-instantiate failed: %w\nadditionally, while reading output: %w", err2, err)) + } + panic(fmt.Errorf("problem reading nix output: %w", err)) + } + err = cmd.Wait() + if err != nil { + panic(fmt.Errorf("nix-instantiate failed: %w", err)) + } + + _ = derivPath + + if *modeFlag == "print" { + if enabledPathTypes[pb.PathType_STORE] { + for k, _ := range results[nixStorePath] { + if *relativePath { + k = strings.TrimPrefix(k, *nixStoreRoot) + k = strings.TrimPrefix(k, "/") + } + fmt.Println(k) + } + } + if enabledPathTypes[pb.PathType_DEPOT] { + for k, _ := range results[depotPath] { + if *relativeFlag { + k = strings.TrimPrefix(k, *depotRoot) + k = strings.TrimPrefix(k, "/") + } + fmt.Println(k) + } + } + if enabledPathTypes[pb.PathType_CORE] { + for k, _ := range results[corePkgsPath] { + // TODO relativeFlag + fmt.Println(k) + } + } + if enabledPathTypes[pb.PathType_UNKNOWN] { + for k, _ := range results[unknownPath] { + fmt.Println(k) + } + } + } else { + panic("unimplemented") + } +} + +func envOr(envVar, def string) string { + v := os.Getenv(envVar) + if v == "" { + return def + } + return v +} + +func checkDepotRoot() { + if *depotRoot == "" { + fmt.Fprintln(os.Stderr, "error: DEPOT_ROOT / -depot not set") + os.Exit(2) + } + _, err := os.Stat(*depotRoot) + if os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "error: %q does not exist\ndid you forget to set DEPOT_ROOT / --depot ?\n", *depotRoot) + os.Exit(1) + } else if err != nil { + fmt.Fprintf(os.Stderr, "error: could not stat %q: %v\n", *depotRoot, err) + os.Exit(1) + } + +} diff --git a/tools/emacs-pkgs/buildEmacsPackage.nix b/tools/emacs-pkgs/buildEmacsPackage.nix new file mode 100644 index 000000000000..160c0626136d --- /dev/null +++ b/tools/emacs-pkgs/buildEmacsPackage.nix @@ -0,0 +1,34 @@ +# Builder for depot-internal Emacs packages. Packages built using this +# builder are added into the Emacs packages fixpoint under +# `emacsPackages.tvlPackages`, which in turn makes it possible to use +# them with special Emacs features like native compilation. +# +# Arguments passed to the builder are the same as +# emacsPackages.trivialBuild, except: +# +# * packageRequires is not used +# +# * externalRequires takes a selection function for packages from +# emacsPackages +# +# * internalRequires takes other depot packages +{ pkgs, ... }: + +buildArgs: + +pkgs.callPackage({ emacsPackages }: + +let + # Select external dependencies from the emacsPackages set + externalDeps = (buildArgs.externalRequires or (_: [])) emacsPackages; + + # Override emacsPackages for depot-internal packages + internalDeps = map (p: p.override { inherit emacsPackages; }) + (buildArgs.internalRequires or []); + + trivialBuildArgs = builtins.removeAttrs buildArgs [ + "externalRequires" "internalRequires" + ] // { + packageRequires = externalDeps ++ internalDeps; + }; +in emacsPackages.trivialBuild trivialBuildArgs) {} diff --git a/tools/emacs-pkgs/defzone/defzone.el b/tools/emacs-pkgs/defzone/defzone.el new file mode 100644 index 000000000000..ffd359e5ff83 --- /dev/null +++ b/tools/emacs-pkgs/defzone/defzone.el @@ -0,0 +1,60 @@ +;;; defzone.el --- Generate zone files from Elisp -*- lexical-binding: t; -*- + +(require 'dash) +(require 'dash-functional) +(require 's) + +(defun record-to-record (zone record &optional subdomain) + "Evaluate a record definition and turn it into a zone file + record in ZONE, optionally prefixed with SUBDOMAIN." + + (cl-labels ((plist->alist (plist) + (when plist + (cons + (cons (car plist) (cadr plist)) + (plist->alist (cddr plist)))))) + (let ((name (if subdomain (s-join "." (list subdomain zone)) zone))) + (pcase record + ;; SOA RDATA (RFC 1035; 3.3.13) + ((and `(SOA . (,ttl . ,keys)) + (let (map (:mname mname) (:rname rname) (:serial serial) + (:refresh refresh) (:retry retry) (:expire expire) + (:minimum min)) + (plist->alist keys))) + (if-let ((missing (-filter #'null (not (list mname rname serial + refresh retry expire min))))) + (error "Missing fields in SOA record: %s" missing) + (format "%s %s IN SOA %s %s %s %s %s %s %s" + name ttl mname rname serial refresh retry expire min))) + + (`(NS . (,ttl . ,targets)) + (->> targets + (-map (lambda (target) (format "%s %s IN NS %s" name ttl target))) + (s-join "\n"))) + + (`(MX . (,ttl . ,pairs)) + (->> pairs + (-map (-lambda ((preference . exchange)) + (format "%s %s IN MX %s %s" name ttl preference exchange))) + (s-join "\n"))) + + (`(TXT ,ttl ,text) (format "%s %s IN TXT %s" name ttl (prin1-to-string text))) + + (`(A . (,ttl . ,ips)) + (->> ips + (-map (lambda (ip) (format "%s %s IN A %s" name ttl ip))) + (s-join "\n"))) + + (`(CNAME ,ttl ,target) (format "%s %s IN CNAME %s" name ttl target)) + + ((and `(,sub . ,records) + (guard (stringp sub))) + (s-join "\n" (-map (lambda (r) (record-to-record zone r sub)) records))) + + (_ (error "Invalid record definition: %s" record)))))) + +(defmacro defzone (fqdn &rest records) + "Generate zone file for the zone at FQDN from a simple DSL." + (declare (indent defun)) + + `(s-join "\n" (-map (lambda (r) (record-to-record ,fqdn r)) (quote ,records)))) diff --git a/tools/emacs-pkgs/defzone/example.el b/tools/emacs-pkgs/defzone/example.el new file mode 100644 index 000000000000..e9c86d25eec8 --- /dev/null +++ b/tools/emacs-pkgs/defzone/example.el @@ -0,0 +1,45 @@ +;;; example.el - usage example for defzone macro + +(defzone "tazj.in." + (SOA 21600 + :mname "ns-cloud-a1.googledomains.com." + :rname "cloud-dns-hostmaster.google.com." + :serial 123 + :refresh 21600 + :retry 3600 + :expire 1209600 + :minimum 300) + + (NS 21600 + "ns-cloud-a1.googledomains.com." + "ns-cloud-a2.googledomains.com." + "ns-cloud-a3.googledomains.com." + "ns-cloud-a4.googledomains.com.") + + (MX 300 + (1 . "aspmx.l.google.com.") + (5 . "alt1.aspmx.l.google.com.") + (5 . "alt2.aspmx.l.google.com.") + (10 . "alt3.aspmx.l.google.com.") + (10 . "alt4.aspmx.l.google.com.")) + + (TXT 3600 "google-site-verification=d3_MI1OwD6q2OT42Vvh0I9w2u3Q5KFBu-PieNUE1Fig") + + (A 300 "34.98.120.189") + + ;; Nested record sets are indicated by a list that starts with a + ;; string (this is just joined, so you can nest multiple levels at + ;; once) + ("blog" + ;; Blog "storage engine" is in a separate DNS zone + (NS 21600 + "ns-cloud-c1.googledomains.com." + "ns-cloud-c2.googledomains.com." + "ns-cloud-c3.googledomains.com." + "ns-cloud-c4.googledomains.com.")) + + ("git" + (A 300 "34.98.120.189") + (TXT 300 "<3 edef")) + + ("files" (CNAME 300 "c.storage.googleapis.com."))) diff --git a/tools/emacs-pkgs/dottime/default.nix b/tools/emacs-pkgs/dottime/default.nix new file mode 100644 index 000000000000..b819e9c14d2c --- /dev/null +++ b/tools/emacs-pkgs/dottime/default.nix @@ -0,0 +1,7 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage { + pname = "dottime"; + version = "1.0"; + src = ./dottime.el; +} diff --git a/tools/emacs-pkgs/dottime/dottime.el b/tools/emacs-pkgs/dottime/dottime.el new file mode 100644 index 000000000000..2446f6488f32 --- /dev/null +++ b/tools/emacs-pkgs/dottime/dottime.el @@ -0,0 +1,81 @@ +;;; dottime.el --- use dottime in the modeline +;; +;; Copyright (C) 2019 Google Inc. +;; +;; Author: Vincent Ambo <tazjin@google.com> +;; Version: 1.0 +;; Package-Requires: (cl-lib) +;; +;;; Commentary: +;; +;; This package changes the display of time in the modeline to use +;; dottime (see https://dotti.me/) instead of the standard time +;; display. +;; +;; Modeline dottime display is enabled by calling +;; `dottime-display-mode' and dottime can be used in Lisp code via +;; `dottime-format'. + +(require 'cl-lib) +(require 'time) + +(defun dottime--format-string (&optional offset prefix) + "Creates the dottime format string for `format-time-string' + based on the local timezone." + + (let* ((offset-sec (or offset (car (current-time-zone)))) + (offset-hours (/ offset-sec 60 60)) + (base (concat prefix "%m-%dT%H·%M"))) + (if (/= offset-hours 0) + (concat base (format "%0+3d" offset-hours)) + base))) + +(defun dottime--display-time-update-advice (orig) + "Function used as advice to `display-time-update' with a + rebound definition of `format-time-string' that renders all + timestamps as dottime." + + (cl-letf* ((format-orig (symbol-function 'format-time-string)) + ((symbol-function 'format-time-string) + (lambda (&rest _) + (funcall format-orig (dottime--format-string) nil t)))) + (funcall orig))) + +(defun dottime-format (&optional time offset prefix) + "Format the given TIME in dottime at OFFSET. If TIME is nil, + the current time will be used. PREFIX is prefixed to the format + string verbatim. + + OFFSET can be an integer representing an offset in seconds, or + the argument can be elided in which case the system time zone + is used." + + (format-time-string (dottime--format-string offset prefix) time t)) + +(defun dottime-display-mode (arg) + "Enable time display as dottime. Disables dottime if called + with prefix 0 or nil." + + (interactive "p") + (if (or (eq arg 0) (eq arg nil)) + (advice-remove 'display-time-update #'dottime--display-time-update-advice) + (advice-add 'display-time-update :around #'dottime--display-time-update-advice)) + (display-time-update) + + ;; Amend the time display in telega.el to use dottime. + ;; + ;; This will never display offsets in the chat window, as those are + ;; always visible in the modeline anyways. + (when (featurep 'telega) + (defun telega-ins--dottime-advice (orig timestamp) + (let* ((dtime (decode-time timestamp t)) + (current-ts (time-to-seconds (current-time))) + (ctime (decode-time current-ts)) + (today00 (telega--time-at00 current-ts ctime))) + (if (> timestamp today00) + (telega-ins (format "%02d·%02d" (nth 2 dtime) (nth 1 dtime))) + (funcall orig timestamp)))) + + (advice-add 'telega-ins--date :around #'telega-ins--dottime-advice))) + +(provide 'dottime) diff --git a/tools/emacs-pkgs/nix-util/default.nix b/tools/emacs-pkgs/nix-util/default.nix new file mode 100644 index 000000000000..ffeb1cefade7 --- /dev/null +++ b/tools/emacs-pkgs/nix-util/default.nix @@ -0,0 +1,7 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage { + pname = "nix-util"; + version = "1.0"; + src = ./nix-util.el; +} diff --git a/tools/emacs-pkgs/nix-util/nix-util.el b/tools/emacs-pkgs/nix-util/nix-util.el new file mode 100644 index 000000000000..4b9dd31a022e --- /dev/null +++ b/tools/emacs-pkgs/nix-util/nix-util.el @@ -0,0 +1,103 @@ +;;; nix-util.el --- Utilities for dealing with Nix code. -*- lexical-binding: t; -*- +;; +;; Copyright (C) 2019 Google Inc. +;; +;; Author: Vincent Ambo <tazjin@google.com> +;; Version: 1.0 +;; Package-Requires: (json map) +;; +;;; Commentary: +;; +;; This package adds some functionality that I find useful when +;; working in Nix buffers or programs installed from Nix. + +(require 'json) +(require 'map) + +(defvar nix-depot-path "/home/tazjin/depot") + +(defun nix/prefetch-github (owner repo) ; TODO(tazjin): support different branches + "Fetch the master branch of a GitHub repository and insert the + call to `fetchFromGitHub' at point." + + (interactive "sOwner: \nsRepository: ") + + (let* (;; Keep these vars around for output insertion + (point (point)) + (buffer (current-buffer)) + (name (concat "github-fetcher/" owner "/" repo)) + (outbuf (format "*%s*" name)) + (errbuf (get-buffer-create "*github-fetcher/errors*")) + (cleanup (lambda () + (kill-buffer outbuf) + (kill-buffer errbuf) + (with-current-buffer buffer + (read-only-mode -1)))) + (prefetch-handler + (lambda (_process event) + (unwind-protect + (pcase event + ("finished\n" + (let* ((json-string (with-current-buffer outbuf + (buffer-string))) + (result (json-read-from-string json-string))) + (with-current-buffer buffer + (goto-char point) + (map-let (("rev" rev) ("sha256" sha256)) result + (read-only-mode -1) + (insert (format "fetchFromGitHub { + owner = \"%s\"; + repo = \"%s\"; + rev = \"%s\"; + sha256 = \"%s\"; +};" owner repo rev sha256)) + (indent-region point (point)))))) + (_ (with-current-buffer errbuf + (error "Failed to prefetch %s/%s: %s" + owner repo (buffer-string))))) + (funcall cleanup))))) + + ;; Fetching happens asynchronously, but we'd like to make sure the + ;; point stays in place while that happens. + (read-only-mode) + (make-process :name name + :buffer outbuf + :command `("nix-prefetch-github" ,owner ,repo) + :stderr errbuf + :sentinel prefetch-handler))) + +(defun nix/sly-from-depot (attribute) + "Start a Sly REPL configured with a Lisp matching a derivation + from my depot. + + The derivation invokes nix.buildLisp.sbclWith and is built + asynchronously. The build output is included in the error + thrown on build failures." + + (interactive "sAttribute: ") + (lexical-let* ((outbuf (get-buffer-create (format "*depot-out/%s*" attribute))) + (errbuf (get-buffer-create (format "*depot-errors/%s*" attribute))) + (expression (format "let depot = import <depot> {}; in depot.nix.buildLisp.sbclWith [ depot.%s ]" attribute)) + ;; TODO(tazjin): use <depot> + (command (list "nix-build" "--no-out-link" "-I" (format "depot=%s" nix-depot-path) "-E" expression))) + + (message "Acquiring Lisp for <depot>.%s" attribute) + (make-process :name (format "depot-nix-build/%s" attribute) + :buffer outbuf + :stderr errbuf + :command command + :sentinel + (lambda (process event) + (unwind-protect + (pcase event + ("finished\n" + (let* ((outpath (s-trim (with-current-buffer outbuf (buffer-string)))) + (lisp-path (s-concat outpath "/bin/sbcl"))) + (message "Acquired Lisp for <depot>.%s at %s" attribute lisp-path) + (sly lisp-path))) + (_ (with-current-buffer errbuf + (error "Failed to build '%s':\n%s" attribute (buffer-string))))) + (kill-buffer outbuf) + (kill-buffer errbuf)))))) + +(provide 'nix-util) diff --git a/tools/emacs-pkgs/notable/OWNERS b/tools/emacs-pkgs/notable/OWNERS new file mode 100644 index 000000000000..f7da62ecf709 --- /dev/null +++ b/tools/emacs-pkgs/notable/OWNERS @@ -0,0 +1,2 @@ +owners: + - tazjin diff --git a/tools/emacs-pkgs/notable/default.nix b/tools/emacs-pkgs/notable/default.nix new file mode 100644 index 000000000000..8c6935fe886b --- /dev/null +++ b/tools/emacs-pkgs/notable/default.nix @@ -0,0 +1,15 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage rec { + pname = "notable"; + version = "1.0"; + src = ./notable.el; + + externalRequires = epkgs: with epkgs; [ + f ht s + ]; + + internalRequires = [ + depot.tools.emacs-pkgs.dottime + ]; +} diff --git a/tools/emacs-pkgs/notable/notable.el b/tools/emacs-pkgs/notable/notable.el new file mode 100644 index 000000000000..4668dd333c99 --- /dev/null +++ b/tools/emacs-pkgs/notable/notable.el @@ -0,0 +1,251 @@ +;;; notable.el --- a simple note-taking app -*- lexical-binding: t; -*- +;; +;; Copyright (C) 2020 The TVL Contributors +;; +;; Author: Vincent Ambo <mail@tazj.in> +;; Version: 1.0 +;; Package-Requires: (cl-lib dash f rx s subr-x) +;; +;;; Commentary: +;; +;; This package provides a simple note-taking application which can be +;; invoked from anywhere in Emacs, with several interactive +;; note-taking functions included. +;; +;; As is tradition for my software, the idea here is to reduce +;; friction which I see even with tools like `org-capture', because +;; `org-mode' does a ton of things I don't care about. +;; +;; Notable stores its notes in simple JSON files in the folder +;; specified by `notable-note-dir'. + +(require 'cl-lib) +(require 'dottime) +(require 'f) +(require 'ht) +(require 'rx) +(require 's) +(require 'subr-x) + +;; User-facing customisation options + +(defgroup notable nil + "Simple note-taking application." + :group 'applications) + +;; TODO(tazjin): Use whatever the XDG state dir thing is for these by +;; default. +(defcustom notable-note-dir (expand-file-name "~/.notable/") + "File path to the directory containing notable's notes." + :type 'string + :group 'notable) + +;; Package internal definitions + +(cl-defstruct (notable--note (:constructor notable--make-note)) + "Structure containing the fields of a single notable note." + time ;; UNIX timestamp at which the note was taken + content ;; Textual content of the note + ) + +(defvar notable--note-lock (make-mutex "notable-notes") + "Exclusive lock for note operations with shared state.") + +(defvar notable--note-regexp + (rx "note-" + (group (one-or-more (any num))) + ".json") + "Regular expression to match note file names.") + +(defvar notable--next-note + (let ((next 0)) + (dolist (file (f-entries notable-note-dir)) + (when-let* ((match (string-match notable--note-regexp file)) + (id (string-to-number + (match-string 1 file))) + (larger (> id next))) + (setq next id))) + (+ 1 next)) + "Next ID to use for notes. Initial value is determined based on + the existing notes files.") + +(defun notable--serialize-note (note) + "Serialise NOTE into JSON format." + (check-type note notable--note) + (json-serialize (ht ("time" (notable--note-time note)) + ("content" (notable--note-content note))))) + +(defun notable--deserialize-note (json) + "Deserialise JSON into a notable note." + (check-type json string) + (let ((parsed (json-parse-string json))) + (unless (and (ht-contains? parsed "time") + (ht-contains-p parsed "content")) + (error "Missing required keys in note structure!")) + (notable--make-note :time (ht-get parsed "time") + :content (ht-get parsed "content")))) + +(defun notable--next-id () + "Return the next note ID and increment the counter." + (with-mutex notable--note-lock + (let ((id notable--next-note)) + (setq notable--next-note (+ 1 id)) + id))) + +(defun notable--note-path (id) + (check-type id integer) + (f-join notable-note-dir (format "note-%d.json" id))) + +(defun notable--archive-path (id) + (check-type id integer) + (f-join notable-note-dir (format "archive-%d.json" id))) + +(defun notable--add-note (content) + "Add a note with CONTENT to the note store." + (let* ((id (notable--next-id)) + (note (notable--make-note :time (time-convert nil 'integer) + :content content)) + (path (notable--note-path id))) + (when (f-exists? path) (error "Note file '%s' already exists!" path)) + (f-write-text (notable--serialize-note note) 'utf-8 path) + (message "Saved note %d" id))) + +(defun notable--archive-note (id) + "Archive the note with ID." + (check-type id integer) + + (unless (f-exists? (notable--note-path id)) + (error "There is no note with ID %d." id)) + + (when (f-exists? (notable--archive-path id)) + (error "Oh no, a note with ID %d has already been archived!" id)) + + (f-move (notable--note-path id) (notable--archive-path id)) + (message "Archived note with ID %d." id)) + +(defun notable--list-note-ids () + "List all note IDs (not contents) from `notable-note-dir'" + (cl-loop for file in (f-entries notable-note-dir) + with res = nil + if (string-match notable--note-regexp file) + do (push (string-to-number (match-string 1 file)) res) + finally return res)) + +(defun notable--get-note (id) + (let ((path (notable--note-path id))) + (unless (f-exists? path) + (error "No note with ID %s in note storage!" id)) + (notable--deserialize-note (f-read-text path 'utf-8)))) + +;; Note view buffer implementation + +(defvar-local notable--buffer-note nil "The note ID displayed by this buffer.") + +(define-derived-mode notable-note-mode fundamental-mode "notable-note" + "Major mode displaying a single Notable note." + (set (make-local-variable 'scroll-preserve-screen-position) t) + (setq truncate-lines t) + (setq buffer-read-only t) + (setq buffer-undo-list t)) + +(setq notable-note-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "q" 'kill-current-buffer) + map)) + +(defun notable--show-note (id) + "Display a single note in a separate buffer." + (check-type id integer) + + (let ((note (notable--get-note id)) + (buffer (get-buffer-create (format "*notable: %d*" id))) + (inhibit-read-only t)) + (with-current-buffer buffer + (notable-note-mode) + (erase-buffer) + (setq notable--buffer-note id) + (setq header-line-format + (format "Note from %s" + (dottime-format + (seconds-to-time (notable--note-time note)))))) + (switch-to-buffer buffer) + (goto-char (point-min)) + (insert (notable--note-content note)))) + +(defun notable--show-note-at-point () + (interactive) + (notable--show-note (get-text-property (point) 'notable-note-id))) + +(defun notable--archive-note-at-point () + (interactive) + (notable--archive-note (get-text-property (point) 'notable-note-id))) + +;; Note list buffer implementation + +(define-derived-mode notable-list-mode fundamental-mode "notable" + "Major mode displaying the Notable note list." + ;; TODO(tazjin): `imenu' functions? + + (set (make-local-variable 'scroll-preserve-screen-position) t) + (setq truncate-lines t) + (setq buffer-read-only t) + (setq buffer-undo-list t) + (hl-line-mode t)) + +(setq notable-list-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "a" 'notable--archive-note-at-point) + (define-key map "q" 'kill-current-buffer) + (define-key map "g" 'notable-list-notes) + (define-key map (kbd "RET") 'notable--show-note-at-point) + map)) + +(defun notable--render-note (id note) + (check-type id integer) + (check-type note notable--note) + + (let* ((start (point)) + (date (dottime-format (seconds-to-time + (notable--note-time note)))) + (first-line (truncate-string-to-width + (car (s-lines (notable--note-content note))) + ;; Length of the window, minus the date prefix: + (- (window-width) (+ 2 (length date))) + nil nil 1))) + (insert (propertize (s-concat date " " first-line) + 'notable-note-id id)) + (insert "\n"))) + +(defun notable--render-notes (notes) + "Retrieve each note in NOTES by ID and insert its contents into +the list buffer. + +For larger notes only the first line is displayed." + (dolist (id notes) + (notable--render-note id (notable--get-note id)))) + +;; User-facing functions + +(defun notable-take-note (content) + "Interactively prompt the user for a note that should be stored +in Notable." + (interactive "sEnter note: ") + (check-type content string) + (notable--add-note content)) + +(defun notable-list-notes () + "Open a buffer listing all active notes." + (interactive) + + (let ((buffer (get-buffer-create "*notable*")) + (notes (notable--list-note-ids)) + (inhibit-read-only t)) + (with-current-buffer buffer + (notable-list-mode) + (erase-buffer) + (setq header-line-format "Notable notes")) + (switch-to-buffer buffer) + (goto-char (point-min)) + (notable--render-notes notes))) + +(provide 'notable) diff --git a/tools/emacs-pkgs/passively/OWNERS b/tools/emacs-pkgs/passively/OWNERS new file mode 100644 index 000000000000..56853aed59e7 --- /dev/null +++ b/tools/emacs-pkgs/passively/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - tazjin diff --git a/tools/emacs-pkgs/passively/README.md b/tools/emacs-pkgs/passively/README.md new file mode 100644 index 000000000000..052c496b324d --- /dev/null +++ b/tools/emacs-pkgs/passively/README.md @@ -0,0 +1,76 @@ +<!-- SPDX-License-Identifier: MIT --> +passively +========= + +Passively is an Emacs Lisp library for passively learning new +information in an Emacs instance. + +Passively works by displaying a random piece of information to be +learned in the Emacs echoline whenever Emacs is idle for a set amount +of time. + +It was designed to aid in language acquisition by passively displaying +new vocabulary to learn. + +Passively is configured with a corpus of information (a hash table +mapping string keys to string values) and maintains a set of terms +that the user already learned in a file on disk. + +## Configuration & usage + +Configure passively like this: + +```lisp +;; Configure the terms to learn. Each term should have a key and a +;; string value which is displayed. +(setq passively-learn-terms + (ht ("забыть" "забыть - to forget") + ("действительно" "действительно - indeed, really"))) + +;; Configure a file in which passively should store its state +;; (defaults to $user-emacs-directory/passively.el) +(setq passively-store-state "/persist/tazjin/passively.el") + +;; Configure after how many seconds of idle time passively should +;; display a new piece of information. +;; (defaults to 4 seconds) +(setq passively-show-after-idle-for 5) + +;; Once this configuration has been set up, start passively: +(passively-enable) + +;; Or, if it annoys you, disable it again: +(passively-disable) +``` + +These variables are registered with `customize` and may be customised +through its interface. + +### Known terms + +Passively exposes the interactive function +`passively-mark-last-as-known` which marks the previously displayed +term as known. This means that it will not be included in the random +selection anymore. + +### Last term + +Passively stores the key of the last known term in +`passively-last-displayed`. + +## Installation + +Inside of the TVL depot, you can install passively from +`pkgs.emacsPackages.tvlPackages.passively`. Outside of the depot, you +can clone passively like this: + + git clone https://code.tvl.fyi/depot.git:/tools/emacs-pkgs/passively.git + +Passively depends on `ht.el`. + +Feel free to contribute patches by emailing them to `depot@tazj.in` + +## Use-cases + +I'm using passively to learn Russian vocabulary. Once I've cleaned up +my configuration for that, my Russian term list will be linked here. diff --git a/tools/emacs-pkgs/passively/default.nix b/tools/emacs-pkgs/passively/default.nix new file mode 100644 index 000000000000..ec59cc85fd8f --- /dev/null +++ b/tools/emacs-pkgs/passively/default.nix @@ -0,0 +1,8 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage { + pname = "passively"; + version = "1.0"; + src = ./passively.el; + externalRequires = (epkgs: with epkgs; [ ht ]); +} diff --git a/tools/emacs-pkgs/passively/passively.el b/tools/emacs-pkgs/passively/passively.el new file mode 100644 index 000000000000..75586e4e3fe5 --- /dev/null +++ b/tools/emacs-pkgs/passively/passively.el @@ -0,0 +1,120 @@ +;;; passively.el --- Passively learn new information -*- lexical-binding: t; -*- +;; +;; SPDX-License-Identifier: MIT +;; Copyright (C) 2020 The TVL Contributors +;; +;; Author: Vincent Ambo <tazjin@tvl.su> +;; Version: 1.0 +;; Package-Requires: (ht seq) +;; URL: https://code.tvl.fyi/about/tools/emacs-pkgs/passively/ +;; +;; This file is not part of GNU Emacs. + +(require 'ht) +(require 'seq) + +;; Customisation options + +(defgroup passively nil + "Customisation options for passively" + :group 'applications) + +(defcustom passively-learn-terms nil + "Terms that passively should randomly display to the user. The +format of this variable is a hash table with a string key that +uniquely identifies the term, and a string value that is +displayed to the user. + +For example, a possible value could be: + + (ht (\"забыть\" \"забыть - to forget\") + (\"действительно\" \"действительно - indeed, really\"))) +" + ;; TODO(tazjin): No hash-table type in customization.el? + :type '(sexp) + :group 'passively) + +(defcustom passively-store-state (format "%spassively.el" user-emacs-directory) + "File in which passively should store its state (e.g. known terms)" + :type '(file) + :group 'passively) + +(defcustom passively-show-after-idle-for 4 + "Number of seconds after Emacs goes idle that passively should +wait before displaying a term." + :type '(integer) + :group 'passively) + +;; Implementation of state persistence +(defvar passively-last-displayed nil + "Key of the last displayed passively term.") + +(defvar passively--known-terms (make-hash-table) + "Set of terms that are already known.") + +(defun passively--persist-known-terms () + "Persist the set of known passively terms to disk." + (with-temp-file passively-store-state + (insert (prin1-to-string (ht-keys passively--known-terms))))) + +(defun passively--load-known-terms () + "Load the set of known passively terms from disk." + (with-temp-buffer + (insert-file-contents passively-store-state) + (let ((keys (read (current-buffer)))) + (setq passively--known-terms (make-hash-table)) + (seq-do + (lambda (key) (ht-set passively--known-terms key t)) + keys))) + (message "passively: loaded %d known words" + (seq-length (ht-keys passively--known-terms)))) + +(defun passively-mark-last-as-known () + "Mark the last term that passively displayed as known. It will +not be displayed again." + (interactive) + + (ht-set passively--known-terms passively-last-displayed t) + (passively--persist-known-terms) + (message "passively: Marked '%s' as known" passively-last-displayed)) + +;; Implementation of main display logic +(defvar passively--display-timer nil + "idle-timer used for displaying terms by passively") + +(defun passively--random-term (timeout) + ;; This is stupid, calculate set intersections instead. + (if (< 1000 timeout) + (error "It seems you already know all the terms?") + (seq-random-elt (ht-keys passively-learn-terms)))) + +(defun passively--display-random-term () + (let* ((timeout 1) + (term (passively--random-term timeout))) + (while (ht-contains? passively--known-terms term) + (setq timeout (+ 1 timeout)) + (setq term (passively--random-term timeout))) + (setq passively-last-displayed term) + (message (ht-get passively-learn-terms term)))) + +(defun passively-enable () + "Enable automatic display of terms via passively." + (interactive) + (if passively--display-timer + (error "passively: Already running!") + (setq passively--display-timer + (run-with-idle-timer passively-show-after-idle-for t + #'passively--display-random-term)) + (message "passively: Now running after %s seconds of idle time" + passively-show-after-idle-for))) + +(defun passively-disable () + "Turn off automatic display of terms via passively." + (interactive) + (unless passively--display-timer + (error "passively: Not running!")) + (cancel-timer passively--display-timer) + (setq passively--display-timer nil) + (message "passively: Now disabled")) + +(provide 'passively) diff --git a/tools/emacs-pkgs/term-switcher/default.nix b/tools/emacs-pkgs/term-switcher/default.nix new file mode 100644 index 000000000000..e775de5cdbe8 --- /dev/null +++ b/tools/emacs-pkgs/term-switcher/default.nix @@ -0,0 +1,8 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage { + pname = "term-switcher"; + version = "1.0"; + src = ./term-switcher.el; + externalRequires = epkgs: with epkgs; [ dash ivy s vterm ]; +} diff --git a/tools/emacs-pkgs/term-switcher/term-switcher.el b/tools/emacs-pkgs/term-switcher/term-switcher.el new file mode 100644 index 000000000000..0055f87fd67f --- /dev/null +++ b/tools/emacs-pkgs/term-switcher/term-switcher.el @@ -0,0 +1,57 @@ +;;; term-switcher.el --- Easily switch between open vterms +;; +;; Copyright (C) 2019 Google Inc. +;; +;; Author: Vincent Ambo <tazjin@google.com> +;; Version: 1.1 +;; Package-Requires: (dash ivy s vterm) +;; +;;; Commentary: +;; +;; This package adds a function that lets users quickly switch between +;; different open vterms via ivy. + +(require 'dash) +(require 'ivy) +(require 's) +(require 'vterm) + +(defgroup term-switcher nil + "Customization options `term-switcher'.") + +(defcustom term-switcher-buffer-prefix "vterm<" + "String prefix for vterm terminal buffers. For example, if you + set your titles to match `vterm<...>' a useful prefix might be + `vterm<'." + :type '(string) + :group 'term-switcher) + +(defun ts/open-or-create-vterm (buffer-name) + "Switch to the buffer with BUFFER-NAME or create a new vterm + buffer." + (if (equal "New vterm" buffer-name) + (vterm) + (if-let ((buffer (get-buffer buffer-name))) + (switch-to-buffer buffer) + (error "Could not find vterm buffer: %s" buffer-name)))) + +(defun ts/is-vterm-buffer (buffer) + "Determine whether BUFFER runs a vterm." + (equal 'vterm-mode (buffer-local-value 'major-mode buffer))) + +(defun ts/switch-to-terminal () + "Switch to an existing vterm buffer or create a new one." + + (interactive) + (let ((terms (-map #'buffer-name + (-filter #'ts/is-vterm-buffer (buffer-list))))) + (if terms + (ivy-read "Switch to vterm: " + (cons "New vterm" terms) + :caller 'ts/switch-to-terminal + :preselect (s-concat "^" term-switcher-buffer-prefix) + :require-match t + :action #'ts/open-or-create-vterm) + (vterm)))) + +(provide 'term-switcher) diff --git a/tools/emacs-pkgs/tvl/OWNERS b/tools/emacs-pkgs/tvl/OWNERS new file mode 100644 index 000000000000..ce7e0e37ee4f --- /dev/null +++ b/tools/emacs-pkgs/tvl/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - grfn diff --git a/tools/emacs-pkgs/tvl/default.nix b/tools/emacs-pkgs/tvl/default.nix new file mode 100644 index 000000000000..5dcc184bb521 --- /dev/null +++ b/tools/emacs-pkgs/tvl/default.nix @@ -0,0 +1,8 @@ +{ depot, ... }: + +depot.tools.emacs-pkgs.buildEmacsPackage { + pname = "tvl"; + version = "1.0"; + src = ./tvl.el; + externalRequires = (epkgs: with epkgs; [ magit s ]); +} diff --git a/tools/emacs-pkgs/tvl/tvl.el b/tools/emacs-pkgs/tvl/tvl.el new file mode 100644 index 000000000000..3938628edc8e --- /dev/null +++ b/tools/emacs-pkgs/tvl/tvl.el @@ -0,0 +1,155 @@ +;;; tvl.el --- description -*- lexical-binding: t; -*- +;; +;; Copyright (C) 2020 Griffin Smith +;; Copyright (C) 2020 The TVL Contributors +;; +;; Author: Griffin Smith <grfn@gws.fyi> +;; Version: 0.0.1 +;; Package-Requires: (s dash magit) +;; +;; This file is not part of GNU Emacs. +;; +;;; Commentary: +;; +;; This file provides shared utilities for interacting with the TVL monorepo +;; +;;; Code: + +(require 'magit) +(require 's) + +(defgroup tvl nil + "Customisation options for TVL functionality.") + +(defcustom tvl-gerrit-remote "origin" + "Name of the git remote for gerrit" + :type '(string) + :group 'tvl) + +(defcustom tvl-depot-path "/depot" + "Location at which the TVL depot is checked out." + :type '(string) + :group 'tvl) + +(defcustom tvl-target-branch "canon" + "Branch to use to target CLs" + :group 'tvl + :type '(string) + :safe (lambda (_) t)) + +(defun tvl--gerrit-ref (target-branch &optional flags) + (let ((flag-suffix (if flags (format "%%l=%s" (s-join "," flags)) + ""))) + (format "HEAD:refs/for/%s%s" target-branch flag-suffix))) + +(transient-define-suffix magit-gerrit-push-for-review () + "Push to Gerrit for review." + (interactive) + (magit-push-refspecs tvl-gerrit-remote + (tvl--gerrit-ref tvl-target-branch) + nil)) + +(transient-append-suffix + #'magit-push ["r"] + (list "R" "push to Gerrit for review" #'magit-gerrit-push-for-review)) + +(transient-define-suffix magit-gerrit-push-wip () + "Push to Gerrit as a work-in-progress." + (interactive) + (magit-push-refspecs tvl-gerrit-remote + (concat (tvl--gerrit-ref tvl-target-branch) "%wip") + nil)) + +(transient-append-suffix + #'magit-push ["r"] + (list "W" "push to Gerrit as a work-in-progress" #'magit-gerrit-push-wip)) + +(transient-define-suffix magit-gerrit-submit () + "Push to Gerrit for review." + (interactive) + (magit-push-refspecs tvl-gerrit-remote + (tvl--gerrit-ref tvl-target-branch '("submit")) + nil)) + +(transient-append-suffix + #'magit-push ["r"] + (list "S" "push to Gerrit to submit" #'magit-gerrit-submit)) + + +(transient-define-suffix magit-gerrit-rubberstamp () + "Push, automatically approve and submit to Gerrit. This +rubberstamp operation is dangerous and should only be used in +`//users'." + (interactive) + (magit-push-refspecs tvl-gerrit-remote + (tvl--gerrit-ref tvl-target-branch + '("Code-Review+2" "publish-comments")) + nil)) + +(transient-append-suffix + #'magit-push ["r"] + (list "P" "push & rubberstamp to Gerrit" #'magit-gerrit-rubberstamp)) + +(defvar magit-cl-history nil) +(defun magit-read-cl (remote) + (let* ((refs (prog2 (message "Determining available refs...") + (magit-remote-list-refs remote) + (message "Determining available refs...done"))) + (change-refs (-filter + (apply-partially #'string-prefix-p "refs/changes/") + refs)) + (cl-number-to-refs + (-group-by + (lambda (change-ref) + ;; refs/changes/34/1234/1 + ;; ^ ^ ^ ^ ^ + ;; 1 2 3 4 5 + ;; ^-- this one + (cadddr + (split-string change-ref (rx "/")))) + change-refs)) + (cl-numbers + (-map + (lambda (cl-to-refs) + (let ((latest-patchset-ref + (-max-by + (-on #'> (lambda (ref) + (string-to-number + (fifth (split-string ref (rx "/")))))) + (-remove + (apply-partially #'s-ends-with-p "meta") + (cdr cl-to-refs))))) + (propertize (car cl-to-refs) 'ref latest-patchset-ref))) + cl-number-to-refs))) + (get-text-property + 0 + 'ref + (magit-completing-read + "Checkout CL" cl-numbers nil t nil 'magit-cl-history)))) + +(transient-define-suffix magit-gerrit-checkout (remote cl-refspec) + "Prompt for a CL number and checkout the latest patchset of that CL with + detached HEAD" + (interactive + (let* ((remote tvl-gerrit-remote) + (cl (magit-read-cl remote))) + (list remote cl))) + (magit-fetch-refspec remote cl-refspec (magit-fetch-arguments)) + ;; That runs async, so wait for it to finish (this is how magit does it) + (while (and magit-this-process + (eq (process-status magit-this-process) 'run)) + (sleep-for 0.005)) + (magit-checkout "FETCH_HEAD" (magit-branch-arguments)) + (message "HEAD detached at %s" cl-refspec)) + +(transient-append-suffix + #'magit-branch ["l"] + (list "g" "gerrit CL" #'magit-gerrit-checkout)) + +(defun tvl-depot-status () + "Open the TVL monorepo in magit." + (interactive) + (magit-status-setup-buffer tvl-depot-path)) + +(provide 'tvl) +;;; tvl.el ends here diff --git a/tools/eprintf.nix b/tools/eprintf.nix new file mode 100644 index 000000000000..eeacca4c8c72 --- /dev/null +++ b/tools/eprintf.nix @@ -0,0 +1,9 @@ +{ depot, pkgs, ... }: + +let + bins = depot.nix.getBins pkgs.coreutils [ "printf" ]; + +# printf(1), but redirect to stderr +in depot.nix.writeExecline "eprintf" {} [ + "fdmove" "-c" "1" "2" bins.printf "$@" +] diff --git a/tools/gerrit-cli.nix b/tools/gerrit-cli.nix new file mode 100644 index 000000000000..1606155a8068 --- /dev/null +++ b/tools/gerrit-cli.nix @@ -0,0 +1,13 @@ +# Utility script to run a gerrit command on the depot host via ssh. +# Reads the username from TVL_USERNAME, or defaults to $(whoami) +{ pkgs, ... }: + +pkgs.writeShellScriptBin "gerrit" '' + TVL_USERNAME=''${TVL_USERNAME:-$(whoami)} + if which ssh &>/dev/null; then + ssh=ssh + else + ssh="${pkgs.openssh}/bin/ssh" + fi + exec $ssh $TVL_USERNAME@code.tvl.fyi -p 29418 -- gerrit $@ +'' diff --git a/tools/gerrit-update.nix b/tools/gerrit-update.nix new file mode 100644 index 000000000000..e4efd89ea597 --- /dev/null +++ b/tools/gerrit-update.nix @@ -0,0 +1,34 @@ +# Utility script to perform a Gerrit update. +{ pkgs, ... }: + +pkgs.writeShellScriptBin "gerrit-update" '' + set -euo pipefail + + if [[ $EUID -ne 0 ]]; then + echo "Oh no! Only root is allowed to update Gerrit!" >&2 + exit 1 + fi + + gerrit_war="$(find "${pkgs.gerrit}/webapps" -name 'gerrit*.war')" + java="${pkgs.jdk}/bin/java" + backup_path="/root/gerrit_preupgrade-$(date +"%Y-%m-%d").tar.bz2" + + # Take a safety backup of Gerrit into /root's homedir. Just in case. + echo "Backing up Gerrit to $backup_path" + tar -cjf "$backup_path" /var/lib/gerrit + + # Stop Gerrit (and its activation socket). + echo "Stopping Gerrit" + systemctl stop gerrit.service gerrit.socket + + # Ask Gerrit to do a schema upgrade... + echo "Performing schema upgrade" + "$java" -jar "$gerrit_war" \ + init --no-auto-start --batch --skip-plugins --site-path "/var/lib/gerrit" + + # Restart Gerrit. + echo "Restarting Gerrit" + systemctl start gerrit.socket gerrit.service + + echo "...done" +'' diff --git a/tools/hash-password.nix b/tools/hash-password.nix new file mode 100644 index 000000000000..9893d521787e --- /dev/null +++ b/tools/hash-password.nix @@ -0,0 +1,7 @@ +# Utility for invoking slappasswd with the correct options for +# creating an ARGON2 password hash. +{ pkgs, ... }: + +pkgs.writeShellScriptBin "hash-password" '' + ${pkgs.openldap}/bin/slappasswd -o module-load=pw-argon2 -h '{ARGON2}' +'' diff --git a/tools/nsfv-setup/default.nix b/tools/nsfv-setup/default.nix new file mode 100644 index 000000000000..98dcc61b7bc1 --- /dev/null +++ b/tools/nsfv-setup/default.nix @@ -0,0 +1,28 @@ +# Configures a running Pulseaudio instance with an LADSP filter that +# creates a noise-cancelling sink. +# +# This can be used to, for example, cancel noise from an incoming +# video conferencing audio stream. +# +# There are some caveats, for example this will not distinguish +# between noise from different participants and I have no idea what +# happens if the default sink goes away. +# +# If this script is run while an NSFV sink exists, the existing sink +# will first be removed. +{ depot, pkgs, ... }: + +let + inherit (pkgs) ripgrep pulseaudio; + inherit (depot.third_party) nsfv; +in pkgs.writeShellScriptBin "nsfv-setup" '' + export PATH="${ripgrep}/bin:${pulseaudio}/bin:$PATH" + + if pacmd list-sinks | rg librnnoise_ladspa.so >/dev/null; then + pactl unload-module module-ladspa-sink + fi + + SINK=$(${pulseaudio}/bin/pacmd info | ${ripgrep}/bin/rg -r '$1' '^Default sink name: (.*)$') + echo "Setting up NSFV filtering to sink ''${SINK}" + ${pulseaudio}/bin/pacmd load-module module-ladspa-sink sink_name=NSFV sink_master=''${SINK} label=noise_suppressor_mono plugin=${nsfv}/lib/ladspa/librnnoise_ladspa.so control=42 rate=48000 +'' diff --git a/tools/perf-flamegraph.nix b/tools/perf-flamegraph.nix new file mode 100644 index 000000000000..b472b746ff14 --- /dev/null +++ b/tools/perf-flamegraph.nix @@ -0,0 +1,12 @@ +# Script that collects perf timing for the execution of a command and writes a +# flamegraph to stdout +{ pkgs, ... }: + +pkgs.writeShellScriptBin "perf-flamegraph" '' + set -euo pipefail + + ${pkgs.linuxPackages.perf}/bin/perf record -g --call-graph dwarf -F max "$@" + ${pkgs.linuxPackages.perf}/bin/perf script \ + | ${pkgs.flamegraph}/bin/stackcollapse-perf.pl \ + | ${pkgs.flamegraph}/bin/flamegraph.pl +'' diff --git a/tools/rust-crates-advisory/OWNERS b/tools/rust-crates-advisory/OWNERS new file mode 100644 index 000000000000..a742d0d22bf6 --- /dev/null +++ b/tools/rust-crates-advisory/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - Profpatsch diff --git a/tools/rust-crates-advisory/check-security-advisory.rs b/tools/rust-crates-advisory/check-security-advisory.rs new file mode 100644 index 000000000000..3fd9bc2dd947 --- /dev/null +++ b/tools/rust-crates-advisory/check-security-advisory.rs @@ -0,0 +1,67 @@ +extern crate semver; +extern crate toml; + +use std::io::Write; + +/// reads a security advisory of the form +/// https://github.com/RustSec/advisory-db/blob/a24932e220dfa9be8b0b501210fef8a0bc7ef43e/EXAMPLE_ADVISORY.md +/// and a crate version number, +/// and returns 0 if the crate version is patched +/// and returns 1 if the crate version is *not* patched +/// +/// If PRINT_ADVISORY is set, the advisory is printed if it matches. + +fn main() { + let mut args = std::env::args_os(); + let file = args.nth(1).expect("security advisory md file is $1"); + let crate_version = + args.nth(0).expect("crate version is $2") + .into_string().expect("crate version string not utf8") + ; + let crate_version = semver::Version::parse(&crate_version).expect(&format!("this is not a semver version: {}", &crate_version)); + let filename = file.to_string_lossy(); + + let content = std::fs::read(&file).expect(&format!("could not read {}", filename)); + let content = + std::str::from_utf8(&content).expect(&format!("file {} was not encoded as utf-8", filename)); + let content = content.trim_start(); + + let toml_start = content + .strip_prefix("```toml").expect(&format!("file did not start with ```toml: {}", filename)); + let toml_end_index = toml_start.find("```").expect(&format!("the toml section did not end, no `` found: {}", filename)); + let toml = &toml_start[..toml_end_index]; + let toml : toml::Value = toml::de::from_slice(toml.as_bytes()).expect(&format!("could not parse toml: {}", filename)); + + let versions = toml + .as_table().expect(&format!("the toml is not a table: {}", filename)) + .get("versions").expect(&format!("the toml does not contain the versions field: {}", filename)) + .as_table().expect(&format!("the toml versions field must be a table: {}", filename)); + + let unaffected = match versions.get("unaffected") { + Some(u) => u + .as_array().expect(&format!("the toml versions.unaffected field must be a list of semvers: {}", filename)) + .iter() + .map(|v| semver::VersionReq::parse(v.as_str().expect(&format!("the version field {} is not a string", v))).expect(&format!("the version field {} is not a valid semver VersionReq", v))) + .collect(), + None => vec![] + }; + + let mut patched : Vec<semver::VersionReq> = versions.get("patched").expect(&format!("the toml versions.patched field must exist: {}", filename)) + .as_array().expect(&format!("the toml versions.patched field must be a list of semvers: {}", filename)) + .iter() + .map(|v| semver::VersionReq::parse(v.as_str().expect(&format!("the version field {} is not a string", v))).expect(&format!("the version field {} is not a valid semver VersionReq", v))) + .collect(); + + patched.extend_from_slice(&unaffected[..]); + let is_patched_or_unaffected = patched.iter().any(|req| req.matches(&crate_version)); + + if is_patched_or_unaffected { + std::process::exit(0); + } else { + if std::env::var_os("PRINT_ADVISORY").is_some() { + write!(std::io::stderr(), "Advisory {} matched!\n{}\n", filename, content).unwrap(); + } + std::process::exit(1); + } + +} diff --git a/tools/rust-crates-advisory/default.nix b/tools/rust-crates-advisory/default.nix new file mode 100644 index 000000000000..c0cd4dc03e05 --- /dev/null +++ b/tools/rust-crates-advisory/default.nix @@ -0,0 +1,90 @@ +{ depot, pkgs, lib, ... }: + +let + + bins = + depot.nix.getBins pkgs.s6-portable-utils [ "s6-ln" "s6-cat" "s6-echo" "s6-mkdir" "s6-test" "s6-touch" ] + // depot.nix.getBins pkgs.lr [ "lr" ] + ; + + crate-advisories = "${depot.third_party.rustsec-advisory-db}/crates"; + + our-crates = lib.filter (v: v ? outPath) + (builtins.attrValues depot.third_party.rust-crates); + + check-security-advisory = depot.nix.writers.rustSimple { + name = "parse-security-advisory"; + dependencies = [ + depot.third_party.rust-crates.toml + depot.third_party.rust-crates.semver + ]; + } (builtins.readFile ./check-security-advisory.rs); + + # $1 is the directory with advisories for crate $2 with version $3 + check-crate-advisory = depot.nix.writeExecline "check-crate-advisory" { readNArgs = 3; } [ + "pipeline" [ bins.lr "-0" "-t" "depth == 1" "$1" ] + "forstdin" "-0" "-Eo" "0" "advisory" + "if" [ depot.tools.eprintf "advisory %s\n" "$advisory" ] + check-security-advisory "$advisory" "$3" + ]; + + # Run through everything in the `crate-advisories` repository + # and check whether we can parse all the advisories without crashing. + test-parsing-all-security-advisories = depot.nix.runExecline "check-all-our-crates" {} [ + "pipeline" [ bins.lr "-0" "-t" "depth == 1" crate-advisories ] + "if" [ + # this will succeed as long as check-crate-advisory doesn’t `panic!()` (status 101) + "forstdin" "-0" "-E" "-x" "101" "crate_advisories" + check-crate-advisory "$crate_advisories" "foo" "0.0.0" + ] + "importas" "out" "out" + bins.s6-touch "$out" + ]; + + + check-all-our-crates = depot.nix.runExecline "check-all-our-crates" { + stdin = lib.concatStrings + (map + (crate: + depot.nix.netstring.fromString + ( depot.nix.netstring.fromString crate.crateName + + depot.nix.netstring.fromString crate.version )) + our-crates); + } [ + "if" [ + "forstdin" "-o" "0" "-Ed" "" "crateNetstring" + "multidefine" "-d" "" "$crateNetstring" [ "crate" "crate_version" ] + "if" [ depot.tools.eprintf "checking %s, version %s\n" "$crate" "$crate_version" ] + + "ifthenelse" [ bins.s6-test "-d" "${crate-advisories}/\${crate}" ] + [ # also print the full advisory text if it matches + "export" "PRINT_ADVISORY" "1" + check-crate-advisory "${crate-advisories}/\${crate}" "$crate" "$crate_version" + ] + [ depot.tools.eprintf "No advisories found for crate %s\n" "$crate" ] + "importas" "-ui" "ret" "?" + # put a marker in ./failed to read at the end + "ifelse" [ bins.s6-test "$ret" "-eq" "1" ] + [ bins.s6-touch "./failed" ] + "if" [ depot.tools.eprintf "\n" ] + "exit" "$ret" + ] + "ifelse" [ bins.s6-test "-f" "./failed" ] + [ "if" [ depot.tools.eprintf "Error: Found active advisories!" ] + "exit" "1" + ] + "importas" "out" "out" + bins.s6-touch "$out" + ]; + +in depot.nix.readTree.drvTargets { + + check-all-our-crates = + depot.nix.drvSeqL + [ test-parsing-all-security-advisories ] + check-all-our-crates; + + inherit + check-crate-advisory + ; +} diff --git a/tools/tvlc/OWNERS b/tools/tvlc/OWNERS new file mode 100644 index 000000000000..9e7830ab215e --- /dev/null +++ b/tools/tvlc/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - riking diff --git a/tools/tvlc/common.sh b/tools/tvlc/common.sh new file mode 100644 index 000000000000..fe7605857fd3 --- /dev/null +++ b/tools/tvlc/common.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -eu +set -o pipefail + +source path-scripts + +XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}" +tvlc_root="$XDG_DATA_HOME/tvlc" + +nice_checkout_root= +if [ -f "$tvlc_root"/nice_checkout_root ]; then + nice_checkout_root="$(cat "$tvlc_root"/nice_checkout_root)" +fi +nice_checkout_root="${nice_checkout_root:-$HOME/tvlc}" + +depot_root= +if [ -f "$tvlc_root/depot_root" ]; then + depot_root="$(cat "$tvlc_root/depot_root")" +fi +if [ -d /depot ]; then + # don't require config on tvl nixos servers + depot_root="${depot_root:-/depot}" +fi +if [ -n "$depot_root" ]; then + export DEPOT_ROOT="$depot_root" +fi + +if [ ! -d "$tvlc_root" ]; then + echo "tvlc: setup required" + echo "please run 'tvlc setup' from the depot root" + exit 1 +fi diff --git a/tools/tvlc/default.nix b/tools/tvlc/default.nix new file mode 100644 index 000000000000..f40f30a44e33 --- /dev/null +++ b/tools/tvlc/default.nix @@ -0,0 +1,50 @@ +{ pkgs, depot, ... }: + +let + pathScripts = pkgs.writeShellScript "imports" '' + export tvix_instantiate="${depot.third_party.nix}/bin/nix-instantiate" + export depot_scanner="${depot.tools.depot-scanner}/bin/depot-scanner" + ''; + + # setup: git rev-parse --show-toplevel > $tvlc_root/depot_root + # setup: mkdir $tvlc_root/clients + # setup: echo 1 > $tvlc_root/next_clientid + + commonsh = pkgs.stdenv.mkDerivation { + name = "common.sh"; + src = ./common.sh; + doCheck = true; + unpackPhase = "true"; + buildPhase = '' + substitute ${./common.sh} $out --replace path-scripts ${pathScripts} + ''; + checkPhase = '' + ${pkgs.shellcheck}/bin/shellcheck $out ${pathScripts} && echo "SHELLCHECK OK" + ''; + installPhase = '' + chmod +x $out + ''; + }; + + tvlcNew = pkgs.stdenv.mkDerivation { + name = "tvlc-new"; + src = ./tvlc-new; + doCheck = true; + + unpackPhase = "true"; + buildPhase = '' + substitute ${./tvlc-new} $out --replace common.sh ${commonsh} + ''; + checkPhase = '' + ${pkgs.shellcheck}/bin/shellcheck $out ${commonsh} ${pathScripts} && echo "SHELLCHECK OK" + ''; + installPhase = '' + chmod +x $out + ''; + }; + +in { + inherit pathScripts; + inherit commonsh; + inherit tvlcNew; +} diff --git a/tools/tvlc/tvlc-new b/tools/tvlc/tvlc-new new file mode 100755 index 000000000000..4ef0df5d33b2 --- /dev/null +++ b/tools/tvlc/tvlc-new @@ -0,0 +1,103 @@ +#!/bin/bash + +source common.sh + +set -eu +set -o pipefail + +function usage() { + echo "tvlc new [-n|--name CLIENTNAME] [derivation...]" + echo "" + cat <<EOF + The 'new' command creates a new git sparse checkout with the given name, and + contents needed to build the Nix derivation(s) specified on the command line. + + Options: + -n/--name client-name: Sets the git branch and nice checkout name for the + workspace. If the option is not provided, the name will be based on the + first non-option command-line argument. + --branch branch-name: Sets the git branch name only. +EOF +} + +checkout_name= +branch_name= + +options=$(getopt -o 'n:' --long debug --long name: -- "$@") +eval set -- "$options" +while true; do + case "$1" in + -h) + usage + exit 0 + ;; + -v) + version + exit 0 + ;; + -n|--name) + shift + checkout_name="$1" + if [ -z "$branch_name" ]; then + branch_name=tvlc-"$1" + fi + ;; + --branch) + shift + branch_name="$1" + ;; + --) + shift + break + ;; + esac + shift +done + +if [ $# -eq 0 ]; then + echo "error: workspace name, target derivations required" + exit 1 +fi + +if [ -z "$checkout_name" ]; then + # TODO(riking): deduce + echo "error: workspace name (-n) required" + exit 1 +fi + +if [ -d "$nice_checkout_root/$checkout_name" ]; then + echo "error: checkout $checkout_name already exists" + # nb: shellescape checkout_name because we expect the user to copy-paste it + # shellcheck disable=SC1003 + echo "consider deleting it with tvlc remove '${checkout_name/'/\'}'" + exit 1 +fi +if [ -f "$DEPOT_ROOT/.git/refs/heads/$branch_name" ]; then + echo "error: branch $branch_name already exists in git" + # shellcheck disable=SC1003 + echo "consider deleting it with cd $DEPOT_ROOT; git branch -d '${checkout_name/'/\'}'" + exit 1 +fi + +# The big one: call into Nix to figure out what paths the desired derivations depend on. +readarray -t includedPaths < <("$depot_scanner" --mode 'print' --only 'DEPOT' --relpath --depot "$DEPOT_ROOT" --nix-bin "$tvix_instantiate" "$@") + +# bash math +checkout_id=$(("$(cat "$tvlc_root/next_clientid")")) +next_checkout_id=$(("$checkout_id"+1)) +echo "$next_checkout_id" > "$tvlc_root/next_clientid" + +checkout_dir="$tvlc_root/clients/$checkout_id" +mkdir "$checkout_dir" +cd "$DEPOT_ROOT" +git worktree add --no-checkout -b "$branch_name" "$checkout_dir" +# BUG: git not creating the /info/ subdir +mkdir "$DEPOT_ROOT/.git/worktrees/$checkout_id/info" + +cd "$checkout_dir" +git sparse-checkout init --cone +git sparse-checkout set "${includedPaths[@]}" + +ln -s "$checkout_dir" "$nice_checkout_root"/"$checkout_name" + +echo "$nice_checkout_root/$checkout_name" diff --git a/tvix/LICENSE b/tvix/LICENSE new file mode 100644 index 000000000000..f288702d2fa1 --- /dev/null +++ b/tvix/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/tvix/OWNERS b/tvix/OWNERS new file mode 100644 index 000000000000..3005e4f6d34b --- /dev/null +++ b/tvix/OWNERS @@ -0,0 +1,5 @@ +inherited: false +owners: + - adisbladis + - flokli + - tazjin diff --git a/tvix/README.md b/tvix/README.md new file mode 100644 index 000000000000..9569cedf33f4 --- /dev/null +++ b/tvix/README.md @@ -0,0 +1,20 @@ +Tvix +==== + +For more information about Tvix, contact one of the project owners. We +are interested in people who would like to help us review designs, +brainstorm and describe requirements that we may not yet have +considered. + +## License structure + +All code implemented for Tvix is licensed under the GPL-3.0, with the +exception of the protocol buffer definitions used for communication +between services which are available under a more permissive license +(MIT). + +The idea behind this structure is that any direct usage of our code +(e.g. linking to it, embedding the evaluator, etc.) will fall under +the terms of the GPL3, but users are free to implement their own +components speaking these protocols under the terms of the MIT +license. diff --git a/tvix/docs/.gitignore b/tvix/docs/.gitignore new file mode 100644 index 000000000000..77699ee8a3f7 --- /dev/null +++ b/tvix/docs/.gitignore @@ -0,0 +1,2 @@ +*.svg +*.html diff --git a/tvix/docs/Makefile b/tvix/docs/Makefile new file mode 100644 index 000000000000..ba9e2bdef6d3 --- /dev/null +++ b/tvix/docs/Makefile @@ -0,0 +1,12 @@ +all: build + +puml: + plantuml *.puml -tsvg + +html: + pandoc *.md -f markdown --self-contained -t html -s -o tvix.html --csl=${CSL} + +build: puml html + +clean: + rm -f *.tex *.pdf *.png *.svg diff --git a/tvix/docs/component-flow.puml b/tvix/docs/component-flow.puml new file mode 100644 index 000000000000..3bcddbe7464e --- /dev/null +++ b/tvix/docs/component-flow.puml @@ -0,0 +1,74 @@ +@startuml + +title Tvix build flow + +actor User +participant CLI +participant "Coordinator" as Coord +participant "Evaluator" as Eval +database Store +participant "Builder" as Build + +note over CLI,Eval + Typically runs locally on the invoking machine +end note +/ note over Store, Build + Can be either local or remote +end note + +User-->CLI: User initiates build of `hello` (analogous to `nix-build -f '<nixpkgs>' -A hello`) + +CLI-->Coord: CLI invokes coordinator + +Coord-->Eval: Sends message to start evaluation of `<nixpkgs>` (path lookup) with attribute `hello` +note right: The paths to the evaluator are local file system paths + +Coord<--Eval: Yields derivations to be built +note right + Immediately starts streaming derivations as they are instantiated across + the dependency graph so they can be built while the evaluation is still running. + + There are two types of build requests: One for regular "fire and forget" builds + and another for IFD (import from derivation). + + These are distinct because IFD needs to be fed back into the evaluator for + further processing while a regular build does not. +end note + +loop while has more derivations + + Coord-->Store: Check if desired paths are in store + alt Store has path + Coord<--Store: Success response + else Store does not have path + Coord-->Build: Request derivation to be built + note left + The build request optionally includes a desired store. + If a builder is aware of how to push to the store it will do so + directly when the build is finished. + + If the store is not known by the builder results will be streamed + back to the coordinator for store addition. + end note + + alt Build failure + Coord<--Build: Fail response + note left: It's up to the coordinator whether to exit on build failure + else Build success + alt Known store + Build-->Store: Push outputs to store + Build<--Coord: Send success & pushed response + else Unknown store + Build<--Coord: Send success & not pushed response + Coord<--Build: Stream build outputs + Coord-->Store: Push outputs to store + end + end + + end +end + +CLI<--Coord: Respond success/fail +User<--CLI: Exit success/fail + +@enduml diff --git a/tvix/docs/components.md b/tvix/docs/components.md new file mode 100644 index 000000000000..19e7baa3ec8a --- /dev/null +++ b/tvix/docs/components.md @@ -0,0 +1,114 @@ +--- +title: "Tvix - Architecture & data flow" +numbersections: true +author: +- adisbladis +- flokli +- tazjin +email: +- adis@blad.is +- mail@tazj.in +lang: en-GB +classoption: +- twocolumn +header-includes: +- \usepackage{caption, graphicx, tikz, aeguill, pdflscape} +--- + +# Background + +We intend for Tvix tooling to be more decoupled than the existing, +monolithic Nix implementation. In practice, we expect to gain several +benefits from this, such as: + +- Ability to use different builders +- Ability to use different store implementations +- No monopolisation of the implementation, allowing users to replace + components that they are unhappy with (up to and including the + language evaluator) +- Less hidden intra-dependencies between tools due to explicit RPC/IPC + boundaries + +Communication between different components of the system will use +gRPC. The rest of this document outlines the components. + +# Components + +## Coordinator + +*Purpose:* The coordinator (in the simplest case, the Tvix CLI tool) +oversees the flow of a build process and delegates tasks to the right +subcomponents. For example, if a user runs the equivalent of +`nix-build` in a folder containing a `default.nix` file, the +coordinator will invoke the evaluator, pass the resulting derivations +to the builder and coordinate any necessary store interactions (for +substitution and other purposes). + +While many users are likely to use the CLI tool as their primary +method of interacting with Tvix, it is not unlikely that alternative +coordinators (e.g. for a distributed, "Nix-native" CI system) would be +implemented. To facilitate this, we are considering implementing the +coordinator on top of a state-machine model that would make it +possible to reuse the FSM logic without tying it to any particular +kind of application. + +## Evaluator + +*Purpose:* Eval takes care of evaluating Nix code. In a typical build +flow it would be responsible for producing derivations. It can also be +used as a standalone tool, for example, in use-cases where Nix is used +to generate configuration without any build or store involvement. + +*Requirements:* For now, it will run on the machine invoking the build +command itself. We give it filesystem access to handle things like +imports or `builtins.readFile`. + +In the future, we might abstract away raw filesystem access by +allowing the evaluator to request files from the coordinator (which +will query the store for it). This might get messy, and the benefits +are questionable. We might be okay with running the evaluator with +filesystem access for now and can extend the interface if the need +arises. + +## Builder + +*Purpose:* A builder receives derivations from the coordinator and +builds them. + +By making builder a standardised interface it's possible to make the +sandboxing mechanism used by the build process pluggable. + +Nix is currently using a hard-coded +[libseccomp](https://github.com/seccomp/libseccomp) based sandboxing +mechanism and another one based on +[sandboxd](https://www.unix.com/man-page/mojave/8/sandboxd/) on macOS. +These are only separated by [compiler preprocessor +macros](https://gcc.gnu.org/onlinedocs/cpp/Ifdef.html) within the same +source files despite having very little in common with each other. + +This makes experimentation with alternative backends difficult and +porting Nix to other platforms harder than it has to be. We want to +write a new Linux builder which uses +[OCI](https://github.com/opencontainers/runtime-spec), the current +dominant Linux containerisation technology, by default. + +With a well-defined builder abstraction, it's also easy to imagine +other backends such as a Kubernetes-based one in the future. + +## Store + +*Purpose:* Store takes care of storing build results. It provides a +unified interface to get file paths and upload new ones. + +Most likely, we will end up with multiple implementations of store, a +few possible ones that come to mind are: + +- Local +- SSH +- GCP +- S3 +- Ceph + +# Figures + + diff --git a/tvix/docs/default.nix b/tvix/docs/default.nix new file mode 100644 index 000000000000..4b09f8d2dae2 --- /dev/null +++ b/tvix/docs/default.nix @@ -0,0 +1,46 @@ +{ pkgs, lib, ... }: + +let + + tl = pkgs.texlive.combine { + inherit (pkgs.texlive) scheme-medium wrapfig ulem capt-of + titlesec preprint enumitem paralist ctex environ svg + beamer trimspaces zhnumber changepage framed pdfpages + fvextra minted upquote ifplatform xstring; + }; + + csl = pkgs.fetchurl { + name = "numeric.csl"; + url = "https://gist.githubusercontent.com/bwiernik/8c6f39cf51ceb3a03107/raw/1d75c2d62113ffbba6ed03a47ad99bde86934f2b/APA%2520Numeric"; + sha256 = "1yfhhnhbzvhrv93baz98frmgsx5y442nzhb0l956l4j35fb0cc3h"; + }; + +in pkgs.stdenv.mkDerivation { + pname = "tvix-doc"; + version = "0.1"; + + outputs = [ "out" "svg" ]; + + src = lib.cleanSource ./.; + + CSL = csl; + + nativeBuildInputs = [ + pkgs.pandoc + pkgs.plantuml + tl + ]; + + installPhase = '' + runHook preInstall + + mkdir -p $out + cp -v *.html $out/ + + mkdir -p $svg + cp -v *.svg $svg/ + + runHook postSubmit + ''; + +} diff --git a/tvix/docs/language-spec.md b/tvix/docs/language-spec.md new file mode 100644 index 000000000000..a71437493307 --- /dev/null +++ b/tvix/docs/language-spec.md @@ -0,0 +1,78 @@ +--- +title: "Specification of the Nix language" +numbersections: true +author: +- tazjin +email: +- tazjin@tvl.su +lang: en-GB +--- + +The Nix Language +================ + +WARNING: This document is a work in progress. Please keep an eye on +[`topic:nix-spec`](https://cl.tvl.fyi/q/topic:nix-spec) for ongoing +CLs. + +Nix is a general-purpose, functional programming language which this +document aims to describe. + +## Background + +Nix was designed and implemented as part of the [Nix package +manager](https://nixos.org/nix). It is primarily used for generating +so-called [*derivations*](#derivations), which are data structures +describing how to build a package. + +The language has been described in the +[thesis](https://edolstra.github.io/pubs/phd-thesis.pdf) introducing +the package manager, but only on a high-level. At the time of writing, +Nix is informally specified (via its only complete implementation in +the package manager) and there is no complete overview over its - +sometimes surprising - semantics. + +The primary project written in Nix is +[nixpkgs](https://github.com/NixOS/nixpkgs/). Uncertainties in the +process of writing this specification are resolved by investigating +patterns in nixpkgs, which we consider canonical. The code in nixpkgs +uses a reasonable subset of the features exposed by the current +implementation, some of which are *accidental*, and is thus more +useful for specifying how the language should work. + +## Introduction to Nix + +Nix is a general-purpose, partially lazy, functional programming +language which provides higher-order functions, type reflection, +primitive data types such as integers, strings and floats, and +compound data structures such as lists and attribute sets. + +Nix has syntactic sugar for common operations, such as those for +attribute sets, and also provides a wide range of built-in functions +which have organically accumulated over time. + +Nix has a variety of legacy features that are not in practical use, +but are documented in sections of this specification for the sake of +completeness. + +This document describes the syntax and abstract semantics of the Nix +language, but leaves out implementation details about how Nix can be +interpreted/compiled/analysed etc. + +### Program structure + +This section describes the semantic structure of Nix, and how it +relates to the rest of the specification. + +Each Nix program is a single [*expression*](#expressions) denoting a +[*value*](#values) (commonly a [*function*](#functions)). Each value +has a [*type*](#types), however this type is not statically known. + +Nix code is modularised through the use of the +[*import*](#builtins-import) built-in function. No separate module +system exists. + +In addition to chapters describing the building blocks mentioned +above, this specificiation also describes the [*syntax*](#syntax), the +available [built-in functions](#builtins), [*error handling*](#errors) +and known [*deficiencies*](#deficiencies) in the language. diff --git a/tvix/proto/LICENSE b/tvix/proto/LICENSE new file mode 100644 index 000000000000..36878fe4cbf1 --- /dev/null +++ b/tvix/proto/LICENSE @@ -0,0 +1,21 @@ +Copyright © 2021 The Tvix Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +“Software”), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/tvix/proto/default.nix b/tvix/proto/default.nix new file mode 100644 index 000000000000..7ff03ba3b74f --- /dev/null +++ b/tvix/proto/default.nix @@ -0,0 +1,9 @@ +# Build protocol buffer definitions to ensure that protos are valid in +# CI. Note that the output of this build target is not actually used +# anywhere, it just functions as a CI check for now. +{ pkgs, ... }: + +pkgs.runCommandNoCC "tvix-cc-proto" {} '' + mkdir $out + ${pkgs.protobuf}/bin/protoc -I ${./.} evaluator.proto --cpp_out=$out +'' diff --git a/tvix/proto/evaluator.proto b/tvix/proto/evaluator.proto new file mode 100644 index 000000000000..710a28fb9df7 --- /dev/null +++ b/tvix/proto/evaluator.proto @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +// Copyright © 2021 The Tvix Authors +syntax = "proto3"; + +package tvix.proto.evaluator.v1; + +service EvaluatorService { + rpc Evaluate(stream EvaluateRequest) returns (stream EvaluateResponse) {} +} + +// +// Message types for EvaluateRequest +// + +message EvaluateFile { + // Absolute path at which the evaluator can find the file to be + // evaluated. + string file_path = 1; + + // Optional attribute that should be evaluated within the file, + // assuming that the value it evaluates to is an attribute set. + optional string attribute = 2; + + // Additional arguments to pass into the evaluation, with which the + // file's top-level function will be auto-called. + map<string, NixValue> arguments = 3; +} + +message EvaluateExpression { + // Literal Nix expression to evaluate. + string expression = 1; + + // Working directory in which the expression should be evaluated. + string working_directory = 2; +} + +message BuildResultChunk { + string drv_hash = 1; + string output = 2; + bytes data = 3; + + // This field may be set on the first build result chunk returned + // to the evaluator, indicating the total size of the output that + // is going to be streamed in bytes. + // + // If set, the evaluator can use this to appropriately allocate a + // buffer for the output. + optional int64 output_size = 4; +} + +// Indicates that a single build has completed successfully. In case +// that the build outputs were required by the evaluator this also +// indicates that the output has been returned completely. +message BuildSuccess { + string drv_hash = 1; + string output = 2; +} + +// Describes an error that occured during a single build. +// +// TODO: We might want a more sophisticated error type. +message BuildError { + string drv_hash = 1; + string output = 2; + string error = 3; +} + +message BuildResult { + oneof build_result { + BuildSuccess build_success = 1; + BuildError build_error = 2; + } +} + + +/// Messages sent to the evaluator by the build coordinator. +message EvaluateRequest { + oneof message { + // Ask the evaluator to evaluate the specified file, and + // optionally attribute within that file. Must be the first + // message. + EvaluateFile evaluate_file = 1; + + // Ask the evaluator to evaluate the specified Nix expression. + // Must be the first message. + EvaluateExpression evaluate_expression = 2; + + // Send the chunks of a build result, in response to a + // BuildRequest. + // + // Note: This message might change as the store protocol is + // designed, as it is possible that mechanisms for transferring + // files might be reused between the protocols. + BuildResultChunk build_result_chunk = 3; + + // Indicate the result of a single build. See the documentation + // for the message types defined above for semantic details. + BuildResult build_result = 4; + } +} + +// +// Message types for EvaluateResponse +// + +// TODO: Placeholder type. +message Derivation { + string drv = 1; +} + +// TODO: Placeholder type. +message NixValue { + string value = 1; +} + +// TODO: Placeholder type. +message NixError { + string value = 1; +} + +message BuildRequest { + Derivation drv = 1; + string output = 2; +} + +// Messages returned to the coordinator by the evaluator. +message EvaluateResponse { + oneof message { + // A derivation that was instantiated while reducing the graph, + // and whose output is not required by the evaluator. + Derivation derivation = 1; + + // A derivation that was instantiated while reducing the graph, + // and whose output is required by the evaluator (IFD). + BuildRequest build_request = 2; + + // The final value yielded by the evaluation. Stream is closed + // after this. + NixValue done = 3; + + // Evaluation error. Stream is closed after this. + NixError error = 4; + } +} diff --git a/users/Profpatsch/OWNERS b/users/Profpatsch/OWNERS new file mode 100644 index 000000000000..5a73d4c3a1fd --- /dev/null +++ b/users/Profpatsch/OWNERS @@ -0,0 +1,4 @@ +inherited: false +owners: + - Profpatsch + - sterni diff --git a/users/Profpatsch/advent-of-code/2020/01/main.py b/users/Profpatsch/advent-of-code/2020/01/main.py new file mode 100644 index 000000000000..e636017a54d5 --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/01/main.py @@ -0,0 +1,22 @@ +import sys + +l = [] +with open('./input', 'r') as f: + for line in f: + l.append(int(line)) + +s = set(l) + +res=None +for el in s: + for el2 in s: + if (2020-(el+el2)) in s: + res=(el, el2, 2020-(el+el2)) + break + +if res == None: + sys.exit("could not find a number that adds to 2020") + +print(res) + +print(res[0] * res[1] * res[2]) diff --git a/users/Profpatsch/advent-of-code/2020/02/main.py b/users/Profpatsch/advent-of-code/2020/02/main.py new file mode 100644 index 000000000000..e3b27c382a21 --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/02/main.py @@ -0,0 +1,77 @@ +import sys + +def parse(line): + a = line.split(sep=" ", maxsplit=1) + assert len(a) == 2 + fromto = a[0].split(sep="-") + assert len(fromto) == 2 + (from_, to) = (int(fromto[0]), int(fromto[1])) + charpass = a[1].split(sep=": ") + assert len(charpass) == 2 + char = charpass[0] + assert len(char) == 1 + pass_ = charpass[1] + assert pass_.endswith("\n") + pass_ = pass_[:-1] + return { + "from": from_, + "to": to, + "char": char, + "pass": pass_ + } + +def char_in_pass(char, pass_): + return pass_.count(char) + +def validate_01(entry): + no = char_in_pass(entry["char"], entry["pass"]) + if no < entry["from"]: + return { "too-small": entry } + elif no > entry["to"]: + return { "too-big": entry } + else: + return { "ok": entry } + +def char_at_pos(char, pos, pass_): + assert pos <= len(pass_) + return pass_[pos-1] == char + +def validate_02(entry): + one = char_at_pos(entry["char"], entry["from"], entry["pass"]) + two = char_at_pos(entry["char"], entry["to"], entry["pass"]) + if one and two: + return { "both": entry } + elif one: + return { "one": entry } + elif two: + return { "two": entry } + else: + return { "none": entry } + + +res01 = [] +res02 = [] +with open("./input", 'r') as f: + for line in f: + p = parse(line) + res01.append(validate_01(p)) + res02.append(validate_02(p)) + +count01=0 +for r in res01: + print(r) + if r.get("ok", False): + count01=count01+1 + +count02=0 +for r in res02: + print(r) + if r.get("one", False): + count02=count02+1 + elif r.get("two", False): + count02=count02+1 + else: + pass + +print("count 1: {}".format(count01)) +print("count 2: {}".format(count02)) diff --git a/users/Profpatsch/advent-of-code/2020/03/main.py b/users/Profpatsch/advent-of-code/2020/03/main.py new file mode 100644 index 000000000000..4d6baf946c3e --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/03/main.py @@ -0,0 +1,66 @@ +import itertools +import math + +def tree_line(init): + return { + "init-len": len(init), + "known": '', + "rest": itertools.repeat(init) + } + +def tree_line_at(pos, tree_line): + needed = (pos + 1) - len(tree_line["known"]) + # internally advance the tree line to the position requested + if needed > 0: + tree_line["known"] = tree_line["known"] \ + + ''.join( + itertools.islice( + tree_line["rest"], + 1+math.floor(needed / tree_line["init-len"]))) + # print(tree_line) + return tree_line["known"][pos] == '#' + +def tree_at(linepos, pos, trees): + return tree_line_at(pos, trees[linepos]) + +def slope_positions(trees, right, down): + line = 0 + pos = 0 + while line < len(trees): + yield (line, pos) + line = line + down + pos = pos + right + +trees = [] +with open("./input", 'r') as f: + for line in f: + line = line.rstrip() + trees.append(tree_line(line)) + +# print(list(itertools.islice(trees[0], 5))) +# print(list(map( +# lambda x: tree_at(0, x, trees), +# range(100) +# ))) +# print(list(slope_positions(trees, right=3, down=1))) + +def count_slope_positions(trees, slope): + count = 0 + for (line, pos) in slope: + if tree_at(line, pos, trees): + count = count + 1 + return count + +print( + count_slope_positions(trees, slope_positions(trees, right=1, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=3, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=5, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=7, down=1)) + * + count_slope_positions(trees, slope_positions(trees, right=1, down=2)) +) + +# I realized I could have just used a modulo instead … diff --git a/users/Profpatsch/advent-of-code/2020/04/main.py b/users/Profpatsch/advent-of-code/2020/04/main.py new file mode 100644 index 000000000000..36bbed7146d6 --- /dev/null +++ b/users/Profpatsch/advent-of-code/2020/04/main.py @@ -0,0 +1,104 @@ +import sys +import itertools +import re +import pprint + +def get_entry(fd): + def to_dict(keyval): + res = {} + for (k, v) in keyval: + assert k not in res + res[k] = v + return res + + res = [] + for line in fd: + if line == "\n": + yield to_dict(res) + res = [] + else: + line = line.rstrip() + items = line.split(" ") + for i in items: + res.append(i.split(":", maxsplit=2)) + +def val_hgt(hgt): + m = re.fullmatch(r'([0-9]+)(cm|in)', hgt) + if m: + (i, what) = m.group(1,2) + i = int(i) + if what == "cm": + return i >= 150 and i <= 193 + elif what == "in": + return i >= 59 and i <= 76 + else: + return False + +required_fields = [ + { "name": "byr", + "check": lambda s: int(s) >= 1920 and int(s) <= 2002 + }, + { "name": "iyr", + "check": lambda s: int(s) >= 2010 and int(s) <= 2020 + }, + { "name": "eyr", + "check": lambda s: int(s) >= 2020 and int(s) <= 2030, + }, + { "name": "hgt", + "check": lambda s: val_hgt(s) + }, + { "name": "hcl", + "check": lambda s: re.fullmatch(r'#[0-9a-f]{6}', s) + }, + { "name": "ecl", + "check": lambda s: re.fullmatch(r'amb|blu|brn|gry|grn|hzl|oth', s) + }, + { "name": "pid", + "check": lambda s: re.fullmatch(r'[0-9]{9}', s) + }, + # we should treat it as not required + # "cid" +] + +required_dict = {} +for f in required_fields: + required_dict[f["name"]] = f + +def validate(keyval): + if keyval[0] not in required_dict: + return { "ok": keyval } + if required_dict[keyval[0]]["check"](keyval[1]): + return { "ok": keyval } + else: + return { "validation": keyval } + +def all_fields(entry): + missing = [] + for r in required_dict: + if r not in e: + missing.append(r) + if missing == []: + return { "ok": entry } + else: + return { "missing": missing } + +count=0 +for e in get_entry(sys.stdin): + a = all_fields(e) + if a.get("ok", False): + res = {} + bad = False + for keyval in e.items(): + r = validate(keyval) + if r.get("validation", False): + bad = True + res[keyval[0]] = r + if bad: + pprint.pprint({ "validation": res }) + else: + pprint.pprint({ "ok": e }) + count = count+1 + else: + pprint.pprint(a) + +print(count) diff --git a/users/Profpatsch/arglib/netencode.nix b/users/Profpatsch/arglib/netencode.nix new file mode 100644 index 000000000000..50f4c11c2d8f --- /dev/null +++ b/users/Profpatsch/arglib/netencode.nix @@ -0,0 +1,40 @@ +{ depot, pkgs, lib, ... }: + +let + netencode = { + rust = depot.nix.writers.rustSimpleLib { + name = "arglib-netencode"; + dependencies = [ + depot.users.Profpatsch.execline.exec-helpers + depot.users.Profpatsch.netencode.netencode-rs + ]; + } '' + extern crate netencode; + extern crate exec_helpers; + + use netencode::{T}; + use std::os::unix::ffi::OsStrExt; + + pub fn arglib_netencode(prog_name: &str, env: Option<&std::ffi::OsStr>) -> T { + let env = match env { + None => std::ffi::OsStr::from_bytes("ARGLIB_NETENCODE".as_bytes()), + Some(a) => a + }; + let t = match std::env::var_os(env) { + None => exec_helpers::die_user_error(prog_name, format!("could not read args, envvar {} not set", env.to_string_lossy())), + // TODO: good error handling for the different parser errors + Some(soup) => match netencode::parse::t_t(soup.as_bytes()) { + Ok((remainder, t)) => match remainder.is_empty() { + true => t, + false => exec_helpers::die_environment_problem(prog_name, format!("arglib: there was some unparsed bytes remaining: {:?}", remainder)) + }, + Err(err) => exec_helpers::die_environment_problem(prog_name, format!("arglib parsing error: {:?}", err)) + } + }; + std::env::remove_var(env); + t + } + ''; + }; + +in depot.nix.readTree.drvTargets netencode diff --git a/users/Profpatsch/atomically-write.nix b/users/Profpatsch/atomically-write.nix new file mode 100644 index 000000000000..d5039d3e46b9 --- /dev/null +++ b/users/Profpatsch/atomically-write.nix @@ -0,0 +1,28 @@ +{ depot, pkgs, ... }: +# Atomically write a file (just `>` redirection in bash +# empties a file even if the command crashes). +# +# Maybe there is an existing tool for that? +# But it’s easy enough to implement. +# +# Example: +# atomically-write +# ./to +# echo "foo" +# +# will atomically write the string "foo" into ./to +let + atomically-write = pkgs.writers.writeDash "atomically-write" '' + set -e + to=$1 + shift + # assumes that the tempfile is on the same file system, (or in memory) + # for the `mv` at the end to be more-or-less atomic. + tmp=$(${pkgs.coreutils}/bin/mktemp -d) + trap 'rm -r "$tmp"' EXIT + "$@" \ + > "$tmp/out" + mv "$tmp/out" "$to" + ''; + +in atomically-write diff --git a/users/Profpatsch/blog/default.nix b/users/Profpatsch/blog/default.nix new file mode 100644 index 000000000000..6ac3c3eb5104 --- /dev/null +++ b/users/Profpatsch/blog/default.nix @@ -0,0 +1,373 @@ +{ depot, pkgs, lib, ... }: + +let + bins = depot.nix.getBins pkgs.lowdown [ "lowdown" ] + // depot.nix.getBins pkgs.cdb [ "cdbget" "cdbmake" "cdbdump" ] + // depot.nix.getBins pkgs.coreutils [ "mv" "cat" "printf" "test" ] + // depot.nix.getBins pkgs.s6-networking [ "s6-tcpserver" ] + // depot.nix.getBins pkgs.time [ "time" ] + ; + + # / + # TODO: use + toplevel = [ + { + route = [ "notes" ]; + name = "Notes"; + page = {cssFile}: router cssFile; + } + { + route = [ "projects" ]; + name = "Projects"; + # page = projects; + } + ]; + + # /notes/* + notes = [ + { + route = [ "notes" "preventing-oom" ]; + name = "Preventing out-of-memory (OOM) errors on Linux"; + page = {cssFile}: markdownToHtml { + name = "preventing-oom"; + markdown = ./notes/preventing-oom.md; + inherit cssFile; + }; + } + { + route = [ "notes" "rust-string-conversions" ]; + name = "Converting between different String types in Rust"; + page = {cssFile}: markdownToHtml { + name = "rust-string-conversions"; + markdown = ./notes/rust-string-conversions.md; + inherit cssFile; + }; + } + ]; + + projects = [ + { + name = "lorri"; + description = "<code>nix-shell</code> replacement for projects"; + link = "https://github.com/nix-community/lorri"; + } + { + name = "netencode"; + description = ''A human-readble nested data exchange format inspired by <a href="https://en.wikipedia.org/wiki/Netstring">netstrings</a> and <a href="https://en.wikipedia.org/wiki/Bencode">bencode</a>.''; + link = depotCgitLink { relativePath = "users/Profpatsch/netencode/README.md"; }; + } + { + name = "yarn2nix"; + description = ''nix dependency generator for the <a href="https://yarnpkg.com/"><code>yarn</code> Javascript package manager</a>''; + link = "https://github.com/Profpatsch/yarn2nix"; + } + ]; + + posts = [ + { + date = "2017-05-04"; + title = "Ligature Emulation in Emacs"; + subtitle = "It’s not pretty, but the results are"; + description = "How to set up ligatures using <code>prettify-symbols-mode</code> and the Hasklig/FiraCode fonts."; + page = {cssFile}: markdownToHtml { + name = "2017-05-04-ligature-emluation-in-emacs"; + markdown = ./posts/2017-05-04-ligature-emulation-in-emacs.md; + inherit cssFile; + }; + route = [ "posts" "2017-05-04-ligature-emluation-in-emacs" ]; + tags = ["emacs"]; + } + ]; + + # convert a markdown file to html via lowdown + markdownToHtml = { + name, + # the file to convert + markdown, + # css file to add to the final result, as { route } + cssFile + }: + depot.nix.runExecline "${name}.html" {} ([ + "importas" "out" "out" + (depot.users.Profpatsch.lib.debugExec "") + bins.lowdown + "-s" "-Thtml" + ] ++ + (lib.optional (cssFile != null) (["-M" "css=${mkRoute cssFile.route}"])) + ++ [ + "-o" "$out" + markdown + ]); + + # takes a { route … } attrset and converts the route lists to an absolute path + fullRoute = attrs: lib.pipe attrs [ + (map (x@{route, ...}: x // { route = mkRoute route; })) + ]; + + # a cdb from route to a netencoded version of data for each route + router = cssFile: lib.pipe (notes ++ posts) [ + (map (r: with depot.users.Profpatsch.lens; + lib.pipe r [ + (over (field "route") mkRoute) + (over (field "page") (_ { inherit cssFile; })) + ])) + (map (x: { + name = x.route; + value = depot.users.Profpatsch.netencode.gen.dwim x; + })) + lib.listToAttrs + (cdbMake "router") + ]; + + # Create a link to the given source file/directory, given the relative path in the depot repo. + # Checks that the file exists at evaluation time. + depotCgitLink = { + # relative path from the depot root (without leading /). + relativePath + }: + assert + (lib.assertMsg + (builtins.pathExists (depot.path + "/" + relativePath)) + "depotCgitLink: path /${relativePath} does not exist in depot"); + "https://code.tvl.fyi/tree/${relativePath}"; + + # look up a route by path ($1) + router-lookup = cssFile: depot.nix.writeExecline "router-lookup" { readNArgs = 1; } [ + cdbLookup (router cssFile) "$1" + ]; + + runExeclineStdout = name: args: cmd: depot.nix.runExecline name args ([ + "importas" "-ui" "out" "out" + "redirfd" "-w" "1" "$out" + ] ++ cmd); + + notes-index-html = + let o = fullRoute notes; + in '' + <ul> + ${scope o (o: '' + <li><a href="${str o.route}">${esc o.name}</a></li> + '')} + </ul> + ''; + + notes-index = pkgs.writeText "notes-index.html" notes-index-html; + + # A simple mustache-inspired string interpolation combinator + # that takes an object and a template (a function from o to string) + # and returns a string. + scope = o: tpl: + if builtins.typeOf o == "list" then + lib.concatMapStringsSep "\n" tpl o + else if builtins.typeOf o == "set" then + tpl o + else throw "${lib.generators.toPretty {} o} not allowed in template"; + + # string-escape html (TODO) + str = s: s; + # html-escape (TODO) + esc = s: s; + html = s: s; + + projects-index-html = + let o = projects; + in '' + <dl> + ${scope o (o: '' + <dt><a href="${str o.link}">${esc o.name}</a></dt> + <dd>${html o.description}</dd> + '')} + </dl> + ''; + + projects-index = pkgs.writeText "projects-index.html" projects-index-html; + + posts-index-html = + let o = fullRoute posts; + in '' + <dl> + ${scope o (o: '' + <dt>${str o.date} <a href="${str o.route}">${esc o.title}</a></dt> + <dd>${html o.description}</dd> + '')} + </dl> + ''; + + posts-index = pkgs.writeText "projects-index.html" posts-index-html; + + arglibNetencode = val: depot.nix.writeExecline "arglib-netencode" { } [ + "export" "ARGLIB_NETENCODE" (depot.users.Profpatsch.netencode.gen.dwim val) + "$@" + ]; + + # A simple http server that serves the site. Yes, it’s horrible. + site-server = { cssFile, port }: depot.nix.writeExecline "blog-server" {} [ + (depot.users.Profpatsch.lib.runInEmptyEnv [ "PATH" ]) + bins.s6-tcpserver "127.0.0.1" port + bins.time "--format=time: %es" "--" + runOr return400 + "pipeline" [ + (arglibNetencode { + what = "request"; + }) + depot.users.Profpatsch.read-http + ] + depot.users.Profpatsch.netencode.record-splice-env + runOr return500 + "importas" "-i" "path" "path" + "if" [ depot.tools.eprintf "GET \${path}\n" ] + runOr return404 + "backtick" "-ni" "TEMPLATE_DATA" [ + # TODO: factor this out of here, this is routing not serving + "ifelse" [ bins.test "$path" "=" "/notes" ] + [ "export" "content-type" "text/html" + "export" "serve-file" notes-index + depot.users.Profpatsch.netencode.env-splice-record + ] + "ifelse" [ bins.test "$path" "=" "/projects" ] + [ "export" "content-type" "text/html" + "export" "serve-file" projects-index + depot.users.Profpatsch.netencode.env-splice-record + ] + "ifelse" [ bins.test "$path" "=" "/posts" ] + [ "export" "content-type" "text/html" + "export" "serve-file" posts-index + depot.users.Profpatsch.netencode.env-splice-record + ] + # TODO: ignore potential query arguments. See 404 message + "pipeline" [ (router-lookup cssFile) "$path" ] + depot.users.Profpatsch.netencode.record-splice-env + "importas" "-ui" "page" "page" + "export" "content-type" "text/html" + "export" "serve-file" "$page" + depot.users.Profpatsch.netencode.env-splice-record + ] + runOr return500 + "if" [ + "pipeline" [ bins.printf '' + HTTP/1.1 200 OK + Content-Type: {{{content-type}}}; charset=UTF-8 + Connection: close + + '' ] + depot.users.Profpatsch.netencode.netencode-mustache + ] + "pipeline" [ "importas" "t" "TEMPLATE_DATA" bins.printf "%s" "$t" ] + depot.users.Profpatsch.netencode.record-splice-env + "importas" "-ui" "serve-file" "serve-file" + bins.cat "$serve-file" + ]; + + # run argv or $1 if argv returns a failure status code. + runOr = depot.nix.writeExecline "run-or" { readNArgs = 1; } [ + "foreground" [ "$@" ] + "importas" "?" "?" + "ifelse" [ bins.test "$?" "-eq" "0" ] + [] + "if" [ depot.tools.eprintf "runOr: exited \${?}, running \${1}\n" ] + "$1" + ]; + + return400 = depot.nix.writeExecline "return400" {} [ + bins.printf "%s" '' + HTTP/1.1 400 Bad Request + Content-Type: text/plain; charset=UTF-8 + Connection: close + + '' + ]; + + return404 = depot.nix.writeExecline "return404" {} [ + bins.printf "%s" '' + HTTP/1.1 404 Not Found + Content-Type: text/plain; charset=UTF-8 + Connection: close + + This page doesn’t exist! Query arguments are not handled at the moment. + '' + ]; + + return500 = depot.nix.writeExecline "return500" {} [ + bins.printf "%s" '' + HTTP/1.1 500 Internal Server Error + Content-Type: text/plain; charset=UTF-8 + Connection: close + + Encountered an internal server error. Please try again. + '' + ]; + + capture-stdin = depot.nix.writers.rustSimple { + name = "capture-stdin"; + dependencies = [ depot.users.Profpatsch.execline.exec-helpers ]; + } '' + extern crate exec_helpers; + use std::io::Read; + fn main() { + let (args, prog) = exec_helpers::args_for_exec("capture-stdin", 1); + let valname = &args[1]; + let mut v : Vec<u8> = vec![]; + std::io::stdin().lock().read_to_end(&mut v).unwrap(); + exec_helpers::exec_into_args("capture-stdin", prog, vec![(valname, v)]); + } + ''; + + # go from a list of path elements to an absolute route string + mkRoute = route: "/" + lib.concatMapStringsSep "/" urlencodeAscii route; + + # urlencodes, but only ASCII characters + # https://en.wikipedia.org/wiki/Percent-encoding + urlencodeAscii = urlPiece: + let + raw = [ "!" "#" "$" "%" "&" "'" "(" ")" "*" "+" "," "/" ":" ";" "=" "?" "@" "[" "]" ]; + enc = [ "%21" "%23" "%24" "%25" "%26" "%27" "%28" "%29" "%2A" "%2B" "%2C" "%2F" "%3A" "%3B" "%3D" "%3F" "%40" "%5B" "%5D" ]; + rest = [ "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" "a" "b" "c" "d" "e" "f" "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "0" "1" "2" "3" "4" "5" "6" "7" "8" "9" "-" "_" "." "~" ]; + in + assert lib.assertMsg (lib.all (c: builtins.elem c (raw ++ rest)) (lib.stringToCharacters urlPiece)) + "urlencodeAscii: the urlPiece must only contain valid url ASCII characters, was: ${urlPiece}"; + builtins.replaceStrings raw enc urlPiece; + + + # create a cdb record entry, as required by the cdbmake tool + cdbRecord = key: val: + "+${toString (builtins.stringLength key)},${toString (builtins.stringLength val)}:" + + "${key}->${val}\n"; + + # create a full cdbmake input from an attribute set of keys to values (strings) + cdbRecords = + with depot.nix.yants; + defun [ (attrs (either drv string)) string ] + (attrs: + (lib.concatStrings (lib.mapAttrsToList cdbRecord attrs)) + "\n"); + + # run cdbmake on a list of key/value pairs (strings + cdbMake = name: attrs: depot.nix.runExecline "${name}.cdb" { + stdin = cdbRecords attrs; + } [ + "importas" "out" "out" + depot.users.Profpatsch.lib.eprint-stdin + "if" [ bins.cdbmake "db" "tmp" ] + bins.mv "db" "$out" + ]; + + # look up a key ($2) in the given cdb ($1) + cdbLookup = depot.nix.writeExecline "cdb-lookup" { readNArgs = 2; } [ + # cdb ($1) on stdin + "redirfd" "-r" "0" "$1" + # key ($2) lookup + bins.cdbget "$2" + ]; + +in depot.nix.readTree.drvTargets { + inherit + router + depotCgitLink + site-server + notes-index + notes-index-html + projects-index + projects-index-html + posts-index-html + ; + +} diff --git a/users/Profpatsch/blog/notes/preventing-oom.md b/users/Profpatsch/blog/notes/preventing-oom.md new file mode 100644 index 000000000000..59ea4f747700 --- /dev/null +++ b/users/Profpatsch/blog/notes/preventing-oom.md @@ -0,0 +1,33 @@ +tags: linux +date: 2020-01-25 +certainty: likely +status: initial +title: Preventing out-of-memory (OOM) errors on Linux + +# Preventing out-of-memory (OOM) errors on Linux + +I’ve been running out of memory more and more often lately. I don’t use any swap space because I am of the opinion that 16GB of memory should be sufficient for most daily and professional tasks. Which is generally true, however sometimes I have a runaway filling my memory. Emacs is very good at doing this for example, prone to filling your RAM when you open json files with very long lines. + +In theory, the kernel OOM killer should come in and save the day, but the Linux OOM killer is notorious for being extremely … conservative. It will try to free every internal structure it can before even thinking about touching any userspace processes. At that point, the desktop usually stopped responding minutes ago. + +Luckily the kernel provides memory statistics for the whole system, as well as single process, and the [`earlyoom`](https://github.com/rfjakob/earlyoom) tool uses those to keep memory usage under a certain limit. It will start killing processes, “heaviest” first, until the given upper memory limit is satisfied again. + +On NixOS, I set: + +```nix +{ + services.earlyoom = { + enable = true; + freeMemThreshold = 5; # <%5 free + }; +} +``` + +and after activation, this simple test shows whether the daemon is working: + +```shell +$ tail /dev/zero +fish: “tail /dev/zero” terminated by signal SIGTERM (Polite quit request) +``` + +`tail /dev/zero` searches for the last line of the file `/dev/zero`, and since it cannot know that there is no next line and no end to the stream of `\0` this file produces, it will fill the RAM as quickly as physically possible. Before it can fill it completely, `earlyoom` recognizes that the limit was breached, singles out the `tail` command as the process using the most amount of memory, and sends it a `SIGTERM`. diff --git a/users/Profpatsch/blog/notes/rust-string-conversions.md b/users/Profpatsch/blog/notes/rust-string-conversions.md new file mode 100644 index 000000000000..99071ef9d370 --- /dev/null +++ b/users/Profpatsch/blog/notes/rust-string-conversions.md @@ -0,0 +1,53 @@ +# Converting between different String types in Rust + +``` +let s: String = ... +let st: &str = ... +let u: &[u8] = ... +let b: [u8; 3] = b"foo" +let v: Vec<u8> = ... +let os: OsString = ... +let ost: OsStr = ... + +From To Use Comment +---- -- --- ------- +&str -> String String::from(st) +&str -> &[u8] st.as_bytes() +&str -> Vec<u8> st.as_bytes().to_owned() via &[u8] +&str -> &OsStr OsStr::new(st) + +String -> &str &s alt. s.as_str() +String -> &[u8] s.as_bytes() +String -> Vec<u8> s.into_bytes() +String -> OsString OsString::from(s) + +&[u8] -> &str str::from_utf8(u).unwrap() +&[u8] -> String String::from_utf8(u).unwrap() +&[u8] -> Vec<u8> u.to_owned() +&[u8] -> &OsStr OsStr::from_bytes(u) use std::os::unix::ffi::OsStrExt; + +[u8; 3] -> &[u8] &b[..] byte literal +[u8; 3] -> &[u8] "foo".as_bytes() alternative via utf8 literal + +Vec<u8> -> &str str::from_utf8(&v).unwrap() via &[u8] +Vec<u8> -> String String::from_utf8(v) +Vec<u8> -> &[u8] &v +Vec<u8> -> OsString OsString::from_vec(v) use std::os::unix::ffi::OsStringExt; + +&OsStr -> &str ost.to_str().unwrap() +&OsStr -> String ost.to_os_string().into_string() via OsString + .unwrap() +&OsStr -> Cow<str> ost.to_string_lossy() Unicode replacement characters +&OsStr -> OsString ost.to_os_string() +&OsStr -> &[u8] ost.as_bytes() use std::os::unix::ffi::OsStringExt; + +OsString -> String os.into_string().unwrap() returns original OsString on failure +OsString -> &str os.to_str().unwrap() +OsString -> &OsStr os.as_os_str() +OsString -> Vec<u8> os.into_vec() use std::os::unix::ffi::OsStringExt; +``` + + +## Source + +Original source is [this document on Pastebin](https://web.archive.org/web/20190710121935/https://pastebin.com/Mhfc6b9i) diff --git a/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md b/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md new file mode 100644 index 000000000000..ba80888badd8 --- /dev/null +++ b/users/Profpatsch/blog/posts/2017-05-04-ligature-emulation-in-emacs.md @@ -0,0 +1,123 @@ +title: Ligature Emulation in Emacs +date: 2017-05-04 + +Monday was (yet another) +[NixOS hackathon][hackathon] at [OpenLab Augsburg][ola]. +[Maximilian][mhuber] was there and to my amazement +he got working ligatures in his Haskell files in Emacs! Ever since Hasklig +updated its format to use ligatures and private Unicode code points a while ago, +the hack I had used in my config stopped working. + +Encouraged by that I decided to take a look on Tuesday. Long story short, I was +able to [get it working in a pretty satisfying way][done]. + +[hackathon]: https://www.meetup.com/Munich-NixOS-Meetup/events/239077247/ +[mhuber]: https://github.com/maximilianhuber +[ola]: https://openlab-augsburg.de +[done]: https://github.com/i-tu/Hasklig/issues/84#issuecomment-298803495 + +What’s left to do is package it into a module and push to melpa. + + +### elisp still sucks, but it’s bearable, sometimes + +I’m the kind of person who, when trying to fix something elisp related, normally +gives up two hours later and three macro calls deep. Yes, homoiconic, +non-lexically-scoped, self-rewriting code is not exactly my fetish. +This time the task and the library (`prettify-symbols-mode`) were simple enough +for that to not happen. + +Some interesting technical trivia: + +- elisp literal character syntax is `?c`. `?\t` is the tab character +- You join characters by `(string c1 c2 c3 ...)` +- [dash.el][dash] is pretty awesome and does what a functional programmer + expects. Also, Rainbow Dash. +- Hasklig and FiraCode multi-column symbols actually [only occupy one column, on + the far right of the glyph][glyph]. `my-correct-symbol-bounds` fixes emacs’ + rendering in that case. + + +[dash]: https://github.com/magnars/dash.el +[glyph]: https://github.com/tonsky/FiraCode/issues/211#issuecomment-239082368 + + +## Appendix A + +For reference, here’s the complete code as it stands now. Feel free to paste +into your config; let’s make it [MIT][mit]. Maybe link to this site, in case there are +updates. + +[mit]: https://opensource.org/licenses/MIT + +```elisp + (defun my-correct-symbol-bounds (pretty-alist) + "Prepend a TAB character to each symbol in this alist, +this way compose-region called by prettify-symbols-mode +will use the correct width of the symbols +instead of the width measured by char-width." + (mapcar (lambda (el) + (setcdr el (string ?\t (cdr el))) + el) + pretty-alist)) + + (defun my-ligature-list (ligatures codepoint-start) + "Create an alist of strings to replace with +codepoints starting from codepoint-start." + (let ((codepoints (-iterate '1+ codepoint-start (length ligatures)))) + (-zip-pair ligatures codepoints))) + + ; list can be found at https://github.com/i-tu/Hasklig/blob/master/GlyphOrderAndAliasDB#L1588 + (setq my-hasklig-ligatures + (let* ((ligs '("&&" "***" "*>" "\\\\" "||" "|>" "::" + "==" "===" "==>" "=>" "=<<" "!!" ">>" + ">>=" ">>>" ">>-" ">-" "->" "-<" "-<<" + "<*" "<*>" "<|" "<|>" "<$>" "<>" "<-" + "<<" "<<<" "<+>" ".." "..." "++" "+++" + "/=" ":::" ">=>" "->>" "<=>" "<=<" "<->"))) + (my-correct-symbol-bounds (my-ligature-list ligs #Xe100)))) + + ;; nice glyphs for haskell with hasklig + (defun my-set-hasklig-ligatures () + "Add hasklig ligatures for use with prettify-symbols-mode." + (setq prettify-symbols-alist + (append my-hasklig-ligatures prettify-symbols-alist)) + (prettify-symbols-mode)) + + (add-hook 'haskell-mode-hook 'my-set-hasklig-ligatures) +``` + +## Appendix B (Update 1): FiraCode integration + +I also created a mapping for [FiraCode][fira]. You need to grab the [additional +symbol font][symbol] that adds (most) ligatures to the unicode private use area. +Consult your system documentation on how to add it to your font cache. +Next add `"Fira Code"` and `"Fira Code Symbol"` to your font preferences. Symbol +only contains the additional characters, so you need both. + +If you are on NixOS, the font package should be on the main branch shortly, [I +added a package][symbol-pkg]. + +[fira]: https://github.com/tonsky/FiraCode/ +[symbol]: https://github.com/tonsky/FiraCode/issues/211#issuecomment-239058632 +[symbol-pkg]: https://github.com/NixOS/nixpkgs/pull/25517 + +Here’s the mapping adjusted for FiraCode: + +```elisp + (setq my-fira-code-ligatures + (let* ((ligs '("www" "**" "***" "**/" "*>" "*/" "\\\\" "\\\\\\" + "{-" "[]" "::" ":::" ":=" "!!" "!=" "!==" "-}" + "--" "---" "-->" "->" "->>" "-<" "-<<" "-~" + "#{" "#[" "##" "###" "####" "#(" "#?" "#_" "#_(" + ".-" ".=" ".." "..<" "..." "?=" "??" ";;" "/*" + "/**" "/=" "/==" "/>" "//" "///" "&&" "||" "||=" + "|=" "|>" "^=" "$>" "++" "+++" "+>" "=:=" "==" + "===" "==>" "=>" "=>>" "<=" "=<<" "=/=" ">-" ">=" + ">=>" ">>" ">>-" ">>=" ">>>" "<*" "<*>" "<|" "<|>" + "<$" "<$>" "<!--" "<-" "<--" "<->" "<+" "<+>" "<=" + "<==" "<=>" "<=<" "<>" "<<" "<<-" "<<=" "<<<" "<~" + "<~~" "</" "</>" "~@" "~-" "~=" "~>" "~~" "~~>" "%%" + "x" ":" "+" "+" "*"))) + (my-correct-symbol-bounds (my-ligature-list ligs #Xe100)))) +``` diff --git a/users/Profpatsch/cdb.nix b/users/Profpatsch/cdb.nix new file mode 100644 index 000000000000..8cfaa3ea7ac6 --- /dev/null +++ b/users/Profpatsch/cdb.nix @@ -0,0 +1,91 @@ +{ depot, pkgs, ... }: + +let + cdbListToNetencode = depot.nix.writers.rustSimple { + name = "cdb-list-to-netencode"; + dependencies = [ + depot.third_party.rust-crates.nom + depot.users.Profpatsch.execline.exec-helpers + depot.users.Profpatsch.netencode.netencode-rs + ]; + } '' + extern crate nom; + extern crate exec_helpers; + extern crate netencode; + use std::collections::HashMap; + use std::io::BufRead; + use nom::{IResult}; + use nom::sequence::{tuple}; + use nom::bytes::complete::{tag, take}; + use nom::character::complete::{digit1, char}; + use nom::error::{context, ErrorKind, ParseError}; + use nom::combinator::{map_res}; + use netencode::{T, Tag}; + + fn usize_t(s: &[u8]) -> IResult<&[u8], usize> { + context( + "usize", + map_res( + map_res(digit1, |n| std::str::from_utf8(n)), + |s| s.parse::<usize>()) + )(s) + } + + fn parse_cdb_record(s: &[u8]) -> IResult<&[u8], (&[u8], &[u8])> { + let (s, (_, klen, _, vlen, _)) = tuple(( + char('+'), + usize_t, + char(','), + usize_t, + char(':') + ))(s)?; + let (s, (key, _, val)) = tuple(( + take(klen), + tag("->"), + take(vlen), + ))(s)?; + Ok((s, (key, val))) + } + + fn main() { + let mut res = vec![]; + let stdin = std::io::stdin(); + let mut lines = stdin.lock().split(b'\n'); + loop { + match lines.next() { + None => exec_helpers::die_user_error("cdb-list-to-netencode", "stdin ended but we didn’t receive the empty line to signify the end of the cdbdump input!"), + Some(Err(err)) => exec_helpers::die_temporary("cdb-list-to-netencode", format!("could not read from stdin: {}", err)), + Some(Ok(line)) => + if &line == b"" { + // the cdbdump input ends after an empty line (double \n) + break; + } else { + match parse_cdb_record(&line) { + Ok((b"", (key, val))) => { + let (key, val) = match + std::str::from_utf8(key) + .and_then(|k| std::str::from_utf8(val).map(|v| (k, v))) { + Ok((key, val)) => (key.to_owned(), val.to_owned()), + Err(err) => exec_helpers::die_user_error("cdb-list-to-netencode", format!("cannot decode line {:?}, we only support utf8-encoded key/values pairs for now: {}", String::from_utf8_lossy(&line), err)), + }; + let _ = res.push((key, val)); + }, + Ok((rest, _)) => exec_helpers::die_user_error("cdb-list-to-netencode", format!("could not decode record line {:?}, had some trailing bytes", String::from_utf8_lossy(&line))), + Err(err) => exec_helpers::die_user_error("cdb-list-to-netencode", format!("could not decode record line {:?}: {:?}", String::from_utf8_lossy(&line), err)), + } + } + } + } + let list = T::List(res.into_iter().map( + |(k, v)| T::Record(vec![(String::from("key"), T::Text(k)), (String::from("val"), T::Text(v))].into_iter().collect()) + ).collect()); + netencode::encode(&mut std::io::stdout(), &list.to_u()); + } + + ''; + +in { + inherit + cdbListToNetencode + ; +} diff --git a/users/Profpatsch/emacs-tree-sitter-move/default.nix b/users/Profpatsch/emacs-tree-sitter-move/default.nix new file mode 100644 index 000000000000..fdc059c089b6 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/default.nix @@ -0,0 +1,3 @@ +# nothing yet (TODO: expose shell & tool) +{...}: +{} diff --git a/users/Profpatsch/emacs-tree-sitter-move/shell.nix b/users/Profpatsch/emacs-tree-sitter-move/shell.nix new file mode 100644 index 000000000000..81d622ac73e5 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/shell.nix @@ -0,0 +1,16 @@ +{ pkgs ? import ../../../third_party {}, ... }: +let + inherit (pkgs) lib; + + treeSitterGrammars = pkgs.runCommandLocal "grammars" {} '' + mkdir -p $out/bin + ${lib.concatStringsSep "\n" + (lib.mapAttrsToList (name: src: "ln -s ${src}/parser $out/bin/${name}.so") pkgs.tree-sitter.builtGrammars)}; + ''; + +in pkgs.mkShell { + buildInputs = [ + pkgs.tree-sitter.builtGrammars.python + ]; + TREE_SITTER_GRAMMAR_DIR = treeSitterGrammars; +} diff --git a/users/Profpatsch/emacs-tree-sitter-move/test.json b/users/Profpatsch/emacs-tree-sitter-move/test.json new file mode 100644 index 000000000000..d9f8075976d6 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/test.json @@ -0,0 +1,14 @@ +{ + "foo": { + "x": [ 1, 2, 3, 4 ], + "bar": "test" + }, + "foo": { + "x": [ 1, 2, 3, 4 ], + "bar": "test" + }, + "foo": { + "x": [ 1, 2, 3, 4 ], + "bar": "test" + } +} diff --git a/users/Profpatsch/emacs-tree-sitter-move/test.py b/users/Profpatsch/emacs-tree-sitter-move/test.py new file mode 100644 index 000000000000..0f57bae035da --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/test.py @@ -0,0 +1,13 @@ +(4 + 5 + 5) + +def foo(a, b, c) + +def bar(a, b): + 4 + 4 + 4 + +[1, 4, 5, 10] + +def foo(): + pass diff --git a/users/Profpatsch/emacs-tree-sitter-move/test.sh b/users/Profpatsch/emacs-tree-sitter-move/test.sh new file mode 100644 index 000000000000..681081f5909d --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/test.sh @@ -0,0 +1,14 @@ +function foo () { + local x=123 +} + +function bar () { + local x=123 +} + +echo abc def \ + gef gef + +printf \ + "%s\n" \ + haha diff --git a/users/Profpatsch/emacs-tree-sitter-move/tmp.el b/users/Profpatsch/emacs-tree-sitter-move/tmp.el new file mode 100644 index 000000000000..88d13fa45b81 --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/tmp.el @@ -0,0 +1,28 @@ +(defun tree-sitter-load-from-grammar-dir (grammar-dir sym lang-name) + (tree-sitter-load + sym + (format "%s/bin/%s" + (getenv grammar-dir) + lang-name))) + +(defun tree-sitter-init-tmp-langs (alist) + (mapcar + (lambda (lang) + (pcase-let ((`(,name ,sym ,mode) lang)) + (tree-sitter-load-from-grammar-dir "TREE_SITTER_GRAMMAR_DIR" sym name) + (cons mode sym))) + alist)) + + +(setq tree-sitter-major-mode-language-alist + (tree-sitter-init-tmp-langs + '(("python" python python-mode) + ("json" json js-mode) + ("bash" bash sh-mode) + ))) + +(define-key evil-normal-state-map (kbd "C-.") #'tree-sitter-move-reset) +(define-key evil-normal-state-map (kbd "C-<right>") #'tree-sitter-move-right) +(define-key evil-normal-state-map (kbd "C-<left>") #'tree-sitter-move-left) +(define-key evil-normal-state-map (kbd "C-<up>") #'tree-sitter-move-up) +(define-key evil-normal-state-map (kbd "C-<down>") #'tree-sitter-move-down) diff --git a/users/Profpatsch/emacs-tree-sitter-move/tree-sitter-move.el b/users/Profpatsch/emacs-tree-sitter-move/tree-sitter-move.el new file mode 100644 index 000000000000..907e1e4081bc --- /dev/null +++ b/users/Profpatsch/emacs-tree-sitter-move/tree-sitter-move.el @@ -0,0 +1,139 @@ +;; this is not an actual cursor, just a node. +;; It’s not super efficient, but cursors can’t be *set* to an arbitrary +;; subnode, because they can’t access the parent otherwise. +;; We’d need a way to reset the cursor and walk down to the node?! +(defvar-local tree-sitter-move--cursor nil + "the buffer-local cursor used for movement") + +(defvar-local tree-sitter-move--debug-overlay nil + "an overlay used to visually display the region currently marked by the cursor") + +;;;;; TODO: should everything use named nodes? Only some things? +;;;;; maybe there should be a pair of functions for everything? +;;;;; For now restrict to named nodes. + +(defun tree-sitter-move--setup () + ;; TODO + (progn + ;; TODO: if tree-sitter-mode fails to load, display a better error + (tree-sitter-mode t) + (setq tree-sitter-move--cursor (tsc-root-node tree-sitter-tree)) + (add-variable-watcher + 'tree-sitter-move--cursor + #'tree-sitter-move--debug-overlay-update))) + +(defun tree-sitter-move--debug-overlay-update (sym newval &rest _args) + "variable-watcher to update the debug overlay when the cursor changes" + (let ((start (tsc-node-start-position newval)) + (end (tsc-node-end-position newval))) + (symbol-macrolet ((o tree-sitter-move--debug-overlay)) + (if o + (move-overlay o start end) + (setq o (make-overlay start end)) + (overlay-put o 'face 'highlight) + )))) + +(defun tree-sitter-move--debug-overlay-teardown () + "Turn of the overlay visibility and delete the overlay object" + (when tree-sitter-move--debug-overlay + (delete-overlay tree-sitter-move--debug-overlay) + (setq tree-sitter-move--debug-overlay nil))) + +(defun tree-sitter-move--teardown () + (setq tree-sitter-move--cursor nil) + (tree-sitter-move--debug-overlay-teardown) + (tree-sitter-mode nil)) + +;; Get the syntax node the cursor is on. +(defun tsc-get-named-node-at-point () + (let ((p (point))) + (tsc-get-named-descendant-for-position-range + (tsc-root-node tree-sitter-tree) p p))) + +;; TODO: is this function necessary? +;; Maybe tree-sitter always guarantees that parents are named? +(defun tsc-get-named-parent (node) + (when-let ((parent (tsc-get-parent node))) + (while (and parent (not (tsc-node-named-p parent))) + (setq parent (tsc-get-parent parent))) + parent)) + +(defun tsc-get-first-named-node-with-siblings-up (node) + "Returns the first 'upwards' node that has siblings. That includes the current + node, so if the given node has siblings, it is returned. Returns nil if there + is no such node until the root" + (when-let ((has-siblings-p + (lambda (parent-node) + (> (tsc-count-named-children parent-node) + 1))) + (cur node) + (parent (tsc-get-named-parent node))) + (while (and parent (not (funcall has-siblings-p parent))) + (setq cur parent) + (setq parent (tsc-get-named-parent cur))) + cur)) + +(defun tree-sitter-move--set-cursor-to-node (node) + (setq tree-sitter-move--cursor node)) + +(defun tree-sitter-move--set-cursor-to-node-at-point () + (tree-sitter-move--set-cursor-to-node (tsc-get-named-node-at-point))) + +(defun tree-sitter-move--move-point-to-node (node) + (set-window-point + (selected-window) + (tsc-node-start-position node))) + + +;; interactive commands (“do what I expect” section) + +(defun tree-sitter-move-reset () + (interactive) + (tree-sitter-move--set-cursor-to-node-at-point)) + +(defun tree-sitter-move-right () + (interactive) + (tree-sitter-move--move-skip-non-sibling-nodes 'tsc-get-next-named-sibling)) + +(defun tree-sitter-move-left () + (interactive) + (tree-sitter-move--move-skip-non-sibling-nodes 'tsc-get-prev-named-sibling)) + +(defun tree-sitter-move-up () + (interactive) + (tree-sitter-move--move-skip-non-sibling-nodes 'tsc-get-parent)) + +;; TODO: does not skip siblings yet, because the skip function only goes up (not down) +(defun tree-sitter-move-down () + (interactive) + (tree-sitter-move--move-if-possible (lambda (n) (tsc-get-nth-named-child n 0)))) + +(defun tree-sitter-move--move-skip-non-sibling-nodes (move-fn) + "Moves to the sidewards next sibling. If the current node does not have siblings, go + upwards until something has siblings and then move to the side (right or left)." + (tree-sitter-move--move-if-possible + (lambda (cur) + (when-let ((with-siblings + (tsc-get-first-named-node-with-siblings-up cur))) + (funcall move-fn with-siblings))))) + +(defun tree-sitter-move--move-if-possible (dir-fn) + (let ((next (funcall dir-fn tree-sitter-move--cursor))) + (when next + (tree-sitter-move--set-cursor-to-node next) + (tree-sitter-move--move-point-to-node next)))) + +; mostly stolen from tree-sitter-mode +;;;###autoload +(define-minor-mode tree-sitter-move-mode + "Minor mode to do cursor movements via tree-sitter" + :init-value nil + :lighter " tree-sitter-move" + (if tree-sitter-move-mode + (tree-sitter--error-protect + (progn + (tree-sitter-move--setup)) + (setq tree-sitter-move-mode nil) + (tree-sitter-move--teardown)) + (lambda ()) + (tree-sitter-move--teardown))) diff --git a/users/Profpatsch/execline/default.nix b/users/Profpatsch/execline/default.nix new file mode 100644 index 000000000000..1f75b97591fc --- /dev/null +++ b/users/Profpatsch/execline/default.nix @@ -0,0 +1,12 @@ +{ depot, pkgs, lib, ... }: + +let + exec-helpers = depot.nix.writers.rustSimpleLib { + name = "exec-helpers"; + } (builtins.readFile ./exec_helpers.rs); + +in depot.nix.readTree.drvTargets { + inherit + exec-helpers + ; +} diff --git a/users/Profpatsch/execline/exec_helpers.rs b/users/Profpatsch/execline/exec_helpers.rs new file mode 100644 index 000000000000..b9e1f5797386 --- /dev/null +++ b/users/Profpatsch/execline/exec_helpers.rs @@ -0,0 +1,113 @@ +use std::os::unix::process::CommandExt; +use std::ffi::OsStr; +use std::os::unix::ffi::{OsStringExt, OsStrExt}; + +pub fn no_args(current_prog_name: &str) -> () { + let mut args = std::env::args_os(); + // remove argv[0] + let _ = args.nth(0); + if args.len() > 0 { + die_user_error(current_prog_name, format!("Expected no arguments, got {:?}", args.collect::<Vec<_>>())) + } +} + +pub fn args(current_prog_name: &str, no_of_positional_args: usize) -> Vec<Vec<u8>> { + let mut args = std::env::args_os(); + // remove argv[0] + let _ = args.nth(0); + if args.len() != no_of_positional_args { + die_user_error(current_prog_name, format!("Expected {} arguments, got {}, namely {:?}", no_of_positional_args, args.len(), args.collect::<Vec<_>>())) + } + args.map(|arg| arg.into_vec()).collect() +} + +pub fn args_for_exec(current_prog_name: &str, no_of_positional_args: usize) -> (Vec<Vec<u8>>, Vec<Vec<u8>>) { + let mut args = std::env::args_os(); + // remove argv[0] + let _ = args.nth(0); + let mut args = args.map(|arg| arg.into_vec()); + let mut pos_args = vec![]; + // get positional args + for i in 1..no_of_positional_args+1 { + pos_args.push( + args.nth(0).expect( + &format!("{}: expects {} positional args, only got {}", current_prog_name, no_of_positional_args, i)) + ); + } + // prog... is the rest of the iterator + let prog : Vec<Vec<u8>> = args.collect(); + (pos_args, prog) +} + +pub fn exec_into_args<'a, 'b, Args, Arg, Env, Key, Val>(current_prog_name: &str, args: Args, env_additions: Env) -> ! + where + Args: IntoIterator<Item = Arg>, + Arg: AsRef<[u8]>, + Env: IntoIterator<Item = (Key, Val)>, + Key: AsRef<[u8]>, + Val: AsRef<[u8]>, +{ + // TODO: is this possible without collecting into a Vec first, just leaving it an IntoIterator? + let args = args.into_iter().collect::<Vec<Arg>>(); + let mut args = args.iter().map(|v| OsStr::from_bytes(v.as_ref())); + let prog = args.nth(0).expect(&format!("{}: first argument must be an executable", current_prog_name)); + // TODO: same here + let env = env_additions.into_iter().collect::<Vec<(Key, Val)>>(); + let env = env.iter().map(|(k,v)| (OsStr::from_bytes(k.as_ref()), OsStr::from_bytes(v.as_ref()))); + let err = std::process::Command::new(prog).args(args).envs(env).exec(); + die_missing_executable(current_prog_name, format!("exec failed: {}, while trying to execing into {:?}", err, prog)); +} + +/// Exit 1 to signify a generic expected error +/// (e.g. something that sometimes just goes wrong, like a nix build). +pub fn die_expected_error<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(1, current_prog_name, msg) +} + +/// Exit 100 to signify a user error (“the user is holding it wrong”). +/// This is a permanent error, if the program is executed the same way +/// it should crash with 100 again. +pub fn die_user_error<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(100, current_prog_name, msg) +} + +/// Exit 101 to signify an unexpected crash (failing assertion or panic). +/// This is the same exit code that `panic!()` emits. +pub fn die_panic<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(101, current_prog_name, msg) +} + +/// Exit 111 to signify a temporary error (such as resource exhaustion) +pub fn die_temporary<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(111, current_prog_name, msg) +} + +/// Exit 126 to signify an environment problem +/// (the user has set up stuff incorrectly so the program cannot work) +pub fn die_environment_problem<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(126, current_prog_name, msg) +} + +/// Exit 127 to signify a missing executable. +pub fn die_missing_executable<S>(current_prog_name: &str, msg: S) -> ! +where S: AsRef<str> +{ + die_with(127, current_prog_name, msg) +} + +fn die_with<S>(status: i32, current_prog_name: &str, msg: S) -> ! + where S: AsRef<str> +{ + eprintln!("{}: {}", current_prog_name, msg.as_ref()); + std::process::exit(status) +} diff --git a/users/Profpatsch/imap-idle.nix b/users/Profpatsch/imap-idle.nix new file mode 100644 index 000000000000..3ad5375d89af --- /dev/null +++ b/users/Profpatsch/imap-idle.nix @@ -0,0 +1,14 @@ +{ depot, pkgs, lib, ... }: + +let + imap-idle = depot.nix.writers.rustSimple { + name = "imap-idle"; + dependencies = [ + depot.users.Profpatsch.arglib.netencode.rust + depot.third_party.rust-crates.imap + depot.third_party.rust-crates.epoll + depot.users.Profpatsch.execline.exec-helpers + ]; + } (builtins.readFile ./imap-idle.rs); + +in imap-idle diff --git a/users/Profpatsch/imap-idle.rs b/users/Profpatsch/imap-idle.rs new file mode 100644 index 000000000000..9dce736d0d8a --- /dev/null +++ b/users/Profpatsch/imap-idle.rs @@ -0,0 +1,132 @@ +extern crate exec_helpers; +// extern crate arglib_netencode; +// extern crate netencode; +extern crate imap; +extern crate epoll; + +// use netencode::dec; +use std::convert::TryFrom; +use std::io::{Read, Write}; +use std::fs::File; +use std::os::unix::io::{FromRawFd, AsRawFd, RawFd}; +use std::time::Duration; +use imap::extensions::idle::SetReadTimeout; + +/// Implements an UCSPI client that wraps fd 6 & 7 +/// and implements Write and Read with a timeout. +/// See https://cr.yp.to/proto/ucspi.txt +#[derive(Debug)] +struct UcspiClient { + read: File, + read_epoll_fd: RawFd, + read_timeout: Option<Duration>, + write: File, +} + +impl UcspiClient { + /// Use fd 6 and 7 to connect to the net, as is specified. + /// Unsafe because fd 6 and 7 are global resources and we don’t mutex them. + pub unsafe fn new_from_6_and_7() -> std::io::Result<Self> { + unsafe { + let read_epoll_fd = epoll::create(false)?; + Ok(UcspiClient { + read: File::from_raw_fd(6), + read_epoll_fd, + read_timeout: None, + write: File::from_raw_fd(7) + }) + } + } +} + +/// Emulates set_read_timeout() like on a TCP socket with an epoll on read. +/// The BSD socket API is rather bad, so fd != fd, +/// and if we cast the `UcspiClient` fds to `TcpStream` instead of `File`, +/// we’d break any UCSPI client programs that *don’t* connect to TCP. +/// Instead we use the (linux) `epoll` API in read to wait on the timeout. +impl SetReadTimeout for UcspiClient { + fn set_read_timeout(&mut self, timeout: Option<Duration>) -> imap::Result<()> { + self.read_timeout = timeout; + Ok(()) + } +} + +impl Read for UcspiClient { + // TODO: test the epoll code with a short timeout + fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> { + const NO_DATA : u64 = 0; + // in order to implement the read_timeout, + // we use epoll to wait for either data or time out + epoll::ctl( + self.read_epoll_fd, + epoll::ControlOptions::EPOLL_CTL_ADD, + self.read.as_raw_fd(), + epoll::Event::new(epoll::Events::EPOLLIN, NO_DATA) + )?; + let UNUSED = epoll::Event::new(epoll::Events::EPOLLIN, NO_DATA); + let wait = epoll::wait( + self.read_epoll_fd, + match self.read_timeout { + Some(duration) => i32::try_from(duration.as_millis()).expect("duration too big for epoll"), + None => -1 // infinite + }, + // event that was generated; but we don’t care + &mut vec![UNUSED; 1][..], + ); + // Delete the listen fd from the epoll fd before reacting + // (otherwise it fails on the next read with `EPOLL_CTL_ADD`) + epoll::ctl( + self.read_epoll_fd, + epoll::ControlOptions::EPOLL_CTL_DEL, + self.read.as_raw_fd(), + UNUSED + )?; + match wait { + // timeout happened (0 events) + Ok(0) => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, "ucspi read timeout")), + // its ready for reading, we can read + Ok(_) => self.read.read(buf), + // error + err => err, + } + } +} + +/// Just proxy through the `Write` of the write fd. +impl Write for UcspiClient { + fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> { + self.write.write(buf) + } + fn flush(&mut self) -> std::io::Result<()> { + self.write.flush() + } +} + +/// Connect to IMAP account and listen for new mails on the INBOX. +fn main() { + exec_helpers::no_args("imap-idle"); + + // TODO: use arglib_netencode + let username = std::env::var("IMAP_USERNAME").expect("username"); + let password = std::env::var("IMAP_PASSWORD").expect("password"); + + let net = unsafe { + UcspiClient::new_from_6_and_7().expect("no ucspi client for you") + }; + let client = imap::Client::new(net); + let mut session = client.login(username, password).map_err(|(err, _)| err).expect("unable to login"); + eprintln!("{:#?}", session); + let list = session.list(None, Some("*")); + eprintln!("{:#?}", list); + let mailbox = session.examine("INBOX"); + eprintln!("{:#?}", mailbox); + fn now() -> String { + String::from_utf8_lossy(&std::process::Command::new("date").output().unwrap().stdout).trim_right().to_string() + } + loop { + eprintln!("{}: idling on INBOX", now()); + let mut handle = session.idle().expect("cannot idle on INBOX"); + let () = handle.wait_keepalive().expect("waiting on idle failed"); + eprintln!("{}: The mailbox has changed!", now()); + } +} diff --git a/users/Profpatsch/lens.nix b/users/Profpatsch/lens.nix new file mode 100644 index 000000000000..58d9c27f5242 --- /dev/null +++ b/users/Profpatsch/lens.nix @@ -0,0 +1,136 @@ +{ ... }: +let + id = x: x; + + const = x: y: x; + + comp = f: g: x: f (g x); + + _ = v: f: f v; + + # Profunctor (p :: Type -> Type -> Type) + Profunctor = rec { + # dimap :: (a -> b) -> (c -> d) -> p b c -> p a d + dimap = f: g: x: lmap f (rmap g x); + # lmap :: (a -> b) -> p b c -> p a c + lmap = f: dimap f id; + # rmap :: (c -> d) -> p b c -> p b d + rmap = g: dimap id g; + }; + + # Profunctor (->) + profunctorFun = Profunctor // { + # dimap :: (a -> b) -> (c -> d) -> (b -> c) -> a -> d + dimap = ab: cd: bc: a: cd (bc (ab a)); + # lmap :: (a -> b) -> (b -> c) -> (a -> c) + lmap = ab: bc: a: bc (ab a); + # rmap :: (c -> d) -> (b -> c) -> (b -> d) + rmap = cd: bc: b: cd (bc b); + }; + + tuple = fst: snd: { + inherit fst snd; + }; + + swap = {fst, snd}: { + fst = snd; + snd = fst; + }; + + # Profunctor p => Strong (p :: Type -> Type -> Type) + Strong = pro: pro // rec { + # firstP :: p a b -> p (a, c) (b, c) + firstP = pab: pro.dimap swap swap (pro.secondP pab); + # secondP :: p a b -> p (c, a) (c, b) + secondP = pab: pro.dimap swap swap (pro.firstP pab); + }; + + # Strong (->) + strongFun = Strong profunctorFun // { + # firstP :: (a -> b) -> (a, c) -> (b, c) + firstP = f: { fst, snd }: { fst = f fst; inherit snd; }; + # secondP :: (a -> b) -> (c, a) -> (c, b) + secondP = f: { snd, fst }: { snd = f snd; inherit fst; }; + }; + + # Iso s t a b :: forall p. Profunctor p -> p a b -> p s t + + # iso :: (s -> a) -> (b -> t) -> Iso s t a b + iso = pro: pro.dimap; + + # Lens s t a b :: forall p. Strong p -> p a b -> p s t + + # lens :: (s -> a) -> (s -> b -> t) -> Lens s t a b + lens = strong: get: set: pab: + lensP + strong + (s: tuple (get s) (b: set s b)) + pab; + + # lensP :: (s -> (a, b -> t)) -> Lens s t a b + lensP = strong: to: pab: + strong.dimap + to + ({fst,snd}: snd fst) + (strong.firstP pab); + + # first element of a tuple + # _1 :: Lens (a, c) (b, c) a b + _1 = strong: strong.firstP; + + # second element of a tuple + # _2 :: Lens (c, a) (c, b) a b + _2 = strong: strong.secondP; + + # a the given field in the record + # field :: (f :: String) -> Lens { f :: a; ... } { f :: b; ... } a b + field = name: strong: + lens + strong + (attrs: attrs.${name}) + (attrs: a: attrs // { ${name} = a; }); + + # Setter :: (->) a b -> (->) s t + # Setter :: (a -> b) -> (s -> t) + + + # Subclasses of profunctor for (->). + # We only have Strong for now, but when we implement Choice we need to add it here. + profunctorSubclassesFun = strongFun; + + # over :: Setter s t a b -> (a -> b) -> s -> t + over = setter: + # A setter needs to be instanced to the profunctor-subclass instances of (->). + (setter profunctorSubclassesFun); + + # set :: Setter s t a b -> b -> s -> t + set = setter: b: over setter (const b); + + # combine a bunch of optics, for the subclass instance of profunctor you give it. + optic = accessors: profunctorSubclass: + builtins.foldl' comp id + (map (accessor: accessor profunctorSubclass) accessors); + + +in { + inherit + id + _ + const + comp + Profunctor + profunctorFun + Strong + strongFun + iso + lens + optic + _1 + _2 + field + tuple + swap + over + set + ; +} diff --git a/users/Profpatsch/lib.nix b/users/Profpatsch/lib.nix new file mode 100644 index 000000000000..e3d59b7d8beb --- /dev/null +++ b/users/Profpatsch/lib.nix @@ -0,0 +1,81 @@ +{ depot, pkgs, ... }: +let + bins = depot.nix.getBins pkgs.coreutils [ "printf" "echo" "cat" "printenv" "tee" ] + // depot.nix.getBins pkgs.bash [ "bash" ] + // depot.nix.getBins pkgs.fdtools [ "multitee" ] + ; + + # Print `msg` and and argv to stderr, then execute into argv + debugExec = msg: depot.nix.writeExecline "debug-exec" {} [ + "if" [ + "fdmove" "-c" "1" "2" + "if" [ bins.printf "%s: " msg ] + "if" [ bins.echo "$@" ] + ] + "$@" + ]; + + # Print stdin to stderr and stdout + eprint-stdin = depot.nix.writeExecline "eprint-stdin" {} [ + "pipeline" [ bins.multitee "0-1,2" ] "$@" + ]; + + # Assume the input on stdin is netencode, pretty print it to stderr and forward it to stdout + eprint-stdin-netencode = depot.nix.writeExecline "eprint-stdin-netencode" {} [ + "pipeline" [ + # move stdout to 3 + "fdmove" "3" "1" + # the multitee copies stdin to 1 (the other pipeline end) and 3 (the stdout of the outer pipeline block) + "pipeline" [ bins.multitee "0-1,3" ] + # make stderr the stdout of pretty, merging with the stderr of pretty + "fdmove" "-c" "1" "2" + depot.users.Profpatsch.netencode.pretty + ] + "$@" + ]; + + # print the given environment variable in $1 to stderr, then execute into the rest of argv + eprintenv = depot.nix.writeExecline "eprintenv" { readNArgs = 1; } [ + "ifelse" [ "fdmove" "-c" "1" "2" bins.printenv "$1" ] + [ "$@" ] + "if" [ depot.tools.eprintf "eprintenv: could not find \"\${1}\" in the environment\n" ] + "$@" + ]; + + # Split stdin into two commands, given by a block and the rest of argv + # + # Example (execline): + # + # pipeline [ echo foo ] + # split-stdin [ fdmove 1 2 foreground [ cat ] echo "bar" ] cat + # + # stdout: foo\n + # stderr: foo\nbar\n + split-stdin = depot.nix.writeExecline "split-stdin" { argMode = "env"; } [ + "pipeline" [ + # this is horrible yes but the quickest way I knew how to implement it + "runblock" "1" bins.bash "-c" ''${bins.tee} >("$@")'' "bash-split-stdin" + ] + "runblock" "-r" "1" + ]; + + # remove everything but a few selected environment variables + runInEmptyEnv = keepVars: + let + importas = pkgs.lib.concatMap (var: [ "importas" "-i" var var ]) keepVars; + # we have to explicitely call export here, because PATH is probably empty + export = pkgs.lib.concatMap (var: [ "${pkgs.execline}/bin/export" var ''''${${var}}'' ]) keepVars; + in depot.nix.writeExecline "empty-env" {} + (importas ++ [ "emptyenv" ] ++ export ++ [ "${pkgs.execline}/bin/exec" "$@" ]); + + +in { + inherit + debugExec + eprint-stdin + eprint-stdin-netencode + eprintenv + split-stdin + runInEmptyEnv + ; +} diff --git a/users/Profpatsch/netencode/README.md b/users/Profpatsch/netencode/README.md new file mode 100644 index 000000000000..67cb843a58c7 --- /dev/null +++ b/users/Profpatsch/netencode/README.md @@ -0,0 +1,111 @@ +# netencode 0.1-unreleased + +[bencode][] and [netstring][]-inspired pipe format that should be trivial to generate correctly in every context (only requires a `byte_length()` and a `printf()`), easy to parse (100 lines of code or less), mostly human-decipherable for easy debugging, and support nested record and sum types. + + +## scalars + +Scalars have the format `[type prefix][size]:[value],`. + +where size is a natural number without leading zeroes. + +### unit + +The unit (`u`) has only one value. + +* The unit is: `u,` + +### numbers + +Naturals (`n`) and Integers (`i`), with a maximum size in bits. + +Bit sizes are specified in 2^n increments, 1 to 9 (`n1`..`n9`, `i1`..`n9`). + +* Natural `1234` that fits in 32 bits (2^5): `n5:1234,` +* Integer `-42` that fits in 8 bits (2^3): `i3:-42,` +* Integer `23` that fits in 64 bits (2^6): `i6:23,` +* Integer `-1` that fits in 512 bits (2^9): `i9:-1,` +* Natural `0` that fits in 1 bit (2^1): `n1:0,` + +An implementation can define the biggest numbers it supports, and has to throw an error for anything bigger. It has to support everything smaller, so for example if you support up to i6/n6, you have to support 1–6 as well. An implementation could support up to the current architecture’s wordsize for example. + +Floats are not supported, you can implement fixed-size decimals or ratios using integers. + +### booleans + +A boolean is represented as `n1`. + +* `n1:0,`: false +* `n1:1,`: true + +TODO: should we add `f,` and `t,`? + +### text + +Text (`t`) that *must* be encoded as UTF-8, starting with its length in bytes: + +* The string `hello world` (11 bytes): `t11:hello world,` +* The string `今日は` (9 bytes): `t9:今日は,` +* The string `:,` (2 bytes): `t2::,,` +* The empty sting `` (0 bytes): `t0:,` + +### binary + +Arbitrary binary strings (`b`) that can contain any data, starting with its length in bytes. + +* The ASCII string `hello world` as binary data (11 bytes): `b11:hello world,` +* The empty binary string (0 bytes): `b0:,` +* The bytestring with `^D` (1 byte): `b1:,` + +Since the binary strings are length-prefixd, they can contain `\0` and no escaping is required. Care has to be taken in languages with `\0`-terminated bytestrings. + +Use text (`t`) if you have utf-8 encoded data. + +## tagged values + +### tags + +A tag (`<`) gives a value a name. The tag is UTF-8 encoded, starting with its length in bytes and proceeding with the value. + +* The tag `foo` (3 bytes) tagging the text `hello` (5 bytes): `<3:foo|t5:hello,` +* The tag `` (0 bytes) tagging the 8-bit integer 0: `<0:|i3:0,` + +### records (products/records), also maps + +A record (`{`) is a concatenation of tags (`<`). It needs to be closed with `}`. +If tag names repeat the later ones should be ignored. Ordering does not matter. + +Similar to text, records start with the length of their *whole encoded content*, in bytes. This makes it possible to treat their contents as opaque bytestrings. + +* There is no empty record. (TODO: make the empty record the unit type, remove `u,`?) +* A record with one empty field, `foo`: `{9:<3:foo|u,}` +* A record with two fields, `foo` and `x`: `{21:<3:foo|u,<1:x|t3:baz,}` +* The same record: `{21:<1:x|t3:baz,<3:foo|u,}` +* The same record (later occurences of fields are ignored): `{28:<1:x|t3:baz,<3:foo|u,<1:x|u,}` + +### sums (tagged unions) + +Simply a tagged value. The tag marker `<` indicates it is a sum if it appears outside of a record. + +## lists + +A list (`[`) imposes an ordering on a sequence of values. It needs to be closed with `]`. Values in it are simply concatenated. + +Similar to records, lists start with the length of their whole encoded content. + +* The empty list: `[0:]` +* The list with one element, the string `foo`: `[7:t3:foo,]` +* The list with text `foo` followed by i3 `-42`: `[14:t3:foo,i3:-42,]` +* The list with `Some` and `None` tags: `[33:<4:Some|t3:foo,<4None|u,<4None|u,]` + +## motivation + +TODO + +## guarantees + +TODO: do I want unique representation (bijection like bencode?) This would put more restrictions on the generator, like sorting records in lexicographic order, but would make it possible to compare without decoding + + +[bencode]: https://en.wikipedia.org/wiki/Bencode +[netstring]: https://en.wikipedia.org/wiki/Netstring diff --git a/users/Profpatsch/netencode/default.nix b/users/Profpatsch/netencode/default.nix new file mode 100644 index 000000000000..739bda3d78c9 --- /dev/null +++ b/users/Profpatsch/netencode/default.nix @@ -0,0 +1,149 @@ +{ depot, pkgs, lib, ... }: + +let + netencode-rs = depot.nix.writers.rustSimpleLib { + name = "netencode"; + dependencies = [ + depot.third_party.rust-crates.nom + depot.users.Profpatsch.execline.exec-helpers + ]; + } (builtins.readFile ./netencode.rs); + + gen = import ./gen.nix { inherit lib; }; + + pretty-rs = depot.nix.writers.rustSimpleLib { + name = "netencode-pretty"; + dependencies = [ + netencode-rs + ]; + } (builtins.readFile ./pretty.rs); + + pretty = depot.nix.writers.rustSimple { + name = "netencode-pretty"; + dependencies = [ + netencode-rs + pretty-rs + depot.users.Profpatsch.execline.exec-helpers + ]; + } '' + extern crate netencode; + extern crate netencode_pretty; + extern crate exec_helpers; + + fn main() { + let (_, prog) = exec_helpers::args_for_exec("netencode-pretty", 0); + let mut buf = vec![]; + let u = netencode::u_from_stdin_or_die_user_error("netencode-pretty", &mut buf); + match netencode_pretty::Pretty::from_u(u).print_multiline(&mut std::io::stdout()) { + Ok(()) => {}, + Err(err) => exec_helpers::die_temporary("netencode-pretty", format!("could not write to stdout: {}", err)) + } + } + ''; + + netencode-mustache = depot.nix.writers.rustSimple { + name = "netencode_mustache"; + dependencies = [ + depot.users.Profpatsch.arglib.netencode.rust + netencode-rs + depot.third_party.rust-crates.mustache + ]; + } (builtins.readFile ./netencode-mustache.rs); + + + record-get = depot.nix.writers.rustSimple { + name = "record-get"; + dependencies = [ + netencode-rs + depot.users.Profpatsch.execline.exec-helpers + depot.users.Profpatsch.arglib.netencode.rust + ]; + } '' + extern crate netencode; + extern crate arglib_netencode; + extern crate exec_helpers; + use netencode::{encode, dec}; + use netencode::dec::{Decoder, DecodeError}; + + fn main() { + let mut buf = vec![]; + let args = exec_helpers::args("record-get", 1); + let field = match std::str::from_utf8(&args[0]) { + Ok(f) => f, + Err(_e) => exec_helpers::die_user_error("record-get", format!("The field name needs to be valid unicode")) + }; + let u = netencode::u_from_stdin_or_die_user_error("record-get", &mut buf); + match (dec::RecordDot {field, inner: dec::AnyU }).dec(u) { + Ok(u) => encode(&mut std::io::stdout(), &u).expect("encoding to stdout failed"), + Err(DecodeError(err)) => exec_helpers::die_user_error("record-get", err) + } + } + ''; + + record-splice-env = depot.nix.writers.rustSimple { + name = "record-splice-env"; + dependencies = [ + netencode-rs + depot.users.Profpatsch.execline.exec-helpers + ]; + } '' + extern crate netencode; + extern crate exec_helpers; + use netencode::dec::{Record, Try, ScalarAsBytes, Decoder, DecodeError}; + + fn main() { + let mut buf = vec![]; + let u = netencode::u_from_stdin_or_die_user_error("record-splice-env", &mut buf); + let (_, prog) = exec_helpers::args_for_exec("record-splice-env", 0); + match Record(Try(ScalarAsBytes)).dec(u) { + Ok(map) => { + exec_helpers::exec_into_args( + "record-splice-env", + prog, + // some elements can’t be decoded as scalars, so just ignore them + map.into_iter().filter_map(|(k, v)| v.map(|v2| (k, v2))) + ); + }, + Err(DecodeError(err)) => exec_helpers::die_user_error("record-splice-env", err), + } + } + ''; + + env-splice-record = depot.nix.writers.rustSimple { + name = "env-splice-record"; + dependencies = [ + netencode-rs + depot.users.Profpatsch.execline.exec-helpers + ]; + } '' + extern crate netencode; + extern crate exec_helpers; + use netencode::{T}; + use std::os::unix::ffi::OsStringExt; + + fn main() { + exec_helpers::no_args("env-splice-record"); + let mut res = std::collections::HashMap::new(); + for (key, val) in std::env::vars_os() { + match (String::from_utf8(key.into_vec()), String::from_utf8(val.into_vec())) { + (Ok(k), Ok(v)) => { let _ = res.insert(k, T::Text(v)); }, + // same as in record-splice-env, we ignore non-utf8 variables + (_, _) => {}, + } + } + netencode::encode(&mut std::io::stdout(), &T::Record(res).to_u()).unwrap() + } + ''; + +in depot.nix.readTree.drvTargets { + inherit + netencode-rs + pretty-rs + pretty + netencode-mustache + record-get + record-splice-env + env-splice-record + gen + ; +} diff --git a/users/Profpatsch/netencode/gen.nix b/users/Profpatsch/netencode/gen.nix new file mode 100644 index 000000000000..305ff7b08dd6 --- /dev/null +++ b/users/Profpatsch/netencode/gen.nix @@ -0,0 +1,69 @@ +{ lib }: +let + + netstring = tag: suffix: s: + "${tag}${toString (builtins.stringLength s)}:${s}${suffix}"; + + unit = "u,"; + + n1 = b: if b then "n1:1," else "n1:0,"; + + n = i: n: "n${toString i}:${toString n},"; + i = i: n: "i${toString i}:${toString n},"; + + n3 = n 3; + n6 = n 6; + n7 = n 7; + + i3 = i 3; + i6 = i 6; + i7 = i 7; + + text = netstring "t" ","; + binary = netstring "b" ","; + + tag = key: val: netstring "<" "|" key + val; + + concatStrings = builtins.concatStringsSep ""; + + record = lokv: netstring "{" "}" + (concatStrings (map ({key, val}: tag key val) lokv)); + + list = l: netstring "[" "]" (concatStrings l); + + dwim = val: + let match = { + "bool" = n1; + "int" = i6; + "string" = text; + "set" = attrs: + # it could be a derivation, then just return the path + if attrs.type or "" == "derivation" then text "${attrs}" + else + record (lib.mapAttrsToList + (k: v: { + key = k; + val = dwim v; + }) attrs); + "list" = l: list (map dwim l); + }; + in match.${builtins.typeOf val} val; + +in { + inherit + unit + n1 + n3 + n6 + n7 + i3 + i6 + i7 + text + binary + tag + record + list + dwim + ; +} diff --git a/users/Profpatsch/netencode/netencode-mustache.rs b/users/Profpatsch/netencode/netencode-mustache.rs new file mode 100644 index 000000000000..ee7bafed2250 --- /dev/null +++ b/users/Profpatsch/netencode/netencode-mustache.rs @@ -0,0 +1,53 @@ +extern crate netencode; +extern crate mustache; +extern crate arglib_netencode; + +use mustache::{Data}; +use netencode::{T}; +use std::collections::HashMap; +use std::os::unix::ffi::{OsStrExt}; +use std::io::{Read}; + +fn netencode_to_mustache_data_dwim(t: T) -> Data { + match t { + // TODO: good idea? + T::Unit => Data::Null, + T::N1(b) => Data::Bool(b), + T::N3(u) => Data::String(u.to_string()), + T::N6(u) => Data::String(u.to_string()), + T::N7(u) => Data::String(u.to_string()), + T::I3(i) => Data::String(i.to_string()), + T::I6(i) => Data::String(i.to_string()), + T::I7(i) => Data::String(i.to_string()), + T::Text(s) => Data::String(s), + T::Binary(b) => unimplemented!(), + T::Sum(tag) => unimplemented!(), + T::Record(xs) => Data::Map( + xs.into_iter() + .map(|(key, val)| (key, netencode_to_mustache_data_dwim(val))) + .collect::<HashMap<_,_>>() + ), + T::List(xs) => Data::Vec( + xs.into_iter() + .map(|x| netencode_to_mustache_data_dwim(x)) + .collect::<Vec<_>>() + ), + } +} + +pub fn from_stdin() -> () { + let data = netencode_to_mustache_data_dwim( + arglib_netencode::arglib_netencode("netencode-mustache", Some(std::ffi::OsStr::new("TEMPLATE_DATA"))) + ); + let mut stdin = String::new(); + std::io::stdin().read_to_string(&mut stdin).unwrap(); + mustache::compile_str(&stdin) + .and_then(|templ| templ.render_data( + &mut std::io::stdout(), + &data + )).unwrap() +} + +pub fn main() { + from_stdin() +} diff --git a/users/Profpatsch/netencode/netencode.rs b/users/Profpatsch/netencode/netencode.rs new file mode 100644 index 000000000000..5bd43f992f0b --- /dev/null +++ b/users/Profpatsch/netencode/netencode.rs @@ -0,0 +1,844 @@ +extern crate nom; +extern crate exec_helpers; + +use std::collections::HashMap; +use std::io::{Write, Read}; +use std::fmt::{Display, Debug}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum T { + // Unit + Unit, + // Boolean + N1(bool), + // Naturals + N3(u8), + N6(u64), + N7(u128), + // Integers + I3(i8), + I6(i64), + I7(i128), + // Text + // TODO: make into &str + Text(String), + // TODO: rename to Bytes + Binary(Vec<u8>), + // Tags + // TODO: make into &str + // TODO: rename to Tag + Sum(Tag<String, T>), + // TODO: make into &str + Record(HashMap<String, T>), + List(Vec<T>), +} + +impl T { + pub fn to_u<'a>(&'a self) -> U<'a> { + match self { + T::Unit => U::Unit, + T::N1(b) => U::N1(*b), + T::N3(u) => U::N3(*u), + T::N6(u) => U::N6(*u), + T::N7(u) => U::N7(*u), + T::I3(i) => U::I3(*i), + T::I6(i) => U::I6(*i), + T::I7(i) => U::I7(*i), + T::Text(t) => U::Text(t.as_str()), + T::Binary(v) => U::Binary(v), + T::Sum(Tag { tag, val }) => U::Sum( + Tag { tag: tag.as_str(), val: Box::new(val.to_u()) } + ), + T::Record(map) => U::Record( + map.iter().map(|(k, v)| (k.as_str(), v.to_u())).collect() + ), + T::List(l) => U::List( + l.iter().map(|v| v.to_u()).collect::<Vec<U<'a>>>() + ), + } + } + + pub fn encode<'a>(&'a self) -> Vec<u8> { + match self { + // TODO: don’t go via U, inefficient + o => o.to_u().encode() + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum U<'a> { + Unit, + // Boolean + N1(bool), + // Naturals + N3(u8), + N6(u64), + N7(u128), + // Integers + I3(i8), + I6(i64), + I7(i128), + // Text + Text(&'a str), + Binary(&'a [u8]), + // TODO: the U-recursion we do here means we can’t be breadth-lazy anymore + // like we originally planned; maybe we want to go `U<'a>` → `&'a [u8]` again? + // Tags + // TODO: rename to Tag + Sum(Tag<&'a str, U<'a>>), + Record(HashMap<&'a str, U<'a>>), + List(Vec<U<'a>>), +} + +impl<'a> U<'a> { + pub fn encode(&self) -> Vec<u8> { + let mut c = std::io::Cursor::new(vec![]); + encode(&mut c, self); + c.into_inner() + } + + pub fn to_t(&self) -> T { + match self { + U::Unit => T::Unit, + U::N1(b) => T::N1(*b), + U::N3(u) => T::N3(*u), + U::N6(u) => T::N6(*u), + U::N7(u) => T::N7(*u), + U::I3(i) => T::I3(*i), + U::I6(i) => T::I6(*i), + U::I7(i) => T::I7(*i), + U::Text(t) => T::Text((*t).to_owned()), + U::Binary(v) => T::Binary((*v).to_owned()), + U::Sum(Tag { tag, val }) => T::Sum( + Tag { tag: (*tag).to_owned(), val: Box::new(val.to_t()) } + ), + U::Record(map) => T::Record( + map.iter().map(|(k, v)| ((*k).to_owned(), v.to_t())).collect::<HashMap<String, T>>() + ), + U::List(l) => T::List( + l.iter().map(|v| v.to_t()).collect::<Vec<T>>() + ), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Tag<S, A> { + // TODO: make into &str + pub tag: S, + pub val: Box<A> +} + +impl<S, A> Tag<S, A> { + fn map<F, B>(self, f: F) -> Tag<S, B> + where F: Fn(A) -> B { + Tag { + tag: self.tag, + val: Box::new(f(*self.val)) + } + } +} + +fn encode_tag<W: Write>(w: &mut W, tag: &str, val: &U) -> std::io::Result<()> { + write!(w, "<{}:{}|", tag.len(), tag)?; + encode(w, val)?; + Ok(()) +} + +pub fn encode<W: Write>(w: &mut W, u: &U) -> std::io::Result<()> { + match u { + U::Unit => write!(w, "u,"), + U::N1(b) => if *b { write!(w, "n1:1,") } else { write!(w, "n1:0,") }, + U::N3(n) => write!(w, "n3:{},", n), + U::N6(n) => write!(w, "n6:{},", n), + U::N7(n) => write!(w, "n7:{},", n), + U::I3(i) => write!(w, "i3:{},", i), + U::I6(i) => write!(w, "i6:{},", i), + U::I7(i) => write!(w, "i7:{},", i), + U::Text(s) => { + write!(w, "t{}:", s.len()); + w.write_all(s.as_bytes()); + write!(w, ",") + } + U::Binary(s) => { + write!(w, "b{}:", s.len()); + w.write_all(&s); + write!(w, ",") + }, + U::Sum(Tag{tag, val}) => encode_tag(w, tag, val), + U::Record(m) => { + let mut c = std::io::Cursor::new(vec![]); + for (k, v) in m { + encode_tag(&mut c, k, v)?; + } + write!(w, "{{{}:", c.get_ref().len())?; + w.write_all(c.get_ref())?; + write!(w, "}}") + }, + U::List(l) => { + let mut c = std::io::Cursor::new(vec![]); + for u in l { + encode(&mut c, u)?; + } + write!(w, "[{}:", c.get_ref().len())?; + w.write_all(c.get_ref())?; + write!(w, "]") + } + } +} + +pub fn text(s: String) -> T { + T::Text(s) +} + +pub fn u_from_stdin_or_die_user_error<'a>(prog_name: &'_ str, stdin_buf: &'a mut Vec<u8>) -> U<'a> { + std::io::stdin().lock().read_to_end(stdin_buf); + let u = match parse::u_u(stdin_buf) { + Ok((rest, u)) => match rest { + b"" => u, + _ => exec_helpers::die_user_error(prog_name, format!("stdin contained some soup after netencode value: {:?}", String::from_utf8_lossy(rest))) + }, + Err(err) => exec_helpers::die_user_error(prog_name, format!("unable to parse netencode from stdin: {:?}", err)) + }; + u +} + +pub mod parse { + use super::{T, Tag, U}; + + use std::str::FromStr; + use std::ops::Neg; + use std::collections::HashMap; + + use nom::{IResult}; + use nom::branch::{alt}; + use nom::bytes::streaming::{tag, take}; + use nom::character::streaming::{digit1, char}; + use nom::sequence::{tuple}; + use nom::combinator::{map, map_res, flat_map, map_parser, opt}; + use nom::error::{context, ErrorKind, ParseError}; + + fn unit_t(s: &[u8]) -> IResult<&[u8], ()> { + let (s, _) = context("unit", tag("u,"))(s)?; + Ok((s, ())) + } + + fn usize_t(s: &[u8]) -> IResult<&[u8], usize> { + context( + "usize", + map_res( + map_res(digit1, |n| std::str::from_utf8(n)), + |s| s.parse::<usize>()) + )(s) + } + + fn sized(begin: char, end: char) -> impl Fn(&[u8]) -> IResult<&[u8], &[u8]> { + move |s: &[u8]| { + // This is the point where we check the descriminator; + // if the beginning char does not match, we can immediately return. + let (s, _) = char(begin)(s)?; + let (s, (len, _)) = tuple(( + usize_t, + char(':') + ))(s)?; + let (s, (res, _)) = tuple(( + take(len), + char(end) + ))(s)?; + Ok((s, res)) + } + } + + + fn uint_t<'a, I: FromStr + 'a>(t: &'static str) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], I> { + move |s: &'a [u8]| { + let (s, (_, _, int, _)) = tuple(( + tag(t.as_bytes()), + char(':'), + map_res( + map_res(digit1, |n: &[u8]| std::str::from_utf8(n)), + |s| s.parse::<I>() + ), + char(',') + ))(s)?; + Ok((s, int)) + } + } + + fn bool_t<'a>() -> impl Fn(&'a [u8]) -> IResult<&'a [u8], bool> { + context("bool", alt(( + map(tag("n1:0,"), |_| false), + map(tag("n1:1,"), |_| true), + ))) + } + + fn int_t<'a, I: FromStr + Neg<Output=I>>(t: &'static str) -> impl Fn(&'a [u8]) -> IResult<&[u8], I> { + context( + t, + move |s: &'a [u8]| { + let (s, (_, _, neg, int, _)) = tuple(( + tag(t.as_bytes()), + char(':'), + opt(char('-')), + map_res( + map_res(digit1, |n: &[u8]| std::str::from_utf8(n)), + |s| s.parse::<I>() + ), + char(',') + ))(s)?; + let res = match neg { + Some(_) => -int, + None => int, + }; + Ok((s, res)) + } + ) + } + + fn tag_t(s: &[u8]) -> IResult<&[u8], Tag<String, T>> { + // recurses into the main parser + map(tag_g(t_t), + |Tag {tag, val}| + Tag { + tag: tag.to_string(), + val + })(s) + } + + fn tag_g<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Tag<&'a str, O>> + where + P: Fn(&'a [u8]) -> IResult<&'a [u8], O> + { + move |s: &[u8]| { + let (s, tag) = sized('<', '|')(s)?; + let (s, val) = inner(s)?; + Ok((s, Tag { + tag: std::str::from_utf8(tag) + .map_err(|_| nom::Err::Failure((s, ErrorKind::Char)))?, + val: Box::new(val) + })) + + } + } + + /// parse text scalar (`t5:hello,`) + fn text(s: &[u8]) -> IResult<&[u8], T> { + let (s, res) = text_g(s)?; + Ok((s, T::Text(res.to_string()))) + } + + fn text_g(s: &[u8]) -> IResult<&[u8], &str> { + let (s, res) = sized('t', ',')(s)?; + Ok((s, + std::str::from_utf8(res) + .map_err(|_| nom::Err::Failure((s, ErrorKind::Char)))?, + )) + } + + fn binary<'a>() -> impl Fn(&'a [u8]) -> IResult<&'a [u8], T> { + map(binary_g(), |b| T::Binary(b.to_owned())) + } + + fn binary_g() -> impl Fn(&[u8]) -> IResult<&[u8], &[u8]> { + sized('b', ',') + } + + fn list_t(s: &[u8]) -> IResult<&[u8], Vec<T>> { + list_g(t_t)(s) + } + + /// Wrap the inner parser of an `many0`/`fold_many0`, so that the parser + /// is not called when the `s` is empty already, preventing it from + /// returning `Incomplete` on streaming parsing. + fn inner_no_empty_string<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], O> + where + O: Clone, + P: Fn(&'a [u8]) -> IResult<&'a [u8], O>, + { + move |s: &'a [u8]| { + if s.is_empty() { + // This is a bit hacky, `many0` considers the inside done + // when a parser returns `Err::Error`, ignoring the actual error content + Err(nom::Err::Error((s, nom::error::ErrorKind::Many0))) + } else { + inner(s) + } + } + } + + fn list_g<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], Vec<O>> + where + O: Clone, + P: Fn(&'a [u8]) -> IResult<&'a [u8], O>, + { + map_parser( + sized('[', ']'), + nom::multi::many0(inner_no_empty_string(inner)) + ) + } + + fn record_t<'a>(s: &'a [u8]) -> IResult<&'a [u8], HashMap<String, T>> { + let (s, r) = record_g(t_t)(s)?; + Ok((s, + r.into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect::<HashMap<_,_>>())) + } + + fn record_g<'a, P, O>(inner: P) -> impl Fn(&'a [u8]) -> IResult<&'a [u8], HashMap<&'a str, O>> + where + O: Clone, + P: Fn(&'a [u8]) -> IResult<&'a [u8], O> + { + move |s: &'a [u8]| { + let (s, map) = map_parser( + sized('{', '}'), + nom::multi::fold_many0( + inner_no_empty_string(tag_g(&inner)), + HashMap::new(), + |mut acc: HashMap<_,_>, Tag { tag, mut val }| { + // ignore duplicated tag names that appear later + // according to netencode spec + if ! acc.contains_key(tag) { + acc.insert(tag, *val); + } + acc + } + ) + )(s)?; + if map.is_empty() { + // records must not be empty, according to the spec + Err(nom::Err::Failure((s,nom::error::ErrorKind::Many1))) + } else { + Ok((s, map)) + } + } + } + + pub fn u_u(s: &[u8]) -> IResult<&[u8], U> { + alt(( + map(text_g, U::Text), + map(binary_g(), U::Binary), + map(unit_t, |()| U::Unit), + map(tag_g(u_u), |t| U::Sum(t)), + map(list_g(u_u), U::List), + map(record_g(u_u), U::Record), + + map(bool_t(), |u| U::N1(u)), + map(uint_t("n3"), |u| U::N3(u)), + map(uint_t("n6"), |u| U::N6(u)), + map(uint_t("n7"), |u| U::N7(u)), + map(int_t("i3"), |u| U::I3(u)), + map(int_t("i6"), |u| U::I6(u)), + map(int_t("i7"), |u| U::I7(u)), + + // less common + map(uint_t("n2"), |u| U::N3(u)), + map(uint_t("n4"), |u| U::N6(u)), + map(uint_t("n5"), |u| U::N6(u)), + map(int_t("i1"), |u| U::I3(u)), + map(int_t("i2"), |u| U::I3(u)), + map(int_t("i4"), |u| U::I6(u)), + map(int_t("i5"), |u| U::I6(u)), + // TODO: 8, 9 not supported + ))(s) + } + + pub fn t_t(s: &[u8]) -> IResult<&[u8], T> { + alt(( + text, + binary(), + map(unit_t, |_| T::Unit), + map(tag_t, |t| T::Sum(t)), + map(list_t, |l| T::List(l)), + map(record_t, |p| T::Record(p)), + + map(bool_t(), |u| T::N1(u)), + // 8, 64 and 128 bit + map(uint_t("n3"), |u| T::N3(u)), + map(uint_t("n6"), |u| T::N6(u)), + map(uint_t("n7"), |u| T::N7(u)), + map(int_t("i3"), |u| T::I3(u)), + map(int_t("i6"), |u| T::I6(u)), + map(int_t("i7"), |u| T::I7(u)), + + // less common + map(uint_t("n2"), |u| T::N3(u)), + map(uint_t("n4"), |u| T::N6(u)), + map(uint_t("n5"), |u| T::N6(u)), + map(int_t("i1"), |u| T::I3(u)), + map(int_t("i2"), |u| T::I3(u)), + map(int_t("i4"), |u| T::I6(u)), + map(int_t("i5"), |u| T::I6(u)), + // TODO: 8, 9 not supported + ))(s) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn test_parse_unit_t() { + assert_eq!( + unit_t("u,".as_bytes()), + Ok(("".as_bytes(), ())) + ); + } + + #[test] + fn test_parse_bool_t() { + assert_eq!( + bool_t()("n1:0,".as_bytes()), + Ok(("".as_bytes(), false)) + ); + assert_eq!( + bool_t()("n1:1,".as_bytes()), + Ok(("".as_bytes(), true)) + ); + } + + #[test] + fn test_parse_usize_t() { + assert_eq!( + usize_t("32foo".as_bytes()), + Ok(("foo".as_bytes(), 32)) + ); + } + + #[test] + fn test_parse_int_t() { + assert_eq!( + uint_t::<u8>("n3")("n3:42,abc".as_bytes()), + Ok(("abc".as_bytes(), 42)) + ); + assert_eq!( + uint_t::<u8>("n3")("n3:1024,abc".as_bytes()), + Err(nom::Err::Error(("1024,abc".as_bytes(), nom::error::ErrorKind::MapRes))) + ); + assert_eq!( + int_t::<i64>("i6")("i6:-23,abc".as_bytes()), + Ok(("abc".as_bytes(), -23)) + ); + assert_eq!( + int_t::<i128>("i3")("i3:0,:abc".as_bytes()), + Ok((":abc".as_bytes(), 0)) + ); + assert_eq!( + uint_t::<u8>("n7")("n7:09,".as_bytes()), + Ok(("".as_bytes(), 9)) + ); + // assert_eq!( + // length("c"), + // Err(nom::Err::Error(("c", nom::error::ErrorKind::Digit))) + // ); + // assert_eq!( + // length(":"), + // Err(nom::Err::Error((":", nom::error::ErrorKind::Digit))) + // ); + } + + #[test] + fn test_parse_text() { + assert_eq!( + text("t5:hello,".as_bytes()), + Ok(("".as_bytes(), T::Text("hello".to_owned()))), + "{}", r"t5:hello," + ); + assert_eq!( + text("t4:fo".as_bytes()), + // The content of the text should be 4 long + Err(nom::Err::Incomplete(nom::Needed::Size(4))), + "{}", r"t4:fo," + ); + assert_eq!( + text("t9:今日は,".as_bytes()), + Ok(("".as_bytes(), T::Text("今日は".to_owned()))), + "{}", r"t9:今日は," + ); + } + + #[test] + fn test_parse_binary() { + assert_eq!( + binary()("b5:hello,".as_bytes()), + Ok(("".as_bytes(), T::Binary(Vec::from("hello".to_owned())))), + "{}", r"b5:hello," + ); + assert_eq!( + binary()("b4:fo".as_bytes()), + // The content of the byte should be 4 long + Err(nom::Err::Incomplete(nom::Needed::Size(4))), + "{}", r"b4:fo," + ); + assert_eq!( + binary()("b4:foob".as_bytes()), + // The content is 4 bytes now, but the finishing , is missing + Err(nom::Err::Incomplete(nom::Needed::Size(1))), + "{}", r"b4:fo," + ); + assert_eq!( + binary()("b9:今日は,".as_bytes()), + Ok(("".as_bytes(), T::Binary(Vec::from("今日は".as_bytes())))), + "{}", r"b9:今日は," + ); + } + + #[test] + fn test_list() { + assert_eq!( + list_t("[0:]".as_bytes()), + Ok(("".as_bytes(), vec![])), + "{}", r"[0:]" + ); + assert_eq!( + list_t("[6:u,u,u,]".as_bytes()), + Ok(("".as_bytes(), vec![ + T::Unit, + T::Unit, + T::Unit, + ])), + "{}", r"[6:u,u,u,]" + ); + assert_eq!( + list_t("[15:u,[7:t3:foo,]u,]".as_bytes()), + Ok(("".as_bytes(), vec![ + T::Unit, + T::List(vec![T::Text("foo".to_owned())]), + T::Unit, + ])), + "{}", r"[15:u,[7:t3:foo,]u,]" + ); + } + + #[test] + fn test_record() { + assert_eq!( + record_t("{21:<1:a|u,<1:b|u,<1:c|u,}".as_bytes()), + Ok(("".as_bytes(), vec![ + ("a".to_owned(), T::Unit), + ("b".to_owned(), T::Unit), + ("c".to_owned(), T::Unit), + ].into_iter().collect::<HashMap<String, T>>())), + "{}", r"{21:<1:a|u,<1:b|u,<1:c|u,}" + ); + // duplicated keys are ignored (first is taken) + assert_eq!( + record_t("{25:<1:a|u,<1:b|u,<1:a|i1:-1,}".as_bytes()), + Ok(("".as_bytes(), vec![ + ("a".to_owned(), T::Unit), + ("b".to_owned(), T::Unit), + ].into_iter().collect::<HashMap<_,_>>())), + "{}", r"{25:<1:a|u,<1:b|u,<1:a|i1:-1,}" + ); + // empty records are not allowed + assert_eq!( + record_t("{0:}".as_bytes()), + Err(nom::Err::Failure(("".as_bytes(), nom::error::ErrorKind::Many1))), + "{}", r"{0:}" + ); + } + + #[test] + fn test_parse() { + assert_eq!( + t_t("n3:255,".as_bytes()), + Ok(("".as_bytes(), T::N3(255))), + "{}", r"n3:255," + ); + assert_eq!( + t_t("t6:halloo,".as_bytes()), + Ok(("".as_bytes(), T::Text("halloo".to_owned()))), + "{}", r"t6:halloo," + ); + assert_eq!( + t_t("<3:foo|t6:halloo,".as_bytes()), + Ok(("".as_bytes(), T::Sum (Tag { + tag: "foo".to_owned(), + val: Box::new(T::Text("halloo".to_owned())) + }))), + "{}", r"<3:foo|t6:halloo," + ); + // { a: Unit + // , foo: List <A: Unit | B: List i3> } + assert_eq!( + t_t("{52:<1:a|u,<3:foo|[33:<1:A|u,<1:A|n1:1,<1:B|[7:i3:127,]]}".as_bytes()), + Ok(("".as_bytes(), T::Record(vec![ + ("a".to_owned(), T::Unit), + ("foo".to_owned(), T::List(vec![ + T::Sum(Tag { tag: "A".to_owned(), val: Box::new(T::Unit) }), + T::Sum(Tag { tag: "A".to_owned(), val: Box::new(T::N1(true)) }), + T::Sum(Tag { tag: "B".to_owned(), val: Box::new(T::List(vec![T::I3(127)])) }), + ])) + ].into_iter().collect::<HashMap<String, T>>()))), + "{}", r"{52:<1:a|u,<3:foo|[33:<1:A|u,<1:A|n1:1,<1:B|[7:i3:127,]]}" + ); + } + + } +} + +pub mod dec { + use super::*; + use std::collections::HashMap; + + pub struct DecodeError(pub String); + + pub trait Decoder<'a> { + type A; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError>; + } + + /// Any netencode, as `T`. + #[derive(Clone, Copy)] + pub struct AnyT; + /// Any netencode, as `U`. + #[derive(Clone, Copy)] + pub struct AnyU; + + impl<'a> Decoder<'a> for AnyT { + type A = T; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + Ok(u.to_t()) + } + } + + impl<'a> Decoder<'a> for AnyU { + type A = U<'a>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + Ok(u) + } + } + + /// A text + #[derive(Clone, Copy)] + pub struct Text; + + /// A bytestring + // TODO: rename to Bytes + #[derive(Clone, Copy)] + pub struct Binary; + + impl<'a> Decoder<'a> for Text { + type A = &'a str; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::Text(t) => Ok(t), + other => Err(DecodeError(format!("Cannot decode {:?} into Text", other))), + } + } + } + + impl<'a> Decoder<'a> for Binary { + type A = &'a [u8]; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::Binary(b) => Ok(b), + other => Err(DecodeError(format!("Cannot decode {:?} into Binary", other))), + } + } + } + + /// Any scalar, converted to bytes. + #[derive(Clone, Copy)] + pub struct ScalarAsBytes; + + impl<'a> Decoder<'a> for ScalarAsBytes { + type A = Vec<u8>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::N3(u) => Ok(format!("{}", u).into_bytes()), + U::N6(u) => Ok(format!("{}", u).into_bytes()), + U::N7(u) => Ok(format!("{}", u).into_bytes()), + U::I3(i) => Ok(format!("{}", i).into_bytes()), + U::I6(i) => Ok(format!("{}", i).into_bytes()), + U::I7(i) => Ok(format!("{}", i).into_bytes()), + U::Text(t) => Ok(t.as_bytes().to_owned()), + U::Binary(b) => Ok(b.to_owned()), + o => Err(DecodeError(format!("Cannot decode {:?} into scalar", o))), + } + } + } + + /// A map of Ts (TODO: rename to map) + #[derive(Clone, Copy)] + pub struct Record<T>(pub T); + + impl<'a, Inner> Decoder<'a> for Record<Inner> + where Inner: Decoder<'a> + { + type A = HashMap<&'a str, Inner::A>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match u { + U::Record(map) => + map.into_iter() + .map(|(k, v)| self.0.dec(v).map(|v2| (k, v2))) + .collect::<Result<Self::A, _>>(), + o => Err(DecodeError(format!("Cannot decode {:?} into record", o))) + } + } + } + + /// Assume a record and project out the field with the given name and type. + #[derive(Clone, Copy)] + pub struct RecordDot<'a, T> { + pub field: &'a str, + pub inner: T + } + + impl <'a, Inner> Decoder<'a> for RecordDot<'_, Inner> + where Inner: Decoder<'a> + Clone + { + type A = Inner::A; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match Record(self.inner.clone()).dec(u) { + Ok(mut map) => match map.remove(self.field) { + Some(inner) => Ok(inner), + None => Err(DecodeError(format!("Cannot find `{}` in record map", self.field))), + }, + Err(err) => Err(err), + } + } + } + + /// Equals one of the listed `A`s exactly, after decoding. + #[derive(Clone)] + pub struct OneOf<T, A>{ + pub inner: T, + pub list: Vec<A>, + } + + impl <'a, Inner> Decoder<'a> for OneOf<Inner, Inner::A> + where Inner: Decoder<'a>, + Inner::A: Display + Debug + PartialEq + { + type A = Inner::A; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match self.inner.dec(u) { + Ok(inner) => match self.list.iter().any(|x| x.eq(&inner)) { + true => Ok(inner), + false => Err(DecodeError(format!("{} is not one of {:?}", inner, self.list))) + }, + Err(err) => Err(err) + } + } + } + + /// Try decoding as `T`. + #[derive(Clone)] + pub struct Try<T>(pub T); + + impl <'a, Inner> Decoder<'a> for Try<Inner> + where Inner: Decoder<'a> + { + type A = Option<Inner::A>; + fn dec(&self, u: U<'a>) -> Result<Self::A, DecodeError> { + match self.0.dec(u) { + Ok(inner) => Ok(Some(inner)), + Err(err) => Ok(None) + } + } + } + +} diff --git a/users/Profpatsch/netencode/pretty.rs b/users/Profpatsch/netencode/pretty.rs new file mode 100644 index 000000000000..8fec24a60e1a --- /dev/null +++ b/users/Profpatsch/netencode/pretty.rs @@ -0,0 +1,140 @@ +extern crate netencode; + +use netencode::{U, T, Tag}; + +pub enum Pretty { + Single { + r#type: char, + length: String, + val: String, + trailer: char, + }, + Tag { + r#type: char, + length: String, + key: String, + inner: char, + val: Box<Pretty>, + }, + Multi { + r#type: char, + length: String, + vals: Vec<Pretty>, + trailer: char + }, +} + +impl Pretty { + pub fn from_u<'a>(u: U<'a>) -> Pretty { + match u { + U::Unit => Self::scalar('u', "", ""), + U::N1(b) => Self::scalar('n', "1:", if b { "1" } else { "0" }), + U::N3(n) => Self::scalar('n', "3:", n), + U::N6(n) => Self::scalar('n', "6:", n), + U::N7(n) => Self::scalar('n', "7:", n), + U::I3(i) => Self::scalar('i', "3:", i), + U::I6(i) => Self::scalar('i', "6:", i), + U::I7(i) => Self::scalar('i', "7:", i), + U::Text(s) => Pretty::Single { + r#type: 't', + length: format!("{}:", s.len()), + val: s.to_string(), + trailer: ',' + }, + U::Binary(s) => Pretty::Single { + r#type: 'b', + length: format!("{}:", s.len()), + // For pretty printing we want the string to be visible obviously. + // Instead of not supporting binary, let’s use lossy conversion. + val: String::from_utf8_lossy(s).into_owned(), + trailer: ',' + }, + U::Sum(Tag{tag, val}) => Self::pretty_tag(tag, Self::from_u(*val)), + U::Record(m) => Pretty::Multi { + r#type: '{', + // TODO: we are losing the size here, should we recompute it? Keep it? + length: String::from(""), + vals: m.into_iter().map(|(k, v)| Self::pretty_tag(k, Self::from_u(v))).collect(), + trailer: '}' + }, + U::List(l) => Pretty::Multi { + r#type: '[', + // TODO: we are losing the size here, should we recompute it? Keep it? + length: String::from(""), + vals: l.into_iter().map(|v| Self::from_u(v)).collect(), + trailer: ']', + }, + } + } + + fn scalar<D>(r#type: char, length: &str, d: D) -> Pretty + where D: std::fmt::Display + { + Pretty::Single { + r#type, + length: length.to_string(), + val: format!("{}", d), + trailer: ',' + } + } + + fn pretty_tag(tag: &str, val: Pretty) -> Pretty { + Pretty::Tag { + r#type: '<', + length: format!("{}:", tag.len()), + key: tag.to_string(), + inner: '|', + val: Box::new(val), + } + } + + pub fn print_multiline<W>(&self, mut w: &mut W) -> std::io::Result<()> + where W: std::io::Write + { + Self::go(&mut w, self, 0, true); + write!(w, "\n") + } + + fn go<W>(mut w: &mut W, p: &Pretty, depth: usize, is_newline: bool) -> std::io::Result<()> + where W: std::io::Write + { + const full : usize = 4; + const half : usize = 2; + let i = &vec![b' '; depth*full]; + let iandhalf = &vec![b' '; depth*full + half]; + let (i, iandhalf) = unsafe {( + std::str::from_utf8_unchecked(i), + std::str::from_utf8_unchecked(iandhalf), + )}; + if is_newline { + write!(&mut w, "{}", i); + } + match p { + Pretty::Single {r#type, length, val, trailer} => + write!(&mut w, "{} {}{}", r#type, val, trailer), + Pretty::Tag { r#type, length, key, inner, val } => { + write!(&mut w, "{} {} {}", r#type, key, inner)?; + Self::go::<W>(&mut w, val, depth, false) + }, + // if the length is 0 or 1, we print on one line, + // only if there’s more than one element we split the resulting value. + // we never break lines on arbitrary column sizes, since that is just silly. + Pretty::Multi {r#type, length, vals, trailer} => match vals.len() { + 0 => write!(&mut w, "{} {}", r#type, trailer), + 1 => { + write!(&mut w, "{} ", r#type); + Self::go::<W>(&mut w, &vals[0], depth, false)?; + write!(&mut w, "{}", trailer) + }, + more => { + write!(&mut w, "\n{}{} \n", iandhalf, r#type)?; + for v in vals { + Self::go::<W>(&mut w, v, depth + 1, true)?; + write!(&mut w, "\n")?; + } + write!(&mut w, "{}{}", iandhalf, trailer) + } + }, + } + } +} diff --git a/users/Profpatsch/netstring/README.md b/users/Profpatsch/netstring/README.md new file mode 100644 index 000000000000..b8daea11d158 --- /dev/null +++ b/users/Profpatsch/netstring/README.md @@ -0,0 +1,18 @@ +# Netstring + +Netstrings are a djb invention. They are intended as a serialization format. Instead of inline control characters like `\n` or `\0` to signal the end of a string, they use a run-length encoding given as the number of bytes, encoded in ASCII, at the beginning of the string. + +``` +hello -> 5:hello, +foo! -> 4:foo!, +こんにちは -> 15:こんにちは, +``` + +They can be used to encode e.g. lists by simply concatenating and reading them in one-by-one. + +If you need a more complex encoding, you could start encoding e.g. tuples as netstrings-in-netstrings, or you could use [`netencode`](../netencode/README.md) instead, which is what-if-json-but-netstrings, and takes the idea of netstrings to their logical conclusion. + +Resources: + +Spec: http://cr.yp.to/proto/netstrings.txt +Wiki: https://en.wikipedia.org/wiki/Netstring diff --git a/users/Profpatsch/netstring/default.nix b/users/Profpatsch/netstring/default.nix new file mode 100644 index 000000000000..dcc29d7e6fec --- /dev/null +++ b/users/Profpatsch/netstring/default.nix @@ -0,0 +1,63 @@ +{ lib, pkgs, depot, ... }: +let + toNetstring = s: + "${toString (builtins.stringLength s)}:${s},"; + + toNetstringList = xs: + lib.concatStrings (map toNetstring xs); + + toNetstringKeyVal = attrs: + lib.concatStrings + (lib.mapAttrsToList + (k: v: toNetstring (toNetstring k + toNetstring v)) + attrs); + + python-netstring = depot.users.Profpatsch.writers.python3Lib { + name = "netstring"; + } '' + def read_netstring(bytes): + (int_length, rest) = bytes.split(sep=b':', maxsplit=1) + val = rest[:int(int_length)] + # has to end on a , + assert(rest[len(val)] == ord(',')) + return (val, rest[len(val) + 1:]) + + def read_netstring_key_val(bytes): + (keyvalnet, rest) = read_netstring(bytes) + (key, valnet) = read_netstring(keyvalnet) + (val, nothing) = read_netstring(valnet) + assert(nothing == b"") + return (key, val, rest) + + def read_netstring_key_val_list(bytes): + rest = bytes + res = {} + while rest != b"": + (key, val, r) = read_netstring_key_val(rest) + rest = r + res[key] = val + return res + ''; + + rust-netstring = depot.nix.writers.rustSimpleLib { + name = "netstring"; + } '' + pub fn to_netstring(s: &[u8]) -> Vec<u8> { + let len = s.len(); + // length of the integer as ascii + let i_len = ((len as f64).log10() as usize) + 1; + let ns_len = i_len + 1 + len + 1; + let mut res = Vec::with_capacity(ns_len); + res.extend_from_slice(format!("{}:", len).as_bytes()); + res.extend_from_slice(s); + res.push(b','); + res + } + ''; + +in depot.nix.readTree.drvTargets { + inherit + python-netstring + rust-netstring + ; +} diff --git a/users/Profpatsch/netstring/tests/default.nix b/users/Profpatsch/netstring/tests/default.nix new file mode 100644 index 000000000000..710ba3d30526 --- /dev/null +++ b/users/Profpatsch/netstring/tests/default.nix @@ -0,0 +1,61 @@ +{ depot, lib, pkgs, ... }: + +let + + python-netstring-test = depot.users.Profpatsch.writers.python3 { + name = "python-netstring-test"; + libraries = p: [ + depot.users.Profpatsch.netstring.python-netstring + ]; + } '' + import netstring + + def assEq(left, right): + assert left == right, "{} /= {}".format(str(left), str(right)) + + assEq( + netstring.read_netstring(b"""${depot.nix.netstring.fromString "hi!"}"""), + (b"hi!", b"") + ) + + assEq( + netstring.read_netstring_key_val( + b"""${depot.nix.netstring.attrsToKeyValList { foo = "42"; }}""" + ), + (b'foo', b'42', b"") + ) + + assEq( + netstring.read_netstring_key_val_list( + b"""${depot.nix.netstring.attrsToKeyValList { foo = "42"; bar = "hi"; }}""" + ), + { b'foo': b'42', b'bar': b'hi' } + ) + ''; + + rust-netstring-test = depot.nix.writers.rustSimple { + name = "rust-netstring-test"; + dependencies = [ + depot.users.Profpatsch.netstring.rust-netstring + ]; + } '' + extern crate netstring; + + fn main() { + assert_eq!( + std::str::from_utf8(&netstring::to_netstring(b"hello")).unwrap(), + r##"${depot.nix.netstring.fromString "hello"}"## + ); + assert_eq!( + std::str::from_utf8(&netstring::to_netstring("こんにちは".as_bytes())).unwrap(), + r##"${depot.nix.netstring.fromString "こんにちは"}"## + ); + } + ''; + +in depot.nix.readTree.drvTargets { + inherit + python-netstring-test + rust-netstring-test + ; +} diff --git a/users/Profpatsch/nixpkgs-rewriter/MetaStdenvLib.hs b/users/Profpatsch/nixpkgs-rewriter/MetaStdenvLib.hs new file mode 100644 index 000000000000..3ed96a7b6eac --- /dev/null +++ b/users/Profpatsch/nixpkgs-rewriter/MetaStdenvLib.hs @@ -0,0 +1,80 @@ +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE NamedFieldPuns #-} +import Nix.Parser +import Nix.Expr.Types +import Nix.Expr.Types.Annotated +import System.Environment (getArgs) +import System.Exit (die) +import Data.Fix (Fix(..)) +import qualified Data.Text as Text +import qualified Data.ByteString.Lazy.Char8 as BL +import qualified Data.Aeson as A +import qualified Data.Aeson.Encoding as A +import Data.Function ((&)) +import qualified System.IO as IO +import qualified Text.Megaparsec.Pos as MP + +main = do + (nixFile:_) <- getArgs + (parseNixFileLoc nixFile :: IO _) >>= \case + Failure err -> do + ePutStrLn $ show err + die "oh no" + Success expr -> do + case snd $ match expr of + NoArguments -> do + ePutStrLn $ "NoArguments in " <> nixFile + printPairs mempty + YesLib vars -> do + ePutStrLn $ "lib in " <> show vars <> " in " <> nixFile + printPairs mempty + NoLib vars srcSpan -> do + ePutStrLn $ nixFile <> " needs lib added" + printPairs + $ "fileName" A..= nixFile + <> "fromLine" A..= (srcSpan & spanBegin & sourceLine) + <> "fromColumn" A..= (srcSpan & spanBegin & sourceColumn) + <> "toLine" A..= (srcSpan & spanEnd & sourceLine) + <> "toColumn" A..= (srcSpan & spanEnd & sourceColumn) + +printPairs pairs = BL.putStrLn $ A.encodingToLazyByteString $ A.pairs pairs + +ePutStrLn = IO.hPutStrLn IO.stderr + +data Descend = YesDesc | NoDesc + deriving Show +data Matched = NoArguments | NoLib [VarName] SrcSpan | YesLib [VarName] + deriving Show + +match :: Fix (Compose (Ann SrcSpan) NExprF) -> (Descend, Matched) +match = \case + (AnnE outerSpan (NAbs (ParamSet params _ _) (AnnE innerSpan _))) -> (NoDesc, + let vars = map fst params in + case (any (== "lib") vars) of + True -> YesLib vars + False -> + -- The span of the arglist is from the beginning of the match + -- to the beginning of the inner expression + let varSpan = SrcSpan + { spanBegin = outerSpan & spanBegin + -- -1 to prevent the spans from overlapping + , spanEnd = sourcePosMinus1 (innerSpan & spanBegin) } + in NoLib vars varSpan) + _ -> (NoDesc, NoArguments) + +-- | Remove one from a source positon. +-- +-- That means if the current position is at the very beginning of a line, +-- jump to the previous line. +sourcePosMinus1 :: SourcePos -> SourcePos +sourcePosMinus1 src@(SourcePos { sourceLine, sourceColumn }) = + let + col = MP.mkPos $ max (MP.unPos sourceColumn - 1) 1 + line = MP.mkPos $ case MP.unPos sourceColumn of + 1 -> max (MP.unPos sourceLine - 1) 1 + _ -> MP.unPos sourceLine + in src + { sourceLine = line + , sourceColumn = col } diff --git a/users/Profpatsch/nixpkgs-rewriter/default.nix b/users/Profpatsch/nixpkgs-rewriter/default.nix new file mode 100644 index 000000000000..787162d4973a --- /dev/null +++ b/users/Profpatsch/nixpkgs-rewriter/default.nix @@ -0,0 +1,112 @@ +{ depot, pkgs, ... }: +let + inherit (depot.nix) + writeExecline + ; + inherit (depot.users.Profpatsch.lib) + debugExec + ; + + bins = depot.nix.getBins pkgs.coreutils [ "head" "shuf" ] + // depot.nix.getBins pkgs.jq [ "jq" ] + // depot.nix.getBins pkgs.findutils [ "xargs" ] + // depot.nix.getBins pkgs.gnused [ "sed" ] + ; + + export-json-object = pkgs.writers.writePython3 "export-json-object" {} '' + import json + import sys + import os + + d = json.load(sys.stdin) + + if d == {}: + sys.exit(0) + + for k, v in d.items(): + os.environ[k] = str(v) + + os.execvp(sys.argv[1], sys.argv[1:]) + ''; + + meta-stdenv-lib = pkgs.writers.writeHaskell "meta-stdenv-lib" { + libraries = [ + pkgs.haskellPackages.hnix + pkgs.haskellPackages.aeson + ]; + } ./MetaStdenvLib.hs; + + replace-between-lines = writeExecline "replace-between-lines" { readNArgs = 1; } [ + "importas" "-ui" "file" "fileName" + "importas" "-ui" "from" "fromLine" + "importas" "-ui" "to" "toLine" + "if" [ depot.tools.eprintf "%s-%s\n" "$from" "$to" ] + (debugExec "adding lib") + bins.sed + "-e" "\${from},\${to} \${1}" + "-i" "$file" + ]; + + add-lib-if-necessary = writeExecline "add-lib-if-necessary" { readNArgs = 1; } [ + "pipeline" [ meta-stdenv-lib "$1" ] + export-json-object + # first replace any stdenv.lib mentions in the arg header + # if this is not done, the replace below kills these. + # Since we want it anyway ultimately, let’s do it here. + "if" [ replace-between-lines "s/stdenv\.lib/lib/" ] + # then add the lib argument + # (has to be before stdenv, otherwise default arguments might be in the way) + replace-between-lines "s/stdenv/lib, stdenv/" + ]; + + metaString = ''meta = with stdenv.lib; {''; + + replace-stdenv-lib = pkgs.writers.writeBash "replace-stdenv-lib" '' + set -euo pipefail + sourceDir="$1" + for file in $( + ${pkgs.ripgrep}/bin/rg \ + --files-with-matches \ + --fixed-strings \ + -e '${metaString}' \ + "$sourceDir" + ) + do + echo "replacing stdenv.lib meta in $file" >&2 + ${bins.sed} -e '/${metaString}/ s/stdenv.lib/lib/' \ + -i "$file" + ${add-lib-if-necessary} "$file" + done + ''; + + instantiate-nixpkgs-randomly = writeExecline "instantiate-nixpkgs-randomly" { readNArgs = 1; } [ + "export" "NIXPKGS_ALLOW_BROKEN" "1" + "export" "NIXPKGS_ALLOW_UNFREE" "1" + "export" "NIXPKGS_ALLOW_INSECURE" "1" + "export" "NIXPKGS_ALLOW_UNSUPPORTED_SYSTEM" "1" + "pipeline" [ + "nix" + "eval" + "--raw" + ''( + let pkgs = import ''${1} {}; + in builtins.toJSON (builtins.attrNames pkgs) + )'' + ] + "pipeline" [ bins.jq "-r" ".[]" ] + "pipeline" [ bins.shuf ] + "pipeline" [ bins.head "-n" "1000" ] + bins.xargs "-I" "{}" "-n1" + "if" [ depot.tools.eprintf "instantiating %s\n" "{}" ] + "nix-instantiate" "$1" "-A" "{}" + ]; + +in depot.nix.readTree.drvTargets { + inherit + instantiate-nixpkgs-randomly + # requires hnix, which we don’t want in tvl for now + # uncomment manually if you want to use it. + # meta-stdenv-lib + # replace-stdenv-lib + ; +} diff --git a/users/Profpatsch/read-http.nix b/users/Profpatsch/read-http.nix new file mode 100644 index 000000000000..854a11b7d099 --- /dev/null +++ b/users/Profpatsch/read-http.nix @@ -0,0 +1,16 @@ +{ depot, pkgs, ... }: + +let + + read-http = depot.nix.writers.rustSimple { + name = "read-http"; + dependencies = [ + depot.third_party.rust-crates.ascii + depot.third_party.rust-crates.httparse + depot.users.Profpatsch.netencode.netencode-rs + depot.users.Profpatsch.arglib.netencode.rust + depot.users.Profpatsch.execline.exec-helpers + ]; + } (builtins.readFile ./read-http.rs); + +in read-http diff --git a/users/Profpatsch/read-http.rs b/users/Profpatsch/read-http.rs new file mode 100644 index 000000000000..50ff663b994a --- /dev/null +++ b/users/Profpatsch/read-http.rs @@ -0,0 +1,211 @@ +extern crate httparse; +extern crate netencode; +extern crate arglib_netencode; +extern crate ascii; +extern crate exec_helpers; + +use std::os::unix::io::FromRawFd; +use std::io::Read; +use std::io::Write; +use std::collections::HashMap; +use exec_helpers::{die_user_error, die_expected_error, die_temporary}; + +use netencode::{U, T, dec}; +use netencode::dec::Decoder; + +enum What { + Request, + Response +} + +// reads a http request (stdin), and writes all headers to stdout, as netencoded record. +// The keys are text, but can be lists of text iff headers appear multiple times, so beware. +fn main() -> std::io::Result<()> { + + exec_helpers::no_args("read-http"); + + let args = dec::RecordDot { + field: "what", + inner: dec::OneOf { + list: vec!["request", "response"], + inner: dec::Text + } + }; + let what : What = match args.dec(arglib_netencode::arglib_netencode("read-http", None).to_u()) { + Ok("request") => What::Request, + Ok("response") => What::Response, + Ok(v) => panic!("shouldn’t happen!, value was: {}", v), + Err(dec::DecodeError(err)) => die_user_error("read-http", err), + }; + + fn read_stdin_to_complete<F>(mut parse: F) -> () + where F: FnMut(&[u8]) -> httparse::Result<usize> + { + let mut res = httparse::Status::Partial; + loop { + if let httparse::Status::Complete(_) = res { + return; + } + let mut buf = [0; 2048]; + match std::io::stdin().read(&mut buf[..]) { + Ok(size) => if size == 0 { + break; + }, + Err(err) => die_temporary("read-http", format!("could not read from stdin, {:?}", err)) + } + match parse(&buf) { + Ok(status) => { + res = status; + } + Err(err) => die_temporary("read-http", format!("httparse parsing failed: {:#?}", err)) + } + } + } + + fn normalize_headers<'a>(headers: &'a [httparse::Header]) -> HashMap<String, U<'a>> { + let mut res = HashMap::new(); + for httparse::Header { name, value } in headers { + let val = ascii::AsciiStr::from_ascii(*value) + .expect(&format!("read-http: we require header values to be ASCII, but the header {} was {:?}", name, value)) + .as_str(); + // lowercase the header names, since the standard doesn’t care + // and we want unique strings to match against + let name_lower = name.to_lowercase(); + match res.insert(name_lower, U::Text(val)) { + None => (), + Some(U::Text(t)) => { + let name_lower = name.to_lowercase(); + let _ = res.insert(name_lower, U::List(vec![U::Text(t), U::Text(val)])); + () + }, + Some(U::List(mut l)) => { + let name_lower = name.to_lowercase(); + l.push(U::Text(val)); + let _ = res.insert(name_lower, U::List(l)); + () + }, + Some(o) => panic!("read-http: header not text nor list: {:?}", o), + } + } + res + } + + // tries to read until the end of the http header (deliniated by two newlines "\r\n\r\n") + fn read_till_end_of_header<R: Read>(buf: &mut Vec<u8>, reader: R) -> Option<()> { + let mut chonker = Chunkyboi::new(reader, 4096); + loop { + // TODO: attacker can send looooong input, set upper maximum + match chonker.next() { + Some(Ok(chunk)) => { + buf.extend_from_slice(&chunk); + if chunk.windows(4).any(|c| c == b"\r\n\r\n" ) { + return Some(()); + } + }, + Some(Err(err)) => die_temporary("read-http", format!("error reading from stdin: {:?}", err)), + None => return None + } + } + } + + // max header size chosen arbitrarily + let mut headers = [httparse::EMPTY_HEADER; 128]; + let stdin = std::io::stdin(); + + match what { + Request => { + let mut req = httparse::Request::new(&mut headers); + let mut buf: Vec<u8> = vec![]; + match read_till_end_of_header(&mut buf, stdin.lock()) { + Some(()) => match req.parse(&buf) { + Ok(httparse::Status::Complete(_body_start)) => {}, + Ok(httparse::Status::Partial) => die_expected_error("read-http", "httparse should have gotten a full header"), + Err(err) => die_expected_error("read-http", format!("httparse response parsing failed: {:#?}", err)) + }, + None => die_expected_error("read-http", format!("httparse end of stdin reached before able to parse request headers")) + } + let method = req.method.expect("method must be filled on complete parse"); + let path = req.path.expect("path must be filled on complete parse"); + write_dict_req(method, path, &normalize_headers(req.headers)) + }, + Response => { + let mut resp = httparse::Response::new(&mut headers); + let mut buf: Vec<u8> = vec![]; + match read_till_end_of_header(&mut buf, stdin.lock()) { + Some(()) => match resp.parse(&buf) { + Ok(httparse::Status::Complete(_body_start)) => {}, + Ok(httparse::Status::Partial) => die_expected_error("read-http", "httparse should have gotten a full header"), + Err(err) => die_expected_error("read-http", format!("httparse response parsing failed: {:#?}", err)) + }, + None => die_expected_error("read-http", format!("httparse end of stdin reached before able to parse response headers")) + } + let code = resp.code.expect("code must be filled on complete parse"); + let reason = resp.reason.expect("reason must be filled on complete parse"); + write_dict_resp(code, reason, &normalize_headers(resp.headers)) + } + } +} + +fn write_dict_req<'a, 'buf>(method: &'buf str, path: &'buf str, headers: &'a HashMap<String, U<'a>>) -> std::io::Result<()> { + let mut http = vec![ + ("method", U::Text(method)), + ("path", U::Text(path)), + ].into_iter().collect(); + write_dict(http, headers) +} + +fn write_dict_resp<'a, 'buf>(code: u16, reason: &'buf str, headers: &'a HashMap<String, U<'a>>) -> std::io::Result<()> { + let mut http = vec![ + ("status", U::N6(code as u64)), + ("status-text", U::Text(reason)), + ].into_iter().collect(); + write_dict(http, headers) +} + + +fn write_dict<'buf, 'a>(mut http: HashMap<&str, U<'a>>, headers: &'a HashMap<String, U<'a>>) -> std::io::Result<()> { + match http.insert("headers", U::Record( + headers.iter().map(|(k,v)| (k.as_str(), v.clone())).collect() + )) { + None => (), + Some(_) => panic!("read-http: headers already in dict"), + }; + netencode::encode( + &mut std::io::stdout(), + &U::Record(http) + )?; + Ok(()) +} + + +// iter helper + +struct Chunkyboi<T> { + inner: T, + buf: Vec<u8>, +} + +impl<R: Read> Chunkyboi<R> { + fn new(inner: R, chunksize: usize) -> Self { + let buf = vec![0; chunksize]; + Chunkyboi { + inner, + buf + } + } +} + +impl<R: Read> Iterator for Chunkyboi<R> { + type Item = std::io::Result<Vec<u8>>; + + fn next(&mut self) -> Option<std::io::Result<Vec<u8>>> { + match self.inner.read(&mut self.buf) { + Ok(0) => None, + Ok(read) => { + // clone a new buffer so we can reuse the internal one + Some(Ok(self.buf[..read].to_owned())) + } + Err(err) => Some(Err(err)) + } + } +} diff --git a/users/Profpatsch/reverse-haskell-deps.hs b/users/Profpatsch/reverse-haskell-deps.hs new file mode 100644 index 000000000000..6b644df9ecc6 --- /dev/null +++ b/users/Profpatsch/reverse-haskell-deps.hs @@ -0,0 +1,72 @@ +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE ScopedTypeVariables #-} +import qualified Text.HTML.TagSoup as Tag +import qualified Data.Text as Text +import Data.Text (Text) +import qualified Data.List as List +import Data.Maybe +import Text.Nicify +import qualified Text.Read as Read +import Numeric.Natural +import Data.Either +import qualified Data.ByteString as ByteString +import qualified Data.Text.Encoding + +parseNat :: Text.Text -> Maybe Natural +parseNat = Read.readMaybe . Text.unpack + +printNice :: Show a => a -> IO () +printNice = putStrLn . nicify . show + +type Tag = Tag.Tag Text.Text + +main = do + reverseHtml <- readStdinUtf8 + printNice $ List.sortOn snd $ packagesAndReverseDeps reverseHtml + + where + readStdinUtf8 = Data.Text.Encoding.decodeUtf8 <$> ByteString.getContents + +-- | reads the table provided by https://packdeps.haskellers.com/reverse +-- figuring out all sections (starting with the link to the package name), +-- then figuring out the name of the package and the first column, +-- which is the number of reverse dependencies of the package +packagesAndReverseDeps reverseHtml = do + let tags = Tag.parseTags reverseHtml + let sections = Tag.partitions (isJust . reverseLink) tags + let sectionNames = map (fromJust . reverseLink . head) sections + mapMaybe + (\(name :: Text.Text, sect) -> do + reverseDeps <- firstNaturalNumber sect + pure (sectionPackageName name sect, reverseDeps) :: Maybe (Text.Text, Natural)) + $ zip sectionNames sections + + + where + reverseLink = \case + Tag.TagOpen "a" attrs -> mapFind attrReverseLink attrs + _ -> Nothing + + attrReverseLink = \case + ("href", lnk) -> if + | "packdeps.haskellers.com/reverse/" `Text.isInfixOf` lnk -> Just lnk + | otherwise -> Nothing + _ -> Nothing + + sectionPackageName :: Text -> [Tag] -> Text + sectionPackageName sectionName = \case + (_: Tag.TagText name : _) -> name + (_: el : _) -> sectionName + xs -> sectionName + + + firstNaturalNumber :: [Tag] -> Maybe Natural + firstNaturalNumber = + mapFind (\case + Tag.TagText t -> parseNat t + _ -> Nothing) + + mapFind :: (a -> Maybe b) -> [a] -> Maybe b + mapFind f xs = fromJust . f <$> List.find (isJust . f) xs diff --git a/users/Profpatsch/reverse-haskell-deps.nix b/users/Profpatsch/reverse-haskell-deps.nix new file mode 100644 index 000000000000..b47347ea9fea --- /dev/null +++ b/users/Profpatsch/reverse-haskell-deps.nix @@ -0,0 +1,26 @@ +{ depot, pkgs, ... }: + +# Parses https://packdeps.haskellers.com/reverse +# and outputs the amount of reverse dependencies of each hackage package. + +let + + rev = depot.nix.writeExecline "reverse-haskell-deps" {} [ + "pipeline" [ + "${pkgs.curl}/bin/curl" "-L" "https://packdeps.haskellers.com/reverse" + ] + rev-hs + + ]; + + rev-hs = pkgs.writers.writeHaskell "revers-haskell-deps-hs" { + libraries = [ + pkgs.haskellPackages.nicify-lib + pkgs.haskellPackages.tagsoup + ]; + + } + ./reverse-haskell-deps.hs; + + +in rev diff --git a/users/Profpatsch/struct-edit/default.nix b/users/Profpatsch/struct-edit/default.nix new file mode 100644 index 000000000000..970cdd4d028b --- /dev/null +++ b/users/Profpatsch/struct-edit/default.nix @@ -0,0 +1,13 @@ +{ depot, ... }: +depot.nix.buildGo.program { + name = "struct-edit"; + srcs = [ + ./main.go + ]; + deps = [ + depot.third_party.gopkgs."github.com".charmbracelet.bubbletea + depot.third_party.gopkgs."github.com".charmbracelet.lipgloss + depot.third_party.gopkgs."github.com".muesli.termenv + depot.third_party.gopkgs."github.com".mattn.go-isatty + ]; +} diff --git a/users/Profpatsch/struct-edit/main.go b/users/Profpatsch/struct-edit/main.go new file mode 100644 index 000000000000..7e43074266d9 --- /dev/null +++ b/users/Profpatsch/struct-edit/main.go @@ -0,0 +1,431 @@ +package main + +import ( + json "encoding/json" + "fmt" + "log" + "os" + "strings" + "sort" + + tea "github.com/charmbracelet/bubbletea" + lipgloss "github.com/charmbracelet/lipgloss" + // termenv "github.com/muesli/termenv" + // isatty "github.com/mattn/go-isatty" +) + +// Keeps the full data structure and a path that indexes our current position into it. +type model struct { + path []index + data val +} + +// an index into a value, uint for lists and string for maps. +// nil for any scalar value. +// TODO: use an actual interface for these +type index interface{} + +/// recursive value that we can represent. +type val struct { + // the “type” of value; see tag const belove + tag tag + // last known position of our cursor + last_index index + // documentation (TODO) + doc string + // the actual value; + // the actual structure is behind a pointer so we can replace the struct. + // determined by the tag + // tagString -> *string + // tagFloat -> *float64 + // tagList -> *[]val + // tagMap -> *map[string]val + val interface{} +} + +type tag string + +const ( + tagString tag = "string" + tagFloat tag = "float" + tagList tag = "list" + tagMap tag = "map" +) + +// print a value, flat +func (v val) Render() string { + s := "" + switch v.tag { + case tagString: + s += *v.val.(*string) + case tagFloat: + s += fmt.Sprint(*v.val.(*float64)) + case tagList: + s += "[ " + vs := []string{} + for _, enum := range v.enumerate() { + vs = append(vs, enum.v.Render()) + } + s += strings.Join(vs, ", ") + s += " ]" + case tagMap: + s += "{ " + vs := []string{} + for _, enum := range v.enumerate() { + vs = append(vs, fmt.Sprintf("%s: %s", enum.i.(string), enum.v.Render())) + } + s += strings.Join(vs, ", ") + s += " }" + default: + s += fmt.Sprintf("<unknown: %v>", v) + } + return s +} + +// render an index, depending on the type +func renderIndex(i index) (s string) { + switch i := i.(type) { + case nil: + s = "" + // list index + case uint: + s = "*" + // map index + case string: + s = i + ":" + } + return +} + +// take an arbitrary (within restrictions) go value and construct a val from it +func makeVal(i interface{}) val { + var v val + switch i := i.(type) { + case string: + v = val{ + tag: tagString, + last_index: index(nil), + doc: "", + val: &i, + } + case float64: + v = val{ + tag: tagFloat, + last_index: index(nil), + doc: "", + val: &i, + } + case []interface{}: + ls := []val{} + for _, i := range i { + ls = append(ls, makeVal(i)) + } + v = val{ + tag: tagList, + last_index: pos1Inner(tagList, &ls), + doc: "", + val: &ls, + } + case map[string]interface{}: + ls := map[string]val{} + for k, i := range i { + ls[k] = makeVal(i) + } + v = val{ + tag: tagMap, + last_index: pos1Inner(tagMap, &ls), + doc: "", + val: &ls, + } + default: + log.Fatalf("makeVal: cannot read json of type %T", i) + } + return v +} + +// return an index that points at the first entry in val +func (v val) pos1() index { + return v.enumerate()[0].i +} + +func pos1Inner(tag tag, v interface{}) index { + return enumerateInner(tag, v)[0].i +} + +type enumerate struct { + i index + v val +} + +// enumerate gives us a stable ordering of elements in this val. +// for scalars it’s just a nil index & the val itself. +// Guaranteed to always return at least one element. +func (v val) enumerate() (e []enumerate) { + e = enumerateInner(v.tag, v.val) + if e == nil { + e = append(e, enumerate{ + i: nil, + v: v, + }) + } + return +} + +// like enumerate, but returns an empty slice for scalars without inner vals. +func enumerateInner(tag tag, v interface{}) (e []enumerate) { + switch tag { + case tagString: + fallthrough + case tagFloat: + e = nil + case tagList: + for i, v := range *v.(*[]val) { + e = append(e, enumerate{i: index(uint(i)), v: v}) + } + case tagMap: + // map sorting order is not stable (actually randomized thank jabber) + // so let’s sort them + keys := []string{} + m := *v.(*map[string]val) + for k, _ := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + e = append(e, enumerate{i: index(k), v: m[k]}) + } + default: + log.Fatalf("unknown val tag %s, %v", tag, v) + } + return +} + +func (m model) PathString() string { + s := "/ " + var is []string + for _, v := range m.path { + is = append(is, fmt.Sprintf("%v", v)) + } + s += strings.Join(is, " / ") + return s +} + +// walk the given path down in data, to get the value at that point. +// Assumes that all path indexes are valid indexes into data. +// Returns a pointer to the value at point, in order to be able to change it. +func walk(data *val, path []index) (*val, bool, error) { + res := data + atPath := func(index int) string { + return fmt.Sprintf("at path %v", path[:index+1]) + } + errf := func(ty string, val interface{}, index int) error { + return fmt.Errorf("walk: can’t walk into %s %v %s", ty, val, atPath(index)) + } + for i, p := range path { + switch res.tag { + case tagString: + return nil, true, nil + case tagFloat: + return nil, true, nil + case tagList: + switch p := p.(type) { + case uint: + list := *res.val.(*[]val) + if int(p) >= len(list) || p < 0 { + return nil, false, fmt.Errorf("index out of bounds %s", atPath(i)) + } + res = &list[p] + default: + return nil, false, fmt.Errorf("not a list index %s", atPath(i)) + } + case tagMap: + switch p := p.(type) { + case string: + m := *res.val.(*map[string]val) + if a, ok := m[p]; ok { + res = &a + } else { + return nil, false, fmt.Errorf("index %s not in map %s", p, atPath(i)) + } + default: + return nil, false, fmt.Errorf("not a map index %v %s", p, atPath(i)) + } + + default: + return nil, false, errf(string(res.tag), res.val, i) + } + } + return res, false, nil +} + +// descend into the selected index. Assumes that the index is valid. +// Will not descend into scalars. +func (m model) descend() (model, error) { + // TODO: two walks?! + this, _, err := walk(&m.data, m.path) + if err != nil { + return m, err + } + newPath := append(m.path, this.last_index) + _, bounce, err := walk(&m.data, newPath) + if err != nil { + return m, err + } + // only descend if we *can* + if !bounce { + m.path = newPath + } + return m, nil +} + +// ascend to one level up. stops at the root. +func (m model) ascend() (model, error) { + if len(m.path) > 0 { + m.path = m.path[:len(m.path)-1] + _, _, err := walk(&m.data, m.path) + return m, err + } + return m, nil +} + +/// go to the next item, or wraparound +func (min model) next() (m model, err error) { + m = min + this, _, err := walk(&m.data, m.path) + if err != nil { + return + } + enumL := this.enumerate() + setNext := false + for _, enum := range enumL { + if setNext { + this.last_index = enum.i + setNext = false + break + } + if enum.i == this.last_index { + setNext = true + } + } + // wraparound + if setNext { + this.last_index = enumL[0].i + } + return +} + +/// go to the previous item, or wraparound +func (min model) prev() (m model, err error) { + m = min + this, _, err := walk(&m.data, m.path) + if err != nil { + return + } + enumL := this.enumerate() + // last element, wraparound + prevIndex := enumL[len(enumL)-1].i + for _, enum := range enumL { + if enum.i == this.last_index { + this.last_index = prevIndex + break + } + prevIndex = enum.i + } + return +} + +/// bubbletea implementations + +func (m model) Init() tea.Cmd { + return nil +} + +func initialModel(v interface{}) model { + val := makeVal(v) + return model{ + path: []index{}, + data: val, + } +} + +func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + var err error + switch msg := msg.(type) { + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "q": + return m, tea.Quit + + case "up": + m, err = m.prev() + + case "down": + m, err = m.next() + + case "right": + m, err = m.descend() + + case "left": + m, err = m.ascend() + + // case "enter": + // _, ok := m.selected[m.cursor] + // if ok { + // delete(m.selected, m.cursor) + // } else { + // m.selected[m.cursor] = struct{}{} + // } + } + + } + if err != nil { + log.Fatal(err) + } + return m, nil +} + +var pathColor = lipgloss.NewStyle(). + // light blue + Foreground(lipgloss.Color("12")) + +var selectedColor = lipgloss.NewStyle(). + Bold(true) + +func (m model) View() string { + s := pathColor.Render(m.PathString()) + cur, _, err := walk(&m.data, m.path) + if err != nil { + log.Fatal(err) + } + s += cur.doc + "\n" + s += "\n" + for _, enum := range cur.enumerate() { + is := renderIndex(enum.i) + if is != "" { + s += is + " " + } + if enum.i == cur.last_index { + s += selectedColor.Render(enum.v.Render()) + } else { + s += enum.v.Render() + } + s += "\n" + } + + // s += fmt.Sprintf("%v\n", m) + // s += fmt.Sprintf("%v\n", cur) + + return s +} + +func main() { + var input interface{} + err := json.NewDecoder(os.Stdin).Decode(&input) + if err != nil { + log.Fatal("json from stdin: ", err) + } + p := tea.NewProgram(initialModel(input)) + if err := p.Start(); err != nil { + log.Fatal("bubbletea TUI error: ", err) + } +} diff --git a/users/Profpatsch/tree-sitter.nix b/users/Profpatsch/tree-sitter.nix new file mode 100644 index 000000000000..4f81b8e7a77c --- /dev/null +++ b/users/Profpatsch/tree-sitter.nix @@ -0,0 +1,179 @@ +{ depot, pkgs, lib, ... }: + +let + bins = depot.nix.getBins pkgs.coreutils [ "head" "printf" "cat" ] + // depot.nix.getBins pkgs.ncurses [ "tput" ] + // depot.nix.getBins pkgs.bc [ "bc" ] + // depot.nix.getBins pkgs.ocamlPackages.sexp [ "sexp" ]; + + print-ast = depot.nix.writers.rustSimple { + name = "print-ast"; + dependencies = with depot.third_party.rust-crates; [ + libloading + tree-sitter + ]; + } '' + extern crate libloading; + extern crate tree_sitter; + use std::mem; + use std::io::{Read}; + use libloading::{Library, Symbol}; + use tree_sitter::{Language, Parser}; + + /// Load the shared lib FILE and return the language under SYMBOL-NAME. + /// Inspired by the rust source of emacs-tree-sitter. + fn _load_language(file: String, symbol_name: String) -> Result<Language, libloading::Error> { + let lib = Library::new(file)?; + let tree_sitter_lang: Symbol<'_, unsafe extern "C" fn() -> _> = + unsafe { lib.get(symbol_name.as_bytes())? }; + let language: Language = unsafe { tree_sitter_lang() }; + // Avoid segmentation fault by not unloading the lib, as language is a static piece of data. + // TODO: Attach an Rc<Library> to Language instead. + mem::forget(lib); + Ok(language) + } + + fn main() { + let mut args = std::env::args(); + let so = args.nth(1).unwrap(); + let symbol_name = args.nth(0).unwrap(); + let file = args.nth(0).unwrap(); + let mut parser = Parser::new(); + let lang = _load_language(so, symbol_name).unwrap(); + parser.set_language(lang).unwrap(); + let bytes = std::fs::read(&file).unwrap(); + print!("{}", parser.parse(&bytes, None).unwrap().root_node().to_sexp()); + } + + + ''; + + tree-sitter-nix = buildTreeSitterGrammar { + language = "tree-sitter-nix"; + source = pkgs.fetchFromGitHub { + owner = "cstrahan"; + repo = "tree-sitter-nix"; + rev = "791b5ff0e4f0da358cbb941788b78d436a2ca621"; + sha256 = "1y5b3wh3fcmbgq8r2i97likzfp1zp02m58zacw5a1cjqs5raqz66"; + }; + }; + + watch-file-modified = depot.nix.writers.rustSimple { + name = "watch-file-modified"; + dependencies = [ + depot.third_party.rust-crates.inotify + depot.users.Profpatsch.netstring.rust-netstring + ]; + } '' + extern crate inotify; + extern crate netstring; + use inotify::{EventMask, WatchMask, Inotify}; + use std::io::Write; + + fn main() { + let mut inotify = Inotify::init() + .expect("Failed to initialize inotify"); + + let file = std::env::args().nth(1).unwrap(); + + let file_watch = inotify + .add_watch( + &file, + WatchMask::MODIFY + ) + .expect("Failed to add inotify watch"); + + let mut buffer = [0u8; 4096]; + loop { + let events = inotify + .read_events_blocking(&mut buffer) + .expect("Failed to read inotify events"); + + for event in events { + if event.wd == file_watch { + std::io::stdout().write(&netstring::to_netstring(file.as_bytes())); + std::io::stdout().flush(); + } + } + } + } + + ''; + + # clear screen and set LINES and COLUMNS to terminal height & width + clear-screen = depot.nix.writeExecline "clear-screen" {} [ + "if" [ bins.tput "clear" ] + "backtick" "-in" "LINES" [ bins.tput "lines" ] + "backtick" "-in" "COLUMNS" [ bins.tput "cols" ] + "$@" + ]; + + print-nix-file = depot.nix.writeExecline "print-nix-file" { readNArgs = 1; } [ + "pipeline" [ print-ast "${tree-sitter-nix}/parser" "tree_sitter_nix" "$1" ] + "pipeline" [ bins.sexp "print" ] + clear-screen + "importas" "-ui" "lines" "LINES" + "backtick" "-in" "ls" [ + "pipeline" + # when you pull out bc to decrement an integer it’s time to switch to python lol + [ bins.printf "x=%s; --x\n" "$lines" ] + bins.bc + ] + "importas" "-ui" "l" "ls" + bins.head "-n\${l}" + ]; + + print-nix-file-on-update = depot.nix.writeExecline "print-nix-file-on-update" { readNArgs = 1; } [ + "if" [ print-nix-file "$1" ] + "pipeline" [ watch-file-modified "$1" ] + "forstdin" "-d" "" "file" + "importas" "file" "file" + print-nix-file "$file" + ]; + + # copied from nixpkgs + buildTreeSitterGrammar = + { + # language name + language + # source for the language grammar + , source + }: + + pkgs.stdenv.mkDerivation { + + pname = "${language}-grammar"; + inherit (pkgs.tree-sitter) version; + + src = source; + + buildInputs = [ pkgs.tree-sitter ]; + + dontUnpack = true; + configurePhase= ":"; + buildPhase = '' + runHook preBuild + scanner_cc="$src/src/scanner.cc" + if [ ! -f "$scanner_cc" ]; then + scanner_cc="" + fi + $CXX -I$src/src/ -c $scanner_cc + $CC -I$src/src/ -shared -o parser -Os scanner.o $src/src/parser.c -lstdc++ + runHook postBuild + ''; + installPhase = '' + runHook preInstall + mkdir $out + mv parser $out/ + runHook postInstall + ''; + }; + +in depot.nix.readTree.drvTargets { + inherit + print-ast + tree-sitter-nix + print-nix-file-on-update + watch-file-modified + ; +} diff --git a/users/Profpatsch/writers/default.nix b/users/Profpatsch/writers/default.nix new file mode 100644 index 000000000000..3151a9d3bd44 --- /dev/null +++ b/users/Profpatsch/writers/default.nix @@ -0,0 +1,76 @@ +{ depot, pkgs, lib, ... }: +let + bins = depot.nix.getBins pkgs.s6-portable-utils ["s6-mkdir" "s6-cat" "s6-ln" "s6-ls" "s6-touch" ] + // depot.nix.getBins pkgs.coreutils ["printf" ]; + + inherit (depot.nix.yants) defun struct restrict attrs list string drv any; + + inherit (depot.nix) drvSeqL; + + FlakeError = + restrict + "flake error" + (s: lib.any (prefix: (builtins.substring 0 1 s) == prefix) + [ "E" "W" ]) + string; + Libraries = defun [ (attrs any) (list drv) ]; + + python3 = { + name, + libraries ? (_: []), + flakeIgnore ? [] + }: pkgs.writers.writePython3 name { + libraries = Libraries libraries pkgs.python3Packages; + flakeIgnore = + let ignoreTheseErrors = [ + # whitespace after { + "E201" + # whitespace before } + "E202" + # fuck 4-space indentation + "E121" "E111" + # who cares about blank lines … + # … at end of files + "W391" + # … between functions + "E302" "E305" + ]; + in list FlakeError (ignoreTheseErrors ++ flakeIgnore); + }; + + # TODO: add the same flake check as the pyhon3 writer + python3Lib = { name, libraries ? (_: []) }: moduleString: + let srcTree = depot.nix.runExecline.local name { stdin = moduleString; } [ + "importas" "out" "out" + "if" [ bins.s6-mkdir "-p" "\${out}/${name}" ] + "if" [ + "redirfd" "-w" "1" "\${out}/setup.py" + bins.printf '' + from distutils.core import setup + + setup( + name='%s', + packages=['%s'] + ) + '' name name + ] + "if" [ + # redirect stdin to the init py + "redirfd" "-w" "1" "\${out}/${name}/__init__.py" + bins.s6-cat + ] + ]; + in pkgs.python3Packages.buildPythonPackage { + inherit name; + src = srcTree; + propagatedBuildInputs = libraries pkgs.python3Packages; + doCheck = false; + }; + + +in { + inherit + python3 + python3Lib + ; +} diff --git a/users/Profpatsch/writers/tests/default.nix b/users/Profpatsch/writers/tests/default.nix new file mode 100644 index 000000000000..dc760af9e16e --- /dev/null +++ b/users/Profpatsch/writers/tests/default.nix @@ -0,0 +1,48 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.users.Profpatsch.writers) + python3Lib + python3 + ; + + inherit (pkgs) + coreutils + ; + + run = drv: depot.nix.runExecline.local "run-${drv.name}" {} [ + "if" [ drv ] + "importas" "out" "out" + "${coreutils}/bin/touch" "$out" + ]; + + pythonTransitiveLib = python3Lib { + name = "transitive"; + } '' + def transitive(s): + return s + " 1 2 3" + ''; + + pythonTestLib = python3Lib { + name = "test_lib"; + libraries = _: [ pythonTransitiveLib ]; + } '' + import transitive + def test(): + return transitive.transitive("test") + ''; + + pythonWithLib = run (python3 { + name = "python-with-lib"; + libraries = _: [ pythonTestLib ]; + } '' + import test_lib + + assert(test_lib.test() == "test 1 2 3") + ''); + +in depot.nix.readTree.drvTargets { + inherit + pythonWithLib + ; +} diff --git a/users/cynthia/OWNERS b/users/cynthia/OWNERS new file mode 100644 index 000000000000..da62f3777af0 --- /dev/null +++ b/users/cynthia/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - cynthia diff --git a/users/cynthia/keys.nix b/users/cynthia/keys.nix new file mode 100644 index 000000000000..bac8dc1c57ae --- /dev/null +++ b/users/cynthia/keys.nix @@ -0,0 +1,7 @@ +{ ... }: + +{ + all = [ + "cert-authority ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICsj3W6QczgxE3s5GGT8qg0aLrCM+QeRnSq9RkiZtKvz meow" + ]; +} \ No newline at end of file diff --git a/users/edef/OWNERS b/users/edef/OWNERS new file mode 100644 index 000000000000..05f7639c8947 --- /dev/null +++ b/users/edef/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - edef diff --git a/users/edef/depot-scan/default.nix b/users/edef/depot-scan/default.nix new file mode 100644 index 000000000000..a9c0f382ff45 --- /dev/null +++ b/users/edef/depot-scan/default.nix @@ -0,0 +1,12 @@ +{ pkgs, ... }: + +pkgs.writeShellScriptBin "depot-scan" '' + set -euo pipefail + + path="''${1:-$(git rev-parse --show-prefix)}" + path="''${path%%/}" + attr="''${path//\//.}" + root="$(git rev-parse --show-toplevel)" + echo "scanning //$path" >&2 + nix-instantiate -E "import ${./wrap.nix} $root" -A "$attr" -vv 2> >(${pkgs.perl}/bin/perl ${./depot-scan.pl}) >&2 +'' diff --git a/users/edef/depot-scan/depot-scan.pl b/users/edef/depot-scan/depot-scan.pl new file mode 100755 index 000000000000..8808e2eb0023 --- /dev/null +++ b/users/edef/depot-scan/depot-scan.pl @@ -0,0 +1,11 @@ +#! /usr/bin/env -S perl -ln +use strict; + +if (/^evaluating file '(.*)'$/ or + /^copied source '(.*)' -> '.*'$/ or + /^trace: depot-scan '(.*)'$/) { + print $1; + next; +} + +print STDERR unless /^instantiated '.*' -> '.*'$/; diff --git a/users/edef/depot-scan/wrap.nix b/users/edef/depot-scan/wrap.nix new file mode 100644 index 000000000000..dcb557a24b1e --- /dev/null +++ b/users/edef/depot-scan/wrap.nix @@ -0,0 +1,15 @@ +# this wraps import to override readFile and readDir to trace the files it touches +# technique inspired by lorri +let + + global = { + import = global.scopedImport {}; + scopedImport = x: builtins.scopedImport (global // x); + builtins = builtins // { + inherit (global) import scopedImport; + readFile = path: builtins.trace "depot-scan '${toString path}'" (builtins.readFile path); + readDir = path: builtins.trace "depot-scan '${toString path}'" (builtins.readDir path); + }; + }; + +in global.import diff --git a/users/edef/keys.nix b/users/edef/keys.nix new file mode 100644 index 000000000000..53e88c9e7345 --- /dev/null +++ b/users/edef/keys.nix @@ -0,0 +1,7 @@ +{ ... }: + +{ + all = [ + "cert-authority ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCvb/7ojfcbKvHIyjnrNUOOgzy44tCkgXY9HLuyFta1jQOE9pFIK19B4dR9bOglPKf145CCL0mSFJNNqmNwwavU2uRn+TQrW+U1dQAk8Gt+gh3O49YE854hwwyMU+xD6bIuUdfxPr+r5al/Ov5Km28ZMlHOs3FoAP0hInK+eAibioxL5rVJOtgicrOVCkGoXEgnuG+LRbOYTwzdClhRUxiPjK8alCbcJQ53AeZHO4G6w9wTr+W5ILCfvW4OmUXCX01sKzaBiQuuFCF6M/H4LlnsPWLMra2twXxkOIhZblwC+lncps9lQaUgiD4koZeOCORvHW00G0L39ilFbbnVcL6Itp/m8RRWm/xRxS4RMnsdV/AhvpRLrhL3lfQ7E2oCeSM36v1S9rdg6a47zcnpL+ahG76Gz39Y7KmVRQciNx7ezbwxj3Q5lZtFykgdfGIAN+bT8ijXMO6m68g60i9Bz4IoMZGkiJGqMYLTxMQ+oRgR3Ro5lbj7E11YBHyeimoBYXYGHMkiuxopQZ7lIj3plxIzhmUlXJBA4jMw9KGHdYaLhaicIYhvQmCTAjrkt2HvxEe6lU8iws2Qv+pB6tAGundN36RVVWAckeQPZ4ZsgDP8V2FfibZ1nsrQ+zBKqaslYMAHs01Cf0Hm0PnCqagf230xaobu0iooNuXx44QKoDnB+w== openpgp:0x803010E7" + ]; +} diff --git a/users/ericvolp12/OWNERS b/users/ericvolp12/OWNERS new file mode 100644 index 000000000000..5a012a695bf3 --- /dev/null +++ b/users/ericvolp12/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - ericvolp12 diff --git a/users/eta/OWNERS b/users/eta/OWNERS new file mode 100644 index 000000000000..f212e89e2ae9 --- /dev/null +++ b/users/eta/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - eta diff --git a/users/eta/keys.nix b/users/eta/keys.nix new file mode 100644 index 000000000000..247a182843f1 --- /dev/null +++ b/users/eta/keys.nix @@ -0,0 +1,12 @@ +{ ... }: + +let + keys = { + yubikey4 = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQD6E1wuWaXQARNoLnmlOJndwI7/ms3Ga7MJxsUvFtaSiy3g8h/hz4WgyR7YT+hUYjFihh/YkGS9Zy9aEqAa5zBGLcZtgj1O0qOl2joynm679zdlcwAart74fXSJYYupT9tFeXXeWLO1g054lVJ5xZ9KLpBBk+6yzlmmm5KuoitKBqBbadzsqAeKhNn1Nq9ITPU4vxTFk+sXp/nxk/JoUOM8S2N4YuoX9OVenDHKh9DtOcvDZhlosGmunO33/YaU2XB95ZE6cNhEtVlkbyR3a2SsAYz1qGgfH0HSyoK3LJoAM4Aiz99ktuKiI/zMy4k4TV00OCi1sCPEjzUoijZRZt5FMH/TVr9dJROVjHcL9g9//fW3jwqojf7uuJFlTJb47RxjTk4Jb4F6K7HhOs7bgh3WuOjvhyRYbCYcg+RfnwjJk+hfM5GcjZ8J4UZdNc5LyIcfH8W1v9DADBCgz7QcmfrfMloYtEgjK/5XVrtBtiMtUOgpfKujawF55d1Vj26+CxeID8NHMXzZYEMeyRpi/WXlC+lq1Wx4Fj8gvideOw/3gAdj2G3SJWdSPk8XpIFQ1fm3tXB0ltyV5TszIJhfMnmsKJeEm3YlTCR1sMW7nr3wEdMqa6mpcWZTWU+dppmAGr2c+OGSnXkCi7Z2h/YJE6X+izrOrqRspG2fCM8GlfRFWw== cardno:000607469311"; + yubikey5 = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKCJx23px0Vknw1NlD+arcqeVXxcogPUMJgF/PGp6wA/tg7hHUKs2udC+gDMYlxQ9IpnWOwZ/9yvqzTDwUU3R/4= YubiKey #15026444 PIV Slot 9a"; + }; + configs = { + whitby = [ keys.yubikey4 keys.yubikey5 ]; + }; +in + configs diff --git a/users/firefly/OWNERS b/users/firefly/OWNERS new file mode 100644 index 000000000000..55d62a5723a5 --- /dev/null +++ b/users/firefly/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - firefly diff --git a/users/firefly/keys.nix b/users/firefly/keys.nix new file mode 100644 index 000000000000..1d7467a0747c --- /dev/null +++ b/users/firefly/keys.nix @@ -0,0 +1,7 @@ +{ ... }: + +rec { + as = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN9i8fs10/BjNEqFXD+3fQeQ0SuHnQx4WpuqUg4caeed firefly@as"; + + whitby = [ as ]; +} diff --git a/users/flokli/OWNERS b/users/flokli/OWNERS new file mode 100644 index 000000000000..63e0fbda3c7f --- /dev/null +++ b/users/flokli/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - flokli diff --git a/users/flokli/keys.nix b/users/flokli/keys.nix new file mode 100644 index 000000000000..790c9862f824 --- /dev/null +++ b/users/flokli/keys.nix @@ -0,0 +1,7 @@ +{ ... }: + +{ + all = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTVTXOutUZZjXLB0lUSgeKcSY/8mxKkC0ingGK1whD2 flokli" + ]; +} diff --git a/users/grfn/OWNERS b/users/grfn/OWNERS new file mode 100644 index 000000000000..da7ac5cb9ef9 --- /dev/null +++ b/users/grfn/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - grfn diff --git a/users/grfn/achilles/.envrc b/users/grfn/achilles/.envrc new file mode 100644 index 000000000000..051d09d292a8 --- /dev/null +++ b/users/grfn/achilles/.envrc @@ -0,0 +1 @@ +eval "$(lorri direnv)" diff --git a/users/grfn/achilles/.gitignore b/users/grfn/achilles/.gitignore new file mode 100644 index 000000000000..ea8c4bf7f35f --- /dev/null +++ b/users/grfn/achilles/.gitignore @@ -0,0 +1 @@ +/target diff --git a/users/grfn/achilles/Cargo.lock b/users/grfn/achilles/Cargo.lock new file mode 100644 index 000000000000..30e5e021f147 --- /dev/null +++ b/users/grfn/achilles/Cargo.lock @@ -0,0 +1,868 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "achilles" +version = "0.1.0" +dependencies = [ + "anyhow", + "bimap", + "clap", + "crate-root", + "derive_more", + "inkwell", + "itertools", + "lazy_static", + "llvm-sys", + "nom", + "nom-trace", + "pratt", + "pretty_assertions", + "proptest", + "test-strategy", + "thiserror", + "void", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bimap" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92b72b8f03128773278bf74418b9205f3d2a12c39a61f92395f47af390c32bf" + +[[package]] +name = "bit-set" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e11e16035ea35e4e5997b393eacbf6f63983188f7a2ad25bfb13465f5ad59de" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "bitvec" +version = "0.19.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8942c8d352ae1838c9dda0b0ca2ab657696ef2232a20147cf1b30ae1a9cb4321" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "byteorder" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd1061998a501ee7d4b6d449020df3266ca3124b941ec56cf2005c3779ca142" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "strsim", + "termcolor", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "clap_derive" +version = "3.0.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "370f715b81112975b1b69db93e0b56ea4cd4e5002ac43b2da8474106a54096a1" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "crate-root" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59c6fe4622b269032d2c5140a592d67a9c409031d286174fcde172fbed86f0d3" + +[[package]] +name = "ctor" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "diff" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "inkwell" +version = "0.1.0" +source = "git+https://github.com/TheDan64/inkwell?branch=master#a2db15b0bd1c06d71763585ae10d9ea4e775da0c" +dependencies = [ + "either", + "inkwell_internals", + "libc", + "llvm-sys", + "once_cell", + "parking_lot", + "regex", +] + +[[package]] +name = "inkwell_internals" +version = "0.3.0" +source = "git+https://github.com/TheDan64/inkwell?branch=master#a2db15b0bd1c06d71763585ae10d9ea4e775da0c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "itertools" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +dependencies = [ + "either", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lexical-core" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if", + "ryu", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b07a082330a35e43f63177cc01689da34fbffa0105e1246cf0311472cac73a" + +[[package]] +name = "llvm-sys" +version = "110.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ede189444b8c78907e5d36da5dabcf153170fcff9c1dba48afc4b33c7e19f0" +dependencies = [ + "cc", + "lazy_static", + "libc", + "regex", + "semver", +] + +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "nom" +version = "6.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2" +dependencies = [ + "bitvec", + "funty", + "lexical-core", + "memchr", + "version_check", +] + +[[package]] +name = "nom-trace" +version = "0.2.1" +source = "git+https://github.com/glittershark/nom-trace?branch=nom-6#6168d2e15cc51efd12d80260159b76a764dba138" +dependencies = [ + "nom", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + +[[package]] +name = "os_str_bytes" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb2e1c3ee07430c2cf76151675e583e0f19985fa6efae47d6848a3e2c824f85" + +[[package]] +name = "output_vt100" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53cdc5b785b7a58c5aad8216b3dfa114df64b0b06ae6e1501cef91df2fbdf8f9" +dependencies = [ + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "pratt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bbc12f7936a7b195790dd6d9b982b66c54f45ff6766decf25c44cac302dce" + +[[package]] +name = "pretty_assertions" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f297542c27a7df8d45de2b0e620308ab883ad232d06c14b76ac3e144bda50184" +dependencies = [ + "ansi_term", + "ctor", + "diff", + "output_vt100", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "proptest" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5" +dependencies = [ + "bit-set", + "bitflags", + "byteorder", + "lazy_static", + "num-traits", + "quick-error 2.0.0", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-error" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ac73b1112776fc109b2e61909bc46c7e1bf0d7f690ffb1676553acce16d5cda" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error 1.2.3", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed22b90a0e734a23a7610f4283ac9e5acfb96cbb30dfefa540d66f866f1c09c5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "test-strategy" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2328963c69243416e811c88066d18f670792b2e36e17fa57f4b1a124f85d18a8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "textwrap" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "203008d98caf094106cfaba70acfed15e18ed3ddb7d94e49baec153a2b462789" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" diff --git a/users/grfn/achilles/Cargo.toml b/users/grfn/achilles/Cargo.toml new file mode 100644 index 000000000000..f091399a0dd4 --- /dev/null +++ b/users/grfn/achilles/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "achilles" +version = "0.1.0" +authors = ["Griffin Smith <root@gws.fyi>"] +edition = "2018" + +[dependencies] +anyhow = "1.0.38" +bimap = "0.6.0" +clap = "3.0.0-beta.2" +derive_more = "0.99.11" +inkwell = { git = "https://github.com/TheDan64/inkwell", branch = "master", features = ["llvm11-0"] } +itertools = "0.10.0" +lazy_static = "1.4.0" +llvm-sys = "110.0.1" +nom = "6.1.2" +nom-trace = { git = "https://github.com/glittershark/nom-trace", branch = "nom-6" } +pratt = "0.3.0" +proptest = "1.0.0" +test-strategy = "0.1.1" +thiserror = "1.0.24" +void = "1.0.2" + +[dev-dependencies] +crate-root = "0.1.3" +pretty_assertions = "0.7.1" diff --git a/users/grfn/achilles/ach/.gitignore b/users/grfn/achilles/ach/.gitignore new file mode 100644 index 000000000000..ac5296ebbd74 --- /dev/null +++ b/users/grfn/achilles/ach/.gitignore @@ -0,0 +1,7 @@ +*.ll +*.o + +functions +simple +externs +units diff --git a/users/grfn/achilles/ach/Makefile b/users/grfn/achilles/ach/Makefile new file mode 100644 index 000000000000..3a8cd2865e87 --- /dev/null +++ b/users/grfn/achilles/ach/Makefile @@ -0,0 +1,15 @@ +default: simple + +%.ll: %.ach + cargo run -- compile $< -o $@ -f llvm + +%.o: %.ll + llc $< -o $@ -filetype=obj + +%: %.o + clang $< -o $@ + +.PHONY: clean + +clean: + @rm -f *.ll *.o simple functions diff --git a/users/grfn/achilles/ach/externs.ach b/users/grfn/achilles/ach/externs.ach new file mode 100644 index 000000000000..faf8ce90e353 --- /dev/null +++ b/users/grfn/achilles/ach/externs.ach @@ -0,0 +1,5 @@ +extern puts : fn cstring -> int + +fn main = + let _ = puts "foobar" + in 0 diff --git a/users/grfn/achilles/ach/functions.ach b/users/grfn/achilles/ach/functions.ach new file mode 100644 index 000000000000..dc6e7a1f3e34 --- /dev/null +++ b/users/grfn/achilles/ach/functions.ach @@ -0,0 +1,8 @@ +ty id : fn a -> a +fn id x = x + +ty plus : fn int -> int +fn plus (x: int) (y: int) = x + y + +ty main : fn -> int +fn main = plus (id 2) 7 diff --git a/users/grfn/achilles/ach/simple.ach b/users/grfn/achilles/ach/simple.ach new file mode 100644 index 000000000000..20f1677235c0 --- /dev/null +++ b/users/grfn/achilles/ach/simple.ach @@ -0,0 +1 @@ +fn main = let x = 2; y = 3 in x + y diff --git a/users/grfn/achilles/ach/units.ach b/users/grfn/achilles/ach/units.ach new file mode 100644 index 000000000000..70635d978c7c --- /dev/null +++ b/users/grfn/achilles/ach/units.ach @@ -0,0 +1,7 @@ +extern puts : fn cstring -> int + +ty print : fn cstring -> () +fn print x = let _ = puts x in () + +ty main : fn -> int +fn main = let _ = print "hi" in 0 diff --git a/users/grfn/achilles/default.nix b/users/grfn/achilles/default.nix new file mode 100644 index 000000000000..5245049d4a32 --- /dev/null +++ b/users/grfn/achilles/default.nix @@ -0,0 +1,24 @@ +{ depot, pkgs, ... }: + +let + llvmPackages = pkgs.llvmPackages_11; +in + +depot.third_party.naersk.buildPackage { + src = ./.; + + buildInputs = [ + llvmPackages.clang + llvmPackages.llvm + llvmPackages.bintools + llvmPackages.libclang.lib + ] ++ (with pkgs; [ + zlib + ncurses + libxml2 + libffi + pkgconfig + ]); + + doCheck = true; +} diff --git a/users/grfn/achilles/shell.nix b/users/grfn/achilles/shell.nix new file mode 100644 index 000000000000..f32dce3ba39d --- /dev/null +++ b/users/grfn/achilles/shell.nix @@ -0,0 +1,18 @@ +with (import ../../.. {}).third_party.nixpkgs; + +mkShell { + buildInputs = [ + clang_11 + llvm_11.lib + llvmPackages_11.bintools + llvmPackages_11.clang + llvmPackages_11.libclang.lib + zlib + ncurses + libxml2 + libffi + pkg-config + ]; + + LLVM_SYS_110_PREFIX = llvmPackages_11.bintools; +} diff --git a/users/grfn/achilles/src/ast/hir.rs b/users/grfn/achilles/src/ast/hir.rs new file mode 100644 index 000000000000..cdfaef567d7a --- /dev/null +++ b/users/grfn/achilles/src/ast/hir.rs @@ -0,0 +1,364 @@ +use std::collections::HashMap; + +use itertools::Itertools; + +use super::{BinaryOperator, Ident, Literal, UnaryOperator}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Pattern<'a, T> { + Id(Ident<'a>, T), + Tuple(Vec<Pattern<'a, T>>), +} + +impl<'a, T> Pattern<'a, T> { + pub fn to_owned(&self) -> Pattern<'static, T> + where + T: Clone, + { + match self { + Pattern::Id(id, t) => Pattern::Id(id.to_owned(), t.clone()), + Pattern::Tuple(pats) => { + Pattern::Tuple(pats.into_iter().map(Pattern::to_owned).collect()) + } + } + } + + pub fn traverse_type<F, U, E>(self, f: F) -> Result<Pattern<'a, U>, E> + where + F: Fn(T) -> Result<U, E> + Clone, + { + match self { + Pattern::Id(id, t) => Ok(Pattern::Id(id, f(t)?)), + Pattern::Tuple(pats) => Ok(Pattern::Tuple( + pats.into_iter() + .map(|pat| pat.traverse_type(f.clone())) + .collect::<Result<Vec<_>, _>>()?, + )), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Binding<'a, T> { + pub pat: Pattern<'a, T>, + pub body: Expr<'a, T>, +} + +impl<'a, T> Binding<'a, T> { + fn to_owned(&self) -> Binding<'static, T> + where + T: Clone, + { + Binding { + pat: self.pat.to_owned(), + body: self.body.to_owned(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Expr<'a, T> { + Ident(Ident<'a>, T), + + Literal(Literal<'a>, T), + + Tuple(Vec<Expr<'a, T>>, T), + + UnaryOp { + op: UnaryOperator, + rhs: Box<Expr<'a, T>>, + type_: T, + }, + + BinaryOp { + lhs: Box<Expr<'a, T>>, + op: BinaryOperator, + rhs: Box<Expr<'a, T>>, + type_: T, + }, + + Let { + bindings: Vec<Binding<'a, T>>, + body: Box<Expr<'a, T>>, + type_: T, + }, + + If { + condition: Box<Expr<'a, T>>, + then: Box<Expr<'a, T>>, + else_: Box<Expr<'a, T>>, + type_: T, + }, + + Fun { + type_args: Vec<Ident<'a>>, + args: Vec<(Ident<'a>, T)>, + body: Box<Expr<'a, T>>, + type_: T, + }, + + Call { + fun: Box<Expr<'a, T>>, + type_args: HashMap<Ident<'a>, T>, + args: Vec<Expr<'a, T>>, + type_: T, + }, +} + +impl<'a, T> Expr<'a, T> { + pub fn type_(&self) -> &T { + match self { + Expr::Ident(_, t) => t, + Expr::Literal(_, t) => t, + Expr::Tuple(_, t) => t, + Expr::UnaryOp { type_, .. } => type_, + Expr::BinaryOp { type_, .. } => type_, + Expr::Let { type_, .. } => type_, + Expr::If { type_, .. } => type_, + Expr::Fun { type_, .. } => type_, + Expr::Call { type_, .. } => type_, + } + } + + pub fn traverse_type<F, U, E>(self, f: F) -> Result<Expr<'a, U>, E> + where + F: Fn(T) -> Result<U, E> + Clone, + { + match self { + Expr::Ident(id, t) => Ok(Expr::Ident(id, f(t)?)), + Expr::Literal(lit, t) => Ok(Expr::Literal(lit, f(t)?)), + Expr::UnaryOp { op, rhs, type_ } => Ok(Expr::UnaryOp { + op, + rhs: Box::new(rhs.traverse_type(f.clone())?), + type_: f(type_)?, + }), + Expr::BinaryOp { + lhs, + op, + rhs, + type_, + } => Ok(Expr::BinaryOp { + lhs: Box::new(lhs.traverse_type(f.clone())?), + op, + rhs: Box::new(rhs.traverse_type(f.clone())?), + type_: f(type_)?, + }), + Expr::Let { + bindings, + body, + type_, + } => Ok(Expr::Let { + bindings: bindings + .into_iter() + .map(|Binding { pat, body }| { + Ok(Binding { + pat: pat.traverse_type(f.clone())?, + body: body.traverse_type(f.clone())?, + }) + }) + .collect::<Result<Vec<_>, E>>()?, + body: Box::new(body.traverse_type(f.clone())?), + type_: f(type_)?, + }), + Expr::If { + condition, + then, + else_, + type_, + } => Ok(Expr::If { + condition: Box::new(condition.traverse_type(f.clone())?), + then: Box::new(then.traverse_type(f.clone())?), + else_: Box::new(else_.traverse_type(f.clone())?), + type_: f(type_)?, + }), + Expr::Fun { + args, + type_args, + body, + type_, + } => Ok(Expr::Fun { + args: args + .into_iter() + .map(|(id, t)| Ok((id, f.clone()(t)?))) + .collect::<Result<Vec<_>, E>>()?, + type_args, + body: Box::new(body.traverse_type(f.clone())?), + type_: f(type_)?, + }), + Expr::Call { + fun, + type_args, + args, + type_, + } => Ok(Expr::Call { + fun: Box::new(fun.traverse_type(f.clone())?), + type_args: type_args + .into_iter() + .map(|(id, ty)| Ok((id, f.clone()(ty)?))) + .collect::<Result<HashMap<_, _>, E>>()?, + args: args + .into_iter() + .map(|e| e.traverse_type(f.clone())) + .collect::<Result<Vec<_>, E>>()?, + type_: f(type_)?, + }), + Expr::Tuple(members, t) => Ok(Expr::Tuple( + members + .into_iter() + .map(|t| t.traverse_type(f.clone())) + .try_collect()?, + f(t)?, + )), + } + } + + pub fn to_owned(&self) -> Expr<'static, T> + where + T: Clone, + { + match self { + Expr::Ident(id, t) => Expr::Ident(id.to_owned(), t.clone()), + Expr::Literal(lit, t) => Expr::Literal(lit.to_owned(), t.clone()), + Expr::UnaryOp { op, rhs, type_ } => Expr::UnaryOp { + op: *op, + rhs: Box::new((**rhs).to_owned()), + type_: type_.clone(), + }, + Expr::BinaryOp { + lhs, + op, + rhs, + type_, + } => Expr::BinaryOp { + lhs: Box::new((**lhs).to_owned()), + op: *op, + rhs: Box::new((**rhs).to_owned()), + type_: type_.clone(), + }, + Expr::Let { + bindings, + body, + type_, + } => Expr::Let { + bindings: bindings.iter().map(|b| b.to_owned()).collect(), + body: Box::new((**body).to_owned()), + type_: type_.clone(), + }, + Expr::If { + condition, + then, + else_, + type_, + } => Expr::If { + condition: Box::new((**condition).to_owned()), + then: Box::new((**then).to_owned()), + else_: Box::new((**else_).to_owned()), + type_: type_.clone(), + }, + Expr::Fun { + args, + type_args, + body, + type_, + } => Expr::Fun { + args: args + .iter() + .map(|(id, t)| (id.to_owned(), t.clone())) + .collect(), + type_args: type_args.iter().map(|arg| arg.to_owned()).collect(), + body: Box::new((**body).to_owned()), + type_: type_.clone(), + }, + Expr::Call { + fun, + type_args, + args, + type_, + } => Expr::Call { + fun: Box::new((**fun).to_owned()), + type_args: type_args + .iter() + .map(|(id, t)| (id.to_owned(), t.clone())) + .collect(), + args: args.iter().map(|e| e.to_owned()).collect(), + type_: type_.clone(), + }, + Expr::Tuple(members, t) => { + Expr::Tuple(members.into_iter().map(Expr::to_owned).collect(), t.clone()) + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Decl<'a, T> { + Fun { + name: Ident<'a>, + type_args: Vec<Ident<'a>>, + args: Vec<(Ident<'a>, T)>, + body: Box<Expr<'a, T>>, + type_: T, + }, + + Extern { + name: Ident<'a>, + arg_types: Vec<T>, + ret_type: T, + }, +} + +impl<'a, T> Decl<'a, T> { + pub fn name(&self) -> &Ident<'a> { + match self { + Decl::Fun { name, .. } => name, + Decl::Extern { name, .. } => name, + } + } + + pub fn set_name(&mut self, new_name: Ident<'a>) { + match self { + Decl::Fun { name, .. } => *name = new_name, + Decl::Extern { name, .. } => *name = new_name, + } + } + + pub fn type_(&self) -> Option<&T> { + match self { + Decl::Fun { type_, .. } => Some(type_), + Decl::Extern { .. } => None, + } + } + + pub fn traverse_type<F, U, E>(self, f: F) -> Result<Decl<'a, U>, E> + where + F: Fn(T) -> Result<U, E> + Clone, + { + match self { + Decl::Fun { + name, + type_args, + args, + body, + type_, + } => Ok(Decl::Fun { + name, + type_args, + args: args + .into_iter() + .map(|(id, t)| Ok((id, f(t)?))) + .try_collect()?, + body: Box::new(body.traverse_type(f.clone())?), + type_: f(type_)?, + }), + Decl::Extern { + name, + arg_types, + ret_type, + } => Ok(Decl::Extern { + name, + arg_types: arg_types.into_iter().map(f.clone()).try_collect()?, + ret_type: f(ret_type)?, + }), + } + } +} diff --git a/users/grfn/achilles/src/ast/mod.rs b/users/grfn/achilles/src/ast/mod.rs new file mode 100644 index 000000000000..5438d29d2cf7 --- /dev/null +++ b/users/grfn/achilles/src/ast/mod.rs @@ -0,0 +1,484 @@ +pub(crate) mod hir; + +use std::borrow::Cow; +use std::collections::HashMap; +use std::convert::TryFrom; +use std::fmt::{self, Display, Formatter}; + +use itertools::Itertools; + +#[derive(Debug, PartialEq, Eq)] +pub struct InvalidIdentifier<'a>(Cow<'a, str>); + +#[derive(Debug, PartialEq, Eq, Hash, Clone)] +pub struct Ident<'a>(pub Cow<'a, str>); + +impl<'a> From<&'a Ident<'a>> for &'a str { + fn from(id: &'a Ident<'a>) -> Self { + id.0.as_ref() + } +} + +impl<'a> Display for Ident<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl<'a> Ident<'a> { + pub fn to_owned(&self) -> Ident<'static> { + Ident(Cow::Owned(self.0.clone().into_owned())) + } + + /// Construct an identifier from a &str without checking that it's a valid identifier + pub fn from_str_unchecked(s: &'a str) -> Self { + debug_assert!(is_valid_identifier(s)); + Self(Cow::Borrowed(s)) + } + + pub fn from_string_unchecked(s: String) -> Self { + debug_assert!(is_valid_identifier(&s)); + Self(Cow::Owned(s)) + } +} + +pub fn is_valid_identifier<S>(s: &S) -> bool +where + S: AsRef<str> + ?Sized, +{ + s.as_ref() + .chars() + .any(|c| !c.is_alphanumeric() || !"_".contains(c)) +} + +impl<'a> TryFrom<&'a str> for Ident<'a> { + type Error = InvalidIdentifier<'a>; + + fn try_from(s: &'a str) -> Result<Self, Self::Error> { + if is_valid_identifier(s) { + Ok(Ident(Cow::Borrowed(s))) + } else { + Err(InvalidIdentifier(Cow::Borrowed(s))) + } + } +} + +impl<'a> TryFrom<String> for Ident<'a> { + type Error = InvalidIdentifier<'static>; + + fn try_from(s: String) -> Result<Self, Self::Error> { + if is_valid_identifier(&s) { + Ok(Ident(Cow::Owned(s))) + } else { + Err(InvalidIdentifier(Cow::Owned(s))) + } + } +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum BinaryOperator { + /// `+` + Add, + + /// `-` + Sub, + + /// `*` + Mul, + + /// `/` + Div, + + /// `^` + Pow, + + /// `==` + Equ, + + /// `!=` + Neq, +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum UnaryOperator { + /// ! + Not, + + /// - + Neg, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Literal<'a> { + Unit, + Int(u64), + Bool(bool), + String(Cow<'a, str>), +} + +impl<'a> Literal<'a> { + pub fn to_owned(&self) -> Literal<'static> { + match self { + Literal::Int(i) => Literal::Int(*i), + Literal::Bool(b) => Literal::Bool(*b), + Literal::String(s) => Literal::String(Cow::Owned(s.clone().into_owned())), + Literal::Unit => Literal::Unit, + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Pattern<'a> { + Id(Ident<'a>), + Tuple(Vec<Pattern<'a>>), +} + +impl<'a> Pattern<'a> { + pub fn to_owned(&self) -> Pattern<'static> { + match self { + Pattern::Id(id) => Pattern::Id(id.to_owned()), + Pattern::Tuple(pats) => Pattern::Tuple(pats.iter().map(Pattern::to_owned).collect()), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Binding<'a> { + pub pat: Pattern<'a>, + pub type_: Option<Type<'a>>, + pub body: Expr<'a>, +} + +impl<'a> Binding<'a> { + fn to_owned(&self) -> Binding<'static> { + Binding { + pat: self.pat.to_owned(), + type_: self.type_.as_ref().map(|t| t.to_owned()), + body: self.body.to_owned(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Expr<'a> { + Ident(Ident<'a>), + + Literal(Literal<'a>), + + UnaryOp { + op: UnaryOperator, + rhs: Box<Expr<'a>>, + }, + + BinaryOp { + lhs: Box<Expr<'a>>, + op: BinaryOperator, + rhs: Box<Expr<'a>>, + }, + + Let { + bindings: Vec<Binding<'a>>, + body: Box<Expr<'a>>, + }, + + If { + condition: Box<Expr<'a>>, + then: Box<Expr<'a>>, + else_: Box<Expr<'a>>, + }, + + Fun(Box<Fun<'a>>), + + Call { + fun: Box<Expr<'a>>, + args: Vec<Expr<'a>>, + }, + + Tuple(Vec<Expr<'a>>), + + Ascription { + expr: Box<Expr<'a>>, + type_: Type<'a>, + }, +} + +impl<'a> Expr<'a> { + pub fn to_owned(&self) -> Expr<'static> { + match self { + Expr::Ident(ref id) => Expr::Ident(id.to_owned()), + Expr::Literal(ref lit) => Expr::Literal(lit.to_owned()), + Expr::Tuple(ref members) => { + Expr::Tuple(members.into_iter().map(Expr::to_owned).collect()) + } + Expr::UnaryOp { op, rhs } => Expr::UnaryOp { + op: *op, + rhs: Box::new((**rhs).to_owned()), + }, + Expr::BinaryOp { lhs, op, rhs } => Expr::BinaryOp { + lhs: Box::new((**lhs).to_owned()), + op: *op, + rhs: Box::new((**rhs).to_owned()), + }, + Expr::Let { bindings, body } => Expr::Let { + bindings: bindings.iter().map(|binding| binding.to_owned()).collect(), + body: Box::new((**body).to_owned()), + }, + Expr::If { + condition, + then, + else_, + } => Expr::If { + condition: Box::new((**condition).to_owned()), + then: Box::new((**then).to_owned()), + else_: Box::new((**else_).to_owned()), + }, + Expr::Fun(fun) => Expr::Fun(Box::new((**fun).to_owned())), + Expr::Call { fun, args } => Expr::Call { + fun: Box::new((**fun).to_owned()), + args: args.iter().map(|arg| arg.to_owned()).collect(), + }, + Expr::Ascription { expr, type_ } => Expr::Ascription { + expr: Box::new((**expr).to_owned()), + type_: type_.to_owned(), + }, + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Arg<'a> { + pub ident: Ident<'a>, + pub type_: Option<Type<'a>>, +} + +impl<'a> Arg<'a> { + pub fn to_owned(&self) -> Arg<'static> { + Arg { + ident: self.ident.to_owned(), + type_: self.type_.as_ref().map(Type::to_owned), + } + } +} + +impl<'a> TryFrom<&'a str> for Arg<'a> { + type Error = <Ident<'a> as TryFrom<&'a str>>::Error; + + fn try_from(value: &'a str) -> Result<Self, Self::Error> { + Ok(Arg { + ident: Ident::try_from(value)?, + type_: None, + }) + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Fun<'a> { + pub args: Vec<Arg<'a>>, + pub body: Expr<'a>, +} + +impl<'a> Fun<'a> { + pub fn to_owned(&self) -> Fun<'static> { + Fun { + args: self.args.iter().map(|arg| arg.to_owned()).collect(), + body: self.body.to_owned(), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Decl<'a> { + Fun { + name: Ident<'a>, + body: Fun<'a>, + }, + Ascription { + name: Ident<'a>, + type_: Type<'a>, + }, + Extern { + name: Ident<'a>, + type_: FunctionType<'a>, + }, +} + +//// + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct FunctionType<'a> { + pub args: Vec<Type<'a>>, + pub ret: Box<Type<'a>>, +} + +impl<'a> FunctionType<'a> { + pub fn to_owned(&self) -> FunctionType<'static> { + FunctionType { + args: self.args.iter().map(|a| a.to_owned()).collect(), + ret: Box::new((*self.ret).to_owned()), + } + } +} + +impl<'a> Display for FunctionType<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "fn {} -> {}", self.args.iter().join(", "), self.ret) + } +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Type<'a> { + Int, + Float, + Bool, + CString, + Unit, + Tuple(Vec<Type<'a>>), + Var(Ident<'a>), + Function(FunctionType<'a>), +} + +impl<'a> Type<'a> { + pub fn to_owned(&self) -> Type<'static> { + match self { + Type::Int => Type::Int, + Type::Float => Type::Float, + Type::Bool => Type::Bool, + Type::CString => Type::CString, + Type::Unit => Type::Unit, + Type::Var(v) => Type::Var(v.to_owned()), + Type::Function(f) => Type::Function(f.to_owned()), + Type::Tuple(members) => Type::Tuple(members.iter().map(Type::to_owned).collect()), + } + } + + pub fn alpha_equiv(&self, other: &Self) -> bool { + fn do_alpha_equiv<'a>( + substs: &mut HashMap<&'a Ident<'a>, &'a Ident<'a>>, + lhs: &'a Type, + rhs: &'a Type, + ) -> bool { + match (lhs, rhs) { + (Type::Var(v1), Type::Var(v2)) => substs.entry(v1).or_insert(v2) == &v2, + ( + Type::Function(FunctionType { + args: args1, + ret: ret1, + }), + Type::Function(FunctionType { + args: args2, + ret: ret2, + }), + ) => { + args1.len() == args2.len() + && args1 + .iter() + .zip(args2) + .all(|(a1, a2)| do_alpha_equiv(substs, a1, a2)) + && do_alpha_equiv(substs, ret1, ret2) + } + _ => lhs == rhs, + } + } + + let mut substs = HashMap::new(); + do_alpha_equiv(&mut substs, self, other) + } + + pub fn traverse_type_vars<'b, F>(self, mut f: F) -> Type<'b> + where + F: FnMut(Ident<'a>) -> Type<'b> + Clone, + { + match self { + Type::Var(tv) => f(tv), + Type::Function(FunctionType { args, ret }) => Type::Function(FunctionType { + args: args + .into_iter() + .map(|t| t.traverse_type_vars(f.clone())) + .collect(), + ret: Box::new(ret.traverse_type_vars(f)), + }), + Type::Int => Type::Int, + Type::Float => Type::Float, + Type::Bool => Type::Bool, + Type::CString => Type::CString, + Type::Tuple(members) => Type::Tuple( + members + .into_iter() + .map(|t| t.traverse_type_vars(f.clone())) + .collect(), + ), + Type::Unit => Type::Unit, + } + } + + pub fn as_tuple(&self) -> Option<&Vec<Type<'a>>> { + if let Self::Tuple(v) = self { + Some(v) + } else { + None + } + } +} + +impl<'a> Display for Type<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Type::Int => f.write_str("int"), + Type::Float => f.write_str("float"), + Type::Bool => f.write_str("bool"), + Type::CString => f.write_str("cstring"), + Type::Unit => f.write_str("()"), + Type::Var(v) => v.fmt(f), + Type::Function(ft) => ft.fmt(f), + Type::Tuple(ms) => write!(f, "({})", ms.iter().join(", ")), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn type_var(n: &str) -> Type<'static> { + Type::Var(Ident::try_from(n.to_owned()).unwrap()) + } + + mod alpha_equiv { + use super::*; + + #[test] + fn trivial() { + assert!(Type::Int.alpha_equiv(&Type::Int)); + assert!(!Type::Int.alpha_equiv(&Type::Bool)); + } + + #[test] + fn simple_type_var() { + assert!(type_var("a").alpha_equiv(&type_var("b"))); + } + + #[test] + fn function_with_type_vars_equiv() { + assert!(Type::Function(FunctionType { + args: vec![type_var("a")], + ret: Box::new(type_var("b")), + }) + .alpha_equiv(&Type::Function(FunctionType { + args: vec![type_var("b")], + ret: Box::new(type_var("a")), + }))) + } + + #[test] + fn function_with_type_vars_non_equiv() { + assert!(!Type::Function(FunctionType { + args: vec![type_var("a")], + ret: Box::new(type_var("a")), + }) + .alpha_equiv(&Type::Function(FunctionType { + args: vec![type_var("b")], + ret: Box::new(type_var("a")), + }))) + } + } +} diff --git a/users/grfn/achilles/src/codegen/llvm.rs b/users/grfn/achilles/src/codegen/llvm.rs new file mode 100644 index 000000000000..9a71ac954e00 --- /dev/null +++ b/users/grfn/achilles/src/codegen/llvm.rs @@ -0,0 +1,486 @@ +use std::convert::{TryFrom, TryInto}; +use std::path::Path; +use std::result; + +use inkwell::basic_block::BasicBlock; +use inkwell::builder::Builder; +pub use inkwell::context::Context; +use inkwell::module::Module; +use inkwell::support::LLVMString; +use inkwell::types::{BasicType, BasicTypeEnum, FunctionType, IntType, StructType}; +use inkwell::values::{AnyValueEnum, BasicValueEnum, FunctionValue, StructValue}; +use inkwell::{AddressSpace, IntPredicate}; +use itertools::Itertools; +use thiserror::Error; + +use crate::ast::hir::{Binding, Decl, Expr, Pattern}; +use crate::ast::{BinaryOperator, Ident, Literal, Type, UnaryOperator}; +use crate::common::env::Env; + +#[derive(Debug, PartialEq, Eq, Error)] +pub enum Error { + #[error("Undefined variable {0}")] + UndefinedVariable(Ident<'static>), + + #[error("LLVM Error: {0}")] + LLVMError(String), +} + +impl From<LLVMString> for Error { + fn from(s: LLVMString) -> Self { + Self::LLVMError(s.to_string()) + } +} + +pub type Result<T> = result::Result<T, Error>; + +pub struct Codegen<'ctx, 'ast> { + context: &'ctx Context, + pub module: Module<'ctx>, + builder: Builder<'ctx>, + env: Env<&'ast Ident<'ast>, AnyValueEnum<'ctx>>, + function_stack: Vec<FunctionValue<'ctx>>, + identifier_counter: u32, +} + +impl<'ctx, 'ast> Codegen<'ctx, 'ast> { + pub fn new(context: &'ctx Context, module_name: &str) -> Self { + let module = context.create_module(module_name); + let builder = context.create_builder(); + Self { + context, + module, + builder, + env: Default::default(), + function_stack: Default::default(), + identifier_counter: 0, + } + } + + pub fn new_function<'a>( + &'a mut self, + name: &str, + ty: FunctionType<'ctx>, + ) -> &'a FunctionValue<'ctx> { + self.function_stack + .push(self.module.add_function(name, ty, None)); + let basic_block = self.append_basic_block("entry"); + self.builder.position_at_end(basic_block); + self.function_stack.last().unwrap() + } + + pub fn finish_function(&mut self, res: Option<&BasicValueEnum<'ctx>>) -> FunctionValue<'ctx> { + self.builder.build_return(match res { + // lol + Some(val) => Some(val), + None => None, + }); + self.function_stack.pop().unwrap() + } + + pub fn append_basic_block(&self, name: &str) -> BasicBlock<'ctx> { + self.context + .append_basic_block(*self.function_stack.last().unwrap(), name) + } + + fn bind_pattern(&mut self, pat: &'ast Pattern<'ast, Type>, val: AnyValueEnum<'ctx>) { + match pat { + Pattern::Id(id, _) => self.env.set(id, val), + Pattern::Tuple(pats) => { + for (i, pat) in pats.iter().enumerate() { + let member = self + .builder + .build_extract_value( + StructValue::try_from(val).unwrap(), + i as _, + "pat_bind", + ) + .unwrap(); + self.bind_pattern(pat, member.into()); + } + } + } + } + + pub fn codegen_expr( + &mut self, + expr: &'ast Expr<'ast, Type>, + ) -> Result<Option<AnyValueEnum<'ctx>>> { + match expr { + Expr::Ident(id, _) => self + .env + .resolve(id) + .cloned() + .ok_or_else(|| Error::UndefinedVariable(id.to_owned())) + .map(Some), + Expr::Literal(lit, ty) => { + let ty = self.codegen_int_type(ty); + match lit { + Literal::Int(i) => Ok(Some(AnyValueEnum::IntValue(ty.const_int(*i, false)))), + Literal::Bool(b) => Ok(Some(AnyValueEnum::IntValue( + ty.const_int(if *b { 1 } else { 0 }, false), + ))), + Literal::String(s) => Ok(Some( + self.builder + .build_global_string_ptr(s, "s") + .as_pointer_value() + .into(), + )), + Literal::Unit => Ok(None), + } + } + Expr::UnaryOp { op, rhs, .. } => { + let rhs = self.codegen_expr(rhs)?.unwrap(); + match op { + UnaryOperator::Not => unimplemented!(), + UnaryOperator::Neg => Ok(Some(AnyValueEnum::IntValue( + self.builder.build_int_neg(rhs.into_int_value(), "neg"), + ))), + } + } + Expr::BinaryOp { lhs, op, rhs, .. } => { + let lhs = self.codegen_expr(lhs)?.unwrap(); + let rhs = self.codegen_expr(rhs)?.unwrap(); + match op { + BinaryOperator::Add => { + Ok(Some(AnyValueEnum::IntValue(self.builder.build_int_add( + lhs.into_int_value(), + rhs.into_int_value(), + "add", + )))) + } + BinaryOperator::Sub => { + Ok(Some(AnyValueEnum::IntValue(self.builder.build_int_sub( + lhs.into_int_value(), + rhs.into_int_value(), + "add", + )))) + } + BinaryOperator::Mul => { + Ok(Some(AnyValueEnum::IntValue(self.builder.build_int_sub( + lhs.into_int_value(), + rhs.into_int_value(), + "add", + )))) + } + BinaryOperator::Div => Ok(Some(AnyValueEnum::IntValue( + self.builder.build_int_signed_div( + lhs.into_int_value(), + rhs.into_int_value(), + "add", + ), + ))), + BinaryOperator::Pow => unimplemented!(), + BinaryOperator::Equ => Ok(Some(AnyValueEnum::IntValue( + self.builder.build_int_compare( + IntPredicate::EQ, + lhs.into_int_value(), + rhs.into_int_value(), + "eq", + ), + ))), + BinaryOperator::Neq => todo!(), + } + } + Expr::Let { bindings, body, .. } => { + self.env.push(); + for Binding { pat, body, .. } in bindings { + if let Some(val) = self.codegen_expr(body)? { + self.bind_pattern(pat, val); + } + } + let res = self.codegen_expr(body); + self.env.pop(); + res + } + Expr::If { + condition, + then, + else_, + type_, + } => { + let then_block = self.append_basic_block("then"); + let else_block = self.append_basic_block("else"); + let join_block = self.append_basic_block("join"); + let condition = self.codegen_expr(condition)?.unwrap(); + self.builder.build_conditional_branch( + condition.into_int_value(), + then_block, + else_block, + ); + self.builder.position_at_end(then_block); + let then_res = self.codegen_expr(then)?; + self.builder.build_unconditional_branch(join_block); + + self.builder.position_at_end(else_block); + let else_res = self.codegen_expr(else_)?; + self.builder.build_unconditional_branch(join_block); + + self.builder.position_at_end(join_block); + if let Some(phi_type) = self.codegen_type(type_) { + let phi = self.builder.build_phi(phi_type, "join"); + phi.add_incoming(&[ + ( + &BasicValueEnum::try_from(then_res.unwrap()).unwrap(), + then_block, + ), + ( + &BasicValueEnum::try_from(else_res.unwrap()).unwrap(), + else_block, + ), + ]); + Ok(Some(phi.as_basic_value().into())) + } else { + Ok(None) + } + } + Expr::Call { fun, args, .. } => { + if let Expr::Ident(id, _) = &**fun { + let function = self + .module + .get_function(id.into()) + .or_else(|| self.env.resolve(id)?.clone().try_into().ok()) + .ok_or_else(|| Error::UndefinedVariable(id.to_owned()))?; + let args = args + .iter() + .map(|arg| Ok(self.codegen_expr(arg)?.unwrap().try_into().unwrap())) + .collect::<Result<Vec<_>>>()?; + Ok(self + .builder + .build_call(function, &args, "call") + .try_as_basic_value() + .left() + .map(|val| val.into())) + } else { + todo!() + } + } + Expr::Fun { args, body, .. } => { + let fname = self.fresh_ident("f"); + let cur_block = self.builder.get_insert_block().unwrap(); + let env = self.env.save(); // TODO: closures + let function = self.codegen_function(&fname, args, body)?; + self.builder.position_at_end(cur_block); + self.env.restore(env); + Ok(Some(function.into())) + } + Expr::Tuple(members, ty) => { + let values = members + .into_iter() + .map(|expr| self.codegen_expr(expr)) + .collect::<Result<Vec<_>>>()? + .into_iter() + .filter_map(|x| x) + .map(|x| x.try_into().unwrap()) + .collect_vec(); + let field_types = ty.as_tuple().unwrap(); + let tuple_type = self.codegen_tuple_type(field_types); + Ok(Some(tuple_type.const_named_struct(&values).into())) + } + } + } + + pub fn codegen_function( + &mut self, + name: &str, + args: &'ast [(Ident<'ast>, Type)], + body: &'ast Expr<'ast, Type>, + ) -> Result<FunctionValue<'ctx>> { + let arg_types = args + .iter() + .filter_map(|(_, at)| self.codegen_type(at)) + .collect::<Vec<_>>(); + + self.new_function( + name, + match self.codegen_type(body.type_()) { + Some(ret_ty) => ret_ty.fn_type(&arg_types, false), + None => self.context.void_type().fn_type(&arg_types, false), + }, + ); + self.env.push(); + for (i, (arg, _)) in args.iter().enumerate() { + self.env.set( + arg, + self.cur_function().get_nth_param(i as u32).unwrap().into(), + ); + } + let res = self.codegen_expr(body)?; + self.env.pop(); + Ok(self.finish_function(res.map(|av| av.try_into().unwrap()).as_ref())) + } + + pub fn codegen_extern( + &mut self, + name: &str, + args: &'ast [Type], + ret: &'ast Type, + ) -> Result<()> { + let arg_types = args + .iter() + .map(|t| self.codegen_type(t).unwrap()) + .collect::<Vec<_>>(); + self.module.add_function( + name, + match self.codegen_type(ret) { + Some(ret_ty) => ret_ty.fn_type(&arg_types, false), + None => self.context.void_type().fn_type(&arg_types, false), + }, + None, + ); + Ok(()) + } + + pub fn codegen_decl(&mut self, decl: &'ast Decl<'ast, Type>) -> Result<()> { + match decl { + Decl::Fun { + name, args, body, .. + } => { + self.codegen_function(name.into(), args, body)?; + Ok(()) + } + Decl::Extern { + name, + arg_types, + ret_type, + } => self.codegen_extern(name.into(), arg_types, ret_type), + } + } + + pub fn codegen_main(&mut self, expr: &'ast Expr<'ast, Type>) -> Result<()> { + self.new_function("main", self.context.i64_type().fn_type(&[], false)); + let res = self.codegen_expr(expr)?; + if *expr.type_() != Type::Int { + self.builder + .build_return(Some(&self.context.i64_type().const_int(0, false))); + } else { + self.finish_function(res.map(|r| r.try_into().unwrap()).as_ref()); + } + Ok(()) + } + + fn codegen_type(&self, type_: &'ast Type) -> Option<BasicTypeEnum<'ctx>> { + // TODO + match type_ { + Type::Int => Some(self.context.i64_type().into()), + Type::Float => Some(self.context.f64_type().into()), + Type::Bool => Some(self.context.bool_type().into()), + Type::CString => Some( + self.context + .i8_type() + .ptr_type(AddressSpace::Generic) + .into(), + ), + Type::Function(_) => todo!(), + Type::Var(_) => unreachable!(), + Type::Unit => None, + Type::Tuple(ts) => Some(self.codegen_tuple_type(ts).into()), + } + } + + fn codegen_tuple_type(&self, ts: &'ast [Type]) -> StructType<'ctx> { + self.context.struct_type( + ts.iter() + .filter_map(|t| self.codegen_type(t)) + .collect_vec() + .as_slice(), + false, + ) + } + + fn codegen_int_type(&self, type_: &'ast Type) -> IntType<'ctx> { + // TODO + self.context.i64_type() + } + + pub fn print_to_file<P>(&self, path: P) -> Result<()> + where + P: AsRef<Path>, + { + Ok(self.module.print_to_file(path)?) + } + + pub fn binary_to_file<P>(&self, path: P) -> Result<()> + where + P: AsRef<Path>, + { + if self.module.write_bitcode_to_path(path.as_ref()) { + Ok(()) + } else { + Err(Error::LLVMError( + "Error writing bitcode to output path".to_owned(), + )) + } + } + + fn fresh_ident(&mut self, prefix: &str) -> String { + self.identifier_counter += 1; + format!("{}{}", prefix, self.identifier_counter) + } + + fn cur_function(&self) -> &FunctionValue<'ctx> { + self.function_stack.last().unwrap() + } +} + +#[cfg(test)] +mod tests { + use inkwell::execution_engine::JitFunction; + use inkwell::OptimizationLevel; + + use super::*; + + fn jit_eval<T>(expr: &str) -> anyhow::Result<T> { + let expr = crate::parser::expr(expr).unwrap().1; + + let expr = crate::tc::typecheck_expr(expr).unwrap(); + + let context = Context::create(); + let mut codegen = Codegen::new(&context, "test"); + let execution_engine = codegen + .module + .create_jit_execution_engine(OptimizationLevel::None) + .unwrap(); + + codegen.codegen_function("test", &[], &expr)?; + + unsafe { + let fun: JitFunction<unsafe extern "C" fn() -> T> = + execution_engine.get_function("test")?; + Ok(fun.call()) + } + } + + #[test] + fn add_literals() { + assert_eq!(jit_eval::<i64>("1 + 2").unwrap(), 3); + } + + #[test] + fn variable_shadowing() { + assert_eq!( + jit_eval::<i64>("let x = 1 in (let x = 2 in x) + x").unwrap(), + 3 + ); + } + + #[test] + fn eq() { + assert_eq!( + jit_eval::<i64>("let x = 1 in if x == 1 then 2 else 4").unwrap(), + 2 + ); + } + + #[test] + fn function_call() { + let res = jit_eval::<i64>("let id = fn x = x in id 1").unwrap(); + assert_eq!(res, 1); + } + + #[test] + fn bind_tuple_pattern() { + let res = jit_eval::<i64>("let (x, y) = (1, 2) in x + y").unwrap(); + assert_eq!(res, 3); + } +} diff --git a/users/grfn/achilles/src/codegen/mod.rs b/users/grfn/achilles/src/codegen/mod.rs new file mode 100644 index 000000000000..8ef057dba04f --- /dev/null +++ b/users/grfn/achilles/src/codegen/mod.rs @@ -0,0 +1,25 @@ +pub mod llvm; + +use inkwell::execution_engine::JitFunction; +use inkwell::OptimizationLevel; +pub use llvm::*; + +use crate::ast::hir::Expr; +use crate::ast::Type; +use crate::common::Result; + +pub fn jit_eval<T>(expr: &Expr<Type>) -> Result<T> { + let context = Context::create(); + let mut codegen = Codegen::new(&context, "eval"); + let execution_engine = codegen + .module + .create_jit_execution_engine(OptimizationLevel::None) + .map_err(Error::from)?; + codegen.codegen_function("test", &[], &expr)?; + + unsafe { + let fun: JitFunction<unsafe extern "C" fn() -> T> = + execution_engine.get_function("eval").unwrap(); + Ok(fun.call()) + } +} diff --git a/users/grfn/achilles/src/commands/check.rs b/users/grfn/achilles/src/commands/check.rs new file mode 100644 index 000000000000..0bea482c1478 --- /dev/null +++ b/users/grfn/achilles/src/commands/check.rs @@ -0,0 +1,39 @@ +use clap::Clap; +use std::path::PathBuf; + +use crate::ast::Type; +use crate::{parser, tc, Result}; + +/// Typecheck a file or expression +#[derive(Clap)] +pub struct Check { + /// File to check + path: Option<PathBuf>, + + /// Expression to check + #[clap(long, short = 'e')] + expr: Option<String>, +} + +fn run_expr(expr: String) -> Result<Type<'static>> { + let (_, parsed) = parser::expr(&expr)?; + let hir_expr = tc::typecheck_expr(parsed)?; + Ok(hir_expr.type_().to_owned()) +} + +fn run_path(path: PathBuf) -> Result<Type<'static>> { + todo!() +} + +impl Check { + pub fn run(self) -> Result<()> { + let type_ = match (self.path, self.expr) { + (None, None) => Err("Must specify either a file or expression to check".into()), + (Some(_), Some(_)) => Err("Cannot specify both a file and expression to check".into()), + (None, Some(expr)) => run_expr(expr), + (Some(path), None) => run_path(path), + }?; + println!("type: {}", type_); + Ok(()) + } +} diff --git a/users/grfn/achilles/src/commands/compile.rs b/users/grfn/achilles/src/commands/compile.rs new file mode 100644 index 000000000000..be8767575ab5 --- /dev/null +++ b/users/grfn/achilles/src/commands/compile.rs @@ -0,0 +1,31 @@ +use std::path::PathBuf; + +use clap::Clap; + +use crate::common::Result; +use crate::compiler::{self, CompilerOptions}; + +/// Compile a source file +#[derive(Clap)] +pub struct Compile { + /// File to compile + file: PathBuf, + + /// Output file + #[clap(short = 'o')] + out_file: PathBuf, + + #[clap(flatten)] + options: CompilerOptions, +} + +impl Compile { + pub fn run(self) -> Result<()> { + eprintln!( + ">>> {} -> {}", + &self.file.to_string_lossy(), + self.out_file.to_string_lossy() + ); + compiler::compile_file(&self.file, &self.out_file, &self.options) + } +} diff --git a/users/grfn/achilles/src/commands/eval.rs b/users/grfn/achilles/src/commands/eval.rs new file mode 100644 index 000000000000..61a712c08a8e --- /dev/null +++ b/users/grfn/achilles/src/commands/eval.rs @@ -0,0 +1,32 @@ +use clap::Clap; + +use crate::codegen; +use crate::interpreter; +use crate::parser; +use crate::tc; +use crate::Result; + +/// Evaluate an expression and print its result +#[derive(Clap)] +pub struct Eval { + /// JIT-compile with LLVM instead of interpreting + #[clap(long)] + jit: bool, + + /// Expression to evaluate + expr: String, +} + +impl Eval { + pub fn run(self) -> Result<()> { + let (_, parsed) = parser::expr(&self.expr)?; + let hir = tc::typecheck_expr(parsed)?; + let result = if self.jit { + codegen::jit_eval::<i64>(&hir)?.into() + } else { + interpreter::eval(&hir)? + }; + println!("{}", result); + Ok(()) + } +} diff --git a/users/grfn/achilles/src/commands/mod.rs b/users/grfn/achilles/src/commands/mod.rs new file mode 100644 index 000000000000..fd0a822708c2 --- /dev/null +++ b/users/grfn/achilles/src/commands/mod.rs @@ -0,0 +1,7 @@ +pub mod check; +pub mod compile; +pub mod eval; + +pub use check::Check; +pub use compile::Compile; +pub use eval::Eval; diff --git a/users/grfn/achilles/src/common/env.rs b/users/grfn/achilles/src/common/env.rs new file mode 100644 index 000000000000..59a5e46c466f --- /dev/null +++ b/users/grfn/achilles/src/common/env.rs @@ -0,0 +1,59 @@ +use std::borrow::Borrow; +use std::collections::HashMap; +use std::hash::Hash; +use std::mem; + +/// A lexical environment +#[derive(Debug, PartialEq, Eq)] +pub struct Env<K: Eq + Hash, V>(Vec<HashMap<K, V>>); + +impl<K, V> Default for Env<K, V> +where + K: Eq + Hash, +{ + fn default() -> Self { + Self::new() + } +} + +impl<K, V> Env<K, V> +where + K: Eq + Hash, +{ + pub fn new() -> Self { + Self(vec![Default::default()]) + } + + pub fn push(&mut self) { + self.0.push(Default::default()); + } + + pub fn pop(&mut self) { + self.0.pop(); + } + + pub fn save(&mut self) -> Self { + mem::take(self) + } + + pub fn restore(&mut self, saved: Self) { + *self = saved; + } + + pub fn set(&mut self, k: K, v: V) { + self.0.last_mut().unwrap().insert(k, v); + } + + pub fn resolve<'a, Q>(&'a self, k: &Q) -> Option<&'a V> + where + K: Borrow<Q>, + Q: Hash + Eq + ?Sized, + { + for ctx in self.0.iter().rev() { + if let Some(res) = ctx.get(k) { + return Some(res); + } + } + None + } +} diff --git a/users/grfn/achilles/src/common/error.rs b/users/grfn/achilles/src/common/error.rs new file mode 100644 index 000000000000..51575a895e91 --- /dev/null +++ b/users/grfn/achilles/src/common/error.rs @@ -0,0 +1,59 @@ +use std::{io, result}; + +use thiserror::Error; + +use crate::{codegen, interpreter, parser, tc}; + +#[derive(Error, Debug)] +pub enum Error { + #[error(transparent)] + IOError(#[from] io::Error), + + #[error("Error parsing input: {0}")] + ParseError(#[from] parser::Error), + + #[error("Error evaluating expression: {0}")] + EvalError(#[from] interpreter::Error), + + #[error("Compile error: {0}")] + CodegenError(#[from] codegen::Error), + + #[error("Type error: {0}")] + TypeError(#[from] tc::Error), + + #[error("{0}")] + Message(String), +} + +impl From<String> for Error { + fn from(s: String) -> Self { + Self::Message(s) + } +} + +impl<'a> From<&'a str> for Error { + fn from(s: &'a str) -> Self { + Self::Message(s.to_owned()) + } +} + +impl<'a> From<nom::Err<nom::error::Error<&'a str>>> for Error { + fn from(e: nom::Err<nom::error::Error<&'a str>>) -> Self { + use nom::error::Error as NomError; + use nom::Err::*; + + Self::ParseError(match e { + Incomplete(i) => Incomplete(i), + Error(NomError { input, code }) => Error(NomError { + input: input.to_owned(), + code, + }), + Failure(NomError { input, code }) => Failure(NomError { + input: input.to_owned(), + code, + }), + }) + } +} + +pub type Result<T> = result::Result<T, Error>; diff --git a/users/grfn/achilles/src/common/mod.rs b/users/grfn/achilles/src/common/mod.rs new file mode 100644 index 000000000000..8368a6dd180f --- /dev/null +++ b/users/grfn/achilles/src/common/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod env; +pub(crate) mod error; +pub(crate) mod namer; + +pub use error::{Error, Result}; +pub use namer::{Namer, NamerOf}; diff --git a/users/grfn/achilles/src/common/namer.rs b/users/grfn/achilles/src/common/namer.rs new file mode 100644 index 000000000000..016e9f6ed99a --- /dev/null +++ b/users/grfn/achilles/src/common/namer.rs @@ -0,0 +1,122 @@ +use std::fmt::Display; +use std::marker::PhantomData; + +pub struct Namer<T, F> { + make_name: F, + counter: u64, + _phantom: PhantomData<T>, +} + +impl<T, F> Namer<T, F> { + pub fn new(make_name: F) -> Self { + Namer { + make_name, + counter: 0, + _phantom: PhantomData, + } + } +} + +impl Namer<String, Box<dyn Fn(u64) -> String>> { + pub fn with_prefix<T>(prefix: T) -> Self + where + T: Display + 'static, + { + Namer::new(move |i| format!("{}{}", prefix, i)).boxed() + } + + pub fn with_suffix<T>(suffix: T) -> Self + where + T: Display + 'static, + { + Namer::new(move |i| format!("{}{}", i, suffix)).boxed() + } + + pub fn alphabetic() -> Self { + Namer::new(|i| { + if i <= 26 { + std::char::from_u32((i + 96) as u32).unwrap().to_string() + } else { + format!( + "{}{}", + std::char::from_u32(((i % 26) + 96) as u32).unwrap(), + i - 26 + ) + } + }) + .boxed() + } +} + +impl<T, F> Namer<T, F> +where + F: Fn(u64) -> T, +{ + pub fn make_name(&mut self) -> T { + self.counter += 1; + (self.make_name)(self.counter) + } + + pub fn boxed(self) -> NamerOf<T> + where + F: 'static, + { + Namer { + make_name: Box::new(self.make_name), + counter: self.counter, + _phantom: self._phantom, + } + } + + pub fn map<G, U>(self, f: G) -> NamerOf<U> + where + G: Fn(T) -> U + 'static, + T: 'static, + F: 'static, + { + Namer { + counter: self.counter, + make_name: Box::new(move |x| f((self.make_name)(x))), + _phantom: PhantomData, + } + } +} + +pub type NamerOf<T> = Namer<T, Box<dyn Fn(u64) -> T>>; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prefix() { + let mut namer = Namer::with_prefix("t"); + assert_eq!(namer.make_name(), "t1"); + assert_eq!(namer.make_name(), "t2"); + } + + #[test] + fn suffix() { + let mut namer = Namer::with_suffix("t"); + assert_eq!(namer.make_name(), "1t"); + assert_eq!(namer.make_name(), "2t"); + } + + #[test] + fn alphabetic() { + let mut namer = Namer::alphabetic(); + assert_eq!(namer.make_name(), "a"); + assert_eq!(namer.make_name(), "b"); + (0..25).for_each(|_| { + namer.make_name(); + }); + assert_eq!(namer.make_name(), "b2"); + } + + #[test] + fn custom_callback() { + let mut namer = Namer::new(|n| n + 1); + assert_eq!(namer.make_name(), 2); + assert_eq!(namer.make_name(), 3); + } +} diff --git a/users/grfn/achilles/src/compiler.rs b/users/grfn/achilles/src/compiler.rs new file mode 100644 index 000000000000..45b215473d7f --- /dev/null +++ b/users/grfn/achilles/src/compiler.rs @@ -0,0 +1,89 @@ +use std::fmt::{self, Display}; +use std::path::Path; +use std::str::FromStr; +use std::{fs, result}; + +use clap::Clap; +use test_strategy::Arbitrary; + +use crate::codegen::{self, Codegen}; +use crate::common::Result; +use crate::passes::hir::{monomorphize, strip_positive_units}; +use crate::{parser, tc}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Arbitrary)] +pub enum OutputFormat { + LLVM, + Bitcode, +} + +impl Default for OutputFormat { + fn default() -> Self { + Self::Bitcode + } +} + +impl FromStr for OutputFormat { + type Err = String; + + fn from_str(s: &str) -> result::Result<Self, Self::Err> { + match s { + "llvm" => Ok(Self::LLVM), + "binary" => Ok(Self::Bitcode), + _ => Err(format!( + "Invalid output format {}, expected one of {{llvm, binary}}", + s + )), + } + } +} + +impl Display for OutputFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OutputFormat::LLVM => f.write_str("llvm"), + OutputFormat::Bitcode => f.write_str("binary"), + } + } +} + +#[derive(Clap, Debug, PartialEq, Eq, Default)] +pub struct CompilerOptions { + #[clap(long, short = 'f', default_value)] + format: OutputFormat, +} + +pub fn compile_file(input: &Path, output: &Path, options: &CompilerOptions) -> Result<()> { + let src = fs::read_to_string(input)?; + let (_, decls) = parser::toplevel(&src)?; + let mut decls = tc::typecheck_toplevel(decls)?; + monomorphize::run_toplevel(&mut decls); + strip_positive_units::run_toplevel(&mut decls); + + let context = codegen::Context::create(); + let mut codegen = Codegen::new( + &context, + &input + .file_stem() + .map_or("UNKNOWN".to_owned(), |s| s.to_string_lossy().into_owned()), + ); + for decl in &decls { + codegen.codegen_decl(decl)?; + } + match options.format { + OutputFormat::LLVM => codegen.print_to_file(output)?, + OutputFormat::Bitcode => codegen.binary_to_file(output)?, + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use test_strategy::proptest; + + #[proptest] + fn output_format_display_from_str_round_trip(of: OutputFormat) { + assert_eq!(OutputFormat::from_str(&of.to_string()), Ok(of)); + } +} diff --git a/users/grfn/achilles/src/interpreter/error.rs b/users/grfn/achilles/src/interpreter/error.rs new file mode 100644 index 000000000000..268d6f479a1e --- /dev/null +++ b/users/grfn/achilles/src/interpreter/error.rs @@ -0,0 +1,19 @@ +use std::result; + +use thiserror::Error; + +use crate::ast::{Ident, Type}; + +#[derive(Debug, PartialEq, Eq, Error)] +pub enum Error { + #[error("Undefined variable {0}")] + UndefinedVariable(Ident<'static>), + + #[error("Unexpected type {actual}, expected type {expected}")] + InvalidType { + actual: Type<'static>, + expected: Type<'static>, + }, +} + +pub type Result<T> = result::Result<T, Error>; diff --git a/users/grfn/achilles/src/interpreter/mod.rs b/users/grfn/achilles/src/interpreter/mod.rs new file mode 100644 index 000000000000..70df7a0724a5 --- /dev/null +++ b/users/grfn/achilles/src/interpreter/mod.rs @@ -0,0 +1,203 @@ +mod error; +mod value; + +use itertools::Itertools; +use value::Val; + +pub use self::error::{Error, Result}; +pub use self::value::{Function, Value}; +use crate::ast::hir::{Binding, Expr, Pattern}; +use crate::ast::{BinaryOperator, FunctionType, Ident, Literal, Type, UnaryOperator}; +use crate::common::env::Env; + +#[derive(Debug, Default)] +pub struct Interpreter<'a> { + env: Env<&'a Ident<'a>, Value<'a>>, +} + +impl<'a> Interpreter<'a> { + pub fn new() -> Self { + Self::default() + } + + fn resolve(&self, var: &'a Ident<'a>) -> Result<Value<'a>> { + self.env + .resolve(var) + .cloned() + .ok_or_else(|| Error::UndefinedVariable(var.to_owned())) + } + + fn bind_pattern(&mut self, pattern: &'a Pattern<'a, Type>, value: Value<'a>) { + match pattern { + Pattern::Id(id, _) => self.env.set(id, value), + Pattern::Tuple(pats) => { + for (pat, val) in pats.iter().zip(value.as_tuple().unwrap().clone()) { + self.bind_pattern(pat, val); + } + } + } + } + + pub fn eval(&mut self, expr: &'a Expr<'a, Type>) -> Result<Value<'a>> { + let res = match expr { + Expr::Ident(id, _) => self.resolve(id), + Expr::Literal(Literal::Int(i), _) => Ok((*i).into()), + Expr::Literal(Literal::Bool(b), _) => Ok((*b).into()), + Expr::Literal(Literal::String(s), _) => Ok(s.clone().into()), + Expr::Literal(Literal::Unit, _) => unreachable!(), + Expr::UnaryOp { op, rhs, .. } => { + let rhs = self.eval(rhs)?; + match op { + UnaryOperator::Neg => -rhs, + _ => unimplemented!(), + } + } + Expr::BinaryOp { lhs, op, rhs, .. } => { + let lhs = self.eval(lhs)?; + let rhs = self.eval(rhs)?; + match op { + BinaryOperator::Add => lhs + rhs, + BinaryOperator::Sub => lhs - rhs, + BinaryOperator::Mul => lhs * rhs, + BinaryOperator::Div => lhs / rhs, + BinaryOperator::Pow => todo!(), + BinaryOperator::Equ => Ok(lhs.eq(&rhs).into()), + BinaryOperator::Neq => todo!(), + } + } + Expr::Let { bindings, body, .. } => { + self.env.push(); + for Binding { pat, body, .. } in bindings { + let val = self.eval(body)?; + self.bind_pattern(pat, val); + } + let res = self.eval(body)?; + self.env.pop(); + Ok(res) + } + Expr::If { + condition, + then, + else_, + .. + } => { + let condition = self.eval(condition)?; + if *(condition.as_type::<bool>()?) { + self.eval(then) + } else { + self.eval(else_) + } + } + Expr::Call { ref fun, args, .. } => { + let fun = self.eval(fun)?; + let expected_type = FunctionType { + args: args.iter().map(|_| Type::Int).collect(), + ret: Box::new(Type::Int), + }; + + let Function { + args: function_args, + body, + .. + } = fun.as_function(expected_type)?; + let arg_values = function_args.iter().zip( + args.iter() + .map(|v| self.eval(v)) + .collect::<Result<Vec<_>>>()?, + ); + let mut interpreter = Interpreter::new(); + for (arg_name, arg_value) in arg_values { + interpreter.env.set(arg_name, arg_value); + } + Ok(Value::from(*interpreter.eval(body)?.as_type::<i64>()?)) + } + Expr::Fun { + type_args: _, + args, + body, + type_, + } => { + let type_ = match type_ { + Type::Function(ft) => ft.clone(), + _ => unreachable!("Function expression without function type"), + }; + + Ok(Value::from(value::Function { + // TODO + type_, + args: args.iter().map(|(arg, _)| arg.to_owned()).collect(), + body: (**body).to_owned(), + })) + } + Expr::Tuple(members, _) => Ok(Val::Tuple( + members + .into_iter() + .map(|expr| self.eval(expr)) + .try_collect()?, + ) + .into()), + }?; + debug_assert_eq!(&res.type_(), expr.type_()); + Ok(res) + } +} + +pub fn eval<'a>(expr: &'a Expr<'a, Type>) -> Result<Value<'a>> { + let mut interpreter = Interpreter::new(); + interpreter.eval(expr) +} + +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use super::value::{TypeOf, Val}; + use super::*; + use BinaryOperator::*; + + fn int_lit(i: u64) -> Box<Expr<'static, Type<'static>>> { + Box::new(Expr::Literal(Literal::Int(i), Type::Int)) + } + + fn do_eval<T>(src: &str) -> T + where + for<'a> &'a T: TryFrom<&'a Val<'a>>, + T: Clone + TypeOf, + { + let expr = crate::parser::expr(src).unwrap().1; + let hir = crate::tc::typecheck_expr(expr).unwrap(); + let res = eval(&hir).unwrap(); + res.as_type::<T>().unwrap().clone() + } + + #[test] + fn simple_addition() { + let expr = Expr::BinaryOp { + lhs: int_lit(1), + op: Mul, + rhs: int_lit(2), + type_: Type::Int, + }; + let res = eval(&expr).unwrap(); + assert_eq!(*res.as_type::<i64>().unwrap(), 2); + } + + #[test] + fn variable_shadowing() { + let res = do_eval::<i64>("let x = 1 in (let x = 2 in x) + x"); + assert_eq!(res, 3); + } + + #[test] + fn conditional_with_equals() { + let res = do_eval::<i64>("let x = 1 in if x == 1 then 2 else 4"); + assert_eq!(res, 2); + } + + #[test] + #[ignore] + fn function_call() { + let res = do_eval::<i64>("let id = fn x = x in id 1"); + assert_eq!(res, 1); + } +} diff --git a/users/grfn/achilles/src/interpreter/value.rs b/users/grfn/achilles/src/interpreter/value.rs new file mode 100644 index 000000000000..272d1167a33c --- /dev/null +++ b/users/grfn/achilles/src/interpreter/value.rs @@ -0,0 +1,224 @@ +use std::borrow::Cow; +use std::convert::TryFrom; +use std::fmt::{self, Display}; +use std::ops::{Add, Div, Mul, Neg, Sub}; +use std::rc::Rc; +use std::result; + +use derive_more::{Deref, From, TryInto}; +use itertools::Itertools; + +use super::{Error, Result}; +use crate::ast::hir::Expr; +use crate::ast::{FunctionType, Ident, Type}; + +#[derive(Debug, Clone)] +pub struct Function<'a> { + pub type_: FunctionType<'a>, + pub args: Vec<Ident<'a>>, + pub body: Expr<'a, Type<'a>>, +} + +#[derive(From, TryInto)] +#[try_into(owned, ref)] +pub enum Val<'a> { + Int(i64), + Float(f64), + Bool(bool), + String(Cow<'a, str>), + Tuple(Vec<Value<'a>>), + Function(Function<'a>), +} + +impl<'a> TryFrom<Val<'a>> for String { + type Error = (); + + fn try_from(value: Val<'a>) -> result::Result<Self, Self::Error> { + match value { + Val::String(s) => Ok(s.into_owned()), + _ => Err(()), + } + } +} + +impl<'a> fmt::Debug for Val<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Val::Int(x) => f.debug_tuple("Int").field(x).finish(), + Val::Float(x) => f.debug_tuple("Float").field(x).finish(), + Val::Bool(x) => f.debug_tuple("Bool").field(x).finish(), + Val::String(s) => f.debug_tuple("String").field(s).finish(), + Val::Function(Function { type_, .. }) => { + f.debug_struct("Function").field("type_", type_).finish() + } + Val::Tuple(members) => f.debug_tuple("Tuple").field(members).finish(), + } + } +} + +impl<'a> PartialEq for Val<'a> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Val::Int(x), Val::Int(y)) => x == y, + (Val::Float(x), Val::Float(y)) => x == y, + (Val::Bool(x), Val::Bool(y)) => x == y, + (Val::Function(_), Val::Function(_)) => false, + (_, _) => false, + } + } +} + +impl<'a> From<u64> for Val<'a> { + fn from(i: u64) -> Self { + Self::from(i as i64) + } +} + +impl<'a> Display for Val<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Val::Int(x) => x.fmt(f), + Val::Float(x) => x.fmt(f), + Val::Bool(x) => x.fmt(f), + Val::String(s) => write!(f, "{:?}", s), + Val::Function(Function { type_, .. }) => write!(f, "<{}>", type_), + Val::Tuple(members) => write!(f, "({})", members.iter().join(", ")), + } + } +} + +impl<'a> Val<'a> { + pub fn type_(&self) -> Type { + match self { + Val::Int(_) => Type::Int, + Val::Float(_) => Type::Float, + Val::Bool(_) => Type::Bool, + Val::String(_) => Type::CString, + Val::Function(Function { type_, .. }) => Type::Function(type_.clone()), + Val::Tuple(members) => Type::Tuple(members.iter().map(|expr| expr.type_()).collect()), + } + } + + pub fn as_type<'b, T>(&'b self) -> Result<&'b T> + where + T: TypeOf + 'b + Clone, + &'b T: TryFrom<&'b Self>, + { + <&T>::try_from(self).map_err(|_| Error::InvalidType { + actual: self.type_().to_owned(), + expected: <T as TypeOf>::type_of(), + }) + } + + pub fn as_function<'b>(&'b self, function_type: FunctionType) -> Result<&'b Function<'a>> { + match self { + Val::Function(f) if f.type_ == function_type => Ok(&f), + _ => Err(Error::InvalidType { + actual: self.type_().to_owned(), + expected: Type::Function(function_type.to_owned()), + }), + } + } + + pub fn as_tuple(&self) -> Option<&Vec<Value<'a>>> { + if let Self::Tuple(v) = self { + Some(v) + } else { + None + } + } + + pub fn try_into_tuple(self) -> result::Result<Vec<Value<'a>>, Self> { + if let Self::Tuple(v) = self { + Ok(v) + } else { + Err(self) + } + } +} + +#[derive(Debug, PartialEq, Clone, Deref)] +pub struct Value<'a>(Rc<Val<'a>>); + +impl<'a> Display for Value<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl<'a, T> From<T> for Value<'a> +where + Val<'a>: From<T>, +{ + fn from(x: T) -> Self { + Self(Rc::new(x.into())) + } +} + +impl<'a> Neg for Value<'a> { + type Output = Result<Value<'a>>; + + fn neg(self) -> Self::Output { + Ok((-self.as_type::<i64>()?).into()) + } +} + +impl<'a> Add for Value<'a> { + type Output = Result<Value<'a>>; + + fn add(self, rhs: Self) -> Self::Output { + Ok((self.as_type::<i64>()? + rhs.as_type::<i64>()?).into()) + } +} + +impl<'a> Sub for Value<'a> { + type Output = Result<Value<'a>>; + + fn sub(self, rhs: Self) -> Self::Output { + Ok((self.as_type::<i64>()? - rhs.as_type::<i64>()?).into()) + } +} + +impl<'a> Mul for Value<'a> { + type Output = Result<Value<'a>>; + + fn mul(self, rhs: Self) -> Self::Output { + Ok((self.as_type::<i64>()? * rhs.as_type::<i64>()?).into()) + } +} + +impl<'a> Div for Value<'a> { + type Output = Result<Value<'a>>; + + fn div(self, rhs: Self) -> Self::Output { + Ok((self.as_type::<f64>()? / rhs.as_type::<f64>()?).into()) + } +} + +pub trait TypeOf { + fn type_of() -> Type<'static>; +} + +impl TypeOf for i64 { + fn type_of() -> Type<'static> { + Type::Int + } +} + +impl TypeOf for bool { + fn type_of() -> Type<'static> { + Type::Bool + } +} + +impl TypeOf for f64 { + fn type_of() -> Type<'static> { + Type::Float + } +} + +impl TypeOf for String { + fn type_of() -> Type<'static> { + Type::CString + } +} diff --git a/users/grfn/achilles/src/main.rs b/users/grfn/achilles/src/main.rs new file mode 100644 index 000000000000..5ae1b59b3a8e --- /dev/null +++ b/users/grfn/achilles/src/main.rs @@ -0,0 +1,36 @@ +use clap::Clap; + +pub mod ast; +pub mod codegen; +pub(crate) mod commands; +pub(crate) mod common; +pub mod compiler; +pub mod interpreter; +pub(crate) mod passes; +#[macro_use] +pub mod parser; +pub mod tc; + +pub use common::{Error, Result}; + +#[derive(Clap)] +struct Opts { + #[clap(subcommand)] + subcommand: Command, +} + +#[derive(Clap)] +enum Command { + Eval(commands::Eval), + Compile(commands::Compile), + Check(commands::Check), +} + +fn main() -> anyhow::Result<()> { + let opts = Opts::parse(); + match opts.subcommand { + Command::Eval(eval) => Ok(eval.run()?), + Command::Compile(compile) => Ok(compile.run()?), + Command::Check(check) => Ok(check.run()?), + } +} diff --git a/users/grfn/achilles/src/parser/expr.rs b/users/grfn/achilles/src/parser/expr.rs new file mode 100644 index 000000000000..f596b18970aa --- /dev/null +++ b/users/grfn/achilles/src/parser/expr.rs @@ -0,0 +1,718 @@ +use std::borrow::Cow; + +use nom::alt; +use nom::character::complete::{digit1, multispace0, multispace1}; +use nom::{ + call, char, complete, delimited, do_parse, flat_map, many0, map, named, opt, parse_to, + preceded, separated_list0, separated_list1, tag, tuple, +}; +use pratt::{Affix, Associativity, PrattParser, Precedence}; + +use super::util::comma; +use crate::ast::{BinaryOperator, Binding, Expr, Fun, Literal, Pattern, UnaryOperator}; +use crate::parser::{arg, ident, type_}; + +#[derive(Debug)] +enum TokenTree<'a> { + Prefix(UnaryOperator), + // Postfix(char), + Infix(BinaryOperator), + Primary(Expr<'a>), + Group(Vec<TokenTree<'a>>), +} + +named!(prefix(&str) -> TokenTree, map!(alt!( + complete!(char!('-')) => { |_| UnaryOperator::Neg } | + complete!(char!('!')) => { |_| UnaryOperator::Not } +), TokenTree::Prefix)); + +named!(infix(&str) -> TokenTree, map!(alt!( + complete!(tag!("==")) => { |_| BinaryOperator::Equ } | + complete!(tag!("!=")) => { |_| BinaryOperator::Neq } | + complete!(char!('+')) => { |_| BinaryOperator::Add } | + complete!(char!('-')) => { |_| BinaryOperator::Sub } | + complete!(char!('*')) => { |_| BinaryOperator::Mul } | + complete!(char!('/')) => { |_| BinaryOperator::Div } | + complete!(char!('^')) => { |_| BinaryOperator::Pow } +), TokenTree::Infix)); + +named!(primary(&str) -> TokenTree, alt!( + do_parse!( + multispace0 >> + char!('(') >> + multispace0 >> + group: group >> + multispace0 >> + char!(')') >> + multispace0 >> + (TokenTree::Group(group)) + ) | + delimited!(multispace0, simple_expr, multispace0) => { |s| TokenTree::Primary(s) } +)); + +named!( + rest(&str) -> Vec<(TokenTree, Vec<TokenTree>, TokenTree)>, + many0!(tuple!( + infix, + delimited!(multispace0, many0!(prefix), multispace0), + primary + // many0!(postfix) + )) +); + +named!(group(&str) -> Vec<TokenTree>, do_parse!( + prefix: many0!(prefix) + >> primary: primary + // >> postfix: many0!(postfix) + >> rest: rest + >> ({ + let mut res = prefix; + res.push(primary); + // res.append(&mut postfix); + for (infix, mut prefix, primary/*, mut postfix*/) in rest { + res.push(infix); + res.append(&mut prefix); + res.push(primary); + // res.append(&mut postfix); + } + res + }) +)); + +fn token_tree(i: &str) -> nom::IResult<&str, Vec<TokenTree>> { + group(i) +} + +struct ExprParser; + +impl<'a, I> PrattParser<I> for ExprParser +where + I: Iterator<Item = TokenTree<'a>>, +{ + type Error = pratt::NoError; + type Input = TokenTree<'a>; + type Output = Expr<'a>; + + fn query(&mut self, input: &Self::Input) -> Result<Affix, Self::Error> { + use BinaryOperator::*; + use UnaryOperator::*; + + Ok(match input { + TokenTree::Infix(Add) => Affix::Infix(Precedence(6), Associativity::Left), + TokenTree::Infix(Sub) => Affix::Infix(Precedence(6), Associativity::Left), + TokenTree::Infix(Mul) => Affix::Infix(Precedence(7), Associativity::Left), + TokenTree::Infix(Div) => Affix::Infix(Precedence(7), Associativity::Left), + TokenTree::Infix(Pow) => Affix::Infix(Precedence(8), Associativity::Right), + TokenTree::Infix(Equ) => Affix::Infix(Precedence(4), Associativity::Right), + TokenTree::Infix(Neq) => Affix::Infix(Precedence(4), Associativity::Right), + TokenTree::Prefix(Neg) => Affix::Prefix(Precedence(6)), + TokenTree::Prefix(Not) => Affix::Prefix(Precedence(6)), + TokenTree::Primary(_) => Affix::Nilfix, + TokenTree::Group(_) => Affix::Nilfix, + }) + } + + fn primary(&mut self, input: Self::Input) -> Result<Self::Output, Self::Error> { + Ok(match input { + TokenTree::Primary(expr) => expr, + TokenTree::Group(group) => self.parse(&mut group.into_iter()).unwrap(), + _ => unreachable!(), + }) + } + + fn infix( + &mut self, + lhs: Self::Output, + op: Self::Input, + rhs: Self::Output, + ) -> Result<Self::Output, Self::Error> { + let op = match op { + TokenTree::Infix(op) => op, + _ => unreachable!(), + }; + Ok(Expr::BinaryOp { + lhs: Box::new(lhs), + op, + rhs: Box::new(rhs), + }) + } + + fn prefix(&mut self, op: Self::Input, rhs: Self::Output) -> Result<Self::Output, Self::Error> { + let op = match op { + TokenTree::Prefix(op) => op, + _ => unreachable!(), + }; + + Ok(Expr::UnaryOp { + op, + rhs: Box::new(rhs), + }) + } + + fn postfix( + &mut self, + _lhs: Self::Output, + _op: Self::Input, + ) -> Result<Self::Output, Self::Error> { + unreachable!() + } +} + +named!(int(&str) -> Literal, map!(flat_map!(digit1, parse_to!(u64)), Literal::Int)); + +named!(bool_(&str) -> Literal, alt!( + complete!(tag!("true")) => { |_| Literal::Bool(true) } | + complete!(tag!("false")) => { |_| Literal::Bool(false) } +)); + +fn string_internal(i: &str) -> nom::IResult<&str, Cow<'_, str>, nom::error::Error<&str>> { + // TODO(grfn): use String::split_once when that's stable + let (s, rem) = if let Some(pos) = i.find('"') { + (&i[..pos], &i[(pos + 1)..]) + } else { + return Err(nom::Err::Error(nom::error::Error::new( + i, + nom::error::ErrorKind::Tag, + ))); + }; + + Ok((rem, Cow::Borrowed(s))) +} + +named!(string(&str) -> Literal, preceded!( + complete!(char!('"')), + map!( + string_internal, + |s| Literal::String(s) + ) +)); + +named!(unit(&str) -> Literal, map!(complete!(tag!("()")), |_| Literal::Unit)); + +named!(literal(&str) -> Literal, alt!(int | bool_ | string | unit)); + +named!(literal_expr(&str) -> Expr, map!(literal, Expr::Literal)); + +named!(tuple(&str) -> Expr, do_parse!( + complete!(tag!("(")) + >> multispace0 + >> fst: expr + >> comma + >> rest: separated_list0!( + comma, + expr + ) + >> multispace0 + >> tag!(")") + >> ({ + let mut members = Vec::with_capacity(rest.len() + 1); + members.push(fst); + members.append(&mut rest.clone()); + Expr::Tuple(members) + }) +)); + +named!(tuple_pattern(&str) -> Pattern, do_parse!( + complete!(tag!("(")) + >> multispace0 + >> pats: separated_list0!( + comma, + pattern + ) + >> multispace0 + >> tag!(")") + >> (Pattern::Tuple(pats)) +)); + +named!(pattern(&str) -> Pattern, alt!( + ident => { |id| Pattern::Id(id) } | + tuple_pattern +)); + +named!(binding(&str) -> Binding, do_parse!( + multispace0 + >> pat: pattern + >> multispace0 + >> type_: opt!(preceded!(tuple!(tag!(":"), multispace0), type_)) + >> multispace0 + >> char!('=') + >> multispace0 + >> body: expr + >> (Binding { + pat, + type_, + body + }) +)); + +named!(let_(&str) -> Expr, do_parse!( + tag!("let") + >> multispace0 + >> bindings: separated_list1!(alt!(char!(';') | char!('\n')), binding) + >> multispace0 + >> tag!("in") + >> multispace0 + >> body: expr + >> (Expr::Let { + bindings, + body: Box::new(body) + }) +)); + +named!(if_(&str) -> Expr, do_parse! ( + tag!("if") + >> multispace0 + >> condition: expr + >> multispace0 + >> tag!("then") + >> multispace0 + >> then: expr + >> multispace0 + >> tag!("else") + >> multispace0 + >> else_: expr + >> (Expr::If { + condition: Box::new(condition), + then: Box::new(then), + else_: Box::new(else_) + }) +)); + +named!(ident_expr(&str) -> Expr, map!(ident, Expr::Ident)); + +fn ascripted<'a>( + p: impl Fn(&'a str) -> nom::IResult<&'a str, Expr, nom::error::Error<&'a str>> + 'a, +) -> impl Fn(&'a str) -> nom::IResult<&str, Expr, nom::error::Error<&'a str>> { + move |i| { + do_parse!( + i, + expr: p + >> multispace0 + >> complete!(tag!(":")) + >> multispace0 + >> type_: type_ + >> (Expr::Ascription { + expr: Box::new(expr), + type_ + }) + ) + } +} + +named!(paren_expr(&str) -> Expr, + delimited!(complete!(tag!("(")), expr, complete!(tag!(")")))); + +named!(funcref(&str) -> Expr, alt!( + ident_expr | + tuple | + paren_expr +)); + +named!(no_arg_call(&str) -> Expr, do_parse!( + fun: funcref + >> complete!(tag!("()")) + >> (Expr::Call { + fun: Box::new(fun), + args: vec![], + }) +)); + +named!(fun_expr(&str) -> Expr, do_parse!( + tag!("fn") + >> multispace1 + >> args: separated_list0!(multispace1, arg) + >> multispace0 + >> char!('=') + >> multispace0 + >> body: expr + >> (Expr::Fun(Box::new(Fun { + args, + body + }))) +)); + +named!(fn_arg(&str) -> Expr, alt!( + ident_expr | + literal_expr | + tuple | + paren_expr +)); + +named!(call_with_args(&str) -> Expr, do_parse!( + fun: funcref + >> multispace1 + >> args: separated_list1!(multispace1, fn_arg) + >> (Expr::Call { + fun: Box::new(fun), + args + }) +)); + +named!(simple_expr_unascripted(&str) -> Expr, alt!( + let_ | + if_ | + fun_expr | + literal_expr | + ident_expr | + tuple +)); + +named!(simple_expr(&str) -> Expr, alt!( + call!(ascripted(simple_expr_unascripted)) | + simple_expr_unascripted +)); + +named!(pub expr(&str) -> Expr, alt!( + no_arg_call | + call_with_args | + map!(token_tree, |tt| { + ExprParser.parse(&mut tt.into_iter()).unwrap() + }) | + simple_expr +)); + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use crate::ast::{Arg, Ident, Pattern, Type}; + use std::convert::TryFrom; + use BinaryOperator::*; + use Expr::{BinaryOp, If, Let, UnaryOp}; + use UnaryOperator::*; + + pub(crate) fn ident_expr(s: &str) -> Box<Expr> { + Box::new(Expr::Ident(Ident::try_from(s).unwrap())) + } + + mod operators { + use super::*; + + #[test] + fn mul_plus() { + let (rem, res) = expr("x*y+z").unwrap(); + assert!(rem.is_empty()); + assert_eq!( + res, + BinaryOp { + lhs: Box::new(BinaryOp { + lhs: ident_expr("x"), + op: Mul, + rhs: ident_expr("y") + }), + op: Add, + rhs: ident_expr("z") + } + ) + } + + #[test] + fn mul_plus_ws() { + let (rem, res) = expr("x * y + z").unwrap(); + assert!(rem.is_empty(), "non-empty remainder: \"{}\"", rem); + assert_eq!( + res, + BinaryOp { + lhs: Box::new(BinaryOp { + lhs: ident_expr("x"), + op: Mul, + rhs: ident_expr("y") + }), + op: Add, + rhs: ident_expr("z") + } + ) + } + + #[test] + fn unary() { + let (rem, res) = expr("x * -z").unwrap(); + assert!(rem.is_empty(), "non-empty remainder: \"{}\"", rem); + assert_eq!( + res, + BinaryOp { + lhs: ident_expr("x"), + op: Mul, + rhs: Box::new(UnaryOp { + op: Neg, + rhs: ident_expr("z"), + }) + } + ) + } + + #[test] + fn mul_literal() { + let (rem, res) = expr("x * 3").unwrap(); + assert!(rem.is_empty()); + assert_eq!( + res, + BinaryOp { + lhs: ident_expr("x"), + op: Mul, + rhs: Box::new(Expr::Literal(Literal::Int(3))), + } + ) + } + + #[test] + fn equ() { + let res = test_parse!(expr, "x * 7 == 7"); + assert_eq!( + res, + BinaryOp { + lhs: Box::new(BinaryOp { + lhs: ident_expr("x"), + op: Mul, + rhs: Box::new(Expr::Literal(Literal::Int(7))) + }), + op: Equ, + rhs: Box::new(Expr::Literal(Literal::Int(7))) + } + ) + } + } + + #[test] + fn unit() { + assert_eq!(test_parse!(expr, "()"), Expr::Literal(Literal::Unit)); + } + + #[test] + fn bools() { + assert_eq!( + test_parse!(expr, "true"), + Expr::Literal(Literal::Bool(true)) + ); + assert_eq!( + test_parse!(expr, "false"), + Expr::Literal(Literal::Bool(false)) + ); + } + + #[test] + fn tuple() { + assert_eq!( + test_parse!(expr, "(1, \"seven\")"), + Expr::Tuple(vec![ + Expr::Literal(Literal::Int(1)), + Expr::Literal(Literal::String(Cow::Borrowed("seven"))) + ]) + ) + } + + #[test] + fn simple_string_lit() { + assert_eq!( + test_parse!(expr, "\"foobar\""), + Expr::Literal(Literal::String(Cow::Borrowed("foobar"))) + ) + } + + #[test] + fn let_complex() { + let res = test_parse!(expr, "let x = 1; y = x * 7 in (x + y) * 4"); + assert_eq!( + res, + Let { + bindings: vec![ + Binding { + pat: Pattern::Id(Ident::try_from("x").unwrap()), + type_: None, + body: Expr::Literal(Literal::Int(1)) + }, + Binding { + pat: Pattern::Id(Ident::try_from("y").unwrap()), + type_: None, + body: Expr::BinaryOp { + lhs: ident_expr("x"), + op: Mul, + rhs: Box::new(Expr::Literal(Literal::Int(7))) + } + } + ], + body: Box::new(Expr::BinaryOp { + lhs: Box::new(Expr::BinaryOp { + lhs: ident_expr("x"), + op: Add, + rhs: ident_expr("y"), + }), + op: Mul, + rhs: Box::new(Expr::Literal(Literal::Int(4))), + }) + } + ) + } + + #[test] + fn if_simple() { + let res = test_parse!(expr, "if x == 8 then 9 else 20"); + assert_eq!( + res, + If { + condition: Box::new(BinaryOp { + lhs: ident_expr("x"), + op: Equ, + rhs: Box::new(Expr::Literal(Literal::Int(8))), + }), + then: Box::new(Expr::Literal(Literal::Int(9))), + else_: Box::new(Expr::Literal(Literal::Int(20))) + } + ) + } + + #[test] + fn no_arg_call() { + let res = test_parse!(expr, "f()"); + assert_eq!( + res, + Expr::Call { + fun: ident_expr("f"), + args: vec![] + } + ); + } + + #[test] + fn unit_call() { + let res = test_parse!(expr, "f ()"); + assert_eq!( + res, + Expr::Call { + fun: ident_expr("f"), + args: vec![Expr::Literal(Literal::Unit)] + } + ) + } + + #[test] + fn call_with_args() { + let res = test_parse!(expr, "f x 1"); + assert_eq!( + res, + Expr::Call { + fun: ident_expr("f"), + args: vec![*ident_expr("x"), Expr::Literal(Literal::Int(1))] + } + ) + } + + #[test] + fn call_funcref() { + let res = test_parse!(expr, "(let x = 1 in x) 2"); + assert_eq!( + res, + Expr::Call { + fun: Box::new(Expr::Let { + bindings: vec![Binding { + pat: Pattern::Id(Ident::try_from("x").unwrap()), + type_: None, + body: Expr::Literal(Literal::Int(1)) + }], + body: ident_expr("x") + }), + args: vec![Expr::Literal(Literal::Int(2))] + } + ) + } + + #[test] + fn anon_function() { + let res = test_parse!(expr, "let id = fn x = x in id 1"); + assert_eq!( + res, + Expr::Let { + bindings: vec![Binding { + pat: Pattern::Id(Ident::try_from("id").unwrap()), + type_: None, + body: Expr::Fun(Box::new(Fun { + args: vec![Arg::try_from("x").unwrap()], + body: *ident_expr("x") + })) + }], + body: Box::new(Expr::Call { + fun: ident_expr("id"), + args: vec![Expr::Literal(Literal::Int(1))], + }) + } + ); + } + + #[test] + fn tuple_binding() { + let res = test_parse!(expr, "let (x, y) = (1, 2) in x"); + assert_eq!( + res, + Expr::Let { + bindings: vec![Binding { + pat: Pattern::Tuple(vec![ + Pattern::Id(Ident::from_str_unchecked("x")), + Pattern::Id(Ident::from_str_unchecked("y")) + ]), + body: Expr::Tuple(vec![ + Expr::Literal(Literal::Int(1)), + Expr::Literal(Literal::Int(2)) + ]), + type_: None + }], + body: Box::new(Expr::Ident(Ident::from_str_unchecked("x"))) + } + ) + } + + mod ascriptions { + use super::*; + + #[test] + fn bare_ascription() { + let res = test_parse!(expr, "1: float"); + assert_eq!( + res, + Expr::Ascription { + expr: Box::new(Expr::Literal(Literal::Int(1))), + type_: Type::Float + } + ) + } + + #[test] + fn fn_body_ascription() { + let res = test_parse!(expr, "let const_1 = fn x = 1: int in const_1 2"); + assert_eq!( + res, + Expr::Let { + bindings: vec![Binding { + pat: Pattern::Id(Ident::try_from("const_1").unwrap()), + type_: None, + body: Expr::Fun(Box::new(Fun { + args: vec![Arg::try_from("x").unwrap()], + body: Expr::Ascription { + expr: Box::new(Expr::Literal(Literal::Int(1))), + type_: Type::Int, + } + })) + }], + body: Box::new(Expr::Call { + fun: ident_expr("const_1"), + args: vec![Expr::Literal(Literal::Int(2))] + }) + } + ) + } + + #[test] + fn let_binding_ascripted() { + let res = test_parse!(expr, "let x: int = 1 in x"); + assert_eq!( + res, + Expr::Let { + bindings: vec![Binding { + pat: Pattern::Id(Ident::try_from("x").unwrap()), + type_: Some(Type::Int), + body: Expr::Literal(Literal::Int(1)) + }], + body: ident_expr("x") + } + ) + } + } +} diff --git a/users/grfn/achilles/src/parser/macros.rs b/users/grfn/achilles/src/parser/macros.rs new file mode 100644 index 000000000000..406e5c0e699e --- /dev/null +++ b/users/grfn/achilles/src/parser/macros.rs @@ -0,0 +1,16 @@ +#[cfg(test)] +#[macro_use] +macro_rules! test_parse { + ($parser: ident, $src: expr) => {{ + let res = $parser($src); + nom_trace::print_trace!(); + let (rem, res) = res.unwrap(); + assert!( + rem.is_empty(), + "non-empty remainder: \"{}\", parsed: {:?}", + rem, + res + ); + res + }}; +} diff --git a/users/grfn/achilles/src/parser/mod.rs b/users/grfn/achilles/src/parser/mod.rs new file mode 100644 index 000000000000..e088cbca10a5 --- /dev/null +++ b/users/grfn/achilles/src/parser/mod.rs @@ -0,0 +1,240 @@ +use nom::character::complete::{multispace0, multispace1}; +use nom::error::{ErrorKind, ParseError}; +use nom::{alt, char, complete, do_parse, eof, many0, named, separated_list0, tag, terminated}; + +#[macro_use] +pub(crate) mod macros; +mod expr; +mod type_; +mod util; + +use crate::ast::{Arg, Decl, Fun, Ident}; +pub use expr::expr; +use type_::function_type; +pub use type_::type_; + +pub type Error = nom::Err<nom::error::Error<String>>; + +pub(crate) fn is_reserved(s: &str) -> bool { + matches!( + s, + "if" | "then" + | "else" + | "let" + | "in" + | "fn" + | "ty" + | "int" + | "float" + | "bool" + | "true" + | "false" + | "cstring" + ) +} + +pub(crate) fn ident<'a, E>(i: &'a str) -> nom::IResult<&'a str, Ident, E> +where + E: ParseError<&'a str>, +{ + let mut chars = i.chars(); + if let Some(f) = chars.next() { + if f.is_alphabetic() || f == '_' { + let mut idx = 1; + for c in chars { + if !(c.is_alphanumeric() || c == '_') { + break; + } + idx += 1; + } + let id = &i[..idx]; + if is_reserved(id) { + Err(nom::Err::Error(E::from_error_kind(i, ErrorKind::Satisfy))) + } else { + Ok((&i[idx..], Ident::from_str_unchecked(id))) + } + } else { + Err(nom::Err::Error(E::from_error_kind(i, ErrorKind::Satisfy))) + } + } else { + Err(nom::Err::Error(E::from_error_kind(i, ErrorKind::Eof))) + } +} + +named!(ascripted_arg(&str) -> Arg, do_parse!( + complete!(char!('(')) >> + multispace0 >> + ident: ident >> + multispace0 >> + complete!(char!(':')) >> + multispace0 >> + type_: type_ >> + multispace0 >> + complete!(char!(')')) >> + (Arg { + ident, + type_: Some(type_) + }) +)); + +named!(arg(&str) -> Arg, alt!( + ident => { |ident| Arg {ident, type_: None}} | + ascripted_arg +)); + +named!(extern_decl(&str) -> Decl, do_parse!( + complete!(tag!("extern")) + >> multispace1 + >> name: ident + >> multispace0 + >> char!(':') + >> multispace0 + >> type_: function_type + >> multispace0 + >> (Decl::Extern { + name, + type_ + }) +)); + +named!(fun_decl(&str) -> Decl, do_parse!( + complete!(tag!("fn")) + >> multispace1 + >> name: ident + >> multispace1 + >> args: separated_list0!(multispace1, arg) + >> multispace0 + >> char!('=') + >> multispace0 + >> body: expr + >> (Decl::Fun { + name, + body: Fun { + args, + body + } + }) +)); + +named!(ascription_decl(&str) -> Decl, do_parse!( + complete!(tag!("ty")) + >> multispace1 + >> name: ident + >> multispace0 + >> complete!(char!(':')) + >> multispace0 + >> type_: type_ + >> multispace0 + >> (Decl::Ascription { + name, + type_ + }) +)); + +named!(pub decl(&str) -> Decl, alt!( + ascription_decl | + fun_decl | + extern_decl +)); + +named!(pub toplevel(&str) -> Vec<Decl>, do_parse!( + decls: many0!(decl) + >> multispace0 + >> eof!() + >> (decls))); + +#[cfg(test)] +mod tests { + use std::convert::TryInto; + + use crate::ast::{BinaryOperator, Expr, FunctionType, Literal, Type}; + + use super::*; + use expr::tests::ident_expr; + + #[test] + fn fn_decl() { + let res = test_parse!(decl, "fn id x = x"); + assert_eq!( + res, + Decl::Fun { + name: "id".try_into().unwrap(), + body: Fun { + args: vec!["x".try_into().unwrap()], + body: *ident_expr("x"), + } + } + ) + } + + #[test] + fn ascripted_fn_args() { + test_parse!(ascripted_arg, "(x : int)"); + let res = test_parse!(decl, "fn plus1 (x : int) = x + 1"); + assert_eq!( + res, + Decl::Fun { + name: "plus1".try_into().unwrap(), + body: Fun { + args: vec![Arg { + ident: "x".try_into().unwrap(), + type_: Some(Type::Int), + }], + body: Expr::BinaryOp { + lhs: ident_expr("x"), + op: BinaryOperator::Add, + rhs: Box::new(Expr::Literal(Literal::Int(1))), + } + } + } + ); + } + + #[test] + fn multiple_decls() { + let res = test_parse!( + toplevel, + "fn id x = x + fn plus x y = x + y + fn main = plus (id 2) 7" + ); + assert_eq!(res.len(), 3); + let res = test_parse!( + toplevel, + "fn id x = x\nfn plus x y = x + y\nfn main = plus (id 2) 7\n" + ); + assert_eq!(res.len(), 3); + } + + #[test] + fn top_level_ascription() { + let res = test_parse!(toplevel, "ty id : fn a -> a"); + assert_eq!( + res, + vec![Decl::Ascription { + name: "id".try_into().unwrap(), + type_: Type::Function(FunctionType { + args: vec![Type::Var("a".try_into().unwrap())], + ret: Box::new(Type::Var("a".try_into().unwrap())) + }) + }] + ) + } + + #[test] + fn return_unit() { + assert_eq!( + test_parse!(decl, "fn g _ = ()"), + Decl::Fun { + name: "g".try_into().unwrap(), + body: Fun { + args: vec![Arg { + ident: "_".try_into().unwrap(), + type_: None, + }], + body: Expr::Literal(Literal::Unit), + }, + } + ) + } +} diff --git a/users/grfn/achilles/src/parser/type_.rs b/users/grfn/achilles/src/parser/type_.rs new file mode 100644 index 000000000000..b80f0e0860a1 --- /dev/null +++ b/users/grfn/achilles/src/parser/type_.rs @@ -0,0 +1,152 @@ +use nom::character::complete::{multispace0, multispace1}; +use nom::{alt, delimited, do_parse, map, named, opt, separated_list0, tag, terminated, tuple}; + +use super::ident; +use super::util::comma; +use crate::ast::{FunctionType, Type}; + +named!(pub function_type(&str) -> FunctionType, do_parse!( + tag!("fn") + >> multispace1 + >> args: map!(opt!(terminated!(separated_list0!( + comma, + type_ + ), multispace1)), |args| args.unwrap_or_default()) + >> tag!("->") + >> multispace1 + >> ret: type_ + >> (FunctionType { + args, + ret: Box::new(ret) + }) +)); + +named!(tuple_type(&str) -> Type, do_parse!( + tag!("(") + >> multispace0 + >> fst: type_ + >> comma + >> rest: separated_list0!( + comma, + type_ + ) + >> multispace0 + >> tag!(")") + >> ({ + let mut members = Vec::with_capacity(rest.len() + 1); + members.push(fst); + members.append(&mut rest.clone()); + Type::Tuple(members) + }) +)); + +named!(pub type_(&str) -> Type, alt!( + tag!("int") => { |_| Type::Int } | + tag!("float") => { |_| Type::Float } | + tag!("bool") => { |_| Type::Bool } | + tag!("cstring") => { |_| Type::CString } | + tag!("()") => { |_| Type::Unit } | + tuple_type | + function_type => { |ft| Type::Function(ft) }| + ident => { |id| Type::Var(id) } | + delimited!( + tuple!(tag!("("), multispace0), + type_, + tuple!(tag!(")"), multispace0) + ) +)); + +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use super::*; + use crate::ast::Ident; + + #[test] + fn simple_types() { + assert_eq!(test_parse!(type_, "int"), Type::Int); + assert_eq!(test_parse!(type_, "float"), Type::Float); + assert_eq!(test_parse!(type_, "bool"), Type::Bool); + assert_eq!(test_parse!(type_, "cstring"), Type::CString); + assert_eq!(test_parse!(type_, "()"), Type::Unit); + } + + #[test] + fn no_arg_fn_type() { + assert_eq!( + test_parse!(type_, "fn -> int"), + Type::Function(FunctionType { + args: vec![], + ret: Box::new(Type::Int) + }) + ); + } + + #[test] + fn fn_type_with_args() { + assert_eq!( + test_parse!(type_, "fn int, bool -> int"), + Type::Function(FunctionType { + args: vec![Type::Int, Type::Bool], + ret: Box::new(Type::Int) + }) + ); + } + + #[test] + fn fn_taking_fn() { + assert_eq!( + test_parse!(type_, "fn fn int, bool -> bool, float -> float"), + Type::Function(FunctionType { + args: vec![ + Type::Function(FunctionType { + args: vec![Type::Int, Type::Bool], + ret: Box::new(Type::Bool) + }), + Type::Float + ], + ret: Box::new(Type::Float) + }) + ) + } + + #[test] + fn parenthesized() { + assert_eq!( + test_parse!(type_, "fn (fn int, bool -> bool), float -> float"), + Type::Function(FunctionType { + args: vec![ + Type::Function(FunctionType { + args: vec![Type::Int, Type::Bool], + ret: Box::new(Type::Bool) + }), + Type::Float + ], + ret: Box::new(Type::Float) + }) + ) + } + + #[test] + fn tuple() { + assert_eq!( + test_parse!(type_, "(int, int)"), + Type::Tuple(vec![Type::Int, Type::Int]) + ) + } + + #[test] + fn type_vars() { + assert_eq!( + test_parse!(type_, "fn x, y -> x"), + Type::Function(FunctionType { + args: vec![ + Type::Var(Ident::try_from("x").unwrap()), + Type::Var(Ident::try_from("y").unwrap()), + ], + ret: Box::new(Type::Var(Ident::try_from("x").unwrap())), + }) + ) + } +} diff --git a/users/grfn/achilles/src/parser/util.rs b/users/grfn/achilles/src/parser/util.rs new file mode 100644 index 000000000000..bb53fb7fff50 --- /dev/null +++ b/users/grfn/achilles/src/parser/util.rs @@ -0,0 +1,8 @@ +use nom::character::complete::multispace0; +use nom::{complete, map, named, tag, tuple}; + +named!(pub(crate) comma(&str) -> (), map!(tuple!( + multispace0, + complete!(tag!(",")), + multispace0 +) ,|_| ())); diff --git a/users/grfn/achilles/src/passes/hir/mod.rs b/users/grfn/achilles/src/passes/hir/mod.rs new file mode 100644 index 000000000000..872c449eb020 --- /dev/null +++ b/users/grfn/achilles/src/passes/hir/mod.rs @@ -0,0 +1,211 @@ +use std::collections::HashMap; + +use crate::ast::hir::{Binding, Decl, Expr, Pattern}; +use crate::ast::{BinaryOperator, Ident, Literal, UnaryOperator}; + +pub(crate) mod monomorphize; +pub(crate) mod strip_positive_units; + +pub(crate) trait Visitor<'a, 'ast, T: 'ast>: Sized + 'a { + type Error; + + fn visit_type(&mut self, _type: &mut T) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_ident(&mut self, _ident: &mut Ident<'ast>) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_literal(&mut self, _literal: &mut Literal<'ast>) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_unary_operator(&mut self, _op: &mut UnaryOperator) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_binary_operator(&mut self, _op: &mut BinaryOperator) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_pattern(&mut self, _pat: &mut Pattern<'ast, T>) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_binding(&mut self, binding: &mut Binding<'ast, T>) -> Result<(), Self::Error> { + self.visit_pattern(&mut binding.pat)?; + self.visit_expr(&mut binding.body)?; + Ok(()) + } + + fn post_visit_call( + &mut self, + _fun: &mut Expr<'ast, T>, + _type_args: &mut HashMap<Ident<'ast>, T>, + _args: &mut Vec<Expr<'ast, T>>, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn pre_visit_call( + &mut self, + _fun: &mut Expr<'ast, T>, + _type_args: &mut HashMap<Ident<'ast>, T>, + _args: &mut Vec<Expr<'ast, T>>, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_tuple(&mut self, members: &mut Vec<Expr<'ast, T>>) -> Result<(), Self::Error> { + for expr in members { + self.visit_expr(expr)?; + } + Ok(()) + } + + fn pre_visit_expr(&mut self, _expr: &mut Expr<'ast, T>) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_expr(&mut self, expr: &mut Expr<'ast, T>) -> Result<(), Self::Error> { + self.pre_visit_expr(expr)?; + match expr { + Expr::Ident(id, t) => { + self.visit_ident(id)?; + self.visit_type(t)?; + } + Expr::Literal(lit, t) => { + self.visit_literal(lit)?; + self.visit_type(t)?; + } + Expr::UnaryOp { op, rhs, type_ } => { + self.visit_unary_operator(op)?; + self.visit_expr(rhs)?; + self.visit_type(type_)?; + } + Expr::BinaryOp { + lhs, + op, + rhs, + type_, + } => { + self.visit_expr(lhs)?; + self.visit_binary_operator(op)?; + self.visit_expr(rhs)?; + self.visit_type(type_)?; + } + Expr::Let { + bindings, + body, + type_, + } => { + for binding in bindings.iter_mut() { + self.visit_binding(binding)?; + } + self.visit_expr(body)?; + self.visit_type(type_)?; + } + Expr::If { + condition, + then, + else_, + type_, + } => { + self.visit_expr(condition)?; + self.visit_expr(then)?; + self.visit_expr(else_)?; + self.visit_type(type_)?; + } + Expr::Fun { + args, + body, + type_args, + type_, + } => { + for (ident, t) in args { + self.visit_ident(ident)?; + self.visit_type(t)?; + } + for ta in type_args { + self.visit_ident(ta)?; + } + self.visit_expr(body)?; + self.visit_type(type_)?; + } + Expr::Call { + fun, + args, + type_args, + type_, + } => { + self.pre_visit_call(fun, type_args, args)?; + self.visit_expr(fun)?; + for arg in args.iter_mut() { + self.visit_expr(arg)?; + } + self.visit_type(type_)?; + self.post_visit_call(fun, type_args, args)?; + } + Expr::Tuple(tup, type_) => { + self.visit_tuple(tup)?; + self.visit_type(type_)?; + } + } + + Ok(()) + } + + fn post_visit_decl(&mut self, _decl: &'a Decl<'ast, T>) -> Result<(), Self::Error> { + Ok(()) + } + + fn post_visit_fun_decl( + &mut self, + _name: &mut Ident<'ast>, + _type_args: &mut Vec<Ident>, + _args: &mut Vec<(Ident, T)>, + _body: &mut Box<Expr<T>>, + _type_: &mut T, + ) -> Result<(), Self::Error> { + Ok(()) + } + + fn visit_decl(&mut self, decl: &'a mut Decl<'ast, T>) -> Result<(), Self::Error> { + match decl { + Decl::Fun { + name, + type_args, + args, + body, + type_, + } => { + self.visit_ident(name)?; + for type_arg in type_args.iter_mut() { + self.visit_ident(type_arg)?; + } + for (arg, t) in args.iter_mut() { + self.visit_ident(arg)?; + self.visit_type(t)?; + } + self.visit_expr(body)?; + self.visit_type(type_)?; + self.post_visit_fun_decl(name, type_args, args, body, type_)?; + } + Decl::Extern { + name, + arg_types, + ret_type, + } => { + self.visit_ident(name)?; + for arg_t in arg_types { + self.visit_type(arg_t)?; + } + self.visit_type(ret_type)?; + } + } + + self.post_visit_decl(decl)?; + Ok(()) + } +} diff --git a/users/grfn/achilles/src/passes/hir/monomorphize.rs b/users/grfn/achilles/src/passes/hir/monomorphize.rs new file mode 100644 index 000000000000..251a988f4f6f --- /dev/null +++ b/users/grfn/achilles/src/passes/hir/monomorphize.rs @@ -0,0 +1,139 @@ +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::convert::TryInto; +use std::mem; + +use void::{ResultVoidExt, Void}; + +use crate::ast::hir::{Decl, Expr}; +use crate::ast::{self, Ident}; + +use super::Visitor; + +#[derive(Default)] +pub(crate) struct Monomorphize<'a, 'ast> { + decls: HashMap<&'a Ident<'ast>, &'a Decl<'ast, ast::Type<'ast>>>, + extra_decls: Vec<Decl<'ast, ast::Type<'ast>>>, + remove_decls: HashSet<Ident<'ast>>, +} + +impl<'a, 'ast> Monomorphize<'a, 'ast> { + pub(crate) fn new() -> Self { + Default::default() + } +} + +impl<'a, 'ast> Visitor<'a, 'ast, ast::Type<'ast>> for Monomorphize<'a, 'ast> { + type Error = Void; + + fn post_visit_call( + &mut self, + fun: &mut Expr<'ast, ast::Type<'ast>>, + type_args: &mut HashMap<Ident<'ast>, ast::Type<'ast>>, + args: &mut Vec<Expr<'ast, ast::Type<'ast>>>, + ) -> Result<(), Self::Error> { + let new_fun = match fun { + Expr::Ident(id, _) => { + let decl: Decl<_> = (**self.decls.get(id).unwrap()).clone(); + let name = RefCell::new(id.to_string()); + let type_args = mem::take(type_args); + let mut monomorphized = decl + .traverse_type(|ty| -> Result<_, Void> { + Ok(ty.clone().traverse_type_vars(|v| { + let concrete = type_args.get(&v).unwrap(); + name.borrow_mut().push_str(&concrete.to_string()); + concrete.clone() + })) + }) + .void_unwrap(); + let name: Ident = name.into_inner().try_into().unwrap(); + if name != *id { + self.remove_decls.insert(id.clone()); + monomorphized.set_name(name.clone()); + let type_ = monomorphized.type_().unwrap().clone(); + self.extra_decls.push(monomorphized); + Some(Expr::Ident(name, type_)) + } else { + None + } + } + _ => todo!(), + }; + if let Some(new_fun) = new_fun { + *fun = new_fun; + } + Ok(()) + } + + fn post_visit_decl( + &mut self, + decl: &'a Decl<'ast, ast::Type<'ast>>, + ) -> Result<(), Self::Error> { + self.decls.insert(decl.name(), decl); + Ok(()) + } +} + +pub(crate) fn run_toplevel<'a>(toplevel: &mut Vec<Decl<'a, ast::Type<'a>>>) { + let mut pass = Monomorphize::new(); + for decl in toplevel.iter_mut() { + pass.visit_decl(decl).void_unwrap(); + } + let remove_decls = mem::take(&mut pass.remove_decls); + let mut extra_decls = mem::take(&mut pass.extra_decls); + toplevel.retain(|decl| !remove_decls.contains(decl.name())); + extra_decls.append(toplevel); + *toplevel = extra_decls; +} + +#[cfg(test)] +mod tests { + use std::convert::TryFrom; + + use super::*; + use crate::parser::toplevel; + use crate::tc::typecheck_toplevel; + + #[test] + fn call_id_decl() { + let (_, program) = toplevel( + "ty id : fn a -> a + fn id x = x + + ty main : fn -> int + fn main = id 0", + ) + .unwrap(); + let mut program = typecheck_toplevel(program).unwrap(); + run_toplevel(&mut program); + + let find_decl = |ident: &str| { + program.iter().find(|decl| { + matches!(decl, Decl::Fun {name, ..} if name == &Ident::try_from(ident).unwrap()) + }).unwrap() + }; + + let main = find_decl("main"); + let body = match main { + Decl::Fun { body, .. } => body, + _ => unreachable!(), + }; + + let expected_type = ast::Type::Function(ast::FunctionType { + args: vec![ast::Type::Int], + ret: Box::new(ast::Type::Int), + }); + + match &**body { + Expr::Call { fun, .. } => { + let fun = match &**fun { + Expr::Ident(fun, _) => fun, + _ => unreachable!(), + }; + let called_decl = find_decl(fun.into()); + assert_eq!(called_decl.type_().unwrap(), &expected_type); + } + _ => unreachable!(), + } + } +} diff --git a/users/grfn/achilles/src/passes/hir/strip_positive_units.rs b/users/grfn/achilles/src/passes/hir/strip_positive_units.rs new file mode 100644 index 000000000000..85ee1cce4859 --- /dev/null +++ b/users/grfn/achilles/src/passes/hir/strip_positive_units.rs @@ -0,0 +1,191 @@ +use std::collections::HashMap; +use std::mem; + +use ast::hir::{Binding, Pattern}; +use ast::Literal; +use void::{ResultVoidExt, Void}; + +use crate::ast::hir::{Decl, Expr}; +use crate::ast::{self, Ident}; + +use super::Visitor; + +/// Strip all values with a unit type in positive (non-return) position +pub(crate) struct StripPositiveUnits {} + +impl<'a, 'ast> Visitor<'a, 'ast, ast::Type<'ast>> for StripPositiveUnits { + type Error = Void; + + fn pre_visit_expr( + &mut self, + expr: &mut Expr<'ast, ast::Type<'ast>>, + ) -> Result<(), Self::Error> { + let mut extracted = vec![]; + if let Expr::Call { args, .. } = expr { + // TODO(grfn): replace with drain_filter once it's stabilized + let mut i = 0; + while i != args.len() { + if args[i].type_() == &ast::Type::Unit { + let expr = args.remove(i); + if !matches!(expr, Expr::Literal(Literal::Unit, _)) { + extracted.push(expr) + }; + } else { + i += 1 + } + } + } + + if !extracted.is_empty() { + let body = mem::replace(expr, Expr::Literal(Literal::Unit, ast::Type::Unit)); + *expr = Expr::Let { + bindings: extracted + .into_iter() + .map(|expr| Binding { + pat: Pattern::Id( + Ident::from_str_unchecked("___discarded"), + expr.type_().clone(), + ), + body: expr, + }) + .collect(), + type_: body.type_().clone(), + body: Box::new(body), + }; + } + + Ok(()) + } + + fn post_visit_call( + &mut self, + _fun: &mut Expr<'ast, ast::Type<'ast>>, + _type_args: &mut HashMap<Ident<'ast>, ast::Type<'ast>>, + args: &mut Vec<Expr<'ast, ast::Type<'ast>>>, + ) -> Result<(), Self::Error> { + args.retain(|arg| arg.type_() != &ast::Type::Unit); + Ok(()) + } + + fn visit_type(&mut self, type_: &mut ast::Type<'ast>) -> Result<(), Self::Error> { + if let ast::Type::Function(ft) = type_ { + ft.args.retain(|a| a != &ast::Type::Unit); + } + Ok(()) + } + + fn post_visit_fun_decl( + &mut self, + _name: &mut Ident<'ast>, + _type_args: &mut Vec<Ident>, + args: &mut Vec<(Ident, ast::Type<'ast>)>, + _body: &mut Box<Expr<ast::Type<'ast>>>, + _type_: &mut ast::Type<'ast>, + ) -> Result<(), Self::Error> { + args.retain(|(_, ty)| ty != &ast::Type::Unit); + Ok(()) + } +} + +pub(crate) fn run_toplevel<'a>(toplevel: &mut Vec<Decl<'a, ast::Type<'a>>>) { + let mut pass = StripPositiveUnits {}; + for decl in toplevel.iter_mut() { + pass.visit_decl(decl).void_unwrap(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parser::toplevel; + use crate::tc::typecheck_toplevel; + use pretty_assertions::assert_eq; + + #[test] + fn unit_only_arg() { + let (_, program) = toplevel( + "ty f : fn () -> int + fn f _ = 1 + + ty main : fn -> int + fn main = f ()", + ) + .unwrap(); + + let (_, expected) = toplevel( + "ty f : fn -> int + fn f = 1 + + ty main : fn -> int + fn main = f()", + ) + .unwrap(); + let expected = typecheck_toplevel(expected).unwrap(); + + let mut program = typecheck_toplevel(program).unwrap(); + run_toplevel(&mut program); + + assert_eq!(program, expected); + } + + #[test] + fn unit_and_other_arg() { + let (_, program) = toplevel( + "ty f : fn (), int -> int + fn f _ x = x + + ty main : fn -> int + fn main = f () 1", + ) + .unwrap(); + + let (_, expected) = toplevel( + "ty f : fn int -> int + fn f x = x + + ty main : fn -> int + fn main = f 1", + ) + .unwrap(); + let expected = typecheck_toplevel(expected).unwrap(); + + let mut program = typecheck_toplevel(program).unwrap(); + run_toplevel(&mut program); + + assert_eq!(program, expected); + } + + #[test] + fn unit_expr_and_other_arg() { + let (_, program) = toplevel( + "ty f : fn (), int -> int + fn f _ x = x + + ty g : fn int -> () + fn g _ = () + + ty main : fn -> int + fn main = f (g 2) 1", + ) + .unwrap(); + + let (_, expected) = toplevel( + "ty f : fn int -> int + fn f x = x + + ty g : fn int -> () + fn g _ = () + + ty main : fn -> int + fn main = let ___discarded = g 2 in f 1", + ) + .unwrap(); + assert_eq!(expected.len(), 6); + let expected = typecheck_toplevel(expected).unwrap(); + + let mut program = typecheck_toplevel(program).unwrap(); + run_toplevel(&mut program); + + assert_eq!(program, expected); + } +} diff --git a/users/grfn/achilles/src/passes/mod.rs b/users/grfn/achilles/src/passes/mod.rs new file mode 100644 index 000000000000..306869bef1d5 --- /dev/null +++ b/users/grfn/achilles/src/passes/mod.rs @@ -0,0 +1 @@ +pub(crate) mod hir; diff --git a/users/grfn/achilles/src/tc/mod.rs b/users/grfn/achilles/src/tc/mod.rs new file mode 100644 index 000000000000..5825bab1fbe9 --- /dev/null +++ b/users/grfn/achilles/src/tc/mod.rs @@ -0,0 +1,808 @@ +use bimap::BiMap; +use derive_more::From; +use itertools::Itertools; +use std::cell::RefCell; +use std::collections::HashMap; +use std::convert::{TryFrom, TryInto}; +use std::fmt::{self, Display}; +use std::{mem, result}; +use thiserror::Error; + +use crate::ast::{self, hir, Arg, BinaryOperator, Ident, Literal, Pattern}; +use crate::common::env::Env; +use crate::common::{Namer, NamerOf}; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Undefined variable {0}")] + UndefinedVariable(Ident<'static>), + + #[error("Mismatched types: expected {expected}, but got {actual}")] + TypeMismatch { expected: Type, actual: Type }, + + #[error("Mismatched types, expected numeric type, but got {0}")] + NonNumeric(Type), + + #[error("Ambiguous type {0}")] + AmbiguousType(TyVar), +} + +pub type Result<T> = result::Result<T, Error>; + +#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)] +pub struct TyVar(u64); + +impl Display for TyVar { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "t{}", self.0) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Hash)] +pub struct NullaryType(String); + +impl Display for NullaryType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum PrimType { + Int, + Float, + Bool, + CString, +} + +impl<'a> From<PrimType> for ast::Type<'a> { + fn from(pr: PrimType) -> Self { + match pr { + PrimType::Int => ast::Type::Int, + PrimType::Float => ast::Type::Float, + PrimType::Bool => ast::Type::Bool, + PrimType::CString => ast::Type::CString, + } + } +} + +impl Display for PrimType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PrimType::Int => f.write_str("int"), + PrimType::Float => f.write_str("float"), + PrimType::Bool => f.write_str("bool"), + PrimType::CString => f.write_str("cstring"), + } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, From)] +pub enum Type { + #[from(ignore)] + Univ(TyVar), + #[from(ignore)] + Exist(TyVar), + Nullary(NullaryType), + Prim(PrimType), + Tuple(Vec<Type>), + Unit, + Fun { + args: Vec<Type>, + ret: Box<Type>, + }, +} + +impl<'a> TryFrom<Type> for ast::Type<'a> { + type Error = Type; + + fn try_from(value: Type) -> result::Result<Self, Self::Error> { + match value { + Type::Unit => Ok(ast::Type::Unit), + Type::Univ(_) => todo!(), + Type::Exist(_) => Err(value), + Type::Nullary(_) => todo!(), + Type::Prim(p) => Ok(p.into()), + Type::Tuple(members) => Ok(ast::Type::Tuple( + members.into_iter().map(|ty| ty.try_into()).try_collect()?, + )), + Type::Fun { ref args, ref ret } => Ok(ast::Type::Function(ast::FunctionType { + args: args + .clone() + .into_iter() + .map(Self::try_from) + .try_collect() + .map_err(|_| value.clone())?, + ret: Box::new((*ret.clone()).try_into().map_err(|_| value.clone())?), + })), + } + } +} + +const INT: Type = Type::Prim(PrimType::Int); +const FLOAT: Type = Type::Prim(PrimType::Float); +const BOOL: Type = Type::Prim(PrimType::Bool); +const CSTRING: Type = Type::Prim(PrimType::CString); + +impl Display for Type { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Type::Nullary(nt) => nt.fmt(f), + Type::Prim(p) => p.fmt(f), + Type::Univ(TyVar(n)) => write!(f, "∀{}", n), + Type::Exist(TyVar(n)) => write!(f, "∃{}", n), + Type::Fun { args, ret } => write!(f, "fn {} -> {}", args.iter().join(", "), ret), + Type::Tuple(members) => write!(f, "({})", members.iter().join(", ")), + Type::Unit => write!(f, "()"), + } + } +} + +struct Typechecker<'ast> { + ty_var_namer: NamerOf<TyVar>, + ctx: HashMap<TyVar, Type>, + env: Env<Ident<'ast>, Type>, + + /// AST type var -> type + instantiations: Env<Ident<'ast>, Type>, + + /// AST type-var -> universal TyVar + type_vars: RefCell<(BiMap<Ident<'ast>, TyVar>, NamerOf<Ident<'static>>)>, +} + +impl<'ast> Typechecker<'ast> { + fn new() -> Self { + Self { + ty_var_namer: Namer::new(TyVar).boxed(), + type_vars: RefCell::new(( + Default::default(), + Namer::alphabetic().map(|n| Ident::try_from(n).unwrap()), + )), + ctx: Default::default(), + env: Default::default(), + instantiations: Default::default(), + } + } + + fn bind_pattern( + &mut self, + pat: Pattern<'ast>, + type_: Type, + ) -> Result<hir::Pattern<'ast, Type>> { + match pat { + Pattern::Id(ident) => { + self.env.set(ident.clone(), type_.clone()); + Ok(hir::Pattern::Id(ident, type_)) + } + Pattern::Tuple(members) => { + let mut tys = Vec::with_capacity(members.len()); + let mut hir_members = Vec::with_capacity(members.len()); + for pat in members { + let ty = self.fresh_ex(); + hir_members.push(self.bind_pattern(pat, ty.clone())?); + tys.push(ty); + } + let tuple_type = Type::Tuple(tys); + self.unify(&tuple_type, &type_)?; + Ok(hir::Pattern::Tuple(hir_members)) + } + } + } + + pub(crate) fn tc_expr(&mut self, expr: ast::Expr<'ast>) -> Result<hir::Expr<'ast, Type>> { + match expr { + ast::Expr::Ident(ident) => { + let type_ = self + .env + .resolve(&ident) + .ok_or_else(|| Error::UndefinedVariable(ident.to_owned()))? + .clone(); + Ok(hir::Expr::Ident(ident, type_)) + } + ast::Expr::Literal(lit) => { + let type_ = match lit { + Literal::Int(_) => Type::Prim(PrimType::Int), + Literal::Bool(_) => Type::Prim(PrimType::Bool), + Literal::String(_) => Type::Prim(PrimType::CString), + Literal::Unit => Type::Unit, + }; + Ok(hir::Expr::Literal(lit.to_owned(), type_)) + } + ast::Expr::Tuple(members) => { + let members = members + .into_iter() + .map(|expr| self.tc_expr(expr)) + .collect::<Result<Vec<_>>>()?; + let type_ = Type::Tuple(members.iter().map(|expr| expr.type_().clone()).collect()); + Ok(hir::Expr::Tuple(members, type_)) + } + ast::Expr::UnaryOp { op, rhs } => todo!(), + ast::Expr::BinaryOp { lhs, op, rhs } => { + let lhs = self.tc_expr(*lhs)?; + let rhs = self.tc_expr(*rhs)?; + let type_ = match op { + BinaryOperator::Equ | BinaryOperator::Neq => { + self.unify(lhs.type_(), rhs.type_())?; + Type::Prim(PrimType::Bool) + } + BinaryOperator::Add | BinaryOperator::Sub | BinaryOperator::Mul => { + let ty = self.unify(lhs.type_(), rhs.type_())?; + // if !matches!(ty, Type::Int | Type::Float) { + // return Err(Error::NonNumeric(ty)); + // } + ty + } + BinaryOperator::Div => todo!(), + BinaryOperator::Pow => todo!(), + }; + Ok(hir::Expr::BinaryOp { + lhs: Box::new(lhs), + op, + rhs: Box::new(rhs), + type_, + }) + } + ast::Expr::Let { bindings, body } => { + self.env.push(); + let bindings = bindings + .into_iter() + .map( + |ast::Binding { pat, type_, body }| -> Result<hir::Binding<Type>> { + let body = self.tc_expr(body)?; + if let Some(type_) = type_ { + let type_ = self.type_from_ast_type(type_); + self.unify(body.type_(), &type_)?; + } + let pat = self.bind_pattern(pat, body.type_().clone())?; + Ok(hir::Binding { pat, body }) + }, + ) + .collect::<Result<Vec<hir::Binding<Type>>>>()?; + let body = self.tc_expr(*body)?; + self.env.pop(); + Ok(hir::Expr::Let { + bindings, + type_: body.type_().clone(), + body: Box::new(body), + }) + } + ast::Expr::If { + condition, + then, + else_, + } => { + let condition = self.tc_expr(*condition)?; + self.unify(&Type::Prim(PrimType::Bool), condition.type_())?; + let then = self.tc_expr(*then)?; + let else_ = self.tc_expr(*else_)?; + let type_ = self.unify(then.type_(), else_.type_())?; + Ok(hir::Expr::If { + condition: Box::new(condition), + then: Box::new(then), + else_: Box::new(else_), + type_, + }) + } + ast::Expr::Fun(f) => { + let ast::Fun { args, body } = *f; + self.env.push(); + let args: Vec<_> = args + .into_iter() + .map(|Arg { ident, type_ }| { + let ty = match type_ { + Some(t) => self.type_from_ast_type(t), + None => self.fresh_ex(), + }; + self.env.set(ident.clone(), ty.clone()); + (ident, ty) + }) + .collect(); + let body = self.tc_expr(body)?; + self.env.pop(); + Ok(hir::Expr::Fun { + type_: Type::Fun { + args: args.iter().map(|(_, ty)| ty.clone()).collect(), + ret: Box::new(body.type_().clone()), + }, + type_args: vec![], // TODO fill in once we do let generalization + args, + body: Box::new(body), + }) + } + ast::Expr::Call { fun, args } => { + let ret_ty = self.fresh_ex(); + let arg_tys = args.iter().map(|_| self.fresh_ex()).collect::<Vec<_>>(); + let ft = Type::Fun { + args: arg_tys.clone(), + ret: Box::new(ret_ty.clone()), + }; + let fun = self.tc_expr(*fun)?; + self.instantiations.push(); + self.unify(&ft, fun.type_())?; + let args = args + .into_iter() + .zip(arg_tys) + .map(|(arg, ty)| { + let arg = self.tc_expr(arg)?; + self.unify(&ty, arg.type_())?; + Ok(arg) + }) + .try_collect()?; + let type_args = self.commit_instantiations(); + Ok(hir::Expr::Call { + fun: Box::new(fun), + type_args, + args, + type_: ret_ty, + }) + } + ast::Expr::Ascription { expr, type_ } => { + let expr = self.tc_expr(*expr)?; + let type_ = self.type_from_ast_type(type_); + self.unify(expr.type_(), &type_)?; + Ok(expr) + } + } + } + + pub(crate) fn tc_decl( + &mut self, + decl: ast::Decl<'ast>, + ) -> Result<Option<hir::Decl<'ast, Type>>> { + match decl { + ast::Decl::Fun { name, body } => { + let mut expr = ast::Expr::Fun(Box::new(body)); + if let Some(type_) = self.env.resolve(&name) { + expr = ast::Expr::Ascription { + expr: Box::new(expr), + type_: self.finalize_type(type_.clone())?, + }; + } + + self.env.push(); + let body = self.tc_expr(expr)?; + let type_ = body.type_().clone(); + self.env.set(name.clone(), type_); + self.env.pop(); + match body { + hir::Expr::Fun { + type_args, + args, + body, + type_, + } => Ok(Some(hir::Decl::Fun { + name, + type_args, + args, + body, + type_, + })), + _ => unreachable!(), + } + } + ast::Decl::Ascription { name, type_ } => { + let type_ = self.type_from_ast_type(type_); + self.env.set(name.clone(), type_); + Ok(None) + } + ast::Decl::Extern { name, type_ } => { + let type_ = self.type_from_ast_type(ast::Type::Function(type_)); + self.env.set(name.clone(), type_.clone()); + let (arg_types, ret_type) = match type_ { + Type::Fun { args, ret } => (args, *ret), + _ => unreachable!(), + }; + Ok(Some(hir::Decl::Extern { + name, + arg_types, + ret_type, + })) + } + } + } + + fn fresh_tv(&mut self) -> TyVar { + self.ty_var_namer.make_name() + } + + fn fresh_ex(&mut self) -> Type { + Type::Exist(self.fresh_tv()) + } + + fn fresh_univ(&mut self) -> Type { + Type::Univ(self.fresh_tv()) + } + + fn unify(&mut self, ty1: &Type, ty2: &Type) -> Result<Type> { + match (ty1, ty2) { + (Type::Unit, Type::Unit) => Ok(Type::Unit), + (Type::Exist(tv), ty) | (ty, Type::Exist(tv)) => match self.resolve_tv(*tv)? { + Some(existing_ty) if self.types_match(ty, &existing_ty) => Ok(ty.clone()), + Some(var @ ast::Type::Var(_)) => { + let var = self.type_from_ast_type(var); + self.unify(&var, ty) + } + Some(existing_ty) => match ty { + Type::Exist(_) => { + let rhs = self.type_from_ast_type(existing_ty); + self.unify(ty, &rhs) + } + _ => Err(Error::TypeMismatch { + expected: ty.clone(), + actual: self.type_from_ast_type(existing_ty), + }), + }, + None => match self.ctx.insert(*tv, ty.clone()) { + Some(existing) => self.unify(&existing, ty), + None => Ok(ty.clone()), + }, + }, + (Type::Univ(u1), Type::Univ(u2)) if u1 == u2 => Ok(ty2.clone()), + (Type::Univ(u), ty) | (ty, Type::Univ(u)) => { + let ident = self.name_univ(*u); + match self.instantiations.resolve(&ident) { + Some(existing_ty) if ty == existing_ty => Ok(ty.clone()), + Some(existing_ty) => Err(Error::TypeMismatch { + expected: ty.clone(), + actual: existing_ty.clone(), + }), + None => { + self.instantiations.set(ident, ty.clone()); + Ok(ty.clone()) + } + } + } + (Type::Prim(p1), Type::Prim(p2)) if p1 == p2 => Ok(ty2.clone()), + (Type::Tuple(t1), Type::Tuple(t2)) if t1.len() == t2.len() => { + let ts = t1 + .iter() + .zip(t2.iter()) + .map(|(t1, t2)| self.unify(t1, t2)) + .try_collect()?; + Ok(Type::Tuple(ts)) + } + ( + Type::Fun { + args: args1, + ret: ret1, + }, + Type::Fun { + args: args2, + ret: ret2, + }, + ) => { + let args = args1 + .iter() + .zip(args2) + .map(|(t1, t2)| self.unify(t1, t2)) + .try_collect()?; + let ret = self.unify(ret1, ret2)?; + Ok(Type::Fun { + args, + ret: Box::new(ret), + }) + } + (Type::Nullary(_), _) | (_, Type::Nullary(_)) => todo!(), + _ => Err(Error::TypeMismatch { + expected: ty1.clone(), + actual: ty2.clone(), + }), + } + } + + fn finalize_expr( + &self, + expr: hir::Expr<'ast, Type>, + ) -> Result<hir::Expr<'ast, ast::Type<'ast>>> { + expr.traverse_type(|ty| self.finalize_type(ty)) + } + + fn finalize_decl( + &mut self, + decl: hir::Decl<'ast, Type>, + ) -> Result<hir::Decl<'ast, ast::Type<'ast>>> { + let res = decl.traverse_type(|ty| self.finalize_type(ty))?; + if let Some(type_) = res.type_() { + let ty = self.type_from_ast_type(type_.clone()); + self.env.set(res.name().clone(), ty); + } + Ok(res) + } + + fn finalize_type(&self, ty: Type) -> Result<ast::Type<'static>> { + let ret = match ty { + Type::Exist(tv) => self.resolve_tv(tv)?.ok_or(Error::AmbiguousType(tv)), + Type::Univ(tv) => Ok(ast::Type::Var(self.name_univ(tv))), + Type::Unit => Ok(ast::Type::Unit), + Type::Nullary(_) => todo!(), + Type::Prim(pr) => Ok(pr.into()), + Type::Tuple(members) => Ok(ast::Type::Tuple( + members + .into_iter() + .map(|ty| self.finalize_type(ty)) + .try_collect()?, + )), + Type::Fun { args, ret } => Ok(ast::Type::Function(ast::FunctionType { + args: args + .into_iter() + .map(|ty| self.finalize_type(ty)) + .try_collect()?, + ret: Box::new(self.finalize_type(*ret)?), + })), + }; + ret + } + + fn resolve_tv(&self, tv: TyVar) -> Result<Option<ast::Type<'static>>> { + let mut res = &Type::Exist(tv); + Ok(loop { + match res { + Type::Exist(tv) => { + res = match self.ctx.get(tv) { + Some(r) => r, + None => return Ok(None), + }; + } + Type::Univ(tv) => { + let ident = self.name_univ(*tv); + if let Some(r) = self.instantiations.resolve(&ident) { + res = r; + } else { + break Some(ast::Type::Var(ident)); + } + } + Type::Nullary(_) => todo!(), + Type::Prim(pr) => break Some((*pr).into()), + Type::Unit => break Some(ast::Type::Unit), + Type::Fun { args, ret } => todo!(), + Type::Tuple(_) => break Some(self.finalize_type(res.clone())?), + } + }) + } + + fn type_from_ast_type(&mut self, ast_type: ast::Type<'ast>) -> Type { + match ast_type { + ast::Type::Unit => Type::Unit, + ast::Type::Int => INT, + ast::Type::Float => FLOAT, + ast::Type::Bool => BOOL, + ast::Type::CString => CSTRING, + ast::Type::Tuple(members) => Type::Tuple( + members + .into_iter() + .map(|ty| self.type_from_ast_type(ty)) + .collect(), + ), + ast::Type::Function(ast::FunctionType { args, ret }) => Type::Fun { + args: args + .into_iter() + .map(|t| self.type_from_ast_type(t)) + .collect(), + ret: Box::new(self.type_from_ast_type(*ret)), + }, + ast::Type::Var(id) => Type::Univ({ + let opt_tv = { self.type_vars.borrow_mut().0.get_by_left(&id).copied() }; + opt_tv.unwrap_or_else(|| { + let tv = self.fresh_tv(); + self.type_vars + .borrow_mut() + .0 + .insert_no_overwrite(id, tv) + .unwrap(); + tv + }) + }), + } + } + + fn name_univ(&self, tv: TyVar) -> Ident<'static> { + let mut vars = self.type_vars.borrow_mut(); + vars.0 + .get_by_right(&tv) + .map(Ident::to_owned) + .unwrap_or_else(|| { + let name = loop { + let name = vars.1.make_name(); + if !vars.0.contains_left(&name) { + break name; + } + }; + vars.0.insert_no_overwrite(name.clone(), tv).unwrap(); + name + }) + } + + fn commit_instantiations(&mut self) -> HashMap<Ident<'ast>, Type> { + let mut res = HashMap::new(); + let mut ctx = mem::take(&mut self.ctx); + for (_, v) in ctx.iter_mut() { + if let Type::Univ(tv) = v { + let tv_name = self.name_univ(*tv); + if let Some(concrete) = self.instantiations.resolve(&tv_name) { + res.insert(tv_name, concrete.clone()); + *v = concrete.clone(); + } + } + } + self.ctx = ctx; + self.instantiations.pop(); + res + } + + fn types_match(&self, type_: &Type, ast_type: &ast::Type<'ast>) -> bool { + match (type_, ast_type) { + (Type::Univ(u), ast::Type::Var(v)) => { + Some(u) == self.type_vars.borrow().0.get_by_left(v) + } + (Type::Univ(_), _) => false, + (Type::Exist(_), _) => false, + (Type::Unit, ast::Type::Unit) => true, + (Type::Unit, _) => false, + (Type::Nullary(_), _) => todo!(), + (Type::Prim(pr), ty) => ast::Type::from(*pr) == *ty, + (Type::Tuple(members), ast::Type::Tuple(members2)) => members + .iter() + .zip(members2.iter()) + .all(|(t1, t2)| self.types_match(t1, t2)), + (Type::Tuple(members), _) => false, + (Type::Fun { args, ret }, ast::Type::Function(ft)) => { + args.len() == ft.args.len() + && args + .iter() + .zip(&ft.args) + .all(|(a1, a2)| self.types_match(a1, &a2)) + && self.types_match(&*ret, &*ft.ret) + } + (Type::Fun { .. }, _) => false, + } + } +} + +pub fn typecheck_expr(expr: ast::Expr) -> Result<hir::Expr<ast::Type>> { + let mut typechecker = Typechecker::new(); + let typechecked = typechecker.tc_expr(expr)?; + typechecker.finalize_expr(typechecked) +} + +pub fn typecheck_toplevel(decls: Vec<ast::Decl>) -> Result<Vec<hir::Decl<ast::Type>>> { + let mut typechecker = Typechecker::new(); + let mut res = Vec::with_capacity(decls.len()); + for decl in decls { + if let Some(hir_decl) = typechecker.tc_decl(decl)? { + let hir_decl = typechecker.finalize_decl(hir_decl)?; + res.push(hir_decl); + } + typechecker.ctx.clear(); + } + Ok(res) +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! assert_type { + ($expr: expr, $type: expr) => { + use crate::parser::{expr, type_}; + let parsed_expr = test_parse!(expr, $expr); + let parsed_type = test_parse!(type_, $type); + let res = typecheck_expr(parsed_expr).unwrap_or_else(|e| panic!("{}", e)); + assert!( + res.type_().alpha_equiv(&parsed_type), + "{} inferred type {}, but expected {}", + $expr, + res.type_(), + $type + ); + }; + + (toplevel($program: expr), $($decl: ident => $type: expr),+ $(,)?) => {{ + use crate::parser::{toplevel, type_}; + let program = test_parse!(toplevel, $program); + let res = typecheck_toplevel(program).unwrap_or_else(|e| panic!("{}", e)); + $( + let parsed_type = test_parse!(type_, $type); + let ident = Ident::try_from(::std::stringify!($decl)).unwrap(); + let decl = res.iter().find(|decl| { + matches!(decl, crate::ast::hir::Decl::Fun { name, .. } if name == &ident) + }).unwrap_or_else(|| panic!("Could not find declaration for {}", ident)); + assert!( + decl.type_().unwrap().alpha_equiv(&parsed_type), + "inferred type {} for {}, but expected {}", + decl.type_().unwrap(), + ident, + $type + ); + )+ + }}; + } + + macro_rules! assert_type_error { + ($expr: expr) => { + use crate::parser::expr; + let parsed_expr = test_parse!(expr, $expr); + let res = typecheck_expr(parsed_expr); + assert!( + res.is_err(), + "Expected type error, but got type: {}", + res.unwrap().type_() + ); + }; + } + + #[test] + fn literal_int() { + assert_type!("1", "int"); + } + + #[test] + fn conditional() { + assert_type!("if 1 == 2 then 3 else 4", "int"); + } + + #[test] + #[ignore] + fn add_bools() { + assert_type_error!("true + false"); + } + + #[test] + fn call_generic_function() { + assert_type!("(fn x = x) 1", "int"); + } + + #[test] + fn call_let_bound_generic() { + assert_type!("let id = fn x = x in id 1", "int"); + } + + #[test] + fn universal_ascripted_let() { + assert_type!("let id: fn a -> a = fn x = x in id 1", "int"); + } + + #[test] + fn call_generic_function_toplevel() { + assert_type!( + toplevel( + "ty id : fn a -> a + fn id x = x + + fn main = id 0" + ), + main => "fn -> int", + id => "fn a -> a", + ); + } + + #[test] + #[ignore] + fn let_generalization() { + assert_type!("let id = fn x = x in if id true then id 1 else 2", "int"); + } + + #[test] + fn concrete_function() { + assert_type!("fn x = x + 1", "fn int -> int"); + } + + #[test] + fn arg_ascriptions() { + assert_type!("fn (x: int) = x", "fn int -> int"); + } + + #[test] + fn call_concrete_function() { + assert_type!("(fn x = x + 1) 2", "int"); + } + + #[test] + fn conditional_non_bool() { + assert_type_error!("if 3 then true else false"); + } + + #[test] + fn let_int() { + assert_type!("let x = 1 in x", "int"); + } +} diff --git a/users/grfn/achilles/tests/compile.rs b/users/grfn/achilles/tests/compile.rs new file mode 100644 index 000000000000..0f1086bfd8e1 --- /dev/null +++ b/users/grfn/achilles/tests/compile.rs @@ -0,0 +1,79 @@ +use std::process::Command; + +use crate_root::root; + +struct Fixture { + name: &'static str, + exit_code: i32, + expected_output: &'static str, +} + +const FIXTURES: &[Fixture] = &[ + Fixture { + name: "simple", + exit_code: 5, + expected_output: "", + }, + Fixture { + name: "functions", + exit_code: 9, + expected_output: "", + }, + Fixture { + name: "externs", + exit_code: 0, + expected_output: "foobar\n", + }, + Fixture { + name: "units", + exit_code: 0, + expected_output: "hi\n", + }, +]; + +#[test] +fn compile_and_run_files() { + let ach = root().unwrap().join("ach"); + + println!("Running: `make clean`"); + assert!( + Command::new("make") + .arg("clean") + .current_dir(&ach) + .spawn() + .unwrap() + .wait() + .unwrap() + .success(), + "make clean failed" + ); + + for Fixture { + name, + exit_code, + expected_output, + } in FIXTURES + { + println!(">>> Testing: {}", name); + + println!(" Running: `make {}`", name); + assert!( + Command::new("make") + .arg(name) + .current_dir(&ach) + .spawn() + .unwrap() + .wait() + .unwrap() + .success(), + "make failed" + ); + + let out_path = ach.join(name); + println!(" Running: `{}`", out_path.to_str().unwrap()); + let output = Command::new(out_path).output().unwrap(); + assert_eq!(output.status.code().unwrap(), *exit_code,); + assert_eq!(output.stdout, expected_output.as_bytes()); + println!(" OK"); + } +} diff --git a/users/grfn/emacs.d/+bindings.el b/users/grfn/emacs.d/+bindings.el new file mode 100644 index 000000000000..05c83435a596 --- /dev/null +++ b/users/grfn/emacs.d/+bindings.el @@ -0,0 +1,1427 @@ +;; -*- lexical-binding: t; -*- + +(load! "utils") +(require 'f) +(require 'predd) + +(undefine-key! :keymaps 'doom-leader-map "/") + +(defmacro find-file-in! (path &optional project-p) + "Returns an interactive function for searching files." + `(lambda () (interactive) + (let ((default-directory ,path)) + (call-interactively + ',(command-remapping + (if project-p + #'projectile-find-file + #'find-file)))))) + +(defun dired-mode-p () (eq 'dired-mode major-mode)) + +(defun grfn/dired-minus () + (interactive) + (if (dired-mode-p) + (dired-up-directory) + (when buffer-file-name + (-> (buffer-file-name) + (f-dirname) + (dired))))) + +(defmacro define-move-and-insert + (name &rest body) + `(defun ,name (count &optional vcount skip-empty-lines) + ;; Following interactive form taken from the source for `evil-insert' + (interactive + (list (prefix-numeric-value current-prefix-arg) + (and (evil-visual-state-p) + (memq (evil-visual-type) '(line block)) + (save-excursion + (let ((m (mark))) + ;; go to upper-left corner temporarily so + ;; `count-lines' yields accurate results + (evil-visual-rotate 'upper-left) + (prog1 (count-lines evil-visual-beginning evil-visual-end) + (set-mark m))))) + (evil-visual-state-p))) + (atomic-change-group + ,@body + (evil-insert count vcount skip-empty-lines)))) + +(define-move-and-insert grfn/insert-at-sexp-end + (when (not (equal (get-char) "(")) + (backward-up-list)) + (forward-sexp) + (backward-char)) + +(define-move-and-insert grfn/insert-at-sexp-start + (backward-up-list) + (forward-char)) + +(define-move-and-insert grfn/insert-at-form-start + (backward-sexp) + (backward-char) + (insert " ")) + +(define-move-and-insert grfn/insert-at-form-end + (forward-sexp) + (insert " ")) + +(load! "splitjoin") + +(defun +hlissner/install-snippets () + "Install my snippets from https://github.com/hlissner/emacs-snippets into +private/hlissner/snippets." + (interactive) + (doom-fetch :github "hlissner/emacs-snippets" + (expand-file-name "snippets" (doom-module-path :private 'hlissner)))) + +(defun +hlissner/yank-buffer-filename () + "Copy the current buffer's path to the kill ring." + (interactive) + (if-let* ((filename (or buffer-file-name (bound-and-true-p list-buffers-directory)))) + (message (kill-new (abbreviate-file-name filename))) + (error "Couldn't find filename in current buffer"))) + +(defmacro +def-finder! (name dir) + "Define a pair of find-file and browse functions." + `(progn + (defun ,(intern (format "+find-in-%s" name)) () + (interactive) + (let ((default-directory ,dir) + projectile-project-name + projectile-require-project-root + projectile-cached-buffer-file-name + projectile-cached-project-root) + (call-interactively #'projectile-find-file))) + (defun ,(intern (format "+hlissner/browse-%s" name)) () + (interactive) + (let ((default-directory ,dir)) + (call-interactively (command-remapping #'find-file)))))) + +(+def-finder! templates +file-templates-dir) +(+def-finder! snippets +grfn-snippets-dir) +(+def-finder! dotfiles (expand-file-name ".dotfiles" "~")) +(+def-finder! doomd (expand-file-name ".doom.d" "~")) +(+def-finder! notes +org-dir) +(+def-finder! home-config (expand-file-name "code/system/home" "~")) +(+def-finder! system-config (expand-file-name "code/system/system" "~")) + +(defun +grfn/paxedit-kill (&optional n) + (interactive "p") + (or (paxedit-comment-kill) + (when (paxedit-symbol-cursor-within?) + (paxedit-symbol-kill)) + (paxedit-implicit-sexp-kill n) + (paxedit-sexp-kill n) + (message paxedit-message-kill))) + +;;; + +(evil-set-command-property 'flycheck-next-error :repeat nil) +(evil-set-command-property 'flycheck-prev-error :repeat nil) + +;;; + +(map! + [remap evil-jump-to-tag] #'projectile-find-tag + [remap find-tag] #'projectile-find-tag + ;; ensure there are no conflicts + :nmvo doom-leader-key nil + :nmvo doom-localleader-key nil) + +(undefine-key! :keymaps 'doom-leader-map "/") + +(map! + ;; --- Global keybindings --------------------------- + ;; Make M-x available everywhere + :gnvime "M-x" #'execute-extended-command + :gnvime "A-x" #'execute-extended-command + ;; Emacs debug utilities + :gnvime "M-;" #'eval-expression + :gnvime "M-:" #'doom/open-scratch-buffer + ;; Text-scaling + "M-+" (λ! (text-scale-set 0)) + "M-=" #'text-scale-increase + "M--" #'text-scale-decrease + ;; Simple window navigation/manipulation + "C-`" #'doom/popup-toggle + "C-~" #'doom/popup-raise + "M-t" #'+workspace/new + "M-T" #'+workspace/display + "M-w" #'delete-window + "M-W" #'+workspace/close-workspace-or-frame + "M-n" #'evil-buffer-new + "M-N" #'make-frame + "M-1" (λ! (+workspace/switch-to 0)) + "M-2" (λ! (+workspace/switch-to 1)) + "M-3" (λ! (+workspace/switch-to 2)) + "M-4" (λ! (+workspace/switch-to 3)) + "M-5" (λ! (+workspace/switch-to 4)) + "M-6" (λ! (+workspace/switch-to 5)) + "M-7" (λ! (+workspace/switch-to 6)) + "M-8" (λ! (+workspace/switch-to 7)) + "M-9" (λ! (+workspace/switch-to 8)) + "M-0" #'+workspace/switch-to-last + ;; Other sensible, textmate-esque global bindings + :ne "M-r" #'+eval/buffer + :ne "M-R" #'+eval/region-and-replace + :ne "M-b" #'+eval/build + :ne "M-a" #'mark-whole-buffer + :ne "M-c" #'evil-yank + :ne "M-q" (if (daemonp) #'delete-frame #'save-buffers-kill-emacs) + :ne "M-f" #'swiper + :ne "C-M-f" #'doom/toggle-fullscreen + :n "M-s" #'save-buffer + :m "A-j" #'+hlissner:multi-next-line + :m "A-k" #'+hlissner:multi-previous-line + :nv "C-SPC" #'+evil:fold-toggle + :gnvimer "M-v" #'clipboard-yank + ;; Easier window navigation + :en "C-h" #'evil-window-left + :en "C-j" #'evil-window-down + :en "C-k" #'evil-window-up + :en "C-l" #'evil-window-right + :n "U" #'undo-tree-visualize + + "C-x p" #'doom/other-popup + + :n "K" #'+lookup/documentation + :n "g d" #'+lookup/definition + + + ;; --- <leader> ------------------------------------- + (:leader + :desc "Ex command" :nv ";" #'evil-ex + :desc "M-x" :nv ":" #'execute-extended-command + :desc "Pop up scratch buffer" :nv "x" #'doom/open-scratch-buffer + :desc "Org Capture" :nv "X" #'org-capture + :desc "Org Capture" :nv "a" #'org-capture + + ;; Most commonly used + :desc "Find file in project" :n "SPC" #'projectile-find-file + :desc "Switch workspace buffer" :n "," #'persp-switch-to-buffer + :desc "Switch buffer" :n "<" #'switch-to-buffer + :desc "Browse files" :n "." #'find-file + :desc "Toggle last popup" :n "~" #'doom/popup-toggle + :desc "Eval expression" :n "`" #'eval-expression + :desc "Blink cursor line" :n "DEL" #'+doom/blink-cursor + :desc "Jump to bookmark" :n "RET" #'bookmark-jump + + ;; C-u is used by evil + :desc "Universal argument" :n "u" #'universal-argument + :desc "window" :n "w" evil-window-map + + (:desc "previous..." :prefix "[" + :desc "Text size" :nv "[" #'text-scale-decrease + :desc "Buffer" :nv "b" #'doom/previous-buffer + :desc "Diff Hunk" :nv "d" #'git-gutter:previous-hunk + :desc "Todo" :nv "t" #'hl-todo-previous + :desc "Error" :nv "e" #'flycheck-previous-error + :desc "Workspace" :nv "w" #'+workspace/switch-left + :desc "Smart jump" :nv "h" #'smart-backward + :desc "Spelling error" :nv "s" #'evil-prev-flyspell-error + :desc "Spelling correction" :n "S" #'flyspell-correct-previous-word-generic + :desc "Git conflict" :n "n" #'smerge-prev) + + (:desc "next..." :prefix "]" + :desc "Text size" :nv "]" #'text-scale-increase + :desc "Buffer" :nv "b" #'doom/next-buffer + :desc "Diff Hunk" :nv "d" #'git-gutter:next-hunk + :desc "Todo" :nv "t" #'hl-todo-next + :desc "Error" :nv "e" #'flycheck-next-error + :desc "Workspace" :nv "w" #'+workspace/switch-right + :desc "Smart jump" :nv "l" #'smart-forward + :desc "Spelling error" :nv "s" #'evil-next-flyspell-error + :desc "Spelling correction" :n "S" #'flyspell-correct-word-generic + :desc "Git conflict" :n "n" #'smerge-next) + + (:desc "search" :prefix "/" + :desc "Swiper" :nv "/" #'swiper + :desc "Imenu" :nv "i" #'imenu + :desc "Imenu across buffers" :nv "I" #'imenu-anywhere + :desc "Online providers" :nv "o" #'+lookup/online-select) + + (:desc "workspace" :prefix "TAB" + :desc "Display tab bar" :n "TAB" #'+workspace/display + :desc "New workspace" :n "n" #'+workspace/new + :desc "Load workspace from file" :n "l" #'+workspace/load + :desc "Load last session" :n "L" (λ! (+workspace/load-session)) + :desc "Save workspace to file" :n "s" #'+workspace/save + :desc "Autosave current session" :n "S" #'+workspace/save-session + :desc "Switch workspace" :n "." #'+workspace/switch-to + :desc "Kill all buffers" :n "x" #'doom/kill-all-buffers + :desc "Delete session" :n "X" #'+workspace/kill-session + :desc "Delete this workspace" :n "d" #'+workspace/delete + :desc "Load session" :n "L" #'+workspace/load-session + :desc "Next workspace" :n "]" #'+workspace/switch-right + :desc "Previous workspace" :n "[" #'+workspace/switch-left + :desc "Switch to 1st workspace" :n "1" (λ! (+workspace/switch-to 0)) + :desc "Switch to 2nd workspace" :n "2" (λ! (+workspace/switch-to 1)) + :desc "Switch to 3rd workspace" :n "3" (λ! (+workspace/switch-to 2)) + :desc "Switch to 4th workspace" :n "4" (λ! (+workspace/switch-to 3)) + :desc "Switch to 5th workspace" :n "5" (λ! (+workspace/switch-to 4)) + :desc "Switch to 6th workspace" :n "6" (λ! (+workspace/switch-to 5)) + :desc "Switch to 7th workspace" :n "7" (λ! (+workspace/switch-to 6)) + :desc "Switch to 8th workspace" :n "8" (λ! (+workspace/switch-to 7)) + :desc "Switch to 9th workspace" :n "9" (λ! (+workspace/switch-to 8)) + :desc "Switch to last workspace" :n "0" #'+workspace/switch-to-last) + + (:desc "buffer" :prefix "b" + :desc "New empty buffer" :n "n" #'evil-buffer-new + :desc "Switch workspace buffer" :n "b" #'switch-to-buffer + :desc "Switch buffer" :n "B" #'switch-to-buffer + :desc "Kill buffer" :n "k" #'doom/kill-this-buffer + :desc "Kill other buffers" :n "o" #'doom/kill-other-buffers + :desc "Save buffer" :n "s" #'save-buffer + :desc "Pop scratch buffer" :n "x" #'doom/open-scratch-buffer + :desc "Bury buffer" :n "z" #'bury-buffer + :desc "Next buffer" :n "]" #'doom/next-buffer + :desc "Previous buffer" :n "[" #'doom/previous-buffer + :desc "Sudo edit this file" :n "S" #'doom/sudo-this-file) + + (:desc "code" :prefix "c" + :desc "List errors" :n "x" #'flycheck-list-errors + :desc "Evaluate buffer/region" :n "e" #'+eval/buffer + :v "e" #'+eval/region + :desc "Evaluate & replace region" :nv "E" #'+eval:replace-region + :desc "Build tasks" :nv "b" #'+eval/build + :desc "Jump to definition" :n "d" #'+lookup/definition + :desc "Jump to references" :n "D" #'+lookup/references + :desc "Open REPL" :n "r" #'+eval/open-repl + :v "r" #'+eval:repl) + + (:desc "file" :prefix "f" + :desc "Find file" :n "." #'find-file + :desc "Sudo find file" :n ">" #'doom/sudo-find-file + :desc "Find file in project" :n "/" #'projectile-find-file + :desc "Find file from here" :n "?" #'counsel-file-jump + :desc "Find other file" :n "a" #'projectile-find-other-file + :desc "Open project editorconfig" :n "c" #'editorconfig-find-current-editorconfig + :desc "Find file in dotfiles" :n "d" #'+find-in-dotfiles + :desc "Find file in system config" :n "s" #'+find-in-system-config + :desc "Find file in home config" :n "h" #'+find-in-home-config + :desc "Browse dotfiles" :n "D" #'+hlissner/browse-dotfiles + :desc "Find file in emacs.d" :n "e" #'+find-in-doomd + :desc "Browse emacs.d" :n "E" #'+hlissner/browse-doomd + :desc "Recent files" :n "r" #'recentf-open-files + :desc "Recent project files" :n "R" #'projectile-recentf + :desc "Yank filename" :n "y" #'+hlissner/yank-buffer-filename) + + (:desc "git" :prefix "g" + :desc "Git status" :n "S" #'magit-status + :desc "Git blame" :n "b" #'magit-blame + :desc "Git time machine" :n "t" #'git-timemachine-toggle + :desc "Git stage hunk" :n "s" #'git-gutter:stage-hunk + :desc "Git revert hunk" :n "r" #'git-gutter:revert-hunk + :desc "Git revert buffer" :n "R" #'vc-revert + ;; :desc "List gists" :n "g" #'+gist:list + :desc "Git grep" :n "g" #'counsel-projectile-rg + :desc "Checkout Branch" :n "c" #'counsel-git-checkout + :desc "Next hunk" :nv "]" #'git-gutter:next-hunk + :desc "Previous hunk" :nv "[" #'git-gutter:previous-hunk + + (:desc "smerge" :prefix "m" + :desc "Keep Current" :n "SPC" #'smerge-keep-current + :desc "Keep All" :n "a" #'smerge-keep-all + :desc "Keep Upper" :n "u" #'smerge-keep-upper + :desc "Keep Lower" :n "l" #'smerge-keep-lower)) + + (:desc "help" :prefix "h" + :n "h" help-map + :desc "Apropos" :n "a" #'apropos + :desc "Reload theme" :n "R" #'doom//reload-theme + :desc "Find library" :n "l" #'find-library + :desc "Toggle Emacs log" :n "m" #'doom/popup-toggle-messages + :desc "Command log" :n "L" #'global-command-log-mode + :desc "Describe function" :n "f" #'describe-function + :desc "Describe key" :n "k" #'describe-key + :desc "Describe char" :n "c" #'describe-char + :desc "Describe mode" :n "M" #'describe-mode + :desc "Describe variable" :n "v" #'describe-variable + :desc "Describe face" :n "F" #'describe-face + :desc "Describe DOOM setting" :n "s" #'doom/describe-setting + :desc "Describe DOOM module" :n "d" #'doom/describe-module + :desc "Find definition" :n "." #'+lookup/definition + :desc "Find references" :n "/" #'+lookup/references + :desc "Find documentation" :n "h" #'+lookup/documentation + :desc "What face" :n "'" #'doom/what-face + :desc "What minor modes" :n ";" #'doom/what-minor-mode + :desc "Info" :n "i" #'info + :desc "Toggle profiler" :n "p" #'doom/toggle-profiler) + + (:desc "insert" :prefix "i" + :desc "From kill-ring" :nv "y" #'counsel-yank-pop + :desc "From snippet" :nv "s" #'yas-insert-snippet) + + (:desc "notes" :prefix "n" + :desc "Agenda" :n "a" #'org-agenda + :desc "Find file in notes" :n "n" #'+find-in-notes + :desc "Store link" :n "l" #'org-store-link + :desc "Browse notes" :n "N" #'+hlissner/browse-notes + :desc "Org capture" :n "x" #'+org-capture/open + :desc "Create clubhouse story" :n "c" #'org-tracker-create-issue + :desc "Archive subtree" :n "k" #'org-archive-subtree + :desc "Goto clocked-in note" :n "g" #'org-clock-goto + :desc "Clock Out" :n "o" #'org-clock-out) + + + (:desc "open" :prefix "o" + :desc "Default browser" :n "b" #'browse-url-of-file + :desc "Debugger" :n "d" #'+debug/open + :desc "Terminal in project" :n "T" #'+term/open-popup-in-project + + :desc "Slack IM" :n "i" #'slack-im-select + :desc "Slack Channel" :n "c" #'slack-channel-select + :desc "Slack Group" :n "g" #'slack-group-select + :desc "Slack Unreads" :n "u" #'slack-select-unread-rooms + :desc "Slack Threads" :n "r" #'slack-all-threads + + :desc "Email" :n "m" #'notmuch-tree-jump-search + + (:desc "ERC" :prefix "e" + :desc "Channel" :n "c" #'erc-switch-to-buffer) + + ;; applications + :desc "APP: elfeed" :n "E" #'=rss + :desc "APP: twitter" :n "T" #'=twitter + + (:desc "spotify" :prefix "s" + :desc "Search track" :n "t" #'counsel-spotify-search-track + :desc "Search album" :n "a" #'counsel-spotify-search-album + :desc "Search artist" :n "A" #'counsel-spotify-search-artist) + + ;; macos + (:when IS-MAC + :desc "Reveal in Finder" :n "o" #'+macos/reveal-in-finder + :desc "Reveal project in Finder" :n "O" #'+macos/reveal-project-in-finder + :desc "Send to Transmit" :n "u" #'+macos/send-to-transmit + :desc "Send project to Transmit" :n "U" #'+macos/send-project-to-transmit + :desc "Send to Launchbar" :n "l" #'+macos/send-to-launchbar + :desc "Send project to Launchbar" :n "L" #'+macos/send-project-to-launchbar)) + + (:desc "Email" :prefix "M" + :desc "Compose" :n "m" #'+notmuch/compose) + + (:desc "project" :prefix "p" + :desc "Browse project" :n "." (find-file-in! (doom-project-root)) + :desc "Find file in project" :n "/" #'projectile-find-file + :desc "Run cmd in project root" :nv "!" #'projectile-run-shell-command-in-root + :desc "Switch project" :n "p" #'projectile-switch-project + :desc "Recent project files" :n "r" #'projectile-recentf + :desc "List project tasks" :n "t" #'+ivy/tasks + :desc "Pop term in project" :n "o" #'+term/open-popup-in-project + :desc "Invalidate cache" :n "x" #'projectile-invalidate-cache) + + (:desc "quit" :prefix "q" + :desc "Quit" :n "q" #'evil-save-and-quit + :desc "Quit (forget session)" :n "Q" #'+workspace/kill-session-and-quit) + + (:desc "remote" :prefix "r" + :desc "Upload local" :n "u" #'+upload/local + :desc "Upload local (force)" :n "U" (λ! (+upload/local t)) + :desc "Download remote" :n "d" #'+upload/remote-download + :desc "Diff local & remote" :n "D" #'+upload/diff + :desc "Browse remote files" :n "." #'+upload/browse + :desc "Detect remote changes" :n ">" #'+upload/check-remote) + + (:desc "snippets" :prefix "s" + :desc "New snippet" :n "n" #'yas-new-snippet + :desc "Insert snippet" :nv "i" #'yas-insert-snippet + :desc "Find snippet for mode" :n "s" #'yas-visit-snippet-file + :desc "Find snippet" :n "S" #'+find-in-snippets) + + (:desc "toggle" :prefix "t" + :desc "Flyspell" :n "s" #'flyspell-mode + :desc "Flycheck" :n "f" #'flycheck-mode + :desc "Line numbers" :n "l" #'doom/toggle-line-numbers + :desc "Fullscreen" :n "f" #'doom/toggle-fullscreen + :desc "Indent guides" :n "i" #'highlight-indentation-mode + :desc "Indent guides (column)" :n "I" #'highlight-indentation-current-column-mode + :desc "Impatient mode" :n "h" #'+impatient-mode/toggle + :desc "Big mode" :n "b" #'doom-big-font-mode + :desc "Evil goggles" :n "g" #'+evil-goggles/toggle)) + + + ;; --- vim-vinegar + :n "-" #'grfn/dired-minus + (:after dired-mode + (:map dired-mode-map + "-" #'grfn/dired-minus)) + + (:map smartparens-mode-map + :n "g o" #'sp-raise-sexp) + + ;; --- vim-sexp-mappings-for-regular-people + (:after paxedit + (:map paxedit-mode-map + :i ";" #'paxedit-insert-semicolon + :i "(" #'paxedit-open-round + :i "[" #'paxedit-open-bracket + :i "{" #'paxedit-open-curly + :n [remap evil-yank-line] #'paxedit-copy + :n [remap evil-delete-line] #'+grfn/paxedit-kill + :n "g o" #'paxedit-sexp-raise + :n [remap evil-join-whitespace] #'paxedit-compress + :n "g S" #'paxedit-format-1 + :n "g k" #'paxedit-backward-up + :n "g j" #'paxedit-backward-end)) + + ;; --- vim-splitjoin + :n [remap evil-join-whitespace] #'+splitjoin/join + :n "gS" #'+splitjoin/split + + ;; --- Personal vim-esque bindings ------------------ + :n "zx" #'doom/kill-this-buffer + :n "ZX" #'bury-buffer + :n "]b" #'doom/next-buffer + :n "[b" #'doom/previous-buffer + :n "]w" #'+workspace/switch-right + :n "[w" #'+workspace/switch-left + :m "gt" #'+workspace/switch-right + :m "gT" #'+workspace/switch-left + :m "gd" #'+lookup/definition + :m "gD" #'+lookup/references + :m "K" #'+lookup/documentation + :n "gp" #'+evil/reselect-paste + :n "gr" #'+eval:region + :n "gR" #'+eval/buffer + :v "gR" #'+eval:replace-region + :v "@" #'+evil:macro-on-all-lines + :n "g@" #'+evil:macro-on-all-lines + ;; repeat in visual mode (FIXME buggy) + :v "." #'evil-repeat + ;; don't leave visual mode after shifting + ;; :v "<" #'+evil/visual-dedent ; vnoremap < <gv + ;; :v ">" #'+evil/visual-indent ; vnoremap > >gv + ;; paste from recent yank register (which isn't overwritten) + :v "C-p" "\"0p" + + (:map evil-window-map ; prefix "C-w" + ;; Navigation + "C-h" #'evil-window-left + "C-j" #'evil-window-down + "C-k" #'evil-window-up + "C-l" #'evil-window-right + "C-w" #'ace-window + ;; Swapping windows + "H" #'+evil/window-move-left + "J" #'+evil/window-move-down + "K" #'+evil/window-move-up + "L" #'+evil/window-move-right + "C-S-w" #'ace-swap-window + ;; Window undo/redo + "u" #'winner-undo + "C-u" #'winner-undo + "C-r" #'winner-redo + "o" #'doom/window-enlargen + ;; Delete window + "c" #'+workspace/close-window-or-workspace + "C-C" #'ace-delete-window + ;; Popups + "p" #'doom/popup-toggle + "m" #'doom/popup-toggle-messages + "P" #'doom/popup-close-all) + + + ;; --- Plugin bindings ------------------------------ + ;; auto-yasnippet + :i [C-tab] #'aya-expand + :nv [C-tab] #'aya-create + + ;; company-mode (vim-like omnicompletion) + :i "C-SPC" #'+company/complete + (:prefix "C-x" + :i "C-l" #'+company/whole-lines + :i "C-k" #'+company/dict-or-keywords + :i "C-f" #'company-files + :i "C-]" #'company-etags + :i "s" #'company-ispell + :i "C-s" #'company-yasnippet + :i "C-o" #'company-capf + :i "C-n" #'company-dabbrev-code + :i "C-p" #'+company/dabbrev-code-previous) + (:after company + (:map company-active-map + ;; Don't interfere with `evil-delete-backward-word' in insert mode + "C-w" nil + "C-o" #'company-search-kill-others + "C-n" #'company-select-next + "C-p" #'company-select-previous + "C-h" #'company-quickhelp-manual-begin + "C-S-h" #'company-show-doc-buffer + "C-S-s" #'company-search-candidates + "C-s" #'company-filter-candidates + "C-SPC" #'company-complete-common + "C-h" #'company-quickhelp-manual-begin + [tab] #'company-complete-common-or-cycle + [backtab] #'company-select-previous + [escape] (λ! (company-abort) (evil-normal-state 1))) + ;; Automatically applies to `company-filter-map' + (:map company-search-map + "C-n" #'company-search-repeat-forward + "C-p" #'company-search-repeat-backward + "C-s" (λ! (company-search-abort) (company-filter-candidates)) + [escape] #'company-search-abort)) + + ;; counsel +; (:after counsel +; (:map counsel-ag-map +; [backtab] #'+ivy/wgrep-occur ; search/replace on results +; "C-SPC" #'ivy-call-and-recenter ; preview)) + + ;; evil-commentary + ;; :n "gc" #'evil-commentary + + ;; evil-exchange + :n "gx" #'evil-exchange + + ;; evil-magit + (:after evil-magit + :map (magit-status-mode-map magit-revision-mode-map) + :n "C-j" nil + :n "C-k" nil) + + ;; Smerge + :n "]n" #'smerge-next + :n "[n" #'smerge-prev + + ;; evil-mc + (:prefix "gz" + :nv "m" #'evil-mc-make-all-cursors + :nv "u" #'evil-mc-undo-all-cursors + :nv "z" #'+evil/mc-make-cursor-here + :nv "t" #'+evil/mc-toggle-cursors + :nv "n" #'evil-mc-make-and-goto-next-cursor + :nv "p" #'evil-mc-make-and-goto-prev-cursor + :nv "N" #'evil-mc-make-and-goto-last-cursor + :nv "P" #'evil-mc-make-and-goto-first-cursor + :nv "d" #'evil-mc-make-and-goto-next-match + :nv "D" #'evil-mc-make-and-goto-prev-match) + (:after evil-mc + :map evil-mc-key-map + :nv "C-n" #'evil-mc-make-and-goto-next-cursor + :nv "C-N" #'evil-mc-make-and-goto-last-cursor + :nv "C-p" #'evil-mc-make-and-goto-prev-cursor + :nv "C-P" #'evil-mc-make-and-goto-first-cursor) + + ;; evil-multiedit + :v "R" #'evil-multiedit-match-all + :n "M-d" #'evil-multiedit-match-symbol-and-next + :n "M-D" #'evil-multiedit-match-symbol-and-prev + :v "M-d" #'evil-multiedit-match-and-next + :v "M-D" #'evil-multiedit-match-and-prev + :nv "C-M-d" #'evil-multiedit-restore + (:after evil-multiedit + (:map evil-multiedit-state-map + "M-d" #'evil-multiedit-match-and-next + "M-D" #'evil-multiedit-match-and-prev + "RET" #'evil-multiedit-toggle-or-restrict-region) + (:map (evil-multiedit-state-map evil-multiedit-insert-state-map) + "C-n" #'evil-multiedit-next + "C-p" #'evil-multiedit-prev)) + + ;; evil-snipe + (:after evil-snipe + ;; Binding to switch to evil-easymotion/avy after a snipe + :map evil-snipe-parent-transient-map + "C-;" (λ! (require 'evil-easymotion) + (call-interactively + (evilem-create #'evil-snipe-repeat + :bind ((evil-snipe-scope 'whole-buffer) + (evil-snipe-enable-highlight) + (evil-snipe-enable-incremental-highlight)))))) + + ;; evil-surround + :v "S" #'evil-surround-region + :o "s" #'evil-surround-edit + :o "S" #'evil-Surround-edit + + ;; expand-region + :v "v" #'er/expand-region + :v "V" #'er/contract-region + + ;; flycheck + :m "]e" #'flycheck-next-error + :m "[e" #'flycheck-previous-error + (:after flycheck + :map flycheck-error-list-mode-map + :n "C-n" #'flycheck-error-list-next-error + :n "C-p" #'flycheck-error-list-previous-error + :n "j" #'flycheck-error-list-next-error + :n "k" #'flycheck-error-list-previous-error + :n "RET" #'flycheck-error-list-goto-error) + + ;; flyspell + :m "]S" #'flyspell-correct-word-generic + :m "[S" #'flyspell-correct-previous-word-generic + + ;; git-gutter + :m "]d" #'git-gutter:next-hunk + :m "[d" #'git-gutter:previous-hunk + + ;; git-timemachine + (:after git-timemachine + (:map git-timemachine-mode-map + :n "C-p" #'git-timemachine-show-previous-revision + :n "C-n" #'git-timemachine-show-next-revision + :n "[[" #'git-timemachine-show-previous-revision + :n "]]" #'git-timemachine-show-next-revision + :n "q" #'git-timemachine-quit + :n "gb" #'git-timemachine-blame)) + + ;; gist + (:after gist + :map gist-list-menu-mode-map + :n "RET" #'+gist/open-current + :n "b" #'gist-browse-current-url + :n "c" #'gist-add-buffer + :n "d" #'gist-kill-current + :n "f" #'gist-fork + :n "q" #'quit-window + :n "r" #'gist-list-reload + :n "s" #'gist-star + :n "S" #'gist-unstar + :n "y" #'gist-print-current-url) + + ;; helm + (:after helm + (:map helm-map + "ESC" nil + "C-S-n" #'helm-next-source + "C-S-p" #'helm-previous-source + "C-u" #'helm-delete-minibuffer-contents + "C-w" #'backward-kill-word + "C-r" #'evil-paste-from-register ; Evil registers in helm! Glorious! + "C-b" #'backward-word + [left] #'backward-char + [right] #'forward-char + [escape] #'helm-keyboard-quit + [tab] #'helm-execute-persistent-action) + + (:after helm-files + (:map helm-generic-files-map + :e "ESC" #'helm-keyboard-quit) + (:map helm-find-files-map + "C-w" #'helm-find-files-up-one-level + "TAB" #'helm-execute-persistent-action)) + + (:after helm-ag + (:map helm-ag-map + "<backtab>" #'helm-ag-edit))) + + ;; hl-todo + :m "]t" #'hl-todo-next + :m "[t" #'hl-todo-previous + + ;; ivy + (:after ivy + :map ivy-minibuffer-map + [escape] #'keyboard-escape-quit + "C-SPC" #'ivy-call-and-recenter + "TAB" #'ivy-partial + "M-v" #'yank + "M-z" #'undo + "C-r" #'evil-paste-from-register + "C-k" #'ivy-previous-line + "C-j" #'ivy-next-line + "C-l" #'ivy-alt-done + "C-w" #'ivy-backward-kill-word + "C-u" #'ivy-kill-line + "C-b" #'backward-word + "C-f" #'forward-word) + + ;; neotree + (:after neotree + :map neotree-mode-map + :n "g" nil + :n [tab] #'neotree-quick-look + :n "RET" #'neotree-enter + :n [backspace] #'evil-window-prev + :n "c" #'neotree-create-node + :n "r" #'neotree-rename-node + :n "d" #'neotree-delete-node + :n "j" #'neotree-next-line + :n "k" #'neotree-previous-line + :n "n" #'neotree-next-line + :n "p" #'neotree-previous-line + :n "h" #'+neotree/collapse-or-up + :n "l" #'+neotree/expand-or-open + :n "J" #'neotree-select-next-sibling-node + :n "K" #'neotree-select-previous-sibling-node + :n "H" #'neotree-select-up-node + :n "L" #'neotree-select-down-node + :n "G" #'evil-goto-line + :n "gg" #'evil-goto-first-line + :n "v" #'neotree-enter-vertical-split + :n "s" #'neotree-enter-horizontal-split + :n "q" #'neotree-hide + :n "R" #'neotree-refresh) + + ;; realgud + (:after realgud + :map realgud:shortkey-mode-map + :n "j" #'evil-next-line + :n "k" #'evil-previous-line + :n "h" #'evil-backward-char + :n "l" #'evil-forward-char + :m "n" #'realgud:cmd-next + :m "b" #'realgud:cmd-break + :m "B" #'realgud:cmd-clear + :n "c" #'realgud:cmd-continue) + + ;; rotate-text + :n "gs" #'rotate-text + + ;; smart-forward + :m "g]" #'smart-forward + :m "g[" #'smart-backward + + ;; undo-tree -- undo/redo for visual regions + :v "C-u" #'undo-tree-undo + :v "C-r" #'undo-tree-redo + + ;; yasnippet + (:after yasnippet + (:map yas-keymap + "C-e" #'+snippets/goto-end-of-field + "C-a" #'+snippets/goto-start-of-field + "<M-right>" #'+snippets/goto-end-of-field + "<M-left>" #'+snippets/goto-start-of-field + "<M-backspace>" #'+snippets/delete-to-start-of-field + [escape] #'evil-normal-state + [backspace] #'+snippets/delete-backward-char + [delete] #'+snippets/delete-forward-char-or-field) + (:map yas-minor-mode-map + :i "<tab>" yas-maybe-expand + :v "<tab>" #'+snippets/expand-on-region)) + + + ;; --- Major mode bindings -------------------------- + + ;; Markdown + (:after markdown-mode + (:map markdown-mode-map + ;; fix conflicts with private bindings + "<backspace>" nil + "<M-left>" nil + "<M-right>" nil)) + + ;; Rust + (:after rust + (:map rust-mode-map + "K" #'racer-describe + "g RET" #'cargo-process-test)) + + ;; Elixir + (:after alchemist + (:map elixir-mode-map + :n "K" #'alchemist-help-search-at-point + :n "g RET" #'alchemist-project-run-tests-for-current-file + :n "g \\" #'alchemist-mix-test-at-point + :n "g SPC" #'alchemist-mix-compile)) + + ;; Haskell + (:after haskell-mode + (:map haskell-mode-map + ;; :n "K" #'intero-info + :n "K" #'lsp-describe-thing-at-point + ;; :n "g d" #'lsp-ui-peek-find-definitions + :n "g d" #'lsp-ui-peek-find-definitions + :n "g R" #'lsp-find-references + ;; :n "g SPC" #'intero-repl-load + ;; :n "g y" #'lsp-ui- + )) + + ;; Javascript + ;; (:after rjsx-mode + ;; (:map rjsx-mode-map + ;; :n "g d" #'flow-minor-jump-to-definition + ;; :n "K" #'flow-minor-type-at-pos)) + + (:after js2-mode + (:map js2-mode-map + :n "g d" #'flow-minor-jump-to-definition + :n "K" #'flow-minor-type-at-pos)) + + ;; Elisp + (:map emacs-lisp-mode-map + :n "g SPC" #'eval-buffer + :n "g RET" (λ! () (ert t))) + + + ;; --- Custom evil text-objects --------------------- + :textobj "a" #'evil-inner-arg #'evil-outer-arg + :textobj "B" #'evil-textobj-anyblock-inner-block #'evil-textobj-anyblock-a-block + :textobj "i" #'evil-indent-plus-i-indent #'evil-indent-plus-a-indent + :textobj "I" #'evil-indent-plus-i-indent-up #'evil-indent-plus-a-indent-up + :textobj "J" #'evil-indent-plus-i-indent-up-down #'evil-indent-plus-a-indent-up-down + + + ;; --- Built-in plugins ----------------------------- + (:after comint + ;; TAB auto-completion in term buffers + :map comint-mode-map [tab] #'company-complete) + + (:after debug + ;; For elisp debugging + :map debugger-mode-map + :n "RET" #'debug-help-follow + :n "e" #'debugger-eval-expression + :n "n" #'debugger-step-through + :n "c" #'debugger-continue) + + (:map help-mode-map + :n "[[" #'help-go-back + :n "]]" #'help-go-forward + :n "o" #'ace-link-help + :n "q" #'quit-window + :n "Q" #'+ivy-quit-and-resume) + + (:after vc-annotate + :map vc-annotate-mode-map + :n "q" #'kill-this-buffer + :n "d" #'vc-annotate-show-diff-revision-at-line + :n "D" #'vc-annotate-show-changeset-diff-revision-at-line + :n "SPC" #'vc-annotate-show-log-revision-at-line + :n "]]" #'vc-annotate-next-revision + :n "[[" #'vc-annotate-prev-revision + :n "TAB" #'vc-annotate-toggle-annotation-visibility + :n "RET" #'vc-annotate-find-revision-at-line)) + +;; evil-easymotion +(after! evil-easymotion + (let ((prefix (concat doom-leader-key " /"))) + ;; NOTE `evilem-default-keybinds' unsets all other keys on the prefix (in + ;; motion state) + (evilem-default-keybindings prefix) + (evilem-define (kbd (concat prefix " n")) #'evil-ex-search-next) + (evilem-define (kbd (concat prefix " N")) #'evil-ex-search-previous) + (evilem-define (kbd (concat prefix " s")) #'evil-snipe-repeat + :pre-hook (save-excursion (call-interactively #'evil-snipe-s)) + :bind ((evil-snipe-scope 'buffer) + (evil-snipe-enable-highlight) + (evil-snipe-enable-incremental-highlight))) + (evilem-define (kbd (concat prefix " S")) #'evil-snipe-repeat-reverse + :pre-hook (save-excursion (call-interactively #'evil-snipe-s)) + :bind ((evil-snipe-scope 'buffer) + (evil-snipe-enable-highlight) + (evil-snipe-enable-incremental-highlight))))) + + +;; +;; Keybinding fixes +;; + +;; This section is dedicated to "fixing" certain keys so that they behave +;; properly, more like vim, or how I like it. + +(map! (:map input-decode-map + [S-iso-lefttab] [backtab] + (:unless window-system "TAB" [tab])) ; Fix TAB in terminal + + ;; I want C-a and C-e to be a little smarter. C-a will jump to + ;; indentation. Pressing it again will send you to the true bol. Same goes + ;; for C-e, except it will ignore comments and trailing whitespace before + ;; jumping to eol. + :i "C-a" #'doom/backward-to-bol-or-indent + :i "C-e" #'doom/forward-to-last-non-comment-or-eol + :i "C-u" #'doom/backward-kill-to-bol-and-indent + + ;; Emacsien motions for insert mode + :i "C-b" #'backward-word + :i "C-f" #'forward-word + + ;; Highjacks space/backspace to: + ;; a) balance spaces inside brackets/parentheses ( | ) -> (|) + ;; b) delete space-indented blocks intelligently + ;; c) do none of this when inside a string + ;; :i "SPC" #'doom/inflate-space-maybe + ;; :i [remap delete-backward-char] #'doom/deflate-space-maybe + ;; :i [remap newline] #'doom/newline-and-indent + + (:map org-mode-map + :i [remap doom/inflate-space-maybe] #'org-self-insert-command + "C-c C-x C-i" #'org-clock-in + "C-c C-x <C-i>" #'org-clock-in) + + (:map org-agenda-mode-map + "C-c C-x C-i" #'org-agenda-clock-in + "C-c C-x <C-i>" #'org-agenda-clock-in) + + ;; Restore common editing keys (and ESC) in minibuffer + (:map (minibuffer-local-map + minibuffer-local-ns-map + minibuffer-local-completion-map + minibuffer-local-must-match-map + minibuffer-local-isearch-map + evil-ex-completion-map + evil-ex-search-keymap + read-expression-map) + ;; [escape] #'abort-recursive-edit + "C-r" #'evil-paste-from-register + "C-a" #'move-beginning-of-line + "C-w" #'doom/minibuffer-kill-word + "C-u" #'doom/minibuffer-kill-line + "C-b" #'backward-word + "C-f" #'forward-word + "M-z" #'doom/minibuffer-undo) + + (:map messages-buffer-mode-map + "M-;" #'eval-expression + "A-;" #'eval-expression) + + (:map tabulated-list-mode-map + [remap evil-record-macro] #'doom/popup-close-maybe) + + (:after view + (:map view-mode-map "<escape>" #'View-quit-all))) + +(defun +sexp-transpose () + (interactive) + (case evil-this-operator + ('evil-shift-right (paxedit-transpose-forward)) + ('evil-shift-left (paxedit-transpose-backward)))) + +;; (defun nmap (&rest keys-and-ops) +;; (->> +;; (seq-partition keys-and-ops 2) +;; (seq-map +;; (lambda (k-op) +;; (let* ((k (car k-op)) +;; (op (cadr k-op)) +;; (prefix (substring k 0 1)) +;; (prefix-sym (lookup-key evil-normal-state-map prefix)) +;; (keyseq (substring k 1))) +;; (list keyseq prefix-sym op)))) +;; (seq-group-by #'car) +;; (seq-map +;; (lambda (k-ops) +;; (let* ((keyseq (car k-ops)) +;; (ops (cdr k-ops)) +;; (existing-binding (lookup-key evil-operator-state-map keyseq)) +;; (handler (λ! () +;; (if-let +;; ((oplist +;; (seq-find (lambda (op) +;; (equal (nth 1 op) +;; evil-this-operator)) +;; ops))) +;; (message "calling oplist") +;; (->> oplist (nth 2) funcall) +;; (when existing-binding +;; (funcall existing-binding)))))) +;; (if existing-binding +;; (progn +;; (define-key evil-operator-state-map +;; (vector 'remap existing-binding) +;; handler) +;; (define-key evil-motion-state-map +;; (vector 'remap existing-binding) +;; handler)) +;; (define-key evil-operator-state-map keyseq handler))))))) + +;; (nmap +;; ">e" #'paxedit-transpose-forward +;; "<e" #'paxedit-transpose-backward) + +(require 'paxedit) +(require 'general) +(general-evil-setup t) + +(nmap + ">" (general-key-dispatch 'evil-shift-right + "e" 'paxedit-transpose-forward + ")" 'sp-forward-slurp-sexp + "(" 'sp-backward-barf-sexp + "I" 'grfn/insert-at-sexp-end + ;; "a" 'grfn/insert-at-form-end + )) + +(nmap + "<" (general-key-dispatch 'evil-shift-left + "e" 'paxedit-transpose-backward + ")" 'sp-forward-barf-sexp + "(" 'sp-backward-slurp-sexp + "I" 'grfn/insert-at-sexp-start + ;; "a" 'grfn/insert-at-form-start + )) + + +(defmacro saving-excursion (&rest body) + `(λ! () (save-excursion ,@body))) + +(nmap "c" (general-key-dispatch 'evil-change + "r c" (saving-excursion (string-inflection-lower-camelcase)) + "r C" (saving-excursion (string-inflection-camelcase)) + "r m" (saving-excursion (string-inflection-camelcase)) + "r s" (saving-excursion (string-inflection-underscore)) + "r u" (saving-excursion (string-inflection-upcase)) + "r -" (saving-excursion (string-inflection-kebab-case)) + "r k" (saving-excursion (string-inflection-kebab-case)) + ;; "r ." (saving-excursion (string-inflection-dot-case)) + ;; "r ." (saving-excursion (string-inflection-space-case)) + ;; "r ." (saving-excursion (string-inflection-title-case)) + )) + + +(predd-defmulti eval-sexp (lambda (form) major-mode)) + +(predd-defmethod eval-sexp 'clojure-mode (form) + (cider-interactive-eval form)) + +(predd-defmethod eval-sexp 'emacs-lisp-mode (form) + (pp-eval-expression form)) + +(predd-defmulti eval-sexp-region (lambda (_beg _end) major-mode)) + +(predd-defmethod eval-sexp-region 'clojure-mode (beg end) + (cider-interactive-eval nil nil (list beg end))) + +(predd-defmethod eval-sexp-region 'emacs-lisp-mode (beg end) + (pp-eval-expression (read (buffer-substring beg end)))) + +(predd-defmulti eval-sexp-region-context (lambda (_beg _end _context) major-mode)) + +(predd-defmethod eval-sexp-region-context 'clojure-mode (beg end context) + (cider--eval-in-context (buffer-substring beg end))) + +(defun pp-eval-context-region (beg end context) + (interactive "r\nxContext: ") + (let* ((inner-expr (read (buffer-substring beg end))) + (full-expr (list 'let* context inner-expr))) + (pp-eval-expression full-expr))) + +(predd-defmethod eval-sexp-region-context 'emacs-lisp-mode (beg end context) + (pp-eval-context-region beg end context)) + +(predd-defmulti preceding-sexp (lambda () major-mode)) + +(predd-defmethod preceding-sexp 'clojure-mode () + (cider-last-sexp)) + +(predd-defmethod preceding-sexp 'emacs-lisp-mode () + (elisp--preceding-sexp)) + +(defun eval-sexp-at-point () + (interactive) + (let ((bounds (bounds-of-thing-at-point 'sexp))) + (eval-sexp-region (car bounds) + (cdr bounds)))) + +(defun eval-last-sexp (_) + (interactive) + (eval-sexp (preceding-sexp))) + +;;; + +(defun cider-insert-current-sexp-in-repl (&optional arg) + "Insert the expression at point in the REPL buffer. +If invoked with a prefix ARG eval the expression after inserting it" + (interactive "P") + (cider-insert-in-repl (cider-sexp-at-point) arg)) + +(evil-define-operator fireplace-send (beg end) + (cider-insert-current-sexp-in-repl nil nil (list beg end))) + +(defun +clojure-pprint-expr (form) + (format "(with-out-str (clojure.pprint/pprint %s))" + form)) + +(defun cider-eval-read-and-print-handler (&optional buffer) + "Make a handler for evaluating and reading then printing result in BUFFER." + (nrepl-make-response-handler + (or buffer (current-buffer)) + (lambda (buffer value) + (let ((value* (read value))) + (with-current-buffer buffer + (insert + (if (derived-mode-p 'cider-clojure-interaction-mode) + (format "\n%s\n" value*) + value*))))) + (lambda (_buffer out) (cider-emit-interactive-eval-output out)) + (lambda (_buffer err) (cider-emit-interactive-eval-err-output err)) + '())) + +(defun cider-eval-and-replace (beg end) + "Evaluate the expression in region and replace it with its result" + (interactive "r") + (let ((form (buffer-substring beg end))) + (cider-nrepl-sync-request:eval form) + (kill-region beg end) + (cider-interactive-eval + (+clojure-pprint-expr form) + (cider-eval-read-and-print-handler)))) + +(defun cider-eval-current-sexp-and-replace () + "Evaluate the expression at point and replace it with its result" + (interactive) + (apply #'cider-eval-and-replace (cider-sexp-at-point 'bounds))) + +;;; + +(evil-define-operator fireplace-eval (beg end) + (eval-sexp-region beg end)) + +(evil-define-operator fireplace-replace (beg end) + (cider-eval-and-replace beg end)) + +(evil-define-operator fireplace-eval-context (beg end) + (eval-sexp-region-context beg end)) + +;;; fireplace-esque eval binding +(nmap :keymaps 'cider-mode-map + "c" (general-key-dispatch 'evil-change + "p" (general-key-dispatch 'fireplace-eval + "p" 'cider-eval-sexp-at-point + "c" 'cider-eval-last-sexp + "d" 'cider-eval-defun-at-point + "r" 'cider-test-run-test) + "q" (general-key-dispatch 'fireplace-send + "q" 'cider-insert-current-sexp-in-repl + "c" 'cider-insert-last-sexp-in-repl) + "x" (general-key-dispatch 'fireplace-eval-context + "x" 'cider-eval-sexp-at-point-in-context + "c" 'cider-eval-last-sexp-in-context) + "!" (general-key-dispatch 'fireplace-replace + "!" 'cider-eval-current-sexp-and-replace + "c" 'cider-eval-last-sexp-and-replace) + "y" 'cider-copy-last-result)) + +;;; + +(nmap :keymaps 'emacs-lisp-mode-map + "c" (general-key-dispatch 'evil-change + "p" (general-key-dispatch 'fireplace-eval + "p" 'eval-sexp-at-point + "c" 'eval-last-sexp + "d" 'eval-defun + "r" 'cider-test-run-test) + "x" (general-key-dispatch 'fireplace-eval-context + "x" 'cider-eval-sexp-at-point-in-context + "c" 'cider-eval-last-sexp-in-context) + "!" (general-key-dispatch 'fireplace-replace + "!" 'cider-eval-current-sexp-and-replace + "c" 'cider-eval-last-sexp-and-replace) + "y" 'cider-copy-last-result)) + +(nmap :keymaps 'sly-mode-map + "c" (general-key-dispatch 'evil-change + "p" (general-key-dispatch 'sly-eval + ;; "p" 'eval-sexp-at-point + "c" 'sly-eval-last-expression + "d" 'sly-eval-defun + ;; "r" 'cider-test-run-test + ) + ;; "x" (general-key-dispatch 'fireplace-eval-context + ;; "x" 'cider-eval-sexp-at-point-in-context + ;; "c" 'cider-eval-last-sexp-in-context + ;; ) + ;; "!" (general-key-dispatch 'fireplace-replace + ;; "!" 'cider-eval-current-sexp-and-replace + ;; "c" 'cider-eval-last-sexp-and-replace) + ;; "y" 'cider-copy-last-result + )) + + +;; >) ; slurp forward +;; <) ; barf forward +;; <( ; slurp backward +;; >( ; slurp backward + +;; (require 'doom-themes) +(defun grfn/haskell-test-file-p () + (string-match-p (rx (and "Spec.hs" eol)) + (buffer-file-name))) + +(require 'haskell) + +(defun grfn/intero-run-main () + (interactive) + (intero-repl-load) + (intero-with-repl-buffer nil + (comint-simple-send + (get-buffer-process (current-buffer)) + "main"))) + +(defun grfn/run-clj-or-cljs-test () + (interactive) + (message "Running tests...") + (cl-case (cider-repl-type-for-buffer) + ('cljs + (cider-interactive-eval + "(with-out-str (cljs.test/run-tests))" + (nrepl-make-response-handler + (current-buffer) + (lambda (_ value) + (with-output-to-temp-buffer "*cljs-test-results*" + (print + (->> value + (s-replace "\"" "") + (s-replace "\\n" "\n"))))) + nil nil nil))) + (('clj 'multi) + (funcall-interactively + #'cider-test-run-ns-tests + nil)))) + +(defun cider-copy-last-result () + (interactive) + (cider-interactive-eval + "*1" + (nrepl-make-response-handler + (current-buffer) + (lambda (_ value) + (kill-new value) + (message "Copied last result (%s) to clipboard" + (if (= (length value) 1) "1 char" + (format "%d chars" (length value))))) + nil nil nil))) + + +(defun grfn/insert-new-src-block () + (interactive) + (let* ((current-src-block (org-element-at-point)) + (src-block-head (save-excursion + (goto-char (org-element-property + :begin current-src-block)) + (let ((line (thing-at-point 'line t))) + (if (not (s-starts-with? "#+NAME:" (s-trim line))) + line + (forward-line) + (thing-at-point 'line t))))) + (point-to-insert + (if-let (results-loc (org-babel-where-is-src-block-result)) + (save-excursion + (goto-char results-loc) + (org-element-property + :end + (org-element-at-point))) + (org-element-property :end (org-element-at-point))))) + (goto-char point-to-insert) + (insert "\n") + (insert src-block-head) + (let ((contents (point-marker))) + (insert "\n#+END_SRC\n") + (goto-char contents)))) + +(defun grfn/+org-insert-item (orig direction) + (interactive) + (if (and (org-in-src-block-p) + (equal direction 'below)) + (grfn/insert-new-src-block) + (funcall orig direction))) + +(advice-add #'+org--insert-item :around #'grfn/+org-insert-item) +;; (advice-add #'+org/insert-item-below :around +;; (lambda (orig) (grfn/+org-insert-item orig 'below))) + +(defun set-pdb-trace () + (interactive) + (end-of-line) + (insert (format "\n%simport pdb;pdb.set_trace()" + (make-string (python-indent-calculate-indentation) + ?\s))) + (evil-indent (line-beginning-position) + (line-end-position))) + +(map! + + (:map magit-mode-map + :n "#" 'forge-dispatch) + + (:map haskell-mode-map + :n "K" 'lsp-info-under-point + :n "g d" 'lsp-ui-peek-find-definitions + :n "g r" 'lsp-ui-peek-find-references + :n "g \\" '+haskell/repl + ;; :n "K" 'intero-info + ;; :n "g d" 'intero-goto-definition + ;; :n "g SPC" 'intero-repl-load + ;; :n "g \\" 'intero-repl + ;; :n "g y" 'intero-type-at + ;; :n "g RET" 'grfn/run-sputnik-test-for-file + + (:localleader + :desc "Apply action" :n "e" 'intero-repl-eval-region + :desc "Rename symbol" :n "r" 'intero-apply-suggestions)) + + (:map python-mode-map + :n "K" #'anaconda-mode-show-doc + :n "g SPC" #'+eval/buffer + :n "g RET" #'python-pytest-file + :n "g \\" #'+python/open-ipython-repl + [remap evil-commentary-yank] #'set-pdb-trace) + + (:after agda2-mode + (:map agda2-mode-map + :n "g SPC" 'agda2-load + :n "g d" 'agda2-goto-definition-keyboard + :n "] g" 'agda2-next-goal + :n "[ g" 'agda2-previous-goal + + (:localleader + :desc "Give" :n "SPC" 'agda2-give + :desc "Case Split" :n "c" 'agda2-make-case + :desc "Make Helper" :n "h" 'agda2-helper-function-type + :desc "Refine" :n "r" 'agda2-refine + :desc "Auto" :n "a" 'agda2-auto-maybe-all + :desc "Goal type and context" :n "t" 'agda2-goal-and-context + :desc "Goal type and context and inferred" :n ";" 'agda2-goal-and-context-and-inferred))) + + (:after clojure-mode + (:map clojure-mode-map + :n "] f" 'forward-sexp + :n "[ f" 'backward-sexp)) + + (:after cider-mode + (:map cider-mode-map + :n "g SPC" 'cider-eval-buffer + :n "g \\" 'cider-switch-to-repl-buffer + :n "K" 'cider-doc + :n "g K" 'cider-grimoire + :n "g d" 'cider-find-dwim + :n "C-w ]" 'cider-find-dwim-other-window + ;; :n "g RET" 'cider-test-run-ns-tests + :n "g RET" 'grfn/run-clj-or-cljs-test + + "C-c C-r r" 'cljr-add-require-to-ns + "C-c C-r i" 'cljr-add-import-to-ns + + (:localleader + ;; :desc "Inspect last result" :n "i" 'cider-inspect-last-result + ;; :desc "Search for documentation" :n "h s" 'cider-apropos-doc + :desc "Add require to ns" :n "n r" 'cljr-add-require-to-ns + :desc "Add import to ns" :n "n i" 'cljr-add-import-to-ns)) + (:map cider-repl-mode-map + :n "g \\" 'cider-switch-to-last-clojure-buffer)) + + (:after w3m + (:map w3m-mode-map + "/" #'evil-search-forward + "?" #'evil-search-backward + "r" #'w3m-reload-this-page)) + + (:after slack + (:map slack-message-buffer-mode-map + :i "<up>" #'slack-message-edit)) + + (:after org + :n "C-c C-x C-o" #'org-clock-out + (:map org-mode-map + [remap counsel-imenu] #'counsel-org-goto + "M-k" #'org-move-subtree-up + "M-j" #'org-move-subtree-down + (:localleader + :n "g" #'counsel-org-goto)) + + (:map org-capture-mode-map + :n "g RET" #'org-capture-finalize + :n "g \\" #'org-captue-refile)) + + (:map lsp-mode-map + :n "K" #'lsp-describe-thing-at-point + :n "g r" #'lsp-rename + (:localleader + :n "a" #'lsp-execute-code-action)) + + ) diff --git a/users/grfn/emacs.d/+commands.el b/users/grfn/emacs.d/+commands.el new file mode 100644 index 000000000000..518f185cb9c2 --- /dev/null +++ b/users/grfn/emacs.d/+commands.el @@ -0,0 +1,149 @@ +;; -*- lexical-binding: t; -*- + +(defalias 'ex! 'evil-ex-define-cmd) + +(defun delete-file-and-buffer () + "Kill the current buffer and deletes the file it is visiting." + (interactive) + (let ((filename (buffer-file-name))) + (when filename + (if (vc-backend filename) + (vc-delete-file filename) + (progn + (delete-file filename) + (message "Deleted file %s" filename) + (kill-buffer)))))) + +;;; Commands defined elsewhere +;;(ex! "al[ign]" #'+evil:align) +;;(ex! "g[lobal]" #'+evil:global) + +;;; Custom commands +;; Editing +(ex! "@" #'+evil:macro-on-all-lines) ; TODO Test me +(ex! "al[ign]" #'+evil:align) +(ex! "enhtml" #'+web:encode-html-entities) +(ex! "dehtml" #'+web:decode-html-entities) +(ex! "mc" #'+evil:mc) +(ex! "iedit" #'evil-multiedit-ex-match) +(ex! "na[rrow]" #'+evil:narrow-buffer) +(ex! "retab" #'+evil:retab) + +(ex! "glog" #'magit-log-buffer-file) + +;; External resources +;; TODO (ex! "db" #'doom:db) +;; TODO (ex! "dbu[se]" #'doom:db-select) +;; TODO (ex! "go[ogle]" #'doom:google-search) +(ex! "lo[okup]" #'+jump:online) +(ex! "dash" #'+lookup:dash) +(ex! "dd" #'+lookup:devdocs) +(ex! "http" #'httpd-start) ; start http server +(ex! "repl" #'+eval:repl) ; invoke or send to repl +;; TODO (ex! "rx" 'doom:regex) ; open re-builder +(ex! "sh[ell]" #'+eshell:run) +(ex! "t[mux]" #'+tmux:run) ; send to tmux +(ex! "tcd" #'+tmux:cd-here) ; cd to default-directory in tmux +(ex! "x" #'doom/open-project-scratch-buffer) + +;; GIT +(ex! "gist" #'+gist:send) ; send current buffer/region to gist +(ex! "gistl" #'+gist:list) ; list gists by user +(ex! "gbrowse" #'+vcs/git-browse) ; show file in github/gitlab +(ex! "gissues" #'+vcs/git-browse-issues) ; show github issues +(ex! "git" #'magit-status) ; open magit status window +(ex! "gstage" #'magit-stage) +(ex! "gunstage" #'magit-unstage) +(ex! "gblame" #'magit-blame) +(ex! "grevert" #'git-gutter:revert-hunk) + +;; Dealing with buffers +(ex! "clean[up]" #'doom/cleanup-buffers) +(ex! "k[ill]" #'doom/kill-this-buffer) +(ex! "k[ill]all" #'+hlissner:kill-all-buffers) +(ex! "k[ill]m" #'+hlissner:kill-matching-buffers) +(ex! "k[ill]o" #'doom/kill-other-buffers) +(ex! "l[ast]" #'doom/popup-restore) +(ex! "m[sg]" #'view-echo-area-messages) +(ex! "pop[up]" #'doom/popup-this-buffer) + +;; Project navigation +(ex! "a" #'projectile-toggle-between-implementation-and-test) +(ex! "as" #'projectile-find-implementation-or-test-other-window) +(ex! "av" #'projectile-find-implementation-or-test-other-window) +(ex! "cd" #'+hlissner:cd) +(cond ((featurep! :completion ivy) + (ex! "ag" #'+ivy:ag) + (ex! "agc[wd]" #'+ivy:ag-cwd) + (ex! "rg" #'+ivy:rg) + (ex! "rgc[wd]" #'+ivy:rg-cwd) + (ex! "sw[iper]" #'+ivy:swiper) + (ex! "todo" #'+ivy:todo)) + ((featurep! :completion helm) + (ex! "ag" #'+helm:ag) + (ex! "agc[wd]" #'+helm:ag-cwd) + (ex! "rg" #'+helm:rg) + (ex! "rgc[wd]" #'+helm:rg-cwd) + (ex! "sw[oop]" #'+helm:swoop) + (ex! "todo" #'+helm:todo))) + +;; Project tools +(ex! "build" #'+eval/build) +(ex! "debug" #'+debug/run) +(ex! "er[rors]" #'flycheck-list-errors) + +;; File operations +(ex! "cp" #'+evil:copy-this-file) +(ex! "mv" #'+evil:move-this-file) +(ex! "rm" #'+evil:delete-this-file) + +;; Sessions/tabs +(ex! "sclear" #'+workspace/kill-session) +(ex! "sl[oad]" #'+workspace:load-session) +(ex! "ss[ave]" #'+workspace:save-session) +(ex! "tabcl[ose]" #'+workspace:delete) +(ex! "tabclear" #'doom/kill-all-buffers) +(ex! "tabl[ast]" #'+workspace/switch-to-last) +(ex! "tabload" #'+workspace:load) +(ex! "tabn[ew]" #'+workspace:new) +(ex! "tabn[ext]" #'+workspace:switch-next) +(ex! "tabp[rev]" #'+workspace:switch-previous) +(ex! "tabr[ename]" #'+workspace:rename) +(ex! "tabs" #'+workspace/display) +(ex! "tabsave" #'+workspace:save) + +(ex! "scr[atch]" #'cider-scratch) + +;; Org-mode +(ex! "cap" #'+org-capture/dwim) + +(evil-define-command evil-alembic-revision (args) + (interactive "<a>") + (apply + #'generate-alembic-migration + (read-string "Message: ") + (s-split "\\s+" (or args "")))) +(ex! "arev[ision]" #'evil-alembic-revision) + +(evil-define-command evil-alembic-upgrade (&optional revision) + (interactive "<a>") + (alembic-upgrade (or revision "head"))) + +(ex! "aup[grade]" #'evil-alembic-upgrade) + +(evil-define-command evil-alembic-downgrade (&optional revision) + (interactive "<a>") + (alembic-downgrade revision)) + +(ex! "adown[grade]" #'evil-alembic-downgrade) + +(evil-define-command evil-alembic (args) + (interactive "<a>") + (run-alembic args)) + +(ex! "alemb[ic]" #'evil-alembic) + +;; Elixir +(add-hook! elixir-mode + (ex! "AV" #'alchemist-project-toggle-file-and-tests-other-window) + (ex! "A" #'alchemist-project-toggle-file-and-tests)) diff --git a/users/grfn/emacs.d/+private.el.gpg b/users/grfn/emacs.d/+private.el.gpg new file mode 100644 index 000000000000..6273c67d6ecd --- /dev/null +++ b/users/grfn/emacs.d/+private.el.gpg Binary files differdiff --git a/users/grfn/emacs.d/.gitignore b/users/grfn/emacs.d/.gitignore new file mode 100644 index 000000000000..1fd0e3988771 --- /dev/null +++ b/users/grfn/emacs.d/.gitignore @@ -0,0 +1,2 @@ +.authinfo.gpg ++private.el diff --git a/users/grfn/emacs.d/autoload/evil.el b/users/grfn/emacs.d/autoload/evil.el new file mode 100644 index 000000000000..319c93c05e47 --- /dev/null +++ b/users/grfn/emacs.d/autoload/evil.el @@ -0,0 +1,37 @@ +;;; /autoload/evil.el -*- lexical-binding: t; -*- +;;;###if (featurep! :feature evil) + +;;;###autoload (autoload '+hlissner:multi-next-line "/autoload/evil" nil t) +(evil-define-motion +hlissner:multi-next-line (count) + "Move down 6 lines." + :type line + (let ((line-move-visual (or visual-line-mode (derived-mode-p 'text-mode)))) + (evil-line-move (* 6 (or count 1))))) + +;;;###autoload (autoload '+hlissner:multi-previous-line "/autoload/evil" nil t) +(evil-define-motion +hlissner:multi-previous-line (count) + "Move up 6 lines." + :type line + (let ((line-move-visual (or visual-line-mode (derived-mode-p 'text-mode)))) + (evil-line-move (- (* 6 (or count 1)))))) + +;;;###autoload (autoload '+hlissner:cd "/autoload/evil" nil t) +(evil-define-command +hlissner:cd () + "Change `default-directory' with `cd'." + (interactive "<f>") + (cd input)) + +;;;###autoload (autoload '+hlissner:kill-all-buffers "/autoload/evil" nil t) +(evil-define-command +hlissner:kill-all-buffers (&optional bang) + "Kill all buffers. If BANG, kill current session too." + (interactive "<!>") + (if bang + (+workspace/kill-session) + (doom/kill-all-buffers))) + +;;;###autoload (autoload '+hlissner:kill-matching-buffers "/autoload/evil" nil t) +(evil-define-command +hlissner:kill-matching-buffers (&optional bang pattern) + "Kill all buffers matching PATTERN regexp. If BANG, only match project +buffers." + (interactive "<a>") + (doom/kill-matching-buffers pattern bang)) diff --git a/users/grfn/emacs.d/autoload/hlissner.el b/users/grfn/emacs.d/autoload/hlissner.el new file mode 100644 index 000000000000..87b2236d12c7 --- /dev/null +++ b/users/grfn/emacs.d/autoload/hlissner.el @@ -0,0 +1,53 @@ +;;; autoload/hlissner.el -*- lexical-binding: t; -*- + +;;;###autoload +(defun +hlissner/install-snippets () + "Install my snippets from https://github.com/hlissner/emacs-snippets into +private/hlissner/snippets." + (interactive) + (doom-fetch :github "hlissner/emacs-snippets" + (expand-file-name "snippets" (doom-module-path :private 'hlissner)))) + +;;;###autoload +(defun +hlissner/yank-buffer-filename () + "Copy the current buffer's path to the kill ring." + (interactive) + (if-let* ((filename (or buffer-file-name (bound-and-true-p list-buffers-directory)))) + (message (kill-new (abbreviate-file-name filename))) + (error "Couldn't find filename in current buffer"))) + +(defmacro +hlissner-def-finder! (name dir) + "Define a pair of find-file and browse functions." + `(progn + (defun ,(intern (format "+hlissner/find-in-%s" name)) () + (interactive) + (let ((default-directory ,dir) + projectile-project-name + projectile-require-project-root + projectile-cached-buffer-file-name + projectile-cached-project-root) + (call-interactively (command-remapping #'projectile-find-file)))) + (defun ,(intern (format "+hlissner/browse-%s" name)) () + (interactive) + (let ((default-directory ,dir)) + (call-interactively (command-remapping #'find-file)))))) + +;;;###autoload (autoload '+hlissner/find-in-templates "autoload/hlissner" nil t) +;;;###autoload (autoload '+hlissner/browse-templates "autoload/hlissner" nil t) +(+hlissner-def-finder! templates +file-templates-dir) + +;;;###autoload (autoload '+hlissner/find-in-snippets "autoload/hlissner" nil t) +;;;###autoload (autoload '+hlissner/browse-snippets "autoload/hlissner" nil t) +(+hlissner-def-finder! snippets +hlissner-snippets-dir) + +;;;###autoload (autoload '+hlissner/find-in-dotfiles "autoload/hlissner" nil t) +;;;###autoload (autoload '+hlissner/browse-dotfiles "autoload/hlissner" nil t) +(+hlissner-def-finder! dotfiles (expand-file-name ".dotfiles" "~")) + +;;;###autoload (autoload '+hlissner/find-in-emacsd "autoload/hlissner" nil t) +;;;###autoload (autoload '+hlissner/browse-emacsd "autoload/hlissner" nil t) +(+hlissner-def-finder! emacsd doom-emacs-dir) + +;;;###autoload (autoload '+hlissner/find-in-notes "autoload/hlissner" nil t) +;;;###autoload (autoload '+hlissner/browse-notes "autoload/hlissner" nil t) +(+hlissner-def-finder! notes +org-dir) diff --git a/users/grfn/emacs.d/clocked-in-elt.el b/users/grfn/emacs.d/clocked-in-elt.el new file mode 100644 index 000000000000..be4161441dce --- /dev/null +++ b/users/grfn/emacs.d/clocked-in-elt.el @@ -0,0 +1,17 @@ +;;; -*- lexical-binding: t; -*- +(load (expand-file-name "init" (or (getenv "EMACSDIR") + (expand-file-name + "../.emacs.d" + (file-name-directory (file-truename load-file-name)))))) + +(require 'org-clock) +(require 'org-element) + +(let ((item (or org-clock-marker + (car org-clock-history)))) + (when item + (with-current-buffer (marker-buffer item) + (goto-char (marker-position item)) + (let ((element (org-element-at-point))) + (when (eq 'headline (car element)) + (message "%s" (plist-get (cadr element) :raw-value))))))) diff --git a/users/grfn/emacs.d/clojure.el b/users/grfn/emacs.d/clojure.el new file mode 100644 index 000000000000..f001a3e12b68 --- /dev/null +++ b/users/grfn/emacs.d/clojure.el @@ -0,0 +1,53 @@ +;;; -*- lexical-binding: t; -*- + +(defun clojure-thing-at-point-setup () + (interactive) + ;; Used by cider-find-dwim to parse the symbol at point + (setq-local + thing-at-point-file-name-chars + (concat thing-at-point-file-name-chars + "><!?"))) + +(defun +grfn/clojure-setup () + ;; (flycheck-select-checker 'clj-kondo) + (require 'flycheck) + (push 'clojure-cider-kibit flycheck-disabled-checkers) + (push 'clojure-cider-eastwood flycheck-disabled-checkers) + (push 'clojure-cider-typed flycheck-disabled-checkers) + ) + +(after! clojure-mode + (define-clojure-indent + (PUT 2) + (POST 2) + (GET 2) + (PATCH 2) + (DELETE 2) + (context 2) + (checking 3) + (match 1) + (domonad 0) + (describe 1) + (before 1) + (it 2)) + + (add-hook 'clojure-mode-hook #'clojure-thing-at-point-setup) + (add-hook 'clojure-mode-hook #'+grfn/clojure-setup)) + +(use-package! flycheck-clojure + ;; :disabled t + :after (flycheck cider) + :config + (flycheck-clojure-setup)) + +(after! clj-refactor + (setq cljr-magic-requires :prompt + cljr-clojure-test-declaration "[clojure.test :refer :all]" + cljr-cljc-clojure-test-declaration"#?(:clj [clojure.test :refer :all] +:cljs [cljs.test :refer-macros [deftest is testing]])" + ) + (add-to-list + 'cljr-magic-require-namespaces + '("s" . "clojure.spec.alpha"))) + +(set-popup-rule! "^\\*cider-test-report" :size 0.4) diff --git a/users/grfn/emacs.d/company-sql.el b/users/grfn/emacs.d/company-sql.el new file mode 100644 index 000000000000..e623aa2de131 --- /dev/null +++ b/users/grfn/emacs.d/company-sql.el @@ -0,0 +1,299 @@ +;;; Commentary: +;;; TODO + +;;; Code: + +(require 'emacsql) +(require 'emacsql-psql) +(require 'dash) +(require 's) +(require 'cl-lib) + +;;; Config + +(defvar-local company-sql-db-host "localhost" + "Host of the postgresql database to query for autocomplete information") + +(defvar-local company-sql-db-port 5432 + "Port of the postgresql database to query for autocomplete information") + +(defvar-local company-sql-db-user "postgres" + "Username of the postgresql database to query for autocomplete information") + +(defvar-local company-sql-db-name nil + "PostgreSQL database name to query for autocomplete information") + +;;; DB Connection + +(defvar-local company-sql/connection nil) + +(defun company-sql/connect () + (unless company-sql/connection + (setq-local company-sql/connection + (emacsql-psql company-sql-db-name + :hostname company-sql-db-host + :username company-sql-db-user + :port (number-to-string company-sql-db-port)))) + company-sql/connection) + +;;; Utils + +(defmacro comment (&rest _)) + +(defun ->string (x) + (cond + ((stringp x) x) + ((symbolp x) (symbol-name x)))) + +(defun alist-get-equal (key alist) + "Like `alist-get', but uses `equal' instead of `eq' for comparing keys" + (->> alist + (-find (lambda (pair) (equal key (car pair)))) + (cdr))) + +;;; Listing relations + +(cl-defun company-sql/list-tables (conn) + (with-timeout (3) + (-map (-compose 'symbol-name 'car) + (emacsql conn + [:select [tablename] + :from pg_catalog:pg_tables + :where (and (!= schemaname '"information_schema") + (!= schemaname '"pg_catalog"))])))) + +(cl-defun company-sql/list-columns (conn) + (with-timeout (3) + (-map + (lambda (row) + (propertize (symbol-name (nth 0 row)) + 'table-name (nth 1 row) + 'data-type (nth 2 row))) + (emacsql conn + [:select [column_name + table_name + data_type] + :from information_schema:columns])))) + +;;; Keywords + +(defvar company-postgresql/keywords + (list +"a" "abort" "abs" "absent" "absolute" "access" "according" "action" "ada" "add" +"admin" "after" "aggregate" "all" "allocate" "also" "alter" "always" "analyse" +"analyze" "and" "any" "are" "array" "array_agg" "array_max_cardinality" "as" +"asc" "asensitive" "assertion" "assignment" "asymmetric" "at" "atomic" "attach" +"attribute" "attributes" "authorization" "avg" "backward" "base64" "before" +"begin" "begin_frame" "begin_partition" "bernoulli" "between" "bigint" "binary" +"bit" "bit_length" "blob" "blocked" "bom" "boolean" "both" "breadth" "by" "c" +"cache" "call" "called" "cardinality" "cascade" "cascaded" "case" "cast" +"catalog" "catalog_name" "ceil" "ceiling" "chain" "char" "character" +"characteristics" "characters" "character_length" "character_set_catalog" +"character_set_name" "character_set_schema" "char_length" "check" "checkpoint" +"class" "class_origin" "clob" "close" "cluster" "coalesce" "cobol" "collate" +"collation" "collation_catalog" "collation_name" "collation_schema" "collect" +"column" "columns" "column_name" "command_function" "command_function_code" +"comment" "comments" "commit" "committed" "concurrently" "condition" +"condition_number" "configuration" "conflict" "connect" "connection" +"connection_name" "constraint" "constraints" "constraint_catalog" +"constraint_name" "constraint_schema" "constructor" "contains" "content" +"continue" "control" "conversion" "convert" "copy" "corr" "corresponding" "cost" +"count" "covar_pop" "covar_samp" "create" "cross" "csv" "cube" "cume_dist" +"current" "current_catalog" "current_date" "current_default_transform_group" +"current_path" "current_role" "current_row" "current_schema" "current_time" +"current_timestamp" "current_transform_group_for_type" "current_user" "cursor" +"cursor_name" "cycle" "data" "database" "datalink" "date" +"datetime_interval_code" "datetime_interval_precision" "day" "db" "deallocate" +"dec" "decimal" "declare" "default" "defaults" "deferrable" "deferred" "defined" +"definer" "degree" "delete" "delimiter" "delimiters" "dense_rank" "depends" +"depth" "deref" "derived" "desc" "describe" "descriptor" "detach" +"deterministic" "diagnostics" "dictionary" "disable" "discard" "disconnect" +"dispatch" "distinct" "dlnewcopy" "dlpreviouscopy" "dlurlcomplete" +"dlurlcompleteonly" "dlurlcompletewrite" "dlurlpath" "dlurlpathonly" +"dlurlpathwrite" "dlurlscheme" "dlurlserver" "dlvalue" "do" "document" "domain" +"double" "drop" "dynamic" "dynamic_function" "dynamic_function_code" "each" +"element" "else" "empty" "enable" "encoding" "encrypted" "end" "end-exec" +"end_frame" "end_partition" "enforced" "enum" "equals" "escape" "event" "every" +"except" "exception" "exclude" "excluding" "exclusive" "exec" "execute" "exists" +"exp" "explain" "expression" "extension" "external" "extract" "false" "family" +"fetch" "file" "filter" "final" "first" "first_value" "flag" "float" "floor" +"following" "for" "force" "foreign" "fortran" "forward" "found" "frame_row" +"free" "freeze" "from" "fs" "full" "function" "functions" "fusion" "g" "general" +"generated" "get" "global" "go" "goto" "grant" "granted" "greatest" "group" +"grouping" "groups" "handler" "having" "header" "hex" "hierarchy" "hold" "hour" +"id" "identity" "if" "ignore" "ilike" "immediate" "immediately" "immutable" +"implementation" "implicit" "import" "in" "include" "including" "increment" +"indent" "index" "indexes" "indicator" "inherit" "inherits" "initially" "inline" +"inner" "inout" "input" "insensitive" "insert" "instance" "instantiable" +"instead" "int" "integer" "integrity" "intersect" "intersection" "interval" +"into" "invoker" "is" "isnull" "isolation" "join" "k" "key" "key_member" +"key_type" "label" "lag" "language" "large" "last" "last_value" "lateral" "lead" +"leading" "leakproof" "least" "left" "length" "level" "library" "like" +"like_regex" "limit" "link" "listen" "ln" "load" "local" "localtime" +"localtimestamp" "location" "locator" "lock" "locked" "logged" "lower" "m" "map" +"mapping" "match" "matched" "materialized" "max" "maxvalue" "max_cardinality" +"member" "merge" "message_length" "message_octet_length" "message_text" "method" +"min" "minute" "minvalue" "mod" "mode" "modifies" "module" "month" "more" "move" +"multiset" "mumps" "name" "names" "namespace" "national" "natural" "nchar" +"nclob" "nesting" "new" "next" "nfc" "nfd" "nfkc" "nfkd" "nil" "no" "none" +"normalize" "normalized" "not" "nothing" "notify" "notnull" "nowait" "nth_value" +"ntile" "null" "nullable" "nullif" "nulls" "number" "numeric" "object" +"occurrences_regex" "octets" "octet_length" "of" "off" "offset" "oids" "old" +"on" "only" "open" "operator" "option" "options" "or" "order" "ordering" +"ordinality" "others" "out" "outer" "output" "over" "overlaps" "overlay" +"overriding" "owned" "owner" "p" "pad" "parallel" "parameter" "parameter_mode" +"parameter_name" "parameter_ordinal_position" "parameter_specific_catalog" +"parameter_specific_name" "parameter_specific_schema" "parser" "partial" +"partition" "pascal" "passing" "passthrough" "password" "path" "percent" +"percentile_cont" "percentile_disc" "percent_rank" "period" "permission" +"placing" "plans" "pli" "policy" "portion" "position" "position_regex" "power" +"precedes" "preceding" "precision" "prepare" "prepared" "preserve" "primary" +"prior" "privileges" "procedural" "procedure" "procedures" "program" "public" +"publication" "quote" "range" "rank" "read" "reads" "real" "reassign" "recheck" +"recovery" "recursive" "ref" "references" "referencing" "refresh" "regr_avgx" +"regr_avgy" "regr_count" "regr_intercept" "regr_r2" "regr_slope" "regr_sxx" +"regr_sxy" "regr_syy" "reindex" "relative" "release" "rename" "repeatable" +"replace" "replica" "requiring" "reset" "respect" "restart" "restore" "restrict" +"result" "return" "returned_cardinality" "returned_length" +"returned_octet_length" "returned_sqlstate" "returning" "returns" "revoke" +"right" "role" "rollback" "rollup" "routine" "routines" "routine_catalog" +"routine_name" "routine_schema" "row" "rows" "row_count" "row_number" "rule" +"savepoint" "scale" "schema" "schemas" "schema_name" "scope" "scope_catalog" +"scope_name" "scope_schema" "scroll" "search" "second" "section" "security" +"select" "selective" "self" "sensitive" "sequence" "sequences" "serializable" +"server" "server_name" "session" "session_user" "set" "setof" "sets" "share" +"show" "similar" "simple" "size" "skip" "smallint" "snapshot" "some" "source" +"space" "specific" "specifictype" "specific_name" "sql" "sqlcode" "sqlerror" +"sqlexception" "sqlstate" "sqlwarning" "sqrt" "stable" "standalone" "start" +"state" "statement" "static" "statistics" "stddev_pop" "stddev_samp" "stdin" +"stdout" "storage" "strict" "strip" "structure" "style" "subclass_origin" +"submultiset" "subscription" "substring" "substring_regex" "succeeds" "sum" +"symmetric" "sysid" "system" "system_time" "system_user" "t" "table" "tables" +"tablesample" "tablespace" "table_name" "temp" "template" "temporary" "text" +"then" "ties" "time" "timestamp" "timezone_hour" "timezone_minute" "to" "token" +"top_level_count" "trailing" "transaction" "transactions_committed" +"transactions_rolled_back" "transaction_active" "transform" "transforms" +"translate" "translate_regex" "translation" "treat" "trigger" "trigger_catalog" +"trigger_name" "trigger_schema" "trim" "trim_array" "true" "truncate" "trusted" +"type" "types" "uescape" "unbounded" "uncommitted" "under" "unencrypted" "union" +"unique" "unknown" "unlink" "unlisten" "unlogged" "unnamed" "unnest" "until" +"untyped" "update" "upper" "uri" "usage" "user" "user_defined_type_catalog" +"user_defined_type_code" "user_defined_type_name" "user_defined_type_schema" +"using" "vacuum" "valid" "validate" "validator" "value" "values" "value_of" +"varbinary" "varchar" "variadic" "varying" "var_pop" "var_samp" "verbose" +"version" "versioning" "view" "views" "volatile" "when" "whenever" "where" +"whitespace" "width_bucket" "window" "with" "within" "without" "work" "wrapper" +"write" "xml" "xmlagg" "xmlattributes" "xmlbinary" "xmlcast" "xmlcomment" +"xmlconcat" "xmldeclaration" "xmldocument" "xmlelement" "xmlexists" "xmlforest" +"xmliterate" "xmlnamespaces" "xmlparse" "xmlpi" "xmlquery" "xmlroot" "xmlschema" +"xmlserialize" "xmltable" "xmltext" "xmlvalidate" "year" "yes" "zone")) + +;;; Company backend + +(cl-defun company-postgresql/candidates (prefix conn) + (-filter + (apply-partially #'s-starts-with? prefix) + (append (-map (lambda (s) + (propertize s 'company-postgresql-annotation "table")) + + (-map (lambda (s) + (propertize s 'company-postgresql-annotation + (format "%s.%s %s" + (get-text-property 0 'table-name s) + s + (-> + (get-text-property 0 'data-type s) + (->string) + (upcase))))) + (company-sql/list-columns conn)) + (-map (lambda (s) + (propertize s 'company-postgresql-annotation "keyword")) + company-postgresql/keywords))))) + +(defun company-postgresql (command &optional arg &rest _) + (interactive (list 'interactive)) + (cl-case command + (interactive (company-begin-backend 'company-postgresql)) + (init (company-sql/connect)) + (prefix (company-grab-symbol)) + (annotation + (get-text-property 0 'company-postgresql-annotation arg)) + (candidates (company-postgresql/candidates + arg + (company-sql/connect))) + (duplicates t) + (ignore-case t))) + +;;; org-babel company sql + +(defvar-local org-company-sql/connections + ()) + +(defun org-company-sql/connect (conn-params) + (or (alist-get-equal conn-params org-company-sql/connections) + (let ((conn (apply 'emacsql-psql conn-params))) + (add-to-list 'org-company-sql/connections (cons conn-params conn)) + conn))) + +(defun org-company-sql/in-sql-source-block-p () + (let ((org-elt (org-element-at-point))) + (and (eq 'src-block (car org-elt)) + (equal "sql" (plist-get (cadr org-elt) + :language))))) + +(defun org-company-sql/parse-cmdline (cmdline) + (let* ((lexed (s-split (rx (one-or-more blank)) cmdline)) + (go (lambda (state tokens) + (if (null tokens) () + (let ((token (car tokens)) + (tokens (cdr tokens))) + (if (null state) + (if (s-starts-with? "-" token) + (funcall go token tokens) + (cons token (funcall go state tokens))) + (cons (cons state token) ; ("-h" . "localhost") + (funcall go nil tokens))))))) + (opts (funcall go nil lexed))) + opts)) + +(defun org-company-sql/source-block-conn-params () + (let* ((block-info (org-babel-get-src-block-info)) + (params (caddr block-info)) + (cmdline (alist-get :cmdline params)) + (parsed (org-company-sql/parse-cmdline cmdline)) + (opts (-filter #'listp parsed)) + (positional (-filter #'stringp parsed)) + (host (alist-get-equal "-h" opts)) + (port (or (alist-get-equal "-p" opts) + "5432")) + (dbname (or (alist-get-equal "-d" opts) + (car positional))) + (username (or (alist-get-equal "-U" opts) + (cadr positional)))) + (list dbname + :hostname host + :username username + :port port))) + +(defun org-company-sql/connection-for-source-block () + (org-company-sql/connect + (org-company-sql/source-block-conn-params))) + + +(defun company-ob-postgresql (command &optional arg &rest _) + (interactive (list 'interactive)) + (cl-case command + (interactive (company-begin-backend 'company-ob-postgresql)) + (prefix (and (org-company-sql/in-sql-source-block-p) + (company-grab-symbol))) + (annotation (get-text-property 0 'company-postgresql-annotation arg)) + (candidates + (company-postgresql/candidates + arg + (org-company-sql/connection-for-source-block))) + (duplicates t) + (ignore-case t))) + +;;; + +(provide 'company-sql) diff --git a/users/grfn/emacs.d/config.el b/users/grfn/emacs.d/config.el new file mode 100644 index 000000000000..5312fd92b119 --- /dev/null +++ b/users/grfn/emacs.d/config.el @@ -0,0 +1,1083 @@ +;;; -*- lexical-binding: t; -*- + +;; I've swapped these keys on my keyboard +(setq x-super-keysym 'alt + x-alt-keysym 'meta) + +(setq user-mail-address "root@gws.fyi" + user-full-name "Griffin Smith") + +(let ((font-family (pcase system-type + ('darwin "MesloLGSDZ NF") + ('gnu/linux "Meslo LGSDZ Nerd Font")))) + (setq doom-font (font-spec :family font-family :size 14) + doom-big-font (font-spec :family font-family :size 24) + doom-big-font-increment 5 + doom-variable-pitch-font (font-spec :family "DejaVu Sans") + doom-unicode-font (font-spec :family font-family))) + +(require 's) + +(undefine-key! :keymaps 'doom-leader-map "/") + +(load! "utils") +(load! "company-sql") +(load! "show-matching-paren") +(load! "irc") +(load! "github-org") +(load! "org-gcal") +(load! "grid") +(load! "nix") +(load! "email") +(load! "cpp") +(load! "lisp") +(load! "clojure") +(load! "rust") +(load! "terraform") + +(require 'tvl) + +(add-hook! elixir-mode + (require 'flycheck-credo) + (setq flycheck-elixir-credo-strict t) + (flycheck-credo-setup) + + (require 'flycheck-mix) (flycheck-mix-setup) + + (require 'flycheck-dialyxir) (flycheck-dialyxir-setup) + + (flycheck-mode)) + +(setq +solarized-s-base03 "#002b36" + +solarized-s-base02 "#073642" + ;; emphasized content + +solarized-s-base01 "#586e75" + ;; primary content + +solarized-s-base00 "#657b83" + +solarized-s-base0 "#839496" + ;; comments + +solarized-s-base1 "#93a1a1" + ;; background highlight light + +solarized-s-base2 "#eee8d5" + ;; background light + +solarized-s-base3 "#fdf6e3" + + ;; Solarized accented colors + +solarized-yellow "#b58900" + +solarized-orange "#cb4b16" + +solarized-red "#dc322f" + +solarized-magenta "#d33682" + +solarized-violet "#6c71c4" + +solarized-blue "#268bd2" + +solarized-cyan "#2aa198" + +solarized-green "#859900" + + ;; Darker and lighter accented colors + ;; Only use these in exceptional circumstances! + +solarized-yellow-d "#7B6000" + +solarized-yellow-l "#DEB542" + +solarized-orange-d "#8B2C02" + +solarized-orange-l "#F2804F" + +solarized-red-d "#990A1B" + +solarized-red-l "#FF6E64" + +solarized-magenta-d "#93115C" + +solarized-magenta-l "#F771AC" + +solarized-violet-d "#3F4D91" + +solarized-violet-l "#9EA0E5" + +solarized-blue-d "#00629D" + +solarized-blue-l "#69B7F0" + +solarized-cyan-d "#00736F" + +solarized-cyan-l "#69CABF" + +solarized-green-d "#546E00" + +solarized-green-l "#B4C342") + +(defcustom theme-overrides nil + "Association list of override faces to set for different custom themes.") + +(defadvice load-theme (after theme-set-overrides activate) + (dolist (theme-settings theme-overrides) + (let ((theme (car theme-settings)) + (faces (cadr theme-settings))) + (if (member theme custom-enabled-themes) + (progn + (dolist (face faces) + (custom-theme-set-faces theme face))))))) + +(defun alist-set (alist-symbol key value) + "Set VALUE of a KEY in ALIST-SYMBOL." + (set alist-symbol (cons (list key value) (assq-delete-all key (eval alist-symbol))))) + +(comment + (custom-theme-set-faces 'grfn-solarized-light + `(font-lock-doc-face + ((t (:foreground ,+solarized-s-base1))))) + ++solarized-s-base1 +(custom-theme-) + (custom-face-get-current-spec 'font-lock-doc-face) + + ) + +(alist-set 'theme-overrides 'grfn-solarized-light + `((font-lock-doc-face ((t (:foreground ,+solarized-s-base1)))) + (font-lock-preprocessor-face ((t (:foreground ,+solarized-red)))) + (font-lock-keyword-face ((t (:foreground ,+solarized-green :bold nil)))) + (font-lock-builtin-face ((t (:foreground ,+solarized-s-base01 + :bold t)))) + + (elixir-attribute-face ((t (:foreground ,+solarized-blue)))) + (elixir-atom-face ((t (:foreground ,+solarized-cyan)))) + (linum ((t (:background ,+solarized-s-base2 :foreground ,+solarized-s-base1)))) + (line-number ((t (:background ,+solarized-s-base2 :foreground ,+solarized-s-base1)))) + (line-number-current-line ((t (:background ,+solarized-s-base2 :foreground ,+solarized-s-base1)))) + + (haskell-operator-face ((t (:foreground ,+solarized-green)))) + (haskell-keyword-face ((t (:foreground ,+solarized-cyan)))) + + (org-drawer ((t (:foreground ,+solarized-s-base1 + :bold t)))))) + +(setq solarized-use-variable-pitch nil + solarized-scale-org-headlines nil + solarized-use-less-bold t) + +(add-to-list 'custom-theme-load-path "~/.doom.d/themes") +(load-theme 'grfn-solarized-light t) + +(defface haskell-import-face `((t (:foreground ,+solarized-magenta))) "") + +(setq doom-theme 'grfn-solarized-light) +; (setq doom-theme 'doom-solarized-light) + +(add-hook! doom-post-init + (set-face-attribute 'bold nil :weight 'ultra-light) + (set-face-bold 'bold nil) + (enable-theme 'grfn-solarized-light)) + +(defun rx-words (&rest words) + (rx-to-string + `(and symbol-start (group (or ,@words)) symbol-end))) + +(font-lock-add-keywords + 'elixir-mode + `((,(rx-words "def" + "defp" + "test" + "describe" + "property" + "defrecord" + "defmodule" + "defstruct" + "defdelegate" + "defprotocol" + "defimpl" + "use" + "import" + "alias" + "require" + "assert" + "refute" + "assert_raise") + . + 'font-lock-preprocessor-face))) + +(font-lock-add-keywords + 'elixir-mode + `((,(rx-words "def" + "defp" + "test" + "describe" + "property" + "defrecord" + "defmodule" + "defstruct" + "defdelegate" + "use" + "import" + "alias" + "require" + "assert" + "refute" + "assert_raise") + . + 'font-lock-preprocessor-face))) + +(font-lock-add-keywords + 'haskell-mode + `((,(rx-words "import") . 'haskell-import-face))) + +;; (font-lock-add-keywords +;; 'haskell-mode +;; `((,(rx "-- |") . 'haskell-keyword-face))) + + +;; (load-file (let ((coding-system-for-read 'utf-8)) +;; (shell-command-to-string "agda-mode locate"))) + +(defvar +grfn-dir (file-name-directory load-file-name)) +(defvar +grfn-snippets-dir (expand-file-name "snippets/" +grfn-dir)) + +;; +(load! "+bindings") +(load! "+commands") +(load! "cpp") + + +(add-to-list 'load-path "/home/grfn/code/org-tracker") +(require 'org-tracker) +(use-package! org-tracker + :hook (org-mode . org-tracker-mode) + :config + (setq org-tracker-state-alist '(("INBOX" . "Inbox") + ("BACKLOG" . "Backlog") + ("TODO" . "Selected for Development") + ("ACTIVE" . "In Progress") + ("PR" . "Code Review") + ("DONE" . "Done")) + org-tracker-username "griffin@readyset.io" + org-tracker-claim-ticket-on-status-update '("ACTIVE" "PR" "DONE") + org-tracker-create-stories-with-labels 'existing) + + (defun org-tracker-headlines-from-assigned-to-me (level) + (interactive "*nLevel: ") + (funcall-interactively + #'org-tracker-headlines-from-search + level + "assignee = currentUser() and statusCategory = 2"))) + +(load! "+private") + +(require 'dash) + +(use-package! predd) + + +;; +;; Global config +;; + +(setq doom-modeline-buffer-file-name-style 'relative-to-project + doom-modeline-modal-icon nil + doom-modeline-github t) + +;; +;; Modules +;; + +(after! smartparens + ;; Auto-close more conservatively and expand braces on RET + (let ((unless-list '(sp-point-before-word-p + sp-point-after-word-p + sp-point-before-same-p))) + (sp-pair "'" nil :unless unless-list) + (sp-pair "\"" nil :unless unless-list)) + (sp-pair "{" nil :post-handlers '(("||\n[i]" "RET") ("| " " ")) + :unless '(sp-point-before-word-p sp-point-before-same-p)) + (sp-pair "(" nil :post-handlers '(("||\n[i]" "RET") ("| " " ")) + :unless '(sp-point-before-word-p sp-point-before-same-p)) + (sp-pair "[" nil :post-handlers '(("| " " ")) + :unless '(sp-point-before-word-p sp-point-before-same-p))) + +;; feature/snippets +(after! yasnippet + ;; Don't use default snippets, use mine. + (setq yas-snippet-dirs + (append (list '+grfn-snippets-dir) + (delq 'yas-installed-snippets-dir yas-snippet-dirs)))) + +(after! company + (setq company-idle-delay 0.2 + company-minimum-prefix-length 1)) + +(setq doom-modeline-height 12) + + + +;; Should really figure out which of these is correct, eventually + +(setq +solarized-s-base03 "#002b36" + +solarized-s-base02 "#073642" + ;; emphasized content + +solarized-s-base01 "#586e75" + ;; primary content + +solarized-s-base00 "#657b83" + +solarized-s-base0 "#839496" + ;; comments + +solarized-s-base1 "#93a1a1" + ;; background highlight light + +solarized-s-base2 "#eee8d5" + ;; background light + +solarized-s-base3 "#fdf6e3" + + ;; Solarized accented colors + +solarized-yellow "#b58900" + +solarized-orange "#cb4b16" + +solarized-red "#dc322f" + +solarized-magenta "#d33682" + +solarized-violet "#6c71c4" + +solarized-blue "#268bd2" + +solarized-cyan "#2aa198" + +solarized-green "#859900" + + ;; Darker and lighter accented colors + ;; Only use these in exceptional circumstances! + +solarized-yellow-d "#7B6000" + +solarized-yellow-l "#DEB542" + +solarized-orange-d "#8B2C02" + +solarized-orange-l "#F2804F" + +solarized-red-d "#990A1B" + +solarized-red-l "#FF6E64" + +solarized-magenta-d "#93115C" + +solarized-magenta-l "#F771AC" + +solarized-violet-d "#3F4D91" + +solarized-violet-l "#9EA0E5" + +solarized-blue-d "#00629D" + +solarized-blue-l "#69B7F0" + +solarized-cyan-d "#00736F" + +solarized-cyan-l "#69CABF" + +solarized-green-d "#546E00" + +solarized-green-l "#B4C342") + +(set-cursor-color +solarized-s-base02) + +(after! doom-theme + (set-face-foreground 'font-lock-doc-face +solarized-s-base1) + (set-face-foreground 'org-block +solarized-s-base00) + (set-face-foreground 'slack-message-output-header +solarized-s-base01) + (set-face-attribute 'slack-message-output-header nil :underline nil) + (set-face-attribute 'slack-message-output-text nil :height 1.0) + ) + +(after! solarized-theme + (set-face-foreground 'font-lock-doc-face +solarized-s-base1) + (set-face-foreground 'org-block +solarized-s-base00) + + (set-face-foreground 'slack-message-output-header +solarized-s-base01) + (set-face-attribute 'slack-message-output-header nil :underline nil) + (set-face-attribute 'slack-message-output-text nil :height 1.0) + ) + +(after! evil + (setq evil-shift-width 2)) + +(after! org + (load! "org-query") + (load! "org-config")) + +(after! magit + (setq git-commit-summary-max-length 50)) + +(after! ivy + ;; (setq ivy-re-builders-alist + ;; '((t . ivy--regex-fuzzy))) + ) + +(add-hook 'before-save-hook 'delete-trailing-whitespace) + +(after! paxedit + (add-hook! emacs-lisp-mode #'paxedit-mode) + (add-hook! clojure-mode #'paxedit-mode) + (add-hook! common-lisp-mode #'paxedit-mode)) + +(require 'haskell) + +(let ((m-symbols + '(("`mappend`" . "⊕") + ("<>" . "⊕") + ("`elem`" . "∈") + ("`notElem`" . "∉")))) + (dolist (item m-symbols) (add-to-list 'haskell-font-lock-symbols-alist item))) + +(setq haskell-font-lock-symbols t) + + +(add-hook! haskell-mode + ;; (intero-mode) + (lsp-mode) + ;; (flycheck-add-next-checker + ;; 'intero + ;; 'haskell-hlint) + (set-fill-column 80) + (setq evil-shift-width 2)) + +(auth-source-pass-enable) + +(require 'fill-column-indicator) +;;; * Column Marker +(defun sanityinc/fci-enabled-p () (symbol-value 'fci-mode)) + +(defvar sanityinc/fci-mode-suppressed nil) +(make-variable-buffer-local 'sanityinc/fci-mode-suppressed) + +(defadvice popup-create (before suppress-fci-mode activate) + "Suspend fci-mode while popups are visible" + (let ((fci-enabled (sanityinc/fci-enabled-p))) + (when fci-enabled + (setq sanityinc/fci-mode-suppressed fci-enabled) + (turn-off-fci-mode)))) + +(defadvice popup-delete (after restore-fci-mode activate) + "Restore fci-mode when all popups have closed" + (when (and sanityinc/fci-mode-suppressed + (null popup-instances)) + (setq sanityinc/fci-mode-suppressed nil) + (turn-on-fci-mode))) + + +;;; Javascript + +(require 'smartparens) + +(setq js-indent-level 2) + +(require 'prettier-js) +(after! prettier-js + (add-hook! rjsx-mode #'prettier-js-mode) + (add-hook! js2-mode #'prettier-js-mode) + (add-hook! json-mode #'prettier-js-mode) + (add-hook! css-mode #'prettier-js-mode)) + +(remove-hook 'js2-mode-hook 'tide-setup t) + +;; Set this to the mode you use, I use rjsx-mode +(add-hook 'rjsx-mode-hook #'flow/set-flow-executable t) + + +;; Auto-format Haskell on save, with a combination of hindent + brittany + +; (define-minor-mode brittany-haskell-mode +; :init-value nil +; :group 'haskell +; :lighter "Brittany-Haskell" +; :keymap '() +; ) + + +(require 'alert) +(setq alert-default-style 'libnotify) + +;; (setq slack-buffer-function #'switch-to-buffer) + +(setq projectile-test-suffix-function + (lambda (project-type) + (case project-type + ('haskell-stack "Test") + ('npm ".test") + (otherwise (projectile-test-suffix project-type))))) + +(setq projectile-create-missing-test-files 't) + +(after! magit + (map! :map magit-mode-map + ;; :n "] ]" #'magit-section-forward + ;; :n "[ [" #'magit-section-backward + ) + + (define-suffix-command magit-commit-wip () + (interactive) + (magit-commit-create '("-m" "wip"))) + + (transient-append-suffix + #'magit-commit + ["c"] + (list "W" "Commit WIP" #'magit-commit-wip)) + + (define-suffix-command magit-reset-head-back () + (interactive) + (magit-reset-mixed "HEAD~")) + + (define-suffix-command magit-reset-head-previous () + (interactive) + (magit-reset-mixed "HEAD@{1}")) + + (transient-append-suffix + #'magit-reset + ["f"] + (list "b" "Reset HEAD~" #'magit-reset-head-back)) + (transient-append-suffix + #'magit-reset + ["f"] + (list "o" "Reset HEAD@{1}" #'magit-reset-head-previous)) + + (defun magit-read-org-tracker-branch-name () + (when-let ((issue-id (org-tracker-clocked-in-issue-id))) + (let ((desc + (magit-read-string-ns + (format "Issue description (to go after gs/%s/)" + issue-id)))) + (format "gs/%s/%s" issue-id desc)))) + + (defun magit-read-org-tracker-branch-args () + (if-let ((issue-id (org-tracker-clocked-in-issue-id))) + (let ((start-point (magit-read-starting-point + "Create and checkout branch for Tracker issue" + nil + "origin/master"))) + (if (magit-rev-verify start-point) + (when-let ((desc (magit-read-org-tracker-branch-name))) + (list desc start-point)) + (user-error "Not a valid starting point: %s" choice))) + (user-error "No currently clocked-in tracker issue"))) + + (transient-define-suffix magit-checkout-org-tracker-branch (branch start-point) + (interactive (magit-read-org-tracker-branch-args)) + (magit-branch-and-checkout branch start-point)) + + (transient-define-suffix magit-rename-org-tracker-branch (old new) + (interactive + (let ((branch (magit-read-local-branch "Rename branch"))) + (list branch (magit-read-org-tracker-branch-name)))) + (when (and old new) + (magit-branch-rename old new))) + + (transient-append-suffix + #'magit-branch + ["c"] + (list "C" "Checkout Tracker branch" #'magit-checkout-org-tracker-branch)) + (transient-append-suffix + #'magit-branch + ["c"] + (list "M" "Rename branch to Tracker ticket" #'magit-rename-org-tracker-branch))) + +;; (defun grfn/split-window-more-sensibly (&optional window) +;; (let ((window (or window (selected-window)))) +;; (or (and (window-splittable-p window) +;; ;; Split window vertically. +;; (with-selected-window window +;; (split-window-right))) +;; (and (window-splittable-p window t) +;; ;; Split window horizontally. +;; (with-selected-window window +;; (split-window-right))) +;; (and (eq window (frame-root-window (window-frame window))) +;; (not (window-minibuffer-p window)) +;; ;; If WINDOW is the only window on its frame and is not the +;; ;; minibuffer window, try to split it vertically disregarding +;; ;; the value of `split-height-threshold'. +;; (let ((split-height-threshold 0)) +;; (when (window-splittable-p window) +;; (with-selected-window window +;; (split-window-below)))))))) + +(use-package! lsp-mode + :after (:any haskell-mode) + :config + (setq lsp-response-timeout 60) + :hook + (haskell-mode . lsp-mode)) + +(use-package! lsp-ui + :after lsp-mode + :config + (defun +grfn/lsp-ui-doc-frame-hook (frame window) + (set-frame-font (if doom-big-font-mode doom-big-font doom-font) + nil (list frame))) + (setq lsp-ui-flycheck-enable t + lsp-ui-doc-header nil + lsp-ui-doc-position 'top + lsp-ui-doc-alignment 'window + lsp-ui-doc-frame-hook '+grfn/lsp-ui-doc-frame-hook + lsp-ui-doc-max-width 150 + lsp-ui-doc-max-height 13) + (setq imenu-auto-rescan t) + (set-face-background 'lsp-ui-doc-background +solarized-s-base2) + (set-face-background 'lsp-face-highlight-read +solarized-s-base2) + (set-face-background 'lsp-face-highlight-write +solarized-s-base2) + :hook + (lsp-mode . lsp-ui-mode) + (lsp-ui-mode . flycheck-mode)) + +(use-package! company-lsp + :after (lsp-mode lsp-ui) + :config + (add-to-list #'company-backends #'company-lsp) + (setq company-lsp-async t)) + +(defun +grfn/haskell-mode-setup () + (interactive) + (flymake-mode -1) + (add-to-list 'flycheck-disabled-checkers 'haskell-ghc) + + (flycheck-remove-next-checker 'lsp 'haskell-ghc) + (flycheck-add-next-checker 'lsp '(warning . haskell-hlint)) + + ;; If there’s a 'hie.sh' defined locally by a project + ;; (e.g. to run HIE in a nix-shell), use it… + (when-let ((project-dir (locate-dominating-file default-directory "hie.sh"))) + (cl-flet + ((which (cmd) + (s-trim + (shell-command-to-string + (concat + "nix-shell " + (expand-file-name "shell.nix" project-dir) + " --run \"which " cmd "\" 2>/dev/null"))))) + (setq-local + lsp-haskell-process-path-hie (expand-file-name "hie.sh" project-dir) + haskell-hoogle-command (which "hoogle")))) + ;; … and only then setup the LSP. + (lsp)) + +(defun never-flymake-mode (orig &rest args) + (when (and (bound-and-true-p flymake-mode)) + (funcall orig 0) + (message "disabled flymake-mode"))) +(advice-add #'flymake-mode :around #'never-flymake-mode) + +(defun +grfn/wrap-lsp-haskell-process (argv) + (let* ((project-dir (locate-dominating-file + (buffer-file-name) + "hie.yaml")) + (shell-dot-nix (expand-file-name "shell.nix" project-dir))) + ;; (when (string-equal default-directory "/home/grfn/code/depot") + ;; (debug)) + (message "%s %s %s %s" + (buffer-file-name) + default-directory + project-dir + shell-dot-nix) + (if (file-exists-p shell-dot-nix) + `("bash" "-c" + ,(format "cd %s && nix-shell %s --run '%s'" + project-dir + shell-dot-nix + (s-join " " argv))) + argv))) + +(use-package! lsp-haskell + :after (lsp-mode lsp-ui haskell-mode) + ;; :hook + ;; (haskell-mode . lsp-haskell-enable) + :config + (setq lsp-haskell-process-path-hie "haskell-language-server-wrapper" + lsp-haskell-process-args-hie + '("-d" "-l" "/tmp/hie.log" "+RTS" "-M4G" "-H1G" "-K4G" "-A16M" "-RTS") + lsp-haskell-process-wrapper-function + #'+grfn/wrap-lsp-haskell-process) + (add-hook 'haskell-mode-hook #'+grfn/haskell-mode-setup 't)) + +(use-package! lsp-imenu + :after (lsp-mode lsp-ui) + :hook + (lsp-after-open . lsp-enable-imenu)) + +;; (use-package! counsel-etags +;; :ensure t +;; :init +;; (add-hook 'haskell-mode-hook +;; (lambda () +;; (add-hook 'after-save-hook +;; 'counsel-etags-virtual-update-tags 'append 'local))) +;; :config +;; (setq counsel-etags-update-interval 60) +;; ;; (push "build" counsel-etags-ignore-directories) +;; ) + +;; (use-package! evil-magit +;; :after (magit)) + +(use-package! writeroom-mode) + +(use-package! graphql-mode) + + +(require 'whitespace) +(setq whitespace-style '(face lines-tail)) +(global-whitespace-mode t) +(add-hook 'org-mode-hook (lambda () (whitespace-mode -1)) t) + +(set-face-foreground 'whitespace-line +solarized-red) +(set-face-attribute 'whitespace-line nil :underline 't) + +;; (set-face-background 'ivy-posframe +solarized-s-base3) +;; (set-face-foreground 'ivy-posframe +solarized-s-base01) + +(let ((base03 "#002b36") + (base02 "#073642") + (base01 "#586e75") + (base00 "#657b83") + (base0 "#839496") + (base1 "#93a1a1") + (base2 "#eee8d5") + (base3 "#fdf6e3") + (yellow "#b58900") + (orange "#cb4b16") + (red "#dc322f") + (magenta "#d33682") + (violet "#6c71c4") + (blue "#268bd2") + (cyan "#2aa198") + (green "#859900")) + (custom-set-faces + `(agda2-highlight-keyword-face ((t (:foreground ,green)))) + `(agda2-highlight-string-face ((t (:foreground ,cyan)))) + `(agda2-highlight-number-face ((t (:foreground ,violet)))) + `(agda2-highlight-symbol-face ((((background ,base3)) (:foreground ,base01)))) + `(agda2-highlight-primitive-type-face ((t (:foreground ,blue)))) + `(agda2-highlight-bound-variable-face ((t nil))) + `(agda2-highlight-inductive-constructor-face ((t (:foreground ,green)))) + `(agda2-highlight-coinductive-constructor-face ((t (:foreground ,yellow)))) + `(agda2-highlight-datatype-face ((t (:foreground ,blue)))) + `(agda2-highlight-field-face ((t (:foreground ,red)))) + `(agda2-highlight-function-face ((t (:foreground ,blue)))) + `(agda2-highlight-module-face ((t (:foreground ,yellow)))) + `(agda2-highlight-postulate-face ((t (:foreground ,blue)))) + `(agda2-highlight-primitive-face ((t (:foreground ,blue)))) + `(agda2-highlight-record-face ((t (:foreground ,blue)))) + `(agda2-highlight-dotted-face ((t nil))) + `(agda2-highlight-operator-face ((t nil))) + `(agda2-highlight-error-face ((t (:foreground ,red :underline t)))) + `(agda2-highlight-unsolved-meta-face ((t (:background ,base2)))) + `(agda2-highlight-unsolved-constraint-face ((t (:background ,base2)))) + `(agda2-highlight-termination-problem-face ((t (:background ,orange :foreground ,base03)))) + `(agda2-highlight-incomplete-pattern-face ((t (:background ,orange :foreground ,base03)))) + `(agda2-highlight-typechecks-face ((t (:background ,cyan :foreground ,base03)))))) + + +(after! cider + (setq cider-prompt-for-symbol nil + cider-font-lock-dynamically 't + cider-save-file-on-load 't) + ) + +(defun +org-clocked-in-element () + (when-let ((item (car org-clock-history))) + (save-mark-and-excursion + (with-current-buffer (marker-buffer item) + (goto-char (marker-position item)) + (org-element-at-point))))) + +(comment + (setq elt (+org-clocked-in-item)) + + (eq 'headline (car elt)) + (plist-get (cadr elt) :raw-value) + ) + +(defun +org-headline-title (headline) + (when (eq 'headline (car elt)) + (plist-get (cadr elt) :raw-value))) + +(setq +ligatures-extra-symbols + (append +ligatures-extra-symbols + '(:equal "≡" + :not-equal "≠" + :is "≣" + :isnt "≢" + :lte "≤" + :gte "≥" + :subseteq "⊆" + ))) + +(after! python + (set-pretty-symbols! 'python-mode :merge t + :equal "==" + :not-equal "!=" + :lte "<=" + :gte ">=" + :is "is" + :isnt "is not" + :subseteq "issubset" + + ;; doom builtins + + ;; Functional + :def "def" + :lambda "lambda" + ;; Types + :null "None" + :true "True" :false "False" + :int "int" :str "str" + :float "float" + :bool "bool" + :tuple "tuple" + ;; Flow + :not "not" + :in "in" :not-in "not in" + :and "and" :or "or" + :for "for" + :return "return" :yield "yield")) + +(use-package! sqlup-mode + :hook + (sql-mode-hook . sqlup-mode) + (sql-interactive-mode-hook . sqlup-mode)) + +(use-package! emacsql) +(use-package! emacsql-psql + :after (emacsql)) + +(use-package! pyimport + :after (python)) + +(use-package! blacken + :after (python) + :init + (add-hook #'python-mode-hook #'blacken-mode) + :config + (setq blacken-only-if-project-is-blackened t + blacken-allow-py36 t + blacken-line-length 100)) + +(after! python + (defun +python-setup () + (setq-local fill-column 100 + whitespace-line-column 100 + flycheck-disabled-checkers '(python-flake8) + flycheck-checker 'python-pylint)) + + (add-hook #'python-mode-hook #'+python-setup) + (add-hook #'python-mode-hook #'lsp) + (remove-hook #'python-mode-hook #'pipenv-mode)) + +; (use-package! w3m +; :config +; (setq browse-url-browser-function +; `(("^https://app.clubhouse.io.*" . browse-url-firefox) +; ("^https://github.com.*" . browse-url-firefox) +; (".*" . browse-url-firefox)))) + +(use-package! ob-http + :config + (add-to-list 'org-babel-load-languages '(http . t))) + +;; (use-package! ob-ipython +;; :after (pyimport) +;; :config +;; (add-to-list 'org-babel-load-languages '(ipython . t)) +;; (setq ob-ipython-command + ;; "/home/griffin/code/urb/ciml-video-classifier/bin/jupyter")) + +(use-package! counsel-spotify) + +(after! counsel + (map! [remap counsel-org-capture] #'org-capture + [remap org-capture] #'org-capture)) + +(use-package! evil-snipe :disabled t) +(evil-snipe-mode -1) + +(use-package! rainbow-mode) + +(use-package! org-alert + :disabled t + :config + (org-alert-enable) + (setq alert-default-style 'libnotify + org-alert-headline-title "org")) + +(use-package! ob-async) + +(use-package! org-recent-headings + :config + (map! :n "SPC n r" #'org-recent-headings-ivy)) + +(use-package! org-sticky-header + :after (org) + :hook (org-mode-hook . org-sticky-header-mode) + :config + (setq-default org-sticky-header-heading-star "●")) + +(enable-theme 'grfn-solarized-light) + +;;; this needs to be *after the theme*, or else I get no agenda items. +;;; whuuu?? +(load! "org-config") + + +;;; word-char +(add-hook! prog-mode + (modify-syntax-entry ?_ "w")) + +(add-hook! lisp-mode + (modify-syntax-entry ?- "w")) + +(after! flycheck + (put 'flycheck-python-pylint-executable 'safe-local-variable (lambda (_) t)) + (setq flycheck-error-list-minimum-level 'warn + flycheck-navigation-minimum-level 'warn)) + +(defvar alembic-command "alembic" + "Command to execute when running alembic") + +(defvar alembic-dir-fun (lambda () default-directory) + "Reference to a function whose return value will be used as the directory to + run Alembic in") + +(put 'alembic-command 'safe-local-variable (lambda (_) t)) +(put 'alembic-dir-fun 'safe-local-variable (lambda (_) t)) + +(defun make-alembic-command (args) + (if (functionp alembic-command) + (funcall alembic-command args) + (concat alembic-command " " args))) + +(defun +grfn/extract-alembic-migration-name (output) + (unless (string-match (rx (0+ anything) "Generating " + (group (one-or-more (not (syntax whitespace)))) + " ..." (one-or-more (syntax whitespace)) "done" + (0+ anything)) + output) + (user-error "Error: %s" output)) + (match-string-no-properties 1 output)) + +(defun -run-alembic (args) + (let* ((default-directory (funcall alembic-dir-fun)) + (command (make-alembic-command args)) + ;; (format "nix-shell --run 'alembic %s'" args) + ;; (format "%s %s" alembic-command args) + (res + (with-temp-buffer + (cons + (shell-command command t) + (s-replace-regexp + "^.*Nix search path entry.*$" "" + (buffer-string))))) + (exit-code (car res)) + (out (cdr res))) + ;; (if (= 0 exit-code) + ;; out + ;; (error "Error running %s: %s" command out)) + out + )) + +(comment + --exit-code + --bs + ) + +(defun run-alembic (args) + (interactive "sAlembic command: ") + (message "%s" (-run-alembic args))) + +(defun generate-alembic-migration (msg &rest args) + (interactive "sMessage: ") + (-> + (format "revision %s -m \"%s\"" + (s-join " " args) + msg) + (-run-alembic) + (+grfn/extract-alembic-migration-name) + (find-file-other-window))) + +(cl-defun alembic-upgrade (&optional revision &key namespace) + (interactive "sRevision: ") + (let ((default-directory (funcall alembic-dir-fun))) + (run-alembic (format "%s upgrade %s" + (if namespace (concat "-n " namespace) "") + (or revision "head"))))) + +(defun alembic-downgrade (revision) + (interactive "sRevision: ") + (let ((default-directory (funcall alembic-dir-fun))) + (run-alembic (format "downgrade %s" (or revision "head"))))) + +(use-package! gnuplot) +(use-package! gnuplot-mode :after gnuplot) +(use-package! string-inflection) + +(after! anaconda-mode + ;; (set-company-backend! 'anaconda-mode #'company-yasnippet) + ) + +;; (add-hook! python-mode +;; (capf)) + +(cl-defstruct pull-request url number title author repository) + +(defun grfn/num-inbox-items () + (length (org-elements-agenda-match "inbox" t))) + +(use-package! dhall-mode + :mode "\\.dhall\\'") + +(use-package! github-review + :after forge) + +(after! forge + (set-popup-rule! + "^\\*forge" + :size 0.75)) + +(defun grfn/org-add-db-connection-params () + (interactive) + (ivy-read + "DB to connect to: " + (-map (lambda (opts) + (propertize (symbol-name (car opts)) + 'header-args (cdr opts))) + db-connection-param-options) + :require-match t + :action + (lambda (opt) + (let ((header-args (get-text-property 0 'header-args opt))) + (org-set-property "header-args" header-args))))) + +(use-package! kubernetes + :commands (kubernetes-overview)) + +(use-package! k8s-mode + :hook (k8s-mode . yas-minor-mode)) + +(use-package! sx) + +;; (use-package! nix-update +;; :config +;; (map! (:map nix-mode-map +;; (:leader +;; :desc "Update fetcher" :nv #'nix-update-fetch)))) + + +(after! lsp-haskell + (lsp-register-client + (make-lsp--client + :new-connection (lsp-stdio-connection (lambda () (lsp-haskell--hie-command))) + :major-modes '(haskell-mode) + :server-id 'hie + ;; :multi-root t + ;; :initialization-options 'lsp-haskell--make-init-options + ) + ) + ) + +(solaire-global-mode -1) + +(use-package! wsd-mode) + +(use-package! metal-mercury-mode) +(use-package! flycheck-mercury + :after (metal-mercury-mode flycheck-mercury)) + +(use-package! direnv + :config (direnv-mode)) + +(after! erc + ;; (setq erc-autojoin-channels-alist '(("freenode.net" "#nixos" "#haskell" "##tvl"))) + ) + +(defun evil-disable-insert-state-bindings () + evil-disable-insert-state-bindings) + +;; (use-package! terraform-mode) +;; (use-package! company-terraform +;; :after terraform-mode +;; :config (company-terraform-init)) + +(use-package! znc + :config + (setq znc-servers + '(("znc.gws.fyi" 5000 t + ((freenode "glittershark" "Ompquy")))))) + +(use-package! jsonnet-mode + :config + (map! + (:map jsonnet-mode-map + (:n "g SPC" #'jsonnet-eval-buffer)))) + +(add-to-list 'safe-local-variable-values + '(truncate-lines . t)) + +(set-popup-rule! + "^\\*gud-" + :quit nil) diff --git a/users/grfn/emacs.d/cpp.el b/users/grfn/emacs.d/cpp.el new file mode 100644 index 000000000000..5b5dc8ead652 --- /dev/null +++ b/users/grfn/emacs.d/cpp.el @@ -0,0 +1,39 @@ +;;; -*- lexical-binding: t; -*- + + +(load! "google-c-style") + +(after! flycheck + (add-to-list 'flycheck-disabled-checkers 'c/c++-gcc) + (add-to-list 'flycheck-disabled-checkers 'c/c++-clang)) + +(defun +grfn/cpp-setup () + (when (s-starts-with? + "/home/grfn/code/depot/third_party/nix" + (buffer-file-name)) + (setq lsp-clients-clangd-executable "/home/grfn/code/depot/users/grfn/emacs.d/nix-clangd.sh" + lsp-clients-clangd-args nil) + (google-set-c-style) + (lsp) + (add-to-list 'flycheck-disabled-checkers 'c/c++-gcc) + (add-to-list 'flycheck-disabled-checkers 'c/c++-clang))) + +(add-hook 'c++-mode-hook #'+grfn/cpp-setup) + +(use-package! protobuf-mode) + +(use-package! clang-format+ + :config + (add-hook 'c-mode-common-hook #'clang-format+-mode)) + +(map! + (:map c++-mode-map + :leader + (:n "/ i" #'counsel-semantic-or-imenu))) + +(comment + (setq + lsp-clients-clangd-executable + "/home/grfn/code/depot/third_party/nix/clangd.sh" + lsp-clients-clangd-args nil) + ) diff --git a/users/grfn/emacs.d/email.el b/users/grfn/emacs.d/email.el new file mode 100644 index 000000000000..83076898b4dc --- /dev/null +++ b/users/grfn/emacs.d/email.el @@ -0,0 +1,42 @@ +;;; -*- lexical-binding: t; -*- + +(after! notmuch + (setq notmuch-saved-searches + '((:name "inbox" :query "tag:inbox tag:important not tag:trash" :key "i") + (:name "flagged" :query "tag:flagged" :key "f") + (:name "sent" :query "tag:sent" :key "s") + (:name "drafts" :query "tag:draft" :key "d") + + (:name "work" :query "tag:inbox and tag:important and path:work/**" + :key "w") + (:name "personal" :query "tag:inbox and tag:important and path:personal/**" + :key "p")) + message-send-mail-function 'message-send-mail-with-sendmail + message-sendmail-f-is-evil 't + message-sendmail-envelope-from 'header + message-sendmail-extra-arguments '("--read-envelope-from")) + + (add-hook! notmuch-message-mode-hook #'notmuch-company-setup)) + +(setq notmuch-saved-searches + '((:name "inbox" :query "tag:inbox tag:important not tag:trash" :key "i") + (:name "flagged" :query "tag:flagged" :key "f") + (:name "sent" :query "tag:sent" :key "s") + (:name "drafts" :query "tag:draft" :key "d") + + (:name "work" :query "tag:inbox and tag:important and path:work/**" + :key "w") + (:name "personal" :query "tag:inbox and tag:important and path:personal/**" + :key "p")) + message-send-mail-function 'message-send-mail-with-sendmail + message-sendmail-f-is-evil 't + message-sendmail-envelope-from 'header + message-sendmail-extra-arguments '("--read-envelope-from")) + +(set-popup-rule! "^\\*notmuch-saved-search-" + :ignore t) + +(set-popup-rule! (lambda (_ action) + (eq (car action) + 'display-buffer-same-window)) + :ignore t) diff --git a/users/grfn/emacs.d/github-org.el b/users/grfn/emacs.d/github-org.el new file mode 100644 index 000000000000..f4f9d2e37069 --- /dev/null +++ b/users/grfn/emacs.d/github-org.el @@ -0,0 +1,99 @@ +;;; -*- lexical-binding: t; -*- + +(require 'ghub) + +(defun grfn/alist->plist (alist) + (->> alist + (-mapcat (lambda (pair) + (list (intern (concat ":" (symbol-name (car pair)))) + (cdr pair)))))) + +;;; + +(cl-defstruct pull-request url number title author repository) + +(defun grfn/query-pulls (query) + (let ((resp (ghub-graphql "query reviewRequests($query: String!) { + reviewRequests: search( + type:ISSUE, + query: $query, + first: 100 + ) { + issueCount + nodes { + ... on PullRequest { + url + number + title + author { + login + ... on User { name } + } + repository { + name + owner { login } + } + } + } + } + }" `((query . ,query))))) + (->> resp + (alist-get 'data) + (alist-get 'reviewRequests) + (alist-get 'nodes) + (-map + (lambda (pr) + (apply + #'make-pull-request + (grfn/alist->plist pr))))))) + +(defun grfn/requested-changes ()) + +(defun grfn/pull-request->org-headline (format-string level pr) + (check-type format-string string) + (check-type level integer) + (check-type pr pull-request) + (s-format (concat (make-string level ?*) " " format-string) + 'aget + `((author . ,(or (->> pr (pull-request-author) (alist-get 'name)) + "no author")) + (owner . ,(->> pr (pull-request-repository) + (alist-get 'owner) + (alist-get 'login))) + (repo . ,(->> pr (pull-request-repository) (alist-get 'name))) + (pr-link . ,(org-make-link-string + (pull-request-url pr) + (pull-request-title pr))) + (today . ,(format-time-string "%Y-%m-%d %a"))))) + +(defun grfn/org-headlines-from-review-requests (level) + "Create org-mode headlines at LEVEL from all review-requested PRs on Github" + (interactive "*nLevel: ") + (let* ((prs (grfn/query-pulls + "is:open is:pr review-requested:glittershark archived:false")) + (text (mapconcat + (apply-partially + #'grfn/pull-request->org-headline + "TODO Review ${author}'s PR on ${owner}/${repo}: ${pr-link} :pr: +SCHEDULED: <${today}>" + level) prs "\n"))) + (save-mark-and-excursion + (insert text)) + (org-align-tags 't))) + +(defun grfn/org-headlines-from-requested-changes (level) + "Create org-mode headlines at LEVEL from all PRs with changes requested + on Github" + (interactive "*nLevel: ") + (let* ((prs (grfn/query-pulls + (concat "is:pr is:open author:glittershark archived:false " + "sort:updated-desc review:changes-requested"))) + (text (mapconcat + (apply-partially + #'grfn/pull-request->org-headline + "TODO Address review comments on ${pr-link} :pr: +SCHEDULED: <${today}>" + level) prs "\n"))) + (save-mark-and-excursion + (insert text)) + (org-align-tags 't))) diff --git a/users/grfn/emacs.d/google-c-style.el b/users/grfn/emacs.d/google-c-style.el new file mode 100644 index 000000000000..9bb12c61aae4 --- /dev/null +++ b/users/grfn/emacs.d/google-c-style.el @@ -0,0 +1,151 @@ +;;; google-c-style.el --- Google's C/C++ style for c-mode + +;; Keywords: c, tools + +;; google-c-style.el is Copyright (C) 2008 Google Inc. All Rights Reserved. +;; +;; It is free software; you can redistribute it and/or modify it under the +;; terms of either: +;; +;; a) the GNU General Public License as published by the Free Software +;; Foundation; either version 1, or (at your option) any later version, or +;; +;; b) the "Artistic License". + +;;; Commentary: + +;; Provides the google C/C++ coding style. You may wish to add +;; `google-set-c-style' to your `c-mode-common-hook' after requiring this +;; file. For example: +;; +;; (add-hook 'c-mode-common-hook 'google-set-c-style) +;; +;; If you want the RETURN key to go to the next line and space over +;; to the right place, add this to your .emacs right after the load-file: +;; +;; (add-hook 'c-mode-common-hook 'google-make-newline-indent) + +;;; Code: + +;; For some reason 1) c-backward-syntactic-ws is a macro and 2) under Emacs 22 +;; bytecode cannot call (unexpanded) macros at run time: +(eval-when-compile (require 'cc-defs)) + +;; Wrapper function needed for Emacs 21 and XEmacs (Emacs 22 offers the more +;; elegant solution of composing a list of lineup functions or quantities with +;; operators such as "add") +(defun google-c-lineup-expression-plus-4 (langelem) + "Indents to the beginning of the current C expression plus 4 spaces. + +This implements title \"Function Declarations and Definitions\" +of the Google C++ Style Guide for the case where the previous +line ends with an open parenthese. + +\"Current C expression\", as per the Google Style Guide and as +clarified by subsequent discussions, means the whole expression +regardless of the number of nested parentheses, but excluding +non-expression material such as \"if(\" and \"for(\" control +structures. + +Suitable for inclusion in `c-offsets-alist'." + (save-excursion + (back-to-indentation) + ;; Go to beginning of *previous* line: + (c-backward-syntactic-ws) + (back-to-indentation) + (cond + ;; We are making a reasonable assumption that if there is a control + ;; structure to indent past, it has to be at the beginning of the line. + ((looking-at "\\(\\(if\\|for\\|while\\)\\s *(\\)") + (goto-char (match-end 1))) + ;; For constructor initializer lists, the reference point for line-up is + ;; the token after the initial colon. + ((looking-at ":\\s *") + (goto-char (match-end 0)))) + (vector (+ 4 (current-column))))) + +;;;###autoload +(defconst google-c-style + `((c-recognize-knr-p . nil) + (c-enable-xemacs-performance-kludge-p . t) ; speed up indentation in XEmacs + (c-basic-offset . 2) + (indent-tabs-mode . nil) + (c-comment-only-line-offset . 0) + (c-hanging-braces-alist . ((defun-open after) + (defun-close before after) + (class-open after) + (class-close before after) + (inexpr-class-open after) + (inexpr-class-close before) + (namespace-open after) + (inline-open after) + (inline-close before after) + (block-open after) + (block-close . c-snug-do-while) + (extern-lang-open after) + (extern-lang-close after) + (statement-case-open after) + (substatement-open after))) + (c-hanging-colons-alist . ((case-label) + (label after) + (access-label after) + (member-init-intro before) + (inher-intro))) + (c-hanging-semi&comma-criteria + . (c-semi&comma-no-newlines-for-oneline-inliners + c-semi&comma-inside-parenlist + c-semi&comma-no-newlines-before-nonblanks)) + (c-indent-comments-syntactically-p . t) + (comment-column . 40) + (c-indent-comment-alist . ((other . (space . 2)))) + (c-cleanup-list . (brace-else-brace + brace-elseif-brace + brace-catch-brace + empty-defun-braces + defun-close-semi + list-close-comma + scope-operator)) + (c-offsets-alist . ((arglist-intro google-c-lineup-expression-plus-4) + (func-decl-cont . ++) + (member-init-intro . ++) + (inher-intro . ++) + (comment-intro . 0) + (arglist-close . c-lineup-arglist) + (topmost-intro . 0) + (block-open . 0) + (inline-open . 0) + (substatement-open . 0) + (statement-cont + . + (,(when (fboundp 'c-no-indent-after-java-annotations) + 'c-no-indent-after-java-annotations) + ,(when (fboundp 'c-lineup-assignments) + 'c-lineup-assignments) + ++)) + (label . /) + (case-label . +) + (statement-case-open . +) + (statement-case-intro . +) ; case w/o { + (access-label . /) + (innamespace . 0)))) + "Google C/C++ Programming Style.") + +;;;###autoload +(defun google-set-c-style () + "Set the current buffer's c-style to Google C/C++ Programming + Style. Meant to be added to `c-mode-common-hook'." + (interactive) + (make-local-variable 'c-tab-always-indent) + (setq c-tab-always-indent t) + (c-add-style "Google" google-c-style t)) + +;;;###autoload +(defun google-make-newline-indent () + "Sets up preferred newline behavior. Not set by default. Meant + to be added to `c-mode-common-hook'." + (interactive) + (define-key c-mode-base-map "\C-m" 'newline-and-indent) + (define-key c-mode-base-map [ret] 'newline-and-indent)) + +(provide 'google-c-style) +;;; google-c-style.el ends here diff --git a/users/grfn/emacs.d/grid.el b/users/grfn/emacs.d/grid.el new file mode 100644 index 000000000000..75776a38cd9d --- /dev/null +++ b/users/grfn/emacs.d/grid.el @@ -0,0 +1,128 @@ +;;; -*- lexical-binding: t; -*- + +(require 's) + +(defun grfn/all-match-groups (s) + (loop for n from 1 + for x = (match-string n s) + while x + collect x)) + +(defun projectile-grid-ff (path &optional ask) + "Call `find-file' function on PATH when it is not nil and the file exists. +If file does not exist and ASK in not nil it will ask user to proceed." + (if (or (and path (file-exists-p path)) + (and ask (yes-or-no-p + (s-lex-format + "File does not exists. Create a new buffer ${path} ?")))) + (find-file path))) + +(defun projectile-grid-goto-file (filepath &optional ask) + "Find FILEPATH after expanding root. ASK is passed straight to `projectile-grid-ff'." + (projectile-grid-ff (projectile-expand-root filepath) ask)) + +(defun projectile-grid-choices (ds) + "Uses `projectile-dir-files' function to find files in directories. +The DIRS is list of lists consisting of a directory path and regexp to filter files from that directory. +Optional third element can be present in the DS list. The third element will be a prefix to be placed before +the filename in the resulting choice. +Returns a hash table with keys being short names (choices) and values being relative paths to the files." + (loop with hash = (make-hash-table :test 'equal) + for (dir re prefix) in ds do + (loop for file in (projectile-dir-files (projectile-expand-root dir)) do + (when (string-match re file) + (puthash + (concat (or prefix "") + (s-join "/" (grfn/all-match-groups file))) + (concat dir file) + hash))) + finally return hash)) + +(defmacro projectile-grid-find-resource (prompt dirs &optional newfile-template) + "Presents files from DIRS with PROMPT to the user using `projectile-completing-read'. +If users chooses a non existant file and NEWFILE-TEMPLATE is not nil +it will use that variable to interpolate the name for the new file. +NEWFILE-TEMPLATE will be the argument for `s-lex-format'. +The bound variable is \"filename\"." + `(lexical-let ((choices (projectile-grid-choices ,dirs))) + (projectile-completing-read + ,prompt + (hash-table-keys choices) + :action + (lambda (c) + (let* ((filepath (gethash c choices)) + (filename c)) ;; so `s-lex-format' can interpolate FILENAME + (if filepath + (projectile-grid-goto-file filepath) + (when-let ((newfile-template ,newfile-template)) + (projectile-grid-goto-file + (funcall newfile-template filepath) + ;; (cond + ;; ((functionp newfile-template) (funcall newfile-template filepath)) + ;; ((stringp newfile-template) (s-lex-format newfile-template))) + t)))))))) + +(defun projectile-grid-find-model () + "Find a model." + (interactive) + (projectile-grid-find-resource + "model: " + '(("python/urbint_lib/models/" + "\\(.+\\)\\.py$") + ("python/urbint_lib/" + "\\(.+\\)/models/\\(.+\\).py$")) + (lambda (filename) + (pcase (s-split "/" filename) + (`(,model) + (s-lex-format "python/urbint_lib/models/${model}.py")) + (`(,app ,model) + (s-lex-format "python/urbint_lib/${app}/models/${model}.py")))))) + +(defun projectile-grid-find-repository () + "Find a repository." + (interactive) + (projectile-grid-find-resource + "repository: " + '(("python/urbint_lib/repositories/" + "\\(.+\\)\\.py$") + ("python/urbint_lib/" + "\\(.+\\)/repositories/\\(.+\\).py$")) + (lambda (filename) + (pcase (s-split "/" filename) + (`(,repository) + (s-lex-format "python/urbint_lib/repositories/${repository}.py")) + (`(,app ,repository) + (s-lex-format "python/urbint_lib/${app}/repositories/${repository}.py")))))) + +(defun projectile-grid-find-controller () + "Find a controller." + (interactive) + (projectile-grid-find-resource + "controller: " + '(("backend/src/grid/api/controllers/" + "\\(.+\\)\\.py$") + ("backend/src/grid/api/apps/" + "\\(.+\\)/controllers/\\(.+\\).py$")) + (lambda (filename) + (pcase (s-split "/" filename) + (`(,controller) + (s-lex-format "backend/src/grid/api/controllers/${controller}.py")) + (`(,app ,controller) + (s-lex-format "backend/src/grid/api/apps/${app}/controllers/${controller}.py")))))) + +(setq projectile-grid-mode-map + (let ((map (make-keymap))) + (map! + (:map map + (:leader + (:desc "Edit..." :prefix "e" + :desc "Model" :n "m" #'projectile-grid-find-model + :desc "Controller" :n "c" #'projectile-grid-find-controller + :desc "Repository" :n "r" #'projectile-grid-find-repository)))) + map)) + +(define-minor-mode projectile-grid-mode + "Minor mode for finding files in GRID" + :init-value nil + :lighter " GRID" + :keymap projectile-grid-mode-map) diff --git a/users/grfn/emacs.d/init.el b/users/grfn/emacs.d/init.el new file mode 100644 index 000000000000..563b455dff15 --- /dev/null +++ b/users/grfn/emacs.d/init.el @@ -0,0 +1,233 @@ +;;; -*- lexical-binding: t; -*- + +(doom! :completion + company ; the ultimate code completion backend + (ivy +fuzzy + +prescient) ; a search engine for love and life + + :ui + ;;deft ; notational velocity for Emacs + doom ; what makes DOOM look the way it does + ;doom-dashboard ; a nifty splash screen for Emacs + doom-quit ; DOOM quit-message prompts when you quit Emacs + ;fill-column ; a `fill-column' indicator + hl-todo ; highlight TODO/FIXME/NOTE tags + ;;indent-guides ; highlighted indent columns + modeline ; snazzy, Atom-inspired modeline, plus API + nav-flash ; blink the current line after jumping + ;;neotree ; a project drawer, like NERDTree for vim + ophints ; highlight the region an operation acts on + (popup ; tame sudden yet inevitable temporary windows + +all ; catch all popups that start with an asterix + +defaults) ; default popup rules + ligatures ; replace bits of code with pretty symbols + ;;tabbar ; FIXME an (incomplete) tab bar for Emacs + ;;treemacs ; a project drawer, like neotree but cooler + unicode ; extended unicode support for various languages + vc-gutter ; vcs diff in the fringe + vi-tilde-fringe ; fringe tildes to mark beyond EOB + window-select ; visually switch windows + workspaces ; tab emulation, persistence & separate workspaces + + :editor + (evil +everywhere); come to the dark side, we have cookies + file-templates ; auto-snippets for empty files + fold ; (nigh) universal code folding + ;;(format +onsave) ; automated prettiness + ;;lispy ; vim for lisp, for people who dont like vim + multiple-cursors ; editing in many places at once + ;;parinfer ; turn lisp into python, sort of + rotate-text ; cycle region at point between text candidates + snippets ; my elves. They type so I don't have to + word-wrap + + :emacs + (dired ; making dired pretty [functional] + ;;+ranger ; bringing the goodness of ranger to dired + ;;+icons ; colorful icons for dired-mode + ) + electric ; smarter, keyword-based electric-indent + ;;eshell ; a consistent, cross-platform shell (WIP) + ;;term ; terminals in Emacs + vc ; version-control and Emacs, sitting in a tree + (undo +tree) + + :tools + ;;ansible + ;;debugger ; FIXME stepping through code, to help you add bugs + ;;direnv + docker + ;;editorconfig ; let someone else argue about tabs vs spaces + ;; ein ; tame Jupyter notebooks with emacs + (eval +overlay) ; run code, run (also, repls) + gist ; interacting with github gists + (lookup ; helps you navigate your code and documentation + +docsets) ; ...or in Dash docsets locally + lsp + ;;macos ; MacOS-specific commands + magit ; a git porcelain for Emacs + make ; run make tasks from Emacs + pass ; password manager for nerds + pdf ; pdf enhancements + ;;prodigy ; FIXME managing external services & code builders + ;;rgb ; creating color strings + ;;terraform ; infrastructure as code + ;;tmux ; an API for interacting with tmux + ;;upload ; map local to remote projects via ssh/ftp + ;;wakatime + ;;vterm ; another terminals in Emacs + + :checkers + syntax ; tasing you for every semicolon you forget + ; spell ; tasing you for misspelling mispelling + + :lang + agda ; types of types of types of types... + ;;assembly ; assembly for fun or debugging + cc ; C/C++/Obj-C madness + clojure ; java with a lisp + common-lisp ; if you've seen one lisp, you've seen them all + ; coq ; proofs-as-programs + ;;crystal ; ruby at the speed of c + ;;csharp ; unity, .NET, and mono shenanigans + data ; config/data formats + erlang ; an elegant language for a more civilized age + elixir ; erlang done right + ;;elm ; care for a cup of TEA? + emacs-lisp ; drown in parentheses + ;;ess ; emacs speaks statistics + ;;go ; the hipster dialect + ;; (haskell +intero) ; a language that's lazier than I am + haskell ; a language that's lazier than I am + ;;hy ; readability of scheme w/ speed of python + ;; idris ; + ;;(java +meghanada) ; the poster child for carpal tunnel syndrome + javascript ; all(hope(abandon(ye(who(enter(here)))))) + julia ; a better, faster MATLAB + ;;kotlin ; a better, slicker Java(Script) + latex ; writing papers in Emacs has never been so fun + ;;ledger ; an accounting system in Emacs + lua ; one-based indices? one-based indices + markdown ; writing docs for people to ignore + ;;nim ; python + lisp at the speed of c + nix ; I hereby declare "nix geht mehr!" + ;;ocaml ; an objective camel + (org ; organize your plain life in plain text + +dragndrop ; drag & drop files/images into org buffers + +attach ; custom attachment system + +babel ; running code in org + +capture ; org-capture in and outside of Emacs + +export ; Exporting org to whatever you want + ;; +habit ; Keep track of your habits + +present ; Emacs for presentations + +pretty + +brain + +protocol) ; Support for org-protocol:// links + ;;perl ; write code no one else can comprehend + ;;php ; perl's insecure younger brother + ;;plantuml ; diagrams for confusing people more + purescript ; javascript, but functional + (python +lsp) ; beautiful is better than ugly + ;;qt ; the 'cutest' gui framework ever + racket ; a DSL for DSLs + rest ; Emacs as a REST client + ;;ruby ; 1.step do {|i| p "Ruby is #{i.even? ? 'love' : 'life'}"} + rust ; Fe2O3.unwrap().unwrap().unwrap().unwrap() + ;;scala ; java, but good + (sh +fish) ; she sells (ba|z|fi)sh shells on the C xor + ;;solidity ; do you need a blockchain? No. + ;;swift ; who asked for emoji variables? + ;;terra ; Earth and Moon in alignment for performance. + ;;web ; the tubes + ;;vala ; GObjective-C + + ;; Applications are complex and opinionated modules that transform Emacs + ;; toward a specific purpose. They may have additional dependencies and + ;; should be loaded late. + :app + ;;(email +gmail) ; emacs as an email client + irc ; how neckbeards socialize + ;;(rss +org) ; emacs as an RSS reader + twitter ; twitter client https://twitter.com/vnought + ;;(write ; emacs as a word processor (latex + org + markdown) + ;; +wordnut ; wordnet (wn) search + ;; +langtool) ; a proofreader (grammar/style check) for Emacs + + :email + ;; (mu4e +gmail) + notmuch + + :collab + ;;floobits ; peer programming for a price + ;;impatient-mode ; show off code over HTTP + + :config + ;; For literate config users. This will tangle+compile a config.org + ;; literate config in your `doom-private-dir' whenever it changes. + ;;literate + + ;; The default module sets reasonable defaults for Emacs. It also + ;; provides a Spacemacs-inspired keybinding scheme and a smartparens + ;; config. Use it as a reference for your own modules. + (default +bindings +smartparens)) +(custom-set-variables + ;; custom-set-variables was added by Custom. + ;; If you edit it by hand, you could mess it up, so be careful. + ;; Your init file should contain only one such instance. + ;; If there is more than one, they won't work right. + '(doom-big-font-mode nil) + '(flycheck-javascript-flow-args nil) + '(org-agenda-files + '("/home/griffin/notes/personal.org" "/home/griffin/notes/2020-01-27-data-pipeline-deploy-mismatch.org" "/home/griffin/notes/architecture.org" "/home/griffin/notes/cooking.org" "/home/griffin/notes/culture-survey.org" "/home/griffin/notes/dir-structure.org" "/home/griffin/notes/dnd.org" "/home/griffin/notes/inbox.org" "/home/griffin/notes/misc-todo.org" "/home/griffin/notes/nix-talk.org" "/home/griffin/notes/notes.org" "/home/griffin/notes/one-on-one.org" "/home/griffin/notes/work.org" "/home/griffin/notes/xanthous.org" "/home/griffin/notes/xgboost.org")) + '(safe-local-variable-values + '((intero-stack-yaml . "/home/griffin/code/mlem/stack.yaml") + (elisp-lint-indent-specs + (if-let* . 2) + (when-let* . 1) + (let* . defun) + (nrepl-dbind-response . 2) + (cider-save-marker . 1) + (cider-propertize-region . 1) + (cider-map-repls . 1) + (cider--jack-in . 1) + (cider--make-result-overlay . 1) + (insert-label . defun) + (insert-align-label . defun) + (insert-rect . defun) + (cl-defun . 2) + (with-parsed-tramp-file-name . 2) + (thread-first . 1) + (thread-last . 1)) + (checkdoc-package-keywords-flag) + (cider-jack-in-default . "shadow-cljs") + (projectile-project-root . "/home/griffin/code/urb/grid/backend/src") + (python-pytest-executable . "/home/griffin/code/urb/grid/backend/src/.venv/bin/pytest")))) +(custom-set-faces + ;; custom-set-faces was added by Custom. + ;; If you edit it by hand, you could mess it up, so be careful. + ;; Your init file should contain only one such instance. + ;; If there is more than one, they won't work right. + '(default ((((class color) (min-colors 89)) (:foreground "#657b83" :background "#fdf6e3")))) + '(agda2-highlight-bound-variable-face ((t nil))) + '(agda2-highlight-coinductive-constructor-face ((t (:foreground "#b58900")))) + '(agda2-highlight-datatype-face ((t (:foreground "#268bd2")))) + '(agda2-highlight-dotted-face ((t nil))) + '(agda2-highlight-error-face ((t (:foreground "#dc322f" :underline t)))) + '(agda2-highlight-field-face ((t (:foreground "#dc322f")))) + '(agda2-highlight-function-face ((t (:foreground "#268bd2")))) + '(agda2-highlight-incomplete-pattern-face ((t (:background "#cb4b16" :foreground "#002b36")))) + '(agda2-highlight-inductive-constructor-face ((t (:foreground "#859900")))) + '(agda2-highlight-keyword-face ((t (:foreground "#859900")))) + '(agda2-highlight-module-face ((t (:foreground "#b58900")))) + '(agda2-highlight-number-face ((t (:foreground "#6c71c4")))) + '(agda2-highlight-operator-face ((t nil))) + '(agda2-highlight-postulate-face ((t (:foreground "#268bd2")))) + '(agda2-highlight-primitive-face ((t (:foreground "#268bd2")))) + '(agda2-highlight-primitive-type-face ((t (:foreground "#268bd2")))) + '(agda2-highlight-record-face ((t (:foreground "#268bd2")))) + '(agda2-highlight-string-face ((t (:foreground "#2aa198")))) + '(agda2-highlight-symbol-face ((((background "#fdf6e3")) (:foreground "#586e75")))) + '(agda2-highlight-termination-problem-face ((t (:background "#cb4b16" :foreground "#002b36")))) + '(agda2-highlight-typechecks-face ((t (:background "#2aa198" :foreground "#002b36")))) + '(agda2-highlight-unsolved-constraint-face ((t (:background "#eee8d5")))) + '(agda2-highlight-unsolved-meta-face ((t (:background "#eee8d5"))))) diff --git a/users/grfn/emacs.d/irc.el b/users/grfn/emacs.d/irc.el new file mode 100644 index 000000000000..117869599d83 --- /dev/null +++ b/users/grfn/emacs.d/irc.el @@ -0,0 +1,131 @@ +;;; -*- lexical-binding: t; -*- + +(require 'erc) +(require 'alert) + +(defvar irc-servers + '("hackint" + "libera")) + +(defun irc-connect (server) + (interactive + (list (ivy-read "Server: " irc-servers))) + (let ((pw (s-trim (shell-command-to-string + (format "pass irccloud/%s" server)))) + (gnutls-verify-error nil)) + (erc-tls :server "bnc.irccloud.com" + :port 6697 + :nick "grfn" + :password (concat "bnc@" + (s-trim (shell-command-to-string "hostname")) + ":" + pw)))) + + +(defgroup erc-alert nil + "Alert me using alert.el for important ERC messages" + :group 'erc) + +(defcustom erc-noise-regexp + "\\(Logging in:\\|Signing off\\|You're now away\\|Welcome back\\)" + "This regexp matches unwanted noise." + :type 'regexp + :group 'erc) + +(setq tvl-enabled? t) + +(defun disable-tvl-notifications () + (interactive) + (setq tvl-enabled? nil)) + +(defun enable-tvl-notifications () + (interactive) + (setq tvl-enabled? t)) + +(defun erc-alert-important-p (info) + (let ((message (plist-get info :message)) + (erc-message (-> info (plist-get :data) (plist-get :message))) + (erc-channel (-> info (plist-get :data) (plist-get :channel)))) + (and erc-message + (not (or (string-match "^\\** *Users on #" message) + (string-match erc-noise-regexp + message))) + (or (and tvl-enabled? + (string-equal erc-channel "#tvl")) + (string-match "grfn" message))))) + +(comment + last-info + erc-noise-regexp + (setq tvl-enabled? nil) + ) + +(defun my-erc-hook (&optional match-type nick message) + "Shows a notification, when user's nick was mentioned. +If the buffer is currently not visible, makes it sticky." + (setq last-message message) + (if (or (null match-type) (not (eq match-type 'fool))) + (let (alert-log-messages) + (alert (or message (buffer-string)) + :severity (if (string-match "grfn" (or message "")) + 'high 'low) + :title (or nick (buffer-name)) + :data `(:message ,(or message (buffer-string)) + :channel ,(or nick (buffer-name))))))) + +(add-hook 'erc-text-matched-hook 'my-erc-hook) +(add-hook 'erc-insert-modify-hook 'my-erc-hook) + +(defun my-erc-define-alerts (&rest ignore) + ;; Unless the user has recently typed in the ERC buffer, highlight the fringe + (alert-add-rule + :status '(buried visible idle) + :severity '(moderate high urgent) + :mode 'erc-mode + :predicate + #'(lambda (info) + (and (not (eq (current-buffer) (plist-get info :buffer))) + (string-match "grfn:" (plist-get info :message)))) + :persistent + #'(lambda (info) + ;; If the buffer is buried, or the user has been idle for + ;; `alert-reveal-idle-time' seconds, make this alert + ;; persistent. Normally, alerts become persistent after + ;; `alert-persist-idle-time' seconds. + (memq (plist-get info :status) '(buried idle))) + :style 'message + :continue t) + + (alert-add-rule + :status 'buried + :mode 'erc-mode + :predicate #'erc-alert-important-p + :style 'libnotify + :append t) + + (alert-add-rule + :status 'buried + :mode 'erc-mode + :predicate #'erc-alert-important-p + :style 'message + :append t) + + (alert-add-rule + :mode 'erc-mode + :predicate #'erc-alert-important-p + :style 'log + :append t) + + (alert-add-rule :mode 'erc-mode :style 'ignore :append t)) + +(add-hook 'erc-connect-pre-hook 'my-erc-define-alerts) + +(defun fix-irc-message (msg) + (let ((msg (s-trim msg))) + (if (string-equal msg ":q") "" msg))) + +(advice-add #'erc-user-input :filter-return #'fix-irc-message) + +(comment + (my-erc-define-alerts) + ) diff --git a/users/grfn/emacs.d/lisp.el b/users/grfn/emacs.d/lisp.el new file mode 100644 index 000000000000..c45cc7e6e381 --- /dev/null +++ b/users/grfn/emacs.d/lisp.el @@ -0,0 +1,38 @@ +;;; -*- lexical-binding: t; -*- + +(defun grfn/sly-panettone () + (interactive) + (sly + (concat + (s-trim + (shell-command-to-string + "nix-build -o sbcl -E 'with import ~/code/depot {}; nix.buildLisp.sbclWith [web.panettone]'")) + "/bin/sbcl"))) + +(defun grfn/setup-lisp () + (interactive) + (unless paxedit-mode (paxedit-mode 1)) + (rainbow-delimiters-mode) + (flycheck-mode -1)) + +(add-hook 'common-lisp-lisp-mode-hook #'grfn/setup-lisp) + +(defun sly-run-tests () + (interactive) + ;; TODO: handle other test frameworks + (let ((orig-window (get-buffer-window))) + (sly-eval '(fiveam:run!)) + (funcall-interactively #'sly-mrepl-sync) + (select-window orig-window))) + +(map! + (:map sly-mode-map + :n "g \\" #'sly-mrepl-sync + :n "g d" #'sly-edit-definition + :n "K" #'sly-documentation + :n "g SPC" #'sly-compile-and-load-file + :n "g RET" #'sly-run-tests) + + (:map sly-mrepl-mode-map + "C-k" #'sly-mrepl-previous-prompt + "C-r" #'isearch-backward)) diff --git a/users/grfn/emacs.d/nix-clangd.sh b/users/grfn/emacs.d/nix-clangd.sh new file mode 100755 index 000000000000..16f6252d8b27 --- /dev/null +++ b/users/grfn/emacs.d/nix-clangd.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +CLANGD_FLAGS=--compile-commands-dir=/home/grfn/builds/tvix \ + nix-shell /home/grfn/code/depot \ + -A third_party.nix \ + --run nix-clangd diff --git a/users/grfn/emacs.d/nix.el b/users/grfn/emacs.d/nix.el new file mode 100644 index 000000000000..ec5b474af218 --- /dev/null +++ b/users/grfn/emacs.d/nix.el @@ -0,0 +1,30 @@ +;;; -*- lexical-binding: t; -*- + +(defun nix-buffer-type () + "Returns: + +'home-manager, if the current buffer is a home-manager module +'nixos, if the current buffer is a nixos module +nil, if none of the above are the case" + (when buffer-file-name + (pcase buffer-file-name + ((rx (0+ nonl) "system/home" (0+ nonl) ".nix" eos) + 'home-manager) + ((rx (0+ nonl) "system/system" (0+ nonl) ".nix" eos) + 'nixos)))) + +(defun set-nix-compile-command () + "Set the compile command for the current buffer based on the type of nix +buffer it is, per `nix-buffer-type'" + (interactive) + (when-let ((btype (nix-buffer-type))) + (setq-local + compile-command + (case btype + ('home-manager "home-manager switch") + ('nixos "sudo nixos-rebuild switch"))))) + +(add-hook 'nix-mode-hook #'set-nix-compile-command) + +(map! (:map nix-mode-map + (:n "g SPC" #'compile))) diff --git a/users/grfn/emacs.d/org-alerts.el b/users/grfn/emacs.d/org-alerts.el new file mode 100644 index 000000000000..8e6c3e0417ff --- /dev/null +++ b/users/grfn/emacs.d/org-alerts.el @@ -0,0 +1,188 @@ +;;; -*- lexical-binding: t; -*- + +;;; Commentary: + +;;; Code: + +(require 's) +(require 'dash) +(require 'alert) +(require 'org-agenda) + + +(defvar grfn/org-alert-interval 300 + "Interval in seconds to recheck and display deadlines.") + + +(defvar grfn/org-alert-notification-title "*org*" + "Title to be sent with notify-send.") + +(defvar grfn/org-alert-headline-regexp "\\(Sched.+:.+\\|Deadline:.+\\)" + "Regexp for headlines to search in agenda buffer.") + +(defun grfn/org-alert--strip-prefix (headline) + "Remove the scheduled/deadline prefix from HEADLINE." + (replace-regexp-in-string ".*:\s+" "" headline)) + + +(defun grfn/org-alert--unique-headlines (regexp agenda) + "Return unique headlines from the results of REGEXP in AGENDA." + (let ((matches (-distinct (-flatten (s-match-strings-all regexp agenda))))) + (--map (grfn/org-alert--strip-prefix it) matches))) + + +(defun grfn/org-alert--get-headlines () + "Return the current org agenda as text only." + (with-temp-buffer + (let ((org-agenda-sticky nil) + (org-agenda-buffer-tmp-name (buffer-name))) + (ignore-errors (org-agenda-list nil "TODAY" 1)) + (grfn/org-alert--unique-headlines + grfn/org-alert-headline-regexp + (buffer-substring-no-properties (point-min) (point-max)))))) + +(defun grfn/parse-range-string (str) + (when + (string-match (rx (group (repeat 2 (any digit)) + ":" + (repeat 2 (any digit))) + (optional + (and + "-" + (group (repeat 2 (any digit)) + ":" + (repeat 2 (any digit)))))) + str) + (list + (org-read-date nil t + (match-string 1 str)) + (when-let ((et (match-string 2 str))) (org-read-date nil t et))))) + +(defun grfn/start-time-from-range-string (str) + (pcase-let ((`(,start-time . _) (grfn/parse-range-string str))) + start-time)) + +(comment + (org-agenda-list nil "TODAY" 1) + + (grfn/org-alert--get-headlines) + (setq --src + (with-temp-buffer + (let ((org-agenda-sticky nil) + (org-agenda-buffer-tmp-name (buffer-name))) + (ignore-errors (org-agenda-list nil "TODAY" 1)) + (buffer-substring-no-properties (point-min) (point-max))))) + + (setq --entries + (with-temp-buffer + (let ((inhibit-redisplay t) + (org-agenda-sticky nil) + (org-agenda-buffer-tmp-name (buffer-name)) + (org-agenda-buffer-name (buffer-name)) + (org-agenda-buffer (current-buffer))) + (org-agenda-get-day-entries + (cadr (org-agenda-files nil 'ifmode)) + (calendar-gregorian-from-absolute + (time-to-days (org-read-date nil t "TODAY"))))))) + + (loop for k in (text-properties-at 0 (car --entries)) + by #'cddr + collect k) + + (--map (substring-no-properties (get-text-property 0 'txt it)) --entries) + (--map (get-text-property 0 'time it) --entries) + (current-time) + + (format-time-string "%R" (org-read-date nil t "10:00-11:00")) + + (grfn/start-time-from-range-string "10:00") + + (current-time-string (org-read-date nil t "10:00-11:00")) + + (todo-state + org-habit-p + priority + warntime + ts-date + date + type + org-hd-marker + org-marker + face + undone-face + help-echo + mouse-face + done-face + org-complex-heading-regexp + org-todo-regexp + org-not-done-regexp + dotime + format + extra + time + level + txt + breadcrumbs + duration + time-of-day + org-lowest-priority + org-highest-priority + tags + org-category) + + (propertize) + + --src + ) + + +(defun grfn/org-alert--headline-complete? (headline) + "Return whether HEADLINE has been completed." + (--any? (s-starts-with? it headline) org-done-keywords-for-agenda)) + + +(defun grfn/org-alert--filter-active (deadlines) + "Remove any completed headings from the provided DEADLINES." + (-remove 'grfn/org-alert--headline-complete? deadlines)) + + +(defun grfn/org-alert--strip-states (deadlines) + "Remove the todo states from DEADLINES." + (--map (s-trim (s-chop-prefixes org-todo-keywords-for-agenda it)) deadlines)) + + +(defun grfn/org-alert-check () + "Check for active, due deadlines and initiate notifications." + (interactive) + ;; avoid interrupting current command. + (unless (minibufferp) + (save-window-excursion + (save-excursion + (save-restriction + (let ((active (grfn/org-alert--filter-active (grfn/org-alert--get-headlines)))) + (dolist (dl (grfn/org-alert--strip-states active)) + (alert dl :title grfn/org-alert-notification-title)))))) + (when (get-buffer org-agenda-buffer-name) + (ignore-errors + (with-current-buffer org-agenda-buffer-name + (org-agenda-redo t)))))) + + +(defun grfn/org-alert-enable () + "Enable the notification timer. Cancels existing timer if running." + (interactive) + (grfn/org-alert-disable) + (run-at-time 0 grfn/org-alert-interval 'grfn/org-alert-check)) + + +(defun grfn/org-alert-disable () + "Cancel the running notification timer." + (interactive) + (dolist (timer timer-list) + (if (eq (elt timer 5) 'grfn/org-alert-check) + (cancel-timer timer)))) + + + +(provide 'grfn/org-alert) +;;; grfn/org-alert.el ends here diff --git a/users/grfn/emacs.d/org-config.el b/users/grfn/emacs.d/org-config.el new file mode 100644 index 000000000000..81fb35a1cbda --- /dev/null +++ b/users/grfn/emacs.d/org-config.el @@ -0,0 +1,193 @@ +;;; -*- lexical-binding: t; -*- + +(defun notes-file (f) + (concat org-directory (if (string-prefix-p "/" f) "" "/") f)) + +(defun grfn/org-project-tag->key (tag) + (s-replace-regexp "^project__" "" tag)) + +(defun grfn/org-project-tag->name (tag) + (s-titleized-words + (s-join " " (s-split "_" (grfn/org-project-tag->key tag))))) + +(defun grfn/org-project-tag->keys (tag) + (s-join "" (cons "p" + (-map (lambda (s) (substring-no-properties s 0 1)) + (s-split "_" (grfn/org-project-tag->key tag)))))) + +(defun grfn/org-projects->agenda-commands (project-tags) + (loop for tag in project-tags + collect `(,(grfn/org-project-tag->keys tag) + ,(grfn/org-project-tag->name tag) + tags-todo + ,tag))) + +(defun grfn/org-projects () + (loop for (tag) in + (org-global-tags-completion-table + (directory-files-recursively "~/notes" "\\.org$")) + when (s-starts-with-p "project__" tag) + collect tag)) + +(comment + (grfn/org-projects->agenda-commands (grfn/org-projects)) + ) + +(setq + org-directory (expand-file-name "~/notes") + +org-dir (expand-file-name "~/notes") + org-default-notes-file (concat org-directory "/inbox.org") + +org-default-todo-file (concat org-directory "/inbox.org") + org-agenda-files (directory-files-recursively + "~/notes" "\\.org$") + org-refile-targets '((org-agenda-files :maxlevel . 3)) + org-outline-path-complete-in-steps nil + org-refile-use-outline-path t + org-file-apps `((auto-mode . emacs) + (,(rx (or (and "." (optional "x") (optional "htm") (optional "l") buffer-end) + (and buffer-start "http" (optional "s") "://"))) + . "firefox %s") + (,(rx ".pdf" buffer-end) . "apvlv %s") + (,(rx "." (or "png" + "jpg" + "jpeg" + "gif" + "tif" + "tiff") + buffer-end) + . "feh %s")) + org-log-done 'time + org-archive-location "~/notes/trash::* From %s" + org-cycle-separator-lines 2 + org-hidden-keywords '(title) + org-tags-column -130 + org-ellipsis "…" + org-imenu-depth 9 + org-capture-templates + `(("t" "Todo" entry + (file +org-default-todo-file) + "* TODO %?\n%i" + :kill-buffer t) + + ("m" "Email" entry + (file +org-default-todo-file) + "* TODO [[%L][%:subject]] :email:\n%i") + + ("n" "Notes" entry + (file +org-default-todo-file) + "* %U %?\n%i" + :prepend t + :kill-buffer t) + + ("c" "Task note" entry + (clock) + "* %U %?\n%i[%l[Context]]\n" + :kill-buffer t + :unnarrowed t) + + ("p" "Projects") + ("px" "Xanthous" entry + (file+headline ,(notes-file "xanthous.org") "Backlog") + "* TODO %?\nContext %a\nIn task: %K") + ("pt" "Tvix" entry + (file+headline ,(notes-file "tvix.org") "Tvix TODO") + "* TODO %?\nContext %a\nIn task: %K") + ("pw" "Windtunnel" entry + (file+headline ,(notes-file "windtunnel.org") "Tasks") + "* TODO %i%?\nContext: %a\nIn task: %K") + + ("d" "Data recording") + ) + + org-capture-templates-contexts + `(("px" ((in-file . "/home/grfn/code/depot/users/grfn/xanthous/.*")))) + + org-deadline-warning-days 1 + org-agenda-skip-scheduled-if-deadline-is-shown 'todo + org-todo-keywords '((sequence "TODO(t)" "ACTIVE(a)" "|" "DONE(d)" "RUNNING(r)") + (sequence "NEXT(n)" "WAITING(w)" "LATER(l)" "|" "CANCELLED(c)")) + org-agenda-custom-commands + `(("S" "Sprint Tasks" tags-todo "sprint") + ("i" "Inbox" tags "inbox") + ("r" "Running jobs" todo "RUNNING") + ("w" "@Work" tags-todo "@work") + ("n" . "Next...") + ("np" "Next Sprint" tags-todo "next_sprint|sprint_planning") + + ("p" . "Project...") + ,@(grfn/org-projects->agenda-commands (grfn/org-projects))) + + org-agenda-dim-blocked-tasks nil + org-enforce-todo-dependencies nil + + org-babel-clojure-backend 'cider) + + +(defun +grfn/insert-work-template () + (interactive) + (goto-char (point-min)) + (forward-line) + (insert "#+TODO: TODO(t) NEXT(n) ACTIVE(a) | DONE(d) PR(p) RUNNING(r) TESTING(D) +#+TODO: BLOCKED(b) BACKLOG(l) PROPOSED(o) | CANCELLED(c) +#+FILETAGS: @work +#+FILETAGS: @work +#+PROPERTY: Effort_ALL 0 4:00 8:00 12:00 20:00 32:00 +#+PROPERTY: ESTIMATE_ALL 0 1 2 3 5 8 +#+PROPERTY: STORY-TYPE_ALL Feature Bug Chore +#+PROPERTY: NOBLOCKING t +#+COLUMNS: %TODO %40ITEM(Task) %17EFFORT(Estimated){:} %CLOCKSUM(Time Spent) %17STORY-TYPE(Type) %TAGS")) + +(defun +grfn/insert-org-template () + (interactive) + (pcase (buffer-file-name) + ((s-contains "/work/") (+grfn/insert-work-template)))) + +;;; TODO: this doesn't work? +(define-auto-insert "\\.org?$" #'grfn/insert-org-template t) + +(defun forge--post-submit-around---link-pr-to-org-item + (orig) + (let ((cb (funcall orig))) + (lambda (value headers status req) + (prog1 (funcall cb value headers status req) + (grfn/at-org-clocked-in-item + (let ((url (alist-get 'html_url value)) + (number (alist-get 'number value))) + (org-set-property + "pull-request" + (org-make-link-string + url + (format "%s/%s/%d" + (->> value + (alist-get 'base) + (alist-get 'repo) + (alist-get 'name)) + (->> value + (alist-get 'base) + (alist-get 'repo) + (alist-get 'owner) + (alist-get 'login)) + number))))))))) + +(advice-add + #'forge--post-submit-callback + :around #'forge--post-submit-around---link-pr-to-org-item) + +(defun +grfn/org-setup () + (setq-local truncate-lines -1) + (display-line-numbers-mode -1) + (line-number-mode -1)) + +(add-hook 'org-mode-hook #'+grfn/org-setup) + +(set-face-foreground 'org-block +solarized-s-base00) +(setq whitespace-global-modes '(not org-mode magit-mode vterm-mode)) +(setf (alist-get 'file org-link-frame-setup) 'find-file-other-window) +(set-face-foreground 'org-block +solarized-s-base00) + +;; (add-hook! org-mode +;; (set-company-backend! 'org-mode +;; '(:separate company-ob-postgresql +;; company-dabbrev +;; company-yasnippet +;; company-ispell))) diff --git a/users/grfn/emacs.d/org-gcal.el b/users/grfn/emacs.d/org-gcal.el new file mode 100644 index 000000000000..3e315c5e6046 --- /dev/null +++ b/users/grfn/emacs.d/org-gcal.el @@ -0,0 +1,181 @@ +;;; -*- lexical-binding: t; -*- + +(require 'aio) +(require 'parse-time) + +(setq-local lexical-binding t) +(setq plstore-cache-passphrase-for-symmetric-encryption t) + +(defvar gcal-client-id) +(defvar gcal-client-secret) + +(defvar google-calendar-readonly-scope + "https://www.googleapis.com/auth/calendar.readonly") + +(defvar events-file "/home/grfn/notes/events.org") + +(defun google--get-token (scope client-id client-secret) + (oauth2-auth-and-store + "https://accounts.google.com/o/oauth2/v2/auth" + "https://oauth2.googleapis.com/token" + scope + client-id + client-secret)) + +(cl-defun google--request (url &key method params scope) + (let ((p (aio-promise)) + (auth-token (google--get-token scope gcal-client-id gcal-client-secret))) + (oauth2-refresh-access auth-token) + (oauth2-url-retrieve + auth-token + url + (lambda (&rest _) + (goto-char (point-min)) + (re-search-forward "^$") + (let ((resp (json-parse-buffer :object-type 'alist))) + (aio-resolve p (lambda () resp)))) + nil + (or method "GET") + params) + p)) + +(cl-defun list-events (&key min-time max-time) + (google--request + (concat + "https://www.googleapis.com/calendar/v3/calendars/griffin@urbint.com/events" + "?timeMin=" (format-time-string "%Y-%m-%dT%T%z" min-time) + "&timeMax=" (format-time-string "%Y-%m-%dT%T%z" max-time)) + :scope google-calendar-readonly-scope)) + + +(defun last-week-events () + (list-events :min-time (time-subtract + (current-time) + (seconds-to-time + (* 60 60 24 7))) + :max-time (current-time))) + +(defun next-week-events () + (list-events :min-time (current-time) + :max-time (time-add + (current-time) + (seconds-to-time + (* 60 60 24 7))))) + +(defun attending-event? (event) + (let* ((attendees (append (alist-get 'attendees event) nil)) + (self (--find (alist-get 'self it) attendees))) + (equal "accepted" (alist-get 'responseStatus self)))) + +(defun event->org-headline (event level) + (cl-flet ((make-time + (key) + (when-let ((raw-time (->> event (alist-get key) (alist-get 'dateTime)))) + (format-time-string + (org-time-stamp-format t) + (parse-iso8601-time-string raw-time))))) + (if-let ((start-time (make-time 'start)) + (end-time (make-time 'end))) + (s-format + "${headline} [[${htmlLink}][${summary}]] :event: +${startTime}--${endTime} +:PROPERTIES: +${location-prop} +:EVENT: ${htmlLink} +:END: + +${description}" + (function + (lambda (k m) + (or (alist-get (intern k) m) + (format "key not found: %s" k)))) + (append + event + `((headline . ,(make-string level ?*)) + (startTime . ,start-time) + (endTime . ,end-time) + (location-prop + . ,(if-let ((location (alist-get 'location event))) + (s-lex-format ":LOCATION: ${location}") + ""))))) + ""))) + +(comment + (alist-get 'foo nil) + ) + +(defun write-events (events) + (with-current-buffer (find-file-noselect events-file) + (save-mark-and-excursion + (save-restriction + (widen) + (erase-buffer) + (goto-char (point-min)) + (insert "#+TITLE: Events") + (newline) (newline) + (prog1 + (loop for event in (append events nil) + when (attending-event? event) + do + (insert (event->org-headline event 1)) + (newline) + sum 1) + (org-align-tags t)))))) + +(defun +grfn/sync-events () + (interactive) + (let* ((events (alist-get 'items (aio-wait-for (next-week-events)))) + (num-written (write-events events))) + (message "Successfully wrote %d events" num-written))) + +(comment + ((kind . "calendar#event") + (etag . "\"3174776941020000\"") + (id . "SNIP") + (status . "confirmed") + (htmlLink . "https://www.google.com/calendar/event?eid=SNIP") + (created . "2020-04-01T13:30:09.000Z") + (updated . "2020-04-20T13:14:30.510Z") + (summary . "SNIP") + (description . "SNIP") + (location . "SNIP") + (creator + (email . "griffin@urbint.com") + (self . t)) + (organizer + (email . "griffin@urbint.com") + (self . t)) + (start + (dateTime . "2020-04-01T12:00:00-04:00") + (timeZone . "America/New_York")) + (end + (dateTime . "2020-04-01T12:30:00-04:00") + (timeZone . "America/New_York")) + (recurrence . + ["RRULE:FREQ=WEEKLY;UNTIL=20200408T035959Z;BYDAY=WE"]) + (iCalUID . "SNIP") + (sequence . 0) + (attendees . + [((email . "griffin@urbint.com") + (organizer . t) + (self . t) + (responseStatus . "accepted")) + ((email . "SNIP") + (displayName . "SNIP") + (responseStatus . "needsAction"))]) + (extendedProperties + (private + (origRecurringId . "309q48kc1dihsvbi13pnlimb5a")) + (shared + (origRecurringId . "309q48kc1dihsvbi13pnlimb5a"))) + (reminders + (useDefault . t))) + + (require 'icalendar) + + (icalendar--convert-recurring-to-diary + nil + "RRULE:FREQ=WEEKLY;UNTIL=20200408T035959Z;BYDAY=WE" + ) + + ) diff --git a/users/grfn/emacs.d/org-query.el b/users/grfn/emacs.d/org-query.el new file mode 100644 index 000000000000..0b987280f2a1 --- /dev/null +++ b/users/grfn/emacs.d/org-query.el @@ -0,0 +1,100 @@ +;;; -*- lexical-binding: t; -*- + +(require 'org) +(require 'org-agenda) +(require 'inflections) + +(defun grfn/org-agenda-entry->element (agenda-entry) + ;; ??? + ()) + +(defun org-elements-agenda-match (match &optional todo-only) + (setq match + (propertize match 'inherited t)) + (with-temp-buffer + (let ((inhibit-redisplay (not debug-on-error)) + (org-agenda-sticky nil) + (org-agenda-buffer-tmp-name (buffer-name)) + (org-agenda-buffer-name (buffer-name)) + (org-agenda-buffer (current-buffer)) + (matcher (org-make-tags-matcher match)) + result) + (org-agenda-prepare (concat "TAGS " match)) + (setq match (car matcher) + matcher (cdr matcher)) + (dolist (file (org-agenda-files nil 'ifmode) + result) + (catch 'nextfile + (org-check-agenda-file file) + (when-let ((buffer (if (file-exists-p file) + (org-get-agenda-file-buffer file) + (error "No such file %s" file)))) + (with-current-buffer buffer + (unless (derived-mode-p 'org-mode) + (error "Agenda file %s is not in Org mode" file)) + (save-excursion + (save-restriction + (if (eq buffer org-agenda-restrict) + (narrow-to-region org-agenda-restrict-begin + org-agenda-restrict-end) + (widen)) + (setq result + (append result (org-scan-tags + 'agenda + matcher + todo-only)))))))))))) + +(defun grfn/num-inbox-items () + (length (org-elements-agenda-match "inbox" t))) + +(defun grfn/num-inbox-items-message () + (let ((n (grfn/num-inbox-items))) + (if (zerop n) "" + (format "%d %s" + n + (if (= 1 n) "item" "items"))))) + +(defmacro grfn/at-org-clocked-in-item (&rest body) + `(when (org-clocking-p) + (let ((m org-clock-marker)) + (with-current-buffer (marker-buffer m) + (save-mark-and-excursion + (goto-char m) + (org-back-to-heading t) + ,@body))))) + +(defun grfn/org-element-clocked-in-task () + (grfn/at-org-clocked-in-item + (org-element-at-point))) + +(comment + (grfn/org-element-clocked-in-task) + (org-element-property :title (grfn/org-element-clocked-in-task)) + ) + +(defun grfn/minutes->hours:minutes (minutes) + (format "%d:%02d" + (floor (/ minutes 60)) + (mod minutes 60))) + +(comment + (grfn/minutes->hours:minutes 1) ; => "0:01" + (grfn/minutes->hours:minutes 15) ; => "0:15" + (grfn/minutes->hours:minutes 130) ; => "2:10" + ) + +(defun grfn/org-current-clocked-in-task-message () + (if (org-clocking-p) + (format "(%s) [%s]" + (->> (grfn/org-element-clocked-in-task) + (org-element-property :title) + (car) + (substring-no-properties) + (s-trim)) + (grfn/minutes->hours:minutes + (org-clock-get-clocked-time))) + "")) + +(comment + (grfn/org-current-clocked-in-task-message) + ) diff --git a/users/grfn/emacs.d/packages.el b/users/grfn/emacs.d/packages.el new file mode 100644 index 000000000000..5a580cad17e0 --- /dev/null +++ b/users/grfn/emacs.d/packages.el @@ -0,0 +1,152 @@ +;; -*- no-byte-compile: t; -*- +;;; private/grfn/packages.el + +(package! moody) + +;; Editor +(package! solarized-theme) +(package! fill-column-indicator) +(package! flx) +(package! general + :recipe (:host github :repo "noctuid/general.el")) +(package! fill-column-indicator) +(package! writeroom-mode) +(package! dash) +(package! w3m) +(package! rainbow-mode) +(package! string-inflection) + +;;; Org +(package! org-tracker + :recipe (:host file + :local-repo "~/code/org-tracker")) +(package! org-alert) +(package! ob-http) +(package! ob-ipython) +(package! ob-async) +(package! org-recent-headings) +(package! org-sticky-header) +(package! gnuplot) +(package! gnuplot-mode) + +;; Presentation +(package! epresent) +(package! org-tree-slide) +(package! ox-reveal) + +;; Slack etc +(package! slack) +(package! alert) + +;; Git +(package! evil-magit) +(package! marshal) +(package! forge) +(package! + github-review + :recipe + (:host github + :repo "charignon/github-review" + :files ("github-review.el"))) + +;; Elisp +(package! dash) +(package! dash-functional) +(package! s) +(package! request) +(package! predd + :recipe (:host github :repo "skeeto/predd")) +(package! aio) + +;; Haskell +(package! lsp-haskell) +(package! counsel-etags) + +;;; LSP +(package! lsp-mode) +(package! lsp-ui :recipe (:host github :repo "emacs-lsp/lsp-ui")) +(package! company-lsp) +(package! lsp-treemacs) + +;; Rust +(package! rustic :disable t) +;; (package! racer :disable t) +(package! cargo) + +;; Lisp +(package! paxedit) + +;; Javascript +(package! flow-minor-mode) +(package! flycheck-flow) +(package! company-flow) +(package! prettier-js) + +;; GraphQL +(package! graphql-mode) + +;; Haskell +(package! lsp-mode) +(package! lsp-ui) +(package! lsp-haskell) +(package! company-lsp) +;; (package! lsp-imenu) + +;; Clojure +(package! flycheck-clojure) + +;; SQL +(package! sqlup-mode) +(package! emacsql) +(package! emacsql-psql) + +;;; Python +(package! pyimport) +;; (package! yapfify) +(package! blacken) + + +;;; Desktop interaction +(package! counsel-spotify) + +;;; Dhall +(package! dhall-mode) + +;;; Kubernetes +(package! kubernetes) +(package! kubernetes-evil) +(package! k8s-mode) + +;;; Stack Exchange +(package! sx) + +;;; Nix +(package! nix-update + :recipe (:host github + :repo "glittershark/nix-update-el")) +(package! direnv) + +;;; Sequence diagrams +(package! wsd-mode + :recipe (:host github + :repo "josteink/wsd-mode")) + +;;; logic? +(package! metal-mercury-mode + :recipe (:host github + :repo "ahungry/metal-mercury-mode")) +(package! flycheck-mercury) + +(package! terraform-mode) +(package! company-terraform) + +(package! jsonnet-mode) + +;;; +(package! znc + :recipe (:host github + :repo "sshirokov/ZNC.el")) + +;;; cpp +(package! protobuf-mode) +(package! clang-format+) diff --git a/users/grfn/emacs.d/rust.el b/users/grfn/emacs.d/rust.el new file mode 100644 index 000000000000..689d843e78e1 --- /dev/null +++ b/users/grfn/emacs.d/rust.el @@ -0,0 +1,39 @@ +;;; -*- lexical-binding: t; -*- + +(add-to-list 'auto-mode-alist '("\\.rs$" . rust-mode)) + +(defun grfn/rust-setup () + (interactive) + + (push '(?> . ("<" . ">")) evil-surround-pairs-alist) + (push '(?< . ("< " . " >")) evil-surround-pairs-alist) + + (setq lsp-rust-server 'rust-analyzer) + (setq-local whitespace-line-column 100 + fill-column 100) + (setq rust-format-show-buffer nil) + (setq lsp-rust-analyzer-import-merge-behaviour "last" + lsp-rust-analyzer-cargo-watch-command "clippy" + lsp-rust-analyzer-cargo-watch-args ["--target-dir" "/home/grfn/code/readyset/readyset/target/rust-analyzer"] + lsp-ui-doc-enable t) + (rust-enable-format-on-save) + (lsp)) + +(add-hook 'rust-mode-hook #'grfn/rust-setup) + +(map! + (:map rust-mode-map + :n "g RET" #'lsp-rust-analyzer-run + :n "g R" #'lsp-find-references + :n "g d" #'lsp-find-definition + (:localleader + "m" #'lsp-rust-analyzer-expand-macro))) + +(comment + (flycheck-get-next-checkers 'lsp) + (flycheck-add-next-checker) + (flycheck-get-next-checkers 'lsp) + ) + +(set-company-backend! 'rust-mode + '(:separate company-capf company-yasnippet)) diff --git a/users/grfn/emacs.d/show-matching-paren.el b/users/grfn/emacs.d/show-matching-paren.el new file mode 100644 index 000000000000..ab65a912a8d1 --- /dev/null +++ b/users/grfn/emacs.d/show-matching-paren.el @@ -0,0 +1,61 @@ +;;; -*- lexical-binding: t; -*- + +;;; https://with-emacs.com/posts/ui-hacks/show-matching-lines-when-parentheses-go-off-screen/ + +;; we will call `blink-matching-open` ourselves... +(remove-hook 'post-self-insert-hook + #'blink-paren-post-self-insert-function) +;; this still needs to be set for `blink-matching-open` to work +(setq blink-matching-paren 'show) + +(let ((ov nil)) ; keep track of the overlay + (advice-add + #'show-paren-function + :after + (defun show-paren--off-screen+ (&rest _args) + "Display matching line for off-screen paren." + (when (overlayp ov) + (delete-overlay ov)) + ;; check if it's appropriate to show match info, + ;; see `blink-paren-post-self-insert-function' + (when (and (overlay-buffer show-paren--overlay) + (not (or cursor-in-echo-area + executing-kbd-macro + noninteractive + (minibufferp) + this-command)) + (and (not (bobp)) + (memq (char-syntax (char-before)) '(?\) ?\$))) + (= 1 (logand 1 (- (point) + (save-excursion + (forward-char -1) + (skip-syntax-backward "/\\") + (point)))))) + ;; rebind `minibuffer-message' called by + ;; `blink-matching-open' to handle the overlay display + (cl-letf (((symbol-function #'minibuffer-message) + (lambda (msg &rest args) + (let ((msg (apply #'format-message msg args))) + (setq ov (display-line-overlay+ + (window-start) msg )))))) + (blink-matching-open)))))) + +(defun display-line-overlay+ (pos str &optional face) + "Display line at POS as STR with FACE. + +FACE defaults to inheriting from default and highlight." + (let ((ol (save-excursion + (goto-char pos) + (make-overlay (line-beginning-position) + (line-end-position))))) + (overlay-put ol 'display str) + (overlay-put ol 'face + (or face '(:inherit default :inherit highlight))) + ol)) + +(setq show-paren-style 'paren + show-paren-delay 0.03 + show-paren-highlight-openparen t + show-paren-when-point-inside-paren nil + show-paren-when-point-in-periphery t) +(show-paren-mode 1) diff --git a/users/grfn/emacs.d/slack-snippets.el b/users/grfn/emacs.d/slack-snippets.el new file mode 100644 index 000000000000..b5bd4db7482c --- /dev/null +++ b/users/grfn/emacs.d/slack-snippets.el @@ -0,0 +1,227 @@ +;;; -*- lexical-binding: t; -*- + +(require 'dash) +(require 'dash-functional) +(require 'request) + +;;; +;;; Configuration +;;; + +(defvar slack/token nil + "Legacy (https://api.slack.com/custom-integrations/legacy-tokens) access token") + +(defvar slack/include-public-channels 't + "Whether or not to inclue public channels in the list of conversations") + +(defvar slack/include-private-channels 't + "Whether or not to inclue public channels in the list of conversations") + +(defvar slack/include-im 't + "Whether or not to inclue IMs (private messages) in the list of conversations") + +(defvar slack/include-mpim nil + "Whether or not to inclue multi-person IMs (multi-person private messages) in + the list of conversations") + +;;; +;;; Utilities +;;; + +(defmacro comment (&rest _body) + "Comment out one or more s-expressions" + nil) + +(defun ->list (vec) (append vec nil)) + +(defun json-truthy? (x) (and x (not (equal :json-false x)))) + +;;; +;;; Generic API integration +;;; + +(defvar slack/base-url "https://slack.com/api") + +(defun slack/get (path params &optional callback) + "params is an alist of query parameters" + (let* ((params-callback (if (functionp params) `(() . ,params) (cons params callback))) + (params (car params-callback)) (callback (cdr params-callback)) + (params (append `(("token" . ,slack/token)) params)) + (url (concat (file-name-as-directory slack/base-url) path))) + (request url + :type "GET" + :params params + :parser 'json-read + :success (cl-function + (lambda (&key data &allow-other-keys) + (funcall callback data)))))) + +(defun slack/post (path params &optional callback) + (let* ((params-callback (if (functionp params) `(() . ,params) (cons params callback))) + (params (car params-callback)) (callback (cdr params-callback)) + (url (concat (file-name-as-directory slack/base-url) path))) + (request url + :type "POST" + :data (json-encode params) + :headers `(("Content-Type" . "application/json") + ("Authorization" . ,(format "Bearer %s" slack/token))) + :success (cl-function + (lambda (&key data &allow-other-keys) + (funcall callback data)))))) + + +;;; +;;; Specific API endpoints +;;; + +;; Users + +(defun slack/users (cb) + "Returns users as (id . name) pairs" + (slack/get + "users.list" + (lambda (data) + (->> data + (assoc-default 'members) + ->list + (-map (lambda (user) + (cons (assoc-default 'id user) + (assoc-default 'real_name user)))) + (-filter #'cdr) + (funcall cb))))) + +(comment + (slack/get + "users.list" + (lambda (data) (setq response-data data))) + + (slack/users (lambda (data) (setq --users data))) + + ) + +;; Conversations + +(defun slack/conversation-types () + (->> + (list (when slack/include-public-channels "public_channel") + (when slack/include-private-channels "private_channel") + (when slack/include-im "im") + (when slack/include-mpim "mpim")) + (-filter #'identity) + (s-join ","))) + +(defun channel-label (chan users-alist) + (cond + ((json-truthy? (assoc-default 'is_channel chan)) + (format "#%s" (assoc-default 'name chan))) + ((json-truthy? (assoc-default 'is_im chan)) + (let ((user-id (assoc-default 'user chan))) + (format "Private message with %s" (assoc-default user-id users-alist)))) + ((json-truthy? (assoc-default 'is_mpim chan)) + (->> chan + (assoc-default 'purpose) + (assoc-default 'value))))) + +(defun slack/conversations (cb) + "Calls `cb' with (id . '((label . \"label\") '(topic . \"topic\") '(purpose . \"purpose\"))) pairs" + (slack/get + "conversations.list" + `(("types" . ,(slack/conversation-types)) + ("exclude-archived" . "true")) + (lambda (data) + (setq --data data) + (slack/users + (lambda (users) + (->> data + (assoc-default 'channels) + ->list + (-map + (lambda (chan) + (cons (assoc-default 'id chan) + `((label . ,(channel-label chan users)) + (topic . ,(->> chan + (assoc-default 'topic) + (assoc-default 'value))) + (purpose . ,(->> chan + (assoc-default 'purpose) + (assoc-default 'value))))))) + (funcall cb))))))) + +(comment + (slack/get + "conversations.list" + '(("types" . "public_channel,private_channel,im,mpim")) + (lambda (data) (setq response-data data))) + + (slack/get + "conversations.list" + '(("types" . "im")) + (lambda (data) (setq response-data data))) + + (slack/conversations + (lambda (convos) (setq --conversations convos))) + + ) + +;; Messages + +(cl-defun slack/post-message + (&key text channel-id (on-success #'identity)) + (slack/post "chat.postMessage" + `((text . ,text) + (channel . ,channel-id) + (as_user . t)) + on-success)) + +(comment + + (slack/post-message + :text "hi slackbot" + :channel-id slackbot-channel-id + :on-success (lambda (data) (setq resp data))) + + ) + +;;; +;;; Posting code snippets to slack +;;; + +(defun prompt-for-channel (cb) + (slack/conversations + (lambda (conversations) + (ivy-read + "Select channel: " + ;; TODO want to potentially use purpose / topic stuff here + (->> conversations + (-filter (lambda (c) (assoc-default 'label (cdr c)))) + (-map (lambda (chan) (let ((label (assoc-default 'label (cdr chan))) + (id (car chan))) + (propertize label 'channel-id id))))) + :history 'slack/channel-history + :action (lambda (selected) + (let ((channel-id (get-text-property 0 'channel-id selected))) + (funcall cb channel-id) + (message "Sent message to %s" selected)))))) + nil) + +(comment + (prompt-for-channel #'message) + (->> --convos + (-filter (lambda (c) (assoc-default 'label (cdr c)))) + (-map (lambda (chan) (let ((label (assoc-default 'label (cdr chan))) + (id (car chan))) + (propertize label 'channel-id id))))) + + (->> --convos (car) (cdr) (assoc-default 'label)) + ) + +(defun slack-send-code-snippet (&optional snippet-text) + (interactive + (list (buffer-substring-no-properties (mark) (point)))) + (prompt-for-channel + (lambda (channel-id) + (slack/post-message + :text (format "```\n%s```" snippet-text) + :channel-id channel-id)))) + +(provide 'slack-snippets) diff --git a/users/grfn/emacs.d/slack.el b/users/grfn/emacs.d/slack.el new file mode 100644 index 000000000000..54d3b40b099c --- /dev/null +++ b/users/grfn/emacs.d/slack.el @@ -0,0 +1,24 @@ +;;; -*- lexical-binding: t; -*- + +(after! slack + (set-face-foreground 'slack-message-output-header +solarized-s-base01) + (set-face-attribute 'slack-message-output-header nil :underline nil) + (set-face-attribute 'slack-message-output-text nil :height 1.0)) + +(require 'slack) +(setq slack-buffer-emojify 't + slack-prefer-current-team 't + slack-thread-also-send-to-room nil) + +(set-popup-rule! "^\\*Slack" + :quit nil + :select t + :side 'bottom + :ttl nil + :size 0.5) + +(add-hook #'slack-message-buffer-mode-hook + (lambda () (toggle-truncate-lines -1))) + +(map! (:map slack-message-buffer-mode-map + :n "q" #'delete-window)) diff --git a/users/grfn/emacs.d/snippets/haskell-mode/annotation b/users/grfn/emacs.d/snippets/haskell-mode/annotation new file mode 100644 index 000000000000..8a2854d759df --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/annotation @@ -0,0 +1,5 @@ +# key: ann +# name: annotation +# expand-env: ((yas-indent-line 'fixed)) +# -- +{-# ANN ${1:module} ("${2:HLint: ignore ${3:Reduce duplication}}" :: String) #-} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/benchmark-module b/users/grfn/emacs.d/snippets/haskell-mode/benchmark-module new file mode 100644 index 000000000000..cbb1646e41d1 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/benchmark-module @@ -0,0 +1,26 @@ +# key: bench +# name: benchmark-module +# expand-env: ((yas-indent-line (quote fixed))) +# -- +-------------------------------------------------------------------------------- +module ${1:`(if (not buffer-file-name) "Module" + (let ((name (file-name-sans-extension (buffer-file-name))) + (case-fold-search nil)) + (if (cl-search "bench/" name) + (replace-regexp-in-string "/" "." + (replace-regexp-in-string "^\/[^A-Z]*" "" + (car (last (split-string name "src"))))) + (file-name-nondirectory name))))`} ( benchmark, main ) where +-------------------------------------------------------------------------------- +import Bench.Prelude +-------------------------------------------------------------------------------- +import ${1:$(s-chop-suffix "Bench" yas-text)} +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain [benchmark] + +-------------------------------------------------------------------------------- + +benchmark :: Benchmark +benchmark = bgroup "${1:$(->> yas-text (s-chop-suffix "Bench") (s-split ".") -last-item)}" [bench "something dumb" $ nf (1 +) (1 :: Int)] diff --git a/users/grfn/emacs.d/snippets/haskell-mode/header b/users/grfn/emacs.d/snippets/haskell-mode/header new file mode 100644 index 000000000000..fdd8250d86ca --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/header @@ -0,0 +1,5 @@ +# key: hh +# name: header +# expand-env: ((yas-indent-line 'fixed)) +# -- +--------------------------------------------------------------------------------$2 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/hedgehog-generator b/users/grfn/emacs.d/snippets/haskell-mode/hedgehog-generator new file mode 100644 index 000000000000..68863f70542b --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/hedgehog-generator @@ -0,0 +1,8 @@ +# key: gen +# name: Hedgehog Generator +# expand-env: ((yas-indent-line (quote fixed))) +# -- +gen${1:Foo} :: Gen $1 +gen$1 = do + $2 + pure $1{..} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/hedgehog-property b/users/grfn/emacs.d/snippets/haskell-mode/hedgehog-property new file mode 100644 index 000000000000..bf39a2a3eecb --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/hedgehog-property @@ -0,0 +1,9 @@ +# -*- mode: snippet -*- +# name: Hedgehog Property +# key: hprop +# expand-env: ((yas-indent-line 'fixed)) +# -- +hprop_${1:somethingIsAlwaysTrue} :: Property +hprop_$1 = property $ do + ${2:x} <- forAll ${3:Gen.int $ Range.linear 1 100} + ${4:x === x} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/hlint b/users/grfn/emacs.d/snippets/haskell-mode/hlint new file mode 100644 index 000000000000..74b63dc672e4 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/hlint @@ -0,0 +1,8 @@ +# -*- mode: snippet -*- +# name: hlint +# uuid: +# expand-env: ((yas-indent-line 'fixed)) +# key: hlint +# condition: t +# -- +{-# ANN module ("Hlint: ignore $1" :: String) #- } \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/import-i b/users/grfn/emacs.d/snippets/haskell-mode/import-i new file mode 100644 index 000000000000..4a7fca2c2fd6 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/import-i @@ -0,0 +1,4 @@ +# key: i +# name: import-i +# -- +import ${1:Prelude} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/inl b/users/grfn/emacs.d/snippets/haskell-mode/inl new file mode 100644 index 000000000000..6e17b83d7114 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/inl @@ -0,0 +1,6 @@ +# -*- mode: snippet -*- +# name: inl +# key: inl +# expand-env: ((yas-indent-line 'fixed)) +# -- +{-# INLINE $1 #-} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/inline b/users/grfn/emacs.d/snippets/haskell-mode/inline new file mode 100644 index 000000000000..1beafbe50b56 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/inline @@ -0,0 +1,5 @@ +# key: inline +# name: inline +# expand-env: ((yas-indent-line 'fixed)) +# -- +{-# INLINE $1 #-} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/language pragma b/users/grfn/emacs.d/snippets/haskell-mode/language pragma new file mode 100644 index 000000000000..6f84720f4511 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/language pragma @@ -0,0 +1,6 @@ +# -*- mode: snippet -*- +# name: language pragma +# key: lang +# expand-env: ((yas-indent-line 'fixed)) +# -- +{-# LANGUAGE $1 #-} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/lens.field b/users/grfn/emacs.d/snippets/haskell-mode/lens.field new file mode 100644 index 000000000000..b22ea3d2e888 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/lens.field @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: lens.field +# key: lens +# expand-env: ((yas-indent-line 'fixed)) +# -- +${1:field} :: Lens' ${2:Source} ${3:Target} +$1 = lens _${4:sourceField} $ \\${2:$(-> yas-text s-word-initials s-downcase)} ${4:$(-> yas-text s-word-initials s-downcase)} -> ${2:$(-> yas-text s-word-initials s-downcase)} { _$4 = ${4:$(-> yas-text s-word-initials s-downcase)} } \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/module b/users/grfn/emacs.d/snippets/haskell-mode/module new file mode 100644 index 000000000000..4554d33f9ba7 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/module @@ -0,0 +1,32 @@ +# -*- mode: snippet -*- +# key: module +# name: module +# condition: (= (length "module") (current-column)) +# expand-env: ((yas-indent-line 'fixed)) +# contributor: Luke Hoersten <luke@hoersten.org> +# -- +-------------------------------------------------------------------------------- +-- | +-- Module : $1 +-- Description : $2 +-- Maintainer : Griffin Smith <grfn@urbint.com> +-- Maturity : ${3:Draft, Usable, Maintained, OR MatureAF} +-- +-- $4 +-------------------------------------------------------------------------------- +module ${1:`(if (not buffer-file-name) "Module" + (let ((name (file-name-sans-extension (buffer-file-name))) + (case-fold-search nil)) + (if (or (cl-search "src/" name) + (cl-search "test/" name)) + (replace-regexp-in-string "/" "." + (replace-regexp-in-string "^\/[^A-Z]*" "" + (car (last (split-string name "src"))))) + (file-name-nondirectory name))))`} + ( + ) where +-------------------------------------------------------------------------------- +import Prelude +-------------------------------------------------------------------------------- + +$0 diff --git a/users/grfn/emacs.d/snippets/haskell-mode/shut up, hlint b/users/grfn/emacs.d/snippets/haskell-mode/shut up, hlint new file mode 100644 index 000000000000..fccff1d66f29 --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/shut up, hlint @@ -0,0 +1,6 @@ +# -*- mode: snippet -*- +# name: shut up, hlint +# key: dupl +# expand-env: ((yas-indent-line 'fixed)) +# -- +{-# ANN module ("HLint: ignore Reduce duplication" :: String) #-} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/test-group b/users/grfn/emacs.d/snippets/haskell-mode/test-group new file mode 100644 index 000000000000..948e90d9e02e --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/test-group @@ -0,0 +1,9 @@ +# -*- mode: snippet -*- +# name: test-group +# uuid: +# key: testGroup +# condition: t +# -- +testGroup "${1:name}" +[ $0 +] \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/test-module b/users/grfn/emacs.d/snippets/haskell-mode/test-module new file mode 100644 index 000000000000..036b0ae9983a --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/test-module @@ -0,0 +1,27 @@ +# -*- mode: snippet -*- +# name: test-module +# key: test +# expand-env: ((yas-indent-line 'fixed)) +# -- +-------------------------------------------------------------------------------- +module ${1:`(if (not buffer-file-name) "Module" + (let ((name (file-name-sans-extension (buffer-file-name))) + (case-fold-search nil)) + (if (cl-search "test/" name) + (replace-regexp-in-string "/" "." + (replace-regexp-in-string "^\/[^A-Z]*" "" + (car (last (split-string name "src"))))) + (file-name-nondirectory name))))`} (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import ${1:$(s-chop-suffix "Spec" yas-text)} +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "$1" + [ $0 + ] \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/haskell-mode/undefined b/users/grfn/emacs.d/snippets/haskell-mode/undefined new file mode 100644 index 000000000000..7bcd99b5716c --- /dev/null +++ b/users/grfn/emacs.d/snippets/haskell-mode/undefined @@ -0,0 +1,6 @@ +# -*- mode: snippet -*- +# name: undefined +# key: u +# expand-env: ((yas-indent-line 'fixed) (yas-wrap-around-region 'nil)) +# -- +undefined$1 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/action-type b/users/grfn/emacs.d/snippets/js2-mode/action-type new file mode 100644 index 000000000000..ef8d1a3863ee --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/action-type @@ -0,0 +1,4 @@ +# key: at +# name: action-type +# -- +export const ${1:FOO_BAR$(->> yas-text s-upcase (s-replace-all '(("-" . "_") (" " . "_"))))}: '${3:ns}/${1:$(-> yas-text s-dashed-words)}' = '$3/${1:$(-> yas-text s-dashed-words)}'$5 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/before b/users/grfn/emacs.d/snippets/js2-mode/before new file mode 100644 index 000000000000..4569b6583143 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/before @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: before +# key: bef +# -- +before(function() { + $1 +}) diff --git a/users/grfn/emacs.d/snippets/js2-mode/context b/users/grfn/emacs.d/snippets/js2-mode/context new file mode 100644 index 000000000000..d83809f3c35e --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/context @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: context +# key: context +# -- +context('$1', function() { + $2 +}) diff --git a/users/grfn/emacs.d/snippets/js2-mode/describe b/users/grfn/emacs.d/snippets/js2-mode/describe new file mode 100644 index 000000000000..bd0198181d02 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/describe @@ -0,0 +1,6 @@ +# key: desc +# name: describe +# -- +describe('$1', () => { + $2 +}) \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/expect b/users/grfn/emacs.d/snippets/js2-mode/expect new file mode 100644 index 000000000000..eba41ef3309d --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/expect @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: expect +# key: ex +# -- +expect($1).$2 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/function b/users/grfn/emacs.d/snippets/js2-mode/function new file mode 100644 index 000000000000..b423044b4410 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/function @@ -0,0 +1,6 @@ +# key: f +# name: function +# -- +function $1($2) { + $3 +} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/header b/users/grfn/emacs.d/snippets/js2-mode/header new file mode 100644 index 000000000000..3e303764cb0b --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/header @@ -0,0 +1,6 @@ +# -*- mode: snippet -*- +# name: header +# key: hh +# expand-env: ((yas-indent-line 'fixed)) +# -- +//////////////////////////////////////////////////////////////////////////////// diff --git a/users/grfn/emacs.d/snippets/js2-mode/it b/users/grfn/emacs.d/snippets/js2-mode/it new file mode 100644 index 000000000000..a451cfc08a90 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/it @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: it +# key: it +# -- +it('$1', () => { + $2 +}) \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/it-pending b/users/grfn/emacs.d/snippets/js2-mode/it-pending new file mode 100644 index 000000000000..00da312e1096 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/it-pending @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: it-pending +# key: xi +# -- +it('$1')$0 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/module b/users/grfn/emacs.d/snippets/js2-mode/module new file mode 100644 index 000000000000..dc79819d8979 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/module @@ -0,0 +1,12 @@ +# key: module +# name: module +# expand-env: ((yas-indent-line (quote fixed))) +# condition: (= (length "module") (current-column)) +# -- +/** + * @fileOverview $1 + * @name ${2:`(file-name-nondirectory (buffer-file-name))`} + * @author Griffin Smith + * @license Proprietary + */ +$3 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/record b/users/grfn/emacs.d/snippets/js2-mode/record new file mode 100644 index 000000000000..0bb0f024367b --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/record @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: record +# key: rec +# -- +export default class $1 extends Record({ + $2 +}) {} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/js2-mode/test b/users/grfn/emacs.d/snippets/js2-mode/test new file mode 100644 index 000000000000..938d490a74e8 --- /dev/null +++ b/users/grfn/emacs.d/snippets/js2-mode/test @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: test +# key: test +# -- +test('$1', () => { + $2 +}) \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/nix-mode/fetchFromGitHub b/users/grfn/emacs.d/snippets/nix-mode/fetchFromGitHub new file mode 100644 index 000000000000..9b9373573048 --- /dev/null +++ b/users/grfn/emacs.d/snippets/nix-mode/fetchFromGitHub @@ -0,0 +1,12 @@ +# -*- mode: snippet -*- +# name: fetchFromGitHub +# uuid: +# key: fetchFromGitHub +# condition: t +# -- +fetchFromGitHub { + owner = "$1"; + repo = "$2"; + rev = "$3"; + sha256 = "0000000000000000000000000000000000000000000000000000"; +} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/nix-mode/pythonPackage b/users/grfn/emacs.d/snippets/nix-mode/pythonPackage new file mode 100644 index 000000000000..0a74c21e1857 --- /dev/null +++ b/users/grfn/emacs.d/snippets/nix-mode/pythonPackage @@ -0,0 +1,16 @@ +# key: pypkg +# name: pythonPackage +# condition: t +# -- +${1:pname} = buildPythonPackage rec { + name = "\${pname}-\${version}"; + pname = "$1"; + version = "${2:1.0.0}"; + src = fetchPypi { + inherit pname version; + sha256 = "0000000000000000000000000000000000000000000000000000"; + }; + propagatedBuildInputs = with pythonSelf; [ + $3 + ]; +}; \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/nix-mode/sha256 b/users/grfn/emacs.d/snippets/nix-mode/sha256 new file mode 100644 index 000000000000..e3d52e1c0201 --- /dev/null +++ b/users/grfn/emacs.d/snippets/nix-mode/sha256 @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: sha256 +# uuid: +# key: sha256 +# condition: t +# -- +sha256 = "0000000000000000000000000000000000000000000000000000"; \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/org-mode/SQL source block b/users/grfn/emacs.d/snippets/org-mode/SQL source block new file mode 100644 index 000000000000..b5d43fd6bc01 --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/SQL source block @@ -0,0 +1,6 @@ +# key: sql +# name: SQL source block +# -- +#+BEGIN_SRC sql ${1::async} +$2 +#+END_SRC diff --git a/users/grfn/emacs.d/snippets/org-mode/combat b/users/grfn/emacs.d/snippets/org-mode/combat new file mode 100644 index 000000000000..ef46062d09b4 --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/combat @@ -0,0 +1,13 @@ +# -*- mode: snippet -*- +# name: combat +# uuid: +# key: combat +# condition: t +# -- +| | initiative | max hp | current hp | status | | +|-------------+------------+--------+------------+--------+------| +| Barty Barty | | | | | <--- | +| Hectoroth | | | | | | +| Xanadu | | | | | | +| Aurora | | | | | | +| EFB | | | | | | \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/org-mode/date b/users/grfn/emacs.d/snippets/org-mode/date new file mode 100644 index 000000000000..297529cdac64 --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/date @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# key: date +# name: date.org +# -- +[`(format-time-string "%Y-%m-%d")`]$0 diff --git a/users/grfn/emacs.d/snippets/org-mode/date-time b/users/grfn/emacs.d/snippets/org-mode/date-time new file mode 100644 index 000000000000..fde469276c3f --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/date-time @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: date-time +# key: dt +# -- +[`(format-time-string "%Y-%m-%d %H:%m:%S")`] \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/org-mode/description b/users/grfn/emacs.d/snippets/org-mode/description new file mode 100644 index 000000000000..a43bc95cc3ed --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/description @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: description +# key: desc +# -- +:DESCRIPTION: +$1 +:END: diff --git a/users/grfn/emacs.d/snippets/org-mode/nologdone b/users/grfn/emacs.d/snippets/org-mode/nologdone new file mode 100644 index 000000000000..e5be85d6b3c0 --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/nologdone @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: nologdone +# key: nologdone +# -- +#+STARTUP: nologdone$0 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/org-mode/python source block b/users/grfn/emacs.d/snippets/org-mode/python source block new file mode 100644 index 000000000000..247ae51b0b78 --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/python source block @@ -0,0 +1,6 @@ +# key: py +# name: Python source block +# -- +#+BEGIN_SRC python +$0 +#+END_SRC \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/org-mode/reveal b/users/grfn/emacs.d/snippets/org-mode/reveal new file mode 100644 index 000000000000..1bdbdfa5dc36 --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/reveal @@ -0,0 +1,6 @@ +# key: reveal +# name: reveal +# condition: t +# -- +#+ATTR_REVEAL: :frag ${1:roll-in} +$0 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/org-mode/transaction b/users/grfn/emacs.d/snippets/org-mode/transaction new file mode 100644 index 000000000000..37f2dd31caff --- /dev/null +++ b/users/grfn/emacs.d/snippets/org-mode/transaction @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: transaction +# key: begin +# -- +BEGIN; +$0 +ROLLBACK; \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/python-mode/add_column b/users/grfn/emacs.d/snippets/python-mode/add_column new file mode 100644 index 000000000000..47e83850d5b7 --- /dev/null +++ b/users/grfn/emacs.d/snippets/python-mode/add_column @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: add_column +# key: op.add_column +# -- +op.add_column('${1:table}', sa.Column('${2:name}', sa.${3:String()}))$0 diff --git a/users/grfn/emacs.d/snippets/python-mode/decorate b/users/grfn/emacs.d/snippets/python-mode/decorate new file mode 100644 index 000000000000..9448b45c9623 --- /dev/null +++ b/users/grfn/emacs.d/snippets/python-mode/decorate @@ -0,0 +1,15 @@ +# -*- mode: snippet -*- +# name: decorate +# uuid: +# key: decorate +# condition: t +# -- +def wrap(inner): + @wraps(inner) + def wrapped(*args, **kwargs): + ret = inner(*args, **kwargs) + return ret + + return wrapped + +return wrap \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/python-mode/dunder b/users/grfn/emacs.d/snippets/python-mode/dunder new file mode 100644 index 000000000000..c49ec40a15cc --- /dev/null +++ b/users/grfn/emacs.d/snippets/python-mode/dunder @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: dunder +# uuid: +# key: du +# condition: t +# -- +__$1__$0 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/python-mode/name b/users/grfn/emacs.d/snippets/python-mode/name new file mode 100644 index 000000000000..eca6d60b481f --- /dev/null +++ b/users/grfn/emacs.d/snippets/python-mode/name @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: name +# uuid: +# key: name +# condition: t +# -- +__name__ \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/python-mode/op.get_bind.execute b/users/grfn/emacs.d/snippets/python-mode/op.get_bind.execute new file mode 100644 index 000000000000..aba801c6baf9 --- /dev/null +++ b/users/grfn/emacs.d/snippets/python-mode/op.get_bind.execute @@ -0,0 +1,7 @@ +# key: exec +# name: op.get_bind.execute +# -- +op.get_bind().execute( + """ + `(progn (sqlup-mode) "")`$1 + """) diff --git a/users/grfn/emacs.d/snippets/python-mode/pdb b/users/grfn/emacs.d/snippets/python-mode/pdb new file mode 100644 index 000000000000..6b5c0bbc0a73 --- /dev/null +++ b/users/grfn/emacs.d/snippets/python-mode/pdb @@ -0,0 +1,7 @@ +# -*- mode: snippet -*- +# name: pdb +# uuid: +# key: pdb +# condition: t +# -- +import pdb; pdb.set_trace() \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/rust-mode/#[macro_use] b/users/grfn/emacs.d/snippets/rust-mode/#[macro_use] new file mode 100644 index 000000000000..fea942a337f6 --- /dev/null +++ b/users/grfn/emacs.d/snippets/rust-mode/#[macro_use] @@ -0,0 +1,5 @@ +# key: macro_use +# name: #[macro_use] +# -- +#[macro_use] +${1:extern crate} ${2:something};$0 diff --git a/users/grfn/emacs.d/snippets/rust-mode/async test b/users/grfn/emacs.d/snippets/rust-mode/async test new file mode 100644 index 000000000000..8465225663fb --- /dev/null +++ b/users/grfn/emacs.d/snippets/rust-mode/async test @@ -0,0 +1,10 @@ +# -*- mode: snippet -*- +# name: async test +# uuid: +# key: atest +# condition: t +# -- +#[tokio::test(flavor = "multi_thread")] +async fn ${1:test_name}() { + `%`$0 +} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/rust-mode/benchmark b/users/grfn/emacs.d/snippets/rust-mode/benchmark new file mode 100644 index 000000000000..f1446923a0e4 --- /dev/null +++ b/users/grfn/emacs.d/snippets/rust-mode/benchmark @@ -0,0 +1,10 @@ +# -*- mode: snippet -*- +# name: benchmark +# uuid: +# key: bench +# condition: t +# -- +#[bench] +fn ${1:benchmark_name}(b: &mut Bencher) { + `%`b.iter(|| $0); +} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/rust-mode/proptest b/users/grfn/emacs.d/snippets/rust-mode/proptest new file mode 100644 index 000000000000..377b3cfcf60c --- /dev/null +++ b/users/grfn/emacs.d/snippets/rust-mode/proptest @@ -0,0 +1,10 @@ +# -*- mode: snippet -*- +# name: proptest +# uuid: +# key: proptest +# condition: t +# -- +#[proptest] +fn ${1:test_name}($2) { + `%`$0 +} \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/rust-mode/tests b/users/grfn/emacs.d/snippets/rust-mode/tests new file mode 100644 index 000000000000..0a476ab58661 --- /dev/null +++ b/users/grfn/emacs.d/snippets/rust-mode/tests @@ -0,0 +1,9 @@ +# key: tests +# name: test module +# -- +#[cfg(test)] +mod ${1:tests} { + use super::*; + + $0 +} diff --git a/users/grfn/emacs.d/snippets/snippet-mode/indent b/users/grfn/emacs.d/snippets/snippet-mode/indent new file mode 100644 index 000000000000..d38ffceafbad --- /dev/null +++ b/users/grfn/emacs.d/snippets/snippet-mode/indent @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: indent +# key: indent +# -- +# expand-env: ((yas-indent-line 'fixed)) \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/sql-mode/count(*) group by b/users/grfn/emacs.d/snippets/sql-mode/count(*) group by new file mode 100644 index 000000000000..6acc46ff397a --- /dev/null +++ b/users/grfn/emacs.d/snippets/sql-mode/count(*) group by @@ -0,0 +1,5 @@ +# -*- mode: snippet -*- +# name: count(*) group by +# key: countby +# -- +SELECT count(*), ${1:column} FROM ${2:table} GROUP BY $1; diff --git a/users/grfn/emacs.d/snippets/terraform-mode/variable b/users/grfn/emacs.d/snippets/terraform-mode/variable new file mode 100644 index 000000000000..e64175200f44 --- /dev/null +++ b/users/grfn/emacs.d/snippets/terraform-mode/variable @@ -0,0 +1,11 @@ +# -*- mode: snippet -*- +# name: variable +# uuid: +# key: var +# condition: t +# -- +variable "${1:name}" { + type = ${2:string} + ${3:default = ${4:default}} +} +$0 \ No newline at end of file diff --git a/users/grfn/emacs.d/snippets/text-mode/date b/users/grfn/emacs.d/snippets/text-mode/date new file mode 100644 index 000000000000..7b9431147011 --- /dev/null +++ b/users/grfn/emacs.d/snippets/text-mode/date @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# name: date +# key: date +# -- +`(format-time-string "%Y-%m-%d")`$0 \ No newline at end of file diff --git a/users/grfn/emacs.d/splitjoin.el b/users/grfn/emacs.d/splitjoin.el new file mode 100644 index 000000000000..dbc9704d7909 --- /dev/null +++ b/users/grfn/emacs.d/splitjoin.el @@ -0,0 +1,192 @@ +;;; -*- lexical-binding: t; -*- + +(require 'dash) +(load! "utils") + +;;; +;;; Vars +;;; + +(defvar +splitjoin/split-callbacks '() + "Alist mapping major mode symbol names to lists of split callbacks") + +(defvar +splitjoin/join-callbacks '() + "Alist mapping major mode symbol names to lists of join callbacks") + + + +;;; +;;; Definition macros +;;; + +(defmacro +splitjoin/defsplit (mode name &rest body) + `(setf + (alist-get ',name (alist-get ,mode +splitjoin/split-callbacks)) + (λ! () ,@body))) + +(defmacro +splitjoin/defjoin (mode name &rest body) + `(setf + (alist-get ',name (alist-get ,mode +splitjoin/join-callbacks)) + (λ! () ,@body))) + +;;; +;;; Commands +;;; + +(defun +splitjoin/split () + (interactive) + (when-let (callbacks (->> +splitjoin/split-callbacks + (alist-get major-mode) + (-map #'cdr))) + (find-if #'funcall callbacks))) + +(defun +splitjoin/join () + (interactive) + (when-let (callbacks (->> +splitjoin/join-callbacks + (alist-get major-mode) + (-map #'cdr))) + (find-if #'funcall callbacks))) + + +;;; +;;; Splits and joins +;;; TODO: this should probably go in a file-per-language +;;; + +(+splitjoin/defjoin + 'elixir-mode + join-do + (let* ((function-pattern (rx (and (zero-or-more whitespace) + "do" + (zero-or-more whitespace) + (optional (and "#" (zero-or-more anything))) + eol))) + (end-pattern (rx bol + (zero-or-more whitespace) + "end" + (zero-or-more whitespace) + eol)) + (else-pattern (rx bol + (zero-or-more whitespace) + "else" + (zero-or-more whitespace) + eol)) + (lineno (line-number-at-pos)) + (line (thing-at-point 'line t))) + (when-let ((do-start-pos (string-match function-pattern line))) + (cond + ((string-match-p end-pattern (get-line (inc lineno))) + (modify-then-indent + (goto-line-char do-start-pos) + (insert ",") + (goto-char (line-end-position)) + (insert ": nil") + (line-move 1) + (delete-line)) + t) + + ((string-match-p end-pattern (get-line (+ 2 lineno))) + (modify-then-indent + (goto-line-char do-start-pos) + (insert ",") + (goto-char (line-end-position)) + (insert ":") + (join-line t) + (line-move 1) + (delete-line)) + t) + + ((and (string-match-p else-pattern (get-line (+ 2 lineno))) + (string-match-p end-pattern (get-line (+ 4 lineno)))) + (modify-then-indent + (goto-line-char do-start-pos) + (insert ",") + (goto-char (line-end-position)) + (insert ":") + (join-line t) + (goto-eol) + (insert ",") + (join-line t) + (goto-eol) + (insert ":") + (join-line t) + (line-move 1) + (delete-line)) + t))))) + +(comment + (string-match (rx (and bol + "if " + (one-or-more anything) + "," + (zero-or-more whitespace) + "do:" + (one-or-more anything) + "," + (zero-or-more whitespace) + "else:" + (one-or-more anything))) + "if 1, do: nil, else: nil") + + ) + +(+splitjoin/defsplit + 'elixir-mode + split-do-with-optional-else + (let* ((if-with-else-pattern (rx (and bol + (one-or-more anything) + "," + (zero-or-more whitespace) + "do:" + (one-or-more anything) + (optional + "," + (zero-or-more whitespace) + "else:" + (one-or-more anything))))) + (current-line (get-line))) + (when (string-match if-with-else-pattern current-line) + (modify-then-indent + (assert (goto-regex-on-line ",[[:space:]]*do:")) + (delete-char 1) + (assert (goto-regex-on-line ":")) + (delete-char 1) + (insert "\n") + (when (goto-regex-on-line-r ",[[:space:]]*else:") + (delete-char 1) + (insert "\n") + (assert (goto-regex-on-line ":")) + (delete-char 1) + (insert "\n")) + (goto-eol) + (insert "\nend")) + t))) + +(comment + (+splitjoin/defsplit 'elixir-mode split-def + (let ((function-pattern (rx (and "," + (zero-or-more whitespace) + "do:"))) + (line (thing-at-point 'line t))) + (when-let (idx (string-match function-pattern line)) + (let ((beg (line-beginning-position)) + (orig-line-char (- (point) (line-beginning-position)))) + (save-mark-and-excursion + (goto-line-char idx) + (delete-char 1) + (goto-line-char (string-match ":" (thing-at-point 'line t))) + (delete-char 1) + (insert "\n") + (goto-eol) + (insert "\n") + (insert "end") + (evil-indent beg (+ (line-end-position) 1)))) + (goto-line-char orig-line-char) + t)))) + +(+splitjoin/defjoin + 'elixir-mode + join-if-with-else + (let* ((current-line (thing-at-point 'line))))) + +(provide 'splitjoin) diff --git a/users/grfn/emacs.d/sql-strings.el b/users/grfn/emacs.d/sql-strings.el new file mode 100644 index 000000000000..eef397a24ea6 --- /dev/null +++ b/users/grfn/emacs.d/sql-strings.el @@ -0,0 +1,75 @@ +;;; -*- lexical-binding: t; -*- + +;;; https://www.emacswiki.org/emacs/StringAtPoint +(defun ourcomments-string-or-comment-bounds-1 (what) + (save-restriction + (widen) + (let* ((here (point)) + ;; Fix-me: when on end-point, how to handle that and which should be last hit point? + (state (parse-partial-sexp (point-min) (1+ here))) + (type (if (nth 3 state) + 'string + (if (nth 4 state) + 'comment))) + (start (when type (nth 8 state))) + end) + (unless start + (setq state (parse-partial-sexp (point-min) here)) + (setq type (if (nth 3 state) + 'string + (if (nth 4 state) + 'comment))) + (setq start (when type (nth 8 state)))) + (unless (or (not what) + (eq what type)) + (setq start nil)) + (if (not start) + (progn + (goto-char here) + nil) + (setq state (parse-partial-sexp (1+ start) (point-max) + nil nil state 'syntax-table)) + (setq end (point)) + (goto-char here) + (cons start end))))) + +(defun ourcomments-bounds-of-string-at-point () + "Return bounds of string at point if any." + (ourcomments-string-or-comment-bounds-1 'string)) + +(put 'string 'bounds-of-thing-at-point 'ourcomments-bounds-of-string-at-point) + +(defun -sanitize-sql-string (str) + (->> str + (downcase) + (s-trim) + (replace-regexp-in-string + (rx (or (and string-start (or "\"\"\"" + "\"")) + (and (or "\"\"\"" + "\"") + string-end))) + "") + (s-trim))) + +(defun sql-string-p (str) + "Returns 't if STR looks like a string literal for a SQL statement" + (setq str (-sanitize-sql-string str)) + (or (s-starts-with? "select" str))) + +;;; tests + +(require 'ert) + +(ert-deftest sanitize-sql-string-test () + (should (string-equal "select * from foo;" + (-sanitize-sql-string + "\"\"\"SELECT * FROM foo;\n\n\"\"\"")))) + +(ert-deftest test-sql-string-p () + (dolist (str '("SELECT * FROM foo;" + "select * from foo;")) + (should (sql-string-p str))) + + (dolist (str '("not a QUERY")) + (should-not (sql-string-p str)))) diff --git a/users/grfn/emacs.d/terraform.el b/users/grfn/emacs.d/terraform.el new file mode 100644 index 000000000000..2d69c9bad9db --- /dev/null +++ b/users/grfn/emacs.d/terraform.el @@ -0,0 +1,31 @@ +;;; -*- lexical-binding: t; -*- + +(add-hook 'terraform-mode-hook #'terraform-format-on-save-mode) + +(defun packer-format-buffer () + (interactive) + (let ((buf (get-buffer-create "*packer-fmt*"))) + (if (zerop (call-process-region (point-min) (point-max) + "packer" nil buf nil "fmt" "-")) + (let ((point (point)) + (window-start (window-start))) + (erase-buffer) + (insert-buffer-substring buf) + (goto-char point) + (set-window-start nil window-start)) + (message "packer fmt failed: %s" (with-current-buffer buf (buffer-string)))) + (kill-buffer buf))) + +(define-minor-mode packer-format-on-save-mode + "Run packer-format-buffer before saving the current buffer" + :lighter nil + (if packer-format-on-save-mode + (add-hook 'before-save-hook #'packer-format-buffer nil t) + (remove-hook 'before-save-hook #'packer-format-buffer t))) + +(defun maybe-init-packer () + (interactive) + (when (s-ends-with-p ".pkr" (file-name-base (buffer-file-name))) + (packer-format-on-save-mode))) + +(add-hook 'hcl-mode-hook #'maybe-init-packer) diff --git a/users/grfn/emacs.d/tests/splitjoin_test.el b/users/grfn/emacs.d/tests/splitjoin_test.el new file mode 100644 index 000000000000..6495a1a5952e --- /dev/null +++ b/users/grfn/emacs.d/tests/splitjoin_test.el @@ -0,0 +1,68 @@ +;;; private/grfn/tests/splitjoin_test.el -*- lexical-binding: t; -*- + +(require 'ert) +;; (load! 'splitjoin) +;; (load! 'utils) +; (require 'splitjoin) + +;;; Helpers + +(defvar *test-buffer* nil) +(make-variable-buffer-local '*test-buffer*) + +(defun test-buffer () + (when (not *test-buffer*) + (setq *test-buffer* (get-buffer-create "test-buffer"))) + *test-buffer*) + +(defmacro with-test-buffer (&rest body) + `(with-current-buffer (test-buffer) + ,@body)) + +(defun set-test-buffer-mode (mode) + (let ((mode (if (functionp mode) mode + (-> mode symbol-name (concat "-mode") intern)))) + (assert (functionp mode)) + (with-test-buffer (funcall mode)))) + +(defmacro set-test-buffer-contents (contents) + (with-test-buffer + (erase-buffer) + (insert contents))) + +(defun test-buffer-contents () + (with-test-buffer (substring-no-properties (buffer-string)))) + +(defmacro assert-test-buffer-contents (expected-contents) + `(should (equal (string-trim (test-buffer-contents)) + (string-trim ,expected-contents)))) + +(defmacro should-join-to (mode original-contents expected-contents) + `(progn + (set-test-buffer-mode ,mode) + (set-test-buffer-contents ,original-contents) + (with-test-buffer (+splitjoin/join)) + (assert-test-buffer-contents ,expected-contents))) + +(defmacro should-split-to (mode original-contents expected-contents) + `(progn + (set-test-buffer-mode ,mode) + (set-test-buffer-contents ,original-contents) + (with-test-buffer (+splitjoin/split)) + (assert-test-buffer-contents ,expected-contents))) + +(defmacro should-splitjoin (mode joined-contents split-contents) + `(progn + (should-split-to ,mode ,joined-contents ,split-contents) + (should-join-to ,mode ,split-contents ,joined-contents))) + +;;; Tests + +;; Elixir +(ert-deftest elixir-if-splitjoin-test () + (should-splitjoin 'elixir + "if predicate?(), do: result" + "if predicate?() do + result +end")) + diff --git a/users/grfn/emacs.d/themes/grfn-solarized-light-theme.el b/users/grfn/emacs.d/themes/grfn-solarized-light-theme.el new file mode 100644 index 000000000000..ae00b6b5fc75 --- /dev/null +++ b/users/grfn/emacs.d/themes/grfn-solarized-light-theme.el @@ -0,0 +1,115 @@ +(require 'solarized) +(eval-when-compile + (require 'solarized-palettes)) + +;; (defun grfn-solarized-theme () +;; (custom-theme-set-faces +;; theme-name +;; `(font-lock-doc-face ((,class (:foreground ,s-base1)))) +;; `(font-lock-preprocessor-face ((,class (:foreground ,red)))) +;; `(font-lock-keyword-face ((,class (:foreground ,green)))) + +;; `(elixir-attribute-face ((,class (:foreground ,blue)))) +;; `(elixir-atom-face ((,class (:foreground ,cyan)))))) + +(setq +solarized-s-base03 "#002b36" + +solarized-s-base02 "#073642" + ;; emphasized content + +solarized-s-base01 "#586e75" + ;; primary content + +solarized-s-base00 "#657b83" + +solarized-s-base0 "#839496" + ;; comments + +solarized-s-base1 "#93a1a1" + ;; background highlight light + +solarized-s-base2 "#eee8d5" + ;; background light + +solarized-s-base3 "#fdf6e3" + + ;; Solarized accented colors + +solarized-yellow "#b58900" + +solarized-orange "#cb4b16" + +solarized-red "#dc322f" + +solarized-magenta "#d33682" + +solarized-violet "#6c71c4" + +solarized-blue "#268bd2" + +solarized-cyan "#2aa198" + +solarized-green "#859900" + + ;; Darker and lighter accented colors + ;; Only use these in exceptional circumstances! + +solarized-yellow-d "#7B6000" + +solarized-yellow-l "#DEB542" + +solarized-orange-d "#8B2C02" + +solarized-orange-l "#F2804F" + +solarized-red-d "#990A1B" + +solarized-red-l "#FF6E64" + +solarized-magenta-d "#93115C" + +solarized-magenta-l "#F771AC" + +solarized-violet-d "#3F4D91" + +solarized-violet-l "#9EA0E5" + +solarized-blue-d "#00629D" + +solarized-blue-l "#69B7F0" + +solarized-cyan-d "#00736F" + +solarized-cyan-l "#69CABF" + +solarized-green-d "#546E00" + +solarized-green-l "#B4C342") + + +(deftheme grfn-solarized-light "The light variant of Griffin's solarized theme") + +(setq grfn-solarized-faces + '("Griffin's solarized theme customization" + (custom-theme-set-faces + theme-name + `(font-lock-doc-face ((t (:foreground ,+solarized-s-base1)))) + `(font-lock-preprocessor-face ((t (:foreground ,+solarized-red)))) + `(font-lock-keyword-face ((t (:foreground ,+solarized-green)))) + + `(elixir-attribute-face ((t (:foreground ,+solarized-blue)))) + `(elixir-atom-face ((t (:foreground ,+solarized-cyan)))) + `(agda2-highlight-keyword-face ((t (:foreground ,green)))) + `(agda2-highlight-string-face ((t (:foreground ,cyan)))) + `(agda2-highlight-number-face ((t (:foreground ,violet)))) + `(agda2-highlight-symbol-face ((((background ,base3)) (:foreground ,base01)))) + `(agda2-highlight-primitive-type-face ((t (:foreground ,blue)))) + `(agda2-highlight-bound-variable-face ((t nil))) + `(agda2-highlight-inductive-constructor-face ((t (:foreground ,green)))) + `(agda2-highlight-coinductive-constructor-face ((t (:foreground ,yellow)))) + `(agda2-highlight-datatype-face ((t (:foreground ,blue)))) + `(agda2-highlight-field-face ((t (:foreground ,red)))) + `(agda2-highlight-function-face ((t (:foreground ,blue)))) + `(agda2-highlight-module-face ((t (:foreground ,yellow)))) + `(agda2-highlight-postulate-face ((t (:foreground ,blue)))) + `(agda2-highlight-primitive-face ((t (:foreground ,blue)))) + `(agda2-highlight-record-face ((t (:foreground ,blue)))) + `(agda2-highlight-dotted-face ((t nil))) + `(agda2-highlight-operator-face ((t nil))) + `(agda2-highlight-error-face ((t (:foreground ,red :underline t)))) + `(agda2-highlight-unsolved-meta-face ((t (:background ,base2)))) + `(agda2-highlight-unsolved-constraint-face ((t (:background ,base2)))) + `(agda2-highlight-termination-problem-face ((t (:background ,orange :foreground ,base03)))) + `(agda2-highlight-incomplete-pattern-face ((t (:background ,orange :foreground ,base03)))) + `(agda2-highlight-typechecks-face ((t (:background ,cyan :foreground ,base03)))) + + `(font-lock-doc-face ((t (:foreground ,+solarized-s-base1)))) + `(font-lock-preprocessor-face ((t (:foreground ,+solarized-red)))) + `(font-lock-keyword-face ((t (:foreground ,+solarized-green :bold nil)))) + `(font-lock-builtin-face ((t (:foreground ,+solarized-s-base01 + :bold t)))) + + `(elixir-attribute-face ((t (:foreground ,+solarized-blue)))) + `(elixir-atom-face ((t (:foreground ,+solarized-cyan)))) + `(linum ((t (:background ,+solarized-s-base2 :foreground ,+solarized-s-base1)))) + `(line-number ((t (:background ,+solarized-s-base2 :foreground ,+solarized-s-base1)))) + + `(haskell-operator-face ((t (:foreground ,+solarized-green)))) + `(haskell-keyword-face ((t (:foreground ,+solarized-cyan)))) + + `(org-drawer ((t (:foreground ,+solarized-s-base1 + :bold t))))))) + +(solarized-with-color-variables + 'light 'grfn-solarized-light solarized-light-color-palette-alist) + +(provide-theme 'grfn-solarized-light) diff --git a/users/grfn/emacs.d/utils.el b/users/grfn/emacs.d/utils.el new file mode 100644 index 000000000000..21192753a268 --- /dev/null +++ b/users/grfn/emacs.d/utils.el @@ -0,0 +1,114 @@ +;;; -*- lexical-binding: t; -*- + + +;; Elisp Extras + +(defmacro comment (&rest _body) + "Comment out one or more s-expressions" + nil) + +(defun inc (x) "Returns x + 1" (+ 1 x)) +(defun dec (x) "Returns x - 1" (- x 1)) + +(defun average (ns) + "Arithmetic mean of xs" + (if (null ns) nil + (/ (apply #'+ ns) + (length ns)))) + +(comment + (average (list 1 2 3 4)) + ) + +;; +;; Text editing utils +;; + +;; Reading strings + +(defun get-char (&optional point) + "Get the character at the given `point' (defaulting to the current point), +without properties" + (let ((point (or point (point)))) + (buffer-substring-no-properties point (+ 1 point)))) + +(defun get-line (&optional lineno) + "Read the line number `lineno', or the current line if `lineno' is nil, and +return it as a string stripped of all text properties" + (let ((current-line (line-number-at-pos))) + (if (or (not lineno) + (= current-line lineno)) + (thing-at-point 'line t) + (save-mark-and-excursion + (line-move (- lineno (line-number-at-pos))) + (thing-at-point 'line t))))) + +(defun get-line-point () + "Get the position in the current line of the point" + (- (point) (line-beginning-position))) + +;; Moving in the file + +(defun goto-line-char (pt) + "Moves the point to the given position expressed as an offset from the start +of the line" + (goto-char (+ (line-beginning-position) pt))) + +(defun goto-eol () + "Moves to the end of the current line" + (goto-char (line-end-position))) + +(defun goto-regex-on-line (regex) + "Moves the point to the first occurrence of `regex' on the current line. +Returns nil if the regex did not match, non-nil otherwise" + (when-let ((current-line (get-line)) + (line-char (string-match regex current-line))) + (goto-line-char line-char))) + +(defun goto-regex-on-line-r (regex) + "Moves the point to the *last* occurrence of `regex' on the current line. +Returns nil if the regex did not match, non-nil otherwise" + (when-let ((current-line (get-line)) + (modified-regex (concat ".*\\(" regex "\\)")) + (_ (string-match modified-regex current-line)) + (match-start (match-beginning 1))) + (goto-line-char match-start))) + +(comment + (progn + (string-match (rx (and (zero-or-more anything) + (group "foo" "foo"))) + "foofoofoo") + (match-beginning 1))) + +;; Changing file contents + +(defun delete-line () + "Remove the line at the current point" + (delete-region (line-beginning-position) + (inc (line-end-position)))) + +(defmacro modify-then-indent (&rest body) + "Modify text in the buffer according to body, then re-indent from where the + cursor started to where the cursor ended up, then return the cursor to where + it started." + `(let ((beg (line-beginning-position)) + (orig-line-char (- (point) (line-beginning-position)))) + (atomic-change-group + (save-mark-and-excursion + ,@body + (evil-indent beg (+ (line-end-position) 1)))) + (goto-line-char orig-line-char))) + +(pcase-defmacro s-starts-with (prefix) + `(pred (s-starts-with-p ,prefix))) + +(pcase-defmacro s-contains (needle &optional ignore-case) + `(pred (s-contains-p ,needle + ,@(when ignore-case (list ignore-case))))) + +(comment + (pcase "foo" + ((s-contains "bar") 1) + ((s-contains "o") 2)) + ) diff --git a/users/grfn/emacs.d/vterm.el b/users/grfn/emacs.d/vterm.el new file mode 100644 index 000000000000..a7fdea46da32 --- /dev/null +++ b/users/grfn/emacs.d/vterm.el @@ -0,0 +1,24 @@ +;;; -*- lexical-binding: t; -*- + +(defun require-vterm () + (add-to-list + 'load-path + (concat + (s-trim + (shell-command-to-string + "nix-build --no-out-link ~/code/depot -A third_party.emacs.vterm")) + "/share/emacs/site-lisp/elpa/vterm-20200515.1412")) + (require 'vterm)) + +(defun +grfn/vterm-setup () + (hide-mode-line-mode) + (setq-local evil-collection-vterm-send-escape-to-vterm-p t)) + +(add-hook 'vterm-mode-hook #'+grfn/vterm-setup) + +(map! (:map vterm-mode-map + "<C-escape>" #'evil-normal-state)) + +(comment + (require-vterm) + ) diff --git a/users/grfn/gws.fyi/.envrc b/users/grfn/gws.fyi/.envrc new file mode 100644 index 000000000000..be81feddb1a5 --- /dev/null +++ b/users/grfn/gws.fyi/.envrc @@ -0,0 +1 @@ +eval "$(lorri direnv)" \ No newline at end of file diff --git a/users/grfn/gws.fyi/.gitignore b/users/grfn/gws.fyi/.gitignore new file mode 100644 index 000000000000..2b72eaed2988 --- /dev/null +++ b/users/grfn/gws.fyi/.gitignore @@ -0,0 +1,3 @@ +result +letsencrypt +index.html diff --git a/users/grfn/gws.fyi/Makefile b/users/grfn/gws.fyi/Makefile new file mode 100644 index 000000000000..d6c9f40c9536 --- /dev/null +++ b/users/grfn/gws.fyi/Makefile @@ -0,0 +1,31 @@ +.PHONY: deploy + +deploy: + @$(shell nix-build `git rev-parse --show-toplevel` -A 'users.grfn."gws.fyi"') + +renew: + @echo Renewing... + @certbot certonly \ + --manual \ + --domain www.gws.fyi \ + --preferred-challenges dns \ + --server https://acme-v02.api.letsencrypt.org/directory \ + --agree-tos \ + --work-dir $(shell pwd)/letsencrypt/work \ + --logs-dir $(shell pwd)/letsencrypt/logs \ + --config-dir $(shell pwd)/letsencrypt/config + @echo "Reimporting certificate" + @aws acm import-certificate \ + --profile personal \ + --certificate file://letsencrypt/config/live/www.gws.fyi/cert.pem \ + --certificate-chain file://letsencrypt/config/live/www.gws.fyi/fullchain.pem \ + --private-key file://letsencrypt/config/live/www.gws.fyi/privkey.pem \ + --certificate-arn arn:aws:acm:us-east-1:797089351721:certificate/628e54f3-55f9-49c0-811a-eba516b68e30 \ + --region us-east-1 + +backup: + @tarsnap -cf $(shell uname -n)-letsencrypt-$(shell date +%Y-%m-%d_%H-%M-%S) \ + letsencrypt/ + +open: + $$BROWSER "https://www.gws.fyi" diff --git a/users/grfn/gws.fyi/config.el b/users/grfn/gws.fyi/config.el new file mode 100644 index 000000000000..b05d897d3ddb --- /dev/null +++ b/users/grfn/gws.fyi/config.el @@ -0,0 +1,6 @@ +(require 'org) + +(setq org-html-postamble nil) + +(defadvice org-export-grab-title-from-buffer + (around org-export-grab-title-from-buffer-disable activate)) diff --git a/users/grfn/gws.fyi/default.nix b/users/grfn/gws.fyi/default.nix new file mode 100644 index 000000000000..5b7d8fc0e19a --- /dev/null +++ b/users/grfn/gws.fyi/default.nix @@ -0,0 +1,35 @@ +args@{ pkgs, depot, ... }: +with pkgs; +let + site = import ./site.nix args; + resume = import ../resume args; + bucket = "s3://gws.fyi"; + distributionID = "E2ST43JNBH8C64"; + + css = runCommand "main.css" { + buildInputs = [ pkgs.minify ]; + } '' + minify --type css < ${./main.css} > $out + ''; + + keys = runCommand "ssh-keys" {} '' + touch $out + echo "${depot.users.grfn.keys.main}" >> $out + ''; + + website = + runCommand "gws.fyi" { } '' + mkdir -p $out + cp ${css} $out/main.css + cp ${site.index} $out/index.html + cp -r ${site.recipes} $out/recipes + cp ${resume} $out/resume.pdf + cp ${keys} $out/keys + ''; + +in (writeShellScript "deploy.sh" '' + ${awscli2}/bin/aws --profile personal s3 sync ${website}/ ${bucket} + echo "Deployed to http://gws.fyi" +'') // { + inherit website site; +} diff --git a/users/grfn/gws.fyi/index.org b/users/grfn/gws.fyi/index.org new file mode 100644 index 000000000000..fa47b7c03668 --- /dev/null +++ b/users/grfn/gws.fyi/index.org @@ -0,0 +1,39 @@ +#+OPTIONS: title:nil toc:nil num:nil +#+HTML_HEAD: <title>griffin smith</title> +#+HTML_HEAD: <link rel="stylesheet" href="./main.css"> + +my name is griffin ward smith (aka grfn, glittershark, gws) and i'm a software +engineer and musician + +* code + +- [[https://github.com/glittershark/][github]] +- [[https://cs.tvl.fyi/depot/-/tree/users/grfn][my directory in the tvl monorepo]] + +* work + +i'm currently working for a relatively new stealth-mode startup. i can't talk much +about it yet, but if you're interested in rust, distributed systems, and +databases please reach out + +* projects + +- [[https://windtunnel.ci/][windtunnel]], a continuous benchmarking software-as-a-service currently accepting early alpha users (send me an email if you want to try it out!) +- [[https://cs.tvl.fyi/depot/-/tree/users/grfn/achilles][achilles]], a compiler for (what I plan to become) a dependently typed, low-level functional programming language targeting LLVM +- [[https://github.com/glittershark/org-clubhouse][org-clubhouse]], an emacs package for lightweight integration between [[https://orgmode.org/][org-mode]] and [[https://clubhouse.io/][the clubhouse project management tool]] +- [[https://cs.tvl.fyi/depot/-/tree/users/grfn/xanthous][xanthous]], a terminal roguelike in haskell that I work on intermittently and exclusively for fun \\ + you can now try xanthous out over ssh by running ~ssh -p 2222 xanthous.gws.fyi~. if you do so I'd love if you send me an email about it + +* music + +- https://sacrosanct.bandcamp.com/, a post-rock project with a [[https://bandcamp.com/h34rken][friend of mine]] +- [[https://soundcloud.com/missingggg][my current soundcloud]], releasing instrumental hip-hop under the name *missing* +- you can also find a log of all the music I listen to [[https://www.last.fm/user/wildgriffin45][on last.fm]] + +* contact + +- [[mailto:web@gws.fyi][web@gws.fyi]] +- [[https://twitter.com/glittershark1][twitter]] +- https://keybase.io/glittershark +- grfn on IRC (hackint or libera.chat) +- [[http://keys.gnupg.net/pks/lookup?op=get&search=0x44EF5B5E861C09A7][gpg key: 0F11A989879E8BBBFDC1E23644EF5B5E861C09A7]] diff --git a/users/grfn/gws.fyi/main.css b/users/grfn/gws.fyi/main.css new file mode 100644 index 000000000000..cdcd440766ca --- /dev/null +++ b/users/grfn/gws.fyi/main.css @@ -0,0 +1,139 @@ +@import url(https://fonts.googleapis.com/css?family=Inconsolata|Inter&display=swap); + +body { + margin-top: 40px; + max-width: 900px; + line-height: 1.6; + font-size: 16px; + background: #f8f3ff; + color: #3a1616; + padding: 0 10px; + font-family: Inter, sans-serif; +} + +@media (min-width: 1050px) { + body { + margin-left: 150px; + } +} + +@media (min-width: 2000px) { + body { + margin-left: 300px; + } +} + +input { + padding: 10px 16px; + margin: 2px 0; + box-sizing: border-box; + border: 2px solid #dabebe; + border-radius: 6px; + background: #f8f3ff; + color: #3a1616; + font-size: 16px; + -webkit-transition: 0.5s; + transition: 0.5s; + outline: 0; +} + +input:focus { + border: 2px solid #3a1616; +} + +.button { + background-color: #f8f3ff; + border: none; + color: #000; + padding: 6px 14px; + text-align: center; + text-decoration: none; + display: inline-block; + font-size: 16px; + margin: 4px 2px; + transition-duration: 0.4s; + cursor: pointer; + border: 2px solid #3a1616; + border-radius: 6px; +} + +.button:hover { + background-color: #3a1616; + color: #fff; +} + +.isa_error, +.isa_info, +.isa_success, +.isa_warning { + width: 90%; + margin: 10px 0; + padding: 12px; +} + +.isa_info { + color: #00529b; + background-color: #bde5f8; +} + +.isa_success { + color: #4f8a10; + background-color: #dff2bf; +} + +.isa_warning { + color: #9f6000; + background-color: #feefb3; +} + +.isa_error { + color: #d8000c; + background-color: #ffd2d2; +} + +h1, +h2, +h3 { + line-height: 1.2; + font-family: Inter, sans-serif; +} + +h1.title, +h2.title, +h3.title { + text-align: left; + margin-bottom: 1.5em; +} + +h2 { + font-size: 18px; +} + +img { + max-width: 750px; + border-radius: 10px; +} + +a { + cursor: pointer; + color: #217ab7; + line-height: inherit; +} + +a:hover { + background-color: #e3d6ff; +} + +a:visited { + color: #43458b; + border-color: #43458b; +} + +pre { + font-family: Inconsolata, monospace; +} + +::selection { + color: #fff; + background: #ff4081; +} diff --git a/users/grfn/gws.fyi/orgExportHTML.nix b/users/grfn/gws.fyi/orgExportHTML.nix new file mode 100644 index 000000000000..ac28580a5926 --- /dev/null +++ b/users/grfn/gws.fyi/orgExportHTML.nix @@ -0,0 +1,70 @@ +{ pkgs, depot, ... }: + +with pkgs; +with lib; + +let + + emacsWithPackages = (pkgs.emacsPackagesGen pkgs.emacs27).emacsWithPackages; + + emacs = emacsWithPackages (p: with p; [ + org + ]); + +in + +opts: + +let + src = if isAttrs opts then opts.src else opts; + headline = if isAttrs opts then opts.headline else null; + + bn = builtins.baseNameOf src; + filename = elemAt (splitString "." bn) 0; + + outName = + if isNull headline + then + let bn = builtins.baseNameOf src; + filename = elemAt (splitString "." bn) 0; + in + if depot.nix.utils.isDirectory src + then filename + else filename + ".html" + else "${filename}-${replaceStrings [" "] ["-"] filename}.html"; + + escapeDoubleQuotes = replaceStrings ["\""] ["\\\""]; + + navToHeadline = optionalString (! isNull headline) '' + (search-forward "${escapeDoubleQuotes headline}") + (org-narrow-to-subtree) + ''; + +in + +runCommand outName { inherit src; } '' + buildFile() { + cp "$1" file.org + ${emacs}/bin/emacs --batch \ + --load ${./config.el} \ + --visit file.org \ + --eval "(progn + ${escapeDoubleQuotes navToHeadline} + (org-html-export-to-html))" \ + --kill + rm file.org + substitute file.html "$2" \ + --replace '<title>‎</title>' "" + rm file.html + } + + if [ -d $src ]; then + for file in $src/*; do + result=''${file/$src/$out} + mkdir -p $(dirname $result) + buildFile $file ''${result/.org/.html} + done + else + buildFile $src $out + fi +'' diff --git a/users/grfn/gws.fyi/recipes/tomato-sauce.org b/users/grfn/gws.fyi/recipes/tomato-sauce.org new file mode 100644 index 000000000000..74e9b103c670 --- /dev/null +++ b/users/grfn/gws.fyi/recipes/tomato-sauce.org @@ -0,0 +1,102 @@ +#+TITLE: Tomato Sauce +#+OPTIONS: toc:nil num:nil +#+HTML_HEAD: <link rel="stylesheet" href="../main.css"> + +This is a general, all-purpose framework for turning some form of tomatoes into +some form of sauce. You can use fresh tomatoes or canned (the latter are really +quite surprisingly good sometimes), and include or omit garlic, basil, or other +add-ins. The only real non-negotiable ingredients are tomatoes (duh), onion, and +some kind of fat (I prefer butter). + +* Sauce + +1. *Prep*. If starting with canned tomatoes, skip this step. if starting with + whole tomatoes (which you should really only ever do if you grew them + yourself or got them fresh at a farmers market, grocery store tomatoes are + kinda sad), first, peel the tomatoes. The easiest way to do this is to score + them with an X pattern cut as shallow as possible while still breaking the + skin, trying to cover the whole surface area of the tomato, blanch them + briefly in boiling water, then dunk into an ice bath. After this, the skins + will slip right off. After peeling, cut out the stem, core, and any green or + brown bits, and go to the next step + +2. *Base layer*. Couple of variables here, though a perfectly good (in fact, my + usual go-to) tomato sauce can also skip this entire step: + - If you want meat with your sauce (pancetta/guanciale/bacon for an + amatriciana, ground beef or pork for a bolognese) you'll start out by + sautéing that in some sort of fat (probably olive oil), less fat for meat + with a lot of fat already in it, to brown and render out fat from the meat + - If you want onion in the final sauce, you'll chop them finely and sauté + them with whatever fat you've got (either from the meat, or olive oil or + butter if you're not making a meat sauce). Remember to always add a *bit* + of salt when sautéing onion like this, not for flavor but to draw out the + moisture. If you just want onion flavor but not bits of onion in the final + sauce, it's added whole later (so ignore this bullet point). + - If you feel like it (sometimes I do, usually I don't) you can also mince + garlic here and sauté that in with everything else. Add a little after the + onion, as garlic cooks slower than onion unless you want something roastier + (usually you don't for tomato sauce) + - The traditional (so I'm told) thing to do with amatriciana, but also nice + with all variations, is to add in a little crushed red pepper with the + fat to flavor it slightly, but do this late so it doesn't burn + - If you have tomato paste on hand and feel like using it, it's also nice to + fry that in the oil for a little bit - usually I'd do that around the same + time as the garlic + + If you're making tomato *paste* from your sauce, skip all of this - paste is + an ingredient, not a sauce on its own, so imo should be as neutral as + possible (i.e. just tomato). + +3. *Tomato layer*. Not a whole lot to do here, just add all of your tomatoes - + either your peeled and de-cored tomatoes from step 1 if you're using whole + tomatoes, or an entire can of whole, peeled san marzano tomatoes, including + the juice in the can - to a pot over medium-high heat. If you need more fat + or if you skipped step 2, this is where you'd add it - a classic and my + personal favorite is like 2/3rds to 3/4ths of a stick of butter, but you can + also go with olive oil. If you skipped the onion in step 2, add that here + too - usually that'd just be a fist-sized amount of onion or so peeled but + left with the stem on so you can fish it out from your final sauce later (and + snack on it!). Also salt here, again not to taste but primarily to draw out + moisture from the various ingredients. + +4. You can cook that for a wide variety of times, especially depending on how + hot you make your stove - there ends up being *lot* of liquid in there, so + you can go (in my experience) a reasonable amount hotter than you expect + without burning the sauce, though obviously your mileage may vary. The main + thing you're looking for is the whole chunks of tomato to break down, and the + whole sauce to get a texture that looks like it'll end up sticking to pasta + nicely. In all versions of this, stir pretty regularly with a wooden spoon, + and use the spoon to crush the big chunks of tomato occasionally. + +5. *Final layer*. Usually I don't do anything here - but if you feel + like it, usually right as you take stuff off the heat is where you'd add + basil, if you're using it. You can also add sugar to balance out too much + acidity from an especially acidic tomato here - I'm not going to tell anyone. + Also salt, but make sure to account for the extra salt you're gonna get from + the pasta water (see step 6) + +6. *Pasta*. You know how to cook pasta, I'm not going to tell you that. But, + like, salt your water until it tastes too salty, and remember to move the + pasta itself *directly* into the sauce pot from the pasta pot before it's + completely done cooking and without straining, bringing along some of the + pasta water (and a little extra for good measure) then finishing the pasta in + the sauce. You know, the thing you do for pasta. Remember the pasta water + will have salt in it, so adjust for that when salting the sauce overall (I + have made this mistake and ended up with too-salty pasta sauce). + +* Paste + +Start with the above recipe for tomato sauce, noting especially that (in my +opinion) you should skip step 2 entirely. Keep cooking the sauce until it's +*too* thick for pasta sauce (but don't burn it!), then spread it out across some +sort of lined sheet pan (like a silpat, if you've got one) and bake in the oven +at like 250-300 degrees for a *hell* of a long time - I've seen this take like +10 hours, for an especially juicy batch of tomatoes, but obviously keep a close +eye on it because it *definitely will burn* eventually. You're looking for the +end result to be the texture of tomato paste, because that's what the recipe is +for. Especially if you're using garden-grown or otherwise fresh tomatoes, +you'll notice quite a few seeds in the final product - don't worry too much +about those, they've never bothered me. Once everything's done and cooled down, +store in a jar in a fridge, topped with olive oil to seal things off and prevent +oxidation. Use in all your future endeavors, including the tomato sauce recipe +above itself. Tomato sauce is a beautiful oroborous. diff --git a/users/grfn/gws.fyi/shell.nix b/users/grfn/gws.fyi/shell.nix new file mode 100644 index 000000000000..846bdb6677a3 --- /dev/null +++ b/users/grfn/gws.fyi/shell.nix @@ -0,0 +1,9 @@ +with import <nixpkgs> { config.allowUnfree = true; }; +mkShell { + buildInputs = [ + awscli + gnumake + letsencrypt + tarsnap + ]; +} diff --git a/users/grfn/gws.fyi/site.nix b/users/grfn/gws.fyi/site.nix new file mode 100644 index 000000000000..33d4a71e7b1f --- /dev/null +++ b/users/grfn/gws.fyi/site.nix @@ -0,0 +1,12 @@ +args@{ pkgs ? import <nixpkgs> {}, ... }: + +let + + orgExportHTML = import ./orgExportHTML.nix args; + +in + +{ + index = orgExportHTML ./index.org; + recipes = orgExportHTML ./recipes; +} diff --git a/users/grfn/keyboard/.gitignore b/users/grfn/keyboard/.gitignore new file mode 100644 index 000000000000..b2be92b7db01 --- /dev/null +++ b/users/grfn/keyboard/.gitignore @@ -0,0 +1 @@ +result diff --git a/users/grfn/keyboard/README.org b/users/grfn/keyboard/README.org new file mode 100644 index 000000000000..b085883a1049 --- /dev/null +++ b/users/grfn/keyboard/README.org @@ -0,0 +1,10 @@ +This repository contains the source of the keyboard layout for my Ergodox EZ, +plus build tooling based on Nix. + +To flash to an Ergodox EZ that's connected to your computer via USB, run: + +#+BEGIN_SRC shell +./flash +#+END_SRC + +then press the reset switch on the keyboard. diff --git a/users/grfn/keyboard/default.nix b/users/grfn/keyboard/default.nix new file mode 100644 index 000000000000..dda1d9682cbf --- /dev/null +++ b/users/grfn/keyboard/default.nix @@ -0,0 +1,63 @@ +{ pkgs, ... }: + +with pkgs; + +let avrlibc = pkgsCross.avr.libcCross; in + +rec { + qmkSource = fetchgit { + url = "https://github.com/qmk/qmk_firmware"; + rev = "ab1650606c36f85018257aba65d9c3ff8ec42e71"; + sha256 = "1k59flkvhjzmfl0yz9z37lqhvad7m9r5wy1p1sjk5274rsmylh79"; + fetchSubmodules = true; + }; + + layout = stdenv.mkDerivation rec { + name = "ergodox_ez_grfn.hex"; + + src = qmkSource; + + buildInputs = [ + dfu-programmer + dfu-util + diffutils + git + python3 + pkgsCross.avr.buildPackages.binutils + pkgsCross.avr.buildPackages.gcc + avrlibc + avrdude + ]; + + AVR_CFLAGS = [ + "-isystem ${avrlibc}/avr/include" + "-L${avrlibc}/avr/lib/avr5" + ]; + + AVR_ASFLAGS = AVR_CFLAGS; + + patches = [ ./increase-tapping-delay.patch ]; + + postPatch = '' + mkdir keyboards/ergodox_ez/keymaps/grfn + cp ${./keymap.c} keyboards/ergodox_ez/keymaps/grfn/keymap.c + ''; + + buildPhase = '' + make ergodox_ez:grfn + ''; + + installPhase = '' + cp ergodox_ez_grfn.hex $out + ''; + }; + + flash = writeShellScript "flash.sh" '' + ${teensy-loader-cli}/bin/teensy-loader-cli \ + -v \ + --mcu=atmega32u4 \ + -w ${layout} + ''; + + meta.targets = [ "layout" ]; +} diff --git a/users/grfn/keyboard/flash b/users/grfn/keyboard/flash new file mode 100755 index 000000000000..76def36f9ca8 --- /dev/null +++ b/users/grfn/keyboard/flash @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +exec "$(nix-build --no-out-link ../../.. -A users.grfn.keyboard.flash)" diff --git a/users/grfn/keyboard/increase-tapping-delay.patch b/users/grfn/keyboard/increase-tapping-delay.patch new file mode 100644 index 000000000000..316c435fed6c --- /dev/null +++ b/users/grfn/keyboard/increase-tapping-delay.patch @@ -0,0 +1,13 @@ +diff --git a/keyboards/ergodox_ez/config.h b/keyboards/ergodox_ez/config.h +index ae70c4f2e..776110c09 100644 +--- a/keyboards/ergodox_ez/config.h ++++ b/keyboards/ergodox_ez/config.h +@@ -45,7 +45,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>. + /* define if matrix has ghost */ + //#define MATRIX_HAS_GHOST + +-#define TAPPING_TERM 200 ++#define TAPPING_TERM 150 + #define IGNORE_MOD_TAP_INTERRUPT // this makes it possible to do rolling combos (zx) with keys that convert to other keys on hold (z becomes ctrl when you hold it, and when this option isn't enabled, z rapidly followed by x actually sends Ctrl-x. That's bad.) + + /* Mechanical locking support. Use KC_LCAP, KC_LNUM or KC_LSCR instead in keymap */ diff --git a/users/grfn/keyboard/keymap.c b/users/grfn/keyboard/keymap.c new file mode 100644 index 000000000000..fbb28c9aac59 --- /dev/null +++ b/users/grfn/keyboard/keymap.c @@ -0,0 +1,206 @@ +#include QMK_KEYBOARD_H +#include "debug.h" +#include "action_layer.h" +#include "version.h" + + +#include "keymap_german.h" + +#include "keymap_nordic.h" + + + +enum custom_keycodes { + PLACEHOLDER = SAFE_RANGE, // can always be here + EPRM, + VRSN, + RGB_SLD, + + EX_PIPE, // |> + THIN_ARROW, // -> + FAT_ARROW, // => +}; + + + +#define LAMBDA UC(0x03BB) + +const uint16_t PROGMEM keymaps[][MATRIX_ROWS][MATRIX_COLS] = { + + [0] = LAYOUT_ergodox( + KC_EQUAL, KC_1, KC_2, KC_3, KC_4, KC_5, KC_LEFT, + KC_TAB, KC_Q, KC_W, KC_E, KC_R, KC_T, KC_LALT, + KC_ESCAPE, KC_A, KC_S, KC_D, KC_F, KC_G, + KC_RSFT, CTL_T(KC_Z), KC_X, KC_C, KC_V, KC_B, KC_TAB, + LT(1,KC_GRAVE), KC_QUOTE, LALT(KC_LSHIFT),KC_LEFT,KC_RIGHT, + ALT_T(KC_APPLICATION), LAMBDA, + KC_LBRACKET, + GUI_T(KC_NO), LSFT_T(KC_BSPACE), KC_COLN, + + KC_MY_COMPUTER, KC_6, KC_7, KC_8, KC_9, KC_0, KC_MINUS, + KC_RALT, KC_Y, KC_U, KC_I, KC_O, KC_P, KC_BSLASH, + KC_H, KC_J, KC_K, KC_L, LT(2,KC_SCOLON), LT(1,KC_QUOTE), + KC_MINUS, KC_N, KC_M, KC_COMMA, KC_DOT, CTL_T(KC_SLASH), KC_RSFT, + KC_DOWN,KC_UP, KC_LBRACKET,KC_RBRACKET,MO(1), + + KC_PAUSE, TG(3), + KC_RBRACKET, + KC_COLN, RSFT_T(KC_ENTER), KC_SPACE + ), + + [1] = LAYOUT_ergodox( + KC_ESCAPE, KC_F1, KC_F2, KC_F3, KC_F4, KC_F5, KC_TRANSPARENT, + KC_TRANSPARENT, KC_EXLM, KC_AT, KC_LCBR, KC_RCBR, KC_PIPE, KC_RABK, + KC_TRANSPARENT, KC_HASH, KC_DLR, KC_LPRN, KC_RPRN, KC_UNDERSCORE, + KC_LABK, KC_PERC, KC_CIRC, KC_LBRACKET, KC_RBRACKET, KC_TILD, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + RGB_MOD, KC_TRANSPARENT, + KC_TRANSPARENT, + RGB_VAD, RGB_VAI, EX_PIPE, + + KC_TRANSPARENT, KC_F6, KC_F7, KC_F8, KC_F9, KC_F10, KC_F11, + KC_PGUP, KC_UP, KC_7, KC_8, KC_9, KC_ASTR, KC_F12, + KC_DOWN, KC_4, KC_5, KC_6, KC_PLUS, KC_TRANSPARENT, + KC_PGDOWN, KC_AMPR, KC_1, KC_2, KC_3, KC_BSLASH, KC_TRANSPARENT, + KC_TRANSPARENT, KC_DOT, KC_0, KC_EQUAL, KC_TRANSPARENT, + RGB_TOG, RGB_SLD, + THIN_ARROW, + EX_PIPE, RGB_HUD, RGB_HUI + ), + + [2] = LAYOUT_ergodox( + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_MS_UP, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_MS_LEFT, KC_MS_DOWN, KC_MS_RIGHT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_MS_BTN1, KC_MS_BTN2, + KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, + KC_MS_BTN1, KC_MS_BTN2, KC_TRANSPARENT, + + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_MS_WH_DOWN, KC_MS_WH_UP, KC_TRANSPARENT, KC_TRANSPARENT, KC_MEDIA_PLAY_PAUSE, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_MEDIA_PREV_TRACK, KC_MEDIA_NEXT_TRACK, KC_TRANSPARENT, KC_TRANSPARENT, + KC_AUDIO_VOL_DOWN, KC_AUDIO_VOL_UP, KC_AUDIO_MUTE, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_WWW_BACK), + + // FPS layout + [3] = LAYOUT_ergodox( + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, + KC_SPACE, KC_TRANSPARENT, KC_TRANSPARENT, + + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT, + KC_TRANSPARENT, TG(3), + KC_TRANSPARENT, + KC_TRANSPARENT, KC_TRANSPARENT, KC_TRANSPARENT), +}; + +const uint16_t PROGMEM fn_actions[] = { + [1] = ACTION_LAYER_TAP_TOGGLE(1) +}; + +// leaving this in place for compatibilty with old keymaps cloned and re-compiled. +const macro_t *action_get_macro(keyrecord_t *record, uint8_t id, uint8_t opt) +{ + switch(id) { + case 0: + if (record->event.pressed) { + SEND_STRING (QMK_KEYBOARD "/" QMK_KEYMAP " @ " QMK_VERSION); + } + break; + } + return MACRO_NONE; +}; + +bool process_record_user(uint16_t keycode, keyrecord_t *record) { + switch (keycode) { + // dynamically generate these. + case EPRM: + if (record->event.pressed) { + eeconfig_init(); + } + return false; + break; + case VRSN: + if (record->event.pressed) { + SEND_STRING (QMK_KEYBOARD "/" QMK_KEYMAP " @ " QMK_VERSION); + } + return false; + break; + case RGB_SLD: + if (record->event.pressed) { + rgblight_mode(1); + } + return false; + break; + case EX_PIPE: + if (record->event.pressed) { + SEND_STRING ( "|> " ); + } + return false; + break; + case THIN_ARROW: + if (record->event.pressed) { + SEND_STRING ( "-> " ); + } + return false; + break; + + + } + return true; +} + +void matrix_scan_user(void) { + + uint8_t layer = biton32(layer_state); + + ergodox_board_led_off(); + ergodox_right_led_1_off(); + ergodox_right_led_2_off(); + ergodox_right_led_3_off(); + switch (layer) { + case 1: + ergodox_right_led_1_on(); + break; + case 2: + ergodox_right_led_2_on(); + break; + case 3: + ergodox_right_led_3_on(); + break; + case 4: + ergodox_right_led_1_on(); + ergodox_right_led_2_on(); + break; + case 5: + ergodox_right_led_1_on(); + ergodox_right_led_3_on(); + break; + case 6: + ergodox_right_led_2_on(); + ergodox_right_led_3_on(); + break; + case 7: + ergodox_right_led_1_on(); + ergodox_right_led_2_on(); + ergodox_right_led_3_on(); + break; + default: + break; + } + +}; diff --git a/users/grfn/keys.nix b/users/grfn/keys.nix new file mode 100644 index 000000000000..c52229b3a898 --- /dev/null +++ b/users/grfn/keys.nix @@ -0,0 +1,5 @@ +{ ... }: +{ + whitby = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDIwl+xQYRCk6Ijz/Ll8eXKZrcTH9/7xwlvIowiuqDSFtGkf+73QJkwVJ0YiKHWAPwIUWMzCEO/Ab2g6j4PcR+XYu8kXbrwT5aW65L/AK1oaav2RfV1bnQEVUP9FRPL52BN42J0ibI2QJZKJVws9JF7vxTWPPG0V0eoxcaRMk1ZEqq+/k3GuN8D69VSV8xo9lB8yZEvTxs0YQRiiF7Q6t/3jhYtz6lCdazQviRcSEOj5AVsDjcf1XIAPOcLK4Q4OEXL49T3UaitSYMyKIO8hzNLiyGAUlSbshAnutPXdyNBypkCs6FrSPSRdBfFjzUVE/a+JWCPmx0q0xAVd497Efxby+Vsa2/TPMp7tSisPaqk3MpPmjBS7eI/y4Pl2GpAB4OVANEBNd1Q6K2/37Pk+PrZtIUBiRG8sM0Od36BjwLCxvG0G5P/UYZ93aC8GzqkRf4evOBMiJCvR2o9CDEDycNyTm1y5dyJzQewOTWX9nsiF1rllc92W0ZALvpO03+W2+k= grfn@chupacabra"; + main = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDHPiNpPB6Uqs/VSW/C8tR/Z5wCQxKppNL2iETb1ucsYsFf1B2apG5txj06NMT6IWXwWpZXq7ld+/sA+a2I03lO2INP7S1Dto5nAwpNhhKN/UBXk76qYTdY5tEvb9J89S2ZzfQWR30aZ0CEDDrcbc+YktU1eSLdluu6QH+M/uPBweSiVn5wNHkc5sRdbyiVsZSQJ41MO7PQrzGpe7Pxola/ghOHdEFlESJMKA5uoRpCGboxtDE9tMJwG5MxNwHERpfI9FjvvLsJRrp9dRf6A/RQjlV/nb1GmpX0I8pvrXEPxm/l0rOAgE81VSsM+BxJ7ZvCe8u/YqMYJ8xVfskzlVsf griffin@MacBook-Pro"; +} diff --git a/users/grfn/org-clubhouse/.gitignore b/users/grfn/org-clubhouse/.gitignore new file mode 100644 index 000000000000..2a7dd97debf1 --- /dev/null +++ b/users/grfn/org-clubhouse/.gitignore @@ -0,0 +1,3 @@ +# Spacemacs +org-clubhouse-autoloads.el +org-clubhouse-pkg.el diff --git a/users/grfn/org-clubhouse/CODE_OF_CONDUCT.org b/users/grfn/org-clubhouse/CODE_OF_CONDUCT.org new file mode 100644 index 000000000000..f15e387d5464 --- /dev/null +++ b/users/grfn/org-clubhouse/CODE_OF_CONDUCT.org @@ -0,0 +1,101 @@ +* Contributor Covenant Code of Conduct + :PROPERTIES: + :CUSTOM_ID: contributor-covenant-code-of-conduct + :END: + +** Our Pledge + :PROPERTIES: + :CUSTOM_ID: our-pledge + :END: + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our +project and our community a harassment-free experience for everyone, +regardless of age, body size, disability, ethnicity, sex +characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +** Our Standards + :PROPERTIES: + :CUSTOM_ID: our-standards + :END: + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual + attention or advances +- Trolling, insulting/derogatory comments, and personal or political + attacks +- Public or private harassment +- Publishing others' private information, such as a physical or + electronic address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +** Our Responsibilities + :PROPERTIES: + :CUSTOM_ID: our-responsibilities + :END: + +Project maintainers are responsible for clarifying the standards of +acceptable behavior and are expected to take appropriate and fair +corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, +or reject comments, commits, code, wiki edits, issues, and other +contributions that are not aligned to this Code of Conduct, or to ban +temporarily or permanently any contributor for other behaviors that they +deem inappropriate, threatening, offensive, or harmful. + +** Scope + :PROPERTIES: + :CUSTOM_ID: scope + :END: + +This Code of Conduct applies within all project spaces, and it also +applies when an individual is representing the project or its community +in public spaces. Examples of representing a project or community +include using an official project e-mail address, posting via an +official social media account, or acting as an appointed representative +at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +** Enforcement + :PROPERTIES: + :CUSTOM_ID: enforcement + :END: + +Instances of abusive, harassing, or otherwise unacceptable behavior may +be reported by contacting the project team at root@gws.fyi. All +complaints will be reviewed and investigated and will result in a +response that is deemed necessary and appropriate to the circumstances. +The project team is obligated to maintain confidentiality with regard to +the reporter of an incident. Further details of specific enforcement +policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in +good faith may face temporary or permanent repercussions as determined +by other members of the project's leadership. + +** Attribution + :PROPERTIES: + :CUSTOM_ID: attribution + :END: + +This Code of Conduct is adapted from the +[[https://www.contributor-covenant.org][Contributor Covenant]], version +1.4, available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/users/grfn/org-clubhouse/LICENSE b/users/grfn/org-clubhouse/LICENSE new file mode 100644 index 000000000000..1777f0fac3ea --- /dev/null +++ b/users/grfn/org-clubhouse/LICENSE @@ -0,0 +1,7 @@ +Copyright (C) 2018 Off Market Data, Inc. DBA Urbint + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/users/grfn/org-clubhouse/README.org b/users/grfn/org-clubhouse/README.org new file mode 100644 index 000000000000..9cd8fbe8921d --- /dev/null +++ b/users/grfn/org-clubhouse/README.org @@ -0,0 +1,142 @@ +#+TITLE:Org-Clubhouse + +Simple, unopinionated integration between Emacs's [[https://orgmode.org/][org-mode]] and the [[https://clubhouse.io/][Clubhouse]] +issue tracker + +(This used to be at urbint/org-clubhouse, by the way, but moved here as it's +more of a personal project than a company one) + +* Installation + +** [[https://github.com/quelpa/quelpa][Quelpa]] + +#+BEGIN_SRC emacs-lisp +(quelpa '(org-clubhouse + :fetcher github + :repo "glittershark/org-clubhouse")) +#+END_SRC + +** [[https://github.com/hlissner/doom-emacs/][DOOM Emacs]] + +#+BEGIN_SRC emacs-lisp +;; in packages.el +(package! org-clubhouse + :recipe (:fetcher github + :repo "glittershark/org-clubhouse" + :files ("*"))) + +;; in config.el +(def-package! org-clubhouse) +#+END_SRC + +** [[http://spacemacs.org/][Spacemacs]] +#+BEGIN_SRC emacs-lisp +;; in .spacemacs (SPC+fed) + dotspacemacs-additional-packages + '((org-clubhouse :location (recipe :fetcher github :repo "glittershark/org-clubhouse"))) +#+END_SRC + + +* Setup + +Once installed, you'll need to set three global config vars: + +#+BEGIN_SRC emacs-lisp +(setq org-clubhouse-auth-token "<your-token>" + org-clubhouse-team-name "<your-team-name>" + org-clubhouse-username "<your-username>") +#+END_SRC + +You can generate a new personal API token by going to the "API Tokens" tab on +the "Settings" page in the clubhouse UI. + +Note that ~org-clubhouse-username~ needs to be set to your *mention name*, not +your username, as currently there's no way to get the ID of a user given their +username in the clubhouse API + +* Usage + +** Reading from clubhouse + +- ~org-clubhouse-headlines-from-query~ + Create org-mode headlines from a [[https://help.clubhouse.io/hc/en-us/articles/360000046646-Searching-in-Clubhouse-Story-Search][clubhouse query]] at the cursor's current + position, prompting for the headline indentation level and clubhouse query + text +- ~org-clubhouse-headline-from-story~ + Prompts for headline indentation level and the title of a story (which will + complete using the titles of all stories in your Clubhouse workspace) and + creates an org-mode headline from that story +- ~org-clubhouse-headline-from-story-id~ + Creates an org-mode headline directly from the ID of a clubhouse story + +** Writing to clubhouse + +- ~org-clubhouse-create-story~ + Creates a new Clubhouse story from the current headline, or if a region of + headlines is selected bulk-creates stories with all those headlines +- ~org-clubhouse-create-epic~ + Creates a new Clubhouse epic from the current headline, or if a region of + headlines is selected bulk-creates epics with all those headlines +- ~org-clubhouse-create-story-with-task-list~ + Creates a Clubhouse story from the current headline, making all direct + children of the headline into tasks in the task list of the story +- ~org-clubhouse-push-task-list~ + Writes each child element of the current clubhouse element as a task list + item of the associated clubhouse ID. +- ~org-clubhouse-update-story-title~ + Updates the title of the Clubhouse story linked to the current headline with + the text of the headline +- ~org-clubhouse-update-description~ + Update the status of the Clubhouse story linked to the current element with + the contents of a drawer inside the element called DESCRIPTION, if any exists +- ~org-clubhouse-claim~ + Adds the user configured in ~org-clubhouse-username~ as the owner of the + clubhouse story associated with the headline at point + +*** Automatically updating Clubhouse story statuses + +Org-clubhouse can be configured to update the status of stories as you update +their todo-keyword in org-mode. To opt-into this behavior, set the +~org-clubhouse-mode~ minor-mode: + +#+BEGIN_SRC emacs-lisp +(add-hook 'org-mode-hook #'org-clubhouse-mode nil nil) +#+END_SRC + +The mapping from org-mode todo-keywords is configured via the +~org-clubhouse-state-alist~ variable, which should be an [[https://www.gnu.org/software/emacs/manual/html_node/elisp/Association-Lists.html][alist]] mapping (string) +[[https://orgmode.org/manual/Workflow-states.html][org-mode todo-keywords]] to the (string) names of their corresponding workflow +state. You can have todo-keywords that don't map to a workflow state (I use this +in my workflow extensively) and org-clubhouse will just preserve the previous +state of the story when moving to that state. + +An example config: + +#+BEGIN_SRC emacs-lisp +(setq org-clubhouse-state-alist + '(("TODO" . "To Do") + ("ACTIVE" . "In Progress") + ("DONE" . "Done"))) +#+END_SRC + +* Philosophy + +I use org-mode every single day to manage tasks, notes, literate programming, +etc. Part of what that means for me is that I already have a system for the +structure of my .org files, and I don't want to sacrifice that system for any +external tool. Updating statuses, ~org-clubhouse-create-story~, and +~org-clubhouse-headline-from-story~ are my bread and butter for that reason - +rather than having some sort of bidirectional sync that pulls down full lists of +all the stories in Clubhouse (or whatever issue tracker / project management +tool I'm using at the time). I can be in a mode where I'm taking meeting notes, +think of something that I need to do, make it a TODO headline, and make that +TODO headline a clubhouse story. That's the same reason for the DESCRIPTION +drawers rather than just sending the entire contents of a headline to +Clubhouse - I almost always want to write things like personal notes, literate +code, etc inside of the tasks I'm working on, and don't always want to share +that with Clubhouse. + +* Configuration + +Refer to the beginning of the [[https://github.com/urbint/org-clubhouse/blob/master/org-clubhouse.el][org-clubhouse.el]] file in this repository for +documentation on all supported configuration variables diff --git a/users/grfn/org-clubhouse/org-clubhouse.el b/users/grfn/org-clubhouse/org-clubhouse.el new file mode 100644 index 000000000000..e6e29b575187 --- /dev/null +++ b/users/grfn/org-clubhouse/org-clubhouse.el @@ -0,0 +1,1241 @@ +;;; org-clubhouse.el --- Simple, unopinionated integration between org-mode and +;;; Clubhouse + +;;; Copyright (C) 2018 Off Market Data, Inc. DBA Urbint +;;; Permission is hereby granted, free of charge, to any person obtaining a copy +;;; of this software and associated documentation files (the "Software"), to +;;; deal in the Software without restriction, including without limitation the +;;; rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +;;; sell copies of the Software, and to permit persons to whom the Software is +;;; furnished to do so, subject to the following conditions: + +;;; The above copyright notice and this permission notice shall be included in +;;; all copies or substantial portions of the Software. + +;;; THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +;;; IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +;;; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +;;; AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +;;; LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +;;; FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +;;; IN THE SOFTWARE. + +;;; Commentary: +;;; org-clubhouse provides simple, unopinionated integration between Emacs's +;;; org-mode and the Clubhouse issue tracker +;;; +;;; To configure org-clubhouse, create an authorization token in Cluhbouse's +;;; settings, then place the following configuration somewhere private: +;;; +;;; (setq org-clubhouse-auth-token "<auth_token>" +;;; org-clubhouse-team-name "<team-name>") +;;; + +;;; Code: + +(require 'cl-macs) +(require 'dash) +(require 'dash-functional) +(require 's) +(require 'org) +(require 'org-element) +(require 'subr-x) +(require 'ivy) +(require 'json) + +;;; +;;; Configuration +;;; + +(defvar org-clubhouse-auth-token nil + "Authorization token for the Clubhouse API.") + +(defvar org-clubhouse-username nil + "Username for the current Clubhouse user. + +Unfortunately, the Clubhouse API doesn't seem to provide this via the API given +an API token, so we need to configure this for +`org-clubhouse-claim-story-on-status-updates' to work") + +(defvar org-clubhouse-team-name nil + "Team name to use in links to Clubhouse. +ie https://app.clubhouse.io/<TEAM_NAME>/stories") + +(defvar org-clubhouse-project-ids nil + "Specific list of project IDs to synchronize with clubhouse. +If unset all projects will be synchronized") + +(defvar org-clubhouse-workflow-name "Default") + +(defvar org-clubhouse-default-story-type nil + "Sets the default story type. If set to 'nil', it will interactively prompt +the user each and every time a new story is created. If set to 'feature', +'bug', or 'chore', that value will be used as the default and the user will +not be prompted") + +(defvar org-clubhouse-state-alist + '(("LATER" . "Unscheduled") + ("[ ]" . "Ready for Development") + ("TODO" . "Ready for Development") + ("OPEN" . "Ready for Development") + ("ACTIVE" . "In Development") + ("PR" . "Review") + ("DONE" . "Merged") + ("[X]" . "Merged") + ("CLOSED" . "Merged")) + "Alist mapping org-mode todo keywords to their corresponding states in + Clubhouse. In `org-clubhouse-mode', moving headlines to these todo keywords + will update to the corresponding status in Clubhouse") + +(defvar org-clubhouse-story-types + '(("feature" . "Feature") + ("bug" . "Bug") + ("chore" . "Chore"))) + +(defvar org-clubhouse-default-story-types + '(("feature" . "Feature") + ("bug" . "Bug") + ("chore" . "Chore") + ("prompt" . "**Prompt each time (do not set a default story type)**"))) + +(defvar org-clubhouse-default-state "Proposed" + "Default state to create all new stories in.") + +(defvar org-clubhouse-claim-story-on-status-update 't + "Controls the assignee behavior of stories on status update. + +If set to 't, will mark the current user as the owner of any clubhouse +stories on any update to the status. + +If set to nil, will never automatically update the assignee of clubhouse +stories. + +If set to a list of todo-state's, will mark the current user as the owner of +clubhouse stories whenever updating the status to one of those todo states.") + +(defvar org-clubhouse-create-stories-with-labels nil + "Controls the way org-clubhouse creates stories with labels based on org tags. + +If set to 't, will create labels for all org tags on headlines when stories are +created. + +If set to 'existing, will set labels on created stories only if the label +already exists in clubhouse + +If set to nil, will never create stories with labels") + +;;; +;;; Utilities +;;; + +(defmacro comment (&rest _) + "Comment out one or more s-expressions." + nil) + +(defun ->list (vec) (append vec nil)) + +(defun reject-archived (item-list) + (-reject (lambda (item) (equal :json-true (alist-get 'archived item))) item-list)) + +(defun alist->plist (key-map alist) + (->> key-map + (-map (lambda (key-pair) + (let ((alist-key (car key-pair)) + (plist-key (cdr key-pair))) + (list plist-key (alist-get alist-key alist))))) + (-flatten-n 1))) + +(defun alist-get-equal (key alist) + "Like `alist-get', but uses `equal' instead of `eq' for comparing keys" + (->> alist + (-find (lambda (pair) (equal key (car pair)))) + (cdr))) + +(defun invert-alist (alist) + "Invert the keys and values of ALIST." + (-map (lambda (cell) (cons (cdr cell) (car cell))) alist)) + +(comment + + (alist->plist + '((foo . :foo) + (bar . :something)) + + '((foo . "foo") (bar . "bar") (ignored . "ignoreme!"))) + ;; => (:foo "foo" :something "bar") + + ) + +(defun find-match-in-alist (target alist) + (->> alist + (-find (lambda (key-value) + (string-equal (cdr key-value) target))) + car)) + +(defun org-clubhouse-collect-headlines (beg end) + "Collects the headline at point or the headlines in a region. Returns a list." + (if (and beg end) + (org-clubhouse-get-headlines-in-region beg end) + (list (org-element-find-headline)))) + + +(defun org-clubhouse-get-headlines-in-region (beg end) + "Collects the headlines from BEG to END" + (save-excursion + ;; This beg/end clean up pulled from `reverse-region`. + ;; it expands the region to include the full lines from the selected region. + + ;; put beg at the start of a line and end and the end of one -- + ;; the largest possible region which fits this criteria + (goto-char beg) + (or (bolp) (forward-line 1)) + (setq beg (point)) + (goto-char end) + ;; the test for bolp is for those times when end is on an empty line; + ;; it is probably not the case that the line should be included in the + ;; reversal; it isn't difficult to add it afterward. + (or (and (eolp) (not (bolp))) (progn (forward-line -1) (end-of-line))) + (setq end (point-marker)) + + ;; move to the beginning + (goto-char beg) + ;; walk by line until past end + (let ((headlines '()) + (before-end 't)) + (while before-end + (add-to-list 'headlines (org-element-find-headline)) + (let ((before (point))) + (org-forward-heading-same-level 1) + (setq before-end (and (not (eq before (point))) (< (point) end))))) + (reverse headlines)))) + +;;; +;;; Org-element interaction +;;; + +;; (defun org-element-find-headline () +;; (let ((current-elt (org-element-at-point))) +;; (if (equal 'headline (car current-elt)) +;; current-elt +;; (let* ((elt-attrs (cadr current-elt)) +;; (parent (plist-get elt-attrs :post-affiliated))) +;; (goto-char parent) +;; (org-element-find-headline))))) + +(defun org-element-find-headline () + (save-mark-and-excursion + (when (not (outline-on-heading-p)) (org-back-to-heading)) + (let ((current-elt (org-element-at-point))) + (when (equal 'headline (car current-elt)) + (cadr current-elt))))) + +(defun org-element-extract-clubhouse-id (elt &optional property) + (when-let* ((clubhouse-id-link (plist-get elt (or property :CLUBHOUSE-ID)))) + (cond + ((string-match + (rx "[[" (one-or-more anything) "]" + "[" (group (one-or-more digit)) "]]") + clubhouse-id-link) + (string-to-number (match-string 1 clubhouse-id-link))) + ((string-match-p + (rx buffer-start + (one-or-more digit) + buffer-end) + clubhouse-id-link) + (string-to-number clubhouse-id-link))))) + +(comment + (let ((strn "[[https://app.clubhouse.io/example/story/2330][2330]]")) + (string-match + (rx "[[" (one-or-more anything) "]" + "[" (group (one-or-more digit)) "]]") + strn) + (string-to-number (match-string 1 strn))) + ) + +(defun org-element-clubhouse-id () + (org-element-extract-clubhouse-id + (org-element-find-headline))) + +(defun org-clubhouse-clocked-in-story-id () + "Return the clubhouse story-id of the currently clocked-in org entry, if any." + (save-mark-and-excursion + (save-current-buffer + (when (org-clocking-p) + (set-buffer (marker-buffer org-clock-marker)) + (save-restriction + (when (or (< org-clock-marker (point-min)) + (> org-clock-marker (point-max))) + (widen)) + (goto-char org-clock-marker) + (org-element-clubhouse-id)))))) + +(comment + (org-clubhouse-clocked-in-story-id) + ) + +(defun org-element-and-children-at-point () + (let* ((elt (org-element-find-headline)) + (contents-begin (or (plist-get elt :contents-begin) + (plist-get elt :begin))) + (end (plist-get elt :end)) + (level (plist-get elt :level)) + (children '())) + (save-excursion + (goto-char (+ contents-begin (length (plist-get elt :title)))) + (while (< (point) end) + (let* ((next-elt (org-element-at-point)) + (elt-type (car next-elt)) + (elt (cadr next-elt))) + (when (and (eql 'headline elt-type) + (eql (+ 1 level) (plist-get elt :level))) + (push elt children)) + (goto-char (plist-get elt :end))))) + (append elt `(:children ,(reverse children))))) + +(defun +org-element-contents (elt) + (if-let ((begin (plist-get (cadr elt) :contents-begin)) + (end (plist-get (cadr elt) :contents-end))) + (buffer-substring-no-properties begin end) + "")) + +(defun org-clubhouse-find-description-drawer () + "Try to find a DESCRIPTION drawer in the current element." + (let ((elt (org-element-at-point))) + (cl-case (car elt) + ('drawer (+org-element-contents elt)) + ('headline + (when-let ((drawer-pos (string-match + ":DESCRIPTION:" + (+org-element-contents elt)))) + (save-excursion + (goto-char (+ (plist-get (cadr elt) :contents-begin) + drawer-pos)) + (org-clubhouse-find-description-drawer))))))) + +(defun org-clubhouse--labels-for-elt (elt) + "Return the Clubhouse labels based on the tags of ELT and the user's config." + (unless (eq nil org-clubhouse-create-stories-with-labels) + (let ((tags (org-get-tags (plist-get elt :contents-begin)))) + (-map (lambda (l) `((name . ,l))) + (cl-case org-clubhouse-create-stories-with-labels + ('t tags) + ('existing (-filter (lambda (tag) (-some (lambda (l) + (string-equal tag (cdr l))) + (org-clubhouse-labels))) + tags))))))) + +;;; +;;; API integration +;;; + +(defvar org-clubhouse-base-url* "https://api.clubhouse.io/api/v3") + +(defun org-clubhouse-auth-url (url &optional params) + (concat url + "?" + (url-build-query-string + (cons `("token" ,org-clubhouse-auth-token) params)))) + +(defun org-clubhouse-baseify-url (url) + (if (s-starts-with? org-clubhouse-base-url* url) url + (concat org-clubhouse-base-url* + (if (s-starts-with? "/" url) url + (concat "/" url))))) + +(cl-defun org-clubhouse-request (method url &key data (params '())) + (message "%s %s %s" method url (prin1-to-string data)) + (let* ((url-request-method method) + (url-request-extra-headers + '(("Content-Type" . "application/json"))) + (url-request-data data) + (buf)) + + (setq url (-> url + org-clubhouse-baseify-url + (org-clubhouse-auth-url params))) + + (setq buf (url-retrieve-synchronously url)) + + (with-current-buffer buf + (goto-char url-http-end-of-headers) + (prog1 (json-read) (kill-buffer))))) + +(cl-defun to-id-name-pairs + (seq &optional (id-attr 'id) (name-attr 'name)) + (->> seq + ->list + (-map (lambda (resource) + (cons (alist-get id-attr resource) + (alist-get name-attr resource)))))) + +(cl-defun org-clubhouse-fetch-as-id-name-pairs + (resource &optional + (id-attr 'id) + (name-attr 'name)) + "Returns the given resource from clubhouse as (id . name) pairs" + (let ((resp-json (org-clubhouse-request "GET" resource))) + (-> resp-json + ->list + reject-archived + (to-id-name-pairs id-attr name-attr)))) + +(defun org-clubhouse-get-story + (clubhouse-id) + (org-clubhouse-request "GET" (format "/stories/%s" clubhouse-id))) + +(defun org-clubhouse-link-to-story (story-id) + (format "https://app.clubhouse.io/%s/story/%d" + org-clubhouse-team-name + story-id)) + +(defun org-clubhouse-link-to-epic (epic-id) + (format "https://app.clubhouse.io/%s/epic/%d" + org-clubhouse-team-name + epic-id)) + +(defun org-clubhouse-link-to-milestone (milestone-id) + (format "https://app.clubhouse.io/%s/milestone/%d" + org-clubhouse-team-name + milestone-id)) + +(defun org-clubhouse-link-to-project (project-id) + (format "https://app.clubhouse.io/%s/project/%d" + org-clubhouse-team-name + project-id)) + +;;; +;;; Caching +;;; + +(comment + (defcache org-clubhouse-projects + (org-sync-clubhouse-fetch-as-id-name-pairs "projectx")) + + (clear-org-clubhouse-projects-cache) + (clear-org-clubhouse-cache) + ) + +(defvar org-clubhouse-cache-clear-functions ()) + +(defmacro defcache (name &optional docstring &rest body) + (let* ((doc (when docstring (list docstring))) + (cache-var-name (intern (concat (symbol-name name) + "-cache"))) + (clear-cache-function-name + (intern (concat "clear-" (symbol-name cache-var-name))))) + `(progn + (defvar ,cache-var-name :no-cache) + (defun ,name () + ,@doc + (when (equal :no-cache ,cache-var-name) + (setq ,cache-var-name (progn ,@body))) + ,cache-var-name) + (defun ,clear-cache-function-name () + (interactive) + (setq ,cache-var-name :no-cache)) + + (push (quote ,clear-cache-function-name) + org-clubhouse-cache-clear-functions)))) + +(defun org-clubhouse-clear-cache () + (interactive) + (-map #'funcall org-clubhouse-cache-clear-functions)) + +;;; +;;; API resource functions +;;; + +(defcache org-clubhouse-projects + "Returns projects as (project-id . name)" + (org-clubhouse-fetch-as-id-name-pairs "projects")) + +(defcache org-clubhouse-epics + "Returns epics as (epic-id . name)" + (org-clubhouse-fetch-as-id-name-pairs "epics")) + +(defcache org-clubhouse-milestones + "Returns milestone-id . name)" + (org-clubhouse-fetch-as-id-name-pairs "milestones")) + +(defcache org-clubhouse-workflow-states + "Returns worflow states as (name . id) pairs" + (let* ((resp-json (org-clubhouse-request "GET" "workflows")) + (workflows (->list resp-json)) + ;; just assume it exists, for now + (workflow (-find (lambda (workflow) + (equal org-clubhouse-workflow-name + (alist-get 'name workflow))) + workflows)) + (states (->list (alist-get 'states workflow)))) + (to-id-name-pairs states + 'name + 'id))) + +(defcache org-clubhouse-labels + "Returns labels as (label-id . name)" + (org-clubhouse-fetch-as-id-name-pairs "labels")) + +(defcache org-clubhouse-whoami + "Returns the ID of the logged in user" + (->> (org-clubhouse-request + "GET" + "/members") + ->list + (find-if (lambda (m) + (->> m + (alist-get 'profile) + (alist-get 'mention_name) + (equal org-clubhouse-username)))) + (alist-get 'id))) + +(defcache org-clubhouse-iterations + "Returns iterations as (iteration-id . name)" + (org-clubhouse-fetch-as-id-name-pairs "iterations")) + +(defun org-clubhouse-stories-in-project (project-id) + "Return the stories in the given PROJECT-ID as org headlines." + (let ((resp-json (org-clubhouse-request "GET" (format "/projects/%d/stories" project-id)))) + (->> resp-json ->list reject-archived + (-reject (lambda (story) (equal :json-true (alist-get 'completed story)))) + (-map (lambda (story) + (cons + (cons 'status + (cond + ((equal :json-true (alist-get 'started story)) + 'started) + ((equal :json-true (alist-get 'completed story)) + 'completed) + ('t + 'open))) + story))) + (-map (-partial #'alist->plist + '((name . :title) + (id . :id) + (status . :status))))))) + +(defun org-clubhouse-workflow-state-id-to-todo-keyword (workflow-state-id) + "Convert the named clubhouse WORKFLOW-STATE-ID to an org todo keyword." + (let* ((state-name (alist-get-equal + workflow-state-id + (invert-alist (org-clubhouse-workflow-states)))) + (inv-state-name-alist + (-map (lambda (cell) (cons (cdr cell) (car cell))) + org-clubhouse-state-alist))) + (or (alist-get-equal state-name inv-state-name-alist) + (if state-name (s-upcase state-name) "UNKNOWN")))) + +;;; +;;; Prompting +;;; + +(defun org-clubhouse-prompt-for-project (cb) + (ivy-read + "Select a project: " + (-map #'cdr (org-clubhouse-projects)) + :require-match t + :history 'org-clubhouse-project-history + :action (lambda (selected) + (let ((project-id + (find-match-in-alist selected (org-clubhouse-projects)))) + (funcall cb project-id))))) + +(defun org-clubhouse-prompt-for-epic (cb) + "Prompt the user for an epic using ivy and call CB with its ID." + (ivy-read + "Select an epic: " + (-map #'cdr (append '((nil . "No Epic")) (org-clubhouse-epics))) + :history 'org-clubhouse-epic-history + :action (lambda (selected) + (let ((epic-id + (find-match-in-alist selected (org-clubhouse-epics)))) + (funcall cb epic-id))))) + +(defun org-clubhouse-prompt-for-milestone (cb) + "Prompt the user for a milestone using ivy and call CB with its ID." + (ivy-read + "Select a milestone: " + (-map #'cdr (append '((nil . "No Milestone")) (org-clubhouse-milestones))) + :require-match t + :history 'org-clubhouse-milestone-history + :action (lambda (selected) + (let ((milestone-id + (find-match-in-alist selected (org-clubhouse-milestones)))) + (funcall cb milestone-id))))) + +(defun org-clubhouse-prompt-for-story-type (cb) + (ivy-read + "Select a story type: " + (-map #'cdr org-clubhouse-story-types) + :history 'org-clubhouse-story-history + :action (lambda (selected) + (let ((story-type + (find-match-in-alist selected org-clubhouse-story-types))) + (funcall cb story-type))))) + +(defun org-clubhouse-prompt-for-default-story-type () + (interactive) + (ivy-read + "Select a default story type: " + (-map #'cdr org-clubhouse-default-story-types) + :history 'org-clubhouse-default-story-history + :action (lambda (selected) + (let ((story-type + (find-match-in-alist selected org-clubhouse-default-story-types))) + (if (string-equal story-type "prompt") + (setq org-clubhouse-default-story-type nil) + (setq org-clubhouse-default-story-type story-type)))))) + +;;; +;;; Epic creation +;;; + +(cl-defun org-clubhouse-create-epic-internal + (title &key milestone-id) + (cl-assert (and (stringp title) + (or (null milestone-id) + (integerp milestone-id)))) + (org-clubhouse-request + "POST" + "epics" + :data + (json-encode + `((name . ,title) + (milestone_id . ,milestone-id))))) + +(defun org-clubhouse-populate-created-epic (elt epic) + (let ((elt-start (plist-get elt :begin)) + (epic-id (alist-get 'id epic)) + (milestone-id (alist-get 'milestone_id epic))) + (save-excursion + (goto-char elt-start) + + (org-set-property "clubhouse-epic-id" + (org-link-make-string + (org-clubhouse-link-to-epic epic-id) + (number-to-string epic-id))) + + (when milestone-id + (org-set-property "clubhouse-milestone" + (org-link-make-string + (org-clubhouse-link-to-milestone milestone-id) + (alist-get milestone-id (org-clubhouse-milestones)))))))) + +(defun org-clubhouse-create-epic (&optional beg end) + "Creates a clubhouse epic using selected headlines. +Will pull the title from the headline at point, or create epics for all the +headlines in the selected region. + +All epics are added to the same milestone, as selected via a prompt. +If the epics already have a CLUBHOUSE-EPIC-ID, they are filtered and ignored." + (interactive + (when (use-region-p) + (list (region-beginning region-end)))) + + (let* ((elts (org-clubhouse-collect-headlines beg end)) + (elts (-remove (lambda (elt) (plist-get elt :CLUBHOUSE-EPIC-ID)) elts))) + (org-clubhouse-prompt-for-milestone + (lambda (milestone-id) + (dolist (elt elts) + (let* ((title (plist-get elt :title)) + (epic (org-clubhouse-create-epic-internal + title + :milestone-id milestone-id))) + (org-clubhouse-populate-created-epic elt epic)) + elts))))) + +;;; +;;; Story creation +;;; + +(defun org-clubhouse-default-state-id () + (alist-get-equal org-clubhouse-default-state (org-clubhouse-workflow-states))) + +(cl-defun org-clubhouse-create-story-internal + (title &key project-id epic-id story-type description labels) + (cl-assert (and (stringp title) + (integerp project-id) + (or (null epic-id) (integerp epic-id)) + (or (null description) (stringp description)))) + (let ((workflow-state-id (org-clubhouse-default-state-id)) + (params `((name . ,title) + (project_id . ,project-id) + (epic_id . ,epic-id) + (story_type . ,story-type) + (description . ,(or description "")) + (labels . ,labels)))) + + (when workflow-state-id + (push `(workflow_state_id . ,workflow-state-id) params)) + + (org-clubhouse-request + "POST" + "stories" + :data + (json-encode params)))) + +(cl-defun org-clubhouse-populate-created-story (elt story &key extra-properties) + (let ((elt-start (plist-get elt :begin)) + (story-id (alist-get 'id story)) + (epic-id (alist-get 'epic_id story)) + (project-id (alist-get 'project_id story)) + (story-type (alist-get 'story_type story))) + + (save-excursion + (goto-char elt-start) + + (org-set-property "clubhouse-id" + (org-link-make-string + (org-clubhouse-link-to-story story-id) + (number-to-string story-id))) + (when epic-id + (org-set-property "clubhouse-epic" + (org-link-make-string + (org-clubhouse-link-to-epic epic-id) + (alist-get epic-id (org-clubhouse-epics))))) + + (org-set-property "clubhouse-project" + (org-link-make-string + (org-clubhouse-link-to-project project-id) + (alist-get project-id (org-clubhouse-projects)))) + + (org-set-property "story-type" + (alist-get-equal story-type org-clubhouse-story-types)) + + (dolist (extra-prop extra-properties) + (org-set-property (car extra-prop) + (alist-get (cdr extra-prop) story))) + + (org-todo "TODO")))) + +(defun org-clubhouse-create-story (&optional beg end &key then) + "Creates a clubhouse story using selected headlines. + +Will pull the title from the headline at point, +or create cards for all the headlines in the selected region. + +All stories are added to the same project and epic, as selected via two prompts. +If the stories already have a CLUBHOUSE-ID, they are filtered and ignored." + (interactive + (when (use-region-p) + (list (region-beginning) (region-end)))) + + (let* ((elts (org-clubhouse-collect-headlines beg end)) + (new-elts (-remove (lambda (elt) (plist-get elt :CLUBHOUSE-ID)) elts))) + (org-clubhouse-prompt-for-project + (lambda (project-id) + (when project-id + (org-clubhouse-prompt-for-epic + (lambda (epic-id) + (let ((create-story + (lambda (story-type) + (-map + (lambda (elt) + (let* ((title (plist-get elt :title)) + (description + (save-mark-and-excursion + (goto-char (plist-get elt :begin)) + (org-clubhouse-find-description-drawer))) + (labels (org-clubhouse--labels-for-elt elt)) + (story (org-clubhouse-create-story-internal + title + :project-id project-id + :epic-id epic-id + :story-type story-type + :description description + :labels labels))) + (org-clubhouse-populate-created-story elt story) + (when (functionp then) + (funcall then story)))) + new-elts)))) + (if org-clubhouse-default-story-type + (funcall create-story org-clubhouse-default-story-type) + (org-clubhouse-prompt-for-story-type create-story)))))))))) + +(defun org-clubhouse-create-story-with-task-list (&optional beg end) + "Creates a clubhouse story using the selected headline, making all direct +children of that headline into tasks in the task list of the story." + (interactive + (when (use-region-p) + (list (region-beginning) (region-end)))) + + (let* ((elt (org-element-and-children-at-point))) + (org-clubhouse-create-story nil nil + :then (lambda (story) + (pp story) + (org-clubhouse-push-task-list + (alist-get 'id story) + (plist-get elt :children)))))) + +;;; +;;; Task creation +;;; + +(cl-defun org-clubhouse-create-task (title &key story-id) + (cl-assert (and (stringp title) + (integerp story-id))) + (org-clubhouse-request + "POST" + (format "/stories/%d/tasks" story-id) + :data (json-encode `((description . ,title))))) + +(defun org-clubhouse-push-task-list (&optional parent-clubhouse-id child-elts) + "Writes each child of the element at point as a task list item. + +When called as (org-clubhouse-push-task-list PARENT-CLUBHOUSE-ID CHILD-ELTS), +allows manually passing a clubhouse ID and list of org-element plists to write" + (interactive) + (let* ((elt (org-element-and-children-at-point)) + (parent-clubhouse-id (or parent-clubhouse-id + (org-element-extract-clubhouse-id elt))) + (child-elts (or child-elts (plist-get elt :children))) + (story (org-clubhouse-get-story parent-clubhouse-id)) + (existing-tasks (alist-get 'tasks story)) + (task-exists + (lambda (task-name) + (cl-some (lambda (task) + (string-equal task-name (alist-get 'description task))) + existing-tasks))) + (elts-with-starts + (-map (lambda (e) (cons (set-marker (make-marker) + (plist-get e :begin)) + e)) + child-elts))) + (dolist (child-elt-and-start elts-with-starts) + (let* ((start (car child-elt-and-start)) + (child-elt (cdr child-elt-and-start)) + (task-name (plist-get child-elt :title))) + (unless (funcall task-exists task-name) + (let ((task (org-clubhouse-create-task + task-name + :story-id parent-clubhouse-id))) + (org-clubhouse-populate-created-task child-elt task start))))))) + +(defun org-clubhouse-populate-created-task (elt task &optional begin) + (let ((elt-start (or begin (plist-get elt :begin))) + (task-id (alist-get 'id task)) + (story-id (alist-get 'story_id task))) + + (save-excursion + (goto-char elt-start) + + (org-set-property "clubhouse-task-id" (format "%d" task-id)) + + (org-set-property "clubhouse-story-id" + (org-link-make-string + (org-clubhouse-link-to-story story-id) + (number-to-string story-id))) + + (org-todo "TODO")))) + +;;; +;;; Task Updates +;;; + +(cl-defun org-clubhouse-update-task-internal + (story-id task-id &rest attrs) + (cl-assert (and (integerp story-id) + (integerp task-id) + (listp attrs))) + (org-clubhouse-request + "PUT" + (format "stories/%d/tasks/%d" story-id task-id) + :data + (json-encode attrs))) + +;;; +;;; Story updates +;;; + +(cl-defun org-clubhouse-update-story-internal + (story-id &rest attrs) + (cl-assert (and (integerp story-id) + (listp attrs))) + (org-clubhouse-request + "PUT" + (format "stories/%d" story-id) + :data + (json-encode attrs))) + +(cl-defun org-clubhouse-update-story-at-point (&rest attrs) + (when-let* ((clubhouse-id (org-element-clubhouse-id))) + (apply + #'org-clubhouse-update-story-internal + (cons clubhouse-id attrs)) + t)) + +(defun org-clubhouse-update-story-title () + "Update the title of the Clubhouse story linked to the current headline. + +Update the title of the story linked to the current headline with the text of +the headline." + (interactive) + + (let* ((elt (org-element-find-headline)) + (title (plist-get elt :title)) + (clubhouse-id (org-element-clubhouse-id))) + (and + (org-clubhouse-update-story-at-point + clubhouse-id + :name title) + (message "Successfully updated story title to \"%s\"" + title)))) + +(defun org-clubhouse-update-status () + "Update the status of the Clubhouse story linked to the current element. + +Update the status of the Clubhouse story linked to the current element with the +entry in `org-clubhouse-state-alist' corresponding to the todo-keyword of the +element." + (interactive) + (let* ((elt (org-element-find-headline)) + (todo-keyword (-> elt + (plist-get :todo-keyword) + (substring-no-properties))) + + (clubhouse-id (org-element-extract-clubhouse-id elt)) + (task-id (plist-get elt :CLUBHOUSE-TASK-ID))) + (cond + (clubhouse-id + (let* ((todo-keyword (-> elt + (plist-get :todo-keyword) + (substring-no-properties)))) + (when-let* ((clubhouse-workflow-state + (alist-get-equal todo-keyword org-clubhouse-state-alist)) + (workflow-state-id + (alist-get-equal clubhouse-workflow-state + (org-clubhouse-workflow-states)))) + (let ((update-assignee? + (if (or (eq 't org-clubhouse-claim-story-on-status-update) + (member todo-keyword + org-clubhouse-claim-story-on-status-update)) + (if org-clubhouse-username + 't + (warn "Not claiming story since `org-clubhouse-username' + is not set") + nil)))) + + (if update-assignee? + (org-clubhouse-update-story-internal + clubhouse-id + :workflow_state_id workflow-state-id + :owner_ids (if update-assignee? + (list (org-clubhouse-whoami)) + (list))) + (org-clubhouse-update-story-internal + clubhouse-id + :workflow_state_id workflow-state-id)) + (message + (if update-assignee? + "Successfully claimed story and updated clubhouse status to \"%s\"" + "Successfully updated clubhouse status to \"%s\"") + clubhouse-workflow-state))))) + + (task-id + (let ((story-id (org-element-extract-clubhouse-id + elt + :CLUBHOUSE-STORY-ID)) + (done? (member todo-keyword org-done-keywords))) + (org-clubhouse-update-task-internal + story-id + (string-to-number task-id) + :complete (if done? 't :json-false)) + (message "Successfully marked clubhouse task status as %s" + (if done? "complete" "incomplete"))))))) + +(defun org-clubhouse-update-description () + "Update the description of the Clubhouse story linked to the current element. + +Update the status of the Clubhouse story linked to the current element with the +contents of a drawer inside the element called DESCRIPTION, if any." + (interactive) + (when-let* ((new-description (org-clubhouse-find-description-drawer))) + (and + (org-clubhouse-update-story-at-point + :description new-description) + (message "Successfully updated story description")))) + +(defun org-clubhouse-update-labels () + "Update the labels of the Clubhouse story linked to the current element. + +Will use the value of `org-clubhouse-create-stories-with-labels' to determine +which labels to set." + (interactive) + (when-let* ((elt (org-element-find-headline)) + (new-labels (org-clubhouse--labels-for-elt elt))) + (and + (org-clubhouse-update-story-at-point + :labels new-labels) + (message "Successfully updated story labels to :%s:" + (->> new-labels + (-map #'cdar) + (s-join ":")))))) + + +;;; +;;; Creating headlines from existing stories +;;; + +(defun org-clubhouse--task-to-headline-text (level task) + (format "%s %s %s +:PROPERTIES: +:clubhouse-task-id: %s +:clubhouse-story-id: %s +:END:" + (make-string level ?*) + (if (equal :json-false (alist-get 'complete task)) + "TODO" "DONE") + (alist-get 'description task) + (alist-get 'id task) + (let ((story-id (alist-get 'story_id task))) + (org-link-make-string + (org-clubhouse-link-to-story story-id) + story-id)))) + +(defun org-clubhouse--story-to-headline-text (level story) + (let ((story-id (alist-get 'id story))) + (format + "%s %s %s %s +:PROPERTIES: +:clubhouse-id: %s +:END: +%s +%s +" + (make-string level ?*) + (org-clubhouse-workflow-state-id-to-todo-keyword + (alist-get 'workflow_state_id story)) + (alist-get 'name story) + (if-let ((labels (->> story + (alist-get 'labels) + ->list + (-map (apply-partially #'alist-get 'name))))) + (format ":%s:" (s-join ":" labels)) + "") + (org-link-make-string + (org-clubhouse-link-to-story story-id) + (number-to-string story-id)) + (let ((desc (alist-get 'description story))) + (if (= 0 (length desc)) "" + (format ":DESCRIPTION:\n%s\n:END:" desc))) + (if-let ((tasks (seq-sort-by + (apply-partially #'alist-get 'position) + #'< + (or (alist-get 'tasks story) + (alist-get 'tasks + (org-clubhouse-get-story story-id)))))) + (mapconcat (apply-partially #'org-clubhouse--task-to-headline-text + (1+ level)) + tasks + "\n") + "")))) + +(defun org-clubhouse-headline-from-my-tasks (level) + "Prompt my active stories and create a single `org-mode' headline at LEVEL." + (interactive "*nLevel: \n") + (if org-clubhouse-username + (let* ((story-list (org-clubhouse--search-stories + (format "owner:%s !is:done !is:archived" + org-clubhouse-username))) + (stories (to-id-name-pairs story-list))) + (org-clubhouse-headline-from-story-id level + (find-match-in-alist + (ivy-read "Select Story: " + (-map #'cdr stories)) + stories))) + (warn "Can't fetch my tasks if `org-clubhouse-username' is unset"))) + +(defun org-clubhouse-headline-from-story-id (level story-id) + "Create a single `org-mode' headline at LEVEL based on the given clubhouse STORY-ID." + (interactive "*nLevel: \nnStory ID: ") + (let* ((story (org-clubhouse-get-story story-id))) + (if (equal '((message . "Resource not found.")) story) + (message "Story ID not found: %d" story-id) + (save-mark-and-excursion + (insert (org-clubhouse--story-to-headline-text level story)) + (org-align-tags))))) + +(defun org-clubhouse--search-stories (query) + (unless (string= "" query) + (-> (org-clubhouse-request "GET" "search/stories" :params `((query ,query))) + cdadr + (append nil) + reject-archived))) + +(defun org-clubhouse-prompt-for-iteration (cb) + "Prompt for iteration and call CB with that iteration" + (ivy-read + "Select an interation: " + (-map #'cdr (org-clubhouse-iterations)) + :require-match t + :history 'org-clubhouse-iteration-history + :action (lambda (selected) + (let ((iteration-id + (find-match-in-alist selected (org-clubhouse-iterations)))) + (funcall cb iteration-id))))) + +(defun org-clubhouse--get-iteration (iteration-id) + (-> (org-clubhouse-request "GET" (format "iterations/%d/stories" iteration-id)) + (append nil))) + +(defun org-clubhouse-headlines-from-iteration (level) + "Create `org-mode' headlines from a clubhouse iteration. + +Create `org-mode' headlines from all the resulting stories at headline level LEVEL." + (interactive "*nLevel: ") + (org-clubhouse-prompt-for-iteration + (lambda (iteration-id) + (let ((story-list (org-clubhouse--get-iteration iteration-id))) + (if (null story-list) + (message "Iteration id returned no stories: %d" iteration-id) + (let ((text (mapconcat (apply-partially + #'org-clubhouse--story-to-headline-text + level) + (reject-archived story-list) "\n"))) + (save-mark-and-excursion + (insert text) + (org-align-all-tags)) + text)))))) + +(defun org-clubhouse-headlines-from-query (level query) + "Create `org-mode' headlines from a clubhouse query. + +Submits QUERY to clubhouse, and creates `org-mode' headlines from all the +resulting stories at headline level LEVEL." + (interactive + "*nLevel: \nMQuery: ") + (let* ((story-list (org-clubhouse--search-stories query))) + (if (null story-list) + (message "Query returned no stories: %s" query) + (let ((text (mapconcat (apply-partially + #'org-clubhouse--story-to-headline-text + level) + (reject-archived story-list) "\n"))) + (if (called-interactively-p) + (save-mark-and-excursion + (insert text) + (org-align-all-tags)) + text))))) + +(defun org-clubhouse-prompt-for-story (cb) + "Prompt the user for a clubhouse story, then call CB with the full story." + (ivy-read "Story title: " + (lambda (search-term) + (let* ((stories (org-clubhouse--search-stories + (if search-term (format "\"%s\"" search-term) + "")))) + (-map (lambda (story) + (propertize (alist-get 'name story) 'story story)) + stories))) + :dynamic-collection t + :history 'org-clubhouse-story-prompt + :action (lambda (s) (funcall cb (get-text-property 0 'story s))) + :require-match t)) + +(defun org-clubhouse-headline-from-story (level) + "Prompt for a story, and create an org headline at LEVEL from that story." + (interactive "*nLevel: ") + (org-clubhouse-prompt-for-story + (lambda (story) + (save-mark-and-excursion + (insert (org-clubhouse--story-to-headline-text level story)) + (org-align-tags))))) + + +(defun org-clubhouse-link () + "Link the current `org-mode' headline with an existing clubhouse story." + (interactive) + (org-clubhouse-prompt-for-story + (lambda (story) + (org-clubhouse-populate-created-story + (org-element-find-headline) + story + :extra-properties '(("clubhouse-story-name" . name))) + (org-todo + (org-clubhouse-workflow-state-id-to-todo-keyword + (alist-get 'workflow_state_id story)))))) + +(defun org-clubhouse-claim () + "Assign the clubhouse story associated with the headline at point to yourself." + (interactive) + (if org-clubhouse-username + (and + (org-clubhouse-update-story-at-point + :owner_ids (list (org-clubhouse-whoami))) + (message "Successfully claimed story")) + (warn "Can't claim story if `org-clubhouse-username' is unset"))) + +(defun org-clubhouse-sync-status (&optional beg end) + "Pull the status(es) for the story(ies) in region and update the todo state. + +Uses `org-clubhouse-state-alist'. Operates over stories from BEG to END" + (interactive + (when (use-region-p) + (list (region-beginning) (region-end)))) + (let ((elts (-filter (lambda (e) (plist-get e :CLUBHOUSE-ID)) + (org-clubhouse-collect-headlines beg end)))) + (save-mark-and-excursion + (dolist (e elts) + (goto-char (plist-get e :begin)) + (let* ((clubhouse-id (org-element-extract-clubhouse-id e)) + (story (org-clubhouse-get-story clubhouse-id)) + (workflow-state-id (alist-get 'workflow_state_id story)) + (todo-keyword (org-clubhouse-workflow-state-id-to-todo-keyword + workflow-state-id))) + (let ((org-after-todo-state-change-hook + (remove 'org-clubhouse-update-status + org-after-todo-state-change-hook))) + (org-todo todo-keyword))))) + (message "Successfully synchronized status of %d stories from Clubhouse" + (length elts)))) + +(cl-defun org-clubhouse-set-epic (&optional story-id epic-id cb &key beg end) + "Set the epic of clubhouse story STORY-ID to EPIC-ID, then call CB. + +When called interactively, prompt for an epic and set the story of the clubhouse +stor{y,ies} at point or region" + (interactive + (when (use-region-p) + (list nil nil nil + :beg (region-beginning) + :end (region-end)))) + (if (and story-id epic-id) + (progn + (org-clubhouse-update-story-internal + story-id :epic-id epic-id) + (when cb (funcall cb))) + (let ((elts (-filter (lambda (elt) (plist-get elt :CLUBHOUSE-ID)) + (org-clubhouse-collect-headlines beg end)))) + (org-clubhouse-prompt-for-epic + (lambda (epic-id) + (-map + (lambda (elt) + (let ((story-id (org-element-extract-clubhouse-id elt))) + (org-clubhouse-set-epic + story-id epic-id + (lambda () + (org-set-property + "clubhouse-epic" + (org-link-make-string + (org-clubhouse-link-to-epic epic-id) + (alist-get epic-id (org-clubhouse-epics)))) + (message "Successfully set the epic on story %d to %d" + story-id epic-id)))))) + elts))))) + +;;; + +(define-minor-mode org-clubhouse-mode + "If enabled, updates to the todo keywords on org headlines will update the +linked ticket in Clubhouse." + :group 'org + :lighter "Org-Clubhouse" + :keymap '() + (add-hook 'org-after-todo-state-change-hook + 'org-clubhouse-update-status + nil + t)) + +(provide 'org-clubhouse) + +;;; org-clubhouse.el ends here diff --git a/users/grfn/resume/chimera.png b/users/grfn/resume/chimera.png new file mode 100644 index 000000000000..6dde989c53b0 --- /dev/null +++ b/users/grfn/resume/chimera.png Binary files differdiff --git a/users/grfn/resume/collection.sty b/users/grfn/resume/collection.sty new file mode 100644 index 000000000000..4f1540a9d214 --- /dev/null +++ b/users/grfn/resume/collection.sty @@ -0,0 +1,85 @@ +%% start of file `collection.sty'. +%% Copyright 2013-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{collection}[2013/03/28 v1.0.0 collections] + + +%------------------------------------------------------------------------------- +% requirements +%------------------------------------------------------------------------------- + + +\RequirePackage{ifthen} + + +%------------------------------------------------------------------------------- +% code +%------------------------------------------------------------------------------- + +% creates a new collection +% usage: \collectionnew{<collection name>} +\newcommand*{\collectionnew}[1]{% + \newcounter{collection@#1@count}} + +% adds an item to a collection +% usage: \collectionadd[<optional key>]{<collection name>}{<item to add>} +\newcommand*{\collectionadd}[3][]{% + \expandafter\def\csname collection@#2@item\roman{collection@#2@count}\endcsname{#3}% + \if\relax\noexpand#1\relax% if #1 is empty + \else\expandafter\def\csname collection@#2@key\roman{collection@#2@count}\endcsname{#1}\fi% + \stepcounter{collection@#2@count}} + +% returns the number of items in a collection +% usage: \collectioncount{<collection name>} +\newcommand*{\collectioncount}[1]{% + \value{collection@#1@count}} + +% gets an item from a collection +% usage: \collectiongetitem{<collection name>}{<element id>} +% where <element id> is an integer between 0 and (collectioncount-1) +\newcommand*{\collectiongetitem}[2]{% + \csname collection@#1@item\romannumeral #2\endcsname} + +% gets a key from a collection +% usage: \collectiongetkey{<collection name>}{<element id>} +% where <element id> is an integer between 0 and (collectioncount-1) +\newcommand*{\collectiongetkey}[2]{% + \csname collection@#1@key\romannumeral #2\endcsname} + +% loops through a collection and perform the given operation on every element +% usage: \collectionloop{<collection name>}{<operation sequence>} +% where <operation sequence> is the code sequence to be evaluated for each collection item, +% code which can refer to \collectionloopid, \collectionloopkey, \collectionloopitem and +% \collectionloopbreak +\newcounter{collection@iterator} +\newcommand*{\collectionloopbreak}{\let\iterate\relax} +\newcommand*{\collectionloop}[2]{% + \setcounter{collection@iterator}{0}% + \loop\ifnum\value{collection@iterator}<\value{collection@#1@count}% + \def\collectionloopid{\arabic{collection@iterator}}% + \def\collectionloopitem{\collectiongetitem{#1}{\collectionloopid}}% + \def\collectionloopkey{\collectiongetkey{#1}{\collectionloopid}}% + #2% + \stepcounter{collection@iterator}% + \repeat} + +% loops through a collection and finds the (first) element matching the given key +% usage: \collectionfindbykey{<collection name>}{key>} +\newcommand*{\collectionfindbykey}[2]{% + \collectionloop{#1}{% + \ifthenelse{\equal{\collectionloopkey}{#2}}{\collectionloopitem\collectionloopbreak}{}}} + + +\endinput + + +%% end of file `collection.cls'. diff --git a/users/grfn/resume/default.nix b/users/grfn/resume/default.nix new file mode 100644 index 000000000000..2db6a650bc6e --- /dev/null +++ b/users/grfn/resume/default.nix @@ -0,0 +1,37 @@ +{ pkgs, ... }: + +with pkgs.lib; + +pkgs.runCommandNoCC "resume.pdf" { + buildInputs = [(pkgs.texlive.combine { + inherit (pkgs.texlive) + capt-of + collection-fontsrecommended + enumitem + etoolbox + fancyvrb + float + fncychap + framed + l3packages + microtype + needspace + parskip + scheme-basic + tabulary + titlesec + ulem + upquote + varwidth + wrapfig + xcolor + ; + })]; +} '' + cp ${builtins.filterSource (path: type: + type == "regular" && + any (ext: hasSuffix ext path) [".sty" ".cls" ".tex" ".png"] + ) ./.}/* . + pdflatex ./resume.tex + cp resume.pdf $out +'' diff --git a/users/grfn/resume/helvetica.sty b/users/grfn/resume/helvetica.sty new file mode 100644 index 000000000000..dacc129a1025 --- /dev/null +++ b/users/grfn/resume/helvetica.sty @@ -0,0 +1,32 @@ +%% +%% This is file `helvetica.sty', based on helvet.sty extended to include +%% definitions for rm and tt. This means commands such as \textbf, \textit, +%% etc. will appear in Helvetica. +%% Changes added by Harriet Borton on <1995/12/11> +%% +%% The original source files were: +%% +%% psfonts.dtx (with options: `helvet') +%% +%% Copyright (C) 1994 Sebastian Rahtz +%% All rights reserved. +%% +%% The original file is part of the PSNFSS2e package. +%% ----------------------------------------- +%% +%% This is a generated file. Permission is granted to to customize the +%% declarations in this file to serve the needs of your installation. +%% However, no permission is granted to distribute a modified version of +%% this file under its original name. +\def\fileversion{4.2} +\def\filedate{94/11/11} +\def\docdate {94/11/06} +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{helvetica}[\filedate\space\fileversion\space +Helvetica PSNFSS2e package] +\renewcommand{\sfdefault}{phv} +\renewcommand{\rmdefault}{phv} +\renewcommand{\ttdefault}{pcr} +\endinput +%% +%% End of file `helvetica.sty'. diff --git a/users/grfn/resume/moderncv.cls b/users/grfn/resume/moderncv.cls new file mode 100644 index 000000000000..a40f80733736 --- /dev/null +++ b/users/grfn/resume/moderncv.cls @@ -0,0 +1,585 @@ +%% start of file `moderncv.cls'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesClass{moderncv}[2013/02/09 v1.3.0 modern curriculum vitae and letter document class] + + +%------------------------------------------------------------------------------- +% class options +% +% (need to be done before the external package loading, for example because +% we need \paperwidth, \paperheight and \@ptsize to be defined before loading +% geometry and fancyhdr) +%------------------------------------------------------------------------------- +% paper size option +\DeclareOption{a4paper}{ + \setlength\paperheight{297mm} + \setlength\paperwidth{210mm}} +\DeclareOption{a5paper}{ + \setlength\paperheight{210mm} + \setlength\paperwidth{148mm}} +\DeclareOption{b5paper}{ + \setlength\paperheight{250mm} + \setlength\paperwidth{176mm}} +\DeclareOption{letterpaper}{ + \setlength\paperheight{11in} + \setlength\paperwidth{8.5in}} +\DeclareOption{legalpaper}{ + \setlength\paperheight{14in} + \setlength\paperwidth{8.5in}} +\DeclareOption{executivepaper}{ + \setlength\paperheight{10.5in} + \setlength\paperwidth{7.25in}} +\DeclareOption{landscape}{ + \setlength\@tempdima{\paperheight} + \setlength\paperheight{\paperwidth} + \setlength\paperwidth{\@tempdima}} + +% font size options +\newcommand\@ptsize{} +\DeclareOption{10pt}{\renewcommand\@ptsize{0}} +\DeclareOption{11pt}{\renewcommand\@ptsize{1}} +\DeclareOption{12pt}{\renewcommand\@ptsize{2}} + +% font type options +\DeclareOption{sans}{\AtBeginDocument{\renewcommand{\familydefault}{\sfdefault}}} +\DeclareOption{roman}{\AtBeginDocument{\renewcommand{\familydefault}{\rmdefault}}} + +% draft/final option +\DeclareOption{draft}{\setlength\overfullrule{5pt}} +\DeclareOption{final}{\setlength\overfullrule{0pt}} + +% execute default options +\ExecuteOptions{a4paper,11pt,final} + +% process given options +\ProcessOptions\relax +\input{size1\@ptsize.clo} + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- +% \AtEndPreamble hook (loading etoolbox instead of defining the macro, as to avoid incompatibilities with etoolbox (and packages relying on it) defining the macro too) +\RequirePackage{etoolbox} +%\let\@endpreamblehook\@empty +%\def\AtEndPreamble{\g@addto@macro\@endpreamblehook} +%\let\document@original\document +%\def\document{\endgroup\@endpreamblehook\begingroup\document@original} + +% if... then... else... constructs +\RequirePackage{ifthen} +% TODO: move to xifthen and \isempty{<arg>} instead of \equal{<arg>}{} + +% color +\RequirePackage{xcolor} + +% font loading +%\RequirePackage{ifxetex,ifluatex} +%\newif\ifxetexorluatex +%\ifxetex +% \xetexorluatextrue +%\else +% \ifluatex +% \xetexorluatextrue +% \else +% \xetexorluatexfalse +% \fi +%\fi +% automatic loading of latin modern fonts +%\ifxetexorluatex +% \RequirePackage{fontspec} +% \defaultfontfeatures{Ligatures=TeX} +% \RequirePackage{unicode-math} +% \setmainfont{Latin Modern} +% \setsansfont{Latin Modern Sans} +% \setmathfont{Latin Modern Math} +%\else + \RequirePackage[T1]{fontenc} + \IfFileExists{lmodern.sty}% + {\RequirePackage{lmodern}}% + {} +%\fi + +% hyper links (hyperref is loaded at the end of the preamble to pass options required by loaded packages such as CJK) +\newcommand*\pdfpagemode{UseNone}% do not show thumbnails or bookmarks on opening (on supporting browsers); set \pdfpagemode to "UseOutlines" to show bookmarks +\RequirePackage{url} +\urlstyle{tt} +\AtEndPreamble{ + \pagenumbering{arabic}% has to be issued before loading hyperref, as to set \thepage and hence to avoid hyperref issuing a warning and setting pdfpagelabels=false + \RequirePackage[unicode]{hyperref}% unicode is required for unicode pdf metadata + \hypersetup{ + breaklinks, + baseurl = http://, + pdfborder = 0 0 0, + pdfpagemode = \pdfpagemode, + pdfstartpage = 1, + pdfcreator = {\LaTeX{} with 'moderncv' package}, +% pdfproducer = {\LaTeX{}},% will/should be set automatically to the correct TeX engine used + bookmarksopen = true, + bookmarksdepth= 2,% to show sections and subsections + pdfauthor = {\@firstname{}~\@lastname{}}, + pdftitle = {\@firstname{}~\@lastname{} -- \@title{}}, + pdfsubject = {Resum\'{e} of \@firstname{}~\@lastname{}}, + pdfkeywords = {\@firstname{}~\@lastname{}, curriculum vit\ae{}, resum\'{e}}}} + +% graphics +\RequirePackage{graphicx} + +% headers and footers +\RequirePackage{fancyhdr} +\fancypagestyle{plain}{ + \renewcommand{\headrulewidth}{0pt} + \renewcommand{\footrulewidth}{0pt} + \fancyhf{}} +% page numbers in footer if more than 1 page +\newif\if@displaypagenumbers\@displaypagenumberstrue +\newcommand*{\nopagenumbers}{\@displaypagenumbersfalse} +\AtEndPreamble{% + \AtBeginDocument{% + \if@displaypagenumbers% + \@ifundefined{r@lastpage}{}{% + \ifthenelse{\pageref{lastpage}>1}{% + \newlength{\pagenumberwidth}% + \settowidth{\pagenumberwidth}{\color{color2}\addressfont\itshape\strut\thepage/\pageref{lastpage}}% + \fancypagestyle{plain}{% + \fancyfoot[r]{\parbox[b]{\pagenumberwidth}{\color{color2}\pagenumberfont\strut\thepage/\pageref{lastpage}}}}% the parbox is required to ensure alignment with a possible center footer (e.g., as in the casual style) + \pagestyle{plain}}{}}% + \AtEndDocument{\label{lastpage}}\else\fi}} +\pagestyle{plain} + +% reduced list spacing +% package providing hooks into lists +% originally developped by Jakob Schiøtz (see http://dcwww.camd.dtu.dk/~schiotz/comp/LatexTips/tweaklist.sty) +% modified and distributed with moderncv(not available otherwise on ctan) +\RequirePackage{tweaklist} +\renewcommand*{\itemhook}{% + \@minipagetrue% removes spacing before lists as they use \addvspace, which doesn't add vertical space inside minipages + \@noparlisttrue% removes spacing at end of lists, caused by \par + \setlength{\topsep}{0pt}% normally not required thanks to \@minipagetrue + \setlength{\partopsep}{0pt}% normally not required thanks to \@minipagetrue + \setlength{\parsep}{0pt}% not required when \itemsep and \parskip are set to 0pt (?) + \setlength{\parskip}{0pt}% + \setlength{\itemsep}{0pt}} +\renewcommand*{\enumhook}{\itemhook{}} +\renewcommand*{\deschook}{\itemhook{}} + +% lengths calculations +\RequirePackage{calc} + +% advanced command arguments (LaTeX 3) +\RequirePackage{xparse} +% TODO (?): replace all \newcommand by \NewDocumentCommand + +% micro-typography (e.g., character protrusion, font expansion, hyphenatable letterspacing) +\RequirePackage{microtype} + +% compatibility package with older versions of moderncv +\RequirePackageWithOptions{moderncvcompatibility} + + +%------------------------------------------------------------------------------- +% class definition +%------------------------------------------------------------------------------- +% minimal base settings +\setlength\lineskip{1\p@} +\setlength\normallineskip{1\p@} +\renewcommand\baselinestretch{} +\setlength{\parindent}{0\p@} +\setlength{\parskip}{0\p@} +\setlength\columnsep{10\p@} +\setlength\columnseprule{0\p@} +\setlength\fboxsep{3\p@} +\setlength\fboxrule{.4\p@} +\setlength\arrayrulewidth{.4\p@} +\setlength\doublerulesep{2\p@} + +% not set on purpose +%\setlength\arraycolsep{5\p@} +%\setlength\tabcolsep{6\p@} +%\setlength\tabbingsep{\labelsep} + +\raggedbottom +\onecolumn + + +%------------------------------------------------------------------------------- +% overall design commands definitions +%------------------------------------------------------------------------------- +% elements +% defines one's name +% usage: \name{<firstname>}{<lastname>} +\newcommand*{\name}[2]{\def\@firstname{#1}\def\@lastname{#2}} +% defines one's title (optional) +% usage: \title{<title>} +\renewcommand*{\title}[1]{\def\@title{#1}} +% defines one's address (optional) +% usage: \address{<street>}{<city>}{<country>} +% where the <city> and <country> arguments can be omitted or provided empty +\NewDocumentCommand{\address}{mG{}G{}}{\def\@addressstreet{#1}\def\@addresscity{#2}\def\@addresscountry{#3}} +% adds a mobile/fixed/fax number to one's personal information (optional) +% usage: \phone[<optional type>]{<number>} +% where <optional type> should be either "mobile", "fixed" or "fax +\RequirePackage{collection} +\collectionnew{phones} +\newcommand*{\phone}[2][fixed]{\collectionadd[#1]{phones}{#2}} +\newcommand*{\email}[1]{\def\@email{#1}} +% defines one's home page (optional) +% usage: \homepage{<url>} +\newcommand*{\homepage}[1]{\def\@homepage{#1}} +% defines one's github (optional) +% usage: \homepage{<url>} +\newcommand*{\github}[1]{\def\@github{#1}} +% defines additional personal information (optional) +% usage: \extrainfo{<text>} +\newcommand*{\extrainfo}[1]{\def\@extrainfo{#1}} + +% colors +\definecolor{color0}{rgb}{0,0,0}% main default color, normally left to black +\definecolor{color1}{rgb}{0,0,0}% primary theme color +\definecolor{color2}{rgb}{0,0,0}% secondary theme color +\definecolor{color3}{rgb}{0,0,0}% tertiary theme color + +% symbols +% itemize labels (the struts were added to correct inter-item spacing (works for single line items, until a solution is found for multi-line ones...) +\newcommand*{\labelitemi}{\strut\textcolor{color1}{\large\rmfamily\textbullet}}% the \rmfamily is required to force Latin Modern fonts when using sans serif, as OMS/lmss/m/n is not defined and gets substituted by OMS/cmsy/m/n +\newcommand*{\labelitemii}{\strut\textcolor{color1}{\large\bfseries-}} +\newcommand*{\labelitemiii}{\strut\textcolor{color1}{\rmfamily\textperiodcentered}}% alternative: \textasteriskcentered; the \rmfamily is required to force Latin Modern fonts when using sans serif, as OMS/lmss/m/n is not defined and gets substituted by OMS/cmsy/m/n +\newcommand*{\labelitemiv}{\labelitemiii} +% enumerate labels +\renewcommand{\theenumi}{\@arabic\c@enumi} +\renewcommand{\theenumii}{\@alph\c@enumii} +\renewcommand{\theenumiii}{\@roman\c@enumiii} +\renewcommand{\theenumiv}{\@Alph\c@enumiv} +% other symbols +\newcommand*{\listitemsymbol}{\labelitemi~} +\newcommand*{\addresssymbol}{} +\newcommand*{\mobilephonesymbol}{} +\newcommand*{\fixedphonesymbol}{} +\newcommand*{\faxphonesymbol}{} +\newcommand*{\emailsymbol}{} +\newcommand*{\homepagesymbol}{} + +% fonts +\AtBeginDocument{\normalfont\color{color0}} + +% strings for internationalisation +\newcommand*{\refname}{Publications} +\newcommand*{\enclname}{Enclosure} + +% makes the footer (normally used both for the resume and the letter) +% usage: \makefooter +\newcommand*{\makefooter}{}% + +% loads a style variant +% usage: \moderncvstyle{<style variant name>} +\newcommand*{\moderncvstyle}[1]{ + \RequirePackage{moderncvstyle#1}} + +% loads a color scheme +% usage: \moderncvcolor{<color scheme name>} +\newcommand*{\moderncvcolor}[1]{ + \RequirePackage{moderncvcolor#1}} + +% loads an icons set +% usage: \moderncvicons{<icon set name>} +\newcommand*{\moderncvicons}[1]{ + \RequirePackage{moderncvicons#1}} + +% recomputes all automatic lengths +\newcommand*{\recomputelengths}{\recomputecvlengths} +\AtBeginDocument{\recomputelengths{}} + +% creates a length if not yet defined +\newcommand*{\@initializelength}[1]{% + \ifdefined#1\else\newlength{#1}\fi} + + +%------------------------------------------------------------------------------- +% resume design commands definitions +%------------------------------------------------------------------------------- +% elements +% defines one's picture (optional) +% usage: photo[<picture width>][<picture frame thickness>]{<picture filename>} +\NewDocumentCommand{\photo}{O{64pt}O{0.4pt}m}{\def\@photowidth{#1}\def\@photoframewidth{#2}\def\@photo{#3}} +\newcommand*{\quote}[1]{\def\@quote{#1}} + +% fonts +\newcommand*{\namefont}{} +\newcommand*{\titlefont}{} +\newcommand*{\addressfont}{} +\newcommand*{\quotefont}{} +\newcommand*{\sectionfont}{} +\newcommand*{\subsectionfont}{} +\newcommand*{\hintfont}{} +\newcommand*{\pagenumberfont}{\addressfont\itshape} + +% styles +\newcommand*{\namestyle}[1]{{\namefont#1}} +\newcommand*{\titlestyle}[1]{{\titlefont#1}} +\newcommand*{\addressstyle}[1]{{\addressfont#1}} +\newcommand*{\quotestyle}[1]{{\quotefont#1}} +\newcommand*{\sectionstyle}[1]{{\sectionfont#1}} +\newcommand*{\subsectionstyle}[1]{{\subsectionfont#1}} +\newcommand*{\hintstyle}[1]{{\hintfont#1}} +\newcommand*{\pagenumberstyle}[1]{{\pagenumberfont#1}} + +% recompute all resume lengths +\newcommand*{\recomputecvlengths}{} + +% internal maketitle command to issue a new line only when required +\newif\if@firstdetailselement\@firstdetailselementtrue +\newcommand*{\makenewline}{ + \if@firstdetailselement% + \strut% to ensure baseline alignment, e.g. with when put in the margin vs sections that also contains a \strut + \else% + \\\fi% + \@firstdetailselementfalse} + +% makes the resume title +% usage: \makecvtitle +\newcommand*{\makecvtitle}{} + +% makes the resume footer +% usage: \makecvfooter +\newcommand*{\makecvfooter}{\makefooter} + +% makes a resume section +% usage: \section{<title>} +% identical starred and non-starred variants should be defined for compatibility with other packages (e.g. with natbib, that uses \section*{} for the bibliography header) +\NewDocumentCommand{\section}{sm}{} + +% makes a resume subsection +% usage: \subsection{title} +\NewDocumentCommand{\subsection}{sm}{} + +% makes a resume line with a header and a corresponding text +% usage: \cvitem[spacing]{header}{text} +\newcommand*{\cvitem}[3][.25em]{} + +% makes a resume line 2 headers and their corresponding text +% usage: \cvdoubleitem[spacing]{header1}{text1}{header2}{text2} +\newcommand*{\cvdoubleitem}[5][.25em]{} + +% makes a resume line with a list item +% usage: \cvlistitem[label]{item} +\newcommand*{\cvlistitem}[2][\listitemsymbol]{} + +% makes a resume line with 2 list items +% usage: \cvlistdoubleitem[label]{item1}{item2} +\newcommand*{\cvlistdoubleitem}[3][\listitemsymbol]{} + +% makes a typical resume job / education entry +% usage: \cventry[spacing]{years}{degree/job title}{institution/employer}{localization}{optionnal: grade/...}{optional: comment/job description} +\newcommand*{\cventry}[7][.25em]{} + +% makes a resume entry with a proficiency comment +% usage: \cvitemwithcomment[spacing]{header}{text}{comment} +\newcommand*{\cvitemwithcomment}[4][.25em]{} + +% makes a generic hyperlink +% usage: \link[optional text]{link} +\newcommand*{\link}[2][]{% + \ifthenelse{\equal{#1}{}}% + {\href{#2}{#2}}% + {\href{#2}{#1}}} + +% makes a http hyperlink +% usage: \httplink[optional text]{link} +\newcommand*{\httplink}[2][]{% + \ifthenelse{\equal{#1}{}}% + {\href{http://#2}{#2}}% + {\href{http://#2}{#1}}} + +% makes an email hyperlink +% usage: \emaillink[optional text]{link} +\newcommand*{\emaillink}[2][]{% + \ifthenelse{\equal{#1}{}}% + {\href{mailto:#2}{#2}}% + {\href{mailto:#2}{#1}}} + +% cvcolumns environment, where every column is created through \cvcolumn +% usage: \begin{cvcolumns} +% \cvcolumn[width]{head}{content} +% \cvcolumn[width]{head}{content} +% ... +% \end{cvcolumns} +% where "width" is the width as a fraction of the line length (between 0 and 1), "head" is the column header and "content" its content +\newcounter{cvcolumnscounter}% counter for the number of columns +\newcounter{cvcolumnsautowidthcounter}% counter for the number of columns with no column width provided, and which will then be equally distributed +\newcounter{tmpiteratorcounter}% counter for any temporary purpose (e.g., iterating loops) +\newlength{\cvcolumnsdummywidth}\setlength{\cvcolumnsdummywidth}{1000pt}% dummy width for total width, in order to enable arithmetics (TeX has no float variables, only integer counters or lengths) +\newlength{\cvcolumnswidth}% total width available for head / content +\newlength{\cvcolumnsautowidth}% total width of columns with no explicit width provided +\newlength{\cvcolumnautowidth}% width of one of the columns with no explicit width provided (based on equal distribution of remaining space) +\newif\if@cvcolumns@head@empty% whether or not at least one of the columns has a header +\newenvironment*{cvcolumns}% + {% at environment opening: reset counters, lengths and ifs + \setcounter{cvcolumnscounter}{0}% + \setcounter{cvcolumnsautowidthcounter}{0}% + \setlength{\cvcolumnsautowidth}{\cvcolumnsdummywidth}% + \setlength{\cvcolumnautowidth}{0pt}% + \@cvcolumns@head@emptytrue}% + {% at environment closing: typeset environment + % compute the width of each cvcolumn, considering a spacing of \separatorcolumnwidth and the columns with set width + \ifnum\thecvcolumnscounter>0% + \setlength{\cvcolumnswidth}{\maincolumnwidth-\value{cvcolumnscounter}\separatorcolumnwidth+\separatorcolumnwidth}% + \setlength{\cvcolumnautowidth}{\cvcolumnswidth*\ratio{\cvcolumnsautowidth}{\cvcolumnsdummywidth}/\value{cvcolumnsautowidthcounter}}\fi% + % pre-aggregate the tabular definition, heading and content (required before creating the tabular, as the tabular environment doesn't like loops --- probably because "&" generates a \endgroup) + % - the tabular definition is the aggregation of the different "\cvcolumn<i>@def" (by default "p{\cvcolumnautowidth}"), separated by "@{\hspace*{\separatorcolumnwidth}}" + % - the tabular heading is the aggregation of the different "\cvcolumn<i>@head", separated by "&" + % - the tabular content is the aggregation of the different "\cvcolumn<i>@content", separated by "&" + % to aggregate the different elements, \protected@edef or \g@addto@macro is required to avoid that \cvcolumns@def, -@head and -@content get expanded in subsequent redefinitions, which would cause errors due to the expansions of \hspace, of \subsectionstyle and possibly of user content/argument such as font commands + \def\cvcolumns@def{}% + \def\cvcolumns@head{}% + \def\cvcolumns@content{}% + \setcounter{tmpiteratorcounter}{0}% + % loop based on \g@addto@macro + \loop\ifnum\thetmpiteratorcounter<\thecvcolumnscounter% + \ifnum\thetmpiteratorcounter=0\else% + \g@addto@macro\cvcolumns@def{@{\hspace*{\separatorcolumnwidth}}}% + \g@addto@macro\cvcolumns@head{&}% + \g@addto@macro\cvcolumns@content{&}\fi% + \expandafter\g@addto@macro\expandafter\cvcolumns@def\expandafter{\csname cvcolumn\roman{tmpiteratorcounter}@def\endcsname}% + \expandafter\g@addto@macro\expandafter\cvcolumns@head\expandafter{\csname cvcolumn\roman{tmpiteratorcounter}@head\endcsname}% + \expandafter\g@addto@macro\expandafter\cvcolumns@content\expandafter{\csname cvcolumn\roman{tmpiteratorcounter}@content\endcsname}% + \stepcounter{tmpiteratorcounter}% + \repeat% +% % same loop based on \protected@edef +% \loop\ifnum\thetmpiteratorcounter<\thecvcolumnscounter% +% \ifnum\thetmpiteratorcounter=0\else% +% \protected@edef\cvcolumns@def{\cvcolumns@def @{\hspace*{\separatorcolumnwidth}}}% +% \protected@edef\cvcolumns@head{\cvcolumns@head &}% +% \protected@edef\cvcolumns@content{\cvcolumns@content &}\fi% +% \expandafter\protected@edef\expandafter\cvcolumns@def\expandafter{\expandafter\cvcolumns@def\expandafter\protect\csname cvcolumn\roman{tmpiteratorcounter}@def\endcsname}% +% \expandafter\protected@edef\expandafter\cvcolumns@head\expandafter{\expandafter\cvcolumns@head\expandafter\protect\csname cvcolumn\roman{tmpiteratorcounter}@head\endcsname}% +% \expandafter\protected@edef\expandafter\cvcolumns@content\expandafter{\expandafter\cvcolumns@content\expandafter\protect\csname cvcolumn\roman{tmpiteratorcounter}@content\endcsname}% +% \stepcounter{tmpiteratorcounter}% +% \repeat% + % create the tabular + \cvitem{}{% + \begin{tabular}{\cvcolumns@def}% + \if@cvcolumns@head@empty\else% + \cvcolumns@head\\[-.8em]% + {\color{color1}\rule{\maincolumnwidth}{.25pt}}\\\fi% + \cvcolumns@content% + \end{tabular}}} + +% cvcolumn command, to create a column inside a cvcolumns environment +% usage: \cvcolumn[width]{head}{content} +% where "width" is the width as a fraction of the line length (between 0 and 1), "head" is the column header and "content" its content ("head" and "content" can contain "\\", "\newline" or any other paragraph command such as "itemize") +\newcommand*{\cvcolumn}[3][\cvcolumnautowidth]{% +% \def\cvcolumn@width{}% + \ifthenelse{\equal{#1}{\cvcolumnautowidth}}% + {% if no width fraction is provided, count this column as auto-adjusted and set its width to \cvcolumnsautowidth + \stepcounter{cvcolumnsautowidthcounter}% + \expandafter\expandafter\expandafter\def\expandafter\csname cvcolumn\roman{cvcolumnscounter}@def\endcsname{p{\cvcolumnautowidth}}% + \expandafter\expandafter\expandafter\def\expandafter\csname cvcolumn\roman{cvcolumnscounter}@head\endcsname{\protect\parbox[b]{\cvcolumnautowidth}{\protect\subsectionstyle{#2}}}}% + {% if a width is provided, set the width of the column to it and decrease the available space for auto-adjusted columns + \addtolength{\cvcolumnsautowidth}{-#1\cvcolumnsdummywidth}% + \expandafter\expandafter\expandafter\def\expandafter\csname cvcolumn\roman{cvcolumnscounter}@def\endcsname{p{#1\cvcolumnswidth}}% + \expandafter\expandafter\expandafter\def\expandafter\csname cvcolumn\roman{cvcolumnscounter}@head\endcsname{\protect\parbox[b]{#1\cvcolumnswidth}{\protect\subsectionstyle{#2}}}}% + \ifthenelse{\equal{#2}{}}{}{\@cvcolumns@head@emptyfalse}% + \expandafter\expandafter\expandafter\def\expandafter\csname cvcolumn\roman{cvcolumnscounter}@content\endcsname{\protect\cvcolumncell{#3}}% + \stepcounter{cvcolumnscounter}} + +% internal cvcolumncell command, that enables a cvcolumn cell to contain paragraph commands (lists, newlines, etc) +\newcommand*{\cvcolumncell}[1]{{% put cell inside a group, so that command redefinitions are only local + % roughly restore \\ to its regular definition (outside of tabular) + \renewcommand*{\\}{\newline}% + % enclose the contents of the cell inside a vertical box, to allow paragraph commands + \protect\vtop{#1}}} + +% thebibliography environment, for use with BibTeX and possibly multibib +\newlength{\bibindent} +\setlength{\bibindent}{1.5em} +% bibliography item label +\newcommand*{\bibliographyitemlabel}{}% use \@biblabel{\arabic{enumiv}} for BibTeX labels +%\newif\if@multibibfirstbib\@multibibfirstbibfalse +% bibliography head (section, etc}, depending on whether multibib is used +\newcommand*{\bibliographyhead}[1]{\section{#1}} +\AtEndPreamble{\@ifpackageloaded{multibib}{\renewcommand*{\bibliographyhead}[1]{\subsection{#1}}}{}} +% thebibliography environment definition +\newenvironment{thebibliography}[1]{}{} +\newcommand*{\newblock}{\hskip .11em\@plus.33em\@minus.07em} +\let\@openbib@code\@empty + +% itemize, enumerate and description environment +\setlength{\leftmargini} {1em} +\leftmargin\leftmargini +\setlength{\leftmarginii} {\leftmargini} +\setlength{\leftmarginiii} {\leftmargini} +\setlength{\leftmarginiv} {\leftmargini} +\setlength{\leftmarginv} {\leftmargini} +\setlength{\leftmarginvi} {\leftmargini} +\setlength{\labelsep} {.5em}% this is the distance between the label and the body, but it pushes the label to the left rather than pushing the body to the right (to do the latter, modify \leftmargin(i) +\setlength{\labelwidth} {\leftmargini}% unfortunately, \labelwidth is not defined by item level (i.e. no \labeliwidth, \labeliiwidth, etc) +\addtolength{\labelwidth} {-\labelsep} +\@beginparpenalty -\@lowpenalty +\@endparpenalty -\@lowpenalty +\@itempenalty -\@lowpenalty +\newcommand\labelenumi{\theenumi.} +\newcommand\labelenumii{(\theenumii)} +\newcommand\labelenumiii{\theenumiii.} +\newcommand\labelenumiv{\theenumiv.} +\renewcommand\p@enumii{\theenumi} +\renewcommand\p@enumiii{\p@enumii(\theenumii)} +\renewcommand\p@enumiv{\p@enumiii\theenumiii} +% description label +\newcommand*\descriptionlabel[1]{\hspace\labelsep\normalfont\bfseries#1} + +% classical \today definition +\def\today{\ifcase\month\or + January\or February\or March\or April\or May\or June\or + July\or August\or September\or October\or November\or December\fi + \space\number\day, \number\year} + +%\newcommand{\widthofautobox}[1]{% +% \widthof{\begin{tabular}{@{}l@{}}#1\end{tabular}}} + +%\newcommand{\autobox}[2][b]{% +% \parbox[#1]{\widthofautobox{#2}}{#2}} + + +%------------------------------------------------------------------------------- +% letter design commands definitions +%------------------------------------------------------------------------------- +% elements +\newcommand*{\recipient}[2]{\def\@recipientname{#1}\def\@recipientaddress{#2}} +\renewcommand*{\date}[1]{\def\@date{#1}}\date{\today} +\newcommand*{\opening}[1]{\def\@opening{#1}} +\newcommand*{\closing}[1]{\def\@closing{#1}} +\newcommand*{\enclosure}[2][]{% + % if an optional argument is provided, use it to redefine \enclname + \ifthenelse{\equal{#1}{}}{}{\renewcommand*{\enclname}{#1}}% + \def\@enclosure{#2}} + +% recompute all letter lengths +\newcommand*{\recomputeletterlengths}{} + +% makes the letter title +% usage: \makelettertitle +\newcommand*{\makelettertitle}{} + +% makes the letter footer +% usage: \makeletterfooter +\newcommand*{\makeletterfooter}{\makefooter} + +% makes the letter closing +% usage: \makeletterclosing +\newcommand*{\makeletterclosing}{} + + +\endinput + + +%% end of file `moderncv.cls'. diff --git a/users/grfn/resume/moderncvcolorblack.sty b/users/grfn/resume/moderncvcolorblack.sty new file mode 100644 index 000000000000..3a6e1477f322 --- /dev/null +++ b/users/grfn/resume/moderncvcolorblack.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolorblack.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolorblack}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: black] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0,0,0}% black +\definecolor{color2}{rgb}{0,0,0}% black + + +\endinput + + +%% end of file `moderncvcolorblack.sty'. diff --git a/users/grfn/resume/moderncvcolorblue.sty b/users/grfn/resume/moderncvcolorblue.sty new file mode 100644 index 000000000000..7b949c704acd --- /dev/null +++ b/users/grfn/resume/moderncvcolorblue.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolorblue.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolorblue}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: blue] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0.22,0.45,0.70}% light blue +\definecolor{color2}{rgb}{0.45,0.45,0.45}% dark grey + + +\endinput + + +%% end of file `moderncvcolorblue.sty'. diff --git a/users/grfn/resume/moderncvcolorgreen.sty b/users/grfn/resume/moderncvcolorgreen.sty new file mode 100644 index 000000000000..4de7f848a04e --- /dev/null +++ b/users/grfn/resume/moderncvcolorgreen.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolorgreen.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolorgreen}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: green] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0.35,0.70,0.30}% green +\definecolor{color2}{rgb}{0.45,0.45,0.45}% dark grey + + +\endinput + + +%% end of file `moderncvcolorgreen.sty'. diff --git a/users/grfn/resume/moderncvcolorgrey.sty b/users/grfn/resume/moderncvcolorgrey.sty new file mode 100644 index 000000000000..9018726a2384 --- /dev/null +++ b/users/grfn/resume/moderncvcolorgrey.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolorgrey.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolorgrey}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: grey] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0.55,0.55,0.55}% dark grey +\definecolor{color2}{rgb}{0.55,0.55,0.55}% dark grey + + +\endinput + + +%% end of file `moderncvcolorgrey.sty'. diff --git a/users/grfn/resume/moderncvcolororange.sty b/users/grfn/resume/moderncvcolororange.sty new file mode 100644 index 000000000000..134ae2401133 --- /dev/null +++ b/users/grfn/resume/moderncvcolororange.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolororange.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolororange}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: orange] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0.95,0.55,0.15}% orange +\definecolor{color2}{rgb}{0.45,0.45,0.45}% dark grey + + +\endinput + + +%% end of file `moderncvcolororange.sty'. diff --git a/users/grfn/resume/moderncvcolorpurple.sty b/users/grfn/resume/moderncvcolorpurple.sty new file mode 100644 index 000000000000..d3dc5345b080 --- /dev/null +++ b/users/grfn/resume/moderncvcolorpurple.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolorpurple.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolorpurple}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: purple] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0.50,0.33,0.80}% purple +\definecolor{color2}{rgb}{0.45,0.45,0.45}% dark grey + + +\endinput + + +%% end of file `moderncvcolorpurple.sty'. diff --git a/users/grfn/resume/moderncvcolorred.sty b/users/grfn/resume/moderncvcolorred.sty new file mode 100644 index 000000000000..681181997d38 --- /dev/null +++ b/users/grfn/resume/moderncvcolorred.sty @@ -0,0 +1,27 @@ +%% start of file `moderncvcolorred.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcolorred}[2013/02/09 v1.3.0 modern curriculum vitae and letter color scheme: red] + + +%------------------------------------------------------------------------------- +% color scheme definition +%------------------------------------------------------------------------------- +\definecolor{color0}{rgb}{0,0,0}% black +\definecolor{color1}{rgb}{0.95,0.20,0.20}% red +\definecolor{color2}{rgb}{0.45,0.45,0.45}% dark grey + + +\endinput + + +%% end of file `moderncvcolorred.sty'. diff --git a/users/grfn/resume/moderncvcompatibility.sty b/users/grfn/resume/moderncvcompatibility.sty new file mode 100644 index 000000000000..1fc53f2180e1 --- /dev/null +++ b/users/grfn/resume/moderncvcompatibility.sty @@ -0,0 +1,104 @@ +%% start of file `moderncvcompatibility.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvcompatibility}[2013/02/09 v1.3.0 modern curriculum vitae and letter compatibility patches] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- + + +%------------------------------------------------------------------------------- +% package options +%------------------------------------------------------------------------------- +% old casual option (version 0.1) +%\DeclareOption{casual}{\input{moderncvstylecasual.sty}} + +% old classic option (version 0.1) +%\DeclareOption{classic}{\input{moderncvstyleclassic.sty}} + +\DeclareOption*{} + +% process given options +\ProcessOptions\relax + +%------------------------------------------------------------------------------- +% definitions +%------------------------------------------------------------------------------- +% compatibility with version 0.1 +\newcommand*{\cvresume}[2]{\cvlistdoubleitem{#1}{#2}} + +% compatibility with versions <= 0.2 +% section, cvline, ... with width argument... +%\newcommand*{\section}[2][0.825]{% +% \closesection{}% +% \@sectionopentrue% +% \addcontentsline{toc}{part}{#2} +% \begin{longtable}[t]{@{}r@{\hspace{.025\textwidth}}@{}p{#1\textwidth}@{}}% +%% \colorrule{.15\textwidth}&\mbox{\color{sectiontitlecolor}\sectionfont#2}\\[1ex]}% +% {\color{sectionrectanglecolor}\rule{0.15\textwidth}{1ex}}&\mbox{\color{sectiontitlecolor}\sectionfont#2}\\[1ex]}% +%\newcommand*{\cvline}[3][.825]{% +% \begin{minipage}[t]{\hintscolumnwidth}\raggedleft\small\sffamily#2\end{minipage}&\begin{minipage}[t]{\maincolumnwidth}#3\end{minipage}\\} +%\newcommand*{\cvitem}[3][.825]{% +% \cvline[#1]{#2}{#3\vspace*{.75em}}} % the \vspace*{} inside the cvline environment is a hack... (should conceptually be outside the environment) + +% compatibility with versions <= 0.5 +%\newcommand*{\cvitem}[2]{\cvline{#1}{#2}} +%\newcommand*{\moderncvstyle}[1]{\moderncvtheme{#1}} + +% compatibility with versions <= 0.7 +\newcommand*{\closesection}{} +\newcommand*{\emptysection}{} +\newcommand*{\sethintscolumnlength}[1]{% + \setlength{\hintscolumnwidth}{#1}% + \recomputelengths} +\newcommand*{\sethintscolumntowidth}[1]{% + \settowidth{\hintscolumnwidth}{#1}% + \recomputelengths} + +% compatibility with versions <= 0.15 +\newcommand*{\cvline}[2]{\cvitem{#1}{#2}} +\newcommand*{\cvlanguage}[3]{\cvitemwithcomment{#1}{#2}{#3}} +\newcommand*{\cvcomputer}[4]{\cvdoubleitem{#1}{\small#2}{#3}{\small#4}} +\newcommand*{\moderncvtheme}[2][blue]{% + \moderncvcolor{#1}% + \moderncvstyle{#2}} + +% compatibility with versions <= 0.19 +\newcommand*{\maketitle}{\makecvtitle}% +\title{}% to avoid LaTeX complaining that \maketitle is a called without first a call to \title +\newcommand*{\maketitlenamewidth}{\makecvtitlenamewidth} + +% compatibility with versions <= 1.3.0 +\newcommand*{\firstname}[1]{\def\@firstname{#1}} +\newcommand*{\lastname}[1]{\def\@lastname{#1}} +\newcommand*{\givenname}[1]{\def\@firstname{#1}} +\newcommand*{\familyname}[1]{\def\@lastname{#1}} +\def\@familyname{\@lastname} + +% compatibility with versions <= 1.4.0 +\newcommand*{\mobile}[1]{\collectionadd[mobile]{phones}{#1}} +%\newcommand*{\phone}[1]{\collectionadd[fixed]{phones}{#1}}% implicit, as \phone{...} defaults to \phone[fixed]{...} +\newcommand*{\fax}[1]{\collectionadd[fax]{phones}{#1}} +\newcommand*{\@mobile}{\collectionfindbykey{phones}{mobile}} +\newcommand*{\@phone}{\collectionfindbykey{phones}{fixed}} +\newcommand*{\@fax}{\collectionfindbykey{phones}{fax}} +\newcommand*{\phonesymbol}{\fixedphonesymbol} +\newcommand*{\mobilesymbol}{\mobilephonesymbol} +\newcommand*{\faxsymbol}{\faxphonesymbol} + + +\endinput + + +%% end of file `moderncvcompatibility.sty'. diff --git a/users/grfn/resume/moderncviconsletters.sty b/users/grfn/resume/moderncviconsletters.sty new file mode 100644 index 000000000000..0a4e2864be29 --- /dev/null +++ b/users/grfn/resume/moderncviconsletters.sty @@ -0,0 +1,50 @@ +%% start of file `moderncviconsletters.sty'. +%% Copyright 2013-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncviconsmarvosym}[2013/02/09 v1.3.0 modern curriculum vitae and letter icons: letters] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- +% MarVoSym font +%\RequirePackage{marvosym} +\newcommand*{\marvosymbol}[1]{} +%\ifxetexorluatex +% \renewcommand*{\marvosymbol}[1]{{\fontspec{MarVoSym}\char#1}} +%\else + \renewcommand*{\marvosymbol}[1]{{\fontfamily{mvs}\fontencoding{U}\fontseries{m}\fontshape{n}\selectfont\char#1}} +%\fi + + +%------------------------------------------------------------------------------- +% symbols definition +%------------------------------------------------------------------------------- +\renewcommand*{\labelitemi}{\strut\textcolor{color1}{\marvosymbol{123}}}% equivalent to \Neutral from marvosym package; alternative: \fontencoding{U}\fontfamily{ding}\selectfont\tiny\symbol{'102} +%\renewcommand*{\labelitemii}{\strut\textcolor{color1}{\large\bfseries-}}% no change from default in moderncv.cls +%\renewcommand*{\labelitemiii}{\strut\textcolor{color1}{\rmfamily\textperiodcentered}}% no change from default in moderncv.cls +%\renewcommand*{\labelitemiv}{\labelitemiii}% no change from default in moderncv.cls + +\renewcommand*{\addresssymbol}{} +\renewcommand*{\mobilephonesymbol}{\textbf{M}~} +\renewcommand*{\fixedphonesymbol}{\textbf{T}~} +\renewcommand*{\faxphonesymbol}{\textbf{F}~} +\renewcommand*{\emailsymbol}{\textbf{E}~} +\renewcommand*{\homepagesymbol}{} + +\renewcommand*{\listitemsymbol}{\labelitemi~} + + +\endinput + + +%% end of file `moderncviconsletters.sty'. diff --git a/users/grfn/resume/moderncviconsmarvosym.sty b/users/grfn/resume/moderncviconsmarvosym.sty new file mode 100644 index 000000000000..eb1b1ec727bb --- /dev/null +++ b/users/grfn/resume/moderncviconsmarvosym.sty @@ -0,0 +1,48 @@ +%% start of file `moderncviconsmarvosym.sty'. +%% Copyright 2013-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncviconsmarvosym}[2013/02/09 v1.3.0 modern curriculum vitae and letter icons: marvosym] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- +% MarVoSym font +%\RequirePackage{marvosym} +\newcommand*{\marvosymbol}[1]{} +%\ifxetexorluatex +% \renewcommand*{\marvosymbol}[1]{{\fontspec{MarVoSym}\char#1}} +%\else + \renewcommand*{\marvosymbol}[1]{{\fontfamily{mvs}\fontencoding{U}\fontseries{m}\fontshape{n}\selectfont\char#1}} +%\fi + + +%------------------------------------------------------------------------------- +% symbols definition +%------------------------------------------------------------------------------- +\renewcommand*{\labelitemi}{\strut\textcolor{color1}{\marvosymbol{123}}}% equivalent to \Neutral from marvosym package; alternative: \fontencoding{U}\fontfamily{ding}\selectfont\tiny\symbol{'102} +%\renewcommand*{\labelitemii}{\strut\textcolor{color1}{\large\bfseries-}}% no change from default in moderncv.cls +%\renewcommand*{\labelitemiii}{\strut\textcolor{color1}{\rmfamily\textperiodcentered}}% no change from default in moderncv.cls +%\renewcommand*{\labelitemiv}{\labelitemiii}% no change from default in moderncv.cls + +\renewcommand*{\addresssymbol}{} +\renewcommand*{\mobilephonesymbol}{\marvosymbol{72}~} +\renewcommand*{\fixedphonesymbol}{\marvosymbol{84}~} +\renewcommand*{\faxphonesymbol}{\marvosymbol{117}~} +\renewcommand*{\emailsymbol}{\marvosymbol{66}~} +\renewcommand*{\homepagesymbol}{{\Large\marvosymbol{205}}~} + + +\endinput + + +%% end of file `moderncviconsmarvosym.sty'. diff --git a/users/grfn/resume/moderncvstylebanking.sty b/users/grfn/resume/moderncvstylebanking.sty new file mode 100644 index 000000000000..fb0b70fdcd10 --- /dev/null +++ b/users/grfn/resume/moderncvstylebanking.sty @@ -0,0 +1,287 @@ +%% start of file `moderncvstylebanking.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvstylebanking}[2013/02/09 v1.3.0 modern curriculum vitae and letter style scheme: banking] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- + + +%------------------------------------------------------------------------------- +% overall style definition +%------------------------------------------------------------------------------- +% fonts +%\ifxetexorluatex +% \setmainfont{Tex-Gyre Pagella} +% \setsansfont{Tex-Gyre Pagella} +% \setmathfont{Tex-Gyre Pagella} +% \setmathfont[range=\mathit,\mathsfit]{Tex-Gyre Pagella Italic} +% \setmathfont[range=\mathbfup,\mathbfsfup]{Tex-Gyre Pagella Bold} +% \setmathfont[range=\mathbfit,\mathbfsfit]{Tex-Gyre Pagella Bold Italic} +%\else + \IfFileExists{tgpagella.sty}% + {% + \RequirePackage{tgpagella}% + \renewcommand*{\familydefault}{\rmdefault}}% + {} +%\fi + +% symbols +\moderncvicons{marvosym} + +% commands +\newcommand*{\maketitlesymbol}{% + {~~~{\rmfamily\textbullet}~~~}}% the \rmfamily is required to force Latin Modern fonts when using sans serif, as OMS/lmss/m/n is not defined and gets substituted by OMS/cmsy/m/n +% internal command to add an element to the footer +% it collects the elements in a temporary box, and checks when to flush the box +\newsavebox{\maketitlebox}% +\newsavebox{\maketitletempbox}% +\newlength{\maketitlewidth}% +\newlength{\maketitleboxwidth}% +\newif\if@firstmaketitleelement\@firstmaketitleelementtrue% +% adds an element to the maketitle, separated by maketitlesymbol +% usage: \addtomaketitle[maketitlesymbol]{element} +\newcommand*{\addtomaketitle}[2][\maketitlesymbol]{% + \if@firstmaketitleelement% + \savebox{\maketitletempbox}{\usebox{\maketitlebox}#2}% + \else% + \savebox{\maketitletempbox}{\usebox{\maketitlebox}#1#2}\fi% + \settowidth{\maketitleboxwidth}{\usebox{\maketitletempbox}}% + \ifnum\maketitleboxwidth<\maketitlewidth% + \savebox{\maketitlebox}{\usebox{\maketitletempbox}}% + \@firstmaketitleelementfalse% + \else% + \flushmaketitle{}\\% + \savebox{\maketitlebox}{#2}% + \savebox{\maketitletempbox}{#2}% + \settowidth{\maketitleboxwidth}{\usebox{\maketitlebox}}% + \@firstmaketitleelementfalse\fi} +% internal command to flush the maketitle +\newcommand*{\flushmaketitle}{% + \strut\usebox{\maketitlebox}% + \savebox{\maketitlebox}{}% + \savebox{\maketitletempbox}{}% + \setlength{\maketitleboxwidth}{0pt}} +\renewcommand*{\maketitle}{% + \setlength{\maketitlewidth}{0.8\textwidth}% + \hfil% + \parbox{\maketitlewidth}{% + \centering% + % name and title + \namestyle{\@firstname~\@lastname}% + \ifthenelse{\equal{\@title}{}}{}{\titlestyle{~|~\@title}}\\% \isundefined doesn't work on \@title, as LaTeX itself defines \@title (before it possibly gets redefined by \title) + % detailed information + \addressfont\color{color2}% + \ifthenelse{\isundefined{\@addressstreet}}{}{\addtomaketitle{\addresssymbol\@addressstreet}% + \ifthenelse{\equal{\@addresscity}{}}{}{\addtomaketitle[~--~]{\@addresscity}}% if \addresstreet is defined, \addresscity and \addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\addtomaketitle[~--~]{\@addresscountry}}% + \flushmaketitle\@firstmaketitleelementtrue\\}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \addtomaketitle{\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}}% + \ifthenelse{\isundefined{\@email}}{}{\addtomaketitle{\emailsymbol\emaillink{\@email}}}% + \ifthenelse{\isundefined{\@homepage}}{}{\addtomaketitle{\homepagesymbol\httplink{\@homepage}}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\addtomaketitle{\@extrainfo}}% + \flushmaketitle}\\[2.5em]}% need to force a \par after this to avoid weird spacing bug at the first section if no blank line is left after \maketitle + + +%------------------------------------------------------------------------------- +% resume style definition +%------------------------------------------------------------------------------- +% fonts +\renewcommand*{\namefont}{\Huge\bfseries\upshape} +\renewcommand*{\titlefont}{\Huge\mdseries\upshape} +\renewcommand*{\addressfont}{\normalsize\mdseries\upshape} +\renewcommand*{\quotefont}{\large\slshape} +\renewcommand*{\sectionfont}{\Large\bfseries\upshape} +\renewcommand*{\subsectionfont}{\large\upshape\fontseries{sb}\selectfont} +\renewcommand*{\hintfont}{\bfseries} + +% styles +\renewcommand*{\namestyle}[1]{{\namefont\textcolor{color1}{#1}}} +\renewcommand*{\titlestyle}[1]{{\titlefont\textcolor{color2!85}{#1}}} +\renewcommand*{\addressstyle}[1]{{\addressfont\textcolor{color1}{#1}}} +\renewcommand*{\quotestyle}[1]{{\quotefont\textcolor{color1}{#1}}} +\renewcommand*{\sectionstyle}[1]{{\sectionfont\textcolor{color1}{#1}}} +\renewcommand*{\subsectionstyle}[1]{{\subsectionfont\textcolor{color1}{#1}}} +\renewcommand*{\hintstyle}[1]{{\hintfont\textcolor{color0}{#1}}} + +% lengths +\newlength{\quotewidth} +\newlength{\hintscolumnwidth} +\setlength{\hintscolumnwidth}{0.3\textwidth}% +\newlength{\separatorcolumnwidth} +\setlength{\separatorcolumnwidth}{0.025\textwidth}% +\newlength{\maincolumnwidth} +\newlength{\doubleitemcolumnwidth} +\newlength{\listitemsymbolwidth} +\settowidth{\listitemsymbolwidth}{\listitemsymbol} +\newlength{\listitemmaincolumnwidth} +\newlength{\listdoubleitemmaincolumnwidth} + +% commands +\renewcommand*{\recomputecvlengths}{% + \setlength{\quotewidth}{0.65\textwidth}% + % main lenghts + \setlength{\maincolumnwidth}{\textwidth}% + % listitem lengths + \setlength{\listitemmaincolumnwidth}{\maincolumnwidth-\listitemsymbolwidth}% + % doubleitem lengths + \setlength{\doubleitemcolumnwidth}{\maincolumnwidth-\separatorcolumnwidth}% + \setlength{\doubleitemcolumnwidth}{0.5\doubleitemcolumnwidth}% + % listdoubleitem lengths + \setlength{\listdoubleitemmaincolumnwidth}{\maincolumnwidth-\listitemsymbolwidth-\separatorcolumnwidth-\listitemsymbolwidth}% + \setlength{\listdoubleitemmaincolumnwidth}{0.5\listdoubleitemmaincolumnwidth}% + % fancyhdr lengths + \renewcommand{\headwidth}{\textwidth}% + % regular lengths + \setlength{\parskip}{0\p@}} + +\renewcommand*{\makecvtitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputecvlengths% + \maketitle% + % optional quote + \ifthenelse{\isundefined{\@quote}}% + {}% + {{\centering\begin{minipage}{\quotewidth}\centering\quotestyle{\@quote}\end{minipage}\\[2.5em]}}% + \par}% to avoid weird spacing bug at the first section if no blank line is left after \maketitle} + +\RenewDocumentCommand{\section}{sm}{% + \par\addvspace{2.5ex}% + \phantomsection{}% reset the anchor for hyperrefs + \addcontentsline{toc}{section}{#2}% + \strut\sectionstyle{#2}% + {\color{color1}\hrule}% + \par\nobreak\addvspace{1ex}\@afterheading} + +\newcommand{\subsectionfill}{\xleaders\hbox to 0.35em{\scriptsize.}\hfill}% different subsectionfills will not be perfectly aligned, but remaining space at the end of the fill will be distributed evenly between leaders, so it will be barely visible +\RenewDocumentCommand{\subsection}{sm}{% + \par\addvspace{1ex}% + \phantomsection{}% + \addcontentsline{toc}{subsection}{#2}% + \strut\subsectionstyle{#2}{\color{color1}{\subsectionfill}}% + \par\nobreak\addvspace{0.5ex}\@afterheading} + +\renewcommand*{\cvitem}[3][.25em]{% + \ifthenelse{\equal{#2}{}}{}{\hintstyle{#2}: }{#3}% + \par\addvspace{#1}} + +\renewcommand*{\cvdoubleitem}[5][.25em]{% + \begin{minipage}[t]{\doubleitemcolumnwidth}\hintstyle{#2}: #3\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \begin{minipage}[t]{\doubleitemcolumnwidth}\ifthenelse{\equal{#4}{}}{}{\hintstyle{#4}: }#5\end{minipage}% + \par\addvspace{#1}} + +\renewcommand*{\cvlistitem}[2][.25em]{% + \listitemsymbol\begin{minipage}[t]{\listitemmaincolumnwidth}#2\end{minipage}% + \par\addvspace{#1}} + +\renewcommand*{\cvlistdoubleitem}[3][.25em]{% + \cvitem[#1]{}{\listitemsymbol\begin{minipage}[t]{\listdoubleitemmaincolumnwidth}#2\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \ifthenelse{\equal{#3}{}}% + {}% + {\listitemsymbol\begin{minipage}[t]{\listdoubleitemmaincolumnwidth}#3\end{minipage}}}} + +\renewcommand*{\cventry}[7][.25em]{ + \begin{tabular*}{\textwidth}{l@{\extracolsep{\fill}}r}% + {\bfseries #4} & {\bfseries #5} \\% + {\itshape #3\ifthenelse{\equal{#6}{}}{}{, #6}} & {\itshape #2}\\% + \end{tabular*}% + \ifx&% + \else{\\\vbox{\small#7}}\fi% + \par\addvspace{#1}} + +\newbox{\cvitemwithcommentmainbox} +\newlength{\cvitemwithcommentmainlength} +\newlength{\cvitemwithcommentcommentlength} +\renewcommand*{\cvitemwithcomment}[4][.25em]{% + \savebox{\cvitemwithcommentmainbox}{\ifthenelse{\equal{#2}{}}{}{\hintstyle{#2}: }#3}% + \setlength{\cvitemwithcommentmainlength}{\widthof{\usebox{\cvitemwithcommentmainbox}}}% + \setlength{\cvitemwithcommentcommentlength}{\maincolumnwidth-\separatorcolumnwidth-\cvitemwithcommentmainlength}% + \begin{minipage}[t]{\cvitemwithcommentmainlength}\ifthenelse{\equal{#2}{}}{}{\hintstyle{#2}: }#3\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \begin{minipage}[t]{\cvitemwithcommentcommentlength}\raggedleft\small\itshape#4\end{minipage}% + \par\addvspace{#1}} + +\renewenvironment{thebibliography}[1]% + {% + \bibliographyhead{\refname}% +% \small% + \begin{list}{\bibliographyitemlabel}% + {% + \setlength{\topsep}{0pt}% + \setlength{\labelwidth}{0pt}% + \setlength{\labelsep}{0pt}% + \leftmargin\labelwidth% + \advance\leftmargin\labelsep% + \@openbib@code% + \usecounter{enumiv}% + \let\p@enumiv\@empty% + \renewcommand\theenumiv{\@arabic\c@enumiv}}% + \sloppy\clubpenalty4000\widowpenalty4000% +% \sfcode`\.\@m% +% \sfcode `\=1000\relax% + }% + {% + \def\@noitemerr{\@latex@warning{Empty `thebibliography' environment}}% + \end{list}% + } + + +%------------------------------------------------------------------------------- +% letter style definition +%------------------------------------------------------------------------------- +% commands +\renewcommand*{\recomputeletterlengths}{ + \recomputecvlengths% + \setlength{\parskip}{6\p@}} + +\renewcommand*{\makelettertitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputeletterlengths% + % sender block + \maketitle% + \par% + % recipient block + \begin{minipage}[t]{.5\textwidth} + \raggedright% + \addressfont% + {\bfseries\upshape\@recipientname}\\% + \@recipientaddress% + \end{minipage} + % date + \hfill % US style +% \\[1em] % UK style + \@date\\[2em]% US informal style: "April 6, 2006"; UK formal style: "05/04/2006" + % opening + \raggedright% + \@opening\\[1.5em]% + % ensure no extra spacing after \makelettertitle due to a possible blank line +% \ignorespacesafterend% not working + \hspace{0pt}\par\vspace{-\baselineskip}\vspace{-\parskip}} + +\renewcommand*{\makeletterclosing}{ + \@closing\\[3em]% + {\bfseries \@firstname~\@lastname}% + \ifthenelse{\isundefined{\@enclosure}}{}{% + \\% + \vfill% + {\color{color2}\itshape\enclname: \@enclosure}}} + + +\endinput + + +%% end of file `moderncvstylebanking.sty'. diff --git a/users/grfn/resume/moderncvstylecasual.sty b/users/grfn/resume/moderncvstylecasual.sty new file mode 100644 index 000000000000..e375e7612a5a --- /dev/null +++ b/users/grfn/resume/moderncvstylecasual.sty @@ -0,0 +1,182 @@ +%% start of file `moderncvstylecasual.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvstylecasual}[2013/02/09 v1.3.0 modern curriculum vitae and letter style scheme: casual] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- +\RequirePackage{moderncvstyleclassic} + + +%------------------------------------------------------------------------------- +% overall style definition +%------------------------------------------------------------------------------- +% commands +% footer symbol used to separate footer elements +\newcommand*{\footersymbol}{% + {~~~{\rmfamily\textbullet}~~~}}% the \rmfamily is required to force Latin Modern fonts when using sans serif, as OMS/lmss/m/n is not defined and gets substituted by OMS/cmsy/m/n +% internal command to add an element to the footer +% it collects the elements in a temporary box, and checks when to flush the box +\newsavebox{\footerbox}% +\newsavebox{\footertempbox}% +\newlength{\footerwidth}% +\newlength{\footerboxwidth}% +\newif\if@firstfooterelement\@firstfooterelementtrue% +% adds an element to the footer, separated by footersymbol +% usage: \addtofooter[footersymbol]{element} +\newcommand*{\addtofooter}[2][\footersymbol]{% + \if@firstfooterelement% + \savebox{\footertempbox}{\usebox{\footerbox}#2}% + \else% + \savebox{\footertempbox}{\usebox{\footerbox}#1#2}\fi% + \settowidth{\footerboxwidth}{\usebox{\footertempbox}}% + \ifnum\footerboxwidth<\footerwidth% + \savebox{\footerbox}{\usebox{\footertempbox}}% + \@firstfooterelementfalse% + \else% + \flushfooter\\% + \savebox{\footerbox}{#2}% + \savebox{\footertempbox}{#2}% + \settowidth{\footerboxwidth}{\usebox{\footerbox}}% + \@firstfooterelementfalse\fi} +% internal command to flush the footer +\newcommand*{\flushfooter}{% + \strut\usebox{\footerbox}% + \savebox{\footerbox}{}% + \savebox{\footertempbox}{}% + \setlength{\footerboxwidth}{0pt}} + + +%------------------------------------------------------------------------------- +% resume style definition +%------------------------------------------------------------------------------- +% fonts +\renewcommand*{\namefont}{\fontsize{38}{40}\mdseries\upshape} +\renewcommand*{\addressfont}{\normalsize\mdseries\slshape} + +% commands +\renewcommand*{\makecvtitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputecvlengths% + % ensure footer with personal information + \makecvfooter% + % optional picture + \newbox{\makecvtitlepicturebox}% + \savebox{\makecvtitlepicturebox}{% + \ifthenelse{\isundefined{\@photo}}% + {}% + {% + \setlength\fboxrule{\@photoframewidth}% + \ifdim\@photoframewidth=0pt% + \setlength{\fboxsep}{0pt}\fi% + {\color{color1}\framebox{\includegraphics[width=\@photowidth]{\@photo}}}}}% + \usebox{\makecvtitlepicturebox}% + % name + \@initializelength{\makecvtitlepicturewidth}% + \settowidth{\makecvtitlepicturewidth}{\usebox{\makecvtitlepicturebox}}% + \parbox[b]{\textwidth-\makecvtitlepicturewidth}{% + \raggedleft\namefont{\color{color2!50}\@firstname} {\color{color2}\@lastname}}\\[-.35em]% alternate design: \MakeLowercase and no space + {\color{color2!50}\rule{\textwidth}{.25ex}}% + % optional title + \ifthenelse{\equal{\@title}{}}{}{\\[1.25em]\null\hfill\titlestyle{\@title}}\\[2.5em]% \null is required as there is no box on the line after \\, so glue (and leaders) disappears; this is in contrast to after \par, where the next line starts with an indent box (even after \noindent). + % optional quote + \ifthenelse{\isundefined{\@quote}}% + {}% + {{\null\hfill\begin{minipage}{\quotewidth}\centering\quotestyle{\@quote}\end{minipage}\hfill\null\\[2.5em]}}% + \par}% to avoid weird spacing bug at the first section if no blank line is left after \maketitle + +\renewcommand*{\makecvfooter}{% + \setlength{\footerwidth}{0.8\textwidth}% + \fancypagestyle{plain}{% + \fancyfoot[c]{% + \parbox[b]{\footerwidth}{% + \centering% + \color{color2}\addressfont% + \ifthenelse{\isundefined{\@addressstreet}}{}{\addtofooter[]{\addresssymbol\@addressstreet}% + \ifthenelse{\equal{\@addresscity}{}}{}{\addtofooter[~--~]{\@addresscity}}% if \addresstreet is defined, \addresscity and \addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\addtofooter[~--~]{\@addresscountry}}% + \flushfooter\@firstfooterelementtrue\\}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \addtofooter{\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}}% + \ifthenelse{\isundefined{\@email}}{}{\addtofooter{\emailsymbol\emaillink{\@email}}}% + \ifthenelse{\isundefined{\@homepage}}{}{\addtofooter{\homepagesymbol\httplink{\@homepage}}}% + \ifthenelse{\isundefined{\@github}}{}{\addtofooter{\httplink{http://github.com/\@github}}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\addtofooter{\@extrainfo}}% + \ifthenelse{\lengthtest{\footerboxwidth=0pt}}{}{\flushfooter}% the lengthtest is required to avoid flushing an empty footer, which could cause a blank line due to the \\ after the address, if no other personal info is used + }}}% + \pagestyle{plain}} + + +%------------------------------------------------------------------------------- +% letter style definition +%------------------------------------------------------------------------------- +\renewcommand*{\makelettertitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputeletterlengths% + % ensure footer with personal information + \makeletterfooter% + % recipient block + \begin{minipage}[t]{.5\textwidth} + \raggedright% + \addressfont% + {\bfseries\upshape\@recipientname}\\% + \@recipientaddress% + \end{minipage} + % date + \hfill% US style +% \\[1em]% UK style + \@date\\[2em]% US informal style: "April 6, 2006"; UK formal style: "05/04/2006" + % opening + \raggedright% + \@opening\\[1.5em]% + % ensure no extra spacing after \makelettertitle due to a possible blank line +% \ignorespacesafterend% not working + \hspace{0pt}\par\vspace{-\baselineskip}\vspace{-\parskip}} + +\renewcommand*{\makeletterfooter}{% + \setlength{\footerwidth}{0.8\textwidth}% + \fancypagestyle{plain}{% + \fancyfoot[c]{% + \parbox[b]{\footerwidth}{% + \centering% + \addressfont\color{color2}% + \vspace{-\baselineskip}% to cancel out the extra vertical space taken by the name (below) and ensure perfect alignment of letter and cv footers + \strut{\bfseries\upshape\@firstname~\@lastname}\\% the \strut is required to ensure the line is exactly \baselineskip tall + \ifthenelse{\isundefined{\@addressstreet}}{}{\addtofooter[]{\addresssymbol\@addressstreet}% + \ifthenelse{\equal{\@addresscity}{}}{}{\addtofooter[~--~]{\@addresscity}}% if \addresstreet is defined, \addresscity and addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\addtofooter[~--~]{\@addresscountry}}% + \flushfooter\@firstfooterelementtrue\\}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \addtofooter{\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}}% + \ifthenelse{\isundefined{\@email}}{}{\addtofooter{\emailsymbol\emaillink{\@email}}}% + \ifthenelse{\isundefined{\@homepage}}{}{\addtofooter{\homepagesymbol\httplink{\@homepage}}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\addtofooter{\@extrainfo}}% + \ifthenelse{\lengthtest{\footerboxwidth=0pt}}{}{\flushfooter}% the lengthtest is required to avoid flushing an empty footer, which could cause a blank line due to the \\ after the address, if no other personal info is used + }}}% + \pagestyle{plain}} + +\renewcommand*{\makeletterclosing}{ + \@closing\\[3em]% + {\bfseries\@firstname~\@lastname}% + \ifthenelse{\isundefined{\@enclosure}}{}{% + \\% + \vfil% + {\color{color2}\itshape\enclname: \@enclosure}}% + \vfil} + + +\endinput + + +%% end of file `moderncvstylecasual.sty'. diff --git a/users/grfn/resume/moderncvstyleclassic.sty b/users/grfn/resume/moderncvstyleclassic.sty new file mode 100644 index 000000000000..63cf97aa3b7d --- /dev/null +++ b/users/grfn/resume/moderncvstyleclassic.sty @@ -0,0 +1,294 @@ +%% start of file `moderncvstyleclassic.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvstyleclassic}[2013/02/09 v1.3.0 modern curriculum vitae and letter style scheme: classic] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- +% Latin Modern fonts +%\ifxetexorluatex +% \setmainfont{Latin Modern Roman} +% \setsansfont{Latin Modern Sans} +% \setmathfont{Latin Modern Math} +%\else + \IfFileExists{lmodern.sty}% + {\RequirePackage{lmodern}}% + {} +%\fi + + +%------------------------------------------------------------------------------- +% overall style definition +%------------------------------------------------------------------------------- +% symbols +\moderncvicons{marvosym} + + +%------------------------------------------------------------------------------- +% resume style definition +%------------------------------------------------------------------------------- +% fonts +\renewcommand*{\namefont}{\fontsize{34}{36}\mdseries\upshape} +\renewcommand*{\titlefont}{\LARGE\mdseries\slshape} +\renewcommand*{\addressfont}{\small\mdseries\slshape} +\renewcommand*{\quotefont}{\large\slshape} +\renewcommand*{\sectionfont}{\Large\mdseries\upshape} +\renewcommand*{\subsectionfont}{\large\mdseries\upshape} +\renewcommand*{\hintfont}{} + +% styles +\renewcommand*{\namestyle}[1]{{\namefont\textcolor{color0}{#1}}} +\renewcommand*{\titlestyle}[1]{{\titlefont\textcolor{color2}{#1}}} +\renewcommand*{\addressstyle}[1]{{\addressfont\textcolor{color1}{#1}}} +\renewcommand*{\quotestyle}[1]{{\quotefont\textcolor{color1}{#1}}} +\renewcommand*{\sectionstyle}[1]{{\sectionfont\textcolor{color1}{#1}}} +\renewcommand*{\subsectionstyle}[1]{{\subsectionfont\textcolor{color1}{#1}}} +\renewcommand*{\hintstyle}[1]{{\hintfont\textcolor{color0}{#1}}} + +% lengths +\newlength{\quotewidth} +\newlength{\hintscolumnwidth} +\setlength{\hintscolumnwidth}{0.175\textwidth} +\newlength{\separatorcolumnwidth} +\setlength{\separatorcolumnwidth}{0.025\textwidth} +\newlength{\maincolumnwidth} +\newlength{\doubleitemmaincolumnwidth} +\newlength{\listitemsymbolwidth} +\settowidth{\listitemsymbolwidth}{\listitemsymbol} +\newlength{\listitemmaincolumnwidth} +\newlength{\listdoubleitemmaincolumnwidth} + +% commands +\renewcommand*{\recomputecvlengths}{% + \setlength{\quotewidth}{0.65\textwidth}% + % main lenghts + \setlength{\maincolumnwidth}{\textwidth-\separatorcolumnwidth-\hintscolumnwidth}% + % listitem lengths + \setlength{\listitemmaincolumnwidth}{\maincolumnwidth-\listitemsymbolwidth}% + % doubleitem lengths + \setlength{\doubleitemmaincolumnwidth}{\maincolumnwidth-\hintscolumnwidth-\separatorcolumnwidth-\separatorcolumnwidth}% + \setlength{\doubleitemmaincolumnwidth}{0.5\doubleitemmaincolumnwidth}% + % listdoubleitem lengths + \setlength{\listdoubleitemmaincolumnwidth}{\maincolumnwidth-\listitemsymbolwidth-\separatorcolumnwidth-\listitemsymbolwidth}% + \setlength{\listdoubleitemmaincolumnwidth}{0.5\listdoubleitemmaincolumnwidth}% + % fancyhdr lengths + \renewcommand{\headwidth}{\textwidth}% + % regular lengths + \setlength{\parskip}{0\p@}} + +% optional maketitle width to force a certain width (if set to 0pt, the width is calculated automatically) +\newlength{\makecvtitlenamewidth} +\setlength{\makecvtitlenamewidth}{0pt}% dummy value +\renewcommand*{\makecvtitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputecvlengths% + % optional detailed information (pre-rendering) + \def\phonesdetails{}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \protected@edef\phonesdetails{\phonesdetails\protect\makenewline\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}}% + \newbox{\makecvtitledetailsbox}% + \savebox{\makecvtitledetailsbox}{% + \addressfont\color{color2}% + \begin{tabular}[b]{@{}r@{}}% + \ifthenelse{\isundefined{\@addressstreet}}{}{\makenewline\addresssymbol\@addressstreet% + \ifthenelse{\equal{\@addresscity}{}}{}{\makenewline\@addresscity}% if \addresstreet is defined, \addresscity and addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\makenewline\@addresscountry}}% + \phonesdetails% needed to be pre-rendered as loops and tabulars seem to conflict + \ifthenelse{\isundefined{\@email}}{}{\makenewline\emailsymbol\emaillink{\@email}}% + \ifthenelse{\isundefined{\@homepage}}{}{\makenewline\homepagesymbol\httplink{\@homepage}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\makenewline\@extrainfo}% + \end{tabular} + }% + % optional photo (pre-rendering) + \newbox{\makecvtitlepicturebox}% + \savebox{\makecvtitlepicturebox}{% + \ifthenelse{\isundefined{\@photo}}% + {}% + {% + \hspace*{\separatorcolumnwidth}% + \color{color1}% + \setlength{\fboxrule}{\@photoframewidth}% + \ifdim\@photoframewidth=0pt% + \setlength{\fboxsep}{0pt}\fi% + \framebox{\includegraphics[width=\@photowidth]{\@photo}}}}% + % name and title + \newlength{\makecvtitledetailswidth}\settowidth{\makecvtitledetailswidth}{\usebox{\makecvtitledetailsbox}}% + \newlength{\makecvtitlepicturewidth}\settowidth{\makecvtitlepicturewidth}{\usebox{\makecvtitlepicturebox}}% + \ifthenelse{\lengthtest{\makecvtitlenamewidth=0pt}}% check for dummy value (equivalent to \ifdim\makecvtitlenamewidth=0pt) + {\setlength{\makecvtitlenamewidth}{\textwidth-\makecvtitledetailswidth-\makecvtitlepicturewidth}}% + {}% + \begin{minipage}[b]{\makecvtitlenamewidth}% + \namestyle{\@firstname\ \@lastname}% + \ifthenelse{\equal{\@title}{}}{}{\\[1.25em]\titlestyle{\@title}}% + \end{minipage}% + \hfill% + % optional detailed information (rendering) + \llap{\usebox{\makecvtitledetailsbox}}% \llap is used to suppress the width of the box, allowing overlap if the value of makecvtitlenamewidth is forced + % optional photo (rendering) + \usebox{\makecvtitlepicturebox}\\[2.5em]% + % optional quote + \ifthenelse{\isundefined{\@quote}}% + {}% + {{\centering\begin{minipage}{\quotewidth}\centering\quotestyle{\@quote}\end{minipage}\\[2.5em]}}% + \par}% to avoid weird spacing bug at the first section if no blank line is left after \makecvtitle + +\newlength{\baseletterheight} +\settoheight{\baseletterheight}{\sectionstyle{o}} +\setlength{\baseletterheight}{\baseletterheight-0.95ex} +\RenewDocumentCommand{\section}{sm}{% + \par\addvspace{2.5ex}% + \phantomsection{}% reset the anchor for hyperrefs + \addcontentsline{toc}{section}{#2}% + \parbox[t]{\hintscolumnwidth}{\strut\raggedleft\raisebox{\baseletterheight}{\color{color1}\rule{\hintscolumnwidth}{0.95ex}}}% + \hspace{\separatorcolumnwidth}% + \parbox[t]{\maincolumnwidth}{\strut\sectionstyle{#2}}% + \par\nobreak\addvspace{1ex}\@afterheading}% to avoid a pagebreak after the heading + +\RenewDocumentCommand{\subsection}{sm}{% + \par\addvspace{1ex}% + \phantomsection{}% reset the anchor for hyperrefs + \addcontentsline{toc}{subsection}{#2}% + \begin{tabular}{@{}p{\hintscolumnwidth}@{\hspace{\separatorcolumnwidth}}p{\maincolumnwidth}@{}}% + \raggedleft\hintstyle{} &{\strut\subsectionstyle{#2}}% + \end{tabular}% + \par\nobreak\addvspace{0.5ex}\@afterheading}% to avoid a pagebreak after the heading + +\renewcommand*{\cvitem}[3][.25em]{% + \begin{tabular}{@{}p{\hintscolumnwidth}@{\hspace{\separatorcolumnwidth}}p{\maincolumnwidth}@{}}% + \raggedleft\hintstyle{#2} &{#3}% + \end{tabular}% + \par\addvspace{#1}} + +\renewcommand*{\cvdoubleitem}[5][.25em]{% + \cvitem[#1]{#2}{% + \begin{minipage}[t]{\doubleitemmaincolumnwidth}#3\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \begin{minipage}[t]{\hintscolumnwidth}\raggedleft\hintstyle{#4}\end{minipage}% + \hspace*{\separatorcolumnwidth}% + \begin{minipage}[t]{\doubleitemmaincolumnwidth}#5\end{minipage}}} + +\renewcommand*{\cvlistitem}[2][.25em]{% + \cvitem[#1]{}{\listitemsymbol\begin{minipage}[t]{\listitemmaincolumnwidth}#2\end{minipage}}} + +\renewcommand*{\cvlistdoubleitem}[3][.25em]{% + \cvitem[#1]{}{\listitemsymbol\begin{minipage}[t]{\listdoubleitemmaincolumnwidth}#2\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \ifthenelse{\equal{#3}{}}% + {}% + {\listitemsymbol\begin{minipage}[t]{\listdoubleitemmaincolumnwidth}#3\end{minipage}}}} + +\renewcommand*{\cventry}[7][.25em]{% + \cvitem[#1]{#2}{% + {\bfseries#3}% + \ifthenelse{\equal{#4}{}}{}{, {\slshape#4}}% + \ifthenelse{\equal{#5}{}}{}{, #5}% + \ifthenelse{\equal{#6}{}}{}{, #6}% + .\strut% + \ifx&% + \else{\newline{}\begin{minipage}[t]{\linewidth}\small#7\end{minipage}}\fi}} + +\newbox{\cvitemwithcommentmainbox} +\newlength{\cvitemwithcommentmainlength} +\newlength{\cvitemwithcommentcommentlength} +\renewcommand*{\cvitemwithcomment}[4][.25em]{% + \savebox{\cvitemwithcommentmainbox}{{\bfseries#3}}% + \setlength{\cvitemwithcommentmainlength}{\widthof{\usebox{\cvitemwithcommentmainbox}}}% + \setlength{\cvitemwithcommentcommentlength}{\maincolumnwidth-\separatorcolumnwidth-\cvitemwithcommentmainlength}% + \cvitem[#1]{#2}{% + \begin{minipage}[t]{\cvitemwithcommentmainlength}\bfseries#3\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \begin{minipage}[t]{\cvitemwithcommentcommentlength}\raggedleft\small\itshape#4\end{minipage}}} + +\renewenvironment{thebibliography}[1]% + {% + \bibliographyhead{\refname}% +% \small% + \begin{list}{\bibliographyitemlabel}% + {% + \setlength{\topsep}{0pt}% + \setlength{\labelwidth}{\hintscolumnwidth}% + \setlength{\labelsep}{\separatorcolumnwidth}% + \leftmargin\labelwidth% + \advance\leftmargin\labelsep% + \@openbib@code% + \usecounter{enumiv}% + \let\p@enumiv\@empty% + \renewcommand\theenumiv{\@arabic\c@enumiv}}% + \sloppy\clubpenalty4000\widowpenalty4000% +% \sfcode`\.\@m% +% \sfcode `\=1000\relax% + }% + {% + \def\@noitemerr{\@latex@warning{Empty `thebibliography' environment}}% + \end{list}% + } + + +%------------------------------------------------------------------------------- +% letter style definition +%------------------------------------------------------------------------------- +% commands +\renewcommand*{\recomputeletterlengths}{% + \recomputecvlengths% + \setlength{\parskip}{6\p@}} + +\renewcommand*{\makelettertitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputeletterlengths% + % sender contact info + \hfill% + \begin{minipage}{.5\textwidth}% + \raggedleft% + \addressfont\textcolor{color2}{% + {\bfseries\upshape\@firstname~\@lastname}\@firstdetailselementfalse% + \ifthenelse{\isundefined{\@addressstreet}}{}{\makenewline\addresssymbol\@addressstreet% + \ifthenelse{\equal{\@addresscity}{}}{}{\makenewline\@addresscity}% if \addresstreet is defined, \addresscity and addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\makenewline\@addresscountry}}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \makenewline\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}% + \ifthenelse{\isundefined{\@email}}{}{\makenewline\emailsymbol\emaillink{\@email}}% + \ifthenelse{\isundefined{\@homepage}}{}{\makenewline\homepagesymbol\httplink{\@homepage}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\makenewline\@extrainfo}}% + \end{minipage}\\[1em] + % recipient block + \begin{minipage}[t]{.5\textwidth} + \raggedright% + \addressfont% + {\bfseries\upshape\@recipientname}\\% + \@recipientaddress% + \end{minipage} + % date + \hfill% US style +% \\[1em]% UK style + \@date\\[2em]% US informal style: "January 1, 1900"; UK formal style: "01/01/1900" + % opening + \raggedright% + \@opening\\[1.5em]% + % ensure no extra spacing after \makelettertitle due to a possible blank line +% \ignorespacesafterend% not working + \hspace{0pt}\par\vspace{-\baselineskip}\vspace{-\parskip}} + +\renewcommand*{\makeletterclosing}{ + \@closing\\[3em]% + {\bfseries \@firstname~\@lastname}% + \ifthenelse{\isundefined{\@enclosure}}{}{% + \\% + \vfill% + {\color{color2}\itshape\enclname: \@enclosure}}} + + +\endinput + + +%% end of file `moderncvstyleclassic.sty'. diff --git a/users/grfn/resume/moderncvstyleempty.sty b/users/grfn/resume/moderncvstyleempty.sty new file mode 100644 index 000000000000..85932464d1c5 --- /dev/null +++ b/users/grfn/resume/moderncvstyleempty.sty @@ -0,0 +1,34 @@ +%% start of file `moderncvstyleempty.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvstyleempty}[2013/02/09 v1.3.0 modern curriculum vitae scheme: empty] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- + + +%------------------------------------------------------------------------------- +% package options +%------------------------------------------------------------------------------- + + +%------------------------------------------------------------------------------- +% style definition +%------------------------------------------------------------------------------- +% see moderncv.cls for command declarations that needs to be implemented, e.g. \maketitle, \section, \subsections, \cvline, etc + +\endinput + + +%% end of file `moderncvstyleempty.sty'. diff --git a/users/grfn/resume/moderncvstyleoldstyle.sty b/users/grfn/resume/moderncvstyleoldstyle.sty new file mode 100644 index 000000000000..ff732f4e2af5 --- /dev/null +++ b/users/grfn/resume/moderncvstyleoldstyle.sty @@ -0,0 +1,306 @@ +%% start of file `moderncvstyleoldstyle.sty'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +%------------------------------------------------------------------------------- +% identification +%------------------------------------------------------------------------------- +\NeedsTeXFormat{LaTeX2e} +\ProvidesPackage{moderncvstyleoldstyle}[2013/02/09 v1.3.0 modern curriculum vitae and letter style scheme: oldstyle] + + +%------------------------------------------------------------------------------- +% required packages +%------------------------------------------------------------------------------- +% change the layout of the page on the fly, for resume or letter layout +\RequirePackage{changepage} + + +%------------------------------------------------------------------------------- +% overall style definition +%------------------------------------------------------------------------------- +% fonts +%\ifxetexorluatex +% \setmainfont[Numbers={OldStyle,Proportional}, BoldFont={Kurier Bold}, ItalicFont={Kurier Light Italic}, BoldItalicFont={Kurier Bold Italic}]{Kurier Light} +% \setsansfont[Numbers={OldStyle,Proportional}, BoldFont={Kurier Bold}, ItalicFont={Kurier Light Italic}, BoldItalicFont={Kurier Bold Italic}]{Kurier Light} +% \setmathfont{Kurier Light} +% \setmathfont[range=\mathit,\mathsfit]{Kurier Light Italic} +% \setmathfont[range=\mathbfup,\mathbfsfup]{Kurier Bold} +% \setmathfont[range=\mathbfit,\mathbfsfit]{Kurier Bold Italic} +%\else + \IfFileExists{kurier.sty}% + {\RequirePackage[light,math]{kurier}}% + {} +%\fi + +% symbols +\moderncvicons{letters} + + +%------------------------------------------------------------------------------- +% resume style definition +%------------------------------------------------------------------------------- +% fonts +\renewcommand*{\namefont}{\fontsize{34}{36}\mdseries\upshape} +\renewcommand*{\titlefont}{\LARGE\mdseries\slshape} +\renewcommand*{\addressfont}{\small\mdseries} +\renewcommand*{\quotefont}{\large\itshape} +\renewcommand*{\sectionfont}{\Large\bfseries\upshape} +\renewcommand*{\subsectionfont}{\large\bfseries\itshape} +\renewcommand*{\hintfont}{\bfseries} + +% styles +\renewcommand*{\namestyle}[1]{{\namefont\textcolor{color0}{#1}}} +\renewcommand*{\titlestyle}[1]{{\titlefont\textcolor{color2}{#1}}} +\renewcommand*{\addressstyle}[1]{{\addressfont\textcolor{color2}{#1}}} +\renewcommand*{\quotestyle}[1]{{\quotefont\textcolor{color1}{#1}}} +\renewcommand*{\sectionstyle}[1]{{\sectionfont\textcolor{color1}{#1}}} +\renewcommand*{\subsectionstyle}[1]{{\subsectionfont\textcolor{color1}{#1}}} +\renewcommand*{\hintstyle}[1]{{\hintfont\textcolor{color0}{#1}}} + +% lengths +\newlength{\quotewidth} +\newlength{\hintscolumnwidth} +\setlength{\hintscolumnwidth}{0.3\textwidth}% +\newlength{\separatorcolumnwidth} +\setlength{\separatorcolumnwidth}{0.025\textwidth}% +\newlength{\maincolumnwidth} +\newlength{\doubleitemcolumnwidth} +\newlength{\listitemsymbolwidth} +\settowidth{\listitemsymbolwidth}{\listitemsymbol} +\newlength{\listitemmaincolumnwidth} +\newlength{\listdoubleitemmaincolumnwidth} + +% commands +\setlength{\marginparwidth}{0\p@}% +\setlength{\marginparsep}{0\p@} +\renewcommand*{\recomputecvlengths}{% + % regular lengths + \changepage{}{+\marginparwidth+\marginparsep}{}{}{}{}{}{}{}% if a letter was typeset before the resume, \marginparwidth and \marginparsep will be non-zero; otherwise, this has no effect + \setlength{\marginparwidth}{0\p@}% + \setlength{\marginparsep}{0\p@} + \setlength{\parskip}{0\p@}% + % maketitle lengths + \setlength{\quotewidth}{0.65\textwidth}% + % main lenghts + \setlength{\maincolumnwidth}{\textwidth-\hintscolumnwidth-\separatorcolumnwidth}% + % listitem lengths + \setlength{\listitemmaincolumnwidth}{\maincolumnwidth-\listitemsymbolwidth}% + % doubleitem lengths + \setlength{\doubleitemcolumnwidth}{\maincolumnwidth-\separatorcolumnwidth}% + \setlength{\doubleitemcolumnwidth}{0.5\doubleitemcolumnwidth}% + % listdoubleitem lengths + \setlength{\listdoubleitemmaincolumnwidth}{\maincolumnwidth-\listitemsymbolwidth-\separatorcolumnwidth-\listitemsymbolwidth}% + \setlength{\listdoubleitemmaincolumnwidth}{0.5\listdoubleitemmaincolumnwidth}% + % fancyhdr lengths + \renewcommand{\headwidth}{\textwidth}} + +\newcommand{\makecvinfo}[1]{% + \newbox{\makecvinfobox}% + \savebox{\makecvinfobox}{\parbox[t]{\hintscolumnwidth}{#1}}% + \newlength{\makecvinfoheight}% + \setlength{\makecvinfoheight}{\totalheightof{\usebox{\makecvinfobox}}}% the total height of the parbox is the sum of its height (\the\ht\makeinfobox) and its depth (\the\dp\makeinfobox); the \totalheightof command is provided by the "calc" package + \usebox{\makecvinfobox}\vspace{-\makecvinfoheight}% + \newlength{\leftcolumnwidth}% + \setlength{\leftcolumnwidth}{\hintscolumnwidth+\separatorcolumnwidth}% + \par\vspace{-\baselineskip}\vspace{-\parskip}\leftskip=\leftcolumnwidth} + +\renewcommand*{\makecvtitle}{ + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputecvlengths% + % optional picture box + \newbox{\makecvtitlepicturebox}% + \savebox{\makecvtitlepicturebox}{% + \ifthenelse{\isundefined{\@photo}}% + {}% + {% + \color{color1}% + \setlength\fboxrule{\@photoframewidth}% + \ifdim\@photoframewidth=0pt% + \setlength{\fboxsep}{0pt}\fi% + \framebox{\includegraphics[width=\@photowidth]{\@photo}}}}% + % name and title + \newlength{\makecvtitlepicturewidth}\settowidth{\makecvtitlepicturewidth}{\usebox{\makecvtitlepicturebox}}% + \newlength{\makecvtitlenamewidth}\setlength{\makecvtitlenamewidth}{\textwidth-\makecvtitlepicturewidth}% + \begin{minipage}[b]{\makecvtitlenamewidth}% + \namestyle{\@firstname\ \@lastname}% + \ifthenelse{\equal{\@title}{}}{}{\\[1.25em]\titlestyle{\@title}}% + \end{minipage}% + % optional photo + \usebox{\makecvtitlepicturebox}\\[2.5em]% + % optional quote + \ifthenelse{\isundefined{\@quote}}% + {}% + {{\centering\begin{minipage}{\quotewidth}\centering\quotestyle{\@quote}\end{minipage}\\[2.5em]}}% + % optional details + \makecvinfo{% + \addressfont\color{color2}% + \ifthenelse{\isundefined{\@addressstreet}}{}{\makenewline\addresssymbol\@addressstreet% + \ifthenelse{\equal{\@addresscity}{}}{}{\makenewline\@addresscity}% if \addresstreet is defined, \addresscity and \addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\makenewline\@addresscountry}}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \makenewline\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}% + \ifthenelse{\isundefined{\@email}}{}{\makenewline\emailsymbol\emaillink{\@email}}% + \ifthenelse{\isundefined{\@homepage}}{}{\makenewline\homepagesymbol\httplink{\@homepage}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\makenewline\@extrainfo}}} + +\RenewDocumentCommand{\section}{sm}{% + \par\addvspace{2.5ex}% + \phantomsection{}% reset the anchor for hyperrefs + \addcontentsline{toc}{section}{#2}% + \strut\sectionstyle{#2}% + \par\nobreak\addvspace{1ex}\@afterheading}% to avoid a pagebreak after the heading + +\RenewDocumentCommand{\subsection}{sm}{% + \par\addvspace{1ex}% + \phantomsection{}% reset the anchor for hyperrefs + \addcontentsline{toc}{subsection}{#2}% + \strut\subsectionstyle{#2}% + \par\nobreak\addvspace{0.5ex}\@afterheading}% to avoid a pagebreak after the heading + +\renewcommand*{\cvitem}[3][.25em]{% + \ifthenelse{\equal{#2}{}}{}{\hintstyle{#2}: }{#3}% + \par\addvspace{#1}} + +\renewcommand*{\cvdoubleitem}[5][.25em]{% + \begin{minipage}[t]{\doubleitemcolumnwidth}\hintstyle{#2}: #3\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \begin{minipage}[t]{\doubleitemcolumnwidth}\ifthenelse{\equal{#4}{}}{}{\hintstyle{#4}: }#5\end{minipage}% + \par\addvspace{#1}} + +\renewcommand*{\cvlistitem}[2][.25em]{% + \cvitem[#1]{}{\listitemsymbol\begin{minipage}[t]{\listitemmaincolumnwidth}#2\end{minipage}}} + +\renewcommand*{\cvlistdoubleitem}[3][.25em]{% + \cvitem[#1]{}{\listitemsymbol\begin{minipage}[t]{\listdoubleitemmaincolumnwidth}#2\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \ifthenelse{\equal{#3}{}}% + {}% + {\listitemsymbol\begin{minipage}[t]{\listdoubleitemmaincolumnwidth}#3\end{minipage}}}} + +\newbox{\cventryyearbox} +\newlength{\cventrytitleboxwidth} +\renewcommand*{\cventry}[7][.25em]{% + \savebox{\cventryyearbox}{% + \hspace*{2\separatorcolumnwidth}% + \hintstyle{#2}}% + \setlength{\cventrytitleboxwidth}{\widthof{\usebox{\cventryyearbox}}}% + \setlength{\cventrytitleboxwidth}{\maincolumnwidth-\cventrytitleboxwidth}% + \begin{minipage}{\maincolumnwidth}% + \parbox[t]{\cventrytitleboxwidth}{% + \strut% + {\bfseries#3}% + \ifthenelse{\equal{#4}{}}{}{, {\slshape#4}}% + \ifthenelse{\equal{#5}{}}{}{, #5}% + \ifthenelse{\equal{#6}{}}{}{, #6}% + .\strut}% + \usebox{\cventryyearbox}% + \end{minipage}% + \ifx&% + \else{% + \newline{}% + \begin{minipage}[t]{\maincolumnwidth}% + \small% + #7% + \end{minipage}}\fi% + \par\addvspace{#1}} + +\newbox{\cvitemwithcommentmainbox} +\newlength{\cvitemwithcommentmainlength} +\newlength{\cvitemwithcommentcommentlength} +\renewcommand*{\cvitemwithcomment}[4][.25em]{% + \savebox{\cvitemwithcommentmainbox}{\ifthenelse{\equal{#2}{}}{}{\hintstyle{#2}: }#3}% + \setlength{\cvitemwithcommentmainlength}{\widthof{\usebox{\cvitemwithcommentmainbox}}}% + \setlength{\cvitemwithcommentcommentlength}{\maincolumnwidth-\separatorcolumnwidth-\cvitemwithcommentmainlength}% + \begin{minipage}[t]{\cvitemwithcommentmainlength}\ifthenelse{\equal{#2}{}}{}{\hintstyle{#2}: }#3\end{minipage}% + \hfill% fill of \separatorcolumnwidth + \begin{minipage}[t]{\cvitemwithcommentcommentlength}\raggedleft\small\itshape#4\end{minipage}% + \par\addvspace{#1}} + +\renewenvironment{thebibliography}[1]% + {% + \bibliographyhead{\refname}% +% \small% + \begin{list}{\bibliographyitemlabel}% + {% + \setlength{\topsep}{0pt}% + \setlength{\labelwidth}{\hintscolumnwidth}% + \setlength{\labelsep}{\separatorcolumnwidth}% + \leftmargin\labelwidth% + \advance\leftmargin\labelsep% + \@openbib@code% + \usecounter{enumiv}% + \let\p@enumiv\@empty% + \renewcommand\theenumiv{\@arabic\c@enumiv}}% + \sloppy\clubpenalty4000\widowpenalty4000% +% \sfcode`\.\@m% +% \sfcode `\=1000\relax% + }% + {% + \def\@noitemerr{\@latex@warning{Empty `thebibliography' environment}}% + \end{list}% + } + + +%------------------------------------------------------------------------------- +% letter style definition +%------------------------------------------------------------------------------- +% commands +%\newlength{\textwidthdelta}% +\renewcommand*{\recomputeletterlengths}{% + \recomputecvlengths% + \setlength{\parskip}{6\p@}% + \leftskip=0pt% +% \setlength{\textwidthdelta}{+\marginparwidth+\marginparsep}% + \setlength{\marginparwidth}{\hintscolumnwidth}% + \setlength{\marginparsep}{2\separatorcolumnwidth}% +% \addtolength{\textwidthdelta}{-\marginparwidth-\marginparsep}% +% \changepage{}{\textwidthdelta}{-\textwidthdelta}{}{}{}{}{}{}%\changepage{<textheight>}{<textwidth>}{<evensidemargin>}{<oddsidemargin>}{<columnsep>}{<topmargin>}{<headheight>}{<headsep>}{<footskip>} + \changepage{}{-\marginparwidth-\marginparsep}{}{}{}{}{}{}{}%\changepage{<textheight>}{<textwidth>}{<evensidemargin>}{<oddsidemargin>}{<columnsep>}{<topmargin>}{<headheight>}{<headsep>}{<footskip>} + } + +\renewcommand*{\makelettertitle}{% + % recompute lengths (in case we are switching from letter to resume, or vice versa) + \recomputeletterlengths% + % recipient block + {\addressfont% + {\bfseries\upshape\@recipientname}\\% + \@recipientaddress}\\[1em]% + % date + \@date\\[2em]% + % opening + \@opening\\[1.5em]% + % sender contact info + \hspace{0pt}% + \marginpar{% + \addressfont\textcolor{color2}{% + {\bfseries\@firstname~\@lastname}\@firstdetailselementfalse% + \ifthenelse{\isundefined{\@addressstreet}}{}{\makenewline\addresssymbol\@addressstreet% + \ifthenelse{\equal{\@addresscity}{}}{}{\makenewline\@addresscity}% if \addresstreet is defined, \addresscity and \addresscountry will always be defined but could be empty + \ifthenelse{\equal{\@addresscountry}{}}{}{\makenewline\@addresscountry}}% + \collectionloop{phones}{% the key holds the phone type (=symbol command prefix), the item holds the number + \makenewline\csname\collectionloopkey phonesymbol\endcsname\collectionloopitem}% + \ifthenelse{\isundefined{\@email}}{}{\makenewline\emailsymbol\emaillink{\@email}}% + \ifthenelse{\isundefined{\@homepage}}{}{\makenewline\homepagesymbol\httplink{\@homepage}}% + \ifthenelse{\isundefined{\@extrainfo}}{}{\makenewline\@extrainfo}}}% + % ensure no extra spacing after \makelettertitle due to a possible blank line +% \ignorespacesafterend% not working + \par\vspace{-\baselineskip}\vspace{-\parskip}} + +\renewcommand*{\makeletterclosing}{ + \@closing\\[3em]% + {\bfseries\@firstname~\@lastname}% + \ifthenelse{\isundefined{\@enclosure}}{}{% + \\% + \vfill% + {\color{color2}\itshape\enclname: \@enclosure}}} + + +\endinput + + +%% end of file `moderncvstyleoldstyle.sty'. diff --git a/users/grfn/resume/picture.png b/users/grfn/resume/picture.png new file mode 100644 index 000000000000..63b21b5320ed --- /dev/null +++ b/users/grfn/resume/picture.png Binary files differdiff --git a/users/grfn/resume/resume.tex b/users/grfn/resume/resume.tex new file mode 100644 index 000000000000..933558d570b6 --- /dev/null +++ b/users/grfn/resume/resume.tex @@ -0,0 +1,212 @@ +%% start of file `template.tex'. +%% Copyright 2006-2013 Xavier Danaux (xdanaux@gmail.com). +%% Copyright 2014-2020 Griffin Smith (wildgriffin45@gmail.com). +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +\documentclass[10pt,a4paper,sans]{moderncv} % possible options include font size ('10pt', '11pt' and '12pt'), paper size ('a4paper', 'letterpaper', 'a5paper', 'legalpaper', 'executivepaper' and 'landscape') and font family ('sans' and 'roman') + +\usepackage[inline]{enumitem} + + +% moderncv themes +% style options are 'casual' (default), 'classic', 'oldstyle' and 'banking' +\moderncvstyle{casual} +% color options 'blue' (default), 'orange', 'green', 'red', 'purple', 'grey' and 'black' +\moderncvcolor{black} +% to set the default font; use '\sfdefault' for the default sans serif font, +% '\rmdefault' for the default roman one, or any tex font name +%\renewcommand{\familydefault}{\sfdefault} +\nopagenumbers{} + +\usepackage[utf8]{inputenc} + +\usepackage[scale=0.8, margin=0.65in]{geometry} +\setlength{\hintscolumnwidth}{2.6cm} + +\name{Griffin}{Smith} +\title{Software Engineer} +\phone[mobile]{(720) 206-7218} +\email{grfn@gws.fyi} +\homepage{https://www.gws.fyi} +\extrainfo{References available upon request} + +\begin{document} +\makecvtitle{} +\section{Skills} +\cvitem{Clojure}{Extensive experience architecting, deploying, and building +complex web applications in Clojure and Clojurescript, with a focus on +Re-Frame and Reagent.} +\cvitem{Haskell}{Passionate love for pure functional programming as a hobbyist +pursuit, but also practical experience building production systems in Haskell at +scale, and using Haskell's advanced type system extensions where appropriate to +deliver increased ergonomics and safety.} +\cvitem{Nix}{Experience with adopting and teaching nix at scale in a production +stack both for local development dependencies and for configuring and building +production software. Core contributer to a fork of the nix implementation itself +(tvix) aimed at providing increased safety, performance, and flexibility.} +\cvitem{Scala}{Understanding of Scala from the perspective of a functional +programmer rather than a Java programmer. Experience building production +big-data processing systems using Akka, and deep programming with Scala's type +system using Shapeless.} +\cvitem{Unix/Linux}{Experience with administrating highly available distributed +systems. Passion for the Unix philosophy of discrete, composable units of +functionality.} +\cvitem{Ruby}{Experience building both full-stack applications with Ruby on +Rails in addition to smaller microservices and custom frameworks. Deep +understanding of the internals of the Ruby interpreter and object system.} +\cvitem{Javascript}{Experience developing real-time responsive single-page web +applications using React, in addition to significant contributions to the React +open-source community.} +\cvitem{SQL}{Deep understanding of relational databases, including experience +designing the database schema in Postgres for an application with over a decade +of usage, hundreds of gigabytes of data, complex, multi-tiered hierarchical data +structures, as well as experience writing and optimizing large, complex queries +against that database.} + +\subsection{Additional Tools} +\cvitem{}{\footnotesize + \begin{itemize*} + \item Vim + \item Kubernetes + \item Git + \item Puppet + \item AWS + \item Reagent + \item Datomic + \item Elasticsearch + \item Redis + \item DynamoDB + \item Docker + \item JIRA + \item Java + \item QuickCheck (and similar tools) + \item Python + \item Elixir + \end{itemize*} + \newline + \textbf{Novice Level:} + \begin{itemize*} + \item Rust + \item C++ + \item Erlang + \item Prolog + \item Idris + \item Agda + \item Tensorflow + \end{itemize*}} + +\section{Experience} +\subsection{Employment} +\cventry{2019-present}{Engineering Manager}{Urbint}{New York, NY}{} +{\begin{itemize} + \item Lead of the platform team with two direct reports - a senior SRE and + a senior software engineer. + \item Performed user research on developers, project managers, product + managers, and other internal stakeholders to build the roadmap for the + platform team. + \item Built and maintained a system to deploy one-off full stack + application instances from pull requests to enable easier testing. + \item Led a large, multi-project migration between CI systems that resulted + in a decrease of average build times from 2 hours to less than 10 minutes. + \item Maintained and extended Nix-based build and development + infrastructure for both software engineers and machine learning engineers. + \end{itemize}} +\cventry{2018--2019}{Senior Software Engineer}{Urbint}{New York, NY}{} +{\begin{itemize} + \item Built, trained, and maintained a large, deep-learning-based + image-detection model for semi-automated (human-in-the-loop) video + classification. + \item Designed, built, and maintained a novel in-house tool for collection of + training data. + \item Maintained and guaranteed reliability of a large data pipeline for + video processing and classification. + \end{itemize}} +\cventry{2017--2018}{Senior Software Engineer}{Urbint}{New York, NY}{} +{\begin{itemize} + \item Integral in the architecture of a novel, serializable ACID + transactional graph database built on RocksDB, first in Elixir then in + Haskell. + \item Helped ship customer deliverables involving multi-day data + processing jobs for disparate data sources. + \item Instructed other developers in the use of and theory behind Haskell + \item Brought computational graph theory to bear on the problem of unifying + disparate, highly heterogeneous data sources across the world of open data. + \end{itemize}} +\cventry{2016--2017}{Senior Software Engineer}{SecurityScorecard, Inc.}{New York, NY}{} +{Lead frontend developer for a rapidly-moving and growing security software startup. + \begin{itemize} + \item Took part in collaborative product design meetings to make UX + tradeoffs with product designers and managers. + \item Drove application architecture for a large, complex, data-driven frontend + application. + \item Championed increased use of production monitoring and alerting. + \item Worked with business stakeholders to set long- and short-term priorities for + application development. + \item Mentored junior team members. + \end{itemize}} +\cventry{2015--2016}{Lead Developer}{Nomi, Inc.}{New York, NY}{} +{Lead web services developer transitioning to a full-stack role implementing + shared software components and architecting a large, complex microservices + application ingesting hundreds of gigabytes of IoT data per week. + \begin{itemize} + \item Lead application architecture of the majority of the backend services to + encourage consistent REST API design and code sharing. + \item Championed the use of Haskell for rapid, safe development of the API Gateway + service. + \item Took ownership of operations and server maintenance of a >100-instance AWS + account using Puppet. + \end{itemize}} +\cventry{2014--2015}{Lead Developer}{LandlordsNY, LLC}{New York, NY}{} +{Sole engineer for a small startup connecting landlords and property managers and + facilitating the online sharing of information in a historically technology-averse + industry. + \begin{itemize} + \item Drove product design, visual design, and UX architecture for a major revamping + of the core product. + \item Interfaced with customers to set priorities for new feature development. + \item Conducted hiring and recruiting to build out an engineering team. + \end{itemize}} +\cventry{2012--2014}{Associate Developer}{Visionlink Inc.}{Boulder, CO}{} +{Integral member of an agile development team building the nation's most-used Information + and Referral platform for organizations such as United Way Worldwide and the American Red + Cross. + \begin{itemize} + \item Refactored and revamped legacy code to increase performance and long-term + maintainablity. + \item Worked on several triage-teams to rapidly fix production bugs with strict deadlines. + \item Built a complex, yet highly-performant tool for searching human services by category. + \item Acted as a core designer and developer of a major product revamp. + \begin{itemize} + \item Drove a complete rethinking of the data model in the product, leading to greater + unification, simplicity, and consistency; + \item Championed the adoption of a test-driven-development model; + \item Drove product documentation and code standardization. + \end{itemize} + \end{itemize}} + +\section{Project Highlights} +\newcommand{\project}[3]{\item \textbf{#1} -- \textit{#2}\newline{}#3} +\cvitem{}{\begin{itemize} + \project{Github Bug Bounty}{https://bounty.github.com/researchers/glittershark.html}{ + Discovered and responsibly disclosed a persistent XSS on Github's main + website} + \project{Tvix}{https://cs.tvl.fyi/depot/-/blob/third\_party/nix/README.md}{ + Fork of the Nix build tool delivering increased reliability, code + quality, and pluggability} + \project{Panettone}{https://cs.tvl.fyi/depot/-/tree/web/panettone}{ + Aggressively simple bug-tracker developed in Common Lisp for the community + involved in the development of Tvix. Hosted at https://b.tvl.fyi} + \project{Org-Clubhouse}{https://github.com/glittershark/org-clubhouse}{ + Emacs library for integration between org-mode and the Clubhouse issue + tracker} + \project{core-async-storage}{https://github.com/glittershark/core-async-storage}{ + Simple Clojurescript wrapper around React Native's AsyncStorage using + core.async} +\end{itemize}} + +\end{document} +% vim: set tw=95 colorcolumn=-1: diff --git a/users/grfn/resume/tweaklist.sty b/users/grfn/resume/tweaklist.sty new file mode 100644 index 000000000000..adc939893261 --- /dev/null +++ b/users/grfn/resume/tweaklist.sty @@ -0,0 +1,56 @@ +%% start of file `tweaklist.sty'. +%% Original by Jakob Schiøtz, downloaded from http://dcwww.camd.dtu.dk/~schiotz/comp/LatexTips/tweaklist.sty; not found on ctan. +%% Modified by Xavier Danaux (xdanaux@gmail.com). +% +% The tweaklist.sty package redefines the itemize, enumerate and description packages, so that all parameters can be adjusted. +% This was done by copying the original definitions, and adding "hook commands" that are executed when entering the environment. +% The hook commands are initially empty, but can be redefined with \renewcommand. +% +% This work may be distributed and/or modified under the +% conditions of the LaTeX Project Public License version 1.3c, +% available at http://www.latex-project.org/lppl/. + + +% hooks for the itemize environment +\def\itemhook{} +\def\itemhooki{} +\def\itemhookii{} +\def\itemhookiii{} +\def\itemhookiv{} +% hooks for the enumerate environment +\def\enumhook{} +\def\enumhooki{} +\def\enumhookii{} +\def\enumhookiii{} +\def\enumhookiv{} +% hook for the description environment +\def\deschook{} +% original environment definitions, with hooks added +\def\enumerate{% + \ifnum \@enumdepth >\thr@@\@toodeep\else + \advance\@enumdepth\@ne + \edef\@enumctr{enum\romannumeral\the\@enumdepth}% + \expandafter + \list + \csname label\@enumctr\endcsname + {% + \enumhook \csname enumhook\romannumeral\the\@enumdepth\endcsname% + \usecounter\@enumctr\def\makelabel##1{\hss\llap{##1}}% + }% + \fi} +\def\itemize{% + \ifnum \@itemdepth >\thr@@\@toodeep\else + \advance\@itemdepth\@ne + \edef\@itemitem{labelitem\romannumeral\the\@itemdepth}% + \expandafter + \list + \csname\@itemitem\endcsname + {% + \itemhook \csname itemhook\romannumeral\the\@itemdepth\endcsname% + \def\makelabel##1{\hss\llap{##1}}% + }% + \fi} +\newenvironment{description} + {\list{}{\deschook\labelwidth\z@ \itemindent-\leftmargin + \let\makelabel\descriptionlabel}} + {\endlist} diff --git a/users/grfn/system/.gitignore b/users/grfn/system/.gitignore new file mode 100644 index 000000000000..41fbeb02c47d --- /dev/null +++ b/users/grfn/system/.gitignore @@ -0,0 +1 @@ +**/result diff --git a/users/grfn/system/home/.skip-subtree b/users/grfn/system/home/.skip-subtree new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/users/grfn/system/home/.skip-subtree diff --git a/users/grfn/system/home/common/solarized.nix b/users/grfn/system/home/common/solarized.nix new file mode 100644 index 000000000000..e94693edc566 --- /dev/null +++ b/users/grfn/system/home/common/solarized.nix @@ -0,0 +1,18 @@ +rec { + base03 = "#002B36"; + base02 = "#073642"; + base01 = "#586e75"; + base00 = "#657b83"; + base0 = "#839496"; + base1 = "#93a1a1"; + base2 = "#eee8d5"; + base3 = "#fdf6e3"; + yellow = "#b58900"; + orange = "#cb4b16"; + red = "#dc322f"; + magenta = "#d33682"; + violet = "#6c71c4"; + blue = "#268bd2"; + cyan = "#2aa198"; + green = "#859900"; +} diff --git a/users/grfn/system/home/default.nix b/users/grfn/system/home/default.nix new file mode 100644 index 000000000000..f62704152b28 --- /dev/null +++ b/users/grfn/system/home/default.nix @@ -0,0 +1,32 @@ +{ pkgs, depot, lib, ... }: + +with lib; + +rec { + home = confPath: (import "${pkgs.home-manager.src}/modules" { + inherit pkgs; + + configuration = { config, lib, ... }: { + imports = [ confPath ]; + lib.depot = depot; + + # home-manager exposes no API to override the package set that + # is used, unless called from the NixOS module. + # + # To get around it, the module argument is overridden here. + _module.args.pkgs = mkForce pkgs; + }; + }); + + dobharchu = home ./machines/dobharchu.nix; + + dobharchuHome = dobharchu.activation-script; + + yeren = home ./machines/yeren.nix; + + yerenHome = yeren.activation-script; + + meta.targets = [ + "yerenHome" + ]; +} diff --git a/users/grfn/system/home/home.nix b/users/grfn/system/home/home.nix new file mode 100644 index 000000000000..39045c147d76 --- /dev/null +++ b/users/grfn/system/home/home.nix @@ -0,0 +1,20 @@ +{ config, pkgs, ... }: + +{ + imports = [ + (throw "Pick a machine from ./machines") + ]; + + # Let Home Manager install and manage itself. + programs.home-manager.enable = true; + + # This value determines the Home Manager release that your + # configuration is compatible with. This helps avoid breakage + # when a new Home Manager release introduces backwards + # incompatible changes. + # + # You can update Home Manager without changing this value. See + # the Home Manager release notes for a list of state version + # changes in each release. + home.stateVersion = "19.09"; +} diff --git a/users/grfn/system/home/machines/dobharchu.nix b/users/grfn/system/home/machines/dobharchu.nix new file mode 100644 index 000000000000..0b8503a00e98 --- /dev/null +++ b/users/grfn/system/home/machines/dobharchu.nix @@ -0,0 +1,17 @@ +{ config, lib, pkgs, ... }: + +{ + imports = [ + ../platforms/darwin.nix + ../modules/common.nix + ../modules/games.nix + ]; + + home.packages = with pkgs; [ + coreutils + gnupg + nix-prefetch-github + pass + pinentry_mac + ]; +} diff --git a/users/grfn/system/home/machines/roswell.nix b/users/grfn/system/home/machines/roswell.nix new file mode 100644 index 000000000000..f68f3a7be36f --- /dev/null +++ b/users/grfn/system/home/machines/roswell.nix @@ -0,0 +1,54 @@ +{ pkgs, lib, config, ... }: + +let + laptopKeyboardId = "5"; +in + +{ + imports = [ + ../platforms/linux.nix + ../modules/shell.nix + ../modules/development.nix + ../modules/emacs.nix + ../modules/vim.nix + ]; + + home.packages = with pkgs; [ + # System utilities + bat + htop + killall + bind + zip unzip + tree + ncat + bc + pv + + # Security + gnupg + keybase + openssl + + # Nix things + nixfmt + nix-prefetch-github + nix-review + cachix + ]; + + programs.password-store.enable = true; + + programs.home-manager.enable = true; + home.stateVersion = "20.03"; + + xsession.enable = lib.mkForce false; + + services.lorri.enable = true; + + programs.direnv = { + enable = true; + enableBashIntegration = true; + enableZshIntegration = true; + }; +} diff --git a/users/grfn/system/home/machines/yeren.nix b/users/grfn/system/home/machines/yeren.nix new file mode 100644 index 000000000000..0c89e9d12393 --- /dev/null +++ b/users/grfn/system/home/machines/yeren.nix @@ -0,0 +1,77 @@ +{ pkgs, lib, config, ... }: + +let + inherit (builtins) pathExists; + laptopKeyboardId = "5"; +in + +{ + imports = [ + ../platforms/linux.nix + ../modules/common.nix + ../modules/development/readyset.nix + ] ++ (lib.optional (pathExists ../modules/private.nix) ../modules/private.nix); + + # for when hacking + programs.home-manager.enable = true; + home.stateVersion = "20.03"; + + system.machine = { + wirelessInterface = "wlp0s20f3"; + i3FontSize = 9; + }; + + home.packages = with pkgs; [ + zoom-us + slack + mysql + graphviz + mypaint + xdot + tdesktop + subsurface + + (discord.override rec { + version = "0.0.16"; + src = fetchurl { + url = "https://dl.discordapp.net/apps/linux/${version}/discord-${version}.tar.gz"; + sha256 = "1s9qym58cjm8m8kg3zywvwai2i3adiq6sdayygk2zv72ry74ldai"; + }; + }) + + steam + + awscli2 + ]; + + systemd.user.services.laptop-keyboard = { + Unit = { + Description = "Swap caps+escape and alt+super, but only on the built-in laptop keyboard"; + After = [ "graphical-session-pre.target" ]; + PartOf = [ "graphical-session.target" ]; + }; + + Install = { WantedBy = [ "graphical-session.target" ]; }; + + Service = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = ( + "${pkgs.xorg.setxkbmap}/bin/setxkbmap " + + "-device ${laptopKeyboardId} " + + "-option caps:swapescape " + + "-option compose:ralt " + + "-option altwin:swap_alt_win" + ); + }; + }; + + xsession.windowManager.i3.config.keybindings.F9 = "exec lock"; + + xdg.mimeApps.defaultApplications."x-scheme-handler/tg" = + "telegramdesktop.desktop"; + + programs.zsh.shellAliases = { + "graph" = "curl -s localhost:6033/graph | dot -Tpng | feh -"; + }; +} diff --git a/users/grfn/system/home/modules/alacritty.nix b/users/grfn/system/home/modules/alacritty.nix new file mode 100644 index 000000000000..67d6638a31f8 --- /dev/null +++ b/users/grfn/system/home/modules/alacritty.nix @@ -0,0 +1,56 @@ +{ config, lib, pkgs, ... }: + +{ + programs.alacritty = { + enable = true; + settings = { + font.size = 6; + font.normal.family = "Meslo LGSDZ Nerd Font"; + + draw_bold_text_with_bright_colors = false; + + key_bindings = [ + { + key = "Escape"; + mods = "Control"; + action = "ToggleViMode"; + } + ]; + + colors = with import ../common/solarized.nix; rec { + # Default colors + primary = { + background = base3; + foreground = base00; + }; + + cursor = { + text = base3; + cursor = base00; + }; + + # Normal colors + normal = { + inherit red green yellow blue magenta cyan; + black = base02; + white = base2; + }; + + # Bright colors + # bright = normal; + bright = { + black = base03; + red = orange; + green = base01; + yellow = base00; + blue = base0; + magenta = violet; + cyan = base1; + white = base3; + }; + + vi_mode_cursor.cursor = red; + }; + }; + }; +} diff --git a/users/grfn/system/home/modules/alsi.nix b/users/grfn/system/home/modules/alsi.nix new file mode 100644 index 000000000000..c4a14e683c39 --- /dev/null +++ b/users/grfn/system/home/modules/alsi.nix @@ -0,0 +1,58 @@ +{ config, lib, pkgs, ... }: +{ + home.packages = [ config.lib.depot.third_party.alsi ]; + + xdg.configFile."alsi/alsi.logo" = { + source = ./nixos-logo.txt; + force = true; + }; + + xdg.configFile."alsi/alsi.conf" = { + force = true; + text = '' + #!${pkgs.perl}/bin/perl + + scalar { + ALSI_VERSION => "0.4.8", + COLORS_FILE => "/home/grfn/.config/alsi/alsi.colors", + DE_FILE => "/home/grfn/.config/alsi/alsi.de", + DEFAULT_COLOR_BOLD => "blue", + DEFAULT_COLOR_NORMAL => "blue", + DF_COMMAND => "df -Th -x sys -x tmpfs -x devtmpfs &>/dev/stdout", + GTK2_RC_FILE => "/home/grfn/.gtkrc-2.0", + GTK3_RC_FILE => "/home/grfn/.config/gtk-3.0/settings.ini", + LOGO_FILE => "/home/grfn/.config/alsi/alsi.logo", + OUTPUT_FILE => "/home/grfn/.config/alsi/alsi.output", + # PACKAGES_PATH => "/var/lib/pacman/local/", + PS_COMMAND => "ps -A", + USAGE_COLORS => 0, + USAGE_COLORS_BOLD => 0, + USAGE_PRECENT_GREEN => 50, + USAGE_PRECENT_RED => 100, + USAGE_PRECENT_YELLOW => 85, + USE_LOGO_FROM_FILE => 1, + USE_VALUES_COLOR => 0, + WM_FILE => "/home/grfn/.config/alsi/alsi.wm", + } + ''; + }; + + xdg.configFile."alsi/alsi.colors".text = '' + #!${pkgs.perl}/bin/perl + + # Colors for alsi + + scalar { + black => {normal => "\e[0;30m", bold => "\e[1;30m"}, + red => {normal => "\e[0;31m", bold => "\e[1;31m"}, + green => {normal => "\e[0;32m", bold => "\e[1;32m"}, + yellow => {normal => "\e[0;33m", bold => "\e[1;33m"}, + default => {normal => "\e[0;34m", bold => "\e[1;34m"}, + blue => {normal => "\e[0;34m", bold => "\e[1;34m"}, + purple => {normal => "\e[0;35m", bold => "\e[1;35m"}, + cyan => {normal => "\e[0;36m", bold => "\e[1;36m"}, + white => {normal => "\e[0;37m", bold => "\e[1;37m"}, + reset => "\e[0m", + } + ''; +} diff --git a/users/grfn/system/home/modules/common.nix b/users/grfn/system/home/modules/common.nix new file mode 100644 index 000000000000..86e4ba400e7c --- /dev/null +++ b/users/grfn/system/home/modules/common.nix @@ -0,0 +1,105 @@ +{ config, lib, pkgs, ... }: + +# Everything in here needs to work on linux or darwin + +{ + imports = [ + ../modules/shell.nix + ../modules/development.nix + ../modules/emacs.nix + ../modules/vim.nix + ../modules/tarsnap.nix + ../modules/twitter.nix + ../modules/lib/cloneRepo.nix + ]; + + programs.password-store.enable = true; + + grfn.impure.clonedRepos.passwordStore = { + github = "glittershark/pass"; + path = ".local/share/password-store"; + }; + + home.packages = with pkgs; [ + # System utilities + bat + htop + killall + bind + zip unzip + tree + ncat + bc + pv + + # Security + gnupg + keybase + openssl + + # Nix things + nixfmt + nix-prefetch-github + nix-review + cachix + (writeShellScriptBin "rebuild-mugwump" '' + set -eo pipefail + cd ~/code/depot + nix build -f . users.grfn.system.system.mugwumpSystem -o /tmp/mugwump + nix copy -f . users.grfn.system.system.mugwumpSystem \ + --to ssh://mugwump + system=$(readlink -ef /tmp/mugwump) + ssh mugwump sudo nix-env -p /nix/var/nix/profiles/system --set $system + ssh mugwump sudo $system/bin/switch-to-configuration switch + '') + (writeShellScriptBin "rebuild-home" '' + set -eo pipefail + cd ~/code/depot + nix build -f . users.grfn.system.home.$(hostname)Home -o /tmp/home + /tmp/home/activate + '') + ]; + + programs.ssh = { + enable = true; + + matchBlocks = { + "home" = { + host = "home.gws.fyi"; + forwardAgent = true; + }; + + "dobharchu" = { + host = "dobharchu"; + hostname = "172.16.0.4"; + forwardAgent = true; + user = "griffin"; + }; + + "cerberus" = { + host = "cerberus"; + hostname = "172.16.0.3"; + forwardAgent = true; + user = "griffin"; + }; + + "mugwump" = { + host = "mugwump"; + hostname = "172.16.0.5"; + forwardAgent = true; + }; + + "roswell" = { + host = "roswell"; + hostname = "18.223.118.13"; + forwardAgent = true; + }; + }; + }; + + programs.direnv = { + enable = true; + enableBashIntegration = true; + enableZshIntegration = true; + }; +} diff --git a/users/grfn/system/home/modules/development.nix b/users/grfn/system/home/modules/development.nix new file mode 100644 index 000000000000..1a5e912916f8 --- /dev/null +++ b/users/grfn/system/home/modules/development.nix @@ -0,0 +1,212 @@ +{ config, lib, pkgs, ... }: + +let + + clj2nix = pkgs.callPackage (pkgs.fetchFromGitHub { + owner = "hlolli"; + repo = "clj2nix"; + rev = "3ab3480a25e850b35d1f532a5e4e7b3202232383"; + sha256 = "1lry026mlpxp1j563qs13nhxf37i2zpl7lh0lgfdwc44afybqka6"; + }) {}; + + pg-dump-upsert = pkgs.buildGoModule rec { + pname = "pg-dump-upsert"; + version = "165258deaebded5e9b88f7a0acf3a4b7350e7bf4"; + + src = pkgs.fetchFromGitHub { + owner = "tomyl"; + repo = "pg-dump-upsert"; + rev = version; + sha256 = "1an4h8jjbj3r618ykjwk9brii4h9cxjqy47c4c8rivnvhimgf4wm"; + }; + + vendorSha256 = "1a5fx6mrv30cl46kswicd8lf5i5shn1fykchvbnbhdpgxhbz6qi4"; + }; + +in + +with lib; + +{ + imports = [ + ./lib/zshFunctions.nix + ./development/kube.nix + # TODO(grfn): agda build is broken in the nixpkgs checkout + # ./development/agda.nix + ./development/rust.nix + ]; + + home.packages = with pkgs; [ + jq + yq + gron + gitAndTools.hub + gitAndTools.tig + gitAndTools.gh + shellcheck + httpie + entr + gnumake + inetutils + tokei + jsonnet + ngrok + amber + + gdb + lldb + hyperfine + clang-tools_11 + (rr.overrideAttrs (_: rec { + version = "f25671d094edac8059cec56b98d7f10f2c740697"; + src = pkgs.fetchFromGitHub { + owner = "rr-debugger"; + repo = "rr"; + rev = version; + sha256 = "149s4mw8vl8d3nx15sfp62z0izp2dibz99k720j75rjnkwk2bq2z"; + fetchSubmodules = true; + }; + })) + + clj2nix + clojure + leiningen + clj-kondo + + pg-dump-upsert + + nodePackages.prettier + ] ++ optionals (stdenv.isLinux) [ + # TODO(grfn): replace with stable again once the current julia debacle + # is resolved upstream, see https://github.com/NixOS/nixpkgs/pull/121114 + julia_16-bin + valgrind + ]; + + programs.git = { + enable = true; + package = pkgs.gitFull; + userEmail = "root@gws.fyi"; + userName = "Griffin Smith"; + ignores = [ + "*.sw*" + ".classpath" + ".project" + ".settings/" + ".dir-locals.el" + ".stack-work-profiling" + ".projectile" + ]; + extraConfig = { + github.user = "glittershark"; + merge.conflictstyle = "diff3"; + rerere.enabled = "true"; + }; + + delta = { + enable = true; + options = { + syntax-theme = "Solarized (light)"; + hunk-style = "plain"; + commit-style = "box"; + }; + }; + }; + + home.file.".gdbinit".text = '' + set history filename ~/.gdb_history + set history save on + set history size unlimited + set history remove-duplicates unlimited + set history expansion on + ''; + + home.file.".psqlrc".text = '' + \set QUIET 1 + \timing + \set ON_ERROR_ROLLBACK interactive + \set VERBOSITY verbose + \x auto + \set PROMPT1 '%[%033[1m%]%M/%/%R%[%033[0m%]%# ' + \set PROMPT2 '...%# ' + \set HISTFILE ~/.psql_history- :DBNAME + \set HISTCONTROL ignoredups + \pset null [null] + \unset QUIET + ''; + + programs.readline = { + enable = true; + extraConfig = '' + set editing-mode vi + ''; + }; + + programs.zsh = { + shellAliases = { + # Git + "gwip" = "git add . && git commit -am wip"; + "gpr" = "g pull-request"; + "gcl" = "git clone"; + "grs" = "gr --soft"; + "grhh" = "grh HEAD"; + "grh" = "gr --hard"; + "gr" = "git reset"; + "gcb" = "gc -b"; + "gco" = "gc"; + "gcd" = "gc development"; + "gcm" = "gc master"; + "gcc" = "gc canon"; + "gc" = "git checkout"; + "gbg" = "git branch | grep"; + "gba" = "git branch -a"; + "gb" = "git branch"; + "gcv" = "git commit --verbose"; + "gci" = "git commit"; + "gm" = "git merge"; + "gdc" = "gd --cached"; + "gd" = "git diff"; + "gsl" = "git stash list"; + "gss" = "git show stash"; + "gsad" = "git stash drop"; + "gsa" = "git stash"; + "gst" = "gs"; + "gs" = "git status"; + "gg" = "gl --decorate --oneline --graph --date-order --all"; + "gl" = "git log"; + "gf" = "git fetch"; + "gur" = "gu --rebase"; + "gu" = "git pull"; + "gpf" = "gp -f"; + "gpa" = "gp --all"; + "gpu" = "git push -u origin \"$(git symbolic-ref --short HEAD)\""; + "gp" = "git push"; + "ganw" = "git diff -w --no-color | git apply --cached --ignore-whitespace"; + "ga" = "git add"; + "gnp" = "git --no-pager"; + "g" = "git"; + "git" = "hub"; + "grim" = "git fetch && git rebase -i --autostash origin/master"; + "grom" = "git fetch && git rebase --autostash origin/master"; + "groc" = "git fetch && git rebase --autostash origin/canon"; + "grc" = "git rebase --continue"; + "gcan" = "git commit --amend --no-edit"; + "grl" = "git reflog"; + + # Haskell + "crl" = "cabal repl"; + "cr" = "cabal run"; + "cnb" = "cabal new-build"; + "cob" = "cabal old-build"; + "cnr" = "cabal new-run"; + "cor" = "cabal old-run"; + "ho" = "hoogle"; + }; + + functions = { + gdelmerged = '' + git branch --merged | egrep -v 'master' | tr -d '+ ' | xargs git branch -d + ''; + }; + }; +} diff --git a/users/grfn/system/home/modules/development/agda.nix b/users/grfn/system/home/modules/development/agda.nix new file mode 100644 index 000000000000..afd22a306dc9 --- /dev/null +++ b/users/grfn/system/home/modules/development/agda.nix @@ -0,0 +1,58 @@ +{ config, lib, pkgs, ... }: + +let + agda-categories = with pkgs.agdaPackages; mkDerivation rec { + pname = "agda-categories"; + version = "2128fab"; + src = pkgs.fetchFromGitHub { + owner = "agda"; + repo = "agda-categories"; + rev = version; + sha256 = "08mc20qaz9vp5rhi60rh8wvjkg5aby3bgwwdhfnxha1663qf1q24"; + }; + + buildInputs = [ standard-library ]; + }; + +in + +{ + imports = [ + ../lib/cloneRepo.nix + ]; + + home.packages = with pkgs; [ + (pkgs.agda.withPackages + (p: with p; [ + p.standard-library + + ])) + ]; + + grfn.impure.clonedRepos = { + agda-stdlib = { + github = "agda/agda-stdlib"; + path = "code/agda-stdlib"; + }; + + agda-categories = { + github = "agda/agda-categories"; + path = "code/agda-categories"; + }; + + categories-examples = { + github = "agda/categories-examples"; + path = "code/categories-examples"; + }; + }; + + home.file.".agda/defaults".text = '' + standard-library + ''; + + home.file.".agda/libraries".text = '' + /home/grfn/code/agda-stdlib/standard-library.agda-lib + /home/grfn/code/agda-categories/agda-categories.agda-lib + ''; + +} diff --git a/users/grfn/system/home/modules/development/kube.nix b/users/grfn/system/home/modules/development/kube.nix new file mode 100644 index 000000000000..97ae4760d43b --- /dev/null +++ b/users/grfn/system/home/modules/development/kube.nix @@ -0,0 +1,34 @@ +{ config, lib, pkgs, ... }: +{ + home.packages = with pkgs; [ + kubectl + kubetail + sops + kubie + # pkgs-unstable.argocd # provided by urbos + ]; + + programs.zsh.shellAliases = { + "kc" = "kubectl"; + "kg" = "kc get"; + "kga" = "kc get --all-namespaces"; + "kpd" = "kubectl get pods"; + "kpa" = "kubectl get pods --all-namespaces"; + "klf" = "kubectl logs -f"; + "kdep" = "kubectl get deployments"; + "ked" = "kubectl edit deployment"; + "kpw" = "kubectl get pods -w"; + "kew" = "kubectl get events -w"; + "kdel" = "kubectl delete"; + "knw" = "kubectl get nodes -w"; + "kev" = "kubectl get events --sort-by='.metadata.creationTimestamp'"; + + "arsy" = "argocd app sync --prune"; + }; + + home.file.".kube/kubie.yaml".text = '' + shell: zsh + prompt: + zsh_use_rps1: true + ''; +} diff --git a/users/grfn/system/home/modules/development/readyset.nix b/users/grfn/system/home/modules/development/readyset.nix new file mode 100644 index 000000000000..19f44b770355 --- /dev/null +++ b/users/grfn/system/home/modules/development/readyset.nix @@ -0,0 +1,16 @@ +{ config, lib, pkgs, ... }: + +{ + imports = [ + ./rust.nix + ]; + + home.packages = with pkgs; [ + # This goes in $PATH so I can run it from rofi and parent to my WM + (writeShellScriptBin "dotclip" "xclip -out -selection clipboard | dot -Tpng | feh -") + ]; + + programs.zsh.shellAliases = { + "tf" = "terraform"; + }; +} diff --git a/users/grfn/system/home/modules/development/rust.nix b/users/grfn/system/home/modules/development/rust.nix new file mode 100644 index 000000000000..018b338bff2a --- /dev/null +++ b/users/grfn/system/home/modules/development/rust.nix @@ -0,0 +1,30 @@ +{ config, lib, pkgs, ... }: + + +{ + home.packages = with pkgs; [ + rustup + rust-analyzer + cargo-edit + cargo-expand + cargo-flamegraph + cargo-rr + cargo-udeps + cargo-bloat + sccache + evcxr + ]; + + programs.zsh.shellAliases = { + "cg" = "cargo"; + "cb" = "cargo build"; + "ct" = "cargo test"; + "ctw" = "fd -e rs | entr cargo test"; + "cch" = "cargo check"; + }; + + home.file.".cargo/config".text = '' + [build] + rustc-wrapper = "${pkgs.sccache}/bin/sccache" + ''; +} diff --git a/users/grfn/system/home/modules/emacs.nix b/users/grfn/system/home/modules/emacs.nix new file mode 100644 index 000000000000..6cc38bc7ab05 --- /dev/null +++ b/users/grfn/system/home/modules/emacs.nix @@ -0,0 +1,108 @@ +{ pkgs, lib, config, ... }: + +with lib; + +let + # doom-emacs = pkgs.callPackage (builtins.fetchTarball { + # url = https://github.com/vlaci/nix-doom-emacs/archive/master.tar.gz; + # }) { + # doomPrivateDir = ./doom.d; # Directory containing your config.el init.el + # # and packages.el files + # }; + + depot = config.lib.depot; + +in { + imports = [ + ./lib/cloneRepo.nix + ]; + + # home.packages = [ doom-emacs ]; + # home.file.".emacs.d/init.el".text = '' + # (load "default.el") + # ''; + # + + config = mkMerge [ + { + home.packages = with pkgs; [ + # LaTeX (for org export) + (pkgs.texlive.combine { + inherit (pkgs.texlive) + capt-of + collection-fontsrecommended + dvipng + fancyvrb + float + fncychap + framed + mathpartir + needspace + parskip + scheme-basic + semantic + tabulary + titlesec + ulem + upquote + varwidth + wrapfig + ; + }) + + ispell + + ripgrep + coreutils + fd + clang + gnutls + emacsPackages.telega + ]; + + programs.emacs = { + enable = true; + package = pkgs.emacsUnstable; + extraPackages = (epkgs: + (with epkgs; [ + tvlPackages.dottime + tvlPackages.tvl + vterm + telega + ]) + ); + }; + + grfn.impure.clonedRepos = { + orgClubhouse = { + github = "glittershark/org-clubhouse"; + path = "code/org-clubhouse"; + }; + + doomEmacs = { + github = "hlissner/doom-emacs"; + path = ".emacs.d"; + after = ["emacs.d"]; + onClone = "bin/doom install"; + }; + + "emacs.d" = { + github = "glittershark/emacs.d"; + path = ".doom.d"; + after = ["orgClubhouse"]; + }; + }; + + programs.zsh.shellAliases = { + "ec" = "emacsclient"; + }; + } + (mkIf pkgs.stdenv.isLinux { + # Notes + services.syncthing = { + enable = true; + tray = true; + }; + }) + ]; +} diff --git a/users/grfn/system/home/modules/email.nix b/users/grfn/system/home/modules/email.nix new file mode 100644 index 000000000000..0a3e58205ba6 --- /dev/null +++ b/users/grfn/system/home/modules/email.nix @@ -0,0 +1,87 @@ +{ lib, pkgs, config, ... }: + +with lib; + +let + + # from home-manager/modules/services/lieer.nix + escapeUnitName = name: + let + good = upperChars ++ lowerChars ++ stringToCharacters "0123456789-_"; + subst = c: if any (x: x == c) good then c else "-"; + in stringAsChars subst name; + + accounts = { + personal = { + primary = true; + address = "root@gws.fyi"; + aliases = [ "grfn@gws.fyi" ]; + passEntry = "root-gws-msmtp"; + }; + + work = { + address = "griffin@readyset.io"; + passEntry = "readyset/msmtp"; + }; + + }; + +in { + programs.lieer.enable = true; + programs.notmuch.enable = true; + services.lieer.enable = true; + programs.msmtp.enable = true; + + home.packages = with pkgs; [ + mu + msmtp + ]; + + systemd.user.services = mapAttrs' (name: account: { + name = escapeUnitName "lieer-${name}"; + value.Service = { + ExecStart = mkForce "${pkgs.writeShellScript "sync-${name}" '' + ${pkgs.gmailieer}/bin/gmi sync --path ~/mail/${name} + ''}"; + Environment = "NOTMUCH_CONFIG=${config.home.sessionVariables.NOTMUCH_CONFIG}"; + }; + + }) accounts; + + # xdg.configFile."notifymuch/notifymuch.cfg".text = generators.toINI {} { + # notifymuch = { + # query = "is:unread and is:important"; + # mail_client = ""; + # recency_interval_hours = "48"; + # hidden_tags = "inbox unread attachment replied sent encrypted signed"; + # }; + # }; + + accounts.email.maildirBasePath = "mail"; + accounts.email.accounts = mapAttrs (_: params@{ passEntry, ... }: { + realName = "Griffin Smith"; + passwordCommand = "pass ${passEntry}"; + + flavor = "gmail.com"; + + imapnotify = { + enable = true; + boxes = [ "Inbox" ]; + }; + + gpg = { + key = "0F11A989879E8BBBFDC1E23644EF5B5E861C09A7"; + signByDefault = true; + }; + + notmuch.enable = true; + lieer = { + enable = true; + sync = { + enable = true; + frequency = "*:*"; + }; + }; + msmtp.enable = true; + } // builtins.removeAttrs params ["passEntry"]) accounts; +} diff --git a/users/grfn/system/home/modules/firefox.nix b/users/grfn/system/home/modules/firefox.nix new file mode 100644 index 000000000000..c7e78685a5a3 --- /dev/null +++ b/users/grfn/system/home/modules/firefox.nix @@ -0,0 +1,22 @@ +{ config, lib, pkgs, ... }: + +{ + + xdg.mimeApps = rec { + enable = true; + defaultApplications = { + "text/html" = [ "firefox.desktop" ]; + "x-scheme-handler/http" = [ "firefox.desktop" ]; + "x-scheme-handler/https" = [ "firefox.desktop" ]; + "x-scheme-handler/ftp" = [ "firefox.desktop" ]; + "x-scheme-handler/chrome" = [ "firefox.desktop" ]; + "application/x-extension-htm" = [ "firefox.desktop" ]; + "application/x-extension-html" = [ "firefox.desktop" ]; + "application/x-extension-shtml" = [ "firefox.desktop" ]; + "application/xhtml+xml" = [ "firefox.desktop" ]; + "application/x-extension-xhtml" = [ "firefox.desktop" ]; + "application/x-extension-xht" = [ "firefox.desktop" ]; + }; + associations.added = defaultApplications; + }; +} diff --git a/users/grfn/system/home/modules/games.nix b/users/grfn/system/home/modules/games.nix new file mode 100644 index 000000000000..da54f99e5b39 --- /dev/null +++ b/users/grfn/system/home/modules/games.nix @@ -0,0 +1,59 @@ +{ config, lib, pkgs, ... }: + +with pkgs; +with lib; + +let + + df-orig = dwarf-fortress-packages.dwarf-fortress-original; + + df-full = (dwarf-fortress-packages.dwarf-fortress-full.override { + theme = null; + enableIntro = false; + enableFPS = true; + }); + + init = runCommand "init.txt" {} '' + substitute "${df-orig}/data/init/init.txt" $out \ + --replace "[INTRO:YES]" "[INTRO:NO]" \ + --replace "[VOLUME:255]" "[VOLUME:0]" \ + --replace "[FPS:NO]" "[FPS:YES]" + ''; + + d_init = runCommand "d_init.txt" {} '' + substitute "${df-orig}/data/init/d_init.txt" $out \ + --replace "[AUTOSAVE:NONE]" "[AUTOSAVE:SEASONAL]" \ + --replace "[AUTOSAVE_PAUSE:NO]" "[AUTOSAVE_PAUSE:YES]" \ + --replace "[INITIAL_SAVE:NO]" "[INITIAL_SAVE:YES]" \ + --replace "[EMBARK_WARNING_ALWAYS:NO]" "[EMBARK_WARNING_ALWAYS:YES]" \ + --replace "[VARIED_GROUND_TILES:YES]" "[VARIED_GROUND_TILES:NO]" \ + --replace "[SHOW_FLOW_AMOUNTS:NO]" "[SHOW_FLOW_AMOUNTS:YES]" + ''; + + df = runCommand "dwarf-fortress" {} '' + mkdir -p $out/bin + sed \ + -e '4icp -f ${init} "$DF_DIR/data/init/init.txt"' \ + -e '4icp -f ${d_init} "$DF_DIR/data/init/d_init.txt"' \ + < "${df-full}/bin/dwarf-fortress" >"$out/bin/dwarf-fortress" + + shopt -s extglob + ln -s ${df-full}/bin/!(dwarf-fortress) $out/bin + + chmod +x $out/bin/dwarf-fortress + ''; + +in mkMerge [ + { + home.packages = [ + crawl + xonotic + ]; + } + (mkIf stdenv.isLinux { + home.packages = [ + df + multimc + ]; + }) +] diff --git a/users/grfn/system/home/modules/i3.nix b/users/grfn/system/home/modules/i3.nix new file mode 100644 index 000000000000..df0859b46aef --- /dev/null +++ b/users/grfn/system/home/modules/i3.nix @@ -0,0 +1,366 @@ +{ config, lib, pkgs, ... }: +let + mod = "Mod4"; + solarized = import ../common/solarized.nix; + # TODO pull this out into lib + emacsclient = eval: pkgs.writeShellScript "emacsclient-eval" '' + msg=$(emacsclient --eval '${eval}' 2>&1) + echo "''${msg:1:-1}" + ''; + screenlayout = { + home = pkgs.writeShellScript "screenlayout_home.sh" '' + xrandr \ + --output eDP-1 --mode 1920x1200 --pos 0x2160 --rotate normal \ + --output DP-1 --off \ + --output DP-2 --off \ + --output DP-3 --mode 3840x2160 --pos 0x0 --rotate normal \ + --output DP-4 --off + ''; + }; + + inherit (builtins) map; + inherit (lib) mkMerge range; +in { + options = with lib; { + system.machine.wirelessInterface = mkOption { + description = '' + Name of the primary wireless interface. Used by i3status, etc. + ''; + default = "wlp3s0"; + type = types.str; + }; + + system.machine.i3FontSize = mkOption { + description = "Font size to use in i3 window decorations etc."; + default = 6; + type = types.int; + }; + }; + + config = + let decorationFont = "MesloLGSDZ ${toString config.system.machine.i3FontSize}"; in + { + home.packages = with pkgs; [ + rofi + rofi-pass + python38Packages.py3status + i3lock + i3status + dconf # for gtk + + # Screenshots + maim + + # GIFs + picom + peek + + (pkgs.writeShellScriptBin "lock" '' + playerctl pause + ${pkgs.i3lock}/bin/i3lock -c 222222 + '') + ]; + + xsession.scriptPath = ".xsession"; + + xsession.windowManager.i3 = { + enable = true; + config = { + modifier = mod; + keybindings = + mkMerge ( + (map + (n: { + "${mod}+${toString n}" = + "workspace ${toString n}"; + "${mod}+Shift+${toString n}" = + "move container to workspace ${toString n}"; + }) + (range 0 9)) + ++ [(rec { + "${mod}+h" = "focus left"; + "${mod}+j" = "focus down"; + "${mod}+k" = "focus up"; + "${mod}+l" = "focus right"; + "${mod}+semicolon" = "focus parent"; + + "${mod}+Shift+h" = "move left"; + "${mod}+Shift+j" = "move down"; + "${mod}+Shift+k" = "move up"; + "${mod}+Shift+l" = "move right"; + + "${mod}+Shift+x" = "kill"; + + "${mod}+Return" = "exec alacritty"; + + "${mod}+Shift+s" = "split h"; + "${mod}+Shift+v" = "split v"; + "${mod}+e" = "layout toggle split"; + "${mod}+w" = "layout tabbed"; + "${mod}+s" = "layout stacking"; + + "${mod}+f" = "fullscreen"; + + "${mod}+Shift+r" = "restart"; + + "${mod}+r" = "mode resize"; + + # Marks + "${mod}+Shift+m" = ''exec i3-input -F "mark %s" -l 1 -P 'Mark: ' ''; + "${mod}+m" = ''exec i3-input -F '[con_mark="%s"] focus' -l 1 -P 'Go to: ' ''; + + # Screenshots + "${mod}+q" = "exec \"maim | xclip -selection clipboard -t image/png\""; + "${mod}+Shift+q" = "exec \"maim -s | xclip -selection clipboard -t image/png\""; + "${mod}+Ctrl+q" = "exec ${pkgs.writeShellScript "peek.sh" '' + ${pkgs.picom}/bin/picom & + picom_pid=$! + ${pkgs.peek}/bin/peek || true + kill -SIGINT $picom_pid + ''}"; + + # Launching applications + "${mod}+u" = "exec ${pkgs.writeShellScript "rofi" '' + rofi \ + -modi 'combi' \ + -combi-modi "window,drun,ssh,run" \ + -font '${decorationFont}' \ + -show combi + ''}"; + + # Passwords + "${mod}+p" = "exec rofi-pass -font '${decorationFont}'"; + + # Media + "XF86AudioPlay" = "exec playerctl play-pause"; + "XF86AudioNext" = "exec playerctl next"; + "XF86AudioPrev" = "exec playerctl previous"; + "XF86AudioRaiseVolume" = "exec pulseaudio-ctl up"; + "XF86AudioLowerVolume" = "exec pulseaudio-ctl down"; + "XF86AudioMute" = "exec pulseaudio-ctl mute"; + + # Lock + Pause = "exec lock"; + + # Brightness + "XF86MonBrightnessDown" = "exec ${pkgs.brightnessctl}/bin/brightnessctl -q s 5%-"; + "XF86MonBrightnessUp" = "exec ${pkgs.brightnessctl}/bin/brightnessctl -q s 5%+"; + + # Sleep/hibernate + # "${mod}+Escape" = "exec systemctl suspend"; + # "${mod}+Shift+Escape" = "exec systemctl hibernate"; + + # Scratch buffer + "${mod}+minus" = "scratchpad show"; + "${mod}+Shift+minus" = "move scratchpad"; + "${mod}+space" = "focus mode_toggle"; + "${mod}+Shift+space" = "floating toggle"; + + # Screen Layout + "${mod}+Shift+t" = "exec xrandr --auto"; + "${mod}+t" = "exec ${screenlayout.home}"; + "${mod}+Ctrl+t" = "exec ${pkgs.writeShellScript "fix_term.sh" '' + xrandr --output eDP-1 --off && ${screenlayout.home} + ''}"; + + # Notifications + "${mod}+Shift+n" = "exec killall -SIGUSR1 .dunst-wrapped"; + "${mod}+n" = "exec killall -SIGUSR2 .dunst-wrapped"; + "Control+space" = "exec ${pkgs.dunst}/bin/dunstctl close"; + "Control+Shift+space" = "exec ${pkgs.dunst}/bin/dunstctl close-all"; + "Control+grave" = "exec ${pkgs.dunst}/bin/dunstctl history-pop"; + "Control+Shift+period" = "exec ${pkgs.dunst}/bin/dunstctl action"; + })]); + + fonts = [ decorationFont ]; + + colors = with solarized; rec { + focused = { + border = base01; + background = base01; + text = base3; + indicator = red; + childBorder = base02; + }; + focusedInactive = focused // { + border = base03; + background = base03; + # text = base1; + }; + unfocused = focusedInactive; + background = base03; + }; + + modes.resize = { + l = "resize shrink width 5 px or 5 ppt"; + k = "resize grow height 5 px or 5 ppt"; + j = "resize shrink height 5 px or 5 ppt"; + h = "resize grow width 5 px or 5 ppt"; + + Return = "mode \"default\""; + }; + + bars = [{ + statusCommand = + let i3status-conf = pkgs.writeText "i3status.conf" '' + general { + output_format = i3bar + colors = true + color_good = "#859900" + + interval = 1 + } + + order += "external_script current_task" + order += "external_script inbox" + order += "spotify" + order += "volume_status" + order += "wireless ${config.system.machine.wirelessInterface}" + # order += "ethernet enp3s0f0" + order += "cpu_usage" + order += "battery 0" + # order += "volume master" + order += "time" + order += "tztime utc" + + mpd { + format = "%artist - %album - %title" + } + + wireless ${config.system.machine.wirelessInterface} { + format_up = "W: (%quality - %essid - %bitrate) %ip" + format_down = "W: -" + } + + ethernet enp3s0f0 { + format_up = "E: %ip" + format_down = "E: -" + } + + battery 0 { + format = "%status %percentage" + path = "/sys/class/power_supply/BAT%d/uevent" + low_threshold = 10 + } + + cpu_usage { + format = "CPU: %usage" + } + + load { + format = "%5min" + } + + time { + format = " %a %h %d ⌚ %I:%M " + } + + spotify { + color_playing = "#fdf6e3" + color_paused = "#93a1a1" + format_stopped = "" + format_down = "" + format = "{title} - {artist} ({album})" + } + + external_script inbox { + script_path = '${emacsclient "(grfn/num-inbox-items-message)"}' + format = 'Inbox: {output}' + cache_timeout = 120 + color = "#93a1a1" + } + + external_script current_task { + script_path = '${emacsclient "(grfn/org-current-clocked-in-task-message)"}' + # format = '{output}' + cache_timeout = 60 + color = "#93a1a1" + } + + tztime utc { + timezone = "UTC" + format = " %H·%M " + } + + volume_status { + format = "☊ {percentage}" + format_muted = "☊ X" + # device = "default" + # mixer_idx = 0 + } + ''; + in "py3status -c ${i3status-conf}"; + fonts = [ decorationFont ]; + position = "top"; + colors = with solarized; rec { + background = base03; + statusline = base3; + separator = base1; + activeWorkspace = { + border = base03; + background = base1; + text = base3; + }; + focusedWorkspace = activeWorkspace; + inactiveWorkspace = activeWorkspace // { + background = base01; + }; + urgentWorkspace = activeWorkspace // { + background = red; + }; + }; + }]; + }; + }; + + services.dunst = { + enable = true; + settings = with solarized; { + global = { + font = "MesloLGSDZ ${toString (config.system.machine.i3FontSize * 1.5)}"; + allow_markup = true; + format = "<b>%s</b>\n%b"; + sort = true; + alignment = "left"; + geometry = "600x15-40+40"; + idle_threshold = 120; + separator_color = "frame"; + separator_height = 1; + word_wrap = true; + padding = 8; + horizontal_padding = 8; + max_icon_size = 45; + }; + + frame = { + width = 0; + color = "#aaaaaa"; + }; + + urgency_low = { + background = base03; + foreground = base3; + timeout = 5; + }; + + urgency_normal = { + background = base02; + foreground = base3; + timeout = 7; + }; + + urgency_critical = { + background = red; + foreground = base3; + timeout = 0; + }; + }; + }; + + gtk = { + enable = true; + iconTheme.name = "Adwaita"; + theme.name = "Adwaita"; + }; + }; +} diff --git a/users/grfn/system/home/modules/lib/cloneRepo.nix b/users/grfn/system/home/modules/lib/cloneRepo.nix new file mode 100644 index 000000000000..dc487dc6bd05 --- /dev/null +++ b/users/grfn/system/home/modules/lib/cloneRepo.nix @@ -0,0 +1,67 @@ +{ lib, config, ... }: +with lib; +{ + options = { + grfn.impure.clonedRepos = mkOption { + description = "Repositories to clone"; + default = {}; + type = with types; loaOf ( + let sm = submodule { + options = { + url = mkOption { + type = nullOr str; + description = "URL of repository to clone"; + default = null; + }; + + github = mkOption { + type = nullOr str; + description = "Github owner/repo of repository to clone"; + default = null; + }; + + path = mkOption { + type = str; + description = "Path to clone to"; + }; + + onClone = mkOption { + type = str; + description = '' + Shell command to run after cloning the repo for the first time. + Runs inside the repo itself. + ''; + default = ""; + }; + + after = mkOption { + type = listOf str; + description = "Activation hooks that this repository must be cloned after"; + default = []; + }; + }; + }; + in addCheck sm (cr: (! isNull cr.url || ! isNull cr.github)) + ); + }; + }; + + config = { + home.activation = + mapAttrs + (_: { + url, path, github, onClone, after, ... + }: + let repoURL = if isNull url then "git@github.com:${github}" else url; + in hm.dag.entryAfter (["writeBoundary"] ++ after) '' + $DRY_RUN_CMD mkdir -p $(dirname "${path}") + if [[ ! -d ${path} ]]; then + $DRY_RUN_CMD git clone "${repoURL}" "${path}" + pushd ${path} + $DRY_RUN_CMD ${onClone} + popd + fi + '') + config.grfn.impure.clonedRepos; + }; +} diff --git a/users/grfn/system/home/modules/lib/zshFunctions.nix b/users/grfn/system/home/modules/lib/zshFunctions.nix new file mode 100644 index 000000000000..7c39b3478cfd --- /dev/null +++ b/users/grfn/system/home/modules/lib/zshFunctions.nix @@ -0,0 +1,21 @@ +{ config, lib, pkgs, ... }: + +with lib; + +{ + options = { + programs.zsh.functions = mkOption { + description = "An attribute set that maps function names to their source"; + default = {}; + type = with types; attrsOf (either str path); + }; + }; + + config.programs.zsh.initExtra = concatStringsSep "\n" ( + mapAttrsToList (name: funSrc: '' + function ${name}() { + ${funSrc} + } + '') config.programs.zsh.functions + ); +} diff --git a/users/grfn/system/home/modules/nixos-logo.txt b/users/grfn/system/home/modules/nixos-logo.txt new file mode 100644 index 000000000000..d4b16b44f0bf --- /dev/null +++ b/users/grfn/system/home/modules/nixos-logo.txt @@ -0,0 +1,26 @@ + [38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m/ + [38;5;m,[38;5;068m([38;5;067m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;061m/[38;5;m( [38;5;m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;m# [38;5;m.[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;m# + [38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;061m/[38;5;061m/[38;5;m/ [38;5;m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;m. [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m/ + [38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;061m/[38;5;067m([38;5;061m/[38;5;061m/[38;5;m, [38;5;m/[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# + [38;5;m([38;5;067m([38;5;067m([38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/ [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m( + [38;5;m.[38;5;068m([38;5;068m([38;5;068m([38;5;068m([38;5;068m([38;5;068m([38;5;068m([38;5;068m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;061m([38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m#[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m([38;5;m( + [38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/[38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m.[38;5;m([38;5;067m([38;5;067m([38;5;m( + [38;5;m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;061m/[38;5;061m([38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/[38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;068m([38;5;067m([38;5;m( + [38;5;m([38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;074m#[38;5;m# [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m([38;5;067m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( + [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m/[38;5;m([38;5;067m([38;5;068m([38;5;067m([38;5;068m([38;5;067m([38;5;068m([38;5;067m([38;5;068m([38;5;m( + [38;5;m*[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m.[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m#[38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( + [38;5;m#[38;5;m#[38;5;m#[38;5;m%[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m([38;5;061m/[38;5;061m/[38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m([38;5;m([38;5;m([38;5;m([38;5;m([38;5;m([38;5;m([38;5;m([38;5;m([38;5;m( +[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m.[38;5;m#[38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( +[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m([38;5;m( [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( + [38;5;m#[38;5;m#[38;5;m#[38;5;m%[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m#[38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m. [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m/[38;5;m([38;5;m([38;5;m([38;5;m( + [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m( [38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/ + [38;5;m.[38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/ [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m, + [38;5;m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;m* [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m( [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m/ + [38;5;m.[38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;m# [38;5;m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m([38;5;m#[38;5;074m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m, + [38;5;m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m# [38;5;m([38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m% + [38;5;m.[38;5;m# [38;5;m([38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m/[38;5;061m([38;5;061m/[38;5;061m/[38;5;067m([38;5;m([38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;m% + [38;5;m([38;5;061m/[38;5;061m/[38;5;061m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m( [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m%[38;5;110m%[38;5;110m%[38;5;m( + [38;5;m/[38;5;061m/[38;5;067m([38;5;061m/[38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;m# + [38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( [38;5;m([38;5;067m([38;5;067m([38;5;067m([38;5;068m([38;5;067m([38;5;067m([38;5;067m([38;5;068m( [38;5;m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m/ + [38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;m( [38;5;m/[38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;067m([38;5;068m([38;5;067m([38;5;m( [38;5;m.[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m#[38;5;110m#[38;5;110m#[38;5;m% + [38;5;m([38;5;068m([38;5;067m([38;5;068m([38;5;067m([38;5;m( [38;5;m([38;5;068m([38;5;067m([38;5;068m([38;5;067m([38;5;068m([38;5;067m([38;5;068m([38;5;m( [38;5;m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;110m%[38;5;110m#[38;5;m/ diff --git a/users/grfn/system/home/modules/obs.nix b/users/grfn/system/home/modules/obs.nix new file mode 100644 index 000000000000..d1dade477ccc --- /dev/null +++ b/users/grfn/system/home/modules/obs.nix @@ -0,0 +1,66 @@ +{ config, lib, pkgs, ... }: + +with pkgs; + +let + libuiohook = stdenv.mkDerivation rec { + pname = "libuiohook"; + version = "1.1"; + src = fetchFromGitHub { + owner = "kwhat"; + repo = "libuiohook"; + rev = version; + sha256 = "1isfxn3cfrdqq22d3mlz2lzm4asf9gprs7ww2xy9c3j3srk9kd7r"; + }; + + preConfigure = '' + ./bootstrap.sh + ''; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ + libtool autoconf automake + x11 + xorg.libXtst + xorg.libXinerama + xorg.libxkbfile + libxkbcommon + ]; + }; + + obs-input-overlay = stdenv.mkDerivation rec { + pname = "obs-input-overlay"; + version = "4.8"; + src = fetchFromGitHub { + owner = "univrsal"; + repo = "input-overlay"; + rev = "v${version}"; + sha256 = "1dklg0dx9ijwyhgwcaqz859rbpaivmqxqvh9w3h4byrh5pnkz8bf"; + fetchSubmodules = true; + }; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ obs-studio libuiohook ]; + + postPatch = '' + sed -i CMakeLists.txt \ + -e '2iinclude(${obs-studio.src}/cmake/Modules/ObsHelpers.cmake)' \ + -e '2ifind_package(LibObs REQUIRED)' + ''; + + cmakeFlags = [ + "-Wno-dev" + ]; + }; +in +{ + home.packages = [ + obs-studio + obs-input-overlay + ]; + + xdg.configFile."obs-studio/plugins/input-overlay/bin/64bit/input-overlay.so".source = + "${obs-input-overlay}/lib/obs-plugins/input-overlay.so"; + xdg.configFile."obs-studio/plugins/input-overlay/data".source = + "${obs-input-overlay}/share/obs/obs-plugins/input-overlay"; +} diff --git a/users/grfn/system/home/modules/ptt.nix b/users/grfn/system/home/modules/ptt.nix new file mode 100644 index 000000000000..436c8f261797 --- /dev/null +++ b/users/grfn/system/home/modules/ptt.nix @@ -0,0 +1,44 @@ +{ config, lib, pkgs, ... }: + +let + + pttKeycode = "152"; + sourceID = "3"; + + mute = pkgs.writeShellScript "mute-mic" '' + xset -r ${pttKeycode} + ${pkgs.pulseaudio}/bin/pactl set-source-mute ${sourceID} 1 + ''; + + unmute = pkgs.writeShellScript "unmute-mic" '' + xset -r ${pttKeycode} + ${pkgs.pulseaudio}/bin/pactl set-source-mute ${sourceID} 0 + ''; + +in + +{ + home.packages = with pkgs; [ + xbindkeys + ]; + + + home.file.".xbindkeysrc.scm".text = '' + (xbindkey '("c:${pttKeycode}") "${unmute}") + (xbindkey '(release "c:${pttKeycode}") "${mute}") + ''; + + systemd.user.services."xbindkeys" = { + Unit = { + Description = "Keybind daemon for push-to-talk"; + After = [ "graphical-session-pre.target" ]; + PartOf = [ "graphical-session.target" ]; + }; + + Install = { WantedBy = [ "graphical-session.target" ]; }; + + Service = { + ExecStart = "${pkgs.xbindkeys}/bin/xbindkeys -n -v"; + }; + }; +} diff --git a/users/grfn/system/home/modules/pure.zsh-theme b/users/grfn/system/home/modules/pure.zsh-theme new file mode 100755 index 000000000000..b4776e81596d --- /dev/null +++ b/users/grfn/system/home/modules/pure.zsh-theme @@ -0,0 +1,151 @@ +#!/bin/zsh -f +# vim: ft=zsh: +# MIT License +# For my own and others sanity +# git: +# %b => current branch +# %a => current action (rebase/merge) +# prompt: +# %F => color dict +# %f => reset color +# %~ => current path +# %* => time +# %n => username +# %m => shortname host +# %(?..) => prompt conditional - %(condition.true.false) + +# turns seconds into human readable time +# 165392 => 1d 21h 56m 32s +prompt_pure_human_time() { + local tmp=$1 + local days=$(( tmp / 60 / 60 / 24 )) + local hours=$(( tmp / 60 / 60 % 24 )) + local minutes=$(( tmp / 60 % 60 )) + local seconds=$(( tmp % 60 )) + (( $days > 0 )) && echo -n "${days}d " + (( $hours > 0 )) && echo -n "${hours}h " + (( $minutes > 0 )) && echo -n "${minutes}m " + echo "${seconds}s" +} + +is_git_repo() { + command git rev-parse --is-inside-work-tree &>/dev/null + return $? +} + +# fastest possible way to check if repo is dirty +prompt_pure_git_dirty() { + # check if we're in a git repo + is_git_repo || return + # check if it's dirty + [[ "$PURE_GIT_UNTRACKED_DIRTY" == 0 ]] && local umode="-uno" || local umode="-unormal" + command test -n "$(git status --porcelain --ignore-submodules ${umode})" + + (($? == 0)) && echo '*' +} + +prompt_pure_git_wip() { + is_git_repo || return + local subject="$(command git show --pretty=%s --quiet HEAD 2>/dev/null)" + [ "$subject" == 'wip' ] && echo '[WIP]' +} + +# displays the exec time of the last command if set threshold was exceeded +prompt_pure_cmd_exec_time() { + local stop=$EPOCHSECONDS + local start=${cmd_timestamp:-$stop} + integer elapsed=$stop-$start + (($elapsed > ${PURE_CMD_MAX_EXEC_TIME:=5})) && prompt_pure_human_time $elapsed +} + +prompt_pure_preexec() { + cmd_timestamp=$EPOCHSECONDS + + # shows the current dir and executed command in the title when a process is active + print -Pn "\e]0;" + echo -nE "$PWD:t: $2" + print -Pn "\a" +} + +# string length ignoring ansi escapes +prompt_pure_string_length() { + echo ${#${(S%%)1//(\%([KF1]|)\{*\}|\%[Bbkf])}} +} + +prompt_pure_nix_info() { + local packages_info='' + if [[ -z $NIX_SHELL_PACKAGES ]]; then + packages_info='[nix-shell]' + else + packages_info="{ $NIX_SHELL_PACKAGES }" + fi + + case $IN_NIX_SHELL in + 'pure') + echo "$fg_bold[green][nix-shell] " + ;; + 'impure') + echo "$fg_bold[magenta][nix-shell] " + ;; + *) ;; + esac +} + +prompt_pure_precmd() { + # shows the full path in the title + print -Pn '\e]0;%~\a' + + # git info + vcs_info + + local prompt_pure_preprompt="\n$(prompt_pure_nix_info)$fg_bold[green]$prompt_pure_username%F{blue}%~%F{yellow}$vcs_info_msg_0_`prompt_pure_git_dirty` $fg_no_bold[red]`prompt_pure_git_wip`%f %F{yellow}`prompt_pure_cmd_exec_time`%f " + print -P $prompt_pure_preprompt + + # check async if there is anything to pull + # (( ${PURE_GIT_PULL:-1} )) && { + # # check if we're in a git repo + # command git rev-parse --is-inside-work-tree &>/dev/null && + # # make sure working tree is not $HOME + # [[ "$(command git rev-parse --show-toplevel)" != "$HOME" ]] && + # # check check if there is anything to pull + # command git fetch &>/dev/null && + # # check if there is an upstream configured for this branch + # command git rev-parse --abbrev-ref @'{u}' &>/dev/null && { + # local arrows='' + # (( $(command git rev-list --right-only --count HEAD...@'{u}' 2>/dev/null) > 0 )) && arrows='⇣' + # (( $(command git rev-list --left-only --count HEAD...@'{u}' 2>/dev/null) > 0 )) && arrows+='⇡' + # print -Pn "\e7\e[A\e[1G\e[`prompt_pure_string_length $prompt_pure_preprompt`C%F{cyan}${arrows}%f\e8" + # } + # } &! + + # reset value since `preexec` isn't always triggered + unset cmd_timestamp +} + + +prompt_pure_setup() { + # prevent percentage showing up + # if output doesn't end with a newline + export PROMPT_EOL_MARK='' + + prompt_opts=(cr subst percent) + + zmodload zsh/datetime + autoload -Uz add-zsh-hook + autoload -Uz vcs_info + + add-zsh-hook precmd prompt_pure_precmd + add-zsh-hook preexec prompt_pure_preexec + + zstyle ':vcs_info:*' enable git + zstyle ':vcs_info:git*' formats ' %b' + zstyle ':vcs_info:git*' actionformats ' %b|%a' + + # show username@host if logged in through SSH + [[ "$SSH_CONNECTION" != '' ]] && prompt_pure_username='%n@%m ' + + # prompt turns red if the previous command didn't exit with 0 + PROMPT='%(?.%F{green}.%F{red})❯%f ' +} + +prompt_pure_setup "$@" diff --git a/users/grfn/system/home/modules/rtlsdr.nix b/users/grfn/system/home/modules/rtlsdr.nix new file mode 100644 index 000000000000..a1c717617a62 --- /dev/null +++ b/users/grfn/system/home/modules/rtlsdr.nix @@ -0,0 +1,21 @@ +{ config, lib, pkgs, ... }: + +let + + nixpkgs-gnuradio = import (pkgs.fetchFromGitHub { + owner = "doronbehar"; + repo = "nixpkgs"; + rev = "712561aa5f10bfe6112a1726a912585612a70d1f"; + sha256 = "04yqflbwjcfl9vlplphpj82csqqz9k6m3nj1ybhwgmsc4by7vivl"; + }) {}; + +in + +{ + home.packages = with pkgs; [ + rtl-sdr + nixpkgs-gnuradio.gnuradio + nixpkgs-gnuradio.gnuradio.plugins.osmosdr + nixpkgs-gnuradio.gqrx + ]; +} diff --git a/users/grfn/system/home/modules/shell.nix b/users/grfn/system/home/modules/shell.nix new file mode 100644 index 000000000000..088fe9238c69 --- /dev/null +++ b/users/grfn/system/home/modules/shell.nix @@ -0,0 +1,184 @@ +{ config, lib, pkgs, ... }: +let + shellAliases = rec { + # NixOS stuff + hms = "home-manager switch"; + nor = "sudo nixos-rebuild switch"; + nrs = nor; + nrb = "sudo nixos-rebuild boot"; + ncg = "nix-collect-garbage"; + vihome = "vim ~/.config/nixpkgs/home.nix && home-manager switch"; + virc = "vim ~/code/system/home/modules/shell.nix && home-manager switch && source ~/.zshrc"; + visystem = "sudo vim /etc/nixos/configuration.nix && sudo nixos-rebuild switch"; + + # Nix + ns = "nix-shell"; + nb = "nix build -f ."; + nbl = "nix build -f . --builders ''"; # nix build local + lwo = "lorri watch --once"; + + # Docker and friends + "dcu" = "docker-compose up"; + "dcud" = "docker-compose up -d"; + "dc" = "docker-compose"; + "dcr" = "docker-compose restart"; + "dclf" = "docker-compose logs -f"; + "dck" = "docker"; + "dockerclean" = "dockercleancontainers && dockercleanimages"; + "dockercleanimages" = "docker images -a --no-trunc | grep none | awk '{print \$$3}' | xargs -L 1 -r docker rmi"; + "dockercleancontainers" = "docker ps -a --no-trunc| grep 'Exit' | awk '{print \$$1}' | xargs -L 1 -r docker rm"; + + # Directories + stck = "dirs -v"; + b= "cd ~1"; + ".." = "cd .."; + "..." = "cd ../.."; + "...." = "cd ../../.."; + "....." = "cd ../../../.."; + + # Aliases from old config + "http" = "http --style solarized"; + "grep" = "grep $GREP_OPTIONS"; + "bak" = "~/bin/backup.sh"; + "xmm" = "xmodmap ~/.Xmodmap"; + "asdflkj" = "asdf"; + "asdf" = "asdfghjkl"; + "asdfghjkl" = "echo \"Having some trouble?\""; + "ift" = "sudo iftop -i wlp3s0"; + "first" = "awk '{print \$$1}'"; + "cmt" = "git log --oneline | fzf-tmux | awk '{print \$$1}'"; + "workmon" = "xrandr --output DP-2 --pos 1440x900 --primary"; + "vi" = "vim"; + "adbdev" = "adb devices"; + "adbcon" = "adb connect $GNEX_IP"; + "mpalb" = "mpc search album"; + "mpart" = "mpc search artist"; + "mps" = "mpc search"; + "mpa" = "mpc add"; + "mpt" = "mpc toggle"; + "mpl" = "mpc playlist"; + "dsstore" = "find . -name '*.DS_Store' -type f -ls -delete"; + "df" = "df -h"; + "fs" = "stat -f '%z bytes'"; + "ll" = "ls -al"; + "la" = "ls -a"; + }; +in { + home.packages = with pkgs; [ + zsh + autojump + ntfy + ]; + + home.sessionVariables = { + EDITOR = "vim"; + LS_COLORS = "no=00:fi=00:di=01;34:ln=01;36:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arj=01;31:*.taz=01;31:*.lzh=01;31:*.zip=01;31:*.z=01;31:*.Z=01;31:*.gz=01;31:*.bz2=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.jpg=01;35:*.jpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.avi=01;35:*.fli=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.ogg=01;35:*.mp3=01;35:*.wav=01;35:"; + BROWSER = "firefox"; + BAT_THEME = "ansi-light"; + }; + + programs.bash = { + enable = true; + inherit shellAliases; + }; + + programs.zsh = { + enable = true; + enableAutosuggestions = true; + autocd = true; + + inherit shellAliases; + + history = rec { + save = 100000; + size = save; + }; + + oh-my-zsh = { + enable = true; + + plugins = [ + "battery" + "colorize" + "command-not-found" + "github" + "gitignore" + "postgres" + "systemd" + "themes" + "vi-mode" + ]; + + custom = "${pkgs.stdenv.mkDerivation { + name = "oh-my-zsh-custom"; + unpackPhase = ":"; + installPhase = '' + mkdir -p $out/themes + mkdir -p $out/custom/plugins + ln -s ${./pure.zsh-theme} $out/themes/pure.zsh-theme + ''; + }}"; + + theme = "pure"; + }; + + plugins = [{ + name = "pure-theme"; + src = pkgs.fetchFromGitHub { + owner = "sindresorhus"; + repo = "pure"; + rev = "0a92b02dd4172f6c64fdc9b81fe6cd4bddb0a23b"; + sha256 = "0l8jqhmmjn7p32hdjnv121xsjnqd2c0plhzgydv2yzrmqgyvx7cc"; + }; + }]; + + initExtraBeforeCompInit = '' + zstyle ':completion:*' completer _complete _ignored _correct _approximate + zstyle ':completion:*' matcher-list \'\' 'm:{[:lower:]}={[:upper:]} m:{[:lower:][:upper:]}={[:upper:][:lower:]} r:|[._- :]=** r:|=**' 'l:|=* r:|=*' + zstyle ':completion:*' max-errors 5 + zstyle ':completion:*' use-cache yes + zstyle ':completion::complete:grunt::options:' expire 1 + zstyle ':completion:*' prompt '%e errors' + # zstyle :compinstall filename '~/.zshrc' + autoload -Uz compinit + ''; + + initExtra = '' + source ${./zshrc} + source ${pkgs.fetchFromGitHub { + owner = "zsh-users"; + repo = "zsh-syntax-highlighting"; + rev = "7678a8a22780141617f809002eeccf054bf8f448"; + sha256 = "0xh4fbd54kvwwpqvabk8lpw7m80phxdzrd75q3y874jw0xx1a9q6"; + }}/zsh-syntax-highlighting.zsh + source ${pkgs.autojump}/share/autojump/autojump.zsh + source ${pkgs.fetchFromGitHub { + owner = "chisui"; + repo = "zsh-nix-shell"; + rev = "a65382a353eaee5a98f068c330947c032a1263bb"; + sha256 = "0l41ac5b7p8yyjvpfp438kw7zl9dblrpd7icjg1v3ig3xy87zv0n"; + }}/nix-shell.plugin.zsh + + eval "$(${pkgs.ntfy}/bin/ntfy shell-integration)" + + export RPS1="" + autoload -U promptinit; promptinit + prompt pure + + if [[ "$TERM" == "dumb" ]]; then + unsetopt zle + unsetopt prompt_cr + unsetopt prompt_subst + unfunction precmd + unfunction preexec + export PS1='$ ' + fi + ''; + }; + + programs.fzf = { + enable = true; + enableBashIntegration = true; + enableZshIntegration = true; + }; +} diff --git a/users/grfn/system/home/modules/tarsnap.nix b/users/grfn/system/home/modules/tarsnap.nix new file mode 100644 index 000000000000..4bff19910f05 --- /dev/null +++ b/users/grfn/system/home/modules/tarsnap.nix @@ -0,0 +1,64 @@ +{ config, lib, pkgs, ... }: + +{ + home.packages = with pkgs; [ + tarsnap + ]; + + home.file.".tarsnaprc".text = '' + ### Recommended options + + # Tarsnap cache directory + cachedir /home/grfn/.cache/tarsnap + + # Tarsnap key file + keyfile /home/grfn/.private/tarsnap.key + + # Don't archive files which have the nodump flag set. + nodump + + # Print statistics when creating or deleting archives. + print-stats + + # Create a checkpoint once per GB of uploaded data. + checkpoint-bytes 1G + + ### Commonly useful options + + # Use SI prefixes to make numbers printed by --print-stats more readable. + humanize-numbers + + ### Other options, not applicable to most systems + + # Aggressive network behaviour: Use multiple TCP connections when + # writing archives. Use of this option is recommended only in + # cases where TCP congestion control is known to be the limiting + # factor in upload performance. + #aggressive-networking + + # Exclude files and directories matching specified patterns. + # Only one file or directory per command; multiple "exclude" + # commands may be given. + #exclude + + # Include only files and directories matching specified patterns. + # Only one file or directory per command; multiple "include" + # commands may be given. + #include + + # Attempt to reduce tarsnap memory consumption. This option + # will slow down the process of creating archives, but may help + # on systems where the average size of files being backed up is + # less than 1 MB. + #lowmem + + # Try even harder to reduce tarsnap memory consumption. This can + # significantly slow down tarsnap, but reduces its memory usage + # by an additional factor of 2 beyond what the lowmem option does. + #verylowmem + + # Snapshot time. Use this option if you are backing up files + # from a filesystem snapshot rather than from a "live" filesystem. + #snaptime <file> + ''; +} diff --git a/users/grfn/system/home/modules/twitter.nix b/users/grfn/system/home/modules/twitter.nix new file mode 100644 index 000000000000..3cb2e90adc34 --- /dev/null +++ b/users/grfn/system/home/modules/twitter.nix @@ -0,0 +1,23 @@ +{ pkgs, lib, ... }: + +{ + home.packages = with pkgs; [ + t + ]; + + home.sessionVariables = { + TWITTER_WHOAMI = "glittershark1"; + }; + + programs.zsh = { + shellAliases = { + "mytl" = "t tl $TWITTER_WHOAMI"; + }; + + functions = { + favelast = "t fave $(t tl -l $1 | head -n1 | cut -d' ' -f1)"; + rtlast = "t rt $(t tl -l $1 | head -n1 | cut -d' ' -f1)"; + tthread = "t reply $(t tl -l $TWITTER_WHOAMI | head -n1 | cut -d' ' -f1) $@"; + }; + }; +} diff --git a/users/grfn/system/home/modules/vim.nix b/users/grfn/system/home/modules/vim.nix new file mode 100644 index 000000000000..b87cb09ad125 --- /dev/null +++ b/users/grfn/system/home/modules/vim.nix @@ -0,0 +1,48 @@ +{ config, pkgs, ... }: +{ + programs.neovim = { + enable = true; + viAlias = true; + vimAlias = true; + plugins = with pkgs.vimPlugins; [ + ctrlp + deoplete-nvim + syntastic + vim-abolish + vim-airline + vim-airline-themes + vim-bufferline + vim-closetag + # vim-colors-solarized + # solarized + (pkgs.vimUtils.buildVimPlugin { + pname = "vim-colors-solarized"; + version = "git"; + src = pkgs.fetchFromGitHub { + owner = "glittershark"; + repo = "vim-colors-solarized"; + rev = "4857c3221ec3f2693a45855154cb61a2cefb514d"; + sha256 = "0kqp5w14g7adaiinmixm7z3x4w74lv1lcgbqjbirx760f0wivf9y"; + }; + }) + vim-commentary + vim-dispatch + vim-endwise + vim-repeat + vim-fugitive + vim-markdown + vim-nix + vim-rhubarb + vim-sexp + vim-sexp-mappings-for-regular-people + vim-sleuth + vim-startify + vim-surround + vim-unimpaired + vinegar + ]; + extraConfig = '' + source ${./vimrc} + ''; + }; +} diff --git a/users/grfn/system/home/modules/vimrc b/users/grfn/system/home/modules/vimrc new file mode 100644 index 000000000000..3e33b5e2bee7 --- /dev/null +++ b/users/grfn/system/home/modules/vimrc @@ -0,0 +1,1121 @@ +" vim:set fdm=marker fmr={{{,}}} ts=2 sts=2 sw=2 expandtab: + + +" Basic Options {{{ +set nocompatible +set modeline +set modelines=10 +syntax enable +filetype plugin indent on +set ruler +set showcmd +set number +set incsearch +set smartcase +set ignorecase +set scrolloff=10 +set tabstop=4 +set shiftwidth=4 +set softtabstop=4 +set nosmartindent +set expandtab +set noerrorbells visualbell t_vb= +set laststatus=2 +set hidden +let mapleader = ',' +let maplocalleader = '\' +set undofile +" set undodir=~/.vim/undo +set wildignore=*.pyc,*.o,.git +set clipboard=unnamedplus +" set backupdir=$HOME/.vim/backup +" set directory=$HOME/.vim/tmp +set foldmarker={{{,}}} +set colorcolumn=+1 +set concealcursor= +set formatoptions+=j +set wildmenu +set wildmode=longest,list:full +set noincsearch +" }}} + +" GUI options {{{ +set go-=m +set go-=T +set go-=r +set go-=L +set go-=e +set guifont=Meslo\ LG\ S\ DZ\ 9 +" }}} + +" Colors {{{ +" set t_Co=256 + +fu! ReverseBackground() + if &bg=="light" + se bg=dark + else + se bg=light + endif +endf +com! BgToggle call ReverseBackground() +nm <F12> :BgToggle<CR> + +set background=light +colorscheme solarized +" }}} + +" --------------------------------------------------------------------------- + +" CtrlP {{{ +let g:ctrlp_custom_ignore = { + \ 'dir': '(node_modules|target)' + \ } +let g:ctrlp_max_files = 0 +let g:ctrlp_max_depth = 100 +" }}} + +" YouCompleteMe {{{ +let g:ycm_semantic_triggers = { + \ 'c' : ['->', '.'], + \ 'objc' : ['->', '.'], + \ 'ocaml' : ['.', '#'], + \ 'cpp,objcpp' : ['->', '.', '::'], + \ 'perl' : ['->'], + \ 'php' : ['->', '::'], + \ 'cs,java,javascript,d,python,perl6,scala,vb,elixir,go' : ['.'], + \ 'vim' : ['re![_a-zA-Z]+[_\w]*\.'], + \ 'lua' : ['.', ':'], + \ 'erlang' : [':'], + \ 'clojure' : [], + \ 'haskell' : ['re!.*', '.', ' ', '('] + \ } + " \ 'haskell' : ['.', '(', ' '] + " \ 'ruby' : ['.', '::'], + " \ 'clojure' : ['(', '.', '/', '['] +" }}} + +" Neocomplete {{{ +if !has('nvim') + " Use neocomplete. + let g:neocomplete#enable_at_startup = 1 + " Use smartcase. + let g:neocomplete#enable_smart_case = 1 + " Set minimum syntax keyword length. + let g:neocomplete#sources#syntax#min_keyword_length = 3 + let g:neocomplete#lock_buffer_name_pattern = '\*ku\*' + + " Define dictionary. + " let g:neocomplete#sources#dictionary#dictionaries = { + " \ 'default' : '', + " \ 'vimshell' : $HOME.'/.vimshell_hist', + " \ 'scheme' : $HOME.'/.gosh_completions' + " \ } + + " Define keyword. + if !exists('g:neocomplete#keyword_patterns') + let g:neocomplete#keyword_patterns = {} + endif + let g:neocomplete#keyword_patterns['default'] = '\h\w*' + + " Plugin key-mappings. + inoremap <expr><C-g> neocomplete#undo_completion() + inoremap <expr><C-l> neocomplete#complete_common_string() + + " Recommended key-mappings. + " <CR>: close popup and save indent. + inoremap <silent> <CR> <C-r>=<SID>my_cr_function()<CR> + function! s:my_cr_function() + return (pumvisible() ? "\<C-y>" : "" ) . "\<CR>" + " For no inserting <CR> key. + "return pumvisible() ? "\<C-y>" : "\<CR>" + endfunction + " <TAB>: completion. + inoremap <expr><TAB> pumvisible() ? "\<C-n>" : "\<TAB>" + " <C-h>, <BS>: close popup and delete backword char. + inoremap <expr><C-h> neocomplete#smart_close_popup()."\<C-h>" + inoremap <expr><BS> neocomplete#smart_close_popup()."\<C-h>" + " Close popup by <Space>. + "inoremap <expr><Space> pumvisible() ? "\<C-y>" : "\<Space>" + + " AutoComplPop like behavior. + "let g:neocomplete#enable_auto_select = 1 + + " Shell like behavior(not recommended). + "set completeopt+=longest + "let g:neocomplete#enable_auto_select = 1 + "let g:neocomplete#disable_auto_complete = 1 + "inoremap <expr><TAB> pumvisible() ? "\<Down>" : "\<C-x>\<C-u>" + + " Enable omni completion. + " autocmd FileType css setlocal omnifunc=csscomplete#CompleteCSS + " autocmd FileType html,markdown setlocal omnifunc=htmlcomplete#CompleteTags + " autocmd FileType javascript setlocal omnifunc=javascriptcomplete#CompleteJS + " autocmd FileType python setlocal omnifunc=pythoncomplete#Complete + " autocmd FileType xml setlocal omnifunc=xmlcomplete#CompleteTags + + " Enable heavy omni completion. + if !exists('g:neocomplete#sources#omni#input_patterns') + let g:neocomplete#sources#omni#input_patterns = {} + endif +endif +" }}} + +" Deoplete {{{ +if has('nvim') + let g:deoplete#enable_at_startup = 1 + + inoremap <silent> <CR> <C-r>=<SID>my_cr_function()<CR> + function! s:my_cr_function() + return (pumvisible() ? "\<C-y>" : "" ) . "\<CR>" + " For no inserting <CR> key. + "return pumvisible() ? "\<C-y>" : "\<CR>" + endfunction + " <TAB>: completion. + inoremap <expr><TAB> pumvisible() ? "\<C-n>" : "\<TAB>" + inoremap <expr><S-TAB> pumvisible() ? "\<C-p>" : "\<TAB>" +endif +" }}} + +" Neovim Terminal mode {{{ +if has('nvim') + tnoremap <Esc> <C-\><C-n> + nnoremap \\ :tabedit term://zsh<CR> + nnoremap q\ :call <SID>OpenRepl()<CR> + + if !exists('g:repl_size') + let g:repl_size=9 + endif + + function! s:OpenRepl() " {{{ + " Check if buffer exists and is open + if exists('s:repl_bufname') && bufexists(s:repl_bufname) && bufwinnr(s:repl_bufname) >=? 0 + " If so, just switch to it + execute bufwinnr(s:repl_bufname) . 'wincmd' 'w' + norm i + return + endif + + if !exists('b:console') + let b:console=$SHELL + endif + + let l:console_cmd = b:console + + execute 'bot' g:repl_size . 'new' + set winfixheight nobuflisted + call termopen(l:console_cmd) + let s:repl_bufname = bufname('%') + norm i + endfunction " }}} +endif +" }}} + +" Tagbar options {{{ +let g:tagbar_autoclose = 1 +let g:tagbar_autofocus = 1 +let g:tagbar_compact = 1 +" }}} + +" delimitMate options {{{ +let g:delimitMate_expand_cr = 1 +" }}} + +" UltiSnips options {{{ +let g:UltiSnipsExpandTrigger = '<c-j>' + "g:UltiSnipsJumpForwardTrigger <c-j> + "g:UltiSnipsJumpBackwardTrigger <c-k> +" }}} + +" VDebug Options {{{ +let g:vdebug_options = {'server': '192.168.56.1'} +" }}} + +" Statusline {{{ +let g:airline_powerline_fonts=1 + +if !exists('g:airline_symbols') + let g:airline_symbols = {} +endif +let g:airline_symbols.space = "\ua0" + +let g:airline#extensions#tagbar#flags = 'f' +let g:airline#extensions#tabline#enabled = 1 +let g:airline#extensions#tabline#show_buffers = 0 +let g:airline#extensions#tabline#show_tabs = 1 +let g:airline#extensions#tabline#tab_min_count = 2 +let g:airline#extensions#tmuxline#enabled = 0 + +let g:tmuxline_theme = 'airline' +let g:tmuxline_preset = 'full' + +"set statusline= +"set statusline+=%2*[%n%H%M%R%W]%*\ " flags and buf no +"set statusline+=%-40f%<\ " path +"set statusline+=%=%40{fugitive#statusline()}\ " Vim status +"set statusline+=%1*%y%*%*\ " file type +"set statusline+=%10((%l,%c)%)\ " line and column +"set statusline+=%P " percentage of file +" }}} + +" Code review mode {{{ +fun! GetFontName() + return substitute(&guifont, '^\(.\{-}\)[0-9]*$', '\1', '') +endfun + +fun! <SID>CodeReviewMode() + let &guifont = GetFontName() . ' 15' +endfun +com! CodeReviewMode call <SID>CodeReviewMode() +" }}} + +" Syntastic {{{ +let g:syntastic_enable_signs = 0 + +" Python {{{ +let g:syntastic_python_checkers = ['flake8'] +let g:syntastic_python_flake8_post_args = "--ignore=E101,E223,E224,E301,E302,E303,E501,E701,W,F401,E111,E261" + +" }}} +" Javascript {{{ +let g:syntastic_javascript_checkers = ['eslint'] +let g:flow#autoclose = 1 +let g:flow#enable = 1 + +" augroup syntastic_javascript_jsx +" autocmd! +" autocmd BufReadPre,BufNewFile *.js +" autocmd BufReadPre,BufNewFile *.jsx +" \ let g:syntastic_javascript_checkers = ['jsxhint'] +" augroup END + +" }}} +" Haml {{{ +let g:syntastic_haml_checkers = ['haml_lint'] + +" }}} +" Html {{{ +let g:syntastic_html_checkers = [] + +" }}} +" Ruby {{{ +let g:syntastic_ruby_checkers = ['rubocop'] +" }}} +" SASS/SCSS {{{ +let g:syntastic_scss_checkers = ['scss_lint'] +" }}} +" Haskell {{{ +" let g:syntastic_haskell_checkers = ['ghc-mod'] +" }}} +" Elixir {{{ +let g:syntastic_elixir_checkers = ['elixir'] +let g:syntastic_enable_elixir_checker = 1 +" }}} +" }}} + +" Bufferline {{{ +let g:bufferline_echo=0 +" }}} + +" Eclim {{{ +let g:EclimCompletionMethod = 'omnifunc' +augroup eclim + au! + au FileType java call <SID>JavaSetup() + au FileType java set textwidth=120 +augroup END + +function! s:JavaSetup() abort + noremap <C-I> :JavaImport<CR> + nnoremap K :JavaDocPreview<CR> + nnoremap ]d :JavaSearchContext<CR> + nnoremap [d :JavaSearchContext<CR> + nnoremap g<CR> :JUnit<CR> + nnoremap g\ :Mvn test<CR> +endfunction +" }}} + +" Signify options {{{ +let g:signify_mapping_next_hunk = ']h' +let g:signify_mapping_prev_hunk = '[h' +let g:signify_vcs_list = ['git'] +let g:signify_sign_change = '~' +let g:signify_sign_delete = '-' +" }}} + +" Simplenote {{{ +let g:SimplenoteFiletype = 'markdown' +let g:SimplenoteSortOrder = 'pinned,modifydate,tagged,createdate' +let g:SimplenoteVertical = 1 + +nnoremap <Leader>nn :Simplenote -n<CR> +nnoremap <Leader>nl :Simplenote -l<CR> +nnoremap <Leader>nw :Simplenote -l work<CR> +nnoremap <Leader>nt :Simplenote -t<CR> +" }}} + +" Emmet {{{ +" Expand abbreviation +let g:user_emmet_leader_key = '<C-y>' +" }}} + +" Startify {{{ +let g:startify_bookmarks=[ '~/.vimrc', '~/.zshrc' ] +" }}} + +" Abolish {{{ +let g:abolish_save_file = expand('~/.vim/after/plugin/abolish.vim') +" }}} + +" Rails projections {{{ + +if !exists('g:rails_projections') + let g:rails_projections = {} +endif + +call extend(g:rails_projections, { + \ "config/routes.rb": { "command": "routes" }, + \ "config/structure.sql": { "command": "structure" } + \ }, 'keep') + +if !exists('g:rails_gem_projections') + let g:rails_gem_projections = {} +endif + +call extend(g:rails_gem_projections, { + \ "active_model_serializers": { + \ "app/serializers/*_serializer.rb": { + \ "command": "serializer", + \ "template": "class %SSerializer < ActiveModel::Serializer\nend", + \ "affinity": "model"}}, + \ "react-rails": { + \ "app/assets/javascripts/components/*.jsx": { + \ "command": "component", + \ "template": "var %S = window.%S = React.createClass({\n render: function() {\n }\n});", + \ "alternate": "spec/javascripts/components/%s_spec.jsx" }, + \ "spec/javascripts/components/*_spec.jsx": { + \ "alternate": "app/assets/javascripts/components/{}.jsx" }}, + \ "rspec": { + \ "spec/**/support/*.rb": { + \ "command": "support"}}, + \ "cucumber": { + \ "features/*.feature": { + \ "command": "feature", + \ "template": "Feature: %h"}, + \ "features/support/*.rb": { + \ "command": "support"}, + \ "features/support/env.rb": { + \ "command": "support"}, + \ "features/step_definitions/*_steps.rb": { + \ "command": "steps"}}, + \ "carrierwave": { + \ "app/uploaders/*_uploader.rb": { + \ "command": "uploader", + \ "template": "class %SUploader < CarrierWave::Uploader::Base\nend"}}, + \ "draper": { + \ "app/decorators/*_decorator.rb": { + \ "command": "decorator", + \ "affinity": "model", + \ "template": "class %SDecorator < Draper::Decorator\nend"}}, + \ "fabrication": { + \ "spec/fabricators/*_fabricator.rb": { + \ "command": ["fabricator", "factory"], + \ "alternate": "app/models/%s.rb", + \ "related": "db/schema.rb#%p", + \ "test": "spec/models/%s_spec.rb", + \ "template": "Fabricator :%s do\nend", + \ "affinity": "model"}}, + \ "factory_girl": { + \ "spec/factories/*.rb": { + \ "command": "factory", + \ "alternate": "app/models/%i.rb", + \ "related": "db/structure.sql#%s", + \ "test": "spec/models/%s_spec.rb", + \ "template": "FactoryGirl.define do\n factory :%i do\n end\nend", + \ "affinity": "model"}, + \ "spec/factories.rb": { + \ "command": "factory"}, + \ "test/factories.rb": { + \ "command": "factory"}} + \ }, 'keep') +" }}} + +" Other projections {{{ +let g:projectionist_heuristics = { + \ "config.ru&docker-compose.yml&app/&config/&OWNERS": { + \ "app/jobs/*.rb": { + \ "type": "job", + \ "alternate": "spec/jobs/{}_spec.rb" + \ }, + \ "app/models/*.rb": { + \ "type": "model", + \ "alternate": "spec/models/{}_spec.rb" + \ }, + \ "app/resources/*_resource.rb": { + \ "type": "resource", + \ "alternate": "spec/resources/{}_resource_spec.rb" + \ }, + \ "config/*.yml": { + \ "type": "config" + \ }, + \ "spec/*_spec.rb": { + \ "type": "spec", + \ "alternate": "app/{}.rb" + \ }, + \ "spec/factories/*.rb": { + \ "type": "factory", + \ } + \ }, + \ "svc-gateway.cabal": { + \ "src/*.hs": { + \ "type": "src", + \ "alternate": "test/{}Spec.hs" + \ }, + \ "test/*Spec.hs": { + \ "type": "spec", + \ "alternate": "src/{}.hs", + \ "template": [ + \ "module Gateway.Resource.HierarchySpec (main, spec) where", + \ "", + \ "import Prelude", + \ "import Test.Hspec", + \ "import Data.Aeson", + \ "", + \ "import Gateway.Resource.Hierarchy", + \ "", + \ "main :: IO ()", + \ "main = hspec spec", + \ "", + \ "spec :: Spec", + \ "spec = do", + \ " describe \"something\" $ undefined" + \ ] + \ }, + \ "svc-gateway.cabal": { + \ "type": "cabal" + \ } + \ }, + \ "package.json&.flowconfig": { + \ "src/*.*": { + \ "type": "src", + \ "alternate": "test/{}_spec.js" + \ } + \ }, + \ "pom.xml&src/main/clj/|src/main/cljs": { + \ "*": { + \ "start": "USE_NREPL=1 bin/run -m elephant.dev-system" , + \ "connect": "nrepl://localhost:5554", + \ "piggieback": "(figwheel-sidecar.repl-api/repl-env)" + \ }, + \ "pom.xml": { "type": "pom" }, + \ "src/main/clj/*.clj": { + \ "alternate": "src/test/clj/{}_test.clj", + \ "template": ["(ns {dot|hyphenate})"] + \ }, + \ "src/test/clj/*_test.clj": { + \ "alternate": "src/main/clj/{}.clj", + \ "dispatch": ":RunTests {dot|hyphenate}-test", + \ "template": ["(ns {dot|hyphenate}-test", + \ " (:require [clojure.test :refer :all]))"] + \ }, + \ "src/main/cljs/*.cljs": { + \ "alternate": "src/test/cljs/{}_test.cljs" + \ }, + \ "src/main/cljs/*_test.cljs": { + \ "alternate": "src/main/cljs/{}.cljs", + \ "dispatch": ":RunTests {dot|hyphenate}-test" + \ }, + \ "src/main/clj/*.cljc": { + \ "alternate": "src/test/clj/{}_test.cljc" + \ }, + \ "src/main/clj/*_test.cljc": { + \ "alternate": "src/test/clj/{}.cljc", + \ "dispatch": ":RunTests {dot|hyphenate}-test" + \ } + \ }} +" }}} + +" AutoPairs {{{ +let g:AutoPairsCenterLine = 0 +" }}} + +" Filetypes {{{ + +" Python {{{ +aug Python + au! + au FileType python set tabstop=4 shiftwidth=4 softtabstop=4 expandtab +aug END +let g:python_highlight_all=1 +" }}} + +" PHP {{{ +aug PHP + au! + "au FileType php setlocal fdm=marker fmr={{{,}}} +aug END " }}} + +" Mail {{{ +aug Mail + au FileType mail setlocal spell +aug END " }}} + +" Haskell {{{ +let g:haskell_conceal_wide = 1 +let g:haskellmode_completion_ghc = 0 +let g:necoghc_enable_detailed_browse = 1 + +augroup Haskell + autocmd! + autocmd FileType haskell setlocal textwidth=110 shiftwidth=2 + autocmd FileType haskell setlocal omnifunc=necoghc#omnifunc + autocmd FileType haskell call <SID>HaskellSetup() + autocmd FileType haskell setlocal keywordprg=hoogle\ -cie +augroup END + +function! s:HaskellSetup() + set sw=4 + " compiler cabal + " let b:start='cabal run' + " let b:console='cabal repl' + " let b:dispatch='cabal test' + compiler stack + let b:start='stack run' + let b:console='stack ghci' + let b:dispatch='stack test' + nnoremap <buffer> gy :HdevtoolsType<CR> + nnoremap <buffer> yu :HdevtoolsClear<CR> +endfunction +" }}} + +" Ruby {{{ + +function! s:RSpecSyntax() + syn keyword rspecMethod describe context it its specify shared_context + \ shared_examples shared_examples_for shared_context include_examples + \ include_context it_should_behave_like it_behaves_like before after + \ around fixtures controller_name helper_name scenario feature + \ background given described_class + syn match rspecMethod '\<let\>!\=' + syn match rspecMethod '\<subject\>!\=' + syn keyword rspecMethod violated pending expect expect_any_instance_of allow + \ allow_any_instance_of double instance_double mock mock_model + \ stub_model xit + syn match rspecMethod '\.\@<!\<stub\>!\@!' + + call s:RSpecHiDefaults() +endfunction + +function! s:RSpecHiDefaults() + hi def link rspecMethod rubyFunction +endfunction + +augroup Ruby + au! + " au FileType ruby let b:surround_114 = "\\(module|class,def,if,unless,case,while,until,begin,do) \r end" + " au FileType ruby set fdm=syntax + au FileType ruby set tw=110 + au FileType ruby set omnifunc= + au FileType ruby nnoremap <buffer> gy orequire 'pry'; binding.pry<ESC>^ + au FileType ruby nnoremap <buffer> gY Orequire 'pry'; binding.pry<ESC>^ + au FileType ruby nnoremap <buffer> yu :g/require 'pry'; binding.pry/d<CR> + au BufNewFile,BufRead *_spec.rb call <SID>RSpecSyntax() +augroup END + +let ruby_operators = 1 +let ruby_space_errors = 1 + +let g:rubycomplete_rails = 1 +command! -range ConvertHashSyntax <line1>,<line2>s/:(\S{-})(\s{-})=> /\1:\2/ +" }}} + +" Clojure {{{ + +aug Clojure + au! + autocmd FileType clojure nnoremap <C-S> :Slamhound<CR> + autocmd FileType clojure nnoremap <silent> gr :w <bar> Require <bar> e<CR> + let g:clojure_align_multiline_strings = 1 + let g:clojure_fuzzy_indent_patterns = + \ ['^with', '^def', '^let', '^fact'] + let g:clojure_special_indent_words = + \ 'deftype,defrecord,reify,proxy,extend-type,extend-protocol,letfn,html' + + autocmd FileType clojure setlocal textwidth=80 + autocmd FileType clojure setlocal lispwords+=GET,POST,PATCH,PUT,DELETE | + \ setlocal lispwords+=context,select + autocmd BufNewFile,BufReadPost *.cljx setfiletype clojure + autocmd BufNewFile,BufReadPost *.cljx setlocal omnifunc= + autocmd BufNewFile,BufReadPost *.cljs setlocal omnifunc= + autocmd FileType clojure call <SID>TangentInit() + autocmd FileType clojure call <SID>sexp_mappings() + autocmd BufRead *.cljc ClojureHighlightReferences + autocmd FileType clojure let b:AutoPairs = { + \ '"': '"', + \ '{': '}', + \ '(': ')', + \ '[': ']'} + " Don't auto-pair quote reader macros + " \'`': '`', + " \ '''': '''', + + autocmd User ProjectionistActivate call s:projectionist_connect() + + function! s:projectionist_connect() abort + let connected = !empty(fireplace#path()) + if !connected + for [root, value] in projectionist#query('connect') + try + silent execute "FireplaceConnect" value root + let connected = 1 + break + catch /.*Connection refused.*/ + endtry + endfor + endif + + " if connected && exists(':Piggieback') + " for [root, value] in projectionist#query('piggieback') + " silent execute "Piggieback" value + " break + " endfor + " endif + endfunction + + " autocmd BufNewFile,BufReadPost *.cljx setlocal omnifunc= + " autocmd BufNewFile,BufReadPost *.cljs setlocal omnifunc= + + autocmd FileType clojure let b:console='lein repl' + autocmd FileType clojure call <SID>ClojureMaps() + + function! s:ClojureMaps() abort + nnoremap <silent> <buffer> [m :call search('^(def', 'Wzb')<CR> + nnoremap <silent> <buffer> ]m :call search('^(def', 'Wz')<CR> + endfunction + + command! Scratch call <SID>OpenScratch() + autocmd FileType clojure nnoremap <buffer> \s :Scratch<CR> + + let g:scratch_buffer_name = 'SCRATCH' + + function! s:OpenScratch() + if bufwinnr(g:scratch_buffer_name) > 0 + execute bufwinnr(g:scratch_buffer_name) . 'wincmd' 'w' + return + endif + + vsplit SCRATCH + set buftype=nofile + set filetype=clojure + let b:scratch = 1 + endfunction +aug END + +function! s:sexp_mappings() abort + if !exists('g:sexp_loaded') + return + endif + + nmap <buffer> cfo <Plug>(sexp_raise_list) + nmap <buffer> cfO <Plug>(sexp_raise_element) + nmap <buffer> cfe <Plug>(sexp_raise_element) +endfunction + +function! s:TangentInit() abort + set textwidth=80 + command! TReset call fireplace#session_eval('(user/reset)') + command! TGo call fireplace#session_eval('(user/go)') + command! TMigrate call fireplace#session_eval('(user/migrate)') + command! TRollback call fireplace#session_eval('(user/rollback)') + nnoremap g\ :TReset<CR> +endfunction + +" }}} + +" Go {{{ + +let g:go_highlight_functions = 1 +let g:go_highlight_methods = 1 +let g:go_highlight_structs = 1 +let g:go_highlight_operators = 1 +let g:go_highlight_build_constraints = 1 + +augroup Go + autocmd! + autocmd FileType go setlocal omnifunc=go#complete#Complete + autocmd FileType go setlocal foldmethod=syntax + autocmd FileType go setlocal foldlevel=100 + autocmd FileType go nnoremap <buffer> <F9> :GoTest<CR> + autocmd FileType go inoremap <buffer> <F9> <ESC>:GoTest<CR>i +augroup END + +" }}} + +" RAML {{{ + +function! s:buffer_syntax() " {{{ + syn keyword ramlRAML RAML contained + syn match ramlVersionString '^#%RAML \d\.\d' contains=ramlRAML +endfunction " }}} + +augroup RAML + autocmd! + autocmd BufRead,BufNewFile *.raml set filetype=yaml + autocmd BufRead,BufNewFile *.raml call s:buffer_syntax() +augroup END + +hi def link ramlVersionString Special +hi def link ramlRAML Error +" }}} + +" Mustache/Handlebars {{{ +let g:mustache_abbreviations = 1 +" }}} + +" Netrw {{{ +augroup netrw + autocmd! + autocmd FileType netrw nnoremap <buffer> Q :Rexplore<CR> + + " Hee hee, oil and vinegar + function! s:setup_oil() abort + nnoremap <buffer> q <C-6> + xnoremap <buffer> q <C-6> + endfunction +augroup END +" }}} +" }}} + +" Remove trailing whitespace {{{ +fun! <SID>StripTrailingWhitespaces() + let l = line(".") + let c = col(".") + %s/\s\+$//e + call cursor(l, c) +endfun + +augroup striptrailingwhitespaces " {{{ +autocmd FileType c,cpp,java,php,ruby,python,sql,javascript,sh,jst,less,haskell,haml,coffee,scss,clojure,objc,elixir,yaml,json,eruby + \ autocmd BufWritePre <buffer> :call <SID>StripTrailingWhitespaces() +augroup END " }}} + +" }}} + +" Goyo {{{ +let g:limelight_conceal_ctermfg = "10" +let g:limelight_conceal_guifg = "#586e75" +autocmd! User GoyoEnter Limelight +autocmd! User GoyoLeave Limelight! +" }}} + +"----------------------------------------------------------------------------- + +" Commands {{{ + +" Edit temporary SQL files {{{ +let s:curr_sql = 0 +fun! <SID>EditSqlTempFile() + let l:fname = '/tmp/q' . s:curr_sql . '.sql' + execute 'edit' l:fname + let s:curr_sql = s:curr_sql + 1 +endfun +com! EditSqlTempFile call <SID>EditSqlTempFile() +" }}} + +" Double Indentation +command! -range DoubleIndentation <line1>,<line2>s/^\(\s.\{-}\)\(\S\)/\1\1\2/ + +" Quick-and-dirty fix capitalization of sql files +command! -range FixSqlCapitalization <line1>,<line2>v/\v(^\s*--.*$)|(TG_)/norm guu + +" VimPipe Commands {{{ +" let g:sql_type_default = 'pgsql' +command! SqlLive let b:vimpipe_command="vagrant ssh -c '~/mysql'" +command! SqlRails let b:vimpipe_command="bin/rails dbconsole" +command! SqlHeroku let b:vimpipe_command="heroku pg:psql" +command! SqlEntities let b:vimpipe_command="psql -h 127.1 entities nomi" +command! SqlUsers let b:vimpipe_command="psql -h 127.1 users nomi" +command! SqlTangent let b:vimpipe_command="psql -h local.docker tangent super" +" }}} + +" Git commands {{{ +command! -nargs=* Gpf Gpush -f <args> +command! -nargs=* Gcv Gcommit --verbose <args> +" }}} + +" Focus dispatch to only the last failures +command! -nargs=* FocusFailures FocusDispatch rspec --only-failures <args> + +" }}} + +" Autocommands {{{ + +augroup fugitive " {{{ + au! + autocmd BufNewFile,BufRead fugitive://* set bufhidden=delete +augroup END " }}} + +augroup omni " {{{ + au! + " autocmd FileType javascript setlocal omnifunc=tern#Complete + "autocmd FileType python setlocal omnifunc=pythoncomplete#Complete + autocmd FileType php setlocal omnifunc= +augroup END " }}} + +augroup sql " {{{ + au! + autocmd FileType sql let b:vimpipe_command="psql -h 127.0.0.1 landlordsny_development landlordsny" + autocmd FileType sql let b:vimpipe_filetype="postgresql" + autocmd FileType sql set syntax=postgresql + autocmd FileType postgresql set nowrap + autocmd BufNewFile,BufReadPost *.sql set syntax=pgsql +augroup END " }}} + +augroup markdown " {{{ + au! + autocmd FileType markdown let b:vimpipe_command='markdown' + autocmd FileType markdown let b:vimpipe_filetype='html' + autocmd FileType markdown set tw=80 +augroup END " }}} + +augroup typescript " {{{ + au! + autocmd FileType typescript let b:vimpipe_command='tsc' + autocmd FileType typescript let b:vimpipe_filetype='javascript' + autocmd FileType typescript TSSstarthere + autocmd FileType typescript nnoremap <buffer> gd :TSSdef<CR> +augroup END " }}} + +augroup jsx " {{{ + au! + " autocmd FileType jsx set syntax=javascript + autocmd FileType javascript set filetype=javascript.jsx +augroup END " }}} + +augroup nicefoldmethod " {{{ + au! + " Don't screw up folds when inserting text that might affect them, until + " leaving insert mode. Foldmethod is local to the window. Protect against + " screwing up folding when switching between windows. + autocmd InsertEnter * + \ if !exists('w:last_fdm') | + \ let w:last_fdm=&foldmethod | + \ setlocal foldmethod=manual | + \ endif + autocmd InsertLeave,WinLeave * + \ if exists('w:last_fdm') | + \ let &l:foldmethod=w:last_fdm | + \ unlet w:last_fdm | + \ endif +augroup END " }}} + +augroup visualbell " {{{ + au! + autocmd GUIEnter * set visualbell t_vb= +augroup END +" }}} + +augroup quickfix " {{{ + au! + autocmd QuickFixCmdPost grep cwindow +augroup END " }}} + +augroup php " {{{ + au! +augroup END "}}} + +augroup rubylang " {{{ + au! + autocmd FileType ruby compiler rake +augroup END " }}} + +augroup javascript "{{{ + au! + autocmd FileType javascript let &errorformat = + \ '%E%.%#%n) %s:,' . + \ '%C%.%#Error: %m,' . + \ '%C%.%#at %s (%f:%l:%c),' . + \ '%Z%.%#at %s (%f:%l:%c),' . + \ '%-G%.%#,' +augroup END " }}} + +augroup git " {{{ + autocmd! + autocmd FileType gitcommit set textwidth=72 +augroup END +" }}} +" }}} + +" Leader commands {{{ + +" Edit specific files {{{ +nnoremap <silent> <leader>ev :split $MYVIMRC<CR> +nnoremap <silent> <leader>eb :split ~/.vim_bundles<CR> +nnoremap <silent> <leader>es :UltiSnipsEdit<CR> +nnoremap <silent> <leader>ea :split ~/.vim/after/plugin/abolish.vim<CR> + +nnoremap <silent> <leader>sv :so $MYVIMRC<CR> +nnoremap <silent> <leader>sb :so ~/.vim_bundles<CR> +nnoremap <silent> <leader>sa :so ~/.vim/after/plugin/abolish.vim<CR> + +nnoremap <Leader>el :EditSqlTempFile<CR> +" }}} + +" Toggle navigation panels {{{ +nnoremap <Leader>l :TagbarToggle<CR> +nnoremap <Leader>mb :MBEToggle<CR> +nnoremap <Leader>u :GundoToggle<CR> + +nnoremap <Leader>t :CtrlP<CR> +nnoremap <Leader>z :FZF<CR> +nnoremap <Leader>b :CtrlPBuffer<CR> +nnoremap <Leader>a :CtrlPTag<CR> +nnoremap <Leader>r :CtrlPGitBranch<CR> +" }}} + +" CtrlP {{{ +let g:ctrlp_custom_ignore = { + \ 'dir': 'node_modules', + \ } +" }}} + +" Git leader commands {{{ +noremap <Leader>g :Git<SPACE> +noremap <Leader>gu :Gpull<CR> +noremap <Leader>gp :Gpush<CR> +noremap <Leader>s :Gstatus<CR> +noremap <Leader>cv :Gcommit --verbose<CR> +noremap <Leader>ca :Gcommit --verbose --amend<CR> + +nnoremap <Leader>dl :diffg LOCAL<CR> +nnoremap <Leader>dr :diffg REMOTE<CR> +nnoremap <Leader>db :diffg BASE<CR> +nnoremap <Leader>du :diffu<CR> +nnoremap <Leader>dg :diffg<CR> + +nnoremap <Leader>d2 :diffg //2<CR>:diffu<CR> +nnoremap <Leader>d3 :diffg //3<CR>:diffu<CR> + +nnoremap <Leader>yt :SignifyToggle<CR> +" }}} + +" Breakpoint Leader Commands {{{ +nnoremap <Leader>x :Breakpoint<CR> +nnoremap <Leader>dx :BreakpointRemove *<CR> +" }}} + +" Tabularize {{{ + " Leader Commands {{{ + nnoremap <localleader>t= :Tabularize /=<CR> + vmap <localleader>t= :Tabularize /=<CR> + + nnoremap <localleader>t> :Tabularize /=><CR> + vmap <localleader>t> :Tabularize /=><CR> + " }}} + + " => Aligning {{{ + function! s:rocketalign() + let l:p = '^.*=>\s.*$' + echo l:p + if exists(':Tabularize') && getline('.') =~# '^.*=' && + \ (getline(line('.')-1) =~# l:p || getline(line('.')+1) =~# l:p) + let column = strlen(substitute(getline('.')[0:col('.')],'[^=>]','','g')) + let position = strlen(matchstr(getline('.')[0:col('.')],'.*=>\s*\zs.*')) + Tabularize/=>/l1 + normal! $ + call search(repeat('[^=>]*=>',column).'\s\{-\}'.repeat('.',position),'ce',line('.')) + endif + endfunction + "inoremap <buffer> <space>=><space> =><Esc>:call <SID>rocketalign()<CR>a + " }}} + + " = Aligning {{{ + function! s:eqalign() + let l:p = '^.*=\s.*$' + if exists(':Tabularize') && getline('.') =~# '^.*=' && + \ (getline(line('.')-1) =~# l:p || getline(line('.')+1) =~# l:p) + let column = strlen(substitute(getline('.')[0:col('.')],'[^=]','','g')) + let position = strlen(matchstr(getline('.')[0:col('.')],'.*=\s*\zs.*')) + Tabularize/=/l1 + normal! $ + call search(repeat('[^=]*=',column).'\s\{-\}'.repeat('.',position),'ce',line('.')) + endif + endfunction + "inoremap <buffer><silent> <space>=<space> =<Esc>:call <SID>eqalign()<CR>a + " }}} + + " : Aligning {{{ + function! s:colonalign() + let l:p : '^.*:\s.*$' + if exists(':Tabularize') && getline('.') :~# '^.*:' && + \ (getline(line('.')-1) :~# l:p || getline(line('.')+1) :~# l:p) + let column : strlen(substitute(getline('.')[0:col('.')],'[^:]','','g')) + let position : strlen(matchstr(getline('.')[0:col('.')],'.*:\s*\zs.*')) + Tabularize/:/l1 + normal! $ + call search(repeat('[^:]*:',column).'\s\{-\}'.repeat('.',position),'ce',line('.')) + endif + endfunction + "inoremap <buffer><silent> <space>:<space> :<Esc>:call <SID>colonalign()<CR>a + " }}} +" }}} + +" }}} + +" Mappings {{{ +" 'delete current' +nnoremap dc 0d$ +nnoremap com :silent !tmux set status<CR> +nnoremap <F9> :Make<CR> +nnoremap g<CR> :Dispatch<CR> +nnoremap g\ :Start<CR> +inoremap <F9> <ESC>:Make<CR>i + +" Navigate buffers {{{ +nnoremap gb :bn<CR> +nnoremap gB :bp<CR> +" }}} + +" Window Navigation {{{ +nnoremap <space>w <C-w> +nnoremap <space>h <C-w>h +nnoremap <space>j <C-w>j +nnoremap <space>k <C-w>k +nnoremap <space>l <C-w>l +nnoremap <space>z <C-w>z +" }}} + + +" Sort with motion {{{ +if !exists("g:sort_motion_flags") + let g:sort_motion_flags = "" +endif +function! s:sort_motion(mode) abort + if a:mode == 'line' + execute "'[,']sort " . g:sort_motion_flags + elseif a:mode == 'char' + execute "normal! `[v`]y" + let sorted = join(sort(split(@@, ', ')), ', ') + execute "normal! v`]c" . sorted + elseif a:mode == 'V' || a:mode == '' + execute "'<,'>sort " . g:sort_motion_flags + endif +endfunction + +function! s:sort_lines() + let beginning = line('.') + let end = v:count + beginning - 1 + execute beginning . ',' . end . 'sort' +endfunction + +xnoremap <silent> <Plug>SortMotionVisual :<C-U>call <SID>sort_motion(visualmode())<CR> +nnoremap <silent> <Plug>SortMotion :<C-U>set opfunc=<SID>sort_motion<CR>g@ +nnoremap <silent> <Plug>SortLines :<C-U>call <SID>sort_lines()<CR> + +map go <Plug>SortMotion +vmap go <Plug>SortMotionVisual +map goo <Plug>SortLines +" }}} +" }}} + +let g:hare_executable = 'cabal exec -- ghc-hare' diff --git a/users/grfn/system/home/modules/zshrc b/users/grfn/system/home/modules/zshrc new file mode 100644 index 000000000000..a12173d6842d --- /dev/null +++ b/users/grfn/system/home/modules/zshrc @@ -0,0 +1,327 @@ +#!/usr/bin/zsh +# vim: set fdm=marker fmr={{{,}}}: + +stty -ixon + +# Compinstall {{{ +zstyle ':completion:*' completer _complete _ignored _correct _approximate +zstyle ':completion:*' matcher-list '' 'm:{[:lower:]}={[:upper:]} m:{[:lower:][:upper:]}={[:upper:][:lower:]} r:|[._- :]=** r:|=**' 'l:|=* r:|=*' +zstyle ':completion:*' max-errors 5 +zstyle ':completion:*' use-cache yes +zstyle ':completion::complete:grunt::options:' expire 1 +zstyle ':completion:*' prompt '%e errors' +zstyle :compinstall filename '~/.zshrc' +autoload -Uz compinit +compinit +# }}} + +# Zsh-newuser-install {{{ +HISTFILE=~/.histfile +HISTSIZE=1000 +SAVEHIST=1000 +setopt appendhistory autocd extendedglob notify autopushd +unsetopt beep nomatch +bindkey -v +# }}} + +# Basic options {{{ +set -o vi +umask 022 +export VIRTUAL_ENV_DISABLE_PROMPT=1 +# export PATH=~/.local/bin:~/.cabal/bin:$PATH:~/code/go/bin:~/bin:~/npm/bin:~/.gem/ruby/2.1.0/bin:~/.gem/ruby/2.0.0/bin:/home/smith/bin +# }}} + +# Zsh highlight highlighters {{{ +ZSH_HIGHLIGHT_HIGHLIGHTERS=(main brackets pattern root) +# }}} + +# More basic options {{{ +setopt no_hist_verify +setopt histignorespace +# }}} + +# Utility Functions {{{ + +# Set the terminal's title bar. +function titlebar() { +echo -ne "\033]0;$*\007" +} + +function quiet() { +"$@" >/dev/null +} + +function quieter() { +"$@" >/dev/null 2>&1 +} + +# From http://stackoverflow.com/questions/370047/#370255 +function path_remove() { +IFS=: +# convert it to an array +t=($PATH) +unset IFS +# perform any array operations to remove elements from the array +t=(${t[@]%%$1}) +IFS=: +# output the new array +echo "${t[*]}" +} + +# }}} + +# Force screen to use zsh {{{ +# }}} + +# Environment {{{ +# }}} + +# Directory Stuff {{{ + +# Always use color output for `ls` + +# Directory listing + +# Easier navigation: .., ..., - + +# File size + +# Recursively delete `.DS_Store` files + +# Create a new directory and enter it +function md() { + mkdir -p "$@" && cd "$@" +} + +# }}} + +# MPD/MPC stuff {{{ +function mp() { +# Test if drive is already mounted +if ! lsblk | grep /media/external >/dev/null; then + if ! sudo mount /media/external; then + echo "External drive not plugged in, or could not mount" + return 1 + fi +fi +if (mpc >/dev/null 2>&1); then + ncmpcpp +else + mpd && + (pgrep mpdscribble || mpdscribble) && + ncmpcpp +fi +} + +# kill mp +function kmp() { +killall ncmpcpp +mpd --kill + +local files + +if (files=$(lsof 2>&1 | grep -v docker | grep external)); then + echo + echo "==> Still processes using external drive:" + echo + echo $files +else + sudo umount /media/external +fi +} + + +function mppal() { +mpc search album "$1" | mpc add && + mpc play; +} +# }}} + +# Git stuff {{{ +# function ga() { git add "${@:-.}"; } # Add all files by default +# Add non-whitespace changes +# function gc() { git checkout "${@:-master}"; } # Checkout master by default + +# open all changed files (that still actually exist) in the editor +function ged() { +local files=() +for f in $(git diff --name-only "$@"); do + [[ -e "$f" ]] && files=("${files[@]}" "$f") +done +local n=${#files[@]} +echo "Opening $n $([[ "$@" ]] || echo "modified ")file$([[ $n != 1 ]] && \ + echo s)${@:+ modified in }$@" +q "${files[@]}" +} + +# git find-replace +function gfr() { +if [[ "$#" == "0" ]]; then + echo 'Usage:' + echo ' gg_replace term replacement file_mask' + echo + echo 'Example:' + echo ' gg_replace cappuchino cappuccino *.html' + echo +else + find=$1; shift + replace=$1; shift + + ORIG_GLOBIGNORE=$GLOBIGNORE + GLOBIGNORE=*.* + if [[ "$#" = "0" ]]; then + set -- ' ' $@ + fi + + while [[ "$#" -gt "0" ]]; do + for file in `git grep -l $find -- $1`; do + sed -e "s/$find/$replace/g" -i'' $file + done + shift + done + + GLOBIGNORE=$ORIG_GLOBIGNORE +fi +} + +function vconflicts() { +$EDITOR $(git status --porcelain | awk '/^UU/ { print $2 }') +} +# }}} + +# fzf {{{ +v() { + local file + file=$(fzf-tmux --query="$1" --select-1 --exit-0) + [ -n "$file" ] && ${EDITOR:-vim} "$file" +} + +c() { + local dir + dir=$(find ${1:-*} -path '*/\.*' -prune -o -type d -print 2> /dev/null | fzf +m) && cd "$dir" +} + +co() { + local branch + branch=$(git branch -a | sed -s "s/\s*\**//g" | fzf --query="$1" --select-1 --exit-0) && git checkout "$branch" +} + + +# fh - repeat history +# h() { +# eval $(([ -n "$ZSH_NAME" ] && fc -l 1 || history) | fzf +s | sed 's/ *[0-9]* *//') +# } + +# fkill - kill process +fkill() { + ps -ef | sed 1d | fzf-tmux -m | awk '{print $2}' | xargs kill -${1:-9} +} +# }}} + +# Tmux utils {{{ +kill_detached() { + for sess in $(tmux ls | grep -v attached | sed -s "s/:.*$//"); do + tmux kill-session -t $sess; + done +} +# }}} + +# Docker {{{ + + +# dbp foo/bar . +function dbp () { + docker build -t $1 ${@:2} && docker push $1 +} + +# }}} + +# Twitter! {{{ + + +# favelast <username> +function favelast() { + t fave $(t tl -l $1 | head -n1 | first) +} + +function rtlast() { + t rt $(t tl -l $1 | head -n1 | first) +} + +function tthread() { + t reply $(t tl -l $TWITTER_WHOAMI | head -n1 | first) $@ +} +# }}} + +# Geeknote {{{ +gnc() { + gn create --title $1 --content '' && + gn find --count=1 "$1" + gn edit 1 +} +# }}} + +# Misc aliases {{{ + +function fw() { # fix white + local substitution + local substitution='s/\x1b\[90m/\x1b[92m/g' + $@ > >(perl -pe "$substitution") 2> >(perl -pe "$substitution" 1>&2) +} +# }}} + +# Grep options {{{ +unset GREP_OPTIONS +export GREP_OPTIONS= +# }}} + + +# Run docker containers {{{ + # -d \ + # -v $HOME/.pentadactyl:/home/firefox/.pentadactyl:rw \ + # -v $HOME/.pentadactylrc:/home/firefox/.pentadactylrc:rw \ + # -v $HOME/.mozilla:/home/firefox/.mozilla:rw \ + # -v $HOME/.config:/home/firefox/.config \ + # -v $HOME/Downloads:/home/firefox/Downloads:rw \ + # -v /etc/fonts:/etc/fonts \ + # -v /tmp/.X11-unix:/tmp/.X11-unix \ + # -v /dev/snd:/dev/snd \ + # --net=host \ + # -v $XDG_RUNTIME_DIR:$XDG_RUNTIME_DIR \ + # -e uid=$(id -u) \ + # -e gid=$(id -g) \ + # -e DISPLAY=$DISPLAY \ + # -e XDG_RUNTIME_DIR=$XDG_RUNTIME_DIR \ + # --name firefox \ + # --rm -it \ + # glittershark/firefox +# }}} + +# Change cursor shape on insert/normal mode {{{ +# (https://unix.stackexchange.com/q/433273/64261) + +KEYTIMEOUT=5 + +_fix_cursor() { + echo -ne '\e[5 q' +} + +precmd_functions+=(_fix_cursor) + +function zle-keymap-select { + if [[ ${KEYMAP} == vicmd ]] || + [[ $1 = 'block' ]]; then + echo -ne '\e[1 q' + + elif [[ ${KEYMAP} == main ]] || + [[ ${KEYMAP} == viins ]] || + [[ ${KEYMAP} = '' ]] || + [[ $1 = 'beam' ]]; then + echo -ne '\e[5 q' + fi +} +zle -N zle-keymap-select + +# }}} + +[ -f ./.localrc ] && source ./.localrc diff --git a/users/grfn/system/home/platforms/darwin.nix b/users/grfn/system/home/platforms/darwin.nix new file mode 100644 index 000000000000..cf0375e94162 --- /dev/null +++ b/users/grfn/system/home/platforms/darwin.nix @@ -0,0 +1,26 @@ +{ config, lib, pkgs, ... }: + +with lib; + +{ + config = { + home.packages = with pkgs; [ + coreutils + gnupg + pinentry_mac + ]; + + home.activation.linkApplications = lib.hm.dag.entryAfter ["writeBoundary"] '' + $DRY_RUN_CMD ln -sf $VERBOSE_ARG \ + ~/.nix-profile/Applications/* ~/Applications/ + ''; + + programs.zsh.initExtra = '' + export NIX_PATH=$HOME/.nix-defexpr/channels:$NIX_PATH + + if [[ "$TERM" == "alacritty" ]]; then + export TERM="xterm-256color" + fi + ''; + }; +} diff --git a/users/grfn/system/home/platforms/linux.nix b/users/grfn/system/home/platforms/linux.nix new file mode 100644 index 000000000000..92924e419973 --- /dev/null +++ b/users/grfn/system/home/platforms/linux.nix @@ -0,0 +1,93 @@ +{ config, pkgs, ... }: + +let + + depot = config.lib.depot; + +in + +{ + imports = [ + ../modules/alacritty.nix + ../modules/alsi.nix + ../modules/development.nix + ../modules/emacs.nix + ../modules/email.nix + ../modules/firefox.nix + ../modules/games.nix + ../modules/obs.nix + ../modules/i3.nix + ../modules/shell.nix + ../modules/tarsnap.nix + ../modules/vim.nix + ]; + + xsession.enable = true; + + home.packages = with pkgs; [ + (import (fetchTarball "https://github.com/ashkitten/nixpkgs/archive/init-glimpse.tar.gz") {}).glimpse + + # Desktop stuff + arandr + firefox + feh + chromium + xclip + xorg.xev + picom + peek + signal-desktop + apvlv # pdf viewer + vlc + irssi + gnutls + pandoc + barrier + depot.tools.nsfv-setup + + # System utilities + powertop + usbutils + pciutils + gdmap + lsof + tree + ncat + iftop + + # Security + gnupg + keybase + openssl + yubikey-manager + yubikey-manager-qt + + # Spotify...etc + spotify + playerctl + ]; + + services.redshift = { + enable = true; + provider = "geoclue2"; + }; + + services.pasystray.enable = true; + + services.gpg-agent = { + enable = true; + }; + + gtk = { + enable = true; + gtk3.bookmarks = [ + "file:///home/grfn/code" + ]; + }; + + programs.zsh.initExtra = '' + [[ ! $IN_NIX_SHELL ]] && alsi -l + ''; + + services.lorri.enable = true; +} diff --git a/users/grfn/system/install b/users/grfn/system/install new file mode 100755 index 000000000000..a9a45953da07 --- /dev/null +++ b/users/grfn/system/install @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eo pipefail + +if [[ -f /etc/nixos/.system-installed ]]; then + echo "=== System config already installed, skipping" +else + echo "==> Installing system config" + + [[ -d /etc/nixos ]] && sudo mv /etc/nixos{,.bak} + sudo mkdir -p /etc/nixos + sudo cp /etc/nixos.bak/hardware-configuration.nix /etc/nixos + + sudo cp ./system/configuration.nix /etc/nixos/ + sudo ln -s $(pwd)/system/{machines,modules,pkgs} /etc/nixos + sudo touch /etc/nixos/.system-installed + + echo "==> System config installed, your old configuration is at /etc/nixos.bak" +fi +echo + +if [[ -f ~/.config/nixpkgs/system-installed ]]; then + echo "=== home-manager config already installed, skipping" +else + echo "==> Installing home-manager config" + nix-channel --add https://github.com/rycee/home-manager/archive/master.tar.gz home-manager + nix-channel --update + # nix-shell '<home-manager>' -A install + + [[ -d ~/.config/nixpkgs ]] && mv ~/.config/{nixpkgs,nixpkgs.bak} + mkdir -p ~/.config/nixpkgs + ln -s $(pwd)/home/* ~/.config/nixpkgs + + echo "==> home-manager config installed" +fi diff --git a/users/grfn/system/system/.skip-subtree b/users/grfn/system/system/.skip-subtree new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/users/grfn/system/system/.skip-subtree diff --git a/users/grfn/system/system/configuration.nix b/users/grfn/system/system/configuration.nix new file mode 100644 index 000000000000..eae567015b73 --- /dev/null +++ b/users/grfn/system/system/configuration.nix @@ -0,0 +1,11 @@ +{ config, pkgs, ... }: + +let machine = throw "Pick a machine from ./machines"; in +{ + imports = + [ + /etc/nixos/hardware-configuration.nix + ./modules/common.nix + machine + ]; +} diff --git a/users/grfn/system/system/default.nix b/users/grfn/system/system/default.nix new file mode 100644 index 000000000000..445d4ad2438d --- /dev/null +++ b/users/grfn/system/system/default.nix @@ -0,0 +1,38 @@ +args @ { depot, pkgs, ... }: + +rec { + mugwump = import ./machines/mugwump.nix; + + mugwumpSystem = (depot.ops.nixos.nixosFor mugwump).system; + + roswell = import ./machines/roswell.nix; + + roswellSystem = (depot.ops.nixos.nixosFor ({ ... }: { + imports = [ + ./machines/roswell.nix + "${pkgs.home-manager.src}/nixos" + ]; + + # Use the same nixpkgs as everything else + home-manager.useGlobalPkgs = true; + + home-manager.users.grfn = { config, lib, ... }: { + imports = [ ../home/machines/roswell.nix ]; + lib.depot = depot; + }; + })).system; + + yeren = import ./machines/yeren.nix; + + yerenSystem = (depot.ops.nixos.nixosFor yeren).system; + + iso = import ./iso.nix args; + + meta.targets = [ + "mugwumpSystem" + "roswellSystem" + "yerenSystem" + + "iso" + ]; +} diff --git a/users/grfn/system/system/iso.nix b/users/grfn/system/system/iso.nix new file mode 100644 index 000000000000..4adccebfb8a2 --- /dev/null +++ b/users/grfn/system/system/iso.nix @@ -0,0 +1,17 @@ +{ depot, lib, pkgs, ... }: + +let + configuration = { ... }: { + imports = [ + "${pkgs.path}/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix" + "${pkgs.path}/nixos/modules/installer/cd-dvd/channel.nix" + ]; + + networking.networkmanager.enable = true; + networking.useDHCP = false; + networking.firewall.enable = false; + networking.wireless.enable = lib.mkForce false; + }; +in (depot.third_party.nixos { + inherit configuration; +}).config.system.build.isoImage diff --git a/users/grfn/system/system/machines/bumblebee.nix b/users/grfn/system/system/machines/bumblebee.nix new file mode 100644 index 000000000000..0fec21409255 --- /dev/null +++ b/users/grfn/system/system/machines/bumblebee.nix @@ -0,0 +1,23 @@ +{ config, lib, pkgs, ... }: +{ + imports = [ + ../modules/reusable/battery.nix + ]; + + networking.hostName = "bumblebee"; + + powerManagement = { + enable = true; + cpuFreqGovernor = "powersave"; + powertop.enable = true; + }; + + # Hibernate on low battery + laptop.onLowBattery = { + enable = true; + action = "hibernate"; + thresholdPercentage = 5; + }; + + services.xserver.xkbOptions = "caps:swapescape"; +} diff --git a/users/grfn/system/system/machines/mugwump.nix b/users/grfn/system/system/machines/mugwump.nix new file mode 100644 index 000000000000..aae9b0387ff5 --- /dev/null +++ b/users/grfn/system/system/machines/mugwump.nix @@ -0,0 +1,260 @@ +{ config, lib, pkgs, modulesPath, depot, ... }: + +with lib; + +{ + imports = [ + ../modules/common.nix + (modulesPath + "/installer/scan/not-detected.nix") + "${depot.path}/ops/modules/prometheus-fail2ban-exporter.nix" + "${depot.path}/users/grfn/xanthous/server/module.nix" + ]; + + networking.hostName = "mugwump"; + + boot = { + loader.systemd-boot.enable = true; + + kernelModules = [ "kvm-intel" ]; + extraModulePackages = [ ]; + + initrd = { + availableKernelModules = [ "xhci_pci" "ehci_pci" "ahci" "usb_storage" "usbhid" "sd_mod" ]; + kernelModules = [ + "uas" "usbcore" "usb_storage" "vfat" "nls_cp437" "nls_iso8859_1" + ]; + + postDeviceCommands = pkgs.lib.mkBefore '' + mkdir -m 0755 -p /key + sleep 2 + mount -n -t vfat -o ro `findfs UUID=9048-A9D5` /key + ''; + + luks.devices."cryptroot" = { + device = "/dev/disk/by-uuid/803a9028-339c-4617-a213-4fe138161f6d"; + keyFile = "/key/keyfile"; + preLVM = false; + }; + }; + }; + + fileSystems = { + "/" = { + device = "/dev/mapper/cryptroot"; + fsType = "btrfs"; + }; + "/boot" = { + device = "/dev/disk/by-uuid/7D74-0E4B"; + fsType = "vfat"; + }; + }; + + networking.interfaces = { + enp0s25.useDHCP = false; + wlp2s0.useDHCP = false; + }; + + networking.firewall.enable = true; + networking.firewall.allowedTCPPorts = [ 22 80 443 ]; + + security.sudo.extraRules = [{ + groups = ["wheel"]; + commands = [{ command = "ALL"; options = ["NOPASSWD"]; }]; + }]; + + nix.gc.dates = "monthly"; + + services.fail2ban = { + enable = true; + ignoreIP = [ + "172.16.0.0/16" + ]; + }; + + services.openssh = { + allowSFTP = false; + passwordAuthentication = false; + permitRootLogin = "no"; + }; + + services.grafana = { + enable = true; + port = 3000; + domain = "metrics.gws.fyi"; + rootUrl = "https://metrics.gws.fyi"; + dataDir = "/var/lib/grafana"; + analytics.reporting.enable = false; + + provision = { + enable = true; + datasources = [{ + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + }]; + }; + }; + + security.acme.email = "root@gws.fyi"; + security.acme.acceptTerms = true; + + services.nginx = { + enable = true; + statusPage = true; + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedTlsSettings = true; + + virtualHosts = { + "metrics.gws.fyi" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://localhost:${toString config.services.grafana.port}"; + }; + }; + }; + }; + + services.ddclient = { + enable = true; + domains = [ "home.gws.fyi" ]; + interval = "1d"; + zone = "gws.fyi"; + protocol = "cloudflare"; + username = "root@gws.fyi"; + quiet = true; + }; + + systemd.services.ddclient.serviceConfig = { + EnvironmentFile = "/etc/secrets/cloudflare.env"; + DynamicUser = lib.mkForce false; + ExecStart = lib.mkForce ( + let runtimeDir = + config.systemd.services.ddclient.serviceConfig.RuntimeDirectory; + in pkgs.writeShellScript "ddclient" '' + set -eo pipefail + + ${pkgs.gnused}/bin/sed -i -s s/password=/password=$CLOUDFLARE_API_KEY/ /run/${runtimeDir}/ddclient.conf + exec ${pkgs.ddclient}/bin/ddclient \ + -file /run/${runtimeDir}/ddclient.conf \ + -login=$CLOUDFLARE_EMAIL \ + ''); + }; + + security.acme.certs."metrics.gws.fyi" = { + dnsProvider = "cloudflare"; + credentialsFile = "/etc/secrets/cloudflare.env"; + webroot = mkForce null; + }; + + services.prometheus = { + enable = true; + exporters = { + node = { + enable = true; + openFirewall = false; + + enabledCollectors = [ + "processes" + "systemd" + "tcpstat" + "wifi" + ]; + }; + + nginx = { + enable = true; + openFirewall = true; + sslVerify = false; + constLabels = [ "host=mugwump" ]; + }; + + blackbox = { + enable = true; + openFirewall = true; + configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON { + modules = { + https_2xx = { + prober = "http"; + http = { + method = "GET"; + fail_if_ssl = false; + fail_if_not_ssl = true; + preferred_ip_protocol = "ip4"; + }; + }; + }; + }); + }; + }; + + scrapeConfigs = [{ + job_name = "node"; + scrape_interval = "5s"; + static_configs = [{ + targets = ["localhost:${toString config.services.prometheus.exporters.node.port}"]; + }]; + } { + job_name = "nginx"; + scrape_interval = "5s"; + static_configs = [{ + targets = ["localhost:${toString config.services.prometheus.exporters.nginx.port}"]; + }]; + } { + job_name = "xanthous_server"; + scrape_interval = "1s"; + static_configs = [{ + targets = ["localhost:${toString config.services.xanthous-server.metricsPort}"]; + }]; + } { + job_name = "blackbox"; + metrics_path = "/probe"; + params.module = ["https_2xx"]; + scrape_interval = "5s"; + static_configs = [{ + targets = [ + "https://gws.fyi" + "https://windtunnel.ci" + "https://app.windtunnel.ci" + "https://metrics.gws.fyi" + ]; + }]; + relabel_configs = [{ + source_labels = ["__address__"]; + target_label = "__param_target"; + } { + source_labels = ["__param_target"]; + target_label = "instance"; + } { + target_label = "__address__"; + replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}"; + }]; + }]; + }; + + services.xanthous-server.enable = true; + + virtualisation.docker.enable = true; + + services.buildkite-agents = listToAttrs (map (n: rec { + name = "mugwump-${toString n}"; + value = { + inherit name; + enable = true; + tokenPath = "/etc/secrets/buildkite-agent-token"; + privateSshKeyPath = "/etc/secrets/buildkite-ssh-key"; + runtimePackages = with pkgs; [ + docker + nix + gnutar + gzip + ]; + }; + }) (range 1 1)); + + users.users."buildkite-agent-mugwump-1" = { + isSystemUser = true; + extraGroups = [ "docker" ]; + }; +} diff --git a/users/grfn/system/system/machines/roswell.nix b/users/grfn/system/system/machines/roswell.nix new file mode 100644 index 000000000000..6eb4a510b8cd --- /dev/null +++ b/users/grfn/system/system/machines/roswell.nix @@ -0,0 +1,17 @@ +{ depot, config, lib, pkgs, modulesPath, ... }: + +{ + imports = [ + ../modules/common.nix + "${modulesPath}/installer/scan/not-detected.nix" + "${modulesPath}/virtualisation/amazon-image.nix" + ]; + + ec2.hvm = true; + + networking.hostName = "roswell"; + + users.users.grfn.openssh.authorizedKeys.keys = [ + depot.users.grfn.keys.main + ]; +} diff --git a/users/grfn/system/system/machines/yeren.nix b/users/grfn/system/system/machines/yeren.nix new file mode 100644 index 000000000000..c6aff886de2f --- /dev/null +++ b/users/grfn/system/system/machines/yeren.nix @@ -0,0 +1,132 @@ +{ depot, modulesPath, config, lib, pkgs, ... }: + +{ + imports = [ + (modulesPath + "/installer/scan/not-detected.nix") + ../modules/common.nix + ../modules/laptop.nix + ../modules/xserver.nix + ../modules/fonts.nix + ../modules/sound.nix + ../modules/tvl.nix + ../modules/development.nix + ../modules/work/kolide.nix + ]; + + networking.hostName = "yeren"; + + system.stateVersion = "21.03"; + + boot = { + initrd = { + availableKernelModules = [ "xhci_pci" "thunderbolt" "nvme" "usb_storage" "sd_mod" "rtsx_pci_sdmmc" ]; + kernelModules = [ ]; + + luks.devices = { + "cryptroot".device = "/dev/disk/by-uuid/dcfbc22d-e0d2-411b-8dd3-96704d3aae2e"; + }; + }; + + kernelPackages = pkgs.linuxPackages_5_14; + + kernelModules = [ "kvm-intel" ]; + blacklistedKernelModules = [ "psmouse" ]; + extraModulePackages = [ + config.boot.kernelPackages.digimend + ]; + kernelParams = [ + "i915.preliminary_hw_support=1" + "pcie_aspm=force" + ]; + + # https://bbs.archlinux.org/viewtopic.php?pid=1933643#p1933643 + extraModprobeConfig = '' + options snd-intel-dspcfg dsp_driver=1 + ''; + + kernel.sysctl = { + "kernel.perf_event_paranoid" = -1; + }; + }; + + fileSystems = { + "/" = { + device = "/dev/mapper/cryptroot"; + fsType = "btrfs"; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/53A9-248B"; + fsType = "vfat"; + }; + }; + + swapDevices = [{ + device = "/dev/disk/by-uuid/b627cb0e-0451-4f25-94d0-6497e01f0da4"; + }]; + + services.xserver = { + exportConfiguration = true; + extraConfig = '' + Section "Device" + Identifier "Intel Graphics" + Driver "intel" + Option "TripleBuffer" "true" + Option "TearFree" "true" + Option "DRI" "true" + Option "AccelMethod" "sna" + EndSection + ''; + }; + + hardware.firmware = with pkgs; [ + alsa-firmware + sof-firmware + ]; + + hardware.opengl.extraPackages = with pkgs; [ + vaapiIntel + vaapiVdpau + libvdpau-va-gl + intel-media-driver + ]; + + # Disabled for now until libfprint-tod can get a version bump + # services.fprintd = { + # enable = true; + # package = pkgs.fprintd-tod; + # }; + + systemd.services.fprintd.environment.FP_TOD_DRIVERS_DIR = + "${pkgs.libfprint-2-tod1-goodix}/usr/lib/libfprint-2/tod-1"; + + security.pam.loginLimits = [ + { + domain = "grfn"; + type = "soft"; + item = "nofile"; + value = "65535"; + } + ]; + + security.pam.services = { + login.fprintAuth = true; + sudo.fprintAuth = true; + i3lock.fprintAuth = false; + i3lock-color.fprintAuth = false; + lightdm.fprintAuth = true; + lightdm-greeter.fprintAuth = true; + }; + + hardware.opengl.driSupport32Bit = true; + + hardware.pulseaudio.extraConfig = '' + load-module module-remap-source source_name=KompleteAudio6_1 source_properties=device.description=KompleteAudio6Input1 master=alsa_input.usb-Native_Instruments_Komplete_Audio_6_458E0FFD-00.multichannel-input remix=no channels=1 master_channel_map=front-left channel_map=mono + load-module module-remap-source source_name=KompleteAudio6_2 source_properties=device.description=KompleteAudio6Input2 master=alsa_input.usb-Native_Instruments_Komplete_Audio_6_458E0FFD-00.multichannel-input remix=no channels=1 master_channel_map=front-right channel_map=mono + load-module module-remap-sink sink_name=KompleteAudio6_12 sink_properties=device.description=KompleteAudio6_12 remix=no master=alsa_output.usb-Native_Instruments_Komplete_Audio_6_458E0FFD-00.analog-surround-21 channels=2 master_channel_map=front-left,front-right channel_map=front-left,front-right + ''; + + services.fwupd.enable = true; + + services.tailscale.enable = true; +} diff --git a/users/grfn/system/system/modules/common.nix b/users/grfn/system/system/modules/common.nix new file mode 100644 index 000000000000..a91584680d52 --- /dev/null +++ b/users/grfn/system/system/modules/common.nix @@ -0,0 +1,80 @@ +{ config, lib, pkgs, ... }: + +let + + depot = import ../../../../.. {}; + +in + +with lib; + +{ + boot.loader.systemd-boot.enable = true; + boot.loader.efi.canTouchEfiVariables = true; + + networking.useDHCP = false; + networking.networkmanager.enable = true; + + i18n = { + defaultLocale = "en_US.UTF-8"; + }; + + time.timeZone = lib.mkDefault "America/New_York"; + + environment.systemPackages = with pkgs; [ + wget + vim + zsh + git + w3m + libnotify + file + lm_sensors + dnsutils + htop + man-pages + man-pages-posix + ]; + + documentation.dev.enable = true; + documentation.man.generateCaches = true; + + services.openssh.enable = true; + + programs.ssh.startAgent = true; + + networking.firewall.enable = mkDefault false; + + users.mutableUsers = true; + programs.zsh.enable = true; + environment.pathsToLink = [ "/share/zsh" ]; + users.users.grfn = { + isNormalUser = true; + initialPassword = "password"; + extraGroups = [ + "wheel" + "networkmanager" + "audio" + "docker" + ]; + shell = pkgs.zsh; + }; + + nix = { + trustedUsers = [ "grfn" ]; + autoOptimiseStore = true; + distributedBuilds = true; + + gc = { + automatic = true; + dates = mkDefault "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + services.udev.packages = with pkgs; [ + yubikey-personalization + ]; + + services.pcscd.enable = true; +} diff --git a/users/grfn/system/system/modules/desktop.nix b/users/grfn/system/system/modules/desktop.nix new file mode 100644 index 000000000000..3adbd9d9b07f --- /dev/null +++ b/users/grfn/system/system/modules/desktop.nix @@ -0,0 +1,19 @@ +{ config, lib, pkgs, ... }: + +{ + imports = [ + ./xserver.nix + ./fonts.nix + ./sound.nix + ./kernel.nix + ]; + + programs.nm-applet.enable = true; + + users.users.grfn.extraGroups = [ + "audio" + "video" + ]; + + services.geoclue2.enable = true; +} diff --git a/users/grfn/system/system/modules/development.nix b/users/grfn/system/system/modules/development.nix new file mode 100644 index 000000000000..bfa0e22cff0a --- /dev/null +++ b/users/grfn/system/system/modules/development.nix @@ -0,0 +1,6 @@ +{ config, lib, pkgs, ... }: + +{ + virtualisation.docker.enable = true; + users.users.grfn.extraGroups = [ "docker" ]; +} diff --git a/users/grfn/system/system/modules/fcitx.nix b/users/grfn/system/system/modules/fcitx.nix new file mode 100644 index 000000000000..812f598f9f47 --- /dev/null +++ b/users/grfn/system/system/modules/fcitx.nix @@ -0,0 +1,10 @@ +{ config, lib, pkgs, ... }: + +{ + i18n.inputMethod = { + enabled = "fcitx"; + fcitx.engines = with pkgs.fcitx-engines; [ + cloudpinyin + ]; + }; +} diff --git a/users/grfn/system/system/modules/fonts.nix b/users/grfn/system/system/modules/fonts.nix new file mode 100644 index 000000000000..babe30d4271f --- /dev/null +++ b/users/grfn/system/system/modules/fonts.nix @@ -0,0 +1,12 @@ +{ config, lib, pkgs, ... }: +{ + fonts = { + fonts = with pkgs; [ + nerdfonts + noto-fonts-emoji + twitter-color-emoji + ]; + + fontconfig.defaultFonts.emoji = ["Twitter Color Emoji"]; + }; +} diff --git a/users/grfn/system/system/modules/kernel.nix b/users/grfn/system/system/modules/kernel.nix new file mode 100644 index 000000000000..dd213fcca31c --- /dev/null +++ b/users/grfn/system/system/modules/kernel.nix @@ -0,0 +1,39 @@ +{ config, lib, pkgs, ... }: +with lib.versions; +let + inherit (pkgs) stdenvNoCC; + kernelRelease = config.boot.kernelPackages.kernel.version; + mj = major kernelRelease; + mm = majorMinor kernelRelease; + patched-linux-ck = stdenvNoCC.mkDerivation { + name = "linux-ck"; + src = builtins.fetchurl { + name = "linux-ck-patch-${mm}-ck1.xz"; + # example: http://ck.kolivas.org/patches/5.0/5.4/5.4-ck1/patch-5.4-ck1.xz + url = "http://ck.kolivas.org/patches/${mj}.0/${mm}/${mm}-ck1/patch-${mm}-ck1.xz"; + sha256 = "1kka38rmjcqsv4j2anczrsni0bf6yfdx2vsxbna3ic84nh3rz434"; + }; + + unpackPhase = '' + ${pkgs.xz}/bin/unxz -kfdc $src > patch-${mm}-ck1 + ''; + + installPhase = '' + cp patch-${mm}-ck1 $out + ''; + }; +in +{ + boot.kernelPackages = pkgs.linuxPackages_5_10.extend (self: super: { + kernel = super.kernel.override { + ignoreConfigErrors = true; + kernelPatches = super.kernel.kernelPatches ++ [{ + name = "linux-ck"; + patch = patched-linux-ck; + }]; + argsOverride = { + modDirVersion = super.kernel.modDirVersion + "-ck1"; + }; + }; + }); +} diff --git a/users/grfn/system/system/modules/laptop.nix b/users/grfn/system/system/modules/laptop.nix new file mode 100644 index 000000000000..05c5333e513f --- /dev/null +++ b/users/grfn/system/system/modules/laptop.nix @@ -0,0 +1,15 @@ +{ config, lib, pkgs, ... }: + +{ + imports = [ + ./reusable/battery.nix + ]; + + laptop.onLowBattery.enable = true; + + services.logind.extraConfig = '' + HandlePowerKey=hibernate + ''; + + services.tlp.enable = true; +} diff --git a/users/grfn/system/system/modules/reusable/README.org b/users/grfn/system/system/modules/reusable/README.org new file mode 100644 index 000000000000..34d9bfdcb729 --- /dev/null +++ b/users/grfn/system/system/modules/reusable/README.org @@ -0,0 +1,2 @@ +This directory contains things I'm eventually planning on contributing upstream +to nixpkgs diff --git a/users/grfn/system/system/modules/reusable/battery.nix b/users/grfn/system/system/modules/reusable/battery.nix new file mode 100644 index 000000000000..ca92e0c3f61c --- /dev/null +++ b/users/grfn/system/system/modules/reusable/battery.nix @@ -0,0 +1,32 @@ +{ config, lib, pkgs, ... }: +with lib; +{ + options = { + laptop.onLowBattery = { + enable = mkEnableOption "Perform action on low battery"; + + thresholdPercentage = mkOption { + description = "Threshold battery percentage on which to perform the action"; + default = 8; + type = types.int; + }; + + action = mkOption { + description = "Action to perform on low battery"; + default = "hibernate"; + type = types.enum [ "hibernate" "suspend" "suspend-then-hibernate" ]; + }; + }; + }; + + config = + let cfg = config.laptop.onLowBattery; + in mkIf cfg.enable { + services.udev.extraRules = concatStrings [ + ''SUBSYSTEM=="power_supply", '' + ''ATTR{status}=="Discharging", '' + ''ATTR{capacity}=="[0-${toString cfg.thresholdPercentage}]", '' + ''RUN+="${pkgs.systemd}/bin/systemctl ${cfg.action}"'' + ]; + }; +} diff --git a/users/grfn/system/system/modules/rtlsdr.nix b/users/grfn/system/system/modules/rtlsdr.nix new file mode 100644 index 000000000000..ce58ebb0dcda --- /dev/null +++ b/users/grfn/system/system/modules/rtlsdr.nix @@ -0,0 +1,17 @@ +{ config, lib, pkgs, ... }: + +{ + + environment.systemPackages = with pkgs; [ + rtl-sdr + ]; + + services.udev.packages = with pkgs; [ + rtl-sdr + ]; + + # blacklist for rtl-sdr + boot.blacklistedKernelModules = [ + "dvb_usb_rtl28xxu" + ]; +} diff --git a/users/grfn/system/system/modules/sound.nix b/users/grfn/system/system/modules/sound.nix new file mode 100644 index 000000000000..07a67a1ec43b --- /dev/null +++ b/users/grfn/system/system/modules/sound.nix @@ -0,0 +1,16 @@ +{ config, lib, pkgs, ... }: + +{ + # Enable sound. + sound.enable = true; + hardware.pulseaudio.enable = true; + + environment.systemPackages = with pkgs; [ + pulseaudio-ctl + paprefs + pasystray + pavucontrol + ]; + + hardware.pulseaudio.package = pkgs.pulseaudioFull; +} diff --git a/users/grfn/system/system/modules/tvl.nix b/users/grfn/system/system/modules/tvl.nix new file mode 100644 index 000000000000..905ec8ced537 --- /dev/null +++ b/users/grfn/system/system/modules/tvl.nix @@ -0,0 +1,37 @@ +{ config, lib, pkgs, ... }: + +{ + nix = { + buildMachines = [{ + hostName = "whitby.tvl.fyi"; + sshUser = "grfn"; + sshKey = "/root/.ssh/id_rsa"; + system = "x86_64-linux"; + maxJobs = 64; + supportedFeatures = ["big-parallel" "kvm" "nixos-test" "benchmark"]; + }]; + + extraOptions = '' + builders-use-substitutes = true + ''; + + binaryCaches = [ + "https://cache.nixos.org" + "ssh://nix-ssh@whitby.tvl.fyi" + ]; + trustedBinaryCaches = [ + "https://cache.nixos.org" + "ssh://nix-ssh@whitby.tvl.fyi" + ]; + binaryCachePublicKeys = [ + "cache.tvl.fyi:fd+9d1ceCPvDX/xVhcfv8nAa6njEhAGAEe+oGJDEeoc=" + ]; + }; + + programs.ssh.knownHosts.whitby = { + hostNames = [ "whitby" "whitby.tvl.fyi" "49.12.129.211"]; + publicKeyFile = pkgs.writeText "whitby.pub" '' + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILNh/w4BSKov0jdz3gKBc98tpoLta5bb87fQXWBhAl2I + ''; + }; +} diff --git a/users/grfn/system/system/modules/work/kolide.deb b/users/grfn/system/system/modules/work/kolide.deb new file mode 100644 index 000000000000..a319a5806fca --- /dev/null +++ b/users/grfn/system/system/modules/work/kolide.deb Binary files differdiff --git a/users/grfn/system/system/modules/work/kolide.nix b/users/grfn/system/system/modules/work/kolide.nix new file mode 100644 index 000000000000..29ee0a0d7ce4 --- /dev/null +++ b/users/grfn/system/system/modules/work/kolide.nix @@ -0,0 +1,49 @@ +{ config, lib, pkgs, ... }: + +let + deb = ./kolide.deb; + + kolide = pkgs.runCommand "kolide-data" { + buildInputs = [ pkgs.binutils-unwrapped ]; + } '' + cp ${deb} ./kolide.deb + ar x kolide.deb + mkdir result + tar xzf data.tar.gz -C result + patchelf \ + --set-interpreter ${pkgs.glibc}/lib/ld-linux-x86-64.so.2 \ + --set-rpath "${lib.makeLibraryPath (with pkgs; [ + zlib + ])}" \ + result/usr/local/kolide-k2/bin/osqueryd + mv result $out + ''; + +in { + systemd.services."launcher.kolide-k2" = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" "syslog.service" ]; + description = "The Kolide Launcher"; + serviceConfig = { + ExecStart = '' + ${kolide}/usr/local/kolide-k2/bin/launcher \ + -config \ + ${pkgs.writeText "launcher.flags" '' + with_initial_runner + control + autoupdate + root_directory /var/lib/kolide + osqueryd_path ${kolide}/usr/local/kolide-k2/bin/osqueryd + enroll_secret_path ${kolide}/etc/kolide-k2/secret + control_hostname k2control.kolide.com + update_channel stable + transport jsonrpc + hostname k2device.kolide.com + ''} + ''; + StateDirectory = "kolide"; + Restart = "on-failure"; + RestartSec = 3; + }; + }; +} diff --git a/users/grfn/system/system/modules/xserver.nix b/users/grfn/system/system/modules/xserver.nix new file mode 100644 index 000000000000..35ee44112ea1 --- /dev/null +++ b/users/grfn/system/system/modules/xserver.nix @@ -0,0 +1,16 @@ +{ config, pkgs, ... }: +{ + # Enable the X11 windowing system. + services.xserver = { + enable = true; + layout = "us"; + + libinput.enable = true; + + displayManager = { + defaultSession = "none+i3"; + }; + + windowManager.i3.enable = true; + }; +} diff --git a/users/grfn/wigglydonke.rs/index.html b/users/grfn/wigglydonke.rs/index.html new file mode 100644 index 000000000000..4fd7f25fcf8c --- /dev/null +++ b/users/grfn/wigglydonke.rs/index.html @@ -0,0 +1,16 @@ +<!DOCTYPE html> +<html> + <head> + <title>Kids Love Wiggly Donkers!</title> + <style type="text/css"> + #wiggly-donkers { + width: 100%; + } + </style> + </head> + <body> + <a href="https://tvl.fyi"> + <img src="/wd.png" id="wiggly-donkers"/> + </a> + </body> +</html> diff --git a/users/grfn/wigglydonke.rs/wd.png b/users/grfn/wigglydonke.rs/wd.png new file mode 100644 index 000000000000..217443e2df82 --- /dev/null +++ b/users/grfn/wigglydonke.rs/wd.png Binary files differdiff --git a/users/grfn/xanthous/.envrc b/users/grfn/xanthous/.envrc new file mode 100644 index 000000000000..be81feddb1a5 --- /dev/null +++ b/users/grfn/xanthous/.envrc @@ -0,0 +1 @@ +eval "$(lorri direnv)" \ No newline at end of file diff --git a/users/grfn/xanthous/.github/actions/nix-build/Dockerfile b/users/grfn/xanthous/.github/actions/nix-build/Dockerfile new file mode 100644 index 000000000000..cfe8e35df091 --- /dev/null +++ b/users/grfn/xanthous/.github/actions/nix-build/Dockerfile @@ -0,0 +1,23 @@ +FROM lnl7/nix:2.1.2 + +LABEL name="Nix Build for GitHub Actions" +LABEL version="1.0" +LABEL repository="http://github.com/glittershark/xanthous" +LABEL homepage="http://github.com/glittershark/xanthous" +LABEL maintainer="Griffin Smith <root at gws dot fyi>" + +LABEL "com.github.actions.name"="Nix Build" +LABEL "com.github.actions.description"="Runs 'nix-build'" +LABEL "com.github.actions.icon"="cpu" +LABEL "com.github.actions.color"="purple" + +RUN nix-env -iA \ + nixpkgs.gnutar nixpkgs.gzip \ + nixpkgs.gnugrep nixpkgs.git && \ + mkdir -p /etc/nix && \ + (echo "binary-caches = https://cache.nixos.org/" | tee -a /etc/nix/nix.conf) && \ + (echo "trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" | tee -a /etc/nix/nix.conf) + +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT [ "/entrypoint.sh" ] +CMD [ "--help" ] diff --git a/users/grfn/xanthous/.github/actions/nix-build/entrypoint.sh b/users/grfn/xanthous/.github/actions/nix-build/entrypoint.sh new file mode 100755 index 000000000000..cb7aca541a3f --- /dev/null +++ b/users/grfn/xanthous/.github/actions/nix-build/entrypoint.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +# Entrypoint that runs nix-build and, optionally, copies Docker image tarballs +# to real files. The reason this is necessary is because once a Nix container +# exits, you must copy out the artifacts to the working directory before exit. + +[ "$DEBUG" = "1" ] && set -x +[ "$QUIET" = "1" ] && QUIET_ARG="-Q" + +set -e + +# file to build (e.g. release.nix) +file="$1" + +[ "$file" = "" ] && echo "No .nix file to build specified!" && exit 1 +[ ! -e "$file" ] && echo "File $file not exist!" && exit 1 + +echo "Building all attrs in $file..." +nix-build --no-link ${QUIET_ARG} "$file" "${@:2}" + +echo "Copying build closure to $(pwd)/store..." +mapfile -t storePaths < <(nix-build ${QUIET_ARG} --no-link "$file" | grep -v cache-deps) +printf '%s\n' "${storePaths[@]}" > store.roots +nix copy --to "file://$(pwd)/store" "${storePaths[@]}" diff --git a/users/grfn/xanthous/.github/workflows/haskell.yml b/users/grfn/xanthous/.github/workflows/haskell.yml new file mode 100644 index 000000000000..df82de3e8caf --- /dev/null +++ b/users/grfn/xanthous/.github/workflows/haskell.yml @@ -0,0 +1,15 @@ +name: Haskell CI + +on: [push] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: Nix Build + with: + args: default.nix --arg failOnWarnings true + uses: ./.github/actions/nix-build diff --git a/users/grfn/xanthous/.gitignore b/users/grfn/xanthous/.gitignore new file mode 100644 index 000000000000..2ad31c01d443 --- /dev/null +++ b/users/grfn/xanthous/.gitignore @@ -0,0 +1,37 @@ +dist +dist-* +cabal-dev +*.o +*.hi +*.hie +*.chi +*.chs.h +*.dyn_o +*.dyn_hi +.hpc +.hsenv +.cabal-sandbox/ +cabal.sandbox.config +*.prof +*.aux +*.hp +*.eventlog +.stack-work/ +cabal.project.local +cabal.project.local~ +cabal.project.local~* +.HTF/ +.ghc.environment.* + + +# from nix-build +result + +# grr +*_flymake.hs + +# app-specific +debug.log +data +*.save +.tasty-rerun-log diff --git a/users/grfn/xanthous/LICENSE b/users/grfn/xanthous/LICENSE new file mode 100644 index 000000000000..45644ff76449 --- /dev/null +++ b/users/grfn/xanthous/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<http://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<http://www.gnu.org/philosophy/why-not-lgpl.html>. diff --git a/users/grfn/xanthous/README.org b/users/grfn/xanthous/README.org new file mode 100644 index 000000000000..7e1fedb069b1 --- /dev/null +++ b/users/grfn/xanthous/README.org @@ -0,0 +1,36 @@ +#+TITLE: Xanthous + +* Building + +#+BEGIN_SRC shell +$ nix build +#+END_SRC + +* Running + +#+BEGIN_SRC shell +$ ./result/bin/xanthous [--help] +#+END_SRC + +** Keyboard commands + +Keyboard commands are currently undocumented, but can be found in [[[https://github.com/glittershark/xanthous/blob/master/src/Xanthous/Command.hs#L26][this file]]. +Movement uses the nethack-esque hjklybnu. + +* Development + +Use [[https://github.com/target/lorri][lorri]], or run everything in a ~nix-shell~ + +#+BEGIN_SRC shell +# Build (for dev) +$ cabal new-build + +# Run the game +$ cabal new-run xanthous + +# Run tests +$ cabal new-run test + +# Run a repl +$ cabal new-repl +#+END_SRC diff --git a/users/grfn/xanthous/Setup.hs b/users/grfn/xanthous/Setup.hs new file mode 100644 index 000000000000..9a994af677b0 --- /dev/null +++ b/users/grfn/xanthous/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/users/grfn/xanthous/app/Main.hs b/users/grfn/xanthous/app/Main.hs new file mode 100644 index 000000000000..c771a0d932cb --- /dev/null +++ b/users/grfn/xanthous/app/Main.hs @@ -0,0 +1,171 @@ +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Main ( main ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (finally) +import Brick +import qualified Brick.BChan +import qualified Graphics.Vty as Vty +import qualified Options.Applicative as Opt +import System.Random +import Control.Monad.Random (getRandom) +import Control.Exception (finally) +import System.Exit (die) +-------------------------------------------------------------------------------- +import qualified Xanthous.Game as Game +import Xanthous.Game.Env (GameEnv(..)) +import qualified Xanthous.Game.Env as Game +import Xanthous.App +import Xanthous.Generators.Level + ( GeneratorInput + , parseGeneratorInput + , generateFromInput + , showCells + ) +import qualified Xanthous.Entities.Character as Character +import Xanthous.Generators.Level.Util (regions) +import Xanthous.Generators.Level.LevelContents +import Xanthous.Data (Dimensions, Dimensions'(Dimensions)) +import Data.Array.IArray ( amap ) +-------------------------------------------------------------------------------- + +parseGameConfig :: Opt.Parser Game.Config +parseGameConfig = Game.Config + <$> Opt.switch + ( Opt.long "disable-saving" + <> Opt.help "Disallow saving games" + ) + +data RunParams = RunParams + { seed :: Maybe Int + , characterName :: Maybe Text + , gameConfig :: Game.Config + } + deriving stock (Show, Eq) + +parseRunParams :: Opt.Parser RunParams +parseRunParams = RunParams + <$> optional (Opt.option Opt.auto + ( Opt.long "seed" + <> Opt.help "Random seed for the game." + )) + <*> optional (Opt.strOption + ( Opt.short 'n' + <> Opt.long "name" + <> Opt.help + ( "Name for the character. If not set on the command line, " + <> "will be prompted for at runtime" + ) + )) + <*> parseGameConfig + +data Command + = Run RunParams + | Load FilePath + | Generate GeneratorInput Dimensions (Maybe Int) + +parseDimensions :: Opt.Parser Dimensions +parseDimensions = Dimensions + <$> Opt.option Opt.auto + ( Opt.short 'w' + <> Opt.long "width" + <> Opt.metavar "TILES" + ) + <*> Opt.option Opt.auto + ( Opt.short 'h' + <> Opt.long "height" + <> Opt.metavar "TILES" + ) + + +parseCommand :: Opt.Parser Command +parseCommand = (<|> Run <$> parseRunParams) $ Opt.subparser + $ Opt.command "run" + (Opt.info + (Run <$> parseRunParams) + (Opt.progDesc "Run the game")) + <> Opt.command "load" + (Opt.info + (Load <$> Opt.argument Opt.str (Opt.metavar "FILE")) + (Opt.progDesc "Load a saved game")) + <> Opt.command "generate" + (Opt.info + (Generate + <$> parseGeneratorInput + <*> parseDimensions + <*> optional + (Opt.option Opt.auto (Opt.long "seed")) + <**> Opt.helper + ) + (Opt.progDesc "Generate a sample level")) + +optParser :: Opt.ParserInfo Command +optParser = Opt.info + (parseCommand <**> Opt.helper) + (Opt.header "Xanthous: a WIP TUI RPG") + +thanks :: IO () +thanks = putStr "\n\n" >> putStrLn "Thanks for playing Xanthous!" + +newGame :: RunParams -> IO () +newGame rparams = do + gameSeed <- maybe getRandom pure $ seed rparams + when (isNothing $ seed rparams) + . putStrLn + $ "Seed: " <> tshow gameSeed + let initialState = Game.initialStateFromSeed gameSeed &~ do + for_ (characterName rparams) $ \cn -> + Game.character . Character.characterName ?= cn + runGame NewGame (gameConfig rparams) initialState `finally` do + thanks + when (isNothing $ seed rparams) + . putStrLn + $ "Seed: " <> tshow gameSeed + putStr "\n\n" + +loadGame :: FilePath -> IO () +loadGame saveFile = do + gameState <- maybe (die "Invalid save file!") pure . Game.loadGame . fromStrict + =<< readFile @IO saveFile + gameState `deepseq` runGame (LoadGame saveFile) Game.defaultConfig gameState + +runGame :: RunType -> Game.Config -> Game.GameState -> IO () +runGame rt _config gameState = do + _eventChan <- Brick.BChan.newBChan 10 + let gameEnv = GameEnv {..} + app <- makeApp gameEnv rt + let buildVty = Vty.mkVty Vty.defaultConfig + initialVty <- buildVty + _game' <- customMain + initialVty + buildVty + (Just _eventChan) + app + gameState + pure () + +runGenerate :: GeneratorInput -> Dimensions -> Maybe Int -> IO () +runGenerate input dims mSeed = do + putStrLn "Generating..." + genSeed <- maybe getRandom pure mSeed + let randGen = mkStdGen genSeed + res = generateFromInput input dims randGen + rs = regions $ amap not res + when (isNothing mSeed) + . putStrLn + $ "Seed: " <> tshow genSeed + putStr "num regions: " + print $ length rs + putStr "region lengths: " + print $ length <$> rs + putStr "character position: " + print =<< chooseCharacterPosition res + putStrLn $ showCells res + +runCommand :: Command -> IO () +runCommand (Run runParams) = newGame runParams +runCommand (Load saveFile) = loadGame saveFile +runCommand (Generate input dims mSeed) = runGenerate input dims mSeed + +main :: IO () +main = runCommand =<< Opt.execParser optParser diff --git a/users/grfn/xanthous/bench/Bench.hs b/users/grfn/xanthous/bench/Bench.hs new file mode 100644 index 000000000000..5889618ee432 --- /dev/null +++ b/users/grfn/xanthous/bench/Bench.hs @@ -0,0 +1,12 @@ +-------------------------------------------------------------------------------- +module Main where +-------------------------------------------------------------------------------- +import Bench.Prelude +-------------------------------------------------------------------------------- +import qualified Xanthous.RandomBench +import qualified Xanthous.Generators.UtilBench + +main :: IO () +main = defaultMain + [ Xanthous.Generators.UtilBench.benchmark + ] diff --git a/users/grfn/xanthous/bench/Bench/Prelude.hs b/users/grfn/xanthous/bench/Bench/Prelude.hs new file mode 100644 index 000000000000..c553abd6d5d0 --- /dev/null +++ b/users/grfn/xanthous/bench/Bench/Prelude.hs @@ -0,0 +1,9 @@ +-------------------------------------------------------------------------------- +module Bench.Prelude + ( module Xanthous.Prelude + , module Criterion.Main + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Criterion.Main +-------------------------------------------------------------------------------- diff --git a/users/grfn/xanthous/bench/Xanthous/Generators/UtilBench.hs b/users/grfn/xanthous/bench/Xanthous/Generators/UtilBench.hs new file mode 100644 index 000000000000..56310e691c33 --- /dev/null +++ b/users/grfn/xanthous/bench/Xanthous/Generators/UtilBench.hs @@ -0,0 +1,37 @@ +-------------------------------------------------------------------------------- +module Xanthous.Generators.UtilBench (benchmark, main) where +-------------------------------------------------------------------------------- +import Bench.Prelude +-------------------------------------------------------------------------------- +import Data.Array.IArray +import Data.Array.Unboxed +import System.Random (getStdGen) +-------------------------------------------------------------------------------- +import Xanthous.Generators.Util +import qualified Xanthous.Generators.CaveAutomata as CaveAutomata +import Xanthous.Data (Dimensions'(..)) +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain [benchmark] + +-------------------------------------------------------------------------------- + +benchmark :: Benchmark +benchmark = bgroup "Generators.Util" + [ bgroup "floodFill" + [ env (NFWrapper <$> cells) $ \(NFWrapper ir) -> + bench "checkerboard" $ nf (floodFill ir) (1,0) + ] + ] + where + cells :: IO Cells + cells = CaveAutomata.generate + CaveAutomata.defaultParams + (Dimensions 50 50) + <$> getStdGen + +newtype NFWrapper a = NFWrapper a + +instance NFData (NFWrapper a) where + rnf (NFWrapper x) = x `seq` () diff --git a/users/grfn/xanthous/bench/Xanthous/RandomBench.hs b/users/grfn/xanthous/bench/Xanthous/RandomBench.hs new file mode 100644 index 000000000000..fae4af92a7a5 --- /dev/null +++ b/users/grfn/xanthous/bench/Xanthous/RandomBench.hs @@ -0,0 +1,32 @@ +-------------------------------------------------------------------------------- +module Xanthous.RandomBench (benchmark, main) where +-------------------------------------------------------------------------------- +import Bench.Prelude +-------------------------------------------------------------------------------- +import Control.Parallel.Strategies +import Control.Monad.Random +-------------------------------------------------------------------------------- +import Xanthous.Random +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain [benchmark] + +-------------------------------------------------------------------------------- + +benchmark :: Benchmark +benchmark = bgroup "Random" + [ bgroup "chooseSubset" + [ bench "serially" $ + nf (evalRand $ chooseSubset (0.5 :: Double) [1 :: Int ..1000000]) + (mkStdGen 1234) + ] + , bgroup "choose weightedBy" + [ bench "serially" $ + nf (evalRand + . choose + . weightedBy (\n -> product [n, pred n .. 1]) + $ [1 :: Int ..1000000]) + (mkStdGen 1234) + ] + ] diff --git a/users/grfn/xanthous/build/generic-arbitrary-export-garbitrary.patch b/users/grfn/xanthous/build/generic-arbitrary-export-garbitrary.patch new file mode 100644 index 000000000000..f0c936bfca18 --- /dev/null +++ b/users/grfn/xanthous/build/generic-arbitrary-export-garbitrary.patch @@ -0,0 +1,12 @@ +diff --git a/src/Test/QuickCheck/Arbitrary/Generic.hs b/src/Test/QuickCheck/Arbitrary/Generic.hs +index fed6ab3..91f59f1 100644 +--- a/src/Test/QuickCheck/Arbitrary/Generic.hs ++++ b/src/Test/QuickCheck/Arbitrary/Generic.hs +@@ -23,6 +23,7 @@ The generated 'arbitrary' method is equivalent to + + module Test.QuickCheck.Arbitrary.Generic + ( Arbitrary(..) ++ , GArbitrary + , genericArbitrary + , genericShrink + ) where diff --git a/users/grfn/xanthous/build/hgeometry-fix-haddock.patch b/users/grfn/xanthous/build/hgeometry-fix-haddock.patch new file mode 100644 index 000000000000..748c65b3e0db --- /dev/null +++ b/users/grfn/xanthous/build/hgeometry-fix-haddock.patch @@ -0,0 +1,13 @@ +diff --git a/src/Data/Geometry/PlanarSubdivision/Merge.hs b/src/Data/Geometry/PlanarSubdivision/Merge.hs +index 1136114..3f4e7bb 100644 +--- a/src/Data/Geometry/PlanarSubdivision/Merge.hs ++++ b/src/Data/Geometry/PlanarSubdivision/Merge.hs +@@ -153,7 +153,7 @@ mergeWith' mergeFaces p1 p2 = PlanarSubdivision cs vd rd rf + -- we have to shift the number of the *Arcs*. Since every dart + -- consists of two arcs, we have to shift by numDarts / 2 + -- Furthermore, we take numFaces - 1 since we want the first +- -- *internal* face of p2 (the one with FaceId 1) to correspond with the first free ++ -- /internal/ face of p2 (the one with FaceId 1) to correspond with the first free + -- position (at index numFaces) + + cs = p1^.components <> p2'^.components diff --git a/users/grfn/xanthous/build/update-comonad-extras.patch b/users/grfn/xanthous/build/update-comonad-extras.patch new file mode 100644 index 000000000000..cd1dbe24d361 --- /dev/null +++ b/users/grfn/xanthous/build/update-comonad-extras.patch @@ -0,0 +1,92 @@ +diff --git a/comonad-extras.cabal b/comonad-extras.cabal +index fc3745a..77a2f0d 100644 +--- a/comonad-extras.cabal ++++ b/comonad-extras.cabal +@@ -1,7 +1,7 @@ + name: comonad-extras + category: Control, Comonads +-version: 4.0 ++version: 5.0 + x-revision: 1 + license: BSD3 + cabal-version: >= 1.6 + license-file: LICENSE +@@ -34,8 +34,8 @@ library + build-depends: + array >= 0.3 && < 0.6, +- base >= 4 && < 4.7, +- containers >= 0.4 && < 0.6, +- comonad >= 4 && < 5, ++ base >= 4 && < 5, ++ containers >= 0.6 && < 0.7, ++ comonad >= 5 && < 6, + distributive >= 0.3.2 && < 1, +- semigroupoids >= 4 && < 5, +- transformers >= 0.2 && < 0.4 ++ semigroupoids >= 5 && < 6, ++ transformers >= 0.5 && < 0.6 + + exposed-modules: + Control.Comonad.Store.Zipper +diff --git a/src/Control/Comonad/Store/Pointer.hs b/src/Control/Comonad/Store/Pointer.hs +index 5044a1e..8d4c62d 100644 +--- a/src/Control/Comonad/Store/Pointer.hs ++++ b/src/Control/Comonad/Store/Pointer.hs +@@ -41,7 +41,6 @@ module Control.Comonad.Store.Pointer + , module Control.Comonad.Store.Class + ) where + +-import Control.Applicative + import Control.Comonad + import Control.Comonad.Hoist.Class + import Control.Comonad.Trans.Class +@@ -51,27 +50,8 @@ import Control.Comonad.Env.Class + import Data.Functor.Identity + import Data.Functor.Extend + import Data.Array +- + #ifdef __GLASGOW_HASKELL__ + import Data.Typeable +-instance (Typeable i, Typeable1 w) => Typeable1 (PointerT i w) where +- typeOf1 diwa = mkTyConApp storeTTyCon [typeOf (i diwa), typeOf1 (w diwa)] +- where +- i :: PointerT i w a -> i +- i = undefined +- w :: PointerT i w a -> w a +- w = undefined +- +-instance (Typeable i, Typeable1 w, Typeable a) => Typeable (PointerT i w a) where +- typeOf = typeOfDefault +- +-storeTTyCon :: TyCon +-#if __GLASGOW_HASKELL__ < 704 +-storeTTyCon = mkTyCon "Control.Comonad.Trans.Store.Pointer.PointerT" +-#else +-storeTTyCon = mkTyCon3 "comonad-extras" "Control.Comonad.Trans.Store.Pointer" "PointerT" +-#endif +-{-# NOINLINE storeTTyCon #-} + #endif + + type Pointer i = PointerT i Identity +@@ -83,6 +63,9 @@ runPointer :: Pointer i a -> (Array i a, i) + runPointer (PointerT (Identity f) i) = (f, i) + + data PointerT i w a = PointerT (w (Array i a)) i ++#ifdef __GLASGOW_HASKELL__ ++ deriving Typeable ++#endif + + runPointerT :: PointerT i w a -> (w (Array i a), i) + runPointerT (PointerT g i) = (g, i) +diff --git a/src/Control/Comonad/Store/Zipper.hs b/src/Control/Comonad/Store/Zipper.hs +index 3b70c86..decc378 100644 +--- a/src/Control/Comonad/Store/Zipper.hs ++++ b/src/Control/Comonad/Store/Zipper.hs +@@ -15,7 +15,6 @@ + module Control.Comonad.Store.Zipper + ( Zipper, zipper, zipper1, unzipper, size) where + +-import Control.Applicative + import Control.Comonad (Comonad(..)) + import Data.Functor.Extend + import Data.Foldable diff --git a/users/grfn/xanthous/default.nix b/users/grfn/xanthous/default.nix new file mode 100644 index 000000000000..c0eca446c9ed --- /dev/null +++ b/users/grfn/xanthous/default.nix @@ -0,0 +1,26 @@ +{ depot ? (import ../../../. {}) +, pkgs ? depot.third_party.nixpkgs +, ... }: + +let + ignore = depot.third_party.gitignoreSource.gitignoreFilter ./.; + src = builtins.path { + name = "xanthous-source"; + path = ./.; + filter = path: type: + !(type == "directory" && builtins.baseNameOf path == "server") + && !(type == "directory" && builtins.baseNameOf path == "docs") + && (ignore path type + || builtins.baseNameOf path == "package.yaml"); + }; + # generated by cabal2nix + basePkg = pkgs.haskellPackages.callPackage ./pkg.nix { }; +in + +pkgs.haskell.lib.overrideCabal basePkg (default: { + inherit src; + version = "canon"; + configureFlags = [ + "--ghc-option=-Wall --ghc-option=-Werror" + ] ++ (default.configureFlags or []); +}) diff --git a/users/grfn/xanthous/docs/raw-types.org b/users/grfn/xanthous/docs/raw-types.org new file mode 100644 index 000000000000..e5bcda04268f --- /dev/null +++ b/users/grfn/xanthous/docs/raw-types.org @@ -0,0 +1,24 @@ +#+TITLE: Raw Types (WIP) + + +* Raw Types +** Item +*** Attributes +| name | type | commentary | +|-----------------+---------------------------+------------------------------------------------------------------| +| name | string | | +| description | string | Not capitalized, should usually start with an indefinite article | +| longDescription | string | Capitalized, should usually start with an indefinite article | +| char | [[*EntityChar][EntityChar]] | | +| wieldable | [[*EntityWieldable][EntityWieldable]] | | +| density | number , [number, number] | Density, or range for random density, in g/m³ | +| volume | number , [number, number] | Volume, or range for random volume, in m³ | +* Data Types +** EntityChar +*** Attributes +| name | type | commentary | +|-------+------+-------------------------------------------------------| +| char | char | How the entity is displayed when dropped on the floor | +| style | Attr | | +** TODO EntityWieldable +** TODO Attr diff --git a/users/grfn/xanthous/hie.yaml b/users/grfn/xanthous/hie.yaml new file mode 100644 index 000000000000..e7cf01d158e5 --- /dev/null +++ b/users/grfn/xanthous/hie.yaml @@ -0,0 +1,10 @@ +cradle: + cabal: + - path: './src' + component: 'lib:xanthous' + - path: './test' + component: 'test:test' + - path: './app' + component: 'exe:xanthous' + - path: './bench' + component: 'bench:benchmark' diff --git a/users/grfn/xanthous/nixpkgs.nix b/users/grfn/xanthous/nixpkgs.nix new file mode 100644 index 000000000000..7d7c16440545 --- /dev/null +++ b/users/grfn/xanthous/nixpkgs.nix @@ -0,0 +1,3 @@ +args: +let pkgs = (import ../../../. args).third_party; +in pkgs // { inherit pkgs; } diff --git a/users/grfn/xanthous/package.yaml b/users/grfn/xanthous/package.yaml new file mode 100644 index 000000000000..630dc69c11d3 --- /dev/null +++ b/users/grfn/xanthous/package.yaml @@ -0,0 +1,156 @@ +name: xanthous +version: 0.1.0.0 +github: "glittershark/xanthous" +license: GPL-3 +author: "Griffin Smith" +maintainer: "root@gws.fyi" +copyright: "2019 Griffin Smith" + +extra-source-files: +- README.org + +synopsis: A WIP TUI RPG +category: Game + +description: Please see the README on GitHub at <https://github.com/glittershark/xanthous> + +dependencies: +- base + +- aeson +- array +- async +- QuickCheck +- quickcheck-text +- quickcheck-instances +- brick +- bifunctors +- checkers +- classy-prelude +- comonad +- comonad-extras +- constraints +- containers +- criterion +- data-default +- data-interval +- deepseq +- directory +- fgl +- fgl-arbitrary +- file-embed +- filepath +- generic-arbitrary +- generic-lens +- groups +- hgeometry +- hgeometry-combinatorial +- JuicyPixels +- lens +- lifted-async +- linear +- megaparsec +- mmorph +- monad-control +- MonadRandom +- mtl +- optparse-applicative +- parallel +- parser-combinators +- pointed +- random +- random-fu +- random-extras +- random-source +- raw-strings-qq +- reflection +- Rasterific +- splitmix +- streams +- stache +- semigroups +- semigroupoids +- tomland +- transformers +- text +- text-zipper +- vector +- vty +- witherable +- yaml +- zlib + +default-extensions: +- BlockArguments +- ConstraintKinds +- DataKinds +- DeriveAnyClass +- DeriveGeneric +- DerivingStrategies +- DerivingVia +- FlexibleContexts +- FlexibleInstances +- FunctionalDependencies +- GADTSyntax +- GeneralizedNewtypeDeriving +- KindSignatures +- StandaloneKindSignatures +- LambdaCase +- MultiWayIf +- NoImplicitPrelude +- NoStarIsType +- OverloadedStrings +- PolyKinds +- RankNTypes +- ScopedTypeVariables +- TupleSections +- TypeApplications +- TypeFamilies +- TypeOperators +- ViewPatterns + +ghc-options: +- -Wall + +library: + source-dirs: src + +executable: + source-dirs: app + main: Main.hs + dependencies: + - xanthous + ghc-options: + - -threaded + - -rtsopts + - -with-rtsopts=-N + - -O2 + +tests: + test: + main: Spec.hs + source-dirs: test + ghc-options: + - -threaded + - -rtsopts + - -with-rtsopts=-N + - -O0 + dependencies: + - xanthous + - tasty + - tasty-hunit + - tasty-quickcheck + - tasty-rerun + - lens-properties + +benchmarks: + benchmark: + main: Bench.hs + source-dirs: bench + ghc-options: + - -threaded + - -rtsopts + - -with-rtsopts=-N + dependencies: + - xanthous + - criterion diff --git a/users/grfn/xanthous/pkg.nix b/users/grfn/xanthous/pkg.nix new file mode 100644 index 000000000000..0f0dbfc9822d --- /dev/null +++ b/users/grfn/xanthous/pkg.nix @@ -0,0 +1,80 @@ +{ mkDerivation, aeson, array, async, base, bifunctors, brick +, checkers, classy-prelude, comonad, comonad-extras, constraints +, containers, criterion, data-default, data-interval, deepseq +, directory, fgl, fgl-arbitrary, file-embed, filepath +, generic-arbitrary, generic-lens, groups, hgeometry +, hgeometry-combinatorial, hpack, JuicyPixels, lens +, lens-properties, lib, lifted-async, linear, megaparsec, mmorph +, monad-control, MonadRandom, mtl, optparse-applicative, parallel +, parser-combinators, pointed, QuickCheck, quickcheck-instances +, quickcheck-text, random, random-extras, random-fu, random-source +, Rasterific, raw-strings-qq, reflection, semigroupoids, semigroups +, splitmix, stache, streams, tasty, tasty-hunit, tasty-quickcheck +, tasty-rerun, text, text-zipper, tomland, transformers, vector +, vty, witherable, yaml, zlib +}: +mkDerivation { + pname = "xanthous"; + version = "0.1.0.0"; + src = ./.; + isLibrary = true; + isExecutable = true; + libraryHaskellDepends = [ + aeson array async base bifunctors brick checkers classy-prelude + comonad comonad-extras constraints containers criterion + data-default data-interval deepseq directory fgl fgl-arbitrary + file-embed filepath generic-arbitrary generic-lens groups hgeometry + hgeometry-combinatorial JuicyPixels lens lifted-async linear + megaparsec mmorph monad-control MonadRandom mtl + optparse-applicative parallel parser-combinators pointed QuickCheck + quickcheck-instances quickcheck-text random random-extras random-fu + random-source Rasterific raw-strings-qq reflection semigroupoids + semigroups splitmix stache streams text text-zipper tomland + transformers vector vty witherable yaml zlib + ]; + libraryToolDepends = [ hpack ]; + executableHaskellDepends = [ + aeson array async base bifunctors brick checkers classy-prelude + comonad comonad-extras constraints containers criterion + data-default data-interval deepseq directory fgl fgl-arbitrary + file-embed filepath generic-arbitrary generic-lens groups hgeometry + hgeometry-combinatorial JuicyPixels lens lifted-async linear + megaparsec mmorph monad-control MonadRandom mtl + optparse-applicative parallel parser-combinators pointed QuickCheck + quickcheck-instances quickcheck-text random random-extras random-fu + random-source Rasterific raw-strings-qq reflection semigroupoids + semigroups splitmix stache streams text text-zipper tomland + transformers vector vty witherable yaml zlib + ]; + testHaskellDepends = [ + aeson array async base bifunctors brick checkers classy-prelude + comonad comonad-extras constraints containers criterion + data-default data-interval deepseq directory fgl fgl-arbitrary + file-embed filepath generic-arbitrary generic-lens groups hgeometry + hgeometry-combinatorial JuicyPixels lens lens-properties + lifted-async linear megaparsec mmorph monad-control MonadRandom mtl + optparse-applicative parallel parser-combinators pointed QuickCheck + quickcheck-instances quickcheck-text random random-extras random-fu + random-source Rasterific raw-strings-qq reflection semigroupoids + semigroups splitmix stache streams tasty tasty-hunit + tasty-quickcheck tasty-rerun text text-zipper tomland transformers + vector vty witherable yaml zlib + ]; + benchmarkHaskellDepends = [ + aeson array async base bifunctors brick checkers classy-prelude + comonad comonad-extras constraints containers criterion + data-default data-interval deepseq directory fgl fgl-arbitrary + file-embed filepath generic-arbitrary generic-lens groups hgeometry + hgeometry-combinatorial JuicyPixels lens lifted-async linear + megaparsec mmorph monad-control MonadRandom mtl + optparse-applicative parallel parser-combinators pointed QuickCheck + quickcheck-instances quickcheck-text random random-extras random-fu + random-source Rasterific raw-strings-qq reflection semigroupoids + semigroups splitmix stache streams text text-zipper tomland + transformers vector vty witherable yaml zlib + ]; + prePatch = "hpack"; + homepage = "https://github.com/glittershark/xanthous#readme"; + description = "A WIP TUI RPG"; + license = lib.licenses.gpl3Only; +} diff --git a/users/grfn/xanthous/server/.envrc b/users/grfn/xanthous/server/.envrc new file mode 100644 index 000000000000..051d09d292a8 --- /dev/null +++ b/users/grfn/xanthous/server/.envrc @@ -0,0 +1 @@ +eval "$(lorri direnv)" diff --git a/users/grfn/xanthous/server/.gitignore b/users/grfn/xanthous/server/.gitignore new file mode 100644 index 000000000000..2f7896d1d136 --- /dev/null +++ b/users/grfn/xanthous/server/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/users/grfn/xanthous/server/Cargo.lock b/users/grfn/xanthous/server/Cargo.lock new file mode 100644 index 000000000000..46488d4575f2 --- /dev/null +++ b/users/grfn/xanthous/server/Cargo.lock @@ -0,0 +1,1937 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher", + "cpufeatures", + "ctr", + "opaque-debug", +] + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "atomic-shim" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d20fdac7156779a1a30d970e838195558b4810dd06aa69e7c7461bdc518edf9b" +dependencies = [ + "crossbeam", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "321629d8ba6513061f26707241fa9bc89524ff1cd7a915a97ef0c62c666ce1b6" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64ct" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b4d9b1225d28d360ec6a231d65af1fd99a2a095154c8040689617290569c5c" + +[[package]] +name = "bcrypt-pbkdf" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c38c03b9506bd92bf1ef50665a81eda156f615438f7654bffba58907e6149d7" +dependencies = [ + "blowfish", + "crypto-mac", + "pbkdf2", + "sha2", + "zeroize", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-modes" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb03d1bed155d89dce0f845b7899b18a9a163e148fd004e1c28421a783e2d8e" +dependencies = [ + "block-padding", + "cipher", +] + +[[package]] +name = "block-padding" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" + +[[package]] +name = "blowfish" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe3ff3fc1de48c1ac2e3341c4df38b0d1bfb8fdf04632a187c8b75aaa319a7ab" +dependencies = [ + "byteorder", + "cipher", + "opaque-debug", +] + +[[package]] +name = "bumpalo" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "cc" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "winapi", +] + +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", +] + +[[package]] +name = "clap" +version = "3.0.0-beta.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feff3878564edb93745d58cf63e17b63f24142506e7a20c87a5521ed7bfb1d63" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "indexmap", + "lazy_static", + "os_str_bytes", + "strsim", + "termcolor", + "textwrap", + "unicase", +] + +[[package]] +name = "clap_derive" +version = "3.0.0-beta.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b15c6b4f786ffb6192ffe65a36855bc1fc2444bcd0945ae16748dcd6ed7d0d3" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "color-eyre" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f1885697ee8a177096d42f158922251a41973117f6d8a234cee94b9509157b7" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6eee477a4a8a72f4addd4de416eb56d54bc307b284d6601bafdee1f4ea462d1" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + +[[package]] +name = "cpufeatures" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch 0.8.2", + "crossbeam-queue", + "crossbeam-utils 0.7.2", +] + +[[package]] +name = "crossbeam-channel" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" +dependencies = [ + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20ff29ded3204c5106278a81a38f4b482636ed4fa1e6cfbeef193291beb29ed" +dependencies = [ + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset 0.5.6", + "scopeguard", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.5", + "lazy_static", + "memoffset 0.6.4", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "cryptovec" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc7fa13a6bbb2322d325292c57f4c8e7291595506f8289968a0eb61c3130bdf" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher", +] + +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + +[[package]] +name = "data-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dirs" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30baa043103c9d0c2a57cf537cc2f35623889dc0d405e6c3cccfadbc81c71309" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + +[[package]] +name = "eyre" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221239d1d5ea86bf5d6f91c9d6bc3646ffe471b08ff9b0f91c44f115ac969d2b" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "futures" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" + +[[package]] +name = "futures-executor" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" + +[[package]] +name = "futures-macro" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +dependencies = [ + "autocfg", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" + +[[package]] +name = "futures-task" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" + +[[package]] +name = "futures-util" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +dependencies = [ + "autocfg", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "proc-macro-hack", + "proc-macro-nested", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hmac" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "http" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503" + +[[package]] +name = "httpdate" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + +[[package]] +name = "hyper" +version = "0.14.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b91bb1f221b6ea1f1e4371216b70f40748774c2fb5971b450c07773fb92d26b" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + +[[package]] +name = "indexmap" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + +[[package]] +name = "itoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "js-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9ffccd38c451a86bf13657df244e9c3f37493cce8e5e21e940963777acc84" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe5e23404da5b4f555ef85ebed98fb4083e55a00c317800bc2a50ede9f3d219" + +[[package]] +name = "libsodium-sys" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b779387cd56adfbc02ea4a668e704f729be8d6a6abd2c27ca5ee537849a92fd" +dependencies = [ + "cc", + "libc", + "pkg-config", + "walkdir", +] + +[[package]] +name = "lock_api" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "mach" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" +dependencies = [ + "libc", +] + +[[package]] +name = "matchers" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f099785f7595cc4b4553a174ce30dd7589ef93391ff414dbb67f62392b9e0ce1" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + +[[package]] +name = "memchr" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "memoffset" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" +dependencies = [ + "autocfg", +] + +[[package]] +name = "metrics" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00f42f354a2ed4894db863b3a4db47aef2d2e4435b937221749bd37a8a7aaa8" +dependencies = [ + "ahash", + "metrics-macros", + "proc-macro-hack", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343a5ceb38235928e7a5687412590f07e6d281522dcd9ff51246f8856eef5fe5" +dependencies = [ + "hyper", + "ipnet", + "metrics", + "metrics-util", + "parking_lot", + "quanta", + "thiserror", + "tokio", +] + +[[package]] +name = "metrics-macros" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa72e4a3d157986dd2565c82ecbddcc23941513669a3766b938f6b72eb87f3f" +dependencies = [ + "lazy_static", + "proc-macro-hack", + "proc-macro2", + "quote", + "regex", + "syn", +] + +[[package]] +name = "metrics-util" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c9b6aee519e1461b678952d3671652bb341d0664b1188f895a436a4e2e6ffa" +dependencies = [ + "ahash", + "aho-corasick", + "atomic-shim", + "crossbeam-epoch 0.9.5", + "crossbeam-utils 0.8.5", + "dashmap", + "hashbrown", + "indexmap", + "metrics", + "num_cpus", + "ordered-float", + "parking_lot", + "quanta", + "radix_trie", + "sketches-ddsketch", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8067b404fe97c70829f082dec8bcf4f71225d7eaea1d8645349cb76fa06205cc" +dependencies = [ + "libc", + "log", + "miow", + "ntapi", + "winapi", +] + +[[package]] +name = "miow" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" +dependencies = [ + "winapi", +] + +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] + +[[package]] +name = "nix" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f305c2c2e4c39a82f7bf0bf65fb557f9070ce06781d4f2454295cc34b1c43188" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset 0.6.4", +] + +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi", +] + +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67ac1d3f9a1d3616fd9a60c8d74296f22406a238b6a72f5cc1e6f314df4ffbf9" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "ordered-float" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97c9d06878b3a851e8026ef94bf7fef9ba93062cd412601da4d9cf369b1cc62d" +dependencies = [ + "num-traits", +] + +[[package]] +name = "os_str_bytes" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addaa943333a514159c80c97ff4a93306530d965d27e139188283cd13e06a799" +dependencies = [ + "memchr", +] + +[[package]] +name = "owo-colors" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2386b4ebe91c2f7f51082d4cefa145d030e33a1842a96b12e4885cc3c01f7a55" + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "password-hash" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e0b28ace46c5a396546bcf443bf422b57049617433d8854227352a4a9b24e7" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + +[[package]] +name = "pbkdf2" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" +dependencies = [ + "base64ct", + "crypto-mac", + "hmac", + "password-hash", + "sha2", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12295df4f294471248581bc09bef3c38a5e46f1e36d6a37353621a0c6c357e1f" + +[[package]] +name = "ppv-lite86" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro-nested" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" + +[[package]] +name = "proc-macro2" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba508cc11742c0dc5c1659771673afbab7a0efab23aa17e854cbab0837ed0b43" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils 0.8.5", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + +[[package]] +name = "quote" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bc8cc6a5f2e3655e0899c1b848643b2562f853f114bfec7be120678e3ace05" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + +[[package]] +name = "rand" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "raw-cpuid" +version = "10.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "929f54e29691d4e6a9cc558479de70db7aa3d98cd6fe7ab86d7507aa2886b9d2" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_syscall" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" +dependencies = [ + "getrandom", + "redox_syscall", +] + +[[package]] +name = "regex" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913" + +[[package]] +name = "serde_derive" +version = "1.0.130" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e466864e431129c7e0d3476b92f20458e5879919a0596c6472738d9fa2d342f8" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b69f9a4c9740d74c5baa3fd2e547f9525fa8088a8a958e0ca2409a514e33f5fa" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpufeatures", + "digest", + "opaque-debug", +] + +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "sketches-ddsketch" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a77a8fd93886010f05e7ea0720e569d6d16c65329dbe3ec033bbbccccb017b" + +[[package]] +name = "slab" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" + +[[package]] +name = "smallvec" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309" + +[[package]] +name = "socket2" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0066c8d12af8b5acd21e00547c3797fde4e8677254a7ee429176ccebbe93dd80" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + +[[package]] +name = "thrussh" +version = "0.33.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6540238a9adf83df6e66541c182a52acf892ab335595ca965c229ade8536f8" +dependencies = [ + "bitflags", + "byteorder", + "cryptovec", + "digest", + "flate2", + "futures", + "generic-array", + "log", + "rand", + "sha2", + "thiserror", + "thrussh-keys", + "thrussh-libsodium", + "tokio", +] + +[[package]] +name = "thrussh-keys" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a72cc51a2932b18d92f7289332d8564cec4a5014063722a9d3fdca52c5d8f5ab" +dependencies = [ + "aes", + "bcrypt-pbkdf", + "bit-vec", + "block-modes", + "byteorder", + "cryptovec", + "data-encoding", + "dirs", + "futures", + "hmac", + "log", + "md5", + "num-bigint", + "num-integer", + "pbkdf2", + "rand", + "serde", + "serde_derive", + "sha2", + "thiserror", + "thrussh-libsodium", + "tokio", + "tokio-stream", + "yasna", +] + +[[package]] +name = "thrussh-libsodium" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe89c70d27b1cb92e13bc8af63493e890d0de46dae4df0e28233f62b4ed9500" +dependencies = [ + "lazy_static", + "libc", + "libsodium-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "tokio" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "588b2d10a336da58d877567cd8fb8a14b463e2104910f8132cd054b4b96e29ee" +dependencies = [ + "autocfg", + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "pin-project-lite", + "signal-hook-registry", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "114383b041aa6212c579467afa0075fbbdd0718de036100bc0ba7961d8cb9095" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4ed65637b8390770814083d20756f87bfa2c21bf2f110babdc5438351746e4" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "tracing-error" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d7c0b83d4a500748fa5879461652b361edf5c9d51ede2a2ac03875ca185e24" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-log" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6923477a48e41c1951f1999ef8bb5a3023eb723ceadafe78ffb65dc366761e3" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb65ea441fbb84f9f6748fd496cf7f63ec9af5bca94dd86456978d055e8eb28b" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" +dependencies = [ + "ansi_term", + "chrono", + "lazy_static", + "matchers", + "regex", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasm-bindgen" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "632f73e236b219150ea279196e54e610f5dbafa5d61786303d4da54f84e47fce" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a317bf8f9fba2476b4b2c85ef4c4af8ff39c3c7f0cdfeed4f82c34a880aa837b" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56146e7c495528bf6587663bea13a8eb588d39b36b679d83972e1a2dbbdacf9" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e0eea25835f8abdc585cd3021b3deb11543c6fe226dcd30b228857c5c5ab" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0237232789cf037d5480773fe568aac745bfe2afbc11a863e97901780a6b47cc" + +[[package]] +name = "web-sys" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38eb105f1c59d9eaa6b5cdc92b859d85b926e82cb2e0945cd0c9259faa6fe9fb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xanthous-server" +version = "0.1.0" +dependencies = [ + "base64ct", + "clap", + "color-eyre", + "eyre", + "futures", + "libc", + "metrics", + "metrics-exporter-prometheus", + "nix", + "pbkdf2", + "tempfile", + "thrussh", + "thrussh-keys", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "yasna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262a29d0e61ccf2b6190d7050d4b237535fc76ce4c1210d9caa316f71dffa75" +dependencies = [ + "bit-vec", + "num-bigint", +] + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" diff --git a/users/grfn/xanthous/server/Cargo.toml b/users/grfn/xanthous/server/Cargo.toml new file mode 100644 index 000000000000..adb2a02391bd --- /dev/null +++ b/users/grfn/xanthous/server/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "xanthous-server" +version = "0.1.0" +edition = "2018" + +[dependencies] +clap = "3.0.0-beta.5" +color-eyre = "0.5.11" +eyre = "0.6.5" +thrussh = "0.33.5" +thrussh-keys = "0.21.0" +tracing = "0.1.29" +tracing-subscriber = "0.2.25" +metrics = "0.17.0" +metrics-exporter-prometheus = "0.6.1" +futures = "0.3.17" +libc = "0.2.103" +nix = "0.23.0" + +# Pins for rust 1.55 (2018 edition) until we have 1.56 in nixpkgs-unstable +pbkdf2 = "<0.9" +base64ct = "<1.2" + +[dependencies.tokio] +version = "1.13" +features = ["rt", "rt-multi-thread", "macros", "net", "process", "fs", "signal"] + +[dev-dependencies] +tempfile = "3.2.0" diff --git a/users/grfn/xanthous/server/default.nix b/users/grfn/xanthous/server/default.nix new file mode 100644 index 000000000000..0b3900e4d5fe --- /dev/null +++ b/users/grfn/xanthous/server/default.nix @@ -0,0 +1,14 @@ +args@{ + depot ? import ../../../.. {} +, pkgs ? depot.third_party.nixpkgs +, ... +}: + +depot.third_party.naersk.buildPackage { + name = "xanthous-server"; + version = "0.0.1"; + src = depot.third_party.gitignoreSource ./.; + passthru = { + docker = import ./docker.nix args; + }; +} diff --git a/users/grfn/xanthous/server/docker.nix b/users/grfn/xanthous/server/docker.nix new file mode 100644 index 000000000000..a62943c2b077 --- /dev/null +++ b/users/grfn/xanthous/server/docker.nix @@ -0,0 +1,19 @@ +{ depot ? import ../../../.. {} +, pkgs ? depot.third_party.nixpkgs +, ... +}: + +let + inherit (depot.users.grfn) xanthous; + xanthous-server = xanthous.server; +in pkgs.dockerTools.buildLayeredImage { + name = "xanthous-server"; + tag = "latest"; + contents = [ xanthous xanthous-server ]; + config = { + Cmd = [ + "${xanthous-server}/bin/xanthous-server" + "--xanthous-binary-path" "${xanthous}/bin/xanthous" + ]; + }; +} diff --git a/users/grfn/xanthous/server/module.nix b/users/grfn/xanthous/server/module.nix new file mode 100644 index 000000000000..73ac276caf4a --- /dev/null +++ b/users/grfn/xanthous/server/module.nix @@ -0,0 +1,48 @@ +{ config, lib, pkgs, depot, ... }: + +let + cfg = config.services.xanthous-server; +in { + options = with lib; { + services.xanthous-server = { + enable = mkEnableOption "xanthous server"; + + port = mkOption { + type = types.int; + default = 2222; + description = "Port to listen to for SSH connections"; + }; + + metricsPort = mkOption { + type = types.int; + default = 9000; + description = "Port to listen to for prometheus metrics"; + }; + + image = mkOption { + type = types.package; + default = depot.users.grfn.xanthous.server.docker; + description = "OCI image file to run"; + }; + + ed25519SecretKeyFile = mkOption { + type = with types; uniq string; + description = "Path to the ed25519 secret key for the server"; + }; + }; + }; + + config = lib.mkIf cfg.enable { + virtualisation.oci-containers.containers."xanthous-server" = { + autoStart = true; + image = "${cfg.image.imageName}:${cfg.image.imageTag}"; + imageFile = cfg.image; + ports = [ + "${toString cfg.port}:22" + "${toString cfg.metricsPort}:9000" + ]; + environment.SECRET_KEY_FILE = "/secret-key"; + volumes = [ "/etc/secrets/xanthous-server-secret-key:/secret-key" ]; + }; + }; +} diff --git a/users/grfn/xanthous/server/shell.nix b/users/grfn/xanthous/server/shell.nix new file mode 100644 index 000000000000..a6747175f105 --- /dev/null +++ b/users/grfn/xanthous/server/shell.nix @@ -0,0 +1,11 @@ +let + depot = import ../../../.. {}; + pkgs = depot.third_party.nixpkgs; +in + +pkgs.mkShell { + buildInputs = with pkgs; [ + rustup + rust-analyzer + ]; +} diff --git a/users/grfn/xanthous/server/src/main.rs b/users/grfn/xanthous/server/src/main.rs new file mode 100644 index 000000000000..ed8f831c7d3f --- /dev/null +++ b/users/grfn/xanthous/server/src/main.rs @@ -0,0 +1,388 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::pin::Pin; +use std::process::Command; +use std::str; +use std::sync::Arc; + +use clap::Parser; +use color_eyre::eyre::Result; +use eyre::{bail, Context}; +use futures::future::{ready, Ready}; +use futures::Future; +use metrics_exporter_prometheus::PrometheusBuilder; +use nix::pty::Winsize; +use pty::ChildHandle; +use thrussh::ChannelId; +use thrussh::{ + server::{self, Auth, Session}, + CryptoVec, +}; +use thrussh_keys::decode_secret_key; +use thrussh_keys::key::KeyPair; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::select; +use tokio::time::Instant; +use tracing::{debug, error, info, info_span, trace, warn, Instrument}; +use tracing_subscriber::EnvFilter; + +use crate::pty::WaitPid; + +mod metrics; +mod pty; + +use crate::metrics::reported::*; +use crate::metrics::{decrement_gauge, histogram, increment_counter, increment_gauge}; + +/// SSH-compatible server for playing Xanthous +#[derive(Parser, Debug)] +struct Opts { + /// Address to bind to + #[clap(long, short = 'a', default_value = "0.0.0.0:22")] + address: String, + + /// Address to listen to for metrics + #[clap(long, default_value = "0.0.0.0:9000")] + metrics_address: SocketAddr, + + /// Format to use when emitting log events + #[clap( + long, + env = "LOG_FORMAT", + default_value = "full", + possible_values = &["compact", "full", "pretty", "json"] + )] + log_format: String, + + /// Full path to the xanthous binary + #[clap(long, env = "XANTHOUS_BINARY_PATH")] + xanthous_binary_path: String, + + /// Path to a file containing the ed25519 secret key for the server + #[clap(long, env = "SECRET_KEY_FILE")] + secret_key_file: PathBuf, + + /// Level to log at + #[clap(long, env = "LOG_LEVEL", default_value = "info")] + log_level: String, +} + +impl Opts { + async fn read_secret_key(&self) -> Result<KeyPair> { + let mut file = File::open(&self.secret_key_file) + .await + .context("Reading secret key file")?; + let mut secret_key = Vec::with_capacity(464); + file.read_to_end(&mut secret_key).await?; + Ok(decode_secret_key(str::from_utf8(&secret_key)?, None)?) + } + + async fn ssh_server_config(&self) -> Result<server::Config> { + let key_pair = self.read_secret_key().await?; + + Ok(server::Config { + server_id: "SSH-2.0-xanthous".to_owned(), + keys: vec![key_pair], + ..Default::default() + }) + } + + fn init_logging(&self) -> Result<()> { + let filter = EnvFilter::try_new(&self.log_level)?; + let s = tracing_subscriber::fmt().with_env_filter(filter); + + match self.log_format.as_str() { + "compact" => s.compact().init(), + "full" => s.init(), + "pretty" => s.pretty().init(), + "json" => s.json().with_current_span(true).init(), + _ => bail!("Invalid log format `{}`"), + } + + Ok(()) + } +} + +struct Handler { + address: SocketAddr, + xanthous_binary_path: &'static str, + username: Option<String>, + child: Option<ChildHandle>, +} + +async fn run_child( + mut child: pty::Child, + mut server_handle: server::Handle, + channel_id: ChannelId, +) -> Result<()> { + let mut buf = [0; 2048]; + loop { + select! { + r = child.tty.read(&mut buf) => { + let read_bytes = r?; + if read_bytes == 0 { + info!("EOF received from process"); + let _ = server_handle.close(channel_id).await; + return Ok(()) + } else { + trace!(?read_bytes, "read bytes from child"); + let _ = server_handle.data(channel_id, CryptoVec::from_slice(&buf[..read_bytes])).await; + } + } + status = WaitPid::new(child.pid) => { + match status { + Ok(_status) => info!("Child exited"), + Err(error) => error!(%error, "Child failed"), + } + let _ = server_handle.close(channel_id).await; + return Ok(()) + } + } + } +} + +impl Handler { + async fn spawn_shell( + &mut self, + mut handle: server::Handle, + channel_id: ChannelId, + term: String, + winsize: Winsize, + ) -> Result<()> { + let mut cmd = Command::new(self.xanthous_binary_path); + cmd.env("TERM", term); + if let Some(username) = &self.username { + cmd.args(["--name", username]); + } + cmd.arg("--disable-saving"); + + let child = pty::spawn(cmd, Some(winsize), None).await?; + info!(pid = %child.pid, "Spawned child"); + increment_gauge!(RUNNING_PROCESSES, 1.0); + self.child = Some(child.handle().await?); + tokio::spawn( + async move { + let span = info_span!("child", pid = %child.pid); + if let Err(error) = run_child(child, handle.clone(), channel_id) + .instrument(span.clone()) + .await + { + span.in_scope(|| error!(%error, "Error running child")); + let _ = handle.close(channel_id).await; + } + decrement_gauge!(RUNNING_PROCESSES, 1.0); + } + .in_current_span(), + ); + Ok(()) + } +} + +#[allow(clippy::type_complexity)] +impl server::Handler for Handler { + type Error = eyre::Error; + type FutureAuth = Ready<Result<(Self, Auth)>>; + type FutureUnit = Pin<Box<dyn Future<Output = Result<(Self, Session)>> + Send + 'static>>; + type FutureBool = Ready<Result<(Self, Session, bool)>>; + + fn finished_auth(self, auth: Auth) -> Self::FutureAuth { + ready(Ok((self, auth))) + } + + fn finished_bool(self, b: bool, session: Session) -> Self::FutureBool { + ready(Ok((self, session, b))) + } + + fn finished(self, session: Session) -> Self::FutureUnit { + Box::pin(ready(Ok((self, session)))) + } + + fn auth_none(mut self, username: &str) -> Self::FutureAuth { + info!(%username, "Accepted new connection"); + self.username = Some(username.to_owned()); + self.finished_auth(Auth::Accept) + } + + fn auth_password(mut self, username: &str, _password: &str) -> Self::FutureAuth { + info!(%username, "Accepted new connection"); + self.username = Some(username.to_owned()); + self.finished_auth(Auth::Accept) + } + + fn auth_publickey( + mut self, + username: &str, + _: &thrussh_keys::key::PublicKey, + ) -> Self::FutureAuth { + info!(%username, "Accepted new connection"); + self.username = Some(username.to_owned()); + self.finished_auth(Auth::Accept) + } + + fn pty_request( + mut self, + channel: thrussh::ChannelId, + term: &str, + col_width: u32, + row_height: u32, + pix_width: u32, + pix_height: u32, + modes: &[(thrussh::Pty, u32)], + session: Session, + ) -> Self::FutureUnit { + let term = term.to_owned(); + let modes = modes.to_vec(); + Box::pin(async move { + debug!( + %term, + %col_width, + %row_height, + %pix_width, + %pix_height, + ?modes, + "PTY Requested" + ); + + self.spawn_shell( + session.handle(), + channel, + term, + Winsize { + ws_row: row_height as _, + ws_col: col_width as _, + ws_xpixel: pix_width as _, + ws_ypixel: pix_height as _, + }, + ) + .await?; + + Ok((self, session)) + }) + } + + fn window_change_request( + mut self, + _channel: ChannelId, + col_width: u32, + row_height: u32, + pix_width: u32, + pix_height: u32, + session: Session, + ) -> Self::FutureUnit { + Box::pin(async move { + if let Some(child) = self.child.as_mut() { + trace!(%row_height, %col_width, "Window resize request received"); + child + .resize_window(Winsize { + ws_row: row_height as _, + ws_col: col_width as _, + ws_xpixel: pix_width as _, + ws_ypixel: pix_height as _, + }) + .await?; + } else { + warn!("Resize request received without child process; ignoring"); + } + + Ok((self, session)) + }) + } + + fn data( + mut self, + _channel: thrussh::ChannelId, + data: &[u8], + session: Session, + ) -> Self::FutureUnit { + trace!(data = %String::from_utf8_lossy(data), raw_data = ?data); + let data = data.to_owned(); + Box::pin(async move { + if let Some(child) = self.child.as_mut() { + child.write_all(&data).await?; + } else { + warn!("Data received without child process; ignoring"); + } + + Ok((self, session)) + }) + } +} + +#[tokio::main] +async fn main() -> Result<()> { + color_eyre::install()?; + let opts = Box::leak::<'static>(Box::new(Opts::parse())); + opts.init_logging()?; + PrometheusBuilder::new() + .listen_address(opts.metrics_address) + .install()?; + metrics::register(); + + let config = Arc::new(opts.ssh_server_config().await?); + info!(address = %opts.address, "Listening for new SSH connections"); + let listener = TcpListener::bind(&opts.address).await?; + + loop { + let (stream, address) = listener.accept().await?; + increment_counter!(CONNECTIONS_ACCEPTED); + increment_gauge!(ACTIVE_CONNECTIONS, 1.0); + let config = config.clone(); + let handler = Handler { + xanthous_binary_path: &opts.xanthous_binary_path, + address, + username: None, + child: None, + }; + tokio::spawn(async move { + let span = info_span!("client", address = %handler.address); + let start = Instant::now(); + if let Err(error) = server::run_stream(config, stream, handler) + .instrument(span.clone()) + .await + { + span.in_scope(|| error!(%error)); + } + let duration = start.elapsed(); + span.in_scope(|| info!(duration_ms = %duration.as_millis(), "Client disconnected")); + histogram!(CONNECTION_DURATION, duration); + decrement_gauge!(ACTIVE_CONNECTIONS, 1.0); + }); + } +} + +#[cfg(test)] +mod tests { + use tempfile::NamedTempFile; + + use super::*; + + #[tokio::test] + async fn read_secret_key() { + use std::io::Write; + + let mut file = NamedTempFile::new().unwrap(); + file.write_all( + b" +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW +QyNTUxOQAAACAYz80xcK7jYxZMAl6apIHKRtB0Z2U78gG39c1QaIhgMwAAAJB9vxK9fb8S +vQAAAAtzc2gtZWQyNTUxOQAAACAYz80xcK7jYxZMAl6apIHKRtB0Z2U78gG39c1QaIhgMw +AAAEDNZ0d3lLNBGU6Im4JOpr490TOjm+cB7kMVXjVg3iCowBjPzTFwruNjFkwCXpqkgcpG +0HRnZTvyAbf1zVBoiGAzAAAACHRlc3Qta2V5AQIDBAU= +-----END OPENSSH PRIVATE KEY----- +", + ) + .unwrap(); + + let opts: Opts = Opts::parse_from(&[ + "xanthous-server".as_ref(), + "--xanthous-binary-path".as_ref(), + "/bin/xanthous".as_ref(), + "--secret-key-file".as_ref(), + file.path().as_os_str(), + ]); + opts.read_secret_key().await.unwrap(); + } +} diff --git a/users/grfn/xanthous/server/src/metrics.rs b/users/grfn/xanthous/server/src/metrics.rs new file mode 100644 index 000000000000..6912cdd9c9ee --- /dev/null +++ b/users/grfn/xanthous/server/src/metrics.rs @@ -0,0 +1,24 @@ +pub use ::metrics::*; + +pub mod reported { + /// Counter: Connections accepted on the TCP listener + pub const CONNECTIONS_ACCEPTED: &str = "ssh.connections.accepted"; + + /// Histogram: Connection duration + pub const CONNECTION_DURATION: &str = "ssh.connections.duration"; + + /// Gauge: Currently active connections + pub const ACTIVE_CONNECTIONS: &str = "ssh.connections.active"; + + /// Gauge: Currently running xanthous processes + pub const RUNNING_PROCESSES: &str = "ssh.child.processes"; +} + +pub fn register() { + use reported::*; + + register_counter!(CONNECTIONS_ACCEPTED); + register_histogram!(CONNECTION_DURATION); + register_gauge!(ACTIVE_CONNECTIONS); + register_gauge!(RUNNING_PROCESSES); +} diff --git a/users/grfn/xanthous/server/src/pty.rs b/users/grfn/xanthous/server/src/pty.rs new file mode 100644 index 000000000000..611130f5bcd9 --- /dev/null +++ b/users/grfn/xanthous/server/src/pty.rs @@ -0,0 +1,173 @@ +use std::io::{self}; +use std::os::unix::prelude::{AsRawFd, CommandExt, FromRawFd}; +use std::pin::Pin; +use std::process::{abort, Command}; +use std::task::{Context, Poll}; + +use eyre::{bail, Result}; +use futures::Future; +use nix::pty::forkpty; +use nix::pty::Winsize; +use nix::sys::termios::Termios; +use nix::sys::wait::{waitpid, WaitPidFlag, WaitStatus}; +use nix::unistd::{ForkResult, Pid}; +use tokio::fs::File; +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::signal::unix::{signal, Signal, SignalKind}; +use tokio::task::spawn_blocking; + +mod ioctl { + use super::Winsize; + use libc::TIOCSWINSZ; + use nix::ioctl_write_ptr_bad; + + ioctl_write_ptr_bad!(tiocswinsz, TIOCSWINSZ, Winsize); +} + +async fn asyncify<F, T>(f: F) -> Result<T> +where + F: FnOnce() -> Result<T> + Send + 'static, + T: Send + 'static, +{ + match spawn_blocking(f).await { + Ok(res) => res, + Err(_) => bail!("background task failed",), + } +} + +pub struct Child { + pub tty: File, + pub pid: Pid, +} + +pub struct ChildHandle { + pub tty: File, +} + +pub struct WaitPid { + pid: Pid, + signal: Signal, +} + +impl WaitPid { + pub fn new(pid: Pid) -> Self { + Self { + pid, + signal: signal(SignalKind::child()).unwrap(), + } + } +} + +impl Future for WaitPid { + type Output = nix::Result<WaitStatus>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + let _ = self.signal.poll_recv(cx); + match waitpid(self.pid, Some(WaitPidFlag::WNOHANG)) { + Ok(WaitStatus::StillAlive) => Poll::Pending, + result => Poll::Ready(result), + } + } +} + +impl Child { + pub async fn handle(&self) -> io::Result<ChildHandle> { + Ok(ChildHandle { + tty: self.tty.try_clone().await?, + }) + } +} + +impl ChildHandle { + pub async fn resize_window(&mut self, winsize: Winsize) -> Result<()> { + let fd = self.tty.as_raw_fd(); + asyncify(move || unsafe { + ioctl::tiocswinsz(fd, &winsize as *const Winsize)?; + Ok(()) + }) + .await + } +} + +pub async fn spawn( + mut cmd: Command, + winsize: Option<Winsize>, + termios: Option<Termios>, +) -> Result<Child> { + asyncify(move || unsafe { + let res = forkpty(winsize.as_ref(), termios.as_ref())?; + match res.fork_result { + ForkResult::Parent { child } => Ok(Child { + pid: child, + tty: File::from_raw_fd(res.master), + }), + ForkResult::Child => { + cmd.exec(); + abort(); + } + } + }) + .await +} + +impl AsyncRead for Child { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + Pin::new(&mut self.tty).poll_read(cx, buf) + } +} + +impl AsyncWrite for Child { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll<Result<usize, io::Error>> { + Pin::new(&mut self.tty).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> { + Pin::new(&mut self.tty).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll<Result<(), io::Error>> { + Pin::new(&mut self.tty).poll_shutdown(cx) + } +} + +impl AsyncRead for ChildHandle { + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> Poll<io::Result<()>> { + Pin::new(&mut self.tty).poll_read(cx, buf) + } +} + +impl AsyncWrite for ChildHandle { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll<Result<usize, io::Error>> { + Pin::new(&mut self.tty).poll_write(cx, buf) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> { + Pin::new(&mut self.tty).poll_flush(cx) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll<Result<(), io::Error>> { + Pin::new(&mut self.tty).poll_shutdown(cx) + } +} diff --git a/users/grfn/xanthous/shell.nix b/users/grfn/xanthous/shell.nix new file mode 100644 index 000000000000..572ed211bcf4 --- /dev/null +++ b/users/grfn/xanthous/shell.nix @@ -0,0 +1,23 @@ +let + depot = import ../../../. {}; + inherit (depot) third_party; + pkgs = third_party.nixpkgs; +in + +(pkgs.haskellPackages.extend (pkgs.haskell.lib.packageSourceOverrides { + xanthous = third_party.gitignoreSource ./.; +})).shellFor { + packages = p: [p.xanthous]; + withHoogle = true; + doBenchmark = true; + buildInputs = (with pkgs.haskellPackages; [ + cabal-install + ghc-prof-flamegraph + hp2pretty + hlint + haskell-language-server + cabal2nix + ]) ++ (with pkgs; [ + qpdf + ]); +} diff --git a/users/grfn/xanthous/src/Data/Aeson/Generic/DerivingVia.hs b/users/grfn/xanthous/src/Data/Aeson/Generic/DerivingVia.hs new file mode 100644 index 000000000000..e89fcd621157 --- /dev/null +++ b/users/grfn/xanthous/src/Data/Aeson/Generic/DerivingVia.hs @@ -0,0 +1,168 @@ +{-# LANGUAGE ConstraintKinds, DataKinds, DeriveGeneric, DerivingVia #-} +{-# LANGUAGE ExplicitNamespaces, FlexibleContexts, FlexibleInstances #-} +{-# LANGUAGE GADTs, GeneralizedNewtypeDeriving, MultiParamTypeClasses #-} +{-# LANGUAGE PolyKinds, ScopedTypeVariables, StandaloneDeriving #-} +{-# LANGUAGE TypeApplications, TypeFamilies, TypeInType, TypeOperators #-} +{-# LANGUAGE UndecidableInstances #-} +{-# OPTIONS_GHC -Wall #-} +-- | https://gist.github.com/konn/27c00f784dd883ec2b90eab8bc84a81d +module Data.Aeson.Generic.DerivingVia + ( StrFun(..), Setting(..), SumEncoding'(..), DefaultOptions, WithOptions(..) + , -- Utility type synonyms to save ticks (') before promoted data constructors + type Drop, type CamelTo2, type UserDefined + , type TaggedObj, type UntaggedVal, type ObjWithSingleField, type TwoElemArr + , type FieldLabelModifier + , type ConstructorTagModifier + , type AllNullaryToStringTag + , type OmitNothingFields + , type SumEnc + , type UnwrapUnaryRecords + , type TagSingleConstructors + ) + where + +import Prelude +import Data.Aeson (FromJSON (..), GFromJSON, GToJSON, + ToJSON (..)) +import Data.Aeson (Options (..), Zero, camelTo2, + genericParseJSON) +import Data.Aeson (defaultOptions, genericToJSON) +import qualified Data.Aeson as Aeson +import Data.Kind (Constraint, Type) +import Data.Proxy (Proxy (..)) +import Data.Reflection (Reifies (..)) +import GHC.Generics (Generic, Rep) +import GHC.TypeLits (KnownNat, KnownSymbol, natVal, symbolVal) +import GHC.TypeLits (Nat, Symbol) + +newtype WithOptions options a = WithOptions { runWithOptions :: a } + +data StrFun = Drop Nat + | CamelTo2 Symbol + | forall p. UserDefined p + +type Drop = 'Drop +type CamelTo2 = 'CamelTo2 +type UserDefined = 'UserDefined + +type family Demoted a where + Demoted Symbol = String + Demoted StrFun = String -> String + Demoted [a] = [Demoted a] + Demoted Setting = Options -> Options + Demoted SumEncoding' = Aeson.SumEncoding + Demoted a = a + +data SumEncoding' = TaggedObj {tagFieldName' :: Symbol, contentsFieldName :: Symbol } + | UntaggedVal + | ObjWithSingleField + | TwoElemArr + +type TaggedObj = 'TaggedObj +type UntaggedVal = 'UntaggedVal +type ObjWithSingleField = 'ObjWithSingleField +type TwoElemArr = 'TwoElemArr + +data Setting = FieldLabelModifier [StrFun] + | ConstructorTagModifier [StrFun] + | AllNullaryToStringTag Bool + | OmitNothingFields Bool + | SumEnc SumEncoding' + | UnwrapUnaryRecords Bool + | TagSingleConstructors Bool + +type FieldLabelModifier = 'FieldLabelModifier +type ConstructorTagModifier = 'ConstructorTagModifier +-- | If 'True' the constructors of a datatype, with all nullary constructors, +-- will be encoded to just a string with the constructor tag. If 'False' the +-- encoding will always follow the 'SumEncoding'. +type AllNullaryToStringTag = 'AllNullaryToStringTag +type OmitNothingFields = 'OmitNothingFields +type SumEnc = 'SumEnc +-- | Hide the field name when a record constructor has only one field, like a +-- newtype. +type UnwrapUnaryRecords = 'UnwrapUnaryRecords +-- | Encode types with a single constructor as sums, so that +-- 'AllNullaryToStringTag' and 'SumEncoding' apply. +type TagSingleConstructors = 'TagSingleConstructors + +class Demotable (a :: k) where + demote :: proxy a -> Demoted k + +type All :: (Type -> Constraint) -> [Type] -> Constraint +type family All p xs where + All p '[] = () + All p (x ': xs) = (p x, All p xs) + +instance Reifies f (String -> String) => Demotable ('UserDefined f) where + demote _ = reflect @f Proxy + +instance KnownSymbol sym => Demotable sym where + demote = symbolVal + +instance (KnownSymbol s, KnownSymbol t) => Demotable ('TaggedObj s t) where + demote _ = Aeson.TaggedObject (symbolVal @s Proxy) (symbolVal @t Proxy) + +instance Demotable 'UntaggedVal where + demote _ = Aeson.UntaggedValue + +instance Demotable 'ObjWithSingleField where + demote _ = Aeson.ObjectWithSingleField + +instance Demotable 'TwoElemArr where + demote _ = Aeson.TwoElemArray + +instance Demotable xs => Demotable ('FieldLabelModifier xs) where + demote _ o = o { fieldLabelModifier = foldr (.) id (demote (Proxy @xs)) } + +instance Demotable xs => Demotable ('ConstructorTagModifier xs) where + demote _ o = o { constructorTagModifier = foldr (.) id (demote (Proxy @xs)) } + +instance Demotable b => Demotable ('AllNullaryToStringTag b) where + demote _ o = o { allNullaryToStringTag = demote (Proxy @b) } + +instance Demotable b => Demotable ('OmitNothingFields b) where + demote _ o = o { omitNothingFields = demote (Proxy @b) } + +instance Demotable b => Demotable ('UnwrapUnaryRecords b) where + demote _ o = o { unwrapUnaryRecords = demote (Proxy @b) } + +instance Demotable b => Demotable ('TagSingleConstructors b) where + demote _ o = o { tagSingleConstructors = demote (Proxy @b) } + +instance Demotable b => Demotable ('SumEnc b) where + demote _ o = o { sumEncoding = demote (Proxy @b) } + +instance Demotable 'True where + demote _ = True + +instance Demotable 'False where + demote _ = False + +instance KnownNat n => Demotable ('Drop n) where + demote _ = drop (fromIntegral $ natVal (Proxy :: Proxy n)) + +instance KnownSymbol sym => Demotable ('CamelTo2 sym) where + demote _ = camelTo2 $ head $ symbolVal @sym Proxy + +instance {-# OVERLAPPING #-} Demotable ('[] :: [k]) where + demote _ = [] + +instance (Demotable (x :: k), Demotable (xs :: [k])) => Demotable (x ': xs) where + demote _ = demote (Proxy @x) : demote (Proxy @xs) + +type DefaultOptions = ('[] :: [Setting]) + +reflectOptions :: forall xs proxy. Demotable (xs :: [Setting]) => proxy xs -> Options +reflectOptions pxy = foldr (.) id (demote pxy) defaultOptions + +instance (Demotable (options :: [Setting])) => Reifies options Options where + reflect = reflectOptions + +instance (Generic a, GToJSON Zero (Rep a), Reifies (options :: k) Options) + => ToJSON (WithOptions options a) where + toJSON = genericToJSON (reflect (Proxy @options)) . runWithOptions + +instance (Generic a, GFromJSON Zero (Rep a), Reifies (options :: k) Options) + => FromJSON (WithOptions options a) where + parseJSON = fmap WithOptions . genericParseJSON (reflect (Proxy @options)) diff --git a/users/grfn/xanthous/src/Xanthous/AI/Gormlak.hs b/users/grfn/xanthous/src/Xanthous/AI/Gormlak.hs new file mode 100644 index 000000000000..1f2b513ffe0e --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/AI/Gormlak.hs @@ -0,0 +1,201 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE UndecidableInstances #-} +-------------------------------------------------------------------------------- +module Xanthous.AI.Gormlak + ( HasVisionRadius(..) + , GormlakBrain(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (lines) +-------------------------------------------------------------------------------- +import Control.Monad.State +import Control.Monad.Random +import Data.Aeson (object) +import qualified Data.Aeson as A +import Data.Generics.Product.Fields +-------------------------------------------------------------------------------- +import Xanthous.Data + ( Positioned(..), positioned, position, _Position + , diffPositions, stepTowards, isUnit + , Ticks, (|*|), invertedRate + ) +import Xanthous.Data.EntityMap +import Xanthous.Entities.Creature.Hippocampus +import Xanthous.Entities.Character (Character) +import qualified Xanthous.Entities.Character as Character +import qualified Xanthous.Entities.RawTypes as Raw +import Xanthous.Entities.RawTypes + ( CreatureType, HasLanguage(language), getLanguage + , HasAttacks (attacks), creatureAttackMessage + ) +import Xanthous.Entities.Common + ( wielded, Inventory, wieldedItems, WieldedItem (WieldedItem) ) +import Xanthous.Game.State +import Xanthous.Game.Lenses + ( entitiesCollision, collisionAt + , character, characterPosition, positionIsCharacterVisible + , hearingRadius + ) +import Xanthous.Data.EntityMap.Graphics (linesOfSight, canSee) +import Xanthous.Random +import Xanthous.Monad (say, message) +import Xanthous.Generators.Speech (word) +import qualified Linear.Metric as Metric +import qualified Xanthous.Messages as Messages +-------------------------------------------------------------------------------- + +-- TODO move the following two classes to a more central location + +class HasVisionRadius a where visionRadius :: a -> Word + +type IsCreature entity = + ( HasVisionRadius entity + , HasField "_hippocampus" entity entity Hippocampus Hippocampus + , HasField "_creatureType" entity entity CreatureType CreatureType + , HasField "_inventory" entity entity Inventory Inventory + , A.ToJSON entity + ) + +-------------------------------------------------------------------------------- + +stepGormlak + :: forall entity m. + ( MonadState GameState m, MonadRandom m + , IsCreature entity + ) + => Ticks + -> Positioned entity + -> m (Positioned entity) +stepGormlak ticks pe@(Positioned pos creature) = do + canSeeCharacter <- uses entities $ canSee (entityIs @Character) pos vision + + let selectDestination pos' creature' = destinationFromPos <$> do + if canSeeCharacter + then do + charPos <- use characterPosition + if isUnit (pos' `diffPositions` charPos) + then attackCharacter $> pos' + else pure $ pos' `stepTowards` charPos + else do + lines <- map (takeWhile (isNothing . entitiesCollision . map snd . snd) + -- the first item on these lines is always the creature itself + . fromMaybe mempty . tailMay) + . linesOfSight pos' (visionRadius creature') + <$> use entities + line <- choose $ weightedBy length lines + pure $ fromMaybe pos' $ fmap fst . headMay =<< line + + pe' <- if canSeeCharacter && not (creature ^. creatureGreeted) + then yellAtCharacter $> (pe & positioned . creatureGreeted .~ True) + else pure pe + + dest <- maybe (selectDestination pos creature) pure + . mfilter (\(Destination p _) -> p /= pos) + $ creature ^. hippocampus . destination + let progress' = + dest ^. destinationProgress + + creature ^. creatureType . Raw.speed . invertedRate |*| ticks + if progress' < 1 + then pure + $ pe' + & positioned . hippocampus . destination + ?~ (dest & destinationProgress .~ progress') + else do + let newPos = dest ^. destinationPosition + remainingSpeed = progress' - 1 + newDest <- selectDestination newPos creature + <&> destinationProgress +~ remainingSpeed + let pe'' = pe' & positioned . hippocampus . destination ?~ newDest + collisionAt newPos >>= \case + Nothing -> pure $ pe'' & position .~ newPos + Just Stop -> pure pe'' + Just Combat -> do + ents <- use $ entities . atPosition newPos + when (any (entityIs @Character) ents) attackCharacter + pure pe' + where + vision = visionRadius creature + attackCharacter = do + dmg <- case creature ^? inventory . wielded . wieldedItems of + Just (WieldedItem item wi) -> do + let msg = fromMaybe + (Messages.lookup ["combat", "creatureAttack", "genericWeapon"]) + $ wi ^. creatureAttackMessage + message msg $ object [ "creature" A..= creature + , "item" A..= item + ] + pure $ wi ^. Raw.damage + Nothing -> do + attack <- choose $ creature ^. creatureType . attacks + attackDescription <- Messages.render (attack ^. Raw.description) + $ object [] + say ["combat", "creatureAttack", "natural"] + $ object [ "creature" A..= creature + , "attackDescription" A..= attackDescription + ] + pure $ attack ^. Raw.damage + + character %= Character.damage dmg + + yellAtCharacter = for_ (creature ^. creatureType . language) + $ \lang -> do + utterance <- fmap (<> "!") . word $ getLanguage lang + creatureSaysText pe utterance + + creatureGreeted :: Lens' entity Bool + creatureGreeted = hippocampus . greetedCharacter + + +-- | A creature sends some text +-- +-- If that creature is visible to the character, its description will be +-- included, otherwise if it's within earshot the character will just hear the +-- sound +creatureSaysText + :: (MonadState GameState m, MonadRandom m, IsCreature entity) + => Positioned entity + -> Text + -> m () +creatureSaysText ent txt = do + let entPos = ent ^. position . _Position . to (fmap fromIntegral) + charPos <- use $ characterPosition . _Position . to (fmap fromIntegral) + let dist :: Int + dist = round $ Metric.distance @_ @Double entPos charPos + audible = dist <= fromIntegral hearingRadius + when audible $ do + visible <- positionIsCharacterVisible $ ent ^. position + let path = ["entities", "say", "creature"] + <> [if visible then "visible" else "invisible"] + params = object [ "creature" A..= (ent ^. positioned) + , "message" A..= txt + ] + say path params + +newtype GormlakBrain entity = GormlakBrain { _unGormlakBrain :: entity } + +instance (IsCreature entity) => Brain (GormlakBrain entity) where + step ticks + = fmap (fmap GormlakBrain) + . stepGormlak ticks + . fmap _unGormlakBrain + entityCanMove = const True + +hippocampus :: HasField "_hippocampus" s t a b => Lens s t a b +hippocampus = field @"_hippocampus" + +creatureType :: HasField "_creatureType" s t a b => Lens s t a b +creatureType = field @"_creatureType" + +inventory :: HasField "_inventory" s t a b => Lens s t a b +inventory = field @"_inventory" + +-------------------------------------------------------------------------------- + +-- instance Brain Creature where +-- step = brainVia GormlakBrain +-- entityCanMove = const True + +-- instance Entity Creature where +-- blocksVision _ = False +-- description = view $ Creature.creatureType . Raw.description +-- entityChar = view $ Creature.creatureType . char diff --git a/users/grfn/xanthous/src/Xanthous/App.hs b/users/grfn/xanthous/src/Xanthous/App.hs new file mode 100644 index 000000000000..a251833955cd --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/App.hs @@ -0,0 +1,607 @@ +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +{-# OPTIONS_GHC -Wno-deferred-type-errors #-} +module Xanthous.App + ( makeApp + , RunType(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Brick hiding (App, halt, continue, raw) +import qualified Brick +import Graphics.Vty.Attributes (defAttr) +import Graphics.Vty.Input.Events (Event(EvKey)) +import Control.Monad.State (get, gets) +import Control.Monad.State.Class (modify) +import Data.Aeson (object, ToJSON) +import qualified Data.Aeson as A +import qualified Data.Vector as V +import System.Exit +import System.Directory (doesFileExist) +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Vector.Lens (toVectorOf) +-------------------------------------------------------------------------------- +import Xanthous.App.Common +import Xanthous.App.Time +import Xanthous.App.Prompt +import Xanthous.App.Autocommands +import Xanthous.Command +import Xanthous.Data + ( move + , Dimensions'(Dimensions) + , positioned + , position + , Position + , (|*|) + , Tiles(..), Hitpoints, fromScalar + ) +import Xanthous.Data.App (ResourceName, Panel(..), AppEvent(..)) +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Data.Levels (prevLevel, nextLevel) +import qualified Xanthous.Data.Levels as Levels +import Xanthous.Data.Entities (blocksObject) +import Xanthous.Game +import Xanthous.Game.State +import Xanthous.Game.Env +import Xanthous.Game.Draw (drawGame) +import Xanthous.Game.Prompt hiding (Fire) +import qualified Xanthous.Messages as Messages +import Xanthous.Random +import Xanthous.Util (removeVectorIndex, useListOf) +import Xanthous.Util.Inflection (toSentence) +import Xanthous.Physics (throwDistance, bluntThrowDamage) +import Xanthous.Data.EntityMap.Graphics (lineOfSight) +import Xanthous.Data.EntityMap (EntityID) +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +import Xanthous.Entities.Common + ( InventoryPosition, describeInventoryPosition, backpack + , wieldableItem, wieldedItems, wielded, itemsWithPosition + , removeItemFromPosition, asWieldedItem, inRightHand + , wieldedItem, items + ) +import qualified Xanthous.Entities.Character as Character +import Xanthous.Entities.Character hiding (pickUpItem) +import Xanthous.Entities.Item (Item, weight) +import qualified Xanthous.Entities.Item as Item +import Xanthous.Entities.Creature (Creature) +import qualified Xanthous.Entities.Creature as Creature +import Xanthous.Entities.Environment + (Door, open, closed, locked, GroundMessage(..), Staircase(..)) +import Xanthous.Entities.RawTypes + ( edible, eatMessage, hitpointsHealed + , attackMessage + ) +import Xanthous.Generators.Level +import qualified Xanthous.Generators.Level.CaveAutomata as CaveAutomata +import qualified Xanthous.Generators.Level.Dungeon as Dungeon +-------------------------------------------------------------------------------- + +type App = Brick.App GameState AppEvent ResourceName + +data RunType = NewGame | LoadGame FilePath + deriving stock (Eq) + +makeApp :: GameEnv -> RunType -> IO App +makeApp env rt = pure $ Brick.App + { appDraw = drawGame + , appChooseCursor = const headMay + , appHandleEvent = \game event -> runAppM (handleEvent event) env game + , appStartEvent = case rt of + NewGame -> runAppM (startEvent >> get) env + LoadGame save -> pure . (savefile ?~ save) + , appAttrMap = const $ attrMap defAttr [] + } + +runAppM :: AppM a -> GameEnv -> GameState -> EventM ResourceName a +runAppM appm ge = fmap fst . runAppT appm ge + +startEvent :: AppM () +startEvent = do + initLevel + modify updateCharacterVision + use (character . characterName) >>= \case + Nothing -> prompt_ @'StringPrompt ["character", "namePrompt"] Uncancellable + $ \(StringResult s) -> do + character . characterName ?= s + say ["welcome"] =<< use character + Just n -> say ["welcome"] $ object [ "characterName" A..= n ] + +initLevel :: AppM () +initLevel = do + level <- genLevel 0 + entities <>= levelToEntityMap level + characterPosition .= level ^. levelCharacterPosition + +-------------------------------------------------------------------------------- + +handleEvent :: BrickEvent ResourceName AppEvent -> AppM (Next GameState) +handleEvent ev = use promptState >>= \case + NoPrompt -> handleNoPromptEvent ev + WaitingPrompt msg pr -> handlePromptEvent msg pr ev + + +handleNoPromptEvent :: BrickEvent ResourceName AppEvent -> AppM (Next GameState) +handleNoPromptEvent (VtyEvent (EvKey k mods)) + | Just command <- commandFromKey k mods + = do messageHistory %= nextTurn + cancelAutocommand + handleCommand command +handleNoPromptEvent (AppEvent AutoContinue) = do + preuse (autocommand . _ActiveAutocommand . _1) >>= traverse_ autoStep + continue +handleNoPromptEvent _ = continue + +handleCommand :: Command -> AppM (Next GameState) +handleCommand Quit = confirm_ ["quit", "confirm"] (liftIO exitSuccess) >> continue +handleCommand (Move dir) = do + newPos <- uses characterPosition $ move dir + collisionAt newPos >>= \case + Nothing -> do + characterPosition .= newPos + stepGameBy =<< uses (character . speed) (|*| Tiles 1) + describeEntitiesAt newPos + Just Combat -> attackAt newPos + Just Stop -> pure () + continue + +handleCommand PickUp = do + pos <- use characterPosition + uses entities (entitiesAtPositionWithType @Item pos) >>= \case + [] -> say_ ["pickUp", "nothingToPickUp"] + [item] -> pickUpItem item + items' -> + menu_ ["pickUp", "menu"] Cancellable (entityMenu_ items') + $ \(MenuResult item) -> pickUpItem item + continue + where + pickUpItem (itemID, item) = do + character %= Character.pickUpItem item + entities . at itemID .= Nothing + say ["pickUp", "pickUp"] $ object [ "item" A..= item ] + stepGameBy 100 -- TODO + +handleCommand Drop = do + takeItemFromInventory_ ["drop", "menu"] Cancellable id + (say_ ["drop", "nothing"]) + $ \(MenuResult item) -> do + entitiesAtCharacter %= (SomeEntity item <|) + say ["drop", "dropped"] $ object [ "item" A..= item ] + continue + +handleCommand PreviousMessage = do + messageHistory %= previousMessage + continue + +handleCommand Open = do + prompt_ @'DirectionPrompt ["open", "prompt"] Cancellable + $ \(DirectionResult dir) -> do + pos <- move dir <$> use characterPosition + doors <- uses entities $ entitiesAtPositionWithType @Door pos + if | null doors -> say_ ["open", "nothingToOpen"] + | any (view $ _2 . locked) doors -> say_ ["open", "locked"] + | all (view $ _2 . open) doors -> say_ ["open", "alreadyOpen"] + | otherwise -> do + for_ doors $ \(eid, _) -> + entities . ix eid . positioned . _SomeEntity . open .= True + say_ ["open", "success"] + pure () + stepGame -- TODO + continue + +handleCommand Close = do + prompt_ @'DirectionPrompt ["close", "prompt"] Cancellable + $ \(DirectionResult dir) -> do + pos <- move dir <$> use characterPosition + (nonDoors, doors) <- uses entities + $ partitionEithers + . toList + . map ( (matching . aside $ _SomeEntity @Door) + . over _2 (view positioned) + ) + . EntityMap.atPositionWithIDs pos + if | null doors -> say_ ["close", "nothingToClose"] + | all (view $ _2 . closed) doors -> say_ ["close", "alreadyClosed"] + | any (view blocksObject . entityAttributes . snd) nonDoors -> + say ["close", "blocked"] + $ object [ "entityDescriptions" + A..= ( toSentence + . map description + . filter (view blocksObject . entityAttributes) + . map snd + ) nonDoors + , "blockOrBlocks" + A..= ( if length nonDoors == 1 + then "blocks" + else "block" + :: Text) + ] + | otherwise -> do + for_ doors $ \(eid, _) -> + entities . ix eid . positioned . _SomeEntity . closed .= True + for_ nonDoors $ \(eid, _) -> + entities . ix eid . position %= move dir + say_ ["close", "success"] + pure () + stepGame -- TODO + continue + +handleCommand Look = do + prompt_ @'PointOnMap ["look", "prompt"] Cancellable + $ \(PointOnMapResult pos) -> revealedEntitiesAtPosition pos >>= \case + Empty -> say_ ["look", "nothing"] + ents -> describeEntities ents + continue + +handleCommand Wait = stepGame >> continue + +handleCommand Eat = do + uses (character . inventory . backpack) + (V.mapMaybe (\item -> (item,) <$> item ^. Item.itemType . edible)) + >>= \case + Empty -> say_ ["eat", "noFood"] + food -> + let foodMenuItem idx (item, edibleItem) + = ( item ^. Item.itemType . char . char + , MenuOption (description item) (idx, item, edibleItem)) + -- TODO refactor to use entityMenu_ + menuItems = mkMenuItems $ imap foodMenuItem food + in menu_ ["eat", "menuPrompt"] Cancellable menuItems + $ \(MenuResult (idx, item, edibleItem)) -> do + character . inventory . backpack %= removeVectorIndex idx + let msg = fromMaybe (Messages.lookup ["eat", "eat"]) + $ edibleItem ^. eatMessage + character . characterHitpoints' += + edibleItem ^. hitpointsHealed . to fromIntegral + message msg $ object ["item" A..= item] + stepGame -- TODO + continue + +handleCommand Read = do + -- TODO allow reading things in the inventory (combo direction+menu prompt?) + prompt_ @'DirectionPrompt ["read", "prompt"] Cancellable + $ \(DirectionResult dir) -> do + pos <- uses characterPosition $ move dir + uses entities + (fmap snd . entitiesAtPositionWithType @GroundMessage pos) >>= \case + Empty -> say_ ["read", "nothing"] + GroundMessage msg :< Empty -> + say ["read", "result"] $ object ["message" A..= msg] + msgs -> + let readAndContinue Empty = pure () + readAndContinue (msg :< msgs') = + prompt @'Continue + ["read", "result"] + (object ["message" A..= msg]) + Cancellable + . const + $ readAndContinue msgs' + readAndContinue _ = error "this is total" + in readAndContinue msgs + continue + +handleCommand ShowInventory = showPanel InventoryPanel >> continue + +handleCommand DescribeInventory = do + selectItemFromInventory_ ["inventory", "describe", "select"] Cancellable id + (say_ ["inventory", "describe", "nothing"]) + $ \(MenuResult (invPos, item)) -> showPanel . ItemDescriptionPanel + $ Item.fullDescription item + <> "\n\n" <> describeInventoryPosition invPos + continue + + +handleCommand Wield = do + takeItemFromInventory_ ["wield", "menu"] Cancellable asWieldedItem + (say_ ["wield", "nothing"]) + $ \(MenuResult item) -> do + prevItems <- character . inventory . wielded <<.= inRightHand item + character . inventory . backpack + <>= fromList (prevItems ^.. wieldedItems . wieldedItem) + say ["wield", "wielded"] item + continue + +handleCommand Fire = do + selectItemFromInventory_ ["fire", "menu"] Cancellable id + (say_ ["fire", "nothing"]) + $ \(MenuResult (invPos, item)) -> + let wt = weight item + dist = throwDistance wt + dam = bluntThrowDamage wt + in if dist < fromScalar 1 + then say_ ["fire", "zeroRange"] + else firePrompt_ ["fire", "target"] Cancellable dist $ + \(FireResult targetPos) -> do + charPos <- use characterPosition + mTarget <- uses entities $ firstEnemy . lineOfSight charPos targetPos + case mTarget of + Just target -> do + creature' <- damageCreature target dam + unless (Creature.isDead creature') $ + let msgPath = ["fire", "fired"] <> [if dam == 0 + then "noDamage" + else "someDamage"] + in say msgPath $ object [ "item" A..= item + , "creature" A..= creature' + ] + Nothing -> + say ["fire", "fired", "noTarget"] $ object [ "item" A..= item ] + character . inventory %= removeItemFromPosition invPos item + entities . EntityMap.atPosition targetPos %= (SomeEntity item <|) + stepGame -- TODO(grfn): should this be based on distance? + continue + where + firstEnemy + :: [(Position, Vector (EntityID, SomeEntity))] + -> Maybe (EntityID, Creature) + firstEnemy los = + let enemies = los >>= \(_, es) -> toList $ headMay es + in enemies ^? folded . below _SomeEntity + +handleCommand Save = + view (config . disableSaving) >>= \case + True -> say_ ["save", "disabled"] >> continue + False -> do + -- TODO default save locations / config file? + use savefile >>= \case + Just filepath -> + stringPromptWithDefault_ + ["save", "location"] + Cancellable + (pack filepath) + promptCallback + Nothing -> prompt_ @'StringPrompt ["save", "location"] Cancellable promptCallback + continue + where + promptCallback :: PromptResult 'StringPrompt -> AppM () + promptCallback (StringResult filename) = do + sf <- use savefile + exists <- liftIO . doesFileExist $ unpack filename + if exists && sf /= Just (unpack filename) + then confirm ["save", "overwrite"] (object ["filename" A..= filename]) + $ doSave filename + else doSave filename + doSave filename = do + src <- gets saveGame + lift . liftIO $ do + writeFile (unpack filename) $ toStrict src + exitSuccess + +handleCommand GoUp = do + hasStairs <- uses entitiesAtCharacter $ elem (SomeEntity UpStaircase) + if hasStairs + then uses levels prevLevel >>= \case + Just levs' -> do + cEID <- use characterEntityID + pCharacter <- entities . at cEID <<.= Nothing + levels .= levs' + charPos <- use characterPosition + entities . at cEID .= pCharacter + characterPosition .= charPos + Nothing -> + -- TODO in nethack, this leaves the game. Maybe something similar here? + say_ ["cant", "goUp"] + else say_ ["cant", "goUp"] + + continue + +handleCommand GoDown = do + hasStairs <- uses entitiesAtCharacter $ elem (SomeEntity DownStaircase) + + if hasStairs + then do + levs <- use levels + let newLevelNum = Levels.pos levs + 1 + levs' <- nextLevel (levelToGameLevel <$> genLevel newLevelNum) levs + cEID <- use characterEntityID + pCharacter <- entities . at cEID <<.= Nothing + levels .= levs' + entities . at cEID .= pCharacter + characterPosition .= extract levs' ^. upStaircasePosition + else say_ ["cant", "goDown"] + + continue + +handleCommand (StartAutoMove dir) = do + runAutocommand $ AutoMove dir + continue + +handleCommand Rest = do + say_ ["autocommands", "resting"] + runAutocommand AutoRest + continue + +-- + +handleCommand ToggleRevealAll = do + val <- debugState . allRevealed <%= not + say ["debug", "toggleRevealAll"] $ object [ "revealAll" A..= val ] + continue + +-------------------------------------------------------------------------------- +attackAt :: Position -> AppM () +attackAt pos = + uses entities (entitiesAtPositionWithType @Creature pos) >>= \case + Empty -> say_ ["combat", "nothingToAttack"] + (creature :< Empty) -> attackCreature creature + creatures -> + menu_ ["combat", "menu"] Cancellable (entityMenu_ creatures) + $ \(MenuResult creature) -> attackCreature creature + where + attackCreature creature = do + charDamage <- uses character characterDamage + creature' <- damageCreature creature charDamage + msg <- uses character getAttackMessage + unless (Creature.isDead creature') + . message msg $ object ["creature" A..= creature'] + whenM (uses character $ isNothing . weapon) handleFists + stepGame + weapon chr = chr ^? inventory . wielded . wieldedItems . wieldableItem + getAttackMessage chr = + case weapon chr of + Just wi -> + fromMaybe (Messages.lookup ["combat", "hit", "generic"]) + $ wi ^. attackMessage + Nothing -> + Messages.lookup ["combat", "hit", "fists"] + + handleFists = do + damageChance <- use $ character . body . knuckles . to fistDamageChance + whenM (chance damageChance) $ do + damageAmount <- use $ character . body . knuckles . to fistfightingDamage + say_ [ "combat" , if damageAmount > 1 + then "fistExtraSelfDamage" + else "fistSelfDamage" ] + character %= Character.damage damageAmount + character . body . knuckles %= damageKnuckles + +damageCreature :: (EntityID, Creature) -> Hitpoints -> AppM Creature +damageCreature (creatureID, creature) dam = do + let creature' = Creature.damage dam creature + msgParams = object ["creature" A..= creature'] + if Creature.isDead creature' + then do + say ["combat", "killed"] msgParams + floorItems <- useListOf + $ entities + . ix creatureID + . positioned + . _SomeEntity @Creature + . inventory + . items + mCreaturePos <- preuse $ entities . ix creatureID . position + entities . at creatureID .= Nothing + for_ mCreaturePos $ \creaturePos -> + entities . EntityMap.atPosition creaturePos + %= (<> fromList (SomeEntity <$> floorItems)) + else entities . ix creatureID . positioned .= SomeEntity creature' + pure creature' + + +entityMenu_ + :: (Comonad w, Entity entity) + => [w entity] + -> Map Char (MenuOption (w entity)) +entityMenu_ = mkMenuItems @[_] . map entityMenuItem + where + entityMenuItem wentity + = let entity = extract wentity + in (entityMenuChar entity, MenuOption (description entity) wentity) + + +entityMenuChar :: Entity a => a -> Char +entityMenuChar entity + = let ec = entityChar entity ^. char + in if ec `elem` (['a'..'z'] ++ ['A'..'Z']) + then ec + else 'a' + +-- | Prompt with an item to select out of the inventory and call callback with +-- it +selectItemFromInventory + :: forall item params. + (ToJSON params) + => [Text] -- ^ Menu message + -> params -- ^ Menu message params + -> PromptCancellable -- ^ Is the menu cancellable? + -> Prism' Item item -- ^ Attach some extra information to the item, in a + -- recoverable fashion. Prism vs iso so we can discard + -- items. + -> AppM () -- ^ Action to take if there are no items matching + -> (PromptResult ('Menu (InventoryPosition, item)) -> AppM ()) + -> AppM () +selectItemFromInventory msgPath msgParams cancellable extraInfo onEmpty cb = do + uses (character . inventory) + (V.mapMaybe (_2 $ preview extraInfo) . toVectorOf itemsWithPosition) + >>= \case + Empty -> onEmpty + items' -> menu msgPath msgParams cancellable (itemMenu items') cb + where + itemMenu = mkMenuItems . map itemMenuItem + itemMenuItem (invPos, extraInfoItem) = + let item = extraInfo # extraInfoItem + in ( entityMenuChar item + , MenuOption + (description item <> " (" <> describeInventoryPosition invPos <> ")") + (invPos, extraInfoItem) + ) + +-- | Prompt with an item to select out of the inventory and call callback with +-- it +selectItemFromInventory_ + :: forall item. + [Text] -- ^ Menu message + -> PromptCancellable -- ^ Is the menu cancellable? + -> Prism' Item item -- ^ Attach some extra information to the item, in a + -- recoverable fashion. Prism vs iso so we can discard + -- items. + -> AppM () -- ^ Action to take if there are no items matching + -> (PromptResult ('Menu (InventoryPosition, item)) -> AppM ()) + -> AppM () +selectItemFromInventory_ msgPath = selectItemFromInventory msgPath () + +-- | Prompt with an item to select out of the inventory, remove it from the +-- inventory, and call callback with it +takeItemFromInventory + :: forall item params. + (ToJSON params) + => [Text] -- ^ Menu message + -> params -- ^ Menu message params + -> PromptCancellable -- ^ Is the menu cancellable? + -> Prism' Item item -- ^ Attach some extra information to the item, in a + -- recoverable fashion. Prism vs iso so we can discard + -- items. + -> AppM () -- ^ Action to take if there are no items matching + -> (PromptResult ('Menu item) -> AppM ()) + -> AppM () +takeItemFromInventory msgPath msgParams cancellable extraInfo onEmpty cb = + selectItemFromInventory msgPath msgParams cancellable extraInfo onEmpty + $ \(MenuResult (invPos, item)) -> do + character . inventory + %= removeItemFromPosition invPos (item ^. re extraInfo) + cb $ MenuResult item + +takeItemFromInventory_ + :: forall item. + [Text] -- ^ Menu message + -> PromptCancellable -- ^ Is the menu cancellable? + -> Prism' Item item -- ^ Attach some extra information to the item, in a + -- recoverable fashion. Prism vs iso so we can discard + -- items. + -> AppM () -- ^ Action to take if there are no items matching + -> (PromptResult ('Menu item) -> AppM ()) + -> AppM () +takeItemFromInventory_ msgPath = takeItemFromInventory msgPath () + +-- entityMenu :: Entity entity => [entity] -> Map Char (MenuOption entity) +-- entityMenu = map (map runIdentity) . entityMenu_ . fmap Identity + +showPanel :: Panel -> AppM () +showPanel panel = do + activePanel ?= panel + prompt_ @'Continue ["generic", "continue"] Uncancellable + . const + $ activePanel .= Nothing + +-------------------------------------------------------------------------------- + +genLevel + :: Word -- ^ Level number, starting at 0 + -> AppM Level +genLevel num = do + let dims = Dimensions 80 80 + generator <- choose $ CaveAutomata :| [Dungeon] + let + doGen = case generator of + CaveAutomata -> generateLevel SCaveAutomata CaveAutomata.defaultParams + Dungeon -> generateLevel SDungeon Dungeon.defaultParams + level <- doGen dims num + pure $!! level + +levelToGameLevel :: Level -> GameLevel +levelToGameLevel level = + let _levelEntities = levelToEntityMap level + _upStaircasePosition = level ^. levelCharacterPosition + _levelRevealedPositions = mempty + in GameLevel {..} diff --git a/users/grfn/xanthous/src/Xanthous/App/Autocommands.hs b/users/grfn/xanthous/src/Xanthous/App/Autocommands.hs new file mode 100644 index 000000000000..5d4db1a47465 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/App/Autocommands.hs @@ -0,0 +1,76 @@ +-------------------------------------------------------------------------------- +module Xanthous.App.Autocommands + ( runAutocommand + , autoStep + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Control.Concurrent (threadDelay) +import qualified Data.Aeson as A +import Data.Aeson (object) +import Data.List.NonEmpty (nonEmpty) +import qualified Data.List.NonEmpty as NE +import Control.Monad.State (gets) +-------------------------------------------------------------------------------- +import Xanthous.App.Common +import Xanthous.App.Time +import Xanthous.Data +import Xanthous.Data.App +import Xanthous.Entities.Character (speed, isFullyHealed) +import Xanthous.Entities.Creature (Creature, creatureType) +import Xanthous.Entities.RawTypes (hostile) +import Xanthous.Game.State +-------------------------------------------------------------------------------- + +-- | Step the given autocommand forward once +autoStep :: Autocommand -> AppM () +autoStep (AutoMove dir) = do + newPos <- uses characterPosition $ move dir + collisionAt newPos >>= \case + Nothing -> do + characterPosition .= newPos + stepGameBy =<< uses (character . speed) (|*| (1 :: Tiles)) + describeEntitiesAt newPos + cancelIfDanger + Just _ -> cancelAutocommand + +autoStep AutoRest = do + done <- uses character isFullyHealed + if done + then say_ ["autocommands", "doneResting"] >> cancelAutocommand + else stepGame >> cancelIfDanger + +-- | Cancel the autocommand if the character is in danger +cancelIfDanger :: AppM () +cancelIfDanger = do + maybeVisibleEnemies <- nonEmpty <$> enemiesInSight + for_ maybeVisibleEnemies $ \visibleEnemies -> do + say ["autocommands", "enemyInSight"] + $ object [ "firstEntity" A..= NE.head visibleEnemies ] + cancelAutocommand + where + enemiesInSight :: AppM [Creature] + enemiesInSight = do + ents <- gets characterVisibleEntities + pure $ ents + ^.. folded + . _SomeEntity @Creature + . filtered (view $ creatureType . hostile) + +-------------------------------------------------------------------------------- + +autocommandIntervalμs :: Int +autocommandIntervalμs = 1000 * 50 -- 50 ms + +runAutocommand :: Autocommand -> AppM () +runAutocommand ac = do + env <- ask + tid <- liftIO . async $ runReaderT go env + autocommand .= ActiveAutocommand ac tid + where + go = everyμs autocommandIntervalμs $ sendEvent AutoContinue + +-- | Perform 'act' every μs microseconds forever +everyμs :: MonadIO m => Int -> m () -> m () +everyμs μs act = act >> liftIO (threadDelay μs) >> everyμs μs act diff --git a/users/grfn/xanthous/src/Xanthous/App/Common.hs b/users/grfn/xanthous/src/Xanthous/App/Common.hs new file mode 100644 index 000000000000..69ba6f0e0596 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/App/Common.hs @@ -0,0 +1,67 @@ +-------------------------------------------------------------------------------- +module Xanthous.App.Common + ( describeEntities + , describeEntitiesAt + , entitiesAtPositionWithType + + -- * Re-exports + , MonadState + , MonadRandom + , EntityMap + , module Xanthous.Game.Lenses + , module Xanthous.Monad + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson (object) +import qualified Data.Aeson as A +import Control.Monad.State (MonadState) +import Control.Monad.Random (MonadRandom) +-------------------------------------------------------------------------------- +import Xanthous.Data (Position, positioned) +import Xanthous.Data.EntityMap (EntityMap) +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Game +import Xanthous.Game.Lenses +import Xanthous.Game.State +import Xanthous.Monad +import Xanthous.Entities.Character (Character) +import Xanthous.Util.Inflection (toSentence) +-------------------------------------------------------------------------------- + +entitiesAtPositionWithType + :: forall a. (Entity a, Typeable a) + => Position + -> EntityMap SomeEntity + -> [(EntityMap.EntityID, a)] +entitiesAtPositionWithType pos em = + let someEnts = EntityMap.atPositionWithIDs pos em + in flip foldMap someEnts $ \(eid, view positioned -> se) -> + case downcastEntity @a se of + Just e -> [(eid, e)] + Nothing -> [] + +describeEntitiesAt :: (MonadState GameState m, MonadRandom m) => Position -> m () +describeEntitiesAt pos = + use ( entities + . EntityMap.atPosition pos + . to (filter (not . entityIs @Character)) + ) >>= \case + Empty -> pure () + ents -> describeEntities ents + +describeEntities + :: ( Entity entity + , MonadRandom m + , MonadState GameState m + , MonoFoldable (f Text) + , Functor f + , Element (f Text) ~ Text + ) + => f entity + -> m () +describeEntities ents = + let descriptions = description <$> ents + in say ["entities", "description"] + $ object ["entityDescriptions" A..= toSentence descriptions] diff --git a/users/grfn/xanthous/src/Xanthous/App/Prompt.hs b/users/grfn/xanthous/src/Xanthous/App/Prompt.hs new file mode 100644 index 000000000000..799281a1c2fd --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/App/Prompt.hs @@ -0,0 +1,228 @@ +{-# LANGUAGE UndecidableInstances #-} +-------------------------------------------------------------------------------- +module Xanthous.App.Prompt + ( handlePromptEvent + , clearPrompt + , prompt + , prompt_ + , stringPromptWithDefault + , stringPromptWithDefault_ + , confirm_ + , confirm + , menu + , menu_ + , firePrompt_ + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Brick (BrickEvent(..), Next) +import Brick.Widgets.Edit (handleEditorEvent) +import Data.Aeson (ToJSON, object) +import Graphics.Vty.Input.Events (Event(EvKey), Key(..)) +-------------------------------------------------------------------------------- +import Xanthous.App.Common +import Xanthous.Data (move, Tiles, Position, positioned, _Position) +import qualified Xanthous.Data as Data +import Xanthous.Command (directionFromChar) +import Xanthous.Data.App (ResourceName, AppEvent) +import Xanthous.Game.Prompt +import Xanthous.Game.State +import qualified Xanthous.Messages as Messages +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Entities.Creature (creatureType, Creature) +import Xanthous.Entities.RawTypes (hostile) +import qualified Linear.Metric as Metric +-------------------------------------------------------------------------------- + +handlePromptEvent + :: Text -- ^ Prompt message + -> Prompt AppM + -> BrickEvent ResourceName AppEvent + -> AppM (Next GameState) + +handlePromptEvent _ (Prompt Cancellable _ _ _ _) (VtyEvent (EvKey KEsc [])) + = clearPrompt >> continue +handlePromptEvent _ pr (VtyEvent (EvKey KEnter [])) + = clearPrompt >> submitPrompt pr >> continue + +handlePromptEvent _ pr@(Prompt _ SConfirm _ _ _) (VtyEvent (EvKey (KChar 'y') [])) + = clearPrompt >> submitPrompt pr >> continue + +handlePromptEvent _ (Prompt _ SConfirm _ _ _) (VtyEvent (EvKey (KChar 'n') [])) + = clearPrompt >> continue + +handlePromptEvent + msg + (Prompt c SStringPrompt (StringPromptState edit) pri cb) + (VtyEvent ev) + = do + edit' <- lift $ handleEditorEvent ev edit + let prompt' = Prompt c SStringPrompt (StringPromptState edit') pri cb + promptState .= WaitingPrompt msg prompt' + continue + +handlePromptEvent _ (Prompt _ SDirectionPrompt _ _ cb) + (VtyEvent (EvKey (KChar (directionFromChar -> Just dir)) [])) + = clearPrompt >> cb (DirectionResult dir) >> continue +handlePromptEvent _ (Prompt _ SDirectionPrompt _ _ _) _ = continue + +handlePromptEvent _ (Prompt _ SMenu _ items' cb) (VtyEvent (EvKey (KChar chr) [])) + | Just (MenuOption _ res) <- items' ^. at chr + = clearPrompt >> cb (MenuResult res) >> continue + | otherwise + = continue + +handlePromptEvent + msg + (Prompt c SPointOnMap (PointOnMapPromptState pos) pri cb) + (VtyEvent (EvKey (KChar (directionFromChar -> Just dir)) [])) + = let pos' = move dir pos + prompt' = Prompt c SPointOnMap (PointOnMapPromptState pos') pri cb + in promptState .= WaitingPrompt msg prompt' + >> continue +handlePromptEvent _ (Prompt _ SPointOnMap _ _ _) _ = continue + +handlePromptEvent + msg + (Prompt c SFire (FirePromptState pos) pri@(origin, range) cb) + (VtyEvent (EvKey (KChar (directionFromChar -> Just dir)) [])) + = do + let pos' = move dir pos + prompt' = Prompt c SFire (FirePromptState pos') pri cb + when (Data.distance origin pos' <= range) $ + promptState .= WaitingPrompt msg prompt' + continue + +handlePromptEvent + _ + (Prompt Cancellable _ _ _ _) + (VtyEvent (EvKey (KChar 'q') [])) + = clearPrompt >> continue +handlePromptEvent _ _ _ = continue + +clearPrompt :: AppM () +clearPrompt = promptState .= NoPrompt + +type PromptParams :: PromptType -> Type +type family PromptParams pt where + PromptParams ('Menu a) = Map Char (MenuOption a) -- Menu items + PromptParams 'Fire = Tiles -- Range + PromptParams _ = () + +prompt + :: forall (pt :: PromptType) (params :: Type). + (ToJSON params, SingPromptType pt, PromptParams pt ~ ()) + => [Text] -- ^ Message key + -> params -- ^ Message params + -> PromptCancellable + -> (PromptResult pt -> AppM ()) -- ^ Prompt promise handler + -> AppM () +prompt msgPath params cancellable cb = do + let pt = singPromptType @pt + msg <- Messages.message msgPath params + mp :: Maybe (Prompt AppM) <- case pt of + SPointOnMap -> do + charPos <- use characterPosition + pure . Just $ mkPointOnMapPrompt cancellable charPos cb + SStringPrompt -> pure . Just $ mkStringPrompt cancellable cb + SConfirm -> pure . Just $ mkPrompt cancellable pt cb + SDirectionPrompt -> pure . Just $ mkPrompt cancellable pt cb + SContinue -> pure . Just $ mkPrompt cancellable pt cb + for_ mp $ \p -> promptState .= WaitingPrompt msg p + +prompt_ + :: forall (pt :: PromptType). + (SingPromptType pt, PromptParams pt ~ ()) + => [Text] -- ^ Message key + -> PromptCancellable + -> (PromptResult pt -> AppM ()) -- ^ Prompt promise handler + -> AppM () +prompt_ msg = prompt msg $ object [] + +stringPromptWithDefault + :: forall (params :: Type). (ToJSON params) + => [Text] -- ^ Message key + -> params -- ^ Message params + -> PromptCancellable + -> Text -- ^ Prompt default + -> (PromptResult 'StringPrompt -> AppM ()) -- ^ Prompt promise handler + -> AppM () +stringPromptWithDefault msgPath params cancellable def cb = do + msg <- Messages.message msgPath params + let p = mkStringPromptWithDefault cancellable def cb + promptState .= WaitingPrompt msg p + +stringPromptWithDefault_ + :: [Text] -- ^ Message key + -> PromptCancellable + -> Text -- ^ Prompt default + -> (PromptResult 'StringPrompt -> AppM ()) -- ^ Prompt promise handler + -> AppM () +stringPromptWithDefault_ msg = stringPromptWithDefault msg $ object [] + +confirm + :: ToJSON params + => [Text] -- ^ Message key + -> params + -> AppM () + -> AppM () +confirm msgPath params + = prompt @'Confirm msgPath params Cancellable . const + +confirm_ :: [Text] -> AppM () -> AppM () +confirm_ msgPath = confirm msgPath $ object [] + +menu :: forall (a :: Type) (params :: Type). + (ToJSON params) + => [Text] -- ^ Message key + -> params -- ^ Message params + -> PromptCancellable + -> Map Char (MenuOption a) -- ^ Menu items + -> (PromptResult ('Menu a) -> AppM ()) -- ^ Menu promise handler + -> AppM () +menu msgPath params cancellable items' cb = do + msg <- Messages.message msgPath params + let p = mkMenu cancellable items' cb + promptState .= WaitingPrompt msg p + +menu_ :: forall (a :: Type). + [Text] -- ^ Message key + -> PromptCancellable + -> Map Char (MenuOption a) -- ^ Menu items + -> (PromptResult ('Menu a) -> AppM ()) -- ^ Menu promise handler + -> AppM () +menu_ msgPath = menu msgPath $ object [] + +firePrompt_ + :: [Text] -- ^ Message key + -> PromptCancellable + -> Tiles -- ^ Range + -> (PromptResult 'Fire -> AppM ()) -- ^ Promise handler + -> AppM () +firePrompt_ msgPath cancellable range cb = do + msg <- Messages.message msgPath $ object [] + initialPos <- maybe (use characterPosition) pure =<< nearestEnemyPosition + let p = mkFirePrompt cancellable initialPos range cb + promptState .= WaitingPrompt msg p + +-- | Returns the position of the nearest visible hostile creature, if any +nearestEnemyPosition :: AppM (Maybe Position) +nearestEnemyPosition = do + charPos <- use characterPosition + em <- use entities + ps <- characterVisiblePositions + let candidates = toList ps >>= \p -> + let ents = EntityMap.atPositionWithIDs p em + in ents + ^.. folded + . _2 + . positioned + . _SomeEntity @Creature + . creatureType + . filtered (view hostile) + . to (const (distance charPos p, p)) + pure . headMay . fmap snd $ sortOn fst candidates + where + distance :: Position -> Position -> Double + distance = Metric.distance `on` (fmap fromIntegral . view _Position) diff --git a/users/grfn/xanthous/src/Xanthous/App/Time.hs b/users/grfn/xanthous/src/Xanthous/App/Time.hs new file mode 100644 index 000000000000..cca352858d9c --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/App/Time.hs @@ -0,0 +1,42 @@ +-------------------------------------------------------------------------------- +module Xanthous.App.Time + ( stepGame + , stepGameBy + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import System.Exit +-------------------------------------------------------------------------------- +import Xanthous.Data (Ticks) +import Xanthous.App.Prompt +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Entities.Character (isDead) +import Xanthous.Game.State +import Xanthous.Game.Prompt +import Xanthous.Game.Lenses +import Control.Monad.State (modify) +import qualified Xanthous.Game.Memo as Memo +-------------------------------------------------------------------------------- + + +stepGameBy :: Ticks -> AppM () +stepGameBy ticks = do + ents <- uses entities EntityMap.toEIDsAndPositioned + for_ ents $ \(eid, pEntity) -> do + pEntity' <- step ticks pEntity + entities . ix eid .= pEntity' + + clearMemo Memo.characterVisiblePositions + modify updateCharacterVision + + whenM (uses character isDead) + . prompt_ @'Continue ["dead"] Uncancellable + . const . lift . liftIO + $ exitSuccess + +ticksPerTurn :: Ticks +ticksPerTurn = 100 + +stepGame :: AppM () +stepGame = stepGameBy ticksPerTurn diff --git a/users/grfn/xanthous/src/Xanthous/Command.hs b/users/grfn/xanthous/src/Xanthous/Command.hs new file mode 100644 index 000000000000..187e5c16d7da --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Command.hs @@ -0,0 +1,84 @@ +-------------------------------------------------------------------------------- +module Xanthous.Command where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (Left, Right, Down) +-------------------------------------------------------------------------------- +import Graphics.Vty.Input (Key(..), Modifier(..)) +import qualified Data.Char as Char +-------------------------------------------------------------------------------- +import Xanthous.Data (Direction(..)) +-------------------------------------------------------------------------------- + +data Command + = Quit + | Move Direction + | StartAutoMove Direction + | PreviousMessage + | PickUp + | Drop + | Open + | Close + | Wait + | Eat + | Look + | Save + | Read + | ShowInventory + | DescribeInventory + | Wield + | Fire + | GoUp + | GoDown + | Rest + + -- | TODO replace with `:` commands + | ToggleRevealAll + +commandFromKey :: Key -> [Modifier] -> Maybe Command +commandFromKey (KChar 'q') [] = Just Quit +commandFromKey (KChar '.') [] = Just Wait +commandFromKey (KChar (directionFromChar -> Just dir)) [] = Just $ Move dir +commandFromKey (KChar c) [] + | Char.isUpper c + , Just dir <- directionFromChar $ Char.toLower c + = Just $ StartAutoMove dir +commandFromKey (KChar 'p') [MCtrl] = Just PreviousMessage +commandFromKey (KChar ',') [] = Just PickUp +commandFromKey (KChar 'd') [] = Just Drop +commandFromKey (KChar 'o') [] = Just Open +commandFromKey (KChar 'c') [] = Just Close +commandFromKey (KChar ';') [] = Just Look +commandFromKey (KChar 'e') [] = Just Eat +commandFromKey (KChar 'S') [] = Just Save +commandFromKey (KChar 'r') [] = Just Read +commandFromKey (KChar 'i') [] = Just ShowInventory +commandFromKey (KChar 'I') [] = Just DescribeInventory +commandFromKey (KChar 'w') [] = Just Wield +commandFromKey (KChar 'f') [] = Just Fire +commandFromKey (KChar '<') [] = Just GoUp +commandFromKey (KChar '>') [] = Just GoDown +commandFromKey (KChar 'R') [] = Just Rest + +commandFromKey KUp [] = Just $ Move Up +commandFromKey KDown [] = Just $ Move Down +commandFromKey KLeft [] = Just $ Move Left +commandFromKey KRight [] = Just $ Move Right + +-- DEBUG COMMANDS -- +commandFromKey (KChar 'r') [MMeta] = Just ToggleRevealAll + +commandFromKey _ _ = Nothing + +-------------------------------------------------------------------------------- + +directionFromChar :: Char -> Maybe Direction +directionFromChar 'h' = Just Left +directionFromChar 'j' = Just Down +directionFromChar 'k' = Just Up +directionFromChar 'l' = Just Right +directionFromChar 'y' = Just UpLeft +directionFromChar 'u' = Just UpRight +directionFromChar 'b' = Just DownLeft +directionFromChar 'n' = Just DownRight +directionFromChar '.' = Just Here +directionFromChar _ = Nothing diff --git a/users/grfn/xanthous/src/Xanthous/Data.hs b/users/grfn/xanthous/src/Xanthous/Data.hs new file mode 100644 index 000000000000..1b67e0f160db --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data.hs @@ -0,0 +1,818 @@ +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE RoleAnnotations #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE DeriveTraversable #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE NoTypeSynonymInstances #-} +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE AllowAmbiguousTypes #-} +-------------------------------------------------------------------------------- +-- | Common data types for Xanthous ------------------------------------------------------------------------------ +module Xanthous.Data + ( Opposite(..) + + -- * + , Position'(..) + , Position + , x + , y + + -- ** + , Positioned(..) + , _Positioned + , position + , positioned + , loc + , _Position + , positionFromPair + , positionFromV2 + , addPositions + , diffPositions + , stepTowards + , isUnit + , distance + + -- * Boxes + , Box(..) + , topLeftCorner + , bottomRightCorner + , setBottomRightCorner + , dimensions + , inBox + , boxIntersects + , boxCenter + , boxEdge + , module Linear.V2 + + -- * Unit math + , Scalar(..) + , Per(..) + , invertRate + , invertedRate + , (|+|) + , (|*|) + , (|/|) + , (:+:) + , (:*:) + , (:/:) + , (:**:)(..) + , Ticks(..) + , Tiles(..) + , TicksPerTile + , TilesPerTick + , timesTiles + , Square(..) + , squared + , Cubic(..) + , Grams + , Meters + , Uno(..) + , Unit(..) + , UnitSymbol(..) + + -- * + , Dimensions'(..) + , Dimensions + , HasWidth(..) + , HasHeight(..) + + -- * + , Direction(..) + , move + , asPosition + , directionOf + , Cardinal(..) + + -- * + , Corner(..) + , Edge(..) + , cornerEdges + + -- * + , Neighbors(..) + , edges + , neighborDirections + , neighborPositions + , neighborCells + , arrayNeighbors + , rotations + , HasTopLeft(..) + , HasTop(..) + , HasTopRight(..) + , HasLeft(..) + , HasRight(..) + , HasBottomLeft(..) + , HasBottom(..) + , HasBottomRight(..) + + -- * + , Hitpoints(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (Left, Down, Right, (.=), elements) +-------------------------------------------------------------------------------- +import Linear.V2 hiding (_x, _y) +import qualified Linear.V2 as L +import Linear.V4 hiding (_x, _y) +import Test.QuickCheck (CoArbitrary, Function, elements) +import Test.QuickCheck.Arbitrary.Generic +import Data.Group +import Brick (Location(Location), Edges(..)) +import Data.Monoid (Product(..), Sum(..)) +import Data.Array.IArray +import Data.Aeson.Generic.DerivingVia +import Data.Aeson + ( ToJSON(..), FromJSON(..), object, (.=), (.:), withObject) +import Data.Random (Distribution) +import Data.Coerce +import Data.Proxy (Proxy(Proxy)) +-------------------------------------------------------------------------------- +import Xanthous.Util (EqEqProp(..), EqProp, between) +import Xanthous.Util.QuickCheck (GenericArbitrary(..)) +import Xanthous.Orphans () +import Xanthous.Util.Graphics +import qualified Linear.Metric as Metric +-------------------------------------------------------------------------------- + +-- | opposite ∘ opposite ≡ id +class Opposite x where + opposite :: x -> x + +-------------------------------------------------------------------------------- + +-- fromScalar ∘ scalar ≡ id +class Scalar a where + scalar :: a -> Double + fromScalar :: Double -> a + +instance Scalar Double where + scalar = id + fromScalar = id + +newtype ScalarIntegral a = ScalarIntegral a + deriving newtype (Eq, Ord, Num, Enum, Real, Integral) +instance Integral a => Scalar (ScalarIntegral a) where + scalar = fromIntegral + fromScalar = floor + +deriving via (ScalarIntegral Integer) instance Scalar Integer +deriving via (ScalarIntegral Word) instance Scalar Word + +-- | Units of measure +class Unit a where + unitSuffix :: Text +type UnitSymbol :: Symbol -> Type -> Type +newtype UnitSymbol suffix a = UnitSymbol a +instance KnownSymbol suffix => Unit (UnitSymbol suffix a) where + unitSuffix = pack $ symbolVal @suffix Proxy + +newtype ShowUnitSuffix a b = ShowUnitSuffix a +instance (Show b, Unit a, Coercible a b) => Show (ShowUnitSuffix a b) where + show a = show (coerce @_ @b a) <> " " <> unpack (unitSuffix @a) + +-------------------------------------------------------------------------------- + +data Position' a where + Position :: { _x :: a + , _y :: a + } -> (Position' a) + deriving stock (Show, Eq, Generic, Ord, Functor, Foldable, Traversable) + deriving anyclass (NFData, Hashable, CoArbitrary, Function) + deriving EqProp via EqEqProp (Position' a) + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + (Position' a) + +x, y :: Lens' (Position' a) a +x = lens (\(Position xx _) -> xx) (\(Position _ yy) xx -> Position xx yy) +y = lens (\(Position _ yy) -> yy) (\(Position xx _) yy -> Position xx yy) + +type Position = Position' Int + +instance Arbitrary a => Arbitrary (Position' a) where + arbitrary = genericArbitrary + shrink (Position px py) = Position <$> shrink px <*> shrink py + + +instance Num a => Semigroup (Position' a) where + (Position x₁ y₁) <> (Position x₂ y₂) = Position (x₁ + x₂) (y₁ + y₂) + +instance Num a => Monoid (Position' a) where + mempty = Position 0 0 + +instance Num a => Group (Position' a) where + invert (Position px py) = Position (negate px) (negate py) + +-- | Positions convert to scalars by discarding their orientation and just +-- measuring the length from the origin +instance (Ord a, Num a, Scalar a) => Scalar (Position' a) where + scalar = fromIntegral . length . line 0 . view _Position + fromScalar n = Position (fromScalar n) (fromScalar n) + +data Positioned a where + Positioned :: Position -> a -> Positioned a + deriving stock (Show, Eq, Ord, Functor, Foldable, Traversable, Generic) + deriving anyclass (NFData, CoArbitrary, Function) +type role Positioned representational + +_Positioned :: Iso (Position, a) (Position, b) (Positioned a) (Positioned b) +_Positioned = iso hither yon + where + hither (pos, a) = Positioned pos a + yon (Positioned pos b) = (pos, b) + +instance Arbitrary a => Arbitrary (Positioned a) where + arbitrary = Positioned <$> arbitrary <*> arbitrary + +instance ToJSON a => ToJSON (Positioned a) where + toJSON (Positioned pos val) = object + [ "position" .= pos + , "data" .= val + ] + +instance FromJSON a => FromJSON (Positioned a) where + parseJSON = withObject "Positioned" $ \obj -> + Positioned <$> obj .: "position" <*> obj .: "data" + +position :: Lens' (Positioned a) Position +position = lens + (\(Positioned pos _) -> pos) + (\(Positioned _ a) pos -> Positioned pos a) + +positioned :: Lens (Positioned a) (Positioned b) a b +positioned = lens + (\(Positioned _ x') -> x') + (\(Positioned pos _) x' -> Positioned pos x') + +loc :: Iso' Position Location +loc = iso hither yon + where + hither (Position px py) = Location (px, py) + yon (Location (lx, ly)) = Position lx ly + +_Position :: Iso' (Position' a) (V2 a) +_Position = iso hither yon + where + hither (Position px py) = V2 px py + yon (V2 lx ly) = Position lx ly + +positionFromPair :: (Num a, Integral i, Integral j) => (i, j) -> Position' a +positionFromPair (i, j) = Position (fromIntegral i) (fromIntegral j) + +positionFromV2 :: (Num a, Integral i) => V2 i -> Position' a +positionFromV2 (V2 xx yy) = Position (fromIntegral xx) (fromIntegral yy) + +-- | Add two positions +-- +-- Operation for the additive group on positions +addPositions :: Num a => Position' a -> Position' a -> Position' a +addPositions = (<>) + +-- | Subtract two positions. +-- +-- diffPositions pos₁ pos₂ = pos₁ `addPositions` (invert pos₂) +diffPositions :: Num a => Position' a -> Position' a -> Position' a +diffPositions (Position x₁ y₁) (Position x₂ y₂) = Position (x₁ - x₂) (y₁ - y₂) + +-- | Is this position a unit position? or: When taken as a difference, does this +-- position represent a step of one tile? +-- +-- ∀ dir :: Direction. isUnit ('asPosition' dir) +isUnit :: (Eq a, Num a) => Position' a -> Bool +isUnit (Position px py) = + abs px `elem` [0,1] && abs py `elem` [0, 1] && (px, py) /= (0, 0) + +-------------------------------------------------------------------------------- + +data Dimensions' a = Dimensions + { _width :: a + , _height :: a + } + deriving stock (Show, Eq, Functor, Generic) + deriving anyclass (CoArbitrary, Function) +makeFieldsNoPrefix ''Dimensions' + +instance Arbitrary a => Arbitrary (Dimensions' a) where + arbitrary = Dimensions <$> arbitrary <*> arbitrary + +type Dimensions = Dimensions' Word + +-------------------------------------------------------------------------------- + +data Direction where + Up :: Direction + Down :: Direction + Left :: Direction + Right :: Direction + UpLeft :: Direction + UpRight :: Direction + DownLeft :: Direction + DownRight :: Direction + Here :: Direction + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (CoArbitrary, Function, NFData, ToJSON, FromJSON, Hashable) + deriving Arbitrary via GenericArbitrary Direction + +instance Opposite Direction where + opposite Up = Down + opposite Down = Up + opposite Left = Right + opposite Right = Left + opposite UpLeft = DownRight + opposite UpRight = DownLeft + opposite DownLeft = UpRight + opposite DownRight = UpLeft + opposite Here = Here + +move :: Num a => Direction -> Position' a -> Position' a +move Up = y -~ 1 +move Down = y +~ 1 +move Left = x -~ 1 +move Right = x +~ 1 +move UpLeft = move Up . move Left +move UpRight = move Up . move Right +move DownLeft = move Down . move Left +move DownRight = move Down . move Right +move Here = id + +asPosition :: Direction -> Position +asPosition dir = move dir mempty + +-- | Returns the direction that a given position is from a given source position +directionOf + :: Position -- ^ Source + -> Position -- ^ Target + -> Direction +directionOf (Position x₁ y₁) (Position x₂ y₂) = + case (x₁ `compare` x₂, y₁ `compare` y₂) of + (EQ, EQ) -> Here + (EQ, LT) -> Down + (EQ, GT) -> Up + (LT, EQ) -> Right + (GT, EQ) -> Left + + (LT, LT) -> DownRight + (GT, LT) -> DownLeft + + (LT, GT) -> UpRight + (GT, GT) -> UpLeft + +-- | Take one (potentially diagonal) step towards the given position +-- +-- ∀ src tgt. isUnit (src `diffPositions` (src `stepTowards tgt`)) +stepTowards + :: Position -- ^ Source + -> Position -- ^ Target + -> Position +stepTowards (view _Position -> p₁) (view _Position -> p₂) + | p₁ == p₂ = _Position # p₁ + | otherwise = + let (_:p:_) = line p₁ p₂ + in _Position # p + +-- | Newtype controlling arbitrary generation to only include cardinal +-- directions ('Up', 'Down', 'Left', 'Right') +newtype Cardinal = Cardinal { getCardinal :: Direction } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, Function, CoArbitrary) + deriving newtype (Opposite) + +instance Arbitrary Cardinal where + arbitrary = Cardinal <$> elements [Up, Down, Left, Right] + +-------------------------------------------------------------------------------- + +data Corner + = TopLeft + | TopRight + | BottomLeft + | BottomRight + deriving stock (Show, Eq, Ord, Enum, Bounded, Generic) + deriving Arbitrary via GenericArbitrary Corner + +instance Opposite Corner where + opposite TopLeft = BottomRight + opposite TopRight = BottomLeft + opposite BottomLeft = TopRight + opposite BottomRight = TopLeft + +data Edge + = TopEdge + | LeftEdge + | RightEdge + | BottomEdge + deriving stock (Show, Eq, Ord, Enum, Bounded, Generic) + deriving Arbitrary via GenericArbitrary Edge + +instance Opposite Edge where + opposite TopEdge = BottomEdge + opposite BottomEdge = TopEdge + opposite LeftEdge = RightEdge + opposite RightEdge = LeftEdge + +cornerEdges :: Corner -> (Edge, Edge) +cornerEdges TopLeft = (TopEdge, LeftEdge) +cornerEdges TopRight = (TopEdge, RightEdge) +cornerEdges BottomLeft = (BottomEdge, LeftEdge) +cornerEdges BottomRight = (BottomEdge, RightEdge) + +-------------------------------------------------------------------------------- + +data Neighbors a = Neighbors + { _topLeft + , _top + , _topRight + , _left + , _right + , _bottomLeft + , _bottom + , _bottomRight :: a + } + deriving stock (Show, Eq, Ord, Functor, Foldable, Traversable, Generic) + deriving anyclass (NFData, CoArbitrary, Function, MonoFoldable) + deriving Arbitrary via GenericArbitrary (Neighbors a) + +type instance Element (Neighbors a) = a + +makeFieldsNoPrefix ''Neighbors + +instance Applicative Neighbors where + pure α = Neighbors + { _topLeft = α + , _top = α + , _topRight = α + , _left = α + , _right = α + , _bottomLeft = α + , _bottom = α + , _bottomRight = α + } + nf <*> nx = Neighbors + { _topLeft = nf ^. topLeft $ nx ^. topLeft + , _top = nf ^. top $ nx ^. top + , _topRight = nf ^. topRight $ nx ^. topRight + , _left = nf ^. left $ nx ^. left + , _right = nf ^. right $ nx ^. right + , _bottomLeft = nf ^. bottomLeft $ nx ^. bottomLeft + , _bottom = nf ^. bottom $ nx ^. bottom + , _bottomRight = nf ^. bottomRight $ nx ^. bottomRight + } + +edges :: Neighbors a -> Edges a +edges neighs = Edges + { eTop = neighs ^. top + , eBottom = neighs ^. bottom + , eLeft = neighs ^. left + , eRight = neighs ^. right + } + +neighborDirections :: Neighbors Direction +neighborDirections = Neighbors + { _topLeft = UpLeft + , _top = Up + , _topRight = UpRight + , _left = Left + , _right = Right + , _bottomLeft = DownLeft + , _bottom = Down + , _bottomRight = DownRight + } + +neighborPositions :: Num a => Position' a -> Neighbors (Position' a) +neighborPositions pos = (`move` pos) <$> neighborDirections + +neighborCells :: Num a => V2 a -> Neighbors (V2 a) +neighborCells = map (view _Position) . neighborPositions . review _Position + +arrayNeighbors + :: (IArray a e, Ix i, Num i) + => a (V2 i) e + -> V2 i + -> Neighbors (Maybe e) +arrayNeighbors arr center = arrLookup <$> neighborPositions (_Position # center) + where + arrLookup (view _Position -> pos) + | inRange (bounds arr) pos = Just $ arr ! pos + | otherwise = Nothing + +-- | Returns a list of all 4 90-degree rotations of the given neighbors +rotations :: Neighbors a -> V4 (Neighbors a) +rotations orig@(Neighbors tl t tr l r bl b br) = V4 + orig -- tl t tr + -- l r + -- bl b br + + (Neighbors bl l tl b t br r tr) -- bl l tl + -- b t + -- br r tr + + (Neighbors br b bl r l tr t tl) -- br b bl + -- r l + -- tr t tl + + (Neighbors tr r br t b tl l bl) -- tr r br + -- t b + -- tl l bl + +-------------------------------------------------------------------------------- + +newtype Per a b = Rate Double + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (Arbitrary, Num, Ord, Enum, Real, Fractional, ToJSON, FromJSON) + via Double + deriving (Semigroup, Monoid) via Product Double + deriving Show via ShowUnitSuffix (Per a b) Double +deriving via Double + instance ( Distribution d Double + , forall xx yy. Coercible xx yy => Coercible (d xx) (d yy) + ) + => Distribution d (Per a b) + +instance (Unit a, Unit b) => Unit (a `Per` b) where + unitSuffix = unitSuffix @a <> "/" <> unitSuffix @b + +invertRate :: a `Per` b -> b `Per` a +invertRate (Rate p) = Rate $ 1 / p + +invertedRate :: Iso (a `Per` b) (b' `Per` a') (b `Per` a) (a' `Per` b') +invertedRate = iso invertRate invertRate + +type (:+:) :: Type -> Type -> Type +type family (:+:) a b where + a :+: a = a + a :+: (Uno b) = a + +infixl 6 |+| +class AddUnit a b where + (|+|) :: a -> b -> a :+: b + +instance Scalar a => AddUnit a a where + x' |+| y' = fromScalar $ scalar x' + scalar y' + +instance (Scalar a, Scalar b) => AddUnit a (Uno b) where + x' |+| y' = fromScalar $ scalar x' + scalar y' + +type (:*:) :: Type -> Type -> Type +type family (:*:) a b where + (a `Per` b) :*: b = a + (Square a) :*: a = Cubic a + a :*: a = Square a + a :*: Uno b = a + a :*: b = a :**: b + +infixl 7 |*| +class MulUnit a b where + (|*|) :: a -> b -> a :*: b + +instance (Scalar a, Scalar b) => MulUnit (a `Per` b) b where + (Rate rate) |*| b = fromScalar $ rate * scalar b + +instance forall a. (Scalar a, a :*: a ~ Square a) => MulUnit a a where + x' |*| y' = Square @a . fromScalar $ scalar x' * scalar y' + +instance forall a. (Scalar a) => MulUnit (Square a) a where + x' |*| y' = Cubic @a . fromScalar $ scalar x' * scalar y' + +instance {-# INCOHERENT #-} forall a b. + (Scalar a, Scalar b, Scalar (a :*: Uno b)) + => MulUnit a (Uno b) where + x' |*| y' = fromScalar $ scalar x' * scalar y' + +type (:/:) :: Type -> Type -> Type +type family (:/:) a b where + (Square a) :/: a = a + (Cubic a) :/: a = Square a + (Cubic a) :/: (Square a) = a + (a :**: b) :/: b = a + (a :**: b) :/: a = b + a :/: Uno b = a + a :/: b = a `Per` b + +infixl 7 |/| +class DivUnit a b where + (|/|) :: a -> b -> a :/: b + +instance Scalar a => DivUnit (Square a) a where + (Square a) |/| b = fromScalar $ scalar a / scalar b + +instance Scalar a => DivUnit (Cubic a) a where + (Cubic a) |/| b = fromScalar $ scalar a / scalar b + +instance (Scalar a, Cubic a :/: Square a ~ a) + => DivUnit (Cubic a) (Square a) where + (Cubic a) |/| (Square b) = fromScalar $ scalar a / scalar b + +instance (Scalar a, Scalar b) => DivUnit (a :**: b) b where + (Times a) |/| b = fromScalar $ scalar a / scalar b + +instance (Scalar a, Scalar b) => DivUnit (a :**: b) a where + (Times a) |/| b = fromScalar $ scalar a / scalar b + +instance {-# INCOHERENT #-} forall a b. + (Scalar a, Scalar b, Scalar (a :/: Uno b)) + => DivUnit a (Uno b) where + x' |/| y' = fromScalar $ scalar x' / scalar y' + +-- | Dimensionless quantitites (mass per unit mass, radians, etc) +-- +-- see <https://en.wikipedia.org/wiki/Parts-per_notation#Uno> +newtype Uno a = Uno a + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving ( Arbitrary, Num, Ord, Enum, Real, Fractional, ToJSON, FromJSON + , Scalar, Show + ) + via a + deriving Unit via UnitSymbol "" (Uno a) + +newtype Square a = Square a + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving ( Arbitrary, Num, Ord, Enum, Real, Fractional, ToJSON, FromJSON + , Scalar + ) + via a +deriving via (a :: Type) + instance ( Distribution d a + , forall xx yy. Coercible xx yy => Coercible (d xx) (d yy) + ) + => Distribution d (Square a) + +instance Unit a => Unit (Square a) where + unitSuffix = unitSuffix @a <> "²" + +instance Show a => Show (Square a) where + show (Square n) = show n <> "²" + +squared :: (Scalar a, a :*: a ~ Square a) => a -> Square a +squared v = v |*| v + +newtype Cubic a = Cubic a + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving ( Arbitrary, Num, Ord, Enum, Real, Fractional, ToJSON, FromJSON + , Scalar + ) + via a +deriving via (a :: Type) + instance ( Distribution d a + , forall xx yy. Coercible xx yy => Coercible (d xx) (d yy) + ) + => Distribution d (Cubic a) + +instance Unit a => Unit (Cubic a) where + unitSuffix = unitSuffix @a <> "³" + +instance Show a => Show (Cubic a) where + show (Cubic n) = show n <> "³" + +newtype (:**:) a b = Times Double + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (Arbitrary, Num, Ord, Enum, Real, Fractional, ToJSON, FromJSON) + via Double + deriving (Semigroup, Monoid) via Sum Double + deriving Show via ShowUnitSuffix (a :**: b) Double +deriving via Double + instance ( Distribution d Double + , forall xx yy. Coercible xx yy => Coercible (d xx) (d yy) + ) + => Distribution d (a :**: b) + +instance (Unit a, Unit b) => Unit (a :**: b) where + unitSuffix = unitSuffix @a <> " " <> unitSuffix @b + +-------------------------------------------------------------------------------- + +newtype Ticks = Ticks Word + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (Num, Ord, Bounded, Enum, Integral, Real, ToJSON, FromJSON) via Word + deriving (Semigroup, Monoid) via (Sum Word) + deriving Scalar via ScalarIntegral Ticks + deriving Arbitrary via GenericArbitrary Ticks + deriving Unit via UnitSymbol "ticks" Ticks + deriving Show via ShowUnitSuffix Ticks Word +deriving via Word + instance ( Distribution d Word + , forall xx yy. Coercible xx yy => Coercible (d xx) (d yy) + ) + => Distribution d Ticks + +newtype Tiles = Tiles Double + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (Num, Ord, Enum, Real, ToJSON, FromJSON, Scalar) via Double + deriving (Semigroup, Monoid) via (Sum Double) + deriving Arbitrary via GenericArbitrary Tiles + deriving Unit via UnitSymbol "m" Tiles + deriving Show via ShowUnitSuffix Tiles Double +deriving via Double + instance ( Distribution d Double + , forall xx yy. Coercible xx yy => Coercible (d xx) (d yy) + ) + => Distribution d Tiles + +type TicksPerTile = Ticks `Per` Tiles +type TilesPerTick = Tiles `Per` Ticks + +timesTiles :: TicksPerTile -> Tiles -> Ticks +timesTiles = (|*|) + +-- | Calculate the (cartesian) distance between two 'Position's, floored and +-- represented as a number of 'Tile's +-- +-- Note that this is imprecise, and may be different than the length of a +-- bresenham's line between the points +distance :: Position -> Position -> Tiles +distance + = (fromScalar .) . (Metric.distance `on` (fmap fromIntegral . view _Position)) + +-------------------------------------------------------------------------------- + +newtype Hitpoints = Hitpoints Word + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving ( Arbitrary, Num, Ord, Bounded, Enum, Integral, Real, Scalar + , ToJSON, FromJSON + ) + via Word + deriving (Semigroup, Monoid) via Sum Word + deriving Unit via UnitSymbol "hp" Hitpoints + deriving Show via ShowUnitSuffix Hitpoints Word + +-------------------------------------------------------------------------------- + +-- | Grams, the fundamental measure of weight in Xanthous. +newtype Grams = Grams Double + deriving stock (Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving ( Arbitrary, Num, Ord, Enum, Real, Floating, Fractional, RealFloat + , RealFrac, Scalar, ToJSON, FromJSON + ) + via Double + deriving (Semigroup, Monoid) via Sum Double + deriving Unit via UnitSymbol "g" Grams + deriving Show via ShowUnitSuffix Grams Double + +-- | Every tile is 1 meter +type Meters = Tiles + +-------------------------------------------------------------------------------- + +data Box a = Box + { _topLeftCorner :: V2 a + , _dimensions :: V2 a + } + deriving stock (Show, Eq, Ord, Functor, Generic) + deriving Arbitrary via GenericArbitrary (Box a) +makeFieldsNoPrefix ''Box + +bottomRightCorner :: Num a => Box a -> V2 a +bottomRightCorner box = + V2 (box ^. topLeftCorner . L._x + box ^. dimensions . L._x) + (box ^. topLeftCorner . L._y + box ^. dimensions . L._y) + +setBottomRightCorner :: (Num a, Ord a) => Box a -> V2 a -> Box a +setBottomRightCorner box br@(V2 brx bry) + | brx < box ^. topLeftCorner . L._x || bry < box ^. topLeftCorner . L._y + = box & topLeftCorner .~ br + & dimensions . L._x .~ ((box ^. topLeftCorner . L._x) - brx) + & dimensions . L._y .~ ((box ^. topLeftCorner . L._y) - bry) + | otherwise + = box & dimensions . L._x .~ (brx - (box ^. topLeftCorner . L._x)) + & dimensions . L._y .~ (bry - (box ^. topLeftCorner . L._y)) + +inBox :: (Ord a, Num a) => Box a -> V2 a -> Bool +inBox box pt = flip all [L._x, L._y] $ \component -> + between (box ^. topLeftCorner . component) + (box ^. to bottomRightCorner . component) + (pt ^. component) + +boxIntersects :: (Ord a, Num a) => Box a -> Box a -> Bool +boxIntersects box₁ box₂ + = any (inBox box₁) [box₂ ^. topLeftCorner, bottomRightCorner box₂] + +boxCenter :: (Fractional a) => Box a -> V2 a +boxCenter box = V2 cx cy + where + cx = box ^. topLeftCorner . L._x + (box ^. dimensions . L._x / 2) + cy = box ^. topLeftCorner . L._y + (box ^. dimensions . L._y / 2) + +boxEdge :: (Enum a, Num a) => Box a -> Edge -> [V2 a] +boxEdge box LeftEdge = + V2 (box ^. topLeftCorner . L._x) + <$> [box ^. topLeftCorner . L._y .. box ^. to bottomRightCorner . L._y] +boxEdge box RightEdge = + V2 (box ^. to bottomRightCorner . L._x) + <$> [box ^. to bottomRightCorner . L._y .. box ^. to bottomRightCorner . L._y] +boxEdge box TopEdge = + flip V2 (box ^. topLeftCorner . L._y) + <$> [box ^. topLeftCorner . L._x .. box ^. to bottomRightCorner . L._x] +boxEdge box BottomEdge = + flip V2 (box ^. to bottomRightCorner . L._y) + <$> [box ^. topLeftCorner . L._x .. box ^. to bottomRightCorner . L._x] diff --git a/users/grfn/xanthous/src/Xanthous/Data/App.hs b/users/grfn/xanthous/src/Xanthous/Data/App.hs new file mode 100644 index 000000000000..a2cfcb8001cb --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/App.hs @@ -0,0 +1,45 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.App + ( Panel(..) + , ResourceName(..) + , AppEvent(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Test.QuickCheck +import Test.QuickCheck.Instances.Text () +import Data.Aeson (ToJSON, FromJSON) +-------------------------------------------------------------------------------- +import Xanthous.Util.QuickCheck +-------------------------------------------------------------------------------- + +-- | Enum for "panels" displayed in the game's UI. +data Panel + = -- | A panel displaying the character's inventory + InventoryPanel + | -- | A panel describing an item in the inventory in detail + -- + -- The argument is the full description of the item + ItemDescriptionPanel Text + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function, ToJSON, FromJSON) + deriving Arbitrary via GenericArbitrary Panel + + +data ResourceName + = MapViewport -- ^ The main viewport where we display the game content + | Character -- ^ The character + | MessageBox -- ^ The box where we display messages to the user + | Prompt -- ^ The game's prompt + | Panel Panel -- ^ A panel in the game + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function, ToJSON, FromJSON) + deriving Arbitrary via GenericArbitrary ResourceName + +data AppEvent + = AutoContinue -- ^ Continue whatever autocommand has been requested by the + -- user + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function, ToJSON, FromJSON) + deriving Arbitrary via GenericArbitrary AppEvent diff --git a/users/grfn/xanthous/src/Xanthous/Data/Entities.hs b/users/grfn/xanthous/src/Xanthous/Data/Entities.hs new file mode 100644 index 000000000000..39953410f2f3 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/Entities.hs @@ -0,0 +1,68 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.Entities + ( -- * Collisions + Collision(..) + , _Stop + , _Combat + -- * Entity Attributes + , EntityAttributes(..) + , blocksVision + , blocksObject + , collision + , defaultEntityAttributes + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson (ToJSON(..), FromJSON(..), (.:?), (.!=), withObject) +import Data.Aeson.Generic.DerivingVia +import Xanthous.Util.QuickCheck (GenericArbitrary(..)) +import Test.QuickCheck +-------------------------------------------------------------------------------- + +data Collision + = Stop -- ^ Can't move through this + | Combat -- ^ Moving into this equates to hitting it with a stick + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Collision + deriving (ToJSON, FromJSON) + via WithOptions '[ AllNullaryToStringTag 'True ] + Collision +makePrisms ''Collision + +-- | Attributes of an entity +data EntityAttributes = EntityAttributes + { _blocksVision :: Bool + -- | Does this entity block a large object from being put in the same tile as + -- it - eg a a door being closed on it + , _blocksObject :: Bool + -- | What type of collision happens when moving into this entity? + , _collision :: Collision + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary EntityAttributes + deriving (ToJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + EntityAttributes +makeLenses ''EntityAttributes + +instance FromJSON EntityAttributes where + parseJSON = withObject "EntityAttributes" $ \o -> do + _blocksVision <- o .:? "blocksVision" + .!= _blocksVision defaultEntityAttributes + _blocksObject <- o .:? "blocksObject" + .!= _blocksObject defaultEntityAttributes + _collision <- o .:? "collision" + .!= _collision defaultEntityAttributes + pure EntityAttributes {..} + +defaultEntityAttributes :: EntityAttributes +defaultEntityAttributes = EntityAttributes + { _blocksVision = False + , _blocksObject = False + , _collision = Stop + } diff --git a/users/grfn/xanthous/src/Xanthous/Data/EntityChar.hs b/users/grfn/xanthous/src/Xanthous/Data/EntityChar.hs new file mode 100644 index 000000000000..855a3462daee --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/EntityChar.hs @@ -0,0 +1,56 @@ +{-# LANGUAGE RoleAnnotations #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.EntityChar + ( EntityChar(..) + , HasChar(..) + , HasStyle(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding ((.=)) +-------------------------------------------------------------------------------- +import qualified Graphics.Vty.Attributes as Vty +import Test.QuickCheck +import Data.Aeson +-------------------------------------------------------------------------------- +import Xanthous.Orphans () +import Xanthous.Util.QuickCheck (GenericArbitrary(..)) +-------------------------------------------------------------------------------- + + +class HasChar s a | s -> a where + char :: Lens' s a + {-# MINIMAL char #-} + +data EntityChar = EntityChar + { _char :: Char + , _style :: Vty.Attr + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary EntityChar +makeFieldsNoPrefix ''EntityChar + +instance FromJSON EntityChar where + parseJSON (String (chr :< Empty)) = pure $ EntityChar chr Vty.defAttr + parseJSON (Object o) = do + (EntityChar _char _) <- o .: "char" + _style <- o .:? "style" .!= Vty.defAttr + pure EntityChar {..} + parseJSON _ = fail "Invalid type, expected string or object" + +instance ToJSON EntityChar where + toJSON (EntityChar chr styl) + | styl == Vty.defAttr = String $ chr <| Empty + | otherwise = object + [ "char" .= chr + , "style" .= styl + ] + +instance IsString EntityChar where + fromString [ch] = EntityChar ch Vty.defAttr + fromString _ = error "Entity char must only be a single character" diff --git a/users/grfn/xanthous/src/Xanthous/Data/EntityMap.hs b/users/grfn/xanthous/src/Xanthous/Data/EntityMap.hs new file mode 100644 index 000000000000..1d9c4d46cdc9 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/EntityMap.hs @@ -0,0 +1,277 @@ +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE DeriveTraversable #-} +{-# LANGUAGE TupleSections #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE DeriveFunctor #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.EntityMap + ( EntityMap + , _EntityMap + , EntityID + , emptyEntityMap + , insertAt + , insertAtReturningID + , fromEIDsAndPositioned + , toEIDsAndPositioned + , atPosition + , atPositionWithIDs + , positions + , lookup + , lookupWithPosition + , positionOf + -- , positionedEntities + , neighbors + , Deduplicate(..) + + -- * debug + , byID + , byPosition + , lastID + + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (lookup) +import Xanthous.Data + ( Position + , Positioned(..) + , positioned + , Neighbors(..) + , neighborPositions, position + ) +import Xanthous.Data.VectorBag +import Xanthous.Orphans () +import Xanthous.Util (EqEqProp(..)) +-------------------------------------------------------------------------------- +import Data.Monoid (Endo(..)) +import Test.QuickCheck (Arbitrary(..), CoArbitrary, Function) +import Test.QuickCheck.Checkers (EqProp) +import Test.QuickCheck.Instances.UnorderedContainers () +import Test.QuickCheck.Instances.Vector () +import Text.Show (showString, showParen) +import Data.Aeson +-------------------------------------------------------------------------------- + +type EntityID = Word32 +type NonNullSet a = NonNull (Set a) + +data EntityMap a where + EntityMap :: + { _byPosition :: Map Position (NonNullSet EntityID) + , _byID :: HashMap EntityID (Positioned a) + , _lastID :: EntityID + } -> EntityMap a + deriving stock (Functor, Foldable, Traversable, Generic) + deriving anyclass (NFData, CoArbitrary, Function) +deriving via (EqEqProp (EntityMap a)) instance (Eq a, Ord a) => EqProp (EntityMap a) +makeLenses ''EntityMap + +instance ToJSON a => ToJSON (EntityMap a) where + toJSON = toJSON . toEIDsAndPositioned + + +instance FromJSON a => FromJSON (EntityMap a) where + parseJSON = fmap (fromEIDsAndPositioned @[_]) . parseJSON + +byIDInvariantError :: forall a. a +byIDInvariantError = error $ "Invariant violation: All EntityIDs in byPosition " + <> "must point to entityIDs in byID" + +instance (Ord a, Eq a) => Eq (EntityMap a) where + -- em₁ == em₂ = em₁ ^. _EntityMap == em₂ ^. _EntityMap + (==) = (==) `on` view (_EntityMap . to sort) + +deriving stock instance (Ord a) => Ord (EntityMap a) + +instance Show a => Show (EntityMap a) where + showsPrec pr em + = showParen (pr > 10) + $ showString + . ("fromEIDsAndPositioned " <>) + . show + . toEIDsAndPositioned + $ em + +instance Arbitrary a => Arbitrary (EntityMap a) where + arbitrary = review _EntityMap <$> arbitrary + shrink em = review _EntityMap <$> shrink (em ^. _EntityMap) + +type instance Index (EntityMap a) = EntityID +type instance IxValue (EntityMap a) = (Positioned a) +instance Ixed (EntityMap a) where ix eid = at eid . traverse + +instance At (EntityMap a) where + at eid = lens (view $ byID . at eid) setter + where + setter :: EntityMap a -> Maybe (Positioned a) -> EntityMap a + setter m Nothing = fromMaybe m $ do + Positioned pos _ <- m ^. byID . at eid + pure $ m + & removeEIDAtPos pos + & byID . at eid .~ Nothing + setter m (Just pe@(Positioned pos _)) = m + & (case lookupWithPosition eid m of + Nothing -> id + Just (Positioned origPos _) -> removeEIDAtPos origPos + ) + & byID . at eid ?~ pe + & byPosition . at pos %~ \case + Nothing -> Just $ opoint eid + Just es -> Just $ ninsertSet eid es + removeEIDAtPos pos = + byPosition . at pos %~ (>>= fromNullable . ndeleteSet eid) + +instance Semigroup (EntityMap a) where + em₁ <> em₂ = alaf Endo foldMap (uncurry insertAt) (em₂ ^. _EntityMap) em₁ + +instance Monoid (EntityMap a) where + mempty = emptyEntityMap + +instance FunctorWithIndex EntityID EntityMap + +instance FoldableWithIndex EntityID EntityMap + +instance TraversableWithIndex EntityID EntityMap where + itraversed = byID . itraversed . rmap sequenceA . distrib + itraverse = itraverseOf itraversed + +type instance Element (EntityMap a) = a +instance MonoFoldable (EntityMap a) + +emptyEntityMap :: EntityMap a +emptyEntityMap = EntityMap mempty mempty 0 + +newtype Deduplicate a = Deduplicate (EntityMap a) + deriving stock (Show, Traversable, Generic) + deriving newtype (Eq, Functor, Foldable, EqProp, Arbitrary) + +instance Semigroup (Deduplicate a) where + (Deduplicate em₁) <> (Deduplicate em₂) = + let _byID = em₁ ^. byID <> em₂ ^. byID + _byPosition = mempty &~ do + ifor_ _byID $ \eid (Positioned pos _) -> + at pos %= \case + Just eids -> Just $ ninsertSet eid eids + Nothing -> Just $ opoint eid + _lastID = fromMaybe 1 $ maximumOf (ifolded . asIndex) _byID + in Deduplicate EntityMap{..} + + +-------------------------------------------------------------------------------- + +_EntityMap :: Iso' (EntityMap a) [(Position, a)] +_EntityMap = iso hither yon + where + hither :: EntityMap a -> [(Position, a)] + hither em = do + (pos, eids) <- em ^. byPosition . _Wrapped + eid <- toList eids + ent <- em ^.. byID . at eid . folded . positioned + pure (pos, ent) + yon :: [(Position, a)] -> EntityMap a + yon poses = alaf Endo foldMap (uncurry insertAt) poses emptyEntityMap + + +insertAtReturningID :: forall a. Position -> a -> EntityMap a -> (EntityID, EntityMap a) +insertAtReturningID pos e em = + let (eid, em') = em & lastID <+~ 1 + in em' + & byID . at eid ?~ Positioned pos e + & byPosition . at pos %~ \case + Nothing -> Just $ opoint eid + Just es -> Just $ ninsertSet eid es + & (eid, ) + +insertAt :: forall a. Position -> a -> EntityMap a -> EntityMap a +insertAt pos e = snd . insertAtReturningID pos e + +atPosition :: forall a. (Ord a, Show a) => Position -> Lens' (EntityMap a) (VectorBag a) +atPosition pos = lens getter setter + where + getter em = + let eids :: VectorBag EntityID + eids = maybe mempty (VectorBag . toVector . toNullable) + $ em ^. byPosition . at pos + in getEIDAssume em <$> eids + setter em Empty = em & byPosition . at pos .~ Nothing + setter em (sort -> entities) = + let origEIDs = maybe Empty toNullable $ em ^. byPosition . at pos + origEntitiesWithIDs = + sortOn snd $ toList origEIDs <&> \eid -> (eid, getEIDAssume em eid) + go alles₁@((eid, e₁) :< es₁) -- orig + (e₂ :< es₂) -- new + | e₁ == e₂ + -- same, do nothing + = let (eids, lastEID, byID') = go es₁ es₂ + in (insertSet eid eids, lastEID, byID') + | otherwise + -- e₂ is new, generate a new ID for it + = let (eids, lastEID, byID') = go alles₁ es₂ + eid' = succ lastEID + in (insertSet eid' eids, eid', byID' & at eid' ?~ Positioned pos e₂) + go Empty Empty = (mempty, em ^. lastID, em ^. byID) + go orig Empty = + let byID' = foldr deleteMap (em ^. byID) $ map fst orig + in (mempty, em ^. lastID, byID') + go Empty (new :< news) = + let (eids, lastEID, byID') = go Empty news + eid' = succ lastEID + in (insertSet eid' eids, eid', byID' & at eid' ?~ Positioned pos new) + go _ _ = error "unreachable" + (eidsAtPosition, newLastID, newByID) = go origEntitiesWithIDs entities + in em & byPosition . at pos .~ fromNullable eidsAtPosition + & byID .~ newByID + & lastID .~ newLastID + +getEIDAssume :: EntityMap a -> EntityID -> a +getEIDAssume em eid = fromMaybe byIDInvariantError + $ em ^? byID . ix eid . positioned + +atPositionWithIDs :: Position -> EntityMap a -> Vector (EntityID, Positioned a) +atPositionWithIDs pos em = + let eids = maybe mempty (toVector . toNullable) + $ em ^. byPosition . at pos + in (id &&& Positioned pos . getEIDAssume em) <$> eids + +fromEIDsAndPositioned + :: forall mono a. (MonoFoldable mono, Element mono ~ (EntityID, Positioned a)) + => mono + -> EntityMap a +fromEIDsAndPositioned eps = newLastID $ alaf Endo foldMap insert' eps mempty + where + insert' (eid, pe@(Positioned pos _)) + = (byID . at eid ?~ pe) + . (byPosition . at pos %~ \case + Just eids -> Just $ ninsertSet eid eids + Nothing -> Just $ opoint eid + ) + newLastID em = em & lastID + .~ fromMaybe 1 + (maximumOf (ifolded . asIndex) (em ^. byID)) + +toEIDsAndPositioned :: EntityMap a -> [(EntityID, Positioned a)] +toEIDsAndPositioned = itoListOf $ byID . ifolded + +positions :: EntityMap a -> [Position] +positions = toListOf $ byPosition . to keys . folded + +lookupWithPosition :: EntityID -> EntityMap a -> Maybe (Positioned a) +lookupWithPosition eid = view $ byID . at eid + +lookup :: EntityID -> EntityMap a -> Maybe a +lookup eid = fmap (view positioned) . lookupWithPosition eid + +-- unlawful :( +-- positionedEntities :: IndexedTraversal EntityID (EntityMap a) (EntityMap b) (Positioned a) (Positioned b) +-- positionedEntities = byID . itraversed + +neighbors :: (Ord a, Show a) => Position -> EntityMap a -> Neighbors (VectorBag a) +neighbors pos em = (\p -> view (atPosition p) em) <$> neighborPositions pos + +-- | Traversal to the position of the entity with the given ID +positionOf :: EntityID -> Traversal' (EntityMap a) Position +positionOf eid = ix eid . position + +-------------------------------------------------------------------------------- +makeWrapped ''Deduplicate diff --git a/users/grfn/xanthous/src/Xanthous/Data/EntityMap/Graphics.hs b/users/grfn/xanthous/src/Xanthous/Data/EntityMap/Graphics.hs new file mode 100644 index 000000000000..1398c611cf20 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/EntityMap/Graphics.hs @@ -0,0 +1,72 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.EntityMap.Graphics + ( visiblePositions + , visibleEntities + , lineOfSight + , linesOfSight + , canSee + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (lines) +-------------------------------------------------------------------------------- +import Xanthous.Util (takeWhileInclusive) +import Xanthous.Data +import Xanthous.Data.Entities +import Xanthous.Data.EntityMap +import Xanthous.Game.State +import Xanthous.Util.Graphics (circle, line) +-------------------------------------------------------------------------------- + +-- | Returns a set of positions that are visible, when taking into account +-- 'blocksVision', from the given position, within the given radius. +visiblePositions + :: Entity e + => Position + -> Word -- ^ Vision radius + -> EntityMap e + -> Set Position +visiblePositions pos radius + = setFromList . positions . visibleEntities pos radius + +-- | Returns a list of entities on the *line of sight* from the first position +-- to the second position +lineOfSight + :: forall e. Entity e + => Position -- ^ Origin + -> Position -- ^ Destination + -> EntityMap e + -> [(Position, Vector (EntityID, e))] +lineOfSight (view _Position -> origin) (view _Position -> destination) em = + takeWhileInclusive (none (view blocksVision . entityAttributes . snd) . snd) + $ getPositionedAt <$> line origin destination + where + getPositionedAt :: V2 Int -> (Position, Vector (EntityID, e)) + getPositionedAt (review _Position -> p) = + (p, over _2 (view positioned) <$> atPositionWithIDs p em) + +-- | Returns a list of individual lines of sight, each of which is a list of +-- entities at positions on that line of sight +linesOfSight + :: forall e. Entity e + => Position -- ^ Centerpoint + -> Word -- ^ Radius + -> EntityMap e + -> [[(Position, Vector (EntityID, e))]] +linesOfSight pos visionRadius em = + radius <&> \edge -> lineOfSight pos (_Position # edge) em + where + radius = circle (pos ^. _Position) $ fromIntegral visionRadius + +-- | Given a point and a radius of vision, returns a list of all entities that +-- are *visible* (eg, not blocked by an entity that obscures vision) from that +-- point +visibleEntities :: Entity e => Position -> Word -> EntityMap e -> EntityMap e +visibleEntities pos visionRadius + = fromEIDsAndPositioned + . foldMap (\(p, es) -> over _2 (Positioned p) <$> es) + . fold + . linesOfSight pos visionRadius + +canSee :: Entity e => (e -> Bool) -> Position -> Word -> EntityMap e -> Bool +canSee match pos radius = any match . visibleEntities pos radius +-- ^ this might be optimizable diff --git a/users/grfn/xanthous/src/Xanthous/Data/Levels.hs b/users/grfn/xanthous/src/Xanthous/Data/Levels.hs new file mode 100644 index 000000000000..13251d8afdf2 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/Levels.hs @@ -0,0 +1,180 @@ +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.Levels + ( Levels + , allLevels + , numLevels + , nextLevel + , prevLevel + , mkLevels1 + , mkLevels + , oneLevel + , current + , ComonadStore(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding ((<.>), Empty, foldMap) +import Xanthous.Util (between, EqProp, EqEqProp(..)) +import Xanthous.Util.Comonad (current) +import Xanthous.Orphans () +-------------------------------------------------------------------------------- +import Control.Comonad.Store +import Control.Comonad.Store.Zipper +import Data.Aeson (ToJSON(..), FromJSON(..)) +import Data.Aeson.Generic.DerivingVia +import Data.Functor.Apply +import Data.Foldable (foldMap) +import Data.List.NonEmpty (NonEmpty) +import qualified Data.List.NonEmpty as NE +import Data.Maybe (fromJust) +import Data.Sequence (Seq((:<|), Empty)) +import Data.Semigroup.Foldable.Class +import Data.Text (replace) +import Test.QuickCheck +-------------------------------------------------------------------------------- + +-- | Collection of levels plus a pointer to the current level +-- +-- Navigation is via the 'Comonad' instance. We can get the current level with +-- 'extract': +-- +-- extract @Levels :: Levels level -> level +-- +-- For access to and modification of the level, use +-- 'Xanthous.Util.Comonad.current' +newtype Levels a = Levels { levelZipper :: Zipper Seq a } + deriving stock (Generic) + deriving (Functor, Comonad, Foldable) via (Zipper Seq) + +type instance Element (Levels a) = a +instance MonoFoldable (Levels a) +instance MonoFunctor (Levels a) +instance MonoTraversable (Levels a) + +instance ComonadStore Word Levels where + pos = toEnum . pos . levelZipper + peek i = peek (fromEnum i) . levelZipper + +instance Traversable Levels where + traverse f (Levels z) = Levels <$> traverse f z + +instance Foldable1 Levels + +instance Traversable1 Levels where + traverse1 f levs@(Levels z) = seek (pos levs) . partialMkLevels <$> go (unzipper z) + where + go Empty = error "empty seq, unreachable" + go (x :<| xs) = (<|) <$> f x <.> go xs + +-- | Always takes the position of the latter element +instance Semigroup (Levels a) where + levs₁ <> levs₂ + = seek (pos levs₂) + . partialMkLevels + $ allLevels levs₁ <> allLevels levs₂ + +-- | The number of levels stored in 'Levels' +-- +-- Equivalent to 'Data.Foldable.length', but likely faster +numLevels :: Levels a -> Word +numLevels = toEnum . size . levelZipper + +-- | Make Levels from a Seq. Throws an error if the seq is not empty +partialMkLevels :: Seq a -> Levels a +partialMkLevels = Levels . fromJust . zipper + +-- | Make Levels from a possibly-empty structure +mkLevels :: Foldable1 f => f level -> Maybe (Levels level) +mkLevels = fmap Levels . zipper . foldMap pure + +-- | Make Levels from a non-empty structure +mkLevels1 :: Foldable1 f => f level -> Levels level +mkLevels1 = fromJust . mkLevels + +oneLevel :: a -> Levels a +oneLevel = mkLevels1 . Identity + +-- | Get a sequence of all the levels +allLevels :: Levels a -> Seq a +allLevels = unzipper . levelZipper + +-- | Step to the next level, generating a new level if necessary using the given +-- applicative action +nextLevel + :: Applicative m + => m level -- ^ Generate a new level, if necessary + -> Levels level + -> m (Levels level) +nextLevel genLevel levs + | succ (pos levs) < numLevels levs + = pure $ seeks succ levs + | otherwise + = genLevel <&> \level -> + seek (pos levs + 1) . partialMkLevels $ allLevels levs |> level + +-- | Go to the previous level. Returns Nothing if 'pos' is 0 +prevLevel :: Levels level -> Maybe (Levels level) +prevLevel levs | pos levs == 0 = Nothing + | otherwise = Just $ seeks pred levs + +-------------------------------------------------------------------------------- + +-- | alternate, slower representation of Levels we can Iso into to perform +-- various operations +data AltLevels a = AltLevels + { _levels :: NonEmpty a + , _currentLevel :: Word -- ^ invariant: is within the bounds of _levels + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + (AltLevels a) +makeLenses ''AltLevels + +alt :: Iso (Levels a) (Levels b) (AltLevels a) (AltLevels b) +alt = iso hither yon + where + hither levs = AltLevels (NE.fromList . toList $ allLevels levs) (pos levs) + yon (AltLevels levs curr) = seek curr $ mkLevels1 levs + +instance Eq a => Eq (Levels a) where + (==) = (==) `on` view alt + +deriving via EqEqProp (Levels a) instance Eq a => EqProp (Levels a) + +instance Show a => Show (Levels a) where + show = unpack . replace "AltLevels" "Levels" . pack . show . view alt + +instance NFData a => NFData (Levels a) where + rnf = rnf . view alt + +instance ToJSON a => ToJSON (Levels a) where + toJSON = toJSON . view alt + +instance FromJSON a => FromJSON (Levels a) where + parseJSON = fmap (review alt) . parseJSON + +instance Arbitrary a => Arbitrary (AltLevels a) where + arbitrary = do + _levels <- arbitrary + _currentLevel <- choose (0, pred . toEnum . length $ _levels) + pure AltLevels {..} + shrink als = do + _levels <- shrink $ als ^. levels + _currentLevel <- filter (between 0 $ pred . toEnum . length $ _levels) + $ shrink $ als ^. currentLevel + pure AltLevels {..} + + +instance Arbitrary a => Arbitrary (Levels a) where + arbitrary = review alt <$> arbitrary + shrink = fmap (review alt) . shrink . view alt + +instance CoArbitrary a => CoArbitrary (Levels a) where + coarbitrary = coarbitrary . view alt + +instance Function a => Function (Levels a) where + function = functionMap (view alt) (review alt) diff --git a/users/grfn/xanthous/src/Xanthous/Data/Memo.hs b/users/grfn/xanthous/src/Xanthous/Data/Memo.hs new file mode 100644 index 000000000000..2b2ee0f96028 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/Memo.hs @@ -0,0 +1,98 @@ +-------------------------------------------------------------------------------- +-- | Memoized values +-------------------------------------------------------------------------------- +module Xanthous.Data.Memo + ( Memoized(UnMemoized) + , memoizeWith + , getMemoized + , runMemoized + , fillWith + , fillWithM + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Data.Aeson (FromJSON, ToJSON) +import Test.QuickCheck (Arbitrary (arbitrary), oneof, CoArbitrary, Function) +import Test.QuickCheck.Checkers (EqProp) +import Xanthous.Util (EqEqProp(EqEqProp)) +import Control.Monad.State.Class (MonadState) +-------------------------------------------------------------------------------- + +-- | A memoized value, keyed by a key +-- +-- If key is different than what is stored here, then val is invalid +data Memoized key val = Memoized key val | UnMemoized + deriving stock (Show, Eq, Generic) + deriving anyclass (Hashable, FromJSON, ToJSON, NFData, CoArbitrary, Function) + deriving EqProp via EqEqProp (Memoized key val) + +instance (Arbitrary k, Arbitrary v) => Arbitrary (Memoized k v) where + arbitrary = oneof [ pure UnMemoized + , Memoized <$> arbitrary <*> arbitrary + ] + +-- | Construct a memoized value with the given key +memoizeWith :: forall key val. key -> val -> Memoized key val +memoizeWith = Memoized +{-# INLINE memoizeWith #-} + +-- | Retrieve a memoized value providing the key. If the value is unmemoized or +-- the keys do not match, returns Nothing. +-- +-- >>> getMemoized 1 (memoizeWith @Int @Int 1 2) +-- Just 2 +-- +-- >>> getMemoized 2 (memoizeWith @Int @Int 1 2) +-- Nothing +-- +-- >>> getMemoized 1 (UnMemoized :: Memoized Int Int) +-- Nothing +getMemoized :: Eq key => key -> Memoized key val -> Maybe val +getMemoized key (Memoized key' v) + | key == key' = Just v + | otherwise = Nothing +getMemoized _ UnMemoized = Nothing +{-# INLINE getMemoized #-} + +-- | Get a memoized value using an applicative action to obtain the key +runMemoized + :: (Eq key, Applicative m) + => Memoized key val + -> m key + -> m (Maybe val) +runMemoized m mk = getMemoized <$> mk <*> pure m + +-- | In a monadic state containing a 'MemoState', look up the current memoized +-- target of some lens keyed by k, filling it with v if not present and +-- returning either the new or old value +fillWith + :: forall m s k v. + (MonadState s m, Eq k) + => Lens' s (Memoized k v) + -> k + -> v + -> m v +fillWith l k v' = do + uses l (getMemoized k) >>= \case + Just v -> pure v + Nothing -> do + l .= memoizeWith k v' + pure v' + +-- | In a monadic state, look up the current memoized target of some lens keyed +-- by k, filling it with the result of some monadic action v if not present and +-- returning either the new or old value +fillWithM + :: forall m s k v. + (MonadState s m, Eq k) + => Lens' s (Memoized k v) + -> k + -> m v + -> m v +fillWithM l k mv = do + uses l (getMemoized k) >>= \case + Just v -> pure v + Nothing -> do + v' <- mv + l .= memoizeWith k v' + pure v' diff --git a/users/grfn/xanthous/src/Xanthous/Data/NestedMap.hs b/users/grfn/xanthous/src/Xanthous/Data/NestedMap.hs new file mode 100644 index 000000000000..1b875d448302 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/NestedMap.hs @@ -0,0 +1,227 @@ +{-# LANGUAGE PartialTypeSignatures #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE PolyKinds #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.NestedMap + ( NestedMapVal(..) + , NestedMap(..) + , lookup + , lookupVal + , insert + + -- * + , (:->) + , BifunctorFunctor'(..) + , BifunctorMonad'(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (lookup, foldMap) +import qualified Xanthous.Prelude as P +-------------------------------------------------------------------------------- +import Test.QuickCheck +import Data.Aeson +import Data.Function (fix) +import Data.Foldable (Foldable(..)) +import Data.List.NonEmpty (NonEmpty(..)) +import qualified Data.List.NonEmpty as NE +-------------------------------------------------------------------------------- + +-- | Natural transformations on bifunctors +type (:->) p q = forall a b. p a b -> q a b +infixr 0 :-> + +class (forall b. Bifunctor b => Bifunctor (t b)) => BifunctorFunctor' t where + bifmap' :: (Bifunctor p, Bifunctor q) => (p :-> q) -> t p :-> t q + +class BifunctorFunctor' t => BifunctorMonad' t where + bireturn' :: (Bifunctor p) => p :-> t p + + bibind' :: (Bifunctor p, Bifunctor q) => (p :-> t q) -> t p :-> t q + bibind' f = bijoin' . bifmap' f + + bijoin' :: (Bifunctor p) => t (t p) :-> t p + bijoin' = bibind' id + + {-# MINIMAL bireturn', (bibind' | bijoin') #-} + +-------------------------------------------------------------------------------- + +data NestedMapVal m k v = Val v | Nested (NestedMap m k v) + +deriving stock instance + ( forall k' v'. (Show k', Show v') => Show (m k' v') + , Show k + , Show v + ) => Show (NestedMapVal m k v) + +deriving stock instance + ( forall k' v'. (Eq k', Eq v') => Eq (m k' v') + , Eq k + , Eq v + ) => Eq (NestedMapVal m k v) + +instance + forall m k v. + ( Arbitrary (m k v) + , Arbitrary (m k (NestedMapVal m k v)) + , Arbitrary k + , Arbitrary v + , IsMap (m k (NestedMapVal m k v)) + , MapValue (m k (NestedMapVal m k v)) ~ (NestedMapVal m k v) + , ContainerKey (m k (NestedMapVal m k v)) ~ k + ) => Arbitrary (NestedMapVal m k v) where + arbitrary = sized . fix $ \gen n -> + let nst = fmap (NestedMap . mapFromList) + . listOf + $ (,) <$> arbitrary @k <*> gen (n `div` 2) + in if n == 0 + then Val <$> arbitrary + else oneof [ Val <$> arbitrary + , Nested <$> nst] + shrink (Val v) = Val <$> shrink v + shrink (Nested mkv) = Nested <$> shrink mkv + +instance Functor (m k) => Functor (NestedMapVal m k) where + fmap f (Val v) = Val $ f v + fmap f (Nested m) = Nested $ fmap f m + +instance Bifunctor m => Bifunctor (NestedMapVal m) where + bimap _ g (Val v) = Val $ g v + bimap f g (Nested m) = Nested $ bimap f g m + +instance BifunctorFunctor' NestedMapVal where + bifmap' _ (Val v) = Val v + bifmap' f (Nested m) = Nested $ bifmap' f m + +instance (ToJSONKey k, ToJSON v, ToJSON (m k (NestedMapVal m k v))) + => ToJSON (NestedMapVal m k v) where + toJSON (Val v) = toJSON v + toJSON (Nested m) = toJSON m + +instance Foldable (m k) => Foldable (NestedMapVal m k) where + foldMap f (Val v) = f v + foldMap f (Nested m) = foldMap f m + +-- _NestedMapVal +-- :: forall m k v m' k' v'. +-- ( IsMap (m k v), IsMap (m' k' v') +-- , IsMap (m [k] v), IsMap (m' [k'] v') +-- , ContainerKey (m k v) ~ k, ContainerKey (m' k' v') ~ k' +-- , ContainerKey (m [k] v) ~ [k], ContainerKey (m' [k'] v') ~ [k'] +-- , MapValue (m k v) ~ v, MapValue (m' k' v') ~ v' +-- , MapValue (m [k] v) ~ v, MapValue (m' [k'] v') ~ v' +-- ) +-- => Iso (NestedMapVal m k v) +-- (NestedMapVal m' k' v') +-- (m [k] v) +-- (m' [k'] v') +-- _NestedMapVal = iso hither yon +-- where +-- hither :: NestedMapVal m k v -> m [k] v +-- hither (Val v) = singletonMap [] v +-- hither (Nested m) = bimap _ _ $ m ^. _NestedMap +-- yon = _ + +-------------------------------------------------------------------------------- + +newtype NestedMap m k v = NestedMap (m k (NestedMapVal m k v)) + +deriving stock instance + ( forall k' v'. (Eq k', Eq v') => Eq (m k' v') + , Eq k + , Eq v + ) => Eq (NestedMap m k v) + +deriving stock instance + ( forall k' v'. (Show k', Show v') => Show (m k' v') + , Show k + , Show v + ) => Show (NestedMap m k v) + +instance Arbitrary (m k (NestedMapVal m k v)) + => Arbitrary (NestedMap m k v) where + arbitrary = NestedMap <$> arbitrary + shrink (NestedMap m) = NestedMap <$> shrink m + +instance Functor (m k) => Functor (NestedMap m k) where + fmap f (NestedMap m) = NestedMap $ fmap (fmap f) m + +instance Bifunctor m => Bifunctor (NestedMap m) where + bimap f g (NestedMap m) = NestedMap $ bimap f (bimap f g) m + +instance BifunctorFunctor' NestedMap where + bifmap' f (NestedMap m) = NestedMap . f $ bimap id (bifmap' f) m + +instance (ToJSONKey k, ToJSON v, ToJSON (m k (NestedMapVal m k v))) + => ToJSON (NestedMap m k v) where + toJSON (NestedMap m) = toJSON m + +instance Foldable (m k) => Foldable (NestedMap m k) where + foldMap f (NestedMap m) = foldMap (foldMap f) m + +-------------------------------------------------------------------------------- + +lookup + :: ( IsMap (m k (NestedMapVal m k v)) + , MapValue (m k (NestedMapVal m k v)) ~ (NestedMapVal m k v) + , ContainerKey (m k (NestedMapVal m k v)) ~ k + ) + => NonEmpty k + -> NestedMap m k v + -> Maybe (NestedMapVal m k v) +lookup (p :| []) (NestedMap vs) = P.lookup p vs +lookup (p :| (p₁ : ps)) (NestedMap vs) = P.lookup p vs >>= \case + (Val _) -> Nothing + (Nested vs') -> lookup (p₁ :| ps) vs' + +lookupVal + :: ( IsMap (m k (NestedMapVal m k v)) + , MapValue (m k (NestedMapVal m k v)) ~ (NestedMapVal m k v) + , ContainerKey (m k (NestedMapVal m k v)) ~ k + ) + => NonEmpty k + -> NestedMap m k v + -> Maybe v +lookupVal ks m + | Just (Val v) <- lookup ks m = Just v + | otherwise = Nothing + +insert + :: ( IsMap (m k (NestedMapVal m k v)) + , MapValue (m k (NestedMapVal m k v)) ~ (NestedMapVal m k v) + , ContainerKey (m k (NestedMapVal m k v)) ~ k + ) + => NonEmpty k + -> v + -> NestedMap m k v + -> NestedMap m k v +insert (k :| []) v (NestedMap m) = NestedMap $ P.insertMap k (Val v) m +insert (k₁ :| (k₂ : ks)) v (NestedMap m) = NestedMap $ alterMap upd k₁ m + where + upd (Just (Nested nm)) = Just . Nested $ insert (k₂ :| ks) v nm + upd _ = Just $ + let (kΩ :| ks') = NE.reverse (k₂ :| ks) + in P.foldl' + (\m' k -> Nested . NestedMap . singletonMap k $ m') + (Nested . NestedMap . singletonMap kΩ $ Val v) + ks' + +-- _NestedMap +-- :: ( IsMap (m k v), IsMap (m' k' v') +-- , IsMap (m (NonEmpty k) v), IsMap (m' (NonEmpty k') v') +-- , ContainerKey (m k v) ~ k, ContainerKey (m' k' v') ~ k' +-- , ContainerKey (m (NonEmpty k) v) ~ (NonEmpty k) +-- , ContainerKey (m' (NonEmpty k') v') ~ (NonEmpty k') +-- , MapValue (m k v) ~ v, MapValue (m' k' v') ~ v' +-- , MapValue (m (NonEmpty k) v) ~ v, MapValue (m' (NonEmpty k') v') ~ v' +-- ) +-- => Iso (NestedMap m k v) +-- (NestedMap m' k' v') +-- (m (NonEmpty k) v) +-- (m' (NonEmpty k') v') +-- _NestedMap = iso undefined yon +-- where +-- hither (NestedMap m) = undefined . mapToList $ m +-- yon mkv = undefined diff --git a/users/grfn/xanthous/src/Xanthous/Data/VectorBag.hs b/users/grfn/xanthous/src/Xanthous/Data/VectorBag.hs new file mode 100644 index 000000000000..2e6d48062a45 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Data/VectorBag.hs @@ -0,0 +1,100 @@ +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE DeriveTraversable #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.VectorBag + (VectorBag(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Data.Aeson +import qualified Data.Vector as V +import Test.QuickCheck +import Test.QuickCheck.Instances.Vector () +-------------------------------------------------------------------------------- + +-- | Acts exactly like a Vector, except ignores order when testing for equality +newtype VectorBag a = VectorBag (Vector a) + deriving stock + ( Traversable + , Generic + ) + deriving newtype + ( Show + , Read + , Foldable + , FromJSON + , FromJSON1 + , ToJSON + , Reversing + , Applicative + , Functor + , Monad + , Monoid + , Semigroup + , Arbitrary + , CoArbitrary + , Filterable + ) +makeWrapped ''VectorBag + +instance Function a => Function (VectorBag a) where + function = functionMap (\(VectorBag v) -> v) VectorBag + +type instance Element (VectorBag a) = a +deriving via (Vector a) instance MonoFoldable (VectorBag a) +deriving via (Vector a) instance GrowingAppend (VectorBag a) +deriving via (Vector a) instance SemiSequence (VectorBag a) +deriving via (Vector a) instance MonoPointed (VectorBag a) +deriving via (Vector a) instance MonoFunctor (VectorBag a) + +instance Cons (VectorBag a) (VectorBag b) a b where + _Cons = prism (\(x, VectorBag xs) -> VectorBag $ x <| xs) $ \(VectorBag v) -> + if V.null v + then Left (VectorBag mempty) + else Right (V.unsafeHead v, VectorBag $ V.unsafeTail v) + +instance AsEmpty (VectorBag a) where + _Empty = prism' (const $ VectorBag Empty) $ \case + (VectorBag Empty) -> Just () + _ -> Nothing + +instance Witherable VectorBag where + wither f (VectorBag v) = VectorBag <$> wither f v + witherM f (VectorBag v) = VectorBag <$> witherM f v + filterA p (VectorBag v) = VectorBag <$> filterA p v + +{- + TODO: + , Ixed + , FoldableWithIndex + , FunctorWithIndex + , TraversableWithIndex + , Snoc + , Each +-} + +instance Ord a => Eq (VectorBag a) where + (==) = (==) `on` (view _Wrapped . sort) + +instance Ord a => Ord (VectorBag a) where + compare = compare `on` (view _Wrapped . sort) + +instance MonoTraversable (VectorBag a) where + otraverse f (VectorBag v) = VectorBag <$> otraverse f v + +instance IsSequence (VectorBag a) where + fromList = VectorBag . fromList + break prd (VectorBag v) = bimap VectorBag VectorBag $ break prd v + span prd (VectorBag v) = bimap VectorBag VectorBag $ span prd v + dropWhile prd (VectorBag v) = VectorBag $ dropWhile prd v + takeWhile prd (VectorBag v) = VectorBag $ takeWhile prd v + splitAt idx (VectorBag v) = bimap VectorBag VectorBag $ splitAt idx v + unsafeSplitAt idx (VectorBag v) = + bimap VectorBag VectorBag $ unsafeSplitAt idx v + take n (VectorBag v) = VectorBag $ take n v + unsafeTake n (VectorBag v) = VectorBag $ unsafeTake n v + drop n (VectorBag v) = VectorBag $ drop n v + unsafeDrop n (VectorBag v) = VectorBag $ unsafeDrop n v + partition p (VectorBag v) = bimap VectorBag VectorBag $ partition p v diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Character.hs b/users/grfn/xanthous/src/Xanthous/Entities/Character.hs new file mode 100644 index 000000000000..d405cb40d3eb --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Character.hs @@ -0,0 +1,240 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.Character + + ( -- * Character datatype + Character(..) + , characterName + , HasInventory(..) + , characterDamage + , characterHitpoints' + , characterHitpoints + , hitpointRecoveryRate + , speed + , body + + -- *** Body + , Body(..) + , initialBody + , knuckles + , Knuckles(..) + , fistDamageChance + , damageKnuckles + , fistfightingDamage + + -- * Character functions + , mkCharacter + , pickUpItem + , isDead + , isFullyHealed + , damage + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Brick +import Data.Aeson.Generic.DerivingVia +import Data.Aeson (ToJSON, FromJSON) +import Data.Coerce (coerce) +import Test.QuickCheck +import Test.QuickCheck.Instances.Vector () +import Test.QuickCheck.Arbitrary.Generic +import Test.QuickCheck.Gen (chooseUpTo) +import Test.QuickCheck.Checkers (EqProp) +import Control.Monad.State.Lazy (execState) +import Control.Monad.Trans.State.Lazy (execStateT) +-------------------------------------------------------------------------------- +import Xanthous.Util.QuickCheck +import Xanthous.Game.State +import Xanthous.Entities.Item +import Xanthous.Entities.Common +import Xanthous.Data + ( TicksPerTile, Hitpoints, Per, Ticks, (|*|), positioned ) +import qualified Xanthous.Entities.RawTypes as Raw +import Xanthous.Util (EqEqProp(EqEqProp), modifyKL) +import Xanthous.Monad (say_) +-------------------------------------------------------------------------------- + +-- | The status of the character's knuckles +-- +-- This struct is used to track the damage and then eventual build-up of +-- calluses when the character is fighting with their fists +data Knuckles = Knuckles + { -- | How damaged are the knuckles currently, from 0 to 5? + -- + -- At 0, no calluses will form + -- At 1 and up, the character will form calluses after a while + -- At 5, continuing to fistfight will deal the character even more damage + _knuckleDamage :: !Word + -- | How built-up are the character's calluses, from 0 to 5? + -- + -- Each level of calluses decreases the likelihood of being damaged when + -- fistfighting by 1%, up to 5 where the character will never be damaged + -- fistfighting + , _knuckleCalluses :: !Word + + -- | Number of turns that have passed since the last time the knuckles were + -- damaged + , _ticksSinceDamaged :: Ticks + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving EqProp via EqEqProp Knuckles + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Knuckles +makeLenses ''Knuckles + +instance Semigroup Knuckles where + (Knuckles d₁ c₁ t₁) <> (Knuckles d₂ c₂ t₂) = Knuckles + (min (d₁ + d₂) 5) + (min (c₁ + c₂) 5) + (max t₁ t₂) + +instance Monoid Knuckles where + mempty = Knuckles 0 0 0 + +instance Arbitrary Knuckles where + arbitrary = do + _knuckleDamage <- fromIntegral <$> chooseUpTo 5 + _knuckleCalluses <- fromIntegral <$> chooseUpTo 5 + _ticksSinceDamaged <- arbitrary + pure Knuckles{..} + +-- | Likelihood that the character fighting with their fists will damage +-- themselves +fistDamageChance :: Knuckles -> Float +fistDamageChance knuckles + | calluses == 5 = 0 + | otherwise = baseChance - (0.01 * fromIntegral calluses) + where + baseChance = 0.08 + calluses = knuckles ^. knuckleCalluses + +-- | Damage the knuckles by a level (capping at the max knuckle damage) +damageKnuckles :: Knuckles -> Knuckles +damageKnuckles = execState $ do + knuckleDamage %= min 5 . succ + ticksSinceDamaged .= 0 + +-- | Damage taken when fistfighting and 'fistDamageChance' has occurred +fistfightingDamage :: Knuckles -> Hitpoints +fistfightingDamage knuckles + | knuckles ^. knuckleDamage == 5 = 2 + | otherwise = 1 + +stepKnuckles :: Ticks -> Knuckles -> AppM Knuckles +stepKnuckles ticks = execStateT . whenM (uses knuckleDamage (> 0)) $ do + ticksSinceDamaged += ticks + whenM (uses ticksSinceDamaged (>= 2000)) $ do + dam <- knuckleDamage <<.= 0 + knuckleCalluses %= min 5 . (+ dam) + ticksSinceDamaged .= 0 + lift $ say_ ["character", "body", "knuckles", "calluses"] + + +-- | Status of the character's body +data Body = Body + { _knuckles :: !Knuckles + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Body + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Body +makeLenses ''Body + +initialBody :: Body +initialBody = Body { _knuckles = mempty } + +-------------------------------------------------------------------------------- + +data Character = Character + { _inventory :: !Inventory + , _characterName :: !(Maybe Text) + , _characterHitpoints' :: !Double + , _speed :: !TicksPerTile + , _body :: !Body + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Character +makeFieldsNoPrefix ''Character + +characterHitpoints :: Character -> Hitpoints +characterHitpoints = views characterHitpoints' floor + +scrollOffset :: Int +scrollOffset = 5 + +instance Draw Character where + draw _ = visibleRegion rloc rreg $ str "@" + where + rloc = Location (negate scrollOffset, negate scrollOffset) + rreg = (2 * scrollOffset, 2 * scrollOffset) + drawPriority = const maxBound -- Character should always be on top, for now + +instance Brain Character where + step ticks = execStateT $ do + positioned . characterHitpoints' %= \hp -> + if hp > fromIntegral initialHitpoints + then hp + else hp + hitpointRecoveryRate |*| ticks + modifyKL (positioned . body . knuckles) $ lift . stepKnuckles ticks + +instance Entity Character where + description _ = "yourself" + entityChar _ = "@" + +instance Arbitrary Character where + arbitrary = genericArbitrary + +initialHitpoints :: Hitpoints +initialHitpoints = 10 + +hitpointRecoveryRate :: Double `Per` Ticks +hitpointRecoveryRate = 1.0 / (15 * coerce defaultSpeed) + +defaultSpeed :: TicksPerTile +defaultSpeed = 100 + +mkCharacter :: Character +mkCharacter = Character + { _inventory = mempty + , _characterName = Nothing + , _characterHitpoints' = fromIntegral initialHitpoints + , _speed = defaultSpeed + , _body = initialBody + } + +defaultCharacterDamage :: Hitpoints +defaultCharacterDamage = 1 + +-- | Returns the damage that the character currently does with an attack +-- TODO use double-handed/left-hand/right-hand here +characterDamage :: Character -> Hitpoints +characterDamage + = fromMaybe defaultCharacterDamage + . preview (inventory . wielded . wieldedItems . wieldableItem . Raw.damage) + +-- | Is the character fully healed up to or past their initial hitpoints? +isFullyHealed :: Character -> Bool +isFullyHealed = (>= initialHitpoints) . characterHitpoints + +-- | Is the character dead? +isDead :: Character -> Bool +isDead = (== 0) . characterHitpoints + +pickUpItem :: Item -> Character -> Character +pickUpItem it = inventory . backpack %~ (it <|) + +damage :: Hitpoints -> Character -> Character +damage (fromIntegral -> amount) = characterHitpoints' %~ \case + n | n <= amount -> 0 + | otherwise -> n - amount + +{-# ANN module ("Hlint: ignore Use newtype instead of data" :: String) #-} diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Common.hs b/users/grfn/xanthous/src/Xanthous/Entities/Common.hs new file mode 100644 index 000000000000..becd1b1ef62e --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Common.hs @@ -0,0 +1,250 @@ +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +-- | +-- Module : Xanthous.Entities.Common +-- Description : Common data type definitions and utilities for entities +-- +-------------------------------------------------------------------------------- +module Xanthous.Entities.Common + ( -- * Inventory + Inventory(..) + , HasInventory(..) + , backpack + , wielded + , items + , InventoryPosition(..) + , describeInventoryPosition + , inventoryPosition + , itemsWithPosition + , removeItemFromPosition + + -- ** Wielded items + , Wielded(..) + , hands + , leftHand + , rightHand + , inLeftHand + , inRightHand + , doubleHanded + , wieldedItems + , WieldedItem(..) + , wieldedItem + , wieldableItem + , asWieldedItem + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson (ToJSON, FromJSON) +import Data.Aeson.Generic.DerivingVia +import Test.QuickCheck +import Test.QuickCheck.Checkers (EqProp) +-------------------------------------------------------------------------------- +import Xanthous.Data (Positioned(..), positioned) +import Xanthous.Util.QuickCheck +import Xanthous.Game.State +import Xanthous.Entities.Item +import Xanthous.Entities.RawTypes (WieldableItem, wieldable) +import Xanthous.Util (removeFirst, EqEqProp(..)) +-------------------------------------------------------------------------------- + +data WieldedItem = WieldedItem + { _wieldedItem :: Item + , _wieldableItem :: WieldableItem + -- ^ Invariant: item ^. itemType . wieldable ≡ Just wieldableItem + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + WieldedItem +makeFieldsNoPrefix ''WieldedItem + +asWieldedItem :: Prism' Item WieldedItem +asWieldedItem = prism' hither yon + where + yon item = WieldedItem item <$> item ^. itemType . wieldable + hither (WieldedItem item _) = item + +instance Brain WieldedItem where + step ticks (Positioned p wi) = + over positioned (\i -> WieldedItem i $ wi ^. wieldableItem) + <$> step ticks (Positioned p $ wi ^. wieldedItem) + +instance Draw WieldedItem where + draw = draw . view wieldedItem + +instance Entity WieldedItem where + entityAttributes = entityAttributes . view wieldedItem + description = description . view wieldedItem + entityChar = entityChar . view wieldedItem + +instance Arbitrary WieldedItem where + arbitrary = genericArbitrary <&> \wi -> + wi & wieldedItem . itemType . wieldable ?~ wi ^. wieldableItem + +data Wielded + = DoubleHanded WieldedItem + | Hands { _leftHand :: !(Maybe WieldedItem) + , _rightHand :: !(Maybe WieldedItem) + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Wielded + deriving (ToJSON, FromJSON) + via WithOptions '[ 'SumEnc 'ObjWithSingleField ] + Wielded + +nothingWielded :: Wielded +nothingWielded = Hands Nothing Nothing + +hands :: Prism' Wielded (Maybe WieldedItem, Maybe WieldedItem) +hands = prism' (uncurry Hands) $ \case + Hands l r -> Just (l, r) + _ -> Nothing + +leftHand :: Traversal' Wielded (Maybe WieldedItem) +leftHand = hands . _1 + +inLeftHand :: WieldedItem -> Wielded +inLeftHand wi = Hands (Just wi) Nothing + +rightHand :: Traversal' Wielded (Maybe WieldedItem) +rightHand = hands . _2 + +inRightHand :: WieldedItem -> Wielded +inRightHand wi = Hands Nothing (Just wi) + +doubleHanded :: Prism' Wielded WieldedItem +doubleHanded = prism' DoubleHanded $ \case + DoubleHanded i -> Just i + _ -> Nothing + +wieldedItems :: Traversal' Wielded WieldedItem +wieldedItems k (DoubleHanded wielded) = DoubleHanded <$> k wielded +wieldedItems k (Hands l r) = Hands <$> _Just k l <*> _Just k r + +data Inventory = Inventory + { _backpack :: Vector Item + , _wielded :: Wielded + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Inventory + deriving EqProp via EqEqProp Inventory + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Inventory +makeFieldsNoPrefix ''Inventory + +items :: Traversal' Inventory Item +items k (Inventory bp w) = Inventory + <$> traversed k bp + <*> (wieldedItems . wieldedItem) k w + +type instance Element Inventory = Item + +instance MonoFunctor Inventory where + omap = over items + +instance MonoFoldable Inventory where + ofoldMap = foldMapOf items + ofoldr = foldrOf items + ofoldl' = foldlOf' items + otoList = toListOf items + oall = allOf items + oany = anyOf items + onull = nullOf items + ofoldr1Ex = foldr1Of items + ofoldl1Ex' = foldl1Of' items + headEx = headEx . toListOf items + lastEx = lastEx . toListOf items + +instance MonoTraversable Inventory where + otraverse = traverseOf items + +instance Semigroup Inventory where + inv₁ <> inv₂ = + let backpack' = inv₁ ^. backpack <> inv₂ ^. backpack + (wielded', backpack'') = case (inv₁ ^. wielded, inv₂ ^. wielded) of + (wielded₁, wielded₂@(DoubleHanded _)) -> + (wielded₂, backpack' <> fromList (wielded₁ ^.. wieldedItems . wieldedItem)) + (wielded₁, wielded₂@(Hands (Just _) (Just _))) -> + (wielded₂, backpack' <> fromList (wielded₁ ^.. wieldedItems . wieldedItem)) + (wielded₁, Hands Nothing Nothing) -> (wielded₁, backpack') + (Hands Nothing Nothing, wielded₂) -> (wielded₂, backpack') + (Hands (Just l₁) Nothing, Hands Nothing (Just r₂)) -> + (Hands (Just l₁) (Just r₂), backpack') + (wielded₁@(DoubleHanded _), wielded₂) -> + (wielded₁, backpack' <> fromList (wielded₂ ^.. wieldedItems . wieldedItem)) + (Hands Nothing (Just r₁), Hands Nothing (Just r₂)) -> + (Hands Nothing (Just r₂), r₁ ^. wieldedItem <| backpack') + (Hands Nothing r₁, Hands (Just l₂) Nothing) -> + (Hands (Just l₂) r₁, backpack') + (Hands (Just l₁) Nothing, Hands (Just l₂) Nothing) -> + (Hands (Just l₂) Nothing, l₁ ^. wieldedItem <| backpack') + (Hands (Just l₁) (Just r₁), Hands Nothing (Just r₂)) -> + (Hands (Just l₁) (Just r₂), r₁ ^. wieldedItem <| backpack') + (Hands (Just l₁) (Just r₁), Hands (Just l₂) Nothing) -> + (Hands (Just l₂) (Just r₁), l₁ ^. wieldedItem <| backpack') + in Inventory backpack'' wielded' + +instance Monoid Inventory where + mempty = Inventory mempty $ Hands Nothing Nothing + +class HasInventory s a | s -> a where + inventory :: Lens' s a + {-# MINIMAL inventory #-} + +-- | Representation for where in the inventory an item might be +data InventoryPosition + = Backpack + | LeftHand + | RightHand + | BothHands + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary InventoryPosition + +-- | Return a human-readable description of the given 'InventoryPosition' +describeInventoryPosition :: InventoryPosition -> Text +describeInventoryPosition Backpack = "In backpack" +describeInventoryPosition LeftHand = "Wielded, in left hand" +describeInventoryPosition RightHand = "Wielded, in right hand" +describeInventoryPosition BothHands = "Wielded, in both hands" + +-- | Given a position in the inventory, return a traversal on the inventory over +-- all the items in that position +inventoryPosition :: InventoryPosition -> Traversal' Inventory Item +inventoryPosition Backpack = backpack . traversed +inventoryPosition LeftHand = wielded . leftHand . _Just . wieldedItem +inventoryPosition RightHand = wielded . leftHand . _Just . wieldedItem +inventoryPosition BothHands = wielded . doubleHanded . wieldedItem + +-- | A fold over all the items in the inventory accompanied by their position in +-- the inventory +-- +-- Invariant: This will return items in the same order as 'items' +itemsWithPosition :: Fold Inventory (InventoryPosition, Item) +itemsWithPosition = folding $ (<>) <$> backpackItems <*> handItems + where + backpackItems = toListOf $ backpack . folded . to (Backpack ,) + handItems inv = case inv ^. wielded of + DoubleHanded i -> pure (BothHands, i ^. wieldedItem) + Hands l r -> (l ^.. folded . wieldedItem . to (LeftHand ,)) + <> (r ^.. folded . wieldedItem . to (RightHand ,)) + +-- | Remove the first item equal to 'Item' from the given position in the +-- inventory +removeItemFromPosition :: InventoryPosition -> Item -> Inventory -> Inventory +removeItemFromPosition Backpack item inv + = inv & backpack %~ removeFirst (== item) +removeItemFromPosition LeftHand item inv + = inv & wielded . leftHand %~ filter ((/= item) . view wieldedItem) +removeItemFromPosition RightHand item inv + = inv & wielded . rightHand %~ filter ((/= item) . view wieldedItem) +removeItemFromPosition BothHands item inv + | has (wielded . doubleHanded . wieldedItem . filtered (== item)) inv + = inv & wielded .~ nothingWielded + | otherwise + = inv diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Creature.hs b/users/grfn/xanthous/src/Xanthous/Entities/Creature.hs new file mode 100644 index 000000000000..3ea610795e98 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Creature.hs @@ -0,0 +1,88 @@ +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.Creature + ( -- * Creature + Creature(..) + -- ** Lenses + , creatureType + , hitpoints + , hippocampus + , inventory + + -- ** Creature functions + , damage + , isDead + , visionRadius + + -- * Hippocampus + , Hippocampus(..) + -- ** Lenses + , destination + -- ** Destination + , Destination(..) + , destinationFromPos + -- *** Lenses + , destinationPosition + , destinationProgress + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Test.QuickCheck +import Data.Aeson.Generic.DerivingVia +import Data.Aeson (ToJSON, FromJSON) +-------------------------------------------------------------------------------- +import Xanthous.AI.Gormlak +import Xanthous.Entities.RawTypes hiding + (Creature, description, damage) +import qualified Xanthous.Entities.RawTypes as Raw +import Xanthous.Game.State +import Xanthous.Data +import Xanthous.Data.Entities +import Xanthous.Entities.Creature.Hippocampus +import Xanthous.Util.QuickCheck (GenericArbitrary(..)) +import Xanthous.Entities.Common (Inventory, HasInventory(..)) +-------------------------------------------------------------------------------- + +data Creature = Creature + { _creatureType :: !CreatureType + , _hitpoints :: !Hitpoints + , _hippocampus :: !Hippocampus + , _inventory :: !Inventory + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Draw via DrawRawCharPriority "_creatureType" 1000 Creature + deriving Arbitrary via GenericArbitrary Creature + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Creature +makeFieldsNoPrefix ''Creature + +instance HasVisionRadius Creature where + visionRadius = const 50 -- TODO + +instance Brain Creature where + step = brainVia GormlakBrain + entityCanMove = const True + +instance Entity Creature where + entityAttributes _ = defaultEntityAttributes + & blocksObject .~ True + description = view $ creatureType . Raw.description + entityChar = view $ creatureType . char + entityCollision = const $ Just Combat + +-------------------------------------------------------------------------------- + +damage :: Hitpoints -> Creature -> Creature +damage amount = hitpoints %~ \hp -> + if hp <= amount + then 0 + else hp - amount + +isDead :: Creature -> Bool +isDead = views hitpoints (== 0) + +{-# ANN module ("Hlint: ignore Use newtype instead of data" :: String) #-} diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Creature/Hippocampus.hs b/users/grfn/xanthous/src/Xanthous/Entities/Creature/Hippocampus.hs new file mode 100644 index 000000000000..9d5cc134517f --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Creature/Hippocampus.hs @@ -0,0 +1,72 @@ +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.Creature.Hippocampus + (-- * Hippocampus + Hippocampus(..) + , initialHippocampus + -- ** Lenses + , destination + , greetedCharacter + -- ** Destination + , Destination(..) + , destinationFromPos + -- *** Lenses + , destinationPosition + , destinationProgress + ) +where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson.Generic.DerivingVia +import Data.Aeson (ToJSON, FromJSON) +import Test.QuickCheck +import Test.QuickCheck.Arbitrary.Generic +-------------------------------------------------------------------------------- +import Xanthous.Data +import Xanthous.Util.QuickCheck +-------------------------------------------------------------------------------- + + +data Destination = Destination + { _destinationPosition :: !Position + -- | The progress towards the destination, tracked as an offset from the + -- creature's original position. + -- + -- When this value reaches >= 1, the creature has reached their destination + , _destinationProgress :: !Tiles + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Destination +instance Arbitrary Destination where arbitrary = genericArbitrary +makeLenses ''Destination + +destinationFromPos :: Position -> Destination +destinationFromPos _destinationPosition = + let _destinationProgress = 0 + in Destination{..} + +data Hippocampus = Hippocampus + { _destination :: !(Maybe Destination) + , -- | Has this creature greeted the character in any way yet? + -- + -- Some creature types ignore this field + _greetedCharacter :: !Bool + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Hippocampus + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Hippocampus +makeLenses ''Hippocampus + +initialHippocampus :: Hippocampus +initialHippocampus = Hippocampus + { _destination = Nothing + , _greetedCharacter = False + } diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Draw/Util.hs b/users/grfn/xanthous/src/Xanthous/Entities/Draw/Util.hs new file mode 100644 index 000000000000..aa6c5fa4fc47 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Draw/Util.hs @@ -0,0 +1,31 @@ +module Xanthous.Entities.Draw.Util where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Brick.Widgets.Border.Style +import Brick.Types (Edges(..)) +-------------------------------------------------------------------------------- + +borderFromEdges :: BorderStyle -> Edges Bool -> Char +borderFromEdges bstyle edges = ($ bstyle) $ case edges of + Edges False False False False -> const '☐' + + Edges True False False False -> bsVertical + Edges False True False False -> bsVertical + Edges False False True False -> bsHorizontal + Edges False False False True -> bsHorizontal + + Edges True True False False -> bsVertical + Edges True False True False -> bsCornerBR + Edges True False False True -> bsCornerBL + + Edges False True True False -> bsCornerTR + Edges False True False True -> bsCornerTL + Edges False False True True -> bsHorizontal + + Edges False True True True -> bsIntersectT + Edges True False True True -> bsIntersectB + Edges True True False True -> bsIntersectL + Edges True True True False -> bsIntersectR + + Edges True True True True -> bsIntersectFull diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Entities.hs b/users/grfn/xanthous/src/Xanthous/Entities/Entities.hs new file mode 100644 index 000000000000..a0c037a1b4ed --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Entities.hs @@ -0,0 +1,63 @@ +{-# LANGUAGE StandaloneDeriving #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.Entities () where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Test.QuickCheck +import qualified Test.QuickCheck.Gen as Gen +import Data.Aeson +-------------------------------------------------------------------------------- +import Xanthous.Entities.Character +import Xanthous.Entities.Item +import Xanthous.Entities.Creature +import Xanthous.Entities.Environment +import Xanthous.Entities.Marker +import Xanthous.Game.State +import Xanthous.Util.QuickCheck +import Data.Aeson.Generic.DerivingVia +-------------------------------------------------------------------------------- + +instance Arbitrary SomeEntity where + arbitrary = Gen.oneof + [ SomeEntity <$> arbitrary @Character + , SomeEntity <$> arbitrary @Item + , SomeEntity <$> arbitrary @Creature + , SomeEntity <$> arbitrary @Wall + , SomeEntity <$> arbitrary @Door + , SomeEntity <$> arbitrary @GroundMessage + , SomeEntity <$> arbitrary @Staircase + , SomeEntity <$> arbitrary @Marker + ] + +instance FromJSON SomeEntity where + parseJSON = withObject "Entity" $ \obj -> do + (entityType :: Text) <- obj .: "type" + case entityType of + "Character" -> SomeEntity @Character <$> obj .: "data" + "Item" -> SomeEntity @Item <$> obj .: "data" + "Creature" -> SomeEntity @Creature <$> obj .: "data" + "Wall" -> SomeEntity @Wall <$> obj .: "data" + "Door" -> SomeEntity @Door <$> obj .: "data" + "GroundMessage" -> SomeEntity @GroundMessage <$> obj .: "data" + "Staircase" -> SomeEntity @Staircase <$> obj .: "data" + "Marker" -> SomeEntity @Marker <$> obj .: "data" + _ -> fail . unpack $ "Invalid entity type \"" <> entityType <> "\"" + +deriving via WithOptions '[ FieldLabelModifier '[Drop 1] ] GameLevel + instance FromJSON GameLevel +deriving via WithOptions '[ FieldLabelModifier '[Drop 1] ] GameState + instance FromJSON GameState + +instance Entity SomeEntity where + entityAttributes (SomeEntity ent) = entityAttributes ent + description (SomeEntity ent) = description ent + entityChar (SomeEntity ent) = entityChar ent + entityCollision (SomeEntity ent) = entityCollision ent + +instance Function SomeEntity where + function = functionJSON + +instance CoArbitrary SomeEntity where + coarbitrary = coarbitrary . encode diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Entities.hs-boot b/users/grfn/xanthous/src/Xanthous/Entities/Entities.hs-boot new file mode 100644 index 000000000000..519a862c6a5a --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Entities.hs-boot @@ -0,0 +1,14 @@ +{-# OPTIONS_GHC -fno-warn-orphans #-} +module Xanthous.Entities.Entities where + +import Test.QuickCheck +import Data.Aeson +import Xanthous.Game.State (SomeEntity, GameState, Entity) + +instance Arbitrary SomeEntity +instance Function SomeEntity +instance CoArbitrary SomeEntity +instance FromJSON SomeEntity +instance Entity SomeEntity + +instance FromJSON GameState diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Environment.hs b/users/grfn/xanthous/src/Xanthous/Entities/Environment.hs new file mode 100644 index 000000000000..b45a91eabed2 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Environment.hs @@ -0,0 +1,160 @@ +{-# LANGUAGE TemplateHaskell #-} +module Xanthous.Entities.Environment + ( + -- * Walls + Wall(..) + + -- * Doors + , Door(..) + , open + , closed + , locked + , unlockedDoor + + -- * Messages + , GroundMessage(..) + + -- * Stairs + , Staircase(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Test.QuickCheck +import Brick (str) +import Brick.Widgets.Border.Style (unicode) +import Brick.Types (Edges(..)) +import Data.Aeson +import Data.Aeson.Generic.DerivingVia +-------------------------------------------------------------------------------- +import Xanthous.Entities.Draw.Util +import Xanthous.Data +import Xanthous.Data.Entities +import Xanthous.Game.State +import Xanthous.Util.QuickCheck +-------------------------------------------------------------------------------- + +data Wall = Wall + deriving stock (Show, Eq, Ord, Generic, Enum) + deriving anyclass (NFData, CoArbitrary, Function) + +instance ToJSON Wall where + toJSON = const $ String "Wall" + +instance FromJSON Wall where + parseJSON = withText "Wall" $ \case + "Wall" -> pure Wall + _ -> fail "Invalid Wall: expected Wall" + +instance Brain Wall where step = brainVia Brainless + +instance Entity Wall where + entityAttributes _ = defaultEntityAttributes + & blocksVision .~ True + & blocksObject .~ True + description _ = "a wall" + entityChar _ = "┼" + +instance Arbitrary Wall where + arbitrary = pure Wall + +wallEdges :: (MonoFoldable mono, Element mono ~ SomeEntity) + => Neighbors mono -> Edges Bool +wallEdges neighs = any (entityIs @Wall) <$> edges neighs + +instance Draw Wall where + drawWithNeighbors neighs _wall = + str . pure . borderFromEdges unicode $ wallEdges neighs + +data Door = Door + { _open :: Bool + , _locked :: Bool + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function, ToJSON, FromJSON) + deriving Arbitrary via GenericArbitrary Door +makeLenses ''Door + +instance Draw Door where + drawWithNeighbors neighs door + = str . pure . ($ door ^. open) $ case wallEdges neighs of + Edges True False False False -> vertDoor + Edges False True False False -> vertDoor + Edges True True False False -> vertDoor + Edges False False True False -> horizDoor + Edges False False False True -> horizDoor + Edges False False True True -> horizDoor + _ -> allsidesDoor + where + horizDoor True = '␣' + horizDoor False = 'ᚔ' + vertDoor True = '[' + vertDoor False = 'ǂ' + allsidesDoor True = '+' + allsidesDoor False = '▥' + +instance Brain Door where step = brainVia Brainless + +instance Entity Door where + entityAttributes door = defaultEntityAttributes + & blocksVision .~ not (door ^. open) + description door | door ^. open = "an open door" + | otherwise = "a closed door" + entityChar _ = "d" + entityCollision door | door ^. open = Nothing + | otherwise = Just Stop + +closed :: Lens' Door Bool +closed = open . involuted not + +-- | A closed, unlocked door +unlockedDoor :: Door +unlockedDoor = Door + { _open = False + , _locked = False + } + +-------------------------------------------------------------------------------- + +newtype GroundMessage = GroundMessage Text + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary GroundMessage + deriving (ToJSON, FromJSON) + via WithOptions '[ 'TagSingleConstructors 'True + , 'SumEnc 'ObjWithSingleField + ] + GroundMessage + deriving Draw + via DrawStyledCharacter ('Just 'Yellow) 'Nothing "≈" + GroundMessage +instance Brain GroundMessage where step = brainVia Brainless + +instance Entity GroundMessage where + description = const "a message on the ground. Press r. to read it." + entityChar = const "≈" + entityCollision = const Nothing + +-------------------------------------------------------------------------------- + +data Staircase = UpStaircase | DownStaircase + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Staircase + deriving (ToJSON, FromJSON) + via WithOptions '[ 'TagSingleConstructors 'True + , 'SumEnc 'ObjWithSingleField + ] + Staircase +instance Brain Staircase where step = brainVia Brainless + +instance Draw Staircase where + draw UpStaircase = str "<" + draw DownStaircase = str ">" + +instance Entity Staircase where + description UpStaircase = "a staircase leading upwards" + description DownStaircase = "a staircase leading downwards" + entityChar UpStaircase = "<" + entityChar DownStaircase = ">" + entityCollision = const Nothing diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Item.hs b/users/grfn/xanthous/src/Xanthous/Entities/Item.hs new file mode 100644 index 000000000000..eadd62569663 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Item.hs @@ -0,0 +1,76 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.Item + ( Item(..) + , itemType + , density + , volume + , newWithType + , isEdible + , weight + , fullDescription + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Test.QuickCheck (Arbitrary, CoArbitrary, Function) +import Data.Aeson (ToJSON, FromJSON) +import Data.Aeson.Generic.DerivingVia +import Control.Monad.Random (MonadRandom) +-------------------------------------------------------------------------------- +import Xanthous.Entities.RawTypes (ItemType) +import qualified Xanthous.Entities.RawTypes as Raw +import Xanthous.Game.State +import Xanthous.Data (Grams, Per, Cubic, Meters, (|*|)) +import Xanthous.Util.QuickCheck (GenericArbitrary(GenericArbitrary)) +import Xanthous.Random (choose, FiniteInterval(..)) +-------------------------------------------------------------------------------- + +data Item = Item + { _itemType :: ItemType + , _density :: Grams `Per` Cubic Meters + , _volume :: Cubic Meters + } + deriving stock (Eq, Show, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Draw via DrawRawChar "_itemType" Item + deriving Arbitrary via GenericArbitrary Item + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + Item +makeLenses ''Item + +-- deriving via (Brainless Item) instance Brain Item +instance Brain Item where step = brainVia Brainless + +instance Entity Item where + description = view $ itemType . Raw.description + entityChar = view $ itemType . Raw.char + entityCollision = const Nothing + +newWithType :: MonadRandom m => ItemType -> m Item +newWithType _itemType = do + _density <- choose . FiniteInterval $ _itemType ^. Raw.density + _volume <- choose . FiniteInterval $ _itemType ^. Raw.volume + pure Item {..} + +isEdible :: Item -> Bool +isEdible = Raw.isEdible . view itemType + +-- | The weight of this item, calculated by multiplying its volume by the +-- density of its material +weight :: Item -> Grams +weight item = (item ^. density) |*| (item ^. volume) + +-- | Describe the item in full detail +fullDescription :: Item -> Text +fullDescription item = unlines + [ item ^. itemType . Raw.description + , "" + , item ^. itemType . Raw.longDescription + , "" + , "volume: " <> tshow (item ^. volume) + , "density: " <> tshow (item ^. density) + , "weight: " <> tshow (weight item) + ] diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Marker.hs b/users/grfn/xanthous/src/Xanthous/Entities/Marker.hs new file mode 100644 index 000000000000..14d02872ed4e --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Marker.hs @@ -0,0 +1,41 @@ +-------------------------------------------------------------------------------- +module Xanthous.Entities.Marker ( Marker(..) ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson +import Test.QuickCheck +import qualified Graphics.Vty.Attributes as Vty +import qualified Graphics.Vty.Image as Vty +import Brick.Widgets.Core (raw) +-------------------------------------------------------------------------------- +import Xanthous.Game.State +import Xanthous.Data.Entities (EntityAttributes(..)) +-------------------------------------------------------------------------------- + +-- | Mark on the map - for use in debugging / development only. +newtype Marker = Marker Text + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (Semigroup, Monoid, ToJSON, FromJSON, Arbitrary) via Text + +instance Brain Marker where step = brainVia Brainless + +instance Entity Marker where + entityAttributes = const EntityAttributes + { _blocksVision = False + , _blocksObject = False + , _collision = Stop + } + description (Marker m) = "[M] " <> m + entityChar = const $ "X" & style .~ markerStyle + entityCollision = const Nothing + +instance Draw Marker where + draw = const . raw $ Vty.char markerStyle 'X' + drawPriority = const maxBound + +markerStyle :: Vty.Attr +markerStyle = Vty.defAttr + `Vty.withForeColor` Vty.red + `Vty.withBackColor` Vty.black diff --git a/users/grfn/xanthous/src/Xanthous/Entities/RawTypes.hs b/users/grfn/xanthous/src/Xanthous/Entities/RawTypes.hs new file mode 100644 index 000000000000..9f5cabecdca7 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/RawTypes.hs @@ -0,0 +1,277 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE DuplicateRecordFields #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.RawTypes + ( + EntityRaw(..) + , _Creature + , _Item + + -- * Creatures + , CreatureType(..) + , hostile + -- ** Generation parameters + , CreatureGenerateParams(..) + , canGenerate + -- ** Language + , LanguageName(..) + , getLanguage + -- ** Attacks + , Attack(..) + + -- * Items + , ItemType(..) + -- ** Item sub-types + -- *** Edible + , EdibleItem(..) + , isEdible + -- *** Wieldable + , WieldableItem(..) + , isWieldable + + -- * Lens classes + , HasAttackMessage(..) + , HasAttacks(..) + , HasChance(..) + , HasChar(..) + , HasCreatureAttackMessage(..) + , HasDamage(..) + , HasDensity(..) + , HasDescription(..) + , HasEatMessage(..) + , HasEdible(..) + , HasEntityName(..) + , HasEquippedItem(..) + , HasFriendly(..) + , HasGenerateParams(..) + , HasHitpointsHealed(..) + , HasLanguage(..) + , HasLevelRange(..) + , HasLongDescription(..) + , HasMaxHitpoints(..) + , HasName(..) + , HasSayVerb(..) + , HasSpeed(..) + , HasVolume(..) + , HasWieldable(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Test.QuickCheck +import Data.Aeson.Generic.DerivingVia +import Data.Aeson (ToJSON, FromJSON) +import Data.Interval (Interval, lowerBound', upperBound') +import qualified Data.Interval as Interval +-------------------------------------------------------------------------------- +import Xanthous.Messages (Message(..)) +import Xanthous.Data (TicksPerTile, Hitpoints, Per, Grams, Cubic, Meters) +import Xanthous.Data.EntityChar +import Xanthous.Util.QuickCheck +import Xanthous.Generators.Speech (Language, gormlak, english) +import Xanthous.Orphans () +import Xanthous.Util (EqProp, EqEqProp(..)) +-------------------------------------------------------------------------------- + +-- | Identifiers for languages that creatures can speak. +-- +-- Non-verbal or non-sentient creatures have Nothing as their language +-- +-- At some point, we will likely want to make languages be defined in data files +-- somewhere, and reference them that way instead. +data LanguageName = Gormlak | English + deriving stock (Show, Eq, Ord, Generic, Enum, Bounded) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary LanguageName + deriving (ToJSON, FromJSON) + via WithOptions '[ AllNullaryToStringTag 'True ] + LanguageName + +-- | Resolve a 'LanguageName' into an actual 'Language' +getLanguage :: LanguageName -> Language +getLanguage Gormlak = gormlak +getLanguage English = english + +-- | Natural attacks for creature types +data Attack = Attack + { -- | the @{{creature}}@ @{{description}}@ + _description :: !Message + -- | Damage dealt + , _damage :: !Hitpoints + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Attack + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] + , OmitNothingFields 'True + ] + Attack +makeFieldsNoPrefix ''Attack + +-- | Description for generating an item equipped to a creature +data CreatureEquippedItem = CreatureEquippedItem + { -- | Name of the entity type to generate + _entityName :: !Text + -- | Chance of generating the item when generating the creature + -- + -- A chance of 1.0 will always generate the item + , _chance :: !Double + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary CreatureEquippedItem + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] + , OmitNothingFields 'True + ] + CreatureEquippedItem +makeFieldsNoPrefix ''CreatureEquippedItem + + +data CreatureGenerateParams = CreatureGenerateParams + { -- | Range of dungeon levels at which to generate this creature + _levelRange :: !(Interval Word) + -- | Item equipped to the creature + , _equippedItem :: !(Maybe CreatureEquippedItem) + } + deriving stock (Eq, Show, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary CreatureGenerateParams + deriving EqProp via EqEqProp CreatureGenerateParams + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + CreatureGenerateParams +makeFieldsNoPrefix ''CreatureGenerateParams + +instance Ord CreatureGenerateParams where + compare + = (compare `on` lowerBound' . _levelRange) + <> (compare `on` upperBound' . _levelRange) + <> (compare `on` _equippedItem) + +-- | Can a creature with these generate params be generated on this level? +canGenerate + :: Word -- ^ Level number + -> CreatureGenerateParams + -> Bool +canGenerate levelNumber gps = Interval.member levelNumber $ gps ^. levelRange + +data CreatureType = CreatureType + { _name :: !Text + , _description :: !Text + , _char :: !EntityChar + , _maxHitpoints :: !Hitpoints + , _friendly :: !Bool + , _speed :: !TicksPerTile + , _language :: !(Maybe LanguageName) + , -- | The verb, in present tense, for when the creature says something + _sayVerb :: !(Maybe Text) + , -- | The creature's natural attacks + _attacks :: !(NonNull (Vector Attack)) + -- | Parameters for generating the creature in levels + , _generateParams :: !(Maybe CreatureGenerateParams) + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary CreatureType + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] + , OmitNothingFields 'True + ] + CreatureType +makeFieldsNoPrefix ''CreatureType + +hostile :: Lens' CreatureType Bool +hostile = friendly . involuted not + +-------------------------------------------------------------------------------- + +data EdibleItem = EdibleItem + { _hitpointsHealed :: !Int + , _eatMessage :: !(Maybe Message) + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary EdibleItem + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + EdibleItem +makeFieldsNoPrefix ''EdibleItem + +data WieldableItem = WieldableItem + { _damage :: !Hitpoints + , _attackMessage :: !(Maybe Message) + -- | Message to use when a creature is using this item to attack the + -- character. + -- + -- Grammatically, should be of the form "The creature slashes you with its + -- dagger". + -- + -- = Parameters + -- + -- [@creature@ (type: 'Creature')] The creature doing the attacking + -- [@item@ (type: 'Item')] The item itself + , _creatureAttackMessage :: !(Maybe Message) + } + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary WieldableItem + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + WieldableItem +makeFieldsNoPrefix ''WieldableItem + +-------------------------------------------------------------------------------- + +data ItemType = ItemType + { _name :: !Text + , _description :: !Text + , _longDescription :: !Text + , _char :: !EntityChar + , _density :: !(Interval (Grams `Per` Cubic Meters)) + , _volume :: !(Interval (Cubic Meters)) + , _edible :: !(Maybe EdibleItem) + , _wieldable :: !(Maybe WieldableItem) + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary ItemType + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + ItemType +makeFieldsNoPrefix ''ItemType + +instance Ord ItemType where + compare x y + = compareOf name x y + <> compareOf description x y + <> compareOf longDescription x y + <> compareOf char x y + <> compareOf (density . to extractInterval) x y + <> compareOf (volume . to extractInterval) x y + <> compareOf edible x y + <> compareOf wieldable x y + where + compareOf l = comparing (view l) + extractInterval = lowerBound' &&& upperBound' + +-- | Can this item be eaten? +isEdible :: ItemType -> Bool +isEdible = has $ edible . _Just + +-- | Can this item be used as a weapon? +isWieldable :: ItemType -> Bool +isWieldable = has $ wieldable . _Just + +-------------------------------------------------------------------------------- + +data EntityRaw + = Creature !CreatureType + | Item !ItemType + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) + deriving Arbitrary via GenericArbitrary EntityRaw + deriving (FromJSON) + via WithOptions '[ SumEnc ObjWithSingleField ] + EntityRaw +makePrisms ''EntityRaw diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws.hs b/users/grfn/xanthous/src/Xanthous/Entities/Raws.hs new file mode 100644 index 000000000000..10f0d831934e --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws.hs @@ -0,0 +1,49 @@ +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.Raws + ( raws + , raw + , RawType(..) + , rawsWithType + ) where +-------------------------------------------------------------------------------- +import Data.FileEmbed +import qualified Data.Yaml as Yaml +import Xanthous.Prelude +import System.FilePath.Posix +-------------------------------------------------------------------------------- +import Xanthous.Entities.RawTypes +import Xanthous.AI.Gormlak () +-------------------------------------------------------------------------------- +rawRaws :: [(FilePath, ByteString)] +rawRaws = $(embedDir "src/Xanthous/Entities/Raws") + +raws :: HashMap Text EntityRaw +raws + = mapFromList + . map (bimap + (pack . takeBaseName) + (either (error . Yaml.prettyPrintParseException) id + . Yaml.decodeEither')) + $ rawRaws + +raw :: Text -> Maybe EntityRaw +raw n = raws ^. at n + +class RawType (a :: Type) where + _RawType :: Prism' EntityRaw a + +instance RawType CreatureType where + _RawType = prism' Creature $ \case + Creature c -> Just c + _ -> Nothing + +instance RawType ItemType where + _RawType = prism' Item $ \case + Item i -> Just i + _ -> Nothing + +rawsWithType :: forall a. RawType a => HashMap Text a +rawsWithType = mapFromList . itoListOf (ifolded . _RawType) $ raws + +-------------------------------------------------------------------------------- diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/broken-dagger.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/broken-dagger.yaml new file mode 100644 index 000000000000..2d30e6986b6e --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/broken-dagger.yaml @@ -0,0 +1,24 @@ +Item: + name: broken dagger + description: a short, broken dagger + longDescription: A short dagger with a twisted, chipped blade + char: + char: † + style: + foreground: black + wieldable: + damage: 3 + attackMessage: + - You slash at the {{creature.creatureType.name}} with your dagger. + - You stab the {{creature.creatureType.name}} with your dagger. + creatureAttackMessage: + - The {{creature.creatureType.name}} slashes at you with its dagger. + - The {{creature.creatureType.name}} stabs you with its dagger. + # Just the steel, not the handle, for now + density: [7750 , 8050000] + # 15cm – 45cm + # × + # 2cm – 3cm + # × + # .5cm – 1cm + volume: [0.15, 1.35] diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/gormlak.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/gormlak.yaml new file mode 100644 index 000000000000..ad3d9cb147da --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/gormlak.yaml @@ -0,0 +1,20 @@ +Creature: + name: gormlak + description: a gormlak + longDescription: | + A chittering imp-like creature with bright yellow horns and sharp claws. It + adores shiny objects and gathers in swarms. + char: + char: g + style: + foreground: red + maxHitpoints: 5 + speed: 125 + friendly: false + language: Gormlak + sayVerb: yells + attacks: + - description: + - claws you + - slashes you with its claws + damage: 1 diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/husk.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/husk.yaml new file mode 100644 index 000000000000..cdfcde616d21 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/husk.yaml @@ -0,0 +1,26 @@ +Creature: + name: husk + description: an empty husk of some humanoid creature + longDescription: | + An empty husk of a humanoid creature. All semblance of sentience has long + left its eyes; instead it shambles about aimlessly, always hungering for the + warmth of life. + char: + char: h + style: + foreground: black + maxHitpoints: 6 + speed: 110 + friendly: false + attacks: + - description: + - swings its arms at you + - elbows you + damage: 1 + - description: kicks you + damage: 2 + generateParams: + levelRange: [1, PosInf] + equippedItem: + entityName: broken-dagger + chance: 0.9 diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/noodles.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/noodles.yaml new file mode 100644 index 000000000000..c0501a18a8e0 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/noodles.yaml @@ -0,0 +1,14 @@ +Item: + name: noodles + description: "a big bowl o' noodles" + longDescription: You know exactly what kind of noodles + char: + char: 'n' + style: + foreground: yellow + edible: + hitpointsHealed: 2 + eatMessage: + - You slurp up the noodles. Yumm! + density: 500000 + volume: 0.001 diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/ooze.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/ooze.yaml new file mode 100644 index 000000000000..fe427c94abf7 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/ooze.yaml @@ -0,0 +1,15 @@ +Creature: + name: ooze + description: an ooze + longDescription: | + A jiggling, amorphous, bright green caustic blob + char: + char: o + style: + foreground: green + maxHitpoints: 3 + speed: 100 + friendly: false + attacks: + - description: slams into you + damage: 1 diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/rock.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/rock.yaml new file mode 100644 index 000000000000..e7492bf5fb6f --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/rock.yaml @@ -0,0 +1,10 @@ +Item: + name: rock + description: a rock + longDescription: a medium-sized rock made out of some unknown stone + char: . + wieldable: + damage: 1 + attackMessage: you hit the {{creature.creatureType.name}} in the head with your rock. + density: [ 1500000, 2500000 ] + volume: [ 0.000125, 0.001 ] diff --git a/users/grfn/xanthous/src/Xanthous/Entities/Raws/stick.yaml b/users/grfn/xanthous/src/Xanthous/Entities/Raws/stick.yaml new file mode 100644 index 000000000000..a7eae9c36666 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Entities/Raws/stick.yaml @@ -0,0 +1,22 @@ +Item: + name: stick + description: a wooden stick + longDescription: A sturdy branch broken off from some sort of tree + char: + char: ∤ + style: + foreground: yellow + wieldable: + damage: 2 + attackMessage: + - You bonk the {{creature.creatureType.name}} over the head with your stick. + - You bash the {{creature.creatureType.name}} on the noggin with your stick. + - You whack the {{creature.creatureType.name}} with your stick. + creatureAttackMessage: + - The {{creature.creatureType.name}} bonks you over the head with its stick. + - The {{creature.creatureType.name}} bashes you on the noggin with its stick. + - The {{creature.creatureType.name}} whacks you with its stick. + # https://www.sciencedirect.com/topics/agricultural-and-biological-sciences/wood-density + # it's a hard stick. so it's dense wood. + density: 890000 # g/m³ + volume: [ 0.003, 0.006 ] # ≈3.5 cm radius × ≈1m length diff --git a/users/grfn/xanthous/src/Xanthous/Game.hs b/users/grfn/xanthous/src/Xanthous/Game.hs new file mode 100644 index 000000000000..89c23f0de850 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game.hs @@ -0,0 +1,73 @@ +module Xanthous.Game + ( GameState(..) + , levels + , entities + , revealedPositions + , messageHistory + , randomGen + , promptState + , GamePromptState(..) + + , getInitialState + , initialStateFromSeed + + , positionedCharacter + , character + , characterPosition + , updateCharacterVision + , characterVisiblePositions + , entitiesAtCharacter + , revealedEntitiesAtPosition + + -- * Messages + , MessageHistory(..) + , HasMessages(..) + , HasTurn(..) + , HasDisplayedTurn(..) + , pushMessage + , previousMessage + , nextTurn + + -- * Collisions + , Collision(..) + , collisionAt + + -- * App monad + , AppT(..) + + -- * Saving the game + , saveGame + , loadGame + , saved + + -- * Debug State + , DebugState(..) + , debugState + , allRevealed + ) where +-------------------------------------------------------------------------------- +import qualified Codec.Compression.Zlib as Zlib +import Codec.Compression.Zlib.Internal (DecompressError) +import qualified Data.Aeson as JSON +import System.IO.Unsafe +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Xanthous.Game.State +import Xanthous.Game.Lenses +import Xanthous.Game.Arbitrary () +import Xanthous.Entities.Entities () +-------------------------------------------------------------------------------- + +saveGame :: GameState -> LByteString +saveGame = Zlib.compress . JSON.encode + +loadGame :: LByteString -> Maybe GameState +loadGame = JSON.decode <=< decompressZlibMay + where + decompressZlibMay bs + = unsafeDupablePerformIO + $ (let r = Zlib.decompress bs in r `seq` pure (Just r)) + `catch` \(_ :: DecompressError) -> pure Nothing + +saved :: Prism' LByteString GameState +saved = prism' saveGame loadGame diff --git a/users/grfn/xanthous/src/Xanthous/Game/Arbitrary.hs b/users/grfn/xanthous/src/Xanthous/Game/Arbitrary.hs new file mode 100644 index 000000000000..679bfe54597f --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/Arbitrary.hs @@ -0,0 +1,53 @@ +{-# LANGUAGE UndecidableInstances #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Xanthous.Game.Arbitrary where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (foldMap) +-------------------------------------------------------------------------------- +import Test.QuickCheck +import System.Random +import Data.Foldable (foldMap) +-------------------------------------------------------------------------------- +import Xanthous.Data.Levels +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Entities.Entities () +import Xanthous.Entities.Character +import Xanthous.Game.State +import Xanthous.Orphans () +import Xanthous.Util.QuickCheck (GenericArbitrary(..)) +-------------------------------------------------------------------------------- + +deriving via GenericArbitrary GameLevel instance Arbitrary GameLevel + +instance Arbitrary GameState where + arbitrary = do + chr <- arbitrary @Character + _upStaircasePosition <- arbitrary + _messageHistory <- arbitrary + levs <- arbitrary @(Levels GameLevel) + _levelRevealedPositions <- + fmap setFromList + . sublistOf + . foldMap (EntityMap.positions . _levelEntities) + $ levs + let (_characterEntityID, _levelEntities) = + EntityMap.insertAtReturningID _upStaircasePosition (SomeEntity chr) + $ levs ^. current . levelEntities + _levels = levs & current .~ GameLevel {..} + _randomGen <- mkStdGen <$> arbitrary + let _promptState = NoPrompt -- TODO + _activePanel <- arbitrary + _debugState <- arbitrary + let _autocommand = NoAutocommand + _memo <- arbitrary + _savefile <- arbitrary + pure $ GameState {..} + + +instance CoArbitrary GameLevel +instance Function GameLevel +instance CoArbitrary GameState +instance Function GameState diff --git a/users/grfn/xanthous/src/Xanthous/Game/Draw.hs b/users/grfn/xanthous/src/Xanthous/Game/Draw.hs new file mode 100644 index 000000000000..53ea1c96f8af --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/Draw.hs @@ -0,0 +1,151 @@ +-------------------------------------------------------------------------------- +module Xanthous.Game.Draw + ( drawGame + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Brick hiding (loc, on) +import Brick.Widgets.Border +import Brick.Widgets.Border.Style +import Brick.Widgets.Edit +import Control.Monad.State.Lazy (evalState) +import Control.Monad.State.Class ( get, MonadState, gets ) +-------------------------------------------------------------------------------- +import Xanthous.Data +import Xanthous.Data.App (ResourceName, Panel(..)) +import qualified Xanthous.Data.App as Resource +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Game.State +import Xanthous.Entities.Common (Wielded(..), wielded, backpack) +import Xanthous.Entities.Character +import Xanthous.Entities.Item (Item) +import Xanthous.Game + ( characterPosition + , character + , revealedEntitiesAtPosition + ) +import Xanthous.Game.Prompt +import Xanthous.Orphans () +-------------------------------------------------------------------------------- + +cursorPosition :: GameState -> Widget ResourceName -> Widget ResourceName +cursorPosition game + | WaitingPrompt _ (Prompt _ _ (preview promptStatePosition -> Just pos) _ _) + <- game ^. promptState + = showCursor Resource.Prompt (pos ^. loc) + | otherwise + = showCursor Resource.Character (game ^. characterPosition . loc) + +drawMessages :: MessageHistory -> Widget ResourceName +drawMessages = txtWrap . (<> " ") . unwords . reverse . oextract + +drawPromptState :: GamePromptState m -> Widget ResourceName +drawPromptState NoPrompt = emptyWidget +drawPromptState (WaitingPrompt msg (Prompt _ pt ps pri _)) = + case (pt, ps, pri) of + (SStringPrompt, StringPromptState edit, mDef) -> + txt msg + <+> txt (maybe "" (\def -> "(default: " <> def <> ") ") mDef) + <+> renderEditor (txt . fold) True edit + (SDirectionPrompt, DirectionPromptState, _) -> txtWrap msg + (SMenu, _, menuItems) -> + txtWrap msg + <=> foldl' (<=>) emptyWidget (map drawMenuItem $ itoList menuItems) + _ -> txtWrap msg + where + drawMenuItem (chr, MenuOption m _) = + str ("[" <> pure chr <> "] ") <+> txtWrap m + +drawEntities + :: forall m. MonadState GameState m + => m (Widget ResourceName) +drawEntities = do + allEnts <- use entities + let entityPositions = EntityMap.positions allEnts + maxY = fromMaybe 0 $ maximumOf (folded . y) entityPositions + maxX = fromMaybe 0 $ maximumOf (folded . x) entityPositions + rows = traverse mkRow [0..maxY] + mkRow rowY = hBox <$> traverse (renderEntityAt . flip Position rowY) [0..maxX] + renderEntityAt pos + = renderTopEntity pos <$> revealedEntitiesAtPosition pos + renderTopEntity pos ents + = let neighbors = EntityMap.neighbors pos allEnts + in maybe (str " ") (drawWithNeighbors neighbors) + $ maximumBy (compare `on` drawPriority) + <$> fromNullable ents + vBox <$> rows + +drawMap :: MonadState GameState m => m (Widget ResourceName) +drawMap = do + cursorPos <- gets cursorPosition + viewport Resource.MapViewport Both . cursorPos <$> drawEntities + +bullet :: Char +bullet = '•' + +drawInventoryPanel :: GameState -> Widget ResourceName +drawInventoryPanel game + = drawWielded (game ^. character . inventory . wielded) + <=> drawBackpack (game ^. character . inventory . backpack) + where + drawWielded (Hands Nothing Nothing) = emptyWidget + drawWielded (DoubleHanded i) = + txtWrap $ "You are holding " <> description i <> " in both hands" + drawWielded (Hands l r) = drawHand "left" l <=> drawHand "right" r + drawHand side = maybe emptyWidget $ \i -> + txtWrap ( "You are holding " + <> description i + <> " in your " <> side <> " hand" + ) + <=> txt " " + + drawBackpack :: Vector Item -> Widget ResourceName + drawBackpack Empty = txtWrap "Your backpack is empty right now." + drawBackpack backpackItems + = txtWrap ( "You are currently carrying the following items in your " + <> "backpack:") + <=> txt " " + <=> foldl' (<=>) emptyWidget + (map + (txtWrap . ((bullet <| " ") <>) . description) + backpackItems) + + +drawPanel :: GameState -> Panel -> Widget ResourceName +drawPanel game panel + = border + . hLimit 35 + . viewport (Resource.Panel panel) Vertical + . case panel of + InventoryPanel -> drawInventoryPanel + ItemDescriptionPanel desc -> const $ txtWrap desc + $ game + +drawCharacterInfo :: Character -> Widget ResourceName +drawCharacterInfo ch = txt " " <+> charName <+> charHitpoints + where + charName | Just n <- ch ^. characterName + = txt $ n <> " " + | otherwise + = emptyWidget + charHitpoints + = txt "Hitpoints: " + <+> txt (tshow $ let Hitpoints hp = characterHitpoints ch in hp) + +drawGame :: GameState -> [Widget ResourceName] +drawGame = evalState $ do + game <- get + drawnMap <- drawMap + pure + . pure + . withBorderStyle unicode + $ case game ^. promptState of + NoPrompt -> drawMessages (game ^. messageHistory) + _ -> emptyWidget + <=> drawPromptState (game ^. promptState) + <=> + (maybe emptyWidget (drawPanel game) (game ^. activePanel) + <+> border drawnMap + ) + <=> drawCharacterInfo (game ^. character) diff --git a/users/grfn/xanthous/src/Xanthous/Game/Env.hs b/users/grfn/xanthous/src/Xanthous/Game/Env.hs new file mode 100644 index 000000000000..5d7b275c8a0b --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/Env.hs @@ -0,0 +1,37 @@ +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Game.Env + ( Config(..) + , defaultConfig + , disableSaving + , GameEnv(..) + , eventChan + , config + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Brick.BChan (BChan) +import Xanthous.Data.App (AppEvent) +-------------------------------------------------------------------------------- + +data Config = Config + { _disableSaving :: Bool + } + deriving stock (Generic, Show, Eq) +makeLenses ''Config +{-# ANN Config ("HLint: ignore Use newtype instead of data" :: String) #-} + +defaultConfig :: Config +defaultConfig = Config + { _disableSaving = False + } + +-------------------------------------------------------------------------------- + +data GameEnv = GameEnv + { _eventChan :: BChan AppEvent + , _config :: Config + } + deriving stock (Generic) +makeLenses ''GameEnv diff --git a/users/grfn/xanthous/src/Xanthous/Game/Lenses.hs b/users/grfn/xanthous/src/Xanthous/Game/Lenses.hs new file mode 100644 index 000000000000..c692a3b47944 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/Lenses.hs @@ -0,0 +1,178 @@ +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE AllowAmbiguousTypes #-} +-------------------------------------------------------------------------------- +module Xanthous.Game.Lenses + ( clearMemo + , positionedCharacter + , character + , characterPosition + , updateCharacterVision + , characterVisiblePositions + , characterVisibleEntities + , positionIsCharacterVisible + , getInitialState + , initialStateFromSeed + , entitiesAtCharacter + , revealedEntitiesAtPosition + , hearingRadius + + -- * Collisions + , Collision(..) + , entitiesCollision + , collisionAt + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import System.Random +import Control.Monad.State +import Control.Monad.Random (getRandom) +-------------------------------------------------------------------------------- +import Xanthous.Game.State +import qualified Xanthous.Game.Memo as Memo +import Xanthous.Data +import Xanthous.Data.Levels +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Data.EntityMap.Graphics + (visiblePositions, visibleEntities) +import Xanthous.Data.VectorBag +import Xanthous.Entities.Character (Character, mkCharacter) +import {-# SOURCE #-} Xanthous.Entities.Entities () +import Xanthous.Game.Memo (emptyMemoState, MemoState) +import Xanthous.Data.Memo (fillWithM, Memoized) +-------------------------------------------------------------------------------- + +getInitialState :: IO GameState +getInitialState = initialStateFromSeed <$> getRandom + +initialStateFromSeed :: Int -> GameState +initialStateFromSeed seed = + let _randomGen = mkStdGen seed + chr = mkCharacter + _upStaircasePosition = Position 0 0 + (_characterEntityID, _levelEntities) + = EntityMap.insertAtReturningID + _upStaircasePosition + (SomeEntity chr) + mempty + _levelRevealedPositions = mempty + level = GameLevel {..} + _levels = oneLevel level + _messageHistory = mempty + _promptState = NoPrompt + _activePanel = Nothing + _debugState = DebugState + { _allRevealed = False + } + _savefile = Nothing + _autocommand = NoAutocommand + _memo = emptyMemoState + in GameState {..} + +clearMemo :: MonadState GameState m => Lens' MemoState (Memoized k v) -> m () +clearMemo l = memo %= Memo.clear l + +positionedCharacter :: Lens' GameState (Positioned Character) +positionedCharacter = lens getPositionedCharacter setPositionedCharacter + where + setPositionedCharacter :: GameState -> Positioned Character -> GameState + setPositionedCharacter game chr + = game + & entities . at (game ^. characterEntityID) + ?~ fmap SomeEntity chr + + getPositionedCharacter :: GameState -> Positioned Character + getPositionedCharacter game + = over positioned + ( fromMaybe (error "Invariant error: Character was not a character!") + . downcastEntity + ) + . fromMaybe (error "Invariant error: Character not found!") + $ EntityMap.lookupWithPosition + (game ^. characterEntityID) + (game ^. entities) + + +character :: Lens' GameState Character +character = positionedCharacter . positioned + +characterPosition :: Lens' GameState Position +characterPosition = positionedCharacter . position + +-- TODO make this dynamic +visionRadius :: Word +visionRadius = 12 + +-- TODO make this dynamic +hearingRadius :: Word +hearingRadius = 12 + +-- | Update the revealed entities at the character's position based on their +-- vision +updateCharacterVision :: GameState -> GameState +updateCharacterVision = execState $ do + positions <- characterVisiblePositions + revealedPositions <>= positions + +characterVisiblePositions :: MonadState GameState m => m (Set Position) +characterVisiblePositions = do + charPos <- use characterPosition + fillWithM + (memo . Memo.characterVisiblePositions) + charPos + (uses entities $ visiblePositions charPos visionRadius) + +characterVisibleEntities :: GameState -> EntityMap.EntityMap SomeEntity +characterVisibleEntities game = + let charPos = game ^. characterPosition + in visibleEntities charPos visionRadius $ game ^. entities + +positionIsCharacterVisible :: MonadState GameState m => Position -> m Bool +positionIsCharacterVisible p = (p `elem`) <$> characterVisiblePositions +-- ^ TODO optimize + +entitiesCollision + :: ( Functor f + , forall xx. MonoFoldable (f xx) + , Element (f SomeEntity) ~ SomeEntity + , Element (f (Maybe Collision)) ~ Maybe Collision + , Show (f (Maybe Collision)) + , Show (f SomeEntity) + ) + => f SomeEntity + -> Maybe Collision +entitiesCollision = join . maximumMay . fmap entityCollision + +collisionAt :: MonadState GameState m => Position -> m (Maybe Collision) +collisionAt p = uses (entities . EntityMap.atPosition p) entitiesCollision + +entitiesAtCharacter :: Lens' GameState (VectorBag SomeEntity) +entitiesAtCharacter = lens getter setter + where + getter gs = gs ^. entities . EntityMap.atPosition (gs ^. characterPosition) + setter gs ents = gs + & entities . EntityMap.atPosition (gs ^. characterPosition) .~ ents + +-- | Returns all entities at the given position that are revealed to the +-- character. +-- +-- Concretely, this is either entities that are *currently* visible to the +-- character, or entities, that are immobile and that the character has seen +-- before +revealedEntitiesAtPosition + :: MonadState GameState m + => Position + -> m (VectorBag SomeEntity) +revealedEntitiesAtPosition p = do + allRev <- use $ debugState . allRevealed + cvps <- characterVisiblePositions + entitiesAtPosition <- use $ entities . EntityMap.atPosition p + revealed <- use revealedPositions + let immobileEntitiesAtPosition = filter (not . entityCanMove) entitiesAtPosition + pure $ if | allRev || p `member` cvps + -> entitiesAtPosition + | p `member` revealed + -> immobileEntitiesAtPosition + | otherwise + -> mempty diff --git a/users/grfn/xanthous/src/Xanthous/Game/Memo.hs b/users/grfn/xanthous/src/Xanthous/Game/Memo.hs new file mode 100644 index 000000000000..9e483a8d4af7 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/Memo.hs @@ -0,0 +1,52 @@ +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +-- | Memoized versions of calculations +-------------------------------------------------------------------------------- +module Xanthous.Game.Memo + ( MemoState + , emptyMemoState + , clear + -- ** Memo lenses + , characterVisiblePositions + + -- * Memoized values + , Memoized(UnMemoized) + , memoizeWith + , getMemoized + , runMemoized + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson (ToJSON, FromJSON) +import Data.Aeson.Generic.DerivingVia +import Test.QuickCheck (CoArbitrary, Function, Arbitrary) +-------------------------------------------------------------------------------- +import Xanthous.Data (Position) +import Xanthous.Data.Memo +import Xanthous.Util.QuickCheck (GenericArbitrary(GenericArbitrary)) +-------------------------------------------------------------------------------- + +-- | Memoized calculations on the game state +data MemoState = MemoState + { -- | Memoized version of 'Xanthous.Game.Lenses.characterVisiblePositions', + -- memoized with the position of the character + _characterVisiblePositions :: Memoized Position (Set Position) + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary MemoState + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + MemoState +makeLenses ''MemoState + +emptyMemoState :: MemoState +emptyMemoState = MemoState { _characterVisiblePositions = UnMemoized } +{-# INLINE emptyMemoState #-} + +clear :: Lens' MemoState (Memoized k v) -> MemoState -> MemoState +clear = flip set UnMemoized +{-# INLINE clear #-} + +{-# ANN module ("Hlint: ignore Use newtype instead of data" :: String) #-} diff --git a/users/grfn/xanthous/src/Xanthous/Game/Prompt.hs b/users/grfn/xanthous/src/Xanthous/Game/Prompt.hs new file mode 100644 index 000000000000..2d6c0a280f41 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/Prompt.hs @@ -0,0 +1,359 @@ +{-# LANGUAGE DeriveFunctor #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE GADTs #-} +-------------------------------------------------------------------------------- +module Xanthous.Game.Prompt + ( PromptType(..) + , SPromptType(..) + , SingPromptType(..) + , PromptCancellable(..) + , PromptResult(..) + , PromptState(..) + , promptStatePosition + , MenuOption(..) + , mkMenuItems + , PromptInput + , Prompt(..) + , mkPrompt + , mkStringPrompt + , mkStringPromptWithDefault + , mkMenu + , mkPointOnMapPrompt + , mkFirePrompt + , isCancellable + , submitPrompt + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Brick.Widgets.Edit (Editor, editorText, getEditContents) +import Test.QuickCheck +import Test.QuickCheck.Arbitrary.Generic +-------------------------------------------------------------------------------- +import Xanthous.Util (smallestNotIn, AlphaChar (..)) +import Xanthous.Data (Direction, Position, Tiles) +import Xanthous.Data.App (ResourceName) +import qualified Xanthous.Data.App as Resource +-------------------------------------------------------------------------------- + +data PromptType where + StringPrompt :: PromptType + Confirm :: PromptType + Menu :: Type -> PromptType + DirectionPrompt :: PromptType + PointOnMap :: PromptType + -- | Throw an item or fire a projectile weapon. Prompt is to select the + -- direction + Fire :: PromptType + Continue :: PromptType + deriving stock (Generic) + +instance Show PromptType where + show StringPrompt = "StringPrompt" + show Confirm = "Confirm" + show (Menu _) = "Menu" + show DirectionPrompt = "DirectionPrompt" + show PointOnMap = "PointOnMap" + show Continue = "Continue" + show Fire = "Fire" + +data SPromptType :: PromptType -> Type where + SStringPrompt :: SPromptType 'StringPrompt + SConfirm :: SPromptType 'Confirm + SMenu :: SPromptType ('Menu a) + SDirectionPrompt :: SPromptType 'DirectionPrompt + SPointOnMap :: SPromptType 'PointOnMap + SContinue :: SPromptType 'Continue + SFire :: SPromptType 'Fire + +instance NFData (SPromptType pt) where + rnf SStringPrompt = () + rnf SConfirm = () + rnf SMenu = () + rnf SDirectionPrompt = () + rnf SPointOnMap = () + rnf SContinue = () + rnf SFire = () + +class SingPromptType pt where singPromptType :: SPromptType pt +instance SingPromptType 'StringPrompt where singPromptType = SStringPrompt +instance SingPromptType 'Confirm where singPromptType = SConfirm +instance SingPromptType 'DirectionPrompt where singPromptType = SDirectionPrompt +instance SingPromptType 'PointOnMap where singPromptType = SPointOnMap +instance SingPromptType 'Continue where singPromptType = SContinue +instance SingPromptType 'Fire where singPromptType = SFire + +instance Show (SPromptType pt) where + show SStringPrompt = "SStringPrompt" + show SConfirm = "SConfirm" + show SMenu = "SMenu" + show SDirectionPrompt = "SDirectionPrompt" + show SPointOnMap = "SPointOnMap" + show SContinue = "SContinue" + show SFire = "SFire" + +data PromptCancellable + = Cancellable + | Uncancellable + deriving stock (Show, Eq, Ord, Enum, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + +instance Arbitrary PromptCancellable where + arbitrary = genericArbitrary + +data PromptResult (pt :: PromptType) where + StringResult :: Text -> PromptResult 'StringPrompt + ConfirmResult :: Bool -> PromptResult 'Confirm + MenuResult :: forall a. a -> PromptResult ('Menu a) + DirectionResult :: Direction -> PromptResult 'DirectionPrompt + PointOnMapResult :: Position -> PromptResult 'PointOnMap + FireResult :: Position -> PromptResult 'Fire + ContinueResult :: PromptResult 'Continue + +instance Arbitrary (PromptResult 'StringPrompt) where + arbitrary = StringResult <$> arbitrary + +instance Arbitrary (PromptResult 'Confirm) where + arbitrary = ConfirmResult <$> arbitrary + +instance Arbitrary a => Arbitrary (PromptResult ('Menu a)) where + arbitrary = MenuResult <$> arbitrary + +instance Arbitrary (PromptResult 'DirectionPrompt) where + arbitrary = DirectionResult <$> arbitrary + +instance Arbitrary (PromptResult 'PointOnMap) where + arbitrary = PointOnMapResult <$> arbitrary + +instance Arbitrary (PromptResult 'Continue) where + arbitrary = pure ContinueResult + +instance Arbitrary (PromptResult 'Fire) where + arbitrary = FireResult <$> arbitrary + +-------------------------------------------------------------------------------- + +data PromptState pt where + StringPromptState + :: Editor Text ResourceName -> PromptState 'StringPrompt + DirectionPromptState :: PromptState 'DirectionPrompt + ContinuePromptState :: PromptState 'Continue + ConfirmPromptState :: PromptState 'Confirm + MenuPromptState :: forall a. PromptState ('Menu a) + PointOnMapPromptState :: Position -> PromptState 'PointOnMap + FirePromptState :: Position -> PromptState 'Fire + +instance NFData (PromptState pt) where + rnf sps@(StringPromptState ed) = sps `deepseq` ed `deepseq` () + rnf DirectionPromptState = () + rnf ContinuePromptState = () + rnf ConfirmPromptState = () + rnf MenuPromptState = () + rnf pomps@(PointOnMapPromptState pos) = pomps `deepseq` pos `deepseq` () + rnf fps@(FirePromptState pos) = fps `deepseq` pos `deepseq` () + +instance Arbitrary (PromptState 'StringPrompt) where + arbitrary = StringPromptState <$> arbitrary + +instance Arbitrary (PromptState 'DirectionPrompt) where + arbitrary = pure DirectionPromptState + +instance Arbitrary (PromptState 'Continue) where + arbitrary = pure ContinuePromptState + +instance Arbitrary (PromptState ('Menu a)) where + arbitrary = pure MenuPromptState + +instance Arbitrary (PromptState 'Fire) where + arbitrary = FirePromptState <$> arbitrary + +instance CoArbitrary (PromptState 'StringPrompt) where + coarbitrary (StringPromptState ed) = coarbitrary ed + +instance CoArbitrary (PromptState 'DirectionPrompt) where + coarbitrary DirectionPromptState = coarbitrary () + +instance CoArbitrary (PromptState 'Continue) where + coarbitrary ContinuePromptState = coarbitrary () + +instance CoArbitrary (PromptState ('Menu a)) where + coarbitrary MenuPromptState = coarbitrary () + +instance CoArbitrary (PromptState 'Fire) where + coarbitrary (FirePromptState pos) = coarbitrary pos + +deriving stock instance Show (PromptState pt) + +-- | Traversal over the position for the prompt types with positions in their +-- prompt state (currently 'Fire' and 'PointOnMap') +promptStatePosition :: forall pt. Traversal' (PromptState pt) Position +promptStatePosition _ ps@(StringPromptState _) = pure ps +promptStatePosition _ DirectionPromptState = pure DirectionPromptState +promptStatePosition _ ContinuePromptState = pure ContinuePromptState +promptStatePosition _ ConfirmPromptState = pure ConfirmPromptState +promptStatePosition _ MenuPromptState = pure MenuPromptState +promptStatePosition f (PointOnMapPromptState p) = PointOnMapPromptState <$> f p +promptStatePosition f (FirePromptState p) = FirePromptState <$> f p + +data MenuOption a = MenuOption Text a + deriving stock (Eq, Generic, Functor) + deriving anyclass (NFData, CoArbitrary, Function) + +instance Comonad MenuOption where + extract (MenuOption _ x) = x + extend cok mo@(MenuOption text _) = MenuOption text (cok mo) + +mkMenuItems :: (MonoFoldable f, Element f ~ (Char, MenuOption a)) + => f + -> Map Char (MenuOption a) +mkMenuItems = flip foldl' mempty $ \items (chr, option) -> + let chr' = if has (ix chr) items + then getAlphaChar . smallestNotIn . map AlphaChar $ keys items + else chr + in items & at chr' ?~ option + +instance Show (MenuOption a) where + show (MenuOption m _) = show m + +type family PromptInput (pt :: PromptType) :: Type where + PromptInput ('Menu a) = Map Char (MenuOption a) + PromptInput 'PointOnMap = Position -- Character pos + PromptInput 'Fire = (Position, Tiles) -- Nearest enemy, range + PromptInput 'StringPrompt = Maybe Text -- Default value + PromptInput _ = () + +data Prompt (m :: Type -> Type) where + Prompt + :: forall (pt :: PromptType) + (m :: Type -> Type). + PromptCancellable + -> SPromptType pt + -> PromptState pt + -> PromptInput pt + -> (PromptResult pt -> m ()) + -> Prompt m + +instance Show (Prompt m) where + show (Prompt c pt ps pri _) + = "(Prompt " + <> show c <> " " + <> show pt <> " " + <> show ps <> " " + <> showPri + <> " <function>)" + where showPri = case pt of + SMenu -> show pri + _ -> "()" + +instance NFData (Prompt m) where + rnf (Prompt c SMenu ps pri cb) + = c + `deepseq` ps + `deepseq` pri + `seq` cb + `seq` () + rnf (Prompt c spt ps pri cb) + = c + `deepseq` spt + `deepseq` ps + `deepseq` pri + `seq` cb + `seq` () + +instance CoArbitrary (m ()) => CoArbitrary (Prompt m) where + coarbitrary (Prompt c SStringPrompt ps pri cb) = + variant @Int 1 . coarbitrary (c, ps, pri, cb) + coarbitrary (Prompt c SConfirm _ pri cb) = -- TODO fill in prompt state + variant @Int 2 . coarbitrary (c, pri, cb) + coarbitrary (Prompt c SMenu _ps _pri _cb) = + variant @Int 3 . coarbitrary c {-, ps, pri, cb -} + coarbitrary (Prompt c SDirectionPrompt ps pri cb) = + variant @Int 4 . coarbitrary (c, ps, pri, cb) + coarbitrary (Prompt c SPointOnMap _ pri cb) = -- TODO fill in prompt state + variant @Int 5 . coarbitrary (c, pri, cb) + coarbitrary (Prompt c SContinue ps pri cb) = + variant @Int 6 . coarbitrary (c, ps, pri, cb) + coarbitrary (Prompt c SFire ps pri cb) = + variant @Int 7 . coarbitrary (c, ps, pri, cb) + +-- instance Function (Prompt m) where +-- function = functionMap toTuple _fromTuple +-- where +-- toTuple (Prompt c pt ps pri cb) = (c, pt, ps, pri, cb) + + +mkPrompt + :: (PromptInput pt ~ ()) + => PromptCancellable -- ^ Is the prompt cancellable or not? + -> SPromptType pt -- ^ The type of the prompt + -> (PromptResult pt -> m ()) -- ^ Function to call when the prompt is complete + -> Prompt m +mkPrompt c pt@SDirectionPrompt cb = Prompt c pt DirectionPromptState () cb +mkPrompt c pt@SContinue cb = Prompt c pt ContinuePromptState () cb +mkPrompt c pt@SConfirm cb = Prompt c pt ConfirmPromptState () cb + +mkStringPrompt + :: PromptCancellable -- ^ Is the prompt cancellable or not? + -> (PromptResult 'StringPrompt -> m ()) -- ^ Function to call when the prompt is complete + -> Prompt m +mkStringPrompt c = + let ps = StringPromptState $ editorText Resource.Prompt (Just 1) "" + in Prompt c SStringPrompt ps Nothing + +mkStringPromptWithDefault + :: PromptCancellable -- ^ Is the prompt cancellable or not? + -> Text -- ^ Default value for the prompt + -> (PromptResult 'StringPrompt -> m ()) -- ^ Function to call when the prompt is complete + -> Prompt m +mkStringPromptWithDefault c def = + let ps = StringPromptState $ editorText Resource.Prompt (Just 1) "" + in Prompt c SStringPrompt ps (Just def) + +mkMenu + :: forall a m. + PromptCancellable + -> Map Char (MenuOption a) -- ^ Menu items + -> (PromptResult ('Menu a) -> m ()) + -> Prompt m +mkMenu c = Prompt c SMenu MenuPromptState + +mkPointOnMapPrompt + :: PromptCancellable + -> Position + -> (PromptResult 'PointOnMap -> m ()) + -> Prompt m +mkPointOnMapPrompt c pos = Prompt c SPointOnMap (PointOnMapPromptState pos) pos + +mkFirePrompt + :: PromptCancellable + -> Position -- ^ Initial position + -> Tiles -- ^ Range + -> (PromptResult 'Fire -> m ()) + -> Prompt m +mkFirePrompt c pos range = Prompt c SFire (FirePromptState pos) (pos, range) + +isCancellable :: Prompt m -> Bool +isCancellable (Prompt Cancellable _ _ _ _) = True +isCancellable (Prompt Uncancellable _ _ _ _) = False + +submitPrompt :: Applicative m => Prompt m -> m () +submitPrompt (Prompt _ pt ps pri cb) = + case (pt, ps, pri) of + (SStringPrompt, StringPromptState edit, mDef) -> + let inputVal = mconcat . getEditContents $ edit + val | null inputVal, Just def <- mDef = def + | otherwise = inputVal + in cb $ StringResult val + (SDirectionPrompt, DirectionPromptState, _) -> + pure () -- Don't use submit with a direction prompt + (SContinue, ContinuePromptState, _) -> + cb ContinueResult + (SMenu, MenuPromptState, _) -> + pure () -- Don't use submit with a menu prompt + (SPointOnMap, PointOnMapPromptState pos, _) -> + cb $ PointOnMapResult pos + (SConfirm, ConfirmPromptState, _) -> + cb $ ConfirmResult True + (SFire, FirePromptState pos, _) -> + cb $ FireResult pos diff --git a/users/grfn/xanthous/src/Xanthous/Game/State.hs b/users/grfn/xanthous/src/Xanthous/Game/State.hs new file mode 100644 index 000000000000..3025eb15be5a --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Game/State.hs @@ -0,0 +1,573 @@ +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE AllowAmbiguousTypes #-} +-------------------------------------------------------------------------------- +module Xanthous.Game.State + ( GameState(..) + , entities + , levels + , revealedPositions + , messageHistory + , randomGen + , activePanel + , promptState + , characterEntityID + , autocommand + , savefile + , memo + , GamePromptState(..) + + -- * Game Level + , GameLevel(..) + , levelEntities + , upStaircasePosition + , levelRevealedPositions + + -- * Messages + , MessageHistory(..) + , HasMessages(..) + , HasTurn(..) + , HasDisplayedTurn(..) + , pushMessage + , previousMessage + , nextTurn + + -- * Autocommands + , Autocommand(..) + , AutocommandState(..) + , _NoAutocommand + , _ActiveAutocommand + + -- * App monad + , AppT(..) + , AppM + , runAppT + + -- * Entities + , Draw(..) + , Brain(..) + , Brainless(..) + , brainVia + , Collision(..) + , Entity(..) + , SomeEntity(..) + , downcastEntity + , _SomeEntity + , entityIs + , entityTypeName + + -- ** Vias + , Color(..) + , DrawNothing(..) + , DrawRawChar(..) + , DrawRawCharPriority(..) + , DrawCharacter(..) + , DrawStyledCharacter(..) + , DeriveEntity(..) + -- ** Field classes + , HasChar(..) + , HasStyle(..) + + -- * Debug State + , DebugState(..) + , debugState + , allRevealed + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.List.NonEmpty ( NonEmpty((:|))) +import qualified Data.List.NonEmpty as NonEmpty +import Data.Typeable +import Data.Coerce +import System.Random +import Test.QuickCheck +import Test.QuickCheck.Arbitrary.Generic +import Control.Monad.Random.Class +import Control.Monad.State +import Control.Monad.Trans.Control (MonadTransControl(..)) +import Control.Monad.Trans.Compose +import Control.Monad.Morph (MFunctor(..)) +import Brick (EventM, Widget, raw, str, emptyWidget) +import Data.Aeson (ToJSON(..), FromJSON(..), Value(Null)) +import qualified Data.Aeson as JSON +import Data.Aeson.Generic.DerivingVia +import Data.Generics.Product.Fields +import qualified Graphics.Vty.Attributes as Vty +import qualified Graphics.Vty.Image as Vty +-------------------------------------------------------------------------------- +import Xanthous.Util (KnownBool(..)) +import Xanthous.Util.QuickCheck (GenericArbitrary(..)) +import Xanthous.Data +import Xanthous.Data.App +import Xanthous.Data.Levels +import Xanthous.Data.EntityMap (EntityMap, EntityID) +import Xanthous.Data.EntityChar +import Xanthous.Data.VectorBag +import Xanthous.Data.Entities +import Xanthous.Orphans () +import Xanthous.Game.Prompt +import Xanthous.Game.Env +import Xanthous.Game.Memo (MemoState) +-------------------------------------------------------------------------------- + +data MessageHistory + = MessageHistory + { _messages :: Map Word (NonEmpty Text) + , _turn :: Word + , _displayedTurn :: Maybe Word + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary MessageHistory + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + MessageHistory +makeFieldsNoPrefix ''MessageHistory + +instance Semigroup MessageHistory where + (MessageHistory msgs₁ turn₁ dt₁) <> (MessageHistory msgs₂ turn₂ dt₂) = + MessageHistory (msgs₁ <> msgs₂) (max turn₁ turn₂) $ case (dt₁, dt₂) of + (_, Nothing) -> Nothing + (Just t, _) -> Just t + (Nothing, Just t) -> Just t + +instance Monoid MessageHistory where + mempty = MessageHistory mempty 0 Nothing + +type instance Element MessageHistory = [Text] +instance MonoFunctor MessageHistory where + omap f mh@(MessageHistory _ t _) = + mh & messages . at t %~ (NonEmpty.nonEmpty . f . toList =<<) + +instance MonoComonad MessageHistory where + oextract (MessageHistory ms t dt) = maybe [] toList $ ms ^. at (fromMaybe t dt) + oextend cok mh@(MessageHistory _ t dt) = + mh & messages . at (fromMaybe t dt) .~ NonEmpty.nonEmpty (cok mh) + +pushMessage :: Text -> MessageHistory -> MessageHistory +pushMessage msg mh@(MessageHistory _ turn' _) = + mh + & messages . at turn' %~ \case + Nothing -> Just $ msg :| mempty + Just msgs -> Just $ msg <| msgs + & displayedTurn .~ Nothing + +nextTurn :: MessageHistory -> MessageHistory +nextTurn = (turn +~ 1) . (displayedTurn .~ Nothing) + +previousMessage :: MessageHistory -> MessageHistory +previousMessage mh = mh & displayedTurn .~ maximumOf + (messages . ifolded . asIndex . filtered (< mh ^. turn)) + mh + + +-------------------------------------------------------------------------------- + +data GamePromptState m where + NoPrompt :: GamePromptState m + WaitingPrompt :: Text -> Prompt m -> GamePromptState m + deriving stock (Show, Generic) + deriving anyclass (NFData) + +-- | Non-injective! We never try to serialize waiting prompts, since: +-- +-- * they contain callback functions +-- * we can't save the game when in a prompt anyway +instance ToJSON (GamePromptState m) where + toJSON _ = Null + +-- | Always expects Null +instance FromJSON (GamePromptState m) where + parseJSON Null = pure NoPrompt + parseJSON _ = fail "Invalid GamePromptState; expected null" + +instance CoArbitrary (GamePromptState m) where + coarbitrary NoPrompt = variant @Int 1 + coarbitrary (WaitingPrompt txt _) = variant @Int 2 . coarbitrary txt + +instance Function (GamePromptState m) where + function = functionMap onlyNoPrompt (const NoPrompt) + where + onlyNoPrompt NoPrompt = () + onlyNoPrompt (WaitingPrompt _ _) = + error "Can't handle prompts in Function!" + +-------------------------------------------------------------------------------- + +newtype AppT m a + = AppT { unAppT :: ReaderT GameEnv (StateT GameState m) a } + deriving ( Functor + , Applicative + , Monad + , MonadState GameState + , MonadReader GameEnv + , MonadIO + ) + via (ReaderT GameEnv (StateT GameState m)) + deriving ( MonadTrans + , MFunctor + ) + via (ReaderT GameEnv `ComposeT` StateT GameState) + +type AppM = AppT (EventM ResourceName) + +-------------------------------------------------------------------------------- + +class Draw a where + drawWithNeighbors :: Neighbors (VectorBag SomeEntity) -> a -> Widget n + drawWithNeighbors = const draw + + draw :: a -> Widget n + draw = drawWithNeighbors $ pure mempty + + -- | higher priority gets drawn on top + drawPriority :: a -> Word + drawPriority = const minBound + +instance Draw a => Draw (Positioned a) where + drawWithNeighbors ns (Positioned _ a) = drawWithNeighbors ns a + draw (Positioned _ a) = draw a + +newtype DrawCharacter (char :: Symbol) (a :: Type) where + DrawCharacter :: a -> DrawCharacter char a + +instance KnownSymbol char => Draw (DrawCharacter char a) where + draw _ = str $ symbolVal @char Proxy + +data Color = Black | Red | Green | Yellow | Blue | Magenta | Cyan | White + +class KnownColor (color :: Color) where + colorVal :: forall proxy. proxy color -> Vty.Color + +instance KnownColor 'Black where colorVal _ = Vty.black +instance KnownColor 'Red where colorVal _ = Vty.red +instance KnownColor 'Green where colorVal _ = Vty.green +instance KnownColor 'Yellow where colorVal _ = Vty.yellow +instance KnownColor 'Blue where colorVal _ = Vty.blue +instance KnownColor 'Magenta where colorVal _ = Vty.magenta +instance KnownColor 'Cyan where colorVal _ = Vty.cyan +instance KnownColor 'White where colorVal _ = Vty.white + +class KnownMaybeColor (maybeColor :: Maybe Color) where + maybeColorVal :: forall proxy. proxy maybeColor -> Maybe Vty.Color + +instance KnownMaybeColor 'Nothing where maybeColorVal _ = Nothing +instance KnownColor color => KnownMaybeColor ('Just color) where + maybeColorVal _ = Just $ colorVal @color Proxy + +newtype DrawStyledCharacter (fg :: Maybe Color) (bg :: Maybe Color) (char :: Symbol) (a :: Type) where + DrawStyledCharacter :: a -> DrawStyledCharacter fg bg char a + +instance + ( KnownMaybeColor fg + , KnownMaybeColor bg + , KnownSymbol char + ) + => Draw (DrawStyledCharacter fg bg char a) where + draw _ = raw $ Vty.string attr $ symbolVal @char Proxy + where attr = Vty.Attr + { Vty.attrStyle = Vty.Default + , Vty.attrForeColor = maybe Vty.Default Vty.SetTo + $ maybeColorVal @fg Proxy + , Vty.attrBackColor = maybe Vty.Default Vty.SetTo + $ maybeColorVal @bg Proxy + , Vty.attrURL = Vty.Default + } + +instance Draw EntityChar where + draw EntityChar{..} = raw $ Vty.string _style [_char] + +-------------------------------------------------------------------------------- + +newtype DrawNothing (a :: Type) = DrawNothing a + +instance Draw (DrawNothing a) where + draw = const emptyWidget + drawPriority = const 0 + +newtype DrawRawChar (rawField :: Symbol) (a :: Type) = DrawRawChar a + +instance + forall rawField a raw. + ( HasField rawField a a raw raw + , HasChar raw EntityChar + ) => Draw (DrawRawChar rawField a) where + draw (DrawRawChar e) = draw $ e ^. field @rawField . char + +newtype DrawRawCharPriority + (rawField :: Symbol) + (priority :: Nat) + (a :: Type) + = DrawRawCharPriority a + +instance + forall rawField priority a raw. + ( HasField rawField a a raw raw + , KnownNat priority + , HasChar raw EntityChar + ) => Draw (DrawRawCharPriority rawField priority a) where + draw (DrawRawCharPriority e) = draw $ e ^. field @rawField . char + drawPriority = const . fromIntegral $ natVal @priority Proxy + + +-------------------------------------------------------------------------------- + +class Brain a where + step :: Ticks -> Positioned a -> AppM (Positioned a) + -- | Does this entity ever move on its own? + entityCanMove :: a -> Bool + entityCanMove = const False + +newtype Brainless a = Brainless a + +instance Brain (Brainless a) where + step = const pure + +-- | Workaround for the inability to use DerivingVia on Brain due to the lack of +-- higher-order roles (specifically AppT not having its last type argument have +-- role representational bc of StateT) +brainVia + :: forall brain entity. (Coercible entity brain, Brain brain) + => (entity -> brain) -- ^ constructor, ignored + -> (Ticks -> Positioned entity -> AppM (Positioned entity)) +brainVia _ ticks = fmap coerce . step ticks . coerce @_ @(Positioned brain) + +-------------------------------------------------------------------------------- + +class ( Show a, Eq a, Ord a, NFData a + , ToJSON a, FromJSON a + , Draw a, Brain a + ) => Entity a where + entityAttributes :: a -> EntityAttributes + entityAttributes = const defaultEntityAttributes + description :: a -> Text + entityChar :: a -> EntityChar + entityCollision :: a -> Maybe Collision + entityCollision = const $ Just Stop + +data SomeEntity where + SomeEntity :: forall a. (Entity a, Typeable a) => a -> SomeEntity + +instance Show SomeEntity where + show (SomeEntity e) = "SomeEntity (" <> show e <> ")" + +instance Eq SomeEntity where + (SomeEntity (a :: ea)) == (SomeEntity (b :: eb)) = case eqT @ea @eb of + Just Refl -> a == b + _ -> False + +instance Ord SomeEntity where + compare (SomeEntity (a :: ea)) (SomeEntity (b :: eb)) = case eqT @ea @eb of + Just Refl -> compare a b + _ -> compare (typeRep $ Proxy @ea) (typeRep $ Proxy @eb) + + +instance NFData SomeEntity where + rnf (SomeEntity ent) = ent `deepseq` () + +instance ToJSON SomeEntity where + toJSON (SomeEntity ent) = entityToJSON ent + where + entityToJSON :: forall entity. (Entity entity, Typeable entity) + => entity -> JSON.Value + entityToJSON entity = JSON.object + [ "type" JSON..= tshow (typeRep @_ @entity Proxy) + , "data" JSON..= toJSON entity + ] + +instance Draw SomeEntity where + drawWithNeighbors ns (SomeEntity ent) = drawWithNeighbors ns ent + drawPriority (SomeEntity ent) = drawPriority ent + +instance Brain SomeEntity where + step ticks (Positioned p (SomeEntity ent)) = + fmap SomeEntity <$> step ticks (Positioned p ent) + entityCanMove (SomeEntity ent) = entityCanMove ent + +downcastEntity :: forall (a :: Type). (Typeable a) => SomeEntity -> Maybe a +downcastEntity (SomeEntity e) = cast e + +entityIs :: forall (a :: Type). (Typeable a) => SomeEntity -> Bool +entityIs = isJust . downcastEntity @a + +_SomeEntity :: forall a. (Entity a, Typeable a) => Prism' SomeEntity a +_SomeEntity = prism' SomeEntity downcastEntity + +-- | Get the name of the type of 'SomeEntity' as a string +entityTypeName :: SomeEntity -> Text +entityTypeName (SomeEntity e) = pack . tyConName . typeRepTyCon $ typeOf e + +newtype DeriveEntity + (blocksVision :: Bool) + (description :: Symbol) + (entityChar :: Symbol) + (entity :: Type) + = DeriveEntity entity + deriving newtype (Show, Eq, Ord, NFData, ToJSON, FromJSON, Draw) + +instance Brain entity => Brain (DeriveEntity b d c entity) where + step = brainVia $ \(DeriveEntity e) -> e + +instance + ( KnownBool blocksVision + , KnownSymbol description + , KnownSymbol entityChar + , Show entity, Eq entity, Ord entity, NFData entity + , ToJSON entity, FromJSON entity + , Draw entity, Brain entity + ) + => Entity (DeriveEntity blocksVision description entityChar entity) where + entityAttributes _ = defaultEntityAttributes + & blocksVision .~ boolVal @blocksVision + description _ = pack . symbolVal $ Proxy @description + entityChar _ = fromString . symbolVal $ Proxy @entityChar + +-------------------------------------------------------------------------------- + +data GameLevel = GameLevel + { _levelEntities :: !(EntityMap SomeEntity) + , _upStaircasePosition :: !Position + , _levelRevealedPositions :: !(Set Position) + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) + deriving (ToJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + GameLevel + +-------------------------------------------------------------------------------- + +data Autocommand + = AutoMove Direction + | AutoRest + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (NFData, Hashable, ToJSON, FromJSON, CoArbitrary, Function) + deriving Arbitrary via GenericArbitrary Autocommand +{-# ANN module ("HLint: ignore Use newtype instead of data" :: String) #-} + +data AutocommandState + = NoAutocommand + | ActiveAutocommand Autocommand (Async ()) + deriving stock (Eq, Ord, Generic) + deriving anyclass (Hashable) + +instance Show AutocommandState where + show NoAutocommand = "NoAutocommand" + show (ActiveAutocommand ac _) = + "(ActiveAutocommand " <> show ac <> " <Async>)" + +instance ToJSON AutocommandState where + toJSON = const Null + +instance FromJSON AutocommandState where + parseJSON Null = pure NoAutocommand + parseJSON _ = fail "Invalid AutocommandState; expected null" + +instance NFData AutocommandState where + rnf NoAutocommand = () + rnf (ActiveAutocommand ac t) = ac `deepseq` t `seq` () + +instance CoArbitrary AutocommandState where + coarbitrary NoAutocommand = variant @Int 1 + coarbitrary (ActiveAutocommand ac t) + = variant @Int 2 + . coarbitrary ac + . coarbitrary (hash t) + +instance Function AutocommandState where + function = functionMap onlyNoAC (const NoAutocommand) + where + onlyNoAC NoAutocommand = () + onlyNoAC _ = error "Can't handle autocommands in Function" + +-------------------------------------------------------------------------------- + + +data DebugState = DebugState + { _allRevealed :: !Bool + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving (ToJSON, FromJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + DebugState +{-# ANN DebugState ("HLint: ignore Use newtype instead of data" :: String) #-} + +instance Arbitrary DebugState where + arbitrary = genericArbitrary + +data GameState = GameState + { _levels :: !(Levels GameLevel) + , _characterEntityID :: !EntityID + , _messageHistory :: !MessageHistory + , _randomGen :: !StdGen + + -- | The active panel displayed in the UI, if any + , _activePanel :: !(Maybe Panel) + + , _promptState :: !(GamePromptState AppM) + , _debugState :: !DebugState + , _autocommand :: !AutocommandState + + -- | The path to the savefile that was loaded for this game, if any + , _savefile :: !(Maybe FilePath) + + , _memo :: MemoState + } + deriving stock (Show, Generic) + deriving anyclass (NFData) + deriving (ToJSON) + via WithOptions '[ FieldLabelModifier '[Drop 1] ] + GameState + +makeLenses ''GameLevel +makeLenses ''GameState + +entities :: Lens' GameState (EntityMap SomeEntity) +entities = levels . current . levelEntities + +revealedPositions :: Lens' GameState (Set Position) +revealedPositions = levels . current . levelRevealedPositions + +instance Eq GameState where + (==) = (==) `on` \gs -> + ( gs ^. entities + , gs ^. revealedPositions + , gs ^. characterEntityID + , gs ^. messageHistory + , gs ^. activePanel + , gs ^. debugState + ) + +-------------------------------------------------------------------------------- + +runAppT :: Monad m => AppT m a -> GameEnv -> GameState -> m (a, GameState) +runAppT appt env initialState + = flip runStateT initialState + . flip runReaderT env + . unAppT + $ appt + +instance (Monad m) => MonadRandom (AppT m) where + getRandomR rng = randomGen %%= randomR rng + getRandom = randomGen %%= random + getRandomRs rng = uses randomGen $ randomRs rng + getRandoms = uses randomGen randoms + +instance MonadTransControl AppT where + type StT AppT a = (a, GameState) + liftWith f + = AppT + . ReaderT $ \e + -> StateT $ \s + -> (,s) <$> f (\action -> runAppT action e s) + restoreT = AppT . ReaderT . const . StateT . const + +-------------------------------------------------------------------------------- + +makeLenses ''DebugState +makePrisms ''AutocommandState diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Level.hs b/users/grfn/xanthous/src/Xanthous/Generators/Level.hs new file mode 100644 index 000000000000..fc57402e7d8e --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Level.hs @@ -0,0 +1,172 @@ +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE GADTs #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level + ( generate + , Generator(..) + , SGenerator(..) + , GeneratorInput(..) + , generateFromInput + , parseGeneratorInput + , showCells + , Level(..) + , levelWalls + , levelItems + , levelCreatures + , levelDoors + , levelCharacterPosition + , levelTutorialMessage + , levelExtra + , generateLevel + , levelToEntityMap + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Data.Array.Unboxed +import qualified Options.Applicative as Opt +import Control.Monad.Random +-------------------------------------------------------------------------------- +import qualified Xanthous.Generators.Level.CaveAutomata as CaveAutomata +import qualified Xanthous.Generators.Level.Dungeon as Dungeon +import Xanthous.Generators.Level.Util +import Xanthous.Generators.Level.LevelContents +import Xanthous.Generators.Level.Village as Village +import Xanthous.Data (Dimensions, Position'(Position), Position) +import Xanthous.Data.EntityMap (EntityMap, _EntityMap) +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Entities.Environment +import Xanthous.Entities.Item (Item) +import Xanthous.Entities.Creature (Creature) +import Xanthous.Game.State (SomeEntity(..)) +import Linear.V2 +-------------------------------------------------------------------------------- + +data Generator + = CaveAutomata + | Dungeon + deriving stock (Show, Eq) + +data SGenerator (gen :: Generator) where + SCaveAutomata :: SGenerator 'CaveAutomata + SDungeon :: SGenerator 'Dungeon + +type family Params (gen :: Generator) :: Type where + Params 'CaveAutomata = CaveAutomata.Params + Params 'Dungeon = Dungeon.Params + +generate + :: RandomGen g + => SGenerator gen + -> Params gen + -> Dimensions + -> g + -> Cells +generate SCaveAutomata = CaveAutomata.generate +generate SDungeon = Dungeon.generate + +data GeneratorInput where + GeneratorInput :: forall gen. SGenerator gen -> Params gen -> GeneratorInput + +generateFromInput :: RandomGen g => GeneratorInput -> Dimensions -> g -> Cells +generateFromInput (GeneratorInput sg ps) = generate sg ps + +parseGeneratorInput :: Opt.Parser GeneratorInput +parseGeneratorInput = Opt.subparser + $ generatorCommand SCaveAutomata + "cave" + "Cellular-automata based cave generator" + CaveAutomata.parseParams + <> generatorCommand SDungeon + "dungeon" + "Classic dungeon map generator" + Dungeon.parseParams + where + generatorCommand sgen name desc parseParams = + Opt.command name + (Opt.info + (GeneratorInput sgen <$> parseParams) + (Opt.progDesc desc) + ) + + +showCells :: Cells -> Text +showCells arr = + let (V2 minX minY, V2 maxX maxY) = bounds arr + showCellVal True = "x" + showCellVal False = " " + showCell = showCellVal . (arr !) + row r = foldMap (showCell . (`V2` r)) [minX..maxX] + rows = row <$> [minY..maxY] + in intercalate "\n" rows + +cellsToWalls :: Cells -> EntityMap Wall +cellsToWalls cells = foldl' maybeInsertWall mempty . assocs $ cells + where + maybeInsertWall em (pos@(V2 x y), True) + | not (surroundedOnAllSides pos) = + let x' = fromIntegral x + y' = fromIntegral y + in EntityMap.insertAt (Position x' y') Wall em + maybeInsertWall em _ = em + surroundedOnAllSides pos = numAliveNeighbors cells pos == 8 + +-------------------------------------------------------------------------------- + +data Level = Level + { _levelWalls :: !(EntityMap Wall) + , _levelDoors :: !(EntityMap Door) + , _levelItems :: !(EntityMap Item) + , _levelCreatures :: !(EntityMap Creature) + , _levelTutorialMessage :: !(EntityMap GroundMessage) + , _levelStaircases :: !(EntityMap Staircase) + , _levelExtra :: !(EntityMap SomeEntity) -- ^ TODO this is a bit of a hack... + , _levelCharacterPosition :: !Position + } + deriving stock (Generic) + deriving anyclass (NFData) +makeLenses ''Level + +generateLevel + :: MonadRandom m + => SGenerator gen + -> Params gen + -> Dimensions + -> Word -- ^ Level number, starting at 0 + -> m Level +generateLevel gen ps dims num = do + rand <- mkStdGen <$> getRandom + let cells = generate gen ps dims rand + _levelWalls = cellsToWalls cells + village <- generateVillage cells gen + let _levelExtra = village + _levelItems <- randomItems cells + _levelCreatures <- randomCreatures num cells + _levelDoors <- randomDoors cells + _levelCharacterPosition <- chooseCharacterPosition cells + let upStaircase = _EntityMap # [(_levelCharacterPosition, UpStaircase)] + downStaircase <- placeDownStaircase cells + let _levelStaircases = upStaircase <> downStaircase + _levelTutorialMessage <- + if num == 0 + then tutorialMessage cells _levelCharacterPosition + else pure mempty + pure Level {..} + +levelToEntityMap :: Level -> EntityMap SomeEntity +levelToEntityMap level + = (SomeEntity <$> level ^. levelWalls) + <> (SomeEntity <$> level ^. levelDoors) + <> (SomeEntity <$> level ^. levelItems) + <> (SomeEntity <$> level ^. levelCreatures) + <> (SomeEntity <$> level ^. levelTutorialMessage) + <> (SomeEntity <$> level ^. levelStaircases) + <> (level ^. levelExtra) + +generateVillage + :: MonadRandom m + => Cells -- ^ Wall positions + -> SGenerator gen + -> m (EntityMap SomeEntity) +generateVillage wallPositions SCaveAutomata = Village.fromCave wallPositions +generateVillage _ _ = pure mempty diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Level/CaveAutomata.hs b/users/grfn/xanthous/src/Xanthous/Generators/Level/CaveAutomata.hs new file mode 100644 index 000000000000..03d534ca39b3 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Level/CaveAutomata.hs @@ -0,0 +1,112 @@ +{-# LANGUAGE MultiWayIf #-} +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level.CaveAutomata + ( Params(..) + , defaultParams + , parseParams + , generate + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Control.Monad.Random (RandomGen, runRandT) +import Data.Array.ST +import Data.Array.Unboxed +import qualified Options.Applicative as Opt +-------------------------------------------------------------------------------- +import Xanthous.Util (between) +import Xanthous.Util.Optparse +import Xanthous.Data (Dimensions, width, height) +import Xanthous.Generators.Level.Util +import Linear.V2 +-------------------------------------------------------------------------------- + +data Params = Params + { _aliveStartChance :: Double + , _birthLimit :: Word + , _deathLimit :: Word + , _steps :: Word + } + deriving stock (Show, Eq, Generic) +makeLenses ''Params + +defaultParams :: Params +defaultParams = Params + { _aliveStartChance = 0.6 + , _birthLimit = 3 + , _deathLimit = 4 + , _steps = 4 + } + +parseParams :: Opt.Parser Params +parseParams = Params + <$> Opt.option parseChance + ( Opt.long "alive-start-chance" + <> Opt.value (defaultParams ^. aliveStartChance) + <> Opt.showDefault + <> Opt.help ( "Chance for each cell to start alive at the beginning of " + <> "the cellular automata" + ) + <> Opt.metavar "CHANCE" + ) + <*> Opt.option parseNeighbors + ( Opt.long "birth-limit" + <> Opt.value (defaultParams ^. birthLimit) + <> Opt.showDefault + <> Opt.help "Minimum neighbor count required for birth of a cell" + <> Opt.metavar "NEIGHBORS" + ) + <*> Opt.option parseNeighbors + ( Opt.long "death-limit" + <> Opt.value (defaultParams ^. deathLimit) + <> Opt.showDefault + <> Opt.help "Maximum neighbor count required for death of a cell" + <> Opt.metavar "NEIGHBORS" + ) + <*> Opt.option Opt.auto + ( Opt.long "steps" + <> Opt.value (defaultParams ^. steps) + <> Opt.showDefault + <> Opt.help "Number of generations to run the automata for" + <> Opt.metavar "STEPS" + ) + <**> Opt.helper + where + parseChance = readWithGuard + (between 0 1) + $ \res -> "Chance must be in the range [0,1], got: " <> show res + + parseNeighbors = readWithGuard + (between 0 8) + $ \res -> "Neighbors must be in the range [0,8], got: " <> show res + +generate :: RandomGen g => Params -> Dimensions -> g -> Cells +generate params dims gen + = runSTUArray + $ fmap fst + $ flip runRandT gen + $ generate' params dims + +generate' :: RandomGen g => Params -> Dimensions -> CellM g s (MCells s) +generate' params dims = do + cells <- randInitialize dims $ params ^. aliveStartChance + let steps' = params ^. steps + when (steps' > 0) + $ for_ [0 .. pred steps'] . const $ stepAutomata cells dims params + -- Remove all but the largest contiguous region of unfilled space + (_: smallerRegions) <- lift $ regions @UArray . amap not <$> freeze cells + lift $ fillAllM (fold smallerRegions) cells + lift $ fillOuterEdgesM cells + pure cells + +stepAutomata :: forall s g. MCells s -> Dimensions -> Params -> CellM g s () +stepAutomata cells dims params = do + origCells <- lift $ cloneMArray @_ @(STUArray s) cells + for_ (range (0, V2 (dims ^. width) (dims ^. height))) $ \pos -> do + neighs <- lift $ numAliveNeighborsM origCells pos + origValue <- lift $ readArray origCells pos + lift . writeArray cells pos + $ if origValue + then neighs >= params ^. deathLimit + else neighs > params ^. birthLimit diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Level/Dungeon.hs b/users/grfn/xanthous/src/Xanthous/Generators/Level/Dungeon.hs new file mode 100644 index 000000000000..0be7c0435c5a --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Level/Dungeon.hs @@ -0,0 +1,190 @@ +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level.Dungeon + ( Params(..) + , defaultParams + , parseParams + , generate + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding ((:>)) +-------------------------------------------------------------------------------- +import Control.Monad.Random +import Data.Array.ST +import Data.Array.IArray (amap) +import Data.Stream.Infinite (Stream(..)) +import qualified Data.Stream.Infinite as Stream +import qualified Data.Graph.Inductive.Graph as Graph +import Data.Graph.Inductive.PatriciaTree +import qualified Data.List.NonEmpty as NE +import Data.Maybe (fromJust) +import Linear.V2 +import Linear.Metric +import qualified Options.Applicative as Opt +-------------------------------------------------------------------------------- +import Xanthous.Random +import Xanthous.Data hiding (x, y, _x, _y, edges, distance) +import Xanthous.Generators.Level.Util +import Xanthous.Util.Graphics (delaunay, straightLine) +import Xanthous.Util.Graph (mstSubGraph) +-------------------------------------------------------------------------------- + +data Params = Params + { _numRoomsRange :: (Word, Word) + , _roomDimensionRange :: (Word, Word) + , _connectednessRatioRange :: (Double, Double) + } + deriving stock (Show, Eq, Ord, Generic) +makeLenses ''Params + +defaultParams :: Params +defaultParams = Params + { _numRoomsRange = (6, 8) + , _roomDimensionRange = (3, 12) + , _connectednessRatioRange = (0.1, 0.15) + } + +parseParams :: Opt.Parser Params +parseParams = Params + <$> parseRange + "num-rooms" + "number of rooms to generate in the dungeon" + "ROOMS" + (defaultParams ^. numRoomsRange) + <*> parseRange + "room-size" + "size in tiles of one of the sides of a room" + "TILES" + (defaultParams ^. roomDimensionRange) + <*> parseRange + "connectedness-ratio" + ( "ratio of edges from the delaunay triangulation to re-add to the " + <> "minimum-spanning-tree") + "RATIO" + (defaultParams ^. connectednessRatioRange) + <**> Opt.helper + where + parseRange name desc metavar (defMin, defMax) = + (,) + <$> Opt.option Opt.auto + ( Opt.long ("min-" <> name) + <> Opt.value defMin + <> Opt.showDefault + <> Opt.help ("Minimum " <> desc) + <> Opt.metavar metavar + ) + <*> Opt.option Opt.auto + ( Opt.long ("max-" <> name) + <> Opt.value defMax + <> Opt.showDefault + <> Opt.help ("Maximum " <> desc) + <> Opt.metavar metavar + ) + +generate :: RandomGen g => Params -> Dimensions -> g -> Cells +generate params dims gen + = amap not + $ runSTUArray + $ fmap fst + $ flip runRandT gen + $ generate' params dims + +-------------------------------------------------------------------------------- + +generate' :: RandomGen g => Params -> Dimensions -> CellM g s (MCells s) +generate' params dims = do + cells <- initializeEmpty dims + rooms <- genRooms params dims + for_ rooms $ fillRoom cells + + let fullRoomGraph = delaunayRoomGraph rooms + mst = mstSubGraph fullRoomGraph + mstEdges = Graph.edges mst + nonMSTEdges = filter (\(n₁, n₂, _) -> (n₁, n₂) `notElem` mstEdges) + $ Graph.labEdges fullRoomGraph + + reintroEdgeCount <- floor . (* fromIntegral (length nonMSTEdges)) + <$> getRandomR (params ^. connectednessRatioRange) + let reintroEdges = take reintroEdgeCount nonMSTEdges + corridorGraph = Graph.insEdges reintroEdges mst + + corridors <- traverse + ( uncurry corridorBetween + . over both (fromJust . Graph.lab corridorGraph) + ) $ Graph.edges corridorGraph + + for_ (join corridors) $ \pt -> lift $ writeArray cells pt True + + pure cells + +type Room = Box Word + +genRooms :: MonadRandom m => Params -> Dimensions -> m [Room] +genRooms params dims = do + numRooms <- fromIntegral <$> getRandomR (params ^. numRoomsRange) + subRand . fmap (Stream.take numRooms . removeIntersecting []) . infinitely $ do + roomWidth <- getRandomR $ params ^. roomDimensionRange + roomHeight <- getRandomR $ params ^. roomDimensionRange + xPos <- getRandomR (0, dims ^. width - roomWidth) + yPos <- getRandomR (0, dims ^. height - roomHeight) + pure Box + { _topLeftCorner = V2 xPos yPos + , _dimensions = V2 roomWidth roomHeight + } + where + removeIntersecting seen (room :> rooms) + | any (boxIntersects room) seen + = removeIntersecting seen rooms + | otherwise + = room :> removeIntersecting (room : seen) rooms + streamRepeat x = x :> streamRepeat x + infinitely = sequence . streamRepeat + +delaunayRoomGraph :: [Room] -> Gr Room Double +delaunayRoomGraph rooms = + Graph.insEdges edges . Graph.insNodes nodes $ Graph.empty + where + edges = map (\((n₁, room₁), (n₂, room₂)) -> (n₁, n₂, roomDist room₁ room₂)) + . over (mapped . both) snd + . delaunay @Double + . NE.fromList + . map (\p@(_, room) -> (boxCenter $ fromIntegral <$> room, p)) + $ nodes + nodes = zip [0..] rooms + roomDist = distance `on` (boxCenter . fmap fromIntegral) + +fillRoom :: MCells s -> Room -> CellM g s () +fillRoom cells room = + let V2 posx posy = room ^. topLeftCorner + V2 dimx dimy = room ^. dimensions + in for_ [posx .. posx + dimx] $ \x -> + for_ [posy .. posy + dimy] $ \y -> + lift $ writeArray cells (V2 x y) True + +corridorBetween :: MonadRandom m => Room -> Room -> m [V2 Word] +corridorBetween originRoom destinationRoom + = straightLine <$> origin <*> destination + where + origin = choose . NE.fromList =<< originEdge + destination = choose . NE.fromList =<< destinationEdge + originEdge = pickEdge originRoom originCorner + destinationEdge = pickEdge destinationRoom destinationCorner + pickEdge room corner = choose . over both (boxEdge room) $ cornerEdges corner + originCorner = + case ( compare (originRoom ^. topLeftCorner . _x) + (destinationRoom ^. topLeftCorner . _x) + , compare (originRoom ^. topLeftCorner . _y) + (destinationRoom ^. topLeftCorner . _y) + ) of + (LT, LT) -> BottomRight + (LT, GT) -> TopRight + (GT, LT) -> BottomLeft + (GT, GT) -> TopLeft + + (EQ, LT) -> BottomLeft + (EQ, GT) -> TopRight + (GT, EQ) -> TopLeft + (LT, EQ) -> BottomRight + (EQ, EQ) -> TopLeft -- should never happen + + destinationCorner = opposite originCorner diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Level/LevelContents.hs b/users/grfn/xanthous/src/Xanthous/Generators/Level/LevelContents.hs new file mode 100644 index 000000000000..4f8a2f42ee16 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Level/LevelContents.hs @@ -0,0 +1,182 @@ +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level.LevelContents + ( chooseCharacterPosition + , randomItems + , randomCreatures + , randomDoors + , placeDownStaircase + , tutorialMessage + , entityFromRaw + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (any, toList) +-------------------------------------------------------------------------------- +import Control.Monad.Random +import Data.Array.IArray (amap, bounds, rangeSize, (!)) +import qualified Data.Array.IArray as Arr +import Data.Foldable (any, toList) +import Linear.V2 +-------------------------------------------------------------------------------- +import Xanthous.Generators.Level.Util +import Xanthous.Random hiding (chance) +import qualified Xanthous.Random as Random +import Xanthous.Data + ( positionFromV2, Position, _Position + , rotations, arrayNeighbors, Neighbors(..) + , neighborPositions + ) +import Xanthous.Data.EntityMap (EntityMap, _EntityMap) +import Xanthous.Entities.Raws (rawsWithType, RawType, raw) +import qualified Xanthous.Entities.Item as Item +import Xanthous.Entities.Item (Item) +import qualified Xanthous.Entities.Creature as Creature +import Xanthous.Entities.Creature (Creature) +import Xanthous.Entities.Environment + (GroundMessage(..), Door(..), unlockedDoor, Staircase(..)) +import Xanthous.Messages (message_) +import Xanthous.Util.Graphics (circle) +import Xanthous.Entities.RawTypes +import Xanthous.Entities.Creature.Hippocampus (initialHippocampus) +import Xanthous.Entities.Common (inRightHand, asWieldedItem, wielded) +import Xanthous.Game.State (SomeEntity(SomeEntity)) +-------------------------------------------------------------------------------- + +chooseCharacterPosition :: MonadRandom m => Cells -> m Position +chooseCharacterPosition = randomPosition + +randomItems :: MonadRandom m => Cells -> m (EntityMap Item) +randomItems = randomEntities (fmap Identity . Item.newWithType) (0.0004, 0.001) + +placeDownStaircase :: MonadRandom m => Cells -> m (EntityMap Staircase) +placeDownStaircase cells = do + pos <- randomPosition cells + pure $ _EntityMap # [(pos, DownStaircase)] + +randomDoors :: MonadRandom m => Cells -> m (EntityMap Door) +randomDoors cells = do + doorRatio <- getRandomR subsetRange + let numDoors = floor $ doorRatio * fromIntegral (length candidateCells) + doorPositions = + removeAdjacent . fmap positionFromV2 . take numDoors $ candidateCells + doors = zip doorPositions $ repeat unlockedDoor + pure $ _EntityMap # doors + where + removeAdjacent = + foldr (\pos acc -> + if pos `elem` (acc >>= toList . neighborPositions) + then acc + else pos : acc + ) [] + candidateCells = filter doorable $ Arr.indices cells + subsetRange = (0.8 :: Double, 1.0) + doorable pos = + not (fromMaybe True $ cells ^? ix pos) + && any (teeish . fmap (fromMaybe True)) + (rotations $ arrayNeighbors cells pos) + -- only generate doors at the *ends* of hallways, eg (where O is walkable, + -- X is a wall, and D is a door): + -- + -- O O O + -- X D X + -- O + teeish (fmap not -> (Neighbors tl t tr l r _ b _ )) = + and [tl, t, tr, b] && (and . fmap not) [l, r] + +randomCreatures + :: MonadRandom m + => Word -- ^ Level number, starting at 0 + -> Cells + -> m (EntityMap Creature) +randomCreatures levelNumber + = randomEntities maybeNewCreature (0.0007, 0.002) + where + maybeNewCreature cType + | maybe True (canGenerate levelNumber) $ cType ^. generateParams + = Just <$> newCreatureWithType cType + | otherwise + = pure Nothing + +newCreatureWithType :: MonadRandom m => CreatureType -> m Creature +newCreatureWithType _creatureType = do + let _hitpoints = _creatureType ^. maxHitpoints + _hippocampus = initialHippocampus + + equipped <- fmap join + . traverse genEquipped + $ _creatureType + ^.. generateParams . _Just . equippedItem . _Just + let _inventory = maybe id (\ei -> wielded .~ inRightHand ei) (headMay equipped) mempty + pure Creature.Creature {..} + where + genEquipped cei = do + doGen <- Random.chance $ cei ^. chance + let entName = cei ^. entityName + itemType = + fromMaybe (error $ "raw \"" <> unpack entName <> "\" not of type Item") + . preview _Item + . fromMaybe (error $ "Could not find raw: " <> unpack entName) + $ raw entName + item <- Item.newWithType itemType + if doGen + then pure [fromMaybe (error $ "raw \"" <> unpack entName <> "\" not wieldable") + $ preview asWieldedItem item] + else pure [] + + +tutorialMessage :: MonadRandom m + => Cells + -> Position -- ^ CharacterPosition + -> m (EntityMap GroundMessage) +tutorialMessage cells characterPosition = do + let distance = 2 + pos <- fmap (fromMaybe (error "No valid positions for tutorial message?")) + . choose . ChooseElement + $ accessiblePositionsWithin distance cells characterPosition + msg <- message_ ["tutorial", "message1"] + pure $ _EntityMap # [(pos, GroundMessage msg)] + where + accessiblePositionsWithin :: Int -> Cells -> Position -> [Position] + accessiblePositionsWithin dist valid pos = + review _Position + <$> filter + (\pt -> not $ valid ! (fromIntegral <$> pt)) + (circle (pos ^. _Position) dist) + +randomEntities + :: forall entity raw m t. (MonadRandom m, RawType raw, Functor t, Foldable t) + => (raw -> m (t entity)) + -> (Float, Float) + -> Cells + -> m (EntityMap entity) +randomEntities newWithType sizeRange cells = + case fromNullable $ rawsWithType @raw of + Nothing -> pure mempty + Just raws -> do + let len = rangeSize $ bounds cells + (numEntities :: Int) <- + floor . (* fromIntegral len) <$> getRandomR sizeRange + entities <- for [0..numEntities] $ const $ do + pos <- randomPosition cells + r <- choose raws + entities <- newWithType r + pure $ (pos, ) <$> entities + pure $ _EntityMap # (entities >>= toList) + +randomPosition :: MonadRandom m => Cells -> m Position +randomPosition = fmap positionFromV2 . choose . impureNonNull . cellCandidates + +-- cellCandidates :: Cells -> Cells +cellCandidates :: Cells -> Set (V2 Word) +cellCandidates + -- find the largest contiguous region of cells in the cave. + = maximumBy (compare `on` length) + . fromMaybe (error "No regions generated! this should never happen.") + . fromNullable + . regions + -- cells ends up with true = wall, we want true = can put an item here + . amap not + +entityFromRaw :: MonadRandom m => EntityRaw -> m SomeEntity +entityFromRaw (Creature ct) = SomeEntity <$> newCreatureWithType ct +entityFromRaw (Item it) = SomeEntity <$> Item.newWithType it diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Level/Util.hs b/users/grfn/xanthous/src/Xanthous/Generators/Level/Util.hs new file mode 100644 index 000000000000..0008eb965c42 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Level/Util.hs @@ -0,0 +1,236 @@ +{-# LANGUAGE QuantifiedConstraints #-} +{-# LANGUAGE AllowAmbiguousTypes #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level.Util + ( MCells + , Cells + , CellM + , randInitialize + , initializeEmpty + , numAliveNeighborsM + , numAliveNeighbors + , fillOuterEdgesM + , cloneMArray + , floodFill + , regions + , fillAll + , fillAllM + , fromPoints + , fromPointsM + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (Foldable, toList, for_) +-------------------------------------------------------------------------------- +import Data.Array.ST +import Data.Array.Unboxed +import Control.Monad.ST +import Control.Monad.Random +import Data.Monoid +import Data.Foldable (Foldable, toList, for_) +import qualified Data.Set as Set +import Data.Semigroup.Foldable +import Linear.V2 +-------------------------------------------------------------------------------- +import Xanthous.Util (foldlMapM', maximum1, minimum1) +import Xanthous.Data (Dimensions, width, height) +-------------------------------------------------------------------------------- + +type MCells s = STUArray s (V2 Word) Bool +type Cells = UArray (V2 Word) Bool +type CellM g s a = RandT g (ST s) a + +randInitialize :: RandomGen g => Dimensions -> Double -> CellM g s (MCells s) +randInitialize dims aliveChance = do + res <- initializeEmpty dims + for_ [0..dims ^. width] $ \i -> + for_ [0..dims ^. height] $ \j -> do + val <- (>= aliveChance) <$> getRandomR (0, 1) + lift $ writeArray res (V2 i j) val + pure res + +initializeEmpty :: RandomGen g => Dimensions -> CellM g s (MCells s) +initializeEmpty dims = + lift $ newArray (0, V2 (dims ^. width) (dims ^. height)) False + +-- | Returns the number of neighbors of the given point in the given array that +-- are True. +-- +-- Behavior if point is out-of-bounds for the array is undefined, but will not +-- error +numAliveNeighborsM + :: forall a i m + . (MArray a Bool m, Ix i, Integral i) + => a (V2 i) Bool + -> V2 i + -> m Word +numAliveNeighborsM cells pt@(V2 x y) = do + cellBounds <- getBounds cells + getSum <$> foldlMapM' + (fmap (Sum . fromIntegral . fromEnum) . boundedGet cellBounds) + neighborPositions + + where + boundedGet :: (V2 i, V2 i) -> (Int, Int) -> m Bool + boundedGet bnds _ + | not (inRange bnds pt) + = pure True + boundedGet (V2 minX minY, V2 maxX maxY) (i, j) + | (x <= minX && i < 0) + || (y <= minY && j < 0) + || (x >= maxX && i > 0) + || (y >= maxY && j > 0) + = pure True + | otherwise = + let nx = fromIntegral $ fromIntegral x + i + ny = fromIntegral $ fromIntegral y + j + in readArray cells $ V2 nx ny + +-- | Returns the number of neighbors of the given point in the given array that +-- are True. +-- +-- Behavior if point is out-of-bounds for the array is undefined, but will not +-- error +numAliveNeighbors + :: forall a i + . (IArray a Bool, Ix i, Integral i) + => a (V2 i) Bool + -> V2 i + -> Word +numAliveNeighbors cells pt@(V2 x y) = + let cellBounds = bounds cells + in getSum $ foldMap + (Sum . fromIntegral . fromEnum . boundedGet cellBounds) + neighborPositions + + where + boundedGet :: (V2 i, V2 i) -> (Int, Int) -> Bool + boundedGet bnds _ + | not (inRange bnds pt) + = True + boundedGet (V2 minX minY, V2 maxX maxY) (i, j) + | (x <= minX && i < 0) + || (y <= minY && j < 0) + || (x >= maxX && i > 0) + || (y >= maxY && j > 0) + = True + | otherwise = + let nx = fromIntegral $ fromIntegral x + i + ny = fromIntegral $ fromIntegral y + j + in cells ! V2 nx ny + +neighborPositions :: [(Int, Int)] +neighborPositions = [(i, j) | i <- [-1..1], j <- [-1..1], (i, j) /= (0, 0)] + +fillOuterEdgesM :: (MArray a Bool m, Ix i) => a (V2 i) Bool -> m () +fillOuterEdgesM arr = do + (V2 minX minY, V2 maxX maxY) <- getBounds arr + for_ (range (minX, maxX)) $ \x -> do + writeArray arr (V2 x minY) True + writeArray arr (V2 x maxY) True + for_ (range (minY, maxY)) $ \y -> do + writeArray arr (V2 minX y) True + writeArray arr (V2 maxX y) True + +cloneMArray + :: forall a a' i e m. + ( Ix i + , MArray a e m + , MArray a' e m + , IArray UArray e + ) + => a i e + -> m (a' i e) +cloneMArray = thaw @_ @UArray <=< freeze + +-------------------------------------------------------------------------------- + +-- | Flood fill a cell array starting at a point, returning a list of all the +-- (true) cell locations reachable from that point +floodFill :: forall a i. + ( IArray a Bool + , Ix i + , Enum i + , Bounded i + , Eq i + ) + => a (V2 i) Bool -- ^ array + -> (V2 i) -- ^ position + -> Set (V2 i) +floodFill = go mempty + where + go :: Set (V2 i) -> a (V2 i) Bool -> (V2 i) -> Set (V2 i) + go res arr@(bounds -> arrBounds) idx@(V2 x y) + | not (inRange arrBounds idx) = res + | not (arr ! idx) = res + | otherwise = + let neighbors + = filter (inRange arrBounds) + . filter (/= idx) + . filter (`notMember` res) + $ V2 + <$> [(if x == minBound then x else pred x) + .. + (if x == maxBound then x else succ x)] + <*> [(if y == minBound then y else pred y) + .. + (if y == maxBound then y else succ y)] + in foldl' (\r idx' -> + if arr ! idx' + then r <> (let r' = r & contains idx' .~ True + in r' `seq` go r' arr idx') + else r) + (res & contains idx .~ True) neighbors +{-# SPECIALIZE floodFill :: UArray (V2 Word) Bool -> (V2 Word) -> Set (V2 Word) #-} + +-- | Gives a list of all the disconnected regions in a cell array, represented +-- each as lists of points +regions :: forall a i. + ( IArray a Bool + , Ix i + , Enum i + , Bounded i + , Eq i + ) + => a (V2 i) Bool + -> [Set (V2 i)] +regions arr + | Just firstPoint <- findFirstPoint arr = + let region = floodFill arr firstPoint + arr' = fillAll region arr + in region : regions arr' + | otherwise = [] + where + findFirstPoint :: a (V2 i) Bool -> Maybe (V2 i) + findFirstPoint = fmap fst . headMay . filter snd . assocs +{-# SPECIALIZE regions :: UArray (V2 Word) Bool -> [Set (V2 Word)] #-} + +fillAll :: (IArray a Bool, Ix i, Foldable f) => f i -> a i Bool -> a i Bool +fillAll ixes a = accum (const fst) a $ (, (False, ())) <$> toList ixes + +fillAllM :: (MArray a Bool m, Ix i, Foldable f) => f i -> a i Bool -> m () +fillAllM ixes a = for_ ixes $ \i -> writeArray a i False + +fromPoints + :: forall a f i. + ( IArray a Bool + , Ix i + , Functor f + , Foldable1 f + ) + => f (i, i) + -> a (i, i) Bool +fromPoints points = + let pts = Set.fromList $ toList points + dims = ( (minimum1 $ fst <$> points, minimum1 $ snd <$> points) + , (maximum1 $ fst <$> points, maximum1 $ snd <$> points) + ) + in array dims $ range dims <&> \i -> (i, i `member` pts) + +fromPointsM + :: (MArray a Bool m, Ix i, Element f ~ i, MonoFoldable f) + => NonNull f + -> m (a i Bool) +fromPointsM points = do + arr <- newArray (minimum points, maximum points) False + fillAllM (otoList points) arr + pure arr diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Level/Village.hs b/users/grfn/xanthous/src/Xanthous/Generators/Level/Village.hs new file mode 100644 index 000000000000..ab7de95e6806 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Level/Village.hs @@ -0,0 +1,126 @@ +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level.Village + ( fromCave + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (any, failing, toList) +-------------------------------------------------------------------------------- +import Control.Monad.Random (MonadRandom) +import Control.Monad.State (execStateT, MonadState, modify) +import Control.Monad.Trans.Maybe +import Control.Parallel.Strategies +import Data.Array.IArray +import Data.Foldable (any, toList) +-------------------------------------------------------------------------------- +import Xanthous.Data +import Xanthous.Data.EntityMap (EntityMap) +import qualified Xanthous.Data.EntityMap as EntityMap +import Xanthous.Entities.Environment +import Xanthous.Generators.Level.Util +import Xanthous.Game.State (SomeEntity(..)) +import Xanthous.Random +-------------------------------------------------------------------------------- + +fromCave :: MonadRandom m + => Cells -- ^ The positions of all the walls + -> m (EntityMap SomeEntity) +fromCave wallPositions = execStateT (fromCave' wallPositions) mempty + +fromCave' :: forall m. (MonadRandom m, MonadState (EntityMap SomeEntity) m) + => Cells + -> m () +fromCave' wallPositions = failing (pure ()) $ do + Just villageRegion <- + choose + . (`using` parTraversable rdeepseq) + . weightedBy (\reg -> let circSize = length $ circumference reg + in if circSize == 50 + then (1.0 :: Double) + else 1.0 / (fromIntegral . abs $ circSize - 50)) + $ regions closedHallways + + let circ = setFromList . circumference $ villageRegion + + centerPoints <- chooseSubset (0.1 :: Double) $ toList circ + + roomTiles <- foldM + (flip $ const $ stepOut circ) + (map pure centerPoints) + [0 :: Int ..2] + + let roomWalls = circumference . setFromList @(Set _) <$> roomTiles + allWalls = join roomWalls + + doorPositions <- fmap join . for roomWalls $ \room -> + let candidates = filter (`notMember` circ) room + in fmap toList . choose $ ChooseElement candidates + + let entryways = + filter (\pt -> + let ncs = neighborCells pt + in any ((&&) <$> (not . (wallPositions !)) + <*> (`notMember` villageRegion)) ncs + && any ((&&) <$> (`member` villageRegion) + <*> (`notElem` allWalls)) ncs) + $ toList villageRegion + + Just entryway <- choose $ ChooseElement entryways + + for_ (filter ((&&) <$> (`notElem` doorPositions) <*> (/= entryway)) allWalls) + $ insertEntity Wall + for_ (filter (/= entryway) doorPositions) $ insertEntity unlockedDoor + insertEntity unlockedDoor entryway + + + where + insertEntity e pt = modify $ EntityMap.insertAt (ptToPos pt) $ SomeEntity e + ptToPos pt = _Position # (fromIntegral <$> pt) + + stepOut :: Set (V2 Word) -> [[V2 Word]] -> MaybeT m [[V2 Word]] + stepOut circ rooms = for rooms $ \room -> + let nextLevels = hashNub $ toList . neighborCells =<< room + in pure + . (<> room) + $ filter ((&&) <$> (`notMember` circ) <*> (`notElem` join rooms)) + nextLevels + + circumference pts = + filter (any (`notMember` pts) . neighborCells) $ toList pts + closedHallways = closeHallways livePositions + livePositions = amap not wallPositions + +-------------------------------------------------------------------------------- + +closeHallways :: Cells -> Cells +closeHallways livePositions = + livePositions // mapMaybe closeHallway (assocs livePositions) + where + closeHallway (_, False) = Nothing + closeHallway (pos, _) + | isHallway pos = Just (pos, False) + | otherwise = Nothing + isHallway pos = any ((&&) <$> not . view left <*> not . view right) + . rotations + . fmap (fromMaybe False) + $ arrayNeighbors livePositions pos + +failing :: Monad m => m a -> MaybeT m a -> m a +failing result = (maybe result pure =<<) . runMaybeT + +{- + +import Xanthous.Generators.Village +import Xanthous.Generators +import Xanthous.Data +import System.Random +import qualified Data.Text +import qualified Xanthous.Generators.CaveAutomata as CA +let gi = GeneratorInput SCaveAutomata CA.defaultParams +wallPositions <- generateFromInput gi (Dimensions 80 50) <$> getStdGen +putStrLn . Data.Text.unpack $ showCells wallPositions + +import Data.Array.IArray +let closedHallways = closeHallways . amap not $ wallPositions +putStrLn . Data.Text.unpack . showCells $ amap not closedHallways + +-} diff --git a/users/grfn/xanthous/src/Xanthous/Generators/Speech.hs b/users/grfn/xanthous/src/Xanthous/Generators/Speech.hs new file mode 100644 index 000000000000..8abc00b6a2fc --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Generators/Speech.hs @@ -0,0 +1,181 @@ +{-# LANGUAGE TemplateHaskell #-} +{-# LANGUAGE OverloadedLists #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Speech + ( -- * Language definition + Language(..) + -- ** Lenses + , phonotactics + , syllablesPerWord + + -- ** Phonotactics + , Phonotactics(..) + -- *** Lenses + , onsets + , nuclei + , codas + , numOnsets + , numNuclei + , numCodas + + -- * Language generation + , syllable + , word + + -- * Languages + , english + , gormlak + + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (replicateM) +import Data.Interval (Interval, (<=..<=)) +import qualified Data.Interval as Interval +import Control.Monad.Random.Class (MonadRandom) +import Xanthous.Random (chooseRange, choose, ChooseElement (..), Weighted (Weighted)) +import Control.Monad (replicateM) +import Test.QuickCheck (Arbitrary, CoArbitrary, Function) +import Test.QuickCheck.Instances.Text () +import Data.List.NonEmpty (NonEmpty) +-------------------------------------------------------------------------------- + +newtype Phoneme = Phoneme Text + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData, CoArbitrary, Function) + deriving newtype (IsString, Semigroup, Monoid, Arbitrary) + +-- | The phonotactics of a language +-- +-- The phonotactics of a language represent the restriction on the phonemes in +-- the syllables of a language. +-- +-- Syllables in a language consist of an onset, a nucleus, and a coda (the +-- nucleus and the coda together representing the "rhyme" of the syllable). +data Phonotactics = Phonotactics + { _onsets :: [Phoneme] -- ^ The permissible onsets, or consonant clusters + -- at the beginning of a syllable + , _nuclei :: [Phoneme] -- ^ The permissible nuclei, or vowel clusters in + -- the middle of a syllable + , _codas :: [Phoneme] -- ^ The permissible codas, or consonant clusters at + -- the end of a syllable + , _numOnsets :: Interval Word -- ^ The range of number of allowable onsets + , _numNuclei :: Interval Word -- ^ The range of number of allowable nuclei + , _numCodas :: Interval Word -- ^ The range of number of allowable codas + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) +makeLenses ''Phonotactics + +-- | Randomly generate a syllable with the given 'Phonotactics' +syllable :: MonadRandom m => Phonotactics -> m Text +syllable phonotactics = do + let genPart num choices = do + n <- fromIntegral . fromMaybe 0 <$> chooseRange (phonotactics ^. num) + fmap (fromMaybe mempty . mconcat) + . replicateM n + . choose . ChooseElement + $ phonotactics ^. choices + + (Phoneme onset) <- genPart numOnsets onsets + (Phoneme nucleus) <- genPart numNuclei nuclei + (Phoneme coda) <- genPart numCodas codas + + pure $ onset <> nucleus <> coda + +-- | A definition for a language +-- +-- Currently this provides enough information to generate multi-syllabic words, +-- but in the future will likely also include grammar-related things. +data Language = Language + { _phonotactics :: Phonotactics + , _syllablesPerWord :: Weighted Int NonEmpty Int + } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) +makeLenses ''Language + +word :: MonadRandom m => Language -> m Text +word lang = do + numSyllables <- choose $ lang ^. syllablesPerWord + mconcat <$> replicateM numSyllables (syllable $ lang ^. phonotactics) + +-------------------------------------------------------------------------------- + +-- <https://en.wikipedia.org/wiki/English_phonology#Phonotactics> +englishPhonotactics :: Phonotactics +englishPhonotactics = Phonotactics + { _onsets = [ "pl" , "bl" , "kl" , "gl" , "pr" , "br" , "tr" , "dr" , "kr" + , "gr" , "tw" , "dw" , "gw" , "kw" , "pw" + + , "fl" , "sl" , {- "thl", -} "shl" {- , "vl" -} + , "p", "b", "t", "d", "k", "ɡ", "m", "n", "f", "v", "th", "s" + , "z", "h", "l", "w" + + , "sp", "st", "sk" + + , "sm", "sn" + + , "sf", "sth" + + , "spl", "skl", "spr", "str", "skr", "skw", "sm", "sp", "st", "sk" + ] + , _nuclei = [ "a", "e", "i", "o", "u", "ur", "ar", "or", "ear", "are", "ure" + , "oa", "ee", "oo", "ei", "ie", "oi", "ou" + ] + , _codas = [ "m", "n", "ng", "p", "t", "tsh", "k", "f", "sh", "s", "th", "x" + , "v", "z", "zh", "l", "r", "w" + + , "lk", "lb", "lt", "ld", "ltsh", "ldsh", "lk" + , "rp", "rb", "rt", "rd", "rtsh", "rdsh", "rk", "rɡ" + , "lf", "lv", "lth", "ls", "lz", "lsh", "lth" + , "rf", "rv", "rth", "rs", "rz", "rth" + , "lm", "ln" + , "rm", "rn", "rl" + , "mp", "nt", "nd", "nth", "nsh", "nk" + , "mf", "ms", "mth", "nf", "nth", "ns", "nz", "nth" + , "ft", "sp", "st", "sk" + , "fth" + , "pt", "kt" + , "pth", "ps", "th", "ts", "dth", "dz", "ks" + , "lpt", "lps", "lfth", "lts", "lst", "lkt", "lks" + , "rmth", "rpt", "rps", "rts", "rst", "rkt" + , "mpt", "mps", "ndth", "nkt", "nks", "nkth" + , "ksth", "kst" + ] + , _numOnsets = 0 <=..<= 1 + , _numNuclei = Interval.singleton 1 + , _numCodas = 0 <=..<= 1 + } + +english :: Language +english = Language + { _phonotactics = englishPhonotactics + , _syllablesPerWord = Weighted [(20, 1), + (7, 2), + (2, 3), + (1, 4)] + } + +gormlakPhonotactics :: Phonotactics +gormlakPhonotactics = Phonotactics + { _onsets = [ "h", "l", "g", "b", "m", "n", "ng" + , "gl", "bl", "fl" + ] + , _numOnsets = Interval.singleton 1 + , _nuclei = [ "a", "o", "aa", "u" ] + , _numNuclei = Interval.singleton 1 + , _codas = [ "r", "l", "g", "m", "n" + , "rl", "gl", "ml", "rm" + , "n", "k" + ] + , _numCodas = Interval.singleton 1 + } + +gormlak :: Language +gormlak = Language + { _phonotactics = gormlakPhonotactics + , _syllablesPerWord = Weighted [ (5, 2) + , (5, 1) + , (1, 3) + ] + } diff --git a/users/grfn/xanthous/src/Xanthous/Messages.hs b/users/grfn/xanthous/src/Xanthous/Messages.hs new file mode 100644 index 000000000000..c273d650821b --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Messages.hs @@ -0,0 +1,114 @@ +{-# LANGUAGE TemplateHaskell #-} +-------------------------------------------------------------------------------- +module Xanthous.Messages + ( Message(..) + , resolve + , MessageMap(..) + , lookupMessage + + -- * Game messages + , messages + , render + , render_ + , lookup + , message + , message_ + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (lookup) +-------------------------------------------------------------------------------- +import Control.Monad.Random.Class (MonadRandom) +import Data.Aeson (FromJSON, ToJSON, toJSON, object) +import qualified Data.Aeson as JSON +import Data.Aeson.Generic.DerivingVia +import Data.FileEmbed +import Data.List.NonEmpty +import Test.QuickCheck hiding (choose) +import Test.QuickCheck.Instances.UnorderedContainers () +import Text.Mustache +import qualified Data.Yaml as Yaml +-------------------------------------------------------------------------------- +import Xanthous.Random +import Xanthous.Orphans () +-------------------------------------------------------------------------------- + +data Message = Single Template | Choice (NonEmpty Template) + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (CoArbitrary, Function, NFData) + deriving (ToJSON, FromJSON) + via WithOptions '[ SumEnc UntaggedVal ] + Message + +instance Arbitrary Message where + arbitrary = + frequency [ (10, Single <$> arbitrary) + , (1, Choice <$> arbitrary) + ] + shrink = genericShrink + +resolve :: MonadRandom m => Message -> m Template +resolve (Single t) = pure t +resolve (Choice ts) = choose ts + +data MessageMap = Direct Message | Nested (HashMap Text MessageMap) + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (CoArbitrary, Function, NFData) + deriving (ToJSON, FromJSON) + via WithOptions '[ SumEnc UntaggedVal ] + MessageMap + +instance Arbitrary MessageMap where + arbitrary = frequency [ (10, Direct <$> arbitrary) + , (1, Nested <$> arbitrary) + ] + +lookupMessage :: [Text] -> MessageMap -> Maybe Message +lookupMessage [] (Direct msg) = Just msg +lookupMessage (k:ks) (Nested m) = lookupMessage ks =<< m ^. at k +lookupMessage _ _ = Nothing + +type instance Index MessageMap = [Text] +type instance IxValue MessageMap = Message +instance Ixed MessageMap where + ix [] f (Direct msg) = Direct <$> f msg + ix (k:ks) f (Nested m) = case m ^. at k of + Just m' -> ix ks f m' <&> \m'' -> + Nested $ m & at k ?~ m'' + Nothing -> pure $ Nested m + ix _ _ m = pure m + +-------------------------------------------------------------------------------- + +rawMessages :: ByteString +rawMessages = $(embedFile "src/Xanthous/messages.yaml") + +messages :: MessageMap +messages + = either (error . Yaml.prettyPrintParseException) id + $ Yaml.decodeEither' rawMessages + +render :: (MonadRandom m, ToJSON params) => Message -> params -> m Text +render msg params = do + tpl <- resolve msg + pure . toStrict . renderMustache tpl $ toJSON params + +-- | Render a message with an empty set of params +render_ :: (MonadRandom m) => Message -> m Text +render_ msg = render msg $ object [] + +lookup :: [Text] -> Message +lookup path = fromMaybe notFound $ messages ^? ix path + where notFound + = Single + $ compileMustacheText "template" "Message not found" + ^?! _Right + +message :: (MonadRandom m, ToJSON params) => [Text] -> params -> m Text +message path params = maybe notFound (`render` params) $ messages ^? ix path + where + notFound = pure "Message not found" + +message_ :: (MonadRandom m) => [Text] -> m Text +message_ path = maybe notFound (`render` JSON.object []) $ messages ^? ix path + where + notFound = pure "Message not found" diff --git a/users/grfn/xanthous/src/Xanthous/Messages/Template.hs b/users/grfn/xanthous/src/Xanthous/Messages/Template.hs new file mode 100644 index 000000000000..5176880355f4 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Messages/Template.hs @@ -0,0 +1,275 @@ +{-# LANGUAGE DeriveDataTypeable #-} +-------------------------------------------------------------------------------- +module Xanthous.Messages.Template + ( -- * Template AST + Template(..) + , Substitution(..) + , Filter(..) + + -- ** Template AST transformations + , reduceTemplate + + -- * Template parser + , template + , runParser + , errorBundlePretty + + -- * Template pretty-printer + , ppTemplate + + -- * Rendering templates + , TemplateVar(..) + , nested + , TemplateVars(..) + , vars + , RenderError + , render + ) +where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding + (many, concat, try, elements, some, parts) +-------------------------------------------------------------------------------- +import Test.QuickCheck hiding (label) +import Test.QuickCheck.Instances.Text () +import Test.QuickCheck.Instances.Semigroup () +import Test.QuickCheck.Checkers (EqProp) +import Control.Monad.Combinators.NonEmpty +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Data +import Text.Megaparsec hiding (sepBy1, some) +import Text.Megaparsec.Char +import qualified Text.Megaparsec.Char.Lexer as L +import Data.Function (fix) +-------------------------------------------------------------------------------- +import Xanthous.Util (EqEqProp(..)) +-------------------------------------------------------------------------------- + +genIdentifier :: Gen Text +genIdentifier = pack <$> listOf1 (elements identifierChars) + +identifierChars :: String +identifierChars = ['a'..'z'] <> ['A'..'Z'] <> ['-', '_'] + +newtype Filter = FilterName Text + deriving stock (Show, Eq, Ord, Generic, Data) + deriving anyclass (NFData) + deriving (IsString) via Text + +instance Arbitrary Filter where + arbitrary = FilterName <$> genIdentifier + shrink (FilterName fn) = fmap FilterName . filter (not . null) $ shrink fn + +data Substitution + = SubstPath (NonEmpty Text) + | SubstFilter Substitution Filter + deriving stock (Show, Eq, Ord, Generic, Data) + deriving anyclass (NFData) + +instance Arbitrary Substitution where + arbitrary = sized . fix $ \gen n -> + let leaves = + [ SubstPath <$> ((:|) <$> genIdentifier <*> listOf genIdentifier)] + subtree = gen $ n `div` 2 + in if n == 0 + then oneof leaves + else oneof $ leaves <> [ SubstFilter <$> subtree <*> arbitrary ] + shrink (SubstPath pth) = + fmap SubstPath + . filter (not . any ((||) <$> null <*> any (`notElem` identifierChars))) + $ shrink pth + shrink (SubstFilter s f) + = shrink s + <> (uncurry SubstFilter <$> shrink (s, f)) + +data Template + = Literal Text + | Subst Substitution + | Concat Template Template + deriving stock (Show, Generic, Data) + deriving anyclass (NFData) + deriving EqProp via EqEqProp Template + +instance Plated Template where + plate _ tpl@(Literal _) = pure tpl + plate _ tpl@(Subst _) = pure tpl + plate f (Concat tpl₁ tpl₂) = Concat <$> f tpl₁ <*> f tpl₂ + +reduceTemplate :: Template -> Template +reduceTemplate = transform $ \case + (Concat (Literal t₁) (Literal t₂)) -> Literal (t₁ <> t₂) + (Concat (Literal "") t) -> t + (Concat t (Literal "")) -> t + (Concat t₁ (Concat t₂ t₃)) -> Concat (Concat t₁ t₂) t₃ + (Concat (Concat t₁ (Literal t₂)) (Literal t₃)) -> (Concat t₁ (Literal $ t₂ <> t₃)) + t -> t + +instance Eq Template where + tpl₁ == tpl₂ = case (reduceTemplate tpl₁, reduceTemplate tpl₂) of + (Literal t₁, Literal t₂) -> t₁ == t₂ + (Subst s₁, Subst s₂) -> s₁ == s₂ + (Concat ta₁ ta₂, Concat tb₁ tb₂) -> ta₁ == tb₁ && ta₂ == tb₂ + _ -> False + +instance Arbitrary Template where + arbitrary = sized . fix $ \gen n -> + let leaves = [ Literal . pack . filter (`notElem` ['\\', '{']) <$> arbitrary + , Subst <$> arbitrary + ] + subtree = gen $ n `div` 2 + genConcat = Concat <$> subtree <*> subtree + in if n == 0 + then oneof leaves + else oneof $ genConcat : leaves + shrink (Literal t) = Literal <$> shrink t + shrink (Subst s) = Subst <$> shrink s + shrink (Concat t₁ t₂) + = shrink t₁ + <> shrink t₂ + <> (Concat <$> shrink t₁ <*> shrink t₂) + +instance Semigroup Template where + (<>) = Concat + +instance Monoid Template where + mempty = Literal "" + +-------------------------------------------------------------------------------- + +type Parser = Parsec Void Text + +sc :: Parser () +sc = L.space space1 empty empty + +lexeme :: Parser a -> Parser a +lexeme = L.lexeme sc + +symbol :: Text -> Parser Text +symbol = L.symbol sc + +identifier :: Parser Text +identifier = lexeme . label "identifier" $ do + firstChar <- letterChar <|> oneOf ['-', '_'] + restChars <- many $ alphaNumChar <|> oneOf ['-', '_'] + pure $ firstChar <| pack restChars + +filterName :: Parser Filter +filterName = FilterName <$> identifier + +substitutionPath :: Parser Substitution +substitutionPath = SubstPath <$> sepBy1 identifier (char '.') + +substitutionFilter :: Parser Substitution +substitutionFilter = do + path <- substitutionPath + fs <- some $ symbol "|" *> filterName + pure $ foldl' SubstFilter path fs + -- pure $ SubstFilter path f + +substitutionContents :: Parser Substitution +substitutionContents + = try substitutionFilter + <|> substitutionPath + +substitution :: Parser Substitution +substitution = between (string "{{") (string "}}") substitutionContents + +literal :: Parser Template +literal = Literal <$> + ( (string "\\{" $> "{") + <|> takeWhile1P Nothing (`notElem` ['\\', '{']) + ) + +subst :: Parser Template +subst = Subst <$> substitution + +template' :: Parser Template +template' = do + parts <- many $ literal <|> subst + pure $ foldr Concat (Literal "") parts + + +template :: Parser Template +template = reduceTemplate <$> template' <* eof + +-------------------------------------------------------------------------------- + +ppSubstitution :: Substitution -> Text +ppSubstitution (SubstPath substParts) = intercalate "." substParts +ppSubstitution (SubstFilter s (FilterName f)) = ppSubstitution s <> " | " <> f + +ppTemplate :: Template -> Text +ppTemplate (Literal txt) = txt +ppTemplate (Subst s) = "{{" <> ppSubstitution s <> "}}" +ppTemplate (Concat tpl₁ tpl₂) = ppTemplate tpl₁ <> ppTemplate tpl₂ + +-------------------------------------------------------------------------------- + +data TemplateVar + = Val Text + | Nested (Map Text TemplateVar) + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) + +nested :: [(Text, TemplateVar)] -> TemplateVar +nested = Nested . mapFromList + +instance Arbitrary TemplateVar where + arbitrary = sized . fix $ \gen n -> + let nst = fmap mapFromList . listOf $ (,) <$> arbitrary <*> gen (n `div` 2) + in if n == 0 + then Val <$> arbitrary + else oneof [ Val <$> arbitrary + , Nested <$> nst] + +newtype TemplateVars = Vars { getTemplateVars :: Map Text TemplateVar } + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) + deriving (Arbitrary) via (Map Text TemplateVar) + +type instance Index TemplateVars = Text +type instance IxValue TemplateVars = TemplateVar +instance Ixed TemplateVars where + ix k f (Vars vs) = Vars <$> ix k f vs +instance At TemplateVars where + at k f (Vars vs) = Vars <$> at k f vs + +vars :: [(Text, TemplateVar)] -> TemplateVars +vars = Vars . mapFromList + +lookupVar :: TemplateVars -> NonEmpty Text -> Maybe TemplateVar +lookupVar vs (p :| []) = vs ^. at p +lookupVar vs (p :| (p₁ : ps)) = vs ^. at p >>= \case + (Val _) -> Nothing + (Nested vs') -> lookupVar (Vars vs') $ p₁ :| ps + +data RenderError + = NoSuchVariable (NonEmpty Text) + | NestedFurther (NonEmpty Text) + | NoSuchFilter Filter + deriving stock (Show, Eq, Generic) + deriving anyclass (NFData) + +renderSubst + :: Map Filter (Text -> Text) -- ^ Filters + -> TemplateVars + -> Substitution + -> Either RenderError Text +renderSubst _ vs (SubstPath pth) = + case lookupVar vs pth of + Just (Val v) -> Right v + Just (Nested _) -> Left $ NestedFurther pth + Nothing -> Left $ NoSuchVariable pth +renderSubst fs vs (SubstFilter s fn) = + case fs ^. at fn of + Just filterFn -> filterFn <$> renderSubst fs vs s + Nothing -> Left $ NoSuchFilter fn + +render + :: Map Filter (Text -> Text) -- ^ Filters + -> TemplateVars -- ^ Template variables + -> Template -- ^ Template + -> Either RenderError Text +render _ _ (Literal s) = pure s +render fs vs (Concat t₁ t₂) = (<>) <$> render fs vs t₁ <*> render fs vs t₂ +render fs vs (Subst s) = renderSubst fs vs s diff --git a/users/grfn/xanthous/src/Xanthous/Monad.hs b/users/grfn/xanthous/src/Xanthous/Monad.hs new file mode 100644 index 000000000000..db602de56f3a --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Monad.hs @@ -0,0 +1,76 @@ +-------------------------------------------------------------------------------- +module Xanthous.Monad + ( AppT(..) + , AppM + , runAppT + , continue + , halt + + -- * Messages + , say + , say_ + , message + , message_ + , writeMessage + + -- * Autocommands + , cancelAutocommand + + -- * Events + , sendEvent + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Control.Monad.Random +import Control.Monad.State +import qualified Brick +import Brick (EventM, Next) +import Brick.BChan (writeBChan) +import Data.Aeson (ToJSON, object) +-------------------------------------------------------------------------------- +import Xanthous.Data.App (AppEvent) +import Xanthous.Game.State +import Xanthous.Game.Env +import Xanthous.Messages (Message) +import qualified Xanthous.Messages as Messages +-------------------------------------------------------------------------------- + +halt :: AppT (EventM n) (Next GameState) +halt = lift . Brick.halt =<< get + +continue :: AppT (EventM n) (Next GameState) +continue = lift . Brick.continue =<< get + +-------------------------------------------------------------------------------- + +say :: (MonadRandom m, ToJSON params, MonadState GameState m) + => [Text] -> params -> m () +say msgPath = writeMessage <=< Messages.message msgPath + +say_ :: (MonadRandom m, MonadState GameState m) => [Text] -> m () +say_ msgPath = say msgPath $ object [] + +message :: (MonadRandom m, ToJSON params, MonadState GameState m) + => Message -> params -> m () +message msg = writeMessage <=< Messages.render msg + +message_ :: (MonadRandom m, MonadState GameState m) + => Message -> m () +message_ msg = message msg $ object [] + +writeMessage :: MonadState GameState m => Text -> m () +writeMessage m = messageHistory %= pushMessage m + +-- | Cancel the currently active autocommand, if any +cancelAutocommand :: (MonadState GameState m, MonadIO m) => m () +cancelAutocommand = do + traverse_ (liftIO . cancel . snd) =<< preuse (autocommand . _ActiveAutocommand) + autocommand .= NoAutocommand + +-------------------------------------------------------------------------------- + +-- | Send an event to the app in an environment where the game env is available +sendEvent :: (MonadReader GameEnv m, MonadIO m) => AppEvent -> m () +sendEvent evt = do + ec <- view eventChan + liftIO $ writeBChan ec evt diff --git a/users/grfn/xanthous/src/Xanthous/Orphans.hs b/users/grfn/xanthous/src/Xanthous/Orphans.hs new file mode 100644 index 000000000000..e9cfddc0e69c --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Orphans.hs @@ -0,0 +1,483 @@ +{-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE PackageImports #-} +{-# OPTIONS_GHC -Wno-orphans #-} +{-# OPTIONS_GHC -Wno-type-defaults #-} +-------------------------------------------------------------------------------- +module Xanthous.Orphans + ( ppTemplate + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (elements, (.=)) +-------------------------------------------------------------------------------- +import Data.Aeson +import Data.Aeson.Types (typeMismatch) +import Data.List.NonEmpty (NonEmpty(..)) +import Graphics.Vty.Attributes +import Brick.Widgets.Edit +import Data.Text.Zipper.Generic (GenericTextZipper) +import Brick.Widgets.Core (getName) +import System.Random.Internal (StdGen (..)) +import System.Random.SplitMix (SMGen ()) +import Test.QuickCheck +import "quickcheck-instances" Test.QuickCheck.Instances () +import Text.Megaparsec (errorBundlePretty) +import Text.Megaparsec.Pos +import Text.Mustache +import Text.Mustache.Type ( showKey ) +import Control.Monad.State +import Linear +import qualified Data.Interval as Interval +import Data.Interval ( Interval, Extended (..), Boundary (..) + , lowerBound', upperBound', (<=..<), (<=..<=) + , interval) +import Test.QuickCheck.Checkers (EqProp ((=-=))) +-------------------------------------------------------------------------------- +import Xanthous.Util.JSON +import Xanthous.Util.QuickCheck +import Xanthous.Util (EqEqProp(EqEqProp)) +-------------------------------------------------------------------------------- + +instance forall s a. + ( Cons s s a a + , IsSequence s + , Element s ~ a + ) => Cons (NonNull s) (NonNull s) a a where + _Cons = prism hither yon + where + hither :: (a, NonNull s) -> NonNull s + hither (a, ns) = + let s = toNullable ns + in impureNonNull $ a <| s + + yon :: NonNull s -> Either (NonNull s) (a, NonNull s) + yon ns = case nuncons ns of + (_, Nothing) -> Left ns + (x, Just xs) -> Right (x, xs) + +instance forall a. Cons (NonEmpty a) (NonEmpty a) a a where + _Cons = prism hither yon + where + hither :: (a, NonEmpty a) -> NonEmpty a + hither (a, x :| xs) = a :| (x : xs) + + yon :: NonEmpty a -> Either (NonEmpty a) (a, NonEmpty a) + yon ns@(x :| xs) = case xs of + (y : ys) -> Right (x, y :| ys) + [] -> Left ns + + +instance Arbitrary PName where + arbitrary = PName . pack <$> listOf1 (elements ['a'..'z']) + +instance Arbitrary Key where + arbitrary = Key <$> listOf1 arbSafeText + where arbSafeText = pack <$> listOf1 (elements ['a'..'z']) + shrink (Key []) = error "unreachable" + shrink k@(Key [_]) = pure k + shrink (Key (p:ps)) = Key . (p :) <$> shrink ps + +instance Arbitrary Pos where + arbitrary = mkPos . succ . abs <$> arbitrary + shrink (unPos -> 1) = [] + shrink (unPos -> x) = mkPos <$> [x..1] + +instance Arbitrary Node where + arbitrary = scale (`div` 10) $ sized node + where + node n | n > 0 = oneof $ leaves ++ branches (n `div` 4) + node _ = oneof leaves + branches n = + [ Section <$> arbitrary <*> subnodes n + , InvertedSection <$> arbitrary <*> subnodes n + ] + subnodes = fmap concatTextBlocks . listOf . node + leaves = + [ TextBlock . pack <$> listOf1 (elements ['a'..'z']) + , EscapedVar <$> arbitrary + , UnescapedVar <$> arbitrary + -- TODO fix pretty-printing of mustache partials + -- , Partial <$> arbitrary <*> arbitrary + ] + shrink = genericShrink + +concatTextBlocks :: [Node] -> [Node] +concatTextBlocks [] = [] +concatTextBlocks [x] = [x] +concatTextBlocks (TextBlock txt₁ : TextBlock txt₂ : xs) + = concatTextBlocks $ TextBlock (txt₁ <> txt₂) : concatTextBlocks xs +concatTextBlocks (x : xs) = x : concatTextBlocks xs + +instance Arbitrary Template where + arbitrary = scale (`div` 8) $ do + template <- concatTextBlocks <$> arbitrary + -- templateName <- arbitrary + -- rest <- arbitrary + let templateName = "template" + rest = mempty + pure $ Template + { templateActual = templateName + , templateCache = rest & at templateName ?~ template + } + shrink (Template actual cache) = + let Just tpl = cache ^. at actual + in do + cache' <- shrink cache + tpl' <- shrink tpl + actual' <- shrink actual + pure $ Template + { templateActual = actual' + , templateCache = cache' & at actual' ?~ tpl' + } + +instance CoArbitrary Template where + coarbitrary = coarbitrary . ppTemplate + +instance Function Template where + function = functionMap ppTemplate parseTemplatePartial + where + parseTemplatePartial txt + = compileMustacheText "template" txt ^?! _Right + +ppNode :: Map PName [Node] -> Node -> Text +ppNode _ (TextBlock txt) = txt +ppNode _ (EscapedVar k) = "{{" <> showKey k <> "}}" +ppNode ctx (Section k body) = + let sk = showKey k + in "{{#" <> sk <> "}}" <> foldMap (ppNode ctx) body <> "{{/" <> sk <> "}}" +ppNode _ (UnescapedVar k) = "{{{" <> showKey k <> "}}}" +ppNode ctx (InvertedSection k body) = + let sk = showKey k + in "{{^" <> sk <> "}}" <> foldMap (ppNode ctx) body <> "{{/" <> sk <> "}}" +ppNode _ (Partial n _) = "{{> " <> unPName n <> "}}" + +ppTemplate :: Template -> Text +ppTemplate (Template actual cache) = + case cache ^. at actual of + Nothing -> error "Template not found?" + Just nodes -> foldMap (ppNode cache) nodes + +instance ToJSON Template where + toJSON = String . ppTemplate + +instance FromJSON Template where + parseJSON + = withText "Template" + $ either (fail . errorBundlePretty) pure + . compileMustacheText "template" + +deriving anyclass instance NFData Node +deriving anyclass instance NFData Template + +instance FromJSON Color where + parseJSON (String "black") = pure black + parseJSON (String "red") = pure red + parseJSON (String "green") = pure green + parseJSON (String "yellow") = pure yellow + parseJSON (String "blue") = pure blue + parseJSON (String "magenta") = pure magenta + parseJSON (String "cyan") = pure cyan + parseJSON (String "white") = pure white + parseJSON (String "brightBlack") = pure brightBlack + parseJSON (String "brightRed") = pure brightRed + parseJSON (String "brightGreen") = pure brightGreen + parseJSON (String "brightYellow") = pure brightYellow + parseJSON (String "brightBlue") = pure brightBlue + parseJSON (String "brightMagenta") = pure brightMagenta + parseJSON (String "brightCyan") = pure brightCyan + parseJSON (String "brightWhite") = pure brightWhite + parseJSON n@(Number _) = Color240 <$> parseJSON n + parseJSON x = typeMismatch "Color" x + +instance ToJSON Color where + toJSON color + | color == black = "black" + | color == red = "red" + | color == green = "green" + | color == yellow = "yellow" + | color == blue = "blue" + | color == magenta = "magenta" + | color == cyan = "cyan" + | color == white = "white" + | color == brightBlack = "brightBlack" + | color == brightRed = "brightRed" + | color == brightGreen = "brightGreen" + | color == brightYellow = "brightYellow" + | color == brightBlue = "brightBlue" + | color == brightMagenta = "brightMagenta" + | color == brightCyan = "brightCyan" + | color == brightWhite = "brightWhite" + | Color240 num <- color = toJSON num + | otherwise = error $ "unimplemented: " <> show color + +instance (Eq a, Show a, Read a, FromJSON a) => FromJSON (MaybeDefault a) where + parseJSON Null = pure Default + parseJSON (String "keepCurrent") = pure KeepCurrent + parseJSON x = SetTo <$> parseJSON x + +instance ToJSON a => ToJSON (MaybeDefault a) where + toJSON Default = Null + toJSON KeepCurrent = String "keepCurrent" + toJSON (SetTo x) = toJSON x + +-------------------------------------------------------------------------------- + +instance Arbitrary Color where + arbitrary = oneof [ Color240 <$> choose (0, 239) + , ISOColor <$> choose (0, 15) + ] + +deriving anyclass instance CoArbitrary Color +deriving anyclass instance Function Color + +instance (Eq a, Show a, Read a, Arbitrary a) => Arbitrary (MaybeDefault a) where + arbitrary = oneof [ pure Default + , pure KeepCurrent + , SetTo <$> arbitrary + ] + +instance CoArbitrary a => CoArbitrary (MaybeDefault a) where + coarbitrary Default = variant @Int 1 + coarbitrary KeepCurrent = variant @Int 2 + coarbitrary (SetTo x) = variant @Int 3 . coarbitrary x + +instance (Eq a, Show a, Read a, Function a) => Function (MaybeDefault a) where + function = functionShow + +deriving via (EqEqProp Attr) instance EqProp Attr + +instance Arbitrary Attr where + arbitrary = do + attrStyle <- arbitrary + attrForeColor <- arbitrary + attrBackColor <- arbitrary + attrURL <- arbitrary + pure Attr {..} + +deriving anyclass instance CoArbitrary Attr +deriving anyclass instance Function Attr + +instance ToJSON Attr where + toJSON Attr{..} = object + [ "style" .= maybeDefaultToJSONWith styleToJSON attrStyle + , "foreground" .= attrForeColor + , "background" .= attrBackColor + , "url" .= attrURL + ] + where + maybeDefaultToJSONWith _ Default = Null + maybeDefaultToJSONWith _ KeepCurrent = String "keepCurrent" + maybeDefaultToJSONWith tj (SetTo x) = tj x + styleToJSON style + | style == standout = "standout" + | style == underline = "underline" + | style == reverseVideo = "reverseVideo" + | style == blink = "blink" + | style == dim = "dim" + | style == bold = "bold" + | style == italic = "italic" + | otherwise = toJSON style + +instance FromJSON Attr where + parseJSON = withObject "Attr" $ \obj -> do + attrStyle <- parseStyle =<< obj .:? "style" .!= Default + attrForeColor <- obj .:? "foreground" .!= Default + attrBackColor <- obj .:? "background" .!= Default + attrURL <- obj .:? "url" .!= Default + pure Attr{..} + + where + parseStyle (SetTo (String "standout")) = pure (SetTo standout) + parseStyle (SetTo (String "underline")) = pure (SetTo underline) + parseStyle (SetTo (String "reverseVideo")) = pure (SetTo reverseVideo) + parseStyle (SetTo (String "blink")) = pure (SetTo blink) + parseStyle (SetTo (String "dim")) = pure (SetTo dim) + parseStyle (SetTo (String "bold")) = pure (SetTo bold) + parseStyle (SetTo (String "italic")) = pure (SetTo italic) + parseStyle (SetTo n@(Number _)) = SetTo <$> parseJSON n + parseStyle (SetTo v) = typeMismatch "Style" v + parseStyle Default = pure Default + parseStyle KeepCurrent = pure KeepCurrent + +deriving stock instance Ord Color +deriving stock instance Ord a => Ord (MaybeDefault a) +deriving stock instance Ord Attr + +-------------------------------------------------------------------------------- + +instance (SemiSequence a, Arbitrary (Element a), Arbitrary a) + => Arbitrary (NonNull a) where + arbitrary = ncons <$> arbitrary <*> arbitrary + +instance ToJSON a => ToJSON (NonNull a) where + toJSON = toJSON . toNullable + +instance (FromJSON a, MonoFoldable a) => FromJSON (NonNull a) where + parseJSON = maybe (fail "Found empty list") pure . fromNullable <=< parseJSON + +instance NFData a => NFData (NonNull a) where + rnf xs = xs `seq` toNullable xs `deepseq` () + +-------------------------------------------------------------------------------- + +instance forall t name. (NFData t, Monoid t, NFData name) + => NFData (Editor t name) where + rnf ed = getName @_ @name ed `deepseq` getEditContents ed `deepseq` () + +deriving via (ReadShowJSON SMGen) instance ToJSON SMGen +deriving via (ReadShowJSON SMGen) instance FromJSON SMGen + +instance ToJSON StdGen where + toJSON = toJSON . unStdGen + toEncoding = toEncoding . unStdGen + +instance FromJSON StdGen where + parseJSON = fmap StdGen . parseJSON + +-------------------------------------------------------------------------------- + +instance CoArbitrary a => CoArbitrary (NonNull a) where + coarbitrary = coarbitrary . toNullable + +instance (MonoFoldable a, Function a) => Function (NonNull a) where + function = functionMap toNullable $ fromMaybe (error "null") . fromNullable + +instance (Arbitrary t, Arbitrary n, GenericTextZipper t) + => Arbitrary (Editor t n) where + arbitrary = editor <$> arbitrary <*> arbitrary <*> arbitrary + +instance forall t n. (CoArbitrary t, CoArbitrary n, Monoid t) + => CoArbitrary (Editor t n) where + coarbitrary ed = coarbitrary (getName @_ @n ed, getEditContents ed) + +instance CoArbitrary StdGen where + coarbitrary = coarbitrary . show + +instance Function StdGen where + function = functionMap unStdGen StdGen + +instance Function SMGen where + function = functionShow + +-------------------------------------------------------------------------------- + +deriving newtype instance (Arbitrary s, CoArbitrary (m (a, s))) + => CoArbitrary (StateT s m a) + +-------------------------------------------------------------------------------- + +deriving via (GenericArbitrary (V2 a)) instance Arbitrary a => Arbitrary (V2 a) +instance CoArbitrary a => CoArbitrary (V2 a) +instance Function a => Function (V2 a) + +-------------------------------------------------------------------------------- + +instance CoArbitrary Boundary +instance Function Boundary + +instance Arbitrary a => Arbitrary (Extended a) where + arbitrary = oneof [ pure NegInf + , pure PosInf + , Finite <$> arbitrary + ] + +instance CoArbitrary a => CoArbitrary (Extended a) where + coarbitrary NegInf = variant 1 + coarbitrary PosInf = variant 2 + coarbitrary (Finite x) = variant 3 . coarbitrary x + +instance (Function a) => Function (Extended a) where + function = functionMap g h + where + g NegInf = Left True + g (Finite a) = Right a + g PosInf = Left False + h (Left False) = PosInf + h (Left True) = NegInf + h (Right a) = Finite a + +instance ToJSON a => ToJSON (Extended a) where + toJSON NegInf = String "NegInf" + toJSON PosInf = String "PosInf" + toJSON (Finite x) = toJSON x + +instance FromJSON a => FromJSON (Extended a) where + parseJSON (String "NegInf") = pure NegInf + parseJSON (String "PosInf") = pure PosInf + parseJSON val = Finite <$> parseJSON val + +instance (EqProp a, Show a) => EqProp (Extended a) where + NegInf =-= NegInf = property True + PosInf =-= PosInf = property True + (Finite x) =-= (Finite y) = x =-= y + x =-= y = counterexample (show x <> " /= " <> show y) False + +instance Arbitrary Interval.Boundary where + arbitrary = elements [ Interval.Open , Interval.Closed ] + +instance (Ord r, Arbitrary r) => Arbitrary (Interval r) where + arbitrary = do + lower <- arbitrary + upper <- arbitrary + pure $ (if upper < lower then flip else id) + Interval.interval + lower + upper + +instance CoArbitrary a => CoArbitrary (Interval a) where + coarbitrary int = coarbitrary (lowerBound' int) . coarbitrary (upperBound' int) + +instance (Function a, Ord a) => Function (Interval a) where + function = functionMap g h + where + g = lowerBound' &&& upperBound' + h = uncurry interval + +deriving via (EqEqProp (Interval a)) instance Eq a => (EqProp (Interval a)) + +instance ToJSON a => ToJSON (Interval a) where + toJSON x = Array . fromList $ + [ object [ lowerKey .= lowerVal ] + , object [ upperKey .= upperVal ] + ] + where + (lowerVal, lowerBoundary) = lowerBound' x + (upperVal, upperBoundary) = upperBound' x + upperKey = boundaryToKey upperBoundary + lowerKey = boundaryToKey lowerBoundary + boundaryToKey Open = "Excluded" + boundaryToKey Closed = "Included" + +instance forall a. (FromJSON a, Ord a) => FromJSON (Interval a) where + parseJSON x = + boundPairWithBoundary x + <|> boundPairWithoutBoundary x + <|> singleVal x + where + boundPairWithBoundary = withArray "Bound pair" $ \arr -> do + checkLength arr + lower <- parseBound $ arr ^?! ix 0 + upper <- parseBound $ arr ^?! ix 1 + pure $ interval lower upper + parseBound = withObject "Bound" $ \obj -> do + when (length obj /= 1) $ fail "Expected an object with a single key" + let [(k, v)] = obj ^@.. ifolded + boundary <- case k of + "Excluded" -> pure Open + "Open" -> pure Open + "Included" -> pure Closed + "Closed" -> pure Closed + _ -> fail "Invalid boundary specification" + val <- parseJSON v + pure (val, boundary) + boundPairWithoutBoundary = withArray "Bound pair" $ \arr -> do + checkLength arr + lower <- parseJSON $ arr ^?! ix 0 + upper <- parseJSON $ arr ^?! ix 1 + pure $ lower <=..< upper + singleVal v = do + val <- parseJSON v + pure $ val <=..<= val + checkLength arr = + when (length arr /= 2) $ fail "Expected array of length 2" diff --git a/users/grfn/xanthous/src/Xanthous/Physics.hs b/users/grfn/xanthous/src/Xanthous/Physics.hs new file mode 100644 index 000000000000..37530cbbc21b --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Physics.hs @@ -0,0 +1,71 @@ +-------------------------------------------------------------------------------- +module Xanthous.Physics + ( throwDistance + , bluntThrowDamage + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Xanthous.Data + ( Meters + , (:**:)(..) + , Square + , Grams + , (|*|) + , (|/|) + , Hitpoints + , Per (..) + , squared + , Uno(..), (|+|) + ) +-------------------------------------------------------------------------------- + +-- university shotputter can put a 16 lb shot about 14 meters +-- ≈ 7.25 kg 14 meters +-- 14m = x / (7.25kg × y + z)² +-- 14m = x / (7250g × y + z)² +-- +-- we don't want to scale down too much: +-- +-- 10 kg 10 meters +-- = 10000 g 10 meters +-- +-- 15 kg w meters +-- = 15000 g w meters +-- +-- 14m = x / (7250g × y + z)² +-- 10m = x / (10000g × y + z)² +-- wm = x / (15000g × y + z)² +-- +-- w≈0.527301 ∧ y≈0.000212178 sqrt(x) ∧ z≈1.80555 sqrt(x) ∧ 22824.1 sqrt(x)!=0 +-- +-- x = 101500 +-- y = 0.0675979 +-- z = 575.231 +-- + +-- TODO make this dynamic +strength :: Meters :**: Square Grams +strength = Times 10150000 + +yCoeff :: Uno Double +yCoeff = Uno 0.0675979 + +zCoeff :: Uno Double +zCoeff = Uno 575.231 + +-- | Calculate the maximum distance an object with the given weight can be +-- thrown +throwDistance + :: Grams -- ^ Weight of the object + -> Meters -- ^ Max distance thrown +throwDistance weight = strength |/| squared (weight |*| yCoeff |+| zCoeff) + +-- | Returns the damage dealt by a blunt object with the given weight when +-- thrown +bluntThrowDamage + :: Grams + -> Hitpoints +bluntThrowDamage weight = throwDamageRatio |*| weight + where + throwDamageRatio :: Hitpoints `Per` Grams + throwDamageRatio = Rate $ 1 / 5000 diff --git a/users/grfn/xanthous/src/Xanthous/Prelude.hs b/users/grfn/xanthous/src/Xanthous/Prelude.hs new file mode 100644 index 000000000000..2cb4299303ba --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Prelude.hs @@ -0,0 +1,48 @@ +-------------------------------------------------------------------------------- +module Xanthous.Prelude + ( module ClassyPrelude + , Type + , Constraint + , module GHC.TypeLits + , module Control.Lens + , module Data.Void + , module Control.Comonad + , module Witherable + , fail + + , (&!) + + -- * Classy-Prelude addons + , ninsertSet + , ndeleteSet + , toVector + ) where +-------------------------------------------------------------------------------- +import ClassyPrelude hiding + ( return, (<|), unsnoc, uncons, cons, snoc, index, (<.>), Index, say + , catMaybes, filter, mapMaybe, hashNub, ordNub + , Memoized, runMemoized + ) +import Data.Kind +import GHC.TypeLits hiding (Text) +import Control.Lens hiding (levels, Level) +import Data.Void +import Control.Comonad +import Witherable +import Control.Monad.Fail (fail) +-------------------------------------------------------------------------------- + +ninsertSet + :: (IsSet set, MonoPointed set) + => Element set -> NonNull set -> NonNull set +ninsertSet x xs = impureNonNull $ opoint x `union` toNullable xs + +ndeleteSet :: IsSet b => Element b -> NonNull b -> b +ndeleteSet x = deleteSet x . toNullable + +toVector :: (MonoFoldable (f a), Element (f a) ~ a) => f a -> Vector a +toVector = fromList . toList + +infixl 1 &! +(&!) :: a -> (a -> b) -> b +(&!) = flip ($!) diff --git a/users/grfn/xanthous/src/Xanthous/Random.hs b/users/grfn/xanthous/src/Xanthous/Random.hs new file mode 100644 index 000000000000..329b321b8bda --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Random.hs @@ -0,0 +1,186 @@ +-------------------------------------------------------------------------------- +{-# LANGUAGE UndecidableInstances #-} +{-# LANGUAGE StandaloneDeriving #-} +{-# OPTIONS_GHC -fno-warn-orphans #-} +-------------------------------------------------------------------------------- +module Xanthous.Random + ( Choose(..) + , ChooseElement(..) + , Weighted(..) + , evenlyWeighted + , weightedBy + , subRand + , chance + , chooseSubset + , chooseRange + , FiniteInterval(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.List.NonEmpty (NonEmpty(..)) +import Control.Monad.Random.Class (MonadRandom(getRandomR, getRandom)) +import Control.Monad.Random (Rand, evalRand, mkStdGen, StdGen) +import Data.Functor.Compose +import Data.Random.Shuffle.Weighted +import Data.Random.Distribution +import Data.Random.Distribution.Uniform +import Data.Random.Distribution.Uniform.Exclusive +import Data.Random.Sample +import qualified Data.Random.Source as DRS +import Data.Interval ( Interval, lowerBound', Extended (Finite) + , upperBound', Boundary (Closed), lowerBound, upperBound + ) +-------------------------------------------------------------------------------- + +instance {-# INCOHERENT #-} (Monad m, MonadRandom m) => DRS.MonadRandom m where + getRandomWord8 = getRandom + getRandomWord16 = getRandom + getRandomWord32 = getRandom + getRandomWord64 = getRandom + getRandomDouble = getRandom + getRandomNByteInteger n = getRandomR (0, 256 ^ n) + +class Choose a where + type RandomResult a + choose :: MonadRandom m => a -> m (RandomResult a) + +newtype ChooseElement a = ChooseElement a + +instance MonoFoldable a => Choose (ChooseElement a) where + type RandomResult (ChooseElement a) = Maybe (Element a) + choose (ChooseElement xs) = do + chosenIdx <- getRandomR (0, olength xs - 1) + let pick _ (Just x) = Just x + pick (x, i) Nothing + | i == chosenIdx = Just x + | otherwise = Nothing + pure $ ofoldr pick Nothing $ zip (toList xs) [0..] + +instance MonoFoldable a => Choose (NonNull a) where + type RandomResult (NonNull a) = Element a + choose + = fmap (fromMaybe (error "unreachable")) -- why not lol + . choose + . ChooseElement + . toNullable + +instance Choose (NonEmpty a) where + type RandomResult (NonEmpty a) = a + choose = choose . fromNonEmpty @[_] + +instance Choose (a, a) where + type RandomResult (a, a) = a + choose (x, y) = choose (x :| [y]) + +newtype Weighted w t a = Weighted (t (w, a)) + deriving (Functor, Foldable) via (t `Compose` (,) w) + +deriving newtype instance Eq (t (w, a)) => Eq (Weighted w t a) +deriving newtype instance Show (t (w, a)) => Show (Weighted w t a) +deriving newtype instance NFData (t (w, a)) => NFData (Weighted w t a) + +instance Traversable t => Traversable (Weighted w t) where + traverse f (Weighted twa) = Weighted <$> (traverse . traverse) f twa + +evenlyWeighted :: [a] -> Weighted Int [] a +evenlyWeighted = Weighted . itoList + +-- | Weight the elements of some functor by a function. Larger values of 'w' per +-- its 'Ord' instance will be more likely to be generated +weightedBy :: Functor t => (a -> w) -> t a -> Weighted w t a +weightedBy weighting xs = Weighted $ (weighting &&& id) <$> xs + +instance (Num w, Ord w, Distribution Uniform w, Excludable w) + => Choose (Weighted w [] a) where + type RandomResult (Weighted w [] a) = Maybe a + choose (Weighted ws) = sample $ headMay <$> weightedSample 1 ws + +instance (Num w, Ord w, Distribution Uniform w, Excludable w) + => Choose (Weighted w NonEmpty a) where + type RandomResult (Weighted w NonEmpty a) = a + choose (Weighted ws) = + sample + $ fromMaybe (error "unreachable") . headMay + <$> weightedSample 1 (toList ws) + +subRand :: MonadRandom m => Rand StdGen a -> m a +subRand sub = evalRand sub . mkStdGen <$> getRandom + +-- | Has a @n@ chance of returning 'True' +-- +-- eg, chance 0.5 will return 'True' half the time +chance + :: (Num w, Ord w, Distribution Uniform w, Excludable w, MonadRandom m) + => w + -> m Bool +chance n = choose $ weightedBy (bool 1 (n * 2)) bools + +-- | Choose a random subset of *about* @w@ of the elements of the given +-- 'Witherable' structure +chooseSubset :: ( Num w, Ord w, Distribution Uniform w, Excludable w + , Witherable t + , MonadRandom m + ) => w -> t a -> m (t a) +chooseSubset = filterA . const . chance + +-- | Choose a random @n@ in the given interval +chooseRange + :: ( MonadRandom m + , Distribution Uniform n + , Enum n + , Bounded n + , Ord n + ) + => Interval n + -> m (Maybe n) +chooseRange int = traverse sample distribution + where + (lower, lowerBoundary) = lowerBound' int + lowerR = case lower of + Finite x -> if lowerBoundary == Closed + then x + else succ x + _ -> minBound + (upper, upperBoundary) = upperBound' int + upperR = case upper of + Finite x -> if upperBoundary == Closed + then x + else pred x + _ -> maxBound + distribution + | lowerR <= upperR = Just $ Uniform lowerR upperR + | otherwise = Nothing + +instance ( Distribution Uniform n + , Enum n + , Bounded n + , Ord n + ) + => Choose (Interval n) where + type RandomResult (Interval n) = n + choose = fmap (fromMaybe $ error "Invalid interval") . chooseRange + +newtype FiniteInterval a + = FiniteInterval { unwrapFiniteInterval :: (Interval a) } + +instance ( Distribution Uniform n + , Ord n + ) + => Choose (FiniteInterval n) where + type RandomResult (FiniteInterval n) = n + -- TODO broken with open/closed right now + choose + = sample + . uncurry Uniform + . over both getFinite + . (lowerBound &&& upperBound) + . unwrapFiniteInterval + where + getFinite (Finite x) = x + getFinite _ = error "Infinite value" + +-------------------------------------------------------------------------------- + +bools :: NonEmpty Bool +bools = True :| [False] diff --git a/users/grfn/xanthous/src/Xanthous/Util.hs b/users/grfn/xanthous/src/Xanthous/Util.hs new file mode 100644 index 000000000000..f918340f055b --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util.hs @@ -0,0 +1,351 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE AllowAmbiguousTypes #-} +{-# LANGUAGE QuantifiedConstraints #-} +-------------------------------------------------------------------------------- +module Xanthous.Util + ( EqEqProp(..) + , EqProp(..) + , foldlMapM + , foldlMapM' + , between + + , appendVia + + -- * Foldable + -- ** Uniqueness + -- *** Predicates on uniqueness + , isUniqueOf + , isUnique + -- *** Removing all duplicate elements in n * log n time + , uniqueOf + , unique + -- *** Removing sequentially duplicate elements in linear time + , uniqOf + , uniq + -- ** Bag sequence algorithms + , takeWhileInclusive + , smallestNotIn + , removeVectorIndex + , removeFirst + , maximum1 + , minimum1 + + -- * Combinators + , times, times_, endoTimes + + -- * State utilities + , modifyK, modifyKL, useListOf + + -- * Type-level programming utils + , KnownBool(..) + + -- * + , AlphaChar(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (foldr) +-------------------------------------------------------------------------------- +import Test.QuickCheck.Checkers +import Data.Foldable (foldr) +import Data.Monoid +import Data.Proxy +import qualified Data.Vector as V +import Data.Semigroup (Max(..), Min(..)) +import Data.Semigroup.Foldable +import Control.Monad.State.Class +import Control.Monad.State (evalState) +-------------------------------------------------------------------------------- + +newtype EqEqProp a = EqEqProp a + deriving newtype Eq + +instance Eq a => EqProp (EqEqProp a) where + (=-=) = eq + +foldlMapM :: forall g b a m. (Foldable g, Monoid b, Applicative m) => (a -> m b) -> g a -> m b +foldlMapM f = foldr f' (pure mempty) + where + f' :: a -> m b -> m b + f' x = liftA2 mappend (f x) + +-- Strict in the monoidal accumulator. For monads strict +-- in the left argument of bind, this will run in constant +-- space. +foldlMapM' :: forall g b a m. (Foldable g, Monoid b, Monad m) => (a -> m b) -> g a -> m b +foldlMapM' f xs = foldr f' pure xs mempty + where + f' :: a -> (b -> m b) -> b -> m b + f' x k bl = do + br <- f x + let !b = mappend bl br + k b + +-- | Returns whether the third argument is in the range given by the first two +-- arguments, inclusive +-- +-- >>> between (0 :: Int) 2 2 +-- True +-- +-- >>> between (0 :: Int) 2 3 +-- False +between + :: Ord a + => a -- ^ lower bound + -> a -- ^ upper bound + -> a -- ^ scrutinee + -> Bool +between lower upper x = x >= lower && x <= upper + +-- | +-- >>> appendVia Sum 1 2 +-- 3 +appendVia :: (Rewrapping s t, Semigroup s) => (Unwrapped s -> s) -> Unwrapped s -> Unwrapped s -> Unwrapped s +appendVia wrap x y = op wrap $ wrap x <> wrap y + +-------------------------------------------------------------------------------- + +-- | Returns True if the targets of the given 'Fold' are unique per the 'Ord' instance for @a@ +-- +-- >>> isUniqueOf (folded . _1) ([(1, 2), (2, 2), (3, 2)] :: [(Int, Int)]) +-- True +-- +-- >>> isUniqueOf (folded . _2) ([(1, 2), (2, 2), (3, 2)] :: [(Int, Int)]) +-- False +-- +-- @ +-- 'isUniqueOf' :: Ord a => 'Getter' s a -> s -> 'Bool' +-- 'isUniqueOf' :: Ord a => 'Fold' s a -> s -> 'Bool' +-- 'isUniqueOf' :: Ord a => 'Lens'' s a -> s -> 'Bool' +-- 'isUniqueOf' :: Ord a => 'Iso'' s a -> s -> 'Bool' +-- 'isUniqueOf' :: Ord a => 'Traversal'' s a -> s -> 'Bool' +-- 'isUniqueOf' :: Ord a => 'Prism'' s a -> s -> 'Bool' +-- @ +isUniqueOf :: Ord a => Getting (Endo (Set a, Bool)) s a -> s -> Bool +isUniqueOf aFold = orOf _2 . foldrOf aFold rejectUnique (mempty, True) + where + rejectUnique x (seen, acc) + | seen ^. contains x = (seen, False) + | otherwise = (seen & contains x .~ True, acc) + +-- | Returns true if the given 'Foldable' container contains only unique +-- elements, as determined by the 'Ord' instance for @a@ +-- +-- >>> isUnique ([3, 1, 2] :: [Int]) +-- True +-- +-- >>> isUnique ([1, 1, 2, 2, 3, 1] :: [Int]) +-- False +isUnique :: (Foldable f, Ord a) => f a -> Bool +isUnique = isUniqueOf folded + + +-- | O(n * log n). Returns a monoidal, 'Cons'able container (a list, a Set, +-- etc.) consisting of the unique (per the 'Ord' instance for @a@) targets of +-- the given 'Fold' +-- +-- >>> uniqueOf (folded . _2) ([(1, 2), (2, 2), (3, 2), (4, 3)] :: [(Int, Int)]) :: [Int] +-- [2,3] +-- +-- @ +-- 'uniqueOf' :: Ord a => 'Getter' s a -> s -> [a] +-- 'uniqueOf' :: Ord a => 'Fold' s a -> s -> [a] +-- 'uniqueOf' :: Ord a => 'Lens'' s a -> s -> [a] +-- 'uniqueOf' :: Ord a => 'Iso'' s a -> s -> [a] +-- 'uniqueOf' :: Ord a => 'Traversal'' s a -> s -> [a] +-- 'uniqueOf' :: Ord a => 'Prism'' s a -> s -> [a] +-- @ +uniqueOf + :: (Monoid c, Ord w, Cons c c w w) => Getting (Endo (Set w, c)) a w -> a -> c +uniqueOf aFold = snd . foldrOf aFold rejectUnique (mempty, mempty) + where + rejectUnique x (seen, acc) + | seen ^. contains x = (seen, acc) + | otherwise = (seen & contains x .~ True, cons x acc) + +-- | Returns a monoidal, 'Cons'able container (a list, a Set, etc.) consisting +-- of the unique (per the 'Ord' instance for @a@) contents of the given +-- 'Foldable' container +-- +-- >>> unique [1, 1, 2, 2, 3, 1] :: [Int] +-- [2,3,1] + +-- >>> unique [1, 1, 2, 2, 3, 1] :: Set Int +-- fromList [3,2,1] +unique :: (Foldable f, Cons c c a a, Ord a, Monoid c) => f a -> c +unique = uniqueOf folded + +-------------------------------------------------------------------------------- + +-- | O(n). Returns a monoidal, 'Cons'able container (a list, a Vector, etc.) +-- consisting of the targets of the given 'Fold' with sequential duplicate +-- elements removed +-- +-- This function (sorry for the confusing name) differs from 'uniqueOf' in that +-- it only compares /sequentially/ duplicate elements (and thus operates in +-- linear time). +-- cf 'Data.Vector.uniq' and POSIX @uniq@ for the name +-- +-- >>> uniqOf (folded . _2) ([(1, 2), (2, 2), (3, 1), (4, 2)] :: [(Int, Int)]) :: [Int] +-- [2,1,2] +-- +-- @ +-- 'uniqOf' :: Eq a => 'Getter' s a -> s -> [a] +-- 'uniqOf' :: Eq a => 'Fold' s a -> s -> [a] +-- 'uniqOf' :: Eq a => 'Lens'' s a -> s -> [a] +-- 'uniqOf' :: Eq a => 'Iso'' s a -> s -> [a] +-- 'uniqOf' :: Eq a => 'Traversal'' s a -> s -> [a] +-- 'uniqOf' :: Eq a => 'Prism'' s a -> s -> [a] +-- @ +uniqOf :: (Monoid c, Cons c c w w, Eq w) => Getting (Endo (Maybe w, c)) a w -> a -> c +uniqOf aFold = snd . foldrOf aFold rejectSeen (Nothing, mempty) + where + rejectSeen x (Nothing, acc) = (Just x, x <| acc) + rejectSeen x tup@(Just a, acc) + | x == a = tup + | otherwise = (Just x, x <| acc) + +-- | O(n). Returns a monoidal, 'Cons'able container (a list, a Vector, etc.) +-- consisting of the targets of the given 'Foldable' container with sequential +-- duplicate elements removed +-- +-- This function (sorry for the confusing name) differs from 'unique' in that +-- it only compares /sequentially/ unique elements (and thus operates in linear +-- time). +-- cf 'Data.Vector.uniq' and POSIX @uniq@ for the name +-- +-- >>> uniq [1, 1, 1, 2, 2, 2, 3, 3, 1] :: [Int] +-- [1,2,3,1] +-- +-- >>> uniq [1, 1, 1, 2, 2, 2, 3, 3, 1] :: Vector Int +-- [1,2,3,1] +-- +uniq :: (Foldable f, Eq a, Cons c c a a, Monoid c) => f a -> c +uniq = uniqOf folded + +-- | Like 'takeWhile', but inclusive +takeWhileInclusive :: (a -> Bool) -> [a] -> [a] +takeWhileInclusive _ [] = [] +takeWhileInclusive p (x:xs) = x : if p x then takeWhileInclusive p xs else [] + +-- | Returns the smallest value not in a list +smallestNotIn :: (Ord a, Bounded a, Enum a) => [a] -> a +smallestNotIn xs = case uniq $ sort xs of + [] -> minBound + xs'@(x : _) + | x > minBound -> minBound + | otherwise + -> snd . headEx . filter (uncurry (/=)) $ zip (xs' ++ [minBound]) [minBound..] + +-- | Remove the element at the given index, if any, from the given vector +removeVectorIndex :: Int -> Vector a -> Vector a +removeVectorIndex idx vect = + let (before, after) = V.splitAt idx vect + in before <> fromMaybe Empty (tailMay after) + +-- | Remove the first element in a sequence that matches a given predicate +removeFirst :: IsSequence seq => (Element seq -> Bool) -> seq -> seq +removeFirst p + = flip evalState False + . filterM (\x -> do + found <- get + let matches = p x + when matches $ put True + pure $ found || not matches) + +maximum1 :: (Ord a, Foldable1 f) => f a -> a +maximum1 = getMax . foldMap1 Max + +minimum1 :: (Ord a, Foldable1 f) => f a -> a +minimum1 = getMin . foldMap1 Min + +times :: (Applicative f, Num n, Enum n) => n -> (n -> f b) -> f [b] +times n f = traverse f [1..n] + +times_ :: (Applicative f, Num n, Enum n) => n -> f a -> f [a] +times_ n fa = times n (const fa) + +-- | Multiply an endomorphism by an integral +-- +-- >>> endoTimes (4 :: Int) succ (5 :: Int) +-- 9 +endoTimes :: Integral n => n -> (a -> a) -> a -> a +endoTimes n f = appEndo $ stimes n (Endo f) + +-------------------------------------------------------------------------------- + +-- | This class gives a boolean associated with a type-level bool, a'la +-- 'KnownSymbol', 'KnownNat' etc. +class KnownBool (bool :: Bool) where + boolVal' :: forall proxy. proxy bool -> Bool + boolVal' _ = boolVal @bool + + boolVal :: Bool + boolVal = boolVal' $ Proxy @bool + +instance KnownBool 'True where boolVal = True +instance KnownBool 'False where boolVal = False + +-------------------------------------------------------------------------------- + +-- | Modify some monadic state via the application of a kleisli endomorphism on +-- the state itself +-- +-- Note that any changes made to the state during execution of @k@ will be +-- overwritten +-- +-- @@ +-- modifyK pure === pure () +-- @@ +modifyK :: MonadState s m => (s -> m s) -> m () +modifyK k = get >>= k >>= put + +-- | Modify some monadic state via the application of a kleisli endomorphism on +-- the target of a lens +-- +-- Note that any changes made to the state during execution of @k@ will be +-- overwritten +-- +-- @@ +-- modifyKL id pure === pure () +-- @@ +modifyKL :: MonadState s m => LensLike m s s a b -> (a -> m b) -> m () +modifyKL l k = get >>= traverseOf l k >>= put + +-- | Use a list of all the targets of a 'Fold' in the current state +-- +-- @@ +-- evalState (useListOf folded) === toList +-- @@ +useListOf :: MonadState s m => Getting (Endo [a]) s a -> m [a] +useListOf = gets . toListOf + +-------------------------------------------------------------------------------- + +-- | A newtype wrapper around 'Char' whose 'Enum' and 'Bounded' instances only +-- include the characters @[a-zA-Z]@ +-- +-- >>> succ (AlphaChar 'z') +-- 'A' +newtype AlphaChar = AlphaChar { getAlphaChar :: Char } + deriving stock Show + deriving (Eq, Ord) via Char + +instance Enum AlphaChar where + toEnum n + | between 0 25 n + = AlphaChar . toEnum $ n + fromEnum 'a' + | between 26 51 n + = AlphaChar . toEnum $ n - 26 + fromEnum 'A' + | otherwise + = error $ "Tag " <> show n <> " out of range [0, 51] for enum AlphaChar" + fromEnum (AlphaChar chr) + | between 'a' 'z' chr + = fromEnum chr - fromEnum 'a' + | between 'A' 'Z' chr + = fromEnum chr - fromEnum 'A' + | otherwise + = error $ "Invalid value for alpha char: " <> show chr + +instance Bounded AlphaChar where + minBound = AlphaChar 'a' + maxBound = AlphaChar 'Z' diff --git a/users/grfn/xanthous/src/Xanthous/Util/Comonad.hs b/users/grfn/xanthous/src/Xanthous/Util/Comonad.hs new file mode 100644 index 000000000000..9e158cc8e2d4 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/Comonad.hs @@ -0,0 +1,24 @@ +-------------------------------------------------------------------------------- +module Xanthous.Util.Comonad + ( -- * Store comonad utils + replace + , current + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Control.Comonad.Store.Class +-------------------------------------------------------------------------------- + +-- | Replace the current position of a store comonad with a new value by +-- comparing positions +replace :: (Eq i, ComonadStore i w) => w a -> a -> w a +replace w x = w =>> \w' -> if pos w' == pos w then x else extract w' +{-# INLINE replace #-} + +-- | Lens into the current position of a store comonad. +-- +-- current = lens extract replace +current :: (Eq i, ComonadStore i w) => Lens' (w a) a +current = lens extract replace +{-# INLINE current #-} diff --git a/users/grfn/xanthous/src/Xanthous/Util/Graph.hs b/users/grfn/xanthous/src/Xanthous/Util/Graph.hs new file mode 100644 index 000000000000..8e5c04f4bfa9 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/Graph.hs @@ -0,0 +1,33 @@ +-------------------------------------------------------------------------------- +module Xanthous.Util.Graph where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Graph.Inductive.Query.MST (msTree) +import qualified Data.Graph.Inductive.Graph as Graph +import Data.Graph.Inductive.Graph +import Data.Graph.Inductive.Basic (undir) +import Data.Set (isSubsetOf) +-------------------------------------------------------------------------------- + +mstSubGraph + :: forall gr node edge. (DynGraph gr, Real edge, Show edge) + => gr node edge -> gr node edge +mstSubGraph graph = insEdges mstEdges . insNodes (labNodes graph) $ Graph.empty + where + mstEdges = ordNub $ do + LP path <- msTree $ undir graph + case path of + [] -> [] + [_] -> [] + ((n₂, edgeWeight) : (n₁, _) : _) -> + pure (n₁, n₂, edgeWeight) + +isSubGraphOf + :: (Graph gr1, Graph gr2, Ord node, Ord edge) + => gr1 node edge + -> gr2 node edge + -> Bool +isSubGraphOf graph₁ graph₂ + = setFromList (labNodes graph₁) `isSubsetOf` setFromList (labNodes graph₂) + && setFromList (labEdges graph₁) `isSubsetOf` setFromList (labEdges graph₂) diff --git a/users/grfn/xanthous/src/Xanthous/Util/Graphics.hs b/users/grfn/xanthous/src/Xanthous/Util/Graphics.hs new file mode 100644 index 000000000000..0cb009f45ad0 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/Graphics.hs @@ -0,0 +1,177 @@ +{-# LANGUAGE TemplateHaskell #-} +-- | Graphics algorithms and utils for rendering things in 2D space +-------------------------------------------------------------------------------- +module Xanthous.Util.Graphics + ( circle + , filledCircle + , line + , straightLine + , delaunay + + -- * Debugging and testing tools + , renderBooleanGraphics + , showBooleanGraphics + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +-- https://github.com/noinia/hgeometry/issues/28 +-- import qualified Algorithms.Geometry.DelaunayTriangulation.DivideAndConquer +-- as Geometry +import qualified Algorithms.Geometry.DelaunayTriangulation.Naive + as Geometry +import qualified Algorithms.Geometry.DelaunayTriangulation.Types as Geometry +import Control.Monad.State (execState, State) +import qualified Data.Geometry.Point as Geometry +import Data.Ext ((:+)(..)) +import Data.List (unfoldr) +import Data.List.NonEmpty (NonEmpty((:|))) +import qualified Data.List.NonEmpty as NE +import Data.Ix (Ix) +import Linear.V2 +-------------------------------------------------------------------------------- + + +-- | Generate a circle centered at the given point and with the given radius +-- using the <midpoint circle algorithm +-- https://en.wikipedia.org/wiki/Midpoint_circle_algorithm>. +-- +-- Code taken from <https://rosettacode.org/wiki/Bitmap/Midpoint_circle_algorithm#Haskell> +circle :: (Num i, Ord i) + => V2 i -- ^ center + -> i -- ^ radius + -> [V2 i] +circle (V2 x₀ y₀) radius + -- Four initial points, plus the generated points + = V2 x₀ (y₀ + radius) + : V2 x₀ (y₀ - radius) + : V2 (x₀ + radius) y₀ + : V2 (x₀ - radius) y₀ + : points + where + -- Creates the (x, y) octet offsets, then maps them to absolute points in all octets. + points = concatMap generatePoints $ unfoldr step initialValues + + generatePoints (V2 x y) + = [ V2 (x₀ `xop` x') (y₀ `yop` y') + | (x', y') <- [(x, y), (y, x)] + , xop <- [(+), (-)] + , yop <- [(+), (-)] + ] + + initialValues = (1 - radius, 1, (-2) * radius, 0, radius) + + step (f, ddf_x, ddf_y, x, y) + | x >= y = Nothing + | otherwise = Just (V2 x' y', (f', ddf_x', ddf_y', x', y')) + where + (f', ddf_y', y') | f >= 0 = (f + ddf_y' + ddf_x', ddf_y + 2, y - 1) + | otherwise = (f + ddf_x, ddf_y, y) + ddf_x' = ddf_x + 2 + x' = x + 1 + + +data FillState i + = FillState + { _inCircle :: Bool + , _result :: NonEmpty (V2 i) + } +makeLenses ''FillState + +runFillState :: NonEmpty (V2 i) -> State (FillState i) a -> [V2 i] +runFillState circumference s + = toList + . view result + . execState s + $ FillState False circumference + +-- | Generate a *filled* circle centered at the given point and with the given +-- radius by filling a circle generated with 'circle' +filledCircle :: (Num i, Integral i, Ix i) + => V2 i -- ^ center + -> i -- ^ radius + -> [V2 i] +filledCircle center radius = + case NE.nonEmpty (circle center radius) of + Nothing -> [] + Just circumference -> runFillState circumference $ + -- the first and last lines of all circles are solid, so the whole "in the + -- circle, out of the circle" thing doesn't work... but that's fine since + -- we don't need to fill them. So just skip them + for_ [succ minX..pred maxX] $ \x -> + for_ [minY..maxY] $ \y -> do + let pt = V2 x y + next = V2 x $ succ y + whenM (use inCircle) $ result %= NE.cons pt + + when (pt `elem` circumference && next `notElem` circumference) + $ inCircle %= not + + where + (V2 minX minY, V2 maxX maxY) = minmaxes circumference + +-- | Draw a line between two points using Bresenham's line drawing algorithm +-- +-- Code taken from <https://wiki.haskell.org/Bresenham%27s_line_drawing_algorithm> +line :: (Num i, Ord i) => V2 i -> V2 i -> [V2 i] +line pa@(V2 xa ya) pb@(V2 xb yb) + = (if maySwitch pa < maySwitch pb then id else reverse) points + where + points = map maySwitch . unfoldr go $ (x₁, y₁, 0) + steep = abs (yb - ya) > abs (xb - xa) + maySwitch = if steep then view _yx else id + [V2 x₁ y₁, V2 x₂ y₂] = sort [maySwitch pa, maySwitch pb] + δx = x₂ - x₁ + δy = abs (y₂ - y₁) + ystep = if y₁ < y₂ then 1 else -1 + go (xTemp, yTemp, err) + | xTemp > x₂ = Nothing + | otherwise = Just (V2 xTemp yTemp, (xTemp + 1, newY, newError)) + where + tempError = err + δy + (newY, newError) = if (2 * tempError) >= δx + then (yTemp + ystep, tempError - δx) + else (yTemp, tempError) +{-# SPECIALIZE line :: V2 Int -> V2 Int -> [V2 Int] #-} +{-# SPECIALIZE line :: V2 Word -> V2 Word -> [V2 Word] #-} + +straightLine :: (Num i, Ord i) => V2 i -> V2 i -> [V2 i] +straightLine pa@(V2 xa _) pb@(V2 _ yb) = line pa midpoint ++ line midpoint pb + where midpoint = V2 xa yb + +delaunay + :: (Ord n, Fractional n) + => NonEmpty (V2 n, p) + -> [((V2 n, p), (V2 n, p))] +delaunay + = map (over both fromPoint) + . Geometry.edgesAsPoints + . Geometry.delaunayTriangulation + . map toPoint + where + toPoint (V2 px py, pid) = Geometry.Point2 px py :+ pid + fromPoint (Geometry.Point2 px py :+ pid) = (V2 px py, pid) + +-------------------------------------------------------------------------------- + +renderBooleanGraphics :: forall i. (Num i, Ord i, Enum i) => [V2 i] -> String +renderBooleanGraphics [] = "" +renderBooleanGraphics (pt : pts') = intercalate "\n" rows + where + rows = row <$> [minX..maxX] + row x = [minY..maxY] <&> \y -> if V2 x y `member` ptSet then 'X' else ' ' + (V2 minX minY, V2 maxX maxY) = minmaxes pts + pts = pt :| pts' + ptSet :: Set (V2 i) + ptSet = setFromList $ toList pts + +showBooleanGraphics :: forall i. (Num i, Ord i, Enum i) => [V2 i] -> IO () +showBooleanGraphics = putStrLn . pack . renderBooleanGraphics + +minmaxes :: forall i. (Ord i) => NonEmpty (V2 i) -> (V2 i, V2 i) +minmaxes xs = + ( V2 (minimum1Of (traverse1 . _x) xs) + (minimum1Of (traverse1 . _y) xs) + , V2 (maximum1Of (traverse1 . _x) xs) + (maximum1Of (traverse1 . _y) xs) + ) diff --git a/users/grfn/xanthous/src/Xanthous/Util/Inflection.hs b/users/grfn/xanthous/src/Xanthous/Util/Inflection.hs new file mode 100644 index 000000000000..724f2339dd21 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/Inflection.hs @@ -0,0 +1,14 @@ + +module Xanthous.Util.Inflection + ( toSentence + ) where + +import Xanthous.Prelude + +toSentence :: (MonoFoldable mono, Element mono ~ Text) => mono -> Text +toSentence xs = case reverse . toList $ xs of + [] -> "" + [x] -> x + [b, a] -> a <> " and " <> b + (final : butlast) -> + intercalate ", " (reverse butlast) <> ", and " <> final diff --git a/users/grfn/xanthous/src/Xanthous/Util/JSON.hs b/users/grfn/xanthous/src/Xanthous/Util/JSON.hs new file mode 100644 index 000000000000..91d1328e4a10 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/JSON.hs @@ -0,0 +1,19 @@ +-------------------------------------------------------------------------------- +module Xanthous.Util.JSON + ( ReadShowJSON(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import Data.Aeson +-------------------------------------------------------------------------------- + +newtype ReadShowJSON a = ReadShowJSON a + deriving newtype (Read, Show) + +instance Show a => ToJSON (ReadShowJSON a) where + toJSON = toJSON . show + +instance Read a => FromJSON (ReadShowJSON a) where + parseJSON = withText "readable" + $ maybe (fail "Could not read") pure . readMay diff --git a/users/grfn/xanthous/src/Xanthous/Util/Optparse.hs b/users/grfn/xanthous/src/Xanthous/Util/Optparse.hs new file mode 100644 index 000000000000..dfa65372351d --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/Optparse.hs @@ -0,0 +1,21 @@ +-------------------------------------------------------------------------------- +module Xanthous.Util.Optparse + ( readWithGuard + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +-------------------------------------------------------------------------------- +import qualified Options.Applicative as Opt +-------------------------------------------------------------------------------- + +readWithGuard + :: Read b + => (b -> Bool) + -> (b -> String) + -> Opt.ReadM b +readWithGuard predicate errmsg = do + res <- Opt.auto + unless (predicate res) + $ Opt.readerError + $ errmsg res + pure res diff --git a/users/grfn/xanthous/src/Xanthous/Util/QuickCheck.hs b/users/grfn/xanthous/src/Xanthous/Util/QuickCheck.hs new file mode 100644 index 000000000000..be12bc294513 --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/Util/QuickCheck.hs @@ -0,0 +1,42 @@ +{-# LANGUAGE UndecidableInstances #-} +module Xanthous.Util.QuickCheck + ( functionShow + , FunctionShow(..) + , functionJSON + , FunctionJSON(..) + , genericArbitrary + , GenericArbitrary(..) + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude +import Test.QuickCheck +import Test.QuickCheck.Function +import Test.QuickCheck.Instances.ByteString () +import Test.QuickCheck.Arbitrary.Generic +import Data.Aeson +import GHC.Generics (Rep) +-------------------------------------------------------------------------------- + +newtype FunctionShow a = FunctionShow a + deriving newtype (Show, Read) + +instance (Show a, Read a) => Function (FunctionShow a) where + function = functionShow + +functionJSON :: (ToJSON a, FromJSON a) => (a -> c) -> a :-> c +functionJSON = functionMap encode (headEx . decode) + +newtype FunctionJSON a = FunctionJSON a + deriving newtype (ToJSON, FromJSON) + +instance (ToJSON a, FromJSON a) => Function (FunctionJSON a) where + function = functionJSON + +-------------------------------------------------------------------------------- + +newtype GenericArbitrary a = GenericArbitrary a + deriving newtype Generic + +instance (Generic a, GArbitrary rep, Rep a ~ rep) + => Arbitrary (GenericArbitrary a) where + arbitrary = genericArbitrary diff --git a/users/grfn/xanthous/src/Xanthous/messages.yaml b/users/grfn/xanthous/src/Xanthous/messages.yaml new file mode 100644 index 000000000000..27ee841dd9db --- /dev/null +++ b/users/grfn/xanthous/src/Xanthous/messages.yaml @@ -0,0 +1,161 @@ +welcome: Welcome to Xanthous, {{characterName}}! It's dangerous out there, why not stay inside? Use hjklybnu to move. +dead: + - You have died... + - You die... + - You perish... + - You have perished... + +generic: + continue: Press enter to continue... + +save: + disabled: "Sorry, saving is currently disabled" + location: "Enter filename to save to: " + overwrite: "A file named {{filename}} already exists. Would you like to overwrite it? " + +quit: + confirm: Really quit without saving? + +entities: + description: You see here {{entityDescriptions}} + say: + creature: + visible: The {{creature.creatureType.name}} {{creature.creatureType.sayVerb}} "{{message}}" + invisible: You hear something yell "{{message}}" in the distance + +pickUp: + menu: What would you like to pick up? + pickUp: You pick up the {{item.itemType.name}}. + nothingToPickUp: "There's nothing here to pick up" + +cant: + goUp: + - You can't go up here + - There's nothing here that would let you go up + goDown: + - You can't go down here + - There's nothing here that would let you go down + +open: + prompt: Direction to open (hjklybnu.)? + success: "You open the door." + locked: "That door is locked" + nothingToOpen: "There's nothing to open there." + alreadyOpen: "That door is already open." + +close: + prompt: Direction to close (hjklybnu.)? + success: + - You close the door. + - You shut the door. + nothingToClose: "There's nothing to close there." + alreadyClosed: "That door is already closed." + blocked: "The {{entityDescriptions}} {{blockOrBlocks}} the door!" + +look: + prompt: Select a position on the map to describe (use Enter to confirm) + nothing: There's nothing there + +character: + namePrompt: "What's your name? " + body: + knuckles: + calluses: + - You've started developing calluses on your knuckles from all the punching you've been doing. + - You've been fighting with your fists so much they're starting to develop calluses. + +combat: + nothingToAttack: There's nothing to attack there. + menu: Which creature would you like to attack? + fistSelfDamage: + - You hit so hard with your fists you hurt yourself! + - The punch leaves your knuckles bloody! + fistExtraSelfDamage: + - You hurt your already-bloody fists with the strike! + - Ouch! Your fists were already bleeding! + hit: + fists: + - You punch the {{creature.creatureType.name}} with your bare fists! It hurts. A lot. + - You strike the {{creature.creatureType.name}} with your bare fists! It leaves a bit of a bruise on your knuckles. + generic: + - You hit the {{creature.creatureType.name}}. + - You attack the {{creature.creatureType.name}}. + creatureAttack: + natural: The {{creature.creatureType.name}} {{attackDescription}}. + genericWeapon: The {{creature.creatureType.name}} attacks you with its {{item.itemType.name}}. + killed: + - You kill the {{creature.creatureType.name}}! + - You've killed the {{creature.creatureType.name}}! + +debug: + toggleRevealAll: revealAll now set to {{revealAll}} + +eat: + noFood: + - You have nothing edible. + - You don't have any food. + - You don't have anything to eat. + - You search your pockets for something edible, and come up short. + menuPrompt: What would you like to eat? + eat: You eat the {{item.itemType.name}}. + +read: + prompt: Direction to read (hjklybnu.)? + nothing: "There's nothing there to read" + result: "\"{{message}}\"" + +inventory: + describe: + select: Select an item in your inventory to describe + nothing: You aren't carrying anything + +wield: + nothing: + - You aren't carrying anything you can wield + - You can't wield anything in your backpack + - You can't wield anything currently in your backpack + menu: What would you like to wield? + # TODO: use actual hands + wielded : You wield the {{wieldedItem.itemType.name}} in your right hand. + +fire: + nothing: + - You don't currently have anything you can throw + - You don't have anything to throw + zeroRange: + - That item is too heavy to throw! + - That's too heavy to throw + - You're not strong enough to throw that any meaningful distance + menu: What would you like to throw? + target: Choose a target + atRange: + - It's too heavy for you to throw any further than this + fired: + noTarget: + - You throw the {{item.itemType.name}} at the ground + noDamage: + - You throw the {{item.itemType.name}} at the {{creature.creatureType.name}}. It doesn't seem to care. + - You throw the {{item.itemType.name}} at the {{creature.creatureType.name}}. It doesn't seem to do anything. + - You throw the {{item.itemType.name}} at the {{creature.creatureType.name}}. It doesn't seem to hurt it. + someDamage: + - You throw the {{item.itemType.name}} at the {{creature.creatureType.name}}. It hits it on the head!. + +drop: + nothing: You aren't carrying anything + menu: What would you like to drop? + # TODO: use actual hands + dropped: + - You drop the {{item.itemType.name}}. + - You drop the {{item.itemType.name}} on the ground. + - You put the {{item.itemType.name}} on the ground. + - You take the {{item.itemType.name}} out of your backpack and put it on the ground. + - You take the {{item.itemType.name}} out of your backpack and drop it on the ground. + +autocommands: + enemyInSight: There's a {{firstEntity.creatureType.name}} nearby! + resting: Resting... + doneResting: Done resting +### + +tutorial: + message1: The caves are dark and full of nightmarish creatures - and you are likely to perish without food. Seek out sustenance! You can pick items up with ,. diff --git a/users/grfn/xanthous/test/Spec.hs b/users/grfn/xanthous/test/Spec.hs new file mode 100644 index 000000000000..64c10cf21e20 --- /dev/null +++ b/users/grfn/xanthous/test/Spec.hs @@ -0,0 +1,59 @@ +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import qualified Xanthous.Data.EntitiesSpec +import qualified Xanthous.Data.EntityCharSpec +import qualified Xanthous.Data.EntityMap.GraphicsSpec +import qualified Xanthous.Data.EntityMapSpec +import qualified Xanthous.Data.LevelsSpec +import qualified Xanthous.Data.MemoSpec +import qualified Xanthous.Data.NestedMapSpec +import qualified Xanthous.DataSpec +import qualified Xanthous.Entities.CommonSpec +import qualified Xanthous.Entities.RawsSpec +import qualified Xanthous.Entities.RawTypesSpec +import qualified Xanthous.Entities.CharacterSpec +import qualified Xanthous.GameSpec +import qualified Xanthous.Game.StateSpec +import qualified Xanthous.Game.PromptSpec +import qualified Xanthous.Generators.Level.UtilSpec +import qualified Xanthous.MessageSpec +import qualified Xanthous.Messages.TemplateSpec +import qualified Xanthous.OrphansSpec +import qualified Xanthous.RandomSpec +import qualified Xanthous.Util.GraphSpec +import qualified Xanthous.Util.GraphicsSpec +import qualified Xanthous.Util.InflectionSpec +import qualified Xanthous.UtilSpec +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMainWithRerun test + +test :: TestTree +test = testGroup "Xanthous" + [ Xanthous.Data.EntitiesSpec.test + , Xanthous.Data.EntityMap.GraphicsSpec.test + , Xanthous.Data.EntityMapSpec.test + , Xanthous.Data.LevelsSpec.test + , Xanthous.Data.MemoSpec.test + , Xanthous.Data.NestedMapSpec.test + , Xanthous.DataSpec.test + , Xanthous.Entities.CommonSpec.test + , Xanthous.Entities.RawsSpec.test + , Xanthous.Entities.CharacterSpec.test + , Xanthous.Entities.RawTypesSpec.test + , Xanthous.GameSpec.test + , Xanthous.Game.StateSpec.test + , Xanthous.Game.PromptSpec.test + , Xanthous.Generators.Level.UtilSpec.test + , Xanthous.MessageSpec.test + , Xanthous.Messages.TemplateSpec.test + , Xanthous.OrphansSpec.test + , Xanthous.RandomSpec.test + , Xanthous.Util.GraphSpec.test + , Xanthous.Util.GraphicsSpec.test + , Xanthous.Util.InflectionSpec.test + , Xanthous.UtilSpec.test + , Xanthous.Data.EntityCharSpec.test + ] diff --git a/users/grfn/xanthous/test/Test/Prelude.hs b/users/grfn/xanthous/test/Test/Prelude.hs new file mode 100644 index 000000000000..75c1ebf5e76a --- /dev/null +++ b/users/grfn/xanthous/test/Test/Prelude.hs @@ -0,0 +1,34 @@ +{-# LANGUAGE AllowAmbiguousTypes #-} +-------------------------------------------------------------------------------- +module Test.Prelude + ( module Xanthous.Prelude + , module Test.Tasty + , module Test.Tasty.HUnit + , module Test.Tasty.QuickCheck + , module Test.Tasty.Ingredients.Rerun + , module Test.QuickCheck.Classes + , testBatch + , jsonRoundTrip + ) where +-------------------------------------------------------------------------------- +import Xanthous.Prelude hiding (assert, elements) +-------------------------------------------------------------------------------- +import Test.Tasty +import Test.Tasty.QuickCheck +import Test.Tasty.HUnit +import Test.Tasty.Ingredients.Rerun +import Test.QuickCheck.Classes +import Test.QuickCheck.Checkers (TestBatch, EqProp ((=-=))) +import Test.QuickCheck.Instances.ByteString () +-------------------------------------------------------------------------------- +import qualified Data.Aeson as JSON +import Data.Aeson (ToJSON, FromJSON) +-------------------------------------------------------------------------------- + +testBatch :: TestBatch -> TestTree +testBatch (name, tests) = testGroup name $ uncurry testProperty <$> tests + +jsonRoundTrip + :: forall a. (ToJSON a, FromJSON a, EqProp a, Arbitrary a, Show a) => TestTree +jsonRoundTrip = testProperty "JSON round trip" $ \(x :: a) -> + JSON.decode (JSON.encode x) =-= Just x diff --git a/users/grfn/xanthous/test/Xanthous/Data/EntitiesSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/EntitiesSpec.hs new file mode 100644 index 000000000000..e403503743c0 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/EntitiesSpec.hs @@ -0,0 +1,28 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.EntitiesSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import qualified Data.Aeson as JSON +-------------------------------------------------------------------------------- +import Xanthous.Data.Entities +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data.Entities" + [ testGroup "Collision" + [ testProperty "JSON round-trip" $ \(c :: Collision) -> + JSON.decode (JSON.encode c) === Just c + , testGroup "JSON encoding examples" + [ testCase "Stop" $ JSON.encode Stop @?= "\"Stop\"" + , testCase "Combat" $ JSON.encode Combat @?= "\"Combat\"" + ] + ] + , testGroup "EntityAttributes" + [ testProperty "JSON round-trip" $ \(ea :: EntityAttributes) -> + JSON.decode (JSON.encode ea) === Just ea + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Data/EntityCharSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/EntityCharSpec.hs new file mode 100644 index 000000000000..9e8024c9d223 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/EntityCharSpec.hs @@ -0,0 +1,18 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.EntityCharSpec where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import qualified Data.Aeson as JSON +-------------------------------------------------------------------------------- +import Xanthous.Data.EntityChar +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data.EntityChar" + [ testProperty "JSON round-trip" $ \(ec :: EntityChar) -> + JSON.decode (JSON.encode ec) === Just ec + ] diff --git a/users/grfn/xanthous/test/Xanthous/Data/EntityMap/GraphicsSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/EntityMap/GraphicsSpec.hs new file mode 100644 index 000000000000..fd37548ce864 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/EntityMap/GraphicsSpec.hs @@ -0,0 +1,57 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.EntityMap.GraphicsSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +import Data.Aeson +-------------------------------------------------------------------------------- +import Xanthous.Game.State +import Xanthous.Data +import Xanthous.Data.EntityMap +import Xanthous.Data.EntityMap.Graphics +import Xanthous.Entities.Environment (Wall(..)) +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data.EntityMap.Graphics" + [ testGroup "visiblePositions" + [ testProperty "one step in each cardinal direction is always visible" + $ \pos (Cardinal dir) (Positive r) (wallPositions :: Set Position)-> + pos `notMember` wallPositions ==> + let em = review _EntityMap . map (, Wall) . toList $ wallPositions + em' = em & atPosition (move dir pos) %~ (Wall <|) + poss = visiblePositions pos r em' + in counterexample ("visiblePositions: " <> show poss) + $ move dir pos `member` poss + , testGroup "bugs" + [ testCase "non-contiguous bug 1" + $ let charPos = Position 20 20 + gormlakPos = Position 17 19 + em = insertAt gormlakPos TestEntity + . insertAt charPos TestEntity + $ mempty + visPositions = visiblePositions charPos 12 em + in (gormlakPos `member` visPositions) @? + ( "not (" + <> show gormlakPos <> " `member` " + <> show visPositions + <> ")" + ) + ] + ] + ] + +-------------------------------------------------------------------------------- + +data TestEntity = TestEntity + deriving stock (Show, Eq, Ord, Generic) + deriving anyclass (ToJSON, FromJSON, NFData) + +instance Brain TestEntity where + step _ = pure +instance Draw TestEntity +instance Entity TestEntity where + description _ = "" + entityChar _ = "e" diff --git a/users/grfn/xanthous/test/Xanthous/Data/EntityMapSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/EntityMapSpec.hs new file mode 100644 index 000000000000..7c5cad019616 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/EntityMapSpec.hs @@ -0,0 +1,69 @@ +{-# LANGUAGE ApplicativeDo #-} +-------------------------------------------------------------------------------- +module Xanthous.Data.EntityMapSpec where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import qualified Data.Aeson as JSON +-------------------------------------------------------------------------------- +import Xanthous.Data.EntityMap +import Xanthous.Data (Positioned(..)) +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = localOption (QuickCheckTests 20) + $ testGroup "Xanthous.Data.EntityMap" + [ testBatch $ monoid @(EntityMap Int) mempty + , testGroup "Deduplicate" + [ testGroup "Semigroup laws" + [ testProperty "associative" $ \(a :: Deduplicate (EntityMap Int)) b c -> + a <> (b <> c) === (a <> b) <> c + ] + ] + , testGroup "Eq laws" + [ testProperty "reflexivity" $ \(em :: EntityMap Int) -> + em == em + , testProperty "symmetric" $ \(em₁ :: EntityMap Int) em₂ -> + (em₁ == em₂) == (em₂ == em₁) + , testProperty "transitive" $ \(em₁ :: EntityMap Int) em₂ em₃ -> + if (em₁ == em₂ && em₂ == em₃) + then (em₁ == em₃) + else True + ] + , testGroup "JSON encoding/decoding" + [ testProperty "round-trips" $ \(em :: EntityMap Int) -> + let em' = JSON.decode (JSON.encode em) + in counterexample (show (em' ^? _Just . lastID, em ^. lastID + , em' ^? _Just . byID == em ^. byID . re _Just + , em' ^? _Just . byPosition == em ^. byPosition . re _Just + , em' ^? _Just . _EntityMap == em ^. _EntityMap . re _Just + )) + $ em' === Just em + , testProperty "Preserves IDs" $ \(em :: EntityMap Int) -> + let Just em' = JSON.decode $ JSON.encode em + in toEIDsAndPositioned em' === toEIDsAndPositioned em + ] + + , localOption (QuickCheckTests 50) + $ testGroup "atPosition" + [ testProperty "setget" $ \pos (em :: EntityMap Int) es -> + view (atPosition pos) (set (atPosition pos) es em) === es + , testProperty "getset" $ \pos (em :: EntityMap Int) -> + set (atPosition pos) (view (atPosition pos) em) em === em + , testProperty "setset" $ \pos (em :: EntityMap Int) es -> + (set (atPosition pos) es . set (atPosition pos) es) em + === + set (atPosition pos) es em + -- testProperty "lens laws" $ \pos -> isLens $ atPosition @Int pos + , testProperty "preserves IDs" $ \(em :: EntityMap Int) e1 e2 p -> + let (eid, em') = insertAtReturningID p e1 em + em'' = em' & atPosition p %~ (e2 <|) + in + counterexample ("em': " <> show em') + . counterexample ("em'': " <> show em'') + $ em'' ^. at eid === Just (Positioned p e1) + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Data/LevelsSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/LevelsSpec.hs new file mode 100644 index 000000000000..a7528331627d --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/LevelsSpec.hs @@ -0,0 +1,66 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.LevelsSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import qualified Data.Aeson as JSON +-------------------------------------------------------------------------------- +import Xanthous.Util (between) +import Xanthous.Data.Levels +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data.Levels" + [ testGroup "current" + [ testProperty "view is extract" $ \(levels :: Levels Int) -> + levels ^. current === extract levels + , testProperty "set replaces current" $ \(levels :: Levels Int) new -> + extract (set current new levels) === new + , testProperty "set extract is id" $ \(levels :: Levels Int) -> + set current (extract levels) levels === levels + , testProperty "set y ∘ set x ≡ set y" $ \(levels :: Levels Int) x y -> + set current y (set current x levels) === set current y levels + ] + , localOption (QuickCheckTests 20) + $ testBatch $ semigroup @(Levels Int) (error "unused", 1 :: Int) + , testGroup "next/prev" + [ testGroup "nextLevel" + [ testProperty "seeks forwards" $ \(levels :: Levels Int) genned -> + (pos . runIdentity . nextLevel (Identity genned) $ levels) + === pos levels + 1 + , testProperty "maintains the invariant" $ \(levels :: Levels Int) genned -> + let levels' = runIdentity . nextLevel (Identity genned) $ levels + in between 0 (toEnum $ length levels') $ pos levels' + , testProperty "extract is total" $ \(levels :: Levels Int) genned -> + let levels' = runIdentity . nextLevel (Identity genned) $ levels + in total $ extract levels' + , testProperty "uses the generated level as the next level" + $ \(levels :: Levels Int) genned -> + let levels' = seek (toEnum $ length levels - 1) levels + levels'' = runIdentity . nextLevel (Identity genned) $ levels' + in counterexample (show levels'') + $ extract levels'' === genned + ] + , testGroup "prevLevel" + [ testProperty "seeks backwards" $ \(levels :: Levels Int) -> + case prevLevel levels of + Nothing -> property Discard + Just levels' -> pos levels' === pos levels - 1 + , testProperty "maintains the invariant" $ \(levels :: Levels Int) -> + case prevLevel levels of + Nothing -> property Discard + Just levels' -> property $ between 0 (toEnum $ length levels') $ pos levels' + , testProperty "extract is total" $ \(levels :: Levels Int) -> + case prevLevel levels of + Nothing -> property Discard + Just levels' -> total $ extract levels' + ] + ] + , testGroup "JSON" + [ testProperty "toJSON/parseJSON round-trip" $ \(levels :: Levels Int) -> + JSON.decode (JSON.encode levels) === Just levels + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Data/MemoSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/MemoSpec.hs new file mode 100644 index 000000000000..ad81f1984d8f --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/MemoSpec.hs @@ -0,0 +1,19 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.MemoSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +import Test.QuickCheck.Instances.Text () +-------------------------------------------------------------------------------- +import Xanthous.Data.Memo +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data.MemoSpec" + [ testGroup "getMemoized" + [ testProperty "when key matches" $ \k v -> + getMemoized @Int @Int k (memoizeWith k v) === Just v + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Data/NestedMapSpec.hs b/users/grfn/xanthous/test/Xanthous/Data/NestedMapSpec.hs new file mode 100644 index 000000000000..acf7a67268f4 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Data/NestedMapSpec.hs @@ -0,0 +1,20 @@ +-------------------------------------------------------------------------------- +module Xanthous.Data.NestedMapSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Test.QuickCheck.Instances.Semigroup () +-------------------------------------------------------------------------------- +import qualified Xanthous.Data.NestedMap as NM +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data.NestedMap" + [ testProperty "insert/lookup" $ \nm ks v -> + let nm' = NM.insert ks v nm + in counterexample ("inserted: " <> show nm') + $ NM.lookup @Map @Int @Int ks nm' === Just (NM.Val v) + ] diff --git a/users/grfn/xanthous/test/Xanthous/DataSpec.hs b/users/grfn/xanthous/test/Xanthous/DataSpec.hs new file mode 100644 index 000000000000..9e67505ba928 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/DataSpec.hs @@ -0,0 +1,109 @@ +-------------------------------------------------------------------------------- +module Xanthous.DataSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude hiding (Right, Left, Down, toList, all) +import Data.Group +import Data.Foldable (toList, all) +-------------------------------------------------------------------------------- +import Xanthous.Data +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Data" + [ testGroup "Position" + [ testBatch $ monoid @Position mempty + , testProperty "group laws" $ \(pos :: Position) -> + pos <> invert pos == mempty && invert pos <> pos == mempty + , testGroup "stepTowards laws" + [ testProperty "takes only one step" $ \src tgt -> + src /= tgt ==> + isUnit (src `diffPositions` (src `stepTowards` tgt)) + -- , testProperty "moves in the right direction" $ \src tgt -> + -- stepTowards src tgt == move (directionOf src tgt) src + ] + , testProperty "directionOf laws" $ \pos dir -> + directionOf pos (move dir pos) == dir + , testProperty "diffPositions is add inverse" $ \(pos₁ :: Position) pos₂ -> + diffPositions pos₁ pos₂ == addPositions pos₁ (invert pos₂) + , testGroup "isUnit" + [ testProperty "double direction is never unit" $ \dir -> + not . isUnit $ move dir (asPosition dir) + , testCase "examples" $ do + isUnit (Position @Int 1 1) @? "not . isUnit $ Position 1 1" + isUnit (Position @Int 0 (-1)) @? "not . isUnit $ Position 0 (-1)" + (not . isUnit) (Position @Int 1 13) @? "isUnit $ Position 1 13" + ] + ] + + , testGroup "Direction" + [ testProperty "opposite is involutive" $ \(dir :: Direction) -> + opposite (opposite dir) == dir + , testProperty "opposite provides inverse" $ \dir -> + invert (asPosition dir) === asPosition (opposite dir) + , testProperty "asPosition isUnit" $ \dir -> + dir /= Here ==> isUnit (asPosition dir) + , testGroup "Move" + [ testCase "Up" $ move Up mempty @?= Position @Int 0 (-1) + , testCase "Down" $ move Down mempty @?= Position @Int 0 1 + , testCase "Left" $ move Left mempty @?= Position @Int (-1) 0 + , testCase "Right" $ move Right mempty @?= Position @Int 1 0 + , testCase "UpLeft" $ move UpLeft mempty @?= Position @Int (-1) (-1) + , testCase "UpRight" $ move UpRight mempty @?= Position @Int 1 (-1) + , testCase "DownLeft" $ move DownLeft mempty @?= Position @Int (-1) 1 + , testCase "DownRight" $ move DownRight mempty @?= Position @Int 1 1 + ] + ] + + , testGroup "Corner" + [ testGroup "instance Opposite" + [ testProperty "involutive" $ \(corner :: Corner) -> + opposite (opposite corner) === corner + ] + ] + + , testGroup "Edge" + [ testGroup "instance Opposite" + [ testProperty "involutive" $ \(edge :: Edge) -> + opposite (opposite edge) === edge + ] + ] + + , testGroup "Box" + [ testGroup "boxIntersects" + [ testProperty "True" $ \dims -> + boxIntersects (Box @Word (V2 1 1) (V2 2 2)) + (Box (V2 2 2) dims) + , testProperty "False" $ \dims -> + not $ boxIntersects (Box @Word (V2 1 1) (V2 2 2)) + (Box (V2 4 2) dims) + ] + ] + + , testGroup "Neighbors" + [ testGroup "rotations" + [ testProperty "always has the same members" + $ \(neighs :: Neighbors Int) -> + all (\ns -> sort (toList ns) == sort (toList neighs)) + $ rotations neighs + , testProperty "all rotations have the same rotations" + $ \(neighs :: Neighbors Int) -> + let rots = rotations neighs + in all (\ns -> sort (toList $ rotations ns) == sort (toList rots)) + rots + ] + ] + + , testGroup "units" + [ testGroup "unit suffixes" + [ testCase "density" + $ tshow (10000 :: Grams `Per` Cubic Meters) @?= "10000.0 g/m³" + , testCase "volume" + $ tshow (5 :: Cubic Meters) @?= "5.0 m³" + , testCase "area" + $ tshow (5 :: Square Meters) @?= "5.0 m²" + ] + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Entities/CharacterSpec.hs b/users/grfn/xanthous/test/Xanthous/Entities/CharacterSpec.hs new file mode 100644 index 000000000000..734cce1efbbe --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Entities/CharacterSpec.hs @@ -0,0 +1,24 @@ +{-# OPTIONS_GHC -Wno-type-defaults #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.CharacterSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Xanthous.Entities.Character +import Xanthous.Util (endoTimes) +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Entities.CharacterSpec" + [ testGroup "Knuckles" + [ testBatch $ monoid @Knuckles mempty + , testGroup "damageKnuckles" + [ testCase "caps at 5" $ + let knuckles' = endoTimes 6 damageKnuckles mempty + in _knuckleDamage knuckles' @?= 5 + ] + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Entities/CommonSpec.hs b/users/grfn/xanthous/test/Xanthous/Entities/CommonSpec.hs new file mode 100644 index 000000000000..ba27e3cbca4e --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Entities/CommonSpec.hs @@ -0,0 +1,32 @@ +-------------------------------------------------------------------------------- +module Xanthous.Entities.CommonSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +import Data.Vector.Lens (toVectorOf) +-------------------------------------------------------------------------------- +import Xanthous.Entities.Common +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Entities.CommonSpec" + [ testGroup "Inventory" + [ testProperty "items === itemsWithPosition . _2" $ \inv -> + inv ^.. items === inv ^.. itemsWithPosition . _2 + , testGroup "removeItemFromPosition" $ + let rewield w inv = + let (old, inv') = inv & wielded <<.~ w + in inv' & backpack <>~ toVectorOf (wieldedItems . wieldedItem) old + in [ (Backpack, \item -> backpack %~ (item ^. wieldedItem <|)) + , (LeftHand, rewield . inLeftHand) + , (RightHand, rewield . inRightHand) + , (BothHands, rewield . review doubleHanded) + ] <&> \(pos, addItem) -> + testProperty (show pos) $ \inv item -> + let inv' = addItem item inv + inv'' = removeItemFromPosition pos (item ^. wieldedItem) inv' + in inv'' ^.. items === inv ^.. items + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Entities/RawTypesSpec.hs b/users/grfn/xanthous/test/Xanthous/Entities/RawTypesSpec.hs new file mode 100644 index 000000000000..e23f7faba3a6 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Entities/RawTypesSpec.hs @@ -0,0 +1,45 @@ +{-# LANGUAGE RecordWildCards #-} +-------------------------------------------------------------------------------- +module Xanthous.Entities.RawTypesSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Data.Interval (Extended(..), (<=..<=)) +-------------------------------------------------------------------------------- +import Xanthous.Entities.RawTypes +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Entities.RawTypesSpec" + [ testGroup "CreatureGenerateParams" + [ testGroup "Ord laws" + [ testProperty "comparability" $ \(a :: CreatureGenerateParams) b -> + a <= b || b <= a + , testProperty "transitivity" $ \(a :: CreatureGenerateParams) b c -> + a <= b && b <= c ==> a <= c + , testProperty "reflexivity" $ \(a :: CreatureGenerateParams) -> + a <= a + , testProperty "antisymmetry" $ \(a :: CreatureGenerateParams) b -> + (a <= b && b <= a) == (a == b) + ] + , testGroup "canGenerate" $ + let makeParams minB maxB = + let _levelRange = maybe NegInf Finite minB <=..<= maybe PosInf Finite maxB + _equippedItem = Nothing + in CreatureGenerateParams {..} + in + [ testProperty "no bounds" $ \level -> + let gps = makeParams Nothing Nothing + in canGenerate level gps + , testProperty "min bound" $ \level minB -> + let gps = makeParams (Just minB) Nothing + in canGenerate level gps === (level >= minB) + , testProperty "max bound" $ \level maxB -> + let gps = makeParams Nothing (Just maxB) + in canGenerate level gps === (level <= maxB) + ] + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Entities/RawsSpec.hs b/users/grfn/xanthous/test/Xanthous/Entities/RawsSpec.hs new file mode 100644 index 000000000000..b6c80be51be7 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Entities/RawsSpec.hs @@ -0,0 +1,30 @@ +-- | + +module Xanthous.Entities.RawsSpec (main, test) where + +import Test.Prelude +import Xanthous.Entities.Raws +import Xanthous.Entities.RawTypes + (_Creature, entityName, generateParams, HasEquippedItem (equippedItem)) + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Entities.Raws" + [ testGroup "raws" + [ testCase "are all valid" $ raws `deepseq` pure () + , testCase "all CreatureEquippedItems reference existent entity names" $ + let notFound + = raws + ^.. folded + . _Creature + . generateParams + . _Just + . equippedItem + . _Just + . entityName + . filtered (isNothing . raw) + in null notFound @? ("Some entities weren't found: " <> show notFound) + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Game/PromptSpec.hs b/users/grfn/xanthous/test/Xanthous/Game/PromptSpec.hs new file mode 100644 index 000000000000..d7a3df4acafa --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Game/PromptSpec.hs @@ -0,0 +1,19 @@ +-------------------------------------------------------------------------------- +module Xanthous.Game.PromptSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Xanthous.Game.Prompt +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Game.PromptSpec" + [ testGroup "mkMenuItems" + [ testCase "with duplicate items" + $ mkMenuItems @[_] [('a', MenuOption @Int "a" 1), ('a', MenuOption "a" 2)] + @?= mapFromList [('a', MenuOption "a" 1), ('b', MenuOption "a" 2)] + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Game/StateSpec.hs b/users/grfn/xanthous/test/Xanthous/Game/StateSpec.hs new file mode 100644 index 000000000000..34584f73b2ad --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Game/StateSpec.hs @@ -0,0 +1,30 @@ +-------------------------------------------------------------------------------- +module Xanthous.Game.StateSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Xanthous.Game.State +import Xanthous.Entities.Raws (raws) +import Xanthous.Generators.Level.LevelContents (entityFromRaw) +import Control.Monad.Random (evalRandT) +import System.Random (getStdGen) +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Game.StateSpec" + [ testGroup "entityTypeName" + [ testCase "for a creature" $ do + let gormlakRaw = raws ^?! ix "gormlak" + creature <- runRand $ entityFromRaw gormlakRaw + entityTypeName creature @?= "Creature" + , testCase "for an item" $ do + let stickRaw = raws ^?! ix "stick" + item <- runRand $ entityFromRaw stickRaw + entityTypeName item @?= "Item" + ] + ] + where + runRand x = evalRandT x =<< getStdGen diff --git a/users/grfn/xanthous/test/Xanthous/GameSpec.hs b/users/grfn/xanthous/test/Xanthous/GameSpec.hs new file mode 100644 index 000000000000..2fa8527d0e59 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/GameSpec.hs @@ -0,0 +1,55 @@ +module Xanthous.GameSpec where + +import Test.Prelude hiding (Down) +import Xanthous.Game +import Xanthous.Game.State +import Control.Lens.Properties +import Xanthous.Data (move, Direction(Down)) +import Xanthous.Data.EntityMap (atPosition) + +main :: IO () +main = defaultMain test + +test :: TestTree +test + = localOption (QuickCheckTests 10) + . localOption (QuickCheckMaxSize 10) + $ testGroup "Xanthous.Game" + [ testGroup "positionedCharacter" + [ testProperty "lens laws" $ isLens positionedCharacter + , testCase "updates the position of the character" $ do + initialGame <- getInitialState + let initialPos = initialGame ^. characterPosition + updatedGame = initialGame & characterPosition %~ move Down + updatedPos = updatedGame ^. characterPosition + updatedPos @?= move Down initialPos + updatedGame ^. entities . atPosition initialPos @?= fromList [] + updatedGame ^. entities . atPosition updatedPos + @?= fromList [SomeEntity $ initialGame ^. character] + ] + , testGroup "characterPosition" + [ testProperty "lens laws" $ isLens characterPosition + ] + , testGroup "character" + [ testProperty "lens laws" $ isLens character + ] + , testGroup "MessageHistory" + [ testGroup "MonoComonad laws" + [ testProperty "oextend oextract ≡ id" + $ \(mh :: MessageHistory) -> oextend oextract mh === mh + , testProperty "oextract ∘ oextend f ≡ f" + $ \(mh :: MessageHistory) f -> (oextract . oextend f) mh === f mh + , testProperty "oextend f ∘ oextend g ≡ oextend (f . oextend g)" + $ \(mh :: MessageHistory) f g -> + (oextend f . oextend g) mh === oextend (f . oextend g) mh + ] + ] + , testGroup "Saving the game" + [ testProperty "forms a prism" $ isPrism saved + , testProperty "round-trips" $ \gs -> + loadGame (saveGame gs) === Just gs + , testProperty "preserves the character ID" $ \gs -> + let Just gs' = loadGame $ saveGame gs + in gs' ^. character === gs ^. character + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Generators/Level/UtilSpec.hs b/users/grfn/xanthous/test/Xanthous/Generators/Level/UtilSpec.hs new file mode 100644 index 000000000000..b53c657f7559 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Generators/Level/UtilSpec.hs @@ -0,0 +1,127 @@ +{-# LANGUAGE PackageImports #-} +-------------------------------------------------------------------------------- +module Xanthous.Generators.Level.UtilSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +import System.Random (mkStdGen) +import Control.Monad.Random (runRandT) +import Data.Array.ST (STUArray, runSTUArray, thaw) +import Data.Array.IArray (bounds, array) +import Data.Array.MArray (newArray, readArray, writeArray) +import Data.Array (Array, range, listArray, Ix) +import Control.Monad.ST (ST, runST) +import "checkers" Test.QuickCheck.Instances.Array () +import Linear.V2 +-------------------------------------------------------------------------------- +import Xanthous.Util +import Xanthous.Data (width, height) +-------------------------------------------------------------------------------- +import Xanthous.Generators.Level.Util +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +-------------------------------------------------------------------------------- + +newtype GenArray a b = GenArray (Array a b) + deriving stock (Show, Eq) + +instance (Ix a, Arbitrary a, CoArbitrary a, Arbitrary b) + => Arbitrary (GenArray a b) where + arbitrary = GenArray <$> do + (mkElem :: a -> b) <- arbitrary + minDims <- arbitrary + maxDims <- arbitrary + let bnds = (minDims, maxDims) + pure $ listArray bnds $ mkElem <$> range bnds + +test :: TestTree +test = testGroup "Xanthous.Generators.Util" + [ testGroup "randInitialize" + [ testProperty "returns an array of the correct dimensions" + $ \dims seed aliveChance -> + let gen = mkStdGen seed + res = runSTUArray + $ fmap fst + $ flip runRandT gen + $ randInitialize dims aliveChance + in bounds res === (0, V2 (dims ^. width) (dims ^. height)) + ] + , testGroup "numAliveNeighborsM" + [ testProperty "maxes out at 8" + $ \(GenArray (arr :: Array (V2 Word) Bool)) loc -> + let + act :: forall s. ST s Word + act = do + mArr <- thaw @_ @_ @_ @(STUArray s) arr + numAliveNeighborsM mArr loc + res = runST act + in counterexample (show res) $ between 0 8 res + , testCase "on the outer x edge" $ + let act :: forall s. ST s Word + act = do + cells <- thaw @_ @_ @_ @(STUArray s) $ array @Array @Bool @(V2 Word) + (V2 0 0, V2 2 2) + [ (V2 0 0, True), (V2 1 0, True), (V2 2 0, True) + , (V2 0 1, False), (V2 1 1, False), (V2 2 1, True) + , (V2 0 2, True), (V2 1 2, True), (V2 2 2, True) + ] + numAliveNeighborsM cells (V2 0 1) + res = runST act + in res @?= 7 + , testCase "on the outer y edge" $ + let act :: forall s. ST s Word + act = do + cells <- thaw @_ @_ @_ @(STUArray s) $ array @Array @Bool @(V2 Word) + (V2 0 0, V2 2 2) + [ (V2 0 0, True), (V2 1 0, True), (V2 2 0, True) + , (V2 0 1, False), (V2 1 1, False), (V2 2 1, True) + , (V2 0 2, True), (V2 1 2, True), (V2 2 2, True) + ] + numAliveNeighborsM cells (V2 1 0) + res = runST act + in res @?= 6 + ] + , testGroup "numAliveNeighbors" + [ testProperty "is equivalient to runST . numAliveNeighborsM . thaw" $ + \(GenArray (arr :: Array (V2 Word) Bool)) loc -> + let + act :: forall s. ST s Word + act = do + mArr <- thaw @_ @_ @_ @(STUArray s) arr + numAliveNeighborsM mArr loc + res = runST act + in numAliveNeighbors arr loc === res + , testCase "on the outer x edge" $ + let cells = + array @Array @Bool @(V2 Word) + (V2 0 0, V2 2 2) + [ (V2 0 0, True), (V2 1 0, True), (V2 2 0, True) + , (V2 0 1, False), (V2 1 1, False), (V2 2 1, True) + , (V2 0 2, True), (V2 1 2, True), (V2 2 2, True) + ] + in numAliveNeighbors cells (V2 0 1) @?= 7 + , testCase "on the outer y edge" $ + let cells = + array @Array @Bool @(V2 Word) + (V2 0 0, V2 2 2) + [ (V2 0 0, True), (V2 1 0, True), (V2 2 0, True) + , (V2 0 1, False), (V2 1 1, False), (V2 2 1, True) + , (V2 0 2, True), (V2 1 2, True), (V2 2 2, True) + ] + in numAliveNeighbors cells (V2 1 0) @?= 6 + ] + , testGroup "cloneMArray" + [ testCase "clones the array" $ runST $ + let + go :: forall s. ST s Assertion + go = do + arr <- newArray @(STUArray s) (0 :: Int, 5) (1 :: Int) + arr' <- cloneMArray @_ @(STUArray s) arr + writeArray arr' 0 1234 + x <- readArray arr 0 + pure $ x @?= 1 + in go + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/MessageSpec.hs b/users/grfn/xanthous/test/Xanthous/MessageSpec.hs new file mode 100644 index 000000000000..2068e338bafe --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/MessageSpec.hs @@ -0,0 +1,59 @@ +{-# LANGUAGE OverloadedLists #-} +module Xanthous.MessageSpec ( main, test ) where + +import Test.Prelude +import Xanthous.Messages +import Data.Aeson +import Text.Mustache +import Control.Lens.Properties + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Messages" + [ testGroup "Message" + [ testGroup "JSON decoding" + [ testCase "Single" + $ decode "\"Test Single Template\"" + @?= Just (Single + $ compileMustacheText "template" "Test Single Template" + ^?! _Right) + , testCase "Choice" + $ decode "[\"Choice 1\", \"Choice 2\"]" + @?= Just + (Choice + [ compileMustacheText "template" "Choice 1" ^?! _Right + , compileMustacheText "template" "Choice 2" ^?! _Right + ]) + ] + ] + , localOption (QuickCheckTests 50) + . localOption (QuickCheckMaxSize 10) + $ testGroup "MessageMap" + [ testGroup "instance Ixed" + [ testProperty "traversal laws" $ \k -> + isTraversal $ ix @MessageMap k + , testCase "preview when exists" $ + let + Right tpl = compileMustacheText "foo" "bar" + msg = Single tpl + mm = Nested [("foo", Direct msg)] + in mm ^? ix ["foo"] @?= Just msg + ] + , testGroup "lookupMessage" + [ testProperty "is equivalent to preview ix" $ \msgMap path -> + lookupMessage path msgMap === msgMap ^? ix path + ] + ] + + , testGroup "Messages" + [ testCase "are all valid" $ messages `deepseq` pure () + ] + + , testGroup "Template" + [ testGroup "eq" + [ testProperty "reflexive" $ \(tpl :: Template) -> tpl == tpl + ] + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Messages/TemplateSpec.hs b/users/grfn/xanthous/test/Xanthous/Messages/TemplateSpec.hs new file mode 100644 index 000000000000..2a3873c3b016 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Messages/TemplateSpec.hs @@ -0,0 +1,80 @@ +-------------------------------------------------------------------------------- +module Xanthous.Messages.TemplateSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +import Test.QuickCheck.Instances.Text () +import Data.List.NonEmpty (NonEmpty(..)) +import Data.Function (fix) +-------------------------------------------------------------------------------- +import Xanthous.Messages.Template +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Messages.Template" + [ testGroup "parsing" + [ testProperty "literals" $ forAll genLiteral $ \s -> + testParse template s === Right (Literal s) + , parseCase "escaped curlies" + "foo\\{" + $ Literal "foo{" + , parseCase "simple substitution" + "foo {{bar}}" + $ Literal "foo " `Concat` Subst (SubstPath $ "bar" :| []) + , parseCase "substitution with filters" + "foo {{bar | baz}}" + $ Literal "foo " + `Concat` Subst (SubstFilter (SubstPath $ "bar" :| []) + (FilterName "baz")) + , parseCase "substitution with multiple filters" + "foo {{bar | baz | qux}}" + $ Literal "foo " + `Concat` Subst (SubstFilter (SubstFilter (SubstPath $ "bar" :| []) + (FilterName "baz")) + (FilterName "qux")) + , parseCase "two substitutions and a literal" + "{{a}}{{b}}c" + $ Subst (SubstPath $ "a" :| []) + `Concat` Subst (SubstPath $ "b" :| []) + `Concat` Literal "c" + , localOption (QuickCheckTests 10) + $ testProperty "round-trips with ppTemplate" $ \tpl -> + testParse template (ppTemplate tpl) === Right tpl + ] + , testBatch $ monoid @Template mempty + , testGroup "rendering" + [ testProperty "rendering literals renders literally" + $ forAll genLiteral $ \s fs vs -> + render fs vs (Literal s) === Right s + , testProperty "rendering substitutions renders substitutions" + $ forAll genPath $ \ident val fs -> + let tpl = Subst (SubstPath ident) + tvs = varsWith ident val + in render fs tvs tpl === Right val + , testProperty "filters filter" $ forAll genPath + $ \ident filterName filterFn val -> + let tpl = Subst (SubstFilter (SubstPath ident) filterName) + fs = mapFromList [(filterName, filterFn)] + vs = varsWith ident val + in render fs vs tpl === Right (filterFn val) + ] + ] + where + genLiteral = pack . filter (`notElem` ['\\', '{']) <$> arbitrary + parseCase name input expected = + testCase name $ testParse template input @?= Right expected + testParse p = over _Left errorBundlePretty . runParser p "<test>" + genIdentifier = pack @Text <$> listOf1 (elements identifierChars) + identifierChars = ['a'..'z'] <> ['A'..'Z'] <> ['-', '_'] + + varsWith (p :| []) val = vars [(p, Val val)] + varsWith (phead :| ps) val = vars . pure . (phead ,) . flip fix ps $ + \next pth -> case pth of + [] -> Val val + p : ps' -> nested [(p, next ps')] + + genPath = (:|) <$> genIdentifier <*> listOf genIdentifier + +-- diff --git a/users/grfn/xanthous/test/Xanthous/OrphansSpec.hs b/users/grfn/xanthous/test/Xanthous/OrphansSpec.hs new file mode 100644 index 000000000000..0d800e8a91de --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/OrphansSpec.hs @@ -0,0 +1,72 @@ +{-# LANGUAGE BlockArguments #-} +{-# LANGUAGE OverloadedLists #-} +-------------------------------------------------------------------------------- +module Xanthous.OrphansSpec where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Text.Mustache +import Text.Megaparsec (errorBundlePretty) +import Graphics.Vty.Attributes +import qualified Data.Aeson as JSON +import Data.Interval (Interval, (<=..<=), (<=..<), (<..<=)) +import Data.Aeson ( ToJSON(toJSON), object, Value(Array) ) +import Data.Aeson.Types (fromJSON) +import Data.IntegerInterval (Extended(Finite)) +-------------------------------------------------------------------------------- +import Xanthous.Orphans +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Orphans" + [ localOption (QuickCheckTests 50) + . localOption (QuickCheckMaxSize 10) + $ testGroup "Template" + [ testProperty "ppTemplate / compileMustacheText " \tpl -> + let src = ppTemplate tpl + res :: Either String Template + res = over _Left errorBundlePretty + $ compileMustacheText (templateActual tpl) src + expected = templateCache tpl ^?! at (templateActual tpl) + in + counterexample (unpack src) + $ Right expected === do + (Template actual cache) <- res + maybe (Left "Template not found") Right $ cache ^? at actual + , testProperty "JSON round trip" $ \(tpl :: Template) -> + counterexample (unpack $ ppTemplate tpl) + $ JSON.decode (JSON.encode tpl) === Just tpl + ] + , testGroup "Attr" + [ jsonRoundTrip @Attr ] + , testGroup "Extended" + [ jsonRoundTrip @(Extended Int) ] + , testGroup "Interval" + [ testGroup "JSON" + [ jsonRoundTrip @(Interval Int) + , testCase "parses a single value as a length-1 interval" $ + getSuccess (fromJSON $ toJSON (1 :: Int)) + @?= Just (Finite (1 :: Int) <=..<= Finite 1) + , testCase "parses a pair of values as a single-ended interval" $ + getSuccess (fromJSON $ toJSON ([1, 2] :: [Int])) + @?= Just (Finite (1 :: Int) <=..< Finite (2 :: Int)) + , testCase "parses the full included/excluded syntax" $ + getSuccess (fromJSON $ Array [ object [ "Excluded" JSON..= (1 :: Int) ] + , object [ "Included" JSON..= (4 :: Int) ] + ]) + @?= Just (Finite (1 :: Int) <..<= Finite (4 :: Int)) + , testCase "parses open/closed as aliases" $ + getSuccess (fromJSON $ Array [ object [ "Open" JSON..= (1 :: Int) ] + , object [ "Closed" JSON..= (4 :: Int) ] + ]) + @?= Just (Finite (1 :: Int) <..<= Finite (4 :: Int)) + ] + ] + ] + where + getSuccess :: JSON.Result a -> Maybe a + getSuccess (JSON.Error _) = Nothing + getSuccess (JSON.Success r) = Just r diff --git a/users/grfn/xanthous/test/Xanthous/RandomSpec.hs b/users/grfn/xanthous/test/Xanthous/RandomSpec.hs new file mode 100644 index 000000000000..c88bd9562928 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/RandomSpec.hs @@ -0,0 +1,45 @@ +-------------------------------------------------------------------------------- +module Xanthous.RandomSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Control.Monad.Random +-------------------------------------------------------------------------------- +import Xanthous.Random +import Xanthous.Orphans () +import qualified Data.Interval as Interval +import Data.Interval (Interval, Extended (Finite), (<=..<=)) +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Random" + [ testGroup "chooseSubset" + [ testProperty "chooses a subset" + $ \(l :: [Int]) (Positive (r :: Double)) -> randomTest $ do + ss <- chooseSubset r l + pure $ all (`elem` l) ss + ] + , testGroup "chooseRange" + [ testProperty "chooses in the range" + $ \(rng :: Interval Int) -> + not (Interval.null rng) + ==> randomTest ( do + chooseRange rng >>= \case + Just r -> pure + . counterexample (show r) + $ r `Interval.member` rng + Nothing -> pure $ property Discard + ) + , testProperty "nonEmpty range is never empty" + $ \ (lower :: Int) (NonZero diff) -> randomTest $ do + let upper = lower + diff + r <- chooseRange (Finite lower <=..<= Finite upper) + pure $ isJust r + + ] + ] + where + randomTest prop = evalRandT prop . mkStdGen =<< arbitrary diff --git a/users/grfn/xanthous/test/Xanthous/Util/GraphSpec.hs b/users/grfn/xanthous/test/Xanthous/Util/GraphSpec.hs new file mode 100644 index 000000000000..35ff090b28b9 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Util/GraphSpec.hs @@ -0,0 +1,39 @@ +module Xanthous.Util.GraphSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude +-------------------------------------------------------------------------------- +import Xanthous.Util.Graph +import Data.Graph.Inductive.Basic +import Data.Graph.Inductive.Graph (labNodes, size, order) +import Data.Graph.Inductive.PatriciaTree +import Data.Graph.Inductive.Arbitrary +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Util.Graph" + [ testGroup "mstSubGraph" + [ testProperty "always produces a subgraph" + $ \(CG _ (graph :: Gr Int Int)) -> + let msg = mstSubGraph $ undir graph + in counterexample (show msg) + $ msg `isSubGraphOf` undir graph + , testProperty "returns a graph with the same nodes" + $ \(CG _ (graph :: Gr Int Int)) -> + let msg = mstSubGraph graph + in counterexample (show msg) + $ labNodes msg === labNodes graph + , testProperty "has nodes - 1 edges" + $ \(CG _ (graph :: Gr Int Int)) -> + order graph > 1 ==> + let msg = mstSubGraph graph + in counterexample (show msg) + $ size msg === order graph - 1 + , testProperty "always produces a simple graph" + $ \(CG _ (graph :: Gr Int Int)) -> + let msg = mstSubGraph graph + in counterexample (show msg) $ isSimple msg + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/Util/GraphicsSpec.hs b/users/grfn/xanthous/test/Xanthous/Util/GraphicsSpec.hs new file mode 100644 index 000000000000..61e589280362 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Util/GraphicsSpec.hs @@ -0,0 +1,72 @@ +module Xanthous.Util.GraphicsSpec (main, test) where +-------------------------------------------------------------------------------- +import Test.Prelude hiding (head) +-------------------------------------------------------------------------------- +import Data.List (nub, head) +import Data.Set (isSubsetOf) +import Linear.V2 +-------------------------------------------------------------------------------- +import Xanthous.Util.Graphics +import Xanthous.Util +import Xanthous.Orphans () +-------------------------------------------------------------------------------- + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Util.Graphics" + [ testGroup "circle" + [ testCase "radius 1, origin 2,2" + {- + | | 0 | 1 | 2 | 3 | + |---+---+---+---+---| + | 0 | | | | | + | 1 | | | x | | + | 2 | | x | | x | + | 3 | | | x | | + -} + $ (sort . unique @[] @[_]) (circle @Int (V2 2 2) 1) + @?= [ V2 1 2 + , V2 2 1, V2 2 3 + , V2 3 2 + ] + , testCase "radius 12, origin 0" + $ (sort . nub) (circle @Int 0 12) + @?= (sort . nub) + [ V2 (-12) (-4), V2 (-12) (-3), V2 (-12) (-2), V2 (-12) (-1) + , V2 (-12) 0, V2 (-12) 1, V2 (-12) 2, V2 (-12) 3, V2 (-12) 4 + , V2 (-11) (-6), V2 (-11) (-5), V2 (-11) 5, V2 (-11) 6, V2 (-10) (-7) + , V2 (-10) 7, V2 (-9) (-9), V2 (-9) (-8), V2 (-9) 8, V2 (-9) 9 + , V2 (-8) (-9), V2 (-8) 9, V2 (-7) (-10), V2 (-7) 10, V2 (-6) (-11) + , V2 (-6) 11, V2 (-5) (-11), V2 (-5) 11, V2 (-4) (-12), V2 (-4) 12 + , V2 (-3) (-12), V2 (-3) 12, V2 (-2) (-12), V2 (-2) 12, V2 (-1) (-12) + , V2 (-1) 12, V2 0 (-12), V2 0 12, V2 1 (-12), V2 1 12, V2 2 (-12) + , V2 2 12, V2 3 (-12), V2 3 12, V2 4 (-12), V2 4 12, V2 5 (-11) + , V2 5 11, V2 6 (-11), V2 6 11, V2 7 (-10), V2 7 10, V2 8 (-9), V2 8 9 + , V2 9 (-9), V2 9 (-8), V2 9 8, V2 9 9, V2 10 (-7), V2 10 7 + , V2 11 (-6), V2 11 (-5), V2 11 5, V2 11 6, V2 12 (-4), V2 12 (-3) + , V2 12 (-2), V2 12 (-1), V2 12 0, V2 12 1, V2 12 2, V2 12 3, V2 12 4 + ] + ] + , testGroup "filledCircle" + [ testProperty "is a superset of circle" $ \center radius -> + let circ = circle @Int center radius + filledCirc = filledCircle center radius + in counterexample ( "circle: " <> show circ + <> "\nfilledCircle: " <> show filledCirc) + $ setFromList circ `isSubsetOf` setFromList filledCirc + -- TODO later + -- , testProperty "is always contiguous" $ \center radius -> + -- let filledCirc = filledCircle center radius + -- in counterexample (renderBooleanGraphics filledCirc) $ + ] + , testGroup "line" + [ testProperty "starts and ends at the start and end points" $ \start end -> + let ℓ = line @Int start end + in counterexample ("line: " <> show ℓ) + $ length ℓ > 2 ==> (head ℓ === start) .&&. (head (reverse ℓ) === end) + ] + ] + +-------------------------------------------------------------------------------- diff --git a/users/grfn/xanthous/test/Xanthous/Util/InflectionSpec.hs b/users/grfn/xanthous/test/Xanthous/Util/InflectionSpec.hs new file mode 100644 index 000000000000..fad841043152 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/Util/InflectionSpec.hs @@ -0,0 +1,18 @@ +module Xanthous.Util.InflectionSpec (main, test) where + +import Test.Prelude +import Xanthous.Util.Inflection + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Util.Inflection" + [ testGroup "toSentence" + [ testCase "empty" $ toSentence [] @?= "" + , testCase "single" $ toSentence ["x"] @?= "x" + , testCase "two" $ toSentence ["x", "y"] @?= "x and y" + , testCase "three" $ toSentence ["x", "y", "z"] @?= "x, y, and z" + , testCase "four" $ toSentence ["x", "y", "z", "w"] @?= "x, y, z, and w" + ] + ] diff --git a/users/grfn/xanthous/test/Xanthous/UtilSpec.hs b/users/grfn/xanthous/test/Xanthous/UtilSpec.hs new file mode 100644 index 000000000000..684a03b2c7a0 --- /dev/null +++ b/users/grfn/xanthous/test/Xanthous/UtilSpec.hs @@ -0,0 +1,46 @@ +module Xanthous.UtilSpec (main, test) where + +import Test.Prelude +import Xanthous.Util +import Control.Monad.State.Lazy (execState) + +main :: IO () +main = defaultMain test + +test :: TestTree +test = testGroup "Xanthous.Util" + [ testGroup "smallestNotIn" + [ testCase "examples" $ do + smallestNotIn [7 :: Word, 3, 7] @?= 0 + smallestNotIn [7 :: Word, 0, 1, 3, 7] @?= 2 + , testProperty "returns an element not in the list" $ \(xs :: [Word]) -> + smallestNotIn xs `notElem` xs + , testProperty "pred return is in the list" $ \(xs :: [Word]) -> + let res = smallestNotIn xs + in res /= 0 ==> pred res `elem` xs + , testProperty "ignores order" $ \(xs :: [Word]) -> + forAll (shuffle xs) $ \shuffledXs -> + smallestNotIn xs === smallestNotIn shuffledXs + ] + , testGroup "takeWhileInclusive" + [ testProperty "takeWhileInclusive (const True) ≡ id" + $ \(xs :: [Int]) -> takeWhileInclusive (const True) xs === xs + ] + , testGroup "endoTimes" + [ testCase "endoTimes 4 succ 5" + $ endoTimes (4 :: Int) succ (5 :: Int) @?= 9 + ] + , testGroup "modifyKL" + [ testCase "_1 += 1" + $ execState (modifyKL _1 $ pure . succ) (1 :: Int, 2 :: Int) @?= (2, 2) + ] + , testGroup "removeFirst" + [ testCase "example" $ + removeFirst @[Int] (> 5) [1..10] @?= [1, 2, 3, 4, 5, 7, 8, 9, 10] + , testProperty "the result is the right length" $ \(xs :: [Int]) p -> + length (removeFirst p xs) `elem` [length xs, length xs - 1] + ] + , testGroup "AlphaChar" + [ testCase "succ 'z'" $ succ (AlphaChar 'z') @?= AlphaChar 'A' + ] + ] diff --git a/users/grfn/xanthous/xanthous.cabal b/users/grfn/xanthous/xanthous.cabal new file mode 100644 index 000000000000..987e1f48f693 --- /dev/null +++ b/users/grfn/xanthous/xanthous.cabal @@ -0,0 +1,528 @@ +cabal-version: 1.12 + +-- This file has been generated from package.yaml by hpack version 0.34.5. +-- +-- see: https://github.com/sol/hpack +-- +-- hash: 8cae8550487b6092c18c82a0dc29bf22980d416771c66f6fca3e151875c66495 + +name: xanthous +version: 0.1.0.0 +synopsis: A WIP TUI RPG +description: Please see the README on GitHub at <https://github.com/glittershark/xanthous> +category: Game +homepage: https://github.com/glittershark/xanthous#readme +bug-reports: https://github.com/glittershark/xanthous/issues +author: Griffin Smith +maintainer: root@gws.fyi +copyright: 2019 Griffin Smith +license: GPL-3 +license-file: LICENSE +build-type: Simple +extra-source-files: + README.org + +source-repository head + type: git + location: https://github.com/glittershark/xanthous + +library + exposed-modules: + Data.Aeson.Generic.DerivingVia + Xanthous.AI.Gormlak + Xanthous.App + Xanthous.App.Autocommands + Xanthous.App.Common + Xanthous.App.Prompt + Xanthous.App.Time + Xanthous.Command + Xanthous.Data + Xanthous.Data.App + Xanthous.Data.Entities + Xanthous.Data.EntityChar + Xanthous.Data.EntityMap + Xanthous.Data.EntityMap.Graphics + Xanthous.Data.Levels + Xanthous.Data.Memo + Xanthous.Data.NestedMap + Xanthous.Data.VectorBag + Xanthous.Entities.Character + Xanthous.Entities.Common + Xanthous.Entities.Creature + Xanthous.Entities.Creature.Hippocampus + Xanthous.Entities.Draw.Util + Xanthous.Entities.Entities + Xanthous.Entities.Environment + Xanthous.Entities.Item + Xanthous.Entities.Marker + Xanthous.Entities.Raws + Xanthous.Entities.RawTypes + Xanthous.Game + Xanthous.Game.Arbitrary + Xanthous.Game.Draw + Xanthous.Game.Env + Xanthous.Game.Lenses + Xanthous.Game.Memo + Xanthous.Game.Prompt + Xanthous.Game.State + Xanthous.Generators.Level + Xanthous.Generators.Level.CaveAutomata + Xanthous.Generators.Level.Dungeon + Xanthous.Generators.Level.LevelContents + Xanthous.Generators.Level.Util + Xanthous.Generators.Level.Village + Xanthous.Generators.Speech + Xanthous.Messages + Xanthous.Messages.Template + Xanthous.Monad + Xanthous.Orphans + Xanthous.Physics + Xanthous.Prelude + Xanthous.Random + Xanthous.Util + Xanthous.Util.Comonad + Xanthous.Util.Graph + Xanthous.Util.Graphics + Xanthous.Util.Inflection + Xanthous.Util.JSON + Xanthous.Util.Optparse + Xanthous.Util.QuickCheck + other-modules: + Paths_xanthous + hs-source-dirs: + src + default-extensions: + BlockArguments + ConstraintKinds + DataKinds + DeriveAnyClass + DeriveGeneric + DerivingStrategies + DerivingVia + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTSyntax + GeneralizedNewtypeDeriving + KindSignatures + StandaloneKindSignatures + LambdaCase + MultiWayIf + NoImplicitPrelude + NoStarIsType + OverloadedStrings + PolyKinds + RankNTypes + ScopedTypeVariables + TupleSections + TypeApplications + TypeFamilies + TypeOperators + ViewPatterns + ghc-options: -Wall + build-depends: + JuicyPixels + , MonadRandom + , QuickCheck + , Rasterific + , aeson + , array + , async + , base + , bifunctors + , brick + , checkers + , classy-prelude + , comonad + , comonad-extras + , constraints + , containers + , criterion + , data-default + , data-interval + , deepseq + , directory + , fgl + , fgl-arbitrary + , file-embed + , filepath + , generic-arbitrary + , generic-lens + , groups + , hgeometry + , hgeometry-combinatorial + , lens + , lifted-async + , linear + , megaparsec + , mmorph + , monad-control + , mtl + , optparse-applicative + , parallel + , parser-combinators + , pointed + , quickcheck-instances + , quickcheck-text + , random + , random-extras + , random-fu + , random-source + , raw-strings-qq + , reflection + , semigroupoids + , semigroups + , splitmix + , stache + , streams + , text + , text-zipper + , tomland + , transformers + , vector + , vty + , witherable + , yaml + , zlib + default-language: Haskell2010 + +executable xanthous + main-is: Main.hs + other-modules: + Paths_xanthous + hs-source-dirs: + app + default-extensions: + BlockArguments + ConstraintKinds + DataKinds + DeriveAnyClass + DeriveGeneric + DerivingStrategies + DerivingVia + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTSyntax + GeneralizedNewtypeDeriving + KindSignatures + StandaloneKindSignatures + LambdaCase + MultiWayIf + NoImplicitPrelude + NoStarIsType + OverloadedStrings + PolyKinds + RankNTypes + ScopedTypeVariables + TupleSections + TypeApplications + TypeFamilies + TypeOperators + ViewPatterns + ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -O2 + build-depends: + JuicyPixels + , MonadRandom + , QuickCheck + , Rasterific + , aeson + , array + , async + , base + , bifunctors + , brick + , checkers + , classy-prelude + , comonad + , comonad-extras + , constraints + , containers + , criterion + , data-default + , data-interval + , deepseq + , directory + , fgl + , fgl-arbitrary + , file-embed + , filepath + , generic-arbitrary + , generic-lens + , groups + , hgeometry + , hgeometry-combinatorial + , lens + , lifted-async + , linear + , megaparsec + , mmorph + , monad-control + , mtl + , optparse-applicative + , parallel + , parser-combinators + , pointed + , quickcheck-instances + , quickcheck-text + , random + , random-extras + , random-fu + , random-source + , raw-strings-qq + , reflection + , semigroupoids + , semigroups + , splitmix + , stache + , streams + , text + , text-zipper + , tomland + , transformers + , vector + , vty + , witherable + , xanthous + , yaml + , zlib + default-language: Haskell2010 + +test-suite test + type: exitcode-stdio-1.0 + main-is: Spec.hs + other-modules: + Test.Prelude + Xanthous.Data.EntitiesSpec + Xanthous.Data.EntityCharSpec + Xanthous.Data.EntityMap.GraphicsSpec + Xanthous.Data.EntityMapSpec + Xanthous.Data.LevelsSpec + Xanthous.Data.MemoSpec + Xanthous.Data.NestedMapSpec + Xanthous.DataSpec + Xanthous.Entities.CharacterSpec + Xanthous.Entities.CommonSpec + Xanthous.Entities.RawsSpec + Xanthous.Entities.RawTypesSpec + Xanthous.Game.PromptSpec + Xanthous.Game.StateSpec + Xanthous.GameSpec + Xanthous.Generators.Level.UtilSpec + Xanthous.Messages.TemplateSpec + Xanthous.MessageSpec + Xanthous.OrphansSpec + Xanthous.RandomSpec + Xanthous.Util.GraphicsSpec + Xanthous.Util.GraphSpec + Xanthous.Util.InflectionSpec + Xanthous.UtilSpec + Paths_xanthous + hs-source-dirs: + test + default-extensions: + BlockArguments + ConstraintKinds + DataKinds + DeriveAnyClass + DeriveGeneric + DerivingStrategies + DerivingVia + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTSyntax + GeneralizedNewtypeDeriving + KindSignatures + StandaloneKindSignatures + LambdaCase + MultiWayIf + NoImplicitPrelude + NoStarIsType + OverloadedStrings + PolyKinds + RankNTypes + ScopedTypeVariables + TupleSections + TypeApplications + TypeFamilies + TypeOperators + ViewPatterns + ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -O0 + build-depends: + JuicyPixels + , MonadRandom + , QuickCheck + , Rasterific + , aeson + , array + , async + , base + , bifunctors + , brick + , checkers + , classy-prelude + , comonad + , comonad-extras + , constraints + , containers + , criterion + , data-default + , data-interval + , deepseq + , directory + , fgl + , fgl-arbitrary + , file-embed + , filepath + , generic-arbitrary + , generic-lens + , groups + , hgeometry + , hgeometry-combinatorial + , lens + , lens-properties + , lifted-async + , linear + , megaparsec + , mmorph + , monad-control + , mtl + , optparse-applicative + , parallel + , parser-combinators + , pointed + , quickcheck-instances + , quickcheck-text + , random + , random-extras + , random-fu + , random-source + , raw-strings-qq + , reflection + , semigroupoids + , semigroups + , splitmix + , stache + , streams + , tasty + , tasty-hunit + , tasty-quickcheck + , tasty-rerun + , text + , text-zipper + , tomland + , transformers + , vector + , vty + , witherable + , xanthous + , yaml + , zlib + default-language: Haskell2010 + +benchmark benchmark + type: exitcode-stdio-1.0 + main-is: Bench.hs + other-modules: + Bench.Prelude + Xanthous.Generators.UtilBench + Xanthous.RandomBench + Paths_xanthous + hs-source-dirs: + bench + default-extensions: + BlockArguments + ConstraintKinds + DataKinds + DeriveAnyClass + DeriveGeneric + DerivingStrategies + DerivingVia + FlexibleContexts + FlexibleInstances + FunctionalDependencies + GADTSyntax + GeneralizedNewtypeDeriving + KindSignatures + StandaloneKindSignatures + LambdaCase + MultiWayIf + NoImplicitPrelude + NoStarIsType + OverloadedStrings + PolyKinds + RankNTypes + ScopedTypeVariables + TupleSections + TypeApplications + TypeFamilies + TypeOperators + ViewPatterns + ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N + build-depends: + JuicyPixels + , MonadRandom + , QuickCheck + , Rasterific + , aeson + , array + , async + , base + , bifunctors + , brick + , checkers + , classy-prelude + , comonad + , comonad-extras + , constraints + , containers + , criterion + , data-default + , data-interval + , deepseq + , directory + , fgl + , fgl-arbitrary + , file-embed + , filepath + , generic-arbitrary + , generic-lens + , groups + , hgeometry + , hgeometry-combinatorial + , lens + , lifted-async + , linear + , megaparsec + , mmorph + , monad-control + , mtl + , optparse-applicative + , parallel + , parser-combinators + , pointed + , quickcheck-instances + , quickcheck-text + , random + , random-extras + , random-fu + , random-source + , raw-strings-qq + , reflection + , semigroupoids + , semigroups + , splitmix + , stache + , streams + , text + , text-zipper + , tomland + , transformers + , vector + , vty + , witherable + , xanthous + , yaml + , zlib + default-language: Haskell2010 diff --git a/users/isomer/OWNERS b/users/isomer/OWNERS new file mode 100644 index 000000000000..6997cd391d9c --- /dev/null +++ b/users/isomer/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - isomer diff --git a/users/isomer/keys.nix b/users/isomer/keys.nix new file mode 100644 index 000000000000..8c29e27895db --- /dev/null +++ b/users/isomer/keys.nix @@ -0,0 +1,7 @@ +# SSH public keys +{ ... }: + +rec { + perry = "cert-authority,principals=perry ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCXWKN+FXlQAQ36R4+FHJ9f15Tz/48xLK1f85Yf9eBrvJJVMn6ge3Cy8AJ2nymBtVvCC86q616yl4Mn+CrKBH/vHr4jY9nxJ7HHgKI8ERr+7KpLIAiiaeIBljWwCy918lK3MijRCuj0P0d3v8CEFJjyCsiyglDVcNhsW87VqqZE6lUg4Alw1CGAmNjamxdoIZxjZAM9vJtZrlYnUiu+X7vTl5ttTaZkLCCfu+/bJAKFBWPG5BPaNjjfGVuTKqEc4plkI3JeZBu3Or3LzlYxcvp71i+eKGJ8F/nMBlo25iQsQpi8ZS7JYAhj3mYVrstw7j+nkgbordvDOK5NbDMi6GzX"; + all = [ perry ]; +} diff --git a/users/lukegb/OWNERS b/users/lukegb/OWNERS new file mode 100644 index 000000000000..676fbf185649 --- /dev/null +++ b/users/lukegb/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - lukegb diff --git a/users/lukegb/hgext/gerrithook.py b/users/lukegb/hgext/gerrithook.py new file mode 100644 index 000000000000..ef02126ba0f8 --- /dev/null +++ b/users/lukegb/hgext/gerrithook.py @@ -0,0 +1,63 @@ +"""Bizarre hacks to make Gerrit better.""" + +import collections +import re +import random +import mercurial + +_ = mercurial.i18n._ + +cmdtable = {} +command = mercurial.registrar.command(cmdtable) + +testedwith = '5.3.1' + +_changeid_regex = re.compile(b'^Change-Id: (I.*)$', re.M) + +def random_hash(): + """Returns a random SHA1-like hex string.""" + return b"%040x" % random.getrandbits(160) + +def reposetup(ui, repo): + + class GerritRepo(repo.__class__): + def commitctx(self, ctx, *args, **kwargs): + match = _changeid_regex.search(ctx._text) + if not match: + ctx._text = ctx._text.rstrip(b'\n') + ctx._text += b'\n\nChange-Id: I' + random_hash() + return super().commitctx(ctx, *args, **kwargs) + + repo.__class__ = GerritRepo + + +@command(b'gerrit-obsolete', [], _(b'[options]')) +def gerritobsolete(ui, repo, **opts): + """Mark draft commits as obsolete by public commits based on Gerrit Change-Id tag.""" + if repo.obsstore.readonly: + ui.error(b'obsstore is readonly') + return + changesets = collections.defaultdict(set) + drafts = set() + for draft in repo.set('draft() - obsolete()'): + match = _changeid_regex.search(draft.description()) + if not match: + continue + changesets[match.groups()[0]].add(draft) + drafts.add(draft) + if not drafts: + return + publicparent = next(repo.set( + b'ancestor((public() and bookmark("canon")), %s)' % ( + b', '.join(x.hex() for x in drafts)))) + megare = b're:(?ms)^Change-Id: (%s)$' % (b'|'.join(changesets.keys()),) + markers = [] + for public in repo.set('(%s..(public() and canon)) and desc(%s)', publicparent, megare): + match = _changeid_regex.search(public.description()) + if not match: + continue + drafts = changesets[match.groups()[0]] + if not drafts: + continue + markers.append((tuple(drafts), (public,))) + mercurial.obsolete.createmarkers(repo, markers, operation=b'gerrit-obsolete') diff --git a/users/lukegb/keys.nix b/users/lukegb/keys.nix new file mode 100644 index 000000000000..e54009122f92 --- /dev/null +++ b/users/lukegb/keys.nix @@ -0,0 +1,10 @@ +# My SSH public keys +{ ... }: + +rec { + termius = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINytpHct7PLdLNp6MoaOPP7ccBPUQKymVNMqix//Wt1f"; + porcorosso-wsl = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMhQ3yjf59eQjOfVXzXz5u8BS5c6hdL1yY8GqccaIjx3"; + porcorosso-nixos = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILid+1rq3k3k7Kbaw8X63vrPrQdanH55TucQwp3ZWfo+"; + clouvider-lon01-nix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINQU7Y+Ha5m0ebwUjA55xXT/xbWZAWx1fVNFufle+vQj"; + all = [ termius porcorosso-wsl porcorosso-nixos clouvider-lon01-nix ]; +} diff --git a/users/qyliss/OWNERS b/users/qyliss/OWNERS new file mode 100644 index 000000000000..d54ea3622d21 --- /dev/null +++ b/users/qyliss/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - qyliss diff --git a/users/qyliss/keys.nix b/users/qyliss/keys.nix new file mode 100644 index 000000000000..d0837a7c6744 --- /dev/null +++ b/users/qyliss/keys.nix @@ -0,0 +1,8 @@ +# Public key from https://github.com/alyssais.keys +{ ... }: + +{ + all = [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDO11Pr7jKaRZ2It1yB312SKFN8mCV7aVYdry16LNwtnA6EDgFxyshG4Zmhl9asxQ9wa1lT3tdKB6ArA+VKxXMZB0zm15jYSLKpHQxMT7T3SqtTluJQpJD9zRtWeHbW/e1mtgn3tPYTHERB4HVGKIeGk97eOR2YOdXPHOIWhOXpogDtUlyt1bmWl0gyRHbWhViLeReHYhsu0KbZlo+ntN9aN7lPVkDfa7gUARv6IeGE5hAYHPRWmQ3VJCDaQnzsTtesLPFiNmV6Pq7qtWbHVNOG9XQLXJhD/305+yDZ2y/+KuBEQCroiWF8fPY/8gutfkZ0ZLjdGbXl38j5v+yRjreh+wjcN5MYWCWM18hMdutpoMd9D7PXaZz90V2vS+mRC81t3zXKrAy3Ke+LQBmlWSWxmKWdDoOTGOHjyPuCC/q+In7Q8hetB9/b9WUXTwEaaE3lUsa7y5JHAekNmdSoN3WD10nGYVUMvRRPGAlyqZTQdvxhn+6Pyu2piwIv/TMmC1CwiHr+fLbHxXQF745sOBQNmrdfiOzqDsKleybNB6i0AdDm5UZcYRcMLuxmryxN8O8qNUdMjMGoCeFcGwAIieqM+0xkPiByKr8ky2yV2lwOaZ4jrp/3j5GsGoQlvNKIPdCA/GQFad6vuqvhlbWcbdfiNpawrppLcJBsGB2NVjGbNQ==" + ]; +} diff --git a/users/riking/OWNERS b/users/riking/OWNERS new file mode 100644 index 000000000000..a39f4cd9f0ae --- /dev/null +++ b/users/riking/OWNERS @@ -0,0 +1,3 @@ +inherit: false +owners: + - riking diff --git a/users/riking/adventofcode-2020/.gitignore b/users/riking/adventofcode-2020/.gitignore new file mode 100644 index 000000000000..076ff412156a --- /dev/null +++ b/users/riking/adventofcode-2020/.gitignore @@ -0,0 +1,2 @@ +*/target +*/input.txt diff --git a/users/riking/adventofcode-2020/day01/Cargo.lock b/users/riking/adventofcode-2020/day01/Cargo.lock new file mode 100644 index 000000000000..a1a18948a7ea --- /dev/null +++ b/users/riking/adventofcode-2020/day01/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf8dcb5b4bbaa28653b647d8c77bd4ed40183b48882e130c1f1ffb73de069fd7" + +[[package]] +name = "day01" +version = "0.1.0" +dependencies = [ + "anyhow", +] diff --git a/users/riking/adventofcode-2020/day01/Cargo.toml b/users/riking/adventofcode-2020/day01/Cargo.toml new file mode 100644 index 000000000000..d90ab548bb7b --- /dev/null +++ b/users/riking/adventofcode-2020/day01/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "day01" +version = "0.1.0" +authors = ["Kane York <kanepyork@gmail.com>"] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0.34" diff --git a/users/riking/adventofcode-2020/day01/default.nix b/users/riking/adventofcode-2020/day01/default.nix new file mode 100644 index 000000000000..0648a05af683 --- /dev/null +++ b/users/riking/adventofcode-2020/day01/default.nix @@ -0,0 +1,10 @@ +{ depot, ... }: + +with depot.third_party; + +naersk.buildPackage { + src = ./.; + + buildInputs = []; + doCheck = true; +} diff --git a/users/riking/adventofcode-2020/day01/src/main.rs b/users/riking/adventofcode-2020/day01/src/main.rs new file mode 100644 index 000000000000..3e6b339d7c2b --- /dev/null +++ b/users/riking/adventofcode-2020/day01/src/main.rs @@ -0,0 +1,85 @@ +use anyhow::anyhow; +use std::fs::File; +use std::io::prelude::*; +use std::io::BufReader; + +const PART_2: bool = true; + +fn day01(is_part2: bool, numbers: &Vec<i64>) -> Result<String, anyhow::Error> { + //println!("{:?}", numbers); + + for n1 in numbers.iter() { + for n2 in numbers.iter() { + if is_part2 { + for n3 in numbers.iter() { + if n1 + n2 + n3 == 2020 { + return Ok((n1 * n2 * n3).to_string()); + } + } + } else { + if n1 + n2 == 2020 { + return Ok((n1 * n2).to_string()); + } + } + } + } + + Err(anyhow!("no solution found")) +} + +fn parse(filename: &str) -> Result<Vec<i64>, anyhow::Error> { + let f = File::open(filename)?; + let mut reader = BufReader::new(f); + + let mut values = Vec::<i64>::new(); + + let mut line = String::new(); + loop { + line.clear(); + reader.read_line(&mut line)?; + let trimmed_line = line.trim(); + if trimmed_line.is_empty() { + break; + } + + values.push(trimmed_line.parse()?); + } + Ok(values) +} + +fn main() -> anyhow::Result<()> { + let args: Vec<String> = std::env::args().collect(); + + //println!("{:?}", args); + if args.len() != 2 { + return Err(anyhow!("usage: day01 input_file")); + } + let filename = args.into_iter().skip(1).next().expect("args len == 1"); + + let numbers = parse(&filename)?; + + println!("{}", day01(PART_2, &numbers)?); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::day01; + + #[test] + fn test_part1() { + let vec = vec![1721, 979, 366, 299, 675, 1456]; + let result = day01(false, &vec).unwrap(); + + assert_eq!(result, 514579.to_string()); + } + + #[test] + fn test_part2() { + let vec = vec![1721, 979, 366, 299, 675, 1456]; + let result = day01(true, &vec).unwrap(); + + assert_eq!(result, 241861950.to_string()); + } +} diff --git a/users/riking/dotfiles/.mybashrc b/users/riking/dotfiles/.mybashrc new file mode 100644 index 000000000000..c5ebc34a1f4f --- /dev/null +++ b/users/riking/dotfiles/.mybashrc @@ -0,0 +1,53 @@ + +# BEGIN: __USER_FUNCTIONS__ +function gh-clone() { + if [[ "x$2" == "x" ]]; then + IFS='/' read -ra PARTS <<< "$1" + user="${PARTS[0]}" + repo="${PARTS[1]}" + else + user="$1" + repo="$2" + fi + if [[ -d ~/go/src/github.com/"$user"/"$repo" ]]; then + cd ~/go/src/github.com/"${user}"/"${repo}" + return 0 + fi + mkdir -p ~/go/src/github.com/"${user}" + cd ~/go/src/github.com/"${user}" + git clone git@github.com:"${user}"/"${repo}".git + cd ~/go/src/github.com/"${user}"/"${repo}" +} + +function download() { + cd "${HOME}/Downloads" + wget "$@" +} + +# todo: only one password pls +function prodaccess() { + (ssh-add -L | grep -q 'ZgEu6S3SLatYN') || ssh-add "$HOME"/.ssh/id_ed25519 + (ssh-add -L | grep -q 'Gfh2S3kUwZ8A6') || ssh-add "$HOME"/.ssh/id_rsa.discourse + echo "signing test" | gpg --clearsign > /dev/null +} + +function reset-audio() { + pulseaudio -k && sudo alsa force-reload +} + +function tvl-push() { + git push origin HEAD:refs/for/canon +} + +# END: __USER_FUNCTIONS__ + +# BEGIN: __USER_ENV__ +GOPATH=$HOME/go +CDPATH=$HOME/go/src +export GPG_TTY="$(tty)" + +export PATH="/usr/local/go/bin:$HOME/go/bin:$HOME/.rbenv/bin:$PATH" + +eval "$(rbenv init -)" +# END: __USER_ENV__ + diff --git a/users/riking/dotfiles/fish/conf.d/nix-env.fish b/users/riking/dotfiles/fish/conf.d/nix-env.fish new file mode 100644 index 000000000000..6f79f9752855 --- /dev/null +++ b/users/riking/dotfiles/fish/conf.d/nix-env.fish @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Unlicense +# https://raw.githubusercontent.com/lilyball/nix-env.fish/master/conf.d/nix-env.fish + +# Setup Nix + +# We need to distinguish between single-user and multi-user installs. +# This is difficult because there's no official way to do this. +# We could look for the presence of /nix/var/nix/daemon-socket/socket but this will fail if the +# daemon hasn't started yet. /nix/var/nix/daemon-socket will exist if the daemon has ever run, but +# I don't think there's any protection against accidentally running `nix-daemon` as a user. +# We also can't just look for /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh because +# older single-user installs used the default profile instead of a per-user profile. +# We can still check for it first, because all multi-user installs should have it, and so if it's +# not present that's a pretty big indicator that this is a single-user install. If it does exist, +# we still need to verify the install type. To that end we'll look for a root owner and sticky bit +# on /nix/store. Multi-user installs set both, single-user installs don't. It's certainly possible +# someone could do a single-user install as root and then manually set the sticky bit but that +# would be extremely unusual. + +set -l nix_profile_path /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh +set -l single_user_profile_path ~/.nix-profile/etc/profile.d/nix.sh +if test -e $nix_profile_path + # The path exists. Double-check that this is a multi-user install. + # We can't just check for ~/.nix-profile/… because this may be a single-user install running as + # the wrong user. + + # stat is not portable. Splitting the output of ls -nd is reliable on most platforms. + set -l owner (string split -n ' ' (ls -nd /nix/store 2>/dev/null))[3] + if not test -k /nix/store -a $owner -eq 0 + # /nix/store is either not owned by root or not sticky. Assume single-user. + set nix_profile_path $single_user_profile_path + end +else + # The path doesn't exist. Assume single-user + set nix_profile_path $single_user_profile_path +end + +if test -e $nix_profile_path + # Source the nix setup script + # We're going to run the regular Nix profile under bash and then print out a few variables + for line in (env -u BASH_ENV bash -c '. "$0"; for name in PATH "${!NIX_@}"; do printf "%s=%s\0" "$name" "${!name}"; done' $nix_profile_path | string split0) + set -xg (string split -m 1 = $line) + end + + # Insert Nix's fish share directories into fish's special variables. + # nixpkgs-installed fish tries to set these up already if NIX_PROFILES is defined, which won't + # be the case when sourcing $__fish_data_dir/share/config.fish normally, but might be for a + # recursive invocation. To guard against that, we'll only insert paths that don't already exit. + # Furthermore, for the vendor_conf.d sourcing, we'll use the pre-existing presence of a path in + # $fish_function_path to determine whether we want to source the relevant vendor_conf.d folder. + + # To start, let's locally define NIX_PROFILES if it doesn't already exist. + set -al NIX_PROFILES + if test (count $NIX_PROFILES) -eq 0 + set -a NIX_PROFILES $HOME/.nix-profile + end + # Replicate the logic from nixpkgs version of $__fish_data_dir/__fish_build_paths.fish. + set -l __nix_profile_paths (string split ' ' -- $NIX_PROFILES)[-1..1] + set -l __extra_completionsdir \ + $__nix_profile_paths/etc/fish/completions \ + $__nix_profile_paths/share/fish/vendor_completions.d + set -l __extra_functionsdir \ + $__nix_profile_paths/etc/fish/functions \ + $__nix_profile_paths/share/fish/vendor_functions.d + set -l __extra_confdir \ + $__nix_profile_paths/etc/fish/conf.d \ + $__nix_profile_paths/share/fish/vendor_conf.d \ + + ### Configure fish_function_path ### + # Remove any of our extra paths that may already exist. + # Record the equivalent __extra_confdir path for any function path that exists. + set -l existing_conf_paths + for path in $__extra_functionsdir + if set -l idx (contains --index -- $path $fish_function_path) + set -e fish_function_path[$idx] + set -a existing_conf_paths $__extra_confdir[(contains --index -- $path $__extra_functionsdir)] + end + end + # Insert the paths before $__fish_data_dir. + if set -l idx (contains --index -- $__fish_data_dir/functions $fish_function_path) + # Fish has no way to simply insert into the middle of an array. + set -l new_path $fish_function_path[1..$idx] + set -e new_path[$idx] + set -a new_path $__extra_functionsdir + set fish_function_path $new_path $fish_function_path[$idx..-1] + else + set -a fish_function_path $__extra_functionsdir + end + + ### Configure fish_complete_path ### + # Remove any of our extra paths that may already exist. + for path in $__extra_completionsdir + if set -l idx (contains --index -- $path $fish_complete_path) + set -e fish_complete_path[$idx] + end + end + # Insert the paths before $__fish_data_dir. + if set -l idx (contains --index -- $__fish_data_dir/completions $fish_complete_path) + set -l new_path $fish_complete_path[1..$idx] + set -e new_path[$idx] + set -a new_path $__extra_completionsdir + set fish_complete_path $new_path $fish_complete_path[$idx..-1] + else + set -a fish_complete_path $__extra_completionsdir + end + + ### Source conf directories ### + # The built-in directories were already sourced during shell initialization. + # Any __extra_confdir that came from $__fish_data_dir/__fish_build_paths.fish was also sourced. + # As explained above, we're using the presence of pre-existing paths in $fish_function_path as a + # signal that the corresponding conf dir has also already been sourced. + # In order to simulate this, we'll run through the same algorithm as found in + # $__fish_data_dir/config.fish except we'll avoid sourcing the file if it comes from an + # already-sourced location. + # Caveats: + # * Files will be sourced in a different order than we'd ideally do (because we're coming in + # after the fact to source them). + # * If there are existing extra conf paths, files in them may have been sourced that should have + # been suppressed by paths we're inserting in front. + # * Similarly any files in $__fish_data_dir/vendor_conf.d that should have been suppressed won't + # have been. + set -l sourcelist + for file in $__fish_config_dir/conf.d/*.fish $__fish_sysconf_dir/conf.d/*.fish + # We know these paths were sourced already. Just record them. + set -l basename (string replace -r '^.*/' '' -- $file) + contains -- $basename $sourcelist + or set -a sourcelist $basename + end + for root in $__extra_confdir + for file in $root/*.fish + set -l basename (string replace -r '^.*/' '' -- $file) + contains -- $basename $sourcelist + and continue + set -a sourcelist $basename + contains -- $root $existing_conf_paths + and continue # this is a pre-existing path, it will have been sourced already + [ -f $file -a -r $file ] + and source $file + end + end +end diff --git a/users/riking/dotfiles/fish/config.fish b/users/riking/dotfiles/fish/config.fish new file mode 100644 index 000000000000..c2454762bddf --- /dev/null +++ b/users/riking/dotfiles/fish/config.fish @@ -0,0 +1,8 @@ +set -gx GOPATH "$HOME/go" +set -gx GPG_TTY (tty) +set -gx DEPOT_ROOT "$GOPATH/src/code.tvl.fyi" + +set -gx PATH '/usr/local/go/bin' "$HOME/.cargo/bin" "$HOME/.rbenv/bin" $PATH +status --is-interactive; and rbenv init - | source +source ~/.opsrc.fish # work +set -gx PATH "$HOME/go/bin" $PATH diff --git a/users/riking/dotfiles/fish/fish_variables b/users/riking/dotfiles/fish/fish_variables new file mode 100644 index 000000000000..fa8bff919f40 --- /dev/null +++ b/users/riking/dotfiles/fish/fish_variables @@ -0,0 +1,32 @@ +# This file contains fish universal variable definitions. +# VERSION: 3.0 +SETUVAR __fish_initialized:3100 +SETUVAR fish_color_autosuggestion:555\x1ebrblack +SETUVAR fish_color_cancel:\x2dr +SETUVAR fish_color_command:005fd7 +SETUVAR fish_color_comment:990000 +SETUVAR fish_color_cwd:green +SETUVAR fish_color_cwd_root:red +SETUVAR fish_color_end:009900 +SETUVAR fish_color_error:ff0000 +SETUVAR fish_color_escape:00a6b2 +SETUVAR fish_color_history_current:\x2d\x2dbold +SETUVAR fish_color_host:normal +SETUVAR fish_color_host_remote:yellow +SETUVAR fish_color_match:\x2d\x2dbackground\x3dbrblue +SETUVAR fish_color_normal:normal +SETUVAR fish_color_operator:00a6b2 +SETUVAR fish_color_param:00afff +SETUVAR fish_color_quote:999900 +SETUVAR fish_color_redirection:00afff +SETUVAR fish_color_search_match:bryellow\x1e\x2d\x2dbackground\x3dbrblack +SETUVAR fish_color_selection:white\x1e\x2d\x2dbold\x1e\x2d\x2dbackground\x3dbrblack +SETUVAR fish_color_status:red +SETUVAR fish_color_user:brgreen +SETUVAR fish_color_valid_path:\x2d\x2dunderline +SETUVAR fish_greeting:Welcome\x20to\x20fish\x2c\x20the\x20friendly\x20interactive\x20shell\x0aType\x20\x60help\x60\x20for\x20instructions\x20on\x20how\x20to\x20use\x20fish +SETUVAR fish_key_bindings:fish_default_key_bindings +SETUVAR fish_pager_color_completion:\x1d +SETUVAR fish_pager_color_description:B3A06D\x1eyellow +SETUVAR fish_pager_color_prefix:white\x1e\x2d\x2dbold\x1e\x2d\x2dunderline +SETUVAR fish_pager_color_progress:brwhite\x1e\x2d\x2dbackground\x3dcyan diff --git a/users/riking/dotfiles/fish/functions/ddate.fish b/users/riking/dotfiles/fish/functions/ddate.fish new file mode 100644 index 000000000000..8152d31680e7 --- /dev/null +++ b/users/riking/dotfiles/fish/functions/ddate.fish @@ -0,0 +1,3 @@ +function ddate --description 'current date in Discourse format' + TZ=UTC date '+[date=%Y-%m-%d time=%H:%M:%S timezone=\"%Z\"]' +end diff --git a/users/riking/dotfiles/fish/functions/gh-clone.fish b/users/riking/dotfiles/fish/functions/gh-clone.fish new file mode 100644 index 000000000000..109ec353f6db --- /dev/null +++ b/users/riking/dotfiles/fish/functions/gh-clone.fish @@ -0,0 +1,18 @@ +function gh-clone --description 'Clone and CD to a github repository' + if test (count $argv) -eq 1 + set user (string split "/" -- $argv[1])[1] + set repo (string split "/" -- $argv[1])[2] + else + set user $argv[1] + set repo $argv[2] + end + + if test -d "$HOME/go/src/github.com/$user/$repo" + cd "$HOME/go/src/github.com/$user/$repo" + return 0 + end + mkdir -p "$HOME/go/src/github.com/$user" + cd "$HOME/go/src/github.com/$user" + git clone "git@github.com:$user/$repo.git" + cd "$HOME/go/src/github.com/$user/$repo" +end diff --git a/users/riking/dotfiles/fish/functions/prodaccess.fish b/users/riking/dotfiles/fish/functions/prodaccess.fish new file mode 100644 index 000000000000..876c14c5e31e --- /dev/null +++ b/users/riking/dotfiles/fish/functions/prodaccess.fish @@ -0,0 +1,6 @@ +function prodaccess + ssh-add "$HOME/.ssh/id_ecdsa_sk" + begin; ssh-add -L | grep -q 'ZgEu6S3SLatYN'; end || ssh-add "$HOME"/.ssh/id_ed25519 + begin; ssh-add -L | grep -q 'Gfh2S3kUwZ8A6'; end || ssh-add "$HOME"/.ssh/id_rsa.discourse + echo "signing test" | gpg --clearsign > /dev/null +end diff --git a/users/riking/dotfiles/fish/functions/reset-audio.fish b/users/riking/dotfiles/fish/functions/reset-audio.fish new file mode 100644 index 000000000000..eb48578a52bc --- /dev/null +++ b/users/riking/dotfiles/fish/functions/reset-audio.fish @@ -0,0 +1,4 @@ +function reset-audio --description "Resets pulse and alsa" + pulseaudio -k + sudo alsa force-reload +end diff --git a/users/riking/dotfiles/fish/functions/tvl-push.fish b/users/riking/dotfiles/fish/functions/tvl-push.fish new file mode 100644 index 000000000000..f04ac830c005 --- /dev/null +++ b/users/riking/dotfiles/fish/functions/tvl-push.fish @@ -0,0 +1,3 @@ +function tvl-push + git push origin HEAD:refs/for/canon +end diff --git a/users/riking/dotfiles/regolith/Xresources b/users/riking/dotfiles/regolith/Xresources new file mode 100644 index 000000000000..f47b93511af0 --- /dev/null +++ b/users/riking/dotfiles/regolith/Xresources @@ -0,0 +1,5 @@ +#include "/etc/regolith/styles/ubuntu/root" + +i3-wm.program.lock: xset s activate +i3-wm.program.1: /bin/sh $HOME/.config/regolith/initrc + diff --git a/users/riking/dotfiles/regolith/flags/first-time-setup-r1-4-1 b/users/riking/dotfiles/regolith/flags/first-time-setup-r1-4-1 new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/users/riking/dotfiles/regolith/flags/first-time-setup-r1-4-1 diff --git a/users/riking/dotfiles/regolith/flags/show-shortcuts b/users/riking/dotfiles/regolith/flags/show-shortcuts new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/users/riking/dotfiles/regolith/flags/show-shortcuts diff --git a/users/riking/dotfiles/regolith/flags/term-profile b/users/riking/dotfiles/regolith/flags/term-profile new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/users/riking/dotfiles/regolith/flags/term-profile diff --git a/users/riking/dotfiles/regolith/flags/ui-fingerprint b/users/riking/dotfiles/regolith/flags/ui-fingerprint new file mode 100644 index 000000000000..b35aedd2dc0f --- /dev/null +++ b/users/riking/dotfiles/regolith/flags/ui-fingerprint @@ -0,0 +1 @@ +ec33ee15ff705ac4b167ba6b7f6df3c2 diff --git a/users/riking/dotfiles/regolith/initrc b/users/riking/dotfiles/regolith/initrc new file mode 100755 index 000000000000..9b14613cd4eb --- /dev/null +++ b/users/riking/dotfiles/regolith/initrc @@ -0,0 +1,3 @@ + +xset s 900 5 +( xss-lock -n /usr/lib/xsecurelock/dimmer -l -- sh -c "XSECURELOCK_PASSWORD_PROMPT=time_hex XSECURELOCK_SHOW_DATETIME=1 XSECURELOCK_SAVER=saver_mpv XSECURELOCK_IMAGE_DURATION_SECONDS=10 XSECURELOCK_LIST_VIDEOS_COMMAND='find ~/Videos/Screensaver -type f' xsecurelock" )& diff --git a/users/riking/dotfiles/tmux.conf b/users/riking/dotfiles/tmux.conf new file mode 100644 index 000000000000..1f253cb27f0c --- /dev/null +++ b/users/riking/dotfiles/tmux.conf @@ -0,0 +1,6 @@ + +set -g mouse on +set-option -g prefix C-a +bind-key C-a send-prefix +bind | split-window -h +bind - split-window -v diff --git a/users/riking/keys.nix b/users/riking/keys.nix new file mode 100644 index 000000000000..6dd2ff18a30f --- /dev/null +++ b/users/riking/keys.nix @@ -0,0 +1,20 @@ +# SSH public keys +{ ... }: + +rec { + sk-ecljg09 = "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBBwJ7dJJUkvIK+bDsVsCsCZSlbs90aOLsHN7XesC8/AmLA5rIRLO8I5ADoOjsWAXl/WAgxqOMmB4LxZjoXWa1a0AAAAEc3NoOg== riking@sk-ECLJG09"; + sk-portable1 = "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBCfA8/0nKk4jXclWHjRZIuicPeyIo9oDwahpnWjEATr7YaFDAo632KTSgqlW0lpx8lX9alLsJRhFV2XaSurYw/EAAAAEc3NoOg== riking@sk-portable1"; + sk-portable2 = "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBEX3DXreQR93SR68QZHTdaVd5RjlRM8C0jcZ4kI4OZwqk7xuk68w3g22q2OM7O+chj+n1N3u0hLxi82QfRnwyasAAAAEc3NoOg== riking@sk-portable2"; + sk-desktop = "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBB+JvN8nAxD+yo49Ohf/UDq7Z049yvkURJIA1XNbvKaAkvfWnCN5m9vTC1FyGxTyCwy4QpD1pFP5fIn0X/kvvfgAAAAEc3NoOg== riking@sk-kane-DAN-A4"; + + u2f = [sk-ecljg09 sk-portable1 sk-portable2 sk-desktop]; + + ed1 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAjWIfFH2bAWMZG+HudV1MVHWUl83M/ZgEu6S3SLatYN riking@kane-DAN-A4"; + ed2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICBblB4C9IgAijv+qN6Zs8TM2Sz7phQvVmRrcDn4VYNo riking@ECLJG09"; + + passworded = [ed1 ed2]; + + unprotected = []; + + all = u2f ++ passworded ++ unprotected; +} diff --git a/users/sterni/OWNERS b/users/sterni/OWNERS new file mode 100644 index 000000000000..cace4d0f3759 --- /dev/null +++ b/users/sterni/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - sterni diff --git a/users/sterni/clhs-lookup/README.md b/users/sterni/clhs-lookup/README.md new file mode 100644 index 000000000000..1f42ff43a210 --- /dev/null +++ b/users/sterni/clhs-lookup/README.md @@ -0,0 +1,13 @@ +# clhs-lookup + +Simple cli to lookup symbols' documentation in a local copy of the +Common Lisp HyperSpec. + +## usage + +``` +clhs-lookup [--print] symbol [symbol [...]] + + --print Print documentation paths to stdout instead of + opening them with $BROWSER (defaults to xdg-open). +``` diff --git a/users/sterni/clhs-lookup/clhs-lookup.lisp b/users/sterni/clhs-lookup/clhs-lookup.lisp new file mode 100644 index 000000000000..0e61dd901f93 --- /dev/null +++ b/users/sterni/clhs-lookup/clhs-lookup.lisp @@ -0,0 +1,46 @@ +(in-package :clhs-lookup) +(declaim (optimize (safety 3))) + +(defun find-symbols-paths (syms clhs) + "Find pathnames to HyperSpec files describing the listed + symbol names (as strings). Paths are returned in the order + of the symbols given with missing entries removed." + (check-type syms list) + (check-type clhs pathname) + (let* ((data-dir (merge-pathnames "HyperSpec/Data/" clhs)) + (data (merge-pathnames "Map_Sym.txt" data-dir)) + (found (make-hash-table :test #'equal)) + (syms (mapcar #'string-upcase syms))) + (with-open-file (s data :direction :input) + (loop + with missing = syms + for symbol-line = (read-line s nil :eof) + for path-line = (read-line s nil :eof) + until (or (eq symbol-line :eof) + (eq path-line :eof) + (null missing)) + for pos = (position symbol-line missing :test #'equal) + when pos + do (progn + (delete symbol-line missing) + (setf (gethash symbol-line found) path-line))) + ; TODO(sterni): get rid of Data/../ in path + (mapcar + (lambda (x) (merge-pathnames x data-dir)) + (remove nil + (mapcar (lambda (x) (gethash x found)) syms)))))) + +(defun main () + (let* ((browser (or (uiop:getenvp "BROWSER") "xdg-open")) + (args (uiop:command-line-arguments)) + (prin (member "--print" args :test #'equal)) + (syms (remove-if (lambda (x) (eq (char x 0) #\-)) args)) + (paths (find-symbols-paths syms *clhs-path*))) + (if (null paths) + (uiop:quit 1) + (dolist (p paths) + (if prin + (format t "~A~%" p) + (uiop:launch-program + (format nil "~A ~A" browser p) + :force-shell t)))))) diff --git a/users/sterni/clhs-lookup/default.nix b/users/sterni/clhs-lookup/default.nix new file mode 100644 index 000000000000..b6a0bd06790f --- /dev/null +++ b/users/sterni/clhs-lookup/default.nix @@ -0,0 +1,39 @@ +{ pkgs, depot, ... }: + +let + inherit (pkgs) fetchzip writeText; + inherit (depot.nix) buildLisp; + inherit (builtins) replaceStrings; + + clhsVersion = "7-0"; + + clhs = fetchzip { + name = "HyperSpec-${replaceStrings [ "-" ] [ "." ] clhsVersion}"; + url = "ftp://ftp.lispworks.com/pub/software_tools/reference/HyperSpec-${clhsVersion}.tar.gz"; + sha256 = "1zsi35245m5sfb862ibzy0pzlph48wvlggnqanymhgqkpa1v20ak"; + stripRoot = false; + }; + + clhs-path = writeText "clhs-path.lisp" '' + (in-package :clhs-lookup.clhs-path) + (defparameter *clhs-path* (pathname "${clhs}/")) + ''; + + clhs-lookup = buildLisp.program { + name = "clhs-lookup"; + + deps = [ + { + default = buildLisp.bundled "asdf"; + sbcl = buildLisp.bundled "uiop"; + } + ]; + + srcs = [ + ./packages.lisp + clhs-path + ./clhs-lookup.lisp + ]; + }; +in + clhs-lookup diff --git a/users/sterni/clhs-lookup/packages.lisp b/users/sterni/clhs-lookup/packages.lisp new file mode 100644 index 000000000000..d059b96ce9f0 --- /dev/null +++ b/users/sterni/clhs-lookup/packages.lisp @@ -0,0 +1,10 @@ +(defpackage :clhs-lookup.clhs-path + (:use :cl) + (:export :*clhs-path*)) + +(defpackage clhs-lookup + (:use :cl :uiop) + (:import-from :clhs-lookup.clhs-path :*clhs-path*) + (:export :main + :find-symbols-paths)) + diff --git a/users/sterni/dot-time-man-pages/OWNERS b/users/sterni/dot-time-man-pages/OWNERS new file mode 100644 index 000000000000..980c17b424f2 --- /dev/null +++ b/users/sterni/dot-time-man-pages/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - edef diff --git a/users/sterni/dot-time-man-pages/default.nix b/users/sterni/dot-time-man-pages/default.nix new file mode 100644 index 000000000000..bf7d63dbd797 --- /dev/null +++ b/users/sterni/dot-time-man-pages/default.nix @@ -0,0 +1,70 @@ +{ depot, lib, ... }: + +let + # TODO(sterni): find a better place for this: is dot time //fun? + + # get the email address of a depot user from //ops/users + findEmail = user: + let + res = builtins.filter ({ username, ... }: username == user) depot.ops.users; + len = builtins.length res; + in + if len == 1 + then (builtins.head res).email + else builtins.throw "findEmail: got ${toString len} results instead of 1"; + + # dot-time(7) man page, ported from dotti.me + dot-time = rec { + name = "dot-time"; + section = 7; + content = '' + .Dd $Mdocdate$ + .Dt ${lib.toUpper name} ${toString section} + .Os + .Sh NAME + .Nm ${name} + .Nd a universal convention for conveying time + .Sh DESCRIPTION + For those of us who travel often or coordinate across many timezones, + working with local time is frequently impractical. + ISO8601, in all its wisdom, allows for time zone designators, + but still represents the hours and minutes as local time, + thus making it inconvenient for quickly comparing timestamps from + different locations. + .Pp + Dot time instead uses UTC for all date, hour, and minute indications, + and while it allows for time zone designators, they are optional + information that can be dropped without changing the indicated time. + It uses an alternate hour separator to make it easy to distinguish from + regular ISO8601. + When a time zone designator is provided, one can easily obtain + the matching local time by adding the UTC offset to the UTC time. + .Sh EXAMPLES + These timestamps all represent the same point in time. + .TS + allbox tab(|); + lb | lb | lb + l | l | l. + dot time|ISO8601|RFC3339 + 2019-06-19T22·13-04|2019-06-19T18:13-04|2019-06-19T18:13:00-04:00 + 2019-06-19T22·13+00|2019-06-19T22:13+00|2019-06-19T22:13:00Z + 2019-06-19T22·13+02|2019-06-20T00:13+02|2019-06-20T00:13:00+02:00 + .TE + .Sh SEE ALSO + .Lk https://dotti.me dotti.me + .Sh AUTHORS + .An -nosplit + .Sy dot time + has been proposed and documented by + .An edef Aq Mt ${findEmail "edef"} + and ported to + .Xr mdoc 7 + by + .An sterni Aq Mt ${findEmail "sterni"} . + ''; + }; + +in + depot.nix.buildManPages "dot-time" {} [ + dot-time + ] diff --git a/users/sterni/emacs/default.nix b/users/sterni/emacs/default.nix new file mode 100644 index 000000000000..f7bdf21a025e --- /dev/null +++ b/users/sterni/emacs/default.nix @@ -0,0 +1,43 @@ +{ depot, pkgs, ... }: + +let + inherit (pkgs.emacsGcc.pkgs) withPackages; + + emacs = withPackages (epkgs: [ + # basic setup + epkgs.elpaPackages.undo-tree + epkgs.melpaPackages.evil + epkgs.melpaPackages.evil-collection + epkgs.melpaPackages.use-package + # languages + epkgs.bqn-mode + epkgs.elpaPackages.ada-mode + epkgs.melpaPackages.adoc-mode + epkgs.melpaPackages.dockerfile-mode + epkgs.melpaPackages.haskell-mode + epkgs.melpaPackages.jq-mode + epkgs.melpaPackages.markdown-mode + epkgs.melpaPackages.nix-mode + epkgs.melpaPackages.sly + epkgs.melpaPackages.yaml-mode + epkgs.rust-mode + epkgs.urweb-mode + # misc + epkgs.melpaPackages.hl-todo + epkgs.elpaPackages.rainbow-mode + epkgs.melpaPackages.rainbow-delimiters + # beyond text editing + epkgs.melpaPackages.elfeed + epkgs.melpaPackages.magit + epkgs.tvlPackages.tvl + ]); +in + +# sadly we can't give an init-file via the command line +pkgs.writeShellScriptBin "emacs" '' + exec ${emacs}/bin/emacs \ + --no-init-file \ + --directory ${./.} \ + --eval "(require 'init)" \ + "$@" +'' diff --git a/users/sterni/emacs/init.el b/users/sterni/emacs/init.el new file mode 100644 index 000000000000..4b868cb242d1 --- /dev/null +++ b/users/sterni/emacs/init.el @@ -0,0 +1,184 @@ +;; Set default font and fallback font via set-fontset-font +;; TODO(sterni): Investigate non-emoji representation of some glyphs +(let ((mono-font "Bitstream Vera Sans Mono-12") + (emoji-font "Noto Color Emoji-12")) + (setq default-frame-alist `((font . ,mono-font))) + (set-frame-font mono-font t t) + (set-fontset-font t nil emoji-font)) + +(setq inhibit-startup-message t + display-time-24hr-format t + select-enable-clipboard t) + +;; Reload files +(global-auto-revert-mode 1) + +;; Indent +(set-default 'indent-tabs-mode nil) +(setq tab-width 2) + +;; UTF-8 +(setq locale-coding-system 'utf-8) +(set-terminal-coding-system 'utf-8) +(set-keyboard-coding-system 'utf-8) +(set-selection-coding-system 'utf-8) +(prefer-coding-system 'utf-8) + +;; Disable unnecessary GUI elements +(scroll-bar-mode 0) +(menu-bar-mode 0) +(tool-bar-mode 0) + +(add-hook 'after-make-frame-functions + (lambda (frame) (scroll-bar-mode 0))) + +;; don't center on cursor when scrolling +(setq scroll-conservatively 1) + +;; type less +(defalias 'yes-or-no-p 'y-or-n-p) + +;; Extra settings when graphical session +(when window-system + (setq frame-title-format '(buffer-file-name "%f" ("%b"))) + (mouse-wheel-mode t) + (blink-cursor-mode -1)) + +;; TODO(sterni): prevent some remaining backup files +(setq auto-save-file-name-transforms + `((".*" ,temporary-file-directory t))) +(setq backup-directory-alist + `((".*" . ,temporary-file-directory))) + +;; buffers +;; unique component should come first for better completion +(setq uniquify-buffer-name-style 'forward) + +;; Display column numbers +(column-number-mode t) +(setq-default fill-column 80) +(setq display-fill-column-indicator-column t) +(add-hook 'prog-mode-hook #'display-fill-column-indicator-mode) + +;; whitespace +(setq whitespace-style '(face trailing tabs) + whitespace-line-column fill-column) +(add-hook 'prog-mode-hook #'whitespace-mode) + +;;; Configure built in modes + +;; Perl +(setq perl-indent-level 2) +(setq perl-continued-statement-offset 0) +(setq perl-continued-brace-offset 0) + +;;; Configure packages +(require 'use-package) + +(package-initialize) + +(use-package undo-tree + :config + (global-undo-tree-mode)) + +(use-package magit + :after evil + :config + ; reset (buffer-local) fill-column value to emacs' default + ; gerrit doesn't like 80 column commit messages… + (add-hook 'git-commit-mode-hook (lambda () (setq fill-column 72))) + (evil-define-key 'normal 'global (kbd "<leader>gr") 'magit-status)) +(use-package tvl :after magit) + +(setq ediff-split-window-function 'split-window-horizontally) + +(use-package evil + :init + (setq evil-want-integration t) + (setq evil-want-keybinding nil) + (setq evil-shift-width 2) + (setq evil-split-window-below t) + (setq evil-split-window-right t) + (setq evil-undo-system 'undo-tree) + :config + (evil-mode 1) + (evil-set-leader 'normal ",") ;; TODO(sterni): space would be nice, but… + (evil-set-leader 'visual ",") + ;; buffer management + (evil-define-key 'normal 'global (kbd "<leader>bk") 'kill-buffer) + (evil-define-key 'normal 'global (kbd "<leader>bb") 'switch-to-buffer) + ;; window management + (evil-define-key 'normal 'global (kbd "<leader>wk") 'delete-window) + (evil-define-key 'normal 'global (kbd "<leader>wo") 'delete-other-window) + (evil-define-key 'normal 'global (kbd "<leader>wh") 'split-window-below) + (evil-define-key 'normal 'global (kbd "<leader>wv") 'split-window-right) + (evil-define-key 'normal 'global (kbd "<leader>ww") 'other-window) + ;; emacs + (evil-define-key 'visual 'global (kbd "<leader>ee") 'eval-region) + (evil-define-key 'normal 'global (kbd "<leader>ee") 'eval-last-sexp) + (evil-define-key 'normal 'global (kbd "<leader>ep") 'eval-print-last-sexp) + (evil-define-key 'normal 'global (kbd "<leader>eh") 'help) + ;; modify what is displayed + (evil-define-key 'normal 'global (kbd "<leader>dw") + (lambda () + (interactive) + (whitespace-mode 'toggle) + (display-fill-column-indicator-mode 'toggle))) + ;; elfeed bindings for evil (can't use-package elfeed apparently) + (evil-define-key 'normal 'global (kbd "<leader>ff") 'elfeed) + (evil-define-key '(normal visual) elfeed-search-mode-map + (kbd "o") 'elfeed-search-browse-url + (kbd "r") 'elfeed-search-untag-all-unread + (kbd "u") 'elfeed-search-tag-all-unread + (kbd "<leader>ff") 'elfeed-search-fetch + (kbd "<leader>fc") 'elfeed-db-compact + (kbd "<leader>fr") 'elfeed-search-update--force)) + +(use-package evil-collection + :after evil + :config + (evil-collection-init)) + +(use-package rainbow-delimiters + :hook (prog-mode . rainbow-delimiters-mode)) + +(use-package nix-mode :mode "\\.nix\\'") +(use-package nix-drv-mode :mode "\\.drv\\'") + +(use-package haskell-mode) +(use-package urweb-mode) +(use-package bqn-mode + :mode "\\.bqn\\'" + :custom bqn-mode-map-prefix "C-s-") ; probably rather using C-\ +(use-package yaml-mode) +(use-package dockerfile-mode) +(use-package jq-mode + :config (add-to-list 'auto-mode-alist '("\\.jq\\'" . jq-mode))) +(use-package rust-mode) +(use-package sly + :after evil + :hook ((sly-mrepl-mode . (lambda () (rainbow-delimiters-mode-enable)))) + :config + (evil-define-key 'normal sly-mrepl-mode-map (kbd "C-r") 'isearch-backward)) + +(use-package ada-mode) + +(use-package rainbow-mode) +(use-package hl-todo + :hook ((prog-mode . hl-todo-mode)) + :config + (setq hl-todo-keyword-faces + '(("TODO" . "#FF0000") + ("FIXME" . "#FF0000") + ("HACK" . "#7f7f7f") + ("XXX" . "#aa0000")))) + +(use-package markdown-mode + :commands (markdown-mode gfm-mode) + :mode (("\\.md\\'" . markdown-mode))) +(use-package adoc-mode + :mode (("\\.adoc\\'" . adoc-mode))) + +(require 'subscriptions) + +(provide 'init) diff --git a/users/sterni/emacs/subscriptions.el b/users/sterni/emacs/subscriptions.el new file mode 100644 index 000000000000..bf890a5ab8b3 --- /dev/null +++ b/users/sterni/emacs/subscriptions.el @@ -0,0 +1,88 @@ +;;; elfeed subscriptions + +(setq elfeed-feeds + (append + ;; immutable subscriptions tracked in git + '(("https://repology.org/maintainer/sternenseemann%40systemli.org/feed-for-repo/nix_unstable/atom" dashboard releases) + ("http://hundimbuero.blogspot.com/feeds/posts/default?alt=rss" blog cool-and-nice) + ("gopher://text.causal.agency/0feed.atom" blog) + ("http://xsteadfastx.org/feed/" blog cool-and-nice) + ("https://tvl.fyi/feed.atom" blog cool-and-nice) + ("https://hannes.robur.coop/atom" blog) + ("https://stevelosh.com/rss.xml" blog) + ("https://planet.lisp.org/rss20.xml" blog) + ("https://hyperthings.garden/rss/all-posts.xml" blog) + ("https://blog.benjojo.co.uk/rss.xml" blog) + ("https://leahneukirchen.org/blog/index.atom" blog cool-and-nice) + ("https://leahneukirchen.org/trivium/index.atom" blog links cool-and-nice) + ("https://firefly.nu/feeds/all.atom.xml" blog cool-and-nice) + ("https://tazj.in/feed.atom" blog cool-and-nice) + ("https://alyssa.is/feed.xml" blog cool-and-nice) + ("https://eta.st/feed.xml" blog cool-and-nice) + ("https://h.eta.st/rss" noisy cool-and-nice) + ("https://spectrum-os.org/git/www/atom/bibliography.html" links blog) + ("https://rachelbythebay.com/w/atom.xml" blog) + ("http://evrl.com/feed.xml" blog) + ("https://vulns.xyz/feed.xml" blog) + ("https://www.german-foreign-policy.com/?type=9818" news) + ("https://niedzejkob.p4.team/rss.xml" blog) + ("https://grahamc.com/feed/" blog) + ("https://michael.stapelberg.ch/feed.xml" blog) + ("https://kazu-yamamoto.hatenablog.jp/feed" blog) + ("https://ariadne.space/feed/" blog) + ("https://bodil.lol/rss.xml" blog) + ("http://blog.nullspace.io/feed.xml" blog) + ("https://blog.kingcons.io/rss.xml" blog) + ("http://jaspervdj.be/rss.xml" blog) + ("https://christine.website/blog.rss" blog) + ("https://drewdevault.com/feed.xml" blog) + ("https://www.imperialviolet.org/iv-rss.xml" blog) + ("https://latacora.micro.blog/feed.xml" blog) + ("https://22gato.tumblr.com/rss" pictures cool-and-nice) + ("https://theprofoundprogrammer.com/rss" blog) + ("https://wiki.openlab-augsburg.de/_feed" openlab) + ("http://shitopenlabsays.tumblr.com/rss" openlab) + ("http://suckless.org/atom.xml" releases) + ("https://kristaps.bsd.lv/lowdown/atom.xml" releases) + ("https://www.tweag.io/rss.xml" blog) + ("http://planet.haskell.org/atom.xml" planet blog) + ("http://0pointer.net/blog/index.atom" blog) + ("https://emacsninja.com/feed.atom" blog) + ("https://emacshorrors.com/feed.atom" blog) + ("http://therealmntmn.tumblr.com/rss" blog) + ("http://blog.duangle.com/feeds/posts/default" blog) + ("http://blog.johl.io/atom.xml" blog) + ("http://blog.z3bra.org/rss/feed.xml" blog) + ("http://ccc.de/de/rss/updates.xml" news) + ;; ("http://fabienne.us/feed/" blog) ; database error + ("http://feeds.feedburner.com/baschtcom" blog) + ("http://ffaaaffaffaffaa.tumblr.com/rss" pictures) + ("http://fnordig.de/feed.xml" blog) + ("http://fotografiona.tumblr.com/rss" pictures) + ("https://grandhotel-cosmopolis.org/de/feed" news) + ("http://guteaussicht.org/rss" pictures) + ("http://konvergenzfehler.de/feed/" blog) + ("https://markuscisler.com/feed.xml" blog) + ("http://n00bcore.de/feed/" podcast) + ("http://spacethatneverwas.tumblr.com/rss" pictures) + ("http://theresa.someserver.de/blog/?feed=rss2" blog) + ("http://www.frumble.de/blog/feed/" blog) + ("http://www.plomlompom.de/PlomRogue/plomwiki.php?action=Blog_Atom" blog) + ("http://www.whvrt.de/rss" pictures) + ("http://www.windytan.com/feeds/posts/default" blog) + ("https://echtsuppe.wordpress.com/feed/" blog defunct) + ("https://mgsloan.com/feed.xml" blog) + ("https://notes.sterni.lv/atom.xml" me) + ("http://arduina.net/feed/" defunct blog) + ("https://anchor.fm/s/94bb000/podcast/rss" podcast)) + ;; http://www.wollenzin.de/feed/ ;_; + + ;; add more feeds from an untracked file in $HOME + (let ((file (concat (getenv "HOME") + "/.config/emacs-custom/mutable-subscriptions.el"))) + (when (file-exists-p file) + (read (with-temp-buffer + (insert-file-contents file) + (buffer-string))))))) + +(provide 'subscriptions) diff --git a/users/sterni/exercises/aoc/.gitignore b/users/sterni/exercises/aoc/.gitignore new file mode 100644 index 000000000000..de53cfc531cb --- /dev/null +++ b/users/sterni/exercises/aoc/.gitignore @@ -0,0 +1 @@ +/*/input \ No newline at end of file diff --git a/users/sterni/exercises/aoc/2021/solutions.bqn b/users/sterni/exercises/aoc/2021/solutions.bqn new file mode 100755 index 000000000000..98d70f8f15cf --- /dev/null +++ b/users/sterni/exercises/aoc/2021/solutions.bqn @@ -0,0 +1,31 @@ +#!/usr/bin/env BQN + +# +# Utilities +# + +ReadInt ← (10⊸×+⊣)´∘⌽-⟜'0' # stolen from leah2 + +ReadInput ← {ReadInt¨•file.Lines ∾ •path‿"/input/day"‿(•Fmt 𝕩)} + +# +# 2021-12-01 +# + +# part 1 + +day1ExampleData ← 199‿200‿208‿210‿200‿207‿240‿269‿260‿263 + +# NB: Because distance from the ground is never smaller than zero, it's +# no problem that nudge inserts a zero at the end of the right list +PositiveDeltaCount ← +´∘(⊢<«)+˝˘∘↕ + +! 7 = 1 PositiveDeltaCount day1ExampleData + +•Out "Day 1.1: "∾•Fmt 1 PositiveDeltaCount ReadInput 1 + +# part 2 + +! 5 = 3 PositiveDeltaCount day1ExampleData + +•Out "Day 1.2: "∾•Fmt 3 PositiveDeltaCount ReadInput 1 diff --git a/users/sterni/htmlman/README.md b/users/sterni/htmlman/README.md new file mode 100644 index 000000000000..258233d4c4d2 --- /dev/null +++ b/users/sterni/htmlman/README.md @@ -0,0 +1,36 @@ +# htmlman + +static site generator for man pages intended for +rendering man page documentation viewable using +a web browser. + +## usage + +If you have a nix expression, `doc.nix`, like this: + +```nix +{ depot, ... }: + +depot.users.sterni.htmlman { + title = "foo project"; + pages = [ + { + name = "foo"; + section = 1; + } + { + name = "foo"; + section = 3; + path = ../devman/foo.3; + } + ]; + manDir = ../man; +} +``` + +You can run the following to directly deploy the resulting +documentation output to a specific target directory: + +```sh +nix-build -A deploy doc.nix && ./result target_directory +``` diff --git a/users/sterni/htmlman/default.nix b/users/sterni/htmlman/default.nix new file mode 100644 index 000000000000..b88bc264103b --- /dev/null +++ b/users/sterni/htmlman/default.nix @@ -0,0 +1,234 @@ +{ depot, lib, pkgs, ... }: + +let + inherit (depot.nix) + getBins + runExecline + yants + ; + + inherit (depot.tools) + cheddar + ; + + inherit (pkgs) + mandoc + coreutils + fetchurl + writers + ; + + bins = getBins cheddar [ "cheddar" ] + // getBins mandoc [ "mandoc" ] + // getBins coreutils [ "cat" "mv" "mkdir" ] + ; + + normalizeDrv = fetchurl { + url = "https://necolas.github.io/normalize.css/8.0.1/normalize.css"; + sha256 = "04jmvybwh2ks4dlnfa70sb3a3z3ig4cv0ya9rizjvm140xq1h22q"; + }; + + execlineStdoutInto = target: line: [ + "redirfd" "-w" "1" target + ] ++ line; + + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + # I will not write a pure nix markdown renderer + markdown = md: + let + html = runExecline.local "rendered-markdown" { + stdin = md; + } ([ + "importas" "-iu" "out" "out" + ] ++ execlineStdoutInto "$out" [ + bins.cheddar "--about-filter" "description.md" + ]); + in builtins.readFile html; + + indexTemplate = { title, description, pages ? [] }: '' + <!doctype html> + <html> + <head> + <meta charset="utf-8"> + <title>${title}</title> + <link rel="stylesheet" type="text/css" href="style.css"/> + </head> + <body> + <div class="index-text"> + <h1>${title}</h1> + ${markdown description} + <h2>man pages</h2> + <ul> + ${lib.concatMapStrings ({ name, section, ... }: '' + <li><a href="${name}.${toString section}.html">${name}(${toString section})</a></li> + '') pages} + </ul> + </div> + </body> + </html> + ''; + + defaultStyle = import ./defaultStyle.nix { }; + + # This deploy script automatically copies the build result into + # a TARGET directory and marks it as writeable optionally. + # It is exposed as the deploy attribute of the result of + # htmlman, so an htmlman expression can be used like this: + # nix-build -A deploy htmlman.nix && ./result target_dir + deployScript = title: drv: writers.writeDash "deploy-${title}" '' + usage() { + printf 'Usage: %s [-w] TARGET\n\n' "$0" + printf 'Deploy htmlman documentation to TARGET directory.\n\n' + printf ' -h Display this help message\n' + printf ' -w Make TARGET directory writeable\n' + } + + if test "$#" -lt 1; then + usage + exit 100 + fi + + writeable=false + + while test "$#" -gt 0; do + case "$1" in + -h) + usage + exit 0 + ;; + -w) + writeable=true + ;; + -*) + usage + exit 100 + ;; + *) + if test -z "$target"; then + target="$1" + else + echo "Too many arguments" + exit 100 + fi + ;; + esac + + shift + done + + if test -z "$target"; then + echo "Missing TARGET" + usage + exit 100 + fi + + set -ex + + mkdir -p "$target" + cp -RTL --reflink=auto "${drv}" "$target" + + if $writeable; then + chmod -R +w "$target" + fi + ''; + + htmlman = + { title + # title of the index page + , description ? "" + # description which is displayed after + # the main heading on the index page + , pages ? [] + # man pages of the following structure: + # { + # name : string; + # section : int; + # path : either path string; + # } + # path is optional, if it is not given, + # the man page source must be located at + # "${manDir}/${name}.${toString section}" + , manDir ? null + # directory in which man page sources are located + , style ? defaultStyle + # CSS to use as a string + , normalizeCss ? true + # whether to include normalize.css before the custom CSS + , linkXr ? "all" + # How to handle cross references in the html output: + # + # * none: don't convert cross references into hyperlinks + # * all: link all cross references as if they were + # rendered into $out by htmlman + # * inManDir: link to all man pages which have their source + # in `manDir` and use the format string defined + # in linkXrFallback for all other cross references. + , linkXrFallback ? "https://manpages.debian.org/unstable/%N.%S.en.html" + # fallback link to use if linkXr == "inManDir" and the man + # page is not in ${manDir}. Placeholders %N (name of page) + # and %S (section of page) can be used. See mandoc(1) for + # more information. + }: + + let + linkXrEnum = yants.enum "linkXr" [ "all" "inManDir" "none" ]; + + index = indexTemplate { + inherit title description pages; + }; + + resolvePath = { path ? null, name, section }: + if path != null + then path + else "${manDir}/${name}.${toString section}"; + + mandocOpts = lib.concatStringsSep "," ([ + "style=style.css" + ] ++ linkXrEnum.match linkXr { + all = [ "man=./%N.%S.html" ]; + inManDir = [ "man=./%N.%S.html;${linkXrFallback}" ]; + none = [ ]; + }); + + html = + runExecline.local "htmlman-${title}" { + derivationArgs = { + inherit index style; + passAsFile = [ "index" "style" ]; + }; + } ([ + "multisubstitute" [ + "importas" "-iu" "out" "out" + "importas" "-iu" "index" "indexPath" + "importas" "-iu" "style" "stylePath" + ] + "if" [ bins.mkdir "-p" "$out" ] + "if" [ bins.mv "$index" "\${out}/index.html" ] + "if" (execlineStdoutInto "\${out}/style.css" [ + "if" ([ + bins.cat + ] ++ lib.optional normalizeCss normalizeDrv + ++ [ + "$style" + ]) + ]) + # let mandoc check for available man pages + "execline-cd" "${manDir}" + ] ++ lib.concatMap ({ name, section, ... }@p: + execlineStdoutInto "\${out}/${name}.${toString section}.html" [ + "if" [ + bins.mandoc + "-mdoc" + "-T" "html" + "-O" mandocOpts + (resolvePath p) + ] + ]) pages); + in html // { + deploy = deployScript title html; + }; +in + htmlman diff --git a/users/sterni/htmlman/defaultStyle.nix b/users/sterni/htmlman/defaultStyle.nix new file mode 100644 index 000000000000..a44b5ef06934 --- /dev/null +++ b/users/sterni/htmlman/defaultStyle.nix @@ -0,0 +1,49 @@ +{ ... }: + +'' + body { + font-size: 1em; + line-height: 1.5; + font-family: serif; + background-color: #efefef; + } + + h1, h2, h3, h4, h5, h6 { + font-family: sans-serif; + font-size: 1em; + margin: 5px 0; + } + + h1 { + margin-top: 0; + } + + a:link, a:visited { + color: #3e7eff; + } + + h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + text-decoration: none; + } + + .manual-text, .index-text { + padding: 20px; + max-width: 800px; + background-color: white; + margin: 0 auto; + } + + table.head, table.foot { + display: none; + } + + .Nd { + display: inline; + } + + /* use same as cheddar for man pages */ + pre { + padding: 16px; + background-color: #f6f8fa; + } +'' diff --git a/users/sterni/keys.nix b/users/sterni/keys.nix new file mode 100644 index 000000000000..815f62ee080e --- /dev/null +++ b/users/sterni/keys.nix @@ -0,0 +1,7 @@ +{ ... }: + +{ + all = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJk+KvgvI2oJTppMASNUfMcMkA2G5ZNt+HnWDzaXKLlo lukas@wolfgang" + ]; +} diff --git a/users/sterni/mblog/cli.lisp b/users/sterni/mblog/cli.lisp new file mode 100644 index 000000000000..93be7e8b8e44 --- /dev/null +++ b/users/sterni/mblog/cli.lisp @@ -0,0 +1,17 @@ +(in-package :mblog) +(declaim (optimize (safety 3))) + +(defparameter +synopsis+ "mnote-html FILE [FILE [ ... ]]") + +;; TODO(sterni): handle relevant conditions +(defun main () + (let* ((args (uiop:command-line-arguments)) + (help-p (or (not args) + (find-if (lambda (x) + (member x '("-h" "--help" "--usage") + :test #'string=)) + args)))) + (if help-p (format *error-output* "Usage: ~A~%" +synopsis+) + (loop for arg in args + do (apple-note-html-fragment + (mime:mime-message (pathname arg)) *standard-output*))))) diff --git a/users/sterni/mblog/default.nix b/users/sterni/mblog/default.nix new file mode 100644 index 000000000000..16ae573ba78c --- /dev/null +++ b/users/sterni/mblog/default.nix @@ -0,0 +1,31 @@ +{ depot, pkgs, ... }: + +depot.nix.buildLisp.program { + name = "mnote-html"; + + srcs = [ + ./packages.lisp + ./transformer.lisp + ./note.lisp + ./cli.lisp + ]; + + deps = [ + { + sbcl = depot.nix.buildLisp.bundled "uiop"; + default = depot.nix.buildLisp.bundled "asdf"; + } + depot.third_party.lisp.alexandria + depot.third_party.lisp.closure-html + depot.third_party.lisp.cl-who + depot.third_party.lisp.mime4cl + ]; + + main = "mblog:main"; + + # due to sclf + brokenOn = [ + "ccl" + "ecl" + ]; +} diff --git a/users/sterni/mblog/note.lisp b/users/sterni/mblog/note.lisp new file mode 100644 index 000000000000..fa4de0956ffb --- /dev/null +++ b/users/sterni/mblog/note.lisp @@ -0,0 +1,60 @@ +(in-package :mblog) +(declaim (optimize (safety 3))) + +;;; util + +(defun html-escape-stream (in out) + "Escape characters read from stream IN and write them to + stream OUT escaped using WHO:ESCAPE-CHAR-MINIMAL." + (loop for char = (read-char in nil nil) + while char + do (write-string (who:escape-char-minimal char) out))) + +(defun cid-header-value (cid) + "Takes a Content-ID as present in Apple Notes' <object> tags and properly + surrounds them with angle brackets for a MIME header" + (concatenate 'string "<" cid ">")) + +;;; main implementation + +;; TODO(sterni): make this a “parser” instead of a predicate +(defun apple-note-p (msg) + "Checks X-Uniform-Type-Identifier of a MIME:MIME-MESSAGE + to determine if a given mime message is an Apple Note." + (when-let (uniform-id (assoc "X-Uniform-Type-Identifier" + (mime:mime-message-headers msg) + :test #'string=)) + (string= (cdr uniform-id) "com.apple.mail-note"))) + +(defun apple-note-html-fragment (msg out) + "Takes a MIME:MIME-MESSAGE and writes its text content as HTML to + the OUT stream. The <object> tags are resolved to <img> which + refer to the respective attachment's filename as a relative path, + but extraction of the attachments must be done separately. The + surrounding <html> and <body> tags are stripped and <head> + discarded completely, so only a fragment which can be included + in custom templates remains." + (let ((text (find-mime-text-part msg))) + (cond + ;; Sanity checking of the note + ((not (apple-note-p msg)) + (error "Unsupported or missing X-Uniform-Type-Identifier")) + ((not text) (error "Malformed Apple Note: no text part")) + ;; notemap creates text/plain notes we need to handle properly. + ;; Additionally we *could* check X-Mailer which notemap sets + ((string= (mime:mime-subtype text) "plain") + (html-escape-stream (mime:mime-body-stream text :binary nil) out)) + ;; Notes.app creates text/html parts + ((string= (mime:mime-subtype text) "html") + (closure-html:parse + (mime:mime-body-stream text) + (make-instance + 'apple-note-transformer + :cid-lookup + (lambda (cid) + (when-let* ((part (mime:find-mime-part-by-id msg (cid-header-value cid))) + (file (mime:mime-part-file-name part))) + file)) + :next-handler + (closure-html:make-character-stream-sink out)))) + (t (error "Malformed Apple Note: unknown mime type"))))) diff --git a/users/sterni/mblog/packages.lisp b/users/sterni/mblog/packages.lisp new file mode 100644 index 000000000000..ca2e41b6827f --- /dev/null +++ b/users/sterni/mblog/packages.lisp @@ -0,0 +1,15 @@ +(defpackage :mblog + (:use + :common-lisp + :mime4cl + :closure-html + :who + :uiop) + (:shadow :with-html-output) ; conflict between closure-html and who + (:import-from + :alexandria + :when-let* + :when-let + :starts-with-subseq + :ends-with-subseq) + (:export :main)) diff --git a/users/sterni/mblog/transformer.lisp b/users/sterni/mblog/transformer.lisp new file mode 100644 index 000000000000..f26c5652a266 --- /dev/null +++ b/users/sterni/mblog/transformer.lisp @@ -0,0 +1,127 @@ +(in-package :mblog) +(declaim (optimize (safety 3))) + +;; Throw away these tags and all of their children +(defparameter +discard-tags-with-children+ '("HEAD")) +;; Only “strip” these tags and leave their content as is +(defparameter +discard-tags-only+ '("BODY" "HTML")) + +;; This is basically the same as cxml's PROXY-HANDLER. +;; Couldn't be bothered to make a BROADCAST-HANDLER because I +;; only need to pass through to one handler. It accepts every +;; event and passes it on to NEXT-HANDLER. This is useful for +;; subclassing mostly where an event can be modified or passed +;; on as is via CALL-NEXT-METHOD. +(defclass hax-proxy-handler (hax:default-handler) + ((next-handler + :initarg :next-handler + :accessor proxy-next-handler))) + +;; Define the trivial handlers which just call themselves for NEXT-HANDLER +(macrolet ((def-proxy-handler (name (&rest args)) + `(defmethod ,name ((h hax-proxy-handler) ,@args) + (,name (proxy-next-handler h) ,@args)))) + (def-proxy-handler hax:start-document (name p-id s-id)) + (def-proxy-handler hax:end-document ()) + (def-proxy-handler hax:start-element (name attrs)) + (def-proxy-handler hax:end-element (name)) + (def-proxy-handler hax:characters (data)) + (def-proxy-handler hax:unescaped (data)) + (def-proxy-handler hax:comment (data))) + +(defclass apple-note-transformer (hax-proxy-handler) + ((cid-lookup + :initarg :cid-lookup + :initform (lambda (cid) nil) + :accessor transformer-cid-lookup) + (discard-until + :initarg :discard-until + :initform nil + :accessor transformer-discard-until) + (depth + :initarg :depth + :initform 0 + :accessor transformer-depth)) + (:documentation + "HAX handler that strips unnecessary tags from the HTML of a com.apple.mail-note + and resolves references to attachments to IMG tags.")) + +;; Define the “boring” handlers which just call the next method (i. e. the next +;; handler) unless discard-until is not nil in which case the event is dropped. +(macrolet ((def-filter-handler (name (&rest args)) + `(defmethod ,name ((h apple-note-transformer) ,@args) + (when (not (transformer-discard-until h)) + (call-next-method))))) + (def-filter-handler hax:start-document (name p-id s-id)) + (def-filter-handler hax:end-document ()) + (def-filter-handler hax:characters (data)) + (def-filter-handler hax:unescaped (data)) + (def-filter-handler hax:comment (data))) + +(defun parse-content-id (attrlist) + (when-let (data (find-if (lambda (x) + (string= (hax:attribute-name x) "DATA")) + attrlist)) + (multiple-value-bind (starts-with-cid-p suffix) + (starts-with-subseq "cid:" (hax:attribute-value data) + :return-suffix t :test #'char=) + (if starts-with-cid-p suffix data)))) + +(defmethod hax:start-element ((handler apple-note-transformer) name attrs) + (with-accessors ((discard-until transformer-discard-until) + (next-handler proxy-next-handler) + (cid-lookup transformer-cid-lookup) + (depth transformer-depth)) + handler + + (cond + ;; If we are discarding, any started element is dropped, + ;; since the end-condition only is reached via END-ELEMENT. + (discard-until nil) + ;; If we are not discarding any outer elements, we can set + ;; up a new discard condition if we encounter an appropriate + ;; element. + ((member name +discard-tags-with-children+ :test #'string=) + (setf discard-until (cons name depth))) + ;; Only drop this event, must be mirrored in END-ELEMENT to + ;; avoid invalidly nested HTML. + ((member name +discard-tags-only+ :test #'string=) nil) + ;; If we encounter an object tag, we drop it and its contents, + ;; but only after inspecting its attributes and emitting new + ;; events representing an img tag which includes the respective + ;; attachment via its filename. + ((string= name "OBJECT") + (progn + (setf discard-until (cons "OBJECT" depth)) + ;; TODO(sterni): check type and only resolve images, raise error + ;; otherwise. We should only encounter images anyways, since + ;; other types are only supported for iCloud which doesn't seem + ;; to use IMAP for sync these days. + (when-let* ((cid (parse-content-id attrs)) + (file (apply cid-lookup (list cid))) + (src (hax:make-attribute "SRC" file))) + (hax:start-element next-handler "IMG" (list src)) + (hax:end-element next-handler "IMG")))) + ;; In all other cases, we use HAX-PROXY-HANDLER to pass the event on. + (t (call-next-method))) + (setf depth (1+ depth)))) + +(defmethod hax:end-element ((handler apple-note-transformer) name) + (with-accessors ((discard-until transformer-discard-until) + (depth transformer-depth)) + handler + + (setf depth (1- depth)) + (cond + ;; If we are discarding and encounter the same tag again at the same + ;; depth, we can stop, but still have to discard the current tag. + ((and discard-until + (string= (car discard-until) name) + (= (cdr discard-until) depth)) + (setf discard-until nil)) + ;; In all other cases, we drop properly. + (discard-until nil) + ;; Mirrored tag stripping as in START-ELEMENT + ((member name +discard-tags-only+ :test #'string=) nil) + ;; In all other cases, we use HAX-PROXY-HANDLER to pass the event on. + (t (call-next-method))))) diff --git a/users/sterni/nix/char/all-chars.bin b/users/sterni/nix/char/all-chars.bin new file mode 100644 index 000000000000..017b909e8e8e --- /dev/null +++ b/users/sterni/nix/char/all-chars.bin @@ -0,0 +1,2 @@ + + !"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ \ No newline at end of file diff --git a/users/sterni/nix/char/default.nix b/users/sterni/nix/char/default.nix new file mode 100644 index 000000000000..aacfc9dcbe4d --- /dev/null +++ b/users/sterni/nix/char/default.nix @@ -0,0 +1,95 @@ +{ depot, lib, pkgs, ... }: + +let + + inherit (depot.users.sterni.nix.flow) + cond + ; + + inherit (depot.nix) + yants + ; + + inherit (depot.users.sterni.nix) + string + ; + + # A char is the atomic element of a nix string + # which is essentially an array of arbitrary bytes + # as long as they are not a NUL byte. + # + # A char is neither a byte nor a unicode codepoint! + char = yants.restrict "char" (s: builtins.stringLength s == 1) yants.string; + + # integer representation of char + charval = yants.restrict "charval" (i: i >= 1 && i < 256) yants.int; + + allChars = builtins.readFile ./all-chars.bin; + + # Originally I searched a list for this, but came to the + # conclusion that this can never be fast enough in Nix. + # We therefore use a solution similar to infinisil's. + ordMap = builtins.listToAttrs + (lib.imap1 (i: v: { name = v; value = i; }) + (string.toChars allChars)); + + # Note on performance: + # chr and ord have been benchmarked using the following cases: + # + # builtins.map ord (lib.stringToCharacters allChars) + # builtins.map chr (builtins.genList (int.add 1) 255 + # + # The findings are as follows: + # 1. Searching through either strings using recursion is + # unbearably slow in Nix, leading to evaluation times + # of up to 3s for the following very small test case. + # This is why we use the trusty attribute set for ord. + # 2. String indexing is much faster than list indexing which + # is why we use the former for chr. + ord = c: ordMap."${c}"; + + chr = i: string.charAt (i - 1) allChars; + + asciiAlpha = c: + let + v = ord c; + in (v >= 65 && v <= 90) + || (v >= 97 && v <= 122); + + asciiNum = c: + let + v = ord c; + in v >= 48 && v <= 57; + + asciiAlphaNum = c: asciiAlpha c || asciiNum c; + +in { + inherit + allChars + char + charval + ord + chr + asciiAlpha + asciiNum + asciiAlphaNum + ; + + # originally I generated a nix file containing a list of + # characters, but infinisil uses a better way which I adapt + # which is using builtins.readFile instead of import. + __generateAllChars = pkgs.runCommandCC "generate-all-chars" { + source = '' + #include <stdio.h> + + int main(void) { + for(int i = 1; i <= 0xff; i++) { + putchar(i); + } + } + ''; + passAsFile = [ "source" ]; + } '' + $CC -o "$out" -x c "$sourcePath" + ''; +} diff --git a/users/sterni/nix/char/tests/default.nix b/users/sterni/nix/char/tests/default.nix new file mode 100644 index 000000000000..49b439adbb84 --- /dev/null +++ b/users/sterni/nix/char/tests/default.nix @@ -0,0 +1,31 @@ +{ depot, ... }: + +let + inherit (depot.nix.runTestsuite) + it + assertEq + runTestsuite + ; + + inherit (depot.users.sterni.nix) + char + string + int + fun + ; + + charList = string.toChars char.allChars; + + testAllCharConversion = it "tests conversion of all chars" [ + (assertEq "char.chr converts to char.allChars" + (builtins.genList (fun.rl char.chr (int.add 1)) 255) + charList) + (assertEq "char.ord converts from char.allChars" + (builtins.genList (int.add 1) 255) + (builtins.map char.ord charList)) + ]; + +in + runTestsuite "char" [ + testAllCharConversion + ] diff --git a/users/sterni/nix/flow/default.nix b/users/sterni/nix/flow/default.nix new file mode 100644 index 000000000000..b5783bd86deb --- /dev/null +++ b/users/sterni/nix/flow/default.nix @@ -0,0 +1,82 @@ +{ depot, ... }: + +let + + inherit (depot.nix) + yants + ; + + inherit (depot.users.sterni.nix) + fun + ; + + # we must avoid evaluating any of the sublists + # as they may contain conditions that throw + condition = yants.restrict "condition" + (ls: builtins.length ls == 2) + (yants.list yants.any); + + /* Like the common lisp macro: takes a list + of two elemented lists whose first element + is a boolean. The second element of the + first list that has true as its first + element is returned. + + Type: [ [ bool a ] ] -> a + + Example: + + cond [ + [ (builtins.isString true) 12 ] + [ (3 == 2) 13 ] + [ true 42 ] + ] + + => 42 + */ + cond = conds: switch true conds; + + /* Generic pattern match-ish construct for nix. + Takes a bunch of lists which are of length + two and checks the first element for either + a predicate or a value. The second value of + the first list which either has a value equal + to or a function that evaluates to true for + the given value. + + Type: a -> [ [ (function | a) b ] ] -> b + + Example: + + switch "foo" [ + [ "smol" "SMOL!!!" ] + [ (x: builtins.stringLength x <= 3) "smol-ish" ] + [ (fun.const true) "not smol" ] + ] + + => "smol-ish" + */ + switch = x: conds: + if builtins.length conds == 0 + then builtins.throw "exhausted all conditions" + else + let + c = condition (builtins.head conds); + s = builtins.head c; + b = + if builtins.isFunction s + then s x + else x == s; + in + if b + then builtins.elemAt c 1 + else switch x (builtins.tail conds); + + + +in { + inherit + cond + switch + ; +} diff --git a/users/sterni/nix/flow/tests/default.nix b/users/sterni/nix/flow/tests/default.nix new file mode 100644 index 000000000000..54cea01858e7 --- /dev/null +++ b/users/sterni/nix/flow/tests/default.nix @@ -0,0 +1,39 @@ +{ depot, ... }: + +let + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + ; + + inherit (depot.users.sterni.nix.flow) + cond + match + ; + + dontEval = builtins.throw "this should not get evaluated"; + + testCond = it "tests cond" [ + (assertThrows "malformed cond list" + (cond [ [ true 1 2 ] [ false 1 ] ])) + (assertEq "last is true" "last" + (cond [ + [ false dontEval] + [ false dontEval ] + [ true "last" ] + ])) + (assertEq "first is true" 1 + (cond [ + [ true 1 ] + [ true dontEval ] + [ true dontEval ] + ])) + ]; + +in + runTestsuite "nix.flow" [ + testCond + ] diff --git a/users/sterni/nix/fun/default.nix b/users/sterni/nix/fun/default.nix new file mode 100644 index 000000000000..6b3541ed4c65 --- /dev/null +++ b/users/sterni/nix/fun/default.nix @@ -0,0 +1,59 @@ +{ depot, lib, ... }: + +let + + inherit (lib) + id + ; + + # Simple function composition, + # application is right to left. + rl = f1: f2: + (x: f1 (f2 x)); + + # Compose a list of functions, + # application is right to left. + rls = fs: + builtins.foldl' (fOut: f: lr f fOut) id fs; + + # Simple function composition, + # application is left to right. + lr = f1: f2: + (x: f2 (f1 x)); + + # Compose a list of functions, + # application is left to right + lrs = x: fs: + builtins.foldl' (v: f: f v) x fs; + + # Warning: cursed function + # + # Check if a function has an attribute + # set pattern with an ellipsis as its argument. + # + # s/o to puck for discovering that you could use + # builtins.toXML to introspect functions more than + # you should be able to in Nix. + hasEllipsis = f: + builtins.isFunction f && + builtins.match ".*<attrspat ellipsis=\"1\">.*" + (builtins.toXML f) != null; + +in + +{ + inherit (lib) + fix + flip + const + ; + + inherit + id + rl + rls + lr + lrs + hasEllipsis + ; +} diff --git a/users/sterni/nix/fun/tests/default.nix b/users/sterni/nix/fun/tests/default.nix new file mode 100644 index 000000000000..6492554306e1 --- /dev/null +++ b/users/sterni/nix/fun/tests/default.nix @@ -0,0 +1,29 @@ +{ depot, ... }: + +let + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + inherit (depot.users.sterni.nix) + fun + ; + + hasEllipsisTests = it "checks fun.hasEllipsis" [ + (assertEq "Malicious string" false + (fun.hasEllipsis (builtins.toXML ({ foo, ... }: 12)))) + (assertEq "No function" false + (fun.hasEllipsis 23)) + (assertEq "No attribute set pattern" false + (fun.hasEllipsis (a: a + 2))) + (assertEq "No ellipsis" false + (fun.hasEllipsis ({ foo, bar }: foo + bar))) + (assertEq "Ellipsis" true + (fun.hasEllipsis ({ depot, pkgs, ... }: 42))) + ]; +in + runTestsuite "nix.fun" [ + hasEllipsisTests + ] diff --git a/users/sterni/nix/html/README.md b/users/sterni/nix/html/README.md new file mode 100644 index 000000000000..0349e466a166 --- /dev/null +++ b/users/sterni/nix/html/README.md @@ -0,0 +1,148 @@ +# html.nix — _the_ most cursed Nix HTML DSL + +A quick example to show you what it looks like: + +```nix +# Note: this example is for standalone usage out of depot +{ pkgs ? import <nixpkgs> {} }: + +let + # zero dependency, one file implementation + htmlNix = import ./path/to/html.nix { }; + + # make the magic work + inherit (htmlNix) __findFile esc withDoctype; +in + +pkgs.writeText "example.html" (withDoctype (<html> {} [ + (<head> {} [ + (<meta> { charset = "utf-8"; } null) + (<title> {} (esc "hello world")) + ]) + (<body> {} [ + (<h1> {} (esc "hello world")) + (<p> { class = "intro"; } (esc '' + welcome to the land of sillyness! + '')) + (<ul> {} [ + (<li> {} [ + (esc "check out ") + (<a> { href = "https://code.tvl.fyi"; } "depot") + ]) + (<li> {} [ + (esc "find ") + (<a> { href = "https://cl.tvl.fyi/q/hashtag:cursed"; } "cursed things") + ]) + ]) + ]) +])) +``` + +Convince yourself it works: + +```console +$ $BROWSER $(nix-build example.nix) +``` + +Alternatively, in depot: + +```console +$ $BROWSER $(nix-build -A users.sterni.nix.html.tests) +``` + +## Creating tags + +An empty tag is passed `null` as its content argument: + +```nix +<link> { + rel = "stylesheet"; + href = "/main.css"; + type = "text/css"; +} null + +# => "<link href=\"/main.css\" rel=\"stylesheet\" type=\"text/css\"/>" +``` + +Content is expected to be HTML: + +```nix +<div> { class = "foo"; } "<strong>hi</strong>" + +# => "<div class=\"foo\"><strong>hi</strong></div>" +``` + +If it's not, be sure to escape it: + +```nix +<p> {} (esc "A => B") + +# => "<p>A => B</p>" +``` + +Nesting tags works of course: + +```nix +<div> {} (<strong> {} (<em> {} "hi")) + +# => "<div><strong><em>hi</em></strong></div>" +``` + +If the content of a tag is a list, it's concatenated: + +```nix +<h1> {} [ + (esc "The ") + (<strong> {} "Nix") + (esc " ") + (<em> {} "Expression") + (esc " Language") +] + +# => "<h1>The <strong>Nix</strong> <em>Expression</em> Language</h1>" +``` + +More detailed documentation can be found in `nixdoc`-compatible +comments in the source file (`default.nix` in this directory). + +## How does this work? + +*Theoretically* expressions like `<nixpkgs>` are just ordinary paths — +their actual value is determined from `NIX_PATH`. `html.nix` works +because of how this is actually implemented: At [parse time][spath-parsing] +Nix transparently translates an expression like `<foo>` into +`__findFile __nixPath "foo"`: + +``` +nix-repl> <nixpkgs> +/nix/var/nix/profiles/per-user/root/channels/vuizvui/nixpkgs + +nix-repl> __findFile __nixPath "nixpkgs" +/nix/var/nix/profiles/per-user/root/channels/vuizvui/nixpkgs +``` + +This translation doesn't take any scoping issues into account -- +so we can just shadow `__findFile` and make it return anything, +even a function: + +``` +nix-repl> __findFile = nixPath: str: + /**/ if str == "double" then x: x * 2 + else if str == "triple" then x: x * 3 + else throw "what?" + +nix-repl> <double> 2 +4 + +nix-repl> <triple> 3 +9 + +nix-repl> <quadruple> 4 +error: what? +``` + +Exactly this is what we are doing in `html.nix`: +Using `let inherit (htmlNix) __findFile; in` we shadow the builtin `__findFile` +with a function which returns a function rendering a particular HTML tag. + +[spath-parsing]: https://github.com/NixOS/nix/blob/293220bed5a75efc963e33c183787e87e55e28d9/src/libexpr/parser.y#L410-L416 diff --git a/users/sterni/nix/html/default.nix b/users/sterni/nix/html/default.nix new file mode 100644 index 000000000000..2498d832aadf --- /dev/null +++ b/users/sterni/nix/html/default.nix @@ -0,0 +1,119 @@ +# Copyright © 2021 sterni +# SPDX-License-Identifier: MIT +# +# This file provides a cursed HTML DSL for nix which works by overloading +# the NIX_PATH lookup operation via angle bracket operations, e. g. `<nixpkgs>`. + +{ ... }: + +let + /* Escape everything we have to escape in an HTML document if either + in a normal context or an attribute string (`<>&"'`). + + A shorthand for this function called `esc` is also provided. + + Type: string -> string + + Example: + + escapeMinimal "<hello>" + => "<hello>" + */ + escapeMinimal = builtins.replaceStrings + [ "<" ">" "&" "\"" "'" ] + [ "<" ">" "&" """ "'" ]; + + /* Return a string with a correctly rendered tag of the given name, + with the given attributes which are automatically escaped. + + If the content argument is `null`, the tag will have no children nor a + closing element. If the content argument is a string it is used as the + content as is (unescaped). If the content argument is a list, its + elements are concatenated. + + `renderTag` is only an internal function which is reexposed as `__findFile` + to allow for much neater syntax than calling `renderTag` everywhere: + + ```nix + { depot, ... }: + let + inherit (depot.users.sterni.nix.html) __findFile esc; + in + + <html> {} [ + (<head> {} (<title> {} (esc "hello world"))) + (<body> {} [ + (<h1> {} (esc "hello world")) + (<p> {} (esc "foo bar")) + ]) + ] + + ``` + + As you can see, the need to call a function disappears, instead the + `NIX_PATH` lookup operation via `<foo>` is overloaded, so it becomes + `renderTag "foo"` automatically. + + Since the content argument may contain the result of other `renderTag` + calls, we can't escape it automatically. Instead this must be done manually + using `esc`. + + Type: string -> attrs<string> -> (list<string> | string | null) -> string + + Example: + + <link> { + rel = "stylesheet"; + href = "/css/main.css"; + type = "text/css"; + } null + + renderTag "link" { + rel = "stylesheet"; + href = "/css/main.css"; + type = "text/css"; + } null + + => "<link href=\"/css/main.css\" rel=\"stylesheet\" type=\"text/css\"/>" + + <p> {} [ + "foo " + (<strong> {} "bar") + ] + + renderTag "p" {} "foo <strong>bar</strong>" + => "<p>foo <strong>bar</strong></p>" + */ + renderTag = tag: attrs: content: + let + attrs' = builtins.concatStringsSep "" ( + builtins.map (n: + " ${escapeMinimal n}=\"${escapeMinimal (toString attrs.${n})}\"" + ) (builtins.attrNames attrs) + ); + content' = + if builtins.isList content + then builtins.concatStringsSep "" content + else content; + in + if content == null + then "<${tag}${attrs'}/>" + else "<${tag}${attrs'}>${content'}</${tag}>"; + + /* Prepend "<!DOCTYPE html>" to a string. + + Type: string -> string + + Example: + + withDoctype (<body> {} (esc "hello")) + => "<!DOCTYPE html><body>hello</body>" + */ + withDoctype = doc: "<!DOCTYPE html>" + doc; + +in { + inherit escapeMinimal renderTag withDoctype; + + __findFile = _: renderTag; + esc = escapeMinimal; +} diff --git a/users/sterni/nix/html/tests/default.nix b/users/sterni/nix/html/tests/default.nix new file mode 100644 index 000000000000..8688b6937130 --- /dev/null +++ b/users/sterni/nix/html/tests/default.nix @@ -0,0 +1,84 @@ +{ depot, pkgs, ... }: + +let + inherit (depot.users.sterni.nix.html) + __findFile + esc + withDoctype + ; + + exampleDocument = withDoctype (<html> { lang = "en"; } [ + (<head> {} [ + (<meta> { charset = "utf-8"; } null) + (<title> {} "html.nix example document") + (<link> { + rel = "license"; + href = "https://code.tvl.fyi/about/LICENSE"; + type = "text/html"; + } null) + (<style> {} (esc '' + hgroup h2 { + font-weight: normal; + } + + dd { + margin: 0; + } + '')) + ]) + (<body> {} [ + (<main> {} [ + (<hgroup> {} [ + (<h1> {} (esc "html.nix")) + (<h2> {} [ + (<em> {} "the") + (esc " most cursed HTML DSL ever!") + ]) + ]) + (<dl> {} [ + (<dt> {} [ + (esc "Q: Wait, it's all ") + (<a> { + href = "https://cl.tvl.fyi/q/hashtag:cursed"; + } (esc "cursed")) + (esc " nix hacks?") + ]) + (<dd> {} (esc "A: Always has been. 🔫")) + (<dt> {} (esc "Q: Why does this work?")) + (<dd> {} [ + (esc "Because nix ") + (<a> { + href = "https://github.com/NixOS/nix/blob/293220bed5a75efc963e33c183787e87e55e28d9/src/libexpr/parser.y#L410-L416"; + } (esc "translates ")) + (<a> { + href = "https://github.com/NixOS/nix/blob/293220bed5a75efc963e33c183787e87e55e28d9/src/libexpr/lexer.l#L100"; + } (esc "SPATH tokens")) + (esc " like ") + (<code> {} (esc "<nixpkgs>")) + (esc " into calls to ") + (<code> {} (esc "__findFile")) + (esc " in the ") + (<em> {} (esc "current")) + (esc " scope.") + ]) + ]) + ]) + ]) + ]); +in + +pkgs.runCommandNoCC "html.nix.html" { + passAsFile = [ "exampleDocument" ]; + inherit exampleDocument; + nativeBuildInputs = [ pkgs.html5validator ]; +} '' + set -x + test "${esc "<> && \" \'"}" = "<> && " '" + + # slow as hell unfortunately + html5validator "$exampleDocumentPath" + + mv "$exampleDocumentPath" "$out" + + set +x +'' diff --git a/users/sterni/nix/int/default.nix b/users/sterni/nix/int/default.nix new file mode 100644 index 000000000000..b3157571272f --- /dev/null +++ b/users/sterni/nix/int/default.nix @@ -0,0 +1,124 @@ +{ depot, lib, ... }: + +let + + # TODO(sterni): implement nix.float and figure out which of these + # functions can be split out into a common nix.num + # library. + + inherit (depot.users.sterni.nix) + string + ; + + inherit (builtins) + bitOr + bitAnd + bitXor + mul + div + add + sub + ; + + abs = i: if i < 0 then -i else i; + + exp = base: pow: + if pow > 0 + then base * (exp base (pow - 1)) + else if pow < 0 + then 1.0 / exp base (abs pow) + else 1; + + bitShiftR = bit: count: + if count == 0 + then bit + else div (bitShiftR bit (count - 1)) 2; + + bitShiftL = bit: count: + if count == 0 + then bit + else 2 * (bitShiftL bit (count - 1)); + + hexdigits = "0123456789ABCDEF"; + + toHex = int: + let + go = i: + if i == 0 + then "" + else go (bitShiftR i 4) + + string.charAt (bitAnd i 15) hexdigits; + sign = lib.optionalString (int < 0) "-"; + in + if int == 0 + then "0" + else "${sign}${go (abs int)}"; + + fromHexMap = builtins.listToAttrs + (lib.imap0 (i: c: { name = c; value = i; }) + (lib.stringToCharacters hexdigits)); + + fromHex = literal: + let + negative = string.charAt 0 literal == "-"; + start = if negative then 1 else 0; + len = builtins.stringLength literal; + # reversed list of all digits + digits = builtins.genList + (i: string.charAt (len - 1 - i) literal) + (len - start); + parsed = builtins.foldl' + (v: d: { + val = v.val + (fromHexMap."${d}" * v.mul); + mul = v.mul * 16; + }) + { val = 0; mul = 1; } digits; + in + if negative + then -parsed.val + else parsed.val; + + # A nix integer is a 64bit signed integer + maxBound = 9223372036854775807; + + # fun fact: -9223372036854775808 is the lower bound + # for a nix integer (as you would expect), but you can't + # use it as an integer literal or you'll be greeted with: + # error: invalid integer '9223372036854775808' + # This is because all int literals when parsing are + # positive, negative "literals" are positive literals + # which are preceded by the arithmetric negation operator. + minBound = -9223372036854775807 - 1; + + odd = x: bitAnd x 1 == 1; + even = x: bitAnd x 1 == 0; + + # div and mod behave like quot and rem in Haskell, + # i. e. they truncate towards 0 + mod = a: b: let res = a / b; in a - (res * b); + + inRange = a: b: x: x >= a && x <= b; + +in { + inherit + maxBound + minBound + abs + exp + odd + even + add + sub + mul + div + mod + bitShiftR + bitShiftL + bitOr + bitAnd + bitXor + toHex + fromHex + inRange + ; +} diff --git a/users/sterni/nix/int/tests/default.nix b/users/sterni/nix/int/tests/default.nix new file mode 100644 index 000000000000..fac45dd251e1 --- /dev/null +++ b/users/sterni/nix/int/tests/default.nix @@ -0,0 +1,203 @@ +{ depot, lib, ... }: + +let + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + ; + + inherit (depot.users.sterni.nix) + int + string + fun + ; + + testBounds = it "checks minBound and maxBound" [ + # this is gonna blow up in my face because + # integer overflow is undefined behavior in + # C++, so most likely anything could happen? + (assertEq "maxBound is the maxBound" true + (int.maxBound + 1 < int.maxBound)) + (assertEq "minBound is the minBound" true + (int.minBound - 1 > int.minBound)) + (assertEq "maxBound overflows to minBound" + (int.maxBound + 1) + int.minBound) + (assertEq "minBound overflows to maxBound" + (int.minBound - 1) + int.maxBound) + ]; + + expectedBytes = [ + "00" "01" "02" "03" "04" "05" "06" "07" "08" "09" "0A" "0B" "0C" "0D" "0E" "0F" + "10" "11" "12" "13" "14" "15" "16" "17" "18" "19" "1A" "1B" "1C" "1D" "1E" "1F" + "20" "21" "22" "23" "24" "25" "26" "27" "28" "29" "2A" "2B" "2C" "2D" "2E" "2F" + "30" "31" "32" "33" "34" "35" "36" "37" "38" "39" "3A" "3B" "3C" "3D" "3E" "3F" + "40" "41" "42" "43" "44" "45" "46" "47" "48" "49" "4A" "4B" "4C" "4D" "4E" "4F" + "50" "51" "52" "53" "54" "55" "56" "57" "58" "59" "5A" "5B" "5C" "5D" "5E" "5F" + "60" "61" "62" "63" "64" "65" "66" "67" "68" "69" "6A" "6B" "6C" "6D" "6E" "6F" + "70" "71" "72" "73" "74" "75" "76" "77" "78" "79" "7A" "7B" "7C" "7D" "7E" "7F" + "80" "81" "82" "83" "84" "85" "86" "87" "88" "89" "8A" "8B" "8C" "8D" "8E" "8F" + "90" "91" "92" "93" "94" "95" "96" "97" "98" "99" "9A" "9B" "9C" "9D" "9E" "9F" + "A0" "A1" "A2" "A3" "A4" "A5" "A6" "A7" "A8" "A9" "AA" "AB" "AC" "AD" "AE" "AF" + "B0" "B1" "B2" "B3" "B4" "B5" "B6" "B7" "B8" "B9" "BA" "BB" "BC" "BD" "BE" "BF" + "C0" "C1" "C2" "C3" "C4" "C5" "C6" "C7" "C8" "C9" "CA" "CB" "CC" "CD" "CE" "CF" + "D0" "D1" "D2" "D3" "D4" "D5" "D6" "D7" "D8" "D9" "DA" "DB" "DC" "DD" "DE" "DF" + "E0" "E1" "E2" "E3" "E4" "E5" "E6" "E7" "E8" "E9" "EA" "EB" "EC" "ED" "EE" "EF" + "F0" "F1" "F2" "F3" "F4" "F5" "F6" "F7" "F8" "F9" "FA" "FB" "FC" "FD" "FE" "FF" + ]; + + hexByte = i: string.fit { width = 2; char = "0"; } (int.toHex i); + + hexInts = [ + { left = 0; right = "0"; } + { left = 1; right = "1"; } + { left = 11; right = "B"; } + { left = 123; right = "7B"; } + { left = 9000; right = "2328"; } + { left = 2323; right = "913"; } + { left = 4096; right = "1000"; } + { left = int.maxBound; right = "7FFFFFFFFFFFFFFF"; } + { left = int.minBound; right = "-8000000000000000"; } + ]; + + testHex = it "checks conversion to hex" (lib.flatten [ + (lib.imap0 (i: hex: [ + (assertEq "hexByte ${toString i} == ${hex}" (hexByte i) hex) + (assertEq "${toString i} == fromHex ${hex}" i (int.fromHex hex)) + ]) expectedBytes) + (builtins.map ({ left, right }: [ + (assertEq "toHex ${toString left} == ${right}" (int.toHex left) right) + (assertEq "${toString left} == fromHex ${right}" left (int.fromHex right)) + ]) hexInts) + ]); + + testBasic = it "checks basic int operations" [ + (assertEq "122 is even" (int.even 122 && !(int.odd 122)) true) + (assertEq "123 is odd" (int.odd 123 && !(int.even 123)) true) + (assertEq "abs -4959" (int.abs (-4959)) 4959) + ]; + + expNumbers = [ + { left = -3; right = 0.125; } + { left = -2; right = 0.25; } + { left = -1; right = 0.5; } + { left = 0; right = 1; } + { left = 1; right = 2; } + { left = 2; right = 4; } + { left = 3; right = 8; } + { left = 4; right = 16; } + { left = 5; right = 32; } + { left = 16; right = 65536; } + ]; + + testExp = it "checks exponentiation" + (builtins.map ({ left, right }: + assertEq + "2 ^ ${toString left} == ${toString right}" + (int.exp 2 left) right) expNumbers); + + shifts = [ + { a = 2; b = 5; c = 64; op = "<<"; } + { a = -2; b = 5; c = -64; op = "<<"; } + { a = 123; b = 4; c = 1968; op = "<<"; } + { a = 1; b = 8; c = 256; op = "<<"; } + { a = 256; b = 8; c = 1; op = ">>"; } + { a = 374; b = 2; c = 93; op = ">>"; } + { a = 2; b = 2; c = 0; op = ">>"; } + { a = 99; b = 9; c = 0; op = ">>"; } + ]; + + checkShift = { a, b, c, op }@args: + let + f = string.match op { + "<<" = int.bitShiftL; + ">>" = int.bitShiftR; + }; + in assertEq "${toString a} ${op} ${toString b} == ${toString c}" (f a b) c; + + checkShiftRDivExp = n: + assertEq "${toString n} >> 5 == ${toString n} / 2 ^ 5" + (int.bitShiftR n 5) (int.div n (int.exp 2 5)); + + checkShiftLMulExp = n: + assertEq "${toString n} >> 6 == ${toString n} * 2 ^ 6" + (int.bitShiftL n 5) (int.mul n (int.exp 2 5)); + + testBit = it "checks bitwise operations" (lib.flatten [ + (builtins.map checkShift shifts) + (builtins.map checkShiftRDivExp [ + 1 + 2 + 3 + 5 + 7 + 23 + 1623 + 238 + 34 + 348 + 2834 + 834 + 348 + ]) + (builtins.map checkShiftLMulExp [ + 1 + 2 + 3 + 5 + 7 + 23 + 384 + 3 + 2 + 5991 + 85109 + 38 + ]) + ]); + + divisions = [ + { a = 2; b = 1; c = 2; mod = 0;} + { a = 2; b = 2; c = 1; mod = 0;} + { a = 20; b = 10; c = 2; mod = 0;} + { a = 12; b = 5; c = 2; mod = 2;} + { a = 23; b = 4; c = 5; mod = 3;} + ]; + + checkDiv = n: { a, b, c, mod }: [ + (assertEq "${n}: div result" (int.div a b) c) + (assertEq "${n}: mod result" (int.mod a b) mod) + (assertEq "${n}: divMod law" ((int.div a b) * b + (int.mod a b)) a) + ]; + + testDivMod = it "checks integer division and modulo" + (lib.flatten [ + (builtins.map (checkDiv "+a / +b") divisions) + (builtins.map (fun.rl (checkDiv "-a / +b") (x: x // { + a = -x.a; + c = -x.c; + mod = -x.mod; + })) divisions) + (builtins.map (fun.rl (checkDiv "+a / -b") (x: x // { + b = -x.b; + c = -x.c; + })) divisions) + (builtins.map (fun.rl (checkDiv "-a / -b") (x: x // { + a = -x.a; + b = -x.b; + mod = -x.mod; + })) divisions) + ]); + +in + runTestsuite "nix.int" [ + testBounds + testHex + testBasic + testExp + testBit + testDivMod + ] diff --git a/users/sterni/nix/string/default.nix b/users/sterni/nix/string/default.nix new file mode 100644 index 000000000000..19d2cec243c0 --- /dev/null +++ b/users/sterni/nix/string/default.nix @@ -0,0 +1,114 @@ +{ depot, lib, ... }: + +let + + inherit (depot.users.sterni.nix.char) + chr + ord + ; + + inherit (depot.users.sterni.nix) + int + flow + ; + + take = n: s: + builtins.substring 0 n s; + + drop = n: s: + builtins.substring n int.maxBound s; + + charAt = i: s: + let + r = builtins.substring i 1 s; + in if r == "" then null else r; + + charIndex = char: s: + let + len = builtins.stringLength s; + go = i: + flow.cond [ + [ (i >= len) null ] + [ (charAt i s == char) i ] + [ true (go (i + 1)) ] + ]; + in go 0; + + toChars = lib.stringToCharacters; + fromChars = lib.concatStrings; + + toBytes = str: + builtins.map ord (toChars str); + + fromBytes = is: lib.concatMapStrings chr is; + + pad = { left ? 0, right ? 0, char ? " " }: s: + let + leftS = fromChars (builtins.genList (_: char) left); + rightS = fromChars (builtins.genList (_: char) right); + in "${leftS}${s}${rightS}"; + + fit = { char ? " ", width, side ? "left" }: s: + let + diff = width - builtins.stringLength s; + in + if diff <= 0 + then s + else pad { inherit char; "${side}" = diff; } s; + + # pattern matching for strings only + match = val: matcher: matcher."${val}"; + + /* Bare-bones printf implementation. Supported format specifiers: + + * `%%` escapes `%` + * `%s` is substituted by a string + + As expected, the first argument is a format string and the values + for its format specifiers need to provided as the next arguments + in order. + + Type: string -> (printfVal : either string (a -> printfVal)) + */ + printf = formatString: + let + specifierWithArg = token: builtins.elem token [ + "%s" + ]; + isSpecifier = lib.hasPrefix "%"; + + tokens = lib.flatten (builtins.split "(%.)" formatString); + argsNeeded = builtins.length (builtins.filter specifierWithArg tokens); + + format = args: (builtins.foldl' ({ out ? "", argIndex ? 0 }: token: { + argIndex = argIndex + (if specifierWithArg token then 1 else 0); + out = + /**/ if token == "%s" then out + builtins.elemAt args argIndex + else if token == "%%" then out + "%" + else if isSpecifier token then throw "Unsupported format specifier ${token}" + else out + token; + }) {} tokens).out; + + accumulateArgs = argCount: args: + if argCount > 0 + then arg: accumulateArgs (argCount - 1) (args ++ [ arg ]) + else format args; + in + accumulateArgs argsNeeded []; + +in { + inherit + take + drop + charAt + charIndex + toBytes + fromBytes + toChars + fromChars + pad + fit + match + printf + ; +} diff --git a/users/sterni/nix/string/tests/default.nix b/users/sterni/nix/string/tests/default.nix new file mode 100644 index 000000000000..c8aec9464077 --- /dev/null +++ b/users/sterni/nix/string/tests/default.nix @@ -0,0 +1,72 @@ +{ depot, ... }: + +let + + inherit (depot.users.sterni.nix) + string + ; + + inherit (depot.nix.runTestsuite) + it + assertEq + runTestsuite + ; + + testTakeDrop = it "tests take and drop" [ + (assertEq "take" + (string.take 5 "five and more") + "five ") + (assertEq "drop" + (string.drop 2 "coin") + "in") + (assertEq "take out of bounds" + (string.take 100 "foo") + "foo") + (assertEq "drop out of bounds" + (string.drop 42 "lol") + "") + ]; + + testIndexing = it "tests string indexing" [ + (assertEq "normal charAt" + (string.charAt 3 "helo") + "o") + (assertEq "out of bounds charAt" + (string.charAt 5 "helo") + null) + ]; + + testFinding = it "tests finding in strings" [ + (assertEq "normal charIndex" + (string.charIndex "d" "abcdefghijkl") + 3) + (assertEq "charIndex no match" + (string.charIndex "w" "zZzZzzzZZZ") + null) + ]; + + dontEval = builtins.throw "this should not get evaluated"; + + testMatch = it "tests match" [ + (assertEq "basic match usage" 42 + (string.match "answer" { + "answer" = 42; + "banana" = dontEval; + "maleur" = dontEval; + })) + ]; + + f = "f"; + testPrintf = it "prints f" [ + (assertEq "basic %s usage" "print ${f}" (string.printf "print %s" f)) + (assertEq "% escaping" "100%" (string.printf "100%%")) + ]; + +in + runTestsuite "nix.string" [ + testTakeDrop + testIndexing + testFinding + testMatch + testPrintf + ] diff --git a/users/sterni/nix/url/default.nix b/users/sterni/nix/url/default.nix new file mode 100644 index 000000000000..37bd0de66ac9 --- /dev/null +++ b/users/sterni/nix/url/default.nix @@ -0,0 +1,81 @@ +{ depot, lib, ... }: + +let + + inherit (depot.users.sterni.nix) + char + int + string + flow + ; + + reserved = c: builtins.elem c [ + "!" "#" "$" "&" "'" "(" ")" + "*" "+" "," "/" ":" ";" "=" + "?" "@" "[" "]" + ]; + + unreserved = c: char.asciiAlphaNum c + || builtins.elem c [ "-" "_" "." "~" ]; + + percentEncode = c: + if unreserved c + then c + else "%" + (string.fit { + width = 2; + char = "0"; + side = "left"; + } (int.toHex (char.ord c))); + + encode = { leaveReserved ? false }: s: + let + chars = lib.stringToCharacters s; + tr = c: + if leaveReserved && reserved c + then c + else percentEncode c; + in lib.concatStrings (builtins.map tr chars); + + decode = s: + let + tokens = builtins.split "%" s; + decodeStep = + { result ? "" + , inPercent ? false + }: s: + flow.cond [ + [ + (builtins.isList s) + { + inherit result; + inPercent = true; + } + ] + [ + inPercent + { + inPercent = false; + # first two characters came after an % + # the rest is the string until the next % + result = result + + char.chr (int.fromHex (string.take 2 s)) + + (string.drop 2 s); + } + ] + [ + (!inPercent) + { + result = result + s; + } + ] + ]; + + in + (builtins.foldl' decodeStep {} tokens).result; + +in { + inherit + encode + decode + ; +} diff --git a/users/sterni/nix/url/tests/default.nix b/users/sterni/nix/url/tests/default.nix new file mode 100644 index 000000000000..7cf53cde1555 --- /dev/null +++ b/users/sterni/nix/url/tests/default.nix @@ -0,0 +1,56 @@ +{ depot, ... }: + +let + + inherit (depot.nix.runTestsuite) + it + assertEq + runTestsuite + ; + + inherit (depot.users.sterni.nix) + url + ; + + checkEncoding = args: { left, right }: + assertEq "encode ${builtins.toJSON left} == ${builtins.toJSON right}" + (url.encode args left) right; + + checkDecoding = { left, right }: + assertEq "${builtins.toJSON left} == decode ${builtins.toJSON right}" + (url.decode left) right; + + unreserved = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.-_~"; + + encodeExpected = [ + { left = "Laguna Beach"; right = "Laguna%20Beach"; } + { left = "👾 Exterminate!"; right = "%F0%9F%91%BE%20Exterminate%21"; } + { left = unreserved; right = unreserved; } + { + left = "`!@#$%^&*()+={}[]:;'\\|<>,?/ \""; + right = "%60%21%40%23%24%25%5E%26%2A%28%29%2B%3D%7B%7D%5B%5D%3A%3B%27%5C%7C%3C%3E%2C%3F%2F%20%22"; + } + ]; + + testEncode = it "checks url.encode" + (builtins.map (checkEncoding {}) encodeExpected); + + testDecode = it "checks url.decode" + (builtins.map checkDecoding encodeExpected); + + testLeaveReserved = it "checks that leaveReserved is like id for valid URLs" + (builtins.map (x: checkEncoding { leaveReserved = true; } { left = x; right = x; }) [ + "ftp://ftp.is.co.za/rfc/rfc1808.txt" + "http://www.ietf.org/rfc/rfc2396.txt" + "ldap://[2001:db8::7]/c=GB?objectClass?one" + "mailto:John.Doe@example.com" + "news:comp.infosystems.www.servers.unix" + "tel:+1-816-555-1212" + "telnet://192.0.2.16:80/" + "urn:oasis:names:specification:docbook:dtd:xml:4.1.2" + ]); +in + runTestsuite "nix.url" [ + testEncode + testLeaveReserved + ] diff --git a/users/sterni/nix/utf8/default.nix b/users/sterni/nix/utf8/default.nix new file mode 100644 index 000000000000..270da934b6a6 --- /dev/null +++ b/users/sterni/nix/utf8/default.nix @@ -0,0 +1,313 @@ +{ depot, lib, ... }: + +let + + inherit (depot.users.sterni.nix) + char + flow + fun + int + string + util + ; + + /* (Internal) function to determine the amount + bytes left in a UTF-8 byte sequence from the + first byte. + + This function will throw if the given first + byte is ill-formed, but will not detect all + cases of ill-formed-ness. + + Based on table 3-6. from The Unicode Standard, + Version 13.0, section 3.9. + + Type: integer -> integer + */ + byteCount = i: flow.cond [ + [ (int.bitAnd i 128 == 0) 1 ] + [ (int.bitAnd i 224 == 192) 2 ] + [ (int.bitAnd i 240 == 224) 3 ] + [ (int.bitAnd i 248 == 240) 4 ] + [ true (builtins.throw "Ill-formed first byte ${int.toHex i}") ] + ]; + + /* (Internal) function to check if a given byte in + an UTF-8 byte sequence is well-formed. + + Based on table 3-7. from The Unicode Standard, + Version 13.0, section 3.9. + + Type: integer -> integer -> integer -> bool + */ + wellFormedByte = + # first byte's integer value + first: + # byte position as an index starting with 0 + pos: + let + defaultRange = int.inRange 128 191; + + secondBytePredicate = flow.switch first [ + [ (int.inRange 194 223) defaultRange ] # C2..DF + [ 224 (int.inRange 160 191) ] # E0 + [ (int.inRange 225 236) defaultRange ] # E1..EC + [ 237 (int.inRange 128 159) ] # ED + [ (int.inRange 238 239) defaultRange ] # EE..EF + [ 240 (int.inRange 144 191) ] # F0 + [ (int.inRange 241 243) defaultRange ] # F1..F3 + [ 244 (int.inRange 128 143) ] # F4 + [ (fun.const true) null ] + ]; + + firstBytePredicate = byte: assert first == byte; + first < 128 || secondBytePredicate != null; + in + # Either ASCII or in one of the byte ranges of Table 3-6. + if pos == 0 then firstBytePredicate + # return predicate according to Table 3-6. + else if pos == 1 then assert secondBytePredicate != null; secondBytePredicate + # 3rd and 4th byte have only one validity rule + else defaultRange; + + /* Iteration step for decoding an UTF-8 byte sequence. + It decodes incrementally, i. e. it has to be fed + one byte at a time and then returns either a + new state or a final result. + + If the resulting attribute set contains the attribute + result, it is finished and the decoded codepoint is + contained in that attribute. In all other cases, + pass the returned set to step again along with + a new byte. The initial state to pass is the empty + set. + + Extra attributes are always passed through, so you + can pass extra state. Be sure not to use result, + pos, code, first or count. + + This function will throw with a fairly detailed + message if it encounters ill-formed bytes. + + The implementation is based on The Unicode Standard, + Version 13.0, section 3.9, especially table 3-6. + + Type: { ... } -> string -> ({ result :: integer, ... } | { ... }) + + Example: utf8.step {} "f" + => { result = 102; } + */ + step = { pos ? 0, code ? 0, ... }@args: byte: + let + value = char.ord byte; + # first byte is context for well-formed-ness + first = args.first or value; + count = args.count or (byteCount first); + newCode = + if count == 1 + then int.bitAnd 127 first # ascii character + else # multi byte UTF-8 sequence + let + # Calculate the bitmask for extracting the + # codepoint data in the current byte. + # If the codepoint is not ASCII, the bits + # used for codepoint data differ depending + # on the byte position and overall byte + # count. The first byte always ignores + # the (count + 1) most significant bits. + # For all subsequent bytes, the 2 most + # significant bits need to be ignored. + # See also table 3-6. + mask = + if pos == 0 + then int.exp 2 (8 - (count + 1)) - 1 + else 63; + # UTF-8 uses the 6 least significant bits in all + # subsequent bytes after the first one. Therefore + # We can determine the amount we need to shift + # the current value by the amount of bytes left. + offset = (count - (pos + 1)) * 6; + in + code + (int.bitShiftL (int.bitAnd mask value) offset); + illFormedMsg = + "Ill-formed byte ${int.toHex value} at position ${toString pos} in ${toString count} byte UTF-8 sequence"; + in + if !(wellFormedByte first pos value) then builtins.throw illFormedMsg + else if pos + 1 == count + then (builtins.removeAttrs args [ # allow extra state being passed through + "count" + "code" + "pos" + "first" + ]) // { result = newCode; } + else (builtins.removeAttrs args [ "result" ]) // { + inherit count first; + code = newCode; + pos = pos + 1; + }; + + /* Decode an UTF-8 string into a list of codepoints. + + Throws if the string is ill-formed UTF-8. + + Type: string -> [ integer ] + */ + # TODO(sterni): option to fallback to replacement char instead of failure + decode = s: + let + stringLength = builtins.stringLength s; + iterResult = builtins.genericClosure { + startSet = [ + { + key = "start"; + stringIndex = -1; + state = {}; + codepoint = null; + } + ]; + operator = { state, stringIndex, ... }: + let + # updated values for current iteration step + newIndex = stringIndex + 1; + newState = step state (builtins.substring newIndex 1 s); + in lib.optional (newIndex < stringLength) { + # unique keys to make genericClosure happy + key = toString newIndex; + # carryover state for the next step + stringIndex = newIndex; + state = newState; + # actual payload for later, steps with value null are filtered out + codepoint = newState.result or null; + }; + }; + in + # extract all steps that yield a code point into a list + builtins.map (v: v.codepoint) ( + builtins.filter ( + { codepoint, stringIndex, state, ... }: + + let + # error message in case we are missing bytes at the end of input + earlyEndMsg = + if state ? count && state ? pos + then "Missing ${toString (with state; count - pos)} bytes at end of input" + else "Unexpected end of input"; + in + + # filter out all iteration steps without a codepoint value + codepoint != null + # if we are at the iteration step of a non-empty input string, throw + # an error if no codepoint was returned, as it indicates an incomplete + # UTF-8 sequence. + || (stringLength > 0 && stringIndex == stringLength - 1 && throw earlyEndMsg) + + ) iterResult + ); + + /* Pretty prints a Unicode codepoint in the U+<HEX> notation. + + Type: integer -> string + */ + formatCodepoint = cp: "U+" + string.fit { + width = 4; + char = "0"; + } (int.toHex cp); + + encodeCodepoint = cp: + let + # Find the amount of bytes needed to encode the given codepoint. + # Note that this doesn't check if the Unicode codepoint is allowed, + # but rather allows all theoretically UTF-8-encodeable ones. + count = flow.switch cp [ + [ (int.inRange 0 127) 1 ] # 00000000 0xxxxxxx + [ (int.inRange 128 2047) 2 ] # 00000yyy yyxxxxxx + [ (int.inRange 2048 65535) 3 ] # zzzzyyyy yyxxxxxx + [ (int.inRange 65536 1114111) 4 ] # 000uuuuu zzzzyyyy yyxxxxxx, + # capped at U+10FFFF + + [ (fun.const true) (builtins.throw invalidCodepointMsg) ] + ]; + + invalidCodepointMsg = "${formatCodepoint cp} is not a Unicode codepoint"; + + # Extract the bit ranges x, y, z and u from the given codepoint + # according to Table 3-6. from The Unicode Standard, Version 13.0, + # section 3.9. u is split into uh and ul since they are used in + # different bytes in the end. + components = lib.mapAttrs (_: { mask, offset }: + int.bitAnd (int.bitShiftR cp offset) mask + ) { + x = { + mask = if count > 1 then 63 else 127; + offset = 0; + }; + y = { + mask = if count > 2 then 63 else 31; + offset = 6; + }; + z = { + mask = 15; + offset = 12; + }; + # u which belongs into the second byte + ul = { + mask = 3; + offset = 16; + }; + # u which belongs into the first byte + uh = { + mask = 7; + offset = 18; + }; + }; + inherit (components) x y z ul uh; + + # Finally construct the byte sequence for the given codepoint. This is + # usually done by using the component and adding a few bits as a prefix + # which depends on the length of the sequence. The longer the sequence, + # the further back each component is pushed. To simplify this, we + # always construct a 4 element list and take the last `count` elements. + # Thanks to laziness the bogus values created by this are never evaluated. + # + # Based on table 3-6. from The Unicode Standard, + # Version 13.0, section 3.9. + bytes = lib.sublist (4 - count) count [ + # 11110uuu + (uh + 240) + # 10uuzzzz or 1110zzzz + (z + (if count > 3 then 128 + int.bitShiftL ul 4 else 224)) + # 10yyyyyy or 110yyyyy + (y + (if count > 2 then 128 else 192)) + # 10xxxxxx or 0xxxxxxx + (x + (if count > 1 then 128 else 0)) + ]; + + firstByte = builtins.head bytes; + + unableToEncodeMessage = "Can't encode ${formatCodepoint cp} as UTF-8"; + + in string.fromBytes ( + builtins.genList (i: + let + byte = builtins.elemAt bytes i; + in + if wellFormedByte firstByte i byte + then byte + else builtins.throw unableToEncodeMessage + ) count + ); + + /* Encode a list of Unicode codepoints into an UTF-8 string. + + Type: [ integer ] -> string + */ + encode = lib.concatMapStrings encodeCodepoint; + +in { + inherit + encode + decode + step + formatCodepoint + ; +} diff --git a/users/sterni/nix/utf8/tests/default.nix b/users/sterni/nix/utf8/tests/default.nix new file mode 100644 index 000000000000..ddcd34208a6d --- /dev/null +++ b/users/sterni/nix/utf8/tests/default.nix @@ -0,0 +1,141 @@ +{ depot, pkgs, lib, ... }: + +let + + inherit (pkgs) + runCommandLocal + ; + + inherit (depot.nix.runTestsuite) + runTestsuite + it + assertEq + assertThrows + assertDoesNotThrow + ; + + inherit (depot.nix.writers) + rustSimple + ; + + inherit (depot.users.sterni.nix) + int + utf8 + string + char + ; + + rustDecoder = rustSimple { + name = "utf8-decode"; + } '' + use std::io::{self, Read}; + fn main() -> std::io::Result<()> { + let mut buffer = String::new(); + io::stdin().read_to_string(&mut buffer)?; + + print!("[ "); + + for c in buffer.chars() { + print!("{} ", u32::from(c)); + } + + print!("]"); + + Ok(()) + } + ''; + + rustDecode = s: + let + expr = runCommandLocal "${s}-decoded" {} '' + printf '%s' ${lib.escapeShellArg s} | ${rustDecoder} > $out + ''; + in import expr; + + hexDecode = l: + utf8.decode (string.fromBytes (builtins.map int.fromHex l)); + + hexEncode = l: utf8.encode (builtins.map int.fromHex l); + + testFailures = it "checks UTF-8 decoding failures" ([ + (assertThrows "truncated UTF-8 string throws" (hexDecode [ "F0" "9F" ])) + # examples from The Unicode Standard + (assertThrows "ill-formed: C0 AF" (hexDecode [ "C0" "AF" ])) + (assertThrows "ill-formed: E0 9F 80" (hexDecode [ "E0" "9F" "80" ])) + (assertEq "well-formed: F4 80 83 92" (hexDecode [ "F4" "80" "83" "92" ]) [ 1048786 ]) + (assertThrows "Codepoint out of range: 0xFFFFFF" (hexEncode [ "FFFFFF" ])) + (assertThrows "Codepoint out of range: -0x02" (hexEncode [ "-02" ])) + ] ++ builtins.genList (i: + let + cp = i + int.fromHex "D800"; + in + assertThrows "Can't encode UTF-16 reserved characters: ${utf8.formatCodepoint cp}" + (utf8.encode [ cp ]) + ) (int.fromHex "07FF")); + + testAscii = it "checks decoding of ascii strings" + (builtins.map (s: assertEq "ASCII decoding is equal to UTF-8 decoding for \"${s}\"" + (string.toBytes s) (utf8.decode s)) [ + "foo bar" + "hello\nworld" + "carriage\r\nreturn" + "1238398494829304 []<><>({})[]!!)" + (string.take 127 char.allChars) + ]); + + randomUnicode = [ + "" # empty string should yield empty list + "🥰👨👨👧👦🐈⬛👩🏽🦰" + # https://kermitproject.org/utf8.html + "ᚠᛇᚻ᛫ᛒᛦᚦ᛫ᚠᚱᚩᚠᚢᚱ᛫ᚠᛁᚱᚪ᛫ᚷᛖᚻᚹᛦᛚᚳᚢᛗ" + "An preost wes on leoden, Laȝamon was ihoten" + "Sîne klâwen durh die wolken sint geslagen," + "Τὴ γλῶσσα μοῦ ἔδωσαν ἑλληνικὴ" + "На берегу пустынных волн" + "ვეპხის ტყაოსანი შოთა რუსთაველი" + "யாமறிந்த மொழிகளிலே தமிழ்மொழி போல் இனிதாவது எங்கும் காணோம், " + "ಬಾ ಇಲ್ಲಿ ಸಂಭವಿಸು " + ]; + + # https://kermitproject.org/utf8.html + glassSentences = [ + "Euro Symbol: €." + "Greek: Μπορώ να φάω σπασμένα γυαλιά χωρίς να πάθω τίποτα." + "Íslenska / Icelandic: Ég get etið gler án þess að meiða mig." + "Polish: Mogę jeść szkło, i mi nie szkodzi." + "Romanian: Pot să mănânc sticlă și ea nu mă rănește." + "Ukrainian: Я можу їсти шкло, й воно мені не пошкодить." + "Armenian: Կրնամ ապակի ուտել և ինծի անհանգիստ չըներ։" + "Georgian: მინას ვჭამ და არა მტკივა." + "Hindi: मैं काँच खा सकता हूँ, मुझे उस से कोई पीडा नहीं होती." + "Hebrew(2): אני יכול לאכול זכוכית וזה לא מזיק לי." + "Yiddish(2): איך קען עסן גלאָז און עס טוט מיר נישט װײ." + "Arabic(2): أنا قادر على أكل الزجاج و هذا لا يؤلمني." + "Japanese: 私はガラスを食べられます。それは私を傷つけません。" + "Thai: ฉันกินกระจกได้ แต่มันไม่ทำให้ฉันเจ็บ " + ]; + + testDecoding = it "checks decoding of UTF-8 strings against Rust's String" + (builtins.map + (s: assertEq "Decoding of “${s}” is correct" (utf8.decode s) (rustDecode s)) + (lib.flatten [ + glassSentences + randomUnicode + ])); + + testDecodingEncoding = it "checks that decoding and then encoding forms an identity" + (builtins.map + (s: assertEq "Decoding and then encoding “${s}” yields itself" + (utf8.encode (utf8.decode s)) s) + (lib.flatten [ + glassSentences + randomUnicode + ])); + +in + runTestsuite "nix.utf8" [ + testFailures + testAscii + testDecoding + testDecodingEncoding + ] diff --git a/users/sterni/nixpkgs-crate-holes/default.nix b/users/sterni/nixpkgs-crate-holes/default.nix new file mode 100644 index 000000000000..a022568dc941 --- /dev/null +++ b/users/sterni/nixpkgs-crate-holes/default.nix @@ -0,0 +1,284 @@ +{ depot, pkgs, lib, ... }: + +let + # dependency imports + + inherit (depot.nix) getBins; + inherit (depot.third_party) rustsec-advisory-db; + + bins = getBins pkgs.jq [ + "jq" + ] // getBins pkgs.coreutils [ + "cat" + "printf" + "tee" + "test" + "wc" + ] // getBins pkgs.gnugrep [ + "grep" + ] // getBins pkgs.cargo-audit [ + "cargo-audit" + ] // getBins pkgs.ansi2html [ + "ansi2html" + ] // { + eprintf = depot.tools.eprintf; + }; + + # list of maintainers we may @mention on GitHub + maintainerWhitelist = builtins.attrValues { + inherit (lib.maintainers) + sternenseemann + qyliss + jk + symphorien + erictapen + expipiplus1 + ; + }; + + # buildRustPackage handling + + /* Predicate by which we identify rust packages we are interested in, + i. e. built using `buildRustPackage`. + + Type :: drv -> bool + */ + isRustPackage = v: v ? cargoDeps; + + /* Takes a buildRustPackage derivation and returns a derivation which + builds extracts the `Cargo.lock` of its `cargoDeps` derivation or + `null` if it has none. + + Type: drv -> option<drv> + */ + # TODO(sterni): support cargoVendorDir? + extractCargoLock = drv: + if !(drv ? cargoDeps.outPath) + then null + else pkgs.runCommandNoCC "${drv.name}-Cargo.lock" {} '' + if test -d "${drv.cargoDeps}"; then + cp "${drv.cargoDeps}/Cargo.lock" "$out" + fi + + if test -f "${drv.cargoDeps}"; then + tar -xO \ + --no-wildcards-match-slash --wildcards \ + -f "${drv.cargoDeps}" \ + '*/Cargo.lock' \ + > "$out" + fi + ''; + + # nixpkgs traversal + + # Condition for us to recurse: Either at top-level or recurseForDerivation. + recurseInto = path: x: path == [] || + (lib.isAttrs x && (x.recurseForDerivations or false)); + + # Returns the value or false if an eval error occurs. + tryEvalOrFalse = v: (builtins.tryEval v).value; + + /* Traverses nixpkgs as instructed by `recurseInto` and collects + the attribute and lockfile derivation of every rust package it + encounters into a list. + + Type :: attrs + -> list { + attr :: list<str>; + lock :: option<drv>; + maintainers :: list<maintainer>; + } + */ + allLockFiles = + let + go = path: x: + let + isDrv = tryEvalOrFalse (lib.isDerivation x); + doRec = tryEvalOrFalse (recurseInto path x); + isRust = tryEvalOrFalse (isRustPackage x); + in + if doRec then lib.concatLists ( + lib.mapAttrsToList (n: go (path ++ [ n ])) x + ) else if isDrv && isRust then [ + { + attr = path; + lock = extractCargoLock x; + maintainers = x.meta.maintainers or []; + } + ] else []; + in go []; + + # Report generation and formatting + + reportFor = { attr, lock, maintainers ? [] }: let + # naïve attribute path to Nix syntax conversion + strAttr = lib.concatStringsSep "." attr; + strMaintainers = lib.concatMapStringsSep " " (m: "@${m.github}") ( + builtins.filter (x: builtins.elem x maintainerWhitelist) maintainers + ); + in + if lock == null + then pkgs.emptyFile + else depot.nix.runExecline "${strAttr}-vulnerability-report" {} [ + "pipeline" [ + bins.cargo-audit + "audit" "--json" + "-n" "--db" rustsec-advisory-db + "-f" lock + ] + "importas" "out" "out" + "redirfd" "-w" "1" "$out" + bins.jq "-rj" "-f" ./format-audit-result.jq + "--arg" "attr" strAttr + "--arg" "maintainers" strMaintainers + ]; + + # GHMF in issues splits paragraphs on newlines + description = lib.concatMapStringsSep "\n\n" ( + builtins.replaceStrings [ "\n" ] [ " " ] + ) [ + '' + The vulnerability report below was generated by + [nixpkgs-crate-holes](https://code.tvl.fyi/tree/users/sterni/nixpkgs-crate-holes) + which extracts the `Cargo.lock` file of each package in nixpkgs with a + `cargoDeps` attribute and passes it to + [cargo-audit](https://github.com/RustSec/rustsec/tree/main/cargo-audit) + using RustSec's + [advisory-db at ${builtins.substring 0 7 rustsec-advisory-db.rev}](https://github.com/RustSec/advisory-db/tree/${rustsec-advisory-db.rev}/). + '' + '' + Feel free to report any problems or suggest improvements (I have an email + address on my profile and hang out on Matrix/libera.chat as sterni)! + Tick off any reports that have been fixed in the meantime. + '' + '' + Note: A vulnerability in a dependency does not necessarily mean the dependent + package is vulnerable, e. g. when a vulnerable function isn't used. + '' + ]; + + runInstructions = '' + <details> + <summary> + Generating Cargo.lock vulnerability reports + + </summary> + + If you have a checkout of [depot](https://code.tvl.fyi/about/), you can generate this report using: + + ``` + nix-build -A users.sterni.nixpkgs-crate-holes.full \ + --argstr nixpkgsPath /path/to/nixpkgs + ``` + + If you want a more detailed report for a single attribute of nixpkgs, use: + + ``` + nix-build -A users.sterni.nixpkgs-crate-holes.single \ + --argstr nixpkgsPath /path/to/nixpkgs --arg attr '[ "ripgrep" ]' + ``` + + </details> + ''; + + defaultNixpkgsArgs = { allowBroken = false; }; + + reportForNixpkgs = + { nixpkgsPath + , nixpkgsArgs ? defaultNixpkgsArgs + }@args: + + let + reports = builtins.map reportFor ( + allLockFiles (import nixpkgsPath nixpkgsArgs) + ); + in + + depot.nix.runExecline "nixpkgs-rust-pkgs-vulnerability-report.md" { + stdin = lib.concatMapStrings (report: "${report}\n") reports; + } [ + "importas" "out" "out" + "redirfd" "-w" "1" "$out" + # Print introduction paragraph for the issue + "if" [ bins.printf "%s\n\n" description ] + # Print all reports + "foreground" [ + "forstdin" "-E" "report" bins.cat "$report" + ] + # Print stats at the end (mostly as a gimmick), we already know how many + # attributes there are and count the attributes with vulnerability by + # finding the number of checkable list entries in the output. + "backtick" "-E" "vulnerableCount" [ + "pipeline" [ + bins.grep "^- \\[ \\]" "$out" + ] + bins.wc "-l" + ] + "if" [ + bins.printf + "\n%s of %s checked attributes have vulnerable dependencies.\n\n" + "$vulnerableCount" + (toString (builtins.length reports)) + ] + "if" [ + bins.printf "%s\n\n" runInstructions + ] + ]; + + singleReport = + { # Attribute to check: string or list of strings (attr path) + attr + # Path to importable nixpkgs checkout + , nixpkgsPath + # Arguments to pass to nixpkgs + , nixpkgsArgs ? defaultNixpkgsArgs + }: + + let + attr' = if builtins.isString attr then [ attr ] else attr; + drv = lib.getAttrFromPath attr' (import nixpkgsPath nixpkgsArgs); + lockFile = extractCargoLock drv; + strAttr = lib.concatStringsSep "." attr'; + in + + depot.nix.runExecline "${strAttr}-report.html" {} [ + "importas" "out" "out" + "backtick" "-I" "-E" "-N" "report" [ + bins.cargo-audit "audit" + "--quiet" + "-n" "--db" rustsec-advisory-db + "-f" lockFile + ] + "pipeline" [ + "ifte" [ + bins.printf "%s" "$report" + ] [ + bins.printf "%s\n" "No vulnerabilities found" + ] + bins.test "-n" "$report" + ] + "pipeline" [ + bins.tee "/dev/stderr" + ] + "redirfd" "-w" "1" "$out" + bins.ansi2html + ]; + +in { + full = reportForNixpkgs; + single = singleReport; + + inherit + extractCargoLock + allLockFiles + ; + + # simple sanity check, doesn't cover everything, but testing the full report + # is quite expensive in terms of evaluation. + testSingle = singleReport { + nixpkgsPath = depot.third_party.nixpkgs.path; + attr = [ "ripgrep" ]; + }; + + meta.targets = [ "testSingle" ]; +} diff --git a/users/sterni/nixpkgs-crate-holes/format-audit-result.jq b/users/sterni/nixpkgs-crate-holes/format-audit-result.jq new file mode 100644 index 000000000000..e3147b8016c1 --- /dev/null +++ b/users/sterni/nixpkgs-crate-holes/format-audit-result.jq @@ -0,0 +1,61 @@ +# Link to human-readable advisory info for a given vulnerability +def link: + [ "https://rustsec.org/advisories/", .advisory.id, ".html" ] | add; + +# Format a list of version constraints +def version_list: + [ .[] | "`" + . + "`" ] | join("; "); + +# show paths to fixing this vulnerability: +# +# - if there are patched releases, show them (the version we are using presumably +# predates the vulnerability discovery, so we likely want to upgrade to a +# patched release). +# - if there are no patched releases, show the unaffected versions (in case we +# want to downgrade). +# - otherwise we state that no unaffected versions are available at this time. +# +# This logic should be useful, but is slightly dumber than cargo-audit's +# suggestion when using the non-JSON output. +def patched: + if .versions.patched == [] then + if .versions.unaffected != [] then + "unaffected: " + (.versions.unaffected | version_list) + else + "no unaffected version available" + end + else + "patched: " + (.versions.patched | version_list) + end; + +# if the vulnerability has aliases (like CVE-*) emit them in parens +def aliases: + if .advisory.aliases == [] then + "" + else + [ " (", (.advisory.aliases | join(", ")), ")" ] | add + end; + +# each vulnerability is rendered as a (normal) sublist item +def format_vulnerability: + [ " - " + , .package.name, " ", .package.version, ": " + , "[", .advisory.id, "](", link, ")" + , aliases + , ", ", patched + , "\n" + ] | add; + +# be quiet if no found vulnerabilities, otherwise render a GHFM checklist item +if .vulnerabilities.found | not then + "" +else + ([ "- [ ] " + , "`", $attr, "`: " + , (.vulnerabilities.count | tostring) + , " vulnerabilities in Cargo.lock" + , if $maintainers != "" then " (cc " + $maintainers + ")" else "" end + , "\n" + ] + (.vulnerabilities.list | map(format_vulnerability)) + ) | add +end diff --git a/users/tazjin/OWNERS b/users/tazjin/OWNERS new file mode 100644 index 000000000000..c86f6eaa6adb --- /dev/null +++ b/users/tazjin/OWNERS @@ -0,0 +1,3 @@ +inherited: false +owners: + - tazjin diff --git a/users/tazjin/aoc2019/default.nix b/users/tazjin/aoc2019/default.nix new file mode 100644 index 000000000000..ce3146d1f74e --- /dev/null +++ b/users/tazjin/aoc2019/default.nix @@ -0,0 +1,22 @@ +# Solutions for Advent of Code 2019, written in Emacs Lisp. +# +# For each day a new file is created as "solution-day$n.el". +{ depot, ... }: + +let + inherit (builtins) attrNames filter head listToAttrs match readDir; + dir = readDir ./.; + matchSolution = match "solution-(.*)\.el"; + isSolution = f: (matchSolution f) != null; + getDay = f: head (matchSolution f); + + solutionFiles = filter (e: dir."${e}" == "regular" && isSolution e) (attrNames dir); + solutions = map (f: let day = getDay f; in { + name = day; + value = depot.nix.writeElispBin { + name = "aoc2019"; + deps = p: with p; [ dash s ht ]; + src = ./. + ("/" + f); + }; + }) solutionFiles; +in listToAttrs solutions diff --git a/users/tazjin/aoc2019/solution-day1.el b/users/tazjin/aoc2019/solution-day1.el new file mode 100644 index 000000000000..d805c22ec870 --- /dev/null +++ b/users/tazjin/aoc2019/solution-day1.el @@ -0,0 +1,28 @@ +;; Advent of Code 2019 - Day 1 +(require 'dash) + +;; Puzzle 1: + +(defvar day-1/input + '(83285 96868 121640 51455 128067 128390 141809 52325 68310 140707 124520 149678 + 87961 52040 133133 52203 117483 85643 84414 86558 65402 122692 88565 61895 + 126271 128802 140363 109764 53600 114391 98973 124467 99574 69140 144856 + 56809 149944 138738 128823 82776 77557 51994 74322 64716 114506 124074 + 73096 97066 96731 149307 135626 121413 69575 98581 50570 60754 94843 72165 + 146504 53290 63491 50936 79644 119081 70218 85849 133228 114550 131943 + 67288 68499 80512 148872 99264 119723 68295 90348 146534 52661 99146 95993 + 130363 78956 126736 82065 77227 129950 97946 132345 107137 79623 148477 + 88928 118911 75277 97162 80664 149742 88983 74518)) + +(defun calculate-fuel (mass) + (- (/ mass 3) 2)) + +(message "Solution to day1/1: %d" (apply #'+ (-map #'calculate-fuel day-1/input))) + +;; Puzzle 2: +(defun calculate-recursive-fuel (mass) + (let ((fuel (calculate-fuel mass))) + (if (< fuel 0) 0 + (+ fuel (calculate-recursive-fuel fuel))))) + +(message "Solution to day1/2: %d" (apply #'+ (-map #'calculate-recursive-fuel day-1/input))) diff --git a/users/tazjin/aoc2019/solution-day2.el b/users/tazjin/aoc2019/solution-day2.el new file mode 100644 index 000000000000..6ecac1e2016c --- /dev/null +++ b/users/tazjin/aoc2019/solution-day2.el @@ -0,0 +1,53 @@ +;; -*- lexical-binding: t; -*- +;; Advent of Code 2019 - Day 2 +(require 'dash) +(require 'ht) + +(defvar day2/input + [1 0 0 3 1 1 2 3 1 3 4 3 1 5 0 3 2 1 9 19 1 19 5 23 1 13 23 27 1 27 6 31 + 2 31 6 35 2 6 35 39 1 39 5 43 1 13 43 47 1 6 47 51 2 13 51 55 1 10 55 + 59 1 59 5 63 1 10 63 67 1 67 5 71 1 71 10 75 1 9 75 79 2 13 79 83 1 9 + 83 87 2 87 13 91 1 10 91 95 1 95 9 99 1 13 99 103 2 103 13 107 1 107 10 + 111 2 10 111 115 1 115 9 119 2 119 6 123 1 5 123 127 1 5 127 131 1 10 + 131 135 1 135 6 139 1 10 139 143 1 143 6 147 2 147 13 151 1 5 151 155 1 + 155 5 159 1 159 2 163 1 163 9 0 99 2 14 0 0]) + +;; Puzzle 1 + +(defun day2/single-op (f state idx) + (let* ((a (aref state (aref state (+ 1 idx)))) + (b (aref state (aref state (+ 2 idx)))) + (p (aref state (+ 3 idx))) + (result (funcall f a b))) + (aset state p (funcall f a b)))) + +(defun day2/operate (state idx) + (pcase (aref state idx) + (99 (aref state 0)) + (1 (day2/single-op #'+ state idx) + (day2/operate state (+ 4 idx))) + (2 (day2/single-op #'* state idx) + (day2/operate state (+ 4 idx))) + (other (error "Unknown opcode: %s" other)))) + +(defun day2/program-with-inputs (noun verb) + (let* ((input (copy-tree day2/input t))) + (aset input 1 noun) + (aset input 2 verb) + (day2/operate input 0))) + +(message "Solution to day2/1: %s" (day2/program-with-inputs 12 2)) + +;; Puzzle 2 +(let* ((used (ht)) + (noun 0) + (verb 0) + (result (day2/program-with-inputs noun verb))) + (while (/= 19690720 result) + (setq noun (random 100)) + (setq verb (random 100)) + (unless (ht-get used (format "%d%d" noun verb)) + (ht-set used (format "%d%d" noun verb) t) + (setq result (day2/program-with-inputs noun verb)))) + + (message "Solution to day2/2: %s%s" noun verb)) diff --git a/users/tazjin/aoc2019/solution-day3.el b/users/tazjin/aoc2019/solution-day3.el new file mode 100644 index 000000000000..b7dfdd245fb1 --- /dev/null +++ b/users/tazjin/aoc2019/solution-day3.el @@ -0,0 +1,64 @@ +;; -*- lexical-binding: t; -*- +;; Advent of Code 2019 - Day 3 + +(require 'cl-lib) +(require 'dash) +(require 'ht) +(require 's) + +(defvar day3/input/wire1 + "R1010,D422,L354,U494,L686,U894,R212,U777,L216,U9,L374,U77,R947,U385,L170,U916,R492,D553,L992,D890,L531,U360,R128,U653,L362,U522,R817,U198,L126,D629,L569,U300,L241,U145,R889,D196,L450,D576,L319,D147,R985,U889,L941,U837,L608,D77,L864,U911,L270,D869,R771,U132,L249,U603,L36,D328,L597,U992,L733,D370,L947,D595,L308,U536,L145,U318,R55,D773,R175,D505,R483,D13,R780,U778,R445,D107,R490,U245,L587,U502,R446,U639,R150,U35,L455,D522,R866,U858,R394,D975,R513,D378,R58,D646,L374,D675,R209,U228,R530,U543,L480,U677,L912,D164,L573,U587,L784,D626,L994,U250,L215,U985,R684,D79,L877,U811,L766,U617,L665,D246,L408,U800,L360,D272,L436,U138,R240,U735,L681,U68,L608,D59,R532,D808,L104,U968,R887,U819,R346,U698,L317,U582,R516,U55,L303,U607,L457,U479,L510,D366,L583,U519,R878,D195,R970,D267,R842,U784,R9,D946,R833,D238,L232,D94,L860,D47,L346,U951,R491,D745,R849,U273,R263,U392,L341,D808,R696,U326,R886,D296,L865,U833,R241,U644,R729,D216,R661,D712,L466,D699,L738,U5,L556,D693,R912,D13,R48,U63,L877,U628,L689,D929,R74,U924,R612,U153,R417,U425,L879,D378,R79,D248,L3,U519,R366,U281,R439,D823,R149,D668,R326,D342,L213,D735,R504,U265,L718,D842,L565,U105,L214,U963,R518,D681,R642,U170,L111,U6,R697,U572,R18,U331,L618,D255,R534,D322,L399,U595,L246,U651,L836,U757,R417,D795,R291,U759,L568,U965,R828,D570,R350,U317,R338,D173,L74,D833,L650,D844,L70,U913,R594,U407,R674,D684,L481,D564,L128,D277,R851,D274,L435,D582,R469,U729,R387,D818,R443,U504,R414,U8,L842,U845,R275,U986,R53,U660,R661,D225,R614,U159,R477") + +(defvar day3/input/wire2 + "L1010,D698,R442,U660,L719,U702,L456,D86,R938,D177,L835,D639,R166,D285,L694,U468,L569,D104,L234,D574,L669,U299,L124,D275,L179,D519,R617,U72,L985,D248,R257,D276,L759,D834,R490,U864,L406,U181,R911,U873,R261,D864,R260,U759,R648,U158,R308,D386,L835,D27,L745,U91,R840,U707,R275,U543,L663,U736,L617,D699,R924,U103,R225,U455,R708,U319,R569,U38,R315,D432,L179,D975,R519,D546,L295,U680,L685,U603,R262,D250,R7,U171,R261,U519,L832,U534,L471,U431,L474,U886,R10,D179,L79,D555,R452,U452,L832,U863,L367,U538,L237,D160,R441,U605,R942,U259,L811,D552,R646,D353,L225,D94,L35,D307,R752,U23,R698,U610,L379,D932,R698,D751,R178,D347,R325,D156,R471,D555,R558,D593,R773,U2,L955,U764,L735,U438,R364,D640,L757,U534,R919,U409,R361,U407,R336,D808,R877,D648,R610,U198,R340,U94,R795,D667,R811,U975,L965,D224,R565,D681,L64,U567,R621,U922,L665,U329,R242,U592,L727,D481,L339,U402,R213,D280,R656,U169,R976,D962,L294,D505,L251,D689,L497,U133,R230,D441,L90,D220,L896,D657,L500,U331,R502,U723,R762,D613,L447,D256,L226,U309,L935,U384,L740,D459,R309,D707,R952,D747,L304,D105,R977,D539,R941,D21,R291,U216,R132,D543,R515,U453,L854,D42,R982,U102,L469,D639,R559,D68,R302,U734,R980,D214,R107,D191,L730,D793,L63,U17,R807,U196,R412,D592,R330,D941,L87,D291,L44,D94,L272,D780,R968,U837,L712,D704,R163,U981,R537,U778,R220,D303,L196,D951,R163,D446,R11,D623,L72,D778,L158,U660,L189,D510,L247,D716,L89,U887,L115,U114,L36,U81,R927,U293,L265,U183,R331,D267,R745,D298,L561,D918,R299,U810,L322,U679,L739,D854,L581,U34,L862,D779,R23") + +;; Puzzle 1 + +(defun wire-from (raw) + (-map (lambda (s) + (cons (substring s 0 1) (string-to-number (substring s 1)))) + (s-split "," raw))) + +(defun day3/move (x y next) + (cl-flet ((steps (by op) + (-map op (reverse (number-sequence 1 by))))) + (pcase next + (`("L" . ,by) (steps by (lambda (n) (cons (- x n) y)))) + (`("R" . ,by) (steps by (lambda (n) (cons (+ x n) y)))) + (`("U" . ,by) (steps by (lambda (n) (cons x (+ y n))))) + (`("D" . ,by) (steps by (lambda (n) (cons x (- y n)))))))) + +(defun day3/wire-points (wire) + (let ((points (ht)) + (point-list (-reduce-from + (lambda (acc point) + (-let* (((x . y) (car acc)) + (next (day3/move x y point))) + (-concat next acc))) + '((0 . 0)) wire))) + (-map (-lambda ((s . p)) (ht-set! points p s)) + (-zip (reverse (number-sequence 0 (- (length point-list) 1))) point-list)) + (ht-remove! points '(0 . 0)) + points)) + +(defun day3/closest-intersection (crossed-points) + (car (-sort #'< + (-map (-lambda ((x . y)) + (+ (abs x) (abs y))) + crossed-points)))) + +(defun day3/minimum-steps (wire1 wire2 crossed) + (car (-sort #'< + (-map (-lambda (p) + (+ (ht-get wire1 p) (ht-get wire2 p))) + crossed)))) + +;; Example: +(let* ((wire1-points (day3/wire-points (wire-from day3/input/wire1))) + (wire2-points (day3/wire-points (wire-from day3/input/wire2))) + (crossed-points (-filter (lambda (p) (ht-contains? wire1-points p)) + (ht-keys wire2-points)))) + (message "Solution for day3/1: %d" (day3/closest-intersection crossed-points)) + (message "Solution for day3/2: %d" (day3/minimum-steps wire1-points + wire2-points + crossed-points))) diff --git a/users/tazjin/aoc2019/solution-day4.el b/users/tazjin/aoc2019/solution-day4.el new file mode 100644 index 000000000000..2805f3f4e9cd --- /dev/null +++ b/users/tazjin/aoc2019/solution-day4.el @@ -0,0 +1,73 @@ +;; -*- lexical-binding: t; -*- +;; Advent of Code 2019 - Day 4 + +(require 'cl-lib) +(require 'dash) + +;; Puzzle 1 + +(defun day4/to-digits (num) + "Convert NUM to a list of its digits." + (cl-labels ((steps (n digits) + (if (= n 0) digits + (steps (/ n 10) (cons (% n 10) digits))))) + (steps num '()))) + +(defvar day4/input (-map #'day4/to-digits (number-sequence 128392 643281))) + +(defun day4/filter-password (digits) + "Determines whether the given rules match the supplied + number." + + (and + ;; It is a six digit number + (= 6 (length digits)) + + ;; Value is within the range given in puzzle input + ;; (noop because the range is generated from the input) + + ;; Two adjacent digits are the same (like 22 in 122345). + (car (-reduce-from (-lambda ((acc . prev) next) + (cons (or acc (= prev next)) next)) + '(nil . 0) digits)) + + ;; Going from left to right, the digits never decrease; they only + ;; ever increase or stay the same (like 111123 or 135679). + (car (-reduce-from (-lambda ((acc . prev) next) + (cons (and acc (>= next prev)) next)) + '(t . 0) digits)))) + +;; Puzzle 2 +;; +;; Additional criteria: If there's matching digits, they're not in a group. + +(cl-defstruct day4/acc state prev count) + +(defun day4/filter-longer-groups (digits) + (let ((res (-reduce-from + (lambda (acc next) + (cond ;; sequence is broken and count was at 1 -> + ;; match! + ((and (= (day4/acc-count acc) 2) + (/= (day4/acc-prev acc) next)) + (setf (day4/acc-state acc) t)) + + ;; sequence continues, counter increment! + ((= (day4/acc-prev acc) next) + (setf (day4/acc-count acc) (+ 1 (day4/acc-count acc)))) + + ;; sequence broken, reset counter + ((/= (day4/acc-prev acc) next) + (setf (day4/acc-count acc) 1))) + + (setf (day4/acc-prev acc) next) + acc) + (make-day4/acc :prev 0 :count 0) digits))) + (or (day4/acc-state res) + (= 2 (day4/acc-count res))))) + +(let* ((simple (-filter #'day4/filter-password day4/input)) + (complex (-filter #'day4/filter-longer-groups simple))) + (message "Solution to day4/1: %d" (length simple)) + (message "Solution to day4/2: %d" (length complex))) + diff --git a/users/tazjin/aoc2020/default.nix b/users/tazjin/aoc2020/default.nix new file mode 100644 index 000000000000..7a7309ac5aaa --- /dev/null +++ b/users/tazjin/aoc2020/default.nix @@ -0,0 +1,22 @@ +# Solutions for Advent of Code 2020, written in Emacs Lisp. +# +# For each day a new file is created as "solution-day$n.el". +{ depot, pkgs, ... }: + +let + inherit (builtins) attrNames filter head listToAttrs match readDir; + dir = readDir ./.; + matchSolution = match "solution-(.*)\.el"; + isSolution = f: (matchSolution f) != null; + getDay = f: head (matchSolution f); + + solutionFiles = filter (e: dir."${e}" == "regular" && isSolution e) (attrNames dir); + solutions = map (f: let day = getDay f; in depot.nix.writeElispBin { + name = day; + deps = p: with p; [ dash s ht p.f ]; + src = ./. + ("/" + f); + }) solutionFiles; +in pkgs.symlinkJoin { + name = "aoc2020"; + paths = solutions; +} diff --git a/users/tazjin/aoc2020/solution-day1.el b/users/tazjin/aoc2020/solution-day1.el new file mode 100644 index 000000000000..a04f43d15197 --- /dev/null +++ b/users/tazjin/aoc2020/solution-day1.el @@ -0,0 +1,44 @@ +;; Advent of Code 2020 - Day 1 +(require 'cl) +(require 'ht) +(require 'dash) + +(defmacro hash-set (&rest elements) + "Define a hash-table with empty values, for use as a set." + (cons 'ht (-map (lambda (x) (list x nil)) elements))) + +;; Puzzle 1: + +(defvar day1/input + (hash-set 1645 1995 1658 1062 1472 1710 1424 1823 1518 1656 1811 1511 1320 1521 1395 + 1996 1724 1666 1637 1504 1766 534 1738 1791 1372 1225 1690 1949 1495 1436 1166 + 1686 1861 1889 1887 997 1202 1478 833 1497 1459 1717 1272 1047 1751 1549 1204 + 1230 1260 1611 1506 1648 1354 1415 1615 1327 1622 1592 1807 1601 1026 1757 1376 + 1707 1514 1905 1660 1578 1963 1292 390 1898 1019 1580 1499 1830 1801 1881 1764 + 1442 1838 1088 1087 1040 1349 1644 1908 1697 1115 1178 1224 1810 1445 1594 1894 + 1287 1676 1435 1294 1796 1350 1685 1118 1488 1726 1696 1190 1538 1780 1806 1207 + 1346 1705 983 1249 1455 2002 1466 1723 1227 1390 1281 1715 1603 1862 1744 1774 + 1385 1312 1654 1872 1142 1273 1508 1639 1827 1461 1795 1533 1304 1417 1984 28 + 1693 1951 1391 1931 1179 1278 1400 1361 1369 1343 1416 1426 314 1510 1933 1239 + 1218 1918 1797 1255 1399 1229 723 1992 1595 1191 1916 1525 1605 1524 1869 1652 + 1874 1756 1246 1310 1219 1482 1429 1244 1554 1575 1123 1194 1408 1917 1613 1773 + 1809 1987 1733 1844 1423 1718 1714 1923 1503)) + +(message "Solution to day1/1: %s" + (cl-loop for first being the hash-keys of day1/input + for second = (- 2020 first) + when (ht-contains? day1/input second) + return (* first second))) + +;; Puzzle 2: + +(message "Solution to day1/1: %s" + (cl-loop for first being the hash-keys of day1/input + for result = + (cl-loop + for second being the elements of (-drop 1 (ht-keys day1/input)) + for third = (- 2020 first second) + when (ht-contains? day1/input third) + return (* first second third)) + + when result return result)) diff --git a/users/tazjin/aoc2020/solution-day2.el b/users/tazjin/aoc2020/solution-day2.el new file mode 100644 index 000000000000..5993bf3407e4 --- /dev/null +++ b/users/tazjin/aoc2020/solution-day2.el @@ -0,0 +1,54 @@ +;; Advent of Code 2020 - Day 2 + +(require 'cl-lib) +(require 'f) +(require 'ht) +(require 's) +(require 'seq) + +(defvar day2/input + ;; This one was too large to inline. + (s-lines (f-read "/tmp/aoc/day2.txt"))) + +(defun day2/count-letters (password) + (let ((table (ht-create))) + (cl-loop for char across password + for current = (ht-get table char) + do (ht-set table char + (if current (+ 1 current) 1))) + table)) + +(defun day2/parse (input) + (let* ((split (s-split " " input)) + (range (s-split "-" (car split)))) + (list (string-to-number (car range)) + (string-to-number (cadr range)) + (string-to-char (cadr split)) + (caddr split)))) + +(defun day2/count-with-validation (func) + (length (-filter + (lambda (password) + (and (not (seq-empty-p password)) + (apply func (day2/parse password)))) + day2/input))) + +;; Puzzle 1 + +(defun day2/validate-oldjob (min max char password) + (let ((count (ht-get (day2/count-letters password) char))) + (when count + (and (>= count min) + (<= count max))))) + +(message "Solution to day2/1: %s" + (day2/count-with-validation #'day2/validate-oldjob)) + +;; Puzzle 2 + +(defun day2/validate-toboggan (pos1 pos2 char password) + (xor (= char (aref password (- pos1 1))) + (= char (aref password (- pos2 1))))) + +(message "Solution to day2/2: %s" + (day2/count-with-validation #'day2/validate-toboggan)) diff --git a/users/tazjin/aoc2020/solution-day3.el b/users/tazjin/aoc2020/solution-day3.el new file mode 100644 index 000000000000..80ea4a226405 --- /dev/null +++ b/users/tazjin/aoc2020/solution-day3.el @@ -0,0 +1,43 @@ +;; Advent of Code 2020 - Day 3 + +(require 'cl-lib) +(require 'dash) +(require 'f) +(require 's) +(require 'seq) + +(setq day3/input + (-filter (lambda (s) (not (seq-empty-p s))) + (s-lines (f-read "/tmp/aoc/day3.txt")))) + +(setq day3/input-width (length (elt day3/input 0))) +(setq day3/input-height (length day3/input)) + +(defun day3/thing-at-point (x y) + "Pun intentional." + (when (>= day3/input-height y) + (let ((x-repeated (mod (- x 1) day3/input-width))) + (elt (elt day3/input (- y 1)) x-repeated)))) + +(defun day3/slope (x-steps y-steps) + "Produce the objects encountered through this slope until the + bottom of the map." + (cl-loop for x from 1 by x-steps + for y from 1 to day3/input-height by y-steps + collect (day3/thing-at-point x y))) + +;; Puzzle 1 + +(defun day3/count-trees (x-steps y-steps) + (cl-loop for thing being the elements of (day3/slope x-steps y-steps) + count (= thing ?#))) + +(message "Solution to day3/1: One encounters %s trees" (day3/count-trees 3 1)) + +;; Puzzle 2 + +(message "Solution to day3/2 %s" (* (day3/count-trees 1 1) + (day3/count-trees 3 1) + (day3/count-trees 5 1) + (day3/count-trees 7 1) + (day3/count-trees 1 2))) diff --git a/users/tazjin/aoc2020/solution-day4.el b/users/tazjin/aoc2020/solution-day4.el new file mode 100644 index 000000000000..034a40a9558d --- /dev/null +++ b/users/tazjin/aoc2020/solution-day4.el @@ -0,0 +1,98 @@ +;; Advent of Code 2020 - Day 4 + +(require 'cl-lib) +(require 's) +(require 'dash) +(require 'f) + +(cl-defstruct day4/passport + byr ;; Birth Year + iyr ;; Issue Year + eyr ;; Expiration Year + hgt ;; Height + hcl ;; Hair Color + ecl ;; Eye Color + pid ;; Passport ID + cid ;; Country ID + ) + +(defun day4/parse-passport (input) + (let* ((pairs (s-split " " (s-replace "\n" " " input) t)) + (slots + (-map + (lambda (pair) + (pcase-let ((`(,key ,value) (s-split ":" (s-trim pair)))) + (list (intern (format ":%s" key)) value))) + pairs))) + (apply #'make-day4/passport (-flatten slots)))) + +(defun day4/parse-passports (input) + (-map #'day4/parse-passport (s-split "\n\n" input t))) + +(setq day4/input (day4/parse-passports (f-read "/tmp/aoc/day4.txt"))) + +;; Puzzle 1 + +(defun day4/validate (passport) + "Check that all fields except CID are present." + (cl-check-type passport day4/passport) + (and (day4/passport-byr passport) + (day4/passport-iyr passport) + (day4/passport-eyr passport) + (day4/passport-hgt passport) + (day4/passport-hcl passport) + (day4/passport-ecl passport) + (day4/passport-pid passport))) + +(message "Solution to day4/1: %s" (cl-loop for passport being the elements of day4/input + count (day4/validate passport))) + +;; Puzzle 2 + +(defun day4/year-bound (min max value) + (and + (s-matches? (rx (= 4 digit)) value) + (<= min (string-to-number value) max))) + +(defun day4/check-unit (unit min max value) + (and + (string-match (rx (group (+? digit)) (literal unit)) value) + (<= min (string-to-number (match-string 1 value)) max))) + +(defun day4/properly-validate (passport) + "Opting for readable rather than clever here." + (and + (day4/validate passport) + + ;; byr (Birth Year) - four digits; at least 1920 and at most 2002. + (day4/year-bound 1920 2002 (day4/passport-byr passport)) + + ;; iyr (Issue Year) - four digits; at least 2010 and at most 2020. + (day4/year-bound 2010 2020 (day4/passport-iyr passport)) + + ;; eyr (Expiration Year) - four digits; at least 2020 and at most 2030. + (day4/year-bound 2020 2030 (day4/passport-eyr passport)) + + ;; hgt (Height) - a number followed by either cm or in: + ;; If cm, the number must be at least 150 and at most 193. + ;; If in, the number must be at least 59 and at most 76. + (or (day4/check-unit "cm" 150 193 (day4/passport-hgt passport)) + (day4/check-unit "in" 59 76 (day4/passport-hgt passport))) + + ;; hcl (Hair Color) - a # followed by exactly six characters 0-9 or a-f. + (s-matches? (rx ?# (= 6 hex)) (day4/passport-hcl passport)) + + ;; ecl (Eye Color) - exactly one of: amb blu brn gry grn hzl oth. + (-contains? '("amb" "blu" "brn" "gry" "grn" "hzl" "oth") + (day4/passport-ecl passport)) + + ;; pid (Passport ID) - a nine-digit number, including leading zeroes. + (s-matches? (rx line-start (= 9 digit) line-end) + (day4/passport-pid passport)) + + ;; cid (Country ID) - ignored, missing or not. + )) + +(message "Solution to day4/2: %s" + (cl-loop for passport being the elements of day4/input + count (day4/properly-validate passport))) diff --git a/users/tazjin/aoc2020/solution-day5.el b/users/tazjin/aoc2020/solution-day5.el new file mode 100644 index 000000000000..9bba322902b0 --- /dev/null +++ b/users/tazjin/aoc2020/solution-day5.el @@ -0,0 +1,61 @@ +;; Advent of Code 2020 - Day 5 + +(require 'cl-lib) +(require 'dash) +(require 'f) +(require 'ht) +(require 's) +(require 'seq) + +(defvar day5/input + (-filter (lambda (s) (not (seq-empty-p s))) + (s-lines (f-read "/tmp/aoc/day5.txt")))) + +(defun day5/lower (sequence) + (seq-subseq sequence 0 (/ (length sequence) 2))) + +(defun day5/upper (sequence) + (seq-subseq sequence (/ (length sequence) 2))) + +(defun day5/seat-id (column row) + (+ column (* 8 row))) + +(defun day5/find-seat (boarding-pass) + (let ((rows (number-sequence 0 127)) + (columns (number-sequence 0 7))) + (cl-loop for char across boarding-pass + do (pcase char + (?F (setq rows (day5/lower rows))) + (?B (setq rows (day5/upper rows))) + (?R (setq columns (day5/upper columns))) + (?L (setq columns (day5/lower columns)))) + finally return (day5/seat-id (car columns) (car rows))))) + +;; Puzzle 1 + +(message "Solution to day5/1: %s" + (cl-loop for boarding-pass in day5/input + maximize (day5/find-seat boarding-pass))) + +;; Puzzle 2 + +(defun day5/all-seats-in (row) + (-map (lambda (column) (day5/seat-id column row)) + (number-sequence 0 7))) + +(message "Solution to day5/2: %s" + (let ((all-seats (ht-create))) + (-each (-mapcat #'day5/all-seats-in (number-sequence 1 126)) + (lambda (seat) (ht-set all-seats seat nil))) + + (cl-loop for boarding-pass in day5/input + do (ht-remove all-seats (day5/find-seat boarding-pass)) + + ;; Remove seats that lack adjacent entries, those + ;; are missing on the plane. + finally return + (car + (-filter (lambda (seat) + (and (not (ht-contains? all-seats (- seat 1))) + (not (ht-contains? all-seats (+ seat 1))))) + (ht-keys all-seats)))))) diff --git a/users/tazjin/aoc2020/solution-day6.el b/users/tazjin/aoc2020/solution-day6.el new file mode 100644 index 000000000000..8179c79af2bd --- /dev/null +++ b/users/tazjin/aoc2020/solution-day6.el @@ -0,0 +1,40 @@ +;; Advent of Code 2020 - Day 6 + +(require 'cl-lib) +(require 'dash) +(require 'f) +(require 'ht) +(require 's) + +(defvar day6/input (s-split "\n\n" (f-read "/tmp/aoc/day6.txt") t) + "Input, split into groups (with people in each group still distinct)") + +;; Puzzle 1 + +(defun day6/count-answers (group-answers) + "I suspect doing it this way will be useful in puzzle 2." + (let ((table (ht-create))) + (-each group-answers + (lambda (answer) + (cl-loop for char across answer + do (ht-set table char (+ 1 (or (ht-get table char) + 0)))))) + table)) + +(message "Solution to day6/1: %s" + (cl-loop for group being the elements of day6/input + sum (length + (ht-keys + (day6/count-answers (s-lines group)))))) + +;; Puzzle 2 + +(defun day6/count-unanimous-answers (answers) + (ht-reject (lambda (_key value) (not (= value (length answers)))) + (day6/count-answers answers))) + +(message "Solution to day6/2: %s" + (cl-loop for group being the elements of day6/input + sum (length + (ht-keys + (day6/count-unanimous-answers (s-split "\n" group t)))))) diff --git a/users/tazjin/aoc2020/solution-day7.el b/users/tazjin/aoc2020/solution-day7.el new file mode 100644 index 000000000000..251a85fede02 --- /dev/null +++ b/users/tazjin/aoc2020/solution-day7.el @@ -0,0 +1,92 @@ +;; Advent of Code 2020 - Day 7 + +(require 'cl-lib) +(require 'dash) +(require 'f) +(require 's) +(require 'ht) + +(defvar day7/input + (s-lines (s-chomp (f-read "/tmp/aoc/day7.txt")))) + +(defun day7/parse-bag (input) + (string-match (rx line-start + (group (one-or-more (or letter space))) + "s contain " + (group (one-or-more anything)) + "." line-end) + input) + (cons (match-string 1 input) + (-map + (lambda (content) + (unless (equal content "no other bags") + (progn + (string-match + (rx (group (one-or-more digit)) + space + (group (one-or-more anything) "bag")) + content) + (cons (match-string 2 content) + (string-to-number (match-string 1 content)))))) + (s-split ", " (match-string 2 input))))) + +(defun day7/id-or-next (table bag-type) + (unless (ht-contains? table bag-type) + (ht-set table bag-type (length (ht-keys table)))) + (ht-get table bag-type)) + +(defun day7/build-graph (input &optional flip) + "Represent graph mappings directionally using an adjacency + matrix, because that's probably easiest. + + By default an edge means 'contains', with optional argument + FLIP edges are inverted and mean 'contained by'." + + (let ((bag-mapping (ht-create)) + (graph (let ((length (length input))) + (apply #'vector + (-map (lambda (_) (make-vector length 0)) input))))) + (cl-loop for bag in (-map #'day7/parse-bag input) + for bag-id = (day7/id-or-next bag-mapping (car bag)) + do (-each (-filter #'identity (cdr bag)) + (pcase-lambda (`(,contained-type . ,count)) + (let ((contained-id (day7/id-or-next bag-mapping contained-type))) + (if flip + (aset (aref graph contained-id) bag-id count) + (aset (aref graph bag-id) contained-id count)))))) + (cons bag-mapping graph))) + +;; Puzzle 1 + +(defun day7/find-ancestors (visited graph start) + (ht-set visited start t) + (cl-loop for bag-count being the elements of (aref graph start) + using (index bag-id) + when (and (> bag-count 0) + (not (ht-contains? visited bag-id))) + do (day7/find-ancestors visited graph bag-id))) + +(message + "Solution to day7/1: %s" + (pcase-let* ((`(,mapping . ,graph) (day7/build-graph day7/input t)) + (shiny-gold-id (ht-get mapping "shiny gold bag")) + (visited (ht-create))) + (day7/find-ancestors visited graph shiny-gold-id) + (- (length (ht-keys visited)) 1))) + +;; Puzzle 2 + +(defun ht-find-by-value (table value) + (ht-find (lambda (_key item-value) (equal item-value value)) table)) + +(defun day7/count-contained-bags (mapping graph start) + (cl-loop for bag-count being the elements of (aref graph start) + using (index bag-id) + when (> bag-count 0) + sum (+ bag-count + (* bag-count (day7/count-contained-bags mapping graph bag-id))))) + +(message "Solution to day7/2: %s" + (pcase-let* ((`(,mapping . ,graph) (day7/build-graph day7/input)) + (shiny-gold-id (ht-get mapping "shiny gold bag"))) + (day7/count-contained-bags mapping graph shiny-gold-id))) diff --git a/users/tazjin/aoc2020/solution-day8.el b/users/tazjin/aoc2020/solution-day8.el new file mode 100644 index 000000000000..591a07fbf3a0 --- /dev/null +++ b/users/tazjin/aoc2020/solution-day8.el @@ -0,0 +1,63 @@ +;; Advent of Code 2020 - Day + +(require 'cl-lib) +(require 'dash) +(require 'f) +(require 's) + +(setq day8/input + (apply #'vector + (-map (lambda (s) + (pcase-let ((`(,op ,val) (s-split " " s t))) + (cons (intern op) (string-to-number val)))) + (s-lines (s-chomp (f-read "/tmp/aoc/day8.txt")))))) + +(defun day8/step (code position acc) + (if (>= position (length code)) + (cons 'final acc) + + (let ((current (aref code position))) + (aset code position :done) + (pcase current + (:done (cons 'loop acc)) + (`(nop . ,val) (cons (+ position 1) acc)) + (`(acc . ,val) (cons (+ position 1) (+ acc val))) + (`(jmp . ,val) (cons (+ position val) acc)))))) + +;; Puzzle 1 + +(message "Solution to day8/1: %s" + (let ((code (copy-sequence day8/input)) + (position 0) + (acc 0)) + (cl-loop for next = (day8/step code position acc) + when (equal 'loop (car next)) return (cdr next) + do (setq position (car next)) + do (setq acc (cdr next))))) + +;; Puzzle 2 + +(defun day8/flip-at (code pos) + (pcase (aref code pos) + (`(nop . ,val) (aset code pos `(jmp . ,val))) + (`(jmp . ,val) (aset code pos `(nop . ,val))) + (other (error "Unexpected flip op: %s" other)))) + +(defun day8/try-flip (flip-at code position acc) + (day8/flip-at code flip-at) + (cl-loop for next = (day8/step code position acc) + when (equal 'loop (car next)) return nil + when (equal 'final (car next)) return (cdr next) + do (setq position (car next)) + do (setq acc (cdr next)))) + +(message "Solution to day8/2: %s" + (let ((flip-options (cl-loop for op being the elements of day8/input + using (index idx) + for opcode = (car op) + when (or (equal 'nop opcode) + (equal 'jmp opcode)) + collect idx))) + (cl-loop for flip-at in flip-options + for result = (day8/try-flip flip-at (copy-sequence day8/input) 0 0) + when result return result))) diff --git a/users/tazjin/avatar.jpeg b/users/tazjin/avatar.jpeg new file mode 100644 index 000000000000..f6888e01c7dc --- /dev/null +++ b/users/tazjin/avatar.jpeg Binary files differdiff --git a/users/tazjin/blog/.skip-subtree b/users/tazjin/blog/.skip-subtree new file mode 100644 index 000000000000..e7fa50d49bdd --- /dev/null +++ b/users/tazjin/blog/.skip-subtree @@ -0,0 +1 @@ +Subdirectories contain blog posts and static assets only diff --git a/users/tazjin/blog/default.nix b/users/tazjin/blog/default.nix new file mode 100644 index 000000000000..6ac89e46656a --- /dev/null +++ b/users/tazjin/blog/default.nix @@ -0,0 +1,43 @@ +{ depot, lib, pkgs, ... }: + +with depot.nix.yants; + +let + inherit (builtins) hasAttr filter; + + config = { + name = "tazjin's blog"; + baseUrl = "https://tazj.in/blog"; + + footer = '' + <p class="footer"> + <a class="uncoloured-link" href="https://tazj.in">homepage</a> + | + <a class="uncoloured-link" href="https://cs.tvl.fyi/">code</a> + </p> + <p class="lod">ಠ_ಠ</p> + ''; + }; + + inherit (depot.web.blog) post includePost renderPost; + + posts = filter includePost (list post (import ./posts.nix)); + + rendered = pkgs.runCommandNoCC "tazjins-blog" {} '' + mkdir -p $out + + ${lib.concatStringsSep "\n" (map (post: + "cp ${renderPost config post} $out/${post.key}.html" + ) posts)} + ''; + +in { + inherit posts rendered config; + + # Generate embeddable nginx configuration for redirects from old post URLs + oldRedirects = lib.concatStringsSep "\n" (map (post: '' + location ~* ^(/en)?/${post.oldKey} { + return 301 https://tazj.in/blog/${post.key}; + } + '') (filter (hasAttr "oldKey") posts)); +} diff --git a/users/tazjin/blog/posts.nix b/users/tazjin/blog/posts.nix new file mode 100644 index 000000000000..b43598d01358 --- /dev/null +++ b/users/tazjin/blog/posts.nix @@ -0,0 +1,57 @@ +# This file defines all the blog posts. +[ + { + key = "emacs-is-underrated"; + title = "Emacs is the most underrated tool"; + date = 1581286656; + content = ./posts/emacs-is-underrated.md; + draft = true; + } + { + key = "best-tools"; + title = "tazjin's best tools"; + date = 1576800001; + content = ./posts/best-tools.md; + } + { + key = "nixery-layers"; + title = "Nixery: Improved Layering Design"; + date = 1565391600; + content = ./posts/nixery-layers.md; + } + { + key = "reversing-watchguard-vpn"; + title = "Reverse-engineering WatchGuard Mobile VPN"; + date = 1486830338; + content = ./posts/reversing-watchguard-vpn.md; + oldKey = "1486830338"; + } + { + key = "make-object-t-again"; + title = "Make Object <T> Again!"; + date = 1476807384; + content = ./posts/make-object-t-again.md; + oldKey = "1476807384"; + } + { + key = "the-smu-problem"; + title = "The SMU-problem of messaging apps"; + date = 1450354078; + content =./posts/the-smu-problem.md; + oldKey = "1450354078"; + } + { + key = "sick-in-sweden"; + title = "Being sick in Sweden"; + date = 1423995834; + content = ./posts/sick-in-sweden.md; + oldKey = "1423995834"; + } + { + key = "nsa-zettabytes"; + title = "The NSA's 5 zettabytes of data"; + date = 1375310627; + content = ./posts/nsa-zettabytes.md; + oldKey = "1375310627"; + } +] diff --git a/users/tazjin/blog/posts/best-tools.md b/users/tazjin/blog/posts/best-tools.md new file mode 100644 index 000000000000..e4bad8f4cd07 --- /dev/null +++ b/users/tazjin/blog/posts/best-tools.md @@ -0,0 +1,160 @@ +In the spirit of various other "Which X do you use?"-pages I thought it would be +fun to have a little post here that describes which tools I've found to work +well for myself. + +When I say "tools" here, it's not about software - it's about real, physical +tools! + +If something goes on this list that's because I think it's seriously a +best-in-class type of product. + +<!-- markdown-toc start - Don't edit this section. Run M-x markdown-toc-refresh-toc --> +- [Media & Tech](#media--tech) + - [Keyboard](#keyboard) + - [Speakers](#speakers) + - [Headphones](#headphones) + - [Earphones](#earphones) + - [Phone](#phone) +- [Other stuff](#other-stuff) + - [Toothbrush](#toothbrush) + - [Shavers](#shavers) + - [Shoulder bag](#shoulder-bag) + - [Wallet](#wallet) +<!-- markdown-toc end --> + +--------- + +# Media & Tech + +## Keyboard + +The best keyboard that money will buy you at the moment is the [Kinesis +Advantage][advantage]. There's a variety of contoured & similarly shaped +keyboards on the market, but the Kinesis is the only one I've tried that has +properly implemented the keywell concept. + +I struggle with RSI issues and the Kinesis actually makes it possible for me to +type for longer periods of time, which always leads to extra discomfort on +laptop keyboards and such. + +Honestly, the Kinesis is probably the best piece of equipment on this entire +list. I own several of them and there will probably be more in the future. They +last forever and your wrists will thank you in the future, even if you do not +suffer from RSI yet. + +[advantage]: https://kinesis-ergo.com/shop/advantage2/ + +## Speakers + +The speakers that I've hooked up to my audio setup (including both record player +& Chromecast / TV) are the [Teufel Motiv 2][motiv-2]. I've had these for over a +decade and they're incredibly good, but unfortunately Teufel no longer makes +them. + +It's possible to grab a pair on eBay occasionally, so keep an eye out if you're +interested! + +[motiv-2]: https://www.teufelaudio.com/uk/pc/motiv-2-p167.html + +## Headphones + +I use the [Bose QC35][qc35] (note: link goes to a newer generation than the one +I own) for their outstanding noise cancelling functionality and decent sound. + +When I first bought them I didn't expect them to end up on this list as the +firmware had issues that made them only barely usable, but Bose has managed to +iron these problems out over time. + +I avoid using Bluetooth when outside and fortunately the QC35 come with an +optional cable that you can plug into any good old 3.5mm jack. + +[qc35]: https://www.bose.co.uk/en_gb/products/headphones/over_ear_headphones/quietcomfort-35-wireless-ii.html + +### Earphones + +Actually, to follow up on the above - most of the time I'm not using (over-ear) +headphones, but (in-ear) earphones - specifically the (**wired!!!**) [Apple +EarPods][earpods]. + +Apple will probably stop selling these soon because they've gotten into the +habit of cancelling all of their good products, so I have a stash of these +around. You will usually find no fewer than 3-4 of them lying around in my +flat. + +[earpods]: https://www.apple.com/uk/shop/product/MNHF2ZM/A/earpods-with-35mm-headphone-plug + +## Phone + +The best phone I have used in recent years is the [iPhone SE][se]. It was the +*last* phone that had a reasonable size (up to 4") *and* a 3.5mm headphone jack. + +Unfortunately, it runs iOS. Despite owning a whole bunch of SEs, I have finally +moved on to an Android phone that is only moderately larger (still by an +annoying amount), but does at least have a headphone jack: The [Samsung Galaxy +S10e][s10e]. + +It has pretty good hardware and I can almost reach 70% of the screen, which is +better than other phones out there right now. Unfortunately it runs Samsung's +impossible-to-remove bloatware on top of Android, but that is still less +annoying to use than iOS. + +QUESTION: This is the only item on this list for which I am actively seeking a +replacement, so if you have any tips about new phones that might fit these +criteria that I've missed please let me know! + +[se]: https://en.wikipedia.org/wiki/IPhone_SE +[s10e]: https://www.phonearena.com/phones/Samsung-Galaxy-S10e_id11114 + +# Other stuff + +## Toothbrush + +The [Philips Sonicare][sonicare] (note: link goes to a newer generation than +mine) is excellent and well worth its money. + +I've had it for a few years and whereas I occasionally had minor teeth issues +before, they seem to be mostly gone now. According to my dentist the state of my +teeth is now usually pretty good and I draw a direct correlation back to this +thing. + +The newer generations come with flashy features like apps and probably more +LEDs, but I suspect that those can just be ignored. + +[sonicare]: https://www.philips.co.uk/c-m-pe/electric-toothbrushes + +## Shavers + +The [Philipps SensoTouch 3D][sensotouch] is excellent. Super-comfortable close +face shave in no time and leaves absolutely no mess around, as far as I can +tell! I've had this for ~5 years and it's not showing any signs of aging yet. + +Another bonus is that its battery time is effectively infinite. I've never had +to worry when bringing it on a longer trip! + +[sensotouch]: https://www.philips.co.uk/c-p/1250X_40/norelco-sensotouch-3d-wet-and-dry-electric-razor-with-precision-trimmer + +## Shoulder bag + +When I moved to London I wanted to stop using backpacks most of the time, as +those are just annoying to deal with when commuting on the tube. + +To work around this I wanted a good shoulder bag with a vertical format (to save +space), but it turned out that there's very few of those around that reach any +kind of quality standard. + +The one I settled on is the [Waterfield Muzetto][muzetto] leather bag. It's one +of those things that comes with a bit of a price tag attached, but it's well +worth it! + +[muzetto]: https://www.sfbags.com/collections/shoulder-messenger-bags/products/muzetto-leather-bag + +## Wallet + +My wallet is the [Bellroy Slim Sleeve][slim-sleeve]. I don't carry cash unless +I'm attending an event in Germany and this wallet fits that lifestyle perfectly. + +It's near indestructible, looks great, is very slim and fits a ton of cards, +business cards, receipts and whatever else you want to be lugging around with +you! + +[slim-sleeve]: https://bellroy.com/products/slim-sleeve-wallet/default/charcoal diff --git a/users/tazjin/blog/posts/emacs-is-underrated.md b/users/tazjin/blog/posts/emacs-is-underrated.md new file mode 100644 index 000000000000..afb8dc889e53 --- /dev/null +++ b/users/tazjin/blog/posts/emacs-is-underrated.md @@ -0,0 +1,233 @@ +TIP: Hello, and thanks for offering to review my draft! This post +intends to convey to people what the point of Emacs is. Not to convert +them to use it, but at least with opening their minds to the +possibility that it might contain valuable things. I don't know if I'm +on track in the right direction, and your input will help me figure it +out. Thanks! + +TODO(tazjin): Restructure sections: Intro -> Introspectability (and +story) -> text-based UIs (which lead to fluidity, muscle memory across +programs and "translatability" of workflows) -> Outro. It needs more +flow! + +TODO(tazjin): Highlight more that it's not about editing: People can +derive useful things from Emacs by just using magit/org/notmuch/etc.! + +TODO(tazjin): Note that there's value in trying Emacs even if people +don't end up using it, similar to how learning languages like Lisp or +Haskell helps grow as a programmer even without using them day-to-day. + +*Real post starts below!* + +--------- + +There are two kinds of people: Those who use Emacs, and those who +think it is a text editor. This post is aimed at those in the second +category. + +Emacs is the most critical piece of software I run. My [Emacs +configuration][emacs-config] has steadily evolved for almost a decade. +Emacs is my window manager, mail client, terminal, git client, +information management system and - perhaps unsurprisingly - text +editor. + +Before going into why I chose to invest so much into this program, +follow me along on a little thought experiment: + +---------- + +Lets say you use a proprietary spreadsheet program. You find that +there are features in it that *almost, but not quite* do what you +want. + +What can you do? You can file a feature request to the company that +makes it and hope they listen, but for the likes of Apple and +Microsoft chances are they won't and there is nothing you can do. + +Let's say you are also running an open-source program for image +manipulation. You again find that some of its features are subtly +different from what you would want them to do. + +Things look a bit different this time - after all, the program is +open-source! You can go and fetch its source code, figure out its +internal structure and wrangle various layers of code into submission +until you find the piece that implements the functionality you want to +change. If you know the language it is written in; you can modify the +feature. + +Now all that's left is figuring out its build system[^1], building and +installing it and moving over to the new version. + +Realistically you are not going to do this much in the real world. The +friction to contributing to projects, especially complex ones, is +often quite high. For minor inconveniences, you might often find +yourself just shrugging and working around them. + +What if it didn't have to be this way? + +------------- + +One of the core properties of Emacs is that it is *introspective* and +*self-documenting*. + +For example: A few years ago, I had just switched over to using +[EXWM][], the Emacs X Window Manager. To launch applications I was +using an Emacs program called Helm that let me select installed +programs interactively and press <kbd>RET</kbd> to execute them. + +This was very useful - until I discovered that if I tried to open a +second terminal window, it would display an error: + + Error: urxvt is already running + +Had this been dmenu, I might have had to go through the whole process +described above to fix the issue. But it wasn't dmenu - it was an +Emacs program, and I did the following things: + +1. I pressed <kbd>C-h k</kbd>[^2] (which means "please tell me what + the following key does"), followed by <kbd>s-d</kbd> (which was my + keybinding for launching programs). + +2. Emacs displayed a new buffer saying, roughly: + + ``` + s-d runs the command helm-run-external-command (found in global-map), + which is an interactive autoloaded compiled Lisp function in + ‘.../helm-external.el’. + + It is bound to s-d. + ``` + + I clicked on the filename. + +3. Emacs opened the file and jumped to the definition of + `helm-run-external-command`. After a few seconds of reading through + the code, I found this snippet: + + ```lisp + (if (get-process proc) + (if helm-raise-command + (shell-command (format helm-raise-command real-com)) + (error "Error: %s is already running" real-com)) + ;; ... the actual code to launch programs followed below ... + ) + ``` + +4. I deleted the outer if-expression which implemented the behaviour I + didn't want, pressed <kbd>C-M-x</kbd> to reload the code and saved + the file. + +The whole process took maybe a minute, and the problem was now gone. + +Emacs isn't just "open-source", it actively encourages the user to +modify it, discover what to modify and experiment while it is running. + +In some sense it is like the experience of the old Lisp machines, a +paradigm that we have completely forgotten. + +--------------- + +Circling back to my opening statement: If Emacs is not a text editor, +then what *is* it? + +The Emacs website says this: + +> [Emacs] is an interpreter for Emacs Lisp, a dialect of the Lisp +> programming language with extensions to support text editing + +The core of Emacs implements the language and the functionality needed +to evaluate and run it, as well as various primitives for user +interface construction such as buffers, windows and frames. + +Every other feature of Emacs is implemented *in Emacs Lisp*. + +The Emacs distribution ships with rudimentary text editing +functionality (and some language-specific support for the most popular +languages), but it also brings with it two IRC clients, a Tetris +implementation, a text-mode web browser, [org-mode][] and many other +tools. + +Outside of the core distribution there is a myriad of available +programs for Emacs: [magit][] (the famous git porcelain), text-based +[HTTP clients][], even interactive [Kubernetes frontends][k8s]. + +What all of these tools have in common is that they use text-based +user interfaces (UI elements like images are used only sparingly in +Emacs), and that they can be introspected and composed like everything +else in Emacs. + +If magit does not expose a git flag I need, it's trivial to add. If I +want a keybinding to jump from a buffer showing me a Kubernetes pod to +a magit buffer for the source code of the container, it only takes a +few lines of Emacs Lisp to implement. + +As proficiency with Emacs Lisp ramps up, the environment becomes +malleable like clay and evolves along with the user's taste and needs. +Muscle memory learned for one program translates seamlessly to others, +and the overall effect is an improvement in *workflow fluidity* that +is difficult to overstate. + +Also, workflows based on Emacs are *stable*. Moving my window +management to Emacs has meant that I'm not subject to the whim of some +third-party developer changing my window layouting features (as they +often do on MacOS). + +To illustrate this: Emacs has development history back to the 1970s, +continuous git history that survived multiple VCS migrations [since +1985][first-commit] (that's 22 years before git itself was released!) +and there is code[^3] implementing interactive functionality that has +survived unmodified in Emacs *since then*. + +--------------- + +Now, what is the point of this post? + +I decided to write this after a recent [tweet][] by @IanColdwater (in +the context of todo-management apps): + +> The fact that it's 2020 and the most viable answer to this appears +> to be Emacs might be the saddest thing I've ever heard + +What bothers me is that people see this as *sad*. Emacs being around +for this long and still being unparalleled for many of the UX +paradigms implemented by its programs is, in my book, incredible - and +not sad. + +How many other paradigms have survived this long? How many other tools +still have fervent followers, amazing [developer tooling][] and a +[vibrant ecosystem][] at this age? + +Steve Yegge [said it best][babel][^5]: Emacs has the Quality Without a +Name. + +What I wish you, the reader, should take away from this post is the +following: + +TODO(tazjin): Figure out what people should take away from this post. +I need to sleep on it. It's something about not dismissing tools just +because of their age, urging them to explore paradigms that might seem +unfamiliar and so on. Ideas welcome. + +--------------- + +[^1]: Wouldn't it be a joy if every project just used Nix? I digress ... +[^2]: These are keyboard shortcuts written in [Emacs Key Notation][ekn]. +[^3]: For example, [functionality for online memes][studly] that + wouldn't be invented for decades to come! +[^4]: ... and some things wrong, but that is an issue for a separate post! +[^5]: And I really *do* urge you to read that post's section on Emacs. + +[emacs-config]: https://git.tazj.in/tree/tools/emacs +[EXWM]: https://github.com/ch11ng/exwm +[helm]: https://github.com/emacs-helm/helm +[ekn]: https://www.gnu.org/software/emacs/manual/html_node/efaq/Basic-keys.html +[org-mode]: https://orgmode.org/ +[magit]: https://magit.vc +[HTTP clients]: https://github.com/pashky/restclient.el +[k8s]: https://github.com/jypma/kubectl +[first-commit]: http://git.savannah.gnu.org/cgit/emacs.git/commit/?id=ce5584125c44a1a2fbb46e810459c50b227a95e2 +[studly]: http://git.savannah.gnu.org/cgit/emacs.git/commit/?id=47bdd84a0a9d20aab934482a64b84d0db63e7532 +[tweet]: https://twitter.com/IanColdwater/status/1220824466525229056 +[developer tooling]: https://github.com/alphapapa/emacs-package-dev-handbook +[vibrant ecosystem]: https://github.com/emacs-tw/awesome-emacs +[babel]: https://sites.google.com/site/steveyegge2/tour-de-babel#TOC-Lisp diff --git a/users/tazjin/blog/posts/make-object-t-again.md b/users/tazjin/blog/posts/make-object-t-again.md new file mode 100644 index 000000000000..420b57c0fde9 --- /dev/null +++ b/users/tazjin/blog/posts/make-object-t-again.md @@ -0,0 +1,98 @@ +A few minutes ago I found myself debugging a strange Java issue related +to Jackson, one of the most common Java JSON serialization libraries. + +The gist of the issue was that a short wrapper using some types from +[Javaslang](http://www.javaslang.io/) was causing unexpected problems: + +```java +public <T> Try<T> readValue(String json, TypeReference type) { + return Try.of(() -> objectMapper.readValue(json, type)); +} +``` + +The signature of this function was based on the original Jackson +`readValue` type signature: + +```java +public <T> T readValue(String content, TypeReference valueTypeRef) +``` + +While happily using my wrapper function I suddenly got an unexpected +error telling me that `Object` is incompatible with the type I was +asking Jackson to de-serialize, which got me to re-evaluate the above +type signature again. + +Lets look for a second at some code that will *happily compile* if you +are using Jackson\'s own `readValue`: + +```java +// This shouldn't compile! +Long l = objectMapper.readValue("\"foo\"", new TypeReference<String>(){}); +``` + +As you can see there we ask Jackson to decode the JSON into a `String` +as enclosed in the `TypeReference`, but assign the result to a `Long`. +And it compiles. And it failes at runtime with +`java.lang.ClassCastException: java.lang.String cannot be cast to java.lang.Long`. +Huh? + +Looking at the Jackson `readValue` implementation it becomes clear +what\'s going on here: + +```java +@SuppressWarnings({ "unchecked", "rawtypes" }) +public <T> T readValue(String content, TypeReference valueTypeRef) + throws IOException, JsonParseException, JsonMappingException +{ + return (T) _readMapAndClose(/* whatever */); +} +``` + +The function is parameterised over the type `T`, however the only place +where `T` occurs in the signature is in the parameter declaration and +the function return type. Java will happily let you use generic +functions and types without specifying type parameters: + +```java +// Compiles fine! +final List myList = List.of(1,2,3); + +// Type is now myList : List<Object> +``` + +Meaning that those parameters default to `Object`. Now in the code above +Jackson also explicitly casts the return value of its inner function +call to `T`. + +What ends up happening is that Java infers the expected return type from +the context of the `readValue` and then happily uses the unchecked cast +to fit that return type. If the type hints of the context aren\'t strong +enough we simply get `Object` back. + +So what\'s the fix for this? It\'s quite simple: + +```java +public <T> T readValue(String content, TypeReference<T> valueTypeRef) +``` + +By also making the parameter appear in the `TypeReference` we \"bind\" +`T` to the type enclosed in the type reference. The cast can then also +safely be removed. + +The cherries on top of this are: + +1. `@SuppressWarnings({ "rawtypes" })` explicitly disables a + warning that would\'ve caught this + +2. the `readValue` implementation using the less powerful `Class` + class to carry the type parameter does this correctly: `public <T> + T readValue(String content, Class<T> valueType)` + +The big question I have about this is *why* does Jackson do it this way? +Obviously the warning did not just appear there by chance, so somebody +must have thought about this? + +If anyone knows what the reason is, I\'d be happy to hear from you. + +PS: Shoutout to David & Lucia for helping me not lose my sanity over +this. diff --git a/users/tazjin/blog/posts/nixery-layers.md b/users/tazjin/blog/posts/nixery-layers.md new file mode 100644 index 000000000000..3f25ceadce7b --- /dev/null +++ b/users/tazjin/blog/posts/nixery-layers.md @@ -0,0 +1,272 @@ +TIP: This blog post was originally published as a design document for +[Nixery][] and is not written in the same style +as other blog posts. + +Thanks to my colleagues at Google and various people from the Nix community for +reviewing this. + +------ + +# Nixery: Improved Layering + +**Authors**: tazjin@ + +**Reviewers**: so...@, en...@, pe...@ + +**Status**: Implemented + +**Last Updated**: 2019-08-10 + +## Introduction + +This document describes a design for an improved image layering method for use +in Nixery. The algorithm [currently used][grhmc] is designed for a slightly +different use-case and we can improve upon it by making use of more of the +available data. + +## Background / Motivation + +Nixery is a service that uses the [Nix package manager][nix] to build container +images (for runtimes such as Docker), that are served on-demand via the +container [registry protocols][]. A demo instance is available at +[nixery.dev][]. + +In practice this means users can simply issue a command such as `docker pull +nixery.dev/shell/git` and receive an image that was built ad-hoc containing a +shell environment and git. + +One of the major advantages of building container images via Nix (as described +for `buildLayeredImage` in [this blog post][grhmc]) is that the +content-addressable nature of container image layers can be used to provide more +efficient caching characteristics (caching based on layer content) than what is +common with Dockerfiles and other image creation methods (caching based on layer +creation method). + +However, this is constrained by the maximum number of layers supported in an +image (125). A naive approach such as putting each included package (any +library, binary, etc.) in its own layer quickly runs into this limitation due to +the large number of dependencies more complex systems tend to have. In addition, +users wanting to extend images created by Nixery (e.g. via `FROM nixery.dev/…`) +share this layer maximum with the created image - limiting extensibility if all +layers are used up by Nixery. + +In theory the layering strategy of `buildLayeredImage` should already provide +good caching characteristics, but in practice we are seeing many images with +significantly more packages than the number of layers configured, leading to +more frequent cache-misses than desired. + +The current implementation of `buildLayeredImage` inspects a graph of image +dependencies and determines the total number of references (direct & indirect) +to any node in the graph. It then sorts all dependencies by this popularity +metric and puts the first `n - 2` (for `n` being the maximum number of layers) +packages in their own layers, all remaining packages in one layer and the image +configuration in the final layer. + +## Design / Proposal + +## (Close-to) ideal layer-layout using more data + +We start out by considering what a close to ideal layout of layers would look +like for a simple use-case. + + + +In this example, counting the total number of references to each node in the +graph yields the following result: + +| pkg | refs | +|-------|------| +| E | 3 | +| D | 2 | +| F | 2 | +| A,B,C | 1 | + +Assuming we are constrained to 4 layers, the current algorithm would yield these layers: + +``` +L1: E +L2: D +L3: F +L4: A, B, C +``` + +The initial proposal for this design is that additional data should be +considered in addition to the total number of references, in particular a +distinction should be made between direct and indirect references. Packages that +are only referenced indirectly should be merged with their parents. + +This yields the following table: + +| pkg | direct | indirect | +|-------|--------|----------| +| E | 3 | 3 | +| D | 2 | 2 | +| F | *1* | 2 | +| A,B,C | 1 | 1 | + +Despite having two indirect references, F is in fact only being referred to +once. Assuming that we have no other data available outside of this graph, we +have no reason to assume that F has any popularity outside of the scope of D. +This might yield the following layers: + +``` +L1: E +L2: D, F +L3: A +L4: B, C +``` + +D and F were grouped, while the top-level references (i.e. the packages +explicitly requested by the user) were split up. + +An assumption is introduced here to justify this split: The top-level packages +is what the user is modifying directly, and those groupings are likely +unpredictable. Thus it is opportune to not group top-level packages in the same +layer. + +This raises a new question: Can we make better decisions about where to split +the top-level? + +## (Even closer to) ideal layering using (even) more data + +So far when deciding layer layouts, only information immediately available in +the build graph of the image has been considered. We do however have much more +information available, as we have both the entire nixpkgs-tree and potentially +other information (such as download statistics). + +We can calculate the total number of references to any derivation in nixpkgs and +use that to rank the popularity of each package. Packages within some percentile +can then be singled out as good candidates for a separate layer. + +When faced with a splitting decision such as in the last section, this data can +aid the decision. Assume for example that package B in the above is actually +`openssl`, which is a very popular package. Taking this into account would +instead yield the following layers: + +``` +L1: E, +L2: D, F +L3: B, +L4: A, C +``` + +## Layer budgets and download size considerations + +As described in the introduction, there is a finite amount of layers available +for each image (the “layer budget”). When calculating the layer distribution, we +might end up with the “ideal” list of layers that we would like to create. Using +our previous example: + +``` +L1: E, +L2: D, F +L3: A +L4: B +L5: C +``` + +If we only have a layer budget of 4 available, something needs to be merged into +the same layer. To make a decision here we could consider only the package +popularity, but there is in fact another piece of information that has not come +up yet: The actual size of the package. + +Presumably a user would not mind downloading a library that is a few kilobytes +in size repeatedly, but they would if it was a 200 megabyte binary instead. + +Conversely if a large binary was successfully cached, but an extremely popular +small library is not, the total download size might also grow to irritating +levels. + +To avoid this we can calculate a merge rating: + + merge_rating(pkg) = popularity_percentile(pkg) × size(pkg.subtree) + +Packages with a low merge rating would be merged together before packages with +higher merge ratings. + +## Implementation + +There are two primary components of the implementation: + +1. The layering component which, given an image specification, decides the image + layers. + +2. The popularity component which, given the entire nixpkgs-tree, calculates the + popularity of packages. + +## Layering component + +It turns out that graph theory’s concept of [dominator trees][] maps reasonably +well onto the proposed idea of separating direct and indirect dependencies. This +becomes visible when creating the dominator tree of a simple example: + + + +Before calculating the dominator tree, we inspect each node and insert extra +edges from the root for packages that match a certain popularity or size +threshold. In this example, G is popular and an extra edge is inserted: + + + +Calculating the dominator tree of this graph now yields our ideal layer +distribution: + + + +The nodes immediately dominated by the root node can now be “harvested” as image +layers, and merging can be performed as described above until the result fits +into the layer budget. + +To implement this, the layering component uses the [gonum/graph][] library which +supports calculating dominator trees. The program is fed with Nix’s +`exportReferencesGraph` (which contains the runtime dependency graph and runtime +closure size) as well as the popularity data and layer budget. It returns a list +of layers, each specifying the paths it should contain. + +Nix invokes this program and uses the output to create a derivation for each +layer, which is then built and returned to Nixery as usual. + +TIP: This is implemented in [`layers.go`][layers.go] in Nixery. The file starts +with an explanatory comment that talks through the process in detail. + +## Popularity component + +The primary issue in calculating the popularity of each package in the tree is +that we are interested in the runtime dependencies of a derivation, not its +build dependencies. + +To access information about the runtime dependency, the derivation actually +needs to be built by Nix - it can not be inferred because Nix does not know +which store paths will still be referenced by the build output. + +However for packages that are cached in the NixOS cache, we can simply inspect +the `narinfo`-files and use those to determine popularity. + +Not every package in nixpkgs is cached, but we can expect all *popular* packages +to be cached. Relying on the cache should therefore be reasonable and avoids us +having to rebuild/download all packages. + +The implementation will read the `narinfo` for each store path in the cache at a +given commit and create a JSON-file containing the total reference count per +package. + +For the public Nixery instance, these popularity files will be distributed via a +GCS bucket. + +TIP: This is implemented in [popcount][] in Nixery. + +-------- + +Hopefully this detailed design review was useful to you. You can also watch [my +NixCon talk][talk] about Nixery for a review of some of this, and some demos. + +[Nixery]: https://github.com/google/nixery +[grhmc]: https://grahamc.com/blog/nix-and-layered-docker-images +[Nix]: https://nixos.org/nix +[registry protocols]: https://github.com/opencontainers/distribution-spec/blob/master/spec.md +[nixery.dev]: https://nixery.dev +[dominator trees]: https://en.wikipedia.org/wiki/Dominator_(graph_theory) +[gonum/graph]: https://godoc.org/gonum.org/v1/gonum/graph +[layers.go]: https://github.com/google/nixery/blob/master/builder/layers.go +[popcount]: https://github.com/google/nixery/tree/master/popcount +[talk]: https://www.youtube.com/watch?v=pOI9H4oeXqA diff --git a/users/tazjin/blog/posts/nsa-zettabytes.md b/users/tazjin/blog/posts/nsa-zettabytes.md new file mode 100644 index 000000000000..f8b326f2fb42 --- /dev/null +++ b/users/tazjin/blog/posts/nsa-zettabytes.md @@ -0,0 +1,93 @@ +I've been reading a few discussions on Reddit about the new NSA data +centre that is being built and stumbled upon [this +post](http://www.reddit.com/r/restorethefourth/comments/1jf6cx/the_guardian_releases_another_leaked_document_nsa/cbe5hnc), +putting its alleged storage capacity at *5 zettabytes*. + +That seems to be a bit much which I tried to explain to that guy, but I +was quickly blocked by the common conspiracy argument that government +technology is somehow far beyond the wildest dreams of us mere mortals - +thus I wrote a very long reply that will most likely never be seen by +anybody. Therefore I've decided to repost it here. + +------------------------------------------------------------------------ + +I feel like I've entered /r/conspiracy. Please have some facts (and do +read them!) + +A one terabyte SSD (I assume that\'s what you meant by flash-drive) +would require 5000000000 of those. That is *five billion* of those flash +drives. Can you visualise how much five billion flash-drives are? + +A single SSD is roughly 2cm\*13cm\*13cm with an approximate weight of +80g. That would make 400 000 metric tons of SSDs, a weight equivalent to +*over one thousand Boeing 747 airplanes*. Even if we assume that they +solder the flash chips directly onto some kind of controller (which also +weighs something), the raw material for that would be completely insane. + +Another visualization: If you stacked 5 billion SSDs on top of each +other you would get an SSD tower that is a hundred thousand kilometres +high, that is equivalent to 2,5 x the equatorial circumference of +*Earth* or 62000 miles. + +The volume of those SSDs would be clocking in at 1690000000 cubic +metres, more than the Empire State building. Are you still with me? + +Lets speak cost. The Samsung SSD that I assume you are referring to will +clock in at \$600, lets assume that the NSA gets a discount when buying +*five billion* of those and gets them at the cheap price of \$250. That +makes 1.25 trillion dollars. That would be a significant chunk of the +current US national debt. + +And all of this is just SSDs to stick into servers and storage units, +which need a whole bunch of other equipment as well to support them - +the cost would probably shoot up to something like 8 trillion dollars if +they were to build this. It would with very high certainty be more than +the annual production of SSDs (I can\'t find numbers on that +unfortunately) and take up *slightly* more space than they have in the +Utah data centre (assuming you\'re not going to tell me that it is in +fact attached to an underground base that goes down to the core of the +Earth). + +Lets look at the \"But the government has better technologies!\" idea. + +Putting aside the fact that the military *most likely* does not have a +secret base on Mars that deals with advanced science that the rest of us +can only dream of, and doing this under the assumption that they do have +this base, lets assume that they build a storage chip that stores 100TB. +This reduces the amount of needed chips to \"just\" 50 million, lets say +they get 10 of those into a server / some kind of specialized storage +unit and we only need 5 million of those specially engineered servers, +with custom connectors, software, chips, storage, most likely also power +sources and whatever - 10 million completely custom units built with +technology that is not available to the market. Google is estimated to +have about a million servers in total, I don\'t know exactly in how many +data centres those are placed but numbers I heard recently said that +it\'s about 40. When Apple assembles a new iPhone model they need +massive factories with thousands of workers and supplies from many +different countries, over several months, to assemble just a few million +units for their launch month. + +You are seriously proposing that the NSA is better than Google and Apple +and the rest of the tech industry, world-wide, combined at designing +*everything* in tech, manufacturing *everything* in tech, without *any* +information about that leaking and without *any* of the science behind +it being known? That\'s not just insane, that\'s outright impossible. + +And we haven\'t even touched upon how they would route the necessary +amounts of bandwidth (crazy insane) to save *the entire internet* into +that data center. + +------------------------------------------------------------------------ + +I\'m not saying that the NSA is not building a data center to store +surveillance information, to have more capacity to spy on people and all +that - I\'m merely making the point that the extent in which conspiracy +sites say they do this vastly overestimates their actual abilities. They +don\'t have magic available to them! Instead of making up insane figures +like that you should focus on what we actually know about their +operations, because using those figures in a debate with somebody who is +responsible for this (and knows what they\'re talking about) will end +with you being destroyed - nobody will listen to the rest of what +you\'re saying when that happens. + +\"Stick to the facts\" is valid for our side as well. diff --git a/users/tazjin/blog/posts/reversing-watchguard-vpn.md b/users/tazjin/blog/posts/reversing-watchguard-vpn.md new file mode 100644 index 000000000000..8968dc864590 --- /dev/null +++ b/users/tazjin/blog/posts/reversing-watchguard-vpn.md @@ -0,0 +1,158 @@ +TIP: WatchGuard has +[responded](https://www.reddit.com/r/netsec/comments/5tg0f9/reverseengineering_watchguard_mobile_vpn/dds6knx/) +to this post on Reddit. If you haven\'t read the post yet I\'d recommend +doing that first before reading the response to have the proper context. + +------------------------------------------------------------------------ + +One of my current clients makes use of +[WatchGuard](http://www.watchguard.com/help/docs/fireware/11/en-US/Content/en-US/mvpn/ssl/mvpn_ssl_client-install_c.html) +Mobile VPN software to provide access to the internal network. + +Currently WatchGuard only provides clients for OS X and Windows, neither +of which I am very fond of. In addition an OpenVPN configuration file is +provided, but it quickly turned out that this was only a piece of the +puzzle. + +The problem is that this VPN setup is secured using 2-factor +authentication (good!), but it does not use OpenVPN's default +[challenge/response](https://openvpn.net/index.php/open-source/documentation/miscellaneous/79-management-interface.html) +functionality to negotiate the credentials. + +Connecting with the OpenVPN config that the website supplied caused the +VPN server to send me a token to my phone, but I simply couldn't figure +out how to supply it back to the server. In a normal challenge/response +setting the token would be supplied as the password on the second +authentication round, but the VPN server kept rejecting that. + +Other possibilities were various combinations of username&password +(I've seen a lot of those around) so I tried a whole bunch, for example +`$password:$token` or even a `sha1(password, token)` - to no avail. + +At this point it was time to crank out +[Hopper](https://www.hopperapp.com/) and see what's actually going on +in the official OS X client - which uses OpenVPN under the hood! + +Diving into the client +---------------------- + +The first surprise came up right after opening the executable: It had +debug symbols in it - and was written in Objective-C! + + + +A good first step when looking at an application binary is going through +the strings that are included in it, and the WatchGuard client had a lot +to offer. Among the most interesting were a bunch of URIs that looked +important: + + + +I started with the first one + + %@?action=sslvpn_download&filename=%@&fw_password=%@&fw_username=%@ + +and just curled it on the VPN host, replacing the username and +password fields with bogus data and the filename field with +`client.wgssl` - another string in the executable that looked like a +filename. + +To my surprise this endpoint immediately responded with a GZIPed file +containing the OpenVPN config, CA certificate, and the client +*certificate and key*, which I previously thought was only accessible +after logging in to the web UI - oh well. + +The next endpoint I tried ended up being a bit more interesting still: + + /?action=sslvpn_logon&fw_username=%@&fw_password=%@&style=fw_logon_progress.xsl&fw_logon_type=logon&fw_domain=Firebox-DB + +Inserting the correct username and password into the query parameters +actually triggered the process that sent a token to my phone. The +response was a simple XML blob: + +```xml +<?xml version="1.0" encoding="UTF-8"?> +<resp> + <action>sslvpn_logon</action> + <logon_status>4</logon_status> + <auth-domain-list> + <auth-domain> + <name>RADIUS</name> + </auth-domain> + </auth-domain-list> + <logon_id>441</logon_id> + <chaStr>Enter Your 6 Digit Passcode </chaStr> +</resp> +``` + +Somewhat unsurprisingly that `chaStr` field is actually the challenge +string displayed in the client when logging in. + +This was obviously going in the right direction so I proceeded to the +procedures making use of this string. The first step was a relatively +uninteresting function called `-[VPNController sslvpnLogon]` which +formatted the URL, opened it and checked whether the `logon_status` was +`4` before proceeding with the `logon_id` and `chaStr` contained in the +response. + +*(Code snippets from here on are Hopper's pseudo-Objective-C)* + + + +It proceeded to the function `-[VPNController processTokenPrompt]` which +showed the dialog window into which the user enters the token, sent it +off to the next URL and checked the `logon_status` again: + +(`r12` is the reference to the `VPNController` instance, i.e. `self`). + + + +If the `logon_status` was `1` (apparently \"success\" here) it proceeded +to do something quite interesting: + + + +The user's password was overwritten with the (verified) OTP token - +before OpenVPN had even been started! + +Reading a bit more of the code in the subsequent +`-[VPNController doLogin]` method revealed that it shelled out to +`openvpn` and enabled the management socket, which makes it possible to +remotely control an `openvpn` process by sending it commands over TCP. + +It then simply sent the username and the OTP token as the credentials +after configuring OpenVPN with the correct config file: + + + +... and the OpenVPN connection then succeeds. + +TL;DR +----- + +Rather than using OpenVPN's built-in challenge/response mechanism, the +WatchGuard client validates user credentials *outside* of the VPN +connection protocol and then passes on the OTP token, which seems to be +temporarily in a 'blessed' state after verification, as the user's +password. + +I didn't check to see how much verification of this token is performed +(does it check the source IP against the IP that performed the challenge +validation?), but this certainly seems like a bit of a security issue - +considering that an attacker on the same network would, if they time the +attack right, only need your username and 6-digit OTP token to +authenticate. + +Don't roll your own security, folks! + +Bonus +----- + +The whole reason why I set out to do this is so I could connect to this +VPN from Linux, so this blog post wouldn't be complete without a +solution for that. + +To make this process really easy I've written a [little +tool](https://github.com/tazjin/watchblob) that performs the steps +mentioned above from the CLI and lets users know when they can +authenticate using their OTP token. diff --git a/users/tazjin/blog/posts/sick-in-sweden.md b/users/tazjin/blog/posts/sick-in-sweden.md new file mode 100644 index 000000000000..0c43c5832d73 --- /dev/null +++ b/users/tazjin/blog/posts/sick-in-sweden.md @@ -0,0 +1,26 @@ +I\'ve been sick more in the two years in Sweden than in the ten years +before that. + +Why? I have a theory about it and after briefly discussing it with one +of my roommates (who is experiencing the same thing) I\'d like to share +it with you: + +Normally when people get sick, are coughing, have a fever and so on they +take a few days off from work and stay at home. The reasons are twofold: +You want to rest a bit in order to get rid of the disease and you want +to *avoid infecting your co-workers*. + +In Sweden people will drag themselves into work anyways, because of a +concept called the +[karensdag](https://www.forsakringskassan.se/wps/portal/sjukvard/sjukskrivning_och_sjukpenning/karensdag_och_forstadagsintyg). +The TL;DR of this is \'if you take days off sick you won\'t get paid for +the first day, and only 80% of your salary on the remaining days\'. + +Many people are not willing to take that financial hit. In combination +with Sweden\'s rather mediocre healthcare system you end up constantly +being surrounded by sick people, not just in your own office but also on +public transport and basically all other public places. + +Oh and the best thing about this? Swedish politicians [often ignore +this](https://www.aftonbladet.se/nyheter/article10506886.ab) rule and +just don\'t report their sick days. Nice. diff --git a/users/tazjin/blog/posts/the-smu-problem.md b/users/tazjin/blog/posts/the-smu-problem.md new file mode 100644 index 000000000000..f411e3116046 --- /dev/null +++ b/users/tazjin/blog/posts/the-smu-problem.md @@ -0,0 +1,151 @@ +After having tested countless messaging apps over the years, being +unsatisfied with most of them and finally getting stuck with +[Telegram](https://telegram.org/) I have developed a little theory about +messaging apps. + +SMU stands for *Security*, *Multi-Device* and *Usability*. Quite like +the [CAP-theorem](https://en.wikipedia.org/wiki/CAP_theorem) I believe +that you can - using current models - only solve two out of three things +on this list. Let me elaborate what I mean by the individual points: + +**Security**: This is mainly about encryption of messages, not so much +about hiding identities to third-parties. Commonly some kind of +asymmetric encryption scheme. Verification of keys used must be possible +for the user. + +**Multi-Device**: Messaging-app clients for multiple devices, with +devices being linked to the same identifier, receiving the same messages +and being independent of each other. A nice bonus is also an open +protocol (like Telegram\'s) that would let people write new clients. + +**Usability**: Usability is a bit of a broad term, but what I mean by it +here is handling contacts and identities. It should be easy to create +accounts, give contact information to people and have everything just +work in a somewhat automated fashion. + +Some categorisation of popular messaging apps: + +**SU**: Threema + +**MU**: Telegram, Google Hangouts, iMessage, Facebook Messenger + +**SM**: +[Signal](https://gist.github.com/TheBlueMatt/d2fcfb78d29faca117f5) + +*Side note: The most popular messaging app - WhatsApp - only scores a +single letter (U). This makes it completely uninteresting to me.* + +Let\'s talk about **SM** - which might contain the key to solving SMU. +Two approaches are interesting here. + +The single key model +-------------------- + +In Signal there is a single identity key which can be used to register a +device on the server. There exists a process for sharing this identity +key from a primary device to a secondary one, so that the secondary +device can register itself (see the link above for a description). + +This *almost* breaks M because there is still a dependence on a primary +device and newly onboarded devices can not be used to onboard further +devices. However, for lack of a better SM example I\'ll give it a pass. + +The other thing it obviously breaks is U as the process for setting it +up is annoying and having to rely on the primary device is a SPOF (there +might be a way to recover from a lost primary device, but I didn\'t find +any information so far). + +The multiple key model +---------------------- + +In iMessage every device that a user logs into creates a new key pair +and submits its public key to a per-account key pool. Senders fetch all +available public keys for a recipient and encrypt to all of the keys. + +Devices that join can catch up on history by receiving it from other +devices that use its public key. + +This *almost* solves all of SMU, but its compliance with S breaks due to +the fact that the key pool is not auditable, and controlled by a +third-party (Apple). How can you verify that they don\'t go and add +another key to your pool? + +A possible solution +------------------- + +Out of these two approaches I believe the multiple key one looks more +promising. If there was a third-party handling the key pool but in a way +that is verifiable, transparent and auditable that model could be used +to solve SMU. + +The technology I have been thinking about for this is some kind of +blockchain model and here\'s how I think it could work: + +1. Bob installs the app and begins onboarding. The first device + generates its keypair, submits the public key and an account + creation request. + +2. Bob\'s account is created on the messaging apps\' servers and a + unique identifier plus the fingerprint of the first device\'s public + key is written to the chain. + +3. Alice sends a message to Bob, her device asks the messaging service + for Bob\'s account\'s identity and public keys. Her device verifies + the public key fingerprint against the one in the blockchain before + encrypting to it and sending the message. + +4. Bob receives Alice\'s message on his first device. + +5. Bob logs in to his account on a second device. The device generates + a key pair and sends the public key to the service, the service + writes it to the blockchain using its identifier. + +6. The messaging service requests that Bob\'s first device signs the + second device\'s key and triggers a simple confirmation popup. + +7. Bob confirms the second device on his first device. It signs the key + and writes the signature to the chain. + +8. Alice sends another message, her device requests Bob\'s current keys + and receives the new key. It verifies that both the messaging + service and one of Bob\'s older devices have confirmed this key in + the chain. It encrypts the message to both keys and sends it on. + +9. Bob receives Alice\'s message on both devices. + +After this the second device can request conversation history from the +first one to synchronise old messages. + +Further devices added to an account can be confirmed by any of the +devices already in the account. + +The messaging service could not add new keys for an account on its own +because it does not control any of the private keys confirmed by the +chain. + +In case all devices were lost, the messaging service could associate the +account with a fresh identity in the block chain. Message history +synchronisation would of course be impossible. + +Feedback welcome +---------------- + +I would love to hear some input on this idea, especially if anyone knows +of an attempt to implement a similar model already. Possible attack +vectors would also be really interesting. + +Until something like this comes to fruition, I\'ll continue using +Telegram with GPG as the security layer when needed. + +**Update:** WhatsApp has launched an integration with the Signal guys +and added their protocol to the official WhatsApp app. This means +WhatsApp now firmly sits in the SU-category, but it still does not solve +this problem. + +**Update 2:** Facebook Messenger has also integrated with Signal, but +their secret chats do not support multi-device well (it is Signal +afterall). This means it scores either SU or MU depending on which mode +you use it in. + +An interesting service I have not yet evaluated properly is +[Matrix](http://matrix.org/). diff --git a/users/tazjin/covid/us_mortality.jq b/users/tazjin/covid/us_mortality.jq new file mode 100644 index 000000000000..584be3ef9afe --- /dev/null +++ b/users/tazjin/covid/us_mortality.jq @@ -0,0 +1,36 @@ +# This turns the CDC mortality data[0] into a format useful for my +# excess mortality spreadsheet. The US format is by far the worst one +# I have dealt with, as expected. +# +# This requires miller for transforming the CSV appropriately. +# +# Params: +# state: abbreviation of the state to extract ('US' for whole country) +# period: time period (either "2020" for current data, or anything else +# for historical averages) +# +# Call as: +# mlr --icsv --ojson cat weekly.csv | \ +# jq -rsf us_mortality.jq --arg state US --arg period 2020 +# +# [0]: https://www.cdc.gov/nchs/nvss/vsrr/covid19/excess_deaths.htm + +def filter_period(period): + if period == "2020" + then . | map(select(.["Time Period"] == 2020)) + else . | map(select(.["Time Period"] == "2015-2019")) + end; + +def collate_weeks(period): + (. | map(.["Number of Deaths"]) | add) as $count + | { + count: (if period == "2020" then $count else $count / 5 end), + week: .[0].Week, + }; + +. | map(select(.Type == "Predicted (weighted)")) + | map(select(.["State Abbreviation"] == $state)) + | filter_period($period) + | group_by(.Week) + | map(collate_weeks($period)) + | .[] | "week \(.week): \(.count)" diff --git a/users/tazjin/dns/default.nix b/users/tazjin/dns/default.nix new file mode 100644 index 000000000000..da92b88beade --- /dev/null +++ b/users/tazjin/dns/default.nix @@ -0,0 +1,12 @@ +# Performs simple (local-only) validity checks on DNS zones. +{ depot, pkgs, ... }: + +let + checkZone = zone: file: pkgs.runCommandNoCC "${zone}-check" {} '' + ${pkgs.bind}/bin/named-checkzone -i local ${zone} ${file} | tee $out + ''; + +in depot.nix.readTree.drvTargets { + kontemplate-works = checkZone "kontemplate.works"./kontemplate.works.zone; + tazj-in = checkZone "tazj.in" ./tazj.in.zone; +} diff --git a/users/tazjin/dns/import b/users/tazjin/dns/import new file mode 100755 index 000000000000..8ea1d694c9a1 --- /dev/null +++ b/users/tazjin/dns/import @@ -0,0 +1,12 @@ +#!/bin/sh +set -ue + +# Imports a zone file into Google Cloud DNS +readonly ZONE="${1}" +readonly FILE="${2}" + +gcloud dns record-sets import "${FILE}" \ + --project composite-watch-759 \ + --zone-file-format \ + --delete-all-existing \ + --zone "${ZONE}" diff --git a/users/tazjin/dns/kontemplate.works.zone b/users/tazjin/dns/kontemplate.works.zone new file mode 100644 index 000000000000..326a129d2105 --- /dev/null +++ b/users/tazjin/dns/kontemplate.works.zone @@ -0,0 +1,15 @@ +;; -*- mode: zone; -*- +;; Do not delete these +kontemplate.works. 21600 IN NS ns-cloud-d1.googledomains.com. +kontemplate.works. 21600 IN NS ns-cloud-d2.googledomains.com. +kontemplate.works. 21600 IN NS ns-cloud-d3.googledomains.com. +kontemplate.works. 21600 IN NS ns-cloud-d4.googledomains.com. +kontemplate.works. 21600 IN SOA ns-cloud-d1.googledomains.com. cloud-dns-hostmaster.google.com. 4 21600 3600 259200 300 + +;; Github site setup +kontemplate.works. 60 IN A 185.199.108.153 +kontemplate.works. 60 IN A 185.199.109.153 +kontemplate.works. 60 IN A 185.199.110.153 +kontemplate.works. 60 IN A 185.199.111.153 + +www.kontemplate.works. 60 IN CNAME tazjin.github.io. diff --git a/users/tazjin/dns/tazj.in.zone b/users/tazjin/dns/tazj.in.zone new file mode 100644 index 000000000000..43db5834a0ca --- /dev/null +++ b/users/tazjin/dns/tazj.in.zone @@ -0,0 +1,33 @@ +;; -*- mode: zone; -*- +;; Do not delete these +tazj.in. 21600 IN NS ns-cloud-a1.googledomains.com. +tazj.in. 21600 IN NS ns-cloud-a2.googledomains.com. +tazj.in. 21600 IN NS ns-cloud-a3.googledomains.com. +tazj.in. 21600 IN NS ns-cloud-a4.googledomains.com. +tazj.in. 21600 IN SOA ns-cloud-a1.googledomains.com. cloud-dns-hostmaster.google.com. 123 21600 3600 1209600 300 + +;; Email setup +tazj.in. 300 IN MX 1 aspmx.l.google.com. +tazj.in. 300 IN MX 5 alt1.aspmx.l.google.com. +tazj.in. 300 IN MX 5 alt2.aspmx.l.google.com. +tazj.in. 300 IN MX 10 alt3.aspmx.l.google.com. +tazj.in. 300 IN MX 10 alt4.aspmx.l.google.com. +tazj.in. 300 IN TXT "v=spf1 include:_spf.google.com ~all" +google._domainkey.tazj.in. 21600 IN TXT "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA9AphX/WJf8zVXQB5Jk0Ry1MI6ARa6vEyAoJtpjpt9Nbm7XU4qVWFRJm+L0VFd5EZ5YDPJTIZ90lJE3/B8vae2ipnoGbJbj8LaVSzzIPMbWmhPhX3fkLJFdkv7xRDMDn730iYXRlfkgv6GsqbS8vZt7mzxx4mpnePTI323yjRVkwRW8nGVbsmB25ZoG1/0985" "kg4mSYxzWeJ2ozCPFhT4sfMtZMXe/4QEkJz/zkod29KZfFJmLgEaf73WLdBX8kdwbhuh2PYXt/PwzUrRzF5ujVCsSaTZwdRVPErcf+yo4NvedelTjjs8rFVfoJiaDD1q2bQ3w0gDEBWPdC2VP7k9zwIDAQAB" + +;; Site verifications +tazj.in. 3600 IN TXT "keybase-site-verification=gC4kzEmnLzY7F669PjN-pw2Cf__xHqcxQ08Gb-W9dhE" +tazj.in. 300 IN TXT "google-site-verification=d3_MI1OwD6q2OT42Vvh0I9w2u3Q5KFBu-PieNUE1Fig" +www.tazj.in. 3600 IN TXT "keybase-site-verification=ER8m_byyqAhzeIy9TyzkAU1H2p2yHtpvImuB_XrRF2U" + +;; Blog "storage engine" +blog.tazj.in. 21600 IN NS ns-cloud-c1.googledomains.com. +blog.tazj.in. 21600 IN NS ns-cloud-c2.googledomains.com. +blog.tazj.in. 21600 IN NS ns-cloud-c3.googledomains.com. +blog.tazj.in. 21600 IN NS ns-cloud-c4.googledomains.com. + +;; Webpage records setup +tazj.in. 300 IN A 34.98.120.189 +www.tazj.in. 300 IN A 34.98.120.189 +git.tazj.in. 300 IN A 34.98.120.189 +files.tazj.in. 300 IN CNAME c.storage.googleapis.com. diff --git a/users/tazjin/dotfiles/config.fish b/users/tazjin/dotfiles/config.fish new file mode 100644 index 000000000000..de2c99ae6007 --- /dev/null +++ b/users/tazjin/dotfiles/config.fish @@ -0,0 +1,40 @@ +# Configure classic prompt +set fish_color_user --bold blue +set fish_color_cwd --bold white + +# Enable colour hints in VCS prompt: +set __fish_git_prompt_showcolorhints yes +set __fish_git_prompt_color_prefix purple +set __fish_git_prompt_color_suffix purple + +# Fish configuration +set fish_greeting "" +set PATH $HOME/.local/bin $HOME/.cargo/bin $PATH + +# Editor configuration +set -gx EDITOR "emacsclient" +set -gx ALTERNATE_EDITOR "emacs -q -nw" +set -gx VISUAL "emacsclient" + +# Miscellaneous +eval (direnv hook fish) + +# Useful command aliases +alias gpr 'git pull --rebase' +alias gco 'git checkout' +alias gf 'git fetch' +alias gap 'git add -p' +alias pbcopy 'xclip -selection clipboard' +alias edit 'emacsclient -n' +alias servedir 'nix-shell -p haskellPackages.wai-app-static --run warp' + +# Old habits die hard (also ls is just easier to type): +alias ls 'exa' + +# Fix up nix-env & friends for Nix 2.0 +export NIX_REMOTE=daemon + +# Fix display of fish in emacs' term-mode: +function fish_title + true +end diff --git a/users/tazjin/dotfiles/default.nix b/users/tazjin/dotfiles/default.nix new file mode 100644 index 000000000000..9b783a9c857c --- /dev/null +++ b/users/tazjin/dotfiles/default.nix @@ -0,0 +1,3 @@ +_: { + dunstrc = ./dunstrc; +} diff --git a/users/tazjin/dotfiles/dunstrc b/users/tazjin/dotfiles/dunstrc new file mode 100644 index 000000000000..2aa1141b6ec2 --- /dev/null +++ b/users/tazjin/dotfiles/dunstrc @@ -0,0 +1,54 @@ +[global] +font = Iosevka Term 11 +origin = top-left +markup = yes +plain_text = no +format = "<b>%s</b>\n%b" +sort = no +indicate_hidden = yes +alignment = center +bounce_freq = 0 +show_age_threshold = -1 +word_wrap = yes +ignore_newline = no +stack_duplicates = yes +hide_duplicate_count = yes +geometry = "300x50-15+49" +shrink = no +transparency = 5 +idle_threshold = 0 +monitor = 0 +follow = keyboard +sticky_history = yes +history_length = 15 +show_indicators = no +line_height = 3 +separator_height = 2 +padding = 6 +horizontal_padding = 6 +separator_color = frame +startup_notification = false +dmenu = /usr/bin/dmenu -p dunst: +browser = /usr/bin/firefox -new-tab +icon_position = off +max_icon_size = 80 +frame_width = 3 +frame_color = "#8EC07C" + +[urgency_low] +frame_color = "#3B7C87" +foreground = "#3B7C87" +background = "#191311" +timeout = 4 + +[urgency_normal] +frame_color = "#5B8234" +foreground = "#5B8234" +background = "#191311" +timeout = 6 + +[urgency_critical] +frame_color = "#B7472A" +foreground = "#B7472A" +background = "#191311" +timeout = 8 diff --git a/users/tazjin/dotfiles/msmtprc b/users/tazjin/dotfiles/msmtprc new file mode 100644 index 000000000000..2af3b9433a6d --- /dev/null +++ b/users/tazjin/dotfiles/msmtprc @@ -0,0 +1,15 @@ +defaults +port 587 +tls on +tls_trust_file /etc/ssl/certs/ca-certificates.crt + +# GSuite for tazj.in +account tazjin +host smtp.gmail.com +port 587 +from mail@tazj.in +auth oauthbearer +user mail@tazj.in +passwordeval "cat ~/mail/account.tazjin/.credentials.gmailieer.json | jq -r '.access_token'" + +account default : tazjin diff --git a/users/tazjin/dotfiles/notmuch-config b/users/tazjin/dotfiles/notmuch-config new file mode 100644 index 000000000000..a490774e635f --- /dev/null +++ b/users/tazjin/dotfiles/notmuch-config @@ -0,0 +1,21 @@ +# .notmuch-config - Configuration file for the notmuch mail system +# +# For more information about notmuch, see https://notmuchmail.org + +[database] +path=/home/vincent/mail + +[user] +name=Vincent Ambo +primary_email=mail@tazj.in +other_email=tazjin@gmail.com; + +[new] +tags=unread;inbox; +ignore= + +[search] +exclude_tags=deleted;spam;draft; + +[maildir] +synchronize_flags=true diff --git a/users/tazjin/dt/CMakeLists.txt b/users/tazjin/dt/CMakeLists.txt new file mode 100644 index 000000000000..85b659fea862 --- /dev/null +++ b/users/tazjin/dt/CMakeLists.txt @@ -0,0 +1,16 @@ +# -*- mode: cmake; -*- +cmake_minimum_required(VERSION 3.16) +project(dt) +add_executable(dt dt.cc) +find_package(absl REQUIRED) + +target_link_libraries(dt + absl::flags + absl::flags_parse + absl::hash + absl::time + absl::strings + farmhash +) + +install(TARGETS dt DESTINATION bin) diff --git a/users/tazjin/dt/README.md b/users/tazjin/dt/README.md new file mode 100644 index 000000000000..ee43d5606409 --- /dev/null +++ b/users/tazjin/dt/README.md @@ -0,0 +1,11 @@ +dt +== + +It's got a purpose. + +## Usage: + +``` +nix-build -E '(import (builtins.fetchGit "https://git.tazj.in/") {}).fun.dt' +./result/bin/dt --one ... --two ... +``` diff --git a/users/tazjin/dt/default.nix b/users/tazjin/dt/default.nix new file mode 100644 index 000000000000..8a728062db21 --- /dev/null +++ b/users/tazjin/dt/default.nix @@ -0,0 +1,13 @@ +{ depot, pkgs, ... }: + +let stdenv = with pkgs; overrideCC clangStdenv clang_11; +in stdenv.mkDerivation { + name = "dt"; + src = ./.; + nativeBuildInputs = [ pkgs.cmake ]; + buildInputs = with depot.third_party; [ + abseil_cpp + farmhash + ]; + meta.ci = false; +} diff --git a/users/tazjin/dt/dt.cc b/users/tazjin/dt/dt.cc new file mode 100644 index 000000000000..5c4c3da76853 --- /dev/null +++ b/users/tazjin/dt/dt.cc @@ -0,0 +1,79 @@ +#include <iostream> +#include <vector> + +#include "absl/flags/flag.h" +#include "absl/flags/parse.h" +#include "absl/hash/hash.h" +#include "absl/strings/str_cat.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" +#include "absl/types/optional.h" +#include "farmhash.h" + +ABSL_FLAG(std::vector<std::string>, words, {}, "words to use"); + +struct Result { + std::string a; + int ec; + absl::optional<std::string> p; +}; + +std::string which(const std::vector<std::string>& words) { + uint64_t fp; + std::string word; + + for (const auto& w : words) { + auto nfp = util::Fingerprint64(w); + if (nfp > fp) { + fp = nfp; + word = w; + } + } + + return word; +} + +Result decide(const std::vector<std::string>& words) { + auto input = absl::FormatTime("%Y%m%d", absl::Now(), absl::UTCTimeZone()); + for (const auto& w : words) { + input += w; + } + + auto base = util::Fingerprint64(input); + Result result = { "nope" }; + + if (base % 10 == 0) { + result.a = "ca"; + } else if (base % 8 == 0) { + result.a = "c1"; + result.p = which(words); + } else if (base % 6 == 0) { + result.a = "skip"; + } else if (base % 3 == 0) { + result.a = "e1"; + result.ec = base % 10; + result.p = which(words); + } else if (base % 2 == 0) { + result.a = "ea"; + result.ec = base % 10; + } + + return result; +} + +int main(int argc, char *argv[]) { + absl::ParseCommandLine(argc, argv); + + auto words = absl::GetFlag(FLAGS_words); + if (words.size() < 2) { + std::cerr << "needs at least two!" << std::endl; + return 1; + } + + auto result = decide(words); + std::cout << result.a + << (result.p.has_value() ? absl::StrCat(" ", "(", result.p.value(), ")") + : "") + << (result.ec > 0 ? absl::StrCat(": ", result.ec) : "") + << std::endl; +} diff --git a/users/tazjin/emacs/.gitignore b/users/tazjin/emacs/.gitignore new file mode 100644 index 000000000000..7b666905f847 --- /dev/null +++ b/users/tazjin/emacs/.gitignore @@ -0,0 +1,11 @@ +.smex-items +*token* +auto-save-list/ +clones/ +elpa/ +irc.el +local.el +other/ +scripts/ +themes/ +*.elc diff --git a/users/tazjin/emacs/README.md b/users/tazjin/emacs/README.md new file mode 100644 index 000000000000..5c667333962e --- /dev/null +++ b/users/tazjin/emacs/README.md @@ -0,0 +1,7 @@ +tools/emacs +=========== + +This sub-folder builds my Emacs configuration, supplying packages from +Nix and configuration from this folder. + +I use Emacs for many things (including as my desktop environment). diff --git a/users/tazjin/emacs/config/bindings.el b/users/tazjin/emacs/config/bindings.el new file mode 100644 index 000000000000..21cca06cc83d --- /dev/null +++ b/users/tazjin/emacs/config/bindings.el @@ -0,0 +1,62 @@ +;; Font size +(define-key global-map (kbd "C-=") 'increase-default-text-scale) ;; '=' because there lies '+' +(define-key global-map (kbd "C--") 'decrease-default-text-scale) +(define-key global-map (kbd "C-x C-0") 'set-default-text-scale) + +;; What does <tab> do? Well, it depends ... +(define-key prog-mode-map (kbd "<tab>") #'company-indent-or-complete-common) + +;; imenu instead of insert-file +(global-set-key (kbd "C-x i") 'imenu) + +;; Window switching. (C-x o goes to the next window) +(windmove-default-keybindings) ;; Shift+direction + +;; Start eshell or switch to it if it's active. +(global-set-key (kbd "C-x m") 'eshell) + +(global-set-key (kbd "C-x C-p") 'browse-repositories) +(global-set-key (kbd "M-g M-g") 'goto-line-with-feedback) + +;; Miscellaneous editing commands +(global-set-key (kbd "C-c w") 'whitespace-cleanup) +(global-set-key (kbd "C-c a") 'align-regexp) +(global-set-key (kbd "C-c m") 'mc/mark-dwim) + +;; Browse URLs (very useful for Gitlab's SSH output!) +(global-set-key (kbd "C-c b p") 'browse-url-at-point) +(global-set-key (kbd "C-c b b") 'browse-url) + +;; C-x REALLY QUIT (idea by @magnars) +(global-set-key (kbd "C-x r q") 'save-buffers-kill-terminal) +(global-set-key (kbd "C-x C-c") 'ignore) + +;; Open a file in project: +(global-set-key (kbd "C-c f") 'project-find-file) + +;; Search in a project +(global-set-key (kbd "C-c r g") 'rg-in-project) + +;; Open a file via magit: +(global-set-key (kbd "C-c C-f") #'magit-find-file-worktree) + +;; Insert TODO comments +(global-set-key (kbd "C-c t") 'insert-todo-comment) + +;; Make sharing music easier +(global-set-key (kbd "s-s w") #'songwhip-lookup-url) + +;; Open the depot +(global-set-key (kbd "s-s d") #'tvl-depot-status) + +;; Add subthread collapsing to notmuch-show. +;; +;; C-, closes a thread, C-. opens a thread. This mirrors stepping +;; in/out of definitions. +(define-key notmuch-show-mode-map (kbd "C-,") 'notmuch-show-open-or-close-subthread) +(define-key notmuch-show-mode-map (kbd "C-.") + (lambda () + (interactive) + (notmuch-show-open-or-close-subthread t))) ;; open + +(provide 'bindings) diff --git a/users/tazjin/emacs/config/custom.el b/users/tazjin/emacs/config/custom.el new file mode 100644 index 000000000000..2bb7ad489600 --- /dev/null +++ b/users/tazjin/emacs/config/custom.el @@ -0,0 +1,25 @@ +(custom-set-variables + ;; custom-set-variables was added by Custom. + ;; If you edit it by hand, you could mess it up, so be careful. + ;; Your init file should contain only one such instance. + ;; If there is more than one, they won't work right. + '(ac-auto-show-menu 0.8) + '(ac-delay 0.2) + '(avy-background t) + '(cargo-process--enable-rust-backtrace 1) + '(company-auto-complete (quote (quote company-explicit-action-p))) + '(company-idle-delay 0.5) + '(custom-enabled-themes (quote (gruber-darker))) + '(display-time-default-load-average nil) + '(display-time-interval 30) + '(elnode-send-file-program "/run/current-system/sw/bin/cat") + '(frame-brackground-mode (quote dark)) + '(global-auto-complete-mode t) + '(kubernetes-commands-display-buffer-function (quote display-buffer)) + '(lsp-gopls-server-path "/home/tazjin/go/bin/gopls") + '(magit-log-show-gpg-status t) + '(ns-alternate-modifier (quote none)) + '(ns-command-modifier (quote control)) + '(ns-right-command-modifier (quote meta)) + '(require-final-newline (quote visit-save)) + '(tls-program (quote ("gnutls-cli --x509cafile %t -p %p %h")))) diff --git a/users/tazjin/emacs/config/desktop.el b/users/tazjin/emacs/config/desktop.el new file mode 100644 index 000000000000..3a4983f629f0 --- /dev/null +++ b/users/tazjin/emacs/config/desktop.el @@ -0,0 +1,261 @@ +;; -*- lexical-binding: t; -*- +;; +;; Configure desktop environment settings, including both +;; window-management (EXWM) as well as additional system-wide +;; commands. + +(require 's) +(require 'f) +(require 'dash) +(require 'exwm) +(require 'exwm-config) +(require 'exwm-randr) +(require 'exwm-systemtray) + +(defcustom tazjin--screen-lock-command "tazjin-screen-lock" + "Command to execute for locking the screen." + :group 'tazjin) + +(defcustom tazjin--backlight-increase-command "light -A 4" + "Command to increase screen brightness." + :group 'tazjin) + +(defcustom tazjin--backlight-decrease-command "light -U 4" + "Command to decrease screen brightness." + :group 'tazjin) + +(defun pactl (cmd) + (shell-command (concat "pactl " cmd)) + (message "Volume command: %s" cmd)) + +(defun volume-mute () (interactive) (pactl "set-sink-mute @DEFAULT_SINK@ toggle")) +(defun volume-up () (interactive) (pactl "set-sink-volume @DEFAULT_SINK@ +5%")) +(defun volume-down () (interactive) (pactl "set-sink-volume @DEFAULT_SINK@ -5%")) + +(defun brightness-up () + (interactive) + (shell-command tazjin--backlight-increase-command) + (message "Brightness increased")) + +(defun brightness-down () + (interactive) + (shell-command tazjin--backlight-decrease-command) + (message "Brightness decreased")) + +(defun set-xkb-layout (layout) + "Set the current X keyboard layout." + + (shell-command (format "setxkbmap %s" layout)) + (shell-command "setxkbmap -option caps:super") + (message "Set X11 keyboard layout to '%s'" layout)) + +(defun lock-screen () + (interactive) + (set-xkb-layout "us") + (shell-command tazjin--screen-lock-command)) + +(defun create-window-name () + "Construct window names to be used for EXWM buffers by + inspecting the window's X11 class and title. + + A lot of commonly used applications either create titles that + are too long by default, or in the case of web + applications (such as Cider) end up being constructed in + awkward ways. + + To avoid this issue, some rewrite rules are applied for more + human-accessible titles." + + (pcase (list (or exwm-class-name "unknown") (or exwm-title "unknown")) + ;; In Cider windows, rename the class and keep the workspace/file + ;; as the title. + (`("Google-chrome" ,(and (pred (lambda (title) (s-ends-with? " - Cider" title))) title)) + (format "Cider<%s>" (s-chop-suffix " - Cider" title))) + (`("Google-chrome" ,(and (pred (lambda (title) (s-ends-with? " - Cider V" title))) title)) + (format "Cider V<%s>" (s-chop-suffix " - Cider V" title))) + + ;; Attempt to detect IRCCloud windows via their title, which is a + ;; combination of the channel name and network. + ;; + ;; This is what would often be referred to as a "hack". The regexp + ;; will not work if a network connection buffer is selected in + ;; IRCCloud, but since the title contains no other indication that + ;; we're dealing with an IRCCloud window + (`("Google-chrome" + ,(and (pred (lambda (title) + (s-matches? "^[\*\+]\s#[a-zA-Z0-9/\-]+\s\|\s[a-zA-Z\.]+$" title))) + title)) + (format "IRCCloud<%s>" title)) + + ;; For other Chrome windows, make the title shorter. + (`("Google-chrome" ,title) + (format "Chrome<%s>" (s-truncate 42 (s-chop-suffix " - Google Chrome" title)))) + + ;; Gnome-terminal -> Term + (`("Gnome-terminal" ,title) + ;; fish-shell buffers contain some unnecessary whitespace and + ;; such before the current working directory. This can be + ;; stripped since most of my terminals are fish shells anyways. + (format "Term<%s>" (s-trim-left (s-chop-prefix "fish" title)))) + + ;; Quassel buffers + ;; + ;; These have a title format that looks like: + ;; "Quassel IRC - #tvl (hackint) — Quassel IRC" + (`("quassel" ,title) + (progn + (if (string-match + (rx "Quassel IRC - " + (group (one-or-more (any alnum "[" "]" "&" "-" "#"))) ;; <-- channel name + " (" (group (one-or-more (any ascii space))) ")" ;; <-- network name + " — Quassel IRC") + title) + (format "Quassel<%s>" (match-string 2 title)) + title))) + + ;; For any other application, a name is constructed from the + ;; window's class and name. + (`(,class ,title) (format "%s<%s>" class (s-truncate 12 title))))) + +;; EXWM launch configuration +;; +;; This used to use use-package, but when something breaks use-package +;; it doesn't exactly make debugging any easier. + +(let ((titlef (lambda () + (exwm-workspace-rename-buffer (create-window-name))))) + (add-hook 'exwm-update-class-hook titlef) + (add-hook 'exwm-update-title-hook titlef)) + +(fringe-mode 3) +(exwm-enable) + +;; 's-N': Switch to certain workspace +(setq exwm-workspace-number 10) +(dotimes (i 10) + (exwm-input-set-key (kbd (format "s-%d" i)) + `(lambda () + (interactive) + (exwm-workspace-switch-create ,i)))) + +;; Launch applications / any command with completion (dmenu style!) +(exwm-input-set-key (kbd "s-d") #'counsel-linux-app) +(exwm-input-set-key (kbd "s-x") #'run-external-command) +(exwm-input-set-key (kbd "s-p") #'password-store-lookup) + +;; Add X11 terminal selector to a key +(exwm-input-set-key (kbd "C-x t") #'ts/switch-to-terminal) + +;; Toggle between line-mode / char-mode +(exwm-input-set-key (kbd "C-c C-t C-t") #'exwm-input-toggle-keyboard) + +;; Volume keys +(exwm-input-set-key (kbd "<XF86AudioMute>") #'volume-mute) +(exwm-input-set-key (kbd "<XF86AudioRaiseVolume>") #'volume-up) +(exwm-input-set-key (kbd "<XF86AudioLowerVolume>") #'volume-down) + +;; Brightness keys +(exwm-input-set-key (kbd "<XF86MonBrightnessDown>") #'brightness-down) +(exwm-input-set-key (kbd "<XF86MonBrightnessUp>") #'brightness-up) +(exwm-input-set-key (kbd "<XF86Display>") #'lock-screen) + +;; Shortcuts for switching between keyboard layouts +(defmacro bind-xkb (lang key) + `(exwm-input-set-key (kbd (format "s-%s" ,key)) + (lambda () + (interactive) + (set-xkb-layout ,lang)))) + +(bind-xkb "us" "k u") +(bind-xkb "de" "k d") +(bind-xkb "no" "k n") +(bind-xkb "ru" "k r") +(bind-xkb "se" "k s") + +;; These are commented out because Emacs no longer starts (??) if +;; they're set at launch. +;; +(bind-xkb "us" "л г") +(bind-xkb "de" "л в") +(bind-xkb "no" "л т") +(bind-xkb "ru" "л к") + +;; Line-editing shortcuts +(exwm-input-set-simulation-keys + '(([?\C-d] . delete) + ([?\C-w] . ?\C-c))) + +;; Show time & battery status in the mode line +(display-time-mode) +(display-battery-mode) + +;; enable display of X11 system tray within Emacs +(exwm-systemtray-enable) + +;; Configure xrandr (multi-monitor setup). + +(defun set-randr-config (screens) + (setq exwm-randr-workspace-monitor-plist + (-flatten (-map (lambda (screen) + (-map (lambda (screen-id) (list screen-id (car screen))) (cdr screen))) + screens)))) + +;; Layouts for Tverskoy (X13 AMD laptop) +(defun randr-tverskoy-layout-single () + "Laptop screen only!" + (interactive) + (set-randr-config '(("eDP" (number-sequence 0 9)))) + (shell-command "xrandr --output eDP --auto --primary") + (shell-command "xrandr --output HDMI-A-0 --off") + (exwm-randr-refresh)) + +(defun randr-tverskoy-split-workspace () + "Split the workspace across two screens, assuming external to the left." + (interactive) + (set-randr-config + '(("HDMI-A-0" 1 2 3 4 5 6 7) + ("eDP" 8 9 0))) + + (shell-command "xrandr --output HDMI-A-0 --left-of eDP --auto") + (exwm-randr-refresh)) + +;; Layouts for frog (desktop) + +(defun randr-frog-layout-right-only () + "Use only the right screen on frog." + (interactive) + (set-randr-config `(("DisplayPort-0" ,(number-sequence 0 9)))) + (shell-command "xrandr --output DisplayPort-0 --off") + (shell-command "xrandr --output DisplayPort-1 --auto --primary")) + +(defun randr-frog-layout-both () + "Use the left and right screen on frog." + (interactive) + (set-randr-config `(("DisplayPort-0" 1 2 3 4 5) + ("DisplayPort-1" 6 7 8 9 0))) + + (shell-command "xrandr --output DisplayPort-0 --auto --primary --left-of DisplayPort-1") + (shell-command "xrandr --output DisplayPort-1 --auto --right-of DisplayPort-0 --rotate left")) + +(pcase (s-trim (shell-command-to-string "hostname")) + ("tverskoy" + (exwm-input-set-key (kbd "s-m s") #'randr-tverskoy-layout-single) + (exwm-input-set-key (kbd "s-m 2") #'randr-tverskoy-split-workspace)) + + ("frog" + (exwm-input-set-key (kbd "s-m b") #'randr-frog-layout-both) + (exwm-input-set-key (kbd "s-m r") #'randr-frog-layout-right-only))) + +;; Notmuch shortcuts as EXWM globals +;; (g m => gmail) +(exwm-input-set-key (kbd "s-g m") #'notmuch) +(exwm-input-set-key (kbd "s-g M") #'counsel-notmuch) + +(exwm-randr-enable) + +;; Let buffers move seamlessly between workspaces by making them +;; accessible in selectors on all frames. +(setq exwm-workspace-show-all-buffers t) +(setq exwm-layout-show-all-buffers t) + +(provide 'desktop) diff --git a/users/tazjin/emacs/config/eshell-setup.el b/users/tazjin/emacs/config/eshell-setup.el new file mode 100644 index 000000000000..0b23c5a2d1bc --- /dev/null +++ b/users/tazjin/emacs/config/eshell-setup.el @@ -0,0 +1,68 @@ +;; EShell configuration + +(require 'eshell) + +;; Generic settings +;; Hide banner message ... +(setq eshell-banner-message "") + +;; Prompt configuration +(defun clean-pwd (path) + "Turns a path of the form /foo/bar/baz into /f/b/baz + (inspired by fish shell)" + (let* ((hpath (replace-regexp-in-string home-dir + "~" + path)) + (current-dir (split-string hpath "/")) + (cdir (last current-dir)) + (head (butlast current-dir))) + (concat (mapconcat (lambda (s) + (if (string= "" s) nil + (substring s 0 1))) + head + "/") + (if head "/" nil) + (car cdir)))) + +(defun vcprompt (&optional args) + "Call the external vcprompt command with optional arguments. + VCPrompt" + (replace-regexp-in-string + "\n" "" + (shell-command-to-string (concat "vcprompt" args)))) + +(defmacro with-face (str &rest properties) + `(propertize ,str 'face (list ,@properties))) + +(defun prompt-f () + "EShell prompt displaying VC info and such" + (concat + (with-face (concat (clean-pwd (eshell/pwd)) " ") :foreground "#96a6c8") + (if (= 0 (user-uid)) + (with-face "#" :foreground "#f43841") + (with-face "$" :foreground "#73c936")) + (with-face " " :foreground "#95a99f"))) + + +(setq eshell-prompt-function 'prompt-f) +(setq eshell-highlight-prompt nil) +(setq eshell-prompt-regexp "^.+? \\((\\(git\\|svn\\|hg\\|darcs\\|cvs\\|bzr\\):.+?) \\)?[$#] ") + +;; Ignore version control folders in autocompletion +(setq eshell-cmpl-cycle-completions nil + eshell-save-history-on-exit t + eshell-cmpl-dir-ignore "\\`\\(\\.\\.?\\|CVS\\|\\.svn\\|\\.git\\)/\\'") + +;; Load some EShell extensions +(eval-after-load 'esh-opt + '(progn + (require 'em-term) + (require 'em-cmpl) + ;; More visual commands! + (add-to-list 'eshell-visual-commands "ssh") + (add-to-list 'eshell-visual-commands "tail") + (add-to-list 'eshell-visual-commands "sl"))) + +(setq eshell-directory-name "~/.config/eshell/") + +(provide 'eshell-setup) diff --git a/users/tazjin/emacs/config/functions.el b/users/tazjin/emacs/config/functions.el new file mode 100644 index 000000000000..5963d142c14b --- /dev/null +++ b/users/tazjin/emacs/config/functions.el @@ -0,0 +1,328 @@ +(require 'chart) +(require 'dash) +(require 'map) + +(defun load-file-if-exists (filename) + (if (file-exists-p filename) + (load filename))) + +(defun goto-line-with-feedback () + "Show line numbers temporarily, while prompting for the line number input" + (interactive) + (unwind-protect + (progn + (setq-local display-line-numbers t) + (let ((target (read-number "Goto line: "))) + (avy-push-mark) + (goto-line target))) + (setq-local display-line-numbers nil))) + +;; These come from the emacs starter kit + +(defun esk-add-watchwords () + (font-lock-add-keywords + nil '(("\\<\\(FIX\\(ME\\)?\\|TODO\\|DEBUG\\|HACK\\|REFACTOR\\|NOCOMMIT\\)" + 1 font-lock-warning-face t)))) + +(defun esk-sudo-edit (&optional arg) + (interactive "p") + (if (or arg (not buffer-file-name)) + (find-file (concat "/sudo:root@localhost:" (read-file-name "File: "))) + (find-alternate-file (concat "/sudo:root@localhost:" buffer-file-name)))) + +;; Open the NixOS man page +(defun nixos-man () + (interactive) + (man "configuration.nix")) + +;; Get the nix store path for a given derivation. +;; If the derivation has not been built before, this will trigger a build. +(defun nix-store-path (derivation) + (let ((expr (concat "with import <nixos> {}; " derivation))) + (s-chomp (shell-command-to-string (concat "nix-build -E '" expr "'"))))) + +(defun insert-nix-store-path () + (interactive) + (let ((derivation (read-string "Derivation name (in <nixos>): "))) + (insert (nix-store-path derivation)))) + +(defun toggle-force-newline () + "Buffer-local toggle for enforcing final newline on save." + (interactive) + (setq-local require-final-newline (not require-final-newline)) + (message "require-final-newline in buffer %s is now %s" + (buffer-name) + require-final-newline)) + +(defun list-external-commands () + "Creates a list of all external commands available on $PATH + while filtering NixOS wrappers." + (cl-loop + for dir in (split-string (getenv "PATH") path-separator) + when (and (file-exists-p dir) (file-accessible-directory-p dir)) + for lsdir = (cl-loop for i in (directory-files dir t) + for bn = (file-name-nondirectory i) + when (and (not (s-contains? "-wrapped" i)) + (not (member bn completions)) + (not (file-directory-p i)) + (file-executable-p i)) + collect bn) + append lsdir into completions + finally return (sort completions 'string-lessp))) + +(defvar external-command-flag-overrides + '(("google-chrome" . "--force-device-scale-factor=1.4")) + + "This setting lets me add additional flags to specific commands + that are run interactively via `run-external-command'.") + +(defun run-external-command--handler (cmd) + "Execute the specified command and notify the user when it + finishes." + (let* ((extra-flags (cdr (assoc cmd external-command-flag-overrides))) + (cmd (if extra-flags (s-join " " (list cmd extra-flags)) cmd))) + (message "Starting %s..." cmd) + (set-process-sentinel + (start-process-shell-command cmd nil cmd) + (lambda (process event) + (when (string= event "finished\n") + (message "%s process finished." process)))))) + +(defun run-external-command () + "Prompts the user with a list of all installed applications and + lets them select one to launch." + + (interactive) + (let ((external-commands-list (list-external-commands))) + (run-external-command--handler + (completing-read "Command: " external-commands-list + nil ;; predicate + t ;; require-match + nil ;; initial-input + ;; hist + 'external-commands-history)))) + +(defun password-store-lookup (&optional password-store-dir) + "Interactive password-store lookup function that actually uses +the GPG agent correctly." + + (interactive) + + (let* ((entry (completing-read "Copy password of entry: " + (password-store-list (or password-store-dir + (password-store-dir))) + nil ;; predicate + t ;; require-match + )) + (password (auth-source-pass-get 'secret entry))) + (password-store-clear) + (kill-new password) + (setq password-store-kill-ring-pointer kill-ring-yank-pointer) + (message "Copied %s to the kill ring. Will clear in %s seconds." + entry (password-store-timeout)) + (setq password-store-timeout-timer + (run-at-time (password-store-timeout) + nil 'password-store-clear)))) + +(defun browse-repositories () + "Select a git repository and open its associated magit buffer." + + (interactive) + (magit-status + (completing-read "Repository: " (magit-list-repos)))) + +(defun bottom-right-window-p () + "Determines whether the last (i.e. bottom-right) window of the + active frame is showing the buffer in which this function is + executed." + (let* ((frame (selected-frame)) + (right-windows (window-at-side-list frame 'right)) + (bottom-windows (window-at-side-list frame 'bottom)) + (last-window (car (seq-intersection right-windows bottom-windows)))) + (eq (current-buffer) (window-buffer last-window)))) + +(defhydra mc/mark-more-hydra (:color pink) + ("<up>" mmlte--up "Mark previous like this") + ("<down>" mc/mmlte--down "Mark next like this") + ("<left>" mc/mmlte--left (if (eq mc/mark-more-like-this-extended-direction 'up) + "Skip past the cursor furthest up" + "Remove the cursor furthest down")) + ("<right>" mc/mmlte--right (if (eq mc/mark-more-like-this-extended-direction 'up) + "Remove the cursor furthest up" + "Skip past the cursor furthest down")) + ("f" nil "Finish selecting")) + +;; Mute the message that mc/mmlte wants to print on its own +(advice-add 'mc/mmlte--message :around (lambda (&rest args) (ignore))) + +(defun mc/mark-dwim (arg) + "Select multiple things, but do what I mean." + + (interactive "p") + (if (not (region-active-p)) (mc/mark-next-lines arg) + (if (< 1 (count-lines (region-beginning) + (region-end))) + (mc/edit-lines arg) + ;; The following is almost identical to `mc/mark-more-like-this-extended', + ;; but uses a hydra (`mc/mark-more-hydra') instead of a transient key map. + (mc/mmlte--down) + (mc/mark-more-hydra/body)))) + +(defun memespace-region () + "Make a meme out of it." + + (interactive) + (let* ((start (region-beginning)) + (end (region-end)) + (memed + (message + (s-trim-right + (apply #'string + (-flatten + (nreverse + (-reduce-from (lambda (acc x) + (cons (cons x (-repeat (+ 1 (length acc)) 32)) acc)) + '() + (string-to-list (buffer-substring-no-properties start end)))))))))) + + (save-excursion (delete-region start end) + (goto-char start) + (insert memed)))) + +(defun insert-todo-comment (prefix todo) + "Insert a comment at point with something for me to do." + + (interactive "P\nsWhat needs doing? ") + (save-excursion + (move-end-of-line nil) + (insert (format " %s TODO(%s): %s" + (s-trim-right comment-start) + (if prefix (read-string "Who needs to do this? ") + (getenv "USER")) + todo)))) + +;; Custom text scale adjustment functions that operate on the entire instance +(defun modify-text-scale (factor) + (set-face-attribute 'default nil + :height (+ (* factor 5) (face-attribute 'default :height)))) + +(defun increase-default-text-scale (prefix) + "Increase default text scale in all Emacs frames, or just the + current frame if PREFIX is set." + + (interactive "P") + (if prefix (text-scale-increase 1) + (modify-text-scale 1))) + +(defun decrease-default-text-scale (prefix) + "Increase default text scale in all Emacs frames, or just the + current frame if PREFIX is set." + + (interactive "P") + (if prefix (text-scale-decrease 1) + (modify-text-scale -1))) + +(defun set-default-text-scale (prefix &optional to) + "Set the default text scale to the specified value, or the + default. Restores current frame's text scale only, if PREFIX is + set." + + (interactive "P") + (if prefix (text-scale-adjust 0) + (set-face-attribute 'default nil :height (or to 120)))) + +(defun scrot-select () + "Take a screenshot based on a mouse-selection and save it to + ~/screenshots." + (interactive) + (shell-command "scrot '$a_%Y-%m-%d_%s.png' -s -e 'mv $f ~/screenshots/'")) + +(defun graph-unread-mails () + "Create a bar chart of unread mails based on notmuch tags. + Certain tags are excluded from the overview." + + (interactive) + (let ((tag-counts + (-keep (-lambda ((name . search)) + (let ((count + (string-to-number + (s-trim + (notmuch-command-to-string "count" search "and" "tag:unread"))))) + (when (>= count 1) (cons name count)))) + (notmuch-hello-generate-tag-alist '("unread" "signed" "attachment" "important"))))) + + (chart-bar-quickie + (if (< (length tag-counts) 6) + 'vertical 'horizontal) + "Unread emails" + (-map #'car tag-counts) "Tag:" + (-map #'cdr tag-counts) "Count:"))) + +(defun notmuch-show-open-or-close-subthread (&optional prefix) + "Open or close the subthread from (and including) the message at point." + (interactive "P") + (save-excursion + (let ((current-depth (map-elt (notmuch-show-get-message-properties) :depth 0))) + (loop do (notmuch-show-message-visible (notmuch-show-get-message-properties) prefix) + until (or (not (notmuch-show-goto-message-next)) + (= (map-elt (notmuch-show-get-message-properties) :depth) current-depth))))) + (force-window-update)) + +(defun vterm-send-ctrl-x () + "Sends `C-x' to the libvterm." + (interactive) + (vterm-send-key "x" nil nil t)) + +(defun find-depot-project (dir) + "Function used in the `project-find-functions' hook list to + determine the current project root of a depot project." + (when (s-starts-with? "/depot" dir) + (if (f-exists-p (f-join dir "default.nix")) + (cons 'transient dir) + (find-depot-project (f-parent dir))))) + +(add-to-list 'project-find-functions #'find-depot-project) + +(defun magit-find-file-worktree () + (interactive) + "Find a file in the current (ma)git worktree." + (magit-find-file--internal "{worktree}" + (magit-read-file-from-rev "HEAD" "Find file") + #'pop-to-buffer-same-window)) + +(defun songwhip--handle-result (status &optional cbargs) + ;; TODO(tazjin): Inspect status, which looks different in practice + ;; than the manual claims. + (if-let* ((response (json-parse-string + (buffer-substring url-http-end-of-headers (point-max)))) + (sw-path (ht-get* response "data" "path")) + (link (format "https://songwhip.com/%s" sw-path)) + (select-enable-clipboard t)) + (progn + (kill-new link) + (message "Copied Songwhip link (%s)" link)) + (warn "Something went wrong while retrieving Songwhip link!") + ;; For debug purposes, the buffer is persisted in this case. + (setq songwhip--debug-buffer (current-buffer)))) + +(defun songwhip-lookup-url (url) + "Look up URL on Songwhip and copy the resulting link to the clipboard." + (interactive "sEnter source URL: ") + (let ((songwhip-url "https://songwhip.com/api/") + (url-request-method "POST") + (url-request-extra-headers '(("Content-Type" . "application/json"))) + (url-request-data + (json-serialize `((country . "GB") + (url . ,url))))) + (url-retrieve "https://songwhip.com/api/" #'songwhip--handle-result nil t t) + (message "Requesting Songwhip URL ... please hold the line."))) + +(defun rg-in-project (&optional prefix) + "Interactively call ripgrep in the current project, or fall + back to ripgrep default behaviour if prefix is set." + (interactive "P") + (counsel-rg nil (unless prefix + (if-let ((pr (project-current))) + (project-root pr))))) + +(provide 'functions) diff --git a/users/tazjin/emacs/config/init.el b/users/tazjin/emacs/config/init.el new file mode 100644 index 000000000000..9c638d826744 --- /dev/null +++ b/users/tazjin/emacs/config/init.el @@ -0,0 +1,313 @@ +;;; init.el --- Package bootstrapping. -*- lexical-binding: t; -*- + +;; Disable annoying warnings from native compilation. +(setq native-comp-async-report-warnings-errors nil + warning-suppress-log-types '((comp))) + +;; Packages are installed via Nix configuration, this file only +;; initialises the newly loaded packages. + +(require 'use-package) +(require 'seq) + +;; TODO(tazjin): Figure out what's up with vc. +;; +;; Leaving vc enabled breaks all find-file operations with messages +;; about .git folders being absent, but in random places. +(require 'vc) +(setq vc-handled-backends nil) + +(package-initialize) + +;; Initialise all packages installed via Nix. +;; +;; TODO: Generate this section in Nix for all packages that do not +;; require special configuration. + +;; +;; Packages providing generic functionality. +;; + +(use-package ace-window + :bind (("C-x o" . ace-window)) + :config + (setq aw-keys '(?f ?j ?d ?k ?s ?l ?a) + aw-scope 'frame)) + +(use-package auth-source-pass :config (auth-source-pass-enable)) + +(use-package avy + :bind (("M-j" . avy-goto-char) + ("M-p" . avy-pop-mark) + ("M-g g" . avy-goto-line))) + +(use-package browse-kill-ring) + +(use-package company + :hook ((prog-mode . company-mode)) + :config (setq company-tooltip-align-annotations t)) + +(use-package counsel + :after (ivy) + :config (counsel-mode 1)) + +(use-package dash) +(use-package gruber-darker-theme) + +(use-package eglot + :custom + (eglot-autoshutdown t) + (eglot-send-changes-idle-time 0.3)) + +(use-package elfeed + :config + (setq elfeed-feeds + '("https://lobste.rs/rss" + "https://www.anti-spiegel.ru/feed/" + "https://www.reddit.com/r/lockdownskepticism/.rss" + "https://www.reddit.com/r/rust/.rss" + "https://news.ycombinator.com/rss" + ("https://xkcd.com/atom.xml" media) + + ;; vlogcreations + ("https://www.youtube.com/feeds/videos.xml?channel_id=UCR0VLWitB2xM4q7tjkoJUPw" media) + ))) + +(use-package ht) + +(use-package hydra) +(use-package idle-highlight-mode :hook ((prog-mode . idle-highlight-mode))) + +(use-package ivy + :config + (ivy-mode 1) + (setq enable-recursive-minibuffers t) + (setq ivy-use-virtual-buffers t)) + +(use-package ivy-prescient + :after (ivy prescient) + :config + (ivy-prescient-mode) + ;; Fixes an issue with how regexes are passed to ripgrep from counsel, + ;; see raxod502/prescient.el#43 + (setf (alist-get 'counsel-rg ivy-re-builders-alist) #'ivy--regex-plus)) + +(use-package multiple-cursors) + +(use-package notmuch + :custom + (notmuch-search-oldest-first nil) + (notmuch-show-all-tags-list t) + (notmuch-hello-tag-list-make-query "tag:unread")) + +(use-package paredit :hook ((lisp-mode . paredit-mode) + (emacs-lisp-mode . paredit-mode))) + +(use-package pinentry + :config + (setq epa-pinentry-mode 'loopback) + (pinentry-start)) + +(use-package prescient + :after (ivy counsel) + :config (prescient-persist-mode)) + +(use-package rainbow-delimiters + :hook (prog-mode . rainbow-delimiters-mode) + :custom-face + (rainbow-delimiters-depth-1-face ((t (:foreground "#2aa198")))) + (rainbow-delimiters-depth-2-face ((t (:foreground "#b58900")))) + (rainbow-delimiters-depth-3-face ((t (:foreground "#268bd2")))) + (rainbow-delimiters-depth-4-face ((t (:foreground "#dc322f")))) + (rainbow-delimiters-depth-5-face ((t (:foreground "#859900")))) + (rainbow-delimiters-depth-6-face ((t (:foreground "#268bd2")))) + (rainbow-delimiters-depth-7-face ((t (:foreground "#cb4b16")))) + (rainbow-delimiters-depth-8-face ((t (:foreground "#d33682")))) + (rainbow-delimiters-depth-9-face ((t (:foreground "#839496"))))) + +(use-package rainbow-mode) +(use-package s) +(use-package string-edit) + +(use-package swiper + :after (counsel ivy) + :bind (("C-s" . swiper))) + +(use-package telephone-line) ;; configuration happens outside of use-package +(use-package term-switcher) +(use-package undo-tree :config (global-undo-tree-mode)) +(use-package uuidgen) +(use-package which-key :config (which-key-mode t)) + +;; +;; Applications in emacs +;; + +(use-package magit + :bind ("C-c g" . magit-status) + :config (setq magit-repository-directories '(("/home/tazjin/projects" . 2) + ("/home/tazjin" . 1)))) + +(use-package password-store) +(use-package restclient) + +(use-package vterm + :config (progn + (setq vterm-shell "fish") + (setq vterm-exit-functions + (lambda (&rest _) (kill-buffer (current-buffer)))) + (setq vterm-kill-buffer-on-exit t)) + :custom-face + (term-color-black ((t (:background "#282828" :foreground "#282828")))) + (term-color-blue ((t (:background "#96a6c8" :foreground "#96a6c8")))) + (term-color-cyan ((t (:background "#1fad83" :foreground "#1fad83")))) + (term-color-green ((t (:background "#73c936" :foreground "#73c936")))) + (term-color-magenta ((t (:background "#9e95c7" :foreground "#9e95c7")))) + (term-color-red ((t (:background "#f43841" :foreground "#f43841")))) + (term-color-white ((t (:background "#f5f5f5" :foreground "#f5f5f5")))) + (term-color-yellow ((t (:background "#ffdd33" :foreground "#ffdd33"))))) + +;; vterm removed the ability to set a custom title generator function +;; via the public API, so this overrides its private title generation +;; function instead +(defun vterm--set-title (title) + (rename-buffer + (generate-new-buffer-name + (format "vterm<%s>" + (s-trim-left + (s-chop-prefix "fish" title)))))) + +;; +;; Packages providing language-specific functionality +;; + +(use-package cargo + :hook ((rust-mode . cargo-minor-mode) + (cargo-process-mode . visual-line-mode)) + :bind (:map cargo-mode-map ("C-c C-c C-l" . ignore))) + +(use-package dockerfile-mode) + +(use-package erlang + :hook ((erlang-mode . (lambda () + ;; Don't indent after '>' while I'm writing + (local-set-key ">" 'self-insert-command))))) + +(use-package f) + +(use-package go-mode + :bind (:map go-mode-map ("C-c C-r" . recompile)) + :hook ((go-mode . (lambda () + (setq tab-width 2) + (setq-local compile-command + (concat "go build " buffer-file-name)))))) + +(use-package haskell-mode) + +(use-package ielm + :hook ((inferior-emacs-lisp-mode . (lambda () + (paredit-mode) + (rainbow-delimiters-mode-enable) + (company-mode))))) + +(use-package jq-mode + :config (add-to-list 'auto-mode-alist '("\\.jq\\'" . jq-mode))) + +(use-package kotlin-mode + :hook ((kotlin-mode . (lambda () + (setq indent-line-function #'indent-relative))))) + +(use-package lsp-mode) + +(use-package markdown-mode + :config + (add-to-list 'auto-mode-alist '("\\.markdown\\'" . markdown-mode)) + (add-to-list 'auto-mode-alist '("\\.md\\'" . markdown-mode))) + +(use-package markdown-toc) + +(use-package nix-mode + :hook ((nix-mode . (lambda () + (setq indent-line-function #'nix-indent-line))))) + +(use-package nix-util) +(use-package nginx-mode) +(use-package rust-mode) + +(use-package sly + :hook ((sly-mrepl-mode . (lambda () + (paredit-mode) + (rainbow-delimiters-mode-enable) + (company-mode)))) + :config + (setq common-lisp-hyperspec-root "file:///home/tazjin/docs/lisp/")) + +(use-package telega + :bind (:map global-map ("s-t" . telega)) + :config + (telega-mode-line-mode 1) + (add-hook 'telega-msg-ignore-predicates 'telega-msg-from-blocked-sender-p)) + +(use-package terraform-mode) +(use-package toml-mode) + +(use-package tvl + :custom + (tvl-gerrit-remote "gerrit")) + +(use-package web-mode) +(use-package yaml-mode) + +(use-package passively + :custom + (passively-store-state "/persist/tazjin/known-russian-words.el")) + +;; Initialise midnight.el, which by default automatically cleans up +;; unused buffers at midnight. +(require 'midnight) + +(defgroup tazjin nil + "Settings related to my configuration") + +(defcustom depot-path "/depot" + "Local path to the depot checkout" + :group 'tazjin) + +;; Configuration changes in `customize` can not actually be persisted +;; to the customise file that Emacs is currently using (since it comes +;; from the Nix store). +;; +;; The way this will work for now is that Emacs will *write* +;; configuration to the file tracked in my repository, while not +;; actually *reading* it from there (unless Emacs is rebuilt). +(setq custom-file (expand-file-name "~/depot/tools/emacs/config/custom.el")) +(load-library "custom") + +(defvar home-dir (expand-file-name "~")) + +;; Seed RNG +(random t) + +;; Load all other Emacs configuration. These configurations are +;; added to `load-path' by Nix. +(mapc 'require '(desktop + mail-setup + look-and-feel + functions + settings + modes + bindings + eshell-setup)) +(telephone-line-setup) +(ace-window-display-mode) + +;; If a local configuration library exists, it should be loaded. +;; +;; This can be provided by calling my Emacs derivation with +;; `withLocalConfig'. +(if-let (local-file (locate-library "local")) + (load local-file)) + +(require 'dottime) + +(provide 'init) diff --git a/users/tazjin/emacs/config/look-and-feel.el b/users/tazjin/emacs/config/look-and-feel.el new file mode 100644 index 000000000000..8cca6e1bf08e --- /dev/null +++ b/users/tazjin/emacs/config/look-and-feel.el @@ -0,0 +1,122 @@ +;;; -*- lexical-binding: t; -*- + +;; Hide those ugly tool bars: +(tool-bar-mode 0) +(scroll-bar-mode 0) +(menu-bar-mode 0) +(add-hook 'after-make-frame-functions + (lambda (frame) (scroll-bar-mode 0))) + +;; Don't do any annoying things: +(setq ring-bell-function 'ignore) +(setq initial-scratch-message "") + +;; Remember layout changes +(winner-mode 1) + +;; Usually emacs will run as a proper GUI application, in which case a few +;; extra settings are nice-to-have: +(when window-system + (setq frame-title-format '(buffer-file-name "%f" ("%b"))) + (mouse-wheel-mode t) + (blink-cursor-mode -1)) + +;; Configure Emacs fonts. +(let ((font (if (equal "frog" (s-trim (shell-command-to-string "hostname"))) + ;; For unclear reasons, frog refuses to render the + ;; regular font weight - everything ends up bold, + ;; which makes it hard to distinguish e.g. read/unread + ;; emails. + ;; + ;; Semi-bold looks a little different than on vauxhall + ;; and other machines, but it's alright. + (format "JetBrains Mono Semi Light-%d" 12) + (format "JetBrains Mono-%d" 12)))) + (setq default-frame-alist `((font . ,font))) + (set-frame-font font t t)) + +;; Configure telephone-line +(defun telephone-misc-if-last-window () + "Renders the mode-line-misc-info string for display in the + mode-line if the currently active window is the last one in the + frame. + + The idea is to not display information like the current time, + load, battery levels on all buffers." + + (when (bottom-right-window-p) + (telephone-line-raw mode-line-misc-info t))) + +(defun telephone-line-setup () + (telephone-line-defsegment telephone-line-last-window-segment () + (telephone-misc-if-last-window)) + + ;; Display the current EXWM workspace index in the mode-line + (telephone-line-defsegment telephone-line-exwm-workspace-index () + (when (bottom-right-window-p) + (format "[%s]" exwm-workspace-current-index))) + + ;; Define a highlight font for ~ important ~ information in the last + ;; window. + (defface special-highlight '((t (:foreground "white" :background "#5f627f"))) "") + (add-to-list 'telephone-line-faces + '(highlight . (special-highlight . special-highlight))) + + (setq telephone-line-lhs + '((nil . (telephone-line-position-segment)) + (accent . (telephone-line-buffer-segment)))) + + (setq telephone-line-rhs + '((accent . (telephone-line-major-mode-segment)) + (nil . (telephone-line-last-window-segment + telephone-line-exwm-workspace-index)) + + ;; TODO(tazjin): lets not do this particular thing while I + ;; don't actually run notmuch, there are too many things + ;; that have a dependency on the modeline drawing correctly + ;; (including randr operations!) + ;; + ;; (highlight . (telephone-line-notmuch-counts)) + )) + + (setq telephone-line-primary-left-separator 'telephone-line-tan-left + telephone-line-primary-right-separator 'telephone-line-tan-right + telephone-line-secondary-left-separator 'telephone-line-tan-hollow-left + telephone-line-secondary-right-separator 'telephone-line-tan-hollow-right) + + (telephone-line-mode 1)) + +;; Auto refresh buffers +(global-auto-revert-mode 1) + +;; Use clipboard properly +(setq select-enable-clipboard t) + +;; Show in-progress chords in minibuffer +(setq echo-keystrokes 0.1) + +;; Show column numbers in all buffers +(column-number-mode t) + +(defalias 'yes-or-no-p 'y-or-n-p) +(defalias 'auto-tail-revert-mode 'tail-mode) + +;; Style line numbers (shown with M-g g) +(setq linum-format + (lambda (line) + (propertize + (format (concat " %" + (number-to-string + (length (number-to-string + (line-number-at-pos (point-max))))) + "d ") + line) + 'face 'linum))) + +;; Display tabs as 2 spaces +(setq tab-width 2) + +;; Don't wrap around when moving between buffers +(setq windmove-wrap-around nil) + +(provide 'look-and-feel) diff --git a/users/tazjin/emacs/config/mail-setup.el b/users/tazjin/emacs/config/mail-setup.el new file mode 100644 index 000000000000..7fbece1b102a --- /dev/null +++ b/users/tazjin/emacs/config/mail-setup.el @@ -0,0 +1,85 @@ +(require 'notmuch) +(require 'counsel-notmuch) + +;; (global-set-key (kbd "C-c m") 'notmuch-hello) +;; (global-set-key (kbd "C-c C-m") 'counsel-notmuch) +;; (global-set-key (kbd "C-c C-e n") 'notmuch-mua-new-mail) + +(setq notmuch-cache-dir (format "%s/.cache/notmuch" (getenv "HOME"))) +(make-directory notmuch-cache-dir t) + +;; Cache addresses for completion: +(setq notmuch-address-save-filename (concat notmuch-cache-dir "/addresses")) + +;; Don't spam my home folder with drafts: +(setq notmuch-draft-folder "drafts") ;; relative to notmuch database + +;; Mark things as read when archiving them: +(setq notmuch-archive-tags '("-inbox" "-unread" "+archive")) + +;; Show me saved searches that I care about: +(setq notmuch-saved-searches + '((:name "inbox" :query "tag:inbox" :count-query "tag:inbox AND tag:unread" :key "i") + (:name "sent" :query "tag:sent" :key "t") + (:name "drafts" :query "tag:draft"))) +(setq notmuch-show-empty-saved-searches t) + +;; Mail sending configuration +(setq sendmail-program "gmi") ;; lieer binary supports sendmail emulation +(setq message-sendmail-extra-arguments + '("send" "--quiet" "-t" "-C" "~/mail/account.tazjin")) +(setq send-mail-function 'sendmail-send-it) +(setq notmuch-mua-user-agent-function + (lambda () (format "Emacs %s; notmuch.el %s" emacs-version notmuch-emacs-version))) +(setq mail-host-address (system-name)) +(setq notmuch-mua-cite-function #'message-cite-original-without-signature) +(setq notmuch-fcc-dirs nil) ;; Gmail does this server-side +(setq message-signature nil) ;; Insert message signature manually with C-c C-w + +;; Close mail buffers after sending mail +(setq message-kill-buffer-on-exit t) + +;; Ensure sender is correctly passed to msmtp +(setq mail-specify-envelope-from t + message-sendmail-envelope-from 'header + mail-envelope-from 'header) + +;; Store sent mail in the correct folder per account +(setq notmuch-maildir-use-notmuch-insert nil) + +;; I don't use drafts but I instinctively hit C-x C-s constantly, lets +;; handle that gracefully. +(define-key notmuch-message-mode-map (kbd "C-x C-s") #'ignore) + +;; Define a telephone-line segment for displaying the count of unread, +;; important mails in the last window's mode-line: +(defvar *last-notmuch-count-redraw* 0) +(defvar *current-notmuch-count* nil) + +(defun update-display-notmuch-counts () + "Update and render the current state of the notmuch unread + count for display in the mode-line. + + The offlineimap-timer runs every 2 minutes, so it does not make + sense to refresh this much more often than that." + + (when (> (- (float-time) *last-notmuch-count-redraw*) 30) + (setq *last-notmuch-count-redraw* (float-time)) + (let* ((inbox-unread (notmuch-saved-search-count "tag:inbox and tag:unread")) + (notmuch-count (format "I: %s; D: %s" inbox-unread))) + (setq *current-notmuch-count* notmuch-count))) + + (when (and (bottom-right-window-p) + ;; Only render if the initial update is done and there + ;; are unread mails: + *current-notmuch-count* + (not (equal *current-notmuch-count* "I: 0; D: 0"))) + *current-notmuch-count*)) + +(telephone-line-defsegment telephone-line-notmuch-counts () + "This segment displays the count of unread notmuch messages in + the last window's mode-line (if unread messages are present)." + + (update-display-notmuch-counts)) + +(provide 'mail-setup) diff --git a/users/tazjin/emacs/config/modes.el b/users/tazjin/emacs/config/modes.el new file mode 100644 index 000000000000..69fb523d0d91 --- /dev/null +++ b/users/tazjin/emacs/config/modes.el @@ -0,0 +1,37 @@ +;; Initializes modes I use. + +(add-hook 'prog-mode-hook 'esk-add-watchwords) +(add-hook 'prog-mode-hook 'hl-line-mode) + +;; Use auto-complete as completion at point +(defun set-auto-complete-as-completion-at-point-function () + (setq completion-at-point-functions '(auto-complete))) + +(add-hook 'auto-complete-mode-hook + 'set-auto-complete-as-completion-at-point-function) + +;; Enable rainbow-delimiters for all things programming +(add-hook 'prog-mode-hook 'rainbow-delimiters-mode) + +;; Enable Paredit & Company in Emacs Lisp mode +(add-hook 'emacs-lisp-mode-hook 'company-mode) + +;; Always highlight matching brackets +(show-paren-mode 1) + +;; Always auto-close parantheses and other pairs +(electric-pair-mode) + +;; Keep track of recent files +(recentf-mode) + +;; Easily navigate sillycased words +(global-subword-mode 1) + +;; Transparently open compressed files +(auto-compression-mode t) + +;; Configure go-mode for Go2 Alpha +(add-to-list 'auto-mode-alist '("\\.go2$" . go-mode)) + +(provide 'modes) diff --git a/users/tazjin/emacs/config/settings.el b/users/tazjin/emacs/config/settings.el new file mode 100644 index 000000000000..8b15b6cda183 --- /dev/null +++ b/users/tazjin/emacs/config/settings.el @@ -0,0 +1,48 @@ +(require 'uniquify) + +;; We don't live in the 80s, but we're also not a shitty web app. +(setq gc-cons-threshold 20000000) + +(setq uniquify-buffer-name-style 'forward) + +; Fix some defaults +(setq visible-bell nil + inhibit-startup-message t + color-theme-is-global t + sentence-end-double-space nil + shift-select-mode nil + uniquify-buffer-name-style 'forward + whitespace-style '(face trailing lines-tail tabs) + whitespace-line-column 80 + default-directory "~" + fill-column 80 + ediff-split-window-function 'split-window-horizontally + initial-major-mode 'emacs-lisp-mode) + +(add-to-list 'safe-local-variable-values '(lexical-binding . t)) +(add-to-list 'safe-local-variable-values '(whitespace-line-column . 80)) + +(set-default 'indent-tabs-mode nil) + +;; UTF-8 please +(setq locale-coding-system 'utf-8) ; pretty +(set-terminal-coding-system 'utf-8) ; pretty +(set-keyboard-coding-system 'utf-8) ; pretty +(set-selection-coding-system 'utf-8) ; please +(prefer-coding-system 'utf-8) ; with sugar on top + +;; Make emacs behave sanely (overwrite selected text) +(delete-selection-mode 1) + +;; Keep your temporary files in tmp, emacs! +(setq auto-save-file-name-transforms + `((".*" ,temporary-file-directory t))) +(setq backup-directory-alist + `((".*" . ,temporary-file-directory))) + +(remove-hook 'kill-buffer-query-functions 'server-kill-buffer-query-function) + +;; Show time in 24h format +(setq display-time-24hr-format t) + +(provide 'settings) diff --git a/users/tazjin/emacs/default.nix b/users/tazjin/emacs/default.nix new file mode 100644 index 000000000000..48805a42f220 --- /dev/null +++ b/users/tazjin/emacs/default.nix @@ -0,0 +1,135 @@ +# This file builds an Emacs pre-configured with the packages I need +# and my personal Emacs configuration. +# +# On NixOS machines, this Emacs currently does not support +# Imagemagick, see https://github.com/NixOS/nixpkgs/issues/70631. +# +# Forcing Emacs to link against Imagemagick currently causes libvterm +# to segfault, which is a lot less desirable than not having telega +# render images correctly. +{ lib, pkgs, ... }: + +pkgs.makeOverridable({ emacs ? pkgs.emacsGcc }: +let + emacsWithPackages = (pkgs.emacsPackagesGen emacs).emacsWithPackages; + + # $PATH for binaries that need to be available to Emacs + emacsBinPath = lib.makeBinPath [ pkgs.emacsPackages.telega ]; + + identity = x: x; + + tazjinsEmacs = pkgfun: (emacsWithPackages(epkgs: pkgfun(with epkgs; [ + ace-link + ace-window + avy + bazel + browse-kill-ring + cargo + clojure-mode + cmake-mode + company + counsel + counsel-notmuch + d-mode + direnv + dockerfile-mode + eglot + elfeed + elixir-mode + elm-mode + erlang + exwm + flymake + go-mode + google-c-style + gruber-darker-theme + haskell-mode + ht + hydra + idle-highlight-mode + ivy + ivy-prescient + jq-mode + kotlin-mode + lsp-mode + magit + markdown-toc + meson-mode + multi-term + multiple-cursors + nginx-mode + nix-mode + notmuch + paredit + password-store + pinentry + polymode + prescient + protobuf-mode + rainbow-delimiters + rainbow-mode + refine + request + restclient + rust-mode + sly + string-edit + swiper + telega + telephone-line + terraform-mode + toml-mode + transient + undo-tree + use-package + uuidgen + vterm + web-mode + websocket + which-key + xelb + yaml-mode + yasnippet + + # Custom depot packages (either ours, or overridden ones) + tvlPackages.dottime + tvlPackages.nix-util + tvlPackages.passively + tvlPackages.rcirc + tvlPackages.term-switcher + tvlPackages.tvl + ]))); +in lib.fix(self: l: f: pkgs.writeShellScriptBin "tazjins-emacs" '' + export PATH="${emacsBinPath}:$PATH" + exec ${tazjinsEmacs f}/bin/emacs \ + --debug-init \ + --no-site-file \ + --no-site-lisp \ + --no-init-file \ + --directory ${./config} ${if l != null then "--directory ${l}" else ""} \ + --eval "(require 'init)" $@ + '' // { + # Call overrideEmacs with a function (pkgs -> pkgs) to modify the + # packages that should be included in this Emacs distribution. + overrideEmacs = f': self l f'; + + # Call withLocalConfig with the path to a *folder* containing a + # `local.el` which provides local system configuration. + withLocalConfig = confDir: self confDir f; + + # Build a derivation that uses the specified local Emacs (i.e. + # built outside of Nix) instead + withLocalEmacs = emacsBin: pkgs.writeShellScriptBin "tazjins-emacs" '' + export PATH="${emacsBinPath}:$PATH" + export EMACSLOADPATH="${(tazjinsEmacs f).deps}/share/emacs/site-lisp:" + exec ${emacsBin} \ + --debug-init \ + --no-site-file \ + --no-site-lisp \ + --no-init-file \ + --directory ${./config} \ + ${if l != null then "--directory ${l}" else ""} \ + --eval "(require 'init)" $@ + ''; + }) null identity +) {} diff --git a/users/tazjin/finito/.gitignore b/users/tazjin/finito/.gitignore new file mode 100644 index 000000000000..548206b0b297 --- /dev/null +++ b/users/tazjin/finito/.gitignore @@ -0,0 +1,3 @@ +.envrc +/target/ +**/*.rs.bk diff --git a/users/tazjin/finito/Cargo.lock b/users/tazjin/finito/Cargo.lock new file mode 100644 index 000000000000..7427a6b11c42 --- /dev/null +++ b/users/tazjin/finito/Cargo.lock @@ -0,0 +1,773 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602d785912f476e480434627e8732e6766b760c045bbf897d9dfaa9f4fbd399c" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler32" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b077b825e468cc974f0020d4082ee6e03132512f207ef1a02fd5d00d1f32d" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "backtrace" +version = "0.3.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05100821de9e028f12ae3d189176b41ee198341eb8f369956407fea2f5cc666c" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96434f987501f0ed4eb336a411e0631ecd1afa11574fe148587adc4ff96143c9" +dependencies = [ + "byteorder", + "safemem", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "block-buffer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a076c298b9ecdb530ed9d967e74a6027d6a7478924520acddcddc24c1c8ab3ab" +dependencies = [ + "arrayref", + "byte-tools", +] + +[[package]] +name = "byte-tools" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "560c32574a12a89ecd91f5e742165893f86e3ab98d21f8ea548658eb9eef5f40" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "chrono" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2" +dependencies = [ + "num-integer", + "num-traits", + "time", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crypto-mac" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0999b4ff4d3446d4ddb19a63e9e00c1876e75cd7000d20e57a693b4b3f08d958" +dependencies = [ + "constant_time_eq", + "generic-array", +] + +[[package]] +name = "digest" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b072242a8cbaf9c145665af9d250c59af3b958f83ed6824e13533cf76d5b90" +dependencies = [ + "generic-array", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", + "synstructure", +] + +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + +[[package]] +name = "fallible-iterator" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb7217124812dc5672b7476d0c2d20cfe9f7c0f1ba0904b674a9762a0212f72e" + +[[package]] +name = "finito" +version = "0.1.0" +dependencies = [ + "serde", +] + +[[package]] +name = "finito-door" +version = "0.1.0" +dependencies = [ + "failure", + "finito", + "serde", + "serde_derive", +] + +[[package]] +name = "finito-postgres" +version = "0.1.0" +dependencies = [ + "chrono", + "finito", + "finito-door", + "postgres", + "postgres-derive", + "r2d2_postgres", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "generic-array" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25c5683767570c2bbd7deba372926a55eaae9982d7726ee2a1050239d45b9d" +dependencies = [ + "typenum", +] + +[[package]] +name = "gimli" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c" + +[[package]] +name = "hex" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" + +[[package]] +name = "hmac" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f3bdb08579d99d7dc761c0e266f13b5f2ab8c8c703b9fc9ef333cd8f48f55e" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itoa" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" + +[[package]] +name = "libc" +version = "0.2.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49" + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "md5" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48" + +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435" +dependencies = [ + "adler32", +] + +[[package]] +name = "num-integer" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + +[[package]] +name = "parking_lot" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +dependencies = [ + "cfg-if", + "cloudabi", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", +] + +[[package]] +name = "postgres" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115dde90ef51af573580c035857badbece2aa5cde3de1dfb3c932969ca92a6c5" +dependencies = [ + "bytes", + "fallible-iterator", + "log", + "postgres-protocol", + "postgres-shared", + "socket2", +] + +[[package]] +name = "postgres-derive" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44ef42ae50f1547dde36aa78d5e44189cbf21f4e77ce6ddc2bbaa068337fc221" +dependencies = [ + "quote 0.5.2", + "syn 0.13.11", +] + +[[package]] +name = "postgres-protocol" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2487e66455bf88a1b247bf08a3ce7fe5197ac6d67228d920b0ee6a0e97fd7312" +dependencies = [ + "base64", + "byteorder", + "bytes", + "fallible-iterator", + "generic-array", + "hmac", + "md5", + "memchr", + "rand 0.3.23", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-shared" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffac35b3e0029b404c24a3b82149b4e904f293e8ca4a327eefa24d3ca50df36f" +dependencies = [ + "chrono", + "fallible-iterator", + "hex", + "phf", + "postgres-protocol", + "serde_json", + "uuid", +] + +[[package]] +name = "proc-macro2" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa" +dependencies = [ + "unicode-xid 0.2.1", +] + +[[package]] +name = "quote" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" +dependencies = [ + "proc-macro2 0.3.8", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2 1.0.18", +] + +[[package]] +name = "r2d2" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1497e40855348e4a8a40767d8e55174bce1e445a3ac9254ad44ad468ee0485af" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_postgres" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c7fe9c0c3d2c298cf262bc3ce4b89cdf0eab620fd9fe759f65b34a1a00fb93" +dependencies = [ + "postgres", + "postgres-shared", + "r2d2", +] + +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0988d7fdf88d5e5fcf5923a0f1e8ab345f3e98ab4bc6bc45a2d5ff7f7458fbf6" +dependencies = [ + "parking_lot", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "serde" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" + +[[package]] +name = "serde_derive" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", +] + +[[package]] +name = "serde_json" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" +dependencies = [ + "block-buffer", + "byte-tools", + "digest", + "fake-simd", +] + +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + +[[package]] +name = "smallvec" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7cb5678e1615754284ec264d9bb5b4c27d2018577fd90ac0ceb578591ed5ee4" + +[[package]] +name = "socket2" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "syn" +version = "0.13.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f9bf6292f3a61d2c716723fdb789a41bbe104168e6f496dc6497e531ea1b9b" +dependencies = [ + "proc-macro2 0.3.8", + "quote 0.5.2", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "unicode-xid 0.2.1", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2 1.0.18", + "quote 1.0.7", + "syn 1.0.33", + "unicode-xid 0.2.1", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53953d2d3a5ad81d9f844a32f14ebb121f50b650cd59d0ee2a07cf13c617efed" + +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "uuid" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc7e3b898aa6f6c08e5295b6c89258d1331e9ac578cc992fb818759951bdc22" +dependencies = [ + "rand 0.3.23", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/users/tazjin/finito/Cargo.toml b/users/tazjin/finito/Cargo.toml new file mode 100644 index 000000000000..310133abeebb --- /dev/null +++ b/users/tazjin/finito/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] +members = [ + "finito-core", + "finito-door", + "finito-postgres" +] diff --git a/users/tazjin/finito/README.md b/users/tazjin/finito/README.md new file mode 100644 index 000000000000..5acd67d3bea7 --- /dev/null +++ b/users/tazjin/finito/README.md @@ -0,0 +1,27 @@ +Finito +====== + +This is a Rust port of the Haskell state-machine library Finito. It is +slightly less featureful because it loses the ability to ensure that +side-effects are contained and because of a slight reduction in +expressivity, which makes it a bit more restrictive. + +However, it still implements the FSM model well enough. + +# Components + +Finito is split up into multiple independent components (note: not all +of these exist yet), separating functionality related to FSM +persistence from other things. + +* `finito`: Core abstraction implemented by Finito +* `finito-door`: Example implementation of a simple, lockable door +* `finito-postgres`: Persistent state-machines using Postgres + +**Note**: The `finito` core library does not contain any tests. Its +coverage is instead provided by the `finito-door` library, which +actually implements an example FSM. + +These are split out because the documentation for `finito-door` is +interesting regardless and because other Finito packages also need an +example implementation. diff --git a/users/tazjin/finito/default.nix b/users/tazjin/finito/default.nix new file mode 100644 index 000000000000..e50ac32be452 --- /dev/null +++ b/users/tazjin/finito/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; +} diff --git a/users/tazjin/finito/finito-core/Cargo.toml b/users/tazjin/finito/finito-core/Cargo.toml new file mode 100644 index 000000000000..1d7bdb8b01fe --- /dev/null +++ b/users/tazjin/finito/finito-core/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "finito" +version = "0.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] + +[dependencies] +serde = "1.0" diff --git a/users/tazjin/finito/finito-core/src/lib.rs b/users/tazjin/finito/finito-core/src/lib.rs new file mode 100644 index 000000000000..517bfad2bc74 --- /dev/null +++ b/users/tazjin/finito/finito-core/src/lib.rs @@ -0,0 +1,243 @@ +//! Finito's core finite-state machine abstraction. +//! +//! # What & why? +//! +//! Most processes that occur in software applications can be modeled +//! as finite-state machines (FSMs), however the actual states, the +//! transitions between them and the model's interaction with the +//! external world is often implicit. +//! +//! Making the states of a process explicit using a simple language +//! that works for both software developers and other people who may +//! have opinions on processes makes it easier to synchronise thoughts, +//! extend software and keep a good level of control over what is going +//! on. +//! +//! This library aims to provide functionality for implementing +//! finite-state machines in a way that balances expressivity and +//! safety. +//! +//! Finito does not aim to prevent every possible incorrect +//! transition, but aims for somewhere "safe-enough" (please don't +//! lynch me) that is still easily understood. +//! +//! # Conceptual overview +//! +//! The core idea behind Finito can be expressed in a single line and +//! will potentially look familiar if you have used Erlang in a +//! previous life. The syntax used here is the type-signature notation +//! of Haskell. +//! +//! ```text +//! advance :: state -> event -> (state, [action]) +//! ``` +//! +//! In short, every FSM is made up of three distinct types: +//! +//! * a state type representing all possible states of the machine +//! +//! * an event type representing all possible events in the machine +//! +//! * an action type representing a description of all possible +//! side-effects of the machine +//! +//! Using the definition above we can now say that a transition in a +//! state-machine, involving these three types, takes an initial state +//! and an event to apply it to and returns a new state and a list of +//! actions to execute. +//! +//! With this definition most processes can already be modeled quite +//! well. Two additional functions are required to make it all work: +//! +//! ```text +//! -- | The ability to cause additional side-effects after entering +//! -- a new state. +//! > enter :: state -> [action] +//! ``` +//! +//! as well as +//! +//! ```text +//! -- | An interpreter for side-effects +//! act :: action -> m [event] +//! ``` +//! +//! **Note**: This library is based on an original Haskell library. In +//! Haskell, side-effects can be controlled via the type system which +//! is impossible in Rust. +//! +//! Some parts of Finito make assumptions about the programmer not +//! making certain kinds of mistakes, which are pointed out in the +//! documentation. Unfortunately those assumptions are not +//! automatically verifiable in Rust. +//! +//! ## Example +//! +//! Please consult `finito-door` for an example representing a simple, +//! lockable door as a finite-state machine. This gives an overview +//! over Finito's primary features. +//! +//! If you happen to be the kind of person who likes to learn about +//! libraries by reading code, you should familiarise yourself with the +//! door as it shows up as the example in other finito-related +//! libraries, too. +//! +//! # Persistence, side-effects and mud +//! +//! These three things are inescapable in the fateful realm of +//! computers, but Finito separates them out into separate libraries +//! that you can drag in as you need them. +//! +//! Currently, those libraries include: +//! +//! * `finito`: Core components and classes of Finito +//! +//! * `finito-in-mem`: In-memory implementation of state machines +//! that do not need to live longer than an application using +//! standard library concurrency primitives. +//! +//! * `finito-postgres`: Postgres-backed, persistent implementation +//! of state machines that, well, do need to live longer. Uses +//! Postgres for concurrency synchronisation, so keep that in +//! mind. +//! +//! Which should cover most use-cases. Okay, enough prose, lets dive +//! in. +//! +//! # Does Finito make you want to scream? +//! +//! Please reach out! I want to know why! + +extern crate serde; + +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::fmt::Debug; +use std::mem; + +/// Primary trait that needs to be implemented for every state type +/// representing the states of an FSM. +/// +/// This trait is used to implement transition logic and to "tie the +/// room together", with the room being our triplet of types. +pub trait FSM where Self: Sized { + /// A human-readable string uniquely describing what this FSM + /// models. This is used in log messages, database tables and + /// various other things throughout Finito. + const FSM_NAME: &'static str; + + /// The associated event type of an FSM represents all possible + /// events that can occur in the state-machine. + type Event; + + /// The associated action type of an FSM represents all possible + /// actions that can occur in the state-machine. + type Action; + + /// The associated error type of an FSM represents failures that + /// can occur during action processing. + type Error: Debug; + + /// The associated state type of an FSM describes the state that + /// is made available to the implementation of action + /// interpretations. + type State; + + /// `handle` deals with any incoming events to cause state + /// transitions and emit actions. This function is the core logic + /// of any state machine. + /// + /// Implementations of this function **must not** cause any + /// side-effects to avoid breaking the guarantees of Finitos + /// conceptual model. + fn handle(self, event: Self::Event) -> (Self, Vec<Self::Action>); + + /// `enter` is called when a new state is entered, allowing a + /// state to produce additional side-effects. + /// + /// This is useful for side-effects that event handlers do not + /// need to know about and for resting assured that a certain + /// action has been caused when a state is entered. + /// + /// FSM state types are expected to be enum (i.e. sum) types. A + /// state is considered "new" and enter calls are run if is of a + /// different enum variant. + fn enter(&self) -> Vec<Self::Action>; + + /// `act` interprets and executes FSM actions. This is the only + /// part of an FSM in which side-effects are allowed. + fn act(Self::Action, &Self::State) -> Result<Vec<Self::Event>, Self::Error>; +} + +/// This function is the primary function used to advance a state +/// machine. It takes care of both running the event handler as well +/// as possible state-enter calls and returning the result. +/// +/// Users of Finito should basically always use this function when +/// advancing state-machines manually, and never call FSM-trait +/// methods directly. +pub fn advance<S: FSM>(state: S, event: S::Event) -> (S, Vec<S::Action>) { + // Determine the enum variant of the initial state (used to + // trigger enter calls). + let old_discriminant = mem::discriminant(&state); + + let (new_state, mut actions) = state.handle(event); + + // Compare the enum variant of the resulting state to the old one + // and run `enter` if they differ. + let new_discriminant = mem::discriminant(&new_state); + let mut enter_actions = if old_discriminant != new_discriminant { + new_state.enter() + } else { + vec![] + }; + + actions.append(&mut enter_actions); + + (new_state, actions) +} + +/// This trait is implemented by Finito backends. Backends are +/// expected to be able to keep track of the current state of an FSM +/// and retrieve it / apply updates transactionally. +/// +/// See the `finito-postgres` and `finito-in-mem` crates for example +/// implementations of this trait. +/// +/// Backends must be parameterised over an additional (user-supplied) +/// state type which can be used to track application state that must +/// be made available to action handlers, for example to pass along +/// database connections. +pub trait FSMBackend<S: 'static> { + /// Key type used to identify individual state machines in this + /// backend. + /// + /// TODO: Should be parameterised over FSM type after rustc + /// #44265. + type Key; + + /// Error type for all potential failures that can occur when + /// interacting with this backend. + type Error: Debug; + + /// Insert a new state-machine into the backend's storage and + /// return its newly allocated key. + fn insert_machine<F>(&self, initial: F) -> Result<Self::Key, Self::Error> + where F: FSM + Serialize + DeserializeOwned; + + /// Retrieve the current state of an FSM by its key. + fn get_machine<F: FSM>(&self, key: Self::Key) -> Result<F, Self::Error> + where F: FSM + Serialize + DeserializeOwned; + + /// Advance a state machine by applying an event and persisting it + /// as well as any resulting actions. + /// + /// **Note**: Whether actions are automatically executed depends + /// on the backend used. Please consult the backend's + /// documentation for details. + fn advance<'a, F: FSM>(&'a self, key: Self::Key, event: F::Event) -> Result<F, Self::Error> + where F: FSM + Serialize + DeserializeOwned, + F::State: From<&'a S>, + F::Event: Serialize + DeserializeOwned, + F::Action: Serialize + DeserializeOwned; +} diff --git a/users/tazjin/finito/finito-door/Cargo.toml b/users/tazjin/finito/finito-door/Cargo.toml new file mode 100644 index 000000000000..32c0a5a7c4ef --- /dev/null +++ b/users/tazjin/finito/finito-door/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "finito-door" +version = "0.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] + +[dependencies] +failure = "0.1" +serde = "1.0" +serde_derive = "1.0" + +[dependencies.finito] +path = "../finito-core" diff --git a/users/tazjin/finito/finito-door/src/lib.rs b/users/tazjin/finito/finito-door/src/lib.rs new file mode 100644 index 000000000000..68542c0bc448 --- /dev/null +++ b/users/tazjin/finito/finito-door/src/lib.rs @@ -0,0 +1,327 @@ +//! Example implementation of a lockable door in Finito +//! +//! # What & why? +//! +//! This module serves as a (hopefully simple) example of how to +//! implement finite-state machines using Finito. Note that the +//! concepts of Finito itself won't be explained in detail here, +//! consult its library documentation for that. +//! +//! Reading through this module should give you a rough idea of how to +//! work with Finito and get you up and running modeling things +//! *quickly*. +//! +//! Note: The generated documentation for this module will display the +//! various components of the door, but it will not inform you about +//! the actual transition logic and all that stuff. Read the source, +//! too! +//! +//! # The Door +//! +//! My favourite example when explaining these state-machines +//! conceptually has been to use a simple, lockable door. Our door has +//! a keypad next to it which can be used to lock the door by entering +//! a code, after which the same code must be entered to unlock it +//! again. +//! +//! The door can only be locked if it is closed. Oh, and it has a few +//! extra features: +//! +//! * whenever the door's state changes, an IRC channel receives a +//! message about that +//! +//! * the door calls the police if the code is intered incorrectly more +//! than a specified number of times (mhm, lets say, three) +//! +//! * if the police is called the door can not be interacted with +//! anymore (and honestly, for the sake of this example, we don't +//! care how its functionality is restored) +//! +//! ## The Door - Visualized +//! +//! Here's a rough attempt at drawing a state diagram in ASCII. The +//! bracketed words denote states, the arrows denote events: +//! +//! ```text +//! <--Open--- <--Unlock-- correct code? --Unlock--> +//! [Opened] [Closed] [Locked] [Disabled] +//! --Close--> ----Lock--> +//! ``` +//! +//! I'm so sorry for that drawing. +//! +//! ## The Door - Usage example +//! +//! An interaction session with our final door could look like this: +//! +//! ```rust,ignore +//! use finito_postgres::{insert_machine, advance}; +//! +//! let door = insert_machine(&conn, &DoorState::Opened)?; +//! +//! advance(&conn, &door, DoorEvent::Close)?; +//! advance(&conn, &door, DoorEvent::Lock(1337))?; +//! +//! format!("Door is now: {}", get_machine(&conn, &door)?); +//! ``` +//! +//! Here we have created, closed and then locked a door and inspected +//! its state. We will see that it is locked, has the locking code we +//! gave it and three remaining attempts to open it. +//! +//! Alright, enough foreplay, lets dive in! + +#[macro_use] extern crate serde_derive; + +extern crate failure; +extern crate finito; + +use finito::FSM; + +/// Type synonym to represent the code with which the door is locked. This +/// exists only for clarity in the signatures below and please do not email me +/// about the fact that an integer is not actually a good representation of +/// numerical digits. Thanks! +type Code = usize; + +/// Type synonym to represent the remaining number of unlock attempts. +type Attempts = usize; + +/// This type represents the possible door states and the data that they carry. +/// We can infer this from the "diagram" in the documentation above. +/// +/// This type is the one for which `finito::FSM` will be implemented, making it +/// the wooden (?) heart of our door. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub enum DoorState { + /// In `Opened` state, the door is wide open and anyone who fits through can + /// go through. + Opened, + + /// In `Closed` state, the door is shut but does not prevent anyone from + /// opening it. + Closed, + + /// In `Locked` state, the door is locked and waiting for someone to enter + /// its locking code on the keypad. + /// + /// This state contains the code that the door is locked with, as well as + /// the remaining number of attempts before the door calls the police and + /// becomes unusable. + Locked { code: Code, attempts: Attempts }, + + /// This state represents a disabled door after the police has been called. + /// The police will need to unlock it manually! + Disabled, +} + +/// This type represents the events that can occur in our door, i.e. the input +/// and interactions it receives. +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub enum DoorEvent { + /// `Open` means someone is opening the door! + Open, + + /// `Close` means, you guessed it, the exact opposite. + Close, + + /// `Lock` means somebody has entered a locking code on the + /// keypad. + Lock(Code), + + /// `Unlock` means someone has attempted to unlock the door. + Unlock(Code), +} + +/// This type represents the possible actions, a.k.a. everything our door "does" +/// that does not just impact itself, a.k.a. side-effects. +/// +/// **Note**: This type by itself *is not* a collection of side-effects, it +/// merely describes the side-effects we want to occur (which are then +/// interpreted by the machinery later). +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub enum DoorAction { + /// `NotifyIRC` is used to display some kind of message on the + /// aforementioned IRC channel that is, for some reason, very interested in + /// the state of the door. + NotifyIRC(String), + + /// `CallThePolice` does what you think it does. + /// + /// **Note**: For safety reasons, causing this action is not recommended for + /// users inside the US! + CallThePolice, +} + +/// This trait implementation turns our 'DoorState' into a type actually +/// representing a finite-state machine. To implement it, we need to do three +/// main things: +/// +/// * Define what our associated `Event` and `Action` type should be +/// +/// * Define the event-handling and state-entering logic (i.e. the meat of the +/// ... door) +/// +/// * Implement the interpretation of our actions, i.e. implement actual +/// side-effects +impl FSM for DoorState { + const FSM_NAME: &'static str = "door"; + + // As you might expect, our `Event` type is 'DoorEvent' and our `Action` + // type is 'DoorAction'. + type Event = DoorEvent; + type Action = DoorAction; + type State = (); + + // For error handling, the door simply uses `failure` which provides a + // generic, chainable error type. In real-world implementations you may want + // to use a custom error type or similar. + type Error = failure::Error; + + // The implementation of `handle` provides us with the actual transition + // logic of the door. + // + // The door is conceptually not that complicated so it is relatively short. + fn handle(self, event: DoorEvent) -> (Self, Vec<DoorAction>) { + match (self, event) { + // An opened door can be closed: + (DoorState::Opened, DoorEvent::Close) => return (DoorState::Closed, vec![]), + + // A closed door can be opened: + (DoorState::Closed, DoorEvent::Open) => return (DoorState::Opened, vec![]), + + // A closed door can also be locked, in which case the locking code + // is stored with the next state and the unlock attempts default to + // three: + (DoorState::Closed, DoorEvent::Lock(code)) => { + return (DoorState::Locked { code, attempts: 3 }, vec![]) + } + + // A locked door receiving an `Unlock`-event can do several + // different things ... + (DoorState::Locked { code, attempts }, DoorEvent::Unlock(unlock_code)) => { + // In the happy case, entry of a correct code leads to the door + // becoming unlocked (i.e. transitioning back to `Closed`). + if code == unlock_code { + return (DoorState::Closed, vec![]); + } + + // If the code wasn't correct and the fraudulent unlocker ran + // out of attempts (i.e. there was only one attempt remaining), + // it's time for some consequences. + if attempts == 1 { + return (DoorState::Disabled, vec![DoorAction::CallThePolice]); + } + + // If the code wasn't correct, but there are still some + // remaining attempts, the user doesn't have to face the police + // quite yet but IRC gets to laugh about it. + return ( + DoorState::Locked { + code, + attempts: attempts - 1, + }, + vec![DoorAction::NotifyIRC("invalid code entered".into())], + ); + } + + // This actually already concludes our event-handling logic. Our + // uncaring door does absolutely nothing if you attempt to do + // something with it that it doesn't support, so the last handler is + // a simple fallback. + // + // In a real-world state machine, especially one that receives + // events from external sources, you may want fallback handlers to + // actually do something. One example could be creating an action + // that logs information about unexpected events, alerts a + // monitoring service, or whatever else. + (current, _) => (current, vec![]), + } + } + + // The implementation of `enter` lets door states cause additional actions + // they are transitioned to. In the door example we use this only to notify + // IRC about what is going on. + fn enter(&self) -> Vec<DoorAction> { + let msg = match self { + DoorState::Opened => "door was opened", + DoorState::Closed => "door was closed", + DoorState::Locked { .. } => "door was locked", + DoorState::Disabled => "door was disabled", + }; + + vec![DoorAction::NotifyIRC(msg.into())] + } + + // The implementation of `act` lets us perform actual side-effects. + // + // Again, for the sake of educational simplicity, this does not deal with + // all potential (or in fact any) error cases that can occur during this toy + // implementation of actions. + // + // Additionally the `act` function can return new events. This is useful for + // a sort of "callback-like" pattern (cause an action to fetch some data, + // receive it as an event) but is not used in this example. + fn act(action: DoorAction, _state: &()) -> Result<Vec<DoorEvent>, failure::Error> { + match action { + DoorAction::NotifyIRC(msg) => { + use std::fs::OpenOptions; + use std::io::Write; + + let mut file = OpenOptions::new() + .append(true) + .create(true) + .open("/tmp/door-irc.log")?; + + write!(file, "<doorbot> {}\n", msg)?; + Ok(vec![]) + } + + DoorAction::CallThePolice => { + // TODO: call the police + println!("The police was called! For real!"); + Ok(vec![]) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use finito::advance; + + fn test_fsm<S: FSM>(initial: S, events: Vec<S::Event>) -> (S, Vec<S::Action>) { + events.into_iter().fold((initial, vec![]), |(state, mut actions), event| { + let (new_state, mut new_actions) = advance(state, event); + actions.append(&mut new_actions); + (new_state, actions) + }) + } + + #[test] + fn test_door() { + let initial = DoorState::Opened; + let events = vec![ + DoorEvent::Close, + DoorEvent::Open, + DoorEvent::Close, + DoorEvent::Lock(1234), + DoorEvent::Unlock(1234), + DoorEvent::Lock(4567), + DoorEvent::Unlock(1234), + ]; + let (final_state, actions) = test_fsm(initial, events); + + assert_eq!(final_state, DoorState::Locked { code: 4567, attempts: 2 }); + assert_eq!(actions, vec![ + DoorAction::NotifyIRC("door was closed".into()), + DoorAction::NotifyIRC("door was opened".into()), + DoorAction::NotifyIRC("door was closed".into()), + DoorAction::NotifyIRC("door was locked".into()), + DoorAction::NotifyIRC("door was closed".into()), + DoorAction::NotifyIRC("door was locked".into()), + DoorAction::NotifyIRC("invalid code entered".into()), + ]); + } +} diff --git a/users/tazjin/finito/finito-postgres/Cargo.toml b/users/tazjin/finito/finito-postgres/Cargo.toml new file mode 100644 index 000000000000..dd8d1d000304 --- /dev/null +++ b/users/tazjin/finito/finito-postgres/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "finito-postgres" +version = "0.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] + +[dependencies] +chrono = "0.4" +postgres-derive = "0.3" +serde = "1.0" +serde_json = "1.0" +r2d2_postgres = "0.14" + +[dependencies.postgres] +version = "0.15" +features = [ "with-uuid", "with-chrono", "with-serde_json" ] + +[dependencies.uuid] +version = "0.5" +features = [ "v4" ] + +[dependencies.finito] +path = "../finito-core" + +[dev-dependencies.finito-door] +path = "../finito-door" diff --git a/users/tazjin/finito/finito-postgres/migrations/2018-09-26-160621_bootstrap_finito_schema/down.sql b/users/tazjin/finito/finito-postgres/migrations/2018-09-26-160621_bootstrap_finito_schema/down.sql new file mode 100644 index 000000000000..9b56f9d35abe --- /dev/null +++ b/users/tazjin/finito/finito-postgres/migrations/2018-09-26-160621_bootstrap_finito_schema/down.sql @@ -0,0 +1,4 @@ +DROP TABLE actions; +DROP TYPE ActionStatus; +DROP TABLE events; +DROP TABLE machines; diff --git a/users/tazjin/finito/finito-postgres/migrations/2018-09-26-160621_bootstrap_finito_schema/up.sql b/users/tazjin/finito/finito-postgres/migrations/2018-09-26-160621_bootstrap_finito_schema/up.sql new file mode 100644 index 000000000000..18ace393b8d9 --- /dev/null +++ b/users/tazjin/finito/finito-postgres/migrations/2018-09-26-160621_bootstrap_finito_schema/up.sql @@ -0,0 +1,37 @@ +-- Creates the initial schema required by finito-postgres. + +CREATE TABLE machines ( + id UUID PRIMARY KEY, + created TIMESTAMPTZ NOT NULL DEFAULT NOW(), + fsm TEXT NOT NULL, + state JSONB NOT NULL +); + +CREATE TABLE events ( + id UUID PRIMARY KEY, + created TIMESTAMPTZ NOT NULL DEFAULT NOW(), + fsm TEXT NOT NULL, + fsm_id UUID NOT NULL REFERENCES machines(id), + event JSONB NOT NULL +); +CREATE INDEX idx_events_machines ON events(fsm_id); + +CREATE TYPE ActionStatus AS ENUM ( + 'Pending', + 'Completed', + 'Failed' +); + +CREATE TABLE actions ( + id UUID PRIMARY KEY, + created TIMESTAMPTZ NOT NULL DEFAULT NOW(), + fsm TEXT NOT NULL, + fsm_id UUID NOT NULL REFERENCES machines(id), + event_id UUID NOT NULL REFERENCES events(id), + content JSONB NOT NULL, + status ActionStatus NOT NULL, + error TEXT +); + +CREATE INDEX idx_actions_machines ON actions(fsm_id); +CREATE INDEX idx_actions_events ON actions(event_id); diff --git a/users/tazjin/finito/finito-postgres/src/error.rs b/users/tazjin/finito/finito-postgres/src/error.rs new file mode 100644 index 000000000000..e130d18361f1 --- /dev/null +++ b/users/tazjin/finito/finito-postgres/src/error.rs @@ -0,0 +1,109 @@ +//! This module defines error types and conversions for issue that can +//! occur while dealing with persisted state machines. + +use std::result; +use std::fmt; +use uuid::Uuid; +use std::error::Error as StdError; + +// errors to chain: +use postgres::Error as PgError; +use r2d2_postgres::r2d2::Error as PoolError; +use serde_json::Error as JsonError; + +pub type Result<T> = result::Result<T, Error>; + +#[derive(Debug)] +pub struct Error { + pub kind: ErrorKind, + pub context: Option<String>, +} + +#[derive(Debug)] +pub enum ErrorKind { + /// Errors occuring during JSON serialization of FSM types. + Serialization(String), + + /// Errors occuring during communication with the database. + Database(String), + + /// Errors with the database connection pool. + DBPool(String), + + /// State machine could not be found. + FSMNotFound(Uuid), + + /// Action could not be found. + ActionNotFound(Uuid), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use ErrorKind::*; + let msg = match &self.kind { + Serialization(err) => + format!("JSON serialization error: {}", err), + + Database(err) => + format!("PostgreSQL error: {}", err), + + DBPool(err) => + format!("Database connection pool error: {}", err), + + FSMNotFound(id) => + format!("FSM with ID {} not found", id), + + ActionNotFound(id) => + format!("Action with ID {} not found", id), + }; + + match &self.context { + None => write!(f, "{}", msg), + Some(ctx) => write!(f, "{}: {}", ctx, msg), + } + } +} + +impl StdError for Error {} + +impl <E: Into<ErrorKind>> From<E> for Error { + fn from(err: E) -> Error { + Error { + kind: err.into(), + context: None, + } + } +} + +impl From<JsonError> for ErrorKind { + fn from(err: JsonError) -> ErrorKind { + ErrorKind::Serialization(err.to_string()) + } +} + +impl From<PgError> for ErrorKind { + fn from(err: PgError) -> ErrorKind { + ErrorKind::Database(err.to_string()) + } +} + +impl From<PoolError> for ErrorKind { + fn from(err: PoolError) -> ErrorKind { + ErrorKind::DBPool(err.to_string()) + } +} + +/// Helper trait that makes it possible to supply contextual +/// information with an error. +pub trait ResultExt<T> { + fn context<C: fmt::Display>(self, ctx: C) -> Result<T>; +} + +impl <T, E: Into<Error>> ResultExt<T> for result::Result<T, E> { + fn context<C: fmt::Display>(self, ctx: C) -> Result<T> { + self.map_err(|err| Error { + context: Some(format!("{}", ctx)), + .. err.into() + }) + } +} diff --git a/users/tazjin/finito/finito-postgres/src/lib.rs b/users/tazjin/finito/finito-postgres/src/lib.rs new file mode 100644 index 000000000000..ae147f751f88 --- /dev/null +++ b/users/tazjin/finito/finito-postgres/src/lib.rs @@ -0,0 +1,431 @@ +//! PostgreSQL-backed persistence for Finito state machines +//! +//! This module implements ... TODO when I can write again. +//! +//! TODO: events & actions should have `SERIAL` keys + +#[macro_use] extern crate postgres; +#[macro_use] extern crate postgres_derive; + +extern crate chrono; +extern crate finito; +extern crate r2d2_postgres; +extern crate serde; +extern crate serde_json; +extern crate uuid; + +#[cfg(test)] mod tests; +#[cfg(test)] extern crate finito_door; + +mod error; +pub use error::{Result, Error, ErrorKind}; + +use chrono::prelude::{DateTime, Utc}; +use error::ResultExt; +use finito::{FSM, FSMBackend}; +use postgres::transaction::Transaction; +use postgres::GenericConnection; +use serde::Serialize; +use serde::de::DeserializeOwned; +use serde_json::Value; +use std::marker::PhantomData; +use uuid::Uuid; +use r2d2_postgres::{r2d2, PostgresConnectionManager}; + +type DBPool = r2d2::Pool<PostgresConnectionManager>; +type DBConn = r2d2::PooledConnection<PostgresConnectionManager>; + +/// This struct represents rows in the database table in which events +/// are persisted. +#[derive(Debug, ToSql, FromSql)] +struct EventT { + /// ID of the persisted event. + id: Uuid, + + /// Timestamp at which the event was stored. + created: DateTime<Utc>, + + /// Name of the type of FSM that this state belongs to. + fsm: String, + + /// ID of the state machine belonging to this event. + fsm_id: Uuid, + + /// Serialised content of the event. + event: Value, +} + +/// This enum represents the possible statuses an action can be in. +#[derive(Debug, PartialEq, ToSql, FromSql)] +#[postgres(name = "actionstatus")] +enum ActionStatus { + /// The action was requested but has not run yet. + Pending, + + /// The action completed successfully. + Completed, + + /// The action failed to run. Information about the error will + /// have been persisted in Postgres. + Failed, +} + +/// This struct represents rows in the database table in which actions +/// are persisted. +#[derive(Debug, ToSql, FromSql)] +struct ActionT { + /// ID of the persisted event. + id: Uuid, + + /// Timestamp at which the event was stored. + created: DateTime<Utc>, + + /// Name of the type of FSM that this state belongs to. + fsm: String, + + /// ID of the state machine belonging to this event. + fsm_id: Uuid, + + /// ID of the event that resulted in this action. + event_id: Uuid, + + /// Serialised content of the action. + #[postgres(name = "content")] // renamed because 'action' is a keyword in PG + action: Value, + + /// Current status of the action. + status: ActionStatus, + + /// Detailed (i.e. Debug-trait formatted) error message, if an + /// error occured during action processing. + error: Option<String>, +} + +// The following functions implement the public interface of +// `finito-postgres`. + +/// TODO: Write docs for this type, brain does not want to do it right +/// now. +pub struct FinitoPostgres<S> { + state: S, + + db_pool: DBPool, +} + +impl <S> FinitoPostgres<S> { + pub fn new(state: S, db_pool: DBPool, _pool_size: usize) -> Self { + FinitoPostgres { + state, db_pool, + } + } +} + +impl <State: 'static> FSMBackend<State> for FinitoPostgres<State> { + type Key = Uuid; + type Error = Error; + + fn insert_machine<S: FSM + Serialize>(&self, initial: S) -> Result<Uuid> { + let query = r#" + INSERT INTO machines (id, fsm, state) + VALUES ($1, $2, $3) + "#; + + let id = Uuid::new_v4(); + let fsm = S::FSM_NAME.to_string(); + let state = serde_json::to_value(initial).context("failed to serialise FSM")?; + + self.conn()?.execute(query, &[&id, &fsm, &state]).context("failed to insert FSM")?; + + return Ok(id); + + } + + fn get_machine<S: FSM + DeserializeOwned>(&self, key: Uuid) -> Result<S> { + get_machine_internal(&*self.conn()?, key, false) + } + + /// Advance a persisted state machine by applying an event, and + /// storing the event as well as all resulting actions. + /// + /// This function holds a database-lock on the state's row while + /// advancing the machine. + /// + /// **Note**: This function returns the new state of the machine + /// immediately after applying the event, however this does not + /// necessarily equate to the state of the machine after all related + /// processing is finished as running actions may result in additional + /// transitions. + fn advance<'a, S>(&'a self, key: Uuid, event: S::Event) -> Result<S> + where S: FSM + Serialize + DeserializeOwned, + S::State: From<&'a State>, + S::Event: Serialize + DeserializeOwned, + S::Action: Serialize + DeserializeOwned { + let conn = self.conn()?; + let tx = conn.transaction().context("could not begin transaction")?; + let state = get_machine_internal(&tx, key, true)?; + + // Advancing the FSM consumes the event, so it is persisted first: + let event_id = insert_event::<_, S>(&tx, key, &event)?; + + // Core advancing logic is run: + let (new_state, actions) = finito::advance(state, event); + + // Resulting actions are persisted (TODO: and interpreted) + let mut action_ids = vec![]; + for action in actions { + let action_id = insert_action::<_, S>(&tx, key, event_id, &action)?; + action_ids.push(action_id); + } + + // And finally the state is updated: + update_state(&tx, key, &new_state)?; + tx.commit().context("could not commit transaction")?; + + self.run_actions::<S>(key, action_ids); + + Ok(new_state) + } +} + +impl <State: 'static> FinitoPostgres<State> { + /// Execute several actions at the same time, each in a separate + /// thread. Note that actions returning further events, causing + /// further transitions, returning further actions and so on will + /// potentially cause multiple threads to get created. + fn run_actions<'a, S>(&'a self, fsm_id: Uuid, action_ids: Vec<Uuid>) where + S: FSM + Serialize + DeserializeOwned, + S::Event: Serialize + DeserializeOwned, + S::Action: Serialize + DeserializeOwned, + S::State: From<&'a State> { + let state: S::State = (&self.state).into(); + let conn = self.conn().expect("TODO"); + + for action_id in action_ids { + let tx = conn.transaction().expect("TODO"); + + // TODO: Determine which concurrency setup we actually want. + if let Ok(events) = run_action(tx, action_id, &state, PhantomData::<S>) { + for event in events { + self.advance::<S>(fsm_id, event).expect("TODO"); + } + } + } + } + + /// Retrieve a single connection from the database connection pool. + fn conn(&self) -> Result<DBConn> { + self.db_pool.get().context("failed to retrieve connection from pool") + } +} + + + +/// Insert a single state-machine into the database and return its +/// newly allocated, random UUID. +pub fn insert_machine<C, S>(conn: &C, initial: S) -> Result<Uuid> where + C: GenericConnection, + S: FSM + Serialize { + let query = r#" + INSERT INTO machines (id, fsm, state) + VALUES ($1, $2, $3) + "#; + + let id = Uuid::new_v4(); + let fsm = S::FSM_NAME.to_string(); + let state = serde_json::to_value(initial).context("failed to serialize FSM")?; + + conn.execute(query, &[&id, &fsm, &state])?; + + return Ok(id); +} + +/// Insert a single event into the database and return its UUID. +fn insert_event<C, S>(conn: &C, + fsm_id: Uuid, + event: &S::Event) -> Result<Uuid> +where + C: GenericConnection, + S: FSM, + S::Event: Serialize { + let query = r#" + INSERT INTO events (id, fsm, fsm_id, event) + VALUES ($1, $2, $3, $4) + "#; + + let id = Uuid::new_v4(); + let fsm = S::FSM_NAME.to_string(); + let event_value = serde_json::to_value(event) + .context("failed to serialize event")?; + + conn.execute(query, &[&id, &fsm, &fsm_id, &event_value])?; + return Ok(id) +} + +/// Insert a single action into the database and return its UUID. +fn insert_action<C, S>(conn: &C, + fsm_id: Uuid, + event_id: Uuid, + action: &S::Action) -> Result<Uuid> where + C: GenericConnection, + S: FSM, + S::Action: Serialize { + let query = r#" + INSERT INTO actions (id, fsm, fsm_id, event_id, content, status) + VALUES ($1, $2, $3, $4, $5, $6) + "#; + + let id = Uuid::new_v4(); + let fsm = S::FSM_NAME.to_string(); + let action_value = serde_json::to_value(action) + .context("failed to serialize action")?; + + conn.execute( + query, + &[&id, &fsm, &fsm_id, &event_id, &action_value, &ActionStatus::Pending] + )?; + + return Ok(id) +} + +/// Update the state of a specified machine. +fn update_state<C, S>(conn: &C, + fsm_id: Uuid, + state: &S) -> Result<()> where + C: GenericConnection, + S: FSM + Serialize { + let query = r#" + UPDATE machines SET state = $1 WHERE id = $2 + "#; + + let state_value = serde_json::to_value(state).context("failed to serialize FSM")?; + let res_count = conn.execute(query, &[&state_value, &fsm_id])?; + + if res_count != 1 { + Err(ErrorKind::FSMNotFound(fsm_id).into()) + } else { + Ok(()) + } +} + +/// Conditionally alter SQL statement to append locking clause inside +/// of a transaction. +fn alter_for_update(alter: bool, query: &str) -> String { + match alter { + false => query.to_string(), + true => format!("{} FOR UPDATE", query), + } +} + +/// Retrieve the current state of a state machine from the database, +/// optionally locking the machine state for the duration of some +/// enclosing transaction. +fn get_machine_internal<C, S>(conn: &C, + id: Uuid, + for_update: bool) -> Result<S> where + C: GenericConnection, + S: FSM + DeserializeOwned { + let query = alter_for_update(for_update, r#" + SELECT state FROM machines WHERE id = $1 + "#); + + let rows = conn.query(&query, &[&id]).context("failed to retrieve FSM")?; + + if let Some(row) = rows.into_iter().next() { + Ok(serde_json::from_value(row.get(0)).context("failed to deserialize FSM")?) + } else { + Err(ErrorKind::FSMNotFound(id).into()) + } +} + +/// Retrieve an action from the database, optionally locking it for +/// the duration of some enclosing transaction. +fn get_action<C, S>(conn: &C, id: Uuid) -> Result<(ActionStatus, S::Action)> where + C: GenericConnection, + S: FSM, + S::Action: DeserializeOwned { + let query = alter_for_update(true, r#" + SELECT status, content FROM actions + WHERE id = $1 AND fsm = $2 + "#); + + let rows = conn.query(&query, &[&id, &S::FSM_NAME])?; + + if let Some(row) = rows.into_iter().next() { + let action = serde_json::from_value(row.get(1)) + .context("failed to deserialize FSM action")?; + Ok((row.get(0), action)) + } else { + Err(ErrorKind::ActionNotFound(id).into()) + } +} + +/// Update the status of an action after an attempt to run it. +fn update_action_status<C, S>(conn: &C, + id: Uuid, + status: ActionStatus, + error: Option<String>, + _fsm: PhantomData<S>) -> Result<()> where + C: GenericConnection, + S: FSM { + let query = r#" + UPDATE actions SET status = $1, error = $2 + WHERE id = $3 AND fsm = $4 + "#; + + let result = conn.execute(&query, &[&status, &error, &id, &S::FSM_NAME])?; + + if result != 1 { + Err(ErrorKind::ActionNotFound(id).into()) + } else { + Ok(()) + } +} + +/// Execute a single action in case it is pending or retryable. Holds +/// a lock on the action's database row while performing the action +/// and writes back the status afterwards. +/// +/// Should the execution of an action fail cleanly (i.e. without a +/// panic), the error will be persisted. Should it fail by panicking +/// (which developers should never do explicitly in action +/// interpreters) its status will not be changed. +fn run_action<S>(tx: Transaction, id: Uuid, state: &S::State, _fsm: PhantomData<S>) + -> Result<Vec<S::Event>> where + S: FSM, + S::Action: DeserializeOwned { + let (status, action) = get_action::<Transaction, S>(&tx, id)?; + + let result = match status { + ActionStatus::Pending => { + match S::act(action, state) { + // If the action succeeded, update its status to + // completed and return the created events. + Ok(events) => { + update_action_status( + &tx, id, ActionStatus::Completed, None, PhantomData::<S> + )?; + events + }, + + // If the action failed, persist the debug message and + // return nothing. + Err(err) => { + let msg = Some(format!("{:?}", err)); + update_action_status( + &tx, id, ActionStatus::Failed, msg, PhantomData::<S> + )?; + vec![] + }, + } + }, + + _ => { + // TODO: Currently only pending actions are run because + // retryable actions are not yet implemented. + vec![] + }, + }; + + tx.commit().context("failed to commit transaction")?; + Ok(result) +} diff --git a/users/tazjin/finito/finito-postgres/src/tests.rs b/users/tazjin/finito/finito-postgres/src/tests.rs new file mode 100644 index 000000000000..b1b5821be3c4 --- /dev/null +++ b/users/tazjin/finito/finito-postgres/src/tests.rs @@ -0,0 +1,47 @@ +use super::*; + +use finito_door::*; +use postgres::{Connection, TlsMode}; + +// TODO: read config from environment +fn open_test_connection() -> Connection { + Connection::connect("postgres://finito:finito@localhost/finito", TlsMode::None) + .expect("Failed to connect to test database") +} + +#[test] +fn test_insert_machine() { + let conn = open_test_connection(); + let initial = DoorState::Opened; + let door = insert_machine(&conn, initial).expect("Failed to insert door"); + let result = get_machine(&conn, &door, false).expect("Failed to fetch door"); + + assert_eq!(result, DoorState::Opened, "Inserted door state should match"); +} + +#[test] +fn test_advance() { + let conn = open_test_connection(); + + let initial = DoorState::Opened; + let events = vec![ + DoorEvent::Close, + DoorEvent::Open, + DoorEvent::Close, + DoorEvent::Lock(1234), + DoorEvent::Unlock(1234), + DoorEvent::Lock(4567), + DoorEvent::Unlock(1234), + ]; + + let door = insert_machine(&conn, initial).expect("Failed to insert door"); + + for event in events { + advance(&conn, &door, event).expect("Failed to advance door FSM"); + } + + let result = get_machine(&conn, &door, false).expect("Failed to fetch door"); + let expected = DoorState::Locked { code: 4567, attempts: 2 }; + + assert_eq!(result, expected, "Advanced door state should match"); +} diff --git a/users/tazjin/gruber-darker.qss b/users/tazjin/gruber-darker.qss new file mode 100644 index 000000000000..16f4c2f32991 --- /dev/null +++ b/users/tazjin/gruber-darker.qss @@ -0,0 +1,508 @@ +/** +** Gruber Darker theme for Quassel. +** +** This theme derives from multiple different things: +** +** - Quassel DarkSolarized (https://gist.github.com/Zren/e91ad5197f9d6b6d410f) +** - Quassel Dracula (https://github.com/dracula/quassel) +** - gruber-darker for Emacs (https://github.com/rexim/gruber-darker-theme) +** - Original Gruber theme for BBEdit (https://daringfireball.net/projects/bbcolors/schemes/) +** +** This is a work-in-progress as I haven't figured out the point of +** all of the colours yet, and what I want them to be instead. +** +**/ + +/** +** Helpful Links: +** - QT: +** http://qt-project.org/doc/qt-4.8/stylesheet-syntax.html +** http://doc.qt.nokia.com/4.7-snapshot/stylesheet-reference.html +** http://doc.qt.nokia.com/4.7-snapshot/stylesheet-examples.html +** - Plastique Client Style: +** https://qt.gitorious.org/qt/qt/source/src/gui/styles/qplastiquestyle.cpp +** https://github.com/mirror/qt/blob/4.8/src/gui/styles/qplastiquestyle.cpp +** - Quassel Stylesheet Gallery: +** http://bugs.quassel-irc.org/projects/1/wiki/Stylesheet_Gallery +** http://bugs.quassel-irc.org/projects/1/wiki/Stylesheet_Gallery#DarkMonokaiqss +*/ + +/** +** - QSS Notes: +** Quassel stylesheets also support Palette { role: color; } for setting the system +** palette. See the QPalette docs for available roles, and convert them into qss-style +** attributes, so ButtonText would become button-text or see qssparser.cpp In fact, +** qssparser.cpp is the authorative source for Quassel's qss syntax that contains all +** the extensions over standard Qt qss syntax. +** See: +** http://qt-project.org/doc/qt-4.8/qpalette.html#ColorRole-enum +** https://github.com/quassel/quassel/blob/master/src/uisupport/qssparser.cpp +** +*/ + +Palette { + /* Window colors */ + window: #282828; + background: #181818; + foreground: #f4f4f4; + + base: #181818; + alternate-base: #282828; + + /* Just setting palette(tooltip-base) doesn't work as intended so we set it in + ** a QTooltip{} rule as well. + */ + tooltip-base: #282a36; // palette(base) TODO + tooltip-text: white; // palette(text) TODO + + /* The following attributes should be done in a scale */ + light: #444444; // Tab Borders, Scrollbar handle grips, Titled Panel border (Settings) + midlight: #333333; // ? + button: #292929; // Menu BG, Scrollbar and Button base. + mid: #252525; // Titled Panel border (Settings) + dark: #202020; // TreeView [-] and ... color (Also various borders in Windows Client Style) + shadow: #1d1d1d; // ? + + + /* Text colors */ + text: white; + button-text: #f8f8f2; + + highlight: #44475a; + + /* Link colors */ + link: #ff79c6; + link-visited: #bd93f9; + + /* Color of the marker line in the chat view. BG Node that is overlayed on the first new ChatLine. */ + // 0 -> 0.1 (sharp line) + marker-line: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1, stop: 0 #586e75, stop: 0.1 #586e75, stop: 0.1 transparent); +} + +/* +** Base Object Colors +*/ + +/* Tables */ +// QTreeView#settingsTree -> Tree in the Settings popup. + +QTreeView, QTableView { + alternate-background-color: #282a36; + // background-color: palette(shadow); + border: 0px; +} + +QTreeView { + selection-background-color: transparent; +} + +QTreeView::item { + border-left: 2px solid palette(base); +} + +QTreeView::item:focus { + border-width: 0 0 0 2px; + outline: none; +} + +QTreeView::item:selected { + border-width: 0 0 0 2px; + color: palette(button-text); +} + +QTreeView::item:hover { + background: palette(dark); +} + + +QTreeView::item:selected:active{ + color: palette(button-text); + background: palette(dark); + border-color: palette(highlight); +} + +QTreeView::item:selected:!active { + color: palette(button-text); + background: palette(dark); + border-color: palette(highlight); +} + +/* Scrollbar */ +/* From Quassel Wiki: http://sprunge.us/iZGB */ +QScrollBar { + //background: transparent; + background: palette(base); + margin: 0; +} +QScrollBar:hover { + /* Optional: Subtle accent of scrolling area on hover */ + background: #161616; /* base +2 */ +} +QScrollBar:vertical { + width: 8px; +} +QScrollBar:horizontal { + height: 8px; +} + +QScrollBar::handle { + padding: 0; + margin: 2px; + border-radius: 2px; + border: 2px solid palette(midlight); + background: palette(midlight); +} + +QScrollBar::handle:vertical { + min-height: 20px; + min-width: 0px; +} + +QScrollBar::handle:horizontal { + min-width: 20px; + min-height: 0px; +} +QScrollBar::handle:hover { + border-color: palette(light); + background: palette(light); +} +QScrollBar::handle:pressed { + background: palette(highlight); + border-color: palette(highlight); +} + +QScrollBar::add-line , QScrollBar::sub-line { + height: 0px; + border: 0px; +} +QScrollBar::up-arrow, QScrollBar::down-arrow { + border: 0px; + width: 0px; + height: 0px; +} + +QScrollBar::add-page, QScrollBar::sub-page { + background: none; +} + +/* Input Box */ +MultiLineEdit { + //background: palette(base); + //color: palette(foreground); +} + +/* Widgets */ +/* http://doc.qt.nokia.com/4.7-snapshot/qdockwidget.html */ +//QMainWindow, +QMainWindow QAbstractScrollArea { + //border: 0; // Remove borders. + border: 1px solid palette(shadow); +} + +QMainWindow { + //background: palette(mid); // Main window trim +} + +/* Splitter */ +/* The splits between QDockWidgets and QMainWindow is a different element. */ +QSplitter::handle, +QMainWindow::separator { + background: palette(dark); +} +QSplitter::handle:horizontal:hover, +QMainWindow::separator:vertical:hover { + background: qlineargradient(x1: 0, y1: 0, x2: 1, y2: 0, stop: 0 palette(window), stop: 0.5 palette(light), stop: 1 palette(window)); +} + +QSplitter::handle:vertical:hover, +QMainWindow::separator:horizontal:hover { + background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1, stop: 0 palette(window), stop: 0.5 palette(light), stop: 1 palette(window)); +} + +/* Menu Bar / Context Menues */ +QMenu { + margin: 5px; // A bit of nice padding around menu items. +} + +/* ToolTip */ +/* Note: You cannot create transparent sections in the popup box without a mask set. Thus the black edges outside the rounded borders. */ +QToolTip { + border: 2px solid #202020; // palette(dark) + border-radius: 2px; + background: #282a36; // palette(base) + color: white; // palette(text) +} + +/* Tabs */ +/* + The palette is designed for the selected one to be darker. So we need to change it. Decided to do a simple line. + tab:bottom and tab:top reverse y1 and y2 on the linear gradients. + + Tab Shadow: #444444 (light) + Tab Hover: #666 + Tab Selected: palette(highlight) +*/ + +QTabWidget::tab-bar { + alignment: center; +} + +QTabBar::tab { + min-width: 30px; + height: 20px; +} + +QTabBar::tab:bottom:selected { + background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1, stop: 0 palette(highlight), stop: 0.2 palette(highlight), stop: 0.2 transparent); +} + +QTabBar::tab:top:selected { + background: qlineargradient(x1: 0, y1: 1, x2: 0, y2: 0, stop: 0 palette(highlight), stop: 0.2 palette(highlight), stop: 0.2 transparent); +} + +QTabBar::tab:!selected { + color: #888; +} + +QTabBar::tab:bottom:!selected { + background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1, stop: 0 palette(light), stop: 0.2 palette(light), stop: 0.2 transparent); +} + +QTabBar::tab:top:!selected { + background: qlineargradient(x1: 0, y1: 1, x2: 0, y2: 0, stop: 0 palette(light), stop: 0.2 palette(light), stop: 0.2 transparent); +} + +QTabBar::tab:!selected:hover { + color: #aaa; +} + +QTabBar::tab:bottom:!selected:hover { + background: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 1, stop: 0 #666, stop: 0.2 #666, stop: 0.2 transparent); +} + +QTabBar::tab:top:!selected:hover { + background: qlineargradient(x1: 0, y1: 1, x2: 0, y2: 0, stop: 0 #666, stop: 0.2 #666, stop: 0.2 transparent); +} + +/* +** Quassel CSS +*/ + +/* Main Chat Background Override */ +ChatView { + background: #181818; +} +ChatView QScrollBar { + background: #282a36; +} +ChatView QScrollBar:hover { + background: #282a36; +} + +ChatView QScrollBar::handle { + border-color: #44475a; + background: #44475a; +} + +ChatView QScrollBar::handle:hover { + border-color: #44475a; + background: #44475a; +} + +/**/ +QStatusBar {} +QStatusBar::item { + border: none; +} +QStatusBar QLabel { + color: #888; +} + +/* https://github.com/quassel/quassel/blob/master/src/qtui/ui/msgprocessorstatuswidget.ui */ +QStatusBar MsgProcessorStatusWidget {} +QStatusBar MsgProcessorStatusWidget QLabel#label {} +QStatusBar MsgProcessorStatusWidget QProgressBar#progressBar {} + +/* https://github.com/quassel/quassel/blob/master/src/qtui/ui/coreconnectionstatuswidget.ui */ +QStatusBar CoreConnectionStatusWidget {} +QStatusBar CoreConnectionStatusWidget QLabel#messageLabel {} +QStatusBar CoreConnectionStatusWidget QProgressBar#progressBar {} +QStatusBar CoreConnectionStatusWidget QLabel#lagLabel {} +QStatusBar CoreConnectionStatusWidget QLabel#sslLabel { + qproperty-pixmap: none; /* Hide the SSL status icon */ +} + + +/* Font */ +// Will not override if selectors are doubled up eg: "ChatLine, MultiLineEdit {}" +// These will override anything set in Quassel's Settings. +/** + * Don't bold or style MultiLineEdit text in any way otherwise you will be + * prone to get weird behaviour in submitting from the Input box. + * It will randomly bold your input if you do. + */ +ChatLine { + //font-family: "MingLiU_HKSCS-ExtB", "Courier New", Courier, Monotype; + + //font-size: 13pt; + //font-weight: bold; + } +MultiLineEdit { + //font-family: "MingLiU_HKSCS-ExtB", "Courier New", Courier, Monotype; + + //font-size: 20px; + //font-weight: normal; + } +ChatLine#plain { + //font-weight: bold; + } + +/* Font: UI Global Font */ +QWidget { + //font-family: consolas; + } +ChatListItem { + font-family: consolas; + } +NickListItem { + font-family: consolas; + } +StyledLabel#topicLabel { + font-family: consolas; + font-size: 14px; + } + + +/* Topic Box */ +StyledLabel#topicLabel { background: palette(base); font-family: consolas; } + +/* Buffer / Channel List */ +/** + state: inactive, channel-event, unread-message, highlighted + type: query, channel, network +**/ +ChatListItem { foreground: #f8f8f2; } +ChatListItem[state="inactive"] { foreground: #44475a; } +ChatListItem[state="channel-event"] { foreground: #6272a4; } /* palette(button-text) */ +ChatListItem[state="unread-message"] { foreground: #f8f8f2; } +ChatListItem[state="highlighted"] { foreground: #44475a; } + +ChatListItem[type="network", state="unread-message"] { foreground: #44475a; } +ChatListItem[type="network", state="highlighted"] { foreground: #44475a; } +ChatListItem[type="query", state="unread-message"] { foreground: #44475a; } + + +/* Nick List */ +/** + state: away + type: user, category +**/ +NickListItem[type="category"] { foreground: #6272a4; } +NickListItem[type="user"] { foreground: #f8f8f2 } +NickListItem[type="user", state="away"] { foreground: #44475a; } + + + +/* Chatbox Line Formatting */ +ChatLine[label="highlight"] { + foreground: #f5f5f5; + background: #282828; +} + +/* +** Option: Bold highlighted text, but not the timestamp. +*/ +/* +ChatLine[label="highlight"] { font-weight: bold; } +ChatLine::timestamp[label="highlight"]{ font-weight: normal; } +*/ + +ChatLine::timestamp[label="highlight"] { foreground: #44475a; } + +ChatLine::timestamp { } + +/* ::contents == Message */ +ChatLine::contents { + /* Can only set background */ +} + +ChatLine#plain { foreground: #f8f8f2; } +ChatLine#notice { foreground: #44475a; } +ChatLine#action { foreground: #565f73; font-style: italic; font-weight: bold; } +ChatLine#nick { foreground: #6272a4; } +ChatLine#mode { foreground: #6272a4; } +ChatLine#join { foreground: #6272a4; } +ChatLine#part { foreground: #6272a4; } +ChatLine#quit { foreground: #6272a4; } +ChatLine#kick { foreground: #6272a4; } +ChatLine#kill { foreground: #6272a4; } +ChatLine#server { foreground: #44475a; } +ChatLine#info { foreground: #44475a; } +ChatLine#error { foreground: #ff5555; } +ChatLine#daychange { foreground: #44475a; } +ChatLine#topic { foreground: #f1fa8c; } +ChatLine#netsplit-join { foreground: #44475a; } +ChatLine#netsplit-quit { foreground: #44475a; } + +ChatLine::timestamp { + foreground: #586e75; + // Resets the timestemp font during #action and other possible formatting. + font-style: normal; + font-weight: normal; +} + +ChatLine::url { + foreground: palette(link); + //font-style: underline; // Uncomment if you always want an underline on links. +} + +/* Sender Colors */ +ChatLine::sender#plain[sender="self"] { foreground: #586e75; } + +/** + * The following are the sixteen colours used for the senders. + * The names are calculated by taking the hash of the nickname. + * Then take the modulo (the remainder) when divided by 16. + * Preview: http://i.imgur.com/xeRKI4H.png + */ +ChatLine::sender#plain[sender="0"] { foreground: #96a6c8; } +ChatLine::sender#plain[sender="1"] { foreground: #73c936; } +ChatLine::sender#plain[sender="2"] { foreground: #ffdd33; } +ChatLine::sender#plain[sender="3"] { foreground: #cc8c3c; } +ChatLine::sender#plain[sender="4"] { foreground: #ff4f58; } +ChatLine::sender#plain[sender="5"] { foreground: #9e95c7; } +ChatLine::sender#plain[sender="6"] { foreground: #95a99f; } +ChatLine::sender#plain[sender="7"] { foreground: #8be9fd; } + +/* +32 */ +ChatLine::sender#plain[sender="8"] { foreground: #96a6c8; } +ChatLine::sender#plain[sender="9"] { foreground: #73c936; } +ChatLine::sender#plain[sender="a"] { foreground: #ffdd33; } +ChatLine::sender#plain[sender="b"] { foreground: #cc8c3c; } +ChatLine::sender#plain[sender="c"] { foreground: #ff4f58; } +ChatLine::sender#plain[sender="d"] { foreground: #9e95c7; } +ChatLine::sender#plain[sender="e"] { foreground: #95a99f; } +ChatLine::sender#plain[sender="f"] { foreground: #8be9fd; } + +/* +** mIRC formats +*/ +ChatLine[format="bold"] { font-weight: bold;} +ChatLine[format="italic"] { font-style: italic; } +ChatLine[format="underline"] { font-style: underline; } + +/* Blues are hard to read. */ +ChatLine[fg-color="2"] { foreground: #15a; } +ChatLine[bg-color="2"] { background: #15a; } +ChatLine[fg-color="c"] { foreground: #15f; } +ChatLine[bg-color="c"] { background: #15f; } + +/* +** Experimental +*/ +BufferViewDock[active=true] { + /* The circle is hardcoded into the title. */ + /* Color only changes on a refresh (F5) (so it's pointless). */ + /* Also colors the border in Breeze. */ + //color: palette(highlight); +} diff --git a/users/tazjin/hanebuschtag.txt b/users/tazjin/hanebuschtag.txt new file mode 100644 index 000000000000..fc5c06522040 --- /dev/null +++ b/users/tazjin/hanebuschtag.txt @@ -0,0 +1,63 @@ +bazurschnaburkini +buchweizengrütze +burkischnurkischnurzelwutz +burwurgurken +burwurka +gaschnurzel +gezwurkel +gurzelschnurzelgurke +hanemazurka +hanemazurkelgurkel +haneschlawitzka +haneschnaburkeln +haneschnawurkagurka +haneschnawurkel +haneschnuren +haneschnurkissima +hanewurka +hanewurkini +hanewurzeln +ronzelschlawonzel +ronzelwonzel +schlagurkelwini +schlaraffenwurburzel +schlawiburschnurschlakini +schlawonzel +schlawurkinischnagurka +schlawurzelgegurkel +schlawurzeltrollurzel +schlunzelgarfunzel +schmonzelgafonzel +schmotzrotzel +schnaburka +schnaburkel +schnaburkini +schnackel +schnarkelbarkel +schnarwurzelka +schnawurkeln +schnawurzelgackschnurschnacksschnicks +schnawurzini +schniepel +schnirkelschini +schnöckel +schnockelgockel +schnorchel +schnörk +schnorkelbusch +schnörkelknörkel +schnorkelorgel +schnörks +schnotzelgekrotzel +schnudelwurkini +schnurburka +schnurkini +schnurkinihanfini +schnurzelgawurzel +schnurzelwurzelwutz +schnurzelwutz +strazurkeln +wazurka +wurkelgurkel +wurkelschnurrini +wurzelchakramahurka diff --git a/users/tazjin/homepage/default.nix b/users/tazjin/homepage/default.nix new file mode 100644 index 000000000000..2ce1cf632255 --- /dev/null +++ b/users/tazjin/homepage/default.nix @@ -0,0 +1,77 @@ +# Assembles the website index and configures an nginx instance to +# serve it. +# +# The website is made up of a simple header&footer and content +# elements for things such as blog posts and projects. +# +# Content for the blog is in //users/tazjin/blog instead of here. +{ depot, lib, pkgs, ... }@args: + +with depot; +with nix.yants; + +let + inherit (builtins) readFile replaceStrings sort; + inherit (pkgs) writeFile runCommandNoCC; + + # The different types of entries on the homepage. + entryClass = enum "entryClass" [ "blog" "project" "misc" ]; + + # The definition of a single entry. + entry = struct "entry" { + class = entryClass; + title = string; + url = string; + date = int; # epoch + description = option string; + }; + + escape = replaceStrings [ "<" ">" "&" "'" ] [ "<" ">" "&" "'" ]; + + postToEntry = defun [ web.blog.post entry ] (post: { + class = "blog"; + title = post.title; + url = "/blog/${post.key}"; + date = post.date; + }); + + formatDate = defun [ int string ] (date: readFile (runCommandNoCC "date" {} '' + date --date='@${toString date}' '+%Y-%m-%d' > $out + '')); + + formatEntryDate = defun [ entry string ] (entry: entryClass.match entry.class { + blog = "Blog post from ${formatDate entry.date}"; + project = "Project from ${formatDate entry.date}"; + misc = "Posted on ${formatDate entry.date}"; + }); + + entryToDiv = defun [ entry string ] (entry: '' + <a href="${entry.url}" class="entry ${entry.class}"> + <div> + <p class="entry-title">${escape entry.title}</p> + ${ + lib.optionalString ((entry ? description) && (entry.description != null)) + "<p class=\"entry-description\">${escape entry.description}</p>" + } + <p class="entry-date">${formatEntryDate entry}</p> + </div> + </a> + ''); + + index = entries: pkgs.writeText "index.html" (lib.concatStrings ( + [ (builtins.readFile ./header.html) ] + ++ (map entryToDiv (sort (a: b: a.date > b.date) entries)) + ++ [ (builtins.readFile ./footer.html) ] + )); + + pageEntries = import ./entries.nix; + homepage = index ((map postToEntry users.tazjin.blog.posts) ++ pageEntries); + atomFeed = import ./feed.nix (args // { inherit entry pageEntries; }); +in runCommandNoCC "website" {} '' + mkdir $out + cp ${homepage} $out/index.html + cp ${atomFeed} $out/feed.atom + mkdir $out/static + cp -r ${depot.web.static}/* $out/static + cp -rf ${./static}/* $out/static +'' diff --git a/users/tazjin/homepage/entries.nix b/users/tazjin/homepage/entries.nix new file mode 100644 index 000000000000..1e2b0b03dfc9 --- /dev/null +++ b/users/tazjin/homepage/entries.nix @@ -0,0 +1,74 @@ +[ + { + class = "misc"; + title = "Interview with Joscha Bach"; + url = "https://www.youtube.com/watch?v=P-2P3MSZrBM"; + date = 1594594800; + description = '' + A fascinating, mind-bending interview by Lex Fridman with Joscha + Bach about the Nature of the Universe. + ''; + } + { + class = "misc"; + title = "The Virus Lounge"; + url = "https://tvl.fyi"; + date = 1587435629; + description = "A daily social video call in these trying pandemic times. Join us!"; + } + { + class = "project"; + title = "depot"; + url = "https://code.tvl.fyi/about"; + date = 1576800000; + description = "Merging all of my projects into a single, Nix-based monorepo"; + } + { + class = "project"; + title = "Nixery"; + url = "https://github.com/google/nixery"; + date = 1565132400; + description = "A Nix-backed container registry that builds container images on demand"; + } + { + class = "project"; + title = "kontemplate"; + url = "https://code.tvl.fyi/about/ops/kontemplate"; + date = 1486550940; + description = "Simple file templating tool built for Kubernetes resources"; + } + { + class = "misc"; + title = "dottime"; + url = "https://dotti.me/"; + date = 1560898800; + description = "A universal convention for conveying time (by edef <3)"; + } + { + class = "project"; + title = "journaldriver"; + url = "https://code.tvl.fyi/about/ops/journaldriver"; + date = 1527375600; + description = "Small daemon to forward logs from journald to Stackdriver Logging"; + } + { + class = "misc"; + title = "Principia Discordia"; + url = "https://principiadiscordia.com/book/1.php"; + date = 1495494000; + description = '' + The Principia is a short book I read as a child, and didn't + understand until much later. It shaped much of my world view. + ''; + } + { + class = "misc"; + title = "This Week in Virology"; + url = "http://www.microbe.tv/twiv/"; + date = 1585517557; + description = '' + Podcast with high-quality information about virology, + epidemiology and so on. Highly relevant to COVID19. + ''; + } +] diff --git a/users/tazjin/homepage/feed.nix b/users/tazjin/homepage/feed.nix new file mode 100644 index 000000000000..2a033444e8ba --- /dev/null +++ b/users/tazjin/homepage/feed.nix @@ -0,0 +1,42 @@ +# Creates the Atom feed for my homepage. +{ depot, lib, pkgs, entry, pageEntries, ... }: + +with depot.nix.yants; + +let + inherit (builtins) map readFile; + inherit (lib) max singleton; + inherit (pkgs) writeText; + inherit (depot.web) blog atom-feed; + + pageEntryToEntry = defun [ entry atom-feed.entry ] (e: { + id = "tazjin:${e.class}:${toString e.date}"; + updated = e.date; + published = e.date; + title = e.title; + summary = e.description; + + links = singleton { + rel = "alternate"; + href = e.url; + }; + }); + + allEntries = (with depot.users.tazjin.blog; map (blog.toFeedEntry config) posts) + ++ (map pageEntryToEntry pageEntries); + + feed = { + id = "https://tazj.in/"; + title = "tazjin's interblag"; + subtitle = "my posts, projects and other interesting things"; + rights = "© 2020 tazjin"; + authors = [ "tazjin" ]; + + links = singleton { + rel = "self"; + href = "https://tazjin/feed.atom"; + }; + + entries = allEntries; + }; +in writeText "feed.atom" (atom-feed.renderFeed feed) diff --git a/users/tazjin/homepage/footer.html b/users/tazjin/homepage/footer.html new file mode 100644 index 000000000000..2f17135066e8 --- /dev/null +++ b/users/tazjin/homepage/footer.html @@ -0,0 +1,2 @@ + </div> +</body> diff --git a/users/tazjin/homepage/header.html b/users/tazjin/homepage/header.html new file mode 100644 index 000000000000..2a5aed4fed57 --- /dev/null +++ b/users/tazjin/homepage/header.html @@ -0,0 +1,33 @@ +<!DOCTYPE html> +<head><meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content="tazjin's blog"> + <link rel="stylesheet" type="text/css" href="static/tvl.css" media="all"> + <link rel="icon" type="image/webp" href="/static/favicon.webp"> + <link rel="alternate" type="application/atom+xml" href="/feed.atom"> + <title>tazjin's interblag</title> +</head> +<body class="dark"> + <header> + <h1> + <a class="interblag-title" href="/">tazjin's interblag</a> + </h1> + <hr> + </header> + <div class="introduction"> + <p>Hello, illuminated visitor.</p> + <p> + I'm tazjin. Usually you can find + me <a class="dark-link" href="https://git.tazj.in/about">programming computers</a> + using tools such as <a class="dark-link" href="https://nixos.org/nix">Nix</a> + and <a class="dark-link" href="https://www.gnu.org/software/emacs/">Emacs</a>. + </p> + <p> + Below is a collection of + my <span class="project">projects</span>, <span class="blog">blog + posts</span> and some <span class="misc">random things</span> by + me or others. If you'd like to get in touch about anything, send + me a mail at mail@[this domain] or ping me on IRC. + </p> + </div> + <div class="entry-container"> diff --git a/users/tazjin/homepage/static/favicon.webp b/users/tazjin/homepage/static/favicon.webp new file mode 100644 index 000000000000..f99c9085340b --- /dev/null +++ b/users/tazjin/homepage/static/favicon.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/nixery/dominator.webp b/users/tazjin/homepage/static/img/nixery/dominator.webp new file mode 100644 index 000000000000..2d8569a6ca21 --- /dev/null +++ b/users/tazjin/homepage/static/img/nixery/dominator.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/nixery/example_extra.webp b/users/tazjin/homepage/static/img/nixery/example_extra.webp new file mode 100644 index 000000000000..101f0f633aef --- /dev/null +++ b/users/tazjin/homepage/static/img/nixery/example_extra.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/nixery/example_plain.webp b/users/tazjin/homepage/static/img/nixery/example_plain.webp new file mode 100644 index 000000000000..a2b90b3e21d5 --- /dev/null +++ b/users/tazjin/homepage/static/img/nixery/example_plain.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/nixery/ideal_layout.webp b/users/tazjin/homepage/static/img/nixery/ideal_layout.webp new file mode 100644 index 000000000000..0e9f74556682 --- /dev/null +++ b/users/tazjin/homepage/static/img/nixery/ideal_layout.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/watchblob_1.webp b/users/tazjin/homepage/static/img/watchblob_1.webp new file mode 100644 index 000000000000..27e588e1a145 --- /dev/null +++ b/users/tazjin/homepage/static/img/watchblob_1.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/watchblob_2.webp b/users/tazjin/homepage/static/img/watchblob_2.webp new file mode 100644 index 000000000000..b2dea98b4fb4 --- /dev/null +++ b/users/tazjin/homepage/static/img/watchblob_2.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/watchblob_3.webp b/users/tazjin/homepage/static/img/watchblob_3.webp new file mode 100644 index 000000000000..99b49373b5b4 --- /dev/null +++ b/users/tazjin/homepage/static/img/watchblob_3.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/watchblob_4.webp b/users/tazjin/homepage/static/img/watchblob_4.webp new file mode 100644 index 000000000000..41dbdb6be1cf --- /dev/null +++ b/users/tazjin/homepage/static/img/watchblob_4.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/watchblob_5.webp b/users/tazjin/homepage/static/img/watchblob_5.webp new file mode 100644 index 000000000000..c42a4ce1bc0f --- /dev/null +++ b/users/tazjin/homepage/static/img/watchblob_5.webp Binary files differdiff --git a/users/tazjin/homepage/static/img/watchblob_6.webp b/users/tazjin/homepage/static/img/watchblob_6.webp new file mode 100644 index 000000000000..1440761859dd --- /dev/null +++ b/users/tazjin/homepage/static/img/watchblob_6.webp Binary files differdiff --git a/users/tazjin/keys.nix b/users/tazjin/keys.nix new file mode 100644 index 000000000000..d6941b66d9d8 --- /dev/null +++ b/users/tazjin/keys.nix @@ -0,0 +1,9 @@ +# My SSH public keys +{ ... }: + +let withAll = keys: keys // { all = builtins.attrValues keys; }; +in withAll { + # frog = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKMZzRdcrHTuCPoaFy36MPr5IW/hnImlse/OBOn6udL/ tazjin@frog"; + s10e = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDf7CNlYoauHcSYsMNnCZt5h9QSYH/7keYkg8g3hT32+"; + tverskoy = "sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBAWvA3RpXpMAqruUbB+eVgvvHCzhs5R9khFRza3YSLeFiIqOxVVgyhzW/BnCSD9t/5JrqRdJIGQLnkQU9m4REhUAAAAEc3NoOg== tazjin@tverskoy"; +} diff --git a/users/tazjin/nisp/transform.el b/users/tazjin/nisp/transform.el new file mode 100644 index 000000000000..89b2bb104d27 --- /dev/null +++ b/users/tazjin/nisp/transform.el @@ -0,0 +1,137 @@ +;; Nix as a Lisp + +(require 'cl-lib) +(require 'json) +(require 's) +(require 'dash) + +(defun nisp/expr (form) + "Entrypoint for Nisp->Nix transformation. Will translate FORM +into Nix code, if it is a valid Nisp expression. + +To make code generation slightly easier, each +expression (including literals) is wrapped in an extra pair of +parens." + (concat + "(" + (pcase form + ;; Special keywords + ('() "null") + (`(let . ,rest) (nisp/let form)) + (`(fn . ,rest) (nisp/fn form)) + (`(if ,cond ,then ,else) (nisp/if cond then else)) + + ;; Nix operators & builtins that need special handling + (`(or ,lhs ,rhs) (nisp/infix "||" lhs rhs)) + (`(and ,lhs ,rhs) (nisp/infix "&&" lhs rhs)) + (`(> ,lhs ,rhs) (nisp/infix ">" lhs rhs)) + (`(< ,lhs ,rhs) (nisp/infix "<" lhs rhs)) + (`(>= ,lhs ,rhs) (nisp/infix ">=" lhs rhs)) + (`(<= ,lhs ,rhs) (nisp/infix "<=" lhs rhs)) + (`(+ ,lhs ,rhs) (nisp/infix "+" lhs rhs)) + (`(- ,lhs ,rhs) (nisp/infix "-" lhs rhs)) + (`(* ,lhs ,rhs) (nisp/infix "*" lhs rhs)) + (`(/ ,lhs ,rhs) (nisp/infix "/" lhs rhs)) + (`(-> ,lhs ,rhs) (nisp/infix "->" lhs rhs)) + (`(? ,lhs ,rhs) (nisp/infix "?" lhs rhs)) + (`(// ,lhs ,rhs) (nisp/infix "//" lhs rhs)) + (`(++ ,lhs ,rhs) (nisp/infix "++" lhs rhs)) + (`(== ,lhs ,rhs) (nisp/infix "==" lhs rhs)) + (`(!= ,lhs ,rhs) (nisp/infix "!=" lhs rhs)) + (`(! ,term) (concat "!" (nisp/expr term))) + (`(- ,term) (concat "-" (nisp/expr term))) + + ;; Attribute sets + (`(attrs . ,rest) (nisp/attribute-set form)) + + ;; Function calls + ((and `(,func . ,args) + (guard (symbolp func))) + (nisp/funcall func args)) + + ;; Primitives + ((pred stringp) (json-encode-string form)) + ((pred numberp) (json-encode-number form)) + ((pred keywordp) (substring (symbol-name form) 1)) + ((pred symbolp) (symbol-name form)) + + ;; Lists + ((pred arrayp) (nisp/list form)) + + (other (error "Encountered unhandled form: %s" other))) + ")")) + +(defun nisp/infix (op lhs rhs) + (concat (nisp/expr lhs) " " op " " (nisp/expr rhs))) + +(defun nisp/funcall (func args) + (concat (symbol-name func) " " (s-join " " (-map #'nisp/expr args)))) + +(defun nisp/let (form) + (pcase form + (`(let . (,bindings . (,body . ()))) (concat "let " + (nisp/let bindings) + (nisp/expr body))) + (`((:inherit . ,inherits) . ,rest) (concat (nisp/inherit (car form)) + " " + (nisp/let rest))) + (`((,name . (,value . ())) .,rest) (concat (symbol-name name) " = " + (nisp/expr value) "; " + (nisp/let rest))) + ('() "in ") + (other (error "malformed form '%s' in let expression" other)))) + +(defun nisp/inherit (form) + (pcase form + (`(:inherit . ,rest) (concat "inherit " (nisp/inherit rest))) + (`((,source) . ,rest) (concat "(" (symbol-name source) ") " (nisp/inherit rest))) + (`(,item . ,rest) (concat (symbol-name item) " " (nisp/inherit rest))) + ('() ";"))) + +(defun nisp/if (cond then else) + (concat "if " (nisp/expr cond) + " then " (nisp/expr then) + " else " (nisp/expr else))) + +(defun nisp/list (form) + (cl-check-type form array) + (concat "[ " + (mapconcat #'nisp/expr form " ") + "]")) + + +(defun nisp/attribute-set (form) + "Attribute sets have spooky special handling because they are +not supported by the reader." + (pcase form + (`(attrs . ,rest) (concat "{ " (nisp/attribute-set rest))) + ((and `(,name . (,value . ,rest)) + (guard (keywordp name))) + (concat (substring (symbol-name name) 1) " = " + (nisp/expr value) "; " + (nisp/attribute-set rest))) + ('() "}"))) + +(defun nisp/fn (form) + (pcase form + (`(fn ,args ,body) (concat + (cl-loop for arg in args + concat (format "%s: " arg)) + (nisp/expr body))))) + +;; The following functions are not part of the transform. + +(defun nisp/eval (form) + (interactive "sExpression: ") + (when (stringp form) + (setq form (read form))) + + (message + ;; TODO(tazjin): Construct argv manually to avoid quoting issues. + (s-chomp + (shell-command-to-string + (concat "nix-instantiate --eval -E '" (nisp/expr form) "'"))))) + +(defun nisp/eval-last-sexp () + (interactive) + (nisp/eval (edebug-last-sexp))) diff --git a/users/tazjin/nittredir/background.js b/users/tazjin/nittredir/background.js new file mode 100644 index 000000000000..8d07f8528520 --- /dev/null +++ b/users/tazjin/nittredir/background.js @@ -0,0 +1,10 @@ +/** + * Register a URL change handler that redirects twitter.com links to nitter.net + */ + +chrome.webRequest.onBeforeRequest.addListener(function(details) { + let url = new URL(details.url); + return { + redirectUrl: ('https://nitter.net' + url.pathname) + }; +}, {urls: ['*://twitter.com/*'], types: ['main_frame']}, ['blocking']); diff --git a/users/tazjin/nittredir/manifest.json b/users/tazjin/nittredir/manifest.json new file mode 100644 index 000000000000..4efe1a6cc245 --- /dev/null +++ b/users/tazjin/nittredir/manifest.json @@ -0,0 +1,15 @@ +{ + "manifest_version": 2, + "name": "nittredir", + "version": "1.0", + "description": "Redirect twitter.com to nitter.net", + "background": { + "scripts": ["background.js"], + "persistent": true + }, + "permissions": [ + "webRequest", + "webRequestBlocking", + "*://twitter.com/*" + ] +} diff --git a/users/tazjin/nix.svg b/users/tazjin/nix.svg new file mode 100644 index 000000000000..4da795a4362b --- /dev/null +++ b/users/tazjin/nix.svg @@ -0,0 +1,50 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<svg + xmlns="http://www.w3.org/2000/svg" + width="141.5919mm" + height="122.80626mm" + viewBox="0 0 501.70361 435.14028" + id="svg2" + version="1.1"> + <g transform="translate(-156.33871,933.1905)" visibility="hidden"> + <path + id="lambda-path" + d="m 309.54892,-710.38827 122.19683,211.67512 -56.15706,0.5268 -32.6236,-56.8692 -32.85645,56.5653 -27.90237,-0.011 -14.29086,-24.6896 46.81047,-80.4901 -33.22946,-57.8257 z" + /> + <use + id="lambda-1" + href="#lambda-path" + visibility="visible" + transform="rotate(180,407.41868,-715.7565)" + fill="#f8f8ff" /> + <use + id="lambda-2" + visibility="visible" + transform="rotate(-120,407.28823,-715.86995)" + href="#lambda-path" + fill="#0039a6" /> + <use + id="lambda-3" + transform="rotate(-60,407.31177,-715.70016)" + href="#lambda-path" + visibility="visible" + fill="#d52b1e" /> + <use + id="lambda-4" + href="#lambda-path" + visibility="visible" + fill="#f8f8ff" /> + <use + id="lambda-5" + transform="rotate(60,407.11155,-715.78724)" + href="#lambda-path" + visibility="visible" + fill="#0039a6" /> + <use + transform="rotate(120,407.33916,-716.08356)" + id="lambda-6" + href="#lambda-path" + visibility="visible" + fill="#d52b1e" /> + </g> +</svg> diff --git a/users/tazjin/nixos/README.md b/users/tazjin/nixos/README.md new file mode 100644 index 000000000000..662f2a36acac --- /dev/null +++ b/users/tazjin/nixos/README.md @@ -0,0 +1,17 @@ +NixOS configuration +=================== + +My NixOS configurations! It configures most of the packages I require +on my systems, sets up Emacs the way I need and does a bunch of other +interesting things. + +System configuration lives in folders, and some of the modules stem +from `//ops/modules`. + +Machines are deployed with the script at `ops.nixos.rebuild-system`. + +## Configured hosts: + +* `tverskoy` - X13 AMD that's travelling around with me +* `frog` - weapon of mass computation (in storage in London) +* `camden` - NUC formerly serving tazj.in (in storage in London) diff --git a/users/tazjin/nixos/camden/default.nix b/users/tazjin/nixos/camden/default.nix new file mode 100644 index 000000000000..ebd4caac75c7 --- /dev/null +++ b/users/tazjin/nixos/camden/default.nix @@ -0,0 +1,351 @@ +# This file configures camden.tazj.in, my homeserver. +{ depot, pkgs, lib, ... }: + +config: let + nginxRedirect = { from, to, acmeHost }: { + serverName = from; + useACMEHost = acmeHost; + forceSSL = true; + + extraConfig = "return 301 https://${to}$request_uri;"; + }; +in lib.fix(self: { + # Disable the current ACME module and use the old one from 19.09 + # instead, until the various regressions have been sorted out. + # TODO(tazjin): Remove this once the new ACME module works. + disabledModules = [ "security/acme.nix" ]; + imports = + let oldChannel = fetchTarball { + # NixOS 19.09 on 2020-10-04 + url = "https://github.com/NixOS/nixpkgs-channels/archive/75f4ba05c63be3f147bcc2f7bd4ba1f029cedcb1.tar.gz"; + sha256 = "157c64220lf825ll4c0cxsdwg7cxqdx4z559fdp7kpz0g6p8fhhr"; + }; + in [ + "${depot.path}/ops/modules/quassel.nix" + "${depot.path}/ops/modules/smtprelay.nix" + "${oldChannel}/nixos/modules/security/acme.nix" + ]; + + # camden is intended to boot unattended, despite having an encrypted + # root partition. + # + # The below configuration uses an externally connected USB drive + # that contains a LUKS key file to unlock the disk automatically at + # boot. + # + # TODO(tazjin): Configure LUKS unlocking via SSH instead. + boot = { + initrd = { + availableKernelModules = [ + "ahci" "xhci_pci" "usbhid" "usb_storage" "sd_mod" "sdhci_pci" + "rtsx_usb_sdmmc" "r8169" + ]; + + kernelModules = [ "dm-snapshot" ]; + + luks.devices.camden-crypt = { + fallbackToPassword = true; + device = "/dev/disk/by-label/camden-crypt"; + keyFile = "/dev/sdb"; + keyFileSize = 4096; + }; + }; + + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + }; + + cleanTmpDir = true; + }; + + fileSystems = { + "/" = { + device = "/dev/disk/by-label/camden-root"; + fsType = "ext4"; + }; + + "/home" = { + device = "/dev/disk/by-label/camden-home"; + fsType = "ext4"; + }; + + "/boot" = { + device = "/dev/disk/by-label/BOOT"; + fsType = "vfat"; + }; + }; + + nix = { + maxJobs = lib.mkDefault 4; + + trustedUsers = [ "root" "tazjin" ]; + + binaryCaches = [ + "https://tazjin.cachix.org" + ]; + + binaryCachePublicKeys = [ + "tazjin.cachix.org-1:IZkgLeqfOr1kAZjypItHMg1NoBjm4zX9Zzep8oRSh7U=" + ]; + }; + + powerManagement.cpuFreqGovernor = lib.mkDefault "powersave"; + + networking = { + hostName = "camden"; + interfaces.enp1s0.useDHCP = true; + interfaces.enp1s0.ipv6.addresses = [ + { + address = "2a01:4b00:821a:ce02::5"; + prefixLength = 64; + } + ]; + + firewall.enable = false; + }; + + time.timeZone = "UTC"; + + # System-wide application setup + programs.fish.enable = true; + programs.mosh.enable = true; + + fonts = { + fonts = [ pkgs.jetbrains-mono ]; + fontconfig.defaultFonts.monospace = [ "JetBrains Mono" ]; + }; + + environment.systemPackages = + # programs from the depot + (with depot; [ + fun.idual.script + fun.idual.setAlarm + ]) ++ + + # programs from nixpkgs + (with pkgs; [ + bat + curl + direnv + emacs27-nox + fswebcam + git + gnupg + google-cloud-sdk + htop + jq + pass + pciutils + restic + ripgrep + screen + ]); + + users = { + # Set up my own user for logging in and doing things ... + users.tazjin = { + isNormalUser = true; + uid = 1000; + extraGroups = [ "git" "wheel" "quassel" "video" ]; + shell = pkgs.fish; + }; + + # Set up a user & group for general git shenanigans + groups.git = {}; + users.git = { + group = "git"; + isSystemUser = true; + }; + }; + + # Services setup + services.openssh.enable = true; + services.haveged.enable = true; + + # Join Tailscale into home network + services.tailscale.enable = true; + + # Allow sudo-ing via the forwarded SSH agent. + security.pam.enableSSHAgentAuth = true; + + # NixOS 20.03 broke nginx and I can't be bothered to debug it + # anymore, all solution attempts have failed, so here's a + # brute-force fix. + systemd.services.fix-nginx = { + script = "${pkgs.coreutils}/bin/chown -R nginx: /var/spool/nginx /var/cache/nginx"; + + serviceConfig = { + User = "root"; + Type = "oneshot"; + }; + }; + + systemd.timers.fix-nginx = { + wantedBy = [ "multi-user.target" ]; + timerConfig = { + OnCalendar = "minutely"; + }; + }; + + # Provision a TLS certificate outside of nginx to avoid + # nixpkgs#38144 + security.acme = { + # acceptTerms = true; + + certs."tazj.in" = { + email = "mail@tazj.in"; + user = "nginx"; + group = "nginx"; + webroot = "/var/lib/acme/acme-challenge"; + extraDomains = { + "cs.tazj.in" = null; + "git.tazj.in" = null; + "www.tazj.in" = null; + + # Local domains (for this machine only) + "camden.tazj.in" = null; + }; + postRun = "systemctl reload nginx"; + }; + + certs."quassel.tazj.in" = { + email = "mail@tazj.in"; + webroot = "/var/lib/acme/challenge-quassel"; + user = "nginx"; # required because of a bug in the ACME module + group = "quassel"; + allowKeysForGroup = true; + }; + }; + + # Forward logs to Google Cloud Platform + services.journaldriver = { + enable = true; + logStream = "home"; + googleCloudProject = "tazjins-infrastructure"; + applicationCredentials = "/etc/gcp/key.json"; + }; + + services.depot.quassel = { + enable = true; + acmeHost = "quassel.tazj.in"; + bindAddresses = [ + "0.0.0.0" + ]; + }; + + services.bitlbee = { + enable = false; + portNumber = 2337; # bees + }; + + # serve my website(s) + services.nginx = { + enable = true; + enableReload = true; + package = with pkgs; nginx.override { + modules = [ nginxModules.rtmp ]; + }; + + recommendedTlsSettings = true; + recommendedGzipSettings = true; + recommendedProxySettings = true; + + appendConfig = '' + rtmp_auto_push on; + rtmp { + server { + listen 1935; + chunk_size 4000; + + application tvl { + live on; + + allow publish 88.98.195.213; + allow publish 10.0.1.0/24; + deny publish all; + + allow play all; + } + } + } + ''; + + commonHttpConfig = '' + log_format json_combined escape=json + '{' + '"remote_addr":"$remote_addr",' + '"method":"$request_method",' + '"uri":"$request_uri",' + '"status":$status,' + '"request_size":$request_length,' + '"response_size":$body_bytes_sent,' + '"response_time":$request_time,' + '"referrer":"$http_referer",' + '"user_agent":"$http_user_agent"' + '}'; + + access_log syslog:server=unix:/dev/log,nohostname json_combined; + ''; + + virtualHosts.homepage = { + serverName = "tazj.in"; + serverAliases = [ "camden.tazj.in" ]; + default = true; + useACMEHost = "tazj.in"; + root = depot.users.tazjin.homepage; + forceSSL = true; + + extraConfig = '' + ${depot.users.tazjin.blog.oldRedirects} + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload" always; + + location ~* \.(webp|woff2)$ { + add_header Cache-Control "public, max-age=31536000"; + } + + location /blog/ { + alias ${depot.users.tazjin.blog.rendered}/; + + if ($request_uri ~ ^/(.*)\.html$) { + return 302 /$1; + } + + try_files $uri $uri.html $uri/ =404; + } + + location = /tazjin { + return 200 "tazjin"; + } + + location /blobs/ { + alias /var/www/blobs/; + } + ''; + }; + + virtualHosts.cgit-old = nginxRedirect { + from = "git.tazj.in"; + to = "code.tvl.fyi"; + acmeHost = "tazj.in"; + }; + + virtualHosts.cs-old = nginxRedirect { + from = "cs.tazj.in"; + to = "cs.tvl.fyi"; + acmeHost = "tazj.in"; + }; + }; + + # Timer units that can be started with systemd-run to set my alarm. + systemd.user.services.light-alarm = { + script = "${depot.fun.idual.script}/bin/idualctl wakey"; + postStart = "${pkgs.systemd}/bin/systemctl --user stop light-alarm.timer"; + serviceConfig = { + Type = "oneshot"; + }; + }; + + system.stateVersion = "19.09"; +}) diff --git a/users/tazjin/nixos/default.nix b/users/tazjin/nixos/default.nix new file mode 100644 index 000000000000..04123a3b5ab1 --- /dev/null +++ b/users/tazjin/nixos/default.nix @@ -0,0 +1,10 @@ +{ depot, lib, ... }: + +let systemFor = sys: (depot.ops.nixos.nixosFor sys).system; +in { + camdenSystem = systemFor depot.users.tazjin.nixos.camden; + frogSystem = systemFor depot.users.tazjin.nixos.frog; + tverskoySystem = systemFor depot.users.tazjin.nixos.tverskoy; + + meta.targets = [ "camdenSystem" "frogSystem" "tverskoySystem" ]; +} diff --git a/users/tazjin/nixos/frog/default.nix b/users/tazjin/nixos/frog/default.nix new file mode 100644 index 000000000000..b3c803c87131 --- /dev/null +++ b/users/tazjin/nixos/frog/default.nix @@ -0,0 +1,285 @@ +{ depot, lib, pkgs, ... }: + +config: let + inherit (pkgs) lieer; + + quasselClient = pkgs.quassel.override { + client = true; + enableDaemon = false; + monolithic = false; + }; +in lib.fix(self: { + imports = [ + "${depot.path}/ops/modules/v4l2loopback.nix" + ]; + + boot = { + tmpOnTmpfs = true; + kernelModules = [ "kvm-amd" ]; + + loader = { + systemd-boot.enable = true; + efi.canTouchEfiVariables = true; + }; + + initrd = { + luks.devices.frog-crypt.device = "/dev/disk/by-label/frog-crypt"; + availableKernelModules = [ "xhci_pci" "ahci" "nvme" "usb_storage" "usbhid" "sd_mod" ]; + kernelModules = [ "dm-snapshot" ]; + }; + + kernelPackages = pkgs.linuxPackages_latest; + kernel.sysctl = { + "kernel.perf_event_paranoid" = -1; + }; + + # Enable this again if frog is put back into use ... + # + # kernelPatches = [ + # depot.third_party.kernelPatches.trx40_usb_audio + # ]; + }; + + hardware = { + cpu.amd.updateMicrocode = true; + enableRedistributableFirmware = true; + opengl = { + enable = true; + driSupport = true; + driSupport32Bit = true; + }; + + pulseaudio = { + enable = true; + package = pkgs.pulseaudioFull; + }; + + bluetooth = { + enable = true; + }; + }; + + nix = { + maxJobs = 48; + binaryCaches = ["ssh://nix-ssh@whitby.tvl.fyi"]; + binaryCachePublicKeys = ["cache.tvl.fyi:fd+9d1ceCPvDX/xVhcfv8nAa6njEhAGAEe+oGJDEeoc="]; + }; + + networking = { + hostName = "frog"; + useDHCP = true; + + # Don't use ISP's DNS servers: + nameservers = [ + "8.8.8.8" + "8.8.4.4" + ]; + + firewall.enable = false; + }; + + # Generate an immutable /etc/resolv.conf from the nameserver settings + # above (otherwise DHCP overwrites it): + environment.etc."resolv.conf" = with lib; { + source = pkgs.writeText "resolv.conf" '' + ${concatStringsSep "\n" (map (ns: "nameserver ${ns}") self.networking.nameservers)} + options edns0 + ''; + }; + + time.timeZone = "Europe/London"; + + fileSystems = { + "/".device = "/dev/disk/by-label/frog-root"; + "/boot".device = "/dev/disk/by-label/BOOT"; + "/home".device = "/dev/disk/by-label/frog-home"; + }; + + # Configure user account + users.extraUsers.tazjin = { + extraGroups = [ "wheel" "audio" "docker" ]; + isNormalUser = true; + uid = 1000; + shell = pkgs.fish; + }; + + security.sudo = { + enable = true; + extraConfig = "wheel ALL=(ALL:ALL) SETENV: ALL"; + }; + + fonts = { + fonts = with pkgs; [ + corefonts + dejavu_fonts + jetbrains-mono + noto-fonts-cjk + noto-fonts-emoji + ]; + + fontconfig = { + hinting.enable = true; + subpixel.lcdfilter = "light"; + + defaultFonts = { + monospace = [ "JetBrains Mono" ]; + }; + }; + }; + + # Configure location (Vauxhall, London) for services that need it. + location = { + latitude = 51.4819109; + longitude = -0.1252998; + }; + + programs.fish.enable = true; + programs.ssh.startAgent = true; + + services.redshift.enable = true; + services.openssh.enable = true; + services.fstrim.enable = true; + services.blueman.enable = true; + + # Required for Yubikey usage as smartcard + services.pcscd.enable = true; + services.udev.packages = [ + pkgs.yubikey-personalization + ]; + + # Enable Docker for Nixery testing + virtualisation.docker = { + enable = true; + autoPrune.enable = true; + }; + + services.xserver = { + enable = true; + layout = "us"; + xkbOptions = "caps:super"; + exportConfiguration = true; + videoDrivers = [ "amdgpu" ]; + displayManager = { + # Give EXWM permission to control the session. + sessionCommands = "${pkgs.xorg.xhost}/bin/xhost +SI:localuser:$USER"; + + lightdm.enable = true; + lightdm.greeters.gtk.clock-format = "%H·%M"; # TODO(tazjin): TZ? + }; + + windowManager.session = lib.singleton { + name = "exwm"; + start = "${depot.users.tazjin.emacs}/bin/tazjins-emacs"; + }; + }; + + # Do not restart the display manager automatically + systemd.services.display-manager.restartIfChanged = lib.mkForce false; + + # clangd needs more than ~2GB in the runtime directory to start up + services.logind.extraConfig = '' + RuntimeDirectorySize=16G + ''; + + # Configure email setup + systemd.user.services.lieer-tazjin = { + description = "Synchronise mail@tazj.in via lieer"; + script = "${lieer}/bin/gmi sync"; + + serviceConfig = { + WorkingDirectory = "%h/mail/account.tazjin"; + Type = "oneshot"; + }; + }; + + systemd.user.timers.lieer-tazjin = { + wantedBy = [ "timers.target" ]; + + timerConfig = { + OnActiveSec = "1"; + OnUnitActiveSec = "180"; + }; + }; + + environment.systemPackages = + # programs from the depot + (with depot; [ + fun.idual.script + fun.uggc + lieer + ops.kontemplate + quasselClient + third_party.git + tools.nsfv-setup + users.tazjin.emacs + ]) ++ + + # programs from nixpkgs + (with pkgs; [ + age + bat + chromium + clang-manpages + clang-tools_11 + clang_11 + curl + direnv + dnsutils + emacs27 # mostly for emacsclient + exa + fd + file + gdb + gnupg + go + google-chrome + google-cloud-sdk + htop + hyperfine + i3lock + iftop + imagemagick + jq + kubectl + linuxPackages.perf + manpages + miller + msmtp + nix-prefetch-github + notmuch + obs-studio + openssh + openssl + pass + pavucontrol + pciutils + pinentry + pinentry-emacs + pmutils + pwgen + ripgrep + rustup + screen + scrot + spotify + tokei + transmission + tree + unzip + usbutils + v4l-utils + vlc + xclip + xsecurelock + yubico-piv-tool + yubikey-personalization + zoxide + + # Commented out because of interim breakage: + # steam + # lutris + ]); + + # ... and other nonsense. + system.stateVersion = "20.03"; +}) diff --git a/users/tazjin/nixos/tverskoy/default.nix b/users/tazjin/nixos/tverskoy/default.nix new file mode 100644 index 000000000000..6dad04d609ce --- /dev/null +++ b/users/tazjin/nixos/tverskoy/default.nix @@ -0,0 +1,415 @@ +{ depot, lib, pkgs, ... }: + +config: let + quasselClient = pkgs.quassel.override { + client = true; + enableDaemon = false; + monolithic = false; + }; + + # Use a screen lock command that resets the keyboard layout + # before locking, to avoid locking me out when the layout is + # in Russian. + screenLock = pkgs.writeShellScriptBin "tazjin-screen-lock" '' + ${pkgs.xorg.setxkbmap}/bin/setxkbmap us + ${pkgs.xorg.setxkbmap}/bin/setxkbmap -option caps:super + exec ${pkgs.xsecurelock}/bin/xsecurelock + ''; +in lib.fix(self: { + imports = [ + "${depot.third_party.impermanence}/nixos.nix" + "${depot.path + "/ops/modules/automatic-gc.nix"}" + "${pkgs.home-manager.src}/nixos" + ]; + + nix = { + binaryCachePublicKeys = [ + "cache.tvl.su:kjc6KOMupXc1vHVufJUoDUYeLzbwSr9abcAKdn/U1Jk=" + ]; + + binaryCaches = [ + "https://cache.tvl.su" + ]; + }; + + boot = rec { + initrd.availableKernelModules = [ "nvme" "ehci_pci" "xhci_pci" "usb_storage" "sd_mod" "rtsx_pci_sdmmc" ]; + initrd.kernelModules = [ ]; + + # Restore /home to the blank snapshot, erasing all ephemeral data. + initrd.postDeviceCommands = lib.mkAfter '' + zfs rollback -r zpool/ephemeral/home@tazjin-clean + ''; + + # Install thinkpad modules for TLP + extraModulePackages = [ kernelPackages.acpi_call ]; + + kernelModules = [ "kvm-amd" "i2c_dev" ]; + kernelPackages = pkgs.linuxPackages_latest; + loader.systemd-boot.enable = true; + loader.efi.canTouchEfiVariables = true; + zfs.enableUnstable = true; + }; + + fileSystems = { + "/" = { + device = "tmpfs"; + fsType = "tmpfs"; + options = [ "defaults" "size=4G" "mode=755" ]; + }; + + "/home" = { + device = "zpool/ephemeral/home"; + fsType = "zfs"; + }; + + "/nix" = { + device = "zpool/local/nix"; + fsType = "zfs"; + }; + + "/depot" = { + device = "zpool/safe/depot"; + fsType = "zfs"; + }; + + "/persist" = { + device = "zpool/safe/persist"; + fsType = "zfs"; + neededForBoot = true; + }; + + # SD card + "/mnt" = { + device = "/dev/disk/by-uuid/c602d703-f1b9-4a44-9e45-94dfe24bdaa8"; + fsType = "ext4"; + }; + + "/boot" = { + device = "/dev/disk/by-uuid/BF4F-388B"; + fsType = "vfat"; + }; + }; + + hardware = { + cpu.amd.updateMicrocode = true; + enableRedistributableFirmware = true; + bluetooth.enable = true; + + opengl = { + enable = true; + extraPackages = with pkgs; [ + vaapiVdpau + libvdpau-va-gl + ]; + }; + }; + + networking = { + hostName = "tverskoy"; + hostId = "3c91827f"; + domain = "tvl.su"; + useDHCP = false; + networkmanager.enable = true; + firewall.enable = false; + + nameservers = [ + "8.8.8.8" + "8.8.4.4" + ]; + }; + + fonts = { + fonts = with pkgs; [ + corefonts + dejavu_fonts + jetbrains-mono + noto-fonts-cjk + noto-fonts-emoji + ]; + + fontconfig = { + hinting.enable = true; + subpixel.lcdfilter = "light"; + + defaultFonts = { + monospace = [ "JetBrains Mono" ]; + }; + }; + }; + + environment.persistence."/persist" = { + directories = [ + "/etc/NetworkManager/system-connections" + "/etc/mullvad-vpn" + "/var/cache/mullvad-vpn" + "/var/lib/bluetooth" + "/var/lib/systemd/coredump" + "/var/log" + ]; + files = [ + "/etc/machine-id" + ]; + }; + + security.rtkit.enable = true; + + services = { + pipewire = { + enable = true; + alsa.enable = true; + pulse.enable = true; + }; + + redshift.enable = true; + blueman.enable = true; + mullvad-vpn.enable = true; + fwupd.enable = true; + printing.enable = true; + + # expose i2c device as /dev/i2c-amdgpu-dm and make it user-accessible + # this is required for sending control commands to the Dasung screen. + udev.extraRules = '' + SUBSYSTEM=="i2c-dev", ACTION=="add", DEVPATH=="/devices/pci0000:00/0000:00:08.1/0000:06:00.0/i2c-5/i2c-dev/i2c-5", SYMLINK+="i2c-amdgpu-dm", TAG+="uaccess" + ''; + + # Configure TLP to keep battery charge between 40-70% while + # plugged in to the wall (thanks etu for the recommendation). + tlp = { + enable = true; + settings.START_CHARGE_THRESH_BAT0 = 40; + settings.STOP_CHARGE_THRESH_BAT0 = 70; + }; + + xserver = { + enable = true; + layout = "us"; + xkbOptions = "caps:super"; + videoDrivers = [ "amdgpu" ]; + + libinput.enable = true; + + displayManager = { + # Give EXWM permission to control the session. + sessionCommands = "${pkgs.xorg.xhost}/bin/xhost +SI:localuser:$USER"; + lightdm.enable = true; + # lightdm.greeters.gtk.clock-format = "%H:%M"; # TODO(tazjin): TZ? + }; + + windowManager.session = lib.singleton { + name = "exwm"; + start = "${depot.users.tazjin.emacs}/bin/tazjins-emacs"; + }; + }; + + # Automatically collect garbage from the Nix store. + depot.automatic-gc = { + enable = true; + interval = "1 hour"; + diskThreshold = 16; # GiB + maxFreed = 10; # GiB + preserveGenerations = "14d"; + }; + }; + + # Automatically detect location to use for redshift + location.provider = "geoclue2"; + + # Do not restart the display manager automatically + systemd.services.display-manager.restartIfChanged = lib.mkForce false; + + time.timeZone = "Europe/Moscow"; + + users.users.tazjin = { + isNormalUser = true; + createHome = true; + extraGroups = [ "wheel" "networkmanager" "video" "adbusers" ]; + uid = 1000; + shell = pkgs.fish; + initialHashedPassword = "$6$d3FywUNCuZnJ4l.$ZW2ul59MLYon1v1xhC3lTJZfZ91lWW6Tpi13MpME0cJcYZNrsx7ABdgQRn.K05awruG2Y9ARAzURnmiJ31WTS1"; + }; + + programs = { + fish.enable = true; + light.enable = true; + ssh.startAgent = true; + mosh.enable = true; + steam.enable = true; + adb.enable = true; + + # Required by impermanence + fuse.userAllowOther = true; + }; + + environment.systemPackages = + # programs from the depot + (with depot; [ + screenLock + tools.nsfv-setup + users.tazjin.emacs + third_party.agenix.cli + third_party.dfmt + ]) ++ + + # programs from nixpkgs + (with pkgs; [ + amber + bat + chromium + curl + ddcutil + direnv + dmd + dnsutils + emacs27-nox # emacsclient + exa + fd + file + firefox + gdb + gh + git + gnupg + google-chrome + gtk3 # for gtk-launch + htop + hyperfine + iftop + imagemagick + jq + lieer + manpages + mosh + msmtp + mullvad-vpn + networkmanagerapplet + nix-prefetch-github + nmap + notmuch + openssh + openssl + paperlike-go + pass + pavucontrol + pinentry + pinentry-emacs + pulseaudioLight # for pactl + pwgen + quasselClient + rink + ripgrep + rustup + screen + scrot + spotify + syncthing + tig + tokei + tree + unzip + vlc + whois + xsecurelock + zoxide + ]); + + systemd.user.services.lieer-tazjin = { + description = "Synchronise mail@tazj.in via lieer"; + script = "${pkgs.lieer}/bin/gmi sync"; + + serviceConfig = { + WorkingDirectory = "%h/mail/account.tazjin"; + Type = "oneshot"; + }; + }; + + systemd.user.timers.lieer-tazjin = { + wantedBy = [ "timers.target" ]; + + timerConfig = { + OnActiveSec = "1"; + OnUnitActiveSec = "180"; + }; + }; + + home-manager.useGlobalPkgs = true; + home-manager.users.tazjin = { config, lib, ... }: { + imports = [ "${depot.third_party.impermanence}/home-manager.nix" ]; + + home.persistence."/persist/tazjin/home" = { + allowOther = true; + + directories = [ + ".cargo" + ".config/google-chrome" + ".config/quassel-irc.org" + ".config/spotify" + ".config/syncthing" + ".elfeed" + ".gnupg" + ".local/share/Steam" + ".local/share/direnv" + ".local/share/fish" + ".local/share/zoxide" + ".mozilla/firefox" + ".password-store" + ".rustup" + ".ssh" + ".steam" + ".telega" + "go" + "mail" + ]; + + files = [ + ".config/mimeapps.list" + ".notmuch-config" + ]; + }; + + programs.git = { + enable = true; + userName = "Vincent Ambo"; + userEmail = "mail@tazj.in"; + extraConfig = { + pull.rebase = true; + init.defaultBranch = "canon"; + }; + }; + + programs.fish = { + enable = true; + interactiveShellInit = '' + ${pkgs.zoxide}/bin/zoxide init fish | source + ''; + }; + + services.screen-locker = { + enable = true; + enableDetectSleep = true; + inactiveInterval = 10; # minutes + lockCmd = "${screenLock}/bin/tazjin-screen-lock"; + }; + + services.picom = { + enable = true; + vSync = true; + backend = "glx"; + }; + + # Enable the dunst notification daemon, but force the + # configuration file separately instead of going via the strange + # Nix->dunstrc encoding route. + services.dunst.enable = true; + xdg.configFile."dunst/dunstrc" = { + source = depot.users.tazjin.dotfiles.dunstrc; + onChange = '' + ${pkgs.procps}/bin/pkill -u "$USER" ''${VERBOSE+-e} dunst || true + ''; + }; + + systemd.user.startServices = true; + }; + + system.stateVersion = "20.09"; +}) diff --git a/users/tazjin/presentations/bootstrapping-2018/README.md b/users/tazjin/presentations/bootstrapping-2018/README.md new file mode 100644 index 000000000000..e9573ae3f2e1 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/README.md @@ -0,0 +1,5 @@ +These are the slides for a talk I gave at the Norwegian Unix User Group on +2018-03-13. + +There is more information and a recording on the [event +page](https://www.nuug.no/aktiviteter/20180313-reproduible-compiler/). diff --git a/users/tazjin/presentations/bootstrapping-2018/default.nix b/users/tazjin/presentations/bootstrapping-2018/default.nix new file mode 100644 index 000000000000..0dff14b2a1a6 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/default.nix @@ -0,0 +1,50 @@ +# This derivation builds the LaTeX presentation. + +{ pkgs, ... }: + +with pkgs; + +let tex = texlive.combine { + inherit (texlive) + beamer + beamertheme-metropolis + etoolbox + euenc + extsizes + fontspec + lualibs + luaotfload + luatex + minted + ms + pgfopts + scheme-basic + translator; +}; +in stdenv.mkDerivation { + name = "nuug-bootstrapping-slides"; + src = ./.; + + FONTCONFIG_FILE = makeFontsConf { + fontDirectories = [ fira fira-code fira-mono ]; + }; + + buildInputs = [ tex fira fira-code fira-mono ]; + buildPhase = '' + # LaTeX needs a cache folder in /home/ ... + mkdir home + export HOME=$PWD/home + # ${tex}/bin/luaotfload-tool -ufv + + # As usual, TeX needs to be run twice ... + function run() { + ${tex}/bin/lualatex presentation.tex + } + run && run + ''; + + installPhase = '' + mkdir -p $out + cp presentation.pdf $out/ + ''; +} diff --git a/users/tazjin/presentations/bootstrapping-2018/drake-meme.png b/users/tazjin/presentations/bootstrapping-2018/drake-meme.png new file mode 100644 index 000000000000..4b036754384f --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/drake-meme.png Binary files differdiff --git a/users/tazjin/presentations/bootstrapping-2018/nixos-logo.png b/users/tazjin/presentations/bootstrapping-2018/nixos-logo.png new file mode 100644 index 000000000000..ce0c98c2cabb --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/nixos-logo.png Binary files differdiff --git a/users/tazjin/presentations/bootstrapping-2018/notes.org b/users/tazjin/presentations/bootstrapping-2018/notes.org new file mode 100644 index 000000000000..363d75352e62 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/notes.org @@ -0,0 +1,89 @@ +#+TITLE: Bootstrapping, reproducibility, etc. +#+AUTHOR: Vincent Ambo +#+DATE: <2018-03-10 Sat> + +* Compiler bootstrapping + This section contains notes about compiler bootstrapping, the + history thereof, which compilers need it - and so on: + +** C + +** Haskell + - self-hosted compiler (GHC) + +** Common Lisp + CL is fairly interesting in this space because it is a language + that is defined via an ANSI standard that compiler implementations + normally actually follow! + + CL has several ecosystem components that focus on making + abstracting away implementation-specific calls and if a self-hosted + compiler is written in CL using those components it can be + cross-bootstrapped. + +** Python + +* A note on runtimes + Sometimes the compiler just isn't enough ... + +** LLVM +** JVM + +* References + https://github.com/mame/quine-relay + https://manishearth.github.io/blog/2016/12/02/reflections-on-rusting-trust/ + https://tests.reproducible-builds.org/debian/reproducible.html + +* Slide thoughts: + 1. Hardware trust has been discussed here a bunch, most recently + during the puri.sm talk. Hardware trust is important, as we see + with IME, but it's striking that people often take a leap to "I'm + now on my trusted Debian with free software". + + Unless you built it yourself from scratch (Spoiler: you haven't) + you're placing trust in what is basically foreign binary blobs. + + Agenda: Implications/attack vectors of this, state of the chicken + & egg, the topic of reproducibility, what can you do? (Nix!) + + 2. Chicken-and-egg issue + + It's an important milestone for a language to become self-hosted: + You begin doing a kind of dogfeeding, you begin to enforce + reliability & consistency guarantees to avoid having to redo your + own codebase constantly and so on. + + However, the implication is now that you need your own compiler + to compile itself. + + Common examples: + - C/C++ compilers needed to build C/C++ compilers: + + GCC 4.7 was the last version of GCC that could be built with a + standard C-compiler, nowadays it is mostly written in C++. + + Certain versions of GCC can be built with LLVM/Clang. + + Clang/LLVM can be compiled by itself and also GCC. + + - Rust was originally written in OCAML but moved to being + self-hosted in 2011. Currently rustc-releases are always built + with a copy of the previous release. + + It's relatively new so we can build the chain all the way. + + Notable exceptions: Some popular languages are not self-hosted, + for example Clojure. Languages also have runtimes, which may be + written in something else (e.g. Haskell -> C runtime) +* How to help: + Most of this advice is about reproducible builds, not bootstrapping, + as that is a much harder project. + + - fix reproducibility issues listed in Debian's issue tracker (focus + on non-Debian specific ones though) + - experiment with NixOS / GuixSD to get a better grasp on the + problem space of reproducibility + + If you want to contribute to bootstrapping, look at + bootstrappable.org and their wiki. Several initiatives such as MES + could need help! diff --git a/users/tazjin/presentations/bootstrapping-2018/presentation.pdf b/users/tazjin/presentations/bootstrapping-2018/presentation.pdf new file mode 100644 index 000000000000..7f435fe5b539 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/presentation.pdf Binary files differdiff --git a/users/tazjin/presentations/bootstrapping-2018/presentation.tex b/users/tazjin/presentations/bootstrapping-2018/presentation.tex new file mode 100644 index 000000000000..d3aa61337554 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/presentation.tex @@ -0,0 +1,251 @@ +\documentclass[12pt]{beamer} +\usetheme{metropolis} +\newenvironment{code}{\ttfamily}{\par} +\title{Where does \textit{your} compiler come from?} +\date{2018-03-13} +\author{Vincent Ambo} +\institute{Norwegian Unix User Group} +\begin{document} + \maketitle + + %% Slide 1: + \section{Introduction} + + %% Slide 2: + \begin{frame}{Chicken and egg} + Self-hosted compilers are often built using themselves, for example: + + \begin{itemize} + \item C-family compilers bootstrap themselves \& each other + \item (Some!) Common Lisp compilers can bootstrap each other + \item \texttt{rustc} bootstraps itself with a previous version + \item ... same for many other languages! + \end{itemize} + \end{frame} + + \begin{frame}{Chicken, egg and ... lizard?} + It's not just compilers: Languages have runtimes, too. + + \begin{itemize} + \item JVM is implemented in C++ + \item Erlang-VM is C + \item Haskell runtime is C + \end{itemize} + + ... we can't ever get away from C, can we? + \end{frame} + + %% Slide 3: + \begin{frame}{Trusting Trust} + \begin{center} + \huge{Could this be exploited?} + \end{center} + \end{frame} + + %% Slide 4: + \begin{frame}{Short interlude: A quine} + \begin{center} + \begin{code} + ((lambda (x) (list x (list 'quote x))) + \newline\vspace*{6mm} '(lambda (x) (list x (list 'quote x)))) + \end{code} + \end{center} + \end{frame} + + %% Slide 5: + \begin{frame}{Short interlude: Quine Relay} + \begin{center} + \includegraphics[ + keepaspectratio=true, + height=\textheight + ]{quine-relay.png} + \end{center} + \end{frame} + + %% Slide 6: + \begin{frame}{Trusting Trust} + An attack described by Ken Thompson in 1983: + + \begin{enumerate} + \item Modify a compiler to detect when it's compiling itself. + \item Let the modification insert \textit{itself} into the new compiler. + \item Add arbitrary attack code to the modification. + \item \textit{Optional!} Remove the attack from the source after compilation. + \end{enumerate} + \end{frame} + + %% Slide 7: + \begin{frame}{Damage potential?} + \begin{center} + \large{Let your imagination run wild!} + \end{center} + \end{frame} + + %% Slide 8: + \section{Countermeasures} + + %% Slide 9: + \begin{frame}{Diverse Double-Compiling} + Assume we have: + + \begin{itemize} + \item Target language compilers $A$ and $T$ + \item The source code of $A$: $ S_{A} $ + \end{itemize} + \end{frame} + + %% Slide 10: + \begin{frame}{Diverse Double-Compiling} + Apply the first stage (functional equivalence): + + \begin{itemize} + \item $ X = A(S_{A})$ + \item $ Y = T(S_{A})$ + \end{itemize} + + Apply the second stage (bit-for-bit equivalence): + + \begin{itemize} + \item $ V = X(S_{A})$ + \item $ W = Y(S_{A})$ + \end{itemize} + + Now we have a new problem: Reproducibility! + \end{frame} + + %% Slide 11: + \begin{frame}{Reproducibility} + Bit-for-bit equivalent output is hard, for example: + + \begin{itemize} + \item Timestamps in output artifacts + \item Non-deterministic linking order in concurrent builds + \item Non-deterministic VM \& memory states in outputs + \item Randomness in builds (sic!) + \end{itemize} + \end{frame} + + \begin{frame}{Reproducibility} + \begin{center} + Without reproducibility, we can never trust that any shipped + binary matches the source code! + \end{center} + \end{frame} + + %% Slide 12: + \section{(Partial) State of the Union} + + \begin{frame}{The Desired State} + \begin{center} + \begin{enumerate} + \item Full-source bootstrap! + \item All packages reproducible! + \end{enumerate} + \end{center} + \end{frame} + + %% Slide 13: + \begin{frame}{Bootstrapping Debian} + \begin{itemize} + \item Sparse information on the Debian-wiki + \item Bootstrapping discussions mostly resolve around new architectures + \item GCC is compiled by depending on previous versions of GCC + \end{itemize} + \end{frame} + + \begin{frame}{Reproducing Debian} + Debian has a very active effort for reproducible builds: + + \begin{itemize} + \item Organised information about reproducibility status + \item Over 90\% reproducibility in Debian package base! + \end{itemize} + \end{frame} + + \begin{frame}{Short interlude: Nix} + \begin{center} + \includegraphics[ + keepaspectratio=true, + height=0.7\textheight + ]{nixos-logo.png} + \end{center} + \end{frame} + + \begin{frame}{Short interlude: Nix} + \begin{center} + \includegraphics[ + keepaspectratio=true, + height=0.90\textheight + ]{drake-meme.png} + \end{center} + \end{frame} + + \begin{frame}{Short interlude: Nix} + \begin{center} + \includegraphics[ + keepaspectratio=true, + height=0.7\textheight + ]{nixos-logo.png} + \end{center} + \end{frame} + + \begin{frame}{Bootstrapping NixOS} + Nix evaluation can not recurse forever: The bootstrap can not + simply depend on a previous GCC. + + Workaround: \texttt{bootstrap-tools} tarball from a previous + binary cache is fetched and used. + + An unfortunate magic binary blob ... + \end{frame} + + \begin{frame}{Reproducing NixOS} + Not all reproducibility patches have been ported from Debian. + + However: Builds are fully repeatable via the Nix fundamentals! + \end{frame} + + \section{Future Developments} + + \begin{frame}{Bootstrappable: stage0} + Hand-rolled ``Cthulhu's Path to Madness'' hex-programs: + + \begin{itemize} + \item No non-auditable binary blobs + \item Aims for understandability by 70\% of programmers + \item End goal is a full-source bootstrap of GCC + \end{itemize} + \end{frame} + + + \begin{frame}{Bootstrappable: MES} + Bootstrapping the ``Maxwell Equations of Software'': + + \begin{itemize} + \item Minimal C-compiler written in Scheme + \item Minimal Scheme-interpreter (currently in C, but intended to + be rewritten in stage0 macros) + \item End goal is full-source bootstrap of the entire GuixSD + \end{itemize} + \end{frame} + + \begin{frame}{Other platforms} + \begin{itemize} + \item Nix for Darwin is actively maintained + \item F-Droid Android repository works towards fully reproducible + builds of (open) Android software + \item Mobile devices (phones, tablets, etc.) are a lost cause at + the moment + \end{itemize} + \end{frame} + + \begin{frame}{Thanks!} + Resources: + \begin{itemize} + \item bootstrappable.org + \item reproducible-builds.org + \end{itemize} + + @tazjin | mail@tazj.in + \end{frame} +\end{document} diff --git a/users/tazjin/presentations/bootstrapping-2018/quine-relay.png b/users/tazjin/presentations/bootstrapping-2018/quine-relay.png new file mode 100644 index 000000000000..5644dc3900e3 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/quine-relay.png Binary files differdiff --git a/users/tazjin/presentations/bootstrapping-2018/result.pdfpc b/users/tazjin/presentations/bootstrapping-2018/result.pdfpc new file mode 100644 index 000000000000..b0fa6c9a0ef8 --- /dev/null +++ b/users/tazjin/presentations/bootstrapping-2018/result.pdfpc @@ -0,0 +1,142 @@ +[file] +result +[last_saved_slide] +10 +[font_size] +20000 +[notes] +### 1 +- previous discussions of hardware trust (e.g. purism presentation) +- people leap to "now I'm on my trusted Debian!" +- unless you built it from scratch (spoiler: you haven't) you're *trusting* someone + +Agenda: Implications of trust with focus on bootstrap paths and reproducibility, plus how you can help.### 2 +self-hosting: +- C-family: GCC pre/post 4.7, Clang +- Common Lisp: Sunshine land! (with SBCL) +- rustc: Bootstrap based on previous versions (C++ transpiler underway!) +- many other languages also work this way! + +(Noteable counterexample: Clojure is written in Java!)### 3 + +- compilers are just one bit, the various runtimes exist, too!### 4 + +Could this be exploited? + +People don't think about where their compiler comes from. + +Even if they do, they may only go so far as to say "I'll just recompile it using <other compiler>". + +Unfortunately, spoiler alert, life isn't that easy in the computer world and yes, exploitation is possible.### 5 + +- describe what a quine is +- classic Lisp quine +- explain demo quine +- demo demo quine + +- this is interesting, but not useful - can quines do more than that?### 6 + +- quine-relay: "art project" with 128-language circular quine + +- show source of quine-relay + +- (demo quine relay?) + +- side-note: this program is very, very trustworthy!### 7 + +Ken Thompson (designer of UNIX and a couple other things!) received Turing award in 1983, and described attack in speech. + +- figure out how to detect self-compilation +- make that modification a quine +- insert modification into new compiler +- add attack code to modification +- remove attack from source, distributed binary will still be compromised! it's like evolution :)### 8 + +damage potential is basically infinite: + +- classic "login" attack +=> also applicable to other credentials + +- attack (weaken) crypto algorithms + +- you can probably think of more!### 10 + +idea being: potential vulnerability would have to work across compilers: + +the more compilers we can introduce (e.g. more architectures, different versions, different compilers), the harder it gets for a vulnerability to survive all of those + +The more compilers, the merrier! Lisps are pretty good at this.### 11 + +if we get a bit-mismatch after DDC, not all hope is lost: Maybe the thing just isn't reproducible! + +- many reasons for failures +- timestamps are a classic! artifacts can be build logs, metadata in ZIP-files or whatever +- non-determinism is the devil +- sometimes people actively introduce build-randomness (NaCl)### 12 + +- Does that binary download on the project's website really match the source? + +- Your Linux packages are signed by someone - cool - but what does that mean?### 13 + +Two things should be achieved - gross oversimplification - to get to the ideal "desired state of the union": + +1. full-source bootstrap: without ever introducing any binaries, go from nothing to a full Linux distribution + +2. when packages are distributed, we should be able to know the expected output of a source package beforehand + +=> suddenly binary distributions become a cache! But more on Nix later.### 14 + +- Debian project does not seem as concerned with bootstrapping as with reproducibility +- Debian mostly bootstraps on new architectures (using cross-compilation and similar techniques, from an existing binary base) +- core bootstrap (GCC & friends) is performed with previous Debian version and depending on GCC### 15 + +... however! Debian cares about reproducibility. + +- automated testing of reproducibility +- information about the status of all packages is made available in repos +- Over 90% packages of packages are reproducible! + +< show reproducible builds website > + +Debian is still fundamentally a binary distribution though, but it doesn't have to be that way.### 16 + +Nix - a purely functional package manager + +It's not a new project (10+ years), been discussed here before, has multiple components: package manager, language, NixOS. + +Instead of describing *how* to build a thing, Nix describes *what* to build:### 17 +### 19 + +In Nix, it's impossible to say "GCC is the result of applying GCC to the GCC source", because that happens to be infinite recursion. + +Bootstrapping in Nix works by introducing a binary pinned by its full-hash, which was built on some previous Nix version. + +Unfortunately also just a magic binary blob ... ### 20 + +NixOS is not actively porting all of Debian's reproducibility patches, but builds are fully repeatable: + +- introducing a malicious compiler would produce a different input hash -> different package + +Future slide: hope is not lost! Things are underway.### 21 + +- bootstrappable.org (demo?) is an umbrella page for several projects working on bootstrappability + +- stage0 is an important piece: manually, small, auditable Hex programs to get to a Hex macro expander + +- end goal is a full-source bootrap, but pieces are missing### 22 + +MES is out of the GuixSD circles (explain Guix, GNU Hurd joke) + +- idea being that once you have a Lisp, you have all of computing (as Alan Key said) + +- includes MesCC in Scheme -> can *almost* make a working tinyCC -> can *almost* make a working gcc 4.7 + +- minimal Scheme interpreter, currently built in C to get the higher-level stuff to work, goal is rewrite in hex +- bootstrapping Guix is the end goal### 23 + +- userspace in Darwin has a Nix project +- unsure about other BSDs, but if anyone knows - input welcome! +- F-Droid has reproducible Android packages, but that's also userspace only +- All other mobile platforms are a lost cause + +Generally, all closed-source software is impossible to trust. diff --git a/users/tazjin/presentations/erlang-2016/.skip-subtree b/users/tazjin/presentations/erlang-2016/.skip-subtree new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/.skip-subtree diff --git a/users/tazjin/presentations/erlang-2016/README.md b/users/tazjin/presentations/erlang-2016/README.md new file mode 100644 index 000000000000..e1b6c83b99cc --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/README.md @@ -0,0 +1,6 @@ +These are the slides for a presentation I gave for the Oslo javaBin meetup in +2016. + +Unfortunately there is no recording of the presentation due to a technical error +(video was recorded, but no audio). This is a bit of a shame because I think +these are some of the best slides I've ever made. diff --git a/users/tazjin/presentations/erlang-2016/presentation.md b/users/tazjin/presentations/erlang-2016/presentation.md new file mode 100644 index 000000000000..526564b88268 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/presentation.md @@ -0,0 +1,222 @@ +slidenumbers: true +Erlang. +====== + +### Fault-tolerant, concurrent programming. + +--- + +## A brief history of Erlang + +--- + + + + +^ Telefontornet in Stockholm, around 1890. Used until 1913. + +--- + + + +^ Telephones were operated manually at Switchboards. Anyone old enough to remember? I'm certainly not. + +--- + + + +^ Eventually we did that in software, and we got better at it over time. Ericsson AXD 301, first commercial Erlang switch. But lets take a step back. + +--- + +## Phone switches must be ... + +Highly concurrent + +Fault-tolerant + +Distributed + +(Fast!) + + + +--- + +## ... and so is Erlang! + +--- + +## Erlang as a whole: + +- Unique process model (actors!) +- Built-in fault-tolerance & error handling +- Distributed processes +- Three parts! + +--- + +## Part 1: Erlang, the language + +- Functional +- Prolog-inspired syntax +- Everything is immutable +- *Extreme* pattern-matching + +--- +### Hello Joe + +```erlang +hello_joe. +``` + +--- +### Hello Joe + +```erlang +-module(hello1). +-export([hello_joe/0]). + +hello_joe() -> + hello_joe. +``` + +--- +### Hello Joe + +```erlang +-module(hello1). +-export([hello_joe/0]). + +hello_joe() -> + hello_joe. + +% 1> c(hello1). +% {ok,hello1} +% 2> hello1:hello_joe(). +% hello_joe +``` + +--- +### Hello Joe + +```erlang +-module(hello2). +-export([hello/1]). + +hello(Name) -> + io:format("Hello ~s!~n", [Name]). + +% 3> c(hello2). +% {ok,hello2} +% 4> hello2:hello("Joe"). +% Hello Joe! +% ok +``` + +--- + +## [fit] Hello ~~world~~ Joe is boring! +## [fit] Lets do it with processes. + +--- +### Hello Server + +```erlang +-module(hello_server). +-export([start_server/0]). + +start_server() -> + spawn(fun() -> server() end). + +server() -> + receive + {greet, Name} -> + io:format("Hello ~s!~n", [Name]), + server() + end. +``` + +--- + +## [fit] Some issues with that ... + +- What about unused messages? +- What if the server crashes? + +--- + +## [fit] Part 2: Open Telecom Platform + +### **It's called Erlang/OTP for a reason.** + +--- + +# OTP: An Application Framework + +- Supervision - keep processes alive! + +- OTP Behaviours - common process patterns + +- Extensive standard library + +- Error handling, debuggers, testing, ... + +- Lots more! + +^ Standard library includes lots of things from simple network libraries over testing frameworks to cryptography, complete LDAP clients etc. + +--- + +# Supervision + + + +^ Supervision keeps processes alive, different restart behaviours, everything should be supervised to avoid "process" (and therefore memory) leaks + +--- + +# OTP Behaviours + +* `gen_server` +* `gen_statem` +* `gen_event` +* `supervisor` + +^ gen = generic. explain server, explain statem, event = event handling with registered handlers, supervisor ... + +--- + +`gen_server` + +--- + +## [fit] Part 3: BEAM + +### Bogdan/Bjørn Erlang Abstract machine + +--- + +## A VM for Erlang + +* Many were written, BEAM survived +* Concurrent garbage-collection +* Lower-level bytecode than JVM +* Very open to new languages + (Elixir, LFE, Joxa, ...) + +--- + +## What next? + +* Ole's talk, obviously! +* Learn You Some Erlang! + www.learnyousomeerlang.com +* Watch *Erlang the Movie* +* (soon!) Join the Oslo BEAM meetup group + +--- + +# [fit] Questions? + +`@tazjin` diff --git a/users/tazjin/presentations/erlang-2016/presentation.pdf b/users/tazjin/presentations/erlang-2016/presentation.pdf new file mode 100644 index 000000000000..ec8d996704b2 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/presentation.pdf Binary files differdiff --git a/users/tazjin/presentations/erlang-2016/src/hello.erl b/users/tazjin/presentations/erlang-2016/src/hello.erl new file mode 100644 index 000000000000..56404a0c5a20 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/src/hello.erl @@ -0,0 +1,5 @@ +-module(hello). +-export([hello_joe/0]). + +hello_joe() -> + hello_joe. diff --git a/users/tazjin/presentations/erlang-2016/src/hello1.erl b/users/tazjin/presentations/erlang-2016/src/hello1.erl new file mode 100644 index 000000000000..ca78261399e1 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/src/hello1.erl @@ -0,0 +1,5 @@ +-module(hello1). +-export([hello_joe/0]). + +hello_joe() -> + hello_joe. diff --git a/users/tazjin/presentations/erlang-2016/src/hello2.erl b/users/tazjin/presentations/erlang-2016/src/hello2.erl new file mode 100644 index 000000000000..2d1f6c84c401 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/src/hello2.erl @@ -0,0 +1,11 @@ +-module(hello2). +-export([hello/1]). + +hello(Name) -> + io:format("Hey ~s!~n", [Name]). + +% 3> c(hello2). +% {ok,hello2} +% 4> hello2:hello("Joe"). +% Hello Joe! +% ok diff --git a/users/tazjin/presentations/erlang-2016/src/hello_server.erl b/users/tazjin/presentations/erlang-2016/src/hello_server.erl new file mode 100644 index 000000000000..01df14ac57d5 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/src/hello_server.erl @@ -0,0 +1,12 @@ +-module(hello_server). +-export([start_server/0, server/0]). + +start_server() -> + spawn(fun() -> server() end). + +server() -> + receive + {greet, Name} -> + io:format("Hello ~s!~n", [Name]), + hello_server:server() + end. diff --git a/users/tazjin/presentations/erlang-2016/src/hello_server2.erl b/users/tazjin/presentations/erlang-2016/src/hello_server2.erl new file mode 100644 index 000000000000..24bb934ee503 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/src/hello_server2.erl @@ -0,0 +1,36 @@ +-module(hello_server2). +-behaviour(gen_server). +-compile(export_all). + +%%% Start callback for supervisor +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +%%% gen_server callbacks + +init([]) -> + {ok, sets:new()}. + +handle_call({greet, Name}, _From, State) -> + io:format("Hello ~s!~n", [Name]), + NewState = sets:add_element(Name, State), + {reply, ok, NewState}; + +handle_call({bye, Name}, _From, State) -> + io:format("Goodbye ~s!~n", [Name]), + NewState = sets:del_element(Name, State), + {reply, ok, NewState}. + +terminate(normal, State) -> + [io:format("Goodbye ~s!~n", [Name]) || Name <- State], + ok. + +%%% Unused gen_server callbacks +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +handle_cast(_Request, State) -> + {noreply, State}. diff --git a/users/tazjin/presentations/erlang-2016/src/hello_sup.erl b/users/tazjin/presentations/erlang-2016/src/hello_sup.erl new file mode 100644 index 000000000000..7fee0928c575 --- /dev/null +++ b/users/tazjin/presentations/erlang-2016/src/hello_sup.erl @@ -0,0 +1,24 @@ +-module(hello_sup). +-behaviour(supervisor). +-export([start_link/0, init/1]). + +%%% Module API + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +%%% Supervisor callbacks + +init([]) -> + Children = [hello_spec()], + {ok, { {one_for_one, 5, 10}, Children}}. + +%%% Private + +hello_spec() -> + #{id => hello_server2, + start => {hello_server2, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + module => [hello_server2]}. diff --git a/users/tazjin/presentations/servant-2016/Makefile b/users/tazjin/presentations/servant-2016/Makefile new file mode 100644 index 000000000000..96115ec2cbfc --- /dev/null +++ b/users/tazjin/presentations/servant-2016/Makefile @@ -0,0 +1,8 @@ +all: slides + +slides: + lualatex --shell-escape slides.tex + +clean: + rm -f slides.aux slides.log slides.nav \ + slides.out slides.toc slides.snm diff --git a/users/tazjin/presentations/servant-2016/README.md b/users/tazjin/presentations/servant-2016/README.md new file mode 100644 index 000000000000..8cfb04a42417 --- /dev/null +++ b/users/tazjin/presentations/servant-2016/README.md @@ -0,0 +1,7 @@ +These are the slides for my presentation about [servant][] at [Oslo Haskell][]. + +A full video recording of the presentation is available [on Vimeo][]. + +[servant]: https://haskell-servant.github.io/ +[Oslo Haskell]: http://www.meetup.com/Oslo-Haskell/events/227107530/ +[on Vimeo]: https://vimeo.com/153901805 diff --git a/users/tazjin/presentations/servant-2016/slides.pdf b/users/tazjin/presentations/servant-2016/slides.pdf new file mode 100644 index 000000000000..842a667e1bcc --- /dev/null +++ b/users/tazjin/presentations/servant-2016/slides.pdf Binary files differdiff --git a/users/tazjin/presentations/servant-2016/slides.pdfpc b/users/tazjin/presentations/servant-2016/slides.pdfpc new file mode 100644 index 000000000000..ed46003768c0 --- /dev/null +++ b/users/tazjin/presentations/servant-2016/slides.pdfpc @@ -0,0 +1,75 @@ +[file] +slides.pdf +[font_size] +10897 +[notes] +### 1 +13### 2 +Let's talk about servant, which is several things: +API description DSL, we'll speak about how this DSL works +and why it's at the type level + +Interpretations of the types resulting from that DSL, for example in +web servers or API clients + +Servant is commonly used or implementing services with APIs, or for accessing +other APIs with a simple, typed client +### 3 +Why type-level DSLs? +Type-level DSL: express *something*, e.g. endpoints of API, on type level by combining types. Types can be uninhabited + +Phil Wadler's: expression problem: things should be extensible both in the cases of a type, and in the functions operating on the type +Normal data types: can't add new constructors easily +Servant lifts thisup to simply allow the declaration of new types that can be included in the DSL, and new interpretations that can be attached to the types through typeclasses + +APIs become first-class citizens, can pass them around, combine them etc, they are separate from interpretations such as server implementations. In contrast, in most webframeworks, API declaration is implicit + +(Mention previous attemps at type-safe web, Yesod / web-routes + boomerang etc) +### 4 +Three extensions are necessary: +TypeOperators lets us use infix operators on the type level as constructors +DataKinds promotes new type declarations to the kind level, makes type-level literals (strings and natural numbers) available, lets us use type-level lists and pairs in combination with typeoperators +TypeFamilies: Type-level functions, map one set of types to another, come in two forms (type families, non-injective; data families, injective), more powerful than associated types +### 5 +Here you can see servant's general syntax, we define an API type as a simple alias of some other type combinations +strings are type-level strings, not actually values, represent path elements +endpoints are separated by :<|>, all endpoints end in a method with content types and return types +Capture captures path segments, but there are other combinators, for example for headers +Everything that is used from the request is expressed in types, enforcing checkability, no "escape hatch" inside handlers to get request +Every combinator has associated interpretations through typeclasses +### 6 +Explain type alias, point out Capture +Server is a type level function (type family), as mentioned earlier +### 7 +If we expand server (in ghci with kind!) we can see the actual type of the +function +### 8 +Lets speak about some interpretations of these things +### 9 +Servant server is the main interpretation that people are interested in, it's used +for taking a type specification and creating a server from it +Based on WAI, the web application interface, common abstraction for web servers which came out of the Yesod project. Implemented by the web server warp, which Yesod runs on +### 10 +Explain snippet, path gets removed from server type (irrelevant for handler), +route extracts string to value level +### 11 +Explain echo server quickly +### 12 +servant client allows generation of Haskell functions that query the API with the same types +this makes for easy to use RPC for example +### 13 +A lot of other interpretations exist for all kinds of things, mock servers for testing, foreign functions in various languages, documentation ... +### 14 +Demo! +1. Go quickly through code +2. Run server, query with curl +3. Open javascript function +4. Show JS code in the thing +5. Open the map itself +6. Open GHCi, use client +7. Generate docs +### 15 +Conclusion +Servant is pretty good, it's very easy to get started and it's great to raise the level of things that the compiler can tell you about when you do them wrong. +### 16 +Drawbacks. diff --git a/users/tazjin/presentations/servant-2016/slides.tex b/users/tazjin/presentations/servant-2016/slides.tex new file mode 100644 index 000000000000..d5947eb9421a --- /dev/null +++ b/users/tazjin/presentations/servant-2016/slides.tex @@ -0,0 +1,137 @@ +\documentclass[12pt]{beamer} +\usetheme{metropolis} +\usepackage{minted} + +\newenvironment{code}{\ttfamily}{\par} + +\title{servant} +\subtitle{Defining web APIs at the type-level} + +\begin{document} +\metroset{titleformat frame=smallcaps} +\setminted{fontsize=\scriptsize} + + +\maketitle + +\section{Introduction} + +\begin{frame}{Type-level DSLs?} + \begin{itemize} + \item (Uninhabited) types with attached ``meaning'' + \item The Expression Problem (Wadler 1998) + \item API representation and interpretation are separated + \item APIs become first-class citizens + \end{itemize} +\end{frame} + +\begin{frame}{Haskell extensions} + \begin{itemize} + \item TypeOperators + \item DataKinds + \item TypeFamilies + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{A servant example} + \begin{minted}{haskell} + type PubAPI = "pubs" :> Get ’[JSON] [Pub] + :<|> "pubs" :> "tagged" + :> Capture "tag" Text + :> Get ’[JSON] [Pub] + \end{minted} +\end{frame} + +\begin{frame}[fragile]{Computed types} + \begin{minted}{haskell} + type TaggedPubs = "tagged" :> Capture "tag" Text :> ... + + taggedPubsHandler :: Server TaggedPubs + taggedPubsHandler tag = ... + \end{minted} +\end{frame} + +\begin{frame}[fragile]{Computed types} + \begin{minted}{haskell} + type TaggedPubs = "tagged" :> Capture "tag" Text :> ... + + taggedPubsHandler :: Server TaggedPubs + taggedPubsHandler tag = ... + + Server TaggedPubs ~ + Text -> EitherT ServantErr IO [Pub] + \end{minted} +\end{frame} + +\section{Interpretations} + +\begin{frame}{servant-server} + The one everyone is interested in! + + \begin{itemize} + \item Based on WAI, can run on warp + \item Interprets combinators with a simple \texttt{HasServer c} class + \item Easy to use! + \end{itemize} +\end{frame} + +\begin{frame}[fragile]{HasServer ...} + \begin{minted}{haskell} + instance (KnownSymbol path, HasServer sublayout) + => HasServer (path :> sublayout) where + type ServerT (path :> sublayout) m = ServerT sublayout m + + route ... + where + pathString = symbolVal (Proxy :: Proxy path) + \end{minted} +\end{frame} + +\begin{frame}[fragile]{Server example} + \begin{minted}{haskell} + type Echo = Capture "echo" Text :> Get ’[PlainText] Text + + echoAPI :: Proxy Echo + echoAPI = Proxy + + echoServer :: Server Echo + echoServer = return + \end{minted} +\end{frame} + +\begin{frame}{servant-client} + \begin{itemize} + \item Generates Haskell client functions for API + \item Same types as API specification: For RPC the whole ``web layer'' is abstracted away + \item Also easy to use! + \end{itemize} +\end{frame} + +\begin{frame}{servant-docs, servant-js ...} + Many other interpretations exist already, for example: + \begin{itemize} + \item Documentation generation + \item Foreign function export (e.g. Elm, JavaScript) + \item Mock-server generation + \end{itemize} +\end{frame} + +\section{Demo} + +\section{Conclusion} + +\begin{frame}{Drawbacks} + \begin{itemize} + \item Haskell has no custom open kinds (yet) + \item Proxies are ugly + \item Errors can be a bit daunting + \end{itemize} +\end{frame} + +\begin{frame}{Questions?} + Ølkartet: github.com/tazjin/pubkartet \\ + Slides: github.com/tazjin/servant-presentation + + @tazjin +\end{frame} +\end{document} diff --git a/users/tazjin/presentations/systemd-2016/.gitignore b/users/tazjin/presentations/systemd-2016/.gitignore new file mode 100644 index 000000000000..1a38620fe9cc --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/.gitignore @@ -0,0 +1,6 @@ +slides.aux +slides.log +slides.nav +slides.out +slides.snm +slides.toc diff --git a/users/tazjin/presentations/systemd-2016/.skip-subtree b/users/tazjin/presentations/systemd-2016/.skip-subtree new file mode 100644 index 000000000000..108b3507ddd1 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/.skip-subtree @@ -0,0 +1 @@ +No Nix files will ever be under this tree ... diff --git a/users/tazjin/presentations/systemd-2016/Makefile b/users/tazjin/presentations/systemd-2016/Makefile new file mode 100644 index 000000000000..ac5dde3cb32f --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/Makefile @@ -0,0 +1,11 @@ +all: slides.pdf + +slides.toc: + lualatex slides.tex + +slides.pdf: slides.toc + lualatex slides.tex + +clean: + rm -f slides.aux slides.log slides.nav \ + slides.out slides.toc slides.snm diff --git a/users/tazjin/presentations/systemd-2016/README.md b/users/tazjin/presentations/systemd-2016/README.md new file mode 100644 index 000000000000..7f004b7d14ca --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/README.md @@ -0,0 +1,6 @@ +This repository contains the slides for my systemd presentation at Hackeriet. + +Requires LaTeX, [beamer][] and the [metropolis][] theme. + +[beamer]: http://mirror.hmc.edu/ctan/macros/latex/contrib/beamer/ +[metropolis]: https://github.com/matze/mtheme diff --git a/users/tazjin/presentations/systemd-2016/demo/demo-error.service b/users/tazjin/presentations/systemd-2016/demo/demo-error.service new file mode 100644 index 000000000000..b2d4c9d34799 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo-error.service @@ -0,0 +1,7 @@ +[Unit] +Description=Demonstrate failing units +OnFailure=demo-notify@%n.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/false diff --git a/users/tazjin/presentations/systemd-2016/demo/demo-limits.slice b/users/tazjin/presentations/systemd-2016/demo/demo-limits.slice new file mode 100644 index 000000000000..998185d26177 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo-limits.slice @@ -0,0 +1,7 @@ +[Unit] +Description=Limited resources demo +DefaultDependencies=no +Before=slices.target + +[Slice] +CPUQuota=10% diff --git a/users/tazjin/presentations/systemd-2016/demo/demo-notify@.service b/users/tazjin/presentations/systemd-2016/demo/demo-notify@.service new file mode 100644 index 000000000000..e25524b4e230 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo-notify@.service @@ -0,0 +1,6 @@ +[Unit] +Description=Demonstrate systemd templating by sending a notification + +[Service] +Type=oneshot +ExecStart=/usr/bin/notify-send 'Systemd notification' '%i' diff --git a/users/tazjin/presentations/systemd-2016/demo/demo-path.path b/users/tazjin/presentations/systemd-2016/demo/demo-path.path new file mode 100644 index 000000000000..87f1342da995 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo-path.path @@ -0,0 +1,6 @@ +[Unit] +Description=Demonstrate systemd path units + +[Path] +DirectoryNotEmpty=/tmp/hackeriet +Unit=demo.service diff --git a/users/tazjin/presentations/systemd-2016/demo/demo-stress.service b/users/tazjin/presentations/systemd-2016/demo/demo-stress.service new file mode 100644 index 000000000000..7e14f13e29d9 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo-stress.service @@ -0,0 +1,6 @@ +[Unit] +Description=Stress test CPU + +[Service] +Slice=demo.slice +ExecStart=/usr/bin/stress -c 5 diff --git a/users/tazjin/presentations/systemd-2016/demo/demo-timer.timer b/users/tazjin/presentations/systemd-2016/demo/demo-timer.timer new file mode 100644 index 000000000000..34eccb98b02a --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo-timer.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Demonstrate systemd timers + +[Timer] +OnActiveSec=2 +OnUnitActiveSec=5 +AccuracySec=5 +Unit=demo.service +# OnCalendar=Thu,Fri 2016-*-1,5 11:12:13 + +[Install] +WantedBy=multi-user.target diff --git a/users/tazjin/presentations/systemd-2016/demo/demo.service b/users/tazjin/presentations/systemd-2016/demo/demo.service new file mode 100644 index 000000000000..fcc710ad933f --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/demo.service @@ -0,0 +1,6 @@ +[Unit] +Description=Demo unit for systemd + +[Service] +Type=oneshot +ExecStart=/usr/bin/echo "Systemd unit activated. Hello Hackeriet." diff --git a/users/tazjin/presentations/systemd-2016/demo/notes.md b/users/tazjin/presentations/systemd-2016/demo/notes.md new file mode 100644 index 000000000000..b4866b1642bb --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/demo/notes.md @@ -0,0 +1,27 @@ +# simple oneshot + +Run `demo-notify@hello.service` + +# simple timer + +Run `demo-timer.timer`, show both + +# enabling + +Enable `demo-timer.timer`, go to symlink folder, disable + +# OnError + +Show & run `demo-error.service` + +# cgroups demo + +Start `demo-stress.service` without, show in htop, stop +Show slice unit, start slice unit +Add Slice=demo-limits.slice +daemon-reload +Start stress again + +# Proper service + +Look at nginx unit diff --git a/users/tazjin/presentations/systemd-2016/slides.pdf b/users/tazjin/presentations/systemd-2016/slides.pdf new file mode 100644 index 000000000000..384db2a6e0af --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/slides.pdf Binary files differdiff --git a/users/tazjin/presentations/systemd-2016/slides.pdfpc b/users/tazjin/presentations/systemd-2016/slides.pdfpc new file mode 100644 index 000000000000..99326bd8bf4e --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/slides.pdfpc @@ -0,0 +1,85 @@ +[file] +slides.pdf +[notes] +### 1 +### 2 +Let's start off by looking at what an init system is, how they used to work and what systemd does different before we go into more systemd-specific details. +### 3 +system processes that are started include for example FS mounts, network settings, powertop... +system services are long-running processes such as daemons, e.g. SSH, database or web servers, session managers, udev ... + +orphans: Process whose parent has finished somehow, gets adopted by init system +-> when a process terminates its parent must call wait() to get its exit() code, if there is no init system adopting orphans the process would become a zombie +### 4 +Before systemd there were simple init systems that just did the tasks listed on the previous slide. +Init scripts -> increased greatly in complexity over time, look at incomprehensible skeleton for Debian service init scripts +Runlevels -> things such as single-user mode, full multiuser mode, reboot, halt + +Init will run all the scripts, but it will not do much more than print information on success/failure of started scripts + +Init scripts run strictly sequential + +Init is unaware of inter-service dependencies, expressed through prefixing scripts with numbers etc. + +Init will not watch processes after system is booted -> crashing daemons will not automatically restart +### 5 +### 6 +How systemd came to be + +Considering the lack of process monitoring, problematic things about init scripts -> legacy init systems have drawbacks + +Apple had already built launchd, a more featured init system that monitored running processes, could automatically restart them and allowed for certain advanced features -> however it is awful to use and wrap your head around + +Lennart Poettering of Pulseaudio fame and Kay Sievers decided to implement a new init system to address these problems, while taking certain clues from Apple's design +### 7 +Systemd's design goals +### 8 +No more init scripts with opaque effects -> services are clearly defined units +Unit dependencies -> systemd can figure out what can be started in parallel +Process supervision: Unit can be configured in many ways, e.g. always restart, only restart on success etc +Service logs: We'll talk more about this later +### 9 +Units are the core component of systemd that users deal with. They define services and everything else that systemd needs to start and manage. +Note that all these are the names of the respective man page on a system with systemd installed +Types: +systemd.service - processes controlled by systemd +systemd.target - equivalent to "runlevels", grouping of units for synchronisation +systemd.timer - more powerful replacement of cron that starts other units +systemd.path - systemd equvialent of inotify, watches files/folders -> launches units +systemd.socket - expose local IPC or network sockets, launch units on connections +systemd.device - trigger units when certain devices are connected +systemd.mount - systemd equivalent of fstab entries +systemd.swap - like mount +systemd.slice - unit groups for resource management purposes +... and a few more specialised ones +### 10 +Linux cgroups are a new resource management feature added quite a long time ago, but not used much. +Cgroups can be created manually and processes can be moved into them in order to control resource utilisation +Few people used them before systemd, limits.conf was often much easier but not as fine-grained +Systemd changed this +### 11 +Systemd collects standard output and stderr from all processes into its journal system +they provide a tool for querying the log, for example grouping service logs together with correct timestamps, querying, +### 12 +Systemd tooling, most important one is systemctl for general service management +journalctl is the query and management tool for journald +systemd-analyze is used for figuring out performance issues, for example by analysing the boot process, can make cool graphs of dependencies +systemd-cgtop is like top, but not on a process level - it's on a cgroup/slice level, shows combined usage of cgroups +systemd-cgls lists contents of systemd's cgroups to see which services are in what group +there also exist a bunch of others that we'll skip for now +### 13 +### 14 +### 15 +Systemd criticism comes from many directions and usually focuses on a few points +feature-creep: systemd is absorbing a lot of different services +### 16 +explain diagram a bit +### 17 +opaque: as a result, systemd has a lot more internal complexity that people can't easily wrap your mind around. However I argue that unless you're using something like suckless' sinit with your own scripts, you probably have no idea what your init does today anyways +unstable: this was definitely true even in the first stable release, with the binary log format getting corrupted for example. I haven't personally experienced any trouble with it recently though. +Another thing is that services start depending on systemd when they shouldn't, a problem for the BSD world (who cares (hey christoph!)) +### 18 +Despite criticism, systemd was adopted rapidly by large portions of the Linux +Initially in RedHat, because Poettering and co work there and it was clear from the beginning that it would be there +ArchLinux (which I'm using) and a few others followed suit quite quickly +Eventually, the big Debian init system discussion - after a lot of flaming - led to Debian adopting it as well, which had a ripple effect for related distros such as Ubuntu which abandoned upstart for it. \ No newline at end of file diff --git a/users/tazjin/presentations/systemd-2016/slides.tex b/users/tazjin/presentations/systemd-2016/slides.tex new file mode 100644 index 000000000000..c613cefd7ec4 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/slides.tex @@ -0,0 +1,160 @@ +\documentclass[12pt]{beamer} +\usetheme{metropolis} + +\newenvironment{code}{\ttfamily}{\par} + +\title{systemd} +\subtitle{The standard Linux init system} + +\begin{document} +\metroset{titleformat frame=smallcaps} + +\maketitle + +\section{Introduction} + +\begin{frame}{What is an init system?} + An init system is the first userspace process (PID 1) started in a UNIX-like system. It handles: + + \begin{itemize} + \item Starting system processes and services to prepare the environment + \item Adopting and ``reaping'' orphaned processes + \end{itemize} +\end{frame} + +\begin{frame}{Classical init systems} + Init systems before systemd - such as SysVinit - were very simple. + + \begin{itemize} + \item Services and processes to run are organised into ``init scripts'' + \item Scripts are linked to specific runlevels + \item Init system is configured to boot into a runlevel + \end{itemize} + +\end{frame} + +\section{systemd} + +\begin{frame}{Can we do better?} + \begin{itemize} + \item ``legacy'' init systems have a lot of drawbacks + \item Apple is taking a different approach on OS X + \item Systemd project was founded to address these issues + \end{itemize} +\end{frame} + +\begin{frame}{Systemd design goals} + \begin{itemize} + \item Expressing service dependencies + \item Monitoring service status + \item Enable parallel service startups + \item Ease of use + \end{itemize} +\end{frame} + +\begin{frame}{Systemd - the basics} + \begin{itemize} + \item No scripts are executed, only declarative units + \item Units have explicit dependencies + \item Processes are supervised + \item cgroups are utilised to apply resource limits + \item Service logs are managed and centrally queryable + \item Much more! + \end{itemize} +\end{frame} + +\begin{frame}{Systemd units} + Units specify how and what to start. Several types exist: + \begin{code} + \small + \begin{columns}[T,onlytextwidth] + \column{0.5\textwidth} + \begin{itemize} + \item systemd.service + \item systemd.target + \item systemd.timer + \item systemd.path + \item systemd.socket + \end{itemize} + \column{0.5\textwidth} + \begin{itemize} + \item systemd.device + \item systemd.mount + \item systemd.swap + \item systemd.slice + \end{itemize} + \end{columns} + \end{code} +\end{frame} + + +\begin{frame}{Resource management} + Systemd utilises Linux \texttt{cgroups} for resource management, specifically CPU, disk I/O and memory usage. + + \begin{itemize} + \item Hierarchical setup of groups makes it easy to limit resources for a set of services + \item Units can be attached to a \texttt{systemd.slice} for controlling resources for a group of services + \item Resource limits can also be specified directly in the unit + \end{itemize} +\end{frame} + +\begin{frame}{journald} + Systemd comes with an integrated log management solution, replacing software such as \texttt{syslog-ng}. + \begin{itemize} + \item All process output is collected in the journal + \item \texttt{journalctl} tool provides many options for querying and tailing logs + \item Children of processes automatically log to the journal as well + \item \textbf{Caveat:} Hard to learn initially + \end{itemize} +\end{frame} + +\begin{frame}{Systemd tooling} + A variety of CLI-tools exist for managing systemd systems. + \begin{code} + \begin{itemize} + \item systemctl + \item journalctl + \item systemd-analyze + \item systemd-cgtop + \item systemd-cgls + \end{itemize} + \end{code} + + Let's look at some of them. +\end{frame} + +\section{Demo} + +\section{Controversies} + +\begin{frame}{Systemd criticism} + Systemd has been heavily criticised, usually focusing around a few points: + \begin{itemize} + \item Feature-creep: Systemd absorbs more and more other services + \end{itemize} +\end{frame} + +\begin{frame}{Systemd criticism} + \includegraphics[keepaspectratio=true,width=\textwidth]{systemdcomponents.png} +\end{frame} + +\begin{frame}{Systemd criticism} + Systemd has been heavily criticised, usually focusing around a few points: + \begin{itemize} + \item Feature-creep: Systemd absorbs more and more other services + \item Opaque: systemd's inner workings are harder to understand than old \texttt{init} + \item Unstable: development is quick and breakage happens + \end{itemize} +\end{frame} + +\begin{frame}{Systemd adoption} + Systemd was initially adopted by RedHat (and related distributions). + + It spread quickly to others, for example ArchLinux. + + Debian and Ubuntu were the last major players who decided to adopt it, but not without drama. +\end{frame} + +\section{Questions?} + +\end{document} diff --git a/users/tazjin/presentations/systemd-2016/systemdcomponents.png b/users/tazjin/presentations/systemd-2016/systemdcomponents.png new file mode 100644 index 000000000000..a22c762f7e13 --- /dev/null +++ b/users/tazjin/presentations/systemd-2016/systemdcomponents.png Binary files differdiff --git a/users/tazjin/rlox/.gitignore b/users/tazjin/rlox/.gitignore new file mode 100644 index 000000000000..29e65519ba35 --- /dev/null +++ b/users/tazjin/rlox/.gitignore @@ -0,0 +1,3 @@ +result +/target +**/*.rs.bk diff --git a/users/tazjin/rlox/Cargo.lock b/users/tazjin/rlox/Cargo.lock new file mode 100644 index 000000000000..d8107726e067 --- /dev/null +++ b/users/tazjin/rlox/Cargo.lock @@ -0,0 +1,6 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "rlox" +version = "0.1.0" + diff --git a/users/tazjin/rlox/Cargo.toml b/users/tazjin/rlox/Cargo.toml new file mode 100644 index 000000000000..b66af6ba85d3 --- /dev/null +++ b/users/tazjin/rlox/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "rlox" +version = "0.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +edition = "2018" + +[features] +# Enables debugging/disassembling in the bytecode interpreter. Off by +# default as it is quite spammy. +disassemble = [] diff --git a/users/tazjin/rlox/README.md b/users/tazjin/rlox/README.md new file mode 100644 index 000000000000..1d2692d09cc1 --- /dev/null +++ b/users/tazjin/rlox/README.md @@ -0,0 +1,7 @@ +This is an interpreter for the Lox language, based on the book "[Crafting +Interpreters](https://craftinginterpreters.com/)". + +The book's original code uses Java, but I don't want to use Java, so I've +decided to take on the extra complexity of porting it to Rust. + +Note: This implements the first of two Lox interpreters. diff --git a/users/tazjin/rlox/default.nix b/users/tazjin/rlox/default.nix new file mode 100644 index 000000000000..e50ac32be452 --- /dev/null +++ b/users/tazjin/rlox/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; +} diff --git a/users/tazjin/rlox/examples/builtins.lox b/users/tazjin/rlox/examples/builtins.lox new file mode 100644 index 000000000000..39af1d73c4c9 --- /dev/null +++ b/users/tazjin/rlox/examples/builtins.lox @@ -0,0 +1 @@ +print clock(); diff --git a/users/tazjin/rlox/examples/fib.lox b/users/tazjin/rlox/examples/fib.lox new file mode 100644 index 000000000000..1b91e9db94ac --- /dev/null +++ b/users/tazjin/rlox/examples/fib.lox @@ -0,0 +1,6 @@ +fun fib(n) { + if (n <= 1) return n; + return fib(n - 2) + fib(n - 1); +} + +print fib(30); diff --git a/users/tazjin/rlox/examples/func.lox b/users/tazjin/rlox/examples/func.lox new file mode 100644 index 000000000000..d197ad11383f --- /dev/null +++ b/users/tazjin/rlox/examples/func.lox @@ -0,0 +1,5 @@ +fun foo(name) { + print("hello " + name); +} + +foo("bar"); diff --git a/users/tazjin/rlox/examples/hello.lox b/users/tazjin/rlox/examples/hello.lox new file mode 100644 index 000000000000..31752d9e2f5e --- /dev/null +++ b/users/tazjin/rlox/examples/hello.lox @@ -0,0 +1,34 @@ +var a = 12; +var b = a * 2; + +{ + var b = a * 3; + a = 42; + print b; +} + +print a; +print b; + +if (5 > 4) + print "it's true"; +else + print "it's false"; + +if (false) + print "it's not true"; + +if (true and false) + print "won't happen"; + +if (true or false) + print "will happen"; + +var n = 5; +while (n > 0) { + print "counting down"; + n = n - 1; +} + +for(var i = 0; i < 10; i = i + 1) + print "bla"; diff --git a/users/tazjin/rlox/examples/if.lox b/users/tazjin/rlox/examples/if.lox new file mode 100644 index 000000000000..5f335c0e8b29 --- /dev/null +++ b/users/tazjin/rlox/examples/if.lox @@ -0,0 +1,7 @@ +if (false) { + print "yes"; +} else { + print "no"; +} + +print "afterwards"; diff --git a/users/tazjin/rlox/examples/scope.lox b/users/tazjin/rlox/examples/scope.lox new file mode 100644 index 000000000000..d563807943ba --- /dev/null +++ b/users/tazjin/rlox/examples/scope.lox @@ -0,0 +1,19 @@ +var a = "global a"; +var b = "global b"; +var c = "global c"; +{ + var a = "outer a"; + var b = "outer b"; + { + var a = "inner a"; + print a; + print b; + print c; + } + print a; + print b; + print c; +} +print a; +print b; +print c; diff --git a/users/tazjin/rlox/examples/scope2.lox b/users/tazjin/rlox/examples/scope2.lox new file mode 100644 index 000000000000..f826c8658803 --- /dev/null +++ b/users/tazjin/rlox/examples/scope2.lox @@ -0,0 +1,10 @@ +var a = "global"; +{ + fun showA() { + print a; + } + + showA(); + var a = "block"; + showA(); +} diff --git a/users/tazjin/rlox/examples/slow.lox b/users/tazjin/rlox/examples/slow.lox new file mode 100644 index 000000000000..dd6fb5e4bf4d --- /dev/null +++ b/users/tazjin/rlox/examples/slow.lox @@ -0,0 +1,9 @@ +fun fib(n) { + if (n < 2) return n; + return fib(n - 1) + fib(n - 2); +} + +var before = clock(); +print fib(40); +var after = clock(); +print after - before; diff --git a/users/tazjin/rlox/examples/var.lox b/users/tazjin/rlox/examples/var.lox new file mode 100644 index 000000000000..7af90b3f0bee --- /dev/null +++ b/users/tazjin/rlox/examples/var.lox @@ -0,0 +1,8 @@ +var a = 10; +var b = 5; + +{ + var b = 10; + var c = 2; + a * b * c; +} diff --git a/users/tazjin/rlox/rustfmt.toml b/users/tazjin/rlox/rustfmt.toml new file mode 100644 index 000000000000..df99c69198f5 --- /dev/null +++ b/users/tazjin/rlox/rustfmt.toml @@ -0,0 +1 @@ +max_width = 80 diff --git a/users/tazjin/rlox/src/bytecode/chunk.rs b/users/tazjin/rlox/src/bytecode/chunk.rs new file mode 100644 index 000000000000..fc5cd34fdf4f --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/chunk.rs @@ -0,0 +1,93 @@ +use std::ops::Index; + +use super::opcode::{CodeIdx, ConstantIdx, OpCode}; +use super::value; + +// In the book, this type is a hand-rolled dynamic array +// implementation in C. The main benefit of following that approach +// would be avoiding issues with OpCode variants not having equal +// sizes, but for the purpose of this I'm going to ignore that +// problem. +#[derive(Debug, Default)] +pub struct Chunk { + pub code: Vec<OpCode>, + lines: Vec<Span>, + constants: Vec<value::Value>, +} + +#[derive(Debug)] +struct Span { + /// Source code line + line: usize, + + /// Number of instructions derived from this line + count: usize, +} + +impl Chunk { + pub fn add_op(&mut self, data: OpCode, line: usize) -> CodeIdx { + let idx = self.code.len(); + self.code.push(data); + self.add_line(line); + CodeIdx(idx) + } + + pub fn add_constant(&mut self, data: value::Value) -> usize { + let idx = self.constants.len(); + self.constants.push(data); + idx + } + + pub fn constant(&self, idx: ConstantIdx) -> &value::Value { + self.constants.index(idx.0) + } + + fn add_line(&mut self, line: usize) { + match self.lines.last_mut() { + Some(span) if span.line == line => span.count += 1, + _ => self.lines.push(Span { line, count: 1 }), + } + } + + pub fn get_line(&self, offset: usize) -> usize { + let mut pos = 0; + for span in &self.lines { + pos += span.count; + if pos > offset { + return span.line; + } + } + + panic!("invalid chunk state: line missing for offset {}", offset); + } +} + +// Disassembler + +/// Print a single disassembled instruction at the specified offset. +/// Some instructions are printed "raw", others have special handling. +#[cfg(feature = "disassemble")] +pub fn disassemble_instruction(chunk: &Chunk, offset: usize) { + print!("{:04} ", offset); + + let line = chunk.get_line(offset); + if offset > 0 && line == chunk.get_line(offset - 1) { + print!(" | "); + } else { + print!("{:4} ", line); + } + + match chunk.code.index(offset) { + OpCode::OpConstant(idx) => { + println!("OpConstant({:?}) '{:?}'", idx, chunk.constant(*idx)) + } + op => println!("{:?}", op), + } +} + +#[cfg(feature = "disassemble")] +pub fn disassemble_chunk(chunk: &Chunk) { + for (idx, _) in chunk.code.iter().enumerate() { + disassemble_instruction(chunk, idx); + } +} diff --git a/users/tazjin/rlox/src/bytecode/compiler.rs b/users/tazjin/rlox/src/bytecode/compiler.rs new file mode 100644 index 000000000000..3e8a80653f91 --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/compiler.rs @@ -0,0 +1,737 @@ +use super::chunk::Chunk; +use super::errors::{Error, ErrorKind, LoxResult}; +use super::interner::{InternedStr, Interner}; +use super::opcode::{CodeIdx, CodeOffset, ConstantIdx, OpCode, StackIdx}; +use super::value::Value; +use crate::scanner::{self, Token, TokenKind}; + +#[cfg(feature = "disassemble")] +use super::chunk; + +#[derive(Debug)] +enum Depth { + Unitialised, + At(usize), +} + +impl Depth { + fn above(&self, theirs: usize) -> bool { + match self { + Depth::Unitialised => false, + Depth::At(ours) => *ours > theirs, + } + } + + fn below(&self, theirs: usize) -> bool { + match self { + Depth::Unitialised => false, + Depth::At(ours) => *ours < theirs, + } + } +} + +#[derive(Debug)] +struct Local { + name: Token, + depth: Depth, +} + +#[derive(Debug, Default)] +struct Locals { + locals: Vec<Local>, + scope_depth: usize, +} + +struct Compiler<T: Iterator<Item = Token>> { + tokens: T, + chunk: Chunk, + panic: bool, + errors: Vec<Error>, + strings: Interner, + locals: Locals, + + current: Option<Token>, + previous: Option<Token>, +} + +#[derive(Debug, PartialEq, PartialOrd)] +enum Precedence { + None, + Assignment, // = + Or, // or + And, // and + Equality, // == != + Comparison, // < > <= >= + Term, // + - + Factor, // * / + Unary, // ! - + Call, // . () + Primary, +} + +type ParseFn<T> = fn(&mut Compiler<T>) -> LoxResult<()>; + +struct ParseRule<T: Iterator<Item = Token>> { + prefix: Option<ParseFn<T>>, + infix: Option<ParseFn<T>>, + precedence: Precedence, +} + +impl<T: Iterator<Item = Token>> ParseRule<T> { + fn new( + prefix: Option<ParseFn<T>>, + infix: Option<ParseFn<T>>, + precedence: Precedence, + ) -> Self { + ParseRule { + prefix, + infix, + precedence, + } + } +} + +impl Precedence { + // Return the next highest precedence, if there is one. + fn next(&self) -> Self { + match self { + Precedence::None => Precedence::Assignment, + Precedence::Assignment => Precedence::Or, + Precedence::Or => Precedence::And, + Precedence::And => Precedence::Equality, + Precedence::Equality => Precedence::Comparison, + Precedence::Comparison => Precedence::Term, + Precedence::Term => Precedence::Factor, + Precedence::Factor => Precedence::Unary, + Precedence::Unary => Precedence::Call, + Precedence::Call => Precedence::Primary, + Precedence::Primary => panic!( + "invalid parser state: no higher precedence than Primary" + ), + } + } +} + +fn rule_for<T: Iterator<Item = Token>>(token: &TokenKind) -> ParseRule<T> { + match token { + TokenKind::LeftParen => { + ParseRule::new(Some(Compiler::grouping), None, Precedence::None) + } + + TokenKind::Minus => ParseRule::new( + Some(Compiler::unary), + Some(Compiler::binary), + Precedence::Term, + ), + + TokenKind::Plus => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Term) + } + + TokenKind::Slash => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Factor) + } + + TokenKind::Star => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Factor) + } + + TokenKind::Number(_) => { + ParseRule::new(Some(Compiler::number), None, Precedence::None) + } + + TokenKind::True => { + ParseRule::new(Some(Compiler::literal), None, Precedence::None) + } + + TokenKind::False => { + ParseRule::new(Some(Compiler::literal), None, Precedence::None) + } + + TokenKind::Nil => { + ParseRule::new(Some(Compiler::literal), None, Precedence::None) + } + + TokenKind::Bang => { + ParseRule::new(Some(Compiler::unary), None, Precedence::None) + } + + TokenKind::BangEqual => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Equality) + } + + TokenKind::EqualEqual => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Equality) + } + + TokenKind::Greater => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Comparison) + } + + TokenKind::GreaterEqual => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Comparison) + } + + TokenKind::Less => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Comparison) + } + + TokenKind::LessEqual => { + ParseRule::new(None, Some(Compiler::binary), Precedence::Comparison) + } + + TokenKind::Identifier(_) => { + ParseRule::new(Some(Compiler::variable), None, Precedence::None) + } + + TokenKind::String(_) => { + ParseRule::new(Some(Compiler::string), None, Precedence::None) + } + + _ => ParseRule::new(None, None, Precedence::None), + } +} + +macro_rules! consume { + ( $self:ident, $expected:pat, $err:expr ) => { + match $self.current().kind { + $expected => $self.advance(), + _ => $self.error_at($self.current().line, $err), + } + }; +} + +impl<T: Iterator<Item = Token>> Compiler<T> { + fn compile(&mut self) -> LoxResult<()> { + self.advance(); + + while !self.match_token(&TokenKind::Eof) { + self.declaration()?; + } + + self.end_compiler() + } + + fn advance(&mut self) { + self.previous = self.current.take(); + self.current = self.tokens.next(); + } + + fn expression(&mut self) -> LoxResult<()> { + self.parse_precedence(Precedence::Assignment) + } + + fn var_declaration(&mut self) -> LoxResult<()> { + let idx = self.parse_variable()?; + + if self.match_token(&TokenKind::Equal) { + self.expression()?; + } else { + self.emit_op(OpCode::OpNil); + } + + self.expect_semicolon("expect ';' after variable declaration")?; + self.define_variable(idx) + } + + fn define_variable(&mut self, var: Option<ConstantIdx>) -> LoxResult<()> { + if self.locals.scope_depth == 0 { + self.emit_op(OpCode::OpDefineGlobal( + var.expect("should be global"), + )); + } else { + self.locals + .locals + .last_mut() + .expect("fatal: variable not yet added at definition") + .depth = Depth::At(self.locals.scope_depth); + } + + Ok(()) + } + + fn declaration(&mut self) -> LoxResult<()> { + if self.match_token(&TokenKind::Var) { + self.var_declaration()?; + } else { + self.statement()?; + } + + if self.panic { + self.synchronise(); + } + + Ok(()) + } + + fn statement(&mut self) -> LoxResult<()> { + if self.match_token(&TokenKind::Print) { + self.print_statement() + } else if self.match_token(&TokenKind::If) { + self.if_statement() + } else if self.match_token(&TokenKind::LeftBrace) { + self.begin_scope(); + self.block()?; + self.end_scope(); + Ok(()) + } else { + self.expression_statement() + } + } + + fn print_statement(&mut self) -> LoxResult<()> { + self.expression()?; + self.expect_semicolon("expect ';' after print statement")?; + self.emit_op(OpCode::OpPrint); + Ok(()) + } + + fn begin_scope(&mut self) { + self.locals.scope_depth += 1; + } + + fn end_scope(&mut self) { + debug_assert!(self.locals.scope_depth > 0, "tried to end global scope"); + self.locals.scope_depth -= 1; + + while self.locals.locals.len() > 0 + && self.locals.locals[self.locals.locals.len() - 1] + .depth + .above(self.locals.scope_depth) + { + self.emit_op(OpCode::OpPop); + self.locals.locals.remove(self.locals.locals.len() - 1); + } + } + + fn block(&mut self) -> LoxResult<()> { + while !self.check(&TokenKind::RightBrace) + && !self.check(&TokenKind::Eof) + { + self.declaration()?; + } + + consume!( + self, + TokenKind::RightBrace, + ErrorKind::ExpectedToken("Expected '}' after block.") + ); + Ok(()) + } + + fn expression_statement(&mut self) -> LoxResult<()> { + self.expression()?; + self.expect_semicolon("expect ';' after expression")?; + // TODO(tazjin): Why did I add this originally? + // self.emit_op(OpCode::OpPop); + Ok(()) + } + + fn if_statement(&mut self) -> LoxResult<()> { + consume!( + self, + TokenKind::LeftParen, + ErrorKind::ExpectedToken("Expected '(' after 'if'") + ); + + self.expression()?; + + consume!( + self, + TokenKind::RightParen, + ErrorKind::ExpectedToken("Expected ')' after condition") + ); + + let then_jump = self.emit_op(OpCode::OpJumpPlaceholder(false)); + self.emit_op(OpCode::OpPop); + self.statement()?; + let else_jump = self.emit_op(OpCode::OpJumpPlaceholder(true)); + self.patch_jump(then_jump); + self.emit_op(OpCode::OpPop); + + if self.match_token(&TokenKind::Else) { + self.statement()?; + } + + self.patch_jump(else_jump); + + Ok(()) + } + + fn number(&mut self) -> LoxResult<()> { + if let TokenKind::Number(num) = self.previous().kind { + self.emit_constant(Value::Number(num), true); + return Ok(()); + } + + unreachable!("internal parser error: entered number() incorrectly") + } + + fn grouping(&mut self) -> LoxResult<()> { + self.expression()?; + consume!( + self, + TokenKind::RightParen, + ErrorKind::ExpectedToken("Expected ')' after expression") + ); + Ok(()) + } + + fn unary(&mut self) -> LoxResult<()> { + // TODO(tazjin): Avoid clone + let kind = self.previous().kind.clone(); + + // Compile the operand + self.parse_precedence(Precedence::Unary)?; + + // Emit operator instruction + match kind { + TokenKind::Bang => self.emit_op(OpCode::OpNot), + TokenKind::Minus => self.emit_op(OpCode::OpNegate), + _ => unreachable!("only called for unary operator tokens"), + }; + + Ok(()) + } + + fn binary(&mut self) -> LoxResult<()> { + // Remember the operator + let operator = self.previous().kind.clone(); + + // Compile the right operand + let rule: ParseRule<T> = rule_for(&operator); + self.parse_precedence(rule.precedence.next())?; + + // Emit operator instruction + match operator { + TokenKind::Minus => self.emit_op(OpCode::OpSubtract), + TokenKind::Plus => self.emit_op(OpCode::OpAdd), + TokenKind::Star => self.emit_op(OpCode::OpMultiply), + TokenKind::Slash => self.emit_op(OpCode::OpDivide), + + TokenKind::BangEqual => { + self.emit_op(OpCode::OpEqual); + self.emit_op(OpCode::OpNot) + } + + TokenKind::EqualEqual => self.emit_op(OpCode::OpEqual), + TokenKind::Greater => self.emit_op(OpCode::OpGreater), + + TokenKind::GreaterEqual => { + self.emit_op(OpCode::OpLess); + self.emit_op(OpCode::OpNot) + } + + TokenKind::Less => self.emit_op(OpCode::OpLess), + TokenKind::LessEqual => { + self.emit_op(OpCode::OpGreater); + self.emit_op(OpCode::OpNot) + } + + _ => unreachable!("only called for binary operator tokens"), + }; + + Ok(()) + } + + fn literal(&mut self) -> LoxResult<()> { + match self.previous().kind { + TokenKind::Nil => self.emit_op(OpCode::OpNil), + TokenKind::True => self.emit_op(OpCode::OpTrue), + TokenKind::False => self.emit_op(OpCode::OpFalse), + _ => unreachable!("only called for literal value tokens"), + }; + + Ok(()) + } + + fn string(&mut self) -> LoxResult<()> { + let val = match &self.previous().kind { + TokenKind::String(s) => s.clone(), + _ => unreachable!("only called for strings"), + }; + + let id = self.strings.intern(val); + self.emit_constant(Value::String(id.into()), true); + + Ok(()) + } + + fn named_variable(&mut self, name: Token) -> LoxResult<()> { + let local_idx = self.resolve_local(&name); + + let ident = if local_idx.is_some() { + None + } else { + Some(self.identifier_constant(&name)?) + }; + + if self.match_token(&TokenKind::Equal) { + self.expression()?; + match local_idx { + Some(idx) => self.emit_op(OpCode::OpSetLocal(idx)), + None => self.emit_op(OpCode::OpSetGlobal(ident.unwrap())), + }; + } else { + match local_idx { + Some(idx) => self.emit_op(OpCode::OpGetLocal(idx)), + None => self.emit_op(OpCode::OpGetGlobal(ident.unwrap())), + }; + } + + Ok(()) + } + + fn variable(&mut self) -> LoxResult<()> { + let name = self.previous().clone(); + self.named_variable(name) + } + + fn parse_precedence(&mut self, precedence: Precedence) -> LoxResult<()> { + self.advance(); + let rule: ParseRule<T> = rule_for(&self.previous().kind); + let prefix_fn = match rule.prefix { + None => unimplemented!("expected expression or something, unclear"), + Some(func) => func, + }; + + prefix_fn(self)?; + + while precedence <= rule_for::<T>(&self.current().kind).precedence { + self.advance(); + match rule_for::<T>(&self.previous().kind).infix { + Some(func) => { + func(self)?; + } + None => { + unreachable!("invalid compiler state: error in parse rules") + } + } + } + + Ok(()) + } + + fn identifier_str(&mut self, token: &Token) -> LoxResult<InternedStr> { + let ident = match &token.kind { + TokenKind::Identifier(ident) => ident.to_string(), + _ => { + return Err(Error { + line: self.current().line, + kind: ErrorKind::ExpectedToken("Expected identifier"), + }) + } + }; + + Ok(self.strings.intern(ident)) + } + + fn identifier_constant(&mut self, name: &Token) -> LoxResult<ConstantIdx> { + let ident = self.identifier_str(name)?; + Ok(self.emit_constant(Value::String(ident.into()), false)) + } + + fn resolve_local(&self, name: &Token) -> Option<StackIdx> { + for (idx, local) in self.locals.locals.iter().enumerate().rev() { + if name.lexeme == local.name.lexeme { + if let Depth::Unitialised = local.depth { + // TODO(tazjin): *return* err + panic!("can't read variable in its own initialiser"); + } + return Some(StackIdx(idx)); + } + } + + None + } + + fn add_local(&mut self, name: Token) { + let local = Local { + name, + depth: Depth::Unitialised, + }; + + self.locals.locals.push(local); + } + + fn declare_variable(&mut self) -> LoxResult<()> { + if self.locals.scope_depth == 0 { + return Ok(()); + } + + let name = self.previous().clone(); + + for local in self.locals.locals.iter().rev() { + if local.depth.below(self.locals.scope_depth) { + break; + } + + if name.lexeme == local.name.lexeme { + return Err(Error { + kind: ErrorKind::VariableShadowed(name.lexeme.into()), + line: name.line, + }); + } + } + + self.add_local(name); + Ok(()) + } + + fn parse_variable(&mut self) -> LoxResult<Option<ConstantIdx>> { + consume!( + self, + TokenKind::Identifier(_), + ErrorKind::ExpectedToken("expected identifier") + ); + + self.declare_variable()?; + if self.locals.scope_depth > 0 { + return Ok(None); + } + + let name = self.previous().clone(); + let id = self.identifier_str(&name)?; + Ok(Some(self.emit_constant(Value::String(id.into()), false))) + } + + fn current_chunk(&mut self) -> &mut Chunk { + &mut self.chunk + } + + fn end_compiler(&mut self) -> LoxResult<()> { + self.emit_op(OpCode::OpReturn); + + #[cfg(feature = "disassemble")] + { + chunk::disassemble_chunk(&self.chunk); + println!("== compilation finished =="); + } + + Ok(()) + } + + fn emit_op(&mut self, op: OpCode) -> CodeIdx { + let line = self.previous().line; + self.current_chunk().add_op(op, line) + } + + fn emit_constant(&mut self, val: Value, with_op: bool) -> ConstantIdx { + let idx = ConstantIdx(self.chunk.add_constant(val)); + + if with_op { + self.emit_op(OpCode::OpConstant(idx)); + } + + idx + } + + fn patch_jump(&mut self, idx: CodeIdx) { + let offset = CodeOffset(self.chunk.code.len() - idx.0 - 1); + + if let OpCode::OpJumpPlaceholder(true) = self.chunk.code[idx.0] { + self.chunk.code[idx.0] = OpCode::OpJump(offset); + return; + } + + if let OpCode::OpJumpPlaceholder(false) = self.chunk.code[idx.0] { + self.chunk.code[idx.0] = OpCode::OpJumpIfFalse(offset); + return; + } + + panic!( + "attempted to patch unsupported op: {:?}", + self.chunk.code[idx.0] + ); + } + + fn previous(&self) -> &Token { + self.previous + .as_ref() + .expect("invalid internal compiler state: missing previous token") + } + + fn current(&self) -> &Token { + self.current + .as_ref() + .expect("invalid internal compiler state: missing current token") + } + + fn error_at(&mut self, line: usize, kind: ErrorKind) { + if self.panic { + return; + } + + self.panic = true; + self.errors.push(Error { kind, line }) + } + + fn match_token(&mut self, token: &TokenKind) -> bool { + if !self.check(token) { + return false; + } + + self.advance(); + true + } + + fn check(&self, token: &TokenKind) -> bool { + return self.current().kind == *token; + } + + fn synchronise(&mut self) { + self.panic = false; + + while self.current().kind != TokenKind::Eof { + if self.previous().kind == TokenKind::Semicolon { + return; + } + + match self.current().kind { + TokenKind::Class + | TokenKind::Fun + | TokenKind::Var + | TokenKind::For + | TokenKind::If + | TokenKind::While + | TokenKind::Print + | TokenKind::Return => return, + + _ => { + self.advance(); + } + } + } + } + + fn expect_semicolon(&mut self, msg: &'static str) -> LoxResult<()> { + consume!(self, TokenKind::Semicolon, ErrorKind::ExpectedToken(msg)); + Ok(()) + } +} + +pub fn compile(code: &str) -> Result<(Interner, Chunk), Vec<Error>> { + let chars = code.chars().collect::<Vec<char>>(); + let tokens = scanner::scan(&chars).map_err(|errors| { + errors.into_iter().map(Into::into).collect::<Vec<Error>>() + })?; + + let mut compiler = Compiler { + tokens: tokens.into_iter().peekable(), + chunk: Default::default(), + panic: false, + errors: vec![], + strings: Interner::with_capacity(1024), + locals: Default::default(), + current: None, + previous: None, + }; + + compiler.compile()?; + + if compiler.errors.is_empty() { + Ok((compiler.strings, compiler.chunk)) + } else { + Err(compiler.errors) + } +} diff --git a/users/tazjin/rlox/src/bytecode/errors.rs b/users/tazjin/rlox/src/bytecode/errors.rs new file mode 100644 index 000000000000..988031f763cf --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/errors.rs @@ -0,0 +1,51 @@ +use crate::scanner::ScannerError; + +use std::fmt; + +#[derive(Debug)] +pub enum ErrorKind { + UnexpectedChar(char), + UnterminatedString, + ExpectedToken(&'static str), + InternalError(&'static str), + TypeError(String), + VariableShadowed(String), +} + +#[derive(Debug)] +pub struct Error { + pub kind: ErrorKind, + pub line: usize, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[line NYI] Error: {:?}", self.kind) + } +} + +impl From<ScannerError> for Error { + fn from(err: ScannerError) -> Self { + match err { + ScannerError::UnexpectedChar { line, unexpected } => Error { + line, + kind: ErrorKind::UnexpectedChar(unexpected), + }, + + ScannerError::UnterminatedString { line } => Error { + line, + kind: ErrorKind::UnterminatedString, + }, + } + } +} + +// Convenience implementation as we're often dealing with vectors of +// errors (to report as many issues as possible before terminating) +impl From<Error> for Vec<Error> { + fn from(err: Error) -> Self { + vec![err] + } +} + +pub type LoxResult<T> = Result<T, Error>; diff --git a/users/tazjin/rlox/src/bytecode/interner/mod.rs b/users/tazjin/rlox/src/bytecode/interner/mod.rs new file mode 100644 index 000000000000..1da1a24b2c5f --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/interner/mod.rs @@ -0,0 +1,87 @@ +//! String-interning implementation for values that are likely to +//! benefit from fast comparisons and deduplication (e.g. instances of +//! variable names). +//! +//! This uses a trick from the typed-arena crate for guaranteeing +//! stable addresses by never resizing the existing String buffer, and +//! collecting full buffers in a vector. + +use std::collections::HashMap; + +#[cfg(test)] +mod tests; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct InternedStr { + id: usize, +} + +#[derive(Default)] +pub struct Interner { + map: HashMap<&'static str, InternedStr>, + vec: Vec<&'static str>, + buf: String, + full: Vec<String>, +} + +impl Interner { + pub fn with_capacity(cap: usize) -> Self { + Interner { + buf: String::with_capacity(cap), + ..Default::default() + } + } + + pub fn intern<S: AsRef<str>>(&mut self, name: S) -> InternedStr { + let name = name.as_ref(); + if let Some(&id) = self.map.get(name) { + return id; + } + + let name = self.alloc(name); + let id = InternedStr { + id: self.vec.len() as usize, + }; + + self.map.insert(name, id); + self.vec.push(name); + + debug_assert!(self.lookup(id) == name); + debug_assert!(self.intern(name) == id); + + id + } + + pub fn lookup<'a>(&'a self, id: InternedStr) -> &'a str { + self.vec[id.id] + } + + fn alloc<'a>(&'a mut self, name: &str) -> &'static str { + let cap = self.buf.capacity(); + if cap < self.buf.len() + name.len() { + let new_cap = (cap.max(name.len()) + 1).next_power_of_two(); + let new_buf = String::with_capacity(new_cap); + let old_buf = std::mem::replace(&mut self.buf, new_buf); + self.full.push(old_buf); + } + + let interned: &'a str = { + let start = self.buf.len(); + self.buf.push_str(name); + &self.buf[start..] + }; + + unsafe { + // This is sound for two reasons: + // + // 1. This function (Interner::alloc) is private, which + // prevents users from allocating a supposedly static + // reference. + // + // 2. Interner::lookup explicitly shortens the lifetime of + // references that are handed out to that of the + // reference to self. + return &*(interned as *const str); + } + } +} diff --git a/users/tazjin/rlox/src/bytecode/interner/tests.rs b/users/tazjin/rlox/src/bytecode/interner/tests.rs new file mode 100644 index 000000000000..b34bf6835389 --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/interner/tests.rs @@ -0,0 +1,24 @@ +use super::*; + +#[test] +fn interns_strings() { + let mut interner = Interner::with_capacity(128); + let id = interner.intern("hello world"); + assert_eq!("hello world", interner.lookup(id)); +} + +#[test] +fn deduplicates_strings() { + let mut interner = Interner::with_capacity(128); + let id_1 = interner.intern("hello world"); + let id_2 = interner.intern("hello world"); + assert_eq!(id_1, id_2); +} + +#[test] +fn ids_survive_growing() { + let mut interner = Interner::with_capacity(16); + let id = interner.intern("hello"); + interner.intern("excessively large string that will cause eallocation"); + assert_eq!("hello", interner.lookup(id)); +} diff --git a/users/tazjin/rlox/src/bytecode/mod.rs b/users/tazjin/rlox/src/bytecode/mod.rs new file mode 100644 index 000000000000..c6f3a737aef8 --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/mod.rs @@ -0,0 +1,33 @@ +//! Bytecode interpreter for Lox. +//! +//! https://craftinginterpreters.com/chunks-of-bytecode.html + +mod chunk; +mod compiler; +mod errors; +mod interner; +mod opcode; +mod value; +mod vm; + +#[cfg(test)] +mod tests; + +pub struct Interpreter {} + +impl crate::Lox for Interpreter { + type Error = errors::Error; + type Value = value::Value; + + fn create() -> Self { + Interpreter {} + } + + fn interpret( + &mut self, + code: String, + ) -> Result<Self::Value, Vec<Self::Error>> { + let (strings, chunk) = compiler::compile(&code)?; + vm::interpret(strings, chunk).map_err(|e| vec![e]) + } +} diff --git a/users/tazjin/rlox/src/bytecode/opcode.rs b/users/tazjin/rlox/src/bytecode/opcode.rs new file mode 100644 index 000000000000..8a106f96917d --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/opcode.rs @@ -0,0 +1,56 @@ +#[derive(Clone, Copy, Debug)] +pub struct ConstantIdx(pub usize); + +#[derive(Clone, Copy, Debug)] +pub struct StackIdx(pub usize); + +#[derive(Clone, Copy, Debug)] +pub struct CodeIdx(pub usize); + +#[derive(Clone, Copy, Debug)] +pub struct CodeOffset(pub usize); + +#[derive(Debug)] +pub enum OpCode { + /// Push a constant onto the stack. + OpConstant(ConstantIdx), + + // Literal pushes + OpNil, + OpTrue, + OpFalse, + + /// Return from the current function. + OpReturn, + + // Boolean & comparison operators + OpNot, + OpEqual, + OpGreater, + OpLess, + + /// Unary negation + OpNegate, + + // Arithmetic operators + OpAdd, + OpSubtract, + OpMultiply, + OpDivide, + + // Built in operations + OpPrint, + OpPop, + + // Variable management + OpDefineGlobal(ConstantIdx), + OpGetGlobal(ConstantIdx), + OpSetGlobal(ConstantIdx), + OpGetLocal(StackIdx), + OpSetLocal(StackIdx), + + // Control flow + OpJumpPlaceholder(bool), + OpJump(CodeOffset), + OpJumpIfFalse(CodeOffset), +} diff --git a/users/tazjin/rlox/src/bytecode/tests.rs b/users/tazjin/rlox/src/bytecode/tests.rs new file mode 100644 index 000000000000..bc7d6cb878f8 --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/tests.rs @@ -0,0 +1,152 @@ +use super::value::Value; +use super::*; + +use crate::Lox; + +fn expect(code: &str, value: Value) { + let result = Interpreter::create() + .interpret(code.into()) + .expect("evaluation failed"); + assert_eq!(result, value); +} + +fn expect_num(code: &str, value: f64) { + expect(code, Value::Number(value)) +} + +fn expect_bool(code: &str, value: bool) { + expect(code, Value::Bool(value)) +} + +fn expect_str(code: &str, value: &str) { + expect(code, Value::String(value.to_string().into())) +} + +#[test] +fn numbers() { + expect_num("1;", 1.0); + expect_num("13.37;", 13.37); +} + +#[test] +fn negative_numbers() { + // Note: This technically tests unary operators. + expect_num("-1;", -1.0); + expect_num("-13.37;", -13.37); +} + +#[test] +fn terms() { + expect_num("1 + 2;", 3.0); + expect_num("3 - 1;", 2.0); + expect_num("0.7 + 0.3;", 1.0); + expect_num("1 + -3;", -2.0); + expect_num("-1 - -1;", 0.0); + expect_num("10 - -10 + 10;", 30.0); +} + +#[test] +fn factors() { + expect_num("1 * 2;", 2.0); + expect_num("10 / 5;", 2.0); + expect_num("0.7 * 4 / 1.4;", 2.0); + expect_num("10 * -10 / 10;", -10.0); +} + +#[test] +fn arithmetic() { + expect_num("10 - 3 * 2;", 4.0); + expect_num("-4 * -4 + (14 - 5);", 25.0); + expect_num("(702 + 408) - ((239 - 734) / -5) + -4;", 1007.0); +} + +#[test] +fn trivial_literals() { + expect("true;", Value::Bool(true)); + expect("false;", Value::Bool(false)); + expect("nil;", Value::Nil); +} + +#[test] +fn negation() { + expect_bool("!true;", false); + expect_bool("!false;", true); + expect_bool("!nil;", true); + expect_bool("!13.5;", false); + expect_bool("!-42;", false); +} + +#[test] +fn equality() { + expect_bool("42 == 42;", true); + expect_bool("42 != 42;", false); + expect_bool("42 == 42.0;", true); + + expect_bool("true == true;", true); + expect_bool("true == false;", false); + expect_bool("true == !false;", true); + expect_bool("true != true;", false); + expect_bool("true != false;", true); + + expect_bool("42 == false;", false); + expect_bool("42 == true;", false); + expect_bool("!42 == !true;", true); +} + +#[test] +fn comparisons() { + expect_bool("42 > 23;", true); + expect_bool("42 < 23;", false); + expect_bool("42 <= 42;", true); + expect_bool("42 <= 23;", false); + expect_bool("42 >= 42;", true); + expect_bool("42 >= 23;", true); +} + +#[test] +fn strings() { + expect_str("\"hello\";", "hello"); + expect_str("\"hello\" + \" world\";", "hello world"); +} + +#[test] +fn global_variables() { + expect_num("var a = 5; a;", 5.0); + expect_num("var a = 5; var b = 2; a * b;", 10.0); + expect_str( + "var greeting = \"hello\"; var name = \"Zubnog\"; greeting + \" \" + name;", + "hello Zubnog", + ); +} + +#[test] +fn global_assignment() { + expect_str( + r#" + var breakfast = "beignets"; + var beverage = "cafe au lait"; + breakfast = "beignets with " + beverage; + breakfast; + "#, + "beignets with cafe au lait", + ); +} + +#[test] +fn local_variables() { + expect_num( + r#" + var a = 10; + var b = 5; + var result = 0; + { + var b = 10; + var c = 2; + result = a * b * c; + } + + result; + "#, + 200.0, + ); +} diff --git a/users/tazjin/rlox/src/bytecode/value.rs b/users/tazjin/rlox/src/bytecode/value.rs new file mode 100644 index 000000000000..4170efadf8fe --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/value.rs @@ -0,0 +1,37 @@ +use super::interner::InternedStr; + +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + Nil, + Bool(bool), + Number(f64), + String(LoxString), +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum LoxString { + Heap(String), + Interned(InternedStr), +} + +impl From<String> for LoxString { + fn from(s: String) -> Self { + LoxString::Heap(s) + } +} + +impl From<InternedStr> for LoxString { + fn from(s: InternedStr) -> Self { + LoxString::Interned(s) + } +} + +impl Value { + pub fn is_falsey(&self) -> bool { + match self { + Value::Nil => true, + Value::Bool(false) => true, + _ => false, + } + } +} diff --git a/users/tazjin/rlox/src/bytecode/vm.rs b/users/tazjin/rlox/src/bytecode/vm.rs new file mode 100644 index 000000000000..d287ec7cb8c5 --- /dev/null +++ b/users/tazjin/rlox/src/bytecode/vm.rs @@ -0,0 +1,277 @@ +use std::collections::HashMap; + +use super::chunk; +use super::errors::*; +use super::interner::Interner; +use super::opcode::OpCode; +use super::value::{LoxString, Value}; + +pub struct VM { + chunk: chunk::Chunk, + + // TODO(tazjin): Accessing array elements constantly is not ideal, + // lets see if something clever can be done with iterators. + ip: usize, + + stack: Vec<Value>, + strings: Interner, + + globals: HashMap<LoxString, Value>, + + // Operations that consume values from the stack without pushing + // anything leave their last value in this slot, which makes it + // possible to return values from interpreters that ran code which + // ended with a statement. + last_drop: Option<Value>, +} + +impl VM { + fn push(&mut self, value: Value) { + self.stack.push(value) + } + + fn pop(&mut self) -> Value { + self.stack.pop().expect("fatal error: stack empty!") + } +} + +macro_rules! with_type { + ( $self:ident, $val:ident, $type:pat, $body:expr ) => { + match $val { + $type => $body, + _ => { + return Err(Error { + line: $self.chunk.get_line($self.ip - 1), + kind: ErrorKind::TypeError(format!( + "Expected type {}, but found value: {:?}", + stringify!($type), + $val, + )), + }) + } + } + }; +} + +macro_rules! binary_op { + ( $vm:ident, $type:tt, $op:tt ) => { + binary_op!($vm, $type, $type, $op) + }; + + ( $vm:ident, $in_type:tt, $out_type:tt, $op:tt ) => {{ + let b = $vm.pop(); + let a = $vm.pop(); + + with_type!($vm, b, Value::$in_type(val_b), { + with_type!($vm, a, Value::$in_type(val_a), { + $vm.push(Value::$out_type(val_a $op val_b)) + }) + }) + }}; +} + +impl VM { + fn run(&mut self) -> LoxResult<Value> { + loop { + let op = &self.chunk.code[self.ip]; + + #[cfg(feature = "disassemble")] + chunk::disassemble_instruction(&self.chunk, self.ip); + + self.ip += 1; + + match op { + OpCode::OpReturn => { + if !self.stack.is_empty() { + let val = self.pop(); + return Ok(self.return_value(val)); + } else if self.last_drop.is_some() { + let val = self.last_drop.take().unwrap(); + return Ok(self.return_value(val)); + } else { + return Ok(Value::Nil); + } + } + + OpCode::OpConstant(idx) => { + let c = self.chunk.constant(*idx).clone(); + self.push(c); + } + + OpCode::OpNil => self.push(Value::Nil), + OpCode::OpTrue => self.push(Value::Bool(true)), + OpCode::OpFalse => self.push(Value::Bool(false)), + + OpCode::OpNot => { + let v = self.pop(); + self.push(Value::Bool(v.is_falsey())); + } + + OpCode::OpEqual => { + let b = self.pop(); + let a = self.pop(); + self.push(Value::Bool(a == b)); + } + + OpCode::OpLess => binary_op!(self, Number, Bool, <), + OpCode::OpGreater => binary_op!(self, Number, Bool, >), + + OpCode::OpNegate => { + let v = self.pop(); + with_type!( + self, + v, + Value::Number(num), + self.push(Value::Number(-num)) + ); + } + + OpCode::OpSubtract => binary_op!(self, Number, -), + OpCode::OpMultiply => binary_op!(self, Number, *), + OpCode::OpDivide => binary_op!(self, Number, /), + + OpCode::OpAdd => { + let b = self.pop(); + let a = self.pop(); + + match (a, b) { + (Value::String(s_a), Value::String(s_b)) => { + let mut new_s = self.resolve_str(&s_a).to_string(); + new_s.push_str(self.resolve_str(&s_b)); + self.push(Value::String(new_s.into())); + } + + (Value::Number(n_a), Value::Number(n_b)) => + self.push(Value::Number(n_a + n_b)), + + _ => return Err(Error { + line: self.chunk.get_line(self.ip - 1), + kind: ErrorKind::TypeError( + "'+' operator only works on strings and numbers".into() + ), + }) + } + } + + OpCode::OpPrint => { + let val = self.pop(); + println!("{}", self.print_value(val)); + } + + OpCode::OpPop => { + self.last_drop = Some(self.pop()); + } + + OpCode::OpDefineGlobal(name_idx) => { + let name = self.chunk.constant(*name_idx); + with_type!(self, name, Value::String(name), { + let name = name.clone(); + let val = self.pop(); + self.globals.insert(name, val); + }); + } + + OpCode::OpGetGlobal(name_idx) => { + let name = self.chunk.constant(*name_idx); + with_type!(self, name, Value::String(name), { + let val = match self.globals.get(name) { + None => unimplemented!("variable not found error"), + Some(val) => val.clone(), + }; + self.push(val) + }); + } + + OpCode::OpSetGlobal(name_idx) => { + let name = self.chunk.constant(*name_idx).clone(); + let new_val = self.pop(); + with_type!(self, name, Value::String(name), { + match self.globals.get_mut(&name) { + None => unimplemented!("variable not found error"), + Some(val) => { + *val = new_val; + } + } + }); + } + + OpCode::OpGetLocal(local_idx) => { + let value = self.stack[local_idx.0].clone(); + self.push(value); + } + + OpCode::OpSetLocal(local_idx) => { + debug_assert!( + self.stack.len() > local_idx.0, + "stack is not currently large enough for local" + ); + self.stack[local_idx.0] = + self.stack.last().unwrap().clone(); + } + + OpCode::OpJumpPlaceholder(_) => { + panic!("unpatched jump detected - this is a fatal compiler error!"); + } + + OpCode::OpJump(offset) => { + self.ip += offset.0; + } + + OpCode::OpJumpIfFalse(offset) => { + if self + .stack + .last() + .expect("condition should leave a value on the stack") + .is_falsey() + { + self.ip += offset.0; + } + } + } + + #[cfg(feature = "disassemble")] + println!("=> {:?}", self.stack); + } + } + + // For some types of values (e.g. interned strings), returns + // should no longer include any references into the interpreter. + fn return_value(&self, val: Value) -> Value { + match val { + Value::String(string @ LoxString::Interned(_)) => { + Value::String(self.resolve_str(&string).to_string().into()) + } + _ => val, + } + } + + fn resolve_str<'a>(&'a self, string: &'a LoxString) -> &'a str { + match string { + LoxString::Heap(s) => s.as_str(), + LoxString::Interned(id) => self.strings.lookup(*id), + } + } + + fn print_value(&self, val: Value) -> String { + match val { + Value::String(LoxString::Heap(s)) => s, + Value::String(LoxString::Interned(id)) => { + self.strings.lookup(id).into() + } + _ => format!("{:?}", val), + } + } +} + +pub fn interpret(strings: Interner, chunk: chunk::Chunk) -> LoxResult<Value> { + let mut vm = VM { + chunk, + strings, + globals: HashMap::new(), + ip: 0, + stack: vec![], + last_drop: None, + }; + + vm.run() +} diff --git a/users/tazjin/rlox/src/main.rs b/users/tazjin/rlox/src/main.rs new file mode 100644 index 000000000000..2d8cf4f354ea --- /dev/null +++ b/users/tazjin/rlox/src/main.rs @@ -0,0 +1,80 @@ +use std::env; +use std::fs; +use std::io; +use std::io::Write; +use std::process; + +mod bytecode; +mod scanner; +mod treewalk; + +/// Trait for making the different interpreters callable in the same +/// way. +pub trait Lox { + type Value: std::fmt::Debug; + type Error: std::fmt::Display; + + fn create() -> Self; + fn interpret( + &mut self, + source: String, + ) -> Result<Self::Value, Vec<Self::Error>>; +} + +fn main() { + let mut args = env::args(); + if args.len() > 2 { + println!("Usage: rlox [script]"); + process::exit(1); + } + + match env::var("LOX_INTERPRETER").as_ref().map(String::as_str) { + Ok("treewalk") => { + pick::<treewalk::interpreter::Interpreter>(args.nth(1)) + } + _ => pick::<bytecode::Interpreter>(args.nth(1)), + } +} + +fn pick<I: Lox>(file_arg: Option<String>) { + if let Some(file) = file_arg { + run_file::<I>(&file); + } else { + run_prompt::<I>(); + } +} + +// Run Lox code from a file and print results to stdout +fn run_file<I: Lox>(file: &str) { + let contents = + fs::read_to_string(file).expect("failed to read the input file"); + let mut lox = I::create(); + run(&mut lox, contents); +} + +// Evaluate Lox code interactively in a shitty REPL. +fn run_prompt<I: Lox>() { + let mut line = String::new(); + let mut lox = I::create(); + + loop { + print!("> "); + io::stdout().flush().unwrap(); + io::stdin() + .read_line(&mut line) + .expect("failed to read user input"); + run(&mut lox, std::mem::take(&mut line)); + line.clear(); + } +} + +fn run<I: Lox>(lox: &mut I, code: String) { + match lox.interpret(code) { + Ok(result) => println!("=> {:?}", result), + Err(errors) => { + for error in errors { + eprintln!("{}", error); + } + } + } +} diff --git a/users/tazjin/rlox/src/scanner.rs b/users/tazjin/rlox/src/scanner.rs new file mode 100644 index 000000000000..4e8f07b61f5e --- /dev/null +++ b/users/tazjin/rlox/src/scanner.rs @@ -0,0 +1,291 @@ +#[derive(Clone, Debug, PartialEq)] +pub enum TokenKind { + // Single-character tokens. + LeftParen, + RightParen, + LeftBrace, + RightBrace, + Comma, + Dot, + Minus, + Plus, + Semicolon, + Slash, + Star, + + // One or two character tokens. + Bang, + BangEqual, + Equal, + EqualEqual, + Greater, + GreaterEqual, + Less, + LessEqual, + + // Literals. + Identifier(String), + String(String), + Number(f64), + True, + False, + Nil, + + // Keywords. + And, + Class, + Else, + Fun, + For, + If, + Or, + Print, + Return, + Super, + This, + Var, + While, + + // Special things + Eof, +} + +#[derive(Clone, Debug)] +pub struct Token { + pub kind: TokenKind, + pub lexeme: String, + pub line: usize, +} + +pub enum ScannerError { + UnexpectedChar { line: usize, unexpected: char }, + UnterminatedString { line: usize }, +} + +struct Scanner<'a> { + source: &'a [char], + tokens: Vec<Token>, + errors: Vec<ScannerError>, + start: usize, // offset of first character in current lexeme + current: usize, // current offset into source + line: usize, // current line in source +} + +impl<'a> Scanner<'a> { + fn is_at_end(&self) -> bool { + return self.current >= self.source.len(); + } + + fn advance(&mut self) -> char { + self.current += 1; + self.source[self.current - 1] + } + + fn add_token(&mut self, kind: TokenKind) { + let lexeme = &self.source[self.start..self.current]; + self.tokens.push(Token { + kind, + lexeme: lexeme.into_iter().collect(), + line: self.line, + }) + } + + fn scan_token(&mut self) { + match self.advance() { + // simple single-character tokens + '(' => self.add_token(TokenKind::LeftParen), + ')' => self.add_token(TokenKind::RightParen), + '{' => self.add_token(TokenKind::LeftBrace), + '}' => self.add_token(TokenKind::RightBrace), + ',' => self.add_token(TokenKind::Comma), + '.' => self.add_token(TokenKind::Dot), + '-' => self.add_token(TokenKind::Minus), + '+' => self.add_token(TokenKind::Plus), + ';' => self.add_token(TokenKind::Semicolon), + '*' => self.add_token(TokenKind::Star), + + // possible multi-character tokens + '!' => self.add_if_next('=', TokenKind::BangEqual, TokenKind::Bang), + '=' => { + self.add_if_next('=', TokenKind::EqualEqual, TokenKind::Equal) + } + '<' => self.add_if_next('=', TokenKind::LessEqual, TokenKind::Less), + '>' => self.add_if_next( + '=', + TokenKind::GreaterEqual, + TokenKind::Greater, + ), + + '/' => { + // support comments until EOL by discarding characters + if self.match_next('/') { + while self.peek() != '\n' && !self.is_at_end() { + self.advance(); + } + } else { + self.add_token(TokenKind::Slash); + } + } + + // ignore whitespace + ws if ws.is_whitespace() => { + if ws == '\n' { + self.line += 1 + } + } + + '"' => self.scan_string(), + + digit if digit.is_digit(10) => self.scan_number(), + + chr if chr.is_alphabetic() || chr == '_' => self.scan_identifier(), + + unexpected => self.errors.push(ScannerError::UnexpectedChar { + line: self.line, + unexpected, + }), + }; + } + + fn match_next(&mut self, expected: char) -> bool { + if self.is_at_end() || self.source[self.current] != expected { + false + } else { + self.current += 1; + true + } + } + + fn add_if_next(&mut self, expected: char, then: TokenKind, or: TokenKind) { + if self.match_next(expected) { + self.add_token(then); + } else { + self.add_token(or); + } + } + + fn peek(&self) -> char { + if self.is_at_end() { + return '\0'; + } else { + return self.source[self.current]; + } + } + + fn peek_next(&self) -> char { + if self.current + 1 >= self.source.len() { + return '\0'; + } else { + return self.source[self.current + 1]; + } + } + + fn scan_string(&mut self) { + while self.peek() != '"' && !self.is_at_end() { + if self.peek() == '\n' { + self.line += 1; + } + + self.advance(); + } + + if self.is_at_end() { + self.errors + .push(ScannerError::UnterminatedString { line: self.line }); + return; + } + + // closing '"' + self.advance(); + + // add token without surrounding quotes + let string: String = self.source[(self.start + 1)..(self.current - 1)] + .iter() + .collect(); + self.add_token(TokenKind::String(string)); + } + + fn scan_number(&mut self) { + while self.peek().is_digit(10) { + self.advance(); + } + + // Look for a fractional part + if self.peek() == '.' && self.peek_next().is_digit(10) { + // consume '.' + self.advance(); + + while self.peek().is_digit(10) { + self.advance(); + } + } + + let num: f64 = self.source[self.start..self.current] + .iter() + .collect::<String>() + .parse() + .expect("float parsing should always work"); + + self.add_token(TokenKind::Number(num)); + } + + fn scan_identifier(&mut self) { + while self.peek().is_alphanumeric() || self.peek() == '_' { + self.advance(); + } + + let ident: String = + self.source[self.start..self.current].iter().collect(); + + // Determine whether this is an identifier, or a keyword: + let token_kind = match ident.as_str() { + "and" => TokenKind::And, + "class" => TokenKind::Class, + "else" => TokenKind::Else, + "false" => TokenKind::False, + "for" => TokenKind::For, + "fun" => TokenKind::Fun, + "if" => TokenKind::If, + "nil" => TokenKind::Nil, + "or" => TokenKind::Or, + "print" => TokenKind::Print, + "return" => TokenKind::Return, + "super" => TokenKind::Super, + "this" => TokenKind::This, + "true" => TokenKind::True, + "var" => TokenKind::Var, + "while" => TokenKind::While, + _ => TokenKind::Identifier(ident), + }; + + self.add_token(token_kind); + } + + fn scan_tokens(&mut self) { + while !self.is_at_end() { + self.start = self.current; + self.scan_token(); + } + + self.add_token(TokenKind::Eof); + } +} + +pub fn scan<'a>(input: &'a [char]) -> Result<Vec<Token>, Vec<ScannerError>> { + let mut scanner = Scanner { + source: &input, + tokens: vec![], + errors: vec![], + start: 0, + current: 0, + line: 0, + }; + + scanner.scan_tokens(); + + if !scanner.errors.is_empty() { + return Err(scanner.errors); + } + + return Ok(scanner.tokens); +} diff --git a/users/tazjin/rlox/src/treewalk/errors.rs b/users/tazjin/rlox/src/treewalk/errors.rs new file mode 100644 index 000000000000..391663d51b18 --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/errors.rs @@ -0,0 +1,59 @@ +use crate::scanner::ScannerError; +use crate::treewalk::interpreter::Value; + +use std::fmt; + +#[derive(Debug)] +pub enum ErrorKind { + UnexpectedChar(char), + UnterminatedString, + UnmatchedParens, + ExpectedExpression(String), + ExpectedSemicolon, + ExpectedClosingBrace, + ExpectedToken(&'static str), + TypeError(String), + UndefinedVariable(String), + InternalError(String), + InvalidAssignmentTarget(String), + RuntimeError(String), + StaticError(String), + + // This variant is not an error, rather it is used for + // short-circuiting out of a function body that hits a `return` + // statement. + // + // It's implemented this way because in the original book the + // author uses exceptions for control flow, and this is the + // closest equivalent that I had available without diverging too + // much. + FunctionReturn(Value), +} + +#[derive(Debug)] +pub struct Error { + pub line: usize, + pub kind: ErrorKind, +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[line {}] Error: {:?}", self.line, self.kind) + } +} + +impl From<ScannerError> for Error { + fn from(err: ScannerError) -> Self { + match err { + ScannerError::UnexpectedChar { line, unexpected } => Error { + line, + kind: ErrorKind::UnexpectedChar(unexpected), + }, + + ScannerError::UnterminatedString { line } => Error { + line, + kind: ErrorKind::UnterminatedString, + }, + } + } +} diff --git a/users/tazjin/rlox/src/treewalk/interpreter.rs b/users/tazjin/rlox/src/treewalk/interpreter.rs new file mode 100644 index 000000000000..d9fe33661684 --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/interpreter.rs @@ -0,0 +1,556 @@ +use crate::treewalk::errors::{Error, ErrorKind}; +use crate::treewalk::parser::{self, Block, Expr, Literal, Statement}; +use crate::treewalk::resolver; +use crate::treewalk::scanner::{self, TokenKind}; +use crate::Lox; +use std::collections::HashMap; +use std::rc::Rc; +use std::sync::RwLock; + +// Implementation of built-in functions. +mod builtins; + +#[cfg(test)] +mod tests; + +// Tree-walk interpreter + +// Representation of all callables, including builtins & user-defined +// functions. +#[derive(Clone, Debug)] +pub enum Callable { + Builtin(&'static dyn builtins::Builtin), + Function { + func: Rc<parser::Function>, + closure: Rc<RwLock<Environment>>, + }, +} + +impl Callable { + fn arity(&self) -> usize { + match self { + Callable::Builtin(builtin) => builtin.arity(), + Callable::Function { func, .. } => func.params.len(), + } + } + + fn call( + &self, + lox: &mut Interpreter, + args: Vec<Value>, + ) -> Result<Value, Error> { + match self { + Callable::Builtin(builtin) => builtin.call(args), + + Callable::Function { func, closure } => { + let mut fn_env: Environment = Default::default(); + fn_env.enclosing = Some(closure.clone()); + + for (param, value) in func.params.iter().zip(args.into_iter()) { + fn_env.define(param, value)?; + } + + let result = lox.interpret_block_with_env( + Some(Rc::new(RwLock::new(fn_env))), + &func.body, + ); + + match result { + // extract returned values if applicable + Err(Error { + kind: ErrorKind::FunctionReturn(value), + .. + }) => Ok(value), + + // otherwise just return the result itself + _ => result, + } + } + } + } +} + +// Representation of an in-language value. +#[derive(Clone, Debug)] +pub enum Value { + Literal(Literal), + Callable(Callable), +} + +impl PartialEq for Value { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Value::Literal(lhs), Value::Literal(rhs)) => lhs == rhs, + // functions do not have equality + _ => false, + } + } +} + +impl From<Literal> for Value { + fn from(lit: Literal) -> Value { + Value::Literal(lit) + } +} + +impl Value { + fn expect_literal(self) -> Result<Literal, Error> { + match self { + Value::Literal(lit) => Ok(lit), + _ => unimplemented!(), // which error? which line? + } + } +} + +#[derive(Debug, Default)] +pub struct Environment { + enclosing: Option<Rc<RwLock<Environment>>>, + values: HashMap<String, Value>, +} + +impl Environment { + fn define( + &mut self, + name: &scanner::Token, + value: Value, + ) -> Result<(), Error> { + let ident = identifier_str(name)?; + self.values.insert(ident.into(), value); + Ok(()) + } + + fn get( + &self, + ident: &str, + line: usize, + depth: usize, + ) -> Result<Value, Error> { + if depth > 0 { + match &self.enclosing { + None => { + return Err(Error { + line, + kind: ErrorKind::InternalError(format!( + "invalid depth {} for {}", + depth, ident + )), + }) + } + Some(parent) => { + let env = parent + .read() + .expect("fatal: environment lock poisoned"); + return env.get(ident, line, depth - 1); + } + } + } + + self.values + .get(ident) + .map(Clone::clone) + .ok_or_else(|| Error { + line, + kind: ErrorKind::UndefinedVariable(ident.into()), + }) + } + + fn assign( + &mut self, + name: &scanner::Token, + value: Value, + ) -> Result<(), Error> { + let ident = identifier_str(name)?; + + match self.values.get_mut(ident) { + Some(target) => { + *target = value; + Ok(()) + } + None => { + if let Some(parent) = &self.enclosing { + return parent.write().unwrap().assign(name, value); + } + + Err(Error { + line: name.line, + kind: ErrorKind::UndefinedVariable(ident.into()), + }) + } + } + } +} + +fn identifier_str(name: &scanner::Token) -> Result<&str, Error> { + if let TokenKind::Identifier(ident) = &name.kind { + Ok(ident) + } else { + Err(Error { + line: name.line, + kind: ErrorKind::InternalError("unexpected identifier kind".into()), + }) + } +} + +#[derive(Debug)] +pub struct Interpreter { + env: Rc<RwLock<Environment>>, +} + +impl Lox for Interpreter { + type Value = Value; + type Error = Error; + + /// Create a new interpreter and configure the initial global + /// variable set. + fn create() -> Self { + let mut globals = HashMap::new(); + + globals.insert( + "clock".into(), + Value::Callable(Callable::Builtin(&builtins::Clock {})), + ); + + Interpreter { + env: Rc::new(RwLock::new(Environment { + enclosing: None, + values: globals, + })), + } + } + + fn interpret(&mut self, code: String) -> Result<Value, Vec<Error>> { + let chars: Vec<char> = code.chars().collect(); + + let mut program = scanner::scan(&chars) + .map_err(|errors| errors.into_iter().map(Into::into).collect()) + .and_then(|tokens| parser::parse(tokens))?; + + let globals = self + .env + .read() + .expect("static globals lock poisoned") + .values + .keys() + .map(Clone::clone) + .collect::<Vec<String>>(); + + resolver::resolve(&globals, &mut program).map_err(|e| vec![e])?; + self.interpret_block_with_env(None, &program) + .map_err(|e| vec![e]) + } +} + +impl Interpreter { + // Environment modification helpers + fn define_var( + &mut self, + name: &scanner::Token, + value: Value, + ) -> Result<(), Error> { + self.env + .write() + .expect("environment lock is poisoned") + .define(name, value) + } + + fn assign_var( + &mut self, + name: &scanner::Token, + value: Value, + ) -> Result<(), Error> { + self.env + .write() + .expect("environment lock is poisoned") + .assign(name, value) + } + + fn get_var(&mut self, var: &parser::Variable) -> Result<Value, Error> { + let ident = identifier_str(&var.name)?; + let depth = var.depth.ok_or_else(|| Error { + line: var.name.line, + kind: ErrorKind::UndefinedVariable(ident.into()), + })?; + + self.env.read().expect("environment lock is poisoned").get( + ident, + var.name.line, + depth, + ) + } + + /// Interpret the block in the supplied environment. If no + /// environment is supplied, a new one is created using the + /// current one as its parent. + fn interpret_block_with_env( + &mut self, + env: Option<Rc<RwLock<Environment>>>, + block: &parser::Block, + ) -> Result<Value, Error> { + let env = match env { + Some(env) => env, + None => { + let env: Rc<RwLock<Environment>> = Default::default(); + set_enclosing_env(&env, self.env.clone()); + env + } + }; + + let previous = std::mem::replace(&mut self.env, env); + let result = self.interpret_block(block); + + // Swap it back, discarding the child env. + self.env = previous; + + return result; + } + + fn interpret_block(&mut self, program: &Block) -> Result<Value, Error> { + let mut value = Value::Literal(Literal::Nil); + + for stmt in program { + value = self.interpret_stmt(stmt)?; + } + + Ok(value) + } + + fn interpret_stmt(&mut self, stmt: &Statement) -> Result<Value, Error> { + let value = match stmt { + Statement::Expr(expr) => self.eval(expr)?, + Statement::Print(expr) => { + let result = self.eval(expr)?; + let output = format!("{:?}", result); + println!("{}", output); + Value::Literal(Literal::String(output)) + } + Statement::Var(var) => return self.interpret_var(var), + Statement::Block(block) => { + return self.interpret_block_with_env(None, block) + } + Statement::If(if_stmt) => return self.interpret_if(if_stmt), + Statement::While(while_stmt) => { + return self.interpret_while(while_stmt) + } + Statement::Function(func) => { + return self.interpret_function(func.clone()) + } + Statement::Return(ret) => { + return Err(Error { + line: 0, + kind: ErrorKind::FunctionReturn(self.eval(&ret.value)?), + }) + } + }; + + Ok(value) + } + + fn interpret_var(&mut self, var: &parser::Var) -> Result<Value, Error> { + let init = var.initialiser.as_ref().ok_or_else(|| Error { + line: var.name.line, + kind: ErrorKind::InternalError( + "missing variable initialiser".into(), + ), + })?; + let value = self.eval(init)?; + self.define_var(&var.name, value.clone())?; + Ok(value) + } + + fn interpret_if(&mut self, if_stmt: &parser::If) -> Result<Value, Error> { + let condition = self.eval(&if_stmt.condition)?; + + if eval_truthy(&condition) { + self.interpret_stmt(&if_stmt.then_branch) + } else if let Some(else_branch) = &if_stmt.else_branch { + self.interpret_stmt(else_branch) + } else { + Ok(Value::Literal(Literal::Nil)) + } + } + + fn interpret_while( + &mut self, + stmt: &parser::While, + ) -> Result<Value, Error> { + let mut value = Value::Literal(Literal::Nil); + while eval_truthy(&self.eval(&stmt.condition)?) { + value = self.interpret_stmt(&stmt.body)?; + } + + Ok(value) + } + + fn interpret_function( + &mut self, + func: Rc<parser::Function>, + ) -> Result<Value, Error> { + let name = func.name.clone(); + let value = Value::Callable(Callable::Function { + func, + closure: self.env.clone(), + }); + self.define_var(&name, value.clone())?; + Ok(value) + } + + fn eval(&mut self, expr: &Expr) -> Result<Value, Error> { + match expr { + Expr::Assign(assign) => self.eval_assign(assign), + Expr::Literal(lit) => Ok(lit.clone().into()), + Expr::Grouping(grouping) => self.eval(&*grouping.0), + Expr::Unary(unary) => self.eval_unary(unary), + Expr::Binary(binary) => self.eval_binary(binary), + Expr::Variable(var) => self.get_var(var), + Expr::Logical(log) => self.eval_logical(log), + Expr::Call(call) => self.eval_call(call), + } + } + + fn eval_unary(&mut self, expr: &parser::Unary) -> Result<Value, Error> { + let right = self.eval(&*expr.right)?; + + match (&expr.operator.kind, right) { + (TokenKind::Minus, Value::Literal(Literal::Number(num))) => { + Ok(Literal::Number(-num).into()) + } + (TokenKind::Bang, right) => { + Ok(Literal::Boolean(!eval_truthy(&right)).into()) + } + + (op, right) => Err(Error { + line: expr.operator.line, + kind: ErrorKind::TypeError(format!( + "Operator '{:?}' can not be called with argument '{:?}'", + op, right + )), + }), + } + } + + fn eval_binary(&mut self, expr: &parser::Binary) -> Result<Value, Error> { + let left = self.eval(&*expr.left)?.expect_literal()?; + let right = self.eval(&*expr.right)?.expect_literal()?; + + let result = match (&expr.operator.kind, left, right) { + // Numeric + (TokenKind::Minus, Literal::Number(l), Literal::Number(r)) => Literal::Number(l - r), + (TokenKind::Slash, Literal::Number(l), Literal::Number(r)) => Literal::Number(l / r), + (TokenKind::Star, Literal::Number(l), Literal::Number(r)) => Literal::Number(l * r), + (TokenKind::Plus, Literal::Number(l), Literal::Number(r)) => Literal::Number(l + r), + + // Strings + (TokenKind::Plus, Literal::String(l), Literal::String(r)) => { + Literal::String(format!("{}{}", l, r)) + } + + // Comparators (on numbers only?) + (TokenKind::Greater, Literal::Number(l), Literal::Number(r)) => Literal::Boolean(l > r), + (TokenKind::GreaterEqual, Literal::Number(l), Literal::Number(r)) => { + Literal::Boolean(l >= r) + } + (TokenKind::Less, Literal::Number(l), Literal::Number(r)) => Literal::Boolean(l < r), + (TokenKind::LessEqual, Literal::Number(l), Literal::Number(r)) => { + Literal::Boolean(l <= r) + } + + // Equality + (TokenKind::Equal, l, r) => Literal::Boolean(l == r), + (TokenKind::BangEqual, l, r) => Literal::Boolean(l != r), + + (op, left, right) => { + return Err(Error { + line: expr.operator.line, + kind: ErrorKind::TypeError(format!( + "Operator '{:?}' can not be called with arguments '({:?}, {:?})'", + op, left, right + )), + }) + } + }; + + Ok(result.into()) + } + + fn eval_assign(&mut self, assign: &parser::Assign) -> Result<Value, Error> { + let value = self.eval(&assign.value)?; + self.assign_var(&assign.name, value.clone())?; + Ok(value) + } + + fn eval_logical( + &mut self, + logical: &parser::Logical, + ) -> Result<Value, Error> { + let left = eval_truthy(&self.eval(&logical.left)?); + let right = eval_truthy(&self.eval(&logical.right)?); + + match &logical.operator.kind { + TokenKind::And => Ok(Literal::Boolean(left && right).into()), + TokenKind::Or => Ok(Literal::Boolean(left || right).into()), + kind => Err(Error { + line: logical.operator.line, + kind: ErrorKind::InternalError(format!( + "Invalid logical operator: {:?}", + kind + )), + }), + } + } + + fn eval_call(&mut self, call: &parser::Call) -> Result<Value, Error> { + let callable = match self.eval(&call.callee)? { + Value::Callable(c) => c, + Value::Literal(v) => { + return Err(Error { + line: call.paren.line, + kind: ErrorKind::RuntimeError(format!( + "not callable: {:?}", + v + )), + }) + } + }; + + let mut args = vec![]; + for arg in &call.args { + args.push(self.eval(arg)?); + } + + if callable.arity() != args.len() { + return Err(Error { + line: call.paren.line, + kind: ErrorKind::RuntimeError(format!( + "Expected {} arguments, but got {}", + callable.arity(), + args.len(), + )), + }); + } + + callable.call(self, args) + } +} + +// Interpreter functions not dependent on interpreter-state. + +fn eval_truthy(lit: &Value) -> bool { + if let Value::Literal(lit) = lit { + match lit { + Literal::Nil => false, + Literal::Boolean(b) => *b, + _ => true, + } + } else { + false + } +} + +fn set_enclosing_env( + this: &RwLock<Environment>, + parent: Rc<RwLock<Environment>>, +) { + this.write() + .expect("environment lock is poisoned") + .enclosing = Some(parent); +} diff --git a/users/tazjin/rlox/src/treewalk/interpreter/builtins.rs b/users/tazjin/rlox/src/treewalk/interpreter/builtins.rs new file mode 100644 index 000000000000..c502d2a1718a --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/interpreter/builtins.rs @@ -0,0 +1,25 @@ +use std::fmt; +use std::time::{SystemTime, UNIX_EPOCH}; + +use crate::treewalk::errors::Error; +use crate::treewalk::interpreter::Value; +use crate::treewalk::parser::Literal; + +pub trait Builtin: fmt::Debug { + fn arity(&self) -> usize; + fn call(&self, args: Vec<Value>) -> Result<Value, Error>; +} + +// Builtin to return the current timestamp. +#[derive(Debug)] +pub struct Clock {} +impl Builtin for Clock { + fn arity(&self) -> usize { + 0 + } + + fn call(&self, _args: Vec<Value>) -> Result<Value, Error> { + let now = SystemTime::now().duration_since(UNIX_EPOCH).unwrap(); + Ok(Value::Literal(Literal::Number(now.as_secs() as f64))) + } +} diff --git a/users/tazjin/rlox/src/treewalk/interpreter/tests.rs b/users/tazjin/rlox/src/treewalk/interpreter/tests.rs new file mode 100644 index 000000000000..2fc6f4fee978 --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/interpreter/tests.rs @@ -0,0 +1,97 @@ +use super::*; + +/// Evaluate a code snippet, returning a value. +fn parse_eval(code: &str) -> Value { + Interpreter::create() + .interpret(code.into()) + .expect("could not interpret code") +} + +#[test] +fn test_if() { + let result = parse_eval( + r#" +if (42 > 23) + "pass"; +else + "fail"; +"#, + ); + + assert_eq!(Value::Literal(Literal::String("pass".into())), result,); +} + +#[test] +fn test_scope() { + let result = parse_eval( + r#" +var result = ""; + +var a = "global a, "; +var b = "global b, "; +var c = "global c"; + +{ + var a = "outer a, "; + var b = "outer b, "; + + { + var a = "inner a, "; + result = a + b + c; + } +} +"#, + ); + + assert_eq!( + Value::Literal(Literal::String("inner a, outer b, global c".into())), + result, + ); +} + +#[test] +fn test_binary_operators() { + assert_eq!(Value::Literal(Literal::Number(42.0)), parse_eval("40 + 2;")); + + assert_eq!( + Value::Literal(Literal::String("foobar".into())), + parse_eval("\"foo\" + \"bar\";") + ); +} + +#[test] +fn test_functions() { + let result = parse_eval( + r#" +fun add(a, b, c) { + a + b + c; +} + +add(1, 2, 3); +"#, + ); + + assert_eq!(Value::Literal(Literal::Number(6.0)), result); +} + +#[test] +fn test_closure() { + let result = parse_eval( + r#" +fun makeCounter() { + var i = 0; + fun count() { + i = i + 1; + } + + return count; +} + +var counter = makeCounter(); +counter(); // "1". +counter(); // "2". +"#, + ); + + assert_eq!(Value::Literal(Literal::Number(2.0)), result); +} diff --git a/users/tazjin/rlox/src/treewalk/mod.rs b/users/tazjin/rlox/src/treewalk/mod.rs new file mode 100644 index 000000000000..2d82b3320a90 --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/mod.rs @@ -0,0 +1,6 @@ +use crate::scanner; + +mod errors; +pub mod interpreter; +mod parser; +mod resolver; diff --git a/users/tazjin/rlox/src/treewalk/parser.rs b/users/tazjin/rlox/src/treewalk/parser.rs new file mode 100644 index 000000000000..003cc34b4665 --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/parser.rs @@ -0,0 +1,716 @@ +// This implements the grammar of Lox as described starting in the +// Crafting Interpreters chapter "Representing Code". Note that the +// upstream Java implementation works around Java being bad at value +// classes by writing a code generator for Java. +// +// My Rust implementation skips this step because it's unnecessary, we +// have real types. +use crate::treewalk::errors::{Error, ErrorKind}; +use crate::treewalk::scanner::{Token, TokenKind}; +use std::rc::Rc; + +// AST + +#[derive(Debug)] +pub struct Assign { + pub name: Token, + pub value: Box<Expr>, + pub depth: Option<usize>, +} + +#[derive(Debug)] +pub struct Binary { + pub left: Box<Expr>, + pub operator: Token, + pub right: Box<Expr>, +} + +#[derive(Debug)] +pub struct Logical { + pub left: Box<Expr>, + pub operator: Token, + pub right: Box<Expr>, +} + +#[derive(Debug)] +pub struct Grouping(pub Box<Expr>); + +#[derive(Debug, Clone, PartialEq)] +pub enum Literal { + Boolean(bool), + Number(f64), + String(String), + Nil, +} + +#[derive(Debug)] +pub struct Unary { + pub operator: Token, + pub right: Box<Expr>, +} + +#[derive(Debug)] +pub struct Call { + pub callee: Box<Expr>, + pub paren: Token, + pub args: Vec<Expr>, +} + +// Not to be confused with `Var`, which is for assignment. +#[derive(Debug)] +pub struct Variable { + pub name: Token, + pub depth: Option<usize>, +} + +#[derive(Debug)] +pub enum Expr { + Assign(Assign), + Binary(Binary), + Grouping(Grouping), + Literal(Literal), + Unary(Unary), + Call(Call), + Variable(Variable), + Logical(Logical), +} + +// Variable assignment. Not to be confused with `Variable`, which is +// for access. +#[derive(Debug)] +pub struct Var { + pub name: Token, + pub initialiser: Option<Expr>, +} + +#[derive(Debug)] +pub struct Return { + pub value: Expr, +} + +#[derive(Debug)] +pub struct If { + pub condition: Expr, + pub then_branch: Box<Statement>, + pub else_branch: Option<Box<Statement>>, +} + +#[derive(Debug)] +pub struct While { + pub condition: Expr, + pub body: Box<Statement>, +} + +pub type Block = Vec<Statement>; + +#[derive(Debug)] +pub struct Function { + pub name: Token, + pub params: Vec<Token>, + pub body: Block, +} + +#[derive(Debug)] +pub enum Statement { + Expr(Expr), + Print(Expr), + Var(Var), + Block(Block), + If(If), + While(While), + Function(Rc<Function>), + Return(Return), +} + +// Parser + +/* +program → declaration* EOF ; + +declaration → funDecl + | varDecl + | statement ; + +funDecl → "fun" function ; +function → IDENTIFIER "(" parameters? ")" block ; +parameters → IDENTIFIER ( "," IDENTIFIER )* ; + + +statement → exprStmt + | forStmt + | ifStmt + | printStmt + | returnStmt + | whileStmt + | block ; + +forStmt → "for" "(" ( varDecl | exprStmt | ";" ) + expression? ";" + expression? ")" statement ; + +returnStmt → "return" expression? ";" ; + +whileStmt → "while" "(" expression ")" statement ; + +exprStmt → expression ";" ; + +ifStmt → "if" "(" expression ")" statement + ( "else" statement )? ; + +printStmt → "print" expression ";" ; + +expression → assignment ; +assignment → IDENTIFIER "=" assignment + | logic_or ; +logic_or → logic_and ( "or" logic_and )* ; +logic_and → equality ( "and" equality )* ; +equality → comparison ( ( "!=" | "==" ) comparison )* ; +comparison → term ( ( ">" | ">=" | "<" | "<=" ) term )* ; +term → factor ( ( "-" | "+" ) factor )* ; +factor → unary ( ( "/" | "*" ) unary )* ; +unary → ( "!" | "-" ) unary | call ; +call → primary ( "(" arguments? ")" )* ; +arguments → expression ( "," expression )* ; +primary → NUMBER | STRING | "true" | "false" | "nil" + | "(" expression ")" ; +*/ + +struct Parser { + tokens: Vec<Token>, + current: usize, +} + +type ExprResult = Result<Expr, Error>; +type StmtResult = Result<Statement, Error>; + +impl Parser { + // recursive-descent parser functions + + fn declaration(&mut self) -> StmtResult { + if self.match_token(&TokenKind::Fun) { + return self.function(); + } + + if self.match_token(&TokenKind::Var) { + return self.var_declaration(); + } + + self.statement() + } + + fn function(&mut self) -> StmtResult { + let name = self.identifier("Expected function name.")?; + + self.consume( + &TokenKind::LeftParen, + ErrorKind::ExpectedToken("Expect '(' after function name."), + )?; + + let mut params = vec![]; + + if !self.check_token(&TokenKind::RightParen) { + loop { + if params.len() >= 255 { + return Err(Error { + line: self.peek().line, + kind: ErrorKind::InternalError( + "255 parameter limit exceeded.".into(), + ), + }); + } + + params.push(self.identifier("Expected parameter name.")?); + + if !self.match_token(&TokenKind::Comma) { + break; + } + } + } + + self.consume( + &TokenKind::RightParen, + ErrorKind::ExpectedToken("Expect ')' after parameters."), + )?; + + self.consume( + &TokenKind::LeftBrace, + ErrorKind::ExpectedToken("Expect '{' before function body."), + )?; + + Ok(Statement::Function(Rc::new(Function { + name, + params, + body: self.block_statement()?, + }))) + } + + fn var_declaration(&mut self) -> StmtResult { + // Since `TokenKind::Identifier` carries data, we can't use + // `consume`. + let mut var = Var { + name: self.identifier("Expected variable name.")?, + initialiser: None, + }; + + if self.match_token(&TokenKind::Equal) { + var.initialiser = Some(self.expression()?); + } + + self.consume(&TokenKind::Semicolon, ErrorKind::ExpectedSemicolon)?; + Ok(Statement::Var(var)) + } + + fn statement(&mut self) -> StmtResult { + if self.match_token(&TokenKind::Print) { + self.print_statement() + } else if self.match_token(&TokenKind::LeftBrace) { + Ok(Statement::Block(self.block_statement()?)) + } else if self.match_token(&TokenKind::If) { + self.if_statement() + } else if self.match_token(&TokenKind::While) { + self.while_statement() + } else if self.match_token(&TokenKind::For) { + self.for_statement() + } else if self.match_token(&TokenKind::Return) { + self.return_statement() + } else { + self.expr_statement() + } + } + + fn print_statement(&mut self) -> StmtResult { + let expr = self.expression()?; + self.consume(&TokenKind::Semicolon, ErrorKind::ExpectedSemicolon)?; + Ok(Statement::Print(expr)) + } + + fn block_statement(&mut self) -> Result<Block, Error> { + let mut block: Block = vec![]; + + while !self.check_token(&TokenKind::RightBrace) && !self.is_at_end() { + block.push(self.declaration()?); + } + + self.consume(&TokenKind::RightBrace, ErrorKind::ExpectedClosingBrace)?; + + Ok(block) + } + + fn if_statement(&mut self) -> StmtResult { + self.consume( + &TokenKind::LeftParen, + ErrorKind::ExpectedToken("Expected '(' after 'if'"), + )?; + let condition = self.expression()?; + self.consume( + &TokenKind::RightParen, + ErrorKind::ExpectedToken("Expected ')' after condition"), + )?; + + let then_branch = Box::new(self.statement()?); + + let mut stmt = If { + condition, + then_branch, + else_branch: Option::None, + }; + + if self.match_token(&TokenKind::Else) { + stmt.else_branch = Some(Box::new(self.statement()?)); + } + + Ok(Statement::If(stmt)) + } + + fn while_statement(&mut self) -> StmtResult { + self.consume( + &TokenKind::LeftParen, + ErrorKind::ExpectedToken("Expected '(' after 'while'"), + )?; + + let condition = self.expression()?; + + self.consume( + &TokenKind::RightParen, + ErrorKind::ExpectedToken("Expected ')' after 'while'"), + )?; + + Ok(Statement::While(While { + condition, + body: Box::new(self.statement()?), + })) + } + + fn for_statement(&mut self) -> StmtResult { + // Parsing of clauses ... + self.consume( + &TokenKind::LeftParen, + ErrorKind::ExpectedToken("Expected '(' after 'for'"), + )?; + + let initialiser = if self.match_token(&TokenKind::Semicolon) { + None + } else if self.match_token(&TokenKind::Var) { + Some(self.var_declaration()?) + } else { + Some(self.expr_statement()?) + }; + + let condition = if self.check_token(&TokenKind::Semicolon) { + // unspecified condition => infinite loop + Expr::Literal(Literal::Boolean(true)) + } else { + self.expression()? + }; + + self.consume(&TokenKind::Semicolon, ErrorKind::ExpectedSemicolon)?; + + let increment = if self.check_token(&TokenKind::RightParen) { + None + } else { + Some(self.expression()?) + }; + + self.consume( + &TokenKind::RightParen, + ErrorKind::ExpectedToken("Expected ')' after for clauses"), + )?; + + let mut body = self.statement()?; + + // ... desugaring to while + + if let Some(inc) = increment { + body = Statement::Block(vec![body, Statement::Expr(inc)]); + } + + body = Statement::While(While { + condition, + body: Box::new(body), + }); + + if let Some(init) = initialiser { + body = Statement::Block(vec![init, body]); + } + + Ok(body) + } + + fn return_statement(&mut self) -> StmtResult { + let value = self.expression()?; + self.consume(&TokenKind::Semicolon, ErrorKind::ExpectedSemicolon)?; + Ok(Statement::Return(Return { value })) + } + + fn expr_statement(&mut self) -> StmtResult { + let expr = self.expression()?; + self.consume(&TokenKind::Semicolon, ErrorKind::ExpectedSemicolon)?; + Ok(Statement::Expr(expr)) + } + + fn expression(&mut self) -> ExprResult { + self.assignment() + } + + fn assignment(&mut self) -> ExprResult { + let expr = self.logic_or()?; + + if self.match_token(&TokenKind::Equal) { + let equals = self.previous().clone(); + let value = self.assignment()?; + + if let Expr::Variable(Variable { name, .. }) = expr { + return Ok(Expr::Assign(Assign { + name, + value: Box::new(value), + depth: None, + })); + } + + return Err(Error { + line: equals.line, + kind: ErrorKind::InvalidAssignmentTarget(format!( + "{:?}", + equals + )), + }); + } + + Ok(expr) + } + + fn logic_or(&mut self) -> ExprResult { + let mut expr = self.logic_and()?; + + while self.match_token(&TokenKind::Or) { + expr = Expr::Logical(Logical { + left: Box::new(expr), + operator: self.previous().clone(), + right: Box::new(self.logic_and()?), + }) + } + + Ok(expr) + } + + fn logic_and(&mut self) -> ExprResult { + let mut expr = self.equality()?; + + while self.match_token(&TokenKind::And) { + expr = Expr::Logical(Logical { + left: Box::new(expr), + operator: self.previous().clone(), + right: Box::new(self.equality()?), + }) + } + + Ok(expr) + } + + fn equality(&mut self) -> ExprResult { + self.binary_operator( + &[TokenKind::BangEqual, TokenKind::EqualEqual], + Self::comparison, + ) + } + + fn comparison(&mut self) -> ExprResult { + self.binary_operator( + &[ + TokenKind::Greater, + TokenKind::GreaterEqual, + TokenKind::Less, + TokenKind::LessEqual, + ], + Self::term, + ) + } + + fn term(&mut self) -> ExprResult { + self.binary_operator(&[TokenKind::Minus, TokenKind::Plus], Self::factor) + } + + fn factor(&mut self) -> ExprResult { + self.binary_operator(&[TokenKind::Slash, TokenKind::Star], Self::unary) + } + + fn unary(&mut self) -> ExprResult { + if self.match_token(&TokenKind::Bang) + || self.match_token(&TokenKind::Minus) + { + return Ok(Expr::Unary(Unary { + operator: self.previous().clone(), + right: Box::new(self.unary()?), + })); + } + + return self.call(); + } + + fn call(&mut self) -> ExprResult { + let mut expr = self.primary()?; + + loop { + if self.match_token(&TokenKind::LeftParen) { + expr = self.finish_call(expr)?; + } else { + break; + } + } + + Ok(expr) + } + + fn finish_call(&mut self, callee: Expr) -> ExprResult { + let mut args = vec![]; + + if !self.check_token(&TokenKind::RightParen) { + loop { + // TODO(tazjin): Check for max args count + args.push(self.expression()?); + if !self.match_token(&TokenKind::Comma) { + break; + } + } + } + + let paren = self.consume( + &TokenKind::RightParen, + ErrorKind::ExpectedToken("Expect ')' after arguments."), + )?; + + Ok(Expr::Call(Call { + args, + callee: Box::new(callee), + paren, + })) + } + + fn primary(&mut self) -> ExprResult { + let next = self.advance(); + let literal = match next.kind { + TokenKind::True => Literal::Boolean(true), + TokenKind::False => Literal::Boolean(false), + TokenKind::Nil => Literal::Nil, + TokenKind::Number(num) => Literal::Number(num), + TokenKind::String(string) => Literal::String(string), + + TokenKind::LeftParen => { + let expr = self.expression()?; + self.consume( + &TokenKind::RightParen, + ErrorKind::UnmatchedParens, + )?; + return Ok(Expr::Grouping(Grouping(Box::new(expr)))); + } + + TokenKind::Identifier(_) => { + return Ok(Expr::Variable(Variable { + name: next, + depth: None, + })) + } + + unexpected => { + eprintln!("encountered {:?}", unexpected); + return Err(Error { + line: next.line, + kind: ErrorKind::ExpectedExpression(next.lexeme), + }); + } + }; + + Ok(Expr::Literal(literal)) + } + + // internal helpers + + fn identifier(&mut self, err: &'static str) -> Result<Token, Error> { + if let TokenKind::Identifier(_) = self.peek().kind { + Ok(self.advance()) + } else { + Err(Error { + line: self.peek().line, + kind: ErrorKind::ExpectedToken(err), + }) + } + } + + /// Check if the next token is in `oneof`, and advance if it is. + fn match_token(&mut self, token: &TokenKind) -> bool { + if self.check_token(token) { + self.advance(); + return true; + } + + false + } + + /// Return the next token and advance parser state. + fn advance(&mut self) -> Token { + if !self.is_at_end() { + self.current += 1; + } + + return self.previous().clone(); + } + + fn is_at_end(&self) -> bool { + self.check_token(&TokenKind::Eof) + } + + /// Is the next token `token`? + fn check_token(&self, token: &TokenKind) -> bool { + self.peek().kind == *token + } + + fn peek(&self) -> &Token { + &self.tokens[self.current] + } + + fn previous(&self) -> &Token { + &self.tokens[self.current - 1] + } + + fn consume( + &mut self, + kind: &TokenKind, + err: ErrorKind, + ) -> Result<Token, Error> { + if self.check_token(kind) { + return Ok(self.advance()); + } + + Err(Error { + line: self.peek().line, + kind: err, + }) + } + + fn synchronise(&mut self) { + self.advance(); + + while !self.is_at_end() { + if self.previous().kind == TokenKind::Semicolon { + return; + } + + match self.peek().kind { + TokenKind::Class + | TokenKind::Fun + | TokenKind::Var + | TokenKind::For + | TokenKind::If + | TokenKind::While + | TokenKind::Print + | TokenKind::Return => return, + + _ => { + self.advance(); + } + } + } + } + + fn binary_operator( + &mut self, + oneof: &[TokenKind], + each: fn(&mut Parser) -> ExprResult, + ) -> ExprResult { + let mut expr = each(self)?; + + while oneof.iter().any(|t| self.match_token(t)) { + expr = Expr::Binary(Binary { + left: Box::new(expr), + operator: self.previous().clone(), + right: Box::new(each(self)?), + }) + } + + return Ok(expr); + } +} + +pub fn parse(tokens: Vec<Token>) -> Result<Block, Vec<Error>> { + let mut parser = Parser { tokens, current: 0 }; + let mut program: Block = vec![]; + let mut errors: Vec<Error> = vec![]; + + while !parser.is_at_end() { + match parser.declaration() { + Err(err) => { + errors.push(err); + parser.synchronise(); + } + Ok(decl) => { + program.push(decl); + } + } + } + + if errors.is_empty() { + Ok(program) + } else { + Err(errors) + } +} diff --git a/users/tazjin/rlox/src/treewalk/resolver.rs b/users/tazjin/rlox/src/treewalk/resolver.rs new file mode 100644 index 000000000000..8231ce5a9e58 --- /dev/null +++ b/users/tazjin/rlox/src/treewalk/resolver.rs @@ -0,0 +1,214 @@ +// Resolves variable access to their specific instances in the +// environment chain. +// +// https://craftinginterpreters.com/resolving-and-binding.html + +use std::collections::HashMap; +use std::rc::Rc; + +use crate::treewalk::errors::{Error, ErrorKind}; +use crate::treewalk::parser::{self, Expr, Statement}; +use crate::treewalk::scanner::Token; + +#[derive(Default)] +struct Resolver<'a> { + scopes: Vec<HashMap<&'a str, bool>>, +} + +impl<'a> Resolver<'a> { + // AST traversal + fn resolve(&mut self, program: &'a mut parser::Block) -> Result<(), Error> { + self.begin_scope(); + for stmt in program { + self.resolve_stmt(stmt)?; + } + self.end_scope(); + + Ok(()) + } + + fn resolve_stmt(&mut self, stmt: &'a mut Statement) -> Result<(), Error> { + match stmt { + Statement::Expr(expr) => self.resolve_expr(expr), + Statement::Print(expr) => self.resolve_expr(expr), + Statement::Var(var) => self.resolve_var(var), + Statement::Return(ret) => self.resolve_expr(&mut ret.value), + Statement::Block(block) => self.resolve(block), + + Statement::If(if_stmt) => { + self.resolve_expr(&mut if_stmt.condition)?; + self.resolve_stmt(&mut if_stmt.then_branch)?; + + if let Some(branch) = if_stmt.else_branch.as_mut() { + self.resolve_stmt(branch)?; + } + + Ok(()) + } + + Statement::While(while_stmt) => { + self.resolve_expr(&mut while_stmt.condition)?; + self.resolve_stmt(&mut while_stmt.body) + } + + Statement::Function(func) => match Rc::get_mut(func) { + Some(func) => self.resolve_function(func), + // The resolver does not clone references, so unless + // the interpreter is called before the resolver this + // case should never happen. + None => return Err(Error { + line: 0, + kind: ErrorKind::InternalError( + "multiple function references before interpretation" + .into(), + ), + }), + }, + } + } + + fn resolve_var(&mut self, var: &'a mut parser::Var) -> Result<(), Error> { + self.declare(&var.name.lexeme); + + if let Some(init) = &mut var.initialiser { + self.resolve_expr(init)?; + } + + self.define(&var.name.lexeme); + + Ok(()) + } + + fn resolve_function( + &mut self, + func: &'a mut parser::Function, + ) -> Result<(), Error> { + self.declare(&func.name.lexeme); + self.define(&func.name.lexeme); + + self.begin_scope(); + + for param in &func.params { + self.declare(¶m.lexeme); + self.define(¶m.lexeme); + } + + for stmt in &mut func.body { + self.resolve_stmt(stmt)?; + } + + self.end_scope(); + + Ok(()) + } + + fn resolve_expr(&mut self, expr: &'a mut Expr) -> Result<(), Error> { + match expr { + Expr::Variable(var) => self.resolve_variable(var), + Expr::Assign(assign) => self.resolve_assign(assign), + Expr::Grouping(grouping) => self.resolve_expr(&mut grouping.0), + Expr::Call(call) => self.resolve_call(call), + Expr::Literal(_) => Ok(()), + Expr::Unary(unary) => self.resolve_expr(&mut unary.right), + + Expr::Logical(log) => { + self.resolve_expr(&mut log.left)?; + self.resolve_expr(&mut log.right) + } + + Expr::Binary(binary) => { + self.resolve_expr(&mut binary.left)?; + self.resolve_expr(&mut binary.right) + } + } + } + + fn resolve_variable( + &mut self, + var: &'a mut parser::Variable, + ) -> Result<(), Error> { + if let Some(scope) = self.scopes.last_mut() { + if let Some(false) = scope.get(var.name.lexeme.as_str()) { + return Err(Error { + line: var.name.line, + kind: ErrorKind::StaticError( + "can't read local variable in its own initialiser" + .into(), + ), + }); + } + } + + var.depth = self.resolve_local(&var.name); + Ok(()) + } + + fn resolve_assign( + &mut self, + assign: &'a mut parser::Assign, + ) -> Result<(), Error> { + self.resolve_expr(&mut assign.value)?; + assign.depth = self.resolve_local(&assign.name); + Ok(()) + } + + fn resolve_local(&mut self, name: &'a Token) -> Option<usize> { + for (c, scope) in self.scopes.iter().rev().enumerate() { + if scope.contains_key(name.lexeme.as_str()) { + return Some(c); + } + } + + None + } + + fn resolve_call( + &mut self, + call: &'a mut parser::Call, + ) -> Result<(), Error> { + self.resolve_expr(&mut call.callee)?; + + for arg in call.args.iter_mut() { + self.resolve_expr(arg)?; + } + + Ok(()) + } + + // Internal helpers + + fn declare(&mut self, name: &'a str) { + if let Some(scope) = self.scopes.last_mut() { + scope.insert(&name, false); + } + } + + fn define(&mut self, name: &'a str) { + if let Some(scope) = self.scopes.last_mut() { + scope.insert(&name, true); + } + } + + fn begin_scope(&mut self) { + self.scopes.push(Default::default()); + } + + fn end_scope(&mut self) { + self.scopes.pop(); + } +} + +pub fn resolve( + globals: &[String], + block: &mut parser::Block, +) -> Result<(), Error> { + let mut resolver: Resolver = Default::default(); + + // Scope for static globals only starts, never ends. + resolver.begin_scope(); + for global in globals { + resolver.define(global); + } + + resolver.resolve(block) +} diff --git a/users/tazjin/russian/helpers.el b/users/tazjin/russian/helpers.el new file mode 100644 index 000000000000..41d4aa34f47e --- /dev/null +++ b/users/tazjin/russian/helpers.el @@ -0,0 +1,7 @@ +;; Helper functions for creating the other files. + +(defun wiktionary-lookup-at-point (ask-lang) + (interactive "P") + (let ((language (if ask-lang (read-string "Language code? ") "ru"))) + (eww (concat "https://ru.wiktionary.org/wiki/" + (thing-at-point 'word))))) diff --git a/users/tazjin/russian/roots.el b/users/tazjin/russian/roots.el new file mode 100644 index 000000000000..77d09b47261c --- /dev/null +++ b/users/tazjin/russian/roots.el @@ -0,0 +1,28 @@ +;; '(root explanation) +;; +;; All roots without explanations are TODOs. +;; +;; In some cases, roots are not direct morphological roots of their +;; descendent words (e.g. -голов- => главный) + +'(("-весь-" "everything, all, every, etc.") + ("-вид-" "seeing, viewing etc.") + ("-врем-" "time") + ("-говор-" "related to talking") + ("-голов-" "head, main, etc.") + ("-друг-" nil) + ("-дум-" "thinking, thoughts") + ("-жи-" "life") + ("-зна-" "knowing, knowledge") + ("-имя-" "name") + ("-й-" "walking, moving to") + ("-мочь-" "ability, permission") + ("-нов-" "new") + ("-общ-" "common?") + ("-правд-" "truth") + ("-прос-" "question") + ("-сказ-" nil) + ("-смотр-" "watching, viewing") + ("-стран-" "country?") + ("-ход-" "movement") + ("-хорош-" "goodness, niceness")) diff --git a/users/tazjin/russian/russian.el b/users/tazjin/russian/russian.el new file mode 100644 index 000000000000..95af5b0e10ba --- /dev/null +++ b/users/tazjin/russian/russian.el @@ -0,0 +1,97 @@ +(require 'cl-macs) +(require 'ht) +(require 'seq) +(require 's) + +;; Type definitions for Russian structures + +(cl-defstruct russian-word + "Definition and metadata of a single Russian word." + (word nil :type string) + (translations :type list + :documentation "List of lists of strings, each a set of translations.") + + (notes nil :type list ;; of string + :documentation "free-form notes about this word") + + (roots nil :type list ;; of string + :documentation "list of strings that correspond with roots (exact string match)")) + +(defun russian--merge-words (previous new) + "Merge two Russian word definitions together. If no previous + definition exists, only the new one will be returned." + (if (not previous) new + (assert (equal (russian-word-word previous) + (russian-word-word new)) + "different words passed into merge function") + (make-russian-word :word (russian-word-word previous) + :translations (-concat (russian-word-translations previous) + (russian-word-translations new)) + :notes (-concat (russian-word-notes previous) + (russian-word-notes new)) + :roots (-concat (russian-word-roots previous) + (russian-word-roots new))))) + +;; Definitions for creating a data structure of all Russian words. + +(defvar russian-words (make-hash-table) + "Table of all Russian words in the corpus.") + +(defun russian--define-word (word) + "Define a single word in the corpus, optionally merging it with + another entry." + (let ((key (russian-word-word word))) + (ht-set russian-words key (russian--merge-words + (ht-get russian-words key) + word)))) + +(defmacro define-russian-words (&rest words) + "Define the list of all available words. There may be more than + one entry for a word in some cases." + (declare (indent defun)) + + ;; Clear the table before proceeding with insertion + (setq russian-words (make-hash-table)) + + (seq-map + (lambda (word) + (russian--define-word (make-russian-word :word (car word) + :translations (cadr word) + :notes (caddr word) + :roots (cadddr word)))) + words) + + '(message "Defined %s unique words." (ht-size russian-words))) + +;; Helpers to train Russian words through passively. + +(defun russian--format-word (word) + "Format a Russian word suitable for echo display." + (apply #'s-concat + (-flatten + (list (russian-word-word word) + " - " + (s-join ", " (russian-word-translations word)) + (when-let ((roots (russian-word-roots word))) + (list " [" (s-join ", " roots) "]")) + (when-let ((notes (russian-word-notes word))) + (list " (" (s-join "; " notes) ")")))))) + +(defun display-russian-words () + "Convert Russian words to passively terms and start passively." + (interactive) + (setq passively-learn-terms (make-hash-table)) + (ht-map + (lambda (k v) + (ht-set passively-learn-terms k (russian--format-word v))) + russian-words) + (passively-enable)) + +(defun lookup-last-russian-word (in-eww) + "Look up the last Russian word in Wiktionary" + (interactive "P") + (let ((url (concat "https://ru.wiktionary.org/wiki/" passively-last-displayed))) + (if in-eww (eww url) + (browse-url url)))) + +(provide 'russian) diff --git a/users/tazjin/russian/words.el b/users/tazjin/russian/words.el new file mode 100644 index 000000000000..3014aa355335 --- /dev/null +++ b/users/tazjin/russian/words.el @@ -0,0 +1,622 @@ +;; entries :: '(entry ...)' +;; entry :: '(word translations note roots) +;; note :: (or nil string) +;; translations :: '(translation ...) +;; roots :: '(root ...) + +(require 'russian-defs) + +(define-russian-words + ;; 1-50 + ("и" ("and" "though")) + ("в" ("in" "at")) + ("не" ("not")) + ("он" ("he")) + ("на" ("on" "it" "at" "to")) + ("я" ("I")) + ("что" ("what" "that" "why")) + ("тот" ("that")) + ("быть" ("to be")) + ("с" ("with" "and" "from" "of")) + ("а" ("while" "and" "but")) + ("весь" ("all" "everything") nil ("-весь-")) + ("это" ("that" "this" "it")) + ("как" ("how" "what" "as" "like")) + ("она" ("she")) + ("по" ("on" "along" "by")) + ("но" ("but")) + ("они" ("they")) + ("к" ("to" "for" "by")) + ("у" ("by" "with" "of")) + ("ты" ("you")) + ("из" ("from" "of" "in")) + ("мы" ("we")) + ("за" ("behind" "over" "at" "after")) + ("вы" ("you")) + ("так" ("so" "thus" "then")) + ("же" ("and" "as for" "but" "same")) + ("от" ("from" "of" "for")) + ("сказать" ("to say" "to speak") nil ("-сказ-")) + ("этот" ("this")) + ("который" ("which" "who" "that")) + ("мочь" ("be able" "can") nil ("-мочь-")) + ("человек" ("man" "person")) + ("о" ("of" "about" "against")) + ("один" ("one" "some" "alone")) + ("ещё" ("still" "yet")) + ("бы" ("would")) + ("такой" ("such" "so" "some")) + ("только" ("only" "merely" "but")) + ("себя" ("myself" "himself" "herself")) + ("своё" ("one's own" "my" "our")) + ("какой" ("what" "which" "how")) + ("когда" ("when" "while" "as")) + ("уже" ("already" "by now")) + ("для" ("for" "to")) + ("вот" ("here" "there" "this is" "that's") + ("calling attention to something")) + ("кто" ("who" "that" "some")) + ("да" ("yes" "but") ("affirmation (..., right?)")) + ("говорить" ("to say" "to tell" "to speak") nil ("-говор-")) + ("год" ("year")) + + ;; 51 - 100 + ("знать" ("to know" "be aware") nil ("-зна-")) + ("мой" ("my" "mine")) + ("до" ("to" "up to" "about" "before")) + ("или" ("or")) + ("если" ("if")) + ("время" ("time" "season") nil ("-врем-")) + ("рука" ("hand" "arm")) + ("нет" ("no" "not" "but")) + ("самый" ("most" "the very" "the same")) + ("ни" ("not a" "not" "neither ... nor")) + ("стать" ("to become" "begin" "come")) + ("большой" ("big" "large" "important")) + ("даже" ("even")) + ("другой" ("other" "another" "different") nil ("-друг-")) + ("наш" ("our" "ours")) + ("свой" ("one's own")) + ("ну" ("now" "right" "well" "come on")) + ("под" ("under" "for" "towards" "to")) + ("где" ("where")) + ("дело" ("business" "affair" "matter")) + ("есть" ("to eat" "to be")) + ("сам" ("oneself")) + ("раз" ("time" "once" "since")) + ("чтобы" ("that" "in order that")) + ("два" ("two")) + ("там" ("there" "then")) + ("чем" ("than" "instead of") + ("чем ..., тем ...")) + ("глаз" ("eye" "sight")) + ("жизнь" ("life") nil ("-жи-")) + ("первый" ("first" "front" "former")) + ("день" ("day")) + ("тут" ("here" "now" "then")) + ("во" ("in" "at") + ("as particle also: wow, exactly, ...")) + ("ничто" ("nothing")) + ("потом" ("afterwards" "then")) + ("очень" ("very")) + ("со" ("with")) + ("хотеть" ("to want")) + ("ли" ("whether" "if")) + ("при" ("attached to" "in the presence of" "by" "about")) + ("голова" ("head" "mind" "brains") nil ("-голов-")) + ("надо" ("over" "above" "ought to")) + ("без" ("without")) + ("видеть" ("to see") nil ("-вид-")) + ("идти" ("to go" "to come")) + ("теперь" ("now" "nowadays")) + ("тоже" ("also" "as well" "too")) + ("стоять" ("to stand" "be" "stand up")) + ("друг" ("friend")) + ("дом" ("house" "home")) + + ;; 101-150 + ("сейчас" ("now" "presently" "soon")) + ("можно" ("possible" "permitted") nil ("-мочь-")) + ("после" ("after" "afterwards")) + ("слово" ("word")) + ("здесь" ("here")) + ("думать" ("to think" "to believe") nil ("-дум-")) + ("место" ("place" "seat")) + ("спросить" ("to ask") nil ("-прос-")) + ("через" ("through" "across")) + ("лицо" ("face" "person")) + ("что" ("what" "which" "that")) + ("тогда" ("then")) + ("хороший" ("good" "nice") nil ("-хорош-")) + ("каждый" ("every" "each")) + ("новый" ("new" "modern") nil ("-нов-")) + ("жить" ("to live") nil ("-жи-")) + ("должный" ("due" "proper" "should")) + ("смотреть" ("to look" "watch")) + ("почему" ("why")) + ("потому" ("that's why")) + ("сторона" ("side" "party")) + ("просто" ("simply")) + ("нога" ("foot" "leg")) + ("сидеть" ("to sit")) + ("понять" ("to understand" "to realise")) + ("иметь" ("to own" "to have")) + ("конечный" ("final" "last")) + ("делать" ("to do" "make")) + ("вдруг" ("suddenly")) + ("над" ("above" "over")) + ("взять" ("to take")) + ("никто" ("nobody")) + ("понимать" ("to understand")) + ("казаться" ("to seem" "to appear")) + ("работа" ("work" "job")) + ("три" ("three")) + ("ваш" ("yours")) + ("уж" ("really" "already")) + ("земля" ("earth" "land" "soil")) + ("конец" ("end" "distance")) + ("несколько" ("several" "some")) + ("час" ("hour" "time")) + ("голос" ("voice")) + ("город" ("town" "city")) + ("последний" ("last" "the latest" "new")) + + ;; 151-200 + ("пока" ("for the present")) ;; TODO(tazjin): review + ("хорошо" ("well") nil ("-хорош-")) + ("давать" ("to give" "to grant")) + ("вода" ("water")) + ("более" ("more")) + ("хотя" ("although")) + ("всегда" ("always")) + ("второй" ("second")) + ("куда" ("where" "what for" "much")) + ("пойти" ("to go") nil ("-й-")) + ("стол" ("table" "desk" "board")) + ("ребёнок" ("child" "kid" "infant")) + ("увидеть" ("to see")) + ("сила" ("strength" "force")) + ("отец" ("father")) + ("женщина" ("woman")) + ("машина" ("car" "machine" "engine")) + ("случай" ("case" "occasion" "incident")) + ("ночь" ("night")) + ("сразу" ("at once" "right away" "just")) + ("мир" ("world" "peace")) + ("совсем" ("quite" "entirely" "totally")) + ("остаться" ("to remain" "to stay")) + ("об" ("about" "of")) + ("вид" ("appearance" "look" "view")) + ("выйти" ("to go out" "to exit" "to come out" "to appear") nil ("-й-")) + ("дать" ("to give")) + ("работать" ("to work")) + ("любить" ("to work")) + ("старый" ("old")) + ("почти" ("almost")) + ("ряд" ("row" "line")) + ("оказаться" ("find oneself" "turn out")) + ("начало" ("beginning" "origin" "source")) + ("твой" ("your" "yours")) + ("вопрос" ("question" "matter" "problem") nil ("-прос-")) + ("много" ("many" "much")) + ("война" ("war")) + ("снова" ("again")) + ("ответить" ("to answer" "to reply")) + ("между" ("between" "among")) + ("подумать" ("to think")) + ("опять" ("again")) + ("белый" ("white")) + ("деньги" ("money")) + ("значить" ("to mean" "to signify") nil ("-зна-")) + ("про" ("about" "for")) + ("лишь" ("only" "as soon as")) + ("минута" ("minute" "moment")) + ("жена" ("wife")) + + ;; 201-300 + ("посмотреть" ("to watch" "to look" "to inspect") nil ("-смотр-")) + ("правда" ("truth") nil ("-правд-")) + ("главный" ("main" "chief") nil ("-голов-")) + ("страна" ("country") nil ("-стран-")) + ("свет" ("light" "world")) + ("ждать" ("to wait")) + ("мать" ("mother")) + ("будто" ("as if" "as though")) + ("никогда" ("never")) + ("товариш" ("comrade" "friend")) + ("дорога" ("road" "way" "journey")) + ("однако" ("however" "although")) + ("лежать" ("to lie" "to be situated")) + ("именно" ("namely" "just" "exactly") nil ("-имя-")) + ("окно" ("window")) + ("никакой" ("no" "none")) + ("найти" ("to find" "to discover") nil ("-й-")) + ("писать" ("to write")) + ("комната" ("room")) + ("Москва" ("Moscow")) + ("часть" ("part" "share" "department")) + ("вообще" ("in general" "altogether" "on the whole") nil ("-общ-")) + ("книга" ("book")) + ("маленький" ("small" "little")) + ("улица" ("street")) + ("режить" ("to decide" "to solve")) + ("далекий" ("distant" "remote")) + ("душа" ("soul" "spirit")) + ("чуть" ("hardly" "slightly")) + ("вернуться" ("to return")) + ("утро" ("morning")) + ("некоторый" ("some")) + ("считать" ("to count" "to consider")) + ("сколько" ("how much" "how many")) + ("помнить" ("to remember")) + ("вечер" ("evening")) + ("пол" ("floor" "gender")) + ("таки" ("after all")) + ("получить" ("to receive" "to get" "to obtain")) + ("народ" ("people" "nation")) + ("плечо" ("shoulder" "upper arm")) + ("хоть" ("even" "if you want" "though")) + ("сегодня" ("today")) + ("бог" ("god")) + ("вместе" ("together")) + ("взгляд" ("look" "glance" "view")) + ("ходить" ("to go" "to walk") nil ("-ход-")) + ("зачем" ("what for" "why")) + ("советский" ("Soviet")) + ("русский" ("Russian")) + ("бывать" ("to be" "to visit" "to happen")) + ("полный" ("full" "complete" "whole")) + ("прийти" ("to arrive" "to come") nil ("-й-")) + ("палец" ("finger" "toe")) + ("Россия" ("Russia")) + ("любой" ("any" "every")) + ("история" ("history" "story" "event")) + ("наконец" ("finally" "at least")) + ("мысль" ("thought" "idea")) + ("узнать" ("to know" "to learn" "to recognise") nil ("-зна-")) + ("назад" ("back" "backwards" "ago")) + ("общий" ("general" "common") nil ("-общ-")) + ("заметить" ("to notice" "to observe")) + ("словно" ("as if" "like")) + ("прошлый" ("past" "vergangen") nil ("-й-")) + ("уйти" ("to leave" "to go away") nil ("-й-")) + ("известный" ("well-known" "famous")) + ("давно" ("long ago")) + ("слышать" ("to hear")) + ("слушать" ("to listen" "to hear")) + ("бояться" ("to be afraid" "fear")) + ("сын" ("son")) + ("нельзя" ("it is impossible" "can't")) + ("прямо" ("straight" "frankly")) + ("долго" ("for a long time")) + ("быстро" ("fast" "quickly")) + ("лес" ("forest")) + ("похожий" ("similar" "alike") nil ("-ход-")) + ("пора" ("time" "pore")) + ("пять" ("five")) + ("глядеть" ("to look" "to gaze")) + ("оно" ("it")) + ("сесть" ("to sit")) + ("имя" ("name") nil ("-имя-")) + ("ж" ("and" "as for" "but")) + ("разговор" ("conversation" "talk") nil ("-говор-")) + ("тело" ("body")) + ("молодой" ("young")) + ("стена" ("wall")) + ("красный" ("red")) + ("читать" ("to read")) + ("право" ("right")) + ("старик" ("old man")) + ("ранний" ("early")) + ("хотеться" ("to want" "to like")) + ("мама" ("mummy" "mum")) + ("оставаться" ("to remain" "to stay")) + ("высокий" ("tall" "high")) + ("путь" ("way" "track" "path")) + ("поэтому" ("therefore")) + + ;; 301-400 + ("совершенно" ("absolutely" "quite")) + ("кроме" ("except" "besides")) + ("тысяча" ("a thousand")) + ("месяц" ("month")) + ("брать" ("to take" "to hire")) + ("написать" ("to write")) + ("целый" ("intact" "whole" "entire")) + ("огромный" ("huge" "enormous")) + ("начинать" ("to begin")) + ("спина" ("back")) + ("настоящий" ("present" "real" "true")) + ("пусть" ("let's" "though")) + ("язык" ("tongue" "language")) + ("точно" ("exactly")) + ("среди" ("among")) + ("чуствовать" ("to feel")) + ("сердце" ("heart")) + ("вести" ("to lead")) + ("иногда" ("sometimes")) + ("мальчик" ("boy")) + ("успеть" ("to be in time" "to be successful")) + ("небо" ("sky")) + ("живой" ("living" "lively" "alive")) + ("смерть" ("death")) + ("продолжать" ("to continue")) + ("девушка" ("girl")) + ("образ" ("shape" "form" "image")) + ("ко" ("to" "towards" "by")) + ("забыть" ("to forget")) + ("вокруг" ("around")) + ("письмо" ("letter")) + ("власть" ("power")) + ("чёрный" ("black")) + ("пройти" ("to pass" "go by" "be over") nil ("-й-")) + ("появиться" ("to appear" "to show up")) + ("воздух" ("air")) + ("разный" ("different")) + ("выходить" ("to go out" "to exit") ("MR says 'to nurse'??") ("-ход-")) + ("просить" ("to ask")) + ("брат" ("brat")) + ("собственный" ("one's own")) + ("отношение" ("relationship" "attitude")) + ("затем" ("then" "after that")) + ("пытаться" ("to try")) + ("показать" ("to show" "to display")) + ("вспомнить" ("to remember" "to recall")) + ("система" ("system")) + ("четыре" ("four")) + ("квартира" ("flat" "apartment")) + ("держать" ("to hold" "to keep")) + ("также" ("also" "as well" "too")) + ("любовь" ("love")) + ("солдат" ("soldier")) + ("откуда" ("from where")) + ("чтоб" ("that" "in order that")) + ("называть" ("to call" "to name")) + ("третий" ("third")) + ("хозяин" ("master" "boss" "host")) + ("вроде" ("like" "not unlike")) + ("уходить" ("to leave" "to go away") nil ("-ход-")) + ("подойти" ("to approach" "to come up") nil ("-й-")) + ("поднять" ("to lift" "to raise")) + ("спрашивать" ("to ask" "to inquire")) + ("начальник" ("chief" "head" "superior")) + ("оба" ("both")) + ("бросить" ("to throw")) + ("школа" ("school")) + ("парень" ("boy" "fellow" "guy")) + ("кровь" ("blood")) + ("двадцать" ("twenty")) + ("солнце" ("sun")) + ("неделя" ("week")) + ("послать" ("to send" "to dispatch")) + ("находиться" ("to be found" "to turn up") nil ("-ход-")) + ("ребята" ("guys" "children")) + ("поставить" ("to put" "to place" "to set")) + ("встать" ("to get up" "to rise" "to stand up")) + ("например" ("for example" "for instance")) + ("шаг" ("step")) + ("мужчина" ("man" "male")) + ("равно" ("alike" "in like manner")) + ("нос" ("nose")) + ("мало" ("little" "few")) + ("внимание" ("attention")) + ("капитан" ("captain" "master")) + ("ухо" ("ear")) + ("туда" ("to there")) + ("сюда" ("to here")) + ("играть" ("to play")) + ("следовать" ("to follow" "to come next")) + ("рассказать" ("to tell" "to narrate")) + ("великий" ("great")) + ("действительно" ("indeed" "really")) + ("слишком" ("too much")) + ("тяжёлый" ("heavy")) + ("спать" ("to sleep")) + ("оставить" ("to leave" "to abandon")) + ("войти" ("to enter" "to come in") nil ("-й-")) + ("длинный" ("long")) + + ;; 401 - 500 + ("чувство" ("feeling")) + ("иолчать" ("to keep silence" "make no complaint" "say nothing")) + ("рассказывать" ("to tell" "narrate")) + ("отвечать" ("to answer" "to reply")) + ("становиться" ("to stand" "to become")) + ("остановиться" ("to stop")) + ("берег" ("bank" "shore" "coast")) + ("семья" ("family")) + ("искать" ("to search")) + ("генерал" ("general")) + ("момент" ("moment" "instant")) + ("десять" ("ten")) + ("начать" ("to begin")) + ("следуюший" ("next" "following")) + ("личный" ("personal")) + ("труд" ("labour" "work")) + ("верить" ("to believe")) + ("группа" ("group")) + ("немного" ("a little")) + ("впрочем" ("however" "though")) + ("видно" ("evidently" "obviously")) + ("являться" ("to appear")) + ("муж" ("husband")) + ("разве" ("really?" "perhaps") ("when pondering something")) + ("движение" ("movement" "motion")) + ("порядок" ("order")) + ("ответ" ("answer" "reply")) + ("тихо" ("quietly" "silently") ("also as exclamation")) + ("знакомый" ("familiar" "acquainted")) + ("газета" ("newspaper")) + ("помощь" ("help")) + ("сильный" ("strong" "powerful")) + ("скорый" ("quick" "fast")) + ("собака" ("dog")) + ("дерево" ("tree")) + ("снег" ("snow")) + ("сон" ("dream")) + ("смысл" ("sense" "meaning" "purpose") ("making sense" "in the sense")) + ("смочь" ("to be able") ("св")) + ("против" ("against" "opposite" "contrary to")) + ("бежать" ("to run" "to hurry")) + ("двор" ("yard" "court")) + ("форма" ("form" "shape" "uniform")) + ("простой" ("simple" "easy" "plain")) + ("приехать" ("to arrive" "to come")) + ("иной" ("different" "other")) + ("кричать" ("to cry" "to shout")) + ("возможность" ("possibility" "opportunity" "chance")) + ("общество" ("society")) + ("зелёный" ("green")) + ("грудь" ("breast" "chest")) + ("угол" ("corner" "angle")) + ("открыть" ("to open")) + ("происходить" ("to happen" "to occur" "to take place")) + ("ладно" ("well" "all right" "okay")) + ("чёрный" ("black") ("noun (m.): as in 'she wears black'")) + ("век" ("century" "age")) + ("карман" ("pocket")) + ("ехать" ("to go" "ride" "drive" "travel")) + ("немец" ("German")) + ("наверное" ("probably" "most likely")) + ("губа" ("lip")) + ("дядя" ("uncle")) + ("приходить" ("to come" "to arrive")) + ("часто" ("often")) + ("домой" ("home") ("as in direction")) + ("огонь" ("fire")) + ("писатель" ("writer")) + ("армия" ("army")) + ("состояние" ("state" "condition" "fortune")) + ("зуб" ("tooth")) + ("очередь" ("queue" "line" "turn")) + ("кой" ("which") ("old-fashioned, literary (in set expressions)")) + ("подняться" ("to rise" "to climb")) + ("камень" ("stone")) + ("гость" ("guest")) + ("показаться" ("to appear" "to come in sight")) + ("ветер" ("window")) + ("собираться" ("to gather" "to assemble" "to intend") ("TODO: intend??")) + ("попасть" ("to hit" "to find oneself") ("to get (in phrases)")) + ("принять" ("to take" "to admit" "to accept")) + ("сначала" ("at first" "from the beginning")) + ("либо" ("or")) + ("поехать" ("to depart" "to set off")) + ("услышать" ("to hear")) + ("уметь" ("to be able" "know" "can")) + ("случиться" ("to happen")) + ("странный" ("strange")) + ("единственный" ("only" "sole")) + ("рота" ("company") ("(military)")) + ("закон" ("law" "act" "statute")) + ("короткий" ("short")) + ("море" ("sea")) + ("добрый" ("kind")) + ("тёмный" ("dark")) + ("гора" ("mountain" "hill")) + ("врач" ("doctor")) + ("край" ("border, edge" "land, country")) + ("стараться" ("to try" "to endeavour")) + ("лучший" ("better" "best")) + + ;; 501 - 600 + ("река" ("river")) + ("военный" ("military")) + ("мера" ("measure" "step")) + ("страшный" ("terrible" "frightful")) + ("вполне" ("quite" "fully")) + ("звать" ("to call")) + ("произойти" ("to happen" "to occur" "take place")) + ("вперед" ("forward")) + ("медленно" ("slowly")) + ("возле" ("by" "near" "close by")) + ("никак" ("in no way" "by no means")) + ("заниматься" ("to be occupied" "to engage")) + ("действие" ("action" "effort")) + ("довольно" ("enough" "rather")) + ("вещь" ("thing")) + ("необходимый" ("necessary") ("not possible to go around")) + ("ход" ("move")) + ("боль" ("pain")) + ("судьба" ("fate" "fortune" "destiny")) + ("причина" ("cause" "reason" "motive")) + ("положить" ("to lay down" "put down" "place")) + ("едва" ("hardly" "just" "barely")) + ("черта" ("line" "boundary" "trait")) + ("девочка" ("girl" "little girl")) + ("лёгкий" ("light" "easy")) + ("волос" ("hair")) + ("купить" ("to buy" "purchase")) + ("номер" ("number" "size" "room" "issue")) + ("основной" ("main")) + ("широкий" ("wide")) + ("умереть" ("to die")) + ("далеко" ("far" "far off")) + ("плохо" ("badly")) + ("глава" ("head" "chief")) + ("красивый" ("beautiful")) + ("серый" ("grey" "dull")) + ("пить" ("to drink")) + ("командир" ("commander" "officer")) + ("обычно" ("usually")) + ("партия" ("party")) + ("проблема" ("problem" "issue")) + ("страх" ("fear")) + ("проходить" ("to pass" "go" "study")) + ("ясно" ("clear" "clearly")) + ("снять" ("to take away" "take off")) + ("бумага" ("paper")) + ("герой" ("hero")) + ("пара" ("pair" "couple")) + ("государство" ("State")) + ("деревня" ("village")) + ("речь" ("speech")) + ("начаться" ("to begin")) + ("средство" ("means" "remedy")) + ("положение" ("position" "posture" "condition" "state")) + ("связь" ("tie, bond" "connection, relation")) + ("небольшой" ("small" "not great")) + ("представлять" ("to present" "introduce" "imagine")) + ("завтра" ("tomorrow")) + ("объяснить" ("to explain")) + ("пустой" ("empty" "hollow" "idle")) + ("произнести" ("to pronounce" "say" "utter")) + ("человеческий" ("human")) + ("нравиться" ("to please" "be likeable to")) + ("однажды" ("once" "one day")) + ("мимо" ("past" "by")) + ("иначе" ("otherwise" "differently|")) + ("существровать" ("to exist" "to be")) + ("класс" ("class")) + ("удаться" ("turn out well" "succeed" "manage")) + ("толстый" ("thick" "heavy" "fat")) + ("цель" ("goal" "object" "target")) + ("сквозь" ("through")) + ("прийтись" ("to fit" "fall" "have to") ("тебе придётся - you have to")) + ("чистый" ("clean" "pure")) + ("знать" ("to know")) + ("прежний" ("former")) + ("профессор" ("professor")) + ("господин" ("gentleman" "Mr.")) + ("счастье" ("happiness" "luck")) + ("худой" ("thin" "skinny")) + ("дух" ("spirit")) + ("план" ("plan")) + ("чужой" ("somebody else's" "strange" "foreign")) + ("зал" ("hall")) + ("представить" ("to present" "produce" "introduce")) + ("особый" ("special")) + ("директор" ("director" "manager")) + ("бывший" ("former" "ex-")) + ("память" ("memory")) + ("близкий" ("near" "similar" "intimate")) + ("сей" ("this")) + ("результат" ("result" "outcome")) + ("больной" ("sick")) + ("данный" ("given" "present")) + ("кстати" ("to the point" "at the same time")) + ("назвать" ("to call" "name")) + ("след" ("track" "footprint")) + ("улыбаться" ("to smile")) + ("бутылка" ("bottle"))) + +(provide 'russian-words) diff --git a/users/tazjin/wallpapers/bio_thehost_1920.webp b/users/tazjin/wallpapers/bio_thehost_1920.webp new file mode 100644 index 000000000000..1b904c06fadf --- /dev/null +++ b/users/tazjin/wallpapers/bio_thehost_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/busride2_1920.webp b/users/tazjin/wallpapers/busride2_1920.webp new file mode 100644 index 000000000000..ad6ec446f661 --- /dev/null +++ b/users/tazjin/wallpapers/busride2_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/by_belltowers_2880.webp b/users/tazjin/wallpapers/by_belltowers_2880.webp new file mode 100644 index 000000000000..f7477f168914 --- /dev/null +++ b/users/tazjin/wallpapers/by_belltowers_2880.webp Binary files differdiff --git a/users/tazjin/wallpapers/by_crossing_2560.webp b/users/tazjin/wallpapers/by_crossing_2560.webp new file mode 100644 index 000000000000..efa263790b45 --- /dev/null +++ b/users/tazjin/wallpapers/by_crossing_2560.webp Binary files differdiff --git a/users/tazjin/wallpapers/by_gathering3_2880.webp b/users/tazjin/wallpapers/by_gathering3_2880.webp new file mode 100644 index 000000000000..e6b83bdcd430 --- /dev/null +++ b/users/tazjin/wallpapers/by_gathering3_2880.webp Binary files differdiff --git a/users/tazjin/wallpapers/by_mainservers1_1920.webp b/users/tazjin/wallpapers/by_mainservers1_1920.webp new file mode 100644 index 000000000000..f88d237e2b91 --- /dev/null +++ b/users/tazjin/wallpapers/by_mainservers1_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/by_warmachines1_2560.webp b/users/tazjin/wallpapers/by_warmachines1_2560.webp new file mode 100644 index 000000000000..848bf62bd7ed --- /dev/null +++ b/users/tazjin/wallpapers/by_warmachines1_2560.webp Binary files differdiff --git a/users/tazjin/wallpapers/by_warmachines3_1920.webp b/users/tazjin/wallpapers/by_warmachines3_1920.webp new file mode 100644 index 000000000000..6002ad695a1e --- /dev/null +++ b/users/tazjin/wallpapers/by_warmachines3_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/clever-man_2880.webp b/users/tazjin/wallpapers/clever-man_2880.webp new file mode 100644 index 000000000000..eb4d3f1bfa33 --- /dev/null +++ b/users/tazjin/wallpapers/clever-man_2880.webp Binary files differdiff --git a/users/tazjin/wallpapers/december1994_1920.webp b/users/tazjin/wallpapers/december1994_1920.webp new file mode 100644 index 000000000000..d2c4da80187c --- /dev/null +++ b/users/tazjin/wallpapers/december1994_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/flyby_1920.webp b/users/tazjin/wallpapers/flyby_1920.webp new file mode 100644 index 000000000000..8df5b1132e74 --- /dev/null +++ b/users/tazjin/wallpapers/flyby_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/gaussfraktarna_1920_badge.webp b/users/tazjin/wallpapers/gaussfraktarna_1920_badge.webp new file mode 100644 index 000000000000..3274a3a2d21d --- /dev/null +++ b/users/tazjin/wallpapers/gaussfraktarna_1920_badge.webp Binary files differdiff --git a/users/tazjin/wallpapers/kraftahq_1920.webp b/users/tazjin/wallpapers/kraftahq_1920.webp new file mode 100644 index 000000000000..62a6debf476f --- /dev/null +++ b/users/tazjin/wallpapers/kraftahq_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/peripheral2_1920.webp b/users/tazjin/wallpapers/peripheral2_1920.webp new file mode 100644 index 000000000000..e454072ac42a --- /dev/null +++ b/users/tazjin/wallpapers/peripheral2_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/ship14_1920.webp b/users/tazjin/wallpapers/ship14_1920.webp new file mode 100644 index 000000000000..502f5dac903e --- /dev/null +++ b/users/tazjin/wallpapers/ship14_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/shipyard_1920.webp b/users/tazjin/wallpapers/shipyard_1920.webp new file mode 100644 index 000000000000..3d4115305d10 --- /dev/null +++ b/users/tazjin/wallpapers/shipyard_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/specky_1920.webp b/users/tazjin/wallpapers/specky_1920.webp new file mode 100644 index 000000000000..b8246618bebc --- /dev/null +++ b/users/tazjin/wallpapers/specky_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/summerlove2_1920.webp b/users/tazjin/wallpapers/summerlove2_1920.webp new file mode 100644 index 000000000000..d64a1cb867ec --- /dev/null +++ b/users/tazjin/wallpapers/summerlove2_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/t50_1920_badge.webp b/users/tazjin/wallpapers/t50_1920_badge.webp new file mode 100644 index 000000000000..f8cb6107f30c --- /dev/null +++ b/users/tazjin/wallpapers/t50_1920_badge.webp Binary files differdiff --git a/users/tazjin/wallpapers/theflood1_1920.webp b/users/tazjin/wallpapers/theflood1_1920.webp new file mode 100644 index 000000000000..335efb057172 --- /dev/null +++ b/users/tazjin/wallpapers/theflood1_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/thelan_1920.webp b/users/tazjin/wallpapers/thelan_1920.webp new file mode 100644 index 000000000000..55e6c22ad212 --- /dev/null +++ b/users/tazjin/wallpapers/thelan_1920.webp Binary files differdiff --git a/users/tazjin/wallpapers/vadrare_1920_badge.webp b/users/tazjin/wallpapers/vadrare_1920_badge.webp new file mode 100644 index 000000000000..887c891da36c --- /dev/null +++ b/users/tazjin/wallpapers/vadrare_1920_badge.webp Binary files differdiff --git a/users/tvlbot.jpg b/users/tvlbot.jpg new file mode 100644 index 000000000000..f0811418dff5 --- /dev/null +++ b/users/tvlbot.jpg Binary files differdiff --git a/web/atom-feed/default.nix b/web/atom-feed/default.nix new file mode 100644 index 000000000000..1fbcde9bd4ff --- /dev/null +++ b/web/atom-feed/default.nix @@ -0,0 +1,152 @@ +# This file defines functions for generating an Atom feed. + +{ depot, lib, pkgs, ... }: + +with depot.nix.yants; + +let + inherit (builtins) foldl' map readFile replaceStrings sort; + inherit (lib) concatStrings concatStringsSep max removeSuffix; + inherit (pkgs) runCommandNoCC; + + # 'link' describes a related link to a feed, or feed element. + # + # https://validator.w3.org/feed/docs/atom.html#link + link = struct "link" { + rel = string; + href = string; + }; + + # 'entry' describes a feed entry, for example a single post on a + # blog. Some optional fields have been omitted. + # + # https://validator.w3.org/feed/docs/atom.html#requiredEntryElements + entry = struct "entry" { + # Identifies the entry using a universally unique and permanent URI. + id = string; + + # Contains a human readable title for the entry. This value should + # not be blank. + title = string; + + # Content of the entry. + content = option string; + + # Indicates the last time the entry was modified in a significant + # way (in seconds since epoch). + updated = int; + + # Names authors of the entry. Recommended element. + authors = option (list string); + + # Related web pages, such as the web location of a blog post. + links = option (list link); + + # Conveys a short summary, abstract, or excerpt of the entry. + summary = option string; + + # Contains the time of the initial creation or first availability + # of the entry. + published = option int; + + # Conveys information about rights, e.g. copyrights, held in and + # over the entry. + rights = option string; + }; + + # 'feed' describes the metadata of the Atom feed itself. + # + # Some optional fields have been omitted. + # + # https://validator.w3.org/feed/docs/atom.html#requiredFeedElements + feed = struct "feed" { + # Identifies the feed using a universally unique and permanent URI. + id = string; + + # Contains a human readable title for the feed. + title = string; + + # Indicates the last time the feed was modified in a significant + # way (in seconds since epoch). Will be calculated based on most + # recently updated entry if unset. + updated = option int; + + # Entries contained within the feed. + entries = list entry; + + # Names authors of the feed. Recommended element. + authors = option (list string); + + # Related web locations. Recommended element. + links = option (list link); + + # Conveys information about rights, e.g. copyrights, held in and + # over the feed. + rights = option string; + + # Contains a human-readable description or subtitle for the feed. + subtitle = option string; + }; + + # Feed generation functions: + + renderEpoch = epoch: removeSuffix "\n" (readFile (runCommandNoCC "date-${toString epoch}" {} '' + date --date='@${toString epoch}' --utc --iso-8601='seconds' > $out + '')); + + escape = replaceStrings [ "<" ">" "&" "'" ] [ "<" ">" "&" "'" ]; + + elem = name: content: ''<${name}>${escape content}</${name}>''; + + renderLink = defun [ link string ] (l: '' + <link href="${escape l.href}" rel="${escape l.rel}" /> + ''); + + # Technically the author element can also contain 'uri' and 'email' + # fields, but they are not used for the purpose of this feed and are + # omitted. + renderAuthor = author: ''<author><name>${escape author}</name></author>''; + + renderEntry = defun [ entry string ] (e: '' + <entry> + ${elem "title" e.title} + ${elem "id" e.id} + ${elem "updated" (renderEpoch e.updated)} + ${if e ? published + then elem "published" (renderEpoch e.published) + else "" + } + ${if e ? content + then ''<content type="html">${escape e.content}</content>'' + else "" + } + ${if e ? summary then elem "summary" e.summary else ""} + ${concatStrings (map renderAuthor (e.authors or []))} + ${if e ? subtitle then elem "subtitle" e.subtitle else ""} + ${if e ? rights then elem "rights" e.rights else ""} + ${concatStrings (map renderLink (e.links or []))} + </entry> + ''); + + mostRecentlyUpdated = defun [ (list entry) int ] (entries: + foldl' max 0 (map (e: e.updated) entries) + ); + + sortEntries = sort (a: b: a.published > b.published); + + renderFeed = defun [ feed string ] (f: '' + <?xml version="1.0" encoding="utf-8"?> + <feed xmlns="http://www.w3.org/2005/Atom"> + ${elem "id" f.id} + ${elem "title" f.title} + ${elem "updated" (renderEpoch (f.updated or (mostRecentlyUpdated f.entries)))} + ${concatStringsSep "\n" (map renderAuthor (f.authors or []))} + ${if f ? subtitle then elem "subtitle" f.subtitle else ""} + ${if f ? rights then elem "rights" f.rights else ""} + ${concatStrings (map renderLink (f.links or []))} + ${concatStrings (map renderEntry (sortEntries f.entries))} + </feed> + ''); +in { + inherit entry feed renderFeed renderEpoch; +} diff --git a/web/atward/.gitignore b/web/atward/.gitignore new file mode 100644 index 000000000000..29e65519ba35 --- /dev/null +++ b/web/atward/.gitignore @@ -0,0 +1,3 @@ +result +/target +**/*.rs.bk diff --git a/web/atward/Cargo.lock b/web/atward/Cargo.lock new file mode 100644 index 000000000000..74689114d816 --- /dev/null +++ b/web/atward/Cargo.lock @@ -0,0 +1,1001 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "ascii" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109" + +[[package]] +name = "atward" +version = "0.1.0" +dependencies = [ + "regex", + "rouille", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr", + "safemem", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "time", + "winapi", +] + +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +dependencies = [ + "autocfg 1.0.1", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "deflate" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4" +dependencies = [ + "adler32", + "byteorder", + "gzip-header", +] + +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "filetime" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.2.7", + "winapi", +] + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "gzip-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0131feb3d3bb2a5a238d8a4d09f6353b7ebfdc52e77bccbf4ea6eaa751dde639" +dependencies = [ + "crc32fast", +] + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "httparse" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437" + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.14", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "memchr" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" + +[[package]] +name = "mime" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" +dependencies = [ + "log 0.3.9", +] + +[[package]] +name = "mime_guess" +version = "1.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" +dependencies = [ + "mime", + "phf", + "phf_codegen", + "unicase", +] + +[[package]] +name = "multipart" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136eed74cadb9edd2651ffba732b19a450316b680e4f48d6c79e905799e19d01" +dependencies = [ + "buf_redux", + "httparse", + "log 0.4.14", + "mime", + "mime_guess", + "quick-error", + "rand 0.6.5", + "safemem", + "tempfile", + "twoway", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", + "unicase", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc 0.2.0", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85dd92e586f7355c633911e11f77f3d12f04b1b1bd76a198bd34ae3af8341ef2" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5f1ceb7f74abbce32601642fcf8e8508a8a8991e0621c7d750295b9095702b" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rouille" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b0bcae5288ce22b0f2acc33651e915f5f4c79258e41df1e1bfcc7bab1c6bca" +dependencies = [ + "base64 0.10.1", + "brotli2", + "chrono", + "deflate", + "filetime", + "multipart", + "num_cpus", + "percent-encoding", + "rand 0.7.3", + "serde", + "serde_derive", + "serde_json", + "sha1", + "term", + "threadpool", + "time", + "tiny_http", + "url", +] + +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "serde" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" + +[[package]] +name = "serde_derive" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + +[[package]] +name = "syn" +version = "1.0.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad184cc9470f9117b2ac6817bfe297307418819ba40552f9b3846f05c33d5373" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand 0.8.3", + "redox_syscall 0.2.7", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "tiny_http" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c9d60c25d1134335a1f29852c0b0dd93e329054f826c87c69a1cc688edbdff" +dependencies = [ + "ascii", + "chrono", + "chunked_transfer", + "log 0.4.14", + "url", +] + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "twoway" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +dependencies = [ + "memchr", +] + +[[package]] +name = "unicase" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +dependencies = [ + "version_check", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "url" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ccd964113622c8e9322cfac19eb1004a07e636c545f325da085d5cdde6f1f8b" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/web/atward/Cargo.toml b/web/atward/Cargo.toml new file mode 100644 index 000000000000..c9d8d02c0d58 --- /dev/null +++ b/web/atward/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "atward" +version = "0.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +edition = "2018" + +[dependencies] +regex = "1.5" +rouille = "3.0" diff --git a/web/atward/build.rs b/web/atward/build.rs new file mode 100644 index 000000000000..5dadba3bf352 --- /dev/null +++ b/web/atward/build.rs @@ -0,0 +1,50 @@ +//! Build script that can be used outside of Nix builds to inject the +//! ATWARD_INDEX_HTML variable when building in development mode. +//! +//! Note that this script assumes that atward is in a checkout of the +//! TVL depot. + +use std::process::Command; + +static ATWARD_INDEX_HTML: &str = "ATWARD_INDEX_HTML"; +static ERROR_MESSAGE: &str = r#"Failed to build index page. + +When building during development, atward expects to be in a checkout +of the TVL depot. This is required to automatically build the index +page that is needed at compile time. + +As atward can not automatically detect the location of the page, +you must set the `ATWARD_INDEX_HTML` environment variable to the +right path. + +The expected page is build using the files in //web/atward/indexHtml +in the depot."#; + +fn main() { + // Do nothing if the variable is already set (e.g. via Nix) + if let Ok(_) = std::env::var(ATWARD_INDEX_HTML) { + return; + } + + // Otherwise ask Nix to build it and inject the result. + let output = Command::new("nix-build") + .arg("-A").arg("web.atward.indexHtml") + // ... assuming atward is at //web/atward ... + .arg("../..") + .output() + .expect(ERROR_MESSAGE); + + if !output.status.success() { + eprintln!("{}\nNix output: {}", ERROR_MESSAGE, String::from_utf8_lossy(&output.stderr)); + return; + } + + let out_path = String::from_utf8(output.stdout) + .expect("Nix returned invalid output after building index page"); + + // Return an instruction to Cargo that will set the environment + // variable during rustc calls. + // + // https://doc.rust-lang.org/cargo/reference/build-scripts.html#cargorustc-envvarvalue + println!("cargo:rustc-env={}={}", ATWARD_INDEX_HTML, out_path.trim()); +} diff --git a/web/atward/default.nix b/web/atward/default.nix new file mode 100644 index 000000000000..f3ed04345c32 --- /dev/null +++ b/web/atward/default.nix @@ -0,0 +1,8 @@ +{ depot, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; + override = x: { + ATWARD_INDEX_HTML = depot.web.atward.indexHtml; + }; +} diff --git a/web/atward/indexHtml/default.nix b/web/atward/indexHtml/default.nix new file mode 100644 index 000000000000..3af808b89831 --- /dev/null +++ b/web/atward/indexHtml/default.nix @@ -0,0 +1,99 @@ +{ depot, ... }: + +depot.web.tvl.template { + title = "atward"; + content = '' + atward + ====== + + ---------- + + **atward** is [TVL's](https://tvl.fyi/) search + service. It can be configured as a browser search engine for easy + access to TVL bugs, code reviews, code paths and more. + + ### Setting up atward + + To configure atward, add a search engine to your browser with the + following search string: `https://at.tvl.fyi/?q=%s` + Consider setting a shortcut, for example **t** or **tvl**. + You can now quickly access TVL resources by typing something + like <kbd>t b/42</kbd> in your URL bar to get to the bug with ID + 42. + + + ### Supported queries + + The following query types are supported in atward: + + * <kbd>b/42</kbd> - access bugs with ID 42 + * <kbd>cl/3087</kbd> - access changelist with ID 3087 + * <kbd>//web/atward</kbd> - open the **//web/atward** path in TVLs monorepo + * <kbd>r/3002</kbd> - access revision 3002 in cgit + + When given a short host name (e.g. <kbd>todo</kbd> or + <kbd>cl</kbd>), atward will redirect to the appropriate `tvl.fyi` + domain. + + ### Configuration + + Some behaviour of atward can be configured by adding query + parameters to the search string: + + * <kbd>cs=true</kbd> - use Sourcegraph instead of cgit to view code + + + In some browsers (like Firefox) users can not edit query + parameters for search engines. As an alternative configuration can + be supplied via cookies with the same names as the configuration + parameters. + + The form below can set this configuration: + <form class="cheddar-callout cheddar-todo"> + <input type="checkbox" + id="cs-setting" + name="cs-setting" + onchange="saveSetting(this, 'cs');"> + <label for="cs-setting">Use Sourcegraph instead of cgit</label> + </form> + + <noscript> + <p class="cheddar-callout cheddar-warning"> + The form above only works with Javascript enabled. Only a few + lines of Javascript are used, and they are licensed under a + free-software license (MIT). + </p> + </noscript> + + ### Source code + + atward's source code lives at + [//web/atward](https://at.tvl.fyi/?q=%2F%2Fweb%2Fatward). + ''; + extraHead = '' + <script> + /* Initialise the state of all settings. */ + function loadSettings() { + loadSetting(document.getElementById('cs-setting'), 'cs'); + } + + /* Initialise the state of a setting from a cookie. */ + function loadSetting(checkbox, name) { + if (document.cookie.split(';').some(function(cookie) { + return cookie.indexOf(`''${name}=true`) >= 0; + })) { + checkbox.checked = true; + } + } + + /* Persist the state of a checkbox in a cookie */ + function saveSetting(checkbox, name) { + console.log(`setting atward parameter '''''${name}' to ''${checkbox.checked.toString()}`); + document.cookie = `''${name}=''${checkbox.checked.toString()};`; + } + + document.addEventListener('DOMContentLoaded', loadSettings); + </script> + <link rel="search" type="application/opensearchdescription+xml" title="TVL Search" href="https://at.tvl.fyi/opensearch.xml"> + ''; +} diff --git a/web/atward/src/main.rs b/web/atward/src/main.rs new file mode 100644 index 000000000000..26d79cde1a1b --- /dev/null +++ b/web/atward/src/main.rs @@ -0,0 +1,388 @@ +//! Atward implements TVL's redirection service, living at +//! atward.tvl.fyi +//! +//! This service is designed to be added as a search engine to web +//! browsers and attempts to send users to useful locations based on +//! their search query (falling back to another search engine). +use regex::Regex; +use rouille::input::cookies; +use rouille::{Request, Response}; + +/// A query handler supported by atward. It consists of a pattern on +/// which to match and trigger the query, and a function to execute +/// that returns the target URL. +struct Handler { + /// Regular expression on which to match the query string. + pattern: Regex, + + /// Function to construct the target URL. If the pattern matches, + /// this is invoked with the captured matches and the entire URI. + /// + /// Returning `None` causes atward to fall through to the next + /// query (and eventually to the default search engine). + target: for<'s> fn(&Query, regex::Captures<'s>) -> Option<String>, +} + +/// An Atward query supplied by a user. +#[derive(Debug, PartialEq)] +struct Query { + /// Query string itself. + query: String, + + /// Should Sourcegraph be used instead of cgit? + cs: bool, +} + +/// Helper function for setting a parameter based on a query +/// parameter. +fn query_setting(req: &Request, config: &mut bool, param: &str) { + match req.get_param(param) { + Some(s) if s == "true" => *config = true, + Some(s) if s == "false" => *config = false, + _ => {} + } +} + +impl Query { + fn from_request(req: &Request) -> Option<Query> { + // First extract the actual search query ... + let mut query = match req.get_param("q") { + Some(query) => Query { query, cs: false }, + None => return None, + }; + + // ... then apply settings to it. Settings in query parameters + // take precedence over cookies. + for cookie in cookies(req) { + match cookie { + ("cs", "true") => { + query.cs = true; + } + _ => {} + } + } + + query_setting(req, &mut query.cs, "cs"); + + Some(query) + } +} + +#[cfg(test)] +impl From<&str> for Query { + fn from(query: &str) -> Query { + Query { + query: query.to_string(), + cs: false, + } + } +} + +/// Create a URL to a file (and, optionally, specific line) in cgit. +fn cgit_url(path: &str) -> String { + if path.ends_with(".md") { + format!("https://code.tvl.fyi/about/{}", path) + } else { + format!("https://code.tvl.fyi/tree/{}", path) + } +} + +/// Create a URL to a path in Sourcegraph. +fn sourcegraph_path_url(path: &str) -> String { + format!("https://cs.tvl.fyi/depot/-/tree/{}", path) +} +/// Definition of all supported query handlers in atward. +fn handlers() -> Vec<Handler> { + vec![ + // Bug IDs (e.g. b/123) + Handler { + pattern: Regex::new("^b/(?P<bug>\\d+)$").unwrap(), + target: |_, captures| Some(format!("https://b.tvl.fyi/{}", &captures["bug"])), + }, + // Changelists (e.g. cl/42) + Handler { + pattern: Regex::new("^cl/(?P<cl>\\d+)$").unwrap(), + target: |_, captures| Some(format!("https://cl.tvl.fyi/{}", &captures["cl"])), + }, + // Non-parameterised short hostnames should redirect to $host.tvl.fyi + Handler { + pattern: Regex::new("^(?P<host>b|cl|cs|code|at|todo)$").unwrap(), + target: |_, captures| Some(format!("https://{}.tvl.fyi/", &captures["host"])), + }, + // Depot revisions (e.g. r/3002) + Handler { + pattern: Regex::new("^r/(?P<rev>\\d+)$").unwrap(), + target: |_, captures| { + Some(format!( + "https://code.tvl.fyi/commit/?id=refs/r/{}", + &captures["rev"] + )) + }, + }, + // Depot paths (e.g. //web/atward or //ops/nixos/whitby/default.nix) + // TODO(tazjin): Add support for specifying lines in a query parameter + Handler { + pattern: Regex::new("^//(?P<path>[a-zA-Z].*)?$").unwrap(), + target: |query, captures| { + // Pass an empty string if the path is missing, to + // redirect to the depot root. + let path = captures.name("path").map(|m| m.as_str()).unwrap_or(""); + + if query.cs { + Some(sourcegraph_path_url(path)) + } else { + Some(cgit_url(path)) + } + }, + }, + ] +} + +/// Attempt to match against all known query types, and return the +/// destination URL if one is found. +fn dispatch(handlers: &[Handler], query: &Query) -> Option<String> { + for handler in handlers { + if let Some(captures) = handler.pattern.captures(&query.query) { + if let Some(destination) = (handler.target)(query, captures) { + return Some(destination); + } + } + } + + None +} + +/// Return the opensearch.xml file which is required for adding atward +/// as a search engine in Firefox. +fn opensearch() -> Response { + Response::text(include_str!("opensearch.xml")) + .with_unique_header("Content-Type", "application/opensearchdescription+xml") +} + +/// Render the atward index page which gives users some information +/// about how to use the service. +fn index() -> Response { + Response::html(include_str!(env!("ATWARD_INDEX_HTML"))) +} + +/// Render the fallback page which informs users that their query is +/// unsupported. +fn fallback() -> Response { + Response::text("error for emphasis that i am angery and the query whimchst i angery atward") + .with_status_code(404) +} + +fn main() { + let queries = handlers(); + let address = std::env::var("ATWARD_LISTEN_ADDRESS") + .expect("ATWARD_LISTEN_ADDRESS environment variable must be set"); + + rouille::start_server(&address, move |request| { + rouille::log(&request, std::io::stderr(), || { + if request.url() == "/opensearch.xml" { + return opensearch(); + } + + let query = match Query::from_request(&request) { + Some(q) => q, + None => return index(), + }; + + match dispatch(&queries, &query) { + None => fallback(), + Some(destination) => Response::redirect_303(destination), + } + }) + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bug_query() { + assert_eq!( + dispatch(&handlers(), &"b/42".into()), + Some("https://b.tvl.fyi/42".to_string()) + ); + + assert_eq!( + dispatch(&handlers(), &"something only mentioning b/42".into()), + None, + ); + assert_eq!(dispatch(&handlers(), &"b/invalid".into()), None,); + } + + #[test] + fn cl_query() { + assert_eq!( + dispatch(&handlers(), &"cl/42".into()), + Some("https://cl.tvl.fyi/42".to_string()) + ); + + assert_eq!( + dispatch(&handlers(), &"something only mentioning cl/42".into()), + None, + ); + assert_eq!(dispatch(&handlers(), &"cl/invalid".into()), None,); + } + + #[test] + fn depot_path_cgit_query() { + assert_eq!( + dispatch(&handlers(), &"//web/atward/default.nix".into()), + Some("https://code.tvl.fyi/tree/web/atward/default.nix".to_string()), + ); + + assert_eq!( + dispatch(&handlers(), &"//nix/readTree/README.md".into()), + Some("https://code.tvl.fyi/about/nix/readTree/README.md".to_string()), + ); + + assert_eq!(dispatch(&handlers(), &"/not/a/depot/path".into()), None); + } + + #[test] + fn depot_path_sourcegraph_query() { + assert_eq!( + dispatch( + &handlers(), + &Query { + query: "//web/atward/default.nix".to_string(), + cs: true, + } + ), + Some("https://cs.tvl.fyi/depot/-/tree/web/atward/default.nix".to_string()), + ); + + assert_eq!( + dispatch( + &handlers(), + &Query { + query: "/not/a/depot/path".to_string(), + cs: true, + } + ), + None + ); + } + + #[test] + fn depot_root_cgit_query() { + assert_eq!( + dispatch( + &handlers(), + &Query { + query: "//".to_string(), + cs: false, + } + ), + Some("https://code.tvl.fyi/tree/".to_string()), + ); + } + + #[test] + fn plain_host_queries() { + assert_eq!( + dispatch(&handlers(), &"cs".into()), + Some("https://cs.tvl.fyi/".to_string()), + ); + + assert_eq!( + dispatch(&handlers(), &"cl".into()), + Some("https://cl.tvl.fyi/".to_string()), + ); + + assert_eq!( + dispatch(&handlers(), &"b".into()), + Some("https://b.tvl.fyi/".to_string()), + ); + + assert_eq!( + dispatch(&handlers(), &"todo".into()), + Some("https://todo.tvl.fyi/".to_string()), + ); + } + + #[test] + fn request_to_query() { + assert_eq!( + Query::from_request(&Request::fake_http("GET", "/?q=b%2F42", vec![], vec![])) + .expect("request should parse to a query"), + Query { + query: "b/42".to_string(), + cs: false, + }, + ); + + assert_eq!( + Query::from_request(&Request::fake_http("GET", "/", vec![], vec![])), + None + ); + } + + #[test] + fn settings_from_cookie() { + assert_eq!( + Query::from_request(&Request::fake_http( + "GET", + "/?q=b%2F42", + vec![("Cookie".to_string(), "cs=true;".to_string())], + vec![] + )) + .expect("request should parse to a query"), + Query { + query: "b/42".to_string(), + cs: true, + }, + ); + } + + #[test] + fn settings_from_query_parameter() { + assert_eq!( + Query::from_request(&Request::fake_http( + "GET", + "/?q=b%2F42&cs=true", + vec![], + vec![] + )) + .expect("request should parse to a query"), + Query { + query: "b/42".to_string(), + cs: true, + }, + ); + + // Query parameter should override cookie + assert_eq!( + Query::from_request(&Request::fake_http( + "GET", + "/?q=b%2F42&cs=false", + vec![("Cookie".to_string(), "cs=true;".to_string())], + vec![] + )) + .expect("request should parse to a query"), + Query { + query: "b/42".to_string(), + cs: false, + }, + ); + } + + #[test] + fn depot_revision_query() { + assert_eq!( + dispatch(&handlers(), &"r/3002".into()), + Some("https://code.tvl.fyi/commit/?id=refs/r/3002".to_string()) + ); + + assert_eq!( + dispatch(&handlers(), &"something only mentioning r/3002".into()), + None, + ); + + assert_eq!(dispatch(&handlers(), &"r/invalid".into()), None,); + } +} diff --git a/web/atward/src/opensearch.xml b/web/atward/src/opensearch.xml new file mode 100644 index 000000000000..6033987f5b4b --- /dev/null +++ b/web/atward/src/opensearch.xml @@ -0,0 +1,8 @@ +<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/"> + <ShortName>TVL</ShortName> + <Description>The Virus Lounge Search</Description> + <InputEncoding>UTF-8</InputEncoding> + <Url type="text/html" template="https://at.tvl.fyi/"> + <Param name="q" value="{searchTerms}"/> + </Url> +</OpenSearchDescription> diff --git a/web/blog/default.nix b/web/blog/default.nix new file mode 100644 index 000000000000..2cabc09b5524 --- /dev/null +++ b/web/blog/default.nix @@ -0,0 +1,62 @@ +# This creates the static files that make up my blog from the Markdown +# files in this repository. +# +# All blog posts are rendered from Markdown by cheddar. +{ depot, lib, pkgs, ... }@args: + +with depot.nix.yants; + +let + inherit (builtins) readFile; + inherit (depot.nix) renderMarkdown; + inherit (depot.web) atom-feed; + inherit (lib) singleton; + + # Type definition for a single blog post. + post = struct "blog-post" { + key = string; + title = string; + date = int; + + # Optional time at which this post was last updated. + updated = option int; + + # Path to the Markdown file containing the post content. + content = path; + + # Should this post be included in the index? (defaults to true) + listed = option bool; + + # Is this a draft? (adds a banner indicating that the link should + # not be shared) + draft = option bool; + + # Previously each post title had a numeric ID. For these numeric + # IDs, redirects are generated so that old URLs stay compatible. + oldKey = option string; + }; + + # Rendering fragments for the HTML version of the blog. + fragments = import ./fragments.nix args; + + # Functions for generating feeds for these blogs using //web/atom-feed. + toFeedEntry = { baseUrl, ...}: defun [ post atom-feed.entry ] (post: rec { + id = "${baseUrl}/${post.key}"; + title = post.title; + content = readFile (renderMarkdown post.content); + published = post.date; + updated = post.updated or post.date; + + links = singleton { + rel = "alternate"; + href = id; + }; + }); +in { + inherit post toFeedEntry; + inherit (fragments) renderPost; + + # Helper function to determine whether a post should be included in + # listings (on homepages, feeds, ...) + includePost = post: !(fragments.isDraft post) && !(fragments.isUnlisted post); +} diff --git a/web/blog/fragments.nix b/web/blog/fragments.nix new file mode 100644 index 000000000000..63fc1ab4c5cf --- /dev/null +++ b/web/blog/fragments.nix @@ -0,0 +1,95 @@ +# This file defines various fragments of the blog, such as the header +# and footer, as functions that receive arguments to be templated into +# them. +# +# An entire post is rendered by `renderPost`, which assembles the +# fragments together in a runCommand execution. +{ depot, lib, pkgs, ... }: + +let + inherit (builtins) filter map hasAttr replaceStrings; + inherit (pkgs) runCommandNoCC writeText; + inherit (depot.nix) renderMarkdown; + + staticUrl = "https://static.tvl.fyi/${depot.web.static.drvHash}"; + + # Generate a post list for all listed, non-draft posts. + isDraft = post: (hasAttr "draft" post) && post.draft; + isUnlisted = post: (hasAttr "listed" post) && !post.listed; + + escape = replaceStrings [ "<" ">" "&" "'" ] [ "<" ">" "&" "'" ]; + + header = name: title: '' + <!DOCTYPE html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content="${escape name}"> + <link rel="stylesheet" type="text/css" href="${staticUrl}/tvl.css" media="all"> + <link rel="icon" type="image/webp" href="/static/favicon.webp"> + <link rel="alternate" type="application/atom+xml" title="Atom Feed" href="https://tvl.fyi/feed.atom"> + <title>${escape name}: ${escape title}</title> + </head> + <body class="light"> + <header> + <h1><a class="blog-title" href="/">${escape name}</a> </h1> + <hr> + </header> + ''; + + fullFooter = content: '' + <hr> + <footer> + ${content} + </footer> + </body> + ''; + + draftWarning = writeText "draft.html" '' + <p class="cheddar-callout cheddar-warning"> + <b>Note:</b> This post is a <b>draft</b>! Please do not share + the link to it without asking first. + </p> + <hr> + ''; + + unlistedWarning = writeText "unlisted.html" '' + <p class="cheddar-callout cheddar-warning"> + <b>Note:</b> This post is <b>unlisted</b>! Please do not share + the link to it without asking first. + </p> + <hr> + ''; + + renderPost = { name, footer, ... }: post: runCommandNoCC "${post.key}.html" {} '' + cat ${writeText "header.html" (header name post.title)} > $out + + # Write the post title & date + echo '<article><h2 class="inline">${escape post.title}</h2>' >> $out + echo '<aside class="date">' >> $out + date --date="@${toString post.date}" '+%Y-%m-%d' >> $out + ${ + if post ? updated + then ''date --date="@${toString post.updated}" '+ (updated %Y-%m-%d)' >> $out'' + else "" + } + echo '</aside>' >> $out + + ${ + # Add a warning to draft/unlisted posts to make it clear that + # people should not share the post. + + if (isDraft post) then "cat ${draftWarning} >> $out" + else if (isUnlisted post) then "cat ${unlistedWarning} >> $out" + else "# Your ads could be here?" + } + + # Write the actual post through cheddar's about-filter mechanism + cat ${renderMarkdown post.content} >> $out + echo '</article>' >> $out + + cat ${writeText "footer.html" (fullFooter footer)} >> $out + ''; +in { + inherit isDraft isUnlisted renderPost; +} diff --git a/web/bubblegum/OWNERS b/web/bubblegum/OWNERS new file mode 100644 index 000000000000..f16dd105d761 --- /dev/null +++ b/web/bubblegum/OWNERS @@ -0,0 +1,3 @@ +inherited: true +owners: + - sterni diff --git a/web/bubblegum/README.md b/web/bubblegum/README.md new file mode 100644 index 000000000000..48c2ba6e0328 --- /dev/null +++ b/web/bubblegum/README.md @@ -0,0 +1,68 @@ +# //web/bubblegum + +`bubblegum` is a CGI programming library for the Nix expression language. +It provides a few helpers to make writing CGI scripts which are executable +using [//nix/nint](../../nix/nint/README.md) convenient. + +An example nix.cgi script looks like this (don't worry about the shebang +too much, you can use `web.bubblegum.writeCGI` to set this up without +thinking twice): + +```nix +#!/usr/bin/env nint --arg depot '(import /path/to/depot {})' +{ depot, ... }: + +let + inherit (depot.web.bubblegum) + respond + ; +in + +respond "OK" { + "Content-type" = "text/html"; + # further headers… +} '' + <!doctype html> + <html> + <head> + <meta charset="utf-8"> + <title>hello world</title> + </head> + <body> + hello world! + </body> + </html> +'' +``` + +As you can see, the core component of `bubblegum` is the `respond` +function which takes three arguments: + +* The response status as the textual representation which is also + returned to the client in the HTTP protocol, e. g. `"OK"`, + `"Not Found"`, `"Bad Request"`, … + +* An attribute set mapping header names to header values to be sent. + +* The response body as a string. + +Additionally it exposes a few helpers for working with the CGI +environment like `pathInfo` which is a wrapper around +`builtins.getEnv "PATH_INFO"`. The documentation for all exposed +helpers is inlined in [default.nix](./default.nix) (you should be +able to use `nixdoc` to render it). + +For deployment purposes it is recommended to use `writeCGI` which +takes a nix CGI script in the form of a derivation, path or string +and builds an executable nix CGI script which has the correct shebang +set and is automatically passed a version of depot from the nix store, +so the script has access to the `bubblegum` library. + +For example nix CGI scripts and a working deployment using `thttpd` +see the [examples directory](./examples). You can also start a local +server running the examples like this: + +``` +$ nix-build -A web.bubblegum.examples && ./result +# navigate to http://localhost:9000 +``` diff --git a/web/bubblegum/default.nix b/web/bubblegum/default.nix new file mode 100644 index 000000000000..1fbe544bce11 --- /dev/null +++ b/web/bubblegum/default.nix @@ -0,0 +1,251 @@ +{ depot, lib, pkgs, ... }: + +let + + inherit (depot.nix) + runExecline + getBins + utils + sparseTree + nint + ; + + minimalDepot = sparseTree depot.path.origSrc [ + # general depot things + "default.nix" + "nix/readTree" + # nixpkgs for lib and packages + "third_party/nixpkgs" + "third_party/overlays" + # bubblegum and its dependencies + "web/bubblegum" + "nix/runExecline" + "nix/utils" + "nix/sparseTree" + # tvix docs for svg demo + "tvix/docs" + # for blog.nix + "users/sterni/nix" + ]; + + statusCodes = { + # 1xx + "Continue" = 100; + "Switching Protocols" = 101; + "Processing" = 102; + "Early Hints" = 103; + # 2xx + "OK" = 200; + "Created" = 201; + "Accepted" = 202; + "Non-Authoritative Information" = 203; + "No Content" = 204; + "Reset Content" = 205; + "Partial Content" = 206; + "Multi Status" = 207; + "Already Reported" = 208; + "IM Used" = 226; + # 3xx + "Multiple Choices" = 300; + "Moved Permanently" = 301; + "Found" = 302; + "See Other" = 303; + "Not Modified" = 304; + "Use Proxy" = 305; + "Switch Proxy" = 306; + "Temporary Redirect" = 307; + "Permanent Redirect" = 308; + # 4xx + "Bad Request" = 400; + "Unauthorized" = 401; + "Payment Required" = 402; + "Forbidden" = 403; + "Not Found" = 404; + "Method Not Allowed" = 405; + "Not Acceptable" = 406; + "Proxy Authentication Required" = 407; + "Request Timeout" = 408; + "Conflict" = 409; + "Gone" = 410; + "Length Required" = 411; + "Precondition Failed" = 412; + "Payload Too Large" = 413; + "URI Too Long" = 414; + "Unsupported Media Type" = 415; + "Range Not Satisfiable" = 416; + "Expectation Failed" = 417; + "I'm a teapot" = 418; + "Misdirected Request" = 421; + "Unprocessable Entity" = 422; + "Locked" = 423; + "Failed Dependency" = 424; + "Too Early" = 425; + "Upgrade Required" = 426; + "Precondition Required" = 428; + "Too Many Requests" = 429; + "Request Header Fields Too Large" = 431; + "Unavailable For Legal Reasons" = 451; + # 5xx + "Internal Server Error" = 500; + "Not Implemented" = 501; + "Bad Gateway" = 502; + "Service Unavailable" = 503; + "Gateway Timeout" = 504; + "HTTP Version Not Supported" = 505; + "Variant Also Negotiates" = 506; + "Insufficient Storage" = 507; + "Loop Detected" = 508; + "Not Extended" = 510; + "Network Authentication Required" = 511; + }; + + /* Generate a CGI response. Takes three arguments: + + 1. Status of the response as a string which is + the descriptive name in the protocol, e. g. + `"OK"`, `"Not Found"` etc. + 2. Attribute set describing extra headers to + send, keys and values should both be strings. + 3. Response content as a string. + + See the [README](./README.md) for an example. + + Type: either int string -> attrs string -> string -> string + */ + respond = + # response status as an integer (status code) or its + # textual representation in the HTTP protocol. + # See `statusCodes` for a list of valid options. + statusArg: + # headers as an attribute set of strings + headers: + # response body as a string + bodyArg: + let + status = + if builtins.isInt statusArg + then { + code = statusArg; + line = lib.findFirst + (line: statusCodes."${line}" == statusArg) + null + (builtins.attrNames statusCodes); + } else if builtins.isString statusArg then { + code = statusCodes."${statusArg}" or null; + line = statusArg; + } else { + code = null; line = null; + }; + renderedHeaders = lib.concatStrings + (lib.mapAttrsToList (n: v: "${n}: ${toString v}\r\n") headers); + internalError = msg: respond 500 { + Content-type = "text/plain"; + } "bubblegum error: ${msg}"; + body = builtins.tryEval bodyArg; + in + if status.code == null || status.line == null + then internalError "Invalid status ${lib.generators.toPretty {} statusArg}." + else if !body.success + then internalError "Unknown evaluation error in user code" + else lib.concatStrings [ + "Status: ${toString status.code} ${status.line}\r\n" + renderedHeaders + "\r\n" + body.value + ]; + + /* Returns the value of the `SCRIPT_NAME` environment + variable used by CGI. + */ + scriptName = builtins.getEnv "SCRIPT_NAME"; + + /* Returns the value of the `PATH_INFO` environment + variable used by CGI. All cases that could be + considered as the CGI script's root (i. e. + `PATH_INFO` is empty or `/`) is mapped to `"/"` + for convenience. + */ + pathInfo = + let + p = builtins.getEnv "PATH_INFO"; + in + if builtins.stringLength p == 0 + then "/" + else p; + + /* Helper function which converts a path from the + root of the CGI script (i. e. something which + could be the content of `PATH_INFO`) to an + absolute path from the web root by also + utilizing `scriptName`. + + Type: string -> string + */ + absolutePath = path: + if builtins.substring 0 1 path == "/" + then "${scriptName}${path}" + else "${scriptName}/${path}"; + + bins = getBins pkgs.coreutils [ "env" "tee" "cat" "printf" "chmod" ] + // getBins nint [ "nint" ]; + + /* Type: args -> either path derivation string -> derivation + */ + writeCGI = + { # if given sets the `PATH` to search for `nix-instantiate` + # Useful when using for example thttpd which unsets `PATH` + # in the CGI environment. + binPath ? "" + # name of the resulting derivation. Defaults to `baseNameOf` + # the input path or name of the input derivation. + # Must be given if the input is a string. + , name ? null + , ... + }@args: + input: let + drvName = + if builtins.isString input || args ? name + then args.name + else utils.storePathName input; + script = + if builtins.isPath input || lib.isDerivation input + then input + else if builtins.isString input + then pkgs.writeText "${drvName}-source" input + else builtins.throw "Unsupported input: ${lib.generators.toPretty {} input}"; + shebang = lib.concatStringsSep " " ([ + "#!${bins.env}" + # use the slightly cursed /usr/bin/env -S which allows us + # to pass any number of arguments to our interpreter + # instead of maximum one using plain shebang which considers + # everything after the first space as the second argument. + "-S" + ] ++ lib.optionals (builtins.stringLength binPath > 0) [ + "PATH=${binPath}" + ] ++ [ + "${bins.nint}" + # always pass depot so scripts can use this library + "--arg depot '(import ${minimalDepot} {})'" + ]); + in runExecline.local drvName {} [ + "importas" "out" "out" + "pipeline" [ + "foreground" [ + "if" [ bins.printf "%s\n" shebang ] + ] + "if" [ bins.cat script ] + ] + "if" [ bins.tee "$out" ] + "if" [ bins.chmod "+x" "$out" ] + "exit" "0" + ]; + +in { + inherit + respond + pathInfo + scriptName + absolutePath + writeCGI + ; +} diff --git a/web/bubblegum/examples/blog.nix b/web/bubblegum/examples/blog.nix new file mode 100644 index 000000000000..9359d38fa28e --- /dev/null +++ b/web/bubblegum/examples/blog.nix @@ -0,0 +1,134 @@ +{ depot, ... }: + +let + inherit (depot.third_party.nixpkgs) + lib + ; + + inherit (depot.users.sterni.nix) + url + fun + string + ; + + inherit (depot.web.bubblegum) + pathInfo + scriptName + respond + absolutePath + ; + + # substituted using substituteAll in default.nix + blogdir = "@blogdir@"; + # blogdir = toString ./posts; # for local testing + + parseDate = post: + let + matched = builtins.match "/?([0-9]+)-([0-9]+)-([0-9]+)-.+" post; + in + if matched == null + then [ 0 0 0 ] + else builtins.map builtins.fromJSON matched; + + parseTitle = post: + let + matched = builtins.match "/?[0-9]+-[0-9]+-[0-9]+-(.+).html" post; + in + if matched == null + then "no title" + else builtins.head matched; + + dateAtLeast = a: b: + builtins.all fun.id + (lib.zipListsWith (partA: partB: partA >= partB) a b); + + byPostDate = a: b: + dateAtLeast (parseDate a) (parseDate b); + + posts = builtins.sort byPostDate + (builtins.attrNames + (lib.filterAttrs (_: v: v == "regular") + (builtins.readDir blogdir))); + + generic = { title, inner, ... }: '' + <!doctype html> + <html> + <head> + <meta charset="utf-8"> + <title>${title}</title> + <style>a:link, a:visited { color: blue; }</style> + </head> + <body> + ${inner} + </body> + </html> + ''; + + index = posts: '' + <main> + <h1>blog posts</h1> + <ul> + '' + lib.concatMapStrings (post: '' + <li> + <a href="${absolutePath (url.encode {} post)}">${parseTitle post}</a> + </li> + '') posts + '' + </ul> + </main> + ''; + + formatDate = + let + # Assume we never deal with years < 1000 + formatDigit = d: string.fit { + char = "0"; width = 2; + } (toString d); + in lib.concatMapStringsSep "-" formatDigit; + + post = title: post: '' + <main> + <h1>${title}</h1> + <div id="content"> + ${builtins.readFile (blogdir + "/" + post)} + </div> + </main> + <footer> + <p>Posted on ${formatDate (parseDate post)}</p> + <nav><a href="${scriptName}">index</a></nav> + </footer> + ''; + + validatePathInfo = pathInfo: + let + chars = string.toChars pathInfo; + in builtins.length chars > 1 + && !(builtins.elem "/" (builtins.tail chars)); + + response = + if pathInfo == "/" + then { + title = "blog"; + status = 200; + inner = index posts; + } + else if !(validatePathInfo pathInfo) + then { + title = "Bad Request"; + status = 400; + inner = "No slashes in post names 😡"; + } + # CGI should already url.decode for us + else if builtins.pathExists (blogdir + "/" + pathInfo) + then rec { + title = parseTitle pathInfo; + status = 200; + inner = post title pathInfo; + } else { + title = "Not Found"; + status = 404; + inner = "<h1>404 — not found</h1>"; + }; +in + respond response.status { + "Content-type" = "text/html"; + } (generic response) diff --git a/web/bubblegum/examples/default.nix b/web/bubblegum/examples/default.nix new file mode 100644 index 000000000000..3f0f51db6369 --- /dev/null +++ b/web/bubblegum/examples/default.nix @@ -0,0 +1,61 @@ +{ depot, pkgs, lib, ... }: + +let + + scripts = [ + ./hello.nix + ./derivation-svg.nix + (substituteAll { + src = ./blog.nix; + # by making this a plain string this + # can be something outside the nix store! + blogdir = ./posts; + }) + ]; + + inherit (depot.nix) + writeExecline + runExecline + getBins + ; + + inherit (depot.web.bubblegum) + writeCGI + ; + + inherit (pkgs) + runCommandLocal + substituteAll + ; + + bins = (getBins pkgs.thttpd [ "thttpd" ]) + // (getBins pkgs.coreutils [ "printf" "cp" "mkdir" ]); + + webRoot = + let + copyScripts = lib.concatMap + (path: let + cgi = writeCGI { + # assume we are on NixOS since thttpd doesn't set PATH. + # using third_party.nix is tricky because not everyone + # has a tvix daemon running. + binPath = "/run/current-system/sw/bin"; + } path; + in [ + "if" [ bins.cp cgi "\${out}/${cgi.name}" ] + ]) scripts; + in runExecline.local "webroot" {} ([ + "importas" "out" "out" + "if" [ bins.mkdir "-p" "$out" ] + ] ++ copyScripts); + + port = 9000; + +in + writeExecline "serve-examples" {} [ + "foreground" [ + bins.printf "%s\n" "Running on http://localhost:${toString port}" + ] + "${bins.thttpd}" "-D" "-p" (toString port) "-l" "/dev/stderr" + "-c" "*.nix" "-d" webRoot + ] diff --git a/web/bubblegum/examples/derivation-svg.nix b/web/bubblegum/examples/derivation-svg.nix new file mode 100644 index 000000000000..a5f30a2bd155 --- /dev/null +++ b/web/bubblegum/examples/derivation-svg.nix @@ -0,0 +1,11 @@ +# Warning: this is *very* slow on the first request +{ depot, ... }: + +let + inherit (depot.web.bubblegum) + respond + ; +in + respond "OK" { + Content-type = "image/svg+xml"; + } (builtins.readFile "${depot.tvix.docs.svg}/component-flow.svg") diff --git a/web/bubblegum/examples/hello.nix b/web/bubblegum/examples/hello.nix new file mode 100644 index 000000000000..db04d4082371 --- /dev/null +++ b/web/bubblegum/examples/hello.nix @@ -0,0 +1,92 @@ +{ depot, ... }: + +let + inherit (depot.third_party.nixpkgs) + lib + ; + + inherit (depot.web.bubblegum) + pathInfo + respond + absolutePath + ; + + routes = { + "/" = { + status = "OK"; + title = "index"; + content = '' + Hello World! + ''; + }; + "/clock" = { + status = "OK"; + title = "clock"; + content = '' + It is ${toString builtins.currentTime}s since 1970-01-01 00:00 UTC. + ''; + }; + "/coffee" = { + status = "I'm a teapot"; + title = "coffee"; + content = '' + No coffee, I'm afraid + ''; + }; + "/type-error" = { + status = 666; + title = "bad usage"; + content = '' + Never gonna see this. + ''; + }; + "/eval-error" = { + status = "OK"; + title = "evaluation error"; + content = builtins.throw "lol"; + }; + }; + + notFound = { + status = "Not Found"; + title = "404"; + content = '' + This page doesn't exist. + ''; + }; + + navigation = + lib.concatStrings (lib.mapAttrsToList + (p: v: "<li><a href=\"${absolutePath p}\">${v.title}</a></li>") + routes); + + template = { title, content, ... }: '' + <!doctype html> + <html lang="en"> + <head> + <meta charset="utf-8"> + <title>${title}</title> + <style>a:link, a:visited { color: blue; }</style> + </head> + <body> + <hgroup> + <h1><code>//web/bubblegum</code></h1> + <h2>example app</h2> + </hgroup> + <header> + <nav> + <ul>${navigation}</ul> + </nav> + </header> + <main> + <p>${content}</p> + </main> + </body> + ''; + + response = routes."${pathInfo}" or notFound; + +in + respond response.status { + "Content-type" = "text/html"; + } (template response) diff --git a/web/bubblegum/examples/posts/2021-04-01-hello.html b/web/bubblegum/examples/posts/2021-04-01-hello.html new file mode 100644 index 000000000000..3c58be7953f1 --- /dev/null +++ b/web/bubblegum/examples/posts/2021-04-01-hello.html @@ -0,0 +1,3 @@ +<p> + This is it, the peak of cursed. +</p> diff --git a/web/bubblegum/examples/posts/2021-04-02-a second post.html b/web/bubblegum/examples/posts/2021-04-02-a second post.html new file mode 100644 index 000000000000..050586c3029c --- /dev/null +++ b/web/bubblegum/examples/posts/2021-04-02-a second post.html @@ -0,0 +1,7 @@ +<p> + <ul> + <li>✅ sorting</li> + <li>✅ url encoding (admire the spaces!)</li> + <li>✅ classic Nix regex based parsing</li> + </ul> +</p> diff --git a/web/cgit-taz/default.nix b/web/cgit-taz/default.nix new file mode 100644 index 000000000000..a89b96fd92ba --- /dev/null +++ b/web/cgit-taz/default.nix @@ -0,0 +1,74 @@ +# This derivation configures a 'cgit' instance to serve repositories +# from a different source. +{ depot, pkgs, ... }: + +let + inherit (pkgs) + mime-types + thttpd + writeShellScriptBin + writeText + ; + + sourceFilter = writeShellScriptBin "cheddar-about" '' + exec ${depot.tools.cheddar}/bin/cheddar --about-filter $@ + ''; + cgitConfig = writeText "cgitrc" '' + # Global configuration + virtual-root=/ + enable-http-clone=0 + readme=:README.md + about-filter=${sourceFilter}/bin/cheddar-about + source-filter=${depot.tools.cheddar}/bin/cheddar + enable-log-filecount=1 + enable-log-linecount=1 + enable-follow-links=1 + enable-blame=1 + mimetype-file=${mime-types}/etc/mime.types + logo=https://static.tvl.fyi/${depot.web.static.drvHash}/logo-animated.svg + + # Repository configuration + repo.url=depot + repo.path=/var/lib/gerrit/git/depot.git/ + repo.desc=monorepo for the virus lounge + repo.owner=The Virus Lounge + repo.clone-url=https://code.tvl.fyi/depot.git + ''; + + thttpdConfig = writeText "thttpd.conf" '' + port=2448 + dir=${depot.third_party.cgit}/cgit + nochroot + novhost + cgipat=**.cgi + ''; + + # Patched version of thttpd that serves cgit.cgi as the index and + # sets the environment variable for pointing cgit at the correct + # configuration. + # + # Things are done this way because recompilation of thttpd is much + # faster than cgit and I don't want to wait long when iterating on + # config. + thttpdConfigPatch = writeText "thttpd_cgit_conf.patch" '' + diff --git a/libhttpd.c b/libhttpd.c + index c6b1622..eef4b73 100644 + --- a/libhttpd.c + +++ b/libhttpd.c + @@ -3055,4 +3055,6 @@ make_envp( httpd_conn* hc ) + + envn = 0; + + // force cgit to load the correct configuration + + envp[envn++] = "CGIT_CONFIG=${cgitConfig}"; + envp[envn++] = build_env( "PATH=%s", CGI_PATH ); + #ifdef CGI_LD_LIBRARY_PATH + ''; + thttpdCgit = thttpd.overrideAttrs(old: { + patches = [ + ./thttpd_cgi_idx.patch + thttpdConfigPatch + ]; + }); +in writeShellScriptBin "cgit-launch" '' + exec ${thttpdCgit}/bin/thttpd -D -C ${thttpdConfig} +'' diff --git a/web/cgit-taz/thttpd_cgi_idx.patch b/web/cgit-taz/thttpd_cgi_idx.patch new file mode 100644 index 000000000000..67dbc0c7ab80 --- /dev/null +++ b/web/cgit-taz/thttpd_cgi_idx.patch @@ -0,0 +1,13 @@ +diff --git a/config.h b/config.h +index 65ab1e3..cde470f 100644 +--- a/config.h ++++ b/config.h +@@ -327,7 +327,7 @@ + /* CONFIGURE: A list of index filenames to check. The files are searched + ** for in this order. + */ +-#define INDEX_NAMES "index.html", "index.htm", "index.xhtml", "index.xht", "Default.htm", "index.cgi" ++#define INDEX_NAMES "cgit.cgi" + + /* CONFIGURE: If this is defined then thttpd will automatically generate + ** index pages for directories that don't have an explicit index file. diff --git a/web/converse/.gitignore b/web/converse/.gitignore new file mode 100644 index 000000000000..548206b0b297 --- /dev/null +++ b/web/converse/.gitignore @@ -0,0 +1,3 @@ +.envrc +/target/ +**/*.rs.bk diff --git a/web/converse/Cargo.lock b/web/converse/Cargo.lock new file mode 100644 index 000000000000..b022bdd55e43 --- /dev/null +++ b/web/converse/Cargo.lock @@ -0,0 +1,3113 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "actix" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c616db5fa4b0c40702fb75201c2af7f8aa8f3a2e2c1dda3b0655772aa949666" +dependencies = [ + "actix_derive", + "bitflags 1.2.1", + "bytes", + "crossbeam-channel", + "failure", + "fnv", + "futures", + "libc", + "log 0.4.14", + "parking_lot 0.7.1", + "smallvec 0.6.14", + "tokio", + "tokio-codec", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-signal", + "tokio-tcp", + "tokio-timer", + "trust-dns-proto 0.5.0", + "trust-dns-resolver", + "uuid", +] + +[[package]] +name = "actix-net" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bebfbe6629e0131730746718c9e032b58f02c6ce06ed7c982b9fef6c8545acd" +dependencies = [ + "actix", + "bytes", + "futures", + "log 0.4.14", + "mio", + "net2", + "num_cpus", + "slab 0.4.2", + "tokio", + "tokio-codec", + "tokio-current-thread", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-timer", + "tower-service", + "trust-dns-resolver", +] + +[[package]] +name = "actix-web" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0ac60f86c65a50b140139f499f4f7c6e49e4b5d88fbfba08e4e3975991f7bf4" +dependencies = [ + "actix", + "actix-net", + "base64 0.10.1", + "bitflags 1.2.1", + "brotli2", + "byteorder", + "bytes", + "cookie", + "encoding", + "failure", + "flate2", + "futures", + "futures-cpupool", + "h2", + "http", + "httparse", + "language-tags", + "lazy_static", + "lazycell", + "log 0.4.14", + "mime 0.3.16", + "mime_guess 2.0.3", + "mio", + "net2", + "num_cpus", + "parking_lot 0.7.1", + "percent-encoding 1.0.1", + "rand 0.6.5", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "sha1", + "slab 0.4.2", + "smallvec 0.6.14", + "time", + "tokio", + "tokio-current-thread", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-timer", + "url", + "v_htmlescape", + "version_check 0.1.5", +] + +[[package]] +name = "actix_derive" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e9431455322ae393d43a2ba1ef96b8080573c0fc23b196219efedfb6ba69" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "addr2line" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a55f82cfe485775d02112886f4169bde0c5894d75e79ead7eafe7e40a25e45f7" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + +[[package]] +name = "aead" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fc95d1bdb8e6666b2b217308eeeb09f2d6728d104be3e31916cc74d15420331" +dependencies = [ + "generic-array", +] + +[[package]] +name = "aes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" +dependencies = [ + "aes-soft", + "aesni", + "cipher", +] + +[[package]] +name = "aes-gcm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "aes-soft" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aesni" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" +dependencies = [ + "cipher", + "opaque-debug", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr 2.3.4", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "ascii" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97be891acc47ca214468e09425d02cef3af2c94d0d82081cd02061f996802f14" + +[[package]] +name = "askama" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a57e2df69b9fdc5ec981be522d8aff87bd90947b68864f30bf34570e6c5b227a" +dependencies = [ + "askama_derive", + "askama_shared", +] + +[[package]] +name = "askama_derive" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284fd159afab7796f2d15a442bdba4af7e085f8389196b03115018993202fe4e" +dependencies = [ + "askama_shared", + "nom 3.2.1", + "quote 0.5.2", + "syn 0.13.11", +] + +[[package]] +name = "askama_shared" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72764966f241ed1143792ec29805df7eec322800b2b402781d86339e66f512b6" +dependencies = [ + "error-chain 0.11.0", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "autocfg" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc" +dependencies = [ + "addr2line", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "489d6c0ed21b11d038c31b6ceccca973e65d73ba3bd8ecb9a2babf5546164643" +dependencies = [ + "byteorder", + "safemem", +] + +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake2b_simd" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + +[[package]] +name = "buf_redux" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f" +dependencies = [ + "memchr 2.3.4", + "safemem", +] + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "cc" +version = "1.0.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time", + "winapi 0.3.9", +] + +[[package]] +name = "chunked_transfer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87" + +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.2.1", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags 1.2.1", +] + +[[package]] +name = "comrak" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "378397d3ac20b5b0b8b29fa2e1bf20546a6d6cedd1bc18a422dfb41384129e29" +dependencies = [ + "clap", + "entities", + "lazy_static", + "pest", + "pest_derive", + "regex", + "twoway", + "typed-arena", + "unicode_categories", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "converse" +version = "0.1.0" +dependencies = [ + "actix", + "actix-web", + "askama", + "chrono", + "comrak", + "crimp", + "curl", + "diesel", + "env_logger", + "failure", + "futures", + "hyper", + "log 0.4.14", + "md5", + "mime_guess 2.0.3", + "pq-sys", + "pulldown-cmark", + "r2d2", + "rand 0.4.6", + "rouille", + "serde", + "serde_derive", + "serde_json", + "tokio", + "tokio-timer", + "url", + "url_serde", +] + +[[package]] +name = "cookie" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80f6044740a4a516b8aac14c140cdf35c1a640b1bd6b98b6224e49143b2f1566" +dependencies = [ + "aes-gcm", + "base64 0.13.0", + "hkdf", + "hmac", + "percent-encoding 2.1.0", + "rand 0.8.3", + "sha2", + "time", +] + +[[package]] +name = "cpuid-bool" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" + +[[package]] +name = "cpuid-bool" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crimp" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe8f9a320ad9c1a2e3bacedaa281587bd297fb10a10179fd39f777049d04794" +dependencies = [ + "curl", + "serde", + "serde_json", +] + +[[package]] +name = "crossbeam-channel" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ec7fcd21571dc78f96cc96243cab8d8f035247c3efd16c687be154c3fa9efa" +dependencies = [ + "crossbeam-utils 0.6.6", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard 1.1.0", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils 0.7.2", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" +dependencies = [ + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +dependencies = [ + "autocfg 1.0.1", + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crypto-mac" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "ctr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" +dependencies = [ + "cipher", +] + +[[package]] +name = "curl" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a872858e9cb9e3b96c80dd78774ad9e32e44d3b05dc31e142b858d14aebc82c" +dependencies = [ + "curl-sys", + "libc", + "openssl-probe", + "openssl-sys", + "schannel", + "socket2", + "winapi 0.3.9", +] + +[[package]] +name = "curl-sys" +version = "0.4.41+curl-7.75.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec466abd277c7cab2905948f3e94d10bc4963f1f5d47921c1cc4ffd2028fe65" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi 0.3.9", +] + +[[package]] +name = "deflate" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707b6a7b384888a70c8d2e8650b3e60170dfc6a67bb4aa67b6dfca57af4bedb4" +dependencies = [ + "adler32", + "byteorder", + "gzip-header", +] + +[[package]] +name = "diesel" +version = "1.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "047bfc4d5c3bd2ef6ca6f981941046113524b9a9f9a7cbdfdd7ff40f58e6f542" +dependencies = [ + "bitflags 1.2.1", + "byteorder", + "chrono", + "diesel_derives", + "pq-sys", + "r2d2", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.68", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "dirs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd78930633bd1c6e35c4b42b1df7b0cbc6bc191146e512bb3bedf243fcc3901" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + +[[package]] +name = "dtoa" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" + +[[package]] +name = "encoding" +version = "0.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec" +dependencies = [ + "encoding-index-japanese", + "encoding-index-korean", + "encoding-index-simpchinese", + "encoding-index-singlebyte", + "encoding-index-tradchinese", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18" +dependencies = [ + "encoding_index_tests", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569" + +[[package]] +name = "entities" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" + +[[package]] +name = "env_logger" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b0a4d2e39f8420210be8b27eeda28029729e2fd4291019455016c348240c38" +dependencies = [ + "atty", + "humantime", + "log 0.4.14", + "regex", + "termcolor", +] + +[[package]] +name = "error-chain" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6930e04918388a9a2e41d518c25cf679ccafe26733fb4127dbf21993f2575d46" +dependencies = [ + "backtrace", +] + +[[package]] +name = "error-chain" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" +dependencies = [ + "backtrace", +] + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.68", + "synstructure", +] + +[[package]] +name = "filetime" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d34cfa13a63ae058bfa601fe9e313bbdb3746427c1459185464ce0fcf62e1e8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall 0.2.5", + "winapi 0.3.9", +] + +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz-sys", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags 1.2.1", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures-cpupool" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4" +dependencies = [ + "futures", + "num_cpus", +] + +[[package]] +name = "generic-array" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "501466ecc8a30d1d3b7fc9229b122b2ce8ed6e9d9223f1138d4babb253e51817" +dependencies = [ + "typenum", + "version_check 0.9.3", +] + +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", +] + +[[package]] +name = "ghash" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "gimli" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" + +[[package]] +name = "gzip-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0131feb3d3bb2a5a238d8a4d09f6353b7ebfdc52e77bccbf4ea6eaa751dde639" +dependencies = [ + "crc32fast", +] + +[[package]] +name = "h2" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5b34c246847f938a410a03c5458c7fee2274436675e76d8b903c08efc29c462" +dependencies = [ + "byteorder", + "bytes", + "fnv", + "futures", + "http", + "indexmap", + "log 0.4.14", + "slab 0.4.2", + "string", + "tokio-io", +] + +[[package]] +name = "hashbrown" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" + +[[package]] +name = "hermit-abi" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +dependencies = [ + "libc", +] + +[[package]] +name = "hkdf" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest", +] + +[[package]] +name = "hostname" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" +dependencies = [ + "libc", + "match_cfg", + "winapi 0.3.9", +] + +[[package]] +name = "http" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ccf5ede3a895d8856620237b2f02972c1bbc78d2965ad7fe8838d4a0ed41f0" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "hyper" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34a590ca09d341e94cddf8e5af0bbccde205d5fbc2fa3c09dd67c7f85cea59d7" +dependencies = [ + "base64 0.9.3", + "bytes", + "futures", + "futures-cpupool", + "httparse", + "iovec", + "language-tags", + "log 0.4.14", + "mime 0.3.16", + "net2", + "percent-encoding 1.0.1", + "relay", + "time", + "tokio-core", + "tokio-io", + "tokio-proto", + "tokio-service", + "unicase 2.6.0", + "want", +] + +[[package]] +name = "idna" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "indexmap" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +dependencies = [ + "autocfg 1.0.1", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "ipconfig" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f7eadeaf4b52700de180d147c4805f199854600b36faa963d91114827b2ffc" +dependencies = [ + "error-chain 0.8.1", + "socket2", + "widestring", + "winapi 0.3.9", + "winreg", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91d884b6667cd606bb5a69aa0c99ba811a115fc68915e7056ec08a46e93199a" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714" + +[[package]] +name = "libz-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" + +[[package]] +name = "lock_api" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ebf1391f6acad60e5c8b43706dde4582df75c06698ab44511d15016bc2442c" +dependencies = [ + "owning_ref", + "scopeguard 0.3.3", +] + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard 1.1.0", +] + +[[package]] +name = "lock_api" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3c91c24eae6777794bb1997ad98bbb87daf92890acab859f7eaa4320333176" +dependencies = [ + "scopeguard 1.1.0", +] + +[[package]] +name = "log" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +dependencies = [ + "log 0.4.14", +] + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + +[[package]] +name = "match_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" + +[[package]] +name = "matches" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "md5" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c56d6a0b07f9e19282511c83fc5b086364cbae4ba8c7d5f190c3d9b0425a48" + +[[package]] +name = "memchr" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a" +dependencies = [ + "libc", +] + +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "mime" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0" +dependencies = [ + "log 0.3.9", +] + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "mime_guess" +version = "1.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216929a5ee4dd316b1702eedf5e74548c123d370f47841ceaac38ca154690ca3" +dependencies = [ + "mime 0.2.6", + "phf", + "phf_codegen", + "unicase 1.4.2", +] + +[[package]] +name = "mime_guess" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2684d4c2e97d99848d30b324b00c8fcc7e5c897b7cbb5819b09e7c90e8baf212" +dependencies = [ + "mime 0.3.16", + "unicase 2.6.0", +] + +[[package]] +name = "miniz-sys" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9e3ae51cea1576ceba0dde3d484d30e6e5b86dee0b2d412fe3a16a15c98202" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg 1.0.1", +] + +[[package]] +name = "mio" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log 0.4.14", + "miow", + "net2", + "slab 0.4.2", + "winapi 0.2.8", +] + +[[package]] +name = "mio-uds" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" +dependencies = [ + "iovec", + "libc", + "mio", +] + +[[package]] +name = "miow" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "multipart" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adba94490a79baf2d6a23eac897157047008272fa3eecb3373ae6377b91eca28" +dependencies = [ + "buf_redux", + "httparse", + "log 0.4.14", + "mime 0.2.6", + "mime_guess 1.8.8", + "quick-error", + "rand 0.4.6", + "safemem", + "tempdir", + "twoway", +] + +[[package]] +name = "net2" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "nom" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05aec50c70fd288702bcd93284a8444607f3292dbdf2a30de5ea5dcdbe72287b" +dependencies = [ + "memchr 1.0.2", +] + +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr 2.3.4", + "version_check 0.1.5", +] + +[[package]] +name = "num-integer" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" +dependencies = [ + "autocfg 1.0.1", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" +dependencies = [ + "autocfg 1.0.1", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a7ab5d64814df0fe4a4b5ead45ed6c5f181ee3ff04ba344313a6c80446c5d4" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "owning_ref" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" +dependencies = [ + "stable_deref_trait", +] + +[[package]] +name = "parking_lot" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab41b4aed082705d1056416ae4468b6ea99d52599ecf3169b00088d43113e337" +dependencies = [ + "lock_api 0.1.5", + "parking_lot_core 0.4.0", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api 0.3.4", + "parking_lot_core 0.6.2", + "rustc_version", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api 0.4.3", + "parking_lot_core 0.8.3", +] + +[[package]] +name = "parking_lot_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94c8c7923936b28d546dfd14d4472eaf34c99b14e1c973a32b3e6d4eb04298c9" +dependencies = [ + "libc", + "rand 0.6.5", + "rustc_version", + "smallvec 0.6.14", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall 0.1.57", + "rustc_version", + "smallvec 0.6.14", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.5", + "smallvec 1.6.1", + "winapi 0.3.9", +] + +[[package]] +name = "percent-encoding" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pest" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fce5d8b5cc33983fc74f78ad552b5522ab41442c4ca91606e4236eb4b5ceefc" + +[[package]] +name = "pest_derive" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3294f437119209b084c797604295f40227cffa35c57220b1e99a6ff3bf8ee4" +dependencies = [ + "pest", + "quote 0.3.15", + "syn 0.11.11", +] + +[[package]] +name = "phf" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3da44b85f8e8dfaec21adae67f95d93244b2ecf6ad2a692320598dcc8e6dd18" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03e85129e324ad4166b06b2c7491ae27fe3ec353af72e72cd1654c7225d517e" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09364cc93c159b8b06b1f4dd8a4398984503483891b0c26b867cf431fb132662" +dependencies = [ + "phf_shared", + "rand 0.6.5", +] + +[[package]] +name = "phf_shared" +version = "0.7.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234f71a15de2288bcb7e3b6515828d22af7ec8598ee6d24c3b526fa0a80b67a0" +dependencies = [ + "siphasher", + "unicase 1.4.2", +] + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "polyval" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eebcc4aa140b9abd2bc40d9c3f7ccec842679cd79045ac3a7ac698c1a064b7cd" +dependencies = [ + "cpuid-bool 0.2.0", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "pq-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dfb5e575ef93a1b7b2a381d47ba7c5d4e4f73bff37cee932195de769aad9a54" +dependencies = [ + "vcpkg", +] + +[[package]] +name = "proc-macro2" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b06e2f335f48d24442b35a19df506a835fb3547bc3c06ef27340da9acf5cae7" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +dependencies = [ + "unicode-xid 0.2.1", +] + +[[package]] +name = "pulldown-cmark" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6fdf85cda6cadfae5428a54661d431330b312bc767ddbc57adbedc24da66e32" +dependencies = [ + "bitflags 0.9.1", + "getopts", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" + +[[package]] +name = "quote" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8" +dependencies = [ + "proc-macro2 0.3.8", +] + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2 1.0.26", +] + +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log 0.4.14", + "parking_lot 0.11.1", + "scheduled-thread-pool", +] + +[[package]] +name = "rand" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c" +dependencies = [ + "libc", + "rand 0.4.6", +] + +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" +dependencies = [ + "autocfg 0.1.7", + "libc", + "rand_chacha 0.1.1", + "rand_core 0.4.2", + "rand_hc 0.1.0", + "rand_isaac", + "rand_jitter", + "rand_os", + "rand_pcg", + "rand_xorshift", + "winapi 0.3.9", +] + +[[package]] +name = "rand" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +dependencies = [ + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.2", + "rand_hc 0.3.0", +] + +[[package]] +name = "rand_chacha" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.3.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.2", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rand_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +dependencies = [ + "getrandom 0.2.2", +] + +[[package]] +name = "rand_hc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +dependencies = [ + "rand_core 0.6.2", +] + +[[package]] +name = "rand_isaac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rand_jitter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" +dependencies = [ + "libc", + "rand_core 0.4.2", + "winapi 0.3.9", +] + +[[package]] +name = "rand_os" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.4.2", + "rdrand", + "winapi 0.3.9", +] + +[[package]] +name = "rand_pcg" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" +dependencies = [ + "autocfg 0.1.7", + "rand_core 0.4.2", +] + +[[package]] +name = "rand_xorshift" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags 1.2.1", +] + +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.16", + "redox_syscall 0.1.57", + "rust-argon2", +] + +[[package]] +name = "regex" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" +dependencies = [ + "aho-corasick", + "memchr 2.3.4", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" + +[[package]] +name = "relay" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1576e382688d7e9deecea24417e350d3062d97e32e45d70b1cde65994ff1489a" +dependencies = [ + "futures", +] + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "resolv-conf" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11834e137f3b14e309437a8276714eed3a80d1ef894869e510f2c0c0b98b9f4a" +dependencies = [ + "hostname", + "quick-error", +] + +[[package]] +name = "rouille" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112568052ec17fa26c6c11c40acbb30d3ad244bf3d6da0be181f5e7e42e5004f" +dependencies = [ + "base64 0.9.3", + "brotli2", + "chrono", + "deflate", + "filetime", + "multipart", + "num_cpus", + "rand 0.5.6", + "serde", + "serde_derive", + "serde_json", + "sha1", + "term", + "threadpool", + "time", + "tiny_http", + "url", +] + +[[package]] +name = "rust-argon2" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" +dependencies = [ + "base64 0.13.0", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils 0.8.3", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e3bad0ee36814ca07d7968269dd4b7ec89ec2da10c4bb613928d3077083c232" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "safemem" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot 0.11.1", +] + +[[package]] +name = "scoped-tls" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332ffa32bf586782a3efaeb58f127980944bbc8c4d6913a86107ac2a5ab24b28" + +[[package]] +name = "scopeguard" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "serde" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" + +[[package]] +name = "serde_derive" +version = "1.0.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.68", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" +dependencies = [ + "dtoa", + "itoa", + "serde", + "url", +] + +[[package]] +name = "sha1" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" +dependencies = [ + "block-buffer", + "cfg-if 1.0.0", + "cpuid-bool 0.1.2", + "digest", + "opaque-debug", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" + +[[package]] +name = "slab" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b4fcaed89ab08ef143da37bc52adbcc04d4a69014f4c1208d6b51f0c47bc23" + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8cbcd6df1e117c2210e13ab5109635ad68a929fcbb8964dc965b76cb5ee013" + +[[package]] +name = "smallvec" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "smallvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" + +[[package]] +name = "socket2" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "string" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24114bfcceb867ca7f71a0d3fe45d45619ec47a6fbfa98cb14e14250bfa5d6d" +dependencies = [ + "bytes", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "subtle" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" + +[[package]] +name = "syn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" +dependencies = [ + "quote 0.3.15", + "synom", + "unicode-xid 0.0.4", +] + +[[package]] +name = "syn" +version = "0.13.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14f9bf6292f3a61d2c716723fdb789a41bbe104168e6f496dc6497e531ea1b9b" +dependencies = [ + "proc-macro2 0.3.8", + "quote 0.5.2", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ce15dd3ed8aa2f8eeac4716d6ef5ab58b6b9256db41d7e1a0224c2788e8fd87" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "unicode-xid 0.2.1", +] + +[[package]] +name = "synom" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" +dependencies = [ + "unicode-xid 0.0.4", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.68", + "unicode-xid 0.2.1", +] + +[[package]] +name = "take" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b157868d8ac1f56b64604539990685fa7611d8fa9e5476cf0c02cf34d32917c5" + +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + +[[package]] +name = "term" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd106a334b7657c10b7c540a0106114feadeb4dc314513e97df481d5d966f42" +dependencies = [ + "byteorder", + "dirs", + "winapi 0.3.9", +] + +[[package]] +name = "termcolor" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "time" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +dependencies = [ + "libc", + "wasi 0.10.0+wasi-snapshot-preview1", + "winapi 0.3.9", +] + +[[package]] +name = "tiny_http" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1661fa0a44c95d01604bd05c66732a446c657efb62b5164a7a083a3b552b4951" +dependencies = [ + "ascii", + "chrono", + "chunked_transfer", + "log 0.4.14", + "url", +] + +[[package]] +name = "tinyvec" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6" +dependencies = [ + "bytes", + "futures", + "mio", + "num_cpus", + "tokio-codec", + "tokio-current-thread", + "tokio-executor", + "tokio-fs", + "tokio-io", + "tokio-reactor", + "tokio-sync", + "tokio-tcp", + "tokio-threadpool", + "tokio-timer", + "tokio-udp", + "tokio-uds", +] + +[[package]] +name = "tokio-codec" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b2998660ba0e70d18684de5d06b70b70a3a747469af9dea7618cc59e75976b" +dependencies = [ + "bytes", + "futures", + "tokio-io", +] + +[[package]] +name = "tokio-core" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87b1395334443abca552f63d4f61d0486f12377c2ba8b368e523f89e828cffd4" +dependencies = [ + "bytes", + "futures", + "iovec", + "log 0.4.14", + "mio", + "scoped-tls", + "tokio", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-timer", +] + +[[package]] +name = "tokio-current-thread" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e" +dependencies = [ + "futures", + "tokio-executor", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures", +] + +[[package]] +name = "tokio-fs" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297a1206e0ca6302a0eed35b700d292b275256f596e2f3fea7729d5e629b6ff4" +dependencies = [ + "futures", + "tokio-io", + "tokio-threadpool", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes", + "futures", + "log 0.4.14", +] + +[[package]] +name = "tokio-proto" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fbb47ae81353c63c487030659494b295f6cb6576242f907f203473b191b0389" +dependencies = [ + "futures", + "log 0.3.9", + "net2", + "rand 0.3.23", + "slab 0.3.0", + "smallvec 0.2.1", + "take", + "tokio-core", + "tokio-io", + "tokio-service", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures", + "lazy_static", + "log 0.4.14", + "mio", + "num_cpus", + "parking_lot 0.9.0", + "slab 0.4.2", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24da22d077e0f15f55162bdbdc661228c1581892f52074fb242678d015b45162" +dependencies = [ + "futures", +] + +[[package]] +name = "tokio-signal" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0c34c6e548f101053321cba3da7cbb87a610b85555884c41b07da2eb91aff12" +dependencies = [ + "futures", + "libc", + "mio", + "mio-uds", + "signal-hook-registry", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "winapi 0.3.9", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes", + "futures", + "iovec", + "mio", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils 0.7.2", + "futures", + "lazy_static", + "log 0.4.14", + "num_cpus", + "slab 0.4.2", + "tokio-executor", +] + +[[package]] +name = "tokio-timer" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296" +dependencies = [ + "crossbeam-utils 0.7.2", + "futures", + "slab 0.4.2", + "tokio-executor", +] + +[[package]] +name = "tokio-udp" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a0b10e610b39c38b031a2fcab08e4b82f16ece36504988dcbd81dbba650d82" +dependencies = [ + "bytes", + "futures", + "log 0.4.14", + "mio", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-uds" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" +dependencies = [ + "bytes", + "futures", + "iovec", + "libc", + "log 0.4.14", + "mio", + "mio-uds", + "tokio-codec", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tower-service" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987" +dependencies = [ + "futures", +] + +[[package]] +name = "trust-dns-proto" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74" +dependencies = [ + "byteorder", + "failure", + "futures", + "idna", + "lazy_static", + "log 0.4.14", + "rand 0.5.6", + "smallvec 0.6.14", + "socket2", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-timer", + "tokio-udp", + "url", +] + +[[package]] +name = "trust-dns-proto" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09144f0992b0870fa8d2972cc069cbf1e3c0fda64d1f3d45c4d68d0e0b52ad4e" +dependencies = [ + "byteorder", + "failure", + "futures", + "idna", + "lazy_static", + "log 0.4.14", + "rand 0.5.6", + "smallvec 0.6.14", + "socket2", + "tokio-executor", + "tokio-io", + "tokio-reactor", + "tokio-tcp", + "tokio-timer", + "tokio-udp", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9f877f7a1ad821ab350505e1f1b146a4960402991787191d6d8cab2ce2de2c" +dependencies = [ + "cfg-if 0.1.10", + "failure", + "futures", + "ipconfig", + "lazy_static", + "log 0.4.14", + "lru-cache", + "resolv-conf", + "smallvec 0.6.14", + "tokio", + "trust-dns-proto 0.6.3", +] + +[[package]] +name = "try-lock" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2aa4715743892880f70885373966c83d73ef1b0838a664ef0c76fffd35e7c2" + +[[package]] +name = "twoway" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1" +dependencies = [ + "memchr 2.3.4", +] + +[[package]] +name = "typed-arena" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2228007eba4120145f785df0f6c92ea538f5a3635a612ecf4e334c8c1446d" + +[[package]] +name = "typenum" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" + +[[package]] +name = "unicase" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4765f83163b74f957c797ad9253caf97f103fb064d3999aea9568d09fc8a33" +dependencies = [ + "version_check 0.1.5", +] + +[[package]] +name = "unicase" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +dependencies = [ + "version_check 0.9.3", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +dependencies = [ + "matches", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "universal-hash" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "url" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd4e7c0d531266369519a4aa4f399d748bd37043b00bde1e4ff1f60a120b355a" +dependencies = [ + "encoding", + "idna", + "matches", + "percent-encoding 1.0.1", +] + +[[package]] +name = "url_serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74e7d099f1ee52f823d4bdd60c93c3602043c728f5db3b97bdb548467f7bddea" +dependencies = [ + "serde", + "url", +] + +[[package]] +name = "uuid" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90dbc611eb48397705a6b0f6e917da23ae517e4d127123d2cf7674206627d32a" +dependencies = [ + "rand 0.6.5", +] + +[[package]] +name = "v_escape" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "660b101c07b5d0863deb9e7fb3138777e858d6d2a79f9e6049a27d1cc77c6da6" +dependencies = [ + "v_escape_derive", +] + +[[package]] +name = "v_escape_derive" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2ca2a14bc3fc5b64d188b087a7d3a927df87b152e941ccfbc66672e20c467ae" +dependencies = [ + "nom 4.2.3", + "proc-macro2 1.0.26", + "quote 1.0.9", + "syn 1.0.68", +] + +[[package]] +name = "v_htmlescape" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e33e939c0d8cf047514fb6ba7d5aac78bc56677a6938b2ee67000b91f2e97e41" +dependencies = [ + "cfg-if 0.1.10", + "v_escape", +] + +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "version_check" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" + +[[package]] +name = "want" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05d9d966753fa4b5c8db73fcab5eed4549cfe0e1e4e66911e5564a0085c35d1" +dependencies = [ + "futures", + "log 0.4.14", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" + +[[package]] +name = "widestring" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7157704c2e12e3d2189c507b7482c52820a16dfa4465ba91add92f266667cadb" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "winreg" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a27a759395c1195c4cc5cda607ef6f8f6498f64e78f7900f5de0a127a424704a" +dependencies = [ + "winapi 0.3.9", +] + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] diff --git a/web/converse/Cargo.toml b/web/converse/Cargo.toml new file mode 100644 index 000000000000..c77101144d5b --- /dev/null +++ b/web/converse/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "converse" +version = "0.1.0" +authors = ["Vincent Ambo <mail@tazj.in>"] +license = "GPL-3.0" + +[dependencies] +actix = "0.7" +actix-web = "0.7" +askama = "0.6" +chrono = { version = "0.4", features = ["serde"] } +comrak = "0.2" +crimp = "0.2" +diesel = { version = "1.2", features = ["postgres", "chrono", "r2d2"]} +env_logger = "0.5" +failure = "0.1" +futures = "0.1" +hyper = "0.11" +log = "0.4" +md5 = "0.3.7" +mime_guess = "2.0.0-alpha" +pq-sys = "=0.4.4" +r2d2 = "0.8" +rand = "0.4" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +tokio = "0.1" +tokio-timer = "0.2" +url = "1.7" +url_serde = "0.2" +curl = "*" # bounded by crimp +rouille = "3.0" + +[build-dependencies] +pulldown-cmark = "0.1" +askama = "0.6" diff --git a/web/converse/LICENSE b/web/converse/LICENSE new file mode 100644 index 000000000000..f288702d2fa1 --- /dev/null +++ b/web/converse/LICENSE @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/> + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + <one line to give the program's name and a brief idea of what it does.> + Copyright (C) <year> <name of author> + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <https://www.gnu.org/licenses/>. + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + <program> Copyright (C) <year> <name of author> + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +<https://www.gnu.org/licenses/>. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +<https://www.gnu.org/licenses/why-not-lgpl.html>. diff --git a/web/converse/README.md b/web/converse/README.md new file mode 100644 index 000000000000..e7a3f70aec07 --- /dev/null +++ b/web/converse/README.md @@ -0,0 +1,14 @@ +Converse +======== + +Welcome to Converse, a work-in-progress forum software written in +Rust. The intention behind Converse is to provide a simple forum-like +experience. + +There is not a lot of documentation about Converse yet and it has +several known issues. Also note that Converse is being developed for a +specific use-case and is not going to be a forum feature kitchen-sink +like most classical forum softwares. + +Better documentation is forthcoming once the remaining basics have +been taken care of. diff --git a/web/converse/build.rs b/web/converse/build.rs new file mode 100644 index 000000000000..89e3e6b1aba9 --- /dev/null +++ b/web/converse/build.rs @@ -0,0 +1,5 @@ +extern crate askama; + +fn main() { + askama::rerun_if_templates_changed(); +} diff --git a/web/converse/default.nix b/web/converse/default.nix new file mode 100644 index 000000000000..cc2dab7f88f8 --- /dev/null +++ b/web/converse/default.nix @@ -0,0 +1,7 @@ +{ depot, pkgs, ... }: + +depot.third_party.naersk.buildPackage { + src = ./.; + buildInputs = with pkgs; [ openssl postgresql.lib ]; + nativeBuildInputs = [ pkgs.pkgconfig ]; +} diff --git a/web/converse/envrc.example b/web/converse/envrc.example new file mode 100644 index 000000000000..f255d269fec7 --- /dev/null +++ b/web/converse/envrc.example @@ -0,0 +1,14 @@ +# This is an example configuration for running converse during +# development using direnv: +# https://github.com/direnv/direnv +# +# The OIDC actor is configured with bogus values as disabling logins +# never causes it to run anyways. + +export DATABASE_URL=postgres://converse:converse@localhost/converse +export RUST_LOG=info +export OIDC_DISCOVERY_URL=https://does.not.matter.com/ +export OIDC_CLIENT_ID=some-client-id +export OIDC_CLIENT_SECRET='some-client-secret' +export BASE_URL=http://localhost:4567 +export REQUIRE_LOGIN=false diff --git a/web/converse/migrations/.gitkeep b/web/converse/migrations/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/web/converse/migrations/.gitkeep diff --git a/web/converse/migrations/00000000000000_diesel_initial_setup/down.sql b/web/converse/migrations/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 000000000000..a9f526091194 --- /dev/null +++ b/web/converse/migrations/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/web/converse/migrations/00000000000000_diesel_initial_setup/up.sql b/web/converse/migrations/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 000000000000..d68895b1a7b7 --- /dev/null +++ b/web/converse/migrations/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/web/converse/migrations/2018-04-08-133240_create_posts/down.sql b/web/converse/migrations/2018-04-08-133240_create_posts/down.sql new file mode 100644 index 000000000000..8ebcef8e131a --- /dev/null +++ b/web/converse/migrations/2018-04-08-133240_create_posts/down.sql @@ -0,0 +1,2 @@ +DROP TABLE posts; +DROP TABLE threads; diff --git a/web/converse/migrations/2018-04-08-133240_create_posts/up.sql b/web/converse/migrations/2018-04-08-133240_create_posts/up.sql new file mode 100644 index 000000000000..7cf601fa2c54 --- /dev/null +++ b/web/converse/migrations/2018-04-08-133240_create_posts/up.sql @@ -0,0 +1,13 @@ +CREATE TABLE threads ( + id SERIAL PRIMARY KEY, + title VARCHAR NOT NULL, + body TEXT NOT NULL, + posted TIMESTAMPTZ NOT NULL +); + +CREATE TABLE posts ( + id SERIAL PRIMARY KEY, + thread SERIAL REFERENCES threads (id), + body TEXT NOT NULL, + posted TIMESTAMPTZ NOT NULL +); diff --git a/web/converse/migrations/2018-04-08-161017_joinable_posts/down.sql b/web/converse/migrations/2018-04-08-161017_joinable_posts/down.sql new file mode 100644 index 000000000000..cc7874b410ee --- /dev/null +++ b/web/converse/migrations/2018-04-08-161017_joinable_posts/down.sql @@ -0,0 +1 @@ +ALTER TABLE posts RENAME COLUMN thread_id TO thread; diff --git a/web/converse/migrations/2018-04-08-161017_joinable_posts/up.sql b/web/converse/migrations/2018-04-08-161017_joinable_posts/up.sql new file mode 100644 index 000000000000..9713de6cfc4b --- /dev/null +++ b/web/converse/migrations/2018-04-08-161017_joinable_posts/up.sql @@ -0,0 +1 @@ +ALTER TABLE posts RENAME COLUMN thread TO thread_id; diff --git a/web/converse/migrations/2018-04-08-172739_default_posted/down.sql b/web/converse/migrations/2018-04-08-172739_default_posted/down.sql new file mode 100644 index 000000000000..64dc07e0666a --- /dev/null +++ b/web/converse/migrations/2018-04-08-172739_default_posted/down.sql @@ -0,0 +1,2 @@ +ALTER TABLE threads ALTER COLUMN posted DROP DEFAULT; +ALTER TABLE posts ALTER COLUMN posted DROP DEFAULT; diff --git a/web/converse/migrations/2018-04-08-172739_default_posted/up.sql b/web/converse/migrations/2018-04-08-172739_default_posted/up.sql new file mode 100644 index 000000000000..afca8181cc56 --- /dev/null +++ b/web/converse/migrations/2018-04-08-172739_default_posted/up.sql @@ -0,0 +1,2 @@ +ALTER TABLE threads ALTER COLUMN posted SET DEFAULT (NOW() AT TIME ZONE 'UTC'); +ALTER TABLE posts ALTER COLUMN posted SET DEFAULT (NOW() AT TIME ZONE 'UTC'); diff --git a/web/converse/migrations/2018-04-08-182319_add_authors/down.sql b/web/converse/migrations/2018-04-08-182319_add_authors/down.sql new file mode 100644 index 000000000000..8ad817908065 --- /dev/null +++ b/web/converse/migrations/2018-04-08-182319_add_authors/down.sql @@ -0,0 +1,5 @@ +ALTER TABLE threads DROP COLUMN author_name; +ALTER TABLE threads DROP COLUMN author_email; + +ALTER TABLE posts DROP COLUMN author_name; +ALTER TABLE posts DROP COLUMN author_email; diff --git a/web/converse/migrations/2018-04-08-182319_add_authors/up.sql b/web/converse/migrations/2018-04-08-182319_add_authors/up.sql new file mode 100644 index 000000000000..ad5beb9f6785 --- /dev/null +++ b/web/converse/migrations/2018-04-08-182319_add_authors/up.sql @@ -0,0 +1,10 @@ +-- This migration adds an 'author' column to the thread & post table. +-- Authors don't currently exist as independent objects in the +-- database as most user management is simply delegated to the OIDC +-- provider. + +ALTER TABLE threads ADD COLUMN author_name VARCHAR NOT NULL DEFAULT 'anonymous'; +ALTER TABLE threads ADD COLUMN author_email VARCHAR NOT NULL DEFAULT 'unknown@example.org'; + +ALTER TABLE posts ADD COLUMN author_name VARCHAR NOT NULL DEFAULT 'anonymous'; +ALTER TABLE posts ADD COLUMN author_email VARCHAR NOT NULL DEFAULT 'unknown@example.org'; diff --git a/web/converse/migrations/2018-04-14-140818_posts_only_in_posts/down.sql b/web/converse/migrations/2018-04-14-140818_posts_only_in_posts/down.sql new file mode 100644 index 000000000000..bb6528f33d78 --- /dev/null +++ b/web/converse/migrations/2018-04-14-140818_posts_only_in_posts/down.sql @@ -0,0 +1 @@ +ALTER TABLE threads ADD COLUMN body TEXT NOT NULL DEFAULT ''; diff --git a/web/converse/migrations/2018-04-14-140818_posts_only_in_posts/up.sql b/web/converse/migrations/2018-04-14-140818_posts_only_in_posts/up.sql new file mode 100644 index 000000000000..07ff9a1196f6 --- /dev/null +++ b/web/converse/migrations/2018-04-14-140818_posts_only_in_posts/up.sql @@ -0,0 +1,6 @@ +-- Instead of storing the thread OP in the thread table, this will +-- make it a post as well. +-- At the time at which this migration was created no important data +-- existed in any converse instances, so data is not moved. + +ALTER TABLE threads DROP COLUMN body; diff --git a/web/converse/migrations/2018-04-14-153202_add_stickies_improve_index/down.sql b/web/converse/migrations/2018-04-14-153202_add_stickies_improve_index/down.sql new file mode 100644 index 000000000000..a67ada3d4a57 --- /dev/null +++ b/web/converse/migrations/2018-04-14-153202_add_stickies_improve_index/down.sql @@ -0,0 +1,2 @@ +DROP VIEW thread_index; +ALTER TABLE threads DROP COLUMN sticky; diff --git a/web/converse/migrations/2018-04-14-153202_add_stickies_improve_index/up.sql b/web/converse/migrations/2018-04-14-153202_add_stickies_improve_index/up.sql new file mode 100644 index 000000000000..74a559e35d00 --- /dev/null +++ b/web/converse/migrations/2018-04-14-153202_add_stickies_improve_index/up.sql @@ -0,0 +1,21 @@ +-- Add support for stickies in threads +ALTER TABLE threads ADD COLUMN sticky BOOLEAN NOT NULL DEFAULT FALSE; + +-- CREATE a simple view that returns the list of threads ordered by +-- the last post that occured in the thread. +CREATE VIEW thread_index AS + SELECT t.id AS thread_id, + t.title AS title, + t.author_name AS thread_author, + t.posted AS created, + t.sticky AS sticky, + p.id AS post_id, + p.author_name AS post_author, + p.posted AS posted + FROM threads t + JOIN (SELECT DISTINCT ON (thread_id) + id, thread_id, author_name, posted + FROM posts + ORDER BY thread_id, id DESC) AS p + ON t.id = p.thread_id + ORDER BY t.sticky DESC, p.id DESC; diff --git a/web/converse/migrations/2018-04-14-170750_search-index/down.sql b/web/converse/migrations/2018-04-14-170750_search-index/down.sql new file mode 100644 index 000000000000..c57e66290201 --- /dev/null +++ b/web/converse/migrations/2018-04-14-170750_search-index/down.sql @@ -0,0 +1,2 @@ +DROP INDEX idx_fts_search; +DROP MATERIALIZED VIEW search_index; diff --git a/web/converse/migrations/2018-04-14-170750_search-index/up.sql b/web/converse/migrations/2018-04-14-170750_search-index/up.sql new file mode 100644 index 000000000000..6b7d90eca63c --- /dev/null +++ b/web/converse/migrations/2018-04-14-170750_search-index/up.sql @@ -0,0 +1,21 @@ +-- Prepare a materialised view containing the tsvector data for all +-- threads and posts. This view is indexed using a GIN-index to enable +-- performant full-text searches. +-- +-- For now the query language is hardcoded to be English. + +CREATE MATERIALIZED VIEW search_index AS + SELECT p.id AS post_id, + p.author_name AS author, + t.id AS thread_id, + t.title AS title, + p.body AS body, + setweight(to_tsvector('english', t.title), 'B') || + setweight(to_tsvector('english', p.body), 'A') || + setweight(to_tsvector('simple', t.author_name), 'C') || + setweight(to_tsvector('simple', p.author_name), 'C') AS document + FROM posts p + JOIN threads t + ON t.id = p.thread_id; + +CREATE INDEX idx_fts_search ON search_index USING gin(document); diff --git a/web/converse/migrations/2018-05-01-141548_add-users/down.sql b/web/converse/migrations/2018-05-01-141548_add-users/down.sql new file mode 100644 index 000000000000..61fd222e18cf --- /dev/null +++ b/web/converse/migrations/2018-05-01-141548_add-users/down.sql @@ -0,0 +1,63 @@ +-- First restore the old columns: +ALTER TABLE threads ADD COLUMN author_name VARCHAR; +ALTER TABLE threads ADD COLUMN author_email VARCHAR; +ALTER TABLE posts ADD COLUMN author_name VARCHAR; +ALTER TABLE posts ADD COLUMN author_email VARCHAR; + +-- Then select the data back into them: +UPDATE threads SET author_name = users.name, + author_email = users.email + FROM users + WHERE threads.user_id = users.id; + +UPDATE posts SET author_name = users.name, + author_email = users.email + FROM users + WHERE posts.user_id = users.id; + +-- add the constraints back: +ALTER TABLE threads ALTER COLUMN author_name SET NOT NULL; +ALTER TABLE threads ALTER COLUMN author_email SET NOT NULL; +ALTER TABLE posts ALTER COLUMN author_name SET NOT NULL; +ALTER TABLE posts ALTER COLUMN author_email SET NOT NULL; + +-- reset the index view: +CREATE OR REPLACE VIEW thread_index AS + SELECT t.id AS thread_id, + t.title AS title, + t.author_name AS thread_author, + t.posted AS created, + t.sticky AS sticky, + p.id AS post_id, + p.author_name AS post_author, + p.posted AS posted + FROM threads t + JOIN (SELECT DISTINCT ON (thread_id) + id, thread_id, author_name, posted + FROM posts + ORDER BY thread_id, id DESC) AS p + ON t.id = p.thread_id + ORDER BY t.sticky DESC, p.id DESC; + +-- reset the search view: +DROP MATERIALIZED VIEW search_index; +CREATE MATERIALIZED VIEW search_index AS + SELECT p.id AS post_id, + p.author_name AS author, + t.id AS thread_id, + t.title AS title, + p.body AS body, + setweight(to_tsvector('english', t.title), 'B') || + setweight(to_tsvector('english', p.body), 'A') || + setweight(to_tsvector('simple', t.author_name), 'C') || + setweight(to_tsvector('simple', p.author_name), 'C') AS document + FROM posts p + JOIN threads t + ON t.id = p.thread_id; + +CREATE INDEX idx_fts_search ON search_index USING gin(document); + +-- and drop the users table and columns: +ALTER TABLE posts DROP COLUMN user_id; +ALTER TABLE threads DROP COLUMN user_id; +DROP TABLE users; diff --git a/web/converse/migrations/2018-05-01-141548_add-users/up.sql b/web/converse/migrations/2018-05-01-141548_add-users/up.sql new file mode 100644 index 000000000000..fcb7133e8e41 --- /dev/null +++ b/web/converse/migrations/2018-05-01-141548_add-users/up.sql @@ -0,0 +1,83 @@ +-- This query creates a users table and migrates the existing user +-- information (from the posts table) into it. + +CREATE TABLE users ( + id SERIAL PRIMARY KEY, + email VARCHAR NOT NULL UNIQUE, + name VARCHAR NOT NULL, + admin BOOLEAN NOT NULL DEFAULT false +); + +-- Insert the 'anonymous' user explicitly: +INSERT INTO users (name, email) + VALUES ('Anonymous', 'anonymous@nothing.org'); + +INSERT INTO users (id, email, name) + SELECT nextval('users_id_seq'), + author_email AS email, + author_name AS name + FROM posts + WHERE author_email != 'anonymous@nothing.org' + GROUP BY name, email; + +-- Create the 'user_id' column in the relevant tables (initially +-- without a not-null constraint) and populate it with the data +-- selected above: +ALTER TABLE posts ADD COLUMN user_id INTEGER REFERENCES users (id); +UPDATE posts SET user_id = users.id + FROM users + WHERE users.email = posts.author_email; + +ALTER TABLE threads ADD COLUMN user_id INTEGER REFERENCES users (id); +UPDATE threads SET user_id = users.id + FROM users + WHERE users.email = threads.author_email; + +-- Add the constraints: +ALTER TABLE posts ALTER COLUMN user_id SET NOT NULL; +ALTER TABLE threads ALTER COLUMN user_id SET NOT NULL; + +-- Update the index view: +CREATE OR REPLACE VIEW thread_index AS + SELECT t.id AS thread_id, + t.title AS title, + ta.name AS thread_author, + t.posted AS created, + t.sticky AS sticky, + p.id AS post_id, + pa.name AS post_author, + p.posted AS posted + FROM threads t + JOIN (SELECT DISTINCT ON (thread_id) + id, thread_id, user_id, posted + FROM posts + ORDER BY thread_id, id DESC) AS p + ON t.id = p.thread_id + JOIN users ta ON ta.id = t.user_id + JOIN users pa ON pa.id = p.user_id + ORDER BY t.sticky DESC, p.id DESC; + +-- Update the search view: +DROP MATERIALIZED VIEW search_index; +CREATE MATERIALIZED VIEW search_index AS + SELECT p.id AS post_id, + pa.name AS author, + t.id AS thread_id, + t.title AS title, + p.body AS body, + setweight(to_tsvector('english', t.title), 'B') || + setweight(to_tsvector('english', p.body), 'A') || + setweight(to_tsvector('simple', ta.name), 'C') || + setweight(to_tsvector('simple', pa.name), 'C') AS document + FROM posts p + JOIN threads t ON t.id = p.thread_id + JOIN users ta ON ta.id = t.user_id + JOIN users pa ON pa.id = p.user_id; + +CREATE INDEX idx_fts_search ON search_index USING gin(document); + +-- And drop the old fields: +ALTER TABLE posts DROP COLUMN author_name; +ALTER TABLE posts DROP COLUMN author_email; +ALTER TABLE threads DROP COLUMN author_name; +ALTER TABLE threads DROP COLUMN author_email; diff --git a/web/converse/migrations/2018-05-01-183232_simplified-post-view/down.sql b/web/converse/migrations/2018-05-01-183232_simplified-post-view/down.sql new file mode 100644 index 000000000000..0f14732f3845 --- /dev/null +++ b/web/converse/migrations/2018-05-01-183232_simplified-post-view/down.sql @@ -0,0 +1 @@ +DROP VIEW simple_posts; diff --git a/web/converse/migrations/2018-05-01-183232_simplified-post-view/up.sql b/web/converse/migrations/2018-05-01-183232_simplified-post-view/up.sql new file mode 100644 index 000000000000..280fef87003e --- /dev/null +++ b/web/converse/migrations/2018-05-01-183232_simplified-post-view/up.sql @@ -0,0 +1,11 @@ +-- Creates a view for listing posts akin to the post table before +-- splitting out users. This exists to avoid having to do joining +-- logic and such inside of the application. + +CREATE VIEW simple_posts AS + SELECT p.id AS id, + thread_id, body, posted, user_id, + users.name AS author_name, + users.email AS author_email + FROM posts p + JOIN users ON users.id = p.user_id; diff --git a/web/converse/migrations/2018-05-25-160648_add_closed_column/down.sql b/web/converse/migrations/2018-05-25-160648_add_closed_column/down.sql new file mode 100644 index 000000000000..fb2a98c0af42 --- /dev/null +++ b/web/converse/migrations/2018-05-25-160648_add_closed_column/down.sql @@ -0,0 +1 @@ +ALTER TABLE threads DROP COLUMN closed; diff --git a/web/converse/migrations/2018-05-25-160648_add_closed_column/up.sql b/web/converse/migrations/2018-05-25-160648_add_closed_column/up.sql new file mode 100644 index 000000000000..d7d4c44da201 --- /dev/null +++ b/web/converse/migrations/2018-05-25-160648_add_closed_column/up.sql @@ -0,0 +1 @@ +ALTER TABLE threads ADD COLUMN closed BOOLEAN NOT NULL DEFAULT false; diff --git a/web/converse/migrations/2018-05-25-161939_add_closed_to_index/down.sql b/web/converse/migrations/2018-05-25-161939_add_closed_to_index/down.sql new file mode 100644 index 000000000000..1063fdc88224 --- /dev/null +++ b/web/converse/migrations/2018-05-25-161939_add_closed_to_index/down.sql @@ -0,0 +1,30 @@ +-- Update the index view: +DROP VIEW thread_index; +CREATE VIEW thread_index AS + SELECT t.id AS thread_id, + t.title AS title, + ta.name AS thread_author, + t.posted AS created, + t.sticky AS sticky, + p.id AS post_id, + pa.name AS post_author, + p.posted AS posted + FROM threads t + JOIN (SELECT DISTINCT ON (thread_id) + id, thread_id, user_id, posted + FROM posts + ORDER BY thread_id, id DESC) AS p + ON t.id = p.thread_id + JOIN users ta ON ta.id = t.user_id + JOIN users pa ON pa.id = p.user_id + ORDER BY t.sticky DESC, p.id DESC; + +-- Update the post view: +DROP VIEW simple_posts; +CREATE VIEW simple_posts AS + SELECT p.id AS id, + thread_id, body, posted, user_id, + users.name AS author_name, + users.email AS author_email + FROM posts p + JOIN users ON users.id = p.user_id; diff --git a/web/converse/migrations/2018-05-25-161939_add_closed_to_index/up.sql b/web/converse/migrations/2018-05-25-161939_add_closed_to_index/up.sql new file mode 100644 index 000000000000..87580a2f3f9e --- /dev/null +++ b/web/converse/migrations/2018-05-25-161939_add_closed_to_index/up.sql @@ -0,0 +1,35 @@ +-- Update the index view: +DROP VIEW thread_index; +CREATE VIEW thread_index AS + SELECT t.id AS thread_id, + t.title AS title, + ta.name AS thread_author, + t.posted AS created, + t.sticky AS sticky, + t.closed AS closed, + p.id AS post_id, + pa.name AS post_author, + p.posted AS posted + FROM threads t + JOIN (SELECT DISTINCT ON (thread_id) + id, thread_id, user_id, posted + FROM posts + ORDER BY thread_id, id DESC) AS p + ON t.id = p.thread_id + JOIN users ta ON ta.id = t.user_id + JOIN users pa ON pa.id = p.user_id + ORDER BY t.sticky DESC, p.id DESC; + +-- Update post view: +DROP VIEW simple_posts; +CREATE VIEW simple_posts AS + SELECT p.id AS id, + thread_id, body, + p.posted AS posted, + p.user_id AS user_id, + threads.closed AS closed, + users.name AS author_name, + users.email AS author_email + FROM posts p + JOIN users ON users.id = p.user_id + JOIN threads ON threads.id = p.thread_id; diff --git a/web/converse/src/db.rs b/web/converse/src/db.rs new file mode 100644 index 000000000000..ae186bdf4e4d --- /dev/null +++ b/web/converse/src/db.rs @@ -0,0 +1,321 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +//! This module implements the database executor, which holds the +//! database connection and performs queries on it. + +use actix::prelude::*; +use diesel::{self, sql_query}; +use diesel::sql_types::Text; +use diesel::prelude::*; +use diesel::r2d2::{Pool, ConnectionManager}; +use crate::models::*; +use crate::errors::{ConverseError, Result}; + +/// Raw PostgreSQL query used to perform full-text search on posts +/// with a supplied phrase. For now, the query language is hardcoded +/// to English and only "plain" queries (i.e. no searches for exact +/// matches or more advanced query syntax) are supported. +const SEARCH_QUERY: &'static str = r#" +WITH search_query (query) AS (VALUES (plainto_tsquery('english', $1))) +SELECT post_id, + thread_id, + author, + title, + ts_headline('english', body, query) AS headline + FROM search_index, search_query + WHERE document @@ query + ORDER BY ts_rank(document, query) DESC + LIMIT 50 +"#; + +const REFRESH_QUERY: &'static str = "REFRESH MATERIALIZED VIEW search_index"; + +pub struct DbExecutor(pub Pool<ConnectionManager<PgConnection>>); + +impl DbExecutor { + /// Request a list of threads. + // + // TODO(tazjin): This should support pagination. + pub fn list_threads(&self) -> Result<Vec<ThreadIndex>> { + use crate::schema::thread_index::dsl::*; + + let conn = self.0.get()?; + let results = thread_index + .load::<ThreadIndex>(&conn)?; + Ok(results) + } + + /// Look up a user based on their email-address. If the user does + /// not exist, it is created. + pub fn lookup_or_create_user(&self, user_email: &str, user_name: &str) -> Result<User> { + use crate::schema::users; + use crate::schema::users::dsl::*; + + let conn = self.0.get()?; + + let opt_user = users + .filter(email.eq(email)) + .first(&conn).optional()?; + + if let Some(user) = opt_user { + Ok(user) + } else { + let new_user = NewUser { + email: user_email.to_string(), + name: user_name.to_string(), + }; + + let user: User = diesel::insert_into(users::table) + .values(&new_user) + .get_result(&conn)?; + + info!("Created new user {} with ID {}", new_user.email, user.id); + + Ok(user) + } + } + + /// Fetch a specific thread and return it with its posts. + pub fn get_thread(&self, thread_id: i32) -> Result<(Thread, Vec<SimplePost>)> { + use crate::schema::threads::dsl::*; + use crate::schema::simple_posts::dsl::id; + + let conn = self.0.get()?; + let thread_result: Thread = threads + .find(thread_id).first(&conn)?; + + let post_list = SimplePost::belonging_to(&thread_result) + .order_by(id.asc()) + .load::<SimplePost>(&conn)?; + + Ok((thread_result, post_list)) + } + + /// Fetch a specific post. + pub fn get_post(&self, post_id: i32) -> Result<SimplePost> { + use crate::schema::simple_posts::dsl::*; + let conn = self.0.get()?; + Ok(simple_posts.find(post_id).first(&conn)?) + } + + /// Update the content of a post. + pub fn update_post(&self, post_id: i32, post_text: String) -> Result<Post> { + use crate::schema::posts::dsl::*; + let conn = self.0.get()?; + let updated = diesel::update(posts.find(post_id)) + .set(body.eq(post_text)) + .get_result(&conn)?; + + Ok(updated) + } + + /// Create a new thread. + pub fn create_thread(&self, new_thread: NewThread, post_text: String) -> Result<Thread> { + use crate::schema::threads; + use crate::schema::posts; + + let conn = self.0.get()?; + + conn.transaction::<Thread, ConverseError, _>(|| { + // First insert the thread structure itself + let thread: Thread = diesel::insert_into(threads::table) + .values(&new_thread) + .get_result(&conn)?; + + // ... then create the first post in the thread. + let new_post = NewPost { + thread_id: thread.id, + body: post_text, + user_id: new_thread.user_id, + }; + + diesel::insert_into(posts::table) + .values(&new_post) + .execute(&conn)?; + + Ok(thread) + }) + } + + /// Create a new post. + pub fn create_post(&self, new_post: NewPost) -> Result<Post> { + use crate::schema::posts; + + let conn = self.0.get()?; + + let closed: bool = { + use crate::schema::threads::dsl::*; + threads.select(closed) + .find(new_post.thread_id) + .first(&conn)? + }; + + if closed { + return Err(ConverseError::ThreadClosed { + id: new_post.thread_id + }) + } + + Ok(diesel::insert_into(posts::table) + .values(&new_post) + .get_result(&conn)?) + } + + /// Search for posts. + pub fn search_posts(&self, query: String) -> Result<Vec<SearchResult>> { + let conn = self.0.get()?; + + let search_results = sql_query(SEARCH_QUERY) + .bind::<Text, _>(query) + .get_results::<SearchResult>(&conn)?; + + Ok(search_results) + } + + /// Trigger a refresh of the view used for full-text searching. + pub fn refresh_search_view(&self) -> Result<()> { + let conn = self.0.get()?; + debug!("Refreshing search_index view in DB"); + sql_query(REFRESH_QUERY).execute(&conn)?; + Ok(()) + } +} + + +// Old actor implementation: + +impl Actor for DbExecutor { + type Context = SyncContext<Self>; +} + +/// Message used to look up a user based on their email-address. If +/// the user does not exist, it is created. +pub struct LookupOrCreateUser { + pub email: String, + pub name: String, +} + +message!(LookupOrCreateUser, Result<User>); + +impl Handler<LookupOrCreateUser> for DbExecutor { + type Result = <LookupOrCreateUser as Message>::Result; + + fn handle(&mut self, + _: LookupOrCreateUser, + _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message used to fetch a specific thread. Returns the thread and +/// its posts. +pub struct GetThread(pub i32); +message!(GetThread, Result<(Thread, Vec<SimplePost>)>); + +impl Handler<GetThread> for DbExecutor { + type Result = <GetThread as Message>::Result; + + fn handle(&mut self, _: GetThread, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message used to fetch a specific post. +#[derive(Deserialize, Debug)] +pub struct GetPost { pub id: i32 } + +message!(GetPost, Result<SimplePost>); + +impl Handler<GetPost> for DbExecutor { + type Result = <GetPost as Message>::Result; + + fn handle(&mut self, _: GetPost, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message used to update the content of a post. +#[derive(Deserialize)] +pub struct UpdatePost { + pub post_id: i32, + pub post: String, +} + +message!(UpdatePost, Result<Post>); + +impl Handler<UpdatePost> for DbExecutor { + type Result = Result<Post>; + + fn handle(&mut self, _: UpdatePost, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message used to create a new thread +pub struct CreateThread { + pub new_thread: NewThread, + pub post: String, +} +message!(CreateThread, Result<Thread>); + +impl Handler<CreateThread> for DbExecutor { + type Result = <CreateThread as Message>::Result; + + fn handle(&mut self, _: CreateThread, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message used to create a new reply +pub struct CreatePost(pub NewPost); +message!(CreatePost, Result<Post>); + +impl Handler<CreatePost> for DbExecutor { + type Result = <CreatePost as Message>::Result; + + fn handle(&mut self, _: CreatePost, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message used to search for posts +#[derive(Deserialize)] +pub struct SearchPosts { pub query: String } +message!(SearchPosts, Result<Vec<SearchResult>>); + +impl Handler<SearchPosts> for DbExecutor { + type Result = <SearchPosts as Message>::Result; + + fn handle(&mut self, _: SearchPosts, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} + +/// Message that triggers a refresh of the view used for full-text +/// searching. +pub struct RefreshSearchView; +message!(RefreshSearchView, Result<()>); + +impl Handler<RefreshSearchView> for DbExecutor { + type Result = Result<()>; + + fn handle(&mut self, _: RefreshSearchView, _: &mut Self::Context) -> Self::Result { + unimplemented!() + } +} diff --git a/web/converse/src/errors.rs b/web/converse/src/errors.rs new file mode 100644 index 000000000000..32507c51b0c2 --- /dev/null +++ b/web/converse/src/errors.rs @@ -0,0 +1,142 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +//! This module defines custom error types using the `failure`-crate. +//! Links to foreign error types (such as database connection errors) +//! are established in a similar way as was tradition in +//! `error_chain`, albeit manually. + +use std::result; +use actix_web::{ResponseError, HttpResponse}; +use actix_web::http::StatusCode; + +// Modules with foreign errors: +use actix; +use actix_web; +use askama; +use diesel; +use r2d2; +use tokio_timer; + +pub type Result<T> = result::Result<T, ConverseError>; +pub type ConverseResult<T> = result::Result<T, ConverseError>; + +#[derive(Debug, Fail)] +pub enum ConverseError { + #[fail(display = "an internal Converse error occured: {}", reason)] + InternalError { reason: String }, + + #[fail(display = "a database error occured: {}", error)] + Database { error: diesel::result::Error }, + + #[fail(display = "a database connection pool error occured: {}", error)] + ConnectionPool { error: r2d2::Error }, + + #[fail(display = "a template rendering error occured: {}", reason)] + Template { reason: String }, + + #[fail(display = "error occured during request handling: {}", error)] + ActixWeb { error: actix_web::Error }, + + #[fail(display = "error occured running timer: {}", error)] + Timer { error: tokio_timer::Error }, + + #[fail(display = "user {} does not have permission to edit post {}", user, id)] + PostEditForbidden { user: i32, id: i32 }, + + #[fail(display = "thread {} is closed and can not be responded to", id)] + ThreadClosed { id: i32 }, + + #[fail(display = "JSON serialisation failed: {}", error)] + Serialisation { error: serde_json::Error }, + + // This variant is used as a catch-all for wrapping + // actix-web-compatible response errors, such as the errors it + // throws itself. + #[fail(display = "Actix response error: {}", error)] + Actix { error: Box<dyn ResponseError> }, +} + +// Establish conversion links to foreign errors: + +impl From<diesel::result::Error> for ConverseError { + fn from(error: diesel::result::Error) -> ConverseError { + ConverseError::Database { error } + } +} + +impl From<r2d2::Error> for ConverseError { + fn from(error: r2d2::Error) -> ConverseError { + ConverseError::ConnectionPool { error } + } +} + +impl From<askama::Error> for ConverseError { + fn from(error: askama::Error) -> ConverseError { + ConverseError::Template { + reason: format!("{}", error), + } + } +} + +impl From<actix::MailboxError> for ConverseError { + fn from(error: actix::MailboxError) -> ConverseError { + ConverseError::Actix { error: Box::new(error) } + } +} + +impl From<actix_web::Error> for ConverseError { + fn from(error: actix_web::Error) -> ConverseError { + ConverseError::ActixWeb { error } + } +} + +impl From<serde_json::Error> for ConverseError { + fn from(error: serde_json::Error) -> ConverseError { + ConverseError::Serialisation { error } + } +} + +impl From<curl::Error> for ConverseError { + fn from(error: curl::Error) -> ConverseError { + ConverseError::InternalError { + reason: format!("error during HTTP request: {}", error), + } + } +} + +impl From<tokio_timer::Error> for ConverseError { + fn from(error: tokio_timer::Error) -> ConverseError { + ConverseError::Timer { error } + } +} + +// Support conversion of error type into HTTP error responses: + +impl ResponseError for ConverseError { + fn error_response(&self) -> HttpResponse { + // Everything is mapped to internal server errors for now. + match *self { + ConverseError::ThreadClosed { id } => HttpResponse::SeeOther() + .header("Location", format!("/thread/{}#post-reply", id)) + .finish(), + _ => HttpResponse::build(StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("An error occured: {}", self)) + } + } +} diff --git a/web/converse/src/handlers.rs b/web/converse/src/handlers.rs new file mode 100644 index 000000000000..0759cec5c146 --- /dev/null +++ b/web/converse/src/handlers.rs @@ -0,0 +1,345 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +//! This module contains the implementation of converse's actix-web +//! HTTP handlers. +//! +//! Most handlers have an associated rendering function using one of +//! the tera templates stored in the `/templates` directory in the +//! project root. + +use actix::prelude::*; +use actix_web::*; +use actix_web::http::Method; +use actix_web::middleware::identity::RequestIdentity; +use actix_web::middleware::{Started, Middleware}; +use actix_web; +use crate::db::*; +use crate::errors::{ConverseResult, ConverseError}; +use futures::Future; +use crate::models::*; +use crate::oidc::*; +use crate::render::*; + +use rouille::{Request, Response}; + +type ConverseResponse = Box<dyn Future<Item=HttpResponse, Error=ConverseError>>; + +const HTML: &'static str = "text/html"; +const ANONYMOUS: i32 = 1; +const NEW_THREAD_LENGTH_ERR: &'static str = "Title and body can not be empty!"; + +/// Represents the state carried by the web server actors. +pub struct AppState { + /// Address of the database actor + pub db: Addr<DbExecutor>, + + /// Address of the OIDC actor + pub oidc: Addr<OidcExecutor>, + + /// Address of the rendering actor + pub renderer: Addr<Renderer>, +} + +/// Serve the forum's index page. +pub fn forum_index_rouille(db: &DbExecutor) -> ConverseResult<Response> { + let threads = db.list_threads()?; + Ok(Response::html(index_page(threads)?)) +} + +pub fn forum_index(_: State<AppState>) -> ConverseResponse { + unimplemented!() +} + +/// Returns the ID of the currently logged in user. If there is no ID +/// present, the ID of the anonymous user will be returned. +pub fn get_user_id(req: &HttpRequest<AppState>) -> i32 { + if let Some(id) = req.identity() { + // If this .expect() call is triggered, someone is likely + // attempting to mess with their cookies. These requests can + // be allowed to fail without further ado. + id.parse().expect("Session cookie contained invalid data!") + } else { + ANONYMOUS + } +} + +pub fn get_user_id_rouille(_req: &Request) -> i32 { + // TODO(tazjin): Implement session support in rouille somehow. + ANONYMOUS +} + +pub fn forum_thread_rouille(req: &Request, db: &DbExecutor, thread_id: i32) + -> ConverseResult<Response> { + let user = get_user_id_rouille(&req); + let thread = db.get_thread(thread_id)?; + Ok(Response::html(thread_page(user, thread.0, thread.1)?)) +} + +/// This handler retrieves and displays a single forum thread. +pub fn forum_thread(_: State<AppState>, + _: HttpRequest<AppState>, + _: Path<i32>) -> ConverseResponse { + unimplemented!() +} + +/// This handler presents the user with the "New Thread" form. +pub fn new_thread(state: State<AppState>) -> ConverseResponse { + state.renderer.send(NewThreadPage::default()).flatten() + .map(|res| HttpResponse::Ok().content_type(HTML).body(res)) + .responder() +} + +#[derive(Deserialize)] +pub struct NewThreadForm { + pub title: String, + pub post: String, +} + +/// This handler receives a "New thread"-form and redirects the user +/// to the new thread after creation. +pub fn submit_thread((state, input, req): (State<AppState>, + Form<NewThreadForm>, + HttpRequest<AppState>)) -> ConverseResponse { + // Trim whitespace out of inputs: + let input = NewThreadForm { + title: input.title.trim().into(), + post: input.post.trim().into(), + }; + + // Perform simple validation and abort here if it fails: + if input.title.is_empty() || input.post.is_empty() { + return state.renderer + .send(NewThreadPage { + alerts: vec![NEW_THREAD_LENGTH_ERR], + title: Some(input.title), + post: Some(input.post), + }) + .flatten() + .map(|res| HttpResponse::Ok().content_type(HTML).body(res)) + .responder(); + } + + let user_id = get_user_id(&req); + + let new_thread = NewThread { + user_id, + title: input.title, + }; + + let msg = CreateThread { + new_thread, + post: input.post, + }; + + state.db.send(msg) + .from_err() + .and_then(move |res| { + let thread = res?; + info!("Created new thread \"{}\" with ID {}", thread.title, thread.id); + Ok(HttpResponse::SeeOther() + .header("Location", format!("/thread/{}", thread.id)) + .finish()) + }) + .responder() +} + +#[derive(Deserialize)] +pub struct NewPostForm { + pub thread_id: i32, + pub post: String, +} + +/// This handler receives a "Reply"-form and redirects the user to the +/// new post after creation. +pub fn reply_thread(state: State<AppState>, + input: Form<NewPostForm>, + req: HttpRequest<AppState>) -> ConverseResponse { + let user_id = get_user_id(&req); + + let new_post = NewPost { + user_id, + thread_id: input.thread_id, + body: input.post.trim().into(), + }; + + state.db.send(CreatePost(new_post)) + .flatten() + .from_err() + .and_then(move |post| { + info!("Posted reply {} to thread {}", post.id, post.thread_id); + Ok(HttpResponse::SeeOther() + .header("Location", format!("/thread/{}#post-{}", post.thread_id, post.id)) + .finish()) + }) + .responder() +} + +/// This handler presents the user with the form to edit a post. If +/// the user attempts to edit a post that they do not have access to, +/// they are currently ungracefully redirected back to the post +/// itself. +pub fn edit_form(state: State<AppState>, + req: HttpRequest<AppState>, + query: Path<GetPost>) -> ConverseResponse { + let user_id = get_user_id(&req); + + state.db.send(query.into_inner()) + .flatten() + .from_err() + .and_then(move |post| { + if user_id != 1 && post.user_id == user_id { + return Ok(post); + } + + Err(ConverseError::PostEditForbidden { + user: user_id, + id: post.id, + }) + }) + .and_then(move |post| { + let edit_msg = EditPostPage { + id: post.id, + post: post.body, + }; + + state.renderer.send(edit_msg).from_err() + }) + .flatten() + .map(|page| HttpResponse::Ok().content_type(HTML).body(page)) + .responder() +} + +/// This handler "executes" an edit to a post if the current user owns +/// the edited post. +pub fn edit_post(state: State<AppState>, + req: HttpRequest<AppState>, + update: Form<UpdatePost>) -> ConverseResponse { + let user_id = get_user_id(&req); + + state.db.send(GetPost { id: update.post_id }) + .flatten() + .from_err() + .and_then(move |post| { + if user_id != 1 && post.user_id == user_id { + Ok(()) + } else { + Err(ConverseError::PostEditForbidden { + user: user_id, + id: post.id, + }) + } + }) + .and_then(move |_| state.db.send(update.0).from_err()) + .flatten() + .map(|updated| HttpResponse::SeeOther() + .header("Location", format!("/thread/{}#post-{}", + updated.thread_id, updated.id)) + .finish()) + .responder() +} + +/// This handler executes a full-text search on the forum database and +/// displays the results to the user. +pub fn search_forum(state: State<AppState>, + query: Query<SearchPosts>) -> ConverseResponse { + let query_string = query.query.clone(); + state.db.send(query.into_inner()) + .flatten() + .and_then(move |results| state.renderer.send(SearchResultPage { + results, + query: query_string, + }).from_err()) + .flatten() + .map(|res| HttpResponse::Ok().content_type(HTML).body(res)) + .responder() +} + +/// This handler initiates an OIDC login. +pub fn login(state: State<AppState>) -> ConverseResponse { + state.oidc.send(GetLoginUrl) + .from_err() + .and_then(|url| Ok(HttpResponse::TemporaryRedirect() + .header("Location", url) + .finish())) + .responder() +} + +/// This handler handles an OIDC callback (i.e. completed login). +/// +/// Upon receiving the callback, a token is retrieved from the OIDC +/// provider and a user lookup is performed. If a user with a matching +/// email-address is found in the database, it is logged in - +/// otherwise a new user is created. +pub fn callback(state: State<AppState>, + data: Form<CodeResponse>, + req: HttpRequest<AppState>) -> ConverseResponse { + state.oidc.send(RetrieveToken(data.0)).flatten() + .map(|author| LookupOrCreateUser { + email: author.email, + name: author.name, + }) + .and_then(move |msg| state.db.send(msg).from_err()).flatten() + .and_then(move |user| { + info!("Completed login for user {} ({})", user.email, user.id); + req.remember(user.id.to_string()); + Ok(HttpResponse::SeeOther() + .header("Location", "/") + .finish())}) + .responder() +} + +/// This is an extension trait to enable easy serving of embedded +/// static content. +/// +/// It is intended to be called with `include_bytes!()` when setting +/// up the actix-web application. +pub trait EmbeddedFile { + fn static_file(self, path: &'static str, content: &'static [u8]) -> Self; +} + +impl EmbeddedFile for App<AppState> { + fn static_file(self, path: &'static str, content: &'static [u8]) -> Self { + self.route(path, Method::GET, move |_: HttpRequest<_>| { + let mime = format!("{}", mime_guess::from_path(path).first_or_octet_stream()); + HttpResponse::Ok() + .content_type(mime.as_str()) + .body(content) + }) + } +} + +/// Middleware used to enforce logins unceremoniously. +pub struct RequireLogin; + +impl <S> Middleware<S> for RequireLogin { + fn start(&self, req: &HttpRequest<S>) -> actix_web::Result<Started> { + let logged_in = req.identity().is_some(); + let is_oidc_req = req.path().starts_with("/oidc"); + + if !is_oidc_req && !logged_in { + Ok(Started::Response( + HttpResponse::SeeOther() + .header("Location", "/oidc/login") + .finish() + )) + } else { + Ok(Started::Done) + } + } +} diff --git a/web/converse/src/main.rs b/web/converse/src/main.rs new file mode 100644 index 000000000000..6d6e9ac71020 --- /dev/null +++ b/web/converse/src/main.rs @@ -0,0 +1,224 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +extern crate askama; + +#[macro_use] +extern crate diesel; + +#[macro_use] +extern crate failure; + +#[macro_use] +extern crate log; + +#[macro_use] +extern crate serde_derive; + +extern crate rouille; +extern crate actix; +extern crate actix_web; +extern crate chrono; +extern crate comrak; +extern crate crimp; +extern crate curl; +extern crate env_logger; +extern crate futures; +extern crate hyper; +extern crate md5; +extern crate mime_guess; +extern crate r2d2; +extern crate rand; +extern crate serde; +extern crate serde_json; +extern crate tokio; +extern crate tokio_timer; +extern crate url; +extern crate url_serde; + +/// Simple macro used to reduce boilerplate when defining actor +/// message types. +macro_rules! message { + ( $t:ty, $r:ty ) => { + impl Message for $t { + type Result = $r; + } + } +} + +pub mod db; +pub mod errors; +pub mod handlers; +pub mod models; +pub mod oidc; +pub mod render; +pub mod schema; + +use actix::prelude::*; +use actix_web::*; +use actix_web::http::Method; +use actix_web::middleware::Logger; +use actix_web::middleware::identity::{IdentityService, CookieIdentityPolicy}; +use crate::db::*; +use diesel::pg::PgConnection; +use diesel::r2d2::{ConnectionManager, Pool}; +use crate::handlers::*; +use crate::oidc::OidcExecutor; +use rand::{OsRng, Rng}; +use crate::render::Renderer; +use std::env; + +fn config(name: &str) -> String { + env::var(name).expect(&format!("{} must be set", name)) +} + +fn config_default(name: &str, default: &str) -> String { + env::var(name).unwrap_or(default.into()) +} + +fn start_db_executor() -> Addr<DbExecutor> { + info!("Initialising database connection pool ..."); + let db_url = config("DATABASE_URL"); + + let manager = ConnectionManager::<PgConnection>::new(db_url); + let pool = Pool::builder().build(manager).expect("Failed to initialise DB pool"); + + SyncArbiter::start(2, move || DbExecutor(pool.clone())) +} + +fn schedule_search_refresh(db: Addr<DbExecutor>) { + use tokio::prelude::*; + use tokio::timer::Interval; + use std::time::{Duration, Instant}; + use std::thread; + + let task = Interval::new(Instant::now(), Duration::from_secs(60)) + .from_err() + .for_each(move |_| db.send(db::RefreshSearchView).flatten()) + .map_err(|err| error!("Error while updating search view: {}", err)); + + thread::spawn(|| tokio::run(task)); +} + +fn start_oidc_executor(base_url: &str) -> Addr<OidcExecutor> { + info!("Initialising OIDC integration ..."); + let oidc_url = config("OIDC_DISCOVERY_URL"); + let oidc_config = oidc::load_oidc(&oidc_url) + .expect("Failed to retrieve OIDC discovery document"); + + let oidc = oidc::OidcExecutor { + oidc_config, + client_id: config("OIDC_CLIENT_ID"), + client_secret: config("OIDC_CLIENT_SECRET"), + redirect_uri: format!("{}/oidc/callback", base_url), + }; + + oidc.start() +} + +fn start_renderer() -> Addr<Renderer> { + let comrak = comrak::ComrakOptions{ + github_pre_lang: true, + ext_strikethrough: true, + ext_table: true, + ext_autolink: true, + ext_tasklist: true, + ext_footnotes: true, + ext_tagfilter: true, + ..Default::default() + }; + + Renderer{ comrak }.start() +} + +fn gen_session_key() -> [u8; 64] { + let mut key_bytes = [0; 64]; + let mut rng = OsRng::new() + .expect("Failed to retrieve RNG for key generation"); + rng.fill_bytes(&mut key_bytes); + + key_bytes +} + +fn start_http_server(base_url: String, + db_addr: Addr<DbExecutor>, + oidc_addr: Addr<OidcExecutor>, + renderer_addr: Addr<Renderer>) { + info!("Initialising HTTP server ..."); + let bind_host = config_default("CONVERSE_BIND_HOST", "127.0.0.1:4567"); + let key = gen_session_key(); + let require_login = config_default("REQUIRE_LOGIN", "true".into()) == "true"; + + server::new(move || { + let state = AppState { + db: db_addr.clone(), + oidc: oidc_addr.clone(), + renderer: renderer_addr.clone(), + }; + + let identity = IdentityService::new( + CookieIdentityPolicy::new(&key) + .name("converse_auth") + .path("/") + .secure(base_url.starts_with("https")) + ); + + let app = App::with_state(state) + .middleware(Logger::default()) + .middleware(identity) + .resource("/", |r| r.method(Method::GET).with(forum_index)) + .resource("/thread/new", |r| r.method(Method::GET).with(new_thread)) + .resource("/thread/submit", |r| r.method(Method::POST).with(submit_thread)) + .resource("/thread/reply", |r| r.method(Method::POST).with(reply_thread)) + .resource("/thread/{id}", |r| r.method(Method::GET).with(forum_thread)) + .resource("/post/{id}/edit", |r| r.method(Method::GET).with(edit_form)) + .resource("/post/edit", |r| r.method(Method::POST).with(edit_post)) + .resource("/search", |r| r.method(Method::GET).with(search_forum)) + .resource("/oidc/login", |r| r.method(Method::GET).with(login)) + .resource("/oidc/callback", |r| r.method(Method::POST).with(callback)) + .static_file("/static/highlight.css", include_bytes!("../static/highlight.css")) + .static_file("/static/highlight.js", include_bytes!("../static/highlight.js")) + .static_file("/static/styles.css", include_bytes!("../static/styles.css")); + + if require_login { + app.middleware(RequireLogin) + } else { + app + }}) + .bind(&bind_host).expect(&format!("Could not bind on '{}'", bind_host)) + .start(); +} + +fn main() { + env_logger::init(); + + info!("Welcome to Converse! Hold on tight while we're getting ready."); + let sys = actix::System::new("converse"); + + let base_url = config("BASE_URL"); + + let db_addr = start_db_executor(); + let oidc_addr = start_oidc_executor(&base_url); + let renderer_addr = start_renderer(); + + schedule_search_refresh(db_addr.clone()); + + start_http_server(base_url, db_addr, oidc_addr, renderer_addr); + + sys.run(); +} diff --git a/web/converse/src/models.rs b/web/converse/src/models.rs new file mode 100644 index 000000000000..da628f78b5bc --- /dev/null +++ b/web/converse/src/models.rs @@ -0,0 +1,127 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +use chrono::prelude::{DateTime, Utc}; +use crate::schema::{users, threads, posts, simple_posts}; +use diesel::sql_types::{Text, Integer}; + +/// Represents a single user in the Converse database. Converse does +/// not handle logins itself, but rather looks them up based on the +/// email address received from an OIDC provider. +#[derive(Identifiable, Queryable, Serialize)] +pub struct User { + pub id: i32, + pub name: String, + pub email: String, + pub admin: bool, +} + +#[derive(Identifiable, Queryable, Serialize, Associations)] +#[belongs_to(User)] +pub struct Thread { + pub id: i32, + pub title: String, + pub posted: DateTime<Utc>, + pub sticky: bool, + pub user_id: i32, + pub closed: bool, +} + +#[derive(Identifiable, Queryable, Serialize, Associations)] +#[belongs_to(Thread)] +#[belongs_to(User)] +pub struct Post { + pub id: i32, + pub thread_id: i32, + pub body: String, + pub posted: DateTime<Utc>, + pub user_id: i32, +} + +/// This struct is used as the query result type for the simplified +/// post view, which already joins user information in the database. +#[derive(Identifiable, Queryable, Serialize, Associations)] +#[belongs_to(Thread)] +pub struct SimplePost { + pub id: i32, + pub thread_id: i32, + pub body: String, + pub posted: DateTime<Utc>, + pub user_id: i32, + pub closed: bool, + pub author_name: String, + pub author_email: String, +} + +/// This struct is used as the query result type for the thread index +/// view, which lists the index of threads ordered by the last post in +/// each thread. +#[derive(Queryable, Serialize)] +pub struct ThreadIndex { + pub thread_id: i32, + pub title: String, + pub thread_author: String, + pub created: DateTime<Utc>, + pub sticky: bool, + pub closed: bool, + pub post_id: i32, + pub post_author: String, + pub posted: DateTime<Utc>, +} + +#[derive(Deserialize, Insertable)] +#[table_name="threads"] +pub struct NewThread { + pub title: String, + pub user_id: i32, +} + +#[derive(Deserialize, Insertable)] +#[table_name="users"] +pub struct NewUser { + pub email: String, + pub name: String, +} + +#[derive(Deserialize, Insertable)] +#[table_name="posts"] +pub struct NewPost { + pub thread_id: i32, + pub body: String, + pub user_id: i32, +} + +/// This struct models the response of a full-text search query. It +/// does not use a table/schema definition struct like the other +/// tables, as no table of this type actually exists. +#[derive(QueryableByName, Debug, Serialize)] +pub struct SearchResult { + #[sql_type = "Integer"] + pub post_id: i32, + #[sql_type = "Integer"] + pub thread_id: i32, + #[sql_type = "Text"] + pub author: String, + #[sql_type = "Text"] + pub title: String, + + /// Headline represents the result of Postgres' ts_headline() + /// function, which highlights search terms in the search results. + #[sql_type = "Text"] + pub headline: String, +} diff --git a/web/converse/src/oidc.rs b/web/converse/src/oidc.rs new file mode 100644 index 000000000000..9f566c04a71a --- /dev/null +++ b/web/converse/src/oidc.rs @@ -0,0 +1,159 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +//! This module provides authentication via OIDC compliant +//! authentication sources. +//! +//! Currently Converse only supports a single OIDC provider. Note that +//! this has so far only been tested with Office365. + +use actix::prelude::*; +use crate::errors::*; +use crimp::Request; +use url::Url; +use url_serde; +use curl::easy::Form; + +/// This structure represents the contents of an OIDC discovery +/// document. +#[derive(Deserialize, Debug, Clone)] +pub struct OidcConfig { + #[serde(with = "url_serde")] + authorization_endpoint: Url, + token_endpoint: String, + userinfo_endpoint: String, + + scopes_supported: Vec<String>, + issuer: String, +} + +#[derive(Clone, Debug)] +pub struct OidcExecutor { + pub client_id: String, + pub client_secret: String, + pub redirect_uri: String, + pub oidc_config: OidcConfig, +} + +/// This struct represents the form response returned by an OIDC +/// provider with the `code`. +#[derive(Debug, Deserialize)] +pub struct CodeResponse { + pub code: String, +} + +/// This struct represents the data extracted from the ID token and +/// stored in the user's session. +#[derive(Debug, Serialize, Deserialize)] +pub struct Author { + pub name: String, + pub email: String, +} + +impl Actor for OidcExecutor { + type Context = Context<Self>; +} + +/// Message used to request the login URL: +pub struct GetLoginUrl; // TODO: Add a nonce parameter stored in session. +message!(GetLoginUrl, String); + +impl Handler<GetLoginUrl> for OidcExecutor { + type Result = String; + + fn handle(&mut self, _: GetLoginUrl, _: &mut Self::Context) -> Self::Result { + let mut url: Url = self.oidc_config.authorization_endpoint.clone(); + { + let mut params = url.query_pairs_mut(); + params.append_pair("client_id", &self.client_id); + params.append_pair("response_type", "code"); + params.append_pair("scope", "openid"); + params.append_pair("redirect_uri", &self.redirect_uri); + params.append_pair("response_mode", "form_post"); + } + return url.into_string(); + } +} + +/// Message used to request the token from the returned code and +/// retrieve userinfo from the appropriate endpoint. +pub struct RetrieveToken(pub CodeResponse); +message!(RetrieveToken, Result<Author>); + +#[derive(Debug, Deserialize)] +struct TokenResponse { + access_token: String, +} + +// TODO: This is currently hardcoded to Office365 fields. +#[derive(Debug, Deserialize)] +struct Userinfo { + name: String, + unique_name: String, // email in office365 +} + +impl Handler<RetrieveToken> for OidcExecutor { + type Result = Result<Author>; + + fn handle(&mut self, msg: RetrieveToken, _: &mut Self::Context) -> Self::Result { + debug!("Received OAuth2 code, requesting access_token"); + + let mut form = Form::new(); + form.part("client_id").contents(&self.client_id.as_bytes()) + .add().expect("critical error: invalid form data"); + + form.part("client_secret").contents(&self.client_secret.as_bytes()) + .add().expect("critical error: invalid form data"); + + form.part("grant_type").contents("authorization_code".as_bytes()) + .add().expect("critical error: invalid form data"); + + form.part("code").contents(&msg.0.code.as_bytes()) + .add().expect("critical error: invalid form data"); + + form.part("redirect_uri").contents(&self.redirect_uri.as_bytes()) + .add().expect("critical error: invalid form data"); + + let response = Request::post(&self.oidc_config.token_endpoint) + .user_agent(concat!("converse-", env!("CARGO_PKG_VERSION")))? + .form(form) + .send()?; + + debug!("Received token response: {:?}", response); + let token: TokenResponse = response.as_json()?.body; + + let bearer = format!("Bearer {}", token.access_token); + let user: Userinfo = Request::get(&self.oidc_config.userinfo_endpoint) + .user_agent(concat!("converse-", env!("CARGO_PKG_VERSION")))? + .header("Authorization", &bearer)? + .send()? + .as_json()?.body; + + Ok(Author { + name: user.name, + email: user.unique_name, + }) + } +} + +/// Convenience function to attempt loading an OIDC discovery document +/// from a specified URL: +pub fn load_oidc(url: &str) -> Result<OidcConfig> { + let config: OidcConfig = Request::get(url).send()?.as_json()?.body; + Ok(config) +} diff --git a/web/converse/src/render.rs b/web/converse/src/render.rs new file mode 100644 index 000000000000..749e77ef50eb --- /dev/null +++ b/web/converse/src/render.rs @@ -0,0 +1,240 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +//! This module defines a rendering actor used for processing Converse +//! data into whatever format is needed by the templates and rendering +//! them. + +use actix::prelude::*; +use askama::Template; +use crate::errors::*; +use std::fmt; +use md5; +use crate::models::*; +use chrono::prelude::{DateTime, Utc}; +use comrak::{ComrakOptions, markdown_to_html}; + +pub struct Renderer { + pub comrak: ComrakOptions, +} + +impl Actor for Renderer { + type Context = actix::Context<Self>; +} + +/// Represents a data formatted for human consumption +#[derive(Debug)] +struct FormattedDate(DateTime<Utc>); + +impl fmt::Display for FormattedDate { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.0.format("%a %d %B %Y, %R")) + } +} + +#[derive(Debug)] +struct IndexThread { + id: i32, + title: String, + sticky: bool, + closed: bool, + posted: FormattedDate, + author_name: String, + post_author: String, +} + +#[derive(Template)] +#[template(path = "index.html")] +struct IndexPageTemplate { + threads: Vec<IndexThread>, +} + +// "Renderable" structures with data transformations applied. +#[derive(Debug)] +struct RenderablePost { + id: i32, + body: String, + posted: FormattedDate, + author_name: String, + author_gravatar: String, + editable: bool, +} + +/// This structure represents the transformed thread data with +/// Markdown rendering and other changes applied. +#[derive(Template)] +#[template(path = "thread.html")] +struct RenderableThreadPage { + id: i32, + title: String, + closed: bool, + posts: Vec<RenderablePost>, +} + +/// Helper function for computing Gravatar links. +fn md5_hex(input: &[u8]) -> String { + format!("{:x}", md5::compute(input)) +} + +/// The different types of editing modes supported by the editing +/// template: +#[derive(Debug, PartialEq)] +pub enum EditingMode { + NewThread, + PostReply, + EditPost, +} + +impl Default for EditingMode { + fn default() -> EditingMode { EditingMode::NewThread } +} + +/// This is the template used for rendering the new thread, edit post +/// and reply to thread forms. +#[derive(Template, Default)] +#[template(path = "post.html")] +pub struct FormTemplate { + /// Which editing mode is to be used by the template? + pub mode: EditingMode, + + /// Potential alerts to display to the user (e.g. input validation + /// results) + pub alerts: Vec<&'static str>, + + /// Either the title to be used in the subject field or the title + /// of the thread the user is responding to. + pub title: Option<String>, + + /// Body of the post being edited, if present. + pub post: Option<String>, + + /// ID of the thread being replied to or the post being edited. + pub id: Option<i32>, +} + +/// Message used to render new thread page. +/// +/// It can optionally contain a vector of warnings to display to the +/// user in alert boxes, such as input validation errors. +#[derive(Default)] +pub struct NewThreadPage { + pub alerts: Vec<&'static str>, + pub title: Option<String>, + pub post: Option<String>, +} +message!(NewThreadPage, Result<String>); + +impl Handler<NewThreadPage> for Renderer { + type Result = Result<String>; + + fn handle(&mut self, msg: NewThreadPage, _: &mut Self::Context) -> Self::Result { + let ctx = FormTemplate { + alerts: msg.alerts, + title: msg.title, + post: msg.post, + ..Default::default() + }; + ctx.render().map_err(|e| e.into()) + } +} + +/// Message used to render post editing page. +pub struct EditPostPage { + pub id: i32, + pub post: String, +} +message!(EditPostPage, Result<String>); + +impl Handler<EditPostPage> for Renderer { + type Result = Result<String>; + + fn handle(&mut self, msg: EditPostPage, _: &mut Self::Context) -> Self::Result { + let ctx = FormTemplate { + mode: EditingMode::EditPost, + id: Some(msg.id), + post: Some(msg.post), + ..Default::default() + }; + + ctx.render().map_err(|e| e.into()) + } +} + +/// Message used to render search results +#[derive(Template)] +#[template(path = "search.html")] +pub struct SearchResultPage { + pub query: String, + pub results: Vec<SearchResult>, +} +message!(SearchResultPage, Result<String>); + +impl Handler<SearchResultPage> for Renderer { + type Result = Result<String>; + + fn handle(&mut self, msg: SearchResultPage, _: &mut Self::Context) -> Self::Result { + msg.render().map_err(|e| e.into()) + } +} + +// TODO: actor-free implementation below + +/// Render the index page for the given thread list. +pub fn index_page(threads: Vec<ThreadIndex>) -> Result<String> { + let threads: Vec<IndexThread> = threads + .into_iter() + .map(|thread| IndexThread { + id: thread.thread_id, + title: thread.title, // escape_html(&thread.title), + sticky: thread.sticky, + closed: thread.closed, + posted: FormattedDate(thread.posted), + author_name: thread.thread_author, + post_author: thread.post_author, + }) + .collect(); + + let tpl = IndexPageTemplate { threads }; + tpl.render().map_err(|e| e.into()) +} + +// Render the page of a given thread. +pub fn thread_page(user: i32, thread: Thread, posts: Vec<SimplePost>) -> Result<String> { + let posts = posts.into_iter().map(|post| { + let editable = user != 1 && post.user_id == user; + + let comrak = ComrakOptions::default(); // TODO(tazjin): cheddar + RenderablePost { + id: post.id, + body: markdown_to_html(&post.body, &comrak), + posted: FormattedDate(post.posted), + author_name: post.author_name.clone(), + author_gravatar: md5_hex(post.author_email.as_bytes()), + editable, + } + }).collect(); + + let renderable = RenderableThreadPage { + posts, + closed: thread.closed, + id: thread.id, + title: thread.title, + }; + + Ok(renderable.render()?) +} diff --git a/web/converse/src/schema.rs b/web/converse/src/schema.rs new file mode 100644 index 000000000000..7de6d13668c2 --- /dev/null +++ b/web/converse/src/schema.rs @@ -0,0 +1,88 @@ +// Copyright (C) 2018-2021 Vincent Ambo <tazjin@tvl.su> +// +// This file is part of Converse. +// +// This program is free software: you can redistribute it and/or +// modify it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see +// <https://www.gnu.org/licenses/>. + +table! { + posts (id) { + id -> Int4, + thread_id -> Int4, + body -> Text, + posted -> Timestamptz, + user_id -> Int4, + } +} + +table! { + threads (id) { + id -> Int4, + title -> Varchar, + posted -> Timestamptz, + sticky -> Bool, + user_id -> Int4, + closed -> Bool, + } +} + +table! { + users (id) { + id -> Int4, + email -> Varchar, + name -> Varchar, + admin -> Bool, + } +} + +// Note: Manually inserted as print-schema does not add views. +table! { + simple_posts (id) { + id -> Int4, + thread_id -> Int4, + body -> Text, + posted -> Timestamptz, + user_id -> Int4, + closed -> Bool, + author_name -> Text, + author_email -> Text, + } +} + +// Note: Manually inserted as print-schema does not add views. +table! { + thread_index (thread_id) { + thread_id -> Int4, + title -> Text, + thread_author -> Text, + created -> Timestamptz, + sticky -> Bool, + closed -> Bool, + post_id -> Int4, + post_author -> Text, + posted -> Timestamptz, + } +} + +joinable!(posts -> threads (thread_id)); +joinable!(posts -> users (user_id)); +joinable!(threads -> users (user_id)); +joinable!(simple_posts -> threads (thread_id)); + +allow_tables_to_appear_in_same_query!( + posts, + threads, + users, + simple_posts, +); diff --git a/web/converse/static/highlight.css b/web/converse/static/highlight.css new file mode 100644 index 000000000000..791932b87eaf --- /dev/null +++ b/web/converse/static/highlight.css @@ -0,0 +1,99 @@ +/* + +github.com style (c) Vasily Polovnyov <vast@whiteants.net> + +*/ + +.hljs { + display: block; + overflow-x: auto; + padding: 0.5em; + color: #333; + background: #f8f8f8; +} + +.hljs-comment, +.hljs-quote { + color: #998; + font-style: italic; +} + +.hljs-keyword, +.hljs-selector-tag, +.hljs-subst { + color: #333; + font-weight: bold; +} + +.hljs-number, +.hljs-literal, +.hljs-variable, +.hljs-template-variable, +.hljs-tag .hljs-attr { + color: #008080; +} + +.hljs-string, +.hljs-doctag { + color: #d14; +} + +.hljs-title, +.hljs-section, +.hljs-selector-id { + color: #900; + font-weight: bold; +} + +.hljs-subst { + font-weight: normal; +} + +.hljs-type, +.hljs-class .hljs-title { + color: #458; + font-weight: bold; +} + +.hljs-tag, +.hljs-name, +.hljs-attribute { + color: #000080; + font-weight: normal; +} + +.hljs-regexp, +.hljs-link { + color: #009926; +} + +.hljs-symbol, +.hljs-bullet { + color: #990073; +} + +.hljs-built_in, +.hljs-builtin-name { + color: #0086b3; +} + +.hljs-meta { + color: #999; + font-weight: bold; +} + +.hljs-deletion { + background: #fdd; +} + +.hljs-addition { + background: #dfd; +} + +.hljs-emphasis { + font-style: italic; +} + +.hljs-strong { + font-weight: bold; +} diff --git a/web/converse/static/highlight.js b/web/converse/static/highlight.js new file mode 100644 index 000000000000..9ca7d97ba31d --- /dev/null +++ b/web/converse/static/highlight.js @@ -0,0 +1,2 @@ +/*! highlight.js v9.12.0 | BSD3 License | git.io/hljslicense */ +!function(e){var n="object"==typeof window&&window||"object"==typeof self&&self;"undefined"!=typeof exports?e(exports):n&&(n.hljs=e({}),"function"==typeof define&&define.amd&&define([],function(){return n.hljs}))}(function(e){function n(e){return e.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">")}function t(e){return e.nodeName.toLowerCase()}function r(e,n){var t=e&&e.exec(n);return t&&0===t.index}function a(e){return k.test(e)}function i(e){var n,t,r,i,o=e.className+" ";if(o+=e.parentNode?e.parentNode.className:"",t=B.exec(o))return w(t[1])?t[1]:"no-highlight";for(o=o.split(/\s+/),n=0,r=o.length;r>n;n++)if(i=o[n],a(i)||w(i))return i}function o(e){var n,t={},r=Array.prototype.slice.call(arguments,1);for(n in e)t[n]=e[n];return r.forEach(function(e){for(n in e)t[n]=e[n]}),t}function u(e){var n=[];return function r(e,a){for(var i=e.firstChild;i;i=i.nextSibling)3===i.nodeType?a+=i.nodeValue.length:1===i.nodeType&&(n.push({event:"start",offset:a,node:i}),a=r(i,a),t(i).match(/br|hr|img|input/)||n.push({event:"stop",offset:a,node:i}));return a}(e,0),n}function c(e,r,a){function i(){return e.length&&r.length?e[0].offset!==r[0].offset?e[0].offset<r[0].offset?e:r:"start"===r[0].event?e:r:e.length?e:r}function o(e){function r(e){return" "+e.nodeName+'="'+n(e.value).replace('"',""")+'"'}s+="<"+t(e)+E.map.call(e.attributes,r).join("")+">"}function u(e){s+="</"+t(e)+">"}function c(e){("start"===e.event?o:u)(e.node)}for(var l=0,s="",f=[];e.length||r.length;){var g=i();if(s+=n(a.substring(l,g[0].offset)),l=g[0].offset,g===e){f.reverse().forEach(u);do c(g.splice(0,1)[0]),g=i();while(g===e&&g.length&&g[0].offset===l);f.reverse().forEach(o)}else"start"===g[0].event?f.push(g[0].node):f.pop(),c(g.splice(0,1)[0])}return s+n(a.substr(l))}function l(e){return e.v&&!e.cached_variants&&(e.cached_variants=e.v.map(function(n){return o(e,{v:null},n)})),e.cached_variants||e.eW&&[o(e)]||[e]}function s(e){function n(e){return e&&e.source||e}function t(t,r){return new RegExp(n(t),"m"+(e.cI?"i":"")+(r?"g":""))}function r(a,i){if(!a.compiled){if(a.compiled=!0,a.k=a.k||a.bK,a.k){var o={},u=function(n,t){e.cI&&(t=t.toLowerCase()),t.split(" ").forEach(function(e){var t=e.split("|");o[t[0]]=[n,t[1]?Number(t[1]):1]})};"string"==typeof a.k?u("keyword",a.k):x(a.k).forEach(function(e){u(e,a.k[e])}),a.k=o}a.lR=t(a.l||/\w+/,!0),i&&(a.bK&&(a.b="\\b("+a.bK.split(" ").join("|")+")\\b"),a.b||(a.b=/\B|\b/),a.bR=t(a.b),a.e||a.eW||(a.e=/\B|\b/),a.e&&(a.eR=t(a.e)),a.tE=n(a.e)||"",a.eW&&i.tE&&(a.tE+=(a.e?"|":"")+i.tE)),a.i&&(a.iR=t(a.i)),null==a.r&&(a.r=1),a.c||(a.c=[]),a.c=Array.prototype.concat.apply([],a.c.map(function(e){return l("self"===e?a:e)})),a.c.forEach(function(e){r(e,a)}),a.starts&&r(a.starts,i);var c=a.c.map(function(e){return e.bK?"\\.?("+e.b+")\\.?":e.b}).concat([a.tE,a.i]).map(n).filter(Boolean);a.t=c.length?t(c.join("|"),!0):{exec:function(){return null}}}}r(e)}function f(e,t,a,i){function o(e,n){var t,a;for(t=0,a=n.c.length;a>t;t++)if(r(n.c[t].bR,e))return n.c[t]}function u(e,n){if(r(e.eR,n)){for(;e.endsParent&&e.parent;)e=e.parent;return e}return e.eW?u(e.parent,n):void 0}function c(e,n){return!a&&r(n.iR,e)}function l(e,n){var t=N.cI?n[0].toLowerCase():n[0];return e.k.hasOwnProperty(t)&&e.k[t]}function p(e,n,t,r){var a=r?"":I.classPrefix,i='<span class="'+a,o=t?"":C;return i+=e+'">',i+n+o}function h(){var e,t,r,a;if(!E.k)return n(k);for(a="",t=0,E.lR.lastIndex=0,r=E.lR.exec(k);r;)a+=n(k.substring(t,r.index)),e=l(E,r),e?(B+=e[1],a+=p(e[0],n(r[0]))):a+=n(r[0]),t=E.lR.lastIndex,r=E.lR.exec(k);return a+n(k.substr(t))}function d(){var e="string"==typeof E.sL;if(e&&!y[E.sL])return n(k);var t=e?f(E.sL,k,!0,x[E.sL]):g(k,E.sL.length?E.sL:void 0);return E.r>0&&(B+=t.r),e&&(x[E.sL]=t.top),p(t.language,t.value,!1,!0)}function b(){L+=null!=E.sL?d():h(),k=""}function v(e){L+=e.cN?p(e.cN,"",!0):"",E=Object.create(e,{parent:{value:E}})}function m(e,n){if(k+=e,null==n)return b(),0;var t=o(n,E);if(t)return t.skip?k+=n:(t.eB&&(k+=n),b(),t.rB||t.eB||(k=n)),v(t,n),t.rB?0:n.length;var r=u(E,n);if(r){var a=E;a.skip?k+=n:(a.rE||a.eE||(k+=n),b(),a.eE&&(k=n));do E.cN&&(L+=C),E.skip||(B+=E.r),E=E.parent;while(E!==r.parent);return r.starts&&v(r.starts,""),a.rE?0:n.length}if(c(n,E))throw new Error('Illegal lexeme "'+n+'" for mode "'+(E.cN||"<unnamed>")+'"');return k+=n,n.length||1}var N=w(e);if(!N)throw new Error('Unknown language: "'+e+'"');s(N);var R,E=i||N,x={},L="";for(R=E;R!==N;R=R.parent)R.cN&&(L=p(R.cN,"",!0)+L);var k="",B=0;try{for(var M,j,O=0;;){if(E.t.lastIndex=O,M=E.t.exec(t),!M)break;j=m(t.substring(O,M.index),M[0]),O=M.index+j}for(m(t.substr(O)),R=E;R.parent;R=R.parent)R.cN&&(L+=C);return{r:B,value:L,language:e,top:E}}catch(T){if(T.message&&-1!==T.message.indexOf("Illegal"))return{r:0,value:n(t)};throw T}}function g(e,t){t=t||I.languages||x(y);var r={r:0,value:n(e)},a=r;return t.filter(w).forEach(function(n){var t=f(n,e,!1);t.language=n,t.r>a.r&&(a=t),t.r>r.r&&(a=r,r=t)}),a.language&&(r.second_best=a),r}function p(e){return I.tabReplace||I.useBR?e.replace(M,function(e,n){return I.useBR&&"\n"===e?"<br>":I.tabReplace?n.replace(/\t/g,I.tabReplace):""}):e}function h(e,n,t){var r=n?L[n]:t,a=[e.trim()];return e.match(/\bhljs\b/)||a.push("hljs"),-1===e.indexOf(r)&&a.push(r),a.join(" ").trim()}function d(e){var n,t,r,o,l,s=i(e);a(s)||(I.useBR?(n=document.createElementNS("http://www.w3.org/1999/xhtml","div"),n.innerHTML=e.innerHTML.replace(/\n/g,"").replace(/<br[ \/]*>/g,"\n")):n=e,l=n.textContent,r=s?f(s,l,!0):g(l),t=u(n),t.length&&(o=document.createElementNS("http://www.w3.org/1999/xhtml","div"),o.innerHTML=r.value,r.value=c(t,u(o),l)),r.value=p(r.value),e.innerHTML=r.value,e.className=h(e.className,s,r.language),e.result={language:r.language,re:r.r},r.second_best&&(e.second_best={language:r.second_best.language,re:r.second_best.r}))}function b(e){I=o(I,e)}function v(){if(!v.called){v.called=!0;var e=document.querySelectorAll("pre code");E.forEach.call(e,d)}}function m(){addEventListener("DOMContentLoaded",v,!1),addEventListener("load",v,!1)}function N(n,t){var r=y[n]=t(e);r.aliases&&r.aliases.forEach(function(e){L[e]=n})}function R(){return x(y)}function w(e){return e=(e||"").toLowerCase(),y[e]||y[L[e]]}var E=[],x=Object.keys,y={},L={},k=/^(no-?highlight|plain|text)$/i,B=/\blang(?:uage)?-([\w-]+)\b/i,M=/((^(<[^>]+>|\t|)+|(?:\n)))/gm,C="</span>",I={classPrefix:"hljs-",tabReplace:null,useBR:!1,languages:void 0};return e.highlight=f,e.highlightAuto=g,e.fixMarkup=p,e.highlightBlock=d,e.configure=b,e.initHighlighting=v,e.initHighlightingOnLoad=m,e.registerLanguage=N,e.listLanguages=R,e.getLanguage=w,e.inherit=o,e.IR="[a-zA-Z]\\w*",e.UIR="[a-zA-Z_]\\w*",e.NR="\\b\\d+(\\.\\d+)?",e.CNR="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",e.BNR="\\b(0b[01]+)",e.RSR="!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~",e.BE={b:"\\\\[\\s\\S]",r:0},e.ASM={cN:"string",b:"'",e:"'",i:"\\n",c:[e.BE]},e.QSM={cN:"string",b:'"',e:'"',i:"\\n",c:[e.BE]},e.PWM={b:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/},e.C=function(n,t,r){var a=e.inherit({cN:"comment",b:n,e:t,c:[]},r||{});return a.c.push(e.PWM),a.c.push({cN:"doctag",b:"(?:TODO|FIXME|NOTE|BUG|XXX):",r:0}),a},e.CLCM=e.C("//","$"),e.CBCM=e.C("/\\*","\\*/"),e.HCM=e.C("#","$"),e.NM={cN:"number",b:e.NR,r:0},e.CNM={cN:"number",b:e.CNR,r:0},e.BNM={cN:"number",b:e.BNR,r:0},e.CSSNM={cN:"number",b:e.NR+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?",r:0},e.RM={cN:"regexp",b:/\//,e:/\/[gimuy]*/,i:/\n/,c:[e.BE,{b:/\[/,e:/\]/,r:0,c:[e.BE]}]},e.TM={cN:"title",b:e.IR,r:0},e.UTM={cN:"title",b:e.UIR,r:0},e.METHOD_GUARD={b:"\\.\\s*"+e.UIR,r:0},e});hljs.registerLanguage("erlang",function(e){var r="[a-z'][a-zA-Z0-9_']*",c="("+r+":"+r+"|"+r+")",b={keyword:"after and andalso|10 band begin bnot bor bsl bzr bxor case catch cond div end fun if let not of orelse|10 query receive rem try when xor",literal:"false true"},i=e.C("%","$"),n={cN:"number",b:"\\b(\\d+#[a-fA-F0-9]+|\\d+(\\.\\d+)?([eE][-+]?\\d+)?)",r:0},a={b:"fun\\s+"+r+"/\\d+"},d={b:c+"\\(",e:"\\)",rB:!0,r:0,c:[{b:c,r:0},{b:"\\(",e:"\\)",eW:!0,rE:!0,r:0}]},o={b:"{",e:"}",r:0},t={b:"\\b_([A-Z][A-Za-z0-9_]*)?",r:0},f={b:"[A-Z][a-zA-Z0-9_]*",r:0},l={b:"#"+e.UIR,r:0,rB:!0,c:[{b:"#"+e.UIR,r:0},{b:"{",e:"}",r:0}]},s={bK:"fun receive if try case",e:"end",k:b};s.c=[i,a,e.inherit(e.ASM,{cN:""}),s,d,e.QSM,n,o,t,f,l];var u=[i,a,s,d,e.QSM,n,o,t,f,l];d.c[1].c=u,o.c=u,l.c[1].c=u;var h={cN:"params",b:"\\(",e:"\\)",c:u};return{aliases:["erl"],k:b,i:"(</|\\*=|\\+=|-=|/\\*|\\*/|\\(\\*|\\*\\))",c:[{cN:"function",b:"^"+r+"\\s*\\(",e:"->",rB:!0,i:"\\(|#|//|/\\*|\\\\|:|;",c:[h,e.inherit(e.TM,{b:r})],starts:{e:";|\\.",k:b,c:u}},i,{b:"^-",e:"\\.",r:0,eE:!0,rB:!0,l:"-"+e.IR,k:"-module -record -undef -export -ifdef -ifndef -author -copyright -doc -vsn -import -include -include_lib -compile -define -else -endif -file -behaviour -behavior -spec",c:[h]},n,e.QSM,l,t,f,o,{b:/\.$/}]}});hljs.registerLanguage("cs",function(e){var i={keyword:"abstract as base bool break byte case catch char checked const continue decimal default delegate do double enum event explicit extern finally fixed float for foreach goto if implicit in int interface internal is lock long nameof object operator out override params private protected public readonly ref sbyte sealed short sizeof stackalloc static string struct switch this try typeof uint ulong unchecked unsafe ushort using virtual void volatile while add alias ascending async await by descending dynamic equals from get global group into join let on orderby partial remove select set value var where yield",literal:"null false true"},t={cN:"string",b:'@"',e:'"',c:[{b:'""'}]},r=e.inherit(t,{i:/\n/}),a={cN:"subst",b:"{",e:"}",k:i},c=e.inherit(a,{i:/\n/}),n={cN:"string",b:/\$"/,e:'"',i:/\n/,c:[{b:"{{"},{b:"}}"},e.BE,c]},s={cN:"string",b:/\$@"/,e:'"',c:[{b:"{{"},{b:"}}"},{b:'""'},a]},o=e.inherit(s,{i:/\n/,c:[{b:"{{"},{b:"}}"},{b:'""'},c]});a.c=[s,n,t,e.ASM,e.QSM,e.CNM,e.CBCM],c.c=[o,n,r,e.ASM,e.QSM,e.CNM,e.inherit(e.CBCM,{i:/\n/})];var l={v:[s,n,t,e.ASM,e.QSM]},b=e.IR+"(<"+e.IR+"(\\s*,\\s*"+e.IR+")*>)?(\\[\\])?";return{aliases:["csharp"],k:i,i:/::/,c:[e.C("///","$",{rB:!0,c:[{cN:"doctag",v:[{b:"///",r:0},{b:"<!--|-->"},{b:"</?",e:">"}]}]}),e.CLCM,e.CBCM,{cN:"meta",b:"#",e:"$",k:{"meta-keyword":"if else elif endif define undef warning error line region endregion pragma checksum"}},l,e.CNM,{bK:"class interface",e:/[{;=]/,i:/[^\s:]/,c:[e.TM,e.CLCM,e.CBCM]},{bK:"namespace",e:/[{;=]/,i:/[^\s:]/,c:[e.inherit(e.TM,{b:"[a-zA-Z](\\.?\\w)*"}),e.CLCM,e.CBCM]},{cN:"meta",b:"^\\s*\\[",eB:!0,e:"\\]",eE:!0,c:[{cN:"meta-string",b:/"/,e:/"/}]},{bK:"new return throw await else",r:0},{cN:"function",b:"("+b+"\\s+)+"+e.IR+"\\s*\\(",rB:!0,e:/[{;=]/,eE:!0,k:i,c:[{b:e.IR+"\\s*\\(",rB:!0,c:[e.TM],r:0},{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:i,r:0,c:[l,e.CNM,e.CBCM]},e.CLCM,e.CBCM]}]}});hljs.registerLanguage("json",function(e){var i={literal:"true false null"},n=[e.QSM,e.CNM],r={e:",",eW:!0,eE:!0,c:n,k:i},t={b:"{",e:"}",c:[{cN:"attr",b:/"/,e:/"/,c:[e.BE],i:"\\n"},e.inherit(r,{b:/:/})],i:"\\S"},c={b:"\\[",e:"\\]",c:[e.inherit(r)],i:"\\S"};return n.splice(n.length,0,t,c),{c:n,k:i,i:"\\S"}});hljs.registerLanguage("swift",function(e){var i={keyword:"__COLUMN__ __FILE__ __FUNCTION__ __LINE__ as as! as? associativity break case catch class continue convenience default defer deinit didSet do dynamic dynamicType else enum extension fallthrough false fileprivate final for func get guard if import in indirect infix init inout internal is lazy left let mutating nil none nonmutating open operator optional override postfix precedence prefix private protocol Protocol public repeat required rethrows return right self Self set static struct subscript super switch throw throws true try try! try? Type typealias unowned var weak where while willSet",literal:"true false nil",built_in:"abs advance alignof alignofValue anyGenerator assert assertionFailure bridgeFromObjectiveC bridgeFromObjectiveCUnconditional bridgeToObjectiveC bridgeToObjectiveCUnconditional c contains count countElements countLeadingZeros debugPrint debugPrintln distance dropFirst dropLast dump encodeBitsAsWords enumerate equal fatalError filter find getBridgedObjectiveCType getVaList indices insertionSort isBridgedToObjectiveC isBridgedVerbatimToObjectiveC isUniquelyReferenced isUniquelyReferencedNonObjC join lazy lexicographicalCompare map max maxElement min minElement numericCast overlaps partition posix precondition preconditionFailure print println quickSort readLine reduce reflect reinterpretCast reverse roundUpToAlignment sizeof sizeofValue sort split startsWith stride strideof strideofValue swap toString transcode underestimateCount unsafeAddressOf unsafeBitCast unsafeDowncast unsafeUnwrap unsafeReflect withExtendedLifetime withObjectAtPlusZero withUnsafePointer withUnsafePointerToObject withUnsafeMutablePointer withUnsafeMutablePointers withUnsafePointer withUnsafePointers withVaList zip"},t={cN:"type",b:"\\b[A-Z][\\wÀ-ʸ']*",r:0},n=e.C("/\\*","\\*/",{c:["self"]}),r={cN:"subst",b:/\\\(/,e:"\\)",k:i,c:[]},a={cN:"number",b:"\\b([\\d_]+(\\.[\\deE_]+)?|0x[a-fA-F0-9_]+(\\.[a-fA-F0-9p_]+)?|0b[01_]+|0o[0-7_]+)\\b",r:0},o=e.inherit(e.QSM,{c:[r,e.BE]});return r.c=[a],{k:i,c:[o,e.CLCM,n,t,a,{cN:"function",bK:"func",e:"{",eE:!0,c:[e.inherit(e.TM,{b:/[A-Za-z$_][0-9A-Za-z$_]*/}),{b:/</,e:/>/},{cN:"params",b:/\(/,e:/\)/,endsParent:!0,k:i,c:["self",a,o,e.CBCM,{b:":"}],i:/["']/}],i:/\[|%/},{cN:"class",bK:"struct protocol class extension enum",k:i,e:"\\{",eE:!0,c:[e.inherit(e.TM,{b:/[A-Za-z$_][\u00C0-\u02B80-9A-Za-z$_]*/})]},{cN:"meta",b:"(@warn_unused_result|@exported|@lazy|@noescape|@NSCopying|@NSManaged|@objc|@convention|@required|@noreturn|@IBAction|@IBDesignable|@IBInspectable|@IBOutlet|@infix|@prefix|@postfix|@autoclosure|@testable|@available|@nonobjc|@NSApplicationMain|@UIApplicationMain)"},{bK:"import",e:/$/,c:[e.CLCM,n]}]}});hljs.registerLanguage("elixir",function(e){var r="[a-zA-Z_][a-zA-Z0-9_]*(\\!|\\?)?",n="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",b="and false then defined module in return redo retry end for true self when next until do begin unless nil break not case cond alias while ensure or include use alias fn quote",c={cN:"subst",b:"#\\{",e:"}",l:r,k:b},a={cN:"string",c:[e.BE,c],v:[{b:/'/,e:/'/},{b:/"/,e:/"/}]},i={cN:"function",bK:"def defp defmacro",e:/\B\b/,c:[e.inherit(e.TM,{b:r,endsParent:!0})]},l=e.inherit(i,{cN:"class",bK:"defimpl defmodule defprotocol defrecord",e:/\bdo\b|$|;/}),s=[a,e.HCM,l,i,{cN:"symbol",b:":(?!\\s)",c:[a,{b:n}],r:0},{cN:"symbol",b:r+":",r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{cN:"variable",b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{b:"->"},{b:"("+e.RSR+")\\s*",c:[e.HCM,{cN:"regexp",i:"\\n",c:[e.BE,c],v:[{b:"/",e:"/[a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}],r:0}];return c.c=s,{l:r,k:b,c:s}});hljs.registerLanguage("bash",function(e){var t={cN:"variable",v:[{b:/\$[\w\d#@][\w\d_]*/},{b:/\$\{(.*?)}/}]},s={cN:"string",b:/"/,e:/"/,c:[e.BE,t,{cN:"variable",b:/\$\(/,e:/\)/,c:[e.BE]}]},a={cN:"string",b:/'/,e:/'/};return{aliases:["sh","zsh"],l:/\b-?[a-z\._]+\b/,k:{keyword:"if then else elif fi for while in do done case esac function",literal:"true false",built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp",_:"-ne -eq -lt -gt -f -d -e -s -l -a"},c:[{cN:"meta",b:/^#![^\n]+sh\s*$/,r:10},{cN:"function",b:/\w[\w\d_]*\s*\(\s*\)\s*\{/,rB:!0,c:[e.inherit(e.TM,{b:/\w[\w\d_]*/})],r:0},e.HCM,s,a,t]}});hljs.registerLanguage("erlang-repl",function(e){return{k:{built_in:"spawn spawn_link self",keyword:"after and andalso|10 band begin bnot bor bsl bsr bxor case catch cond div end fun if let not of or orelse|10 query receive rem try when xor"},c:[{cN:"meta",b:"^[0-9]+> ",r:10},e.C("%","$"),{cN:"number",b:"\\b(\\d+#[a-fA-F0-9]+|\\d+(\\.\\d+)?([eE][-+]?\\d+)?)",r:0},e.ASM,e.QSM,{b:"\\?(::)?([A-Z]\\w*(::)?)+"},{b:"->"},{b:"ok"},{b:"!"},{b:"(\\b[a-z'][a-zA-Z0-9_']*:[a-z'][a-zA-Z0-9_']*)|(\\b[a-z'][a-zA-Z0-9_']*)",r:0},{b:"[A-Z][a-zA-Z0-9_']*",r:0}]}});hljs.registerLanguage("elm",function(e){var i={v:[e.C("--","$"),e.C("{-","-}",{c:["self"]})]},t={cN:"type",b:"\\b[A-Z][\\w']*",r:0},c={b:"\\(",e:"\\)",i:'"',c:[{cN:"type",b:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},i]},n={b:"{",e:"}",c:c.c};return{k:"let in if then else case of where module import exposing type alias as infix infixl infixr port effect command subscription",c:[{bK:"port effect module",e:"exposing",k:"port effect module where command subscription exposing",c:[c,i],i:"\\W\\.|;"},{b:"import",e:"$",k:"import as exposing",c:[c,i],i:"\\W\\.|;"},{b:"type",e:"$",k:"type alias",c:[t,c,n,i]},{bK:"infix infixl infixr",e:"$",c:[e.CNM,i]},{b:"port",e:"$",k:"port",c:[i]},e.QSM,e.CNM,t,e.inherit(e.TM,{b:"^[_a-z][\\w']*"}),i,{b:"->|<-"}],i:/;/}});hljs.registerLanguage("clojure",function(e){var t={"builtin-name":"def defonce cond apply if-not if-let if not not= = < > <= >= == + / * - rem quot neg? pos? delay? symbol? keyword? true? false? integer? empty? coll? list? set? ifn? fn? associative? sequential? sorted? counted? reversible? number? decimal? class? distinct? isa? float? rational? reduced? ratio? odd? even? char? seq? vector? string? map? nil? contains? zero? instance? not-every? not-any? libspec? -> ->> .. . inc compare do dotimes mapcat take remove take-while drop letfn drop-last take-last drop-while while intern condp case reduced cycle split-at split-with repeat replicate iterate range merge zipmap declare line-seq sort comparator sort-by dorun doall nthnext nthrest partition eval doseq await await-for let agent atom send send-off release-pending-sends add-watch mapv filterv remove-watch agent-error restart-agent set-error-handler error-handler set-error-mode! error-mode shutdown-agents quote var fn loop recur throw try monitor-enter monitor-exit defmacro defn defn- macroexpand macroexpand-1 for dosync and or when when-not when-let comp juxt partial sequence memoize constantly complement identity assert peek pop doto proxy defstruct first rest cons defprotocol cast coll deftype defrecord last butlast sigs reify second ffirst fnext nfirst nnext defmulti defmethod meta with-meta ns in-ns create-ns import refer keys select-keys vals key val rseq name namespace promise into transient persistent! conj! assoc! dissoc! pop! disj! use class type num float double short byte boolean bigint biginteger bigdec print-method print-dup throw-if printf format load compile get-in update-in pr pr-on newline flush read slurp read-line subvec with-open memfn time re-find re-groups rand-int rand mod locking assert-valid-fdecl alias resolve ref deref refset swap! reset! set-validator! compare-and-set! alter-meta! reset-meta! commute get-validator alter ref-set ref-history-count ref-min-history ref-max-history ensure sync io! new next conj set! to-array future future-call into-array aset gen-class reduce map filter find empty hash-map hash-set sorted-map sorted-map-by sorted-set sorted-set-by vec vector seq flatten reverse assoc dissoc list disj get union difference intersection extend extend-type extend-protocol int nth delay count concat chunk chunk-buffer chunk-append chunk-first chunk-rest max min dec unchecked-inc-int unchecked-inc unchecked-dec-inc unchecked-dec unchecked-negate unchecked-add-int unchecked-add unchecked-subtract-int unchecked-subtract chunk-next chunk-cons chunked-seq? prn vary-meta lazy-seq spread list* str find-keyword keyword symbol gensym force rationalize"},r="a-zA-Z_\\-!.?+*=<>&#'",n="["+r+"]["+r+"0-9/;:]*",a="[-+]?\\d+(\\.\\d+)?",o={b:n,r:0},s={cN:"number",b:a,r:0},c=e.inherit(e.QSM,{i:null}),i=e.C(";","$",{r:0}),d={cN:"literal",b:/\b(true|false|nil)\b/},l={b:"[\\[\\{]",e:"[\\]\\}]"},m={cN:"comment",b:"\\^"+n},p=e.C("\\^\\{","\\}"),u={cN:"symbol",b:"[:]{1,2}"+n},f={b:"\\(",e:"\\)"},h={eW:!0,r:0},y={k:t,l:n,cN:"name",b:n,starts:h},b=[f,c,m,p,i,u,l,s,d,o];return f.c=[e.C("comment",""),y,h],h.c=b,l.c=b,p.c=[l],{aliases:["clj"],i:/\S/,c:[f,c,m,p,i,u,l,s,d]}});hljs.registerLanguage("dns",function(d){return{aliases:["bind","zone"],k:{keyword:"IN A AAAA AFSDB APL CAA CDNSKEY CDS CERT CNAME DHCID DLV DNAME DNSKEY DS HIP IPSECKEY KEY KX LOC MX NAPTR NS NSEC NSEC3 NSEC3PARAM PTR RRSIG RP SIG SOA SRV SSHFP TA TKEY TLSA TSIG TXT"},c:[d.C(";","$",{r:0}),{cN:"meta",b:/^\$(TTL|GENERATE|INCLUDE|ORIGIN)\b/},{cN:"number",b:"((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)(\\.(25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]?\\d)){3}))|:)))\\b"},{cN:"number",b:"((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]).){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\b"},d.inherit(d.NM,{b:/\b\d+[dhwm]?/})]}});hljs.registerLanguage("dockerfile",function(e){return{aliases:["docker"],cI:!0,k:"from maintainer expose env arg user onbuild stopsignal",c:[e.HCM,e.ASM,e.QSM,e.NM,{bK:"run cmd entrypoint volume add copy workdir label healthcheck shell",starts:{e:/[^\\]\n/,sL:"bash"}}],i:"</"}});hljs.registerLanguage("css",function(e){var c="[a-zA-Z-][a-zA-Z0-9_-]*",t={b:/[A-Z\_\.\-]+\s*:/,rB:!0,e:";",eW:!0,c:[{cN:"attribute",b:/\S/,e:":",eE:!0,starts:{eW:!0,eE:!0,c:[{b:/[\w-]+\(/,rB:!0,c:[{cN:"built_in",b:/[\w-]+/},{b:/\(/,e:/\)/,c:[e.ASM,e.QSM]}]},e.CSSNM,e.QSM,e.ASM,e.CBCM,{cN:"number",b:"#[0-9A-Fa-f]+"},{cN:"meta",b:"!important"}]}}]};return{cI:!0,i:/[=\/|'\$]/,c:[e.CBCM,{cN:"selector-id",b:/#[A-Za-z0-9_-]+/},{cN:"selector-class",b:/\.[A-Za-z0-9_-]+/},{cN:"selector-attr",b:/\[/,e:/\]/,i:"$"},{cN:"selector-pseudo",b:/:(:)?[a-zA-Z0-9\_\-\+\(\)"'.]+/},{b:"@(font-face|page)",l:"[a-z-]+",k:"font-face page"},{b:"@",e:"[{;]",i:/:/,c:[{cN:"keyword",b:/\w+/},{b:/\s/,eW:!0,eE:!0,r:0,c:[e.ASM,e.QSM,e.CSSNM]}]},{cN:"selector-tag",b:c,r:0},{b:"{",e:"}",i:/\S/,c:[e.CBCM,t]}]}});hljs.registerLanguage("xml",function(s){var e="[A-Za-z0-9\\._:-]+",t={eW:!0,i:/</,r:0,c:[{cN:"attr",b:e,r:0},{b:/=\s*/,r:0,c:[{cN:"string",endsParent:!0,v:[{b:/"/,e:/"/},{b:/'/,e:/'/},{b:/[^\s"'=<>`]+/}]}]}]};return{aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist"],cI:!0,c:[{cN:"meta",b:"<!DOCTYPE",e:">",r:10,c:[{b:"\\[",e:"\\]"}]},s.C("<!--","-->",{r:10}),{b:"<\\!\\[CDATA\\[",e:"\\]\\]>",r:10},{b:/<\?(php)?/,e:/\?>/,sL:"php",c:[{b:"/\\*",e:"\\*/",skip:!0}]},{cN:"tag",b:"<style(?=\\s|>|$)",e:">",k:{name:"style"},c:[t],starts:{e:"</style>",rE:!0,sL:["css","xml"]}},{cN:"tag",b:"<script(?=\\s|>|$)",e:">",k:{name:"script"},c:[t],starts:{e:"</script>",rE:!0,sL:["actionscript","javascript","handlebars","xml"]}},{cN:"meta",v:[{b:/<\?xml/,e:/\?>/,r:10},{b:/<\?\w+/,e:/\?>/}]},{cN:"tag",b:"</?",e:"/?>",c:[{cN:"name",b:/[^\/><\s]+/,r:0},t]}]}});hljs.registerLanguage("nginx",function(e){var r={cN:"variable",v:[{b:/\$\d+/},{b:/\$\{/,e:/}/},{b:"[\\$\\@]"+e.UIR}]},b={eW:!0,l:"[a-z/_]+",k:{literal:"on off yes no true false none blocked debug info notice warn error crit select break last permanent redirect kqueue rtsig epoll poll /dev/poll"},r:0,i:"=>",c:[e.HCM,{cN:"string",c:[e.BE,r],v:[{b:/"/,e:/"/},{b:/'/,e:/'/}]},{b:"([a-z]+):/",e:"\\s",eW:!0,eE:!0,c:[r]},{cN:"regexp",c:[e.BE,r],v:[{b:"\\s\\^",e:"\\s|{|;",rE:!0},{b:"~\\*?\\s+",e:"\\s|{|;",rE:!0},{b:"\\*(\\.[a-z\\-]+)+"},{b:"([a-z\\-]+\\.)+\\*"}]},{cN:"number",b:"\\b\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}(:\\d{1,5})?\\b"},{cN:"number",b:"\\b\\d+[kKmMgGdshdwy]*\\b",r:0},r]};return{aliases:["nginxconf"],c:[e.HCM,{b:e.UIR+"\\s+{",rB:!0,e:"{",c:[{cN:"section",b:e.UIR}],r:0},{b:e.UIR+"\\s",e:";|{",rB:!0,c:[{cN:"attribute",b:e.UIR,starts:b}],r:0}],i:"[^\\s\\}]"}});hljs.registerLanguage("ruby",function(e){var b="[a-zA-Z_]\\w*[!?=]?|[-+~]\\@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?",r={keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor",literal:"true false nil"},c={cN:"doctag",b:"@[A-Za-z]+"},a={b:"#<",e:">"},s=[e.C("#","$",{c:[c]}),e.C("^\\=begin","^\\=end",{c:[c],r:10}),e.C("^__END__","\\n$")],n={cN:"subst",b:"#\\{",e:"}",k:r},t={cN:"string",c:[e.BE,n],v:[{b:/'/,e:/'/},{b:/"/,e:/"/},{b:/`/,e:/`/},{b:"%[qQwWx]?\\(",e:"\\)"},{b:"%[qQwWx]?\\[",e:"\\]"},{b:"%[qQwWx]?{",e:"}"},{b:"%[qQwWx]?<",e:">"},{b:"%[qQwWx]?/",e:"/"},{b:"%[qQwWx]?%",e:"%"},{b:"%[qQwWx]?-",e:"-"},{b:"%[qQwWx]?\\|",e:"\\|"},{b:/\B\?(\\\d{1,3}|\\x[A-Fa-f0-9]{1,2}|\\u[A-Fa-f0-9]{4}|\\?\S)\b/},{b:/<<(-?)\w+$/,e:/^\s*\w+$/}]},i={cN:"params",b:"\\(",e:"\\)",endsParent:!0,k:r},d=[t,a,{cN:"class",bK:"class module",e:"$|;",i:/=/,c:[e.inherit(e.TM,{b:"[A-Za-z_]\\w*(::\\w+)*(\\?|\\!)?"}),{b:"<\\s*",c:[{b:"("+e.IR+"::)?"+e.IR}]}].concat(s)},{cN:"function",bK:"def",e:"$|;",c:[e.inherit(e.TM,{b:b}),i].concat(s)},{b:e.IR+"::"},{cN:"symbol",b:e.UIR+"(\\!|\\?)?:",r:0},{cN:"symbol",b:":(?!\\s)",c:[t,{b:b}],r:0},{cN:"number",b:"(\\b0[0-7_]+)|(\\b0x[0-9a-fA-F_]+)|(\\b[1-9][0-9_]*(\\.[0-9_]+)?)|[0_]\\b",r:0},{b:"(\\$\\W)|((\\$|\\@\\@?)(\\w+))"},{cN:"params",b:/\|/,e:/\|/,k:r},{b:"("+e.RSR+"|unless)\\s*",k:"unless",c:[a,{cN:"regexp",c:[e.BE,n],i:/\n/,v:[{b:"/",e:"/[a-z]*"},{b:"%r{",e:"}[a-z]*"},{b:"%r\\(",e:"\\)[a-z]*"},{b:"%r!",e:"![a-z]*"},{b:"%r\\[",e:"\\][a-z]*"}]}].concat(s),r:0}].concat(s);n.c=d,i.c=d;var l="[>?]>",o="[\\w#]+\\(\\w+\\):\\d+:\\d+>",u="(\\w+-)?\\d+\\.\\d+\\.\\d(p\\d+)?[^>]+>",w=[{b:/^\s*=>/,starts:{e:"$",c:d}},{cN:"meta",b:"^("+l+"|"+o+"|"+u+")",starts:{e:"$",c:d}}];return{aliases:["rb","gemspec","podspec","thor","irb"],k:r,i:/\/\*/,c:s.concat(w).concat(d)}});hljs.registerLanguage("lua",function(e){var t="\\[=*\\[",a="\\]=*\\]",r={b:t,e:a,c:["self"]},n=[e.C("--(?!"+t+")","$"),e.C("--"+t,a,{c:[r],r:10})];return{l:e.UIR,k:{literal:"true false nil",keyword:"and break do else elseif end for goto if in local not or repeat return then until while",built_in:"_G _ENV _VERSION __index __newindex __mode __call __metatable __tostring __len __gc __add __sub __mul __div __mod __pow __concat __unm __eq __lt __le assert collectgarbage dofile error getfenv getmetatable ipairs load loadfile loadstringmodule next pairs pcall print rawequal rawget rawset require select setfenvsetmetatable tonumber tostring type unpack xpcall arg selfcoroutine resume yield status wrap create running debug getupvalue debug sethook getmetatable gethook setmetatable setlocal traceback setfenv getinfo setupvalue getlocal getregistry getfenv io lines write close flush open output type read stderr stdin input stdout popen tmpfile math log max acos huge ldexp pi cos tanh pow deg tan cosh sinh random randomseed frexp ceil floor rad abs sqrt modf asin min mod fmod log10 atan2 exp sin atan os exit setlocale date getenv difftime remove time clock tmpname rename execute package preload loadlib loaded loaders cpath config path seeall string sub upper len gfind rep find match char dump gmatch reverse byte format gsub lower table setn insert getn foreachi maxn foreach concat sort remove"},c:n.concat([{cN:"function",bK:"function",e:"\\)",c:[e.inherit(e.TM,{b:"([_a-zA-Z]\\w*\\.)*([_a-zA-Z]\\w*:)?[_a-zA-Z]\\w*"}),{cN:"params",b:"\\(",eW:!0,c:n}].concat(n)},e.CNM,e.ASM,e.QSM,{cN:"string",b:t,e:a,c:[r],r:5}])}});hljs.registerLanguage("shell",function(s){return{aliases:["console"],c:[{cN:"meta",b:"^\\s{0,3}[\\w\\d\\[\\]()@-]*[>%$#]",starts:{e:"$",sL:"bash"}}]}});hljs.registerLanguage("scheme",function(e){var t="[^\\(\\)\\[\\]\\{\\}\",'`;#|\\\\\\s]+",r="(\\-|\\+)?\\d+([./]\\d+)?",a=r+"[+\\-]"+r+"i",i={"builtin-name":"case-lambda call/cc class define-class exit-handler field import inherit init-field interface let*-values let-values let/ec mixin opt-lambda override protect provide public rename require require-for-syntax syntax syntax-case syntax-error unit/sig unless when with-syntax and begin call-with-current-continuation call-with-input-file call-with-output-file case cond define define-syntax delay do dynamic-wind else for-each if lambda let let* let-syntax letrec letrec-syntax map or syntax-rules ' * + , ,@ - ... / ; < <= = => > >= ` abs acos angle append apply asin assoc assq assv atan boolean? caar cadr call-with-input-file call-with-output-file call-with-values car cdddar cddddr cdr ceiling char->integer char-alphabetic? char-ci<=? char-ci<? char-ci=? char-ci>=? char-ci>? char-downcase char-lower-case? char-numeric? char-ready? char-upcase char-upper-case? char-whitespace? char<=? char<? char=? char>=? char>? char? close-input-port close-output-port complex? cons cos current-input-port current-output-port denominator display eof-object? eq? equal? eqv? eval even? exact->inexact exact? exp expt floor force gcd imag-part inexact->exact inexact? input-port? integer->char integer? interaction-environment lcm length list list->string list->vector list-ref list-tail list? load log magnitude make-polar make-rectangular make-string make-vector max member memq memv min modulo negative? newline not null-environment null? number->string number? numerator odd? open-input-file open-output-file output-port? pair? peek-char port? positive? procedure? quasiquote quote quotient rational? rationalize read read-char real-part real? remainder reverse round scheme-report-environment set! set-car! set-cdr! sin sqrt string string->list string->number string->symbol string-append string-ci<=? string-ci<? string-ci=? string-ci>=? string-ci>? string-copy string-fill! string-length string-ref string-set! string<=? string<? string=? string>=? string>? string? substring symbol->string symbol? tan transcript-off transcript-on truncate values vector vector->list vector-fill! vector-length vector-ref vector-set! with-input-from-file with-output-to-file write write-char zero?"},n={cN:"meta",b:"^#!",e:"$"},c={cN:"literal",b:"(#t|#f|#\\\\"+t+"|#\\\\.)"},l={cN:"number",v:[{b:r,r:0},{b:a,r:0},{b:"#b[0-1]+(/[0-1]+)?"},{b:"#o[0-7]+(/[0-7]+)?"},{b:"#x[0-9a-f]+(/[0-9a-f]+)?"}]},s=e.QSM,o=[e.C(";","$",{r:0}),e.C("#\\|","\\|#")],u={b:t,r:0},p={cN:"symbol",b:"'"+t},d={eW:!0,r:0},m={v:[{b:/'/},{b:"`"}],c:[{b:"\\(",e:"\\)",c:["self",c,s,l,u,p]}]},g={cN:"name",b:t,l:t,k:i},h={b:/lambda/,eW:!0,rB:!0,c:[g,{b:/\(/,e:/\)/,endsParent:!0,c:[u]}]},b={v:[{b:"\\(",e:"\\)"},{b:"\\[",e:"\\]"}],c:[h,g,d]};return d.c=[c,l,s,u,p,m,b].concat(o),{i:/\S/,c:[n,l,s,p,m,b].concat(o)}});hljs.registerLanguage("sql",function(e){var t=e.C("--","$");return{cI:!0,i:/[<>{}*#]/,c:[{bK:"begin end start commit rollback savepoint lock alter create drop rename call delete do handler insert load replace select truncate update set show pragma grant merge describe use explain help declare prepare execute deallocate release unlock purge reset change stop analyze cache flush optimize repair kill install uninstall checksum restore check backup revoke comment",e:/;/,eW:!0,l:/[\w\.]+/,k:{keyword:"abort abs absolute acc acce accep accept access accessed accessible account acos action activate add addtime admin administer advanced advise aes_decrypt aes_encrypt after agent aggregate ali alia alias allocate allow alter always analyze ancillary and any anydata anydataset anyschema anytype apply archive archived archivelog are as asc ascii asin assembly assertion associate asynchronous at atan atn2 attr attri attrib attribu attribut attribute attributes audit authenticated authentication authid authors auto autoallocate autodblink autoextend automatic availability avg backup badfile basicfile before begin beginning benchmark between bfile bfile_base big bigfile bin binary_double binary_float binlog bit_and bit_count bit_length bit_or bit_xor bitmap blob_base block blocksize body both bound buffer_cache buffer_pool build bulk by byte byteordermark bytes cache caching call calling cancel capacity cascade cascaded case cast catalog category ceil ceiling chain change changed char_base char_length character_length characters characterset charindex charset charsetform charsetid check checksum checksum_agg child choose chr chunk class cleanup clear client clob clob_base clone close cluster_id cluster_probability cluster_set clustering coalesce coercibility col collate collation collect colu colum column column_value columns columns_updated comment commit compact compatibility compiled complete composite_limit compound compress compute concat concat_ws concurrent confirm conn connec connect connect_by_iscycle connect_by_isleaf connect_by_root connect_time connection consider consistent constant constraint constraints constructor container content contents context contributors controlfile conv convert convert_tz corr corr_k corr_s corresponding corruption cos cost count count_big counted covar_pop covar_samp cpu_per_call cpu_per_session crc32 create creation critical cross cube cume_dist curdate current current_date current_time current_timestamp current_user cursor curtime customdatum cycle data database databases datafile datafiles datalength date_add date_cache date_format date_sub dateadd datediff datefromparts datename datepart datetime2fromparts day day_to_second dayname dayofmonth dayofweek dayofyear days db_role_change dbtimezone ddl deallocate declare decode decompose decrement decrypt deduplicate def defa defau defaul default defaults deferred defi defin define degrees delayed delegate delete delete_all delimited demand dense_rank depth dequeue des_decrypt des_encrypt des_key_file desc descr descri describ describe descriptor deterministic diagnostics difference dimension direct_load directory disable disable_all disallow disassociate discardfile disconnect diskgroup distinct distinctrow distribute distributed div do document domain dotnet double downgrade drop dumpfile duplicate duration each edition editionable editions element ellipsis else elsif elt empty enable enable_all enclosed encode encoding encrypt end end-exec endian enforced engine engines enqueue enterprise entityescaping eomonth error errors escaped evalname evaluate event eventdata events except exception exceptions exchange exclude excluding execu execut execute exempt exists exit exp expire explain export export_set extended extent external external_1 external_2 externally extract failed failed_login_attempts failover failure far fast feature_set feature_value fetch field fields file file_name_convert filesystem_like_logging final finish first first_value fixed flash_cache flashback floor flush following follows for forall force form forma format found found_rows freelist freelists freepools fresh from from_base64 from_days ftp full function general generated get get_format get_lock getdate getutcdate global global_name globally go goto grant grants greatest group group_concat group_id grouping grouping_id groups gtid_subtract guarantee guard handler hash hashkeys having hea head headi headin heading heap help hex hierarchy high high_priority hosts hour http id ident_current ident_incr ident_seed identified identity idle_time if ifnull ignore iif ilike ilm immediate import in include including increment index indexes indexing indextype indicator indices inet6_aton inet6_ntoa inet_aton inet_ntoa infile initial initialized initially initrans inmemory inner innodb input insert install instance instantiable instr interface interleaved intersect into invalidate invisible is is_free_lock is_ipv4 is_ipv4_compat is_not is_not_null is_used_lock isdate isnull isolation iterate java join json json_exists keep keep_duplicates key keys kill language large last last_day last_insert_id last_value lax lcase lead leading least leaves left len lenght length less level levels library like like2 like4 likec limit lines link list listagg little ln load load_file lob lobs local localtime localtimestamp locate locator lock locked log log10 log2 logfile logfiles logging logical logical_reads_per_call logoff logon logs long loop low low_priority lower lpad lrtrim ltrim main make_set makedate maketime managed management manual map mapping mask master master_pos_wait match matched materialized max maxextents maximize maxinstances maxlen maxlogfiles maxloghistory maxlogmembers maxsize maxtrans md5 measures median medium member memcompress memory merge microsecond mid migration min minextents minimum mining minus minute minvalue missing mod mode model modification modify module monitoring month months mount move movement multiset mutex name name_const names nan national native natural nav nchar nclob nested never new newline next nextval no no_write_to_binlog noarchivelog noaudit nobadfile nocheck nocompress nocopy nocycle nodelay nodiscardfile noentityescaping noguarantee nokeep nologfile nomapping nomaxvalue nominimize nominvalue nomonitoring none noneditionable nonschema noorder nopr nopro noprom nopromp noprompt norely noresetlogs noreverse normal norowdependencies noschemacheck noswitch not nothing notice notrim novalidate now nowait nth_value nullif nulls num numb numbe nvarchar nvarchar2 object ocicoll ocidate ocidatetime ociduration ociinterval ociloblocator ocinumber ociref ocirefcursor ocirowid ocistring ocitype oct octet_length of off offline offset oid oidindex old on online only opaque open operations operator optimal optimize option optionally or oracle oracle_date oradata ord ordaudio orddicom orddoc order ordimage ordinality ordvideo organization orlany orlvary out outer outfile outline output over overflow overriding package pad parallel parallel_enable parameters parent parse partial partition partitions pascal passing password password_grace_time password_lock_time password_reuse_max password_reuse_time password_verify_function patch path patindex pctincrease pctthreshold pctused pctversion percent percent_rank percentile_cont percentile_disc performance period period_add period_diff permanent physical pi pipe pipelined pivot pluggable plugin policy position post_transaction pow power pragma prebuilt precedes preceding precision prediction prediction_cost prediction_details prediction_probability prediction_set prepare present preserve prior priority private private_sga privileges procedural procedure procedure_analyze processlist profiles project prompt protection public publishingservername purge quarter query quick quiesce quota quotename radians raise rand range rank raw read reads readsize rebuild record records recover recovery recursive recycle redo reduced ref reference referenced references referencing refresh regexp_like register regr_avgx regr_avgy regr_count regr_intercept regr_r2 regr_slope regr_sxx regr_sxy reject rekey relational relative relaylog release release_lock relies_on relocate rely rem remainder rename repair repeat replace replicate replication required reset resetlogs resize resource respect restore restricted result result_cache resumable resume retention return returning returns reuse reverse revoke right rlike role roles rollback rolling rollup round row row_count rowdependencies rowid rownum rows rtrim rules safe salt sample save savepoint sb1 sb2 sb4 scan schema schemacheck scn scope scroll sdo_georaster sdo_topo_geometry search sec_to_time second section securefile security seed segment select self sequence sequential serializable server servererror session session_user sessions_per_user set sets settings sha sha1 sha2 share shared shared_pool short show shrink shutdown si_averagecolor si_colorhistogram si_featurelist si_positionalcolor si_stillimage si_texture siblings sid sign sin size size_t sizes skip slave sleep smalldatetimefromparts smallfile snapshot some soname sort soundex source space sparse spfile split sql sql_big_result sql_buffer_result sql_cache sql_calc_found_rows sql_small_result sql_variant_property sqlcode sqldata sqlerror sqlname sqlstate sqrt square standalone standby start starting startup statement static statistics stats_binomial_test stats_crosstab stats_ks_test stats_mode stats_mw_test stats_one_way_anova stats_t_test_ stats_t_test_indep stats_t_test_one stats_t_test_paired stats_wsr_test status std stddev stddev_pop stddev_samp stdev stop storage store stored str str_to_date straight_join strcmp strict string struct stuff style subdate subpartition subpartitions substitutable substr substring subtime subtring_index subtype success sum suspend switch switchoffset switchover sync synchronous synonym sys sys_xmlagg sysasm sysaux sysdate sysdatetimeoffset sysdba sysoper system system_user sysutcdatetime table tables tablespace tan tdo template temporary terminated tertiary_weights test than then thread through tier ties time time_format time_zone timediff timefromparts timeout timestamp timestampadd timestampdiff timezone_abbr timezone_minute timezone_region to to_base64 to_date to_days to_seconds todatetimeoffset trace tracking transaction transactional translate translation treat trigger trigger_nestlevel triggers trim truncate try_cast try_convert try_parse type ub1 ub2 ub4 ucase unarchived unbounded uncompress under undo unhex unicode uniform uninstall union unique unix_timestamp unknown unlimited unlock unpivot unrecoverable unsafe unsigned until untrusted unusable unused update updated upgrade upped upper upsert url urowid usable usage use use_stored_outlines user user_data user_resources users using utc_date utc_timestamp uuid uuid_short validate validate_password_strength validation valist value values var var_samp varcharc vari varia variab variabl variable variables variance varp varraw varrawc varray verify version versions view virtual visible void wait wallet warning warnings week weekday weekofyear wellformed when whene whenev wheneve whenever where while whitespace with within without work wrapped xdb xml xmlagg xmlattributes xmlcast xmlcolattval xmlelement xmlexists xmlforest xmlindex xmlnamespaces xmlpi xmlquery xmlroot xmlschema xmlserialize xmltable xmltype xor year year_to_month years yearweek",literal:"true false null",built_in:"array bigint binary bit blob boolean char character date dec decimal float int int8 integer interval number numeric real record serial serial8 smallint text varchar varying void"},c:[{cN:"string",b:"'",e:"'",c:[e.BE,{b:"''"}]},{cN:"string",b:'"',e:'"',c:[e.BE,{b:'""'}]},{cN:"string",b:"`",e:"`",c:[e.BE]},e.CNM,e.CBCM,t]},e.CBCM,t]}});hljs.registerLanguage("typescript",function(e){var r={keyword:"in if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const class public private protected get set super static implements enum export import declare type namespace abstract as from extends async await",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document any number boolean string void Promise"};return{aliases:["ts"],k:r,c:[{cN:"meta",b:/^\s*['"]use strict['"]/},e.ASM,e.QSM,{cN:"string",b:"`",e:"`",c:[e.BE,{cN:"subst",b:"\\$\\{",e:"\\}"}]},e.CLCM,e.CBCM,{cN:"number",v:[{b:"\\b(0[bB][01]+)"},{b:"\\b(0[oO][0-7]+)"},{b:e.CNR}],r:0},{b:"("+e.RSR+"|\\b(case|return|throw)\\b)\\s*",k:"return throw case",c:[e.CLCM,e.CBCM,e.RM,{cN:"function",b:"(\\(.*?\\)|"+e.IR+")\\s*=>",rB:!0,e:"\\s*=>",c:[{cN:"params",v:[{b:e.IR},{b:/\(\s*\)/},{b:/\(/,e:/\)/,eB:!0,eE:!0,k:r,c:["self",e.CLCM,e.CBCM]}]}]}],r:0},{cN:"function",b:"function",e:/[\{;]/,eE:!0,k:r,c:["self",e.inherit(e.TM,{b:/[A-Za-z$_][0-9A-Za-z$_]*/}),{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:r,c:[e.CLCM,e.CBCM],i:/["'\(]/}],i:/%/,r:0},{bK:"constructor",e:/\{/,eE:!0,c:["self",{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,k:r,c:[e.CLCM,e.CBCM],i:/["'\(]/}]},{b:/module\./,k:{built_in:"module"},r:0},{bK:"module",e:/\{/,eE:!0},{bK:"interface",e:/\{/,eE:!0,k:"interface extends"},{b:/\$[(.]/},{b:"\\."+e.IR,r:0},{cN:"meta",b:"@[A-Za-z]+"}]}});hljs.registerLanguage("lisp",function(b){var e="[a-zA-Z_\\-\\+\\*\\/\\<\\=\\>\\&\\#][a-zA-Z0-9_\\-\\+\\*\\/\\<\\=\\>\\&\\#!]*",c="\\|[^]*?\\|",r="(\\-|\\+)?\\d+(\\.\\d+|\\/\\d+)?((d|e|f|l|s|D|E|F|L|S)(\\+|\\-)?\\d+)?",a={cN:"meta",b:"^#!",e:"$"},l={cN:"literal",b:"\\b(t{1}|nil)\\b"},n={cN:"number",v:[{b:r,r:0},{b:"#(b|B)[0-1]+(/[0-1]+)?"},{b:"#(o|O)[0-7]+(/[0-7]+)?"},{b:"#(x|X)[0-9a-fA-F]+(/[0-9a-fA-F]+)?"},{b:"#(c|C)\\("+r+" +"+r,e:"\\)"}]},i=b.inherit(b.QSM,{i:null}),t=b.C(";","$",{r:0}),s={b:"\\*",e:"\\*"},u={cN:"symbol",b:"[:&]"+e},d={b:e,r:0},f={b:c},m={b:"\\(",e:"\\)",c:["self",l,i,n,d]},o={c:[n,i,s,u,m,d],v:[{b:"['`]\\(",e:"\\)"},{b:"\\(quote ",e:"\\)",k:{name:"quote"}},{b:"'"+c}]},v={v:[{b:"'"+e},{b:"#'"+e+"(::"+e+")*"}]},N={b:"\\(\\s*",e:"\\)"},A={eW:!0,r:0};return N.c=[{cN:"name",v:[{b:e},{b:c}]},A],A.c=[o,v,N,l,n,i,t,s,u,f,d],{i:/\S/,c:[n,a,l,i,t,o,v,N,d]}});hljs.registerLanguage("javascript",function(e){var r="[A-Za-z$_][0-9A-Za-z$_]*",t={keyword:"in of if for while finally var new function do return void else break catch instanceof with throw case default try this switch continue typeof delete let yield const export super debugger as async await static import from as",literal:"true false null undefined NaN Infinity",built_in:"eval isFinite isNaN parseFloat parseInt decodeURI decodeURIComponent encodeURI encodeURIComponent escape unescape Object Function Boolean Error EvalError InternalError RangeError ReferenceError StopIteration SyntaxError TypeError URIError Number Math Date String RegExp Array Float32Array Float64Array Int16Array Int32Array Int8Array Uint16Array Uint32Array Uint8Array Uint8ClampedArray ArrayBuffer DataView JSON Intl arguments require module console window document Symbol Set Map WeakSet WeakMap Proxy Reflect Promise"},a={cN:"number",v:[{b:"\\b(0[bB][01]+)"},{b:"\\b(0[oO][0-7]+)"},{b:e.CNR}],r:0},n={cN:"subst",b:"\\$\\{",e:"\\}",k:t,c:[]},c={cN:"string",b:"`",e:"`",c:[e.BE,n]};n.c=[e.ASM,e.QSM,c,a,e.RM];var s=n.c.concat([e.CBCM,e.CLCM]);return{aliases:["js","jsx"],k:t,c:[{cN:"meta",r:10,b:/^\s*['"]use (strict|asm)['"]/},{cN:"meta",b:/^#!/,e:/$/},e.ASM,e.QSM,c,e.CLCM,e.CBCM,a,{b:/[{,]\s*/,r:0,c:[{b:r+"\\s*:",rB:!0,r:0,c:[{cN:"attr",b:r,r:0}]}]},{b:"("+e.RSR+"|\\b(case|return|throw)\\b)\\s*",k:"return throw case",c:[e.CLCM,e.CBCM,e.RM,{cN:"function",b:"(\\(.*?\\)|"+r+")\\s*=>",rB:!0,e:"\\s*=>",c:[{cN:"params",v:[{b:r},{b:/\(\s*\)/},{b:/\(/,e:/\)/,eB:!0,eE:!0,k:t,c:s}]}]},{b:/</,e:/(\/\w+|\w+\/)>/,sL:"xml",c:[{b:/<\w+\s*\/>/,skip:!0},{b:/<\w+/,e:/(\/\w+|\w+\/)>/,skip:!0,c:[{b:/<\w+\s*\/>/,skip:!0},"self"]}]}],r:0},{cN:"function",bK:"function",e:/\{/,eE:!0,c:[e.inherit(e.TM,{b:r}),{cN:"params",b:/\(/,e:/\)/,eB:!0,eE:!0,c:s}],i:/\[|%/},{b:/\$[(.]/},e.METHOD_GUARD,{cN:"class",bK:"class",e:/[{;=]/,eE:!0,i:/[:"\[\]]/,c:[{bK:"extends"},e.UTM]},{bK:"constructor",e:/\{/,eE:!0}],i:/#(?!!)/}});hljs.registerLanguage("java",function(e){var a="[À-ʸa-zA-Z_$][À-ʸa-zA-Z_$0-9]*",t=a+"(<"+a+"(\\s*,\\s*"+a+")*>)?",r="false synchronized int abstract float private char boolean static null if const for true while long strictfp finally protected import native final void enum else break transient catch instanceof byte super volatile case assert short package default double public try this switch continue throws protected public private module requires exports do",s="\\b(0[bB]([01]+[01_]+[01]+|[01]+)|0[xX]([a-fA-F0-9]+[a-fA-F0-9_]+[a-fA-F0-9]+|[a-fA-F0-9]+)|(([\\d]+[\\d_]+[\\d]+|[\\d]+)(\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))?|\\.([\\d]+[\\d_]+[\\d]+|[\\d]+))([eE][-+]?\\d+)?)[lLfF]?",c={cN:"number",b:s,r:0};return{aliases:["jsp"],k:r,i:/<\/|#/,c:[e.C("/\\*\\*","\\*/",{r:0,c:[{b:/\w+@/,r:0},{cN:"doctag",b:"@[A-Za-z]+"}]}),e.CLCM,e.CBCM,e.ASM,e.QSM,{cN:"class",bK:"class interface",e:/[{;=]/,eE:!0,k:"class interface",i:/[:"\[\]]/,c:[{bK:"extends implements"},e.UTM]},{bK:"new throw return else",r:0},{cN:"function",b:"("+t+"\\s+)+"+e.UIR+"\\s*\\(",rB:!0,e:/[{;=]/,eE:!0,k:r,c:[{b:e.UIR+"\\s*\\(",rB:!0,r:0,c:[e.UTM]},{cN:"params",b:/\(/,e:/\)/,k:r,r:0,c:[e.ASM,e.QSM,e.CNM,e.CBCM]},e.CLCM,e.CBCM]},c,{cN:"meta",b:"@[A-Za-z]+"}]}});hljs.registerLanguage("clojure-repl",function(e){return{c:[{cN:"meta",b:/^([\w.-]+|\s*#_)=>/,starts:{e:/$/,sL:"clojure"}}]}});hljs.registerLanguage("cpp",function(t){var e={cN:"keyword",b:"\\b[a-z\\d_]*_t\\b"},r={cN:"string",v:[{b:'(u8?|U)?L?"',e:'"',i:"\\n",c:[t.BE]},{b:'(u8?|U)?R"',e:'"',c:[t.BE]},{b:"'\\\\?.",e:"'",i:"."}]},s={cN:"number",v:[{b:"\\b(0b[01']+)"},{b:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)(u|U|l|L|ul|UL|f|F|b|B)"},{b:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)"}],r:0},i={cN:"meta",b:/#\s*[a-z]+\b/,e:/$/,k:{"meta-keyword":"if else elif endif define undef warning error line pragma ifdef ifndef include"},c:[{b:/\\\n/,r:0},t.inherit(r,{cN:"meta-string"}),{cN:"meta-string",b:/<[^\n>]*>/,e:/$/,i:"\\n"},t.CLCM,t.CBCM]},a=t.IR+"\\s*\\(",c={keyword:"int float while private char catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignof constexpr decltype noexcept static_assert thread_local restrict _Bool complex _Complex _Imaginary atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and or not",built_in:"std string cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap array shared_ptr abort abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr",literal:"true false nullptr NULL"},n=[e,t.CLCM,t.CBCM,s,r];return{aliases:["c","cc","h","c++","h++","hpp"],k:c,i:"</",c:n.concat([i,{b:"\\b(deque|list|queue|stack|vector|map|set|bitset|multiset|multimap|unordered_map|unordered_set|unordered_multiset|unordered_multimap|array)\\s*<",e:">",k:c,c:["self",e]},{b:t.IR+"::",k:c},{v:[{b:/=/,e:/;/},{b:/\(/,e:/\)/},{bK:"new throw return else",e:/;/}],k:c,c:n.concat([{b:/\(/,e:/\)/,k:c,c:n.concat(["self"]),r:0}]),r:0},{cN:"function",b:"("+t.IR+"[\\*&\\s]+)+"+a,rB:!0,e:/[{;=]/,eE:!0,k:c,i:/[^\w\s\*&]/,c:[{b:a,rB:!0,c:[t.TM],r:0},{cN:"params",b:/\(/,e:/\)/,k:c,r:0,c:[t.CLCM,t.CBCM,r,s,e]},t.CLCM,t.CBCM,i]},{cN:"class",bK:"class struct",e:/[{;:]/,c:[{b:/</,e:/>/,c:["self"]},t.TM]}]),exports:{preprocessor:i,strings:r,k:c}}});hljs.registerLanguage("kotlin",function(e){var t={keyword:"abstract as val var vararg get set class object open private protected public noinline crossinline dynamic final enum if else do while for when throw try catch finally import package is in fun override companion reified inline lateinit initinterface annotation data sealed internal infix operator out by constructor super trait volatile transient native default",built_in:"Byte Short Char Int Long Boolean Float Double Void Unit Nothing",literal:"true false null"},r={cN:"keyword",b:/\b(break|continue|return|this)\b/,starts:{c:[{cN:"symbol",b:/@\w+/}]}},i={cN:"symbol",b:e.UIR+"@"},n={cN:"subst",b:"\\${",e:"}",c:[e.ASM,e.CNM]},a={cN:"variable",b:"\\$"+e.UIR},c={cN:"string",v:[{b:'"""',e:'"""',c:[a,n]},{b:"'",e:"'",i:/\n/,c:[e.BE]},{b:'"',e:'"',i:/\n/,c:[e.BE,a,n]}]},s={cN:"meta",b:"@(?:file|property|field|get|set|receiver|param|setparam|delegate)\\s*:(?:\\s*"+e.UIR+")?"},o={cN:"meta",b:"@"+e.UIR,c:[{b:/\(/,e:/\)/,c:[e.inherit(c,{cN:"meta-string"})]}]};return{k:t,c:[e.C("/\\*\\*","\\*/",{r:0,c:[{cN:"doctag",b:"@[A-Za-z]+"}]}),e.CLCM,e.CBCM,r,i,s,o,{cN:"function",bK:"fun",e:"[(]|$",rB:!0,eE:!0,k:t,i:/fun\s+(<.*>)?[^\s\(]+(\s+[^\s\(]+)\s*=/,r:5,c:[{b:e.UIR+"\\s*\\(",rB:!0,r:0,c:[e.UTM]},{cN:"type",b:/</,e:/>/,k:"reified",r:0},{cN:"params",b:/\(/,e:/\)/,endsParent:!0,k:t,r:0,c:[{b:/:/,e:/[=,\/]/,eW:!0,c:[{cN:"type",b:e.UIR},e.CLCM,e.CBCM],r:0},e.CLCM,e.CBCM,s,o,c,e.CNM]},e.CBCM]},{cN:"class",bK:"class interface trait",e:/[:\{(]|$/,eE:!0,i:"extends implements",c:[{bK:"public protected internal private constructor"},e.UTM,{cN:"type",b:/</,e:/>/,eB:!0,eE:!0,r:0},{cN:"type",b:/[,:]\s*/,e:/[<\(,]|$/,eB:!0,rE:!0},s,o]},c,{cN:"meta",b:"^#!/usr/bin/env",e:"$",i:"\n"},e.CNM]}});hljs.registerLanguage("http",function(e){var t="HTTP/[0-9\\.]+";return{aliases:["https"],i:"\\S",c:[{b:"^"+t,e:"$",c:[{cN:"number",b:"\\b\\d{3}\\b"}]},{b:"^[A-Z]+ (.*?) "+t+"$",rB:!0,e:"$",c:[{cN:"string",b:" ",e:" ",eB:!0,eE:!0},{b:t},{cN:"keyword",b:"[A-Z]+"}]},{cN:"attribute",b:"^\\w",e:": ",eE:!0,i:"\\n|\\s|=",starts:{e:"$",r:0}},{b:"\\n\\n",starts:{sL:[],eW:!0}}]}});hljs.registerLanguage("makefile",function(e){var i={cN:"variable",v:[{b:"\\$\\("+e.UIR+"\\)",c:[e.BE]},{b:/\$[@%<?\^\+\*]/}]},r={cN:"string",b:/"/,e:/"/,c:[e.BE,i]},a={cN:"variable",b:/\$\([\w-]+\s/,e:/\)/,k:{built_in:"subst patsubst strip findstring filter filter-out sort word wordlist firstword lastword dir notdir suffix basename addsuffix addprefix join wildcard realpath abspath error warning shell origin flavor foreach if or and call eval file value"},c:[i]},n={b:"^"+e.UIR+"\\s*[:+?]?=",i:"\\n",rB:!0,c:[{b:"^"+e.UIR,e:"[:+?]?=",eE:!0}]},t={cN:"meta",b:/^\.PHONY:/,e:/$/,k:{"meta-keyword":".PHONY"},l:/[\.\w]+/},l={cN:"section",b:/^[^\s]+:/,e:/$/,c:[i]};return{aliases:["mk","mak"],k:"define endef undefine ifdef ifndef ifeq ifneq else endif include -include sinclude override export unexport private vpath",l:/[\w-]+/,c:[e.HCM,i,r,a,n,t,l]}});hljs.registerLanguage("ini",function(e){var b={cN:"string",c:[e.BE],v:[{b:"'''",e:"'''",r:10},{b:'"""',e:'"""',r:10},{b:'"',e:'"'},{b:"'",e:"'"}]};return{aliases:["toml"],cI:!0,i:/\S/,c:[e.C(";","$"),e.HCM,{cN:"section",b:/^\s*\[+/,e:/\]+/},{b:/^[a-z0-9\[\]_-]+\s*=\s*/,e:"$",rB:!0,c:[{cN:"attr",b:/[a-z0-9\[\]_-]+/},{b:/=/,eW:!0,r:0,c:[{cN:"literal",b:/\bon|off|true|false|yes|no\b/},{cN:"variable",v:[{b:/\$[\w\d"][\w\d_]*/},{b:/\$\{(.*?)}/}]},b,{cN:"number",b:/([\+\-]+)?[\d]+_[\d_]+/},e.NM]}]}]}});hljs.registerLanguage("rust",function(e){var t="([ui](8|16|32|64|128|size)|f(32|64))?",r="alignof as be box break const continue crate do else enum extern false fn for if impl in let loop match mod mut offsetof once priv proc pub pure ref return self Self sizeof static struct super trait true type typeof unsafe unsized use virtual while where yield move default",n="drop i8 i16 i32 i64 i128 isize u8 u16 u32 u64 u128 usize f32 f64 str char bool Box Option Result String Vec Copy Send Sized Sync Drop Fn FnMut FnOnce ToOwned Clone Debug PartialEq PartialOrd Eq Ord AsRef AsMut Into From Default Iterator Extend IntoIterator DoubleEndedIterator ExactSizeIterator SliceConcatExt ToString assert! assert_eq! bitflags! bytes! cfg! col! concat! concat_idents! debug_assert! debug_assert_eq! env! panic! file! format! format_args! include_bin! include_str! line! local_data_key! module_path! option_env! print! println! select! stringify! try! unimplemented! unreachable! vec! write! writeln! macro_rules! assert_ne! debug_assert_ne!";return{aliases:["rs"],k:{keyword:r,literal:"true false Some None Ok Err",built_in:n},l:e.IR+"!?",i:"</",c:[e.CLCM,e.C("/\\*","\\*/",{c:["self"]}),e.inherit(e.QSM,{b:/b?"/,i:null}),{cN:"string",v:[{b:/r(#*)"(.|\n)*?"\1(?!#)/},{b:/b?'\\?(x\w{2}|u\w{4}|U\w{8}|.)'/}]},{cN:"symbol",b:/'[a-zA-Z_][a-zA-Z0-9_]*/},{cN:"number",v:[{b:"\\b0b([01_]+)"+t},{b:"\\b0o([0-7_]+)"+t},{b:"\\b0x([A-Fa-f0-9_]+)"+t},{b:"\\b(\\d[\\d_]*(\\.[0-9_]+)?([eE][+-]?[0-9_]+)?)"+t}],r:0},{cN:"function",bK:"fn",e:"(\\(|<)",eE:!0,c:[e.UTM]},{cN:"meta",b:"#\\!?\\[",e:"\\]",c:[{cN:"meta-string",b:/"/,e:/"/}]},{cN:"class",bK:"type",e:";",c:[e.inherit(e.UTM,{endsParent:!0})],i:"\\S"},{cN:"class",bK:"trait enum struct union",e:"{",c:[e.inherit(e.UTM,{endsParent:!0})],i:"[\\w\\d]"},{b:e.IR+"::",k:{built_in:n}},{b:"->"}]}});hljs.registerLanguage("haskell",function(e){var i={v:[e.C("--","$"),e.C("{-","-}",{c:["self"]})]},a={cN:"meta",b:"{-#",e:"#-}"},l={cN:"meta",b:"^#",e:"$"},c={cN:"type",b:"\\b[A-Z][\\w']*",r:0},n={b:"\\(",e:"\\)",i:'"',c:[a,l,{cN:"type",b:"\\b[A-Z][\\w]*(\\((\\.\\.|,|\\w+)\\))?"},e.inherit(e.TM,{b:"[_a-z][\\w']*"}),i]},s={b:"{",e:"}",c:n.c};return{aliases:["hs"],k:"let in if then else case of where do module import hiding qualified type data newtype deriving class instance as default infix infixl infixr foreign export ccall stdcall cplusplus jvm dotnet safe unsafe family forall mdo proc rec",c:[{bK:"module",e:"where",k:"module where",c:[n,i],i:"\\W\\.|;"},{b:"\\bimport\\b",e:"$",k:"import qualified as hiding",c:[n,i],i:"\\W\\.|;"},{cN:"class",b:"^(\\s*)?(class|instance)\\b",e:"where",k:"class family instance where",c:[c,n,i]},{cN:"class",b:"\\b(data|(new)?type)\\b",e:"$",k:"data family type newtype deriving",c:[a,c,n,s,i]},{bK:"default",e:"$",c:[c,n,i]},{bK:"infix infixl infixr",e:"$",c:[e.CNM,i]},{b:"\\bforeign\\b",e:"$",k:"foreign import export ccall stdcall cplusplus jvm dotnet safe unsafe",c:[c,e.QSM,i]},{cN:"meta",b:"#!\\/usr\\/bin\\/env runhaskell",e:"$"},a,l,e.QSM,e.CNM,c,e.inherit(e.TM,{b:"^[_a-z][\\w']*"}),i,{b:"->|<-"}]}});hljs.registerLanguage("nix",function(e){var r={keyword:"rec with let in inherit assert if else then",literal:"true false or and null",built_in:"import abort baseNameOf dirOf isNull builtins map removeAttrs throw toString derivation"},t={cN:"subst",b:/\$\{/,e:/}/,k:r},i={b:/[a-zA-Z0-9-_]+(\s*=)/,rB:!0,r:0,c:[{cN:"attr",b:/\S+/}]},s={cN:"string",c:[t],v:[{b:"''",e:"''"},{b:'"',e:'"'}]},a=[e.NM,e.HCM,e.CBCM,s,i];return t.c=a,{aliases:["nixos"],k:r,c:a}});hljs.registerLanguage("diff",function(e){return{aliases:["patch"],c:[{cN:"meta",r:10,v:[{b:/^@@ +\-\d+,\d+ +\+\d+,\d+ +@@$/},{b:/^\*\*\* +\d+,\d+ +\*\*\*\*$/},{b:/^\-\-\- +\d+,\d+ +\-\-\-\-$/}]},{cN:"comment",v:[{b:/Index: /,e:/$/},{b:/={3,}/,e:/$/},{b:/^\-{3}/,e:/$/},{b:/^\*{3} /,e:/$/},{b:/^\+{3}/,e:/$/},{b:/\*{5}/,e:/\*{5}$/}]},{cN:"addition",b:"^\\+",e:"$"},{cN:"deletion",b:"^\\-",e:"$"},{cN:"addition",b:"^\\!",e:"$"}]}});hljs.registerLanguage("go",function(e){var t={keyword:"break default func interface select case map struct chan else goto package switch const fallthrough if range type continue for import return var go defer bool byte complex64 complex128 float32 float64 int8 int16 int32 int64 string uint8 uint16 uint32 uint64 int uint uintptr rune",literal:"true false iota nil",built_in:"append cap close complex copy imag len make new panic print println real recover delete"};return{aliases:["golang"],k:t,i:"</",c:[e.CLCM,e.CBCM,{cN:"string",v:[e.QSM,{b:"'",e:"[^\\\\]'"},{b:"`",e:"`"}]},{cN:"number",v:[{b:e.CNR+"[dflsi]",r:1},e.CNM]},{b:/:=/},{cN:"function",bK:"func",e:/\s*\{/,eE:!0,c:[e.TM,{cN:"params",b:/\(/,e:/\)/,k:t,i:/["']/}]}]}});hljs.registerLanguage("gradle",function(e){return{cI:!0,k:{keyword:"task project allprojects subprojects artifacts buildscript configurations dependencies repositories sourceSets description delete from into include exclude source classpath destinationDir includes options sourceCompatibility targetCompatibility group flatDir doLast doFirst flatten todir fromdir ant def abstract break case catch continue default do else extends final finally for if implements instanceof native new private protected public return static switch synchronized throw throws transient try volatile while strictfp package import false null super this true antlrtask checkstyle codenarc copy boolean byte char class double float int interface long short void compile runTime file fileTree abs any append asList asWritable call collect compareTo count div dump each eachByte eachFile eachLine every find findAll flatten getAt getErr getIn getOut getText grep immutable inject inspect intersect invokeMethods isCase join leftShift minus multiply newInputStream newOutputStream newPrintWriter newReader newWriter next plus pop power previous print println push putAt read readBytes readLines reverse reverseEach round size sort splitEachLine step subMap times toInteger toList tokenize upto waitForOrKill withPrintWriter withReader withStream withWriter withWriterAppend write writeLine"},c:[e.CLCM,e.CBCM,e.ASM,e.QSM,e.NM,e.RM]}});hljs.registerLanguage("markdown",function(e){return{aliases:["md","mkdown","mkd"],c:[{cN:"section",v:[{b:"^#{1,6}",e:"$"},{b:"^.+?\\n[=-]{2,}$"}]},{b:"<",e:">",sL:"xml",r:0},{cN:"bullet",b:"^([*+-]|(\\d+\\.))\\s+"},{cN:"strong",b:"[*_]{2}.+?[*_]{2}"},{cN:"emphasis",v:[{b:"\\*.+?\\*"},{b:"_.+?_",r:0}]},{cN:"quote",b:"^>\\s+",e:"$"},{cN:"code",v:[{b:"^```w*s*$",e:"^```s*$"},{b:"`.+?`"},{b:"^( {4}| )",e:"$",r:0}]},{b:"^[-\\*]{3,}",e:"$"},{b:"\\[.+?\\][\\(\\[].*?[\\)\\]]",rB:!0,c:[{cN:"string",b:"\\[",e:"\\]",eB:!0,rE:!0,r:0},{cN:"link",b:"\\]\\(",e:"\\)",eB:!0,eE:!0},{cN:"symbol",b:"\\]\\[",e:"\\]",eB:!0,eE:!0}],r:10},{b:/^\[[^\n]+\]:/,rB:!0,c:[{cN:"symbol",b:/\[/,e:/\]/,eB:!0,eE:!0},{cN:"link",b:/:\s*/,e:/$/,eB:!0}]}]}});hljs.initHighlightingOnLoad(); diff --git a/web/converse/static/styles.css b/web/converse/static/styles.css new file mode 100644 index 000000000000..b57fd899ea8d --- /dev/null +++ b/web/converse/static/styles.css @@ -0,0 +1,145 @@ +* :not(.material-icons) { + font-family: 'Ubuntu', sans-serif; +} + +code, pre, code * { + font-family: 'Ubuntu Mono', monospace !important; +} + +.cvs-title, .thread-link { + text-decoration: none; +} + +.thread-list-item:hover { + background-color: #f5f5f5; +} + +.thread-link { + padding: 5px; + padding-top: 10px; +} + +.thread-title { + padding-right: 15vw; +} + +.search-field { + margin-right: 15px; + max-width: 200px; +} + +.thread-author { + font-style: italic; + font-size: 85%; +} + +@media only screen and (min-width: 768px) { + .converse main { + padding-top: 10px; + padding-bottom: 10px; + } +} + +.mdl-list__item-text-body { + max-height: 40px; +} + +.thread-divider:after { + border-bottom: 1px solid rgba(0,0,0,.13); + content:""; + position: absolute; + width: 80%; +} + +html, body { + margin: 0; + padding: 0; +} +.converse .mdl-layout__header-row { + padding-left: 40px; +} +.converse .mdl-layout.is-small-screen .mdl-layout__header-row h3 { + font-size: inherit; +} +.converse .mdl-card { + height: auto; + display: -webkit-flex; + display: -ms-flexbox; + display: flex; + -webkit-flex-direction: column; + -ms-flex-direction: column; + flex-direction: column; +} +.converse .mdl-card > * { + height: auto; +} +.converse .mdl-card .mdl-card__supporting-text { + margin: 40px; + -webkit-flex-grow: 1; + -ms-flex-positive: 1; + flex-grow: 1; + padding: 0; + color: inherit; + width: calc(100% - 80px); +} +.mdl-demo.converse .mdl-card__supporting-text h4 { + margin-top: 0; + margin-bottom: 20px; +} +.converse .mdl-card__actions { + margin: 0; + padding: 4px 40px; + color: inherit; +} +.converse section.section--center { + max-width: 860px; +} +.converse .mdl-card .avatar-card { + display: flex; + flex-direction: column; + text-align: center; + margin-top: 30px; +} +.desktop-avatar { + width: 80px; + margin-right: auto; + margin-left: auto; +} +.mobile-avatar { + width: 30px; + border-radius: 8px; + margin-bottom: 5px; +} +.mobile-date { + text-decoration: none; +} +.converse .mdl-card .post-box { + margin: 20px; +} +.converse .mdl-card .post-actions { + display: flex; + padding-right: 5px; +} +.post-action { + margin: 5px; + margin-bottom: 10px; +} +.converse section.post-section { + padding: 5px; +} +.post-date { + text-decoration: none; + font-size: 80%; +} +.mdl-layout__content { + flex: 1 0 auto; +} +.converse .reply-box { + padding-top: 10px; +} +.search-result { + margin: 8px; +} +.search-result .mdl-button { + margin: 3px; +} diff --git a/web/converse/templates/index.html b/web/converse/templates/index.html new file mode 100644 index 000000000000..07d6e7bfcf09 --- /dev/null +++ b/web/converse/templates/index.html @@ -0,0 +1,80 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0"> + <title>Converse: Index</title> + + <!-- TODO --> + <meta http-equiv="Content-Security-Policy" content="script-src https://code.getmdl.io 'self';"> + <!-- <link rel="shortcut icon" href="images/favicon.png"> --> + + <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Ubuntu"> + <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons"> + <link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.blue_grey-orange.min.css" /> + <link rel="stylesheet" href="/static/styles.css"> + </head> + <body class="converse mdl-base mdl-color-text--grey-700 mdl-color--grey-100"> + <div class="mdl-layout mdl-layout--fixed-header mdl-js-layout mdl-color--grey-100"> + <header class="mdl-layout__header mdl-layout__header--scroll mdl-color--primary-dark"> + <div class="mdl-layout__header-row"> + <a href="/" class="mdl-layout-title mdl-color-text--blue-grey-50 cvs-title">Converse</a> + <div class="mdl-layout-spacer"></div> + <div class="mdl-textfield mdl-js-textfield mdl-textfield--floating-label mdl-color-text--blue-grey-50 search-field"> + <form method="get" action="/search"> + <input class="mdl-textfield__input" type="search" id="search-query" aria-label="Search" name="query"> + <label class="mdl-textfield__label mdl-color-text--blue-grey-100" for="search-query">Search query...</label> + <input type="submit" hidden /> + </form> + </div> + + <a href="/thread/new"> + <button class="mdl-button mdl-js-button mdl-button--raised mdl-button--accent mdl-js-ripple-effect"> + New Thread + </button> + </a> + </div> + </header> + <main class="mdl-layout__content"> + <section class="section--center mdl-grid mdl-grid--no-spacing mdl-shadow--2dp"> + <div class="mdl-card mdl-cell mdl-cell--12-col"> + <div class="mdl-card__supporting-text mdl-grid"> + <h4 class="mdl-cell mdl-cell--12-col">Latest threads:</h4> + <ul class="mdl-list"> + {% for thread in threads %} + <li class="mdl-list__item thread-list-item mdl-list__item--three-line"> + <a class="thread-link mdl-color-text--grey-800" href="/thread/{{ thread.id }}"> + <span class="mdl-list__item-primary-content {% if loop.index < threads.len() %}thread-divider{% endif %}"> + <button class="mdl-button mdl-js-button mdl-button--fab mdl-button--mini-fab mdl-button--colored mdl-list__item-icon"> + <i class="material-icons"> + {% if thread.sticky %} + announcement + {% else if thread.closed %} + lock + {% else %} + library_books + {% endif %} + </i> + </button> + <span class="thread-title">{{ thread.title }}<span class="thread-author"> by {{ thread.author_name }}</span></span> + <span class="mdl-list__item-text-body"> + Last reply by {{ thread.post_author }} on {{ thread.posted }}. + </span> + </span> + </a> + </li> + {% endfor %} + </ul> + </div> + </div> + </section> + </main> + <footer class="mdl-mini-footer"> + <div class="mdl-mini-footer--right-section"> + <p>Powered by <a href="https://code.tvl.fyi/about/web/converse">Converse</a></p> + </div> + </footer> + </div> + <script src="https://code.getmdl.io/1.3.0/material.min.js"></script> + </body> +</html> diff --git a/web/converse/templates/post.html b/web/converse/templates/post.html new file mode 100644 index 000000000000..8ea2df1cd344 --- /dev/null +++ b/web/converse/templates/post.html @@ -0,0 +1,124 @@ +{# + This template is shared by the new thread, reply and post-editing pages. + + The main display differences between the different editing styles are the + headline of the page ("Submit new thread", "Reply to thread", "Edit post") + and whether or not the subject line field is displayed in the input form. + + Every one of these pages can have a variable length list of alerts submitted + into the template, which will be rendered as Boostrap alert boxes above the + user input form. +#} + +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0"> + <title>Converse: Post</title> + + <meta http-equiv="Content-Security-Policy" content="script-src https://code.getmdl.io 'self';"> + <!-- <link rel="shortcut icon" href="images/favicon.png"> --> + + <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Ubuntu"> + <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons"> + <link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.blue_grey-orange.min.css" /> + <link rel="stylesheet" href="/static/styles.css"> + </head> + <body class="converse mdl-base mdl-color-text--grey-700 mdl-color--grey-100"> + <div class="mdl-layout mdl-layout--fixed-header mdl-js-layout mdl-color--grey-100"> + <header class="mdl-layout__header mdl-layout__header--scroll mdl-color--primary-dark"> + <div class="mdl-layout__header-row"> + <a href="/" class="mdl-layout-title mdl-color-text--blue-grey-50 cvs-title"> + {% match mode %} + {% when EditingMode::NewThread %} + Converse: Submit new thread + {% when EditingMode::PostReply %} + Converse: Reply to thread + {% when EditingMode::EditPost %} + Converse: Edit post + {% endmatch %} + </a> + <div class="mdl-layout-spacer"></div> + <a href="/"> + <button class="mdl-button mdl-js-button mdl-button--raised mdl-button--accent mdl-js-ripple-effect"> + Back to index + </button> + </a> + </div> + </header> + <main class="mdl-layout__content mdl-grid"> + <div class="mdl-card mdl-shadow--2dp mdl-cell--8-col"> + {% match mode %} + {% when EditingMode::NewThread %} + <form action="/thread/submit" method="post"> + {% when EditingMode::PostReply %} + <form action="/thread/reply" method="post"> + {% when EditingMode::EditPost %} + <form action="/post/edit" method="post"> + {% endmatch %} + {% match mode %} + {% when EditingMode::PostReply %} + <input type="hidden" id="thread_id" name="thread_id" value="{{ id.unwrap() }}"> + {% when EditingMode::EditPost %} + <input type="hidden" id="thread_id" name="post_id" value="{{ id.unwrap() }}"> + {% else %} + {# no post ID when making a new thread #} + {% endmatch %} + <div class="mdl-card__supporting-text"> + {% for alert in alerts %} + <span class="mdl-chip mdl-color--red-200"> + <span class="mdl-chip__text">{{ alert }} </span> + </span> + {% endfor %} + {% if mode == EditingMode::NewThread %} + <div class="mdl-textfield mdl-js-textfield mdl-textfield--floating-label mdl-cell--12-col"> + <input class="mdl-textfield__input" type="text" id="title" name="title" aria-label="thread title" required {% match title %}{% when Some with (title_text) %}value="{{ title_text }}"{% else %}{# Nothing! #}{% endmatch %}> + <label class="mdl-textfield__label" for="title">Thread title</label> + </div> + {% endif %} + <div class="mdl-textfield mdl-js-textfield mdl-textfield--floating-label mdl-cell--12-col"> + <textarea class="mdl-textfield__input" type="text" rows="25" id="post" name="post" aria-label="post content" required> + {%- match post -%} + {%- when Some with (post_text) -%} + {{- post_text -}} + {%- else -%} + {# Nothing! #} + {%- endmatch -%} + </textarea> + <label class="mdl-textfield__label" for="body">Content ...</label> + </div> + </div> + <div class="mdl-card__actions"> + <input class="mdl-button mdl-button--raised mdl-button--colored mdl-js-button mdl-js-ripple-effect" type="submit" value="Submit!"> + </div> + </form> + </div> + <div class="mdl-card mdl-shadow--2dp mdl-cell--4-col"> + <div class="mdl-card__title mdl-card--border"> + Quick Markdown primer: + </div> + <div class="mdl-card__supporting-text"> + <p> + Remember that you can use <a href="https://daringfireball.net/projects/markdown/basics"><strong>Markdown</strong></a> when + writing your posts: + </p> + <p><i>*italic text*</i></p> + <p><strong>**bold text**</strong></p> + <p><s>~strikethrough text~</s></p> + <p><code>[link text](https://some.link.com/)</code></p> + <p><code></code></p> + <p>Use <code>*</code> or <code>-</code> to enumerate lists.</p> + <p>See Markdown documentation for more information!</p> + </div> + </div> + </main> + <footer class="mdl-mini-footer"> + <div class="mdl-mini-footer--right-section"> + <p>Powered by <a href="https://code.tvl.fyi/about/web/converse">Converse</a></p> + </div> + </footer> + </div> + <script src="https://code.getmdl.io/1.3.0/material.min.js"></script> + </body> +</html> diff --git a/web/converse/templates/search.html b/web/converse/templates/search.html new file mode 100644 index 000000000000..418440e028ce --- /dev/null +++ b/web/converse/templates/search.html @@ -0,0 +1,67 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0"> + <title>Converse: Search results</title> + + <!-- TODO --> + <meta http-equiv="Content-Security-Policy" content="script-src https://code.getmdl.io 'self';"> + <!-- <link rel="shortcut icon" href="images/favicon.png"> --> + + <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Ubuntu"> + <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons"> + <link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.blue_grey-orange.min.css" /> + <link rel="stylesheet" href="/static/styles.css"> + </head> + <body class="converse mdl-base mdl-color-text--grey-700 mdl-color--grey-100"> + <div class="mdl-layout mdl-layout--fixed-header mdl-js-layout mdl-color--grey-100"> + <header class="mdl-layout__header mdl-layout__header--scroll mdl-color--primary-dark"> + <div class="mdl-layout__header-row"> + <a href="/" class="mdl-layout-title mdl-color-text--blue-grey-50 cvs-title">Converse</a> + <div class="mdl-layout-spacer"></div> + <div class="mdl-textfield mdl-js-textfield mdl-textfield--floating-label mdl-color-text--blue-grey-50 search-field"> + <form method="get" action="/search"> + <input class="mdl-textfield__input" type="search" id="search-query" aria-label="Search" name="query"> + <label class="mdl-textfield__label mdl-color-text--blue-grey-100" for="search-query">Search query...</label> + <input type="submit" hidden /> + </form> + </div> + + <a href="/"> + <button class="mdl-button mdl-js-button mdl-button--raised mdl-button--accent mdl-js-ripple-effect"> + Back to index + </button> + </a> + </div> + </header> + <main class="mdl-layout__content"> + <section class="section--center mdl-grid"> + <div class="mdl-cell--12-col"> + <h4>Search results for '{{ query }}':</h4> + </div> + {% for result in results %} + <div class="mdl-card mdl-cell--6-col search-result mdl-shadow--2dp"> + <div class="mdl-card__supporting-text"> + <p>Posted in '{{ result.title }}' by {{ result.author }}:</p> + <p>{{ result.headline|safe }}</p> + </div> + <div class="mdl-card__actions mdl-card--border post-actions"> + <div class="mdl-layout-spacer"></div> + <a class="mdl-button mdl-js-button mdl-button--fab mdl-button--mini-fab mdl-button--colored" href="/thread/{{ result.thread_id }}#post-{{ result.post_id }}"> + <i class="material-icons">arrow_forward</i> + </a> + </div> + </div> + {% endfor %} + </section> + </main> + <footer class="mdl-mini-footer"> + <div class="mdl-mini-footer--right-section"> + <p>Powered by <a href="https://code.tvl.fyi/about/web/converse">Converse</a></p> + </div> + </footer> + </div> + <script src="https://code.getmdl.io/1.3.0/material.min.js"></script> + </body> +</html> diff --git a/web/converse/templates/thread.html b/web/converse/templates/thread.html new file mode 100644 index 000000000000..66ec23de1f0a --- /dev/null +++ b/web/converse/templates/thread.html @@ -0,0 +1,111 @@ +<!doctype html> +<html lang="en"> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0, minimum-scale=1.0"> + <title>Converse: {{ title }}</title> + + <!-- TODO --> + <meta http-equiv="Content-Security-Policy" content="script-src https://code.getmdl.io 'self';"> + <!-- <link rel="shortcut icon" href="images/favicon.png"> --> + + <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Ubuntu"> + <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Ubuntu+Mono"> + <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons"> + <link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.blue_grey-orange.min.css" /> + <link rel="stylesheet" href="/static/styles.css"> + <!-- Syntax highlighting for code --> + <link rel="stylesheet" href="/static/highlight.css"> + <style>img { max-width:100%; height:auto; }</style> + <script src="/static/highlight.js"></script> + </head> + <body class="converse mdl-base mdl-color-text--grey-700 mdl-color--grey-100"> + <div class="mdl-layout mdl-layout--fixed-header mdl-js-layout mdl-color--grey-100"> + <header class="mdl-layout__header mdl-color--primary-dark"> + <div class="mdl-layout__header-row"> + <a href="/" class="mdl-layout-title mdl-color-text--blue-grey-50 cvs-title">Converse: {{ title }}</a> + <div class="mdl-layout-spacer"></div> + <a href="/"> + <button class="mdl-button mdl-js-button mdl-button--raised mdl-button--accent mdl-js-ripple-effect"> + Back to index + </button> + </a> + </div> + </header> + <main class="mdl-layout__content"> + {% for post in posts -%} + <section id="post-{{ post.id }}" class="section--center mdl-grid mdl-grid--no-spacing"> + <!-- card to display avatars on desktop --> + <div class="mdl-card mdl-shadow--2dp mdl-cell--2-col mdl-cell--hide-phone mdl-cell--hide-tablet avatar-box"> + <div class="avatar-card"> + <img class="desktop-avatar" src="https://www.gravatar.com/avatar/{{ post.author_gravatar }}?d=monsterid&s=160" /> + <p class="user-name">{{ post.author_name }}</p> + </div> + </div> + <!-- card for main post content --> + <div class="mdl-card mdl-shadow--2dp post-box mdl-cell--10-col"> + <!-- card section for displaying user & post information on mobile --> + <div class="mdl-card__supporting-text mdl-card--border mdl-cell--hide-desktop mdl-color-text--blue-grey-500 mobile-user"> + <img class="mobile-avatar" src="https://www.gravatar.com/avatar/{{ post.author_gravatar }}?d=monsterid"/> + <span> {{ post.author_name }} posted on </span> + <a class="mdl-color-text--blue-grey-500 mobile-date" href="/thread/{{ id }}#post-{{ post.id }}">{{ post.posted }}</a> + </div> + <!-- card section to display post date on desktop --> + <div class="mdl-card__menu mdl-cell--hide-phone mdl-cell--hide-tablet"> + <a class="post-date mdl-color-text--blue-grey-500" href="/thread/{{ id }}#post-{{ post.id }}">{{ post.posted }}</a> + </div> + <!-- card section for actual post content --> + <div class="mdl-card__supporting-text post-box">{{ post.body|safe }}</div> + <!-- card section for post actions --> + <div class="mdl-card__actions post-actions"> + <div class="mdl-layout-spacer"></div> + + {% if post.editable %} + <a href="/post/{{ post.id }}/edit" class="mdl-button mdl-js-button mdl-button--accent" id="edit-post-{{ post.id }}" aria-label="Edit post"> + <i class="material-icons">edit</i> + <span class="mdl-tooltip mdl-tooltip--top" for="edit-post-{{ post.id }}">Edit post</span> + </a> + {% endif %} + <button class="mdl-button mdl-js-button mdl-button--accent" id="quote-post-{{ post.id }}" aria-label="Quote post" disabled> + <i class="material-icons">reply</i> + <span class="mdl-tooltip mdl-tooltip--top" for="quote-post-{{ post.id }}">Quote post</span> + </button> + </div> + </div> + </section> + {% endfor %} + + <!-- section for writing a response on the same page --> + <section id="post-reply" class="section--center mdl-grid mdl-grid--no-spacing reply-box"> + <div class="mdl-card mdl-shadow--2dp mdl-cell--12-col"> + {% if closed %} + <div class="mdl-card__supporting-text"> + This thread is <b>closed</b> and can no longer be responded to. + </div> + {% else %} + <form id="reply-form" action="/thread/reply" method="post"> + <input type="hidden" id="thread_id" name="thread_id" value="{{ id }}"> + + <div class="mdl-card__supporting-text"> + <div class="mdl-textfield mdl-js-textfield mdl-textfield--floating-label mdl-cell--12-col"> + <textarea class="mdl-textfield__input" type="text" rows="8" id="post" name="post" aria-label="reply content"></textarea> + <label class="mdl-textfield__label" for="post">Write a reply</label> + </div> + <button class="mdl-button mdl-button--raised mdl-button--primary mdl-js-button mdl-js-ripple-effect" type="submit"> + Post! + </button> + </div> + </form> + {% endif %} + </div> + </section> + </main> + <footer class="mdl-mini-footer"> + <div class="mdl-mini-footer--right-section"> + <p>Powered by <a href="https://code.tvl.fyi/about/web/converse">Converse</a></p> + </div> + </footer> + </div> + <script src="https://code.getmdl.io/1.3.0/material.min.js"></script> + </body> +</html> diff --git a/web/converse/todo.org b/web/converse/todo.org new file mode 100644 index 000000000000..379f10753ae5 --- /dev/null +++ b/web/converse/todo.org @@ -0,0 +1,13 @@ +* DONE Pin *-versions in cargo.toml +* DONE Markdown support +* DONE Post ordering as expected +* DONE Stickies! +* DONE Search +* DONE Post editing +* TODO Configurable number of DB workers +* TODO Match certain types of Diesel errors (esp. for "not found") +* TODO Sketch out categories vs. tags system +* TODO Quote button +* TODO Multiquote buttons +* TODO Pagination +* TODO Multi-thread guest accounts diff --git a/web/panettone/.envrc b/web/panettone/.envrc new file mode 100644 index 000000000000..be81feddb1a5 --- /dev/null +++ b/web/panettone/.envrc @@ -0,0 +1 @@ +eval "$(lorri direnv)" \ No newline at end of file diff --git a/web/panettone/.gitignore b/web/panettone/.gitignore new file mode 100644 index 000000000000..be303db03207 --- /dev/null +++ b/web/panettone/.gitignore @@ -0,0 +1 @@ +*.fasl diff --git a/web/panettone/OWNERS b/web/panettone/OWNERS new file mode 100644 index 000000000000..b2b0acc303c8 --- /dev/null +++ b/web/panettone/OWNERS @@ -0,0 +1,5 @@ +inherited: true +owners: + - grfn + - tazjin + - sterni diff --git a/web/panettone/default.nix b/web/panettone/default.nix new file mode 100644 index 000000000000..a01e0d81c83a --- /dev/null +++ b/web/panettone/default.nix @@ -0,0 +1,55 @@ +{ depot, ... }: + +depot.nix.buildLisp.program { + name = "panettone"; + + deps = with depot.third_party.lisp; [ + bordeaux-threads + cl-json + cl-ppcre + cl-smtp + cl-who + defclass-std + drakma + easy-routes + hunchentoot + lass + local-time + postmodern + trivial-ldap + + depot.lisp.klatre + ]; + + srcs = [ + ./panettone.asd + ./src/packages.lisp + ./src/util.lisp + ./src/css.lisp + ./src/email.lisp + ./src/inline-markdown.lisp + ./src/authentication.lisp + ./src/model.lisp + ./src/irc.lisp + ./src/panettone.lisp + ]; + + tests = { + deps = with depot.third_party.lisp; [ + fiveam + ]; + + srcs = [ + ./test/package.lisp + ./test/model_test.lisp + ./test/inline-markdown_test.lisp + ]; + + expression = "(fiveam:run!)"; + }; + + brokenOn = [ + "ecl" # dependencies use dynamic cffi + "ccl" # The value NIL is not of the expected type STRING. when loading model.lisp + ]; +} diff --git a/web/panettone/docker-compose.yml b/web/panettone/docker-compose.yml new file mode 100644 index 000000000000..84723667e6b9 --- /dev/null +++ b/web/panettone/docker-compose.yml @@ -0,0 +1,11 @@ +version: '3.4' +services: + postgres: + image: postgres:11 + restart: always + environment: + POSTGRES_USER: panettone + POSTGRES_PASSWORD: password + POSTGRES_DB: panettone + ports: + - 127.0.0.1:5432:5432 diff --git a/web/panettone/panettone.asd b/web/panettone/panettone.asd new file mode 100644 index 000000000000..4d44e50fd3e8 --- /dev/null +++ b/web/panettone/panettone.asd @@ -0,0 +1,6 @@ +(asdf:defsystem "panettone" + :description "A simple issue tracker" + :serial t + :components ((:file "packages") + (:file "css") + (:file "pannetone"))) diff --git a/web/panettone/shell.nix b/web/panettone/shell.nix new file mode 100644 index 000000000000..aeafc7afcd99 --- /dev/null +++ b/web/panettone/shell.nix @@ -0,0 +1,15 @@ +{ depot ? import ../.. {} }: + +with depot.third_party; + +mkShell { + buildInputs = [ + docker-compose + postgresql + ]; + + PGPASSWORD = "password"; + PGHOST = "localhost"; + PGUSER = "panettone"; + PGDATABASE = "panettone"; +} diff --git a/web/panettone/src/.gitignore b/web/panettone/src/.gitignore new file mode 100644 index 000000000000..10aa5440d832 --- /dev/null +++ b/web/panettone/src/.gitignore @@ -0,0 +1,2 @@ +# I use this as the out-link for my local lisp dev env +sbcl diff --git a/web/panettone/src/authentication.lisp b/web/panettone/src/authentication.lisp new file mode 100644 index 000000000000..c3353450201d --- /dev/null +++ b/web/panettone/src/authentication.lisp @@ -0,0 +1,115 @@ +(in-package :panettone.authentication) + +(defvar *user* nil + "The currently logged-in user") + +(defvar *ldap* nil + "The ldap connection") + +(defvar *ldap-host* "localhost" + "The host for the ldap connection") + +(defvar *ldap-port* 389 + "The port for the ldap connection") + +(defclass/std user () + ((cn dn mail displayname :type string))) + +(defun connect-ldap (&key + (host "localhost") + (port 389)) + (setq *ldap-host* host + *ldap-port* port + *ldap* (ldap:new-ldap :host host :port port))) + +(defun reconnect-ldap () + (setq *ldap* (ldap:new-ldap + :host *ldap-host* + :port *ldap-port*))) + +(defmacro with-ldap ((&key (max-tries 1)) &body body) + "Execute BODY in a context where ldap connection errors trigger a reconnect +and a retry" + (with-gensyms (n try retry e) + `(flet + ((,try + (,n) + (flet ((,retry (,e) + (if (>= ,n ,max-tries) + (error ,e) + (progn + (reconnect-ldap) + (,try (1+ ,n)))))) + (handler-case + (progn + ,@body) + (end-of-file (,e) (,retry ,e)) + (trivial-ldap:ldap-connection-error (,e) (,retry ,e)))))) + (,try 0)))) + +(defun ldap-entry->user (entry) + (apply + #'make-instance + 'user + :dn (ldap:dn entry) + (alexandria:mappend + (lambda (field) + (list field (car (ldap:attr-value entry field)))) + (list :mail + :cn + :displayname)))) + +(defun find-user/ldap (username) + (check-type username (simple-array character (*))) + (with-ldap () + (ldap:search + *ldap* + `(and (= objectClass organizationalPerson) + (or + (= cn ,username) + (= dn ,username))) + ;; TODO(grfn): make this configurable + :base "ou=users,dc=tvl,dc=fyi") + (ldap:next-search-result *ldap*))) + +(defun find-user (username) + (check-type username (simple-array character (*))) + (when-let ((ldap-entry (find-user/ldap username))) + (ldap-entry->user ldap-entry))) + +(defun find-user-by-dn (dn) + "Look up the user with the given DN in the LDAP database, returning an +instance of `user'" + (with-ldap () + (let ((have-results + (handler-case + (ldap:search *ldap* `(= objectClass organizationalPerson) + :base dn + :scope 'ldap:base) + ; catch ldap-errors generated by trivial-ldap:parse-ldap-message + ; since this is thrown on conditions which we don't want this + ; function to fail like when there are no search results + (trivial-ldap:ldap-error (e) nil)))) + (when have-results + (when-let ((ldap-entry (ldap:next-search-result *ldap*))) + (ldap-entry->user ldap-entry)))))) + +(comment + (find-user-by-dn "cn=grfn,ou=users,dc=tvl,dc=fyi") + ) + +(defun authenticate-user (user-or-username password) + "Checks the given USER-OR-USERNAME has the given PASSWORD, by making a bind +request against the ldap server at *ldap*. Returns the user if authentication is +successful, `nil' otherwise" + (when-let ((user (if (typep user-or-username 'user) user-or-username + (find-user user-or-username)))) + (let ((dn (dn user))) + (let ((code-sym + (nth-value 1 (ldap:bind + (ldap:new-ldap :host (ldap:host *ldap*) + :port (ldap:port *ldap*) + :user dn + :pass password))))) + (when (equalp code-sym 'trivial-ldap:success) + user))))) diff --git a/web/panettone/src/css.lisp b/web/panettone/src/css.lisp new file mode 100644 index 000000000000..aa753cb50fc5 --- /dev/null +++ b/web/panettone/src/css.lisp @@ -0,0 +1,223 @@ +(in-package :panettone.css) +(declaim (optimize (safety 3))) + +(defun button (selector) + `((,selector + :background-color "var(--success)" + :padding "0.5rem" + :text-decoration "none" + :transition "box-shadow" "0.15s" "ease-in-out") + + ((:and ,selector :hover) + :box-shadow "0.25rem" "0.25rem" "0" "0" "rgba(0,0,0,0.08)") + + ((:and ,selector (:or :active :focus)) + :box-shadow "0.1rem" "0.1rem" "0" "0" "rgba(0,0,0,0.05)" + :outline "none" + :border "none"))) + +(defparameter markdown-styles + `((blockquote + :border-left "5px" "solid" "var(--light)"-gray + :padding-left "1rem" + :margin-left "0rem") + (pre + :overflow-x "auto"))) + +(defparameter issue-list-styles + `((.issue-list + :list-style-type "none" + :padding-left 0 + + (.issue-subject + :font-weight "bold") + + (li + :padding-bottom "1rem") + + ((li + li) + :border-top "1px" "solid" "var(--gray)") + + (a + :text-decoration "none" + :display "block") + + ((:and a :hover) + :outline "none" + + (.issue-subject + :color "var(--primary)"))) + + (.comment-count + :color "var(--gray)"))) + +(defparameter issue-history-styles + `((.issue-history + :list-style "none" + :border-top "1px" "solid" "var(--gray)" + :padding-top "1rem" + :padding-left "2rem" + + (.comment-info + :color "var(--gray)" + :margin 0 + :padding-top "1rem" + + (a :text-decoration "none") + ((:and a :hover) + :text-decoration "underline")) + + ((:or .comment .event) + :padding-top "1rem" + :padding-bottom "1rem" + :border-bottom "1px" "solid" "var(--gray)" + + (p :margin 0)) + + ((:and (:or .comment .event) :target) + :border-color "var(--primary)" + :border-bottom-width "3px") + + (.event + :color "var(--gray)")))) + +(defparameter form-styles + `(((:or (:and input (:or (:= type "text") + (:= type "password"))) + textarea) + :width "100%" + :padding "0.5rem" + :outline "none" + :border-top "none" + :border-left "none" + :border-right "none" + :border-bottom "1px" "solid" "var(--gray)" + :margin-bottom "1rem") + + (textarea + :resize "vertical") + + ((:and input (:= type "submit")) + :-webkit-appearance "none" + :border "none" + :cursor "pointer" + :font-size "1rem") + + ,@(button '(:and input (:= type "submit"))) + + (.form-link + ((:and input (:= type "submit")) + :background-color "initial" + :color "inherit" + :padding 0 + :text-decoration "underline") + + ((:and input (:= type "submit") + (:or :hover :active :focus)) + :box-shadow 0 0 0 0)) + + (.form-group + :margin-top "1rem") + + (label.checkbox + :cursor "pointer"))) + +(defparameter issue-styles + `((.issue-info + :display "flex" + :justify-content "space-between" + :align-items "center" + + ,@(button '.edit-issue) + + (.created-by-at + :flex 1) + + (.edit-issue + :background-color "var(--light)"-gray + :flex 0 + :margin-right "0.5rem") + + (.close-issue + :background-color "var(--failure)")))) + +(defparameter styles + `(,@form-styles + ,@issue-list-styles + ,@issue-styles + ,@issue-history-styles + ,@markdown-styles + + (body + :font-family "sans-serif" + :color "var(--text)" + :background "var(--bg)" + :--text "rgb(24, 24, 24)" + :--bg "white" + :--gray "#8D8D8D" + :--primary "rgb(106, 154, 255)" + :--primary-light "rgb(150, 166, 200)" + :--success "rgb(168, 249, 166)" + :--failure "rgb(247, 167, 167)" + :--light-gray "#EEE") + + (:media "(prefers-color-scheme: dark)" + (body + :--text "rgb(240, 240, 240)" + :--bg "black" + :--gray "#8D8D8D" + :--primary "rgb(106, 154, 255)" + :--primary-light "rgb(150, 166, 200)" + :--success "rgb(14, 130, 11)" + :--failure "rgb(124, 14, 14)" + :--light-gray "#222")) + + (a :color "inherit") + + (.content + :max-width "800px" + :margin "0 auto") + + (header + :display "flex" + :align-items "center" + :border-bottom "1px" "solid" "var(--text)" + :margin-bottom "1rem" + + (h1 + :padding 0 + :flex 1) + + (.issue-number + :color "var(--gray)" + :font-size "1.5rem")) + + (nav + :display "flex" + :color "var(--gray)" + :justify-content "space-between" + + (.nav-group + :display "flex" + (>* + :margin-left "0.5rem"))) + + (footer + :border-top "1px" "solid" "var(--gray)" + :padding-top "1rem" + :margin-top "1rem" + :color "var(--gray)") + + ,@(button '.new-issue) + + (.alert + :padding "0.5rem" + :margin-bottom "1rem" + :background-color "var(--failure)") + + (.login-form + :max-width "300px" + :margin "0 auto") + + (.created-by-at + :color "var(--gray)"))) diff --git a/web/panettone/src/email.lisp b/web/panettone/src/email.lisp new file mode 100644 index 000000000000..cb01c488a2a6 --- /dev/null +++ b/web/panettone/src/email.lisp @@ -0,0 +1,48 @@ +(in-package :panettone.email) +(declaim (optimize (safety 3))) + +(defvar *smtp-server* "localhost" + "The host for SMTP connections") + +(defvar *smtp-server-port* 2525 + "The port for SMTP connections") + +(defvar *notification-from* "tvlbot@tazj.in" + "The email address to send email notifications from") + +(defvar *notification-from-display-name* "Panettone" + "The Display Name to use when sending email notifications") + +(defvar *notification-subject-prefix* "[panettone]" + "String to prefix all email subjects with") + +(defun send-email-notification (&key to subject message) + "Sends an email to TO with the given SUBJECT and MESSAGE, using the current +values of `*smtp-server*', `*smtp-server-port*' and `*email-notification-from*'" + (let ((subject (if *notification-subject-prefix* + (format nil "~A ~A" + *notification-subject-prefix* + subject) + subject))) + (cl-smtp:send-email + *smtp-server* + *notification-from* + to + subject + message + :port *smtp-server-port* + :display-name *notification-from-display-name*))) + +(defun user-has-email-notifications-enabled-p (dn) + "Returns T if the user with the given DN has enabled email notifications" + (enable-email-notifications-p (settings-for-user dn))) + +(defun notify-user (dn &key subject message) + "Sends an email notification to the user with DN with the given SUBJECT and + MESSAGE, iff that user has not disabled email notifications" + (when (user-has-email-notifications-enabled-p dn) + (when-let ((user (find-user-by-dn dn))) + (send-email-notification + :to (mail user) + :subject subject + :message message)))) diff --git a/web/panettone/src/inline-markdown.lisp b/web/panettone/src/inline-markdown.lisp new file mode 100644 index 000000000000..fc5f77584fc4 --- /dev/null +++ b/web/panettone/src/inline-markdown.lisp @@ -0,0 +1,130 @@ +(in-package :panettone.inline-markdown) +(declaim (optimize (safety 3))) + +(define-constant +inline-markup-types+ + '(("~~" :del) + ("*" :em) + ("`" :code)) + :test #'equal) + +(defun next-token (mkdn &optional (escaped nil)) + "Parses and returns the next token from the beginning of + an inline markdown string which is not altered. The resulting + tokens are either :normal (normal text), :special (syntactically + significant) or :escaped (escaped using \\). If the string is + empty, a pseudo-token named :endofinput is returned. Return value + is a list where the first element is the token type, the second + the token content and optionally the third the markup type." + ; special tokens are syntactically significant characters + ; or strings for our inline markdown subset. “normal” tokens + ; the strings in between + (let* ((special-toks #.'(cons (list "\\" :escape) +inline-markup-types+)) + (toks (loop + for tok in special-toks + for pos = (search (car tok) mkdn) + when pos collect (cons tok pos))) + (next-tok + (unless (null toks) + (reduce (lambda (a b) (if (< (cdr a) (cdr b)) a b)) toks)))) + (cond + ; end of input + ((= (length mkdn) 0) (list :endofinput "")) + ; no special tokens, just return entire string + ((null next-tok) (list :normal mkdn)) + ; special token, but not at the beginning of the string + ; so we return everything until the special token as + ; a string + ((> (cdr next-tok) 0) (list :normal (subseq mkdn 0 (cdr next-tok)))) + ; \ at the beginning of the string: we get the next + ; token and mark it as escaped unless we are already + ; escaping in which case we just return the backslash + ; as a special token + ((eq (cadr (car next-tok)) :escape) + (if escaped + (list :special "\\") + (list :escaped + (next-token (subseq mkdn 1) t)))) + ; any other special token at the beginning of the string + ; here we also pass the markup type as a third list element + ; to prevent unnecessesary lookups + (t (list :special + (subseq mkdn 0 (length (car (car next-tok)))) + (cadr (car next-tok))))))) + +(defun token-length (tok-type tok-str) + "Returns the string length consumed by a call + to next-token returning the given token type and string." + (check-type tok-type symbol) + (if (eq tok-type :escaped) + ; backslash + length of escaped token + (progn + (check-type tok-str list) + (1+ (token-length (car tok-str) (cadr tok-str)))) + (progn + (check-type tok-str string) + (length tok-str)))) + +(defun write-tag (tag pos &optional (target *standard-output*)) + "Wrapper around who:convert-tag-to-string-list to + only output a single :opening or :closing tag." + (check-type tag symbol) + (check-type pos symbol) + (let + ((index + (cond + ((eq pos :opening) 0) + ((eq pos :closing) 3) + (t (error 'simple-type-error))))) + (dolist + (tag-part (subseq + (who:convert-tag-to-string-list tag nil nil nil) + index (+ index 3))) + (write-string tag-part target)))) + +(defun render-inline-markdown (s &optional (target *standard-output*) (in :normal)) + "Render inline markdown, a subset of markdown safe to render + inside inline elements. The resulting html is directly written + to a specified stream or *standard-output* to integrate well + with cl-who." + (check-type s string) + (check-type target stream) + (loop + for (tok-type tok-str tok-markup) = (next-token s) + do (setq s (subseq s (token-length tok-type tok-str))) + when (eq tok-type :endofinput) + return "" + when (eq tok-type :normal) + do (write-string (who:escape-string tok-str) target) + when (eq tok-type :escaped) + do (progn + ; if normal tokens are escaped we treat the \ as if it were \\ + ; + ; TODO(sterni): maybe also use the :normal behavior in :code except for #\`. + (when (eq (car tok-str) :normal) + (write-char #\\ target)) + (write-string (who:escape-string (cadr tok-str)) target)) + when (eq tok-type :special) + do (cond + ; we are on the outer level and encounter a special token: + ; render surrounding tags and call ourselves to render + ; inner content. + ((eq in :normal) + (progn + (write-tag tok-markup :opening target) + (setq s (render-inline-markdown s target tok-markup)) + (write-tag tok-markup :closing target))) + ; we are on the inner level and encounter the token that initiated + ; our markup again, meaning we need to return to the outer level. + ; we return the remaining string to be consumed. + ((eq in tok-markup) (return s)) + ; remaining case: we are on the inner level and encounter different markup. + + ; we don't support nested markup for simplicity reasons, so instead we + ; just render any nested markdown tokens as if they were escaped. This + ; only eliminates the slight use case for nesting :em inside :del, but + ; shouldn't be too bad. As a side effect this is the precise behavior + ; we want for :code. + ; + ; TODO(sterni): maybe bring back the restart-based system which allowed + ; to skip nested tokens if desired. + (t (write-string (who:escape-string tok-str) target))))) diff --git a/web/panettone/src/irc.lisp b/web/panettone/src/irc.lisp new file mode 100644 index 000000000000..2ab72a2e391e --- /dev/null +++ b/web/panettone/src/irc.lisp @@ -0,0 +1,35 @@ +;;;; Using irccat to send IRC notifications + +(in-package :panettone.irc) + +(defun noping (s) + (format nil "~A~A~A" + (char s 0) + #\ZERO_WIDTH_SPACE + (subseq s 1))) + +(defun get-irccat-config () + "Reads the IRCCATHOST and IRCCATPORT environment variables, and returns them +as two values" + (destructuring-bind (host port) + (mapcar #'uiop:getenvp '("IRCCATHOST" "IRCCATPORT")) + (if (and host port) + (values host (parse-integer port)) + (values "localhost" 4722)))) + +(defun send-irc-notification (body &key channel) + "Sends BODY to the IRC channel CHANNEL (starting with #), +if an IRCCat server is configured (using the IRCCATHOST and IRCCATPORT +environment variables). +May signal a condition if sending fails." + (multiple-value-bind (irchost ircport) (get-irccat-config) + (when irchost + (let ((socket (socket-connect irchost ircport))) + (unwind-protect + (progn + (format (socket-stream socket) "~@[~A ~]~A~A~%" + channel + #\ZERO_WIDTH_SPACE + body) + (finish-output (socket-stream socket))) + (ignore-errors (socket-close socket))))))) diff --git a/web/panettone/src/model.lisp b/web/panettone/src/model.lisp new file mode 100644 index 000000000000..77a7cd6da92f --- /dev/null +++ b/web/panettone/src/model.lisp @@ -0,0 +1,421 @@ +(in-package :panettone.model) +(declaim (optimize (safety 3))) + +(defun connect-postgres (&key + (host (or (uiop:getenvp "PGHOST") "localhost")) + (user (or (uiop:getenvp "PGUSER") "panettone")) + (password (or (uiop:getenvp "PGPASSWORD") "password")) + (database (or (uiop:getenvp "PGDATABASE") "panettone")) + (port (or (integer-env "PGPORT") 5432))) + "Initialize the global postgresql connection for Panettone" + (postmodern:connect-toplevel database user password host :port port)) + +(defun make-thread + (function &rest args) + "Make a new thread as per `BORDEAUX-THREADS:MAKE-THREAD' but with its own, new +database connection." + (let ((spec `(,(or (uiop:getenvp "PGDATABASE") "panettone") + ,(or (uiop:getenvp "PGUSER") "panettone") + ,(or (uiop:getenvp "PGPASSWORD") "password") + ,(or (uiop:getenvp "PGHOST") "localhost") + :port ,(or (integer-env "PGPORT") 5432)))) + (apply #'bt:make-thread + (lambda () + (postmodern:call-with-connection spec function)) + args))) + +;;; +;;; Schema +;;; + +(defclass user-settings () + ((user-dn :col-type string :initarg :user-dn :accessor user-dn) + (enable-email-notifications + :col-type boolean + :initarg :enable-email-notifications + :accessor enable-email-notifications-p + :initform t + :col-default t)) + (:metaclass dao-class) + (:keys user-dn) + (:table-name user_settings) + (:documentation + "Panettone settings for an individual user DN")) + +(deftable (user-settings "user_settings") + (!dao-def)) + +(defun settings-for-user (dn) + "Retrieve the settings for the user with the given DN, creating a new row in + the database if not yet present" + (or + (car + (query-dao + 'user-settings + (:select '* :from 'user-settings :where (:= 'user-dn dn)))) + (insert-dao (make-instance 'user-settings :user-dn dn)))) + +(defun update-user-settings (settings &rest attrs) + "Update the fields of the settings for USER with the given ATTRS, which is a + plist of slot and value" + (check-type settings user-settings) + (when-let ((set-fields + (iter + (for slot in '(enable-email-notifications)) + (for new-value = (getf attrs slot)) + (appending + (progn + (setf (slot-value settings slot) new-value) + (list slot new-value)))))) + (execute + (sql-compile + `(:update user-settings + :set ,@set-fields + :where (:= user-dn ,(user-dn settings))))))) + + +(define-constant +issue-statuses+ '(:open :closed) + :test #'equal) + +(deftype issue-status () + "Type specifier for the status of an `issue'" + (cons 'member +issue-statuses+)) + +(defun ddl/create-issue-status () + "Issue DDL to create the `issue-status' type, if it doesn't exist" + (unless (query (:select (:exists (:select 1 + :from 'pg_type + :where (:= 'typname "issue_status")))) + :single) + (query (sql-compile + `(:create-enum issue-status ,+issue-statuses+))))) + +(defclass has-created-at () + ((created-at :col-type timestamp + :col-default (local-time:now) + :initarg :created-at + :accessor created-at)) + (:metaclass dao-class)) + +(defun created-at->timestamp (object) + (assert (slot-exists-p object 'created-at)) + (unless (or (not (slot-boundp object 'created-at)) + (typep (slot-value object 'created-at) 'local-time:timestamp)) + (setf (slot-value object 'created-at) + (local-time:universal-to-timestamp (created-at object))))) + +(defmethod initialize-instance :after + ((obj has-created-at) &rest initargs &key &allow-other-keys) + (declare (ignore initargs)) + (created-at->timestamp obj)) + +(defclass issue (has-created-at) + ((id :col-type serial :initarg :id :accessor id) + (subject :col-type string :initarg :subject :accessor subject) + (body :col-type string :initarg :body :accessor body :col-default "") + (author-dn :col-type string :initarg :author-dn :accessor author-dn) + (comments :type list :accessor issue-comments) + (events :type list :accessor issue-events) + (num-comments :type integer :accessor num-comments) + (status :col-type issue_status + :initarg :status + :accessor status + :initform :open + :col-default "open")) + (:metaclass dao-class) + (:keys id) + (:table-name issues) + (:documentation + "Issues are the primary entity in the Panettone database. An issue is + reported by a user, has a subject and an optional body, and can be either + open or closed")) + +(defmethod cl-postgres:to-sql-string ((kw (eql :open))) + (cl-postgres:to-sql-string "open")) +(defmethod cl-postgres:to-sql-string ((kw (eql :closed))) + (cl-postgres:to-sql-string "closed")) +(defmethod cl-postgres:to-sql-string ((ts local-time:timestamp)) + (cl-postgres:to-sql-string + (local-time:timestamp-to-unix ts))) + +(defmethod initialize-instance :after + ((issue issue) &rest initargs &key &allow-other-keys) + (declare (ignore initargs)) + (unless (symbolp (status issue)) + (setf (status issue) + (intern (string-upcase (status issue)) + "KEYWORD")))) + +(deftable issue (!dao-def)) + +(defclass issue-comment (has-created-at) + ((id :col-type integer :col-identity t :initarg :id :accessor id) + (body :col-type string :initarg :body :accessor body) + (author-dn :col-type string :initarg :author-dn :accessor author-dn) + (issue-id :col-type integer :initarg :issue-id :accessor :user-id)) + (:metaclass dao-class) + (:keys id) + (:table-name issue_comments) + (:documentation "Comments on an `issue'")) +(deftable (issue-comment "issue_comments") + (!dao-def) + (!foreign 'issues 'issue-id 'id :on-delete :cascade :on-update :cascade)) + +(defclass issue-event (has-created-at) + ((id :col-type integer :col-identity t :initarg :id :accessor id) + (issue-id :col-type integer + :initarg :issue-id + :accessor issue-id) + (acting-user-dn :col-type string + :initarg :acting-user-dn + :accessor acting-user-dn) + (field :col-type (or string db-null) + :initarg :field + :accessor field) + (previous-value :col-type (or string db-null) + :initarg :previous-value + :accessor previous-value) + (new-value :col-type (or string db-null) + :initarg :new-value + :accessor new-value)) + (:metaclass dao-class) + (:keys id) + (:table-name issue_events) + (:documentation "Events that have occurred for an issue. + +If a field has been changed on an issue, the SYMBOL-NAME of that slot will be in +FIELD, its previous value will be formatted using ~A into PREVIOUS-VALUE, and +its new value will be formatted using ~A into NEW-VALUE")) + +(deftable (issue-event "issue_events") + (!dao-def) + (!foreign 'issues 'issue-id 'id :on-delete :cascade :on-update :cascade)) + +(define-constant +all-tables+ + '(issue + issue-comment + issue-event + user-settings) + :test #'equal) + +(defun ddl/create-tables () + "Issue DDL to create all tables, if they don't already exist." + (dolist (table +all-tables+) + (unless (table-exists-p (dao-table-name table)) + (create-table table)))) + +(defun ddl/init () + "Idempotently initialize the full database schema for Panettone" + (ddl/create-issue-status) + (ddl/create-tables)) + +;;; +;;; Querying +;;; + +(define-condition issue-not-found (error) + ((id :type integer + :initarg :id + :reader not-found-id + :documentation "ID of the issue that was not found")) + (:documentation + "Error condition for when an issue requested by ID is not found")) + +(defun get-issue (id) + "Look up the 'issue with the given ID and return it, or signal a condition of +type `ISSUE-NOT-FOUND'." + (restart-case + (or (get-dao 'issue id) + (error 'issue-not-found :id id)) + (different-id (new-id) + :report "Use a different issue ID" + :interactive (lambda () + (format t "Enter a new ID: ") + (multiple-value-list (eval (read)))) + (get-issue new-id)))) + +(defun issue-exists-p (id) + "Returns `T' if an issue with the given ID exists" + (query + (:select (:exists (:select 1 + :from 'issues + :where (:= 'id id)))) + :single)) + +(defun list-issues (&key status (with '(:num-comments))) + "Return a list of all issues with the given STATUS (or all if nil), ordered by + ID descending. If WITH contains `:NUM-COMMENTS' (the default) each issue will + have the `num-comments' slot filled with the number of comments on that issue + (to avoid N+1 queries)." + (let* ((condition (unless (null status) + `(:where (:= status $1)))) + (select (if (find :num-comments with) + `(:select issues.* (:as (:count issue-comments.id) + num-comments) + :from issues + :left-join issue-comments + :on (:= issues.id issue-comments.issue-id) + ,@condition + :group-by issues.id) + `(:select * :from issues ,@condition))) + (query (sql-compile + `(:order-by ,select (:desc id))))) + (with-column-writers ('num_comments 'num-comments) + (query-dao 'issue query status)))) + +(defmethod num-comments ((issue-id integer)) + "Return the number of comments for the given ISSUE-ID." + (query + (:select (:count '*) + :from 'issue-comments + :where (:= 'issue-id issue-id)) + :single)) + +(defmethod slot-unbound (cls (issue issue) (slot (eql 'comments))) + (declare (ignore cls) (ignore slot)) + (setf (issue-comments issue) (issue-comments (id issue)))) + +(defmethod issue-comments ((issue-id integer)) + "Return a list of all comments with the given ISSUE-ID, sorted oldest first. +NOTE: This makes a database query, so be wary of N+1 queries" + (query-dao + 'issue-comment + (:order-by + (:select '* + :from 'issue-comments + :where (:= 'issue-id issue-id)) + (:asc 'created-at)))) + +(defmethod slot-unbound (cls (issue issue) (slot (eql 'events))) + (declare (ignore cls) (ignore slot)) + (setf (issue-events issue) (issue-events (id issue)))) + +(defmethod issue-events ((issue-id integer)) + "Return a list of all events with the given ISSUE-ID, sorted oldest first. +NOTE: This makes a database query, so be wary of N+1 queries" + (query-dao + 'issue-event + (:order-by + (:select '* + :from 'issue-events + :where (:= 'issue-id issue-id)) + (:asc 'created-at)))) + + +;;; +;;; Writing +;;; + +(defun record-issue-event + (issue-id &key + field + previous-value + new-value) + "Record in the database that the user identified by `AUTHN:*USER*' updated +ISSUE-ID, and return the resulting `ISSUE-EVENT'. If no user is currently +authenticated, warn and no-op" + (check-type issue-id (integer)) + (check-type field (or null symbol)) + (if authn:*user* + (insert-dao + (make-instance 'issue-event + :issue-id issue-id + :acting-user-dn (authn:dn authn:*user*) + :field (symbol-name field) + :previous-value (when previous-value + (format nil "~A" previous-value)) + :new-value (when new-value + (format nil "~A" new-value)))) + (warn "Performing operation as unauthenticated user"))) + +(defun create-issue (&rest attrs) + "Insert a new issue into the database with the given ATTRS, which should be +a plist of initforms, and return an instance of `issue'" + (insert-dao (apply #'make-instance 'issue attrs))) + +(defun delete-issue (issue) + (delete-dao issue)) + +(defun set-issue-status (issue-id status) + "Set the status of the issue with the given ISSUE-ID to STATUS in the db. If +the issue doesn't exist, signals `issue-not-found'" + (check-type issue-id integer) + (check-type status issue-status) + (let ((original-status (query (:select 'status + :from 'issues + :where (:= 'id issue-id)) + :single))) + (when (zerop (execute (:update 'issues + :set 'status (cl-postgres:to-sql-string status) + :where (:= 'id issue-id)))) + (error 'issue-not-found :id issue-id)) + (record-issue-event + issue-id + :field 'status + :previous-value (string-upcase original-status) + :new-value status) + (values))) + +(defun update-issue (issue &rest attrs) + "Update the fields of ISSUE with the given ATTRS, which is a plist of slot and +value, and record events for the updates" + (let ((set-fields + (iter (for slot in '(subject body)) + (for new-value = (getf attrs slot)) + (appending + (let ((previous-value (slot-value issue slot))) + (when (and new-value (not (equalp + new-value + previous-value))) + (record-issue-event (id issue) + :field slot + :previous-value previous-value + :new-value new-value) + (setf (slot-value issue slot) new-value) + (list slot new-value))))))) + (execute + (sql-compile + `(:update issues + :set ,@set-fields + :where (:= id ,(id issue))))))) + +(defun create-issue-comment (&rest attrs &key issue-id &allow-other-keys) + "Insert a new issue comment into the database with the given ATTRS and +ISSUE-ID, which should be a plist of initforms, and return an instance of +`issue-comment'. If no issue exists with `ID' ISSUE-ID, signals +`issue-not-found'." + (unless (issue-exists-p issue-id) + (error 'issue-not-found :id issue-id)) + (insert-dao (apply #'make-instance 'issue-comment :issue-id issue-id attrs))) + +(defun issue-commenter-dns (issue-id) + "Returns a list of all the dns of users who have commented on ISSUE-ID" + (query (:select 'author-dn :distinct + :from 'issue-comments + :where (:= 'issue-id issue-id)) + :column)) + +(defun issue-subscribers (issue-id) + "Returns a list of user DNs who should receive notifications for actions taken + on ISSUE-ID. + +Currently this is implemented as the author of issue plus all the users who have +commented on the issue, but in the future we likely want to also allow +explicitly subscribing to / unsubscribing from individual issues." + (let ((issue (get-issue issue-id))) + (adjoin (author-dn issue) + (issue-commenter-dns issue-id) + :test #'equal))) + + +(comment + (connect-postgres) + (ddl/init) + (make-instance 'issue :subject "test") + (create-issue :subject "test" + :author-dn "cn=grfn,ou=users,dc=tvl,dc=fyi") + + (issue-commenter-dns 1) + (issue-subscribers 1) + + ) diff --git a/web/panettone/src/packages.lisp b/web/panettone/src/packages.lisp new file mode 100644 index 000000000000..b0833e454135 --- /dev/null +++ b/web/panettone/src/packages.lisp @@ -0,0 +1,82 @@ +(defpackage panettone.util + (:use :cl :klatre) + (:import-from :alexandria :when-let) + (:export :integer-env)) + +(defpackage panettone.css + (:use :cl :lass) + (:export :styles)) + +(defpackage panettone.inline-markdown + (:use :cl) + (:import-from :alexandria :define-constant) + (:export :render-inline-markdown)) + +(defpackage panettone.irc + (:nicknames :irc) + (:use :cl :usocket) + (:export :noping :send-irc-notification)) + +(defpackage :panettone.authentication + (:nicknames :authn) + (:use :cl :panettone.util :klatre) + (:import-from :defclass-std :defclass/std) + (:import-from :alexandria :when-let :with-gensyms) + (:export + :*user* :*ldap* + :user :cn :dn :mail :displayname + :connect-ldap :find-user :find-user-by-dn :authenticate-user)) + +(defpackage panettone.model + (:nicknames :model) + (:use :cl :panettone.util :klatre :postmodern :iterate) + (:import-from :alexandria :if-let :when-let :define-constant) + (:export + :connect-postgres :ddl/init :make-thread + + :user-settings + :user-dn :enable-email-notifications-p :settings-for-user + :update-user-settings :enable-email-notifications + + :issue :issue-comment :issue-event + :id :subject :body :author-dn :issue-id :status :created-at :acting-user-dn + :field :previous-value :new-value + + :get-issue :issue-exists-p :list-issues :create-issue :set-issue-status + :update-issue :delete-issue :issue-not-found :not-found-id + + :issue-events + + :issue-comments :num-comments :create-issue-comment + :issue-commenter-dns :issue-subscribers)) + +(defpackage panettone.email + (:nicknames :email) + (:use :cl) + (:import-from :alexandria :when-let) + (:import-from :panettone.model + :settings-for-user :enable-email-notifications-p) + (:import-from :panettone.authentication + :find-user-by-dn :mail :displayname) + (:export + :*smtp-server* :*smtp-server-port* :*notification-from* + :*notification-from-display-name* :*notification-subject-prefix* + :notify-user :send-email-notification)) + +(defpackage panettone + (:use :cl :klatre :easy-routes :iterate + :panettone.util + :panettone.authentication + :panettone.inline-markdown) + (:import-from :defclass-std :defclass/std) + (:import-from :alexandria :if-let :when-let :switch :alist-hash-table) + (:import-from :cl-ppcre :split) + (:import-from :bordeaux-threads :make-thread) + (:import-from + :panettone.model + :id :subject :body :author-dn :issue-id :status :created-at + :field :previous-value :new-value :acting-user-dn + :issue-comments :num-comments :issue-events) + (:import-from :panettone.irc :send-irc-notification) + (:shadow :next) + (:export :start-pannetone :config :main)) diff --git a/web/panettone/src/panettone.lisp b/web/panettone/src/panettone.lisp new file mode 100644 index 000000000000..b16171242dde --- /dev/null +++ b/web/panettone/src/panettone.lisp @@ -0,0 +1,684 @@ +(in-package :panettone) +(declaim (optimize (safety 3))) + +(defvar *cheddar-url* "http://localhost:4238") + +(defgeneric render-markdown (markdown) + (:documentation + "Render the argument, or the elements of the argument, as markdown, and return + the same structure")) + +(defun request-markdown-from-cheddar (input) + "Send the CL value INPUT encoded as JSON to cheddar's + markdown endpoint and return the decoded response." + (let ((s (drakma:http-request + (concatenate 'string + *cheddar-url* + "/markdown") + :accept "application/json" + :method :post + :content-type "application/json" + :external-format-out :utf-8 + :content (json:encode-json-to-string input) + :want-stream t))) + (setf (flexi-streams:flexi-stream-external-format s) :utf-8) + (cl-json:decode-json s))) + +(defmethod render-markdown ((markdown string)) + (cdr (assoc :markdown + (request-markdown-from-cheddar + `((markdown . ,markdown)))))) + +(defmethod render-markdown ((markdown hash-table)) + (alist-hash-table + (request-markdown-from-cheddar markdown))) + +(defun markdownify-comment-bodies (comments) + "Convert the bodies of the given list of comments to markdown in-place using + Cheddar, and return nothing" + (let ((in (make-hash-table)) + (comment-table (make-hash-table))) + (dolist (comment comments) + (when (typep comment 'model:issue-comment) + (setf (gethash (id comment) in) (body comment)) + (setf (gethash (id comment) comment-table) comment))) + (let ((res (render-markdown in))) + (iter (for (comment-id markdown-body) in-hashtable res) + (let ((comment-id (parse-integer (symbol-name comment-id)))) + (setf (slot-value (gethash comment-id comment-table) + 'model:body) + markdown-body))))) + (values)) + +;;; +;;; Views +;;; + +(defvar *title* "Panettone") + +(setf (who:html-mode) :html5) + +(defun render/nav () + (who:with-html-output (*standard-output*) + (:nav + (if (find (car (split "\\?" (hunchentoot:request-uri*) :limit 2)) + (list "/" "/issues/closed") + :test #'string=) + (who:htm (:span :class "placeholder")) + (who:htm (:a :href "/" "All Issues"))) + (if *user* + (who:htm + (:div :class "nav-group" + (:a :href "/settings" "Settings") + (:form :class "form-link log-out" + :method "post" + :action "/logout" + (:input :type "submit" :value "Log Out")))) + (who:htm + (:a :href + (format nil + "/login?original-uri=~A" + (drakma:url-encode (hunchentoot:request-uri*) + :utf-8)) + "Log In")))))) + +(defun author (object) + (find-user-by-dn (author-dn object))) + +(defun displayname-if-known (user) + (or (when user (displayname user)) + "unknown")) + +(defmacro render ((&key + (footer t) + (header t)) + &body body) + `(who:with-html-output-to-string (*standard-output* nil :prologue t) + (:html + :lang "en" + (:head + (:title (who:esc *title*)) + (:link :rel "stylesheet" :type "text/css" :href "/main.css") + (:meta :name "viewport" + :content "width=device-width,initial-scale=1")) + (:body + (:div + :class "content" + (when ,header + (who:htm + (render/nav))) + ,@body + (when ,footer + (who:htm + (:footer + (render/nav))))))))) + +(defun form-button (&key + class + input-class + href + label + (method "post")) + (who:with-html-output (*standard-output*) + (:form :class class + :method method + :action href + (:input :type "submit" + :class input-class + :value label)))) + +(defun render/alert (message) + "Render an alert box for MESSAGE, if non-null" + (check-type message (or null string)) + (who:with-html-output (*standard-output*) + (when message + (who:htm (:div :class "alert" (who:esc message)))))) + +(defun render/login (&key message (original-uri "/")) + (render (:footer nil :header nil) + (:div + :class "login-form" + (:header + (:h1 "Login")) + (:main + :class "login-form" + (render/alert message) + (:form + :method :post :action "/login" + (:input :type "hidden" :name "original-uri" + :value (who:escape-string original-uri)) + (:div + (:label :for "username" + "Username") + (:input :type "text" + :name "username" + :id "username" + :placeholder "username")) + (:div + (:label :for "password" + "Password") + (:input :type "password" + :name "password" + :id "password" + :placeholder "password")) + (:input :type "submit" + :value "Submit")))))) + +(defun render/settings () + (let ((settings (model:settings-for-user (dn *user*)))) + (render () + (:div + :class "settings-page" + (:header + (:h1 "Settings")) + (:form + :method :post :action "/settings" + (:div + (:label :class "checkbox" + (:input :type "checkbox" + :name "enable-email-notifications" + :id "enable-email-notifications" + :checked (model:enable-email-notifications-p + settings)) + "Enable Email Notifications")) + (:div :class "form-group" + (:input :type "submit" + :value "Save Settings"))))))) + +(defun created-by-at (issue) + (check-type issue model:issue) + (who:with-html-output (*standard-output*) + (:span :class "created-by-at" + "Opened by " + (:span :class "username" + (who:esc (displayname-if-known + (author issue)))) + " at " + (:span :class "timestamp" + (who:esc + (format-dottime (created-at issue))))))) + +(defun render/issue-list (&key issues) + (who:with-html-output (*standard-output*) + (:ol + :class "issue-list" + (dolist (issue issues) + (let ((issue-id (model:id issue))) + (who:htm + (:li + (:a :href (format nil "/issues/~A" issue-id) + (:p + (:span :class "issue-subject" + (render-inline-markdown (subject issue)))) + (:span :class "issue-number" + (who:esc (format nil "#~A" issue-id))) + " - " + (created-by-at issue) + (let ((num-comments (length (issue-comments issue)))) + (unless (zerop num-comments) + (who:htm + (:span :class "comment-count" + " - " + (who:esc + (format nil "~A comment~:p" num-comments)))))))))))))) + +(defun render/index (&key issues) + (render () + (:header + (:h1 "Issues") + (when *user* + (who:htm + (:a + :class "new-issue" + :href "/issues/new" "New Issue")))) + (:main + (:div + :class "issue-links" + (:a :href "/issues/closed" "View closed issues")) + (render/issue-list :issues issues)))) + +(defun render/closed-issues (&key issues) + (render () + (:header + (:h1 "Closed issues")) + (:main + (:div + :class "issue-links" + (:a :href "/" "View open isues")) + (render/issue-list :issues issues)))) + +(defun render/issue-form (&optional issue message) + (let ((editing (and issue (id issue)))) + (render () + (:header + (:h1 + (who:esc + (if editing "Edit Issue" "New Issue")))) + (:main + (render/alert message) + (:form :method "post" + :action (if editing + (format nil "/issues/~A" + (id issue)) + "/issues") + :class "issue-form" + (:div + (:input :type "text" + :id "subject" + :name "subject" + :placeholder "Subject" + :value (when editing + (who:escape-string + (subject issue))))) + + (:div + (:textarea :name "body" + :placeholder "Description" + :rows 10 + (who:esc + (when editing + (body issue))))) + + (:input :type "submit" + :value + (if editing + "Save Issue" + "Create Issue"))))))) + +(defun render/new-comment (issue-id) + (who:with-html-output (*standard-output*) + (:form + :class "new-comment" + :method "post" + :action (format nil "/issues/~A/comments" issue-id) + (:div + (:textarea :name "body" + :placeholder "Leave a comment" + :rows 5)) + (:input :type "submit" + :value "Comment")))) + +(defgeneric render/issue-history-item (item)) + +(defmethod render/issue-history-item ((comment model:issue-comment)) + (let ((fragment (format nil "comment-~A" (id comment)))) + (who:with-html-output (*standard-output*) + (:li + :class "comment" + :id fragment + (:p (who:str (body comment))) + (:p + :class "comment-info" + (:span :class "username" + (who:esc + (displayname-if-known (author comment))) + " at " + (:a :href (concatenate 'string "#" fragment) + (who:esc (format-dottime (created-at comment)))))))))) + +(defmethod render/issue-history-item ((event model:issue-event)) + (let ((user (find-user-by-dn (acting-user-dn event))) + (fragment (format nil "event-~A" (id event)))) + (who:with-html-output (*standard-output*) + (:li + :class "event" + :id fragment + (who:esc (displayname-if-known user)) + (switch ((field event) :test #'string=) + ("STATUS" + (who:htm + (who:esc + (switch ((new-value event) :test #'string=) + ("OPEN" " reopened ") + ("CLOSED" " closed "))) + " this issue ")) + ("BODY" (who:htm " updated the body of this issue")) + (t + (who:htm + " changed the " + (who:esc (string-downcase (field event))) + " of this issue from \"" + (who:esc (previous-value event)) + "\" to \"" + (who:esc (new-value event)) + "\""))) + " at " + (who:esc (format-dottime (created-at event))))))) + +(defun render/issue (issue) + (check-type issue model:issue) + (let ((issue-id (id issue)) + (issue-status (status issue))) + (render () + (:header + (:h1 (render-inline-markdown (subject issue))) + (:div :class "issue-number" + (who:esc (format nil "#~A" issue-id)))) + (:main + (:div + :class "issue-info" + (created-by-at issue) + + (when *user* + (who:htm + (when (string= (author-dn issue) + (dn *user*)) + (who:htm + (:a :class "edit-issue" + :href (format nil "/issues/~A/edit" + issue-id) + "Edit"))) + (form-button + :class "set-issue-status" + :href (format nil "/issues/~A/~A" + issue-id + (case issue-status + (:open "close") + (:closed "open"))) + :input-class (case issue-status + (:open "close-issue") + (:closed "open-issue")) + :label (case issue-status + (:open "Close") + (:closed "Reopen")))))) + (:p (who:str (render-markdown (body issue)))) + (let* ((comments (issue-comments issue)) + (events (issue-events issue)) + (history (merge 'list + comments + events + #'local-time:timestamp< + :key #'created-at))) + (markdownify-comment-bodies comments) + (who:htm + (:ol + :class "issue-history" + (dolist (item history) + (render/issue-history-item item)) + (when *user* + (render/new-comment (id issue)))))))))) + +(defun render/not-found (entity-type) + (render () + (:h1 (who:esc entity-type) " Not Found"))) + +;;; +;;; HTTP handlers +;;; + +(defun send-email-for-issue + (issue-id &key subject (message "")) + "Send an email notification to all subscribers to the given issue with the +given subject an body (in a thread, to avoid blocking)" + (let ((current-user *user*)) + (model:make-thread + (lambda () + (dolist (user-dn (model:issue-subscribers issue-id)) + (when (not (equal (dn current-user) user-dn)) + (email:notify-user + user-dn + :subject subject + :message message))))))) + +(defun link-to-issue (issue-id) + (format nil "https://b.tvl.fyi/issues/~A" issue-id)) + +(defun @auth-optional (next) + (let ((*user* (hunchentoot:session-value 'user))) + (funcall next))) + +(defun @auth (next) + (if-let ((*user* (hunchentoot:session-value 'user))) + (funcall next) + (hunchentoot:redirect + (format nil "/login?original-uri=~A" + (drakma:url-encode + (hunchentoot:request-uri*) + :utf-8))))) + +(defun @txn (next) + (pomo:with-transaction () + (catch + ;; 'hunchentoot:handler-done is unexported, but is used by functions + ;; like hunchentoot:redirect to nonlocally abort the request handler - + ;; this doesn't mean an error occurred, so we need to catch it here to + ;; make the transaction still get committed + (intern "HANDLER-DONE" "HUNCHENTOOT") + (funcall next)))) + +(defun @handle-issue-not-found (next) + (handler-case (funcall next) + (model:issue-not-found (err) + (render/not-found + (format nil "Issue #~A" (model:not-found-id err)))))) + +(defroute login-form ("/login" :method :get) + (original-uri) + (if (hunchentoot:session-value 'user) + (hunchentoot:redirect (or original-uri "/")) + (render/login :original-uri original-uri))) + +(defroute submit-login ("/login" :method :post) + (&post original-uri username password) + (if-let ((user (authenticate-user username password))) + (progn + (setf (hunchentoot:session-value 'user) user) + (hunchentoot:redirect (or original-uri "/"))) + (render/login :message "Invalid credentials" + :original-uri original-uri))) + +(defroute logout ("/logout" :method :post) () + (hunchentoot:delete-session-value 'user) + (hunchentoot:redirect "/")) + +(defroute index ("/" :decorators (@auth-optional)) () + (let ((issues (model:list-issues :status :open))) + (render/index :issues issues))) + +(defroute settings ("/settings" :method :get :decorators (@auth)) () + (render/settings)) + +(defroute save-settings ("/settings" :method :post :decorators (@auth)) + (&post enable-email-notifications) + (let ((settings (model:settings-for-user (dn *user*)))) + (model:update-user-settings + settings + 'model:enable-email-notifications enable-email-notifications) + (render/settings))) + +(defroute handle-closed-issues + ("/issues/closed" :decorators (@auth-optional)) () + (let ((issues (model:list-issues :status :closed))) + (render/closed-issues :issues issues))) + +(defroute new-issue ("/issues/new" :decorators (@auth)) () + (render/issue-form)) + +(defroute handle-create-issue + ("/issues" :method :post :decorators (@auth @txn)) + (&post subject body) + (if (string= subject "") + (render/issue-form + (make-instance 'model:issue :subject subject :body body) + "Subject is required") + (let ((issue + (model:create-issue :subject subject + :body body + :author-dn (dn *user*)))) + (send-irc-notification + (format nil + "b/~A: \"~A\" opened by ~A - https://b.tvl.fyi/issues/~A" + (id issue) + subject + (irc:noping (cn *user*)) + (id issue)) + :channel (or (uiop:getenvp "ISSUECHANNEL") + "#tvl")) + (hunchentoot:redirect "/")))) + +(defroute show-issue + ("/issues/:id" :decorators (@auth-optional @handle-issue-not-found)) + (&path (id 'integer)) + (let* ((issue (model:get-issue id)) + (*title* (format nil "~A | Panettone" + (subject issue)))) + (render/issue issue))) + +(defroute edit-issue + ("/issues/:id/edit" :decorators (@auth @handle-issue-not-found)) + (&path (id 'integer)) + (let* ((issue (model:get-issue id)) + (*title* "Edit Issue | Panettone")) + (render/issue-form issue))) + +(defroute update-issue + ("/issues/:id" :decorators (@auth @handle-issue-not-found @txn) + ;; NOTE: this should be a put, but we're all HTML forms + ;; right now and those don't support PUT + :method :post) + (&path (id 'integer) &post subject body) + (let ((issue (model:get-issue id))) + ;; only the original author can edit an issue + (if (string-equal (author-dn issue) + (dn *user*)) + (progn + (model:update-issue issue + 'model:subject subject + 'model:body body) + (hunchentoot:redirect (format nil "/issues/~A" id))) + (render/not-found "Issue")))) + +(defroute handle-create-comment + ("/issues/:id/comments" + :decorators (@auth @handle-issue-not-found @txn) + :method :post) + (&path (id 'integer) &post body) + (flet ((redirect-to-issue () + (hunchentoot:redirect (format nil "/issues/~A" id)))) + (cond + ((string= body "") + (redirect-to-issue)) + (:else + (model:create-issue-comment + :issue-id id + :body body + :author-dn (dn *user*)) + + (let ((issue (model:get-issue id))) + (send-email-for-issue + id + :subject (format nil "~A commented on b/~A: \"~A\"" + (displayname *user*) + id + (subject issue)) + :message (format nil "~A~%~%~A" + body + (link-to-issue id)))) + (redirect-to-issue))))) + +(defroute close-issue + ("/issues/:id/close" :decorators (@auth @handle-issue-not-found @txn) + :method :post) + (&path (id 'integer)) + (model:set-issue-status id :closed) + (let ((issue (model:get-issue id))) + (send-irc-notification + (format nil + "b/~A: \"~A\" closed by ~A - ~A" + id + (subject issue) + (irc:noping (cn *user*)) + (link-to-issue id)) + :channel (or (uiop:getenvp "ISSUECHANNEL") + "#tvl")) + (send-email-for-issue + id + :subject (format nil "b/~A: \"~A\" closed by ~A" + id + (subject issue) + (displayname *user*)) + :message (link-to-issue id))) + (hunchentoot:redirect (format nil "/issues/~A" id))) + +(defroute open-issue + ("/issues/:id/open" :decorators (@auth) + :method :post) + (&path (id 'integer)) + (model:set-issue-status id :open) + (let ((issue (model:get-issue id))) + (send-irc-notification + (format nil + "b/~A: \"~A\" reopened by ~A - ~A" + id + (subject issue) + (irc:noping (cn *user*)) + (link-to-issue id)) + :channel (or (uiop:getenvp "ISSUECHANNEL") + "#tvl")) + (send-email-for-issue + id + :subject (format nil "b/~A: \"~A\" reopened by ~A" + id + (subject issue) + (displayname *user*)) + :message (link-to-issue id))) + (hunchentoot:redirect (format nil "/issues/~A" id))) + +(defroute styles ("/main.css") () + (setf (hunchentoot:content-type*) "text/css") + (apply #'lass:compile-and-write panettone.css:styles)) + +(defvar *acceptor* nil + "Hunchentoot acceptor for Panettone's web server.") + +(defun migrate-db () + "Migrate the database to the latest version of the schema" + (model:ddl/init)) + +(defun start-panettone (&key port + (ldap-host "localhost") + (ldap-port 389) + postgres-params + session-secret) + (connect-ldap :host ldap-host + :port ldap-port) + + (apply #'model:connect-postgres postgres-params) + (migrate-db) + + (when session-secret + (setq hunchentoot:*session-secret* session-secret)) + + (setq hunchentoot:*session-max-time* (* 60 60 24 90)) + + (setq *acceptor* + (make-instance 'easy-routes:routes-acceptor :port port)) + (hunchentoot:start *acceptor*)) + +(defun main () + (let ((port (integer-env "PANETTONE_PORT" :default 6161)) + (ldap-port (integer-env "LDAP_PORT" :default 389)) + (cheddar-url (uiop:getenvp "CHEDDAR_URL")) + (session-secret (uiop:getenvp "SESSION_SECRET"))) + (when cheddar-url (setq *cheddar-url* cheddar-url)) + (setq hunchentoot:*show-lisp-backtraces-p* nil) + (setq hunchentoot:*log-lisp-backtraces-p* nil) + + (start-panettone :port port + :ldap-port ldap-port + :session-secret session-secret) + + (sb-thread:join-thread + (find-if (lambda (th) + (string= (sb-thread:thread-name th) + (format nil "hunchentoot-listener-*:~A" port))) + (sb-thread:list-all-threads))))) + +(comment + (setq hunchentoot:*catch-errors-p* nil) + ;; to setup an ssh tunnel to ldap+cheddar+irccat for development: + ;; ssh -NL 3899:localhost:389 -L 4238:localhost:4238 -L 4722:localhost:4722 whitby.tvl.fyi + (start-panettone :port 6161 + :ldap-port 3899 + :session-secret "session-secret") + ) diff --git a/web/panettone/src/util.lisp b/web/panettone/src/util.lisp new file mode 100644 index 000000000000..9fd9ceaa79a4 --- /dev/null +++ b/web/panettone/src/util.lisp @@ -0,0 +1,7 @@ +(in-package :panettone.util) + +(defun integer-env (var &key default) + (or + (when-let ((str (uiop:getenvp var))) + (try-parse-integer str)) + default)) diff --git a/web/panettone/test/inline-markdown_test.lisp b/web/panettone/test/inline-markdown_test.lisp new file mode 100644 index 000000000000..a32464456d48 --- /dev/null +++ b/web/panettone/test/inline-markdown_test.lisp @@ -0,0 +1,54 @@ +(in-package :panettone.tests) +(declaim (optimize (safety 3))) + +(defmacro inline-markdown-unit-test (name input expected) + `(test ,name + (is (equal + ,expected + (with-output-to-string (*standard-output*) + (render-inline-markdown ,input)))))) + +(inline-markdown-unit-test + inline-markdown-typical-test + "hello *world*, here is ~~no~~ `code`!" + "hello <em>world</em>, here is <del>no</del> <code>code</code>!") + +(inline-markdown-unit-test + inline-markdown-two-emphasize-types-test + "*stress* *this*" + "<em>stress</em> <em>this</em>") + +(inline-markdown-unit-test + inline-markdown-html-escaping-test + "<tag>öäü" + "<tag>öäü") + +(inline-markdown-unit-test + inline-markdown-nesting-test + "`inside code *anything* goes`, but also ~~*here*~~" + "<code>inside code *anything* goes</code>, but also <del>*here*</del>") + +(inline-markdown-unit-test + inline-markdown-escaping-test + "A backslash \\\\ shows: \\*, \\` and \\~~" + "A backslash \\ shows: *, ` and ~~") + +(inline-markdown-unit-test + inline-markdown-nested-escaping-test + "`prevent \\`code\\` from ending, but never stand alone \\\\`" + "<code>prevent `code` from ending, but never stand alone \\</code>") + +(inline-markdown-unit-test + inline-markdown-escape-normal-tokens-test + "\\Normal tokens \\escaped?" + "\\Normal tokens \\escaped?") + +(inline-markdown-unit-test + inline-markdown-no-unclosed-tags-test + "A tag, once opened, *must be closed" + "A tag, once opened, <em>must be closed</em>") + +(inline-markdown-unit-test + inline-markdown-unicode-safe + "Does Unicode 👨👨👧👦 break \\👩🏾🦰 tokenization?" + "Does Unicode 👨‍👨‍👧‍👦 break \\👩🏾‍🦰 tokenization?") diff --git a/web/panettone/test/irc_test.lisp b/web/panettone/test/irc_test.lisp new file mode 100644 index 000000000000..0224836cbc32 --- /dev/null +++ b/web/panettone/test/irc_test.lisp @@ -0,0 +1,5 @@ +(in-package :panettone.tests) +(declaim (optimize (safety 3))) + +(test noping-test + (is (not (equal "grfn" (panettone.irc:noping "grfn"))))) diff --git a/web/panettone/test/model_test.lisp b/web/panettone/test/model_test.lisp new file mode 100644 index 000000000000..e4cd78a65a43 --- /dev/null +++ b/web/panettone/test/model_test.lisp @@ -0,0 +1,13 @@ +(in-package :panettone.tests) +(declaim (optimize (safety 3))) + +(test initialize-issue-status-test + (let ((issue (make-instance 'model:issue :status "open"))) + (is (eq :open (model:status issue))))) + +(test initialize-issue-created-at-test + (let* ((time (get-universal-time)) + (issue (make-instance 'model:issue :created-at time))) + (is (local-time:timestamp= + (local-time:universal-to-timestamp time) + (model:created-at issue))))) diff --git a/web/panettone/test/package.lisp b/web/panettone/test/package.lisp new file mode 100644 index 000000000000..d2a2f974208e --- /dev/null +++ b/web/panettone/test/package.lisp @@ -0,0 +1,3 @@ +(defpackage :panettone.tests + (:use :cl :klatre :fiveam + :panettone.inline-markdown)) diff --git a/web/static/default.nix b/web/static/default.nix new file mode 100644 index 000000000000..ea498057f9de --- /dev/null +++ b/web/static/default.nix @@ -0,0 +1,22 @@ +# Expose all static assets as a folder. The derivation contains a +# `drvHash` attribute which can be used for cache-busting. +{ depot, lib, pkgs, ... }: + +let + storeDirLength = with builtins; (stringLength storeDir) + 1; + logo = depot.web.tvl.logo; +in lib.fix(self: pkgs.runCommand "tvl-static" { + passthru = { + # Preserving the string context here makes little sense: While we are + # referencing this derivation, we are not doing so via the nix store, + # so it makes little sense for Nix to police the references. + drvHash = builtins.unsafeDiscardStringContext ( + lib.substring storeDirLength 32 self.drvPath + ); + }; +} '' + mkdir $out + cp -r ${./.}/* $out + cp ${logo.pastelRainbow} $out/logo-animated.svg + cp ${logo.redPng} $out/logo-red.png +'') diff --git a/web/static/favicon.webp b/web/static/favicon.webp new file mode 100644 index 000000000000..741cdbc64ff2 --- /dev/null +++ b/web/static/favicon.webp Binary files differdiff --git a/web/static/files/adisbladis_tazjin_tvix.webp b/web/static/files/adisbladis_tazjin_tvix.webp new file mode 100644 index 000000000000..38e004543da6 --- /dev/null +++ b/web/static/files/adisbladis_tazjin_tvix.webp Binary files differdiff --git a/web/static/files/flokli_tazjin_tvix.webp b/web/static/files/flokli_tazjin_tvix.webp new file mode 100644 index 000000000000..4c0b94903fda --- /dev/null +++ b/web/static/files/flokli_tazjin_tvix.webp Binary files differdiff --git a/web/static/jetbrains-mono-bold-italic.woff2 b/web/static/jetbrains-mono-bold-italic.woff2 new file mode 100644 index 000000000000..34b5c69ae1cf --- /dev/null +++ b/web/static/jetbrains-mono-bold-italic.woff2 Binary files differdiff --git a/web/static/jetbrains-mono-bold.woff2 b/web/static/jetbrains-mono-bold.woff2 new file mode 100644 index 000000000000..84a008af7edb --- /dev/null +++ b/web/static/jetbrains-mono-bold.woff2 Binary files differdiff --git a/web/static/jetbrains-mono-italic.woff2 b/web/static/jetbrains-mono-italic.woff2 new file mode 100644 index 000000000000..85fd4687891e --- /dev/null +++ b/web/static/jetbrains-mono-italic.woff2 Binary files differdiff --git a/web/static/jetbrains-mono.woff2 b/web/static/jetbrains-mono.woff2 new file mode 100644 index 000000000000..d5b94cb9e7db --- /dev/null +++ b/web/static/jetbrains-mono.woff2 Binary files differdiff --git a/web/static/tvl.css b/web/static/tvl.css new file mode 100644 index 000000000000..aea4d426eac3 --- /dev/null +++ b/web/static/tvl.css @@ -0,0 +1,183 @@ +/* Jetbrains Mono font from https://www.jetbrains.com/lp/mono/ + licensed under Apache 2.0. Thanks, Jetbrains! */ +@font-face { + font-family: jetbrains-mono; + src: url(jetbrains-mono.woff2); +} + +@font-face { + font-family: jetbrains-mono; + font-weight: bold; + src: url(jetbrains-mono-bold.woff2); +} + +@font-face { + font-family: jetbrains-mono; + font-style: italic; + src: url(jetbrains-mono-italic.woff2); +} + +@font-face { + font-family: jetbrains-mono; + font-weight: bold; + font-style: italic; + src: url(jetbrains-mono-bold-italic.woff2); +} + +/* Generic-purpose styling */ + +body { + max-width: 800px; + margin: 40px auto; + line-height: 1.6; + font-size: 18px; + padding: 0 10px; + font-family: jetbrains-mono, monospace; +} + +p, a :not(.uncoloured-link) { + color: inherit; +} + +h1, h2, h3 { + line-height: 1.2 +} + +/* Homepage styling */ + +.dark { + background-color: #181818; + color: #e4e4ef; +} + +.dark-link, .interblag-title { + color: #96a6c8; +} + +.entry-container { + display: flex; + flex-direction: row; + flex-wrap: wrap; + justify-content: flex-start; +} + +.interblag-title { + text-decoration: none; +} + +.entry { + width: 42%; + margin: 5px; + padding-left: 7px; + padding-right: 5px; + border: 2px solid; + border-radius: 5px; + flex-grow: 1; + text-decoration: none; +} + +.misc { + color: #73c936; + border-color: #73c936; +} + +.blog { + color: #268bd2; + border-color: #268bd2; +} + +.project { + color: #ff4f58; + border-color: #ff4f58; +} + +.entry-title { + color: inherit !important; + font-weight: bold; + text-decoration: none; +} + +.entry-date { + font-style: italic; +} + +/* Blog styling */ + +.light { + color: #383838; +} + +.blog-title { + color: inherit; + text-decoration: none; +} + +.footer { + text-align: right; +} + +.date { + text-align: right; + font-style: italic; + float: right; +} + +.inline { + display: inline; +} + +.lod { + text-align: center; +} + +.uncoloured-link { + color: inherit; +} + +pre { + width: 100%; + overflow: auto; +} + +img { + max-width: 100%; +} + +.cheddar-callout { + display: block; + padding: 10px; +} + +.cheddar-question { + color: #3367d6; + background-color: #e8f0fe; +} + +.cheddar-todo { + color: #616161; + background-color: #eeeeee; +} + +.cheddar-tip { + color: #00796b; + background-color: #e0f2f1; +} + +.cheddar-warning { + color: #a52714; + background-color: #fbe9e7; +} + +kbd { + background-color: #eee; + border-radius: 3px; + border: 1px solid #b4b4b4; + box-shadow: 0 1px 1px rgba(0, 0, 0, .2), 0 2px 0 0 rgba(255, 255, 255, .7) inset; + color: #333; + display: inline-block; + font-size: .85em; + font-weight: 700; + line-height: 1; + padding: 2px 4px; + white-space: nowrap; +} diff --git a/web/todolist/default.nix b/web/todolist/default.nix new file mode 100644 index 000000000000..938902f31563 --- /dev/null +++ b/web/todolist/default.nix @@ -0,0 +1,111 @@ +# Generates a simple web view of open TODOs in the depot. +# +# Only TODOs that match the form 'TODO($username)' are considered, and +# only for users that are known to us. +{ depot, lib, pkgs, ... }: + +with depot.nix.yants; + +let + inherit (pkgs) + jq + ripgrep + runCommandNoCC + writeText + ; + + inherit (builtins) + elem + filter + fromJSON + head + readFile + map + ; + + inherit (lib) concatStringsSep; + + knownUsers = map (u: u.username) depot.ops.users; + + todo = struct { + file = string; + line = int; + todo = string; + user = string; + }; + + allTodos = fromJSON (readFile (runCommandNoCC "depot-todos.json" {} '' + ${ripgrep}/bin/rg --json 'TODO\(\w+\):.*$' ${depot.path} | \ + ${jq}/bin/jq -s -f ${./extract-todos.jq} > $out + '')); + + knownUserTodos = filter (todos: elem (head todos).user knownUsers) allTodos; + + fileLink = defun [ todo string ] (t: + ''<a style="color: inherit;" + href="https://cs.tvl.fyi/depot/-/blob/${t.file}#L${toString t.line}"> + //${t.file}:${toString t.line}</a>''); + + todoElement = defun [ todo string ] (t: '' + <p>${fileLink t}:</p> + <blockquote>${t.todo}</blockquote> + + ''); + + userParagraph = todos: + let user = (head todos).user; + in '' + <p> + <h3> + <a style="color:inherit; text-decoration: none;" + name="${user}" + href="#${user}">${user}</a> + </h3> + ${concatStringsSep "\n" (map todoElement todos)} + </p> + <hr> + ''; + + staticUrl = "https://static.tvl.fyi/${depot.web.static.drvHash}"; + + todoPage = writeText "index.html" '' + <!DOCTYPE html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content="TVL's todo-list"> + <link rel="stylesheet" type="text/css" media="all" href="${staticUrl}/tvl.css"> + <link rel="icon" type="image/webp" href="${staticUrl}/favicon.webp"> + <title>TVL's todo-list</title> + <style> + svg { + max-width: inherit; + height: auto; + } + </style> + </head> + <body class="dark"> + <header> + <h1><a class="blog-title" href="/">The Virus Lounge's todo-list</a> </h1> + <hr> + </header> + <main> + ${concatStringsSep "\n" (map userParagraph knownUserTodos)} + </main> + <footer> + <p class="footer"> + <a class="uncoloured-link" href="https://tvl.fyi">homepage</a> + | + <a class="uncoloured-link" href="https://cs.tvl.fyi/depot/-/blob/README.md">code</a> + | + <a class="uncoloured-link" href="https://cl.tvl.fyi">reviews</a> + </p> + <p class="lod">ಠ_ಠ</p> + </footer> + </body> + ''; + +in runCommandNoCC "tvl-todos" {} '' + mkdir $out + cp ${todoPage} $out/index.html +'' diff --git a/web/todolist/extract-todos.jq b/web/todolist/extract-todos.jq new file mode 100644 index 000000000000..d4e0476c403a --- /dev/null +++ b/web/todolist/extract-todos.jq @@ -0,0 +1,30 @@ +# Simple jq script to extract all TODO comments in the code base from +# ripgrep's JSON output. +# +# This assumes that the filter used is something like 'TODO\(\w+\):' + +# Capture the username and todo entry from an input string. +def capture_todo: + capture("TODO\\((?<user>\\w+)\\):\\s+(?<todo>.*)$"); + +# Construct a structure with only the fields we need to populate the +# page. +def simplify_match: + .data as $data + | (.data.submatches[0].match.text | capture_todo) as $capture + | { + file: ($data | .path.text | sub("/nix/store/.+-depot/"; "")), + line: ($data | .line_number), + todo: ($capture | .todo), + user: ($capture | .user), + }; + +# Group all matches first by the user and return them in sorted order +# by the file in which they appear. This matches the presentation +# order on the website. +def group_by_user: . + | group_by(.user) + | map(sort_by(.file)); + +# main: +map(select(.type == "match") | simplify_match) | group_by_user diff --git a/web/tvl/blog/default.nix b/web/tvl/blog/default.nix new file mode 100644 index 000000000000..fe8d1c42d6dd --- /dev/null +++ b/web/tvl/blog/default.nix @@ -0,0 +1,18 @@ +{ depot, ... }: + +{ + config = { + name = "TVL's blog"; + footer = depot.web.tvl.footer {}; + baseUrl = "https://tvl.fyi/blog"; + }; + + posts = [ + { + key = "rewriting-nix"; + title = "Tvix: We are rewriting Nix"; + date = 1638381387; + content = ./rewriting-nix.md; + } + ]; +} diff --git a/web/tvl/blog/rewriting-nix.md b/web/tvl/blog/rewriting-nix.md new file mode 100644 index 000000000000..1b143e5b0ed9 --- /dev/null +++ b/web/tvl/blog/rewriting-nix.md @@ -0,0 +1,90 @@ +Evaluating the Nix programming language, used by the Nix package +manager, is currently very slow. This becomes apparent in all projects +written in Nix that are not just simple package definitions, for +example: + +* the NixOS module system +* TVL projects like + [`//nix/yants`](https://at.tvl.fyi/?q=%2F%2Fnix%2Fyants) and + [`//web/bubblegum`](https://at.tvl.fyi/?q=%2F%2Fweb%2Fbubblegum). +* the code that [generates build + instructions](https://at.tvl.fyi/?q=%2F%2Fops%2Fpipelines) for TVL's + [CI setup](https://tvl.fyi/builds) + +Whichever project you pick, they all suffer from issues with the +language implementation. At TVL, it takes us close to a minute to +create the CI instructions for our monorepo at the moment - despite it +being a plain Nix evaluation. Running our Nix-native build systems for +[Go](https://code.tvl.fyi/about/nix/buildGo) and [Common +Lisp](https://code.tvl.fyi/about/nix/buildLisp) takes much more time +than we would like. + +Some time last year a few of us got together and started investigating +ways to modernise the current architecture of Nix and figure out how +to improve the speed of some of the components. We created over [250 +commits](https://cl.tvl.fyi/q/topic:tvix) in our fork of the Nix 2.3 +codebase at the time, tried [performance +experiments](https://cl.tvl.fyi/c/depot/+/1123/) aimed at improving +the current evaluator and fought [gnarly +bugs](https://cl.tvl.fyi/c/depot/+/1504). + +After a while we realised that we were treading water: Some of our +ideas are too architecturally divergent from Nix to be done on top of +the existing codebase, and the memory model of Nix causes significant +headaches when trying to do any kind of larger change. + +We needed an alternative approach and started brainstorming on a bent +whiteboard in a small flat in Hurghada, Egypt. + + + +Half a year later we are now ready to announce our new project: +**Tvix**, a re-imagined Nix with full nixpkgs compatibility. Tvix is +generously funded [by NLNet](https://nlnet.nl/project/Tvix/) (thanks!) +and we are ready to start implementing it. + +The [Tvix +architecture](https://code.tvl.fyi/about/tvix/docs/components.md) is +designed to be modular: It should be possible to write an evaluator +that plugs in the Guile language (for compatibility with GNU Guix), to +use arbitrary builders, and to replace the store implementation. + +Tvix has these high-level goals: + +* Creating an alternative implementation of Nix that is **fully + compatible with nixpkgs**. + + The package collection is an enormous effort with hundreds of + thousands of commits, encoding expert knowledge about lots of + different software and ways of building and managing it. It is a + very valuable piece of software and we must be able to reuse it. + +* More efficient Nix language evaluation, leading to greatly increased + performance. + +* No more strict separation of evaluation and build phases: Generating + Nix data structures from build artefacts ("IFD") should be supported + first-class and not incur significant performance cost. + +* Well-defined interaction protocols for how the three different + components (evaluator, builder, store) interact. + +* A builder implementation using OCI instead of custom sandboxing + code. + + + +Tvix is not intended to *replace* Nix, instead we want to improve the +ecosystem by offering an alternative, fast and reliable implementation +for Nix features that are in use today. + +As things ramp up we will be posting more information on this blog, +for now you can keep an eye on +[`//tvix`](https://cs.tvl.fyi/depot/-/tree/tvix) in the TVL monorepo +and subscribe to [our feed](https://tvl.fyi/feed.atom). + +Stay tuned! + +<span style="font-size: small;">PS: TVL is international, but a lot of +the development will take place in our office in Moscow. Say hi if +you're around and interested!</span> diff --git a/web/tvl/default.nix b/web/tvl/default.nix new file mode 100644 index 000000000000..ee012ca4096a --- /dev/null +++ b/web/tvl/default.nix @@ -0,0 +1,126 @@ +{ depot, lib, pkgs, ... }: + +with depot.nix.yants; + +let + inherit (builtins) filter; + inherit (pkgs) graphviz runCommandNoCC writeText; + inherit (depot.web) atom-feed blog tvl; + + listPosts = defun [ (list blog.post) string ] (posts: + lib.concatStringsSep "\n" (map (p: "* [${p.title}](blog/${p.key})") posts) + ); + + postRenderingCommands = defun [ (list blog.post) string ] (posts: + lib.concatStringsSep "\n" + (map (p: "cp ${blog.renderPost tvl.blog.config p} $out/blog/${p.key}.html") posts) + ); + + tvlGraph = runCommandNoCC "tvl.svg" { + nativeBuildInputs = with pkgs; [ fontconfig freetype cairo jetbrains-mono ]; + } '' + ${graphviz}/bin/neato -Tsvg ${./tvl.dot} > $out + ''; + + publishedPosts = filter blog.includePost tvl.blog.posts; + + feed = { + id = "https://tvl.fyi/"; + title = "TVL blog"; + subtitle = "Thoughts and news from The Virus Lounge"; + authors = [ "tazjin" ]; # TODO(tazjin): Extract from post info + + links = lib.singleton { + rel = "self"; + href = "https://tvl.fyi/feed.atom"; + }; + + entries = map (blog.toFeedEntry tvl.blog.config) publishedPosts; + }; + + atomFeed = writeText "feed.atom" (atom-feed.renderFeed feed); + + homepage = tvl.template { + title = "The Virus Lounge"; + content = '' + The Virus Lounge + ================ + + ---------------- + + <img class="tvl-logo" src="https://static.tvl.fyi/${depot.web.static.drvHash}/logo-animated.svg" + alt="Virus with lambda-shaped spike proteins sitting on an armchair"> + + Welcome to **The Virus Lounge**. We're a group of people who got + together in 2020, when we felt that there was not enough + spontaneous socialising on the internet. + + Because of our shared interests in topics like **build systems** + and **monorepos** we started working on code together, in our + monorepo called the *depot*. + + Feel free to explore the tech we have built so far, all our + systems are linked in the footer. + + We mostly hang out on IRC. You can find us in [`#tvl`][tvl-irc] + on [hackint][], which is also reachable [via XMPP][hackint-xmpp] + at [`#tvl@irc.hackint.org`][tvl-xmpp] (sic!). + + Hackint also provide a [web chat][tvl-webchat]. + + [tvl-irc]: ircs://irc.hackint.org:6697/#tvl + [hackint]: https://hackint.org/ + [hackint-xmpp]: https://hackint.org/transport/xmpp + [tvl-xmpp]: xmpp:#tvl@irc.hackint.org?join + [tvl-webchat]: https://webirc.hackint.org/#ircs://irc.hackint.org/#tvl + + ---------------- + + ## Blog + + Here are the most recent TVL blog posts. + + ${listPosts publishedPosts} + + You can also follow our [atom feed](https://tvl.fyi/feed.atom). + + ---------------- + + ## Where did all these people come from? + + It's pretty straightforward. Feel free to click on people, too. + + <div class="tvl-graph-container"> + <!-- + cheddar leaves HTML inside of HTML alone, + so wrapping the SVG prevents it from messing it up + --> + ${builtins.readFile tvlGraph} + </div> + ''; + extraHead = '' + <style> + .tvl-graph-container { + max-width: inherit; + } + + .tvl-graph-container svg { + max-width: inherit; + height: auto; + } + + .tvl-logo { + width: 60%; + display: block; + margin-left: auto; + margin-right: auto; + } + </style> + ''; + }; +in runCommandNoCC "website" {} '' + mkdir -p $out/blog + cp ${homepage} $out/index.html + ${postRenderingCommands tvl.blog.posts} + cp ${atomFeed} $out/feed.atom +'' diff --git a/web/tvl/footer/default.nix b/web/tvl/footer/default.nix new file mode 100644 index 000000000000..7412d019ee56 --- /dev/null +++ b/web/tvl/footer/default.nix @@ -0,0 +1,21 @@ +# Footer fragment for TVL homepages, used by //web/tvl/template for +# our static pages and also via //web/blog for blog posts. +{ lib, ... }: + +args: '' + <p class="footer"> + <a class="uncoloured-link" href="https://at.tvl.fyi/?q=%2F%2FREADME.md">code</a> + | + <a class="uncoloured-link" href="https://cl.tvl.fyi/">reviews</a> + | + <a class="uncoloured-link" href="https://tvl.fyi/builds">ci</a> + | + <a class="uncoloured-link" href="https://b.tvl.fyi/">bugs</a> + | + <a class="uncoloured-link" href="https://todo.tvl.fyi/">todos</a> + | + <a class="uncoloured-link" href="https://atward.tvl.fyi/">search</a> + '' + lib.optionalString (args ? extraFooter) args.extraFooter + '' + </p> + <p class="lod">ಠ_ಠ</p> +'' diff --git a/web/tvl/logo/default.nix b/web/tvl/logo/default.nix new file mode 100644 index 000000000000..940f67199bf6 --- /dev/null +++ b/web/tvl/logo/default.nix @@ -0,0 +1,93 @@ +# Creates an output containing the logo in SVG format (animated and +# static, one for each background colour) and without animations in +# PNG. +{ depot, lib, pkgs, ... }: + +let + palette = { + purple = "#CC99C9"; + blue = "#9EC1CF"; + green = "#9EE09E"; + yellow = "#FDFD97"; + orange = "#FEB144"; + red = "#FF6663"; + }; + + staticCss = colour: '' + #armchair-background { + fill: ${colour}; + } + ''; + + # Create an animated CSS that equally spreads out the colours over + # the animation duration (1min). + animatedCss = colours: let + # Calculate at which percentage offset each colour should appear. + stepSize = 100 / ((builtins.length colours) - 1); + frames = lib.imap0 (idx: colour: { inherit colour; at = idx * stepSize; }) colours; + frameCss = frame: "${toString frame.at}% { fill: ${frame.colour}; }"; + in '' + #armchair-background { + animation: 30s infinite alternate armchairPalette; + } + + @keyframes armchairPalette { + ${lib.concatStringsSep "\n" (map frameCss frames)} + } + ''; + + # Dark version of the logo, suitable for light backgrounds. + darkCss = armchairCss: '' + .structure { + fill: #383838; + } + #letters { + fill: #fefefe; + } + ${armchairCss} + ''; + + # Light version, suitable for dark backgrounds. + lightCss = armchairCss: '' + .structure { + fill: #e4e4ef; + } + #letters { + fill: #181818; + } + ${armchairCss} + ''; + + logoShapes = builtins.readFile ./logo-shapes.svg; + logoSvg = style: '' + <svg xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="420 860 1640 1500" + xmlns:xlink="http://www.w3.org/1999/xlink"> + <style>${style}</style> + ${logoShapes} + </svg> + ''; + +in depot.nix.readTree.drvTargets(lib.fix (self: { + # Expose the logo construction functions. + inherit palette darkCss lightCss animatedCss staticCss; + + # Create a TVL logo SVG with the specified style. + logoSvg = style: pkgs.writeText "logo.svg" (logoSvg style); + + # Create a PNG of the TVL logo with the specified style and DPI. + logoPng = style: dpi: pkgs.runCommandNoCC "logo.png" {} '' + ${pkgs.inkscape}/bin/inkscape \ + --export-area-drawing \ + --export-background-opacity 0 \ + --export-dpi ${toString dpi} \ + ${self.logoSvg style} -o $out + ''; + + # Animated dark SVG logo with all colours. + pastelRainbow = self.logoSvg (darkCss (animatedCss (lib.attrValues palette))); +} + +# Add individual outputs for static dark logos of each colour. +// (lib.mapAttrs' + (k: v: lib.nameValuePair "${k}Png" + (self.logoPng (darkCss (staticCss v)) 96)) palette))) diff --git a/web/tvl/logo/logo-shapes.svg b/web/tvl/logo/logo-shapes.svg new file mode 100644 index 000000000000..d35971e0a91a --- /dev/null +++ b/web/tvl/logo/logo-shapes.svg @@ -0,0 +1,27 @@ +<polygon id="armchair-background" points="463 2030 567 1814 1904 1814 1978 2030 1935 2169 1720 2155 1590 2311 873 2305 778 2142 570 2186"/> +<g class="structure"> + <path id="virusbody" d="M 707.524,1820.74 701.038,1401.58 970,1100 h 542 l 271.37,310.47 -16.99,419.17 -295.93,284.34 -445.46,-4.15 z"/> +</g> +<g class="structure" id="lambdas"> + <!-- virus lambdas and feet, clockwise starting at the top left --> + <path id="topleft" d="m 1002,1045 38,75 -65,35 -140,-260 h 78 l 47,80 45,-80 h 45 l 17.39,34.968" /> + <use id="topright" xlink:href="#topleft" transform="matrix(-1,0,0,1,2482,0)" /> + <use id="midright" xlink:href="#topleft" transform="matrix(-0.70710678,-0.70710678,-0.70710678,0.70710678,3284.799,1331.4128)" /> + <use id="bottomright" xlink:href="#topleft" transform="matrix(-0.25881905,-0.96592583,-0.96592583,0.25881905,3120.6829,2438.0653)" /> + <use id="rightfoot" xlink:href="#topleft" transform="matrix(-0.60515932,0.14752194,-0.14752194,-0.60515932,2234.5287,2616.7665)" /> + <use id="leftfoot" xlink:href="#topleft" transform="matrix(0.60515932,0.14752194,0.14752194,-0.60515932,253.62404,2616.7665)" /> + <use id="bottomleft" xlink:href="#topleft" transform="rotate(-75,1263.0635,1635.2798)" /> + <use id="midleft" xlink:href="#topleft" transform="rotate(-45,1209.002,1626.9386)" /> +</g> +<g class="structure" id="armchair"> + <path d="M742.781 2172.23s-89.208 93.93-210.767 22.78c-121.56-71.14-124.755-220.09-47.72-318 78.865-100.24 220.899-86.94 221.229-85.38.274 1.3 247.178 196.08 328.597 260.28 16.08 12.68 25.71 20.27 25.71 20.27l-37.68 41.02s-209.519-177.76-290.729-250.45c-9.975 1.38-150.662-67.27-214.983 108.51-24.251 74.65 15.983 145.09 69.889 167.71 91.689 19.32 94.88 1.94 121.523-18.39"/> + <path d="M1738.4 2174.64s91.9 88.75 209.97 16.51c118.07-72.25 115.91-216.85 39.26-313.11-78.47-98.55-217.31-83.5-217.61-81.95-.26 1.29-239.43 197.97-318.3 262.8-15.58 12.8-24.9 20.46-24.9 20.46l37.4 40.26s202.73-184.66 281.29-257.92c9.78 1.23 134.36-50.54 211.78 110.07 28.32 92.64-13.71 144.64-66.18 167.81-89.5 20.38-90.29.61-116.63-19.24"/> + <path d="m899.02 2276.92 680.44-.32 98.56-134.61 51.64 32.46-121.94 160.78-739.1-1.03-125.507-162.22 54.172-39.79 101.735 144.73Z"/> + <path d="m744.143 2173.36 56.05-35.55s-44.914-79.17-102.074-8.6"/> + <path d="M1728.8 2176.06c-7.6 2.16-53.69-30.58-53.69-30.58s43.06-84.48 102.63-21.21c59.57 63.27-52.85 47.65-48.94 51.79Z"/> +</g> +<g id="letters" fill="#fefefe"> + <path id="t" d="M970.081 1776.8c-22.214 0-40.017-6.45-53.41-19.35-13.394-12.9-20.09-30.14-20.09-51.7v-158.27h-75.95v-40.18h75.95v-75.95h44.1v75.95h107.799v40.18H940.681v158.27c0 9.15 2.695 16.58 8.085 22.3 5.39 5.72 12.495 8.57 21.315 8.57h73.499v40.18h-73.499Z"/> + <path id="v" d="M 1205.77 1776.8 L 1112.18 1507.3 L 1157.75 1507.3 L 1235.66 1742.99 L 1311.12 1507.3 L 1357.18 1507.3 L 1263.59 1776.8 L 1205.77 1776.8 L 1205.77 1776.8 Z"/> + <path id="lambda" d="M 1406.18 1776.8 L 1506.14 1511.71 L 1469.88 1419.1 L 1516.92 1419.1 L 1651.18 1776.8 L 1604.14 1776.8 L 1539.95 1601.87 L 1530.64 1571.49 L 1453.71 1776.8 L 1406.18 1776.8 Z"/> +</g> diff --git a/web/tvl/template/default.nix b/web/tvl/template/default.nix new file mode 100644 index 000000000000..6ccc10de6235 --- /dev/null +++ b/web/tvl/template/default.nix @@ -0,0 +1,50 @@ +{ depot, pkgs, lib, ... }: + +{ # content of the <title> tag + title + # main part of the page, usually wrapped with <main> +, content + # optional extra html to inject into <head> +, extraHead ? null + # optional extra html to inject into <footer> +, extraFooter ? null + # URL at which static assets are located +, staticUrl ? "https://static.tvl.fyi/${depot.web.static.drvHash}" +}@args: + +let + inherit (pkgs) runCommandNoCC lib; + inherit (depot.tools) cheddar; +in + +runCommandNoCC "${lib.strings.sanitizeDerivationName title}-index.html" { + headerPart = '' + <!DOCTYPE html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1"> + <meta name="description" content="The Virus Lounge"> + <link rel="stylesheet" type="text/css" href="${staticUrl}/tvl.css" media="all"> + <link rel="icon" type="image/webp" href="${staticUrl}/favicon.webp"> + <link rel="alternate" type="application/atom+xml" title="Atom Feed" href="https://tvl.fyi/feed.atom"> + <title>${title}</title> + '' + lib.optionalString (args ? extraHead) extraHead + '' + </head> + <body class="light"> + ''; + + inherit content; + + footerPart = '' + <hr> + <footer> + ${depot.web.tvl.footer args} + </footer> + </body> + ''; + + passAsFile = [ "headerPart" "content" "footerPart" ]; +} '' + ${cheddar}/bin/cheddar --about-filter content.md < $contentPath > rendered.html + cat $headerPartPath rendered.html $footerPartPath > $out +'' diff --git a/web/tvl/tvl.dot b/web/tvl/tvl.dot new file mode 100644 index 000000000000..b28c529e0c3a --- /dev/null +++ b/web/tvl/tvl.dot @@ -0,0 +1,172 @@ +digraph tvl { + node [fontname = "JetBrains Mono"]; + overlap = false; + splines = polyline; + + TVL [style="bold" href="http://tvl.fyi"]; + tazjin -> TVL [style="bold"]; + + // people + subgraph { + Irenes [href="https://www.pluralpride.com/"]; + adisbladis [href="http://nixos.expert/"]; + andi [label="andi-" href="https://andreas.rammhold.de/"]; + anon1 [color="grey" fontcolor="grey"]; + aurora [href="https://nonegenderleftfox.aventine.se/"]; + benjojo [href="https://benjojo.co.uk/"]; + cynthia [href="https://cynthia.re/"]; + edef [href="https://edef.eu/files/edef.hs"]; + ericvolp [href="https://ericv.me"]; + espes; + eta [href="https://theta.eu.org/"]; + firefly [href="http://firefly.nu/"]; + flokli [href="https://flokli.de/"]; + fzakaria [href="https://fzakaria.com/"]; + ghuntley [href="https://ghuntley.com/"]; + grfn [href="http://gws.fyi"]; + htbf [href="https://htbf.dev/"]; + implr [href="https://twitter.com/implring"]; + isomer [href="https://www.lorier.net/"]; + jusrin [href="https://jusrin.dev/"]; + kn; + lassulus; + leah2 [href="https://leahneukirchen.org/"]; + lukegb [href="https://lukegb.com/"]; + marcusr [href="http://marcus.nordaaker.com/"]; + mdjnsn; + ncl; + nikky [href="http://nikky.moe/"]; + nyanotech [href="https://twitter.com/nyanotech"]; + poigon; + Profpatsch [href="http://profpatsch.de/"]; + qyliss [href="https://alyssa.is"]; + seven [href="https://open.spotify.com/user/so7"]; + spacekookie [href="https://spacekookie.de/"]; + sterni [href="https://sterni.lv/"]; + tazjin [href="https://tazj.in/"]; + wpcarro [href="https://wpcarro.dev/"]; + yuuko; + } + + // companies (blue) + subgraph { + node [color="#4285f4" fontcolor="#4285f4"]; + spotify [href="https://www.spotify.com/"]; + google [href="https://www.google.com/"]; + } + + // communities? (red) + subgraph { + node [color="#db4437" fontcolor="#db4437"]; + eve [href="https://www.eveonline.com/"]; + nix [href="https://nixos.org/nix/"]; + ircv3 [href="https://ircv3.net/"]; + lgbtslack [label="lgbt.tech" href="https://lgbtq.technology/"]; + muccc [label="µccc" href="https://muc.ccc.de/"]; + afra [label="AfRA" href="https://afra-berlin.de/"]; + } + + // special + subgraph { + baby [color="pink" fontcolor="pink" href="https://cynthia.re/s/baby"]; + unspecific [color="grey" fontcolor="grey"]; + } + + // primary edges (how did they end up in TVL?) + subgraph { + // Direct edges + nix -> tazjin; + spotify -> tazjin; + google -> tazjin; + eve -> tazjin; + unspecific -> tazjin; + edef -> tazjin; + + // via nix + adisbladis -> nix; + jusrin -> nix; + ghuntley -> nix; + flokli -> nix; + andi -> nix; + Profpatsch -> nix; + lassulus -> nix; + + // via edef + benjojo -> edef; + espes -> edef; + firefly -> edef; + leah2 -> aurora; + ncl -> edef; + qyliss -> edef; + + // via spotify + seven -> spotify; + + // via google + htbf -> google; + Irenes -> google; + isomer -> google; + lukegb -> google; + wpcarro -> google; + fzakaria -> google; + mdjnsn -> google; + + // random primary + grfn -> wpcarro; + anon1 -> google; + aurora -> eve; + cynthia -> benjojo; + eta -> anon1; + ericvolp -> lukegb; + marcusr -> unspecific; + poigon -> eve; + implr -> lukegb; + afra -> unspecific; + nikky -> afra; + spacekookie -> qyliss; + kn -> flokli; + sterni -> Profpatsch; + yuuko -> ncl; + } + + // secondary edges (how are they connected otherwise?) + subgraph { + edge [weight=0 style="dotted" color="grey" arrowhead="none"]; + + // lgbt slack + aurora -> lgbtslack; + leah2 -> lgbtslack; + edef -> lgbtslack; + + // ircv3 + eta -> ircv3; + firefly -> ircv3; + + // µccc + leah2 -> muccc; + + // random + leah2 -> edef; + lukegb -> isomer; + eta -> firefly; + cynthia -> firefly; + cynthia -> lukegb; + implr -> google; + nyanotech -> google; + lukegb -> benjojo; + espes -> benjojo; + espes -> aurora; + qyliss -> nix; + grfn -> nix; + edef -> nix; + spacekookie -> afra; + qyliss -> afra; + } + + // baby + subgraph { + edge [weight=0 style="dotted" color="pink" arrowhead="none"]; + cynthia -> baby; + eta -> baby; + } +} |
